 .gitignore                                         |     9 +
 .mailmap                                           |     1 +
 Documentation/00-INDEX                             |     4 -
 Documentation/ABI/stable/sysfs-bus-vmbus           |    53 +-
 Documentation/ABI/testing/devlink-resource-mlxsw   |    33 +
 Documentation/ABI/testing/evm                      |    54 +-
 Documentation/ABI/testing/ima_policy               |     3 +-
 Documentation/ABI/testing/rtc-cdev                 |    42 +
 Documentation/ABI/testing/sysfs-ata                |   171 +-
 Documentation/ABI/testing/sysfs-block-device       |    58 +
 Documentation/ABI/testing/sysfs-bus-iio            |    14 +-
 .../ABI/testing/sysfs-bus-iio-dfsdm-adc-stm32      |    16 +
 .../ABI/testing/sysfs-bus-pci-drivers-xhci_hcd     |    25 +
 Documentation/ABI/testing/sysfs-bus-siox           |    87 +
 .../ABI/testing/sysfs-class-led-trigger-netdev     |    45 +
 Documentation/ABI/testing/sysfs-class-net          |    24 +
 Documentation/ABI/testing/sysfs-class-ocxl         |    35 +
 Documentation/ABI/testing/sysfs-class-rtc          |    91 +
 Documentation/ABI/testing/sysfs-class-scsi_host    |    89 +
 Documentation/ABI/testing/sysfs-devices-coredump   |    10 +
 .../ABI/testing/sysfs-devices-platform-dock        |    39 +
 Documentation/ABI/testing/sysfs-devices-system-cpu |    77 +-
 .../ABI/testing/sysfs-driver-samsung-laptop        |     2 +-
 Documentation/ABI/testing/sysfs-fs-f2fs            |     6 +
 Documentation/ABI/testing/sysfs-kernel-livepatch   |    26 +
 Documentation/ABI/testing/sysfs-platform-dptf      |    40 +
 Documentation/IRQ-domain.txt                       |    36 +-
 Documentation/PCI/pci.txt                          |     4 +-
 .../Design/Data-Structures/Data-Structures.html    |    49 +-
 .../RCU/Design/Requirements/Requirements.html      |     7 +-
 Documentation/RCU/rcu_dereference.txt              |     6 +-
 Documentation/RCU/stallwarn.txt                    |    10 +-
 Documentation/RCU/whatisRCU.txt                    |     3 +-
 Documentation/accelerators/ocxl.rst                |   165 +
 Documentation/admin-guide/README.rst               |     5 -
 Documentation/admin-guide/kernel-parameters.txt    |    52 +-
 Documentation/admin-guide/mono.rst                 |     6 +-
 Documentation/admin-guide/thunderbolt.rst          |    66 +-
 Documentation/arm64/cpu-feature-registers.txt      |     4 +-
 Documentation/arm64/elf_hwcaps.txt                 |     4 +
 Documentation/arm64/silicon-errata.txt             |     2 +-
 Documentation/atomic_bitops.txt                    |     7 +-
 Documentation/bpf/bpf_devel_QA.txt                 |   550 +
 Documentation/cgroup-v1/cgroups.txt                |     7 +-
 Documentation/cgroup-v1/memory.txt                 |     4 +-
 Documentation/cgroup-v2.txt                        |    77 +-
 Documentation/circular-buffers.txt                 |     3 +-
 Documentation/conf.py                              |     1 -
 Documentation/{ => core-api}/errseq.rst            |    20 +-
 Documentation/core-api/idr.rst                     |    79 +
 Documentation/core-api/index.rst                   |     4 +
 Documentation/core-api/kernel-api.rst              |    27 +-
 .../printk-formats.rst}                            |   257 +-
 Documentation/core-api/refcount-vs-atomic.rst      |   150 +
 Documentation/cpu-freq/cpu-drivers.txt             |     4 +
 Documentation/device-mapper/cache-policies.txt     |     4 +-
 Documentation/device-mapper/cache.txt              |     9 +-
 Documentation/device-mapper/dm-raid.txt            |     5 +-
 Documentation/device-mapper/snapshot.txt           |     4 +
 Documentation/device-mapper/thin-provisioning.txt  |    14 +-
 Documentation/device-mapper/unstriped.txt          |   124 +
 Documentation/devicetree/bindings/arm/actions.txt  |    16 +
 .../devicetree/bindings/arm/arm-dsu-pmu.txt        |    27 +
 .../devicetree/bindings/arm/atmel-at91.txt         |    32 -
 Documentation/devicetree/bindings/arm/axentia.txt  |     9 +
 .../devicetree/bindings/arm/bcm/brcm,brcmstb.txt   |    22 +-
 Documentation/devicetree/bindings/arm/cpus.txt     |     1 +
 .../devicetree/bindings/arm/firmware/sdei.txt      |    42 +
 .../bindings/arm/marvell/armada-37xx.txt           |    19 +
 .../bindings/arm/mediatek/mediatek,ethsys.txt      |     1 +
 .../devicetree/bindings/arm/samsung/exynos-adc.txt |     6 +-
 .../bindings/arm/samsung/samsung-boards.txt        |     2 +-
 Documentation/devicetree/bindings/arm/shmobile.txt |     4 +
 Documentation/devicetree/bindings/arm/stm32.txt    |     9 +
 .../devicetree/bindings/arm/technologic.txt        |    11 +
 .../bindings/{misc => auxdisplay}/arm-charlcd.txt  |     0
 Documentation/devicetree/bindings/bus/ti-sysc.txt  |    37 +
 Documentation/devicetree/bindings/chosen.txt       |    15 +
 .../bindings/clock/amlogic,gxbb-clkc.txt           |     7 +-
 .../devicetree/bindings/clock/exynos3250-clock.txt |     2 +-
 .../devicetree/bindings/clock/exynos5260-clock.txt |     2 +-
 .../devicetree/bindings/clock/exynos5410-clock.txt |     2 +-
 .../devicetree/bindings/clock/exynos5433-clock.txt |     2 +-
 .../devicetree/bindings/clock/hi3660-clock.txt     |     6 +
 .../devicetree/bindings/clock/qcom,a53pll.txt      |    22 +
 .../devicetree/bindings/clock/qcom,spmi-clkdiv.txt |    59 +
 .../devicetree/bindings/clock/qoriq-clock.txt      |     1 +
 .../devicetree/bindings/clock/silabs,si5351.txt    |     1 +
 Documentation/devicetree/bindings/clock/sprd.txt   |    63 +
 .../devicetree/bindings/clock/sun8i-de2.txt        |     5 +-
 .../devicetree/bindings/crypto/arm-cryptocell.txt  |    22 +
 .../devicetree/bindings/crypto/atmel-crypto.txt    |     2 +-
 .../bindings/crypto/inside-secure-safexcel.txt     |     3 +-
 .../bindings/crypto/samsung,exynos-rng4.txt        |     4 +-
 .../devicetree/bindings/crypto/st,stm32-cryp.txt   |    19 +
 .../bindings/devfreq/event/exynos-nocp.txt         |     2 +-
 .../bindings/display/amlogic,meson-dw-hdmi.txt     |     4 +
 .../bindings/display/amlogic,meson-vpu.txt         |     4 +
 .../bindings/display/exynos/exynos_dsim.txt        |     2 +-
 .../bindings/display/exynos/exynos_hdmi.txt        |     1 +
 .../devicetree/bindings/display/ilitek,ili9225.txt |    25 +
 .../bindings/display/panel/ilitek,ili9322.txt      |    49 +
 .../display/panel/mitsubishi,aa070mc01.txt         |     7 +
 .../bindings/display/panel/panel-common.txt        |    10 +
 .../bindings/display/panel/panel-lvds.txt          |     1 +
 .../bindings/display/panel/sgd,gktw70sdae4se.txt   |    41 +
 .../bindings/display/panel/simple-panel.txt        |     2 +-
 .../bindings/display/panel/tianma,tm070rvhg71.txt  |    29 +
 .../display/panel/toshiba,lt089ac29000.txt         |     2 +-
 .../{toppoly,td028ttec1.txt => tpo,td028ttec1.txt} |     4 +-
 .../devicetree/bindings/display/renesas,du.txt     |    30 +-
 .../bindings/display/rockchip/rockchip-vop.txt     |     1 +
 .../bindings/display/simple-framebuffer-sunxi.txt  |     4 +
 .../bindings/display/sitronix,st7735r.txt          |    35 +
 .../devicetree/bindings/display/st,stih4xx.txt     |     2 +-
 .../devicetree/bindings/display/st,stm32-ltdc.txt  |     6 +-
 .../bindings/display/sunxi/sun4i-drm.txt           |    11 +
 .../display/tegra/nvidia,tegra20-host1x.txt        |    14 +-
 .../devicetree/bindings/display/ti/ti,dra7-dss.txt |     5 +
 .../bindings/display/ti/ti,omap2-dss.txt           |     4 +
 .../bindings/display/ti/ti,omap3-dss.txt           |     4 +
 .../bindings/display/ti/ti,omap4-dss.txt           |     4 +
 .../bindings/display/ti/ti,omap5-dss.txt           |     4 +
 .../devicetree/bindings/dma/mv-xor-v2.txt          |     6 +-
 .../devicetree/bindings/dma/qcom_hidma_mgmt.txt    |     4 +-
 Documentation/devicetree/bindings/dma/snps-dma.txt |     2 +-
 .../devicetree/bindings/dma/ste-dma40.txt          |     2 +-
 Documentation/devicetree/bindings/eeprom/at24.txt  |    78 +
 Documentation/devicetree/bindings/eeprom/at25.txt  |     4 +-
 .../devicetree/bindings/eeprom/eeprom.txt          |    47 -
 .../devicetree/bindings/gpio/abilis,tb10x-gpio.txt |     2 +-
 .../devicetree/bindings/gpio/gpio-axp209.txt       |    49 +-
 .../devicetree/bindings/gpio/gpio-stp-xway.txt     |     2 +-
 Documentation/devicetree/bindings/gpio/gpio.txt    |     2 +-
 .../devicetree/bindings/gpio/renesas,gpio-rcar.txt |     2 +-
 .../devicetree/bindings/gpu/arm,mali-utgard.txt    |     1 +
 .../devicetree/bindings/hwmon/aspeed-pwm-tacho.txt |    14 +-
 .../bindings/i2c/i2c-arb-gpio-challenge.txt        |     2 +-
 .../devicetree/bindings/i2c/i2c-imx-lpi2c.txt      |     2 +-
 .../devicetree/bindings/i2c/i2c-meson.txt          |     6 +-
 Documentation/devicetree/bindings/i2c/i2c-mtk.txt  |     1 +
 .../devicetree/bindings/i2c/i2c-mux-pca954x.txt    |    13 +-
 .../devicetree/bindings/i2c/i2c-mv64xxx.txt        |    20 +
 .../bindings/iio/adc/amlogic,meson-saradc.txt      |     1 -
 .../devicetree/bindings/iio/adc/aspeed_adc.txt     |     4 +-
 .../bindings/iio/adc/at91-sama5d2_adc.txt          |     7 +
 .../bindings/iio/adc/sigma-delta-modulator.txt     |    13 +
 .../devicetree/bindings/iio/adc/st,stm32-adc.txt   |    24 +
 .../bindings/iio/adc/st,stm32-dfsdm-adc.txt        |   128 +
 .../devicetree/bindings/iio/health/max30102.txt    |     8 +-
 .../devicetree/bindings/iio/light/uvis25.txt       |    23 +
 .../devicetree/bindings/input/hid-over-i2c.txt     |     2 +-
 .../devicetree/bindings/input/samsung-keypad.txt   |     2 +-
 .../input/touchscreen/brcm,iproc-touchscreen.txt   |     2 +-
 .../bindings/input/touchscreen/mms114.txt          |    35 +-
 .../bindings/input/touchscreen/silead_gsl1680.txt  |     2 +
 .../interrupt-controller/brcm,bcm2836-l1-intc.txt  |     4 +-
 .../interrupt-controller/google,goldfish-pic.txt   |    30 +
 .../bindings/interrupt-controller/renesas,irqc.txt |     1 +
 .../bindings/iommu/renesas,ipmmu-vmsa.txt          |     3 +
 .../devicetree/bindings/iommu/samsung,sysmmu.txt   |     2 +-
 .../devicetree/bindings/leds/leds-lm3692x.txt      |    49 +
 .../devicetree/bindings/leds/leds-lp8860.txt       |    32 +-
 .../devicetree/bindings/mailbox/omap-mailbox.txt   |     2 +-
 .../bindings/mailbox/qcom,apcs-kpss-global.txt     |    18 +
 .../bindings/mailbox/ti,message-manager.txt        |     4 +-
 .../devicetree/bindings/media/cec-gpio.txt         |     6 +-
 .../devicetree/bindings/media/i2c/mt9m111.txt      |     4 +
 .../devicetree/bindings/media/i2c/ov5640.txt       |    46 +-
 .../devicetree/bindings/media/i2c/ov7740.txt       |    47 +
 .../bindings/media/i2c/toshiba,et8ek8.txt          |     7 +
 .../devicetree/bindings/media/nvidia,tegra-vde.txt |    55 +
 .../devicetree/bindings/media/s5p-cec.txt          |     2 +-
 .../devicetree/bindings/media/samsung-s5c73m3.txt  |     2 +-
 .../bindings/media/xilinx/xlnx,v-tpg.txt           |     2 +-
 .../memory-controllers/nvidia,tegra30-mc.txt       |     2 +
 .../bindings/memory-controllers/ti-aemif.txt       |     2 +-
 .../bindings/memory-controllers/ti/emif.txt        |    17 +-
 .../devicetree/bindings/mfd/atmel-tcb.txt          |    56 +
 Documentation/devicetree/bindings/mfd/cros-ec.txt  |     2 +-
 Documentation/devicetree/bindings/mfd/mc13xxx.txt  |     2 +-
 Documentation/devicetree/bindings/mfd/syscon.txt   |     8 +
 .../bindings/misc/nvidia,tegra186-misc.txt         |    12 +
 Documentation/devicetree/bindings/mmc/mmc.txt      |     4 +-
 Documentation/devicetree/bindings/mmc/mtk-sd.txt   |     2 +
 Documentation/devicetree/bindings/mmc/tmio_mmc.txt |     1 +
 .../devicetree/bindings/mtd/fsl-quadspi.txt        |     2 +-
 .../devicetree/bindings/mtd/gpmc-onenand.txt       |     6 +-
 .../devicetree/bindings/mtd/lpc32xx-mlc.txt        |     2 +-
 .../devicetree/bindings/mtd/marvell-nand.txt       |   123 +
 Documentation/devicetree/bindings/mtd/mtk-nand.txt |    11 +-
 Documentation/devicetree/bindings/mtd/nand.txt     |     1 +
 .../bindings/net/brcm,bcm7445-switch-v4.0.txt      |     5 +-
 .../bindings/net/can/can-transceiver.txt           |    24 +
 .../devicetree/bindings/net/can/fsl-flexcan.txt    |     6 +
 .../devicetree/bindings/net/can/m_can.txt          |     9 +
 .../devicetree/bindings/net/can/rcar_can.txt       |     7 +-
 .../bindings/net/cortina,gemini-ethernet.txt       |    92 +
 Documentation/devicetree/bindings/net/cpsw.txt     |     4 +-
 .../devicetree/bindings/net/davinci-mdio.txt       |     4 +-
 .../devicetree/bindings/net/dsa/marvell.txt        |    48 +-
 Documentation/devicetree/bindings/net/fsl-fec.txt  |     4 +-
 .../devicetree/bindings/net/fsl-tsec-phy.txt       |     2 +-
 .../devicetree/bindings/net/ieee802154/adf7242.txt |     2 +-
 .../devicetree/bindings/net/mediatek-net.txt       |     2 +-
 Documentation/devicetree/bindings/net/phy.txt      |    12 +
 .../devicetree/bindings/net/renesas,ravb.txt       |     7 +-
 Documentation/devicetree/bindings/net/sff,sfp.txt  |    10 +-
 .../bindings/net/socionext,uniphier-ave4.txt       |    48 +
 .../devicetree/bindings/net/socionext-netsec.txt   |    53 +
 .../net/{ti,wilink-st.txt => ti-bluetooth.txt}     |    18 +-
 .../bindings/net/wireless/mediatek,mt76.txt        |    32 +
 .../bindings/net/wireless/qcom,ath10k.txt          |     3 +
 .../devicetree/bindings/nvmem/rockchip-efuse.txt   |     5 +
 Documentation/devicetree/bindings/opp/opp.txt      |    13 +
 .../bindings/opp/ti-omap5-opp-supply.txt           |    63 +
 .../devicetree/bindings/pci/axis,artpec6-pcie.txt  |     5 +-
 .../devicetree/bindings/pci/cdns,cdns-pcie-ep.txt  |    22 +
 .../bindings/pci/cdns,cdns-pcie-host.txt           |    60 +
 .../devicetree/bindings/pci/fsl,imx6q-pcie.txt     |     1 +
 .../bindings/pci/samsung,exynos5440-pcie.txt       |    58 +-
 Documentation/devicetree/bindings/phy/ti-phy.txt   |     2 +-
 .../bindings/pinctrl/abilis,tb10x-iomux.txt        |     6 +-
 .../bindings/pinctrl/cortina,gemini-pinctrl.txt    |     3 +
 .../bindings/pinctrl/fsl,imx6ul-pinctrl.txt        |     3 +-
 .../bindings/pinctrl/img,pistachio-pinctrl.txt     |     2 +-
 .../bindings/pinctrl/lantiq,pinctrl-xway.txt       |     2 +-
 .../devicetree/bindings/pinctrl/meson,pinctrl.txt  |     2 +
 .../bindings/pinctrl/mscc,ocelot-pinctrl.txt       |    39 +
 .../devicetree/bindings/pinctrl/pinctrl-mt65xx.txt |     2 +-
 .../devicetree/bindings/pinctrl/pinctrl-mt7622.txt |   351 +
 .../bindings/pinctrl/qcom,msm8998-pinctrl.txt      |   193 +
 .../bindings/pinctrl/renesas,pfc-pinctrl.txt       |     1 +
 .../pinctrl/socionext,uniphier-pinctrl.txt         |    27 +
 .../bindings/pinctrl/st,stm32-pinctrl.txt          |     2 +
 .../devicetree/bindings/power/actions,owl-sps.txt  |     2 +
 .../devicetree/bindings/power/fsl,imx-gpc.txt      |     3 +
 .../devicetree/bindings/power/fsl,imx-gpcv2.txt    |     4 +-
 .../devicetree/bindings/power/mti,mips-cpc.txt     |     8 +
 .../devicetree/bindings/power/power_domain.txt     |    65 +
 .../bindings/power/reset/imx-snvs-poweroff.txt     |    23 -
 .../devicetree/bindings/power/supply/bq27xxx.txt   |     1 +
 .../devicetree/bindings/power/ti-smartreflex.txt   |     2 +-
 .../devicetree/bindings/power/wakeup-source.txt    |     2 +-
 .../bindings/powerpc/fsl/interlaken-lac.txt        |    12 +-
 .../devicetree/bindings/powerpc/fsl/mpc5200.txt    |     2 +-
 .../devicetree/bindings/pwm/pwm-meson.txt          |     2 +
 .../devicetree/bindings/regulator/regulator.txt    |    12 +-
 .../bindings/regulator/sprd,sc2731-regulator.txt   |    43 +
 .../bindings/regulator/st,stm32-vrefbuf.txt        |     2 +-
 .../bindings/reset/amlogic,meson-reset.txt         |     3 +-
 .../devicetree/bindings/rng/brcm,bcm2835.txt       |    22 +-
 .../devicetree/bindings/rng/brcm,bcm6368.txt       |    17 -
 .../devicetree/bindings/rtc/rtc-mxc_v2.txt         |    17 +
 .../devicetree/bindings/scsi/hisilicon-sas.txt     |     5 +-
 .../devicetree/bindings/serial/fsl-imx-uart.txt    |     4 +-
 .../devicetree/bindings/serial/fsl-lpuart.txt      |     3 +-
 .../devicetree/bindings/serial/ingenic,uart.txt    |     8 +-
 .../devicetree/bindings/serial/lantiq_asc.txt      |     2 +-
 .../devicetree/bindings/serial/maxim,max310x.txt   |    18 +-
 .../devicetree/bindings/serial/mvebu-uart.txt      |    50 +-
 .../devicetree/bindings/serial/omap_serial.txt     |     1 +
 Documentation/devicetree/bindings/serial/rs485.txt |     1 +
 .../bindings/siox/eckelmann,siox-gpio.txt          |    19 +
 Documentation/devicetree/bindings/slimbus/bus.txt  |    50 +
 .../devicetree/bindings/slimbus/slim-qcom-ctrl.txt |    39 +
 .../devicetree/bindings/soc/mediatek/scpsys.txt    |     3 +
 .../devicetree/bindings/soc/qcom/qcom,smp2p.txt    |     8 +-
 .../devicetree/bindings/soc/xilinx/xlnx,vcu.txt    |    31 +
 Documentation/devicetree/bindings/sound/dmic.txt   |     2 +
 .../devicetree/bindings/sound/img,i2s-out.txt      |     2 +-
 .../devicetree/bindings/sound/img,parallel-out.txt |     2 +-
 .../devicetree/bindings/sound/img,spdif-in.txt     |     2 +-
 .../devicetree/bindings/sound/img,spdif-out.txt    |     2 +-
 .../devicetree/bindings/sound/max98373.txt         |    40 +
 .../devicetree/bindings/sound/mt2701-afe-pcm.txt   |   266 +-
 .../bindings/sound/mxs-audio-sgtl5000.txt          |    33 +-
 .../devicetree/bindings/sound/nau8825.txt          |     4 +-
 .../devicetree/bindings/sound/pcm186x.txt          |    42 +
 .../devicetree/bindings/sound/renesas,rsnd.txt     |    15 +-
 .../devicetree/bindings/sound/simple-card.txt      |     3 +
 .../devicetree/bindings/sound/st,sti-asoc-card.txt |     8 +-
 .../devicetree/bindings/sound/st,stm32-adfsdm.txt  |    63 +
 .../devicetree/bindings/sound/st,stm32-sai.txt     |    12 +-
 .../devicetree/bindings/sound/sun4i-i2s.txt        |     2 +
 .../devicetree/bindings/sound/tas5720.txt          |     4 +-
 .../devicetree/bindings/sound/tfa9879.txt          |     8 +-
 .../devicetree/bindings/sound/ti,tas6424.txt       |    20 +
 .../devicetree/bindings/sound/tlv320aic31xx.txt    |     9 +-
 .../devicetree/bindings/sound/tlv320aic3x.txt      |    10 +-
 .../devicetree/bindings/sound/tscs42xx.txt         |    16 +
 .../devicetree/bindings/sound/uniphier,evea.txt    |    26 +
 Documentation/devicetree/bindings/spi/sh-msiof.txt |    16 +-
 .../devicetree/bindings/spi/spi-davinci.txt        |     2 +-
 .../devicetree/bindings/spi/spi-lantiq-ssc.txt     |     2 +-
 .../devicetree/bindings/spi/spi-meson.txt          |     4 +-
 .../devicetree/bindings/spi/spi-orion.txt          |     9 +
 .../devicetree/bindings/spi/spi-xilinx.txt         |     2 +-
 .../devicetree/bindings/thermal/armada-thermal.txt |    37 +-
 .../devicetree/bindings/thermal/exynos-thermal.txt |     2 +-
 .../devicetree/bindings/thermal/imx-thermal.txt    |    25 +
 .../devicetree/bindings/thermal/rcar-thermal.txt   |     1 +
 .../devicetree/bindings/thermal/thermal.txt        |     6 +-
 .../bindings/timer/actions,owl-timer.txt           |     1 +
 .../devicetree/bindings/timer/renesas,tpu.txt      |     2 +-
 .../bindings/timer/samsung,exynos4210-mct.txt      |     2 +-
 .../bindings/timer/spreadtrum,sprd-timer.txt       |    20 +
 .../devicetree/bindings/trivial-devices.txt        |     2 +
 Documentation/devicetree/bindings/usb/dwc2.txt     |     2 +-
 Documentation/devicetree/bindings/usb/dwc3.txt     |     2 +
 .../devicetree/bindings/usb/mediatek,mtk-xhci.txt  |    18 +-
 .../devicetree/bindings/usb/mediatek,mtu3.txt      |    15 +-
 .../devicetree/bindings/usb/renesas_usb3.txt       |     1 +
 .../devicetree/bindings/usb/renesas_usbhs.txt      |     3 +
 .../devicetree/bindings/usb/usb-device.txt         |    68 +-
 Documentation/devicetree/bindings/usb/usb-xhci.txt |     6 +-
 .../devicetree/bindings/vendor-prefixes.txt        |     8 +
 .../bindings/watchdog/cortina,gemini-watchdog.txt  |    17 -
 ...ina,gemin-watchdog.txt => faraday,ftwdt010.txt} |    11 +-
 .../bindings/watchdog/ingenic,jz4740-wdt.txt       |     4 +-
 .../devicetree/bindings/watchdog/pnx4008-wdt.txt   |     2 +-
 .../bindings/watchdog/realtek,rtd119x.txt          |    17 +
 .../devicetree/bindings/watchdog/renesas-wdt.txt   |     3 +-
 .../devicetree/bindings/watchdog/samsung-wdt.txt   |     2 +-
 .../devicetree/bindings/watchdog/sprd-wdt.txt      |    19 +
 .../bindings/watchdog/zii,rave-sp-wdt.txt          |    39 +
 Documentation/devicetree/booting-without-of.txt    |     2 +-
 Documentation/doc-guide/kernel-doc.rst             |   360 +-
 Documentation/driver-api/basics.rst                |    21 +-
 Documentation/driver-api/dmaengine/provider.rst    |    38 +-
 Documentation/driver-api/firmware/built-in-fw.rst  |     7 +-
 .../driver-api/firmware/fallback-mechanisms.rst    |     2 +-
 Documentation/driver-api/iio/hw-consumer.rst       |    51 +
 Documentation/driver-api/iio/index.rst             |     1 +
 Documentation/driver-api/index.rst                 |     2 +
 Documentation/driver-api/pm/devices.rst            |    54 +-
 Documentation/driver-api/s390-drivers.rst          |    32 +-
 Documentation/driver-api/scsi.rst                  |    10 +-
 Documentation/driver-api/slimbus.rst               |   127 +
 Documentation/driver-api/soundwire/index.rst       |    15 +
 Documentation/driver-api/soundwire/summary.rst     |   207 +
 Documentation/driver-api/uio-howto.rst             |    26 +-
 Documentation/driver-api/usb/usb3-debug-port.rst   |    52 +
 .../driver-api/usb/writing_usb_driver.rst          |     2 +-
 Documentation/driver-model/devres.txt              |     3 +
 Documentation/fault-injection/fault-injection.txt  |    70 +-
 .../features/debug/KASAN/arch-support.txt          |     2 +-
 .../features/debug/stackprotector/arch-support.txt |     2 +-
 .../sched/membarrier-sync-core/arch-support.txt    |    62 +
 Documentation/filesystems/afs.txt                  |    17 +
 Documentation/filesystems/dax.txt                  |     1 -
 Documentation/filesystems/ext2.txt                 |     2 -
 Documentation/filesystems/ext4.txt                 |     9 +-
 Documentation/filesystems/fscrypt.rst              |    10 +-
 Documentation/filesystems/nfs/Exporting            |    27 +-
 Documentation/filesystems/overlayfs.txt            |   106 +-
 Documentation/filesystems/sysfs.txt                |     4 +-
 Documentation/filesystems/vfat.txt                 |     2 +-
 Documentation/fpga/fpga-mgr.txt                    |   132 +-
 Documentation/fpga/fpga-region.txt                 |    95 +
 Documentation/fpga/overview.txt                    |    23 +
 Documentation/gpio/board.txt                       |    14 +-
 Documentation/gpio/consumer.txt                    |   107 +-
 Documentation/gpio/driver.txt                      |     4 +
 Documentation/gpio/drivers-on-gpio.txt             |     5 -
 Documentation/gpio/sysfs.txt                       |    11 +
 Documentation/gpu/drm-kms-helpers.rst              |    39 +-
 Documentation/gpu/drm-kms.rst                      |    14 +-
 Documentation/gpu/i915.rst                         |     4 +-
 Documentation/gpu/todo.rst                         |    41 +-
 Documentation/gpu/tve200.rst                       |     2 +-
 Documentation/hwmon/lm25066                        |    20 +-
 Documentation/hwmon/max31785                       |    15 +-
 Documentation/hwmon/w83773g                        |    33 +
 Documentation/i2c/DMA-considerations               |    67 +
 Documentation/i2c/busses/i2c-i801                  |     2 +
 Documentation/i2c/dev-interface                    |    17 +-
 Documentation/i2c/gpio-fault-injection             |    54 +
 Documentation/ia64/serial.txt                      |     2 +-
 Documentation/index.rst                            |    13 +
 Documentation/infiniband/user_verbs.txt            |     2 +-
 Documentation/input/devices/gpio-tilt.rst          |   103 -
 Documentation/input/devices/rotary-encoder.rst     |     6 +-
 Documentation/input/ff.rst                         |     6 +-
 Documentation/input/multi-touch-protocol.rst       |     9 +-
 Documentation/ioctl/ioctl-number.txt               |     1 +
 Documentation/isdn/INTERFACE.CAPI                  |     2 +-
 Documentation/isdn/README                          |     4 +-
 Documentation/isdn/README.FAQ                      |     4 +-
 Documentation/isdn/README.gigaset                  |    16 +-
 Documentation/kbuild/kconfig-language.txt          |    21 +
 Documentation/kernel-doc-nano-HOWTO.txt            |   322 -
 Documentation/kernel-hacking/hacking.rst           |     6 +-
 Documentation/livepatch/livepatch.txt              |   114 +-
 Documentation/locking/locktorture.txt              |     5 -
 Documentation/locking/mutex-design.txt             |    49 +-
 Documentation/maintainer/conf.py                   |    10 +
 Documentation/maintainer/configure-git.rst         |    34 +
 Documentation/maintainer/index.rst                 |    14 +
 Documentation/maintainer/pull-requests.rst         |   178 +
 Documentation/md/raid5-ppl.txt                     |     7 +-
 Documentation/media/dmx.h.rst.exceptions           |    16 +-
 Documentation/media/kapi/cec-core.rst              |    14 +
 Documentation/media/kapi/dtv-ca.rst                |     2 +-
 Documentation/media/kapi/dtv-common.rst            |    11 +-
 Documentation/media/kapi/dtv-demux.rst             |     8 +-
 Documentation/media/kapi/dtv-frontend.rst          |     8 +-
 Documentation/media/kapi/dtv-net.rst               |     2 +-
 Documentation/media/kapi/rc-core.rst               |    82 +-
 Documentation/media/kapi/v4l2-dev.rst              |    17 +-
 Documentation/media/kapi/v4l2-event.rst            |    67 +-
 Documentation/media/lirc.h.rst.exceptions          |    31 +
 Documentation/media/uapi/dvb/dmx-expbuf.rst        |    88 +
 Documentation/media/uapi/dvb/dmx-mmap.rst          |   116 +
 Documentation/media/uapi/dvb/dmx-munmap.rst        |    54 +
 Documentation/media/uapi/dvb/dmx-qbuf.rst          |    84 +
 Documentation/media/uapi/dvb/dmx-querybuf.rst      |    63 +
 Documentation/media/uapi/dvb/dmx-reqbufs.rst       |    74 +
 Documentation/media/uapi/dvb/dmx_fcalls.rst        |     6 +
 .../media/uapi/dvb/fe_property_parameters.rst      |    18 +
 .../dvb/frontend-property-satellite-systems.rst    |     2 +
 Documentation/media/uapi/rc/lirc-dev-intro.rst     |    75 +-
 Documentation/media/uapi/rc/lirc-func.rst          |     1 -
 Documentation/media/uapi/rc/lirc-get-features.rst  |    25 +-
 Documentation/media/uapi/rc/lirc-get-length.rst    |    44 -
 Documentation/media/uapi/rc/lirc-get-rec-mode.rst  |    45 +-
 Documentation/media/uapi/rc/lirc-get-send-mode.rst |    40 +-
 Documentation/media/uapi/rc/lirc-read.rst          |    15 +-
 Documentation/media/uapi/rc/lirc-write.rst         |    19 +-
 Documentation/media/uapi/v4l/meta-formats.rst      |     1 +
 Documentation/media/uapi/v4l/pixfmt-meta-uvc.rst   |    51 +
 Documentation/media/uapi/v4l/pixfmt-rgb.rst        |     1 +
 .../media/uapi/v4l/pixfmt-srggb10-ipu3.rst         |   335 +
 .../media/uapi/v4l/vidioc-g-dv-timings.rst         |     2 +-
 Documentation/memory-barriers.txt                  |    22 +-
 Documentation/mips/AU1xxx_IDE.README               |     7 -
 Documentation/mtd/spi-nor.txt                      |     3 +
 Documentation/networking/00-INDEX                  |     4 +-
 Documentation/networking/batman-adv.rst            |     2 +
 Documentation/networking/can.rst                   |  1437 +
 Documentation/networking/can.txt                   |  1308 -
 Documentation/networking/dsa/dsa.txt               |     5 -
 Documentation/networking/filter.txt                |     2 +-
 Documentation/networking/ieee802154.txt            |    40 +
 Documentation/networking/index.rst                 |     1 +
 Documentation/networking/ip-sysctl.txt             |     5 +-
 Documentation/networking/kapi.rst                  |    24 +
 Documentation/networking/netdev-features.txt       |     9 +
 Documentation/networking/pktgen.txt                |    19 +-
 Documentation/networking/segmentation-offloads.txt |    52 +-
 Documentation/networking/xfrm_device.txt           |   135 +
 Documentation/networking/xfrm_proc.txt             |    20 +-
 Documentation/perf/arm_dsu_pmu.txt                 |    28 +
 Documentation/power/pci.txt                        |    11 +
 Documentation/power/regulator/machine.txt          |    36 +-
 Documentation/process/changes.rst                  |    25 +
 Documentation/process/index.rst                    |     1 +
 Documentation/process/kernel-docs.rst              |     2 +-
 .../process/kernel-enforcement-statement.rst       |     4 +
 Documentation/process/license-rules.rst            |   370 +
 Documentation/process/maintainer-pgp-guide.rst     |   929 +
 Documentation/process/submit-checklist.rst         |     4 +-
 Documentation/rtc.txt                              |    81 +-
 Documentation/security/credentials.rst             |     7 +
 Documentation/security/self-protection.rst         |    15 +
 Documentation/sparc/oradax/dax-hv-api.txt          |  1433 +
 Documentation/sparc/oradax/oracle-dax.txt          |   429 +
 Documentation/speculation.txt                      |    90 +
 Documentation/sphinx/kerneldoc.py                  |     3 +-
 Documentation/sphinx/kfigure.py                    |     6 +-
 Documentation/sysctl/kernel.txt                    |    17 +-
 Documentation/sysctl/net.txt                       |     4 +-
 Documentation/sysctl/user.txt                      |     2 +-
 Documentation/sysctl/vm.txt                        |    25 -
 Documentation/thermal/cpu-cooling-api.txt          |   115 +-
 Documentation/trace/ftrace-uses.rst                |    60 +-
 .../translations/zh_CN/filesystems/sysfs.txt       |     4 +-
 Documentation/usb/chipidea.txt                     |    12 +-
 Documentation/usb/usbip_protocol.txt               |     1 -
 Documentation/virtual/kvm/00-INDEX                 |     3 +
 .../virtual/kvm/amd-memory-encryption.rst          |   247 +
 Documentation/virtual/kvm/api.txt                  |    94 +-
 Documentation/virtual/kvm/arm/vgic-mapped-irqs.txt |   187 -
 Documentation/virtual/kvm/cpuid.txt                |     8 +
 Documentation/virtual/kvm/msr.txt                  |     3 +-
 Documentation/vm/hugetlbpage.txt                   |    27 +-
 Documentation/vm/hwpoison.txt                      |     2 +-
 Documentation/w1/masters/w1-gpio                   |    17 +-
 Documentation/w1/w1.generic                        |     2 +-
 Documentation/watchdog/watchdog-parameters.txt     |    23 -
 Documentation/x86/intel_rdt_ui.txt                 |    15 +-
 Documentation/x86/microcode.txt                    |     5 +-
 Documentation/x86/topology.txt                     |     2 +-
 Documentation/xtensa/mmu.txt                       |    78 +-
 LICENSES/exceptions/Linux-syscall-note             |    25 +
 LICENSES/other/GPL-1.0                             |   260 +
 LICENSES/other/MPL-1.1                             |   478 +
 LICENSES/preferred/BSD-2-Clause                    |    32 +
 LICENSES/preferred/BSD-3-Clause                    |    36 +
 LICENSES/preferred/BSD-3-Clause-Clear              |    41 +
 LICENSES/preferred/GPL-2.0                         |   353 +
 LICENSES/preferred/LGPL-2.0                        |   487 +
 LICENSES/preferred/LGPL-2.1                        |   503 +
 LICENSES/preferred/MIT                             |    30 +
 MAINTAINERS                                        |   626 +-
 Makefile                                           |    93 +-
 arch/Kconfig                                       |    38 +-
 arch/alpha/Kconfig                                 |     1 +
 arch/alpha/include/asm/Kbuild                      |     1 -
 arch/alpha/include/asm/asm-prototypes.h            |     2 +-
 arch/alpha/include/asm/cmpxchg.h                   |     6 -
 arch/alpha/include/asm/futex.h                     |     8 +-
 arch/alpha/include/asm/processor.h                 |     5 +-
 arch/alpha/include/asm/thread_info.h               |     3 -
 arch/alpha/include/asm/xchg.h                      |    38 +-
 arch/alpha/include/uapi/asm/Kbuild                 |     1 +
 arch/alpha/include/uapi/asm/poll.h                 |     2 -
 arch/alpha/include/uapi/asm/termbits.h             |     6 +-
 arch/alpha/kernel/console.c                        |     1 +
 arch/alpha/kernel/osf_sys.c                        |    72 +-
 arch/alpha/kernel/pci.c                            |     2 +-
 arch/alpha/kernel/pci_impl.h                       |     3 +-
 arch/alpha/kernel/process.c                        |     5 +-
 arch/alpha/kernel/sys_nautilus.c                   |     2 +-
 arch/alpha/kernel/traps.c                          |    13 +-
 arch/arc/Kconfig                                   |     4 -
 arch/arc/boot/dts/axs101.dts                       |     2 +-
 arch/arc/boot/dts/axs10x_mb.dtsi                   |     4 +-
 arch/arc/boot/dts/haps_hs_idu.dts                  |     2 +-
 arch/arc/boot/dts/nsim_700.dts                     |     2 +-
 arch/arc/boot/dts/nsim_hs.dts                      |     2 +-
 arch/arc/boot/dts/nsim_hs_idu.dts                  |     2 +-
 arch/arc/boot/dts/nsimosci.dts                     |     2 +-
 arch/arc/boot/dts/nsimosci_hs.dts                  |     2 +-
 arch/arc/boot/dts/nsimosci_hs_idu.dts              |     2 +-
 arch/arc/configs/axs101_defconfig                  |     1 -
 arch/arc/configs/axs103_defconfig                  |     1 -
 arch/arc/configs/axs103_smp_defconfig              |     1 -
 arch/arc/configs/haps_hs_defconfig                 |     1 -
 arch/arc/configs/haps_hs_smp_defconfig             |     1 -
 arch/arc/configs/hsdk_defconfig                    |     1 -
 arch/arc/configs/nsim_700_defconfig                |     1 -
 arch/arc/configs/nsim_hs_defconfig                 |     1 -
 arch/arc/configs/nsim_hs_smp_defconfig             |     1 -
 arch/arc/configs/nsimosci_defconfig                |     1 -
 arch/arc/configs/nsimosci_hs_defconfig             |     1 -
 arch/arc/configs/nsimosci_hs_smp_defconfig         |     1 -
 arch/arc/configs/tb10x_defconfig                   |     1 -
 arch/arc/configs/vdk_hs38_defconfig                |     1 -
 arch/arc/configs/vdk_hs38_smp_defconfig            |     1 -
 arch/arc/include/asm/Kbuild                        |     1 -
 arch/arc/include/asm/bug.h                         |     3 +-
 arch/arc/include/asm/dma-mapping.h                 |     7 -
 arch/arc/include/asm/entry-arcv2.h                 |     2 +-
 arch/arc/include/asm/hugepage.h                    |     3 +
 arch/arc/include/asm/thread_info.h                 |     3 -
 arch/arc/kernel/mcip.c                             |    74 +-
 arch/arc/kernel/setup.c                            |     4 +-
 arch/arc/kernel/smp.c                              |    50 +-
 arch/arc/kernel/traps.c                            |    14 +-
 arch/arc/kernel/unwind.c                           |     2 +-
 arch/arc/mm/cache.c                                |     5 +-
 arch/arc/mm/dma.c                                  |    14 +-
 arch/arm/Kconfig                                   |    17 +-
 arch/arm/Kconfig.debug                             |    34 +-
 arch/arm/Makefile                                  |     6 +-
 arch/arm/boot/compressed/string.c                  |     5 -
 arch/arm/boot/compressed/vmlinux.lds.S             |     8 +
 arch/arm/boot/deflate_xip_data.sh                  |     6 +-
 arch/arm/boot/dts/Makefile                         |    47 +-
 arch/arm/boot/dts/am335x-bone-common.dtsi          |     2 +-
 arch/arm/boot/dts/am335x-boneblue.dts              |    81 +-
 arch/arm/boot/dts/am335x-evm.dts                   |     2 +-
 arch/arm/boot/dts/am335x-evmsk.dts                 |     2 +-
 arch/arm/boot/dts/am335x-pepper.dts                |     2 +-
 arch/arm/boot/dts/am33xx-clocks.dtsi               |   205 +-
 arch/arm/boot/dts/am33xx.dtsi                      |    18 +-
 arch/arm/boot/dts/am3517.dtsi                      |     8 +-
 arch/arm/boot/dts/am4372.dtsi                      |    20 +-
 arch/arm/boot/dts/am437x-gp-evm.dts                |     4 +-
 arch/arm/boot/dts/am437x-idk-evm.dts               |    14 +
 arch/arm/boot/dts/am437x-sk-evm.dts                |     4 +-
 arch/arm/boot/dts/am43x-epos-evm.dts               |    10 +-
 arch/arm/boot/dts/am43xx-clocks.dtsi               |   230 +-
 arch/arm/boot/dts/am571x-idk.dts                   |     4 +
 arch/arm/boot/dts/am572x-idk-common.dtsi           |   100 +
 arch/arm/boot/dts/am572x-idk.dts                   |    93 +-
 arch/arm/boot/dts/am574x-idk.dts                   |    22 +
 arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi    |     4 +-
 arch/arm/boot/dts/animeo_ip.dts                    |    12 +
 arch/arm/boot/dts/arm-realview-eb-mp.dtsi          |     5 -
 arch/arm/boot/dts/armada-38x.dtsi                  |     5 +
 arch/arm/boot/dts/aspeed-ast2500-evb.dts           |     4 +-
 arch/arm/boot/dts/aspeed-bmc-opp-palmetto.dts      |     5 +-
 arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts       |   135 +-
 arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts   |   548 +
 arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts         |   426 +
 arch/arm/boot/dts/aspeed-bmc-quanta-q71l.dts       |   458 +
 arch/arm/boot/dts/aspeed-g4.dtsi                   |   168 +-
 arch/arm/boot/dts/aspeed-g5.dtsi                   |   161 +-
 arch/arm/boot/dts/at91-ariag25.dts                 |    12 +
 arch/arm/boot/dts/at91-ariettag25.dts              |    12 +
 arch/arm/boot/dts/at91-cosino.dtsi                 |    12 +
 arch/arm/boot/dts/at91-foxg20.dts                  |    12 +
 arch/arm/boot/dts/at91-kizbox.dts                  |    12 +
 arch/arm/boot/dts/at91-kizbox2.dts                 |    12 +
 arch/arm/boot/dts/at91-kizboxmini.dts              |    12 +
 arch/arm/boot/dts/at91-linea.dtsi                  |    12 +
 arch/arm/boot/dts/at91-natte.dtsi                  |   244 +
 arch/arm/boot/dts/at91-nattis-2-natte-2.dts        |   258 +
 arch/arm/boot/dts/at91-qil_a9260.dts               |    12 +
 arch/arm/boot/dts/at91-sam9_l9260.dts              |    12 +
 arch/arm/boot/dts/at91-sama5d27_som1_ek.dts        |    12 +
 arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts          |   413 +
 arch/arm/boot/dts/at91-sama5d2_xplained.dts        |    12 +
 arch/arm/boot/dts/at91-sama5d3_xplained.dts        |    12 +
 arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi          |    12 +
 arch/arm/boot/dts/at91-sama5d4_xplained.dts        |    12 +
 arch/arm/boot/dts/at91-sama5d4ek.dts               |    12 +
 arch/arm/boot/dts/at91-vinco.dts                   |    12 +
 arch/arm/boot/dts/at91rm9200.dtsi                  |     8 +-
 arch/arm/boot/dts/at91rm9200ek.dts                 |    12 +
 arch/arm/boot/dts/at91sam9260.dtsi                 |     8 +-
 arch/arm/boot/dts/at91sam9260ek.dts                |    12 +
 arch/arm/boot/dts/at91sam9261.dtsi                 |     6 +-
 arch/arm/boot/dts/at91sam9261ek.dts                |    22 +-
 arch/arm/boot/dts/at91sam9263.dtsi                 |     6 +-
 arch/arm/boot/dts/at91sam9263ek.dts                |    14 +-
 arch/arm/boot/dts/at91sam9g20ek_common.dtsi        |    12 +
 arch/arm/boot/dts/at91sam9g25ek.dts                |     2 +-
 arch/arm/boot/dts/at91sam9g45.dtsi                 |    10 +-
 arch/arm/boot/dts/at91sam9m10g45ek.dts             |    14 +-
 arch/arm/boot/dts/at91sam9n12.dtsi                 |     8 +-
 arch/arm/boot/dts/at91sam9n12ek.dts                |    12 +
 arch/arm/boot/dts/at91sam9rl.dtsi                  |     4 +-
 arch/arm/boot/dts/at91sam9rlek.dts                 |    12 +
 arch/arm/boot/dts/at91sam9x5.dtsi                  |     8 +-
 arch/arm/boot/dts/at91sam9x5cm.dtsi                |    12 +
 arch/arm/boot/dts/axp81x.dtsi                      |    20 +
 arch/arm/boot/dts/bcm11351.dtsi                    |     2 +-
 arch/arm/boot/dts/bcm21664.dtsi                    |     2 +-
 arch/arm/boot/dts/bcm2835-rpi-a-plus.dts           |     4 +-
 arch/arm/boot/dts/bcm2835-rpi-a.dts                |     2 +-
 arch/arm/boot/dts/bcm2835-rpi-b-plus.dts           |     4 +-
 arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts           |     2 +-
 arch/arm/boot/dts/bcm2835-rpi-b.dts                |     2 +-
 arch/arm/boot/dts/bcm2835.dtsi                     |     6 +-
 arch/arm/boot/dts/bcm2836-rpi-2-b.dts              |     4 +-
 arch/arm/boot/dts/bcm2836.dtsi                     |    24 +-
 arch/arm/boot/dts/bcm2837-rpi-3-b.dts              |     2 +-
 arch/arm/boot/dts/bcm2837.dtsi                     |    14 +-
 arch/arm/boot/dts/bcm283x.dtsi                     |     3 +-
 arch/arm/boot/dts/bcm958625hr.dts                  |     2 +-
 arch/arm/boot/dts/dm814x-clocks.dtsi               |    30 +
 arch/arm/boot/dts/dm814x.dtsi                      |     4 +
 arch/arm/boot/dts/dm816x-clocks.dtsi               |    30 +
 arch/arm/boot/dts/dm816x.dtsi                      |     9 +-
 arch/arm/boot/dts/dra62x-j5eco-evm.dts             |     8 +-
 arch/arm/boot/dts/dra7-evm-common.dtsi             |     4 +-
 arch/arm/boot/dts/dra7-evm.dts                     |     2 +-
 arch/arm/boot/dts/dra7.dtsi                        |   169 +-
 arch/arm/boot/dts/dra72-evm-common.dtsi            |     4 +-
 arch/arm/boot/dts/dra72-evm-tps65917.dtsi          |     4 +
 arch/arm/boot/dts/dra72x.dtsi                      |    16 +-
 arch/arm/boot/dts/dra74x.dtsi                      |    18 +-
 arch/arm/boot/dts/dra76-evm.dts                    |    17 +
 arch/arm/boot/dts/dra7xx-clocks.dtsi               |   909 +-
 arch/arm/boot/dts/efm32gg-dk3750.dts               |     2 +-
 arch/arm/boot/dts/ethernut5.dts                    |    12 +
 arch/arm/boot/dts/exynos3250-artik5-eval.dts       |     5 +-
 arch/arm/boot/dts/exynos3250-artik5.dtsi           |     5 +-
 arch/arm/boot/dts/exynos3250-monk.dts              |     5 +-
 arch/arm/boot/dts/exynos3250-pinctrl.dtsi          |     7 +-
 arch/arm/boot/dts/exynos3250-rinato.dts            |     5 +-
 arch/arm/boot/dts/exynos3250.dtsi                  |    39 +-
 arch/arm/boot/dts/exynos4-cpu-thermal.dtsi         |     6 +-
 arch/arm/boot/dts/exynos4.dtsi                     |    62 +-
 arch/arm/boot/dts/exynos4210-origen.dts            |     7 +-
 arch/arm/boot/dts/exynos4210-pinctrl.dtsi          |     7 +-
 arch/arm/boot/dts/exynos4210-smdkv310.dts          |     7 +-
 arch/arm/boot/dts/exynos4210-trats.dts             |     7 +-
 arch/arm/boot/dts/exynos4210-universal_c210.dts    |     7 +-
 arch/arm/boot/dts/exynos4210.dtsi                  |    15 +-
 arch/arm/boot/dts/exynos4412-itop-elite.dts        |     5 +-
 arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi    |     5 +-
 arch/arm/boot/dts/exynos4412-odroid-common.dtsi    |     5 +-
 arch/arm/boot/dts/exynos4412-odroidu3.dts          |     7 +-
 arch/arm/boot/dts/exynos4412-odroidx.dts           |     7 +-
 arch/arm/boot/dts/exynos4412-odroidx2.dts          |     7 +-
 arch/arm/boot/dts/exynos4412-origen.dts            |     7 +-
 arch/arm/boot/dts/exynos4412-pinctrl.dtsi          |     9 +-
 arch/arm/boot/dts/exynos4412-ppmu-common.dtsi      |     5 +-
 arch/arm/boot/dts/exynos4412-prime.dtsi            |     5 +-
 arch/arm/boot/dts/exynos4412-smdk4412.dts          |     7 +-
 arch/arm/boot/dts/exynos4412-trats2.dts            |    29 +-
 arch/arm/boot/dts/exynos4412.dtsi                  |   100 +-
 arch/arm/boot/dts/exynos5.dtsi                     |    50 +-
 arch/arm/boot/dts/exynos5250-arndale.dts           |     5 +-
 arch/arm/boot/dts/exynos5250-pinctrl.dtsi          |     7 +-
 arch/arm/boot/dts/exynos5250-smdk5250.dts          |     5 +-
 arch/arm/boot/dts/exynos5250-snow-common.dtsi      |     5 +-
 arch/arm/boot/dts/exynos5250-snow-rev5.dts         |     5 +-
 arch/arm/boot/dts/exynos5250-snow.dts              |     5 +-
 arch/arm/boot/dts/exynos5250-spring.dts            |     5 +-
 arch/arm/boot/dts/exynos5250.dtsi                  |   133 +-
 arch/arm/boot/dts/exynos5260-pinctrl.dtsi          |     7 +-
 arch/arm/boot/dts/exynos5260-xyref5260.dts         |     7 +-
 arch/arm/boot/dts/exynos5260.dtsi                  |    33 +-
 arch/arm/boot/dts/exynos5410-odroidxu.dts          |     5 +-
 arch/arm/boot/dts/exynos5410-pinctrl.dtsi          |     5 +-
 arch/arm/boot/dts/exynos5410-smdk5410.dts          |     7 +-
 arch/arm/boot/dts/exynos5410.dtsi                  |    21 +-
 arch/arm/boot/dts/exynos5420-arndale-octa.dts      |     7 +-
 arch/arm/boot/dts/exynos5420-cpus.dtsi             |    15 +-
 arch/arm/boot/dts/exynos5420-peach-pit.dts         |     5 +-
 arch/arm/boot/dts/exynos5420-pinctrl.dtsi          |     7 +-
 arch/arm/boot/dts/exynos5420-smdk5420.dts          |     7 +-
 arch/arm/boot/dts/exynos5420-tmu-sensor-conf.dtsi  |     6 +-
 arch/arm/boot/dts/exynos5420-trip-points.dtsi      |     6 +-
 arch/arm/boot/dts/exynos5420.dtsi                  |    76 +-
 arch/arm/boot/dts/exynos5422-cpus.dtsi             |    15 +-
 arch/arm/boot/dts/exynos5422-odroid-core.dtsi      |    13 +-
 arch/arm/boot/dts/exynos5422-odroidhc1.dts         |     7 +-
 arch/arm/boot/dts/exynos5422-odroidxu3-audio.dtsi  |    67 +-
 arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi |     7 +-
 arch/arm/boot/dts/exynos5422-odroidxu3-lite.dts    |     7 +-
 arch/arm/boot/dts/exynos5422-odroidxu3.dts         |     7 +-
 arch/arm/boot/dts/exynos5422-odroidxu4.dts         |    59 +-
 arch/arm/boot/dts/exynos5440-sd5v1.dts             |     7 +-
 arch/arm/boot/dts/exynos5440-ssdk5440.dts          |     7 +-
 arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi  |     6 +-
 arch/arm/boot/dts/exynos5440-trip-points.dtsi      |     6 +-
 arch/arm/boot/dts/exynos5440.dtsi                  |    21 +-
 arch/arm/boot/dts/exynos54xx-odroidxu-leds.dtsi    |     7 +-
 arch/arm/boot/dts/exynos54xx.dtsi                  |    31 +-
 arch/arm/boot/dts/ge863-pro3.dtsi                  |    12 +
 arch/arm/boot/dts/gemini-dlink-dir-685.dts         |    67 +-
 arch/arm/boot/dts/gemini-dlink-dns-313.dts         |   303 +
 arch/arm/boot/dts/gemini-nas4220b.dts              |    69 +
 arch/arm/boot/dts/gemini-rut1xx.dts                |    25 +
 arch/arm/boot/dts/gemini-wbd111.dts                |    47 +
 arch/arm/boot/dts/gemini-wbd222.dts                |    60 +
 arch/arm/boot/dts/gemini.dtsi                      |    44 +-
 arch/arm/boot/dts/imx25.dtsi                       |     6 +-
 arch/arm/boot/dts/imx27-pdk.dts                    |     1 +
 arch/arm/boot/dts/imx27-phytec-phycore-rdk.dts     |     1 +
 arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi    |     1 +
 arch/arm/boot/dts/imx28-ts4600.dts                 |    79 +
 arch/arm/boot/dts/imx28.dtsi                       |     4 +-
 arch/arm/boot/dts/imx35.dtsi                       |     6 +-
 arch/arm/boot/dts/imx51-babbage.dts                |    77 +-
 .../boot/dts/imx51-eukrea-mbimxsd51-baseboard.dts  |     1 +
 arch/arm/boot/dts/imx51-ts4800.dts                 |     6 +-
 arch/arm/boot/dts/imx51-zii-rdu1.dts               |    12 +-
 arch/arm/boot/dts/imx51.dtsi                       |     1 +
 arch/arm/boot/dts/imx53-cx9020.dts                 |     2 +-
 arch/arm/boot/dts/imx53-m53.dtsi                   |     1 -
 arch/arm/boot/dts/imx53-tx53-x03x.dts              |   232 +-
 arch/arm/boot/dts/imx53-tx53.dtsi                  |    14 +-
 arch/arm/boot/dts/imx53.dtsi                       |    52 +-
 arch/arm/boot/dts/imx6dl-aristainetos_4.dts        |    50 +-
 arch/arm/boot/dts/imx6dl-aristainetos_7.dts        |    48 +-
 arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts       |     2 +-
 arch/arm/boot/dts/imx6dl-cubox-i-emmc-som-v15.dts  |    52 +
 arch/arm/boot/dts/imx6dl-cubox-i-som-v15.dts       |    51 +
 arch/arm/boot/dts/imx6dl-cubox-i.dts               |     2 +
 .../boot/dts/imx6dl-hummingboard-emmc-som-v15.dts  |    53 +
 arch/arm/boot/dts/imx6dl-hummingboard-som-v15.dts  |    52 +
 arch/arm/boot/dts/imx6dl-hummingboard.dts          |     2 +
 .../boot/dts/imx6dl-hummingboard2-emmc-som-v15.dts |    55 +
 arch/arm/boot/dts/imx6dl-hummingboard2-som-v15.dts |    54 +
 arch/arm/boot/dts/imx6dl-hummingboard2.dts         |    53 +
 arch/arm/boot/dts/imx6dl-icore-rqs.dts             |     2 +-
 arch/arm/boot/dts/imx6dl-pinfunc.h                 |     1 +
 arch/arm/boot/dts/imx6dl-ts7970.dts                |    50 +
 arch/arm/boot/dts/imx6dl.dtsi                      |    12 +-
 arch/arm/boot/dts/imx6q-apalis-eval.dts            |     2 +-
 arch/arm/boot/dts/imx6q-apalis-ixora-v1.1.dts      |     2 +-
 arch/arm/boot/dts/imx6q-apalis-ixora.dts           |     2 +-
 arch/arm/boot/dts/imx6q-b450v3.dts                 |    52 +
 arch/arm/boot/dts/imx6q-b650v3.dts                 |    52 +
 arch/arm/boot/dts/imx6q-b850v3.dts                 |    75 +
 arch/arm/boot/dts/imx6q-bx50v3.dtsi                |    62 +
 arch/arm/boot/dts/imx6q-cubox-i-emmc-som-v15.dts   |    60 +
 arch/arm/boot/dts/imx6q-cubox-i-som-v15.dts        |    59 +
 arch/arm/boot/dts/imx6q-cubox-i.dts                |     2 +
 arch/arm/boot/dts/imx6q-display5.dtsi              |     2 +-
 arch/arm/boot/dts/imx6q-h100.dts                   |    33 +-
 .../boot/dts/imx6q-hummingboard-emmc-som-v15.dts   |    61 +
 arch/arm/boot/dts/imx6q-hummingboard-som-v15.dts   |    60 +
 arch/arm/boot/dts/imx6q-hummingboard.dts           |     2 +
 .../boot/dts/imx6q-hummingboard2-emmc-som-v15.dts  |    63 +
 arch/arm/boot/dts/imx6q-hummingboard2-som-v15.dts  |    62 +
 arch/arm/boot/dts/imx6q-hummingboard2.dts          |    61 +
 arch/arm/boot/dts/imx6q-pinfunc.h                  |     1 +
 arch/arm/boot/dts/imx6q-ts7970.dts                 |    54 +
 arch/arm/boot/dts/imx6q-var-dt6customboard.dts     |   235 +
 arch/arm/boot/dts/imx6qdl-apalis.dtsi              |     1 -
 arch/arm/boot/dts/imx6qdl-apf6dev.dtsi             |    44 +-
 arch/arm/boot/dts/imx6qdl-aristainetos.dtsi        |    73 +-
 arch/arm/boot/dts/imx6qdl-aristainetos2.dtsi       |    76 +-
 arch/arm/boot/dts/imx6qdl-colibri.dtsi             |     1 -
 arch/arm/boot/dts/imx6qdl-cubox-i.dtsi             |    72 +-
 arch/arm/boot/dts/imx6qdl-hummingboard.dtsi        |   103 +-
 arch/arm/boot/dts/imx6qdl-hummingboard2-emmc.dtsi  |    72 +
 arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi       |   540 +
 arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi       |    10 +-
 arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi          |    17 +
 arch/arm/boot/dts/imx6qdl-sabreauto.dtsi           |     9 +
 ...6qdl-microsom.dtsi => imx6qdl-sr-som-brcm.dtsi} |    41 +-
 arch/arm/boot/dts/imx6qdl-sr-som-emmc.dtsi         |    70 +
 arch/arm/boot/dts/imx6qdl-sr-som-ti.dtsi           |   170 +
 ...dl-microsom-ar8035.dtsi => imx6qdl-sr-som.dtsi} |    36 +-
 arch/arm/boot/dts/imx6qdl-ts7970.dtsi              |   594 +
 arch/arm/boot/dts/imx6qdl-udoo.dtsi                |    25 +-
 arch/arm/boot/dts/imx6qdl-var-dart.dtsi            |   503 +
 arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi     |     5 +
 arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi            |    14 +-
 arch/arm/boot/dts/imx6qdl.dtsi                     |   161 +-
 arch/arm/boot/dts/imx6sl-evk.dts                   |   133 +-
 arch/arm/boot/dts/imx6sl.dtsi                      |    48 +-
 arch/arm/boot/dts/imx6sx-sdb.dtsi                  |   220 +-
 arch/arm/boot/dts/imx6sx.dtsi                      |    93 +-
 arch/arm/boot/dts/imx6ul-14x14-evk.dts             |    19 +-
 arch/arm/boot/dts/imx6ul.dtsi                      |    51 +-
 arch/arm/boot/dts/imx7-colibri-eval-v3.dtsi        |    39 +-
 arch/arm/boot/dts/imx7-colibri.dtsi                |   116 +-
 arch/arm/boot/dts/imx7d-colibri-emmc-eval-v3.dts   |    19 +
 arch/arm/boot/dts/imx7d-colibri-emmc.dtsi          |    21 +
 arch/arm/boot/dts/imx7d-colibri-eval-v3.dts        |    13 +-
 arch/arm/boot/dts/imx7d-colibri.dtsi               |     4 +
 arch/arm/boot/dts/imx7d-pico-pi.dts                |   181 +
 .../boot/dts/{imx7d-pico.dts => imx7d-pico.dtsi}   |   133 -
 arch/arm/boot/dts/imx7d-sdb.dts                    |     2 +-
 arch/arm/boot/dts/imx7d.dtsi                       |    22 +-
 arch/arm/boot/dts/imx7s.dtsi                       |   133 +-
 arch/arm/boot/dts/iwg20d-q7-common.dtsi            |   122 +
 arch/arm/boot/dts/iwg20d-q7-dbcm-ca.dtsi           |    87 +
 arch/arm/boot/dts/keystone-clocks.dtsi             |    59 +-
 arch/arm/boot/dts/keystone-k2e-clocks.dtsi         |    15 +-
 arch/arm/boot/dts/keystone-k2e-evm.dts             |    13 +-
 arch/arm/boot/dts/keystone-k2e-netcp.dtsi          |     7 +-
 arch/arm/boot/dts/keystone-k2e.dtsi                |    11 +-
 arch/arm/boot/dts/keystone-k2g-evm.dts             |   115 +-
 arch/arm/boot/dts/keystone-k2g-ice.dts             |   339 +-
 arch/arm/boot/dts/keystone-k2g.dtsi                |    58 +-
 arch/arm/boot/dts/keystone-k2hk-clocks.dtsi        |    81 +-
 arch/arm/boot/dts/keystone-k2hk-evm.dts            |    59 +-
 arch/arm/boot/dts/keystone-k2hk-netcp.dtsi         |     7 +-
 arch/arm/boot/dts/keystone-k2hk.dtsi               |    11 +-
 arch/arm/boot/dts/keystone-k2l-clocks.dtsi         |    51 +-
 arch/arm/boot/dts/keystone-k2l-evm.dts             |    13 +-
 arch/arm/boot/dts/keystone-k2l-netcp.dtsi          |     7 +-
 arch/arm/boot/dts/keystone-k2l.dtsi                |    55 +-
 arch/arm/boot/dts/keystone.dtsi                    |    18 +-
 arch/arm/boot/dts/kirkwood-linksys-viper.dts       |    10 +-
 arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts   |    17 +
 arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts   |   249 +-
 arch/arm/boot/dts/logicpd-som-lv-baseboard.dtsi    |   256 +
 arch/arm/boot/dts/logicpd-som-lv.dtsi              |    36 +-
 arch/arm/boot/dts/logicpd-torpedo-35xx-devkit.dts  |    17 +
 arch/arm/boot/dts/logicpd-torpedo-37xx-devkit.dts  |   411 +-
 arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi   |   395 +
 arch/arm/boot/dts/logicpd-torpedo-som.dtsi         |    92 +-
 arch/arm/boot/dts/lpc3250-ea3250.dts               |     4 +-
 arch/arm/boot/dts/lpc3250-phy3250.dts              |     4 +-
 arch/arm/boot/dts/lpc4337-ciaa.dts                 |     6 +-
 arch/arm/boot/dts/lpc4350-hitex-eval.dts           |     2 +-
 arch/arm/boot/dts/lpc4357-ea4357-devkit.dts        |     2 +-
 arch/arm/boot/dts/ls1021a-moxa-uc-8410a.dts        |   241 +
 arch/arm/boot/dts/ls1021a-qds.dts                  |    21 +
 arch/arm/boot/dts/ls1021a-twr.dts                  |    20 +
 arch/arm/boot/dts/ls1021a.dtsi                     |    66 +-
 arch/arm/boot/dts/meson8.dtsi                      |    29 +-
 arch/arm/boot/dts/meson8b.dtsi                     |    24 +-
 arch/arm/boot/dts/mpa1600.dts                      |    12 +
 arch/arm/boot/dts/mt2701.dtsi                      |     2 +
 arch/arm/boot/dts/mt7623.dtsi                      |     5 +-
 arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts      |     2 +-
 arch/arm/boot/dts/mt7623n-rfb-nand.dts             |     2 +-
 arch/arm/boot/dts/omap2420-n8x0-common.dtsi        |     1 +
 arch/arm/boot/dts/omap3-cm-t3x.dtsi                |     8 +-
 arch/arm/boot/dts/omap3-evm-37xx.dts               |     8 +-
 arch/arm/boot/dts/omap3-gta04.dtsi                 |     2 +-
 arch/arm/boot/dts/omap3-igep.dtsi                  |    30 +-
 arch/arm/boot/dts/omap3-lilly-a83x.dtsi            |     8 +-
 arch/arm/boot/dts/omap3-n9.dts                     |     7 +
 arch/arm/boot/dts/omap3-n900.dts                   |    11 +-
 arch/arm/boot/dts/omap3-n950-n9.dtsi               |     1 +
 arch/arm/boot/dts/omap3-n950.dts                   |    88 +
 arch/arm/boot/dts/omap3-pandora-common.dtsi        |     2 +-
 arch/arm/boot/dts/omap3.dtsi                       |    14 -
 arch/arm/boot/dts/omap3430-sdp.dts                 |     1 +
 arch/arm/boot/dts/omap34xx.dtsi                    |    39 +
 arch/arm/boot/dts/omap36xx.dtsi                    |    46 +
 arch/arm/boot/dts/omap4-droid4-xt894.dts           |     6 +-
 arch/arm/boot/dts/omap4.dtsi                       |   161 +-
 arch/arm/boot/dts/omap44xx-clocks.dtsi             |   895 +-
 arch/arm/boot/dts/omap5-uevm.dts                   |     2 +-
 arch/arm/boot/dts/omap5.dtsi                       |    46 +-
 arch/arm/boot/dts/omap54xx-clocks.dtsi             |   623 +-
 arch/arm/boot/dts/openbmc-flash-layout.dtsi        |    32 +
 arch/arm/boot/dts/owl-s500-sparky.dts              |    43 +
 .../boot/dts/{wd-mbwe.dts => ox810se-wd-mbwe.dts}  |     0
 ...ts => ox820-cloudengines-pogoplug-series-3.dts} |     0
 arch/arm/boot/dts/pm9g45.dts                       |    12 +
 arch/arm/boot/dts/qcom-msm8974-fairphone-fp2.dts   |    34 +
 arch/arm/boot/dts/r7s72100-genmai.dts              |     2 +-
 arch/arm/boot/dts/r8a73a4.dtsi                     |     5 +-
 arch/arm/boot/dts/r8a7740-armadillo800eva.dts      |     5 +-
 arch/arm/boot/dts/r8a7740.dtsi                     |     5 +-
 arch/arm/boot/dts/r8a7743.dtsi                     |   607 +-
 .../arm/boot/dts/r8a7745-iwg22d-sodimm-dbhd-ca.dts |   158 +
 arch/arm/boot/dts/r8a7745-iwg22d-sodimm.dts        |   183 +-
 arch/arm/boot/dts/r8a7745.dtsi                     |   585 +-
 arch/arm/boot/dts/r8a7778.dtsi                     |    12 +-
 arch/arm/boot/dts/r8a7779.dtsi                     |    12 +-
 arch/arm/boot/dts/r8a7790-lager.dts                |     5 +-
 arch/arm/boot/dts/r8a7790.dtsi                     |    25 +-
 arch/arm/boot/dts/r8a7791-koelsch.dts              |    19 +-
 arch/arm/boot/dts/r8a7791.dtsi                     |    22 +-
 arch/arm/boot/dts/r8a7792.dtsi                     |    67 +-
 arch/arm/boot/dts/r8a7793.dtsi                     |    22 +-
 arch/arm/boot/dts/r8a7794-alt.dts                  |     5 +-
 arch/arm/boot/dts/r8a7794.dtsi                     |    27 +-
 arch/arm/boot/dts/rk3036.dtsi                      |     4 +-
 arch/arm/boot/dts/rk3066a-rayeager.dts             |     1 +
 arch/arm/boot/dts/rk322x.dtsi                      |     6 +-
 arch/arm/boot/dts/rk3288-phycore-som.dtsi          |    20 -
 arch/arm/boot/dts/rk3288.dtsi                      |     2 +
 arch/arm/boot/dts/s3c2416-pinctrl.dtsi             |     5 +-
 arch/arm/boot/dts/s3c2416-smdk2416.dts             |     5 +-
 arch/arm/boot/dts/s3c2416.dtsi                     |     5 +-
 arch/arm/boot/dts/s3c24xx.dtsi                     |     5 +-
 arch/arm/boot/dts/s3c6400.dtsi                     |     7 +-
 arch/arm/boot/dts/s3c6410-mini6410.dts             |     7 +-
 arch/arm/boot/dts/s3c6410-smdk6410.dts             |     7 +-
 arch/arm/boot/dts/s3c6410.dtsi                     |     7 +-
 arch/arm/boot/dts/s3c64xx-pinctrl.dtsi             |     5 +-
 arch/arm/boot/dts/s3c64xx.dtsi                     |     5 +-
 arch/arm/boot/dts/s5pv210-aquila.dts               |     5 +-
 arch/arm/boot/dts/s5pv210-goni.dts                 |     5 +-
 arch/arm/boot/dts/s5pv210-pinctrl.dtsi             |     5 +-
 arch/arm/boot/dts/s5pv210-smdkc110.dts             |     5 +-
 arch/arm/boot/dts/s5pv210-smdkv210.dts             |     5 +-
 arch/arm/boot/dts/s5pv210-torbreck.dts             |     5 +-
 arch/arm/boot/dts/s5pv210.dtsi                     |     8 +-
 arch/arm/boot/dts/sama5d2.dtsi                     |    10 +-
 arch/arm/boot/dts/sama5d3.dtsi                     |     4 +-
 arch/arm/boot/dts/sama5d3_tcb1.dtsi                |     4 +-
 arch/arm/boot/dts/sama5d3xcm.dtsi                  |    12 +
 arch/arm/boot/dts/sama5d3xcm_cmp.dtsi              |    12 +
 arch/arm/boot/dts/sama5d3xmb.dtsi                  |     2 +-
 arch/arm/boot/dts/sama5d3xmb_cmp.dtsi              |     2 +-
 arch/arm/boot/dts/sama5d4.dtsi                     |    18 +-
 arch/arm/boot/dts/sh73a0.dtsi                      |     3 -
 arch/arm/boot/dts/socfpga.dtsi                     |     4 +
 arch/arm/boot/dts/socfpga_arria10_socdk.dtsi       |     1 +
 arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts |     2 +-
 arch/arm/boot/dts/spear1310-evb.dts                |     2 +-
 arch/arm/boot/dts/spear1340.dtsi                   |     4 +-
 arch/arm/boot/dts/spear13xx.dtsi                   |     6 +-
 arch/arm/boot/dts/spear600.dtsi                    |     1 +
 arch/arm/boot/dts/ste-nomadik-stn8815.dtsi         |     1 +
 arch/arm/boot/dts/stih407.dtsi                     |     3 +-
 arch/arm/boot/dts/stih410.dtsi                     |     3 +-
 arch/arm/boot/dts/sun5i-a13-olinuxino.dts          |     4 +
 arch/arm/boot/dts/sun6i-a31s-sinovoip-bpi-m2.dts   |    63 +-
 arch/arm/boot/dts/sun7i-a20-pcduino3-nano.dts      |    25 +
 arch/arm/boot/dts/sun8i-a83t-bananapi-m3.dts       |    19 +
 arch/arm/boot/dts/sun8i-a83t-cubietruck-plus.dts   |    17 +
 arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts          |    61 +
 arch/arm/boot/dts/sun8i-a83t.dtsi                  |   234 +
 arch/arm/boot/dts/sun8i-h2-plus-orangepi-r1.dts    |    91 +
 arch/arm/boot/dts/sun8i-h3-bananapi-m2-plus.dts    |     1 -
 arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts |   196 +
 arch/arm/boot/dts/sun8i-h3-nanopi-m1-plus.dts      |     3 +-
 arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts           |    11 +
 arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts          |    17 +
 arch/arm/boot/dts/sun8i-h3-orangepi-plus.dts       |     1 -
 arch/arm/boot/dts/sun8i-h3.dtsi                    |     4 +
 arch/arm/boot/dts/sunxi-h3-h5.dtsi                 |    38 +
 arch/arm/boot/dts/tegra20.dtsi                     |    35 +
 arch/arm/boot/dts/tny_a9260_common.dtsi            |    12 +
 arch/arm/boot/dts/tny_a9263.dts                    |    12 +
 arch/arm/boot/dts/uniphier-ld4-ref.dts             |     2 +-
 arch/arm/boot/dts/uniphier-ld4.dtsi                |    23 +
 arch/arm/boot/dts/uniphier-ld6b-ref.dts            |     2 +-
 arch/arm/boot/dts/uniphier-pro4-ref.dts            |     2 +-
 arch/arm/boot/dts/uniphier-pro4.dtsi               |    27 +
 arch/arm/boot/dts/uniphier-pro5.dtsi               |    33 +
 arch/arm/boot/dts/uniphier-pxs2.dtsi               |    19 +
 arch/arm/boot/dts/uniphier-sld8-ref.dts            |     2 +-
 arch/arm/boot/dts/uniphier-sld8.dtsi               |    23 +
 arch/arm/boot/dts/usb_a9260_common.dtsi            |    12 +
 arch/arm/boot/dts/usb_a9263.dts                    |    12 +
 arch/arm/boot/dts/vf-colibri-eval-v3.dtsi          |     2 +-
 arch/arm/boot/dts/vf500.dtsi                       |     2 +-
 arch/arm/boot/dts/vf610-zii-dev-rev-b.dts          |    20 +-
 arch/arm/boot/dts/vf610-zii-dev-rev-c.dts          |     4 +-
 arch/arm/boot/dts/vf610-zii-dev.dtsi               |     4 +
 arch/arm/boot/dts/vfxxx.dtsi                       |    10 +-
 arch/arm/boot/dts/zx296702.dtsi                    |    20 +-
 arch/arm/common/bL_switcher_dummy_if.c             |     4 +
 arch/arm/common/sa1111.c                           |   328 +-
 arch/arm/configs/aspeed_g4_defconfig               |     1 -
 arch/arm/configs/aspeed_g5_defconfig               |     1 -
 arch/arm/configs/cns3420vb_defconfig               |     1 -
 arch/arm/configs/davinci_all_defconfig             |     1 +
 arch/arm/configs/exynos_defconfig                  |     7 +
 arch/arm/configs/hisi_defconfig                    |     1 -
 arch/arm/configs/imx_v6_v7_defconfig               |     8 +
 arch/arm/configs/keystone_defconfig                |     7 +
 arch/arm/configs/magician_defconfig                |     1 -
 arch/arm/configs/mini2440_defconfig                |     1 -
 arch/arm/configs/multi_v7_defconfig                |    13 +-
 arch/arm/configs/mv78xx0_defconfig                 |     1 -
 arch/arm/configs/mvebu_v7_defconfig                |     3 +-
 arch/arm/configs/mxs_defconfig                     |     1 -
 arch/arm/configs/omap2plus_defconfig               |    21 +-
 arch/arm/configs/orion5x_defconfig                 |     1 -
 arch/arm/configs/pxa_defconfig                     |     1 -
 arch/arm/configs/qcom_defconfig                    |     3 +
 arch/arm/configs/sama5_defconfig                   |     1 -
 arch/arm/configs/shmobile_defconfig                |     2 +
 arch/arm/configs/tegra_defconfig                   |     2 -
 arch/arm/configs/vf610m4_defconfig                 |     1 -
 arch/arm/configs/vt8500_v6_v7_defconfig            |     1 -
 arch/arm/crypto/aes-neonbs-glue.c                  |    10 +-
 arch/arm/crypto/crc32-ce-glue.c                    |     2 +
 arch/arm/include/asm/Kbuild                        |     1 -
 arch/arm/include/asm/bitops.h                      |     1 +
 arch/arm/include/asm/dma-direct.h                  |    36 +
 arch/arm/include/asm/dma-mapping.h                 |    35 +-
 arch/arm/include/asm/exception.h                   |     3 +-
 arch/arm/include/asm/glue-cache.h                  |     4 +
 arch/arm/include/asm/hardware/cache-b15-rac.h      |    10 +
 arch/arm/include/asm/hardware/sa1111.h             |    32 +-
 arch/arm/include/asm/kvm_emulate.h                 |     2 +-
 arch/arm/include/asm/kvm_host.h                    |    14 +
 arch/arm/include/asm/kvm_hyp.h                     |     3 +-
 arch/arm/include/asm/kvm_mmu.h                     |   116 +-
 arch/arm/include/asm/kvm_psci.h                    |    27 -
 arch/arm/include/asm/memory.h                      |     1 +
 arch/arm/include/asm/pci.h                         |     5 +-
 arch/arm/include/asm/pgtable-3level.h              |     3 +
 arch/arm/include/asm/pgtable.h                     |     4 +-
 arch/arm/include/asm/processor.h                   |    10 +
 arch/arm/include/asm/ptdump.h                      |    43 +
 arch/arm/include/asm/sections.h                    |    21 +
 arch/arm/include/asm/string.h                      |    14 -
 arch/arm/include/asm/thread_info.h                 |     3 -
 arch/arm/include/asm/traps.h                       |    12 -
 arch/arm/include/asm/unified.h                     |    77 +-
 arch/arm/include/asm/vdso.h                        |     2 -
 arch/arm/include/uapi/asm/siginfo.h                |    13 +
 arch/arm/kernel/armksyms.c                         |     1 -
 arch/arm/kernel/bios32.c                           |     2 +-
 arch/arm/kernel/entry-armv.S                       |     6 +-
 arch/arm/kernel/entry-common.S                     |     1 +
 arch/arm/kernel/head-common.S                      |     5 +-
 arch/arm/kernel/hw_breakpoint.c                    |    10 +-
 arch/arm/kernel/process.c                          |     5 +-
 arch/arm/kernel/ptrace.c                           |     8 +-
 arch/arm/kernel/smp.c                              |     3 +
 arch/arm/kernel/stacktrace.c                       |    14 +-
 arch/arm/kernel/time.c                             |     2 +-
 arch/arm/kernel/traps.c                            |     4 +-
 arch/arm/kernel/vdso.c                             |    12 +-
 arch/arm/kernel/vmlinux-xip.lds.S                  |     6 +-
 arch/arm/kernel/vmlinux.lds.S                      |     6 +-
 arch/arm/kvm/handle_exit.c                         |    17 +-
 arch/arm/kvm/hyp/Makefile                          |     5 +
 arch/arm/kvm/hyp/banked-sr.c                       |     4 +
 arch/arm/kvm/hyp/switch.c                          |     1 +
 arch/arm/kvm/hyp/tlb.c                             |     1 +
 arch/arm/lib/Makefile                              |     2 +-
 arch/arm/lib/memzero.S                             |   137 -
 arch/arm/mach-bcm/brcmstb.c                        |     2 -
 arch/arm/mach-clps711x/board-dt.c                  |     2 +-
 arch/arm/mach-davinci/board-da850-evm.c            |     4 +-
 arch/arm/mach-davinci/board-dm355-evm.c            |    17 +-
 arch/arm/mach-davinci/board-dm355-leopard.c        |     2 +-
 arch/arm/mach-davinci/board-dm365-evm.c            |     2 +-
 arch/arm/mach-davinci/board-dm644x-evm.c           |    15 +-
 arch/arm/mach-davinci/board-neuros-osd2.c          |     2 +-
 arch/arm/mach-davinci/board-omapl138-hawk.c        |     4 +-
 arch/arm/mach-davinci/common.c                     |     2 +-
 arch/arm/mach-davinci/da830.c                      |     6 +-
 arch/arm/mach-davinci/da850.c                      |     6 +-
 arch/arm/mach-davinci/devices-da8xx.c              |     4 +-
 arch/arm/mach-davinci/dm355.c                      |     2 +-
 arch/arm/mach-davinci/dm365.c                      |     2 +-
 arch/arm/mach-davinci/dm644x.c                     |     2 +-
 arch/arm/mach-davinci/dm646x.c                     |     4 +-
 arch/arm/mach-davinci/include/mach/common.h        |     2 +-
 arch/arm/mach-ep93xx/Kconfig                       |     7 +
 arch/arm/mach-ep93xx/ts72xx.c                      |   165 +-
 arch/arm/mach-ep93xx/ts72xx.h                      |     9 +
 arch/arm/mach-exynos/Kconfig                       |     4 +-
 arch/arm/mach-exynos/Makefile                      |     4 +-
 arch/arm/mach-exynos/common.h                      |     5 +-
 arch/arm/mach-exynos/exynos-smc.S                  |     5 +-
 arch/arm/mach-exynos/exynos.c                      |    16 +-
 arch/arm/mach-exynos/firmware.c                    |    14 +-
 arch/arm/mach-exynos/headsmp.S                     |     6 +-
 arch/arm/mach-exynos/include/mach/map.h            |     7 +-
 arch/arm/mach-exynos/mcpm-exynos.c                 |    17 +-
 arch/arm/mach-exynos/platsmp.c                     |    21 +-
 arch/arm/mach-exynos/pm.c                          |    24 +-
 arch/arm/mach-exynos/sleep.S                       |    11 +-
 arch/arm/mach-exynos/smc.h                         |     5 +-
 arch/arm/mach-exynos/suspend.c                     |    24 +-
 arch/arm/mach-imx/common.h                         |     9 -
 arch/arm/mach-imx/mmdc.c                           |    11 -
 arch/arm/mach-ixp4xx/vulcan-setup.c                |    13 +-
 arch/arm/mach-mediatek/Kconfig                     |     2 +-
 arch/arm/mach-meson/Kconfig                        |     1 +
 arch/arm/mach-mmp/mmp2.h                           |     2 +-
 arch/arm/mach-mmp/pxa168.h                         |     2 +-
 arch/arm/mach-mmp/pxa910.h                         |     2 +-
 arch/arm/mach-mvebu/Kconfig                        |     5 +-
 arch/arm/mach-omap1/clock.c                        |     6 +-
 arch/arm/mach-omap2/Makefile                       |     3 -
 arch/arm/mach-omap2/clockdomain.c                  |     8 -
 arch/arm/mach-omap2/clockdomain.h                  |     2 -
 arch/arm/mach-omap2/cm-regbits-24xx.h              |    81 -
 arch/arm/mach-omap2/cm-regbits-34xx.h              |   162 -
 arch/arm/mach-omap2/cm.h                           |     7 +-
 arch/arm/mach-omap2/cm2xxx.c                       |     2 +-
 arch/arm/mach-omap2/cm33xx.c                       |     8 +-
 arch/arm/mach-omap2/cm3xxx.c                       |     2 +-
 arch/arm/mach-omap2/cm_common.c                    |    16 +-
 arch/arm/mach-omap2/cminst44xx.c                   |    12 +-
 arch/arm/mach-omap2/gpmc-onenand.c                 |   409 -
 arch/arm/mach-omap2/id.c                           |    17 +-
 arch/arm/mach-omap2/omap-wakeupgen.c               |     4 +-
 arch/arm/mach-omap2/omap_hwmod.c                   |    85 +-
 arch/arm/mach-omap2/omap_hwmod.h                   |    40 +-
 arch/arm/mach-omap2/omap_hwmod_2420_data.c         |    14 -
 arch/arm/mach-omap2/omap_hwmod_2430_data.c         |    28 -
 arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c |    64 -
 arch/arm/mach-omap2/omap_hwmod_3xxx_data.c         |   152 -
 arch/arm/mach-omap2/omap_hwmod_44xx_data.c         |    17 +-
 arch/arm/mach-omap2/omap_hwmod_7xx_data.c          |    34 +-
 arch/arm/mach-omap2/omap_hwmod_81xx_data.c         |     2 +-
 arch/arm/mach-omap2/omap_hwmod_common_data.c       |    40 +-
 arch/arm/mach-omap2/pm.c                           |     4 +-
 arch/arm/mach-omap2/soc.h                          |    16 +
 arch/arm/mach-omap2/timer.c                        |    19 +-
 arch/arm/mach-orion5x/Kconfig                      |     3 -
 arch/arm/mach-orion5x/dns323-setup.c               |    53 +-
 arch/arm/mach-orion5x/tsx09-common.c               |    49 +-
 arch/arm/mach-pxa/Kconfig                          |     1 +
 arch/arm/mach-pxa/balloon3.c                       |     2 +-
 arch/arm/mach-pxa/cm-x300.c                        |     2 +-
 arch/arm/mach-pxa/colibri-evalboard.c              |     2 +-
 arch/arm/mach-pxa/colibri-pxa270-income.c          |     2 +-
 arch/arm/mach-pxa/corgi.c                          |    33 +-
 arch/arm/mach-pxa/csb726.c                         |     2 +-
 arch/arm/mach-pxa/devices.c                        |     2 +-
 arch/arm/mach-pxa/devices.h                        |     9 +
 arch/arm/mach-pxa/em-x270.c                        |     2 +-
 arch/arm/mach-pxa/ezx.c                            |     2 +-
 arch/arm/mach-pxa/hx4700.c                         |     2 +-
 arch/arm/mach-pxa/littleton.c                      |     3 +-
 arch/arm/mach-pxa/lubbock.c                        |    24 +-
 arch/arm/mach-pxa/magician.c                       |     2 +-
 arch/arm/mach-pxa/mainstone.c                      |     2 +-
 arch/arm/mach-pxa/mioa701.c                        |     2 +-
 arch/arm/mach-pxa/mxm8x10.c                        |     2 +-
 arch/arm/mach-pxa/palm27x.c                        |     2 +-
 arch/arm/mach-pxa/pcm990-baseboard.c               |     2 +-
 arch/arm/mach-pxa/poodle.c                         |    30 +-
 arch/arm/mach-pxa/pxa27x.c                         |     2 +-
 arch/arm/mach-pxa/pxa3xx.c                         |     2 +-
 arch/arm/mach-pxa/raumfeld.c                       |    18 +-
 arch/arm/mach-pxa/saar.c                           |     2 +-
 arch/arm/mach-pxa/sharpsl_pm.c                     |     4 +-
 arch/arm/mach-pxa/spitz.c                          |    36 +-
 arch/arm/mach-pxa/stargate2.c                      |     2 +-
 arch/arm/mach-pxa/tosa-bt.c                        |     4 +
 arch/arm/mach-pxa/tosa.c                           |    30 +-
 arch/arm/mach-pxa/trizeps4.c                       |     2 +-
 arch/arm/mach-pxa/viper.c                          |     2 +-
 arch/arm/mach-pxa/vpac270.c                        |     2 +-
 arch/arm/mach-pxa/xcep.c                           |     3 +-
 arch/arm/mach-pxa/z2.c                             |     2 +-
 arch/arm/mach-pxa/zeus.c                           |     3 +-
 arch/arm/mach-pxa/zylonite_pxa300.c                |     3 +-
 arch/arm/mach-s3c24xx/Kconfig                      |     4 +-
 arch/arm/mach-s3c24xx/Makefile                     |     4 +-
 arch/arm/mach-s3c24xx/Makefile.boot                |     2 +
 arch/arm/mach-s3c24xx/anubis.h                     |     7 +-
 arch/arm/mach-s3c24xx/bast-ide.c                   |    17 +-
 arch/arm/mach-s3c24xx/bast-irq.c                   |    28 +-
 arch/arm/mach-s3c24xx/bast.h                       |     7 +-
 arch/arm/mach-s3c24xx/common-smdk.c                |    21 +-
 arch/arm/mach-s3c24xx/common-smdk.h                |     7 +-
 arch/arm/mach-s3c24xx/common.c                     |    29 +-
 arch/arm/mach-s3c24xx/common.h                     |     5 +-
 arch/arm/mach-s3c24xx/cpufreq-utils.c              |    18 +-
 arch/arm/mach-s3c24xx/fb-core.h                    |     5 +-
 arch/arm/mach-s3c24xx/gta02.h                      |     7 +-
 arch/arm/mach-s3c24xx/h1940-bluetooth.c            |    16 +-
 arch/arm/mach-s3c24xx/h1940.h                      |     7 +-
 arch/arm/mach-s3c24xx/include/mach/dma.h           |    10 +-
 arch/arm/mach-s3c24xx/include/mach/fb.h            |     1 +
 arch/arm/mach-s3c24xx/include/mach/gpio-samsung.h  |     7 +-
 arch/arm/mach-s3c24xx/include/mach/hardware.h      |     7 +-
 arch/arm/mach-s3c24xx/include/mach/irqs.h          |    10 +-
 arch/arm/mach-s3c24xx/include/mach/map.h           |    10 +-
 arch/arm/mach-s3c24xx/include/mach/pm-core.h       |     9 +-
 arch/arm/mach-s3c24xx/include/mach/regs-clock.h    |    10 +-
 arch/arm/mach-s3c24xx/include/mach/regs-gpio.h     |    10 +-
 arch/arm/mach-s3c24xx/include/mach/regs-irq.h      |    10 +-
 arch/arm/mach-s3c24xx/include/mach/regs-lcd.h      |    11 +-
 .../mach-s3c24xx/include/mach/regs-s3c2443-clock.h |    10 +-
 arch/arm/mach-s3c24xx/include/mach/rtc-core.h      |     7 +-
 arch/arm/mach-s3c24xx/include/mach/s3c2412.h       |     5 +-
 arch/arm/mach-s3c24xx/iotiming-s3c2410.c           |    18 +-
 arch/arm/mach-s3c24xx/iotiming-s3c2412.c           |    18 +-
 arch/arm/mach-s3c24xx/irq-pm.c                     |    19 +-
 arch/arm/mach-s3c24xx/mach-amlm5900.c              |    35 +-
 arch/arm/mach-s3c24xx/mach-anubis.c                |    15 +-
 arch/arm/mach-s3c24xx/mach-at2440evb.c             |    21 +-
 arch/arm/mach-s3c24xx/mach-bast.c                  |    17 +-
 arch/arm/mach-s3c24xx/mach-gta02.c                 |    33 +-
 arch/arm/mach-s3c24xx/mach-h1940.c                 |    17 +-
 arch/arm/mach-s3c24xx/mach-jive.c                  |    17 +-
 arch/arm/mach-s3c24xx/mach-mini2440.c              |    23 +-
 arch/arm/mach-s3c24xx/mach-n30.c                   |    27 +-
 arch/arm/mach-s3c24xx/mach-nexcoder.c              |    22 +-
 arch/arm/mach-s3c24xx/mach-osiris-dvs.c            |    19 +-
 arch/arm/mach-s3c24xx/mach-osiris.c                |    14 +-
 arch/arm/mach-s3c24xx/mach-otom.c                  |    13 +-
 arch/arm/mach-s3c24xx/mach-qt2410.c                |    27 +-
 arch/arm/mach-s3c24xx/mach-rx1950.c                |    17 +-
 arch/arm/mach-s3c24xx/mach-rx3715.c                |    18 +-
 arch/arm/mach-s3c24xx/mach-s3c2416-dt.c            |    28 +-
 arch/arm/mach-s3c24xx/mach-smdk2410.c              |    39 +-
 arch/arm/mach-s3c24xx/mach-smdk2413.c              |    19 +-
 arch/arm/mach-s3c24xx/mach-smdk2416.c              |    18 +-
 arch/arm/mach-s3c24xx/mach-smdk2440.c              |    23 +-
 arch/arm/mach-s3c24xx/mach-smdk2443.c              |    22 +-
 arch/arm/mach-s3c24xx/mach-tct_hammer.c            |    33 +-
 arch/arm/mach-s3c24xx/mach-vr1000.c                |    19 +-
 arch/arm/mach-s3c24xx/mach-vstms.c                 |    15 +-
 arch/arm/mach-s3c24xx/nand-core.h                  |     7 +-
 arch/arm/mach-s3c24xx/osiris.h                     |     7 +-
 arch/arm/mach-s3c24xx/otom.h                       |     7 +-
 arch/arm/mach-s3c24xx/pll-s3c2410.c                |    30 +-
 arch/arm/mach-s3c24xx/pll-s3c2440-12000000.c       |    20 +-
 arch/arm/mach-s3c24xx/pll-s3c2440-16934400.c       |    20 +-
 arch/arm/mach-s3c24xx/pm-h1940.S                   |    19 +-
 arch/arm/mach-s3c24xx/pm-s3c2410.c                 |    27 +-
 arch/arm/mach-s3c24xx/pm-s3c2412.c                 |    17 +-
 arch/arm/mach-s3c24xx/pm-s3c2416.c                 |    17 +-
 arch/arm/mach-s3c24xx/pm.c                         |    39 +-
 arch/arm/mach-s3c24xx/regs-dsc.h                   |     7 +-
 arch/arm/mach-s3c24xx/regs-mem.h                   |     5 +-
 arch/arm/mach-s3c24xx/s3c2410.c                    |    17 +-
 arch/arm/mach-s3c24xx/s3c2412-power.h              |     5 +-
 arch/arm/mach-s3c24xx/s3c2412.c                    |    16 +-
 arch/arm/mach-s3c24xx/s3c2416.c                    |    31 +-
 arch/arm/mach-s3c24xx/s3c2440.c                    |    17 +-
 arch/arm/mach-s3c24xx/s3c2442.c                    |    29 +-
 arch/arm/mach-s3c24xx/s3c2443.c                    |    17 +-
 arch/arm/mach-s3c24xx/s3c244x.c                    |    17 +-
 arch/arm/mach-s3c24xx/setup-camif.c                |    14 +-
 arch/arm/mach-s3c24xx/setup-i2c.c                  |    17 +-
 arch/arm/mach-s3c24xx/setup-sdhci-gpio.c           |    21 +-
 arch/arm/mach-s3c24xx/setup-spi.c                  |    16 +-
 arch/arm/mach-s3c24xx/setup-ts.c                   |    17 +-
 arch/arm/mach-s3c24xx/simtec-audio.c               |    19 +-
 arch/arm/mach-s3c24xx/simtec-nor.c                 |    19 +-
 arch/arm/mach-s3c24xx/simtec-pm.c                  |    21 +-
 arch/arm/mach-s3c24xx/simtec-usb.c                 |    21 +-
 arch/arm/mach-s3c24xx/simtec.h                     |    10 +-
 arch/arm/mach-s3c24xx/sleep-s3c2410.S              |    20 +-
 arch/arm/mach-s3c24xx/sleep-s3c2412.S              |    20 +-
 arch/arm/mach-s3c24xx/sleep.S                      |    20 +-
 arch/arm/mach-s3c24xx/spi-core.h                   |     5 +-
 arch/arm/mach-s3c24xx/vr1000.h                     |    11 +-
 arch/arm/mach-s3c64xx/Kconfig                      |     5 +-
 arch/arm/mach-s3c64xx/Makefile                     |     4 +-
 arch/arm/mach-s3c64xx/ata-core.h                   |     7 +-
 arch/arm/mach-s3c64xx/backlight.h                  |     5 +-
 arch/arm/mach-s3c64xx/common.c                     |    26 +-
 arch/arm/mach-s3c64xx/common.h                     |     5 +-
 arch/arm/mach-s3c64xx/cpuidle.c                    |    15 +-
 arch/arm/mach-s3c64xx/crag6410.h                   |     5 +-
 arch/arm/mach-s3c64xx/dev-audio.c                  |    13 +-
 arch/arm/mach-s3c64xx/dev-backlight.c              |    16 +-
 arch/arm/mach-s3c64xx/dev-uart.c                   |    22 +-
 arch/arm/mach-s3c64xx/include/mach/gpio-samsung.h  |     7 +-
 arch/arm/mach-s3c64xx/include/mach/hardware.h      |     1 +
 arch/arm/mach-s3c64xx/include/mach/map.h           |    10 +-
 arch/arm/mach-s3c64xx/include/mach/pm-core.h       |     8 +-
 arch/arm/mach-s3c64xx/include/mach/regs-clock.h    |    10 +-
 arch/arm/mach-s3c64xx/include/mach/regs-irq.h      |    10 +-
 arch/arm/mach-s3c64xx/irq-pm.c                     |    21 +-
 arch/arm/mach-s3c64xx/irq-uart.h                   |     7 +-
 arch/arm/mach-s3c64xx/mach-anw6410.c               |    22 +-
 arch/arm/mach-s3c64xx/mach-crag6410-module.c       |    15 +-
 arch/arm/mach-s3c64xx/mach-crag6410.c              |    19 +-
 arch/arm/mach-s3c64xx/mach-hmt.c                   |    14 +-
 arch/arm/mach-s3c64xx/mach-mini6410.c              |    20 +-
 arch/arm/mach-s3c64xx/mach-ncp.c                   |    13 +-
 arch/arm/mach-s3c64xx/mach-real6410.c              |    20 +-
 arch/arm/mach-s3c64xx/mach-s3c64xx-dt.c            |    14 +-
 arch/arm/mach-s3c64xx/mach-smartq.c                |    13 +-
 arch/arm/mach-s3c64xx/mach-smartq.h                |     6 +-
 arch/arm/mach-s3c64xx/mach-smartq5.c               |    13 +-
 arch/arm/mach-s3c64xx/mach-smartq7.c               |    13 +-
 arch/arm/mach-s3c64xx/mach-smdk6400.c              |    15 +-
 arch/arm/mach-s3c64xx/mach-smdk6410.c              |    18 +-
 arch/arm/mach-s3c64xx/onenand-core.h               |     7 +-
 arch/arm/mach-s3c64xx/pl080.c                      |    14 +-
 arch/arm/mach-s3c64xx/pm.c                         |    21 +-
 arch/arm/mach-s3c64xx/regs-modem.h                 |     7 +-
 arch/arm/mach-s3c64xx/regs-srom.h                  |     7 +-
 arch/arm/mach-s3c64xx/s3c6400.c                    |    15 +-
 arch/arm/mach-s3c64xx/s3c6410.c                    |    17 +-
 arch/arm/mach-s3c64xx/setup-fb-24bpp.c             |    21 +-
 arch/arm/mach-s3c64xx/setup-i2c0.c                 |    21 +-
 arch/arm/mach-s3c64xx/setup-i2c1.c                 |    21 +-
 arch/arm/mach-s3c64xx/setup-ide.c                  |    17 +-
 arch/arm/mach-s3c64xx/setup-keypad.c               |    17 +-
 arch/arm/mach-s3c64xx/setup-sdhci-gpio.c           |    19 +-
 arch/arm/mach-s3c64xx/setup-spi.c                  |    13 +-
 arch/arm/mach-s3c64xx/setup-usb-phy.c              |    14 +-
 arch/arm/mach-s3c64xx/sleep.S                      |     7 +-
 arch/arm/mach-s3c64xx/watchdog-reset.h             |     7 +-
 arch/arm/mach-s5pv210/Kconfig                      |     4 +-
 arch/arm/mach-s5pv210/Makefile                     |     4 +-
 arch/arm/mach-s5pv210/common.h                     |     5 +-
 arch/arm/mach-s5pv210/pm.c                         |    25 +-
 arch/arm/mach-s5pv210/regs-clock.h                 |     7 +-
 arch/arm/mach-s5pv210/s5pv210.c                    |    18 +-
 arch/arm/mach-s5pv210/sleep.S                      |     6 +-
 arch/arm/mach-sa1100/Kconfig                       |     1 +
 arch/arm/mach-sa1100/assabet.c                     |    71 +-
 arch/arm/mach-sa1100/neponset.c                    |   159 +-
 arch/arm/mach-ux500/cpu-db8500.c                   |    34 +-
 arch/arm/mach-vt8500/Kconfig                       |     1 -
 arch/arm/mm/Kconfig                                |     8 +
 arch/arm/mm/Makefile                               |     4 +-
 arch/arm/mm/cache-b15-rac.c                        |   356 +
 arch/arm/mm/cache-v7.S                             |    21 +
 arch/arm/mm/dma-mapping-nommu.c                    |    13 +-
 arch/arm/mm/dump.c                                 |   151 +-
 arch/arm/mm/fault.c                                |     5 +-
 arch/arm/mm/idmap.c                                |     4 +-
 arch/arm/mm/init.c                                 |     2 +
 arch/arm/mm/nommu.c                                |     4 +-
 arch/arm/mm/pmsa-v7.c                              |     4 +-
 arch/arm/mm/proc-v7.S                              |     6 +-
 arch/arm/mm/ptdump_debugfs.c                       |    34 +
 arch/arm/net/bpf_jit_32.c                          |    12 +-
 arch/arm/plat-omap/dmtimer.c                       |     7 +-
 arch/arm/plat-omap/include/plat/sram.h             |    11 +-
 arch/arm/plat-omap/sram.c                          |    36 +-
 arch/arm/plat-orion/common.c                       |    23 +-
 arch/arm/plat-samsung/Kconfig                      |     4 +-
 arch/arm/plat-samsung/Makefile                     |     4 +-
 arch/arm/plat-samsung/adc.c                        |    19 +-
 arch/arm/plat-samsung/cpu.c                        |    17 +-
 arch/arm/plat-samsung/dev-uart.c                   |    21 +-
 arch/arm/plat-samsung/devs.c                       |    17 +-
 arch/arm/plat-samsung/gpio-samsung.c               |    26 +-
 arch/arm/plat-samsung/include/plat/adc-core.h      |    10 +-
 arch/arm/plat-samsung/include/plat/adc.h           |    10 +-
 arch/arm/plat-samsung/include/plat/cpu-freq-core.h |    10 +-
 arch/arm/plat-samsung/include/plat/cpu-freq.h      |    10 +-
 arch/arm/plat-samsung/include/plat/cpu.h           |    10 +-
 arch/arm/plat-samsung/include/plat/devs.h          |    10 +-
 arch/arm/plat-samsung/include/plat/fb-s3c2410.h    |     8 +-
 arch/arm/plat-samsung/include/plat/fb.h            |    10 +-
 .../plat-samsung/include/plat/gpio-cfg-helpers.h   |    10 +-
 arch/arm/plat-samsung/include/plat/gpio-cfg.h      |    10 +-
 arch/arm/plat-samsung/include/plat/gpio-core.h     |    10 +-
 arch/arm/plat-samsung/include/plat/iic-core.h      |    10 +-
 arch/arm/plat-samsung/include/plat/keypad.h        |     6 +-
 arch/arm/plat-samsung/include/plat/map-base.h      |    10 +-
 arch/arm/plat-samsung/include/plat/map-s3c.h       |    10 +-
 arch/arm/plat-samsung/include/plat/map-s5p.h       |    10 +-
 arch/arm/plat-samsung/include/plat/pm-common.h     |     7 +-
 arch/arm/plat-samsung/include/plat/pm.h            |    10 +-
 arch/arm/plat-samsung/include/plat/pwm-core.h      |     5 +-
 arch/arm/plat-samsung/include/plat/regs-adc.h      |    10 +-
 arch/arm/plat-samsung/include/plat/regs-irqtype.h  |     8 +-
 arch/arm/plat-samsung/include/plat/regs-spi.h      |    10 +-
 arch/arm/plat-samsung/include/plat/regs-udc.h      |    11 +-
 arch/arm/plat-samsung/include/plat/samsung-time.h  |    10 +-
 arch/arm/plat-samsung/include/plat/sdhci.h         |    10 +-
 arch/arm/plat-samsung/include/plat/usb-phy.h       |     6 +-
 arch/arm/plat-samsung/include/plat/wakeup-mask.h   |    11 +-
 arch/arm/plat-samsung/init.c                       |    19 +-
 arch/arm/plat-samsung/platformdata.c               |    15 +-
 arch/arm/plat-samsung/pm-check.c                   |    22 +-
 arch/arm/plat-samsung/pm-common.c                  |    24 +-
 arch/arm/plat-samsung/pm-debug.c                   |    24 +-
 arch/arm/plat-samsung/pm-gpio.c                    |    22 +-
 arch/arm/plat-samsung/pm.c                         |    21 +-
 arch/arm/plat-samsung/wakeup-mask.c                |    15 +-
 arch/arm/plat-samsung/watchdog-reset.c             |    21 +-
 arch/arm/probes/kprobes/core.c                     |    14 +-
 arch/arm/tools/mach-types                          |     1 +
 arch/arm/vfp/vfpmodule.c                           |     4 +-
 arch/arm64/Kconfig                                 |    96 +-
 arch/arm64/Kconfig.platforms                       |     1 +
 arch/arm64/boot/dts/actions/Makefile               |     2 +
 arch/arm64/boot/dts/actions/s700-cubieboard7.dts   |    46 +
 arch/arm64/boot/dts/actions/s700.dtsi              |   169 +
 .../boot/dts/allwinner/sun50i-a64-bananapi-m64.dts |    36 +
 .../boot/dts/allwinner/sun50i-a64-olinuxino.dts    |    23 +
 .../boot/dts/allwinner/sun50i-a64-orangepi-win.dts |   125 +-
 .../dts/allwinner/sun50i-h5-nanopi-neo-plus2.dts   |    19 +-
 .../boot/dts/allwinner/sun50i-h5-nanopi-neo2.dts   |     2 +-
 .../boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts  |     2 +-
 .../dts/allwinner/sun50i-h5-orangepi-prime.dts     |     2 +-
 .../allwinner/sun50i-h5-orangepi-zero-plus2.dts    |    34 +-
 arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi       |     4 +
 arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi  |    16 +-
 .../boot/dts/altera/socfpga_stratix10_socdk.dts    |     4 +
 arch/arm64/boot/dts/amlogic/meson-axg-s400.dts     |    13 +
 arch/arm64/boot/dts/amlogic/meson-axg.dtsi         |   347 +-
 .../arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi |    12 +
 arch/arm64/boot/dts/amlogic/meson-gx.dtsi          |    29 +-
 .../boot/dts/amlogic/meson-gxbb-nanopi-k2.dts      |     1 +
 .../arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts |    32 +
 arch/arm64/boot/dts/amlogic/meson-gxbb-p200.dts    |     1 +
 arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi        |    46 +-
 .../dts/amlogic/meson-gxl-s905x-libretech-cc.dts   |    12 +
 .../boot/dts/amlogic/meson-gxl-s905x-p212.dtsi     |    12 +
 arch/arm64/boot/dts/amlogic/meson-gxl.dtsi         |    47 +-
 .../boot/dts/amlogic/meson-gxm-khadas-vim2.dts     |    15 +
 arch/arm64/boot/dts/amlogic/meson-gxm-q200.dts     |     3 +-
 arch/arm64/boot/dts/cavium/thunder2-99xx.dtsi      |     3 +-
 arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi     |     5 +-
 arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi |     5 +-
 .../boot/dts/exynos/exynos5433-tm2-common.dtsi     |    19 +-
 arch/arm64/boot/dts/exynos/exynos5433-tm2.dts      |     5 +-
 arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts     |     5 +-
 .../dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi |     5 +-
 .../dts/exynos/exynos5433-tmu-sensor-conf.dtsi     |     5 +-
 arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi     |     5 +-
 arch/arm64/boot/dts/exynos/exynos5433.dtsi         |   147 +-
 arch/arm64/boot/dts/exynos/exynos7-espresso.dts    |     7 +-
 arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi    |     7 +-
 .../boot/dts/exynos/exynos7-tmu-sensor-conf.dtsi   |     6 +-
 .../arm64/boot/dts/exynos/exynos7-trip-points.dtsi |     6 +-
 arch/arm64/boot/dts/exynos/exynos7.dtsi            |     9 +-
 arch/arm64/boot/dts/freescale/fsl-ls1088a-rdb.dts  |     8 +
 arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi     |    97 +
 arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi |    12 +
 arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts  |     1 +
 arch/arm64/boot/dts/hisilicon/hi3660.dtsi          |    30 +-
 .../boot/dts/hisilicon/hi3798cv200-poplar.dts      |     6 +
 arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi     |    12 +
 arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts     |     3 +-
 arch/arm64/boot/dts/marvell/armada-372x.dtsi       |     1 +
 arch/arm64/boot/dts/marvell/armada-37xx.dtsi       |    21 +
 arch/arm64/boot/dts/marvell/armada-7040-db.dts     |    52 +-
 arch/arm64/boot/dts/marvell/armada-70x0.dtsi       |    37 +-
 arch/arm64/boot/dts/marvell/armada-8020.dtsi       |     2 +-
 arch/arm64/boot/dts/marvell/armada-8040-db.dts     |   111 +-
 arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts  |    82 +-
 arch/arm64/boot/dts/marvell/armada-8040.dtsi       |     2 +-
 arch/arm64/boot/dts/marvell/armada-80x0.dtsi       |    93 +-
 arch/arm64/boot/dts/marvell/armada-ap806.dtsi      |    10 +-
 arch/arm64/boot/dts/marvell/armada-common.dtsi     |    10 +
 .../boot/dts/marvell/armada-cp110-master.dtsi      |   446 -
 .../arm64/boot/dts/marvell/armada-cp110-slave.dtsi |   444 -
 arch/arm64/boot/dts/marvell/armada-cp110.dtsi      |   424 +
 arch/arm64/boot/dts/mediatek/mt2712-evb.dts        |    27 +
 arch/arm64/boot/dts/mediatek/mt2712e.dtsi          |   188 +
 arch/arm64/boot/dts/mediatek/mt8173-evb.dts        |    20 +-
 arch/arm64/boot/dts/mediatek/mt8173.dtsi           |   106 +-
 arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts |    51 +
 arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi     |    10 +-
 arch/arm64/boot/dts/nvidia/tegra186.dtsi           |   351 +
 arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi     |    23 +
 arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi     |    23 +
 arch/arm64/boot/dts/nvidia/tegra210.dtsi           |     4 +-
 arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi          |     1 +
 arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi       |     6 +-
 arch/arm64/boot/dts/qcom/msm8916-pins.dtsi         |    88 +-
 arch/arm64/boot/dts/qcom/msm8916.dtsi              |    29 +-
 arch/arm64/boot/dts/qcom/msm8996.dtsi              |     6 +-
 arch/arm64/boot/dts/qcom/pm8916.dtsi               |    82 +-
 arch/arm64/boot/dts/renesas/Makefile               |     3 +-
 .../boot/dts/renesas/r8a7795-es1-salvator-x.dts    |     2 +-
 arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi       |    85 +
 arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts |     2 +-
 .../arm64/boot/dts/renesas/r8a7795-salvator-xs.dts |     2 +-
 arch/arm64/boot/dts/renesas/r8a7795.dtsi           |   379 +-
 arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts |     2 +-
 .../arm64/boot/dts/renesas/r8a7796-salvator-xs.dts |    58 +
 arch/arm64/boot/dts/renesas/r8a7796.dtsi           |   373 +-
 arch/arm64/boot/dts/renesas/r8a77970-eagle.dts     |    23 +-
 arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts     |    55 +
 arch/arm64/boot/dts/renesas/r8a77970.dtsi          |   115 +-
 arch/arm64/boot/dts/renesas/r8a77995.dtsi          |   234 +
 arch/arm64/boot/dts/renesas/salvator-common.dtsi   |    50 +
 arch/arm64/boot/dts/renesas/ulcb-kf.dtsi           |    29 +
 arch/arm64/boot/dts/renesas/ulcb.dtsi              |     1 +
 arch/arm64/boot/dts/rockchip/rk3328-rock64.dts     |     7 +-
 arch/arm64/boot/dts/rockchip/rk3328.dtsi           |    53 +-
 arch/arm64/boot/dts/rockchip/rk3368.dtsi           |     2 +-
 arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts  |    29 +
 arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi       |    58 +-
 arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi  |     2 +-
 arch/arm64/boot/dts/rockchip/rk3399.dtsi           |    81 +-
 .../arm64/boot/dts/socionext/uniphier-ld11-ref.dts |     2 +-
 arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi   |     6 +-
 .../arm64/boot/dts/socionext/uniphier-ld20-ref.dts |     2 +-
 arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi   |     3 +-
 .../arm64/boot/dts/socionext/uniphier-pxs3-ref.dts |    16 +
 arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi   |     3 +-
 arch/arm64/boot/dts/sprd/sc9860.dtsi               |   114 +
 arch/arm64/boot/dts/sprd/whale2.dtsi               |    62 +-
 arch/arm64/configs/defconfig                       |    15 +-
 arch/arm64/crypto/Kconfig                          |    18 +
 arch/arm64/crypto/Makefile                         |    11 +-
 arch/arm64/crypto/aes-ce-core.S                    |    87 +
 .../crypto/{aes-ce-cipher.c => aes-ce-glue.c}      |   115 +-
 arch/arm64/crypto/aes-cipher-core.S                |    19 +-
 arch/arm64/crypto/aes-glue.c                       |     1 +
 arch/arm64/crypto/aes-neon.S                       |     8 +-
 arch/arm64/crypto/crc32-ce-core.S                  |     7 +-
 arch/arm64/crypto/crc32-ce-glue.c                  |     2 +
 arch/arm64/crypto/crct10dif-ce-core.S              |    17 +-
 arch/arm64/crypto/sha1-ce-core.S                   |    20 +-
 arch/arm64/crypto/sha2-ce-core.S                   |     4 +-
 arch/arm64/crypto/sha3-ce-core.S                   |   210 +
 arch/arm64/crypto/sha3-ce-glue.c                   |   161 +
 arch/arm64/crypto/sha512-ce-core.S                 |   204 +
 arch/arm64/crypto/sha512-ce-glue.c                 |   119 +
 arch/arm64/crypto/sha512-glue.c                    |     1 +
 arch/arm64/crypto/sm3-ce-core.S                    |   141 +
 arch/arm64/crypto/sm3-ce-glue.c                    |    92 +
 arch/arm64/include/asm/Kbuild                      |     1 -
 arch/arm64/include/asm/alternative.h               |     2 +
 arch/arm64/include/asm/arm_dsu_pmu.h               |   129 +
 arch/arm64/include/asm/asm-uaccess.h               |    42 +-
 arch/arm64/include/asm/assembler.h                 |   133 +-
 arch/arm64/include/asm/barrier.h                   |    22 +
 arch/arm64/include/asm/cacheflush.h                |     7 +
 arch/arm64/include/asm/compat.h                    |    64 -
 arch/arm64/include/asm/cpucaps.h                   |     6 +-
 arch/arm64/include/asm/cputype.h                   |    11 +-
 arch/arm64/include/asm/dma-mapping.h               |    35 -
 arch/arm64/include/asm/efi.h                       |    12 +-
 arch/arm64/include/asm/esr.h                       |    20 +
 arch/arm64/include/asm/exception.h                 |    14 +
 arch/arm64/include/asm/fixmap.h                    |     5 +
 arch/arm64/include/asm/fpsimd.h                    |     2 +-
 arch/arm64/include/asm/futex.h                     |     9 +-
 arch/arm64/include/asm/hugetlb.h                   |     2 +-
 arch/arm64/include/asm/kasan.h                     |    17 +-
 arch/arm64/include/asm/kernel-pgtable.h            |    47 +-
 arch/arm64/include/asm/kvm_arm.h                   |     2 +
 arch/arm64/include/asm/kvm_asm.h                   |     2 +
 arch/arm64/include/asm/kvm_emulate.h               |    17 +
 arch/arm64/include/asm/kvm_host.h                  |    27 +
 arch/arm64/include/asm/kvm_hyp.h                   |     1 -
 arch/arm64/include/asm/kvm_mmu.h                   |   113 +-
 arch/arm64/include/asm/kvm_psci.h                  |    27 -
 arch/arm64/include/asm/memory.h                    |     3 +-
 arch/arm64/include/asm/mmu.h                       |    49 +
 arch/arm64/include/asm/mmu_context.h               |    31 +-
 arch/arm64/include/asm/percpu.h                    |    11 +-
 arch/arm64/include/asm/pgalloc.h                   |    44 +-
 arch/arm64/include/asm/pgtable-hwdef.h             |    34 +-
 arch/arm64/include/asm/pgtable-prot.h              |    35 +-
 arch/arm64/include/asm/pgtable.h                   |    87 +-
 arch/arm64/include/asm/proc-fns.h                  |     6 -
 arch/arm64/include/asm/processor.h                 |    14 +
 arch/arm64/include/asm/sdei.h                      |    57 +
 arch/arm64/include/asm/sections.h                  |     1 +
 arch/arm64/include/asm/sparsemem.h                 |     2 +-
 arch/arm64/include/asm/spinlock.h                  |     4 +-
 arch/arm64/include/asm/stacktrace.h                |     5 +-
 arch/arm64/include/asm/sysreg.h                    |    92 +-
 arch/arm64/include/asm/thread_info.h               |     2 -
 arch/arm64/include/asm/tlbflush.h                  |    16 +-
 arch/arm64/include/asm/traps.h                     |    54 +
 arch/arm64/include/asm/uaccess.h                   |   193 +-
 arch/arm64/include/asm/vmap_stack.h                |    28 +
 arch/arm64/include/uapi/asm/hwcap.h                |     1 +
 arch/arm64/include/uapi/asm/siginfo.h              |    21 +
 arch/arm64/kernel/Makefile                         |     5 +
 arch/arm64/kernel/acpi.c                           |     6 +-
 arch/arm64/kernel/alternative.c                    |     9 +-
 arch/arm64/kernel/arm64ksyms.c                     |     4 +-
 arch/arm64/kernel/armv8_deprecated.c               |     4 +-
 arch/arm64/kernel/asm-offsets.c                    |    12 +-
 arch/arm64/kernel/bpi.S                            |    83 +
 arch/arm64/kernel/cpu-reset.S                      |     2 +-
 arch/arm64/kernel/cpu_errata.c                     |   248 +-
 arch/arm64/kernel/cpufeature.c                     |   188 +-
 arch/arm64/kernel/cpuidle.c                        |     8 +-
 arch/arm64/kernel/cpuinfo.c                        |     1 +
 arch/arm64/kernel/debug-monitors.c                 |    13 +-
 arch/arm64/kernel/efi.c                            |     6 +-
 arch/arm64/kernel/entry.S                          |   409 +-
 arch/arm64/kernel/fpsimd.c                         |     6 +-
 arch/arm64/kernel/head.S                           |   227 +-
 arch/arm64/kernel/hibernate-asm.S                  |    12 +-
 arch/arm64/kernel/hibernate.c                      |   149 +-
 arch/arm64/kernel/irq.c                            |    13 +-
 arch/arm64/kernel/perf_event.c                     |     9 +-
 arch/arm64/kernel/process.c                        |    24 +-
 arch/arm64/kernel/ptrace.c                         |    44 +-
 arch/arm64/kernel/sdei.c                           |   235 +
 arch/arm64/kernel/signal.c                         |     7 +-
 arch/arm64/kernel/signal32.c                       |    85 +-
 arch/arm64/kernel/sleep.S                          |     2 +-
 arch/arm64/kernel/smp.c                            |    11 +-
 arch/arm64/kernel/stacktrace.c                     |     5 +
 arch/arm64/kernel/suspend.c                        |     4 +-
 arch/arm64/kernel/sys_compat.c                     |     2 +-
 arch/arm64/kernel/time.c                           |     2 +-
 arch/arm64/kernel/topology.c                       |    16 +-
 arch/arm64/kernel/traps.c                          |    61 +-
 arch/arm64/kernel/vmlinux.lds.S                    |    27 +-
 arch/arm64/kvm/guest.c                             |    12 +-
 arch/arm64/kvm/handle_exit.c                       |    46 +-
 arch/arm64/kvm/hyp-init.S                          |    30 +-
 arch/arm64/kvm/hyp/debug-sr.c                      |     1 +
 arch/arm64/kvm/hyp/entry.S                         |    35 +-
 arch/arm64/kvm/hyp/hyp-entry.S                     |    38 +-
 arch/arm64/kvm/hyp/s2-setup.c                      |     2 +
 arch/arm64/kvm/hyp/switch.c                        |    53 +-
 arch/arm64/kvm/hyp/sysreg-sr.c                     |    22 +-
 arch/arm64/kvm/hyp/tlb.c                           |     1 +
 arch/arm64/kvm/inject_fault.c                      |    13 +-
 arch/arm64/kvm/sys_regs.c                          |    11 +
 arch/arm64/lib/clear_user.S                        |    10 +-
 arch/arm64/lib/copy_from_user.S                    |     4 +-
 arch/arm64/lib/copy_in_user.S                      |     9 +-
 arch/arm64/lib/copy_to_user.S                      |     4 +-
 arch/arm64/lib/tishift.S                           |     8 +-
 arch/arm64/mm/cache.S                              |    36 +-
 arch/arm64/mm/context.c                            |    67 +-
 arch/arm64/mm/dma-mapping.c                        |    54 +-
 arch/arm64/mm/dump.c                               |    54 +-
 arch/arm64/mm/fault.c                              |   194 +-
 arch/arm64/mm/hugetlbpage.c                        |    94 +-
 arch/arm64/mm/init.c                               |    62 +-
 arch/arm64/mm/kasan_init.c                         |    73 +-
 arch/arm64/mm/mmu.c                                |   356 +-
 arch/arm64/mm/pageattr.c                           |    32 +-
 arch/arm64/mm/pgd.c                                |     8 +
 arch/arm64/mm/proc.S                               |   274 +-
 arch/arm64/net/bpf_jit_comp.c                      |    91 +-
 arch/arm64/xen/hypercall.S                         |     4 +-
 arch/blackfin/include/asm/clkdev.h                 |    17 -
 arch/blackfin/include/asm/thread_info.h            |     2 -
 arch/blackfin/include/uapi/asm/poll.h              |     2 +-
 arch/blackfin/include/uapi/asm/siginfo.h           |    34 -
 arch/c6x/include/asm/Kbuild                        |     1 -
 arch/c6x/include/asm/thread_info.h                 |     3 -
 arch/c6x/kernel/traps.c                            |     4 +-
 arch/cris/Kconfig                                  |     4 +
 arch/cris/arch-v10/drivers/gpio.c                  |     8 +-
 arch/cris/arch-v10/drivers/sync_serial.c           |    16 +-
 arch/cris/arch-v32/drivers/cryptocop.c             |    23 +-
 arch/cris/arch-v32/drivers/pci/Makefile            |     2 +-
 arch/cris/arch-v32/drivers/pci/dma.c               |    80 -
 arch/cris/arch-v32/drivers/sync_serial.c           |    18 +-
 arch/cris/include/arch-v10/arch/bug.h              |    11 +-
 arch/cris/include/asm/Kbuild                       |     2 +-
 arch/cris/include/asm/dma-mapping.h                |    20 -
 arch/cris/include/asm/processor.h                  |     9 -
 arch/cris/include/asm/thread_info.h                |     9 +-
 arch/cris/kernel/Makefile                          |    14 +-
 arch/cris/kernel/devicetree.c                      |    15 -
 arch/cris/kernel/setup.c                           |     2 +-
 arch/cris/kernel/vmlinux.lds.S                     |     1 +
 arch/frv/include/asm/Kbuild                        |     1 -
 arch/frv/include/asm/thread_info.h                 |     3 -
 arch/frv/include/uapi/asm/Kbuild                   |     1 +
 arch/frv/include/uapi/asm/poll.h                   |     2 -
 arch/frv/include/uapi/asm/siginfo.h                |    13 -
 arch/h8300/Kconfig                                 |     1 +
 arch/h8300/include/asm/Kbuild                      |     2 +-
 arch/h8300/include/asm/byteorder.h                 |     1 -
 arch/h8300/include/asm/dma-mapping.h               |    12 -
 arch/h8300/include/asm/thread_info.h               |     3 -
 arch/h8300/kernel/Makefile                         |     2 +-
 arch/h8300/kernel/dma.c                            |    69 -
 arch/hexagon/include/asm/Kbuild                    |     1 -
 arch/hexagon/include/asm/dma-mapping.h             |     7 -
 arch/hexagon/include/asm/io.h                      |     2 -
 arch/hexagon/include/asm/thread_info.h             |     3 -
 arch/hexagon/kernel/dma.c                          |     1 +
 arch/hexagon/kernel/vmlinux.lds.S                  |     2 +
 arch/ia64/Kconfig                                  |    10 +-
 arch/ia64/Makefile                                 |     2 +-
 arch/ia64/hp/common/hwsw_iommu.c                   |     2 +-
 arch/ia64/include/asm/Kbuild                       |     1 -
 arch/ia64/include/asm/asm-prototypes.h             |     2 +-
 arch/ia64/include/asm/atomic.h                     |    69 +-
 arch/ia64/include/asm/bug.h                        |     6 +-
 arch/ia64/include/asm/dma-mapping.h                |    19 -
 arch/ia64/include/asm/dma.h                        |     2 -
 arch/ia64/include/asm/sections.h                   |    10 +-
 arch/ia64/include/asm/swiotlb.h                    |    18 -
 arch/ia64/include/asm/thread_info.h                |     4 +-
 arch/ia64/include/uapi/asm/Kbuild                  |     1 +
 arch/ia64/include/uapi/asm/poll.h                  |     2 -
 arch/ia64/include/uapi/asm/siginfo.h               |    96 -
 arch/ia64/kernel/Makefile                          |     3 +-
 arch/ia64/kernel/acpi.c                            |     5 +
 arch/ia64/kernel/dma-mapping.c                     |     9 +
 arch/ia64/kernel/err_inject.c                      |     4 +-
 arch/ia64/kernel/init_task.c                       |    44 -
 arch/ia64/kernel/module.c                          |    12 +
 arch/ia64/kernel/pci-dma.c                         |    19 +-
 arch/ia64/kernel/pci-swiotlb.c                     |    68 -
 arch/ia64/kernel/perfmon.c                         |    19 +-
 arch/ia64/kernel/process.c                         |    10 +-
 arch/ia64/kernel/signal.c                          |    52 -
 arch/ia64/kernel/vmlinux.lds.S                     |     3 +
 arch/ia64/mm/contig.c                              |     4 +-
 arch/ia64/mm/discontig.c                           |    14 +-
 arch/ia64/mm/init.c                                |    18 +-
 arch/ia64/scripts/unwcheck.py                      |    16 +-
 arch/m32r/Kconfig                                  |     2 +-
 arch/m32r/include/asm/Kbuild                       |     2 +-
 arch/m32r/include/asm/dma-mapping.h                |    24 -
 arch/m32r/include/asm/io.h                         |     2 -
 arch/m32r/include/asm/thread_info.h                |     3 -
 arch/m32r/include/uapi/asm/Kbuild                  |     1 +
 arch/m32r/include/uapi/asm/poll.h                  |     2 -
 arch/m32r/kernel/traps.c                           |     8 -
 arch/m68k/configs/amiga_defconfig                  |     5 +-
 arch/m68k/configs/apollo_defconfig                 |     5 +-
 arch/m68k/configs/atari_defconfig                  |     5 +-
 arch/m68k/configs/bvme6000_defconfig               |     5 +-
 arch/m68k/configs/hp300_defconfig                  |     5 +-
 arch/m68k/configs/mac_defconfig                    |     5 +-
 arch/m68k/configs/multi_defconfig                  |     5 +-
 arch/m68k/configs/mvme147_defconfig                |     5 +-
 arch/m68k/configs/mvme16x_defconfig                |     5 +-
 arch/m68k/configs/q40_defconfig                    |     5 +-
 arch/m68k/configs/sun3_defconfig                   |     5 +-
 arch/m68k/configs/sun3x_defconfig                  |     5 +-
 arch/m68k/include/asm/Kbuild                       |     1 -
 arch/m68k/include/asm/bitops.h                     |     3 +-
 arch/m68k/include/asm/bug.h                        |     3 +
 arch/m68k/include/asm/macintosh.h                  |     9 +-
 arch/m68k/include/asm/thread_info.h                |     4 -
 arch/m68k/kernel/dma.c                             |     2 -
 arch/m68k/mac/config.c                             |   110 +-
 arch/m68k/mac/oss.c                                |    67 +-
 arch/m68k/mm/fault.c                               |     3 +-
 arch/metag/include/asm/Kbuild                      |     1 -
 arch/metag/include/asm/thread_info.h               |     3 -
 arch/metag/include/uapi/asm/siginfo.h              |     7 +
 arch/metag/kernel/devtree.c                        |    14 -
 arch/metag/kernel/traps.c                          |     2 +-
 arch/microblaze/Kconfig                            |     1 +
 arch/microblaze/Kconfig.platform                   |     1 +
 arch/microblaze/Makefile                           |    17 +-
 arch/microblaze/include/asm/Kbuild                 |     1 -
 arch/microblaze/include/asm/dma-mapping.h          |     4 +-
 arch/microblaze/include/asm/io.h                   |     2 +-
 arch/microblaze/include/asm/setup.h                |     1 -
 arch/microblaze/include/asm/thread_info.h          |     3 -
 arch/microblaze/kernel/dma.c                       |    78 +-
 arch/microblaze/lib/fastcopy.S                     |     4 -
 arch/microblaze/mm/init.c                          |    64 +-
 arch/microblaze/mm/pgtable.c                       |     2 +-
 arch/mips/Kconfig                                  |    26 +-
 arch/mips/Makefile                                 |     8 +-
 arch/mips/ath25/board.c                            |     2 +
 arch/mips/bcm47xx/Platform                         |     1 +
 arch/mips/bcm63xx/boards/Kconfig                   |     1 -
 arch/mips/bcm63xx/dev-enet.c                       |     8 +
 arch/mips/boot/Makefile                            |     1 +
 arch/mips/boot/compressed/Makefile                 |     6 +-
 arch/mips/boot/dts/ingenic/Makefile                |     1 +
 arch/mips/boot/dts/ingenic/gcw0.dts                |    62 +
 arch/mips/boot/dts/ingenic/jz4770.dtsi             |   212 +
 arch/mips/cavium-octeon/Kconfig                    |     1 +
 arch/mips/cavium-octeon/dma-octeon.c               |    29 +-
 arch/mips/cavium-octeon/octeon-irq.c               |     2 +
 arch/mips/configs/ar7_defconfig                    |     1 -
 arch/mips/configs/ath25_defconfig                  |     1 -
 arch/mips/configs/ath79_defconfig                  |     1 -
 arch/mips/configs/bigsur_defconfig                 |     1 -
 arch/mips/configs/gcw0_defconfig                   |    27 +
 arch/mips/configs/generic/board-ranchu.config      |    30 +
 arch/mips/configs/ip27_defconfig                   |     1 -
 arch/mips/configs/ip32_defconfig                   |     1 -
 arch/mips/configs/malta_defconfig                  |     5 +-
 arch/mips/configs/malta_kvm_defconfig              |     4 +-
 arch/mips/configs/malta_kvm_guest_defconfig        |     4 +-
 arch/mips/configs/malta_qemu_32r6_defconfig        |     1 +
 arch/mips/configs/maltaaprp_defconfig              |     1 +
 arch/mips/configs/maltasmvp_defconfig              |     1 +
 arch/mips/configs/maltasmvp_eva_defconfig          |     1 +
 arch/mips/configs/maltaup_defconfig                |     1 +
 arch/mips/configs/maltaup_xpa_defconfig            |     5 +-
 arch/mips/configs/nlm_xlp_defconfig                |     1 -
 arch/mips/configs/nlm_xlr_defconfig                |     1 -
 arch/mips/configs/pic32mzda_defconfig              |     1 -
 arch/mips/configs/pnx8335_stb225_defconfig         |     1 -
 arch/mips/configs/qi_lb60_defconfig                |     1 -
 arch/mips/configs/rm200_defconfig                  |     9 -
 arch/mips/configs/rt305x_defconfig                 |     1 -
 arch/mips/configs/sb1250_swarm_defconfig           |     1 -
 arch/mips/configs/xway_defconfig                   |     1 -
 arch/mips/generic/Kconfig                          |    10 +
 arch/mips/generic/Makefile                         |     1 +
 arch/mips/generic/board-ranchu.c                   |    93 +
 arch/mips/generic/irq.c                            |    18 +-
 arch/mips/include/asm/Kbuild                       |     1 -
 arch/mips/include/asm/asm-prototypes.h             |     2 +-
 arch/mips/include/asm/bootinfo.h                   |     2 +
 arch/mips/include/asm/checksum.h                   |     2 +-
 arch/mips/include/asm/compat.h                     |    74 -
 arch/mips/include/asm/dma-direct.h                 |     1 +
 arch/mips/include/asm/dma-mapping.h                |    10 -
 .../include/asm/mach-bcm63xx/bcm63xx_dev_enet.h    |     4 +
 .../include/asm/mach-cavium-octeon/dma-coherence.h |     8 +
 arch/mips/include/asm/mach-generic/dma-coherence.h |    12 -
 arch/mips/include/asm/mach-loongson64/boot_param.h |     2 +-
 .../include/asm/mach-loongson64/dma-coherence.h    |     8 +
 arch/mips/include/asm/machine.h                    |     2 +-
 arch/mips/include/asm/mipsregs.h                   |   702 +-
 arch/mips/include/asm/msa.h                        |    63 +-
 arch/mips/include/asm/netlogic/common.h            |     3 -
 arch/mips/include/asm/pgtable.h                    |     3 +
 arch/mips/include/asm/thread_info.h                |     3 -
 arch/mips/include/uapi/asm/siginfo.h               |    86 +-
 arch/mips/jazz/setup.c                             |     8 +-
 arch/mips/jz4740/Kconfig                           |    10 +
 arch/mips/jz4740/prom.c                            |    25 +-
 arch/mips/jz4740/setup.c                           |    22 +-
 arch/mips/jz4740/time.c                            |     2 +-
 arch/mips/kernel/branch.c                          |     2 +-
 arch/mips/kernel/cps-vec.S                         |    17 +-
 arch/mips/kernel/ftrace.c                          |     2 +-
 arch/mips/kernel/mips-cpc.c                        |    13 +
 arch/mips/kernel/prom.c                            |     5 -
 arch/mips/kernel/rtlx.c                            |     8 +-
 arch/mips/kernel/setup.c                           |    55 +-
 arch/mips/kernel/signal32.c                        |    67 -
 arch/mips/kernel/smp-bmips.c                       |    10 +-
 arch/mips/kernel/smp-cps.c                         |     2 -
 arch/mips/kernel/traps.c                           |    29 +-
 arch/mips/kernel/watch.c                           |    31 +-
 arch/mips/kvm/Kconfig                              |     1 +
 arch/mips/kvm/mips.c                               |    67 +-
 arch/mips/lantiq/Kconfig                           |     2 +
 arch/mips/lantiq/xway/sysctrl.c                    |     6 +-
 arch/mips/loongson64/Kconfig                       |     9 +-
 arch/mips/loongson64/common/dma-swiotlb.c          |    24 +-
 arch/mips/loongson64/common/mem.c                  |     2 +-
 arch/mips/loongson64/loongson-3/numa.c             |     2 +-
 arch/mips/math-emu/cp1emu.c                        |    28 +-
 arch/mips/math-emu/dp_add.c                        |     3 +-
 arch/mips/math-emu/dp_div.c                        |     1 +
 arch/mips/math-emu/dp_fmax.c                       |     2 +
 arch/mips/math-emu/dp_fmin.c                       |     2 +
 arch/mips/math-emu/dp_maddf.c                      |     8 +-
 arch/mips/math-emu/dp_mul.c                        |     4 +-
 arch/mips/math-emu/dp_sqrt.c                       |     8 +-
 arch/mips/math-emu/dp_sub.c                        |     2 +-
 arch/mips/math-emu/ieee754dp.h                     |     3 +
 arch/mips/math-emu/sp_add.c                        |     3 +-
 arch/mips/math-emu/sp_div.c                        |     1 +
 arch/mips/math-emu/sp_fdp.c                        |     3 +-
 arch/mips/math-emu/sp_fmax.c                       |     2 +
 arch/mips/math-emu/sp_fmin.c                       |     2 +
 arch/mips/math-emu/sp_maddf.c                      |     3 +-
 arch/mips/math-emu/sp_mul.c                        |     1 +
 arch/mips/math-emu/sp_sqrt.c                       |     3 +-
 arch/mips/math-emu/sp_sub.c                        |     1 +
 arch/mips/math-emu/sp_tlong.c                      |     1 -
 arch/mips/mm/dma-default.c                         |     9 -
 arch/mips/mm/sc-mips.c                             |     9 +
 arch/mips/mti-malta/malta-setup.c                  |    10 +-
 arch/mips/net/bpf_jit.c                            |     2 -
 arch/mips/net/ebpf_jit.c                           |    33 +-
 arch/mips/netlogic/Kconfig                         |     5 -
 arch/mips/netlogic/common/Makefile                 |     1 -
 arch/mips/netlogic/common/nlm-dma.c                |    97 -
 arch/mips/ralink/mt7621.c                          |    42 +-
 arch/mips/ralink/reset.c                           |     7 -
 arch/mips/txx9/rbtx4939/setup.c                    |     4 +-
 arch/mn10300/configs/asb2364_defconfig             |     1 -
 arch/mn10300/include/asm/Kbuild                    |     1 -
 arch/mn10300/include/asm/thread_info.h             |     2 -
 arch/mn10300/include/uapi/asm/Kbuild               |     1 +
 arch/mn10300/include/uapi/asm/poll.h               |     2 -
 arch/mn10300/kernel/mn10300-serial.c               |     7 +-
 arch/mn10300/kernel/traps.c                        |     4 +-
 arch/mn10300/mm/dma-alloc.c                        |     3 -
 arch/mn10300/mm/misalignment.c                     |     2 +-
 arch/nios2/Kconfig                                 |     1 -
 arch/nios2/boot/dts/3c120_devboard.dts             |    16 +-
 arch/nios2/configs/10m50_defconfig                 |     1 -
 arch/nios2/configs/3c120_defconfig                 |     1 -
 arch/nios2/include/asm/Kbuild                      |     1 -
 arch/nios2/include/asm/thread_info.h               |     3 -
 arch/nios2/kernel/prom.c                           |     5 -
 arch/nios2/mm/dma-mapping.c                        |     3 -
 arch/openrisc/include/asm/Kbuild                   |     1 -
 arch/openrisc/include/asm/processor.h              |     2 -
 arch/openrisc/include/asm/thread_info.h            |     2 -
 arch/openrisc/kernel/traps.c                       |    13 +-
 arch/openrisc/kernel/vmlinux.lds.S                 |     1 +
 arch/parisc/boot/compressed/vmlinux.lds.S          |     2 +
 arch/parisc/include/asm/Kbuild                     |     1 -
 arch/parisc/include/asm/cacheflush.h               |     1 +
 arch/parisc/include/asm/compat.h                   |    64 -
 arch/parisc/include/asm/processor.h                |     2 +
 arch/parisc/include/asm/sections.h                 |     6 +
 arch/parisc/include/asm/thread_info.h              |     3 -
 arch/parisc/include/uapi/asm/siginfo.h             |     7 +
 arch/parisc/kernel/cache.c                         |    88 +-
 arch/parisc/kernel/head.S                          |    18 +-
 arch/parisc/kernel/module.c                        |    16 +
 arch/parisc/kernel/pacache.S                       |    22 +
 arch/parisc/kernel/pci-dma.c                       |     7 -
 arch/parisc/kernel/pdt.c                           |     2 +-
 arch/parisc/kernel/process.c                       |     9 +
 arch/parisc/kernel/signal32.c                      |   106 -
 arch/parisc/kernel/signal32.h                      |     3 -
 arch/parisc/kernel/smp.c                           |     7 +-
 arch/parisc/kernel/time.c                          |    11 +-
 arch/parisc/kernel/traps.c                         |     2 +-
 arch/parisc/kernel/vmlinux.lds.S                   |     2 +
 arch/parisc/mm/init.c                              |     7 +-
 arch/powerpc/Kconfig                               |    22 +-
 arch/powerpc/Kconfig.debug                         |     6 +-
 arch/powerpc/Makefile                              |     1 +
 arch/powerpc/boot/Makefile                         |    11 +-
 arch/powerpc/boot/dts/a3m071.dts                   |    10 +-
 arch/powerpc/boot/dts/akebono.dts                  |     4 +-
 arch/powerpc/boot/dts/c2k.dts                      |     6 +-
 arch/powerpc/boot/dts/currituck.dts                |     2 +-
 arch/powerpc/boot/dts/fsl/mpc8568mds.dts           |    12 +-
 arch/powerpc/boot/dts/fsl/mpc8569mds.dts           |    20 +-
 arch/powerpc/boot/dts/fsl/p1010si-post.dtsi        |     2 +
 arch/powerpc/boot/dts/fsl/p1021mds.dts             |     6 +-
 arch/powerpc/boot/dts/fsl/p1025rdb.dtsi            |     8 +-
 arch/powerpc/boot/dts/fsl/p1025rdb_32b.dts         |     2 +-
 arch/powerpc/boot/dts/fsl/p1025twr.dtsi            |     8 +-
 arch/powerpc/boot/dts/fsl/t1040rdb.dts             |     2 +-
 arch/powerpc/boot/dts/fsl/t1042d4rdb.dts           |    10 +-
 arch/powerpc/boot/dts/fsl/t1042rdb.dts             |     2 +-
 arch/powerpc/boot/dts/fsl/t104xrdb.dtsi            |     6 +-
 arch/powerpc/boot/dts/fsp2.dts                     |     6 +-
 arch/powerpc/boot/dts/gamecube.dts                 |    14 +-
 arch/powerpc/boot/dts/haleakala.dts                |     2 +-
 arch/powerpc/boot/dts/kilauea.dts                  |     4 +-
 arch/powerpc/boot/dts/kmeter1.dts                  |    10 +-
 arch/powerpc/boot/dts/makalu.dts                   |     4 +-
 arch/powerpc/boot/dts/mpc832x_mds.dts              |    10 +-
 arch/powerpc/boot/dts/mpc832x_rdb.dts              |     8 +-
 arch/powerpc/boot/dts/mpc836x_mds.dts              |     8 +-
 arch/powerpc/boot/dts/sbc8548-altflash.dts         |     8 +-
 arch/powerpc/boot/dts/sbc8548.dts                  |     8 +-
 arch/powerpc/boot/dts/wii.dts                      |    32 +-
 arch/powerpc/boot/serial.c                         |     6 +
 arch/powerpc/configs/44x/warp_defconfig            |     1 -
 arch/powerpc/configs/c2k_defconfig                 |    12 -
 arch/powerpc/configs/fsl-emb-nonhw.config          |     1 -
 arch/powerpc/configs/g5_defconfig                  |    12 -
 arch/powerpc/configs/maple_defconfig               |    12 -
 arch/powerpc/configs/mpc512x_defconfig             |     1 -
 arch/powerpc/configs/mpc866_ads_defconfig          |     1 -
 arch/powerpc/configs/pmac32_defconfig              |    12 -
 arch/powerpc/configs/powernv_defconfig             |     3 +-
 arch/powerpc/configs/ppc64_defconfig               |     1 -
 arch/powerpc/configs/ppc6xx_defconfig              |     1 -
 arch/powerpc/configs/ps3_defconfig                 |     1 -
 arch/powerpc/configs/pseries_defconfig             |     1 -
 arch/powerpc/configs/wii_defconfig                 |     1 -
 arch/powerpc/crypto/crc32c-vpmsum_glue.c           |     1 +
 arch/powerpc/include/asm/Kbuild                    |     1 -
 arch/powerpc/include/asm/book3s/32/pgtable.h       |    24 +
 arch/powerpc/include/asm/book3s/64/hash-4k.h       |    23 +-
 arch/powerpc/include/asm/book3s/64/hash-64k.h      |    83 +-
 arch/powerpc/include/asm/book3s/64/hash.h          |    21 +-
 arch/powerpc/include/asm/book3s/64/mmu-hash.h      |     5 +
 arch/powerpc/include/asm/book3s/64/mmu.h           |    13 +
 arch/powerpc/include/asm/book3s/64/pgalloc.h       |    16 +-
 arch/powerpc/include/asm/book3s/64/pgtable.h       |    96 +-
 arch/powerpc/include/asm/book3s/64/radix.h         |     6 -
 arch/powerpc/include/asm/book3s/64/tlbflush-hash.h |     1 +
 .../powerpc/include/asm/book3s/64/tlbflush-radix.h |    10 +-
 arch/powerpc/include/asm/book3s/64/tlbflush.h      |    38 +
 arch/powerpc/include/asm/bug.h                     |     4 +-
 arch/powerpc/include/asm/code-patching.h           |     2 +
 arch/powerpc/include/asm/compat.h                  |    65 -
 arch/powerpc/include/asm/cpm.h                     |     2 +-
 arch/powerpc/include/asm/cpm1.h                    |     2 +
 arch/powerpc/include/asm/cputable.h                |    22 +-
 arch/powerpc/include/asm/debug.h                   |     2 +-
 arch/powerpc/include/asm/dma-direct.h              |    29 +
 arch/powerpc/include/asm/dma-mapping.h             |    36 +-
 arch/powerpc/include/asm/drmem.h                   |   102 +
 arch/powerpc/include/asm/eeh.h                     |     2 +
 arch/powerpc/include/asm/exception-64s.h           |   103 +-
 arch/powerpc/include/asm/firmware.h                |     5 +-
 arch/powerpc/include/asm/hardirq.h                 |     1 +
 arch/powerpc/include/asm/head-64.h                 |    47 +-
 arch/powerpc/include/asm/hmi.h                     |     4 +
 arch/powerpc/include/asm/hugetlb.h                 |     3 +-
 arch/powerpc/include/asm/hw_irq.h                  |   173 +-
 arch/powerpc/include/asm/imc-pmu.h                 |     9 +-
 arch/powerpc/include/asm/irqflags.h                |    14 +-
 arch/powerpc/include/asm/kexec.h                   |     8 +
 arch/powerpc/include/asm/kvm_book3s.h              |     6 +-
 arch/powerpc/include/asm/kvm_book3s_64.h           |    14 +-
 arch/powerpc/include/asm/kvm_host.h                |     8 +-
 arch/powerpc/include/asm/kvm_ppc.h                 |     6 +-
 arch/powerpc/include/asm/local.h                   |   200 +-
 arch/powerpc/include/asm/machdep.h                 |     8 +
 arch/powerpc/include/asm/membarrier.h              |    27 +
 arch/powerpc/include/asm/mman.h                    |    13 +-
 arch/powerpc/include/asm/mmu-8xx.h                 |    60 +-
 arch/powerpc/include/asm/mmu.h                     |     9 +
 arch/powerpc/include/asm/mmu_context.h             |    40 +-
 arch/powerpc/include/asm/module.h                  |     3 +
 arch/powerpc/include/asm/mpic_timer.h              |     8 +-
 arch/powerpc/include/asm/nmi.h                     |     4 -
 arch/powerpc/include/asm/nohash/32/pgalloc.h       |     3 +-
 arch/powerpc/include/asm/nohash/32/pgtable.h       |     3 +-
 arch/powerpc/include/asm/nohash/32/pte-8xx.h       |    25 +-
 arch/powerpc/include/asm/nohash/64/pgtable.h       |     1 +
 arch/powerpc/include/asm/nohash/pgtable.h          |    27 +-
 arch/powerpc/include/asm/nohash/pte-book3e.h       |     1 +
 arch/powerpc/include/asm/opal-api.h                |     6 +-
 arch/powerpc/include/asm/opal.h                    |     6 +
 arch/powerpc/include/asm/paca.h                    |     5 +-
 arch/powerpc/include/asm/pci-bridge.h              |     9 +-
 arch/powerpc/include/asm/pci.h                     |     2 +
 arch/powerpc/include/asm/pkeys.h                   |   218 +
 arch/powerpc/include/asm/pnv-ocxl.h                |    36 +
 arch/powerpc/include/asm/ppc-opcode.h              |     8 +
 arch/powerpc/include/asm/processor.h               |     5 +
 arch/powerpc/include/asm/prom.h                    |    27 +-
 arch/powerpc/include/asm/pte-common.h              |    37 +-
 arch/powerpc/include/asm/reg.h                     |     6 +-
 arch/powerpc/include/asm/reg_8xx.h                 |    82 -
 arch/powerpc/include/asm/sections.h                |    12 +
 arch/powerpc/include/asm/swiotlb.h                 |     4 +-
 arch/powerpc/include/asm/systbl.h                  |     3 +
 arch/powerpc/include/asm/thread_info.h             |     3 -
 arch/powerpc/include/asm/topology.h                |    13 +
 arch/powerpc/include/asm/unistd.h                  |     6 +-
 arch/powerpc/include/asm/xive-regs.h               |    35 +
 arch/powerpc/include/asm/xive.h                    |    43 +-
 arch/powerpc/include/uapi/asm/elf.h                |     1 +
 arch/powerpc/include/uapi/asm/kvm.h                |     2 +
 arch/powerpc/include/uapi/asm/mman.h               |     6 +
 arch/powerpc/include/uapi/asm/siginfo.h            |    16 +-
 arch/powerpc/include/uapi/asm/unistd.h             |     3 +
 arch/powerpc/kernel/asm-offsets.c                  |    11 +-
 arch/powerpc/kernel/cpu_setup_power.S              |    50 -
 arch/powerpc/kernel/cputable.c                     |    15 -
 arch/powerpc/kernel/crash.c                        |    16 +-
 arch/powerpc/kernel/dma-iommu.c                    |     2 +-
 arch/powerpc/kernel/dma-swiotlb.c                  |    12 +-
 arch/powerpc/kernel/dma.c                          |    73 +-
 arch/powerpc/kernel/dt_cpu_ftrs.c                  |    36 +-
 arch/powerpc/kernel/eeh.c                          |    59 +
 arch/powerpc/kernel/eeh_driver.c                   |    11 +-
 arch/powerpc/kernel/eeh_sysfs.c                    |    64 +-
 arch/powerpc/kernel/entry_32.S                     |    10 +-
 arch/powerpc/kernel/entry_64.S                     |    67 +-
 arch/powerpc/kernel/exceptions-64e.S               |    22 +-
 arch/powerpc/kernel/exceptions-64s.S               |   126 +-
 arch/powerpc/kernel/head_64.S                      |    11 +-
 arch/powerpc/kernel/head_8xx.S                     |   275 +-
 arch/powerpc/kernel/idle_book3e.S                  |     5 +-
 arch/powerpc/kernel/idle_power4.S                  |     5 +-
 arch/powerpc/kernel/irq.c                          |    37 +-
 arch/powerpc/kernel/mce.c                          |   144 +-
 arch/powerpc/kernel/mce_power.c                    |   115 +-
 arch/powerpc/kernel/module.lds                     |     8 +
 arch/powerpc/kernel/module_64.c                    |    49 +-
 arch/powerpc/kernel/optprobes_head.S               |     2 +-
 arch/powerpc/kernel/paca.c                         |    13 +-
 arch/powerpc/kernel/pci-common.c                   |    40 +-
 arch/powerpc/kernel/pci-hotplug.c                  |    20 +-
 arch/powerpc/kernel/pci_32.c                       |     3 +-
 arch/powerpc/kernel/pci_dn.c                       |     6 -
 arch/powerpc/kernel/pci_of_scan.c                  |     9 +-
 arch/powerpc/kernel/proc_powerpc.c                 |     2 +-
 arch/powerpc/kernel/process.c                      |    41 +-
 arch/powerpc/kernel/prom.c                         |   115 +-
 arch/powerpc/kernel/prom_init.c                    |     1 +
 arch/powerpc/kernel/ptrace.c                       |    78 +
 arch/powerpc/kernel/rtas-proc.c                    |    14 +-
 arch/powerpc/kernel/rtas_flash.c                   |     2 +-
 arch/powerpc/kernel/rtasd.c                        |     6 +-
 arch/powerpc/kernel/setup-common.c                 |    23 +-
 arch/powerpc/kernel/setup.h                        |     4 +
 arch/powerpc/kernel/setup_64.c                     |    48 +-
 arch/powerpc/kernel/signal.c                       |     6 +-
 arch/powerpc/kernel/signal_32.c                    |    74 +-
 arch/powerpc/kernel/signal_64.c                    |    11 +-
 arch/powerpc/kernel/smp.c                          |    18 +
 arch/powerpc/kernel/sysfs.c                        |    10 +
 arch/powerpc/kernel/time.c                         |     6 +-
 arch/powerpc/kernel/traps.c                        |    73 +-
 arch/powerpc/kernel/vdso64/gettimeofday.S          |    67 +-
 arch/powerpc/kernel/vmlinux.lds.S                  |     3 +
 arch/powerpc/kernel/watchdog.c                     |   100 +-
 arch/powerpc/kvm/Kconfig                           |     3 +-
 arch/powerpc/kvm/book3s.c                          |    24 +-
 arch/powerpc/kvm/book3s_64_mmu_hv.c                |    38 +-
 arch/powerpc/kvm/book3s_64_mmu_radix.c             |    74 +-
 arch/powerpc/kvm/book3s_hv.c                       |    97 +-
 arch/powerpc/kvm/book3s_hv_ras.c                   |    14 +-
 arch/powerpc/kvm/book3s_hv_rm_mmu.c                |    20 +-
 arch/powerpc/kvm/book3s_hv_rmhandlers.S            |   287 +-
 arch/powerpc/kvm/book3s_interrupts.S               |     4 +-
 arch/powerpc/kvm/book3s_pr.c                       |    20 +-
 arch/powerpc/kvm/book3s_xics.c                     |     2 +-
 arch/powerpc/kvm/book3s_xive.c                     |   113 +-
 arch/powerpc/kvm/book3s_xive.h                     |    15 +-
 arch/powerpc/kvm/booke.c                           |    51 +-
 arch/powerpc/kvm/emulate_loadstore.c               |    36 +
 arch/powerpc/kvm/powerpc.c                         |   202 +-
 arch/powerpc/kvm/timing.c                          |     3 +-
 arch/powerpc/lib/code-patching.c                   |    37 +-
 arch/powerpc/lib/feature-fixups.c                  |     8 +-
 arch/powerpc/mm/8xx_mmu.c                          |     4 +-
 arch/powerpc/mm/Makefile                           |     3 +-
 arch/powerpc/mm/drmem.c                            |   447 +
 arch/powerpc/mm/dump_linuxpagetables.c             |    32 +-
 arch/powerpc/mm/fault.c                            |    53 +-
 arch/powerpc/mm/hash64_4k.c                        |    14 +-
 arch/powerpc/mm/hash64_64k.c                       |   125 +-
 arch/powerpc/mm/hash_native_64.c                   |   113 +-
 arch/powerpc/mm/hash_utils_64.c                    |    98 +-
 arch/powerpc/mm/hugetlbpage-hash64.c               |    22 +-
 arch/powerpc/mm/hugetlbpage.c                      |     8 +-
 arch/powerpc/mm/init-common.c                      |     4 +-
 arch/powerpc/mm/init_64.c                          |    22 +-
 arch/powerpc/mm/mem.c                              |    14 +-
 arch/powerpc/mm/mmu_context.c                      |     7 +
 arch/powerpc/mm/mmu_context_book3s64.c             |     3 +
 arch/powerpc/mm/numa.c                             |   338 +-
 arch/powerpc/mm/pgtable-book3s64.c                 |     7 +-
 arch/powerpc/mm/pgtable-hash64.c                   |    22 -
 arch/powerpc/mm/pgtable-radix.c                    |   138 +-
 arch/powerpc/mm/pgtable.c                          |     3 +-
 arch/powerpc/mm/pgtable_32.c                       |     9 +-
 arch/powerpc/mm/pgtable_64.c                       |    19 +-
 arch/powerpc/mm/pkeys.c                            |   468 +
 arch/powerpc/mm/subpage-prot.c                     |     3 +
 arch/powerpc/mm/tlb-radix.c                        |   237 +-
 arch/powerpc/mm/tlb_hash64.c                       |     9 +-
 arch/powerpc/mm/tlb_nohash.c                       |     5 +-
 arch/powerpc/net/bpf_jit_comp.c                    |     5 +-
 arch/powerpc/net/bpf_jit_comp64.c                  |    13 +-
 arch/powerpc/perf/8xx-pmu.c                        |    52 +-
 arch/powerpc/perf/Makefile                         |     2 +-
 arch/powerpc/perf/core-book3s.c                    |     2 +-
 arch/powerpc/perf/imc-pmu.c                        |   102 +-
 arch/powerpc/platforms/44x/fsp2.c                  |   259 +-
 arch/powerpc/platforms/44x/fsp2.h                  |   272 +
 arch/powerpc/platforms/512x/mpc512x_shared.c       |     4 +-
 arch/powerpc/platforms/52xx/mpc52xx_gpt.c          |    52 +-
 arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c     |     2 +-
 arch/powerpc/platforms/83xx/mpc832x_mds.c          |     2 +-
 arch/powerpc/platforms/83xx/mpc832x_rdb.c          |     2 +-
 arch/powerpc/platforms/83xx/mpc836x_mds.c          |     2 +-
 arch/powerpc/platforms/85xx/socrates_fpga_pic.c    |     7 +-
 arch/powerpc/platforms/86xx/mpc86xx_hpcn.c         |     2 +-
 arch/powerpc/platforms/8xx/Kconfig                 |    12 -
 arch/powerpc/platforms/Kconfig                     |    11 -
 arch/powerpc/platforms/Kconfig.cputype             |     8 -
 arch/powerpc/platforms/cell/interrupt.c            |     3 +-
 arch/powerpc/platforms/cell/iommu.c                |    28 +-
 arch/powerpc/platforms/cell/setup.c                |     3 +-
 arch/powerpc/platforms/cell/spider-pic.c           |     3 +-
 arch/powerpc/platforms/cell/spu_manage.c           |     4 +-
 arch/powerpc/platforms/cell/spufs/backing_ops.c    |    14 +-
 arch/powerpc/platforms/cell/spufs/file.c           |    32 +-
 arch/powerpc/platforms/cell/spufs/hw_ops.c         |    13 +-
 arch/powerpc/platforms/cell/spufs/inode.c          |     4 +-
 arch/powerpc/platforms/cell/spufs/spufs.h          |     5 +-
 arch/powerpc/platforms/maple/time.c                |     2 +-
 arch/powerpc/platforms/pasemi/dma_lib.c            |     4 +-
 arch/powerpc/platforms/pasemi/iommu.c              |     2 +-
 arch/powerpc/platforms/pasemi/setup.c              |     2 +-
 arch/powerpc/platforms/powermac/backlight.c        |     6 +-
 arch/powerpc/platforms/powermac/feature.c          |     5 +-
 arch/powerpc/platforms/powermac/pic.c              |     8 +-
 arch/powerpc/platforms/powermac/smp.c              |     4 +-
 arch/powerpc/platforms/powernv/Makefile            |     1 +
 arch/powerpc/platforms/powernv/eeh-powernv.c       |   101 +-
 arch/powerpc/platforms/powernv/npu-dma.c           |     7 +-
 arch/powerpc/platforms/powernv/ocxl.c              |   515 +
 arch/powerpc/platforms/powernv/opal-dump.c         |     4 +-
 arch/powerpc/platforms/powernv/opal-elog.c         |     4 +-
 arch/powerpc/platforms/powernv/opal-imc.c          |    83 +-
 .../powerpc/platforms/powernv/opal-memory-errors.c |     2 +-
 arch/powerpc/platforms/powernv/opal-prd.c          |     4 +-
 arch/powerpc/platforms/powernv/opal-sysparam.c     |     6 +-
 arch/powerpc/platforms/powernv/opal-wrappers.S     |     3 +
 arch/powerpc/platforms/powernv/opal.c              |    28 +-
 arch/powerpc/platforms/powernv/pci-ioda.c          |    94 +-
 arch/powerpc/platforms/powernv/pci.c               |     4 +
 arch/powerpc/platforms/powernv/pci.h               |     8 +-
 arch/powerpc/platforms/powernv/setup.c             |     4 +
 arch/powerpc/platforms/powernv/smp.c               |    28 +
 arch/powerpc/platforms/powernv/vas-window.c        |    16 +-
 arch/powerpc/platforms/ps3/device-init.c           |    12 +-
 arch/powerpc/platforms/ps3/mm.c                    |     4 +-
 arch/powerpc/platforms/ps3/os-area.c               |     2 +-
 arch/powerpc/platforms/ps3/setup.c                 |     1 +
 arch/powerpc/platforms/pseries/cmm.c               |    16 +-
 arch/powerpc/platforms/pseries/eeh_pseries.c       |   162 +-
 arch/powerpc/platforms/pseries/firmware.c          |     2 +
 arch/powerpc/platforms/pseries/hotplug-cpu.c       |     3 +
 arch/powerpc/platforms/pseries/hotplug-memory.c    |   522 +-
 arch/powerpc/platforms/pseries/hvCall_inst.c       |     2 +-
 arch/powerpc/platforms/pseries/ibmebus.c           |     4 +-
 arch/powerpc/platforms/pseries/iommu.c             |    10 +-
 arch/powerpc/platforms/pseries/lparcfg.c           |     8 +-
 arch/powerpc/platforms/pseries/mobility.c          |     2 +-
 arch/powerpc/platforms/pseries/of_helpers.c        |    60 +
 arch/powerpc/platforms/pseries/pci.c               |   177 +-
 arch/powerpc/platforms/pseries/pseries_energy.c    |   126 +-
 arch/powerpc/platforms/pseries/ras.c               |    31 +-
 arch/powerpc/platforms/pseries/reconfig.c          |     2 +-
 arch/powerpc/platforms/pseries/scanlog.c           |     2 +-
 arch/powerpc/platforms/pseries/setup.c             |   179 +-
 arch/powerpc/platforms/pseries/suspend.c           |     3 +-
 arch/powerpc/platforms/pseries/vio.c               |     2 +-
 arch/powerpc/sysdev/Makefile                       |     4 +-
 arch/powerpc/sysdev/axonram.c                      |   383 -
 arch/powerpc/sysdev/cpm1.c                         |    33 +-
 arch/powerpc/sysdev/cpm2.c                         |    11 -
 arch/powerpc/sysdev/cpm_common.c                   |     5 +-
 arch/powerpc/sysdev/cpm_gpio.c                     |    80 +
 arch/powerpc/sysdev/dart_iommu.c                   |     4 +-
 arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c        |    16 +-
 arch/powerpc/sysdev/fsl_pci.c                      |    18 +-
 arch/powerpc/sysdev/i8259.c                        |     6 +-
 arch/powerpc/sysdev/mpic.c                         |    10 +-
 arch/powerpc/sysdev/mpic_timer.c                   |    55 +-
 arch/powerpc/sysdev/mv64x60_pci.c                  |     6 +-
 arch/powerpc/sysdev/xics/icp-native.c              |    10 +-
 arch/powerpc/sysdev/xics/ics-opal.c                |     4 +-
 arch/powerpc/sysdev/xics/ics-rtas.c                |     4 +-
 arch/powerpc/sysdev/xics/xics-common.c             |     8 +-
 arch/powerpc/sysdev/xive/common.c                  |     8 +-
 arch/powerpc/sysdev/xive/native.c                  |    18 +-
 arch/powerpc/sysdev/xive/spapr.c                   |    16 +-
 arch/powerpc/xmon/ppc-dis.c                        |     4 -
 arch/powerpc/xmon/xmon.c                           |     6 +-
 arch/riscv/Kconfig                                 |    15 +-
 arch/riscv/include/asm/Kbuild                      |     3 +-
 arch/riscv/include/asm/barrier.h                   |     6 +-
 arch/riscv/include/asm/csr.h                       |    14 +-
 arch/riscv/include/asm/dma-mapping.h               |    38 -
 arch/riscv/include/asm/ftrace.h                    |    10 +
 arch/riscv/include/asm/mmu_context.h               |    17 +-
 arch/riscv/include/asm/thread_info.h               |     2 -
 arch/riscv/include/asm/tlbflush.h                  |    20 +-
 arch/riscv/include/asm/unistd.h                    |     1 -
 arch/riscv/kernel/Makefile                         |     7 +
 arch/riscv/kernel/entry.S                          |    14 +-
 arch/riscv/kernel/ftrace.c                         |    41 +
 arch/riscv/kernel/head.S                           |     8 +-
 arch/riscv/kernel/mcount.S                         |   126 +
 arch/riscv/kernel/process.c                        |     2 +-
 arch/riscv/kernel/setup.c                          |    46 +-
 arch/riscv/kernel/vdso.c                           |     2 +-
 arch/riscv/mm/fault.c                              |     5 +-
 arch/riscv/mm/init.c                               |    12 +-
 arch/s390/Kconfig                                  |    49 +-
 arch/s390/Makefile                                 |    24 +-
 arch/s390/boot/compressed/Makefile                 |     1 +
 arch/s390/boot/compressed/vmlinux.lds.S            |    12 +-
 arch/s390/configs/zfcpdump_defconfig               |     1 -
 arch/s390/crypto/crc32-vx.c                        |     3 +
 arch/s390/include/asm/Kbuild                       |     7 +-
 arch/s390/include/asm/barrier.h                    |    24 +
 arch/s390/include/asm/bitops.h                     |     5 +
 arch/s390/include/asm/compat.h                     |    73 -
 arch/s390/include/asm/css_chars.h                  |     4 +-
 arch/s390/include/asm/diag.h                       |    44 +-
 arch/s390/include/asm/dis.h                        |     2 +-
 arch/s390/include/asm/dma-mapping.h                |    26 -
 arch/s390/include/asm/dwarf.h                      |    37 +
 arch/s390/include/asm/eadm.h                       |     2 +-
 arch/s390/include/asm/facility.h                   |    20 +-
 arch/s390/include/asm/kvm_host.h                   |   126 +-
 arch/s390/include/asm/lowcore.h                    |     9 +-
 arch/s390/include/asm/mmu_context.h                |     1 +
 arch/s390/include/asm/nospec-branch.h              |    18 +
 arch/s390/include/asm/pci_dma.h                    |     3 +
 arch/s390/include/asm/pgtable.h                    |     4 +-
 arch/s390/include/asm/processor.h                  |     4 +
 arch/s390/include/asm/runtime_instr.h              |    67 +-
 arch/s390/include/asm/sclp.h                       |     1 +
 arch/s390/include/asm/sysinfo.h                    |     3 +-
 arch/s390/include/asm/thread_info.h                |     6 +-
 arch/s390/include/asm/unistd.h                     |     1 +
 arch/s390/include/uapi/asm/Kbuild                  |     3 +
 arch/s390/include/uapi/asm/runtime_instr.h         |    74 +
 arch/s390/include/uapi/asm/unistd.h                |   401 +-
 arch/s390/kernel/Makefile                          |     4 +
 arch/s390/kernel/alternative.c                     |    26 +-
 arch/s390/kernel/compat_linux.c                    |     8 +-
 arch/s390/kernel/compat_signal.c                   |   100 -
 arch/s390/kernel/early.c                           |     5 +
 arch/s390/kernel/entry.S                           |   255 +-
 arch/s390/kernel/head.S                            |    10 +-
 arch/s390/kernel/ipl.c                             |    13 +-
 arch/s390/kernel/kprobes.c                         |     3 +-
 arch/s390/kernel/module.c                          |    62 +-
 arch/s390/kernel/nospec-branch.c                   |   100 +
 arch/s390/kernel/perf_cpum_cf_events.c             |     2 +-
 arch/s390/kernel/processor.c                       |    18 +
 arch/s390/kernel/runtime_instr.c                   |    10 +-
 arch/s390/kernel/setup.c                           |     8 +-
 arch/s390/kernel/smp.c                             |     9 +-
 arch/s390/kernel/syscalls.S                        |   392 -
 arch/s390/kernel/syscalls/Makefile                 |    52 +
 arch/s390/kernel/syscalls/syscall.tbl              |   390 +
 arch/s390/kernel/syscalls/syscalltbl               |   232 +
 arch/s390/kernel/sysinfo.c                         |     2 +
 arch/s390/kernel/topology.c                        |     3 +-
 arch/s390/kernel/vdso32/Makefile                   |     3 +
 arch/s390/kernel/vdso32/clock_getres.S             |     5 +-
 arch/s390/kernel/vdso32/clock_gettime.S            |    17 +-
 arch/s390/kernel/vdso32/getcpu.S                   |     5 +-
 arch/s390/kernel/vdso32/gettimeofday.S             |     9 +-
 arch/s390/kernel/vdso64/Makefile                   |     3 +
 arch/s390/kernel/vdso64/clock_getres.S             |     5 +-
 arch/s390/kernel/vdso64/clock_gettime.S            |    21 +-
 arch/s390/kernel/vdso64/getcpu.S                   |     5 +-
 arch/s390/kernel/vdso64/gettimeofday.S             |     9 +-
 arch/s390/kernel/vmlinux.lds.S                     |    17 +
 arch/s390/kvm/Kconfig                              |     1 +
 arch/s390/kvm/diag.c                               |     1 +
 arch/s390/kvm/intercept.c                          |    51 +-
 arch/s390/kvm/interrupt.c                          |   401 +-
 arch/s390/kvm/kvm-s390.c                           |   290 +-
 arch/s390/kvm/kvm-s390.h                           |    29 +-
 arch/s390/kvm/priv.c                               |   230 +-
 arch/s390/kvm/sigp.c                               |    18 +-
 arch/s390/kvm/vsie.c                               |   111 +-
 arch/s390/mm/gmap.c                                |    44 +-
 arch/s390/mm/init.c                                |     7 +-
 arch/s390/mm/vmem.c                                |     6 +-
 arch/s390/net/bpf_jit_comp.c                       |    14 +-
 arch/s390/tools/.gitignore                         |     1 +
 arch/s390/tools/Makefile                           |    23 +-
 arch/s390/tools/gen_facilities.c                   |     4 +-
 arch/s390/tools/gen_opcode_table.c                 |     4 +-
 arch/score/include/asm/Kbuild                      |     1 -
 arch/score/include/asm/thread_info.h               |     3 -
 arch/score/include/uapi/asm/Kbuild                 |     1 +
 arch/score/include/uapi/asm/poll.h                 |     7 -
 arch/score/kernel/setup.c                          |     4 +-
 arch/sh/boards/board-espt.c                        |     1 -
 arch/sh/boards/board-sh7757lcr.c                   |     4 -
 arch/sh/boards/mach-ecovec24/setup.c               |     1 -
 arch/sh/boards/mach-se/7724/setup.c                |     1 -
 arch/sh/boards/mach-sh7763rdp/setup.c              |     1 -
 arch/sh/boot/dts/Makefile                          |     4 +-
 arch/sh/configs/polaris_defconfig                  |     1 -
 arch/sh/drivers/push-switch.c                      |     2 +-
 arch/sh/include/asm/Kbuild                         |     1 -
 arch/sh/include/asm/thread_info.h                  |     3 -
 arch/sh/kernel/cpu/sh2/setup-sh7619.c              |     1 -
 arch/sh/kernel/process_32.c                        |     5 +-
 arch/sh/kernel/traps_32.c                          |     3 +-
 arch/sh/mm/init.c                                  |    10 +-
 arch/sparc/Kconfig                                 |     2 +
 arch/sparc/crypto/crc32c_glue.c                    |     1 +
 arch/sparc/include/asm/Kbuild                      |     1 -
 arch/sparc/include/asm/asm-prototypes.h            |     2 +-
 arch/sparc/include/asm/bug.h                       |     6 +-
 arch/sparc/include/asm/compat.h                    |    59 -
 arch/sparc/include/asm/hypervisor.h                |   138 +
 arch/sparc/include/asm/pgtable_64.h                |     2 +-
 arch/sparc/include/asm/thread_info_32.h            |     3 -
 arch/sparc/include/asm/thread_info_64.h            |     3 -
 arch/sparc/include/uapi/asm/oradax.h               |    91 +
 arch/sparc/kernel/hvapi.c                          |     1 +
 arch/sparc/kernel/hvcalls.S                        |    57 +
 arch/sparc/kernel/signal32.c                       |    69 -
 arch/sparc/kernel/signal_64.c                      |     3 -
 arch/sparc/mm/init_64.c                            |     5 +-
 arch/sparc/mm/tlb.c                                |    42 +-
 arch/sparc/net/bpf_jit_comp_32.c                   |     2 -
 arch/sparc/net/bpf_jit_comp_64.c                   |    66 +-
 arch/sparc/vdso/vma.c                              |     2 +-
 arch/tile/Kconfig                                  |     3 +-
 arch/tile/configs/tilegx_defconfig                 |     1 -
 arch/tile/configs/tilepro_defconfig                |     1 -
 arch/tile/include/asm/Kbuild                       |     1 -
 arch/tile/include/asm/compat.h                     |    62 -
 arch/tile/include/asm/dma-mapping.h                |    20 -
 arch/tile/include/asm/thread_info.h                |     3 -
 arch/tile/include/uapi/asm/siginfo.h               |     8 -
 arch/tile/kernel/compat_signal.c                   |    73 -
 arch/tile/kernel/pci-dma.c                         |    38 +-
 arch/tile/kernel/setup.c                           |     8 +-
 arch/tile/kernel/single_step.c                     |    24 +-
 arch/tile/kernel/sysfs.c                           |    12 +-
 arch/tile/kernel/traps.c                           |     4 +-
 arch/tile/kernel/unaligned.c                       |    46 +-
 arch/um/Makefile                                   |     7 +
 arch/um/drivers/hostaudio_kern.c                   |     6 +-
 arch/um/drivers/mconsole_kern.c                    |     3 +-
 arch/um/include/asm/Kbuild                         |     1 -
 arch/um/include/asm/processor-generic.h            |     5 +-
 arch/um/include/asm/thread_info.h                  |     9 +-
 arch/um/include/asm/vmlinux.lds.h                  |     2 +
 arch/um/kernel/dyn.lds.S                           |     3 +-
 arch/um/kernel/trap.c                              |     2 +-
 arch/um/kernel/um_arch.c                           |     2 +-
 arch/um/kernel/uml.lds.S                           |     2 +-
 arch/unicore32/include/asm/Kbuild                  |     1 -
 arch/unicore32/include/asm/bitops.h                |     2 +
 arch/unicore32/include/asm/dma-mapping.h           |    29 +-
 arch/unicore32/include/asm/thread_info.h           |     3 -
 arch/unicore32/kernel/process.c                    |     5 +-
 arch/unicore32/mm/Kconfig                          |     1 +
 arch/unicore32/mm/Makefile                         |     2 -
 arch/unicore32/mm/dma-swiotlb.c                    |    48 -
 arch/x86/.gitignore                                |     1 +
 arch/x86/Kconfig                                   |   112 +-
 arch/x86/Kconfig.cpu                               |    17 +-
 arch/x86/Kconfig.debug                             |     8 -
 arch/x86/Makefile                                  |    16 +-
 arch/x86/boot/compressed/eboot.c                   |     5 +-
 arch/x86/boot/compressed/misc.c                    |     4 +
 arch/x86/crypto/aesni-intel_asm.S                  |   199 +-
 arch/x86/crypto/aesni-intel_glue.c                 |    70 +-
 arch/x86/crypto/chacha20_glue.c                    |     1 -
 arch/x86/crypto/crc32-pclmul_glue.c                |     1 +
 arch/x86/crypto/crc32c-intel_glue.c                |     1 +
 arch/x86/crypto/poly1305_glue.c                    |     2 -
 arch/x86/crypto/salsa20-i586-asm_32.S              |   184 +-
 arch/x86/crypto/salsa20-x86_64-asm_64.S            |   114 -
 arch/x86/crypto/salsa20_glue.c                     |   105 +-
 .../x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c |    10 +-
 arch/x86/crypto/twofish-x86_64-asm_64-3way.S       |   112 +-
 arch/x86/entry/calling.h                           |   121 +-
 arch/x86/entry/common.c                            |    15 +-
 arch/x86/entry/entry_32.S                          |     8 +
 arch/x86/entry/entry_64.S                          |   338 +-
 arch/x86/entry/entry_64_compat.S                   |    65 +-
 arch/x86/entry/syscall_64.c                        |     7 +-
 arch/x86/entry/syscalls/syscall_32.tbl             |    38 +-
 arch/x86/entry/vdso/vdso32/vclock_gettime.c        |     2 -
 arch/x86/entry/vsyscall/vsyscall_64.c              |    18 +-
 arch/x86/events/core.c                             |     3 +-
 arch/x86/events/intel/core.c                       |    16 +-
 arch/x86/events/intel/ds.c                         |    31 +-
 arch/x86/events/intel/lbr.c                        |     2 +-
 arch/x86/events/intel/p6.c                         |     2 +-
 arch/x86/events/intel/uncore_snbep.c               |    34 +-
 arch/x86/events/msr.c                              |    70 +-
 arch/x86/events/perf_event.h                       |     9 +-
 arch/x86/hyperv/hv_init.c                          |   144 +-
 arch/x86/hyperv/mmu.c                              |    12 +-
 arch/x86/ia32/sys_ia32.c                           |    74 +-
 arch/x86/include/asm/Kbuild                        |     1 -
 arch/x86/include/asm/acpi.h                        |     4 +-
 arch/x86/include/asm/alternative.h                 |     4 +-
 arch/x86/include/asm/apm.h                         |     6 +
 arch/x86/include/asm/barrier.h                     |    58 +-
 arch/x86/include/asm/bitops.h                      |    29 +-
 arch/x86/include/asm/bug.h                         |    19 +-
 arch/x86/include/asm/compat.h                      |    86 +-
 arch/x86/include/asm/cpufeature.h                  |    86 +-
 arch/x86/include/asm/cpufeatures.h                 |    27 +-
 arch/x86/include/asm/disabled-features.h           |     3 +-
 arch/x86/include/asm/dma-direct.h                  |    30 +
 arch/x86/include/asm/dma-mapping.h                 |    29 +-
 arch/x86/include/asm/efi.h                         |    17 +-
 arch/x86/include/asm/error-injection.h             |    13 +
 arch/x86/include/asm/fixmap.h                      |     6 +-
 arch/x86/include/asm/fpu/signal.h                  |     6 -
 arch/x86/include/asm/hardirq.h                     |     3 +
 arch/x86/include/asm/hw_irq.h                      |     1 +
 arch/x86/include/asm/hypervisor.h                  |     1 +
 arch/x86/include/asm/i8259.h                       |     5 +
 arch/x86/include/asm/intel-family.h                |     6 +
 arch/x86/include/asm/intel_pmc_ipc.h               |     6 +
 arch/x86/include/asm/io.h                          |    15 -
 arch/x86/include/asm/iosf_mbi.h                    |    25 +
 arch/x86/include/asm/irq_vectors.h                 |     7 +-
 arch/x86/include/asm/jailhouse_para.h              |    26 +
 arch/x86/include/asm/kasan.h                       |    12 +-
 arch/x86/include/asm/kprobes.h                     |     2 +
 arch/x86/include/asm/kvm_host.h                    |    28 +-
 arch/x86/include/asm/mce.h                         |     2 +
 arch/x86/include/asm/microcode.h                   |    10 +-
 arch/x86/include/asm/mmu_context.h                 |     1 +
 arch/x86/include/asm/mpspec_def.h                  |    14 +-
 arch/x86/include/asm/mshyperv.h                    |    36 +-
 arch/x86/include/asm/msr-index.h                   |    14 +
 arch/x86/include/asm/msr.h                         |     3 +-
 arch/x86/include/asm/nospec-branch.h               |   114 +-
 arch/x86/include/asm/page_64.h                     |     4 -
 arch/x86/include/asm/paravirt.h                    |    21 +-
 arch/x86/include/asm/paravirt_types.h              |     7 +-
 arch/x86/include/asm/pat.h                         |     2 +
 arch/x86/include/asm/percpu.h                      |     2 +-
 arch/x86/include/asm/pgtable-3level.h              |    37 +-
 arch/x86/include/asm/pgtable.h                     |    23 +-
 arch/x86/include/asm/pgtable_32.h                  |     3 +-
 arch/x86/include/asm/pgtable_32_types.h            |     5 +-
 arch/x86/include/asm/pgtable_64.h                  |     1 +
 arch/x86/include/asm/pgtable_types.h               |    12 +-
 arch/x86/include/asm/pmc_core.h                    |    27 -
 arch/x86/include/asm/processor.h                   |    15 +-
 arch/x86/include/asm/ptrace.h                      |     5 +
 arch/x86/include/asm/refcount.h                    |     6 +-
 arch/x86/include/asm/required-features.h           |     3 +-
 arch/x86/include/asm/rmwcc.h                       |    16 +-
 arch/x86/include/asm/sections.h                    |     1 +
 arch/x86/include/asm/smp.h                         |     1 +
 arch/x86/include/asm/svm.h                         |     3 +
 arch/x86/include/asm/swiotlb.h                     |     2 -
 arch/x86/include/asm/sync_core.h                   |    28 +
 arch/x86/include/asm/sys_ia32.h                    |    48 +-
 arch/x86/include/asm/syscall.h                     |     6 +-
 arch/x86/include/asm/thread_info.h                 |     5 +-
 arch/x86/include/asm/tlbflush.h                    |    29 +-
 arch/x86/include/asm/uaccess.h                     |    15 +-
 arch/x86/include/asm/uaccess_32.h                  |     6 +-
 arch/x86/include/asm/uaccess_64.h                  |    12 +-
 arch/x86/include/asm/uprobes.h                     |     4 +
 arch/x86/include/asm/uv/uv_bau.h                   |     1 -
 arch/x86/include/asm/uv/uv_hub.h                   |    14 +
 arch/x86/include/asm/uv/uv_mmrs.h                  |   749 +-
 arch/x86/include/asm/vmx.h                         |     1 +
 arch/x86/include/asm/x86_init.h                    |     1 +
 arch/x86/include/uapi/asm/Kbuild                   |     1 +
 arch/x86/include/uapi/asm/bootparam.h              |    22 +
 arch/x86/include/uapi/asm/hyperv.h                 |    27 +
 arch/x86/include/uapi/asm/kvm_para.h               |     5 +
 arch/x86/include/uapi/asm/mce.h                    |     1 +
 arch/x86/include/uapi/asm/poll.h                   |     1 -
 arch/x86/kernel/Makefile                           |     2 +
 arch/x86/kernel/acpi/boot.c                        |    38 +-
 arch/x86/kernel/acpi/sleep.c                       |     2 +
 arch/x86/kernel/alternative.c                      |    28 +-
 arch/x86/kernel/amd_gart_64.c                      |     1 +
 arch/x86/kernel/amd_nb.c                           |     2 +-
 arch/x86/kernel/aperture_64.c                      |    46 +-
 arch/x86/kernel/apic/apic.c                        |     6 +-
 arch/x86/kernel/apic/apic_flat_64.c                |    24 +-
 arch/x86/kernel/apic/io_apic.c                     |    22 +-
 arch/x86/kernel/apic/vector.c                      |    25 +-
 arch/x86/kernel/apic/x2apic_uv_x.c                 |    99 +-
 arch/x86/kernel/apm_32.c                           |     5 +-
 arch/x86/kernel/asm-offsets_32.c                   |     2 +-
 arch/x86/kernel/cpu/amd.c                          |    94 +-
 arch/x86/kernel/cpu/bugs.c                         |   152 +-
 arch/x86/kernel/cpu/centaur.c                      |     8 +-
 arch/x86/kernel/cpu/common.c                       |   110 +-
 arch/x86/kernel/cpu/cyrix.c                        |     2 +-
 arch/x86/kernel/cpu/hypervisor.c                   |     4 +
 arch/x86/kernel/cpu/intel.c                        |    85 +-
 arch/x86/kernel/cpu/intel_rdt.c                    |    70 +-
 arch/x86/kernel/cpu/intel_rdt.h                    |     5 +
 arch/x86/kernel/cpu/intel_rdt_rdtgroup.c           |   118 +-
 arch/x86/kernel/cpu/mcheck/dev-mcelog.c            |     6 +-
 arch/x86/kernel/cpu/mcheck/mce-internal.h          |    15 +
 arch/x86/kernel/cpu/mcheck/mce-severity.c          |    26 +-
 arch/x86/kernel/cpu/mcheck/mce.c                   |    67 +-
 arch/x86/kernel/cpu/mcheck/mce_amd.c               |    29 +-
 arch/x86/kernel/cpu/microcode/amd.c                |    44 +-
 arch/x86/kernel/cpu/microcode/core.c               |   183 +-
 arch/x86/kernel/cpu/microcode/intel.c              |    68 +-
 arch/x86/kernel/cpu/mshyperv.c                     |     6 +
 arch/x86/kernel/cpu/mtrr/generic.c                 |     2 +-
 arch/x86/kernel/cpu/mtrr/main.c                    |     4 +-
 arch/x86/kernel/cpu/proc.c                         |     8 +-
 arch/x86/kernel/cpu/scattered.c                    |     4 +-
 arch/x86/kernel/devicetree.c                       |     6 -
 arch/x86/kernel/dumpstack.c                        |     2 +-
 arch/x86/kernel/early-quirks.c                     |    87 +-
 arch/x86/kernel/head_32.S                          |     4 +-
 arch/x86/kernel/head_64.S                          |     2 +
 arch/x86/kernel/idt.c                              |     5 +-
 arch/x86/kernel/ioport.c                           |     2 +-
 arch/x86/kernel/irq.c                              |     9 +
 arch/x86/kernel/itmt.c                             |     1 -
 arch/x86/kernel/jailhouse.c                        |   211 +
 arch/x86/kernel/kprobes/core.c                     |    10 +-
 arch/x86/kernel/kvm.c                              |    65 +-
 arch/x86/kernel/machine_kexec_64.c                 |     1 +
 arch/x86/kernel/module.c                           |     1 +
 arch/x86/kernel/mpparse.c                          |    25 +-
 arch/x86/kernel/paravirt.c                         |     6 +-
 arch/x86/kernel/pci-dma.c                          |    23 +-
 arch/x86/kernel/pci-nommu.c                        |    21 +-
 arch/x86/kernel/pci-swiotlb.c                      |     8 +-
 arch/x86/kernel/platform-quirks.c                  |     1 +
 arch/x86/kernel/process.c                          |     1 -
 arch/x86/kernel/process_64.c                       |     4 +-
 arch/x86/kernel/ptrace.c                           |     2 +-
 arch/x86/kernel/relocate_kernel_64.S               |     8 +
 arch/x86/kernel/setup.c                            |    18 +-
 arch/x86/kernel/setup_percpu.c                     |    17 +-
 arch/x86/kernel/signal.c                           |     2 +-
 arch/x86/kernel/signal_compat.c                    |   188 +-
 arch/x86/kernel/smpboot.c                          |    17 +-
 arch/x86/kernel/time.c                             |     9 +-
 arch/x86/kernel/traps.c                            |    17 +-
 arch/x86/kernel/tsc.c                              |    61 +-
 arch/x86/kernel/unwind_orc.c                       |     3 +-
 arch/x86/kernel/uprobes.c                          |   107 +-
 arch/x86/kernel/vm86_32.c                          |     3 +-
 arch/x86/kernel/vmlinux.lds.S                      |     2 +
 arch/x86/kvm/Kconfig                               |     8 +
 arch/x86/kvm/cpuid.c                               |    45 +-
 arch/x86/kvm/cpuid.h                               |     1 +
 arch/x86/kvm/emulate.c                             |    71 +-
 arch/x86/kvm/irq.c                                 |     2 +-
 arch/x86/kvm/lapic.c                               |    36 +-
 arch/x86/kvm/lapic.h                               |     4 +-
 arch/x86/kvm/mmu.c                                 |    42 +-
 arch/x86/kvm/mmu_audit.c                           |     2 +-
 arch/x86/kvm/svm.c                                 |  1441 +-
 arch/x86/kvm/vmx.c                                 |  1422 +-
 arch/x86/kvm/vmx_shadow_fields.h                   |    77 +
 arch/x86/kvm/x86.c                                 |   453 +-
 arch/x86/kvm/x86.h                                 |    33 +-
 arch/x86/lib/Makefile                              |     1 +
 arch/x86/lib/cpu.c                                 |     2 +-
 arch/x86/lib/delay.c                               |     2 +-
 arch/x86/lib/error-inject.c                        |    20 +
 arch/x86/lib/getuser.S                             |    10 +
 arch/x86/lib/usercopy_32.c                         |     8 +-
 arch/x86/mm/cpu_entry_area.c                       |     6 +
 arch/x86/mm/extable.c                              |    34 +-
 arch/x86/mm/fault.c                                |    10 +-
 arch/x86/mm/init_32.c                              |    24 +-
 arch/x86/mm/init_64.c                              |    78 +-
 arch/x86/mm/ioremap.c                              |     2 +-
 arch/x86/mm/kmmio.c                                |     2 +-
 arch/x86/mm/mem_encrypt.c                          |     2 +-
 arch/x86/mm/mem_encrypt_boot.S                     |     2 +
 arch/x86/mm/mmio-mod.c                             |     5 +-
 arch/x86/mm/pat.c                                  |    19 +
 arch/x86/mm/pgtable.c                              |    48 +
 arch/x86/mm/pgtable_32.c                           |     2 +-
 arch/x86/mm/pti.c                                  |     2 +-
 arch/x86/mm/tlb.c                                  |    45 +-
 arch/x86/net/bpf_jit_comp.c                        |   118 +-
 arch/x86/oprofile/nmi_int.c                        |     2 +-
 arch/x86/pci/intel_mid_pci.c                       |     1 +
 arch/x86/pci/irq.c                                 |     3 +-
 arch/x86/pci/sta2x11-fixup.c                       |     1 +
 arch/x86/pci/xen.c                                 |     4 +-
 arch/x86/platform/efi/efi_64.c                     |     3 +-
 arch/x86/platform/intel-mid/intel-mid.c            |     4 +-
 arch/x86/platform/intel-mid/sfi.c                  |     5 +-
 arch/x86/platform/intel/iosf_mbi.c                 |    19 +-
 arch/x86/platform/uv/tlb_uv.c                      |     7 +-
 arch/x86/power/hibernate_32.c                      |     2 +-
 arch/x86/power/hibernate_64.c                      |     2 +-
 arch/x86/purgatory/Makefile                        |     2 +-
 arch/x86/realmode/rm/trampoline_64.S               |     2 +-
 arch/x86/tools/Makefile                            |    12 +-
 .../tools/{test_get_len.c => insn_decoder_test.c}  |    43 +-
 .../tools/{distill.awk => objdump_reformat.awk}    |     4 +-
 arch/x86/tools/relocs.c                            |     3 +
 arch/x86/um/asm/barrier.h                          |     4 -
 arch/x86/xen/enlighten_pv.c                        |     6 +-
 arch/x86/xen/mmu_hvm.c                             |     2 +-
 arch/x86/xen/mmu_pv.c                              |     6 +-
 arch/x86/xen/p2m.c                                 |     6 +
 arch/x86/xen/smp.c                                 |     2 +
 arch/x86/xen/spinlock.c                            |     2 -
 arch/x86/xen/suspend.c                             |    16 +
 arch/x86/xen/xen-head.S                            |    16 +
 arch/xtensa/Kconfig                                |     7 +
 arch/xtensa/Makefile                               |     7 +-
 arch/xtensa/boot/boot-redboot/bootstrap.S          |     1 +
 arch/xtensa/boot/lib/Makefile                      |     6 +
 arch/xtensa/configs/audio_kc705_defconfig          |     1 -
 arch/xtensa/configs/cadence_csp_defconfig          |     1 -
 arch/xtensa/configs/generic_kc705_defconfig        |     1 -
 arch/xtensa/configs/nommu_kc705_defconfig          |     1 -
 arch/xtensa/configs/smp_lx200_defconfig            |     1 -
 arch/xtensa/include/asm/Kbuild                     |     1 -
 arch/xtensa/include/asm/asmmacro.h                 |    40 +
 arch/xtensa/include/asm/current.h                  |     4 +-
 arch/xtensa/include/asm/dma-mapping.h              |    10 -
 arch/xtensa/include/asm/fixmap.h                   |     4 +-
 arch/xtensa/include/asm/futex.h                    |    23 +-
 arch/xtensa/include/asm/highmem.h                  |     2 +-
 arch/xtensa/include/asm/kasan.h                    |    39 +
 arch/xtensa/include/asm/kmem_layout.h              |     7 +
 arch/xtensa/include/asm/linkage.h                  |     9 +
 arch/xtensa/include/asm/mmu_context.h              |     1 +
 arch/xtensa/include/asm/nommu_context.h            |     4 +
 arch/xtensa/include/asm/page.h                     |     2 -
 arch/xtensa/include/asm/pgtable.h                  |     3 +-
 arch/xtensa/include/asm/ptrace.h                   |    15 +-
 arch/xtensa/include/asm/regs.h                     |     1 +
 arch/xtensa/include/asm/stackprotector.h           |    40 +
 arch/xtensa/include/asm/string.h                   |    23 +-
 arch/xtensa/include/asm/thread_info.h              |    16 +-
 arch/xtensa/include/asm/traps.h                    |    35 +
 arch/xtensa/include/asm/uaccess.h                  |     9 +-
 arch/xtensa/kernel/Makefile                        |     3 -
 arch/xtensa/kernel/align.S                         |     7 +-
 arch/xtensa/kernel/asm-offsets.c                   |    16 +
 arch/xtensa/kernel/coprocessor.S                   |     3 +-
 arch/xtensa/kernel/entry.S                         |   103 +-
 arch/xtensa/kernel/head.S                          |    10 +-
 arch/xtensa/kernel/module.c                        |    19 +-
 arch/xtensa/kernel/pci-dma.c                       |    40 +-
 arch/xtensa/kernel/pci.c                           |    30 +-
 arch/xtensa/kernel/process.c                       |     6 +
 arch/xtensa/kernel/ptrace.c                        |     8 +-
 arch/xtensa/kernel/setup.c                         |    61 +-
 arch/xtensa/kernel/signal.c                        |     8 +-
 arch/xtensa/kernel/traps.c                         |    64 +-
 arch/xtensa/kernel/vectors.S                       |    17 +-
 arch/xtensa/kernel/vmlinux.lds.S                   |    90 +-
 arch/xtensa/kernel/xtensa_ksyms.c                  |     5 +
 arch/xtensa/lib/checksum.S                         |    74 +-
 arch/xtensa/lib/memcopy.S                          |    81 +-
 arch/xtensa/lib/memset.S                           |    45 +-
 arch/xtensa/lib/pci-auto.c                         |    45 +-
 arch/xtensa/lib/strncpy_user.S                     |    60 +-
 arch/xtensa/lib/strnlen_user.S                     |    28 +-
 arch/xtensa/lib/usercopy.S                         |   138 +-
 arch/xtensa/mm/Makefile                            |     5 +
 arch/xtensa/mm/cache.c                             |     3 -
 arch/xtensa/mm/fault.c                             |    22 +-
 arch/xtensa/mm/init.c                              |   106 +-
 arch/xtensa/mm/kasan_init.c                        |    95 +
 arch/xtensa/mm/mmu.c                               |    31 +-
 arch/xtensa/mm/tlb.c                               |     6 +-
 arch/xtensa/platforms/iss/console.c                |     4 +-
 arch/xtensa/platforms/iss/network.c                |    14 +-
 block/bfq-cgroup.c                                 |     7 +-
 block/bfq-iosched.c                                |   602 +-
 block/bfq-iosched.h                                |    19 +
 block/bfq-wf2q.c                                   |     7 +
 block/bio-integrity.c                              |     1 -
 block/bio.c                                        |    30 +-
 block/blk-cgroup.c                                 |    11 +-
 block/blk-core.c                                   |   103 +-
 block/blk-exec.c                                   |     2 +-
 block/blk-lib.c                                    |    12 +
 block/blk-map.c                                    |     4 +-
 block/blk-merge.c                                  |    42 +-
 block/blk-mq-debugfs.c                             |    22 +-
 block/blk-mq-sched.c                               |     5 +-
 block/blk-mq-sched.h                               |     2 +-
 block/blk-mq-sysfs.c                               |     9 +-
 block/blk-mq-tag.c                                 |    13 +-
 block/blk-mq.c                                     |   690 +-
 block/blk-mq.h                                     |    52 +-
 block/blk-sysfs.c                                  |    47 +-
 block/blk-throttle.c                               |   146 +-
 block/blk-timeout.c                                |    26 +-
 block/blk-wbt.c                                    |    10 +-
 block/blk-zoned.c                                  |    42 +
 block/blk.h                                        |    46 +-
 block/bounce.c                                     |    33 +-
 block/bsg-lib.c                                    |     3 +-
 block/bsg.c                                        |    48 +-
 block/deadline-iosched.c                           |   114 +-
 block/elevator.c                                   |    12 +-
 block/genhd.c                                      |    71 +-
 block/ioctl.c                                      |     2 +-
 block/kyber-iosched.c                              |     1 +
 block/mq-deadline.c                                |   151 +-
 block/partition-generic.c                          |     6 +
 block/partitions/msdos.c                           |     4 +-
 block/scsi_ioctl.c                                 |    34 +-
 block/sed-opal.c                                   |     2 +-
 certs/blacklist_nohashes.c                         |     2 +-
 crypto/Kconfig                                     |     5 +-
 crypto/Makefile                                    |     1 +
 crypto/ablk_helper.c                               |     5 +-
 crypto/aead.c                                      |    19 +-
 crypto/af_alg.c                                    |    30 +-
 crypto/ahash.c                                     |    33 +-
 crypto/algapi.c                                    |    13 +-
 crypto/algif_aead.c                                |    15 +-
 crypto/algif_hash.c                                |    52 +-
 crypto/algif_skcipher.c                            |    60 +-
 crypto/api.c                                       |     6 +-
 crypto/asymmetric_keys/pkcs7_trust.c               |     1 +
 crypto/asymmetric_keys/pkcs7_verify.c              |    12 +-
 crypto/asymmetric_keys/public_key.c                |     4 +-
 crypto/asymmetric_keys/restrict.c                  |    21 +-
 crypto/authenc.c                                   |     4 +-
 crypto/authencesn.c                                |     4 +-
 crypto/blkcipher.c                                 |     1 -
 crypto/camellia_generic.c                          |     3 +-
 crypto/cast5_generic.c                             |     3 +-
 crypto/cast6_generic.c                             |     3 +-
 crypto/chacha20_generic.c                          |    33 +-
 crypto/crc32_generic.c                             |     1 +
 crypto/crc32c_generic.c                            |     1 +
 crypto/cryptd.c                                    |    17 +-
 crypto/crypto_user.c                               |     4 +-
 crypto/ecc.c                                       |     2 +-
 crypto/echainiv.c                                  |     5 -
 crypto/gcm.c                                       |     4 +-
 crypto/gf128mul.c                                  |     2 -
 crypto/ghash-generic.c                             |     6 -
 crypto/internal.h                                  |     8 +-
 crypto/keywrap.c                                   |     4 +-
 crypto/mcryptd.c                                   |    11 +-
 crypto/poly1305_generic.c                          |    27 +-
 crypto/proc.c                                      |     2 +-
 crypto/salsa20_generic.c                           |   240 +-
 crypto/scompress.c                                 |    51 +-
 crypto/seqiv.c                                     |     5 -
 crypto/sha3_generic.c                              |   364 +-
 crypto/shash.c                                     |    25 +-
 crypto/simd.c                                      |     4 +-
 crypto/skcipher.c                                  |    30 +-
 crypto/tcrypt.c                                    |  1085 +-
 crypto/testmgr.c                                   |    41 +-
 crypto/testmgr.h                                   |   550 +
 crypto/twofish_common.c                            |     5 +-
 crypto/twofish_generic.c                           |     5 +-
 crypto/xcbc.c                                      |     3 +-
 drivers/Kconfig                                    |     8 +
 drivers/Makefile                                   |     9 +-
 drivers/acpi/Kconfig                               |    39 +-
 drivers/acpi/acpi_dbg.c                            |     8 +-
 drivers/acpi/acpi_lpit.c                           |     1 +
 drivers/acpi/acpi_lpss.c                           |   139 +
 drivers/acpi/acpi_video.c                          |    16 +-
 drivers/acpi/acpi_watchdog.c                       |     4 +-
 drivers/acpi/acpica/acapps.h                       |     7 +-
 drivers/acpi/acpica/accommon.h                     |     2 +-
 drivers/acpi/acpica/acconvert.h                    |     2 +-
 drivers/acpi/acpica/acdebug.h                      |     6 +-
 drivers/acpi/acpica/acdispat.h                     |     2 +-
 drivers/acpi/acpica/acevents.h                     |     2 +-
 drivers/acpi/acpica/acglobal.h                     |    84 +-
 drivers/acpi/acpica/achware.h                      |     2 +-
 drivers/acpi/acpica/acinterp.h                     |     2 +-
 drivers/acpi/acpica/aclocal.h                      |    17 +-
 drivers/acpi/acpica/acmacros.h                     |     4 +-
 drivers/acpi/acpica/acnamesp.h                     |     5 +-
 drivers/acpi/acpica/acobject.h                     |     2 +-
 drivers/acpi/acpica/acopcode.h                     |     2 +-
 drivers/acpi/acpica/acparser.h                     |     2 +-
 drivers/acpi/acpica/acpredef.h                     |     2 +-
 drivers/acpi/acpica/acresrc.h                      |     2 +-
 drivers/acpi/acpica/acstruct.h                     |     2 +-
 drivers/acpi/acpica/actables.h                     |     2 +-
 drivers/acpi/acpica/acutils.h                      |    25 +-
 drivers/acpi/acpica/amlcode.h                      |     2 +-
 drivers/acpi/acpica/amlresrc.h                     |     2 +-
 drivers/acpi/acpica/dbcmds.c                       |     2 +-
 drivers/acpi/acpica/dbconvert.c                    |     2 +-
 drivers/acpi/acpica/dbdisply.c                     |     2 +-
 drivers/acpi/acpica/dbexec.c                       |   112 +-
 drivers/acpi/acpica/dbfileio.c                     |     6 +-
 drivers/acpi/acpica/dbhistry.c                     |     2 +-
 drivers/acpi/acpica/dbinput.c                      |   147 +-
 drivers/acpi/acpica/dbmethod.c                     |     2 +-
 drivers/acpi/acpica/dbnames.c                      |     2 +-
 drivers/acpi/acpica/dbobject.c                     |     2 +-
 drivers/acpi/acpica/dbstats.c                      |     2 +-
 drivers/acpi/acpica/dbtest.c                       |     2 +-
 drivers/acpi/acpica/dbutils.c                      |     2 +-
 drivers/acpi/acpica/dbxface.c                      |     2 +-
 drivers/acpi/acpica/dsargs.c                       |     2 +-
 drivers/acpi/acpica/dscontrol.c                    |    20 +-
 drivers/acpi/acpica/dsdebug.c                      |     2 +-
 drivers/acpi/acpica/dsfield.c                      |    30 +-
 drivers/acpi/acpica/dsinit.c                       |     2 +-
 drivers/acpi/acpica/dsmethod.c                     |     2 +-
 drivers/acpi/acpica/dsmthdat.c                     |     2 +-
 drivers/acpi/acpica/dsobject.c                     |     6 +-
 drivers/acpi/acpica/dsopcode.c                     |     2 +-
 drivers/acpi/acpica/dspkginit.c                    |    23 +-
 drivers/acpi/acpica/dsutils.c                      |     5 +-
 drivers/acpi/acpica/dswexec.c                      |     2 +-
 drivers/acpi/acpica/dswload.c                      |     8 +-
 drivers/acpi/acpica/dswload2.c                     |    15 +-
 drivers/acpi/acpica/dswscope.c                     |     2 +-
 drivers/acpi/acpica/dswstate.c                     |     2 +-
 drivers/acpi/acpica/evevent.c                      |     2 +-
 drivers/acpi/acpica/evglock.c                      |     2 +-
 drivers/acpi/acpica/evgpe.c                        |     2 +-
 drivers/acpi/acpica/evgpeblk.c                     |     2 +-
 drivers/acpi/acpica/evgpeinit.c                    |     2 +-
 drivers/acpi/acpica/evgpeutil.c                    |     2 +-
 drivers/acpi/acpica/evhandler.c                    |     2 +-
 drivers/acpi/acpica/evmisc.c                       |     2 +-
 drivers/acpi/acpica/evregion.c                     |    12 +-
 drivers/acpi/acpica/evrgnini.c                     |     2 +-
 drivers/acpi/acpica/evsci.c                        |     2 +-
 drivers/acpi/acpica/evxface.c                      |     2 +-
 drivers/acpi/acpica/evxfevnt.c                     |     2 +-
 drivers/acpi/acpica/evxfgpe.c                      |     2 +-
 drivers/acpi/acpica/evxfregn.c                     |     2 +-
 drivers/acpi/acpica/exconcat.c                     |     2 +-
 drivers/acpi/acpica/exconfig.c                     |     2 +-
 drivers/acpi/acpica/exconvrt.c                     |     2 +-
 drivers/acpi/acpica/excreate.c                     |     2 +-
 drivers/acpi/acpica/exdebug.c                      |     2 +-
 drivers/acpi/acpica/exdump.c                       |    13 +-
 drivers/acpi/acpica/exfield.c                      |     2 +-
 drivers/acpi/acpica/exfldio.c                      |     2 +-
 drivers/acpi/acpica/exmisc.c                       |     2 +-
 drivers/acpi/acpica/exmutex.c                      |     2 +-
 drivers/acpi/acpica/exnames.c                      |     2 +-
 drivers/acpi/acpica/exoparg1.c                     |     2 +-
 drivers/acpi/acpica/exoparg2.c                     |     2 +-
 drivers/acpi/acpica/exoparg3.c                     |     2 +-
 drivers/acpi/acpica/exoparg6.c                     |     2 +-
 drivers/acpi/acpica/exprep.c                       |     2 +-
 drivers/acpi/acpica/exregion.c                     |     2 +-
 drivers/acpi/acpica/exresnte.c                     |     2 +-
 drivers/acpi/acpica/exresolv.c                     |     2 +-
 drivers/acpi/acpica/exresop.c                      |     2 +-
 drivers/acpi/acpica/exstore.c                      |     2 +-
 drivers/acpi/acpica/exstoren.c                     |     2 +-
 drivers/acpi/acpica/exstorob.c                     |     2 +-
 drivers/acpi/acpica/exsystem.c                     |     2 +-
 drivers/acpi/acpica/extrace.c                      |     2 +-
 drivers/acpi/acpica/exutils.c                      |     2 +-
 drivers/acpi/acpica/hwacpi.c                       |     2 +-
 drivers/acpi/acpica/hwesleep.c                     |     2 +-
 drivers/acpi/acpica/hwgpe.c                        |     2 +-
 drivers/acpi/acpica/hwpci.c                        |     2 +-
 drivers/acpi/acpica/hwregs.c                       |     2 +-
 drivers/acpi/acpica/hwsleep.c                      |     2 +-
 drivers/acpi/acpica/hwtimer.c                      |    31 +-
 drivers/acpi/acpica/hwvalid.c                      |    16 +-
 drivers/acpi/acpica/hwxface.c                      |     2 +-
 drivers/acpi/acpica/hwxfsleep.c                    |     2 +-
 drivers/acpi/acpica/nsaccess.c                     |    15 +-
 drivers/acpi/acpica/nsalloc.c                      |     2 +-
 drivers/acpi/acpica/nsarguments.c                  |     2 +-
 drivers/acpi/acpica/nsconvert.c                    |     5 +-
 drivers/acpi/acpica/nsdump.c                       |     2 +-
 drivers/acpi/acpica/nsdumpdv.c                     |     2 +-
 drivers/acpi/acpica/nseval.c                       |     2 +-
 drivers/acpi/acpica/nsinit.c                       |     2 +-
 drivers/acpi/acpica/nsload.c                       |     2 +-
 drivers/acpi/acpica/nsnames.c                      |   151 +-
 drivers/acpi/acpica/nsobject.c                     |     2 +-
 drivers/acpi/acpica/nsparse.c                      |     2 +-
 drivers/acpi/acpica/nspredef.c                     |     2 +-
 drivers/acpi/acpica/nsprepkg.c                     |     2 +-
 drivers/acpi/acpica/nsrepair.c                     |     2 +-
 drivers/acpi/acpica/nsrepair2.c                    |     2 +-
 drivers/acpi/acpica/nssearch.c                     |     3 +-
 drivers/acpi/acpica/nsutils.c                      |     2 +-
 drivers/acpi/acpica/nswalk.c                       |     2 +-
 drivers/acpi/acpica/nsxfeval.c                     |    11 +-
 drivers/acpi/acpica/nsxfname.c                     |     2 +-
 drivers/acpi/acpica/nsxfobj.c                      |     2 +-
 drivers/acpi/acpica/psargs.c                       |     4 +-
 drivers/acpi/acpica/psloop.c                       |     2 +-
 drivers/acpi/acpica/psobject.c                     |    12 +-
 drivers/acpi/acpica/psopcode.c                     |     2 +-
 drivers/acpi/acpica/psopinfo.c                     |     2 +-
 drivers/acpi/acpica/psparse.c                      |     2 +-
 drivers/acpi/acpica/psscope.c                      |     2 +-
 drivers/acpi/acpica/pstree.c                       |     2 +-
 drivers/acpi/acpica/psutils.c                      |    16 +-
 drivers/acpi/acpica/pswalk.c                       |     2 +-
 drivers/acpi/acpica/psxface.c                      |     2 +-
 drivers/acpi/acpica/rsaddr.c                       |     2 +-
 drivers/acpi/acpica/rscalc.c                       |     2 +-
 drivers/acpi/acpica/rscreate.c                     |     2 +-
 drivers/acpi/acpica/rsdump.c                       |     2 +-
 drivers/acpi/acpica/rsdumpinfo.c                   |     2 +-
 drivers/acpi/acpica/rsinfo.c                       |     2 +-
 drivers/acpi/acpica/rsio.c                         |     2 +-
 drivers/acpi/acpica/rsirq.c                        |     2 +-
 drivers/acpi/acpica/rslist.c                       |     2 +-
 drivers/acpi/acpica/rsmemory.c                     |     2 +-
 drivers/acpi/acpica/rsmisc.c                       |     2 +-
 drivers/acpi/acpica/rsserial.c                     |     2 +-
 drivers/acpi/acpica/rsutils.c                      |     2 +-
 drivers/acpi/acpica/rsxface.c                      |     2 +-
 drivers/acpi/acpica/tbdata.c                       |     2 +-
 drivers/acpi/acpica/tbfadt.c                       |     2 +-
 drivers/acpi/acpica/tbfind.c                       |     2 +-
 drivers/acpi/acpica/tbinstal.c                     |     2 +-
 drivers/acpi/acpica/tbprint.c                      |     2 +-
 drivers/acpi/acpica/tbutils.c                      |     2 +-
 drivers/acpi/acpica/tbxface.c                      |     2 +-
 drivers/acpi/acpica/tbxfload.c                     |     2 +-
 drivers/acpi/acpica/tbxfroot.c                     |     2 +-
 drivers/acpi/acpica/utaddress.c                    |     2 +-
 drivers/acpi/acpica/utalloc.c                      |     2 +-
 drivers/acpi/acpica/utascii.c                      |     2 +-
 drivers/acpi/acpica/utbuffer.c                     |     2 +-
 drivers/acpi/acpica/utcache.c                      |     2 +-
 drivers/acpi/acpica/utcopy.c                       |     2 +-
 drivers/acpi/acpica/utdebug.c                      |    20 +-
 drivers/acpi/acpica/utdecode.c                     |    13 +-
 drivers/acpi/acpica/utdelete.c                     |     2 +-
 drivers/acpi/acpica/uterror.c                      |    75 +-
 drivers/acpi/acpica/uteval.c                       |     2 +-
 drivers/acpi/acpica/utexcep.c                      |     2 +-
 drivers/acpi/acpica/utglobal.c                     |     2 +-
 drivers/acpi/acpica/uthex.c                        |     2 +-
 drivers/acpi/acpica/utids.c                        |     2 +-
 drivers/acpi/acpica/utinit.c                       |     3 +-
 drivers/acpi/acpica/utlock.c                       |     2 +-
 drivers/acpi/acpica/utmath.c                       |     6 +-
 drivers/acpi/acpica/utmisc.c                       |     2 +-
 drivers/acpi/acpica/utmutex.c                      |    11 +-
 drivers/acpi/acpica/utnonansi.c                    |    13 +-
 drivers/acpi/acpica/utobject.c                     |     2 +-
 drivers/acpi/acpica/utosi.c                        |     4 +-
 drivers/acpi/acpica/utownerid.c                    |     2 +-
 drivers/acpi/acpica/utpredef.c                     |     2 +-
 drivers/acpi/acpica/utprint.c                      |     2 +-
 drivers/acpi/acpica/utresdecode.c                  |     2 +-
 drivers/acpi/acpica/utresrc.c                      |     2 +-
 drivers/acpi/acpica/utstate.c                      |     2 +-
 drivers/acpi/acpica/utstring.c                     |     2 +-
 drivers/acpi/acpica/utstrsuppt.c                   |    44 +-
 drivers/acpi/acpica/utstrtoul64.c                  |     2 +-
 drivers/acpi/acpica/uttrack.c                      |     8 +-
 drivers/acpi/acpica/utuuid.c                       |     2 +-
 drivers/acpi/acpica/utxface.c                      |     2 +-
 drivers/acpi/acpica/utxferror.c                    |    10 +-
 drivers/acpi/acpica/utxfinit.c                     |     2 +-
 drivers/acpi/acpica/utxfmutex.c                    |     2 +-
 drivers/acpi/apei/ghes.c                           |    81 +-
 drivers/acpi/battery.c                             |     4 +-
 drivers/acpi/bus.c                                 |   120 +-
 drivers/acpi/button.c                              |    22 +
 drivers/acpi/cppc_acpi.c                           |     2 +-
 drivers/acpi/device_pm.c                           |    27 +-
 drivers/acpi/device_sysfs.c                        |     6 +-
 drivers/acpi/ec.c                                  |     8 +-
 drivers/acpi/ec_sys.c                              |     2 +-
 drivers/acpi/evged.c                               |    47 +-
 drivers/acpi/internal.h                            |     2 +-
 drivers/acpi/nfit/core.c                           |    30 +
 drivers/acpi/nfit/nfit.h                           |     1 +
 drivers/acpi/numa.c                                |    13 +-
 drivers/acpi/pci_link.c                            |     2 +-
 drivers/acpi/pmic/intel_pmic_bxtwc.c               |     9 +-
 drivers/acpi/pmic/intel_pmic_chtdc_ti.c            |     5 +-
 drivers/acpi/pmic/intel_pmic_chtwc.c               |     9 +-
 drivers/acpi/pmic/intel_pmic_crc.c                 |     7 +-
 drivers/acpi/pmic/intel_pmic_xpower.c              |     7 +-
 drivers/acpi/processor_idle.c                      |     4 +
 drivers/acpi/processor_perflib.c                   |     2 +-
 drivers/acpi/property.c                            |     8 +
 drivers/acpi/sbshc.c                               |     4 +-
 drivers/acpi/scan.c                                |    20 +-
 drivers/acpi/sleep.c                               |    16 +-
 drivers/acpi/spcr.c                                |    30 +-
 drivers/acpi/sysfs.c                               |    26 +-
 drivers/acpi/tables.c                              |     3 +-
 drivers/acpi/utils.c                               |    41 +-
 drivers/android/binder.c                           |   229 +-
 drivers/android/binder_alloc.c                     |    29 +-
 drivers/android/binder_alloc.h                     |     4 +-
 drivers/ata/Kconfig                                |    28 +-
 drivers/ata/Makefile                               |     1 -
 drivers/ata/ahci.c                                 |    98 +-
 drivers/ata/ahci.h                                 |     3 +
 drivers/ata/ahci_brcm.c                            |   120 +-
 drivers/ata/ata_piix.c                             |     2 +-
 drivers/ata/libahci.c                              |    10 +
 drivers/ata/libahci_platform.c                     |     2 +-
 drivers/ata/libata-core.c                          |    26 +-
 drivers/ata/libata-eh.c                            |     3 +-
 drivers/ata/libata-scsi.c                          |    12 +-
 drivers/ata/pata_ali.c                             |     3 +-
 drivers/ata/pata_at32.c                            |   400 -
 drivers/ata/pata_atiixp.c                          |     4 +
 drivers/ata/pata_it821x.c                          |     2 +-
 drivers/ata/pata_pdc2027x.c                        |     4 +-
 drivers/ata/sata_mv.c                              |     2 +-
 drivers/ata/sata_rcar.c                            |    62 +-
 drivers/atm/eni.c                                  |    18 +-
 drivers/atm/he.c                                   |     8 +-
 drivers/atm/iphase.c                               |     2 +-
 drivers/auxdisplay/img-ascii-lcd.c                 |    10 +-
 drivers/auxdisplay/panel.c                         |     6 +-
 drivers/base/Kconfig                               |    29 +-
 drivers/base/arch_topology.c                       |     8 +-
 drivers/base/attribute_container.c                 |     3 +-
 drivers/base/bus.c                                 |     6 +-
 drivers/base/cacheinfo.c                           |    13 +-
 drivers/base/class.c                               |     4 +-
 drivers/base/component.c                           |    83 +-
 drivers/base/container.c                           |     5 +-
 drivers/base/core.c                                |    14 +-
 drivers/base/cpu.c                                 |     1 +
 drivers/base/dd.c                                  |    43 +-
 drivers/base/devcoredump.c                         |    17 +-
 drivers/base/devres.c                              |     3 +-
 drivers/base/dma-contiguous.c                      |     6 +-
 drivers/base/dma-mapping.c                         |     3 +-
 drivers/base/driver.c                              |     6 +-
 drivers/base/firmware.c                            |     3 +-
 drivers/base/firmware_class.c                      |   859 +-
 drivers/base/hypervisor.c                          |     3 +-
 drivers/base/init.c                                |     3 +-
 drivers/base/isa.c                                 |     1 +
 drivers/base/map.c                                 |     2 +-
 drivers/base/memory.c                              |     2 +-
 drivers/base/module.c                              |     4 +-
 drivers/base/pinctrl.c                             |     3 +-
 drivers/base/platform-msi.c                        |    13 +-
 drivers/base/platform.c                            |     3 +-
 drivers/base/power/domain.c                        |   145 +-
 drivers/base/power/main.c                          |   415 +-
 drivers/base/power/power.h                         |    11 +-
 drivers/base/power/runtime.c                       |    84 +-
 drivers/base/power/sysfs.c                         |   182 +-
 drivers/base/power/wakeirq.c                       |    14 +-
 drivers/base/power/wakeup.c                        |    26 +-
 drivers/base/property.c                            |   183 +-
 drivers/base/regmap/Kconfig                        |    10 +-
 drivers/base/regmap/Makefile                       |     2 +
 drivers/base/regmap/internal.h                     |     8 +
 drivers/base/regmap/regcache-flat.c                |    15 +-
 drivers/base/regmap/regmap-debugfs.c               |    12 +
 drivers/base/regmap/regmap-sdw.c                   |    88 +
 drivers/base/regmap/regmap-slimbus.c               |    80 +
 drivers/base/regmap/regmap.c                       |    47 +-
 drivers/base/soc.c                                 |     2 +-
 drivers/base/syscore.c                             |     3 +-
 drivers/base/test/test_async_driver_probe.c        |    10 +-
 drivers/base/topology.c                            |    17 +-
 drivers/base/transport_class.c                     |     3 +-
 drivers/bcma/driver_pcie2.c                        |     3 +-
 drivers/block/DAC960.c                             |   198 +-
 drivers/block/DAC960.h                             |     4 +-
 drivers/block/Kconfig                              |     4 +
 drivers/block/amiflop.c                            |     2 +-
 drivers/block/aoe/aoe.h                            |     3 +-
 drivers/block/aoe/aoecmd.c                         |    48 +-
 drivers/block/ataflop.c                            |     2 +-
 drivers/block/brd.c                                |     2 +-
 drivers/block/drbd/drbd_bitmap.c                   |     2 +-
 drivers/block/drbd/drbd_main.c                     |     8 +-
 drivers/block/drbd/drbd_receiver.c                 |     3 +-
 drivers/block/floppy.c                             |     2 +-
 drivers/block/loop.c                               |     4 +-
 drivers/block/nbd.c                                |     2 +-
 drivers/block/null_blk.c                           |   292 +-
 drivers/block/pktcdvd.c                            |    14 +-
 drivers/block/rbd.c                                |    22 +-
 drivers/block/skd_main.c                           |     7 +-
 drivers/block/smart1,2.h                           |   278 -
 drivers/block/swim.c                               |     2 +-
 drivers/block/virtio_blk.c                         |    34 +-
 drivers/block/xen-blkfront.c                       |    19 +-
 drivers/block/z2ram.c                              |     2 +-
 drivers/block/zram/zram_drv.c                      |     2 +-
 drivers/bluetooth/Kconfig                          |    14 +
 drivers/bluetooth/bluecard_cs.c                    |     8 +-
 drivers/bluetooth/bpa10x.c                         |     2 +-
 drivers/bluetooth/btbcm.c                          |     1 +
 drivers/bluetooth/btbcm.h                          |     2 +-
 drivers/bluetooth/btintel.c                        |   157 +-
 drivers/bluetooth/btintel.h                        |    33 +-
 drivers/bluetooth/btqcomsmd.c                      |     3 +-
 drivers/bluetooth/btsdio.c                         |     9 +
 drivers/bluetooth/btusb.c                          |   185 +-
 drivers/bluetooth/hci_bcm.c                        |   229 +-
 drivers/bluetooth/hci_intel.c                      |   186 +-
 drivers/bluetooth/hci_ldisc.c                      |     2 +-
 drivers/bluetooth/hci_ll.c                         |   107 +-
 drivers/bluetooth/hci_qca.c                        |     3 +
 drivers/bluetooth/hci_serdev.c                     |     1 +
 drivers/bluetooth/hci_vhci.c                       |     6 +-
 drivers/bus/Kconfig                                |     2 +-
 drivers/bus/omap_l3_noc.c                          |     5 +
 drivers/bus/omap_l3_smx.c                          |     6 +
 drivers/bus/ti-sysc.c                              |   485 +-
 drivers/char/agp/intel-gtt.c                       |    16 +-
 drivers/char/agp/nvidia-agp.c                      |    12 +-
 drivers/char/agp/sworks-agp.c                      |     3 +-
 drivers/char/apm-emulation.c                       |     4 +-
 drivers/char/dsp56k.c                              |     4 +-
 drivers/char/dtlk.c                                |    12 +-
 drivers/char/hpet.c                                |     4 +-
 drivers/char/hw_random/Kconfig                     |    45 +-
 drivers/char/hw_random/Makefile                    |     3 +-
 drivers/char/hw_random/bcm2835-rng.c               |   169 +-
 drivers/char/hw_random/bcm63xx-rng.c               |   154 -
 drivers/char/hw_random/core.c                      |     4 +
 drivers/char/hw_random/exynos-trng.c               |   235 +
 drivers/char/hw_random/imx-rngc.c                  |    13 +-
 drivers/char/hw_random/mtk-rng.c                   |     1 +
 drivers/char/hw_random/tpm-rng.c                   |    50 -
 drivers/char/hw_random/via-rng.c                   |     2 +-
 drivers/char/ipmi/bt-bmc.c                         |     8 +-
 drivers/char/ipmi/ipmi_devintf.c                   |     6 +-
 drivers/char/ipmi/ipmi_dmi.c                       |     5 +-
 drivers/char/ipmi/ipmi_msghandler.c                |    19 +-
 drivers/char/ipmi/ipmi_powernv.c                   |     6 +-
 drivers/char/ipmi/ipmi_si_intf.c                   |    13 +-
 drivers/char/ipmi/ipmi_si_platform.c               |     2 +-
 drivers/char/ipmi/ipmi_ssif.c                      |     3 +-
 drivers/char/ipmi/ipmi_watchdog.c                  |    10 +-
 drivers/char/lp.c                                  |    69 +-
 drivers/char/mem.c                                 |    27 +-
 drivers/char/pcmcia/cm4040_cs.c                    |     8 +-
 drivers/char/ppdev.c                               |     6 +-
 drivers/char/random.c                              |    32 +-
 drivers/char/rtc.c                                 |     6 +-
 drivers/char/snsc.c                                |     8 +-
 drivers/char/sonypi.c                              |     4 +-
 drivers/char/tpm/Kconfig                           |    11 +
 drivers/char/tpm/Makefile                          |     5 +-
 drivers/char/tpm/st33zp24/st33zp24.c               |     4 +-
 drivers/char/tpm/tpm-chip.c                        |    67 +-
 drivers/char/tpm/tpm-interface.c                   |   235 +-
 drivers/char/tpm/tpm.h                             |    52 +-
 drivers/char/tpm/tpm1_eventlog.c                   |    13 +-
 drivers/char/tpm/tpm2-cmd.c                        |    16 +-
 drivers/char/tpm/tpm2_eventlog.c                   |     2 +-
 .../char/tpm/{tpm_acpi.c => tpm_eventlog_acpi.c}   |     4 +-
 drivers/char/tpm/tpm_eventlog_efi.c                |    66 +
 drivers/char/tpm/{tpm_of.c => tpm_eventlog_of.c}   |     6 +-
 drivers/char/tpm/tpm_i2c_infineon.c                |    32 +-
 drivers/char/tpm/tpm_i2c_nuvoton.c                 |     8 +-
 drivers/char/tpm/tpm_tis.c                         |   108 -
 drivers/char/tpm/tpm_tis_core.c                    |   198 +-
 drivers/char/tpm/tpm_tis_core.h                    |    16 +
 drivers/char/tpm/tpm_vtpm_proxy.c                  |    10 +-
 drivers/char/tpm/xen-tpmfront.c                    |    61 +
 drivers/char/virtio_console.c                      |    12 +-
 drivers/char/xillybus/Kconfig                      |     4 +-
 drivers/char/xillybus/xillybus_core.c              |    16 +-
 drivers/char/xillybus/xillybus_of.c                |    12 +-
 drivers/clk/Kconfig                                |    13 +
 drivers/clk/Makefile                               |     4 +-
 drivers/clk/at91/clk-programmable.c                |     2 +
 drivers/clk/at91/pmc.c                             |    63 +-
 drivers/clk/at91/pmc.h                             |     2 +
 drivers/clk/bcm/clk-bcm2835.c                      |    12 +-
 drivers/clk/bcm/clk-cygnus.c                       |    25 +-
 drivers/clk/bcm/clk-iproc-pll.c                    |   260 +-
 drivers/clk/bcm/clk-iproc.h                        |     5 +
 drivers/clk/clk-aspeed.c                           |   673 +
 drivers/clk/clk-axi-clkgen.c                       |    39 +-
 drivers/clk/clk-divider.c                          |     7 +-
 drivers/clk/clk-qoriq.c                            |     9 +-
 drivers/clk/clk-si5351.c                           |    68 +-
 drivers/clk/clk-stm32f4.c                          |     2 +-
 drivers/clk/clk-stm32h7.c                          |    19 +-
 drivers/clk/clk.c                                  |   648 +-
 drivers/clk/clk.h                                  |     4 +
 drivers/clk/clkdev.c                               |     2 +-
 drivers/clk/h8300/clk-div.c                        |     4 +-
 drivers/clk/h8300/clk-h8s2678.c                    |     6 +-
 drivers/clk/hisilicon/Kconfig                      |     6 +
 drivers/clk/hisilicon/Makefile                     |     1 +
 drivers/clk/hisilicon/clk-hi3660-stub.c            |   187 +
 drivers/clk/hisilicon/clkdivider-hi6220.c          |     2 +-
 drivers/clk/imx/clk-imx51-imx53.c                  |    32 +-
 drivers/clk/ingenic/Makefile                       |     1 +
 drivers/clk/ingenic/cgu.c                          |    94 +-
 drivers/clk/ingenic/cgu.h                          |     4 +-
 drivers/clk/ingenic/jz4770-cgu.c                   |   483 +
 drivers/clk/ingenic/jz4780-cgu.c                   |     2 +-
 drivers/clk/mediatek/Kconfig                       |    97 +-
 drivers/clk/mediatek/Makefile                      |     3 +-
 drivers/clk/mediatek/clk-mtk.h                     |     8 +-
 drivers/clk/meson/Kconfig                          |     8 +
 drivers/clk/meson/Makefile                         |     1 +
 drivers/clk/meson/axg.c                            |   938 +
 drivers/clk/meson/axg.h                            |   126 +
 drivers/clk/meson/clk-mpll.c                       |     2 +-
 drivers/clk/meson/clkc.h                           |     2 +-
 drivers/clk/meson/gxbb.c                           |   132 +-
 drivers/clk/meson/meson8b.c                        |    24 +-
 drivers/clk/mvebu/armada-37xx-periph.c             |   317 +-
 drivers/clk/nxp/clk-lpc32xx.c                      |     6 +-
 drivers/clk/pxa/clk-pxa3xx.c                       |     6 +-
 drivers/clk/qcom/Kconfig                           |    30 +
 drivers/clk/qcom/Makefile                          |     4 +
 drivers/clk/qcom/a53-pll.c                         |   107 +
 drivers/clk/qcom/apcs-msm8916.c                    |   137 +
 drivers/clk/qcom/clk-alpha-pll.c                   |   516 +-
 drivers/clk/qcom/clk-alpha-pll.h                   |    35 +-
 drivers/clk/qcom/clk-rcg.h                         |    10 -
 drivers/clk/qcom/clk-regmap-divider.c              |    31 +-
 drivers/clk/qcom/clk-regmap-divider.h              |     1 +
 drivers/clk/qcom/clk-regmap-mux-div.c              |   231 +
 drivers/clk/qcom/clk-regmap-mux-div.h              |    44 +
 drivers/clk/qcom/clk-regmap-mux.c                  |     6 +
 drivers/clk/qcom/clk-regmap-mux.h                  |     2 +
 drivers/clk/qcom/clk-spmi-pmic-div.c               |   302 +
 drivers/clk/qcom/common.h                          |    11 +-
 drivers/clk/qcom/gcc-ipq8074.c                     |  4182 ++-
 drivers/clk/qcom/gcc-msm8916.c                     |    13 +-
 drivers/clk/qcom/gcc-msm8994.c                     |     4 +
 drivers/clk/qcom/gcc-msm8996.c                     |     4 +
 drivers/clk/qcom/mmcc-msm8996.c                    |    16 +
 drivers/clk/renesas/clk-mstp.c                     |     2 +-
 drivers/clk/renesas/r8a7796-cpg-mssr.c             |     1 +
 drivers/clk/renesas/r8a77970-cpg-mssr.c            |     1 +
 drivers/clk/renesas/renesas-cpg-mssr.c             |     2 +-
 drivers/clk/spear/clk-frac-synth.c                 |     2 +-
 drivers/clk/spear/clk-gpt-synth.c                  |     2 +-
 drivers/clk/sprd/Kconfig                           |    14 +
 drivers/clk/sprd/Makefile                          |    11 +
 drivers/clk/sprd/common.c                          |    96 +
 drivers/clk/sprd/common.h                          |    38 +
 drivers/clk/sprd/composite.c                       |    60 +
 drivers/clk/sprd/composite.h                       |    51 +
 drivers/clk/sprd/div.c                             |    91 +
 drivers/clk/sprd/div.h                             |    75 +
 drivers/clk/sprd/gate.c                            |   111 +
 drivers/clk/sprd/gate.h                            |    59 +
 drivers/clk/sprd/mux.c                             |    76 +
 drivers/clk/sprd/mux.h                             |    74 +
 drivers/clk/sprd/pll.c                             |   266 +
 drivers/clk/sprd/pll.h                             |   108 +
 drivers/clk/sprd/sc9860-clk.c                      |  1974 ++
 drivers/clk/sunxi-ng/ccu-sun50i-a64.c              |    57 +-
 drivers/clk/sunxi-ng/ccu-sun6i-a31.c               |     6 +-
 drivers/clk/sunxi-ng/ccu-sun8i-a83t.c              |    22 +-
 drivers/clk/sunxi-ng/ccu-sun8i-de2.c               |    53 +-
 drivers/clk/sunxi-ng/ccu_common.h                  |    29 -
 drivers/clk/sunxi-ng/ccu_div.c                     |     2 +-
 drivers/clk/sunxi-ng/ccu_mp.c                      |    20 +-
 drivers/clk/sunxi-ng/ccu_mp.h                      |    24 +
 drivers/clk/sunxi-ng/ccu_nm.c                      |    50 +-
 drivers/clk/sunxi-ng/ccu_nm.h                      |     2 +
 drivers/clk/sunxi/clk-factors.c                    |    26 +-
 drivers/clk/sunxi/clk-factors.h                    |     4 +
 drivers/clk/sunxi/clk-mod0.c                       |     9 +-
 drivers/clk/sunxi/clk-sun8i-apb0.c                 |     5 +-
 drivers/clk/sunxi/clk-sun8i-mbus.c                 |     7 +-
 drivers/clk/sunxi/clk-sun9i-core.c                 |     9 +-
 drivers/clk/sunxi/clk-sunxi.c                      |    36 +-
 drivers/clk/ti/Makefile                            |     4 -
 drivers/clk/ti/apll.c                              |     3 +-
 drivers/clk/ti/clk-33xx.c                          |   279 +-
 drivers/clk/ti/clk-3xxx-legacy.c                   |  4656 ---
 drivers/clk/ti/clk-3xxx.c                          |   263 +-
 drivers/clk/ti/clk-43xx.c                          |   295 +-
 drivers/clk/ti/clk-44xx.c                          |   200 +-
 drivers/clk/ti/clk-54xx.c                          |   697 +-
 drivers/clk/ti/clk-7xx.c                           |  1076 +-
 drivers/clk/ti/clk-814x.c                          |    50 +-
 drivers/clk/ti/clk-816x.c                          |    62 +-
 drivers/clk/ti/clk.c                               |   205 +-
 drivers/clk/ti/clkctrl.c                           |    93 +-
 drivers/clk/ti/clock.h                             |    81 +-
 drivers/clk/ti/composite.c                         |    48 +-
 drivers/clk/ti/dpll.c                              |    93 +-
 drivers/clk/ti/gate.c                              |    48 -
 drivers/clk/ti/interface.c                         |    32 -
 drivers/clk/zte/clk.h                              |    18 -
 drivers/clocksource/Kconfig                        |     9 +
 drivers/clocksource/Makefile                       |     1 +
 drivers/clocksource/arc_timer.c                    |    11 +-
 drivers/clocksource/fsl_ftm_timer.c                |     2 +-
 drivers/clocksource/mips-gic-timer.c               |     6 +-
 drivers/clocksource/owl-timer.c                    |     5 +-
 drivers/clocksource/tcb_clksrc.c                   |     2 +-
 drivers/clocksource/timer-of.c                     |    84 +-
 drivers/clocksource/timer-of.h                     |     1 +
 drivers/clocksource/timer-sprd.c                   |   159 +
 drivers/clocksource/timer-stm32.c                  |   358 +-
 drivers/clocksource/timer-sun5i.c                  |     2 +-
 drivers/cpufreq/Kconfig                            |    10 -
 drivers/cpufreq/Kconfig.arm                        |    88 +-
 drivers/cpufreq/Makefile                           |    10 +-
 drivers/cpufreq/acpi-cpufreq.c                     |     2 +-
 drivers/cpufreq/amd_freq_sensitivity.c             |    11 +-
 drivers/cpufreq/arm_big_little.c                   |    23 +-
 drivers/cpufreq/armada-37xx-cpufreq.c              |   241 +
 drivers/cpufreq/at32ap-cpufreq.c                   |   127 -
 drivers/cpufreq/cpufreq-dt-platdev.c               |     8 +
 drivers/cpufreq/cpufreq-dt.c                       |    27 +-
 drivers/cpufreq/cpufreq.c                          |    58 +-
 drivers/cpufreq/cpufreq_stats.c                    |     3 +-
 drivers/cpufreq/exynos5440-cpufreq.c               |     7 +-
 drivers/cpufreq/freq_table.c                       |     8 +-
 drivers/cpufreq/imx6q-cpufreq.c                    |   172 +-
 drivers/cpufreq/intel_pstate.c                     |    19 +-
 drivers/cpufreq/longhaul.c                         |    12 +-
 drivers/cpufreq/mediatek-cpufreq.c                 |    23 +-
 drivers/cpufreq/mvebu-cpufreq.c                    |    16 +-
 drivers/cpufreq/p4-clockmod.c                      |     2 +-
 drivers/cpufreq/pasemi-cpufreq.c                   |     6 +-
 drivers/cpufreq/powernow-k7.c                      |     2 +-
 drivers/cpufreq/powernv-cpufreq.c                  |   143 +-
 drivers/cpufreq/qoriq-cpufreq.c                    |    14 +-
 drivers/cpufreq/s3c24xx-cpufreq.c                  |     8 +-
 drivers/cpufreq/scpi-cpufreq.c                     |   199 +-
 drivers/cpufreq/speedstep-centrino.c               |     4 +-
 drivers/cpufreq/speedstep-lib.c                    |     6 +-
 drivers/cpufreq/ti-cpufreq.c                       |    51 +-
 drivers/cpuidle/cpuidle-powernv.c                  |     2 +
 drivers/cpuidle/cpuidle-pseries.c                  |    16 +-
 drivers/cpuidle/governor.c                         |     5 +-
 drivers/crypto/Kconfig                             |     1 -
 drivers/crypto/amcc/crypto4xx_alg.c                |     6 +-
 drivers/crypto/amcc/crypto4xx_core.c               |   131 +-
 drivers/crypto/amcc/crypto4xx_core.h               |     4 +-
 drivers/crypto/amcc/crypto4xx_reg_def.h            |     4 +-
 drivers/crypto/amcc/crypto4xx_trng.c               |     2 +-
 drivers/crypto/axis/artpec6_crypto.c               |     8 +-
 drivers/crypto/bcm/cipher.c                        |     1 -
 drivers/crypto/bfin_crc.c                          |     3 +-
 drivers/crypto/caam/caamalg.c                      |   120 +-
 drivers/crypto/caam/caamalg_desc.c                 |   182 +-
 drivers/crypto/caam/caamalg_desc.h                 |    10 +-
 drivers/crypto/caam/caamalg_qi.c                   |    68 +-
 drivers/crypto/caam/caamhash.c                     |    73 +-
 drivers/crypto/caam/ctrl.c                         |    12 +-
 drivers/crypto/caam/desc.h                         |    29 +
 drivers/crypto/caam/desc_constr.h                  |    51 +-
 drivers/crypto/caam/intern.h                       |     1 +
 drivers/crypto/caam/key_gen.c                      |    30 -
 drivers/crypto/caam/key_gen.h                      |    30 +
 drivers/crypto/cavium/cpt/cptvf_reqmanager.c       |     3 +-
 drivers/crypto/cavium/nitrox/nitrox_reqmgr.c       |     1 -
 drivers/crypto/ccp/Kconfig                         |    12 +
 drivers/crypto/ccp/Makefile                        |     1 +
 drivers/crypto/ccp/ccp-crypto-aes-galois.c         |     1 -
 drivers/crypto/ccp/psp-dev.c                       |   805 +
 drivers/crypto/ccp/psp-dev.h                       |    83 +
 drivers/crypto/ccp/sp-dev.c                        |    35 +
 drivers/crypto/ccp/sp-dev.h                        |    28 +-
 drivers/crypto/ccp/sp-pci.c                        |    52 +
 drivers/crypto/chelsio/Kconfig                     |    10 +
 drivers/crypto/chelsio/Makefile                    |     1 +
 drivers/crypto/chelsio/chcr_algo.c                 |   540 +-
 drivers/crypto/chelsio/chcr_algo.h                 |    15 -
 drivers/crypto/chelsio/chcr_core.c                 |    14 +
 drivers/crypto/chelsio/chcr_core.h                 |    38 +
 drivers/crypto/chelsio/chcr_crypto.h               |    76 +-
 drivers/crypto/chelsio/chcr_ipsec.c                |   654 +
 drivers/crypto/exynos-rng.c                        |   108 +-
 drivers/crypto/hifn_795x.c                         |     1 +
 drivers/crypto/inside-secure/safexcel.c            |   370 +-
 drivers/crypto/inside-secure/safexcel.h            |   173 +-
 drivers/crypto/inside-secure/safexcel_cipher.c     |    53 +-
 drivers/crypto/inside-secure/safexcel_hash.c       |   125 +-
 drivers/crypto/ixp4xx_crypto.c                     |     7 +-
 drivers/crypto/marvell/cesa.c                      |    20 +-
 drivers/crypto/nx/nx-842-powernv.c                 |     4 +-
 drivers/crypto/padlock-aes.c                       |     2 +-
 drivers/crypto/picoxcell_crypto.c                  |    27 +-
 drivers/crypto/qat/qat_common/qat_hal.c            |   133 +-
 drivers/crypto/s5p-sss.c                           |    38 +-
 drivers/crypto/stm32/Kconfig                       |    13 +-
 drivers/crypto/stm32/Makefile                      |     5 +-
 drivers/crypto/stm32/stm32-cryp.c                  |  1170 +
 drivers/crypto/stm32/stm32_crc32.c                 |     2 +
 drivers/crypto/sunxi-ss/sun4i-ss-prng.c            |     6 +-
 drivers/crypto/talitos.c                           |     4 +
 drivers/dax/device.c                               |     2 +-
 drivers/dax/pmem.c                                 |    20 +-
 drivers/dax/super.c                                |    16 +-
 drivers/devfreq/devfreq.c                          |     5 +-
 drivers/dma-buf/dma-buf.c                          |    39 +-
 drivers/dma-buf/dma-fence-array.c                  |    14 +-
 drivers/dma-buf/reservation.c                      |    66 +-
 drivers/dma-buf/sync_file.c                        |     4 +-
 drivers/dma/amba-pl08x.c                           |    11 +-
 drivers/dma/bcm2835-dma.c                          |    10 +-
 drivers/dma/cppi41.c                               |     2 +-
 drivers/dma/dma-jz4780.c                           |    10 +-
 drivers/dma/dmatest.c                              |     2 +-
 drivers/dma/edma.c                                 |     7 +-
 drivers/dma/img-mdc-dma.c                          |    17 +-
 drivers/dma/imx-dma.c                              |     2 +-
 drivers/dma/imx-sdma.c                             |     6 +
 drivers/dma/ioat/dma.c                             |     2 -
 drivers/dma/k3dma.c                                |    10 +-
 drivers/dma/mic_x100_dma.c                         |     4 +-
 drivers/dma/mv_xor_v2.c                            |    25 +-
 drivers/dma/omap-dma.c                             |     2 +-
 drivers/dma/qcom/hidma.c                           |    41 +-
 drivers/dma/qcom/hidma_ll.c                        |     9 +-
 drivers/dma/qcom/hidma_mgmt.c                      |    61 +-
 drivers/dma/s3c24xx-dma.c                          |    11 +-
 drivers/dma/sh/rcar-dmac.c                         |    70 +-
 drivers/dma/sprd-dma.c                             |     2 +-
 drivers/dma/stm32-dmamux.c                         |    12 +-
 drivers/dma/tegra20-apb-dma.c                      |    19 +-
 drivers/dma/ti-dma-crossbar.c                      |    10 +-
 drivers/dma/timb_dma.c                             |     2 +-
 drivers/dma/virt-dma.c                             |     5 +-
 drivers/dma/virt-dma.h                             |    44 +
 drivers/dma/xilinx/xilinx_dma.c                    |   302 +-
 drivers/dma/xilinx/zynqmp_dma.c                    |   179 +-
 drivers/edac/Kconfig                               |     7 +
 drivers/edac/Makefile                              |     1 +
 drivers/edac/amd64_edac.c                          |     2 +-
 drivers/edac/mv64x60_edac.c                        |     2 +-
 drivers/edac/octeon_edac-lmc.c                     |     1 +
 drivers/edac/sb_edac.c                             |     2 +-
 drivers/edac/ti_edac.c                             |   341 +
 drivers/eisa/eisa-bus.c                            |    62 +-
 drivers/eisa/pci_eisa.c                            |    10 +-
 drivers/eisa/virtual_root.c                        |    19 +-
 drivers/extcon/extcon-adc-jack.c                   |     2 +-
 drivers/extcon/extcon-axp288.c                     |    47 +-
 drivers/extcon/extcon-intel-int3496.c              |     3 +-
 drivers/extcon/extcon-max77693.c                   |     2 +-
 drivers/extcon/extcon-max8997.c                    |     2 +-
 drivers/extcon/extcon-usbc-cros-ec.c               |   142 +-
 drivers/firewire/core-cdev.c                       |     8 +-
 drivers/firewire/net.c                             |     7 +-
 drivers/firewire/nosy.c                            |     8 +-
 drivers/firewire/ohci.c                            |     8 +-
 drivers/firmware/Kconfig                           |    10 +-
 drivers/firmware/Makefile                          |     1 +
 drivers/firmware/arm_sdei.c                        |  1092 +
 drivers/firmware/dcdbas.c                          |     2 +-
 drivers/firmware/dmi-sysfs.c                       |     2 +-
 drivers/firmware/dmi_scan.c                        |    64 +-
 drivers/firmware/edd.c                             |     8 +-
 drivers/firmware/efi/Kconfig                       |    12 +-
 drivers/firmware/efi/Makefile                      |     3 +-
 drivers/firmware/efi/capsule-loader.c              |     2 +-
 drivers/firmware/efi/cper-arm.c                    |   356 +
 drivers/firmware/efi/cper.c                        |   122 +-
 drivers/firmware/efi/efi.c                         |     6 +-
 drivers/firmware/efi/libstub/Makefile              |     3 +-
 drivers/firmware/efi/libstub/tpm.c                 |    81 +
 drivers/firmware/efi/tpm.c                         |    40 +
 drivers/firmware/iscsi_ibft.c                      |     5 +-
 drivers/firmware/psci.c                            |    57 +-
 drivers/firmware/psci_checker.c                    |    46 +-
 drivers/firmware/qcom_scm.c                        |    24 -
 drivers/firmware/qemu_fw_cfg.c                     |     4 +-
 drivers/firmware/raspberrypi.c                     |     2 +-
 drivers/firmware/ti_sci.c                          |     4 +-
 drivers/fpga/Kconfig                               |   103 +-
 drivers/fpga/Makefile                              |     1 +
 drivers/fpga/fpga-bridge.c                         |   113 +-
 drivers/fpga/fpga-mgr.c                            |   123 +-
 drivers/fpga/fpga-region.c                         |   464 +-
 drivers/fpga/of-fpga-region.c                      |   504 +
 drivers/fpga/socfpga-a10.c                         |     8 +-
 drivers/fsi/Kconfig                                |     6 +-
 drivers/gpio/Kconfig                               |    32 +-
 drivers/gpio/Makefile                              |     3 +-
 drivers/gpio/devres.c                              |    42 +
 drivers/gpio/gpio-74x164.c                         |     5 +-
 drivers/gpio/gpio-adp5520.c                        |     3 +-
 drivers/gpio/gpio-adp5588.c                        |     2 +-
 drivers/gpio/gpio-altera.c                         |     3 +-
 drivers/gpio/gpio-amd8111.c                        |     2 +-
 drivers/gpio/gpio-arizona.c                        |     2 +-
 drivers/gpio/gpio-aspeed.c                         |    41 +-
 drivers/gpio/gpio-ath79.c                          |     3 +
 drivers/gpio/gpio-axp209.c                         |   188 -
 drivers/gpio/gpio-bcm-kona.c                       |     8 +-
 drivers/gpio/gpio-brcmstb.c                        |     1 -
 drivers/gpio/gpio-bt8xx.c                          |     2 +-
 drivers/gpio/gpio-crystalcove.c                    |     2 +-
 drivers/gpio/gpio-cs5535.c                         |     2 +-
 drivers/gpio/gpio-da9052.c                         |     2 +-
 drivers/gpio/gpio-da9055.c                         |     2 +-
 drivers/gpio/gpio-davinci.c                        |     2 +-
 drivers/gpio/gpio-ftgpio010.c                      |     4 +-
 drivers/gpio/gpio-iop.c                            |     4 +
 drivers/gpio/gpio-it87.c                           |     2 +-
 drivers/gpio/gpio-max732x.c                        |     6 +-
 drivers/gpio/gpio-merrifield.c                     |    11 +-
 drivers/gpio/gpio-mockup.c                         |   288 +-
 drivers/gpio/gpio-omap.c                           |    39 +-
 drivers/gpio/gpio-pcie-idio-24.c                   |   447 +
 drivers/gpio/gpio-rcar.c                           |    38 +-
 drivers/gpio/gpio-stmpe.c                          |    54 +-
 drivers/gpio/gpio-thunderx.c                       |     4 +-
 drivers/gpio/gpio-uniphier.c                       |     2 +-
 drivers/gpio/gpio-winbond.c                        |   732 +
 drivers/gpio/gpiolib-acpi.c                        |   132 +-
 drivers/gpio/gpiolib-of.c                          |   132 +-
 drivers/gpio/gpiolib-sysfs.c                       |    33 +-
 drivers/gpio/gpiolib.c                             |   338 +-
 drivers/gpio/gpiolib.h                             |    18 +-
 drivers/gpu/drm/Kconfig                            |    10 +
 drivers/gpu/drm/Makefile                           |     2 +
 drivers/gpu/drm/amd/amdgpu/Makefile                |    10 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu.h                |   229 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c            |     2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c           |     3 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c         |   102 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |     5 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c  |   115 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c  |   188 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c       |   266 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h       |     6 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c   |    58 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c           |     2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c            |     7 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c     |    90 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |    79 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c            |    82 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c        |   792 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h        |    42 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |  1929 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_display.c        |     9 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_display.h        |     6 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h            |     6 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |    35 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c             |    27 -
 drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c          |    67 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c           |    75 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h           |     4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |    34 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c            |    12 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c        |    61 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c             |    11 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c            |   459 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h            |    91 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c             |    10 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h             |     4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c            |    12 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c            |    72 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c            |    24 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h           |     8 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |   109 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.h         |     2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c             |    19 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c            |    22 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c      |    10 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c           |    32 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |    20 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c          |    20 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h          |     2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c           |    37 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h           |     4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h          |    28 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |   287 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h            |    12 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c          |    45 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c            |    53 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h            |     8 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c            |   134 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h            |     4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c            |    32 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h            |     4 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c           |    35 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h           |     9 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |  1050 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |    96 +-
 drivers/gpu/drm/amd/amdgpu/atombios_encoders.c     |     4 +-
 drivers/gpu/drm/amd/amdgpu/atombios_encoders.h     |     5 +
 drivers/gpu/drm/amd/amdgpu/atombios_i2c.c          |    11 +-
 drivers/gpu/drm/amd/amdgpu/ci_dpm.c                |    24 +-
 drivers/gpu/drm/amd/amdgpu/cik.c                   |   212 +-
 drivers/gpu/drm/amd/amdgpu/cik_ih.c                |     2 +-
 drivers/gpu/drm/amd/amdgpu/cik_sdma.c              |    28 +-
 drivers/gpu/drm/amd/amdgpu/cikd.h                  |     2 +-
 drivers/gpu/drm/amd/amdgpu/clearstate_gfx9.h       |     8 +-
 drivers/gpu/drm/amd/amdgpu/cz_ih.c                 |     2 +-
 drivers/gpu/drm/amd/amdgpu/dce_v10_0.c             |    44 +-
 drivers/gpu/drm/amd/amdgpu/dce_v11_0.c             |    50 +-
 drivers/gpu/drm/amd/amdgpu/dce_v6_0.c              |    22 +-
 drivers/gpu/drm/amd/amdgpu/dce_v8_0.c              |    20 +-
 drivers/gpu/drm/amd/amdgpu/dce_virtual.c           |    14 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c              |    38 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c              |    84 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c              |   221 +-
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c              |   329 +-
 drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c           |    51 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c              |    35 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c              |    49 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c              |    91 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c              |   374 +-
 drivers/gpu/drm/amd/amdgpu/iceland_ih.c            |     2 +-
 drivers/gpu/drm/amd/amdgpu/kv_dpm.c                |     8 +-
 drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c            |    63 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c              |    28 +-
 drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c              |    64 +-
 drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c             |   125 +-
 drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h             |    26 +-
 drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c             |   159 +-
 drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h             |    22 +-
 drivers/gpu/drm/amd/amdgpu/psp_v10_0.c             |    16 +-
 drivers/gpu/drm/amd/amdgpu/psp_v3_1.c              |    20 +-
 drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c             |    42 +-
 drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c             |   117 +-
 drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c             |   322 +-
 drivers/gpu/drm/amd/amdgpu/si.c                    |   178 +-
 drivers/gpu/drm/amd/amdgpu/si_dma.c                |    30 +-
 drivers/gpu/drm/amd/amdgpu/si_dpm.c                |    65 +-
 drivers/gpu/drm/amd/amdgpu/si_ih.c                 |     2 +-
 drivers/gpu/drm/amd/amdgpu/soc15.c                 |   278 +-
 drivers/gpu/drm/amd/amdgpu/soc15.h                 |    20 +
 drivers/gpu/drm/amd/amdgpu/soc15_common.h          |    62 +-
 drivers/gpu/drm/amd/amdgpu/tonga_ih.c              |     2 +-
 drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c              |     6 +-
 drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c              |     6 +-
 drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c              |    63 +-
 drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c              |   112 +-
 drivers/gpu/drm/amd/amdgpu/vce_v3_0.c              |    10 +-
 drivers/gpu/drm/amd/amdgpu/vce_v4_0.c              |    74 +-
 drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c              |    88 +-
 drivers/gpu/drm/amd/amdgpu/vega10_ih.c             |    73 +-
 drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c       |    56 +
 drivers/gpu/drm/amd/amdgpu/vi.c                    |   187 +-
 drivers/gpu/drm/amd/amdgpu/vid.h                   |     2 +
 drivers/gpu/drm/amd/amdkfd/Makefile                |     4 +-
 .../gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm  |  1384 +
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c           |    46 +-
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c              |  1267 +
 drivers/gpu/drm/amd/amdkfd/kfd_crat.h              |    42 +-
 drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c            |    14 +-
 drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c           |    75 +
 drivers/gpu/drm/amd/amdkfd/kfd_device.c            |    31 +-
 .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |   144 +-
 .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |     8 +-
 drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c          |     9 +-
 drivers/gpu/drm/amd/amdkfd/kfd_events.c            |    14 +-
 drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c       |     7 +-
 drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c      |     2 +-
 drivers/gpu/drm/amd/amdkfd/kfd_module.c            |    17 +
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h       |     4 +
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c   |    48 +-
 drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c    |   167 +-
 drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c    |    62 +-
 drivers/gpu/drm/amd/amdkfd/kfd_pasid.c             |     2 +-
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |    83 +-
 drivers/gpu/drm/amd/amdkfd/kfd_process.c           |   267 +-
 .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c |    78 +-
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c          |  1061 +-
 drivers/gpu/drm/amd/amdkfd/kfd_topology.h          |    33 +-
 drivers/gpu/drm/amd/display/TODO                   |     3 +
 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |   477 +-
 drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h  |    12 +
 .../drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c  |     2 +-
 .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c  |     7 +-
 .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.c    |    13 +-
 .../drm/amd/display/amdgpu_dm/amdgpu_dm_services.c |    33 +-
 drivers/gpu/drm/amd/display/dc/basics/Makefile     |     2 +-
 drivers/gpu/drm/amd/display/dc/basics/conversion.c |     2 +-
 drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c |    28 +
 .../gpu/drm/amd/display/dc/basics/log_helpers.c    |    10 +-
 drivers/gpu/drm/amd/display/dc/basics/logger.c     |    22 +-
 drivers/gpu/drm/amd/display/dc/bios/bios_parser.c  |    60 +-
 .../gpu/drm/amd/display/dc/bios/command_table.c    |    21 +
 .../gpu/drm/amd/display/dc/bios/command_table2.c   |    13 +
 drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c   |     9 +
 .../gpu/drm/amd/display/dc/calcs/dcn_calc_auto.c   |    17 +-
 .../gpu/drm/amd/display/dc/calcs/dcn_calc_math.c   |    27 +-
 drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c   |   148 +-
 drivers/gpu/drm/amd/display/dc/core/dc.c           |   631 +-
 drivers/gpu/drm/amd/display/dc/core/dc_debug.c     |     4 +
 .../gpu/drm/amd/display/dc/core/dc_hw_sequencer.c  |   145 +
 drivers/gpu/drm/amd/display/dc/core/dc_link.c      |   186 +-
 drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c   |    23 +-
 drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c |    20 +-
 drivers/gpu/drm/amd/display/dc/core/dc_resource.c  |   154 +-
 drivers/gpu/drm/amd/display/dc/core/dc_stream.c    |    95 +-
 drivers/gpu/drm/amd/display/dc/dc.h                |   597 +-
 drivers/gpu/drm/amd/display/dc/dc_dp_types.h       |    28 +-
 drivers/gpu/drm/amd/display/dc/dc_helper.c         |     7 +-
 drivers/gpu/drm/amd/display/dc/dc_hw_types.h       |    27 +-
 drivers/gpu/drm/amd/display/dc/dc_link.h           |   207 +
 drivers/gpu/drm/amd/display/dc/dc_stream.h         |   295 +
 drivers/gpu/drm/amd/display/dc/dc_types.h          |     6 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_abm.c       |    50 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_abm.h       |     8 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c    |     4 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c      |   228 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h      |    15 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h     |    26 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c       |    33 +-
 .../gpu/drm/amd/display/dc/dce/dce_link_encoder.c  |    59 +-
 .../gpu/drm/amd/display/dc/dce/dce_link_encoder.h  |     8 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_opp.c       |     9 +-
 .../drm/amd/display/dc/dce/dce_stream_encoder.c    |    34 +-
 drivers/gpu/drm/amd/display/dc/dce/dce_transform.c |   278 +-
 .../amd/display/dc/dce100/dce100_hw_sequencer.c    |     2 +
 .../drm/amd/display/dc/dce100/dce100_resource.c    |     1 +
 .../amd/display/dc/dce110/dce110_hw_sequencer.c    |   454 +-
 .../amd/display/dc/dce110/dce110_hw_sequencer.h    |     8 +-
 .../drm/amd/display/dc/dce110/dce110_mem_input_v.c |    28 +-
 .../amd/display/dc/dce110/dce110_opp_regamma_v.c   |     2 +-
 .../drm/amd/display/dc/dce110/dce110_resource.c    |    19 +
 .../display/dc/dce110/dce110_timing_generator.c    |   265 +-
 .../display/dc/dce110/dce110_timing_generator.h    |     6 +
 .../drm/amd/display/dc/dce112/dce112_resource.c    |     2 +
 .../amd/display/dc/dce120/dce120_hw_sequencer.c    |     6 +-
 .../drm/amd/display/dc/dce120/dce120_resource.c    |    10 +-
 .../display/dc/dce120/dce120_timing_generator.c    |     6 +-
 .../gpu/drm/amd/display/dc/dce80/dce80_resource.c  |     3 +
 drivers/gpu/drm/amd/display/dc/dcn10/Makefile      |     5 +-
 .../gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c |   294 +-
 .../gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h |    10 +
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c   |   114 +-
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h   |   530 +-
 .../gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c    |   221 +-
 .../gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c  |     8 +
 .../gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c    |   516 +
 .../gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h    |   214 +
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c  |   116 +-
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h  |   569 +-
 .../drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c  |  2131 +-
 .../drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h  |     1 +
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c   |   529 +-
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h   |    90 +-
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c   |   118 +-
 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h   |   125 +-
 .../{dcn10_timing_generator.c => dcn10_optc.c}     |   439 +-
 .../{dcn10_timing_generator.h => dcn10_optc.h}     |   150 +-
 .../gpu/drm/amd/display/dc/dcn10/dcn10_resource.c  |   133 +-
 drivers/gpu/drm/amd/display/dc/dm_services.h       |     7 +
 .../gpu/drm/amd/display/dc/dml/display_mode_lib.c  |    29 -
 .../drm/amd/display/dc/dml/display_mode_structs.h  |     6 +-
 .../gpu/drm/amd/display/dc/dml/display_mode_vba.c  |   131 +-
 .../drm/amd/display/dc/dml/display_rq_dlg_calc.c   |     9 +
 .../amd/display/dc/dml/dml1_display_rq_dlg_calc.c  |     9 +
 .../gpu/drm/amd/display/dc/dml/soc_bounding_box.c  |    10 +
 .../amd/display/dc/gpio/dce120/hw_factory_dce120.c |     6 +-
 .../display/dc/gpio/dce120/hw_translate_dce120.c   |     6 +-
 .../amd/display/dc/gpio/dcn10/hw_factory_dcn10.c   |     6 +-
 .../amd/display/dc/gpio/dcn10/hw_translate_dcn10.c |     6 +-
 .../amd/display/dc/i2caux/dce120/i2caux_dce120.c   |     6 +-
 .../drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c |     6 +-
 drivers/gpu/drm/amd/display/dc/inc/core_types.h    |     3 +-
 drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h     |     6 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/abm.h        |    14 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h       |    21 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h        |    46 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h       |    19 +
 drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h  |    55 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h        |     6 +-
 .../gpu/drm/amd/display/dc/inc/hw/link_encoder.h   |     5 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h        |   157 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/opp.h        |    29 +-
 .../drm/amd/display/dc/inc/hw/timing_generator.h   |    71 +-
 drivers/gpu/drm/amd/display/dc/inc/hw/transform.h  |     6 +-
 drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h  |    33 +-
 .../amd/display/dc/irq/dce110/irq_service_dce110.c |     2 +-
 .../amd/display/dc/irq/dce120/irq_service_dce120.c |     6 +-
 .../amd/display/dc/irq/dcn10/irq_service_dcn10.c   |     6 +-
 drivers/gpu/drm/amd/display/dc/os_types.h          |    14 +-
 .../amd/display/dc/virtual/virtual_link_encoder.c  |     6 +-
 .../drm/amd/display/include/ddc_service_types.h    |    33 -
 drivers/gpu/drm/amd/display/include/fixed31_32.h   |     7 +
 .../amd/display/include/grph_object_ctrl_defs.h    |     5 -
 .../gpu/drm/amd/display/include/grph_object_id.h   |    16 +-
 .../gpu/drm/amd/display/include/logger_interface.h |     5 +
 drivers/gpu/drm/amd/display/include/signal_types.h |     5 +
 .../drm/amd/display/modules/freesync/freesync.c    |    84 +-
 drivers/gpu/drm/amd/include/amd_shared.h           |   172 -
 .../amd/include/asic_reg/athub/athub_1_0_offset.h  |   453 +
 .../amd/include/asic_reg/athub/athub_1_0_sh_mask.h |  2045 ++
 .../asic_reg/{vega10/DC => dce}/dce_12_0_offset.h  |     0
 .../asic_reg/{vega10/DC => dce}/dce_12_0_sh_mask.h |     0
 .../asic_reg/{raven1/DCN => dcn}/dcn_1_0_offset.h  |     0
 .../asic_reg/{raven1/DCN => dcn}/dcn_1_0_sh_mask.h |     0
 .../asic_reg/{vega10/GC => gc}/gc_9_0_default.h    |     7 -
 .../asic_reg/{vega10/GC => gc}/gc_9_0_offset.h     |    14 -
 .../asic_reg/{vega10/GC => gc}/gc_9_0_sh_mask.h    |    45 -
 .../asic_reg/{raven1/GC => gc}/gc_9_1_offset.h     |    14 -
 .../drm/amd/include/asic_reg/hdp/hdp_4_0_offset.h  |   209 +
 .../drm/amd/include/asic_reg/hdp/hdp_4_0_sh_mask.h |   601 +
 .../{vega10/MMHUB => mmhub}/mmhub_1_0_default.h    |     0
 .../{vega10/MMHUB => mmhub}/mmhub_1_0_offset.h     |     0
 .../{vega10/MMHUB => mmhub}/mmhub_1_0_sh_mask.h    |     0
 .../{raven1/MMHUB => mmhub}/mmhub_9_1_offset.h     |     0
 .../{raven1/MMHUB => mmhub}/mmhub_9_1_sh_mask.h    |     0
 .../asic_reg/{raven1/MP => mp}/mp_10_0_default.h   |     0
 .../asic_reg/{raven1/MP => mp}/mp_10_0_offset.h    |     0
 .../asic_reg/{raven1/MP => mp}/mp_10_0_sh_mask.h   |     0
 .../drm/amd/include/asic_reg/mp/mp_9_0_offset.h    |   375 +
 .../drm/amd/include/asic_reg/mp/mp_9_0_sh_mask.h   |  1463 +
 .../{vega10/NBIF => nbif}/nbif_6_1_offset.h        |     0
 .../{vega10/NBIF => nbif}/nbif_6_1_sh_mask.h       |     0
 .../{vega10/NBIO => nbio}/nbio_6_1_default.h       |     0
 .../{vega10/NBIO => nbio}/nbio_6_1_offset.h        |     0
 .../{vega10/NBIO => nbio}/nbio_6_1_sh_mask.h       |     0
 .../{raven1/NBIO => nbio}/nbio_7_0_default.h       |     0
 .../{raven1/NBIO => nbio}/nbio_7_0_offset.h        |     0
 .../{raven1/NBIO => nbio}/nbio_7_0_sh_mask.h       |     0
 .../{vega10/OSSSYS => oss}/osssys_4_0_offset.h     |     0
 .../{vega10/OSSSYS => oss}/osssys_4_0_sh_mask.h    |     0
 .../include/asic_reg/raven1/DCN/dcn_1_0_default.h  |  7988 -----
 .../include/asic_reg/raven1/GC/gc_9_1_default.h    |  4005 ---
 .../include/asic_reg/raven1/GC/gc_9_1_sh_mask.h    | 31191 -------------------
 .../asic_reg/raven1/MMHUB/mmhub_9_1_default.h      |  1028 -
 .../asic_reg/raven1/SDMA0/sdma0_4_1_sh_mask.h      |  1658 -
 .../include/asic_reg/raven1/VCN/vcn_1_0_default.h  |   202 -
 .../amd/include/asic_reg/sdma0/sdma0_4_0_default.h |   286 +
 .../amd/include/asic_reg/sdma0/sdma0_4_0_offset.h  |   547 +
 .../amd/include/asic_reg/sdma0/sdma0_4_0_sh_mask.h |  1852 ++
 .../{raven1/SDMA0 => sdma0}/sdma0_4_1_default.h    |     0
 .../{raven1/SDMA0 => sdma0}/sdma0_4_1_offset.h     |     0
 .../amd/include/asic_reg/sdma1/sdma1_4_0_default.h |   282 +
 .../amd/include/asic_reg/sdma1/sdma1_4_0_offset.h  |   539 +
 .../amd/include/asic_reg/sdma1/sdma1_4_0_sh_mask.h |  1810 ++
 .../{vega10/SMUIO => smuio}/smuio_9_0_offset.h     |     0
 .../{vega10/SMUIO => smuio}/smuio_9_0_sh_mask.h    |     0
 .../{raven1/THM => thm}/thm_10_0_default.h         |     0
 .../asic_reg/{raven1/THM => thm}/thm_10_0_offset.h |     0
 .../{raven1/THM => thm}/thm_10_0_sh_mask.h         |     0
 .../asic_reg/{vega10/THM => thm}/thm_9_0_default.h |     0
 .../asic_reg/{vega10/THM => thm}/thm_9_0_offset.h  |     0
 .../asic_reg/{vega10/THM => thm}/thm_9_0_sh_mask.h |     0
 .../drm/amd/include/asic_reg/umc/umc_6_0_default.h |    31 +
 .../drm/amd/include/asic_reg/umc/umc_6_0_offset.h  |    52 +
 .../drm/amd/include/asic_reg/umc/umc_6_0_sh_mask.h |    36 +
 .../asic_reg/{vega10/UVD => uvd}/uvd_7_0_offset.h  |     0
 .../asic_reg/{vega10/UVD => uvd}/uvd_7_0_sh_mask.h |     0
 .../asic_reg/{vega10/VCE => vce}/vce_4_0_default.h |     0
 .../asic_reg/{vega10/VCE => vce}/vce_4_0_offset.h  |     0
 .../asic_reg/{vega10/VCE => vce}/vce_4_0_sh_mask.h |     0
 .../asic_reg/{raven1/VCN => vcn}/vcn_1_0_offset.h  |     0
 .../asic_reg/{raven1/VCN => vcn}/vcn_1_0_sh_mask.h |     0
 .../asic_reg/vega10/ATHUB/athub_1_0_default.h      |   241 -
 .../asic_reg/vega10/ATHUB/athub_1_0_offset.h       |   453 -
 .../asic_reg/vega10/ATHUB/athub_1_0_sh_mask.h      |  2045 --
 .../include/asic_reg/vega10/DC/dce_12_0_default.h  |  9868 ------
 .../include/asic_reg/vega10/HDP/hdp_4_0_default.h  |   117 -
 .../include/asic_reg/vega10/HDP/hdp_4_0_offset.h   |   209 -
 .../include/asic_reg/vega10/HDP/hdp_4_0_sh_mask.h  |   601 -
 .../include/asic_reg/vega10/MP/mp_9_0_default.h    |   342 -
 .../amd/include/asic_reg/vega10/MP/mp_9_0_offset.h |   375 -
 .../include/asic_reg/vega10/MP/mp_9_0_sh_mask.h    |  1463 -
 .../asic_reg/vega10/NBIF/nbif_6_1_default.h        |  1271 -
 .../asic_reg/vega10/OSSSYS/osssys_4_0_default.h    |   176 -
 .../asic_reg/vega10/SDMA0/sdma0_4_0_default.h      |   286 -
 .../asic_reg/vega10/SDMA0/sdma0_4_0_offset.h       |   547 -
 .../asic_reg/vega10/SDMA0/sdma0_4_0_sh_mask.h      |  1852 --
 .../asic_reg/vega10/SDMA1/sdma1_4_0_default.h      |   282 -
 .../asic_reg/vega10/SDMA1/sdma1_4_0_offset.h       |   539 -
 .../asic_reg/vega10/SDMA1/sdma1_4_0_sh_mask.h      |  1810 --
 .../asic_reg/vega10/SMUIO/smuio_9_0_default.h      |   100 -
 .../include/asic_reg/vega10/UVD/uvd_7_0_default.h  |   127 -
 drivers/gpu/drm/amd/include/dm_pp_interface.h      |   144 +
 drivers/gpu/drm/amd/include/kgd_kfd_interface.h    |    52 +-
 drivers/gpu/drm/amd/include/kgd_pp_interface.h     |   294 +
 .../amd/include/{asic_reg/vega10 => }/soc15ip.h    |     0
 .../include/{asic_reg/vega10 => }/vega10_enum.h    |     0
 drivers/gpu/drm/amd/include/vi_structs.h           |     2 +
 drivers/gpu/drm/amd/powerplay/amd_powerplay.c      |   172 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c     |    29 +-
 .../gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c  |    33 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c        |     1 +
 .../gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h    |     2 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c       |     4 +
 drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c |     2 +-
 .../amd/powerplay/hwmgr/process_pptables_v1_0.c    |     6 +-
 .../gpu/drm/amd/powerplay/hwmgr/processpptables.c  |     6 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c     |   182 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h     |    15 +
 drivers/gpu/drm/amd/powerplay/hwmgr/rv_inc.h       |    18 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c   |    79 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c |    92 +-
 drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h |     1 -
 drivers/gpu/drm/amd/powerplay/hwmgr/vega10_inc.h   |    23 +-
 drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h  |   275 +-
 .../gpu/drm/amd/powerplay/inc/hardwaremanager.h    |     1 +
 drivers/gpu/drm/amd/powerplay/inc/hwmgr.h          |    35 +-
 drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h       |     2 +-
 drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h       |     7 +-
 drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c   |    24 +-
 drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c   |    13 +
 drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c |     5 +-
 .../gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c  |    28 +-
 .../drm/amd/powerplay/smumgr/polaris10_smumgr.c    |     5 +-
 drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.c   |     8 +
 drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.h   |     2 +-
 drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c |     8 +-
 .../gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c    |    15 +-
 .../gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c   |     6 +-
 drivers/gpu/drm/amd/scheduler/gpu_scheduler.h      |   185 -
 drivers/gpu/drm/arc/arcpgu_drv.c                   |     2 +-
 drivers/gpu/drm/arm/hdlcd_crtc.c                   |     8 +-
 drivers/gpu/drm/arm/hdlcd_drv.c                    |     2 +-
 drivers/gpu/drm/arm/malidp_drv.c                   |    63 +-
 drivers/gpu/drm/arm/malidp_drv.h                   |     2 -
 drivers/gpu/drm/arm/malidp_planes.c                |     3 +-
 drivers/gpu/drm/armada/armada_crtc.c               |   395 +-
 drivers/gpu/drm/armada/armada_crtc.h               |    24 +-
 drivers/gpu/drm/armada/armada_drm.h                |     1 -
 drivers/gpu/drm/armada/armada_drv.c                |     8 +-
 drivers/gpu/drm/armada/armada_fb.c                 |    11 +-
 drivers/gpu/drm/armada/armada_fbdev.c              |     8 -
 drivers/gpu/drm/armada/armada_overlay.c            |   282 +-
 drivers/gpu/drm/armada/armada_trace.h              |    24 +-
 drivers/gpu/drm/ast/ast_mode.c                     |     1 +
 drivers/gpu/drm/ast/ast_tables.h                   |     4 +-
 drivers/gpu/drm/ast/ast_ttm.c                      |    15 +-
 drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c       |    26 +-
 drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h       |     2 +-
 drivers/gpu/drm/bochs/bochs_mm.c                   |     7 +-
 drivers/gpu/drm/bridge/analogix-anx78xx.c          |     2 -
 drivers/gpu/drm/bridge/sil-sii8620.c               |    53 +-
 drivers/gpu/drm/bridge/synopsys/dw-hdmi.c          |     6 +-
 drivers/gpu/drm/bridge/tc358767.c                  |     2 +
 drivers/gpu/drm/cirrus/cirrus_main.c               |     9 +
 drivers/gpu/drm/cirrus/cirrus_mode.c               |    40 +-
 drivers/gpu/drm/cirrus/cirrus_ttm.c                |    12 +-
 drivers/gpu/drm/drm_atomic.c                       |    70 +-
 drivers/gpu/drm/drm_atomic_helper.c                |   127 +-
 drivers/gpu/drm/drm_auth.c                         |     6 +
 drivers/gpu/drm/drm_blend.c                        |     8 +-
 drivers/gpu/drm/drm_connector.c                    |    73 +
 drivers/gpu/drm/drm_debugfs.c                      |     8 +
 drivers/gpu/drm/drm_dp_helper.c                    |     1 -
 drivers/gpu/drm/drm_drv.c                          |    47 -
 drivers/gpu/drm/drm_edid.c                         |   131 +-
 drivers/gpu/drm/drm_fb_cma_helper.c                |   156 +-
 drivers/gpu/drm/drm_fb_helper.c                    |   352 +-
 drivers/gpu/drm/drm_file.c                         |     6 +-
 drivers/gpu/drm/drm_framebuffer.c                  |   100 +-
 drivers/gpu/drm/drm_gem.c                          |    21 +-
 drivers/gpu/drm/drm_gem_cma_helper.c               |    53 +-
 drivers/gpu/drm/drm_gem_framebuffer_helper.c       |     6 +-
 drivers/gpu/drm/drm_internal.h                     |     7 +
 drivers/gpu/drm/drm_mm.c                           |    23 +-
 drivers/gpu/drm/drm_mode_config.c                  |     3 +
 drivers/gpu/drm/drm_modeset_helper.c               |    76 +
 .../drm/drm_panel_orientation_quirks.c}            |   104 +-
 drivers/gpu/drm/drm_plane_helper.c                 |   111 +-
 drivers/gpu/drm/drm_prime.c                        |     8 +-
 drivers/gpu/drm/drm_print.c                        |    54 +-
 drivers/gpu/drm/drm_probe_helper.c                 |    23 +-
 drivers/gpu/drm/drm_simple_kms_helper.c            |     9 +-
 drivers/gpu/drm/drm_syncobj.c                      |    56 +-
 drivers/gpu/drm/drm_vblank.c                       |    12 +-
 drivers/gpu/drm/drm_vma_manager.c                  |    15 +-
 drivers/gpu/drm/etnaviv/Kconfig                    |     9 +
 drivers/gpu/drm/etnaviv/etnaviv_buffer.c           |    40 +-
 drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c       |     1 +
 drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c           |    29 +-
 drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h           |    18 +-
 drivers/gpu/drm/etnaviv/etnaviv_drv.c              |    22 +-
 drivers/gpu/drm/etnaviv/etnaviv_drv.h              |    14 +-
 drivers/gpu/drm/etnaviv/etnaviv_dump.c             |    23 +-
 drivers/gpu/drm/etnaviv/etnaviv_gem.c              |   197 +-
 drivers/gpu/drm/etnaviv/etnaviv_gem.h              |    22 +-
 drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c        |     7 +-
 drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c       |   199 +-
 drivers/gpu/drm/etnaviv/etnaviv_gpu.c              |   215 +-
 drivers/gpu/drm/etnaviv/etnaviv_gpu.h              |    11 +-
 drivers/gpu/drm/etnaviv/etnaviv_iommu.c            |     5 +-
 drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c         |     2 +-
 drivers/gpu/drm/etnaviv/etnaviv_mmu.c              |    14 +-
 drivers/gpu/drm/etnaviv/etnaviv_perfmon.c          |     4 +-
 drivers/gpu/drm/etnaviv/etnaviv_perfmon.h          |     2 +-
 drivers/gpu/drm/exynos/Kconfig                     |    11 +-
 drivers/gpu/drm/exynos/Makefile                    |     1 -
 drivers/gpu/drm/exynos/exynos5433_drm_decon.c      |     8 +-
 drivers/gpu/drm/exynos/exynos7_drm_decon.c         |     2 +-
 drivers/gpu/drm/exynos/exynos_drm_drv.c            |    20 +-
 drivers/gpu/drm/exynos/exynos_drm_drv.h            |     2 -
 drivers/gpu/drm/exynos/exynos_drm_fb.c             |     2 +-
 drivers/gpu/drm/exynos/exynos_drm_fbdev.c          |    18 -
 drivers/gpu/drm/exynos/exynos_drm_fbdev.h          |     2 -
 drivers/gpu/drm/exynos/exynos_drm_g2d.c            |    12 +-
 drivers/gpu/drm/exynos/exynos_drm_ipp.c            |  1806 --
 drivers/gpu/drm/exynos/exynos_drm_ipp.h            |   252 -
 drivers/gpu/drm/exynos/exynos_drm_rotator.h        |    19 -
 drivers/gpu/drm/exynos/exynos_hdmi.c               |    10 +-
 .../gpu/drm/exynos/regs-decon5433.h                |     6 +-
 .../gpu/drm/exynos/regs-decon7.h                   |     8 +-
 drivers/gpu/drm/exynos/regs-fimc.h                 |     2 +-
 drivers/gpu/drm/exynos/regs-hdmi.h                 |     2 +-
 drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c          |    25 +-
 drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h          |     1 -
 drivers/gpu/drm/gma500/cdv_device.c                |    16 +-
 drivers/gpu/drm/gma500/framebuffer.c               |     9 +-
 drivers/gpu/drm/gma500/gma_device.c                |     4 +-
 drivers/gpu/drm/gma500/mid_bios.c                  |    12 +-
 drivers/gpu/drm/gma500/mmu.c                       |     2 +-
 drivers/gpu/drm/gma500/psb_drv.c                   |    27 +-
 drivers/gpu/drm/gma500/psb_drv.h                   |    19 +-
 drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c        |    11 +-
 drivers/gpu/drm/i2c/tda998x_drv.c                  |     1 -
 drivers/gpu/drm/i915/Kconfig.debug                 |    29 +
 drivers/gpu/drm/i915/Makefile                      |    32 +-
 drivers/gpu/drm/i915/gvt/Makefile                  |     3 +-
 drivers/gpu/drm/i915/gvt/cfg_space.c               |    18 +-
 drivers/gpu/drm/i915/gvt/cmd_parser.c              |   272 +-
 drivers/gpu/drm/i915/gvt/debug.h                   |    24 +-
 drivers/gpu/drm/i915/gvt/debugfs.c                 |   212 +
 drivers/gpu/drm/i915/gvt/display.c                 |    80 +-
 drivers/gpu/drm/i915/gvt/display.h                 |     2 +
 drivers/gpu/drm/i915/gvt/dmabuf.c                  |   536 +
 drivers/gpu/drm/i915/gvt/dmabuf.h                  |    67 +
 drivers/gpu/drm/i915/gvt/edid.c                    |    22 +-
 drivers/gpu/drm/i915/gvt/execlist.c                |   502 +-
 drivers/gpu/drm/i915/gvt/execlist.h                |     8 +-
 drivers/gpu/drm/i915/gvt/fb_decoder.c              |   514 +
 drivers/gpu/drm/i915/gvt/fb_decoder.h              |   169 +
 drivers/gpu/drm/i915/gvt/firmware.c                |    26 +-
 drivers/gpu/drm/i915/gvt/gtt.c                     |   322 +-
 drivers/gpu/drm/i915/gvt/gtt.h                     |    45 +-
 drivers/gpu/drm/i915/gvt/gvt.c                     |   153 +-
 drivers/gpu/drm/i915/gvt/gvt.h                     |   127 +-
 drivers/gpu/drm/i915/gvt/handlers.c                |   842 +-
 drivers/gpu/drm/i915/gvt/hypercall.h               |     4 +
 drivers/gpu/drm/i915/gvt/kvmgt.c                   |   419 +-
 drivers/gpu/drm/i915/gvt/mmio.c                    |   127 +-
 drivers/gpu/drm/i915/gvt/mmio.h                    |     9 +-
 drivers/gpu/drm/i915/gvt/mmio_context.c            |   415 +
 .../gpu/drm/i915/gvt/{render.h => mmio_context.h}  |     9 +
 drivers/gpu/drm/i915/gvt/mpt.h                     |    96 +-
 drivers/gpu/drm/i915/gvt/opregion.c                |   418 +-
 drivers/gpu/drm/i915/gvt/reg.h                     |     6 +-
 drivers/gpu/drm/i915/gvt/render.c                  |   405 -
 drivers/gpu/drm/i915/gvt/sched_policy.c            |    26 +-
 drivers/gpu/drm/i915/gvt/sched_policy.h            |     2 +
 drivers/gpu/drm/i915/gvt/scheduler.c               |   734 +-
 drivers/gpu/drm/i915/gvt/scheduler.h               |    41 +-
 drivers/gpu/drm/i915/gvt/trace.h                   |    15 +-
 drivers/gpu/drm/i915/gvt/vgpu.c                    |    76 +-
 drivers/gpu/drm/i915/i915_cmd_parser.c             |    30 +-
 drivers/gpu/drm/i915/i915_debugfs.c                |   438 +-
 drivers/gpu/drm/i915/i915_drv.c                    |   198 +-
 drivers/gpu/drm/i915/i915_drv.h                    |   651 +-
 drivers/gpu/drm/i915/i915_gem.c                    |   569 +-
 drivers/gpu/drm/i915/i915_gem.h                    |    12 +-
 drivers/gpu/drm/i915/i915_gem_clflush.c            |     2 +-
 drivers/gpu/drm/i915/i915_gem_context.c            |   370 +-
 drivers/gpu/drm/i915/i915_gem_context.h            |     4 +-
 drivers/gpu/drm/i915/i915_gem_evict.c              |     4 +-
 drivers/gpu/drm/i915/i915_gem_execbuffer.c         |    89 +-
 drivers/gpu/drm/i915/i915_gem_gtt.c                |   249 +-
 drivers/gpu/drm/i915/i915_gem_gtt.h                |    19 +-
 drivers/gpu/drm/i915/i915_gem_internal.c           |     4 +-
 drivers/gpu/drm/i915/i915_gem_object.h             |    13 +-
 drivers/gpu/drm/i915/i915_gem_render_state.c       |   139 +-
 drivers/gpu/drm/i915/i915_gem_render_state.h       |     4 +-
 drivers/gpu/drm/i915/i915_gem_request.c            |    74 +-
 drivers/gpu/drm/i915/i915_gem_request.h            |     2 +
 drivers/gpu/drm/i915/i915_gem_shrinker.c           |   142 +-
 drivers/gpu/drm/i915/i915_gem_stolen.c             |   323 +-
 drivers/gpu/drm/i915/i915_gem_tiling.c             |    19 +-
 drivers/gpu/drm/i915/i915_gem_timeline.c           |    11 +-
 drivers/gpu/drm/i915/i915_gem_timeline.h           |     2 +-
 drivers/gpu/drm/i915/i915_gpu_error.c              |   195 +-
 drivers/gpu/drm/i915/i915_irq.c                    |    14 +-
 drivers/gpu/drm/i915/i915_memcpy.c                 |     7 +-
 drivers/gpu/drm/i915/i915_oa_bdw.c                 |     4 +-
 drivers/gpu/drm/i915/i915_oa_bxt.c                 |     4 +-
 drivers/gpu/drm/i915/i915_oa_cflgt2.c              |     4 +-
 drivers/gpu/drm/i915/i915_oa_cflgt3.c              |   109 +
 drivers/gpu/drm/i915/i915_oa_cflgt3.h              |    34 +
 drivers/gpu/drm/i915/i915_oa_chv.c                 |     4 +-
 drivers/gpu/drm/i915/i915_oa_cnl.c                 |   121 +
 drivers/gpu/drm/i915/i915_oa_cnl.h                 |    34 +
 drivers/gpu/drm/i915/i915_oa_glk.c                 |     4 +-
 drivers/gpu/drm/i915/i915_oa_hsw.c                 |     4 +-
 drivers/gpu/drm/i915/i915_oa_kblgt2.c              |     4 +-
 drivers/gpu/drm/i915/i915_oa_kblgt3.c              |     4 +-
 drivers/gpu/drm/i915/i915_oa_sklgt2.c              |     4 +-
 drivers/gpu/drm/i915/i915_oa_sklgt3.c              |     4 +-
 drivers/gpu/drm/i915/i915_oa_sklgt4.c              |     4 +-
 drivers/gpu/drm/i915/i915_params.c                 |    59 +-
 drivers/gpu/drm/i915/i915_params.h                 |    14 +-
 drivers/gpu/drm/i915/i915_pci.c                    |    98 +-
 drivers/gpu/drm/i915/i915_perf.c                   |   209 +-
 drivers/gpu/drm/i915/i915_pmu.c                    |   920 +
 drivers/gpu/drm/i915/i915_pmu.h                    |   117 +
 drivers/gpu/drm/i915/i915_reg.h                    |    99 +-
 drivers/gpu/drm/i915/i915_selftest.h               |     2 +-
 drivers/gpu/drm/i915/i915_sw_fence.c               |     3 +
 drivers/gpu/drm/i915/i915_sysfs.c                  |    55 +-
 drivers/gpu/drm/i915/i915_trace.h                  |    40 +-
 drivers/gpu/drm/i915/i915_utils.h                  |    22 +-
 drivers/gpu/drm/i915/i915_vma.c                    |    40 +-
 drivers/gpu/drm/i915/i915_vma.h                    |    35 +-
 drivers/gpu/drm/i915/intel_audio.c                 |   208 +-
 drivers/gpu/drm/i915/intel_bios.c                  |   149 +-
 drivers/gpu/drm/i915/intel_breadcrumbs.c           |   145 +-
 drivers/gpu/drm/i915/intel_cdclk.c                 |   421 +-
 drivers/gpu/drm/i915/intel_color.c                 |     4 +-
 drivers/gpu/drm/i915/intel_crt.c                   |    46 +-
 drivers/gpu/drm/i915/intel_csr.c                   |    13 +-
 drivers/gpu/drm/i915/intel_ddi.c                   |   352 +-
 drivers/gpu/drm/i915/intel_device_info.c           |   200 +-
 drivers/gpu/drm/i915/intel_device_info.h           |   183 +
 drivers/gpu/drm/i915/intel_display.c               |  1007 +-
 drivers/gpu/drm/i915/intel_display.h               |   321 +
 drivers/gpu/drm/i915/intel_dp.c                    |   604 +-
 drivers/gpu/drm/i915/intel_dp_link_training.c      |    26 +-
 drivers/gpu/drm/i915/intel_dp_mst.c                |    79 +-
 drivers/gpu/drm/i915/intel_dpio_phy.c              |   122 +-
 drivers/gpu/drm/i915/intel_dpll_mgr.c              |   107 +-
 drivers/gpu/drm/i915/intel_drv.h                   |    80 +-
 drivers/gpu/drm/i915/intel_dsi.c                   |    47 +-
 drivers/gpu/drm/i915/intel_dvo.c                   |     2 +
 drivers/gpu/drm/i915/intel_engine_cs.c             |   497 +-
 drivers/gpu/drm/i915/intel_fbc.c                   |    48 +-
 drivers/gpu/drm/i915/intel_fbdev.c                 |     3 +-
 drivers/gpu/drm/i915/intel_guc.c                   |   152 +-
 drivers/gpu/drm/i915/intel_guc.h                   |    21 +-
 drivers/gpu/drm/i915/intel_guc_ct.c                |     1 +
 drivers/gpu/drm/i915/intel_guc_fw.c                |   248 +-
 drivers/gpu/drm/i915/intel_guc_fw.h                |     2 +-
 drivers/gpu/drm/i915/intel_guc_fwif.h              |    40 +
 drivers/gpu/drm/i915/intel_guc_log.c               |    29 +-
 .../drm/i915/{i915_guc_reg.h => intel_guc_reg.h}   |    14 +-
 ...915_guc_submission.c => intel_guc_submission.c} |   751 +-
 ...915_guc_submission.h => intel_guc_submission.h} |    17 +-
 drivers/gpu/drm/i915/intel_gvt.c                   |     7 +-
 drivers/gpu/drm/i915/intel_hangcheck.c             |    72 +-
 drivers/gpu/drm/i915/intel_hdmi.c                  |   128 +-
 drivers/gpu/drm/i915/intel_huc.c                   |    88 +-
 drivers/gpu/drm/i915/intel_huc.h                   |     6 +-
 drivers/gpu/drm/i915/intel_i2c.c                   |    61 +-
 drivers/gpu/drm/i915/intel_lrc.c                   |   288 +-
 drivers/gpu/drm/i915/intel_lrc.h                   |     5 -
 drivers/gpu/drm/i915/intel_lvds.c                  |     2 +
 drivers/gpu/drm/i915/intel_modes.c                 |     1 -
 drivers/gpu/drm/i915/intel_opregion.c              |     6 +-
 drivers/gpu/drm/i915/intel_opregion.h              |   106 +
 drivers/gpu/drm/i915/intel_overlay.c               |     4 +-
 drivers/gpu/drm/i915/intel_panel.c                 |    88 +-
 drivers/gpu/drm/i915/intel_pipe_crc.c              |     6 +-
 drivers/gpu/drm/i915/intel_pm.c                    |   443 +-
 drivers/gpu/drm/i915/intel_psr.c                   |     4 +-
 drivers/gpu/drm/i915/intel_ringbuffer.c            |   747 +-
 drivers/gpu/drm/i915/intel_ringbuffer.h            |   249 +-
 drivers/gpu/drm/i915/intel_runtime_pm.c            |    10 +-
 drivers/gpu/drm/i915/intel_sdvo.c                  |     6 +-
 drivers/gpu/drm/i915/intel_sprite.c                |    13 +-
 drivers/gpu/drm/i915/intel_tv.c                    |     6 +-
 drivers/gpu/drm/i915/intel_uc.c                    |   265 +-
 drivers/gpu/drm/i915/intel_uc.h                    |    23 +
 drivers/gpu/drm/i915/intel_uc_fw.c                 |     6 +-
 drivers/gpu/drm/i915/intel_uc_fw.h                 |     7 +-
 drivers/gpu/drm/i915/intel_uncore.c                |   172 +-
 drivers/gpu/drm/i915/intel_vbt_defs.h              |    10 +-
 drivers/gpu/drm/i915/selftests/huge_pages.c        |   181 +-
 .../gpu/drm/i915/selftests/i915_gem_coherency.c    |    16 +-
 drivers/gpu/drm/i915/selftests/i915_gem_context.c  |    33 +-
 drivers/gpu/drm/i915/selftests/i915_gem_evict.c    |     2 +
 drivers/gpu/drm/i915/selftests/i915_gem_gtt.c      |    52 +-
 drivers/gpu/drm/i915/selftests/i915_gem_object.c   |     2 +
 drivers/gpu/drm/i915/selftests/i915_gem_request.c  |    22 +-
 drivers/gpu/drm/i915/selftests/i915_gem_timeline.c |     2 +-
 .../gpu/drm/i915/selftests/i915_live_selftests.h   |     1 +
 drivers/gpu/drm/i915/selftests/i915_random.c       |     2 +-
 drivers/gpu/drm/i915/selftests/i915_syncmap.c      |     6 +-
 drivers/gpu/drm/i915/selftests/i915_vma.c          |     2 +-
 drivers/gpu/drm/i915/selftests/intel_guc.c         |   353 +
 drivers/gpu/drm/i915/selftests/intel_hangcheck.c   |   331 +-
 drivers/gpu/drm/i915/selftests/intel_uncore.c      |    11 +-
 drivers/gpu/drm/i915/selftests/mock_engine.c       |    41 +-
 drivers/gpu/drm/i915/selftests/mock_gem_device.c   |    14 +-
 drivers/gpu/drm/i915/selftests/mock_gtt.c          |     4 +-
 drivers/gpu/drm/imx/imx-drm-core.c                 |    34 +-
 drivers/gpu/drm/imx/imx-drm.h                      |     1 -
 drivers/gpu/drm/imx/ipuv3-crtc.c                   |     5 +
 drivers/gpu/drm/imx/ipuv3-plane.c                  |   120 +-
 drivers/gpu/drm/mediatek/mtk_drm_plane.c           |     8 +-
 drivers/gpu/drm/mediatek/mtk_hdmi.c                |     4 +-
 drivers/gpu/drm/meson/meson_crtc.c                 |     6 +
 drivers/gpu/drm/meson/meson_drv.c                  |     9 +
 drivers/gpu/drm/meson/meson_drv.h                  |     3 +
 drivers/gpu/drm/meson/meson_dw_hdmi.c              |    13 +
 drivers/gpu/drm/meson/meson_plane.c                |    15 +-
 drivers/gpu/drm/meson/meson_registers.h            |     4 +
 drivers/gpu/drm/mgag200/mgag200_ttm.c              |    15 +-
 drivers/gpu/drm/msm/adreno/a5xx_gpu.c              |    38 +
 drivers/gpu/drm/msm/adreno/a5xx_power.c            |     8 +-
 drivers/gpu/drm/msm/adreno/adreno_device.c         |   140 +-
 drivers/gpu/drm/msm/adreno/adreno_gpu.c            |    85 +-
 drivers/gpu/drm/msm/adreno/adreno_gpu.h            |     4 -
 drivers/gpu/drm/msm/dsi/pll/dsi_pll_14nm.c         |     2 +-
 drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h            |     7 +-
 drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c            |     2 +-
 drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c          |    19 +-
 drivers/gpu/drm/msm/msm_drv.c                      |    18 +-
 drivers/gpu/drm/msm/msm_drv.h                      |     3 +-
 drivers/gpu/drm/msm/msm_gem.c                      |    14 +-
 drivers/gpu/drm/msm/msm_gpu.c                      |   112 +-
 drivers/gpu/drm/msm/msm_gpu.h                      |    14 +-
 drivers/gpu/drm/nouveau/dispnv04/arb.c             |     4 +-
 drivers/gpu/drm/nouveau/dispnv04/hw.c              |    10 +-
 drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h   |     1 +
 .../gpu/drm/nouveau/include/nvkm/subdev/secboot.h  |     1 +
 .../gpu/drm/nouveau/include/nvkm/subdev/therm.h    |    17 +
 drivers/gpu/drm/nouveau/nouveau_backlight.c        |    14 +-
 drivers/gpu/drm/nouveau/nouveau_bo.c               |    51 +-
 drivers/gpu/drm/nouveau/nouveau_bo.h               |    28 +
 drivers/gpu/drm/nouveau/nouveau_connector.c        |    18 +-
 drivers/gpu/drm/nouveau/nouveau_crtc.h             |     1 -
 drivers/gpu/drm/nouveau/nouveau_display.c          |     3 +-
 drivers/gpu/drm/nouveau/nouveau_drm.c              |     3 +-
 drivers/gpu/drm/nouveau/nouveau_fbcon.c            |    30 +-
 drivers/gpu/drm/nouveau/nouveau_fbcon.h            |     2 -
 drivers/gpu/drm/nouveau/nouveau_gem.c              |     2 +-
 drivers/gpu/drm/nouveau/nouveau_vga.c              |     3 +-
 drivers/gpu/drm/nouveau/nv50_display.c             |   218 +-
 drivers/gpu/drm/nouveau/nvkm/engine/device/base.c  |    30 +-
 drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h     |     1 +
 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c     |   207 +
 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h     |    55 +
 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c     |   155 +
 drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c      |     6 +-
 drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c     |     1 +
 drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c    |     9 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild      |     1 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c     |     6 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c     |    47 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h     |    35 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk110.c     |    71 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h      |     2 +
 drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c   |    10 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c     |     3 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c      |     4 +-
 .../drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h   |   746 +-
 .../drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h   |   802 +-
 .../drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h   |  1006 +-
 .../gpu/drm/nouveau/nvkm/subdev/pmu/fuc/memx.fuc   |    30 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/secboot/Kbuild |     2 +
 drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr.h  |     1 +
 .../gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.c |   144 +
 .../gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.h |    49 +
 .../gpu/drm/nouveau/nvkm/subdev/secboot/acr_r375.c |    93 +-
 .../gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c    |     2 +-
 .../gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c    |    67 +
 drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h |     2 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild   |     2 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c   |    70 +-
 .../nvkm/subdev/therm/gf100.c}                     |    79 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h  |    35 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c  |     8 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c  |   136 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h  |    48 +
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gt215.c  |     2 +-
 drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h   |    21 +-
 .../gpu/drm/omapdrm/displays/connector-analog-tv.c |     2 +-
 drivers/gpu/drm/omapdrm/displays/connector-dvi.c   |     2 +-
 drivers/gpu/drm/omapdrm/displays/connector-hdmi.c  |     2 +-
 drivers/gpu/drm/omapdrm/displays/encoder-opa362.c  |     2 +-
 drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c  |     5 +-
 .../gpu/drm/omapdrm/displays/encoder-tpd12s015.c   |     2 +-
 drivers/gpu/drm/omapdrm/displays/panel-dpi.c       |     2 +-
 drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c    |   291 +-
 .../omapdrm/displays/panel-lgphilips-lb035q02.c    |     2 +-
 .../drm/omapdrm/displays/panel-nec-nl8048hl11.c    |     2 +-
 .../drm/omapdrm/displays/panel-sharp-ls037v7dw01.c |     2 +-
 .../drm/omapdrm/displays/panel-tpo-td028ttec1.c    |    13 +-
 drivers/gpu/drm/omapdrm/dss/base.c                 |    15 +
 drivers/gpu/drm/omapdrm/dss/core.c                 |    55 +-
 drivers/gpu/drm/omapdrm/dss/dispc.c                |    27 +-
 drivers/gpu/drm/omapdrm/dss/dispc.h                |     5 +-
 drivers/gpu/drm/omapdrm/dss/dispc_coefs.c          |     4 +-
 drivers/gpu/drm/omapdrm/dss/display.c              |    16 -
 drivers/gpu/drm/omapdrm/dss/dpi.c                  |     8 +-
 drivers/gpu/drm/omapdrm/dss/dsi.c                  |    14 +-
 drivers/gpu/drm/omapdrm/dss/dss-of.c               |     2 +-
 drivers/gpu/drm/omapdrm/dss/dss.c                  |    25 +-
 drivers/gpu/drm/omapdrm/dss/dss.h                  |    38 +-
 drivers/gpu/drm/omapdrm/dss/hdmi4.c                |    13 +-
 drivers/gpu/drm/omapdrm/dss/hdmi4_core.c           |     3 +-
 drivers/gpu/drm/omapdrm/dss/hdmi5.c                |    14 +-
 drivers/gpu/drm/omapdrm/dss/hdmi5_core.c           |     3 +-
 drivers/gpu/drm/omapdrm/dss/hdmi_phy.c             |     2 +-
 drivers/gpu/drm/omapdrm/dss/hdmi_pll.c             |     2 +-
 drivers/gpu/drm/omapdrm/dss/hdmi_wp.c              |     2 +-
 drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c    |     2 +-
 drivers/gpu/drm/omapdrm/dss/omapdss.h              |    12 +-
 drivers/gpu/drm/omapdrm/dss/output.c               |     2 +-
 drivers/gpu/drm/omapdrm/dss/pll.c                  |     2 +-
 drivers/gpu/drm/omapdrm/dss/sdi.c                  |     2 -
 drivers/gpu/drm/omapdrm/dss/venc.c                 |    18 +-
 drivers/gpu/drm/omapdrm/dss/video-pll.c            |    20 +-
 drivers/gpu/drm/omapdrm/omap_connector.c           |    10 +-
 drivers/gpu/drm/omapdrm/omap_connector.h           |    37 +
 drivers/gpu/drm/omapdrm/omap_crtc.c                |    41 +-
 drivers/gpu/drm/omapdrm/omap_crtc.h                |    43 +
 drivers/gpu/drm/omapdrm/omap_debugfs.c             |     4 +-
 drivers/gpu/drm/omapdrm/omap_dmm_priv.h            |    14 +-
 drivers/gpu/drm/omapdrm/omap_dmm_tiler.c           |    25 +-
 drivers/gpu/drm/omapdrm/omap_dmm_tiler.h           |     1 -
 drivers/gpu/drm/omapdrm/omap_drv.c                 |    45 +-
 drivers/gpu/drm/omapdrm/omap_drv.h                 |   140 +-
 drivers/gpu/drm/omapdrm/omap_encoder.c             |     4 +-
 drivers/gpu/drm/omapdrm/omap_encoder.h             |    33 +
 drivers/gpu/drm/omapdrm/omap_fb.c                  |     4 +-
 drivers/gpu/drm/omapdrm/omap_fb.h                  |    46 +
 drivers/gpu/drm/omapdrm/omap_fbdev.c               |     7 +-
 drivers/gpu/drm/omapdrm/omap_fbdev.h               |    39 +
 drivers/gpu/drm/omapdrm/omap_gem.c                 |    11 +-
 drivers/gpu/drm/omapdrm/omap_gem.h                 |    99 +
 drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c          |     4 +-
 drivers/gpu/drm/omapdrm/omap_irq.c                 |     4 +-
 drivers/gpu/drm/omapdrm/omap_irq.h                 |    39 +
 drivers/gpu/drm/omapdrm/omap_plane.c               |     4 +-
 drivers/gpu/drm/omapdrm/omap_plane.h               |    37 +
 drivers/gpu/drm/omapdrm/tcm-sita.c                 |     4 +-
 drivers/gpu/drm/omapdrm/tcm-sita.h                 |     4 +-
 drivers/gpu/drm/omapdrm/tcm.h                      |     2 -
 drivers/gpu/drm/panel/Kconfig                      |     8 +
 drivers/gpu/drm/panel/Makefile                     |     1 +
 drivers/gpu/drm/panel/panel-ilitek-ili9322.c       |   962 +
 drivers/gpu/drm/panel/panel-lvds.c                 |    30 +
 .../gpu/drm/panel/panel-raspberrypi-touchscreen.c  |     2 +-
 drivers/gpu/drm/panel/panel-simple.c               |    62 +
 drivers/gpu/drm/pl111/pl111_drm.h                  |     1 -
 drivers/gpu/drm/pl111/pl111_drv.c                  |    16 +-
 drivers/gpu/drm/qxl/qxl_display.c                  |    63 +-
 drivers/gpu/drm/qxl/qxl_drv.h                      |     2 +
 drivers/gpu/drm/qxl/qxl_ioctl.c                    |     4 +-
 drivers/gpu/drm/qxl/qxl_object.c                   |     6 +-
 drivers/gpu/drm/qxl/qxl_release.c                  |     6 +-
 drivers/gpu/drm/qxl/qxl_ttm.c                      |    21 +-
 drivers/gpu/drm/r128/r128_state.c                  |    23 +-
 drivers/gpu/drm/radeon/cik.c                       |   145 +-
 drivers/gpu/drm/radeon/cik_reg.h                   |     2 -
 drivers/gpu/drm/radeon/evergreen.c                 |     1 +
 drivers/gpu/drm/radeon/ni.c                        |     1 +
 drivers/gpu/drm/radeon/radeon.h                    |     6 -
 drivers/gpu/drm/radeon/radeon_connectors.c         |   106 +-
 drivers/gpu/drm/radeon/radeon_device.c             |    36 +-
 drivers/gpu/drm/radeon/radeon_display.c            |     9 +-
 drivers/gpu/drm/radeon/radeon_dp_mst.c             |     3 +-
 drivers/gpu/drm/radeon/radeon_fb.c                 |    22 -
 drivers/gpu/drm/radeon/radeon_gem.c                |     5 +-
 drivers/gpu/drm/radeon/radeon_kms.c                |     5 +-
 drivers/gpu/drm/radeon/radeon_mn.c                 |     3 +-
 drivers/gpu/drm/radeon/radeon_mode.h               |     3 -
 drivers/gpu/drm/radeon/radeon_object.c             |    16 +-
 drivers/gpu/drm/radeon/radeon_pm.c                 |     6 +-
 drivers/gpu/drm/radeon/radeon_ttm.c                |    46 +-
 drivers/gpu/drm/radeon/radeon_uvd.c                |     2 +-
 drivers/gpu/drm/radeon/radeon_vm.c                 |     3 +-
 drivers/gpu/drm/radeon/si_dpm.c                    |     5 +
 drivers/gpu/drm/rcar-du/rcar_du_crtc.c             |    38 +-
 drivers/gpu/drm/rcar-du/rcar_du_drv.c              |    62 +-
 drivers/gpu/drm/rcar-du/rcar_du_drv.h              |     1 +
 drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c          |     4 -
 drivers/gpu/drm/rcar-du/rcar_du_plane.c            |    75 +-
 drivers/gpu/drm/rcar-du/rcar_du_plane.h            |     4 +
 drivers/gpu/drm/rcar-du/rcar_du_vsp.c              |    64 +-
 drivers/gpu/drm/rockchip/analogix_dp-rockchip.c    |    28 +-
 drivers/gpu/drm/rockchip/cdn-dp-core.c             |     4 +-
 drivers/gpu/drm/rockchip/inno_hdmi.c               |     1 +
 drivers/gpu/drm/rockchip/rockchip_drm_drv.c        |     9 +-
 drivers/gpu/drm/rockchip/rockchip_drm_fb.c         |     9 +-
 drivers/gpu/drm/rockchip/rockchip_drm_vop.c        |     6 +-
 drivers/gpu/drm/rockchip/rockchip_lvds.c           |     1 +
 drivers/gpu/drm/rockchip/rockchip_vop_reg.c        |    30 +
 drivers/gpu/drm/rockchip/rockchip_vop_reg.h        |     6 +
 drivers/gpu/drm/scheduler/Makefile                 |    26 +
 .../gpu/drm/{amd => }/scheduler/gpu_scheduler.c    |   396 +-
 drivers/gpu/drm/{amd => }/scheduler/sched_fence.c  |   122 +-
 drivers/gpu/drm/selftests/test-drm_mm.c            |     5 +
 drivers/gpu/drm/sti/Kconfig                        |     1 +
 drivers/gpu/drm/sti/sti_awg_utils.c                |     2 +-
 drivers/gpu/drm/sti/sti_awg_utils.h                |     2 +-
 drivers/gpu/drm/sti/sti_compositor.c               |     2 +-
 drivers/gpu/drm/sti/sti_compositor.h               |     2 +-
 drivers/gpu/drm/sti/sti_crtc.c                     |     2 +-
 drivers/gpu/drm/sti/sti_crtc.h                     |     2 +-
 drivers/gpu/drm/sti/sti_cursor.c                   |     2 +-
 drivers/gpu/drm/sti/sti_cursor.h                   |     2 +-
 drivers/gpu/drm/sti/sti_drv.c                      |    29 +-
 drivers/gpu/drm/sti/sti_drv.h                      |     3 +-
 drivers/gpu/drm/sti/sti_dvo.c                      |     2 +-
 drivers/gpu/drm/sti/sti_gdp.c                      |     2 +-
 drivers/gpu/drm/sti/sti_gdp.h                      |     2 +-
 drivers/gpu/drm/sti/sti_hda.c                      |     2 +-
 drivers/gpu/drm/sti/sti_hdmi.c                     |    12 +-
 drivers/gpu/drm/sti/sti_hdmi.h                     |     2 +-
 drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.c         |     2 +-
 drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.h         |     2 +-
 drivers/gpu/drm/sti/sti_hqvdp.c                    |     2 +-
 drivers/gpu/drm/sti/sti_hqvdp_lut.h                |     2 +-
 drivers/gpu/drm/sti/sti_mixer.c                    |     2 +-
 drivers/gpu/drm/sti/sti_mixer.h                    |     2 +-
 drivers/gpu/drm/sti/sti_plane.c                    |     2 +-
 drivers/gpu/drm/sti/sti_plane.h                    |     2 +-
 drivers/gpu/drm/sti/sti_tvout.c                    |     2 +-
 drivers/gpu/drm/sti/sti_vid.c                      |     2 +-
 drivers/gpu/drm/sti/sti_vid.h                      |     2 +-
 drivers/gpu/drm/sti/sti_vtg.c                      |    40 +-
 drivers/gpu/drm/sti/sti_vtg.h                      |     2 +-
 drivers/gpu/drm/stm/drv.c                          |    41 +-
 drivers/gpu/drm/stm/dw_mipi_dsi-stm.c              |    14 +-
 drivers/gpu/drm/stm/ltdc.c                         |    75 +-
 drivers/gpu/drm/stm/ltdc.h                         |     4 +-
 drivers/gpu/drm/sun4i/Makefile                     |     5 +-
 drivers/gpu/drm/sun4i/sun4i_crtc.c                 |     4 +
 drivers/gpu/drm/sun4i/sun4i_dotclock.c             |    15 +-
 drivers/gpu/drm/sun4i/sun4i_drv.c                  |    26 +-
 drivers/gpu/drm/sun4i/sun4i_drv.h                  |     2 -
 drivers/gpu/drm/sun4i/sun4i_framebuffer.c          |    18 +-
 drivers/gpu/drm/sun4i/sun4i_framebuffer.h          |     2 +-
 drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c             |     6 +-
 drivers/gpu/drm/sun4i/sun4i_lvds.c                 |   177 +
 drivers/gpu/drm/sun4i/sun4i_lvds.h                 |    12 +
 drivers/gpu/drm/sun4i/sun4i_rgb.c                  |     2 +
 drivers/gpu/drm/sun4i/sun4i_tcon.c                 |   264 +-
 drivers/gpu/drm/sun4i/sun4i_tcon.h                 |    34 +
 drivers/gpu/drm/sun4i/sun8i_csc.c                  |    93 +
 drivers/gpu/drm/sun4i/sun8i_csc.h                  |    36 +
 drivers/gpu/drm/sun4i/sun8i_layer.c                |   134 -
 drivers/gpu/drm/sun4i/sun8i_layer.h                |    36 -
 drivers/gpu/drm/sun4i/sun8i_mixer.c                |   517 +-
 drivers/gpu/drm/sun4i/sun8i_mixer.h                |   126 +-
 drivers/gpu/drm/sun4i/sun8i_ui_layer.c             |   349 +
 drivers/gpu/drm/sun4i/sun8i_ui_layer.h             |    63 +
 drivers/gpu/drm/sun4i/sun8i_ui_scaler.c            |   172 +
 drivers/gpu/drm/sun4i/sun8i_ui_scaler.h            |    49 +
 drivers/gpu/drm/sun4i/sun8i_vi_layer.c             |   390 +
 drivers/gpu/drm/sun4i/sun8i_vi_layer.h             |    51 +
 drivers/gpu/drm/sun4i/sun8i_vi_scaler.c            |   971 +
 drivers/gpu/drm/sun4i/sun8i_vi_scaler.h            |    58 +
 drivers/gpu/drm/tegra/Makefile                     |     2 +
 drivers/gpu/drm/tegra/dc.c                         |  1942 +-
 drivers/gpu/drm/tegra/dc.h                         |   289 +-
 drivers/gpu/drm/tegra/dpaux.c                      |   120 +-
 drivers/gpu/drm/tegra/drm.c                        |   178 +-
 drivers/gpu/drm/tegra/drm.h                        |    30 +-
 drivers/gpu/drm/tegra/dsi.c                        |   229 +-
 drivers/gpu/drm/tegra/fb.c                         |    50 +-
 drivers/gpu/drm/tegra/gem.c                        |    15 +-
 drivers/gpu/drm/tegra/hdmi.c                       |   504 +-
 drivers/gpu/drm/tegra/hub.c                        |   806 +
 drivers/gpu/drm/tegra/hub.h                        |    81 +
 drivers/gpu/drm/tegra/output.c                     |    25 +-
 drivers/gpu/drm/tegra/plane.c                      |   386 +
 drivers/gpu/drm/tegra/plane.h                      |    70 +
 drivers/gpu/drm/tegra/sor.c                        |  1117 +-
 drivers/gpu/drm/tegra/sor.h                        |    16 +
 drivers/gpu/drm/tegra/vic.c                        |    20 +-
 drivers/gpu/drm/tilcdc/tilcdc_crtc.c               |    51 +-
 drivers/gpu/drm/tilcdc/tilcdc_drv.c                |    28 +-
 drivers/gpu/drm/tilcdc/tilcdc_drv.h                |     2 -
 drivers/gpu/drm/tilcdc/tilcdc_regs.h               |     2 +-
 drivers/gpu/drm/tinydrm/Kconfig                    |    20 +
 drivers/gpu/drm/tinydrm/Makefile                   |     2 +
 drivers/gpu/drm/tinydrm/core/tinydrm-core.c        |   104 +-
 drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c     |     4 +-
 drivers/gpu/drm/tinydrm/ili9225.c                  |   469 +
 drivers/gpu/drm/tinydrm/mi0283qt.c                 |    10 +-
 drivers/gpu/drm/tinydrm/mipi-dbi.c                 |    32 +-
 drivers/gpu/drm/tinydrm/st7586.c                   |     3 +-
 drivers/gpu/drm/tinydrm/st7735r.c                  |   215 +
 drivers/gpu/drm/ttm/ttm_agp_backend.c              |     4 +-
 drivers/gpu/drm/ttm/ttm_bo.c                       |   341 +-
 drivers/gpu/drm/ttm/ttm_bo_util.c                  |    23 +-
 drivers/gpu/drm/ttm/ttm_bo_vm.c                    |    32 +-
 drivers/gpu/drm/ttm/ttm_execbuf_util.c             |     8 +-
 drivers/gpu/drm/ttm/ttm_memory.c                   |    37 +-
 drivers/gpu/drm/ttm/ttm_object.c                   |    13 +-
 drivers/gpu/drm/ttm/ttm_page_alloc.c               |    36 +-
 drivers/gpu/drm/ttm/ttm_page_alloc_dma.c           |    99 +-
 drivers/gpu/drm/ttm/ttm_tt.c                       |     5 +-
 drivers/gpu/drm/tve200/tve200_drm.h                |     1 -
 drivers/gpu/drm/tve200/tve200_drv.c                |    16 +-
 drivers/gpu/drm/udl/udl_fb.c                       |     9 +-
 drivers/gpu/drm/vc4/vc4_drv.c                      |    13 +-
 drivers/gpu/drm/vc4/vc4_drv.h                      |     2 -
 drivers/gpu/drm/vc4/vc4_hdmi.c                     |    16 +-
 drivers/gpu/drm/vc4/vc4_kms.c                      |    18 +-
 drivers/gpu/drm/vc4/vc4_plane.c                    |    38 +-
 drivers/gpu/drm/via/via_drv.h                      |     4 +-
 drivers/gpu/drm/via/via_irq.c                      |    27 +-
 drivers/gpu/drm/virtio/virtgpu_display.c           |    12 +
 drivers/gpu/drm/virtio/virtgpu_ioctl.c             |    30 +-
 drivers/gpu/drm/virtio/virtgpu_kms.c               |     6 +-
 drivers/gpu/drm/virtio/virtgpu_object.c            |     6 +-
 drivers/gpu/drm/virtio/virtgpu_ttm.c               |    13 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_binding.c            |     6 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c             |    17 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_context.c            |     6 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c            |     9 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c             |    21 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_drv.c                |    13 +
 drivers/gpu/drm/vmwgfx/vmwgfx_drv.h                |     3 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c            |     9 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_fence.c              |     6 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c              |     2 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_kms.c                |    79 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_kms.h                |    13 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_mob.c                |    13 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_resource.c           |     6 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c               |     5 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_shader.c             |    21 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c    |     6 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_so.c                 |     6 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c               |     5 +-
 drivers/gpu/drm/vmwgfx/vmwgfx_surface.c            |    12 +-
 drivers/gpu/drm/zte/zx_drm_drv.c                   |    48 +-
 drivers/gpu/drm/zte/zx_hdmi.c                      |     1 +
 drivers/gpu/drm/zte/zx_plane.c                     |    15 +-
 drivers/gpu/host1x/bus.c                           |    11 +-
 drivers/gpu/host1x/dev.c                           |    36 +-
 drivers/gpu/host1x/dev.h                           |     1 +
 drivers/gpu/ipu-v3/Kconfig                         |     4 +-
 drivers/gpu/ipu-v3/ipu-common.c                    |     4 +-
 drivers/gpu/ipu-v3/ipu-cpmem.c                     |     3 +
 drivers/gpu/ipu-v3/ipu-csi.c                       |     2 +
 drivers/gpu/ipu-v3/ipu-ic.c                        |     1 +
 drivers/gpu/ipu-v3/ipu-pre.c                       |    32 +-
 drivers/gpu/ipu-v3/ipu-prg.c                       |    99 +-
 drivers/gpu/ipu-v3/ipu-prv.h                       |     4 +-
 drivers/gpu/vga/vgaarb.c                           |     4 +-
 drivers/hid/Kconfig                                |    12 +
 drivers/hid/Makefile                               |     3 +-
 drivers/hid/hid-asus.c                             |    41 +-
 drivers/hid/hid-core.c                             |   932 +-
 drivers/hid/hid-debug.c                            |     6 +-
 drivers/hid/hid-elecom.c                           |    78 +-
 drivers/hid/hid-elo.c                              |     6 +
 drivers/hid/hid-generic.c                          |    68 +-
 drivers/hid/hid-ids.h                              |    10 +-
 drivers/hid/hid-jabra.c                            |    58 +
 drivers/hid/hid-multitouch.c                       |   137 +-
 drivers/hid/hid-quirks.c                           |  1279 +
 drivers/hid/hid-rmi.c                              |     1 +
 drivers/hid/hid-roccat-kovaplus.c                  |     2 +
 drivers/hid/hid-roccat.c                           |     6 +-
 drivers/hid/hid-sensor-custom.c                    |     6 +-
 drivers/hid/hid-sony.c                             |    93 +-
 drivers/hid/hidraw.c                               |     6 +-
 drivers/hid/i2c-hid/i2c-hid.c                      |    18 +-
 drivers/hid/intel-ish-hid/ipc/hw-ish.h             |     1 +
 drivers/hid/intel-ish-hid/ipc/pci-ish.c            |     1 +
 drivers/hid/uhid.c                                 |     4 +-
 drivers/hid/usbhid/Makefile                        |     2 +-
 drivers/hid/usbhid/hid-core.c                      |    11 +-
 drivers/hid/usbhid/hid-quirks.c                    |   402 -
 drivers/hid/usbhid/hiddev.c                        |     6 +-
 drivers/hid/wacom_sys.c                            |   134 +-
 drivers/hid/wacom_wac.c                            |    67 +-
 drivers/hid/wacom_wac.h                            |     6 +
 drivers/hsi/clients/cmt_speech.c                   |    12 +-
 drivers/hv/hv.c                                    |     3 -
 drivers/hv/hv_utils_transport.c                    |     6 +-
 drivers/hv/ring_buffer.c                           |    75 +-
 drivers/hv/vmbus_drv.c                             |    40 +-
 drivers/hwmon/Kconfig                              |    26 +-
 drivers/hwmon/Makefile                             |     1 +
 drivers/hwmon/aspeed-pwm-tacho.c                   |    22 +
 drivers/hwmon/coretemp.c                           |     9 +-
 drivers/hwmon/dell-smm-hwmon.c                     |    54 +-
 drivers/hwmon/hih6130.c                            |     2 +-
 drivers/hwmon/hwmon-vid.c                          |     2 +-
 drivers/hwmon/hwmon.c                              |    12 +-
 drivers/hwmon/iio_hwmon.c                          |     3 +-
 drivers/hwmon/ina2xx.c                             |    90 +-
 drivers/hwmon/k10temp.c                            |     8 +-
 drivers/hwmon/k8temp.c                             |     2 +-
 drivers/hwmon/lm75.c                               |     2 +-
 drivers/hwmon/pmbus/Kconfig                        |     1 +
 drivers/hwmon/pmbus/ibm-cffps.c                    |   288 +-
 drivers/hwmon/pmbus/ir35221.c                      |   189 -
 drivers/hwmon/pmbus/lm25066.c                      |    67 +-
 drivers/hwmon/pmbus/max31785.c                     |   294 +-
 drivers/hwmon/pmbus/pmbus.h                        |    41 +
 drivers/hwmon/pmbus/pmbus_core.c                   |   273 +-
 drivers/hwmon/sht15.c                              |     1 +
 drivers/hwmon/sht21.c                              |     2 +-
 drivers/hwmon/sht3x.c                              |     7 +
 drivers/hwmon/w83773g.c                            |   329 +
 .../coresight/coresight-dynamic-replicator.c       |     4 +-
 drivers/hwtracing/coresight/coresight-etb10.c      |     1 -
 drivers/hwtracing/coresight/coresight-funnel.c     |     4 +-
 drivers/hwtracing/coresight/coresight-tpiu.c       |    17 +-
 drivers/hwtracing/coresight/coresight.c            |    25 +-
 drivers/hwtracing/coresight/of_coresight.c         |    15 +-
 drivers/i2c/algos/i2c-algo-bit.c                   |     5 +
 drivers/i2c/busses/Kconfig                         |    10 +
 drivers/i2c/busses/i2c-acorn.c                     |     8 +-
 drivers/i2c/busses/i2c-bcm2835.c                   |    21 +-
 drivers/i2c/busses/i2c-davinci.c                   |    37 +-
 drivers/i2c/busses/i2c-designware-common.c         |    20 +-
 drivers/i2c/busses/i2c-designware-core.h           |     4 +-
 drivers/i2c/busses/i2c-designware-master.c         |    59 +-
 drivers/i2c/busses/i2c-designware-platdrv.c        |    68 +-
 drivers/i2c/busses/i2c-designware-slave.c          |    64 -
 drivers/i2c/busses/i2c-exynos5.c                   |     2 +-
 drivers/i2c/busses/i2c-gpio.c                      |   122 +-
 drivers/i2c/busses/i2c-i801.c                      |     1 +
 drivers/i2c/busses/i2c-imx-lpi2c.c                 |    68 +-
 drivers/i2c/busses/i2c-imx.c                       |    20 +-
 drivers/i2c/busses/i2c-ismt.c                      |    42 +-
 drivers/i2c/busses/i2c-meson.c                     |    34 +-
 drivers/i2c/busses/i2c-mpc.c                       |    57 +-
 drivers/i2c/busses/i2c-mt65xx.c                    |    40 +-
 drivers/i2c/busses/i2c-mv64xxx.c                   |    20 +-
 drivers/i2c/busses/i2c-mxs.c                       |     4 +-
 drivers/i2c/busses/i2c-octeon-core.c               |     1 +
 drivers/i2c/busses/i2c-octeon-core.h               |     2 +-
 drivers/i2c/busses/i2c-pxa-pci.c                   |     2 +-
 drivers/i2c/busses/i2c-pxa.c                       |     2 +-
 drivers/i2c/busses/i2c-rcar.c                      |    62 +-
 drivers/i2c/busses/i2c-rk3x.c                      |    17 +-
 drivers/i2c/busses/i2c-sh_mobile.c                 |   191 +-
 drivers/i2c/busses/i2c-sirf.c                      |     4 +-
 drivers/i2c/busses/i2c-stm32f7.c                   |     5 +
 drivers/i2c/i2c-core-base.c                        |   212 +-
 drivers/i2c/i2c-core-smbus.c                       |    45 +-
 drivers/i2c/i2c-dev.c                              |     2 +
 drivers/i2c/muxes/Kconfig                          |     6 +-
 drivers/i2c/muxes/i2c-mux-pca954x.c                |    38 +-
 drivers/i2c/muxes/i2c-mux-reg.c                    |     3 +
 drivers/ide/ide-probe.c                            |     2 +-
 drivers/ide/ide-taskfile.c                         |     1 -
 drivers/ide/sl82c105.c                             |     5 +-
 drivers/iio/accel/bmc150-accel-i2c.c               |    12 +
 drivers/iio/accel/da280.c                          |    31 +-
 drivers/iio/accel/kxsd9-i2c.c                      |     3 +
 drivers/iio/accel/mma8452.c                        |    20 +-
 drivers/iio/accel/st_accel_core.c                  |     2 +-
 drivers/iio/adc/Kconfig                            |    40 +
 drivers/iio/adc/Makefile                           |     3 +
 drivers/iio/adc/aspeed_adc.c                       |    28 +-
 drivers/iio/adc/at91-sama5d2_adc.c                 |   456 +-
 drivers/iio/adc/at91_adc.c                         |     4 +-
 drivers/iio/adc/axp288_adc.c                       |    20 +-
 drivers/iio/adc/hx711.c                            |   134 +-
 drivers/iio/adc/ina2xx-adc.c                       |   317 +-
 drivers/iio/adc/meson_saradc.c                     |    64 +-
 drivers/iio/adc/qcom-vadc-common.c                 |     4 +
 drivers/iio/adc/sd_adc_modulator.c                 |    68 +
 drivers/iio/adc/stm32-adc-core.c                   |    14 +-
 drivers/iio/adc/stm32-adc-core.h                   |    14 +-
 drivers/iio/adc/stm32-adc.c                        |   206 +-
 drivers/iio/adc/stm32-dfsdm-adc.c                  |  1208 +
 drivers/iio/adc/stm32-dfsdm-core.c                 |   310 +
 drivers/iio/adc/stm32-dfsdm.h                      |   310 +
 drivers/iio/adc/ti_am335x_adc.c                    |     2 +-
 drivers/iio/buffer/Kconfig                         |    10 +
 drivers/iio/buffer/Makefile                        |     1 +
 drivers/iio/buffer/industrialio-buffer-cb.c        |    11 +
 drivers/iio/buffer/industrialio-buffer-dma.c       |     4 +-
 drivers/iio/buffer/industrialio-hw-consumer.c      |   247 +
 drivers/iio/chemical/ccs811.c                      |    16 +-
 .../iio/common/cros_ec_sensors/cros_ec_sensors.c   |     2 -
 drivers/iio/common/ssp_sensors/ssp.h               |     2 +-
 drivers/iio/common/ssp_sensors/ssp_dev.c           |     2 +-
 drivers/iio/common/ssp_sensors/ssp_spi.c           |     5 +-
 drivers/iio/counter/stm32-lptimer-cnt.c            |     2 +-
 drivers/iio/dac/mcp4725.c                          |     2 +-
 drivers/iio/dac/stm32-dac-core.c                   |    14 +-
 drivers/iio/dac/stm32-dac-core.h                   |    15 +-
 drivers/iio/dac/stm32-dac.c                        |    15 +-
 drivers/iio/dummy/iio_dummy_evgen.c                |     2 +-
 drivers/iio/gyro/adis16136.c                       |    15 +-
 drivers/iio/gyro/bmg160_core.c                     |     1 -
 drivers/iio/health/max30102.c                      |   308 +-
 drivers/iio/humidity/hts221.h                      |     3 +-
 drivers/iio/humidity/hts221_core.c                 |    18 +-
 drivers/iio/humidity/hts221_i2c.c                  |    18 +-
 drivers/iio/humidity/hts221_spi.c                  |    18 +-
 drivers/iio/iio_core.h                             |     2 +-
 drivers/iio/imu/adis16480.c                        |    28 +-
 drivers/iio/imu/adis_trigger.c                     |     7 +-
 drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c         |     3 +-
 drivers/iio/imu/st_lsm6dsx/Kconfig                 |     2 +
 drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h            |    39 +-
 drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c     |   107 +-
 drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c       |   115 +-
 drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c        |    55 +-
 drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c        |    70 +-
 drivers/iio/industrialio-buffer.c                  |    25 +-
 drivers/iio/industrialio-core.c                    |     1 +
 drivers/iio/industrialio-event.c                   |    10 +-
 drivers/iio/inkern.c                               |    17 +-
 drivers/iio/light/Kconfig                          |    34 +
 drivers/iio/light/Makefile                         |     4 +
 drivers/iio/light/cros_ec_light_prox.c             |     2 -
 drivers/iio/light/st_uvis25.h                      |    37 +
 drivers/iio/light/st_uvis25_core.c                 |   359 +
 drivers/iio/light/st_uvis25_i2c.c                  |    69 +
 drivers/iio/light/st_uvis25_spi.c                  |    68 +
 drivers/iio/light/zopt2201.c                       |   568 +
 drivers/iio/magnetometer/ak8975.c                  |     1 +
 drivers/iio/pressure/bmp280-core.c                 |   203 +-
 drivers/iio/pressure/st_pressure_core.c            |     2 +-
 drivers/iio/proximity/Kconfig                      |     2 +
 drivers/iio/proximity/sx9500.c                     |     1 +
 drivers/iio/trigger/stm32-lptimer-trigger.c        |     3 +-
 drivers/iio/trigger/stm32-timer-trigger.c          |     2 +-
 drivers/infiniband/Kconfig                         |     8 +-
 drivers/infiniband/core/Makefile                   |     2 +-
 drivers/infiniband/core/addr.c                     |   103 +-
 drivers/infiniband/core/cache.c                    |    23 +-
 drivers/infiniband/core/cm.c                       |   227 +-
 drivers/infiniband/core/cma.c                      |   267 +-
 drivers/infiniband/core/cma_configfs.c             |     2 +-
 drivers/infiniband/core/core_priv.h                |    57 +-
 drivers/infiniband/core/cq.c                       |    46 +-
 drivers/infiniband/core/device.c                   |    47 +-
 drivers/infiniband/core/fmr_pool.c                 |    12 +-
 drivers/infiniband/core/iwpm_util.c                |     1 +
 drivers/infiniband/core/mad.c                      |     1 -
 drivers/infiniband/core/netlink.c                  |    10 +-
 drivers/infiniband/core/nldev.c                    |   396 +-
 drivers/infiniband/core/rdma_core.c                |    38 +-
 drivers/infiniband/core/restrack.c                 |   173 +
 drivers/infiniband/core/roce_gid_mgmt.c            |    13 +-
 drivers/infiniband/core/sa_query.c                 |    25 +-
 drivers/infiniband/core/security.c                 |    10 +-
 drivers/infiniband/core/sysfs.c                    |     1 -
 drivers/infiniband/core/ucm.c                      |    79 +-
 drivers/infiniband/core/ucma.c                     |    98 +-
 drivers/infiniband/core/umem.c                     |     2 +-
 drivers/infiniband/core/user_mad.c                 |   129 +-
 drivers/infiniband/core/uverbs_cmd.c               |    58 +-
 drivers/infiniband/core/uverbs_ioctl.c             |    22 +-
 drivers/infiniband/core/uverbs_ioctl_merge.c       |    18 +-
 drivers/infiniband/core/uverbs_main.c              |   134 +-
 drivers/infiniband/core/uverbs_std_types.c         |    17 +-
 drivers/infiniband/core/verbs.c                    |   313 +-
 drivers/infiniband/hw/bnxt_re/bnxt_re.h            |    45 +-
 drivers/infiniband/hw/bnxt_re/hw_counters.c        |   145 +-
 drivers/infiniband/hw/bnxt_re/hw_counters.h        |    39 +
 drivers/infiniband/hw/bnxt_re/ib_verbs.c           |   477 +-
 drivers/infiniband/hw/bnxt_re/ib_verbs.h           |    25 +
 drivers/infiniband/hw/bnxt_re/main.c               |   275 +-
 drivers/infiniband/hw/bnxt_re/qplib_fp.c           |   579 +-
 drivers/infiniband/hw/bnxt_re/qplib_fp.h           |    94 +-
 drivers/infiniband/hw/bnxt_re/qplib_rcfw.c         |    14 +-
 drivers/infiniband/hw/bnxt_re/qplib_rcfw.h         |     8 +-
 drivers/infiniband/hw/bnxt_re/qplib_res.c          |    27 +-
 drivers/infiniband/hw/bnxt_re/qplib_sp.c           |   148 +-
 drivers/infiniband/hw/bnxt_re/qplib_sp.h           |    91 +-
 drivers/infiniband/hw/bnxt_re/roce_hsi.h           |   152 +-
 drivers/infiniband/hw/cxgb4/cm.c                   |    27 +-
 drivers/infiniband/hw/cxgb4/device.c               |    36 +-
 drivers/infiniband/hw/cxgb4/ev.c                   |     2 +-
 drivers/infiniband/hw/cxgb4/iw_cxgb4.h             |     4 +-
 drivers/infiniband/hw/cxgb4/qp.c                   |     6 +-
 drivers/infiniband/hw/cxgb4/t4.h                   |     4 +-
 drivers/infiniband/hw/hfi1/chip.c                  |   169 +-
 drivers/infiniband/hw/hfi1/chip.h                  |     6 +-
 drivers/infiniband/hw/hfi1/debugfs.c               |     9 +-
 drivers/infiniband/hw/hfi1/driver.c                |    67 +-
 drivers/infiniband/hw/hfi1/file_ops.c              |    30 +-
 drivers/infiniband/hw/hfi1/firmware.c              |    64 +-
 drivers/infiniband/hw/hfi1/hfi.h                   |    51 +-
 drivers/infiniband/hw/hfi1/init.c                  |    33 +-
 drivers/infiniband/hw/hfi1/iowait.h                |     9 +
 drivers/infiniband/hw/hfi1/mad.c                   |   133 +-
 drivers/infiniband/hw/hfi1/mad.h                   |    47 +-
 drivers/infiniband/hw/hfi1/mmu_rb.c                |     1 +
 drivers/infiniband/hw/hfi1/pcie.c                  |    23 +-
 drivers/infiniband/hw/hfi1/pio.c                   |    15 +-
 drivers/infiniband/hw/hfi1/qp.c                    |    12 +-
 drivers/infiniband/hw/hfi1/qp.h                    |    13 +
 drivers/infiniband/hw/hfi1/rc.c                    |    63 +-
 drivers/infiniband/hw/hfi1/ruc.c                   |    48 +-
 drivers/infiniband/hw/hfi1/sdma.c                  |    17 +-
 drivers/infiniband/hw/hfi1/sdma.h                  |     1 +
 drivers/infiniband/hw/hfi1/trace.c                 |     8 +-
 drivers/infiniband/hw/hfi1/trace_ibhdrs.h          |    16 +-
 drivers/infiniband/hw/hfi1/trace_rx.h              |    30 +-
 drivers/infiniband/hw/hfi1/uc.c                    |    11 +-
 drivers/infiniband/hw/hfi1/ud.c                    |    41 +-
 drivers/infiniband/hw/hfi1/verbs.c                 |    16 +-
 drivers/infiniband/hw/hfi1/verbs.h                 |    24 +-
 drivers/infiniband/hw/hfi1/verbs_txreq.h           |     7 +
 drivers/infiniband/hw/hns/Makefile                 |     2 +-
 drivers/infiniband/hw/hns/hns_roce_cmd.c           |     1 +
 drivers/infiniband/hw/hns/hns_roce_cmd.h           |    10 +
 drivers/infiniband/hw/hns/hns_roce_common.h        |    17 +-
 drivers/infiniband/hw/hns/hns_roce_cq.c            |    19 +-
 drivers/infiniband/hw/hns/hns_roce_device.h        |   113 +-
 drivers/infiniband/hw/hns/hns_roce_eq.c            |   759 -
 drivers/infiniband/hw/hns/hns_roce_eq.h            |   134 -
 drivers/infiniband/hw/hns/hns_roce_hw_v1.c         |   818 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v1.h         |   302 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c         |  1871 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h         |   518 +-
 drivers/infiniband/hw/hns/hns_roce_main.c          |    19 +-
 drivers/infiniband/hw/hns/hns_roce_qp.c            |    90 +-
 drivers/infiniband/hw/i40iw/Kconfig                |     1 -
 drivers/infiniband/hw/i40iw/i40iw.h                |     3 +
 drivers/infiniband/hw/i40iw/i40iw_cm.c             |    68 +-
 drivers/infiniband/hw/i40iw/i40iw_cm.h             |     8 +-
 drivers/infiniband/hw/i40iw/i40iw_ctrl.c           |    25 +-
 drivers/infiniband/hw/i40iw/i40iw_d.h              |     1 +
 drivers/infiniband/hw/i40iw/i40iw_hw.c             |     3 +-
 drivers/infiniband/hw/i40iw/i40iw_main.c           |    13 +-
 drivers/infiniband/hw/i40iw/i40iw_puda.c           |     5 +-
 drivers/infiniband/hw/i40iw/i40iw_puda.h           |     1 +
 drivers/infiniband/hw/i40iw/i40iw_uk.c             |    18 +-
 drivers/infiniband/hw/i40iw/i40iw_user.h           |     3 +-
 drivers/infiniband/hw/i40iw/i40iw_utils.c          |    50 +-
 drivers/infiniband/hw/i40iw/i40iw_verbs.c          |     5 +-
 drivers/infiniband/hw/mlx4/cq.c                    |     8 +-
 drivers/infiniband/hw/mlx4/main.c                  |    30 +-
 drivers/infiniband/hw/mlx4/qp.c                    |    20 +-
 drivers/infiniband/hw/mlx5/cong.c                  |    83 +-
 drivers/infiniband/hw/mlx5/cq.c                    |    12 +-
 drivers/infiniband/hw/mlx5/mad.c                   |    23 +-
 drivers/infiniband/hw/mlx5/main.c                  |  1425 +-
 drivers/infiniband/hw/mlx5/mlx5_ib.h               |   112 +-
 drivers/infiniband/hw/mlx5/mr.c                    |    20 +-
 drivers/infiniband/hw/mlx5/odp.c                   |     9 -
 drivers/infiniband/hw/mlx5/qp.c                    |   466 +-
 drivers/infiniband/hw/mlx5/srq.c                   |    15 +-
 drivers/infiniband/hw/mthca/mthca_memfree.c        |     7 +-
 drivers/infiniband/hw/mthca/mthca_user.h           |   112 -
 drivers/infiniband/hw/nes/nes_cm.c                 |     2 +-
 drivers/infiniband/hw/nes/nes_cm.h                 |     3 +-
 drivers/infiniband/hw/ocrdma/ocrdma_hw.c           |    19 +-
 drivers/infiniband/hw/ocrdma/ocrdma_stats.c        |     8 +-
 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c        |    10 +-
 drivers/infiniband/hw/qedr/main.c                  |    64 +-
 drivers/infiniband/hw/qedr/qedr_hsi_rdma.h         |   125 +-
 drivers/infiniband/hw/qedr/qedr_iw_cm.c            |    19 +-
 drivers/infiniband/hw/qedr/verbs.c                 |    73 +-
 drivers/infiniband/hw/qib/qib.h                    |     8 +-
 drivers/infiniband/hw/qib/qib_driver.c             |    16 -
 drivers/infiniband/hw/qib/qib_eeprom.c             |     3 +-
 drivers/infiniband/hw/qib/qib_file_ops.c           |    90 +-
 drivers/infiniband/hw/qib/qib_init.c               |     2 +
 drivers/infiniband/hw/qib/qib_keys.c               |   235 -
 drivers/infiniband/hw/qib/qib_rc.c                 |    12 +-
 drivers/infiniband/hw/qib/qib_ruc.c                |     1 -
 drivers/infiniband/hw/qib/qib_uc.c                 |     5 +-
 drivers/infiniband/hw/qib/qib_ud.c                 |     5 +-
 drivers/infiniband/hw/qib/qib_verbs.c              |     2 -
 drivers/infiniband/hw/usnic/usnic_ib_sysfs.c       |     1 -
 drivers/infiniband/hw/usnic/usnic_ib_verbs.c       |     1 -
 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h          |     4 +-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c       |    17 +-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c     |    21 +-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c       |    15 +-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c       |    11 +-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c      |     4 +-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c    |     4 +-
 drivers/infiniband/sw/rdmavt/cq.c                  |    15 +-
 drivers/infiniband/sw/rdmavt/mcast.c               |     4 +-
 drivers/infiniband/sw/rdmavt/mr.c                  |    12 +-
 drivers/infiniband/sw/rdmavt/qp.c                  |    28 +-
 drivers/infiniband/sw/rdmavt/srq.c                 |    16 +-
 drivers/infiniband/sw/rdmavt/trace.h               |     4 +-
 drivers/infiniband/sw/rdmavt/trace_qp.h            |    42 +
 drivers/infiniband/sw/rdmavt/vt.c                  |    10 +-
 drivers/infiniband/sw/rdmavt/vt.h                  |     6 +-
 drivers/infiniband/sw/rxe/Kconfig                  |     4 +-
 drivers/infiniband/sw/rxe/rxe.c                    |     6 -
 drivers/infiniband/sw/rxe/rxe.h                    |     6 +-
 drivers/infiniband/sw/rxe/rxe_av.c                 |    14 +-
 drivers/infiniband/sw/rxe/rxe_loc.h                |    11 +-
 drivers/infiniband/sw/rxe/rxe_net.c                |    18 +-
 drivers/infiniband/sw/rxe/rxe_net.h                |     1 -
 drivers/infiniband/sw/rxe/rxe_qp.c                 |    27 +-
 drivers/infiniband/sw/rxe/rxe_recv.c               |     3 +-
 drivers/infiniband/sw/rxe/rxe_req.c                |     9 +-
 drivers/infiniband/sw/rxe/rxe_resp.c               |     5 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c              |    12 +-
 drivers/infiniband/sw/rxe/rxe_verbs.h              |     3 +
 drivers/infiniband/ulp/ipoib/ipoib_cm.c            |    18 +-
 drivers/infiniband/ulp/ipoib/ipoib_fs.c            |     2 -
 drivers/infiniband/ulp/ipoib/ipoib_ib.c            |     5 +-
 drivers/infiniband/ulp/ipoib/ipoib_main.c          |   101 +-
 drivers/infiniband/ulp/ipoib/ipoib_verbs.c         |     6 +-
 drivers/infiniband/ulp/iser/iscsi_iser.c           |     2 +-
 drivers/infiniband/ulp/iser/iser_initiator.c       |    16 +-
 drivers/infiniband/ulp/isert/ib_isert.c            |     7 +
 drivers/infiniband/ulp/isert/ib_isert.h            |     1 +
 drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c    |     2 +-
 drivers/infiniband/ulp/srp/ib_srp.c                |   795 +-
 drivers/infiniband/ulp/srp/ib_srp.h                |    43 +-
 drivers/infiniband/ulp/srpt/ib_srpt.c              |   962 +-
 drivers/infiniband/ulp/srpt/ib_srpt.h              |   100 +-
 drivers/input/evdev.c                              |    30 +-
 drivers/input/input-compat.c                       |     8 +-
 drivers/input/input-compat.h                       |     3 +-
 drivers/input/input-polldev.c                      |     1 -
 drivers/input/input.c                              |     4 +-
 drivers/input/joydev.c                             |     6 +-
 drivers/input/joystick/xpad.c                      |     7 +-
 drivers/input/keyboard/matrix_keypad.c             |     4 +-
 drivers/input/keyboard/pmic8xxx-keypad.c           |     1 -
 drivers/input/misc/Kconfig                         |    16 +-
 drivers/input/misc/Makefile                        |     1 -
 drivers/input/misc/apanel.c                        |     2 -
 drivers/input/misc/ati_remote2.c                   |     2 -
 drivers/input/misc/gpio_tilt_polled.c              |   210 -
 drivers/input/misc/hp_sdc_rtc.c                    |     4 +-
 drivers/input/misc/keyspan_remote.c                |    11 +-
 drivers/input/misc/uinput.c                        |    10 +-
 drivers/input/misc/yealink.c                       |     6 +-
 drivers/input/mouse/cyapa.c                        |     2 +-
 drivers/input/mouse/cyapa_gen3.c                   |    43 -
 drivers/input/mouse/elan_i2c_core.c                |     2 -
 drivers/input/mouse/psmouse-base.c                 |    34 +-
 drivers/input/mouse/synaptics.c                    |    11 +-
 drivers/input/mousedev.c                           |     8 +-
 drivers/input/rmi4/rmi_bus.c                       |     1 -
 drivers/input/rmi4/rmi_driver.c                    |    14 +-
 drivers/input/rmi4/rmi_driver.h                    |     2 -
 drivers/input/rmi4/rmi_f03.c                       |    73 +-
 drivers/input/rmi4/rmi_f34.c                       |     1 -
 drivers/input/rmi4/rmi_i2c.c                       |     1 -
 drivers/input/rmi4/rmi_spi.c                       |     1 -
 drivers/input/serio/Kconfig                        |    10 -
 drivers/input/serio/Makefile                       |     1 -
 drivers/input/serio/at32psif.c                     |   357 -
 drivers/input/serio/hil_mlc.c                      |    26 +-
 drivers/input/serio/hp_sdc.c                       |    17 +-
 drivers/input/serio/hp_sdc_mlc.c                   |     5 +-
 drivers/input/serio/serio_raw.c                    |     8 +-
 drivers/input/serio/userio.c                       |     4 +-
 drivers/input/sparse-keymap.c                      |     1 -
 drivers/input/tablet/acecad.c                      |    14 +-
 drivers/input/tablet/aiptek.c                      |    11 +-
 drivers/input/tablet/hanwang.c                     |    10 +-
 drivers/input/tablet/kbtab.c                       |    17 +-
 drivers/input/touchscreen/Kconfig                  |    15 -
 drivers/input/touchscreen/Makefile                 |     1 -
 drivers/input/touchscreen/ad7877.c                 |    67 +-
 drivers/input/touchscreen/atmel-wm97xx.c           |   436 -
 drivers/input/touchscreen/auo-pixcir-ts.c          |     6 +-
 drivers/input/touchscreen/colibri-vf50-ts.c        |     2 -
 drivers/input/touchscreen/da9052_tsi.c             |     1 -
 drivers/input/touchscreen/edt-ft5x06.c             |    14 +-
 drivers/input/touchscreen/elants_i2c.c             |     4 +-
 drivers/input/touchscreen/goodix.c                 |   149 +-
 drivers/input/touchscreen/melfas_mip4.c            |     1 -
 drivers/input/touchscreen/mms114.c                 |   255 +-
 drivers/input/touchscreen/raydium_i2c_ts.c         |    14 +-
 drivers/input/touchscreen/silead.c                 |    46 +-
 drivers/input/touchscreen/stmfts.c                 |    11 +-
 drivers/input/touchscreen/usbtouchscreen.c         |     9 +-
 drivers/input/touchscreen/wdt87xx_i2c.c            |     2 -
 drivers/iommu/amd_iommu.c                          |    27 +-
 drivers/iommu/amd_iommu_init.c                     |     9 +-
 drivers/iommu/amd_iommu_types.h                    |     2 +
 drivers/iommu/amd_iommu_v2.c                       |     4 +-
 drivers/iommu/arm-smmu-v3.c                        |     2 +-
 drivers/iommu/arm-smmu.c                           |    12 +-
 drivers/iommu/exynos-iommu.c                       |     9 +-
 drivers/iommu/intel-iommu.c                        |     7 +-
 drivers/iommu/intel-svm.c                          |    35 +-
 drivers/iommu/iommu.c                              |     5 +
 drivers/iommu/ipmmu-vmsa.c                         |    14 +-
 drivers/iommu/msm_iommu.c                          |    16 +-
 drivers/iommu/of_iommu.c                           |    16 -
 drivers/iommu/omap-iommu-debug.c                   |     4 +-
 drivers/iommu/qcom_iommu.c                         |     2 +-
 drivers/iommu/tegra-smmu.c                         |   124 +-
 drivers/irqchip/Kconfig                            |     8 +
 drivers/irqchip/Makefile                           |     1 +
 drivers/irqchip/irq-bcm2836.c                      |    46 +-
 drivers/irqchip/irq-bcm7038-l1.c                   |     3 -
 drivers/irqchip/irq-bcm7120-l2.c                   |     3 -
 drivers/irqchip/irq-brcmstb-l2.c                   |     3 -
 drivers/irqchip/irq-gic-v2m.c                      |    46 +-
 drivers/irqchip/irq-gic-v3-its-pci-msi.c           |     2 +
 drivers/irqchip/irq-gic-v3-its-platform-msi.c      |     2 +
 drivers/irqchip/irq-gic-v3-its.c                   |    15 +-
 drivers/irqchip/irq-gic-v3.c                       |    44 +-
 drivers/irqchip/irq-goldfish-pic.c                 |   139 +
 drivers/irqchip/irq-i8259.c                        |     4 +-
 drivers/irqchip/irq-imx-gpcv2.c                    |    14 -
 drivers/irqchip/irq-mips-gic.c                     |     2 -
 drivers/irqchip/irq-ompic.c                        |     4 +-
 drivers/irqchip/irq-stm32-exti.c                   |     3 +-
 drivers/isdn/capi/capi.c                           |    10 +-
 drivers/isdn/divert/divert_procfs.c                |     8 +-
 drivers/isdn/hardware/eicon/divamnt.c              |     8 +-
 drivers/isdn/hardware/eicon/divasi.c               |    14 +-
 drivers/isdn/hardware/eicon/divasmain.c            |     6 +-
 drivers/isdn/hardware/eicon/divasproc.c            |     4 +-
 drivers/isdn/hysdn/hysdn_proclog.c                 |     6 +-
 drivers/isdn/i4l/isdn_common.c                     |    16 +-
 drivers/isdn/i4l/isdn_ppp.c                        |    12 +-
 drivers/isdn/i4l/isdn_ppp.h                        |     2 +-
 drivers/isdn/mISDN/l1oip_core.c                    |    22 +-
 drivers/isdn/mISDN/timerdev.c                      |     6 +-
 drivers/leds/Kconfig                               |     9 +-
 drivers/leds/Makefile                              |     1 +
 drivers/leds/leds-as3645a.c                        |     7 +-
 drivers/leds/leds-blinkm.c                         |     4 +
 drivers/leds/leds-lm3692x.c                        |   393 +
 drivers/leds/leds-lp8860.c                         |    66 +-
 drivers/leds/leds-pm8058.c                         |     2 +-
 drivers/leds/leds-pwm.c                            |     1 -
 drivers/leds/trigger/Kconfig                       |     9 +-
 drivers/leds/trigger/Makefile                      |     1 +
 drivers/leds/trigger/ledtrig-netdev.c              |   496 +
 drivers/leds/trigger/ledtrig-transient.c           |    33 +-
 drivers/leds/uleds.c                               |     4 +-
 drivers/lightnvm/Kconfig                           |     7 -
 drivers/lightnvm/Makefile                          |     1 -
 drivers/lightnvm/core.c                            |   462 +-
 drivers/lightnvm/pblk-cache.c                      |     5 +
 drivers/lightnvm/pblk-core.c                       |    55 +-
 drivers/lightnvm/pblk-gc.c                         |    23 +-
 drivers/lightnvm/pblk-init.c                       |   104 +-
 drivers/lightnvm/pblk-map.c                        |     2 +-
 drivers/lightnvm/pblk-rb.c                         |   111 +-
 drivers/lightnvm/pblk-read.c                       |    35 +-
 drivers/lightnvm/pblk-recovery.c                   |    43 +-
 drivers/lightnvm/pblk-rl.c                         |    54 +-
 drivers/lightnvm/pblk-sysfs.c                      |    15 +-
 drivers/lightnvm/pblk-write.c                      |    23 +-
 drivers/lightnvm/pblk.h                            |   163 +-
 drivers/lightnvm/rrpc.c                            |  1625 -
 drivers/lightnvm/rrpc.h                            |   290 -
 drivers/macintosh/adb.c                            |    24 +-
 drivers/macintosh/adbhid.c                         |    55 +-
 drivers/macintosh/ams/ams-input.c                  |     4 +-
 drivers/macintosh/macio_asic.c                     |     1 +
 drivers/macintosh/smu.c                            |     6 +-
 drivers/macintosh/therm_adt746x.c                  |     4 +-
 drivers/macintosh/via-pmu-backlight.c              |     2 +-
 drivers/macintosh/via-pmu.c                        |     8 +-
 drivers/macintosh/windfarm_pm112.c                 |     8 +-
 drivers/macintosh/windfarm_pm121.c                 |     5 +-
 drivers/macintosh/windfarm_pm72.c                  |     2 +-
 drivers/macintosh/windfarm_pm81.c                  |     5 +-
 drivers/macintosh/windfarm_pm91.c                  |     5 +-
 drivers/macintosh/windfarm_rm31.c                  |     2 +-
 drivers/mailbox/mailbox-test.c                     |     4 +-
 drivers/mailbox/qcom-apcs-ipc-mailbox.c            |    35 +-
 drivers/mailbox/ti-msgmgr.c                        |    14 +-
 drivers/md/Kconfig                                 |     7 +
 drivers/md/Makefile                                |     1 +
 drivers/md/bcache/alloc.c                          |    23 +-
 drivers/md/bcache/bcache.h                         |    33 +-
 drivers/md/bcache/btree.c                          |    19 +-
 drivers/md/bcache/closure.c                        |    47 +-
 drivers/md/bcache/closure.h                        |    60 +-
 drivers/md/bcache/debug.c                          |     7 +-
 drivers/md/bcache/io.c                             |    13 +-
 drivers/md/bcache/journal.c                        |    52 +-
 drivers/md/bcache/movinggc.c                       |     2 +-
 drivers/md/bcache/request.c                        |    31 +-
 drivers/md/bcache/super.c                          |    77 +-
 drivers/md/bcache/sysfs.c                          |    34 +-
 drivers/md/bcache/util.c                           |    34 +
 drivers/md/bcache/util.h                           |     3 +
 drivers/md/bcache/writeback.c                      |   212 +-
 drivers/md/bcache/writeback.h                      |    15 +-
 drivers/md/dm-bufio.c                              |    53 +-
 drivers/md/dm-core.h                               |     5 +-
 drivers/md/dm-crypt.c                              |     6 +-
 drivers/md/dm-delay.c                              |     2 +
 drivers/md/dm-flakey.c                             |     5 +
 drivers/md/dm-io.c                                 |     3 +-
 drivers/md/dm-ioctl.c                              |     6 +-
 drivers/md/dm-kcopyd.c                             |     6 +-
 drivers/md/dm-log-writes.c                         |     2 +-
 drivers/md/dm-mpath.c                              |   293 +-
 drivers/md/dm-queue-length.c                       |     6 +-
 drivers/md/dm-raid.c                               |   390 +-
 drivers/md/dm-rq.c                                 |    35 +-
 drivers/md/dm-service-time.c                       |     6 +-
 drivers/md/dm-snap.c                               |    84 +-
 drivers/md/dm-stats.c                              |     1 +
 drivers/md/dm-table.c                              |   114 +-
 drivers/md/dm-thin.c                               |     9 +-
 drivers/md/dm-unstripe.c                           |   219 +
 drivers/md/dm-zoned-metadata.c                     |     3 +
 drivers/md/dm-zoned-target.c                       |     3 +
 drivers/md/dm.c                                    |   722 +-
 drivers/md/dm.h                                    |     4 +-
 drivers/md/md-multipath.c                          |     2 +-
 drivers/md/md.c                                    |    94 +-
 drivers/md/md.h                                    |    11 +
 drivers/md/raid1.c                                 |    24 +-
 drivers/md/raid1.h                                 |    12 +
 drivers/md/raid10.c                                |    30 +-
 drivers/md/raid10.h                                |    13 +
 drivers/md/raid5-cache.c                           |    31 +-
 drivers/md/raid5-log.h                             |    31 +
 drivers/md/raid5-ppl.c                             |   177 +-
 drivers/md/raid5.c                                 |    35 +-
 drivers/md/raid5.h                                 |    12 +
 drivers/media/Kconfig                              |    22 +-
 drivers/media/cec/cec-adap.c                       |    81 +-
 drivers/media/cec/cec-api.c                        |    44 +-
 drivers/media/cec/cec-core.c                       |    27 +-
 drivers/media/cec/cec-priv.h                       |     2 +
 drivers/media/common/Kconfig                       |     1 +
 drivers/media/common/Makefile                      |     2 +-
 drivers/media/common/b2c2/Makefile                 |     1 -
 drivers/media/common/b2c2/flexcop-common.h         |     8 +-
 drivers/media/common/saa7146/saa7146_fops.c        |    14 +-
 drivers/media/common/saa7146/saa7146_video.c       |    17 +-
 drivers/media/common/siano/Makefile                |     4 -
 drivers/media/common/siano/smsdvb-debugfs.c        |    36 +-
 drivers/media/common/siano/smsdvb-main.c           |    10 +-
 drivers/media/common/siano/smsir.c                 |    35 +-
 drivers/media/common/siano/smsir.h                 |    37 +-
 drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c    |     8 +-
 drivers/media/common/v4l2-tpg/v4l2-tpg-core.c      |     2 +-
 drivers/media/common/videobuf2/Kconfig             |    34 +
 drivers/media/common/videobuf2/Makefile            |    14 +
 .../{v4l2-core => common/videobuf2}/vb2-trace.c    |     0
 .../videobuf2}/videobuf2-core.c                    |    87 +-
 .../videobuf2}/videobuf2-dma-contig.c              |     6 +-
 .../videobuf2}/videobuf2-dma-sg.c                  |     7 +-
 .../videobuf2}/videobuf2-dvb.c                     |     0
 .../videobuf2}/videobuf2-memops.c                  |     0
 .../videobuf2}/videobuf2-v4l2.c                    |    20 +-
 .../videobuf2}/videobuf2-vmalloc.c                 |     0
 drivers/media/dvb-core/Kconfig                     |    13 +
 drivers/media/dvb-core/Makefile                    |     7 +-
 drivers/media/dvb-core/dmxdev.c                    |   279 +-
 drivers/media/dvb-core/dvb_ca_en50221.c            |    17 +-
 drivers/media/dvb-core/dvb_demux.c                 |   133 +-
 drivers/media/dvb-core/dvb_frontend.c              |   187 +-
 drivers/media/dvb-core/dvb_math.c                  |     2 +-
 drivers/media/dvb-core/dvb_net.c                   |    79 +-
 drivers/media/dvb-core/dvb_ringbuffer.c            |     2 +-
 drivers/media/dvb-core/dvb_vb2.c                   |   443 +
 drivers/media/dvb-core/dvbdev.c                    |     2 +-
 drivers/media/dvb-frontends/Makefile               |     1 -
 drivers/media/dvb-frontends/a8293.h                |     2 +-
 drivers/media/dvb-frontends/af9013_priv.h          |     2 +-
 drivers/media/dvb-frontends/af9033_priv.h          |     4 +-
 drivers/media/dvb-frontends/as102_fe.c             |     2 +-
 drivers/media/dvb-frontends/ascot2e.c              |     6 +-
 drivers/media/dvb-frontends/atbm8830.c             |     2 +-
 drivers/media/dvb-frontends/au8522_common.c        |     2 +-
 drivers/media/dvb-frontends/au8522_dig.c           |     2 +-
 drivers/media/dvb-frontends/au8522_priv.h          |   220 +-
 drivers/media/dvb-frontends/bcm3510.c              |     2 +-
 drivers/media/dvb-frontends/cx22700.c              |     2 +-
 drivers/media/dvb-frontends/cx22702.c              |     2 +-
 drivers/media/dvb-frontends/cx24110.c              |     2 +-
 drivers/media/dvb-frontends/cx24113.c              |     2 +-
 drivers/media/dvb-frontends/cx24116.c              |     4 +-
 drivers/media/dvb-frontends/cx24117.c              |     2 +-
 drivers/media/dvb-frontends/cx24120.c              |     2 +-
 drivers/media/dvb-frontends/cx24123.c              |     2 +-
 drivers/media/dvb-frontends/cxd2820r_priv.h        |     4 +-
 drivers/media/dvb-frontends/cxd2841er.c            |    23 +-
 drivers/media/dvb-frontends/dib0070.c              |     2 +-
 drivers/media/dvb-frontends/dib0090.c              |     2 +-
 drivers/media/dvb-frontends/dib3000mb.c            |     2 +-
 drivers/media/dvb-frontends/dib3000mc.c            |     2 +-
 drivers/media/dvb-frontends/dib7000m.c             |     2 +-
 drivers/media/dvb-frontends/dib7000p.c             |     4 +-
 drivers/media/dvb-frontends/dib8000.c              |     4 +-
 drivers/media/dvb-frontends/dib9000.c              |     4 +-
 drivers/media/dvb-frontends/dibx000_common.c       |     8 +-
 drivers/media/dvb-frontends/drx39xyj/Makefile      |     1 -
 drivers/media/dvb-frontends/drx39xyj/drx39xxj.h    |     2 +-
 drivers/media/dvb-frontends/drx39xyj/drx_driver.h  |     2 +-
 drivers/media/dvb-frontends/drx39xyj/drxj.c        |    16 +-
 drivers/media/dvb-frontends/drxd_hard.c            |    27 +-
 drivers/media/dvb-frontends/drxk.h                 |     6 +-
 drivers/media/dvb-frontends/drxk_hard.c            |     6 +-
 drivers/media/dvb-frontends/ds3000.c               |     2 +-
 drivers/media/dvb-frontends/dvb-pll.h              |     2 +-
 drivers/media/dvb-frontends/dvb_dummy_fe.c         |     2 +-
 drivers/media/dvb-frontends/dvb_dummy_fe.h         |     2 +-
 drivers/media/dvb-frontends/ec100.c                |     2 +-
 drivers/media/dvb-frontends/gp8psk-fe.c            |     2 +-
 drivers/media/dvb-frontends/helene.c               |     6 +-
 drivers/media/dvb-frontends/horus3a.c              |     6 +-
 drivers/media/dvb-frontends/isl6405.c              |     2 +-
 drivers/media/dvb-frontends/isl6421.c              |     2 +-
 drivers/media/dvb-frontends/isl6423.c              |     2 +-
 drivers/media/dvb-frontends/itd1000.c              |     7 +-
 drivers/media/dvb-frontends/ix2505v.h              |     2 +-
 drivers/media/dvb-frontends/l64781.c               |     2 +-
 drivers/media/dvb-frontends/lg2160.h               |     2 +-
 drivers/media/dvb-frontends/lgdt3305.c             |     2 +-
 drivers/media/dvb-frontends/lgdt3305.h             |     2 +-
 drivers/media/dvb-frontends/lgdt3306a.c            |     2 +-
 drivers/media/dvb-frontends/lgdt3306a.h            |     2 +-
 drivers/media/dvb-frontends/lgdt330x.c             |     4 +-
 drivers/media/dvb-frontends/lgs8gl5.c              |     2 +-
 drivers/media/dvb-frontends/lgs8gxx.c              |     2 +-
 drivers/media/dvb-frontends/lnbh25.c               |     2 +-
 drivers/media/dvb-frontends/lnbp21.c               |     2 +-
 drivers/media/dvb-frontends/lnbp22.c               |     2 +-
 drivers/media/dvb-frontends/m88ds3103.c            |     7 +-
 drivers/media/dvb-frontends/m88ds3103_priv.h       |     4 +-
 drivers/media/dvb-frontends/m88rs2000.c            |    13 +-
 drivers/media/dvb-frontends/m88rs2000.h            |     2 +-
 drivers/media/dvb-frontends/mb86a16.c              |    15 +-
 drivers/media/dvb-frontends/mb86a16.h              |     2 +-
 drivers/media/dvb-frontends/mb86a20s.c             |     4 +-
 drivers/media/dvb-frontends/mn88472_priv.h         |     4 +-
 drivers/media/dvb-frontends/mn88473.c              |     2 +-
 drivers/media/dvb-frontends/mn88473_priv.h         |     4 +-
 drivers/media/dvb-frontends/mt312.c                |     7 +-
 drivers/media/dvb-frontends/mt352.c                |     2 +-
 drivers/media/dvb-frontends/mxl5xx.c               |    11 +-
 drivers/media/dvb-frontends/mxl5xx.h               |     2 +-
 drivers/media/dvb-frontends/nxt200x.c              |     2 +-
 drivers/media/dvb-frontends/nxt6000.c              |     2 +-
 drivers/media/dvb-frontends/or51132.c              |     4 +-
 drivers/media/dvb-frontends/or51211.c              |     4 +-
 drivers/media/dvb-frontends/rtl2830_priv.h         |     4 +-
 drivers/media/dvb-frontends/rtl2832_priv.h         |     4 +-
 drivers/media/dvb-frontends/rtl2832_sdr.h          |     2 +-
 drivers/media/dvb-frontends/s5h1409.c              |     2 +-
 drivers/media/dvb-frontends/s5h1411.c              |     2 +-
 drivers/media/dvb-frontends/s5h1420.c              |     2 +-
 drivers/media/dvb-frontends/s5h1432.c              |     2 +-
 drivers/media/dvb-frontends/s921.c                 |     2 +-
 drivers/media/dvb-frontends/si2165.c               |   613 +-
 drivers/media/dvb-frontends/si2165.h               |    37 +-
 drivers/media/dvb-frontends/si2165_priv.h          |   114 +-
 drivers/media/dvb-frontends/si2168.c               |     3 +
 drivers/media/dvb-frontends/si2168_priv.h          |     2 +-
 drivers/media/dvb-frontends/si21xx.c               |     2 +-
 drivers/media/dvb-frontends/si21xx.h               |     2 +-
 drivers/media/dvb-frontends/sp2.h                  |     2 +-
 drivers/media/dvb-frontends/sp2_priv.h             |     2 +-
 drivers/media/dvb-frontends/sp8870.c               |     2 +-
 drivers/media/dvb-frontends/sp887x.c               |     2 +-
 drivers/media/dvb-frontends/stb0899_algo.c         |     3 +-
 drivers/media/dvb-frontends/stb0899_drv.c          |    15 +-
 drivers/media/dvb-frontends/stb0899_drv.h          |     4 +-
 drivers/media/dvb-frontends/stb0899_priv.h         |     4 +-
 drivers/media/dvb-frontends/stb6000.h              |     2 +-
 drivers/media/dvb-frontends/stb6100.c              |     8 +-
 drivers/media/dvb-frontends/stb6100.h              |     2 +-
 drivers/media/dvb-frontends/stb6100_cfg.h          |     2 +-
 drivers/media/dvb-frontends/stb6100_proc.h         |     2 +-
 drivers/media/dvb-frontends/stv0288.c              |     2 +-
 drivers/media/dvb-frontends/stv0288.h              |     2 +-
 drivers/media/dvb-frontends/stv0297.c              |     2 +-
 drivers/media/dvb-frontends/stv0297.h              |     2 +-
 drivers/media/dvb-frontends/stv0299.c              |     2 +-
 drivers/media/dvb-frontends/stv0299.h              |     2 +-
 drivers/media/dvb-frontends/stv0367.c              |     8 +-
 drivers/media/dvb-frontends/stv0367.h              |     2 +-
 drivers/media/dvb-frontends/stv0900.h              |     2 +-
 drivers/media/dvb-frontends/stv0900_core.c         |     2 +-
 drivers/media/dvb-frontends/stv0900_init.h         |    34 +-
 drivers/media/dvb-frontends/stv0900_priv.h         |     2 +-
 drivers/media/dvb-frontends/stv090x.c              |    43 +-
 drivers/media/dvb-frontends/stv090x_priv.h         |     4 +-
 drivers/media/dvb-frontends/stv0910.c              |   227 +-
 drivers/media/dvb-frontends/stv0910_regs.h         |  1854 +-
 drivers/media/dvb-frontends/stv6110.h              |     2 +-
 drivers/media/dvb-frontends/stv6110x.c             |     8 +-
 drivers/media/dvb-frontends/stv6110x_priv.h        |     6 +-
 drivers/media/dvb-frontends/stv6111.c              |    46 +-
 drivers/media/dvb-frontends/tc90522.c              |     2 +-
 drivers/media/dvb-frontends/tc90522.h              |     2 +-
 drivers/media/dvb-frontends/tda10021.c             |     2 +-
 drivers/media/dvb-frontends/tda10023.c             |     4 +-
 drivers/media/dvb-frontends/tda10048.c             |     4 +-
 drivers/media/dvb-frontends/tda1004x.c             |     2 +-
 drivers/media/dvb-frontends/tda10071_priv.h        |     2 +-
 drivers/media/dvb-frontends/tda10086.c             |     2 +-
 drivers/media/dvb-frontends/tda18271c2dd.c         |     3 +-
 drivers/media/dvb-frontends/tda18271c2dd.h         |     4 +-
 drivers/media/dvb-frontends/tda665x.c              |     2 +-
 drivers/media/dvb-frontends/tda8083.c              |     2 +-
 drivers/media/dvb-frontends/tda8261.c              |     2 +-
 drivers/media/dvb-frontends/tda826x.h              |     2 +-
 drivers/media/dvb-frontends/ts2020.c               |     6 +-
 drivers/media/dvb-frontends/tua6100.h              |     2 +-
 drivers/media/dvb-frontends/ves1820.c              |     2 +-
 drivers/media/dvb-frontends/ves1x93.c              |     2 +-
 drivers/media/dvb-frontends/zd1301_demod.h         |     2 +-
 drivers/media/dvb-frontends/zl10036.h              |     2 +-
 drivers/media/dvb-frontends/zl10039.c              |     6 +-
 drivers/media/dvb-frontends/zl10353.c              |     2 +-
 drivers/media/firewire/Makefile                    |     2 -
 drivers/media/firewire/firedtv-avc.c               |     6 +-
 drivers/media/firewire/firedtv-ci.c                |     6 +-
 drivers/media/firewire/firedtv-dvb.c               |     8 +-
 drivers/media/firewire/firedtv-fe.c                |     8 +-
 drivers/media/firewire/firedtv-fw.c                |     2 +-
 drivers/media/firewire/firedtv.h                   |    12 +-
 drivers/media/i2c/Kconfig                          |    26 +-
 drivers/media/i2c/Makefile                         |     2 +-
 drivers/media/i2c/adv7180.c                        |    12 +-
 drivers/media/i2c/adv7343.c                        |     2 +-
 drivers/media/i2c/adv7393.c                        |     2 +-
 drivers/media/i2c/adv748x/adv748x-afe.c            |     1 +
 drivers/media/i2c/adv748x/adv748x-core.c           |    16 +-
 drivers/media/i2c/adv748x/adv748x-csi2.c           |    13 +-
 drivers/media/i2c/adv748x/adv748x.h                |     1 +
 drivers/media/i2c/adv7604.c                        |     1 +
 drivers/media/i2c/as3645a.c                        |   880 -
 drivers/media/i2c/cx25840/Makefile                 |     2 -
 drivers/media/i2c/cx25840/cx25840-core.c           |    35 +-
 drivers/media/i2c/cx25840/cx25840-core.h           |     2 +-
 drivers/media/i2c/cx25840/cx25840-ir.c             |     6 +-
 drivers/media/i2c/dw9714.c                         |    22 +-
 drivers/media/i2c/imx274.c                         |     4 +-
 drivers/media/i2c/ir-kbd-i2c.c                     |   540 +-
 drivers/media/i2c/ks0127.c                         |     2 +-
 drivers/media/i2c/mt9m111.c                        |    49 +-
 drivers/media/i2c/mt9v011.c                        |    13 +-
 drivers/media/i2c/mt9v032.c                        |    21 +-
 drivers/media/i2c/ov2640.c                         |     4 -
 drivers/media/i2c/ov2659.c                         |     4 -
 drivers/media/i2c/ov5640.c                         |   326 +-
 drivers/media/i2c/ov7670.c                         |    90 +-
 drivers/media/i2c/ov7740.c                         |  1214 +
 drivers/media/i2c/ov9650.c                         |    10 +-
 drivers/media/i2c/saa6588.c                        |     2 +-
 drivers/media/i2c/saa6752hs.c                      |     8 +-
 drivers/media/i2c/saa7115.c                        |    60 +-
 drivers/media/i2c/saa711x_regs.h                   |    14 +-
 drivers/media/i2c/saa7127.c                        |   162 +-
 drivers/media/i2c/saa717x.c                        |    12 +-
 drivers/media/i2c/smiapp/smiapp-core.c             |     2 +-
 drivers/media/i2c/tda7432.c                        |     1 -
 drivers/media/i2c/ths7303.c                        |     2 +-
 drivers/media/i2c/tvaudio.c                        |     4 +-
 drivers/media/i2c/tvp514x.c                        |     4 -
 drivers/media/i2c/tvp5150.c                        |   101 +-
 drivers/media/i2c/tvp5150_reg.h                    |     9 +-
 drivers/media/i2c/tvp7002_reg.h                    |     6 +-
 drivers/media/i2c/vpx3220.c                        |     2 +-
 drivers/media/media-devnode.c                      |     4 +-
 drivers/media/mmc/siano/Makefile                   |     2 -
 drivers/media/pci/Kconfig                          |     2 +
 drivers/media/pci/Makefile                         |     3 +-
 drivers/media/pci/b2c2/Makefile                    |     1 -
 drivers/media/pci/bt8xx/Makefile                   |     2 -
 drivers/media/pci/bt8xx/bt878.c                    |     7 +-
 drivers/media/pci/bt8xx/bttv-cards.c               |   273 +-
 drivers/media/pci/bt8xx/bttv-driver.c              |    34 +-
 drivers/media/pci/bt8xx/bttv-input.c               |     8 +-
 drivers/media/pci/bt8xx/bttv.h                     |     5 +-
 drivers/media/pci/bt8xx/bttvp.h                    |     6 +-
 drivers/media/pci/bt8xx/dst.c                      |     2 +-
 drivers/media/pci/bt8xx/dst_ca.c                   |     4 +-
 drivers/media/pci/bt8xx/dvb-bt8xx.c                |     8 +-
 drivers/media/pci/bt8xx/dvb-bt8xx.h                |     4 +-
 drivers/media/pci/cobalt/Kconfig                   |     1 +
 drivers/media/pci/cx18/Makefile                    |     1 -
 drivers/media/pci/cx18/cx18-alsa-pcm.c             |     2 +-
 drivers/media/pci/cx18/cx18-av-audio.c             |     2 +-
 drivers/media/pci/cx18/cx18-av-core.c              |    18 +-
 drivers/media/pci/cx18/cx18-av-core.h              |     2 +-
 drivers/media/pci/cx18/cx18-cards.c                |     8 +-
 drivers/media/pci/cx18/cx18-cards.h                |    40 +-
 drivers/media/pci/cx18/cx18-driver.h               |    58 +-
 drivers/media/pci/cx18/cx18-fileops.c              |    24 +-
 drivers/media/pci/cx18/cx18-fileops.h              |     2 +-
 drivers/media/pci/cx18/cx18-firmware.c             |    96 +-
 drivers/media/pci/cx18/cx18-i2c.c                  |    13 +-
 drivers/media/pci/cx18/cx18-mailbox.c              |     8 +-
 drivers/media/pci/cx18/cx18-streams.c              |     4 +-
 drivers/media/pci/cx18/cx18-vbi.c                  |     2 +-
 drivers/media/pci/cx18/cx23418.h                   |    88 +-
 drivers/media/pci/cx23885/Makefile                 |     2 -
 drivers/media/pci/cx23885/altera-ci.c              |     6 +-
 drivers/media/pci/cx23885/cimax2.c                 |     4 +-
 drivers/media/pci/cx23885/cimax2.h                 |     2 +-
 drivers/media/pci/cx23885/cx23885-cards.c          |     6 +-
 drivers/media/pci/cx23885/cx23885-dvb.c            |     6 +-
 drivers/media/pci/cx23885/cx23885-input.c          |    15 +-
 drivers/media/pci/cx23885/cx23885-video.c          |     2 +-
 drivers/media/pci/cx23885/cx23885.h                |     4 +-
 drivers/media/pci/cx23885/cx23888-ir.c             |     6 +-
 drivers/media/pci/cx25821/Makefile                 |     2 -
 drivers/media/pci/cx25821/cx25821-alsa.c           |     4 +-
 drivers/media/pci/cx88/Makefile                    |     3 -
 drivers/media/pci/cx88/cx88-blackbird.c            |     3 +-
 drivers/media/pci/cx88/cx88-video.c                |    10 +-
 drivers/media/pci/cx88/cx88.h                      |     4 +-
 drivers/media/pci/ddbridge/Makefile                |     5 +-
 drivers/media/pci/ddbridge/ddbridge-ci.c           |   359 +
 drivers/media/pci/ddbridge/ddbridge-ci.h           |    31 +
 drivers/media/pci/ddbridge/ddbridge-core.c         |   659 +-
 drivers/media/pci/ddbridge/ddbridge-hw.c           |     8 +-
 drivers/media/pci/ddbridge/ddbridge-i2c.c          |    16 +-
 drivers/media/pci/ddbridge/ddbridge-main.c         |    23 +-
 .../ddbridge/{ddbridge-maxs8.c => ddbridge-max.c}  |    54 +-
 .../ddbridge/{ddbridge-maxs8.h => ddbridge-max.h}  |    12 +-
 drivers/media/pci/ddbridge/ddbridge-regs.h         |    32 +-
 drivers/media/pci/ddbridge/ddbridge.h              |   105 +-
 drivers/media/pci/dm1105/Makefile                  |     2 +-
 drivers/media/pci/dm1105/dm1105.c                  |    12 +-
 drivers/media/pci/intel/Makefile                   |     5 +
 drivers/media/pci/intel/ipu3/Kconfig               |    19 +
 drivers/media/pci/intel/ipu3/Makefile              |     1 +
 drivers/media/pci/intel/ipu3/ipu3-cio2.c           |  2048 ++
 drivers/media/pci/intel/ipu3/ipu3-cio2.h           |   449 +
 drivers/media/pci/ivtv/Makefile                    |     2 -
 drivers/media/pci/ivtv/ivtv-cards.c                |     2 +-
 drivers/media/pci/ivtv/ivtv-cards.h                |   148 +-
 drivers/media/pci/ivtv/ivtv-driver.h               |   102 +-
 drivers/media/pci/ivtv/ivtv-fileops.c              |    26 +-
 drivers/media/pci/ivtv/ivtv-fileops.h              |     4 +-
 drivers/media/pci/ivtv/ivtv-firmware.c             |    36 +-
 drivers/media/pci/ivtv/ivtv-i2c.c                  |    46 +-
 drivers/media/pci/ivtv/ivtv-ioctl.c                |    74 +-
 drivers/media/pci/ivtv/ivtv-mailbox.c              |   182 +-
 drivers/media/pci/mantis/Makefile                  |     2 +-
 drivers/media/pci/mantis/hopper_cards.c            |    10 +-
 drivers/media/pci/mantis/hopper_vp3028.c           |    10 +-
 drivers/media/pci/mantis/mantis_ca.c               |    10 +-
 drivers/media/pci/mantis/mantis_cards.c            |    10 +-
 drivers/media/pci/mantis/mantis_dma.c              |    10 +-
 drivers/media/pci/mantis/mantis_dvb.c              |    10 +-
 drivers/media/pci/mantis/mantis_evm.c              |    10 +-
 drivers/media/pci/mantis/mantis_hif.c              |    10 +-
 drivers/media/pci/mantis/mantis_i2c.c              |    10 +-
 drivers/media/pci/mantis/mantis_input.c            |    10 +-
 drivers/media/pci/mantis/mantis_ioc.c              |    10 +-
 drivers/media/pci/mantis/mantis_link.h             |     2 +-
 drivers/media/pci/mantis/mantis_pci.c              |    10 +-
 drivers/media/pci/mantis/mantis_pcmcia.c           |    10 +-
 drivers/media/pci/mantis/mantis_reg.h              |     6 +-
 drivers/media/pci/mantis/mantis_uart.c             |    10 +-
 drivers/media/pci/mantis/mantis_vp1033.c           |    10 +-
 drivers/media/pci/mantis/mantis_vp1034.c           |    10 +-
 drivers/media/pci/mantis/mantis_vp1034.h           |     2 +-
 drivers/media/pci/mantis/mantis_vp1041.c           |   220 +-
 drivers/media/pci/mantis/mantis_vp2033.c           |    10 +-
 drivers/media/pci/mantis/mantis_vp2040.c           |    10 +-
 drivers/media/pci/mantis/mantis_vp3028.h           |     2 +-
 drivers/media/pci/mantis/mantis_vp3030.c           |    10 +-
 drivers/media/pci/meye/meye.c                      |     8 +-
 drivers/media/pci/netup_unidvb/Makefile            |     1 -
 drivers/media/pci/netup_unidvb/netup_unidvb.h      |     2 +-
 drivers/media/pci/netup_unidvb/netup_unidvb_core.c |     2 +-
 drivers/media/pci/ngene/Makefile                   |     1 -
 drivers/media/pci/ngene/ngene.h                    |    14 +-
 drivers/media/pci/pluto2/Makefile                  |     2 +-
 drivers/media/pci/pluto2/pluto2.c                  |    14 +-
 drivers/media/pci/pt1/Makefile                     |     2 +-
 drivers/media/pci/pt1/pt1.c                        |    29 +-
 drivers/media/pci/pt1/va1j5jf8007s.c               |     4 +-
 drivers/media/pci/pt1/va1j5jf8007s.h               |     2 +-
 drivers/media/pci/pt1/va1j5jf8007t.c               |     6 +-
 drivers/media/pci/pt1/va1j5jf8007t.h               |     2 +-
 drivers/media/pci/pt3/Makefile                     |     1 -
 drivers/media/pci/pt3/pt3.c                        |     8 +-
 drivers/media/pci/pt3/pt3.h                        |     6 +-
 drivers/media/pci/pt3/pt3_i2c.c                    |     1 -
 drivers/media/pci/saa7134/Makefile                 |     2 -
 drivers/media/pci/saa7134/saa7134-cards.c          |    64 +-
 drivers/media/pci/saa7134/saa7134-dvb.c            |     6 +-
 drivers/media/pci/saa7134/saa7134-input.c          |     3 +-
 drivers/media/pci/saa7134/saa7134-video.c          |    10 +-
 drivers/media/pci/saa7134/saa7134.h                |     8 +-
 drivers/media/pci/saa7146/hexium_gemini.c          |    25 +-
 drivers/media/pci/saa7146/hexium_orion.c           |    21 +-
 drivers/media/pci/saa7146/mxb.c                    |    29 +-
 drivers/media/pci/saa7164/Makefile                 |     4 -
 drivers/media/pci/saa7164/saa7164-encoder.c        |    12 +-
 drivers/media/pci/saa7164/saa7164-vbi.c            |     6 +-
 drivers/media/pci/saa7164/saa7164.h                |    10 +-
 drivers/media/pci/smipcie/Makefile                 |     1 -
 drivers/media/pci/smipcie/smipcie.h                |    12 +-
 drivers/media/pci/solo6x10/solo6x10-core.c         |    17 +-
 drivers/media/pci/solo6x10/solo6x10-gpio.c         |     2 +
 drivers/media/pci/ttpci/Makefile                   |     2 +-
 drivers/media/pci/ttpci/av7110.c                   |     7 +-
 drivers/media/pci/ttpci/av7110.h                   |    16 +-
 drivers/media/pci/ttpci/av7110_av.c                |    24 +-
 drivers/media/pci/ttpci/av7110_ca.c                |     8 +-
 drivers/media/pci/ttpci/budget-av.c                |     8 +-
 drivers/media/pci/ttpci/budget-ci.c                |   212 +-
 drivers/media/pci/ttpci/budget.h                   |    12 +-
 drivers/media/pci/ttpci/dvb_filter.h               |     2 +-
 drivers/media/pci/tw5864/tw5864-video.c            |     2 +-
 drivers/media/pci/zoran/videocodec.c               |     1 -
 drivers/media/pci/zoran/zoran_driver.c             |    58 +-
 drivers/media/pci/zoran/zr36057.h                  |     4 +-
 drivers/media/platform/Kconfig                     |    38 +-
 drivers/media/platform/Makefile                    |    16 +-
 drivers/media/platform/am437x/am437x-vpfe.c        |     6 +-
 drivers/media/platform/arv.c                       |    54 +-
 drivers/media/platform/atmel/atmel-isc.c           |     6 +-
 drivers/media/platform/atmel/atmel-isi.c           |     2 +-
 drivers/media/platform/blackfin/ppi.c              |     2 +-
 drivers/media/platform/coda/coda-bit.c             |    23 +-
 drivers/media/platform/coda/coda-common.c          |    13 +-
 drivers/media/platform/coda/coda_regs.h            |     3 +-
 drivers/media/platform/coda/imx-vdoa.c             |     8 +-
 drivers/media/platform/davinci/dm355_ccdc.c        |     4 +-
 drivers/media/platform/davinci/dm355_ccdc_regs.h   |     6 +-
 drivers/media/platform/davinci/dm644x_ccdc.c       |     6 +-
 drivers/media/platform/davinci/dm644x_ccdc_regs.h  |     4 +-
 drivers/media/platform/davinci/isif_regs.h         |     6 +-
 drivers/media/platform/davinci/vpfe_capture.c      |    10 +-
 drivers/media/platform/davinci/vpif.h              |     4 +-
 drivers/media/platform/davinci/vpif_capture.c      |    10 +-
 drivers/media/platform/davinci/vpss.c              |    10 +-
 drivers/media/platform/exynos-gsc/gsc-m2m.c        |     8 +-
 drivers/media/platform/exynos4-is/Kconfig          |     2 +-
 drivers/media/platform/exynos4-is/fimc-core.c      |     4 +-
 drivers/media/platform/exynos4-is/fimc-core.h      |     2 +-
 drivers/media/platform/exynos4-is/fimc-isp.c       |    14 +-
 drivers/media/platform/exynos4-is/fimc-lite.c      |     7 +-
 drivers/media/platform/exynos4-is/fimc-lite.h      |     4 +-
 drivers/media/platform/exynos4-is/fimc-m2m.c       |    10 +-
 drivers/media/platform/exynos4-is/media-dev.c      |     4 +-
 drivers/media/platform/fsl-viu.c                   |    10 +-
 drivers/media/platform/m2m-deinterlace.c           |    16 +-
 .../media/platform/mtk-vcodec/mtk_vcodec_util.c    |     3 +
 drivers/media/platform/mtk-vcodec/vdec_vpu_if.h    |     2 +-
 drivers/media/platform/mx2_emmaprp.c               |     4 +-
 drivers/media/platform/omap/omap_vout.c            |    14 +-
 drivers/media/platform/omap3isp/isp.c              |     2 +-
 drivers/media/platform/omap3isp/ispvideo.c         |     4 +-
 drivers/media/platform/pxa_camera.c                |    35 +-
 drivers/media/platform/qcom/camss-8x16/camss.c     |     2 +-
 drivers/media/platform/qcom/venus/core.h           |     4 +-
 drivers/media/platform/qcom/venus/hfi_cmds.c       |    73 +-
 drivers/media/platform/qcom/venus/hfi_helper.h     |     4 +-
 drivers/media/platform/qcom/venus/venc.c           |    48 +-
 drivers/media/platform/qcom/venus/venc_ctrls.c     |    59 +-
 drivers/media/platform/rcar-vin/rcar-core.c        |     2 +-
 drivers/media/platform/rcar_drif.c                 |     4 +-
 drivers/media/platform/s3c-camif/camif-capture.c   |     4 +-
 drivers/media/platform/s5p-jpeg/jpeg-core.c        |     2 +-
 drivers/media/platform/s5p-mfc/s5p_mfc.c           |    18 +-
 drivers/media/platform/s5p-mfc/s5p_mfc_common.h    |     3 +
 drivers/media/platform/s5p-mfc/s5p_mfc_ctrl.c      |    10 +-
 drivers/media/platform/s5p-mfc/s5p_mfc_enc.c       |     2 +-
 drivers/media/platform/sh_veu.c                    |     2 +-
 drivers/media/platform/sh_vou.c                    |     2 +-
 .../platform/soc_camera/sh_mobile_ceu_camera.c     |     9 +-
 drivers/media/platform/soc_camera/soc_camera.c     |     8 +-
 drivers/media/platform/soc_camera/soc_scale_crop.c |     4 +
 drivers/media/platform/sti/bdisp/bdisp-debug.c     |     2 +-
 drivers/media/platform/sti/bdisp/bdisp-filter.h    |     2 +-
 drivers/media/platform/sti/bdisp/bdisp-hw.c        |     4 +-
 drivers/media/platform/sti/bdisp/bdisp-reg.h       |     2 +-
 drivers/media/platform/sti/bdisp/bdisp-v4l2.c      |     2 +-
 drivers/media/platform/sti/bdisp/bdisp.h           |     2 +-
 drivers/media/platform/sti/c8sectpfe/Kconfig       |     2 +-
 drivers/media/platform/sti/c8sectpfe/Makefile      |     5 +-
 .../platform/sti/c8sectpfe/c8sectpfe-common.c      |    15 +-
 .../platform/sti/c8sectpfe/c8sectpfe-common.h      |    13 +-
 .../media/platform/sti/c8sectpfe/c8sectpfe-core.c  |    19 +-
 .../media/platform/sti/c8sectpfe/c8sectpfe-core.h  |     5 +-
 .../platform/sti/c8sectpfe/c8sectpfe-debugfs.c     |     9 +-
 .../platform/sti/c8sectpfe/c8sectpfe-debugfs.h     |     9 +-
 .../media/platform/sti/c8sectpfe/c8sectpfe-dvb.c   |    11 +-
 .../media/platform/sti/c8sectpfe/c8sectpfe-dvb.h   |     5 +-
 drivers/media/platform/sti/cec/stih-cec.c          |     5 +-
 drivers/media/platform/sti/delta/delta-cfg.h       |     2 +-
 drivers/media/platform/sti/delta/delta-debug.c     |     2 +-
 drivers/media/platform/sti/delta/delta-debug.h     |     2 +-
 drivers/media/platform/sti/delta/delta-ipc.c       |     2 +-
 drivers/media/platform/sti/delta/delta-ipc.h       |     2 +-
 drivers/media/platform/sti/delta/delta-mem.c       |     2 +-
 drivers/media/platform/sti/delta/delta-mem.h       |     2 +-
 drivers/media/platform/sti/delta/delta-mjpeg-dec.c |     2 +-
 drivers/media/platform/sti/delta/delta-mjpeg-fw.h  |     2 +-
 drivers/media/platform/sti/delta/delta-mjpeg-hdr.c |     2 +-
 drivers/media/platform/sti/delta/delta-mjpeg.h     |     2 +-
 drivers/media/platform/sti/delta/delta-v4l2.c      |     2 +-
 drivers/media/platform/sti/delta/delta.h           |     2 +-
 drivers/media/platform/sti/hva/hva-debugfs.c       |     2 +-
 drivers/media/platform/sti/hva/hva-h264.c          |     2 +-
 drivers/media/platform/sti/hva/hva-hw.c            |     2 +-
 drivers/media/platform/sti/hva/hva-hw.h            |     2 +-
 drivers/media/platform/sti/hva/hva-mem.c           |     2 +-
 drivers/media/platform/sti/hva/hva-mem.h           |     2 +-
 drivers/media/platform/sti/hva/hva-v4l2.c          |     2 +-
 drivers/media/platform/sti/hva/hva.h               |     4 +-
 drivers/media/platform/stm32/stm32-dcmi.c          |     2 +-
 drivers/media/platform/tegra-cec/tegra_cec.c       |    22 +-
 drivers/media/platform/ti-vpe/cal.c                |     5 +-
 drivers/media/platform/via-camera.c                |     2 +-
 drivers/media/platform/via-camera.h                |     2 +-
 drivers/media/platform/vimc/vimc-common.h          |     5 +
 drivers/media/platform/vimc/vimc-sensor.c          |    65 +-
 drivers/media/platform/vivid/vivid-core.c          |     4 +-
 drivers/media/platform/vivid/vivid-core.h          |     5 +-
 drivers/media/platform/vivid/vivid-ctrls.c         |    35 +-
 drivers/media/platform/vivid/vivid-radio-rx.c      |    15 +-
 drivers/media/platform/vivid/vivid-radio-rx.h      |     2 +-
 drivers/media/platform/vivid/vivid-radio-tx.c      |    12 +-
 drivers/media/platform/vivid/vivid-radio-tx.h      |     2 +-
 drivers/media/platform/vivid/vivid-rds-gen.c       |     2 +-
 drivers/media/platform/vivid/vivid-rds-gen.h       |     1 +
 drivers/media/platform/vivid/vivid-vbi-gen.c       |     2 +-
 drivers/media/platform/vivid/vivid-vid-cap.c       |     9 +-
 drivers/media/platform/vsp1/vsp1_drm.c             |    37 +-
 drivers/media/platform/vsp1/vsp1_drv.c             |    16 +-
 drivers/media/platform/xilinx/xilinx-vipp.c        |    18 +-
 drivers/media/radio/radio-aimslab.c                |     2 +-
 drivers/media/radio/radio-aztech.c                 |     2 +-
 drivers/media/radio/radio-cadet.c                  |    14 +-
 drivers/media/radio/radio-gemtek.c                 |     8 +-
 drivers/media/radio/radio-maxiradio.c              |     2 +-
 drivers/media/radio/radio-mr800.c                  |    24 +-
 drivers/media/radio/radio-rtrack2.c                |     2 +-
 drivers/media/radio/radio-sf16fmi.c                |     4 +-
 drivers/media/radio/radio-sf16fmr2.c               |     2 +-
 drivers/media/radio/radio-si476x.c                 |    28 +-
 drivers/media/radio/radio-tea5764.c                |     2 +-
 drivers/media/radio/radio-terratec.c               |     6 +-
 drivers/media/radio/radio-wl1273.c                 |     6 +-
 drivers/media/radio/si470x/radio-si470x-common.c   |    34 +-
 drivers/media/radio/tea575x.c                      |     2 +-
 drivers/media/radio/wl128x/fmdrv_common.h          |    10 +-
 drivers/media/radio/wl128x/fmdrv_v4l2.c            |     4 +-
 drivers/media/rc/Kconfig                           |    35 +-
 drivers/media/rc/Makefile                          |     5 +-
 drivers/media/rc/iguanair.c                        |    19 +-
 drivers/media/rc/imon.c                            |    55 +-
 drivers/media/rc/ir-jvc-decoder.c                  |     1 +
 drivers/media/rc/ir-lirc-codec.c                   |   448 -
 drivers/media/rc/ir-mce_kbd-decoder.c              |    18 +-
 drivers/media/rc/ir-nec-decoder.c                  |    20 +-
 drivers/media/rc/ir-rc5-decoder.c                  |    44 +-
 drivers/media/rc/ir-rc6-decoder.c                  |    32 +-
 drivers/media/rc/ir-sanyo-decoder.c                |    38 +-
 drivers/media/rc/ir-sharp-decoder.c                |     1 +
 drivers/media/rc/ir-sony-decoder.c                 |     1 +
 drivers/media/rc/ir-spi.c                          |    15 +-
 drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c    |    17 +-
 drivers/media/rc/keymaps/rc-apac-viewcomp.c        |    17 +-
 drivers/media/rc/keymaps/rc-asus-pc39.c            |    17 +-
 drivers/media/rc/keymaps/rc-asus-ps3-100.c         |    17 +-
 drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c |    17 +-
 drivers/media/rc/keymaps/rc-avermedia-a16d.c       |    17 +-
 drivers/media/rc/keymaps/rc-avermedia-cardbus.c    |    17 +-
 drivers/media/rc/keymaps/rc-avermedia-dvbt.c       |    17 +-
 drivers/media/rc/keymaps/rc-avermedia-m135a.c      |    15 +-
 drivers/media/rc/keymaps/rc-avermedia.c            |    17 +-
 drivers/media/rc/keymaps/rc-avertv-303.c           |    17 +-
 drivers/media/rc/keymaps/rc-behold-columbus.c      |    23 +-
 drivers/media/rc/keymaps/rc-behold.c               |    17 +-
 drivers/media/rc/keymaps/rc-budget-ci-old.c        |    17 +-
 drivers/media/rc/keymaps/rc-cinergy-1400.c         |    17 +-
 drivers/media/rc/keymaps/rc-cinergy.c              |    17 +-
 drivers/media/rc/keymaps/rc-dib0700-nec.c          |    27 +-
 drivers/media/rc/keymaps/rc-dib0700-rc5.c          |    27 +-
 drivers/media/rc/keymaps/rc-dm1105-nec.c           |    17 +-
 drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c      |    17 +-
 drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c   |    17 +-
 drivers/media/rc/keymaps/rc-em-terratec.c          |    17 +-
 drivers/media/rc/keymaps/rc-encore-enltv-fm53.c    |    17 +-
 drivers/media/rc/keymaps/rc-encore-enltv.c         |    17 +-
 drivers/media/rc/keymaps/rc-encore-enltv2.c        |    17 +-
 drivers/media/rc/keymaps/rc-evga-indtube.c         |    17 +-
 drivers/media/rc/keymaps/rc-eztv.c                 |    17 +-
 drivers/media/rc/keymaps/rc-flydvb.c               |    17 +-
 drivers/media/rc/keymaps/rc-flyvideo.c             |    17 +-
 drivers/media/rc/keymaps/rc-fusionhdtv-mce.c       |    17 +-
 drivers/media/rc/keymaps/rc-gadmei-rm008z.c        |    17 +-
 drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c   |    17 +-
 drivers/media/rc/keymaps/rc-gotview7135.c          |    17 +-
 drivers/media/rc/keymaps/rc-hauppauge.c            |    29 +-
 drivers/media/rc/keymaps/rc-iodata-bctv7e.c        |    17 +-
 drivers/media/rc/keymaps/rc-kaiomy.c               |    17 +-
 drivers/media/rc/keymaps/rc-kworld-315u.c          |    17 +-
 .../media/rc/keymaps/rc-kworld-plus-tv-analog.c    |    17 +-
 drivers/media/rc/keymaps/rc-manli.c                |    17 +-
 drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c  |    17 +-
 drivers/media/rc/keymaps/rc-msi-tvanywhere.c       |    17 +-
 drivers/media/rc/keymaps/rc-nebula.c               |    17 +-
 .../media/rc/keymaps/rc-nec-terratec-cinergy-xs.c  |    17 +-
 drivers/media/rc/keymaps/rc-norwood.c              |    17 +-
 drivers/media/rc/keymaps/rc-npgtech.c              |    17 +-
 drivers/media/rc/keymaps/rc-pctv-sedna.c           |    17 +-
 drivers/media/rc/keymaps/rc-pinnacle-color.c       |    17 +-
 drivers/media/rc/keymaps/rc-pinnacle-grey.c        |    17 +-
 drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c     |    17 +-
 drivers/media/rc/keymaps/rc-pixelview-002t.c       |    17 +-
 drivers/media/rc/keymaps/rc-pixelview-mk12.c       |    17 +-
 drivers/media/rc/keymaps/rc-pixelview-new.c        |    17 +-
 drivers/media/rc/keymaps/rc-pixelview.c            |    17 +-
 .../media/rc/keymaps/rc-powercolor-real-angel.c    |    17 +-
 drivers/media/rc/keymaps/rc-proteus-2309.c         |    17 +-
 drivers/media/rc/keymaps/rc-purpletv.c             |    17 +-
 drivers/media/rc/keymaps/rc-pv951.c                |    17 +-
 .../media/rc/keymaps/rc-real-audio-220-32-keys.c   |    17 +-
 drivers/media/rc/keymaps/rc-tbs-nec.c              |    17 +-
 drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c  |    17 +-
 drivers/media/rc/keymaps/rc-tevii-nec.c            |    17 +-
 drivers/media/rc/keymaps/rc-tt-1500.c              |    17 +-
 drivers/media/rc/keymaps/rc-videomate-s350.c       |    17 +-
 drivers/media/rc/keymaps/rc-videomate-tv-pvr.c     |    17 +-
 drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c |    19 +-
 drivers/media/rc/keymaps/rc-winfast.c              |    17 +-
 drivers/media/rc/lirc_dev.c                        |   981 +-
 drivers/media/rc/rc-core-priv.h                    |    72 +-
 drivers/media/rc/rc-ir-raw.c                       |    86 +-
 drivers/media/rc/rc-main.c                         |   248 +-
 drivers/media/rc/winbond-cir.c                     |     2 +-
 drivers/media/tuners/Kconfig                       |     7 +
 drivers/media/tuners/Makefile                      |     2 +-
 drivers/media/tuners/e4000.h                       |     2 +-
 drivers/media/tuners/fc0011.h                      |     2 +-
 drivers/media/tuners/fc0012.h                      |     2 +-
 drivers/media/tuners/fc0013.h                      |     2 +-
 drivers/media/tuners/fc2580.h                      |     2 +-
 drivers/media/tuners/it913x.h                      |     2 +-
 drivers/media/tuners/m88rs6000t.h                  |     2 +-
 drivers/media/tuners/max2165.c                     |     2 +-
 drivers/media/tuners/mc44s803.c                    |     2 +-
 drivers/media/tuners/mt2060.c                      |     2 +-
 drivers/media/tuners/mt2063.c                      |     4 +-
 drivers/media/tuners/mt2063.h                      |     2 +-
 drivers/media/tuners/mt20xx.h                      |     2 +-
 drivers/media/tuners/mt2131.c                      |     2 +-
 drivers/media/tuners/mt2266.c                      |     2 +-
 drivers/media/tuners/mxl301rf.h                    |     2 +-
 drivers/media/tuners/mxl5005s.c                    |    25 +-
 drivers/media/tuners/mxl5005s.h                    |     2 +-
 drivers/media/tuners/mxl5007t.h                    |     2 +-
 drivers/media/tuners/qm1d1c0042.h                  |     2 +-
 drivers/media/tuners/qt1010.c                      |     4 +-
 drivers/media/tuners/qt1010.h                      |     2 +-
 drivers/media/tuners/r820t.c                       |    69 +-
 drivers/media/tuners/r820t.h                       |     2 +-
 drivers/media/tuners/si2157.c                      |     2 +-
 drivers/media/tuners/si2157.h                      |     2 +-
 drivers/media/tuners/tda18212.h                    |     2 +-
 drivers/media/tuners/tda18218.h                    |     2 +-
 drivers/media/tuners/tda18250.c                    |   902 +
 drivers/media/tuners/tda18250.h                    |    51 +
 drivers/media/tuners/tda18250_priv.h               |   145 +
 drivers/media/tuners/tda18271.h                    |     2 +-
 drivers/media/tuners/tda827x.h                     |     4 +-
 drivers/media/tuners/tda8290.c                     |    78 +-
 drivers/media/tuners/tda8290.h                     |     2 +-
 drivers/media/tuners/tda9887.c                     |     4 +-
 drivers/media/tuners/tda9887.h                     |     2 +-
 drivers/media/tuners/tea5761.c                     |    15 +-
 drivers/media/tuners/tea5761.h                     |     2 +-
 drivers/media/tuners/tea5767.c                     |    21 +-
 drivers/media/tuners/tea5767.h                     |     2 +-
 drivers/media/tuners/tua9001.h                     |     2 +-
 drivers/media/tuners/tuner-i2c.h                   |     2 +-
 drivers/media/tuners/tuner-simple.c                |     2 +-
 drivers/media/tuners/tuner-simple.h                |     2 +-
 drivers/media/tuners/tuner-xc2028-types.h          |     5 +-
 drivers/media/tuners/tuner-xc2028.c                |    26 +-
 drivers/media/tuners/tuner-xc2028.h                |     9 +-
 drivers/media/tuners/xc4000.c                      |     7 +-
 drivers/media/tuners/xc5000.c                      |    24 +-
 drivers/media/usb/as102/Makefile                   |     1 -
 drivers/media/usb/as102/as102_drv.c                |     2 +-
 drivers/media/usb/as102/as102_drv.h                |     6 +-
 drivers/media/usb/as102/as10x_cmd_cfg.c            |     6 +-
 drivers/media/usb/au0828/Kconfig                   |     2 +-
 drivers/media/usb/au0828/Makefile                  |     1 -
 drivers/media/usb/au0828/au0828-cards.h            |     2 +-
 drivers/media/usb/au0828/au0828-input.c            |    25 +-
 drivers/media/usb/au0828/au0828-video.c            |     2 +-
 drivers/media/usb/au0828/au0828.h                  |    18 +-
 drivers/media/usb/b2c2/Makefile                    |     1 -
 drivers/media/usb/cpia2/cpia2.h                    |     2 +-
 drivers/media/usb/cpia2/cpia2_core.c               |     8 +-
 drivers/media/usb/cpia2/cpia2_usb.c                |    14 +-
 drivers/media/usb/cpia2/cpia2_v4l.c                |     8 +-
 drivers/media/usb/cx231xx/Makefile                 |     2 -
 drivers/media/usb/cx231xx/cx231xx-417.c            |    10 +-
 drivers/media/usb/cx231xx/cx231xx-audio.c          |     6 +-
 drivers/media/usb/cx231xx/cx231xx-avcore.c         |     4 +-
 drivers/media/usb/cx231xx/cx231xx-cards.c          |    30 +-
 drivers/media/usb/cx231xx/cx231xx-core.c           |     2 +-
 drivers/media/usb/cx231xx/cx231xx-dvb.c            |     8 +-
 drivers/media/usb/cx231xx/cx231xx-i2c.c            |     2 +-
 drivers/media/usb/cx231xx/cx231xx-input.c          |    28 +-
 drivers/media/usb/cx231xx/cx231xx-pcb-cfg.h        |     2 +-
 drivers/media/usb/cx231xx/cx231xx-reg.h            |    20 +-
 drivers/media/usb/cx231xx/cx231xx-video.c          |    20 +-
 drivers/media/usb/cx231xx/cx231xx.h                |     1 +
 drivers/media/usb/dvb-usb-v2/Makefile              |     1 -
 drivers/media/usb/dvb-usb-v2/anysee.h              |     2 +-
 drivers/media/usb/dvb-usb-v2/az6007.c              |     2 +-
 drivers/media/usb/dvb-usb-v2/dvb_usb.h             |    10 +-
 drivers/media/usb/dvb-usb-v2/dvbsky.c              |    92 +-
 drivers/media/usb/dvb-usb-v2/lmedm04.c             |    39 +-
 drivers/media/usb/dvb-usb-v2/mxl111sf-demod.c      |     9 +-
 drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h      |     2 +-
 drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h      |     2 +-
 drivers/media/usb/dvb-usb/Kconfig                  |     2 +
 drivers/media/usb/dvb-usb/Makefile                 |     1 -
 drivers/media/usb/dvb-usb/az6027.c                 |   218 +-
 drivers/media/usb/dvb-usb/cxusb.c                  |    10 +-
 drivers/media/usb/dvb-usb/dib0700.h                |     2 +
 drivers/media/usb/dvb-usb/dib0700_core.c           |    26 +-
 drivers/media/usb/dvb-usb/dib0700_devices.c        |   110 +-
 drivers/media/usb/dvb-usb/dvb-usb.h                |    10 +-
 drivers/media/usb/dvb-usb/dw2102.c                 |     2 +-
 drivers/media/usb/dvb-usb/friio-fe.c               |     5 +-
 drivers/media/usb/dvb-usb/pctv452e.c               |    10 +-
 drivers/media/usb/dvb-usb/ttusb2.c                 |     2 +-
 drivers/media/usb/em28xx/Kconfig                   |    14 +-
 drivers/media/usb/em28xx/Makefile                  |     2 -
 drivers/media/usb/em28xx/em28xx-cards.c            |     1 -
 drivers/media/usb/em28xx/em28xx-dvb.c              |     9 +-
 drivers/media/usb/gspca/autogain_functions.c       |    28 +-
 drivers/media/usb/gspca/benq.c                     |     8 +-
 drivers/media/usb/gspca/conex.c                    |    13 +-
 drivers/media/usb/gspca/cpia1.c                    |    76 +-
 drivers/media/usb/gspca/dtcs033.c                  |    28 +-
 drivers/media/usb/gspca/etoms.c                    |    38 +-
 drivers/media/usb/gspca/finepix.c                  |     4 +-
 drivers/media/usb/gspca/gl860/gl860.c              |    37 +-
 drivers/media/usb/gspca/gspca.c                    |   171 +-
 drivers/media/usb/gspca/gspca.h                    |     9 +-
 drivers/media/usb/gspca/jeilinj.c                  |    19 +-
 drivers/media/usb/gspca/jl2005bcd.c                |    45 +-
 drivers/media/usb/gspca/kinect.c                   |    11 +-
 drivers/media/usb/gspca/konica.c                   |    28 +-
 drivers/media/usb/gspca/m5602/m5602_core.c         |    34 +-
 drivers/media/usb/gspca/m5602/m5602_mt9m111.c      |    21 +-
 drivers/media/usb/gspca/m5602/m5602_ov7660.c       |    11 +-
 drivers/media/usb/gspca/m5602/m5602_ov9650.c       |    26 +-
 drivers/media/usb/gspca/m5602/m5602_po1030.c       |    27 +-
 drivers/media/usb/gspca/m5602/m5602_s5k4aa.c       |    16 +-
 drivers/media/usb/gspca/m5602/m5602_s5k83a.c       |     2 +-
 drivers/media/usb/gspca/mars.c                     |     4 +-
 drivers/media/usb/gspca/mr97310a.c                 |    29 +-
 drivers/media/usb/gspca/nw80x.c                    |    24 +-
 drivers/media/usb/gspca/ov519.c                    |   155 +-
 drivers/media/usb/gspca/ov534.c                    |    25 +-
 drivers/media/usb/gspca/ov534_9.c                  |    23 +-
 drivers/media/usb/gspca/pac207.c                   |    16 +-
 drivers/media/usb/gspca/pac7302.c                  |     2 +-
 drivers/media/usb/gspca/pac7311.c                  |     2 +-
 drivers/media/usb/gspca/pac_common.h               |     7 +-
 drivers/media/usb/gspca/sn9c2028.c                 |    34 +-
 drivers/media/usb/gspca/sn9c2028.h                 |     7 +-
 drivers/media/usb/gspca/sn9c20x.c                  |     2 +-
 drivers/media/usb/gspca/sonixj.c                   |    58 +-
 drivers/media/usb/gspca/spca1528.c                 |    17 +-
 drivers/media/usb/gspca/spca500.c                  |    66 +-
 drivers/media/usb/gspca/spca501.c                  |    10 +-
 drivers/media/usb/gspca/spca505.c                  |     6 +-
 drivers/media/usb/gspca/spca506.c                  |    16 +-
 drivers/media/usb/gspca/spca508.c                  |    20 +-
 drivers/media/usb/gspca/spca561.c                  |    20 +-
 drivers/media/usb/gspca/sq905.c                    |    16 +-
 drivers/media/usb/gspca/sq905c.c                   |    37 +-
 drivers/media/usb/gspca/sq930x.c                   |    29 +-
 drivers/media/usb/gspca/stk014.c                   |     6 +-
 drivers/media/usb/gspca/stk1135.c                  |    15 +-
 drivers/media/usb/gspca/stv0680.c                  |    36 +-
 drivers/media/usb/gspca/stv06xx/stv06xx.c          |    76 +-
 drivers/media/usb/gspca/stv06xx/stv06xx.h          |     2 +-
 drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c     |    10 +-
 drivers/media/usb/gspca/stv06xx/stv06xx_pb0100.c   |    23 +-
 drivers/media/usb/gspca/stv06xx/stv06xx_st6422.c   |     2 +-
 drivers/media/usb/gspca/stv06xx/stv06xx_vv6410.c   |    16 +-
 drivers/media/usb/gspca/sunplus.c                  |    36 +-
 drivers/media/usb/gspca/t613.c                     |    24 +-
 drivers/media/usb/gspca/topro.c                    |     6 +-
 drivers/media/usb/gspca/touptek.c                  |    89 +-
 drivers/media/usb/gspca/vc032x.c                   |    51 +-
 drivers/media/usb/gspca/w996Xcf.c                  |     9 +-
 drivers/media/usb/gspca/xirlink_cit.c              |    32 +-
 drivers/media/usb/gspca/zc3xx.c                    |    83 +-
 drivers/media/usb/hdpvr/Makefile                   |     4 -
 drivers/media/usb/hdpvr/hdpvr-core.c               |    37 +-
 drivers/media/usb/hdpvr/hdpvr-i2c.c                |    23 +-
 drivers/media/usb/hdpvr/hdpvr-video.c              |    36 +-
 drivers/media/usb/hdpvr/hdpvr.h                    |    19 +-
 drivers/media/usb/pulse8-cec/pulse8-cec.c          |     4 +-
 drivers/media/usb/pvrusb2/Makefile                 |     2 -
 drivers/media/usb/pvrusb2/pvrusb2-devattr.c        |    12 +-
 drivers/media/usb/pvrusb2/pvrusb2-dvb.c            |     2 +-
 drivers/media/usb/pvrusb2/pvrusb2-dvb.h            |     8 +-
 drivers/media/usb/pvrusb2/pvrusb2-hdw.c            |    15 +-
 drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c       |    13 +-
 drivers/media/usb/pvrusb2/pvrusb2-v4l2.c           |    44 +-
 drivers/media/usb/pwc/pwc.h                        |     6 +-
 drivers/media/usb/s2255/s2255drv.c                 |    13 +-
 drivers/media/usb/siano/Makefile                   |     1 -
 drivers/media/usb/siano/smsusb.c                   |     2 +-
 drivers/media/usb/stk1160/Makefile                 |     4 +-
 drivers/media/usb/stkwebcam/stk-sensor.c           |    44 +-
 drivers/media/usb/stkwebcam/stk-webcam.c           |    10 +-
 drivers/media/usb/tm6000/Makefile                  |     2 -
 drivers/media/usb/tm6000/tm6000-alsa.c             |    18 +-
 drivers/media/usb/tm6000/tm6000-cards.c            |    21 +-
 drivers/media/usb/tm6000/tm6000-core.c             |    24 +-
 drivers/media/usb/tm6000/tm6000-i2c.c              |    24 +-
 drivers/media/usb/tm6000/tm6000-regs.h             |    14 +-
 drivers/media/usb/tm6000/tm6000-stds.c             |    18 +-
 drivers/media/usb/tm6000/tm6000-usb-isoc.h         |    14 +-
 drivers/media/usb/tm6000/tm6000-video.c            |    50 +-
 drivers/media/usb/tm6000/tm6000.h                  |    24 +-
 drivers/media/usb/ttusb-budget/Makefile            |     2 +-
 drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c  |    26 +-
 drivers/media/usb/ttusb-dec/Makefile               |     2 -
 drivers/media/usb/ttusb-dec/ttusb_dec.c            |    36 +-
 drivers/media/usb/ttusb-dec/ttusbdecfe.c           |     2 +-
 drivers/media/usb/usbtv/Kconfig                    |    16 +-
 drivers/media/usb/usbvision/Makefile               |     1 -
 drivers/media/usb/usbvision/usbvision-video.c      |     3 +-
 drivers/media/usb/uvc/Makefile                     |     2 +-
 drivers/media/usb/uvc/uvc_driver.c                 |   243 +-
 drivers/media/usb/uvc/uvc_isight.c                 |    12 +-
 drivers/media/usb/uvc/uvc_metadata.c               |   179 +
 drivers/media/usb/uvc/uvc_queue.c                  |    48 +-
 drivers/media/usb/uvc/uvc_status.c                 |     5 +-
 drivers/media/usb/uvc/uvc_v4l2.c                   |    61 +-
 drivers/media/usb/uvc/uvc_video.c                  |   187 +-
 drivers/media/usb/uvc/uvcvideo.h                   |    37 +-
 drivers/media/usb/zr364xx/zr364xx.c                |     4 +-
 drivers/media/v4l2-core/Kconfig                    |    37 +-
 drivers/media/v4l2-core/Makefile                   |    11 +-
 drivers/media/v4l2-core/v4l2-async.c               |    16 +-
 drivers/media/v4l2-core/v4l2-common.c              |    27 +-
 drivers/media/v4l2-core/v4l2-compat-ioctl32.c      |  1040 +-
 drivers/media/v4l2-core/v4l2-ctrls.c               |     4 +-
 drivers/media/v4l2-core/v4l2-dev.c                 |    14 +-
 drivers/media/v4l2-core/v4l2-dv-timings.c          |     2 +-
 drivers/media/v4l2-core/v4l2-fwnode.c              |    10 +-
 drivers/media/v4l2-core/v4l2-ioctl.c               |   212 +-
 drivers/media/v4l2-core/v4l2-mc.c                  |     2 -
 drivers/media/v4l2-core/v4l2-mem2mem.c             |    24 +-
 drivers/media/v4l2-core/v4l2-subdev.c              |     6 +-
 drivers/media/v4l2-core/videobuf-core.c            |    16 +-
 drivers/memory/Kconfig                             |    10 +
 drivers/memory/Makefile                            |     8 +
 drivers/memory/Makefile.asm-offsets                |     5 +
 drivers/memory/brcmstb_dpfe.c                      |    74 +-
 drivers/memory/emif-asm-offsets.c                  |    92 +
 drivers/memory/emif.h                              |    17 +
 drivers/memory/omap-gpmc.c                         |   163 +-
 drivers/memory/tegra/Makefile                      |     1 +
 drivers/memory/tegra/tegra114.c                    |    15 +
 drivers/memory/tegra/tegra124.c                    |    17 +
 drivers/memory/tegra/tegra186.c                    |   600 +
 drivers/memory/tegra/tegra210.c                    |    15 +
 drivers/memory/tegra/tegra30.c                     |    15 +
 drivers/memory/ti-emif-pm.c                        |   324 +
 drivers/memory/ti-emif-sram-pm.S                   |   334 +
 drivers/memstick/host/Kconfig                      |     4 +-
 drivers/memstick/host/rtsx_pci_ms.c                |     2 +-
 drivers/memstick/host/rtsx_usb_ms.c                |     2 +-
 drivers/message/fusion/mptbase.c                   |    59 +-
 drivers/message/fusion/mptctl.c                    |    27 +-
 drivers/message/fusion/mptsas.c                    |     1 -
 drivers/mfd/Kconfig                                |    41 +-
 drivers/mfd/Makefile                               |     7 +-
 drivers/mfd/ab8500-debugfs.c                       |   429 +-
 drivers/mfd/atmel-flexcom.c                        |    63 +-
 drivers/mfd/axp20x.c                               |     4 +
 drivers/mfd/cros_ec.c                              |     4 +-
 drivers/{platform/chrome => mfd}/cros_ec_dev.c     |     8 +-
 drivers/{platform/chrome => mfd}/cros_ec_dev.h     |     0
 drivers/mfd/cros_ec_spi.c                          |    25 +-
 drivers/mfd/intel-lpss.c                           |     6 +-
 drivers/mfd/intel_soc_pmic_core.c                  |     1 -
 drivers/mfd/kempld-core.c                          |     2 +-
 drivers/mfd/lpc_ich.c                              |     5 -
 drivers/mfd/max77843.c                             |     1 -
 drivers/mfd/palmas.c                               |    10 +-
 drivers/mfd/pcf50633-core.c                        |     2 +-
 drivers/mfd/rave-sp.c                              |   710 +
 drivers/mfd/stm32-lptimer.c                        |     6 +-
 drivers/mfd/stm32-timers.c                         |     4 +-
 drivers/mfd/syscon.c                               |    19 +
 drivers/mfd/ti_am335x_tscadc.c                     |     2 +-
 drivers/mfd/tmio_core.c                            |    20 +
 drivers/misc/Kconfig                               |    10 +-
 drivers/misc/Makefile                              |     2 +
 drivers/misc/ad525x_dpot.c                         |    30 +-
 drivers/misc/ad525x_dpot.h                         |    12 +-
 drivers/misc/apds990x.c                            |    11 +
 drivers/misc/cardreader/Kconfig                    |    20 +
 drivers/misc/cardreader/Makefile                   |     4 +
 drivers/{mfd => misc/cardreader}/rtl8411.c         |     2 +-
 drivers/{mfd => misc/cardreader}/rts5209.c         |     2 +-
 drivers/{mfd => misc/cardreader}/rts5227.c         |     2 +-
 drivers/{mfd => misc/cardreader}/rts5229.c         |     2 +-
 drivers/{mfd => misc/cardreader}/rts5249.c         |     3 +-
 drivers/misc/cardreader/rts5260.c                  |   748 +
 drivers/misc/cardreader/rts5260.h                  |    45 +
 drivers/{mfd => misc/cardreader}/rtsx_pcr.c        |   125 +-
 drivers/{mfd => misc/cardreader}/rtsx_pcr.h        |    12 +-
 drivers/{mfd => misc/cardreader}/rtsx_usb.c        |     2 +-
 drivers/misc/cxl/api.c                             |     2 +-
 drivers/misc/cxl/context.c                         |     2 +
 drivers/misc/cxl/cxl.h                             |     5 +-
 drivers/misc/cxl/cxllib.c                          |     3 +-
 drivers/misc/cxl/file.c                            |    23 +-
 drivers/misc/cxl/native.c                          |    13 +-
 drivers/misc/cxl/pci.c                             |     2 -
 drivers/misc/cxl/vphb.c                            |     2 +-
 drivers/misc/ds1682.c                              |    35 +-
 drivers/misc/eeprom/Kconfig                        |     1 +
 drivers/misc/eeprom/at24.c                         |   731 +-
 drivers/misc/eeprom/at25.c                         |     3 +
 drivers/misc/enclosure.c                           |    12 +-
 drivers/misc/fsa9480.c                             |     1 +
 drivers/misc/genwqe/card_base.c                    |    16 +-
 drivers/misc/genwqe/card_base.h                    |    20 +-
 drivers/misc/genwqe/card_ddcb.c                    |    22 +-
 drivers/misc/genwqe/card_debugfs.c                 |     2 +-
 drivers/misc/genwqe/card_dev.c                     |    19 +-
 drivers/misc/genwqe/card_utils.c                   |    25 +-
 drivers/misc/hpilo.c                               |    11 +-
 drivers/misc/hpilo.h                               |     5 +-
 drivers/misc/ics932s401.c                          |     8 +-
 drivers/misc/isl29003.c                            |     7 +
 drivers/misc/lis3lv02d/lis3lv02d.c                 |     4 +-
 drivers/misc/lkdtm.h                               |     4 +-
 drivers/misc/lkdtm_core.c                          |     6 +-
 drivers/misc/lkdtm_heap.c                          |     4 +
 drivers/misc/lkdtm_usercopy.c                      |   101 +-
 drivers/misc/mei/bus.c                             |     4 +-
 drivers/misc/mei/client.c                          |     6 +
 drivers/misc/mei/hw-me-regs.h                      |     5 +
 drivers/misc/mei/hw-me.c                           |     4 +-
 drivers/misc/mei/hw-txe.c                          |     4 +-
 drivers/misc/mei/init.c                            |     4 +-
 drivers/misc/mei/main.c                            |    18 +-
 drivers/misc/mei/pci-me.c                          |    10 +-
 drivers/misc/mic/cosm/cosm_scif_server.c           |    16 +-
 drivers/misc/mic/cosm_client/cosm_scif_client.c    |     4 +-
 drivers/misc/mic/scif/scif_api.c                   |    31 +-
 drivers/misc/mic/scif/scif_epd.h                   |     2 +-
 drivers/misc/mic/scif/scif_fd.c                    |     2 +-
 drivers/misc/mic/vop/vop_main.c                    |    20 +-
 drivers/misc/mic/vop/vop_vringh.c                  |    34 +-
 drivers/misc/ocxl/Kconfig                          |    31 +
 drivers/misc/ocxl/Makefile                         |    11 +
 drivers/misc/ocxl/afu_irq.c                        |   202 +
 drivers/misc/ocxl/config.c                         |   723 +
 drivers/misc/ocxl/context.c                        |   279 +
 drivers/misc/ocxl/file.c                           |   461 +
 drivers/misc/ocxl/link.c                           |   647 +
 drivers/misc/ocxl/main.c                           |    33 +
 drivers/misc/ocxl/ocxl_internal.h                  |   131 +
 drivers/misc/ocxl/pasid.c                          |   107 +
 drivers/misc/ocxl/pci.c                            |   585 +
 drivers/misc/ocxl/sysfs.c                          |   142 +
 drivers/misc/ocxl/trace.c                          |     6 +
 drivers/misc/ocxl/trace.h                          |   182 +
 drivers/misc/phantom.c                             |     8 +-
 drivers/misc/sgi-gru/grutlbpurge.c                 |     1 +
 drivers/misc/vexpress-syscfg.c                     |     4 +-
 drivers/misc/vmw_vmci/vmci_host.c                  |     6 +-
 drivers/misc/vmw_vmci/vmci_queue_pair.c            |   179 +-
 drivers/mmc/core/block.c                           |  1404 +-
 drivers/mmc/core/block.h                           |    12 +-
 drivers/mmc/core/bus.c                             |     2 -
 drivers/mmc/core/card.h                            |     1 +
 drivers/mmc/core/core.c                            |   228 +-
 drivers/mmc/core/core.h                            |    47 +-
 drivers/mmc/core/host.h                            |     6 +-
 drivers/mmc/core/mmc_ops.c                         |     4 -
 drivers/mmc/core/mmc_test.c                        |   221 +-
 drivers/mmc/core/queue.c                           |   504 +-
 drivers/mmc/core/queue.h                           |    64 +-
 drivers/mmc/core/quirks.h                          |     6 +
 drivers/mmc/core/slot-gpio.c                       |    22 +-
 drivers/mmc/host/Kconfig                           |    31 +-
 drivers/mmc/host/Makefile                          |    11 +-
 drivers/mmc/host/android-goldfish.c                |     2 -
 drivers/mmc/host/bcm2835.c                         |     3 +-
 drivers/mmc/host/cqhci.c                           |  1150 +
 drivers/mmc/host/cqhci.h                           |   240 +
 drivers/mmc/host/davinci_mmc.c                     |    17 +-
 drivers/mmc/host/dw_mmc-exynos.c                   |     9 +-
 drivers/mmc/host/dw_mmc-k3.c                       |     4 +
 drivers/mmc/host/dw_mmc-rockchip.c                 |     1 +
 drivers/mmc/host/dw_mmc-zx.c                       |     1 +
 drivers/mmc/host/dw_mmc.c                          |    99 +-
 drivers/mmc/host/dw_mmc.h                          |     2 +
 drivers/mmc/host/meson-gx-mmc.c                    |    22 +-
 drivers/mmc/host/mmci.c                            |   127 +-
 drivers/mmc/host/mmci.h                            |     6 +
 drivers/mmc/host/renesas_sdhi.h                    |    22 +
 drivers/mmc/host/renesas_sdhi_core.c               |    49 +-
 drivers/mmc/host/renesas_sdhi_internal_dmac.c      |    15 +-
 drivers/mmc/host/renesas_sdhi_sys_dmac.c           |    37 +-
 drivers/mmc/host/rtsx_pci_sdmmc.c                  |     2 +-
 drivers/mmc/host/rtsx_usb_sdmmc.c                  |     2 +-
 drivers/mmc/host/s3cmci.c                          |     2 +-
 drivers/mmc/host/sdhci-acpi.c                      |   132 +-
 drivers/mmc/host/sdhci-esdhc-imx.c                 |    22 +-
 drivers/mmc/host/sdhci-of-arasan.c                 |   149 +-
 drivers/mmc/host/sdhci-of-esdhc.c                  |     8 +
 drivers/mmc/host/sdhci-pci-arasan.c                |   331 +
 drivers/mmc/host/sdhci-pci-core.c                  |   253 +-
 drivers/mmc/host/sdhci-pci.h                       |     7 +-
 drivers/mmc/host/sdhci-spear.c                     |     4 +
 drivers/mmc/host/sdhci-xenon.c                     |     7 +
 drivers/mmc/host/sdhci.c                           |   221 +-
 drivers/mmc/host/sdhci.h                           |     5 +-
 drivers/mmc/host/sdhci_f_sdh30.c                   |    52 +-
 drivers/mmc/host/sh_mmcif.c                        |     2 +-
 drivers/mmc/host/sunxi-mmc.c                       |     9 +-
 drivers/mmc/host/tmio_mmc.c                        |    23 +-
 drivers/mmc/host/tmio_mmc.h                        |    43 +-
 drivers/mmc/host/tmio_mmc_core.c                   |   152 +-
 drivers/mtd/chips/jedec_probe.c                    |     2 +
 drivers/mtd/devices/docg3.c                        |    70 -
 drivers/mtd/devices/m25p80.c                       |     9 +
 drivers/mtd/devices/mchp23k256.c                   |    18 +-
 drivers/mtd/mtdchar.c                              |     4 +-
 drivers/mtd/mtdcore.c                              |    36 +-
 drivers/mtd/mtdpart.c                              |    43 +-
 drivers/mtd/mtdswap.c                              |     5 +-
 drivers/mtd/nand/Kconfig                           |    17 +-
 drivers/mtd/nand/Makefile                          |     1 +
 drivers/mtd/nand/atmel/nand-controller.c           |     9 +-
 drivers/mtd/nand/atmel/pmecc.c                     |     2 +-
 drivers/mtd/nand/bf5xx_nand.c                      |     6 +-
 drivers/mtd/nand/brcmnand/brcmnand.c               |    38 +-
 drivers/mtd/nand/cafe_nand.c                       |    52 +-
 drivers/mtd/nand/denali.c                          |    84 +-
 drivers/mtd/nand/denali.h                          |     4 +-
 drivers/mtd/nand/denali_pci.c                      |     4 +
 drivers/mtd/nand/diskonchip.c                      |     4 +-
 drivers/mtd/nand/docg4.c                           |    21 +-
 drivers/mtd/nand/fsl_elbc_nand.c                   |    10 +-
 drivers/mtd/nand/fsl_ifc_nand.c                    |    45 +-
 drivers/mtd/nand/fsmc_nand.c                       |     9 +-
 drivers/mtd/nand/gpmi-nand/gpmi-nand.c             |   111 +-
 drivers/mtd/nand/gpmi-nand/gpmi-nand.h             |    46 +-
 drivers/mtd/nand/hisi504_nand.c                    |     9 +-
 drivers/mtd/nand/jz4740_nand.c                     |    16 +-
 drivers/mtd/nand/lpc32xx_mlc.c                     |     7 +-
 drivers/mtd/nand/lpc32xx_slc.c                     |    33 +-
 drivers/mtd/nand/marvell_nand.c                    |  2896 ++
 drivers/mtd/nand/mtk_ecc.c                         |   126 +-
 drivers/mtd/nand/mtk_ecc.h                         |     3 +-
 drivers/mtd/nand/mtk_nand.c                        |    76 +-
 drivers/mtd/nand/nand_base.c                       |  2603 +-
 drivers/mtd/nand/nand_bbt.c                        |     2 +-
 drivers/mtd/nand/nand_hynix.c                      |   129 +-
 drivers/mtd/nand/nand_micron.c                     |    83 +-
 drivers/mtd/nand/nand_samsung.c                    |    19 +
 drivers/mtd/nand/nand_timings.c                    |    21 +-
 drivers/mtd/nand/omap2.c                           |    28 +-
 drivers/mtd/nand/pxa3xx_nand.c                     |    14 +-
 drivers/mtd/nand/qcom_nandc.c                      |    31 +-
 drivers/mtd/nand/r852.c                            |    11 +-
 drivers/mtd/nand/sh_flctl.c                        |     6 +-
 drivers/mtd/nand/sm_common.h                       |     2 +-
 drivers/mtd/nand/sunxi_nand.c                      |   111 +-
 drivers/mtd/nand/tango_nand.c                      |    27 +-
 drivers/mtd/nand/tmio_nand.c                       |     5 +-
 drivers/mtd/nand/vf610_nfc.c                       |    12 +-
 drivers/mtd/onenand/Kconfig                        |     7 +-
 drivers/mtd/onenand/omap2.c                        |   577 +-
 drivers/mtd/onenand/onenand_base.c                 |    81 -
 drivers/mtd/onenand/samsung.c                      |   185 +-
 drivers/mtd/parsers/sharpslpart.c                  |     6 +-
 drivers/mtd/spi-nor/cadence-quadspi.c              |    55 +-
 drivers/mtd/spi-nor/fsl-quadspi.c                  |     8 +-
 drivers/mtd/spi-nor/intel-spi.c                    |     6 -
 drivers/mtd/spi-nor/mtk-quadspi.c                  |   240 +-
 drivers/mtd/spi-nor/spi-nor.c                      |    75 +-
 drivers/mtd/tests/nandbiterrs.c                    |     2 +-
 drivers/mtd/tests/oobtest.c                        |    21 +
 drivers/mtd/ubi/block.c                            |    42 +-
 drivers/mtd/ubi/build.c                            |    13 +-
 drivers/mtd/ubi/eba.c                              |     2 +-
 drivers/mtd/ubi/fastmap-wl.c                       |     2 +-
 drivers/mtd/ubi/fastmap.c                          |     5 +-
 drivers/mtd/ubi/vmt.c                              |    15 +-
 drivers/mtd/ubi/wl.c                               |    87 +-
 drivers/mtd/ubi/wl.h                               |     2 +-
 drivers/mux/Kconfig                                |     1 +
 drivers/mux/Makefile                               |     1 +
 drivers/mux/adg792a.c                              |     5 +-
 drivers/mux/core.c                                 |     5 +-
 drivers/mux/gpio.c                                 |     5 +-
 drivers/mux/mmio.c                                 |     5 +-
 drivers/net/Kconfig                                |    11 +
 drivers/net/Makefile                               |     1 +
 drivers/net/bonding/bond_main.c                    |    73 +-
 drivers/net/bonding/bond_options.c                 |     6 +-
 drivers/net/caif/caif_hsi.c                        |     1 -
 drivers/net/can/c_can/c_can_pci.c                  |     4 +-
 drivers/net/can/cc770/cc770.c                      |   100 +-
 drivers/net/can/cc770/cc770.h                      |     2 +
 drivers/net/can/dev.c                              |    47 +-
 drivers/net/can/flexcan.c                          |   241 +-
 drivers/net/can/ifi_canfd/ifi_canfd.c              |    75 +-
 drivers/net/can/m_can/m_can.c                      |   190 +-
 drivers/net/can/peak_canfd/peak_canfd.c            |    25 +-
 drivers/net/can/peak_canfd/peak_pciefd_main.c      |     8 +-
 drivers/net/can/rx-offload.c                       |     2 +-
 drivers/net/can/slcan.c                            |     4 +-
 drivers/net/can/spi/mcp251x.c                      |     7 +-
 drivers/net/can/usb/gs_usb.c                       |     4 +-
 drivers/net/can/usb/peak_usb/pcan_usb.c            |     9 +-
 drivers/net/can/usb/peak_usb/pcan_usb_core.c       |    55 +-
 drivers/net/can/usb/peak_usb/pcan_usb_core.h       |     7 +-
 drivers/net/can/usb/peak_usb/pcan_usb_fd.c         |     9 +-
 drivers/net/can/usb/peak_usb/pcan_usb_pro.c        |     9 +-
 drivers/net/can/vcan.c                             |     2 +-
 drivers/net/can/vxcan.c                            |    16 +-
 drivers/net/dsa/Kconfig                            |     2 +-
 drivers/net/dsa/Makefile                           |     5 +-
 drivers/net/dsa/b53/b53_common.c                   |    14 +-
 drivers/net/dsa/b53/b53_priv.h                     |     7 +-
 drivers/net/dsa/bcm_sf2.c                          |    11 +-
 drivers/net/dsa/dsa_loop.c                         |     9 +-
 drivers/net/dsa/lan9303-core.c                     |   138 +-
 drivers/net/dsa/microchip/ksz_common.c             |    12 +-
 drivers/net/dsa/mt7530.c                           |   290 +-
 drivers/net/dsa/mt7530.h                           |    83 +-
 drivers/net/dsa/mv88e6xxx/chip.c                   |    94 +-
 drivers/net/dsa/mv88e6xxx/chip.h                   |     2 +
 drivers/net/dsa/mv88e6xxx/global1.h                |    16 +-
 drivers/net/dsa/mv88e6xxx/global1_atu.c            |    87 +
 drivers/net/dsa/mv88e6xxx/global1_vtu.c            |    74 +
 drivers/net/dummy.c                                |   215 +-
 drivers/net/ethernet/8390/Kconfig                  |     2 +-
 drivers/net/ethernet/8390/mac8390.c                |    33 +-
 drivers/net/ethernet/Kconfig                       |     2 +
 drivers/net/ethernet/Makefile                      |     2 +
 drivers/net/ethernet/alteon/acenic.c               |    14 +-
 drivers/net/ethernet/amazon/ena/ena_eth_com.c      |    11 +
 drivers/net/ethernet/amazon/ena/ena_eth_com.h      |     2 +
 drivers/net/ethernet/amazon/ena/ena_netdev.c       |    68 +-
 drivers/net/ethernet/amazon/ena/ena_netdev.h       |     6 +-
 drivers/net/ethernet/amazon/ena/ena_regs_defs.h    |     2 +
 drivers/net/ethernet/amd/xgbe/xgbe-drv.c           |    26 +-
 drivers/net/ethernet/amd/xgbe/xgbe-pci.c           |     2 +
 drivers/net/ethernet/aquantia/atlantic/Makefile    |     1 +
 drivers/net/ethernet/aquantia/atlantic/aq_cfg.h    |    10 +-
 drivers/net/ethernet/aquantia/atlantic/aq_common.h |    39 +-
 drivers/net/ethernet/aquantia/atlantic/aq_hw.h     |    79 +-
 .../net/ethernet/aquantia/atlantic/aq_hw_utils.c   |    11 +-
 .../net/ethernet/aquantia/atlantic/aq_hw_utils.h   |     3 +
 drivers/net/ethernet/aquantia/atlantic/aq_main.c   |   110 +-
 drivers/net/ethernet/aquantia/atlantic/aq_main.h   |     2 +
 drivers/net/ethernet/aquantia/atlantic/aq_nic.c    |   312 +-
 drivers/net/ethernet/aquantia/atlantic/aq_nic.h    |    63 +-
 .../ethernet/aquantia/atlantic/aq_nic_internal.h   |    45 -
 .../net/ethernet/aquantia/atlantic/aq_pci_func.c   |   432 +-
 .../net/ethernet/aquantia/atlantic/aq_pci_func.h   |    26 +-
 drivers/net/ethernet/aquantia/atlantic/aq_ring.c   |    11 +-
 drivers/net/ethernet/aquantia/atlantic/aq_ring.h   |     4 +-
 drivers/net/ethernet/aquantia/atlantic/aq_utils.h  |     6 -
 drivers/net/ethernet/aquantia/atlantic/aq_vec.c    |    16 +-
 drivers/net/ethernet/aquantia/atlantic/aq_vec.h    |     4 +-
 .../ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c  |   472 +-
 .../ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.h  |    18 +-
 .../aquantia/atlantic/hw_atl/hw_atl_a0_internal.h  |    65 -
 .../ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c  |   506 +-
 .../ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.h  |    28 +-
 .../aquantia/atlantic/hw_atl/hw_atl_b0_internal.h  |    64 -
 .../ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c |  1326 +-
 .../ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h |   544 +-
 .../aquantia/atlantic/hw_atl/hw_atl_llh_internal.h |  1521 +-
 .../aquantia/atlantic/hw_atl/hw_atl_utils.c        |   402 +-
 .../aquantia/atlantic/hw_atl/hw_atl_utils.h        |   134 +-
 .../aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c   |   184 +
 drivers/net/ethernet/aquantia/atlantic/ver.h       |     8 +-
 drivers/net/ethernet/arc/emac_rockchip.c           |     6 +-
 drivers/net/ethernet/broadcom/bcm63xx_enet.c       |    27 +-
 drivers/net/ethernet/broadcom/bcm63xx_enet.h       |     3 -
 drivers/net/ethernet/broadcom/bcmsysport.c         |    45 +-
 drivers/net/ethernet/broadcom/bcmsysport.h         |     2 +-
 drivers/net/ethernet/broadcom/bgmac.c              |    15 -
 drivers/net/ethernet/broadcom/bnx2.c               |     4 +-
 drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c    |    29 +-
 drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c   |    28 +-
 drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c  |    10 +-
 drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h  |     1 +
 drivers/net/ethernet/broadcom/bnxt/Makefile        |     2 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt.c          |   838 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt.h          |    90 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c      |     5 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c      |    32 +
 drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c  |    23 +
 drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h  |     1 +
 drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h      | 11936 ++++---
 drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c    |   221 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c       |    55 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c      |    33 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h      |     6 +
 drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c      |     1 +
 drivers/net/ethernet/broadcom/genet/bcmgenet.c     |     7 +-
 drivers/net/ethernet/broadcom/tg3.c                |    54 +-
 drivers/net/ethernet/broadcom/tg3.h                |    19 +-
 drivers/net/ethernet/cadence/macb.h                |   170 +-
 drivers/net/ethernet/cadence/macb_main.c           |   758 +-
 drivers/net/ethernet/cavium/Kconfig                |    13 +
 drivers/net/ethernet/cavium/Makefile               |     1 +
 drivers/net/ethernet/cavium/common/Makefile        |     1 +
 drivers/net/ethernet/cavium/common/cavium_ptp.c    |   355 +
 drivers/net/ethernet/cavium/common/cavium_ptp.h    |    70 +
 .../net/ethernet/cavium/liquidio/octeon_device.c   |     7 +-
 drivers/net/ethernet/cavium/thunder/nic.h          |    36 +
 drivers/net/ethernet/cavium/thunder/nic_main.c     |    58 +-
 drivers/net/ethernet/cavium/thunder/nic_reg.h      |     1 +
 .../net/ethernet/cavium/thunder/nicvf_ethtool.c    |    31 +-
 drivers/net/ethernet/cavium/thunder/nicvf_main.c   |   187 +-
 drivers/net/ethernet/cavium/thunder/nicvf_queues.c |    30 +
 drivers/net/ethernet/cavium/thunder/nicvf_queues.h |     4 +-
 drivers/net/ethernet/cavium/thunder/thunder_bgx.c  |    31 +-
 drivers/net/ethernet/cavium/thunder/thunder_bgx.h  |     4 +
 drivers/net/ethernet/cavium/thunder/thunder_xcv.c  |     2 +-
 drivers/net/ethernet/chelsio/Kconfig               |     1 +
 drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c    |     1 +
 drivers/net/ethernet/chelsio/cxgb4/Makefile        |     2 +-
 drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c  |    24 +-
 drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h  |   156 +-
 drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h      |    10 +
 drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c     |  1175 +-
 drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h     |    18 +-
 .../net/ethernet/chelsio/cxgb4/cudbg_lib_common.h  |     8 +-
 drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c    |    82 +
 drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h    |    43 +
 drivers/net/ethernet/chelsio/cxgb4/cxgb4.h         |    69 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c   |   119 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h   |     4 +
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c |   306 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c |   107 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c  |   140 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c    |   635 +-
 .../net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c   |    21 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c  |     8 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c     |     1 +
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h     |     5 +-
 drivers/net/ethernet/chelsio/cxgb4/sge.c           |   323 +-
 drivers/net/ethernet/chelsio/cxgb4/t4_hw.c         |   279 +-
 drivers/net/ethernet/chelsio/cxgb4/t4_hw.h         |    45 +-
 drivers/net/ethernet/chelsio/cxgb4/t4_msg.h        |   164 +
 drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h |     2 +
 drivers/net/ethernet/chelsio/cxgb4/t4_regs.h       |    35 +
 drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h      |    52 +-
 drivers/net/ethernet/chelsio/cxgb4vf/adapter.h     |     1 +
 .../net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c    |    12 +-
 drivers/net/ethernet/chelsio/cxgb4vf/sge.c         |    38 +-
 drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h |     1 +
 drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c     |    28 +
 drivers/net/ethernet/cirrus/mac89x0.c              |     6 +-
 drivers/net/ethernet/cisco/enic/enic.h             |     2 +
 drivers/net/ethernet/cisco/enic/enic_ethtool.c     |    12 +
 drivers/net/ethernet/cisco/enic/enic_main.c        |     5 +-
 drivers/net/ethernet/cortina/Kconfig               |    23 +
 drivers/net/ethernet/cortina/Makefile              |     4 +
 drivers/net/ethernet/cortina/gemini.c              |  2594 ++
 drivers/net/ethernet/cortina/gemini.h              |   958 +
 drivers/net/ethernet/emulex/benet/be.h             |     7 +-
 drivers/net/ethernet/emulex/benet/be_cmds.c        |     2 +-
 drivers/net/ethernet/emulex/benet/be_ethtool.c     |     1 +
 drivers/net/ethernet/emulex/benet/be_hw.h          |     1 +
 drivers/net/ethernet/emulex/benet/be_main.c        |   119 +-
 drivers/net/ethernet/freescale/dpaa/dpaa_eth.c     |     8 +-
 drivers/net/ethernet/freescale/fec.h               |     5 +-
 drivers/net/ethernet/freescale/fec_main.c          |    30 +-
 drivers/net/ethernet/freescale/fman/fman_dtsec.c   |     2 +-
 drivers/net/ethernet/freescale/gianfar.c           |    27 +-
 drivers/net/ethernet/hisilicon/Kconfig             |    28 +-
 drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c |     2 +-
 drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c  |     2 +-
 drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c |   109 +-
 drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c  |     2 +-
 drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c  |     2 +-
 drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h  |     7 +-
 drivers/net/ethernet/hisilicon/hns/hns_ethtool.c   |     4 +-
 drivers/net/ethernet/hisilicon/hns3/Makefile       |     7 +
 drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h    |    88 +
 drivers/net/ethernet/hisilicon/hns3/hnae3.c        |    14 +-
 drivers/net/ethernet/hisilicon/hns3/hnae3.h        |    32 +-
 .../hisilicon/hns3/{hns3pf => }/hns3_dcbnl.c       |     2 +-
 .../hisilicon/hns3/{hns3pf => }/hns3_enet.c        |   405 +-
 .../hisilicon/hns3/{hns3pf => }/hns3_enet.h        |    28 +-
 .../hisilicon/hns3/{hns3pf => }/hns3_ethtool.c     |   347 +-
 .../net/ethernet/hisilicon/hns3/hns3pf/Makefile    |     8 +-
 .../net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h |   106 +-
 .../ethernet/hisilicon/hns3/hns3pf/hclge_main.c    |  1632 +-
 .../ethernet/hisilicon/hns3/hns3pf/hclge_main.h    |   129 +-
 .../net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c |   418 +
 .../ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c    |     5 +
 .../net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c  |    64 +-
 .../net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h  |    12 +
 .../net/ethernet/hisilicon/hns3/hns3vf/Makefile    |     9 +
 .../ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c   |   342 +
 .../ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h   |   248 +
 .../ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c  |  1505 +
 .../ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h  |   164 +
 .../ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c   |   181 +
 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c  |    10 +-
 drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h  |     2 +-
 drivers/net/ethernet/ibm/emac/core.c               |    20 +-
 drivers/net/ethernet/ibm/emac/emac.h               |    13 -
 drivers/net/ethernet/ibm/emac/phy.c                |    10 +-
 drivers/net/ethernet/ibm/emac/rgmii.c              |    43 +-
 drivers/net/ethernet/ibm/emac/zmii.c               |    38 +-
 drivers/net/ethernet/ibm/ibmvnic.c                 |   125 +-
 drivers/net/ethernet/ibm/ibmvnic.h                 |     2 +-
 drivers/net/ethernet/intel/e100.c                  |    12 +-
 drivers/net/ethernet/intel/e1000/e1000_ethtool.c   |     4 +-
 drivers/net/ethernet/intel/e1000e/defines.h        |    21 +-
 drivers/net/ethernet/intel/e1000e/ich8lan.c        |    42 +-
 drivers/net/ethernet/intel/e1000e/mac.c            |    25 +-
 drivers/net/ethernet/intel/e1000e/netdev.c         |    41 +-
 drivers/net/ethernet/intel/fm10k/fm10k_iov.c       |     4 +-
 drivers/net/ethernet/intel/fm10k/fm10k_main.c      |    14 +-
 drivers/net/ethernet/intel/fm10k/fm10k_netdev.c    |    54 +-
 drivers/net/ethernet/intel/fm10k/fm10k_pf.c        |     2 +-
 drivers/net/ethernet/intel/i40e/i40e.h             |    67 +-
 drivers/net/ethernet/intel/i40e/i40e_adminq.c      |    17 +-
 drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h  |    30 +-
 drivers/net/ethernet/intel/i40e/i40e_client.c      |    36 +-
 drivers/net/ethernet/intel/i40e/i40e_client.h      |     2 +-
 drivers/net/ethernet/intel/i40e/i40e_common.c      |    74 +-
 drivers/net/ethernet/intel/i40e/i40e_ethtool.c     |   192 +-
 drivers/net/ethernet/intel/i40e/i40e_main.c        |   245 +-
 drivers/net/ethernet/intel/i40e/i40e_nvm.c         |   143 +-
 drivers/net/ethernet/intel/i40e/i40e_prototype.h   |    51 +-
 drivers/net/ethernet/intel/i40e/i40e_status.h      |     1 +
 drivers/net/ethernet/intel/i40e/i40e_txrx.c        |    84 +-
 drivers/net/ethernet/intel/i40e/i40e_txrx.h        |     7 +-
 drivers/net/ethernet/intel/i40e/i40e_type.h        |    54 +-
 drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c |     5 +-
 drivers/net/ethernet/intel/i40evf/i40e_adminq.c    |    15 +-
 .../net/ethernet/intel/i40evf/i40e_adminq_cmd.h    |    30 +-
 drivers/net/ethernet/intel/i40evf/i40e_common.c    |    31 +-
 drivers/net/ethernet/intel/i40evf/i40e_prototype.h |    10 +-
 drivers/net/ethernet/intel/i40evf/i40e_status.h    |     1 +
 drivers/net/ethernet/intel/i40evf/i40e_txrx.c      |    66 +-
 drivers/net/ethernet/intel/i40evf/i40e_txrx.h      |     4 +-
 drivers/net/ethernet/intel/i40evf/i40e_type.h      |    54 +-
 drivers/net/ethernet/intel/i40evf/i40evf.h         |     9 +-
 drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c |    28 +-
 drivers/net/ethernet/intel/i40evf/i40evf_main.c    |   228 +-
 .../net/ethernet/intel/i40evf/i40evf_virtchnl.c    |    83 +-
 drivers/net/ethernet/intel/igb/igb.h               |     1 +
 drivers/net/ethernet/intel/igb/igb_ethtool.c       |    32 +-
 drivers/net/ethernet/intel/igb/igb_main.c          |    72 +-
 drivers/net/ethernet/intel/igb/igb_ptp.c           |     9 +
 drivers/net/ethernet/intel/ixgbe/Makefile          |     1 +
 drivers/net/ethernet/intel/ixgbe/ixgbe.h           |    46 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c     |     3 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c     |    11 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_common.c    |   121 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_common.h    |     6 +
 drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c       |    10 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c |    22 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c |     5 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c    |     2 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c   |     2 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c   |    56 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c      |    10 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c     |   941 +
 drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h     |    93 +
 drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c       |    76 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_main.c      |   597 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c       |    15 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c       |     8 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c     |    63 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_type.h      |    61 +-
 drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c      |    21 +-
 drivers/net/ethernet/intel/ixgbevf/ethtool.c       |     3 +
 drivers/net/ethernet/intel/ixgbevf/ixgbevf.h       |    16 +-
 drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c  |   372 +-
 drivers/net/ethernet/intel/ixgbevf/vf.c            |    34 +-
 drivers/net/ethernet/jme.c                         |     2 +-
 drivers/net/ethernet/marvell/Kconfig               |     2 +-
 drivers/net/ethernet/marvell/mvneta.c              |   688 +-
 drivers/net/ethernet/marvell/mvpp2.c               |   264 +-
 drivers/net/ethernet/marvell/pxa168_eth.c          |    12 +
 drivers/net/ethernet/marvell/sky2.c                |     2 +-
 drivers/net/ethernet/mediatek/mtk_eth_soc.c        |    36 +-
 drivers/net/ethernet/mediatek/mtk_eth_soc.h        |     3 +
 drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c     |    77 +-
 drivers/net/ethernet/mellanox/mlx4/en_ethtool.c    |    50 +-
 drivers/net/ethernet/mellanox/mlx4/en_main.c       |     4 +-
 drivers/net/ethernet/mellanox/mlx4/en_netdev.c     |    10 +-
 drivers/net/ethernet/mellanox/mlx4/en_rx.c         |    69 +-
 drivers/net/ethernet/mellanox/mlx4/mlx4_en.h       |     5 +-
 drivers/net/ethernet/mellanox/mlx4/mr.c            |    40 +-
 drivers/net/ethernet/mellanox/mlx4/qp.c            |     3 +
 .../net/ethernet/mellanox/mlx4/resource_tracker.c  |     1 +
 drivers/net/ethernet/mellanox/mlx5/core/Kconfig    |     2 +-
 drivers/net/ethernet/mellanox/mlx5/core/Makefile   |     2 +-
 .../mellanox/mlx5/core/diag/fs_tracepoint.c        |     8 +-
 drivers/net/ethernet/mellanox/mlx5/core/en.h       |    85 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_dim.c   |    48 +
 .../net/ethernet/mellanox/mlx5/core/en_ethtool.c   |    40 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_fs.c    |   110 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |   318 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_rep.c   |   186 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_rep.h   |    14 +
 drivers/net/ethernet/mellanox/mlx5/core/en_rx.c    |    68 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c |   347 -
 .../net/ethernet/mellanox/mlx5/core/en_selftest.c  |     3 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_stats.c |   282 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_stats.h |    12 +
 drivers/net/ethernet/mellanox/mlx5/core/en_tc.c    |   569 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_tx.c    |     2 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c  |    10 +-
 drivers/net/ethernet/mellanox/mlx5/core/eq.c       |    29 +-
 drivers/net/ethernet/mellanox/mlx5/core/eswitch.c  |   147 +-
 drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |    52 +-
 .../ethernet/mellanox/mlx5/core/eswitch_offloads.c |   216 +-
 .../net/ethernet/mellanox/mlx5/core/fpga/conn.c    |    11 +-
 drivers/net/ethernet/mellanox/mlx5/core/fs_core.c  |   163 +-
 drivers/net/ethernet/mellanox/mlx5/core/fs_core.h  |     6 +-
 .../net/ethernet/mellanox/mlx5/core/fs_counters.c  |     6 +
 drivers/net/ethernet/mellanox/mlx5/core/fw.c       |    10 +-
 drivers/net/ethernet/mellanox/mlx5/core/health.c   |     2 +-
 .../ethernet/mellanox/mlx5/core/ipoib/ethtool.c    |     1 +
 .../net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c  |    29 +-
 .../net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h  |     3 +-
 .../ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c |     7 +
 .../net/ethernet/mellanox/mlx5/core/lib/clock.c    |    56 +
 drivers/net/ethernet/mellanox/mlx5/core/lib/gid.c  |     5 +-
 drivers/net/ethernet/mellanox/mlx5/core/main.c     |    17 +-
 .../net/ethernet/mellanox/mlx5/core/mlx5_core.h    |     3 +-
 drivers/net/ethernet/mellanox/mlx5/core/qp.c       |   125 +-
 drivers/net/ethernet/mellanox/mlx5/core/transobj.c |   214 +
 drivers/net/ethernet/mellanox/mlx5/core/vport.c    |    91 +-
 drivers/net/ethernet/mellanox/mlxsw/core.c         |    92 +-
 drivers/net/ethernet/mellanox/mlxsw/core.h         |    16 +-
 .../mellanox/mlxsw/core_acl_flex_actions.c         |   266 +-
 .../mellanox/mlxsw/core_acl_flex_actions.h         |    14 +-
 .../ethernet/mellanox/mlxsw/core_acl_flex_keys.h   |    20 +-
 drivers/net/ethernet/mellanox/mlxsw/i2c.c          |     5 +-
 drivers/net/ethernet/mellanox/mlxsw/item.h         |     2 +-
 drivers/net/ethernet/mellanox/mlxsw/pci.c          |    98 +-
 drivers/net/ethernet/mellanox/mlxsw/reg.h          |    37 +
 drivers/net/ethernet/mellanox/mlxsw/spectrum.c     |   639 +-
 drivers/net/ethernet/mellanox/mlxsw/spectrum.h     |   106 +-
 drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c |   364 +-
 .../mellanox/mlxsw/spectrum_acl_flex_actions.c     |    66 +
 .../ethernet/mellanox/mlxsw/spectrum_acl_tcam.c    |    44 +-
 .../net/ethernet/mellanox/mlxsw/spectrum_buffers.c |    12 +-
 .../net/ethernet/mellanox/mlxsw/spectrum_dpipe.c   |    81 +-
 drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c |     2 +-
 .../net/ethernet/mellanox/mlxsw/spectrum_flower.c  |    51 +-
 .../net/ethernet/mellanox/mlxsw/spectrum_kvdl.c    |    26 +
 .../net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c |     3 +-
 .../net/ethernet/mellanox/mlxsw/spectrum_qdisc.c   |   551 +-
 .../net/ethernet/mellanox/mlxsw/spectrum_router.c  |   385 +-
 .../ethernet/mellanox/mlxsw/spectrum_switchdev.c   |    29 +-
 drivers/net/ethernet/micrel/ks8851.c               |    13 +-
 drivers/net/ethernet/natsemi/Kconfig               |     6 +-
 drivers/net/ethernet/natsemi/Makefile              |     2 +-
 drivers/net/ethernet/natsemi/macsonic.c            |    38 +-
 drivers/net/ethernet/netronome/nfp/Makefile        |     3 +
 drivers/net/ethernet/netronome/nfp/bpf/cmsg.c      |   453 +
 drivers/net/ethernet/netronome/nfp/bpf/fw.h        |   157 +
 drivers/net/ethernet/netronome/nfp/bpf/jit.c       |   994 +-
 drivers/net/ethernet/netronome/nfp/bpf/main.c      |   257 +-
 drivers/net/ethernet/netronome/nfp/bpf/main.h      |   207 +-
 drivers/net/ethernet/netronome/nfp/bpf/offload.c   |   198 +-
 drivers/net/ethernet/netronome/nfp/bpf/verifier.c  |   159 +-
 drivers/net/ethernet/netronome/nfp/flower/action.c |    79 +-
 drivers/net/ethernet/netronome/nfp/flower/cmsg.c   |    75 +-
 drivers/net/ethernet/netronome/nfp/flower/cmsg.h   |    71 +-
 drivers/net/ethernet/netronome/nfp/flower/main.c   |   156 +-
 drivers/net/ethernet/netronome/nfp/flower/main.h   |    16 +-
 drivers/net/ethernet/netronome/nfp/flower/match.c  |   127 +-
 .../net/ethernet/netronome/nfp/flower/offload.c    |    61 +-
 .../ethernet/netronome/nfp/flower/tunnel_conf.c    |    10 +-
 drivers/net/ethernet/netronome/nfp/nfp_app.c       |    14 +-
 drivers/net/ethernet/netronome/nfp/nfp_app.h       |    93 +-
 drivers/net/ethernet/netronome/nfp/nfp_asm.c       |    95 +-
 drivers/net/ethernet/netronome/nfp/nfp_asm.h       |    28 +-
 drivers/net/ethernet/netronome/nfp/nfp_devlink.c   |    12 +-
 drivers/net/ethernet/netronome/nfp/nfp_main.c      |    25 +-
 drivers/net/ethernet/netronome/nfp/nfp_main.h      |    29 +
 drivers/net/ethernet/netronome/nfp/nfp_net.h       |    28 +-
 .../net/ethernet/netronome/nfp/nfp_net_common.c    |   122 +-
 drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.c  |   135 +
 drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h  |   215 +-
 .../net/ethernet/netronome/nfp/nfp_net_debugdump.c |   811 +
 .../net/ethernet/netronome/nfp/nfp_net_ethtool.c   |   109 +-
 drivers/net/ethernet/netronome/nfp/nfp_net_main.c  |    76 +-
 drivers/net/ethernet/netronome/nfp/nfp_net_repr.c  |    90 +-
 drivers/net/ethernet/netronome/nfp/nfp_net_repr.h  |    16 +-
 .../net/ethernet/netronome/nfp/nfp_netvf_main.c    |     6 -
 drivers/net/ethernet/netronome/nfp/nfp_port.c      |    18 +
 drivers/net/ethernet/netronome/nfp/nfp_port.h      |     6 +
 drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h   |     2 +
 .../net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h   |     1 +
 .../ethernet/netronome/nfp/nfpcore/nfp_cppcore.c   |    62 +-
 .../ethernet/netronome/nfp/nfpcore/nfp_cpplib.c    |    38 +-
 .../ethernet/netronome/nfp/nfpcore/nfp_hwinfo.c    |    10 +
 .../net/ethernet/netronome/nfp/nfpcore/nfp_nsp.c   |    72 +-
 .../net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h   |    44 +-
 .../ethernet/netronome/nfp/nfpcore/nfp_resource.c  |    21 +-
 .../net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c |     4 -
 drivers/net/ethernet/nvidia/forcedeth.c            |    94 +-
 .../net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c   |     6 +-
 drivers/net/ethernet/pasemi/pasemi_mac.c           |     4 +-
 drivers/net/ethernet/qlogic/Kconfig                |     1 +
 drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c |     4 +-
 drivers/net/ethernet/qlogic/qed/qed.h              |    12 +-
 drivers/net/ethernet/qlogic/qed/qed_cxt.c          |    54 +-
 drivers/net/ethernet/qlogic/qed/qed_cxt.h          |     4 +-
 drivers/net/ethernet/qlogic/qed/qed_dcbx.c         |     4 +-
 drivers/net/ethernet/qlogic/qed/qed_debug.c        |  1173 +-
 drivers/net/ethernet/qlogic/qed/qed_dev.c          |    15 +-
 drivers/net/ethernet/qlogic/qed/qed_fcoe.c         |    10 +-
 drivers/net/ethernet/qlogic/qed/qed_hsi.h          | 10603 ++++---
 drivers/net/ethernet/qlogic/qed/qed_hw.c           |    68 +
 drivers/net/ethernet/qlogic/qed/qed_hw.h           |     4 +
 .../net/ethernet/qlogic/qed/qed_init_fw_funcs.c    |  1091 +-
 drivers/net/ethernet/qlogic/qed/qed_init_ops.c     |    22 +-
 drivers/net/ethernet/qlogic/qed/qed_int.c          |     8 +-
 drivers/net/ethernet/qlogic/qed/qed_int.h          |     2 +-
 drivers/net/ethernet/qlogic/qed/qed_iscsi.c        |    58 +-
 drivers/net/ethernet/qlogic/qed/qed_iwarp.c        |    50 +-
 drivers/net/ethernet/qlogic/qed/qed_iwarp.h        |     3 +-
 drivers/net/ethernet/qlogic/qed/qed_l2.c           |   119 +-
 drivers/net/ethernet/qlogic/qed/qed_l2.h           |    33 +-
 drivers/net/ethernet/qlogic/qed/qed_ll2.c          |    28 +-
 drivers/net/ethernet/qlogic/qed/qed_mcp.c          |     2 +-
 drivers/net/ethernet/qlogic/qed/qed_rdma.c         |    13 +-
 drivers/net/ethernet/qlogic/qed/qed_reg_addr.h     |   190 +-
 drivers/net/ethernet/qlogic/qed/qed_sp_commands.c  |     8 +-
 drivers/net/ethernet/qlogic/qed/qed_spq.c          |    12 +-
 drivers/net/ethernet/qlogic/qed/qed_sriov.c        |    22 +-
 drivers/net/ethernet/qlogic/qed/qed_sriov.h        |    24 +
 drivers/net/ethernet/qlogic/qede/qede.h            |    10 +-
 drivers/net/ethernet/qlogic/qede/qede_ethtool.c    |     3 +
 drivers/net/ethernet/qlogic/qede/qede_filter.c     |    47 +-
 drivers/net/ethernet/qlogic/qede/qede_fp.c         |    21 +-
 drivers/net/ethernet/qlogic/qede/qede_main.c       |    41 +-
 drivers/net/ethernet/qlogic/qede/qede_ptp.c        |     2 +-
 .../net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c    |    18 +-
 drivers/net/ethernet/qlogic/qlge/qlge_main.c       |     3 +-
 drivers/net/ethernet/qualcomm/emac/emac-mac.c      |    23 +-
 drivers/net/ethernet/qualcomm/emac/emac-mac.h      |     3 +-
 drivers/net/ethernet/qualcomm/emac/emac.c          |     7 +-
 drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c |   132 +-
 drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h |     3 +-
 .../net/ethernet/qualcomm/rmnet/rmnet_handlers.c   |    70 +-
 drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h    |    32 +-
 .../ethernet/qualcomm/rmnet/rmnet_map_command.c    |    22 +-
 .../net/ethernet/qualcomm/rmnet/rmnet_map_data.c   |   309 +-
 .../net/ethernet/qualcomm/rmnet/rmnet_private.h    |    12 +-
 drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c    |     9 +-
 drivers/net/ethernet/realtek/8139cp.c              |     4 +-
 drivers/net/ethernet/realtek/r8169.c               |   137 +-
 drivers/net/ethernet/renesas/ravb_main.c           |     9 +-
 drivers/net/ethernet/renesas/sh_eth.c              |    55 +-
 drivers/net/ethernet/renesas/sh_eth.h              |    11 -
 drivers/net/ethernet/rocker/rocker_main.c          |    18 +-
 drivers/net/ethernet/rocker/rocker_ofdpa.c         |     1 -
 drivers/net/ethernet/sfc/ef10.c                    |   363 +-
 drivers/net/ethernet/sfc/ef10_regs.h               |    46 +-
 drivers/net/ethernet/sfc/efx.c                     |    66 +-
 drivers/net/ethernet/sfc/efx.h                     |     9 +-
 drivers/net/ethernet/sfc/ethtool.c                 |     6 +-
 drivers/net/ethernet/sfc/farch.c                   |    26 +-
 drivers/net/ethernet/sfc/io.h                      |    19 +-
 drivers/net/ethernet/sfc/mcdi.h                    |     3 +
 drivers/net/ethernet/sfc/mcdi_pcol.h               |  2453 +-
 drivers/net/ethernet/sfc/mcdi_port.c               |   168 +-
 drivers/net/ethernet/sfc/net_driver.h              |    33 +-
 drivers/net/ethernet/sfc/nic.h                     |    28 +
 drivers/net/ethernet/sfc/ptp.c                     |   370 +-
 drivers/net/ethernet/sfc/siena.c                   |    12 +-
 drivers/net/ethernet/sfc/tx.c                      |    21 +-
 drivers/net/ethernet/smsc/Kconfig                  |     2 +-
 drivers/net/ethernet/smsc/smsc911x.c               |     4 +-
 drivers/net/ethernet/socionext/Kconfig             |    34 +
 drivers/net/ethernet/socionext/Makefile            |     6 +
 drivers/net/ethernet/socionext/netsec.c            |  1777 ++
 drivers/net/ethernet/socionext/sni_ave.c           |  1736 ++
 drivers/net/ethernet/stmicro/stmmac/common.h       |     2 +-
 .../net/ethernet/stmicro/stmmac/dwmac-meson8b.c    |   113 +-
 drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c  |     3 +-
 .../net/ethernet/stmicro/stmmac/dwmac1000_core.c   |    14 +-
 .../net/ethernet/stmicro/stmmac/dwmac100_core.c    |    15 +-
 drivers/net/ethernet/stmicro/stmmac/dwmac4.h       |     2 +-
 drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c  |    25 +-
 drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c |     6 +-
 drivers/net/ethernet/stmicro/stmmac/enh_desc.c     |     4 +-
 drivers/net/ethernet/stmicro/stmmac/norm_desc.c    |     2 +-
 drivers/net/ethernet/stmicro/stmmac/stmmac_main.c  |    61 +-
 drivers/net/ethernet/sun/Kconfig                   |     1 +
 drivers/net/ethernet/sun/cassini.c                 |     1 +
 drivers/net/ethernet/sun/cassini.h                 |     1 +
 drivers/net/ethernet/sun/ldmvsw.c                  |     1 +
 drivers/net/ethernet/sun/niu.c                     |     1 +
 drivers/net/ethernet/sun/sunbmac.c                 |     1 +
 drivers/net/ethernet/sun/sungem.c                  |     1 +
 drivers/net/ethernet/sun/sunhme.c                  |     1 +
 drivers/net/ethernet/sun/sunqe.c                   |     1 +
 drivers/net/ethernet/sun/sunvnet.c                 |     3 +-
 drivers/net/ethernet/sun/sunvnet_common.c          |     1 +
 drivers/net/ethernet/synopsys/dwc-xlgmac-common.c  |    24 +-
 drivers/net/ethernet/ti/cpsw.c                     |   115 +-
 drivers/net/ethernet/ti/cpsw.h                     |    23 +-
 drivers/net/ethernet/ti/cpsw_ale.c                 |   109 +-
 drivers/net/ethernet/ti/cpsw_ale.h                 |     1 -
 drivers/net/ethernet/ti/davinci_cpdma.c            |     2 +-
 drivers/net/ethernet/ti/davinci_emac.c             |    20 +-
 drivers/net/ethernet/ti/netcp_ethss.c              |     6 +-
 drivers/net/ethernet/tundra/tsi108_eth.c           |    36 +-
 drivers/net/geneve.c                               |    24 +-
 drivers/net/hyperv/hyperv_net.h                    |    62 +-
 drivers/net/hyperv/netvsc.c                        |   158 +-
 drivers/net/hyperv/netvsc_drv.c                    |   454 +-
 drivers/net/hyperv/rndis_filter.c                  |   115 +-
 drivers/net/ieee802154/adf7242.c                   |    90 +-
 drivers/net/ieee802154/ca8210.c                    |     8 +-
 drivers/net/ipvlan/ipvlan_core.c                   |    17 +-
 drivers/net/ipvlan/ipvlan_main.c                   |    33 +-
 drivers/net/macsec.c                               |    73 +-
 drivers/net/macvlan.c                              |     4 +-
 drivers/net/netdevsim/Makefile                     |    11 +
 drivers/net/netdevsim/bpf.c                        |   642 +
 drivers/net/netdevsim/netdev.c                     |   504 +
 drivers/net/netdevsim/netdevsim.h                  |   109 +
 drivers/net/phy/amd.c                              |     2 -
 drivers/net/phy/at803x.c                           |    44 +-
 drivers/net/phy/bcm-cygnus.c                       |     2 -
 drivers/net/phy/bcm-phy-lib.c                      |     4 +-
 drivers/net/phy/bcm63xx.c                          |     4 -
 drivers/net/phy/bcm7xxx.c                          |     6 -
 drivers/net/phy/broadcom.c                         |    69 +-
 drivers/net/phy/cicada.c                           |     4 -
 drivers/net/phy/davicom.c                          |     5 -
 drivers/net/phy/dp83640.c                          |     2 -
 drivers/net/phy/dp83822.c                          |     2 -
 drivers/net/phy/dp83848.c                          |     2 -
 drivers/net/phy/dp83867.c                          |     2 -
 drivers/net/phy/fixed_phy.c                        |    31 -
 drivers/net/phy/icplus.c                           |     4 -
 drivers/net/phy/intel-xway.c                       |    12 -
 drivers/net/phy/lxt.c                              |     5 -
 drivers/net/phy/marvell.c                          |   604 +-
 drivers/net/phy/marvell10g.c                       |   111 +-
 drivers/net/phy/mdio-bcm-unimac.c                  |     2 +
 drivers/net/phy/mdio_bus.c                         |   100 +-
 drivers/net/phy/mdio_device.c                      |    34 +-
 drivers/net/phy/meson-gxl.c                        |   186 +-
 drivers/net/phy/micrel.c                           |    51 +-
 drivers/net/phy/microchip.c                        |     1 -
 drivers/net/phy/national.c                         |     2 -
 drivers/net/phy/phy-c45.c                          |    33 +
 drivers/net/phy/phy-core.c                         |   258 +-
 drivers/net/phy/phy.c                              |   193 +-
 drivers/net/phy/phy_device.c                       |   156 +-
 drivers/net/phy/phylink.c                          |   448 +-
 drivers/net/phy/qsemi.c                            |     2 -
 drivers/net/phy/realtek.c                          |   107 +-
 drivers/net/phy/rockchip.c                         |     1 -
 drivers/net/phy/sfp-bus.c                          |   221 +-
 drivers/net/phy/sfp.c                              |   119 +-
 drivers/net/phy/smsc.c                             |    11 +-
 drivers/net/phy/ste10Xp.c                          |     4 -
 drivers/net/phy/uPD60620.c                         |     1 -
 drivers/net/phy/vitesse.c                          |    12 -
 drivers/net/ppp/ppp_async.c                        |     2 +-
 drivers/net/ppp/ppp_generic.c                      |    47 +-
 drivers/net/ppp/ppp_synctty.c                      |     2 +-
 drivers/net/slip/slip.c                            |     4 +-
 drivers/net/tap.c                                  |    54 +-
 drivers/net/team/team.c                            |    16 +-
 drivers/net/thunderbolt.c                          |    19 +-
 drivers/net/tun.c                                  |   499 +-
 drivers/net/usb/cdc_ether.c                        |     6 +
 drivers/net/usb/lan78xx.c                          |    35 +-
 drivers/net/usb/qmi_wwan.c                         |    10 +-
 drivers/net/usb/r8152.c                            |    15 +-
 drivers/net/usb/smsc75xx.c                         |     7 +-
 drivers/net/usb/usbnet.c                           |    10 +-
 drivers/net/veth.c                                 |     3 +
 drivers/net/virtio_net.c                           |   345 +-
 drivers/net/vmxnet3/vmxnet3_drv.c                  |    22 +-
 drivers/net/vmxnet3/vmxnet3_int.h                  |    11 +-
 drivers/net/vrf.c                                  |     5 +-
 drivers/net/vxlan.c                                |    26 +-
 drivers/net/wan/cosa.c                             |     2 +-
 drivers/net/wan/hdlc_ppp.c                         |     5 +-
 drivers/net/wimax/i2400m/sysfs.c                   |     3 +-
 drivers/net/wireless/ath/ath10k/Kconfig            |     9 +-
 drivers/net/wireless/ath/ath10k/Makefile           |     3 +-
 drivers/net/wireless/ath/ath10k/ahb.c              |     2 +-
 drivers/net/wireless/ath/ath10k/bmi.c              |     2 +-
 drivers/net/wireless/ath/ath10k/bmi.h              |     2 +-
 drivers/net/wireless/ath/ath10k/ce.c               |   636 +-
 drivers/net/wireless/ath/ath10k/ce.h               |    61 +-
 drivers/net/wireless/ath/ath10k/core.c             |   357 +-
 drivers/net/wireless/ath/ath10k/core.h             |    36 +-
 drivers/net/wireless/ath/ath10k/coredump.c         |   994 +
 drivers/net/wireless/ath/ath10k/coredump.h         |   225 +
 drivers/net/wireless/ath/ath10k/debug.c            |   305 +-
 drivers/net/wireless/ath/ath10k/debug.h            |    40 +-
 drivers/net/wireless/ath/ath10k/debugfs_sta.c      |    29 +-
 drivers/net/wireless/ath/ath10k/hif.h              |     2 +-
 drivers/net/wireless/ath/ath10k/htc.c              |     2 +-
 drivers/net/wireless/ath/ath10k/htc.h              |     2 +-
 drivers/net/wireless/ath/ath10k/htt.c              |     9 +-
 drivers/net/wireless/ath/ath10k/htt.h              |   165 +-
 drivers/net/wireless/ath/ath10k/htt_rx.c           |   365 +-
 drivers/net/wireless/ath/ath10k/htt_tx.c           |   600 +-
 drivers/net/wireless/ath/ath10k/hw.c               |     4 +-
 drivers/net/wireless/ath/ath10k/hw.h               |    24 +-
 drivers/net/wireless/ath/ath10k/mac.c              |    72 +-
 drivers/net/wireless/ath/ath10k/mac.h              |     2 +-
 drivers/net/wireless/ath/ath10k/pci.c              |   236 +-
 drivers/net/wireless/ath/ath10k/pci.h              |     2 +-
 drivers/net/wireless/ath/ath10k/rx_desc.h          |    52 +-
 drivers/net/wireless/ath/ath10k/spectral.c         |     2 +-
 drivers/net/wireless/ath/ath10k/spectral.h         |     6 +-
 drivers/net/wireless/ath/ath10k/swap.c             |     2 +-
 drivers/net/wireless/ath/ath10k/swap.h             |     2 +-
 drivers/net/wireless/ath/ath10k/targaddrs.h        |     2 +-
 drivers/net/wireless/ath/ath10k/testmode.c         |     2 +-
 drivers/net/wireless/ath/ath10k/testmode_i.h       |     2 +-
 drivers/net/wireless/ath/ath10k/thermal.c          |     2 +-
 drivers/net/wireless/ath/ath10k/thermal.h          |     2 +-
 drivers/net/wireless/ath/ath10k/trace.h            |     2 +-
 drivers/net/wireless/ath/ath10k/txrx.c             |     2 +-
 drivers/net/wireless/ath/ath10k/txrx.h             |     2 +-
 drivers/net/wireless/ath/ath10k/wmi-ops.h          |    11 +-
 drivers/net/wireless/ath/ath10k/wmi-tlv.c          |   146 +-
 drivers/net/wireless/ath/ath10k/wmi-tlv.h          |   115 +-
 drivers/net/wireless/ath/ath10k/wmi.c              |    74 +-
 drivers/net/wireless/ath/ath10k/wmi.h              |    14 +-
 drivers/net/wireless/ath/ath10k/wow.c              |     2 +-
 drivers/net/wireless/ath/ath10k/wow.h              |     2 +-
 drivers/net/wireless/ath/ath6kl/cfg80211.c         |     2 -
 drivers/net/wireless/ath/ath6kl/txrx.c             |     2 +-
 drivers/net/wireless/ath/ath9k/Kconfig             |    14 +-
 drivers/net/wireless/ath/ath9k/Makefile            |     4 +-
 drivers/net/wireless/ath/ath9k/ar9003_eeprom.c     |   133 +-
 drivers/net/wireless/ath/ath9k/ar9003_eeprom.h     |    10 +
 drivers/net/wireless/ath/ath9k/calib.c             |    36 +-
 drivers/net/wireless/ath/ath9k/common-spectral.h   |     4 +-
 drivers/net/wireless/ath/ath9k/dfs.c               |     8 +-
 drivers/net/wireless/ath/ath9k/hif_usb.c           |     1 +
 drivers/net/wireless/ath/ath9k/htc_drv_init.c      |     1 +
 drivers/net/wireless/ath/ath9k/htc_drv_main.c      |     4 +
 drivers/net/wireless/ath/ath9k/hw.c                |    33 +-
 drivers/net/wireless/ath/ath9k/hw.h                |     5 +
 drivers/net/wireless/ath/ath9k/init.c              |    57 +
 drivers/net/wireless/ath/ath9k/mac.c               |    47 +
 drivers/net/wireless/ath/ath9k/pci.c               |    21 +-
 drivers/net/wireless/ath/ath9k/recv.c              |     4 +-
 drivers/net/wireless/ath/ath9k/reg.h               |    15 +
 drivers/net/wireless/ath/wcn36xx/dxe.c             |    46 +-
 drivers/net/wireless/ath/wcn36xx/hal.h             |   107 +-
 drivers/net/wireless/ath/wcn36xx/main.c            |    16 +-
 drivers/net/wireless/ath/wcn36xx/smd.c             |   126 +-
 drivers/net/wireless/ath/wcn36xx/smd.h             |     3 +
 drivers/net/wireless/ath/wil6210/boot_loader.h     |     9 +-
 drivers/net/wireless/ath/wil6210/cfg80211.c        |    85 +-
 drivers/net/wireless/ath/wil6210/debugfs.c         |   151 +-
 drivers/net/wireless/ath/wil6210/ethtool.c         |    15 +
 drivers/net/wireless/ath/wil6210/fw.h              |    18 +-
 drivers/net/wireless/ath/wil6210/fw_inc.c          |   234 +-
 drivers/net/wireless/ath/wil6210/interrupt.c       |    30 +-
 drivers/net/wireless/ath/wil6210/main.c            |   304 +-
 drivers/net/wireless/ath/wil6210/netdev.c          |    20 +-
 drivers/net/wireless/ath/wil6210/pcie_bus.c        |   192 +-
 drivers/net/wireless/ath/wil6210/pm.c              |   121 +-
 drivers/net/wireless/ath/wil6210/pmc.c             |    11 +-
 drivers/net/wireless/ath/wil6210/txrx.c            |    14 +-
 drivers/net/wireless/ath/wil6210/wil6210.h         |   112 +-
 drivers/net/wireless/ath/wil6210/wil_crash_dump.c  |    11 +
 drivers/net/wireless/ath/wil6210/wil_platform.h    |    15 +-
 drivers/net/wireless/ath/wil6210/wmi.c             |   635 +-
 drivers/net/wireless/ath/wil6210/wmi.h             |   117 +-
 drivers/net/wireless/broadcom/b43/phy_n.c          |     2 +-
 .../wireless/broadcom/brcm80211/brcmfmac/bcdc.c    |    17 +-
 .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c  |   531 +-
 .../broadcom/brcm80211/brcmfmac/cfg80211.c         |     2 +-
 .../wireless/broadcom/brcm80211/brcmfmac/chip.c    |     1 +
 .../wireless/broadcom/brcm80211/brcmfmac/common.c  |     5 +
 .../wireless/broadcom/brcm80211/brcmfmac/common.h  |     1 +
 .../wireless/broadcom/brcm80211/brcmfmac/core.c    |    57 +
 .../wireless/broadcom/brcm80211/brcmfmac/core.h    |     2 +
 .../wireless/broadcom/brcm80211/brcmfmac/feature.c |    22 +-
 .../wireless/broadcom/brcm80211/brcmfmac/fwil.c    |    25 +-
 .../wireless/broadcom/brcm80211/brcmfmac/msgbuf.c  |    10 +-
 .../net/wireless/broadcom/brcm80211/brcmfmac/p2p.c |    24 +-
 .../wireless/broadcom/brcm80211/brcmfmac/pcie.c    |    20 +-
 .../wireless/broadcom/brcm80211/brcmfmac/proto.h   |    14 +-
 .../wireless/broadcom/brcm80211/brcmfmac/sdio.c    |   487 +-
 .../wireless/broadcom/brcm80211/brcmfmac/sdio.h    |    73 +-
 .../broadcom/brcm80211/brcmsmac/phy/phy_n.c        |     3 +-
 .../broadcom/brcm80211/brcmsmac/phy/phytbl_n.c     |   462 +-
 drivers/net/wireless/intel/iwlwifi/Kconfig         |     1 -
 drivers/net/wireless/intel/iwlwifi/Makefile        |     3 +-
 drivers/net/wireless/intel/iwlwifi/cfg/22000.c     |   216 +
 drivers/net/wireless/intel/iwlwifi/cfg/8000.c      |     4 +-
 drivers/net/wireless/intel/iwlwifi/cfg/9000.c      |    64 +-
 drivers/net/wireless/intel/iwlwifi/cfg/a000.c      |   216 -
 drivers/net/wireless/intel/iwlwifi/fw/api/alive.h  |     4 +-
 drivers/net/wireless/intel/iwlwifi/fw/api/coex.h   |    30 -
 .../net/wireless/intel/iwlwifi/fw/api/commands.h   |     2 +-
 .../net/wireless/intel/iwlwifi/fw/api/datapath.h   |    15 +
 drivers/net/wireless/intel/iwlwifi/fw/api/debug.h  |     4 +-
 .../net/wireless/intel/iwlwifi/fw/api/mac-cfg.h    |    19 +
 drivers/net/wireless/intel/iwlwifi/fw/api/rs.h     |   262 +-
 .../net/wireless/intel/iwlwifi/fw/api/time-event.h |     4 +-
 drivers/net/wireless/intel/iwlwifi/fw/api/tx.h     |     4 +-
 drivers/net/wireless/intel/iwlwifi/fw/dbg.c        |    28 +-
 drivers/net/wireless/intel/iwlwifi/fw/dbg.h        |     3 +
 drivers/net/wireless/intel/iwlwifi/fw/debugfs.c    |   195 +
 drivers/net/wireless/intel/iwlwifi/fw/debugfs.h    |   105 +
 drivers/net/wireless/intel/iwlwifi/fw/file.h       |    39 +-
 drivers/net/wireless/intel/iwlwifi/fw/img.h        |     2 +-
 drivers/net/wireless/intel/iwlwifi/fw/init.c       |    19 +-
 drivers/net/wireless/intel/iwlwifi/fw/runtime.h    |    21 +-
 drivers/net/wireless/intel/iwlwifi/fw/smem.c       |     8 +-
 drivers/net/wireless/intel/iwlwifi/iwl-config.h    |    25 +-
 .../wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h  |     7 +-
 drivers/net/wireless/intel/iwlwifi/iwl-drv.c       |   110 +-
 drivers/net/wireless/intel/iwlwifi/iwl-fh.h        |    12 +-
 drivers/net/wireless/intel/iwlwifi/iwl-trans.h     |    11 +-
 drivers/net/wireless/intel/iwlwifi/mvm/Makefile    |     2 +-
 drivers/net/wireless/intel/iwlwifi/mvm/coex.c      |    33 +-
 drivers/net/wireless/intel/iwlwifi/mvm/d3.c        |   240 +-
 drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c   |    98 +-
 drivers/net/wireless/intel/iwlwifi/mvm/fw.c        |    18 +-
 drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c  |     3 +-
 drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c  |    99 +-
 drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       |    27 +-
 drivers/net/wireless/intel/iwlwifi/mvm/ops.c       |    21 +-
 drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c  |    21 +-
 drivers/net/wireless/intel/iwlwifi/mvm/quota.c     |     4 +
 drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c     |   314 +
 drivers/net/wireless/intel/iwlwifi/mvm/rs.c        |   262 +-
 drivers/net/wireless/intel/iwlwifi/mvm/rs.h        |    48 +
 drivers/net/wireless/intel/iwlwifi/mvm/rx.c        |     9 +-
 drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c      |    49 +-
 drivers/net/wireless/intel/iwlwifi/mvm/scan.c      |    35 +-
 drivers/net/wireless/intel/iwlwifi/mvm/sta.c       |   135 +-
 drivers/net/wireless/intel/iwlwifi/mvm/sta.h       |     7 +-
 .../net/wireless/intel/iwlwifi/mvm/time-event.c    |    36 +-
 drivers/net/wireless/intel/iwlwifi/mvm/tx.c        |    47 +-
 drivers/net/wireless/intel/iwlwifi/mvm/utils.c     |    40 +-
 drivers/net/wireless/intel/iwlwifi/pcie/drv.c      |    76 +-
 drivers/net/wireless/intel/iwlwifi/pcie/internal.h |     7 -
 .../net/wireless/intel/iwlwifi/pcie/trans-gen2.c   |     4 +-
 drivers/net/wireless/intel/iwlwifi/pcie/trans.c    |    81 +-
 drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c  |     2 +-
 drivers/net/wireless/intel/iwlwifi/pcie/tx.c       |     2 +-
 drivers/net/wireless/mac80211_hwsim.c              |   114 +-
 drivers/net/wireless/mac80211_hwsim.h              |    68 +-
 drivers/net/wireless/marvell/mwifiex/cfg80211.c    |     7 +-
 drivers/net/wireless/marvell/mwifiex/cmdevt.c      |    11 +-
 drivers/net/wireless/marvell/mwifiex/debugfs.c     |    13 +-
 drivers/net/wireless/marvell/mwifiex/fw.h          |    11 +
 drivers/net/wireless/marvell/mwifiex/init.c        |    10 +
 drivers/net/wireless/marvell/mwifiex/main.c        |    97 +-
 drivers/net/wireless/marvell/mwifiex/main.h        |    13 +-
 drivers/net/wireless/marvell/mwifiex/pcie.c        |    18 +-
 drivers/net/wireless/marvell/mwifiex/sdio.c        |    14 +-
 drivers/net/wireless/marvell/mwifiex/sta_cmd.c     |     4 +
 drivers/net/wireless/marvell/mwifiex/sta_event.c   |    61 +
 drivers/net/wireless/marvell/mwl8k.c               |     7 +-
 drivers/net/wireless/mediatek/Kconfig              |     1 +
 drivers/net/wireless/mediatek/Makefile             |     1 +
 drivers/net/wireless/mediatek/mt76/Kconfig         |    10 +
 drivers/net/wireless/mediatek/mt76/Makefile        |    15 +
 drivers/net/wireless/mediatek/mt76/agg-rx.c        |   294 +
 drivers/net/wireless/mediatek/mt76/debugfs.c       |    78 +
 drivers/net/wireless/mediatek/mt76/dma.c           |   459 +
 drivers/net/wireless/mediatek/mt76/dma.h           |    38 +
 drivers/net/wireless/mediatek/mt76/eeprom.c        |   112 +
 drivers/net/wireless/mediatek/mt76/mac80211.c      |   555 +
 drivers/net/wireless/mediatek/mt76/mmio.c          |    61 +
 drivers/net/wireless/mediatek/mt76/mt76.h          |   442 +
 drivers/net/wireless/mediatek/mt76/mt76x2.h        |   230 +
 drivers/net/wireless/mediatek/mt76/mt76x2_core.c   |    88 +
 .../net/wireless/mediatek/mt76/mt76x2_debugfs.c    |   133 +
 drivers/net/wireless/mediatek/mt76/mt76x2_dfs.c    |   506 +
 drivers/net/wireless/mediatek/mt76/mt76x2_dfs.h    |    82 +
 drivers/net/wireless/mediatek/mt76/mt76x2_dma.c    |   184 +
 drivers/net/wireless/mediatek/mt76/mt76x2_dma.h    |    68 +
 drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c |   664 +
 drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h |   185 +
 drivers/net/wireless/mediatek/mt76/mt76x2_init.c   |   876 +
 drivers/net/wireless/mediatek/mt76/mt76x2_mac.c    |   839 +
 drivers/net/wireless/mediatek/mt76/mt76x2_mac.h    |   190 +
 drivers/net/wireless/mediatek/mt76/mt76x2_main.c   |   577 +
 drivers/net/wireless/mediatek/mt76/mt76x2_mcu.c    |   453 +
 drivers/net/wireless/mediatek/mt76/mt76x2_mcu.h    |   155 +
 drivers/net/wireless/mediatek/mt76/mt76x2_pci.c    |   110 +
 drivers/net/wireless/mediatek/mt76/mt76x2_phy.c    |   740 +
 drivers/net/wireless/mediatek/mt76/mt76x2_regs.h   |   587 +
 drivers/net/wireless/mediatek/mt76/mt76x2_trace.c  |    23 +
 drivers/net/wireless/mediatek/mt76/mt76x2_trace.h  |   144 +
 drivers/net/wireless/mediatek/mt76/mt76x2_tx.c     |   260 +
 drivers/net/wireless/mediatek/mt76/trace.c         |    23 +
 drivers/net/wireless/mediatek/mt76/trace.h         |    71 +
 drivers/net/wireless/mediatek/mt76/tx.c            |   511 +
 drivers/net/wireless/mediatek/mt76/util.c          |    78 +
 drivers/net/wireless/mediatek/mt76/util.h          |    44 +
 drivers/net/wireless/quantenna/qtnfmac/cfg80211.c  |   191 +-
 drivers/net/wireless/quantenna/qtnfmac/cfg80211.h  |    19 -
 drivers/net/wireless/quantenna/qtnfmac/commands.c  |   581 +-
 drivers/net/wireless/quantenna/qtnfmac/commands.h  |     5 +
 drivers/net/wireless/quantenna/qtnfmac/core.c      |   152 +-
 drivers/net/wireless/quantenna/qtnfmac/core.h      |    23 +-
 drivers/net/wireless/quantenna/qtnfmac/event.c     |    80 +-
 .../net/wireless/quantenna/qtnfmac/pearl/pcie.c    |     7 +-
 drivers/net/wireless/quantenna/qtnfmac/qlink.h     |   351 +-
 .../net/wireless/quantenna/qtnfmac/qlink_util.c    |    78 +-
 .../net/wireless/quantenna/qtnfmac/qlink_util.h    |     4 +
 drivers/net/wireless/quantenna/qtnfmac/util.c      |     8 +-
 drivers/net/wireless/quantenna/qtnfmac/util.h      |     4 +-
 drivers/net/wireless/ralink/rt2x00/rt2800lib.c     |     2 +-
 drivers/net/wireless/ralink/rt2x00/rt2x00debug.c   |     4 +-
 drivers/net/wireless/ralink/rt2x00/rt2x00mac.c     |    32 +-
 drivers/net/wireless/ralink/rt2x00/rt2x00queue.c   |    12 +-
 .../net/wireless/realtek/rtl818x/rtl8187/rtl8225.c |     5 +-
 drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h   |     2 +-
 drivers/net/wireless/realtek/rtlwifi/base.c        |   164 +-
 drivers/net/wireless/realtek/rtlwifi/base.h        |     2 +
 .../realtek/rtlwifi/btcoexist/halbtc8192e2ant.c    |   203 +-
 .../realtek/rtlwifi/btcoexist/halbtc8192e2ant.h    |     3 +-
 .../realtek/rtlwifi/btcoexist/halbtc8723b1ant.c    |   269 +-
 .../realtek/rtlwifi/btcoexist/halbtc8723b1ant.h    |     3 +-
 .../realtek/rtlwifi/btcoexist/halbtc8723b2ant.c    |   206 +-
 .../realtek/rtlwifi/btcoexist/halbtc8723b2ant.h    |     3 +-
 .../realtek/rtlwifi/btcoexist/halbtc8821a1ant.c    |   285 +-
 .../realtek/rtlwifi/btcoexist/halbtc8821a1ant.h    |     3 +-
 .../realtek/rtlwifi/btcoexist/halbtc8821a2ant.c    |   177 +-
 .../realtek/rtlwifi/btcoexist/halbtc8821a2ant.h    |     3 +-
 .../realtek/rtlwifi/btcoexist/halbtcoutsrc.c       |   663 +-
 .../realtek/rtlwifi/btcoexist/halbtcoutsrc.h       |   252 +-
 .../wireless/realtek/rtlwifi/btcoexist/rtl_btc.c   |   325 +-
 .../wireless/realtek/rtlwifi/btcoexist/rtl_btc.h   |    12 +-
 drivers/net/wireless/realtek/rtlwifi/core.c        |    52 +-
 drivers/net/wireless/realtek/rtlwifi/debug.c       |   483 +-
 drivers/net/wireless/realtek/rtlwifi/debug.h       |    12 +
 drivers/net/wireless/realtek/rtlwifi/efuse.c       |     4 +-
 drivers/net/wireless/realtek/rtlwifi/pci.c         |    94 +-
 drivers/net/wireless/realtek/rtlwifi/pci.h         |     5 +-
 drivers/net/wireless/realtek/rtlwifi/ps.c          |    29 +-
 drivers/net/wireless/realtek/rtlwifi/rc.c          |    16 +-
 .../net/wireless/realtek/rtlwifi/rtl8188ee/fw.c    |     2 +
 .../net/wireless/realtek/rtlwifi/rtl8188ee/hw.c    |    11 +-
 .../net/wireless/realtek/rtlwifi/rtl8188ee/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8192ce/hw.c    |    12 +-
 .../net/wireless/realtek/rtlwifi/rtl8192ce/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8192de/hw.c    |    12 +-
 .../net/wireless/realtek/rtlwifi/rtl8192de/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8192ee/hw.c    |    11 +-
 .../net/wireless/realtek/rtlwifi/rtl8192ee/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8192ee/trx.c   |    57 +-
 .../net/wireless/realtek/rtlwifi/rtl8192se/hw.c    |    12 +-
 .../net/wireless/realtek/rtlwifi/rtl8192se/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8723ae/hw.c    |     7 +-
 .../net/wireless/realtek/rtlwifi/rtl8723ae/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8723be/hw.c    |    25 +-
 .../net/wireless/realtek/rtlwifi/rtl8723be/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8723be/phy.c   |    12 +-
 .../net/wireless/realtek/rtlwifi/rtl8723be/table.c |    14 +-
 .../realtek/rtlwifi/rtl8723com/fw_common.c         |     4 +-
 .../net/wireless/realtek/rtlwifi/rtl8821ae/hw.c    |    16 +-
 .../net/wireless/realtek/rtlwifi/rtl8821ae/hw.h    |     3 +-
 .../net/wireless/realtek/rtlwifi/rtl8821ae/table.c |    34 +-
 drivers/net/wireless/realtek/rtlwifi/usb.c         |     1 -
 drivers/net/wireless/realtek/rtlwifi/wifi.h        |    69 +-
 drivers/net/wireless/ti/wl1251/init.c              |     4 +-
 drivers/net/wireless/ti/wl1251/main.c              |     3 +-
 drivers/net/wireless/ti/wlcore/acx.c               |     2 +-
 drivers/net/wireless/ti/wlcore/acx.h               |     2 -
 drivers/net/wireless/ti/wlcore/main.c              |    41 +-
 drivers/net/wireless/ti/wlcore/tx.c                |     4 +-
 drivers/net/xen-netfront.c                         |    53 +-
 drivers/ntb/hw/amd/ntb_hw_amd.c                    |     4 +
 drivers/ntb/hw/idt/ntb_hw_idt.c                    |    37 +-
 drivers/ntb/hw/intel/ntb_hw_intel.c                |   313 +-
 drivers/ntb/hw/intel/ntb_hw_intel.h                |    58 -
 drivers/ntb/hw/mscc/ntb_hw_switchtec.c             |   603 +-
 drivers/ntb/ntb.c                                  |     4 +-
 drivers/ntb/ntb_transport.c                        |     3 +
 drivers/ntb/test/ntb_perf.c                        |  1824 +-
 drivers/ntb/test/ntb_pingpong.c                    |   450 +-
 drivers/ntb/test/ntb_tool.c                        |  1827 +-
 drivers/nubus/Makefile                             |     2 +-
 drivers/nubus/bus.c                                |   117 +
 drivers/nubus/nubus.c                              |   542 +-
 drivers/nubus/proc.c                               |   281 +-
 drivers/nvdimm/blk.c                               |     3 +-
 drivers/nvdimm/btt.c                               |     5 +-
 drivers/nvdimm/bus.c                               |     3 -
 drivers/nvdimm/namespace_devs.c                    |     2 +-
 drivers/nvdimm/nd.h                                |     9 +-
 drivers/nvdimm/pfn_devs.c                          |    29 +-
 drivers/nvdimm/pmem.c                              |    38 +-
 drivers/nvdimm/pmem.h                              |     1 +
 drivers/nvdimm/region_devs.c                       |    22 +
 drivers/nvme/host/Makefile                         |     4 +
 drivers/nvme/host/core.c                           |   201 +-
 drivers/nvme/host/fabrics.c                        |    25 +-
 drivers/nvme/host/fabrics.h                        |    11 +-
 drivers/nvme/host/fc.c                             |   203 +-
 drivers/nvme/host/lightnvm.c                       |   185 +-
 drivers/nvme/host/multipath.c                      |    87 +-
 drivers/nvme/host/nvme.h                           |    20 +-
 drivers/nvme/host/pci.c                            |   265 +-
 drivers/nvme/host/rdma.c                           |    26 +-
 drivers/nvme/host/trace.c                          |   130 +
 drivers/nvme/host/trace.h                          |   165 +
 drivers/nvme/target/Kconfig                        |     2 +
 drivers/nvme/target/core.c                         |    23 +-
 drivers/nvme/target/fabrics-cmd.c                  |     2 +-
 drivers/nvme/target/fc.c                           |    60 +-
 drivers/nvme/target/fcloop.c                       |   244 +-
 drivers/nvme/target/io-cmd.c                       |     7 +-
 drivers/nvme/target/loop.c                         |     7 +-
 drivers/nvme/target/rdma.c                         |    83 +-
 drivers/nvmem/core.c                               |    17 +-
 drivers/nvmem/rockchip-efuse.c                     |    70 +-
 drivers/nvmem/uniphier-efuse.c                     |    10 +-
 drivers/of/Kconfig                                 |    22 +-
 drivers/of/Makefile                                |     2 -
 drivers/of/address.c                               |    11 +-
 drivers/of/base.c                                  |    32 +-
 drivers/of/device.c                                |     1 +
 drivers/of/dynamic.c                               |     1 +
 drivers/of/fdt.c                                   |    21 +-
 drivers/of/fdt_address.c                           |     6 +-
 drivers/of/irq.c                                   |     9 +-
 drivers/of/kobj.c                                  |     2 +-
 drivers/of/of_mdio.c                               |     5 +
 drivers/of/of_numa.c                               |    13 +-
 drivers/of/of_pci.c                                |   384 -
 drivers/of/of_pci_irq.c                            |   131 -
 drivers/of/of_private.h                            |     6 +-
 drivers/of/of_reserved_mem.c                       |     6 +-
 drivers/of/overlay.c                               |     5 +-
 drivers/of/pdt.c                                   |     6 +-
 drivers/of/platform.c                              |    13 +-
 drivers/of/property.c                              |    14 +-
 drivers/of/resolver.c                              |     5 +-
 drivers/of/unittest-data/Makefile                  |    19 +-
 drivers/of/unittest-data/overlay_bad_symbol.dts    |     1 +
 drivers/of/unittest.c                              |    11 +-
 drivers/opp/Makefile                               |     1 +
 drivers/opp/cpu.c                                  |     2 +-
 drivers/opp/ti-opp-supply.c                        |   425 +
 drivers/pci/Kconfig                                |     3 +
 drivers/pci/Makefile                               |    14 +-
 drivers/pci/access.c                               |     9 +-
 drivers/pci/ats.c                                  |     1 +
 drivers/pci/bus.c                                  |     5 +-
 drivers/pci/cadence/Kconfig                        |    27 +
 drivers/pci/cadence/Makefile                       |     4 +
 drivers/pci/cadence/pcie-cadence-ep.c              |   542 +
 drivers/pci/cadence/pcie-cadence-host.c            |   336 +
 drivers/pci/cadence/pcie-cadence.c                 |   126 +
 drivers/pci/cadence/pcie-cadence.h                 |   311 +
 drivers/pci/dwc/Kconfig                            |    70 +-
 drivers/pci/dwc/Makefile                           |     6 +-
 drivers/pci/dwc/pci-dra7xx.c                       |    47 +-
 drivers/pci/dwc/pci-exynos.c                       |   227 +-
 drivers/pci/dwc/pci-imx6.c                         |     5 +-
 drivers/pci/dwc/pci-keystone-dw.c                  |     6 +-
 drivers/pci/dwc/pci-keystone.c                     |    10 +-
 drivers/pci/dwc/pci-keystone.h                     |     6 +-
 drivers/pci/dwc/pci-layerscape.c                   |     5 +-
 drivers/pci/dwc/pcie-armada8k.c                    |     5 +-
 drivers/pci/dwc/pcie-artpec6.c                     |   475 +-
 drivers/pci/dwc/pcie-designware-ep.c               |   140 +-
 drivers/pci/dwc/pcie-designware-host.c             |    22 +-
 drivers/pci/dwc/pcie-designware-plat.c             |     5 +-
 drivers/pci/dwc/pcie-designware.c                  |     7 +-
 drivers/pci/dwc/pcie-designware.h                  |    41 +-
 drivers/pci/dwc/pcie-hisi.c                        |     5 +-
 drivers/pci/dwc/pcie-histb.c                       |     5 +-
 drivers/pci/dwc/pcie-kirin.c                       |     5 +-
 drivers/pci/dwc/pcie-qcom.c                        |    14 +-
 drivers/pci/dwc/pcie-spear13xx.c                   |     5 +-
 drivers/pci/ecam.c                                 |    13 +-
 drivers/pci/endpoint/Kconfig                       |     1 +
 drivers/pci/endpoint/Makefile                      |     1 +
 drivers/pci/endpoint/functions/Kconfig             |     1 +
 drivers/pci/endpoint/functions/Makefile            |     1 +
 drivers/pci/endpoint/functions/pci-epf-test.c      |    54 +-
 drivers/pci/endpoint/pci-ep-cfs.c                  |    78 +-
 drivers/pci/endpoint/pci-epc-core.c                |    85 +-
 drivers/pci/endpoint/pci-epc-mem.c                 |    13 +-
 drivers/pci/endpoint/pci-epf-core.c                |    17 +-
 drivers/pci/host-bridge.c                          |     1 +
 drivers/pci/host/Kconfig                           |     2 +
 drivers/pci/host/Makefile                          |     2 +
 drivers/pci/host/pci-aardvark.c                    |     5 +-
 drivers/pci/host/pci-host-common.c                 |    87 +-
 drivers/pci/host/pci-host-generic.c                |    13 +-
 drivers/pci/host/pci-hyperv.c                      |    12 +-
 drivers/pci/host/pci-mvebu.c                       |     5 +-
 drivers/pci/host/pci-rcar-gen2.c                   |     5 +-
 drivers/pci/host/pci-tegra.c                       |   171 +-
 drivers/pci/host/pci-thunder-ecam.c                |     5 +-
 drivers/pci/host/pci-thunder-pem.c                 |    13 +-
 drivers/pci/host/pci-v3-semi.c                     |     1 +
 drivers/pci/host/pci-versatile.c                   |    12 +-
 drivers/pci/host/pci-xgene-msi.c                   |    11 +-
 drivers/pci/host/pci-xgene.c                       |    13 +-
 drivers/pci/host/pcie-altera-msi.c                 |    13 +-
 drivers/pci/host/pcie-altera.c                     |    13 +-
 drivers/pci/host/pcie-iproc-bcma.c                 |    10 +-
 drivers/pci/host/pcie-iproc-msi.c                  |    10 +-
 drivers/pci/host/pcie-iproc-platform.c             |    17 +-
 drivers/pci/host/pcie-iproc.c                      |    18 +-
 drivers/pci/host/pcie-iproc.h                      |    12 +-
 drivers/pci/host/pcie-mediatek.c                   |    10 +-
 drivers/pci/host/pcie-rcar.c                       |    12 +-
 drivers/pci/host/pcie-rockchip.c                   |     6 +-
 drivers/pci/host/pcie-xilinx-nwl.c                 |     6 +-
 drivers/pci/host/pcie-xilinx.c                     |     6 +-
 drivers/pci/host/vmd.c                             |    10 +-
 drivers/pci/hotplug/Kconfig                        |     1 +
 drivers/pci/hotplug/acpi_pcihp.c                   |    16 +-
 drivers/pci/hotplug/acpiphp.h                      |    16 +-
 drivers/pci/hotplug/acpiphp_core.c                 |    16 +-
 drivers/pci/hotplug/acpiphp_glue.c                 |    20 +-
 drivers/pci/hotplug/acpiphp_ibm.c                  |    21 +-
 drivers/pci/hotplug/cpci_hotplug.h                 |    16 +-
 drivers/pci/hotplug/cpci_hotplug_core.c            |    16 +-
 drivers/pci/hotplug/cpci_hotplug_pci.c             |    16 +-
 drivers/pci/hotplug/cpcihp_generic.c               |    21 +-
 drivers/pci/hotplug/cpcihp_zt5550.c                |    21 +-
 drivers/pci/hotplug/cpcihp_zt5550.h                |    21 +-
 drivers/pci/hotplug/cpqphp.h                       |    16 +-
 drivers/pci/hotplug/cpqphp_core.c                  |    19 +-
 drivers/pci/hotplug/cpqphp_ctrl.c                  |    16 +-
 drivers/pci/hotplug/cpqphp_nvram.c                 |    16 +-
 drivers/pci/hotplug/cpqphp_nvram.h                 |    16 +-
 drivers/pci/hotplug/cpqphp_pci.c                   |    34 +-
 drivers/pci/hotplug/cpqphp_sysfs.c                 |    16 +-
 drivers/pci/hotplug/ibmphp.h                       |    16 +-
 drivers/pci/hotplug/ibmphp_core.c                  |    36 +-
 drivers/pci/hotplug/ibmphp_ebda.c                  |    16 +-
 drivers/pci/hotplug/ibmphp_hpc.c                   |    16 +-
 drivers/pci/hotplug/ibmphp_pci.c                   |    76 +-
 drivers/pci/hotplug/ibmphp_res.c                   |    54 +-
 drivers/pci/hotplug/pci_hotplug_core.c             |    16 +-
 drivers/pci/hotplug/pciehp.h                       |    16 +-
 drivers/pci/hotplug/pciehp_core.c                  |    16 +-
 drivers/pci/hotplug/pciehp_ctrl.c                  |    16 +-
 drivers/pci/hotplug/pciehp_hpc.c                   |    28 +-
 drivers/pci/hotplug/pciehp_pci.c                   |    28 +-
 drivers/pci/hotplug/pcihp_skeleton.c               |    16 +-
 drivers/pci/hotplug/pnv_php.c                      |    45 +-
 drivers/pci/hotplug/rpadlpar.h                     |     6 +-
 drivers/pci/hotplug/rpadlpar_core.c                |    19 +-
 drivers/pci/hotplug/rpadlpar_sysfs.c               |     9 +-
 drivers/pci/hotplug/rpaphp.h                       |    24 +-
 drivers/pci/hotplug/rpaphp_core.c                  |   123 +-
 drivers/pci/hotplug/rpaphp_pci.c                   |    16 +-
 drivers/pci/hotplug/rpaphp_slot.c                  |    16 +-
 drivers/pci/hotplug/s390_pci_hpc.c                 |     3 +-
 drivers/pci/hotplug/sgi_hotplug.c                  |    41 +-
 drivers/pci/hotplug/shpchp.h                       |    24 +-
 drivers/pci/hotplug/shpchp_core.c                  |    21 +-
 drivers/pci/hotplug/shpchp_ctrl.c                  |    16 +-
 drivers/pci/hotplug/shpchp_hpc.c                   |    16 +-
 drivers/pci/hotplug/shpchp_pci.c                   |    28 +-
 drivers/pci/hotplug/shpchp_sysfs.c                 |    16 +-
 drivers/pci/iov.c                                  |    22 +-
 drivers/pci/irq.c                                  |     7 +-
 drivers/pci/mmap.c                                 |     5 +-
 drivers/pci/msi.c                                  |     8 +-
 drivers/pci/of.c                                   |   571 +-
 drivers/pci/pci-acpi.c                             |     3 +-
 drivers/pci/pci-driver.c                           |    25 +-
 drivers/pci/pci-mid.c                              |     5 +-
 drivers/pci/pci-stub.c                             |     5 +-
 drivers/pci/pci-sysfs.c                            |    30 +-
 drivers/pci/pci.c                                  |   183 +-
 drivers/pci/pci.h                                  |    74 +-
 drivers/pci/pcie/Kconfig                           |     3 +-
 drivers/pci/pcie/aer/Kconfig                       |     1 +
 drivers/pci/pcie/aer/Kconfig.debug                 |     1 +
 drivers/pci/pcie/aer/aer_inject.c                  |    22 +-
 drivers/pci/pcie/aer/aerdrv.c                      |     7 +-
 drivers/pci/pcie/aer/aerdrv_core.c                 |    34 +-
 drivers/pci/pcie/aer/aerdrv_errprint.c             |    27 +-
 drivers/pci/pcie/aer/ecrc.c                        |    16 +-
 drivers/pci/pcie/aspm.c                            |    80 +-
 drivers/pci/pcie/pcie-dpc.c                        |   256 +-
 drivers/pci/pcie/pme.c                             |    11 +-
 drivers/pci/pcie/portdrv_acpi.c                    |     5 +-
 drivers/pci/pcie/portdrv_bus.c                     |     1 +
 drivers/pci/pcie/portdrv_core.c                    |     4 +-
 drivers/pci/pcie/portdrv_pci.c                     |     3 +
 drivers/pci/pcie/ptm.c                             |    12 +-
 drivers/pci/probe.c                                |   372 +-
 drivers/pci/quirks.c                               |   247 +-
 drivers/pci/remove.c                               |     1 +
 drivers/pci/rom.c                                  |     9 +-
 drivers/pci/search.c                               |     1 +
 drivers/pci/setup-bus.c                            |    49 +-
 drivers/pci/setup-irq.c                            |     5 +-
 drivers/pci/setup-res.c                            |    44 +-
 drivers/pci/slot.c                                 |     1 +
 drivers/pci/switch/Kconfig                         |     2 +
 drivers/pci/switch/Makefile                        |     1 +
 drivers/pci/switch/switchtec.c                     |    24 +-
 drivers/pci/syscall.c                              |     4 +-
 drivers/pci/vc.c                                   |    23 +-
 drivers/pci/vpd.c                                  |     1 +
 drivers/pci/xen-pcifront.c                         |    23 +-
 drivers/pcmcia/at91_cf.c                           |     4 +-
 drivers/pcmcia/rsrc_nonstatic.c                    |     2 +-
 drivers/pcmcia/soc_common.c                        |     8 +-
 drivers/perf/Kconfig                               |     9 +
 drivers/perf/Makefile                              |     1 +
 drivers/perf/arm_dsu_pmu.c                         |   841 +
 drivers/perf/arm_pmu.c                             |   138 +-
 drivers/perf/arm_pmu_acpi.c                        |    61 +-
 drivers/perf/arm_pmu_platform.c                    |    52 +-
 drivers/perf/arm_spe_pmu.c                         |     9 +
 drivers/phy/broadcom/phy-brcm-sata.c               |    31 +
 drivers/phy/broadcom/phy-brcm-usb-init.c           |    22 +-
 drivers/phy/broadcom/phy-brcm-usb.c                |     4 +-
 drivers/phy/mediatek/phy-mtk-tphy.c                |    35 +-
 drivers/phy/qualcomm/phy-qcom-ufs.c                |     5 +
 drivers/phy/samsung/phy-exynos5-usbdrd.c           |   183 +
 drivers/pinctrl/Kconfig                            |    25 +
 drivers/pinctrl/Makefile                           |     4 +-
 drivers/pinctrl/core.c                             |    24 +-
 drivers/pinctrl/freescale/pinctrl-imx.c            |    81 +-
 drivers/pinctrl/freescale/pinctrl-imx.h            |    13 +-
 drivers/pinctrl/freescale/pinctrl-imx25.c          |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx35.c          |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx50.c          |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx51.c          |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx53.c          |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx6dl.c         |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx6q.c          |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx6sl.c         |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx6sx.c         |     2 +-
 drivers/pinctrl/freescale/pinctrl-imx6ul.c         |    52 +-
 drivers/pinctrl/freescale/pinctrl-imx7d.c          |    10 +-
 drivers/pinctrl/freescale/pinctrl-imx7ulp.c        |     7 +-
 drivers/pinctrl/freescale/pinctrl-vf610.c          |     5 +-
 drivers/pinctrl/intel/pinctrl-baytrail.c           |     6 +
 drivers/pinctrl/intel/pinctrl-cannonlake.c         |    65 +-
 drivers/pinctrl/intel/pinctrl-cherryview.c         |    59 +-
 drivers/pinctrl/intel/pinctrl-intel.c              |   179 +-
 drivers/pinctrl/intel/pinctrl-intel.h              |     3 +
 drivers/pinctrl/intel/pinctrl-merrifield.c         |     7 +
 drivers/pinctrl/mediatek/Kconfig                   |    15 +-
 drivers/pinctrl/mediatek/Makefile                  |     3 +-
 drivers/pinctrl/mediatek/pinctrl-mt7622.c          |  1597 +
 drivers/pinctrl/meson/Kconfig                      |     9 +
 drivers/pinctrl/meson/Makefile                     |     2 +
 drivers/pinctrl/meson/pinctrl-meson-axg-pmx.c      |   118 +
 drivers/pinctrl/meson/pinctrl-meson-axg-pmx.h      |    62 +
 drivers/pinctrl/meson/pinctrl-meson-axg.c          |   975 +
 drivers/pinctrl/meson/pinctrl-meson.h              |     1 +
 drivers/pinctrl/mvebu/pinctrl-armada-37xx.c        |     4 +-
 drivers/pinctrl/mvebu/pinctrl-mvebu.c              |     5 +-
 drivers/pinctrl/nomadik/pinctrl-abx500.c           |    10 +-
 drivers/pinctrl/pinctrl-adi2.c                     |    13 +-
 drivers/pinctrl/pinctrl-at91-pio4.c                |     4 +-
 drivers/pinctrl/pinctrl-at91.c                     |     4 +-
 drivers/pinctrl/pinctrl-axp209.c                   |   478 +
 drivers/pinctrl/pinctrl-gemini.c                   |    84 +-
 drivers/pinctrl/pinctrl-ingenic.c                  |     4 +-
 drivers/pinctrl/pinctrl-mcp23s08.c                 |    54 +-
 drivers/pinctrl/pinctrl-ocelot.c                   |   511 +
 drivers/pinctrl/pinctrl-palmas.c                   |     4 +-
 drivers/pinctrl/pinctrl-rockchip.c                 |    31 +-
 drivers/pinctrl/pinctrl-single.c                   |    10 +-
 drivers/pinctrl/pinctrl-sx150x.c                   |    40 +-
 drivers/pinctrl/pinctrl-tz1090-pdc.c               |     9 +-
 drivers/pinctrl/pinctrl-tz1090.c                   |     9 +-
 drivers/pinctrl/pinctrl-utils.c                    |     4 +-
 drivers/pinctrl/pinctrl-xway.c                     |    10 +-
 drivers/pinctrl/pinmux.c                           |     4 +-
 drivers/pinctrl/pxa/pinctrl-pxa2xx.c               |     4 +
 drivers/pinctrl/qcom/Kconfig                       |     8 +
 drivers/pinctrl/qcom/Makefile                      |     1 +
 drivers/pinctrl/qcom/pinctrl-msm.c                 |     5 +-
 drivers/pinctrl/qcom/pinctrl-msm8998.c             |  1590 +
 drivers/pinctrl/samsung/Kconfig                    |     1 +
 drivers/pinctrl/samsung/pinctrl-exynos-arm.c       |    89 +-
 drivers/pinctrl/samsung/pinctrl-exynos-arm64.c     |    47 +-
 drivers/pinctrl/samsung/pinctrl-exynos.c           |    33 +-
 drivers/pinctrl/samsung/pinctrl-exynos.h           |     6 +-
 drivers/pinctrl/samsung/pinctrl-exynos5440.c       |    21 +-
 drivers/pinctrl/samsung/pinctrl-s3c24xx.c          |    51 +-
 drivers/pinctrl/samsung/pinctrl-s3c64xx.c          |    34 +-
 drivers/pinctrl/samsung/pinctrl-samsung.c          |    98 +-
 drivers/pinctrl/samsung/pinctrl-samsung.h          |    46 +-
 drivers/pinctrl/sh-pfc/Kconfig                     |     5 +
 drivers/pinctrl/sh-pfc/Makefile                    |     1 +
 drivers/pinctrl/sh-pfc/core.c                      |     6 +
 drivers/pinctrl/sh-pfc/pfc-r8a7791.c               |    62 +-
 drivers/pinctrl/sh-pfc/pfc-r8a7794.c               |   473 +
 drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c           |     2 +-
 drivers/pinctrl/sh-pfc/pfc-r8a7795.c               |   194 +-
 drivers/pinctrl/sh-pfc/pfc-r8a7796.c               |    66 +-
 drivers/pinctrl/sh-pfc/pfc-r8a77970.c              |  2329 ++
 drivers/pinctrl/sh-pfc/pfc-r8a77995.c              |    88 +
 drivers/pinctrl/sh-pfc/sh_pfc.h                    |    17 +-
 drivers/pinctrl/spear/pinctrl-plgpio.c             |     8 +-
 drivers/pinctrl/spear/pinctrl-spear.c              |     4 +-
 drivers/pinctrl/sprd/pinctrl-sprd.c                |     2 +-
 drivers/pinctrl/stm32/Kconfig                      |    12 +
 drivers/pinctrl/stm32/Makefile                     |     2 +
 drivers/pinctrl/stm32/pinctrl-stm32.c              |     3 +-
 drivers/pinctrl/stm32/pinctrl-stm32.h              |     3 +-
 drivers/pinctrl/stm32/pinctrl-stm32f429.c          |     3 +-
 drivers/pinctrl/stm32/pinctrl-stm32f469.c          |     6 +-
 drivers/pinctrl/stm32/pinctrl-stm32f746.c          |     3 +-
 drivers/pinctrl/stm32/pinctrl-stm32f769.c          |  1827 ++
 drivers/pinctrl/stm32/pinctrl-stm32h743.c          |     6 +-
 drivers/pinctrl/stm32/pinctrl-stm32mp157.c         |  2188 ++
 drivers/pinctrl/sunxi/pinctrl-sun4i-a10.c          |     2 +-
 drivers/pinctrl/sunxi/pinctrl-sunxi.c              |     7 +-
 drivers/pinctrl/tegra/pinctrl-tegra.c              |     9 +-
 drivers/pinctrl/uniphier/pinctrl-uniphier-core.c   |   176 +-
 drivers/pinctrl/vt8500/pinctrl-vt8500.c            |     4 +-
 drivers/pinctrl/vt8500/pinctrl-wm8505.c            |     4 +-
 drivers/pinctrl/vt8500/pinctrl-wm8650.c            |     4 +-
 drivers/pinctrl/vt8500/pinctrl-wm8750.c            |     4 +-
 drivers/pinctrl/vt8500/pinctrl-wm8850.c            |     4 +-
 drivers/platform/Kconfig                           |     2 +
 drivers/platform/Makefile                          |     1 +
 drivers/platform/chrome/Kconfig                    |    10 +-
 drivers/platform/chrome/Makefile                   |     7 +-
 drivers/platform/chrome/cros_ec_debugfs.c          |    11 +-
 drivers/platform/chrome/cros_ec_debugfs.h          |    27 -
 drivers/platform/chrome/cros_ec_lightbar.c         |     6 +-
 drivers/platform/chrome/cros_ec_lpc.c              |    44 +-
 drivers/platform/chrome/cros_ec_proto.c            |     8 +-
 drivers/platform/chrome/cros_ec_sysfs.c            |     7 +-
 drivers/platform/chrome/cros_ec_vbc.c              |     1 +
 drivers/platform/goldfish/goldfish_pipe.c          |    12 +-
 drivers/platform/mellanox/Kconfig                  |    26 +
 drivers/platform/mellanox/Makefile                 |     6 +
 drivers/platform/mellanox/mlxreg-hotplug.c         |   638 +
 drivers/platform/x86/Kconfig                       |    84 +-
 drivers/platform/x86/Makefile                      |     9 +-
 drivers/platform/x86/acer-wireless.c               |    71 +
 drivers/platform/x86/alienware-wmi.c               |    17 +
 drivers/platform/x86/apple-gmux.c                  |    48 +-
 drivers/platform/x86/asus-nb-wmi.c                 |    13 +-
 drivers/platform/x86/compal-laptop.c               |    18 +-
 drivers/platform/x86/dell-laptop.c                 |   256 +-
 .../x86/{dell-smbios.c => dell-smbios-base.c}      |    39 +-
 drivers/platform/x86/dell-smbios-smm.c             |    18 +-
 drivers/platform/x86/dell-smbios-wmi.c             |    14 +-
 drivers/platform/x86/dell-smbios.h                 |    27 +-
 drivers/platform/x86/dell-wmi.c                    |     5 +-
 drivers/platform/x86/gpd-pocket-fan.c              |   216 +
 drivers/platform/x86/ideapad-laptop.c              |    58 +-
 drivers/platform/x86/intel-hid.c                   |    42 +-
 drivers/platform/x86/intel-vbtn.c                  |    78 +-
 drivers/platform/x86/intel_chtdc_ti_pwrbtn.c       |    93 +
 drivers/platform/x86/intel_int0002_vgpio.c         |     2 +-
 drivers/platform/x86/intel_pmc_core.c              |   280 +-
 drivers/platform/x86/intel_pmc_core.h              |    29 +-
 drivers/platform/x86/intel_pmc_ipc.c               |    33 +-
 drivers/platform/x86/intel_telemetry_debugfs.c     |    83 +-
 drivers/platform/x86/mlx-platform.c                |   676 +-
 drivers/platform/x86/mlxcpld-hotplug.c             |   515 -
 drivers/platform/x86/pmc_atom.c                    |    42 +-
 drivers/platform/x86/samsung-laptop.c              |    18 +-
 drivers/platform/x86/silead_dmi.c                  |   175 +
 drivers/platform/x86/sony-laptop.c                 |     4 +-
 drivers/platform/x86/surfacepro3_button.c          |     4 +-
 drivers/platform/x86/thinkpad_acpi.c               |    18 +-
 drivers/platform/x86/wmi.c                         |     8 +-
 drivers/pnp/pnpbios/core.c                         |     5 +-
 drivers/pnp/quirks.c                               |     1 -
 drivers/power/avs/rockchip-io-domain.c             |    24 +-
 drivers/power/reset/Kconfig                        |     9 -
 drivers/power/reset/Makefile                       |     1 -
 drivers/power/reset/at91-sama5d2_shdwc.c           |     4 +-
 drivers/power/reset/imx-snvs-poweroff.c            |    66 -
 drivers/power/reset/msm-poweroff.c                 |     7 +-
 drivers/power/reset/zx-reboot.c                    |     4 +
 drivers/power/supply/ab8500_charger.c              |     6 +-
 drivers/power/supply/axp20x_ac_power.c             |     8 +-
 drivers/power/supply/axp288_charger.c              |   290 +-
 drivers/power/supply/axp288_fuel_gauge.c           |   203 +-
 drivers/power/supply/bq24190_charger.c             |   126 +-
 drivers/power/supply/bq27xxx_battery.c             |    39 +-
 drivers/power/supply/bq27xxx_battery_i2c.c         |     2 +
 drivers/power/supply/charger-manager.c             |     2 +-
 drivers/power/supply/cpcap-battery.c               |     4 +-
 drivers/power/supply/ltc2941-battery-gauge.c       |    25 +
 drivers/power/supply/max17042_battery.c            |    54 +-
 drivers/power/supply/sbs-manager.c                 |     2 +-
 drivers/powercap/intel_rapl.c                      |    99 +-
 drivers/powercap/powercap_sys.c                    |     6 +-
 drivers/pps/generators/pps_gen_parport.c           |    40 +-
 drivers/pps/pps.c                                  |     4 +-
 drivers/ps3/ps3av.c                                |    11 +-
 drivers/ptp/ptp_chardev.c                          |     4 +-
 drivers/ptp/ptp_private.h                          |     2 +-
 drivers/pwm/pwm-meson.c                            |    26 +
 drivers/pwm/pwm-stmpe.c                            |     2 +-
 drivers/pwm/sysfs.c                                |     1 +
 drivers/rapidio/devices/rio_mport_cdev.c           |     4 +-
 drivers/rapidio/devices/tsi721_dma.c               |    17 +-
 drivers/rapidio/rio.c                              |   110 +-
 drivers/ras/cec.c                                  |     2 +-
 drivers/regulator/Kconfig                          |     7 +
 drivers/regulator/Makefile                         |     1 +
 drivers/regulator/core.c                           |   383 +-
 drivers/regulator/internal.h                       |    27 +-
 drivers/regulator/of_regulator.c                   |    34 +-
 drivers/regulator/qcom_spmi-regulator.c            |    84 +-
 drivers/regulator/sc2731-regulator.c               |   256 +
 drivers/regulator/stm32-vrefbuf.c                  |     2 +-
 drivers/regulator/tps65218-regulator.c             |     5 +-
 drivers/remoteproc/qcom_adsp_pil.c                 |     8 +-
 drivers/remoteproc/qcom_common.c                   |    21 +-
 drivers/remoteproc/qcom_common.h                   |     4 -
 drivers/remoteproc/qcom_q6v5_pil.c                 |    18 +-
 drivers/remoteproc/qcom_wcnss.c                    |     8 +-
 drivers/remoteproc/remoteproc_core.c               |    83 +-
 drivers/remoteproc/remoteproc_elf_loader.c         |    59 +-
 drivers/remoteproc/remoteproc_internal.h           |    57 +-
 drivers/remoteproc/remoteproc_virtio.c             |     2 +-
 drivers/remoteproc/st_slim_rproc.c                 |    32 +-
 drivers/reset/core.c                               |     9 +-
 drivers/reset/reset-meson.c                        |     2 +
 drivers/rpmsg/qcom_glink_native.c                  |     1 +
 drivers/rpmsg/qcom_glink_smem.c                    |     5 +-
 drivers/rpmsg/qcom_smd.c                           |    72 +-
 drivers/rpmsg/rpmsg_char.c                         |     8 +-
 drivers/rpmsg/rpmsg_core.c                         |     2 +-
 drivers/rpmsg/rpmsg_internal.h                     |     2 +-
 drivers/rtc/Kconfig                                |    27 +-
 drivers/rtc/Makefile                               |     3 +-
 drivers/rtc/rtc-ac100.c                            |    25 +-
 drivers/rtc/rtc-at32ap700x.c                       |   287 -
 drivers/rtc/rtc-brcmstb-waketimer.c                |    15 +-
 drivers/rtc/rtc-cros-ec.c                          |   413 +
 drivers/rtc/rtc-dev.c                              |     4 +-
 drivers/rtc/rtc-mxc_v2.c                           |   419 +
 drivers/rtc/rtc-omap.c                             |     5 +-
 drivers/rtc/rtc-opal.c                             |    12 +-
 drivers/rtc/rtc-r7301.c                            |     4 +-
 drivers/rtc/rtc-r9701.c                            |     6 -
 drivers/rtc/rtc-stm32.c                            |     4 +-
 drivers/rtc/rtc-sun6i.c                            |     4 +-
 drivers/s390/block/Kconfig                         |     1 +
 drivers/s390/block/dasd.c                          |    33 +-
 drivers/s390/block/dasd_eckd.c                     |     7 +-
 drivers/s390/block/dasd_eer.c                      |     6 +-
 drivers/s390/block/dcssblk.c                       |     3 +-
 drivers/s390/char/Makefile                         |     2 +
 drivers/s390/char/monreader.c                      |     6 +-
 drivers/s390/char/sclp_early.c                     |     3 +-
 drivers/s390/char/sclp_early_core.c                |     4 +-
 drivers/s390/cio/chp.c                             |    10 +-
 drivers/s390/cio/cio.c                             |     2 +-
 drivers/s390/cio/cmf.c                             |    15 +-
 drivers/s390/cio/css.c                             |     8 +-
 drivers/s390/cio/device.c                          |    10 +-
 drivers/s390/cio/device_fsm.c                      |     7 +-
 drivers/s390/cio/device_ops.c                      |    72 +-
 drivers/s390/cio/io_sch.h                          |     1 +
 drivers/s390/cio/itcw.c                            |     2 +-
 drivers/s390/cio/qdio_main.c                       |     4 +-
 drivers/s390/cio/vfio_ccw_cp.c                     |     2 +-
 drivers/s390/crypto/ap_card.c                      |     2 +-
 drivers/s390/net/Kconfig                           |     3 -
 drivers/s390/net/lcs.c                             |    10 +-
 drivers/s390/net/lcs.h                             |     3 +-
 drivers/s390/net/qeth_core.h                       |    49 +-
 drivers/s390/net/qeth_core_main.c                  |   169 +-
 drivers/s390/net/qeth_core_mpc.h                   |    13 +-
 drivers/s390/net/qeth_l2.h                         |     3 +-
 drivers/s390/net/qeth_l2_main.c                    |    94 +-
 drivers/s390/net/qeth_l3.h                         |    49 +-
 drivers/s390/net/qeth_l3_main.c                    |   621 +-
 drivers/s390/net/qeth_l3_sys.c                     |   114 +-
 drivers/s390/virtio/virtio_ccw.c                   |    29 +
 drivers/sbus/char/Kconfig                          |     8 +
 drivers/sbus/char/Makefile                         |     1 +
 drivers/sbus/char/openprom.c                       |     5 +-
 drivers/sbus/char/oradax.c                         |  1005 +
 drivers/scsi/3w-9xxx.c                             |    26 +-
 drivers/scsi/3w-9xxx.h                             |     2 +-
 drivers/scsi/3w-sas.c                              |    15 +-
 drivers/scsi/Makefile                              |     1 -
 drivers/scsi/aacraid/aachba.c                      |   473 +-
 drivers/scsi/aacraid/aacraid.h                     |    54 +-
 drivers/scsi/aacraid/commctrl.c                    |     6 +-
 drivers/scsi/aacraid/comminit.c                    |    49 +-
 drivers/scsi/aacraid/commsup.c                     |   220 +-
 drivers/scsi/aacraid/linit.c                       |    30 +-
 drivers/scsi/aacraid/sa.c                          |    32 +-
 drivers/scsi/aic7xxx/aiclib.c                      |    34 -
 drivers/scsi/arcmsr/arcmsr.h                       |   567 +-
 drivers/scsi/arcmsr/arcmsr_hba.c                   |  1318 +-
 drivers/scsi/arm/fas216.c                          |     2 +-
 drivers/scsi/bfa/bfa_core.c                        |     2 +-
 drivers/scsi/bfa/bfa_cs.h                          |     6 +-
 drivers/scsi/bfa/bfa_defs_svc.h                    |     3 +-
 drivers/scsi/bfa/bfa_fcbuild.c                     |     8 +-
 drivers/scsi/bfa/bfa_fcpim.c                       |     3 +-
 drivers/scsi/bfa/bfa_fcpim.h                       |     4 +-
 drivers/scsi/bfa/bfa_fcs.c                         |    78 +-
 drivers/scsi/bfa/bfa_fcs_lport.c                   |    62 +-
 drivers/scsi/bfa/bfa_ioc.c                         |    10 +-
 drivers/scsi/bfa/bfa_port.c                        |    15 +-
 drivers/scsi/bfa/bfa_port.h                        |     2 +-
 drivers/scsi/bfa/bfa_svc.c                         |    51 +-
 drivers/scsi/bfa/bfa_svc.h                         |     2 +-
 drivers/scsi/bfa/bfad.c                            |    23 +-
 drivers/scsi/bfa/bfad_attr.c                       |     5 +-
 drivers/scsi/bfa/bfad_bsg.c                        |    10 +-
 drivers/scsi/bfa/bfad_debugfs.c                    |     8 +-
 drivers/scsi/bfa/bfad_im.h                         |    32 +-
 drivers/scsi/bnx2fc/bnx2fc_fcoe.c                  |     4 +-
 drivers/scsi/bnx2fc/bnx2fc_hwi.c                   |    60 +-
 drivers/scsi/bnx2fc/bnx2fc_io.c                    |     1 +
 drivers/scsi/bnx2fc/bnx2fc_tgt.c                   |    51 +-
 drivers/scsi/bnx2i/bnx2i_hwi.c                     |    19 +-
 drivers/scsi/csiostor/csio_init.c                  |     2 +-
 drivers/scsi/csiostor/csio_init.h                  |     1 -
 drivers/scsi/csiostor/csio_lnode.c                 |     2 +-
 drivers/scsi/csiostor/csio_mb.c                    |     6 +-
 drivers/scsi/cxgbi/libcxgbi.c                      |     2 +-
 drivers/scsi/cxlflash/Makefile                     |     2 +-
 drivers/scsi/cxlflash/backend.h                    |    41 +
 drivers/scsi/cxlflash/common.h                     |     8 +-
 drivers/scsi/cxlflash/cxl_hw.c                     |   168 +
 drivers/scsi/cxlflash/main.c                       |   100 +-
 drivers/scsi/cxlflash/superpipe.c                  |    64 +-
 drivers/scsi/cxlflash/superpipe.h                  |     4 +-
 drivers/scsi/device_handler/scsi_dh_alua.c         |    42 +-
 drivers/scsi/fnic/fnic_debugfs.c                   |    22 +-
 drivers/scsi/fnic/fnic_fcs.c                       |     4 +-
 drivers/scsi/fnic/fnic_scsi.c                      |     2 +-
 drivers/scsi/fnic/fnic_stats.h                     |     4 +-
 drivers/scsi/fnic/fnic_trace.c                     |    58 +-
 drivers/scsi/hisi_sas/hisi_sas.h                   |    46 +-
 drivers/scsi/hisi_sas/hisi_sas_main.c              |   267 +-
 drivers/scsi/hisi_sas/hisi_sas_v1_hw.c             |     6 +-
 drivers/scsi/hisi_sas/hisi_sas_v2_hw.c             |   194 +-
 drivers/scsi/hisi_sas/hisi_sas_v3_hw.c             |   332 +-
 drivers/scsi/hosts.c                               |     4 +
 drivers/scsi/hpsa.c                                |   101 +-
 drivers/scsi/hpsa.h                                |     1 +
 drivers/scsi/ibmvscsi/ibmvfc.c                     |    40 +-
 drivers/scsi/ibmvscsi/ibmvfc.h                     |     2 +-
 drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c           |   320 +-
 drivers/scsi/ipr.c                                 |     4 +-
 drivers/scsi/iscsi_tcp.c                           |    10 +
 drivers/scsi/libiscsi.c                            |    28 +-
 drivers/scsi/libiscsi_tcp.c                        |     9 +-
 drivers/scsi/libsas/sas_ata.c                      |     1 -
 drivers/scsi/libsas/sas_discover.c                 |    34 +-
 drivers/scsi/libsas/sas_event.c                    |    86 +-
 drivers/scsi/libsas/sas_expander.c                 |    12 +-
 drivers/scsi/libsas/sas_init.c                     |   107 +-
 drivers/scsi/libsas/sas_internal.h                 |     7 +
 drivers/scsi/libsas/sas_phy.c                      |    69 +-
 drivers/scsi/libsas/sas_port.c                     |    25 +-
 drivers/scsi/libsas/sas_scsi_host.c                |    53 +-
 drivers/scsi/lpfc/lpfc.h                           |     6 +-
 drivers/scsi/lpfc/lpfc_attr.c                      |   123 +-
 drivers/scsi/lpfc/lpfc_crtn.h                      |     2 +
 drivers/scsi/lpfc/lpfc_ct.c                        |    20 +
 drivers/scsi/lpfc/lpfc_debugfs.c                   |    65 +-
 drivers/scsi/lpfc/lpfc_disc.h                      |     2 +
 drivers/scsi/lpfc/lpfc_els.c                       |   116 +-
 drivers/scsi/lpfc/lpfc_hbadisc.c                   |    29 +-
 drivers/scsi/lpfc/lpfc_hw4.h                       |     6 +-
 drivers/scsi/lpfc/lpfc_init.c                      |   294 +-
 drivers/scsi/lpfc/lpfc_nportdisc.c                 |    88 +-
 drivers/scsi/lpfc/lpfc_nvme.c                      |   356 +-
 drivers/scsi/lpfc/lpfc_nvme.h                      |    17 +-
 drivers/scsi/lpfc/lpfc_nvmet.c                     |    71 +-
 drivers/scsi/lpfc/lpfc_nvmet.h                     |    10 +
 drivers/scsi/lpfc/lpfc_sli.c                       |   193 +-
 drivers/scsi/lpfc/lpfc_sli4.h                      |    14 +-
 drivers/scsi/lpfc/lpfc_version.h                   |     2 +-
 drivers/scsi/megaraid/megaraid_sas.h               |    38 +-
 drivers/scsi/megaraid/megaraid_sas_base.c          |   216 +-
 drivers/scsi/megaraid/megaraid_sas_fp.c            |    28 +-
 drivers/scsi/megaraid/megaraid_sas_fusion.c        |   153 +-
 drivers/scsi/megaraid/megaraid_sas_fusion.h        |     6 +
 drivers/scsi/mpt3sas/mpt3sas_base.c                |   220 +-
 drivers/scsi/mpt3sas/mpt3sas_base.h                |    32 +-
 drivers/scsi/mpt3sas/mpt3sas_ctl.c                 |    35 +-
 drivers/scsi/mpt3sas/mpt3sas_scsih.c               |   355 +-
 drivers/scsi/mpt3sas/mpt3sas_warpdrive.c           |    33 +-
 drivers/scsi/pmcraid.c                             |     2 +-
 drivers/scsi/ppa.c                                 |     4 +
 drivers/scsi/qedf/drv_fcoe_fw_funcs.c              |    35 +-
 drivers/scsi/qedf/drv_fcoe_fw_funcs.h              |     2 +-
 drivers/scsi/qedf/qedf.h                           |     4 +-
 drivers/scsi/qedf/qedf_els.c                       |     2 +-
 drivers/scsi/qedf/qedf_hsi.h                       |    68 -
 drivers/scsi/qedf/qedf_io.c                        |    35 +-
 drivers/scsi/qedf/qedf_main.c                      |    15 +-
 drivers/scsi/qedf/qedf_version.h                   |     8 +-
 drivers/scsi/qedi/qedi_debugfs.c                   |     4 +-
 drivers/scsi/qedi/qedi_fw.c                        |    66 +-
 drivers/scsi/qedi/qedi_fw_api.c                    |   139 +-
 drivers/scsi/qedi/qedi_fw_iscsi.h                  |     2 +-
 drivers/scsi/qedi/qedi_gbl.h                       |     5 +-
 drivers/scsi/qedi/qedi_iscsi.c                     |     9 +-
 drivers/scsi/qedi/qedi_iscsi.h                     |     2 +-
 drivers/scsi/qedi/qedi_main.c                      |   130 +-
 drivers/scsi/qedi/qedi_version.h                   |     8 +-
 drivers/scsi/qla2xxx/qla_attr.c                    |     7 +-
 drivers/scsi/qla2xxx/qla_bsg.c                     |     9 +-
 drivers/scsi/qla2xxx/qla_def.h                     |   178 +-
 drivers/scsi/qla2xxx/qla_dfs.c                     |    39 +-
 drivers/scsi/qla2xxx/qla_fw.h                      |     2 +-
 drivers/scsi/qla2xxx/qla_gbl.h                     |    32 +-
 drivers/scsi/qla2xxx/qla_gs.c                      |  1502 +-
 drivers/scsi/qla2xxx/qla_init.c                    |  1113 +-
 drivers/scsi/qla2xxx/qla_inline.h                  |     1 +
 drivers/scsi/qla2xxx/qla_iocb.c                    |    64 +-
 drivers/scsi/qla2xxx/qla_isr.c                     |    62 +-
 drivers/scsi/qla2xxx/qla_mbx.c                     |   151 +-
 drivers/scsi/qla2xxx/qla_mid.c                     |   140 +-
 drivers/scsi/qla2xxx/qla_nx2.c                     |     2 -
 drivers/scsi/qla2xxx/qla_os.c                      |   597 +-
 drivers/scsi/qla2xxx/qla_sup.c                     |     1 +
 drivers/scsi/qla2xxx/qla_target.c                  |   762 +-
 drivers/scsi/qla2xxx/qla_target.h                  |     4 +-
 drivers/scsi/qla2xxx/qla_tmpl.c                    |    40 +-
 drivers/scsi/qla2xxx/qla_version.h                 |     2 +-
 drivers/scsi/qla2xxx/tcm_qla2xxx.c                 |     5 +-
 drivers/scsi/qla4xxx/ql4_def.h                     |     2 +
 drivers/scsi/qla4xxx/ql4_init.c                    |     5 +-
 drivers/scsi/qla4xxx/ql4_mbx.c                     |    21 +-
 drivers/scsi/qla4xxx/ql4_nx.c                      |     5 +-
 drivers/scsi/qla4xxx/ql4_os.c                      |    58 +-
 drivers/scsi/scsi_common.c                         |    14 +-
 drivers/scsi/scsi_debug.c                          |   724 +-
 drivers/scsi/scsi_debugfs.c                        |    40 +-
 drivers/scsi/scsi_devinfo.c                        |    39 +-
 drivers/scsi/scsi_dh.c                             |     5 +-
 drivers/scsi/scsi_error.c                          |    22 +-
 drivers/scsi/scsi_lib.c                            |    69 +-
 drivers/scsi/scsi_priv.h                           |    15 +-
 drivers/scsi/scsi_sysfs.c                          |     3 -
 drivers/scsi/scsi_transport_fc.c                   |     4 +-
 drivers/scsi/scsi_transport_spi.c                  |    16 +-
 drivers/scsi/sd.c                                  |    52 +-
 drivers/scsi/sd.h                                  |    11 -
 drivers/scsi/sd_zbc.c                              |   268 +-
 drivers/scsi/ses.c                                 |    11 +-
 drivers/scsi/sg.c                                  |    18 +-
 drivers/scsi/smartpqi/Makefile                     |     2 +-
 drivers/scsi/st.c                                  |     2 +-
 drivers/scsi/storvsc_drv.c                         |     9 +-
 drivers/scsi/sym53c8xx_2/sym_hipd.c                |     2 +-
 drivers/scsi/ufs/ufshcd.c                          |     2 +
 drivers/scsi/ufs/ufshci.h                          |    16 +
 drivers/scsi/virtio_scsi.c                         |   129 +-
 drivers/scsi/wd719x.c                              |     4 +-
 drivers/sh/clk/core.c                              |     5 +-
 drivers/siox/Kconfig                               |    18 +
 drivers/siox/Makefile                              |     2 +
 drivers/siox/siox-bus-gpio.c                       |   172 +
 drivers/siox/siox-core.c                           |   934 +
 drivers/siox/siox.h                                |    49 +
 drivers/slimbus/Kconfig                            |    24 +
 drivers/slimbus/Makefile                           |    10 +
 drivers/slimbus/core.c                             |   480 +
 drivers/slimbus/messaging.c                        |   332 +
 drivers/slimbus/qcom-ctrl.c                        |   747 +
 drivers/slimbus/sched.c                            |   121 +
 drivers/slimbus/slimbus.h                          |   261 +
 drivers/soc/Kconfig                                |     1 +
 drivers/soc/Makefile                               |     2 +
 drivers/soc/actions/owl-sps.c                      |    42 +
 drivers/soc/bcm/brcmstb/biuctrl.c                  |   192 +-
 drivers/soc/bcm/brcmstb/common.c                   |    43 +-
 drivers/soc/fsl/guts.c                             |     8 +
 drivers/soc/fsl/qbman/qman.c                       |    28 +-
 drivers/soc/gemini/Makefile                        |     2 +
 drivers/soc/gemini/soc-gemini.c                    |    71 +
 drivers/soc/imx/gpc.c                              |    28 +-
 drivers/soc/mediatek/mtk-infracfg.c                |    26 +-
 drivers/soc/mediatek/mtk-scpsys.c                  |   140 +-
 drivers/soc/qcom/Kconfig                           |    10 +
 drivers/soc/qcom/Makefile                          |     2 +
 drivers/soc/qcom/qmi_encdec.c                      |   816 +
 drivers/soc/qcom/qmi_interface.c                   |   848 +
 drivers/soc/qcom/rmtfs_mem.c                       |     4 +
 drivers/soc/qcom/smp2p.c                           |    55 +-
 drivers/soc/qcom/smsm.c                            |     6 +-
 drivers/soc/renesas/rcar-sysc.c                    |     2 +-
 drivers/soc/renesas/renesas-soc.c                  |     3 +
 drivers/soc/samsung/Kconfig                        |     1 +
 drivers/soc/samsung/Makefile                       |     1 +
 drivers/soc/samsung/exynos-pmu.c                   |    16 +-
 drivers/soc/samsung/exynos-pmu.h                   |     5 +-
 drivers/soc/samsung/exynos3250-pmu.c               |    16 +-
 drivers/soc/samsung/exynos4-pmu.c                  |    16 +-
 drivers/soc/samsung/exynos5250-pmu.c               |    16 +-
 drivers/soc/samsung/exynos5420-pmu.c               |    16 +-
 drivers/soc/samsung/pm_domains.c                   |    24 +-
 drivers/soc/tegra/Kconfig                          |     5 +-
 drivers/soc/tegra/Makefile                         |     1 -
 drivers/soc/tegra/fuse/fuse-tegra.c                |     4 +
 drivers/soc/tegra/fuse/fuse-tegra20.c              |    13 +-
 drivers/soc/tegra/fuse/fuse-tegra30.c              |    24 +-
 drivers/soc/tegra/fuse/fuse.h                      |     4 +
 drivers/soc/tegra/fuse/tegra-apbmisc.c             |    11 +-
 drivers/soc/tegra/pmc-tegra186.c                   |   169 -
 drivers/soc/tegra/pmc.c                            |   304 +-
 drivers/soc/ti/knav_qmss_queue.c                   |     4 +-
 drivers/soc/xilinx/Kconfig                         |    20 +
 drivers/soc/xilinx/Makefile                        |     2 +
 drivers/soc/xilinx/xlnx_vcu.c                      |   630 +
 drivers/soundwire/Kconfig                          |    37 +
 drivers/soundwire/Makefile                         |    18 +
 drivers/soundwire/bus.c                            |   997 +
 drivers/soundwire/bus.h                            |    71 +
 drivers/soundwire/bus_type.c                       |   193 +
 drivers/soundwire/cadence_master.c                 |   751 +
 drivers/soundwire/cadence_master.h                 |    48 +
 drivers/soundwire/intel.c                          |   345 +
 drivers/soundwire/intel.h                          |    23 +
 drivers/soundwire/intel_init.c                     |   198 +
 drivers/soundwire/mipi_disco.c                     |   401 +
 drivers/soundwire/slave.c                          |   114 +
 drivers/spi/spi-armada-3700.c                      |   110 +-
 drivers/spi/spi-atmel.c                            |   113 +-
 drivers/spi/spi-bcm53xx.c                          |    26 +-
 drivers/spi/spi-davinci.c                          |     4 +-
 drivers/spi/spi-dw.c                               |     2 -
 drivers/spi/spi-fsl-dspi.c                         |     9 +-
 drivers/spi/spi-imx.c                              |    26 +-
 drivers/spi/spi-jcore.c                            |     4 +-
 drivers/spi/spi-meson-spicc.c                      |     1 +
 drivers/spi/spi-orion.c                            |    21 +-
 drivers/spi/spi-pxa2xx.c                           |     4 +-
 drivers/spi/spi-s3c64xx.c                          |    18 +-
 drivers/spi/spi-sh-msiof.c                         |   128 +-
 drivers/spi/spi-sirf.c                             |     4 +-
 drivers/spi/spi-sun6i.c                            |     2 +-
 drivers/spi/spi-xilinx.c                           |     1 +
 drivers/ssb/Kconfig                                |    11 +-
 drivers/staging/Kconfig                            |     4 +
 drivers/staging/Makefile                           |     2 +
 drivers/staging/android/ashmem.c                   |    69 +-
 drivers/staging/android/ashmem.h                   |     1 +
 drivers/staging/android/ion/ion-ioctl.c            |    16 +-
 drivers/staging/android/ion/ion.c                  |    40 +-
 drivers/staging/android/ion/ion.h                  |    16 +-
 drivers/staging/android/ion/ion_carveout_heap.c    |    11 +-
 drivers/staging/android/ion/ion_chunk_heap.c       |    11 +-
 drivers/staging/android/ion/ion_cma_heap.c         |    28 +-
 drivers/staging/android/ion/ion_heap.c             |    16 +-
 drivers/staging/android/ion/ion_page_pool.c        |    11 +-
 drivers/staging/android/ion/ion_system_heap.c      |    13 +-
 drivers/staging/android/uapi/ashmem.h              |     1 +
 drivers/staging/android/uapi/ion.h                 |    11 +-
 .../devicetree/bindings/crypto/arm-cryptocell.txt  |    27 -
 drivers/staging/ccree/Kconfig                      |     2 +
 drivers/staging/ccree/Makefile                     |     8 +-
 drivers/staging/ccree/TODO                         |    22 +-
 drivers/staging/ccree/{ssi_aead.c => cc_aead.c}    |  1119 +-
 drivers/staging/ccree/{ssi_aead.h => cc_aead.h}    |    56 +-
 .../ccree/{ssi_buffer_mgr.c => cc_buffer_mgr.c}    |  1085 +-
 drivers/staging/ccree/cc_buffer_mgr.h              |    74 +
 .../staging/ccree/{ssi_cipher.c => cc_cipher.c}    |   715 +-
 drivers/staging/ccree/cc_cipher.h                  |    74 +
 drivers/staging/ccree/cc_crypto_ctx.h              |    38 +-
 drivers/staging/ccree/cc_debugfs.c                 |   101 +
 drivers/staging/ccree/cc_debugfs.h                 |    32 +
 .../staging/ccree/{ssi_driver.c => cc_driver.c}    |   346 +-
 drivers/staging/ccree/cc_driver.h                  |   194 +
 drivers/staging/ccree/{ssi_fips.c => cc_fips.c}    |    45 +-
 drivers/staging/ccree/cc_fips.h                    |    37 +
 drivers/staging/ccree/{ssi_hash.c => cc_hash.c}    |  1820 +-
 drivers/staging/ccree/cc_hash.h                    |   114 +
 drivers/staging/ccree/cc_host_regs.h               |   142 +
 drivers/staging/ccree/cc_hw_queue_defs.h           |    40 +-
 drivers/staging/ccree/{ssi_ivgen.c => cc_ivgen.c}  |   139 +-
 drivers/staging/ccree/cc_ivgen.h                   |    55 +
 drivers/staging/ccree/cc_kernel_regs.h             |   167 +
 drivers/staging/ccree/cc_lli_defs.h                |    19 +-
 drivers/staging/ccree/cc_pm.c                      |   122 +
 drivers/staging/ccree/cc_pm.h                      |    57 +
 drivers/staging/ccree/cc_request_mgr.c             |   713 +
 drivers/staging/ccree/cc_request_mgr.h             |    51 +
 .../ccree/{ssi_sram_mgr.c => cc_sram_mgr.c}        |    70 +-
 drivers/staging/ccree/cc_sram_mgr.h                |    65 +
 drivers/staging/ccree/dx_crys_kernel.h             |   180 -
 drivers/staging/ccree/dx_host.h                    |   155 -
 drivers/staging/ccree/dx_reg_common.h              |    26 -
 drivers/staging/ccree/hash_defs.h                  |    36 -
 drivers/staging/ccree/ssi_buffer_mgr.h             |    91 -
 drivers/staging/ccree/ssi_cipher.h                 |    84 -
 drivers/staging/ccree/ssi_config.h                 |    36 -
 drivers/staging/ccree/ssi_driver.h                 |   206 -
 drivers/staging/ccree/ssi_fips.h                   |    49 -
 drivers/staging/ccree/ssi_hash.h                   |   103 -
 drivers/staging/ccree/ssi_ivgen.h                  |    71 -
 drivers/staging/ccree/ssi_pm.c                     |   145 -
 drivers/staging/ccree/ssi_pm.h                     |    43 -
 drivers/staging/ccree/ssi_request_mgr.c            |   610 -
 drivers/staging/ccree/ssi_request_mgr.h            |    60 -
 drivers/staging/ccree/ssi_sram_mgr.h               |    79 -
 drivers/staging/ccree/ssi_sysfs.c                  |   172 -
 drivers/staging/ccree/ssi_sysfs.h                  |    55 -
 .../clocking-wizard/clk-xlnx-clock-wizard.c        |    13 +-
 drivers/staging/comedi/comedi.h                    |    11 +-
 drivers/staging/comedi/comedi_buf.c                |    11 +-
 drivers/staging/comedi/comedi_compat32.c           |    11 +-
 drivers/staging/comedi/comedi_compat32.h           |    11 +-
 drivers/staging/comedi/comedi_fops.c               |    19 +-
 drivers/staging/comedi/comedi_pci.c                |    11 +-
 drivers/staging/comedi/comedi_pci.h                |    11 +-
 drivers/staging/comedi/comedi_pcmcia.c             |    11 +-
 drivers/staging/comedi/comedi_pcmcia.h             |    11 +-
 drivers/staging/comedi/comedi_usb.c                |    11 +-
 drivers/staging/comedi/comedi_usb.h                |    11 +-
 drivers/staging/comedi/comedidev.h                 |    51 +-
 drivers/staging/comedi/comedilib.h                 |    11 +-
 drivers/staging/comedi/drivers.c                   |    14 +-
 drivers/staging/comedi/drivers/8255.c              |    11 +-
 drivers/staging/comedi/drivers/8255.h              |    11 +-
 drivers/staging/comedi/drivers/8255_pci.c          |    11 +-
 drivers/staging/comedi/drivers/addi_apci_1032.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_1500.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_1516.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_1564.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_16xx.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_2032.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_2200.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_3120.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_3501.c    |    11 +-
 drivers/staging/comedi/drivers/addi_apci_3xxx.c    |    11 +-
 drivers/staging/comedi/drivers/addi_watchdog.c     |    11 +-
 drivers/staging/comedi/drivers/adl_pci6208.c       |    11 +-
 drivers/staging/comedi/drivers/adl_pci7x3x.c       |    11 +-
 drivers/staging/comedi/drivers/adl_pci8164.c       |    11 +-
 drivers/staging/comedi/drivers/adl_pci9111.c       |    11 +-
 drivers/staging/comedi/drivers/adl_pci9118.c       |     1 +
 drivers/staging/comedi/drivers/adq12b.c            |    11 +-
 drivers/staging/comedi/drivers/adv_pci1710.c       |     1 +
 drivers/staging/comedi/drivers/adv_pci1720.c       |    11 +-
 drivers/staging/comedi/drivers/adv_pci1723.c       |    11 +-
 drivers/staging/comedi/drivers/adv_pci1724.c       |    11 +-
 drivers/staging/comedi/drivers/adv_pci1760.c       |    13 +-
 drivers/staging/comedi/drivers/adv_pci_dio.c       |     1 +
 drivers/staging/comedi/drivers/aio_aio12_8.c       |    11 +-
 drivers/staging/comedi/drivers/aio_iiro_16.c       |    11 +-
 drivers/staging/comedi/drivers/amplc_dio200.c      |    11 +-
 drivers/staging/comedi/drivers/amplc_dio200.h      |    11 +-
 .../staging/comedi/drivers/amplc_dio200_common.c   |    11 +-
 drivers/staging/comedi/drivers/amplc_dio200_pci.c  |    11 +-
 drivers/staging/comedi/drivers/amplc_pc236.c       |    11 +-
 drivers/staging/comedi/drivers/amplc_pc236.h       |    11 +-
 .../staging/comedi/drivers/amplc_pc236_common.c    |    11 +-
 drivers/staging/comedi/drivers/amplc_pc263.c       |    11 +-
 drivers/staging/comedi/drivers/amplc_pci224.c      |    11 +-
 drivers/staging/comedi/drivers/amplc_pci230.c      |    11 +-
 drivers/staging/comedi/drivers/amplc_pci236.c      |    11 +-
 drivers/staging/comedi/drivers/amplc_pci263.c      |    11 +-
 drivers/staging/comedi/drivers/c6xdigio.c          |    11 +-
 drivers/staging/comedi/drivers/cb_das16_cs.c       |    11 +-
 drivers/staging/comedi/drivers/cb_pcidas.c         |    11 +-
 drivers/staging/comedi/drivers/cb_pcidas64.c       |    11 +-
 drivers/staging/comedi/drivers/cb_pcidda.c         |    11 +-
 drivers/staging/comedi/drivers/cb_pcimdas.c        |    11 +-
 drivers/staging/comedi/drivers/cb_pcimdda.c        |    11 +-
 drivers/staging/comedi/drivers/comedi_8254.c       |    11 +-
 drivers/staging/comedi/drivers/comedi_8254.h       |    11 +-
 drivers/staging/comedi/drivers/comedi_8255.c       |    11 +-
 drivers/staging/comedi/drivers/comedi_bond.c       |    11 +-
 drivers/staging/comedi/drivers/comedi_isadma.c     |    11 +-
 drivers/staging/comedi/drivers/comedi_isadma.h     |    11 +-
 drivers/staging/comedi/drivers/comedi_parport.c    |    11 +-
 drivers/staging/comedi/drivers/comedi_test.c       |    11 +-
 drivers/staging/comedi/drivers/contec_pci_dio.c    |    11 +-
 drivers/staging/comedi/drivers/dac02.c             |    11 +-
 drivers/staging/comedi/drivers/daqboard2000.c      |    11 +-
 drivers/staging/comedi/drivers/das08.c             |    11 +-
 drivers/staging/comedi/drivers/das08.h             |    11 +-
 drivers/staging/comedi/drivers/das08_cs.c          |    11 +-
 drivers/staging/comedi/drivers/das08_isa.c         |    11 +-
 drivers/staging/comedi/drivers/das08_pci.c         |    11 +-
 drivers/staging/comedi/drivers/das16.c             |    11 +-
 drivers/staging/comedi/drivers/das16m1.c           |    11 +-
 drivers/staging/comedi/drivers/das1800.c           |    11 +-
 drivers/staging/comedi/drivers/das6402.c           |    11 +-
 drivers/staging/comedi/drivers/das800.c            |    11 +-
 drivers/staging/comedi/drivers/dmm32at.c           |    11 +-
 drivers/staging/comedi/drivers/dt2801.c            |     1 +
 drivers/staging/comedi/drivers/dt2811.c            |    13 +-
 drivers/staging/comedi/drivers/dt2814.c            |    11 +-
 drivers/staging/comedi/drivers/dt2815.c            |    11 +-
 drivers/staging/comedi/drivers/dt2817.c            |    11 +-
 drivers/staging/comedi/drivers/dt282x.c            |    11 +-
 drivers/staging/comedi/drivers/dt3000.c            |    11 +-
 drivers/staging/comedi/drivers/dt9812.c            |    11 +-
 drivers/staging/comedi/drivers/dyna_pci10xx.c      |    11 +-
 drivers/staging/comedi/drivers/fl512.c             |    11 +-
 drivers/staging/comedi/drivers/gsc_hpdi.c          |    11 +-
 drivers/staging/comedi/drivers/icp_multi.c         |    11 +-
 drivers/staging/comedi/drivers/ii_pci20kc.c        |     1 +
 drivers/staging/comedi/drivers/jr3_pci.c           |    11 +-
 drivers/staging/comedi/drivers/ke_counter.c        |    11 +-
 drivers/staging/comedi/drivers/me4000.c            |    11 +-
 drivers/staging/comedi/drivers/me_daq.c            |    11 +-
 drivers/staging/comedi/drivers/mf6x4.c             |    11 +-
 drivers/staging/comedi/drivers/mite.c              |    11 +-
 drivers/staging/comedi/drivers/mite.h              |    11 +-
 drivers/staging/comedi/drivers/mpc624.c            |    11 +-
 drivers/staging/comedi/drivers/multiq3.c           |    11 +-
 drivers/staging/comedi/drivers/ni_6527.c           |    11 +-
 drivers/staging/comedi/drivers/ni_65xx.c           |    11 +-
 drivers/staging/comedi/drivers/ni_660x.c           |    11 +-
 drivers/staging/comedi/drivers/ni_670x.c           |    11 +-
 drivers/staging/comedi/drivers/ni_at_a2150.c       |    11 +-
 drivers/staging/comedi/drivers/ni_at_ao.c          |    11 +-
 drivers/staging/comedi/drivers/ni_atmio.c          |    11 +-
 drivers/staging/comedi/drivers/ni_atmio16d.c       |    11 +-
 drivers/staging/comedi/drivers/ni_daq_700.c        |    11 +-
 drivers/staging/comedi/drivers/ni_daq_dio24.c      |    11 +-
 drivers/staging/comedi/drivers/ni_labpc.c          |    11 +-
 drivers/staging/comedi/drivers/ni_labpc.h          |    11 +-
 drivers/staging/comedi/drivers/ni_labpc_common.c   |    11 +-
 drivers/staging/comedi/drivers/ni_labpc_cs.c       |    11 +-
 drivers/staging/comedi/drivers/ni_labpc_isadma.c   |    11 +-
 drivers/staging/comedi/drivers/ni_labpc_pci.c      |    11 +-
 drivers/staging/comedi/drivers/ni_mio_common.c     |    11 +-
 drivers/staging/comedi/drivers/ni_mio_cs.c         |    11 +-
 drivers/staging/comedi/drivers/ni_pcidio.c         |    11 +-
 drivers/staging/comedi/drivers/ni_pcimio.c         |    11 +-
 drivers/staging/comedi/drivers/ni_stc.h            |    11 +-
 drivers/staging/comedi/drivers/ni_tio.c            |    11 +-
 drivers/staging/comedi/drivers/ni_tio.h            |    11 +-
 drivers/staging/comedi/drivers/ni_tio_internal.h   |    11 +-
 drivers/staging/comedi/drivers/ni_tiocmd.c         |    11 +-
 drivers/staging/comedi/drivers/ni_usb6501.c        |    11 +-
 drivers/staging/comedi/drivers/pcl711.c            |    11 +-
 drivers/staging/comedi/drivers/pcl724.c            |     1 +
 drivers/staging/comedi/drivers/pcl726.c            |    11 +-
 drivers/staging/comedi/drivers/pcl730.c            |     1 +
 drivers/staging/comedi/drivers/pcl812.c            |     1 +
 drivers/staging/comedi/drivers/pcl816.c            |     1 +
 drivers/staging/comedi/drivers/pcl818.c            |     1 +
 drivers/staging/comedi/drivers/pcm3724.c           |     1 +
 drivers/staging/comedi/drivers/pcmad.c             |    11 +-
 drivers/staging/comedi/drivers/pcmda12.c           |    11 +-
 drivers/staging/comedi/drivers/pcmmio.c            |    11 +-
 drivers/staging/comedi/drivers/pcmuio.c            |    11 +-
 drivers/staging/comedi/drivers/plx9052.h           |    11 +-
 drivers/staging/comedi/drivers/plx9080.h           |     6 +-
 drivers/staging/comedi/drivers/rtd520.c            |    11 +-
 drivers/staging/comedi/drivers/rti800.c            |    11 +-
 drivers/staging/comedi/drivers/rti802.c            |    11 +-
 drivers/staging/comedi/drivers/s526.c              |    11 +-
 drivers/staging/comedi/drivers/s626.c              |    20 +-
 drivers/staging/comedi/drivers/s626.h              |    11 +-
 drivers/staging/comedi/drivers/serial2002.c        |    18 +-
 drivers/staging/comedi/drivers/ssv_dnp.c           |    11 +-
 drivers/staging/comedi/drivers/usbdux.c            |    14 +-
 drivers/staging/comedi/drivers/usbduxfast.c        |    11 +-
 drivers/staging/comedi/drivers/usbduxsigma.c       |    11 +-
 drivers/staging/comedi/drivers/vmk80xx.c           |    11 +-
 .../staging/comedi/kcomedilib/kcomedilib_main.c    |    11 +-
 drivers/staging/comedi/proc.c                      |    11 +-
 drivers/staging/comedi/range.c                     |    11 +-
 drivers/staging/dgnc/Makefile                      |     3 +-
 drivers/staging/dgnc/dgnc_cls.c                    |    38 +-
 drivers/staging/dgnc/dgnc_cls.h                    |    11 +-
 drivers/staging/dgnc/dgnc_driver.c                 |   160 +-
 drivers/staging/dgnc/dgnc_driver.h                 |    52 +-
 drivers/staging/dgnc/dgnc_mgmt.c                   |   248 -
 drivers/staging/dgnc/dgnc_mgmt.h                   |    26 -
 drivers/staging/dgnc/dgnc_neo.c                    |  1690 -
 drivers/staging/dgnc/dgnc_neo.h                    |   175 -
 drivers/staging/dgnc/dgnc_pci.h                    |    69 -
 drivers/staging/dgnc/dgnc_tty.c                    |    20 +-
 drivers/staging/dgnc/dgnc_tty.h                    |    11 +-
 drivers/staging/dgnc/dgnc_utils.c                  |    17 -
 drivers/staging/dgnc/dgnc_utils.h                  |     7 -
 drivers/staging/dgnc/digi.h                        |    88 +-
 drivers/staging/emxx_udc/emxx_udc.c                |    10 +-
 drivers/staging/emxx_udc/emxx_udc.h                |    10 +-
 drivers/staging/fbtft/Kconfig                      |     6 -
 drivers/staging/fbtft/fb_agm1264k-fl.c             |    11 +-
 drivers/staging/fbtft/fb_bd663474.c                |    11 +-
 drivers/staging/fbtft/fb_hx8340bn.c                |    11 +-
 drivers/staging/fbtft/fb_hx8347d.c                 |    11 +-
 drivers/staging/fbtft/fb_hx8353d.c                 |    11 +-
 drivers/staging/fbtft/fb_hx8357d.c                 |    11 +-
 drivers/staging/fbtft/fb_hx8357d.h                 |     1 +
 drivers/staging/fbtft/fb_ili9163.c                 |    11 +-
 drivers/staging/fbtft/fb_ili9320.c                 |    11 +-
 drivers/staging/fbtft/fb_ili9325.c                 |    11 +-
 drivers/staging/fbtft/fb_ili9340.c                 |    11 +-
 drivers/staging/fbtft/fb_ili9341.c                 |    11 +-
 drivers/staging/fbtft/fb_ili9481.c                 |    11 +-
 drivers/staging/fbtft/fb_ili9486.c                 |    11 +-
 drivers/staging/fbtft/fb_pcd8544.c                 |    11 +-
 drivers/staging/fbtft/fb_ra8875.c                  |    11 +-
 drivers/staging/fbtft/fb_s6d02a1.c                 |    11 +-
 drivers/staging/fbtft/fb_s6d1121.c                 |    11 +-
 drivers/staging/fbtft/fb_sh1106.c                  |    11 +-
 drivers/staging/fbtft/fb_ssd1289.c                 |    11 +-
 drivers/staging/fbtft/fb_ssd1305.c                 |    11 +-
 drivers/staging/fbtft/fb_ssd1306.c                 |    11 +-
 drivers/staging/fbtft/fb_ssd1325.c                 |    11 +-
 drivers/staging/fbtft/fb_ssd1331.c                 |     1 +
 drivers/staging/fbtft/fb_ssd1351.c                 |     1 +
 drivers/staging/fbtft/fb_st7735r.c                 |    11 +-
 drivers/staging/fbtft/fb_st7789v.c                 |    11 +-
 drivers/staging/fbtft/fb_tinylcd.c                 |    11 +-
 drivers/staging/fbtft/fb_tls8204.c                 |    11 +-
 drivers/staging/fbtft/fb_uc1611.c                  |    11 +-
 drivers/staging/fbtft/fb_uc1701.c                  |    11 +-
 drivers/staging/fbtft/fb_upd161704.c               |    11 +-
 drivers/staging/fbtft/fb_watterott.c               |    11 +-
 drivers/staging/fbtft/fbtft-core.c                 |    27 +-
 drivers/staging/fbtft/fbtft.h                      |    15 +-
 drivers/staging/fbtft/fbtft_device.c               |    11 +-
 drivers/staging/fbtft/flexfb.c                     |    11 +-
 drivers/staging/fbtft/internal.h                   |    16 +-
 drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.c     |   165 +-
 drivers/staging/fsl-dpaa2/ethernet/dpaa2-eth.h     |    35 +-
 drivers/staging/fsl-dpaa2/ethernet/dpaa2-ethtool.c |     1 +
 drivers/staging/fsl-mc/Kconfig                     |     1 +
 drivers/staging/fsl-mc/Makefile                    |     1 +
 drivers/staging/fsl-mc/README.txt                  |   386 -
 drivers/staging/fsl-mc/bus/Kconfig                 |     5 +-
 drivers/staging/fsl-mc/bus/Makefile                |     3 +-
 drivers/staging/fsl-mc/bus/dpbp-cmd.h              |    28 +-
 drivers/staging/fsl-mc/bus/dpbp.c                  |    44 +-
 drivers/staging/fsl-mc/bus/dpcon-cmd.h             |    34 +-
 drivers/staging/fsl-mc/bus/dpcon.c                 |   115 +-
 drivers/staging/fsl-mc/bus/dpio/Makefile           |     1 +
 drivers/staging/fsl-mc/bus/dpio/dpio-cmd.h         |    28 +-
 drivers/staging/fsl-mc/bus/dpio/dpio-driver.c      |    27 +-
 drivers/staging/fsl-mc/bus/dpio/dpio-service.c     |   129 +-
 drivers/staging/fsl-mc/bus/dpio/dpio.c             |    28 +-
 drivers/staging/fsl-mc/bus/dpio/dpio.h             |    28 +-
 drivers/staging/fsl-mc/bus/dpio/qbman-portal.c     |    27 +-
 drivers/staging/fsl-mc/bus/dpio/qbman-portal.h     |    27 +-
 drivers/staging/fsl-mc/bus/dpmcp-cmd.h             |    56 -
 drivers/staging/fsl-mc/bus/dpmcp.c                 |    63 +-
 drivers/staging/fsl-mc/bus/dpmcp.h                 |    60 -
 drivers/staging/fsl-mc/bus/dpmng-cmd.h             |    58 -
 drivers/staging/fsl-mc/bus/dprc-cmd.h              |   451 -
 drivers/staging/fsl-mc/bus/dprc-driver.c           |    82 +-
 drivers/staging/fsl-mc/bus/dprc.c                  |   268 +-
 drivers/staging/fsl-mc/bus/dprc.h                  |   268 -
 drivers/staging/fsl-mc/bus/fsl-mc-allocator.c      |   117 +-
 drivers/staging/fsl-mc/bus/fsl-mc-bus.c            |   112 +-
 drivers/staging/fsl-mc/bus/fsl-mc-msi.c            |    19 +-
 drivers/staging/fsl-mc/bus/fsl-mc-private.h        |   379 +-
 .../staging/fsl-mc/bus/irq-gic-v3-its-fsl-mc-msi.c |    36 +-
 drivers/staging/fsl-mc/bus/mc-io.c                 |    62 +-
 drivers/staging/fsl-mc/bus/mc-sys.c                |    37 +-
 drivers/staging/fsl-mc/include/dpaa2-fd.h          |    27 +-
 drivers/staging/fsl-mc/include/dpaa2-global.h      |    27 +-
 drivers/staging/fsl-mc/include/dpaa2-io.h          |    33 +-
 drivers/staging/fsl-mc/include/dpbp.h              |    29 +-
 drivers/staging/fsl-mc/include/dpcon.h             |    42 +-
 drivers/staging/fsl-mc/include/mc.h                |    61 +-
 drivers/staging/fsl-mc/overview.rst                |   404 +
 drivers/staging/fwserial/dma_fifo.c                |    11 +-
 drivers/staging/fwserial/dma_fifo.h                |    11 +-
 drivers/staging/fwserial/fwserial.c                |    13 +-
 drivers/staging/greybus/arche-apb-ctrl.c           |    39 +-
 drivers/staging/greybus/arche-platform.c           |    21 +-
 drivers/staging/greybus/audio_codec.c              |    23 +-
 drivers/staging/greybus/audio_codec.h              |    12 +-
 drivers/staging/greybus/authentication.c           |     2 +-
 drivers/staging/greybus/camera.c                   |    16 +-
 drivers/staging/greybus/tools/loopback_test.c      |     4 +-
 drivers/staging/gs_fpgaboot/io.h                   |     5 -
 drivers/staging/iio/adc/ad7192.c                   |    29 +-
 drivers/staging/iio/cdc/ad7152.c                   |    25 +-
 drivers/staging/iio/cdc/ad7746.c                   |    17 +-
 drivers/staging/iio/impedance-analyzer/ad5933.c    |     4 +-
 drivers/staging/iio/light/tsl2x7x.c                |     9 +-
 drivers/staging/iio/trigger/iio-trig-bfin-timer.c  |    19 +-
 {net => drivers/staging}/ipx/Kconfig               |     1 +
 {net => drivers/staging}/ipx/Makefile              |     0
 drivers/staging/ipx/TODO                           |     4 +
 {net => drivers/staging}/ipx/af_ipx.c              |     0
 {net => drivers/staging}/ipx/ipx_proc.c            |     3 -
 {net => drivers/staging}/ipx/ipx_route.c           |     0
 {net => drivers/staging}/ipx/pe2.c                 |     0
 {net => drivers/staging}/ipx/sysctl_net_ipx.c      |     0
 drivers/staging/irda/drivers/pxaficp_ir.c          |     1 -
 drivers/staging/irda/drivers/sh_sir.c              |     4 +-
 drivers/staging/irda/net/af_irda.c                 |    18 +-
 drivers/staging/irda/net/irlmp.c                   |     4 +-
 drivers/staging/irda/net/irnet/irnet_ppp.c         |    16 +-
 drivers/staging/irda/net/irnet/irnet_ppp.h         |     2 +-
 drivers/staging/ks7010/ks_wlan_net.c               |     4 +-
 .../staging/lustre/include/linux/libcfs/libcfs.h   |    13 +-
 .../lustre/include/linux/libcfs/libcfs_cpu.h       |     4 +-
 .../lustre/include/linux/libcfs/libcfs_hash.h      |     6 +-
 .../lustre/include/linux/libcfs/libcfs_private.h   |   130 +-
 .../lustre/include/linux/libcfs/libcfs_string.h    |     5 +-
 .../lustre/include/linux/libcfs/libcfs_workitem.h  |   107 -
 .../lustre/include/linux/libcfs/linux/linux-cpu.h  |     4 +-
 .../staging/lustre/include/linux/lnet/lib-lnet.h   |    63 +-
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c    |   160 +-
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h    |     2 +-
 .../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c |    10 +-
 .../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c  |     4 +-
 .../staging/lustre/lnet/klnds/socklnd/socklnd.c    |    61 +-
 .../staging/lustre/lnet/klnds/socklnd/socklnd_cb.c |     6 +-
 .../lustre/lnet/klnds/socklnd/socklnd_proto.c      |     8 +-
 drivers/staging/lustre/lnet/libcfs/Makefile        |     2 +-
 drivers/staging/lustre/lnet/libcfs/fail.c          |     2 +-
 drivers/staging/lustre/lnet/libcfs/hash.c          |   100 +-
 drivers/staging/lustre/lnet/libcfs/libcfs_cpu.c    |     6 +-
 drivers/staging/lustre/lnet/libcfs/libcfs_lock.c   |     6 +-
 drivers/staging/lustre/lnet/libcfs/libcfs_mem.c    |    24 +-
 drivers/staging/lustre/lnet/libcfs/libcfs_string.c |    32 +-
 .../staging/lustre/lnet/libcfs/linux/linux-cpu.c   |   151 +-
 .../lustre/lnet/libcfs/linux/linux-crypto-adler.c  |     1 +
 .../lustre/lnet/libcfs/linux/linux-module.c        |     4 +-
 drivers/staging/lustre/lnet/libcfs/module.c        |    36 +-
 drivers/staging/lustre/lnet/libcfs/prng.c          |   137 -
 drivers/staging/lustre/lnet/libcfs/tracefile.c     |     2 +-
 drivers/staging/lustre/lnet/libcfs/workitem.c      |   466 -
 drivers/staging/lustre/lnet/lnet/api-ni.c          |    36 +-
 drivers/staging/lustre/lnet/lnet/config.c          |    53 +-
 drivers/staging/lustre/lnet/lnet/lib-eq.c          |    15 +-
 drivers/staging/lustre/lnet/lnet/lib-md.c          |     8 +-
 drivers/staging/lustre/lnet/lnet/lib-me.c          |    10 +-
 drivers/staging/lustre/lnet/lnet/lib-move.c        |    25 +-
 drivers/staging/lustre/lnet/lnet/lib-msg.c         |    20 +-
 drivers/staging/lustre/lnet/lnet/lib-ptl.c         |     8 +-
 drivers/staging/lustre/lnet/lnet/lib-socket.c      |    38 +-
 drivers/staging/lustre/lnet/lnet/net_fault.c       |    48 +-
 drivers/staging/lustre/lnet/lnet/nidstrings.c      |     8 +-
 drivers/staging/lustre/lnet/lnet/peer.c            |    10 +-
 drivers/staging/lustre/lnet/lnet/router.c          |    60 +-
 drivers/staging/lustre/lnet/lnet/router_proc.c     |    42 +-
 drivers/staging/lustre/lnet/selftest/conctl.c      |   191 +-
 drivers/staging/lustre/lnet/selftest/conrpc.c      |    10 +-
 drivers/staging/lustre/lnet/selftest/console.c     |    83 +-
 drivers/staging/lustre/lnet/selftest/framework.c   |    40 +-
 drivers/staging/lustre/lnet/selftest/module.c      |    48 +-
 drivers/staging/lustre/lnet/selftest/rpc.c         |    96 +-
 drivers/staging/lustre/lnet/selftest/selftest.h    |    46 +-
 drivers/staging/lustre/lustre/fld/fld_cache.c      |    13 +-
 .../staging/lustre/lustre/include/lustre_disk.h    |     6 +-
 drivers/staging/lustre/lustre/include/lustre_sec.h |     4 +-
 drivers/staging/lustre/lustre/include/obd_class.h  |   305 +-
 drivers/staging/lustre/lustre/ldlm/ldlm_extent.c   |     4 +-
 drivers/staging/lustre/lustre/ldlm/ldlm_lock.c     |    18 +-
 drivers/staging/lustre/lustre/ldlm/ldlm_lockd.c    |    14 +-
 drivers/staging/lustre/lustre/ldlm/ldlm_pool.c     |     8 +-
 drivers/staging/lustre/lustre/ldlm/ldlm_request.c  |    12 +-
 drivers/staging/lustre/lustre/ldlm/ldlm_resource.c |    20 +-
 drivers/staging/lustre/lustre/llite/dir.c          |     7 +-
 .../staging/lustre/lustre/llite/llite_internal.h   |    12 +-
 drivers/staging/lustre/lustre/llite/llite_lib.c    |    11 +-
 drivers/staging/lustre/lustre/llite/super25.c      |    23 +-
 drivers/staging/lustre/lustre/llite/vvp_dev.c      |     2 +-
 drivers/staging/lustre/lustre/llite/vvp_io.c       |     2 +-
 drivers/staging/lustre/lustre/lov/lov_internal.h   |    12 +-
 drivers/staging/lustre/lustre/lov/lov_obd.c        |     6 +-
 drivers/staging/lustre/lustre/lov/lov_object.c     |     2 +-
 drivers/staging/lustre/lustre/lov/lov_request.c    |    10 +-
 drivers/staging/lustre/lustre/mgc/mgc_request.c    |     4 +-
 drivers/staging/lustre/lustre/obdclass/cl_lock.c   |     9 +-
 drivers/staging/lustre/lustre/obdclass/cl_object.c |     8 +-
 drivers/staging/lustre/lustre/obdclass/cl_page.c   |     4 +-
 drivers/staging/lustre/lustre/obdclass/class_obd.c |     3 +-
 .../lustre/lustre/obdclass/linux/linux-module.c    |     3 +
 .../lustre/lustre/obdclass/lprocfs_status.c        |    21 +-
 drivers/staging/lustre/lustre/obdclass/lu_object.c |    24 +-
 .../lustre/lustre/obdclass/lustre_handles.c        |     9 +-
 .../staging/lustre/lustre/obdclass/obd_config.c    |     2 +-
 drivers/staging/lustre/lustre/obdclass/obd_mount.c |    63 +-
 drivers/staging/lustre/lustre/osc/osc_internal.h   |     6 +-
 drivers/staging/lustre/lustre/osc/osc_request.c    |     4 +-
 drivers/staging/lustre/lustre/ptlrpc/client.c      |     2 +-
 drivers/staging/lustre/lustre/ptlrpc/sec_bulk.c    |     8 +-
 drivers/staging/media/Kconfig                      |    17 +-
 drivers/staging/media/Makefile                     |     2 +-
 drivers/staging/media/atomisp/i2c/atomisp-gc0310.c |    10 +-
 drivers/staging/media/atomisp/i2c/atomisp-gc2235.c |     8 +-
 drivers/staging/media/atomisp/i2c/atomisp-lm3554.c |    38 +-
 .../staging/media/atomisp/i2c/atomisp-mt9m114.c    |     8 +-
 drivers/staging/media/atomisp/i2c/atomisp-ov2680.c |    10 +-
 drivers/staging/media/atomisp/i2c/atomisp-ov2722.c |    17 +-
 drivers/staging/media/atomisp/i2c/ov2680.h         |     1 -
 .../media/atomisp/i2c/ov5693/atomisp-ov5693.c      |    94 +-
 drivers/staging/media/atomisp/i2c/ov5693/ov5693.h  |     2 +-
 drivers/staging/media/atomisp/i2c/ov8858.c         |    43 +-
 .../staging/media/atomisp/include/linux/atomisp.h  |     2 +
 .../atomisp/include/linux/atomisp_gmin_platform.h  |     1 -
 .../media/atomisp/pci/atomisp2/atomisp_drvfs.c     |    17 +-
 .../media/atomisp/pci/atomisp2/atomisp_drvfs.h     |     5 +-
 .../media/atomisp/pci/atomisp2/atomisp_fops.c      |     4 +-
 .../media/atomisp/pci/atomisp2/atomisp_internal.h  |     1 -
 .../media/atomisp/pci/atomisp2/atomisp_ioctl.c     |     5 +-
 .../media/atomisp/pci/atomisp2/atomisp_subdev.c    |     2 +
 .../media/atomisp/pci/atomisp2/atomisp_v4l2.c      |    12 +-
 .../isp/kernels/eed1_8/ia_css_eed1_8.host.c        |    24 +-
 .../css2400/runtime/debug/src/ia_css_debug.c       |     1 +
 .../isp_param/interface/ia_css_isp_param_types.h   |     2 +-
 .../staging/media/atomisp/pci/atomisp2/hmm/hmm.c   |     8 +-
 .../platform/intel-mid/atomisp_gmin_platform.c     |   129 +-
 drivers/staging/media/bcm2048/radio-bcm2048.c      |     6 +-
 drivers/staging/media/cxd2099/Makefile             |     1 -
 drivers/staging/media/cxd2099/cxd2099.c            |    30 +-
 drivers/staging/media/cxd2099/cxd2099.h            |    16 +-
 drivers/staging/media/davinci_vpfe/TODO            |    10 +-
 drivers/staging/media/davinci_vpfe/vpfe_video.c    |     2 +-
 drivers/staging/media/imx/TODO                     |    63 +-
 drivers/staging/media/imx/imx-ic-prp.c             |     4 +-
 drivers/staging/media/imx/imx-media-capture.c      |     8 +-
 drivers/staging/media/imx/imx-media-csi.c          |   197 +-
 drivers/staging/media/imx/imx-media-dev.c          |   400 +-
 drivers/staging/media/imx/imx-media-fim.c          |    30 +-
 drivers/staging/media/imx/imx-media-internal-sd.c  |   253 +-
 drivers/staging/media/imx/imx-media-of.c           |   278 +-
 drivers/staging/media/imx/imx-media-utils.c        |   122 +-
 drivers/staging/media/imx/imx-media.h              |   191 +-
 drivers/staging/media/imx/imx6-mipi-csi2.c         |     4 +-
 drivers/staging/media/lirc/Kconfig                 |    21 -
 drivers/staging/media/lirc/Makefile                |     6 -
 drivers/staging/media/lirc/TODO                    |    36 -
 drivers/staging/media/lirc/lirc_zilog.c            |  1653 -
 drivers/staging/media/omap4iss/iss.c               |     2 +-
 drivers/staging/media/omap4iss/iss_video.c         |     2 +-
 drivers/staging/media/tegra-vde/Kconfig            |     8 +
 drivers/staging/media/tegra-vde/Makefile           |     1 +
 drivers/staging/media/tegra-vde/TODO               |     4 +
 drivers/staging/media/tegra-vde/tegra-vde.c        |  1213 +
 drivers/staging/media/tegra-vde/uapi.h             |    78 +
 .../most/Documentation/ABI/sysfs-bus-most.txt      |   313 +
 .../staging/most/Documentation/driver_usage.txt    |   192 +-
 drivers/staging/most/Kconfig                       |    27 +-
 drivers/staging/most/Makefile                      |    19 +-
 drivers/staging/most/aim-cdev/Makefile             |     4 -
 drivers/staging/most/aim-network/Makefile          |     4 -
 drivers/staging/most/aim-sound/Makefile            |     4 -
 drivers/staging/most/aim-v4l2/Makefile             |     5 -
 drivers/staging/most/{aim-cdev => cdev}/Kconfig    |     6 +-
 drivers/staging/most/cdev/Makefile                 |     4 +
 drivers/staging/most/{aim-cdev => cdev}/cdev.c     |   200 +-
 drivers/staging/most/core.c                        |  1603 +
 .../staging/most/{mostcore/mostcore.h => core.h}   |    77 +-
 drivers/staging/most/{hdm-dim2 => dim2}/Kconfig    |     6 +-
 drivers/staging/most/dim2/Makefile                 |     4 +
 .../most/{hdm-dim2/dim2_hdm.c => dim2/dim2.c}      |    45 +-
 .../most/{hdm-dim2/dim2_hdm.h => dim2/dim2.h}      |    10 +-
 .../most/{hdm-dim2/dim2_errors.h => dim2/errors.h} |    10 +-
 .../most/{hdm-dim2/dim2_hal.c => dim2/hal.c}       |    16 +-
 .../most/{hdm-dim2/dim2_hal.h => dim2/hal.h}       |    12 +-
 .../most/{hdm-dim2/dim2_reg.h => dim2/reg.h}       |    10 +-
 drivers/staging/most/dim2/sysfs.c                  |    49 +
 drivers/staging/most/dim2/sysfs.h                  |    30 +
 drivers/staging/most/hdm-dim2/Makefile             |     5 -
 drivers/staging/most/hdm-dim2/dim2_sysfs.c         |   115 -
 drivers/staging/most/hdm-dim2/dim2_sysfs.h         |    36 -
 drivers/staging/most/hdm-i2c/Makefile              |     3 -
 drivers/staging/most/hdm-usb/Makefile              |     4 -
 drivers/staging/most/{hdm-i2c => i2c}/Kconfig      |     6 +-
 drivers/staging/most/i2c/Makefile                  |     4 +
 .../staging/most/{hdm-i2c/hdm_i2c.c => i2c/i2c.c}  |    19 +-
 drivers/staging/most/mostcore/Kconfig              |    14 -
 drivers/staging/most/mostcore/Makefile             |     3 -
 drivers/staging/most/mostcore/core.c               |  1949 --
 drivers/staging/most/{aim-network => net}/Kconfig  |     6 +-
 drivers/staging/most/net/Makefile                  |     4 +
 .../most/{aim-network/networking.c => net/net.c}   |    71 +-
 drivers/staging/most/{aim-sound => sound}/Kconfig  |     6 +-
 drivers/staging/most/sound/Makefile                |     4 +
 drivers/staging/most/{aim-sound => sound}/sound.c  |    33 +-
 drivers/staging/most/{hdm-usb => usb}/Kconfig      |     7 +-
 drivers/staging/most/usb/Makefile                  |     4 +
 .../staging/most/{hdm-usb/hdm_usb.c => usb/usb.c}  |   276 +-
 drivers/staging/most/{aim-v4l2 => video}/Kconfig   |     6 +-
 drivers/staging/most/video/Makefile                |     4 +
 drivers/staging/most/{aim-v4l2 => video}/video.c   |   171 +-
 drivers/staging/mt29f_spinand/mt29f_spinand.c      |     5 +-
 {fs => drivers/staging}/ncpfs/Kconfig              |     0
 {fs => drivers/staging}/ncpfs/Makefile             |     0
 drivers/staging/ncpfs/TODO                         |     4 +
 {fs => drivers/staging}/ncpfs/dir.c                |     0
 {fs => drivers/staging}/ncpfs/file.c               |     0
 {fs => drivers/staging}/ncpfs/getopt.c             |     0
 {fs => drivers/staging}/ncpfs/getopt.h             |     0
 {fs => drivers/staging}/ncpfs/inode.c              |     3 +-
 {fs => drivers/staging}/ncpfs/ioctl.c              |     0
 {fs => drivers/staging}/ncpfs/mmap.c               |     0
 {fs => drivers/staging}/ncpfs/ncp_fs.h             |     0
 {fs => drivers/staging}/ncpfs/ncp_fs_i.h           |     0
 {fs => drivers/staging}/ncpfs/ncp_fs_sb.h          |     0
 {fs => drivers/staging}/ncpfs/ncplib_kernel.c      |     4 +
 {fs => drivers/staging}/ncpfs/ncplib_kernel.h      |     0
 {fs => drivers/staging}/ncpfs/ncpsign_kernel.c     |     0
 {fs => drivers/staging}/ncpfs/ncpsign_kernel.h     |     0
 {fs => drivers/staging}/ncpfs/sock.c               |     3 +-
 {fs => drivers/staging}/ncpfs/symlink.c            |     0
 drivers/staging/nvec/nvec-keytable.h               |    14 +-
 drivers/staging/nvec/nvec.c                        |     6 +-
 drivers/staging/nvec/nvec.h                        |     6 +-
 drivers/staging/nvec/nvec_kbd.c                    |     6 +-
 drivers/staging/nvec/nvec_paz00.c                  |     6 +-
 drivers/staging/nvec/nvec_power.c                  |     6 +-
 drivers/staging/nvec/nvec_ps2.c                    |     6 +-
 drivers/staging/octeon/Makefile                    |     4 +-
 drivers/staging/octeon/ethernet-defines.h          |     5 +-
 drivers/staging/octeon/ethernet-mdio.c             |     5 +-
 drivers/staging/octeon/ethernet-mdio.h             |     5 +-
 drivers/staging/octeon/ethernet-mem.c              |     5 +-
 drivers/staging/octeon/ethernet-mem.h              |     5 +-
 drivers/staging/octeon/ethernet-rgmii.c            |     5 +-
 drivers/staging/octeon/ethernet-rx.c               |     5 +-
 drivers/staging/octeon/ethernet-rx.h               |     5 +-
 drivers/staging/octeon/ethernet-sgmii.c            |     5 +-
 drivers/staging/octeon/ethernet-spi.c              |     5 +-
 drivers/staging/octeon/ethernet-tx.c               |     5 +-
 drivers/staging/octeon/ethernet-tx.h               |     5 +-
 drivers/staging/octeon/ethernet-util.h             |     5 +-
 drivers/staging/octeon/ethernet.c                  |     5 +-
 drivers/staging/octeon/octeon-ethernet.h           |     5 +-
 drivers/staging/olpc_dcon/olpc_dcon.c              |    30 +-
 drivers/staging/olpc_dcon/olpc_dcon.h              |    30 +-
 drivers/staging/olpc_dcon/olpc_dcon_xo_1.c         |     2 +-
 .../Documentation/devicetree/pi433-overlay.dts     |    32 +-
 drivers/staging/pi433/Documentation/pi433.txt      |    18 +-
 drivers/staging/pi433/pi433_if.c                   |   607 +-
 drivers/staging/pi433/pi433_if.h                   |    38 +-
 drivers/staging/pi433/rf69.c                       |   905 +-
 drivers/staging/pi433/rf69.h                       |    37 +-
 drivers/staging/pi433/rf69_enum.h                  |   298 +-
 drivers/staging/pi433/rf69_registers.h             |    48 +-
 drivers/staging/rtl8188eu/core/rtw_mlme_ext.c      |     4 +-
 drivers/staging/rtl8188eu/hal/pwrseqcmd.c          |     2 +-
 drivers/staging/rtl8192e/rtl8192e/Kconfig          |     1 -
 drivers/staging/rtl8192e/rtllib_rx.c               |     1 -
 drivers/staging/rtl8192e/rtllib_softmac.c          |     2 +-
 drivers/staging/rtl8192u/Kconfig                   |     1 -
 drivers/staging/rtl8192u/ieee80211/dot11d.c        |     2 +-
 drivers/staging/rtl8192u/ieee80211/ieee80211_rx.c  |    12 +-
 .../staging/rtl8192u/ieee80211/ieee80211_softmac.c |     2 +-
 drivers/staging/rtl8192u/r8192U_core.c             |     2 +-
 drivers/staging/rtl8712/hal_init.c                 |     1 -
 drivers/staging/rtl8712/ieee80211.c                |     6 +-
 drivers/staging/rtl8712/os_intfs.c                 |     5 +-
 drivers/staging/rtl8712/rtl8712_cmd.c              |    18 +-
 drivers/staging/rtl8712/rtl8712_recv.c             |    55 +-
 drivers/staging/rtl8712/rtl8712_xmit.c             |     3 +-
 drivers/staging/rtl8712/rtl871x_cmd.c              |    36 +-
 drivers/staging/rtl8712/rtl871x_ioctl_linux.c      |   106 +-
 drivers/staging/rtl8712/rtl871x_ioctl_set.c        |    72 +-
 drivers/staging/rtl8712/rtl871x_mlme.c             |     9 +-
 drivers/staging/rtl8712/rtl871x_mlme.h             |     3 +-
 drivers/staging/rtl8712/rtl871x_mp.c               |     2 +-
 drivers/staging/rtl8712/rtl871x_security.c         |    44 +-
 drivers/staging/rtl8712/usb_ops_linux.c            |     2 +-
 drivers/staging/rtl8723bs/core/rtw_cmd.c           |     4 +-
 drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c     |     2 +-
 drivers/staging/rtl8723bs/hal/HalPwrSeqCmd.c       |     2 +-
 drivers/staging/rtl8723bs/hal/hal_com_phycfg.c     |    16 +-
 drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c    |     2 +-
 drivers/staging/rtl8723bs/hal/sdio_ops.c           |   768 +-
 drivers/staging/rtl8723bs/os_dep/ioctl_linux.c     |     4 +-
 drivers/staging/rtlwifi/Kconfig                    |    12 +-
 drivers/staging/rtlwifi/btcoexist/halbtcoutsrc.c   |    20 +-
 drivers/staging/rtlwifi/cam.c                      |     4 +-
 drivers/staging/rtlwifi/core.c                     |    10 +-
 drivers/staging/rtlwifi/rtl8822be/fw.c             |     2 +
 drivers/staging/rtlwifi/wifi.h                     |     2 +-
 drivers/staging/rts5208/ms.c                       |     3 +-
 drivers/staging/rts5208/rtsx.c                     |    17 -
 drivers/staging/rts5208/rtsx.h                     |     2 -
 drivers/staging/rts5208/sd.h                       |     2 +-
 drivers/staging/sm750fb/TODO                       |     4 +-
 drivers/staging/sm750fb/ddk750_sii164.c            |     8 -
 drivers/staging/sm750fb/ddk750_sii164.h            |     2 -
 drivers/staging/sm750fb/sm750_hw.c                 |    14 +-
 drivers/staging/speakup/buffers.c                  |     1 +
 drivers/staging/speakup/fakekey.c                  |    11 +-
 drivers/staging/speakup/keyhelp.c                  |    11 +-
 drivers/staging/speakup/kobjects.c                 |     1 +
 drivers/staging/speakup/main.c                     |    11 +-
 drivers/staging/speakup/selection.c                |     9 +-
 drivers/staging/speakup/serialio.c                 |     1 +
 drivers/staging/speakup/speakup_acntpc.c           |    10 +-
 drivers/staging/speakup/speakup_acntsa.c           |    11 +-
 drivers/staging/speakup/speakup_apollo.c           |    11 +-
 drivers/staging/speakup/speakup_audptr.c           |    11 +-
 drivers/staging/speakup/speakup_bns.c              |    11 +-
 drivers/staging/speakup/speakup_decext.c           |    11 +-
 drivers/staging/speakup/speakup_decpc.c            |    11 +-
 drivers/staging/speakup/speakup_dectlk.c           |    11 +-
 drivers/staging/speakup/speakup_dtlk.c             |    11 +-
 drivers/staging/speakup/speakup_dummy.c            |    11 +-
 drivers/staging/speakup/speakup_keypc.c            |    11 +-
 drivers/staging/speakup/speakup_ltlk.c             |    11 +-
 drivers/staging/speakup/speakup_soft.c             |    18 +-
 drivers/staging/speakup/speakup_spkout.c           |    11 +-
 drivers/staging/speakup/speakup_txprt.c            |    11 +-
 drivers/staging/speakup/spk_priv.h                 |    11 +-
 drivers/staging/speakup/spk_priv_keyinfo.h         |    11 +-
 drivers/staging/speakup/spk_ttyio.c                |     1 +
 drivers/staging/speakup/synth.c                    |     1 +
 drivers/staging/speakup/thread.c                   |     1 +
 drivers/staging/speakup/varhandlers.c              |     1 +
 drivers/staging/typec/tcpci.c                      |    28 +-
 drivers/staging/typec/tcpci.h                      |    11 +-
 drivers/staging/unisys/Kconfig                     |     4 -
 drivers/staging/unisys/Makefile                    |     1 -
 drivers/staging/unisys/include/iochannel.h         |    15 +-
 drivers/staging/unisys/include/visorchannel.h      |   189 -
 drivers/staging/unisys/visorhba/visorhba_main.c    |    14 +-
 .../staging/unisys/visorinput/ultrainputreport.h   |    11 +-
 drivers/staging/unisys/visorinput/visorinput.c     |    13 +-
 drivers/staging/unisys/visornic/visornic_main.c    |    13 +-
 drivers/staging/vboxvideo/vbox_fb.c                |     2 +-
 drivers/staging/vboxvideo/vbox_main.c              |     2 +-
 drivers/staging/vboxvideo/vbox_ttm.c               |    23 +-
 .../vc04_services/bcm2835-audio/bcm2835-ctl.c      |    15 +-
 .../vc04_services/bcm2835-audio/bcm2835-pcm.c      |    15 +-
 .../vc04_services/bcm2835-audio/bcm2835-vchiq.c    |    15 +-
 .../staging/vc04_services/bcm2835-audio/bcm2835.c  |    16 +-
 .../staging/vc04_services/bcm2835-audio/bcm2835.h  |    15 +-
 .../bcm2835-audio/vc_vchi_audioserv_defs.h         |    15 +-
 .../vc04_services/bcm2835-camera/bcm2835-camera.c  |    61 +-
 .../vc04_services/bcm2835-camera/bcm2835-camera.h  |     7 +-
 .../vc04_services/bcm2835-camera/controls.c        |     5 +-
 .../vc04_services/bcm2835-camera/mmal-common.h     |     5 +-
 .../vc04_services/bcm2835-camera/mmal-encodings.h  |     5 +-
 .../vc04_services/bcm2835-camera/mmal-msg-common.h |     5 +-
 .../vc04_services/bcm2835-camera/mmal-msg-format.h |     5 +-
 .../vc04_services/bcm2835-camera/mmal-msg-port.h   |     5 +-
 .../vc04_services/bcm2835-camera/mmal-msg.h        |     5 +-
 .../vc04_services/bcm2835-camera/mmal-parameters.h |     5 +-
 .../vc04_services/bcm2835-camera/mmal-vchiq.c      |    12 +-
 .../vc04_services/bcm2835-camera/mmal-vchiq.h      |     5 +-
 .../interface/vchiq_arm/vchiq_2835_arm.c           |     6 +-
 .../vc04_services/interface/vchiq_arm/vchiq_arm.c  |    12 +-
 .../vc04_services/interface/vchiq_arm/vchiq_core.c |     9 +-
 .../interface/vchiq_arm/vchiq_kern_lib.c           |    50 +-
 .../vc04_services/interface/vchiq_arm/vchiq_shim.c |     3 +-
 drivers/staging/vme/devices/vme_user.c             |    26 +-
 drivers/staging/vt6655/baseband.c                  |    11 +-
 drivers/staging/vt6655/baseband.h                  |    11 +-
 drivers/staging/vt6655/card.c                      |    11 +-
 drivers/staging/vt6655/card.h                      |    11 +-
 drivers/staging/vt6655/channel.c                   |    11 +-
 drivers/staging/vt6655/channel.h                   |    12 +-
 drivers/staging/vt6655/desc.h                      |    11 +-
 drivers/staging/vt6655/device.h                    |    11 +-
 drivers/staging/vt6655/device_cfg.h                |    11 +-
 drivers/staging/vt6655/device_main.c               |    19 +-
 drivers/staging/vt6655/dpc.c                       |    11 +-
 drivers/staging/vt6655/dpc.h                       |    11 +-
 drivers/staging/vt6655/key.c                       |    11 +-
 drivers/staging/vt6655/key.h                       |    11 +-
 drivers/staging/vt6655/mac.c                       |    11 +-
 drivers/staging/vt6655/mac.h                       |    11 +-
 drivers/staging/vt6655/power.c                     |    11 +-
 drivers/staging/vt6655/power.h                     |    11 +-
 drivers/staging/vt6655/rf.c                        |    11 +-
 drivers/staging/vt6655/rf.h                        |    11 +-
 drivers/staging/vt6655/rxtx.c                      |    11 +-
 drivers/staging/vt6655/rxtx.h                      |    11 +-
 drivers/staging/vt6655/srom.c                      |    11 +-
 drivers/staging/vt6655/srom.h                      |    11 +-
 drivers/staging/vt6655/tmacro.h                    |    11 +-
 drivers/staging/vt6655/upc.h                       |    11 +-
 drivers/staging/vt6656/baseband.c                  |    12 +-
 drivers/staging/vt6656/baseband.h                  |    12 +-
 drivers/staging/vt6656/card.c                      |    12 +-
 drivers/staging/vt6656/card.h                      |    12 +-
 drivers/staging/vt6656/channel.c                   |    12 +-
 drivers/staging/vt6656/channel.h                   |    12 +-
 drivers/staging/vt6656/desc.h                      |    12 +-
 drivers/staging/vt6656/device.h                    |    12 +-
 drivers/staging/vt6656/dpc.c                       |    12 +-
 drivers/staging/vt6656/dpc.h                       |    12 +-
 drivers/staging/vt6656/firmware.c                  |    12 +-
 drivers/staging/vt6656/firmware.h                  |    12 +-
 drivers/staging/vt6656/int.c                       |    12 +-
 drivers/staging/vt6656/int.h                       |    12 +-
 drivers/staging/vt6656/key.c                       |    12 +-
 drivers/staging/vt6656/key.h                       |    12 +-
 drivers/staging/vt6656/mac.c                       |    12 +-
 drivers/staging/vt6656/mac.h                       |    12 +-
 drivers/staging/vt6656/main_usb.c                  |    17 +-
 drivers/staging/vt6656/power.c                     |    12 +-
 drivers/staging/vt6656/power.h                     |    12 +-
 drivers/staging/vt6656/rf.c                        |    12 +-
 drivers/staging/vt6656/rf.h                        |    12 +-
 drivers/staging/vt6656/rxtx.c                      |    12 +-
 drivers/staging/vt6656/rxtx.h                      |    12 +-
 drivers/staging/vt6656/usbpipe.c                   |    12 +-
 drivers/staging/vt6656/usbpipe.h                   |    12 +-
 drivers/staging/vt6656/wcmd.c                      |    12 +-
 drivers/staging/vt6656/wcmd.h                      |    12 +-
 drivers/staging/wilc1000/coreconfigurator.c        |     2 +-
 drivers/staging/wilc1000/host_interface.c          |    40 +-
 drivers/staging/wilc1000/linux_wlan.c              |    11 +-
 drivers/staging/wilc1000/wilc_debugfs.c            |     1 +
 drivers/staging/wilc1000/wilc_sdio.c               |     5 +-
 drivers/staging/wilc1000/wilc_spi.c                |    21 +-
 drivers/staging/wilc1000/wilc_wfi_cfgoperations.c  |   566 +-
 drivers/staging/wilc1000/wilc_wfi_netdevice.h      |    29 +-
 drivers/staging/wilc1000/wilc_wlan.c               |     9 +-
 drivers/staging/wilc1000/wilc_wlan_cfg.c           |     6 +-
 drivers/staging/wilc1000/wilc_wlan_if.h            |   100 +-
 drivers/staging/wlan-ng/hfa384x.h                  |     1 +
 drivers/staging/wlan-ng/hfa384x_usb.c              |     7 +-
 drivers/staging/wlan-ng/p80211conv.c               |     1 +
 drivers/staging/wlan-ng/p80211conv.h               |     1 +
 drivers/staging/wlan-ng/p80211hdr.h                |     1 +
 drivers/staging/wlan-ng/p80211ioctl.h              |     1 +
 drivers/staging/wlan-ng/p80211metadef.h            |     1 +
 drivers/staging/wlan-ng/p80211metastruct.h         |     1 +
 drivers/staging/wlan-ng/p80211mgmt.h               |     1 +
 drivers/staging/wlan-ng/p80211msg.h                |     1 +
 drivers/staging/wlan-ng/p80211netdev.c             |     4 +-
 drivers/staging/wlan-ng/p80211netdev.h             |     1 +
 drivers/staging/wlan-ng/p80211req.c                |     1 +
 drivers/staging/wlan-ng/p80211req.h                |     1 +
 drivers/staging/wlan-ng/p80211types.h              |     1 +
 drivers/staging/wlan-ng/p80211wep.c                |     1 +
 drivers/staging/wlan-ng/prism2fw.c                 |     1 +
 drivers/staging/wlan-ng/prism2mgmt.c               |     1 +
 drivers/staging/wlan-ng/prism2mgmt.h               |     1 +
 drivers/staging/wlan-ng/prism2mib.c                |     1 +
 drivers/staging/wlan-ng/prism2sta.c                |     1 +
 drivers/staging/xgifb/XGI_main.h                   |    13 -
 drivers/staging/xgifb/XGI_main_26.c                |   224 +-
 drivers/staging/xgifb/vb_setmode.c                 |    81 +-
 drivers/staging/xgifb/vb_table.h                   |     4 +-
 drivers/target/Kconfig                             |     1 +
 drivers/target/iscsi/cxgbit/cxgbit_cm.c            |     3 +
 drivers/target/iscsi/iscsi_target_auth.c           |     3 +-
 drivers/target/iscsi/iscsi_target_nego.c           |     3 +
 drivers/target/loopback/tcm_loop.c                 |   145 +-
 drivers/target/sbp/sbp_target.c                    |    13 +-
 drivers/target/target_core_configfs.c              |     6 +
 drivers/target/target_core_device.c                |     4 +-
 drivers/target/target_core_fabric_lib.c            |     6 +-
 drivers/target/target_core_internal.h              |     3 +-
 drivers/target/target_core_pr.c                    |     4 +-
 drivers/target/target_core_sbc.c                   |     8 +-
 drivers/target/target_core_transport.c             |    49 +-
 drivers/target/target_core_user.c                  |   983 +-
 drivers/tee/optee/Makefile                         |     1 +
 drivers/tee/optee/call.c                           |   220 +-
 drivers/tee/optee/core.c                           |   163 +-
 drivers/tee/optee/optee_msg.h                      |    38 +-
 drivers/tee/optee/optee_private.h                  |    72 +-
 drivers/tee/optee/optee_smc.h                      |     7 +
 drivers/tee/optee/rpc.c                            |    81 +-
 drivers/tee/optee/shm_pool.c                       |    75 +
 drivers/tee/optee/shm_pool.h                       |    23 +
 drivers/tee/optee/supp.c                           |   375 +-
 drivers/tee/tee_core.c                             |   113 +-
 drivers/tee/tee_private.h                          |    60 +-
 drivers/tee/tee_shm.c                              |   230 +-
 drivers/tee/tee_shm_pool.c                         |   165 +-
 drivers/thermal/Kconfig                            |     4 +-
 drivers/thermal/armada_thermal.c                   |   253 +-
 drivers/thermal/cpu_cooling.c                      |   201 +-
 drivers/thermal/hisi_thermal.c                     |     2 +-
 drivers/thermal/imx_thermal.c                      |    74 +-
 drivers/thermal/int340x_thermal/int3400_thermal.c  |    12 +-
 drivers/thermal/mtk_thermal.c                      |     9 +-
 drivers/thermal/of-thermal.c                       |     1 -
 drivers/thermal/power_allocator.c                  |     2 +
 drivers/thermal/tegra/soctherm.c                   |   103 +-
 drivers/thermal/thermal_hwmon.c                    |    20 +-
 drivers/thermal/thermal_sysfs.c                    |    17 +-
 drivers/thermal/x86_pkg_temp_thermal.c             |     4 +-
 drivers/tty/Kconfig                                |     6 +-
 drivers/tty/goldfish.c                             |     2 +
 drivers/tty/isicom.c                               |     6 +-
 drivers/tty/moxa.c                                 |    17 +-
 drivers/tty/moxa.h                                 |     2 +
 drivers/tty/n_gsm.c                                |    29 +-
 drivers/tty/n_hdlc.c                               |    14 +-
 drivers/tty/n_r3964.c                              |     8 +-
 drivers/tty/n_tty.c                                |    26 +-
 drivers/tty/pty.c                                  |     4 +-
 drivers/tty/serdev/core.c                          |   118 +-
 drivers/tty/serdev/serdev-ttyport.c                |    32 +-
 drivers/tty/serial/8250/8250_dw.c                  |    33 +-
 drivers/tty/serial/8250/8250_exar.c                |    34 +-
 drivers/tty/serial/8250/8250_ingenic.c             |    17 +-
 drivers/tty/serial/8250/8250_of.c                  |     5 +-
 drivers/tty/serial/8250/8250_omap.c                |     2 +-
 drivers/tty/serial/8250/8250_pci.c                 |    21 +-
 drivers/tty/serial/8250/8250_port.c                |    26 -
 drivers/tty/serial/8250/8250_uniphier.c            |     5 +-
 drivers/tty/serial/8250/Kconfig                    |     1 +
 drivers/tty/serial/Kconfig                         |    14 +-
 drivers/tty/serial/amba-pl011.c                    |    12 +-
 drivers/tty/serial/atmel_serial.c                  |     3 +-
 drivers/tty/serial/earlycon.c                      |    18 +-
 drivers/tty/serial/fsl_lpuart.c                    |    15 +-
 drivers/tty/serial/imx.c                           |    93 +-
 drivers/tty/serial/jsm/jsm_neo.c                   |     6 +-
 drivers/tty/serial/jsm/jsm_tty.c                   |     6 -
 drivers/tty/serial/max310x.c                       |   267 +-
 drivers/tty/serial/meson_uart.c                    |    31 +-
 drivers/tty/serial/mxs-auart.c                     |    11 +-
 drivers/tty/serial/omap-serial.c                   |    18 +-
 drivers/tty/serial/serial_core.c                   |    37 +-
 drivers/tty/serial/sh-sci.c                        |     4 +-
 drivers/tty/serial/stm32-usart.h                   |     2 +-
 drivers/tty/tty_io.c                               |    41 +-
 drivers/tty/tty_ldisc.c                            |     8 +-
 drivers/tty/vt/selection.c                         |     6 +-
 drivers/tty/vt/vc_screen.c                         |     4 +-
 drivers/tty/vt/vt.c                                |     8 +-
 drivers/uio/uio.c                                  |     4 +-
 drivers/uio/uio_hv_generic.c                       |   138 +-
 drivers/usb/Kconfig                                |     6 +
 drivers/usb/atm/cxacru.c                           |    18 +-
 drivers/usb/atm/ueagle-atm.c                       |    18 +-
 drivers/usb/chipidea/ci_hdrc_tegra.c               |     2 +-
 drivers/usb/chipidea/core.c                        |     6 +-
 drivers/usb/chipidea/otg_fsm.c                     |    23 +-
 drivers/usb/class/cdc-acm.c                        |    26 +-
 drivers/usb/class/cdc-wdm.c                        |    12 +-
 drivers/usb/class/usblp.c                          |    12 +-
 drivers/usb/class/usbtmc.c                         |     8 +-
 drivers/usb/core/devices.c                         |     4 +-
 drivers/usb/core/devio.c                           |    16 +-
 drivers/usb/core/driver.c                          |    10 +-
 drivers/usb/core/hub.c                             |    47 +-
 drivers/usb/core/ledtrig-usbport.c                 |    10 +-
 drivers/usb/core/message.c                         |    54 +-
 drivers/usb/core/of.c                              |    95 +-
 drivers/usb/core/quirks.c                          |     6 +-
 drivers/usb/core/urb.c                             |     3 -
 drivers/usb/core/usb.c                             |     3 +-
 drivers/usb/core/usb.h                             |     1 +
 drivers/usb/dwc2/core.h                            |     2 +
 drivers/usb/dwc2/gadget.c                          |    26 +-
 drivers/usb/dwc2/hcd.c                             |     7 +
 drivers/usb/dwc2/hcd.h                             |     9 +
 drivers/usb/dwc2/hcd_intr.c                        |    20 +
 drivers/usb/dwc2/hcd_queue.c                       |    81 +-
 drivers/usb/dwc2/params.c                          |     6 +-
 drivers/usb/dwc2/platform.c                        |    10 +
 drivers/usb/dwc3/core.c                            |    93 +-
 drivers/usb/dwc3/core.h                            |    26 +-
 drivers/usb/dwc3/debug.h                           |     9 +
 drivers/usb/dwc3/dwc3-of-simple.c                  |     1 +
 drivers/usb/dwc3/dwc3-omap.c                       |    16 +
 drivers/usb/dwc3/ep0.c                             |    13 +-
 drivers/usb/dwc3/gadget.c                          |     8 +-
 drivers/usb/dwc3/trace.h                           |     4 +-
 drivers/usb/early/xhci-dbc.c                       |     2 +-
 drivers/usb/gadget/function/f_fs.c                 |    67 +-
 drivers/usb/gadget/function/f_hid.c                |     8 +-
 drivers/usb/gadget/function/f_ncm.c                |    30 +-
 drivers/usb/gadget/function/f_printer.c            |     8 +-
 drivers/usb/gadget/function/f_uac2.c               |     2 +
 drivers/usb/gadget/function/u_serial.c             |   192 +-
 drivers/usb/gadget/function/uvc_queue.c            |     2 +-
 drivers/usb/gadget/function/uvc_queue.h            |     2 +-
 drivers/usb/gadget/function/uvc_v4l2.c             |     2 +-
 drivers/usb/gadget/legacy/inode.c                  |    11 +-
 drivers/usb/gadget/legacy/ncm.c                    |     6 +-
 drivers/usb/gadget/udc/Kconfig                     |     1 -
 drivers/usb/gadget/udc/bcm63xx_udc.c               |    16 +-
 drivers/usb/gadget/udc/bdc/bdc_pci.c               |     1 +
 drivers/usb/gadget/udc/core.c                      |    12 +-
 drivers/usb/gadget/udc/dummy_hcd.c                 |    16 +-
 drivers/usb/gadget/udc/fsl_udc_core.c              |     6 +-
 drivers/usb/gadget/udc/pxa27x_udc.c                |     3 -
 drivers/usb/gadget/udc/renesas_usb3.c              |     2 +-
 drivers/usb/gadget/udc/trace.h                     |     6 +-
 drivers/usb/gadget/udc/udc-xilinx.c                |     2 -
 drivers/usb/host/Kconfig                           |    16 +-
 drivers/usb/host/Makefile                          |     5 +
 drivers/usb/host/ehci-hub.c                        |     4 +-
 drivers/usb/host/ehci-omap.c                       |     3 +-
 drivers/usb/host/ehci-q.c                          |    12 +-
 drivers/usb/host/ehci-st.c                         |     1 +
 drivers/usb/host/ehci-sysfs.c                      |    12 +-
 drivers/usb/host/fotg210-hcd.c                     |    14 +-
 drivers/usb/host/ohci-da8xx.c                      |     2 +-
 drivers/usb/host/ohci-hcd.c                        |    13 +-
 drivers/usb/host/ohci-hub.c                        |     4 +-
 drivers/usb/host/ohci-q.c                          |    17 +-
 drivers/usb/host/pci-quirks.c                      |   109 +
 drivers/usb/host/pci-quirks.h                      |     5 +
 drivers/usb/host/uhci-hcd.c                        |     3 +-
 drivers/usb/host/uhci-hcd.h                        |     3 +
 drivers/usb/host/uhci-platform.c                   |    23 +-
 drivers/usb/host/uhci-q.c                          |     3 +-
 drivers/usb/host/whci/asl.c                        |     2 -
 drivers/usb/host/xhci-dbg.c                        |   261 -
 drivers/usb/host/xhci-dbgcap.c                     |  1000 +
 drivers/usb/host/xhci-dbgcap.h                     |   229 +
 drivers/usb/host/xhci-dbgtty.c                     |   501 +
 drivers/usb/host/xhci-debugfs.c                    |     4 +-
 drivers/usb/host/xhci-hub.c                        |    33 +-
 drivers/usb/host/xhci-mem.c                        |   132 +-
 drivers/usb/host/xhci-mtk.c                        |   186 +-
 drivers/usb/host/xhci-mtk.h                        |     6 +-
 drivers/usb/host/xhci-pci.c                        |    17 +
 drivers/usb/host/xhci-plat.c                       |    16 +-
 drivers/usb/host/xhci-rcar.c                       |     4 +
 drivers/usb/host/xhci-ring.c                       |    14 +-
 drivers/usb/host/xhci-trace.h                      |    69 +
 drivers/usb/host/xhci.c                            |    59 +-
 drivers/usb/host/xhci.h                            |    55 +-
 drivers/usb/misc/chaoskey.c                        |     6 +-
 drivers/usb/misc/cypress_cy7c63.c                  |    12 +-
 drivers/usb/misc/cytherm.c                         |    45 +-
 drivers/usb/misc/iowarrior.c                       |    12 +-
 drivers/usb/misc/ldusb.c                           |    16 +-
 drivers/usb/misc/legousbtower.c                    |    12 +-
 drivers/usb/misc/trancevibrator.c                  |     6 +-
 drivers/usb/misc/usbsevseg.c                       |    24 +-
 drivers/usb/misc/usbtest.c                         |    70 +
 drivers/usb/mon/mon_bin.c                          |     6 +-
 drivers/usb/mon/mon_text.c                         |   126 +-
 drivers/usb/mtu3/mtu3.h                            |    11 +-
 drivers/usb/mtu3/mtu3_dr.h                         |    10 +-
 drivers/usb/mtu3/mtu3_host.c                       |   115 +-
 drivers/usb/mtu3/mtu3_plat.c                       |    10 +-
 drivers/usb/musb/da8xx.c                           |     2 +-
 drivers/usb/musb/musb_core.c                       |    24 +-
 drivers/usb/musb/musb_host.c                       |    13 +-
 drivers/usb/phy/phy-mv-usb.c                       |    18 +-
 drivers/usb/phy/phy-mxs-usb.c                      |     3 +
 drivers/usb/phy/phy-tahvo.c                        |     6 +-
 drivers/usb/phy/phy-twl6030-usb.c                  |     4 +-
 drivers/usb/phy/phy.c                              |    14 +-
 drivers/usb/renesas_usbhs/Makefile                 |     2 +-
 drivers/usb/renesas_usbhs/common.c                 |    22 +
 drivers/usb/renesas_usbhs/common.h                 |     7 +
 drivers/usb/renesas_usbhs/fifo.c                   |    26 +-
 drivers/usb/renesas_usbhs/pipe.c                   |    19 +-
 drivers/usb/renesas_usbhs/pipe.h                   |     2 +
 drivers/usb/renesas_usbhs/rcar3.c                  |    35 +-
 drivers/usb/renesas_usbhs/rza.c                    |    52 +
 drivers/usb/renesas_usbhs/rza.h                    |     4 +
 drivers/usb/serial/Kconfig                         |    79 +-
 drivers/usb/serial/ark3116.c                       |    49 +-
 drivers/usb/serial/f81534.c                        |   392 +-
 drivers/usb/serial/ftdi_sio.c                      |     4 +-
 drivers/usb/serial/io_edgeport.c                   |     1 -
 drivers/usb/serial/iuu_phoenix.c                   |     1 -
 drivers/usb/serial/mos7720.c                       |     4 +-
 drivers/usb/serial/option.c                        |    12 +
 drivers/usb/serial/pl2303.c                        |     1 +
 drivers/usb/serial/pl2303.h                        |     1 +
 drivers/usb/serial/usb-serial-simple.c             |     7 +
 drivers/usb/storage/debug.c                        |     3 -
 drivers/usb/storage/debug.h                        |     3 -
 drivers/usb/storage/initializers.c                 |     3 -
 drivers/usb/storage/initializers.h                 |     3 -
 drivers/usb/storage/protocol.c                     |     3 -
 drivers/usb/storage/protocol.h                     |     3 -
 drivers/usb/storage/scsiglue.c                     |     3 -
 drivers/usb/storage/scsiglue.h                     |     3 -
 drivers/usb/storage/sierra_ms.c                    |     4 +-
 drivers/usb/storage/transport.c                    |     3 -
 drivers/usb/storage/transport.h                    |     3 -
 drivers/usb/storage/uas.c                          |     7 +-
 drivers/usb/storage/unusual_devs.h                 |    15 +-
 drivers/usb/storage/unusual_uas.h                  |    63 -
 drivers/usb/storage/usb.c                          |    11 +-
 drivers/usb/storage/usb.h                          |     3 -
 drivers/usb/storage/usual-tables.c                 |     3 -
 drivers/usb/typec/fusb302/fusb302.c                |    40 +-
 drivers/usb/typec/tcpm.c                           |   130 +-
 drivers/usb/typec/typec_wcove.c                    |     4 +-
 drivers/usb/usbip/stub_dev.c                       |     7 +-
 drivers/usb/usbip/stub_rx.c                        |     3 -
 drivers/usb/usbip/usbip_common.c                   |     1 -
 drivers/usb/usbip/vhci_hcd.c                       |     2 +
 drivers/usb/usbip/vhci_rx.c                        |     2 +-
 drivers/usb/usbip/vhci_sysfs.c                     |    20 +-
 drivers/usb/usbip/vudc_sysfs.c                     |    12 +-
 drivers/uwb/Kconfig                                |     1 +
 drivers/uwb/address.c                              |    14 +-
 drivers/uwb/drp-ie.c                               |    13 +-
 drivers/vfio/mdev/vfio_mdev.c                      |     6 +-
 drivers/vfio/pci/vfio_pci.c                        |    72 +-
 drivers/vfio/vfio.c                                |    52 +-
 drivers/vfio/vfio_iommu_type1.c                    |    18 +-
 drivers/vfio/virqfd.c                              |    12 +-
 drivers/vhost/net.c                                |    84 +-
 drivers/vhost/scsi.c                               |    11 +-
 drivers/vhost/test.c                               |     2 +-
 drivers/vhost/vhost.c                              |   114 +-
 drivers/vhost/vhost.h                              |    15 +-
 drivers/vhost/vsock.c                              |     2 +-
 drivers/video/backlight/apple_bl.c                 |     2 +-
 drivers/video/backlight/corgi_lcd.c                |     2 +-
 drivers/video/backlight/tdo24m.c                   |     2 +-
 drivers/video/backlight/tosa_lcd.c                 |     2 +-
 drivers/video/console/dummycon.c                   |     1 -
 drivers/video/console/vgacon.c                     |    34 +-
 drivers/video/fbdev/Kconfig                        |     4 +-
 drivers/video/fbdev/atmel_lcdfb.c                  |     8 +-
 drivers/video/fbdev/aty/radeon_base.c              |    18 +-
 drivers/video/fbdev/au1200fb.h                     |   286 -
 drivers/video/fbdev/auo_k190x.c                    |     7 +-
 drivers/video/fbdev/core/Makefile                  |     3 -
 drivers/video/fbdev/core/fbcon.c                   |    22 +-
 drivers/video/fbdev/core/fbcon.h                   |     6 -
 drivers/video/fbdev/core/fbsysfs.c                 |     1 +
 drivers/video/fbdev/efifb.c                        |    21 +-
 drivers/video/fbdev/geode/video_gx.c               |     2 +-
 drivers/video/fbdev/intelfb/intelfbhw.c            |     4 +-
 drivers/video/fbdev/macfb.c                        |    10 +-
 drivers/video/fbdev/mmp/core.c                     |     5 +
 drivers/video/fbdev/mxsfb.c                        |    45 +-
 drivers/video/fbdev/nvidia/nv_hw.c                 |    11 +-
 drivers/video/fbdev/nvidia/nv_setup.c              |     3 +-
 .../omap2/omapfb/displays/panel-tpo-td028ttec1.c   |    12 +-
 drivers/video/fbdev/omap2/omapfb/dss/dispc.c       |    39 +-
 drivers/video/fbdev/omap2/omapfb/dss/dss.c         |    46 +-
 drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c       |     5 +-
 drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c    |    33 +-
 drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c    |     1 +
 drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c     |     2 +
 drivers/video/fbdev/pxa3xx-gcu.c                   |    16 +-
 drivers/video/fbdev/riva/fbdev.c                   |     2 +-
 drivers/video/fbdev/riva/nv_driver.c               |     7 +-
 drivers/video/fbdev/riva/riva_hw.c                 |    20 +-
 drivers/video/fbdev/riva/riva_hw.h                 |     3 +-
 drivers/video/fbdev/sbuslib.c                      |     4 +-
 drivers/video/fbdev/smscufx.c                      |     5 +-
 drivers/video/fbdev/udlfb.c                        |   655 +-
 drivers/video/fbdev/vfb.c                          |    17 +
 drivers/video/fbdev/vga16fb.c                      |     1 -
 drivers/video/fbdev/vt8500lcdfb.c                  |     4 +-
 drivers/video/fbdev/w100fb.c                       |     4 +-
 drivers/video/fbdev/wm8505fb.c                     |     4 +-
 drivers/video/hdmi.c                               |    51 +-
 drivers/virt/Kconfig                               |     1 +
 drivers/virt/Makefile                              |     1 +
 drivers/virt/fsl_hypervisor.c                      |     6 +-
 drivers/virt/vboxguest/Kconfig                     |    18 +
 drivers/virt/vboxguest/Makefile                    |     3 +
 drivers/virt/vboxguest/vboxguest_core.c            |  1571 +
 drivers/virt/vboxguest/vboxguest_core.h            |   174 +
 drivers/virt/vboxguest/vboxguest_linux.c           |   466 +
 drivers/virt/vboxguest/vboxguest_utils.c           |   803 +
 drivers/virt/vboxguest/vboxguest_version.h         |    19 +
 drivers/virt/vboxguest/vmmdev.h                    |   449 +
 drivers/virtio/Kconfig                             |     8 +-
 drivers/virtio/virtio.c                            |    18 +-
 drivers/virtio/virtio_balloon.c                    |     4 +
 drivers/virtio/virtio_mmio.c                       |     6 +-
 drivers/virtio/virtio_pci_common.c                 |     8 +-
 drivers/virtio/virtio_ring.c                       |     2 -
 drivers/{staging/unisys => }/visorbus/Kconfig      |     2 +-
 drivers/{staging/unisys => }/visorbus/Makefile     |     2 -
 .../unisys => }/visorbus/controlvmchannel.h        |    14 +-
 .../{staging/unisys => }/visorbus/vbuschannel.h    |    13 +-
 .../{staging/unisys => }/visorbus/visorbus_main.c  |    13 +-
 .../unisys => }/visorbus/visorbus_private.h        |    13 +-
 .../{staging/unisys => }/visorbus/visorchannel.c   |    13 +-
 .../{staging/unisys => }/visorbus/visorchipset.c   |    20 +-
 drivers/vme/vme.c                                  |     2 +-
 drivers/w1/masters/w1-gpio.c                       |   149 +-
 drivers/w1/w1_netlink.h                            |     6 +-
 drivers/watchdog/Kconfig                           |    52 +-
 drivers/watchdog/Makefile                          |     8 +-
 drivers/watchdog/advantechwdt.c                    |     2 +-
 drivers/watchdog/alim1535_wdt.c                    |     2 +-
 drivers/watchdog/aspeed_wdt.c                      |    21 +-
 drivers/watchdog/at32ap700x_wdt.c                  |   433 -
 drivers/watchdog/da9062_wdt.c                      |    48 +-
 drivers/watchdog/davinci_wdt.c                     |    38 +
 drivers/watchdog/dw_wdt.c                          |    18 +-
 drivers/watchdog/eurotechwdt.c                     |     2 +-
 drivers/watchdog/f71808e_wdt.c                     |     7 +-
 drivers/watchdog/ftwdt010_wdt.c                    |   236 +
 drivers/watchdog/gemini_wdt.c                      |   229 -
 drivers/watchdog/gpio_wdt.c                        |    58 +-
 drivers/watchdog/hpwdt.c                           |   507 +-
 drivers/watchdog/i6300esb.c                        |   377 +-
 drivers/watchdog/ib700wdt.c                        |     2 +-
 drivers/watchdog/imx2_wdt.c                        |    20 +-
 drivers/watchdog/jz4740_wdt.c                      |     1 +
 drivers/watchdog/mei_wdt.c                         |     7 +-
 drivers/watchdog/mpc8xxx_wdt.c                     |    83 +-
 drivers/watchdog/mt7621_wdt.c                      |    34 +-
 drivers/watchdog/orion_wdt.c                       |     2 +-
 drivers/watchdog/pcwd_pci.c                        |     2 +-
 drivers/watchdog/pcwd_usb.c                        |     5 +-
 drivers/watchdog/rave-sp-wdt.c                     |   337 +
 drivers/watchdog/rtd119x_wdt.c                     |   168 +
 drivers/watchdog/sbsa_gwdt.c                       |     3 +-
 drivers/watchdog/sp5100_tco.c                      |   710 +-
 drivers/watchdog/sp5100_tco.h                      |    57 +-
 drivers/watchdog/sprd_wdt.c                        |   399 +
 drivers/watchdog/stm32_iwdg.c                      |     6 +-
 drivers/watchdog/sunxi_wdt.c                       |     7 +-
 drivers/watchdog/watchdog_core.c                   |     1 +
 drivers/watchdog/watchdog_dev.c                    |   125 +-
 drivers/watchdog/wdat_wdt.c                        |     2 +-
 drivers/watchdog/wdt_pci.c                         |     2 +-
 drivers/watchdog/xen_wdt.c                         |   255 +-
 drivers/xen/events/events_base.c                   |     4 +-
 drivers/xen/evtchn.c                               |     8 +-
 drivers/xen/grant-table.c                          |     4 +-
 drivers/xen/mcelog.c                               |     4 +-
 drivers/xen/pvcalls-back.c                         |     4 +-
 drivers/xen/pvcalls-front.c                        |   230 +-
 drivers/xen/pvcalls-front.h                        |     2 +-
 drivers/xen/swiotlb-xen.c                          |     2 +-
 drivers/xen/tmem.c                                 |     4 +
 drivers/xen/xenbus/xenbus.h                        |     1 +
 drivers/xen/xenbus/xenbus_comms.c                  |     1 +
 drivers/xen/xenbus/xenbus_dev_frontend.c           |     4 +-
 drivers/xen/xenbus/xenbus_probe.c                  |     5 +-
 drivers/xen/xenbus/xenbus_xs.c                     |     3 +
 fs/9p/Kconfig                                      |     3 -
 fs/Kconfig                                         |    14 +-
 fs/Makefile                                        |     1 -
 fs/affs/amigaffs.c                                 |     5 +-
 fs/affs/dir.c                                      |     5 +-
 fs/affs/super.c                                    |     3 +-
 fs/afs/addr_list.c                                 |    13 +-
 fs/afs/dir.c                                       |   122 +-
 fs/afs/fsclient.c                                  |     3 +-
 fs/afs/inode.c                                     |    53 +-
 fs/afs/internal.h                                  |    12 +-
 fs/afs/mntpt.c                                     |    20 +-
 fs/afs/rotate.c                                    |   293 +-
 fs/afs/server_list.c                               |     3 +-
 fs/afs/super.c                                     |   132 +-
 fs/afs/vlclient.c                                  |    10 +-
 fs/afs/volume.c                                    |    47 +-
 fs/aio.c                                           |    44 +-
 fs/autofs4/waitq.c                                 |     4 +-
 fs/befs/linuxvfs.c                                 |    14 +-
 fs/binfmt_elf.c                                    |     2 +
 fs/block_dev.c                                     |    49 +-
 fs/btrfs/Kconfig                                   |     3 -
 fs/btrfs/Makefile                                  |     2 +-
 fs/btrfs/backref.c                                 |    15 +-
 fs/btrfs/compression.c                             |   141 +-
 fs/btrfs/compression.h                             |     4 +-
 fs/btrfs/ctree.c                                   |    56 +-
 fs/btrfs/ctree.h                                   |    19 +-
 fs/btrfs/delayed-inode.c                           |    66 +-
 fs/btrfs/delayed-ref.c                             |     5 +-
 fs/btrfs/delayed-ref.h                             |     2 +-
 fs/btrfs/dev-replace.c                             |    30 +-
 fs/btrfs/dir-item.c                                |   108 -
 fs/btrfs/disk-io.c                                 |    77 +-
 fs/btrfs/disk-io.h                                 |     3 +
 fs/btrfs/export.c                                  |     5 -
 fs/btrfs/extent-tree.c                             |    25 +-
 fs/btrfs/extent_io.c                               |   154 +-
 fs/btrfs/extent_io.h                               |    50 +-
 fs/btrfs/extent_map.c                              |   132 +
 fs/btrfs/extent_map.h                              |     3 +-
 fs/btrfs/file.c                                    |   392 +-
 fs/btrfs/free-space-cache.c                        |    15 +-
 fs/btrfs/inode-item.c                              |    44 +-
 fs/btrfs/inode.c                                   |   350 +-
 fs/btrfs/ioctl.c                                   |    45 +-
 fs/btrfs/props.c                                   |    13 +-
 fs/btrfs/qgroup.c                                  |    12 +-
 fs/btrfs/raid56.c                                  |   120 +-
 fs/btrfs/ref-verify.c                              |     6 +-
 fs/btrfs/relocation.c                              |    18 +-
 fs/btrfs/root-tree.c                               |     7 -
 fs/btrfs/scrub.c                                   |    95 +-
 fs/btrfs/send.c                                    |     9 +-
 fs/btrfs/super.c                                   |   350 +-
 fs/btrfs/sysfs.c                                   |     2 +-
 fs/btrfs/tests/btrfs-tests.c                       |     3 +
 fs/btrfs/tests/btrfs-tests.h                       |     1 +
 fs/btrfs/tests/extent-map-tests.c                  |   366 +
 fs/btrfs/tests/inode-tests.c                       |    17 +-
 fs/btrfs/transaction.c                             |    22 +-
 fs/btrfs/transaction.h                             |    11 +-
 fs/btrfs/tree-checker.c                            |   142 +
 fs/btrfs/tree-log.c                                |   204 +-
 fs/btrfs/volumes.c                                 |   701 +-
 fs/btrfs/volumes.h                                 |    45 +-
 fs/btrfs/xattr.c                                   |     7 +-
 fs/btrfs/zstd.c                                    |   132 +-
 fs/buffer.c                                        |    12 +-
 fs/cachefiles/daemon.c                             |    12 +-
 fs/ceph/Kconfig                                    |     3 -
 fs/ceph/addr.c                                     |    28 +-
 fs/ceph/caps.c                                     |   196 +-
 fs/ceph/dir.c                                      |   107 +-
 fs/ceph/file.c                                     |    21 +-
 fs/ceph/inode.c                                    |    56 +-
 fs/ceph/mds_client.c                               |    33 +-
 fs/ceph/mds_client.h                               |     3 +
 fs/ceph/snap.c                                     |     8 +-
 fs/ceph/super.c                                    |    27 +-
 fs/ceph/super.h                                    |    55 +-
 fs/cifs/Kconfig                                    |    23 +-
 fs/cifs/Makefile                                   |     2 +
 fs/cifs/cifs_debug.c                               |   208 +-
 fs/cifs/cifsacl.c                                  |     2 +-
 fs/cifs/cifsencrypt.c                              |     3 +-
 fs/cifs/cifsfs.c                                   |    18 +-
 fs/cifs/cifsfs.h                                   |     2 +-
 fs/cifs/cifsglob.h                                 |    35 +-
 fs/cifs/cifsproto.h                                |     4 +
 fs/cifs/cifssmb.c                                  |    26 +-
 fs/cifs/connect.c                                  |   251 +-
 fs/cifs/file.c                                     |    43 +-
 fs/cifs/inode.c                                    |     2 +-
 fs/cifs/misc.c                                     |    14 +-
 fs/cifs/smb1ops.c                                  |     4 +-
 fs/cifs/smb2file.c                                 |     2 +-
 fs/cifs/smb2misc.c                                 |     2 +-
 fs/cifs/smb2ops.c                                  |    77 +-
 fs/cifs/smb2pdu.c                                  |   578 +-
 fs/cifs/smb2pdu.h                                  |   174 +-
 fs/cifs/smb2proto.h                                |     3 +-
 fs/cifs/smbdirect.c                                |  2612 ++
 fs/cifs/smbdirect.h                                |   338 +
 fs/cifs/transport.c                                |    69 +-
 fs/coda/psdev.c                                    |     8 +-
 fs/compat_ioctl.c                                  |    22 +-
 fs/cramfs/Kconfig                                  |     3 +-
 fs/crypto/crypto.c                                 |     1 +
 fs/crypto/fname.c                                  |   140 +-
 fs/crypto/fscrypt_private.h                        |    31 +
 fs/crypto/hooks.c                                  |   158 +
 fs/crypto/keyinfo.c                                |    17 +-
 fs/dax.c                                           |    30 +-
 fs/dcache.c                                        |   184 +-
 fs/debugfs/file.c                                  |     8 +-
 fs/devpts/inode.c                                  |     4 +-
 fs/direct-io.c                                     |    27 +-
 fs/dlm/lowcomms.c                                  |     4 +-
 fs/dlm/plock.c                                     |     6 +-
 fs/dlm/user.c                                      |     4 +-
 fs/ecryptfs/miscdev.c                              |     6 +-
 fs/efivarfs/file.c                                 |     6 +
 fs/eventfd.c                                       |   149 +-
 fs/eventpoll.c                                     |    89 +-
 fs/exofs/dir.c                                     |     9 +-
 fs/exofs/super.c                                   |    10 +-
 fs/ext2/Kconfig                                    |     6 +-
 fs/ext2/dir.c                                      |     9 +-
 fs/ext2/file.c                                     |     2 +-
 fs/ext2/super.c                                    |    26 +-
 fs/ext4/Kconfig                                    |     3 -
 fs/ext4/acl.h                                      |     2 +-
 fs/ext4/balloc.c                                   |     4 +-
 fs/ext4/block_validity.c                           |     6 +-
 fs/ext4/dir.c                                      |     9 +-
 fs/ext4/ext4.h                                     |    22 +-
 fs/ext4/ext4_extents.h                             |    14 +-
 fs/ext4/ext4_jbd2.h                                |     5 +-
 fs/ext4/extents.c                                  |    14 +-
 fs/ext4/extents_status.h                           |     2 +-
 fs/ext4/file.c                                     |    10 +-
 fs/ext4/fsmap.c                                    |    15 +-
 fs/ext4/fsmap.h                                    |    15 +-
 fs/ext4/hash.c                                     |     6 +-
 fs/ext4/ialloc.c                                   |     4 +-
 fs/ext4/inline.c                                   |    17 +-
 fs/ext4/inode.c                                    |    29 +-
 fs/ext4/ioctl.c                                    |     3 +-
 fs/ext4/mballoc.c                                  |    28 +-
 fs/ext4/mballoc.h                                  |     2 +-
 fs/ext4/migrate.c                                  |     9 +-
 fs/ext4/move_extent.c                              |    10 +-
 fs/ext4/namei.c                                    |    69 +-
 fs/ext4/resize.c                                   |     2 +-
 fs/ext4/super.c                                    |    37 +-
 fs/ext4/symlink.c                                  |    43 +-
 fs/ext4/sysfs.c                                    |    65 +-
 fs/ext4/truncate.h                                 |     2 +-
 fs/ext4/xattr.c                                    |     5 +-
 fs/ext4/xattr.h                                    |     2 +-
 fs/f2fs/Kconfig                                    |     6 +-
 fs/f2fs/acl.c                                      |     2 +-
 fs/f2fs/checkpoint.c                               |    10 +-
 fs/f2fs/data.c                                     |   301 +-
 fs/f2fs/debug.c                                    |    12 +-
 fs/f2fs/dir.c                                      |     6 +
 fs/f2fs/f2fs.h                                     |   201 +-
 fs/f2fs/file.c                                     |   248 +-
 fs/f2fs/gc.c                                       |    18 +-
 fs/f2fs/gc.h                                       |     2 +
 fs/f2fs/inode.c                                    |    36 +-
 fs/f2fs/namei.c                                    |   197 +-
 fs/f2fs/node.c                                     |   149 +-
 fs/f2fs/node.h                                     |     4 +
 fs/f2fs/recovery.c                                 |    27 +-
 fs/f2fs/segment.c                                  |   129 +-
 fs/f2fs/segment.h                                  |    92 +-
 fs/f2fs/super.c                                    |   142 +-
 fs/f2fs/sysfs.c                                    |    14 +-
 fs/f2fs/trace.c                                    |    12 +-
 fs/f2fs/xattr.c                                    |    12 +-
 fs/fat/dir.c                                       |     3 +-
 fs/fat/inode.c                                     |     9 +-
 fs/fat/namei_msdos.c                               |     7 +-
 fs/fat/namei_vfat.c                                |    22 +-
 fs/fcntl.c                                         |    21 +-
 fs/fhandle.c                                       |     3 +-
 fs/file.c                                          |     7 +-
 fs/file_table.c                                    |     1 -
 fs/freevxfs/vxfs_super.c                           |     8 +-
 fs/fs-writeback.c                                  |     2 +-
 fs/fuse/dev.c                                      |    10 +-
 fs/fuse/file.c                                     |     8 +-
 fs/fuse/fuse_i.h                                   |     2 +-
 fs/gfs2/Kconfig                                    |     1 +
 fs/gfs2/aops.c                                     |    35 +-
 fs/gfs2/bmap.c                                     |   623 +-
 fs/gfs2/bmap.h                                     |     1 +
 fs/gfs2/dir.c                                      |     3 +-
 fs/gfs2/file.c                                     |    23 +-
 fs/gfs2/glock.c                                    |    70 +-
 fs/gfs2/glops.c                                    |    19 +-
 fs/gfs2/incore.h                                   |     6 +-
 fs/gfs2/inode.c                                    |    11 +-
 fs/gfs2/lock_dlm.c                                 |     4 +-
 fs/gfs2/log.c                                      |   124 +-
 fs/gfs2/log.h                                      |    10 +-
 fs/gfs2/lops.c                                     |    18 +-
 fs/gfs2/lops.h                                     |     3 +
 fs/gfs2/main.c                                     |    90 +-
 fs/gfs2/ops_fstype.c                               |     2 +-
 fs/gfs2/quota.c                                    |     3 +-
 fs/gfs2/recovery.c                                 |   110 +-
 fs/gfs2/rgrp.c                                     |    38 +-
 fs/gfs2/super.c                                    |    19 +-
 fs/gfs2/sys.c                                      |     4 +-
 fs/gfs2/trace_gfs2.h                               |    11 +-
 fs/gfs2/trans.c                                    |     4 +-
 fs/hfsplus/Kconfig                                 |     3 -
 fs/hfsplus/dir.c                                   |     4 +-
 fs/hfsplus/hfsplus_fs.h                            |     3 +-
 fs/hfsplus/inode.c                                 |     7 +-
 fs/hfsplus/super.c                                 |     2 +-
 fs/hugetlbfs/inode.c                               |    56 +-
 fs/inode.c                                         |    12 +-
 fs/iomap.c                                         |    14 +-
 fs/jbd2/checkpoint.c                               |     5 +-
 fs/jbd2/commit.c                                   |     5 +-
 fs/jbd2/journal.c                                  |     5 +-
 fs/jbd2/recovery.c                                 |     5 +-
 fs/jbd2/revoke.c                                   |     5 +-
 fs/jbd2/transaction.c                              |    10 +-
 fs/jffs2/Kconfig                                   |     6 +-
 fs/jffs2/fs.c                                      |     1 -
 fs/jfs/Kconfig                                     |     3 -
 fs/jfs/super.c                                     |     8 +-
 fs/kernfs/file.c                                   |     8 +-
 fs/lockd/clntproc.c                                |    14 +-
 fs/lockd/host.c                                    |    22 +-
 fs/lockd/mon.c                                     |    14 +-
 fs/lockd/svcproc.c                                 |     2 +-
 fs/locks.c                                         |     6 +-
 fs/mbcache.c                                       |     8 +-
 fs/namei.c                                         |   100 +-
 fs/nfs/blocklayout/blocklayout.c                   |    94 +-
 fs/nfs/blocklayout/blocklayout.h                   |     7 +-
 fs/nfs/blocklayout/dev.c                           |     7 +-
 fs/nfs/callback_proc.c                             |    14 +-
 fs/nfs/delegation.c                                |     3 +-
 fs/nfs/direct.c                                    |     6 +-
 fs/nfs/export.c                                    |     5 -
 fs/nfs/filelayout/filelayout.c                     |     4 +-
 fs/nfs/fscache-index.c                             |     5 +-
 fs/nfs/inode.c                                     |    71 +-
 fs/nfs/io.c                                        |     2 +-
 fs/nfs/nfs3proc.c                                  |     2 +-
 fs/nfs/nfs4client.c                                |    30 +-
 fs/nfs/nfs4file.c                                  |     1 -
 fs/nfs/nfs4idmap.c                                 |     6 +-
 fs/nfs/nfs4namespace.c                             |     2 -
 fs/nfs/nfs4proc.c                                  |    52 +-
 fs/nfs/nfs4state.c                                 |     5 +-
 fs/nfs/nfs4sysctl.c                                |     2 +-
 fs/nfs/nfs4xdr.c                                   |    64 +-
 fs/nfs/nfstrace.h                                  |    27 +-
 fs/nfs/pagelist.c                                  |     8 +-
 fs/nfs/pnfs.c                                      |    19 +-
 fs/nfs/pnfs.h                                      |     6 +-
 fs/nfs/pnfs_dev.c                                  |     1 -
 fs/nfs/super.c                                     |     2 +
 fs/nfs/write.c                                     |    93 +-
 fs/nfsd/nfs3xdr.c                                  |    31 +-
 fs/nfsd/nfs4proc.c                                 |     9 +-
 fs/nfsd/nfs4state.c                                |    68 +-
 fs/nfsd/nfs4xdr.c                                  |    16 +-
 fs/nfsd/nfsfh.h                                    |    31 +-
 fs/nfsd/nfsxdr.c                                   |     1 +
 fs/nilfs2/segbuf.c                                 |     2 +-
 fs/nilfs2/segbuf.h                                 |     4 +-
 fs/nilfs2/segment.c                                |     2 +-
 fs/nilfs2/segment.h                                |     2 +-
 fs/nilfs2/sufile.c                                 |     2 +-
 fs/nilfs2/sufile.h                                 |     2 +-
 fs/nilfs2/super.c                                  |     4 +-
 fs/nilfs2/sysfs.c                                  |    21 +-
 fs/nilfs2/the_nilfs.h                              |     8 +-
 fs/notify/fanotify/fanotify_user.c                 |     6 +-
 fs/notify/inotify/inotify_user.c                   |     6 +-
 fs/nsfs.c                                          |    29 +-
 fs/ntfs/inode.c                                    |     9 -
 fs/ntfs/mft.c                                      |     6 -
 fs/ocfs2/acl.c                                     |     6 +
 fs/ocfs2/alloc.c                                   |   261 +-
 fs/ocfs2/alloc.h                                   |     1 +
 fs/ocfs2/aops.c                                    |    10 +-
 fs/ocfs2/cluster/quorum.c                          |     5 +-
 fs/ocfs2/cluster/tcp.c                             |     3 +-
 fs/ocfs2/cluster/tcp_internal.h                    |     2 +-
 fs/ocfs2/dir.c                                     |    17 +-
 fs/ocfs2/dlm/dlmmaster.c                           |     7 -
 fs/ocfs2/dlmfs/dlmfs.c                             |    10 +-
 fs/ocfs2/dlmglue.c                                 |   136 +-
 fs/ocfs2/dlmglue.h                                 |    35 +-
 fs/ocfs2/extent_map.c                              |    45 +
 fs/ocfs2/extent_map.h                              |     3 +
 fs/ocfs2/file.c                                    |   101 +-
 fs/ocfs2/inode.c                                   |     3 +-
 fs/ocfs2/journal.c                                 |    23 +-
 fs/ocfs2/mmap.c                                    |     2 +-
 fs/ocfs2/namei.c                                   |     3 +-
 fs/ocfs2/ocfs2.h                                   |     1 +
 fs/ocfs2/ocfs2_lockid.h                            |     5 +
 fs/ocfs2/ocfs2_trace.h                             |    10 +-
 fs/ocfs2/quota_global.c                            |     3 +-
 fs/ocfs2/suballoc.c                                |     8 +-
 fs/ocfs2/super.c                                   |    13 +-
 fs/ocfs2/xattr.c                                   |     5 +-
 fs/orangefs/dcache.c                               |    19 +-
 fs/orangefs/devorangefs-req.c                      |     6 +-
 fs/orangefs/namei.c                                |    16 +-
 fs/orangefs/orangefs-debugfs.c                     |     4 +-
 fs/orangefs/orangefs-debugfs.h                     |     1 -
 fs/orangefs/orangefs-kernel.h                      |    10 +-
 fs/orangefs/orangefs-utils.c                       |    93 +-
 fs/orangefs/protocol.h                             |    15 -
 fs/orangefs/super.c                                |    23 +-
 fs/overlayfs/Kconfig                               |    45 +-
 fs/overlayfs/Makefile                              |     3 +-
 fs/overlayfs/copy_up.c                             |   188 +-
 fs/overlayfs/dir.c                                 |   175 +-
 fs/overlayfs/export.c                              |   837 +
 fs/overlayfs/inode.c                               |   132 +-
 fs/overlayfs/namei.c                               |   539 +-
 fs/overlayfs/overlayfs.h                           |    67 +-
 fs/overlayfs/ovl_entry.h                           |    11 +-
 fs/overlayfs/readdir.c                             |    57 +-
 fs/overlayfs/super.c                               |   126 +-
 fs/overlayfs/util.c                                |   108 +-
 fs/pipe.c                                          |    83 +-
 fs/posix_acl.c                                     |     6 +-
 fs/proc/array.c                                    |     8 +-
 fs/proc/base.c                                     |    70 +-
 fs/proc/consoles.c                                 |     3 +-
 fs/proc/fd.c                                       |     2 +-
 fs/proc/generic.c                                  |    18 +-
 fs/proc/inode.c                                    |    17 +-
 fs/proc/internal.h                                 |    29 +-
 fs/proc/kcore.c                                    |    22 +-
 fs/proc/kmsg.c                                     |     4 +-
 fs/proc/proc_net.c                                 |     4 +-
 fs/proc/proc_sysctl.c                              |     8 +-
 fs/proc/self.c                                     |     9 +-
 fs/proc/task_mmu.c                                 |    19 +-
 fs/proc/thread_self.c                              |     8 +-
 fs/proc/vmcore.c                                   |     6 +-
 fs/proc_namespace.c                                |     6 +-
 fs/pstore/platform.c                               |     1 -
 fs/reiserfs/Kconfig                                |     6 +-
 fs/select.c                                        |    35 +-
 fs/seq_file.c                                      |     5 +-
 fs/signalfd.c                                      |    23 +-
 fs/super.c                                         |     2 +-
 fs/sysfs/dir.c                                     |     9 +-
 fs/sysfs/file.c                                    |     8 +-
 fs/sysfs/group.c                                   |     6 +-
 fs/sysfs/mount.c                                   |     5 +-
 fs/sysfs/symlink.c                                 |     4 +-
 fs/sysfs/sysfs.h                                   |     3 +-
 fs/timerfd.c                                       |     6 +-
 fs/ubifs/dir.c                                     |   106 +-
 fs/ubifs/file.c                                    |    77 +-
 fs/ubifs/super.c                                   |     4 +-
 fs/ubifs/tnc.c                                     |    21 +-
 fs/ubifs/xattr.c                                   |     2 -
 fs/udf/udftime.c                                   |     5 +
 fs/ufs/dir.c                                       |     9 +-
 fs/ufs/inode.c                                     |     3 +-
 fs/ufs/super.c                                     |    16 +-
 fs/userfaultfd.c                                   |    95 +-
 fs/xfs/Kconfig                                     |     3 -
 fs/xfs/libxfs/xfs_alloc.c                          |   124 +-
 fs/xfs/libxfs/xfs_alloc.h                          |    10 +
 fs/xfs/libxfs/xfs_alloc_btree.c                    |    47 +-
 fs/xfs/libxfs/xfs_attr.c                           |     4 -
 fs/xfs/libxfs/xfs_attr_leaf.c                      |   148 +-
 fs/xfs/libxfs/xfs_attr_leaf.h                      |     1 +
 fs/xfs/libxfs/xfs_attr_remote.c                    |   104 +-
 fs/xfs/libxfs/xfs_bmap.c                           |   120 +-
 fs/xfs/libxfs/xfs_bmap_btree.c                     |    58 +-
 fs/xfs/libxfs/xfs_btree.c                          |   117 +-
 fs/xfs/libxfs/xfs_btree.h                          |    16 +-
 fs/xfs/libxfs/xfs_da_btree.c                       |    70 +-
 fs/xfs/libxfs/xfs_da_format.h                      |     6 +
 fs/xfs/libxfs/xfs_dir2.c                           |     5 +-
 fs/xfs/libxfs/xfs_dir2.h                           |     2 +
 fs/xfs/libxfs/xfs_dir2_block.c                     |    39 +-
 fs/xfs/libxfs/xfs_dir2_data.c                      |   208 +-
 fs/xfs/libxfs/xfs_dir2_leaf.c                      |    89 +-
 fs/xfs/libxfs/xfs_dir2_node.c                      |    89 +-
 fs/xfs/libxfs/xfs_dir2_priv.h                      |    12 +-
 fs/xfs/libxfs/xfs_dir2_sf.c                        |    30 +-
 fs/xfs/libxfs/xfs_dquot_buf.c                      |   174 +-
 fs/xfs/libxfs/xfs_fs.h                             |     7 +
 fs/xfs/libxfs/xfs_ialloc.c                         |   143 +-
 fs/xfs/libxfs/xfs_ialloc.h                         |     6 +
 fs/xfs/libxfs/xfs_ialloc_btree.c                   |    65 +-
 fs/xfs/libxfs/xfs_inode_buf.c                      |   135 +-
 fs/xfs/libxfs/xfs_inode_buf.h                      |     4 +-
 fs/xfs/libxfs/xfs_inode_fork.c                     |   152 +-
 fs/xfs/libxfs/xfs_inode_fork.h                     |    14 +
 fs/xfs/libxfs/xfs_log_rlimit.c                     |     2 +-
 fs/xfs/libxfs/xfs_quota_defs.h                     |     9 +-
 fs/xfs/libxfs/xfs_refcount.c                       |    19 +
 fs/xfs/libxfs/xfs_refcount.h                       |     3 +
 fs/xfs/libxfs/xfs_refcount_btree.c                 |    40 +-
 fs/xfs/libxfs/xfs_rmap.c                           |    67 +
 fs/xfs/libxfs/xfs_rmap.h                           |     5 +
 fs/xfs/libxfs/xfs_rmap_btree.c                     |    40 +-
 fs/xfs/libxfs/xfs_rtbitmap.c                       |    21 +
 fs/xfs/libxfs/xfs_sb.c                             |   113 +-
 fs/xfs/libxfs/xfs_sb.h                             |     4 +
 fs/xfs/libxfs/xfs_shared.h                         |     4 +
 fs/xfs/libxfs/xfs_symlink_remote.c                 |    75 +-
 fs/xfs/libxfs/xfs_trans_resv.c                     |   199 +-
 fs/xfs/scrub/agheader.c                            |   343 +-
 fs/xfs/scrub/alloc.c                               |    81 +
 fs/xfs/scrub/bmap.c                                |   219 +-
 fs/xfs/scrub/btree.c                               |   184 +-
 fs/xfs/scrub/btree.h                               |     9 +
 fs/xfs/scrub/common.c                              |   255 +-
 fs/xfs/scrub/common.h                              |    23 +-
 fs/xfs/scrub/dabtree.c                             |    22 +
 fs/xfs/scrub/dir.c                                 |    44 +-
 fs/xfs/scrub/ialloc.c                              |   194 +-
 fs/xfs/scrub/inode.c                               |   178 +-
 fs/xfs/scrub/parent.c                              |     8 +-
 fs/xfs/scrub/quota.c                               |     7 -
 fs/xfs/scrub/refcount.c                            |   420 +-
 fs/xfs/scrub/rmap.c                                |   123 +
 fs/xfs/scrub/rtbitmap.c                            |    35 +-
 fs/xfs/scrub/scrub.c                               |   203 +-
 fs/xfs/scrub/scrub.h                               |    37 +
 fs/xfs/scrub/trace.h                               |    44 +-
 fs/xfs/xfs_aops.c                                  |    15 +-
 fs/xfs/xfs_bmap_util.c                             |     4 +-
 fs/xfs/xfs_buf.c                                   |    22 +-
 fs/xfs/xfs_buf.h                                   |     8 +-
 fs/xfs/xfs_buf_item.c                              |   156 +-
 fs/xfs/xfs_buf_item.h                              |     7 +-
 fs/xfs/xfs_dir2_readdir.c                          |     4 +-
 fs/xfs/xfs_dquot.c                                 |    62 +-
 fs/xfs/xfs_dquot_item.c                            |     9 +-
 fs/xfs/xfs_error.c                                 |    64 +-
 fs/xfs/xfs_error.h                                 |    14 +-
 fs/xfs/xfs_file.c                                  |     2 +-
 fs/xfs/xfs_fsops.c                                 |    79 +-
 fs/xfs/xfs_fsops.h                                 |     1 -
 fs/xfs/xfs_icache.c                                |    75 +-
 fs/xfs/xfs_inode.c                                 |   107 +-
 fs/xfs/xfs_inode.h                                 |     5 +-
 fs/xfs/xfs_inode_item.c                            |    44 +-
 fs/xfs/xfs_ioctl.c                                 |     5 +-
 fs/xfs/xfs_ioctl32.c                               |     3 +-
 fs/xfs/xfs_iomap.c                                 |    42 +-
 fs/xfs/xfs_linux.h                                 |    14 +
 fs/xfs/xfs_log.c                                   |    17 +-
 fs/xfs/xfs_log_recover.c                           |    58 +-
 fs/xfs/xfs_mount.c                                 |     3 +
 fs/xfs/xfs_qm.c                                    |    33 +-
 fs/xfs/xfs_refcount_item.c                         |     9 +-
 fs/xfs/xfs_reflink.c                               |    95 +-
 fs/xfs/xfs_rmap_item.c                             |     4 +-
 fs/xfs/xfs_rtalloc.h                               |     4 +
 fs/xfs/xfs_super.c                                 |    36 +-
 fs/xfs/xfs_super.h                                 |     7 +
 fs/xfs/xfs_trace.h                                 |    68 +-
 fs/xfs/xfs_trans.c                                 |    22 +
 fs/xfs/xfs_trans.h                                 |     2 +-
 fs/xfs/xfs_trans_buf.c                             |    98 +-
 fs/xfs/xfs_trans_inode.c                           |    16 +-
 include/acpi/acbuffer.h                            |     2 +-
 include/acpi/acconfig.h                            |     6 +-
 include/acpi/acexcep.h                             |    14 +-
 include/acpi/acnames.h                             |     2 +-
 include/acpi/acoutput.h                            |     2 +-
 include/acpi/acpi.h                                |     2 +-
 include/acpi/acpi_bus.h                            |     3 +
 include/acpi/acpiosxf.h                            |     2 +-
 include/acpi/acpixf.h                              |     8 +-
 include/acpi/acrestyp.h                            |     2 +-
 include/acpi/actbl.h                               |     2 +-
 include/acpi/actbl1.h                              |   161 +-
 include/acpi/actbl2.h                              |    17 +-
 include/acpi/actbl3.h                              |     2 +-
 include/acpi/actypes.h                             |    16 +-
 include/acpi/acuuid.h                              |     2 +-
 include/acpi/platform/acenv.h                      |     2 +-
 include/acpi/platform/acenvex.h                    |     2 +-
 include/acpi/platform/acgcc.h                      |     2 +-
 include/acpi/platform/acgccex.h                    |     2 +-
 include/acpi/platform/acintel.h                    |     2 +-
 include/acpi/platform/aclinux.h                    |     4 +-
 include/acpi/platform/aclinuxex.h                  |     2 +-
 include/asm-generic/audit_dir_write.h              |     2 +
 include/asm-generic/bitops/find.h                  |    20 +
 include/asm-generic/bitops/lock.h                  |     3 +-
 include/asm-generic/bug.h                          |     1 +
 include/asm-generic/clkdev.h                       |    30 -
 include/asm-generic/dma-mapping.h                  |    10 +
 include/asm-generic/error-injection.h              |    35 +
 include/asm-generic/pci_iomap.h                    |     6 +-
 include/asm-generic/pgtable.h                      |    35 +-
 include/asm-generic/qrwlock_types.h                |     1 +
 include/asm-generic/sections.h                     |     8 +-
 include/asm-generic/vmlinux.lds.h                  |    14 +
 include/crypto/aead.h                              |    10 +-
 include/crypto/chacha20.h                          |     3 +-
 include/crypto/hash.h                              |    46 +-
 include/crypto/if_alg.h                            |     2 +-
 include/crypto/internal/hash.h                     |     2 +
 include/crypto/internal/scompress.h                |    11 -
 include/crypto/null.h                              |    10 -
 include/crypto/poly1305.h                          |     2 -
 include/crypto/salsa20.h                           |    27 +
 include/crypto/sha3.h                              |     6 +-
 include/crypto/skcipher.h                          |    11 +-
 include/drm/drmP.h                                 |   197 +-
 include/drm/drm_atomic.h                           |    41 +
 include/drm/drm_atomic_helper.h                    |     7 +
 include/drm/drm_connector.h                        |    54 +-
 include/drm/drm_crtc_helper.h                      |     1 +
 include/drm/drm_device.h                           |     9 +
 include/drm/drm_dp_helper.h                        |     1 +
 include/drm/drm_drv.h                              |    23 +-
 include/drm/drm_edid.h                             |     2 +-
 include/drm/drm_encoder.h                          |     6 +-
 include/drm/drm_fb_cma_helper.h                    |    13 +-
 include/drm/drm_fb_helper.h                        |    57 +
 include/drm/drm_file.h                             |     2 +-
 include/drm/drm_framebuffer.h                      |     8 +-
 include/drm/drm_gem_cma_helper.h                   |    16 +-
 include/drm/drm_mm.h                               |     2 +-
 include/drm/drm_mode_config.h                      |    27 +-
 include/drm/drm_modeset_helper.h                   |     3 +
 include/drm/drm_modeset_helper_vtables.h           |     3 -
 include/drm/drm_plane.h                            |    14 +-
 include/drm/drm_plane_helper.h                     |     5 -
 include/drm/drm_print.h                            |   219 +
 include/drm/drm_syncobj.h                          |    36 +-
 include/drm/drm_utils.h                            |    15 +
 include/drm/drm_vma_manager.h                      |     2 +-
 include/drm/gpu_scheduler.h                        |   173 +
 .../drm/gpu_scheduler_trace.h                      |    21 +-
 include/drm/i915_drm.h                             |     3 +
 include/drm/i915_pciids.h                          |    32 +-
 include/drm/intel-gtt.h                            |     3 +-
 include/drm/spsc_queue.h                           |   122 +
 include/drm/tinydrm/mipi-dbi.h                     |     4 +-
 include/drm/tinydrm/tinydrm.h                      |     8 +-
 include/drm/ttm/ttm_bo_api.h                       |   169 +-
 include/drm/ttm/ttm_bo_driver.h                    |   157 +-
 include/drm/ttm/ttm_memory.h                       |    75 +-
 include/drm/ttm/ttm_page_alloc.h                   |    11 +-
 include/dt-bindings/bus/ti-sysc.h                  |    22 +
 include/dt-bindings/clock/am3.h                    |   108 +
 include/dt-bindings/clock/am4.h                    |   113 +
 include/dt-bindings/clock/aspeed-clock.h           |    52 +
 include/dt-bindings/clock/axg-clkc.h               |    71 +
 include/dt-bindings/clock/dm814.h                  |    45 +
 include/dt-bindings/clock/dm816.h                  |    53 +
 include/dt-bindings/clock/dra7.h                   |   172 +
 include/dt-bindings/clock/hi3660-clock.h           |     7 +
 include/dt-bindings/clock/jz4770-cgu.h             |    58 +
 include/dt-bindings/clock/omap5.h                  |   118 +
 include/dt-bindings/clock/qcom,gcc-ipq8074.h       |   222 +
 include/dt-bindings/clock/sprd,sc9860-clk.h        |   404 +
 include/dt-bindings/gpio/aspeed-gpio.h             |    49 +
 include/dt-bindings/gpio/gpio.h                    |     6 +-
 include/dt-bindings/gpio/meson-axg-gpio.h          |   116 +
 include/dt-bindings/memory/tegra186-mc.h           |   111 +
 include/dt-bindings/pinctrl/am43xx.h               |     3 +-
 include/dt-bindings/pinctrl/stm32-pinfunc.h        |     6 +
 include/dt-bindings/power/mt2712-power.h           |    26 +
 include/dt-bindings/power/owl-s700-powergate.h     |    19 +
 .../dt-bindings/reset/amlogic,meson-axg-reset.h    |   124 +
 include/kvm/arm_arch_timer.h                       |     2 +
 include/kvm/arm_psci.h                             |    51 +
 include/kvm/arm_vgic.h                             |    14 +-
 include/linux/acpi.h                               |    28 +-
 include/linux/arch_topology.h                      |     2 +-
 include/linux/arm-smccc.h                          |   165 +-
 include/linux/arm_sdei.h                           |    79 +
 include/linux/ata.h                                |     2 +
 include/linux/backing-dev.h                        |     2 +-
 include/linux/bio.h                                |    28 +-
 include/linux/bitfield.h                           |    46 +
 include/linux/bitmap.h                             |    63 +-
 include/linux/blk-cgroup.h                         |     8 +-
 include/linux/blk-mq.h                             |     3 +-
 include/linux/blk_types.h                          |    46 +
 include/linux/blkdev.h                             |   174 +-
 include/linux/bpf.h                                |    94 +-
 include/linux/bpf_types.h                          |     2 +-
 include/linux/bpf_verifier.h                       |    63 +-
 include/linux/brcmphy.h                            |     1 +
 include/linux/buffer_head.h                        |     6 +-
 include/linux/build_bug.h                          |     2 -
 include/linux/bvec.h                               |     9 +
 include/linux/can/dev.h                            |     7 +
 include/linux/cgroup-defs.h                        |     6 +-
 include/linux/clk-provider.h                       |    43 +-
 include/linux/clk.h                                |    62 +
 include/linux/clkdev.h                             |     7 +-
 include/linux/compat.h                             |   126 +-
 include/linux/compiler-clang.h                     |    13 +
 include/linux/compiler-gcc.h                       |    26 +-
 include/linux/compiler.h                           |    31 +-
 include/linux/cper.h                               |    48 +
 include/linux/cpu_cooling.h                        |    75 +-
 include/linux/cpufreq.h                            |   125 +-
 include/linux/cpuhotplug.h                         |     3 +
 include/linux/cpuidle.h                            |    42 +-
 include/linux/cpumask.h                            |     4 +-
 include/linux/cpuset.h                             |     6 +-
 include/linux/crash_dump.h                         |    12 +-
 include/linux/crc-ccitt.h                          |     7 +
 include/linux/crypto.h                             |    10 +-
 include/linux/dax.h                                |     2 +-
 include/linux/dcache.h                             |     2 +
 include/linux/device-mapper.h                      |    56 +-
 include/linux/device.h                             |    10 +-
 include/linux/dma-buf.h                            |     2 +-
 include/linux/dma-direct.h                         |    47 +
 include/linux/dma-fence-array.h                    |     3 +
 include/linux/dma-fence.h                          |     2 +-
 include/linux/dma-mapping.h                        |    23 +-
 include/linux/dsa/lan9303.h                        |     3 +-
 include/linux/efi.h                                |    46 +
 include/linux/elevator.h                           |     2 -
 include/linux/error-injection.h                    |    27 +
 include/linux/errseq.h                             |     2 +-
 include/linux/etherdevice.h                        |     2 +-
 include/linux/eventfd.h                            |    14 +-
 include/linux/f2fs_fs.h                            |    14 +-
 include/linux/fb.h                                 |    10 +-
 include/linux/fdtable.h                            |     5 +-
 include/linux/filter.h                             |    40 +-
 include/linux/fpga/fpga-bridge.h                   |    14 +-
 include/linux/fpga/fpga-mgr.h                      |    39 +-
 include/linux/fpga/fpga-region.h                   |    40 +
 include/linux/fs.h                                 |    37 +-
 include/linux/fscrypt.h                            |   174 +-
 include/linux/fscrypt_notsupp.h                    |    59 +-
 include/linux/fscrypt_supp.h                       |    68 +-
 include/linux/fsl_ifc.h                            |     6 +-
 include/linux/ftrace.h                             |    12 -
 include/linux/fwnode.h                             |     4 +
 include/linux/genetlink.h                          |     3 +-
 include/linux/genhd.h                              |     9 +-
 include/linux/genl_magic_func.h                    |    12 +-
 include/linux/gpio.h                               |    10 +
 include/linux/gpio/consumer.h                      |    25 +
 include/linux/gpio/driver.h                        |     3 +
 include/linux/gpio/machine.h                       |     4 +-
 include/linux/hid.h                                |    22 +-
 include/linux/hil_mlc.h                            |     6 +-
 include/linux/hp_sdc.h                             |     2 +-
 include/linux/hrtimer.h                            |   113 +-
 include/linux/hugetlb.h                            |    21 +-
 include/linux/hyperv.h                             |    22 -
 include/linux/i2c.h                                |   112 +-
 include/linux/i7300_idle.h                         |    84 -
 include/linux/idr.h                                |   174 +-
 include/linux/if_link.h                            |     2 +
 include/linux/if_macvlan.h                         |     2 +-
 include/linux/if_tap.h                             |     6 +-
 include/linux/if_tun.h                             |    25 +-
 include/linux/if_vlan.h                            |    79 +-
 include/linux/iio/adc/stm32-dfsdm-adc.h            |    18 +
 include/linux/iio/consumer.h                       |    37 +
 include/linux/iio/hw-consumer.h                    |    21 +
 include/linux/iio/iio.h                            |    32 +-
 include/linux/iio/machine.h                        |     7 +
 include/linux/iio/trigger.h                        |     3 +-
 include/linux/iio/types.h                          |    28 +
 include/linux/inetdevice.h                         |     2 +-
 include/linux/init.h                               |     9 +-
 include/linux/init_task.h                          |   258 +-
 include/linux/input/gpio_tilt.h                    |    74 -
 include/linux/integrity.h                          |     1 +
 include/linux/intel-iommu.h                        |     2 +
 include/linux/ioport.h                             |     2 +-
 include/linux/irq_work.h                           |    11 +-
 include/linux/irqchip/arm-gic-v3.h                 |     1 +
 include/linux/irqchip/arm-gic.h                    |     1 +
 include/linux/irqflags.h                           |     2 -
 include/linux/iversion.h                           |   337 +
 include/linux/jbd2.h                               |   431 +-
 include/linux/jump_label.h                         |     7 +-
 include/linux/kallsyms.h                           |    74 +-
 include/linux/kasan.h                              |    15 +-
 include/linux/kconfig.h                            |     9 +
 include/linux/kcore.h                              |     1 +
 include/linux/kernel.h                             |     1 +
 include/linux/kfifo.h                              |     3 +-
 include/linux/kobject.h                            |     3 +-
 include/linux/kobject_ns.h                         |     3 +-
 include/linux/kvm_host.h                           |    20 +-
 include/linux/led-class-flash.h                    |     4 +
 include/linux/libfdt.h                             |     1 -
 include/linux/libnvdimm.h                          |    11 +
 include/linux/lightnvm.h                           |   125 +-
 include/linux/livepatch.h                          |     4 -
 include/linux/lockd/lockd.h                        |     9 +-
 include/linux/lockdep.h                            |     7 +-
 include/linux/lockref.h                            |     2 +-
 include/linux/mdio.h                               |     8 +
 include/linux/memblock.h                           |     5 +-
 include/linux/memcontrol.h                         |   169 +-
 include/linux/memory_hotplug.h                     |    29 +-
 include/linux/memremap.h                           |    77 +-
 include/linux/mfd/axp20x.h                         |     5 -
 include/linux/mfd/cros_ec.h                        |     4 +
 include/linux/mfd/cros_ec_commands.h               |    25 +
 include/linux/mfd/palmas.h                         |     3 +
 include/linux/mfd/rave-sp.h                        |    60 +
 include/linux/mfd/stm32-lptimer.h                  |     6 +-
 include/linux/mfd/stm32-timers.h                   |     4 +-
 include/linux/mfd/tmio.h                           |    20 -
 include/linux/mlx5/device.h                        |    16 +
 include/linux/mlx5/driver.h                        |    49 +-
 include/linux/mlx5/fs.h                            |     4 +
 include/linux/mlx5/mlx5_ifc.h                      |    73 +-
 include/linux/mlx5/qp.h                            |    12 +
 include/linux/mlx5/transobj.h                      |    23 +
 include/linux/mlx5/vport.h                         |     4 +
 include/linux/mm.h                                 |    52 +-
 include/linux/mm_inline.h                          |     6 -
 include/linux/mm_types.h                           |   154 +-
 include/linux/mmc/host.h                           |     5 +-
 include/linux/mmc/slot-gpio.h                      |     1 +
 include/linux/mmu_notifier.h                       |    30 +-
 include/linux/mmzone.h                             |    12 +-
 include/linux/mod_devicetable.h                    |    19 +
 include/linux/module.h                             |    27 +-
 include/linux/mtd/map.h                            |   130 +-
 include/linux/mtd/mtd.h                            |    28 +
 include/linux/mtd/rawnand.h                        |   443 +-
 include/linux/mtd/spi-nor.h                        |    12 +-
 include/linux/mutex.h                              |     9 +-
 include/linux/mux/consumer.h                       |     5 +-
 include/linux/mux/driver.h                         |     5 +-
 include/linux/net.h                                |     4 +-
 include/linux/net_dim.h                            |   380 +
 include/linux/netdev_features.h                    |     3 +
 include/linux/netdevice.h                          |    60 +-
 include/linux/netfilter.h                          |   116 +-
 include/linux/netfilter/ipset/ip_set.h             |     6 +
 include/linux/netfilter/ipset/ip_set_counter.h     |    25 +-
 include/linux/netfilter/nfnetlink.h                |     3 +-
 include/linux/netfilter/x_tables.h                 |     4 +
 include/linux/netfilter_defs.h                     |    12 +-
 include/linux/netfilter_ipv4.h                     |    46 +
 include/linux/netfilter_ipv6.h                     |    19 +
 include/linux/nfs4.h                               |    12 +-
 include/linux/nospec.h                             |    58 +
 include/linux/ntb.h                                |    51 +-
 include/linux/nubus.h                              |   189 +-
 include/linux/nvme.h                               |    22 +-
 include/linux/of.h                                 |    13 +-
 include/linux/of_dma.h                             |     5 +-
 include/linux/of_fdt.h                             |    18 +-
 include/linux/of_gpio.h                            |     8 +-
 include/linux/of_graph.h                           |     5 +-
 include/linux/of_iommu.h                           |     5 +-
 include/linux/of_pci.h                             |    25 +-
 include/linux/of_pdt.h                             |     6 +-
 include/linux/of_platform.h                        |     7 +-
 include/linux/omap-gpmc.h                          |    28 +
 include/linux/page-flags.h                         |     5 -
 include/linux/pagevec.h                            |     6 +-
 include/linux/pci-aspm.h                           |    35 +-
 include/linux/pci-dma-compat.h                     |    27 +-
 include/linux/pci-ecam.h                           |    13 +-
 include/linux/pci-ep-cfs.h                         |     5 +-
 include/linux/pci-epc.h                            |    48 +-
 include/linux/pci-epf.h                            |     5 +-
 include/linux/pci.h                                |   480 +-
 include/linux/pci_hotplug.h                        |    16 +-
 include/linux/pci_ids.h                            |     4 +
 include/linux/percpu-refcount.h                    |    24 +-
 include/linux/percpu_counter.h                     |     6 +-
 include/linux/perf/arm_pmu.h                       |    26 +-
 include/linux/pfn_t.h                              |    13 +
 include/linux/phy.h                                |   151 +-
 include/linux/phy_fixed.h                          |     9 -
 include/linux/phylink.h                            |   201 +-
 include/linux/pinctrl/devinfo.h                    |     2 +
 include/linux/pinctrl/pinconf-generic.h            |     2 +
 include/linux/pinctrl/pinctrl.h                    |     1 +
 include/linux/pipe_fs_i.h                          |     5 +-
 include/linux/platform_data/at24.h                 |     2 +
 include/linux/platform_data/i2c-davinci.h          |     5 +-
 .../{i2c/pxa-i2c.h => platform_data/i2c-pxa.h}     |    11 -
 include/linux/platform_data/mlxcpld-hotplug.h      |    99 -
 include/linux/platform_data/mlxreg.h               |   144 +
 include/linux/platform_data/mms114.h               |    24 -
 include/linux/platform_data/mtd-onenand-omap2.h    |    34 -
 include/linux/platform_data/si5351.h               |     2 +
 include/linux/platform_data/spi-s3c64xx.h          |     6 +-
 include/linux/platform_data/ti-sysc.h              |    86 +
 include/linux/pm.h                                 |    16 +-
 include/linux/pm_wakeup.h                          |     7 +
 include/linux/poll.h                               |    37 +-
 include/linux/posix-clock.h                        |     2 +-
 include/linux/posix-timers.h                       |    25 +-
 include/linux/posix_acl.h                          |     7 +-
 include/linux/power/bq27xxx_battery.h              |     1 +
 include/linux/proc_ns.h                            |     3 +
 include/linux/property.h                           |    23 +-
 include/linux/psci.h                               |    17 +-
 include/linux/psp-sev.h                            |   606 +
 include/linux/ptr_ring.h                           |    94 +-
 include/linux/qcom_scm.h                           |     3 +
 include/linux/qed/common_hsi.h                     |  1264 +-
 include/linux/qed/eth_common.h                     |   396 +-
 include/linux/qed/fcoe_common.h                    |   940 +-
 include/linux/qed/iscsi_common.h                   |  1585 +-
 include/linux/qed/iwarp_common.h                   |    17 +-
 include/linux/qed/qed_eth_if.h                     |    38 +-
 include/linux/qed/qed_if.h                         |    36 +-
 include/linux/qed/qed_iscsi_if.h                   |     2 -
 include/linux/qed/qed_ll2_if.h                     |     2 +-
 include/linux/qed/rdma_common.h                    |    25 +-
 include/linux/qed/roce_common.h                    |    15 +-
 include/linux/qed/storage_common.h                 |    91 +-
 include/linux/qed/tcp_common.h                     |   165 +-
 include/linux/radix-tree.h                         |    17 +-
 include/linux/rcupdate.h                           |    25 +-
 include/linux/rcutiny.h                            |     1 -
 include/linux/rcutree.h                            |     1 -
 include/linux/refcount.h                           |     2 +-
 include/linux/regmap.h                             |    68 +-
 include/linux/regulator/driver.h                   |     2 +
 include/linux/regulator/machine.h                  |    37 +-
 include/linux/remoteproc.h                         |    21 +-
 include/linux/reservation.h                        |    23 +
 include/linux/reset.h                              |    50 +-
 include/linux/rhashtable.h                         |    42 +-
 include/linux/ring_buffer.h                        |     2 +-
 include/linux/rpmsg.h                              |     4 +-
 include/linux/rtc.h                                |     1 -
 include/linux/rtnetlink.h                          |    18 +-
 include/linux/{mfd => }/rtsx_common.h              |     0
 include/linux/{mfd => }/rtsx_pci.h                 |   236 +-
 include/linux/{mfd => }/rtsx_usb.h                 |     0
 include/linux/scatterlist.h                        |    11 +
 include/linux/sched.h                              |    27 +
 include/linux/sched/cpufreq.h                      |     2 -
 include/linux/sched/mm.h                           |    60 +-
 include/linux/sched/signal.h                       |    28 +
 include/linux/sched/task.h                         |    14 +
 include/linux/sched/task_stack.h                   |     2 +-
 include/linux/sched/topology.h                     |    12 +-
 include/linux/sched/user.h                         |     4 +
 include/linux/scif.h                               |    20 +-
 include/linux/sctp.h                               |    37 +
 include/linux/seccomp.h                            |     8 +
 include/linux/semaphore.h                          |     2 +-
 include/linux/seq_file.h                           |    14 +
 include/linux/seqlock.h                            |     3 +-
 include/linux/serdev.h                             |    17 +-
 include/linux/serial_core.h                        |    12 +-
 include/linux/sfp.h                                |    94 +-
 include/linux/sh_eth.h                             |     3 -
 include/linux/shmem_fs.h                           |     6 +-
 include/linux/signal.h                             |    15 +-
 include/linux/siox.h                               |    77 +
 include/linux/skb_array.h                          |     7 +-
 include/linux/skbuff.h                             |    52 +-
 include/linux/slab.h                               |    41 +-
 include/linux/slab_def.h                           |     3 +
 include/linux/slimbus.h                            |   164 +
 include/linux/slub_def.h                           |     3 +
 include/linux/soc/brcmstb/brcmstb.h                |     6 -
 include/linux/soc/mediatek/infracfg.h              |     7 +-
 include/linux/soc/qcom/qmi.h                       |   271 +
 include/linux/sound.h                              |     2 -
 include/linux/soundwire/sdw.h                      |   479 +
 include/linux/soundwire/sdw_intel.h                |    24 +
 include/linux/soundwire/sdw_registers.h            |   194 +
 include/linux/soundwire/sdw_type.h                 |    19 +
 include/linux/spinlock.h                           |     6 +
 include/linux/srcu.h                               |     4 +-
 include/linux/srcutree.h                           |     8 +-
 include/linux/stddef.h                             |    10 +-
 include/linux/string.h                             |     3 +-
 include/linux/sunrpc/clnt.h                        |     1 -
 include/linux/sunrpc/sched.h                       |     3 +
 include/linux/sunrpc/svc_rdma.h                    |     2 -
 include/linux/sunrpc/xprtrdma.h                    |     2 +-
 include/linux/suspend.h                            |    30 +-
 include/linux/swap.h                               |     4 -
 include/linux/swiotlb.h                            |    12 +-
 include/linux/switchtec.h                          |    26 +-
 include/linux/sync_core.h                          |    21 +
 include/linux/sysctl.h                             |     3 -
 include/linux/sysfs.h                              |    14 +-
 include/linux/tcp.h                                |    13 +-
 include/linux/tee_drv.h                            |   196 +-
 include/linux/ti-emif-sram.h                       |    69 +
 include/linux/torture.h                            |     8 +-
 include/linux/tpm.h                                |    39 +-
 {drivers/char/tpm => include/linux}/tpm_eventlog.h |    34 +-
 include/linux/trace_events.h                       |     7 +
 include/linux/tracepoint.h                         |     5 +-
 include/linux/tty.h                                |     3 +
 include/linux/tty_ldisc.h                          |     2 +-
 include/linux/u64_stats_sync.h                     |    22 +
 include/linux/uaccess.h                            |     8 +
 include/linux/usb.h                                |     7 +-
 include/linux/usb/gadget.h                         |     2 +
 include/linux/usb/of.h                             |    21 +-
 include/linux/usb/pd.h                             |     2 +
 include/linux/usb/pd_vdo.h                         |     2 +-
 include/linux/usb/quirks.h                         |     3 +
 include/linux/usb/renesas_usbhs.h                  |     9 +
 include/linux/usb/tcpm.h                           |    16 +-
 include/linux/uuid.h                               |     1 +
 include/linux/vbox_utils.h                         |    79 +
 include/linux/vfio.h                               |     3 +-
 .../unisys/include => include/linux}/visorbus.h    |   178 +-
 include/linux/vmstat.h                             |    17 -
 include/linux/w1-gpio.h                            |     9 +-
 include/linux/wait.h                               |    10 +-
 include/linux/workqueue.h                          |     2 +-
 include/linux/zpool.h                              |     2 +
 include/media/cec.h                                |    18 +-
 {drivers/media/dvb-core => include/media}/demux.h  |    21 +-
 {drivers/media/dvb-core => include/media}/dmxdev.h |    45 +-
 include/media/drv-intf/cx2341x.h                   |   144 +-
 include/media/drv-intf/exynos-fimc.h               |     3 +-
 include/media/drv-intf/msp3400.h                   |    62 +-
 include/media/drv-intf/saa7146.h                   |     2 +-
 .../media/dvb-core => include/media}/dvb-usb-ids.h |     3 +
 .../dvb-core => include/media}/dvb_ca_en50221.h    |     2 +-
 .../media/dvb-core => include/media}/dvb_demux.h   |    29 +-
 .../dvb-core => include/media}/dvb_frontend.h      |    19 +-
 .../media/dvb-core => include/media}/dvb_math.h    |     0
 .../media/dvb-core => include/media}/dvb_net.h     |     2 +-
 .../dvb-core => include/media}/dvb_ringbuffer.h    |     0
 include/media/dvb_vb2.h                            |   280 +
 {drivers/media/dvb-core => include/media}/dvbdev.h |     4 +-
 include/media/i2c-addr.h                           |    43 -
 include/media/i2c/as3645a.h                        |    66 -
 include/media/i2c/bt819.h                          |     4 +-
 include/media/i2c/ir-kbd-i2c.h                     |     6 +-
 include/media/i2c/m52790.h                         |    52 +-
 include/media/i2c/saa7115.h                        |    12 +-
 include/media/i2c/tvaudio.h                        |    17 +-
 include/media/i2c/upd64031a.h                      |     6 +-
 include/media/lirc.h                               |     1 -
 include/media/lirc_dev.h                           |   192 -
 include/media/media-devnode.h                      |     2 +-
 include/media/media-entity.h                       |    11 +-
 include/media/rc-core.h                            |    69 +-
 include/media/rc-map.h                             |    54 +-
 include/media/soc_camera.h                         |     2 +-
 include/media/{ => tpg}/v4l2-tpg.h                 |    45 +-
 include/media/tuner-types.h                        |    15 +
 include/media/v4l2-async.h                         |    39 +-
 include/media/v4l2-common.h                        |   145 +-
 include/media/v4l2-ctrls.h                         |    13 +-
 include/media/v4l2-dev.h                           |   142 +-
 include/media/v4l2-device.h                        |   246 +-
 include/media/v4l2-dv-timings.h                    |    16 +-
 include/media/v4l2-event.h                         |    36 +-
 include/media/v4l2-flash-led-class.h               |    12 +
 include/media/v4l2-fwnode.h                        |    12 +-
 include/media/v4l2-mediabus.h                      |    80 +-
 include/media/v4l2-mem2mem.h                       |     4 +-
 include/media/v4l2-subdev.h                        |   150 +-
 include/media/v4l2-tpg-colors.h                    |    68 -
 include/media/videobuf-core.h                      |     2 +-
 include/media/videobuf-dvb.h                       |    10 +-
 include/media/videobuf2-core.h                     |   521 +-
 include/media/videobuf2-dvb.h                      |    11 +-
 include/media/videobuf2-memops.h                   |     8 +-
 include/media/videobuf2-v4l2.h                     |   117 +-
 include/misc/cxl.h                                 |     2 +-
 include/misc/ocxl-config.h                         |    45 +
 include/misc/ocxl.h                                |   214 +
 include/net/act_api.h                              |    15 +-
 include/net/addrconf.h                             |     2 +-
 include/net/bluetooth/bluetooth.h                  |     2 +-
 include/net/caif/cfpkt.h                           |    27 +
 include/net/cfg80211.h                             |    17 +-
 include/net/devlink.h                              |   127 +
 include/net/dn_route.h                             |     1 +
 include/net/dsa.h                                  |    61 +-
 include/net/dst.h                                  |    39 +-
 include/net/erspan.h                               |   240 +-
 include/net/gen_stats.h                            |     3 +
 include/net/inet_connection_sock.h                 |     6 +-
 include/net/inet_hashtables.h                      |    29 +-
 include/net/inet_sock.h                            |    25 +
 include/net/inet_timewait_sock.h                   |     4 +-
 include/net/ip.h                                   |    20 +-
 include/net/ip6_fib.h                              |    20 +-
 include/net/ip6_route.h                            |    14 +-
 include/net/ip6_tunnel.h                           |     4 +
 include/net/ip_fib.h                               |     1 +
 include/net/ip_tunnels.h                           |     5 +-
 include/net/ip_vs.h                                |     3 +-
 include/net/ipv6.h                                 |    19 +
 include/net/iucv/af_iucv.h                         |     2 +-
 include/net/llc_conn.h                             |     2 +-
 include/net/mac80211.h                             |    16 +-
 include/net/net_namespace.h                        |    10 +-
 include/net/netfilter/ipv4/nf_conntrack_ipv4.h     |    12 +-
 include/net/netfilter/ipv6/nf_conntrack_ipv6.h     |    12 +-
 include/net/netfilter/nf_conntrack.h               |     5 -
 include/net/netfilter/nf_conntrack_count.h         |    17 +
 include/net/netfilter/nf_conntrack_l4proto.h       |    19 +-
 include/net/netfilter/nf_flow_table.h              |   126 +
 include/net/netfilter/nf_queue.h                   |     2 +-
 include/net/netfilter/nf_tables.h                  |   133 +-
 include/net/netfilter/nf_tables_ipv4.h             |    27 +-
 include/net/netfilter/nf_tables_ipv6.h             |    29 +-
 include/net/netns/can.h                            |     4 +-
 include/net/netns/core.h                           |     5 +-
 include/net/netns/netfilter.h                      |    12 +-
 include/net/netns/nftables.h                       |     8 +-
 include/net/netns/sctp.h                           |     5 +-
 include/net/pkt_cls.h                              |   113 +-
 include/net/pkt_sched.h                            |    17 +-
 include/net/regulatory.h                           |     2 +-
 include/net/route.h                                |     8 +-
 include/net/rtnetlink.h                            |     4 +-
 include/net/sch_generic.h                          |   170 +-
 include/net/sctp/constants.h                       |     9 +-
 include/net/sctp/sctp.h                            |     8 +-
 include/net/sctp/sm.h                              |    18 +-
 include/net/sctp/stream_interleave.h               |    61 +
 include/net/sctp/structs.h                         |    77 +-
 include/net/sctp/ulpevent.h                        |    23 +-
 include/net/sctp/ulpqueue.h                        |    10 +-
 include/net/sock.h                                 |    66 +-
 include/net/tc_act/tc_csum.h                       |    16 +-
 include/net/tc_act/tc_mirred.h                     |     6 +-
 include/net/tcp.h                                  |    62 +-
 include/net/tls.h                                  |     2 +
 include/net/udp.h                                  |     2 +-
 include/net/udplite.h                              |     1 +
 include/net/vxlan.h                                |     2 +-
 include/net/wext.h                                 |     4 +-
 include/net/xdp.h                                  |    48 +
 include/net/xfrm.h                                 |    79 +-
 include/rdma/ib_addr.h                             |    40 +-
 include/rdma/ib_hdrs.h                             |    19 +-
 include/rdma/ib_sa.h                               |    10 +-
 include/rdma/ib_verbs.h                            |    80 +-
 include/rdma/opa_addr.h                            |    16 +
 include/rdma/rdma_cm.h                             |    19 +
 include/rdma/rdma_cm_ib.h                          |     8 +-
 include/rdma/rdma_vt.h                             |    31 +-
 include/rdma/restrack.h                            |   153 +
 include/rdma/uverbs_ioctl.h                        |    43 +-
 include/scsi/libsas.h                              |    30 +-
 include/scsi/scsi_cmnd.h                           |     7 +-
 include/scsi/scsi_host.h                           |     3 +
 include/scsi/scsi_proto.h                          |     1 +
 include/scsi/scsi_transport_fc.h                   |     4 +-
 include/scsi/scsi_transport_sas.h                  |     1 +
 include/scsi/srp.h                                 |    17 +
 include/soc/arc/mcip.h                             |     5 +
 include/soc/tegra/mc.h                             |     9 +
 include/soc/tegra/pmc.h                            |    12 +
 include/sound/ac97/regs.h                          |     2 +-
 include/sound/hdaudio_ext.h                        |     4 +-
 include/sound/hwdep.h                              |     2 +-
 include/sound/info.h                               |     2 +-
 include/sound/pcm.h                                |     8 +
 include/sound/rt5514.h                             |     2 +
 include/sound/rt5645.h                             |     3 +
 include/sound/soc-acpi-intel-match.h               |     1 +
 include/sound/soc-acpi.h                           |    14 +-
 include/sound/soc-dai.h                            |     5 +-
 include/sound/soc.h                                |    11 +-
 include/target/target_core_backend.h               |     1 +
 include/target/target_core_base.h                  |     2 +
 include/trace/events/bridge.h                      |     4 +-
 include/trace/events/btrfs.h                       |     1 -
 include/trace/events/f2fs.h                        |     3 +-
 include/trace/events/mmc.h                         |     4 +-
 include/trace/events/net_probe_common.h            |    44 +
 include/trace/events/rcu.h                         |    75 +-
 include/trace/events/rdma.h                        |   129 +
 include/trace/events/rpcrdma.h                     |   890 +
 include/trace/events/sctp.h                        |    99 +
 include/trace/events/siox.h                        |    66 +
 include/trace/events/sock.h                        |   117 +
 include/trace/events/sunrpc.h                      |    12 +-
 include/trace/events/tcp.h                         |    76 +-
 include/trace/events/thermal.h                     |    10 +-
 include/trace/events/timer.h                       |    37 +-
 include/trace/events/vmscan.h                      |    23 +-
 include/trace/events/xen.h                         |     2 +-
 include/uapi/asm-generic/poll.h                    |     4 +-
 include/uapi/asm-generic/siginfo.h                 |   109 +-
 include/uapi/drm/amdgpu_drm.h                      |    12 +
 include/uapi/drm/drm_fourcc.h                      |    38 +-
 include/uapi/drm/exynos_drm.h                      |   192 +-
 include/uapi/drm/i915_drm.h                        |    77 +
 include/uapi/drm/virtgpu_drm.h                     |     1 +
 include/uapi/linux/arm_sdei.h                      |    73 +
 .../packet.h => include/uapi/linux/batadv_packet.h |   269 +-
 include/uapi/linux/batman_adv.h                    |    27 +-
 include/uapi/linux/blktrace_api.h                  |     2 +-
 include/uapi/linux/bpf.h                           |   116 +-
 include/uapi/linux/bpf_common.h                    |     7 +-
 include/uapi/linux/btrfs.h                         |    11 +-
 include/uapi/linux/btrfs_tree.h                    |     2 +
 include/uapi/linux/can/netlink.h                   |     1 +
 include/uapi/linux/devlink.h                       |    25 +
 include/uapi/linux/dvb/dmx.h                       |    98 +-
 include/uapi/linux/dvb/frontend.h                  |    40 +-
 include/uapi/linux/dvb/version.h                   |     2 +-
 include/uapi/linux/dvb/video.h                     |    20 +-
 include/uapi/linux/elf.h                           |     1 +
 include/uapi/linux/erspan.h                        |    52 +
 include/uapi/linux/ethtool.h                       |     1 +
 include/uapi/linux/eventpoll.h                     |    33 +-
 include/uapi/linux/fs.h                            |     6 +-
 include/uapi/linux/gfs2_ondisk.h                   |    62 +-
 include/uapi/linux/i2c.h                           |     3 +
 include/uapi/linux/if_ether.h                      |     8 +-
 include/uapi/linux/if_link.h                       |     5 +
 include/uapi/linux/if_macsec.h                     |     9 +-
 include/uapi/linux/if_tun.h                        |     2 +
 include/uapi/linux/if_tunnel.h                     |     3 +
 include/uapi/linux/inet_diag.h                     |     2 +
 include/uapi/linux/input-event-codes.h             |     1 +
 include/uapi/linux/input.h                         |    11 +
 include/uapi/linux/kfd_ioctl.h                     |    15 +-
 include/uapi/linux/kvm.h                           |    92 +
 include/uapi/linux/l2tp.h                          |     6 +-
 include/uapi/linux/libc-compat.h                   |     6 -
 include/uapi/linux/lightnvm.h                      |     9 +
 include/uapi/linux/lirc.h                          |    81 +
 include/uapi/linux/lp.h                            |    12 +-
 include/uapi/linux/membarrier.h                    |    74 +-
 include/uapi/linux/ndctl.h                         |    56 +-
 include/uapi/linux/netfilter/nf_conntrack_common.h |     6 +-
 include/uapi/linux/netfilter/nf_tables.h           |    76 +
 include/uapi/linux/netfilter/xt_connlimit.h        |     2 +-
 include/uapi/linux/netfilter_arp.h                 |     3 +
 include/uapi/linux/netfilter_decnet.h              |     4 +-
 include/uapi/linux/netfilter_ipv4.h                |     1 +
 include/uapi/linux/netfilter_ipv6.h                |     1 +
 include/uapi/linux/netfilter_ipv6/ip6t_srh.h       |    57 +
 include/uapi/linux/nfs.h                           |     1 +
 include/uapi/linux/nl80211.h                       |     4 +
 include/uapi/linux/nubus.h                         |    23 -
 include/uapi/linux/openvswitch.h                   |     1 +
 include/uapi/linux/pci_regs.h                      |    30 +-
 include/uapi/linux/perf_event.h                    |    32 +-
 include/uapi/linux/psci.h                          |     3 +
 include/uapi/linux/psp-sev.h                       |   142 +
 include/uapi/linux/ptrace.h                        |     6 +
 include/uapi/linux/rtnetlink.h                     |    12 +
 include/uapi/linux/sched.h                         |     5 +
 include/uapi/linux/sctp.h                          |     3 +
 include/uapi/linux/switchtec_ioctl.h               |     3 +-
 include/uapi/linux/tee.h                           |    37 +
 include/uapi/linux/tipc.h                          |     7 +-
 include/uapi/linux/types.h                         |     6 +
 include/uapi/linux/usb/audio.h                     |     4 +-
 include/uapi/linux/usb/ch9.h                       |     4 +-
 include/uapi/linux/usbdevice_fs.h                  |     2 +-
 include/uapi/linux/uuid.h                          |     1 -
 include/uapi/linux/uvcvideo.h                      |    26 +
 include/uapi/linux/v4l2-controls.h                 |    96 +-
 include/uapi/linux/vbox_err.h                      |   151 +
 include/uapi/linux/vbox_vmmdev_types.h             |   226 +
 include/uapi/linux/vboxguest.h                     |   330 +
 include/uapi/linux/vfio.h                          |    72 +
 include/uapi/linux/videodev2.h                     |    63 +-
 include/uapi/linux/virtio_balloon.h                |     3 +-
 include/uapi/linux/virtio_net.h                    |    13 +
 include/uapi/misc/cxl.h                            |    10 +-
 include/uapi/misc/ocxl.h                           |    66 +
 include/uapi/rdma/bnxt_re-abi.h                    |     9 +
 include/uapi/rdma/ib_user_verbs.h                  |    11 +-
 include/uapi/rdma/mlx4-abi.h                       |     7 +-
 include/uapi/rdma/mlx5-abi.h                       |    53 +-
 include/uapi/rdma/rdma_netlink.h                   |    61 +-
 include/uapi/rdma/rdma_user_ioctl.h                |     4 +-
 include/uapi/rdma/vmw_pvrdma-abi.h                 |    12 +-
 include/uapi/sound/asound.h                        |     9 +
 include/uapi/sound/snd_sst_tokens.h                |    17 +-
 include/video/imx-ipu-v3.h                         |     2 +-
 include/video/udlfb.h                              |     3 +-
 init/Kconfig                                       |     9 +
 init/Makefile                                      |     2 -
 init/init_task.c                                   |   170 +-
 init/main.c                                        |     2 +
 ipc/mqueue.c                                       |   182 +-
 ipc/msg.c                                          |    20 +-
 ipc/sem.c                                          |    10 +
 ipc/shm.c                                          |    32 +-
 ipc/util.c                                         |     5 +-
 kernel/Makefile                                    |     1 +
 kernel/async.c                                     |    20 +-
 kernel/bpf/Makefile                                |     2 +
 kernel/bpf/arraymap.c                              |    78 +-
 kernel/bpf/cgroup.c                                |    15 +-
 kernel/bpf/core.c                                  |   441 +-
 kernel/bpf/cpumap.c                                |    33 +-
 kernel/bpf/devmap.c                                |     8 +-
 kernel/bpf/disasm.c                                |    63 +-
 kernel/bpf/disasm.h                                |    29 +-
 kernel/bpf/hashtab.c                               |   103 +-
 kernel/bpf/inode.c                                 |    50 +-
 kernel/bpf/lpm_trie.c                              |   112 +-
 kernel/bpf/offload.c                               |   430 +-
 kernel/bpf/sockmap.c                               |   200 +-
 kernel/bpf/stackmap.c                              |    34 +-
 kernel/bpf/syscall.c                               |   216 +-
 kernel/bpf/verifier.c                              |  1493 +-
 kernel/cgroup/cgroup.c                             |    16 +-
 kernel/cgroup/cpuset.c                             |     4 +-
 kernel/compat.c                                    |    21 +-
 kernel/configs/kvm_guest.config                    |     1 +
 kernel/configs/nopm.config                         |    15 +
 kernel/configs/tiny.config                         |     4 +
 kernel/debug/kdb/kdb_main.c                        |    10 +-
 kernel/debug/kdb/kdb_private.h                     |     2 +-
 kernel/events/callchain.c                          |    15 -
 kernel/events/core.c                               |    94 +-
 kernel/events/hw_breakpoint.c                      |    30 +-
 kernel/events/internal.h                           |     4 -
 kernel/events/ring_buffer.c                        |     2 +-
 kernel/events/uprobes.c                            |    12 +-
 kernel/extable.c                                   |     2 +-
 kernel/fail_function.c                             |   359 +
 kernel/fork.c                                      |   528 +-
 kernel/futex.c                                     |    20 +-
 kernel/irq/Kconfig                                 |    10 -
 kernel/irq/affinity.c                              |    30 +-
 kernel/irq/autoprobe.c                             |     2 +-
 kernel/irq/chip.c                                  |     6 +-
 kernel/irq/debug.h                                 |    14 +-
 kernel/irq/internals.h                             |     2 +-
 kernel/irq/irqdomain.c                             |   136 +-
 kernel/irq/matrix.c                                |    23 +-
 kernel/irq/spurious.c                              |     1 -
 kernel/irq_work.c                                  |     2 +-
 kernel/jump_label.c                                |    29 +-
 kernel/kallsyms.c                                  |    46 -
 kernel/kcov.c                                      |     4 +-
 kernel/kprobes.c                                   |   178 +-
 kernel/livepatch/core.c                            |    76 +-
 kernel/livepatch/transition.c                      |   116 +-
 kernel/livepatch/transition.h                      |     2 +
 kernel/locking/lockdep.c                           |    89 +-
 kernel/locking/locktorture.c                       |   108 +-
 kernel/locking/mutex.c                             |    37 +-
 kernel/locking/qspinlock.c                         |    25 +-
 kernel/locking/rtmutex.c                           |     5 +-
 kernel/memremap.c                                  |   186 +-
 kernel/module.c                                    |    32 +-
 kernel/padata.c                                    |     1 +
 kernel/panic.c                                     |     2 +-
 kernel/pid.c                                       |    27 +-
 kernel/power/main.c                                |    29 +
 kernel/power/power.h                               |     3 -
 kernel/power/snapshot.c                            |     6 +-
 kernel/power/swap.c                                |     6 +-
 kernel/printk/printk.c                             |   226 +-
 kernel/ptrace.c                                    |    36 +-
 kernel/rcu/rcu.h                                   |    27 +-
 kernel/rcu/rcuperf.c                               |     6 +-
 kernel/rcu/rcutorture.c                            |    12 +-
 kernel/rcu/srcutree.c                              |   109 +-
 kernel/rcu/tree.c                                  |   355 +-
 kernel/rcu/tree.h                                  |     5 +-
 kernel/rcu/tree_plugin.h                           |    13 +-
 kernel/rcu/update.c                                |     2 +
 kernel/relay.c                                     |    11 +-
 kernel/resource.c                                  |    39 +-
 kernel/sched/autogroup.c                           |     5 +-
 kernel/sched/core.c                                |   182 +-
 kernel/sched/cpufreq_schedutil.c                   |    95 +-
 kernel/sched/deadline.c                            |   149 +-
 kernel/sched/debug.c                               |    29 +-
 kernel/sched/fair.c                                |   140 +-
 kernel/sched/membarrier.c                          |   177 +-
 kernel/sched/rt.c                                  |    32 +-
 kernel/sched/sched.h                               |   114 +-
 kernel/sched/stats.h                               |     6 +
 kernel/sched/topology.c                            |    13 +
 kernel/seccomp.c                                   |   110 +-
 kernel/signal.c                                    |   354 +-
 kernel/softirq.c                                   |     2 +-
 kernel/sys.c                                       |     2 +-
 kernel/sysctl.c                                    |    40 +-
 kernel/taskstats.c                                 |     6 +-
 kernel/time/hrtimer.c                              |   658 +-
 kernel/time/posix-clock.c                          |     8 +-
 kernel/time/posix-cpu-timers.c                     |    25 +-
 kernel/time/posix-timers.c                         |    13 +-
 kernel/time/tick-internal.h                        |    13 +-
 kernel/time/tick-sched.c                           |     2 +-
 kernel/time/timer.c                                |    96 +-
 kernel/torture.c                                   |    39 +-
 kernel/trace/Kconfig                               |     9 +
 kernel/trace/bpf_trace.c                           |   129 +-
 kernel/trace/ftrace.c                              |     3 -
 kernel/trace/ring_buffer.c                         |     6 +-
 kernel/trace/trace.c                               |    35 +-
 kernel/trace/trace_benchmark.c                     |     2 +-
 kernel/trace/trace_events.c                        |     2 -
 kernel/trace/trace_events_filter.c                 |     9 +-
 kernel/trace/trace_kprobe.c                        |    65 +-
 kernel/trace/trace_probe.c                         |     8 +-
 kernel/trace/trace_probe.h                         |    14 +-
 kernel/trace/trace_selftest_dynamic.c              |     5 +-
 kernel/trace/trace_uprobe.c                        |     2 +-
 kernel/tracepoint.c                                |     9 +-
 kernel/user.c                                      |     3 +
 kernel/workqueue.c                                 |    93 +-
 lib/Kconfig                                        |     6 +-
 lib/Kconfig.debug                                  |    29 +-
 lib/Kconfig.kasan                                  |    11 +
 lib/Makefile                                       |     7 +-
 lib/assoc_array.c                                  |    37 +-
 lib/bitmap.c                                       |   137 +-
 lib/btree.c                                        |    10 +-
 lib/bucket_locks.c                                 |    54 +
 lib/bug.c                                          |     4 +-
 lib/chacha20.c                                     |    71 +-
 lib/cpumask.c                                      |     9 +-
 lib/crc-ccitt.c                                    |    58 +-
 lib/dma-debug.c                                    |    10 +-
 lib/dma-direct.c                                   |   161 +
 lib/dma-noop.c                                     |    68 -
 lib/error-inject.c                                 |   242 +
 lib/errseq.c                                       |    37 +-
 lib/find_bit.c                                     |    59 +-
 lib/{test_find_bit.c => find_bit_benchmark.c}      |    70 +-
 lib/idr.c                                          |   258 +-
 lib/ioremap.c                                      |     6 +-
 lib/kobject.c                                      |     6 +-
 lib/kobject_uevent.c                               |     3 +-
 lib/pci_iomap.c                                    |     1 +
 lib/percpu-refcount.c                              |    10 +-
 lib/radix-tree.c                                   |     5 +-
 lib/rhashtable.c                                   |   164 +-
 lib/sbitmap.c                                      |     2 +-
 lib/scatterlist.c                                  |   127 +
 lib/smp_processor_id.c                             |     3 +-
 lib/stackdepot.c                                   |    19 +-
 lib/string.c                                       |     2 +-
 lib/swiotlb.c                                      |   205 +-
 lib/test_bitmap.c                                  |   295 +-
 lib/test_bpf.c                                     |   147 +-
 lib/test_firmware.c                                |    17 +-
 lib/test_kasan.c                                   |   107 +-
 lib/test_kmod.c                                    |    16 +-
 lib/test_rhashtable.c                              |   140 +-
 lib/test_sort.c                                    |     6 +
 lib/ubsan.c                                        |    64 +-
 lib/ubsan.h                                        |    17 +-
 lib/usercopy.c                                     |     2 +-
 lib/uuid.c                                         |    34 +-
 lib/vsprintf.c                                     |    94 +-
 mm/Kconfig                                         |     7 +-
 mm/bootmem.c                                       |     2 +-
 mm/compaction.c                                    |     2 +-
 mm/fadvise.c                                       |    10 +-
 mm/filemap.c                                       |     1 -
 mm/gup.c                                           |    60 +-
 mm/hmm.c                                           |    17 +-
 mm/huge_memory.c                                   |    91 +-
 mm/hugetlb.c                                       |   384 +-
 mm/hwpoison-inject.c                               |     2 +-
 mm/interval_tree.c                                 |     2 +-
 mm/kasan/kasan.c                                   |   108 +-
 mm/kasan/kasan.h                                   |    55 +-
 mm/kasan/report.c                                  |    11 +-
 mm/khugepaged.c                                    |    30 +-
 mm/kmemleak.c                                      |    13 +-
 mm/ksm.c                                           |    11 +-
 mm/maccess.c                                       |     2 +-
 mm/madvise.c                                       |     2 +-
 mm/memblock.c                                      |    34 +-
 mm/memcontrol.c                                    |   303 +-
 mm/memory-failure.c                                |    50 +-
 mm/memory.c                                        |   102 +-
 mm/memory_hotplug.c                                |    48 +-
 mm/mempolicy.c                                     |    42 +-
 mm/mempool.c                                       |     6 +-
 mm/migrate.c                                       |     3 +-
 mm/mlock.c                                         |    10 +-
 mm/mmu_notifier.c                                  |    31 +
 mm/mprotect.c                                      |     5 +
 mm/nommu.c                                         |     9 +-
 mm/oom_kill.c                                      |    21 +-
 mm/page_alloc.c                                    |   201 +-
 mm/page_ext.c                                      |     2 +
 mm/page_io.c                                       |     4 +-
 mm/page_owner.c                                    |    26 +-
 mm/pagewalk.c                                      |     1 +
 mm/percpu-km.c                                     |     8 +-
 mm/percpu-vm.c                                     |    18 +-
 mm/percpu.c                                        |    67 +-
 mm/pgtable-generic.c                               |     6 +-
 mm/process_vm_access.c                             |    10 +-
 mm/shmem.c                                         |    90 +-
 mm/slab.c                                          |    47 +-
 mm/slab.h                                          |    11 +-
 mm/slab_common.c                                   |   118 +-
 mm/slub.c                                          |    71 +-
 mm/sparse-vmemmap.c                                |    67 +-
 mm/sparse.c                                        |    49 +-
 mm/swap.c                                          |   113 +-
 mm/swapfile.c                                      |     6 +-
 mm/truncate.c                                      |    23 +-
 mm/usercopy.c                                      |   133 +-
 mm/userfaultfd.c                                   |     1 -
 mm/util.c                                          |    36 +-
 mm/vmalloc.c                                       |    10 +-
 mm/vmscan.c                                        |   177 +-
 mm/vmstat.c                                        |     2 +
 mm/z3fold.c                                        |     4 +-
 mm/zbud.c                                          |     4 +-
 mm/zpool.c                                         |    71 +-
 mm/zsmalloc.c                                      |    31 +-
 mm/zswap.c                                         |    97 +-
 net/8021q/vlan_core.c                              |     4 +-
 net/8021q/vlanproc.c                               |     2 -
 net/9p/trans_fd.c                                  |    82 +-
 net/9p/trans_virtio.c                              |     3 +-
 net/Kconfig                                        |    19 +-
 net/Makefile                                       |     1 -
 net/appletalk/aarp.c                               |     1 -
 net/appletalk/atalk_proc.c                         |     3 -
 net/atm/br2684.c                                   |     1 -
 net/atm/common.c                                   |    14 +-
 net/atm/common.h                                   |     2 +-
 net/atm/lec.c                                      |     1 -
 net/atm/mpc.c                                      |     9 +-
 net/atm/mpoa_caches.c                              |    48 +-
 net/atm/mpoa_caches.h                              |     9 +-
 net/atm/mpoa_proc.c                                |    16 +-
 net/atm/proc.c                                     |     1 -
 net/ax25/af_ax25.c                                 |     1 -
 net/ax25/ax25_route.c                              |     1 -
 net/ax25/ax25_uid.c                                |     1 -
 net/batman-adv/Kconfig                             |    17 +
 net/batman-adv/Makefile                            |     2 +-
 net/batman-adv/bat_algo.c                          |    35 +-
 net/batman-adv/bat_algo.h                          |     1 +
 net/batman-adv/bat_iv_ogm.c                        |   133 +-
 net/batman-adv/bat_iv_ogm.h                        |     1 +
 net/batman-adv/bat_v.c                             |    53 +-
 net/batman-adv/bat_v.h                             |     1 +
 net/batman-adv/bat_v_elp.c                         |    29 +-
 net/batman-adv/bat_v_elp.h                         |     1 +
 net/batman-adv/bat_v_ogm.c                         |    39 +-
 net/batman-adv/bat_v_ogm.h                         |     1 +
 net/batman-adv/bitarray.c                          |     3 +-
 net/batman-adv/bitarray.h                          |    10 +-
 net/batman-adv/bridge_loop_avoidance.c             |   136 +-
 net/batman-adv/bridge_loop_avoidance.h             |     5 +-
 net/batman-adv/debugfs.c                           |    30 +-
 net/batman-adv/debugfs.h                           |     1 +
 net/batman-adv/distributed-arp-table.c             |    82 +-
 net/batman-adv/distributed-arp-table.h             |     9 +-
 net/batman-adv/fragmentation.c                     |    28 +-
 net/batman-adv/fragmentation.h                     |     3 +-
 net/batman-adv/gateway_client.c                    |    72 +-
 net/batman-adv/gateway_client.h                    |     1 +
 net/batman-adv/gateway_common.c                    |    30 +-
 net/batman-adv/gateway_common.h                    |     6 +-
 net/batman-adv/hard-interface.c                    |    76 +-
 net/batman-adv/hard-interface.h                    |    59 +-
 net/batman-adv/hash.c                              |    20 +-
 net/batman-adv/hash.h                              |    28 +-
 net/batman-adv/icmp_socket.c                       |    22 +-
 net/batman-adv/icmp_socket.h                       |     1 +
 net/batman-adv/log.c                               |    24 +-
 net/batman-adv/log.h                               |    62 +-
 net/batman-adv/main.c                              |    54 +-
 net/batman-adv/main.h                              |   127 +-
 net/batman-adv/multicast.c                         |    91 +-
 net/batman-adv/multicast.h                         |    17 +-
 net/batman-adv/netlink.c                           |    27 +-
 net/batman-adv/netlink.h                           |     1 +
 net/batman-adv/network-coding.c                    |   126 +-
 net/batman-adv/network-coding.h                    |     1 +
 net/batman-adv/originator.c                        |   158 +-
 net/batman-adv/originator.h                        |    51 +-
 net/batman-adv/routing.c                           |    81 +-
 net/batman-adv/routing.h                           |     1 +
 net/batman-adv/send.c                              |    66 +-
 net/batman-adv/send.h                              |     8 +-
 net/batman-adv/soft-interface.c                    |    74 +-
 net/batman-adv/soft-interface.h                    |     1 +
 net/batman-adv/sysfs.c                             |    58 +-
 net/batman-adv/sysfs.h                             |    14 +
 net/batman-adv/tp_meter.c                          |    77 +-
 net/batman-adv/tp_meter.h                          |     1 +
 net/batman-adv/translation-table.c                 |   234 +-
 net/batman-adv/translation-table.h                 |     1 +
 net/batman-adv/tvlv.c                              |    43 +-
 net/batman-adv/tvlv.h                              |     1 +
 net/batman-adv/types.h                             |  2007 +-
 net/bluetooth/af_bluetooth.c                       |    60 +-
 net/bluetooth/cmtp/capi.c                          |     1 -
 net/bluetooth/hci_debugfs.c                        |   210 +-
 net/bluetooth/hci_request.c                        |    64 +-
 net/bluetooth/hidp/core.c                          |     2 +-
 net/bluetooth/smp.c                                |     8 +-
 net/bpf/test_run.c                                 |     4 +
 net/bridge/br_device.c                             |    10 +
 net/bridge/br_fdb.c                                |   394 +-
 net/bridge/br_mdb.c                                |     6 +-
 net/bridge/br_netfilter_hooks.c                    |     6 +-
 net/bridge/br_nf_core.c                            |     1 -
 net/bridge/br_private.h                            |    18 +-
 net/bridge/br_switchdev.c                          |     8 +-
 net/bridge/br_sysfs_br.c                           |    13 +-
 net/bridge/br_sysfs_if.c                           |     3 +
 net/bridge/br_vlan.c                               |     2 +
 net/bridge/netfilter/Kconfig                       |     2 +
 net/bridge/netfilter/ebt_among.c                   |    65 +-
 net/bridge/netfilter/ebt_limit.c                   |     4 +-
 net/bridge/netfilter/ebtables.c                    |    46 +-
 net/bridge/netfilter/nf_tables_bridge.c            |   120 +-
 net/caif/caif_socket.c                             |    18 +-
 net/caif/cfctrl.c                                  |    50 +-
 net/caif/cfpkt_skbuff.c                            |     1 -
 net/caif/chnl_net.c                                |     1 -
 net/can/Kconfig                                    |     2 +-
 net/can/af_can.c                                   |    20 +-
 net/can/af_can.h                                   |     2 +-
 net/can/bcm.c                                      |     1 -
 net/can/gw.c                                       |    14 +-
 net/can/proc.c                                     |    14 +-
 net/can/raw.c                                      |     2 +
 net/ceph/ceph_common.c                             |    11 +
 net/core/Makefile                                  |     2 +-
 net/core/datagram.c                                |    22 +-
 net/core/dev.c                                     |   309 +-
 net/core/dev_ioctl.c                               |   135 +-
 net/core/devlink.c                                 |   618 +-
 net/core/dst.c                                     |    14 +-
 net/core/ethtool.c                                 |    72 +-
 net/core/filter.c                                  |   415 +-
 net/core/flow_dissector.c                          |    69 +-
 net/core/gen_estimator.c                           |     5 +
 net/core/gen_stats.c                               |     9 +-
 net/core/link_watch.c                              |     2 +-
 net/core/neighbour.c                               |     1 -
 net/core/net-procfs.c                              |     4 -
 net/core/net-sysfs.c                               |    56 +-
 net/core/net_namespace.c                           |    83 +-
 net/core/pktgen.c                                  |   281 +-
 net/core/rtnetlink.c                               |   527 +-
 net/core/skbuff.c                                  |   130 +-
 net/core/sock.c                                    |    98 +-
 net/core/sock_diag.c                               |    12 +-
 net/core/sock_reuseport.c                          |    39 +-
 net/core/stream.c                                  |     4 +-
 net/core/sysctl_net_core.c                         |    60 +-
 net/core/xdp.c                                     |    73 +
 net/dccp/Kconfig                                   |    17 -
 net/dccp/Makefile                                  |     5 +-
 net/dccp/ackvec.c                                  |     2 +-
 net/dccp/dccp.h                                    |     2 +-
 net/dccp/minisocks.c                               |     7 +-
 net/dccp/probe.c                                   |   203 -
 net/dccp/proto.c                                   |    28 +-
 net/dccp/trace.h                                   |    84 +
 net/decnet/af_decnet.c                             |    69 +-
 net/decnet/dn_dev.c                                |    10 +-
 net/decnet/dn_fib.c                                |     6 +-
 net/decnet/dn_neigh.c                              |     1 -
 net/decnet/dn_route.c                              |    43 +-
 net/dns_resolver/dns_query.c                       |    22 +-
 net/dsa/Kconfig                                    |     9 +
 net/dsa/Makefile                                   |     3 +-
 net/dsa/dsa2.c                                     |     9 +-
 net/dsa/dsa_priv.h                                 |    13 +-
 net/dsa/legacy.c                                   |    26 +-
 net/dsa/port.c                                     |   103 +-
 net/dsa/slave.c                                    |    25 +-
 net/dsa/switch.c                                   |   111 +-
 net/dsa/tag_brcm.c                                 |    12 +
 net/dsa/tag_mtk.c                                  |    38 +-
 net/ieee802154/6lowpan/core.c                      |    12 +-
 net/ipv4/Makefile                                  |     1 -
 net/ipv4/af_inet.c                                 |    47 +-
 net/ipv4/arp.c                                     |     1 -
 net/ipv4/devinet.c                                 |    61 +-
 net/ipv4/esp4.c                                    |    36 +-
 net/ipv4/esp4_offload.c                            |    75 +-
 net/ipv4/fib_frontend.c                            |     8 +-
 net/ipv4/fib_semantics.c                           |     5 +
 net/ipv4/fib_trie.c                                |     3 -
 net/ipv4/igmp.c                                    |     6 +-
 net/ipv4/inet_connection_sock.c                    |     9 +-
 net/ipv4/inet_diag.c                               |    11 +-
 net/ipv4/inet_fragment.c                           |     3 +
 net/ipv4/inet_hashtables.c                         |   188 +-
 net/ipv4/inet_timewait_sock.c                      |    31 +-
 net/ipv4/ip_forward.c                              |     2 +-
 net/ipv4/ip_gre.c                                  |   170 +-
 net/ipv4/ip_output.c                               |     2 +-
 net/ipv4/ip_sockglue.c                             |    33 +-
 net/ipv4/ip_tunnel.c                               |    31 +-
 net/ipv4/ip_vti.c                                  |     2 -
 net/ipv4/ipconfig.c                                |    48 +-
 net/ipv4/ipmr.c                                    |     4 +-
 net/ipv4/netfilter.c                               |    62 +-
 net/ipv4/netfilter/Kconfig                         |    12 +
 net/ipv4/netfilter/Makefile                        |     9 +
 net/ipv4/netfilter/arp_tables.c                    |    38 +-
 net/ipv4/netfilter/ip_tables.c                     |    41 +-
 net/ipv4/netfilter/ipt_CLUSTERIP.c                 |    52 +-
 net/ipv4/netfilter/ipt_ECN.c                       |    12 +-
 net/ipv4/netfilter/ipt_REJECT.c                    |     4 +-
 net/ipv4/netfilter/ipt_rpfilter.c                  |     6 +-
 net/ipv4/netfilter/iptable_filter.c                |     6 -
 net/ipv4/netfilter/iptable_mangle.c                |     5 -
 net/ipv4/netfilter/iptable_nat.c                   |     4 +
 net/ipv4/netfilter/iptable_raw.c                   |    37 +-
 net/ipv4/netfilter/iptable_security.c              |     6 -
 net/ipv4/netfilter/nf_conntrack_l3proto_ipv4.c     |    27 +-
 net/ipv4/netfilter/nf_conntrack_proto_icmp.c       |     4 +-
 net/ipv4/netfilter/nf_defrag_ipv4.c                |     2 +
 net/ipv4/netfilter/nf_flow_table_ipv4.c            |   286 +
 net/ipv4/netfilter/nf_nat_l3proto_ipv4.c           |    10 -
 net/ipv4/netfilter/nf_nat_snmp_basic.asn1          |   177 +
 net/ipv4/netfilter/nf_nat_snmp_basic.c             |  1286 -
 net/ipv4/netfilter/nf_nat_snmp_basic_main.c        |   235 +
 net/ipv4/netfilter/nf_socket_ipv4.c                |     6 +-
 net/ipv4/netfilter/nf_tables_arp.c                 |    62 +-
 net/ipv4/netfilter/nf_tables_ipv4.c                |    83 +-
 net/ipv4/netfilter/nft_chain_nat_ipv4.c            |     3 +-
 net/ipv4/netfilter/nft_chain_route_ipv4.c          |     8 +-
 net/ipv4/proc.c                                    |     3 -
 net/ipv4/raw.c                                     |    18 +-
 net/ipv4/route.c                                   |    72 +-
 net/ipv4/syncookies.c                              |     2 +
 net/ipv4/tcp.c                                     |   112 +-
 net/ipv4/tcp_bbr.c                                 |     9 +-
 net/ipv4/tcp_diag.c                                |     2 +-
 net/ipv4/tcp_fastopen.c                            |    30 +-
 net/ipv4/tcp_illinois.c                            |     2 +-
 net/ipv4/tcp_input.c                               |    87 +-
 net/ipv4/tcp_ipv4.c                                |     8 +-
 net/ipv4/tcp_metrics.c                             |     7 +-
 net/ipv4/tcp_minisocks.c                           |     7 +-
 net/ipv4/tcp_nv.c                                  |     4 +-
 net/ipv4/tcp_output.c                              |    52 +-
 net/ipv4/tcp_probe.c                               |   301 -
 net/ipv4/tcp_timer.c                               |    25 +-
 net/ipv4/tcp_ulp.c                                 |    59 +-
 net/ipv4/udp.c                                     |    77 +-
 net/ipv4/udplite.c                                 |     1 -
 net/ipv4/xfrm4_mode_tunnel.c                       |    10 +-
 net/ipv4/xfrm4_output.c                            |     3 +-
 net/ipv4/xfrm4_policy.c                            |     5 +-
 net/ipv6/addrconf.c                                |    90 +-
 net/ipv6/addrlabel.c                               |    25 +-
 net/ipv6/af_inet6.c                                |    11 +-
 net/ipv6/anycast.c                                 |     1 -
 net/ipv6/datagram.c                                |    24 +-
 net/ipv6/esp6.c                                    |    36 +-
 net/ipv6/esp6_offload.c                            |    82 +-
 net/ipv6/ila/ila_xlat.c                            |     4 +-
 net/ipv6/inet6_hashtables.c                        |    77 +-
 net/ipv6/ip6_checksum.c                            |     5 +
 net/ipv6/ip6_fib.c                                 |    66 +-
 net/ipv6/ip6_flowlabel.c                           |     1 -
 net/ipv6/ip6_gre.c                                 |   637 +-
 net/ipv6/ip6_output.c                              |    30 +-
 net/ipv6/ip6_tunnel.c                              |    17 +-
 net/ipv6/ip6_vti.c                                 |    52 +-
 net/ipv6/ip6mr.c                                   |    12 +-
 net/ipv6/ipv6_sockglue.c                           |    27 +-
 net/ipv6/mcast.c                                   |    12 +-
 net/ipv6/ndisc.c                                   |     8 +-
 net/ipv6/netfilter.c                               |    53 +-
 net/ipv6/netfilter/Kconfig                         |    17 +
 net/ipv6/netfilter/Makefile                        |     4 +
 net/ipv6/netfilter/ip6_tables.c                    |    38 +-
 net/ipv6/netfilter/ip6t_REJECT.c                   |     4 +-
 net/ipv6/netfilter/ip6t_rpfilter.c                 |    10 +-
 net/ipv6/netfilter/ip6t_srh.c                      |   163 +
 net/ipv6/netfilter/ip6table_mangle.c               |     8 -
 net/ipv6/netfilter/ip6table_nat.c                  |     4 +
 net/ipv6/netfilter/ip6table_raw.c                  |    31 +-
 net/ipv6/netfilter/nf_conntrack_l3proto_ipv6.c     |    25 +-
 net/ipv6/netfilter/nf_conntrack_proto_icmpv6.c     |     4 +-
 net/ipv6/netfilter/nf_conntrack_reasm.c            |    16 +-
 net/ipv6/netfilter/nf_defrag_ipv6_hooks.c          |     3 +
 net/ipv6/netfilter/nf_flow_table_ipv6.c            |   278 +
 net/ipv6/netfilter/nf_nat_l3proto_ipv6.c           |    12 +-
 net/ipv6/netfilter/nf_socket_ipv6.c                |     6 +-
 net/ipv6/netfilter/nf_tables_ipv6.c                |    82 +-
 net/ipv6/netfilter/nft_chain_nat_ipv6.c            |     3 +-
 net/ipv6/netfilter/nft_chain_route_ipv6.c          |     3 +-
 net/ipv6/netfilter/nft_fib_ipv6.c                  |    24 +-
 net/ipv6/proc.c                                    |     3 -
 net/ipv6/raw.c                                     |     3 +-
 net/ipv6/route.c                                   |   661 +-
 net/ipv6/seg6.c                                    |     4 +-
 net/ipv6/seg6_iptunnel.c                           |    23 +-
 net/ipv6/seg6_local.c                              |     2 +-
 net/ipv6/sit.c                                     |     9 +-
 net/ipv6/syncookies.c                              |     2 +
 net/ipv6/tcp_ipv6.c                                |     9 +-
 net/ipv6/udp.c                                     |    55 +-
 net/ipv6/udplite.c                                 |     1 -
 net/ipv6/xfrm6_mode_tunnel.c                       |    10 +-
 net/ipv6/xfrm6_output.c                            |     2 +-
 net/ipv6/xfrm6_policy.c                            |     7 +-
 net/iucv/af_iucv.c                                 |    28 +-
 net/kcm/kcmproc.c                                  |     2 -
 net/kcm/kcmsock.c                                  |    39 +-
 net/l2tp/l2tp_core.c                               |   242 +-
 net/l2tp/l2tp_core.h                               |    42 +-
 net/l2tp/l2tp_debugfs.c                            |     4 +-
 net/l2tp/l2tp_ip.c                                 |    10 +-
 net/l2tp/l2tp_ip6.c                                |     8 +-
 net/l2tp/l2tp_netlink.c                            |    39 +-
 net/l2tp/l2tp_ppp.c                                |    61 +-
 net/llc/llc_c_ac.c                                 |    15 +-
 net/llc/llc_conn.c                                 |    32 +-
 net/llc/llc_proc.c                                 |     2 -
 net/mac80211/agg-rx.c                              |    30 +-
 net/mac80211/agg-tx.c                              |    34 +-
 net/mac80211/cfg.c                                 |    33 +-
 net/mac80211/debugfs.c                             |     2 +
 net/mac80211/debugfs_sta.c                         |     4 +-
 net/mac80211/driver-ops.h                          |     3 +-
 net/mac80211/ht.c                                  |     1 +
 net/mac80211/ieee80211_i.h                         |     6 +-
 net/mac80211/iface.c                               |     4 +-
 net/mac80211/key.c                                 |    12 +-
 net/mac80211/main.c                                |     3 +
 net/mac80211/mesh.c                                |    19 +-
 net/mac80211/mesh_hwmp.c                           |     1 +
 net/mac80211/mesh_pathtbl.c                        |    34 +-
 net/mac80211/mesh_plink.c                          |     2 +-
 net/mac80211/mlme.c                                |    13 +-
 net/mac80211/offchannel.c                          |     4 +-
 net/mac80211/rx.c                                  |    19 +-
 net/mac80211/spectmgmt.c                           |     7 +-
 net/mac80211/sta_info.c                            |     3 +-
 net/mac80211/tdls.c                                |     6 +-
 net/mac80211/tx.c                                  |    12 +-
 net/mac80211/util.c                                |    19 +-
 net/mac80211/wme.c                                 |     1 +
 net/mac80211/wpa.c                                 |    16 +-
 net/mpls/af_mpls.c                                 |    41 +-
 net/ncsi/ncsi-aen.c                                |    35 +-
 net/netfilter/Kconfig                              |    35 +
 net/netfilter/Makefile                             |     9 +-
 net/netfilter/core.c                               |   263 +-
 net/netfilter/ipset/ip_set_bitmap_gen.h            |    10 +-
 net/netfilter/ipset/ip_set_bitmap_ip.c             |     8 +-
 net/netfilter/ipset/ip_set_bitmap_ipmac.c          |     8 +-
 net/netfilter/ipset/ip_set_bitmap_port.c           |     8 +-
 net/netfilter/ipset/ip_set_core.c                  |    36 +-
 net/netfilter/ipset/ip_set_hash_gen.h              |    38 +-
 net/netfilter/ipset/ip_set_hash_ipportnet.c        |    26 +-
 net/netfilter/ipset/ip_set_hash_net.c              |     9 +-
 net/netfilter/ipset/ip_set_hash_netiface.c         |     9 +-
 net/netfilter/ipset/ip_set_hash_netnet.c           |    28 +-
 net/netfilter/ipset/ip_set_hash_netport.c          |    19 +-
 net/netfilter/ipset/ip_set_hash_netportnet.c       |    35 +-
 net/netfilter/ipset/ip_set_list_set.c              |    21 +-
 net/netfilter/ipvs/ip_vs_app.c                     |     1 -
 net/netfilter/ipvs/ip_vs_conn.c                    |     4 +-
 net/netfilter/ipvs/ip_vs_core.c                    |    12 +-
 net/netfilter/ipvs/ip_vs_ctl.c                     |     3 -
 net/netfilter/ipvs/ip_vs_ftp.c                     |     2 +-
 net/netfilter/ipvs/ip_vs_proto_tcp.c               |     1 +
 net/netfilter/ipvs/ip_vs_proto_udp.c               |     1 +
 net/netfilter/ipvs/ip_vs_sync.c                    |     9 +-
 net/netfilter/nf_conncount.c                       |   373 +
 net/netfilter/nf_conntrack_core.c                  |    28 +-
 net/netfilter/nf_conntrack_expect.c                |     1 -
 net/netfilter/nf_conntrack_h323_asn1.c             |    40 +-
 net/netfilter/nf_conntrack_h323_main.c             |    77 +-
 net/netfilter/nf_conntrack_netlink.c               |    22 +-
 net/netfilter/nf_conntrack_proto.c                 |    18 +-
 net/netfilter/nf_conntrack_proto_dccp.c            |    21 +-
 net/netfilter/nf_conntrack_proto_generic.c         |     4 +-
 net/netfilter/nf_conntrack_proto_gre.c             |     4 +-
 net/netfilter/nf_conntrack_proto_sctp.c            |    21 +-
 net/netfilter/nf_conntrack_proto_tcp.c             |    25 +-
 net/netfilter/nf_conntrack_proto_udp.c             |    10 +-
 net/netfilter/nf_conntrack_standalone.c            |    14 +-
 net/netfilter/nf_flow_table.c                      |   453 +
 net/netfilter/nf_flow_table_inet.c                 |    49 +
 net/netfilter/nf_internals.h                       |     2 +-
 net/netfilter/nf_log.c                             |     1 -
 net/netfilter/nf_nat_proto_common.c                |     7 +-
 net/netfilter/nf_queue.c                           |    96 +-
 net/netfilter/nf_synproxy_core.c                   |     1 -
 net/netfilter/nf_tables_api.c                      |  1741 +-
 net/netfilter/nf_tables_inet.c                     |    88 +-
 net/netfilter/nf_tables_netdev.c                   |    87 +-
 net/netfilter/nfnetlink.c                          |     4 -
 net/netfilter/nfnetlink_acct.c                     |     2 -
 net/netfilter/nfnetlink_cttimeout.c                |     2 -
 net/netfilter/nfnetlink_log.c                      |     1 -
 net/netfilter/nfnetlink_queue.c                    |    10 +-
 net/netfilter/nft_cmp.c                            |     2 +
 net/netfilter/nft_compat.c                         |    26 +-
 net/netfilter/nft_ct.c                             |    16 +-
 net/netfilter/nft_dynset.c                         |     4 +-
 net/netfilter/nft_flow_offload.c                   |   242 +
 net/netfilter/nft_log.c                            |     4 +-
 net/netfilter/nft_masq.c                           |     2 +-
 net/netfilter/nft_meta.c                           |    45 +-
 net/netfilter/nft_nat.c                            |     2 +-
 net/netfilter/nft_redir.c                          |     2 +-
 net/netfilter/nft_rt.c                             |    15 +-
 net/netfilter/nft_set_hash.c                       |    12 +-
 net/netfilter/utils.c                              |    90 +
 net/netfilter/x_tables.c                           |   159 +-
 net/netfilter/xt_AUDIT.c                           |     4 +-
 net/netfilter/xt_CHECKSUM.c                        |     8 +-
 net/netfilter/xt_CONNSECMARK.c                     |    10 +-
 net/netfilter/xt_CT.c                              |    25 +-
 net/netfilter/xt_DSCP.c                            |     4 +-
 net/netfilter/xt_HL.c                              |    13 +-
 net/netfilter/xt_HMARK.c                           |    27 +-
 net/netfilter/xt_IDLETIMER.c                       |    10 +-
 net/netfilter/xt_LED.c                             |    17 +-
 net/netfilter/xt_NFQUEUE.c                         |     8 +-
 net/netfilter/xt_RATEEST.c                         |    22 +-
 net/netfilter/xt_SECMARK.c                         |    18 +-
 net/netfilter/xt_TCPMSS.c                          |    15 +-
 net/netfilter/xt_TPROXY.c                          |     6 +-
 net/netfilter/xt_addrtype.c                        |    48 +-
 net/netfilter/xt_bpf.c                             |     4 +-
 net/netfilter/xt_cgroup.c                          |     9 +-
 net/netfilter/xt_cluster.c                         |     8 +-
 net/netfilter/xt_connbytes.c                       |     4 +-
 net/netfilter/xt_connlabel.c                       |     7 +-
 net/netfilter/xt_connlimit.c                       |   369 +-
 net/netfilter/xt_connmark.c                        |     8 +-
 net/netfilter/xt_conntrack.c                       |     4 +-
 net/netfilter/xt_dscp.c                            |     4 +-
 net/netfilter/xt_ecn.c                             |     4 +-
 net/netfilter/xt_hashlimit.c                       |    47 +-
 net/netfilter/xt_helper.c                          |     4 +-
 net/netfilter/xt_ipcomp.c                          |     4 +-
 net/netfilter/xt_ipvs.c                            |     3 +-
 net/netfilter/xt_l2tp.c                            |    22 +-
 net/netfilter/xt_limit.c                           |     7 +-
 net/netfilter/xt_nat.c                             |     5 +-
 net/netfilter/xt_nfacct.c                          |     7 +-
 net/netfilter/xt_physdev.c                         |     4 +-
 net/netfilter/xt_policy.c                          |    26 +-
 net/netfilter/xt_recent.c                          |    20 +-
 net/netfilter/xt_set.c                             |   169 +-
 net/netfilter/xt_socket.c                          |    10 +-
 net/netfilter/xt_state.c                           |     4 +-
 net/netfilter/xt_statistic.c                       |     1 +
 net/netfilter/xt_time.c                            |     6 +-
 net/netlink/af_netlink.c                           |    79 +-
 net/netlink/diag.c                                 |     8 +-
 net/netlink/genetlink.c                            |    12 +-
 net/netrom/af_netrom.c                             |     1 -
 net/netrom/nr_route.c                              |     2 -
 net/nfc/llcp_commands.c                            |     4 +
 net/nfc/llcp_sock.c                                |    22 +-
 net/nfc/nci/uart.c                                 |     2 +-
 net/nfc/netlink.c                                  |     3 +-
 net/openvswitch/conntrack.c                        |    38 +-
 net/openvswitch/flow.c                             |     6 +-
 net/openvswitch/flow_netlink.c                     |    56 +-
 net/openvswitch/meter.c                            |    14 +-
 net/openvswitch/vport-internal_dev.c               |    10 +-
 net/packet/af_packet.c                             |    12 +-
 net/phonet/pn_netlink.c                            |    21 +-
 net/phonet/socket.c                                |    16 +-
 net/qrtr/qrtr.c                                    |     8 +-
 net/qrtr/smd.c                                     |     1 +
 net/rds/af_rds.c                                   |    20 +-
 net/rds/bind.c                                     |     1 +
 net/rds/cong.c                                     |    10 +-
 net/rds/connection.c                               |    32 +-
 net/rds/ib.c                                       |    26 +-
 net/rds/ib_cm.c                                    |     1 +
 net/rds/rds.h                                      |    17 +-
 net/rds/send.c                                     |    37 +-
 net/rds/tcp.c                                      |   113 +-
 net/rds/tcp.h                                      |     1 +
 net/rds/tcp_connect.c                              |     2 +-
 net/rds/tcp_listen.c                               |    14 +-
 net/rds/tcp_recv.c                                 |     8 +-
 net/rds/tcp_send.c                                 |     5 +-
 net/rds/threads.c                                  |    20 +-
 net/rfkill/core.c                                  |     6 +-
 net/rose/af_rose.c                                 |     1 -
 net/rose/rose_route.c                              |     3 -
 net/rxrpc/af_rxrpc.c                               |     8 +-
 net/rxrpc/conn_client.c                            |     3 +-
 net/rxrpc/conn_event.c                             |     1 +
 net/rxrpc/conn_object.c                            |    16 +-
 net/rxrpc/output.c                                 |     2 +-
 net/rxrpc/proc.c                                   |     2 -
 net/rxrpc/recvmsg.c                                |     5 +-
 net/rxrpc/rxkad.c                                  |    92 +-
 net/sched/Kconfig                                  |     3 -
 net/sched/act_api.c                                |    78 +-
 net/sched/act_bpf.c                                |    12 +-
 net/sched/act_connmark.c                           |     8 +-
 net/sched/act_csum.c                               |    77 +-
 net/sched/act_gact.c                               |     8 +-
 net/sched/act_ife.c                                |    18 +-
 net/sched/act_ipt.c                                |    27 +-
 net/sched/act_mirred.c                             |    27 +-
 net/sched/act_nat.c                                |     8 +-
 net/sched/act_pedit.c                              |    12 +-
 net/sched/act_police.c                             |    14 +-
 net/sched/act_sample.c                             |    13 +-
 net/sched/act_simple.c                             |    12 +-
 net/sched/act_skbedit.c                            |     8 +-
 net/sched/act_skbmod.c                             |    15 +-
 net/sched/act_tunnel_key.c                         |    20 +-
 net/sched/act_vlan.c                               |    15 +-
 net/sched/cls_api.c                                |   704 +-
 net/sched/cls_basic.c                              |    49 +-
 net/sched/cls_bpf.c                                |    88 +-
 net/sched/cls_cgroup.c                             |    12 +-
 net/sched/cls_flow.c                               |    12 +-
 net/sched/cls_flower.c                             |    88 +-
 net/sched/cls_fw.c                                 |    19 +-
 net/sched/cls_matchall.c                           |    35 +-
 net/sched/cls_route.c                              |    16 +-
 net/sched/cls_rsvp.h                               |     9 +-
 net/sched/cls_tcindex.c                            |    17 +-
 net/sched/cls_u32.c                                |   236 +-
 net/sched/sch_api.c                                |   305 +-
 net/sched/sch_atm.c                                |    23 +-
 net/sched/sch_cbq.c                                |    78 +-
 net/sched/sch_cbs.c                                |    31 +-
 net/sched/sch_choke.c                              |     8 +-
 net/sched/sch_codel.c                              |     8 +-
 net/sched/sch_drr.c                                |    40 +-
 net/sched/sch_dsmark.c                             |    19 +-
 net/sched/sch_fifo.c                               |    11 +-
 net/sched/sch_fq.c                                 |     8 +-
 net/sched/sch_fq_codel.c                           |    13 +-
 net/sched/sch_generic.c                            |   564 +-
 net/sched/sch_gred.c                               |    13 +-
 net/sched/sch_hfsc.c                               |    28 +-
 net/sched/sch_hhf.c                                |     8 +-
 net/sched/sch_htb.c                                |    29 +-
 net/sched/sch_ingress.c                            |    99 +-
 net/sched/sch_mq.c                                 |    42 +-
 net/sched/sch_mqprio.c                             |    76 +-
 net/sched/sch_multiq.c                             |    19 +-
 net/sched/sch_netem.c                              |    14 +-
 net/sched/sch_pie.c                                |     8 +-
 net/sched/sch_plug.c                               |     6 +-
 net/sched/sch_prio.c                               |    81 +-
 net/sched/sch_qfq.c                                |    22 +-
 net/sched/sch_red.c                                |    65 +-
 net/sched/sch_sfb.c                                |    20 +-
 net/sched/sch_sfq.c                                |     8 +-
 net/sched/sch_tbf.c                                |    34 +-
 net/sched/sch_teql.c                               |     3 +-
 net/sctp/Kconfig                                   |    12 -
 net/sctp/Makefile                                  |     5 +-
 net/sctp/associola.c                               |     2 +-
 net/sctp/chunk.c                                   |     8 +-
 net/sctp/debug.c                                   |     6 +
 net/sctp/endpointola.c                             |     2 +-
 net/sctp/input.c                                   |    13 +-
 net/sctp/inqueue.c                                 |     2 +-
 net/sctp/ipv6.c                                    |    10 +-
 net/sctp/offload.c                                 |     2 +-
 net/sctp/output.c                                  |     5 +-
 net/sctp/outqueue.c                                |    12 +-
 net/sctp/probe.c                                   |   244 -
 net/sctp/proc.c                                    |     7 +-
 net/sctp/protocol.c                                |    10 +-
 net/sctp/sm_make_chunk.c                           |    79 +-
 net/sctp/sm_sideeffect.c                           |    51 +-
 net/sctp/sm_statefuns.c                            |    50 +-
 net/sctp/sm_statetable.c                           |     5 +-
 net/sctp/socket.c                                  |   253 +-
 net/sctp/stream.c                                  |    49 +-
 net/sctp/stream_interleave.c                       |  1330 +
 net/sctp/stream_sched.c                            |     3 +-
 net/sctp/sysctl.c                                  |     7 +
 net/sctp/ulpevent.c                                |    15 +-
 net/sctp/ulpqueue.c                                |    23 +-
 net/smc/af_smc.c                                   |   247 +-
 net/smc/smc.h                                      |     5 +-
 net/smc/smc_cdc.c                                  |    54 +-
 net/smc/smc_cdc.h                                  |     1 -
 net/smc/smc_clc.c                                  |   102 +-
 net/smc/smc_clc.h                                  |    34 +-
 net/smc/smc_close.c                                |   207 +-
 net/smc/smc_close.h                                |     2 -
 net/smc/smc_core.c                                 |    20 +-
 net/smc/smc_diag.c                                 |     6 +-
 net/smc/smc_ib.c                                   |    38 +-
 net/smc/smc_llc.c                                  |     2 +-
 net/smc/smc_rx.c                                   |     9 +-
 net/smc/smc_tx.c                                   |    36 +-
 net/smc/smc_wr.c                                   |    50 +-
 net/smc/smc_wr.h                                   |     2 -
 net/socket.c                                       |   312 +-
 net/strparser/strparser.c                          |     4 +-
 net/sunrpc/cache.c                                 |    12 +-
 net/sunrpc/clnt.c                                  |    16 -
 net/sunrpc/rpc_pipe.c                              |     8 +-
 net/sunrpc/sched.c                                 |    42 +-
 net/sunrpc/svcsock.c                               |    18 +-
 net/sunrpc/xprt.c                                  |     5 +-
 net/sunrpc/xprtrdma/backchannel.c                  |    78 +-
 net/sunrpc/xprtrdma/fmr_ops.c                      |   157 +-
 net/sunrpc/xprtrdma/frwr_ops.c                     |   329 +-
 net/sunrpc/xprtrdma/module.c                       |    12 +-
 net/sunrpc/xprtrdma/rpc_rdma.c                     |   164 +-
 net/sunrpc/xprtrdma/svc_rdma_backchannel.c         |     5 -
 net/sunrpc/xprtrdma/svc_rdma_recvfrom.c            |     9 +-
 net/sunrpc/xprtrdma/svc_rdma_rw.c                  |    12 +-
 net/sunrpc/xprtrdma/svc_rdma_sendto.c              |     6 -
 net/sunrpc/xprtrdma/svc_rdma_transport.c           |    25 +-
 net/sunrpc/xprtrdma/transport.c                    |   128 +-
 net/sunrpc/xprtrdma/verbs.c                        |   288 +-
 net/sunrpc/xprtrdma/xprt_rdma.h                    |   116 +-
 net/sunrpc/xprtsock.c                              |    63 +-
 net/tipc/bcast.c                                   |    12 +-
 net/tipc/bearer.c                                  |    82 +-
 net/tipc/bearer.h                                  |     4 +
 net/tipc/core.h                                    |     1 -
 net/tipc/group.c                                   |   372 +-
 net/tipc/group.h                                   |    10 +-
 net/tipc/link.c                                    |     2 +-
 net/tipc/msg.c                                     |    55 +-
 net/tipc/msg.h                                     |     3 +-
 net/tipc/name_table.c                              |    57 +-
 net/tipc/name_table.h                              |     9 +-
 net/tipc/net.c                                     |    15 +-
 net/tipc/net.h                                     |     1 +
 net/tipc/netlink_compat.c                          |    43 +-
 net/tipc/server.c                                  |    80 +-
 net/tipc/server.h                                  |    13 +-
 net/tipc/socket.c                                  |   139 +-
 net/tipc/subscr.c                                  |    35 +-
 net/tipc/subscr.h                                  |     2 +-
 net/tls/tls_main.c                                 |    61 +-
 net/tls/tls_sw.c                                   |     8 +-
 net/unix/af_unix.c                                 |    58 +-
 net/vmw_vsock/af_vsock.c                           |    34 +-
 net/wireless/Kconfig                               |    13 +-
 net/wireless/ibss.c                                |     5 +
 net/wireless/mesh.c                                |    25 +-
 net/wireless/mlme.c                                |     6 +-
 net/wireless/nl80211.c                             |    77 +-
 net/wireless/scan.c                                |     5 +
 net/wireless/sme.c                                 |     2 +
 net/wireless/trace.h                               |    12 +-
 net/wireless/wext-core.c                           |    13 +-
 net/wireless/wext-proc.c                           |     1 -
 net/xfrm/xfrm_device.c                             |   201 +-
 net/xfrm/xfrm_input.c                              |     7 +-
 net/xfrm/xfrm_ipcomp.c                             |     2 +-
 net/xfrm/xfrm_output.c                             |     7 +-
 net/xfrm/xfrm_policy.c                             |   144 +-
 net/xfrm/xfrm_proc.c                               |     1 -
 net/xfrm/xfrm_replay.c                             |     7 +-
 net/xfrm/xfrm_state.c                              |    12 +
 net/xfrm/xfrm_user.c                               |    21 +-
 samples/bpf/Makefile                               |    16 +-
 samples/bpf/bpf_load.c                             |   102 -
 samples/bpf/bpf_load.h                             |     2 +-
 samples/bpf/tcbpf2_kern.c                          |   167 +-
 samples/bpf/test_cgrp2_attach2.c                   |    36 +-
 samples/bpf/test_override_return.sh                |    15 +
 samples/bpf/test_tunnel_bpf.sh                     |   128 +-
 samples/bpf/tracex7_kern.c                         |    16 +
 samples/bpf/tracex7_user.c                         |    28 +
 samples/bpf/xdp1_user.c                            |     4 +-
 samples/bpf/xdp2skb_meta.sh                        |   220 +
 samples/bpf/xdp2skb_meta_kern.c                    |   105 +
 samples/bpf/xdp_monitor_kern.c                     |    96 +-
 samples/bpf/xdp_monitor_user.c                     |   416 +-
 samples/bpf/xdp_redirect_cpu_user.c                |     6 +-
 samples/bpf/xdp_redirect_map_user.c                |     8 +-
 samples/bpf/xdp_redirect_user.c                    |     8 +-
 samples/bpf/xdp_router_ipv4_user.c                 |    10 +-
 samples/bpf/xdp_rxq_info_kern.c                    |    96 +
 samples/bpf/xdp_rxq_info_user.c                    |   531 +
 samples/bpf/xdp_tx_iptunnel_user.c                 |     6 +-
 samples/kobject/kobject-example.c                  |     4 +-
 samples/kobject/kset-example.c                     |     4 +-
 samples/livepatch/livepatch-callbacks-demo.c       |    15 -
 samples/livepatch/livepatch-sample.c               |    15 -
 samples/livepatch/livepatch-shadow-fix1.c          |    15 -
 samples/livepatch/livepatch-shadow-fix2.c          |    15 -
 samples/seccomp/Makefile                           |    10 +-
 samples/sockmap/sockmap_user.c                     |   392 +-
 scripts/Makefile.build                             |     8 +
 scripts/Makefile.extrawarn                         |     3 +
 scripts/Makefile.kasan                             |    35 +-
 scripts/Makefile.lib                               |    42 +-
 scripts/Makefile.ubsan                             |     1 -
 scripts/adjust_autoksyms.sh                        |     7 +
 scripts/basic/fixdep.c                             |   247 +-
 scripts/bloat-o-meter                              |     2 +-
 scripts/checkpatch.pl                              |   161 +-
 scripts/coccicheck                                 |     2 +-
 scripts/coccinelle/api/alloc/alloc_cast.cocci      |    92 +-
 scripts/coccinelle/api/alloc/kzalloc-simple.cocci  |    86 -
 scripts/coccinelle/api/alloc/zalloc-simple.cocci   |   448 +
 scripts/coccinelle/api/memdup.cocci                |     5 +-
 scripts/coccinelle/free/devm_free.cocci            |    55 +-
 scripts/coccinelle/free/ifnullfree.cocci           |     2 +-
 scripts/coccinelle/misc/array_size.cocci           |     4 +-
 scripts/coccinelle/null/deref_null.cocci           |     6 +-
 scripts/decodecode                                 |    12 +
 scripts/gcc-plugins/gcc-common.h                   |     4 +
 scripts/gcc-plugins/latent_entropy_plugin.c        |    17 +-
 scripts/gcc-plugins/randomize_layout_plugin.c      |    75 +-
 scripts/gcc-plugins/structleak_plugin.c            |    19 +-
 scripts/kallsyms.c                                 |     2 +-
 scripts/kconfig/.gitignore                         |     1 -
 scripts/kconfig/Makefile                           |    14 +-
 scripts/kconfig/conf.c                             |    59 +-
 scripts/kconfig/confdata.c                         |    10 +-
 scripts/kconfig/expr.c                             |   140 +-
 scripts/kconfig/expr.h                             |    91 +-
 scripts/kconfig/kconf_id.c                         |     1 -
 scripts/kconfig/kxgettext.c                        |     2 +-
 scripts/kconfig/lkc.h                              |     3 +-
 scripts/kconfig/lkc_proto.h                        |     2 +-
 scripts/kconfig/lxdialog/check-lxdialog.sh         |     3 +-
 scripts/kconfig/mconf.c                            |     2 +-
 scripts/kconfig/menu.c                             |   187 +-
 scripts/kconfig/nconf.gui.c                        |     3 +-
 scripts/kconfig/symbol.c                           |    40 +-
 scripts/kconfig/util.c                             |    26 +-
 scripts/kconfig/zconf.l                            |    51 +-
 scripts/kconfig/zconf.lex.c_shipped                |  2473 --
 scripts/kconfig/zconf.tab.c_shipped                |  2471 --
 scripts/kconfig/zconf.y                            |   118 +-
 scripts/kernel-doc                                 |  1506 +-
 scripts/link-vmlinux.sh                            |     2 +-
 scripts/mod/devicetable-offsets.c                  |     4 +
 scripts/mod/file2alias.c                           |    15 +
 scripts/mod/modpost.c                              |    11 +-
 scripts/package/Makefile                           |    26 +-
 scripts/package/builddeb                           |     2 +-
 scripts/package/mkspec                             |     2 +-
 scripts/package/snapcraft.template                 |    14 +
 scripts/tags.sh                                    |     2 +-
 security/Kconfig                                   |    15 +
 security/apparmor/apparmorfs.c                     |     6 +-
 security/integrity/digsig.c                        |     1 +
 security/integrity/evm/evm.h                       |     9 +-
 security/integrity/evm/evm_crypto.c                |    75 +-
 security/integrity/evm/evm_main.c                  |    67 +-
 security/integrity/evm/evm_secfs.c                 |    20 +-
 security/integrity/iint.c                          |     5 +-
 security/integrity/ima/ima_api.c                   |     5 +-
 security/integrity/ima/ima_appraise.c              |    46 +-
 security/integrity/ima/ima_crypto.c                |     2 +-
 security/integrity/ima/ima_init.c                  |     2 +-
 security/integrity/ima/ima_main.c                  |    96 +-
 security/integrity/ima/ima_policy.c                |    32 +-
 security/integrity/ima/ima_queue.c                 |     2 +-
 security/integrity/ima/ima_template.c              |    11 +-
 security/integrity/integrity.h                     |    41 +-
 security/keys/big_key.c                            |   110 +-
 security/keys/keyring.c                            |     7 -
 security/keys/trusted.c                            |    35 +-
 security/selinux/include/netlabel.h                |     3 +-
 security/selinux/netlabel.c                        |     3 +-
 security/selinux/ss/services.c                     |    21 +-
 security/selinux/xfrm.c                            |     2 +-
 security/smack/smack.h                             |     1 +
 security/smack/smack_access.c                      |    40 +-
 security/smack/smack_lsm.c                         |    10 +-
 security/tomoyo/audit.c                            |     8 +-
 security/tomoyo/common.c                           |    18 +-
 security/tomoyo/common.h                           |     6 +-
 security/tomoyo/securityfs_if.c                    |     6 +-
 security/yama/yama_lsm.c                           |    11 +-
 sound/ac97/Kconfig                                 |     1 -
 sound/core/compress_offload.c                      |    16 +-
 sound/core/control.c                               |    23 +-
 sound/core/hwdep.c                                 |     4 +-
 sound/core/info.c                                  |     8 +-
 sound/core/init.c                                  |     4 +-
 sound/core/oss/pcm_oss.c                           |    34 +-
 sound/core/pcm_lib.c                               |     4 +-
 sound/core/pcm_misc.c                              |    19 +-
 sound/core/pcm_native.c                            |    26 +-
 sound/core/rawmidi.c                               |     8 +-
 sound/core/seq/oss/seq_oss.c                       |     4 +-
 sound/core/seq/oss/seq_oss_device.h                |     2 +-
 sound/core/seq/oss/seq_oss_rw.c                    |     8 +-
 sound/core/seq/seq_clientmgr.c                     |    38 +-
 sound/core/seq/seq_fifo.c                          |     2 +-
 sound/core/seq/seq_memory.c                        |    14 +-
 sound/core/seq/seq_memory.h                        |     3 +-
 sound/core/seq/seq_prioq.c                         |    28 +-
 sound/core/seq/seq_prioq.h                         |     6 +-
 sound/core/seq/seq_queue.c                         |    32 +-
 sound/core/seq/seq_timer.c                         |    13 +-
 sound/core/seq/seq_timer.h                         |     2 +-
 sound/core/timer.c                                 |     8 +-
 sound/drivers/aloop.c                              |    17 +-
 sound/drivers/dummy.c                              |    29 +-
 sound/firewire/bebob/bebob_hwdep.c                 |     6 +-
 sound/firewire/dice/dice-hwdep.c                   |     6 +-
 sound/firewire/digi00x/digi00x-hwdep.c             |     6 +-
 sound/firewire/fireface/ff-hwdep.c                 |     6 +-
 sound/firewire/fireworks/fireworks_hwdep.c         |     8 +-
 sound/firewire/motu/motu-hwdep.c                   |     8 +-
 sound/firewire/oxfw/oxfw-hwdep.c                   |     6 +-
 sound/firewire/tascam/tascam-hwdep.c               |     6 +-
 sound/hda/ext/hdac_ext_bus.c                       |     2 +-
 sound/isa/gus/gus_dma.c                            |     5 +-
 sound/mips/hal2.c                                  |     2 +-
 sound/mips/sgio2audio.c                            |     2 +-
 sound/oss/dmasound/dmasound_core.c                 |     6 +-
 sound/pci/hda/Kconfig                              |     1 -
 sound/pci/hda/hda_intel.c                          |    44 +-
 sound/pci/hda/patch_conexant.c                     |     2 +
 sound/pci/hda/patch_realtek.c                      |   273 +-
 sound/pci/ice1712/prodigy_hifi.c                   |   131 +-
 sound/pci/korg1212/korg1212.c                      |     1 -
 sound/soc/Kconfig                                  |     1 +
 sound/soc/Makefile                                 |     1 +
 sound/soc/amd/acp-pcm-dma.c                        |    51 +-
 sound/soc/amd/acp.h                                |     2 +
 sound/soc/atmel/atmel-classd.c                     |     6 -
 sound/soc/atmel/sam9g20_wm8731.c                   |    17 +-
 sound/soc/atmel/sam9x5_wm8731.c                    |    12 +-
 sound/soc/au1x/ac97c.c                             |     6 +-
 sound/soc/bcm/bcm2835-i2s.c                        |    20 +-
 sound/soc/cirrus/ep93xx-ac97.c                     |     6 +-
 sound/soc/codecs/88pm860x-codec.c                  |     9 +-
 sound/soc/codecs/Kconfig                           |    46 +-
 sound/soc/codecs/Makefile                          |    13 +-
 sound/soc/codecs/ak4613.c                          |    78 +
 sound/soc/codecs/cq93vc.c                          |    10 +-
 sound/soc/codecs/cs35l32.c                         |    18 +-
 sound/soc/codecs/cs35l34.c                         |    19 +-
 sound/soc/codecs/cs42l52.c                         |    13 +-
 sound/soc/codecs/cs42l56.c                         |    13 +-
 sound/soc/codecs/cs42l73.c                         |    15 +-
 sound/soc/codecs/cs47l24.c                         |    12 +-
 sound/soc/codecs/cx20442.c                         |    46 +-
 sound/soc/codecs/da7213.c                          |     7 +-
 sound/soc/codecs/da7218.c                          |     9 +-
 sound/soc/codecs/dmic.c                            |    24 +-
 sound/soc/codecs/hdac_hdmi.c                       |   358 +-
 sound/soc/codecs/hdmi-codec.c                      |     7 +-
 sound/soc/codecs/max98373.c                        |   976 +
 sound/soc/codecs/max98373.h                        |   213 +
 sound/soc/codecs/max98926.c                        |     2 +-
 sound/soc/codecs/max98927.c                        |     1 -
 sound/soc/codecs/mc13783.c                         |     9 +-
 sound/soc/codecs/msm8916-wcd-analog.c              |     8 +-
 sound/soc/codecs/nau8540.c                         |    98 +-
 sound/soc/codecs/nau8540.h                         |    20 +
 sound/soc/codecs/nau8824.c                         |    18 +-
 sound/soc/codecs/nau8825.c                         |   101 +-
 sound/soc/codecs/nau8825.h                         |     3 +-
 sound/soc/codecs/pcm186x-i2c.c                     |    69 +
 sound/soc/codecs/pcm186x-spi.c                     |    69 +
 sound/soc/codecs/pcm186x.c                         |   719 +
 sound/soc/codecs/pcm186x.h                         |   220 +
 sound/soc/codecs/pcm512x-spi.c                     |     4 +
 sound/soc/codecs/rl6231.c                          |    93 +-
 sound/soc/codecs/rt5514-spi.c                      |     1 +
 sound/soc/codecs/rt5514.c                          |    85 +
 sound/soc/codecs/rt5514.h                          |     5 +-
 sound/soc/codecs/rt5645.c                          |   187 +-
 sound/soc/codecs/rt5645.h                          |     6 +
 sound/soc/codecs/rt5651.c                          |     1 +
 sound/soc/codecs/sgtl5000.c                        |    33 +-
 sound/soc/codecs/si476x.c                          |     9 +-
 sound/soc/codecs/sn95031.c                         |   936 -
 sound/soc/codecs/sn95031.h                         |   133 -
 sound/soc/codecs/spdif_receiver.c                  |     5 +-
 sound/soc/codecs/spdif_transmitter.c               |     5 +-
 sound/soc/codecs/tas5720.c                         |    61 +-
 sound/soc/codecs/tas5720.h                         |    31 +
 sound/soc/codecs/tas6424.c                         |   707 +
 sound/soc/codecs/tas6424.h                         |   144 +
 sound/soc/codecs/tfa9879.c                         |     1 +
 sound/soc/codecs/tlv320aic31xx.c                   |   310 +-
 sound/soc/codecs/tlv320aic31xx.h                   |   335 +-
 sound/soc/codecs/tlv320aic32x4.c                   |   182 +-
 sound/soc/codecs/tlv320aic32x4.h                   |   308 +-
 sound/soc/codecs/tlv320aic3x.c                     |    15 +-
 sound/soc/codecs/tlv320dac33.c                     |    32 +-
 sound/soc/codecs/ts3a227e.c                        |     2 +-
 sound/soc/codecs/tscs42xx.c                        |  1456 +
 sound/soc/codecs/tscs42xx.h                        |  2693 ++
 sound/soc/codecs/twl4030.c                         |     9 +-
 sound/soc/codecs/twl6040.c                         |    18 +-
 sound/soc/codecs/uda1380.c                         |    42 +-
 sound/soc/codecs/wm0010.c                          |     5 +-
 sound/soc/codecs/wm2000.c                          |     6 +-
 sound/soc/codecs/wm2200.c                          |     9 +-
 sound/soc/codecs/wm5102.c                          |    11 +-
 sound/soc/codecs/wm5110.c                          |    12 +-
 sound/soc/codecs/wm8350.c                          |    10 +-
 sound/soc/codecs/wm8400.c                          |     9 +-
 sound/soc/codecs/wm8903.c                          |    12 +-
 sound/soc/codecs/wm8994.c                          |    10 +-
 sound/soc/codecs/wm8997.c                          |    11 +-
 sound/soc/codecs/wm8998.c                          |    12 +-
 sound/soc/codecs/wm_adsp.c                         |    14 +-
 sound/soc/davinci/davinci-mcasp.c                  |    19 +
 sound/soc/fsl/eukrea-tlv320.c                      |     1 -
 sound/soc/fsl/fsl-asoc-card.c                      |     4 +-
 sound/soc/fsl/fsl_asrc.h                           |     2 +-
 sound/soc/fsl/fsl_dma.c                            |     4 +-
 sound/soc/fsl/fsl_ssi.c                            |  1367 +-
 sound/soc/fsl/fsl_ssi.h                            |   427 +-
 sound/soc/fsl/fsl_ssi_dbg.c                        |    59 +-
 sound/soc/hisilicon/hi6210-i2s.c                   |     1 -
 sound/soc/intel/Kconfig                            |   118 +-
 sound/soc/intel/Makefile                           |     2 +-
 sound/soc/intel/atom/sst/sst_acpi.c                |     3 +
 sound/soc/intel/atom/sst/sst_stream.c              |     8 +-
 sound/soc/intel/boards/Kconfig                     |   196 +-
 sound/soc/intel/boards/bytcht_da7213.c             |     4 +-
 sound/soc/intel/boards/bytcht_es8316.c             |    26 +-
 sound/soc/intel/boards/bytcr_rt5640.c              |     4 +-
 sound/soc/intel/boards/bytcr_rt5651.c              |    50 +-
 sound/soc/intel/boards/cht_bsw_rt5645.c            |    13 +-
 sound/soc/intel/boards/cht_bsw_rt5672.c            |     4 +-
 sound/soc/intel/boards/haswell.c                   |     2 +-
 sound/soc/intel/boards/kbl_rt5663_max98927.c       |    97 +-
 .../soc/intel/boards/kbl_rt5663_rt5514_max98927.c  |     2 +-
 sound/soc/intel/boards/mfld_machine.c              |   428 -
 sound/soc/intel/common/sst-dsp.c                   |     4 +-
 sound/soc/intel/skylake/Makefile                   |     5 +
 sound/soc/intel/skylake/bxt-sst.c                  |     2 +-
 sound/soc/intel/skylake/cnl-sst.c                  |     2 +-
 sound/soc/intel/skylake/skl-i2s.h                  |    95 +
 sound/soc/intel/skylake/skl-messages.c             |    23 +
 sound/soc/intel/skylake/skl-nhlt.c                 |   175 +-
 sound/soc/intel/skylake/skl-pcm.c                  |    14 +-
 sound/soc/intel/skylake/skl-ssp-clk.c              |   429 +
 sound/soc/intel/skylake/skl-ssp-clk.h              |   117 +
 sound/soc/intel/skylake/skl-sst-dsp.c              |    14 +-
 sound/soc/intel/skylake/skl-sst-dsp.h              |     4 +
 sound/soc/intel/skylake/skl-sst-utils.c            |     6 +-
 sound/soc/intel/skylake/skl-sst.c                  |     2 +-
 sound/soc/intel/skylake/skl-topology.c             |    45 +-
 sound/soc/intel/skylake/skl.c                      |   150 +-
 sound/soc/intel/skylake/skl.h                      |    28 +
 sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c  |   552 +-
 sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.h  |    15 +-
 sound/soc/mediatek/mt2701/mt2701-afe-common.h      |    87 +-
 sound/soc/mediatek/mt2701/mt2701-afe-pcm.c         |   215 +-
 sound/soc/mediatek/mt2701/mt2701-reg.h             |    42 +-
 sound/soc/mediatek/mt8173/mt8173-afe-pcm.c         |     6 +-
 sound/soc/mediatek/mt8173/mt8173-rt5650-rt5514.c   |     2 -
 sound/soc/mediatek/mt8173/mt8173-rt5650-rt5676.c   |     2 -
 sound/soc/mediatek/mt8173/mt8173-rt5650.c          |    13 +-
 sound/soc/mxs/mxs-sgtl5000.c                       |    25 +-
 sound/soc/nuc900/nuc900-ac97.c                     |    11 +-
 sound/soc/omap/ams-delta.c                         |     4 +-
 sound/soc/omap/mcbsp.c                             |     4 +-
 sound/soc/qcom/apq8016_sbc.c                       |    10 +-
 sound/soc/rockchip/rk3399_gru_sound.c              |    22 +-
 sound/soc/rockchip/rockchip_i2s.c                  |    11 +-
 sound/soc/samsung/bells.c                          |    40 +-
 sound/soc/sh/rcar/core.c                           |   143 +-
 sound/soc/sh/rcar/dma.c                            |    18 -
 sound/soc/sh/rcar/rsnd.h                           |    15 +-
 sound/soc/sh/rcar/ssi.c                            |   163 +-
 sound/soc/soc-acpi.c                               |    73 +-
 sound/soc/soc-compress.c                           |    81 +-
 sound/soc/soc-core.c                               |   157 +-
 sound/soc/soc-dapm.c                               |     4 +-
 sound/soc/soc-io.c                                 |     6 +
 sound/soc/soc-ops.c                                |     4 +-
 sound/soc/soc-pcm.c                                |     7 +-
 sound/soc/soc-utils.c                              |     2 +-
 sound/soc/stm/Kconfig                              |    18 +-
 sound/soc/stm/Makefile                             |     3 +
 sound/soc/stm/stm32_adfsdm.c                       |   347 +
 sound/soc/stm/stm32_sai.c                          |   114 +-
 sound/soc/sunxi/sun4i-codec.c                      |    29 +-
 sound/soc/sunxi/sun4i-i2s.c                        |    59 +-
 sound/soc/sunxi/sun8i-codec.c                      |    82 +-
 sound/soc/uniphier/Kconfig                         |    19 +
 sound/soc/uniphier/Makefile                        |     3 +
 sound/soc/uniphier/evea.c                          |   567 +
 sound/soc/ux500/mop500.c                           |     4 +
 sound/soc/ux500/ux500_pcm.c                        |     5 +
 sound/sound_core.c                                 |    42 -
 sound/usb/card.c                                   |    21 +-
 sound/usb/mixer.c                                  |    26 +-
 sound/usb/mixer_quirks.c                           |    86 +-
 sound/usb/pcm.c                                    |     9 +
 sound/usb/quirks-table.h                           |    95 +
 sound/usb/quirks.c                                 |     8 +-
 sound/usb/usx2y/us122l.c                           |    51 +-
 sound/usb/usx2y/usX2Yhwdep.c                       |    36 +-
 sound/x86/intel_hdmi_audio.c                       |    37 +-
 tools/arch/alpha/include/uapi/asm/errno.h          |   128 +
 tools/arch/mips/include/asm/errno.h                |    17 +
 tools/arch/mips/include/uapi/asm/errno.h           |   130 +
 tools/arch/parisc/include/uapi/asm/errno.h         |   128 +
 tools/arch/powerpc/include/uapi/asm/errno.h        |    10 +
 tools/arch/powerpc/include/uapi/asm/kvm.h          |    27 +
 tools/arch/s390/include/uapi/asm/kvm.h             |     5 +-
 tools/arch/sparc/include/uapi/asm/errno.h          |   118 +
 tools/arch/x86/include/asm/cpufeatures.h           |    30 +-
 tools/arch/x86/include/asm/disabled-features.h     |    11 +-
 tools/arch/x86/include/asm/required-features.h     |     3 +-
 tools/arch/x86/include/uapi/asm/errno.h            |     1 +
 tools/bpf/Makefile                                 |    29 +
 tools/bpf/bpf_jit_disasm.c                         |    14 +-
 tools/bpf/bpftool/Documentation/Makefile           |    35 +-
 tools/bpf/bpftool/Documentation/bpftool-cgroup.rst |   118 +
 tools/bpf/bpftool/Documentation/bpftool-map.rst    |    11 +-
 tools/bpf/bpftool/Documentation/bpftool-prog.rst   |    16 +-
 tools/bpf/bpftool/Documentation/bpftool.rst        |    12 +-
 tools/bpf/bpftool/Makefile                         |    90 +-
 tools/bpf/bpftool/bash-completion/bpftool          |    78 +-
 tools/bpf/bpftool/cgroup.c                         |   308 +
 tools/bpf/bpftool/common.c                         |   199 +-
 tools/bpf/bpftool/jit_disasm.c                     |    23 +-
 tools/bpf/bpftool/main.c                           |    18 +-
 tools/bpf/bpftool/main.h                           |     9 +-
 tools/bpf/bpftool/map.c                            |    13 +-
 tools/bpf/bpftool/prog.c                           |   232 +-
 tools/build/Makefile.feature                       |     4 +-
 tools/build/feature/Makefile                       |    15 +-
 tools/build/feature/test-all.c                     |    10 +
 tools/build/feature/test-disassembler-four-args.c  |    15 +
 tools/build/feature/test-libopencsd.c              |     8 +
 tools/build/feature/test-pthread-barrier.c         |    12 +
 tools/cgroup/Makefile                              |     1 -
 tools/gpio/Makefile                                |     2 -
 tools/gpio/gpio-event-mon.c                        |    10 +-
 tools/hv/Makefile                                  |    24 +-
 tools/iio/Makefile                                 |     2 -
 tools/include/asm-generic/bitops/find.h            |    16 +
 tools/include/uapi/asm-generic/errno-base.h        |    40 +
 tools/include/uapi/asm-generic/errno.h             |   123 +
 tools/include/uapi/drm/i915_drm.h                  |    77 +
 tools/include/uapi/linux/bpf.h                     |   116 +-
 tools/include/uapi/linux/bpf_common.h              |     7 +-
 tools/include/uapi/linux/if_link.h                 |   944 +
 tools/include/uapi/linux/kvm.h                     |    96 +
 tools/include/uapi/linux/netlink.h                 |   251 +
 tools/include/uapi/linux/perf_event.h              |    32 +-
 tools/include/uapi/linux/sched.h                   |     5 +
 tools/include/uapi/sound/asound.h                  |     9 +
 tools/kvm/kvm_stat/kvm_stat                        |   503 +-
 tools/kvm/kvm_stat/kvm_stat.txt                    |     4 +-
 tools/laptop/freefall/Makefile                     |     1 -
 tools/leds/Makefile                                |     1 -
 tools/lib/bpf/Build                                |     2 +-
 tools/lib/bpf/Makefile                             |    30 +-
 tools/lib/bpf/bpf.c                                |   135 +
 tools/lib/bpf/bpf.h                                |     4 +-
 tools/lib/bpf/libbpf.c                             |   261 +-
 tools/lib/bpf/libbpf.h                             |     6 +
 tools/lib/bpf/nlattr.c                             |   187 +
 tools/lib/bpf/nlattr.h                             |    72 +
 tools/lib/find_bit.c                               |    39 +-
 tools/lib/subcmd/pager.c                           |     5 +-
 tools/lib/traceevent/event-parse.c                 |    62 +-
 tools/lib/traceevent/event-plugin.c                |    24 +-
 tools/lib/traceevent/kbuffer-parse.c               |     4 +-
 tools/lib/traceevent/parse-filter.c                |    22 +-
 tools/objtool/builtin-check.c                      |     6 +-
 tools/objtool/builtin-orc.c                        |     6 +-
 tools/objtool/builtin.h                            |     5 +
 tools/objtool/check.c                              |   245 +-
 tools/objtool/check.h                              |     4 +-
 tools/objtool/orc_gen.c                            |     5 +
 tools/perf/.gitignore                              |     1 +
 tools/perf/Build                                   |     4 +-
 tools/perf/Documentation/perf-buildid-cache.txt    |     3 +
 tools/perf/Documentation/perf-data.txt             |     4 +
 tools/perf/Documentation/perf-evlist.txt           |     4 +
 tools/perf/Documentation/perf-inject.txt           |     4 +
 tools/perf/Documentation/perf-kallsyms.txt         |     2 +-
 tools/perf/Documentation/perf-lock.txt             |     4 +
 tools/perf/Documentation/perf-probe.txt            |    18 +-
 tools/perf/Documentation/perf-record.txt           |     3 +
 tools/perf/Documentation/perf-report.txt           |    37 +-
 tools/perf/Documentation/perf-sched.txt            |     4 +
 tools/perf/Documentation/perf-script.txt           |    47 +-
 tools/perf/Documentation/perf-timechart.txt        |     4 +-
 tools/perf/Documentation/perf-top.txt              |     6 +
 tools/perf/Documentation/perf-trace.txt            |    20 +-
 tools/perf/Documentation/perf.data-file-format.txt |    27 +
 tools/perf/Documentation/tips.txt                  |     2 +
 tools/perf/Makefile.config                         |    65 +-
 tools/perf/Makefile.perf                           |    23 +-
 tools/perf/arch/arm/util/auxtrace.c                |    77 +-
 tools/perf/arch/arm/util/pmu.c                     |     6 +
 tools/perf/arch/arm64/util/Build                   |     5 +-
 tools/perf/arch/arm64/util/arm-spe.c               |   225 +
 tools/perf/arch/arm64/util/header.c                |    65 +
 tools/perf/arch/arm64/util/sym-handling.c          |    22 +
 tools/perf/arch/common.c                           |    44 +-
 tools/perf/arch/common.h                           |     1 -
 tools/perf/arch/powerpc/util/header.c              |     2 +-
 tools/perf/arch/powerpc/util/sym-handling.c        |     8 +
 tools/perf/arch/s390/Makefile                      |    25 +
 tools/perf/arch/s390/annotate/instructions.c       |     3 +-
 tools/perf/arch/s390/entry/syscalls/mksyscalltbl   |    32 +
 tools/perf/arch/s390/entry/syscalls/syscall.tbl    |   390 +
 tools/perf/arch/x86/tests/perf-time-to-tsc.c       |     2 +-
 tools/perf/arch/x86/util/header.c                  |     4 +-
 tools/perf/arch/x86/util/unwind-libunwind.c        |     2 +-
 tools/perf/bench/futex-hash.c                      |    20 +-
 tools/perf/bench/futex-lock-pi.c                   |    23 +-
 tools/perf/bench/futex-requeue.c                   |    22 +-
 tools/perf/bench/futex-wake-parallel.c             |    46 +-
 tools/perf/bench/futex-wake.c                      |    18 +-
 tools/perf/builtin-buildid-cache.c                 |     4 +-
 tools/perf/builtin-c2c.c                           |    20 +-
 tools/perf/builtin-help.c                          |     2 +-
 tools/perf/builtin-inject.c                        |     3 +-
 tools/perf/builtin-kvm.c                           |    13 +-
 tools/perf/builtin-record.c                        |    75 +-
 tools/perf/builtin-report.c                        |   286 +-
 tools/perf/builtin-script.c                        |   266 +-
 tools/perf/builtin-stat.c                          |   232 +-
 tools/perf/builtin-top.c                           |   160 +-
 tools/perf/builtin-trace.c                         |   125 +-
 tools/perf/check-headers.sh                        |    10 +-
 tools/perf/perf-completion.sh                      |    47 +-
 tools/perf/perf.c                                  |     4 +-
 tools/perf/perf.h                                  |     2 +
 .../arch/arm64/cavium/thunderx2-imp-def.json       |    62 +
 .../pmu-events/arch/arm64/cortex-a53/branch.json   |    27 +
 .../perf/pmu-events/arch/arm64/cortex-a53/bus.json |    22 +
 .../pmu-events/arch/arm64/cortex-a53/cache.json    |    27 +
 .../pmu-events/arch/arm64/cortex-a53/memory.json   |    22 +
 .../pmu-events/arch/arm64/cortex-a53/other.json    |    32 +
 .../pmu-events/arch/arm64/cortex-a53/pipeline.json |    52 +
 tools/perf/pmu-events/arch/arm64/mapfile.csv       |    16 +
 tools/perf/pmu-events/arch/powerpc/mapfile.csv     |    12 +-
 .../perf/pmu-events/arch/powerpc/power9/cache.json |     5 -
 .../pmu-events/arch/powerpc/power9/frontend.json   |     7 +-
 .../pmu-events/arch/powerpc/power9/marked.json     |    27 +-
 .../perf/pmu-events/arch/powerpc/power9/other.json |   276 +-
 .../pmu-events/arch/powerpc/power9/pipeline.json   |    14 +-
 tools/perf/pmu-events/arch/powerpc/power9/pmc.json |     2 +-
 .../arch/powerpc/power9/translation.json           |     5 -
 .../perf/pmu-events/arch/x86/broadwell/cache.json  |   555 +-
 .../arch/x86/broadwell/floating-point.json         |   108 +-
 .../pmu-events/arch/x86/broadwell/frontend.json    |   138 +-
 .../perf/pmu-events/arch/x86/broadwell/memory.json |   210 +-
 .../perf/pmu-events/arch/x86/broadwell/other.json  |    20 +-
 .../pmu-events/arch/x86/broadwell/pipeline.json    |  1216 +-
 .../arch/x86/broadwell/virtual-memory.json         |   150 +-
 .../pmu-events/arch/x86/broadwellde/cache.json     |   389 +-
 .../arch/x86/broadwellde/floating-point.json       |   108 +-
 .../pmu-events/arch/x86/broadwellde/frontend.json  |   138 +-
 .../pmu-events/arch/x86/broadwellde/memory.json    |     9 +-
 .../pmu-events/arch/x86/broadwellde/other.json     |    20 +-
 .../pmu-events/arch/x86/broadwellde/pipeline.json  |  1214 +-
 .../arch/x86/broadwellde/virtual-memory.json       |   150 +-
 .../perf/pmu-events/arch/x86/broadwellx/cache.json |   383 +-
 .../arch/x86/broadwellx/floating-point.json        |   108 +-
 .../pmu-events/arch/x86/broadwellx/frontend.json   |   138 +-
 .../pmu-events/arch/x86/broadwellx/memory.json     |    40 +-
 .../perf/pmu-events/arch/x86/broadwellx/other.json |    20 +-
 .../pmu-events/arch/x86/broadwellx/pipeline.json   |  1214 +-
 .../arch/x86/broadwellx/virtual-memory.json        |   150 +-
 tools/perf/pmu-events/arch/x86/goldmont/cache.json |  1244 +-
 .../perf/pmu-events/arch/x86/goldmont/memory.json  |   280 +-
 tools/perf/pmu-events/arch/x86/goldmont/other.json |    54 +-
 .../pmu-events/arch/x86/goldmont/pipeline.json     |   506 +-
 .../arch/x86/goldmont/virtual-memory.json          |    60 +-
 tools/perf/pmu-events/arch/x86/haswell/cache.json  |   365 +-
 .../arch/x86/haswell/floating-point.json           |    20 +-
 .../perf/pmu-events/arch/x86/haswell/frontend.json |   132 +-
 tools/perf/pmu-events/arch/x86/haswell/memory.json |    21 +
 tools/perf/pmu-events/arch/x86/haswell/other.json  |    20 +-
 .../perf/pmu-events/arch/x86/haswell/pipeline.json |  1131 +-
 .../arch/x86/haswell/virtual-memory.json           |   212 +-
 tools/perf/pmu-events/arch/x86/haswellx/cache.json |   377 +-
 .../arch/x86/haswellx/floating-point.json          |    20 +-
 .../pmu-events/arch/x86/haswellx/frontend.json     |   132 +-
 .../perf/pmu-events/arch/x86/haswellx/memory.json  |    28 +
 tools/perf/pmu-events/arch/x86/haswellx/other.json |    20 +-
 .../pmu-events/arch/x86/haswellx/pipeline.json     |  1133 +-
 .../arch/x86/haswellx/virtual-memory.json          |   212 +-
 .../perf/pmu-events/arch/x86/ivybridge/cache.json  |   243 +-
 .../pmu-events/arch/x86/ivybridge/frontend.json    |   122 +-
 .../perf/pmu-events/arch/x86/ivybridge/memory.json |    24 +-
 .../perf/pmu-events/arch/x86/ivybridge/other.json  |    20 +-
 .../pmu-events/arch/x86/ivybridge/pipeline.json    |   822 +-
 .../arch/x86/ivybridge/virtual-memory.json         |    60 +-
 tools/perf/pmu-events/arch/x86/ivytown/cache.json  |   236 +-
 .../perf/pmu-events/arch/x86/ivytown/frontend.json |   122 +-
 tools/perf/pmu-events/arch/x86/ivytown/memory.json |    24 +-
 tools/perf/pmu-events/arch/x86/ivytown/other.json  |    20 +-
 .../perf/pmu-events/arch/x86/ivytown/pipeline.json |   822 +-
 .../arch/x86/ivytown/virtual-memory.json           |    60 +-
 tools/perf/pmu-events/arch/x86/mapfile.csv         |     5 +-
 .../perf/pmu-events/arch/x86/silvermont/cache.json |     3 +-
 tools/perf/pmu-events/arch/x86/skylake/cache.json  |  4390 +--
 .../arch/x86/skylake/floating-point.json           |     5 +-
 .../perf/pmu-events/arch/x86/skylake/frontend.json |   232 +-
 tools/perf/pmu-events/arch/x86/skylake/memory.json |  2118 +-
 tools/perf/pmu-events/arch/x86/skylake/other.json  |    40 +-
 .../perf/pmu-events/arch/x86/skylake/pipeline.json |   973 +-
 .../arch/x86/skylake/virtual-memory.json           |   262 +-
 tools/perf/pmu-events/arch/x86/skylakex/cache.json |   257 +-
 .../arch/x86/skylakex/floating-point.json          |     3 +-
 .../pmu-events/arch/x86/skylakex/frontend.json     |    48 +-
 .../perf/pmu-events/arch/x86/skylakex/memory.json  |   231 +-
 tools/perf/pmu-events/arch/x86/skylakex/other.json |    94 +-
 .../pmu-events/arch/x86/skylakex/pipeline.json     |    44 +-
 .../arch/x86/skylakex/virtual-memory.json          |    42 +-
 tools/perf/pmu-events/jevents.c                    |    39 +-
 tools/perf/scripts/python/bin/mem-phys-addr-record |    19 +
 tools/perf/scripts/python/bin/mem-phys-addr-report |     3 +
 tools/perf/scripts/python/mem-phys-addr.py         |    95 +
 tools/perf/tests/attr.c                            |     6 +
 tools/perf/tests/backward-ring-buffer.c            |     9 +-
 tools/perf/tests/bp_signal.c                       |     2 +-
 tools/perf/tests/bpf-script-example.c              |     4 +-
 tools/perf/tests/bpf.c                             |    68 +-
 tools/perf/tests/builtin-test.c                    |    10 +-
 tools/perf/tests/code-reading.c                    |     2 +-
 tools/perf/tests/dwarf-unwind.c                    |     1 +
 tools/perf/tests/keep-tracking.c                   |     2 +-
 tools/perf/tests/mmap-basic.c                      |     2 +-
 tools/perf/tests/openat-syscall-tp-fields.c        |     5 +-
 tools/perf/tests/parse-events.c                    |     1 -
 tools/perf/tests/perf-record.c                     |     2 +-
 tools/perf/tests/sample-parsing.c                  |     2 +-
 .../perf/tests/shell/trace+probe_libc_inet_pton.sh |    23 +-
 tools/perf/tests/shell/trace+probe_vfs_getname.sh  |     7 +-
 tools/perf/tests/sw-clock.c                        |     2 +-
 tools/perf/tests/switch-tracking.c                 |     2 +-
 tools/perf/tests/task-exit.c                       |     2 +-
 tools/perf/tests/thread-map.c                      |     2 +-
 tools/perf/trace/beauty/Build                      |     1 +
 tools/perf/trace/beauty/arch_errno_names.c         |     1 +
 tools/perf/trace/beauty/arch_errno_names.sh        |   100 +
 tools/perf/trace/beauty/beauty.h                   |     5 +
 tools/perf/trace/beauty/flock.c                    |    10 +-
 tools/perf/trace/beauty/futex_val3.c               |    18 +
 tools/perf/ui/browsers/annotate.c                  |   422 +-
 tools/perf/ui/browsers/hists.c                     |    38 +-
 tools/perf/ui/browsers/hists.h                     |     3 +-
 tools/perf/ui/gtk/annotate.c                       |    25 +-
 tools/perf/util/Build                              |    10 +-
 tools/perf/util/annotate.c                         |   652 +-
 tools/perf/util/annotate.h                         |    76 +-
 tools/perf/util/arm-spe-pkt-decoder.c              |   462 +
 tools/perf/util/arm-spe-pkt-decoder.h              |    43 +
 tools/perf/util/arm-spe.c                          |   231 +
 tools/perf/util/arm-spe.h                          |    31 +
 tools/perf/util/auxtrace.c                         |    23 +-
 tools/perf/util/auxtrace.h                         |     1 +
 tools/perf/util/bpf-loader.c                       |     4 +-
 tools/perf/util/callchain.c                        |    10 +
 tools/perf/util/callchain.h                        |     2 +
 tools/perf/util/cgroup.c                           |     3 +
 tools/perf/util/cs-etm-decoder/Build               |     1 +
 tools/perf/util/cs-etm-decoder/cs-etm-decoder.c    |   513 +
 tools/perf/util/cs-etm-decoder/cs-etm-decoder.h    |   105 +
 tools/perf/util/cs-etm.c                           |  1023 +
 tools/perf/util/cs-etm.h                           |    18 +
 tools/perf/util/data.c                             |    10 -
 tools/perf/util/dso.c                              |     2 +-
 tools/perf/util/env.c                              |    47 +
 tools/perf/util/env.h                              |     2 +
 tools/perf/util/event.c                            |     8 +
 tools/perf/util/event.h                            |     4 +-
 tools/perf/util/evlist.c                           |    87 +-
 tools/perf/util/evlist.h                           |    19 +-
 tools/perf/util/evsel.c                            |   252 +-
 tools/perf/util/evsel.h                            |    26 +-
 tools/perf/util/generate-cmdlist.sh                |     2 +-
 tools/perf/util/header.c                           |   130 +-
 tools/perf/util/header.h                           |     9 +-
 tools/perf/util/hist.h                             |     6 +-
 tools/perf/util/intel-bts.c                        |     6 +-
 tools/perf/util/intel-pt-decoder/Build             |    24 +-
 tools/perf/util/intel-pt.c                         |    11 +-
 tools/perf/util/machine.c                          |     4 +-
 tools/perf/util/map.c                              |     2 +-
 tools/perf/util/metricgroup.c                      |    10 +-
 tools/perf/util/mmap.c                             |   190 +-
 tools/perf/util/mmap.h                             |    14 +-
 tools/perf/util/ordered-events.c                   |     3 +-
 tools/perf/util/ordered-events.h                   |     2 +-
 tools/perf/util/parse-events.c                     |     3 +
 tools/perf/util/path.c                             |    14 +
 tools/perf/util/path.h                             |     3 +
 tools/perf/util/pmu.c                              |    87 +-
 tools/perf/util/pmu.h                              |     2 +-
 tools/perf/util/probe-event.c                      |    85 +-
 tools/perf/util/python-ext-sources                 |     1 +
 tools/perf/util/python.c                           |     2 +-
 tools/perf/util/rblist.c                           |    19 +-
 tools/perf/util/rblist.h                           |     1 +
 tools/perf/util/record.c                           |     8 +-
 .../util/scripting-engines/trace-event-python.c    |     3 +-
 tools/perf/util/session.c                          |    54 +-
 tools/perf/util/session.h                          |     2 +-
 tools/perf/util/sort.c                             |    20 +-
 tools/perf/util/srcline.c                          |     9 +-
 tools/perf/util/srcline.h                          |     5 +-
 tools/perf/util/stat-shadow.c                      |   426 +-
 tools/perf/util/stat.c                             |    15 +-
 tools/perf/util/stat.h                             |    63 +-
 tools/perf/util/string.c                           |    46 +
 tools/perf/util/string2.h                          |     2 +
 tools/perf/util/symbol.c                           |     5 +
 tools/perf/util/symbol.h                           |     1 +
 tools/perf/util/syscalltbl.c                       |     4 +
 tools/perf/util/target.h                           |     7 +
 tools/perf/util/thread_map.c                       |    27 +-
 tools/perf/util/thread_map.h                       |     3 +-
 tools/perf/util/time-utils.c                       |   301 +-
 tools/perf/util/time-utils.h                       |     8 +
 tools/perf/util/tool.h                             |     1 +
 tools/perf/util/trigger.h                          |     9 +-
 tools/perf/util/unwind-libunwind-local.c           |     9 +-
 tools/perf/util/unwind-libunwind.c                 |     4 +-
 tools/perf/util/util.c                             |    26 +-
 tools/perf/util/util.h                             |    10 +
 tools/power/acpi/Makefile.config                   |     3 -
 tools/power/acpi/common/cmfsize.c                  |     2 +-
 tools/power/acpi/common/getopt.c                   |     2 +-
 .../acpi/os_specific/service_layers/oslinuxtbl.c   |     2 +-
 .../acpi/os_specific/service_layers/osunixdir.c    |     2 +-
 .../acpi/os_specific/service_layers/osunixmap.c    |     2 +-
 .../acpi/os_specific/service_layers/osunixxf.c     |     2 +-
 tools/power/acpi/tools/acpidump/acpidump.h         |     2 +-
 tools/power/acpi/tools/acpidump/apdump.c           |     2 +-
 tools/power/acpi/tools/acpidump/apfiles.c          |     2 +-
 tools/power/acpi/tools/acpidump/apmain.c           |    30 +-
 tools/power/cpupower/lib/cpufreq.h                 |     4 -
 .../x86/intel_pstate_tracer/intel_pstate_tracer.py |    15 +
 tools/scripts/Makefile.include                     |    19 +
 tools/spi/Makefile                                 |     2 -
 tools/testing/nvdimm/Kbuild                        |     4 +
 tools/testing/nvdimm/acpi_nfit_test.c              |     8 +
 tools/testing/nvdimm/device_dax_test.c             |     8 +
 tools/testing/nvdimm/libnvdimm_test.c              |     8 +
 tools/testing/nvdimm/pmem_test.c                   |     8 +
 tools/testing/nvdimm/test/iomap.c                  |     7 +-
 tools/testing/nvdimm/test/nfit.c                   |   498 +-
 tools/testing/nvdimm/test/nfit_test.h              |   134 +
 tools/testing/nvdimm/watermark.h                   |    21 +
 tools/testing/radix-tree/idr-test.c                |    81 +-
 tools/testing/radix-tree/linux.c                   |    11 +-
 tools/testing/radix-tree/linux/compiler_types.h    |     0
 tools/testing/radix-tree/linux/gfp.h               |     1 +
 tools/testing/radix-tree/linux/kernel.h            |     2 -
 tools/testing/radix-tree/linux/slab.h              |     6 +
 tools/testing/selftests/Makefile                   |     9 +-
 tools/testing/selftests/android/Makefile           |     8 +-
 tools/testing/selftests/bpf/.gitignore             |     9 +
 tools/testing/selftests/bpf/Makefile               |    34 +-
 tools/testing/selftests/bpf/bpf_helpers.h          |     5 +-
 tools/testing/selftests/bpf/config                 |     2 +
 tools/testing/selftests/bpf/sample_map_ret0.c      |    34 +
 tools/testing/selftests/bpf/sample_ret0.c          |     7 +
 tools/testing/selftests/bpf/tcp_client.py          |    51 +
 tools/testing/selftests/bpf/tcp_server.py          |    83 +
 tools/testing/selftests/bpf/test_align.c           |   178 +-
 tools/testing/selftests/bpf/test_dev_cgroup.c      |    11 +-
 tools/testing/selftests/bpf/test_kmod.sh           |    18 +-
 tools/testing/selftests/bpf/test_l4lb_noinline.c   |   473 +
 tools/testing/selftests/bpf/test_libbpf.sh         |    49 +
 tools/testing/selftests/bpf/test_libbpf_open.c     |   150 +
 tools/testing/selftests/bpf/test_lpm_map.c         |   217 +
 tools/testing/selftests/bpf/test_maps.c            |    34 +-
 tools/testing/selftests/bpf/test_offload.py        |  1085 +
 tools/testing/selftests/bpf/test_progs.c           |   355 +-
 tools/testing/selftests/bpf/test_stacktrace_map.c  |    62 +
 tools/testing/selftests/bpf/test_tcpbpf.h          |    16 +
 tools/testing/selftests/bpf/test_tcpbpf_kern.c     |   114 +
 tools/testing/selftests/bpf/test_tcpbpf_user.c     |   126 +
 tools/testing/selftests/bpf/test_tracepoint.c      |    26 +
 tools/testing/selftests/bpf/test_verifier.c        |  2488 +-
 tools/testing/selftests/bpf/test_xdp_meta.sh       |     1 +
 tools/testing/selftests/bpf/test_xdp_noinline.c    |   833 +
 tools/testing/selftests/bpf/test_xdp_redirect.sh   |     2 +
 tools/testing/selftests/firmware/fw_fallback.sh    |   172 +-
 tools/testing/selftests/firmware/fw_filesystem.sh  |     5 +-
 .../ftrace/test.d/ftrace/func-filter-glob.tc       |     6 +
 .../ftrace/test.d/ftrace/func_set_ftrace_file.tc   |    37 +
 tools/testing/selftests/ftrace/test.d/functions    |    10 +-
 .../ftrace/test.d/kprobe/kprobe_args_string.tc     |    46 +
 .../ftrace/test.d/kprobe/kprobe_args_syntax.tc     |    97 +
 .../ftrace/test.d/kprobe/multiple_kprobes.tc       |    21 +-
 .../selftests/ftrace/test.d/kprobe/probepoint.tc   |    43 +
 tools/testing/selftests/futex/Makefile             |     6 +-
 tools/testing/selftests/gen_kselftest_tar.sh       |     4 +-
 tools/testing/selftests/kselftest.h                |     2 +-
 tools/testing/selftests/kselftest_install.sh       |     4 +-
 tools/testing/selftests/lib.mk                     |     2 +-
 tools/testing/selftests/media_tests/Makefile       |     7 +-
 .../selftests/media_tests/media_device_open.c      |     3 +-
 .../selftests/media_tests/media_device_test.c      |     3 +-
 .../selftests/media_tests/video_device_test.c      |     3 +-
 .../testing/selftests/membarrier/membarrier_test.c |   237 +-
 tools/testing/selftests/memfd/Makefile             |     6 +
 tools/testing/selftests/memfd/common.c             |    46 +
 tools/testing/selftests/memfd/common.h             |     9 +
 tools/testing/selftests/memfd/config               |     1 +
 tools/testing/selftests/memfd/fuse_test.c          |    44 +-
 tools/testing/selftests/memfd/memfd_test.c         |   212 +-
 tools/testing/selftests/memfd/run_fuse_test.sh     |     2 +-
 tools/testing/selftests/memfd/run_tests.sh         |     1 +
 tools/testing/selftests/memory-hotplug/Makefile    |     5 +-
 tools/testing/selftests/net/Makefile               |     1 +
 tools/testing/selftests/net/fib_tests.sh           |   429 +
 tools/testing/selftests/net/msg_zerocopy.c         |    21 +-
 tools/testing/selftests/net/reuseport_bpf.c        |    21 +-
 tools/testing/selftests/net/rtnetlink.sh           |   229 +
 tools/testing/selftests/nsfs/pidns.c               |     2 +-
 tools/testing/selftests/ntb/ntb_test.sh            |   307 +-
 tools/testing/selftests/powerpc/alignment/Makefile |     3 +-
 .../powerpc/alignment/alignment_handler.c          |   491 +
 .../selftests/powerpc/benchmarks/mmap_bench.c      |    53 +-
 tools/testing/selftests/powerpc/mm/.gitignore      |     3 +-
 tools/testing/selftests/powerpc/mm/Makefile        |     2 +-
 tools/testing/selftests/powerpc/mm/segv_errors.c   |    78 +
 tools/testing/selftests/powerpc/mm/subpage_prot.c  |    14 +
 .../selftests/powerpc/ptrace/ptrace-tm-spd-vsx.c   |     4 +-
 .../selftests/powerpc/ptrace/ptrace-tm-spr.c       |     3 +-
 .../selftests/powerpc/ptrace/ptrace-tm-vsx.c       |     2 +-
 tools/testing/selftests/powerpc/tm/.gitignore      |     1 +
 tools/testing/selftests/powerpc/tm/Makefile        |     5 +-
 tools/testing/selftests/powerpc/tm/tm-trap.c       |   331 +
 .../testing/selftests/powerpc/tm/tm-unavailable.c  |    43 +-
 tools/testing/selftests/pstore/config              |     1 +
 tools/testing/selftests/ptp/testptp.c              |     4 +-
 .../selftests/rcutorture/bin/config2frag.sh        |    25 -
 .../testing/selftests/rcutorture/bin/configinit.sh |     2 +-
 .../testing/selftests/rcutorture/bin/kvm-build.sh  |     5 -
 .../selftests/rcutorture/bin/kvm-recheck-lock.sh   |     2 +-
 .../selftests/rcutorture/bin/kvm-recheck-rcu.sh    |     4 +-
 .../rcutorture/bin/kvm-recheck-rcuperf-ftrace.sh   |     2 +-
 .../rcutorture/bin/kvm-recheck-rcuperf.sh          |     4 +-
 .../selftests/rcutorture/bin/kvm-recheck.sh        |     2 +-
 .../selftests/rcutorture/bin/kvm-test-1-run.sh     |     6 +-
 tools/testing/selftests/rcutorture/bin/kvm.sh      |    42 +-
 .../selftests/rcutorture/bin/parse-torture.sh      |     2 +-
 .../rcutorture/configs/lock/ver_functions.sh       |     1 -
 .../rcutorture/configs/rcu/ver_functions.sh        |     1 -
 .../rcutorture/configs/rcuperf/ver_functions.sh    |     1 -
 tools/testing/selftests/seccomp/seccomp_bpf.c      |    65 +-
 tools/testing/selftests/sync/Makefile              |     2 +-
 .../tc-testing/tc-tests/actions/skbmod.json        |     2 +-
 tools/testing/selftests/vDSO/Makefile              |    14 +-
 tools/testing/selftests/vm/.gitignore              |     2 +
 tools/testing/selftests/vm/Makefile                |     5 +-
 tools/testing/selftests/vm/compaction_test.c       |     2 +
 tools/testing/selftests/vm/run_vmtests             |    36 +-
 tools/testing/selftests/vm/va_128TBswitch.c        |   297 +
 tools/testing/selftests/x86/5lvl.c                 |   177 -
 tools/testing/selftests/x86/Makefile               |    34 +-
 tools/testing/selftests/x86/entry_from_vm86.c      |    32 +-
 tools/testing/selftests/x86/mpx-mini-test.c        |    32 +-
 tools/testing/selftests/x86/protection_keys.c      |    28 -
 tools/testing/selftests/x86/ptrace_syscall.c       |     8 +-
 tools/testing/selftests/x86/single_step_syscall.c  |     5 +-
 tools/testing/selftests/x86/test_mremap_vdso.c     |     4 +
 tools/testing/selftests/x86/test_vdso.c            |    55 +-
 tools/testing/selftests/x86/test_vsyscall.c        |    22 +-
 tools/usb/Makefile                                 |     1 -
 tools/usb/usbip/libsrc/usbip_device_driver.c       |     7 +-
 tools/usb/usbip/src/usbip_bind.c                   |     9 +
 tools/usb/usbip/src/usbip_list.c                   |     9 +
 tools/usb/usbip/src/usbipd.c                       |     2 +-
 tools/virtio/linux/kernel.h                        |     2 +-
 tools/virtio/linux/thread_info.h                   |     1 +
 tools/virtio/ringtest/main.h                       |    59 +-
 tools/virtio/ringtest/ptr_ring.c                   |     2 +-
 tools/virtio/ringtest/ring.c                       |    30 +-
 tools/virtio/ringtest/virtio_ring_0_9.c            |    24 +-
 tools/vm/Makefile                                  |     1 -
 tools/vm/page-types.c                              |    28 +-
 tools/wmi/Makefile                                 |     1 -
 virt/kvm/Kconfig                                   |     3 +
 virt/kvm/arm/arch_timer.c                          |   212 +-
 virt/kvm/arm/arm.c                                 |   181 +-
 virt/kvm/arm/hyp/vgic-v2-sr.c                      |     1 +
 virt/kvm/arm/hyp/vgic-v3-sr.c                      |     3 +-
 virt/kvm/arm/mmu.c                                 |    82 +-
 virt/kvm/arm/psci.c                                |   143 +-
 virt/kvm/arm/vgic/vgic-its.c                       |     4 +-
 virt/kvm/arm/vgic/vgic-mmio.c                      |   118 +-
 virt/kvm/arm/vgic/vgic-v2.c                        |    40 +-
 virt/kvm/arm/vgic/vgic-v3.c                        |    38 +-
 virt/kvm/arm/vgic/vgic.c                           |   128 +-
 virt/kvm/arm/vgic/vgic.h                           |    11 +
 virt/kvm/eventfd.c                                 |    12 +-
 virt/kvm/kvm_main.c                                |   117 +-
 12060 files changed, 508425 insertions(+), 314422 deletions(-)
 create mode 100644 Documentation/ABI/testing/devlink-resource-mlxsw
 create mode 100644 Documentation/ABI/testing/rtc-cdev
 create mode 100644 Documentation/ABI/testing/sysfs-block-device
 create mode 100644 Documentation/ABI/testing/sysfs-bus-iio-dfsdm-adc-stm32
 create mode 100644 Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd
 create mode 100644 Documentation/ABI/testing/sysfs-bus-siox
 create mode 100644 Documentation/ABI/testing/sysfs-class-led-trigger-netdev
 create mode 100644 Documentation/ABI/testing/sysfs-class-ocxl
 create mode 100644 Documentation/ABI/testing/sysfs-class-rtc
 create mode 100644 Documentation/ABI/testing/sysfs-devices-coredump
 create mode 100644 Documentation/ABI/testing/sysfs-devices-platform-dock
 create mode 100644 Documentation/ABI/testing/sysfs-platform-dptf
 create mode 100644 Documentation/accelerators/ocxl.rst
 create mode 100644 Documentation/bpf/bpf_devel_QA.txt
 rename Documentation/{ => core-api}/errseq.rst (92%)
 create mode 100644 Documentation/core-api/idr.rst
 rename Documentation/{printk-formats.txt => core-api/printk-formats.rst} (58%)
 create mode 100644 Documentation/core-api/refcount-vs-atomic.rst
 create mode 100644 Documentation/device-mapper/unstriped.txt
 create mode 100644 Documentation/devicetree/bindings/arm/arm-dsu-pmu.txt
 create mode 100644 Documentation/devicetree/bindings/arm/firmware/sdei.txt
 create mode 100644 Documentation/devicetree/bindings/arm/stm32.txt
 rename Documentation/devicetree/bindings/{misc => auxdisplay}/arm-charlcd.txt (100%)
 create mode 100644 Documentation/devicetree/bindings/clock/qcom,a53pll.txt
 create mode 100644 Documentation/devicetree/bindings/clock/qcom,spmi-clkdiv.txt
 create mode 100644 Documentation/devicetree/bindings/clock/sprd.txt
 create mode 100644 Documentation/devicetree/bindings/crypto/arm-cryptocell.txt
 create mode 100644 Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt
 create mode 100644 Documentation/devicetree/bindings/display/ilitek,ili9225.txt
 create mode 100644 Documentation/devicetree/bindings/display/panel/ilitek,ili9322.txt
 create mode 100644 Documentation/devicetree/bindings/display/panel/mitsubishi,aa070mc01.txt
 create mode 100644 Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.txt
 create mode 100644 Documentation/devicetree/bindings/display/panel/tianma,tm070rvhg71.txt
 rename Documentation/devicetree/bindings/display/panel/{toppoly,td028ttec1.txt => tpo,td028ttec1.txt} (84%)
 create mode 100644 Documentation/devicetree/bindings/display/sitronix,st7735r.txt
 create mode 100644 Documentation/devicetree/bindings/eeprom/at24.txt
 delete mode 100644 Documentation/devicetree/bindings/eeprom/eeprom.txt
 create mode 100644 Documentation/devicetree/bindings/iio/adc/sigma-delta-modulator.txt
 create mode 100644 Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt
 create mode 100644 Documentation/devicetree/bindings/iio/light/uvis25.txt
 create mode 100644 Documentation/devicetree/bindings/interrupt-controller/google,goldfish-pic.txt
 create mode 100644 Documentation/devicetree/bindings/leds/leds-lm3692x.txt
 create mode 100644 Documentation/devicetree/bindings/media/i2c/ov7740.txt
 create mode 100644 Documentation/devicetree/bindings/media/nvidia,tegra-vde.txt
 create mode 100644 Documentation/devicetree/bindings/mfd/atmel-tcb.txt
 create mode 100644 Documentation/devicetree/bindings/misc/nvidia,tegra186-misc.txt
 create mode 100644 Documentation/devicetree/bindings/mtd/marvell-nand.txt
 create mode 100644 Documentation/devicetree/bindings/net/can/can-transceiver.txt
 create mode 100644 Documentation/devicetree/bindings/net/cortina,gemini-ethernet.txt
 create mode 100644 Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt
 create mode 100644 Documentation/devicetree/bindings/net/socionext-netsec.txt
 rename Documentation/devicetree/bindings/net/{ti,wilink-st.txt => ti-bluetooth.txt} (64%)
 create mode 100644 Documentation/devicetree/bindings/net/wireless/mediatek,mt76.txt
 create mode 100644 Documentation/devicetree/bindings/opp/ti-omap5-opp-supply.txt
 create mode 100644 Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt
 create mode 100644 Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt
 create mode 100644 Documentation/devicetree/bindings/pinctrl/mscc,ocelot-pinctrl.txt
 create mode 100644 Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
 create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,msm8998-pinctrl.txt
 create mode 100644 Documentation/devicetree/bindings/pinctrl/socionext,uniphier-pinctrl.txt
 create mode 100644 Documentation/devicetree/bindings/power/mti,mips-cpc.txt
 delete mode 100644 Documentation/devicetree/bindings/power/reset/imx-snvs-poweroff.txt
 create mode 100644 Documentation/devicetree/bindings/regulator/sprd,sc2731-regulator.txt
 delete mode 100644 Documentation/devicetree/bindings/rng/brcm,bcm6368.txt
 create mode 100644 Documentation/devicetree/bindings/rtc/rtc-mxc_v2.txt
 create mode 100644 Documentation/devicetree/bindings/siox/eckelmann,siox-gpio.txt
 create mode 100644 Documentation/devicetree/bindings/slimbus/bus.txt
 create mode 100644 Documentation/devicetree/bindings/slimbus/slim-qcom-ctrl.txt
 create mode 100644 Documentation/devicetree/bindings/soc/xilinx/xlnx,vcu.txt
 create mode 100644 Documentation/devicetree/bindings/sound/max98373.txt
 create mode 100644 Documentation/devicetree/bindings/sound/pcm186x.txt
 create mode 100644 Documentation/devicetree/bindings/sound/st,stm32-adfsdm.txt
 create mode 100644 Documentation/devicetree/bindings/sound/ti,tas6424.txt
 create mode 100644 Documentation/devicetree/bindings/sound/tscs42xx.txt
 create mode 100644 Documentation/devicetree/bindings/sound/uniphier,evea.txt
 create mode 100644 Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt
 delete mode 100644 Documentation/devicetree/bindings/watchdog/cortina,gemini-watchdog.txt
 rename Documentation/devicetree/bindings/watchdog/{cortina,gemin-watchdog.txt => faraday,ftwdt010.txt} (55%)
 create mode 100644 Documentation/devicetree/bindings/watchdog/realtek,rtd119x.txt
 create mode 100644 Documentation/devicetree/bindings/watchdog/sprd-wdt.txt
 create mode 100644 Documentation/devicetree/bindings/watchdog/zii,rave-sp-wdt.txt
 create mode 100644 Documentation/driver-api/iio/hw-consumer.rst
 create mode 100644 Documentation/driver-api/slimbus.rst
 create mode 100644 Documentation/driver-api/soundwire/index.rst
 create mode 100644 Documentation/driver-api/soundwire/summary.rst
 create mode 100644 Documentation/features/sched/membarrier-sync-core/arch-support.txt
 create mode 100644 Documentation/fpga/fpga-region.txt
 create mode 100644 Documentation/fpga/overview.txt
 create mode 100644 Documentation/hwmon/w83773g
 create mode 100644 Documentation/i2c/DMA-considerations
 create mode 100644 Documentation/i2c/gpio-fault-injection
 delete mode 100644 Documentation/input/devices/gpio-tilt.rst
 delete mode 100644 Documentation/kernel-doc-nano-HOWTO.txt
 create mode 100644 Documentation/maintainer/conf.py
 create mode 100644 Documentation/maintainer/configure-git.rst
 create mode 100644 Documentation/maintainer/index.rst
 create mode 100644 Documentation/maintainer/pull-requests.rst
 create mode 100644 Documentation/media/uapi/dvb/dmx-expbuf.rst
 create mode 100644 Documentation/media/uapi/dvb/dmx-mmap.rst
 create mode 100644 Documentation/media/uapi/dvb/dmx-munmap.rst
 create mode 100644 Documentation/media/uapi/dvb/dmx-qbuf.rst
 create mode 100644 Documentation/media/uapi/dvb/dmx-querybuf.rst
 create mode 100644 Documentation/media/uapi/dvb/dmx-reqbufs.rst
 delete mode 100644 Documentation/media/uapi/rc/lirc-get-length.rst
 create mode 100644 Documentation/media/uapi/v4l/pixfmt-meta-uvc.rst
 create mode 100644 Documentation/media/uapi/v4l/pixfmt-srggb10-ipu3.rst
 create mode 100644 Documentation/networking/can.rst
 delete mode 100644 Documentation/networking/can.txt
 create mode 100644 Documentation/networking/xfrm_device.txt
 create mode 100644 Documentation/perf/arm_dsu_pmu.txt
 create mode 100644 Documentation/process/license-rules.rst
 create mode 100644 Documentation/process/maintainer-pgp-guide.rst
 create mode 100644 Documentation/sparc/oradax/dax-hv-api.txt
 create mode 100644 Documentation/sparc/oradax/oracle-dax.txt
 create mode 100644 Documentation/speculation.txt
 create mode 100644 Documentation/virtual/kvm/amd-memory-encryption.rst
 delete mode 100644 Documentation/virtual/kvm/arm/vgic-mapped-irqs.txt
 create mode 100644 LICENSES/exceptions/Linux-syscall-note
 create mode 100644 LICENSES/other/GPL-1.0
 create mode 100644 LICENSES/other/MPL-1.1
 create mode 100644 LICENSES/preferred/BSD-2-Clause
 create mode 100644 LICENSES/preferred/BSD-3-Clause
 create mode 100644 LICENSES/preferred/BSD-3-Clause-Clear
 create mode 100644 LICENSES/preferred/GPL-2.0
 create mode 100644 LICENSES/preferred/LGPL-2.0
 create mode 100644 LICENSES/preferred/LGPL-2.1
 create mode 100644 LICENSES/preferred/MIT
 delete mode 100644 arch/alpha/include/uapi/asm/poll.h
 create mode 100644 arch/arm/boot/dts/am572x-idk-common.dtsi
 create mode 100644 arch/arm/boot/dts/am574x-idk.dts
 create mode 100644 arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts
 create mode 100644 arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts
 create mode 100644 arch/arm/boot/dts/aspeed-bmc-quanta-q71l.dts
 create mode 100644 arch/arm/boot/dts/at91-natte.dtsi
 create mode 100644 arch/arm/boot/dts/at91-nattis-2-natte-2.dts
 create mode 100644 arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts
 create mode 100644 arch/arm/boot/dts/gemini-dlink-dns-313.dts
 create mode 100644 arch/arm/boot/dts/imx28-ts4600.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-cubox-i-emmc-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-cubox-i-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-hummingboard-emmc-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-hummingboard-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-hummingboard2-emmc-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-hummingboard2-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-hummingboard2.dts
 create mode 100644 arch/arm/boot/dts/imx6dl-ts7970.dts
 create mode 100644 arch/arm/boot/dts/imx6q-cubox-i-emmc-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6q-cubox-i-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6q-hummingboard-emmc-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6q-hummingboard-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6q-hummingboard2-emmc-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6q-hummingboard2-som-v15.dts
 create mode 100644 arch/arm/boot/dts/imx6q-hummingboard2.dts
 create mode 100644 arch/arm/boot/dts/imx6q-ts7970.dts
 create mode 100644 arch/arm/boot/dts/imx6q-var-dt6customboard.dts
 create mode 100644 arch/arm/boot/dts/imx6qdl-hummingboard2-emmc.dtsi
 create mode 100644 arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi
 rename arch/arm/boot/dts/{imx6qdl-microsom.dtsi => imx6qdl-sr-som-brcm.dtsi} (85%)
 create mode 100644 arch/arm/boot/dts/imx6qdl-sr-som-emmc.dtsi
 create mode 100644 arch/arm/boot/dts/imx6qdl-sr-som-ti.dtsi
 rename arch/arm/boot/dts/{imx6qdl-microsom-ar8035.dtsi => imx6qdl-sr-som.dtsi} (84%)
 create mode 100644 arch/arm/boot/dts/imx6qdl-ts7970.dtsi
 create mode 100644 arch/arm/boot/dts/imx6qdl-var-dart.dtsi
 create mode 100644 arch/arm/boot/dts/imx7d-colibri-emmc-eval-v3.dts
 create mode 100644 arch/arm/boot/dts/imx7d-colibri-emmc.dtsi
 create mode 100644 arch/arm/boot/dts/imx7d-pico-pi.dts
 rename arch/arm/boot/dts/{imx7d-pico.dts => imx7d-pico.dtsi} (71%)
 create mode 100644 arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts
 create mode 100644 arch/arm/boot/dts/logicpd-som-lv-baseboard.dtsi
 create mode 100644 arch/arm/boot/dts/logicpd-torpedo-35xx-devkit.dts
 create mode 100644 arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi
 create mode 100644 arch/arm/boot/dts/ls1021a-moxa-uc-8410a.dts
 create mode 100644 arch/arm/boot/dts/openbmc-flash-layout.dtsi
 create mode 100644 arch/arm/boot/dts/owl-s500-sparky.dts
 rename arch/arm/boot/dts/{wd-mbwe.dts => ox810se-wd-mbwe.dts} (100%)
 rename arch/arm/boot/dts/{cloudengines-pogoplug-series-3.dts => ox820-cloudengines-pogoplug-series-3.dts} (100%)
 create mode 100644 arch/arm/boot/dts/r8a7745-iwg22d-sodimm-dbhd-ca.dts
 create mode 100644 arch/arm/boot/dts/sun8i-h2-plus-orangepi-r1.dts
 create mode 100644 arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts
 create mode 100644 arch/arm/include/asm/dma-direct.h
 create mode 100644 arch/arm/include/asm/hardware/cache-b15-rac.h
 delete mode 100644 arch/arm/include/asm/kvm_psci.h
 create mode 100644 arch/arm/include/asm/ptdump.h
 create mode 100644 arch/arm/include/uapi/asm/siginfo.h
 delete mode 100644 arch/arm/lib/memzero.S
 delete mode 100644 arch/arm/mach-omap2/gpmc-onenand.c
 create mode 100644 arch/arm/mm/cache-b15-rac.c
 create mode 100644 arch/arm/mm/ptdump_debugfs.c
 create mode 100644 arch/arm64/boot/dts/actions/s700-cubieboard7.dts
 create mode 100644 arch/arm64/boot/dts/actions/s700.dtsi
 create mode 100644 arch/arm64/boot/dts/marvell/armada-common.dtsi
 delete mode 100644 arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi
 delete mode 100644 arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi
 create mode 100644 arch/arm64/boot/dts/marvell/armada-cp110.dtsi
 create mode 100644 arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts
 create mode 100644 arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
 create mode 100644 arch/arm64/crypto/aes-ce-core.S
 rename arch/arm64/crypto/{aes-ce-cipher.c => aes-ce-glue.c} (62%)
 create mode 100644 arch/arm64/crypto/sha3-ce-core.S
 create mode 100644 arch/arm64/crypto/sha3-ce-glue.c
 create mode 100644 arch/arm64/crypto/sha512-ce-core.S
 create mode 100644 arch/arm64/crypto/sha512-ce-glue.c
 create mode 100644 arch/arm64/crypto/sm3-ce-core.S
 create mode 100644 arch/arm64/crypto/sm3-ce-glue.c
 create mode 100644 arch/arm64/include/asm/arm_dsu_pmu.h
 delete mode 100644 arch/arm64/include/asm/kvm_psci.h
 create mode 100644 arch/arm64/include/asm/sdei.h
 create mode 100644 arch/arm64/include/asm/vmap_stack.h
 create mode 100644 arch/arm64/kernel/bpi.S
 create mode 100644 arch/arm64/kernel/sdei.c
 delete mode 100644 arch/blackfin/include/asm/clkdev.h
 delete mode 100644 arch/cris/arch-v32/drivers/pci/dma.c
 delete mode 100644 arch/cris/include/asm/dma-mapping.h
 delete mode 100644 arch/cris/kernel/devicetree.c
 delete mode 100644 arch/frv/include/uapi/asm/siginfo.h
 delete mode 100644 arch/h8300/include/asm/dma-mapping.h
 delete mode 100644 arch/h8300/kernel/dma.c
 delete mode 100644 arch/ia64/include/asm/swiotlb.h
 delete mode 100644 arch/ia64/include/uapi/asm/poll.h
 delete mode 100644 arch/ia64/kernel/init_task.c
 delete mode 100644 arch/ia64/kernel/pci-swiotlb.c
 delete mode 100644 arch/m32r/include/asm/dma-mapping.h
 delete mode 100644 arch/m32r/include/uapi/asm/poll.h
 create mode 100644 arch/mips/boot/dts/ingenic/gcw0.dts
 create mode 100644 arch/mips/boot/dts/ingenic/jz4770.dtsi
 create mode 100644 arch/mips/configs/gcw0_defconfig
 create mode 100644 arch/mips/configs/generic/board-ranchu.config
 create mode 100644 arch/mips/generic/board-ranchu.c
 create mode 100644 arch/mips/include/asm/dma-direct.h
 delete mode 100644 arch/mips/netlogic/common/nlm-dma.c
 delete mode 100644 arch/mn10300/include/uapi/asm/poll.h
 create mode 100644 arch/powerpc/include/asm/dma-direct.h
 create mode 100644 arch/powerpc/include/asm/drmem.h
 create mode 100644 arch/powerpc/include/asm/membarrier.h
 create mode 100644 arch/powerpc/include/asm/pkeys.h
 create mode 100644 arch/powerpc/include/asm/pnv-ocxl.h
 create mode 100644 arch/powerpc/kernel/module.lds
 create mode 100644 arch/powerpc/mm/drmem.c
 create mode 100644 arch/powerpc/mm/pkeys.c
 create mode 100644 arch/powerpc/platforms/44x/fsp2.h
 create mode 100644 arch/powerpc/platforms/powernv/ocxl.c
 delete mode 100644 arch/powerpc/sysdev/axonram.c
 create mode 100644 arch/powerpc/sysdev/cpm_gpio.c
 delete mode 100644 arch/riscv/include/asm/dma-mapping.h
 create mode 100644 arch/riscv/include/asm/ftrace.h
 create mode 100644 arch/riscv/kernel/ftrace.c
 create mode 100644 arch/riscv/kernel/mcount.S
 delete mode 100644 arch/s390/include/asm/dma-mapping.h
 create mode 100644 arch/s390/include/asm/dwarf.h
 create mode 100644 arch/s390/include/asm/nospec-branch.h
 create mode 100644 arch/s390/include/uapi/asm/runtime_instr.h
 create mode 100644 arch/s390/kernel/nospec-branch.c
 delete mode 100644 arch/s390/kernel/syscalls.S
 create mode 100644 arch/s390/kernel/syscalls/Makefile
 create mode 100644 arch/s390/kernel/syscalls/syscall.tbl
 create mode 100755 arch/s390/kernel/syscalls/syscalltbl
 delete mode 100644 arch/score/include/uapi/asm/poll.h
 create mode 100644 arch/sparc/include/uapi/asm/oradax.h
 create mode 100644 arch/um/include/asm/vmlinux.lds.h
 delete mode 100644 arch/unicore32/mm/dma-swiotlb.c
 create mode 100644 arch/x86/include/asm/dma-direct.h
 create mode 100644 arch/x86/include/asm/error-injection.h
 create mode 100644 arch/x86/include/asm/jailhouse_para.h
 delete mode 100644 arch/x86/include/asm/pmc_core.h
 create mode 100644 arch/x86/include/asm/sync_core.h
 delete mode 100644 arch/x86/include/uapi/asm/poll.h
 create mode 100644 arch/x86/kernel/jailhouse.c
 create mode 100644 arch/x86/kvm/vmx_shadow_fields.h
 create mode 100644 arch/x86/lib/error-inject.c
 rename arch/x86/tools/{test_get_len.c => insn_decoder_test.c} (81%)
 rename arch/x86/tools/{distill.awk => objdump_reformat.awk} (91%)
 create mode 100644 arch/xtensa/include/asm/kasan.h
 create mode 100644 arch/xtensa/include/asm/linkage.h
 create mode 100644 arch/xtensa/include/asm/stackprotector.h
 create mode 100644 arch/xtensa/mm/kasan_init.c
 delete mode 100644 drivers/ata/pata_at32.c
 create mode 100644 drivers/base/regmap/regmap-sdw.c
 create mode 100644 drivers/base/regmap/regmap-slimbus.c
 delete mode 100644 drivers/block/smart1,2.h
 delete mode 100644 drivers/char/hw_random/bcm63xx-rng.c
 create mode 100644 drivers/char/hw_random/exynos-trng.c
 delete mode 100644 drivers/char/hw_random/tpm-rng.c
 rename drivers/char/tpm/{tpm_acpi.c => tpm_eventlog_acpi.c} (97%)
 create mode 100644 drivers/char/tpm/tpm_eventlog_efi.c
 rename drivers/char/tpm/{tpm_of.c => tpm_eventlog_of.c} (93%)
 create mode 100644 drivers/clk/clk-aspeed.c
 create mode 100644 drivers/clk/hisilicon/clk-hi3660-stub.c
 create mode 100644 drivers/clk/ingenic/jz4770-cgu.c
 create mode 100644 drivers/clk/meson/axg.c
 create mode 100644 drivers/clk/meson/axg.h
 create mode 100644 drivers/clk/qcom/a53-pll.c
 create mode 100644 drivers/clk/qcom/apcs-msm8916.c
 create mode 100644 drivers/clk/qcom/clk-regmap-mux-div.c
 create mode 100644 drivers/clk/qcom/clk-regmap-mux-div.h
 create mode 100644 drivers/clk/qcom/clk-spmi-pmic-div.c
 create mode 100644 drivers/clk/sprd/Kconfig
 create mode 100644 drivers/clk/sprd/Makefile
 create mode 100644 drivers/clk/sprd/common.c
 create mode 100644 drivers/clk/sprd/common.h
 create mode 100644 drivers/clk/sprd/composite.c
 create mode 100644 drivers/clk/sprd/composite.h
 create mode 100644 drivers/clk/sprd/div.c
 create mode 100644 drivers/clk/sprd/div.h
 create mode 100644 drivers/clk/sprd/gate.c
 create mode 100644 drivers/clk/sprd/gate.h
 create mode 100644 drivers/clk/sprd/mux.c
 create mode 100644 drivers/clk/sprd/mux.h
 create mode 100644 drivers/clk/sprd/pll.c
 create mode 100644 drivers/clk/sprd/pll.h
 create mode 100644 drivers/clk/sprd/sc9860-clk.c
 delete mode 100644 drivers/clk/ti/clk-3xxx-legacy.c
 create mode 100644 drivers/clocksource/timer-sprd.c
 create mode 100644 drivers/cpufreq/armada-37xx-cpufreq.c
 delete mode 100644 drivers/cpufreq/at32ap-cpufreq.c
 create mode 100644 drivers/crypto/ccp/psp-dev.c
 create mode 100644 drivers/crypto/ccp/psp-dev.h
 create mode 100644 drivers/crypto/chelsio/chcr_ipsec.c
 create mode 100644 drivers/crypto/stm32/stm32-cryp.c
 create mode 100644 drivers/edac/ti_edac.c
 create mode 100644 drivers/firmware/arm_sdei.c
 create mode 100644 drivers/firmware/efi/cper-arm.c
 create mode 100644 drivers/firmware/efi/tpm.c
 create mode 100644 drivers/fpga/of-fpga-region.c
 delete mode 100644 drivers/gpio/gpio-axp209.c
 create mode 100644 drivers/gpio/gpio-pcie-idio-24.c
 create mode 100644 drivers/gpio/gpio-winbond.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
 mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
 create mode 100644 drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c
 create mode 100644 drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
 create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_crat.c
 create mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
 create mode 100644 drivers/gpu/drm/amd/display/dc/dc_link.h
 create mode 100644 drivers/gpu/drm/amd/display/dc/dc_stream.h
 create mode 100644 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
 create mode 100644 drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
 rename drivers/gpu/drm/amd/display/dc/dcn10/{dcn10_timing_generator.c => dcn10_optc.c} (72%)
 rename drivers/gpu/drm/amd/display/dc/dcn10/{dcn10_timing_generator.h => dcn10_optc.h} (76%)
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_offset.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_sh_mask.h
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/DC => dce}/dce_12_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/DC => dce}/dce_12_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/DCN => dcn}/dcn_1_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/DCN => dcn}/dcn_1_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/GC => gc}/gc_9_0_default.h (99%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/GC => gc}/gc_9_0_offset.h (99%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/GC => gc}/gc_9_0_sh_mask.h (99%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/GC => gc}/gc_9_1_offset.h (99%)
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_offset.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_sh_mask.h
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/MMHUB => mmhub}/mmhub_1_0_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/MMHUB => mmhub}/mmhub_1_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/MMHUB => mmhub}/mmhub_1_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/MMHUB => mmhub}/mmhub_9_1_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/MMHUB => mmhub}/mmhub_9_1_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/MP => mp}/mp_10_0_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/MP => mp}/mp_10_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/MP => mp}/mp_10_0_sh_mask.h (100%)
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_offset.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_sh_mask.h
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/NBIF => nbif}/nbif_6_1_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/NBIF => nbif}/nbif_6_1_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/NBIO => nbio}/nbio_6_1_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/NBIO => nbio}/nbio_6_1_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/NBIO => nbio}/nbio_6_1_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/NBIO => nbio}/nbio_7_0_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/NBIO => nbio}/nbio_7_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/NBIO => nbio}/nbio_7_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/OSSSYS => oss}/osssys_4_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/OSSSYS => oss}/osssys_4_0_sh_mask.h (100%)
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_sh_mask.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_sh_mask.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_default.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_default.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_offset.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_sh_mask.h
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/SDMA0 => sdma0}/sdma0_4_1_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/SDMA0 => sdma0}/sdma0_4_1_offset.h (100%)
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_default.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_offset.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_sh_mask.h
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/SMUIO => smuio}/smuio_9_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/SMUIO => smuio}/smuio_9_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/THM => thm}/thm_10_0_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/THM => thm}/thm_10_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/THM => thm}/thm_10_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/THM => thm}/thm_9_0_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/THM => thm}/thm_9_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/THM => thm}/thm_9_0_sh_mask.h (100%)
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_default.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_offset.h
 create mode 100644 drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_sh_mask.h
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/UVD => uvd}/uvd_7_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/UVD => uvd}/uvd_7_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/VCE => vce}/vce_4_0_default.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/VCE => vce}/vce_4_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{vega10/VCE => vce}/vce_4_0_sh_mask.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/VCN => vcn}/vcn_1_0_offset.h (100%)
 rename drivers/gpu/drm/amd/include/asic_reg/{raven1/VCN => vcn}/vcn_1_0_sh_mask.h (100%)
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_offset.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_sh_mask.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_offset.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_sh_mask.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_offset.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_sh_mask.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_offset.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_sh_mask.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_offset.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_sh_mask.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_default.h
 delete mode 100644 drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_default.h
 create mode 100644 drivers/gpu/drm/amd/include/kgd_pp_interface.h
 rename drivers/gpu/drm/amd/include/{asic_reg/vega10 => }/soc15ip.h (100%)
 rename drivers/gpu/drm/amd/include/{asic_reg/vega10 => }/vega10_enum.h (100%)
 delete mode 100644 drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
 rename drivers/{video/fbdev/core/fbcon_dmi_quirks.c => gpu/drm/drm_panel_orientation_quirks.c} (50%)
 delete mode 100644 drivers/gpu/drm/exynos/exynos_drm_ipp.c
 delete mode 100644 drivers/gpu/drm/exynos/exynos_drm_ipp.h
 delete mode 100644 drivers/gpu/drm/exynos/exynos_drm_rotator.h
 rename include/video/exynos5433_decon.h => drivers/gpu/drm/exynos/regs-decon5433.h (98%)
 rename include/video/exynos7_decon.h => drivers/gpu/drm/exynos/regs-decon7.h (99%)
 create mode 100644 drivers/gpu/drm/i915/gvt/debugfs.c
 create mode 100644 drivers/gpu/drm/i915/gvt/dmabuf.c
 create mode 100644 drivers/gpu/drm/i915/gvt/dmabuf.h
 create mode 100644 drivers/gpu/drm/i915/gvt/fb_decoder.c
 create mode 100644 drivers/gpu/drm/i915/gvt/fb_decoder.h
 create mode 100644 drivers/gpu/drm/i915/gvt/mmio_context.c
 rename drivers/gpu/drm/i915/gvt/{render.h => mmio_context.h} (91%)
 delete mode 100644 drivers/gpu/drm/i915/gvt/render.c
 create mode 100644 drivers/gpu/drm/i915/i915_oa_cflgt3.c
 create mode 100644 drivers/gpu/drm/i915/i915_oa_cflgt3.h
 create mode 100644 drivers/gpu/drm/i915/i915_oa_cnl.c
 create mode 100644 drivers/gpu/drm/i915/i915_oa_cnl.h
 create mode 100644 drivers/gpu/drm/i915/i915_pmu.c
 create mode 100644 drivers/gpu/drm/i915/i915_pmu.h
 create mode 100644 drivers/gpu/drm/i915/intel_device_info.h
 create mode 100644 drivers/gpu/drm/i915/intel_display.h
 rename drivers/gpu/drm/i915/{i915_guc_reg.h => intel_guc_reg.h} (92%)
 rename drivers/gpu/drm/i915/{i915_guc_submission.c => intel_guc_submission.c} (63%)
 rename drivers/gpu/drm/i915/{i915_guc_submission.h => intel_guc_submission.h} (85%)
 create mode 100644 drivers/gpu/drm/i915/intel_opregion.h
 create mode 100644 drivers/gpu/drm/i915/selftests/intel_guc.c
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk110.c
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.c
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.h
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c
 rename drivers/gpu/drm/{amd/display/dc/basics/grph_object_id.c => nouveau/nvkm/subdev/therm/gf100.c} (50%)
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c
 create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_connector.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_crtc.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_encoder.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_fb.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_fbdev.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_gem.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_irq.h
 create mode 100644 drivers/gpu/drm/omapdrm/omap_plane.h
 create mode 100644 drivers/gpu/drm/panel/panel-ilitek-ili9322.c
 create mode 100644 drivers/gpu/drm/scheduler/Makefile
 rename drivers/gpu/drm/{amd => }/scheduler/gpu_scheduler.c (55%)
 rename drivers/gpu/drm/{amd => }/scheduler/sched_fence.c (58%)
 create mode 100644 drivers/gpu/drm/sun4i/sun4i_lvds.c
 create mode 100644 drivers/gpu/drm/sun4i/sun4i_lvds.h
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_csc.c
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_csc.h
 delete mode 100644 drivers/gpu/drm/sun4i/sun8i_layer.c
 delete mode 100644 drivers/gpu/drm/sun4i/sun8i_layer.h
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_ui_layer.c
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_ui_layer.h
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_ui_scaler.c
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_ui_scaler.h
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_vi_layer.c
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_vi_layer.h
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_vi_scaler.c
 create mode 100644 drivers/gpu/drm/sun4i/sun8i_vi_scaler.h
 create mode 100644 drivers/gpu/drm/tegra/hub.c
 create mode 100644 drivers/gpu/drm/tegra/hub.h
 create mode 100644 drivers/gpu/drm/tegra/plane.c
 create mode 100644 drivers/gpu/drm/tegra/plane.h
 create mode 100644 drivers/gpu/drm/tinydrm/ili9225.c
 create mode 100644 drivers/gpu/drm/tinydrm/st7735r.c
 create mode 100644 drivers/hid/hid-jabra.c
 create mode 100644 drivers/hid/hid-quirks.c
 delete mode 100644 drivers/hid/usbhid/hid-quirks.c
 create mode 100644 drivers/hwmon/w83773g.c
 create mode 100644 drivers/iio/adc/sd_adc_modulator.c
 create mode 100644 drivers/iio/adc/stm32-dfsdm-adc.c
 create mode 100644 drivers/iio/adc/stm32-dfsdm-core.c
 create mode 100644 drivers/iio/adc/stm32-dfsdm.h
 create mode 100644 drivers/iio/buffer/industrialio-hw-consumer.c
 create mode 100644 drivers/iio/light/st_uvis25.h
 create mode 100644 drivers/iio/light/st_uvis25_core.c
 create mode 100644 drivers/iio/light/st_uvis25_i2c.c
 create mode 100644 drivers/iio/light/st_uvis25_spi.c
 create mode 100644 drivers/iio/light/zopt2201.c
 create mode 100644 drivers/infiniband/core/restrack.c
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_eq.c
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_eq.h
 delete mode 100644 drivers/infiniband/hw/mthca/mthca_user.h
 delete mode 100644 drivers/infiniband/hw/qib/qib_keys.c
 delete mode 100644 drivers/input/misc/gpio_tilt_polled.c
 delete mode 100644 drivers/input/serio/at32psif.c
 delete mode 100644 drivers/input/touchscreen/atmel-wm97xx.c
 create mode 100644 drivers/irqchip/irq-goldfish-pic.c
 create mode 100644 drivers/leds/leds-lm3692x.c
 create mode 100644 drivers/leds/trigger/ledtrig-netdev.c
 delete mode 100644 drivers/lightnvm/rrpc.c
 delete mode 100644 drivers/lightnvm/rrpc.h
 create mode 100644 drivers/md/dm-unstripe.c
 create mode 100644 drivers/media/common/videobuf2/Kconfig
 create mode 100644 drivers/media/common/videobuf2/Makefile
 rename drivers/media/{v4l2-core => common/videobuf2}/vb2-trace.c (100%)
 rename drivers/media/{v4l2-core => common/videobuf2}/videobuf2-core.c (96%)
 rename drivers/media/{v4l2-core => common/videobuf2}/videobuf2-dma-contig.c (99%)
 rename drivers/media/{v4l2-core => common/videobuf2}/videobuf2-dma-sg.c (99%)
 rename drivers/media/{v4l2-core => common/videobuf2}/videobuf2-dvb.c (100%)
 rename drivers/media/{v4l2-core => common/videobuf2}/videobuf2-memops.c (100%)
 rename drivers/media/{v4l2-core => common/videobuf2}/videobuf2-v4l2.c (98%)
 rename drivers/media/{v4l2-core => common/videobuf2}/videobuf2-vmalloc.c (100%)
 create mode 100644 drivers/media/dvb-core/dvb_vb2.c
 delete mode 100644 drivers/media/i2c/as3645a.c
 create mode 100644 drivers/media/i2c/ov7740.c
 create mode 100644 drivers/media/pci/ddbridge/ddbridge-ci.c
 create mode 100644 drivers/media/pci/ddbridge/ddbridge-ci.h
 rename drivers/media/pci/ddbridge/{ddbridge-maxs8.c => ddbridge-max.c} (92%)
 rename drivers/media/pci/ddbridge/{ddbridge-maxs8.h => ddbridge-max.h} (73%)
 create mode 100644 drivers/media/pci/intel/Makefile
 create mode 100644 drivers/media/pci/intel/ipu3/Kconfig
 create mode 100644 drivers/media/pci/intel/ipu3/Makefile
 create mode 100644 drivers/media/pci/intel/ipu3/ipu3-cio2.c
 create mode 100644 drivers/media/pci/intel/ipu3/ipu3-cio2.h
 delete mode 100644 drivers/media/rc/ir-lirc-codec.c
 create mode 100644 drivers/media/tuners/tda18250.c
 create mode 100644 drivers/media/tuners/tda18250.h
 create mode 100644 drivers/media/tuners/tda18250_priv.h
 create mode 100644 drivers/media/usb/uvc/uvc_metadata.c
 create mode 100644 drivers/memory/Makefile.asm-offsets
 create mode 100644 drivers/memory/emif-asm-offsets.c
 create mode 100644 drivers/memory/tegra/tegra186.c
 create mode 100644 drivers/memory/ti-emif-pm.c
 create mode 100644 drivers/memory/ti-emif-sram-pm.S
 rename drivers/{platform/chrome => mfd}/cros_ec_dev.c (99%)
 rename drivers/{platform/chrome => mfd}/cros_ec_dev.h (100%)
 create mode 100644 drivers/mfd/rave-sp.c
 create mode 100644 drivers/misc/cardreader/Kconfig
 create mode 100644 drivers/misc/cardreader/Makefile
 rename drivers/{mfd => misc/cardreader}/rtl8411.c (99%)
 rename drivers/{mfd => misc/cardreader}/rts5209.c (99%)
 rename drivers/{mfd => misc/cardreader}/rts5227.c (99%)
 rename drivers/{mfd => misc/cardreader}/rts5229.c (99%)
 rename drivers/{mfd => misc/cardreader}/rts5249.c (99%)
 create mode 100644 drivers/misc/cardreader/rts5260.c
 create mode 100644 drivers/misc/cardreader/rts5260.h
 rename drivers/{mfd => misc/cardreader}/rtsx_pcr.c (92%)
 rename drivers/{mfd => misc/cardreader}/rtsx_pcr.h (88%)
 rename drivers/{mfd => misc/cardreader}/rtsx_usb.c (99%)
 create mode 100644 drivers/misc/ocxl/Kconfig
 create mode 100644 drivers/misc/ocxl/Makefile
 create mode 100644 drivers/misc/ocxl/afu_irq.c
 create mode 100644 drivers/misc/ocxl/config.c
 create mode 100644 drivers/misc/ocxl/context.c
 create mode 100644 drivers/misc/ocxl/file.c
 create mode 100644 drivers/misc/ocxl/link.c
 create mode 100644 drivers/misc/ocxl/main.c
 create mode 100644 drivers/misc/ocxl/ocxl_internal.h
 create mode 100644 drivers/misc/ocxl/pasid.c
 create mode 100644 drivers/misc/ocxl/pci.c
 create mode 100644 drivers/misc/ocxl/sysfs.c
 create mode 100644 drivers/misc/ocxl/trace.c
 create mode 100644 drivers/misc/ocxl/trace.h
 create mode 100644 drivers/mmc/host/cqhci.c
 create mode 100644 drivers/mmc/host/cqhci.h
 create mode 100644 drivers/mmc/host/sdhci-pci-arasan.c
 create mode 100644 drivers/mtd/nand/marvell_nand.c
 delete mode 100644 drivers/net/ethernet/aquantia/atlantic/aq_nic_internal.h
 create mode 100644 drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
 create mode 100644 drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c
 create mode 100644 drivers/net/ethernet/cavium/common/Makefile
 create mode 100644 drivers/net/ethernet/cavium/common/cavium_ptp.c
 create mode 100644 drivers/net/ethernet/cavium/common/cavium_ptp.h
 create mode 100644 drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c
 create mode 100644 drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h
 create mode 100644 drivers/net/ethernet/cortina/Kconfig
 create mode 100644 drivers/net/ethernet/cortina/Makefile
 create mode 100644 drivers/net/ethernet/cortina/gemini.c
 create mode 100644 drivers/net/ethernet/cortina/gemini.h
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
 rename drivers/net/ethernet/hisilicon/hns3/{hns3pf => }/hns3_dcbnl.c (97%)
 rename drivers/net/ethernet/hisilicon/hns3/{hns3pf => }/hns3_enet.c (89%)
 rename drivers/net/ethernet/hisilicon/hns3/{hns3pf => }/hns3_enet.h (94%)
 rename drivers/net/ethernet/hisilicon/hns3/{hns3pf => }/hns3_ethtool.c (70%)
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3vf/Makefile
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
 create mode 100644 drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c
 create mode 100644 drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
 create mode 100644 drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h
 create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_dim.c
 delete mode 100644 drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c
 create mode 100644 drivers/net/ethernet/netronome/nfp/bpf/cmsg.c
 create mode 100644 drivers/net/ethernet/netronome/nfp/bpf/fw.h
 create mode 100644 drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.c
 create mode 100644 drivers/net/ethernet/netronome/nfp/nfp_net_debugdump.c
 create mode 100644 drivers/net/ethernet/socionext/Kconfig
 create mode 100644 drivers/net/ethernet/socionext/Makefile
 create mode 100644 drivers/net/ethernet/socionext/netsec.c
 create mode 100644 drivers/net/ethernet/socionext/sni_ave.c
 create mode 100644 drivers/net/netdevsim/Makefile
 create mode 100644 drivers/net/netdevsim/bpf.c
 create mode 100644 drivers/net/netdevsim/netdev.c
 create mode 100644 drivers/net/netdevsim/netdevsim.h
 create mode 100644 drivers/net/wireless/ath/ath10k/coredump.c
 create mode 100644 drivers/net/wireless/ath/ath10k/coredump.h
 create mode 100644 drivers/net/wireless/intel/iwlwifi/cfg/22000.c
 delete mode 100644 drivers/net/wireless/intel/iwlwifi/cfg/a000.c
 create mode 100644 drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
 create mode 100644 drivers/net/wireless/intel/iwlwifi/fw/debugfs.h
 create mode 100644 drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/Kconfig
 create mode 100644 drivers/net/wireless/mediatek/mt76/Makefile
 create mode 100644 drivers/net/wireless/mediatek/mt76/agg-rx.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/debugfs.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/dma.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/dma.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/eeprom.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mac80211.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mmio.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_core.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_debugfs.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_dfs.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_dfs.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_dma.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_dma.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_init.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_mac.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_mac.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_main.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_mcu.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_mcu.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_pci.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_phy.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_regs.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_trace.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_trace.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/mt76x2_tx.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/trace.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/trace.h
 create mode 100644 drivers/net/wireless/mediatek/mt76/tx.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/util.c
 create mode 100644 drivers/net/wireless/mediatek/mt76/util.h
 create mode 100644 drivers/nubus/bus.c
 create mode 100644 drivers/nvme/host/trace.c
 create mode 100644 drivers/nvme/host/trace.h
 delete mode 100644 drivers/of/of_pci.c
 delete mode 100644 drivers/of/of_pci_irq.c
 create mode 100644 drivers/opp/ti-opp-supply.c
 create mode 100644 drivers/pci/cadence/Kconfig
 create mode 100644 drivers/pci/cadence/Makefile
 create mode 100644 drivers/pci/cadence/pcie-cadence-ep.c
 create mode 100644 drivers/pci/cadence/pcie-cadence-host.c
 create mode 100644 drivers/pci/cadence/pcie-cadence.c
 create mode 100644 drivers/pci/cadence/pcie-cadence.h
 create mode 100644 drivers/perf/arm_dsu_pmu.c
 create mode 100644 drivers/pinctrl/mediatek/pinctrl-mt7622.c
 create mode 100644 drivers/pinctrl/meson/pinctrl-meson-axg-pmx.c
 create mode 100644 drivers/pinctrl/meson/pinctrl-meson-axg-pmx.h
 create mode 100644 drivers/pinctrl/meson/pinctrl-meson-axg.c
 create mode 100644 drivers/pinctrl/pinctrl-axp209.c
 create mode 100644 drivers/pinctrl/pinctrl-ocelot.c
 create mode 100644 drivers/pinctrl/qcom/pinctrl-msm8998.c
 create mode 100644 drivers/pinctrl/sh-pfc/pfc-r8a77970.c
 create mode 100644 drivers/pinctrl/stm32/pinctrl-stm32f769.c
 create mode 100644 drivers/pinctrl/stm32/pinctrl-stm32mp157.c
 delete mode 100644 drivers/platform/chrome/cros_ec_debugfs.h
 create mode 100644 drivers/platform/mellanox/Kconfig
 create mode 100644 drivers/platform/mellanox/Makefile
 create mode 100644 drivers/platform/mellanox/mlxreg-hotplug.c
 create mode 100644 drivers/platform/x86/acer-wireless.c
 rename drivers/platform/x86/{dell-smbios.c => dell-smbios-base.c} (95%)
 create mode 100644 drivers/platform/x86/gpd-pocket-fan.c
 create mode 100644 drivers/platform/x86/intel_chtdc_ti_pwrbtn.c
 delete mode 100644 drivers/platform/x86/mlxcpld-hotplug.c
 delete mode 100644 drivers/power/reset/imx-snvs-poweroff.c
 create mode 100644 drivers/regulator/sc2731-regulator.c
 delete mode 100644 drivers/rtc/rtc-at32ap700x.c
 create mode 100644 drivers/rtc/rtc-cros-ec.c
 create mode 100644 drivers/rtc/rtc-mxc_v2.c
 create mode 100644 drivers/sbus/char/oradax.c
 delete mode 100644 drivers/scsi/aic7xxx/aiclib.c
 create mode 100644 drivers/scsi/cxlflash/backend.h
 create mode 100644 drivers/scsi/cxlflash/cxl_hw.c
 create mode 100644 drivers/siox/Kconfig
 create mode 100644 drivers/siox/Makefile
 create mode 100644 drivers/siox/siox-bus-gpio.c
 create mode 100644 drivers/siox/siox-core.c
 create mode 100644 drivers/siox/siox.h
 create mode 100644 drivers/slimbus/Kconfig
 create mode 100644 drivers/slimbus/Makefile
 create mode 100644 drivers/slimbus/core.c
 create mode 100644 drivers/slimbus/messaging.c
 create mode 100644 drivers/slimbus/qcom-ctrl.c
 create mode 100644 drivers/slimbus/sched.c
 create mode 100644 drivers/slimbus/slimbus.h
 create mode 100644 drivers/soc/gemini/Makefile
 create mode 100644 drivers/soc/gemini/soc-gemini.c
 create mode 100644 drivers/soc/qcom/qmi_encdec.c
 create mode 100644 drivers/soc/qcom/qmi_interface.c
 delete mode 100644 drivers/soc/tegra/pmc-tegra186.c
 create mode 100644 drivers/soc/xilinx/Kconfig
 create mode 100644 drivers/soc/xilinx/Makefile
 create mode 100644 drivers/soc/xilinx/xlnx_vcu.c
 create mode 100644 drivers/soundwire/Kconfig
 create mode 100644 drivers/soundwire/Makefile
 create mode 100644 drivers/soundwire/bus.c
 create mode 100644 drivers/soundwire/bus.h
 create mode 100644 drivers/soundwire/bus_type.c
 create mode 100644 drivers/soundwire/cadence_master.c
 create mode 100644 drivers/soundwire/cadence_master.h
 create mode 100644 drivers/soundwire/intel.c
 create mode 100644 drivers/soundwire/intel.h
 create mode 100644 drivers/soundwire/intel_init.c
 create mode 100644 drivers/soundwire/mipi_disco.c
 create mode 100644 drivers/soundwire/slave.c
 delete mode 100644 drivers/staging/ccree/Documentation/devicetree/bindings/crypto/arm-cryptocell.txt
 rename drivers/staging/ccree/{ssi_aead.c => cc_aead.c} (70%)
 rename drivers/staging/ccree/{ssi_aead.h => cc_aead.h} (64%)
 rename drivers/staging/ccree/{ssi_buffer_mgr.c => cc_buffer_mgr.c} (56%)
 create mode 100644 drivers/staging/ccree/cc_buffer_mgr.h
 rename drivers/staging/ccree/{ssi_cipher.c => cc_cipher.c} (58%)
 create mode 100644 drivers/staging/ccree/cc_cipher.h
 create mode 100644 drivers/staging/ccree/cc_debugfs.c
 create mode 100644 drivers/staging/ccree/cc_debugfs.h
 rename drivers/staging/ccree/{ssi_driver.c => cc_driver.c} (52%)
 create mode 100644 drivers/staging/ccree/cc_driver.h
 rename drivers/staging/ccree/{ssi_fips.c => cc_fips.c} (60%)
 create mode 100644 drivers/staging/ccree/cc_fips.h
 rename drivers/staging/ccree/{ssi_hash.c => cc_hash.c} (52%)
 create mode 100644 drivers/staging/ccree/cc_hash.h
 create mode 100644 drivers/staging/ccree/cc_host_regs.h
 rename drivers/staging/ccree/{ssi_ivgen.c => cc_ivgen.c} (62%)
 create mode 100644 drivers/staging/ccree/cc_ivgen.h
 create mode 100644 drivers/staging/ccree/cc_kernel_regs.h
 create mode 100644 drivers/staging/ccree/cc_pm.c
 create mode 100644 drivers/staging/ccree/cc_pm.h
 create mode 100644 drivers/staging/ccree/cc_request_mgr.c
 create mode 100644 drivers/staging/ccree/cc_request_mgr.h
 rename drivers/staging/ccree/{ssi_sram_mgr.c => cc_sram_mgr.c} (50%)
 create mode 100644 drivers/staging/ccree/cc_sram_mgr.h
 delete mode 100644 drivers/staging/ccree/dx_crys_kernel.h
 delete mode 100644 drivers/staging/ccree/dx_host.h
 delete mode 100644 drivers/staging/ccree/dx_reg_common.h
 delete mode 100644 drivers/staging/ccree/hash_defs.h
 delete mode 100644 drivers/staging/ccree/ssi_buffer_mgr.h
 delete mode 100644 drivers/staging/ccree/ssi_cipher.h
 delete mode 100644 drivers/staging/ccree/ssi_config.h
 delete mode 100644 drivers/staging/ccree/ssi_driver.h
 delete mode 100644 drivers/staging/ccree/ssi_fips.h
 delete mode 100644 drivers/staging/ccree/ssi_hash.h
 delete mode 100644 drivers/staging/ccree/ssi_ivgen.h
 delete mode 100644 drivers/staging/ccree/ssi_pm.c
 delete mode 100644 drivers/staging/ccree/ssi_pm.h
 delete mode 100644 drivers/staging/ccree/ssi_request_mgr.c
 delete mode 100644 drivers/staging/ccree/ssi_request_mgr.h
 delete mode 100644 drivers/staging/ccree/ssi_sram_mgr.h
 delete mode 100644 drivers/staging/ccree/ssi_sysfs.c
 delete mode 100644 drivers/staging/ccree/ssi_sysfs.h
 delete mode 100644 drivers/staging/dgnc/dgnc_mgmt.c
 delete mode 100644 drivers/staging/dgnc/dgnc_mgmt.h
 delete mode 100644 drivers/staging/dgnc/dgnc_neo.c
 delete mode 100644 drivers/staging/dgnc/dgnc_neo.h
 delete mode 100644 drivers/staging/dgnc/dgnc_pci.h
 delete mode 100644 drivers/staging/dgnc/dgnc_utils.c
 delete mode 100644 drivers/staging/dgnc/dgnc_utils.h
 delete mode 100644 drivers/staging/fsl-mc/README.txt
 delete mode 100644 drivers/staging/fsl-mc/bus/dpmcp-cmd.h
 delete mode 100644 drivers/staging/fsl-mc/bus/dpmcp.h
 delete mode 100644 drivers/staging/fsl-mc/bus/dpmng-cmd.h
 delete mode 100644 drivers/staging/fsl-mc/bus/dprc-cmd.h
 delete mode 100644 drivers/staging/fsl-mc/bus/dprc.h
 create mode 100644 drivers/staging/fsl-mc/overview.rst
 rename {net => drivers/staging}/ipx/Kconfig (99%)
 rename {net => drivers/staging}/ipx/Makefile (100%)
 create mode 100644 drivers/staging/ipx/TODO
 rename {net => drivers/staging}/ipx/af_ipx.c (100%)
 rename {net => drivers/staging}/ipx/ipx_proc.c (99%)
 rename {net => drivers/staging}/ipx/ipx_route.c (100%)
 rename {net => drivers/staging}/ipx/pe2.c (100%)
 rename {net => drivers/staging}/ipx/sysctl_net_ipx.c (100%)
 delete mode 100644 drivers/staging/lustre/include/linux/libcfs/libcfs_workitem.h
 delete mode 100644 drivers/staging/lustre/lnet/libcfs/prng.c
 delete mode 100644 drivers/staging/lustre/lnet/libcfs/workitem.c
 delete mode 100644 drivers/staging/media/lirc/Kconfig
 delete mode 100644 drivers/staging/media/lirc/Makefile
 delete mode 100644 drivers/staging/media/lirc/TODO
 delete mode 100644 drivers/staging/media/lirc/lirc_zilog.c
 create mode 100644 drivers/staging/media/tegra-vde/Kconfig
 create mode 100644 drivers/staging/media/tegra-vde/Makefile
 create mode 100644 drivers/staging/media/tegra-vde/TODO
 create mode 100644 drivers/staging/media/tegra-vde/tegra-vde.c
 create mode 100644 drivers/staging/media/tegra-vde/uapi.h
 create mode 100644 drivers/staging/most/Documentation/ABI/sysfs-bus-most.txt
 delete mode 100644 drivers/staging/most/aim-cdev/Makefile
 delete mode 100644 drivers/staging/most/aim-network/Makefile
 delete mode 100644 drivers/staging/most/aim-sound/Makefile
 delete mode 100644 drivers/staging/most/aim-v4l2/Makefile
 rename drivers/staging/most/{aim-cdev => cdev}/Kconfig (70%)
 create mode 100644 drivers/staging/most/cdev/Makefile
 rename drivers/staging/most/{aim-cdev => cdev}/cdev.c (67%)
 create mode 100644 drivers/staging/most/core.c
 rename drivers/staging/most/{mostcore/mostcore.h => core.h} (84%)
 rename drivers/staging/most/{hdm-dim2 => dim2}/Kconfig (83%)
 create mode 100644 drivers/staging/most/dim2/Makefile
 rename drivers/staging/most/{hdm-dim2/dim2_hdm.c => dim2/dim2.c} (96%)
 rename drivers/staging/most/{hdm-dim2/dim2_hdm.h => dim2/dim2.h} (54%)
 rename drivers/staging/most/{hdm-dim2/dim2_errors.h => dim2/errors.h} (73%)
 rename drivers/staging/most/{hdm-dim2/dim2_hal.c => dim2/hal.c} (98%)
 rename drivers/staging/most/{hdm-dim2/dim2_hal.h => dim2/hal.h} (88%)
 rename drivers/staging/most/{hdm-dim2/dim2_reg.h => dim2/reg.h} (89%)
 create mode 100644 drivers/staging/most/dim2/sysfs.c
 create mode 100644 drivers/staging/most/dim2/sysfs.h
 delete mode 100644 drivers/staging/most/hdm-dim2/Makefile
 delete mode 100644 drivers/staging/most/hdm-dim2/dim2_sysfs.c
 delete mode 100644 drivers/staging/most/hdm-dim2/dim2_sysfs.h
 delete mode 100644 drivers/staging/most/hdm-i2c/Makefile
 delete mode 100644 drivers/staging/most/hdm-usb/Makefile
 rename drivers/staging/most/{hdm-i2c => i2c}/Kconfig (72%)
 create mode 100644 drivers/staging/most/i2c/Makefile
 rename drivers/staging/most/{hdm-i2c/hdm_i2c.c => i2c/i2c.c} (95%)
 delete mode 100644 drivers/staging/most/mostcore/Kconfig
 delete mode 100644 drivers/staging/most/mostcore/Makefile
 delete mode 100644 drivers/staging/most/mostcore/core.c
 rename drivers/staging/most/{aim-network => net}/Kconfig (69%)
 create mode 100644 drivers/staging/most/net/Makefile
 rename drivers/staging/most/{aim-network/networking.c => net/net.c} (87%)
 rename drivers/staging/most/{aim-sound => sound}/Kconfig (72%)
 create mode 100644 drivers/staging/most/sound/Makefile
 rename drivers/staging/most/{aim-sound => sound}/sound.c (95%)
 rename drivers/staging/most/{hdm-usb => usb}/Kconfig (77%)
 create mode 100644 drivers/staging/most/usb/Makefile
 rename drivers/staging/most/{hdm-usb/hdm_usb.c => usb/usb.c} (85%)
 rename drivers/staging/most/{aim-v4l2 => video}/Kconfig (71%)
 create mode 100644 drivers/staging/most/video/Makefile
 rename drivers/staging/most/{aim-v4l2 => video}/video.c (74%)
 rename {fs => drivers/staging}/ncpfs/Kconfig (100%)
 rename {fs => drivers/staging}/ncpfs/Makefile (100%)
 create mode 100644 drivers/staging/ncpfs/TODO
 rename {fs => drivers/staging}/ncpfs/dir.c (100%)
 rename {fs => drivers/staging}/ncpfs/file.c (100%)
 rename {fs => drivers/staging}/ncpfs/getopt.c (100%)
 rename {fs => drivers/staging}/ncpfs/getopt.h (100%)
 rename {fs => drivers/staging}/ncpfs/inode.c (99%)
 rename {fs => drivers/staging}/ncpfs/ioctl.c (100%)
 rename {fs => drivers/staging}/ncpfs/mmap.c (100%)
 rename {fs => drivers/staging}/ncpfs/ncp_fs.h (100%)
 rename {fs => drivers/staging}/ncpfs/ncp_fs_i.h (100%)
 rename {fs => drivers/staging}/ncpfs/ncp_fs_sb.h (100%)
 rename {fs => drivers/staging}/ncpfs/ncplib_kernel.c (99%)
 rename {fs => drivers/staging}/ncpfs/ncplib_kernel.h (100%)
 rename {fs => drivers/staging}/ncpfs/ncpsign_kernel.c (100%)
 rename {fs => drivers/staging}/ncpfs/ncpsign_kernel.h (100%)
 rename {fs => drivers/staging}/ncpfs/sock.c (99%)
 rename {fs => drivers/staging}/ncpfs/symlink.c (100%)
 delete mode 100644 drivers/staging/unisys/include/visorchannel.h
 create mode 100644 drivers/tee/optee/shm_pool.c
 create mode 100644 drivers/tee/optee/shm_pool.h
 create mode 100644 drivers/usb/host/xhci-dbgcap.c
 create mode 100644 drivers/usb/host/xhci-dbgcap.h
 create mode 100644 drivers/usb/host/xhci-dbgtty.c
 create mode 100644 drivers/usb/renesas_usbhs/rza.c
 create mode 100644 drivers/usb/renesas_usbhs/rza.h
 create mode 100644 drivers/virt/vboxguest/Kconfig
 create mode 100644 drivers/virt/vboxguest/Makefile
 create mode 100644 drivers/virt/vboxguest/vboxguest_core.c
 create mode 100644 drivers/virt/vboxguest/vboxguest_core.h
 create mode 100644 drivers/virt/vboxguest/vboxguest_linux.c
 create mode 100644 drivers/virt/vboxguest/vboxguest_utils.c
 create mode 100644 drivers/virt/vboxguest/vboxguest_version.h
 create mode 100644 drivers/virt/vboxguest/vmmdev.h
 rename drivers/{staging/unisys => }/visorbus/Kconfig (94%)
 rename drivers/{staging/unisys => }/visorbus/Makefile (81%)
 rename drivers/{staging/unisys => }/visorbus/controlvmchannel.h (97%)
 rename drivers/{staging/unisys => }/visorbus/vbuschannel.h (85%)
 rename drivers/{staging/unisys => }/visorbus/visorbus_main.c (98%)
 rename drivers/{staging/unisys => }/visorbus/visorbus_private.h (78%)
 rename drivers/{staging/unisys => }/visorbus/visorchannel.c (95%)
 rename drivers/{staging/unisys => }/visorbus/visorchipset.c (98%)
 delete mode 100644 drivers/watchdog/at32ap700x_wdt.c
 create mode 100644 drivers/watchdog/ftwdt010_wdt.c
 delete mode 100644 drivers/watchdog/gemini_wdt.c
 create mode 100644 drivers/watchdog/rave-sp-wdt.c
 create mode 100644 drivers/watchdog/rtd119x_wdt.c
 create mode 100644 drivers/watchdog/sprd_wdt.c
 create mode 100644 fs/btrfs/tests/extent-map-tests.c
 create mode 100644 fs/cifs/smbdirect.c
 create mode 100644 fs/cifs/smbdirect.h
 create mode 100644 fs/overlayfs/export.c
 delete mode 100644 include/asm-generic/clkdev.h
 create mode 100644 include/asm-generic/dma-mapping.h
 create mode 100644 include/asm-generic/error-injection.h
 create mode 100644 include/crypto/salsa20.h
 create mode 100644 include/drm/drm_utils.h
 create mode 100644 include/drm/gpu_scheduler.h
 rename drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h => include/drm/gpu_scheduler_trace.h (84%)
 create mode 100644 include/drm/spsc_queue.h
 create mode 100644 include/dt-bindings/bus/ti-sysc.h
 create mode 100644 include/dt-bindings/clock/am3.h
 create mode 100644 include/dt-bindings/clock/am4.h
 create mode 100644 include/dt-bindings/clock/aspeed-clock.h
 create mode 100644 include/dt-bindings/clock/axg-clkc.h
 create mode 100644 include/dt-bindings/clock/dm814.h
 create mode 100644 include/dt-bindings/clock/dm816.h
 create mode 100644 include/dt-bindings/clock/dra7.h
 create mode 100644 include/dt-bindings/clock/jz4770-cgu.h
 create mode 100644 include/dt-bindings/clock/omap5.h
 create mode 100644 include/dt-bindings/clock/sprd,sc9860-clk.h
 create mode 100644 include/dt-bindings/gpio/aspeed-gpio.h
 create mode 100644 include/dt-bindings/gpio/meson-axg-gpio.h
 create mode 100644 include/dt-bindings/memory/tegra186-mc.h
 create mode 100644 include/dt-bindings/power/mt2712-power.h
 create mode 100644 include/dt-bindings/power/owl-s700-powergate.h
 create mode 100644 include/dt-bindings/reset/amlogic,meson-axg-reset.h
 create mode 100644 include/kvm/arm_psci.h
 create mode 100644 include/linux/arm_sdei.h
 create mode 100644 include/linux/dma-direct.h
 create mode 100644 include/linux/error-injection.h
 create mode 100644 include/linux/fpga/fpga-region.h
 delete mode 100644 include/linux/i7300_idle.h
 create mode 100644 include/linux/iio/adc/stm32-dfsdm-adc.h
 create mode 100644 include/linux/iio/hw-consumer.h
 delete mode 100644 include/linux/input/gpio_tilt.h
 create mode 100644 include/linux/iversion.h
 create mode 100644 include/linux/mfd/rave-sp.h
 create mode 100644 include/linux/net_dim.h
 create mode 100644 include/linux/nospec.h
 rename include/linux/{i2c/pxa-i2c.h => platform_data/i2c-pxa.h} (90%)
 delete mode 100644 include/linux/platform_data/mlxcpld-hotplug.h
 create mode 100644 include/linux/platform_data/mlxreg.h
 delete mode 100644 include/linux/platform_data/mms114.h
 delete mode 100644 include/linux/platform_data/mtd-onenand-omap2.h
 create mode 100644 include/linux/platform_data/ti-sysc.h
 create mode 100644 include/linux/psp-sev.h
 rename include/linux/{mfd => }/rtsx_common.h (100%)
 rename include/linux/{mfd => }/rtsx_pci.h (83%)
 rename include/linux/{mfd => }/rtsx_usb.h (100%)
 create mode 100644 include/linux/siox.h
 create mode 100644 include/linux/slimbus.h
 create mode 100644 include/linux/soc/qcom/qmi.h
 create mode 100644 include/linux/soundwire/sdw.h
 create mode 100644 include/linux/soundwire/sdw_intel.h
 create mode 100644 include/linux/soundwire/sdw_registers.h
 create mode 100644 include/linux/soundwire/sdw_type.h
 create mode 100644 include/linux/sync_core.h
 create mode 100644 include/linux/ti-emif-sram.h
 rename {drivers/char/tpm => include/linux}/tpm_eventlog.h (78%)
 create mode 100644 include/linux/vbox_utils.h
 rename {drivers/staging/unisys/include => include/linux}/visorbus.h (52%)
 rename {drivers/media/dvb-core => include/media}/demux.h (97%)
 rename {drivers/media/dvb-core => include/media}/dmxdev.h (81%)
 rename {drivers/media/dvb-core => include/media}/dvb-usb-ids.h (99%)
 rename {drivers/media/dvb-core => include/media}/dvb_ca_en50221.h (99%)
 rename {drivers/media/dvb-core => include/media}/dvb_demux.h (93%)
 rename {drivers/media/dvb-core => include/media}/dvb_frontend.h (98%)
 rename {drivers/media/dvb-core => include/media}/dvb_math.h (100%)
 rename {drivers/media/dvb-core => include/media}/dvb_net.h (98%)
 rename {drivers/media/dvb-core => include/media}/dvb_ringbuffer.h (100%)
 create mode 100644 include/media/dvb_vb2.h
 rename {drivers/media/dvb-core => include/media}/dvbdev.h (99%)
 delete mode 100644 include/media/i2c-addr.h
 delete mode 100644 include/media/i2c/as3645a.h
 delete mode 100644 include/media/lirc.h
 delete mode 100644 include/media/lirc_dev.h
 rename include/media/{ => tpg}/v4l2-tpg.h (93%)
 delete mode 100644 include/media/v4l2-tpg-colors.h
 create mode 100644 include/misc/ocxl-config.h
 create mode 100644 include/misc/ocxl.h
 create mode 100644 include/net/netfilter/nf_conntrack_count.h
 create mode 100644 include/net/netfilter/nf_flow_table.h
 create mode 100644 include/net/sctp/stream_interleave.h
 create mode 100644 include/net/xdp.h
 create mode 100644 include/rdma/restrack.h
 create mode 100644 include/trace/events/net_probe_common.h
 create mode 100644 include/trace/events/rdma.h
 create mode 100644 include/trace/events/rpcrdma.h
 create mode 100644 include/trace/events/sctp.h
 create mode 100644 include/trace/events/siox.h
 create mode 100644 include/uapi/linux/arm_sdei.h
 rename net/batman-adv/packet.h => include/uapi/linux/batadv_packet.h (80%)
 create mode 100644 include/uapi/linux/erspan.h
 create mode 100644 include/uapi/linux/netfilter_ipv6/ip6t_srh.h
 create mode 100644 include/uapi/linux/psp-sev.h
 create mode 100644 include/uapi/linux/vbox_err.h
 create mode 100644 include/uapi/linux/vbox_vmmdev_types.h
 create mode 100644 include/uapi/linux/vboxguest.h
 create mode 100644 include/uapi/misc/ocxl.h
 create mode 100644 kernel/configs/nopm.config
 create mode 100644 kernel/fail_function.c
 create mode 100644 lib/bucket_locks.c
 create mode 100644 lib/dma-direct.c
 delete mode 100644 lib/dma-noop.c
 create mode 100644 lib/error-inject.c
 rename lib/{test_find_bit.c => find_bit_benchmark.c} (69%)
 create mode 100644 net/core/xdp.c
 delete mode 100644 net/dccp/probe.c
 create mode 100644 net/dccp/trace.h
 create mode 100644 net/ipv4/netfilter/nf_flow_table_ipv4.c
 create mode 100644 net/ipv4/netfilter/nf_nat_snmp_basic.asn1
 delete mode 100644 net/ipv4/netfilter/nf_nat_snmp_basic.c
 create mode 100644 net/ipv4/netfilter/nf_nat_snmp_basic_main.c
 delete mode 100644 net/ipv4/tcp_probe.c
 create mode 100644 net/ipv6/netfilter/ip6t_srh.c
 create mode 100644 net/ipv6/netfilter/nf_flow_table_ipv6.c
 create mode 100644 net/netfilter/nf_conncount.c
 create mode 100644 net/netfilter/nf_flow_table.c
 create mode 100644 net/netfilter/nf_flow_table_inet.c
 create mode 100644 net/netfilter/nft_flow_offload.c
 create mode 100644 net/netfilter/utils.c
 delete mode 100644 net/sctp/probe.c
 create mode 100644 net/sctp/stream_interleave.c
 create mode 100755 samples/bpf/test_override_return.sh
 create mode 100644 samples/bpf/tracex7_kern.c
 create mode 100644 samples/bpf/tracex7_user.c
 create mode 100755 samples/bpf/xdp2skb_meta.sh
 create mode 100644 samples/bpf/xdp2skb_meta_kern.c
 create mode 100644 samples/bpf/xdp_rxq_info_kern.c
 create mode 100644 samples/bpf/xdp_rxq_info_user.c
 delete mode 100644 scripts/coccinelle/api/alloc/kzalloc-simple.cocci
 create mode 100644 scripts/coccinelle/api/alloc/zalloc-simple.cocci
 delete mode 100644 scripts/kconfig/zconf.lex.c_shipped
 delete mode 100644 scripts/kconfig/zconf.tab.c_shipped
 create mode 100644 scripts/package/snapcraft.template
 create mode 100644 sound/soc/codecs/max98373.c
 create mode 100644 sound/soc/codecs/max98373.h
 create mode 100644 sound/soc/codecs/pcm186x-i2c.c
 create mode 100644 sound/soc/codecs/pcm186x-spi.c
 create mode 100644 sound/soc/codecs/pcm186x.c
 create mode 100644 sound/soc/codecs/pcm186x.h
 delete mode 100644 sound/soc/codecs/sn95031.c
 delete mode 100644 sound/soc/codecs/sn95031.h
 create mode 100644 sound/soc/codecs/tas6424.c
 create mode 100644 sound/soc/codecs/tas6424.h
 create mode 100644 sound/soc/codecs/tscs42xx.c
 create mode 100644 sound/soc/codecs/tscs42xx.h
 delete mode 100644 sound/soc/intel/boards/mfld_machine.c
 create mode 100644 sound/soc/intel/skylake/skl-i2s.h
 create mode 100644 sound/soc/intel/skylake/skl-ssp-clk.c
 create mode 100644 sound/soc/intel/skylake/skl-ssp-clk.h
 create mode 100644 sound/soc/stm/stm32_adfsdm.c
 create mode 100644 sound/soc/uniphier/Kconfig
 create mode 100644 sound/soc/uniphier/Makefile
 create mode 100644 sound/soc/uniphier/evea.c
 create mode 100644 tools/arch/alpha/include/uapi/asm/errno.h
 create mode 100644 tools/arch/mips/include/asm/errno.h
 create mode 100644 tools/arch/mips/include/uapi/asm/errno.h
 create mode 100644 tools/arch/parisc/include/uapi/asm/errno.h
 create mode 100644 tools/arch/powerpc/include/uapi/asm/errno.h
 create mode 100644 tools/arch/sparc/include/uapi/asm/errno.h
 create mode 100644 tools/arch/x86/include/uapi/asm/errno.h
 create mode 100644 tools/bpf/bpftool/Documentation/bpftool-cgroup.rst
 create mode 100644 tools/bpf/bpftool/cgroup.c
 create mode 100644 tools/build/feature/test-disassembler-four-args.c
 create mode 100644 tools/build/feature/test-libopencsd.c
 create mode 100644 tools/build/feature/test-pthread-barrier.c
 create mode 100644 tools/include/uapi/asm-generic/errno-base.h
 create mode 100644 tools/include/uapi/asm-generic/errno.h
 create mode 100644 tools/include/uapi/linux/if_link.h
 create mode 100644 tools/include/uapi/linux/netlink.h
 create mode 100644 tools/lib/bpf/nlattr.c
 create mode 100644 tools/lib/bpf/nlattr.h
 create mode 100644 tools/perf/arch/arm64/util/arm-spe.c
 create mode 100644 tools/perf/arch/arm64/util/header.c
 create mode 100644 tools/perf/arch/arm64/util/sym-handling.c
 create mode 100755 tools/perf/arch/s390/entry/syscalls/mksyscalltbl
 create mode 100644 tools/perf/arch/s390/entry/syscalls/syscall.tbl
 create mode 100644 tools/perf/pmu-events/arch/arm64/cavium/thunderx2-imp-def.json
 create mode 100644 tools/perf/pmu-events/arch/arm64/cortex-a53/branch.json
 create mode 100644 tools/perf/pmu-events/arch/arm64/cortex-a53/bus.json
 create mode 100644 tools/perf/pmu-events/arch/arm64/cortex-a53/cache.json
 create mode 100644 tools/perf/pmu-events/arch/arm64/cortex-a53/memory.json
 create mode 100644 tools/perf/pmu-events/arch/arm64/cortex-a53/other.json
 create mode 100644 tools/perf/pmu-events/arch/arm64/cortex-a53/pipeline.json
 create mode 100644 tools/perf/pmu-events/arch/arm64/mapfile.csv
 create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record
 create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report
 create mode 100644 tools/perf/scripts/python/mem-phys-addr.py
 create mode 100644 tools/perf/trace/beauty/arch_errno_names.c
 create mode 100755 tools/perf/trace/beauty/arch_errno_names.sh
 create mode 100644 tools/perf/trace/beauty/futex_val3.c
 create mode 100644 tools/perf/util/arm-spe-pkt-decoder.c
 create mode 100644 tools/perf/util/arm-spe-pkt-decoder.h
 create mode 100644 tools/perf/util/arm-spe.c
 create mode 100644 tools/perf/util/arm-spe.h
 create mode 100644 tools/perf/util/cs-etm-decoder/Build
 create mode 100644 tools/perf/util/cs-etm-decoder/cs-etm-decoder.c
 create mode 100644 tools/perf/util/cs-etm-decoder/cs-etm-decoder.h
 create mode 100644 tools/perf/util/cs-etm.c
 create mode 100644 tools/testing/nvdimm/acpi_nfit_test.c
 create mode 100644 tools/testing/nvdimm/device_dax_test.c
 create mode 100644 tools/testing/nvdimm/libnvdimm_test.c
 create mode 100644 tools/testing/nvdimm/pmem_test.c
 create mode 100644 tools/testing/nvdimm/watermark.h
 create mode 100644 tools/testing/radix-tree/linux/compiler_types.h
 create mode 100644 tools/testing/selftests/bpf/sample_map_ret0.c
 create mode 100644 tools/testing/selftests/bpf/sample_ret0.c
 create mode 100755 tools/testing/selftests/bpf/tcp_client.py
 create mode 100755 tools/testing/selftests/bpf/tcp_server.py
 create mode 100644 tools/testing/selftests/bpf/test_l4lb_noinline.c
 create mode 100755 tools/testing/selftests/bpf/test_libbpf.sh
 create mode 100644 tools/testing/selftests/bpf/test_libbpf_open.c
 create mode 100755 tools/testing/selftests/bpf/test_offload.py
 create mode 100644 tools/testing/selftests/bpf/test_stacktrace_map.c
 create mode 100644 tools/testing/selftests/bpf/test_tcpbpf.h
 create mode 100644 tools/testing/selftests/bpf/test_tcpbpf_kern.c
 create mode 100644 tools/testing/selftests/bpf/test_tcpbpf_user.c
 create mode 100644 tools/testing/selftests/bpf/test_tracepoint.c
 create mode 100644 tools/testing/selftests/bpf/test_xdp_noinline.c
 create mode 100644 tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_string.tc
 create mode 100644 tools/testing/selftests/ftrace/test.d/kprobe/kprobe_args_syntax.tc
 create mode 100644 tools/testing/selftests/ftrace/test.d/kprobe/probepoint.tc
 create mode 100644 tools/testing/selftests/memfd/common.c
 create mode 100644 tools/testing/selftests/memfd/common.h
 create mode 100644 tools/testing/selftests/memfd/config
 create mode 100755 tools/testing/selftests/net/fib_tests.sh
 create mode 100644 tools/testing/selftests/powerpc/alignment/alignment_handler.c
 create mode 100644 tools/testing/selftests/powerpc/mm/segv_errors.c
 create mode 100644 tools/testing/selftests/powerpc/tm/tm-trap.c
 delete mode 100755 tools/testing/selftests/rcutorture/bin/config2frag.sh
 create mode 100644 tools/testing/selftests/vm/va_128TBswitch.c
 delete mode 100644 tools/testing/selftests/x86/5lvl.c
 create mode 100644 tools/virtio/linux/thread_info.h

diff --git a/.gitignore b/.gitignore
index f6050b88e95b..1be78fd8163b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -65,6 +65,11 @@ modules.builtin
 #
 /debian/
 
+#
+# Snap directory (make snap-pkg)
+#
+/snap/
+
 #
 # tar directory (make tar*-pkg)
 #
@@ -122,3 +127,7 @@ all.config
 
 # Kdevelop4
 *.kdev4
+
+#Automatically generated by ASN.1 compiler
+net/ipv4/netfilter/nf_nat_snmp_basic-asn1.c
+net/ipv4/netfilter/nf_nat_snmp_basic-asn1.h
diff --git a/.mailmap b/.mailmap
index e18cab73e209..a2ce89a456c2 100644
--- a/.mailmap
+++ b/.mailmap
@@ -62,6 +62,7 @@ Frank Zago <fzago@systemfabricworks.com>
 Greg Kroah-Hartman <greg@echidna.(none)>
 Greg Kroah-Hartman <gregkh@suse.de>
 Greg Kroah-Hartman <greg@kroah.com>
+Gregory CLEMENT <gregory.clement@bootlin.com> <gregory.clement@free-electrons.com>
 Henk Vergonet <Henk.Vergonet@gmail.com>
 Henrik Kretzschmar <henne@nachtwindheim.de>
 Henrik Rydberg <rydberg@bitmath.org>
diff --git a/Documentation/00-INDEX b/Documentation/00-INDEX
index 3bec49c33bbb..7f3a0728ccf2 100644
--- a/Documentation/00-INDEX
+++ b/Documentation/00-INDEX
@@ -228,8 +228,6 @@ isdn/
 	- directory with info on the Linux ISDN support, and supported cards.
 kbuild/
 	- directory with info about the kernel build process.
-kernel-doc-nano-HOWTO.txt
-	- outdated info about kernel-doc documentation.
 kdump/
 	- directory with mini HowTo on getting the crash dump code to work.
 doc-guide/
@@ -346,8 +344,6 @@ prctl/
 	- directory with info on the priveledge control subsystem
 preempt-locking.txt
 	- info on locking under a preemptive kernel.
-printk-formats.txt
-	- how to get printk format specifiers right
 process/
 	- how to work with the mainline kernel development process.
 pps/
diff --git a/Documentation/ABI/stable/sysfs-bus-vmbus b/Documentation/ABI/stable/sysfs-bus-vmbus
index d4077cc60d55..e46be65d0e1d 100644
--- a/Documentation/ABI/stable/sysfs-bus-vmbus
+++ b/Documentation/ABI/stable/sysfs-bus-vmbus
@@ -42,72 +42,93 @@ Contact:	K. Y. Srinivasan <kys@microsoft.com>
 Description:	The 16 bit vendor ID of the device
 Users:		tools/hv/lsvmbus and user level RDMA libraries
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/cpu
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN
+Date:		September. 2017
+KernelVersion:	4.14
+Contact:	Stephen Hemminger <sthemmin@microsoft.com>
+Description:	Directory for per-channel information
+		NN is the VMBUS relid associtated with the channel.
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/cpu
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	VCPU (sub)channel is affinitized to
-Users:		tools/hv/lsvmbus and other debuggig tools
+Users:		tools/hv/lsvmbus and other debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/cpu
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/cpu
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	VCPU (sub)channel is affinitized to
-Users:		tools/hv/lsvmbus and other debuggig tools
+Users:		tools/hv/lsvmbus and other debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/in_mask
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/in_mask
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
-Description:	Inbound channel signaling state
+Description:	Host to guest channel interrupt mask
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/latency
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/latency
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Channel signaling latency
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/out_mask
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/out_mask
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
-Description:	Outbound channel signaling state
+Description:	Guest to host channel interrupt mask
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/pending
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/pending
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Channel interrupt pending state
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/read_avail
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/read_avail
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
-Description:	Bytes availabble to read
+Description:	Bytes available to read
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/write_avail
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/write_avail
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
-Description:	Bytes availabble to write
+Description:	Bytes available to write
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/events
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/events
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Number of times we have signaled the host
 Users:		Debugging tools
 
-What:		/sys/bus/vmbus/devices/vmbus_*/channels/relid/interrupts
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/interrupts
 Date:		September. 2017
 KernelVersion:	4.14
 Contact:	Stephen Hemminger <sthemmin@microsoft.com>
 Description:	Number of times we have taken an interrupt (incoming)
 Users:		Debugging tools
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/subchannel_id
+Date:		January. 2018
+KernelVersion:	4.16
+Contact:	Stephen Hemminger <sthemmin@microsoft.com>
+Description:	Subchannel ID associated with VMBUS channel
+Users:		Debugging tools and userspace drivers
+
+What:		/sys/bus/vmbus/devices/vmbus_*/channels/NN/monitor_id
+Date:		January. 2018
+KernelVersion:	4.16
+Contact:	Stephen Hemminger <sthemmin@microsoft.com>
+Description:	Monitor bit associated with channel
+Users:		Debugging tools and userspace drivers
diff --git a/Documentation/ABI/testing/devlink-resource-mlxsw b/Documentation/ABI/testing/devlink-resource-mlxsw
new file mode 100644
index 000000000000..259ed2948ec0
--- /dev/null
+++ b/Documentation/ABI/testing/devlink-resource-mlxsw
@@ -0,0 +1,33 @@
+What: 		/kvd/
+Date:		08-Jan-2018
+KernelVersion:	v4.16
+Contact:	mlxsw@mellanox.com
+Description:	The main database in the Spectrum device is a centralized
+		KVD database used for many of the tables used to configure
+		the chip including L2 FDB, L3 LPM, ECMP and more. The KVD
+		is divided into two sections, the first is hash-based table
+		and the second is a linear access table. The division
+		between the linear and hash-based sections is static and
+		require reload before the changes take effect.
+
+What: 		/kvd/linear
+Date:		08-Jan-2018
+KernelVersion:	v4.16
+Contact:	mlxsw@mellanox.com
+Description:	The linear section of the KVD is managed by software as a
+		flat memory accessed using an index.
+
+What: 		/kvd/hash_single
+Date:		08-Jan-2018
+KernelVersion:	v4.16
+Contact:	mlxsw@mellanox.com
+Description:	The hash based section of the KVD is managed by the switch
+		device. Used in case the key size is smaller or equal to
+		64bit.
+
+What: 		/kvd/hash_double
+Date:		08-Jan-2018
+KernelVersion:	v4.16
+Contact:	mlxsw@mellanox.com
+Description:	The hash based section of the KVD is managed by the switch
+		device. Used in case the key is larger than 64 bit.
diff --git a/Documentation/ABI/testing/evm b/Documentation/ABI/testing/evm
index 9578247e1792..d12cb2eae9ee 100644
--- a/Documentation/ABI/testing/evm
+++ b/Documentation/ABI/testing/evm
@@ -14,30 +14,46 @@ Description:
 		generated either locally or remotely using an
 		asymmetric key. These keys are loaded onto root's
 		keyring using keyctl, and EVM is then enabled by
-		echoing a value to <securityfs>/evm:
+		echoing a value to <securityfs>/evm made up of the
+		following bits:
 
-		1: enable HMAC validation and creation
-		2: enable digital signature validation
-		3: enable HMAC and digital signature validation and HMAC
-		   creation
+		Bit	  Effect
+		0	  Enable HMAC validation and creation
+		1	  Enable digital signature validation
+		2	  Permit modification of EVM-protected metadata at
+			  runtime. Not supported if HMAC validation and
+			  creation is enabled.
+		31	  Disable further runtime modification of EVM policy
 
-		Further writes will be blocked if HMAC support is enabled or
-		if bit 32 is set:
+		For example:
 
-		echo 0x80000002 ><securityfs>/evm
+		echo 1 ><securityfs>/evm
 
-		will enable digital signature validation and block
-		further writes to <securityfs>/evm.
+		will enable HMAC validation and creation
 
-		Until this is done, EVM can not create or validate the
-		'security.evm' xattr, but returns INTEGRITY_UNKNOWN.
-		Loading keys and signaling EVM should be done as early
-		as possible.  Normally this is done in the initramfs,
-		which has already been measured as part of the trusted
-		boot.  For more information on creating and loading
-		existing trusted/encrypted keys, refer to:
+		echo 0x80000003 ><securityfs>/evm
 
-		Documentation/security/keys/trusted-encrypted.rst. Both dracut
-		(via 97masterkey and 98integrity) and systemd (via
+		will enable HMAC and digital signature validation and
+		HMAC creation and disable all further modification of policy.
+
+		echo 0x80000006 ><securityfs>/evm
+
+		will enable digital signature validation, permit
+		modification of EVM-protected metadata and
+		disable all further modification of policy
+
+		Note that once a key has been loaded, it will no longer be
+		possible to enable metadata modification.
+
+		Until key loading has been signaled EVM can not create
+		or validate the 'security.evm' xattr, but returns
+		INTEGRITY_UNKNOWN.  Loading keys and signaling EVM
+		should be done as early as possible.  Normally this is
+		done in the initramfs, which has already been measured
+		as part of the trusted boot.  For more information on
+		creating and loading existing trusted/encrypted keys,
+		refer to:
+		Documentation/security/keys/trusted-encrypted.rst. Both
+		dracut (via 97masterkey and 98integrity) and systemd (via
 		core/ima-setup) have support for loading keys at boot
 		time.
diff --git a/Documentation/ABI/testing/ima_policy b/Documentation/ABI/testing/ima_policy
index e76432b9954d..2028f2d093b2 100644
--- a/Documentation/ABI/testing/ima_policy
+++ b/Documentation/ABI/testing/ima_policy
@@ -17,7 +17,8 @@ Description:
 
 		rule format: action [condition ...]
 
-		action: measure | dont_measure | appraise | dont_appraise | audit
+		action: measure | dont_measure | appraise | dont_appraise |
+			audit | hash | dont_hash
 		condition:= base | lsm  [option]
 			base:	[[func=] [mask=] [fsmagic=] [fsuuid=] [uid=]
 				[euid=] [fowner=]]
diff --git a/Documentation/ABI/testing/rtc-cdev b/Documentation/ABI/testing/rtc-cdev
new file mode 100644
index 000000000000..97447283f13b
--- /dev/null
+++ b/Documentation/ABI/testing/rtc-cdev
@@ -0,0 +1,42 @@
+What:		/dev/rtcX
+Date:		April 2005
+KernelVersion:	2.6.12
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		The ioctl interface to drivers for real-time clocks (RTCs).
+		Following actions are supported:
+
+		* RTC_RD_TIME, RTC_SET_TIME: Read or set the RTC time. Time
+		  format is a Gregorian calendar date and 24 hour wall clock
+		  time.
+
+		* RTC_AIE_ON, RTC_AIE_OFF: Enable or disable the alarm interrupt
+		  for RTCs that support alarms
+
+		* RTC_ALM_READ, RTC_ALM_SET: Read or set the alarm time for
+		  RTCs that support alarms. Can be set upto 24 hours in the
+		  future. Requires a separate RTC_AIE_ON call to enable the
+		  alarm interrupt. (Prefer to use RTC_WKALM_*)
+
+		* RTC_WKALM_RD, RTC_WKALM_SET: For RTCs that support a more
+		  powerful interface, which can issue alarms beyond 24 hours and
+		  enable IRQs in the same request.
+
+		* RTC_PIE_ON, RTC_PIE_OFF: Enable or disable the periodic
+		  interrupt for RTCs that support periodic interrupts.
+
+		* RTC_UIE_ON, RTC_UIE_OFF: Enable or disable the update
+		  interrupt for RTCs that support it.
+
+		* RTC_IRQP_READ, RTC_IRQP_SET: Read or set the frequency for
+		  periodic interrupts for RTCs that support periodic interrupts.
+		  Requires a separate RTC_PIE_ON call to enable the periodic
+		  interrupts.
+
+		The ioctl() calls supported by the older /dev/rtc interface are
+		also supported by the newer RTC class framework. However,
+		because the chips and systems are not standardized, some PC/AT
+		functionality might not be provided. And in the same way, some
+		newer features -- including those enabled by ACPI -- are exposed
+		by the RTC class framework, but can't be supported by the older
+		driver.
diff --git a/Documentation/ABI/testing/sysfs-ata b/Documentation/ABI/testing/sysfs-ata
index aa4296498859..9ab0ef1dd1c7 100644
--- a/Documentation/ABI/testing/sysfs-ata
+++ b/Documentation/ABI/testing/sysfs-ata
@@ -1,110 +1,139 @@
 What:		/sys/class/ata_...
-Date:		August 2008
-Contact:	Gwendal Grignou<gwendal@google.com>
 Description:
-
-Provide a place in sysfs for storing the ATA topology of the system.  This allows
-retrieving various information about ATA objects.
+		Provide a place in sysfs for storing the ATA topology of the
+		system. This allows retrieving various information about ATA
+		objects.
 
 Files under /sys/class/ata_port
 -------------------------------
 
-	For each port, a directory ataX is created where X is the ata_port_id of
-	the port. The device parent is the ata host device.
+For each port, a directory ataX is created where X is the ata_port_id of the
+port. The device parent is the ata host device.
 
-idle_irq (read)
 
-	Number of IRQ received by the port while idle [some ata HBA only].
+What:		/sys/class/ata_port/ataX/nr_pmp_links
+What:		/sys/class/ata_port/ataX/idle_irq
+Date:		May, 2010
+KernelVersion:	v2.6.37
+Contact:	Gwendal Grignou <gwendal@chromium.org>
+Description:
+		nr_pmp_links:	(RO) If a SATA Port Multiplier (PM) is
+				connected, the number of links behind it.
 
-nr_pmp_links (read)
+		idle_irq:	(RO) Number of IRQ received by the port while
+				idle [some ata HBA only].
 
-	If a SATA Port Multiplier (PM) is connected, number of link behind it.
+
+What:		/sys/class/ata_port/ataX/port_no
+Date:		May, 2013
+KernelVersion:	v3.11
+Contact:	Gwendal Grignou <gwendal@chromium.org>
+Description:
+		(RO) Host local port number. While registering host controller,
+		port numbers are tracked based upon number of ports available on
+		the controller. This attribute is needed by udev for composing
+		persistent links in /dev/disk/by-path.
 
 Files under /sys/class/ata_link
 -------------------------------
 
-	Behind each port, there is a ata_link. If there is a SATA PM in the
-	topology, 15 ata_link objects are created.
-
-	If a link is behind a port, the directory name is linkX, where X is
-	ata_port_id of the port.
-	If a link is behind a PM, its name is linkX.Y where X is ata_port_id
-	of the parent port and Y the PM port.
+Behind each port, there is a ata_link. If there is a SATA PM in the topology, 15
+ata_link objects are created.
 
-hw_sata_spd_limit
+If a link is behind a port, the directory name is linkX, where X is ata_port_id
+of the port. If a link is behind a PM, its name is linkX.Y where X is
+ata_port_id of the parent port and Y the PM port.
 
-	Maximum speed supported by the connected SATA device.
 
-sata_spd_limit
+What:		/sys/class/ata_link/linkX[.Y]/hw_sata_spd_limit
+What:		/sys/class/ata_link/linkX[.Y]/sata_spd_limit
+What:		/sys/class/ata_link/linkX[.Y]/sata_spd
+Date:		May, 2010
+KernelVersion:	v2.6.37
+Contact:	Gwendal Grignou <gwendal@chromium.org>
+Description:
+		hw_sata_spd_limit:	(RO) Maximum speed supported by the
+					connected SATA device.
 
-	Maximum speed imposed by libata.
+		sata_spd_limit:		(RO) Maximum speed imposed by libata.
 
-sata_spd
+		sata_spd:		(RO) Current speed of the link
+					eg. 1.5, 3 Gbps etc.
 
-	Current speed of the link [1.5, 3Gps,...].
 
 Files under /sys/class/ata_device
 ---------------------------------
 
-	Behind each link, up to two ata device are created.
-	The name of the directory is devX[.Y].Z where:
-	- X is ata_port_id of the port where the device is connected,
-	- Y the port of the PM if any, and
-	- Z the device id: for PATA, there is usually 2 devices [0,1],
-	only 1 for SATA.
-
-class
-	Device class. Can be "ata" for disk, "atapi" for packet device,
-	"pmp" for PM, or "none" if no device was found behind the link.
-
-dma_mode
+Behind each link, up to two ata devices are created.
+The name of the directory is devX[.Y].Z where:
+- X is ata_port_id of the port where the device is connected,
+- Y the port of the PM if any, and
+- Z the device id: for PATA, there is usually 2 devices [0,1], only 1 for SATA.
+
+
+What:		/sys/class/ata_device/devX[.Y].Z/spdn_cnt
+What:		/sys/class/ata_device/devX[.Y].Z/gscr
+What:		/sys/class/ata_device/devX[.Y].Z/ering
+What:		/sys/class/ata_device/devX[.Y].Z/id
+What:		/sys/class/ata_device/devX[.Y].Z/pio_mode
+What:		/sys/class/ata_device/devX[.Y].Z/xfer_mode
+What:		/sys/class/ata_device/devX[.Y].Z/dma_mode
+What:		/sys/class/ata_device/devX[.Y].Z/class
+Date:		May, 2010
+KernelVersion:	v2.6.37
+Contact:	Gwendal Grignou <gwendal@chromium.org>
+Description:
+		spdn_cnt:	(RO) Number of times libata decided to lower the
+				speed of link due to errors.
 
-	Transfer modes supported by the device when in DMA mode.
-	Mostly used by PATA device.
+		gscr:		(RO) Cached result of the dump of PM GSCR
+				register. Valid registers are:
 
-pio_mode
+				0:      SATA_PMP_GSCR_PROD_ID,
+				1:	SATA_PMP_GSCR_REV,
+				2:      SATA_PMP_GSCR_PORT_INFO,
+				32:     SATA_PMP_GSCR_ERROR,
+				33:     SATA_PMP_GSCR_ERROR_EN,
+				64:     SATA_PMP_GSCR_FEAT,
+				96:     SATA_PMP_GSCR_FEAT_EN,
+				130:    SATA_PMP_GSCR_SII_GPIO
 
-	Transfer modes supported by the device when in PIO mode.
-	Mostly used by PATA device.
+				Only valid if the device is a PM.
 
-xfer_mode
+		ering:		(RO) Formatted output of the error ring of the
+				device.
 
-	Current transfer mode.
+		id:		(RO) Cached result of IDENTIFY command, as
+				described in ATA8 7.16 and 7.17. Only valid if
+				the device is not a PM.
 
-id
+		pio_mode:	(RO) Transfer modes supported by the device when
+				in PIO mode. Mostly used by PATA device.
 
-	Cached result of IDENTIFY command, as described in ATA8 7.16 and 7.17.
-	Only valid if the device is not a PM.
+		xfer_mode:	(RO) Current transfer mode
 
-gscr
+		dma_mode:	(RO) Transfer modes supported by the device when
+				in DMA mode. Mostly used by PATA device.
 
-	Cached result of the dump of PM GSCR register.
-	Valid registers are:
-	0: 	SATA_PMP_GSCR_PROD_ID,
-	1: 	SATA_PMP_GSCR_REV,
-	2: 	SATA_PMP_GSCR_PORT_INFO,
-	32:	SATA_PMP_GSCR_ERROR,
-	33:	SATA_PMP_GSCR_ERROR_EN,
-	64:	SATA_PMP_GSCR_FEAT,
-	96:	SATA_PMP_GSCR_FEAT_EN,
-	130:	SATA_PMP_GSCR_SII_GPIO
-	Only valid if the device is a PM.
+		class:		(RO) Device class. Can be "ata" for disk,
+				"atapi" for packet device, "pmp" for PM, or
+				"none" if no device was found behind the link.
 
-trim
 
-	Shows the DSM TRIM mode currently used by the device. Valid
-	values are:
-	unsupported:		Drive does not support DSM TRIM
-	unqueued:		Drive supports unqueued DSM TRIM only
-	queued:			Drive supports queued DSM TRIM
-	forced_unqueued:	Drive's queued DSM support is known to be
-				buggy and only unqueued TRIM commands
-				are sent
+What:		/sys/class/ata_device/devX[.Y].Z/trim
+Date:		May, 2015
+KernelVersion:	v4.10
+Contact:	Gwendal Grignou <gwendal@chromium.org>
+Description:
+		(RO) Shows the DSM TRIM mode currently used by the device. Valid
+		values are:
 
-spdn_cnt
+		unsupported:		Drive does not support DSM TRIM
 
-	Number of time libata decided to lower the speed of link due to errors.
+		unqueued:               Drive supports unqueued DSM TRIM only
 
-ering
+		queued:                 Drive supports queued DSM TRIM
 
-	Formatted output of the error ring of the device.
+		forced_unqueued:	Drive's queued DSM support is known to
+					be buggy and only unqueued TRIM commands
+					are sent
diff --git a/Documentation/ABI/testing/sysfs-block-device b/Documentation/ABI/testing/sysfs-block-device
new file mode 100644
index 000000000000..82ef6eab042d
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-block-device
@@ -0,0 +1,58 @@
+What:		/sys/block/*/device/sw_activity
+Date:		Jun, 2008
+KernelVersion:	v2.6.27
+Contact:	linux-ide@vger.kernel.org
+Description:
+		(RW) Used by drivers which support software controlled activity
+		LEDs.
+
+		It has the following valid values:
+
+		0	OFF - the LED is not activated on activity
+		1	BLINK_ON - the LED blinks on every 10ms when activity is
+			detected.
+		2	BLINK_OFF - the LED is on when idle, and blinks off
+			every 10ms when activity is detected.
+
+		Note that the user must turn sw_activity OFF it they wish to
+		control the activity LED via the em_message file.
+
+
+What:		/sys/block/*/device/unload_heads
+Date:		Sep, 2008
+KernelVersion:	v2.6.28
+Contact:	linux-ide@vger.kernel.org
+Description:
+		(RW) Hard disk shock protection
+
+		Writing an integer value to this file will take the heads of the
+		respective drive off the platter and block all I/O operations
+		for the specified number of milliseconds.
+
+		- If the device does not support the unload heads feature,
+		  access is denied with -EOPNOTSUPP.
+		- The maximal value accepted for a timeout is 30000
+		  milliseconds.
+		- A previously set timeout can be cancelled and disk can resume
+		  normal operation immediately by specifying a timeout of 0.
+		- Some hard drives only comply with an earlier version of the
+		  ATA standard, but support the unload feature nonetheless.
+		  There is no safe way Linux can detect these devices, so this
+		  is not enabled by default. If it is known that your device
+		  does support the unload feature, then you can tell the kernel
+		  to enable it by writing -1. It can be disabled again by
+		  writing -2.
+		- Values below -2 are rejected with -EINVAL
+
+		For more information, see
+		Documentation/laptops/disk-shock-protection.txt
+
+
+What:		/sys/block/*/device/ncq_prio_enable
+Date:		Oct, 2016
+KernelVersion:	v4.10
+Contact:	linux-ide@vger.kernel.org
+Description:
+		(RW) Write to the file to turn on or off the SATA ncq (native
+		command queueing) support. By default this feature is turned
+		off.
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index 2e3f919485f4..6a5f34b4d5b9 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -32,7 +32,7 @@ Description:
 		Description of the physical chip / device for device X.
 		Typically a part number.
 
-What:		/sys/bus/iio/devices/iio:deviceX/timestamp_clock
+What:		/sys/bus/iio/devices/iio:deviceX/current_timestamp_clock
 KernelVersion:	4.5
 Contact:	linux-iio@vger.kernel.org
 Description:
@@ -1290,7 +1290,7 @@ KernelVersion:	3.4
 Contact:	linux-iio@vger.kernel.org
 Description:
 		Unit-less light intensity. Modifiers both and ir indicate
-		that measurements contains visible and infrared light
+		that measurements contain visible and infrared light
 		components or just infrared light, respectively. Modifier uv indicates
 		that measurements contain ultraviolet light components.
 
@@ -1413,6 +1413,16 @@ Description:
 		the available samples after the timeout expires and thus have a
 		maximum delay guarantee.
 
+What:		/sys/bus/iio/devices/iio:deviceX/buffer/data_available
+KernelVersion: 4.16
+Contact:	linux-iio@vger.kernel.org
+Description:
+		A read-only value indicating the bytes of data available in the
+		buffer. In the case of an output buffer, this indicates the
+		amount of empty space available to write data to. In the case of
+		an input buffer, this indicates the amount of data available for
+		reading.
+
 What:		/sys/bus/iio/devices/iio:deviceX/buffer/hwfifo_enabled
 KernelVersion: 4.2
 Contact:	linux-iio@vger.kernel.org
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-dfsdm-adc-stm32 b/Documentation/ABI/testing/sysfs-bus-iio-dfsdm-adc-stm32
new file mode 100644
index 000000000000..da9822309f07
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-iio-dfsdm-adc-stm32
@@ -0,0 +1,16 @@
+What:		/sys/bus/iio/devices/iio:deviceX/in_voltage_spi_clk_freq
+KernelVersion:	4.14
+Contact:	arnaud.pouliquen@st.com
+Description:
+		For audio purpose only.
+		Used by audio driver to set/get the spi input frequency.
+		This is mandatory if DFSDM is slave on SPI bus, to
+		provide information on the SPI clock frequency during runtime
+		Notice that the SPI frequency should be a multiple of sample
+		frequency to ensure the precision.
+		if DFSDM input is SPI master
+			Reading  SPI clkout frequency,
+			error on writing
+		If DFSDM input is SPI Slave:
+			Reading returns value previously set.
+			Writing value before starting conversions.
\ No newline at end of file
diff --git a/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd b/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd
new file mode 100644
index 000000000000..0088aba4caa8
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-pci-drivers-xhci_hcd
@@ -0,0 +1,25 @@
+What:		/sys/bus/pci/drivers/xhci_hcd/.../dbc
+Date:		June 2017
+Contact:	Lu Baolu <baolu.lu@linux.intel.com>
+Description:
+		xHCI compatible USB host controllers (i.e. super-speed
+		USB3 controllers) are often implemented with the Debug
+		Capability (DbC). It can present a debug device which
+		is fully compliant with the USB framework and provides
+		the equivalent of a very high performance full-duplex
+		serial link for debug purpose.
+
+		The DbC debug device shares a root port with xHCI host.
+		When the DbC is enabled, the root port will be assigned
+		to the Debug Capability. Otherwise, it will be assigned
+		to xHCI.
+
+		Writing "enable" to this attribute will enable the DbC
+		functionality and the shared root port will be assigned
+		to the DbC device. Writing "disable" to this attribute
+		will disable the DbC functionality and the shared root
+		port will roll back to the xHCI.
+
+		Reading this attribute gives the state of the DbC. It
+		can be one of the following states: disabled, enabled,
+		initialized, connected, configured and stalled.
diff --git a/Documentation/ABI/testing/sysfs-bus-siox b/Documentation/ABI/testing/sysfs-bus-siox
new file mode 100644
index 000000000000..fed7c3765a4e
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-siox
@@ -0,0 +1,87 @@
+What:		/sys/bus/siox/devices/siox-X/active
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		On reading represents the current state of the bus. If it
+		contains a "0" the bus is stopped and connected devices are
+		expected to not do anything because their watchdog triggered.
+		When the file contains a "1" the bus is operated and periodically
+		does a push-pull cycle to write and read data from the
+		connected devices.
+		When writing a "0" or "1" the bus moves to the described state.
+
+What:		/sys/bus/siox/devices/siox-X/device_add
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Write-only file. Write
+
+			<type> <inbytes> <outbytes> <statustype>
+
+		to add a new device dynamically. <type> is the name that is used to match
+		to a driver (similar to the platform bus). <inbytes> and <outbytes> define
+		the length of the input and output shift register in bytes respectively.
+		<statustype> defines the 4 bit device type that is check to identify connection
+		problems.
+		The new device is added to the end of the existing chain.
+
+What:		/sys/bus/siox/devices/siox-X/device_remove
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Write-only file. A single write removes the last device in the siox chain.
+
+What:		/sys/bus/siox/devices/siox-X/poll_interval_ns
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Defines the interval between two poll cycles in nano seconds.
+		Note this is rounded to jiffies on writing. On reading the current value
+		is returned.
+
+What:		/sys/bus/siox/devices/siox-X-Y/connected
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Read-only value. "0" means the Yth device on siox bus X isn't "connected" i.e.
+		communication with it is not ensured. "1" signals a working connection.
+
+What:		/sys/bus/siox/devices/siox-X-Y/inbytes
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Read-only value reporting the inbytes value provided to siox-X/device_add
+
+What:		/sys/bus/siox/devices/siox-X-Y/status_errors
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Counts the number of time intervals when the read status byte doesn't yield the
+		expected value.
+
+What:		/sys/bus/siox/devices/siox-X-Y/type
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Read-only value reporting the type value provided to siox-X/device_add.
+
+What:		/sys/bus/siox/devices/siox-X-Y/watchdog
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Read-only value reporting if the watchdog of the siox device is
+		active. "0" means the watchdog is not active and the device is expected to
+		be operational. "1" means the watchdog keeps the device in reset.
+
+What:		/sys/bus/siox/devices/siox-X-Y/watchdog_errors
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Read-only value reporting the number to time intervals when the
+		watchdog was active.
+
+What:		/sys/bus/siox/devices/siox-X-Y/outbytes
+KernelVersion:	4.16
+Contact:	Gavin Schenk <g.schenk@eckelmann.de>, Uwe Kleine-König <u.kleine-koenig@pengutronix.de>
+Description:
+		Read-only value reporting the outbytes value provided to siox-X/device_add.
diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-netdev b/Documentation/ABI/testing/sysfs-class-led-trigger-netdev
new file mode 100644
index 000000000000..451af6d6768c
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-class-led-trigger-netdev
@@ -0,0 +1,45 @@
+What:		/sys/class/leds/<led>/device_name
+Date:		Dec 2017
+KernelVersion:	4.16
+Contact:	linux-leds@vger.kernel.org
+Description:
+		Specifies the network device name to monitor.
+
+What:		/sys/class/leds/<led>/interval
+Date:		Dec 2017
+KernelVersion:	4.16
+Contact:	linux-leds@vger.kernel.org
+Description:
+		Specifies the duration of the LED blink in milliseconds.
+		Defaults to 50 ms.
+
+What:		/sys/class/leds/<led>/link
+Date:		Dec 2017
+KernelVersion:	4.16
+Contact:	linux-leds@vger.kernel.org
+Description:
+		Signal the link state of the named network device.
+		If set to 0 (default), the LED's normal state is off.
+		If set to 1, the LED's normal state reflects the link state
+		of the named network device.
+		Setting this value also immediately changes the LED state.
+
+What:		/sys/class/leds/<led>/tx
+Date:		Dec 2017
+KernelVersion:	4.16
+Contact:	linux-leds@vger.kernel.org
+Description:
+		Signal transmission of data on the named network device.
+		If set to 0 (default), the LED will not blink on transmission.
+		If set to 1, the LED will blink for the milliseconds specified
+		in interval to signal transmission.
+
+What:		/sys/class/leds/<led>/rx
+Date:		Dec 2017
+KernelVersion:	4.16
+Contact:	linux-leds@vger.kernel.org
+Description:
+		Signal reception of data on the named network device.
+		If set to 0 (default), the LED will not blink on reception.
+		If set to 1, the LED will blink for the milliseconds specified
+		in interval to signal reception.
diff --git a/Documentation/ABI/testing/sysfs-class-net b/Documentation/ABI/testing/sysfs-class-net
index 6856da99b6f7..2f1788111cd9 100644
--- a/Documentation/ABI/testing/sysfs-class-net
+++ b/Documentation/ABI/testing/sysfs-class-net
@@ -259,3 +259,27 @@ Contact:	netdev@vger.kernel.org
 Description:
 		Symbolic link to the PHY device this network device is attached
 		to.
+
+What:		/sys/class/net/<iface>/carrier_changes
+Date:		Mar 2014
+KernelVersion:	3.15
+Contact:	netdev@vger.kernel.org
+Description:
+		32-bit unsigned integer counting the number of times the link has
+		seen a change from UP to DOWN and vice versa
+
+What:		/sys/class/net/<iface>/carrier_up_count
+Date:		Jan 2018
+KernelVersion:	4.16
+Contact:	netdev@vger.kernel.org
+Description:
+		32-bit unsigned integer counting the number of times the link has
+		been up
+
+What:		/sys/class/net/<iface>/carrier_down_count
+Date:		Jan 2018
+KernelVersion:	4.16
+Contact:	netdev@vger.kernel.org
+Description:
+		32-bit unsigned integer counting the number of times the link has
+		been down
diff --git a/Documentation/ABI/testing/sysfs-class-ocxl b/Documentation/ABI/testing/sysfs-class-ocxl
new file mode 100644
index 000000000000..b5b1fa197592
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-class-ocxl
@@ -0,0 +1,35 @@
+What:		/sys/class/ocxl/<afu name>/afu_version
+Date:		January 2018
+Contact:	linuxppc-dev@lists.ozlabs.org
+Description:	read only
+		Version of the AFU, in the format <major>:<minor>
+		Reflects what is read in the configuration space of the AFU
+
+What:		/sys/class/ocxl/<afu name>/contexts
+Date:		January 2018
+Contact:	linuxppc-dev@lists.ozlabs.org
+Description:	read only
+		Number of contexts for the AFU, in the format <n>/<max>
+		where:
+			n:	number of currently active contexts, for debug
+			max:	maximum number of contexts supported by the AFU
+
+What:		/sys/class/ocxl/<afu name>/pp_mmio_size
+Date:		January 2018
+Contact:	linuxppc-dev@lists.ozlabs.org
+Description:	read only
+		Size of the per-process mmio area, as defined in the
+		configuration space of the AFU
+
+What:		/sys/class/ocxl/<afu name>/global_mmio_size
+Date:		January 2018
+Contact:	linuxppc-dev@lists.ozlabs.org
+Description:	read only
+		Size of the global mmio area, as defined in the
+		configuration space of the AFU
+
+What:		/sys/class/ocxl/<afu name>/global_mmio_area
+Date:		January 2018
+Contact:	linuxppc-dev@lists.ozlabs.org
+Description:	read/write
+		Give access the global mmio area for the AFU
diff --git a/Documentation/ABI/testing/sysfs-class-rtc b/Documentation/ABI/testing/sysfs-class-rtc
new file mode 100644
index 000000000000..cf60412882f0
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-class-rtc
@@ -0,0 +1,91 @@
+What:		/sys/class/rtc/
+Date:		March 2006
+KernelVersion:	2.6.17
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		The rtc/ class subdirectory belongs to the RTC subsystem.
+
+What:		/sys/class/rtc/rtcX/
+Date:		March 2006
+KernelVersion:	2.6.17
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		The /sys/class/rtc/rtc{0,1,2,3,...} directories correspond
+		to each RTC device.
+
+What:		/sys/class/rtc/rtcX/date
+Date:		March 2006
+KernelVersion:	2.6.17
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RO) RTC-provided date in YYYY-MM-DD format
+
+What:		/sys/class/rtc/rtcX/hctosys
+Date:		September 2009
+KernelVersion:	2.6.32
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RO) 1 if the RTC provided the system time at boot via the
+		CONFIG_RTC_HCTOSYS kernel option, 0 otherwise
+
+What:		/sys/class/rtc/rtcX/max_user_freq
+Date:		October 2007
+KernelVersion:	2.6.24
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RW) The maximum interrupt rate an unprivileged user may request
+		from this RTC.
+
+What:		/sys/class/rtc/rtcX/name
+Date:		March 2006
+KernelVersion:	2.6.17
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RO) The name of the RTC corresponding to this sysfs directory
+
+What:		/sys/class/rtc/rtcX/since_epoch
+Date:		March 2006
+KernelVersion:	2.6.17
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RO) RTC-provided time as the number of seconds since the epoch
+
+What:		/sys/class/rtc/rtcX/time
+Date:		March 2006
+KernelVersion:	2.6.17
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RO) RTC-provided time in 24-hour notation (hh:mm:ss)
+
+What:		/sys/class/rtc/rtcX/*/nvmem
+Date:		February 2016
+KernelVersion:	4.6
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RW) The non volatile storage exported as a raw file, as
+		described in Documentation/nvmem/nvmem.txt
+
+What:		/sys/class/rtc/rtcX/offset
+Date:		February 2016
+KernelVersion:	4.6
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RW) The amount which the rtc clock has been adjusted in
+		firmware. Visible only if the driver supports clock offset
+		adjustment. The unit is parts per billion, i.e. The number of
+		clock ticks which are added to or removed from the rtc's base
+		clock per billion ticks. A positive value makes a day pass more
+		slowly, longer, and a negative value makes a day pass more
+		quickly.
+
+What:		/sys/class/rtc/rtcX/wakealarm
+Date:		February 2007
+KernelVersion:	2.6.20
+Contact:	linux-rtc@vger.kernel.org
+Description:
+		(RW) The time at which the clock will generate a system wakeup
+		event. This is a one shot wakeup event, so must be reset after
+		wake if a daily wakeup is required. Format is seconds since the
+		epoch by default, or if there's a leading +, seconds in the
+		future, or if there is a leading +=, seconds ahead of the
+		current alarm.
diff --git a/Documentation/ABI/testing/sysfs-class-scsi_host b/Documentation/ABI/testing/sysfs-class-scsi_host
index 0eb255e7db12..bafc59fd7b69 100644
--- a/Documentation/ABI/testing/sysfs-class-scsi_host
+++ b/Documentation/ABI/testing/sysfs-class-scsi_host
@@ -27,3 +27,92 @@ Description:	This file contains the current status of the "SSD Smart Path"
 		the direct i/o path to physical devices.  This setting is
 		controller wide, affecting all configured logical drives on the
 		controller.  This file is readable and writable.
+
+What:		/sys/class/scsi_host/hostX/link_power_management_policy
+Date:		Oct, 2007
+KernelVersion:	v2.6.24
+Contact:	linux-ide@vger.kernel.org
+Description:
+		(RW) This parameter allows the user to read and set the link
+		(interface) power management.
+
+		There are four possible options:
+
+		min_power: Tell the controller to try to make the link use the
+		least possible power when possible. This may sacrifice some
+		performance due to increased latency when coming out of lower
+		power states.
+
+		max_performance: Generally, this means no power management.
+		Tell the controller to have performance be a priority over power
+		management.
+
+		medium_power: Tell the controller to enter a lower power state
+		when possible, but do not enter the lowest power state, thus
+		improving latency over min_power setting.
+
+		med_power_with_dipm: Identical to the existing medium_power
+		setting except that it enables dipm (device initiated power
+		management) on top, which makes it match the Windows IRST (Intel
+		Rapid Storage Technology) driver settings. This setting is also
+		close to min_power, except that:
+		a) It does not use host-initiated slumber mode, but it does
+		allow device-initiated slumber
+		b) It does not enable low power device sleep mode (DevSlp).
+
+What:		/sys/class/scsi_host/hostX/em_message
+What:		/sys/class/scsi_host/hostX/em_message_type
+Date:		Jun, 2008
+KernelVersion:	v2.6.27
+Contact:	linux-ide@vger.kernel.org
+Description:
+		em_message: (RW) Enclosure management support. For the LED
+		protocol, writes and reads correspond to the LED message format
+		as defined in the AHCI spec.
+
+		The user must turn sw_activity (under /sys/block/*/device/) OFF
+		it they wish to control the activity LED via the em_message
+		file.
+
+		em_message_type: (RO) Displays the current enclosure management
+		protocol that is being used by the driver (for eg. LED, SAF-TE,
+		SES-2, SGPIO etc).
+
+What:		/sys/class/scsi_host/hostX/ahci_port_cmd
+What:		/sys/class/scsi_host/hostX/ahci_host_caps
+What:		/sys/class/scsi_host/hostX/ahci_host_cap2
+Date:		Mar, 2010
+KernelVersion:	v2.6.35
+Contact:	linux-ide@vger.kernel.org
+Description:
+		[to be documented]
+
+What:		/sys/class/scsi_host/hostX/ahci_host_version
+Date:		Mar, 2010
+KernelVersion:	v2.6.35
+Contact:	linux-ide@vger.kernel.org
+Description:
+		(RO) Display the version of the AHCI spec implemented by the
+		host.
+
+What:		/sys/class/scsi_host/hostX/em_buffer
+Date:		Apr, 2010
+KernelVersion:	v2.6.35
+Contact:	linux-ide@vger.kernel.org
+Description:
+		(RW) Allows access to AHCI EM (enclosure management) buffer
+		directly if the host supports EM.
+
+		For eg. the AHCI driver supports SGPIO EM messages but the
+		SATA/AHCI specs do not define the SGPIO message format of the EM
+		buffer. Different hardware(HW) vendors may have different
+		definitions. With the em_buffer attribute, this issue can be
+		solved by allowing HW vendors to provide userland drivers and
+		tools for their SGPIO initiators.
+
+What:		/sys/class/scsi_host/hostX/em_message_supported
+Date:		Oct, 2009
+KernelVersion:	v2.6.39
+Contact:	linux-ide@vger.kernel.org
+Description:
+		(RO) Displays supported enclosure management message types.
diff --git a/Documentation/ABI/testing/sysfs-devices-coredump b/Documentation/ABI/testing/sysfs-devices-coredump
new file mode 100644
index 000000000000..e459368533a4
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-devices-coredump
@@ -0,0 +1,10 @@
+What:		/sys/devices/.../coredump
+Date:		December 2017
+Contact:	Arend van Spriel <aspriel@gmail.com>
+Description:
+		The /sys/devices/.../coredump attribute is only present when the
+		device is bound to a driver, which provides the .coredump()
+		callback. The attribute is write only. Anything written to this
+		file will trigger the .coredump() callback.
+
+		Available when CONFIG_DEV_COREDUMP is enabled.
diff --git a/Documentation/ABI/testing/sysfs-devices-platform-dock b/Documentation/ABI/testing/sysfs-devices-platform-dock
new file mode 100644
index 000000000000..1d8c18f905c7
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-devices-platform-dock
@@ -0,0 +1,39 @@
+What:		/sys/devices/platform/dock.N/docked
+Date:		Dec, 2006
+KernelVersion:	2.6.19
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) Value 1 or 0 indicates whether the software believes the
+		laptop is docked in a docking station.
+
+What:		/sys/devices/platform/dock.N/undock
+Date:		Dec, 2006
+KernelVersion:	2.6.19
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(WO) Writing to this file causes the software to initiate an
+		undock request to the firmware.
+
+What:		/sys/devices/platform/dock.N/uid
+Date:		Feb, 2007
+KernelVersion:	v2.6.21
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) Displays the docking station the laptop is docked to.
+
+What:		/sys/devices/platform/dock.N/flags
+Date:		May, 2007
+KernelVersion:	v2.6.21
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) Show dock station flags, useful for checking if undock
+		request has been made by the user (from the immediate_undock
+		option).
+
+What:		/sys/devices/platform/dock.N/type
+Date:		Aug, 2008
+KernelVersion:	v2.6.27
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) Display the dock station type- dock_station, ata_bay or
+		battery_bay.
diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu
index bfd29bc8d37a..4ed63b6cfb15 100644
--- a/Documentation/ABI/testing/sysfs-devices-system-cpu
+++ b/Documentation/ABI/testing/sysfs-devices-system-cpu
@@ -108,6 +108,8 @@ Description:	CPU topology files that describe a logical CPU's relationship
 
 What:		/sys/devices/system/cpu/cpuidle/current_driver
 		/sys/devices/system/cpu/cpuidle/current_governer_ro
+		/sys/devices/system/cpu/cpuidle/available_governors
+		/sys/devices/system/cpu/cpuidle/current_governor
 Date:		September 2007
 Contact:	Linux kernel mailing list <linux-kernel@vger.kernel.org>
 Description:	Discover cpuidle policy and mechanism
@@ -119,13 +121,84 @@ Description:	Discover cpuidle policy and mechanism
 		Idle policy (governor) is differentiated from idle mechanism
 		(driver)
 
-		current_driver: displays current idle mechanism
+		current_driver: (RO) displays current idle mechanism
 
-		current_governor_ro: displays current idle policy
+		current_governor_ro: (RO) displays current idle policy
+
+		With the cpuidle_sysfs_switch boot option enabled (meant for
+		developer testing), the following three attributes are visible
+		instead:
+
+		current_driver: same as described above
+
+		available_governors: (RO) displays a space separated list of
+		available governors
+
+		current_governor: (RW) displays current idle policy. Users can
+		switch the governor at runtime by writing to this file.
 
 		See files in Documentation/cpuidle/ for more information.
 
 
+What:		/sys/devices/system/cpu/cpuX/cpuidle/stateN/name
+		/sys/devices/system/cpu/cpuX/cpuidle/stateN/latency
+		/sys/devices/system/cpu/cpuX/cpuidle/stateN/power
+		/sys/devices/system/cpu/cpuX/cpuidle/stateN/time
+		/sys/devices/system/cpu/cpuX/cpuidle/stateN/usage
+Date:		September 2007
+KernelVersion:	v2.6.24
+Contact:	Linux power management list <linux-pm@vger.kernel.org>
+Description:
+		The directory /sys/devices/system/cpu/cpuX/cpuidle contains per
+		logical CPU specific cpuidle information for each online cpu X.
+		The processor idle states which are available for use have the
+		following attributes:
+
+		name: (RO) Name of the idle state (string).
+
+		latency: (RO) The latency to exit out of this idle state (in
+		microseconds).
+
+		power: (RO) The power consumed while in this idle state (in
+		milliwatts).
+
+		time: (RO) The total time spent in this idle state (in microseconds).
+
+		usage: (RO) Number of times this state was entered (a count).
+
+
+What:		/sys/devices/system/cpu/cpuX/cpuidle/stateN/desc
+Date:		February 2008
+KernelVersion:	v2.6.25
+Contact:	Linux power management list <linux-pm@vger.kernel.org>
+Description:
+		(RO) A small description about the idle state (string).
+
+
+What:		/sys/devices/system/cpu/cpuX/cpuidle/stateN/disable
+Date:		March 2012
+KernelVersion:	v3.10
+Contact:	Linux power management list <linux-pm@vger.kernel.org>
+Description:
+		(RW) Option to disable this idle state (bool). The behavior and
+		the effect of the disable variable depends on the implementation
+		of a particular governor. In the ladder governor, for example,
+		it is not coherent, i.e. if one is disabling a light state, then
+		all deeper states are disabled as well, but the disable variable
+		does not reflect it. Likewise, if one enables a deep state but a
+		lighter state still is disabled, then this has no effect.
+
+
+What:		/sys/devices/system/cpu/cpuX/cpuidle/stateN/residency
+Date:		March 2014
+KernelVersion:	v3.15
+Contact:	Linux power management list <linux-pm@vger.kernel.org>
+Description:
+		(RO) Display the target residency i.e. the minimum amount of
+		time (in microseconds) this cpu should spend in this idle state
+		to make the transition worth the effort.
+
+
 What:		/sys/devices/system/cpu/cpu#/cpufreq/*
 Date:		pre-git history
 Contact:	linux-pm@vger.kernel.org
diff --git a/Documentation/ABI/testing/sysfs-driver-samsung-laptop b/Documentation/ABI/testing/sysfs-driver-samsung-laptop
index 63c1ad0212fc..34d3a3359cf4 100644
--- a/Documentation/ABI/testing/sysfs-driver-samsung-laptop
+++ b/Documentation/ABI/testing/sysfs-driver-samsung-laptop
@@ -3,7 +3,7 @@ Date:		January 1, 2010
 KernelVersion:	2.6.33
 Contact:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
 Description:	Some Samsung laptops have different "performance levels"
-		that are can be modified by a function key, and by this
+		that can be modified by a function key, and by this
 		sysfs file.  These values don't always make a whole lot
 		of sense, but some users like to modify them to keep
 		their fans quiet at all costs.  Reading from this file
diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs
index a7799c2fca28..d870b5514d15 100644
--- a/Documentation/ABI/testing/sysfs-fs-f2fs
+++ b/Documentation/ABI/testing/sysfs-fs-f2fs
@@ -186,3 +186,9 @@ Date:		August 2017
 Contact:	"Jaegeuk Kim" <jaegeuk@kernel.org>
 Description:
 		 Controls sleep time of GC urgent mode
+
+What:		/sys/fs/f2fs/<disk>/readdir_ra
+Date:		November 2017
+Contact:	"Sheng Yong" <shengyong1@huawei.com>
+Description:
+		 Controls readahead inode block in readdir.
diff --git a/Documentation/ABI/testing/sysfs-kernel-livepatch b/Documentation/ABI/testing/sysfs-kernel-livepatch
index d5d39748382f..dac7e1e62a8b 100644
--- a/Documentation/ABI/testing/sysfs-kernel-livepatch
+++ b/Documentation/ABI/testing/sysfs-kernel-livepatch
@@ -33,6 +33,32 @@ Description:
 		An attribute which indicates whether the patch is currently in
 		transition.
 
+What:		/sys/kernel/livepatch/<patch>/signal
+Date:		Nov 2017
+KernelVersion:	4.15.0
+Contact:	live-patching@vger.kernel.org
+Description:
+		A writable attribute that allows administrator to affect the
+		course of an existing transition. Writing 1 sends a fake
+		signal to all remaining blocking tasks. The fake signal
+		means that no proper signal is delivered (there is no data in
+		signal pending structures). Tasks are interrupted or woken up,
+		and forced to change their patched state.
+
+What:		/sys/kernel/livepatch/<patch>/force
+Date:		Nov 2017
+KernelVersion:	4.15.0
+Contact:	live-patching@vger.kernel.org
+Description:
+		A writable attribute that allows administrator to affect the
+		course of an existing transition. Writing 1 clears
+		TIF_PATCH_PENDING flag of all tasks and thus forces the tasks to
+		the patched or unpatched state. Administrator should not
+		use this feature without a clearance from a patch
+		distributor. Removal (rmmod) of patch modules is permanently
+		disabled when the feature is used. See
+		Documentation/livepatch/livepatch.txt for more information.
+
 What:		/sys/kernel/livepatch/<patch>/<object>
 Date:		Nov 2014
 KernelVersion:	3.19.0
diff --git a/Documentation/ABI/testing/sysfs-platform-dptf b/Documentation/ABI/testing/sysfs-platform-dptf
new file mode 100644
index 000000000000..325dc0667dbb
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-platform-dptf
@@ -0,0 +1,40 @@
+What:		/sys/bus/platform/devices/INT3407:00/dptf_power/charger_type
+Date:		Jul, 2016
+KernelVersion:	v4.10
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) The charger type - Traditional, Hybrid or NVDC.
+
+What:		/sys/bus/platform/devices/INT3407:00/dptf_power/adapter_rating_mw
+Date:		Jul, 2016
+KernelVersion:	v4.10
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) Adapter rating in milliwatts (the maximum Adapter power).
+		Must be 0 if no AC Adaptor is plugged in.
+
+What:		/sys/bus/platform/devices/INT3407:00/dptf_power/max_platform_power_mw
+Date:		Jul, 2016
+KernelVersion:	v4.10
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) Maximum platform power that can be supported by the battery
+		in milliwatts.
+
+What:		/sys/bus/platform/devices/INT3407:00/dptf_power/platform_power_source
+Date:		Jul, 2016
+KernelVersion:	v4.10
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) Display the platform power source
+		0x00 = DC
+		0x01 = AC
+		0x02 = USB
+		0x03 = Wireless Charger
+
+What:		/sys/bus/platform/devices/INT3407:00/dptf_power/battery_steady_power
+Date:		Jul, 2016
+KernelVersion:	v4.10
+Contact:	linux-acpi@vger.kernel.org
+Description:
+		(RO) The maximum sustained power for battery in milliwatts.
diff --git a/Documentation/IRQ-domain.txt b/Documentation/IRQ-domain.txt
index 4a1cd7645d85..507775cce753 100644
--- a/Documentation/IRQ-domain.txt
+++ b/Documentation/IRQ-domain.txt
@@ -265,37 +265,5 @@ support other architectures, such as ARM, ARM64 etc.
 
 === Debugging ===
 
-If you switch on CONFIG_IRQ_DOMAIN_DEBUG (which depends on
-CONFIG_IRQ_DOMAIN and CONFIG_DEBUG_FS), you will find a new file in
-your debugfs mount point, called irq_domain_mapping. This file
-contains a live snapshot of all the IRQ domains in the system:
-
- name              mapped  linear-max  direct-max  devtree-node
- pl061                  8           8           0  /smb/gpio@e0080000
- pl061                  8           8           0  /smb/gpio@e1050000
- pMSI                   0           0           0  /interrupt-controller@e1101000/v2m@e0080000
- MSI                   37           0           0  /interrupt-controller@e1101000/v2m@e0080000
- GICv2m                37           0           0  /interrupt-controller@e1101000/v2m@e0080000
- GICv2                448         448           0  /interrupt-controller@e1101000
-
-it also iterates over the interrupts to display their mapping in the
-domains, and makes the domain stacking visible:
-
-
-irq    hwirq    chip name        chip data           active  type            domain
-    1  0x00019  GICv2            0xffff00000916bfd8     *    LINEAR          GICv2
-    2  0x0001d  GICv2            0xffff00000916bfd8          LINEAR          GICv2
-    3  0x0001e  GICv2            0xffff00000916bfd8     *    LINEAR          GICv2
-    4  0x0001b  GICv2            0xffff00000916bfd8     *    LINEAR          GICv2
-    5  0x0001a  GICv2            0xffff00000916bfd8          LINEAR          GICv2
-[...]
-   96  0x81808  MSI              0x          (null)           RADIX          MSI
-   96+ 0x00063  GICv2m           0xffff8003ee116980           RADIX          GICv2m
-   96+ 0x00063  GICv2            0xffff00000916bfd8          LINEAR          GICv2
-   97  0x08800  MSI              0x          (null)     *     RADIX          MSI
-   97+ 0x00064  GICv2m           0xffff8003ee116980     *     RADIX          GICv2m
-   97+ 0x00064  GICv2            0xffff00000916bfd8     *    LINEAR          GICv2
-
-Here, interrupts 1-5 are only using a single domain, while 96 and 97
-are build out of a stack of three domain, each level performing a
-particular function.
+Most of the internals of the IRQ subsystem are exposed in debugfs by
+turning CONFIG_GENERIC_IRQ_DEBUGFS on.
diff --git a/Documentation/PCI/pci.txt b/Documentation/PCI/pci.txt
index 611a75e4366e..badb26ac33dc 100644
--- a/Documentation/PCI/pci.txt
+++ b/Documentation/PCI/pci.txt
@@ -570,7 +570,9 @@ your driver if they're helpful, or just use plain hex constants.
 The device IDs are arbitrary hex numbers (vendor controlled) and normally used
 only in a single location, the pci_device_id table.
 
-Please DO submit new vendor/device IDs to http://pciids.sourceforge.net/.
+Please DO submit new vendor/device IDs to http://pci-ids.ucw.cz/.
+There are mirrors of the pci.ids file at http://pciids.sourceforge.net/
+and https://github.com/pciutils/pciids.
 
 
 
diff --git a/Documentation/RCU/Design/Data-Structures/Data-Structures.html b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
index 38d6d800761f..6c06e10bd04b 100644
--- a/Documentation/RCU/Design/Data-Structures/Data-Structures.html
+++ b/Documentation/RCU/Design/Data-Structures/Data-Structures.html
@@ -1097,7 +1097,8 @@ will cause the CPU to disregard the values of its counters on
 its next exit from idle.
 Finally, the <tt>rcu_qs_ctr_snap</tt> field is used to detect
 cases where a given operation has resulted in a quiescent state
-for all flavors of RCU, for example, <tt>cond_resched_rcu_qs()</tt>.
+for all flavors of RCU, for example, <tt>cond_resched()</tt>
+when RCU has indicated a need for quiescent states.
 
 <h5>RCU Callback Handling</h5>
 
@@ -1182,8 +1183,8 @@ CPU (and from tracing) unless otherwise stated.
 Its fields are as follows:
 
 <pre>
-  1   int dynticks_nesting;
-  2   int dynticks_nmi_nesting;
+  1   long dynticks_nesting;
+  2   long dynticks_nmi_nesting;
   3   atomic_t dynticks;
   4   bool rcu_need_heavy_qs;
   5   unsigned long rcu_qs_ctr;
@@ -1191,15 +1192,31 @@ Its fields are as follows:
 </pre>
 
 <p>The <tt>-&gt;dynticks_nesting</tt> field counts the
-nesting depth of normal interrupts.
-In addition, this counter is incremented when exiting dyntick-idle
-mode and decremented when entering it.
+nesting depth of process execution, so that in normal circumstances
+this counter has value zero or one.
+NMIs, irqs, and tracers are counted by the <tt>-&gt;dynticks_nmi_nesting</tt>
+field.
+Because NMIs cannot be masked, changes to this variable have to be
+undertaken carefully using an algorithm provided by Andy Lutomirski.
+The initial transition from idle adds one, and nested transitions
+add two, so that a nesting level of five is represented by a
+<tt>-&gt;dynticks_nmi_nesting</tt> value of nine.
 This counter can therefore be thought of as counting the number
 of reasons why this CPU cannot be permitted to enter dyntick-idle
-mode, aside from non-maskable interrupts (NMIs).
-NMIs are counted by the <tt>-&gt;dynticks_nmi_nesting</tt>
-field, except that NMIs that interrupt non-dyntick-idle execution
-are not counted.
+mode, aside from process-level transitions.
+
+<p>However, it turns out that when running in non-idle kernel context,
+the Linux kernel is fully capable of entering interrupt handlers that
+never exit and perhaps also vice versa.
+Therefore, whenever the <tt>-&gt;dynticks_nesting</tt> field is
+incremented up from zero, the <tt>-&gt;dynticks_nmi_nesting</tt> field
+is set to a large positive number, and whenever the
+<tt>-&gt;dynticks_nesting</tt> field is decremented down to zero,
+the the <tt>-&gt;dynticks_nmi_nesting</tt> field is set to zero.
+Assuming that the number of misnested interrupts is not sufficient
+to overflow the counter, this approach corrects the
+<tt>-&gt;dynticks_nmi_nesting</tt> field every time the corresponding
+CPU enters the idle loop from process context.
 
 </p><p>The <tt>-&gt;dynticks</tt> field counts the corresponding
 CPU's transitions to and from dyntick-idle mode, so that this counter
@@ -1231,14 +1248,16 @@ in response.
 <tr><th>&nbsp;</th></tr>
 <tr><th align="left">Quick Quiz:</th></tr>
 <tr><td>
-	Why not just count all NMIs?
-	Wouldn't that be simpler and less error prone?
+	Why not simply combine the <tt>-&gt;dynticks_nesting</tt>
+	and <tt>-&gt;dynticks_nmi_nesting</tt> counters into a
+	single counter that just counts the number of reasons that
+	the corresponding CPU is non-idle?
 </td></tr>
 <tr><th align="left">Answer:</th></tr>
 <tr><td bgcolor="#ffffff"><font color="ffffff">
-	It seems simpler only until you think hard about how to go about
-	updating the <tt>rcu_dynticks</tt> structure's
-	<tt>-&gt;dynticks</tt> field.
+	Because this would fail in the presence of interrupts whose
+	handlers never return and of handlers that manage to return
+	from a made-up interrupt.
 </font></td></tr>
 <tr><td>&nbsp;</td></tr>
 </table>
diff --git a/Documentation/RCU/Design/Requirements/Requirements.html b/Documentation/RCU/Design/Requirements/Requirements.html
index 62e847bcdcdd..49690228b1c6 100644
--- a/Documentation/RCU/Design/Requirements/Requirements.html
+++ b/Documentation/RCU/Design/Requirements/Requirements.html
@@ -581,7 +581,8 @@ This guarantee was only partially premeditated.
 DYNIX/ptx used an explicit memory barrier for publication, but had nothing
 resembling <tt>rcu_dereference()</tt> for subscription, nor did it
 have anything resembling the <tt>smp_read_barrier_depends()</tt>
-that was later subsumed into <tt>rcu_dereference()</tt>.
+that was later subsumed into <tt>rcu_dereference()</tt> and later
+still into <tt>READ_ONCE()</tt>.
 The need for these operations made itself known quite suddenly at a
 late-1990s meeting with the DEC Alpha architects, back in the days when
 DEC was still a free-standing company.
@@ -2797,7 +2798,7 @@ RCU must avoid degrading real-time response for CPU-bound threads, whether
 executing in usermode (which is one use case for
 <tt>CONFIG_NO_HZ_FULL=y</tt>) or in the kernel.
 That said, CPU-bound loops in the kernel must execute
-<tt>cond_resched_rcu_qs()</tt> at least once per few tens of milliseconds
+<tt>cond_resched()</tt> at least once per few tens of milliseconds
 in order to avoid receiving an IPI from RCU.
 
 <p>
@@ -3128,7 +3129,7 @@ The solution, in the form of
 is to have implicit
 read-side critical sections that are delimited by voluntary context
 switches, that is, calls to <tt>schedule()</tt>,
-<tt>cond_resched_rcu_qs()</tt>, and
+<tt>cond_resched()</tt>, and
 <tt>synchronize_rcu_tasks()</tt>.
 In addition, transitions to and from userspace execution also delimit
 tasks-RCU read-side critical sections.
diff --git a/Documentation/RCU/rcu_dereference.txt b/Documentation/RCU/rcu_dereference.txt
index 1acb26b09b48..ab96227bad42 100644
--- a/Documentation/RCU/rcu_dereference.txt
+++ b/Documentation/RCU/rcu_dereference.txt
@@ -122,11 +122,7 @@ o	Be very careful about comparing pointers obtained from
 		Note that if checks for being within an RCU read-side
 		critical section are not required and the pointer is never
 		dereferenced, rcu_access_pointer() should be used in place
-		of rcu_dereference(). The rcu_access_pointer() primitive
-		does not require an enclosing read-side critical section,
-		and also omits the smp_read_barrier_depends() included in
-		rcu_dereference(), which in turn should provide a small
-		performance gain in some CPUs (e.g., the DEC Alpha).
+		of rcu_dereference().
 
 	o	The comparison is against a pointer that references memory
 		that was initialized "a long time ago."  The reason
diff --git a/Documentation/RCU/stallwarn.txt b/Documentation/RCU/stallwarn.txt
index a08f928c8557..4259f95c3261 100644
--- a/Documentation/RCU/stallwarn.txt
+++ b/Documentation/RCU/stallwarn.txt
@@ -23,12 +23,10 @@ o	A CPU looping with preemption disabled.  This condition can
 o	A CPU looping with bottom halves disabled.  This condition can
 	result in RCU-sched and RCU-bh stalls.
 
-o	For !CONFIG_PREEMPT kernels, a CPU looping anywhere in the
-	kernel without invoking schedule().  Note that cond_resched()
-	does not necessarily prevent RCU CPU stall warnings.  Therefore,
-	if the looping in the kernel is really expected and desirable
-	behavior, you might need to replace some of the cond_resched()
-	calls with calls to cond_resched_rcu_qs().
+o	For !CONFIG_PREEMPT kernels, a CPU looping anywhere in the kernel
+	without invoking schedule().  If the looping in the kernel is
+	really expected and desirable behavior, you might need to add
+	some calls to cond_resched().
 
 o	Booting Linux using a console connection that is too slow to
 	keep up with the boot-time console-message rate.  For example,
diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt
index df62466da4e0..a27fbfb0efb8 100644
--- a/Documentation/RCU/whatisRCU.txt
+++ b/Documentation/RCU/whatisRCU.txt
@@ -600,8 +600,7 @@ don't forget about them when submitting patches making use of RCU!]
 
 	#define rcu_dereference(p) \
 	({ \
-		typeof(p) _________p1 = p; \
-		smp_read_barrier_depends(); \
+		typeof(p) _________p1 = READ_ONCE(p); \
 		(_________p1); \
 	})
 
diff --git a/Documentation/accelerators/ocxl.rst b/Documentation/accelerators/ocxl.rst
new file mode 100644
index 000000000000..ddcc58d01cfb
--- /dev/null
+++ b/Documentation/accelerators/ocxl.rst
@@ -0,0 +1,165 @@
+========================================================
+OpenCAPI (Open Coherent Accelerator Processor Interface)
+========================================================
+
+OpenCAPI is an interface between processors and accelerators. It aims
+at being low-latency and high-bandwidth. The specification is
+developed by the `OpenCAPI Consortium <http://opencapi.org/>`_.
+
+It allows an accelerator (which could be a FPGA, ASICs, ...) to access
+the host memory coherently, using virtual addresses. An OpenCAPI
+device can also host its own memory, that can be accessed from the
+host.
+
+OpenCAPI is known in linux as 'ocxl', as the open, processor-agnostic
+evolution of 'cxl' (the driver for the IBM CAPI interface for
+powerpc), which was named that way to avoid confusion with the ISDN
+CAPI subsystem.
+
+
+High-level view
+===============
+
+OpenCAPI defines a Data Link Layer (DL) and Transaction Layer (TL), to
+be implemented on top of a physical link. Any processor or device
+implementing the DL and TL can start sharing memory.
+
+::
+
+  +-----------+                         +-------------+
+  |           |                         |             |
+  |           |                         | Accelerated |
+  | Processor |                         |  Function   |
+  |           |  +--------+             |    Unit     |  +--------+
+  |           |--| Memory |             |    (AFU)    |--| Memory |
+  |           |  +--------+             |             |  +--------+
+  +-----------+                         +-------------+
+       |                                       |
+  +-----------+                         +-------------+
+  |    TL     |                         |    TLX      |
+  +-----------+                         +-------------+
+       |                                       |
+  +-----------+                         +-------------+
+  |    DL     |                         |    DLX      |
+  +-----------+                         +-------------+
+       |                                       |
+       |                   PHY                 |
+       +---------------------------------------+
+
+
+
+Device discovery
+================
+
+OpenCAPI relies on a PCI-like configuration space, implemented on the
+device. So the host can discover AFUs by querying the config space.
+
+OpenCAPI devices in Linux are treated like PCI devices (with a few
+caveats). The firmware is expected to abstract the hardware as if it
+was a PCI link. A lot of the existing PCI infrastructure is reused:
+devices are scanned and BARs are assigned during the standard PCI
+enumeration. Commands like 'lspci' can therefore be used to see what
+devices are available.
+
+The configuration space defines the AFU(s) that can be found on the
+physical adapter, such as its name, how many memory contexts it can
+work with, the size of its MMIO areas, ...
+
+
+
+MMIO
+====
+
+OpenCAPI defines two MMIO areas for each AFU:
+
+* the global MMIO area, with registers pertinent to the whole AFU.
+* a per-process MMIO area, which has a fixed size for each context.
+
+
+
+AFU interrupts
+==============
+
+OpenCAPI includes the possibility for an AFU to send an interrupt to a
+host process. It is done through a 'intrp_req' defined in the
+Transaction Layer, specifying a 64-bit object handle which defines the
+interrupt.
+
+The driver allows a process to allocate an interrupt and obtain its
+64-bit object handle, that can be passed to the AFU.
+
+
+
+char devices
+============
+
+The driver creates one char device per AFU found on the physical
+device. A physical device may have multiple functions and each
+function can have multiple AFUs. At the time of this writing though,
+it has only been tested with devices exporting only one AFU.
+
+Char devices can be found in /dev/ocxl/ and are named as:
+/dev/ocxl/<AFU name>.<location>.<index>
+
+where <AFU name> is a max 20-character long name, as found in the
+config space of the AFU.
+<location> is added by the driver and can help distinguish devices
+when a system has more than one instance of the same OpenCAPI device.
+<index> is also to help distinguish AFUs in the unlikely case where a
+device carries multiple copies of the same AFU.
+
+
+
+Sysfs class
+===========
+
+An ocxl class is added for the devices representing the AFUs. See
+/sys/class/ocxl. The layout is described in
+Documentation/ABI/testing/sysfs-class-ocxl
+
+
+
+User API
+========
+
+open
+----
+
+Based on the AFU definition found in the config space, an AFU may
+support working with more than one memory context, in which case the
+associated char device may be opened multiple times by different
+processes.
+
+
+ioctl
+-----
+
+OCXL_IOCTL_ATTACH:
+
+  Attach the memory context of the calling process to the AFU so that
+  the AFU can access its memory.
+
+OCXL_IOCTL_IRQ_ALLOC:
+
+  Allocate an AFU interrupt and return an identifier.
+
+OCXL_IOCTL_IRQ_FREE:
+
+  Free a previously allocated AFU interrupt.
+
+OCXL_IOCTL_IRQ_SET_FD:
+
+  Associate an event fd to an AFU interrupt so that the user process
+  can be notified when the AFU sends an interrupt.
+
+OCXL_IOCTL_GET_METADATA:
+
+  Obtains configuration information from the card, such at the size of
+  MMIO areas, the AFU version, and the PASID for the current context.
+
+
+mmap
+----
+
+A process can mmap the per-process MMIO area for interactions with the
+AFU.
diff --git a/Documentation/admin-guide/README.rst b/Documentation/admin-guide/README.rst
index 63066db39910..af5a437198d0 100644
--- a/Documentation/admin-guide/README.rst
+++ b/Documentation/admin-guide/README.rst
@@ -170,11 +170,6 @@ Configuring the kernel
                         your existing ./.config file and asking about
                         new config symbols.
 
-     "make silentoldconfig"
-                        Like above, but avoids cluttering the screen
-                        with questions already answered.
-                        Additionally updates the dependencies.
-
      "make olddefconfig"
                         Like above, but sets new symbols to their default
                         values without prompting.
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 46b26bfee27b..1d1d53f85ddd 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -114,7 +114,6 @@
 			This facility can be used to prevent such uncontrolled
 			GPE floodings.
 			Format: <int>
-			Support masking of GPEs numbered from 0x00 to 0x7f.
 
 	acpi_no_auto_serialize	[HW,ACPI]
 			Disable auto-serialization of AML methods
@@ -223,7 +222,7 @@
 
 	acpi_sleep=	[HW,ACPI] Sleep options
 			Format: { s3_bios, s3_mode, s3_beep, s4_nohwsig,
-				  old_ordering, nonvs, sci_force_enable }
+				  old_ordering, nonvs, sci_force_enable, nobl }
 			See Documentation/power/video.txt for information on
 			s3_bios and s3_mode.
 			s3_beep is for debugging; it makes the PC's speaker beep
@@ -239,6 +238,9 @@
 			sci_force_enable causes the kernel to set SCI_EN directly
 			on resume from S1/S3 (which is against the ACPI spec,
 			but some broken systems don't work without it).
+			nobl causes the internal blacklist of systems known to
+			behave incorrectly in some ways with respect to system
+			suspend and resume to be ignored (use wisely).
 
 	acpi_use_timer_override [HW,ACPI]
 			Use timer override. For some broken Nvidia NF5 boards
@@ -644,6 +646,20 @@
 			console=brl,ttyS0
 		For now, only VisioBraille is supported.
 
+	console_msg_format=
+			[KNL] Change console messages format
+		default
+			By default we print messages on consoles in
+			"[time stamp] text\n" format (time stamp may not be
+			printed, depending on CONFIG_PRINTK_TIME or
+			`printk_time' param).
+		syslog
+			Switch to syslog format: "<%u>[time stamp] text\n"
+			IOW, each message will have a facility and loglevel
+			prefix. The format is similar to one used by syslog()
+			syscall, or to executing "dmesg -S --raw" or to reading
+			from /proc/kmsg.
+
 	consoleblank=	[KNL] The console blank (screen saver) timeout in
 			seconds. A value of 0 disables the blank timer.
                        Defaults to 0.
@@ -915,9 +931,12 @@
 
 	earlycon=	[KNL] Output early console device and options.
 
-			When used with no options, the early console is
-			determined by the stdout-path property in device
-			tree's chosen node.
+			[ARM64] The early console is determined by the
+			stdout-path property in device tree's chosen node,
+			or determined by the ACPI SPCR table.
+
+			[X86] When used with no options the early console is
+			determined by the ACPI SPCR table.
 
 		cdns,<addr>[,options]
 			Start an early, polled-mode console on a Cadence
@@ -2050,9 +2069,6 @@
 			This tests the locking primitive's ability to
 			transition abruptly to and from idle.
 
-	locktorture.torture_runnable= [BOOT]
-			Start locktorture running at boot time.
-
 	locktorture.torture_type= [KNL]
 			Specify the locking implementation to test.
 
@@ -2539,6 +2555,9 @@
 			This is useful when you use a panic=... timeout and
 			need the box quickly up again.
 
+			These settings can be accessed at runtime via
+			the nmi_watchdog and hardlockup_panic sysctls.
+
 	netpoll.carrier_timeout=
 			[NET] Specifies amount of time (in seconds) that
 			netpoll should wait for a carrier. By default netpoll
@@ -2742,8 +2761,6 @@
 	norandmaps	Don't use address space randomization.  Equivalent to
 			echo 0 > /proc/sys/kernel/randomize_va_space
 
-	noreplace-paravirt	[X86,IA-64,PV_OPS] Don't patch paravirt_ops
-
 	noreplace-smp	[X86-32,SMP] Don't replace SMP instructions
 			with UP alternatives
 
@@ -3486,9 +3503,6 @@
 			the same as for rcuperf.nreaders.
 			N, where N is the number of CPUs
 
-	rcuperf.perf_runnable= [BOOT]
-			Start rcuperf running at boot time.
-
 	rcuperf.perf_type= [KNL]
 			Specify the RCU implementation to test.
 
@@ -3622,9 +3636,6 @@
 			Test RCU's dyntick-idle handling.  See also the
 			rcutorture.shuffle_interval parameter.
 
-	rcutorture.torture_runnable= [BOOT]
-			Start rcutorture running at boot time.
-
 	rcutorture.torture_type= [KNL]
 			Specify the RCU implementation to test.
 
@@ -3682,7 +3693,8 @@
 
 	rdt=		[HW,X86,RDT]
 			Turn on/off individual RDT features. List is:
-			cmt, mbmtotal, mbmlocal, l3cat, l3cdp, l2cat, mba.
+			cmt, mbmtotal, mbmlocal, l3cat, l3cdp, l2cat, l2cdp,
+			mba.
 			E.g. to turn on cmt and turn off mba use:
 				rdt=cmt,!mba
 
@@ -3702,7 +3714,11 @@
 			[KNL, SMP] Set scheduler's default relax_domain_level.
 			See Documentation/cgroup-v1/cpusets.txt.
 
-	reserve=	[KNL,BUGS] Force the kernel to ignore some iomem area
+	reserve=	[KNL,BUGS] Force kernel to ignore I/O ports or memory
+			Format: <base1>,<size1>[,<base2>,<size2>,...]
+			Reserve I/O ports or memory so the kernel won't use
+			them.  If <base> is less than 0x10000, the region
+			is assumed to be I/O ports; otherwise it is memory.
 
 	reservetop=	[X86-32]
 			Format: nn[KMG]
diff --git a/Documentation/admin-guide/mono.rst b/Documentation/admin-guide/mono.rst
index cdddc099af64..59e6d59f0ed9 100644
--- a/Documentation/admin-guide/mono.rst
+++ b/Documentation/admin-guide/mono.rst
@@ -9,14 +9,14 @@ This will allow you to execute Mono-based .NET binaries just like any
 other program after you have done the following:
 
 1) You MUST FIRST install the Mono CLR support, either by downloading
-   a binary package, a source tarball or by installing from CVS. Binary
+   a binary package, a source tarball or by installing from Git. Binary
    packages for several distributions can be found at:
 
-	http://go-mono.com/download.html
+	http://www.mono-project.com/download/
 
    Instructions for compiling Mono can be found at:
 
-	http://www.go-mono.com/compiling.html
+	http://www.mono-project.com/docs/compiling-mono/linux/
 
    Once the Mono CLR support has been installed, just check that
    ``/usr/bin/mono`` (which could be located elsewhere, for example
diff --git a/Documentation/admin-guide/thunderbolt.rst b/Documentation/admin-guide/thunderbolt.rst
index 9b55952039a6..9948ec36a204 100644
--- a/Documentation/admin-guide/thunderbolt.rst
+++ b/Documentation/admin-guide/thunderbolt.rst
@@ -3,13 +3,13 @@
 =============
 The interface presented here is not meant for end users. Instead there
 should be a userspace tool that handles all the low-level details, keeps
-database of the authorized devices and prompts user for new connections.
+a database of the authorized devices and prompts users for new connections.
 
 More details about the sysfs interface for Thunderbolt devices can be
 found in ``Documentation/ABI/testing/sysfs-bus-thunderbolt``.
 
 Those users who just want to connect any device without any sort of
-manual work, can add following line to
+manual work can add following line to
 ``/etc/udev/rules.d/99-local.rules``::
 
   ACTION=="add", SUBSYSTEM=="thunderbolt", ATTR{authorized}=="0", ATTR{authorized}="1"
@@ -20,7 +20,7 @@ vulnerable to DMA attacks.
 
 Security levels and how to use them
 -----------------------------------
-Starting from Intel Falcon Ridge Thunderbolt controller there are 4
+Starting with Intel Falcon Ridge Thunderbolt controller there are 4
 security levels available. The reason for these is the fact that the
 connected devices can be DMA masters and thus read contents of the host
 memory without CPU and OS knowing about it. There are ways to prevent
@@ -37,14 +37,14 @@ The security levels are as follows:
   user
     User is asked whether the device is allowed to be connected.
     Based on the device identification information available through
-    ``/sys/bus/thunderbolt/devices``. user then can do the decision.
+    ``/sys/bus/thunderbolt/devices``, the user then can make the decision.
     In BIOS settings this is typically called *Unique ID*.
 
   secure
     User is asked whether the device is allowed to be connected. In
     addition to UUID the device (if it supports secure connect) is sent
     a challenge that should match the expected one based on a random key
-    written to ``key`` sysfs attribute. In BIOS settings this is
+    written to the ``key`` sysfs attribute. In BIOS settings this is
     typically called *One time saved key*.
 
   dponly
@@ -78,7 +78,7 @@ When a device is plugged in it will appear in sysfs as follows::
   /sys/bus/thunderbolt/devices/0-1/unique_id	- e0376f00-0300-0100-ffff-ffffffffffff
 
 The ``authorized`` attribute reads 0 which means no PCIe tunnels are
-created yet. The user can authorize the device by simply::
+created yet. The user can authorize the device by simply entering::
 
   # echo 1 > /sys/bus/thunderbolt/devices/0-1/authorized
 
@@ -86,7 +86,7 @@ This will create the PCIe tunnels and the device is now connected.
 
 If the device supports secure connect, and the domain security level is
 set to ``secure``, it has an additional attribute ``key`` which can hold
-a random 32 byte value used for authorization and challenging the device in
+a random 32-byte value used for authorization and challenging the device in
 future connects::
 
   /sys/bus/thunderbolt/devices/0-3/authorized	- 0
@@ -99,12 +99,12 @@ future connects::
 
 Notice the key is empty by default.
 
-If the user does not want to use secure connect it can just ``echo 1``
+If the user does not want to use secure connect they can just ``echo 1``
 to the ``authorized`` attribute and the PCIe tunnels will be created in
-the same way than in ``user`` security level.
+the same way as in the ``user`` security level.
 
 If the user wants to use secure connect, the first time the device is
-plugged a key needs to be created and send to the device::
+plugged a key needs to be created and sent to the device::
 
   # key=$(openssl rand -hex 32)
   # echo $key > /sys/bus/thunderbolt/devices/0-3/key
@@ -121,27 +121,27 @@ device using the same key::
 
 If the challenge the device returns back matches the one we expect based
 on the key, the device is connected and the PCIe tunnels are created.
-However, if the challenge failed no tunnels are created and error is
+However, if the challenge fails no tunnels are created and error is
 returned to the user.
 
-If the user still wants to connect the device it can either approve
-the device without a key or write new key and write 1 to the
+If the user still wants to connect the device they can either approve
+the device without a key or write a new key and write 1 to the
 ``authorized`` file to get the new key stored on the device NVM.
 
 Upgrading NVM on Thunderbolt device or host
 -------------------------------------------
-Since most of the functionality is handled in a firmware running on a
+Since most of the functionality is handled in firmware running on a
 host controller or a device, it is important that the firmware can be
 upgraded to the latest where possible bugs in it have been fixed.
 Typically OEMs provide this firmware from their support site.
 
-There is also a central site which has links where to download firmwares
+There is also a central site which has links where to download firmware
 for some machines:
 
   `Thunderbolt Updates <https://thunderbolttechnology.net/updates>`_
 
-Before you upgrade firmware on a device or host, please make sure it is
-the suitable. Failing to do that may render the device (or host) in a
+Before you upgrade firmware on a device or host, please make sure it is a
+suitable upgrade. Failing to do that may render the device (or host) in a
 state where it cannot be used properly anymore without special tools!
 
 Host NVM upgrade on Apple Macs is not supported.
@@ -151,7 +151,7 @@ Thunderbolt device so that the host controller appears. It does not
 matter which device is connected (unless you are upgrading NVM on a
 device - then you need to connect that particular device).
 
-Note OEM-specific method to power the controller up ("force power") may
+Note an OEM-specific method to power the controller up ("force power") may
 be available for your system in which case there is no need to plug in a
 Thunderbolt device.
 
@@ -171,7 +171,7 @@ it comes back the driver notices it and initiates a full power cycle.
 After a while the host controller appears again and this time it should
 be fully functional.
 
-We can verify that the new NVM firmware is active by running following
+We can verify that the new NVM firmware is active by running the following
 commands::
 
   # cat /sys/bus/thunderbolt/devices/0-0/nvm_authenticate
@@ -179,38 +179,38 @@ commands::
   # cat /sys/bus/thunderbolt/devices/0-0/nvm_version
   18.0
 
-If ``nvm_authenticate`` contains anything else than 0x0 it is the error
+If ``nvm_authenticate`` contains anything other than 0x0 it is the error
 code from the last authentication cycle, which means the authentication
 of the NVM image failed.
 
 Note names of the NVMem devices ``nvm_activeN`` and ``nvm_non_activeN``
-depends on the order they are registered in the NVMem subsystem. N in
+depend on the order they are registered in the NVMem subsystem. N in
 the name is the identifier added by the NVMem subsystem.
 
 Upgrading NVM when host controller is in safe mode
 --------------------------------------------------
 If the existing NVM is not properly authenticated (or is missing) the
-host controller goes into safe mode which means that only available
-functionality is flashing new NVM image. When in this mode the reading
+host controller goes into safe mode which means that the only available
+functionality is flashing a new NVM image. When in this mode, reading
 ``nvm_version`` fails with ``ENODATA`` and the device identification
 information is missing.
 
 To recover from this mode, one needs to flash a valid NVM image to the
-host host controller in the same way it is done in the previous chapter.
+host controller in the same way it is done in the previous chapter.
 
 Networking over Thunderbolt cable
 ---------------------------------
-Thunderbolt technology allows software communication across two hosts
+Thunderbolt technology allows software communication between two hosts
 connected by a Thunderbolt cable.
 
-It is possible to tunnel any kind of traffic over Thunderbolt link but
+It is possible to tunnel any kind of traffic over a Thunderbolt link but
 currently we only support Apple ThunderboltIP protocol.
 
-If the other host is running Windows or macOS only thing you need to
-do is to connect Thunderbolt cable between the two hosts, the
-``thunderbolt-net`` is loaded automatically. If the other host is also
-Linux you should load ``thunderbolt-net`` manually on one host (it does
-not matter which one)::
+If the other host is running Windows or macOS, the only thing you need to
+do is to connect a Thunderbolt cable between the two hosts; the
+``thunderbolt-net`` driver is loaded automatically. If the other host is
+also Linux you should load ``thunderbolt-net`` manually on one host (it
+does not matter which one)::
 
   # modprobe thunderbolt-net
 
@@ -220,12 +220,12 @@ is built-in to the kernel image, there is no need to do anything.
 The driver will create one virtual ethernet interface per Thunderbolt
 port which are named like ``thunderbolt0`` and so on. From this point
 you can either use standard userspace tools like ``ifconfig`` to
-configure the interface or let your GUI to handle it automatically.
+configure the interface or let your GUI handle it automatically.
 
 Forcing power
 -------------
 Many OEMs include a method that can be used to force the power of a
-thunderbolt controller to an "On" state even if nothing is connected.
+Thunderbolt controller to an "On" state even if nothing is connected.
 If supported by your machine this will be exposed by the WMI bus with
 a sysfs attribute called "force_power".
 
diff --git a/Documentation/arm64/cpu-feature-registers.txt b/Documentation/arm64/cpu-feature-registers.txt
index bd9b3faab2c4..a70090b28b07 100644
--- a/Documentation/arm64/cpu-feature-registers.txt
+++ b/Documentation/arm64/cpu-feature-registers.txt
@@ -110,7 +110,9 @@ infrastructure:
      x--------------------------------------------------x
      | Name                         |  bits   | visible |
      |--------------------------------------------------|
-     | RES0                         | [63-48] |    n    |
+     | RES0                         | [63-52] |    n    |
+     |--------------------------------------------------|
+     | FHM                          | [51-48] |    y    |
      |--------------------------------------------------|
      | DP                           | [47-44] |    y    |
      |--------------------------------------------------|
diff --git a/Documentation/arm64/elf_hwcaps.txt b/Documentation/arm64/elf_hwcaps.txt
index 89edba12a9e0..57324ee55ecc 100644
--- a/Documentation/arm64/elf_hwcaps.txt
+++ b/Documentation/arm64/elf_hwcaps.txt
@@ -158,3 +158,7 @@ HWCAP_SHA512
 HWCAP_SVE
 
     Functionality implied by ID_AA64PFR0_EL1.SVE == 0b0001.
+
+HWCAP_ASIMDFHM
+
+   Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0001.
diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt
index fc1c884fea10..c1d520de6dfe 100644
--- a/Documentation/arm64/silicon-errata.txt
+++ b/Documentation/arm64/silicon-errata.txt
@@ -72,7 +72,7 @@ stable kernels.
 | Hisilicon      | Hip0{6,7}       | #161010701      | N/A                         |
 | Hisilicon      | Hip07           | #161600802      | HISILICON_ERRATUM_161600802 |
 |                |                 |                 |                             |
-| Qualcomm Tech. | Falkor v1       | E1003           | QCOM_FALKOR_ERRATUM_1003    |
+| Qualcomm Tech. | Kryo/Falkor v1  | E1003           | QCOM_FALKOR_ERRATUM_1003    |
 | Qualcomm Tech. | Falkor v1       | E1009           | QCOM_FALKOR_ERRATUM_1009    |
 | Qualcomm Tech. | QDF2400 ITS     | E0065           | QCOM_QDF2400_ERRATUM_0065   |
 | Qualcomm Tech. | Falkor v{1,2}   | E1041           | QCOM_FALKOR_ERRATUM_1041    |
diff --git a/Documentation/atomic_bitops.txt b/Documentation/atomic_bitops.txt
index 5550bfdcce5f..be70b32c95d9 100644
--- a/Documentation/atomic_bitops.txt
+++ b/Documentation/atomic_bitops.txt
@@ -58,7 +58,12 @@ Like with atomic_t, the rule of thumb is:
 
  - RMW operations that have a return value are fully ordered.
 
-Except for test_and_set_bit_lock() which has ACQUIRE semantics and
+ - RMW operations that are conditional are unordered on FAILURE,
+   otherwise the above rules apply. In the case of test_and_{}_bit() operations,
+   if the bit in memory is unchanged by the operation then it is deemed to have
+   failed.
+
+Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics and
 clear_bit_unlock() which has RELEASE semantics.
 
 Since a platform only has a single means of achieving atomic operations
diff --git a/Documentation/bpf/bpf_devel_QA.txt b/Documentation/bpf/bpf_devel_QA.txt
new file mode 100644
index 000000000000..84cbb302f2b5
--- /dev/null
+++ b/Documentation/bpf/bpf_devel_QA.txt
@@ -0,0 +1,550 @@
+This document provides information for the BPF subsystem about various
+workflows related to reporting bugs, submitting patches, and queueing
+patches for stable kernels.
+
+For general information about submitting patches, please refer to
+Documentation/process/. This document only describes additional specifics
+related to BPF.
+
+Reporting bugs:
+---------------
+
+Q: How do I report bugs for BPF kernel code?
+
+A: Since all BPF kernel development as well as bpftool and iproute2 BPF
+   loader development happens through the netdev kernel mailing list,
+   please report any found issues around BPF to the following mailing
+   list:
+
+     netdev@vger.kernel.org
+
+   This may also include issues related to XDP, BPF tracing, etc.
+
+   Given netdev has a high volume of traffic, please also add the BPF
+   maintainers to Cc (from kernel MAINTAINERS file):
+
+     Alexei Starovoitov <ast@kernel.org>
+     Daniel Borkmann <daniel@iogearbox.net>
+
+   In case a buggy commit has already been identified, make sure to keep
+   the actual commit authors in Cc as well for the report. They can
+   typically be identified through the kernel's git tree.
+
+   Please do *not* report BPF issues to bugzilla.kernel.org since it
+   is a guarantee that the reported issue will be overlooked.
+
+Submitting patches:
+-------------------
+
+Q: To which mailing list do I need to submit my BPF patches?
+
+A: Please submit your BPF patches to the netdev kernel mailing list:
+
+     netdev@vger.kernel.org
+
+   Historically, BPF came out of networking and has always been maintained
+   by the kernel networking community. Although these days BPF touches
+   many other subsystems as well, the patches are still routed mainly
+   through the networking community.
+
+   In case your patch has changes in various different subsystems (e.g.
+   tracing, security, etc), make sure to Cc the related kernel mailing
+   lists and maintainers from there as well, so they are able to review
+   the changes and provide their Acked-by's to the patches.
+
+Q: Where can I find patches currently under discussion for BPF subsystem?
+
+A: All patches that are Cc'ed to netdev are queued for review under netdev
+   patchwork project:
+
+     http://patchwork.ozlabs.org/project/netdev/list/
+
+   Those patches which target BPF, are assigned to a 'bpf' delegate for
+   further processing from BPF maintainers. The current queue with
+   patches under review can be found at:
+
+     https://patchwork.ozlabs.org/project/netdev/list/?delegate=77147
+
+   Once the patches have been reviewed by the BPF community as a whole
+   and approved by the BPF maintainers, their status in patchwork will be
+   changed to 'Accepted' and the submitter will be notified by mail. This
+   means that the patches look good from a BPF perspective and have been
+   applied to one of the two BPF kernel trees.
+
+   In case feedback from the community requires a respin of the patches,
+   their status in patchwork will be set to 'Changes Requested', and purged
+   from the current review queue. Likewise for cases where patches would
+   get rejected or are not applicable to the BPF trees (but assigned to
+   the 'bpf' delegate).
+
+Q: How do the changes make their way into Linux?
+
+A: There are two BPF kernel trees (git repositories). Once patches have
+   been accepted by the BPF maintainers, they will be applied to one
+   of the two BPF trees:
+
+     https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/
+     https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/
+
+   The bpf tree itself is for fixes only, whereas bpf-next for features,
+   cleanups or other kind of improvements ("next-like" content). This is
+   analogous to net and net-next trees for networking. Both bpf and
+   bpf-next will only have a master branch in order to simplify against
+   which branch patches should get rebased to.
+
+   Accumulated BPF patches in the bpf tree will regularly get pulled
+   into the net kernel tree. Likewise, accumulated BPF patches accepted
+   into the bpf-next tree will make their way into net-next tree. net and
+   net-next are both run by David S. Miller. From there, they will go
+   into the kernel mainline tree run by Linus Torvalds. To read up on the
+   process of net and net-next being merged into the mainline tree, see
+   the netdev FAQ under:
+
+     Documentation/networking/netdev-FAQ.txt
+
+   Occasionally, to prevent merge conflicts, we might send pull requests
+   to other trees (e.g. tracing) with a small subset of the patches, but
+   net and net-next are always the main trees targeted for integration.
+
+   The pull requests will contain a high-level summary of the accumulated
+   patches and can be searched on netdev kernel mailing list through the
+   following subject lines (yyyy-mm-dd is the date of the pull request):
+
+     pull-request: bpf yyyy-mm-dd
+     pull-request: bpf-next yyyy-mm-dd
+
+Q: How do I indicate which tree (bpf vs. bpf-next) my patch should be
+   applied to?
+
+A: The process is the very same as described in the netdev FAQ, so
+   please read up on it. The subject line must indicate whether the
+   patch is a fix or rather "next-like" content in order to let the
+   maintainers know whether it is targeted at bpf or bpf-next.
+
+   For fixes eventually landing in bpf -> net tree, the subject must
+   look like:
+
+     git format-patch --subject-prefix='PATCH bpf' start..finish
+
+   For features/improvements/etc that should eventually land in
+   bpf-next -> net-next, the subject must look like:
+
+     git format-patch --subject-prefix='PATCH bpf-next' start..finish
+
+   If unsure whether the patch or patch series should go into bpf
+   or net directly, or bpf-next or net-next directly, it is not a
+   problem either if the subject line says net or net-next as target.
+   It is eventually up to the maintainers to do the delegation of
+   the patches.
+
+   If it is clear that patches should go into bpf or bpf-next tree,
+   please make sure to rebase the patches against those trees in
+   order to reduce potential conflicts.
+
+   In case the patch or patch series has to be reworked and sent out
+   again in a second or later revision, it is also required to add a
+   version number (v2, v3, ...) into the subject prefix:
+
+     git format-patch --subject-prefix='PATCH net-next v2' start..finish
+
+   When changes have been requested to the patch series, always send the
+   whole patch series again with the feedback incorporated (never send
+   individual diffs on top of the old series).
+
+Q: What does it mean when a patch gets applied to bpf or bpf-next tree?
+
+A: It means that the patch looks good for mainline inclusion from
+   a BPF point of view.
+
+   Be aware that this is not a final verdict that the patch will
+   automatically get accepted into net or net-next trees eventually:
+
+   On the netdev kernel mailing list reviews can come in at any point
+   in time. If discussions around a patch conclude that they cannot
+   get included as-is, we will either apply a follow-up fix or drop
+   them from the trees entirely. Therefore, we also reserve to rebase
+   the trees when deemed necessary. After all, the purpose of the tree
+   is to i) accumulate and stage BPF patches for integration into trees
+   like net and net-next, and ii) run extensive BPF test suite and
+   workloads on the patches before they make their way any further.
+
+   Once the BPF pull request was accepted by David S. Miller, then
+   the patches end up in net or net-next tree, respectively, and
+   make their way from there further into mainline. Again, see the
+   netdev FAQ for additional information e.g. on how often they are
+   merged to mainline.
+
+Q: How long do I need to wait for feedback on my BPF patches?
+
+A: We try to keep the latency low. The usual time to feedback will
+   be around 2 or 3 business days. It may vary depending on the
+   complexity of changes and current patch load.
+
+Q: How often do you send pull requests to major kernel trees like
+   net or net-next?
+
+A: Pull requests will be sent out rather often in order to not
+   accumulate too many patches in bpf or bpf-next.
+
+   As a rule of thumb, expect pull requests for each tree regularly
+   at the end of the week. In some cases pull requests could additionally
+   come also in the middle of the week depending on the current patch
+   load or urgency.
+
+Q: Are patches applied to bpf-next when the merge window is open?
+
+A: For the time when the merge window is open, bpf-next will not be
+   processed. This is roughly analogous to net-next patch processing,
+   so feel free to read up on the netdev FAQ about further details.
+
+   During those two weeks of merge window, we might ask you to resend
+   your patch series once bpf-next is open again. Once Linus released
+   a v*-rc1 after the merge window, we continue processing of bpf-next.
+
+   For non-subscribers to kernel mailing lists, there is also a status
+   page run by David S. Miller on net-next that provides guidance:
+
+     http://vger.kernel.org/~davem/net-next.html
+
+Q: I made a BPF verifier change, do I need to add test cases for
+   BPF kernel selftests?
+
+A: If the patch has changes to the behavior of the verifier, then yes,
+   it is absolutely necessary to add test cases to the BPF kernel
+   selftests suite. If they are not present and we think they are
+   needed, then we might ask for them before accepting any changes.
+
+   In particular, test_verifier.c is tracking a high number of BPF test
+   cases, including a lot of corner cases that LLVM BPF back end may
+   generate out of the restricted C code. Thus, adding test cases is
+   absolutely crucial to make sure future changes do not accidentally
+   affect prior use-cases. Thus, treat those test cases as: verifier
+   behavior that is not tracked in test_verifier.c could potentially
+   be subject to change.
+
+Q: When should I add code to samples/bpf/ and when to BPF kernel
+   selftests?
+
+A: In general, we prefer additions to BPF kernel selftests rather than
+   samples/bpf/. The rationale is very simple: kernel selftests are
+   regularly run by various bots to test for kernel regressions.
+
+   The more test cases we add to BPF selftests, the better the coverage
+   and the less likely it is that those could accidentally break. It is
+   not that BPF kernel selftests cannot demo how a specific feature can
+   be used.
+
+   That said, samples/bpf/ may be a good place for people to get started,
+   so it might be advisable that simple demos of features could go into
+   samples/bpf/, but advanced functional and corner-case testing rather
+   into kernel selftests.
+
+   If your sample looks like a test case, then go for BPF kernel selftests
+   instead!
+
+Q: When should I add code to the bpftool?
+
+A: The main purpose of bpftool (under tools/bpf/bpftool/) is to provide
+   a central user space tool for debugging and introspection of BPF programs
+   and maps that are active in the kernel. If UAPI changes related to BPF
+   enable for dumping additional information of programs or maps, then
+   bpftool should be extended as well to support dumping them.
+
+Q: When should I add code to iproute2's BPF loader?
+
+A: For UAPI changes related to the XDP or tc layer (e.g. cls_bpf), the
+   convention is that those control-path related changes are added to
+   iproute2's BPF loader as well from user space side. This is not only
+   useful to have UAPI changes properly designed to be usable, but also
+   to make those changes available to a wider user base of major
+   downstream distributions.
+
+Q: Do you accept patches as well for iproute2's BPF loader?
+
+A: Patches for the iproute2's BPF loader have to be sent to:
+
+     netdev@vger.kernel.org
+
+   While those patches are not processed by the BPF kernel maintainers,
+   please keep them in Cc as well, so they can be reviewed.
+
+   The official git repository for iproute2 is run by Stephen Hemminger
+   and can be found at:
+
+     https://git.kernel.org/pub/scm/linux/kernel/git/shemminger/iproute2.git/
+
+   The patches need to have a subject prefix of '[PATCH iproute2 master]'
+   or '[PATCH iproute2 net-next]'. 'master' or 'net-next' describes the
+   target branch where the patch should be applied to. Meaning, if kernel
+   changes went into the net-next kernel tree, then the related iproute2
+   changes need to go into the iproute2 net-next branch, otherwise they
+   can be targeted at master branch. The iproute2 net-next branch will get
+   merged into the master branch after the current iproute2 version from
+   master has been released.
+
+   Like BPF, the patches end up in patchwork under the netdev project and
+   are delegated to 'shemminger' for further processing:
+
+     http://patchwork.ozlabs.org/project/netdev/list/?delegate=389
+
+Q: What is the minimum requirement before I submit my BPF patches?
+
+A: When submitting patches, always take the time and properly test your
+   patches *prior* to submission. Never rush them! If maintainers find
+   that your patches have not been properly tested, it is a good way to
+   get them grumpy. Testing patch submissions is a hard requirement!
+
+   Note, fixes that go to bpf tree *must* have a Fixes: tag included. The
+   same applies to fixes that target bpf-next, where the affected commit
+   is in net-next (or in some cases bpf-next). The Fixes: tag is crucial
+   in order to identify follow-up commits and tremendously helps for people
+   having to do backporting, so it is a must have!
+
+   We also don't accept patches with an empty commit message. Take your
+   time and properly write up a high quality commit message, it is
+   essential!
+
+   Think about it this way: other developers looking at your code a month
+   from now need to understand *why* a certain change has been done that
+   way, and whether there have been flaws in the analysis or assumptions
+   that the original author did. Thus providing a proper rationale and
+   describing the use-case for the changes is a must.
+
+   Patch submissions with >1 patch must have a cover letter which includes
+   a high level description of the series. This high level summary will
+   then be placed into the merge commit by the BPF maintainers such that
+   it is also accessible from the git log for future reference.
+
+Q: What do I need to consider when adding a new instruction or feature
+   that would require BPF JIT and/or LLVM integration as well?
+
+A: We try hard to keep all BPF JITs up to date such that the same user
+   experience can be guaranteed when running BPF programs on different
+   architectures without having the program punt to the less efficient
+   interpreter in case the in-kernel BPF JIT is enabled.
+
+   If you are unable to implement or test the required JIT changes for
+   certain architectures, please work together with the related BPF JIT
+   developers in order to get the feature implemented in a timely manner.
+   Please refer to the git log (arch/*/net/) to locate the necessary
+   people for helping out.
+
+   Also always make sure to add BPF test cases (e.g. test_bpf.c and
+   test_verifier.c) for new instructions, so that they can receive
+   broad test coverage and help run-time testing the various BPF JITs.
+
+   In case of new BPF instructions, once the changes have been accepted
+   into the Linux kernel, please implement support into LLVM's BPF back
+   end. See LLVM section below for further information.
+
+Stable submission:
+------------------
+
+Q: I need a specific BPF commit in stable kernels. What should I do?
+
+A: In case you need a specific fix in stable kernels, first check whether
+   the commit has already been applied in the related linux-*.y branches:
+
+     https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git/
+
+   If not the case, then drop an email to the BPF maintainers with the
+   netdev kernel mailing list in Cc and ask for the fix to be queued up:
+
+     netdev@vger.kernel.org
+
+   The process in general is the same as on netdev itself, see also the
+   netdev FAQ document.
+
+Q: Do you also backport to kernels not currently maintained as stable?
+
+A: No. If you need a specific BPF commit in kernels that are currently not
+   maintained by the stable maintainers, then you are on your own.
+
+   The current stable and longterm stable kernels are all listed here:
+
+     https://www.kernel.org/
+
+Q: The BPF patch I am about to submit needs to go to stable as well. What
+   should I do?
+
+A: The same rules apply as with netdev patch submissions in general, see
+   netdev FAQ under:
+
+     Documentation/networking/netdev-FAQ.txt
+
+   Never add "Cc: stable@vger.kernel.org" to the patch description, but
+   ask the BPF maintainers to queue the patches instead. This can be done
+   with a note, for example, under the "---" part of the patch which does
+   not go into the git log. Alternatively, this can be done as a simple
+   request by mail instead.
+
+Q: Where do I find currently queued BPF patches that will be submitted
+   to stable?
+
+A: Once patches that fix critical bugs got applied into the bpf tree, they
+   are queued up for stable submission under:
+
+     http://patchwork.ozlabs.org/bundle/bpf/stable/?state=*
+
+   They will be on hold there at minimum until the related commit made its
+   way into the mainline kernel tree.
+
+   After having been under broader exposure, the queued patches will be
+   submitted by the BPF maintainers to the stable maintainers.
+
+Testing patches:
+----------------
+
+Q: Which BPF kernel selftests version should I run my kernel against?
+
+A: If you run a kernel xyz, then always run the BPF kernel selftests from
+   that kernel xyz as well. Do not expect that the BPF selftest from the
+   latest mainline tree will pass all the time.
+
+   In particular, test_bpf.c and test_verifier.c have a large number of
+   test cases and are constantly updated with new BPF test sequences, or
+   existing ones are adapted to verifier changes e.g. due to verifier
+   becoming smarter and being able to better track certain things.
+
+LLVM:
+-----
+
+Q: Where do I find LLVM with BPF support?
+
+A: The BPF back end for LLVM is upstream in LLVM since version 3.7.1.
+
+   All major distributions these days ship LLVM with BPF back end enabled,
+   so for the majority of use-cases it is not required to compile LLVM by
+   hand anymore, just install the distribution provided package.
+
+   LLVM's static compiler lists the supported targets through 'llc --version',
+   make sure BPF targets are listed. Example:
+
+     $ llc --version
+     LLVM (http://llvm.org/):
+       LLVM version 6.0.0svn
+       Optimized build.
+       Default target: x86_64-unknown-linux-gnu
+       Host CPU: skylake
+
+       Registered Targets:
+         bpf    - BPF (host endian)
+         bpfeb  - BPF (big endian)
+         bpfel  - BPF (little endian)
+         x86    - 32-bit X86: Pentium-Pro and above
+         x86-64 - 64-bit X86: EM64T and AMD64
+
+   For developers in order to utilize the latest features added to LLVM's
+   BPF back end, it is advisable to run the latest LLVM releases. Support
+   for new BPF kernel features such as additions to the BPF instruction
+   set are often developed together.
+
+   All LLVM releases can be found at: http://releases.llvm.org/
+
+Q: Got it, so how do I build LLVM manually anyway?
+
+A: You need cmake and gcc-c++ as build requisites for LLVM. Once you have
+   that set up, proceed with building the latest LLVM and clang version
+   from the git repositories:
+
+     $ git clone http://llvm.org/git/llvm.git
+     $ cd llvm/tools
+     $ git clone --depth 1 http://llvm.org/git/clang.git
+     $ cd ..; mkdir build; cd build
+     $ cmake .. -DLLVM_TARGETS_TO_BUILD="BPF;X86" \
+                -DBUILD_SHARED_LIBS=OFF           \
+                -DCMAKE_BUILD_TYPE=Release        \
+                -DLLVM_BUILD_RUNTIME=OFF
+     $ make -j $(getconf _NPROCESSORS_ONLN)
+
+   The built binaries can then be found in the build/bin/ directory, where
+   you can point the PATH variable to.
+
+Q: Should I notify BPF kernel maintainers about issues in LLVM's BPF code
+   generation back end or about LLVM generated code that the verifier
+   refuses to accept?
+
+A: Yes, please do! LLVM's BPF back end is a key piece of the whole BPF
+   infrastructure and it ties deeply into verification of programs from the
+   kernel side. Therefore, any issues on either side need to be investigated
+   and fixed whenever necessary.
+
+   Therefore, please make sure to bring them up at netdev kernel mailing
+   list and Cc BPF maintainers for LLVM and kernel bits:
+
+     Yonghong Song <yhs@fb.com>
+     Alexei Starovoitov <ast@kernel.org>
+     Daniel Borkmann <daniel@iogearbox.net>
+
+   LLVM also has an issue tracker where BPF related bugs can be found:
+
+     https://bugs.llvm.org/buglist.cgi?quicksearch=bpf
+
+   However, it is better to reach out through mailing lists with having
+   maintainers in Cc.
+
+Q: I have added a new BPF instruction to the kernel, how can I integrate
+   it into LLVM?
+
+A: LLVM has a -mcpu selector for the BPF back end in order to allow the
+   selection of BPF instruction set extensions. By default the 'generic'
+   processor target is used, which is the base instruction set (v1) of BPF.
+
+   LLVM has an option to select -mcpu=probe where it will probe the host
+   kernel for supported BPF instruction set extensions and selects the
+   optimal set automatically.
+
+   For cross-compilation, a specific version can be select manually as well.
+
+     $ llc -march bpf -mcpu=help
+     Available CPUs for this target:
+
+       generic - Select the generic processor.
+       probe   - Select the probe processor.
+       v1      - Select the v1 processor.
+       v2      - Select the v2 processor.
+     [...]
+
+   Newly added BPF instructions to the Linux kernel need to follow the same
+   scheme, bump the instruction set version and implement probing for the
+   extensions such that -mcpu=probe users can benefit from the optimization
+   transparently when upgrading their kernels.
+
+   If you are unable to implement support for the newly added BPF instruction
+   please reach out to BPF developers for help.
+
+   By the way, the BPF kernel selftests run with -mcpu=probe for better
+   test coverage.
+
+Q: In some cases clang flag "-target bpf" is used but in other cases the
+   default clang target, which matches the underlying architecture, is used.
+   What is the difference and when I should use which?
+
+A: Although LLVM IR generation and optimization try to stay architecture
+   independent, "-target <arch>" still has some impact on generated code:
+
+     - BPF program may recursively include header file(s) with file scope
+       inline assembly codes. The default target can handle this well,
+       while bpf target may fail if bpf backend assembler does not
+       understand these assembly codes, which is true in most cases.
+
+     - When compiled without -g, additional elf sections, e.g.,
+       .eh_frame and .rela.eh_frame, may be present in the object file
+       with default target, but not with bpf target.
+
+     - The default target may turn a C switch statement into a switch table
+       lookup and jump operation. Since the switch table is placed
+       in the global readonly section, the bpf program will fail to load.
+       The bpf target does not support switch table optimization.
+       The clang option "-fno-jump-tables" can be used to disable
+       switch table generation.
+
+   You should use default target when:
+
+     - Your program includes a header file, e.g., ptrace.h, which eventually
+       pulls in some header files containing file scope host assembly codes.
+     - You can add "-fno-jump-tables" to work around the switch table issue.
+
+   Otherwise, you can use bpf target.
+
+Happy BPF hacking!
diff --git a/Documentation/cgroup-v1/cgroups.txt b/Documentation/cgroup-v1/cgroups.txt
index 308e5ff7207a..059f7063eea6 100644
--- a/Documentation/cgroup-v1/cgroups.txt
+++ b/Documentation/cgroup-v1/cgroups.txt
@@ -523,12 +523,7 @@ Accessing a task's cgroup pointer may be done in the following ways:
 Each subsystem should:
 
 - add an entry in linux/cgroup_subsys.h
-- define a cgroup_subsys object called <name>_subsys
-
-If a subsystem can be compiled as a module, it should also have in its
-module initcall a call to cgroup_load_subsys(), and in its exitcall a
-call to cgroup_unload_subsys(). It should also set its_subsys.module =
-THIS_MODULE in its .c file.
+- define a cgroup_subsys object called <name>_cgrp_subsys
 
 Each subsystem may export the following methods. The only mandatory
 methods are css_alloc/free. Any others that are null are presumed to
diff --git a/Documentation/cgroup-v1/memory.txt b/Documentation/cgroup-v1/memory.txt
index cefb63639070..a4af2e124e24 100644
--- a/Documentation/cgroup-v1/memory.txt
+++ b/Documentation/cgroup-v1/memory.txt
@@ -524,9 +524,9 @@ Note:
 	Only anonymous and swap cache memory is listed as part of 'rss' stat.
 	This should not be confused with the true 'resident set size' or the
 	amount of physical memory used by the cgroup.
-	'rss + file_mapped" will give you resident set size of cgroup.
+	'rss + mapped_file" will give you resident set size of cgroup.
 	(Note: file and shmem may be shared among other cgroups. In that case,
-	 file_mapped is accounted only when the memory cgroup is owner of page
+	 mapped_file is accounted only when the memory cgroup is owner of page
 	 cache.)
 
 5.3 swappiness
diff --git a/Documentation/cgroup-v2.txt b/Documentation/cgroup-v2.txt
index 2cddab7efb20..74cdeaed9f7a 100644
--- a/Documentation/cgroup-v2.txt
+++ b/Documentation/cgroup-v2.txt
@@ -53,10 +53,14 @@ v1 is available under Documentation/cgroup-v1/.
        5-3-2. Writeback
      5-4. PID
        5-4-1. PID Interface Files
-     5-5. RDMA
-       5-5-1. RDMA Interface Files
-     5-6. Misc
-       5-6-1. perf_event
+     5-5. Device
+     5-6. RDMA
+       5-6-1. RDMA Interface Files
+     5-7. Misc
+       5-7-1. perf_event
+     5-N. Non-normative information
+       5-N-1. CPU controller root cgroup process behaviour
+       5-N-2. IO controller root cgroup process behaviour
    6. Namespace
      6-1. Basics
      6-2. The Root and Views
@@ -279,7 +283,7 @@ thread mode, the following conditions must be met.
   exempt from this requirement.
 
 Topology-wise, a cgroup can be in an invalid state.  Please consider
-the following toplogy::
+the following topology::
 
   A (threaded domain) - B (threaded) - C (domain, just created)
 
@@ -420,7 +424,9 @@ The root cgroup is exempt from this restriction.  Root contains
 processes and anonymous resource consumption which can't be associated
 with any other cgroups and requires special treatment from most
 controllers.  How resource consumption in the root cgroup is governed
-is up to each controller.
+is up to each controller (for more information on this topic please
+refer to the Non-normative information section in the Controllers
+chapter).
 
 Note that the restriction doesn't get in the way if there is no
 enabled controller in the cgroup's "cgroup.subtree_control".  This is
@@ -1063,10 +1069,10 @@ PAGE_SIZE multiple when read back.
 		reached the limit and allocation was about to fail.
 
 		Depending on context result could be invocation of OOM
-		killer and retrying allocation or failing alloction.
+		killer and retrying allocation or failing allocation.
 
 		Failed allocation in its turn could be returned into
-		userspace as -ENOMEM or siletly ignored in cases like
+		userspace as -ENOMEM or silently ignored in cases like
 		disk readahead.  For now OOM in memory cgroup kills
 		tasks iff shortage has happened inside page fault.
 
@@ -1191,7 +1197,7 @@ PAGE_SIZE multiple when read back.
 	cgroups.  The default is "max".
 
 	Swap usage hard limit.  If a cgroup's swap usage reaches this
-	limit, anonymous meomry of the cgroup will not be swapped out.
+	limit, anonymous memory of the cgroup will not be swapped out.
 
 
 Usage Guidelines
@@ -1429,6 +1435,30 @@ through fork() or clone(). These will return -EAGAIN if the creation
 of a new process would cause a cgroup policy to be violated.
 
 
+Device controller
+-----------------
+
+Device controller manages access to device files. It includes both
+creation of new device files (using mknod), and access to the
+existing device files.
+
+Cgroup v2 device controller has no interface files and is implemented
+on top of cgroup BPF. To control access to device files, a user may
+create bpf programs of the BPF_CGROUP_DEVICE type and attach them
+to cgroups. On an attempt to access a device file, corresponding
+BPF programs will be executed, and depending on the return value
+the attempt will succeed or fail with -EPERM.
+
+A BPF_CGROUP_DEVICE program takes a pointer to the bpf_cgroup_dev_ctx
+structure, which describes the device access attempt: access type
+(mknod/read/write) and device (type, major and minor numbers).
+If the program returns 0, the attempt fails with -EPERM, otherwise
+it succeeds.
+
+An example of BPF_CGROUP_DEVICE program may be found in the kernel
+source tree in the tools/testing/selftests/bpf/dev_cgroup.c file.
+
+
 RDMA
 ----
 
@@ -1481,6 +1511,35 @@ always be filtered by cgroup v2 path.  The controller can still be
 moved to a legacy hierarchy after v2 hierarchy is populated.
 
 
+Non-normative information
+-------------------------
+
+This section contains information that isn't considered to be a part of
+the stable kernel API and so is subject to change.
+
+
+CPU controller root cgroup process behaviour
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When distributing CPU cycles in the root cgroup each thread in this
+cgroup is treated as if it was hosted in a separate child cgroup of the
+root cgroup. This child cgroup weight is dependent on its thread nice
+level.
+
+For details of this mapping see sched_prio_to_weight array in
+kernel/sched/core.c file (values from this array should be scaled
+appropriately so the neutral - nice 0 - value is 100 instead of 1024).
+
+
+IO controller root cgroup process behaviour
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Root cgroup processes are hosted in an implicit leaf child node.
+When distributing IO resources this implicit child node is taken into
+account as if it was a normal child cgroup of the root cgroup with a
+weight value of 200.
+
+
 Namespace
 =========
 
diff --git a/Documentation/circular-buffers.txt b/Documentation/circular-buffers.txt
index d4628174b7c5..53e51caa3347 100644
--- a/Documentation/circular-buffers.txt
+++ b/Documentation/circular-buffers.txt
@@ -220,8 +220,7 @@ before it writes the new tail pointer, which will erase the item.
 
 Note the use of READ_ONCE() and smp_load_acquire() to read the
 opposition index.  This prevents the compiler from discarding and
-reloading its cached value - which some compilers will do across
-smp_read_barrier_depends().  This isn't strictly needed if you can
+reloading its cached value.  This isn't strictly needed if you can
 be sure that the opposition index will _only_ be used the once.
 The smp_load_acquire() additionally forces the CPU to order against
 subsequent memory references.  Similarly, smp_store_release() is used
diff --git a/Documentation/conf.py b/Documentation/conf.py
index 63857d33778c..62ac5a9f3a9f 100644
--- a/Documentation/conf.py
+++ b/Documentation/conf.py
@@ -88,7 +88,6 @@ finally:
     if makefile_version and makefile_patchlevel:
         version = release = makefile_version + '.' + makefile_patchlevel
     else:
-        sys.stderr.write('Warning: Could not extract kernel version\n')
         version = release = "unknown version"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
diff --git a/Documentation/errseq.rst b/Documentation/core-api/errseq.rst
similarity index 92%
rename from Documentation/errseq.rst
rename to Documentation/core-api/errseq.rst
index 4c29bd5afbc5..ff332e272405 100644
--- a/Documentation/errseq.rst
+++ b/Documentation/core-api/errseq.rst
@@ -1,5 +1,7 @@
+=====================
 The errseq_t datatype
 =====================
+
 An errseq_t is a way of recording errors in one place, and allowing any
 number of "subscribers" to tell whether it has changed since a previous
 point where it was sampled.
@@ -21,12 +23,13 @@ a flag to tell whether the value has been sampled since a new value was
 recorded.  That allows us to avoid bumping the counter if no one has
 sampled it since the last time an error was recorded.
 
-Thus we end up with a value that looks something like this::
+Thus we end up with a value that looks something like this:
 
-    bit:  31..13        12        11..0
-    +-----------------+----+----------------+
-    |     counter     | SF |      errno     |
-    +-----------------+----+----------------+
++--------------------------------------+----+------------------------+
+| 31..13                               | 12 | 11..0                  |
++--------------------------------------+----+------------------------+
+| counter                              | SF | errno                  |
++--------------------------------------+----+------------------------+
 
 The general idea is for "watchers" to sample an errseq_t value and keep
 it as a running cursor.  That value can later be used to tell whether
@@ -42,6 +45,7 @@ has ever been an error set since it was first initialized.
 
 API usage
 =========
+
 Let me tell you a story about a worker drone.  Now, he's a good worker
 overall, but the company is a little...management heavy.  He has to
 report to 77 supervisors today, and tomorrow the "big boss" is coming in
@@ -125,6 +129,7 @@ not usable by anyone else.
 
 Serializing errseq_t cursor updates
 ===================================
+
 Note that the errseq_t API does not protect the errseq_t cursor during a
 check_and_advance_operation. Only the canonical error code is handled
 atomically.  In a situation where more than one task might be using the
@@ -147,3 +152,8 @@ errseq_check_and_advance after taking the lock. e.g.::
 
 That avoids the spinlock in the common case where nothing has changed
 since the last time it was checked.
+
+Functions
+=========
+
+.. kernel-doc:: lib/errseq.c
diff --git a/Documentation/core-api/idr.rst b/Documentation/core-api/idr.rst
new file mode 100644
index 000000000000..9078a5c3ac95
--- /dev/null
+++ b/Documentation/core-api/idr.rst
@@ -0,0 +1,79 @@
+.. SPDX-License-Identifier: CC-BY-SA-4.0
+
+=============
+ID Allocation
+=============
+
+:Author: Matthew Wilcox
+
+Overview
+========
+
+A common problem to solve is allocating identifiers (IDs); generally
+small numbers which identify a thing.  Examples include file descriptors,
+process IDs, packet identifiers in networking protocols, SCSI tags
+and device instance numbers.  The IDR and the IDA provide a reasonable
+solution to the problem to avoid everybody inventing their own.  The IDR
+provides the ability to map an ID to a pointer, while the IDA provides
+only ID allocation, and as a result is much more memory-efficient.
+
+IDR usage
+=========
+
+Start by initialising an IDR, either with :c:func:`DEFINE_IDR`
+for statically allocated IDRs or :c:func:`idr_init` for dynamically
+allocated IDRs.
+
+You can call :c:func:`idr_alloc` to allocate an unused ID.  Look up
+the pointer you associated with the ID by calling :c:func:`idr_find`
+and free the ID by calling :c:func:`idr_remove`.
+
+If you need to change the pointer associated with an ID, you can call
+:c:func:`idr_replace`.  One common reason to do this is to reserve an
+ID by passing a ``NULL`` pointer to the allocation function; initialise the
+object with the reserved ID and finally insert the initialised object
+into the IDR.
+
+Some users need to allocate IDs larger than ``INT_MAX``.  So far all of
+these users have been content with a ``UINT_MAX`` limit, and they use
+:c:func:`idr_alloc_u32`.  If you need IDs that will not fit in a u32,
+we will work with you to address your needs.
+
+If you need to allocate IDs sequentially, you can use
+:c:func:`idr_alloc_cyclic`.  The IDR becomes less efficient when dealing
+with larger IDs, so using this function comes at a slight cost.
+
+To perform an action on all pointers used by the IDR, you can
+either use the callback-based :c:func:`idr_for_each` or the
+iterator-style :c:func:`idr_for_each_entry`.  You may need to use
+:c:func:`idr_for_each_entry_continue` to continue an iteration.  You can
+also use :c:func:`idr_get_next` if the iterator doesn't fit your needs.
+
+When you have finished using an IDR, you can call :c:func:`idr_destroy`
+to release the memory used by the IDR.  This will not free the objects
+pointed to from the IDR; if you want to do that, use one of the iterators
+to do it.
+
+You can use :c:func:`idr_is_empty` to find out whether there are any
+IDs currently allocated.
+
+If you need to take a lock while allocating a new ID from the IDR,
+you may need to pass a restrictive set of GFP flags, which can lead
+to the IDR being unable to allocate memory.  To work around this,
+you can call :c:func:`idr_preload` before taking the lock, and then
+:c:func:`idr_preload_end` after the allocation.
+
+.. kernel-doc:: include/linux/idr.h
+   :doc: idr sync
+
+IDA usage
+=========
+
+.. kernel-doc:: lib/idr.c
+   :doc: IDA description
+
+Functions and structures
+========================
+
+.. kernel-doc:: include/linux/idr.h
+.. kernel-doc:: lib/idr.c
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index d5bbe035316d..c670a8031786 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -14,13 +14,17 @@ Core utilities
    kernel-api
    assoc_array
    atomic_ops
+   refcount-vs-atomic
    cpu_hotplug
+   idr
    local_ops
    workqueue
    genericirq
    flexible-arrays
    librs
    genalloc
+   errseq
+   printk-formats
 
 Interfaces for kernel debugging
 ===============================
diff --git a/Documentation/core-api/kernel-api.rst b/Documentation/core-api/kernel-api.rst
index 2d9da6c40a4d..ff335f8aeb39 100644
--- a/Documentation/core-api/kernel-api.rst
+++ b/Documentation/core-api/kernel-api.rst
@@ -103,18 +103,6 @@ CRC Functions
 .. kernel-doc:: lib/crc-itu-t.c
    :export:
 
-idr/ida Functions
------------------
-
-.. kernel-doc:: include/linux/idr.h
-   :doc: idr sync
-
-.. kernel-doc:: lib/idr.c
-   :doc: IDA description
-
-.. kernel-doc:: lib/idr.c
-   :export:
-
 Math Functions in Linux
 =======================
 
@@ -139,6 +127,21 @@ Division Functions
 .. kernel-doc:: lib/gcd.c
    :export:
 
+Sorting
+-------
+
+.. kernel-doc:: lib/sort.c
+   :export:
+
+.. kernel-doc:: lib/list_sort.c
+   :export:
+
+UUID/GUID
+---------
+
+.. kernel-doc:: lib/uuid.c
+   :export:
+
 Memory Management in Linux
 ==========================
 
diff --git a/Documentation/printk-formats.txt b/Documentation/core-api/printk-formats.rst
similarity index 58%
rename from Documentation/printk-formats.txt
rename to Documentation/core-api/printk-formats.rst
index aa0a776c817a..934559b3c130 100644
--- a/Documentation/printk-formats.txt
+++ b/Documentation/core-api/printk-formats.rst
@@ -5,6 +5,7 @@ How to get printk format specifiers right
 :Author: Randy Dunlap <rdunlap@infradead.org>
 :Author: Andrew Murray <amurray@mpc-data.co.uk>
 
+
 Integer types
 =============
 
@@ -25,105 +26,101 @@ Integer types
 		s64			%lld or %llx
 		u64			%llu or %llx
 
-If <type> is dependent on a config option for its size (e.g., ``sector_t``,
-``blkcnt_t``) or is architecture-dependent for its size (e.g., ``tcflag_t``),
-use a format specifier of its largest possible type and explicitly cast to it.
+
+If <type> is dependent on a config option for its size (e.g., sector_t,
+blkcnt_t) or is architecture-dependent for its size (e.g., tcflag_t), use a
+format specifier of its largest possible type and explicitly cast to it.
 
 Example::
 
 	printk("test: sector number/total blocks: %llu/%llu\n",
 		(unsigned long long)sector, (unsigned long long)blockcount);
 
-Reminder: ``sizeof()`` result is of type ``size_t``.
+Reminder: sizeof() returns type size_t.
 
-The kernel's printf does not support ``%n``. For obvious reasons, floating
-point formats (``%e, %f, %g, %a``) are also not recognized. Use of any
+The kernel's printf does not support %n. Floating point formats (%e, %f,
+%g, %a) are also not recognized, for obvious reasons. Use of any
 unsupported specifier or length qualifier results in a WARN and early
-return from vsnprintf.
+return from vsnprintf().
 
-Raw pointer value SHOULD be printed with %p. The kernel supports
-the following extended format specifiers for pointer types:
-
-Pointer Types
+Pointer types
 =============
 
-Pointers printed without a specifier extension (i.e unadorned %p) are
-hashed to give a unique identifier without leaking kernel addresses to user
-space. On 64 bit machines the first 32 bits are zeroed. If you _really_
-want the address see %px below.
+A raw pointer value may be printed with %p which will hash the address
+before printing. The kernel also supports extended specifiers for printing
+pointers of different types.
+
+Plain Pointers
+--------------
 
 ::
 
 	%p	abcdef12 or 00000000abcdef12
 
+Pointers printed without a specifier extension (i.e unadorned %p) are
+hashed to prevent leaking information about the kernel memory layout. This
+has the added benefit of providing a unique identifier. On 64-bit machines
+the first 32 bits are zeroed. If you *really* want the address see %px
+below.
+
 Symbols/Function Pointers
-=========================
+-------------------------
 
 ::
 
+	%pS	versatile_init+0x0/0x110
+	%ps	versatile_init
 	%pF	versatile_init+0x0/0x110
 	%pf	versatile_init
-	%pS	versatile_init+0x0/0x110
 	%pSR	versatile_init+0x9/0x110
 		(with __builtin_extract_return_addr() translation)
-	%ps	versatile_init
 	%pB	prev_fn_of_versatile_init+0x88/0x88
 
-The ``F`` and ``f`` specifiers are for printing function pointers,
-for example, f->func, &gettimeofday. They have the same result as
-``S`` and ``s`` specifiers. But they do an extra conversion on
-ia64, ppc64 and parisc64 architectures where the function pointers
-are actually function descriptors.
 
-The ``S`` and ``s`` specifiers can be used for printing symbols
-from direct addresses, for example, __builtin_return_address(0),
-(void *)regs->ip. They result in the symbol name with (``S``) or
-without (``s``) offsets. If KALLSYMS are disabled then the symbol
-address is printed instead.
+The ``S`` and ``s`` specifiers are used for printing a pointer in symbolic
+format. They result in the symbol name with (S) or without (s)
+offsets. If KALLSYMS are disabled then the symbol address is printed instead.
+
+Note, that the ``F`` and ``f`` specifiers are identical to ``S`` (``s``)
+and thus deprecated. We have ``F`` and ``f`` because on ia64, ppc64 and
+parisc64 function pointers are indirect and, in fact, are function
+descriptors, which require additional dereferencing before we can lookup
+the symbol. As of now, ``S`` and ``s`` perform dereferencing on those
+platforms (when needed), so ``F`` and ``f`` exist for compatibility
+reasons only.
 
 The ``B`` specifier results in the symbol name with offsets and should be
 used when printing stack backtraces. The specifier takes into
 consideration the effect of compiler optimisations which may occur
-when tail-call``s are used and marked with the noreturn GCC attribute.
-
-Examples::
-
-	printk("Going to call: %pF\n", gettimeofday);
-	printk("Going to call: %pF\n", p->func);
-	printk("%s: called from %pS\n", __func__, (void *)_RET_IP_);
-	printk("%s: called from %pS\n", __func__,
-				(void *)__builtin_return_address(0));
-	printk("Faulted at %pS\n", (void *)regs->ip);
-	printk(" %s%pB\n", (reliable ? "" : "? "), (void *)*stack);
+when tail-calls are used and marked with the noreturn GCC attribute.
 
 Kernel Pointers
-===============
+---------------
 
 ::
 
 	%pK	01234567 or 0123456789abcdef
 
 For printing kernel pointers which should be hidden from unprivileged
-users. The behaviour of ``%pK`` depends on the ``kptr_restrict sysctl`` - see
+users. The behaviour of %pK depends on the kptr_restrict sysctl - see
 Documentation/sysctl/kernel.txt for more details.
 
 Unmodified Addresses
-====================
+--------------------
 
 ::
 
 	%px	01234567 or 0123456789abcdef
 
-For printing pointers when you _really_ want to print the address. Please
+For printing pointers when you *really* want to print the address. Please
 consider whether or not you are leaking sensitive information about the
-Kernel layout in memory before printing pointers with %px. %px is
-functionally equivalent to %lx. %px is preferred to %lx because it is more
-uniquely grep'able. If, in the future, we need to modify the way the Kernel
-handles printing pointers it will be nice to be able to find the call
-sites.
+kernel memory layout before printing pointers with %px. %px is functionally
+equivalent to %lx (or %lu). %px is preferred because it is more uniquely
+grep'able. If in the future we need to modify the way the kernel handles
+printing pointers we will be better equipped to find the call sites.
 
 Struct Resources
-================
+----------------
 
 ::
 
@@ -133,32 +130,37 @@ Struct Resources
 		[mem 0x0000000060000000-0x000000006fffffff pref]
 
 For printing struct resources. The ``R`` and ``r`` specifiers result in a
-printed resource with (``R``) or without (``r``) a decoded flags member.
+printed resource with (R) or without (r) a decoded flags member.
+
 Passed by reference.
 
-Physical addresses types ``phys_addr_t``
-========================================
+Physical address types phys_addr_t
+----------------------------------
 
 ::
 
 	%pa[p]	0x01234567 or 0x0123456789abcdef
 
-For printing a ``phys_addr_t`` type (and its derivatives, such as
-``resource_size_t``) which can vary based on build options, regardless of
-the width of the CPU data path. Passed by reference.
+For printing a phys_addr_t type (and its derivatives, such as
+resource_size_t) which can vary based on build options, regardless of the
+width of the CPU data path.
+
+Passed by reference.
 
-DMA addresses types ``dma_addr_t``
-==================================
+DMA address types dma_addr_t
+----------------------------
 
 ::
 
 	%pad	0x01234567 or 0x0123456789abcdef
 
-For printing a ``dma_addr_t`` type which can vary based on build options,
-regardless of the width of the CPU data path. Passed by reference.
+For printing a dma_addr_t type which can vary based on build options,
+regardless of the width of the CPU data path.
+
+Passed by reference.
 
 Raw buffer as an escaped string
-===============================
+-------------------------------
 
 ::
 
@@ -168,8 +170,8 @@ For printing raw buffer as an escaped string. For the following buffer::
 
 		1b 62 20 5c 43 07 22 90 0d 5d
 
-few examples show how the conversion would be done (the result string
-without surrounding quotes)::
+A few examples show how the conversion would be done (excluding surrounding
+quotes)::
 
 		%*pE		"\eb \C\a"\220\r]"
 		%*pEhp		"\x1bb \C\x07"\x90\x0d]"
@@ -179,23 +181,23 @@ The conversion rules are applied according to an optional combination
 of flags (see :c:func:`string_escape_mem` kernel documentation for the
 details):
 
-	- ``a`` - ESCAPE_ANY
-	- ``c`` - ESCAPE_SPECIAL
-	- ``h`` - ESCAPE_HEX
-	- ``n`` - ESCAPE_NULL
-	- ``o`` - ESCAPE_OCTAL
-	- ``p`` - ESCAPE_NP
-	- ``s`` - ESCAPE_SPACE
+	- a - ESCAPE_ANY
+	- c - ESCAPE_SPECIAL
+	- h - ESCAPE_HEX
+	- n - ESCAPE_NULL
+	- o - ESCAPE_OCTAL
+	- p - ESCAPE_NP
+	- s - ESCAPE_SPACE
 
 By default ESCAPE_ANY_NP is used.
 
 ESCAPE_ANY_NP is the sane choice for many cases, in particularly for
 printing SSIDs.
 
-If field width is omitted the 1 byte only will be escaped.
+If field width is omitted then 1 byte only will be escaped.
 
 Raw buffer as a hex string
-==========================
+--------------------------
 
 ::
 
@@ -204,12 +206,12 @@ Raw buffer as a hex string
 	%*phD	00-01-02- ... -3f
 	%*phN	000102 ... 3f
 
-For printing a small buffers (up to 64 bytes long) as a hex string with
-certain separator. For the larger buffers consider to use
+For printing small buffers (up to 64 bytes long) as a hex string with a
+certain separator. For larger buffers consider using
 :c:func:`print_hex_dump`.
 
 MAC/FDDI addresses
-==================
+------------------
 
 ::
 
@@ -220,11 +222,11 @@ MAC/FDDI addresses
 	%pmR	050403020100
 
 For printing 6-byte MAC/FDDI addresses in hex notation. The ``M`` and ``m``
-specifiers result in a printed address with (``M``) or without (``m``) byte
-separators. The default byte separator is the colon (``:``).
+specifiers result in a printed address with (M) or without (m) byte
+separators. The default byte separator is the colon (:).
 
 Where FDDI addresses are concerned the ``F`` specifier can be used after
-the ``M`` specifier to use dash (``-``) separators instead of the default
+the ``M`` specifier to use dash (-) separators instead of the default
 separator.
 
 For Bluetooth addresses the ``R`` specifier shall be used after the ``M``
@@ -234,7 +236,7 @@ of Bluetooth addresses which are in the little endian order.
 Passed by reference.
 
 IPv4 addresses
-==============
+--------------
 
 ::
 
@@ -243,8 +245,8 @@ IPv4 addresses
 	%p[Ii]4[hnbl]
 
 For printing IPv4 dot-separated decimal addresses. The ``I4`` and ``i4``
-specifiers result in a printed address with (``i4``) or without (``I4``)
-leading zeros.
+specifiers result in a printed address with (i4) or without (I4) leading
+zeros.
 
 The additional ``h``, ``n``, ``b``, and ``l`` specifiers are used to specify
 host, network, big or little endian order addresses respectively. Where
@@ -253,7 +255,7 @@ no specifier is provided the default network/big endian order is used.
 Passed by reference.
 
 IPv6 addresses
-==============
+--------------
 
 ::
 
@@ -262,7 +264,7 @@ IPv6 addresses
 	%pI6c	1:2:3:4:5:6:7:8
 
 For printing IPv6 network-order 16-bit hex addresses. The ``I6`` and ``i6``
-specifiers result in a printed address with (``I6``) or without (``i6``)
+specifiers result in a printed address with (I6) or without (i6)
 colon-separators. Leading zeros are always used.
 
 The additional ``c`` specifier can be used with the ``I`` specifier to
@@ -272,7 +274,7 @@ http://tools.ietf.org/html/rfc5952
 Passed by reference.
 
 IPv4/IPv6 addresses (generic, with port, flowinfo, scope)
-=========================================================
+---------------------------------------------------------
 
 ::
 
@@ -282,8 +284,8 @@ IPv4/IPv6 addresses (generic, with port, flowinfo, scope)
 	%pISpc	1.2.3.4:12345	or [1:2:3:4:5:6:7:8]:12345
 	%p[Ii]S[pfschnbl]
 
-For printing an IP address without the need to distinguish whether it``s
-of type AF_INET or AF_INET6, a pointer to a valid ``struct sockaddr``,
+For printing an IP address without the need to distinguish whether it's of
+type AF_INET or AF_INET6. A pointer to a valid struct sockaddr,
 specified through ``IS`` or ``iS``, can be passed to this format specifier.
 
 The additional ``p``, ``f``, and ``s`` specifiers are used to specify port
@@ -309,7 +311,7 @@ Further examples::
 	%pISpfc		1.2.3.4:12345	or [1:2:3:4:5:6:7:8]:12345/123456789
 
 UUID/GUID addresses
-===================
+-------------------
 
 ::
 
@@ -318,33 +320,33 @@ UUID/GUID addresses
 	%pUl	03020100-0504-0706-0809-0a0b0c0e0e0f
 	%pUL	03020100-0504-0706-0809-0A0B0C0E0E0F
 
-For printing 16-byte UUID/GUIDs addresses. The additional 'l', 'L',
-'b' and 'B' specifiers are used to specify a little endian order in
-lower ('l') or upper case ('L') hex characters - and big endian order
-in lower ('b') or upper case ('B') hex characters.
+For printing 16-byte UUID/GUIDs addresses. The additional ``l``, ``L``,
+``b`` and ``B`` specifiers are used to specify a little endian order in
+lower (l) or upper case (L) hex notation - and big endian order in lower (b)
+or upper case (B) hex notation.
 
 Where no additional specifiers are used the default big endian
-order with lower case hex characters will be printed.
+order with lower case hex notation will be printed.
 
 Passed by reference.
 
 dentry names
-============
+------------
 
 ::
 
 	%pd{,2,3,4}
 	%pD{,2,3,4}
 
-For printing dentry name; if we race with :c:func:`d_move`, the name might be
-a mix of old and new ones, but it won't oops.  ``%pd`` dentry is a safer
-equivalent of ``%s`` ``dentry->d_name.name`` we used to use, ``%pd<n>`` prints
-``n`` last components.  ``%pD`` does the same thing for struct file.
+For printing dentry name; if we race with :c:func:`d_move`, the name might
+be a mix of old and new ones, but it won't oops.  %pd dentry is a safer
+equivalent of %s dentry->d_name.name we used to use, %pd<n> prints ``n``
+last components.  %pD does the same thing for struct file.
 
 Passed by reference.
 
 block_device names
-==================
+------------------
 
 ::
 
@@ -353,7 +355,7 @@ block_device names
 For printing name of block_device pointers.
 
 struct va_format
-================
+----------------
 
 ::
 
@@ -375,31 +377,27 @@ correctness of the format string and va_list arguments.
 Passed by reference.
 
 kobjects
-========
+--------
 
 ::
 
-	%pO
+	%pOF[fnpPcCF]
 
-	Base specifier for kobject based structs. Must be followed with
-	character for specific type of kobject as listed below:
 
-	Device tree nodes:
+For printing kobject based structs (device nodes). Default behaviour is
+equivalent to %pOFf.
 
-	%pOF[fnpPcCF]
+	- f - device node full_name
+	- n - device node name
+	- p - device node phandle
+	- P - device node path spec (name + @unit)
+	- F - device node flags
+	- c - major compatible string
+	- C - full compatible string
 
-	For printing device tree nodes. The optional arguments are:
-	    f device node full_name
-	    n device node name
-	    p device node phandle
-	    P device node path spec (name + @unit)
-	    F device node flags
-	    c major compatible string
-	    C full compatible string
-	Without any arguments prints full_name (same as %pOFf)
-	The separator when using multiple arguments is ':'
+The separator when using multiple arguments is ':'
 
-	Examples:
+Examples::
 
 	%pOF	/foo/bar@0			- Node full name
 	%pOFf	/foo/bar@0			- Same as above
@@ -412,11 +410,10 @@ kobjects
 							P - Populated
 							B - Populated bus
 
-	Passed by reference.
-
+Passed by reference.
 
 struct clk
-==========
+----------
 
 ::
 
@@ -424,14 +421,14 @@ struct clk
 	%pCn	pll1
 	%pCr	1560000000
 
-For printing struct clk structures. ``%pC`` and ``%pCn`` print the name
+For printing struct clk structures. %pC and %pCn print the name
 (Common Clock Framework) or address (legacy clock framework) of the
-structure; ``%pCr`` prints the current clock rate.
+structure; %pCr prints the current clock rate.
 
 Passed by reference.
 
 bitmap and its derivatives such as cpumask and nodemask
-=======================================================
+-------------------------------------------------------
 
 ::
 
@@ -439,13 +436,13 @@ bitmap and its derivatives such as cpumask and nodemask
 	%*pbl	0,3-6,8-10
 
 For printing bitmap and its derivatives such as cpumask and nodemask,
-``%*pb`` output the bitmap with field width as the number of bits and ``%*pbl``
+%*pb outputs the bitmap with field width as the number of bits and %*pbl
 output the bitmap as range list with field width as the number of bits.
 
 Passed by reference.
 
 Flags bitfields such as page flags, gfp_flags
-=============================================
+---------------------------------------------
 
 ::
 
@@ -459,14 +456,14 @@ character. Currently supported are [p]age flags, [v]ma_flags (both
 expect ``unsigned long *``) and [g]fp_flags (expects ``gfp_t *``). The flag
 names and print order depends on the particular	type.
 
-Note that this format should not be used directly in :c:func:`TP_printk()` part
-of a tracepoint. Instead, use the ``show_*_flags()`` functions from
-<trace/events/mmflags.h>.
+Note that this format should not be used directly in the
+:c:func:`TP_printk()` part of a tracepoint. Instead, use the show_*_flags()
+functions from <trace/events/mmflags.h>.
 
 Passed by reference.
 
 Network device features
-=======================
+-----------------------
 
 ::
 
@@ -476,8 +473,10 @@ For printing netdev_features_t.
 
 Passed by reference.
 
-If you add other ``%p`` extensions, please extend lib/test_printf.c with
-one or more test cases, if at all feasible.
+Thanks
+======
 
+If you add other %p extensions, please extend <lib/test_printf.c> with
+one or more test cases, if at all feasible.
 
 Thank you for your cooperation and attention.
diff --git a/Documentation/core-api/refcount-vs-atomic.rst b/Documentation/core-api/refcount-vs-atomic.rst
new file mode 100644
index 000000000000..83351c258cdb
--- /dev/null
+++ b/Documentation/core-api/refcount-vs-atomic.rst
@@ -0,0 +1,150 @@
+===================================
+refcount_t API compared to atomic_t
+===================================
+
+.. contents:: :local:
+
+Introduction
+============
+
+The goal of refcount_t API is to provide a minimal API for implementing
+an object's reference counters. While a generic architecture-independent
+implementation from lib/refcount.c uses atomic operations underneath,
+there are a number of differences between some of the ``refcount_*()`` and
+``atomic_*()`` functions with regards to the memory ordering guarantees.
+This document outlines the differences and provides respective examples
+in order to help maintainers validate their code against the change in
+these memory ordering guarantees.
+
+The terms used through this document try to follow the formal LKMM defined in
+github.com/aparri/memory-model/blob/master/Documentation/explanation.txt
+
+memory-barriers.txt and atomic_t.txt provide more background to the
+memory ordering in general and for atomic operations specifically.
+
+Relevant types of memory ordering
+=================================
+
+.. note:: The following section only covers some of the memory
+   ordering types that are relevant for the atomics and reference
+   counters and used through this document. For a much broader picture
+   please consult memory-barriers.txt document.
+
+In the absence of any memory ordering guarantees (i.e. fully unordered)
+atomics & refcounters only provide atomicity and
+program order (po) relation (on the same CPU). It guarantees that
+each ``atomic_*()`` and ``refcount_*()`` operation is atomic and instructions
+are executed in program order on a single CPU.
+This is implemented using :c:func:`READ_ONCE`/:c:func:`WRITE_ONCE` and
+compare-and-swap primitives.
+
+A strong (full) memory ordering guarantees that all prior loads and
+stores (all po-earlier instructions) on the same CPU are completed
+before any po-later instruction is executed on the same CPU.
+It also guarantees that all po-earlier stores on the same CPU
+and all propagated stores from other CPUs must propagate to all
+other CPUs before any po-later instruction is executed on the original
+CPU (A-cumulative property). This is implemented using :c:func:`smp_mb`.
+
+A RELEASE memory ordering guarantees that all prior loads and
+stores (all po-earlier instructions) on the same CPU are completed
+before the operation. It also guarantees that all po-earlier
+stores on the same CPU and all propagated stores from other CPUs
+must propagate to all other CPUs before the release operation
+(A-cumulative property). This is implemented using
+:c:func:`smp_store_release`.
+
+A control dependency (on success) for refcounters guarantees that
+if a reference for an object was successfully obtained (reference
+counter increment or addition happened, function returned true),
+then further stores are ordered against this operation.
+Control dependency on stores are not implemented using any explicit
+barriers, but rely on CPU not to speculate on stores. This is only
+a single CPU relation and provides no guarantees for other CPUs.
+
+
+Comparison of functions
+=======================
+
+case 1) - non-"Read/Modify/Write" (RMW) ops
+-------------------------------------------
+
+Function changes:
+
+ * :c:func:`atomic_set` --> :c:func:`refcount_set`
+ * :c:func:`atomic_read` --> :c:func:`refcount_read`
+
+Memory ordering guarantee changes:
+
+ * none (both fully unordered)
+
+
+case 2) - increment-based ops that return no value
+--------------------------------------------------
+
+Function changes:
+
+ * :c:func:`atomic_inc` --> :c:func:`refcount_inc`
+ * :c:func:`atomic_add` --> :c:func:`refcount_add`
+
+Memory ordering guarantee changes:
+
+ * none (both fully unordered)
+
+case 3) - decrement-based RMW ops that return no value
+------------------------------------------------------
+
+Function changes:
+
+ * :c:func:`atomic_dec` --> :c:func:`refcount_dec`
+
+Memory ordering guarantee changes:
+
+ * fully unordered --> RELEASE ordering
+
+
+case 4) - increment-based RMW ops that return a value
+-----------------------------------------------------
+
+Function changes:
+
+ * :c:func:`atomic_inc_not_zero` --> :c:func:`refcount_inc_not_zero`
+ * no atomic counterpart --> :c:func:`refcount_add_not_zero`
+
+Memory ordering guarantees changes:
+
+ * fully ordered --> control dependency on success for stores
+
+.. note:: We really assume here that necessary ordering is provided as a
+   result of obtaining pointer to the object!
+
+
+case 5) - decrement-based RMW ops that return a value
+-----------------------------------------------------
+
+Function changes:
+
+ * :c:func:`atomic_dec_and_test` --> :c:func:`refcount_dec_and_test`
+ * :c:func:`atomic_sub_and_test` --> :c:func:`refcount_sub_and_test`
+ * no atomic counterpart --> :c:func:`refcount_dec_if_one`
+ * ``atomic_add_unless(&var, -1, 1)`` --> ``refcount_dec_not_one(&var)``
+
+Memory ordering guarantees changes:
+
+ * fully ordered --> RELEASE ordering + control dependency
+
+.. note:: :c:func:`atomic_add_unless` only provides full order on success.
+
+
+case 6) - lock-based RMW
+------------------------
+
+Function changes:
+
+ * :c:func:`atomic_dec_and_lock` --> :c:func:`refcount_dec_and_lock`
+ * :c:func:`atomic_dec_and_mutex_lock` --> :c:func:`refcount_dec_and_mutex_lock`
+
+Memory ordering guarantees changes:
+
+ * fully ordered --> RELEASE ordering + control dependency + hold
+   :c:func:`spin_lock` on success
diff --git a/Documentation/cpu-freq/cpu-drivers.txt b/Documentation/cpu-freq/cpu-drivers.txt
index 434c49cc7330..61546ac578d6 100644
--- a/Documentation/cpu-freq/cpu-drivers.txt
+++ b/Documentation/cpu-freq/cpu-drivers.txt
@@ -291,3 +291,7 @@ For example:
 		/* Do something with pos */
 		pos->frequency = ...
 	}
+
+If you need to work with the position of pos within driver_freq_table,
+do not subtract the pointers, as it is quite costly. Instead, use the
+macros cpufreq_for_each_entry_idx() and cpufreq_for_each_valid_entry_idx().
diff --git a/Documentation/device-mapper/cache-policies.txt b/Documentation/device-mapper/cache-policies.txt
index d3ca8af21a31..86786d87d9a8 100644
--- a/Documentation/device-mapper/cache-policies.txt
+++ b/Documentation/device-mapper/cache-policies.txt
@@ -60,7 +60,7 @@ Memory usage:
 The mq policy used a lot of memory; 88 bytes per cache block on a 64
 bit machine.
 
-smq uses 28bit indexes to implement it's data structures rather than
+smq uses 28bit indexes to implement its data structures rather than
 pointers.  It avoids storing an explicit hit count for each block.  It
 has a 'hotspot' queue, rather than a pre-cache, which uses a quarter of
 the entries (each hotspot block covers a larger area than a single
@@ -84,7 +84,7 @@ resulting in better promotion/demotion decisions.
 
 Adaptability:
 The mq policy maintained a hit count for each cache block.  For a
-different block to get promoted to the cache it's hit count has to
+different block to get promoted to the cache its hit count has to
 exceed the lowest currently in the cache.  This meant it could take a
 long time for the cache to adapt between varying IO patterns.
 
diff --git a/Documentation/device-mapper/cache.txt b/Documentation/device-mapper/cache.txt
index cdfd0feb294e..ff0841711fd5 100644
--- a/Documentation/device-mapper/cache.txt
+++ b/Documentation/device-mapper/cache.txt
@@ -59,7 +59,7 @@ Fixed block size
 The origin is divided up into blocks of a fixed size.  This block size
 is configurable when you first create the cache.  Typically we've been
 using block sizes of 256KB - 1024KB.  The block size must be between 64
-(32KB) and 2097152 (1GB) and a multiple of 64 (32KB).
+sectors (32KB) and 2097152 sectors (1GB) and a multiple of 64 sectors (32KB).
 
 Having a fixed block size simplifies the target a lot.  But it is
 something of a compromise.  For instance, a small part of a block may be
@@ -119,7 +119,7 @@ doing here to avoid migrating during those peak io moments.
 
 For the time being, a message "migration_threshold <#sectors>"
 can be used to set the maximum number of sectors being migrated,
-the default being 204800 sectors (or 100MB).
+the default being 2048 sectors (1MB).
 
 Updating on-disk metadata
 -------------------------
@@ -143,11 +143,6 @@ the policy how big this chunk is, but it should be kept small.  Like the
 dirty flags this data is lost if there's a crash so a safe fallback
 value should always be possible.
 
-For instance, the 'mq' policy, which is currently the default policy,
-uses this facility to store the hit count of the cache blocks.  If
-there's a crash this information will be lost, which means the cache
-may be less efficient until those hit counts are regenerated.
-
 Policy hints affect performance, not correctness.
 
 Policy messaging
diff --git a/Documentation/device-mapper/dm-raid.txt b/Documentation/device-mapper/dm-raid.txt
index 32df07e29f68..390c145f01d7 100644
--- a/Documentation/device-mapper/dm-raid.txt
+++ b/Documentation/device-mapper/dm-raid.txt
@@ -343,5 +343,8 @@ Version History
 1.11.0  Fix table line argument order
 	(wrong raid10_copies/raid10_format sequence)
 1.11.1  Add raid4/5/6 journal write-back support via journal_mode option
-1.12.1  fix for MD deadlock between mddev_suspend() and md_write_start() available
+1.12.1  Fix for MD deadlock between mddev_suspend() and md_write_start() available
 1.13.0  Fix dev_health status at end of "recover" (was 'a', now 'A')
+1.13.1  Fix deadlock caused by early md_stop_writes().  Also fix size an
+	state races.
+1.13.2  Fix raid redundancy validation and avoid keeping raid set frozen
diff --git a/Documentation/device-mapper/snapshot.txt b/Documentation/device-mapper/snapshot.txt
index ad6949bff2e3..b8bbb516f989 100644
--- a/Documentation/device-mapper/snapshot.txt
+++ b/Documentation/device-mapper/snapshot.txt
@@ -49,6 +49,10 @@ The difference between persistent and transient is with transient
 snapshots less metadata must be saved on disk - they can be kept in
 memory by the kernel.
 
+When loading or unloading the snapshot target, the corresponding
+snapshot-origin or snapshot-merge target must be suspended. A failure to
+suspend the origin target could result in data corruption.
+
 
 * snapshot-merge <origin> <COW device> <persistent> <chunksize>
 
diff --git a/Documentation/device-mapper/thin-provisioning.txt b/Documentation/device-mapper/thin-provisioning.txt
index 1699a55b7b70..4bcd4b7f79f9 100644
--- a/Documentation/device-mapper/thin-provisioning.txt
+++ b/Documentation/device-mapper/thin-provisioning.txt
@@ -112,9 +112,11 @@ $low_water_mark is expressed in blocks of size $data_block_size.  If
 free space on the data device drops below this level then a dm event
 will be triggered which a userspace daemon should catch allowing it to
 extend the pool device.  Only one such event will be sent.
-Resuming a device with a new table itself triggers an event so the
-userspace daemon can use this to detect a situation where a new table
-already exceeds the threshold.
+
+No special event is triggered if a just resumed device's free space is below
+the low water mark. However, resuming a device always triggers an
+event; a userspace daemon should verify that free space exceeds the low
+water mark when handling this event.
 
 A low water mark for the metadata device is maintained in the kernel and
 will trigger a dm event if free space on the metadata device drops below
@@ -274,7 +276,8 @@ ii) Status
 
     <transaction id> <used metadata blocks>/<total metadata blocks>
     <used data blocks>/<total data blocks> <held metadata root>
-    [no_]discard_passdown ro|rw
+    ro|rw|out_of_data_space [no_]discard_passdown [error|queue]_if_no_space
+    needs_check|-
 
     transaction id:
 	A 64-bit number used by userspace to help synchronise with metadata
@@ -394,3 +397,6 @@ ii) Status
 	If the pool has encountered device errors and failed, the status
 	will just contain the string 'Fail'.  The userspace recovery
 	tools should then be used.
+
+    In the case where <nr mapped sectors> is 0, there is no highest
+    mapped sector and the value of <highest mapped sector> is unspecified.
diff --git a/Documentation/device-mapper/unstriped.txt b/Documentation/device-mapper/unstriped.txt
new file mode 100644
index 000000000000..0b2a306c54ee
--- /dev/null
+++ b/Documentation/device-mapper/unstriped.txt
@@ -0,0 +1,124 @@
+Introduction
+============
+
+The device-mapper "unstriped" target provides a transparent mechanism to
+unstripe a device-mapper "striped" target to access the underlying disks
+without having to touch the true backing block-device.  It can also be
+used to unstripe a hardware RAID-0 to access backing disks.
+
+Parameters:
+<number of stripes> <chunk size> <stripe #> <dev_path> <offset>
+
+<number of stripes>
+        The number of stripes in the RAID 0.
+
+<chunk size>
+	The amount of 512B sectors in the chunk striping.
+
+<dev_path>
+	The block device you wish to unstripe.
+
+<stripe #>
+        The stripe number within the device that corresponds to physical
+        drive you wish to unstripe.  This must be 0 indexed.
+
+
+Why use this module?
+====================
+
+An example of undoing an existing dm-stripe
+-------------------------------------------
+
+This small bash script will setup 4 loop devices and use the existing
+striped target to combine the 4 devices into one.  It then will use
+the unstriped target ontop of the striped device to access the
+individual backing loop devices.  We write data to the newly exposed
+unstriped devices and verify the data written matches the correct
+underlying device on the striped array.
+
+#!/bin/bash
+
+MEMBER_SIZE=$((128 * 1024 * 1024))
+NUM=4
+SEQ_END=$((${NUM}-1))
+CHUNK=256
+BS=4096
+
+RAID_SIZE=$((${MEMBER_SIZE}*${NUM}/512))
+DM_PARMS="0 ${RAID_SIZE} striped ${NUM} ${CHUNK}"
+COUNT=$((${MEMBER_SIZE} / ${BS}))
+
+for i in $(seq 0 ${SEQ_END}); do
+  dd if=/dev/zero of=member-${i} bs=${MEMBER_SIZE} count=1 oflag=direct
+  losetup /dev/loop${i} member-${i}
+  DM_PARMS+=" /dev/loop${i} 0"
+done
+
+echo $DM_PARMS | dmsetup create raid0
+for i in $(seq 0 ${SEQ_END}); do
+  echo "0 1 unstriped ${NUM} ${CHUNK} ${i} /dev/mapper/raid0 0" | dmsetup create set-${i}
+done;
+
+for i in $(seq 0 ${SEQ_END}); do
+  dd if=/dev/urandom of=/dev/mapper/set-${i} bs=${BS} count=${COUNT} oflag=direct
+  diff /dev/mapper/set-${i} member-${i}
+done;
+
+for i in $(seq 0 ${SEQ_END}); do
+  dmsetup remove set-${i}
+done
+
+dmsetup remove raid0
+
+for i in $(seq 0 ${SEQ_END}); do
+  losetup -d /dev/loop${i}
+  rm -f member-${i}
+done
+
+Another example
+---------------
+
+Intel NVMe drives contain two cores on the physical device.
+Each core of the drive has segregated access to its LBA range.
+The current LBA model has a RAID 0 128k chunk on each core, resulting
+in a 256k stripe across the two cores:
+
+   Core 0:       Core 1:
+  __________    __________
+  | LBA 512|    | LBA 768|
+  | LBA 0  |    | LBA 256|
+  ----------    ----------
+
+The purpose of this unstriping is to provide better QoS in noisy
+neighbor environments. When two partitions are created on the
+aggregate drive without this unstriping, reads on one partition
+can affect writes on another partition.  This is because the partitions
+are striped across the two cores.  When we unstripe this hardware RAID 0
+and make partitions on each new exposed device the two partitions are now
+physically separated.
+
+With the dm-unstriped target we're able to segregate an fio script that
+has read and write jobs that are independent of each other.  Compared to
+when we run the test on a combined drive with partitions, we were able
+to get a 92% reduction in read latency using this device mapper target.
+
+
+Example dmsetup usage
+=====================
+
+unstriped ontop of Intel NVMe device that has 2 cores
+-----------------------------------------------------
+dmsetup create nvmset0 --table '0 512 unstriped 2 256 0 /dev/nvme0n1 0'
+dmsetup create nvmset1 --table '0 512 unstriped 2 256 1 /dev/nvme0n1 0'
+
+There will now be two devices that expose Intel NVMe core 0 and 1
+respectively:
+/dev/mapper/nvmset0
+/dev/mapper/nvmset1
+
+unstriped ontop of striped with 4 drives using 128K chunk size
+--------------------------------------------------------------
+dmsetup create raid_disk0 --table '0 512 unstriped 4 256 0 /dev/mapper/striped 0'
+dmsetup create raid_disk1 --table '0 512 unstriped 4 256 1 /dev/mapper/striped 0'
+dmsetup create raid_disk2 --table '0 512 unstriped 4 256 2 /dev/mapper/striped 0'
+dmsetup create raid_disk3 --table '0 512 unstriped 4 256 3 /dev/mapper/striped 0'
diff --git a/Documentation/devicetree/bindings/arm/actions.txt b/Documentation/devicetree/bindings/arm/actions.txt
index ced764a8549e..d54f33c4e0da 100644
--- a/Documentation/devicetree/bindings/arm/actions.txt
+++ b/Documentation/devicetree/bindings/arm/actions.txt
@@ -21,10 +21,26 @@ Boards:
 
 Root node property compatible must contain, depending on board:
 
+ - Allo.com Sparky: "allo,sparky"
  - Cubietech CubieBoard6: "cubietech,cubieboard6"
  - LeMaker Guitar Base Board rev. B: "lemaker,guitar-bb-rev-b", "lemaker,guitar"
 
 
+S700 SoC
+========
+
+Required root node properties:
+
+- compatible :  must contain "actions,s700"
+
+
+Boards:
+
+Root node property compatible must contain, depending on board:
+
+ - Cubietech CubieBoard7: "cubietech,cubieboard7"
+
+
 S900 SoC
 ========
 
diff --git a/Documentation/devicetree/bindings/arm/arm-dsu-pmu.txt b/Documentation/devicetree/bindings/arm/arm-dsu-pmu.txt
new file mode 100644
index 000000000000..6efabba530f1
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/arm-dsu-pmu.txt
@@ -0,0 +1,27 @@
+* ARM DynamIQ Shared Unit (DSU) Performance Monitor Unit (PMU)
+
+ARM DyanmIQ Shared Unit (DSU) integrates one or more CPU cores
+with a shared L3 memory system, control logic and external interfaces to
+form a multicore cluster. The PMU enables to gather various statistics on
+the operations of the DSU. The PMU provides independent 32bit counters that
+can count any of the supported events, along with a 64bit cycle counter.
+The PMU is accessed via CPU system registers and has no MMIO component.
+
+** DSU PMU required properties:
+
+- compatible	: should be one of :
+
+		"arm,dsu-pmu"
+
+- interrupts	: Exactly 1 SPI must be listed.
+
+- cpus		: List of phandles for the CPUs connected to this DSU instance.
+
+
+** Example:
+
+dsu-pmu-0 {
+	compatible = "arm,dsu-pmu";
+	interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>;
+	cpus = <&cpu_0>, <&cpu_1>;
+};
diff --git a/Documentation/devicetree/bindings/arm/atmel-at91.txt b/Documentation/devicetree/bindings/arm/atmel-at91.txt
index 91cb8e4f2a4f..31220b54d85d 100644
--- a/Documentation/devicetree/bindings/arm/atmel-at91.txt
+++ b/Documentation/devicetree/bindings/arm/atmel-at91.txt
@@ -90,38 +90,6 @@ System Timer (ST) required properties:
 Its subnodes can be:
 - watchdog: compatible should be "atmel,at91rm9200-wdt"
 
-TC/TCLIB Timer required properties:
-- compatible: Should be "atmel,<chip>-tcb".
-  <chip> can be "at91rm9200" or "at91sam9x5"
-- reg: Should contain registers location and length
-- interrupts: Should contain all interrupts for the TC block
-  Note that you can specify several interrupt cells if the TC
-  block has one interrupt per channel.
-- clock-names: tuple listing input clock names.
-	Required elements: "t0_clk", "slow_clk"
-	Optional elements: "t1_clk", "t2_clk"
-- clocks: phandles to input clocks.
-
-Examples:
-
-One interrupt per TC block:
-	tcb0: timer@fff7c000 {
-		compatible = "atmel,at91rm9200-tcb";
-		reg = <0xfff7c000 0x100>;
-		interrupts = <18 4>;
-		clocks = <&tcb0_clk>;
-		clock-names = "t0_clk";
-	};
-
-One interrupt per TC channel in a TC block:
-	tcb1: timer@fffdc000 {
-		compatible = "atmel,at91rm9200-tcb";
-		reg = <0xfffdc000 0x100>;
-		interrupts = <26 4 27 4 28 4>;
-		clocks = <&tcb1_clk>;
-		clock-names = "t0_clk";
-	};
-
 RSTC Reset Controller required properties:
 - compatible: Should be "atmel,<chip>-rstc".
   <chip> can be "at91sam9260" or "at91sam9g45" or "sama5d3"
diff --git a/Documentation/devicetree/bindings/arm/axentia.txt b/Documentation/devicetree/bindings/arm/axentia.txt
index ea3fb96ae465..de58f2463880 100644
--- a/Documentation/devicetree/bindings/arm/axentia.txt
+++ b/Documentation/devicetree/bindings/arm/axentia.txt
@@ -10,6 +10,15 @@ compatible = "axentia,linea",
 and following the rules from atmel-at91.txt for a sama5d31 SoC.
 
 
+Nattis v2 board with Natte v2 power board
+-----------------------------------------
+
+Required root node properties:
+compatible = "axentia,nattis-2", "axentia,natte-2", "axentia,linea",
+	     "atmel,sama5d31", "atmel,sama5d3", "atmel,sama5";
+and following the rules from above for the axentia,linea CPU module.
+
+
 TSE-850 v3 board
 ----------------
 
diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt b/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt
index 790e6b0b8306..c052caad36e8 100644
--- a/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt
+++ b/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt
@@ -17,21 +17,23 @@ Further, syscon nodes that map platform-specific registers used for general
 system control is required:
 
     - compatible: "brcm,bcm<chip_id>-sun-top-ctrl", "syscon"
-    - compatible: "brcm,bcm<chip_id>-hif-cpubiuctrl", "syscon"
+    - compatible: "brcm,bcm<chip_id>-cpu-biu-ctrl",
+		  "brcm,brcmstb-cpu-biu-ctrl",
+		  "syscon"
     - compatible: "brcm,bcm<chip_id>-hif-continuation", "syscon"
 
-hif-cpubiuctrl node
+cpu-biu-ctrl node
 -------------------
-SoCs with Broadcom Brahma15 ARM-based CPUs have a specific Bus Interface Unit
-(BIU) block which controls and interfaces the CPU complex to the different
-Memory Controller Ports (MCP), one per memory controller (MEMC). This BIU block
-offers a feature called Write Pairing which consists in collapsing two adjacent
-cache lines into a single (bursted) write transaction towards the memory
-controller (MEMC) to maximize write bandwidth.
+SoCs with Broadcom Brahma15 ARM-based and Brahma53 ARM64-based CPUs have a
+specific Bus Interface Unit (BIU) block which controls and interfaces the CPU
+complex to the different Memory Controller Ports (MCP), one per memory
+controller (MEMC). This BIU block offers a feature called Write Pairing which
+consists in collapsing two adjacent cache lines into a single (bursted) write
+transaction towards the memory controller (MEMC) to maximize write bandwidth.
 
 Required properties:
 
-    - compatible: must be "brcm,bcm7445-hif-cpubiuctrl", "syscon"
+    - compatible: must be "brcm,bcm7445-cpu-biu-ctrl", "brcm,brcmstb-cpu-biu-ctrl", "syscon"
 
 Optional properties:
 
@@ -52,7 +54,7 @@ example:
         };
 
         hif_cpubiuctrl: syscon@3e2400 {
-            compatible = "brcm,bcm7445-hif-cpubiuctrl", "syscon";
+            compatible = "brcm,bcm7445-cpu-biu-ctrl", "brcm,brcmstb-cpu-biu-ctrl", "syscon";
             reg = <0x3e2400 0x5b4>;
             brcm,write-pairing;
         };
diff --git a/Documentation/devicetree/bindings/arm/cpus.txt b/Documentation/devicetree/bindings/arm/cpus.txt
index a0009b72e9be..f4a777039f03 100644
--- a/Documentation/devicetree/bindings/arm/cpus.txt
+++ b/Documentation/devicetree/bindings/arm/cpus.txt
@@ -169,6 +169,7 @@ described below.
 			    "arm,cortex-r5"
 			    "arm,cortex-r7"
 			    "brcm,brahma-b15"
+			    "brcm,brahma-b53"
 			    "brcm,vulcan"
 			    "cavium,thunder"
 			    "cavium,thunder2"
diff --git a/Documentation/devicetree/bindings/arm/firmware/sdei.txt b/Documentation/devicetree/bindings/arm/firmware/sdei.txt
new file mode 100644
index 000000000000..ee3f0ff49889
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/firmware/sdei.txt
@@ -0,0 +1,42 @@
+* Software Delegated Exception Interface (SDEI)
+
+Firmware implementing the SDEI functions described in ARM document number
+ARM DEN 0054A ("Software Delegated Exception Interface") can be used by
+Linux to receive notification of events such as those generated by
+firmware-first error handling, or from an IRQ that has been promoted to
+a firmware-assisted NMI.
+
+The interface provides a number of API functions for registering callbacks
+and enabling/disabling events. Functions are invoked by trapping to the
+privilege level of the SDEI firmware (specified as part of the binding
+below) and passing arguments in a manner specified by the "SMC Calling
+Convention (ARM DEN 0028B):
+
+	 r0		=> 32-bit Function ID / return value
+	{r1 - r3}	=> Parameters
+
+Note that the immediate field of the trapping instruction must be set
+to #0.
+
+The SDEI_EVENT_REGISTER function registers a callback in the kernel
+text to handle the specified event number.
+
+The sdei node should be a child node of '/firmware' and have required
+properties:
+
+ - compatible    : should contain:
+	* "arm,sdei-1.0" : For implementations complying to SDEI version 1.x.
+
+ - method        : The method of calling the SDEI firmware. Permitted
+                   values are:
+	* "smc" : SMC #0, with the register assignments specified in this
+	          binding.
+	* "hvc" : HVC #0, with the register assignments specified in this
+	          binding.
+Example:
+	firmware {
+		sdei {
+			compatible	= "arm,sdei-1.0";
+			method		= "smc";
+		};
+	};
diff --git a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt
index 51336e5fc761..35c3c3460d17 100644
--- a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt
+++ b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt
@@ -14,3 +14,22 @@ following property before the previous one:
 Example:
 
 compatible = "marvell,armada-3720-db", "marvell,armada3720", "marvell,armada3710";
+
+
+Power management
+----------------
+
+For power management (particularly DVFS and AVS), the North Bridge
+Power Management component is needed:
+
+Required properties:
+- compatible     : should contain "marvell,armada-3700-nb-pm", "syscon";
+- reg            : the register start and length for the North Bridge
+		    Power Management
+
+Example:
+
+nb_pm: syscon@14000 {
+	compatible = "marvell,armada-3700-nb-pm", "syscon";
+	reg = <0x14000 0x60>;
+}
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt
index 7aa3fa167668..6cc7840ff37a 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,ethsys.txt
@@ -20,4 +20,5 @@ ethsys: clock-controller@1b000000 {
 	compatible = "mediatek,mt2701-ethsys", "syscon";
 	reg = <0 0x1b000000 0 0x1000>;
 	#clock-cells = <1>;
+	#reset-cells = <1>;
 };
diff --git a/Documentation/devicetree/bindings/arm/samsung/exynos-adc.txt b/Documentation/devicetree/bindings/arm/samsung/exynos-adc.txt
index ccaaec6014bd..6c49db7f8ad2 100644
--- a/Documentation/devicetree/bindings/arm/samsung/exynos-adc.txt
+++ b/Documentation/devicetree/bindings/arm/samsung/exynos-adc.txt
@@ -55,7 +55,7 @@ Note: child nodes can be added for auto probing from device tree.
 
 Example: adding device info in dtsi file
 
-adc: adc@12D10000 {
+adc: adc@12d10000 {
 	compatible = "samsung,exynos-adc-v1";
 	reg = <0x12D10000 0x100>;
 	interrupts = <0 106 0>;
@@ -71,7 +71,7 @@ adc: adc@12D10000 {
 
 Example: adding device info in dtsi file for Exynos3250 with additional sclk
 
-adc: adc@126C0000 {
+adc: adc@126c0000 {
 	compatible = "samsung,exynos3250-adc", "samsung,exynos-adc-v2;
 	reg = <0x126C0000 0x100>;
 	interrupts = <0 137 0>;
@@ -87,7 +87,7 @@ adc: adc@126C0000 {
 
 Example: Adding child nodes in dts file
 
-adc@12D10000 {
+adc@12d10000 {
 
 	/* NTC thermistor is a hwmon device */
 	ncp15wb473@0 {
diff --git a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
index e13459618581..469ac98ecf8f 100644
--- a/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
+++ b/Documentation/devicetree/bindings/arm/samsung/samsung-boards.txt
@@ -72,7 +72,7 @@ Optional nodes:
         - compatible: only "samsung,secure-firmware" is currently supported
         - reg: address of non-secure SYSRAM used for communication with firmware
 
-	firmware@203F000 {
+	firmware@203f000 {
 		compatible = "samsung,secure-firmware";
 		reg = <0x0203F000 0x1000>;
 	};
diff --git a/Documentation/devicetree/bindings/arm/shmobile.txt b/Documentation/devicetree/bindings/arm/shmobile.txt
index 020d758fc0c5..5c3af7ef0761 100644
--- a/Documentation/devicetree/bindings/arm/shmobile.txt
+++ b/Documentation/devicetree/bindings/arm/shmobile.txt
@@ -104,12 +104,16 @@ Boards:
     compatible = "renesas,salvator-x", "renesas,r8a7796"
   - Salvator-XS (Salvator-X 2nd version, RTP0RC7795SIPB0012S)
     compatible = "renesas,salvator-xs", "renesas,r8a7795"
+  - Salvator-XS (Salvator-X 2nd version, RTP0RC7796SIPB0012S)
+    compatible = "renesas,salvator-xs", "renesas,r8a7796"
   - SILK (RTP0RC7794LCB00011S)
     compatible = "renesas,silk", "renesas,r8a7794"
   - SK-RZG1E (YR8A77450S000BE)
     compatible = "renesas,sk-rzg1e", "renesas,r8a7745"
   - SK-RZG1M (YR8A77430S000BE)
     compatible = "renesas,sk-rzg1m", "renesas,r8a7743"
+  - V3MSK
+    compatible = "renesas,v3msk", "renesas,r8a77970"
   - Wheat
     compatible = "renesas,wheat", "renesas,r8a7792"
 
diff --git a/Documentation/devicetree/bindings/arm/stm32.txt b/Documentation/devicetree/bindings/arm/stm32.txt
new file mode 100644
index 000000000000..05762b08a7bb
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/stm32.txt
@@ -0,0 +1,9 @@
+STMicroelectronics STM32 Platforms Device Tree Bindings
+
+Each device tree must specify which STM32 SoC it uses,
+using one of the following compatible strings:
+
+  st,stm32f429
+  st,stm32f469
+  st,stm32f746
+  st,stm32h743
diff --git a/Documentation/devicetree/bindings/arm/technologic.txt b/Documentation/devicetree/bindings/arm/technologic.txt
index 33797acad846..f1cedc00dcab 100644
--- a/Documentation/devicetree/bindings/arm/technologic.txt
+++ b/Documentation/devicetree/bindings/arm/technologic.txt
@@ -1,6 +1,11 @@
 Technologic Systems Platforms Device Tree Bindings
 --------------------------------------------------
 
+TS-4600 is a System-on-Module based on the Freescale i.MX28 System-on-Chip.
+It can be mounted on a carrier board providing additional peripheral connectors.
+Required root node properties:
+	- compatible = "technologic,imx28-ts4600", "fsl,imx28"
+
 TS-4800 board
 Required root node properties:
 	- compatible = "technologic,imx51-ts4800", "fsl,imx51";
@@ -10,3 +15,9 @@ It can be mounted on a carrier board providing additional peripheral connectors.
 Required root node properties:
 	- compatible = "technologic,imx6dl-ts4900", "fsl,imx6dl"
 	- compatible = "technologic,imx6q-ts4900", "fsl,imx6q"
+
+TS-7970 is a System-on-Module based on the Freescale i.MX6 System-on-Chip.
+It can be mounted on a carrier board providing additional peripheral connectors.
+Required root node properties:
+	- compatible = "technologic,imx6dl-ts7970", "fsl,imx6dl"
+	- compatible = "technologic,imx6q-ts7970", "fsl,imx6q"
diff --git a/Documentation/devicetree/bindings/misc/arm-charlcd.txt b/Documentation/devicetree/bindings/auxdisplay/arm-charlcd.txt
similarity index 100%
rename from Documentation/devicetree/bindings/misc/arm-charlcd.txt
rename to Documentation/devicetree/bindings/auxdisplay/arm-charlcd.txt
diff --git a/Documentation/devicetree/bindings/bus/ti-sysc.txt b/Documentation/devicetree/bindings/bus/ti-sysc.txt
index fb1790e39398..2957a9ae291f 100644
--- a/Documentation/devicetree/bindings/bus/ti-sysc.txt
+++ b/Documentation/devicetree/bindings/bus/ti-sysc.txt
@@ -19,6 +19,7 @@ Required standard properties:
 
 - compatible	shall be one of the following generic types:
 
+		"ti,sysc"
 		"ti,sysc-omap2"
 		"ti,sysc-omap4"
 		"ti,sysc-omap4-simple"
@@ -26,6 +27,8 @@ Required standard properties:
 		or one of the following derivative types for hardware
 		needing special workarounds:
 
+		"ti,sysc-omap2-timer"
+		"ti,sysc-omap4-timer"
 		"ti,sysc-omap3430-sr"
 		"ti,sysc-omap3630-sr"
 		"ti,sysc-omap4-sr"
@@ -49,6 +52,26 @@ Required standard properties:
 
 Optional properties:
 
+- ti,sysc-mask	shall contain mask of supported register bits for the
+		SYSCONFIG register as documented in the Technical Reference
+		Manual (TRM) for the interconnect target module
+
+- ti,sysc-midle	list of master idle modes supported by the interconnect
+		target module as documented in the TRM for SYSCONFIG
+		register MIDLEMODE bits
+
+- ti,sysc-sidle	list of slave idle modes supported by the interconnect
+		target module as documented in the TRM for SYSCONFIG
+		register SIDLEMODE bits
+
+- ti,sysc-delay-us	delay needed after OCP softreset before accssing
+			SYSCONFIG register again
+
+- ti,syss-mask	optional mask of reset done status bits as described in the
+		TRM for SYSSTATUS registers, typically 1 with some devices
+		having separate reset done bits for children like OHCI and
+		EHCI
+
 - clocks	clock specifier for each name in the clock-names as
 		specified in the binding documentation for ti-clkctrl,
 		typically available for all interconnect targets on TI SoCs
@@ -61,6 +84,9 @@ Optional properties:
 - ti,hwmods	optional TI interconnect module name to use legacy
 		hwmod platform data
 
+- ti,no-reset-on-init	interconnect target module should not be reset at init
+
+- ti,no-idle-on-init	interconnect target module should not be idled at init
 
 Example: Single instance of MUSB controller on omap4 using interconnect ranges
 using offsets from l4_cfg second segment (0x4a000000 + 0x80000 = 0x4a0ab000):
@@ -74,6 +100,17 @@ using offsets from l4_cfg second segment (0x4a000000 + 0x80000 = 0x4a0ab000):
 		reg-names = "rev", "sysc", "syss";
 		clocks = <&l3_init_clkctrl OMAP4_USB_OTG_HS_CLKCTRL 0>;
 		clock-names = "fck";
+		ti,sysc-mask = <(SYSC_OMAP2_ENAWAKEUP |
+				 SYSC_OMAP2_SOFTRESET |
+				 SYSC_OMAP2_AUTOIDLE)>;
+		ti,sysc-midle = <SYSC_IDLE_FORCE>,
+				<SYSC_IDLE_NO>,
+				<SYSC_IDLE_SMART>;
+		ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+				<SYSC_IDLE_NO>,
+				<SYSC_IDLE_SMART>,
+				<SYSC_IDLE_SMART_WKUP>;
+		ti,syss-mask = <1>;
 		#address-cells = <1>;
 		#size-cells = <1>;
 		ranges = <0 0x2b000 0x1000>;
diff --git a/Documentation/devicetree/bindings/chosen.txt b/Documentation/devicetree/bindings/chosen.txt
index e3b13ea7d2ae..45e79172a646 100644
--- a/Documentation/devicetree/bindings/chosen.txt
+++ b/Documentation/devicetree/bindings/chosen.txt
@@ -120,3 +120,18 @@ e.g.
 While this property does not represent a real hardware, the address
 and the size are expressed in #address-cells and #size-cells,
 respectively, of the root node.
+
+linux,initrd-start and linux,initrd-end
+---------------------------------------
+
+These properties hold the physical start and end address of an initrd that's
+loaded by the bootloader. Note that linux,initrd-start is inclusive, but
+linux,initrd-end is exclusive.
+e.g.
+
+/ {
+	chosen {
+		linux,initrd-start = <0x82000000>;
+		linux,initrd-end = <0x82800000>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt
index 924040769186..e2b377ed6f91 100644
--- a/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt
+++ b/Documentation/devicetree/bindings/clock/amlogic,gxbb-clkc.txt
@@ -5,8 +5,11 @@ controllers within the SoC.
 
 Required Properties:
 
-- compatible: should be "amlogic,gxbb-clkc" for GXBB SoC,
-	      or "amlogic,gxl-clkc" for GXL and GXM SoC.
+- compatible: should be:
+		"amlogic,gxbb-clkc" for GXBB SoC,
+		"amlogic,gxl-clkc" for GXL and GXM SoC,
+		"amlogic,axg-clkc" for AXG SoC.
+
 - reg: physical base address of the clock controller and length of memory
        mapped region.
 
diff --git a/Documentation/devicetree/bindings/clock/exynos3250-clock.txt b/Documentation/devicetree/bindings/clock/exynos3250-clock.txt
index f1738b88c225..7441ed519f02 100644
--- a/Documentation/devicetree/bindings/clock/exynos3250-clock.txt
+++ b/Documentation/devicetree/bindings/clock/exynos3250-clock.txt
@@ -32,7 +32,7 @@ Example 1: Examples of clock controller nodes are listed below.
 		#clock-cells = <1>;
 	};
 
-	cmu_dmc: clock-controller@105C0000 {
+	cmu_dmc: clock-controller@105c0000 {
 		compatible = "samsung,exynos3250-cmu-dmc";
 		reg = <0x105C0000 0x2000>;
 		#clock-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/exynos5260-clock.txt b/Documentation/devicetree/bindings/clock/exynos5260-clock.txt
index 5496b2fac483..c79d31f7f66e 100644
--- a/Documentation/devicetree/bindings/clock/exynos5260-clock.txt
+++ b/Documentation/devicetree/bindings/clock/exynos5260-clock.txt
@@ -180,7 +180,7 @@ Example 2: UART controller node that consumes the clock generated by the
 		peri clock controller. Refer to the standard clock bindings for
 		information about 'clocks' and 'clock-names' property.
 
-	serial@12C00000 {
+	serial@12c00000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0x12C00000 0x100>;
 		interrupts = <0 146 0>;
diff --git a/Documentation/devicetree/bindings/clock/exynos5410-clock.txt b/Documentation/devicetree/bindings/clock/exynos5410-clock.txt
index c68b0d29b3d0..217beb27c30e 100644
--- a/Documentation/devicetree/bindings/clock/exynos5410-clock.txt
+++ b/Documentation/devicetree/bindings/clock/exynos5410-clock.txt
@@ -41,7 +41,7 @@ Example 2: UART controller node that consumes the clock generated by the clock
 	   controller. Refer to the standard clock bindings for information
 	   about 'clocks' and 'clock-names' property.
 
-	serial@12C20000 {
+	serial@12c20000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0x12C00000 0x100>;
 		interrupts = <0 51 0>;
diff --git a/Documentation/devicetree/bindings/clock/exynos5433-clock.txt b/Documentation/devicetree/bindings/clock/exynos5433-clock.txt
index c473dd38dd55..50d5897c9849 100644
--- a/Documentation/devicetree/bindings/clock/exynos5433-clock.txt
+++ b/Documentation/devicetree/bindings/clock/exynos5433-clock.txt
@@ -472,7 +472,7 @@ Example 2: Examples of clock controller nodes are listed below.
 Example 3: UART controller node that consumes the clock generated by the clock
 	   controller.
 
-	serial_0: serial@14C10000 {
+	serial_0: serial@14c10000 {
 		compatible = "samsung,exynos5433-uart";
 		reg = <0x14C10000 0x100>;
 		interrupts = <0 421 0>;
diff --git a/Documentation/devicetree/bindings/clock/hi3660-clock.txt b/Documentation/devicetree/bindings/clock/hi3660-clock.txt
index 0035a7ecaf20..946da7cee54f 100644
--- a/Documentation/devicetree/bindings/clock/hi3660-clock.txt
+++ b/Documentation/devicetree/bindings/clock/hi3660-clock.txt
@@ -13,12 +13,18 @@ Required Properties:
 	- "hisilicon,hi3660-pmuctrl"
 	- "hisilicon,hi3660-sctrl"
 	- "hisilicon,hi3660-iomcu"
+	- "hisilicon,hi3660-stub-clk"
 
 - reg: physical base address of the controller and length of memory mapped
   region.
 
 - #clock-cells: should be 1.
 
+Optional Properties:
+
+- mboxes: Phandle to the mailbox for sending message to MCU.
+            (See: ../mailbox/hisilicon,hi3660-mailbox.txt for more info)
+
 Each clock is assigned an identifier and client nodes use this identifier
 to specify the clock which they consume.
 
diff --git a/Documentation/devicetree/bindings/clock/qcom,a53pll.txt b/Documentation/devicetree/bindings/clock/qcom,a53pll.txt
new file mode 100644
index 000000000000..e3fa8118eaee
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,a53pll.txt
@@ -0,0 +1,22 @@
+Qualcomm MSM8916 A53 PLL Binding
+--------------------------------
+The A53 PLL on MSM8916 platforms is the main CPU PLL used used for frequencies
+above 1GHz.
+
+Required properties :
+- compatible : Shall contain only one of the following:
+
+		"qcom,msm8916-a53pll"
+
+- reg : shall contain base register location and length
+
+- #clock-cells : must be set to <0>
+
+Example:
+
+	a53pll: clock@b016000 {
+		compatible = "qcom,msm8916-a53pll";
+		reg = <0xb016000 0x40>;
+		#clock-cells = <0>;
+	};
+
diff --git a/Documentation/devicetree/bindings/clock/qcom,spmi-clkdiv.txt b/Documentation/devicetree/bindings/clock/qcom,spmi-clkdiv.txt
new file mode 100644
index 000000000000..7474aba36607
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,spmi-clkdiv.txt
@@ -0,0 +1,59 @@
+Qualcomm Technologies, Inc. SPMI PMIC clock divider (clkdiv)
+
+clkdiv configures the clock frequency of a set of outputs on the PMIC.
+These clocks are typically wired through alternate functions on
+gpio pins.
+
+=======================
+Properties
+=======================
+
+- compatible
+	Usage:      required
+	Value type: <string>
+	Definition: must be "qcom,spmi-clkdiv".
+
+- reg
+	Usage:      required
+	Value type: <prop-encoded-array>
+	Definition: base address of CLKDIV peripherals.
+
+- qcom,num-clkdivs
+	Usage:      required
+	Value type: <u32>
+	Definition: number of CLKDIV peripherals.
+
+- clocks:
+	Usage: required
+	Value type: <prop-encoded-array>
+	Definition: reference to the xo clock.
+
+- clock-names:
+	Usage: required
+	Value type: <stringlist>
+	Definition: must be "xo".
+
+- #clock-cells:
+	Usage: required
+	Value type: <u32>
+	Definition: shall contain 1.
+
+=======
+Example
+=======
+
+pm8998_clk_divs: clock-controller@5b00 {
+	compatible = "qcom,spmi-clkdiv";
+	reg = <0x5b00>;
+	#clock-cells = <1>;
+	qcom,num-clkdivs = <3>;
+	clocks = <&xo_board>;
+	clock-names = "xo";
+
+	assigned-clocks = <&pm8998_clk_divs 1>,
+			  <&pm8998_clk_divs 2>,
+			  <&pm8998_clk_divs 3>;
+	assigned-clock-rates = <9600000>,
+			       <9600000>,
+			       <9600000>;
+};
diff --git a/Documentation/devicetree/bindings/clock/qoriq-clock.txt b/Documentation/devicetree/bindings/clock/qoriq-clock.txt
index 6498e1fdbb33..97f46adac85f 100644
--- a/Documentation/devicetree/bindings/clock/qoriq-clock.txt
+++ b/Documentation/devicetree/bindings/clock/qoriq-clock.txt
@@ -78,6 +78,7 @@ second cell is the clock index for the specified type.
 	2	hwaccel		index (n in CLKCGnHWACSR)
 	3	fman		0 for fm1, 1 for fm2
 	4	platform pll	0=pll, 1=pll/2, 2=pll/3, 3=pll/4
+				4=pll/5, 5=pll/6, 6=pll/7, 7=pll/8
 	5	coreclk		must be 0
 
 3. Example
diff --git a/Documentation/devicetree/bindings/clock/silabs,si5351.txt b/Documentation/devicetree/bindings/clock/silabs,si5351.txt
index a6c4ef343b44..f00191cad8cd 100644
--- a/Documentation/devicetree/bindings/clock/silabs,si5351.txt
+++ b/Documentation/devicetree/bindings/clock/silabs,si5351.txt
@@ -49,6 +49,7 @@ Optional child node properties:
 - silabs,multisynth-source: source pll A(0) or B(1) of corresponding multisynth
   divider.
 - silabs,pll-master: boolean, multisynth can change pll frequency.
+- silabs,pll-reset: boolean, clock output can reset its pll.
 - silabs,disable-state : clock output disable state, shall be
   0 = clock output is driven LOW when disabled
   1 = clock output is driven HIGH when disabled
diff --git a/Documentation/devicetree/bindings/clock/sprd.txt b/Documentation/devicetree/bindings/clock/sprd.txt
new file mode 100644
index 000000000000..e9d179e882d9
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/sprd.txt
@@ -0,0 +1,63 @@
+Spreadtrum Clock Binding
+------------------------
+
+Required properties:
+- compatible: should contain the following compatible strings:
+	- "sprd,sc9860-pmu-gate"
+	- "sprd,sc9860-pll"
+	- "sprd,sc9860-ap-clk"
+	- "sprd,sc9860-aon-prediv"
+	- "sprd,sc9860-apahb-gate"
+	- "sprd,sc9860-aon-gate"
+	- "sprd,sc9860-aonsecure-clk"
+	- "sprd,sc9860-agcp-gate"
+	- "sprd,sc9860-gpu-clk"
+	- "sprd,sc9860-vsp-clk"
+	- "sprd,sc9860-vsp-gate"
+	- "sprd,sc9860-cam-clk"
+	- "sprd,sc9860-cam-gate"
+	- "sprd,sc9860-disp-clk"
+	- "sprd,sc9860-disp-gate"
+	- "sprd,sc9860-apapb-gate"
+
+- #clock-cells: must be 1
+
+- clocks : Should be the input parent clock(s) phandle for the clock, this
+	   property here just simply shows which clock group the clocks'
+	   parents are in, since each clk node would represent many clocks
+	   which are defined in the driver.  The detailed dependency
+	   relationship (i.e. how many parents and which are the parents)
+	   are implemented in driver code.
+
+Optional properties:
+
+- reg:	Contain the registers base address and length. It must be configured
+	only if no 'sprd,syscon' under the node.
+
+- sprd,syscon: phandle to the syscon which is in the same address area with
+	       the clock, and so we can get regmap for the clocks from the
+	       syscon device.
+
+Example:
+
+	pmu_gate: pmu-gate {
+		compatible = "sprd,sc9860-pmu-gate";
+		sprd,syscon = <&pmu_regs>;
+		clocks = <&ext_26m>;
+		#clock-cells = <1>;
+	};
+
+	pll: pll {
+		compatible = "sprd,sc9860-pll";
+		sprd,syscon = <&ana_regs>;
+		clocks = <&pmu_gate 0>;
+		#clock-cells = <1>;
+	};
+
+	ap_clk: clock-controller@20000000 {
+		compatible = "sprd,sc9860-ap-clk";
+		reg = <0 0x20000000 0 0x400>;
+		clocks = <&ext_26m>, <&pll 0>,
+			 <&pmu_gate 0>;
+		#clock-cells = <1>;
+	};
diff --git a/Documentation/devicetree/bindings/clock/sun8i-de2.txt b/Documentation/devicetree/bindings/clock/sun8i-de2.txt
index 631d27cd89d6..f2fa87c4765c 100644
--- a/Documentation/devicetree/bindings/clock/sun8i-de2.txt
+++ b/Documentation/devicetree/bindings/clock/sun8i-de2.txt
@@ -4,13 +4,14 @@ Allwinner Display Engine 2.0 Clock Control Binding
 Required properties :
 - compatible: must contain one of the following compatibles:
 		- "allwinner,sun8i-a83t-de2-clk"
+		- "allwinner,sun8i-h3-de2-clk"
 		- "allwinner,sun8i-v3s-de2-clk"
 		- "allwinner,sun50i-h5-de2-clk"
 
 - reg: Must contain the registers base address and length
 - clocks: phandle to the clocks feeding the display engine subsystem.
 	  Three are needed:
-  - "mod": the display engine module clock
+  - "mod": the display engine module clock (on A83T it's the DE PLL)
   - "bus": the bus clock for the whole display engine subsystem
 - clock-names: Must contain the clock names described just above
 - resets: phandle to the reset control for the display engine subsystem.
@@ -19,7 +20,7 @@ Required properties :
 
 Example:
 de2_clocks: clock@1000000 {
-	compatible = "allwinner,sun8i-a83t-de2-clk";
+	compatible = "allwinner,sun8i-h3-de2-clk";
 	reg = <0x01000000 0x100000>;
 	clocks = <&ccu CLK_BUS_DE>,
 		 <&ccu CLK_DE>;
diff --git a/Documentation/devicetree/bindings/crypto/arm-cryptocell.txt b/Documentation/devicetree/bindings/crypto/arm-cryptocell.txt
new file mode 100644
index 000000000000..cec8d5d74e26
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/arm-cryptocell.txt
@@ -0,0 +1,22 @@
+Arm TrustZone CryptoCell cryptographic engine
+
+Required properties:
+- compatible: Should be "arm,cryptocell-712-ree".
+- reg: Base physical address of the engine and length of memory mapped region.
+- interrupts: Interrupt number for the device.
+
+Optional properties:
+- interrupt-parent: The phandle for the interrupt controller that services
+  interrupts for this device.
+- clocks: Reference to the crypto engine clock.
+- dma-coherent: Present if dma operations are coherent.
+
+Examples:
+
+       arm_cc712: crypto@80000000 {
+               compatible = "arm,cryptocell-712-ree";
+               interrupt-parent = <&intc>;
+               interrupts = < 0 30 4 >;
+               reg = < 0x80000000 0x10000 >;
+
+       };
diff --git a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
index 7de1a9674c70..6b458bb2440d 100644
--- a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
+++ b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt
@@ -75,7 +75,7 @@ Required properties:
 - clock-frequency: must be present in the i2c controller node.
 
 Example:
-atecc508a@C0 {
+atecc508a@c0 {
 	compatible = "atmel,atecc508a";
 	reg = <0xC0>;
 };
diff --git a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt
index fbc07d12322f..30c3ce6b502e 100644
--- a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt
+++ b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt
@@ -1,7 +1,8 @@
 Inside Secure SafeXcel cryptographic engine
 
 Required properties:
-- compatible: Should be "inside-secure,safexcel-eip197".
+- compatible: Should be "inside-secure,safexcel-eip197" or
+              "inside-secure,safexcel-eip97".
 - reg: Base physical address of the engine and length of memory mapped region.
 - interrupts: Interrupt numbers for the rings and engine.
 - interrupt-names: Should be "ring0", "ring1", "ring2", "ring3", "eip", "mem".
diff --git a/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt b/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt
index 4ca8dd4d7e66..a13fbdb4bd88 100644
--- a/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt
+++ b/Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt
@@ -2,7 +2,9 @@ Exynos Pseudo Random Number Generator
 
 Required properties:
 
-- compatible  : Should be "samsung,exynos4-rng".
+- compatible  : One of:
+                - "samsung,exynos4-rng" for Exynos4210 and Exynos4412
+                - "samsung,exynos5250-prng" for Exynos5250+
 - reg         : Specifies base physical address and size of the registers map.
 - clocks      : Phandle to clock-controller plus clock-specifier pair.
 - clock-names : "secss" as a clock name.
diff --git a/Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt b/Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt
new file mode 100644
index 000000000000..970487fa40b8
--- /dev/null
+++ b/Documentation/devicetree/bindings/crypto/st,stm32-cryp.txt
@@ -0,0 +1,19 @@
+* STMicroelectronics STM32 CRYP
+
+Required properties:
+- compatible: Should be "st,stm32f756-cryp".
+- reg: The address and length of the peripheral registers space
+- clocks: The input clock of the CRYP instance
+- interrupts: The CRYP interrupt
+
+Optional properties:
+- resets: The input reset of the CRYP instance
+
+Example:
+crypto@50060000 {
+	compatible = "st,stm32f756-cryp";
+	reg = <0x50060000 0x400>;
+	interrupts = <79>;
+	clocks = <&rcc 0 STM32F7_AHB2_CLOCK(CRYP)>;
+	resets = <&rcc STM32F7_AHB2_RESET(CRYP)>;
+};
diff --git a/Documentation/devicetree/bindings/devfreq/event/exynos-nocp.txt b/Documentation/devicetree/bindings/devfreq/event/exynos-nocp.txt
index fd459f00aa5a..aeaebd425d1f 100644
--- a/Documentation/devicetree/bindings/devfreq/event/exynos-nocp.txt
+++ b/Documentation/devicetree/bindings/devfreq/event/exynos-nocp.txt
@@ -20,7 +20,7 @@ Optional properties:
 
 Example : NoC Probe nodes in Device Tree are listed below.
 
-	nocp_mem0_0: nocp@10CA1000 {
+	nocp_mem0_0: nocp@10ca1000 {
 		compatible = "samsung,exynos5420-nocp";
 		reg = <0x10CA1000 0x200>;
 	};
diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt
index 7f040edc16fe..bf4a18047309 100644
--- a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt
+++ b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt
@@ -48,6 +48,10 @@ Required properties:
   Documentation/devicetree/bindings/reset/reset.txt,
   the reset-names should be "hdmitx_apb", "hdmitx", "hdmitx_phy"
 
+Optional properties:
+- hdmi-supply: Optional phandle to an external 5V regulator to power the HDMI
+  logic, as described in the file ../regulator/regulator.txt
+
 Required nodes:
 
 The connections to the HDMI ports are modeled using the OF graph
diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
index 00f74bad1e95..057b81335775 100644
--- a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
+++ b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
@@ -64,6 +64,10 @@ Required properties:
 - reg-names: should contain the names of the previous memory regions
 - interrupts: should contain the VENC Vsync interrupt number
 
+Optional properties:
+- power-domains: Optional phandle to associated power domain as described in
+	the file ../power/power_domain.txt
+
 Required nodes:
 
 The connections to the VPU output video ports are modeled using the OF graph
diff --git a/Documentation/devicetree/bindings/display/exynos/exynos_dsim.txt b/Documentation/devicetree/bindings/display/exynos/exynos_dsim.txt
index ca5204b3bc21..2fff8b406f4c 100644
--- a/Documentation/devicetree/bindings/display/exynos/exynos_dsim.txt
+++ b/Documentation/devicetree/bindings/display/exynos/exynos_dsim.txt
@@ -54,7 +54,7 @@ Video interfaces:
 
 Example:
 
-	dsi@11C80000 {
+	dsi@11c80000 {
 		compatible = "samsung,exynos4210-mipi-dsi";
 		reg = <0x11C80000 0x10000>;
 		interrupts = <0 79 0>;
diff --git a/Documentation/devicetree/bindings/display/exynos/exynos_hdmi.txt b/Documentation/devicetree/bindings/display/exynos/exynos_hdmi.txt
index 6394ea9e3b9e..58b12e25bbb1 100644
--- a/Documentation/devicetree/bindings/display/exynos/exynos_hdmi.txt
+++ b/Documentation/devicetree/bindings/display/exynos/exynos_hdmi.txt
@@ -16,6 +16,7 @@ Required properties:
 - ddc: phandle to the hdmi ddc node
 - phy: phandle to the hdmi phy node
 - samsung,syscon-phandle: phandle for system controller node for PMU.
+- #sound-dai-cells: should be 0.
 
 Required properties for Exynos 4210, 4212, 5420 and 5433:
 - clocks: list of clock IDs from SoC clock driver.
diff --git a/Documentation/devicetree/bindings/display/ilitek,ili9225.txt b/Documentation/devicetree/bindings/display/ilitek,ili9225.txt
new file mode 100644
index 000000000000..a59feb52015b
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/ilitek,ili9225.txt
@@ -0,0 +1,25 @@
+Ilitek ILI9225 display panels
+
+This binding is for display panels using an Ilitek ILI9225 controller in SPI
+mode.
+
+Required properties:
+- compatible:	"vot,v220hf01a-t", "ilitek,ili9225"
+- rs-gpios:	Register select signal
+- reset-gpios:	Reset pin
+
+The node for this driver must be a child node of a SPI controller, hence
+all mandatory properties described in ../spi/spi-bus.txt must be specified.
+
+Optional properties:
+- rotation:	panel rotation in degrees counter clockwise (0,90,180,270)
+
+Example:
+	display@0{
+		compatible = "vot,v220hf01a-t", "ilitek,ili9225";
+		reg = <0>;
+		spi-max-frequency = <12000000>;
+		rs-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio0 8 GPIO_ACTIVE_HIGH>;
+		rotation = <270>;
+	};
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.txt b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.txt
new file mode 100644
index 000000000000..3d5ce6ad6ec7
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9322.txt
@@ -0,0 +1,49 @@
+Ilitek ILI9322 TFT panel driver with SPI control bus
+
+This is a driver for 320x240 TFT panels, accepting a variety of input
+streams that get adapted and scaled to the panel. The panel output has
+960 TFT source driver pins and 240 TFT gate driver pins, VCOM, VCOML and
+VCOMH outputs.
+
+Required properties:
+  - compatible: "dlink,dir-685-panel", "ilitek,ili9322"
+    (full system-specific compatible is always required to look up configuration)
+  - reg: address of the panel on the SPI bus
+
+Optional properties:
+  - vcc-supply: core voltage supply, see regulator/regulator.txt
+  - iovcc-supply: voltage supply for the interface input/output signals,
+    see regulator/regulator.txt
+  - vci-supply: voltage supply for analog parts, see regulator/regulator.txt
+  - reset-gpios: a GPIO spec for the reset pin, see gpio/gpio.txt
+
+  The following optional properties only apply to RGB and YUV input modes and
+  can be omitted for BT.656 input modes:
+
+  - pixelclk-active: see display/panel/display-timing.txt
+  - de-active: see display/panel/display-timing.txt
+  - hsync-active: see display/panel/display-timing.txt
+  - vsync-active: see display/panel/display-timing.txt
+
+The panel must obey the rules for a SPI slave device as specified in
+spi/spi-bus.txt
+
+The device node can contain one 'port' child node with one child
+'endpoint' node, according to the bindings defined in
+media/video-interfaces.txt. This node should describe panel's video bus.
+
+Example:
+
+panel: display@0 {
+	compatible = "dlink,dir-685-panel", "ilitek,ili9322";
+	reg = <0>;
+	vcc-supply = <&vdisp>;
+	iovcc-supply = <&vdisp>;
+	vci-supply = <&vdisp>;
+
+	port {
+		panel_in: endpoint {
+			remote-endpoint = <&display_out>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/display/panel/mitsubishi,aa070mc01.txt b/Documentation/devicetree/bindings/display/panel/mitsubishi,aa070mc01.txt
new file mode 100644
index 000000000000..7d8f6eeef6d9
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/mitsubishi,aa070mc01.txt
@@ -0,0 +1,7 @@
+Mitsubishi "AA070MC01 7.0" WVGA TFT LCD panel
+
+Required properties:
+- compatible: should be "mitsubishi,aa070mc01-ca1"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff --git a/Documentation/devicetree/bindings/display/panel/panel-common.txt b/Documentation/devicetree/bindings/display/panel/panel-common.txt
index ec52c472c845..557fa765adcb 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-common.txt
+++ b/Documentation/devicetree/bindings/display/panel/panel-common.txt
@@ -78,6 +78,16 @@ used for panels that implement compatible control signals.
   while active. Active high reset signals can be supported by inverting the
   GPIO specifier polarity flag.
 
+Power
+-----
+
+- power-supply: display panels require power to be supplied. While several
+  panels need more than one power supply with panel-specific constraints
+  governing the order and timings of the power supplies, in many cases a single
+  power supply is sufficient, either because the panel has a single power rail,
+  or because all its power rails can be driven by the same supply. In that case
+  the power-supply property specifies the supply powering the panel as a phandle
+  to a regulator.
 
 Backlight
 ---------
diff --git a/Documentation/devicetree/bindings/display/panel/panel-lvds.txt b/Documentation/devicetree/bindings/display/panel/panel-lvds.txt
index b938269f841e..250850a2150b 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-lvds.txt
+++ b/Documentation/devicetree/bindings/display/panel/panel-lvds.txt
@@ -32,6 +32,7 @@ Optional properties:
 - label: See panel-common.txt.
 - gpios: See panel-common.txt.
 - backlight: See panel-common.txt.
+- power-supply: See panel-common.txt.
 - data-mirror: If set, reverse the bit order described in the data mappings
   below on all data lanes, transmitting bits for slots 6 to 0 instead of
   0 to 6.
diff --git a/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.txt b/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.txt
new file mode 100644
index 000000000000..d06644b555bd
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.txt
@@ -0,0 +1,41 @@
+Solomon Goldentek Display GKTW70SDAE4SE LVDS Display Panel
+==========================================================
+
+The GKTW70SDAE4SE is a 7" WVGA TFT-LCD display panel.
+
+These DT bindings follow the LVDS panel bindings defined in panel-lvds.txt
+with the following device-specific properties.
+
+Required properties:
+
+- compatible: Shall contain "sgd,gktw70sdae4se" and "panel-lvds", in that order.
+
+Example
+-------
+
+panel {
+	compatible = "sgd,gktw70sdae4se", "panel-lvds";
+
+	width-mm = <153>;
+	height-mm = <86>;
+
+	data-mapping = "jeida-18";
+
+	panel-timing {
+		clock-frequency = <32000000>;
+		hactive = <800>;
+		vactive = <480>;
+		hback-porch = <39>;
+		hfront-porch = <39>;
+		vback-porch = <29>;
+		vfront-porch = <13>;
+		hsync-len = <47>;
+		vsync-len = <2>;
+	};
+
+	port {
+		panel_in: endpoint {
+			remote-endpoint = <&lvds_encoder>;
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/display/panel/simple-panel.txt b/Documentation/devicetree/bindings/display/panel/simple-panel.txt
index 1341bbf4aa3d..16d8ff088b7d 100644
--- a/Documentation/devicetree/bindings/display/panel/simple-panel.txt
+++ b/Documentation/devicetree/bindings/display/panel/simple-panel.txt
@@ -1,7 +1,7 @@
 Simple display panel
 
 Required properties:
-- power-supply: regulator to provide the supply voltage
+- power-supply: See panel-common.txt
 
 Optional properties:
 - ddc-i2c-bus: phandle of an I2C controller used for DDC EDID probing
diff --git a/Documentation/devicetree/bindings/display/panel/tianma,tm070rvhg71.txt b/Documentation/devicetree/bindings/display/panel/tianma,tm070rvhg71.txt
new file mode 100644
index 000000000000..b25261e63a6d
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/tianma,tm070rvhg71.txt
@@ -0,0 +1,29 @@
+Tianma Micro-electronics TM070RVHG71 7.0" WXGA TFT LCD panel
+
+Required properties:
+- compatible: should be "tianma,tm070rvhg71"
+- power-supply: single regulator to provide the supply voltage
+- backlight: phandle of the backlight device attached to the panel
+
+Required nodes:
+- port: LVDS port mapping to connect this display
+
+This panel needs single power supply voltage. Its backlight is conntrolled
+via PWM signal.
+
+Example:
+--------
+
+Example device-tree definition when connected to iMX6Q based board
+
+	panel: panel-lvds0 {
+		compatible = "tianma,tm070rvhg71";
+		backlight = <&backlight_lvds>;
+		power-supply = <&reg_lvds>;
+
+		port {
+			panel_in_lvds0: endpoint {
+				remote-endpoint = <&lvds0_out>;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/display/panel/toshiba,lt089ac29000.txt b/Documentation/devicetree/bindings/display/panel/toshiba,lt089ac29000.txt
index 4c0caaf246c9..89826116628c 100644
--- a/Documentation/devicetree/bindings/display/panel/toshiba,lt089ac29000.txt
+++ b/Documentation/devicetree/bindings/display/panel/toshiba,lt089ac29000.txt
@@ -1,7 +1,7 @@
 Toshiba 8.9" WXGA (1280x768) TFT LCD panel
 
 Required properties:
-- compatible: should be "toshiba,lt089ac29000.txt"
+- compatible: should be "toshiba,lt089ac29000"
 - power-supply: as specified in the base binding
 
 This binding is compatible with the simple-panel binding, which is specified
diff --git a/Documentation/devicetree/bindings/display/panel/toppoly,td028ttec1.txt b/Documentation/devicetree/bindings/display/panel/tpo,td028ttec1.txt
similarity index 84%
rename from Documentation/devicetree/bindings/display/panel/toppoly,td028ttec1.txt
rename to Documentation/devicetree/bindings/display/panel/tpo,td028ttec1.txt
index 7175dc3740ac..ed34253d9fb1 100644
--- a/Documentation/devicetree/bindings/display/panel/toppoly,td028ttec1.txt
+++ b/Documentation/devicetree/bindings/display/panel/tpo,td028ttec1.txt
@@ -2,7 +2,7 @@ Toppoly TD028TTEC1 Panel
 ========================
 
 Required properties:
-- compatible: "toppoly,td028ttec1"
+- compatible: "tpo,td028ttec1"
 
 Optional properties:
 - label: a symbolic name for the panel
@@ -14,7 +14,7 @@ Example
 -------
 
 lcd-panel: td028ttec1@0 {
-	compatible = "toppoly,td028ttec1";
+	compatible = "tpo,td028ttec1";
 	reg = <0>;
 	spi-max-frequency = <100000>;
 	spi-cpol;
diff --git a/Documentation/devicetree/bindings/display/renesas,du.txt b/Documentation/devicetree/bindings/display/renesas,du.txt
index 4bbd1e9bf3be..cd48aba3bc8c 100644
--- a/Documentation/devicetree/bindings/display/renesas,du.txt
+++ b/Documentation/devicetree/bindings/display/renesas,du.txt
@@ -3,6 +3,8 @@
 Required Properties:
 
   - compatible: must be one of the following.
+    - "renesas,du-r8a7743" for R8A7743 (RZ/G1M) compatible DU
+    - "renesas,du-r8a7745" for R8A7745 (RZ/G1E) compatible DU
     - "renesas,du-r8a7779" for R8A7779 (R-Car H1) compatible DU
     - "renesas,du-r8a7790" for R8A7790 (R-Car H2) compatible DU
     - "renesas,du-r8a7791" for R8A7791 (R-Car M2-W) compatible DU
@@ -27,10 +29,10 @@ Required Properties:
   - clock-names: Name of the clocks. This property is model-dependent.
     - R8A7779 uses a single functional clock. The clock doesn't need to be
       named.
-    - R8A779[0123456] use one functional clock per channel and one clock per
-      LVDS encoder (if available). The functional clocks must be named "du.x"
-      with "x" being the channel numerical index. The LVDS clocks must be
-      named "lvds.x" with "x" being the LVDS encoder numerical index.
+    - All other DU instances use one functional clock per channel and one
+      clock per LVDS encoder (if available). The functional clocks must be
+      named "du.x" with "x" being the channel numerical index. The LVDS clocks
+      must be named "lvds.x" with "x" being the LVDS encoder numerical index.
     - In addition to the functional and encoder clocks, all DU versions also
       support externally supplied pixel clocks. Those clocks are optional.
       When supplied they must be named "dclkin.x" with "x" being the input
@@ -49,16 +51,18 @@ bindings specified in Documentation/devicetree/bindings/graph.txt.
 The following table lists for each supported model the port number
 corresponding to each DU output.
 
-		Port 0		Port1		Port2		Port3
+                      Port0          Port1          Port2          Port3
 -----------------------------------------------------------------------------
- R8A7779 (H1)	DPAD 0		DPAD 1		-		-
- R8A7790 (H2)	DPAD		LVDS 0		LVDS 1		-
- R8A7791 (M2-W)	DPAD		LVDS 0		-		-
- R8A7792 (V2H)	DPAD 0		DPAD 1		-		-
- R8A7793 (M2-N)	DPAD		LVDS 0		-		-
- R8A7794 (E2)	DPAD 0		DPAD 1		-		-
- R8A7795 (H3)	DPAD		HDMI 0		HDMI 1		LVDS
- R8A7796 (M3-W)	DPAD		HDMI		LVDS		-
+ R8A7743 (RZ/G1M)     DPAD 0         LVDS 0         -              -
+ R8A7745 (RZ/G1E)     DPAD 0         DPAD 1         -              -
+ R8A7779 (R-Car H1)   DPAD 0         DPAD 1         -              -
+ R8A7790 (R-Car H2)   DPAD 0         LVDS 0         LVDS 1         -
+ R8A7791 (R-Car M2-W) DPAD 0         LVDS 0         -              -
+ R8A7792 (R-Car V2H)  DPAD 0         DPAD 1         -              -
+ R8A7793 (R-Car M2-N) DPAD 0         LVDS 0         -              -
+ R8A7794 (R-Car E2)   DPAD 0         DPAD 1         -              -
+ R8A7795 (R-Car H3)   DPAD 0         HDMI 0         HDMI 1         LVDS 0
+ R8A7796 (R-Car M3-W) DPAD 0         HDMI 0         LVDS 0         -
 
 
 Example: R8A7795 (R-Car H3) ES2.0 DU
diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.txt b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.txt
index 5d835d9c1ba8..eeda3597011e 100644
--- a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.txt
+++ b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.txt
@@ -7,6 +7,7 @@ buffer to an external LCD interface.
 Required properties:
 - compatible: value should be one of the following
 		"rockchip,rk3036-vop";
+		"rockchip,rk3126-vop";
 		"rockchip,rk3288-vop";
 		"rockchip,rk3368-vop";
 		"rockchip,rk3366-vop";
diff --git a/Documentation/devicetree/bindings/display/simple-framebuffer-sunxi.txt b/Documentation/devicetree/bindings/display/simple-framebuffer-sunxi.txt
index a9168ae6946c..d693b8dc9a62 100644
--- a/Documentation/devicetree/bindings/display/simple-framebuffer-sunxi.txt
+++ b/Documentation/devicetree/bindings/display/simple-framebuffer-sunxi.txt
@@ -15,6 +15,10 @@ Required properties:
   "de_be1-lcd1"
   "de_be0-lcd0-hdmi"
   "de_be1-lcd1-hdmi"
+  "mixer0-lcd0"
+  "mixer0-lcd0-hdmi"
+  "mixer1-lcd1-hdmi"
+  "mixer1-lcd1-tve"
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/display/sitronix,st7735r.txt b/Documentation/devicetree/bindings/display/sitronix,st7735r.txt
new file mode 100644
index 000000000000..f0a5090a3326
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/sitronix,st7735r.txt
@@ -0,0 +1,35 @@
+Sitronix ST7735R display panels
+
+This binding is for display panels using a Sitronix ST7735R controller in SPI
+mode.
+
+Required properties:
+- compatible:	"jianda,jd-t18003-t01", "sitronix,st7735r"
+- dc-gpios:	Display data/command selection (D/CX)
+- reset-gpios:	Reset signal (RSTX)
+
+The node for this driver must be a child node of a SPI controller, hence
+all mandatory properties described in ../spi/spi-bus.txt must be specified.
+
+Optional properties:
+- rotation:	panel rotation in degrees counter clockwise (0,90,180,270)
+- backlight:	phandle of the backlight device attached to the panel
+
+Example:
+
+	backlight: backlight {
+		compatible = "gpio-backlight";
+		gpios = <&gpio 44 GPIO_ACTIVE_HIGH>;
+	}
+
+	...
+
+	display@0{
+		compatible = "jianda,jd-t18003-t01", "sitronix,st7735r";
+		reg = <0>;
+		spi-max-frequency = <32000000>;
+		dc-gpios = <&gpio 43 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpio 80 GPIO_ACTIVE_HIGH>;
+		rotation = <270>;
+		backlight = &backlight;
+	};
diff --git a/Documentation/devicetree/bindings/display/st,stih4xx.txt b/Documentation/devicetree/bindings/display/st,stih4xx.txt
index a352ed30cd70..6778b3e7ad5b 100644
--- a/Documentation/devicetree/bindings/display/st,stih4xx.txt
+++ b/Documentation/devicetree/bindings/display/st,stih4xx.txt
@@ -119,7 +119,7 @@ Example:
 / {
 	...
 
-	vtg_main_slave: sti-vtg-main-slave@fe85A800 {
+	vtg_main_slave: sti-vtg-main-slave@fe85a800 {
 		compatible	= "st,vtg";
 		reg		= <0xfe85A800 0x300>;
 		interrupts	= <GIC_SPI 175 IRQ_TYPE_NONE>;
diff --git a/Documentation/devicetree/bindings/display/st,stm32-ltdc.txt b/Documentation/devicetree/bindings/display/st,stm32-ltdc.txt
index 74b5ac7b26d6..029252253ad4 100644
--- a/Documentation/devicetree/bindings/display/st,stm32-ltdc.txt
+++ b/Documentation/devicetree/bindings/display/st,stm32-ltdc.txt
@@ -10,7 +10,11 @@
       - "lcd" for the clock feeding the output pixel clock & IP clock.
   - resets: reset to be used by the device (defined by use of RCC macro).
   Required nodes:
-    - Video port for RGB output.
+  - Video port for DPI RGB output: ltdc has one video port with up to 2
+    endpoints:
+      - for external dpi rgb panel or bridge, using gpios.
+      - for internal dpi input of the MIPI DSI host controller.
+      Note: These 2 endpoints cannot be activated simultaneously.
 
 * STMicroelectronics STM32 DSI controller specific extensions to Synopsys
   DesignWare MIPI DSI host controller
diff --git a/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt b/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt
index 50cc72ee1168..cd626ee1147a 100644
--- a/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt
+++ b/Documentation/devicetree/bindings/display/sunxi/sun4i-drm.txt
@@ -93,6 +93,7 @@ Required properties:
    * allwinner,sun6i-a31s-tcon
    * allwinner,sun7i-a20-tcon
    * allwinner,sun8i-a33-tcon
+   * allwinner,sun8i-a83t-tcon-lcd
    * allwinner,sun8i-v3s-tcon
  - reg: base address and size of memory-mapped region
  - interrupts: interrupt associated to this IP
@@ -121,6 +122,14 @@ Required properties:
 On SoCs other than the A33 and V3s, there is one more clock required:
    - 'tcon-ch1': The clock driving the TCON channel 1
 
+On SoCs that support LVDS (all SoCs but the A13, H3, H5 and V3s), you
+need one more reset line:
+   - 'lvds': The reset line driving the LVDS logic
+
+And on the A23, A31, A31s and A33, you need one more clock line:
+   - 'lvds-alt': An alternative clock source, separate from the TCON channel 0
+                 clock, that can be used to drive the LVDS clock
+
 DRC
 ---
 
@@ -216,6 +225,7 @@ supported.
 
 Required properties:
   - compatible: value must be one of:
+    * allwinner,sun8i-a83t-de2-mixer-0
     * allwinner,sun8i-v3s-de2-mixer
   - reg: base address and size of the memory-mapped region.
   - clocks: phandles to the clocks feeding the mixer
@@ -245,6 +255,7 @@ Required properties:
     * allwinner,sun6i-a31s-display-engine
     * allwinner,sun7i-a20-display-engine
     * allwinner,sun8i-a33-display-engine
+    * allwinner,sun8i-a83t-display-engine
     * allwinner,sun8i-v3s-display-engine
 
   - allwinner,pipelines: list of phandle to the display engine
diff --git a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.txt b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.txt
index 844e0103fb0d..593be44a53c9 100644
--- a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.txt
+++ b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-host1x.txt
@@ -206,21 +206,33 @@ of the following host1x client modules:
     - "nvidia,tegra132-sor": for Tegra132
     - "nvidia,tegra210-sor": for Tegra210
     - "nvidia,tegra210-sor1": for Tegra210
+    - "nvidia,tegra186-sor": for Tegra186
+    - "nvidia,tegra186-sor1": for Tegra186
   - reg: Physical base address and length of the controller's registers.
   - interrupts: The interrupt outputs from the controller.
   - clocks: Must contain an entry for each entry in clock-names.
     See ../clocks/clock-bindings.txt for details.
   - clock-names: Must include the following entries:
     - sor: clock input for the SOR hardware
-    - source: source clock for the SOR clock
+    - out: SOR output clock
     - parent: input for the pixel clock
     - dp: reference clock for the SOR clock
     - safe: safe reference for the SOR clock during power up
+
+    For Tegra186 and later:
+    - pad: SOR pad output clock (on Tegra186 and later)
+
+    Obsolete:
+    - source: source clock for the SOR clock (obsolete, use "out" instead)
+
   - resets: Must contain an entry for each entry in reset-names.
     See ../reset/reset.txt for details.
   - reset-names: Must include the following entries:
     - sor
 
+  Required properties on Tegra186 and later:
+  - nvidia,interface: index of the SOR interface
+
   Optional properties:
   - nvidia,ddc-i2c-bus: phandle of an I2C controller used for DDC EDID probing
   - nvidia,hpd-gpio: specifies a GPIO used for hotplug detection
diff --git a/Documentation/devicetree/bindings/display/ti/ti,dra7-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,dra7-dss.txt
index c30f9ec189ed..91279f1060fe 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,dra7-dss.txt
+++ b/Documentation/devicetree/bindings/display/ti/ti,dra7-dss.txt
@@ -47,6 +47,11 @@ Required properties:
 - clocks: handle to fclk
 - clock-names: "fck"
 
+Optional properties:
+- max-memory-bandwidth: Input memory (from main memory to dispc) bandwidth limit
+			in bytes per second
+
+
 HDMI
 ----
 
diff --git a/Documentation/devicetree/bindings/display/ti/ti,omap2-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,omap2-dss.txt
index afcd5a86c6a4..ee867c4d1152 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,omap2-dss.txt
+++ b/Documentation/devicetree/bindings/display/ti/ti,omap2-dss.txt
@@ -28,6 +28,10 @@ Required properties:
 - ti,hwmods: "dss_dispc"
 - interrupts: the DISPC interrupt
 
+Optional properties:
+- max-memory-bandwidth: Input memory (from main memory to dispc) bandwidth limit
+			in bytes per second
+
 
 RFBI
 ----
diff --git a/Documentation/devicetree/bindings/display/ti/ti,omap3-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,omap3-dss.txt
index dc66e1447c31..cd02516a40b6 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,omap3-dss.txt
+++ b/Documentation/devicetree/bindings/display/ti/ti,omap3-dss.txt
@@ -37,6 +37,10 @@ Required properties:
 - clocks: handle to fclk
 - clock-names: "fck"
 
+Optional properties:
+- max-memory-bandwidth: Input memory (from main memory to dispc) bandwidth limit
+			in bytes per second
+
 
 RFBI
 ----
diff --git a/Documentation/devicetree/bindings/display/ti/ti,omap4-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,omap4-dss.txt
index bc624db8888d..0f85f6b3a5a8 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,omap4-dss.txt
+++ b/Documentation/devicetree/bindings/display/ti/ti,omap4-dss.txt
@@ -36,6 +36,10 @@ Required properties:
 - clocks: handle to fclk
 - clock-names: "fck"
 
+Optional properties:
+- max-memory-bandwidth: Input memory (from main memory to dispc) bandwidth limit
+			in bytes per second
+
 
 RFBI
 ----
diff --git a/Documentation/devicetree/bindings/display/ti/ti,omap5-dss.txt b/Documentation/devicetree/bindings/display/ti/ti,omap5-dss.txt
index 118a486c47bb..20861218649f 100644
--- a/Documentation/devicetree/bindings/display/ti/ti,omap5-dss.txt
+++ b/Documentation/devicetree/bindings/display/ti/ti,omap5-dss.txt
@@ -36,6 +36,10 @@ Required properties:
 - clocks: handle to fclk
 - clock-names: "fck"
 
+Optional properties:
+- max-memory-bandwidth: Input memory (from main memory to dispc) bandwidth limit
+			in bytes per second
+
 
 RFBI
 ----
diff --git a/Documentation/devicetree/bindings/dma/mv-xor-v2.txt b/Documentation/devicetree/bindings/dma/mv-xor-v2.txt
index 217a90eaabe7..9c38bbe7e6d7 100644
--- a/Documentation/devicetree/bindings/dma/mv-xor-v2.txt
+++ b/Documentation/devicetree/bindings/dma/mv-xor-v2.txt
@@ -11,7 +11,11 @@ Required properties:
   interrupts.
 
 Optional properties:
-- clocks: Optional reference to the clock used by the XOR engine.
+- clocks: Optional reference to the clocks used by the XOR engine.
+- clock-names: mandatory if there is a second clock, in this case the
+   name must be "core" for the first clock and "reg" for the second
+   one
+
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt b/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt
index b3408cc57be6..1ae4748730a8 100644
--- a/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt
+++ b/Documentation/devicetree/bindings/dma/qcom_hidma_mgmt.txt
@@ -47,8 +47,8 @@ When the OS is not in control of the management interface (i.e. it's a guest),
 the channel nodes appear on their own, not under a management node.
 
 Required properties:
-- compatible: must contain "qcom,hidma-1.0" for initial HW or "qcom,hidma-1.1"
-for MSI capable HW.
+- compatible: must contain "qcom,hidma-1.0" for initial HW or
+  "qcom,hidma-1.1"/"qcom,hidma-1.2" for MSI capable HW.
 - reg: Addresses for the transfer and event channel
 - interrupts: Should contain the event interrupt
 - desc-count: Number of asynchronous requests this channel can handle
diff --git a/Documentation/devicetree/bindings/dma/snps-dma.txt b/Documentation/devicetree/bindings/dma/snps-dma.txt
index a122723907ac..99acc712f83a 100644
--- a/Documentation/devicetree/bindings/dma/snps-dma.txt
+++ b/Documentation/devicetree/bindings/dma/snps-dma.txt
@@ -64,6 +64,6 @@ Example:
 		reg = <0xe0000000 0x1000>;
 		interrupts = <0 35 0x4>;
 		dmas = <&dmahost 12 0 1>,
-			<&dmahost 13 0 1 0>;
+			<&dmahost 13 1 0>;
 		dma-names = "rx", "rx";
 	};
diff --git a/Documentation/devicetree/bindings/dma/ste-dma40.txt b/Documentation/devicetree/bindings/dma/ste-dma40.txt
index aa7dbd565ad0..99ab5c4d331e 100644
--- a/Documentation/devicetree/bindings/dma/ste-dma40.txt
+++ b/Documentation/devicetree/bindings/dma/ste-dma40.txt
@@ -15,7 +15,7 @@ Optional properties:
 
 Example:
 
-	dma: dma-controller@801C0000 {
+	dma: dma-controller@801c0000 {
 		compatible = "stericsson,db8500-dma40", "stericsson,dma40";
 		reg = <0x801C0000 0x1000  0x40010000 0x800>;
 		reg-names = "base", "lcpa";
diff --git a/Documentation/devicetree/bindings/eeprom/at24.txt b/Documentation/devicetree/bindings/eeprom/at24.txt
new file mode 100644
index 000000000000..abfae1beca2b
--- /dev/null
+++ b/Documentation/devicetree/bindings/eeprom/at24.txt
@@ -0,0 +1,78 @@
+EEPROMs (I2C)
+
+Required properties:
+
+  - compatible: Must be a "<manufacturer>,<model>" pair. The following <model>
+                values are supported (assuming "atmel" as manufacturer):
+
+                "atmel,24c00",
+                "atmel,24c01",
+                "atmel,24cs01",
+                "atmel,24c02",
+                "atmel,24cs02",
+                "atmel,24mac402",
+                "atmel,24mac602",
+                "atmel,spd",
+                "atmel,24c04",
+                "atmel,24cs04",
+                "atmel,24c08",
+                "atmel,24cs08",
+                "atmel,24c16",
+                "atmel,24cs16",
+                "atmel,24c32",
+                "atmel,24cs32",
+                "atmel,24c64",
+                "atmel,24cs64",
+                "atmel,24c128",
+                "atmel,24c256",
+                "atmel,24c512",
+                "atmel,24c1024",
+
+                If <manufacturer> is not "atmel", then a fallback must be used
+                with the same <model> and "atmel" as manufacturer.
+
+                Example:
+                        compatible = "microchip,24c128", "atmel,24c128";
+
+                Supported manufacturers are:
+
+                "catalyst",
+                "microchip",
+                "nxp",
+                "ramtron",
+                "renesas",
+                "st",
+
+                Some vendors use different model names for chips which are just
+                variants of the above. Known such exceptions are listed below:
+
+                "renesas,r1ex24002" - the fallback is "atmel,24c02"
+
+  - reg: The I2C address of the EEPROM.
+
+Optional properties:
+
+  - pagesize: The length of the pagesize for writing. Please consult the
+              manual of your device, that value varies a lot. A wrong value
+              may result in data loss! If not specified, a safety value of
+              '1' is used which will be very slow.
+
+  - read-only: This parameterless property disables writes to the eeprom.
+
+  - size: Total eeprom size in bytes.
+
+  - no-read-rollover: This parameterless property indicates that the
+                      multi-address eeprom does not automatically roll over
+                      reads to the next slave address. Please consult the
+                      manual of your device.
+
+  - wp-gpios: GPIO to which the write-protect pin of the chip is connected.
+
+Example:
+
+eeprom@52 {
+	compatible = "atmel,24c32";
+	reg = <0x52>;
+	pagesize = <32>;
+	wp-gpios = <&gpio1 3 0>;
+};
diff --git a/Documentation/devicetree/bindings/eeprom/at25.txt b/Documentation/devicetree/bindings/eeprom/at25.txt
index e823d90b802f..b3bde97dc199 100644
--- a/Documentation/devicetree/bindings/eeprom/at25.txt
+++ b/Documentation/devicetree/bindings/eeprom/at25.txt
@@ -11,7 +11,9 @@ Required properties:
 - spi-max-frequency : max spi frequency to use
 - pagesize : size of the eeprom page
 - size : total eeprom size in bytes
-- address-width : number of address bits (one of 8, 16, or 24)
+- address-width : number of address bits (one of 8, 9, 16, or 24).
+  For 9 bits, the MSB of the address is sent as bit 3 of the instruction
+  byte, before the address byte.
 
 Optional properties:
 - spi-cpha : SPI shifted clock phase, as per spi-bus bindings.
diff --git a/Documentation/devicetree/bindings/eeprom/eeprom.txt b/Documentation/devicetree/bindings/eeprom/eeprom.txt
deleted file mode 100644
index 27f2bc15298a..000000000000
--- a/Documentation/devicetree/bindings/eeprom/eeprom.txt
+++ /dev/null
@@ -1,47 +0,0 @@
-EEPROMs (I2C)
-
-Required properties:
-
-  - compatible : should be "<manufacturer>,<type>", like these:
-
-	"atmel,24c00", "atmel,24c01", "atmel,24c02", "atmel,24c04",
-	"atmel,24c08", "atmel,24c16", "atmel,24c32", "atmel,24c64",
-	"atmel,24c128", "atmel,24c256", "atmel,24c512", "atmel,24c1024"
-
-	"catalyst,24c32"
-
-	"microchip,24c128"
-
-	"ramtron,24c64"
-
-	"renesas,r1ex24002"
-
-	The following manufacturers values have been deprecated:
-	"at", "at24"
-
-	 If there is no specific driver for <manufacturer>, a generic
-	 device with <type> and manufacturer "atmel" should be used.
-	 Possible types are:
-	 "24c00", "24c01", "24c02", "24c04", "24c08", "24c16", "24c32", "24c64",
-	 "24c128", "24c256", "24c512", "24c1024", "spd"
-
-  - reg : the I2C address of the EEPROM
-
-Optional properties:
-
-  - pagesize : the length of the pagesize for writing. Please consult the
-               manual of your device, that value varies a lot. A wrong value
-	       may result in data loss! If not specified, a safety value of
-	       '1' is used which will be very slow.
-
-  - read-only: this parameterless property disables writes to the eeprom
-
-  - size: total eeprom size in bytes
-
-Example:
-
-eeprom@52 {
-	compatible = "atmel,24c32";
-	reg = <0x52>;
-	pagesize = <32>;
-};
diff --git a/Documentation/devicetree/bindings/gpio/abilis,tb10x-gpio.txt b/Documentation/devicetree/bindings/gpio/abilis,tb10x-gpio.txt
index 00611aceed3e..a25c87b650e5 100644
--- a/Documentation/devicetree/bindings/gpio/abilis,tb10x-gpio.txt
+++ b/Documentation/devicetree/bindings/gpio/abilis,tb10x-gpio.txt
@@ -21,7 +21,7 @@ Documentation/devicetree/bindings/gpio/gpio.txt
 
 Example:
 
-	gpioa: gpio@FF140000 {
+	gpioa: gpio@ff140000 {
 		compatible = "abilis,tb10x-gpio";
 		interrupt-controller;
 		#interrupt-cells = <1>;
diff --git a/Documentation/devicetree/bindings/gpio/gpio-axp209.txt b/Documentation/devicetree/bindings/gpio/gpio-axp209.txt
index a6611304dd3c..fc42b2caa06d 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-axp209.txt
+++ b/Documentation/devicetree/bindings/gpio/gpio-axp209.txt
@@ -1,10 +1,17 @@
-AXP209 GPIO controller
+AXP209 GPIO & pinctrl controller
 
 This driver follows the usual GPIO bindings found in
 Documentation/devicetree/bindings/gpio/gpio.txt
 
+This driver follows the usual pinctrl bindings found in
+Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
+
+This driver employs the per-pin muxing pattern.
+
 Required properties:
-- compatible: Should be "x-powers,axp209-gpio"
+- compatible: Should be one of:
+	- "x-powers,axp209-gpio"
+	- "x-powers,axp813-gpio"
 - #gpio-cells: Should be two. The first cell is the pin number and the
   second is the GPIO flags.
 - gpio-controller: Marks the device node as a GPIO controller.
@@ -28,3 +35,41 @@ axp209: pmic@34 {
 		#gpio-cells = <2>;
 	};
 };
+
+The GPIOs can be muxed to other functions and therefore, must be a subnode of
+axp_gpio.
+
+Example:
+
+&axp_gpio {
+	gpio0_adc: gpio0-adc {
+		pins = "GPIO0";
+		function = "adc";
+	};
+};
+
+&example_node {
+	pinctrl-names = "default";
+	pinctrl-0 = <&gpio0_adc>;
+};
+
+GPIOs and their functions
+-------------------------
+
+Each GPIO is independent from the other (i.e. GPIO0 in gpio_in function does
+not force GPIO1 and GPIO2 to be in gpio_in function as well).
+
+axp209
+------
+GPIO	|	Functions
+------------------------
+GPIO0	|	gpio_in, gpio_out, ldo, adc
+GPIO1	|	gpio_in, gpio_out, ldo, adc
+GPIO2	|	gpio_in, gpio_out
+
+axp813
+------
+GPIO	|	Functions
+------------------------
+GPIO0	|	gpio_in, gpio_out, ldo, adc
+GPIO1	|	gpio_in, gpio_out, ldo
diff --git a/Documentation/devicetree/bindings/gpio/gpio-stp-xway.txt b/Documentation/devicetree/bindings/gpio/gpio-stp-xway.txt
index 854de130a971..78458adbf4b7 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-stp-xway.txt
+++ b/Documentation/devicetree/bindings/gpio/gpio-stp-xway.txt
@@ -27,7 +27,7 @@ Optional properties:
 
 Example:
 
-gpio1: stp@E100BB0 {
+gpio1: stp@e100bb0 {
 	compatible = "lantiq,gpio-stp-xway";
 	reg = <0xE100BB0 0x40>;
 	#gpio-cells = <2>;
diff --git a/Documentation/devicetree/bindings/gpio/gpio.txt b/Documentation/devicetree/bindings/gpio/gpio.txt
index 802402f6cc5d..b5de08e3b1a2 100644
--- a/Documentation/devicetree/bindings/gpio/gpio.txt
+++ b/Documentation/devicetree/bindings/gpio/gpio.txt
@@ -290,7 +290,7 @@ pins 50..69.
 
 Example 2:
 
-	gpio_pio_i: gpio-controller@14B0 {
+	gpio_pio_i: gpio-controller@14b0 {
 		#gpio-cells = <2>;
 		compatible = "fsl,qe-pario-bank-e", "fsl,qe-pario-bank";
 		reg = <0x1480 0x18>;
diff --git a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt
index a7ac460ad657..9474138d776e 100644
--- a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt
+++ b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt
@@ -5,7 +5,7 @@ Required Properties:
   - compatible: should contain one or more of the following:
     - "renesas,gpio-r8a7743": for R8A7743 (RZ/G1M) compatible GPIO controller.
     - "renesas,gpio-r8a7745": for R8A7745 (RZ/G1E) compatible GPIO controller.
-    - "renesas,gpio-r8a7778": for R8A7778 (R-Mobile M1) compatible GPIO controller.
+    - "renesas,gpio-r8a7778": for R8A7778 (R-Car M1) compatible GPIO controller.
     - "renesas,gpio-r8a7779": for R8A7779 (R-Car H1) compatible GPIO controller.
     - "renesas,gpio-r8a7790": for R8A7790 (R-Car H2) compatible GPIO controller.
     - "renesas,gpio-r8a7791": for R8A7791 (R-Car M2-W) compatible GPIO controller.
diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-utgard.txt b/Documentation/devicetree/bindings/gpu/arm,mali-utgard.txt
index c6814d7cc2b2..ad876548ab5d 100644
--- a/Documentation/devicetree/bindings/gpu/arm,mali-utgard.txt
+++ b/Documentation/devicetree/bindings/gpu/arm,mali-utgard.txt
@@ -17,6 +17,7 @@ Required properties:
       + rockchip,rk3066-mali
       + rockchip,rk3188-mali
       + rockchip,rk3228-mali
+      + rockchip,rk3328-mali
       + stericsson,db8500-mali
 
   - reg: Physical base address and length of the GPU registers
diff --git a/Documentation/devicetree/bindings/hwmon/aspeed-pwm-tacho.txt b/Documentation/devicetree/bindings/hwmon/aspeed-pwm-tacho.txt
index 367c8203213b..3ac02988a1a5 100644
--- a/Documentation/devicetree/bindings/hwmon/aspeed-pwm-tacho.txt
+++ b/Documentation/devicetree/bindings/hwmon/aspeed-pwm-tacho.txt
@@ -22,8 +22,9 @@ Required properties for pwm-tacho node:
 - compatible : should be "aspeed,ast2400-pwm-tacho" for AST2400 and
 	       "aspeed,ast2500-pwm-tacho" for AST2500.
 
-- clocks : a fixed clock providing input clock frequency(PWM
-	   and Fan Tach clock)
+- clocks : phandle to clock provider with the clock number in the second cell
+
+- resets : phandle to reset controller with the reset number in the second cell
 
 fan subnode format:
 ===================
@@ -48,19 +49,14 @@ Required properties for each child node:
 
 Examples:
 
-pwm_tacho_fixed_clk: fixedclk {
-	compatible = "fixed-clock";
-	#clock-cells = <0>;
-	clock-frequency = <24000000>;
-};
-
 pwm_tacho: pwmtachocontroller@1e786000 {
 	#address-cells = <1>;
 	#size-cells = <1>;
 	#cooling-cells = <2>;
 	reg = <0x1E786000 0x1000>;
 	compatible = "aspeed,ast2500-pwm-tacho";
-	clocks = <&pwm_tacho_fixed_clk>;
+	clocks = <&syscon ASPEED_CLK_APB>;
+	resets = <&syscon ASPEED_RESET_PWM>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_pwm0_default &pinctrl_pwm1_default>;
 
diff --git a/Documentation/devicetree/bindings/i2c/i2c-arb-gpio-challenge.txt b/Documentation/devicetree/bindings/i2c/i2c-arb-gpio-challenge.txt
index 248a155414c2..548a73cde796 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-arb-gpio-challenge.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-arb-gpio-challenge.txt
@@ -54,7 +54,7 @@ Optional properties:
 
 
 Example:
-	i2c@12CA0000 {
+	i2c@12ca0000 {
 		compatible = "acme,some-i2c-device";
 		#address-cells = <1>;
 		#size-cells = <0>;
diff --git a/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.txt b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.txt
index 70c054a9a997..60fe90d69f4e 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.txt
@@ -11,7 +11,7 @@ Required properties:
 
 Examples:
 
-lpi2c7: lpi2c7@40A50000 {
+lpi2c7: lpi2c7@40a50000 {
 	compatible = "fsl,imx8dv-lpi2c";
 	reg = <0x40A50000 0x10000>;
 	interrupt-parent = <&intc>;
diff --git a/Documentation/devicetree/bindings/i2c/i2c-meson.txt b/Documentation/devicetree/bindings/i2c/i2c-meson.txt
index 611b934c7e10..13d410de077c 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-meson.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-meson.txt
@@ -1,7 +1,11 @@
 Amlogic Meson I2C controller
 
 Required properties:
- - compatible: must be "amlogic,meson6-i2c" or "amlogic,meson-gxbb-i2c"
+ - compatible: must be:
+	"amlogic,meson6-i2c" for Meson8 and compatible SoCs
+	"amlogic,meson-gxbb-i2c" for GXBB and compatible SoCs
+	"amlogic,meson-axg-i2c"for AXG and compatible SoCs
+
  - reg: physical address and length of the device registers
  - interrupts: a single interrupt specifier
  - clocks: clock for the device
diff --git a/Documentation/devicetree/bindings/i2c/i2c-mtk.txt b/Documentation/devicetree/bindings/i2c/i2c-mtk.txt
index ff7bf37deb43..e199695b1c96 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-mtk.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-mtk.txt
@@ -5,6 +5,7 @@ The MediaTek's I2C controller is used to interface with I2C devices.
 Required properties:
   - compatible: value should be either of the following.
       "mediatek,mt2701-i2c", "mediatek,mt6577-i2c": for MediaTek MT2701
+      "mediatek,mt2712-i2c": for MediaTek MT2712
       "mediatek,mt6577-i2c": for MediaTek MT6577
       "mediatek,mt6589-i2c": for MediaTek MT6589
       "mediatek,mt7622-i2c": for MediaTek MT7622
diff --git a/Documentation/devicetree/bindings/i2c/i2c-mux-pca954x.txt b/Documentation/devicetree/bindings/i2c/i2c-mux-pca954x.txt
index aa097045a10e..34d91501342e 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-mux-pca954x.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-mux-pca954x.txt
@@ -1,10 +1,19 @@
 * NXP PCA954x I2C bus switch
 
+The driver supports NXP PCA954x and PCA984x I2C mux/switch devices.
+
 Required Properties:
 
   - compatible: Must contain one of the following.
-    "nxp,pca9540", "nxp,pca9542", "nxp,pca9543", "nxp,pca9544",
-    "nxp,pca9545", "nxp,pca9546", "nxp,pca9547", "nxp,pca9548"
+    "nxp,pca9540",
+    "nxp,pca9542",
+    "nxp,pca9543",
+    "nxp,pca9544",
+    "nxp,pca9545",
+    "nxp,pca9546", "nxp,pca9846",
+    "nxp,pca9547", "nxp,pca9847",
+    "nxp,pca9548", "nxp,pca9848",
+    "nxp,pca9849"
 
   - reg: The I2C address of the device.
 
diff --git a/Documentation/devicetree/bindings/i2c/i2c-mv64xxx.txt b/Documentation/devicetree/bindings/i2c/i2c-mv64xxx.txt
index 5c30026921ae..0ffe65a316ae 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-mv64xxx.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-mv64xxx.txt
@@ -25,6 +25,15 @@ default frequency is 100kHz
                      whenever you're using the "allwinner,sun6i-a31-i2c"
                      compatible.
 
+ - clocks:	   : pointers to the reference clocks for this device, the
+		     first one is the one used for the clock on the i2c bus,
+		     the second one is the clock used to acces the registers
+		     of the controller
+
+ - clock-names	   : names of used clocks, mandatory if the second clock is
+		     used, the name must be "core", and "reg" (the latter is
+		     only for Armada 7K/8K).
+
 Examples:
 
 	i2c@11000 {
@@ -42,3 +51,14 @@ For the Armada XP:
 		interrupts = <29>;
 		clock-frequency = <100000>;
 	};
+
+For the Armada 7040:
+
+	i2c@701000 {
+		compatible = "marvell,mv78230-i2c";
+		reg = <0x701000 0x20>;
+		interrupts = <29>;
+		clock-frequency = <100000>;
+		clock-names = "core", "reg";
+		clocks = <&core_clock>, <&reg_clock>;
+	};
diff --git a/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt b/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt
index f413e82c8b83..1e6ee3deb4fa 100644
--- a/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt
+++ b/Documentation/devicetree/bindings/iio/adc/amlogic,meson-saradc.txt
@@ -15,7 +15,6 @@ Required properties:
 			- "clkin" for the reference clock (typically XTAL)
 			- "core" for the SAR ADC core clock
 		optional clocks:
-			- "sana" for the analog clock
 			- "adc_clk" for the ADC (sampling) clock
 			- "adc_sel" for the ADC (sampling) clock mux
 - vref-supply:	the regulator supply for the ADC reference voltage
diff --git a/Documentation/devicetree/bindings/iio/adc/aspeed_adc.txt b/Documentation/devicetree/bindings/iio/adc/aspeed_adc.txt
index 674e133b7cd7..034fc2ba100e 100644
--- a/Documentation/devicetree/bindings/iio/adc/aspeed_adc.txt
+++ b/Documentation/devicetree/bindings/iio/adc/aspeed_adc.txt
@@ -8,6 +8,7 @@ Required properties:
 - reg: memory window mapping address and length
 - clocks: Input clock used to derive the sample clock. Expected to be the
           SoC's APB clock.
+- resets: Reset controller phandle
 - #io-channel-cells: Must be set to <1> to indicate channels are selected
                      by index.
 
@@ -15,6 +16,7 @@ Example:
 	adc@1e6e9000 {
 		compatible = "aspeed,ast2400-adc";
 		reg = <0x1e6e9000 0xb0>;
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_ADC>;
 		#io-channel-cells = <1>;
 	};
diff --git a/Documentation/devicetree/bindings/iio/adc/at91-sama5d2_adc.txt b/Documentation/devicetree/bindings/iio/adc/at91-sama5d2_adc.txt
index 552e7a83951d..6469a4cd2a6d 100644
--- a/Documentation/devicetree/bindings/iio/adc/at91-sama5d2_adc.txt
+++ b/Documentation/devicetree/bindings/iio/adc/at91-sama5d2_adc.txt
@@ -17,6 +17,11 @@ Required properties:
   This property uses the IRQ edge types values: IRQ_TYPE_EDGE_RISING ,
   IRQ_TYPE_EDGE_FALLING or IRQ_TYPE_EDGE_BOTH
 
+Optional properties:
+  - dmas: Phandle to dma channel for the ADC.
+  - dma-names: Must be "rx" when dmas property is being used.
+  See ../../dma/dma.txt for details.
+
 Example:
 
 adc: adc@fc030000 {
@@ -31,4 +36,6 @@ adc: adc@fc030000 {
 	vddana-supply = <&vdd_3v3_lp_reg>;
 	vref-supply = <&vdd_3v3_lp_reg>;
 	atmel,trigger-edge-type = <IRQ_TYPE_EDGE_BOTH>;
+	dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(25))>;
+	dma-names = "rx";
 }
diff --git a/Documentation/devicetree/bindings/iio/adc/sigma-delta-modulator.txt b/Documentation/devicetree/bindings/iio/adc/sigma-delta-modulator.txt
new file mode 100644
index 000000000000..ba24ca7ba95e
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/sigma-delta-modulator.txt
@@ -0,0 +1,13 @@
+Device-Tree bindings for sigma delta modulator
+
+Required properties:
+- compatible: should be "ads1201", "sd-modulator". "sd-modulator" can be use
+	as a generic SD modulator if modulator not specified in compatible list.
+- #io-channel-cells = <0>: See the IIO bindings section "IIO consumers".
+
+Example node:
+
+	ads1202: adc@0 {
+		compatible = "sd-modulator";
+		#io-channel-cells = <0>;
+	};
diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
index 48bfcaa3ffcd..e8bb8243e92c 100644
--- a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
+++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.txt
@@ -62,6 +62,15 @@ Required properties:
 - st,adc-channels: List of single-ended channels muxed for this ADC.
   It can have up to 16 channels on stm32f4 or 20 channels on stm32h7, numbered
   from 0 to 15 or 19 (resp. for in0..in15 or in0..in19).
+- st,adc-diff-channels: List of differential channels muxed for this ADC.
+  Depending on part used, some channels can be configured as differential
+  instead of single-ended (e.g. stm32h7). List here positive and negative
+  inputs pairs as <vinp vinn>, <vinp vinn>,... vinp and vinn are numbered
+  from 0 to 19 on stm32h7)
+  Note: At least one of "st,adc-channels" or "st,adc-diff-channels" is required.
+  Both properties can be used together. Some channels can be used as
+  single-ended and some other ones as differential (mixed). But channels
+  can't be configured both as single-ended and differential (invalid).
 - #io-channel-cells = <1>: See the IIO bindings section "IIO consumers" in
   Documentation/devicetree/bindings/iio/iio-bindings.txt
 
@@ -111,3 +120,18 @@ Example:
 		...
 		other adc child nodes follow...
 	};
+
+Example to setup:
+- channel 1 as single-ended
+- channels 2 & 3 as differential (with resp. 6 & 7 negative inputs)
+
+	adc: adc@40022000 {
+		compatible = "st,stm32h7-adc-core";
+		...
+		adc1: adc@0 {
+			compatible = "st,stm32h7-adc";
+			...
+			st,adc-channels = <1>;
+			st,adc-diff-channels = <2 6>, <3 7>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt b/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt
new file mode 100644
index 000000000000..911492da48f3
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.txt
@@ -0,0 +1,128 @@
+STMicroelectronics STM32 DFSDM ADC device driver
+
+
+STM32 DFSDM ADC is a sigma delta analog-to-digital converter dedicated to
+interface external sigma delta modulators to STM32 micro controllers.
+It is mainly targeted for:
+- Sigma delta modulators (motor control, metering...)
+- PDM microphones (audio digital microphone)
+
+It features up to 8 serial digital interfaces (SPI or Manchester) and
+up to 4 filters on stm32h7.
+
+Each child node match with a filter instance.
+
+Contents of a STM32 DFSDM root node:
+------------------------------------
+Required properties:
+- compatible: Should be "st,stm32h7-dfsdm".
+- reg: Offset and length of the DFSDM block register set.
+- clocks: IP and serial interfaces clocking. Should be set according
+		to rcc clock ID and "clock-names".
+- clock-names: Input clock name "dfsdm" must be defined,
+		"audio" is optional. If defined CLKOUT is based on the audio
+		clock, else "dfsdm" is used.
+- #interrupt-cells = <1>;
+- #address-cells = <1>;
+- #size-cells = <0>;
+
+Optional properties:
+- spi-max-frequency: Requested only for SPI master mode.
+		  SPI clock OUT frequency (Hz). This clock must be set according
+		  to "clock" property. Frequency must be a multiple of the rcc
+		  clock frequency. If not, SPI CLKOUT frequency will not be
+		  accurate.
+
+Contents of a STM32 DFSDM child nodes:
+--------------------------------------
+
+Required properties:
+- compatible: Must be:
+	"st,stm32-dfsdm-adc" for sigma delta ADCs
+	"st,stm32-dfsdm-dmic" for audio digital microphone.
+- reg: Specifies the DFSDM filter instance used.
+- interrupts: IRQ lines connected to each DFSDM filter instance.
+- st,adc-channels:	List of single-ended channels muxed for this ADC.
+			valid values:
+				"st,stm32h7-dfsdm" compatibility: 0 to 7.
+- st,adc-channel-names:	List of single-ended channel names.
+- st,filter-order:  SinC filter order from 0 to 5.
+			0: FastSinC
+			[1-5]: order 1 to 5.
+			For audio purpose it is recommended to use order 3 to 5.
+- #io-channel-cells = <1>: See the IIO bindings section "IIO consumers".
+
+Required properties for "st,stm32-dfsdm-adc" compatibility:
+- io-channels: From common IIO binding. Used to pipe external sigma delta
+		modulator or internal ADC output to DFSDM channel.
+		This is not required for "st,stm32-dfsdm-pdm" compatibility as
+		PDM microphone is binded in Audio DT node.
+
+Required properties for "st,stm32-dfsdm-pdm" compatibility:
+- #sound-dai-cells: Must be set to 0.
+- dma: DMA controller phandle and DMA request line associated to the
+		filter instance (specified by the field "reg")
+- dma-names: Must be "rx"
+
+Optional properties:
+- st,adc-channel-types:	Single-ended channel input type.
+			- "SPI_R": SPI with data on rising edge (default)
+			- "SPI_F": SPI with data on falling edge
+			- "MANCH_R": manchester codec, rising edge = logic 0
+			- "MANCH_F": manchester codec, falling edge = logic 1
+- st,adc-channel-clk-src: Conversion clock source.
+			  - "CLKIN": external SPI clock (CLKIN x)
+			  - "CLKOUT": internal SPI clock (CLKOUT) (default)
+			  - "CLKOUT_F": internal SPI clock divided by 2 (falling edge).
+			  - "CLKOUT_R": internal SPI clock divided by 2 (rising edge).
+
+- st,adc-alt-channel: Must be defined if two sigma delta modulator are
+			  connected on same SPI input.
+			  If not set, channel n is connected to SPI input n.
+			  If set, channel n is connected to SPI input n + 1.
+
+- st,filter0-sync: Set to 1 to synchronize with DFSDM filter instance 0.
+		   Used for multi microphones synchronization.
+
+Example of a sigma delta adc connected on DFSDM SPI port 0
+and a pdm microphone connected on DFSDM SPI port 1:
+
+	ads1202: simple_sd_adc@0 {
+		compatible = "ads1202";
+		#io-channel-cells = <1>;
+	};
+
+	dfsdm: dfsdm@40017000 {
+		compatible = "st,stm32h7-dfsdm";
+		reg = <0x40017000 0x400>;
+		clocks = <&rcc DFSDM1_CK>;
+		clock-names = "dfsdm";
+		#interrupt-cells = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		dfsdm_adc0: filter@0 {
+			compatible = "st,stm32-dfsdm-adc";
+			#io-channel-cells = <1>;
+			reg = <0>;
+			interrupts = <110>;
+			st,adc-channels = <0>;
+			st,adc-channel-names = "sd_adc0";
+			st,adc-channel-types = "SPI_F";
+			st,adc-channel-clk-src = "CLKOUT";
+			io-channels = <&ads1202 0>;
+			st,filter-order = <3>;
+		};
+		dfsdm_pdm1: filter@1 {
+			compatible = "st,stm32-dfsdm-dmic";
+			reg = <1>;
+			interrupts = <111>;
+			dmas = <&dmamux1 102 0x400 0x00>;
+			dma-names = "rx";
+			st,adc-channels = <1>;
+			st,adc-channel-names = "dmic1";
+			st,adc-channel-types = "SPI_R";
+			st,adc-channel-clk-src = "CLKOUT";
+			st,filter-order = <5>;
+		};
+	}
diff --git a/Documentation/devicetree/bindings/iio/health/max30102.txt b/Documentation/devicetree/bindings/iio/health/max30102.txt
index 8629c18b0e78..ef2ca0a0306f 100644
--- a/Documentation/devicetree/bindings/iio/health/max30102.txt
+++ b/Documentation/devicetree/bindings/iio/health/max30102.txt
@@ -1,9 +1,11 @@
 Maxim MAX30102 heart rate and pulse oximeter sensor
+Maxim MAX30105 optical particle-sensing module
 
 * https://datasheets.maximintegrated.com/en/ds/MAX30102.pdf
+* https://datasheets.maximintegrated.com/en/ds/MAX30105.pdf
 
 Required properties:
-  - compatible: must be "maxim,max30102"
+  - compatible: must be "maxim,max30102" or "maxim,max30105"
   - reg: the I2C address of the sensor
   - interrupt-parent: should be the phandle for the interrupt controller
   - interrupts: the sole interrupt generated by the device
@@ -12,8 +14,10 @@ Required properties:
   interrupt client node bindings.
 
 Optional properties:
-  - maxim,red-led-current-microamp: configuration for RED LED current
+  - maxim,red-led-current-microamp: configuration for red LED current
   - maxim,ir-led-current-microamp: configuration for IR LED current
+  - maxim,green-led-current-microamp: configuration for green LED current
+    (max30105 only)
 
     Note that each step is approximately 200 microamps, ranging from 0 uA to
     50800 uA.
diff --git a/Documentation/devicetree/bindings/iio/light/uvis25.txt b/Documentation/devicetree/bindings/iio/light/uvis25.txt
new file mode 100644
index 000000000000..3041207e3f3c
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/light/uvis25.txt
@@ -0,0 +1,23 @@
+* ST UVIS25 uv sensor
+
+Required properties:
+- compatible: should be "st,uvis25"
+- reg: i2c address of the sensor / spi cs line
+
+Optional properties:
+- interrupt-parent: should be the phandle for the interrupt controller
+- interrupts: interrupt mapping for IRQ. It should be configured with
+  flags IRQ_TYPE_LEVEL_HIGH, IRQ_TYPE_EDGE_RISING, IRQ_TYPE_LEVEL_LOW or
+  IRQ_TYPE_EDGE_FALLING.
+
+  Refer to interrupt-controller/interrupts.txt for generic interrupt
+  client node bindings.
+
+Example:
+
+uvis25@47 {
+	compatible = "st,uvis25";
+	reg = <0x47>;
+	interrupt-parent = <&gpio0>;
+	interrupts = <0 IRQ_TYPE_EDGE_RISING>;
+};
diff --git a/Documentation/devicetree/bindings/input/hid-over-i2c.txt b/Documentation/devicetree/bindings/input/hid-over-i2c.txt
index 28e8bd8b7d64..4d3da9d91de4 100644
--- a/Documentation/devicetree/bindings/input/hid-over-i2c.txt
+++ b/Documentation/devicetree/bindings/input/hid-over-i2c.txt
@@ -31,7 +31,7 @@ device-specific compatible properties, which should be used in addition to the
 
 - vdd-supply: phandle of the regulator that provides the supply voltage.
 - post-power-on-delay-ms: time required by the device after enabling its regulators
-  before it is ready for communication. Must be used with 'vdd-supply'.
+  or powering it on, before it is ready for communication.
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/input/samsung-keypad.txt b/Documentation/devicetree/bindings/input/samsung-keypad.txt
index 5305e74e5742..4c5c0a82586d 100644
--- a/Documentation/devicetree/bindings/input/samsung-keypad.txt
+++ b/Documentation/devicetree/bindings/input/samsung-keypad.txt
@@ -45,7 +45,7 @@ Optional Properties specific to linux:
 
 
 Example:
-	keypad@100A0000 {
+	keypad@100a0000 {
 		compatible = "samsung,s5pv210-keypad";
 		reg = <0x100A0000 0x100>;
 		interrupts = <173>;
diff --git a/Documentation/devicetree/bindings/input/touchscreen/brcm,iproc-touchscreen.txt b/Documentation/devicetree/bindings/input/touchscreen/brcm,iproc-touchscreen.txt
index ac5dff412e25..f127a2117072 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/brcm,iproc-touchscreen.txt
+++ b/Documentation/devicetree/bindings/input/touchscreen/brcm,iproc-touchscreen.txt
@@ -66,7 +66,7 @@ Example: An example of touchscreen node
 		reg = <0x180a6000 0xc30>;
 	};
 
-	touchscreen: touchscreen@180A6000 {
+	touchscreen: touchscreen@180a6000 {
 		compatible = "brcm,iproc-touchscreen";
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/Documentation/devicetree/bindings/input/touchscreen/mms114.txt b/Documentation/devicetree/bindings/input/touchscreen/mms114.txt
index 89d4c56c5671..2cd954051d29 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/mms114.txt
+++ b/Documentation/devicetree/bindings/input/touchscreen/mms114.txt
@@ -1,17 +1,23 @@
-* MELFAS MMS114 touchscreen controller
+* MELFAS MMS114/MMS152 touchscreen controller
 
 Required properties:
-- compatible: must be "melfas,mms114"
+- compatible: should be one of:
+	- "melfas,mms114"
+	- "melfas,mms152"
 - reg: I2C address of the chip
 - interrupts: interrupt to which the chip is connected
-- x-size: horizontal resolution of touchscreen
-- y-size: vertical resolution of touchscreen
+- touchscreen-size-x: See [1]
+- touchscreen-size-y: See [1]
 
 Optional properties:
-- contact-threshold:
-- moving-threshold:
-- x-invert: invert X axis
-- y-invert: invert Y axis
+- touchscreen-fuzz-x: See [1]
+- touchscreen-fuzz-y: See [1]
+- touchscreen-fuzz-pressure: See [1]
+- touchscreen-inverted-x: See [1]
+- touchscreen-inverted-y: See [1]
+- touchscreen-swapped-x-y: See [1]
+
+[1]: Documentation/devicetree/bindings/input/touchscreen/touchscreen.txt
 
 Example:
 
@@ -22,12 +28,13 @@ Example:
 			compatible = "melfas,mms114";
 			reg = <0x48>;
 			interrupts = <39 0>;
-			x-size = <720>;
-			y-size = <1280>;
-			contact-threshold = <10>;
-			moving-threshold = <10>;
-			x-invert;
-			y-invert;
+			touchscreen-size-x = <720>;
+			touchscreen-size-y = <1280>;
+			touchscreen-fuzz-x = <10>;
+			touchscreen-fuzz-y = <10>;
+			touchscreen-fuzz-pressure = <10>;
+			touchscreen-inverted-x;
+			touchscreen-inverted-y;
 		};
 
 		/* ... */
diff --git a/Documentation/devicetree/bindings/input/touchscreen/silead_gsl1680.txt b/Documentation/devicetree/bindings/input/touchscreen/silead_gsl1680.txt
index 6aa625e0cb8d..84752de12412 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/silead_gsl1680.txt
+++ b/Documentation/devicetree/bindings/input/touchscreen/silead_gsl1680.txt
@@ -23,6 +23,8 @@ Optional properties:
 - touchscreen-inverted-y  : See touchscreen.txt
 - touchscreen-swapped-x-y : See touchscreen.txt
 - silead,max-fingers	  : maximum number of fingers the touchscreen can detect
+- silead,home-button	  : Boolean, set to true on devices which have a
+			    capacitive home-button build into the touchscreen
 - vddio-supply		  : regulator phandle for controller VDDIO
 - avdd-supply		  : regulator phandle for controller AVDD
 
diff --git a/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2836-l1-intc.txt b/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2836-l1-intc.txt
index f320dcd6e69b..8ced1696c325 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2836-l1-intc.txt
+++ b/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2836-l1-intc.txt
@@ -12,7 +12,7 @@ Required properties:
 			  registers
 - interrupt-controller:	Identifies the node as an interrupt controller
 - #interrupt-cells:	Specifies the number of cells needed to encode an
-			  interrupt source. The value shall be 1
+			  interrupt source. The value shall be 2
 
 Please refer to interrupts.txt in this directory for details of the common
 Interrupt Controllers bindings used by client devices.
@@ -32,6 +32,6 @@ local_intc: local_intc {
 	compatible = "brcm,bcm2836-l1-intc";
 	reg = <0x40000000 0x100>;
 	interrupt-controller;
-	#interrupt-cells = <1>;
+	#interrupt-cells = <2>;
 	interrupt-parent = <&local_intc>;
 };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/google,goldfish-pic.txt b/Documentation/devicetree/bindings/interrupt-controller/google,goldfish-pic.txt
new file mode 100644
index 000000000000..35f752706e7d
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/google,goldfish-pic.txt
@@ -0,0 +1,30 @@
+Android Goldfish PIC
+
+Android Goldfish programmable interrupt device used by Android
+emulator.
+
+Required properties:
+
+- compatible : should contain "google,goldfish-pic"
+- reg        : <registers mapping>
+- interrupts : <interrupt mapping>
+
+Example for mips when used in cascade mode:
+
+        cpuintc {
+                #interrupt-cells = <0x1>;
+                #address-cells = <0>;
+                interrupt-controller;
+                compatible = "mti,cpu-interrupt-controller";
+        };
+
+        interrupt-controller@1f000000 {
+                compatible = "google,goldfish-pic";
+                reg = <0x1f000000 0x1000>;
+
+                interrupt-controller;
+                #interrupt-cells = <0x1>;
+
+                interrupt-parent = <&cpuintc>;
+                interrupts = <0x2>;
+        };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.txt b/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.txt
index 33c9a10fdc91..20f121daa910 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.txt
+++ b/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.txt
@@ -14,6 +14,7 @@ Required properties:
     - "renesas,irqc-r8a7794" (R-Car E2)
     - "renesas,intc-ex-r8a7795" (R-Car H3)
     - "renesas,intc-ex-r8a7796" (R-Car M3-W)
+    - "renesas,intc-ex-r8a77965" (R-Car M3-N)
     - "renesas,intc-ex-r8a77970" (R-Car V3M)
     - "renesas,intc-ex-r8a77995" (R-Car D3)
 - #interrupt-cells: has to be <2>: an interrupt index and flags, as defined in
diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
index 857df929a654..1fd5d69647ca 100644
--- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
+++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt
@@ -16,6 +16,9 @@ Required Properties:
     - "renesas,ipmmu-r8a7793" for the R8A7793 (R-Car M2-N) IPMMU.
     - "renesas,ipmmu-r8a7794" for the R8A7794 (R-Car E2) IPMMU.
     - "renesas,ipmmu-r8a7795" for the R8A7795 (R-Car H3) IPMMU.
+    - "renesas,ipmmu-r8a7796" for the R8A7796 (R-Car M3-W) IPMMU.
+    - "renesas,ipmmu-r8a77970" for the R8A77970 (R-Car V3M) IPMMU.
+    - "renesas,ipmmu-r8a77995" for the R8A77995 (R-Car D3) IPMMU.
     - "renesas,ipmmu-vmsa" for generic R-Car Gen2 VMSA-compatible IPMMU.
 
   - reg: Base address and size of the IPMMU registers.
diff --git a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt
index 85f068805dd8..b1682c80b490 100644
--- a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt
+++ b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt
@@ -56,7 +56,7 @@ Examples:
 		iommus = <&sysmmu_gsc0>;
 	};
 
-	sysmmu_gsc0: sysmmu@13E80000 {
+	sysmmu_gsc0: sysmmu@13e80000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x13E80000 0x1000>;
 		interrupt-parent = <&combiner>;
diff --git a/Documentation/devicetree/bindings/leds/leds-lm3692x.txt b/Documentation/devicetree/bindings/leds/leds-lm3692x.txt
new file mode 100644
index 000000000000..6c9074f84a51
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/leds-lm3692x.txt
@@ -0,0 +1,49 @@
+* Texas Instruments - LM3692x Highly Efficient White LED Driver
+
+The LM3692x is an ultra-compact, highly efficient,
+white-LED driver designed for LCD display backlighting.
+
+The main difference between the LM36922 and LM36923 is the number of
+LED strings it supports.  The LM36922 supports two strings while the LM36923
+supports three strings.
+
+Required properties:
+	- compatible:
+		"ti,lm36922"
+		"ti,lm36923"
+	- reg :  I2C slave address
+	- #address-cells : 1
+	- #size-cells : 0
+
+Optional properties:
+	- enable-gpios : gpio pin to enable/disable the device.
+	- vled-supply : LED supply
+
+Required child properties:
+	- reg : 0
+
+Optional child properties:
+	- label : see Documentation/devicetree/bindings/leds/common.txt
+	- linux,default-trigger :
+	   see Documentation/devicetree/bindings/leds/common.txt
+
+Example:
+
+led-controller@36 {
+	compatible = "ti,lm3692x";
+	reg = <0x36>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	enable-gpios = <&gpio1 28 GPIO_ACTIVE_HIGH>;
+	vled-supply = <&vbatt>;
+
+	led@0 {
+		reg = <0>;
+		label = "white:backlight_cluster";
+		linux,default-trigger = "backlight";
+	};
+}
+
+For more product information please see the link below:
+http://www.ti.com/lit/ds/snvsa29/snvsa29.pdf
diff --git a/Documentation/devicetree/bindings/leds/leds-lp8860.txt b/Documentation/devicetree/bindings/leds/leds-lp8860.txt
index aad38dd94d4b..5f0e892ad759 100644
--- a/Documentation/devicetree/bindings/leds/leds-lp8860.txt
+++ b/Documentation/devicetree/bindings/leds/leds-lp8860.txt
@@ -6,23 +6,39 @@ current sinks that can be controlled by a PWM input
 signal, a SPI/I2C master, or both.
 
 Required properties:
-	- compatible:
+	- compatible :
 		"ti,lp8860"
-	- reg -  I2C slave address
-	- label - Used for naming LEDs
+	- reg : I2C slave address
+	- #address-cells : 1
+	- #size-cells : 0
 
 Optional properties:
-	- enable-gpio - gpio pin to enable/disable the device.
-	- supply - "vled" - LED supply
+	- enable-gpios : gpio pin to enable (active high)/disable the device.
+	- vled-supply : LED supply
+
+Required child properties:
+	- reg : 0
+
+Optional child properties:
+	- label : see Documentation/devicetree/bindings/leds/common.txt
+	- linux,default-trigger :
+	   see Documentation/devicetree/bindings/leds/common.txt
 
 Example:
 
-leds: leds@6 {
+led-controller@2d {
 	compatible = "ti,lp8860";
+	#address-cells = <1>;
+	#size-cells = <0>;
 	reg = <0x2d>;
-	label = "display_cluster";
-	enable-gpio = <&gpio1 28 GPIO_ACTIVE_HIGH>;
+	enable-gpios = <&gpio1 28 GPIO_ACTIVE_HIGH>;
 	vled-supply = <&vbatt>;
+
+	led@0 {
+		reg = <0>;
+		label = "white:backlight";
+		linux,default-trigger = "backlight";
+	};
 }
 
 For more product information please see the link below:
diff --git a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
index 9b40c4925aa9..0ef372656a3e 100644
--- a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
+++ b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt
@@ -124,7 +124,7 @@ dsp {
 };
 
 /* AM33xx */
-mailbox: mailbox@480C8000 {
+mailbox: mailbox@480c8000 {
 	compatible = "ti,omap4-mailbox";
 	reg = <0x480C8000 0x200>;
 	interrupts = <77>;
diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.txt b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.txt
index fb961c310f44..16964f0c1773 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.txt
+++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.txt
@@ -15,12 +15,21 @@ platforms.
 	Usage: required
 	Value type: <prop-encoded-array>
 	Definition: must specify the base address and size of the global block
+- clocks:
+	Usage: required if #clocks-cells property is present
+	Value type: <phandle>
+	Definition: phandle to the input PLL, which feeds the APCS mux/divider
 
 - #mbox-cells:
 	Usage: required
 	Value type: <u32>
 	Definition: as described in mailbox.txt, must be 1
 
+- #clock-cells:
+	Usage: optional
+	Value type: <u32>
+	Definition: as described in clock.txt, must be 0
+
 
 = EXAMPLE
 The following example describes the APCS HMSS found in MSM8996 and part of the
@@ -44,3 +53,12 @@ GLINK RPM referencing the "rpm_hlos" doorbell therein.
 		mbox-names = "rpm_hlos";
 	};
 
+Below is another example of the APCS binding on MSM8916 platforms:
+
+	apcs: mailbox@b011000 {
+		compatible = "qcom,msm8916-apcs-kpss-global";
+		reg = <0xb011000 0x1000>;
+		#mbox-cells = <1>;
+		clocks = <&a53pll>;
+		#clock-cells = <0>;
+	};
diff --git a/Documentation/devicetree/bindings/mailbox/ti,message-manager.txt b/Documentation/devicetree/bindings/mailbox/ti,message-manager.txt
index c3b55b3ede8a..ebf0e3710cee 100644
--- a/Documentation/devicetree/bindings/mailbox/ti,message-manager.txt
+++ b/Documentation/devicetree/bindings/mailbox/ti,message-manager.txt
@@ -20,9 +20,9 @@ Required properties:
 		        order referring to the transfer path.
 - interrupt-names:	Contains interrupt names matching the rx transfer path
 			for a given SoC. Receive interrupts shall be of the
-			format: "rx_<QID>_<PID>".
+			format: "rx_<QID>".
 			For ti,k2g-message-manager, this shall contain:
-				"rx_005_002", "rx_057_002"
+				"rx_005", "rx_057"
 - interrupts:		Contains the interrupt information corresponding to
 			interrupt-names property.
 
diff --git a/Documentation/devicetree/bindings/media/cec-gpio.txt b/Documentation/devicetree/bindings/media/cec-gpio.txt
index 46a0bac8b3b9..12fcd55ed153 100644
--- a/Documentation/devicetree/bindings/media/cec-gpio.txt
+++ b/Documentation/devicetree/bindings/media/cec-gpio.txt
@@ -4,6 +4,10 @@ The HDMI CEC GPIO module supports CEC implementations where the CEC line
 is hooked up to a pull-up GPIO line and - optionally - the HPD line is
 hooked up to another GPIO line.
 
+Please note: the maximum voltage for the CEC line is 3.63V, for the HPD
+line it is 5.3V. So you may need some sort of level conversion circuitry
+when connecting them to a GPIO line.
+
 Required properties:
   - compatible: value must be "cec-gpio".
   - cec-gpios: gpio that the CEC line is connected to. The line should be
@@ -21,7 +25,7 @@ the following property is optional:
 
 Example for the Raspberry Pi 3 where the CEC line is connected to
 pin 26 aka BCM7 aka CE1 on the GPIO pin header and the HPD line is
-connected to pin 11 aka BCM17:
+connected to pin 11 aka BCM17 (some level shifter is needed for this!):
 
 #include <dt-bindings/gpio/gpio.h>
 
diff --git a/Documentation/devicetree/bindings/media/i2c/mt9m111.txt b/Documentation/devicetree/bindings/media/i2c/mt9m111.txt
index ed5a334b1e57..6b910036b57e 100644
--- a/Documentation/devicetree/bindings/media/i2c/mt9m111.txt
+++ b/Documentation/devicetree/bindings/media/i2c/mt9m111.txt
@@ -6,6 +6,8 @@ interface.
 
 Required Properties:
 - compatible: value should be "micron,mt9m111"
+- clocks: reference to the master clock.
+- clock-names: shall be "mclk".
 
 For further reading on port node refer to
 Documentation/devicetree/bindings/media/video-interfaces.txt.
@@ -16,6 +18,8 @@ Example:
 		mt9m111@5d {
 			compatible = "micron,mt9m111";
 			reg = <0x5d>;
+			clocks = <&mclk>;
+			clock-names = "mclk";
 
 			remote = <&pxa_camera>;
 			port {
diff --git a/Documentation/devicetree/bindings/media/i2c/ov5640.txt b/Documentation/devicetree/bindings/media/i2c/ov5640.txt
index 540b36c4b1f2..8e36da0d8406 100644
--- a/Documentation/devicetree/bindings/media/i2c/ov5640.txt
+++ b/Documentation/devicetree/bindings/media/i2c/ov5640.txt
@@ -1,4 +1,4 @@
-* Omnivision OV5640 MIPI CSI-2 sensor
+* Omnivision OV5640 MIPI CSI-2 / parallel sensor
 
 Required Properties:
 - compatible: should be "ovti,ov5640"
@@ -18,7 +18,25 @@ The device node must contain one 'port' child node for its digital output
 video port, in accordance with the video interface bindings defined in
 Documentation/devicetree/bindings/media/video-interfaces.txt.
 
-Example:
+OV5640 can be connected to a MIPI CSI-2 bus or a parallel bus endpoint.
+
+Endpoint node required properties for CSI-2 connection are:
+- remote-endpoint: a phandle to the bus receiver's endpoint node.
+- clock-lanes: should be set to <0> (clock lane on hardware lane 0)
+- data-lanes: should be set to <1> or <1 2> (one or two CSI-2 lanes supported)
+
+Endpoint node required properties for parallel connection are:
+- remote-endpoint: a phandle to the bus receiver's endpoint node.
+- bus-width: shall be set to <8> for 8 bits parallel bus
+	     or <10> for 10 bits parallel bus
+- data-shift: shall be set to <2> for 8 bits parallel bus
+	      (lines 9:2 are used) or <0> for 10 bits parallel bus
+- hsync-active: active state of the HSYNC signal, 0/1 for LOW/HIGH respectively.
+- vsync-active: active state of the VSYNC signal, 0/1 for LOW/HIGH respectively.
+- pclk-sample: sample data on rising (1) or falling (0) edge of the pixel clock
+	       signal.
+
+Examples:
 
 &i2c1 {
 	ov5640: camera@3c {
@@ -35,6 +53,7 @@ Example:
 		reset-gpios = <&gpio1 20 GPIO_ACTIVE_LOW>;
 
 		port {
+			/* MIPI CSI-2 bus endpoint */
 			ov5640_to_mipi_csi2: endpoint {
 				remote-endpoint = <&mipi_csi2_from_ov5640>;
 				clock-lanes = <0>;
@@ -43,3 +62,26 @@ Example:
 		};
 	};
 };
+
+&i2c1 {
+	ov5640: camera@3c {
+		compatible = "ovti,ov5640";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ov5640>;
+		reg = <0x3c>;
+		clocks = <&clk_ext_camera>;
+		clock-names = "xclk";
+
+		port {
+			/* Parallel bus endpoint */
+			ov5640_to_parallel: endpoint {
+				remote-endpoint = <&parallel_from_ov5640>;
+				bus-width = <8>;
+				data-shift = <2>; /* lines 9:2 are used */
+				hsync-active = <0>;
+				vsync-active = <0>;
+				pclk-sample = <1>;
+			};
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/media/i2c/ov7740.txt b/Documentation/devicetree/bindings/media/i2c/ov7740.txt
new file mode 100644
index 000000000000..af781c3a5f0e
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/ov7740.txt
@@ -0,0 +1,47 @@
+* Omnivision OV7740 CMOS image sensor
+
+The Omnivision OV7740 image sensor supports multiple output image
+size, such as VGA, and QVGA, CIF and any size smaller. It also
+supports the RAW RGB and YUV output formats.
+
+The common video interfaces bindings (see video-interfaces.txt) should
+be used to specify link to the image data receiver. The OV7740 device
+node should contain one 'port' child node with an 'endpoint' subnode.
+
+Required Properties:
+- compatible:	"ovti,ov7740".
+- reg:		I2C slave address of the sensor.
+- clocks:	Reference to the xvclk input clock.
+- clock-names:	"xvclk".
+
+Optional Properties:
+- reset-gpios: Rreference to the GPIO connected to the reset_b pin,
+  if any. Active low with pull-ip resistor.
+- powerdown-gpios: Reference to the GPIO connected to the pwdn pin,
+  if any. Active high with pull-down resistor.
+
+Endpoint node mandatory properties:
+- remote-endpoint: A phandle to the bus receiver's endpoint node.
+
+Example:
+
+	i2c1: i2c@fc028000 {
+		ov7740: camera@21 {
+			compatible = "ovti,ov7740";
+			reg = <0x21>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_sensor_power &pinctrl_sensor_reset>;
+			clocks = <&isc>;
+			clock-names = "xvclk";
+			assigned-clocks = <&isc>;
+			assigned-clock-rates = <24000000>;
+			reset-gpios = <&pioA 43 GPIO_ACTIVE_LOW>;
+			powerdown-gpios = <&pioA 44 GPIO_ACTIVE_HIGH>;
+
+			port {
+				ov7740_0: endpoint {
+					remote-endpoint = <&isc_0>;
+				};
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/media/i2c/toshiba,et8ek8.txt b/Documentation/devicetree/bindings/media/i2c/toshiba,et8ek8.txt
index 0b7b6a4d84ff..e80d5891b7ed 100644
--- a/Documentation/devicetree/bindings/media/i2c/toshiba,et8ek8.txt
+++ b/Documentation/devicetree/bindings/media/i2c/toshiba,et8ek8.txt
@@ -20,6 +20,13 @@ Mandatory properties
   is in hardware standby mode when the signal is in the low state.
 
 
+Optional properties
+-------------------
+
+- flash-leds: See ../video-interfaces.txt
+- lens-focus: See ../video-interfaces.txt
+
+
 Endpoint node mandatory properties
 ----------------------------------
 
diff --git a/Documentation/devicetree/bindings/media/nvidia,tegra-vde.txt b/Documentation/devicetree/bindings/media/nvidia,tegra-vde.txt
new file mode 100644
index 000000000000..470237ed6fe5
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/nvidia,tegra-vde.txt
@@ -0,0 +1,55 @@
+NVIDIA Tegra Video Decoder Engine
+
+Required properties:
+- compatible : Must contain one of the following values:
+   - "nvidia,tegra20-vde"
+   - "nvidia,tegra30-vde"
+   - "nvidia,tegra114-vde"
+   - "nvidia,tegra124-vde"
+   - "nvidia,tegra132-vde"
+- reg : Must contain an entry for each entry in reg-names.
+- reg-names : Must include the following entries:
+  - sxe
+  - bsev
+  - mbe
+  - ppe
+  - mce
+  - tfe
+  - ppb
+  - vdma
+  - frameid
+- iram : Must contain phandle to the mmio-sram device node that represents
+         IRAM region used by VDE.
+- interrupts : Must contain an entry for each entry in interrupt-names.
+- interrupt-names : Must include the following entries:
+  - sync-token
+  - bsev
+  - sxe
+- clocks : Must include the following entries:
+  - vde
+- resets : Must include the following entries:
+  - vde
+
+Example:
+
+video-codec@6001a000 {
+	compatible = "nvidia,tegra20-vde";
+	reg = <0x6001a000 0x1000 /* Syntax Engine */
+	       0x6001b000 0x1000 /* Video Bitstream Engine */
+	       0x6001c000  0x100 /* Macroblock Engine */
+	       0x6001c200  0x100 /* Post-processing Engine */
+	       0x6001c400  0x100 /* Motion Compensation Engine */
+	       0x6001c600  0x100 /* Transform Engine */
+	       0x6001c800  0x100 /* Pixel prediction block */
+	       0x6001ca00  0x100 /* Video DMA */
+	       0x6001d800  0x300 /* Video frame controls */>;
+	reg-names = "sxe", "bsev", "mbe", "ppe", "mce",
+		    "tfe", "ppb", "vdma", "frameid";
+	iram = <&vde_pool>; /* IRAM region */
+	interrupts = <GIC_SPI  9 IRQ_TYPE_LEVEL_HIGH>, /* Sync token interrupt */
+		     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, /* BSE-V interrupt */
+		     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>; /* SXE interrupt */
+	interrupt-names = "sync-token", "bsev", "sxe";
+	clocks = <&tegra_car TEGRA20_CLK_VDE>;
+	resets = <&tegra_car 61>;
+};
diff --git a/Documentation/devicetree/bindings/media/s5p-cec.txt b/Documentation/devicetree/bindings/media/s5p-cec.txt
index 6f3756da900f..e847291d4aff 100644
--- a/Documentation/devicetree/bindings/media/s5p-cec.txt
+++ b/Documentation/devicetree/bindings/media/s5p-cec.txt
@@ -23,7 +23,7 @@ Optional:
 
 Example:
 
-hdmicec: cec@100B0000 {
+hdmicec: cec@100b0000 {
 	compatible = "samsung,s5p-cec";
 	reg = <0x100B0000 0x200>;
 	interrupts = <0 114 0>;
diff --git a/Documentation/devicetree/bindings/media/samsung-s5c73m3.txt b/Documentation/devicetree/bindings/media/samsung-s5c73m3.txt
index 2c85c4538a6d..21f31fdf5543 100644
--- a/Documentation/devicetree/bindings/media/samsung-s5c73m3.txt
+++ b/Documentation/devicetree/bindings/media/samsung-s5c73m3.txt
@@ -62,7 +62,7 @@ For more details see description of the SPI busses bindings
 
 Example:
 
-i2c@138A000000 {
+i2c@138a000000 {
 	...
 	s5c73m3@3c {
 		compatible = "samsung,s5c73m3";
diff --git a/Documentation/devicetree/bindings/media/xilinx/xlnx,v-tpg.txt b/Documentation/devicetree/bindings/media/xilinx/xlnx,v-tpg.txt
index 9dd86b3db937..439351ab2a79 100644
--- a/Documentation/devicetree/bindings/media/xilinx/xlnx,v-tpg.txt
+++ b/Documentation/devicetree/bindings/media/xilinx/xlnx,v-tpg.txt
@@ -66,6 +66,6 @@ Example:
 				tpg1_out: endpoint {
 					remote-endpoint = <&switch_in0>;
 				};
-			}:
+			};
 		};
 	};
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt
index 8dbe47013c2b..14968b048cd3 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra30-mc.txt
@@ -12,6 +12,8 @@ Required properties:
 - clock-names: Must include the following entries:
   - mc: the module's clock input
 - interrupts: The interrupt outputs from the controller.
+
+Required properties for Tegra30, Tegra114, Tegra124, Tegra132 and Tegra210:
 - #iommu-cells: Should be 1. The single cell of the IOMMU specifier defines
   the SWGROUP of the master.
 
diff --git a/Documentation/devicetree/bindings/memory-controllers/ti-aemif.txt b/Documentation/devicetree/bindings/memory-controllers/ti-aemif.txt
index 9592717f483f..190437a0c146 100644
--- a/Documentation/devicetree/bindings/memory-controllers/ti-aemif.txt
+++ b/Documentation/devicetree/bindings/memory-controllers/ti-aemif.txt
@@ -138,7 +138,7 @@ from the corresponding HW reg.
 
 Example for aemif, davinci nand and nor flash chip select shown below.
 
-memory-controller@21000A00 {
+memory-controller@21000a00 {
 	compatible = "ti,davinci-aemif";
 	#address-cells = <2>;
 	#size-cells = <1>;
diff --git a/Documentation/devicetree/bindings/memory-controllers/ti/emif.txt b/Documentation/devicetree/bindings/memory-controllers/ti/emif.txt
index 152eeccbde1c..621b41c79faa 100644
--- a/Documentation/devicetree/bindings/memory-controllers/ti/emif.txt
+++ b/Documentation/devicetree/bindings/memory-controllers/ti/emif.txt
@@ -23,6 +23,13 @@ Required properties:
   the value shall be "emif<n>" where <n> is the number of the EMIF
   instance with base 1.
 
+Required only for "ti,emif-am3352" and "ti,emif-am4372":
+- sram			: Phandles for generic sram driver nodes,
+  first should be type 'protect-exec' for the driver to use to copy
+  and run PM functions, second should be regular pool to be used for
+  data region for code. See Documentation/devicetree/bindings/sram/sram.txt
+  for more details.
+
 Optional properties:
 - cs1-used		: Have this property if CS1 of this EMIF
   instance has a memory part attached to it. If there is a memory
@@ -44,7 +51,7 @@ Optional properties:
 - hw-caps-temp-alert	: Have this property if the controller
   has capability for generating SDRAM temperature alerts
 
-Example:
+-Examples:
 
 emif1: emif@4c000000 {
 	compatible	= "ti,emif-4d";
@@ -56,3 +63,11 @@ emif1: emif@4c000000 {
 	hw-caps-ll-interface;
 	hw-caps-temp-alert;
 };
+
+/* From am33xx.dtsi */
+emif: emif@4c000000 {
+        compatible = "ti,emif-am3352";
+        reg =   <0x4C000000 0x1000>;
+        sram = <&pm_sram_code
+                &pm_sram_data>;
+};
diff --git a/Documentation/devicetree/bindings/mfd/atmel-tcb.txt b/Documentation/devicetree/bindings/mfd/atmel-tcb.txt
new file mode 100644
index 000000000000..c4a83e364cb6
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/atmel-tcb.txt
@@ -0,0 +1,56 @@
+* Device tree bindings for Atmel Timer Counter Blocks
+- compatible: Should be "atmel,<chip>-tcb", "simple-mfd", "syscon".
+  <chip> can be "at91rm9200" or "at91sam9x5"
+- reg: Should contain registers location and length
+- #address-cells: has to be 1
+- #size-cells: has to be 0
+- interrupts: Should contain all interrupts for the TC block
+  Note that you can specify several interrupt cells if the TC
+  block has one interrupt per channel.
+- clock-names: tuple listing input clock names.
+	Required elements: "t0_clk", "slow_clk"
+	Optional elements: "t1_clk", "t2_clk"
+- clocks: phandles to input clocks.
+
+The TCB can expose multiple subdevices:
+ * a timer
+   - compatible: Should be "atmel,tcb-timer"
+   - reg: Should contain the TCB channels to be used. If the
+     counter width is 16 bits (at91rm9200-tcb), two consecutive
+     channels are needed. Else, only one channel will be used.
+
+Examples:
+
+One interrupt per TC block:
+	tcb0: timer@fff7c000 {
+		compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0xfff7c000 0x100>;
+		interrupts = <18 4>;
+		clocks = <&tcb0_clk>, <&clk32k>;
+		clock-names = "t0_clk", "slow_clk";
+
+		timer@0 {
+			compatible = "atmel,tcb-timer";
+			reg = <0>, <1>;
+		};
+
+		timer@2 {
+			compatible = "atmel,tcb-timer";
+			reg = <2>;
+		};
+	};
+
+One interrupt per TC channel in a TC block:
+	tcb1: timer@fffdc000 {
+		compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0xfffdc000 0x100>;
+		interrupts = <26 4>, <27 4>, <28 4>;
+		clocks = <&tcb1_clk>, <&clk32k>;
+		clock-names = "t0_clk", "slow_clk";
+	};
+
+
diff --git a/Documentation/devicetree/bindings/mfd/cros-ec.txt b/Documentation/devicetree/bindings/mfd/cros-ec.txt
index 136e0c2da44d..6245c9b1a68b 100644
--- a/Documentation/devicetree/bindings/mfd/cros-ec.txt
+++ b/Documentation/devicetree/bindings/mfd/cros-ec.txt
@@ -41,7 +41,7 @@ Optional properties (all):
 
 Example for I2C:
 
-i2c@12CA0000 {
+i2c@12ca0000 {
 	cros-ec@1e {
 		reg = <0x1e>;
 		compatible = "google,cros-ec-i2c";
diff --git a/Documentation/devicetree/bindings/mfd/mc13xxx.txt b/Documentation/devicetree/bindings/mfd/mc13xxx.txt
index ac235fe385fc..8261ea73278a 100644
--- a/Documentation/devicetree/bindings/mfd/mc13xxx.txt
+++ b/Documentation/devicetree/bindings/mfd/mc13xxx.txt
@@ -130,7 +130,7 @@ ecspi@70010000 { /* ECSPI1 */
 			#size-cells = <0>;
 			led-control = <0x000 0x000 0x0e0 0x000>;
 
-			sysled {
+			sysled@3 {
 				reg = <3>;
 				label = "system:red:live";
 				linux,default-trigger = "heartbeat";
diff --git a/Documentation/devicetree/bindings/mfd/syscon.txt b/Documentation/devicetree/bindings/mfd/syscon.txt
index 8b92d4576c42..25d9e9c2fd53 100644
--- a/Documentation/devicetree/bindings/mfd/syscon.txt
+++ b/Documentation/devicetree/bindings/mfd/syscon.txt
@@ -16,9 +16,17 @@ Required properties:
 Optional property:
 - reg-io-width: the size (in bytes) of the IO accesses that should be
   performed on the device.
+- hwlocks: reference to a phandle of a hardware spinlock provider node.
 
 Examples:
 gpr: iomuxc-gpr@20e0000 {
 	compatible = "fsl,imx6q-iomuxc-gpr", "syscon";
 	reg = <0x020e0000 0x38>;
+	hwlocks = <&hwlock1 1>;
+};
+
+hwlock1: hwspinlock@40500000 {
+	...
+	reg = <0x40500000 0x1000>;
+	#hwlock-cells = <1>;
 };
diff --git a/Documentation/devicetree/bindings/misc/nvidia,tegra186-misc.txt b/Documentation/devicetree/bindings/misc/nvidia,tegra186-misc.txt
new file mode 100644
index 000000000000..892ba4384abc
--- /dev/null
+++ b/Documentation/devicetree/bindings/misc/nvidia,tegra186-misc.txt
@@ -0,0 +1,12 @@
+NVIDIA Tegra186 MISC register block
+
+The MISC register block found on Tegra186 SoCs contains registers that can be
+used to identify a given chip and various strapping options.
+
+Required properties:
+- compatible: Must be:
+  - Tegra186: "nvidia,tegra186-misc"
+- reg: Should contain 2 entries: The first entry gives the physical address
+       and length of the register region which contains revision and debug
+       features. The second entry specifies the physical address and length
+       of the register region indicating the strapping options.
diff --git a/Documentation/devicetree/bindings/mmc/mmc.txt b/Documentation/devicetree/bindings/mmc/mmc.txt
index fb11ae8b3b72..467cd7b147ce 100644
--- a/Documentation/devicetree/bindings/mmc/mmc.txt
+++ b/Documentation/devicetree/bindings/mmc/mmc.txt
@@ -67,10 +67,10 @@ logic applies to the "wp-inverted" property.
 CD and WP lines can be implemented on the hardware in one of two ways: as GPIOs,
 specified in cd-gpios and wp-gpios properties, or as dedicated pins. Polarity of
 dedicated pins can be specified, using *-inverted properties. GPIO polarity can
-also be specified using the OF_GPIO_ACTIVE_LOW flag. This creates an ambiguity
+also be specified using the GPIO_ACTIVE_LOW flag. This creates an ambiguity
 in the latter case. We choose to use the XOR logic for GPIO CD and WP lines.
 This means, the two properties are "superimposed," for example leaving the
-OF_GPIO_ACTIVE_LOW flag clear and specifying the respective *-inverted
+GPIO_ACTIVE_LOW flag clear and specifying the respective *-inverted property
 property results in a double-inversion and actually means the "normal" line
 polarity is in effect.
 
diff --git a/Documentation/devicetree/bindings/mmc/mtk-sd.txt b/Documentation/devicetree/bindings/mmc/mtk-sd.txt
index 72d2a734ab85..9b8017670870 100644
--- a/Documentation/devicetree/bindings/mmc/mtk-sd.txt
+++ b/Documentation/devicetree/bindings/mmc/mtk-sd.txt
@@ -12,6 +12,8 @@ Required properties:
 	"mediatek,mt8173-mmc": for mmc host ip compatible with mt8173
 	"mediatek,mt2701-mmc": for mmc host ip compatible with mt2701
 	"mediatek,mt2712-mmc": for mmc host ip compatible with mt2712
+	"mediatek,mt7623-mmc", "mediatek,mt2701-mmc": for MT7623 SoC
+
 - reg: physical base address of the controller and length
 - interrupts: Should contain MSDC interrupt number
 - clocks: Should contain phandle for the clock feeding the MMC controller
diff --git a/Documentation/devicetree/bindings/mmc/tmio_mmc.txt b/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
index 3c6762430fd9..d8685cb83325 100644
--- a/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
+++ b/Documentation/devicetree/bindings/mmc/tmio_mmc.txt
@@ -26,6 +26,7 @@ Required properties:
 		"renesas,sdhi-r8a7794" - SDHI IP on R8A7794 SoC
 		"renesas,sdhi-r8a7795" - SDHI IP on R8A7795 SoC
 		"renesas,sdhi-r8a7796" - SDHI IP on R8A7796 SoC
+		"renesas,sdhi-r8a77995" - SDHI IP on R8A77995 SoC
 		"renesas,sdhi-shmobile" - a generic sh-mobile SDHI controller
 		"renesas,rcar-gen1-sdhi" - a generic R-Car Gen1 SDHI controller
 		"renesas,rcar-gen2-sdhi" - a generic R-Car Gen2 or RZ/G1
diff --git a/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt b/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt
index c34aa6f8a424..63d4d626fbd5 100644
--- a/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt
+++ b/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt
@@ -12,7 +12,7 @@ Required properties:
   - reg-names: Should contain the reg names "QuadSPI" and "QuadSPI-memory"
   - interrupts : Should contain the interrupt for the device
   - clocks : The clocks needed by the QuadSPI controller
-  - clock-names : the name of the clocks
+  - clock-names : Should contain the name of the clocks: "qspi_en" and "qspi".
 
 Optional properties:
   - fsl,qspi-has-second-chip: The controller has two buses, bus A and bus B.
diff --git a/Documentation/devicetree/bindings/mtd/gpmc-onenand.txt b/Documentation/devicetree/bindings/mtd/gpmc-onenand.txt
index b6e8bfd024f4..e9f01a963a0a 100644
--- a/Documentation/devicetree/bindings/mtd/gpmc-onenand.txt
+++ b/Documentation/devicetree/bindings/mtd/gpmc-onenand.txt
@@ -9,13 +9,14 @@ Documentation/devicetree/bindings/memory-controllers/omap-gpmc.txt
 
 Required properties:
 
+ - compatible:		"ti,omap2-onenand"
  - reg:			The CS line the peripheral is connected to
- - gpmc,device-width	Width of the ONENAND device connected to the GPMC
+ - gpmc,device-width:	Width of the ONENAND device connected to the GPMC
 			in bytes. Must be 1 or 2.
 
 Optional properties:
 
- - dma-channel:		DMA Channel index
+ - int-gpios:		GPIO specifier for the INT pin.
 
 For inline partition table parsing (optional):
 
@@ -35,6 +36,7 @@ Example for an OMAP3430 board:
 		#size-cells = <1>;
 
 		onenand@0 {
+			compatible = "ti,omap2-onenand";
 			reg = <0 0 0>; /* CS0, offset 0 */
 			gpmc,device-width = <2>;
 
diff --git a/Documentation/devicetree/bindings/mtd/lpc32xx-mlc.txt b/Documentation/devicetree/bindings/mtd/lpc32xx-mlc.txt
index d0a37252eb22..6d60bc3063f5 100644
--- a/Documentation/devicetree/bindings/mtd/lpc32xx-mlc.txt
+++ b/Documentation/devicetree/bindings/mtd/lpc32xx-mlc.txt
@@ -23,7 +23,7 @@ Optional subnodes:
 
 Example:
 
-	mlc: flash@200A8000 {
+	mlc: flash@200a8000 {
 		compatible = "nxp,lpc3220-mlc";
 		reg = <0x200A8000 0x11000>;
 		interrupts = <11 0>;
diff --git a/Documentation/devicetree/bindings/mtd/marvell-nand.txt b/Documentation/devicetree/bindings/mtd/marvell-nand.txt
new file mode 100644
index 000000000000..c08fb477b3c6
--- /dev/null
+++ b/Documentation/devicetree/bindings/mtd/marvell-nand.txt
@@ -0,0 +1,123 @@
+Marvell NAND Flash Controller (NFC)
+
+Required properties:
+- compatible: can be one of the following:
+    * "marvell,armada-8k-nand-controller"
+    * "marvell,armada370-nand-controller"
+    * "marvell,pxa3xx-nand-controller"
+    * "marvell,armada-8k-nand" (deprecated)
+    * "marvell,armada370-nand" (deprecated)
+    * "marvell,pxa3xx-nand" (deprecated)
+  Compatibles marked deprecated support only the old bindings described
+  at the bottom.
+- reg: NAND flash controller memory area.
+- #address-cells: shall be set to 1. Encode the NAND CS.
+- #size-cells: shall be set to 0.
+- interrupts: shall define the NAND controller interrupt.
+- clocks: shall reference the NAND controller clock.
+- marvell,system-controller: Set to retrieve the syscon node that handles
+  NAND controller related registers (only required with the
+  "marvell,armada-8k-nand[-controller]" compatibles).
+
+Optional properties:
+- label: see partition.txt. New platforms shall omit this property.
+- dmas: shall reference DMA channel associated to the NAND controller.
+  This property is only used with "marvell,pxa3xx-nand[-controller]"
+  compatible strings.
+- dma-names: shall be "rxtx".
+  This property is only used with "marvell,pxa3xx-nand[-controller]"
+  compatible strings.
+
+Optional children nodes:
+Children nodes represent the available NAND chips.
+
+Required properties:
+- reg: shall contain the native Chip Select ids (0-3).
+- nand-rb: see nand.txt (0-1).
+
+Optional properties:
+- marvell,nand-keep-config: orders the driver not to take the timings
+  from the core and leaving them completely untouched. Bootloader
+  timings will then be used.
+- label: MTD name.
+- nand-on-flash-bbt: see nand.txt.
+- nand-ecc-mode: see nand.txt. Will use hardware ECC if not specified.
+- nand-ecc-algo: see nand.txt. This property is essentially useful when
+  not using hardware ECC. Howerver, it may be added when using hardware
+  ECC for clarification but will be ignored by the driver because ECC
+  mode is chosen depending on the page size and the strength required by
+  the NAND chip. This value may be overwritten with nand-ecc-strength
+  property.
+- nand-ecc-strength: see nand.txt.
+- nand-ecc-step-size: see nand.txt. Marvell's NAND flash controller does
+  use fixed strength (1-bit for Hamming, 16-bit for BCH), so the actual
+  step size will shrink or grow in order to fit the required strength.
+  Step sizes are not completely random for all and follow certain
+  patterns described in AN-379, "Marvell SoC NFC ECC".
+
+See Documentation/devicetree/bindings/mtd/nand.txt for more details on
+generic bindings.
+
+
+Example:
+nand_controller: nand-controller@d0000 {
+	compatible = "marvell,armada370-nand-controller";
+	reg = <0xd0000 0x54>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
+	clocks = <&coredivclk 0>;
+
+	nand@0 {
+		reg = <0>;
+		label = "main-storage";
+		nand-rb = <0>;
+		nand-ecc-mode = "hw";
+		marvell,nand-keep-config;
+		nand-on-flash-bbt;
+		nand-ecc-strength = <4>;
+		nand-ecc-step-size = <512>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "Rootfs";
+				reg = <0x00000000 0x40000000>;
+			};
+		};
+	};
+};
+
+
+Note on legacy bindings: One can find, in not-updated device trees,
+bindings slightly different than described above with other properties
+described below as well as the partitions node at the root of a so
+called "nand" node (without clear controller/chip separation).
+
+Legacy properties:
+- marvell,nand-enable-arbiter: To enable the arbiter, all boards blindly
+  used it, this bit was set by the bootloader for many boards and even if
+  it is marked reserved in several datasheets, it might be needed to set
+  it (otherwise it is harmless) so whether or not this property is set,
+  the bit is selected by the driver.
+- num-cs: Number of chip-select lines to use, all boards blindly set 1
+  to this and for a reason, other values would have failed. The value of
+  this property is ignored.
+
+Example:
+
+	nand0: nand@43100000 {
+		compatible = "marvell,pxa3xx-nand";
+		reg = <0x43100000 90>;
+		interrupts = <45>;
+		dmas = <&pdma 97 0>;
+		dma-names = "rxtx";
+		#address-cells = <1>;
+		marvell,nand-keep-config;
+		marvell,nand-enable-arbiter;
+		num-cs = <1>;
+		/* Partitions (optional) */
+       };
diff --git a/Documentation/devicetree/bindings/mtd/mtk-nand.txt b/Documentation/devicetree/bindings/mtd/mtk-nand.txt
index 0431841de781..1c88526dedfc 100644
--- a/Documentation/devicetree/bindings/mtd/mtk-nand.txt
+++ b/Documentation/devicetree/bindings/mtd/mtk-nand.txt
@@ -12,8 +12,10 @@ tree nodes.
 
 The first part of NFC is NAND Controller Interface (NFI) HW.
 Required NFI properties:
-- compatible:			Should be one of "mediatek,mt2701-nfc",
-				"mediatek,mt2712-nfc".
+- compatible:			Should be one of
+				"mediatek,mt2701-nfc",
+				"mediatek,mt2712-nfc",
+				"mediatek,mt7622-nfc".
 - reg:				Base physical address and size of NFI.
 - interrupts:			Interrupts of NFI.
 - clocks:			NFI required clocks.
@@ -142,7 +144,10 @@ Example:
 ==============
 
 Required BCH properties:
-- compatible:	Should be one of "mediatek,mt2701-ecc", "mediatek,mt2712-ecc".
+- compatible:	Should be one of
+		"mediatek,mt2701-ecc",
+		"mediatek,mt2712-ecc",
+		"mediatek,mt7622-ecc".
 - reg:		Base physical address and size of ECC.
 - interrupts:	Interrupts of ECC.
 - clocks:	ECC required clocks.
diff --git a/Documentation/devicetree/bindings/mtd/nand.txt b/Documentation/devicetree/bindings/mtd/nand.txt
index 133f3813719c..8bb11d809429 100644
--- a/Documentation/devicetree/bindings/mtd/nand.txt
+++ b/Documentation/devicetree/bindings/mtd/nand.txt
@@ -43,6 +43,7 @@ Optional NAND chip properties:
 		     This is particularly useful when only the in-band area is
 		     used by the upper layers, and you want to make your NAND
 		     as reliable as possible.
+- nand-rb: shall contain the native Ready/Busy ids.
 
 The ECC strength and ECC step size properties define the correction capability
 of a controller. Together, they say a controller can correct "{strength} bit
diff --git a/Documentation/devicetree/bindings/net/brcm,bcm7445-switch-v4.0.txt b/Documentation/devicetree/bindings/net/brcm,bcm7445-switch-v4.0.txt
index 9a734d808aa7..b7336b9d6a3c 100644
--- a/Documentation/devicetree/bindings/net/brcm,bcm7445-switch-v4.0.txt
+++ b/Documentation/devicetree/bindings/net/brcm,bcm7445-switch-v4.0.txt
@@ -2,7 +2,10 @@
 
 Required properties:
 
-- compatible: should be "brcm,bcm7445-switch-v4.0" or "brcm,bcm7278-switch-v4.0"
+- compatible: should be one of
+	"brcm,bcm7445-switch-v4.0"
+	"brcm,bcm7278-switch-v4.0"
+	"brcm,bcm7278-switch-v4.8"
 - reg: addresses and length of the register sets for the device, must be 6
   pairs of register addresses and lengths
 - interrupts: interrupts for the devices, must be two interrupts
diff --git a/Documentation/devicetree/bindings/net/can/can-transceiver.txt b/Documentation/devicetree/bindings/net/can/can-transceiver.txt
new file mode 100644
index 000000000000..0011f53ff159
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/can/can-transceiver.txt
@@ -0,0 +1,24 @@
+Generic CAN transceiver Device Tree binding
+------------------------------
+
+CAN transceiver typically limits the max speed in standard CAN and CAN FD
+modes. Typically these limitations are static and the transceivers themselves
+provide no way to detect this limitation at runtime. For this situation,
+the "can-transceiver" node can be used.
+
+Required Properties:
+ max-bitrate:	a positive non 0 value that determines the max
+		speed that CAN/CAN-FD can run. Any other value
+		will be ignored.
+
+Examples:
+
+Based on Texas Instrument's TCAN1042HGV CAN Transceiver
+
+m_can0 {
+	....
+	can-transceiver {
+		max-bitrate = <5000000>;
+	};
+	...
+};
diff --git a/Documentation/devicetree/bindings/net/can/fsl-flexcan.txt b/Documentation/devicetree/bindings/net/can/fsl-flexcan.txt
index 56d6cc336e1c..bfc0c433654f 100644
--- a/Documentation/devicetree/bindings/net/can/fsl-flexcan.txt
+++ b/Documentation/devicetree/bindings/net/can/fsl-flexcan.txt
@@ -18,6 +18,12 @@ Optional properties:
 
 - xceiver-supply: Regulator that powers the CAN transceiver
 
+- big-endian: This means the registers of FlexCAN controller are big endian.
+              This is optional property.i.e. if this property is not present in
+              device tree node then controller is assumed to be little endian.
+              if this property is present then controller is assumed to be big
+              endian.
+
 Example:
 
 	can@1c000 {
diff --git a/Documentation/devicetree/bindings/net/can/m_can.txt b/Documentation/devicetree/bindings/net/can/m_can.txt
index 63e90421d029..ed614383af9c 100644
--- a/Documentation/devicetree/bindings/net/can/m_can.txt
+++ b/Documentation/devicetree/bindings/net/can/m_can.txt
@@ -43,6 +43,11 @@ Required properties:
 			  Please refer to 2.4.1 Message RAM Configuration in
 			  Bosch M_CAN user manual for details.
 
+Optional Subnode:
+- can-transceiver	: Can-transceiver subnode describing maximum speed
+			  that can be used for CAN/CAN-FD modes. See
+			  Documentation/devicetree/bindings/net/can/can-transceiver.txt
+			  for details.
 Example:
 SoC dtsi:
 m_can1: can@20e8000 {
@@ -63,4 +68,8 @@ Board dts:
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_m_can1>;
 	status = "enabled";
+
+	can-transceiver {
+		max-bitrate = <5000000>;
+	};
 };
diff --git a/Documentation/devicetree/bindings/net/can/rcar_can.txt b/Documentation/devicetree/bindings/net/can/rcar_can.txt
index 06bb7cc334c8..94a7f33ac5e9 100644
--- a/Documentation/devicetree/bindings/net/can/rcar_can.txt
+++ b/Documentation/devicetree/bindings/net/can/rcar_can.txt
@@ -2,7 +2,9 @@ Renesas R-Car CAN controller Device Tree Bindings
 -------------------------------------------------
 
 Required properties:
-- compatible: "renesas,can-r8a7778" if CAN controller is a part of R8A7778 SoC.
+- compatible: "renesas,can-r8a7743" if CAN controller is a part of R8A7743 SoC.
+	      "renesas,can-r8a7745" if CAN controller is a part of R8A7745 SoC.
+	      "renesas,can-r8a7778" if CAN controller is a part of R8A7778 SoC.
 	      "renesas,can-r8a7779" if CAN controller is a part of R8A7779 SoC.
 	      "renesas,can-r8a7790" if CAN controller is a part of R8A7790 SoC.
 	      "renesas,can-r8a7791" if CAN controller is a part of R8A7791 SoC.
@@ -12,7 +14,8 @@ Required properties:
 	      "renesas,can-r8a7795" if CAN controller is a part of R8A7795 SoC.
 	      "renesas,can-r8a7796" if CAN controller is a part of R8A7796 SoC.
 	      "renesas,rcar-gen1-can" for a generic R-Car Gen1 compatible device.
-	      "renesas,rcar-gen2-can" for a generic R-Car Gen2 compatible device.
+	      "renesas,rcar-gen2-can" for a generic R-Car Gen2 or RZ/G1
+	      compatible device.
 	      "renesas,rcar-gen3-can" for a generic R-Car Gen3 compatible device.
 	      When compatible with the generic version, nodes must list the
 	      SoC-specific version corresponding to the platform first
diff --git a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.txt b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.txt
new file mode 100644
index 000000000000..6c559981d110
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.txt
@@ -0,0 +1,92 @@
+Cortina Systems Gemini Ethernet Controller
+==========================================
+
+This ethernet controller is found in the Gemini SoC family:
+StorLink SL3512 and SL3516, also known as Cortina Systems
+CS3512 and CS3516.
+
+Required properties:
+- compatible: must be "cortina,gemini-ethernet"
+- reg: must contain the global registers and the V-bit and A-bit
+  memory areas, in total three register sets.
+- syscon: a phandle to the system controller
+- #address-cells: must be specified, must be <1>
+- #size-cells: must be specified, must be <1>
+- ranges: should be state like this giving a 1:1 address translation
+  for the subnodes
+
+The subnodes represents the two ethernet ports in this device.
+They are not independent of each other since they share resources
+in the parent node, and are thus children.
+
+Required subnodes:
+- port0: contains the resources for ethernet port 0
+- port1: contains the resources for ethernet port 1
+
+Required subnode properties:
+- compatible: must be "cortina,gemini-ethernet-port"
+- reg: must contain two register areas: the DMA/TOE memory and
+  the GMAC memory area of the port
+- interrupts: should contain the interrupt line of the port.
+  this is nominally a level interrupt active high.
+- resets: this must provide an SoC-integrated reset line for
+  the port.
+- clocks: this should contain a handle to the PCLK clock for
+  clocking the silicon in this port
+- clock-names: must be "PCLK"
+
+Optional subnode properties:
+- phy-mode: see ethernet.txt
+- phy-handle: see ethernet.txt
+
+Example:
+
+mdio-bus {
+	(...)
+	phy0: ethernet-phy@1 {
+		reg = <1>;
+		device_type = "ethernet-phy";
+	};
+	phy1: ethernet-phy@3 {
+		reg = <3>;
+		device_type = "ethernet-phy";
+	};
+};
+
+
+ethernet@60000000 {
+	compatible = "cortina,gemini-ethernet";
+	reg = <0x60000000 0x4000>, /* Global registers, queue */
+	      <0x60004000 0x2000>, /* V-bit */
+	      <0x60006000 0x2000>; /* A-bit */
+	syscon = <&syscon>;
+	#address-cells = <1>;
+	#size-cells = <1>;
+	ranges;
+
+	gmac0: ethernet-port@0 {
+		compatible = "cortina,gemini-ethernet-port";
+		reg = <0x60008000 0x2000>, /* Port 0 DMA/TOE */
+		      <0x6000a000 0x2000>; /* Port 0 GMAC */
+		interrupt-parent = <&intcon>;
+		interrupts = <1 IRQ_TYPE_LEVEL_HIGH>;
+		resets = <&syscon GEMINI_RESET_GMAC0>;
+		clocks = <&syscon GEMINI_CLK_GATE_GMAC0>;
+		clock-names = "PCLK";
+		phy-mode = "rgmii";
+		phy-handle = <&phy0>;
+	};
+
+	gmac1: ethernet-port@1 {
+		compatible = "cortina,gemini-ethernet-port";
+		reg = <0x6000c000 0x2000>, /* Port 1 DMA/TOE */
+		      <0x6000e000 0x2000>; /* Port 1 GMAC */
+		interrupt-parent = <&intcon>;
+		interrupts = <2 IRQ_TYPE_LEVEL_HIGH>;
+		resets = <&syscon GEMINI_RESET_GMAC1>;
+		clocks = <&syscon GEMINI_CLK_GATE_GMAC1>;
+		clock-names = "PCLK";
+		phy-mode = "rgmii";
+		phy-handle = <&phy1>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/net/cpsw.txt b/Documentation/devicetree/bindings/net/cpsw.txt
index 7cc15c96ea95..4cb4925a28ab 100644
--- a/Documentation/devicetree/bindings/net/cpsw.txt
+++ b/Documentation/devicetree/bindings/net/cpsw.txt
@@ -61,7 +61,7 @@ file.
 
 Examples:
 
-	mac: ethernet@4A100000 {
+	mac: ethernet@4a100000 {
 		compatible = "ti,cpsw";
 		reg = <0x4A100000 0x1000>;
 		interrupts = <55 0x4>;
@@ -91,7 +91,7 @@ Examples:
 	};
 
 (or)
-	mac: ethernet@4A100000 {
+	mac: ethernet@4a100000 {
 		compatible = "ti,cpsw";
 		ti,hwmods = "cpgmac0";
 		cpdma_channels = <8>;
diff --git a/Documentation/devicetree/bindings/net/davinci-mdio.txt b/Documentation/devicetree/bindings/net/davinci-mdio.txt
index 621156ca4ffd..e6527de80f10 100644
--- a/Documentation/devicetree/bindings/net/davinci-mdio.txt
+++ b/Documentation/devicetree/bindings/net/davinci-mdio.txt
@@ -21,7 +21,7 @@ file.
 
 Examples:
 
-	mdio: davinci_mdio@4A101000 {
+	mdio: davinci_mdio@4a101000 {
 		compatible = "ti,davinci_mdio";
 		reg = <0x4A101000 0x1000>;
 		bus_freq = <1000000>;
@@ -29,7 +29,7 @@ Examples:
 
 (or)
 
-	mdio: davinci_mdio@4A101000 {
+	mdio: davinci_mdio@4a101000 {
 		compatible = "ti,davinci_mdio";
 		ti,hwmods = "davinci_mdio";
 		bus_freq = <1000000>;
diff --git a/Documentation/devicetree/bindings/net/dsa/marvell.txt b/Documentation/devicetree/bindings/net/dsa/marvell.txt
index 1d4d0f49c9d0..8c033d48e2ba 100644
--- a/Documentation/devicetree/bindings/net/dsa/marvell.txt
+++ b/Documentation/devicetree/bindings/net/dsa/marvell.txt
@@ -50,14 +50,15 @@ Example:
 			compatible = "marvell,mv88e6085";
 			reg = <0>;
 			reset-gpios = <&gpio5 1 GPIO_ACTIVE_LOW>;
-		};
-		mdio {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			switch1phy0: switch1phy0@0 {
-				reg = <0>;
-				interrupt-parent = <&switch0>;
-				interrupts = <0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mdio {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				switch1phy0: switch1phy0@0 {
+					reg = <0>;
+					interrupt-parent = <&switch0>;
+					interrupts = <0 IRQ_TYPE_LEVEL_HIGH>;
+				};
 			};
 		};
 	};
@@ -74,23 +75,24 @@ Example:
 			compatible = "marvell,mv88e6390";
 			reg = <0>;
 			reset-gpios = <&gpio5 1 GPIO_ACTIVE_LOW>;
-		};
-		mdio {
-			#address-cells = <1>;
-			#size-cells = <0>;
-			switch1phy0: switch1phy0@0 {
-				reg = <0>;
-				interrupt-parent = <&switch0>;
-				interrupts = <0 IRQ_TYPE_LEVEL_HIGH>;
+
+			mdio {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				switch1phy0: switch1phy0@0 {
+					reg = <0>;
+					interrupt-parent = <&switch0>;
+					interrupts = <0 IRQ_TYPE_LEVEL_HIGH>;
+				};
 			};
-		};
 
-		mdio1 {
-			compatible = "marvell,mv88e6xxx-mdio-external";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			switch1phy9: switch1phy0@9 {
-				reg = <9>;
+			mdio1 {
+				compatible = "marvell,mv88e6xxx-mdio-external";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				switch1phy9: switch1phy0@9 {
+					reg = <9>;
+				};
 			};
 		};
 	};
diff --git a/Documentation/devicetree/bindings/net/fsl-fec.txt b/Documentation/devicetree/bindings/net/fsl-fec.txt
index f0dc94409107..2d41fb96ce0a 100644
--- a/Documentation/devicetree/bindings/net/fsl-fec.txt
+++ b/Documentation/devicetree/bindings/net/fsl-fec.txt
@@ -59,7 +59,7 @@ ethernet@83fec000 {
 	reg = <0x83fec000 0x4000>;
 	interrupts = <87>;
 	phy-mode = "mii";
-	phy-reset-gpios = <&gpio2 14 0>; /* GPIO2_14 */
+	phy-reset-gpios = <&gpio2 14 GPIO_ACTIVE_LOW>; /* GPIO2_14 */
 	local-mac-address = [00 04 9F 01 1B B9];
 	phy-supply = <&reg_fec_supply>;
 };
@@ -71,7 +71,7 @@ ethernet@83fec000 {
 	reg = <0x83fec000 0x4000>;
 	interrupts = <87>;
 	phy-mode = "mii";
-	phy-reset-gpios = <&gpio2 14 0>; /* GPIO2_14 */
+	phy-reset-gpios = <&gpio2 14 GPIO_ACTIVE_LOW>; /* GPIO2_14 */
 	local-mac-address = [00 04 9F 01 1B B9];
 	phy-supply = <&reg_fec_supply>;
 	phy-handle = <&ethphy>;
diff --git a/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt b/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt
index db74f0dc290c..594982c6b9f9 100644
--- a/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt
+++ b/Documentation/devicetree/bindings/net/fsl-tsec-phy.txt
@@ -136,7 +136,7 @@ Clock Properties:
 
 Example:
 
-	ptp_clock@24E00 {
+	ptp_clock@24e00 {
 		compatible = "fsl,etsec-ptp";
 		reg = <0x24E00 0xB0>;
 		interrupts = <12 0x8 13 0x8>;
diff --git a/Documentation/devicetree/bindings/net/ieee802154/adf7242.txt b/Documentation/devicetree/bindings/net/ieee802154/adf7242.txt
index dea5124cdc52..d24172cc6d32 100644
--- a/Documentation/devicetree/bindings/net/ieee802154/adf7242.txt
+++ b/Documentation/devicetree/bindings/net/ieee802154/adf7242.txt
@@ -1,7 +1,7 @@
 * ADF7242 IEEE 802.15.4 *
 
 Required properties:
-  - compatible:		should be "adi,adf7242"
+  - compatible:		should be "adi,adf7242", "adi,adf7241"
   - spi-max-frequency:	maximal bus speed (12.5 MHz)
   - reg:		the chipselect index
   - interrupts:		the interrupt generated by the device via pin IRQ1.
diff --git a/Documentation/devicetree/bindings/net/mediatek-net.txt b/Documentation/devicetree/bindings/net/mediatek-net.txt
index 214eaa9a6683..53c13ee384a4 100644
--- a/Documentation/devicetree/bindings/net/mediatek-net.txt
+++ b/Documentation/devicetree/bindings/net/mediatek-net.txt
@@ -28,7 +28,7 @@ Required properties:
 - mediatek,sgmiisys: phandle to the syscon node that handles the SGMII setup
 	which is required for those SoCs equipped with SGMII such as MT7622 SoC.
 - mediatek,pctl: phandle to the syscon node that handles the ports slew rate
-	and driver current
+	and driver current: only for MT2701 and MT7623 SoC
 
 Optional properties:
 - interrupt-parent: Should be the phandle for the interrupt controller
diff --git a/Documentation/devicetree/bindings/net/phy.txt b/Documentation/devicetree/bindings/net/phy.txt
index 77d0b2a61ffa..d2169a56f5e3 100644
--- a/Documentation/devicetree/bindings/net/phy.txt
+++ b/Documentation/devicetree/bindings/net/phy.txt
@@ -53,6 +53,14 @@ Optional Properties:
   to ensure the integrated PHY is used. The absence of this property indicates
   the muxers should be configured so that the external PHY is used.
 
+- reset-gpios: The GPIO phandle and specifier for the PHY reset signal.
+
+- reset-assert-us: Delay after the reset was asserted in microseconds.
+  If this property is missing the delay will be skipped.
+
+- reset-deassert-us: Delay after the reset was deasserted in microseconds.
+  If this property is missing the delay will be skipped.
+
 Example:
 
 ethernet-phy@0 {
@@ -60,4 +68,8 @@ ethernet-phy@0 {
 	interrupt-parent = <&PIC>;
 	interrupts = <35 IRQ_TYPE_EDGE_RISING>;
 	reg = <0>;
+
+	reset-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
+	reset-assert-us = <1000>;
+	reset-deassert-us = <2000>;
 };
diff --git a/Documentation/devicetree/bindings/net/renesas,ravb.txt b/Documentation/devicetree/bindings/net/renesas,ravb.txt
index c902261893b9..b4dc455eb155 100644
--- a/Documentation/devicetree/bindings/net/renesas,ravb.txt
+++ b/Documentation/devicetree/bindings/net/renesas,ravb.txt
@@ -18,6 +18,7 @@ Required properties:
       - "renesas,etheravb-r8a7795" for the R8A7795 SoC.
       - "renesas,etheravb-r8a7796" for the R8A7796 SoC.
       - "renesas,etheravb-r8a77970" for the R8A77970 SoC.
+      - "renesas,etheravb-r8a77980" for the R8A77980 SoC.
       - "renesas,etheravb-r8a77995" for the R8A77995 SoC.
       - "renesas,etheravb-rcar-gen3" as a fallback for the above
 		R-Car Gen3 devices.
@@ -26,7 +27,11 @@ Required properties:
 	SoC-specific version corresponding to the platform first followed by
 	the generic version.
 
-- reg: offset and length of (1) the register block and (2) the stream buffer.
+- reg: Offset and length of (1) the register block and (2) the stream buffer.
+       The region for the register block is mandatory.
+       The region for the stream buffer is optional, as it is only present on
+       R-Car Gen2 and RZ/G1 SoCs, and on R-Car H3 (R8A7795), M3-W (R8A7796),
+       and M3-N (R8A77965).
 - interrupts: A list of interrupt-specifiers, one for each entry in
 	      interrupt-names.
 	      If interrupt-names is not present, an interrupt specifier
diff --git a/Documentation/devicetree/bindings/net/sff,sfp.txt b/Documentation/devicetree/bindings/net/sff,sfp.txt
index 60e970ce10ee..f1c441bedf68 100644
--- a/Documentation/devicetree/bindings/net/sff,sfp.txt
+++ b/Documentation/devicetree/bindings/net/sff,sfp.txt
@@ -3,7 +3,9 @@ Transceiver
 
 Required properties:
 
-- compatible : must be "sff,sfp"
+- compatible : must be one of
+  "sff,sfp" for SFP modules
+  "sff,sff" for soldered down SFF modules
 
 Optional Properties:
 
@@ -11,7 +13,8 @@ Optional Properties:
   interface
 
 - mod-def0-gpios : GPIO phandle and a specifier of the MOD-DEF0 (AKA Mod_ABS)
-  module presence input gpio signal, active (module absent) high
+  module presence input gpio signal, active (module absent) high. Must
+  not be present for SFF modules
 
 - los-gpios : GPIO phandle and a specifier of the Receiver Loss of Signal
   Indication input gpio signal, active (signal lost) high
@@ -24,10 +27,11 @@ Optional Properties:
 
 - rate-select0-gpios : GPIO phandle and a specifier of the Rx Signaling Rate
   Select (AKA RS0) output gpio signal, low: low Rx rate, high: high Rx rate
+  Must not be present for SFF modules
 
 - rate-select1-gpios : GPIO phandle and a specifier of the Tx Signaling Rate
   Select (AKA RS1) output gpio signal (SFP+ only), low: low Tx rate, high:
-  high Tx rate
+  high Tx rate. Must not be present for SFF modules
 
 Example #1: Direct serdes to SFP connection
 
diff --git a/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt b/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt
new file mode 100644
index 000000000000..270ea4efff13
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/socionext,uniphier-ave4.txt
@@ -0,0 +1,48 @@
+* Socionext AVE ethernet controller
+
+This describes the devicetree bindings for AVE ethernet controller
+implemented on Socionext UniPhier SoCs.
+
+Required properties:
+ - compatible: Should be
+	- "socionext,uniphier-pro4-ave4" : for Pro4 SoC
+	- "socionext,uniphier-pxs2-ave4" : for PXs2 SoC
+	- "socionext,uniphier-ld11-ave4" : for LD11 SoC
+	- "socionext,uniphier-ld20-ave4" : for LD20 SoC
+ - reg: Address where registers are mapped and size of region.
+ - interrupts: Should contain the MAC interrupt.
+ - phy-mode: See ethernet.txt in the same directory. Allow to choose
+	"rgmii", "rmii", or "mii" according to the PHY.
+ - phy-handle: Should point to the external phy device.
+	See ethernet.txt file in the same directory.
+ - clocks: A phandle to the clock for the MAC.
+
+Optional properties:
+ - resets: A phandle to the reset control for the MAC.
+ - local-mac-address: See ethernet.txt in the same directory.
+
+Required subnode:
+ - mdio: A container for child nodes representing phy nodes.
+         See phy.txt in the same directory.
+
+Example:
+
+	ether: ethernet@65000000 {
+		compatible = "socionext,uniphier-ld20-ave4";
+		reg = <0x65000000 0x8500>;
+		interrupts = <0 66 4>;
+		phy-mode = "rgmii";
+		phy-handle = <&ethphy>;
+		clocks = <&sys_clk 6>;
+		resets = <&sys_rst 6>;
+		local-mac-address = [00 00 00 00 00 00];
+
+		mdio {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			ethphy: ethphy@1 {
+				reg = <1>;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/net/socionext-netsec.txt b/Documentation/devicetree/bindings/net/socionext-netsec.txt
new file mode 100644
index 000000000000..0cff94fb0433
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/socionext-netsec.txt
@@ -0,0 +1,53 @@
+* Socionext NetSec Ethernet Controller IP
+
+Required properties:
+- compatible: Should be "socionext,synquacer-netsec"
+- reg: Address and length of the control register area, followed by the
+       address and length of the EEPROM holding the MAC address and
+       microengine firmware
+- interrupts: Should contain ethernet controller interrupt
+- clocks: phandle to the PHY reference clock
+- clock-names: Should be "phy_ref_clk"
+- phy-mode: See ethernet.txt file in the same directory
+- phy-handle: See ethernet.txt in the same directory.
+
+- mdio device tree subnode: When the Netsec has a phy connected to its local
+		mdio, there must be device tree subnode with the following
+		required properties:
+
+	- #address-cells: Must be <1>.
+	- #size-cells: Must be <0>.
+
+	For each phy on the mdio bus, there must be a node with the following
+	fields:
+	- compatible: Refer to phy.txt
+	- reg: phy id used to communicate to phy.
+
+Optional properties: (See ethernet.txt file in the same directory)
+- dma-coherent: Boolean property, must only be present if memory
+	accesses performed by the device are cache coherent.
+- local-mac-address: See ethernet.txt in the same directory.
+- mac-address: See ethernet.txt in the same directory.
+- max-speed: See ethernet.txt in the same directory.
+- max-frame-size: See ethernet.txt in the same directory.
+
+Example:
+	eth0: ethernet@522d0000 {
+		compatible = "socionext,synquacer-netsec";
+		reg = <0 0x522d0000 0x0 0x10000>, <0 0x10000000 0x0 0x10000>;
+		interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&clk_netsec>;
+		clock-names = "phy_ref_clk";
+		phy-mode = "rgmii";
+		max-speed = <1000>;
+		max-frame-size = <9000>;
+		phy-handle = <&phy1>;
+
+		mdio {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			phy1: ethernet-phy@1 {
+				compatible = "ethernet-phy-ieee802.3-c22";
+				reg = <1>;
+			};
+		};
diff --git a/Documentation/devicetree/bindings/net/ti,wilink-st.txt b/Documentation/devicetree/bindings/net/ti-bluetooth.txt
similarity index 64%
rename from Documentation/devicetree/bindings/net/ti,wilink-st.txt
rename to Documentation/devicetree/bindings/net/ti-bluetooth.txt
index 1649c1f66b07..6d03ff8c7068 100644
--- a/Documentation/devicetree/bindings/net/ti,wilink-st.txt
+++ b/Documentation/devicetree/bindings/net/ti-bluetooth.txt
@@ -1,10 +1,18 @@
-TI WiLink 7/8 (wl12xx/wl18xx) Shared Transport BT/FM/GPS devices
+Texas Instruments Bluetooth Chips
+---------------------------------
+
+This documents the binding structure and common properties for serial
+attached TI Bluetooth devices. The following chips are included in this
+binding:
+
+* TI CC256x Bluetooth devices
+* TI WiLink 7/8 (wl12xx/wl18xx) Shared Transport BT/FM/GPS devices
 
 TI WiLink devices have a UART interface for providing Bluetooth, FM radio,
 and GPS over what's called "shared transport". The shared transport is
 standard BT HCI protocol with additional channels for the other functions.
 
-These devices also have a separate WiFi interface as described in
+TI WiLink devices also have a separate WiFi interface as described in
 wireless/ti,wlcore.txt.
 
 This bindings follows the UART slave device binding in
@@ -12,6 +20,7 @@ This bindings follows the UART slave device binding in
 
 Required properties:
  - compatible: should be one of the following:
+    "ti,cc2560"
     "ti,wl1271-st"
     "ti,wl1273-st"
     "ti,wl1281-st"
@@ -32,6 +41,9 @@ Optional properties:
    See ../clocks/clock-bindings.txt for details.
  - clock-names : Must include the following entry:
    "ext_clock" (External clock provided to the TI combo chip).
+ - nvmem-cells: phandle to nvmem data cell that contains a 6 byte BD address
+   with the most significant byte first (big-endian).
+ - nvmem-cell-names: "bd-address" (required when nvmem-cells is specified)
 
 Example:
 
@@ -43,5 +55,7 @@ Example:
 		enable-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>;
 		clocks = <&clk32k_wl18xx>;
 		clock-names = "ext_clock";
+		nvmem-cells = <&bd_address>;
+		nvmem-cell-names = "bd-address";
 	};
 };
diff --git a/Documentation/devicetree/bindings/net/wireless/mediatek,mt76.txt b/Documentation/devicetree/bindings/net/wireless/mediatek,mt76.txt
new file mode 100644
index 000000000000..0c17a0ec9b7b
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/wireless/mediatek,mt76.txt
@@ -0,0 +1,32 @@
+* MediaTek mt76xx devices
+
+This node provides properties for configuring the MediaTek mt76xx wireless
+device. The node is expected to be specified as a child node of the PCI
+controller to which the wireless chip is connected.
+
+Optional properties:
+
+- mac-address: See ethernet.txt in the parent directory
+- local-mac-address: See ethernet.txt in the parent directory
+- ieee80211-freq-limit: See ieee80211.txt
+- mediatek,mtd-eeprom: Specify a MTD partition + offset containing EEPROM data
+
+Optional nodes:
+- led: Properties for a connected LED
+  Optional properties:
+    - led-sources: See Documentation/devicetree/bindings/leds/common.txt
+
+&pcie {
+	pcie0 {
+		wifi@0,0 {
+			compatible = "mediatek,mt76";
+			reg = <0x0000 0 0 0 0>;
+			ieee80211-freq-limit = <5000000 6000000>;
+			mediatek,mtd-eeprom = <&factory 0x8000>;
+
+			led {
+				led-sources = <2>;
+			};
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt
index 74d7f0af209c..3d2a031217da 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.txt
@@ -41,6 +41,9 @@ Optional properties:
 - qcom,msi_addr: MSI interrupt address.
 - qcom,msi_base: Base value to add before writing MSI data into
 		MSI address register.
+- qcom,ath10k-calibration-variant: string to search for in the board-2.bin
+				   variant list with the same bus and device
+				   specific ids
 - qcom,ath10k-calibration-data : calibration data + board specific data
 				 as an array, the length can vary between
 				 hw versions.
diff --git a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.txt b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.txt
index 60bec4782806..265bdb7dc8aa 100644
--- a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.txt
+++ b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.txt
@@ -6,12 +6,17 @@ Required properties:
   - "rockchip,rk3188-efuse" - for RK3188 SoCs.
   - "rockchip,rk3228-efuse" - for RK3228 SoCs.
   - "rockchip,rk3288-efuse" - for RK3288 SoCs.
+  - "rockchip,rk3328-efuse" - for RK3328 SoCs.
   - "rockchip,rk3368-efuse" - for RK3368 SoCs.
   - "rockchip,rk3399-efuse" - for RK3399 SoCs.
 - reg: Should contain the registers location and exact eFuse size
 - clocks: Should be the clock id of eFuse
 - clock-names: Should be "pclk_efuse"
 
+Optional properties:
+- rockchip,efuse-size: Should be exact eFuse size in byte, the eFuse
+  size in property <reg> will be invalid if define this property.
+
 Deprecated properties:
 - compatible: "rockchip,rockchip-efuse"
   Old efuse compatible value compatible to rk3066a, rk3188 and rk3288
diff --git a/Documentation/devicetree/bindings/opp/opp.txt b/Documentation/devicetree/bindings/opp/opp.txt
index 9d733af26be7..4e4f30288c8b 100644
--- a/Documentation/devicetree/bindings/opp/opp.txt
+++ b/Documentation/devicetree/bindings/opp/opp.txt
@@ -45,6 +45,11 @@ Devices supporting OPPs must set their "operating-points-v2" property with
 phandle to a OPP table in their DT node. The OPP core will use this phandle to
 find the operating points for the device.
 
+This can contain more than one phandle for power domain providers that provide
+multiple power domains. That is, one phandle for each power domain. If only one
+phandle is available, then the same OPP table will be used for all power domains
+provided by the power domain provider.
+
 If required, this can be extended for SoC vendor specific bindings. Such bindings
 should be documented as Documentation/devicetree/bindings/power/<vendor>-opp.txt
 and should have a compatible description like: "operating-points-v2-<vendor>".
@@ -154,6 +159,14 @@ Optional properties:
 
 - status: Marks the node enabled/disabled.
 
+- required-opp: This contains phandle to an OPP node in another device's OPP
+  table. It may contain an array of phandles, where each phandle points to an
+  OPP of a different device. It should not contain multiple phandles to the OPP
+  nodes in the same OPP table. This specifies the minimum required OPP of the
+  device(s), whose OPP's phandle is present in this property, for the
+  functioning of the current device at the current OPP (where this property is
+  present).
+
 Example 1: Single cluster Dual-core ARM cortex A9, switch DVFS states together.
 
 / {
diff --git a/Documentation/devicetree/bindings/opp/ti-omap5-opp-supply.txt b/Documentation/devicetree/bindings/opp/ti-omap5-opp-supply.txt
new file mode 100644
index 000000000000..832346e489a3
--- /dev/null
+++ b/Documentation/devicetree/bindings/opp/ti-omap5-opp-supply.txt
@@ -0,0 +1,63 @@
+Texas Instruments OMAP compatible OPP supply description
+
+OMAP5, DRA7, and AM57 family of SoCs have Class0 AVS eFuse registers which
+contain data that can be used to adjust voltages programmed for some of their
+supplies for more efficient operation. This binding provides the information
+needed to read these values and use them to program the main regulator during
+an OPP transitions.
+
+Also, some supplies may have an associated vbb-supply which is an Adaptive Body
+Bias regulator which much be transitioned in a specific sequence with regards
+to the vdd-supply and clk when making an OPP transition. By supplying two
+regulators to the device that will undergo OPP transitions we can make use
+of the multi regulator binding that is part of the OPP core described here [1]
+to describe both regulators needed by the platform.
+
+[1] Documentation/devicetree/bindings/opp/opp.txt
+
+Required Properties for Device Node:
+- vdd-supply: phandle to regulator controlling VDD supply
+- vbb-supply: phandle to regulator controlling Body Bias supply
+	      (Usually Adaptive Body Bias regulator)
+
+Required Properties for opp-supply node:
+- compatible: Should be one of:
+	"ti,omap-opp-supply" - basic OPP supply controlling VDD and VBB
+	"ti,omap5-opp-supply" - OMAP5+ optimized voltages in efuse(class0)VDD
+			    along with VBB
+	"ti,omap5-core-opp-supply" - OMAP5+ optimized voltages in efuse(class0) VDD
+			    but no VBB.
+- reg: Address and length of the efuse register set for the device (mandatory
+	only for "ti,omap5-opp-supply")
+- ti,efuse-settings: An array of u32 tuple items providing information about
+	optimized efuse configuration. Each item consists of the following:
+	volt: voltage in uV - reference voltage (OPP voltage)
+	efuse_offseet: efuse offset from reg where the optimized voltage is stored.
+- ti,absolute-max-voltage-uv: absolute maximum voltage for the OPP supply.
+
+Example:
+
+/* Device Node (CPU)  */
+cpus {
+	cpu0: cpu@0 {
+		device_type = "cpu";
+
+		...
+
+		vdd-supply = <&vcc>;
+		vbb-supply = <&abb_mpu>;
+	};
+};
+
+/* OMAP OPP Supply with Class0 registers */
+opp_supply_mpu: opp_supply@4a003b20 {
+	compatible = "ti,omap5-opp-supply";
+	reg = <0x4a003b20 0x8>;
+	ti,efuse-settings = <
+	/* uV   offset */
+	1060000 0x0
+	1160000 0x4
+	1210000 0x8
+	>;
+	ti,absolute-max-voltage-uv = <1500000>;
+};
diff --git a/Documentation/devicetree/bindings/pci/axis,artpec6-pcie.txt b/Documentation/devicetree/bindings/pci/axis,artpec6-pcie.txt
index 4e4aee4439ea..979dc7b6cfe8 100644
--- a/Documentation/devicetree/bindings/pci/axis,artpec6-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/axis,artpec6-pcie.txt
@@ -4,7 +4,10 @@ This PCIe host controller is based on the Synopsys DesignWare PCIe IP
 and thus inherits all the common properties defined in designware-pcie.txt.
 
 Required properties:
-- compatible: "axis,artpec6-pcie", "snps,dw-pcie"
+- compatible: "axis,artpec6-pcie", "snps,dw-pcie" for ARTPEC-6 in RC mode;
+	      "axis,artpec6-pcie-ep", "snps,dw-pcie" for ARTPEC-6 in EP mode;
+	      "axis,artpec7-pcie", "snps,dw-pcie" for ARTPEC-7 in RC mode;
+	      "axis,artpec7-pcie-ep", "snps,dw-pcie" for ARTPEC-7 in EP mode;
 - reg: base addresses and lengths of the PCIe controller (DBI),
 	the PHY controller, and configuration address space.
 - reg-names: Must include the following entries:
diff --git a/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt
new file mode 100644
index 000000000000..9a305237fa6e
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-ep.txt
@@ -0,0 +1,22 @@
+* Cadence PCIe endpoint controller
+
+Required properties:
+- compatible: Should contain "cdns,cdns-pcie-ep" to identify the IP used.
+- reg: Should contain the controller register base address and AXI interface
+  region base address respectively.
+- reg-names: Must be "reg" and "mem" respectively.
+- cdns,max-outbound-regions: Set to maximum number of outbound regions
+
+Optional properties:
+- max-functions: Maximum number of functions that can be configured (default 1).
+
+Example:
+
+pcie@fc000000 {
+	compatible = "cdns,cdns-pcie-ep";
+	reg = <0x0 0xfc000000 0x0 0x01000000>,
+	      <0x0 0x80000000 0x0 0x40000000>;
+	reg-names = "reg", "mem";
+	cdns,max-outbound-regions = <16>;
+	max-functions = /bits/ 8 <8>;
+};
diff --git a/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt
new file mode 100644
index 000000000000..20a33f38f69d
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/cdns,cdns-pcie-host.txt
@@ -0,0 +1,60 @@
+* Cadence PCIe host controller
+
+This PCIe controller inherits the base properties defined in
+host-generic-pci.txt.
+
+Required properties:
+- compatible: Should contain "cdns,cdns-pcie-host" to identify the IP used.
+- reg: Should contain the controller register base address, PCIe configuration
+  window base address, and AXI interface region base address respectively.
+- reg-names: Must be "reg", "cfg" and "mem" respectively.
+- #address-cells: Set to <3>
+- #size-cells: Set to <2>
+- device_type: Set to "pci"
+- ranges: Ranges for the PCI memory and I/O regions
+- #interrupt-cells: Set to <1>
+- interrupt-map-mask and interrupt-map: Standard PCI properties to define the
+  mapping of the PCIe interface to interrupt numbers.
+
+Optional properties:
+- cdns,max-outbound-regions: Set to maximum number of outbound regions
+  (default 32)
+- cdns,no-bar-match-nbits: Set into the no BAR match register to configure the
+  number of least significant bits kept during inbound (PCIe -> AXI) address
+  translations (default 32)
+- vendor-id: The PCI vendor ID (16 bits, default is design dependent)
+- device-id: The PCI device ID (16 bits, default is design dependent)
+
+Example:
+
+pcie@fb000000 {
+	compatible = "cdns,cdns-pcie-host";
+	device_type = "pci";
+	#address-cells = <3>;
+	#size-cells = <2>;
+	bus-range = <0x0 0xff>;
+	linux,pci-domain = <0>;
+	cdns,max-outbound-regions = <16>;
+	cdns,no-bar-match-nbits = <32>;
+	vendor-id = /bits/ 16 <0x17cd>;
+	device-id = /bits/ 16 <0x0200>;
+
+	reg = <0x0 0xfb000000  0x0 0x01000000>,
+	      <0x0 0x41000000  0x0 0x00001000>,
+	      <0x0 0x40000000  0x0 0x04000000>;
+	reg-names = "reg", "cfg", "mem";
+
+	ranges = <0x02000000 0x0 0x42000000  0x0 0x42000000  0x0 0x1000000>,
+		 <0x01000000 0x0 0x43000000  0x0 0x43000000  0x0 0x0010000>;
+
+	#interrupt-cells = <0x1>;
+
+	interrupt-map = <0x0 0x0 0x0  0x1  &gic  0x0 0x0 0x0 14 0x1
+			 0x0 0x0 0x0  0x2  &gic  0x0 0x0 0x0 15 0x1
+			 0x0 0x0 0x0  0x3  &gic  0x0 0x0 0x0 16 0x1
+			 0x0 0x0 0x0  0x4  &gic  0x0 0x0 0x0 17 0x1>;
+
+	interrupt-map-mask = <0x0 0x0 0x0  0x7>;
+
+	msi-parent = <&its_pci>;
+};
diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.txt b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.txt
index 149d8f7f86b0..cb33421184a0 100644
--- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.txt
@@ -41,6 +41,7 @@ Optional properties:
 Additional required properties for imx6sx-pcie:
 - clock names: Must include the following additional entries:
 	- "pcie_inbound_axi"
+- power-domains: Must be set to a phandle pointing to the PCIE_PHY power domain
 
 Additional required properties for imx7d-pcie:
 - power-domains: Must be set to a phandle pointing to PCIE_PHY power domain
diff --git a/Documentation/devicetree/bindings/pci/samsung,exynos5440-pcie.txt b/Documentation/devicetree/bindings/pci/samsung,exynos5440-pcie.txt
index 34a11bfbfb60..651d957d1051 100644
--- a/Documentation/devicetree/bindings/pci/samsung,exynos5440-pcie.txt
+++ b/Documentation/devicetree/bindings/pci/samsung,exynos5440-pcie.txt
@@ -6,9 +6,6 @@ and thus inherits all the common properties defined in designware-pcie.txt.
 Required properties:
 - compatible: "samsung,exynos5440-pcie"
 - reg: base addresses and lengths of the PCIe controller,
-	the PHY controller, additional register for the PHY controller.
-	(Registers for the PHY controller are DEPRECATED.
-	 Use the PHY framework.)
 - reg-names : First name should be set to "elbi".
 	And use the "config" instead of getting the configuration address space
 	from "ranges".
@@ -23,49 +20,8 @@ For other common properties, refer to
 
 Example:
 
-SoC-specific DT Entry:
+SoC-specific DT Entry (with using PHY framework):
 
-	pcie@290000 {
-		compatible = "samsung,exynos5440-pcie", "snps,dw-pcie";
-		reg = <0x290000 0x1000
-			0x270000 0x1000
-			0x271000 0x40>;
-		interrupts = <0 20 0>, <0 21 0>, <0 22 0>;
-		clocks = <&clock 28>, <&clock 27>;
-		clock-names = "pcie", "pcie_bus";
-		#address-cells = <3>;
-		#size-cells = <2>;
-		device_type = "pci";
-		ranges = <0x00000800 0 0x40000000 0x40000000 0 0x00001000   /* configuration space */
-			  0x81000000 0 0	  0x40001000 0 0x00010000   /* downstream I/O */
-			  0x82000000 0 0x40011000 0x40011000 0 0x1ffef000>; /* non-prefetchable memory */
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0>;
-		interrupt-map = <0 0 0 0 &gic GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
-		num-lanes = <4>;
-	};
-
-	pcie@2a0000 {
-		compatible = "samsung,exynos5440-pcie", "snps,dw-pcie";
-		reg = <0x2a0000 0x1000
-			0x272000 0x1000
-			0x271040 0x40>;
-		interrupts = <0 23 0>, <0 24 0>, <0 25 0>;
-		clocks = <&clock 29>, <&clock 27>;
-		clock-names = "pcie", "pcie_bus";
-		#address-cells = <3>;
-		#size-cells = <2>;
-		device_type = "pci";
-		ranges = <0x00000800 0 0x60000000 0x60000000 0 0x00001000   /* configuration space */
-			  0x81000000 0 0	  0x60001000 0 0x00010000   /* downstream I/O */
-			  0x82000000 0 0x60011000 0x60011000 0 0x1ffef000>; /* non-prefetchable memory */
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0>;
-		interrupt-map = <0 0 0 0 &gic GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
-		num-lanes = <4>;
-	};
-
-With using PHY framework:
 	pcie_phy0: pcie-phy@270000 {
 		...
 		reg = <0x270000 0x1000>, <0x271000 0x40>;
@@ -74,13 +30,21 @@ With using PHY framework:
 	};
 
 	pcie@290000 {
-		...
+		compatible = "samsung,exynos5440-pcie", "snps,dw-pcie";
 		reg = <0x290000 0x1000>, <0x40000000 0x1000>;
 		reg-names = "elbi", "config";
+		clocks = <&clock 28>, <&clock 27>;
+		clock-names = "pcie", "pcie_bus";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
 		phys = <&pcie_phy0>;
 		ranges = <0x81000000 0 0	  0x60001000 0 0x00010000
 			  0x82000000 0 0x60011000 0x60011000 0 0x1ffef000>;
-		...
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+		num-lanes = <4>;
 	};
 
 Board-specific DT Entry:
diff --git a/Documentation/devicetree/bindings/phy/ti-phy.txt b/Documentation/devicetree/bindings/phy/ti-phy.txt
index cd13e6157088..57dfda8a7a1d 100644
--- a/Documentation/devicetree/bindings/phy/ti-phy.txt
+++ b/Documentation/devicetree/bindings/phy/ti-phy.txt
@@ -120,7 +120,7 @@ usb3phy@4a084400 {
 			"refclk";
 };
 
-sata_phy: phy@4A096000 {
+sata_phy: phy@4a096000 {
 	compatible = "ti,phy-pipe3-sata";
 	reg = <0x4A096000 0x80>, /* phy_rx */
 	      <0x4A096400 0x64>, /* phy_tx */
diff --git a/Documentation/devicetree/bindings/pinctrl/abilis,tb10x-iomux.txt b/Documentation/devicetree/bindings/pinctrl/abilis,tb10x-iomux.txt
index 2c11866221c2..c591b9cb5ba0 100644
--- a/Documentation/devicetree/bindings/pinctrl/abilis,tb10x-iomux.txt
+++ b/Documentation/devicetree/bindings/pinctrl/abilis,tb10x-iomux.txt
@@ -49,7 +49,7 @@ explained in Documentation/devicetree/bindings/gpio/gpio.txt.
 Example
 -------
 
-iomux: iomux@FF10601c {
+iomux: iomux@ff10601c {
 	compatible = "abilis,tb10x-iomux";
 	reg = <0xFF10601c 0x4>;
 	pctl_gpio_a: pctl-gpio-a {
@@ -59,7 +59,7 @@ iomux: iomux@FF10601c {
 		abilis,function = "uart0";
 	};
 };
-uart@FF100000 {
+uart@ff100000 {
 	compatible = "snps,dw-apb-uart";
 	reg = <0xFF100000 0x100>;
 	clock-frequency = <166666666>;
@@ -69,7 +69,7 @@ uart@FF100000 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pctl_uart0>;
 };
-gpioa: gpio@FF140000 {
+gpioa: gpio@ff140000 {
 	compatible = "abilis,tb10x-gpio";
 	reg = <0xFF140000 0x1000>;
 	gpio-controller;
diff --git a/Documentation/devicetree/bindings/pinctrl/cortina,gemini-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/cortina,gemini-pinctrl.txt
index d857b67fab72..4346ff2dd8e6 100644
--- a/Documentation/devicetree/bindings/pinctrl/cortina,gemini-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/cortina,gemini-pinctrl.txt
@@ -17,6 +17,9 @@ and generic pin config nodes.
 
 Supported configurations:
 - skew-delay is supported on the Ethernet pins
+- drive-strength with 4, 8, 12 or 16 mA as argument is supported for
+  entire groups on the groups "idegrp", "gmii_gmac0_grp", "gmii_gmac1_grp"
+  and "pcigrp".
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx6ul-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx6ul-pinctrl.txt
index a81bbf37ed66..7ca4f6118d9a 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx6ul-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx6ul-pinctrl.txt
@@ -4,7 +4,8 @@ Please refer to fsl,imx-pinctrl.txt in this directory for common binding part
 and usage.
 
 Required properties:
-- compatible: "fsl,imx6ul-iomuxc"
+- compatible: "fsl,imx6ul-iomuxc" for main IOMUX controller or
+  "fsl,imx6ull-iomuxc-snvs" for i.MX 6ULL's SNVS IOMUX controller.
 - fsl,pins: each entry consists of 6 integers and represents the mux and config
   setting for one pin.  The first 5 integers <mux_reg conf_reg input_reg mux_val
   input_val> are specified using a PIN_FUNC_ID macro, which can be found in
diff --git a/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt
index 0326154c7925..a72dc3178179 100644
--- a/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/img,pistachio-pinctrl.txt
@@ -152,7 +152,7 @@ resetn
 
 Example:
 --------
-pinctrl@18101C00 {
+pinctrl@18101c00 {
 	compatible = "img,pistachio-system-pinctrl";
 	reg = <0x18101C00 0x400>;
 
diff --git a/Documentation/devicetree/bindings/pinctrl/lantiq,pinctrl-xway.txt b/Documentation/devicetree/bindings/pinctrl/lantiq,pinctrl-xway.txt
index 8e5216bcd748..4658f105fa09 100644
--- a/Documentation/devicetree/bindings/pinctrl/lantiq,pinctrl-xway.txt
+++ b/Documentation/devicetree/bindings/pinctrl/lantiq,pinctrl-xway.txt
@@ -163,7 +163,7 @@ Valid values for xRX300 pin names:
 						io42-io43,io48-io61.
 
 Example:
-	gpio: pinmux@E100B10 {
+	gpio: pinmux@e100b10 {
 		compatible = "lantiq,danube-pinctrl";
 		pinctrl-names = "default";
 		pinctrl-0 = <&state_default>;
diff --git a/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
index 2392557ede27..2c12f9789116 100644
--- a/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/meson,pinctrl.txt
@@ -9,6 +9,8 @@ Required properties for the root node:
 		      "amlogic,meson-gxbb-aobus-pinctrl"
 		      "amlogic,meson-gxl-periphs-pinctrl"
 		      "amlogic,meson-gxl-aobus-pinctrl"
+		      "amlogic,meson-axg-periphs-pinctrl"
+		      "amlogic,meson-axg-aobus-pinctrl"
  - reg: address and size of registers controlling irq functionality
 
 === GPIO sub-nodes ===
diff --git a/Documentation/devicetree/bindings/pinctrl/mscc,ocelot-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/mscc,ocelot-pinctrl.txt
new file mode 100644
index 000000000000..24a210e0c59a
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/mscc,ocelot-pinctrl.txt
@@ -0,0 +1,39 @@
+Microsemi Ocelot pin controller Device Tree Bindings
+----------------------------------------------------
+
+Required properties:
+ - compatible		: Should be "mscc,ocelot-pinctrl"
+ - reg			: Address and length of the register set for the device
+ - gpio-controller	: Indicates this device is a GPIO controller
+ - #gpio-cells		: Must be 2.
+			  The first cell is the pin number and the
+			  second cell specifies GPIO flags, as defined in
+			  <dt-bindings/gpio/gpio.h>.
+ - gpio-ranges		: Range of pins managed by the GPIO controller.
+
+
+The ocelot-pinctrl driver uses the generic pin multiplexing and generic pin
+configuration documented in pinctrl-bindings.txt.
+
+The following generic properties are supported:
+ - function
+ - pins
+
+Example:
+	gpio: pinctrl@71070034 {
+		compatible = "mscc,ocelot-pinctrl";
+		reg = <0x71070034 0x28>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-ranges = <&gpio 0 0 22>;
+
+		uart_pins: uart-pins {
+				pins = "GPIO_6", "GPIO_7";
+				function = "uart";
+		};
+
+		uart2_pins: uart2-pins {
+				pins = "GPIO_12", "GPIO_13";
+				function = "uart2";
+		};
+	};
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
index 231fa1db7c5e..afa8a18ea11a 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
@@ -81,7 +81,7 @@ Examples:
 		reg = <0 0x10005000 0 0x1000>;
 	};
 
-	syscfg_pctl_b: syscfg_pctl_b@1020C020 {
+	syscfg_pctl_b: syscfg_pctl_b@1020c020 {
 		compatible = "mediatek,mt8135-pctl-b-syscfg", "syscon";
 		reg = <0 0x1020C020 0 0x1000>;
 	};
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
new file mode 100644
index 000000000000..f18ed99f6e14
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
@@ -0,0 +1,351 @@
+== MediaTek MT7622 pinctrl controller ==
+
+Required properties for the root node:
+ - compatible: Should be one of the following
+	       "mediatek,mt7622-pinctrl" for MT7622 SoC
+ - reg: offset and length of the pinctrl space
+
+ - gpio-controller: Marks the device node as a GPIO controller.
+ - #gpio-cells: Should be two. The first cell is the pin number and the
+   second is the GPIO flags.
+
+Please refer to pinctrl-bindings.txt in this directory for details of the
+common pinctrl bindings used by client devices, including the meaning of the
+phrase "pin configuration node".
+
+MT7622 pin configuration nodes act as a container for an arbitrary number of
+subnodes. Each of these subnodes represents some desired configuration for a
+pin, a group, or a list of pins or groups. This configuration can include the
+mux function to select on those pin(s)/group(s), and various pin configuration
+parameters, such as pull-up, slew rate, etc.
+
+We support 2 types of configuration nodes. Those nodes can be either pinmux
+nodes or pinconf nodes. Each configuration node can consist of multiple nodes
+describing the pinmux and pinconf options.
+
+The name of each subnode doesn't matter as long as it is unique; all subnodes
+should be enumerated and processed purely based on their content.
+
+== pinmux nodes content ==
+
+The following generic properties as defined in pinctrl-bindings.txt are valid
+to specify in a pinmux subnode:
+
+Required properties are:
+ - groups: An array of strings. Each string contains the name of a group.
+  Valid values for these names are listed below.
+ - function: A string containing the name of the function to mux to the
+  group. Valid values for function names are listed below.
+
+== pinconf nodes content ==
+
+The following generic properties as defined in pinctrl-bindings.txt are valid
+to specify in a pinconf subnode:
+
+Required properties are:
+ - pins: An array of strings. Each string contains the name of a pin.
+  Valid values for these names are listed below.
+ - groups: An array of strings. Each string contains the name of a group.
+  Valid values for these names are listed below.
+
+Optional properies are:
+ bias-disable, bias-pull, bias-pull-down, input-enable,
+ input-schmitt-enable, input-schmitt-disable, output-enable
+ output-low, output-high, drive-strength, slew-rate
+
+ Valid arguments for 'slew-rate' are '0' for no slew rate controlled and '1' for
+ slower slew rate respectively.
+ Valid arguments for 'drive-strength', 4, 8, 12, or 16 in mA.
+
+The following specific properties as defined are valid to specify in a pinconf
+subnode:
+
+Optional properties are:
+ - mediatek,tdsel: An integer describing the steps for output level shifter duty
+   cycle when asserted (high pulse width adjustment). Valid arguments are from 0
+   to 15.
+ - mediatek,rdsel: An integer describing the steps for input level shifter duty
+   cycle when asserted (high pulse width adjustment). Valid arguments are from 0
+   to 63.
+
+== Valid values for pins, function and groups on MT7622 ==
+
+Valid values for pins are:
+pins can be referenced via the pin names as the below table shown and the
+related physical number is also put ahead of those names which helps cross
+references to pins between groups to know whether pins assignment conflict
+happens among devices try to acquire those available pins.
+
+	Pin #:  Valid values for pins
+	-----------------------------
+	PIN 0: "GPIO_A"
+	PIN 1: "I2S1_IN"
+	PIN 2: "I2S1_OUT"
+	PIN 3: "I2S_BCLK"
+	PIN 4: "I2S_WS"
+	PIN 5: "I2S_MCLK"
+	PIN 6: "TXD0"
+	PIN 7: "RXD0"
+	PIN 8: "SPI_WP"
+	PIN 9: "SPI_HOLD"
+	PIN 10: "SPI_CLK"
+	PIN 11: "SPI_MOSI"
+	PIN 12: "SPI_MISO"
+	PIN 13: "SPI_CS"
+	PIN 14: "I2C_SDA"
+	PIN 15: "I2C_SCL"
+	PIN 16: "I2S2_IN"
+	PIN 17: "I2S3_IN"
+	PIN 18: "I2S4_IN"
+	PIN 19: "I2S2_OUT"
+	PIN 20: "I2S3_OUT"
+	PIN 21: "I2S4_OUT"
+	PIN 22: "GPIO_B"
+	PIN 23: "MDC"
+	PIN 24: "MDIO"
+	PIN 25: "G2_TXD0"
+	PIN 26: "G2_TXD1"
+	PIN 27: "G2_TXD2"
+	PIN 28: "G2_TXD3"
+	PIN 29: "G2_TXEN"
+	PIN 30: "G2_TXC"
+	PIN 31: "G2_RXD0"
+	PIN 32: "G2_RXD1"
+	PIN 33: "G2_RXD2"
+	PIN 34: "G2_RXD3"
+	PIN 35: "G2_RXDV"
+	PIN 36: "G2_RXC"
+	PIN 37: "NCEB"
+	PIN 38: "NWEB"
+	PIN 39: "NREB"
+	PIN 40: "NDL4"
+	PIN 41: "NDL5"
+	PIN 42: "NDL6"
+	PIN 43: "NDL7"
+	PIN 44: "NRB"
+	PIN 45: "NCLE"
+	PIN 46: "NALE"
+	PIN 47: "NDL0"
+	PIN 48: "NDL1"
+	PIN 49: "NDL2"
+	PIN 50: "NDL3"
+	PIN 51: "MDI_TP_P0"
+	PIN 52: "MDI_TN_P0"
+	PIN 53: "MDI_RP_P0"
+	PIN 54: "MDI_RN_P0"
+	PIN 55: "MDI_TP_P1"
+	PIN 56: "MDI_TN_P1"
+	PIN 57: "MDI_RP_P1"
+	PIN 58: "MDI_RN_P1"
+	PIN 59: "MDI_RP_P2"
+	PIN 60: "MDI_RN_P2"
+	PIN 61: "MDI_TP_P2"
+	PIN 62: "MDI_TN_P2"
+	PIN 63: "MDI_TP_P3"
+	PIN 64: "MDI_TN_P3"
+	PIN 65: "MDI_RP_P3"
+	PIN 66: "MDI_RN_P3"
+	PIN 67: "MDI_RP_P4"
+	PIN 68: "MDI_RN_P4"
+	PIN 69: "MDI_TP_P4"
+	PIN 70: "MDI_TN_P4"
+	PIN 71: "PMIC_SCL"
+	PIN 72: "PMIC_SDA"
+	PIN 73: "SPIC1_CLK"
+	PIN 74: "SPIC1_MOSI"
+	PIN 75: "SPIC1_MISO"
+	PIN 76: "SPIC1_CS"
+	PIN 77: "GPIO_D"
+	PIN 78: "WATCHDOG"
+	PIN 79: "RTS3_N"
+	PIN 80: "CTS3_N"
+	PIN 81: "TXD3"
+	PIN 82: "RXD3"
+	PIN 83: "PERST0_N"
+	PIN 84: "PERST1_N"
+	PIN 85: "WLED_N"
+	PIN 86: "EPHY_LED0_N"
+	PIN 87: "AUXIN0"
+	PIN 88: "AUXIN1"
+	PIN 89: "AUXIN2"
+	PIN 90: "AUXIN3"
+	PIN 91: "TXD4"
+	PIN 92: "RXD4"
+	PIN 93: "RTS4_N"
+	PIN 94: "CST4_N"
+	PIN 95: "PWM1"
+	PIN 96: "PWM2"
+	PIN 97: "PWM3"
+	PIN 98: "PWM4"
+	PIN 99: "PWM5"
+	PIN 100: "PWM6"
+	PIN 101: "PWM7"
+	PIN 102: "GPIO_E"
+
+Valid values for function are:
+	"emmc", "eth", "i2c", "i2s", "ir", "led", "flash", "pcie",
+	"pmic", "pwm", "sd", "spi", "tdm", "uart", "watchdog"
+
+Valid values for groups are:
+additional data is put followingly with valid value allowing us to know which
+applicable function and which relevant pins (in pin#) are able applied for that
+group.
+
+	Valid value			function	pins (in pin#)
+	-------------------------------------------------------------------------
+	"emmc"				"emmc"		40, 41, 42, 43, 44, 45,
+							47, 48, 49, 50
+	"emmc_rst"			"emmc"		37
+	"esw"				"eth"		51, 52, 53, 54, 55, 56,
+							57, 58, 59, 60, 61, 62,
+							63, 64, 65, 66, 67, 68,
+							69, 70
+	"esw_p0_p1"			"eth"		51, 52, 53, 54, 55, 56,
+							57, 58
+	"esw_p2_p3_p4"			"eth"		59, 60, 61, 62, 63, 64,
+							65, 66, 67, 68, 69, 70
+	"rgmii_via_esw"			"eth"		59, 60, 61, 62, 63, 64,
+							65, 66, 67, 68, 69, 70
+	"rgmii_via_gmac1"		"eth"		59, 60, 61, 62, 63, 64,
+							65, 66, 67, 68, 69, 70
+	"rgmii_via_gmac2"		"eth"		25, 26, 27, 28, 29, 30,
+							31, 32, 33, 34, 35, 36
+	"mdc_mdio"			"eth"		23, 24
+	"i2c0"				"i2c"		14, 15
+	"i2c1_0"			"i2c"		55, 56
+	"i2c1_1"			"i2c"		73, 74
+	"i2c1_2"			"i2c"		87, 88
+	"i2c2_0"			"i2c"		57, 58
+	"i2c2_1"			"i2c"		75, 76
+	"i2c2_2"			"i2c"		89, 90
+	"i2s_in_mclk_bclk_ws"		"i2s"		3, 4, 5
+	"i2s1_in_data"			"i2s"		1
+	"i2s2_in_data"			"i2s"		16
+	"i2s3_in_data"			"i2s"		17
+	"i2s4_in_data"			"i2s"		18
+	"i2s_out_mclk_bclk_ws"		"i2s"		3, 4, 5
+	"i2s1_out_data"			"i2s"		2
+	"i2s2_out_data"			"i2s"		19
+	"i2s3_out_data"			"i2s"		20
+	"i2s4_out_data"			"i2s"		21
+	"ir_0_tx"			"ir"		16
+	"ir_1_tx"			"ir"		59
+	"ir_2_tx"			"ir"		99
+	"ir_0_rx"			"ir"		17
+	"ir_1_rx"			"ir"		60
+	"ir_2_rx"			"ir"		100
+	"ephy_leds"			"led"		86, 91, 92, 93, 94
+	"ephy0_led"			"led"		86
+	"ephy1_led"			"led"		91
+	"ephy2_led"			"led"		92
+	"ephy3_led"			"led"		93
+	"ephy4_led"			"led"		94
+	"wled"				"led"		85
+	"par_nand"			"flash"		37, 38, 39, 40, 41, 42,
+							43, 44, 45, 46, 47, 48,
+							49, 50
+	"snfi"				"flash"		8, 9, 10, 11, 12, 13
+	"spi_nor"			"flash"		8, 9, 10, 11, 12, 13
+	"pcie0_0_waken"			"pcie"		14
+	"pcie0_1_waken"			"pcie"		79
+	"pcie1_0_waken"			"pcie"		14
+	"pcie0_0_clkreq"		"pcie"		15
+	"pcie0_1_clkreq"		"pcie"		80
+	"pcie1_0_clkreq"		"pcie"		15
+	"pcie0_pad_perst"		"pcie"		83
+	"pcie1_pad_perst"		"pcie"		84
+	"pmic_bus"			"pmic"		71, 72
+	"pwm_ch1_0"			"pwm"		51
+	"pwm_ch1_1"			"pwm"		73
+	"pwm_ch1_2"			"pwm"		95
+	"pwm_ch2_0"			"pwm"		52
+	"pwm_ch2_1"			"pwm"		74
+	"pwm_ch2_2"			"pwm"		96
+	"pwm_ch3_0"			"pwm"		53
+	"pwm_ch3_1"			"pwm"		75
+	"pwm_ch3_2"			"pwm"		97
+	"pwm_ch4_0"			"pwm"		54
+	"pwm_ch4_1"			"pwm"		67
+	"pwm_ch4_2"			"pwm"		76
+	"pwm_ch4_3"			"pwm"		98
+	"pwm_ch5_0"			"pwm"		68
+	"pwm_ch5_1"			"pwm"		77
+	"pwm_ch5_2"			"pwm"		99
+	"pwm_ch6_0"			"pwm"		69
+	"pwm_ch6_1"			"pwm"		78
+	"pwm_ch6_2"			"pwm"		81
+	"pwm_ch6_3"			"pwm"		100
+	"pwm_ch7_0"			"pwm"		70
+	"pwm_ch7_1"			"pwm"		82
+	"pwm_ch7_2"			"pwm"		101
+	"sd_0"				"sd"		16, 17, 18, 19, 20, 21
+	"sd_1"				"sd"		25, 26, 27, 28, 29, 30
+	"spic0_0"			"spi"		63, 64, 65, 66
+	"spic0_1"			"spi"		79, 80, 81, 82
+	"spic1_0"			"spi"		67, 68, 69, 70
+	"spic1_1"			"spi"		73, 74, 75, 76
+	"spic2_0_wp_hold"		"spi"		8, 9
+	"spic2_0"			"spi"		10, 11, 12, 13
+	"tdm_0_out_mclk_bclk_ws"	"tdm"		8, 9, 10
+	"tdm_0_in_mclk_bclk_ws"		"tdm"		11, 12, 13
+	"tdm_0_out_data"		"tdm"		20
+	"tdm_0_in_data"			"tdm"		21
+	"tdm_1_out_mclk_bclk_ws"	"tdm"		57, 58, 59
+	"tdm_1_in_mclk_bclk_ws"		"tdm"		60, 61, 62
+	"tdm_1_out_data"		"tdm"		55
+	"tdm_1_in_data"			"tdm"		56
+	"uart0_0_tx_rx"			"uart"		6, 7
+	"uart1_0_tx_rx"			"uart"		55, 56
+	"uart1_0_rts_cts"		"uart"		57, 58
+	"uart1_1_tx_rx"			"uart"		73, 74
+	"uart1_1_rts_cts"		"uart"		75, 76
+	"uart2_0_tx_rx"			"uart"		3, 4
+	"uart2_0_rts_cts"		"uart"		1, 2
+	"uart2_1_tx_rx"			"uart"		51, 52
+	"uart2_1_rts_cts"		"uart"		53, 54
+	"uart2_2_tx_rx"			"uart"		59, 60
+	"uart2_2_rts_cts"		"uart"		61, 62
+	"uart2_3_tx_rx"			"uart"		95, 96
+	"uart3_0_tx_rx"			"uart"		57, 58
+	"uart3_1_tx_rx"			"uart"		81, 82
+	"uart3_1_rts_cts"		"uart"		79, 80
+	"uart4_0_tx_rx"			"uart"		61, 62
+	"uart4_1_tx_rx"			"uart"		91, 92
+	"uart4_1_rts_cts"		"uart"		93, 94
+	"uart4_2_tx_rx"			"uart"		97, 98
+	"uart4_2_rts_cts"		"uart"		95, 96
+	"watchdog"			"watchdog"	78
+
+Example:
+
+	pio: pinctrl@10211000 {
+		compatible = "mediatek,mt7622-pinctrl";
+		reg = <0 0x10211000 0 0x1000>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		pinctrl_eth_default: eth-default {
+			mux-mdio {
+				groups = "mdc_mdio";
+				function = "eth";
+				drive-strength = <12>;
+			};
+
+			mux-gmac2 {
+				groups = "gmac2";
+				function = "eth";
+				drive-strength = <12>;
+			};
+
+			mux-esw {
+				groups = "esw";
+				function = "eth";
+				drive-strength = <8>;
+			};
+
+			conf-mdio {
+				pins = "MDC";
+				bias-pull-up;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,msm8998-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/qcom,msm8998-pinctrl.txt
new file mode 100644
index 000000000000..e70c79bbbc5b
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,msm8998-pinctrl.txt
@@ -0,0 +1,193 @@
+Qualcomm MSM8998 TLMM block
+
+This binding describes the Top Level Mode Multiplexer block found in the
+MSM8998 platform.
+
+- compatible:
+	Usage: required
+	Value type: <string>
+	Definition: must be "qcom,msm8998-pinctrl"
+
+- reg:
+	Usage: required
+	Value type: <prop-encoded-array>
+	Definition: the base address and size of the TLMM register space.
+
+- interrupts:
+	Usage: required
+	Value type: <prop-encoded-array>
+	Definition: should specify the TLMM summary IRQ.
+
+- interrupt-controller:
+	Usage: required
+	Value type: <none>
+	Definition: identifies this node as an interrupt controller
+
+- #interrupt-cells:
+	Usage: required
+	Value type: <u32>
+	Definition: must be 2. Specifying the pin number and flags, as defined
+		    in <dt-bindings/interrupt-controller/irq.h>
+
+- gpio-controller:
+	Usage: required
+	Value type: <none>
+	Definition: identifies this node as a gpio controller
+
+- #gpio-cells:
+	Usage: required
+	Value type: <u32>
+	Definition: must be 2. Specifying the pin number and flags, as defined
+		    in <dt-bindings/gpio/gpio.h>
+
+Please refer to ../gpio/gpio.txt and ../interrupt-controller/interrupts.txt for
+a general description of GPIO and interrupt bindings.
+
+Please refer to pinctrl-bindings.txt in this directory for details of the
+common pinctrl bindings used by client devices, including the meaning of the
+phrase "pin configuration node".
+
+The pin configuration nodes act as a container for an arbitrary number of
+subnodes. Each of these subnodes represents some desired configuration for a
+pin, a group, or a list of pins or groups. This configuration can include the
+mux function to select on those pin(s)/group(s), and various pin configuration
+parameters, such as pull-up, drive strength, etc.
+
+
+PIN CONFIGURATION NODES:
+
+The name of each subnode is not important; all subnodes should be enumerated
+and processed purely based on their content.
+
+Each subnode only affects those parameters that are explicitly listed. In
+other words, a subnode that lists a mux function but no pin configuration
+parameters implies no information about any pin configuration parameters.
+Similarly, a pin subnode that describes a pullup parameter implies no
+information about e.g. the mux function.
+
+
+The following generic properties as defined in pinctrl-bindings.txt are valid
+to specify in a pin configuration subnode:
+
+- pins:
+	Usage: required
+	Value type: <string-array>
+	Definition: List of gpio pins affected by the properties specified in
+		    this subnode.
+
+		    Valid pins are:
+		      gpio0-gpio149
+		        Supports mux, bias and drive-strength
+
+		      sdc2_clk, sdc2_cmd, sdc2_data
+		        Supports bias and drive-strength
+
+		      ufs_reset
+		        Supports bias and drive-strength
+
+- function:
+	Usage: required
+	Value type: <string>
+	Definition: Specify the alternative function to be configured for the
+		    specified pins. Functions are only valid for gpio pins.
+		    Valid values are:
+
+		    gpio, adsp_ext, agera_pll, atest_char, atest_gpsadc0,
+		    atest_gpsadc1, atest_tsens, atest_tsens2, atest_usb1,
+		    atest_usb10, atest_usb11, atest_usb12, atest_usb13,
+		    audio_ref, bimc_dte0, bimc_dte1, blsp10_spi, blsp10_spi_a,
+		    blsp10_spi_b, blsp11_i2c, blsp1_spi, blsp1_spi_a,
+		    blsp1_spi_b, blsp2_spi, blsp9_spi, blsp_i2c1, blsp_i2c2,
+		    blsp_i2c3, blsp_i2c4, blsp_i2c5, blsp_i2c6, blsp_i2c7,
+		    blsp_i2c8, blsp_i2c9, blsp_i2c10, blsp_i2c11, blsp_i2c12,
+		    blsp_spi1, blsp_spi2, blsp_spi3, blsp_spi4, blsp_spi5,
+		    blsp_spi6, blsp_spi7, blsp_spi8, blsp_spi9, blsp_spi10,
+		    blsp_spi11, blsp_spi12, blsp_uart1_a, blsp_uart1_b,
+		    blsp_uart2_a, blsp_uart2_b, blsp_uart3_a, blsp_uart3_b,
+		    blsp_uart7_a, blsp_uart7_b, blsp_uart8, blsp_uart8_a,
+		    blsp_uart8_b, blsp_uart9_a, blsp_uart9_b, blsp_uim1_a,
+		    blsp_uim1_b, blsp_uim2_a, blsp_uim2_b, blsp_uim3_a,
+		    blsp_uim3_b, blsp_uim7_a, blsp_uim7_b, blsp_uim8_a,
+		    blsp_uim8_b, blsp_uim9_a, blsp_uim9_b, bt_reset,
+		    btfm_slimbus, cam_mclk, cci_async, cci_i2c, cci_timer0,
+		    cci_timer1, cci_timer2, cci_timer3, cci_timer4, cri_trng,
+		    cri_trng0, cri_trng1, dbg_out, ddr_bist, edp_hot, edp_lcd,
+		    gcc_gp1_a, gcc_gp1_b, gcc_gp2_a, gcc_gp2_b, gcc_gp3_a,
+		    gcc_gp3_b, hdmi_cec, hdmi_ddc, hdmi_hot, hdmi_rcv,
+		    isense_dbg, jitter_bist, ldo_en, ldo_update, lpass_slimbus,
+		    m_voc, mdp_vsync, mdp_vsync0, mdp_vsync1, mdp_vsync2,
+		    mdp_vsync3, mdp_vsync_a, mdp_vsync_b, modem_tsync, mss_lte,
+		    nav_dr, nav_pps, pa_indicator, pci_e0, phase_flag,
+		    pll_bypassnl, pll_reset, pri_mi2s, pri_mi2s_ws, prng_rosc,
+		    pwr_crypto, pwr_modem, pwr_nav, qdss_cti0_a, qdss_cti0_b,
+		    qdss_cti1_a, qdss_cti1_b, qdss, qlink_enable,
+		    qlink_request, qua_mi2s, sd_card, sd_write, sdc40, sdc41,
+		    sdc42, sdc43, sdc4_clk, sdc4_cmd, sec_mi2s, sp_cmu,
+		    spkr_i2s, ssbi1, ssc_irq, ter_mi2s, tgu_ch0, tgu_ch1,
+		    tsense_pwm1, tsense_pwm2, tsif1_clk, tsif1_data, tsif1_en,
+		    tsif1_error, tsif1_sync, tsif2_clk, tsif2_data, tsif2_en,
+		    tsif2_error, tsif2_sync, uim1_clk, uim1_data, uim1_present,
+		    uim1_reset, uim2_clk, uim2_data, uim2_present, uim2_reset,
+		    uim_batt, usb_phy, vfr_1, vsense_clkout, vsense_data0,
+		    vsense_data1, vsense_mode, wlan1_adc0, wlan1_adc1,
+		    wlan2_adc0, wlan2_adc1,
+
+- bias-disable:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins should be configued as no pull.
+
+- bias-pull-down:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins should be configued as pull down.
+
+- bias-pull-up:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins should be configued as pull up.
+
+- output-high:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins are configured in output mode, driven
+		    high.
+		    Not valid for sdc pins.
+
+- output-low:
+	Usage: optional
+	Value type: <none>
+	Definition: The specified pins are configured in output mode, driven
+		    low.
+		    Not valid for sdc pins.
+
+- drive-strength:
+	Usage: optional
+	Value type: <u32>
+	Definition: Selects the drive strength for the specified pins, in mA.
+		    Valid values are: 2, 4, 6, 8, 10, 12, 14 and 16
+
+Example:
+
+	tlmm: pinctrl@03400000 {
+		compatible = "qcom,msm8998-pinctrl";
+		reg = <0x03400000 0xc00000>;
+		interrupts = <0 208 0>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+
+		uart_console_active: uart_console_active {
+			mux {
+				pins = "gpio4", "gpio5";
+				function = "blsp_uart8_a";
+			};
+
+			config {
+				pins = "gpio4", "gpio5";
+				drive-strength = <2>;
+				bias-disable;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt
index 9b4f8041c36a..bb1790e0b176 100644
--- a/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt
@@ -24,6 +24,7 @@ Required Properties:
     - "renesas,pfc-r8a7794": for R8A7794 (R-Car E2) compatible pin-controller.
     - "renesas,pfc-r8a7795": for R8A7795 (R-Car H3) compatible pin-controller.
     - "renesas,pfc-r8a7796": for R8A7796 (R-Car M3-W) compatible pin-controller.
+    - "renesas,pfc-r8a77970": for R8A77970 (R-Car V3M) compatible pin-controller.
     - "renesas,pfc-r8a77995": for R8A77995 (R-Car D3) compatible pin-controller.
     - "renesas,pfc-sh73a0": for SH73A0 (SH-Mobile AG5) compatible pin-controller.
 
diff --git a/Documentation/devicetree/bindings/pinctrl/socionext,uniphier-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/socionext,uniphier-pinctrl.txt
new file mode 100644
index 000000000000..8173b12138ad
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/socionext,uniphier-pinctrl.txt
@@ -0,0 +1,27 @@
+UniPhier SoCs pin controller
+
+Required properties:
+- compatible: should be one of the following:
+    "socionext,uniphier-ld4-pinctrl"  - for LD4 SoC
+    "socionext,uniphier-pro4-pinctrl" - for Pro4 SoC
+    "socionext,uniphier-sld8-pinctrl" - for sLD8 SoC
+    "socionext,uniphier-pro5-pinctrl" - for Pro5 SoC
+    "socionext,uniphier-pxs2-pinctrl" - for PXs2 SoC
+    "socionext,uniphier-ld6b-pinctrl" - for LD6b SoC
+    "socionext,uniphier-ld11-pinctrl" - for LD11 SoC
+    "socionext,uniphier-ld20-pinctrl" - for LD20 SoC
+    "socionext,uniphier-pxs3-pinctrl" - for PXs3 SoC
+
+Note:
+The UniPhier pinctrl should be a subnode of a "syscon" compatible node.
+
+Example:
+	soc-glue@5f800000 {
+		compatible = "socionext,uniphier-pro4-soc-glue",
+			     "simple-mfd", "syscon";
+		reg = <0x5f800000 0x2000>;
+
+		pinctrl: pinctrl {
+			compatible = "socionext,uniphier-pro4-pinctrl";
+		};
+	};
diff --git a/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt
index 58c2a4c229db..2c46f30b62c5 100644
--- a/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt
@@ -12,6 +12,8 @@ Required properies:
    "st,stm32f469-pinctrl"
    "st,stm32f746-pinctrl"
    "st,stm32h743-pinctrl"
+   "st,stm32mp157-pinctrl"
+   "st,stm32mp157-z-pinctrl"
  - #address-cells: The value of this property must be 1
  - #size-cells	: The value of this property must be 1
  - ranges	: defines mapping between pin controller node (parent) to
diff --git a/Documentation/devicetree/bindings/power/actions,owl-sps.txt b/Documentation/devicetree/bindings/power/actions,owl-sps.txt
index 007b9a7ae723..78edd63641e8 100644
--- a/Documentation/devicetree/bindings/power/actions,owl-sps.txt
+++ b/Documentation/devicetree/bindings/power/actions,owl-sps.txt
@@ -2,10 +2,12 @@ Actions Semi Owl Smart Power System (SPS)
 
 Required properties:
 - compatible          :  "actions,s500-sps" for S500
+                         "actions,s700-sps" for S700
 - reg                 :  Offset and length of the register set for the device.
 - #power-domain-cells :  Must be 1.
                          See macros in:
                           include/dt-bindings/power/owl-s500-powergate.h for S500
+                          include/dt-bindings/power/owl-s700-powergate.h for S700
 
 
 Example:
diff --git a/Documentation/devicetree/bindings/power/fsl,imx-gpc.txt b/Documentation/devicetree/bindings/power/fsl,imx-gpc.txt
index e371b262d709..b31d6bbeee16 100644
--- a/Documentation/devicetree/bindings/power/fsl,imx-gpc.txt
+++ b/Documentation/devicetree/bindings/power/fsl,imx-gpc.txt
@@ -9,6 +9,7 @@ Required properties:
   - fsl,imx6q-gpc
   - fsl,imx6qp-gpc
   - fsl,imx6sl-gpc
+  - fsl,imx6sx-gpc
 - reg: should be register base and length as documented in the
   datasheet
 - interrupts: Should contain one interrupt specifier for the GPC interrupt
@@ -29,6 +30,8 @@ Required properties:
   PU_DOMAIN      1
   The following additional DOMAIN_INDEX value is valid for i.MX6SL:
   DISPLAY_DOMAIN 2
+  The following additional DOMAIN_INDEX value is valid for i.MX6SX:
+  PCI_DOMAIN     3
 
 - #power-domain-cells: Should be 0
 
diff --git a/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.txt b/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.txt
index 02f45c65fd87..9acce75b29ab 100644
--- a/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.txt
+++ b/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.txt
@@ -44,10 +44,10 @@ Example:
 			#address-cells = <1>;
 			#size-cells = <0>;
 
-			pgc_pcie_phy: power-domain@3 {
+			pgc_pcie_phy: power-domain@1 {
 				#power-domain-cells = <0>;
 
-				reg = <IMX7_POWER_DOMAIN_PCIE_PHY>;
+				reg = <1>;
 				power-supply = <&reg_1p0d>;
 			};
 		};
diff --git a/Documentation/devicetree/bindings/power/mti,mips-cpc.txt b/Documentation/devicetree/bindings/power/mti,mips-cpc.txt
new file mode 100644
index 000000000000..c6b82511ae8a
--- /dev/null
+++ b/Documentation/devicetree/bindings/power/mti,mips-cpc.txt
@@ -0,0 +1,8 @@
+Binding for MIPS Cluster Power Controller (CPC).
+
+This binding allows a system to specify where the CPC registers are
+located.
+
+Required properties:
+compatible : Should be "mti,mips-cpc".
+regs: Should describe the address & size of the CPC register region.
diff --git a/Documentation/devicetree/bindings/power/power_domain.txt b/Documentation/devicetree/bindings/power/power_domain.txt
index 14bd9e945ff6..f3355313c020 100644
--- a/Documentation/devicetree/bindings/power/power_domain.txt
+++ b/Documentation/devicetree/bindings/power/power_domain.txt
@@ -40,6 +40,12 @@ Optional properties:
   domain's idle states. In the absence of this property, the domain would be
   considered as capable of being powered-on or powered-off.
 
+- operating-points-v2 : Phandles to the OPP tables of power domains provided by
+  a power domain provider. If the provider provides a single power domain only
+  or all the power domains provided by the provider have identical OPP tables,
+  then this shall contain a single phandle. Refer to ../opp/opp.txt for more
+  information.
+
 Example:
 
 	power: power-controller@12340000 {
@@ -120,4 +126,63 @@ The node above defines a typical PM domain consumer device, which is located
 inside a PM domain with index 0 of a power controller represented by a node
 with the label "power".
 
+Optional properties:
+- required-opp: This contains phandle to an OPP node in another device's OPP
+  table. It may contain an array of phandles, where each phandle points to an
+  OPP of a different device. It should not contain multiple phandles to the OPP
+  nodes in the same OPP table. This specifies the minimum required OPP of the
+  device(s), whose OPP's phandle is present in this property, for the
+  functioning of the current device at the current OPP (where this property is
+  present).
+
+Example:
+- OPP table for domain provider that provides two domains.
+
+	domain0_opp_table: opp-table0 {
+		compatible = "operating-points-v2";
+
+		domain0_opp_0: opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-microvolt = <975000 970000 985000>;
+		};
+		domain0_opp_1: opp-1100000000 {
+			opp-hz = /bits/ 64 <1100000000>;
+			opp-microvolt = <1000000 980000 1010000>;
+		};
+	};
+
+	domain1_opp_table: opp-table1 {
+		compatible = "operating-points-v2";
+
+		domain1_opp_0: opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <975000 970000 985000>;
+		};
+		domain1_opp_1: opp-1300000000 {
+			opp-hz = /bits/ 64 <1300000000>;
+			opp-microvolt = <1000000 980000 1010000>;
+		};
+	};
+
+	power: power-controller@12340000 {
+		compatible = "foo,power-controller";
+		reg = <0x12340000 0x1000>;
+		#power-domain-cells = <1>;
+		operating-points-v2 = <&domain0_opp_table>, <&domain1_opp_table>;
+	};
+
+	leaky-device0@12350000 {
+		compatible = "foo,i-leak-current";
+		reg = <0x12350000 0x1000>;
+		power-domains = <&power 0>;
+		required-opp = <&domain0_opp_0>;
+	};
+
+	leaky-device1@12350000 {
+		compatible = "foo,i-leak-current";
+		reg = <0x12350000 0x1000>;
+		power-domains = <&power 1>;
+		required-opp = <&domain1_opp_1>;
+	};
+
 [1]. Documentation/devicetree/bindings/power/domain-idle-state.txt
diff --git a/Documentation/devicetree/bindings/power/reset/imx-snvs-poweroff.txt b/Documentation/devicetree/bindings/power/reset/imx-snvs-poweroff.txt
deleted file mode 100644
index 1b81fcd9fb72..000000000000
--- a/Documentation/devicetree/bindings/power/reset/imx-snvs-poweroff.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-i.mx6 Poweroff Driver
-
-SNVS_LPCR in SNVS module can power off the whole system by pull
-PMIC_ON_REQ low if PMIC_ON_REQ is connected with external PMIC.
-If you don't want to use PMIC_ON_REQ as power on/off control,
-please set status='disabled' to disable this driver.
-
-Required Properties:
--compatible: "fsl,sec-v4.0-poweroff"
--reg: Specifies the physical address of the SNVS_LPCR register
-
-Example:
-	snvs@20cc000 {
-		compatible = "fsl,sec-v4.0-mon", "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <1>;
-		ranges = <0 0x020cc000 0x4000>;
-		.....
-		snvs_poweroff: snvs-poweroff@38 {
-			compatible = "fsl,sec-v4.0-poweroff";
-			reg = <0x38 0x4>;
-		};
-	}
diff --git a/Documentation/devicetree/bindings/power/supply/bq27xxx.txt b/Documentation/devicetree/bindings/power/supply/bq27xxx.txt
index 6858e1a804ad..615c1cb6889f 100644
--- a/Documentation/devicetree/bindings/power/supply/bq27xxx.txt
+++ b/Documentation/devicetree/bindings/power/supply/bq27xxx.txt
@@ -15,6 +15,7 @@ Required properties:
  * "ti,bq27520g2" - BQ27520-g2
  * "ti,bq27520g3" - BQ27520-g3
  * "ti,bq27520g4" - BQ27520-g4
+ * "ti,bq27521" - BQ27521
  * "ti,bq27530" - BQ27530
  * "ti,bq27531" - BQ27531
  * "ti,bq27541" - BQ27541
diff --git a/Documentation/devicetree/bindings/power/ti-smartreflex.txt b/Documentation/devicetree/bindings/power/ti-smartreflex.txt
index 9780957c9115..21ef14d6af12 100644
--- a/Documentation/devicetree/bindings/power/ti-smartreflex.txt
+++ b/Documentation/devicetree/bindings/power/ti-smartreflex.txt
@@ -7,7 +7,7 @@ Required properties:
 
 compatible: Shall be one of the following:
 	    "ti,omap3-smartreflex-core"
-	    "ti,omap3-smartreflex-iva"
+	    "ti,omap3-smartreflex-mpu-iva"
 	    "ti,omap4-smartreflex-core"
 	    "ti,omap4-smartreflex-mpu"
 	    "ti,omap4-smartreflex-iva"
diff --git a/Documentation/devicetree/bindings/power/wakeup-source.txt b/Documentation/devicetree/bindings/power/wakeup-source.txt
index 3c81f78b5c27..5d254ab13ebf 100644
--- a/Documentation/devicetree/bindings/power/wakeup-source.txt
+++ b/Documentation/devicetree/bindings/power/wakeup-source.txt
@@ -60,7 +60,7 @@ Examples
 		#size-cells = <0>;
 
 		button@1 {
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 			wakeup-source;
 			linux,code = <116>;
 			label = "POWER";
diff --git a/Documentation/devicetree/bindings/powerpc/fsl/interlaken-lac.txt b/Documentation/devicetree/bindings/powerpc/fsl/interlaken-lac.txt
index 641bc13983e1..ed6a414b2e10 100644
--- a/Documentation/devicetree/bindings/powerpc/fsl/interlaken-lac.txt
+++ b/Documentation/devicetree/bindings/powerpc/fsl/interlaken-lac.txt
@@ -223,37 +223,37 @@ lac-portals {
 		reg = <0x9000 0x1000>;
 	};
 
-	lportal10: lac-portal@A000 {
+	lportal10: lac-portal@a000 {
 		compatible = "fsl,interlaken-lac-portal-v1.0";
 		fsl,liodn = <0x20E>;
 		reg = <0xA000 0x1000>;
 	};
 
-	lportal11: lac-portal@B000 {
+	lportal11: lac-portal@b000 {
 		compatible = "fsl,interlaken-lac-portal-v1.0";
 		fsl,liodn = <0x20F>;
 		reg = <0xB000 0x1000>;
 	};
 
-	lportal12: lac-portal@C000 {
+	lportal12: lac-portal@c000 {
 		compatible = "fsl,interlaken-lac-portal-v1.0";
 		fsl,liodn = <0x210>;
 		reg = <0xC000 0x1000>;
 	};
 
-	lportal13: lac-portal@D000 {
+	lportal13: lac-portal@d000 {
 		compatible = "fsl,interlaken-lac-portal-v1.0";
 		fsl,liodn = <0x211>;
 		reg = <0xD000 0x1000>;
 	};
 
-	lportal14: lac-portal@E000 {
+	lportal14: lac-portal@e000 {
 		compatible = "fsl,interlaken-lac-portal-v1.0";
 		fsl,liodn = <0x212>;
 		reg = <0xE000 0x1000>;
 	};
 
-	lportal15: lac-portal@F000 {
+	lportal15: lac-portal@f000 {
 		compatible = "fsl,interlaken-lac-portal-v1.0";
 		fsl,liodn = <0x213>;
 		reg = <0xF000 0x1000>;
diff --git a/Documentation/devicetree/bindings/powerpc/fsl/mpc5200.txt b/Documentation/devicetree/bindings/powerpc/fsl/mpc5200.txt
index 4ccb2cd5df94..d096cf461d81 100644
--- a/Documentation/devicetree/bindings/powerpc/fsl/mpc5200.txt
+++ b/Documentation/devicetree/bindings/powerpc/fsl/mpc5200.txt
@@ -195,4 +195,4 @@ External interrupts:
 
 fsl,mpc5200-mscan nodes
 -----------------------
-See file can.txt in this directory.
+See file Documentation/devicetree/bindings/powerpc/fsl/mpc5200.txt
diff --git a/Documentation/devicetree/bindings/pwm/pwm-meson.txt b/Documentation/devicetree/bindings/pwm/pwm-meson.txt
index 1ee81321c35e..1fa3f7182133 100644
--- a/Documentation/devicetree/bindings/pwm/pwm-meson.txt
+++ b/Documentation/devicetree/bindings/pwm/pwm-meson.txt
@@ -5,6 +5,8 @@ Required properties:
 - compatible: Shall contain "amlogic,meson8b-pwm"
                          or "amlogic,meson-gxbb-pwm"
                          or "amlogic,meson-gxbb-ao-pwm"
+                         or "amlogic,meson-axg-ee-pwm"
+                         or "amlogic,meson-axg-ao-pwm"
 - #pwm-cells: Should be 3. See pwm.txt in this directory for a description of
   the cells format.
 
diff --git a/Documentation/devicetree/bindings/regulator/regulator.txt b/Documentation/devicetree/bindings/regulator/regulator.txt
index 3cbf56ce66ea..2babe15b618d 100644
--- a/Documentation/devicetree/bindings/regulator/regulator.txt
+++ b/Documentation/devicetree/bindings/regulator/regulator.txt
@@ -42,8 +42,16 @@ Optional properties:
 - regulator-state-[mem/disk] node has following common properties:
 	- regulator-on-in-suspend: regulator should be on in suspend state.
 	- regulator-off-in-suspend: regulator should be off in suspend state.
-	- regulator-suspend-microvolt: regulator should be set to this voltage
-	  in suspend.
+	- regulator-suspend-min-microvolt: minimum voltage may be set in
+	  suspend state.
+	- regulator-suspend-max-microvolt: maximum voltage may be set in
+	  suspend state.
+	- regulator-suspend-microvolt: the default voltage which regulator
+	  would be set in suspend. This property is now deprecated, instead
+	  setting voltage for suspend mode via the API which regulator
+	  driver provides is recommended.
+	- regulator-changeable-in-suspend: whether the default voltage and
+	  the regulator on/off in suspend can be changed in runtime.
 	- regulator-mode: operating mode in the given suspend state.
 	  The set of possible operating modes depends on the capabilities of
 	  every hardware so the valid modes are documented on each regulator
diff --git a/Documentation/devicetree/bindings/regulator/sprd,sc2731-regulator.txt b/Documentation/devicetree/bindings/regulator/sprd,sc2731-regulator.txt
new file mode 100644
index 000000000000..63dc07877cd6
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/sprd,sc2731-regulator.txt
@@ -0,0 +1,43 @@
+Spreadtrum SC2731 Voltage regulators
+
+The SC2731 integrates low-voltage and low quiescent current DCDC/LDO.
+14 LDO and 3 DCDCs are designed for external use. All DCDCs/LDOs have
+their own bypass (power-down) control signals. External tantalum or MLCC
+ceramic capacitors are recommended to use with these LDOs.
+
+Required properties:
+ - compatible: should be "sprd,sc27xx-regulator".
+
+List of regulators provided by this controller. It is named according to
+its regulator type, BUCK_<name> and LDO_<name>. The definition for each
+of these nodes is defined using the standard binding for regulators at
+Documentation/devicetree/bindings/regulator/regulator.txt.
+
+The valid names for regulators are:
+BUCK:
+	BUCK_CPU0, BUCK_CPU1, BUCK_RF
+LDO:
+	LDO_CAMA0, LDO_CAMA1, LDO_CAMMOT, LDO_VLDO, LDO_EMMCCORE, LDO_SDCORE,
+	LDO_SDIO, LDO_WIFIPA, LDO_USB33, LDO_CAMD0, LDO_CAMD1, LDO_CON,
+	LDO_CAMIO, LDO_SRAM
+
+Example:
+	regulators {
+		compatible = "sprd,sc27xx-regulator";
+
+		vddarm0: BUCK_CPU0 {
+			regulator-name = "vddarm0";
+			regulator-min-microvolt = <400000>;
+			regulator-max-microvolt = <1996875>;
+			regulator-ramp-delay = <25000>;
+			regulator-always-on;
+		};
+
+		vddcama0: LDO_CAMA0 {
+			regulator-name = "vddcama0";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <3750000>;
+			regulator-enable-ramp-delay = <100>;
+		};
+		...
+	};
diff --git a/Documentation/devicetree/bindings/regulator/st,stm32-vrefbuf.txt b/Documentation/devicetree/bindings/regulator/st,stm32-vrefbuf.txt
index 3944ee3e731e..5ddb8500a929 100644
--- a/Documentation/devicetree/bindings/regulator/st,stm32-vrefbuf.txt
+++ b/Documentation/devicetree/bindings/regulator/st,stm32-vrefbuf.txt
@@ -10,7 +10,7 @@ Required properties:
 - clocks:		Must contain an entry for peripheral clock.
 
 Example:
-	vrefbuf: regulator@58003C00 {
+	vrefbuf: regulator@58003c00 {
 		compatible = "st,stm32-vrefbuf";
 		reg = <0x58003C00 0x8>;
 		clocks = <&rcc VREF_CK>;
diff --git a/Documentation/devicetree/bindings/reset/amlogic,meson-reset.txt b/Documentation/devicetree/bindings/reset/amlogic,meson-reset.txt
index e746b631793a..28ef6c295c76 100644
--- a/Documentation/devicetree/bindings/reset/amlogic,meson-reset.txt
+++ b/Documentation/devicetree/bindings/reset/amlogic,meson-reset.txt
@@ -5,7 +5,8 @@ Please also refer to reset.txt in this directory for common reset
 controller binding usage.
 
 Required properties:
-- compatible: Should be "amlogic,meson8b-reset" or "amlogic,meson-gxbb-reset"
+- compatible: Should be "amlogic,meson8b-reset", "amlogic,meson-gxbb-reset" or
+	"amlogic,meson-axg-reset".
 - reg: should contain the register address base
 - #reset-cells: 1, see below
 
diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt
index 26542690b578..627b29531a32 100644
--- a/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt
+++ b/Documentation/devicetree/bindings/rng/brcm,bcm2835.txt
@@ -1,11 +1,19 @@
-BCM2835 Random number generator
+BCM2835/6368 Random number generator
 
 Required properties:
 
-- compatible : should be "brcm,bcm2835-rng"  or "brcm,bcm-nsp-rng" or
-  "brcm,bcm5301x-rng"
+- compatible : should be one of
+	"brcm,bcm2835-rng"
+	"brcm,bcm-nsp-rng"
+	"brcm,bcm5301x-rng" or
+	"brcm,bcm6368-rng"
 - reg : Specifies base physical address and size of the registers.
 
+Optional properties:
+
+- clocks : phandle to clock-controller plus clock-specifier pair
+- clock-names : "ipsec" as a clock name
+
 Example:
 
 rng {
@@ -17,3 +25,11 @@ rng@18033000 {
 	compatible = "brcm,bcm-nsp-rng";
 	reg = <0x18033000 0x14>;
 };
+
+random: rng@10004180 {
+	compatible = "brcm,bcm6368-rng";
+	reg = <0x10004180 0x14>;
+
+	clocks = <&periph_clk 18>;
+	clock-names = "ipsec";
+};
diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm6368.txt b/Documentation/devicetree/bindings/rng/brcm,bcm6368.txt
deleted file mode 100644
index 4b5ac600bfbd..000000000000
--- a/Documentation/devicetree/bindings/rng/brcm,bcm6368.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-BCM6368 Random number generator
-
-Required properties:
-
-- compatible : should be "brcm,bcm6368-rng"
-- reg : Specifies base physical address and size of the registers
-- clocks : phandle to clock-controller plus clock-specifier pair
-- clock-names : "ipsec" as a clock name
-
-Example:
-	random: rng@10004180 {
-		compatible = "brcm,bcm6368-rng";
-		reg = <0x10004180 0x14>;
-
-		clocks = <&periph_clk 18>;
-		clock-names = "ipsec";
-	};
diff --git a/Documentation/devicetree/bindings/rtc/rtc-mxc_v2.txt b/Documentation/devicetree/bindings/rtc/rtc-mxc_v2.txt
new file mode 100644
index 000000000000..79d7e87b0d91
--- /dev/null
+++ b/Documentation/devicetree/bindings/rtc/rtc-mxc_v2.txt
@@ -0,0 +1,17 @@
+* i.MX53 Secure Real Time Clock (SRTC)
+
+Required properties:
+- compatible: should be: "fsl,imx53-rtc"
+- reg: physical base address of the controller and length of memory mapped
+  region.
+- clocks: should contain the phandle for the rtc clock
+- interrupts: rtc alarm interrupt
+
+Example:
+
+rtc@53fa4000 {
+	compatible = "fsl,imx53-rtc";
+	reg = <0x53fa4000 0x4000>;
+	interrupts = <24>;
+	clocks = <&clks IMX5_CLK_SRTC_GATE>;
+};
diff --git a/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt b/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt
index b6a869f97715..df3bef7998fa 100644
--- a/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt
+++ b/Documentation/devicetree/bindings/scsi/hisilicon-sas.txt
@@ -8,7 +8,10 @@ Main node required properties:
 	(b) "hisilicon,hip06-sas-v2" for v2 hw in hip06 chipset
 	(c) "hisilicon,hip07-sas-v2" for v2 hw in hip07 chipset
   - sas-addr : array of 8 bytes for host SAS address
-  - reg : Address and length of the SAS register
+  - reg : Contains two regions. The first is the address and length of the SAS
+          register. The second is the address and length of CPLD register for
+          SGPIO control. The second is optional, and should be set only when
+          we use a CPLD for directly attached disk LED control.
   - hisilicon,sas-syscon: phandle of syscon used for sas control
   - ctrl-reset-reg : offset to controller reset register in ctrl reg
   - ctrl-reset-sts-reg : offset to controller reset status register in ctrl reg
diff --git a/Documentation/devicetree/bindings/serial/fsl-imx-uart.txt b/Documentation/devicetree/bindings/serial/fsl-imx-uart.txt
index 860a9559839a..afcfbc34e243 100644
--- a/Documentation/devicetree/bindings/serial/fsl-imx-uart.txt
+++ b/Documentation/devicetree/bindings/serial/fsl-imx-uart.txt
@@ -6,10 +6,10 @@ Required properties:
 - interrupts : Should contain uart interrupt
 
 Optional properties:
-- fsl,irda-mode : Indicate the uart supports irda mode
 - fsl,dte-mode : Indicate the uart works in DTE mode. The uart works
                   in DCE mode by default.
-- rs485-rts-delay, rs485-rx-during-tx, linux,rs485-enabled-at-boot-time: see rs485.txt
+- rs485-rts-delay, rs485-rts-active-low, rs485-rx-during-tx,
+  linux,rs485-enabled-at-boot-time: see rs485.txt
 
 Please check Documentation/devicetree/bindings/serial/serial.txt
 for the complete list of generic properties.
diff --git a/Documentation/devicetree/bindings/serial/fsl-lpuart.txt b/Documentation/devicetree/bindings/serial/fsl-lpuart.txt
index 59567b51cf09..6bd3f2e93d61 100644
--- a/Documentation/devicetree/bindings/serial/fsl-lpuart.txt
+++ b/Documentation/devicetree/bindings/serial/fsl-lpuart.txt
@@ -16,7 +16,8 @@ Required properties:
 Optional properties:
 - dmas: A list of two dma specifiers, one for each entry in dma-names.
 - dma-names: should contain "tx" and "rx".
-- rs485-rts-delay, rs485-rx-during-tx, linux,rs485-enabled-at-boot-time: see rs485.txt
+- rs485-rts-delay, rs485-rts-active-low, rs485-rx-during-tx,
+  linux,rs485-enabled-at-boot-time: see rs485.txt
 
 Note: Optional properties for DMA support. Write them both or both not.
 
diff --git a/Documentation/devicetree/bindings/serial/ingenic,uart.txt b/Documentation/devicetree/bindings/serial/ingenic,uart.txt
index 02cb7fe59cb7..c3c6406d5cfe 100644
--- a/Documentation/devicetree/bindings/serial/ingenic,uart.txt
+++ b/Documentation/devicetree/bindings/serial/ingenic,uart.txt
@@ -1,8 +1,12 @@
 * Ingenic SoC UART
 
 Required properties:
-- compatible : "ingenic,jz4740-uart", "ingenic,jz4760-uart",
-	"ingenic,jz4775-uart" or "ingenic,jz4780-uart"
+- compatible : One of:
+  - "ingenic,jz4740-uart",
+  - "ingenic,jz4760-uart",
+  - "ingenic,jz4770-uart",
+  - "ingenic,jz4775-uart",
+  - "ingenic,jz4780-uart".
 - reg : offset and length of the register set for the device.
 - interrupts : should contain uart interrupt.
 - clocks : phandles to the module & baud clocks.
diff --git a/Documentation/devicetree/bindings/serial/lantiq_asc.txt b/Documentation/devicetree/bindings/serial/lantiq_asc.txt
index 5b78591aaa46..3acbd309ab9d 100644
--- a/Documentation/devicetree/bindings/serial/lantiq_asc.txt
+++ b/Documentation/devicetree/bindings/serial/lantiq_asc.txt
@@ -8,7 +8,7 @@ Required properties:
 
 Example:
 
-asc1: serial@E100C00 {
+asc1: serial@e100c00 {
 	compatible = "lantiq,asc";
 	reg = <0xE100C00 0x400>;
 	interrupt-parent = <&icu0>;
diff --git a/Documentation/devicetree/bindings/serial/maxim,max310x.txt b/Documentation/devicetree/bindings/serial/maxim,max310x.txt
index 83a919c241b0..823f77dd7978 100644
--- a/Documentation/devicetree/bindings/serial/maxim,max310x.txt
+++ b/Documentation/devicetree/bindings/serial/maxim,max310x.txt
@@ -24,13 +24,27 @@ Optional properties:
     1 = active low.
 
 Example:
+
+/ {
+	clocks {
+		spi_uart_clk: osc_max14830 {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <3686400>;
+		};
+
+	};
+};
+
+&spi0 {
 	max14830: max14830@0 {
 		compatible = "maxim,max14830";
 		reg = <0>;
-		clocks = <&clk20m>;
+		clocks = <&spi_uart_clk>;
 		clock-names = "osc";
 		interrupt-parent = <&gpio3>;
-		interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
+		interrupts = <7 IRQ_TYPE_LEVEL_LOW>;
 		gpio-controller;
 		#gpio-cells = <2>;
 	};
+};
diff --git a/Documentation/devicetree/bindings/serial/mvebu-uart.txt b/Documentation/devicetree/bindings/serial/mvebu-uart.txt
index d37fabe17bd1..2ae2fee7e023 100644
--- a/Documentation/devicetree/bindings/serial/mvebu-uart.txt
+++ b/Documentation/devicetree/bindings/serial/mvebu-uart.txt
@@ -1,13 +1,53 @@
-* Marvell UART : Non standard UART used in some of Marvell EBU SoCs (e.g., Armada-3700)
+* Marvell UART : Non standard UART used in some of Marvell EBU SoCs
+                 e.g., Armada-3700.
 
 Required properties:
-- compatible: "marvell,armada-3700-uart"
+- compatible:
+    - "marvell,armada-3700-uart" for the standard variant of the UART
+      (32 bytes FIFO, no DMA, level interrupts, 8-bit access to the
+      FIFO, baudrate limited to 230400).
+    - "marvell,armada-3700-uart-ext" for the extended variant of the
+      UART (128 bytes FIFO, DMA, front interrupts, 8-bit or 32-bit
+      accesses to the FIFO, baudrate unlimited by the dividers).
 - reg: offset and length of the register set for the device.
-- interrupts: device interrupt
+- clocks: UART reference clock used to derive the baudrate. If no clock
+      is provided (possible only with the "marvell,armada-3700-uart"
+      compatible string for backward compatibility), it will only work
+      if the baudrate was initialized by the bootloader and no baudrate
+      change will then be possible.
+- interrupts:
+    - Must contain three elements for the standard variant of the IP
+      (marvell,armada-3700-uart): "uart-sum", "uart-tx" and "uart-rx",
+      respectively the UART sum interrupt, the UART TX interrupt and
+      UART RX interrupt. A corresponding interrupt-names property must
+      be defined.
+    - Must contain two elements for the extended variant of the IP
+      (marvell,armada-3700-uart-ext): "uart-tx" and "uart-rx",
+      respectively the UART TX interrupt and the UART RX interrupt. A
+      corresponding interrupts-names property must be defined.
+    - For backward compatibility reasons, a single element interrupts
+      property is also supported for the standard variant of the IP,
+      containing only the UART sum interrupt. This form is deprecated
+      and should no longer be used.
 
 Example:
-	serial@12000 {
+	uart0: serial@12000 {
 		compatible = "marvell,armada-3700-uart";
 		reg = <0x12000 0x200>;
-		interrupts = <43>;
+		clocks = <&xtalclk>;
+		interrupts =
+		<GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>,
+		<GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>,
+		<GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "uart-sum", "uart-tx", "uart-rx";
+	};
+
+	uart1: serial@12200 {
+		compatible = "marvell,armada-3700-uart-ext";
+		reg = <0x12200 0x30>;
+		clocks = <&xtalclk>;
+		interrupts =
+		<GIC_SPI 30 IRQ_TYPE_EDGE_RISING>,
+		<GIC_SPI 31 IRQ_TYPE_EDGE_RISING>;
+		interrupt-names = "uart-tx", "uart-rx";
 	};
diff --git a/Documentation/devicetree/bindings/serial/omap_serial.txt b/Documentation/devicetree/bindings/serial/omap_serial.txt
index 43eac675f21f..4b0f05adb228 100644
--- a/Documentation/devicetree/bindings/serial/omap_serial.txt
+++ b/Documentation/devicetree/bindings/serial/omap_serial.txt
@@ -20,6 +20,7 @@ Optional properties:
          node and a DMA channel number.
 - dma-names : "rx" for receive channel, "tx" for transmit channel.
 - rs485-rts-delay, rs485-rx-during-tx, linux,rs485-enabled-at-boot-time: see rs485.txt
+- rs485-rts-active-high: drive RTS high when sending (default is low).
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/serial/rs485.txt b/Documentation/devicetree/bindings/serial/rs485.txt
index b8415936dfdb..b7c29f74ebb2 100644
--- a/Documentation/devicetree/bindings/serial/rs485.txt
+++ b/Documentation/devicetree/bindings/serial/rs485.txt
@@ -12,6 +12,7 @@ Optional properties:
   * b is the delay between end of data sent and rts signal in milliseconds
       it corresponds to the delay after sending data and actual release of the line.
   If this property is not specified, <0 0> is assumed.
+- rs485-rts-active-low: drive RTS low when sending (default is high).
 - linux,rs485-enabled-at-boot-time: empty property telling to enable the rs485
   feature at boot time. It can be disabled later with proper ioctl.
 - rs485-rx-during-tx: empty property that enables the receiving of data even
diff --git a/Documentation/devicetree/bindings/siox/eckelmann,siox-gpio.txt b/Documentation/devicetree/bindings/siox/eckelmann,siox-gpio.txt
new file mode 100644
index 000000000000..55259cf39c25
--- /dev/null
+++ b/Documentation/devicetree/bindings/siox/eckelmann,siox-gpio.txt
@@ -0,0 +1,19 @@
+Eckelmann SIOX GPIO bus
+
+Required properties:
+- compatible : "eckelmann,siox-gpio"
+- din-gpios, dout-gpios, dclk-gpios, dld-gpios: references gpios for the
+    corresponding bus signals.
+
+Examples:
+
+        siox {
+                compatible = "eckelmann,siox-gpio";
+                pinctrl-names = "default";
+                pinctrl-0 = <&pinctrl_siox>;
+
+                din-gpios = <&gpio6 11 0>;
+                dout-gpios = <&gpio6 8 0>;
+                dclk-gpios = <&gpio6 9 0>;
+                dld-gpios = <&gpio6 10 0>;
+        };
diff --git a/Documentation/devicetree/bindings/slimbus/bus.txt b/Documentation/devicetree/bindings/slimbus/bus.txt
new file mode 100644
index 000000000000..52fa6426388c
--- /dev/null
+++ b/Documentation/devicetree/bindings/slimbus/bus.txt
@@ -0,0 +1,50 @@
+SLIM(Serial Low Power Interchip Media Bus) bus
+
+SLIMbus is a 2-wire bus, and is used to communicate with peripheral
+components like audio-codec.
+
+Required property for SLIMbus controller node:
+- compatible	- name of SLIMbus controller
+
+Child nodes:
+Every SLIMbus controller node can contain zero or more child nodes
+representing slave devices on the bus. Every SLIMbus slave device is
+uniquely determined by the enumeration address containing 4 fields:
+Manufacturer ID, Product code, Device index, and Instance value for
+the device.
+If child node is not present and it is instantiated after device
+discovery (slave device reporting itself present).
+
+In some cases it may be necessary to describe non-probeable device
+details such as non-standard ways of powering up a device. In
+such cases, child nodes for those devices will be present as
+slaves of the SLIMbus controller, as detailed below.
+
+Required property for SLIMbus child node if it is present:
+- reg		- Should be ('Device index', 'Instance ID') from SLIMbus
+		  Enumeration  Address.
+		  Device Index Uniquely identifies multiple Devices within
+		  a single Component.
+		  Instance ID Is for the cases where multiple Devices of the
+		  same type or Class are attached to the bus.
+
+- compatible	-"slimMID,PID". The textual representation of Manufacturer ID,
+	 	  Product Code, shall be in lower case hexadecimal with leading
+		  zeroes suppressed
+
+SLIMbus example for Qualcomm's slimbus manager component:
+
+	slim@28080000 {
+		compatible = "qcom,apq8064-slim", "qcom,slim";
+		reg = <0x28080000 0x2000>,
+		interrupts = <0 33 0>;
+		clocks = <&lcc SLIMBUS_SRC>, <&lcc AUDIO_SLIMBUS_CLK>;
+		clock-names = "iface", "core";
+		#address-cells = <2>;
+		#size-cell = <0>;
+
+		codec: wcd9310@1,0{
+			compatible = "slim217,60";
+			reg = <1 0>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/slimbus/slim-qcom-ctrl.txt b/Documentation/devicetree/bindings/slimbus/slim-qcom-ctrl.txt
new file mode 100644
index 000000000000..922dcb8ff24a
--- /dev/null
+++ b/Documentation/devicetree/bindings/slimbus/slim-qcom-ctrl.txt
@@ -0,0 +1,39 @@
+Qualcomm SLIMbus controller
+This controller is used if applications processor driver controls SLIMbus
+master component.
+
+Required properties:
+
+ - #address-cells - refer to Documentation/devicetree/bindings/slimbus/bus.txt
+ - #size-cells	- refer to Documentation/devicetree/bindings/slimbus/bus.txt
+
+ - reg : Offset and length of the register region(s) for the device
+ - reg-names : Register region name(s) referenced in reg above
+	 Required register resource entries are:
+	 "ctrl": Physical address of controller register blocks
+ 	 "slew": required for "qcom,apq8064-slim" SOC.
+ - compatible : should be "qcom,<SOC-NAME>-slim" for SOC specific compatible
+ 		followed by "qcom,slim" for fallback.
+ - interrupts : Interrupt number used by this controller
+ - clocks : Interface and core clocks used by this SLIMbus controller
+ - clock-names : Required clock-name entries are:
+	"iface" : Interface clock for this controller
+	"core" : Interrupt for controller core's BAM
+
+Example:
+
+	slim@28080000 {
+		compatible = "qcom,apq8064-slim", "qcom,slim";
+		reg = <0x28080000 0x2000>, <0x80207C 4>;
+		reg-names = "ctrl", "slew";
+		interrupts = <0 33 0>;
+		clocks = <&lcc SLIMBUS_SRC>, <&lcc AUDIO_SLIMBUS_CLK>;
+		clock-names = "iface", "core";
+		#address-cells = <2>;
+		#size-cell = <0>;
+
+		wcd9310: audio-codec@1,0{
+			compatible = "slim217,60";
+			reg = <1 0>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt b/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt
index 40056f7990f8..76bf45b893fa 100644
--- a/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt
+++ b/Documentation/devicetree/bindings/soc/mediatek/scpsys.txt
@@ -12,11 +12,13 @@ power/power_domain.txt. It provides the power domains defined in
 - include/dt-bindings/power/mt8173-power.h
 - include/dt-bindings/power/mt6797-power.h
 - include/dt-bindings/power/mt2701-power.h
+- include/dt-bindings/power/mt2712-power.h
 - include/dt-bindings/power/mt7622-power.h
 
 Required properties:
 - compatible: Should be one of:
 	- "mediatek,mt2701-scpsys"
+	- "mediatek,mt2712-scpsys"
 	- "mediatek,mt6797-scpsys"
 	- "mediatek,mt7622-scpsys"
 	- "mediatek,mt8173-scpsys"
@@ -27,6 +29,7 @@ Required properties:
                       These are clocks which hardware needs to be
                       enabled before enabling certain power domains.
 	Required clocks for MT2701: "mm", "mfg", "ethif"
+	Required clocks for MT2712: "mm", "mfg", "venc", "jpgdec", "audio", "vdec"
 	Required clocks for MT6797: "mm", "mfg", "vdec"
 	Required clocks for MT7622: "hif_sel"
 	Required clocks for MT8173: "mm", "mfg", "venc", "venc_lt"
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smp2p.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,smp2p.txt
index af9ca37221ce..a35af2dafdad 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,smp2p.txt
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smp2p.txt
@@ -17,9 +17,15 @@ processor ID) and a string identifier.
 	Value type: <prop-encoded-array>
 	Definition: one entry specifying the smp2p notification interrupt
 
-- qcom,ipc:
+- mboxes:
 	Usage: required
 	Value type: <prop-encoded-array>
+	Definition: reference to the associated doorbell in APCS, as described
+		    in mailbox/mailbox.txt
+
+- qcom,ipc:
+	Usage: required, unless mboxes is specified
+	Value type: <prop-encoded-array>
 	Definition: three entries specifying the outgoing ipc bit used for
 		    signaling the remote end of the smp2p edge:
 		    - phandle to a syscon node representing the apcs registers
diff --git a/Documentation/devicetree/bindings/soc/xilinx/xlnx,vcu.txt b/Documentation/devicetree/bindings/soc/xilinx/xlnx,vcu.txt
new file mode 100644
index 000000000000..6786d6715df0
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/xilinx/xlnx,vcu.txt
@@ -0,0 +1,31 @@
+LogicoreIP designed compatible with Xilinx ZYNQ family.
+-------------------------------------------------------
+
+General concept
+---------------
+
+LogicoreIP design to provide the isolation between processing system
+and programmable logic. Also provides the list of register set to configure
+the frequency.
+
+Required properties:
+- compatible: shall be one of:
+	"xlnx,vcu"
+	"xlnx,vcu-logicoreip-1.0"
+- reg, reg-names: There are two sets of registers need to provide.
+	1. vcu slcr
+	2. Logicore
+	reg-names should contain name for the each register sequence.
+- clocks: phandle for aclk and pll_ref clocksource
+- clock-names: The identification string, "aclk", is always required for
+   the axi clock. "pll_ref" is required for pll.
+Example:
+
+	xlnx_vcu: vcu@a0040000 {
+		compatible = "xlnx,vcu-logicoreip-1.0";
+		reg = <0x0 0xa0040000 0x0 0x1000>,
+			 <0x0 0xa0041000 0x0 0x1000>;
+		reg-names = "vcu_slcr", "logicore";
+		clocks = <&si570_1>, <&clkc 71>;
+		clock-names = "pll_ref", "aclk";
+	};
diff --git a/Documentation/devicetree/bindings/sound/dmic.txt b/Documentation/devicetree/bindings/sound/dmic.txt
index 54c8ef6498a8..f7bf65611453 100644
--- a/Documentation/devicetree/bindings/sound/dmic.txt
+++ b/Documentation/devicetree/bindings/sound/dmic.txt
@@ -7,10 +7,12 @@ Required properties:
 
 Optional properties:
 	- dmicen-gpios: GPIO specifier for dmic to control start and stop
+	- num-channels: Number of microphones on this DAI
 
 Example node:
 
 	dmic_codec: dmic@0 {
 		compatible = "dmic-codec";
 		dmicen-gpios = <&gpio4 3 GPIO_ACTIVE_HIGH>;
+		num-channels = <1>;
 	};
diff --git a/Documentation/devicetree/bindings/sound/img,i2s-out.txt b/Documentation/devicetree/bindings/sound/img,i2s-out.txt
index 0159415b3338..6b0ee9b7e11b 100644
--- a/Documentation/devicetree/bindings/sound/img,i2s-out.txt
+++ b/Documentation/devicetree/bindings/sound/img,i2s-out.txt
@@ -35,7 +35,7 @@ Optional Properties:
 
 Example:
 
-i2s_out: i2s-out@18100A00 {
+i2s_out: i2s-out@18100a00 {
 	compatible = "img,i2s-out";
 	reg = <0x18100A00 0x200>;
 	interrupts = <GIC_SHARED 13 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/Documentation/devicetree/bindings/sound/img,parallel-out.txt b/Documentation/devicetree/bindings/sound/img,parallel-out.txt
index a3015d2a06e0..37a3f94cc126 100644
--- a/Documentation/devicetree/bindings/sound/img,parallel-out.txt
+++ b/Documentation/devicetree/bindings/sound/img,parallel-out.txt
@@ -29,7 +29,7 @@ Optional Properties:
 
 Example:
 
-parallel_out: parallel-out@18100C00 {
+parallel_out: parallel-out@18100c00 {
 	compatible = "img,parallel-out";
 	reg = <0x18100C00 0x100>;
 	interrupts = <GIC_SHARED 19 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/Documentation/devicetree/bindings/sound/img,spdif-in.txt b/Documentation/devicetree/bindings/sound/img,spdif-in.txt
index aab9a81f7e13..f7ea8c87bf34 100644
--- a/Documentation/devicetree/bindings/sound/img,spdif-in.txt
+++ b/Documentation/devicetree/bindings/sound/img,spdif-in.txt
@@ -29,7 +29,7 @@ Optional Properties:
 
 Example:
 
-spdif_in: spdif-in@18100E00 {
+spdif_in: spdif-in@18100e00 {
 	compatible = "img,spdif-in";
 	reg = <0x18100E00 0x100>;
 	interrupts = <GIC_SHARED 20 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/Documentation/devicetree/bindings/sound/img,spdif-out.txt b/Documentation/devicetree/bindings/sound/img,spdif-out.txt
index 470a5191e101..413ed8b01870 100644
--- a/Documentation/devicetree/bindings/sound/img,spdif-out.txt
+++ b/Documentation/devicetree/bindings/sound/img,spdif-out.txt
@@ -29,7 +29,7 @@ Optional Properties:
 
 Example:
 
-spdif_out: spdif-out@18100D00 {
+spdif_out: spdif-out@18100d00 {
 	compatible = "img,spdif-out";
 	reg = <0x18100D00 0x100>;
 	interrupts = <GIC_SHARED 21 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/Documentation/devicetree/bindings/sound/max98373.txt b/Documentation/devicetree/bindings/sound/max98373.txt
new file mode 100644
index 000000000000..456cb1c59353
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/max98373.txt
@@ -0,0 +1,40 @@
+Maxim Integrated MAX98373 Speaker Amplifier
+
+This device supports I2C.
+
+Required properties:
+
+ - compatible : "maxim,max98373"
+
+ - reg : the I2C address of the device.
+
+Optional properties:
+
+  - maxim,vmon-slot-no : slot number used to send voltage information
+                   or in inteleave mode this will be used as
+                   interleave slot.
+                   slot range : 0 ~ 15,  Default : 0
+
+  - maxim,imon-slot-no : slot number used to send current information
+                   slot range : 0 ~ 15,  Default : 0
+
+  - maxim,spkfb-slot-no : slot number used to send speaker feedback information
+                   slot range : 0 ~ 15,  Default : 0
+
+  - maxim,interleave-mode : For cases where a single combined channel
+		   for the I/V sense data is not sufficient, the device can also be configured
+		   to share a single data output channel on alternating frames.
+		   In this configuration, the current and voltage data will be frame interleaved
+		   on a single output channel.
+                   Boolean, define to enable the interleave mode, Default : false
+
+Example:
+
+codec: max98373@31 {
+   compatible = "maxim,max98373";
+   reg = <0x31>;
+   maxim,vmon-slot-no = <0>;
+   maxim,imon-slot-no = <1>;
+   maxim,spkfb-slot-no = <2>;
+   maxim,interleave-mode;
+};
diff --git a/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt b/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt
index 77a57f84bed4..6df87b97f7cb 100644
--- a/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt
+++ b/Documentation/devicetree/bindings/sound/mt2701-afe-pcm.txt
@@ -2,153 +2,143 @@ Mediatek AFE PCM controller for mt2701
 
 Required properties:
 - compatible = "mediatek,mt2701-audio";
-- reg: register location and size
 - interrupts: should contain AFE and ASYS interrupts
 - interrupt-names: should be "afe" and "asys"
 - power-domains: should define the power domain
+- clocks: Must contain an entry for each entry in clock-names
+  See ../clocks/clock-bindings.txt for details
 - clock-names: should have these clock names:
 		"infra_sys_audio_clk",
 		"top_audio_mux1_sel",
 		"top_audio_mux2_sel",
-		"top_audio_mux1_div",
-		"top_audio_mux2_div",
-		"top_audio_48k_timing",
-		"top_audio_44k_timing",
-		"top_audpll_mux_sel",
-		"top_apll_sel",
-		"top_aud1_pll_98M",
-		"top_aud2_pll_90M",
-		"top_hadds2_pll_98M",
-		"top_hadds2_pll_294M",
-		"top_audpll",
-		"top_audpll_d4",
-		"top_audpll_d8",
-		"top_audpll_d16",
-		"top_audpll_d24",
-		"top_audintbus_sel",
-		"clk_26m",
-		"top_syspll1_d4",
-		"top_aud_k1_src_sel",
-		"top_aud_k2_src_sel",
-		"top_aud_k3_src_sel",
-		"top_aud_k4_src_sel",
-		"top_aud_k5_src_sel",
-		"top_aud_k6_src_sel",
-		"top_aud_k1_src_div",
-		"top_aud_k2_src_div",
-		"top_aud_k3_src_div",
-		"top_aud_k4_src_div",
-		"top_aud_k5_src_div",
-		"top_aud_k6_src_div",
-		"top_aud_i2s1_mclk",
-		"top_aud_i2s2_mclk",
-		"top_aud_i2s3_mclk",
-		"top_aud_i2s4_mclk",
-		"top_aud_i2s5_mclk",
-		"top_aud_i2s6_mclk",
-		"top_asm_m_sel",
-		"top_asm_h_sel",
-		"top_univpll2_d4",
-		"top_univpll2_d2",
-		"top_syspll_d5";
+		"top_audio_a1sys_hp",
+		"top_audio_a2sys_hp",
+		"i2s0_src_sel",
+		"i2s1_src_sel",
+		"i2s2_src_sel",
+		"i2s3_src_sel",
+		"i2s0_src_div",
+		"i2s1_src_div",
+		"i2s2_src_div",
+		"i2s3_src_div",
+		"i2s0_mclk_en",
+		"i2s1_mclk_en",
+		"i2s2_mclk_en",
+		"i2s3_mclk_en",
+		"i2so0_hop_ck",
+		"i2so1_hop_ck",
+		"i2so2_hop_ck",
+		"i2so3_hop_ck",
+		"i2si0_hop_ck",
+		"i2si1_hop_ck",
+		"i2si2_hop_ck",
+		"i2si3_hop_ck",
+		"asrc0_out_ck",
+		"asrc1_out_ck",
+		"asrc2_out_ck",
+		"asrc3_out_ck",
+		"audio_afe_pd",
+		"audio_afe_conn_pd",
+		"audio_a1sys_pd",
+		"audio_a2sys_pd",
+		"audio_mrgif_pd";
+- assigned-clocks: list of input clocks and dividers for the audio system.
+		   See ../clocks/clock-bindings.txt for details.
+- assigned-clocks-parents: parent of input clocks of assigned clocks.
+- assigned-clock-rates: list of clock frequencies of assigned clocks.
+
+Must be a subnode of MediaTek audsys device tree node.
+See ../arm/mediatek/mediatek,audsys.txt for details about the parent node.
 
 Example:
 
-	afe: mt2701-afe-pcm@11220000 {
-		compatible = "mediatek,mt2701-audio";
-		reg = <0 0x11220000 0 0x2000>,
-		      <0 0x112A0000 0 0x20000>;
-		interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>,
-			     <GIC_SPI 132 IRQ_TYPE_LEVEL_LOW>;
-		interrupt-names	= "afe", "asys";
-		power-domains = <&scpsys MT2701_POWER_DOMAIN_IFR_MSC>;
-		clocks = <&infracfg CLK_INFRA_AUDIO>,
-			 <&topckgen CLK_TOP_AUD_MUX1_SEL>,
-			 <&topckgen CLK_TOP_AUD_MUX2_SEL>,
-			 <&topckgen CLK_TOP_AUD_MUX1_DIV>,
-			 <&topckgen CLK_TOP_AUD_MUX2_DIV>,
-			 <&topckgen CLK_TOP_AUD_48K_TIMING>,
-			 <&topckgen CLK_TOP_AUD_44K_TIMING>,
-			 <&topckgen CLK_TOP_AUDPLL_MUX_SEL>,
-			 <&topckgen CLK_TOP_APLL_SEL>,
-			 <&topckgen CLK_TOP_AUD1PLL_98M>,
-			 <&topckgen CLK_TOP_AUD2PLL_90M>,
-			 <&topckgen CLK_TOP_HADDS2PLL_98M>,
-			 <&topckgen CLK_TOP_HADDS2PLL_294M>,
-			 <&topckgen CLK_TOP_AUDPLL>,
-			 <&topckgen CLK_TOP_AUDPLL_D4>,
-			 <&topckgen CLK_TOP_AUDPLL_D8>,
-			 <&topckgen CLK_TOP_AUDPLL_D16>,
-			 <&topckgen CLK_TOP_AUDPLL_D24>,
-			 <&topckgen CLK_TOP_AUDINTBUS_SEL>,
-			 <&clk26m>,
-			 <&topckgen CLK_TOP_SYSPLL1_D4>,
-			 <&topckgen CLK_TOP_AUD_K1_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K2_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K3_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K4_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K5_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K6_SRC_SEL>,
-			 <&topckgen CLK_TOP_AUD_K1_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K2_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K3_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K4_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K5_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_K6_SRC_DIV>,
-			 <&topckgen CLK_TOP_AUD_I2S1_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S2_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S3_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S4_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S5_MCLK>,
-			 <&topckgen CLK_TOP_AUD_I2S6_MCLK>,
-			 <&topckgen CLK_TOP_ASM_M_SEL>,
-			 <&topckgen CLK_TOP_ASM_H_SEL>,
-			 <&topckgen CLK_TOP_UNIVPLL2_D4>,
-			 <&topckgen CLK_TOP_UNIVPLL2_D2>,
-			 <&topckgen CLK_TOP_SYSPLL_D5>;
+	audsys: audio-subsystem@11220000 {
+		compatible = "mediatek,mt2701-audsys", "syscon", "simple-mfd";
+		...
+
+		afe: audio-controller {
+			compatible = "mediatek,mt2701-audio";
+			interrupts =  <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>,
+				      <GIC_SPI 132 IRQ_TYPE_LEVEL_LOW>;
+			interrupt-names	= "afe", "asys";
+			power-domains = <&scpsys MT2701_POWER_DOMAIN_IFR_MSC>;
+
+			clocks = <&infracfg CLK_INFRA_AUDIO>,
+				 <&topckgen CLK_TOP_AUD_MUX1_SEL>,
+				 <&topckgen CLK_TOP_AUD_MUX2_SEL>,
+				 <&topckgen CLK_TOP_AUD_48K_TIMING>,
+				 <&topckgen CLK_TOP_AUD_44K_TIMING>,
+				 <&topckgen CLK_TOP_AUD_K1_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K2_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K3_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K4_SRC_SEL>,
+				 <&topckgen CLK_TOP_AUD_K1_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K2_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K3_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_K4_SRC_DIV>,
+				 <&topckgen CLK_TOP_AUD_I2S1_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S2_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S3_MCLK>,
+				 <&topckgen CLK_TOP_AUD_I2S4_MCLK>,
+				 <&audsys CLK_AUD_I2SO1>,
+				 <&audsys CLK_AUD_I2SO2>,
+				 <&audsys CLK_AUD_I2SO3>,
+				 <&audsys CLK_AUD_I2SO4>,
+				 <&audsys CLK_AUD_I2SIN1>,
+				 <&audsys CLK_AUD_I2SIN2>,
+				 <&audsys CLK_AUD_I2SIN3>,
+				 <&audsys CLK_AUD_I2SIN4>,
+				 <&audsys CLK_AUD_ASRCO1>,
+				 <&audsys CLK_AUD_ASRCO2>,
+				 <&audsys CLK_AUD_ASRCO3>,
+				 <&audsys CLK_AUD_ASRCO4>,
+				 <&audsys CLK_AUD_AFE>,
+				 <&audsys CLK_AUD_AFE_CONN>,
+				 <&audsys CLK_AUD_A1SYS>,
+				 <&audsys CLK_AUD_A2SYS>,
+				 <&audsys CLK_AUD_AFE_MRGIF>;
+
+			clock-names = "infra_sys_audio_clk",
+				      "top_audio_mux1_sel",
+				      "top_audio_mux2_sel",
+				      "top_audio_a1sys_hp",
+				      "top_audio_a2sys_hp",
+				      "i2s0_src_sel",
+				      "i2s1_src_sel",
+				      "i2s2_src_sel",
+				      "i2s3_src_sel",
+				      "i2s0_src_div",
+				      "i2s1_src_div",
+				      "i2s2_src_div",
+				      "i2s3_src_div",
+				      "i2s0_mclk_en",
+				      "i2s1_mclk_en",
+				      "i2s2_mclk_en",
+				      "i2s3_mclk_en",
+				      "i2so0_hop_ck",
+				      "i2so1_hop_ck",
+				      "i2so2_hop_ck",
+				      "i2so3_hop_ck",
+				      "i2si0_hop_ck",
+				      "i2si1_hop_ck",
+				      "i2si2_hop_ck",
+				      "i2si3_hop_ck",
+				      "asrc0_out_ck",
+				      "asrc1_out_ck",
+				      "asrc2_out_ck",
+				      "asrc3_out_ck",
+				      "audio_afe_pd",
+				      "audio_afe_conn_pd",
+				      "audio_a1sys_pd",
+				      "audio_a2sys_pd",
+				      "audio_mrgif_pd";
 
-		clock-names = "infra_sys_audio_clk",
-			      "top_audio_mux1_sel",
-			      "top_audio_mux2_sel",
-			      "top_audio_mux1_div",
-			      "top_audio_mux2_div",
-			      "top_audio_48k_timing",
-			      "top_audio_44k_timing",
-			      "top_audpll_mux_sel",
-			      "top_apll_sel",
-			      "top_aud1_pll_98M",
-			      "top_aud2_pll_90M",
-			      "top_hadds2_pll_98M",
-			      "top_hadds2_pll_294M",
-			      "top_audpll",
-			      "top_audpll_d4",
-			      "top_audpll_d8",
-			      "top_audpll_d16",
-			      "top_audpll_d24",
-			      "top_audintbus_sel",
-			      "clk_26m",
-			      "top_syspll1_d4",
-			      "top_aud_k1_src_sel",
-			      "top_aud_k2_src_sel",
-			      "top_aud_k3_src_sel",
-			      "top_aud_k4_src_sel",
-			      "top_aud_k5_src_sel",
-			      "top_aud_k6_src_sel",
-			      "top_aud_k1_src_div",
-			      "top_aud_k2_src_div",
-			      "top_aud_k3_src_div",
-			      "top_aud_k4_src_div",
-			      "top_aud_k5_src_div",
-			      "top_aud_k6_src_div",
-			      "top_aud_i2s1_mclk",
-			      "top_aud_i2s2_mclk",
-			      "top_aud_i2s3_mclk",
-			      "top_aud_i2s4_mclk",
-			      "top_aud_i2s5_mclk",
-			      "top_aud_i2s6_mclk",
-			      "top_asm_m_sel",
-			      "top_asm_h_sel",
-			      "top_univpll2_d4",
-			      "top_univpll2_d2",
-			      "top_syspll_d5";
+			assigned-clocks = <&topckgen CLK_TOP_AUD_MUX1_SEL>,
+					  <&topckgen CLK_TOP_AUD_MUX2_SEL>,
+					  <&topckgen CLK_TOP_AUD_MUX1_DIV>,
+					  <&topckgen CLK_TOP_AUD_MUX2_DIV>;
+			assigned-clock-parents = <&topckgen CLK_TOP_AUD1PLL_98M>,
+						 <&topckgen CLK_TOP_AUD2PLL_90M>;
+			assigned-clock-rates = <0>, <0>, <49152000>, <45158400>;
+		};
 	};
diff --git a/Documentation/devicetree/bindings/sound/mxs-audio-sgtl5000.txt b/Documentation/devicetree/bindings/sound/mxs-audio-sgtl5000.txt
index 601c518eddaa..4eb980bd0287 100644
--- a/Documentation/devicetree/bindings/sound/mxs-audio-sgtl5000.txt
+++ b/Documentation/devicetree/bindings/sound/mxs-audio-sgtl5000.txt
@@ -1,10 +1,31 @@
 * Freescale MXS audio complex with SGTL5000 codec
 
 Required properties:
-- compatible: "fsl,mxs-audio-sgtl5000"
-- model: The user-visible name of this sound complex
-- saif-controllers: The phandle list of the MXS SAIF controller
-- audio-codec: The phandle of the SGTL5000 audio codec
+- compatible		: "fsl,mxs-audio-sgtl5000"
+- model			: The user-visible name of this sound complex
+- saif-controllers	: The phandle list of the MXS SAIF controller
+- audio-codec		: The phandle of the SGTL5000 audio codec
+- audio-routing		: A list of the connections between audio components.
+			  Each entry is a pair of strings, the first being the
+			  connection's sink, the second being the connection's
+			  source. Valid names could be power supplies, SGTL5000
+			  pins, and the jacks on the board:
+
+			  Power supplies:
+			   * Mic Bias
+
+			  SGTL5000 pins:
+			   * MIC_IN
+			   * LINE_IN
+			   * HP_OUT
+			   * LINE_OUT
+
+			  Board connectors:
+			   * Mic Jack
+			   * Line In Jack
+			   * Headphone Jack
+			   * Line Out Jack
+			   * Ext Spk
 
 Example:
 
@@ -14,4 +35,8 @@ sound {
 	model = "imx28-evk-sgtl5000";
 	saif-controllers = <&saif0 &saif1>;
 	audio-codec = <&sgtl5000>;
+	audio-routing =
+		"MIC_IN", "Mic Jack",
+		"Mic Jack", "Mic Bias",
+		"Headphone Jack", "HP_OUT";
 };
diff --git a/Documentation/devicetree/bindings/sound/nau8825.txt b/Documentation/devicetree/bindings/sound/nau8825.txt
index 2f5e973285a6..d16d96839bcb 100644
--- a/Documentation/devicetree/bindings/sound/nau8825.txt
+++ b/Documentation/devicetree/bindings/sound/nau8825.txt
@@ -69,7 +69,7 @@ Optional properties:
   - nuvoton,jack-insert-debounce: number from 0 to 7 that sets debounce time to 2^(n+2) ms
   - nuvoton,jack-eject-debounce: number from 0 to 7 that sets debounce time to 2^(n+2) ms
 
-  - nuvoton,crosstalk-bypass: make crosstalk function bypass if set.
+  - nuvoton,crosstalk-enable: make crosstalk function enable if set.
 
   - clocks: list of phandle and clock specifier pairs according to common clock bindings for the
       clocks described in clock-names
@@ -98,7 +98,7 @@ Example:
       nuvoton,short-key-debounce = <2>;
       nuvoton,jack-insert-debounce = <7>;
       nuvoton,jack-eject-debounce = <7>;
-      nuvoton,crosstalk-bypass;
+      nuvoton,crosstalk-enable;
 
       clock-names = "mclk";
       clocks = <&tegra_car TEGRA210_CLK_CLK_OUT_2>;
diff --git a/Documentation/devicetree/bindings/sound/pcm186x.txt b/Documentation/devicetree/bindings/sound/pcm186x.txt
new file mode 100644
index 000000000000..1087f4855980
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/pcm186x.txt
@@ -0,0 +1,42 @@
+Texas Instruments PCM186x Universal Audio ADC
+
+These devices support both I2C and SPI (configured with pin strapping
+on the board).
+
+Required properties:
+
+ - compatible : "ti,pcm1862",
+                "ti,pcm1863",
+                "ti,pcm1864",
+                "ti,pcm1865"
+
+ - reg : The I2C address of the device for I2C, the chip select
+         number for SPI.
+
+ - avdd-supply: Analog core power supply (3.3v)
+ - dvdd-supply: Digital core power supply
+ - iovdd-supply: Digital IO power supply
+        See regulator/regulator.txt for more information
+
+CODEC input pins:
+ * VINL1
+ * VINR1
+ * VINL2
+ * VINR2
+ * VINL3
+ * VINR3
+ * VINL4
+ * VINR4
+
+The pins can be used in referring sound node's audio-routing property.
+
+Example:
+
+	pcm186x: audio-codec@4a {
+		compatible = "ti,pcm1865";
+		reg = <0x4a>;
+
+		avdd-supply = <&reg_3v3_analog>;
+		dvdd-supply = <&reg_3v3>;
+		iovdd-supply = <&reg_1v8>;
+	};
diff --git a/Documentation/devicetree/bindings/sound/renesas,rsnd.txt b/Documentation/devicetree/bindings/sound/renesas,rsnd.txt
index 085bec364caf..5bed9a595772 100644
--- a/Documentation/devicetree/bindings/sound/renesas,rsnd.txt
+++ b/Documentation/devicetree/bindings/sound/renesas,rsnd.txt
@@ -4,7 +4,7 @@ Renesas R-Car sound
 * Modules
 =============================================
 
-Renesas R-Car sound is constructed from below modules
+Renesas R-Car and RZ/G sound is constructed from below modules
 (for Gen2 or later)
 
  SCU		: Sampling Rate Converter Unit
@@ -197,12 +197,17 @@ Ex)
 	[MEM] -> [SRC2] -> [CTU03] -+
 
 	sound {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
 		compatible = "simple-scu-audio-card";
 		...
-		simple-audio-card,cpu-0 {
+		simple-audio-card,cpu@0 {
+			reg = <0>;
 			sound-dai = <&rcar_sound 0>;
 		};
-		simple-audio-card,cpu-1 {
+		simple-audio-card,cpu@1 {
+			reg = <1>;
 			sound-dai = <&rcar_sound 1>;
 		};
 		simple-audio-card,codec {
@@ -334,9 +339,11 @@ Required properties:
 
 - compatible			: "renesas,rcar_sound-<soctype>", fallbacks
 				  "renesas,rcar_sound-gen1" if generation1, and
-				  "renesas,rcar_sound-gen2" if generation2
+				  "renesas,rcar_sound-gen2" if generation2 (or RZ/G1)
 				  "renesas,rcar_sound-gen3" if generation3
 				  Examples with soctypes are:
+				    - "renesas,rcar_sound-r8a7743" (RZ/G1M)
+				    - "renesas,rcar_sound-r8a7745" (RZ/G1E)
 				    - "renesas,rcar_sound-r8a7778" (R-Car M1A)
 				    - "renesas,rcar_sound-r8a7779" (R-Car H1)
 				    - "renesas,rcar_sound-r8a7790" (R-Car H2)
diff --git a/Documentation/devicetree/bindings/sound/simple-card.txt b/Documentation/devicetree/bindings/sound/simple-card.txt
index 166f2290233b..17c13e74667d 100644
--- a/Documentation/devicetree/bindings/sound/simple-card.txt
+++ b/Documentation/devicetree/bindings/sound/simple-card.txt
@@ -140,6 +140,7 @@ sound {
 	simple-audio-card,name = "Cubox Audio";
 
 	simple-audio-card,dai-link@0 {		/* I2S - HDMI */
+		reg = <0>;
 		format = "i2s";
 		cpu {
 			sound-dai = <&audio1 0>;
@@ -150,6 +151,7 @@ sound {
 	};
 
 	simple-audio-card,dai-link@1 {		/* S/PDIF - HDMI */
+		reg = <1>;
 		cpu {
 			sound-dai = <&audio1 1>;
 		};
@@ -159,6 +161,7 @@ sound {
 	};
 
 	simple-audio-card,dai-link@2 {		/* S/PDIF - S/PDIF */
+		reg = <2>;
 		cpu {
 			sound-dai = <&audio1 1>;
 		};
diff --git a/Documentation/devicetree/bindings/sound/st,sti-asoc-card.txt b/Documentation/devicetree/bindings/sound/st,sti-asoc-card.txt
index 9c1ee52fed5b..4d51f3f5ea98 100644
--- a/Documentation/devicetree/bindings/sound/st,sti-asoc-card.txt
+++ b/Documentation/devicetree/bindings/sound/st,sti-asoc-card.txt
@@ -51,7 +51,7 @@ Optional properties:
 
 Example:
 
-	sti_uni_player1: sti-uni-player@8D81000 {
+	sti_uni_player1: sti-uni-player@8d81000 {
 		compatible = "st,stih407-uni-player-hdmi";
 		#sound-dai-cells = <0>;
 		st,syscfg = <&syscfg_core>;
@@ -63,7 +63,7 @@ Example:
 		st,tdm-mode = <1>;
 	};
 
-	sti_uni_player2: sti-uni-player@8D82000 {
+	sti_uni_player2: sti-uni-player@8d82000 {
 		compatible = "st,stih407-uni-player-pcm-out";
 		#sound-dai-cells = <0>;
 		st,syscfg = <&syscfg_core>;
@@ -74,7 +74,7 @@ Example:
 		dma-names = "tx";
 	};
 
-	sti_uni_player3: sti-uni-player@8D85000 {
+	sti_uni_player3: sti-uni-player@8d85000 {
 		compatible = "st,stih407-uni-player-spdif";
 		#sound-dai-cells = <0>;
 		st,syscfg = <&syscfg_core>;
@@ -85,7 +85,7 @@ Example:
 		dma-names = "tx";
 	};
 
-	sti_uni_reader1: sti-uni-reader@8D84000 {
+	sti_uni_reader1: sti-uni-reader@8d84000 {
 		compatible = "st,stih407-uni-reader-hdmi";
 		#sound-dai-cells = <0>;
 		st,syscfg = <&syscfg_core>;
diff --git a/Documentation/devicetree/bindings/sound/st,stm32-adfsdm.txt b/Documentation/devicetree/bindings/sound/st,stm32-adfsdm.txt
new file mode 100644
index 000000000000..864f5b00b031
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/st,stm32-adfsdm.txt
@@ -0,0 +1,63 @@
+STMicroelectronics Audio Digital Filter Sigma Delta modulators(DFSDM)
+
+The DFSDM allows PDM microphones capture through SPI interface. The Audio
+interface is seems as a sub block of the DFSDM device.
+For details on DFSDM bindings refer to ../iio/adc/st,stm32-dfsdm-adc.txt
+
+Required properties:
+  - compatible: "st,stm32h7-dfsdm-dai".
+
+  - #sound-dai-cells : Must be equal to 0
+
+  - io-channels : phandle to iio dfsdm instance node.
+
+Example of a sound card using audio DFSDM node.
+
+	sound_card {
+		compatible = "audio-graph-card";
+
+		dais = <&cpu_port>;
+	};
+
+	dfsdm: dfsdm@40017000 {
+		compatible = "st,stm32h7-dfsdm";
+		reg = <0x40017000 0x400>;
+		clocks = <&rcc DFSDM1_CK>;
+		clock-names = "dfsdm";
+		#interrupt-cells = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		dfsdm_adc0: filter@0 {
+			compatible = "st,stm32-dfsdm-dmic";
+			reg = <0>;
+			interrupts = <110>;
+			dmas = <&dmamux1 101 0x400 0x00>;
+			dma-names = "rx";
+			st,adc-channels = <1>;
+			st,adc-channel-names = "dmic0";
+			st,adc-channel-types = "SPI_R";
+			st,adc-channel-clk-src = "CLKOUT";
+			st,filter-order = <5>;
+
+			dfsdm_dai0: dfsdm-dai {
+				compatible = "st,stm32h7-dfsdm-dai";
+				#sound-dai-cells = <0>;
+				io-channels = <&dfsdm_adc0 0>;
+				cpu_port: port {
+				dfsdm_endpoint: endpoint {
+					remote-endpoint = <&dmic0_endpoint>;
+				};
+			};
+		};
+	};
+
+	dmic0: dmic@0 {
+		compatible = "dmic-codec";
+		#sound-dai-cells = <0>;
+		port {
+			dmic0_endpoint: endpoint {
+				remote-endpoint = <&dfsdm_endpoint>;
+			};
+		};
+	};
diff --git a/Documentation/devicetree/bindings/sound/st,stm32-sai.txt b/Documentation/devicetree/bindings/sound/st,stm32-sai.txt
index 1f9cd7095337..b1acc1a256ba 100644
--- a/Documentation/devicetree/bindings/sound/st,stm32-sai.txt
+++ b/Documentation/devicetree/bindings/sound/st,stm32-sai.txt
@@ -20,11 +20,6 @@ Required properties:
 
 Optional properties:
   - resets: Reference to a reset controller asserting the SAI
-  - st,sync: specify synchronization mode.
-	By default SAI sub-block is in asynchronous mode.
-	This property sets SAI sub-block as slave of another SAI sub-block.
-	Must contain the phandle and index of the sai sub-block providing
-	the synchronization.
 
 SAI subnodes:
 Two subnodes corresponding to SAI sub-block instances A et B can be defined.
@@ -44,6 +39,13 @@ SAI subnodes required properties:
   - pinctrl-names: should contain only value "default"
   - pinctrl-0: see Documentation/devicetree/bindings/pinctrl/pinctrl-stm32.txt
 
+SAI subnodes Optional properties:
+  - st,sync: specify synchronization mode.
+	By default SAI sub-block is in asynchronous mode.
+	This property sets SAI sub-block as slave of another SAI sub-block.
+	Must contain the phandle and index of the sai sub-block providing
+	the synchronization.
+
 The device node should contain one 'port' child node with one child 'endpoint'
 node, according to the bindings defined in Documentation/devicetree/bindings/
 graph.txt.
diff --git a/Documentation/devicetree/bindings/sound/sun4i-i2s.txt b/Documentation/devicetree/bindings/sound/sun4i-i2s.txt
index 05d7135a8d2f..b9d50d6cdef3 100644
--- a/Documentation/devicetree/bindings/sound/sun4i-i2s.txt
+++ b/Documentation/devicetree/bindings/sound/sun4i-i2s.txt
@@ -8,6 +8,7 @@ Required properties:
 - compatible: should be one of the following:
    - "allwinner,sun4i-a10-i2s"
    - "allwinner,sun6i-a31-i2s"
+   - "allwinner,sun8i-a83t-i2s"
    - "allwinner,sun8i-h3-i2s"
 - reg: physical base address of the controller and length of memory mapped
   region.
@@ -23,6 +24,7 @@ Required properties:
 
 Required properties for the following compatibles:
 	- "allwinner,sun6i-a31-i2s"
+	- "allwinner,sun8i-a83t-i2s"
 	- "allwinner,sun8i-h3-i2s"
 - resets: phandle to the reset line for this codec
 
diff --git a/Documentation/devicetree/bindings/sound/tas5720.txt b/Documentation/devicetree/bindings/sound/tas5720.txt
index 40d94f82beb3..7481653fe8e3 100644
--- a/Documentation/devicetree/bindings/sound/tas5720.txt
+++ b/Documentation/devicetree/bindings/sound/tas5720.txt
@@ -6,10 +6,12 @@ audio playback. For more product information please see the links below:
 
 http://www.ti.com/product/TAS5720L
 http://www.ti.com/product/TAS5720M
+http://www.ti.com/product/TAS5722L
 
 Required properties:
 
-- compatible : "ti,tas5720"
+- compatible : "ti,tas5720",
+               "ti,tas5722"
 - reg : I2C slave address
 - dvdd-supply : phandle to a 3.3-V supply for the digital circuitry
 - pvdd-supply : phandle to a supply used for the Class-D amp and the analog
diff --git a/Documentation/devicetree/bindings/sound/tfa9879.txt b/Documentation/devicetree/bindings/sound/tfa9879.txt
index 23ba522d9e2b..1620e6848436 100644
--- a/Documentation/devicetree/bindings/sound/tfa9879.txt
+++ b/Documentation/devicetree/bindings/sound/tfa9879.txt
@@ -6,18 +6,18 @@ Required properties:
 
 - reg : the I2C address of the device
 
+- #sound-dai-cells : must be 0.
+
 Example:
 
 &i2c1 {
-	clock-frequency = <100000>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_i2c1>;
-	status = "okay";
 
-	codec: tfa9879@6c {
+	amp: amp@6c {
 		#sound-dai-cells = <0>;
 		compatible = "nxp,tfa9879";
 		reg = <0x6c>;
-        };
+	};
 };
 
diff --git a/Documentation/devicetree/bindings/sound/ti,tas6424.txt b/Documentation/devicetree/bindings/sound/ti,tas6424.txt
new file mode 100644
index 000000000000..1c4ada0eef4e
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/ti,tas6424.txt
@@ -0,0 +1,20 @@
+Texas Instruments TAS6424 Quad-Channel Audio amplifier
+
+The TAS6424 serial control bus communicates through I2C protocols.
+
+Required properties:
+	- compatible: "ti,tas6424" - TAS6424
+	- reg: I2C slave address
+	- sound-dai-cells: must be equal to 0
+
+Example:
+
+tas6424: tas6424@6a {
+	compatible = "ti,tas6424";
+	reg = <0x6a>;
+
+	#sound-dai-cells = <0>;
+};
+
+For more product information please see the link below:
+http://www.ti.com/product/TAS6424-Q1
diff --git a/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt b/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt
index 6fbba562eaa7..5b3c33bb99e5 100644
--- a/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt
+++ b/Documentation/devicetree/bindings/sound/tlv320aic31xx.txt
@@ -22,7 +22,7 @@ Required properties:
 
 Optional properties:
 
-- gpio-reset - gpio pin number used for codec reset
+- reset-gpios - GPIO specification for the active low RESET input.
 - ai31xx-micbias-vg - MicBias Voltage setting
         1 or MICBIAS_2_0V - MICBIAS output is powered to 2.0V
         2 or MICBIAS_2_5V - MICBIAS output is powered to 2.5V
@@ -30,6 +30,10 @@ Optional properties:
 	If this node is not mentioned or if the value is unknown, then
 	micbias	is set to 2.0V.
 
+Deprecated properties:
+
+- gpio-reset - gpio pin number used for codec reset
+
 CODEC output pins:
   * HPL
   * HPR
@@ -48,6 +52,7 @@ CODEC input pins:
 The pins can be used in referring sound node's audio-routing property.
 
 Example:
+#include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/sound/tlv320aic31xx-micbias.h>
 
 tlv320aic31xx: tlv320aic31xx@18 {
@@ -56,6 +61,8 @@ tlv320aic31xx: tlv320aic31xx@18 {
 
 	ai31xx-micbias-vg = <MICBIAS_OFF>;
 
+	reset-gpios = <&gpio1 17 GPIO_ACTIVE_LOW>;
+
 	HPVDD-supply = <&regulator>;
 	SPRVDD-supply = <&regulator>;
 	SPLVDD-supply = <&regulator>;
diff --git a/Documentation/devicetree/bindings/sound/tlv320aic3x.txt b/Documentation/devicetree/bindings/sound/tlv320aic3x.txt
index ba5b45c483f5..9796c4639262 100644
--- a/Documentation/devicetree/bindings/sound/tlv320aic3x.txt
+++ b/Documentation/devicetree/bindings/sound/tlv320aic3x.txt
@@ -17,7 +17,7 @@ Required properties:
 
 Optional properties:
 
-- gpio-reset - gpio pin number used for codec reset
+- reset-gpios - GPIO specification for the active low RESET input.
 - ai3x-gpio-func - <array of 2 int> - AIC3X_GPIO1 & AIC3X_GPIO2 Functionality
 				    - Not supported on tlv320aic3104
 - ai3x-micbias-vg - MicBias Voltage required.
@@ -34,6 +34,10 @@ Optional properties:
 - AVDD-supply, IOVDD-supply, DRVDD-supply, DVDD-supply : power supplies for the
   device as covered in Documentation/devicetree/bindings/regulator/regulator.txt
 
+Deprecated properties:
+
+- gpio-reset - gpio pin number used for codec reset
+
 CODEC output pins:
   * LLOUT
   * RLOUT
@@ -61,10 +65,14 @@ The pins can be used in referring sound node's audio-routing property.
 
 Example:
 
+#include <dt-bindings/gpio/gpio.h>
+
 tlv320aic3x: tlv320aic3x@1b {
 	compatible = "ti,tlv320aic3x";
 	reg = <0x1b>;
 
+	reset-gpios = <&gpio1 17 GPIO_ACTIVE_LOW>;
+
 	AVDD-supply = <&regulator>;
 	IOVDD-supply = <&regulator>;
 	DRVDD-supply = <&regulator>;
diff --git a/Documentation/devicetree/bindings/sound/tscs42xx.txt b/Documentation/devicetree/bindings/sound/tscs42xx.txt
new file mode 100644
index 000000000000..2ac2f0996697
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/tscs42xx.txt
@@ -0,0 +1,16 @@
+TSCS42XX Audio CODEC
+
+Required Properties:
+
+	- compatible :	"tempo,tscs42A1" for analog mic
+			"tempo,tscs42A2" for digital mic
+
+	- reg : 	<0x71> for analog mic
+			<0x69> for digital mic
+
+Example:
+
+wookie: codec@69 {
+	compatible = "tempo,tscs42A2";
+	reg = <0x69>;
+};
diff --git a/Documentation/devicetree/bindings/sound/uniphier,evea.txt b/Documentation/devicetree/bindings/sound/uniphier,evea.txt
new file mode 100644
index 000000000000..3f31b235f18b
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/uniphier,evea.txt
@@ -0,0 +1,26 @@
+Socionext EVEA - UniPhier SoC internal codec driver
+
+Required properties:
+- compatible      : should be "socionext,uniphier-evea".
+- reg             : offset and length of the register set for the device.
+- clock-names     : should include following entries:
+                    "evea", "exiv"
+- clocks          : a list of phandle, should contain an entry for each
+                    entries in clock-names.
+- reset-names     : should include following entries:
+                    "evea", "exiv", "adamv"
+- resets          : a list of phandle, should contain reset entries of
+                    reset-names.
+- #sound-dai-cells: should be 1.
+
+Example:
+
+	codec {
+		compatible = "socionext,uniphier-evea";
+		reg = <0x57900000 0x1000>;
+		clock-names = "evea", "exiv";
+		clocks = <&sys_clk 41>, <&sys_clk 42>;
+		reset-names = "evea", "exiv", "adamv";
+		resets = <&sys_rst 41>, <&sys_rst 42>, <&adamv_rst 0>;
+		#sound-dai-cells = <1>;
+	};
diff --git a/Documentation/devicetree/bindings/spi/sh-msiof.txt b/Documentation/devicetree/bindings/spi/sh-msiof.txt
index bdd83959019c..80710f0f0448 100644
--- a/Documentation/devicetree/bindings/spi/sh-msiof.txt
+++ b/Documentation/devicetree/bindings/spi/sh-msiof.txt
@@ -36,7 +36,21 @@ Required properties:
 
 Optional properties:
 - clocks               : Must contain a reference to the functional clock.
-- num-cs               : Total number of chip-selects (default is 1)
+- num-cs               : Total number of chip selects (default is 1).
+			 Up to 3 native chip selects are supported:
+			   0: MSIOF_SYNC
+			   1: MSIOF_SS1
+			   2: MSIOF_SS2
+			 Hardware limitations related to chip selects:
+			   - Native chip selects are always deasserted in
+			     between transfers that are part of the same
+			     message.  Use cs-gpios to work around this.
+			   - All slaves using native chip selects must use the
+			     same spi-cs-high configuration.  Use cs-gpios to
+			     work around this.
+			   - When using GPIO chip selects, at least one native
+			     chip select must be left unused, as it will be
+			     driven anyway.
 - dmas                 : Must contain a list of two references to DMA
 			 specifiers, one for transmission, and one for
 			 reception.
diff --git a/Documentation/devicetree/bindings/spi/spi-davinci.txt b/Documentation/devicetree/bindings/spi/spi-davinci.txt
index 1925277bfc1e..9f5b4c7c0c08 100644
--- a/Documentation/devicetree/bindings/spi/spi-davinci.txt
+++ b/Documentation/devicetree/bindings/spi/spi-davinci.txt
@@ -68,7 +68,7 @@ SPI_SOMI/SIMO+-----------------+                           +-----------
 Example of a NOR flash slave device (n25q032) connected to DaVinci
 SPI controller device over the SPI bus.
 
-spi0:spi@20BF0000 {
+spi0:spi@20bf0000 {
 	#address-cells			= <1>;
 	#size-cells			= <0>;
 	compatible			= "ti,dm6446-spi";
diff --git a/Documentation/devicetree/bindings/spi/spi-lantiq-ssc.txt b/Documentation/devicetree/bindings/spi/spi-lantiq-ssc.txt
index 6069b95a883d..ce3230c8e28d 100644
--- a/Documentation/devicetree/bindings/spi/spi-lantiq-ssc.txt
+++ b/Documentation/devicetree/bindings/spi/spi-lantiq-ssc.txt
@@ -16,7 +16,7 @@ Optional properties:
 Example:
 
 
-spi: spi@E100800 {
+spi: spi@e100800 {
 	compatible = "lantiq,xrx200-spi", "lantiq,xrx100-spi";
 	reg = <0xE100800 0x100>;
 	interrupt-parent = <&icu0>;
diff --git a/Documentation/devicetree/bindings/spi/spi-meson.txt b/Documentation/devicetree/bindings/spi/spi-meson.txt
index 825c39cae74a..b7f5e86fed22 100644
--- a/Documentation/devicetree/bindings/spi/spi-meson.txt
+++ b/Documentation/devicetree/bindings/spi/spi-meson.txt
@@ -27,7 +27,9 @@ The Meson SPICC is generic SPI controller for general purpose Full-Duplex
 communications with dedicated 16 words RX/TX PIO FIFOs.
 
 Required properties:
- - compatible: should be "amlogic,meson-gx-spicc" on Amlogic GX SoCs.
+ - compatible: should be:
+	"amlogic,meson-gx-spicc" on Amlogic GX and compatible SoCs.
+	"amlogic,meson-axg-spicc" on Amlogic AXG and compatible SoCs
  - reg: physical base address and length of the controller registers
  - interrupts: The interrupt specifier
  - clock-names: Must contain "core"
diff --git a/Documentation/devicetree/bindings/spi/spi-orion.txt b/Documentation/devicetree/bindings/spi/spi-orion.txt
index df8ec31f2f07..8434a65fc12a 100644
--- a/Documentation/devicetree/bindings/spi/spi-orion.txt
+++ b/Documentation/devicetree/bindings/spi/spi-orion.txt
@@ -18,8 +18,17 @@ Required properties:
 	The eight register sets following the control registers refer to
 	chip-select lines 0 through 7 respectively.
 - cell-index : Which of multiple SPI controllers is this.
+- clocks : pointers to the reference clocks for this device, the first
+	   one is the one used for the clock on the spi bus, the
+	   second one is optional and is the clock used for the
+	   functional part of the controller
+
 Optional properties:
 - interrupts : Is currently not used.
+- clock-names : names of used clocks, mandatory if the second clock is
+		used, the name must be "core", and "axi" (the latter
+		is only for Armada 7K/8K).
+
 
 Example:
        spi@10600 {
diff --git a/Documentation/devicetree/bindings/spi/spi-xilinx.txt b/Documentation/devicetree/bindings/spi/spi-xilinx.txt
index c7b7856bd528..7bf61efc66c8 100644
--- a/Documentation/devicetree/bindings/spi/spi-xilinx.txt
+++ b/Documentation/devicetree/bindings/spi/spi-xilinx.txt
@@ -2,7 +2,7 @@ Xilinx SPI controller Device Tree Bindings
 -------------------------------------------------
 
 Required properties:
-- compatible		: Should be "xlnx,xps-spi-2.00.a" or "xlnx,xps-spi-2.00.b"
+- compatible		: Should be "xlnx,xps-spi-2.00.a", "xlnx,xps-spi-2.00.b" or "xlnx,axi-quad-spi-1.00.a"
 - reg			: Physical base address and size of SPI registers map.
 - interrupts		: Property with a value describing the interrupt
 			  number.
diff --git a/Documentation/devicetree/bindings/thermal/armada-thermal.txt b/Documentation/devicetree/bindings/thermal/armada-thermal.txt
index 24aacf8948c5..e0d013a2e66d 100644
--- a/Documentation/devicetree/bindings/thermal/armada-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/armada-thermal.txt
@@ -2,22 +2,35 @@
 
 Required properties:
 
-- compatible:	Should be set to one of the following:
-		marvell,armada370-thermal
-		marvell,armada375-thermal
-		marvell,armada380-thermal
-		marvell,armadaxp-thermal
+- compatible: Should be set to one of the following:
+    * marvell,armada370-thermal
+    * marvell,armada375-thermal
+    * marvell,armada380-thermal
+    * marvell,armadaxp-thermal
+    * marvell,armada-ap806-thermal
+    * marvell,armada-cp110-thermal
 
-- reg:		Device's register space.
-		Two entries are expected, see the examples below.
-		The first one is required for the sensor register;
-		the second one is required for the control register
-		to be used for sensor initialization (a.k.a. calibration).
+- reg: Device's register space.
+  Two entries are expected, see the examples below. The first one points
+  to the status register (4B). The second one points to the control
+  registers (8B).
+  Note: The compatibles marvell,armada370-thermal,
+  marvell,armada380-thermal, and marvell,armadaxp-thermal must point to
+  "control MSB/control 1", with size of 4 (deprecated binding), or point
+  to "control LSB/control 0" with size of 8 (current binding). All other
+  compatibles must point to "control LSB/control 0" with size of 8.
 
-Example:
+Examples:
 
+	/* Legacy bindings */
 	thermal@d0018300 {
 		compatible = "marvell,armada370-thermal";
-                reg = <0xd0018300 0x4
+		reg = <0xd0018300 0x4
 		       0xd0018304 0x4>;
 	};
+
+	ap_thermal: thermal@6f8084 {
+		compatible = "marvell,armada-ap806-thermal";
+		reg = <0x6f808C 0x4>,
+		      <0x6f8084 0x8>;
+	};
diff --git a/Documentation/devicetree/bindings/thermal/exynos-thermal.txt b/Documentation/devicetree/bindings/thermal/exynos-thermal.txt
index 9b4c7b017495..1b596fd38dc4 100644
--- a/Documentation/devicetree/bindings/thermal/exynos-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/exynos-thermal.txt
@@ -70,7 +70,7 @@ Following properties are mandatory (depending on SoC):
 
 Example 1):
 
-	tmu@100C0000 {
+	tmu@100c0000 {
 		compatible = "samsung,exynos4412-tmu";
 		interrupt-parent = <&combiner>;
 		reg = <0x100C0000 0x100>;
diff --git a/Documentation/devicetree/bindings/thermal/imx-thermal.txt b/Documentation/devicetree/bindings/thermal/imx-thermal.txt
index 28be51afdb6a..379eb763073e 100644
--- a/Documentation/devicetree/bindings/thermal/imx-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/imx-thermal.txt
@@ -22,7 +22,32 @@ Optional properties:
 - clocks : thermal sensor's clock source.
 
 Example:
+ocotp: ocotp@21bc000 {
+	#address-cells = <1>;
+	#size-cells = <1>;
+	compatible = "fsl,imx6sx-ocotp", "syscon";
+	reg = <0x021bc000 0x4000>;
+	clocks = <&clks IMX6SX_CLK_OCOTP>;
 
+	tempmon_calib: calib@38 {
+		reg = <0x38 4>;
+	};
+
+	tempmon_temp_grade: temp-grade@20 {
+		reg = <0x20 4>;
+	};
+};
+
+tempmon: tempmon {
+	compatible = "fsl,imx6sx-tempmon", "fsl,imx6q-tempmon";
+	interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+	fsl,tempmon = <&anatop>;
+	nvmem-cells = <&tempmon_calib>, <&tempmon_temp_grade>;
+	nvmem-cell-names = "calib", "temp_grade";
+	clocks = <&clks IMX6SX_CLK_PLL3_USB_OTG>;
+};
+
+Legacy method (Deprecated):
 tempmon {
 	compatible = "fsl,imx6q-tempmon";
 	fsl,tempmon = <&anatop>;
diff --git a/Documentation/devicetree/bindings/thermal/rcar-thermal.txt b/Documentation/devicetree/bindings/thermal/rcar-thermal.txt
index a8e52c8ccfcc..349e635f2d87 100644
--- a/Documentation/devicetree/bindings/thermal/rcar-thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/rcar-thermal.txt
@@ -6,6 +6,7 @@ Required properties:
 			   "renesas,rcar-thermal" (without thermal-zone) as fallback.
 			  Examples with soctypes are:
 			    - "renesas,thermal-r8a73a4" (R-Mobile APE6)
+			    - "renesas,thermal-r8a7743" (RZ/G1M)
 			    - "renesas,thermal-r8a7779" (R-Car H1)
 			    - "renesas,thermal-r8a7790" (R-Car H2)
 			    - "renesas,thermal-r8a7791" (R-Car M2-W)
diff --git a/Documentation/devicetree/bindings/thermal/thermal.txt b/Documentation/devicetree/bindings/thermal/thermal.txt
index 44d7cb2cb2c0..1719d47a5e2f 100644
--- a/Documentation/devicetree/bindings/thermal/thermal.txt
+++ b/Documentation/devicetree/bindings/thermal/thermal.txt
@@ -252,7 +252,7 @@ ocp {
 	/*
 	 * A simple IC with a single bandgap temperature sensor.
 	 */
-	bandgap0: bandgap@0000ED00 {
+	bandgap0: bandgap@0000ed00 {
 		...
 		#thermal-sensor-cells = <0>;
 	};
@@ -330,7 +330,7 @@ ocp {
 	/*
 	 * A simple IC with several bandgap temperature sensors.
 	 */
-	bandgap0: bandgap@0000ED00 {
+	bandgap0: bandgap@0000ed00 {
 		...
 		#thermal-sensor-cells = <1>;
 	};
@@ -458,7 +458,7 @@ ocp {
 	/*
 	 * A simple IC with a single bandgap temperature sensor.
 	 */
-	bandgap0: bandgap@0000ED00 {
+	bandgap0: bandgap@0000ed00 {
 		...
 		#thermal-sensor-cells = <0>;
 	};
diff --git a/Documentation/devicetree/bindings/timer/actions,owl-timer.txt b/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
index e3c28da80cb2..977054f87563 100644
--- a/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
+++ b/Documentation/devicetree/bindings/timer/actions,owl-timer.txt
@@ -2,6 +2,7 @@ Actions Semi Owl Timer
 
 Required properties:
 - compatible      :  "actions,s500-timer" for S500
+                     "actions,s700-timer" for S700
                      "actions,s900-timer" for S900
 - reg             :  Offset and length of the register set for the device.
 - interrupts      :  Should contain the interrupts.
diff --git a/Documentation/devicetree/bindings/timer/renesas,tpu.txt b/Documentation/devicetree/bindings/timer/renesas,tpu.txt
index f8b25897fb31..1d46f9de4feb 100644
--- a/Documentation/devicetree/bindings/timer/renesas,tpu.txt
+++ b/Documentation/devicetree/bindings/timer/renesas,tpu.txt
@@ -1,4 +1,4 @@
-* Renesas H8/300 Timer Pluse Unit
+* Renesas H8/300 Timer Pulse Unit
 
 The TPU is a 16bit timer/counter with configurable clock inputs and
 programmable compare match.
diff --git a/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.txt b/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.txt
index 167d5dab9f64..8f78640ad64c 100644
--- a/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.txt
+++ b/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.txt
@@ -51,7 +51,7 @@ Example 2: In this example, the timer interrupts are connected to two separate
 	   interrupt controllers. Hence, an interrupt-map is created to map
 	   the interrupts to the respective interrupt controllers.
 
-	mct@101C0000 {
+	mct@101c0000 {
 		compatible = "samsung,exynos4210-mct";
 		reg = <0x101C0000 0x800>;
 		interrupt-parent = <&mct_map>;
diff --git a/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt b/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt
new file mode 100644
index 000000000000..6d97e7d0f6e8
--- /dev/null
+++ b/Documentation/devicetree/bindings/timer/spreadtrum,sprd-timer.txt
@@ -0,0 +1,20 @@
+Spreadtrum timers
+
+The Spreadtrum SC9860 platform provides 3 general-purpose timers.
+These timers can support 32bit or 64bit counter, as well as supporting
+period mode or one-shot mode, and they are can be wakeup source
+during deep sleep.
+
+Required properties:
+- compatible: should be "sprd,sc9860-timer" for SC9860 platform.
+- reg: The register address of the timer device.
+- interrupts: Should contain the interrupt for the timer device.
+- clocks: The phandle to the source clock (usually a 32.768 KHz fixed clock).
+
+Example:
+	timer@40050000 {
+		compatible = "sprd,sc9860-timer";
+		reg = <0 0x40050000 0 0x20>;
+		interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&ext_32k>;
+	};
diff --git a/Documentation/devicetree/bindings/trivial-devices.txt b/Documentation/devicetree/bindings/trivial-devices.txt
index 5f3143f97098..2e3740f98c41 100644
--- a/Documentation/devicetree/bindings/trivial-devices.txt
+++ b/Documentation/devicetree/bindings/trivial-devices.txt
@@ -63,6 +63,7 @@ fsl,sgtl5000		SGTL5000: Ultra Low-Power Audio Codec
 gmt,g751		G751: Digital Temperature Sensor and Thermal Watchdog with Two-Wire Interface
 infineon,slb9635tt	Infineon SLB9635 (Soft-) I2C TPM (old protocol, max 100khz)
 infineon,slb9645tt	Infineon SLB9645 I2C TPM (new protocol, max 400khz)
+infineon,tlv493d-a1b6	Infineon TLV493D-A1B6 I2C 3D Magnetic Sensor
 isil,isl1208		Intersil ISL1208 Low Power RTC with Battery Backed SRAM
 isil,isl1218		Intersil ISL1218 Low Power RTC with Battery Backed SRAM
 isil,isl12022		Intersil ISL12022 Real-time Clock
@@ -151,6 +152,7 @@ national,lm85		Temperature sensor with integrated fan control
 national,lm92		±0.33°C Accurate, 12-Bit + Sign Temperature Sensor and Thermal Window Comparator with Two-Wire Interface
 nuvoton,npct501		i2c trusted platform module (TPM)
 nuvoton,npct601		i2c trusted platform module (TPM2)
+nuvoton,w83773g		Nuvoton Temperature Sensor
 nxp,pca9556		Octal SMBus and I2C registered interface
 nxp,pca9557		8-bit I2C-bus and SMBus I/O port with reset
 nxp,pcf2127		Real-time clock
diff --git a/Documentation/devicetree/bindings/usb/dwc2.txt b/Documentation/devicetree/bindings/usb/dwc2.txt
index e64d903bcbe8..46da5f184460 100644
--- a/Documentation/devicetree/bindings/usb/dwc2.txt
+++ b/Documentation/devicetree/bindings/usb/dwc2.txt
@@ -19,7 +19,7 @@ Required properties:
   configured in FS mode;
   - "st,stm32f4x9-hsotg": The DWC2 USB HS controller instance in STM32F4x9 SoCs
   configured in HS mode;
-  - "st,stm32f7xx-hsotg": The DWC2 USB HS controller instance in STM32F7xx SoCs
+  - "st,stm32f7-hsotg": The DWC2 USB HS controller instance in STM32F7 SoCs
     configured in HS mode;
 - reg : Should contain 1 register range (address and length)
 - interrupts : Should contain 1 interrupt
diff --git a/Documentation/devicetree/bindings/usb/dwc3.txt b/Documentation/devicetree/bindings/usb/dwc3.txt
index 52fb41046b34..44e8bab159ad 100644
--- a/Documentation/devicetree/bindings/usb/dwc3.txt
+++ b/Documentation/devicetree/bindings/usb/dwc3.txt
@@ -47,6 +47,8 @@ Optional properties:
 			from P0 to P1/P2/P3 without delay.
  - snps,dis-tx-ipgap-linecheck-quirk: when set, disable u2mac linestate check
 			during HS transmit.
+ - snps,dis_metastability_quirk: when set, disable metastability workaround.
+			CAUTION: use only if you are absolutely sure of it.
  - snps,is-utmi-l1-suspend: true when DWC3 asserts output signal
 			utmi_l1_suspend_n, false when asserts utmi_sleep_n
  - snps,hird-threshold: HIRD threshold
diff --git a/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.txt b/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.txt
index 30595964876a..88d9f4a4b280 100644
--- a/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.txt
+++ b/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.txt
@@ -35,10 +35,14 @@ Required properties:
  - phys : a list of phandle + phy specifier pairs
 
 Optional properties:
- - mediatek,wakeup-src : 1: ip sleep wakeup mode; 2: line state wakeup
-	mode;
- - mediatek,syscon-wakeup : phandle to syscon used to access USB wakeup
-	control register, it depends on "mediatek,wakeup-src".
+ - wakeup-source : enable USB remote wakeup;
+ - mediatek,syscon-wakeup : phandle to syscon used to access the register
+	of the USB wakeup glue layer between xHCI and SPM; it depends on
+	"wakeup-source", and has two arguments:
+	- the first one : register base address of the glue layer in syscon;
+	- the second one : hardware version of the glue layer
+		- 1 : used by mt8173 etc
+		- 2 : used by mt2712 etc
  - mediatek,u3p-dis-msk : mask to disable u3ports, bit0 for u3port0,
 	bit1 for u3port1, ... etc;
  - vbus-supply : reference to the VBUS regulator;
@@ -46,6 +50,7 @@ Optional properties:
  - pinctrl-names : a pinctrl state named "default" must be defined
  - pinctrl-0 : pin control group
 	See: Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt
+ - imod-interval-ns: default interrupt moderation interval is 5000ns
 
 Example:
 usb30: usb@11270000 {
@@ -64,8 +69,9 @@ usb30: usb@11270000 {
 	vusb33-supply = <&mt6397_vusb_reg>;
 	vbus-supply = <&usb_p1_vbus>;
 	usb3-lpm-capable;
-	mediatek,syscon-wakeup = <&pericfg>;
-	mediatek,wakeup-src = <1>;
+	mediatek,syscon-wakeup = <&pericfg 0x400 1>;
+	wakeup-source;
+	imod-interval-ns = <10000>;
 };
 
 2nd: dual-role mode with xHCI driver
diff --git a/Documentation/devicetree/bindings/usb/mediatek,mtu3.txt b/Documentation/devicetree/bindings/usb/mediatek,mtu3.txt
index b2271d8e6b50..d589a1ef96a1 100644
--- a/Documentation/devicetree/bindings/usb/mediatek,mtu3.txt
+++ b/Documentation/devicetree/bindings/usb/mediatek,mtu3.txt
@@ -42,9 +42,14 @@ Optional properties:
  - enable-manual-drd : supports manual dual-role switch via debugfs; usually
 	used when receptacle is TYPE-A and also wants to support dual-role
 	mode.
- - mediatek,enable-wakeup : supports ip sleep wakeup used by host mode
- - mediatek,syscon-wakeup : phandle to syscon used to access USB wakeup
-	control register, it depends on "mediatek,enable-wakeup".
+ - wakeup-source: enable USB remote wakeup of host mode.
+ - mediatek,syscon-wakeup : phandle to syscon used to access the register
+	of the USB wakeup glue layer between SSUSB and SPM; it depends on
+	"wakeup-source", and has two arguments:
+	- the first one : register base address of the glue layer in syscon;
+	- the second one : hardware version of the glue layer
+		- 1 : used by mt8173 etc
+		- 2 : used by mt2712 etc
  - mediatek,u3p-dis-msk : mask to disable u3ports, bit0 for u3port0,
 	bit1 for u3port1, ... etc;
 
@@ -71,8 +76,8 @@ ssusb: usb@11271000 {
 	vbus-supply = <&usb_p0_vbus>;
 	extcon = <&extcon_usb>;
 	dr_mode = "otg";
-	mediatek,enable-wakeup;
-	mediatek,syscon-wakeup = <&pericfg>;
+	wakeup-source;
+	mediatek,syscon-wakeup = <&pericfg 0x400 1>;
 	#address-cells = <2>;
 	#size-cells = <2>;
 	ranges;
diff --git a/Documentation/devicetree/bindings/usb/renesas_usb3.txt b/Documentation/devicetree/bindings/usb/renesas_usb3.txt
index 87a45e2f9b7f..2c071bb5801e 100644
--- a/Documentation/devicetree/bindings/usb/renesas_usb3.txt
+++ b/Documentation/devicetree/bindings/usb/renesas_usb3.txt
@@ -4,6 +4,7 @@ Required properties:
   - compatible: Must contain one of the following:
 	- "renesas,r8a7795-usb3-peri"
 	- "renesas,r8a7796-usb3-peri"
+	- "renesas,r8a77965-usb3-peri"
 	- "renesas,rcar-gen3-usb3-peri" for a generic R-Car Gen3 compatible
 	  device
 
diff --git a/Documentation/devicetree/bindings/usb/renesas_usbhs.txt b/Documentation/devicetree/bindings/usb/renesas_usbhs.txt
index 47394ab788e3..43960faf5a88 100644
--- a/Documentation/devicetree/bindings/usb/renesas_usbhs.txt
+++ b/Documentation/devicetree/bindings/usb/renesas_usbhs.txt
@@ -12,9 +12,12 @@ Required properties:
 	- "renesas,usbhs-r8a7794" for r8a7794 (R-Car E2) compatible device
 	- "renesas,usbhs-r8a7795" for r8a7795 (R-Car H3) compatible device
 	- "renesas,usbhs-r8a7796" for r8a7796 (R-Car M3-W) compatible device
+	- "renesas,usbhs-r8a77965" for r8a77965 (R-Car M3-N) compatible device
 	- "renesas,usbhs-r8a77995" for r8a77995 (R-Car D3) compatible device
+	- "renesas,usbhs-r7s72100" for r7s72100 (RZ/A1) compatible device
 	- "renesas,rcar-gen2-usbhs" for R-Car Gen2 or RZ/G1 compatible devices
 	- "renesas,rcar-gen3-usbhs" for R-Car Gen3 compatible device
+	- "renesas,rza1-usbhs" for RZ/A1 compatible device
 
 	When compatible with the generic version, nodes must list the
 	SoC-specific version corresponding to the platform first followed
diff --git a/Documentation/devicetree/bindings/usb/usb-device.txt b/Documentation/devicetree/bindings/usb/usb-device.txt
index 1b27cebb47f4..036be172b1ae 100644
--- a/Documentation/devicetree/bindings/usb/usb-device.txt
+++ b/Documentation/devicetree/bindings/usb/usb-device.txt
@@ -4,8 +4,49 @@ Usually, we only use device tree for hard wired USB device.
 The reference binding doc is from:
 http://www.devicetree.org/open-firmware/bindings/usb/usb-1_0.ps
 
+Four types of device-tree nodes are defined: "host-controller nodes"
+representing USB host controllers, "device nodes" representing USB devices,
+"interface nodes" representing USB interfaces and "combined nodes"
+representing simple USB devices.
 
-Required properties:
+A combined node shall be used instead of a device node and an interface node
+for devices of class 0 or 9 (hub) with a single configuration and a single
+interface.
+
+A "hub node" is a combined node or an interface node that represents a USB
+hub.
+
+
+Required properties for device nodes:
+- compatible: "usbVID,PID", where VID is the vendor id and PID the product id.
+  The textual representation of VID and PID shall be in lower case hexadecimal
+  with leading zeroes suppressed. The other compatible strings from the above
+  standard binding could also be used, but a device adhering to this binding
+  may leave out all except for "usbVID,PID".
+- reg: the number of the USB hub port or the USB host-controller port to which
+  this device is attached. The range is 1-255.
+
+
+Required properties for device nodes with interface nodes:
+- #address-cells: shall be 2
+- #size-cells: shall be 0
+
+
+Required properties for interface nodes:
+- compatible: "usbifVID,PID.configCN.IN", where VID is the vendor id, PID is
+  the product id, CN is the configuration value and IN is the interface
+  number. The textual representation of VID, PID, CN and IN shall be in lower
+  case hexadecimal with leading zeroes suppressed. The other compatible
+  strings from the above standard binding could also be used, but a device
+  adhering to this binding may leave out all except for
+  "usbifVID,PID.configCN.IN".
+- reg: the interface number and configuration value
+
+The configuration component is not included in the textual representation of
+an interface-node unit address for configuration 1.
+
+
+Required properties for combined nodes:
 - compatible: "usbVID,PID", where VID is the vendor id and PID the product id.
   The textual representation of VID and PID shall be in lower case hexadecimal
   with leading zeroes suppressed. The other compatible strings from the above
@@ -31,8 +72,31 @@ Example:
 	#address-cells = <1>;
 	#size-cells = <0>;
 
-	hub@1 {	/* hub connected to port 1 */
+	hub@1 {		/* hub connected to port 1 */
 		compatible = "usb5e3,608";
 		reg = <1>;
 	};
+
+	device@2 {	/* device connected to port 2 */
+		compatible = "usb123,4567";
+		reg = <2>;
+	};
+
+	device@3 { 	/* device connected to port 3 */
+		compatible = "usb123,abcd";
+		reg = <3>;
+
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		interface@0 {	/* interface 0 of configuration 1 */
+			compatible = "usbif123,abcd.config1.0";
+			reg = <0 1>;
+		};
+
+		interface@0,2 {	/* interface 0 of configuration 2 */
+			compatible = "usbif123,abcd.config2.0";
+			reg = <0 2>;
+		};
+	};
 };
diff --git a/Documentation/devicetree/bindings/usb/usb-xhci.txt b/Documentation/devicetree/bindings/usb/usb-xhci.txt
index ae6e484a8d7c..1651483a7048 100644
--- a/Documentation/devicetree/bindings/usb/usb-xhci.txt
+++ b/Documentation/devicetree/bindings/usb/usb-xhci.txt
@@ -7,12 +7,15 @@ Required properties:
     - "marvell,armada3700-xhci" for Armada 37xx SoCs
     - "marvell,armada-375-xhci" for Armada 375 SoCs
     - "marvell,armada-380-xhci" for Armada 38x SoCs
+    - "renesas,xhci-r8a7743" for r8a7743 SoC
     - "renesas,xhci-r8a7790" for r8a7790 SoC
     - "renesas,xhci-r8a7791" for r8a7791 SoC
     - "renesas,xhci-r8a7793" for r8a7793 SoC
     - "renesas,xhci-r8a7795" for r8a7795 SoC
     - "renesas,xhci-r8a7796" for r8a7796 SoC
-    - "renesas,rcar-gen2-xhci" for a generic R-Car Gen2 compatible device
+    - "renesas,xhci-r8a77965" for r8a77965 SoC
+    - "renesas,rcar-gen2-xhci" for a generic R-Car Gen2 or RZ/G1 compatible
+      device
     - "renesas,rcar-gen3-xhci" for a generic R-Car Gen3 compatible device
     - "xhci-platform" (deprecated)
 
@@ -29,6 +32,7 @@ Optional properties:
   - usb2-lpm-disable: indicate if we don't want to enable USB2 HW LPM
   - usb3-lpm-capable: determines if platform is USB3 LPM capable
   - quirk-broken-port-ped: set if the controller has broken port disable mechanism
+  - imod-interval-ns: default interrupt moderation interval is 5000ns
 
 Example:
 	usb@f0931000 {
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt
index 0994bdd82cd3..ae850d6c0ad3 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.txt
+++ b/Documentation/devicetree/bindings/vendor-prefixes.txt
@@ -15,6 +15,7 @@ adi	Analog Devices, Inc.
 advantech	Advantech Corporation
 aeroflexgaisler	Aeroflex Gaisler AB
 al	Annapurna Labs
+allo	Allo.com
 allwinner	Allwinner Technology Co., Ltd.
 alphascale	AlphaScale Integrated Circuits Systems, Inc.
 altr	Altera Corp.
@@ -97,6 +98,7 @@ dptechnics	DPTechnics
 dragino	Dragino Technology Co., Limited
 ea	Embedded Artists AB
 ebv	EBV Elektronik
+eckelmann	Eckelmann AG
 edt	Emerging Display Technologies
 eeti	eGalax_eMPIA Technology Inc
 elan	Elan Microelectronic Corp.
@@ -123,6 +125,7 @@ focaltech	FocalTech Systems Co.,Ltd
 friendlyarm	Guangzhou FriendlyARM Computer Tech Co., Ltd
 fsl	Freescale Semiconductor
 fujitsu	Fujitsu Ltd.
+gcw Game Consoles Worldwide
 ge	General Electric Company
 geekbuying	GeekBuying
 gef	GE Fanuc Intelligent Platforms Embedded Systems, Inc.
@@ -154,6 +157,7 @@ i2se	I2SE GmbH
 ibm	International Business Machines (IBM)
 idt	Integrated Device Technologies, Inc.
 ifi	Ingenieurburo Fur Ic-Technologie (I/F/I)
+ilitek	ILI Technology Corporation (ILITEK)
 img	Imagination Technologies Ltd.
 infineon Infineon Technologies
 inforce	Inforce Computing
@@ -172,6 +176,7 @@ itead	ITEAD Intelligent Systems Co.Ltd
 iwave  iWave Systems Technologies Pvt. Ltd.
 jdi	Japan Display Inc.
 jedec	JEDEC Solid State Technology Association
+jianda	Jiandangjing Technology Co., Ltd.
 karo	Ka-Ro electronics GmbH
 keithkoep	Keith & Koep GmbH
 keymile	Keymile GmbH
@@ -305,6 +310,7 @@ seagate	Seagate Technology PLC
 semtech	Semtech Corporation
 sensirion	Sensirion AG
 sff	Small Form Factor Committee
+sgd	Solomon Goldentek Display Corporation
 sgx	SGX Sensortech
 sharp	Sharp Corporation
 shimafuji	Shimafuji Electric, Inc.
@@ -347,6 +353,7 @@ tcg	Trusted Computing Group
 tcl	Toby Churchill Ltd.
 technexion	TechNexion
 technologic	Technologic Systems
+tempo	Tempo Semiconductor
 terasic	Terasic Inc.
 thine	THine Electronics, Inc.
 ti	Texas Instruments
@@ -379,6 +386,7 @@ virtio	Virtual I/O Device Specification, developed by the OASIS consortium
 vivante	Vivante Corporation
 vocore VoCore Studio
 voipac	Voipac Technologies s.r.o.
+vot	Vision Optical Technology Co., Ltd.
 wd	Western Digital Corp.
 wetek	WeTek Electronics, limited.
 wexler	Wexler
diff --git a/Documentation/devicetree/bindings/watchdog/cortina,gemini-watchdog.txt b/Documentation/devicetree/bindings/watchdog/cortina,gemini-watchdog.txt
deleted file mode 100644
index bc4b865d178b..000000000000
--- a/Documentation/devicetree/bindings/watchdog/cortina,gemini-watchdog.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-Cortina Systems Gemini SoC Watchdog
-
-Required properties:
-- compatible : must be "cortina,gemini-watchdog"
-- reg : shall contain base register location and length
-- interrupts : shall contain the interrupt for the watchdog
-
-Optional properties:
-- timeout-sec : the default watchdog timeout in seconds.
-
-Example:
-
-watchdog@41000000 {
-	compatible = "cortina,gemini-watchdog";
-	reg = <0x41000000 0x1000>;
-	interrupts = <3 IRQ_TYPE_LEVEL_HIGH>;
-};
diff --git a/Documentation/devicetree/bindings/watchdog/cortina,gemin-watchdog.txt b/Documentation/devicetree/bindings/watchdog/faraday,ftwdt010.txt
similarity index 55%
rename from Documentation/devicetree/bindings/watchdog/cortina,gemin-watchdog.txt
rename to Documentation/devicetree/bindings/watchdog/faraday,ftwdt010.txt
index bc4b865d178b..9ecdb502e605 100644
--- a/Documentation/devicetree/bindings/watchdog/cortina,gemin-watchdog.txt
+++ b/Documentation/devicetree/bindings/watchdog/faraday,ftwdt010.txt
@@ -1,7 +1,12 @@
-Cortina Systems Gemini SoC Watchdog
+Faraday Technology FTWDT010 watchdog
+
+This is an IP part from Faraday Technology found in the Gemini
+SoCs and others.
 
 Required properties:
-- compatible : must be "cortina,gemini-watchdog"
+- compatible : must be one of
+  "faraday,ftwdt010"
+  "cortina,gemini-watchdog", "faraday,ftwdt010"
 - reg : shall contain base register location and length
 - interrupts : shall contain the interrupt for the watchdog
 
@@ -11,7 +16,7 @@ Optional properties:
 Example:
 
 watchdog@41000000 {
-	compatible = "cortina,gemini-watchdog";
+	compatible = "faraday,ftwdt010";
 	reg = <0x41000000 0x1000>;
 	interrupts = <3 IRQ_TYPE_LEVEL_HIGH>;
 };
diff --git a/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt b/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt
index 3c7a1cd13b10..cb44918f01a8 100644
--- a/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt
+++ b/Documentation/devicetree/bindings/watchdog/ingenic,jz4740-wdt.txt
@@ -1,7 +1,7 @@
-Ingenic Watchdog Timer (WDT) Controller for JZ4740
+Ingenic Watchdog Timer (WDT) Controller for JZ4740 & JZ4780
 
 Required properties:
-compatible: "ingenic,jz4740-watchdog"
+compatible: "ingenic,jz4740-watchdog" or "ingenic,jz4780-watchdog"
 reg: Register address and length for watchdog registers
 
 Example:
diff --git a/Documentation/devicetree/bindings/watchdog/pnx4008-wdt.txt b/Documentation/devicetree/bindings/watchdog/pnx4008-wdt.txt
index 556d06c17c92..4b76bec62af9 100644
--- a/Documentation/devicetree/bindings/watchdog/pnx4008-wdt.txt
+++ b/Documentation/devicetree/bindings/watchdog/pnx4008-wdt.txt
@@ -10,7 +10,7 @@ Optional properties:
 
 Example:
 
-	watchdog@4003C000 {
+	watchdog@4003c000 {
 		compatible = "nxp,pnx4008-wdt";
 		reg = <0x4003C000 0x1000>;
 		timeout-sec = <10>;
diff --git a/Documentation/devicetree/bindings/watchdog/realtek,rtd119x.txt b/Documentation/devicetree/bindings/watchdog/realtek,rtd119x.txt
new file mode 100644
index 000000000000..05653054bd5b
--- /dev/null
+++ b/Documentation/devicetree/bindings/watchdog/realtek,rtd119x.txt
@@ -0,0 +1,17 @@
+Realtek RTD1295 Watchdog
+========================
+
+Required properties:
+
+- compatible :  Should be "realtek,rtd1295-watchdog"
+- reg        :  Specifies the physical base address and size of registers
+- clocks     :  Specifies one clock input
+
+
+Example:
+
+	watchdog@98007680 {
+		compatible = "realtek,rtd1295-watchdog";
+		reg = <0x98007680 0x100>;
+		clocks = <&osc27M>;
+	};
diff --git a/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt b/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt
index bf6d1ca58af7..74b2f03c1515 100644
--- a/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt
+++ b/Documentation/devicetree/bindings/watchdog/renesas-wdt.txt
@@ -4,10 +4,11 @@ Required properties:
 - compatible : Should be "renesas,<soctype>-wdt", and
 	       "renesas,rcar-gen3-wdt" or "renesas,rza-wdt" as fallback.
 	       Examples with soctypes are:
+	         - "renesas,r7s72100-wdt" (RZ/A1)
 	         - "renesas,r8a7795-wdt" (R-Car H3)
 	         - "renesas,r8a7796-wdt" (R-Car M3-W)
+	         - "renesas,r8a77970-wdt" (R-Car V3M)
 	         - "renesas,r8a77995-wdt" (R-Car D3)
-	         - "renesas,r7s72100-wdt" (RZ/A1)
 
   When compatible with the generic version, nodes must list the SoC-specific
   version corresponding to the platform first, followed by the generic
diff --git a/Documentation/devicetree/bindings/watchdog/samsung-wdt.txt b/Documentation/devicetree/bindings/watchdog/samsung-wdt.txt
index 1f6e101e299a..46dcb48e75b4 100644
--- a/Documentation/devicetree/bindings/watchdog/samsung-wdt.txt
+++ b/Documentation/devicetree/bindings/watchdog/samsung-wdt.txt
@@ -25,7 +25,7 @@ Optional properties:
 
 Example:
 
-watchdog@101D0000 {
+watchdog@101d0000 {
 	compatible = "samsung,exynos5250-wdt";
 	reg = <0x101D0000 0x100>;
 	interrupts = <0 42 0>;
diff --git a/Documentation/devicetree/bindings/watchdog/sprd-wdt.txt b/Documentation/devicetree/bindings/watchdog/sprd-wdt.txt
new file mode 100644
index 000000000000..aeaf3e0caf47
--- /dev/null
+++ b/Documentation/devicetree/bindings/watchdog/sprd-wdt.txt
@@ -0,0 +1,19 @@
+Spreadtrum SoCs Watchdog timer
+
+Required properties:
+- compatible : Should be "sprd,sp9860-wdt".
+- reg : Specifies base physical address and size of the registers.
+- interrupts : Exactly one interrupt specifier.
+- timeout-sec : Contain the default watchdog timeout in seconds.
+- clock-names : Contain the input clock names.
+- clocks : Phandles to input clocks.
+
+Example:
+	watchdog: watchdog@40310000 {
+		compatible = "sprd,sp9860-wdt";
+		reg = <0 0x40310000 0 0x1000>;
+		interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
+		timeout-sec = <12>;
+		clock-names = "enable", "rtc_enable";
+		clocks = <&clk_aon_apb_gates1 8>, <&clk_aon_apb_rtc_gates 9>;
+	};
diff --git a/Documentation/devicetree/bindings/watchdog/zii,rave-sp-wdt.txt b/Documentation/devicetree/bindings/watchdog/zii,rave-sp-wdt.txt
new file mode 100644
index 000000000000..3de96186e92e
--- /dev/null
+++ b/Documentation/devicetree/bindings/watchdog/zii,rave-sp-wdt.txt
@@ -0,0 +1,39 @@
+Zodiac Inflight Innovations RAVE Supervisory Processor Watchdog Bindings
+
+RAVE SP watchdog device is a "MFD cell" device corresponding to
+watchdog functionality of RAVE Supervisory Processor. It is expected
+that its Device Tree node is specified as a child of the node
+corresponding to the parent RAVE SP device (as documented in
+Documentation/devicetree/bindings/mfd/zii,rave-sp.txt)
+
+Required properties:
+
+- compatible: Depending on wire protocol implemented by RAVE SP
+  firmware, should be one of:
+	- "zii,rave-sp-watchdog"
+	- "zii,rave-sp-watchdog-legacy"
+
+Optional properties:
+
+- wdt-timeout:	Two byte nvmem cell specified as per
+		Documentation/devicetree/bindings/nvmem/nvmem.txt
+
+Example:
+
+	rave-sp {
+		compatible = "zii,rave-sp-rdu1";
+		current-speed = <38400>;
+
+		eeprom {
+			wdt_timeout: wdt-timeout@8E {
+				reg = <0x8E 2>;
+			};
+		};
+
+		watchdog {
+			compatible = "zii,rave-sp-watchdog";
+			nvmem-cells = <&wdt_timeout>;
+			nvmem-cell-names = "wdt-timeout";
+		};
+	}
+
diff --git a/Documentation/devicetree/booting-without-of.txt b/Documentation/devicetree/booting-without-of.txt
index 417f91110010..e86bd2f64117 100644
--- a/Documentation/devicetree/booting-without-of.txt
+++ b/Documentation/devicetree/booting-without-of.txt
@@ -1309,7 +1309,7 @@ number and level/sense information. All interrupt children in an
 OpenPIC interrupt domain use 2 cells per interrupt in their interrupts
 property.
 
-The PCI bus binding specifies a #interrupt-cell value of 1 to encode
+The PCI bus binding specifies a #interrupt-cells value of 1 to encode
 which interrupt pin (INTA,INTB,INTC,INTD) is used.
 
 2) interrupt-parent property
diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst
index 0268335414ce..722d4525f7cf 100644
--- a/Documentation/doc-guide/kernel-doc.rst
+++ b/Documentation/doc-guide/kernel-doc.rst
@@ -112,16 +112,17 @@ Example kernel-doc function comment::
 
   /**
    * foobar() - Brief description of foobar.
-   * @arg: Description of argument of foobar.
+   * @argument1: Description of parameter argument1 of foobar.
+   * @argument2: Description of parameter argument2 of foobar.
    *
    * Longer description of foobar.
    *
    * Return: Description of return value of foobar.
    */
-  int foobar(int arg)
+  int foobar(int argument1, char *argument2)
 
 The format is similar for documentation for structures, enums, paragraphs,
-etc. See the sections below for details.
+etc. See the sections below for specific details of each type.
 
 The kernel-doc structure is extracted from the comments, and proper `Sphinx C
 Domain`_ function and type descriptions with anchors are generated for them. The
@@ -130,6 +131,226 @@ cross-references. See below for details.
 
 .. _Sphinx C Domain: http://www.sphinx-doc.org/en/stable/domains.html
 
+
+Parameters and member arguments
+-------------------------------
+
+The kernel-doc function comments describe each parameter to the function and
+function typedefs or each member of struct/union, in order, with the
+``@argument:`` descriptions. For each non-private member argument, one
+``@argument`` definition is needed.
+
+The ``@argument:`` descriptions begin on the very next line following
+the opening brief function description line, with no intervening blank
+comment lines.
+
+The ``@argument:`` descriptions may span multiple lines.
+
+.. note::
+
+   If the ``@argument`` description has multiple lines, the continuation
+   of the description should be starting exactly at the same column as
+   the previous line, e. g.::
+
+      * @argument: some long description
+      *       that continues on next lines
+
+   or::
+
+      * @argument:
+      *		some long description
+      *		that continues on next lines
+
+If a function or typedef parameter argument is ``...`` (e. g. a variable
+number of arguments), its description should be listed in kernel-doc
+notation as::
+
+      * @...: description
+
+Private members
+~~~~~~~~~~~~~~~
+
+Inside a struct or union description, you can use the ``private:`` and
+``public:`` comment tags. Structure fields that are inside a ``private:``
+area are not listed in the generated output documentation.
+
+The ``private:`` and ``public:`` tags must begin immediately following a
+``/*`` comment marker.  They may optionally include comments between the
+``:`` and the ending ``*/`` marker.
+
+Example::
+
+  /**
+   * struct my_struct - short description
+   * @a: first member
+   * @b: second member
+   * @d: fourth member
+   *
+   * Longer description
+   */
+  struct my_struct {
+      int a;
+      int b;
+  /* private: internal use only */
+      int c;
+  /* public: the next one is public */
+      int d;
+  };
+
+Function documentation
+----------------------
+
+The general format of a function and function-like macro kernel-doc comment is::
+
+  /**
+   * function_name() - Brief description of function.
+   * @arg1: Describe the first argument.
+   * @arg2: Describe the second argument.
+   *        One can provide multiple line descriptions
+   *        for arguments.
+   *
+   * A longer description, with more discussion of the function function_name()
+   * that might be useful to those using or modifying it. Begins with an
+   * empty comment line, and may include additional embedded empty
+   * comment lines.
+   *
+   * The longer description may have multiple paragraphs.
+   *
+   * Return: Describe the return value of foobar.
+   *
+   * The return value description can also have multiple paragraphs, and should
+   * be placed at the end of the comment block.
+   */
+
+The brief description following the function name may span multiple lines, and
+ends with an argument description, a blank comment line, or the end of the
+comment block.
+
+Return values
+~~~~~~~~~~~~~
+
+The return value, if any, should be described in a dedicated section
+named ``Return``.
+
+.. note::
+
+  #) The multi-line descriptive text you provide does *not* recognize
+     line breaks, so if you try to format some text nicely, as in::
+
+	* Return:
+	* 0 - OK
+	* -EINVAL - invalid argument
+	* -ENOMEM - out of memory
+
+     this will all run together and produce::
+
+	Return: 0 - OK -EINVAL - invalid argument -ENOMEM - out of memory
+
+     So, in order to produce the desired line breaks, you need to use a
+     ReST list, e. g.::
+
+      * Return:
+      * * 0		- OK to runtime suspend the device
+      * * -EBUSY	- Device should not be runtime suspended
+
+  #) If the descriptive text you provide has lines that begin with
+     some phrase followed by a colon, each of those phrases will be taken
+     as a new section heading, with probably won't produce the desired
+     effect.
+
+Structure, union, and enumeration documentation
+-----------------------------------------------
+
+The general format of a struct, union, and enum kernel-doc comment is::
+
+  /**
+   * struct struct_name - Brief description.
+   * @argument: Description of member member_name.
+   *
+   * Description of the structure.
+   */
+
+On the above, ``struct`` is used to mean structs. You can also use ``union``
+and ``enum``  to describe unions and enums. ``argument`` is used
+to mean struct and union member names as well as enumerations in an enum.
+
+The brief description following the structure name may span multiple lines, and
+ends with a member description, a blank comment line, or the end of the
+comment block.
+
+The kernel-doc data structure comments describe each member of the structure,
+in order, with the member descriptions.
+
+Nested structs/unions
+~~~~~~~~~~~~~~~~~~~~~
+
+It is possible to document nested structs unions, like::
+
+      /**
+       * struct nested_foobar - a struct with nested unions and structs
+       * @arg1: - first argument of anonymous union/anonymous struct
+       * @arg2: - second argument of anonymous union/anonymous struct
+       * @arg3: - third argument of anonymous union/anonymous struct
+       * @arg4: - fourth argument of anonymous union/anonymous struct
+       * @bar.st1.arg1 - first argument of struct st1 on union bar
+       * @bar.st1.arg2 - second argument of struct st1 on union bar
+       * @bar.st2.arg1 - first argument of struct st2 on union bar
+       * @bar.st2.arg2 - second argument of struct st2 on union bar
+      struct nested_foobar {
+        /* Anonymous union/struct*/
+        union {
+          struct {
+            int arg1;
+            int arg2;
+	  }
+          struct {
+            void *arg3;
+            int arg4;
+	  }
+	}
+	union {
+          struct {
+            int arg1;
+            int arg2;
+	  } st1;
+          struct {
+            void *arg1;
+            int arg2;
+	  } st2;
+	} bar;
+      };
+
+.. note::
+
+   #) When documenting nested structs or unions, if the struct/union ``foo``
+      is named, the argument ``bar`` inside it should be documented as
+      ``@foo.bar:``
+   #) When the nested struct/union is anonymous, the argument ``bar`` on it
+      should be documented as ``@bar:``
+
+Typedef documentation
+---------------------
+
+The general format of a typedef kernel-doc comment is::
+
+  /**
+   * typedef type_name - Brief description.
+   *
+   * Description of the type.
+   */
+
+Typedefs with function prototypes can also be documented::
+
+  /**
+   * typedef type_name - Brief description.
+   * @arg1: description of arg1
+   * @arg2: description of arg2
+   *
+   * Description of the type.
+   */
+   typedef void (*type_name)(struct v4l2_ctrl *arg1, void *arg2);
+
+
 Highlights and cross-references
 -------------------------------
 
@@ -201,70 +422,7 @@ cross-references.
 
 For further details, please refer to the `Sphinx C Domain`_ documentation.
 
-Function documentation
-----------------------
-
-The general format of a function and function-like macro kernel-doc comment is::
-
-  /**
-   * function_name() - Brief description of function.
-   * @arg1: Describe the first argument.
-   * @arg2: Describe the second argument.
-   *        One can provide multiple line descriptions
-   *        for arguments.
-   *
-   * A longer description, with more discussion of the function function_name()
-   * that might be useful to those using or modifying it. Begins with an
-   * empty comment line, and may include additional embedded empty
-   * comment lines.
-   *
-   * The longer description may have multiple paragraphs.
-   *
-   * Return: Describe the return value of foobar.
-   *
-   * The return value description can also have multiple paragraphs, and should
-   * be placed at the end of the comment block.
-   */
-
-The brief description following the function name may span multiple lines, and
-ends with an ``@argument:`` description, a blank comment line, or the end of the
-comment block.
-
-The kernel-doc function comments describe each parameter to the function, in
-order, with the ``@argument:`` descriptions. The ``@argument:`` descriptions
-must begin on the very next line following the opening brief function
-description line, with no intervening blank comment lines. The ``@argument:``
-descriptions may span multiple lines. The continuation lines may contain
-indentation. If a function parameter is ``...`` (varargs), it should be listed
-in kernel-doc notation as: ``@...:``.
-
-The return value, if any, should be described in a dedicated section at the end
-of the comment starting with "Return:".
-
-Structure, union, and enumeration documentation
------------------------------------------------
-
-The general format of a struct, union, and enum kernel-doc comment is::
-
-  /**
-   * struct struct_name - Brief description.
-   * @member_name: Description of member member_name.
-   *
-   * Description of the structure.
-   */
-
-Below, "struct" is used to mean structs, unions and enums, and "member" is used
-to mean struct and union members as well as enumerations in an enum.
-
-The brief description following the structure name may span multiple lines, and
-ends with a ``@member:`` description, a blank comment line, or the end of the
-comment block.
 
-The kernel-doc data structure comments describe each member of the structure, in
-order, with the ``@member:`` descriptions. The ``@member:`` descriptions must
-begin on the very next line following the opening brief function description
-line, with no intervening blank comment lines. The ``@member:`` descriptions may
-span multiple lines. The continuation lines may contain indentation.
 
 In-line member documentation comments
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -294,42 +452,6 @@ on a line of their own, like all other kernel-doc comments::
         int foobar;
   }
 
-Private members
-~~~~~~~~~~~~~~~
-
-Inside a struct description, you can use the "private:" and "public:" comment
-tags. Structure fields that are inside a "private:" area are not listed in the
-generated output documentation.  The "private:" and "public:" tags must begin
-immediately following a ``/*`` comment marker.  They may optionally include
-comments between the ``:`` and the ending ``*/`` marker.
-
-Example::
-
-  /**
-   * struct my_struct - short description
-   * @a: first member
-   * @b: second member
-   *
-   * Longer description
-   */
-  struct my_struct {
-      int a;
-      int b;
-  /* private: internal use only */
-      int c;
-  };
-
-
-Typedef documentation
----------------------
-
-The general format of a typedef kernel-doc comment is::
-
-  /**
-   * typedef type_name - Brief description.
-   *
-   * Description of the type.
-   */
 
 Overview documentation comments
 -------------------------------
@@ -376,3 +498,37 @@ file.
 
 Data structures visible in kernel include files should also be documented using
 kernel-doc formatted comments.
+
+How to use kernel-doc to generate man pages
+-------------------------------------------
+
+If you just want to use kernel-doc to generate man pages you can do this
+from the Kernel git tree::
+
+  $ scripts/kernel-doc -man $(git grep -l '/\*\*' |grep -v Documentation/) | ./split-man.pl /tmp/man
+
+Using the small ``split-man.pl`` script below::
+
+
+  #!/usr/bin/perl
+
+  if ($#ARGV < 0) {
+     die "where do I put the results?\n";
+  }
+
+  mkdir $ARGV[0],0777;
+  $state = 0;
+  while (<STDIN>) {
+      if (/^\.TH \"[^\"]*\" 9 \"([^\"]*)\"/) {
+	if ($state == 1) { close OUT }
+	$state = 1;
+	$fn = "$ARGV[0]/$1.9";
+	print STDERR "Creating $fn\n";
+	open OUT, ">$fn" or die "can't open $fn: $!\n";
+	print OUT $_;
+      } elsif ($state != 0) {
+	print OUT $_;
+      }
+  }
+
+  close OUT;
diff --git a/Documentation/driver-api/basics.rst b/Documentation/driver-api/basics.rst
index 73fa7d42bbba..826e85d50a16 100644
--- a/Documentation/driver-api/basics.rst
+++ b/Documentation/driver-api/basics.rst
@@ -13,12 +13,6 @@ Driver device table
 .. kernel-doc:: include/linux/mod_devicetable.h
    :internal:
 
-Atomic and pointer manipulation
--------------------------------
-
-.. kernel-doc:: arch/x86/include/asm/atomic.h
-   :internal:
-
 Delaying, scheduling, and timer routines
 ----------------------------------------
 
@@ -85,6 +79,21 @@ Internal Functions
 .. kernel-doc:: kernel/kthread.c
    :export:
 
+Reference counting
+------------------
+
+.. kernel-doc:: include/linux/refcount.h
+   :internal:
+
+.. kernel-doc:: lib/refcount.c
+   :export:
+
+Atomics
+-------
+
+.. kernel-doc:: arch/x86/include/asm/atomic.h
+   :internal:
+
 Kernel objects manipulation
 ---------------------------
 
diff --git a/Documentation/driver-api/dmaengine/provider.rst b/Documentation/driver-api/dmaengine/provider.rst
index 814acb4d2294..dfc4486b5743 100644
--- a/Documentation/driver-api/dmaengine/provider.rst
+++ b/Documentation/driver-api/dmaengine/provider.rst
@@ -111,40 +111,36 @@ The first thing you need to do in your driver is to allocate this
 structure. Any of the usual memory allocators will do, but you'll also
 need to initialize a few fields in there:
 
-- channels: should be initialized as a list using the
+- ``channels``: should be initialized as a list using the
   INIT_LIST_HEAD macro for example
 
-- src_addr_widths:
+- ``src_addr_widths``:
   should contain a bitmask of the supported source transfer width
 
-- dst_addr_widths:
+- ``dst_addr_widths``:
   should contain a bitmask of the supported destination transfer width
 
-- directions:
+- ``directions``:
   should contain a bitmask of the supported slave directions
   (i.e. excluding mem2mem transfers)
 
-- residue_granularity:
+- ``residue_granularity``:
+  granularity of the transfer residue reported to dma_set_residue.
+  This can be either:
 
-  - Granularity of the transfer residue reported to dma_set_residue.
-    This can be either:
+  - Descriptor:
+    your device doesn't support any kind of residue
+    reporting. The framework will only know that a particular
+    transaction descriptor is done.
 
-  - Descriptor
+  - Segment:
+    your device is able to report which chunks have been transferred
 
-    - Your device doesn't support any kind of residue
-      reporting. The framework will only know that a particular
-      transaction descriptor is done.
+  - Burst:
+    your device is able to report which burst have been transferred
 
-      - Segment
-
-        - Your device is able to report which chunks have been transferred
-
-      - Burst
-
-        - Your device is able to report which burst have been transferred
-
-  - dev: should hold the pointer to the ``struct device`` associated
-    to your current driver instance.
+- ``dev``: should hold the pointer to the ``struct device`` associated
+  to your current driver instance.
 
 Supported transaction types
 ---------------------------
diff --git a/Documentation/driver-api/firmware/built-in-fw.rst b/Documentation/driver-api/firmware/built-in-fw.rst
index 7300e66857f8..396cdf591ac5 100644
--- a/Documentation/driver-api/firmware/built-in-fw.rst
+++ b/Documentation/driver-api/firmware/built-in-fw.rst
@@ -11,13 +11,8 @@ options:
   * CONFIG_EXTRA_FIRMWARE
   * CONFIG_EXTRA_FIRMWARE_DIR
 
-This should not be confused with CONFIG_FIRMWARE_IN_KERNEL, this is for drivers
-which enables firmware to be built as part of the kernel build process. This
-option, CONFIG_FIRMWARE_IN_KERNEL, will build all firmware for all drivers
-enabled which ship its firmware inside the Linux kernel source tree.
-
 There are a few reasons why you might want to consider building your firmware
-into the kernel with CONFIG_EXTRA_FIRMWARE though:
+into the kernel with CONFIG_EXTRA_FIRMWARE:
 
 * Speed
 * Firmware is needed for accessing the boot device, and the user doesn't
diff --git a/Documentation/driver-api/firmware/fallback-mechanisms.rst b/Documentation/driver-api/firmware/fallback-mechanisms.rst
index d19354794e67..4055ac76b288 100644
--- a/Documentation/driver-api/firmware/fallback-mechanisms.rst
+++ b/Documentation/driver-api/firmware/fallback-mechanisms.rst
@@ -71,7 +71,7 @@ via fw_create_instance(). This call creates a new struct device named after
 the firmware requested, and establishes it in the device hierarchy by
 associating the device used to make the request as the device's parent.
 The sysfs directory's file attributes are defined and controlled through
-the new device's class (firmare_class) and group (fw_dev_attr_groups).
+the new device's class (firmware_class) and group (fw_dev_attr_groups).
 This is actually where the original firmware_class.c file name comes from,
 as originally the only firmware loading mechanism available was the
 mechanism we now use as a fallback mechanism.
diff --git a/Documentation/driver-api/iio/hw-consumer.rst b/Documentation/driver-api/iio/hw-consumer.rst
new file mode 100644
index 000000000000..8facce6a6733
--- /dev/null
+++ b/Documentation/driver-api/iio/hw-consumer.rst
@@ -0,0 +1,51 @@
+===========
+HW consumer
+===========
+An IIO device can be directly connected to another device in hardware. in this
+case the buffers between IIO provider and IIO consumer are handled by hardware.
+The Industrial I/O HW consumer offers a way to bond these IIO devices without
+software buffer for data. The implementation can be found under
+:file:`drivers/iio/buffer/hw-consumer.c`
+
+
+* struct :c:type:`iio_hw_consumer` — Hardware consumer structure
+* :c:func:`iio_hw_consumer_alloc` — Allocate IIO hardware consumer
+* :c:func:`iio_hw_consumer_free` — Free IIO hardware consumer
+* :c:func:`iio_hw_consumer_enable` — Enable IIO hardware consumer
+* :c:func:`iio_hw_consumer_disable` — Disable IIO hardware consumer
+
+
+HW consumer setup
+=================
+
+As standard IIO device the implementation is based on IIO provider/consumer.
+A typical IIO HW consumer setup looks like this::
+
+	static struct iio_hw_consumer *hwc;
+
+	static const struct iio_info adc_info = {
+		.read_raw = adc_read_raw,
+	};
+
+	static int adc_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan, int *val,
+				int *val2, long mask)
+	{
+		ret = iio_hw_consumer_enable(hwc);
+
+		/* Acquire data */
+
+		ret = iio_hw_consumer_disable(hwc);
+	}
+
+	static int adc_probe(struct platform_device *pdev)
+	{
+		hwc = devm_iio_hw_consumer_alloc(&iio->dev);
+	}
+
+More details
+============
+.. kernel-doc:: include/linux/iio/hw-consumer.h
+.. kernel-doc:: drivers/iio/buffer/industrialio-hw-consumer.c
+   :export:
+
diff --git a/Documentation/driver-api/iio/index.rst b/Documentation/driver-api/iio/index.rst
index e5c3922d1b6f..7fba341bd8b2 100644
--- a/Documentation/driver-api/iio/index.rst
+++ b/Documentation/driver-api/iio/index.rst
@@ -15,3 +15,4 @@ Contents:
    buffers
    triggers
    triggered-buffers
+   hw-consumer
diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst
index d17a9876b473..e9b41b1634f3 100644
--- a/Documentation/driver-api/index.rst
+++ b/Documentation/driver-api/index.rst
@@ -47,6 +47,8 @@ available subsections can be seen below.
    gpio
    misc_devices
    dmaengine/index
+   slimbus
+   soundwire/index
 
 .. only::  subproject and html
 
diff --git a/Documentation/driver-api/pm/devices.rst b/Documentation/driver-api/pm/devices.rst
index 53c1b0b06da5..1128705a5731 100644
--- a/Documentation/driver-api/pm/devices.rst
+++ b/Documentation/driver-api/pm/devices.rst
@@ -777,17 +777,51 @@ The driver can indicate that by setting ``DPM_FLAG_SMART_SUSPEND`` in
 runtime suspend at the beginning of the ``suspend_late`` phase of system-wide
 suspend (or in the ``poweroff_late`` phase of hibernation), when runtime PM
 has been disabled for it, under the assumption that its state should not change
-after that point until the system-wide transition is over.  If that happens, the
-driver's system-wide resume callbacks, if present, may still be invoked during
-the subsequent system-wide resume transition and the device's runtime power
-management status may be set to "active" before enabling runtime PM for it,
-so the driver must be prepared to cope with the invocation of its system-wide
-resume callbacks back-to-back with its ``->runtime_suspend`` one (without the
-intervening ``->runtime_resume`` and so on) and the final state of the device
-must reflect the "active" status for runtime PM in that case.
+after that point until the system-wide transition is over (the PM core itself
+does that for devices whose "noirq", "late" and "early" system-wide PM callbacks
+are executed directly by it).  If that happens, the driver's system-wide resume
+callbacks, if present, may still be invoked during the subsequent system-wide
+resume transition and the device's runtime power management status may be set
+to "active" before enabling runtime PM for it, so the driver must be prepared to
+cope with the invocation of its system-wide resume callbacks back-to-back with
+its ``->runtime_suspend`` one (without the intervening ``->runtime_resume`` and
+so on) and the final state of the device must reflect the "active" runtime PM
+status in that case.
 
 During system-wide resume from a sleep state it's easiest to put devices into
 the full-power state, as explained in :file:`Documentation/power/runtime_pm.txt`.
-Refer to that document for more information regarding this particular issue as
+[Refer to that document for more information regarding this particular issue as
 well as for information on the device runtime power management framework in
-general.
+general.]
+
+However, it often is desirable to leave devices in suspend after system
+transitions to the working state, especially if those devices had been in
+runtime suspend before the preceding system-wide suspend (or analogous)
+transition.  Device drivers can use the ``DPM_FLAG_LEAVE_SUSPENDED`` flag to
+indicate to the PM core (and middle-layer code) that they prefer the specific
+devices handled by them to be left suspended and they have no problems with
+skipping their system-wide resume callbacks for this reason.  Whether or not the
+devices will actually be left in suspend may depend on their state before the
+given system suspend-resume cycle and on the type of the system transition under
+way.  In particular, devices are not left suspended if that transition is a
+restore from hibernation, as device states are not guaranteed to be reflected
+by the information stored in the hibernation image in that case.
+
+The middle-layer code involved in the handling of the device is expected to
+indicate to the PM core if the device may be left in suspend by setting its
+:c:member:`power.may_skip_resume` status bit which is checked by the PM core
+during the "noirq" phase of the preceding system-wide suspend (or analogous)
+transition.  The middle layer is then responsible for handling the device as
+appropriate in its "noirq" resume callback, which is executed regardless of
+whether or not the device is left suspended, but the other resume callbacks
+(except for ``->complete``) will be skipped automatically by the PM core if the
+device really can be left in suspend.
+
+For devices whose "noirq", "late" and "early" driver callbacks are invoked
+directly by the PM core, all of the system-wide resume callbacks are skipped if
+``DPM_FLAG_LEAVE_SUSPENDED`` is set and the device is in runtime suspend during
+the ``suspend_noirq`` (or analogous) phase or the transition under way is a
+proper system suspend (rather than anything related to hibernation) and the
+device's wakeup settings are suitable for runtime PM (that is, it cannot
+generate wakeup signals at all or it is allowed to wake up the system from
+sleep).
diff --git a/Documentation/driver-api/s390-drivers.rst b/Documentation/driver-api/s390-drivers.rst
index ecf8851d3565..30e6aa7e160b 100644
--- a/Documentation/driver-api/s390-drivers.rst
+++ b/Documentation/driver-api/s390-drivers.rst
@@ -22,9 +22,28 @@ While most I/O devices on a s390 system are typically driven through the
 channel I/O mechanism described here, there are various other methods
 (like the diag interface). These are out of the scope of this document.
 
+The s390 common I/O layer also provides access to some devices that are
+not strictly considered I/O devices. They are considered here as well,
+although they are not the focus of this document.
+
 Some additional information can also be found in the kernel source under
 Documentation/s390/driver-model.txt.
 
+The css bus
+===========
+
+The css bus contains the subchannels available on the system. They fall
+into several categories:
+
+* Standard I/O subchannels, for use by the system. They have a child
+  device on the ccw bus and are described below.
+* I/O subchannels bound to the vfio-ccw driver. See
+  Documentation/s390/vfio-ccw.txt.
+* Message subchannels. No Linux driver currently exists.
+* CHSC subchannels (at most one). The chsc subchannel driver can be used
+  to send asynchronous chsc commands.
+* eADM subchannels. Used for talking to storage class memory.
+
 The ccw bus
 ===========
 
@@ -102,10 +121,15 @@ ccw group devices
 Generic interfaces
 ==================
 
-Some interfaces are available to other drivers that do not necessarily
-have anything to do with the busses described above, but still are
-indirectly using basic infrastructure in the common I/O layer. One
-example is the support for adapter interrupts.
+The following section contains interfaces in use not only by drivers
+dealing with ccw devices, but drivers for various other s390 hardware
+as well.
+
+Adapter interrupts
+------------------
+
+The common I/O layer provides helper functions for dealing with adapter
+interrupts and interrupt vectors.
 
 .. kernel-doc:: drivers/s390/cio/airq.c
    :export:
diff --git a/Documentation/driver-api/scsi.rst b/Documentation/driver-api/scsi.rst
index 9ae03171daca..3ae337929721 100644
--- a/Documentation/driver-api/scsi.rst
+++ b/Documentation/driver-api/scsi.rst
@@ -224,6 +224,14 @@ mid to lowlevel SCSI driver interface
 .. kernel-doc:: drivers/scsi/hosts.c
    :export:
 
+drivers/scsi/scsi_common.c
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+general support functions
+
+.. kernel-doc:: drivers/scsi/scsi_common.c
+   :export:
+
 Transport classes
 -----------------
 
@@ -332,5 +340,5 @@ todo
 ~~~~
 
 Parallel (fast/wide/ultra) SCSI, USB, SATA, SAS, Fibre Channel,
-FireWire, ATAPI devices, Infiniband, I20, iSCSI, Parallel ports,
+FireWire, ATAPI devices, Infiniband, I2O, iSCSI, Parallel ports,
 netlink...
diff --git a/Documentation/driver-api/slimbus.rst b/Documentation/driver-api/slimbus.rst
new file mode 100644
index 000000000000..7555ecd538de
--- /dev/null
+++ b/Documentation/driver-api/slimbus.rst
@@ -0,0 +1,127 @@
+============================
+Linux kernel SLIMbus support
+============================
+
+Overview
+========
+
+What is SLIMbus?
+----------------
+SLIMbus (Serial Low Power Interchip Media Bus) is a specification developed by
+MIPI (Mobile Industry Processor Interface) alliance. The bus uses master/slave
+configuration, and is a 2-wire multi-drop implementation (clock, and data).
+
+Currently, SLIMbus is used to interface between application processors of SoCs
+(System-on-Chip) and peripheral components (typically codec). SLIMbus uses
+Time-Division-Multiplexing to accommodate multiple data channels, and
+a control channel.
+
+The control channel is used for various control functions such as bus
+management, configuration and status updates. These messages can be unicast (e.g.
+reading/writing device specific values), or multicast (e.g. data channel
+reconfiguration sequence is a broadcast message announced to all devices)
+
+A data channel is used for data-transfer between 2 SLIMbus devices. Data
+channel uses dedicated ports on the device.
+
+Hardware description:
+---------------------
+SLIMbus specification has different types of device classifications based on
+their capabilities.
+A manager device is responsible for enumeration, configuration, and dynamic
+channel allocation. Every bus has 1 active manager.
+
+A generic device is a device providing application functionality (e.g. codec).
+
+Framer device is responsible for clocking the bus, and transmitting frame-sync
+and framing information on the bus.
+
+Each SLIMbus component has an interface device for monitoring physical layer.
+
+Typically each SoC contains SLIMbus component having 1 manager, 1 framer device,
+1 generic device (for data channel support), and 1 interface device.
+External peripheral SLIMbus component usually has 1 generic device (for
+functionality/data channel support), and an associated interface device.
+The generic device's registers are mapped as 'value elements' so that they can
+be written/read using SLIMbus control channel exchanging control/status type of
+information.
+In case there are multiple framer devices on the same bus, manager device is
+responsible to select the active-framer for clocking the bus.
+
+Per specification, SLIMbus uses "clock gears" to do power management based on
+current frequency and bandwidth requirements. There are 10 clock gears and each
+gear changes the SLIMbus frequency to be twice its previous gear.
+
+Each device has a 6-byte enumeration-address and the manager assigns every
+device with a 1-byte logical address after the devices report presence on the
+bus.
+
+Software description:
+---------------------
+There are 2 types of SLIMbus drivers:
+
+slim_controller represents a 'controller' for SLIMbus. This driver should
+implement duties needed by the SoC (manager device, associated
+interface device for monitoring the layers and reporting errors, default
+framer device).
+
+slim_device represents the 'generic device/component' for SLIMbus, and a
+slim_driver should implement driver for that slim_device.
+
+Device notifications to the driver:
+-----------------------------------
+Since SLIMbus devices have mechanisms for reporting their presence, the
+framework allows drivers to bind when corresponding devices report their
+presence on the bus.
+However, it is possible that the driver needs to be probed
+first so that it can enable corresponding SLIMbus device (e.g. power it up and/or
+take it out of reset). To support that behavior, the framework allows drivers
+to probe first as well  (e.g. using standard DeviceTree compatibility field).
+This creates the necessity for the driver to know when the device is functional
+(i.e. reported present). device_up callback is used for that reason when the
+device reports present and is assigned a logical address by the controller.
+
+Similarly, SLIMbus devices 'report absent' when they go down. A 'device_down'
+callback notifies the driver when the device reports absent and its logical
+address assignment is invalidated by the controller.
+
+Another notification "boot_device" is used to notify the slim_driver when
+controller resets the bus. This notification allows the driver to take necessary
+steps to boot the device so that it's functional after the bus has been reset.
+
+Driver and Controller APIs:
+--------------------------
+.. kernel-doc:: include/linux/slimbus.h
+   :internal:
+
+.. kernel-doc:: drivers/slimbus/slimbus.h
+   :internal:
+
+.. kernel-doc:: drivers/slimbus/core.c
+   :export:
+
+Clock-pause:
+------------
+SLIMbus mandates that a reconfiguration sequence (known as clock-pause) be
+broadcast to all active devices on the bus before the bus can enter low-power
+mode. Controller uses this sequence when it decides to enter low-power mode so
+that corresponding clocks and/or power-rails can be turned off to save power.
+Clock-pause is exited by waking up framer device (if controller driver initiates
+exiting low power mode), or by toggling the data line (if a slave device wants
+to initiate it).
+
+Clock-pause APIs:
+~~~~~~~~~~~~~~~~~
+.. kernel-doc:: drivers/slimbus/sched.c
+   :export:
+
+Messaging:
+----------
+The framework supports regmap and read/write apis to exchange control-information
+with a SLIMbus device. APIs can be synchronous or asynchronous.
+The header file <linux/slimbus.h> has more documentation about messaging APIs.
+
+Messaging APIs:
+~~~~~~~~~~~~~~~
+.. kernel-doc:: drivers/slimbus/messaging.c
+   :export:
diff --git a/Documentation/driver-api/soundwire/index.rst b/Documentation/driver-api/soundwire/index.rst
new file mode 100644
index 000000000000..647e94654752
--- /dev/null
+++ b/Documentation/driver-api/soundwire/index.rst
@@ -0,0 +1,15 @@
+=======================
+SoundWire Documentation
+=======================
+
+.. toctree::
+   :maxdepth: 1
+
+   summary
+
+.. only::  subproject
+
+   Indices
+   =======
+
+   * :ref:`genindex`
diff --git a/Documentation/driver-api/soundwire/summary.rst b/Documentation/driver-api/soundwire/summary.rst
new file mode 100644
index 000000000000..8193125a2bfb
--- /dev/null
+++ b/Documentation/driver-api/soundwire/summary.rst
@@ -0,0 +1,207 @@
+===========================
+SoundWire Subsystem Summary
+===========================
+
+SoundWire is a new interface ratified in 2015 by the MIPI Alliance.
+SoundWire is used for transporting data typically related to audio
+functions. SoundWire interface is optimized to integrate audio devices in
+mobile or mobile inspired systems.
+
+SoundWire is a 2-pin multi-drop interface with data and clock line. It
+facilitates development of low cost, efficient, high performance systems.
+Broad level key features of SoundWire interface include:
+
+ (1) Transporting all of payload data channels, control information, and setup
+     commands over a single two-pin interface.
+
+ (2) Lower clock frequency, and hence lower power consumption, by use of DDR
+     (Dual Data Rate) data transmission.
+
+ (3) Clock scaling and optional multiple data lanes to give wide flexibility
+     in data rate to match system requirements.
+
+ (4) Device status monitoring, including interrupt-style alerts to the Master.
+
+The SoundWire protocol supports up to eleven Slave interfaces. All the
+interfaces share the common Bus containing data and clock line. Each of the
+Slaves can support up to 14 Data Ports. 13 Data Ports are dedicated to audio
+transport. Data Port0 is dedicated to transport of Bulk control information,
+each of the audio Data Ports (1..14) can support up to 8 Channels in
+transmit or receiving mode (typically fixed direction but configurable
+direction is enabled by the specification).  Bandwidth restrictions to
+~19.2..24.576Mbits/s don't however allow for 11*13*8 channels to be
+transmitted simultaneously.
+
+Below figure shows an example of connectivity between a SoundWire Master and
+two Slave devices. ::
+
+        +---------------+                                       +---------------+
+        |               |                       Clock Signal    |               |
+        |    Master     |-------+-------------------------------|    Slave      |
+        |   Interface   |       |               Data Signal     |  Interface 1  |
+        |               |-------|-------+-----------------------|               |
+        +---------------+       |       |                       +---------------+
+                                |       |
+                                |       |
+                                |       |
+                             +--+-------+--+
+                             |             |
+                             |   Slave     |
+                             | Interface 2 |
+                             |             |
+                             +-------------+
+
+
+Terminology
+===========
+
+The MIPI SoundWire specification uses the term 'device' to refer to a Master
+or Slave interface, which of course can be confusing. In this summary and
+code we use the term interface only to refer to the hardware. We follow the
+Linux device model by mapping each Slave interface connected on the bus as a
+device managed by a specific driver. The Linux SoundWire subsystem provides
+a framework to implement a SoundWire Slave driver with an API allowing
+3rd-party vendors to enable implementation-defined functionality while
+common setup/configuration tasks are handled by the bus.
+
+Bus:
+Implements SoundWire Linux Bus which handles the SoundWire protocol.
+Programs all the MIPI-defined Slave registers. Represents a SoundWire
+Master. Multiple instances of Bus may be present in a system.
+
+Slave:
+Registers as SoundWire Slave device (Linux Device). Multiple Slave devices
+can register to a Bus instance.
+
+Slave driver:
+Driver controlling the Slave device. MIPI-specified registers are controlled
+directly by the Bus (and transmitted through the Master driver/interface).
+Any implementation-defined Slave register is controlled by Slave driver. In
+practice, it is expected that the Slave driver relies on regmap and does not
+request direct register access.
+
+Programming interfaces (SoundWire Master interface Driver)
+==========================================================
+
+SoundWire Bus supports programming interfaces for the SoundWire Master
+implementation and SoundWire Slave devices. All the code uses the "sdw"
+prefix commonly used by SoC designers and 3rd party vendors.
+
+Each of the SoundWire Master interfaces needs to be registered to the Bus.
+Bus implements API to read standard Master MIPI properties and also provides
+callback in Master ops for Master driver to implement its own functions that
+provides capabilities information. DT support is not implemented at this
+time but should be trivial to add since capabilities are enabled with the
+``device_property_`` API.
+
+The Master interface along with the Master interface capabilities are
+registered based on board file, DT or ACPI.
+
+Following is the Bus API to register the SoundWire Bus:
+
+.. code-block:: c
+
+	int sdw_add_bus_master(struct sdw_bus *bus)
+	{
+		if (!bus->dev)
+			return -ENODEV;
+
+		mutex_init(&bus->lock);
+		INIT_LIST_HEAD(&bus->slaves);
+
+		/* Check ACPI for Slave devices */
+		sdw_acpi_find_slaves(bus);
+
+		/* Check DT for Slave devices */
+		sdw_of_find_slaves(bus);
+
+		return 0;
+	}
+
+This will initialize sdw_bus object for Master device. "sdw_master_ops" and
+"sdw_master_port_ops" callback functions are provided to the Bus.
+
+"sdw_master_ops" is used by Bus to control the Bus in the hardware specific
+way. It includes Bus control functions such as sending the SoundWire
+read/write messages on Bus, setting up clock frequency & Stream
+Synchronization Point (SSP). The "sdw_master_ops" structure abstracts the
+hardware details of the Master from the Bus.
+
+"sdw_master_port_ops" is used by Bus to setup the Port parameters of the
+Master interface Port. Master interface Port register map is not defined by
+MIPI specification, so Bus calls the "sdw_master_port_ops" callback
+function to do Port operations like "Port Prepare", "Port Transport params
+set", "Port enable and disable". The implementation of the Master driver can
+then perform hardware-specific configurations.
+
+Programming interfaces (SoundWire Slave Driver)
+===============================================
+
+The MIPI specification requires each Slave interface to expose a unique
+48-bit identifier, stored in 6 read-only dev_id registers. This dev_id
+identifier contains vendor and part information, as well as a field enabling
+to differentiate between identical components. An additional class field is
+currently unused. Slave driver is written for a specific vendor and part
+identifier, Bus enumerates the Slave device based on these two ids.
+Slave device and driver match is done based on these two ids . Probe
+of the Slave driver is called by Bus on successful match between device and
+driver id. A parent/child relationship is enforced between Master and Slave
+devices (the logical representation is aligned with the physical
+connectivity).
+
+The information on Master/Slave dependencies is stored in platform data,
+board-file, ACPI or DT. The MIPI Software specification defines additional
+link_id parameters for controllers that have multiple Master interfaces. The
+dev_id registers are only unique in the scope of a link, and the link_id
+unique in the scope of a controller. Both dev_id and link_id are not
+necessarily unique at the system level but the parent/child information is
+used to avoid ambiguity.
+
+.. code-block:: c
+
+	static const struct sdw_device_id slave_id[] = {
+	        SDW_SLAVE_ENTRY(0x025d, 0x700, 0),
+	        {},
+	};
+	MODULE_DEVICE_TABLE(sdw, slave_id);
+
+	static struct sdw_driver slave_sdw_driver = {
+	        .driver = {
+	                   .name = "slave_xxx",
+	                   .pm = &slave_runtime_pm,
+	                   },
+		.probe = slave_sdw_probe,
+		.remove = slave_sdw_remove,
+		.ops = &slave_slave_ops,
+		.id_table = slave_id,
+	};
+
+
+For capabilities, Bus implements API to read standard Slave MIPI properties
+and also provides callback in Slave ops for Slave driver to implement own
+function that provides capabilities information. Bus needs to know a set of
+Slave capabilities to program Slave registers and to control the Bus
+reconfigurations.
+
+Future enhancements to be done
+==============================
+
+ (1) Bulk Register Access (BRA) transfers.
+
+
+ (2) Multiple data lane support.
+
+Links
+=====
+
+SoundWire MIPI specification 1.1 is available at:
+https://members.mipi.org/wg/All-Members/document/70290
+
+SoundWire MIPI DisCo (Discovery and Configuration) specification is
+available at:
+https://www.mipi.org/specifications/mipi-disco-soundwire
+
+(publicly accessible with registration or directly accessible to MIPI
+members)
+
+MIPI Alliance Manufacturer ID Page: mid.mipi.org
diff --git a/Documentation/driver-api/uio-howto.rst b/Documentation/driver-api/uio-howto.rst
index f73d660b2956..693e3bd84e79 100644
--- a/Documentation/driver-api/uio-howto.rst
+++ b/Documentation/driver-api/uio-howto.rst
@@ -667,27 +667,28 @@ Making the driver recognize the device
 Since the driver does not declare any device GUID's, it will not get
 loaded automatically and will not automatically bind to any devices, you
 must load it and allocate id to the driver yourself. For example, to use
-the network device GUID::
+the network device class GUID::
 
      modprobe uio_hv_generic
      echo "f8615163-df3e-46c5-913f-f2d2f965ed0e" > /sys/bus/vmbus/drivers/uio_hv_generic/new_id
 
 If there already is a hardware specific kernel driver for the device,
 the generic driver still won't bind to it, in this case if you want to
-use the generic driver (why would you?) you'll have to manually unbind
-the hardware specific driver and bind the generic driver, like this::
+use the generic driver for a userspace library you'll have to manually unbind
+the hardware specific driver and bind the generic driver, using the device specific GUID
+like this::
 
-          echo -n vmbus-ed963694-e847-4b2a-85af-bc9cfc11d6f3 > /sys/bus/vmbus/drivers/hv_netvsc/unbind
-          echo -n vmbus-ed963694-e847-4b2a-85af-bc9cfc11d6f3 > /sys/bus/vmbus/drivers/uio_hv_generic/bind
+          echo -n ed963694-e847-4b2a-85af-bc9cfc11d6f3 > /sys/bus/vmbus/drivers/hv_netvsc/unbind
+          echo -n ed963694-e847-4b2a-85af-bc9cfc11d6f3 > /sys/bus/vmbus/drivers/uio_hv_generic/bind
 
 You can verify that the device has been bound to the driver by looking
 for it in sysfs, for example like the following::
 
-        ls -l /sys/bus/vmbus/devices/vmbus-ed963694-e847-4b2a-85af-bc9cfc11d6f3/driver
+        ls -l /sys/bus/vmbus/devices/ed963694-e847-4b2a-85af-bc9cfc11d6f3/driver
 
 Which if successful should print::
 
-      .../vmbus-ed963694-e847-4b2a-85af-bc9cfc11d6f3/driver -> ../../../bus/vmbus/drivers/uio_hv_generic
+      .../ed963694-e847-4b2a-85af-bc9cfc11d6f3/driver -> ../../../bus/vmbus/drivers/uio_hv_generic
 
 Things to know about uio_hv_generic
 -----------------------------------
@@ -697,6 +698,17 @@ prevents the device from generating further interrupts until the bit is
 cleared. The userspace driver should clear this bit before blocking and
 waiting for more interrupts.
 
+When host rescinds a device, the interrupt file descriptor is marked down
+and any reads of the interrupt file descriptor will return -EIO. Similar
+to a closed socket or disconnected serial device.
+
+The vmbus device regions are mapped into uio device resources:
+    0) Channel ring buffers: guest to host and host to guest
+    1) Guest to host interrupt signalling pages
+    2) Guest to host monitor page
+    3) Network receive buffer region
+    4) Network send buffer region
+
 Further information
 ===================
 
diff --git a/Documentation/driver-api/usb/usb3-debug-port.rst b/Documentation/driver-api/usb/usb3-debug-port.rst
index feb1a36a65b7..b9fd131f4723 100644
--- a/Documentation/driver-api/usb/usb3-debug-port.rst
+++ b/Documentation/driver-api/usb/usb3-debug-port.rst
@@ -98,3 +98,55 @@ you to check the sanity of the setup.
 	cat /dev/ttyUSB0
 	done
 	===== end of bash scripts ===============
+
+Serial TTY
+==========
+
+The DbC support has been added to the xHCI driver. You can get a
+debug device provided by the DbC at runtime.
+
+In order to use this, you need to make sure your kernel has been
+configured to support USB_XHCI_DBGCAP. A sysfs attribute under
+the xHCI device node is used to enable or disable DbC. By default,
+DbC is disabled::
+
+	root@target:/sys/bus/pci/devices/0000:00:14.0# cat dbc
+	disabled
+
+Enable DbC with the following command::
+
+	root@target:/sys/bus/pci/devices/0000:00:14.0# echo enable > dbc
+
+You can check the DbC state at anytime::
+
+	root@target:/sys/bus/pci/devices/0000:00:14.0# cat dbc
+	enabled
+
+Connect the debug target to the debug host with a USB 3.0 super-
+speed A-to-A debugging cable. You can see /dev/ttyDBC0 created
+on the debug target. You will see below kernel message lines::
+
+	root@target: tail -f /var/log/kern.log
+	[  182.730103] xhci_hcd 0000:00:14.0: DbC connected
+	[  191.169420] xhci_hcd 0000:00:14.0: DbC configured
+	[  191.169597] xhci_hcd 0000:00:14.0: DbC now attached to /dev/ttyDBC0
+
+Accordingly, the DbC state has been brought up to::
+
+	root@target:/sys/bus/pci/devices/0000:00:14.0# cat dbc
+	configured
+
+On the debug host, you will see the debug device has been enumerated.
+You will see below kernel message lines::
+
+	root@host: tail -f /var/log/kern.log
+	[   79.454780] usb 2-2.1: new SuperSpeed USB device number 3 using xhci_hcd
+	[   79.475003] usb 2-2.1: LPM exit latency is zeroed, disabling LPM.
+	[   79.475389] usb 2-2.1: New USB device found, idVendor=1d6b, idProduct=0010
+	[   79.475390] usb 2-2.1: New USB device strings: Mfr=1, Product=2, SerialNumber=3
+	[   79.475391] usb 2-2.1: Product: Linux USB Debug Target
+	[   79.475392] usb 2-2.1: Manufacturer: Linux Foundation
+	[   79.475393] usb 2-2.1: SerialNumber: 0001
+
+The debug device works now. You can use any communication or debugging
+program to talk between the host and the target.
diff --git a/Documentation/driver-api/usb/writing_usb_driver.rst b/Documentation/driver-api/usb/writing_usb_driver.rst
index 69f077dcdb78..4fe1c06b6a13 100644
--- a/Documentation/driver-api/usb/writing_usb_driver.rst
+++ b/Documentation/driver-api/usb/writing_usb_driver.rst
@@ -321,6 +321,6 @@ linux-usb-devel Mailing List Archives:
 http://marc.theaimsgroup.com/?l=linux-usb-devel
 
 Programming Guide for Linux USB Device Drivers:
-http://usb.cs.tum.edu/usbdoc
+http://lmu.web.psi.ch/docu/manuals/software_manuals/linux_sl/usb_linux_programming_guide.pdf
 
 USB Home Page: http://www.usb.org
diff --git a/Documentation/driver-model/devres.txt b/Documentation/driver-model/devres.txt
index c180045eb43b..7c1bb3d0c222 100644
--- a/Documentation/driver-model/devres.txt
+++ b/Documentation/driver-model/devres.txt
@@ -384,6 +384,9 @@ RESET
   devm_reset_control_get()
   devm_reset_controller_register()
 
+SERDEV
+  devm_serdev_device_open()
+
 SLAVE DMA ENGINE
   devm_acpi_dma_controller_register()
 
diff --git a/Documentation/fault-injection/fault-injection.txt b/Documentation/fault-injection/fault-injection.txt
index 918972babcd8..de1dc35fe500 100644
--- a/Documentation/fault-injection/fault-injection.txt
+++ b/Documentation/fault-injection/fault-injection.txt
@@ -1,7 +1,7 @@
 Fault injection capabilities infrastructure
 ===========================================
 
-See also drivers/md/faulty.c and "every_nth" module option for scsi_debug.
+See also drivers/md/md-faulty.c and "every_nth" module option for scsi_debug.
 
 
 Available fault injection capabilities
@@ -30,6 +30,12 @@ o fail_mmc_request
   injects MMC data errors on devices permitted by setting
   debugfs entries under /sys/kernel/debug/mmc0/fail_mmc_request
 
+o fail_function
+
+  injects error return on specific functions, which are marked by
+  ALLOW_ERROR_INJECTION() macro, by setting debugfs entries
+  under /sys/kernel/debug/fail_function. No boot option supported.
+
 Configure fault-injection capabilities behavior
 -----------------------------------------------
 
@@ -123,6 +129,29 @@ configuration of fault-injection capabilities.
 	default is 'N', setting it to 'Y' will disable failure injections
 	when dealing with private (address space) futexes.
 
+- /sys/kernel/debug/fail_function/inject:
+
+	Format: { 'function-name' | '!function-name' | '' }
+	specifies the target function of error injection by name.
+	If the function name leads '!' prefix, given function is
+	removed from injection list. If nothing specified ('')
+	injection list is cleared.
+
+- /sys/kernel/debug/fail_function/injectable:
+
+	(read only) shows error injectable functions and what type of
+	error values can be specified. The error type will be one of
+	below;
+	- NULL:	retval must be 0.
+	- ERRNO: retval must be -1 to -MAX_ERRNO (-4096).
+	- ERR_NULL: retval must be 0 or -1 to -MAX_ERRNO (-4096).
+
+- /sys/kernel/debug/fail_function/<functiuon-name>/retval:
+
+	specifies the "error" return value to inject to the given
+	function for given function. This will be created when
+	user specifies new injection entry.
+
 o Boot option
 
 In order to inject faults while debugfs is not available (early boot time),
@@ -268,6 +297,45 @@ trap "echo 0 > /sys/kernel/debug/$FAILTYPE/probability" SIGINT SIGTERM EXIT
 echo "Injecting errors into the module $module... (interrupt to stop)"
 sleep 1000000
 
+------------------------------------------------------------------------------
+
+o Inject open_ctree error while btrfs mount
+
+#!/bin/bash
+
+rm -f testfile.img
+dd if=/dev/zero of=testfile.img bs=1M seek=1000 count=1
+DEVICE=$(losetup --show -f testfile.img)
+mkfs.btrfs -f $DEVICE
+mkdir -p tmpmnt
+
+FAILTYPE=fail_function
+FAILFUNC=open_ctree
+echo $FAILFUNC > /sys/kernel/debug/$FAILTYPE/inject
+echo -12 > /sys/kernel/debug/$FAILTYPE/$FAILFUNC/retval
+echo N > /sys/kernel/debug/$FAILTYPE/task-filter
+echo 100 > /sys/kernel/debug/$FAILTYPE/probability
+echo 0 > /sys/kernel/debug/$FAILTYPE/interval
+echo -1 > /sys/kernel/debug/$FAILTYPE/times
+echo 0 > /sys/kernel/debug/$FAILTYPE/space
+echo 1 > /sys/kernel/debug/$FAILTYPE/verbose
+
+mount -t btrfs $DEVICE tmpmnt
+if [ $? -ne 0 ]
+then
+	echo "SUCCESS!"
+else
+	echo "FAILED!"
+	umount tmpmnt
+fi
+
+echo > /sys/kernel/debug/$FAILTYPE/inject
+
+rmdir tmpmnt
+losetup -d $DEVICE
+rm testfile.img
+
+
 Tool to run command with failslab or fail_page_alloc
 ----------------------------------------------------
 In order to make it easier to accomplish the tasks mentioned above, we can use
diff --git a/Documentation/features/debug/KASAN/arch-support.txt b/Documentation/features/debug/KASAN/arch-support.txt
index f377290fe48e..3406fae833c3 100644
--- a/Documentation/features/debug/KASAN/arch-support.txt
+++ b/Documentation/features/debug/KASAN/arch-support.txt
@@ -35,5 +35,5 @@
     |          um: | TODO |
     |   unicore32: | TODO |
     |         x86: |  ok  | 64-bit only
-    |      xtensa: | TODO |
+    |      xtensa: |  ok  |
     -----------------------
diff --git a/Documentation/features/debug/stackprotector/arch-support.txt b/Documentation/features/debug/stackprotector/arch-support.txt
index d7acd7bd3619..59a4c9ffb7f3 100644
--- a/Documentation/features/debug/stackprotector/arch-support.txt
+++ b/Documentation/features/debug/stackprotector/arch-support.txt
@@ -35,5 +35,5 @@
     |          um: | TODO |
     |   unicore32: | TODO |
     |         x86: |  ok  |
-    |      xtensa: | TODO |
+    |      xtensa: |  ok  |
     -----------------------
diff --git a/Documentation/features/sched/membarrier-sync-core/arch-support.txt b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
new file mode 100644
index 000000000000..2c815a7f1ba7
--- /dev/null
+++ b/Documentation/features/sched/membarrier-sync-core/arch-support.txt
@@ -0,0 +1,62 @@
+#
+# Feature name:          membarrier-sync-core
+#         Kconfig:       ARCH_HAS_MEMBARRIER_SYNC_CORE
+#         description:   arch supports core serializing membarrier
+#
+# Architecture requirements
+#
+# * arm64
+#
+# Rely on eret context synchronization when returning from IPI handler, and
+# when returning to user-space.
+#
+# * x86
+#
+# x86-32 uses IRET as return from interrupt, which takes care of the IPI.
+# However, it uses both IRET and SYSEXIT to go back to user-space. The IRET
+# instruction is core serializing, but not SYSEXIT.
+#
+# x86-64 uses IRET as return from interrupt, which takes care of the IPI.
+# However, it can return to user-space through either SYSRETL (compat code),
+# SYSRETQ, or IRET.
+#
+# Given that neither SYSRET{L,Q}, nor SYSEXIT, are core serializing, we rely
+# instead on write_cr3() performed by switch_mm() to provide core serialization
+# after changing the current mm, and deal with the special case of kthread ->
+# uthread (temporarily keeping current mm into active_mm) by issuing a
+# sync_core_before_usermode() in that specific case.
+#
+    -----------------------
+    |         arch |status|
+    -----------------------
+    |       alpha: | TODO |
+    |         arc: | TODO |
+    |         arm: | TODO |
+    |       arm64: |  ok  |
+    |    blackfin: | TODO |
+    |         c6x: | TODO |
+    |        cris: | TODO |
+    |         frv: | TODO |
+    |       h8300: | TODO |
+    |     hexagon: | TODO |
+    |        ia64: | TODO |
+    |        m32r: | TODO |
+    |        m68k: | TODO |
+    |       metag: | TODO |
+    |  microblaze: | TODO |
+    |        mips: | TODO |
+    |     mn10300: | TODO |
+    |       nios2: | TODO |
+    |    openrisc: | TODO |
+    |      parisc: | TODO |
+    |     powerpc: | TODO |
+    |        s390: | TODO |
+    |       score: | TODO |
+    |          sh: | TODO |
+    |       sparc: | TODO |
+    |        tile: | TODO |
+    |          um: | TODO |
+    |   unicore32: | TODO |
+    |         x86: |  ok  |
+    |      xtensa: | TODO |
+    -----------------------
diff --git a/Documentation/filesystems/afs.txt b/Documentation/filesystems/afs.txt
index ba99b5ac4fd8..c5254f6d234d 100644
--- a/Documentation/filesystems/afs.txt
+++ b/Documentation/filesystems/afs.txt
@@ -7,6 +7,7 @@ Contents:
  - Overview.
  - Usage.
  - Mountpoints.
+ - Dynamic root.
  - Proc filesystem.
  - The cell database.
  - Security.
@@ -127,6 +128,22 @@ mounted on /afs in one go by doing:
 	umount /afs
 
 
+============
+DYNAMIC ROOT
+============
+
+A mount option is available to create a serverless mount that is only usable
+for dynamic lookup.  Creating such a mount can be done by, for example:
+
+	mount -t afs none /afs -o dyn
+
+This creates a mount that just has an empty directory at the root.  Attempting
+to look up a name in this directory will cause a mountpoint to be created that
+looks up a cell of the same name, for example:
+
+	ls /afs/grand.central.org/
+
+
 ===============
 PROC FILESYSTEM
 ===============
diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt
index 3be3b266be41..70cb68bed2e8 100644
--- a/Documentation/filesystems/dax.txt
+++ b/Documentation/filesystems/dax.txt
@@ -46,7 +46,6 @@ stall the CPU for an extended period, you should also not attempt to
 implement direct_access.
 
 These block devices may be used for inspiration:
-- axonram: Axon DDR2 device driver
 - brd: RAM backed block device driver
 - dcssblk: s390 dcss block device driver
 - pmem: NVDIMM persistent memory driver
diff --git a/Documentation/filesystems/ext2.txt b/Documentation/filesystems/ext2.txt
index 55755395d3dc..81c0becab225 100644
--- a/Documentation/filesystems/ext2.txt
+++ b/Documentation/filesystems/ext2.txt
@@ -49,12 +49,10 @@ sb=n				Use alternate superblock at this location.
 
 user_xattr			Enable "user." POSIX Extended Attributes
 				(requires CONFIG_EXT2_FS_XATTR).
-				See also http://acl.bestbits.at
 nouser_xattr			Don't support "user." extended attributes.
 
 acl				Enable POSIX Access Control Lists support
 				(requires CONFIG_EXT2_FS_POSIX_ACL).
-				See also http://acl.bestbits.at
 noacl				Don't support POSIX ACLs.
 
 nobh				Do not attach buffer_heads to file pagecache.
diff --git a/Documentation/filesystems/ext4.txt b/Documentation/filesystems/ext4.txt
index 75236c0c2ac2..7f628b9f7c4b 100644
--- a/Documentation/filesystems/ext4.txt
+++ b/Documentation/filesystems/ext4.txt
@@ -202,15 +202,14 @@ inode_readahead_blks=n	This tuning parameter controls the maximum
 			the buffer cache.  The default value is 32 blocks.
 
 nouser_xattr		Disables Extended User Attributes.  See the
-			attr(5) manual page and http://acl.bestbits.at/
-			for more information about extended attributes.
+			attr(5) manual page for more information about
+			extended attributes.
 
 noacl			This option disables POSIX Access Control List
 			support. If ACL support is enabled in the kernel
 			configuration (CONFIG_EXT4_FS_POSIX_ACL), ACL is
 			enabled by default on mount. See the acl(5) manual
-			page and http://acl.bestbits.at/ for more information
-			about acl.
+			page for more information about acl.
 
 bsddf		(*)	Make 'df' act like BSD.
 minixdf			Make 'df' act like Minix.
@@ -233,7 +232,7 @@ data_err=ignore(*)	Just print an error message if an error occurs
 data_err=abort		Abort the journal if an error occurs in a file
 			data buffer in ordered mode.
 
-grpid			Give objects the same group ID as their creator.
+grpid			New objects have the group ID of their parent.
 bsdgroups
 
 nogrpid		(*)	New objects have the group ID of their creator.
diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst
index 776ddc655f79..cfbc18f0d9c9 100644
--- a/Documentation/filesystems/fscrypt.rst
+++ b/Documentation/filesystems/fscrypt.rst
@@ -448,8 +448,14 @@ astute users may notice some differences in behavior:
 
 - The st_size of an encrypted symlink will not necessarily give the
   length of the symlink target as required by POSIX.  It will actually
-  give the length of the ciphertext, which may be slightly longer than
-  the plaintext due to the NUL-padding.
+  give the length of the ciphertext, which will be slightly longer
+  than the plaintext due to NUL-padding and an extra 2-byte overhead.
+
+- The maximum length of an encrypted symlink is 2 bytes shorter than
+  the maximum length of an unencrypted symlink.  For example, on an
+  EXT4 filesystem with a 4K block size, unencrypted symlinks can be up
+  to 4095 bytes long, while encrypted symlinks can only be up to 4093
+  bytes long (both lengths excluding the terminating null).
 
 Note that mmap *is* supported.  This is possible because the pagecache
 for an encrypted file contains the plaintext, not the ciphertext.
diff --git a/Documentation/filesystems/nfs/Exporting b/Documentation/filesystems/nfs/Exporting
index 520a4becb75c..63889149f532 100644
--- a/Documentation/filesystems/nfs/Exporting
+++ b/Documentation/filesystems/nfs/Exporting
@@ -56,13 +56,25 @@ a/ A dentry flag DCACHE_DISCONNECTED which is set on
    any dentry that might not be part of the proper prefix.
    This is set when anonymous dentries are created, and cleared when a
    dentry is noticed to be a child of a dentry which is in the proper
-   prefix. 
-
-b/ A per-superblock list "s_anon" of dentries which are the roots of
-   subtrees that are not in the proper prefix.  These dentries, as
-   well as the proper prefix, need to be released at unmount time.  As
-   these dentries will not be hashed, they are linked together on the
-   d_hash list_head.
+   prefix.  If the refcount on a dentry with this flag set
+   becomes zero, the dentry is immediately discarded, rather than being
+   kept in the dcache.  If a dentry that is not already in the dcache
+   is repeatedly accessed by filehandle (as NFSD might do), an new dentry
+   will be a allocated for each access, and discarded at the end of
+   the access.
+
+   Note that such a dentry can acquire children, name, ancestors, etc.
+   without losing DCACHE_DISCONNECTED - that flag is only cleared when
+   subtree is successfully reconnected to root.  Until then dentries
+   in such subtree are retained only as long as there are references;
+   refcount reaching zero means immediate eviction, same as for unhashed
+   dentries.  That guarantees that we won't need to hunt them down upon
+   umount.
+
+b/ A primitive for creation of secondary roots - d_obtain_root(inode).
+   Those do _not_ bear DCACHE_DISCONNECTED.  They are placed on the
+   per-superblock list (->s_roots), so they can be located at umount
+   time for eviction purposes.
 
 c/ Helper routines to allocate anonymous dentries, and to help attach
    loose directory dentries at lookup time. They are:
@@ -77,7 +89,6 @@ c/ Helper routines to allocate anonymous dentries, and to help attach
       (such as an anonymous one created by d_obtain_alias), if appropriate.
       It returns NULL when the passed-in dentry is used, following the calling
       convention of ->lookup.
-
  
 Filesystem Issues
 -----------------
diff --git a/Documentation/filesystems/overlayfs.txt b/Documentation/filesystems/overlayfs.txt
index e6a5f4912b6d..6ea1e64d1464 100644
--- a/Documentation/filesystems/overlayfs.txt
+++ b/Documentation/filesystems/overlayfs.txt
@@ -190,6 +190,20 @@ Mount options:
     Redirects are not created and not followed (equivalent to "redirect_dir=off"
     if "redirect_always_follow" feature is not enabled).
 
+When the NFS export feature is enabled, every copied up directory is
+indexed by the file handle of the lower inode and a file handle of the
+upper directory is stored in a "trusted.overlay.upper" extended attribute
+on the index entry.  On lookup of a merged directory, if the upper
+directory does not match the file handle stores in the index, that is an
+indication that multiple upper directories may be redirected to the same
+lower directory.  In that case, lookup returns an error and warns about
+a possible inconsistency.
+
+Because lower layer redirects cannot be verified with the index, enabling
+NFS export support on an overlay filesystem with no upper layer requires
+turning off redirect follow (e.g. "redirect_dir=nofollow").
+
+
 Non-directories
 ---------------
 
@@ -281,9 +295,9 @@ filesystem, so both st_dev and st_ino of the file may change.
 
 Any open files referring to this inode will access the old data.
 
-If a file with multiple hard links is copied up, then this will
-"break" the link.  Changes will not be propagated to other names
-referring to the same inode.
+Unless "inode index" feature is enabled, if a file with multiple hard
+links is copied up, then this will "break" the link.  Changes will not be
+propagated to other names referring to the same inode.
 
 Unless "redirect_dir" feature is enabled, rename(2) on a lower or merged
 directory will fail with EXDEV.
@@ -299,6 +313,92 @@ filesystem are not allowed.  If the underlying filesystem is changed,
 the behavior of the overlay is undefined, though it will not result in
 a crash or deadlock.
 
+When the overlay NFS export feature is enabled, overlay filesystems
+behavior on offline changes of the underlying lower layer is different
+than the behavior when NFS export is disabled.
+
+On every copy_up, an NFS file handle of the lower inode, along with the
+UUID of the lower filesystem, are encoded and stored in an extended
+attribute "trusted.overlay.origin" on the upper inode.
+
+When the NFS export feature is enabled, a lookup of a merged directory,
+that found a lower directory at the lookup path or at the path pointed
+to by the "trusted.overlay.redirect" extended attribute, will verify
+that the found lower directory file handle and lower filesystem UUID
+match the origin file handle that was stored at copy_up time.  If a
+found lower directory does not match the stored origin, that directory
+will not be merged with the upper directory.
+
+
+
+NFS export
+----------
+
+When the underlying filesystems supports NFS export and the "nfs_export"
+feature is enabled, an overlay filesystem may be exported to NFS.
+
+With the "nfs_export" feature, on copy_up of any lower object, an index
+entry is created under the index directory.  The index entry name is the
+hexadecimal representation of the copy up origin file handle.  For a
+non-directory object, the index entry is a hard link to the upper inode.
+For a directory object, the index entry has an extended attribute
+"trusted.overlay.upper" with an encoded file handle of the upper
+directory inode.
+
+When encoding a file handle from an overlay filesystem object, the
+following rules apply:
+
+1. For a non-upper object, encode a lower file handle from lower inode
+2. For an indexed object, encode a lower file handle from copy_up origin
+3. For a pure-upper object and for an existing non-indexed upper object,
+   encode an upper file handle from upper inode
+
+The encoded overlay file handle includes:
+ - Header including path type information (e.g. lower/upper)
+ - UUID of the underlying filesystem
+ - Underlying filesystem encoding of underlying inode
+
+This encoding format is identical to the encoding format file handles that
+are stored in extended attribute "trusted.overlay.origin".
+
+When decoding an overlay file handle, the following steps are followed:
+
+1. Find underlying layer by UUID and path type information.
+2. Decode the underlying filesystem file handle to underlying dentry.
+3. For a lower file handle, lookup the handle in index directory by name.
+4. If a whiteout is found in index, return ESTALE. This represents an
+   overlay object that was deleted after its file handle was encoded.
+5. For a non-directory, instantiate a disconnected overlay dentry from the
+   decoded underlying dentry, the path type and index inode, if found.
+6. For a directory, use the connected underlying decoded dentry, path type
+   and index, to lookup a connected overlay dentry.
+
+Decoding a non-directory file handle may return a disconnected dentry.
+copy_up of that disconnected dentry will create an upper index entry with
+no upper alias.
+
+When overlay filesystem has multiple lower layers, a middle layer
+directory may have a "redirect" to lower directory.  Because middle layer
+"redirects" are not indexed, a lower file handle that was encoded from the
+"redirect" origin directory, cannot be used to find the middle or upper
+layer directory.  Similarly, a lower file handle that was encoded from a
+descendant of the "redirect" origin directory, cannot be used to
+reconstruct a connected overlay path.  To mitigate the cases of
+directories that cannot be decoded from a lower file handle, these
+directories are copied up on encode and encoded as an upper file handle.
+On an overlay filesystem with no upper layer this mitigation cannot be
+used NFS export in this setup requires turning off redirect follow (e.g.
+"redirect_dir=nofollow").
+
+The overlay filesystem does not support non-directory connectable file
+handles, so exporting with the 'subtree_check' exportfs configuration will
+cause failures to lookup files over NFS.
+
+When the NFS export feature is enabled, all directory index entries are
+verified on mount time to check that upper file handles are not stale.
+This verification may cause significant overhead in some cases.
+
+
 Testsuite
 ---------
 
diff --git a/Documentation/filesystems/sysfs.txt b/Documentation/filesystems/sysfs.txt
index 9a3658cc399e..a1426cabcef1 100644
--- a/Documentation/filesystems/sysfs.txt
+++ b/Documentation/filesystems/sysfs.txt
@@ -154,8 +154,8 @@ static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr,
         if (dev_attr->show)
                 ret = dev_attr->show(dev, dev_attr, buf);
         if (ret >= (ssize_t)PAGE_SIZE) {
-                print_symbol("dev_attr_show: %s returned bad count\n",
-                                (unsigned long)dev_attr->show);
+                printk("dev_attr_show: %pS returned bad count\n",
+                                dev_attr->show);
         }
         return ret;
 }
diff --git a/Documentation/filesystems/vfat.txt b/Documentation/filesystems/vfat.txt
index cf51360e3a9f..91031298beb1 100644
--- a/Documentation/filesystems/vfat.txt
+++ b/Documentation/filesystems/vfat.txt
@@ -344,4 +344,4 @@ the following:
 	   characters in the final slot are set to Unicode 0xFFFF.
 
 Finally, note that the extended name is stored in Unicode.  Each Unicode
-character takes two bytes.
+character takes either two or four bytes, UTF-16LE encoded.
diff --git a/Documentation/fpga/fpga-mgr.txt b/Documentation/fpga/fpga-mgr.txt
index 78f197fadfd1..cc6413ed6fc9 100644
--- a/Documentation/fpga/fpga-mgr.txt
+++ b/Documentation/fpga/fpga-mgr.txt
@@ -11,61 +11,65 @@ hidden away in a low level driver which registers a set of ops with the core.
 The FPGA image data itself is very manufacturer specific, but for our purposes
 it's just binary data.  The FPGA manager core won't parse it.
 
+The FPGA image to be programmed can be in a scatter gather list, a single
+contiguous buffer, or a firmware file.  Because allocating contiguous kernel
+memory for the buffer should be avoided, users are encouraged to use a scatter
+gather list instead if possible.
+
+The particulars for programming the image are presented in a structure (struct
+fpga_image_info).  This struct contains parameters such as pointers to the
+FPGA image as well as image-specific particulars such as whether the image was
+built for full or partial reconfiguration.
 
 API Functions:
 ==============
 
-To program the FPGA from a file or from a buffer:
--------------------------------------------------
-
-	int fpga_mgr_buf_load(struct fpga_manager *mgr,
-			      struct fpga_image_info *info,
-		              const char *buf, size_t count);
-
-Load the FPGA from an image which exists as a contiguous buffer in
-memory. Allocating contiguous kernel memory for the buffer should be avoided,
-users are encouraged to use the _sg interface instead of this.
-
-        int fpga_mgr_buf_load_sg(struct fpga_manager *mgr,
-				 struct fpga_image_info *info,
-				 struct sg_table *sgt);
+To program the FPGA:
+--------------------
 
-Load the FPGA from an image from non-contiguous in memory. Callers can
-construct a sg_table using alloc_page backed memory.
+	int fpga_mgr_load(struct fpga_manager *mgr,
+			  struct fpga_image_info *info);
 
-	int fpga_mgr_firmware_load(struct fpga_manager *mgr,
-				   struct fpga_image_info *info,
-		                   const char *image_name);
-
-Load the FPGA from an image which exists as a file.  The image file must be on
-the firmware search path (see the firmware class documentation).  If successful,
+Load the FPGA from an image which is indicated in the info.  If successful,
 the FPGA ends up in operating mode.  Return 0 on success or a negative error
 code.
 
-A FPGA design contained in a FPGA image file will likely have particulars that
-affect how the image is programmed to the FPGA.  These are contained in struct
-fpga_image_info.  Currently the only such particular is a single flag bit
-indicating whether the image is for full or partial reconfiguration.
+To allocate or free a struct fpga_image_info:
+---------------------------------------------
+
+	struct fpga_image_info *fpga_image_info_alloc(struct device *dev);
+
+	void fpga_image_info_free(struct fpga_image_info *info);
 
 To get/put a reference to a FPGA manager:
 -----------------------------------------
 
 	struct fpga_manager *of_fpga_mgr_get(struct device_node *node);
 	struct fpga_manager *fpga_mgr_get(struct device *dev);
+	void fpga_mgr_put(struct fpga_manager *mgr);
 
-Given a DT node or device, get an exclusive reference to a FPGA manager.
+Given a DT node or device, get a reference to a FPGA manager.  This pointer
+can be saved until you are ready to program the FPGA.  fpga_mgr_put releases
+the reference.
 
-	void fpga_mgr_put(struct fpga_manager *mgr);
 
-Release the reference.
+To get exclusive control of a FPGA manager:
+-------------------------------------------
+
+	int fpga_mgr_lock(struct fpga_manager *mgr);
+	void fpga_mgr_unlock(struct fpga_manager *mgr);
+
+The user should call fpga_mgr_lock and verify that it returns 0 before
+attempting to program the FPGA.  Likewise, the user should call
+fpga_mgr_unlock when done programming the FPGA.
 
 
 To register or unregister the low level FPGA-specific driver:
 -------------------------------------------------------------
 
 	int fpga_mgr_register(struct device *dev, const char *name,
-		              const struct fpga_manager_ops *mops,
-		              void *priv);
+			      const struct fpga_manager_ops *mops,
+			      void *priv);
 
 	void fpga_mgr_unregister(struct device *dev);
 
@@ -75,62 +79,58 @@ device."
 
 How to write an image buffer to a supported FPGA
 ================================================
-/* Include to get the API */
 #include <linux/fpga/fpga-mgr.h>
 
-/* device node that specifies the FPGA manager to use */
-struct device_node *mgr_node = ...
+struct fpga_manager *mgr;
+struct fpga_image_info *info;
+int ret;
 
-/* FPGA image is in this buffer.  count is size of the buffer. */
-char *buf = ...
-int count = ...
+/*
+ * Get a reference to FPGA manager.  The manager is not locked, so you can
+ * hold onto this reference without it preventing programming.
+ *
+ * This example uses the device node of the manager.  Alternatively, use
+ * fpga_mgr_get(dev) instead if you have the device.
+ */
+mgr = of_fpga_mgr_get(mgr_node);
 
 /* struct with information about the FPGA image to program. */
-struct fpga_image_info info;
+info = fpga_image_info_alloc(dev);
 
 /* flags indicates whether to do full or partial reconfiguration */
-info.flags = 0;
+info->flags = FPGA_MGR_PARTIAL_RECONFIG;
 
-int ret;
+/*
+ * At this point, indicate where the image is. This is pseudo-code; you're
+ * going to use one of these three.
+ */
+if (image is in a scatter gather table) {
 
-/* Get exclusive control of FPGA manager */
-struct fpga_manager *mgr = of_fpga_mgr_get(mgr_node);
+	info->sgt = [your scatter gather table]
 
-/* Load the buffer to the FPGA */
-ret = fpga_mgr_buf_load(mgr, &info, buf, count);
-
-/* Release the FPGA manager */
-fpga_mgr_put(mgr);
-
-
-How to write an image file to a supported FPGA
-==============================================
-/* Include to get the API */
-#include <linux/fpga/fpga-mgr.h>
+} else if (image is in a buffer) {
 
-/* device node that specifies the FPGA manager to use */
-struct device_node *mgr_node = ...
+	info->buf = [your image buffer]
+	info->count = [image buffer size]
 
-/* FPGA image is in this file which is in the firmware search path */
-const char *path = "fpga-image-9.rbf"
+} else if (image is in a firmware file) {
 
-/* struct with information about the FPGA image to program. */
-struct fpga_image_info info;
-
-/* flags indicates whether to do full or partial reconfiguration */
-info.flags = 0;
+	info->firmware_name = devm_kstrdup(dev, firmware_name, GFP_KERNEL);
 
-int ret;
+}
 
 /* Get exclusive control of FPGA manager */
-struct fpga_manager *mgr = of_fpga_mgr_get(mgr_node);
+ret = fpga_mgr_lock(mgr);
 
-/* Get the firmware image (path) and load it to the FPGA */
-ret = fpga_mgr_firmware_load(mgr, &info, path);
+/* Load the buffer to the FPGA */
+ret = fpga_mgr_buf_load(mgr, &info, buf, count);
 
 /* Release the FPGA manager */
+fpga_mgr_unlock(mgr);
 fpga_mgr_put(mgr);
 
+/* Deallocate the image info if you're done with it */
+fpga_image_info_free(info);
 
 How to support a new FPGA device
 ================================
diff --git a/Documentation/fpga/fpga-region.txt b/Documentation/fpga/fpga-region.txt
new file mode 100644
index 000000000000..139a02ba1ff6
--- /dev/null
+++ b/Documentation/fpga/fpga-region.txt
@@ -0,0 +1,95 @@
+FPGA Regions
+
+Alan Tull 2017
+
+CONTENTS
+ - Introduction
+ - The FPGA region API
+ - Usage example
+
+Introduction
+============
+
+This document is meant to be an brief overview of the FPGA region API usage.  A
+more conceptual look at regions can be found in [1].
+
+For the purposes of this API document, let's just say that a region associates
+an FPGA Manager and a bridge (or bridges) with a reprogrammable region of an
+FPGA or the whole FPGA.  The API provides a way to register a region and to
+program a region.
+
+Currently the only layer above fpga-region.c in the kernel is the Device Tree
+support (of-fpga-region.c) described in [1].  The DT support layer uses regions
+to program the FPGA and then DT to handle enumeration.  The common region code
+is intended to be used by other schemes that have other ways of accomplishing
+enumeration after programming.
+
+An fpga-region can be set up to know the following things:
+* which FPGA manager to use to do the programming
+* which bridges to disable before programming and enable afterwards.
+
+Additional info needed to program the FPGA image is passed in the struct
+fpga_image_info [2] including:
+* pointers to the image as either a scatter-gather buffer, a contiguous
+  buffer, or the name of firmware file
+* flags indicating specifics such as whether the image if for partial
+  reconfiguration.
+
+===================
+The FPGA region API
+===================
+
+To register or unregister a region:
+-----------------------------------
+
+	int fpga_region_register(struct device *dev,
+				 struct fpga_region *region);
+	int fpga_region_unregister(struct fpga_region *region);
+
+An example of usage can be seen in the probe function of [3]
+
+To program an FPGA:
+-------------------
+	int fpga_region_program_fpga(struct fpga_region *region);
+
+This function operates on info passed in the fpga_image_info
+(region->info).
+
+This function will attempt to:
+ * lock the region's mutex
+ * lock the region's FPGA manager
+ * build a list of FPGA bridges if a method has been specified to do so
+ * disable the bridges
+ * program the FPGA
+ * re-enable the bridges
+ * release the locks
+
+=============
+Usage example
+=============
+
+First, allocate the info struct:
+
+	info = fpga_image_info_alloc(dev);
+	if (!info)
+		return -ENOMEM;
+
+Set flags as needed, i.e.
+
+	info->flags |= FPGA_MGR_PARTIAL_RECONFIG;
+
+Point to your FPGA image, such as:
+
+	info->sgt = &sgt;
+
+Add info to region and do the programming:
+
+	region->info = info;
+	ret = fpga_region_program_fpga(region);
+
+Then enumerate whatever hardware has appeared in the FPGA.
+
+--
+[1] ../devicetree/bindings/fpga/fpga-region.txt
+[2] ./fpga-mgr.txt
+[3] ../../drivers/fpga/of-fpga-region.c
diff --git a/Documentation/fpga/overview.txt b/Documentation/fpga/overview.txt
new file mode 100644
index 000000000000..0f1236e7e675
--- /dev/null
+++ b/Documentation/fpga/overview.txt
@@ -0,0 +1,23 @@
+Linux kernel FPGA support
+
+Alan Tull 2017
+
+The main point of this project has been to separate the out the upper layers
+that know when to reprogram a FPGA from the lower layers that know how to
+reprogram a specific FPGA device.  The intention is to make this manufacturer
+agnostic, understanding that of course the FPGA images are very device specific
+themselves.
+
+The framework in the kernel includes:
+* low level FPGA manager drivers that know how to program a specific device
+* the fpga-mgr framework they are registered with
+* low level FPGA bridge drivers for hard/soft bridges which are intended to
+  be disable during FPGA programming
+* the fpga-bridge framework they are registered with
+* the fpga-region framework which associates and controls managers and bridges
+  as reconfigurable regions
+* the of-fpga-region support for reprogramming FPGAs when device tree overlays
+  are applied.
+
+I would encourage you the user to add code that creates FPGA regions rather
+that trying to control managers and bridges separately.
diff --git a/Documentation/gpio/board.txt b/Documentation/gpio/board.txt
index a0f61898d493..659bb19f5b3c 100644
--- a/Documentation/gpio/board.txt
+++ b/Documentation/gpio/board.txt
@@ -2,6 +2,7 @@ GPIO Mappings
 =============
 
 This document explains how GPIOs can be assigned to given devices and functions.
+
 Note that it only applies to the new descriptor-based interface. For a
 description of the deprecated integer-based GPIO interface please refer to
 gpio-legacy.txt (actually, there is no real mapping possible with the old
@@ -49,7 +50,7 @@ This property will make GPIOs 15, 16 and 17 available to the driver under the
 
 	power = gpiod_get(dev, "power", GPIOD_OUT_HIGH);
 
-The led GPIOs will be active-high, while the power GPIO will be active-low (i.e.
+The led GPIOs will be active high, while the power GPIO will be active low (i.e.
 gpiod_is_active_low(power) will be true).
 
 The second parameter of the gpiod_get() functions, the con_id string, has to be
@@ -122,9 +123,14 @@ where
 	can be NULL, in which case it will match any function.
   - idx is the index of the GPIO within the function.
   - flags is defined to specify the following properties:
-	* GPIOF_ACTIVE_LOW	- to configure the GPIO as active-low
-	* GPIOF_OPEN_DRAIN	- GPIO pin is open drain type.
-	* GPIOF_OPEN_SOURCE	- GPIO pin is open source type.
+	* GPIO_ACTIVE_HIGH	- GPIO line is active high
+	* GPIO_ACTIVE_LOW	- GPIO line is active low
+	* GPIO_OPEN_DRAIN	- GPIO line is set up as open drain
+	* GPIO_OPEN_SOURCE	- GPIO line is set up as open source
+	* GPIO_PERSISTENT	- GPIO line is persistent during
+				  suspend/resume and maintains its value
+	* GPIO_TRANSITORY	- GPIO line is transitory and may loose its
+				  electrical state during suspend/resume
 
 In the future, these flags might be extended to support more properties.
 
diff --git a/Documentation/gpio/consumer.txt b/Documentation/gpio/consumer.txt
index 63e1bd1d88e3..d53e5b5cfc9c 100644
--- a/Documentation/gpio/consumer.txt
+++ b/Documentation/gpio/consumer.txt
@@ -66,6 +66,15 @@ for the GPIO. Values can be:
 * GPIOD_IN to initialize the GPIO as input.
 * GPIOD_OUT_LOW to initialize the GPIO as output with a value of 0.
 * GPIOD_OUT_HIGH to initialize the GPIO as output with a value of 1.
+* GPIOD_OUT_LOW_OPEN_DRAIN same as GPIOD_OUT_LOW but also enforce the line
+  to be electrically used with open drain.
+* GPIOD_OUT_HIGH_OPEN_DRAIN same as GPIOD_OUT_HIGH but also enforce the line
+  to be electrically used with open drain.
+
+The two last flags are used for use cases where open drain is mandatory, such
+as I2C: if the line is not already configured as open drain in the mappings
+(see board.txt), then open drain will be enforced anyway and a warning will be
+printed that the board configuration needs to be updated to match the use case.
 
 Both functions return either a valid GPIO descriptor, or an error code checkable
 with IS_ERR() (they will never return a NULL pointer). -ENOENT will be returned
@@ -184,7 +193,7 @@ A driver can also query the current direction of a GPIO:
 
 	int gpiod_get_direction(const struct gpio_desc *desc)
 
-This function will return either GPIOF_DIR_IN or GPIOF_DIR_OUT.
+This function returns 0 for output, 1 for input, or an error code in case of error.
 
 Be aware that there is no default direction for GPIOs. Therefore, **using a GPIO
 without setting its direction first is illegal and will result in undefined
@@ -240,59 +249,71 @@ that can't be accessed from hardIRQ handlers, these calls act the same as the
 spinlock-safe calls.
 
 
-Active-low State and Raw GPIO Values
-------------------------------------
-Device drivers like to manage the logical state of a GPIO, i.e. the value their
-device will actually receive, no matter what lies between it and the GPIO line.
-In some cases, it might make sense to control the actual GPIO line value. The
-following set of calls ignore the active-low property of a GPIO and work on the
-raw line value:
-
-	int gpiod_get_raw_value(const struct gpio_desc *desc)
-	void gpiod_set_raw_value(struct gpio_desc *desc, int value)
-	int gpiod_get_raw_value_cansleep(const struct gpio_desc *desc)
-	void gpiod_set_raw_value_cansleep(struct gpio_desc *desc, int value)
-	int gpiod_direction_output_raw(struct gpio_desc *desc, int value)
-
-The active-low state of a GPIO can also be queried using the following call:
-
-	int gpiod_is_active_low(const struct gpio_desc *desc)
-
-Note that these functions should only be used with great moderation ; a driver
-should not have to care about the physical line level.
-
-
-The active-low property
------------------------
-
-As a driver should not have to care about the physical line level, all of the
+The active low and open drain semantics
+---------------------------------------
+As a consumer should not have to care about the physical line level, all of the
 gpiod_set_value_xxx() or gpiod_set_array_value_xxx() functions operate with
-the *logical* value. With this they take the active-low property into account.
-This means that they check whether the GPIO is configured to be active-low,
+the *logical* value. With this they take the active low property into account.
+This means that they check whether the GPIO is configured to be active low,
 and if so, they manipulate the passed value before the physical line level is
 driven.
 
+The same is applicable for open drain or open source output lines: those do not
+actively drive their output high (open drain) or low (open source), they just
+switch their output to a high impedance value. The consumer should not need to
+care. (For details read about open drain in driver.txt.)
+
 With this, all the gpiod_set_(array)_value_xxx() functions interpret the
-parameter "value" as "active" ("1") or "inactive" ("0"). The physical line
+parameter "value" as "asserted" ("1") or "de-asserted" ("0"). The physical line
 level will be driven accordingly.
 
-As an example, if the active-low property for a dedicated GPIO is set, and the
-gpiod_set_(array)_value_xxx() passes "active" ("1"), the physical line level
+As an example, if the active low property for a dedicated GPIO is set, and the
+gpiod_set_(array)_value_xxx() passes "asserted" ("1"), the physical line level
 will be driven low.
 
 To summarize:
 
-Function (example)               active-low property  physical line
-gpiod_set_raw_value(desc, 0);        don't care           low
-gpiod_set_raw_value(desc, 1);        don't care           high
-gpiod_set_value(desc, 0);       default (active-high)     low
-gpiod_set_value(desc, 1);       default (active-high)     high
-gpiod_set_value(desc, 0);             active-low          high
-gpiod_set_value(desc, 1);             active-low          low
-
-Please note again that the set_raw/get_raw functions should be avoided as much
-as possible, especially by drivers which should not care about the actual
-physical line level and worry about the logical value instead.
+Function (example)                 line property          physical line
+gpiod_set_raw_value(desc, 0);      don't care             low
+gpiod_set_raw_value(desc, 1);      don't care             high
+gpiod_set_value(desc, 0);          default (active high)  low
+gpiod_set_value(desc, 1);          default (active high)  high
+gpiod_set_value(desc, 0);          active low             high
+gpiod_set_value(desc, 1);          active low             low
+gpiod_set_value(desc, 0);          default (active high)  low
+gpiod_set_value(desc, 1);          default (active high)  high
+gpiod_set_value(desc, 0);          open drain             low
+gpiod_set_value(desc, 1);          open drain             high impedance
+gpiod_set_value(desc, 0);          open source            high impedance
+gpiod_set_value(desc, 1);          open source            high
+
+It is possible to override these semantics using the *set_raw/'get_raw functions
+but it should be avoided as much as possible, especially by system-agnostic drivers
+which should not need to care about the actual physical line level and worry about
+the logical value instead.
+
+
+Accessing raw GPIO values
+-------------------------
+Consumers exist that need to manage the logical state of a GPIO line, i.e. the value
+their device will actually receive, no matter what lies between it and the GPIO
+line.
+
+The following set of calls ignore the active-low or open drain property of a GPIO and
+work on the raw line value:
+
+	int gpiod_get_raw_value(const struct gpio_desc *desc)
+	void gpiod_set_raw_value(struct gpio_desc *desc, int value)
+	int gpiod_get_raw_value_cansleep(const struct gpio_desc *desc)
+	void gpiod_set_raw_value_cansleep(struct gpio_desc *desc, int value)
+	int gpiod_direction_output_raw(struct gpio_desc *desc, int value)
+
+The active low state of a GPIO can also be queried using the following call:
+
+	int gpiod_is_active_low(const struct gpio_desc *desc)
+
+Note that these functions should only be used with great moderation; a driver
+should not have to care about the physical line level or open drain semantics.
 
 
 Access multiple GPIOs with a single function call
diff --git a/Documentation/gpio/driver.txt b/Documentation/gpio/driver.txt
index d8de1c7de85a..3392a0fd4c23 100644
--- a/Documentation/gpio/driver.txt
+++ b/Documentation/gpio/driver.txt
@@ -88,6 +88,10 @@ ending up in the pin control back-end "behind" the GPIO controller, usually
 closer to the actual pins. This way the pin controller can manage the below
 listed GPIO configurations.
 
+If a pin controller back-end is used, the GPIO controller or hardware
+description needs to provide "GPIO ranges" mapping the GPIO line offsets to pin
+numbers on the pin controller so they can properly cross-reference each other.
+
 
 GPIOs with debounce support
 ---------------------------
diff --git a/Documentation/gpio/drivers-on-gpio.txt b/Documentation/gpio/drivers-on-gpio.txt
index 9a78d385b92e..a2ccbab12eb7 100644
--- a/Documentation/gpio/drivers-on-gpio.txt
+++ b/Documentation/gpio/drivers-on-gpio.txt
@@ -28,11 +28,6 @@ hardware descriptions such as device tree or ACPI:
 - gpio-beeper: drivers/input/misc/gpio-beeper.c is used to provide a beep from
   an external speaker connected to a GPIO line.
 
-- gpio-tilt-polled: drivers/input/misc/gpio_tilt_polled.c provides tilt
-  detection switches using GPIO, which is useful for your homebrewn pinball
-  machine if for nothing else. It can detect different tilt angles of the
-  monitored object.
-
 - extcon-gpio: drivers/extcon/extcon-gpio.c is used when you need to read an
   external connector status, such as a headset line for an audio driver or an
   HDMI connector. It will provide a better userspace sysfs interface than GPIO.
diff --git a/Documentation/gpio/sysfs.txt b/Documentation/gpio/sysfs.txt
index aeab01aa4d00..6cdeab8650cd 100644
--- a/Documentation/gpio/sysfs.txt
+++ b/Documentation/gpio/sysfs.txt
@@ -1,6 +1,17 @@
 GPIO Sysfs Interface for Userspace
 ==================================
 
+THIS ABI IS DEPRECATED, THE ABI DOCUMENTATION HAS BEEN MOVED TO
+Documentation/ABI/obsolete/sysfs-gpio AND NEW USERSPACE CONSUMERS
+ARE SUPPOSED TO USE THE CHARACTER DEVICE ABI. THIS OLD SYSFS ABI WILL
+NOT BE DEVELOPED (NO NEW FEATURES), IT WILL JUST BE MAINTAINED.
+
+Refer to the examples in tools/gpio/* for an introduction to the new
+character device ABI. Also see the userspace header in
+include/uapi/linux/gpio.h
+
+The deprecated sysfs ABI
+------------------------
 Platforms which use the "gpiolib" implementors framework may choose to
 configure a sysfs user interface to GPIOs. This is different from the
 debugfs interface, since it provides control over GPIO direction and
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index 13dd237418cc..e37557b30f62 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -74,15 +74,6 @@ Helper Functions Reference
 .. kernel-doc:: drivers/gpu/drm/drm_atomic_helper.c
    :export:
 
-Legacy CRTC/Modeset Helper Functions Reference
-==============================================
-
-.. kernel-doc:: drivers/gpu/drm/drm_crtc_helper.c
-   :doc: overview
-
-.. kernel-doc:: drivers/gpu/drm/drm_crtc_helper.c
-   :export:
-
 Simple KMS Helper Reference
 ===========================
 
@@ -163,6 +154,9 @@ Panel Helper Reference
 .. kernel-doc:: drivers/gpu/drm/drm_panel.c
    :export:
 
+.. kernel-doc:: drivers/gpu/drm/drm_panel_orientation_quirks.c
+   :export:
+
 Display Port Helper Functions Reference
 =======================================
 
@@ -279,15 +273,6 @@ Flip-work Helper Reference
 .. kernel-doc:: drivers/gpu/drm/drm_flip_work.c
    :export:
 
-Plane Helper Reference
-======================
-
-.. kernel-doc:: drivers/gpu/drm/drm_plane_helper.c
-   :doc: overview
-
-.. kernel-doc:: drivers/gpu/drm/drm_plane_helper.c
-   :export:
-
 Auxiliary Modeset Helpers
 =========================
 
@@ -305,3 +290,21 @@ Framebuffer GEM Helper Reference
 
 .. kernel-doc:: drivers/gpu/drm/drm_gem_framebuffer_helper.c
    :export:
+
+Legacy Plane Helper Reference
+=============================
+
+.. kernel-doc:: drivers/gpu/drm/drm_plane_helper.c
+   :doc: overview
+
+.. kernel-doc:: drivers/gpu/drm/drm_plane_helper.c
+   :export:
+
+Legacy CRTC/Modeset Helper Functions Reference
+==============================================
+
+.. kernel-doc:: drivers/gpu/drm/drm_crtc_helper.c
+   :doc: overview
+
+.. kernel-doc:: drivers/gpu/drm/drm_crtc_helper.c
+   :export:
diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst
index 307284125d7a..2dcf5b42015d 100644
--- a/Documentation/gpu/drm-kms.rst
+++ b/Documentation/gpu/drm-kms.rst
@@ -263,14 +263,20 @@ Taken all together there's two consequences for the atomic design:
 
 - An atomic update is assembled and validated as an entirely free-standing pile
   of structures within the :c:type:`drm_atomic_state <drm_atomic_state>`
-  container. Again drivers can subclass that container for their own state
-  structure tracking needs. Only when a state is committed is it applied to the
-  driver and modeset objects. This way rolling back an update boils down to
-  releasing memory and unreferencing objects like framebuffers.
+  container. Driver private state structures are also tracked in the same
+  structure; see the next chapter.  Only when a state is committed is it applied
+  to the driver and modeset objects. This way rolling back an update boils down
+  to releasing memory and unreferencing objects like framebuffers.
 
 Read on in this chapter, and also in :ref:`drm_atomic_helper` for more detailed
 coverage of specific topics.
 
+Handling Driver Private State
+-----------------------------
+
+.. kernel-doc:: drivers/gpu/drm/drm_atomic.c
+   :doc: handling driver private state
+
 Atomic Mode Setting Function Reference
 --------------------------------------
 
diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst
index e94d3ac2bdd0..41dc881b00dc 100644
--- a/Documentation/gpu/i915.rst
+++ b/Documentation/gpu/i915.rst
@@ -347,10 +347,10 @@ GuC-specific firmware loader
 GuC-based command submission
 ----------------------------
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_guc_submission.c
+.. kernel-doc:: drivers/gpu/drm/i915/intel_guc_submission.c
    :doc: GuC-based command submission
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_guc_submission.c
+.. kernel-doc:: drivers/gpu/drm/i915/intel_guc_submission.c
    :internal:
 
 GuC Firmware Layout
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index 36625aa66c27..1e593370f64f 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -179,8 +179,39 @@ don't do this, drivers used dev_info/warn/err to make this differentiation. We
 now have DRM_DEV_* variants of the drm print macros, so we can start to convert
 those drivers back to using drm-formwatted specific log messages.
 
+Before you start this conversion please contact the relevant maintainers to make
+sure your work will be merged - not everyone agrees that the DRM dmesg macros
+are better.
+
 Contact: Sean Paul, Maintainer of the driver you plan to convert
 
+Convert drivers to use simple modeset suspend/resume
+----------------------------------------------------
+
+Most drivers (except i915 and nouveau) that use
+drm_atomic_helper_suspend/resume() can probably be converted to use
+drm_mode_config_helper_suspend/resume().
+
+Contact: Maintainer of the driver you plan to convert
+
+Convert drivers to use drm_fb_helper_fbdev_setup/teardown()
+-----------------------------------------------------------
+
+Most drivers can use drm_fb_helper_fbdev_setup() except maybe:
+
+- amdgpu which has special logic to decide whether to call
+  drm_helper_disable_unused_functions()
+
+- armada which isn't atomic and doesn't call
+  drm_helper_disable_unused_functions()
+
+- i915 which calls drm_fb_helper_initial_config() in a worker
+
+Drivers that use drm_framebuffer_remove() to clean up the fbdev framebuffer can
+probably use drm_fb_helper_fbdev_teardown().
+
+Contact: Maintainer of the driver you plan to convert
+
 Core refactorings
 =================
 
@@ -382,11 +413,6 @@ those drivers as simple as possible, so lots of room for refactoring:
   one of the ideas for having a shared dsi/dbi helper, abstracting away the
   transport details more.
 
-- tinydrm_lastclose could be drm_fb_helper_lastclose. Only thing we need
-  for that is to store the drm_fb_helper pointer somewhere in
-  drm_device->mode_config. And then we could roll that out to all the
-  drivers.
-
 - tinydrm_gem_cma_prime_import_sg_table should probably go into the cma
   helpers, as a _vmapped variant (since not every driver needs the vmap).
   And tinydrm_gem_cma_free_object could the be merged into
@@ -400,11 +426,6 @@ those drivers as simple as possible, so lots of room for refactoring:
   a drm_device wrong. Doesn't matter, since everyone else gets it wrong
   too :-)
 
-- With the fbdev pointer in dev->mode_config we could also make
-  suspend/resume helpers entirely generic, at least if we add a
-  dev->mode_config.suspend_state. We could even provide a generic pm_ops
-  structure with those.
-
 - also rework the drm_framebuffer_funcs->dirty hook wire-up, see above.
 
 Contact: Noralf Trønnes, Daniel Vetter
diff --git a/Documentation/gpu/tve200.rst b/Documentation/gpu/tve200.rst
index 69b17b324e12..152ea9398f7e 100644
--- a/Documentation/gpu/tve200.rst
+++ b/Documentation/gpu/tve200.rst
@@ -3,4 +3,4 @@
 ==================================
 
 .. kernel-doc:: drivers/gpu/drm/tve200/tve200_drv.c
-   :doc: Faraday TV Encoder 200
+   :doc: Faraday TV Encoder TVE200 DRM Driver
diff --git a/Documentation/hwmon/lm25066 b/Documentation/hwmon/lm25066
index 3fa6bf820c88..51b32aa203a8 100644
--- a/Documentation/hwmon/lm25066
+++ b/Documentation/hwmon/lm25066
@@ -8,11 +8,6 @@ Supported chips:
     Datasheets:
 	http://www.ti.com/lit/gpn/lm25056
 	http://www.ti.com/lit/gpn/lm25056a
-  * TI LM25063
-    Prefix: 'lm25063'
-    Addresses scanned: -
-    Datasheet:
-	To be announced
   * National Semiconductor LM25066
     Prefix: 'lm25066'
     Addresses scanned: -
@@ -42,7 +37,7 @@ Description
 -----------
 
 This driver supports hardware monitoring for National Semiconductor / TI LM25056,
-LM25063, LM25066, LM5064, and LM5066/LM5066I Power Management, Monitoring,
+LM25066, LM5064, and LM5066/LM5066I Power Management, Monitoring,
 Control, and Protection ICs.
 
 The driver is a client driver to the core PMBus driver. Please see
@@ -74,12 +69,8 @@ in1_input		Measured input voltage.
 in1_average		Average measured input voltage.
 in1_min			Minimum input voltage.
 in1_max			Maximum input voltage.
-in1_crit		Critical high input voltage (LM25063 only).
-in1_lcrit		Critical low input voltage (LM25063 only).
 in1_min_alarm		Input voltage low alarm.
 in1_max_alarm		Input voltage high alarm.
-in1_lcrit_alarm		Input voltage critical low alarm (LM25063 only).
-in1_crit_alarm		Input voltage critical high alarm. (LM25063 only).
 
 in2_label		"vmon"
 in2_input		Measured voltage on VAUX pin
@@ -94,16 +85,12 @@ in3_input		Measured output voltage.
 in3_average		Average measured output voltage.
 in3_min			Minimum output voltage.
 in3_min_alarm		Output voltage low alarm.
-in3_highest		Historical minimum output voltage (LM25063 only).
-in3_lowest		Historical maximum output voltage (LM25063 only).
 
 curr1_label		"iin"
 curr1_input		Measured input current.
 curr1_average		Average measured input current.
 curr1_max		Maximum input current.
-curr1_crit		Critical input current (LM25063 only).
 curr1_max_alarm		Input current high alarm.
-curr1_crit_alarm	Input current critical high alarm (LM25063 only).
 
 power1_label		"pin"
 power1_input		Measured input power.
@@ -113,11 +100,6 @@ power1_alarm		Input power alarm
 power1_input_highest	Historical maximum power.
 power1_reset_history	Write any value to reset maximum power history.
 
-power2_label		"pout". LM25063 only.
-power2_input		Measured output power.
-power2_max		Maximum output power limit.
-power2_crit		Critical output power limit.
-
 temp1_input		Measured temperature.
 temp1_max		Maximum temperature.
 temp1_crit		Critical high temperature.
diff --git a/Documentation/hwmon/max31785 b/Documentation/hwmon/max31785
index 45fb6093dec2..270c5f865261 100644
--- a/Documentation/hwmon/max31785
+++ b/Documentation/hwmon/max31785
@@ -17,8 +17,9 @@ management with temperature and remote voltage sensing. Various fan control
 features are provided, including PWM frequency control, temperature hysteresis,
 dual tachometer measurements, and fan health monitoring.
 
-For dual rotor fan configuration, the MAX31785 exposes the slowest rotor of the
-two in the fan[1-4]_input attributes.
+For dual-rotor configurations the MAX31785A exposes the second rotor tachometer
+readings in attributes fan[5-8]_input. By contrast the MAX31785 only exposes
+the slowest rotor measurement, and does so in the fan[1-4]_input attributes.
 
 Usage Notes
 -----------
@@ -31,7 +32,9 @@ Sysfs attributes
 
 fan[1-4]_alarm		Fan alarm.
 fan[1-4]_fault		Fan fault.
-fan[1-4]_input		Fan RPM.
+fan[1-8]_input		Fan RPM. On the MAX31785A, inputs 5-8 correspond to the
+			second rotor of fans 1-4
+fan[1-4]_target		Fan input target
 
 in[1-6]_crit		Critical maximum output voltage
 in[1-6]_crit_alarm	Output voltage critical high alarm
@@ -44,6 +47,12 @@ in[1-6]_max_alarm	Output voltage high alarm
 in[1-6]_min		Minimum output voltage
 in[1-6]_min_alarm	Output voltage low alarm
 
+pwm[1-4]		Fan target duty cycle (0..255)
+pwm[1-4]_enable		0: Full-speed
+			1: Manual PWM control
+			2: Automatic PWM (tach-feedback RPM fan-control)
+			3: Automatic closed-loop (temp-feedback fan-control)
+
 temp[1-11]_crit		Critical high temperature
 temp[1-11]_crit_alarm	Chip temperature critical high alarm
 temp[1-11]_input	Measured temperature
diff --git a/Documentation/hwmon/w83773g b/Documentation/hwmon/w83773g
new file mode 100644
index 000000000000..4cc6c0b8257f
--- /dev/null
+++ b/Documentation/hwmon/w83773g
@@ -0,0 +1,33 @@
+Kernel driver w83773g
+====================
+
+Supported chips:
+  * Nuvoton W83773G
+    Prefix: 'w83773g'
+    Addresses scanned: I2C 0x4c and 0x4d
+    Datasheet: https://www.nuvoton.com/resource-files/W83773G_SG_DatasheetV1_2.pdf
+
+Authors:
+	Lei YU <mine260309@gmail.com>
+
+Description
+-----------
+
+This driver implements support for Nuvoton W83773G temperature sensor
+chip. This chip implements one local and two remote sensors.
+The chip also features offsets for the two remote sensors which get added to
+the input readings. The chip does all the scaling by itself and the driver
+therefore reports true temperatures that don't need any user-space adjustments.
+Temperature is measured in degrees Celsius.
+The chip is wired over I2C/SMBus and specified over a temperature
+range of -40 to +125 degrees Celsius (for local sensor) and -40 to +127
+degrees Celsius (for remote sensors).
+Resolution for both the local and remote channels is 0.125 degree C.
+
+The chip supports only temperature measurement. The driver exports
+the temperature values via the following sysfs files:
+
+temp[1-3]_input
+temp[2-3]_fault
+temp[2-3]_offset
+update_interval
diff --git a/Documentation/i2c/DMA-considerations b/Documentation/i2c/DMA-considerations
new file mode 100644
index 000000000000..966610aa4620
--- /dev/null
+++ b/Documentation/i2c/DMA-considerations
@@ -0,0 +1,67 @@
+=================
+Linux I2C and DMA
+=================
+
+Given that i2c is a low-speed bus, over which the majority of messages
+transferred are small, it is not considered a prime user of DMA access. At this
+time of writing, only 10% of I2C bus master drivers have DMA support
+implemented. And the vast majority of transactions are so small that setting up
+DMA for it will likely add more overhead than a plain PIO transfer.
+
+Therefore, it is *not* mandatory that the buffer of an I2C message is DMA safe.
+It does not seem reasonable to apply additional burdens when the feature is so
+rarely used. However, it is recommended to use a DMA-safe buffer if your
+message size is likely applicable for DMA. Most drivers have this threshold
+around 8 bytes (as of today, this is mostly an educated guess, however). For
+any message of 16 byte or larger, it is probably a really good idea. Please
+note that other subsystems you use might add requirements. E.g., if your
+I2C bus master driver is using USB as a bridge, then you need to have DMA
+safe buffers always, because USB requires it.
+
+Clients
+-------
+
+For clients, if you use a DMA safe buffer in i2c_msg, set the I2C_M_DMA_SAFE
+flag with it. Then, the I2C core and drivers know they can safely operate DMA
+on it. Note that using this flag is optional. I2C host drivers which are not
+updated to use this flag will work like before. And like before, they risk
+using an unsafe DMA buffer. To improve this situation, using I2C_M_DMA_SAFE in
+more and more clients and host drivers is the planned way forward. Note also
+that setting this flag makes only sense in kernel space. User space data is
+copied into kernel space anyhow. The I2C core makes sure the destination
+buffers in kernel space are always DMA capable. Also, when the core emulates
+SMBus transactions via I2C, the buffers for block transfers are DMA safe. Users
+of i2c_master_send() and i2c_master_recv() functions can now use DMA safe
+variants (i2c_master_send_dmasafe() and i2c_master_recv_dmasafe()) once they
+know their buffers are DMA safe. Users of i2c_transfer() must set the
+I2C_M_DMA_SAFE flag manually.
+
+Masters
+-------
+
+Bus master drivers wishing to implement safe DMA can use helper functions from
+the I2C core. One gives you a DMA-safe buffer for a given i2c_msg as long as a
+certain threshold is met::
+
+	dma_buf = i2c_get_dma_safe_msg_buf(msg, threshold_in_byte);
+
+If a buffer is returned, it is either msg->buf for the I2C_M_DMA_SAFE case or a
+bounce buffer. But you don't need to care about that detail, just use the
+returned buffer. If NULL is returned, the threshold was not met or a bounce
+buffer could not be allocated. Fall back to PIO in that case.
+
+In any case, a buffer obtained from above needs to be released. It ensures data
+is copied back to the message and a potentially used bounce buffer is freed::
+
+	i2c_release_dma_safe_msg_buf(msg, dma_buf);
+
+The bounce buffer handling from the core is generic and simple. It will always
+allocate a new bounce buffer. If you want a more sophisticated handling (e.g.
+reusing pre-allocated buffers), you are free to implement your own.
+
+Please also check the in-kernel documentation for details. The i2c-sh_mobile
+driver can be used as a reference example how to use the above helpers.
+
+Final note: If you plan to use DMA with I2C (or with anything else, actually)
+make sure you have CONFIG_DMA_API_DEBUG enabled during development. It can help
+you find various issues which can be complex to debug otherwise.
diff --git a/Documentation/i2c/busses/i2c-i801 b/Documentation/i2c/busses/i2c-i801
index d47702456926..65514c251318 100644
--- a/Documentation/i2c/busses/i2c-i801
+++ b/Documentation/i2c/busses/i2c-i801
@@ -28,8 +28,10 @@ Supported adapters:
   * Intel Wildcat Point (PCH)
   * Intel Wildcat Point-LP (PCH)
   * Intel BayTrail (SOC)
+  * Intel Braswell (SOC)
   * Intel Sunrise Point-H (PCH)
   * Intel Sunrise Point-LP (PCH)
+  * Intel Kaby Lake-H (PCH)
   * Intel DNV (SOC)
   * Intel Broxton (SOC)
   * Intel Lewisburg (PCH)
diff --git a/Documentation/i2c/dev-interface b/Documentation/i2c/dev-interface
index 5ff19447ac44..d04e6e4964ee 100644
--- a/Documentation/i2c/dev-interface
+++ b/Documentation/i2c/dev-interface
@@ -17,13 +17,16 @@ i2c-10, ...). All 256 minor device numbers are reserved for i2c.
 C example
 =========
 
-So let's say you want to access an i2c adapter from a C program. The
-first thing to do is "#include <linux/i2c-dev.h>". Please note that
-there are two files named "i2c-dev.h" out there, one is distributed
-with the Linux kernel and is meant to be included from kernel
-driver code, the other one is distributed with i2c-tools and is
-meant to be included from user-space programs. You obviously want
-the second one here.
+So let's say you want to access an i2c adapter from a C program.
+First, you need to include these two headers:
+
+  #include <linux/i2c-dev.h>
+  #include <i2c/smbus.h>
+
+(Please note that there are two files named "i2c-dev.h" out there. One is
+distributed with the Linux kernel and the other one is included in the
+source tree of i2c-tools. They used to be different in content but since 2012
+they're identical. You should use "linux/i2c-dev.h").
 
 Now, you have to decide which adapter you want to access. You should
 inspect /sys/class/i2c-dev/ or run "i2cdetect -l" to decide this.
diff --git a/Documentation/i2c/gpio-fault-injection b/Documentation/i2c/gpio-fault-injection
new file mode 100644
index 000000000000..e0c4f775e239
--- /dev/null
+++ b/Documentation/i2c/gpio-fault-injection
@@ -0,0 +1,54 @@
+Linux I2C fault injection
+=========================
+
+The GPIO based I2C bus master driver can be configured to provide fault
+injection capabilities. It is then meant to be connected to another I2C bus
+which is driven by the I2C bus master driver under test. The GPIO fault
+injection driver can create special states on the bus which the other I2C bus
+master driver should handle gracefully.
+
+Once the Kconfig option I2C_GPIO_FAULT_INJECTOR is enabled, there will be an
+'i2c-fault-injector' subdirectory in the Kernel debugfs filesystem, usually
+mounted at /sys/kernel/debug. There will be a separate subdirectory per GPIO
+driven I2C bus. Each subdirectory will contain files to trigger the fault
+injection. They will be described now along with their intended use-cases.
+
+"scl"
+-----
+
+By reading this file, you get the current state of SCL. By writing, you can
+change its state to either force it low or to release it again. So, by using
+"echo 0 > scl" you force SCL low and thus, no communication will be possible
+because the bus master under test will not be able to clock. It should detect
+the condition of SCL being unresponsive and report an error to the upper
+layers.
+
+"sda"
+-----
+
+By reading this file, you get the current state of SDA. By writing, you can
+change its state to either force it low or to release it again. So, by using
+"echo 0 > sda" you force SDA low and thus, data cannot be transmitted. The bus
+master under test should detect this condition and trigger a bus recovery (see
+I2C specification version 4, section 3.1.16) using the helpers of the Linux I2C
+core (see 'struct bus_recovery_info'). However, the bus recovery will not
+succeed because SDA is still pinned low until you manually release it again
+with "echo 1 > sda". A test with an automatic release can be done with the
+'incomplete_transfer' file.
+
+"incomplete_transfer"
+---------------------
+
+This file is write only and you need to write the address of an existing I2C
+client device to it. Then, a transfer to this device will be started, but it
+will stop at the ACK phase after the address of the client has been
+transmitted. Because the device will ACK its presence, this results in SDA
+being pulled low by the device while SCL is high. So, similar to the "sda" file
+above, the bus master under test should detect this condition and try a bus
+recovery. This time, however, it should succeed and the device should release
+SDA after toggling SCL. Please note: there are I2C client devices which detect
+a stuck SDA on their side and release it on their own after a few milliseconds.
+Also, there are external devices deglitching and monitoring the I2C bus. They
+can also detect a stuck SDA and will init a bus recovery on their own. If you
+want to implement bus recovery in a bus master driver, make sure you checked
+your hardware setup carefully before.
diff --git a/Documentation/ia64/serial.txt b/Documentation/ia64/serial.txt
index 6869c73de4e2..a63d2c54329b 100644
--- a/Documentation/ia64/serial.txt
+++ b/Documentation/ia64/serial.txt
@@ -111,7 +111,7 @@ TROUBLESHOOTING SERIAL CONSOLE PROBLEMS
 
 	- If you don't have an HCDP, the kernel doesn't know where
 	  your console lives until the driver discovers serial
-	  devices.  Use "console=uart, io,0x3f8" (or appropriate
+	  devices.  Use "console=uart,io,0x3f8" (or appropriate
 	  address for your machine).
 
     Kernel and init script output works fine, but no "login:" prompt:
diff --git a/Documentation/index.rst b/Documentation/index.rst
index cb7f1ba5b3b1..ef5080cbf009 100644
--- a/Documentation/index.rst
+++ b/Documentation/index.rst
@@ -13,6 +13,18 @@ documents into a coherent whole.  Please note that improvements to the
 documentation are welcome; join the linux-doc list at vger.kernel.org if
 you want to help out.
 
+Licensing documentation
+-----------------------
+
+The following describes the license of the Linux kernel source code
+(GPLv2), how to properly mark the license of individual files in the source
+tree, as well as links to the full license text.
+
+.. toctree::
+   :maxdepth: 2
+
+   process/license-rules.rst
+
 User-oriented documentation
 ---------------------------
 
@@ -52,6 +64,7 @@ merged much easier.
    dev-tools/index
    doc-guide/index
    kernel-hacking/index
+   maintainer/index
 
 Kernel API documentation
 ------------------------
diff --git a/Documentation/infiniband/user_verbs.txt b/Documentation/infiniband/user_verbs.txt
index e5092d696da2..df049b9f5b6e 100644
--- a/Documentation/infiniband/user_verbs.txt
+++ b/Documentation/infiniband/user_verbs.txt
@@ -5,7 +5,7 @@ USERSPACE VERBS ACCESS
   described in chapter 11 of the InfiniBand Architecture Specification.
 
   To use the verbs, the libibverbs library, available from
-  http://www.openfabrics.org/, is required.  libibverbs contains a
+  https://github.com/linux-rdma/rdma-core, is required. libibverbs contains a
   device-independent API for using the ib_uverbs interface.
   libibverbs also requires appropriate device-dependent kernel and
   userspace driver for your InfiniBand hardware.  For example, to use
diff --git a/Documentation/input/devices/gpio-tilt.rst b/Documentation/input/devices/gpio-tilt.rst
deleted file mode 100644
index fa6e64570aa7..000000000000
--- a/Documentation/input/devices/gpio-tilt.rst
+++ /dev/null
@@ -1,103 +0,0 @@
-Driver for tilt-switches connected via GPIOs
-============================================
-
-Generic driver to read data from tilt switches connected via gpios.
-Orientation can be provided by one or more than one tilt switches,
-i.e. each tilt switch providing one axis, and the number of axes
-is also not limited.
-
-
-Data structures
----------------
-
-The array of struct gpio in the gpios field is used to list the gpios
-that represent the current tilt state.
-
-The array of struct gpio_tilt_axis describes the axes that are reported
-to the input system. The values set therein are used for the
-input_set_abs_params calls needed to init the axes.
-
-The array of struct gpio_tilt_state maps gpio states to the corresponding
-values to report. The gpio state is represented as a bitfield where the
-bit-index corresponds to the index of the gpio in the struct gpio array.
-In the same manner the values stored in the axes array correspond to
-the elements of the gpio_tilt_axis-array.
-
-
-Example
--------
-
-Example configuration for a single TS1003 tilt switch that rotates around
-one axis in 4 steps and emits the current tilt via two GPIOs::
-
-    static int sg060_tilt_enable(struct device *dev) {
-	    /* code to enable the sensors */
-    };
-
-    static void sg060_tilt_disable(struct device *dev) {
-	    /* code to disable the sensors */
-    };
-
-    static struct gpio sg060_tilt_gpios[] = {
-	    { SG060_TILT_GPIO_SENSOR1, GPIOF_IN, "tilt_sensor1" },
-	    { SG060_TILT_GPIO_SENSOR2, GPIOF_IN, "tilt_sensor2" },
-    };
-
-    static struct gpio_tilt_state sg060_tilt_states[] = {
-	    {
-		    .gpios = (0 << 1) | (0 << 0),
-		    .axes = (int[]) {
-			    0,
-		    },
-	    }, {
-		    .gpios = (0 << 1) | (1 << 0),
-		    .axes = (int[]) {
-			    1, /* 90 degrees */
-		    },
-	    }, {
-		    .gpios = (1 << 1) | (1 << 0),
-		    .axes = (int[]) {
-			    2, /* 180 degrees */
-		    },
-	    }, {
-		    .gpios = (1 << 1) | (0 << 0),
-		    .axes = (int[]) {
-			    3, /* 270 degrees */
-		    },
-	    },
-    };
-
-    static struct gpio_tilt_axis sg060_tilt_axes[] = {
-	    {
-		    .axis = ABS_RY,
-		    .min = 0,
-		    .max = 3,
-		    .fuzz = 0,
-		    .flat = 0,
-	    },
-    };
-
-    static struct gpio_tilt_platform_data sg060_tilt_pdata= {
-	    .gpios = sg060_tilt_gpios,
-	    .nr_gpios = ARRAY_SIZE(sg060_tilt_gpios),
-
-	    .axes = sg060_tilt_axes,
-	    .nr_axes = ARRAY_SIZE(sg060_tilt_axes),
-
-	    .states = sg060_tilt_states,
-	    .nr_states = ARRAY_SIZE(sg060_tilt_states),
-
-	    .debounce_interval = 100,
-
-	    .poll_interval = 1000,
-	    .enable = sg060_tilt_enable,
-	    .disable = sg060_tilt_disable,
-    };
-
-    static struct platform_device sg060_device_tilt = {
-	    .name = "gpio-tilt-polled",
-	    .id = -1,
-	    .dev = {
-		    .platform_data = &sg060_tilt_pdata,
-	    },
-    };
diff --git a/Documentation/input/devices/rotary-encoder.rst b/Documentation/input/devices/rotary-encoder.rst
index b07b20a295ac..810ae02bdaa0 100644
--- a/Documentation/input/devices/rotary-encoder.rst
+++ b/Documentation/input/devices/rotary-encoder.rst
@@ -108,9 +108,9 @@ example below:
 	};
 
 	static const struct property_entry rotary_encoder_properties[] __initconst = {
-		PROPERTY_ENTRY_INTEGER("rotary-encoder,steps-per-period", u32, 24),
-		PROPERTY_ENTRY_INTEGER("linux,axis",			  u32, ABS_X),
-		PROPERTY_ENTRY_INTEGER("rotary-encoder,relative_axis",	  u32, 0),
+		PROPERTY_ENTRY_U32("rotary-encoder,steps-per-period", 24),
+		PROPERTY_ENTRY_U32("linux,axis",		      ABS_X),
+		PROPERTY_ENTRY_U32("rotary-encoder,relative_axis",    0),
 		{ },
 	};
 
diff --git a/Documentation/input/ff.rst b/Documentation/input/ff.rst
index 26d461998e08..0c02e87ee86d 100644
--- a/Documentation/input/ff.rst
+++ b/Documentation/input/ff.rst
@@ -31,7 +31,7 @@ To enable force feedback, you have to:
 
 Before you start, let me WARN you that some devices shake violently during the
 initialisation phase. This happens for example with my "AVB Top Shot Pegasus".
-To stop this annoying behaviour, move you joystick to its limits. Anyway, you
+To stop this annoying behaviour, move your joystick to its limits. Anyway, you
 should keep a hand on your device, in order to avoid it to break down if
 something goes wrong.
 
@@ -121,7 +121,7 @@ uploaded, but not played.
 The content of effect may be modified. In particular, its field "id" is set
 to the unique id assigned by the driver. This data is required for performing
 some operations (removing an effect, controlling the playback).
-This if field must be set to -1 by the user in order to tell the driver to
+The "id" field must be set to -1 by the user in order to tell the driver to
 allocate a new effect.
 
 Effects are file descriptor specific.
@@ -178,7 +178,7 @@ Control of playing is done with write(). Below is an example:
 	stop.code = effect.id;
 	stop.value = 0;
 
-	write(fd, (const void*) &play, sizeof(stop));
+	write(fd, (const void*) &stop, sizeof(stop));
 
 Setting the gain
 ----------------
diff --git a/Documentation/input/multi-touch-protocol.rst b/Documentation/input/multi-touch-protocol.rst
index 8035868c56bc..b51751a0cd5d 100644
--- a/Documentation/input/multi-touch-protocol.rst
+++ b/Documentation/input/multi-touch-protocol.rst
@@ -269,10 +269,11 @@ ABS_MT_ORIENTATION
     The orientation of the touching ellipse. The value should describe a signed
     quarter of a revolution clockwise around the touch center. The signed value
     range is arbitrary, but zero should be returned for an ellipse aligned with
-    the Y axis of the surface, a negative value when the ellipse is turned to
-    the left, and a positive value when the ellipse is turned to the
-    right. When completely aligned with the X axis, the range max should be
-    returned.
+    the Y axis (north) of the surface, a negative value when the ellipse is
+    turned to the left, and a positive value when the ellipse is turned to the
+    right. When aligned with the X axis in the positive direction, the range
+    max should be returned; when aligned with the X axis in the negative
+    direction, the range -max should be returned.
 
     Touch ellipsis are symmetrical by default. For devices capable of true 360
     degree orientation, the reported orientation must exceed the range max to
diff --git a/Documentation/ioctl/ioctl-number.txt b/Documentation/ioctl/ioctl-number.txt
index 3e3fdae5f3ed..6501389d55b9 100644
--- a/Documentation/ioctl/ioctl-number.txt
+++ b/Documentation/ioctl/ioctl-number.txt
@@ -326,6 +326,7 @@ Code  Seq#(hex)	Include File		Comments
 0xB5	00-0F	uapi/linux/rpmsg.h	<mailto:linux-remoteproc@vger.kernel.org>
 0xC0	00-0F	linux/usb/iowarrior.h
 0xCA	00-0F	uapi/misc/cxl.h
+0xCA	10-2F	uapi/misc/ocxl.h
 0xCA	80-BF	uapi/scsi/cxlflash_ioctl.h
 0xCB	00-1F	CBM serial IEC bus	in development:
 					<mailto:michael.klein@puffin.lb.shuttle.de>
diff --git a/Documentation/isdn/INTERFACE.CAPI b/Documentation/isdn/INTERFACE.CAPI
index 1688b5a1fd77..021aa9cf139d 100644
--- a/Documentation/isdn/INTERFACE.CAPI
+++ b/Documentation/isdn/INTERFACE.CAPI
@@ -18,7 +18,7 @@ corresponding hardware driver. Kernel CAPI then forwards CAPI messages in both
 directions between the application and the hardware driver.
 
 Format and semantics of CAPI messages are specified in the CAPI 2.0 standard.
-This standard is freely available from http://www.capi.org.
+This standard is freely available from https://www.capi.org.
 
 
 2. Driver and Device Registration
diff --git a/Documentation/isdn/README b/Documentation/isdn/README
index 32d4e80c2c03..74bd2bdb455b 100644
--- a/Documentation/isdn/README
+++ b/Documentation/isdn/README
@@ -33,10 +33,10 @@ README for the ISDN-subsystem
      de.alt.comm.isdn4linux
 
   There is also a well maintained FAQ in English available at
-     http://www.mhessler.de/i4lfaq/
+     https://www.mhessler.de/i4lfaq/
   It can be viewed online, or downloaded in sgml/text/html format.
   The FAQ can also be viewed online at
-     http://www.isdn4linux.de/faq/
+     https://www.isdn4linux.de/faq/i4lfaq.html
   or downloaded from
      ftp://ftp.isdn4linux.de/pub/isdn4linux/FAQ/
 
diff --git a/Documentation/isdn/README.FAQ b/Documentation/isdn/README.FAQ
index 356f7944641d..e5dd1addacdd 100644
--- a/Documentation/isdn/README.FAQ
+++ b/Documentation/isdn/README.FAQ
@@ -8,9 +8,9 @@ You find it in:
 
 In case you just want to see the FAQ online, or download the newest version,
 you can have a look at my website:
-http://www.mhessler.de/i4lfaq/ (view + download)
+https://www.mhessler.de/i4lfaq/ (view + download)
 or:
-http://www.isdn4linux.de/faq/ (view)
+https://www.isdn4linux.de/faq/4lfaq.html (view)
 
 As the extension tells, the FAQ is in SGML format, and you can convert it
 into text/html/... format by using the sgml2txt/sgml2html/... tools.
diff --git a/Documentation/isdn/README.gigaset b/Documentation/isdn/README.gigaset
index 7534c6039adc..9b1ce277ca3d 100644
--- a/Documentation/isdn/README.gigaset
+++ b/Documentation/isdn/README.gigaset
@@ -29,8 +29,9 @@ GigaSet 307x Device Driver
         T-Com Sinus 721 data
         Chicago 390 USB (KPN)
 
-     See also http://www.erbze.info/sinus_gigaset.htm and
-              http://gigaset307x.sourceforge.net/
+     See also http://www.erbze.info/sinus_gigaset.htm
+       (archived at https://web.archive.org/web/20100717020421/http://www.erbze.info:80/sinus_gigaset.htm ) and
+	http://gigaset307x.sourceforge.net/
 
      We had also reports from users of Gigaset M105 who could use the drivers
      with SX 100 and CX 100 ISDN bases (only in unimodem mode, see section 2.5.)
@@ -52,7 +53,7 @@ GigaSet 307x Device Driver
      to use CAPI 2.0 or ISDN4Linux for ISDN connections (voice or data).
 
      There are some user space tools available at
-     http://sourceforge.net/projects/gigaset307x/
+     https://sourceforge.net/projects/gigaset307x/
      which provide access to additional device specific functions like SMS,
      phonebook or call journal.
 
@@ -202,7 +203,7 @@ GigaSet 307x Device Driver
      You can use some configuration tool of your distribution to configure this
      "modem" or configure pppd/wvdial manually. There are some example ppp
      configuration files and chat scripts in the gigaset-VERSION/ppp directory
-     in the driver packages from http://sourceforge.net/projects/gigaset307x/.
+     in the driver packages from https://sourceforge.net/projects/gigaset307x/.
      Please note that the USB drivers are not able to change the state of the
      control lines. This means you must use "Stupid Mode" if you are using
      wvdial or you should use the nocrtscts option of pppd.
@@ -361,7 +362,7 @@ GigaSet 307x Device Driver
      ---------------------------
      If you can't solve problems with the driver on your own, feel free to
      use one of the forums, bug trackers, or mailing lists on
-         http://sourceforge.net/projects/gigaset307x
+         https://sourceforge.net/projects/gigaset307x
      or write an electronic mail to the maintainers.
 
      Try to provide as much information as possible, such as
@@ -391,11 +392,12 @@ GigaSet 307x Device Driver
 4.   Links, other software
      ---------------------
      - Sourceforge project developing this driver and associated tools
-         http://sourceforge.net/projects/gigaset307x
+         https://sourceforge.net/projects/gigaset307x
      - Yahoo! Group on the Siemens Gigaset family of devices
-         http://de.groups.yahoo.com/group/Siemens-Gigaset
+         https://de.groups.yahoo.com/group/Siemens-Gigaset
      - Siemens Gigaset/T-Sinus compatibility table
          http://www.erbze.info/sinus_gigaset.htm
+	    (archived at https://web.archive.org/web/20100717020421/http://www.erbze.info:80/sinus_gigaset.htm )
 
 
 5.   Credits
diff --git a/Documentation/kbuild/kconfig-language.txt b/Documentation/kbuild/kconfig-language.txt
index c4a293a03c33..f5b9493f04ad 100644
--- a/Documentation/kbuild/kconfig-language.txt
+++ b/Documentation/kbuild/kconfig-language.txt
@@ -77,6 +77,27 @@ applicable everywhere (see syntax).
   Optionally, dependencies only for this default value can be added with
   "if".
 
+ The default value deliberately defaults to 'n' in order to avoid bloating the
+ build. With few exceptions, new config options should not change this. The
+ intent is for "make oldconfig" to add as little as possible to the config from
+ release to release.
+
+ Note:
+	Things that merit "default y/m" include:
+
+	a) A new Kconfig option for something that used to always be built
+	   should be "default y".
+
+	b) A new gatekeeping Kconfig option that hides/shows other Kconfig
+	   options (but does not generate any code of its own), should be
+	   "default y" so people will see those other options.
+
+	c) Sub-driver behavior or similar options for a driver that is
+	   "default n". This allows you to provide sane defaults.
+
+	d) Hardware or infrastructure that everybody expects, such as CONFIG_NET
+	   or CONFIG_BLOCK. These are rare exceptions.
+
 - type definition + default value:
 	"def_bool"/"def_tristate" <expr> ["if" <expr>]
   This is a shorthand notation for a type definition plus a value.
diff --git a/Documentation/kernel-doc-nano-HOWTO.txt b/Documentation/kernel-doc-nano-HOWTO.txt
deleted file mode 100644
index c23e2c5ab80d..000000000000
--- a/Documentation/kernel-doc-nano-HOWTO.txt
+++ /dev/null
@@ -1,322 +0,0 @@
-NOTE: this document is outdated and will eventually be removed.  See
-Documentation/doc-guide/ for current information.
-
-kernel-doc nano-HOWTO
-=====================
-
-How to format kernel-doc comments
----------------------------------
-
-In order to provide embedded, 'C' friendly, easy to maintain,
-but consistent and extractable documentation of the functions and
-data structures in the Linux kernel, the Linux kernel has adopted
-a consistent style for documenting functions and their parameters,
-and structures and their members.
-
-The format for this documentation is called the kernel-doc format.
-It is documented in this Documentation/kernel-doc-nano-HOWTO.txt file.
-
-This style embeds the documentation within the source files, using
-a few simple conventions.  The scripts/kernel-doc perl script, the
-Documentation/sphinx/kerneldoc.py Sphinx extension and other tools understand
-these conventions, and are used to extract this embedded documentation
-into various documents.
-
-In order to provide good documentation of kernel functions and data
-structures, please use the following conventions to format your
-kernel-doc comments in Linux kernel source.
-
-We definitely need kernel-doc formatted documentation for functions
-that are exported to loadable modules using EXPORT_SYMBOL.
-
-We also look to provide kernel-doc formatted documentation for
-functions externally visible to other kernel files (not marked
-"static").
-
-We also recommend providing kernel-doc formatted documentation
-for private (file "static") routines, for consistency of kernel
-source code layout.  But this is lower priority and at the
-discretion of the MAINTAINER of that kernel source file.
-
-Data structures visible in kernel include files should also be
-documented using kernel-doc formatted comments.
-
-The opening comment mark "/**" is reserved for kernel-doc comments.
-Only comments so marked will be considered by the kernel-doc scripts,
-and any comment so marked must be in kernel-doc format.  Do not use
-"/**" to be begin a comment block unless the comment block contains
-kernel-doc formatted comments.  The closing comment marker for
-kernel-doc comments can be either "*/" or "**/", but "*/" is
-preferred in the Linux kernel tree.
-
-Kernel-doc comments should be placed just before the function
-or data structure being described.
-
-Example kernel-doc function comment:
-
-/**
- * foobar() - short function description of foobar
- * @arg1:	Describe the first argument to foobar.
- * @arg2:	Describe the second argument to foobar.
- *		One can provide multiple line descriptions
- *		for arguments.
- *
- * A longer description, with more discussion of the function foobar()
- * that might be useful to those using or modifying it.  Begins with
- * empty comment line, and may include additional embedded empty
- * comment lines.
- *
- * The longer description can have multiple paragraphs.
- *
- * Return: Describe the return value of foobar.
- */
-
-The short description following the subject can span multiple lines
-and ends with an @argument description, an empty line or the end of
-the comment block.
-
-The @argument descriptions must begin on the very next line following
-this opening short function description line, with no intervening
-empty comment lines.
-
-If a function parameter is "..." (varargs), it should be listed in
-kernel-doc notation as:
- * @...: description
-
-The return value, if any, should be described in a dedicated section
-named "Return".
-
-Example kernel-doc data structure comment.
-
-/**
- * struct blah - the basic blah structure
- * @mem1:	describe the first member of struct blah
- * @mem2:	describe the second member of struct blah,
- *		perhaps with more lines and words.
- *
- * Longer description of this structure.
- */
-
-The kernel-doc function comments describe each parameter to the
-function, in order, with the @name lines.
-
-The kernel-doc data structure comments describe each structure member
-in the data structure, with the @name lines.
-
-The longer description formatting is "reflowed", losing your line
-breaks.  So presenting carefully formatted lists within these
-descriptions won't work so well; derived documentation will lose
-the formatting.
-
-See the section below "How to add extractable documentation to your
-source files" for more details and notes on how to format kernel-doc
-comments.
-
-Components of the kernel-doc system
------------------------------------
-
-Many places in the source tree have extractable documentation in the
-form of block comments above functions.  The components of this system
-are:
-
-- scripts/kernel-doc
-
-  This is a perl script that hunts for the block comments and can mark
-  them up directly into DocBook, ReST, man, text, and HTML. (No, not
-  texinfo.)
-
-- scripts/docproc.c
-
-  This is a program for converting SGML template files into SGML
-  files. When a file is referenced it is searched for symbols
-  exported (EXPORT_SYMBOL), to be able to distinguish between internal
-  and external functions.
-  It invokes kernel-doc, giving it the list of functions that
-  are to be documented.
-  Additionally it is used to scan the SGML template files to locate
-  all the files referenced herein. This is used to generate dependency
-  information as used by make.
-
-- Makefile
-
-  The targets 'xmldocs', 'latexdocs', 'pdfdocs', 'epubdocs'and 'htmldocs'
-  are used to build XML DocBook files, LaTeX files, PDF files,
-  ePub files and html files in Documentation/.
-
-How to extract the documentation
---------------------------------
-
-If you just want to read the ready-made books on the various
-subsystems, just type 'make epubdocs', or 'make pdfdocs', or 'make htmldocs',
-depending on your preference.  If you would rather read a different format,
-you can type 'make xmldocs' and then use DocBook tools to convert
-Documentation/output/*.xml to a format of your choice (for example,
-'db2html ...' if 'make htmldocs' was not defined).
-
-If you want to see man pages instead, you can do this:
-
-$ cd linux
-$ scripts/kernel-doc -man $(find -name '*.c') | split-man.pl /tmp/man
-$ scripts/kernel-doc -man $(find -name '*.h') | split-man.pl /tmp/man
-
-Here is split-man.pl:
-
--->
-#!/usr/bin/perl
-
-if ($#ARGV < 0) {
-   die "where do I put the results?\n";
-}
-
-mkdir $ARGV[0],0777;
-$state = 0;
-while (<STDIN>) {
-    if (/^\.TH \"[^\"]*\" 9 \"([^\"]*)\"/) {
-	if ($state == 1) { close OUT }
-	$state = 1;
-	$fn = "$ARGV[0]/$1.9";
-	print STDERR "Creating $fn\n";
-	open OUT, ">$fn" or die "can't open $fn: $!\n";
-	print OUT $_;
-    } elsif ($state != 0) {
-	print OUT $_;
-    }
-}
-
-close OUT;
-<--
-
-If you just want to view the documentation for one function in one
-file, you can do this:
-
-$ scripts/kernel-doc -man -function fn file | nroff -man | less
-
-or this:
-
-$ scripts/kernel-doc -text -function fn file
-
-
-How to add extractable documentation to your source files
----------------------------------------------------------
-
-The format of the block comment is like this:
-
-/**
- * function_name(:)? (- short description)?
-(* @parameterx(space)*: (description of parameter x)?)*
-(* a blank line)?
- * (Description:)? (Description of function)?
- * (section header: (section description)? )*
-(*)?*/
-
-All "description" text can span multiple lines, although the
-function_name & its short description are traditionally on a single line.
-Description text may also contain blank lines (i.e., lines that contain
-only a "*").
-
-"section header:" names must be unique per function (or struct,
-union, typedef, enum).
-
-Use the section header "Return" for sections describing the return value
-of a function.
-
-Avoid putting a spurious blank line after the function name, or else the
-description will be repeated!
-
-All descriptive text is further processed, scanning for the following special
-patterns, which are highlighted appropriately.
-
-'funcname()' - function
-'$ENVVAR' - environment variable
-'&struct_name' - name of a structure (up to two words including 'struct')
-'@parameter' - name of a parameter
-'%CONST' - name of a constant.
-
-NOTE 1:  The multi-line descriptive text you provide does *not* recognize
-line breaks, so if you try to format some text nicely, as in:
-
-  Return:
-    0 - cool
-    1 - invalid arg
-    2 - out of memory
-
-this will all run together and produce:
-
-  Return: 0 - cool 1 - invalid arg 2 - out of memory
-
-NOTE 2:  If the descriptive text you provide has lines that begin with
-some phrase followed by a colon, each of those phrases will be taken as
-a new section heading, which means you should similarly try to avoid text
-like:
-
-  Return:
-    0: cool
-    1: invalid arg
-    2: out of memory
-
-every line of which would start a new section.  Again, probably not
-what you were after.
-
-Take a look around the source tree for examples.
-
-
-kernel-doc for structs, unions, enums, and typedefs
----------------------------------------------------
-
-Beside functions you can also write documentation for structs, unions,
-enums and typedefs. Instead of the function name you must write the name
-of the declaration;  the struct/union/enum/typedef must always precede
-the name. Nesting of declarations is not supported.
-Use the argument mechanism to document members or constants.
-
-Inside a struct description, you can use the "private:" and "public:"
-comment tags.  Structure fields that are inside a "private:" area
-are not listed in the generated output documentation.  The "private:"
-and "public:" tags must begin immediately following a "/*" comment
-marker.  They may optionally include comments between the ":" and the
-ending "*/" marker.
-
-Example:
-
-/**
- * struct my_struct - short description
- * @a: first member
- * @b: second member
- *
- * Longer description
- */
-struct my_struct {
-    int a;
-    int b;
-/* private: internal use only */
-    int c;
-};
-
-
-Including documentation blocks in source files
-----------------------------------------------
-
-To facilitate having source code and comments close together, you can
-include kernel-doc documentation blocks that are free-form comments
-instead of being kernel-doc for functions, structures, unions,
-enums, or typedefs.  This could be used for something like a
-theory of operation for a driver or library code, for example.
-
-This is done by using a DOC: section keyword with a section title.  E.g.:
-
-/**
- * DOC: Theory of Operation
- *
- * The whizbang foobar is a dilly of a gizmo.  It can do whatever you
- * want it to do, at any time.  It reads your mind.  Here's how it works.
- *
- * foo bar splat
- *
- * The only drawback to this gizmo is that is can sometimes damage
- * hardware, software, or its subject(s).
- */
-
-DOC: sections are used in ReST files.
-
-Tim.
-*/ <twaugh@redhat.com>
diff --git a/Documentation/kernel-hacking/hacking.rst b/Documentation/kernel-hacking/hacking.rst
index daf3883b2694..9999c8468293 100644
--- a/Documentation/kernel-hacking/hacking.rst
+++ b/Documentation/kernel-hacking/hacking.rst
@@ -523,7 +523,7 @@ this expression is true, or ``-ERESTARTSYS`` if a signal is received. The
 Waking Up Queued Tasks
 ----------------------
 
-Call :c:func:`wake_up()` (``include/linux/wait.h``);, which will wake
+Call :c:func:`wake_up()` (``include/linux/wait.h``), which will wake
 up every process in the queue. The exception is if one has
 ``TASK_EXCLUSIVE`` set, in which case the remainder of the queue will
 not be woken. There are other variants of this basic function available
@@ -690,8 +690,8 @@ not provide the necessary runtime environment and the include files are
 not tested for it. It is still possible, but not recommended. If you
 really want to do this, forget about exceptions at least.
 
-NUMif
------
+#if
+---
 
 It is generally considered cleaner to use macros in header files (or at
 the top of .c files) to abstract away functions rather than using \`#if'
diff --git a/Documentation/livepatch/livepatch.txt b/Documentation/livepatch/livepatch.txt
index ecdb18104ab0..1ae2de758c08 100644
--- a/Documentation/livepatch/livepatch.txt
+++ b/Documentation/livepatch/livepatch.txt
@@ -72,8 +72,7 @@ example, they add a NULL pointer or a boundary check, fix a race by adding
 a missing memory barrier, or add some locking around a critical section.
 Most of these changes are self contained and the function presents itself
 the same way to the rest of the system. In this case, the functions might
-be updated independently one by one.  (This can be done by setting the
-'immediate' flag in the klp_patch struct.)
+be updated independently one by one.
 
 But there are more complex fixes. For example, a patch might change
 ordering of locking in multiple functions at the same time. Or a patch
@@ -125,12 +124,6 @@ safe to patch tasks:
    b) Patching CPU-bound user tasks.  If the task is highly CPU-bound
       then it will get patched the next time it gets interrupted by an
       IRQ.
-   c) In the future it could be useful for applying patches for
-      architectures which don't yet have HAVE_RELIABLE_STACKTRACE.  In
-      this case you would have to signal most of the tasks on the
-      system.  However this isn't supported yet because there's
-      currently no way to patch kthreads without
-      HAVE_RELIABLE_STACKTRACE.
 
 3. For idle "swapper" tasks, since they don't ever exit the kernel, they
    instead have a klp_update_patch_state() call in the idle loop which
@@ -138,27 +131,16 @@ safe to patch tasks:
 
    (Note there's not yet such an approach for kthreads.)
 
-All the above approaches may be skipped by setting the 'immediate' flag
-in the 'klp_patch' struct, which will disable per-task consistency and
-patch all tasks immediately.  This can be useful if the patch doesn't
-change any function or data semantics.  Note that, even with this flag
-set, it's possible that some tasks may still be running with an old
-version of the function, until that function returns.
+Architectures which don't have HAVE_RELIABLE_STACKTRACE solely rely on
+the second approach. It's highly likely that some tasks may still be
+running with an old version of the function, until that function
+returns. In this case you would have to signal the tasks. This
+especially applies to kthreads. They may not be woken up and would need
+to be forced. See below for more information.
 
-There's also an 'immediate' flag in the 'klp_func' struct which allows
-you to specify that certain functions in the patch can be applied
-without per-task consistency.  This might be useful if you want to patch
-a common function like schedule(), and the function change doesn't need
-consistency but the rest of the patch does.
-
-For architectures which don't have HAVE_RELIABLE_STACKTRACE, the user
-must set patch->immediate which causes all tasks to be patched
-immediately.  This option should be used with care, only when the patch
-doesn't change any function or data semantics.
-
-In the future, architectures which don't have HAVE_RELIABLE_STACKTRACE
-may be allowed to use per-task consistency if we can come up with
-another way to patch kthreads.
+Unless we can come up with another way to patch kthreads, architectures
+without HAVE_RELIABLE_STACKTRACE are not considered fully supported by
+the kernel livepatching.
 
 The /sys/kernel/livepatch/<patch>/transition file shows whether a patch
 is in transition.  Only a single patch (the topmost patch on the stack)
@@ -176,8 +158,31 @@ If a patch is in transition, this file shows 0 to indicate the task is
 unpatched and 1 to indicate it's patched.  Otherwise, if no patch is in
 transition, it shows -1.  Any tasks which are blocking the transition
 can be signaled with SIGSTOP and SIGCONT to force them to change their
-patched state.
-
+patched state. This may be harmful to the system though.
+/sys/kernel/livepatch/<patch>/signal attribute provides a better alternative.
+Writing 1 to the attribute sends a fake signal to all remaining blocking
+tasks. No proper signal is actually delivered (there is no data in signal
+pending structures). Tasks are interrupted or woken up, and forced to change
+their patched state.
+
+Administrator can also affect a transition through
+/sys/kernel/livepatch/<patch>/force attribute. Writing 1 there clears
+TIF_PATCH_PENDING flag of all tasks and thus forces the tasks to the patched
+state. Important note! The force attribute is intended for cases when the
+transition gets stuck for a long time because of a blocking task. Administrator
+is expected to collect all necessary data (namely stack traces of such blocking
+tasks) and request a clearance from a patch distributor to force the transition.
+Unauthorized usage may cause harm to the system. It depends on the nature of the
+patch, which functions are (un)patched, and which functions the blocking tasks
+are sleeping in (/proc/<pid>/stack may help here). Removal (rmmod) of patch
+modules is permanently disabled when the force feature is used. It cannot be
+guaranteed there is no task sleeping in such module. It implies unbounded
+reference count if a patch module is disabled and enabled in a loop.
+
+Moreover, the usage of force may also affect future applications of live
+patches and cause even more harm to the system. Administrator should first
+consider to simply cancel a transition (see above). If force is used, reboot
+should be planned and no more live patches applied.
 
 3.1 Adding consistency model support to new architectures
 ---------------------------------------------------------
@@ -216,13 +221,6 @@ few options:
    a good backup option for those architectures which don't have
    reliable stack traces yet.
 
-In the meantime, patches for such architectures can bypass the
-consistency model by setting klp_patch.immediate to true.  This option
-is perfectly fine for patches which don't change the semantics of the
-patched functions.  In practice, this is usable for ~90% of security
-fixes.  Use of this option also means the patch can't be unloaded after
-it has been disabled.
-
 
 4. Livepatch module
 ===================
@@ -278,9 +276,6 @@ into three levels:
     only for a particular object ( vmlinux or a kernel module ). Note that
     kallsyms allows for searching symbols according to the object name.
 
-    There's also an 'immediate' flag which, when set, patches the
-    function immediately, bypassing the consistency model safety checks.
-
   + struct klp_object defines an array of patched functions (struct
     klp_func) in the same object. Where the object is either vmlinux
     (NULL) or a module name.
@@ -299,9 +294,6 @@ into three levels:
     symbols are found. The only exception are symbols from objects
     (kernel modules) that have not been loaded yet.
 
-    Setting the 'immediate' flag applies the patch to all tasks
-    immediately, bypassing the consistency model safety checks.
-
     For more details on how the patch is applied on a per-task basis,
     see the "Consistency model" section.
 
@@ -316,14 +308,12 @@ section "Livepatch life-cycle" below for more details about these
 two operations.
 
 Module removal is only safe when there are no users of the underlying
-functions. The immediate consistency model is not able to detect this. The
-code just redirects the functions at the very beginning and it does not
-check if the functions are in use. In other words, it knows when the
-functions get called but it does not know when the functions return.
-Therefore it cannot be decided when the livepatch module can be safely
-removed. This is solved by a hybrid consistency model. When the system is
-transitioned to a new patch state (patched/unpatched) it is guaranteed that
-no task sleeps or runs in the old code.
+functions. This is the reason why the force feature permanently disables
+the removal. The forced tasks entered the functions but we cannot say
+that they returned back.  Therefore it cannot be decided when the
+livepatch module can be safely removed. When the system is successfully
+transitioned to a new patch state (patched/unpatched) without being
+forced it is guaranteed that no task sleeps or runs in the old code.
 
 
 5. Livepatch life-cycle
@@ -337,19 +327,12 @@ First, the patch is applied only when all patched symbols for already
 loaded objects are found. The error handling is much easier if this
 check is done before particular functions get redirected.
 
-Second, the immediate consistency model does not guarantee that anyone is not
-sleeping in the new code after the patch is reverted. This means that the new
-code needs to stay around "forever". If the code is there, one could apply it
-again. Therefore it makes sense to separate the operations that might be done
-once and those that need to be repeated when the patch is enabled (applied)
-again.
-
-Third, it might take some time until the entire system is migrated
-when a more complex consistency model is used. The patch revert might
-block the livepatch module removal for too long. Therefore it is useful
-to revert the patch using a separate operation that might be called
-explicitly. But it does not make sense to remove all information
-until the livepatch module is really removed.
+Second, it might take some time until the entire system is migrated with
+the hybrid consistency model being used. The patch revert might block
+the livepatch module removal for too long. Therefore it is useful to
+revert the patch using a separate operation that might be called
+explicitly. But it does not make sense to remove all information until
+the livepatch module is really removed.
 
 
 5.1. Registration
@@ -435,6 +418,9 @@ Information about the registered patches can be found under
 /sys/kernel/livepatch. The patches could be enabled and disabled
 by writing there.
 
+/sys/kernel/livepatch/<patch>/signal and /sys/kernel/livepatch/<patch>/force
+attributes allow administrator to affect a patching operation.
+
 See Documentation/ABI/testing/sysfs-kernel-livepatch for more details.
 
 
diff --git a/Documentation/locking/locktorture.txt b/Documentation/locking/locktorture.txt
index a2ef3a929bf1..6a8df4cd19bf 100644
--- a/Documentation/locking/locktorture.txt
+++ b/Documentation/locking/locktorture.txt
@@ -57,11 +57,6 @@ torture_type	  Type of lock to torture. By default, only spinlocks will
 
 		     o "rwsem_lock": read/write down() and up() semaphore pairs.
 
-torture_runnable  Start locktorture at boot time in the case where the
-		  module is built into the kernel, otherwise wait for
-		  torture_runnable to be set via sysfs before starting.
-		  By default it will begin once the module is loaded.
-
 
 	    ** Torture-framework (RCU + locking) **
 
diff --git a/Documentation/locking/mutex-design.txt b/Documentation/locking/mutex-design.txt
index 60c482df1a38..818aca19612f 100644
--- a/Documentation/locking/mutex-design.txt
+++ b/Documentation/locking/mutex-design.txt
@@ -21,37 +21,23 @@ Implementation
 --------------
 
 Mutexes are represented by 'struct mutex', defined in include/linux/mutex.h
-and implemented in kernel/locking/mutex.c. These locks use a three
-state atomic counter (->count) to represent the different possible
-transitions that can occur during the lifetime of a lock:
-
-	  1: unlocked
-	  0: locked, no waiters
-   negative: locked, with potential waiters
-
-In its most basic form it also includes a wait-queue and a spinlock
-that serializes access to it. CONFIG_SMP systems can also include
-a pointer to the lock task owner (->owner) as well as a spinner MCS
-lock (->osq), both described below in (ii).
+and implemented in kernel/locking/mutex.c. These locks use an atomic variable
+(->owner) to keep track of the lock state during its lifetime.  Field owner
+actually contains 'struct task_struct *' to the current lock owner and it is
+therefore NULL if not currently owned. Since task_struct pointers are aligned
+at at least L1_CACHE_BYTES, low bits (3) are used to store extra state (e.g.,
+if waiter list is non-empty).  In its most basic form it also includes a
+wait-queue and a spinlock that serializes access to it. Furthermore,
+CONFIG_MUTEX_SPIN_ON_OWNER=y systems use a spinner MCS lock (->osq), described
+below in (ii).
 
 When acquiring a mutex, there are three possible paths that can be
 taken, depending on the state of the lock:
 
-(i) fastpath: tries to atomically acquire the lock by decrementing the
-    counter. If it was already taken by another task it goes to the next
-    possible path. This logic is architecture specific. On x86-64, the
-    locking fastpath is 2 instructions:
-
-    0000000000000e10 <mutex_lock>:
-    e21:   f0 ff 0b                lock decl (%rbx)
-    e24:   79 08                   jns    e2e <mutex_lock+0x1e>
-
-   the unlocking fastpath is equally tight:
-
-    0000000000000bc0 <mutex_unlock>:
-    bc8:   f0 ff 07                lock incl (%rdi)
-    bcb:   7f 0a                   jg     bd7 <mutex_unlock+0x17>
-
+(i) fastpath: tries to atomically acquire the lock by cmpxchg()ing the owner with
+    the current task. This only works in the uncontended case (cmpxchg() checks
+    against 0UL, so all 3 state bits above have to be 0). If the lock is
+    contended it goes to the next possible path.
 
 (ii) midpath: aka optimistic spinning, tries to spin for acquisition
      while the lock owner is running and there are no other tasks ready
@@ -143,11 +129,10 @@ Test if the mutex is taken:
 Disadvantages
 -------------
 
-Unlike its original design and purpose, 'struct mutex' is larger than
-most locks in the kernel. E.g: on x86-64 it is 40 bytes, almost twice
-as large as 'struct semaphore' (24 bytes) and tied, along with rwsems,
-for the largest lock in the kernel. Larger structure sizes mean more
-CPU cache and memory footprint.
+Unlike its original design and purpose, 'struct mutex' is among the largest
+locks in the kernel. E.g: on x86-64 it is 32 bytes, where 'struct semaphore'
+is 24 bytes and rw_semaphore is 40 bytes. Larger structure sizes mean more CPU
+cache and memory footprint.
 
 When to use mutexes
 -------------------
diff --git a/Documentation/maintainer/conf.py b/Documentation/maintainer/conf.py
new file mode 100644
index 000000000000..81e9eb7a7884
--- /dev/null
+++ b/Documentation/maintainer/conf.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8; mode: python -*-
+
+project = 'Linux Kernel Development Documentation'
+
+tags.add("subproject")
+
+latex_documents = [
+    ('index', 'maintainer.tex', 'Linux Kernel Development Documentation',
+     'The kernel development community', 'manual'),
+]
diff --git a/Documentation/maintainer/configure-git.rst b/Documentation/maintainer/configure-git.rst
new file mode 100644
index 000000000000..78bbbb0d2c84
--- /dev/null
+++ b/Documentation/maintainer/configure-git.rst
@@ -0,0 +1,34 @@
+.. _configuregit:
+
+Configure Git
+=============
+
+This chapter describes maintainer level git configuration.
+
+Tagged branches used in :ref:`Documentation/maintainer/pull-requests.rst
+<pullrequests>` should be signed with the developers public GPG key. Signed
+tags can be created by passing the ``-u`` flag to ``git tag``. However,
+since you would *usually* use the same key for the same project, you can
+set it once with
+::
+
+	git config user.signingkey "keyname"
+
+Alternatively, edit your ``.git/config`` or ``~/.gitconfig`` file by hand:
+::
+
+	[user]
+		name = Jane Developer
+		email = jd@domain.org
+		signingkey = jd@domain.org
+
+You may need to tell ``git`` to use ``gpg2``
+::
+
+	[gpg]
+		program = /path/to/gpg2
+
+You may also like to tell ``gpg`` which ``tty`` to use (add to your shell rc file)
+::
+
+	export GPG_TTY=$(tty)
diff --git a/Documentation/maintainer/index.rst b/Documentation/maintainer/index.rst
new file mode 100644
index 000000000000..2a14916930cb
--- /dev/null
+++ b/Documentation/maintainer/index.rst
@@ -0,0 +1,14 @@
+==========================
+Kernel Maintainer Handbook
+==========================
+
+This document is the humble beginning of a manual for kernel maintainers.
+There is a lot yet to go here!  Please feel free to propose (and write)
+additions to this manual.
+
+.. toctree::
+   :maxdepth: 2
+
+   configure-git
+   pull-requests
+
diff --git a/Documentation/maintainer/pull-requests.rst b/Documentation/maintainer/pull-requests.rst
new file mode 100644
index 000000000000..a19db3458b56
--- /dev/null
+++ b/Documentation/maintainer/pull-requests.rst
@@ -0,0 +1,178 @@
+.. _pullrequests:
+
+Creating Pull Requests
+======================
+
+This chapter describes how maintainers can create and submit pull requests
+to other maintainers. This is useful for transferring changes from one
+maintainers tree to another maintainers tree.
+
+This document was written by Tobin C. Harding (who at that time, was not an
+experienced maintainer) primarily from comments made by Greg Kroah-Hartman
+and Linus Torvalds on LKML. Suggestions and fixes by Jonathan Corbet and
+Mauro Carvalho Chehab.  Misrepresentation was unintentional but inevitable,
+please direct abuse to Tobin C. Harding <me@tobin.cc>.
+
+Original email thread::
+
+	http://lkml.kernel.org/r/20171114110500.GA21175@kroah.com
+
+
+Create Branch
+-------------
+
+To start with you will need to have all the changes you wish to include in
+the pull request on a separate branch. Typically you will base this branch
+off of a branch in the developers tree whom you intend to send the pull
+request to.
+
+In order to create the pull request you must first tag the branch that you
+have just created. It is recommended that you choose a meaningful tag name,
+in a way that you and others can understand, even after some time.  A good
+practice is to include in the name an indicator of the sybsystem of origin
+and the target kernel version.
+
+Greg offers the following. A pull request with miscellaneous stuff for
+drivers/char, to be applied at the Kernel version 4.15-rc1 could be named
+as ``char-misc-4.15-rc1``. If such tag would be produced from a branch
+named ``char-misc-next``, you would be using the following command::
+
+        git tag -s char-misc-4.15-rc1 char-misc-next
+
+that will create a signed tag called ``char-misc-4.15-rc1`` based on the
+last commit in the ``char-misc-next`` branch, and sign it with your gpg key
+(see :ref:`Documentation/maintainer/configure_git.rst <configuregit>`).
+
+Linus will only accept pull requests based on a signed tag. Other
+maintainers may differ.
+
+When you run the above command ``git`` will drop you into an editor and ask
+you to describe the tag.  In this case, you are describing a pull request,
+so outline what is contained here, why it should be merged, and what, if
+any, testing has been done.  All of this information will end up in the tag
+itself, and then in the merge commit that the maintainer makes if/when they
+merge the pull request. So write it up well, as it will be in the kernel
+tree for forever.
+
+As said by Linus::
+
+	Anyway, at least to me, the important part is the *message*. I want
+	to understand what I'm pulling, and why I should pull it. I also
+	want to use that message as the message for the merge, so it should
+	not just make sense to me, but make sense as a historical record
+	too.
+
+	Note that if there is something odd about the pull request, that
+	should very much be in the explanation. If you're touching files
+	that you don't maintain, explain _why_. I will see it in the
+	diffstat anyway, and if you didn't mention it, I'll just be extra
+	suspicious.  And when you send me new stuff after the merge window
+	(or even bug-fixes, but ones that look scary), explain not just
+	what they do and why they do it, but explain the _timing_. What
+	happened that this didn't go through the merge window..
+
+	I will take both what you write in the email pull request _and_ in
+	the signed tag, so depending on your workflow, you can either
+	describe your work in the signed tag (which will also automatically
+	make it into the pull request email), or you can make the signed
+	tag just a placeholder with nothing interesting in it, and describe
+	the work later when you actually send me the pull request.
+
+	And yes, I will edit the message. Partly because I tend to do just
+	trivial formatting (the whole indentation and quoting etc), but
+	partly because part of the message may make sense for me at pull
+	time (describing the conflicts and your personal issues for sending
+	it right now), but may not make sense in the context of a merge
+	commit message, so I will try to make it all make sense. I will
+	also fix any speeling mistaeks and bad grammar I notice,
+	particularly for non-native speakers (but also for native ones
+	;^). But I may miss some, or even add some.
+
+			Linus
+
+Greg gives, as an example pull request::
+
+	Char/Misc patches for 4.15-rc1
+
+	Here is the big char/misc patch set for the 4.15-rc1 merge window.
+	Contained in here is the normal set of new functions added to all
+	of these crazy drivers, as well as the following brand new
+	subsystems:
+		- time_travel_controller: Finally a set of drivers for the
+		  latest time travel bus architecture that provides i/o to
+		  the CPU before it asked for it, allowing uninterrupted
+		  processing
+		- relativity_shifters: due to the affect that the
+		  time_travel_controllers have on the overall system, there
+		  was a need for a new set of relativity shifter drivers to
+		  accommodate the newly formed black holes that would
+		  threaten to suck CPUs into them.  This subsystem handles
+		  this in a way to successfully neutralize the problems.
+		  There is a Kconfig option to force these to be enabled
+		  when needed, so problems should not occur.
+
+	All of these patches have been successfully tested in the latest
+	linux-next releases, and the original problems that it found have
+	all been resolved (apologies to anyone living near Canberra for the
+	lack of the Kconfig options in the earlier versions of the
+	linux-next tree creations.)
+
+	Signed-off-by: Your-name-here <your_email@domain>
+
+
+The tag message format is just like a git commit id.  One line at the top
+for a "summary subject" and be sure to sign-off at the bottom.
+
+Now that you have a local signed tag, you need to push it up to where it
+can be retrieved::
+
+	git push origin char-misc-4.15-rc1
+
+
+Create Pull Request
+-------------------
+
+The last thing to do is create the pull request message.  ``git`` handily
+will do this for you with the ``git request-pull`` command, but it needs a
+bit of help determining what you want to pull, and on what to base the pull
+against (to show the correct changes to be pulled and the diffstat). The
+following command(s) will generate a pull request::
+
+	git request-pull master git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git/ char-misc-4.15-rc1
+
+Quoting Greg::
+
+	This is asking git to compare the difference from the
+	'char-misc-4.15-rc1' tag location, to the head of the 'master'
+	branch (which in my case points to the last location in Linus's
+	tree that I diverged from, usually a -rc release) and to use the
+	git:// protocol to pull from.  If you wish to use https://, that
+	can be used here instead as well (but note that some people behind
+	firewalls will have problems with https git pulls).
+
+	If the char-misc-4.15-rc1 tag is not present in the repo that I am
+	asking to be pulled from, git will complain saying it is not there,
+	a handy way to remember to actually push it to a public location.
+
+	The output of 'git request-pull' will contain the location of the
+	git tree and specific tag to pull from, and the full text
+	description of that tag (which is why you need to provide good
+	information in that tag).  It will also create a diffstat of the
+	pull request, and a shortlog of the individual commits that the
+	pull request will provide.
+
+Linus responded that he tends to prefer the ``git://`` protocol. Other
+maintainers may have different preferences. Also, note that if you are
+creating pull requests without a signed tag then ``https://`` may be a
+better choice. Please see the original thread for the full discussion.
+
+
+Submit Pull Request
+-------------------
+
+A pull request is submitted in the same way as an ordinary patch. Send as
+inline email to the maintainer and CC LKML and any sub-system specific
+lists if required. Pull requests to Linus typically have a subject line
+something like::
+
+	[GIT PULL] <subsystem> changes for v4.15-rc1
diff --git a/Documentation/md/raid5-ppl.txt b/Documentation/md/raid5-ppl.txt
index 127072b09363..bfa092589e00 100644
--- a/Documentation/md/raid5-ppl.txt
+++ b/Documentation/md/raid5-ppl.txt
@@ -39,6 +39,7 @@ case the behavior is the same as in plain raid5.
 PPL is available for md version-1 metadata and external (specifically IMSM)
 metadata arrays. It can be enabled using mdadm option --consistency-policy=ppl.
 
-Currently, volatile write-back cache should be disabled on all member drives
-when using PPL. Otherwise it cannot guarantee consistency in case of power
-failure.
+There is a limitation of maximum 64 disks in the array for PPL. It allows to
+keep data structures and implementation simple. RAID5 arrays with so many disks
+are not likely due to high risk of multiple disks failure. Such restriction
+should not be a real life limitation.
diff --git a/Documentation/media/dmx.h.rst.exceptions b/Documentation/media/dmx.h.rst.exceptions
index 629db384104a..a8c4239ed95b 100644
--- a/Documentation/media/dmx.h.rst.exceptions
+++ b/Documentation/media/dmx.h.rst.exceptions
@@ -50,7 +50,15 @@ replace typedef dmx_filter_t :c:type:`dmx_filter`
 replace typedef dmx_pes_type_t :c:type:`dmx_pes_type`
 replace typedef dmx_input_t :c:type:`dmx_input`
 
-ignore symbol DMX_OUT_DECODER
-ignore symbol DMX_OUT_TAP
-ignore symbol DMX_OUT_TS_TAP
-ignore symbol DMX_OUT_TSDEMUX_TAP
+replace symbol DMX_BUFFER_FLAG_HAD_CRC32_DISCARD :c:type:`dmx_buffer_flags`
+replace	symbol DMX_BUFFER_FLAG_TEI :c:type:`dmx_buffer_flags`
+replace	symbol DMX_BUFFER_PKT_COUNTER_MISMATCH :c:type:`dmx_buffer_flags`
+replace	symbol DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED :c:type:`dmx_buffer_flags`
+replace	symbol DMX_BUFFER_FLAG_DISCONTINUITY_INDICATOR :c:type:`dmx_buffer_flags`
+
+replace symbol DMX_OUT_DECODER :c:type:`dmx_output`
+replace symbol DMX_OUT_TAP :c:type:`dmx_output`
+replace symbol DMX_OUT_TS_TAP :c:type:`dmx_output`
+replace symbol DMX_OUT_TSDEMUX_TAP :c:type:`dmx_output`
+
+replace ioctl DMX_DQBUF dmx_qbuf
diff --git a/Documentation/media/kapi/cec-core.rst b/Documentation/media/kapi/cec-core.rst
index d37e107f2fde..62b9a1448177 100644
--- a/Documentation/media/kapi/cec-core.rst
+++ b/Documentation/media/kapi/cec-core.rst
@@ -103,6 +103,7 @@ your driver:
 		/* Low-level callbacks */
 		int (*adap_enable)(struct cec_adapter *adap, bool enable);
 		int (*adap_monitor_all_enable)(struct cec_adapter *adap, bool enable);
+		int (*adap_monitor_pin_enable)(struct cec_adapter *adap, bool enable);
 		int (*adap_log_addr)(struct cec_adapter *adap, u8 logical_addr);
 		int (*adap_transmit)(struct cec_adapter *adap, u8 attempts,
 				      u32 signal_free_time, struct cec_msg *msg);
@@ -144,6 +145,19 @@ called if the CEC_CAP_MONITOR_ALL capability is set. This callback is optional
 Note that adap_monitor_all_enable must return 0 if enable is false.
 
 
+To enable/disable the 'monitor pin' mode:
+
+.. c:function::
+	int (*adap_monitor_pin_enable)(struct cec_adapter *adap, bool enable);
+
+If enabled, then the adapter should be put in a mode to also monitor CEC pin
+changes. Not all hardware supports this and this function is only called if
+the CEC_CAP_MONITOR_PIN capability is set. This callback is optional
+(some hardware may always be in 'monitor pin' mode).
+
+Note that adap_monitor_pin_enable must return 0 if enable is false.
+
+
 To program a new logical address:
 
 .. c:function::
diff --git a/Documentation/media/kapi/dtv-ca.rst b/Documentation/media/kapi/dtv-ca.rst
index a4dd700189b0..fded096b937c 100644
--- a/Documentation/media/kapi/dtv-ca.rst
+++ b/Documentation/media/kapi/dtv-ca.rst
@@ -1,4 +1,4 @@
 Digital TV Conditional Access kABI
 ----------------------------------
 
-.. kernel-doc:: drivers/media/dvb-core/dvb_ca_en50221.h
+.. kernel-doc:: include/media/dvb_ca_en50221.h
diff --git a/Documentation/media/kapi/dtv-common.rst b/Documentation/media/kapi/dtv-common.rst
index 40cf1033b5e1..7a9574f03190 100644
--- a/Documentation/media/kapi/dtv-common.rst
+++ b/Documentation/media/kapi/dtv-common.rst
@@ -7,7 +7,7 @@ Math functions
 Provide some commonly-used math functions, usually required in order to
 estimate signal strength and signal to noise measurements in dB.
 
-.. kernel-doc:: drivers/media/dvb-core/dvb_math.h
+.. kernel-doc:: include/media/dvb_math.h
 
 
 DVB devices
@@ -15,7 +15,7 @@ DVB devices
 
 Those functions are responsible for handling the DVB device nodes.
 
-.. kernel-doc:: drivers/media/dvb-core/dvbdev.h
+.. kernel-doc:: include/media/dvbdev.h
 
 Digital TV Ring buffer
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -52,4 +52,9 @@ copy it from/to userspace.
      Resetting the buffer counts as a read and write operation.
      Two or more writers must be locked against each other.
 
-.. kernel-doc:: drivers/media/dvb-core/dvb_ringbuffer.h
+.. kernel-doc:: include/media/dvb_ringbuffer.h
+
+Digital TV VB2 handler
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. kernel-doc:: include/media/dvb_vb2.h
diff --git a/Documentation/media/kapi/dtv-demux.rst b/Documentation/media/kapi/dtv-demux.rst
index 7aa865a2b43f..24857133e4e8 100644
--- a/Documentation/media/kapi/dtv-demux.rst
+++ b/Documentation/media/kapi/dtv-demux.rst
@@ -8,7 +8,7 @@ The Kernel Digital TV Demux kABI defines a driver-internal interface for
 registering low-level, hardware specific driver to a hardware independent
 demux layer. It is only of interest for Digital TV device driver writers.
 The header file for this kABI is named ``demux.h`` and located in
-``drivers/media/dvb-core``.
+``include/media``.
 
 The demux kABI should be implemented for each demux in the system. It is
 used to select the TS source of a demux and to manage the demux resources.
@@ -69,14 +69,14 @@ callbacks.
 Digital TV Demux device registration functions and data structures
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. kernel-doc:: drivers/media/dvb-core/dmxdev.h
+.. kernel-doc:: include/media/dmxdev.h
 
 High-level Digital TV demux interface
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. kernel-doc:: drivers/media/dvb-core/dvb_demux.h
+.. kernel-doc:: include/media/dvb_demux.h
 
 Driver-internal low-level hardware specific driver demux interface
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. kernel-doc:: drivers/media/dvb-core/demux.h
+.. kernel-doc:: include/media/demux.h
diff --git a/Documentation/media/kapi/dtv-frontend.rst b/Documentation/media/kapi/dtv-frontend.rst
index f1a2fdaab5ba..472650cdb100 100644
--- a/Documentation/media/kapi/dtv-frontend.rst
+++ b/Documentation/media/kapi/dtv-frontend.rst
@@ -8,7 +8,7 @@ The Digital TV Frontend kABI defines a driver-internal interface for
 registering low-level, hardware specific driver to a hardware independent
 frontend layer. It is only of interest for Digital TV device driver writers.
 The header file for this API is named ``dvb_frontend.h`` and located in
-``drivers/media/dvb-core``.
+``include/media/``.
 
 Demodulator driver
 ^^^^^^^^^^^^^^^^^^
@@ -17,7 +17,7 @@ The demodulator driver is responsible to talk with the decoding part of the
 hardware. Such driver should implement :c:type:`dvb_frontend_ops`, with
 tells what type of digital TV standards are supported, and points to a
 series of functions that allow the DVB core to command the hardware via
-the code under ``drivers/media/dvb-core/dvb_frontend.c``.
+the code under ``include/media/dvb_frontend.c``.
 
 A typical example of such struct in a driver ``foo`` is::
 
@@ -118,7 +118,7 @@ Satellite TV reception is::
 
 .. |delta|   unicode:: U+00394
 
-The ``drivers/media/dvb-core/dvb_frontend.c`` has a kernel thread with is
+The ``include/media/dvb_frontend.c`` has a kernel thread with is
 responsible for tuning the device. It supports multiple algorithms to
 detect a channel, as defined at enum :c:func:`dvbfe_algo`.
 
@@ -440,4 +440,4 @@ monotonic stats at the right time.
 Digital TV Frontend functions and types
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. kernel-doc:: drivers/media/dvb-core/dvb_frontend.h
+.. kernel-doc:: include/media/dvb_frontend.h
diff --git a/Documentation/media/kapi/dtv-net.rst b/Documentation/media/kapi/dtv-net.rst
index ced991b73d69..158c7cbd7600 100644
--- a/Documentation/media/kapi/dtv-net.rst
+++ b/Documentation/media/kapi/dtv-net.rst
@@ -1,4 +1,4 @@
 Digital TV Network kABI
 -----------------------
 
-.. kernel-doc:: drivers/media/dvb-core/dvb_net.h
+.. kernel-doc:: include/media/dvb_net.h
diff --git a/Documentation/media/kapi/rc-core.rst b/Documentation/media/kapi/rc-core.rst
index a45895886257..4759f020d6b2 100644
--- a/Documentation/media/kapi/rc-core.rst
+++ b/Documentation/media/kapi/rc-core.rst
@@ -4,11 +4,83 @@ Remote Controller devices
 Remote Controller core
 ~~~~~~~~~~~~~~~~~~~~~~
 
-.. kernel-doc:: include/media/rc-core.h
+The remote controller core implements infrastructure to receive and send
+remote controller keyboard keystrokes and mouse events.
 
-.. kernel-doc:: include/media/rc-map.h
+Every time a key is pressed on a remote controller, a scan code is produced.
+Also, on most hardware, keeping a key pressed for more than a few dozens of
+milliseconds produce a repeat key event. That's somewhat similar to what
+a normal keyboard or mouse is handled internally on Linux\ [#f1]_. So, the
+remote controller core is implemented on the top of the linux input/evdev
+interface.
+
+.. [#f1]
+
+   The main difference is that, on keyboard events, the keyboard controller
+   produces one event for a key press and another one for key release. On
+   infrared-based remote controllers, there's no key release event. Instead,
+   an extra code is produced to indicate key repeats.
+
+However, most of the remote controllers use infrared (IR) to transmit signals.
+As there are several protocols used to modulate infrared signals, one
+important part of the core is dedicated to adjust the driver and the core
+system to support the infrared protocol used by the emitter.
+
+The infrared transmission is done by blinking a infrared emitter using a
+carrier. The carrier can be switched on or off by the IR transmitter
+hardware. When the carrier is switched on, it is called *PULSE*.
+When the carrier is switched off, it is called *SPACE*.
+
+In other words, a typical IR transmission can be viewed as a sequence of
+*PULSE* and *SPACE* events, each with a given duration.
+
+The carrier parameters (frequency, duty cycle) and the intervals for
+*PULSE* and *SPACE* events depend on the protocol.
+For example, the NEC protocol uses a carrier of 38kHz, and transmissions
+start with a 9ms *PULSE* and a 4.5ms SPACE. It then transmits 16 bits of
+scan code, being 8 bits for address (usually it is a fixed number for a
+given remote controller), followed by 8 bits of code. A bit "1" is modulated
+with 560µs *PULSE* followed by 1690µs *SPACE* and a bit "0"  is modulated
+with 560µs *PULSE* followed by 560µs *SPACE*.
+
+At receiver, a simple low-pass filter can be used to convert the received
+signal in a sequence of *PULSE/SPACE* events, filtering out the carrier
+frequency. Due to that, the receiver doesn't care about the carrier's
+actual frequency parameters: all it has to do is to measure the amount
+of time it receives *PULSE/SPACE* events.
+So, a simple IR receiver hardware will just provide a sequence of timings
+for those events to the Kernel. The drivers for hardware with such kind of
+receivers are identified by  ``RC_DRIVER_IR_RAW``, as defined by
+:c:type:`rc_driver_type`\ [#f2]_. Other hardware come with a
+microcontroller that decode the *PULSE/SPACE* sequence and return scan
+codes to the Kernel. Such kind of receivers are identified
+by ``RC_DRIVER_SCANCODE``.
 
-LIRC
-~~~~
+.. [#f2]
 
-.. kernel-doc:: include/media/lirc_dev.h
+   The RC core also supports devices that have just IR emitters,
+   without any receivers. Right now, all such devices work only in
+   raw TX mode. Such kind of hardware is identified as
+   ``RC_DRIVER_IR_RAW_TX``.
+
+When the RC core receives events produced by ``RC_DRIVER_IR_RAW`` IR
+receivers, it needs to decode the IR protocol, in order to obtain the
+corresponding scan code. The protocols supported by the RC core are
+defined at enum :c:type:`rc_proto`.
+
+When the RC code receives a scan code (either directly, by a driver
+of the type ``RC_DRIVER_SCANCODE``, or via its IR decoders), it needs
+to convert into a Linux input event code. This is done via a mapping
+table.
+
+The Kernel has support for mapping tables available on most media
+devices. It also supports loading a table in runtime, via some
+sysfs nodes. See the :ref:`RC userspace API <Remote_controllers_Intro>`
+for more details.
+
+Remote controller data structures and functions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. kernel-doc:: include/media/rc-core.h
+
+.. kernel-doc:: include/media/rc-map.h
diff --git a/Documentation/media/kapi/v4l2-dev.rst b/Documentation/media/kapi/v4l2-dev.rst
index b29aa616c267..7bb0505b60f1 100644
--- a/Documentation/media/kapi/v4l2-dev.rst
+++ b/Documentation/media/kapi/v4l2-dev.rst
@@ -196,11 +196,18 @@ device.
 Which device is registered depends on the type argument. The following
 types exist:
 
-- ``VFL_TYPE_GRABBER``: ``/dev/videoX`` for video input/output devices
-- ``VFL_TYPE_VBI``: ``/dev/vbiX`` for vertical blank data (i.e. closed captions, teletext)
-- ``VFL_TYPE_RADIO``: ``/dev/radioX`` for radio tuners
-- ``VFL_TYPE_SDR``: ``/dev/swradioX`` for Software Defined Radio tuners
-- ``VFL_TYPE_TOUCH``: ``/dev/v4l-touchX`` for touch sensors
+========================== ====================	 ==============================
+:c:type:`vfl_devnode_type` Device name		 Usage
+========================== ====================	 ==============================
+``VFL_TYPE_GRABBER``       ``/dev/videoX``       for video input/output devices
+``VFL_TYPE_VBI``           ``/dev/vbiX``         for vertical blank data (i.e.
+						 closed captions, teletext)
+``VFL_TYPE_RADIO``         ``/dev/radioX``       for radio tuners
+``VFL_TYPE_SUBDEV``        ``/dev/v4l-subdevX``  for V4L2 subdevices
+``VFL_TYPE_SDR``           ``/dev/swradioX``     for Software Defined Radio
+						 (SDR) tuners
+``VFL_TYPE_TOUCH``         ``/dev/v4l-touchX``   for touch sensors
+========================== ====================	 ==============================
 
 The last argument gives you a certain amount of control over the device
 device node number used (i.e. the X in ``videoX``). Normally you will pass -1
diff --git a/Documentation/media/kapi/v4l2-event.rst b/Documentation/media/kapi/v4l2-event.rst
index 9938d21ef4d1..5c7e31224ddc 100644
--- a/Documentation/media/kapi/v4l2-event.rst
+++ b/Documentation/media/kapi/v4l2-event.rst
@@ -5,27 +5,68 @@ V4L2 events
 The V4L2 events provide a generic way to pass events to user space.
 The driver must use :c:type:`v4l2_fh` to be able to support V4L2 events.
 
-Events are defined by a type and an optional ID. The ID may refer to a V4L2
-object such as a control ID. If unused, then the ID is 0.
+Events are subscribed per-filehandle. An event specification consists of a
+``type`` and is optionally associated with an object identified through the
+``id`` field. If unused, then the ``id`` is 0. So an event is uniquely
+identified by the ``(type, id)`` tuple.
 
-When the user subscribes to an event the driver will allocate a number of
-kevent structs for that event. So every (type, ID) event tuple will have
-its own set of kevent structs. This guarantees that if a driver is generating
-lots of events of one type in a short time, then that will not overwrite
-events of another type.
+The :c:type:`v4l2_fh` struct has a list of subscribed events on its
+``subscribed`` field.
 
-But if you get more events of one type than the number of kevents that were
-reserved, then the oldest event will be dropped and the new one added.
+When the user subscribes to an event, a :c:type:`v4l2_subscribed_event`
+struct is added to :c:type:`v4l2_fh`\ ``.subscribed``, one for every
+subscribed event.
+
+Each :c:type:`v4l2_subscribed_event` struct ends with a
+:c:type:`v4l2_kevent` ringbuffer, with the size given by the caller
+of :c:func:`v4l2_event_subscribe`. This ringbuffer is used to store any events
+raised by the driver.
+
+So every ``(type, ID)`` event tuple will have its own
+:c:type:`v4l2_kevent` ringbuffer. This guarantees that if a driver is
+generating lots of events of one type in a short time, then that will
+not overwrite events of another type.
+
+But if you get more events of one type than the size of the
+:c:type:`v4l2_kevent` ringbuffer, then the oldest event will be dropped
+and the new one added.
+
+The :c:type:`v4l2_kevent` struct links into the ``available``
+list of the :c:type:`v4l2_fh` struct so :ref:`VIDIOC_DQEVENT` will
+know which event to dequeue first.
+
+Finally, if the event subscription is associated with a particular object
+such as a V4L2 control, then that object needs to know about that as well
+so that an event can be raised by that object. So the ``node`` field can
+be used to link the :c:type:`v4l2_subscribed_event` struct into a list of
+such objects.
+
+So to summarize:
+
+- struct :c:type:`v4l2_fh` has two lists: one of the ``subscribed`` events,
+  and one of the ``available`` events.
+
+- struct :c:type:`v4l2_subscribed_event` has a ringbuffer of raised
+  (pending) events of that particular type.
+
+- If struct :c:type:`v4l2_subscribed_event` is associated with a specific
+  object, then that object will have an internal list of
+  struct :c:type:`v4l2_subscribed_event` so it knows who subscribed an
+  event to that object.
 
 Furthermore, the internal struct :c:type:`v4l2_subscribed_event` has
 ``merge()`` and ``replace()`` callbacks which drivers can set. These
 callbacks are called when a new event is raised and there is no more room.
+
 The ``replace()`` callback allows you to replace the payload of the old event
 with that of the new event, merging any relevant data from the old payload
 into the new payload that replaces it. It is called when this event type has
-only one kevent struct allocated. The ``merge()`` callback allows you to merge
-the oldest event payload into that of the second-oldest event payload. It is
-called when there are two or more kevent structs allocated.
+a ringbuffer with size is one, i.e. only one event can be stored in the
+ringbuffer.
+
+The ``merge()`` callback allows you to merge the oldest event payload into
+that of the second-oldest event payload. It is called when
+the ringbuffer has size is greater than one.
 
 This way no status information is lost, just the intermediate steps leading
 up to that state.
@@ -73,7 +114,7 @@ The ops argument allows the driver to specify a number of callbacks:
 Callback Description
 ======== ==============================================================
 add      called when a new listener gets added (subscribing to the same
-         event twice will only cause this callback to get called once)
+	 event twice will only cause this callback to get called once)
 del      called when a listener stops listening
 replace  replace event 'old' with event 'new'.
 merge    merge event 'old' into event 'new'.
diff --git a/Documentation/media/lirc.h.rst.exceptions b/Documentation/media/lirc.h.rst.exceptions
index c130617a9986..c6e3a35d2c4e 100644
--- a/Documentation/media/lirc.h.rst.exceptions
+++ b/Documentation/media/lirc.h.rst.exceptions
@@ -28,6 +28,36 @@ ignore define LIRC_CAN_SEND_MASK
 ignore define LIRC_CAN_REC_MASK
 ignore define LIRC_CAN_SET_REC_DUTY_CYCLE
 
+# Obsolete ioctls
+
+ignore ioctl LIRC_GET_LENGTH
+
+# rc protocols
+
+ignore symbol RC_PROTO_UNKNOWN
+ignore symbol RC_PROTO_OTHER
+ignore symbol RC_PROTO_RC5
+ignore symbol RC_PROTO_RC5X_20
+ignore symbol RC_PROTO_RC5_SZ
+ignore symbol RC_PROTO_JVC
+ignore symbol RC_PROTO_SONY12
+ignore symbol RC_PROTO_SONY15
+ignore symbol RC_PROTO_SONY20
+ignore symbol RC_PROTO_NEC
+ignore symbol RC_PROTO_NECX
+ignore symbol RC_PROTO_NEC32
+ignore symbol RC_PROTO_SANYO
+ignore symbol RC_PROTO_MCIR2_KBD
+ignore symbol RC_PROTO_MCIR2_MSE
+ignore symbol RC_PROTO_RC6_0
+ignore symbol RC_PROTO_RC6_6A_20
+ignore symbol RC_PROTO_RC6_6A_24
+ignore symbol RC_PROTO_RC6_6A_32
+ignore symbol RC_PROTO_RC6_MCE
+ignore symbol RC_PROTO_SHARP
+ignore symbol RC_PROTO_XMP
+ignore symbol RC_PROTO_CEC
+
 # Undocumented macros
 
 ignore define PULSE_BIT
@@ -40,3 +70,4 @@ ignore define LIRC_VALUE_MASK
 ignore define LIRC_MODE2_MASK
 
 ignore define LIRC_MODE_RAW
+ignore define LIRC_MODE_LIRCCODE
diff --git a/Documentation/media/uapi/dvb/dmx-expbuf.rst b/Documentation/media/uapi/dvb/dmx-expbuf.rst
new file mode 100644
index 000000000000..2d96cfe891df
--- /dev/null
+++ b/Documentation/media/uapi/dvb/dmx-expbuf.rst
@@ -0,0 +1,88 @@
+.. _DMX_EXPBUF:
+
+****************
+ioctl DMX_EXPBUF
+****************
+
+Name
+====
+
+DMX_EXPBUF - Export a buffer as a DMABUF file descriptor.
+
+.. warning:: this API is still experimental
+
+
+Synopsis
+========
+
+.. c:function:: int ioctl( int fd, DMX_EXPBUF, struct dmx_exportbuffer *argp )
+    :name: DMX_EXPBUF
+
+
+Arguments
+=========
+
+``fd``
+    File descriptor returned by :ref:`open() <dmx_fopen>`.
+
+``argp``
+    Pointer to struct :c:type:`dmx_exportbuffer`.
+
+
+Description
+===========
+
+This ioctl is an extension to the memory mapping I/O method.
+It can be used to export a buffer as a DMABUF file at any time after
+buffers have been allocated with the :ref:`DMX_REQBUFS` ioctl.
+
+To export a buffer, applications fill struct :c:type:`dmx_exportbuffer`.
+Applications must set the ``index`` field. Valid index numbers
+range from zero to the number of buffers allocated with :ref:`DMX_REQBUFS`
+(struct :c:type:`dmx_requestbuffers` ``count``) minus one.
+Additional flags may be posted in the ``flags`` field. Refer to a manual
+for open() for details. Currently only O_CLOEXEC, O_RDONLY, O_WRONLY,
+and O_RDWR are supported.
+All other fields must be set to zero. In the
+case of multi-planar API, every plane is exported separately using
+multiple :ref:`DMX_EXPBUF` calls.
+
+After calling :ref:`DMX_EXPBUF` the ``fd`` field will be set by a
+driver, on success. This is a DMABUF file descriptor. The application may
+pass it to other DMABUF-aware devices. It is recommended to close a DMABUF
+file when it is no longer used to allow the associated memory to be reclaimed.
+
+
+Examples
+========
+
+
+.. code-block:: c
+
+    int buffer_export(int v4lfd, enum dmx_buf_type bt, int index, int *dmafd)
+    {
+	struct dmx_exportbuffer expbuf;
+
+	memset(&expbuf, 0, sizeof(expbuf));
+	expbuf.type = bt;
+	expbuf.index = index;
+	if (ioctl(v4lfd, DMX_EXPBUF, &expbuf) == -1) {
+	    perror("DMX_EXPBUF");
+	    return -1;
+	}
+
+	*dmafd = expbuf.fd;
+
+	return 0;
+    }
+
+Return Value
+============
+
+On success 0 is returned, on error -1 and the ``errno`` variable is set
+appropriately. The generic error codes are described at the
+:ref:`Generic Error Codes <gen-errors>` chapter.
+
+EINVAL
+    A queue is not in MMAP mode or DMABUF exporting is not supported or
+    ``flags`` or ``index`` fields are invalid.
diff --git a/Documentation/media/uapi/dvb/dmx-mmap.rst b/Documentation/media/uapi/dvb/dmx-mmap.rst
new file mode 100644
index 000000000000..15d107348b9f
--- /dev/null
+++ b/Documentation/media/uapi/dvb/dmx-mmap.rst
@@ -0,0 +1,116 @@
+.. _dmx-mmap:
+
+*****************
+Digital TV mmap()
+*****************
+
+Name
+====
+
+dmx-mmap - Map device memory into application address space
+
+.. warning:: this API is still experimental
+
+Synopsis
+========
+
+.. code-block:: c
+
+    #include <unistd.h>
+    #include <sys/mman.h>
+
+
+.. c:function:: void *mmap( void *start, size_t length, int prot, int flags, int fd, off_t offset )
+    :name: dmx-mmap
+
+Arguments
+=========
+
+``start``
+    Map the buffer to this address in the application's address space.
+    When the ``MAP_FIXED`` flag is specified, ``start`` must be a
+    multiple of the pagesize and mmap will fail when the specified
+    address cannot be used. Use of this option is discouraged;
+    applications should just specify a ``NULL`` pointer here.
+
+``length``
+    Length of the memory area to map. This must be a multiple of the
+    DVB packet length (188, on most drivers).
+
+``prot``
+    The ``prot`` argument describes the desired memory protection.
+    Regardless of the device type and the direction of data exchange it
+    should be set to ``PROT_READ`` | ``PROT_WRITE``, permitting read
+    and write access to image buffers. Drivers should support at least
+    this combination of flags.
+
+``flags``
+    The ``flags`` parameter specifies the type of the mapped object,
+    mapping options and whether modifications made to the mapped copy of
+    the page are private to the process or are to be shared with other
+    references.
+
+    ``MAP_FIXED`` requests that the driver selects no other address than
+    the one specified. If the specified address cannot be used,
+    :ref:`mmap() <dmx-mmap>` will fail. If ``MAP_FIXED`` is specified,
+    ``start`` must be a multiple of the pagesize. Use of this option is
+    discouraged.
+
+    One of the ``MAP_SHARED`` or ``MAP_PRIVATE`` flags must be set.
+    ``MAP_SHARED`` allows applications to share the mapped memory with
+    other (e. g. child-) processes.
+
+    .. note::
+
+       The Linux Digital TV applications should not set the
+       ``MAP_PRIVATE``, ``MAP_DENYWRITE``, ``MAP_EXECUTABLE`` or ``MAP_ANON``
+       flags.
+
+``fd``
+    File descriptor returned by :ref:`open() <dmx_fopen>`.
+
+``offset``
+    Offset of the buffer in device memory, as returned by
+    :ref:`DMX_QUERYBUF` ioctl.
+
+
+Description
+===========
+
+The :ref:`mmap() <dmx-mmap>` function asks to map ``length`` bytes starting at
+``offset`` in the memory of the device specified by ``fd`` into the
+application address space, preferably at address ``start``. This latter
+address is a hint only, and is usually specified as 0.
+
+Suitable length and offset parameters are queried with the
+:ref:`DMX_QUERYBUF` ioctl. Buffers must be allocated with the
+:ref:`DMX_REQBUFS` ioctl before they can be queried.
+
+To unmap buffers the :ref:`munmap() <dmx-munmap>` function is used.
+
+
+Return Value
+============
+
+On success :ref:`mmap() <dmx-mmap>` returns a pointer to the mapped buffer. On
+error ``MAP_FAILED`` (-1) is returned, and the ``errno`` variable is set
+appropriately. Possible error codes are:
+
+EBADF
+    ``fd`` is not a valid file descriptor.
+
+EACCES
+    ``fd`` is not open for reading and writing.
+
+EINVAL
+    The ``start`` or ``length`` or ``offset`` are not suitable. (E. g.
+    they are too large, or not aligned on a ``PAGESIZE`` boundary.)
+
+    The ``flags`` or ``prot`` value is not supported.
+
+    No buffers have been allocated with the
+    :ref:`DMX_REQBUFS` ioctl.
+
+ENOMEM
+    Not enough physical or virtual memory was available to complete the
+    request.
diff --git a/Documentation/media/uapi/dvb/dmx-munmap.rst b/Documentation/media/uapi/dvb/dmx-munmap.rst
new file mode 100644
index 000000000000..d77218732bb6
--- /dev/null
+++ b/Documentation/media/uapi/dvb/dmx-munmap.rst
@@ -0,0 +1,54 @@
+.. _dmx-munmap:
+
+************
+DVB munmap()
+************
+
+Name
+====
+
+dmx-munmap - Unmap device memory
+
+.. warning:: This API is still experimental.
+
+
+Synopsis
+========
+
+.. code-block:: c
+
+    #include <unistd.h>
+    #include <sys/mman.h>
+
+
+.. c:function:: int munmap( void *start, size_t length )
+    :name: dmx-munmap
+
+Arguments
+=========
+
+``start``
+    Address of the mapped buffer as returned by the
+    :ref:`mmap() <dmx-mmap>` function.
+
+``length``
+    Length of the mapped buffer. This must be the same value as given to
+    :ref:`mmap() <dmx-mmap>`.
+
+
+Description
+===========
+
+Unmaps a previously with the :ref:`mmap() <dmx-mmap>` function mapped
+buffer and frees it, if possible.
+
+
+Return Value
+============
+
+On success :ref:`munmap() <dmx-munmap>` returns 0, on failure -1 and the
+``errno`` variable is set appropriately:
+
+EINVAL
+    The ``start`` or ``length`` is incorrect, or no buffers have been
+    mapped yet.
diff --git a/Documentation/media/uapi/dvb/dmx-qbuf.rst b/Documentation/media/uapi/dvb/dmx-qbuf.rst
new file mode 100644
index 000000000000..be5a4c6f1904
--- /dev/null
+++ b/Documentation/media/uapi/dvb/dmx-qbuf.rst
@@ -0,0 +1,84 @@
+.. _DMX_QBUF:
+
+*************************
+ioctl DMX_QBUF, DMX_DQBUF
+*************************
+
+Name
+====
+
+DMX_QBUF - DMX_DQBUF - Exchange a buffer with the driver
+
+.. warning:: this API is still experimental
+
+
+Synopsis
+========
+
+.. c:function:: int ioctl( int fd, DMX_QBUF, struct dmx_buffer *argp )
+    :name: DMX_QBUF
+
+.. c:function:: int ioctl( int fd, DMX_DQBUF, struct dmx_buffer *argp )
+    :name: DMX_DQBUF
+
+
+Arguments
+=========
+
+``fd``
+    File descriptor returned by :ref:`open() <dmx_fopen>`.
+
+``argp``
+    Pointer to struct :c:type:`dmx_buffer`.
+
+
+Description
+===========
+
+Applications call the ``DMX_QBUF`` ioctl to enqueue an empty
+(capturing) or filled (output) buffer in the driver's incoming queue.
+The semantics depend on the selected I/O method.
+
+To enqueue a buffer applications set the ``index`` field. Valid index
+numbers range from zero to the number of buffers allocated with
+:ref:`DMX_REQBUFS` (struct :c:type:`dmx_requestbuffers` ``count``) minus
+one. The contents of the struct :c:type:`dmx_buffer` returned
+by a :ref:`DMX_QUERYBUF` ioctl will do as well.
+
+When ``DMX_QBUF`` is called with a pointer to this structure, it locks the
+memory pages of the buffer in physical memory, so they cannot be swapped
+out to disk. Buffers remain locked until dequeued, until the
+the device is closed.
+
+Applications call the ``DMX_DQBUF`` ioctl to dequeue a filled
+(capturing) buffer from the driver's outgoing queue.
+They just set the ``index`` field withe the buffer ID to be queued.
+When ``DMX_DQBUF`` is called with a pointer to struct :c:type:`dmx_buffer`,
+the driver fills the remaining fields or returns an error code.
+
+By default ``DMX_DQBUF`` blocks when no buffer is in the outgoing
+queue. When the ``O_NONBLOCK`` flag was given to the
+:ref:`open() <dmx_fopen>` function, ``DMX_DQBUF`` returns
+immediately with an ``EAGAIN`` error code when no buffer is available.
+
+The struct :c:type:`dmx_buffer` structure is specified in
+:ref:`buffer`.
+
+
+Return Value
+============
+
+On success 0 is returned, on error -1 and the ``errno`` variable is set
+appropriately. The generic error codes are described at the
+:ref:`Generic Error Codes <gen-errors>` chapter.
+
+EAGAIN
+    Non-blocking I/O has been selected using ``O_NONBLOCK`` and no
+    buffer was in the outgoing queue.
+
+EINVAL
+    The ``index`` is out of bounds, or no buffers have been allocated yet.
+
+EIO
+    ``DMX_DQBUF`` failed due to an internal error. Can also indicate
+    temporary problems like signal loss or CRC errors.
diff --git a/Documentation/media/uapi/dvb/dmx-querybuf.rst b/Documentation/media/uapi/dvb/dmx-querybuf.rst
new file mode 100644
index 000000000000..89481e24bb86
--- /dev/null
+++ b/Documentation/media/uapi/dvb/dmx-querybuf.rst
@@ -0,0 +1,63 @@
+.. _DMX_QUERYBUF:
+
+******************
+ioctl DMX_QUERYBUF
+******************
+
+Name
+====
+
+DMX_QUERYBUF - Query the status of a buffer
+
+.. warning:: this API is still experimental
+
+
+Synopsis
+========
+
+.. c:function:: int ioctl( int fd, DMX_QUERYBUF, struct dvb_buffer *argp )
+    :name: DMX_QUERYBUF
+
+
+Arguments
+=========
+
+``fd``
+    File descriptor returned by :ref:`open() <dmx_fopen>`.
+
+``argp``
+    Pointer to struct :c:type:`dvb_buffer`.
+
+
+Description
+===========
+
+This ioctl is part of the mmap streaming I/O method. It can
+be used to query the status of a buffer at any time after buffers have
+been allocated with the :ref:`DMX_REQBUFS` ioctl.
+
+Applications set the ``index`` field. Valid index numbers range from zero
+to the number of buffers allocated with :ref:`DMX_REQBUFS`
+(struct :c:type:`dvb_requestbuffers` ``count``) minus one.
+
+After calling :ref:`DMX_QUERYBUF` with a pointer to this structure,
+drivers return an error code or fill the rest of the structure.
+
+On success, the ``offset`` will contain the offset of the buffer from the
+start of the device memory, the ``length`` field its size, and the
+``bytesused`` the number of bytes occupied by data in the buffer (payload).
+
+Return Value
+============
+
+On success 0 is returned, the ``offset`` will contain the offset of the
+buffer from the start of the device memory, the ``length`` field its size,
+and the ``bytesused`` the number of bytes occupied by data in the buffer
+(payload).
+
+On error it returns -1 and the ``errno`` variable is set
+appropriately. The generic error codes are described at the
+:ref:`Generic Error Codes <gen-errors>` chapter.
+
+EINVAL
+    The ``index`` is out of bounds.
diff --git a/Documentation/media/uapi/dvb/dmx-reqbufs.rst b/Documentation/media/uapi/dvb/dmx-reqbufs.rst
new file mode 100644
index 000000000000..14b80d60bf35
--- /dev/null
+++ b/Documentation/media/uapi/dvb/dmx-reqbufs.rst
@@ -0,0 +1,74 @@
+.. _DMX_REQBUFS:
+
+*****************
+ioctl DMX_REQBUFS
+*****************
+
+Name
+====
+
+DMX_REQBUFS - Initiate Memory Mapping and/or DMA buffer I/O
+
+.. warning:: this API is still experimental
+
+
+Synopsis
+========
+
+.. c:function:: int ioctl( int fd, DMX_REQBUFS, struct dmx_requestbuffers *argp )
+    :name: DMX_REQBUFS
+
+
+Arguments
+=========
+
+``fd``
+    File descriptor returned by :ref:`open() <dmx_fopen>`.
+
+``argp``
+    Pointer to struct :c:type:`dmx_requestbuffers`.
+
+Description
+===========
+
+This ioctl is used to initiate a memory mapped or DMABUF based demux I/O.
+
+Memory mapped buffers are located in device memory and must be allocated
+with this ioctl before they can be mapped into the application's address
+space. User buffers are allocated by applications themselves, and this
+ioctl is merely used to switch the driver into user pointer I/O mode and
+to setup some internal structures. Similarly, DMABUF buffers are
+allocated by applications through a device driver, and this ioctl only
+configures the driver into DMABUF I/O mode without performing any direct
+allocation.
+
+To allocate device buffers applications initialize all fields of the
+struct :c:type:`dmx_requestbuffers` structure. They set the  ``count`` field
+to the desired number of buffers,  and ``size`` to the size of each
+buffer.
+
+When the ioctl is called with a pointer to this structure, the driver will
+attempt to allocate the requested number of buffers and it stores the actual
+number allocated in the ``count`` field. The ``count`` can be smaller than the number requested, even zero, when the driver runs out of free memory. A larger
+number is also possible when the driver requires more buffers to
+function correctly. The actual allocated buffer size can is returned
+at ``size``, and can be smaller than what's requested.
+
+When this I/O method is not supported, the ioctl returns an ``EOPNOTSUPP``
+error code.
+
+Applications can call :ref:`DMX_REQBUFS` again to change the number of
+buffers, however this cannot succeed when any buffers are still mapped.
+A ``count`` value of zero frees all buffers, after aborting or finishing
+any DMA in progress.
+
+
+Return Value
+============
+
+On success 0 is returned, on error -1 and the ``errno`` variable is set
+appropriately. The generic error codes are described at the
+:ref:`Generic Error Codes <gen-errors>` chapter.
+
+EOPNOTSUPP
+    The  the requested I/O method is not supported.
diff --git a/Documentation/media/uapi/dvb/dmx_fcalls.rst b/Documentation/media/uapi/dvb/dmx_fcalls.rst
index a17289143220..4c391cf2554f 100644
--- a/Documentation/media/uapi/dvb/dmx_fcalls.rst
+++ b/Documentation/media/uapi/dvb/dmx_fcalls.rst
@@ -13,6 +13,8 @@ Demux Function Calls
     dmx-fclose
     dmx-fread
     dmx-fwrite
+    dmx-mmap
+    dmx-munmap
     dmx-start
     dmx-stop
     dmx-set-filter
@@ -22,3 +24,7 @@ Demux Function Calls
     dmx-get-pes-pids
     dmx-add-pid
     dmx-remove-pid
+    dmx-reqbufs
+    dmx-querybuf
+    dmx-expbuf
+    dmx-qbuf
diff --git a/Documentation/media/uapi/dvb/fe_property_parameters.rst b/Documentation/media/uapi/dvb/fe_property_parameters.rst
index 6eef507fea50..3524dcae4604 100644
--- a/Documentation/media/uapi/dvb/fe_property_parameters.rst
+++ b/Documentation/media/uapi/dvb/fe_property_parameters.rst
@@ -987,3 +987,21 @@ Possible values: 0, 1, LNA_AUTO
 1, LNA on
 
 use the special macro LNA_AUTO to set LNA auto
+
+
+.. _DTV-SCRAMBLING-SEQUENCE-INDEX:
+
+DTV_SCRAMBLING_SEQUENCE_INDEX
+=============================
+
+Used on DVB-S2.
+
+This 18 bit field, when present, carries the index of the DVB-S2 physical
+layer scrambling sequence as defined in clause 5.5.4 of EN 302 307.
+There is no explicit signalling method to convey scrambling sequence index
+to the receiver. If S2 satellite delivery system descriptor is available
+it can be used to read the scrambling sequence index (EN 300 468 table 41).
+
+By default, gold scrambling sequence index 0 is used.
+
+The valid scrambling sequence index range is from 0 to 262142.
diff --git a/Documentation/media/uapi/dvb/frontend-property-satellite-systems.rst b/Documentation/media/uapi/dvb/frontend-property-satellite-systems.rst
index 1f40399c68ff..2929e6999a7a 100644
--- a/Documentation/media/uapi/dvb/frontend-property-satellite-systems.rst
+++ b/Documentation/media/uapi/dvb/frontend-property-satellite-systems.rst
@@ -60,6 +60,8 @@ following parameters:
 
 -  :ref:`DTV_STREAM_ID <DTV-STREAM-ID>`
 
+-  :ref:`DTV_SCRAMBLING_SEQUENCE_INDEX <DTV-SCRAMBLING-SEQUENCE-INDEX>`
+
 In addition, the :ref:`DTV QoS statistics <frontend-stat-properties>`
 are also valid.
 
diff --git a/Documentation/media/uapi/rc/lirc-dev-intro.rst b/Documentation/media/uapi/rc/lirc-dev-intro.rst
index d1936eeb9ce0..3a74fec66d69 100644
--- a/Documentation/media/uapi/rc/lirc-dev-intro.rst
+++ b/Documentation/media/uapi/rc/lirc-dev-intro.rst
@@ -6,11 +6,12 @@
 Introduction
 ************
 
-The LIRC device interface is a bi-directional interface for transporting
-raw IR data between userspace and kernelspace. Fundamentally, it is just
-a chardev (/dev/lircX, for X = 0, 1, 2, ...), with a number of standard
-struct file_operations defined on it. With respect to transporting raw
-IR data to and fro, the essential fops are read, write and ioctl.
+LIRC stands for Linux Infrared Remote Control. The LIRC device interface is
+a bi-directional interface for transporting raw IR and decoded scancodes
+data between userspace and kernelspace. Fundamentally, it is just a chardev
+(/dev/lircX, for X = 0, 1, 2, ...), with a number of standard struct
+file_operations defined on it. With respect to transporting raw IR and
+decoded scancodes to and fro, the essential fops are read, write and ioctl.
 
 Example dmesg output upon a driver registering w/LIRC:
 
@@ -18,7 +19,7 @@ Example dmesg output upon a driver registering w/LIRC:
 
     $ dmesg |grep lirc_dev
     lirc_dev: IR Remote Control driver registered, major 248
-    rc rc0: lirc_dev: driver ir-lirc-codec (mceusb) registered at minor = 0
+    rc rc0: lirc_dev: driver mceusb registered at minor = 0
 
 What you should see for a chardev:
 
@@ -36,6 +37,43 @@ LIRC modes
 LIRC supports some modes of receiving and sending IR codes, as shown
 on the following table.
 
+.. _lirc-mode-scancode:
+.. _lirc-scancode-flag-toggle:
+.. _lirc-scancode-flag-repeat:
+
+``LIRC_MODE_SCANCODE``
+
+    This mode is for both sending and receiving IR.
+
+    For transmitting (aka sending), create a ``struct lirc_scancode`` with
+    the desired scancode set in the ``scancode`` member, :c:type:`rc_proto`
+    set the IR protocol, and all other members set to 0. Write this struct to
+    the lirc device.
+
+    For receiving, you read ``struct lirc_scancode`` from the lirc device,
+    with ``scancode`` set to the received scancode and the IR protocol
+    :c:type:`rc_proto`. If the scancode maps to a valid key code, this is set
+    in the ``keycode`` field, else it is set to ``KEY_RESERVED``.
+
+    The ``flags`` can have ``LIRC_SCANCODE_FLAG_TOGGLE`` set if the toggle
+    bit is set in protocols that support it (e.g. rc-5 and rc-6), or
+    ``LIRC_SCANCODE_FLAG_REPEAT`` for when a repeat is received for protocols
+    that support it (e.g. nec).
+
+    In the Sanyo and NEC protocol, if you hold a button on remote, rather than
+    repeating the entire scancode, the remote sends a shorter message with
+    no scancode, which just means button is held, a "repeat". When this is
+    received, the ``LIRC_SCANCODE_FLAG_REPEAT`` is set and the scancode and
+    keycode is repeated.
+
+    With nec, there is no way to distinguish "button hold" from "repeatedly
+    pressing the same button". The rc-5 and rc-6 protocols have a toggle bit.
+    When a button is released and pressed again, the toggle bit is inverted.
+    If the toggle bit is set, the ``LIRC_SCANCODE_FLAG_TOGGLE`` is set.
+
+    The ``timestamp`` field is filled with the time nanoseconds
+    (in ``CLOCK_MONOTONIC``) when the scancode was decoded.
+
 .. _lirc-mode-mode2:
 
 ``LIRC_MODE_MODE2``
@@ -72,21 +110,6 @@ on the following table.
         this packet will be sent, with the number of microseconds with
         no IR.
 
-.. _lirc-mode-lirccode:
-
-``LIRC_MODE_LIRCCODE``
-
-    This mode can be used for IR receive and send.
-
-    The IR signal is decoded internally by the receiver, or encoded by the
-    transmitter. The LIRC interface represents the scancode as byte string,
-    which might not be a u32, it can be any length. The value is entirely
-    driver dependent. This mode is used by some older lirc drivers.
-
-    The length of each code depends on the driver, which can be retrieved
-    with :ref:`lirc_get_length`. This length is used both
-    for transmitting and receiving IR.
-
 .. _lirc-mode-pulse:
 
 ``LIRC_MODE_PULSE``
@@ -99,3 +122,13 @@ on the following table.
     of entries.
 
     This mode is used only for IR send.
+
+
+**************************
+Remote Controller protocol
+**************************
+
+An enum :c:type:`rc_proto` in the :ref:`lirc_header` lists all the
+supported IR protocols:
+
+.. kernel-doc:: include/uapi/linux/lirc.h
diff --git a/Documentation/media/uapi/rc/lirc-func.rst b/Documentation/media/uapi/rc/lirc-func.rst
index 9b5a772ec96c..ddb4620de294 100644
--- a/Documentation/media/uapi/rc/lirc-func.rst
+++ b/Documentation/media/uapi/rc/lirc-func.rst
@@ -18,7 +18,6 @@ LIRC Function Reference
     lirc-set-send-duty-cycle
     lirc-get-timeout
     lirc-set-rec-timeout
-    lirc-get-length
     lirc-set-rec-carrier
     lirc-set-rec-carrier-range
     lirc-set-send-carrier
diff --git a/Documentation/media/uapi/rc/lirc-get-features.rst b/Documentation/media/uapi/rc/lirc-get-features.rst
index 64f89a4f9d9c..889a8807037b 100644
--- a/Documentation/media/uapi/rc/lirc-get-features.rst
+++ b/Documentation/media/uapi/rc/lirc-get-features.rst
@@ -55,15 +55,24 @@ LIRC features
 
 ``LIRC_CAN_REC_MODE2``
 
-    The driver is capable of receiving using
-    :ref:`LIRC_MODE_MODE2 <lirc-mode-MODE2>`.
+    This is raw IR driver for receiving. This means that
+    :ref:`LIRC_MODE_MODE2 <lirc-mode-MODE2>` is used. This also implies
+    that :ref:`LIRC_MODE_SCANCODE <lirc-mode-SCANCODE>` is also supported,
+    as long as the kernel is recent enough. Use the
+    :ref:`lirc_set_rec_mode` to switch modes.
 
 .. _LIRC-CAN-REC-LIRCCODE:
 
 ``LIRC_CAN_REC_LIRCCODE``
 
-    The driver is capable of receiving using
-    :ref:`LIRC_MODE_LIRCCODE <lirc-mode-LIRCCODE>`.
+    Unused. Kept just to avoid breaking uAPI.
+
+.. _LIRC-CAN-REC-SCANCODE:
+
+``LIRC_CAN_REC_SCANCODE``
+
+    This is a scancode driver for receiving. This means that
+    :ref:`LIRC_MODE_SCANCODE <lirc-mode-SCANCODE>` is used.
 
 .. _LIRC-CAN-SET-SEND-CARRIER:
 
@@ -157,7 +166,10 @@ LIRC features
 ``LIRC_CAN_SEND_PULSE``
 
     The driver supports sending (also called as IR blasting or IR TX) using
-    :ref:`LIRC_MODE_PULSE <lirc-mode-pulse>`.
+    :ref:`LIRC_MODE_PULSE <lirc-mode-pulse>`. This implies that
+    :ref:`LIRC_MODE_SCANCODE <lirc-mode-SCANCODE>` is also supported for
+    transmit, as long as the kernel is recent enough. Use the
+    :ref:`lirc_set_send_mode` to switch modes.
 
 .. _LIRC-CAN-SEND-MODE2:
 
@@ -170,8 +182,7 @@ LIRC features
 
 ``LIRC_CAN_SEND_LIRCCODE``
 
-    The driver supports sending (also called as IR blasting or IR TX) using
-    :ref:`LIRC_MODE_LIRCCODE <lirc-mode-LIRCCODE>`.
+    Unused. Kept just to avoid breaking uAPI.
 
 
 Return Value
diff --git a/Documentation/media/uapi/rc/lirc-get-length.rst b/Documentation/media/uapi/rc/lirc-get-length.rst
deleted file mode 100644
index 3990af5de0e9..000000000000
--- a/Documentation/media/uapi/rc/lirc-get-length.rst
+++ /dev/null
@@ -1,44 +0,0 @@
-.. -*- coding: utf-8; mode: rst -*-
-
-.. _lirc_get_length:
-
-*********************
-ioctl LIRC_GET_LENGTH
-*********************
-
-Name
-====
-
-LIRC_GET_LENGTH - Retrieves the code length in bits.
-
-Synopsis
-========
-
-.. c:function:: int ioctl( int fd, LIRC_GET_LENGTH, __u32 *length )
-    :name: LIRC_GET_LENGTH
-
-Arguments
-=========
-
-``fd``
-    File descriptor returned by open().
-
-``length``
-    length, in bits
-
-
-Description
-===========
-
-Retrieves the code length in bits (only for
-:ref:`LIRC_MODE_LIRCCODE <lirc-mode-lirccode>`).
-Reads on the device must be done in blocks matching the bit count.
-The bit could should be rounded up so that it matches full bytes.
-
-
-Return Value
-============
-
-On success 0 is returned, on error -1 and the ``errno`` variable is set
-appropriately. The generic error codes are described at the
-:ref:`Generic Error Codes <gen-errors>` chapter.
diff --git a/Documentation/media/uapi/rc/lirc-get-rec-mode.rst b/Documentation/media/uapi/rc/lirc-get-rec-mode.rst
index a4eb6c0a26e9..2722118484fa 100644
--- a/Documentation/media/uapi/rc/lirc-get-rec-mode.rst
+++ b/Documentation/media/uapi/rc/lirc-get-rec-mode.rst
@@ -10,15 +10,15 @@ ioctls LIRC_GET_REC_MODE and LIRC_SET_REC_MODE
 Name
 ====
 
-LIRC_GET_REC_MODE/LIRC_SET_REC_MODE - Get/set supported receive modes.
+LIRC_GET_REC_MODE/LIRC_SET_REC_MODE - Get/set current receive mode.
 
 Synopsis
 ========
 
-.. c:function:: int ioctl( int fd, LIRC_GET_REC_MODE, __u32 rx_modes)
+.. c:function:: int ioctl( int fd, LIRC_GET_REC_MODE, __u32 *mode)
 	:name: LIRC_GET_REC_MODE
 
-.. c:function:: int ioctl( int fd, LIRC_SET_REC_MODE, __u32 rx_modes)
+.. c:function:: int ioctl( int fd, LIRC_SET_REC_MODE, __u32 *mode)
 	:name: LIRC_SET_REC_MODE
 
 Arguments
@@ -27,20 +27,41 @@ Arguments
 ``fd``
     File descriptor returned by open().
 
-``rx_modes``
-    Bitmask with the supported transmit modes.
+``mode``
+    Mode used for receive.
 
 Description
 ===========
 
-Get/set supported receive modes. Only :ref:`LIRC_MODE_MODE2 <lirc-mode-mode2>`
-and :ref:`LIRC_MODE_LIRCCODE <lirc-mode-lirccode>` are supported for IR
-receive. Use :ref:`lirc_get_features` to find out which modes the driver
-supports.
+Get and set the current receive mode. Only
+:ref:`LIRC_MODE_MODE2 <lirc-mode-mode2>` and
+:ref:`LIRC_MODE_SCANCODE <lirc-mode-scancode>` are supported.
+Use :ref:`lirc_get_features` to find out which modes the driver supports.
 
 Return Value
 ============
 
-On success 0 is returned, on error -1 and the ``errno`` variable is set
-appropriately. The generic error codes are described at the
-:ref:`Generic Error Codes <gen-errors>` chapter.
+.. tabularcolumns:: |p{2.5cm}|p{15.0cm}|
+
+.. flat-table::
+    :header-rows:  0
+    :stub-columns: 0
+
+
+    -  .. row 1
+
+       -  ``ENODEV``
+
+       -  Device not available.
+
+    -  .. row 2
+
+       -  ``ENOTTY``
+
+       -  Device does not support receiving.
+
+    -  .. row 3
+
+       -  ``EINVAL``
+
+       -  Invalid mode or invalid mode for this device.
diff --git a/Documentation/media/uapi/rc/lirc-get-send-mode.rst b/Documentation/media/uapi/rc/lirc-get-send-mode.rst
index a169b234290e..c44e61a79ad1 100644
--- a/Documentation/media/uapi/rc/lirc-get-send-mode.rst
+++ b/Documentation/media/uapi/rc/lirc-get-send-mode.rst
@@ -10,15 +10,15 @@ ioctls LIRC_GET_SEND_MODE and LIRC_SET_SEND_MODE
 Name
 ====
 
-LIRC_GET_SEND_MODE/LIRC_SET_SEND_MODE - Get/set supported transmit mode.
+LIRC_GET_SEND_MODE/LIRC_SET_SEND_MODE - Get/set current transmit mode.
 
 Synopsis
 ========
 
-.. c:function:: int ioctl( int fd, LIRC_GET_SEND_MODE, __u32 *tx_modes )
+.. c:function:: int ioctl( int fd, LIRC_GET_SEND_MODE, __u32 *mode )
     :name: LIRC_GET_SEND_MODE
 
-.. c:function:: int ioctl( int fd, LIRC_SET_SEND_MODE, __u32 *tx_modes )
+.. c:function:: int ioctl( int fd, LIRC_SET_SEND_MODE, __u32 *mode )
     :name: LIRC_SET_SEND_MODE
 
 Arguments
@@ -27,8 +27,8 @@ Arguments
 ``fd``
     File descriptor returned by open().
 
-``tx_modes``
-    Bitmask with the supported transmit modes.
+``mode``
+    The mode used for transmitting.
 
 
 Description
@@ -37,13 +37,35 @@ Description
 Get/set current transmit mode.
 
 Only :ref:`LIRC_MODE_PULSE <lirc-mode-pulse>` and
-:ref:`LIRC_MODE_LIRCCODE <lirc-mode-lirccode>` is supported by for IR send,
+:ref:`LIRC_MODE_SCANCODE <lirc-mode-scancode>` are supported by for IR send,
 depending on the driver. Use :ref:`lirc_get_features` to find out which
 modes the driver supports.
 
 Return Value
 ============
 
-On success 0 is returned, on error -1 and the ``errno`` variable is set
-appropriately. The generic error codes are described at the
-:ref:`Generic Error Codes <gen-errors>` chapter.
+
+.. tabularcolumns:: |p{2.5cm}|p{15.0cm}|
+
+.. flat-table::
+    :header-rows:  0
+    :stub-columns: 0
+
+
+    -  .. row 1
+
+       -  ``ENODEV``
+
+       -  Device not available.
+
+    -  .. row 2
+
+       -  ``ENOTTY``
+
+       -  Device does not support transmitting.
+
+    -  .. row 3
+
+       -  ``EINVAL``
+
+       -  Invalid mode or invalid mode for this device.
diff --git a/Documentation/media/uapi/rc/lirc-read.rst b/Documentation/media/uapi/rc/lirc-read.rst
index ff14a69104e5..c024aaffb8ad 100644
--- a/Documentation/media/uapi/rc/lirc-read.rst
+++ b/Documentation/media/uapi/rc/lirc-read.rst
@@ -45,13 +45,20 @@ descriptor ``fd`` into the buffer starting at ``buf``.  If ``count`` is zero,
 is greater than ``SSIZE_MAX``, the result is unspecified.
 
 The exact format of the data depends on what :ref:`lirc_modes` a driver
-uses. Use :ref:`lirc_get_features` to get the supported mode.
+uses. Use :ref:`lirc_get_features` to get the supported mode, and use
+:ref:`lirc_set_rec_mode` set the current active mode.
 
-The generally preferred mode for receive is
-:ref:`LIRC_MODE_MODE2 <lirc-mode-mode2>`,
-in which packets containing an int value describing an IR signal are
+The mode :ref:`LIRC_MODE_MODE2 <lirc-mode-mode2>` is for raw IR,
+in which packets containing an unsigned int value describing an IR signal are
 read from the chardev.
 
+Alternatively, :ref:`LIRC_MODE_SCANCODE <lirc-mode-scancode>` can be available,
+in this mode scancodes which are either decoded by software decoders, or
+by hardware decoders. The :c:type:`rc_proto` member is set to the
+protocol used for transmission, and ``scancode`` to the decoded scancode,
+and the ``keycode`` set to the keycode or ``KEY_RESERVED``.
+
+
 Return Value
 ============
 
diff --git a/Documentation/media/uapi/rc/lirc-write.rst b/Documentation/media/uapi/rc/lirc-write.rst
index 2aad0fef4a5b..d4566b0a2015 100644
--- a/Documentation/media/uapi/rc/lirc-write.rst
+++ b/Documentation/media/uapi/rc/lirc-write.rst
@@ -42,21 +42,32 @@ Description
 referenced by the file descriptor ``fd`` from the buffer starting at
 ``buf``.
 
-The exact format of the data depends on what mode a driver uses, use
-:ref:`lirc_get_features` to get the supported mode.
+The exact format of the data depends on what mode a driver is in, use
+:ref:`lirc_get_features` to get the supported modes and use
+:ref:`lirc_set_send_mode` set the mode.
 
 When in :ref:`LIRC_MODE_PULSE <lirc-mode-PULSE>` mode, the data written to
 the chardev is a pulse/space sequence of integer values. Pulses and spaces
 are only marked implicitly by their position. The data must start and end
 with a pulse, therefore, the data must always include an uneven number of
-samples. The write function must block until the data has been transmitted
+samples. The write function blocks until the data has been transmitted
 by the hardware. If more data is provided than the hardware can send, the
 driver returns ``EINVAL``.
 
+When in :ref:`LIRC_MODE_SCANCODE <lirc-mode-scancode>` mode, one
+``struct lirc_scancode`` must be written to the chardev at a time, else
+``EINVAL`` is returned. Set the desired scancode in the ``scancode`` member,
+and the protocol in the :c:type:`rc_proto`: member. All other members must be
+set to 0, else ``EINVAL`` is returned. If there is no protocol encoder
+for the protocol or the scancode is not valid for the specified protocol,
+``EINVAL`` is returned. The write function blocks until the scancode
+is transmitted by the hardware.
+
+
 Return Value
 ============
 
-On success, the number of bytes read is returned. It is not an error if
+On success, the number of bytes written is returned. It is not an error if
 this number is smaller than the number of bytes requested, or the amount
 of data required for one frame.  On error, -1 is returned, and the ``errno``
 variable is set appropriately. The generic error codes are described at the
diff --git a/Documentation/media/uapi/v4l/meta-formats.rst b/Documentation/media/uapi/v4l/meta-formats.rst
index 01e24e3df571..0c4e1ecf5879 100644
--- a/Documentation/media/uapi/v4l/meta-formats.rst
+++ b/Documentation/media/uapi/v4l/meta-formats.rst
@@ -12,5 +12,6 @@ These formats are used for the :ref:`metadata` interface only.
 .. toctree::
     :maxdepth: 1
 
+    pixfmt-meta-uvc
     pixfmt-meta-vsp1-hgo
     pixfmt-meta-vsp1-hgt
diff --git a/Documentation/media/uapi/v4l/pixfmt-meta-uvc.rst b/Documentation/media/uapi/v4l/pixfmt-meta-uvc.rst
new file mode 100644
index 000000000000..b5165dc090c2
--- /dev/null
+++ b/Documentation/media/uapi/v4l/pixfmt-meta-uvc.rst
@@ -0,0 +1,51 @@
+.. -*- coding: utf-8; mode: rst -*-
+
+.. _v4l2-meta-fmt-uvc:
+
+*******************************
+V4L2_META_FMT_UVC ('UVCH')
+*******************************
+
+UVC Payload Header Data
+
+
+Description
+===========
+
+This format describes standard UVC metadata, extracted from UVC packet headers
+and provided by the UVC driver through metadata video nodes. That data includes
+exact copies of the standard part of UVC Payload Header contents and auxiliary
+timing information, required for precise interpretation of timestamps, contained
+in those headers. See section "2.4.3.3 Video and Still Image Payload Headers" of
+the "UVC 1.5 Class specification" for details.
+
+Each UVC payload header can be between 2 and 12 bytes large. Buffers can
+contain multiple headers, if multiple such headers have been transmitted by the
+camera for the respective frame. However, the driver may drop headers when the
+buffer is full, when they contain no useful information (e.g. those without the
+SCR field or with that field identical to the previous header), or generally to
+perform rate limiting when the device sends a large number of headers.
+
+Each individual block contains the following fields:
+
+.. flat-table:: UVC Metadata Block
+    :widths: 1 4
+    :header-rows:  1
+    :stub-columns: 0
+
+    * - Field
+      - Description
+    * - __u64 ts;
+      - system timestamp in host byte order, measured by the driver upon
+        reception of the payload
+    * - __u16 sof;
+      - USB Frame Number in host byte order, also obtained by the driver as
+        close as possible to the above timestamp to enable correlation between
+        them
+    * - :cspan:`1` *The rest is an exact copy of the UVC payload header:*
+    * - __u8 length;
+      - length of the rest of the block, including this field
+    * - __u8 flags;
+      - Flags, indicating presence of other standard UVC fields
+    * - __u8 buf[];
+      - The rest of the header, possibly including UVC PTS and SCR fields
diff --git a/Documentation/media/uapi/v4l/pixfmt-rgb.rst b/Documentation/media/uapi/v4l/pixfmt-rgb.rst
index 4cc27195dc79..cf2ef7df9616 100644
--- a/Documentation/media/uapi/v4l/pixfmt-rgb.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-rgb.rst
@@ -16,6 +16,7 @@ RGB Formats
     pixfmt-srggb10p
     pixfmt-srggb10alaw8
     pixfmt-srggb10dpcm8
+    pixfmt-srggb10-ipu3
     pixfmt-srggb12
     pixfmt-srggb12p
     pixfmt-srggb16
diff --git a/Documentation/media/uapi/v4l/pixfmt-srggb10-ipu3.rst b/Documentation/media/uapi/v4l/pixfmt-srggb10-ipu3.rst
new file mode 100644
index 000000000000..99cde5077519
--- /dev/null
+++ b/Documentation/media/uapi/v4l/pixfmt-srggb10-ipu3.rst
@@ -0,0 +1,335 @@
+.. -*- coding: utf-8; mode: rst -*-
+
+.. _v4l2-pix-fmt-ipu3-sbggr10:
+.. _v4l2-pix-fmt-ipu3-sgbrg10:
+.. _v4l2-pix-fmt-ipu3-sgrbg10:
+.. _v4l2-pix-fmt-ipu3-srggb10:
+
+**********************************************************************************************************************************************
+V4L2_PIX_FMT_IPU3_SBGGR10 ('ip3b'), V4L2_PIX_FMT_IPU3_SGBRG10 ('ip3g'), V4L2_PIX_FMT_IPU3_SGRBG10 ('ip3G'), V4L2_PIX_FMT_IPU3_SRGGB10 ('ip3r')
+**********************************************************************************************************************************************
+
+10-bit Bayer formats
+
+Description
+===========
+
+These four pixel formats are used by Intel IPU3 driver, they are raw
+sRGB / Bayer formats with 10 bits per sample with every 25 pixels packed
+to 32 bytes leaving 6 most significant bits padding in the last byte.
+The format is little endian.
+
+In other respects this format is similar to :ref:`V4L2-PIX-FMT-SRGGB10`.
+Below is an example of a small image in V4L2_PIX_FMT_IPU3_SBGGR10 format.
+
+**Byte Order.**
+Each cell is one byte.
+
+.. tabularcolumns:: |p{0.8cm}|p{4.0cm}|p{4.0cm}|p{4.0cm}|p{4.0cm}|
+
+.. flat-table::
+
+    * - start + 0:
+      - B\ :sub:`0000low`
+      - G\ :sub:`0001low`\ (bits 7--2)
+
+        B\ :sub:`0000high`\ (bits 1--0)
+      - B\ :sub:`0002low`\ (bits 7--4)
+
+        G\ :sub:`0001high`\ (bits 3--0)
+      - G\ :sub:`0003low`\ (bits 7--6)
+
+        B\ :sub:`0002high`\ (bits 5--0)
+    * - start + 4:
+      - G\ :sub:`0003high`
+      - B\ :sub:`0004low`
+      - G\ :sub:`0005low`\ (bits 7--2)
+
+        B\ :sub:`0004high`\ (bits 1--0)
+      - B\ :sub:`0006low`\ (bits 7--4)
+
+        G\ :sub:`0005high`\ (bits 3--0)
+    * - start + 8:
+      - G\ :sub:`0007low`\ (bits 7--6)
+
+        B\ :sub:`0006high`\ (bits 5--0)
+      - G\ :sub:`0007high`
+      - B\ :sub:`0008low`
+      - G\ :sub:`0009low`\ (bits 7--2)
+
+        B\ :sub:`0008high`\ (bits 1--0)
+    * - start + 12:
+      - B\ :sub:`0010low`\ (bits 7--4)
+
+        G\ :sub:`0009high`\ (bits 3--0)
+      - G\ :sub:`0011low`\ (bits 7--6)
+
+        B\ :sub:`0010high`\ (bits 5--0)
+      - G\ :sub:`0011high`
+      - B\ :sub:`0012low`
+    * - start + 16:
+      - G\ :sub:`0013low`\ (bits 7--2)
+
+        B\ :sub:`0012high`\ (bits 1--0)
+      - B\ :sub:`0014low`\ (bits 7--4)
+
+        G\ :sub:`0013high`\ (bits 3--0)
+      - G\ :sub:`0015low`\ (bits 7--6)
+
+        B\ :sub:`0014high`\ (bits 5--0)
+      - G\ :sub:`0015high`
+    * - start + 20
+      - B\ :sub:`0016low`
+      - G\ :sub:`0017low`\ (bits 7--2)
+
+        B\ :sub:`0016high`\ (bits 1--0)
+      - B\ :sub:`0018low`\ (bits 7--4)
+
+        G\ :sub:`0017high`\ (bits 3--0)
+      - G\ :sub:`0019low`\ (bits 7--6)
+
+        B\ :sub:`0018high`\ (bits 5--0)
+    * - start + 24:
+      - G\ :sub:`0019high`
+      - B\ :sub:`0020low`
+      - G\ :sub:`0021low`\ (bits 7--2)
+
+        B\ :sub:`0020high`\ (bits 1--0)
+      - B\ :sub:`0022low`\ (bits 7--4)
+
+        G\ :sub:`0021high`\ (bits 3--0)
+    * - start + 28:
+      - G\ :sub:`0023low`\ (bits 7--6)
+
+        B\ :sub:`0022high`\ (bits 5--0)
+      - G\ :sub:`0023high`
+      - B\ :sub:`0024low`
+      - B\ :sub:`0024high`\ (bits 1--0)
+    * - start + 32:
+      - G\ :sub:`0100low`
+      - R\ :sub:`0101low`\ (bits 7--2)
+
+        G\ :sub:`0100high`\ (bits 1--0)
+      - G\ :sub:`0102low`\ (bits 7--4)
+
+        R\ :sub:`0101high`\ (bits 3--0)
+      - R\ :sub:`0103low`\ (bits 7--6)
+
+        G\ :sub:`0102high`\ (bits 5--0)
+    * - start + 36:
+      - R\ :sub:`0103high`
+      - G\ :sub:`0104low`
+      - R\ :sub:`0105low`\ (bits 7--2)
+
+        G\ :sub:`0104high`\ (bits 1--0)
+      - G\ :sub:`0106low`\ (bits 7--4)
+
+        R\ :sub:`0105high`\ (bits 3--0)
+    * - start + 40:
+      - R\ :sub:`0107low`\ (bits 7--6)
+
+        G\ :sub:`0106high`\ (bits 5--0)
+      - R\ :sub:`0107high`
+      - G\ :sub:`0108low`
+      - R\ :sub:`0109low`\ (bits 7--2)
+
+        G\ :sub:`0108high`\ (bits 1--0)
+    * - start + 44:
+      - G\ :sub:`0110low`\ (bits 7--4)
+
+        R\ :sub:`0109high`\ (bits 3--0)
+      - R\ :sub:`0111low`\ (bits 7--6)
+
+        G\ :sub:`0110high`\ (bits 5--0)
+      - R\ :sub:`0111high`
+      - G\ :sub:`0112low`
+    * - start + 48:
+      - R\ :sub:`0113low`\ (bits 7--2)
+
+        G\ :sub:`0112high`\ (bits 1--0)
+      - G\ :sub:`0114low`\ (bits 7--4)
+
+        R\ :sub:`0113high`\ (bits 3--0)
+      - R\ :sub:`0115low`\ (bits 7--6)
+
+        G\ :sub:`0114high`\ (bits 5--0)
+      - R\ :sub:`0115high`
+    * - start + 52:
+      - G\ :sub:`0116low`
+      - R\ :sub:`0117low`\ (bits 7--2)
+
+        G\ :sub:`0116high`\ (bits 1--0)
+      - G\ :sub:`0118low`\ (bits 7--4)
+
+        R\ :sub:`0117high`\ (bits 3--0)
+      - R\ :sub:`0119low`\ (bits 7--6)
+
+        G\ :sub:`0118high`\ (bits 5--0)
+    * - start + 56:
+      - R\ :sub:`0119high`
+      - G\ :sub:`0120low`
+      - R\ :sub:`0121low`\ (bits 7--2)
+
+        G\ :sub:`0120high`\ (bits 1--0)
+      - G\ :sub:`0122low`\ (bits 7--4)
+
+        R\ :sub:`0121high`\ (bits 3--0)
+    * - start + 60:
+      - R\ :sub:`0123low`\ (bits 7--6)
+
+        G\ :sub:`0122high`\ (bits 5--0)
+      - R\ :sub:`0123high`
+      - G\ :sub:`0124low`
+      - G\ :sub:`0124high`\ (bits 1--0)
+    * - start + 64:
+      - B\ :sub:`0200low`
+      - G\ :sub:`0201low`\ (bits 7--2)
+
+        B\ :sub:`0200high`\ (bits 1--0)
+      - B\ :sub:`0202low`\ (bits 7--4)
+
+        G\ :sub:`0201high`\ (bits 3--0)
+      - G\ :sub:`0203low`\ (bits 7--6)
+
+        B\ :sub:`0202high`\ (bits 5--0)
+    * - start + 68:
+      - G\ :sub:`0203high`
+      - B\ :sub:`0204low`
+      - G\ :sub:`0205low`\ (bits 7--2)
+
+        B\ :sub:`0204high`\ (bits 1--0)
+      - B\ :sub:`0206low`\ (bits 7--4)
+
+        G\ :sub:`0205high`\ (bits 3--0)
+    * - start + 72:
+      - G\ :sub:`0207low`\ (bits 7--6)
+
+        B\ :sub:`0206high`\ (bits 5--0)
+      - G\ :sub:`0207high`
+      - B\ :sub:`0208low`
+      - G\ :sub:`0209low`\ (bits 7--2)
+
+        B\ :sub:`0208high`\ (bits 1--0)
+    * - start + 76:
+      - B\ :sub:`0210low`\ (bits 7--4)
+
+        G\ :sub:`0209high`\ (bits 3--0)
+      - G\ :sub:`0211low`\ (bits 7--6)
+
+        B\ :sub:`0210high`\ (bits 5--0)
+      - G\ :sub:`0211high`
+      - B\ :sub:`0212low`
+    * - start + 80:
+      - G\ :sub:`0213low`\ (bits 7--2)
+
+        B\ :sub:`0212high`\ (bits 1--0)
+      - B\ :sub:`0214low`\ (bits 7--4)
+
+        G\ :sub:`0213high`\ (bits 3--0)
+      - G\ :sub:`0215low`\ (bits 7--6)
+
+        B\ :sub:`0214high`\ (bits 5--0)
+      - G\ :sub:`0215high`
+    * - start + 84:
+      - B\ :sub:`0216low`
+      - G\ :sub:`0217low`\ (bits 7--2)
+
+        B\ :sub:`0216high`\ (bits 1--0)
+      - B\ :sub:`0218low`\ (bits 7--4)
+
+        G\ :sub:`0217high`\ (bits 3--0)
+      - G\ :sub:`0219low`\ (bits 7--6)
+
+        B\ :sub:`0218high`\ (bits 5--0)
+    * - start + 88:
+      - G\ :sub:`0219high`
+      - B\ :sub:`0220low`
+      - G\ :sub:`0221low`\ (bits 7--2)
+
+        B\ :sub:`0220high`\ (bits 1--0)
+      - B\ :sub:`0222low`\ (bits 7--4)
+
+        G\ :sub:`0221high`\ (bits 3--0)
+    * - start + 92:
+      - G\ :sub:`0223low`\ (bits 7--6)
+
+        B\ :sub:`0222high`\ (bits 5--0)
+      - G\ :sub:`0223high`
+      - B\ :sub:`0224low`
+      - B\ :sub:`0224high`\ (bits 1--0)
+    * - start + 96:
+      - G\ :sub:`0300low`
+      - R\ :sub:`0301low`\ (bits 7--2)
+
+        G\ :sub:`0300high`\ (bits 1--0)
+      - G\ :sub:`0302low`\ (bits 7--4)
+
+        R\ :sub:`0301high`\ (bits 3--0)
+      - R\ :sub:`0303low`\ (bits 7--6)
+
+        G\ :sub:`0302high`\ (bits 5--0)
+    * - start + 100:
+      - R\ :sub:`0303high`
+      - G\ :sub:`0304low`
+      - R\ :sub:`0305low`\ (bits 7--2)
+
+        G\ :sub:`0304high`\ (bits 1--0)
+      - G\ :sub:`0306low`\ (bits 7--4)
+
+        R\ :sub:`0305high`\ (bits 3--0)
+    * - start + 104:
+      - R\ :sub:`0307low`\ (bits 7--6)
+
+        G\ :sub:`0306high`\ (bits 5--0)
+      - R\ :sub:`0307high`
+      - G\ :sub:`0308low`
+      - R\ :sub:`0309low`\ (bits 7--2)
+
+        G\ :sub:`0308high`\ (bits 1--0)
+    * - start + 108:
+      - G\ :sub:`0310low`\ (bits 7--4)
+
+        R\ :sub:`0309high`\ (bits 3--0)
+      - R\ :sub:`0311low`\ (bits 7--6)
+
+        G\ :sub:`0310high`\ (bits 5--0)
+      - R\ :sub:`0311high`
+      - G\ :sub:`0312low`
+    * - start + 112:
+      - R\ :sub:`0313low`\ (bits 7--2)
+
+        G\ :sub:`0312high`\ (bits 1--0)
+      - G\ :sub:`0314low`\ (bits 7--4)
+
+        R\ :sub:`0313high`\ (bits 3--0)
+      - R\ :sub:`0315low`\ (bits 7--6)
+
+        G\ :sub:`0314high`\ (bits 5--0)
+      - R\ :sub:`0315high`
+    * - start + 116:
+      - G\ :sub:`0316low`
+      - R\ :sub:`0317low`\ (bits 7--2)
+
+        G\ :sub:`0316high`\ (bits 1--0)
+      - G\ :sub:`0318low`\ (bits 7--4)
+
+        R\ :sub:`0317high`\ (bits 3--0)
+      - R\ :sub:`0319low`\ (bits 7--6)
+
+        G\ :sub:`0318high`\ (bits 5--0)
+    * - start + 120:
+      - R\ :sub:`0319high`
+      - G\ :sub:`0320low`
+      - R\ :sub:`0321low`\ (bits 7--2)
+
+        G\ :sub:`0320high`\ (bits 1--0)
+      - G\ :sub:`0322low`\ (bits 7--4)
+
+        R\ :sub:`0321high`\ (bits 3--0)
+    * - start + 124:
+      - R\ :sub:`0323low`\ (bits 7--6)
+
+        G\ :sub:`0322high`\ (bits 5--0)
+      - R\ :sub:`0323high`
+      - G\ :sub:`0324low`
+      - G\ :sub:`0324high`\ (bits 1--0)
diff --git a/Documentation/media/uapi/v4l/vidioc-g-dv-timings.rst b/Documentation/media/uapi/v4l/vidioc-g-dv-timings.rst
index 2696380626d4..1a034e825161 100644
--- a/Documentation/media/uapi/v4l/vidioc-g-dv-timings.rst
+++ b/Documentation/media/uapi/v4l/vidioc-g-dv-timings.rst
@@ -267,7 +267,7 @@ EBUSY
 	will also be cleared.
     * - ``V4L2_DV_FL_HALF_LINE``
       - Specific to interlaced formats: if set, then the vertical
-	frontporch of field 1 (aka the odd field) is really one half-line
+	backporch of field 1 (aka the odd field) is really one half-line
 	longer and the vertical backporch of field 2 (aka the even field)
 	is really one half-line shorter, so each field has exactly the
 	same number of half-lines. Whether half-lines can be detected or
diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index 479ecec80593..a863009849a3 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -227,17 +227,20 @@ There are some minimal guarantees that may be expected of a CPU:
  (*) On any given CPU, dependent memory accesses will be issued in order, with
      respect to itself.  This means that for:
 
-	Q = READ_ONCE(P); smp_read_barrier_depends(); D = READ_ONCE(*Q);
+	Q = READ_ONCE(P); D = READ_ONCE(*Q);
 
      the CPU will issue the following memory operations:
 
 	Q = LOAD P, D = LOAD *Q
 
-     and always in that order.  On most systems, smp_read_barrier_depends()
-     does nothing, but it is required for DEC Alpha.  The READ_ONCE()
-     is required to prevent compiler mischief.  Please note that you
-     should normally use something like rcu_dereference() instead of
-     open-coding smp_read_barrier_depends().
+     and always in that order.  However, on DEC Alpha, READ_ONCE() also
+     emits a memory-barrier instruction, so that a DEC Alpha CPU will
+     instead issue the following memory operations:
+
+	Q = LOAD P, MEMORY_BARRIER, D = LOAD *Q, MEMORY_BARRIER
+
+     Whether on DEC Alpha or not, the READ_ONCE() also prevents compiler
+     mischief.
 
  (*) Overlapping loads and stores within a particular CPU will appear to be
      ordered within that CPU.  This means that for:
@@ -1815,7 +1818,7 @@ The Linux kernel has eight basic CPU memory barriers:
 	GENERAL		mb()			smp_mb()
 	WRITE		wmb()			smp_wmb()
 	READ		rmb()			smp_rmb()
-	DATA DEPENDENCY	read_barrier_depends()	smp_read_barrier_depends()
+	DATA DEPENDENCY				READ_ONCE()
 
 
 All memory barriers except the data dependency barriers imply a compiler
@@ -2864,7 +2867,10 @@ access depends on a read, not all do, so it may not be relied on.
 
 Other CPUs may also have split caches, but must coordinate between the various
 cachelets for normal memory accesses.  The semantics of the Alpha removes the
-need for coordination in the absence of memory barriers.
+need for hardware coordination in the absence of memory barriers, which
+permitted Alpha to sport higher CPU clock rates back in the day.  However,
+please note that smp_read_barrier_depends() should not be used except in
+Alpha arch-specific code and within the READ_ONCE() macro.
 
 
 CACHE COHERENCY VS DMA
diff --git a/Documentation/mips/AU1xxx_IDE.README b/Documentation/mips/AU1xxx_IDE.README
index 52844a58cc8a..ff675a1b1422 100644
--- a/Documentation/mips/AU1xxx_IDE.README
+++ b/Documentation/mips/AU1xxx_IDE.README
@@ -56,8 +56,6 @@ Following extra configs variables are introduced:
 
   CONFIG_BLK_DEV_IDE_AU1XXX_PIO_DBDMA    - enable the PIO+DBDMA mode
   CONFIG_BLK_DEV_IDE_AU1XXX_MDMA2_DBDMA  - enable the MWDMA mode
-  CONFIG_BLK_DEV_IDE_AU1XXX_BURSTABLE_ON - set Burstable FIFO in DBDMA
-                                           controller
 
 
 SUPPORTED IDE MODES
@@ -82,11 +80,9 @@ CONFIG_IDE_GENERIC=y
 CONFIG_BLK_DEV_IDEPCI=y
 CONFIG_BLK_DEV_GENERIC=y
 CONFIG_BLK_DEV_IDEDMA_PCI=y
-CONFIG_IDEDMA_PCI_AUTO=y
 CONFIG_BLK_DEV_IDE_AU1XXX=y
 CONFIG_BLK_DEV_IDE_AU1XXX_MDMA2_DBDMA=y
 CONFIG_BLK_DEV_IDEDMA=y
-CONFIG_IDEDMA_AUTO=y
 
 Also define 'IDE_AU1XXX_BURSTMODE' in 'drivers/ide/mips/au1xxx-ide.c' to enable
 the burst support on DBDMA controller.
@@ -94,16 +90,13 @@ the burst support on DBDMA controller.
 If the used system need the USB support enable the following kernel configs for
 high IDE to USB throughput.
 
-CONFIG_BLK_DEV_IDEDISK=y
 CONFIG_IDE_GENERIC=y
 CONFIG_BLK_DEV_IDEPCI=y
 CONFIG_BLK_DEV_GENERIC=y
 CONFIG_BLK_DEV_IDEDMA_PCI=y
-CONFIG_IDEDMA_PCI_AUTO=y
 CONFIG_BLK_DEV_IDE_AU1XXX=y
 CONFIG_BLK_DEV_IDE_AU1XXX_MDMA2_DBDMA=y
 CONFIG_BLK_DEV_IDEDMA=y
-CONFIG_IDEDMA_AUTO=y
 
 Also undefine 'IDE_AU1XXX_BURSTMODE' in 'drivers/ide/mips/au1xxx-ide.c' to
 disable the burst support on DBDMA controller.
diff --git a/Documentation/mtd/spi-nor.txt b/Documentation/mtd/spi-nor.txt
index 548d6306ebca..da1fbff5a24c 100644
--- a/Documentation/mtd/spi-nor.txt
+++ b/Documentation/mtd/spi-nor.txt
@@ -60,3 +60,6 @@ The main API is spi_nor_scan(). Before you call the hook, a driver should
 initialize the necessary fields for spi_nor{}. Please see
 drivers/mtd/spi-nor/spi-nor.c for detail. Please also refer to fsl-quadspi.c
 when you want to write a new driver for a SPI NOR controller.
+Another API is spi_nor_restore(), this is used to restore the status of SPI
+flash chip such as addressing mode. Call it whenever detach the driver from
+device or reboot the system.
diff --git a/Documentation/networking/00-INDEX b/Documentation/networking/00-INDEX
index 7a79b3587dd3..2b89d91b376f 100644
--- a/Documentation/networking/00-INDEX
+++ b/Documentation/networking/00-INDEX
@@ -36,8 +36,6 @@ bonding.txt
 	- Linux Ethernet Bonding Driver HOWTO: link aggregation in Linux.
 bridge.txt
 	- where to get user space programs for ethernet bridging with Linux.
-can.txt
-	- documentation on CAN protocol family.
 cdc_mbim.txt
 	- 3G/LTE USB modem (Mobile Broadband Interface Model)
 checksum-offloads.txt
@@ -228,6 +226,8 @@ x25.txt
 	- general info on X.25 development.
 x25-iface.txt
 	- description of the X.25 Packet Layer to LAPB device interface.
+xfrm_device.txt
+	- description of XFRM offload API
 xfrm_proc.txt
 	- description of the statistics package for XFRM.
 xfrm_sync.txt
diff --git a/Documentation/networking/batman-adv.rst b/Documentation/networking/batman-adv.rst
index a342b2cc3dc6..245fb6c0ab6f 100644
--- a/Documentation/networking/batman-adv.rst
+++ b/Documentation/networking/batman-adv.rst
@@ -1,3 +1,5 @@
+.. SPDX-License-Identifier: GPL-2.0
+
 ==========
 batman-adv
 ==========
diff --git a/Documentation/networking/can.rst b/Documentation/networking/can.rst
new file mode 100644
index 000000000000..d23c51abf8c6
--- /dev/null
+++ b/Documentation/networking/can.rst
@@ -0,0 +1,1437 @@
+===================================
+SocketCAN - Controller Area Network
+===================================
+
+Overview / What is SocketCAN
+============================
+
+The socketcan package is an implementation of CAN protocols
+(Controller Area Network) for Linux.  CAN is a networking technology
+which has widespread use in automation, embedded devices, and
+automotive fields.  While there have been other CAN implementations
+for Linux based on character devices, SocketCAN uses the Berkeley
+socket API, the Linux network stack and implements the CAN device
+drivers as network interfaces.  The CAN socket API has been designed
+as similar as possible to the TCP/IP protocols to allow programmers,
+familiar with network programming, to easily learn how to use CAN
+sockets.
+
+
+.. _socketcan-motivation:
+
+Motivation / Why Using the Socket API
+=====================================
+
+There have been CAN implementations for Linux before SocketCAN so the
+question arises, why we have started another project.  Most existing
+implementations come as a device driver for some CAN hardware, they
+are based on character devices and provide comparatively little
+functionality.  Usually, there is only a hardware-specific device
+driver which provides a character device interface to send and
+receive raw CAN frames, directly to/from the controller hardware.
+Queueing of frames and higher-level transport protocols like ISO-TP
+have to be implemented in user space applications.  Also, most
+character-device implementations support only one single process to
+open the device at a time, similar to a serial interface.  Exchanging
+the CAN controller requires employment of another device driver and
+often the need for adaption of large parts of the application to the
+new driver's API.
+
+SocketCAN was designed to overcome all of these limitations.  A new
+protocol family has been implemented which provides a socket interface
+to user space applications and which builds upon the Linux network
+layer, enabling use all of the provided queueing functionality.  A device
+driver for CAN controller hardware registers itself with the Linux
+network layer as a network device, so that CAN frames from the
+controller can be passed up to the network layer and on to the CAN
+protocol family module and also vice-versa.  Also, the protocol family
+module provides an API for transport protocol modules to register, so
+that any number of transport protocols can be loaded or unloaded
+dynamically.  In fact, the can core module alone does not provide any
+protocol and cannot be used without loading at least one additional
+protocol module.  Multiple sockets can be opened at the same time,
+on different or the same protocol module and they can listen/send
+frames on different or the same CAN IDs.  Several sockets listening on
+the same interface for frames with the same CAN ID are all passed the
+same received matching CAN frames.  An application wishing to
+communicate using a specific transport protocol, e.g. ISO-TP, just
+selects that protocol when opening the socket, and then can read and
+write application data byte streams, without having to deal with
+CAN-IDs, frames, etc.
+
+Similar functionality visible from user-space could be provided by a
+character device, too, but this would lead to a technically inelegant
+solution for a couple of reasons:
+
+* **Intricate usage:**  Instead of passing a protocol argument to
+  socket(2) and using bind(2) to select a CAN interface and CAN ID, an
+  application would have to do all these operations using ioctl(2)s.
+
+* **Code duplication:**  A character device cannot make use of the Linux
+  network queueing code, so all that code would have to be duplicated
+  for CAN networking.
+
+* **Abstraction:**  In most existing character-device implementations, the
+  hardware-specific device driver for a CAN controller directly
+  provides the character device for the application to work with.
+  This is at least very unusual in Unix systems for both, char and
+  block devices.  For example you don't have a character device for a
+  certain UART of a serial interface, a certain sound chip in your
+  computer, a SCSI or IDE controller providing access to your hard
+  disk or tape streamer device.  Instead, you have abstraction layers
+  which provide a unified character or block device interface to the
+  application on the one hand, and a interface for hardware-specific
+  device drivers on the other hand.  These abstractions are provided
+  by subsystems like the tty layer, the audio subsystem or the SCSI
+  and IDE subsystems for the devices mentioned above.
+
+  The easiest way to implement a CAN device driver is as a character
+  device without such a (complete) abstraction layer, as is done by most
+  existing drivers.  The right way, however, would be to add such a
+  layer with all the functionality like registering for certain CAN
+  IDs, supporting several open file descriptors and (de)multiplexing
+  CAN frames between them, (sophisticated) queueing of CAN frames, and
+  providing an API for device drivers to register with.  However, then
+  it would be no more difficult, or may be even easier, to use the
+  networking framework provided by the Linux kernel, and this is what
+  SocketCAN does.
+
+The use of the networking framework of the Linux kernel is just the
+natural and most appropriate way to implement CAN for Linux.
+
+
+.. _socketcan-concept:
+
+SocketCAN Concept
+=================
+
+As described in :ref:`socketcan-motivation` the main goal of SocketCAN is to
+provide a socket interface to user space applications which builds
+upon the Linux network layer. In contrast to the commonly known
+TCP/IP and ethernet networking, the CAN bus is a broadcast-only(!)
+medium that has no MAC-layer addressing like ethernet. The CAN-identifier
+(can_id) is used for arbitration on the CAN-bus. Therefore the CAN-IDs
+have to be chosen uniquely on the bus. When designing a CAN-ECU
+network the CAN-IDs are mapped to be sent by a specific ECU.
+For this reason a CAN-ID can be treated best as a kind of source address.
+
+
+.. _socketcan-receive-lists:
+
+Receive Lists
+-------------
+
+The network transparent access of multiple applications leads to the
+problem that different applications may be interested in the same
+CAN-IDs from the same CAN network interface. The SocketCAN core
+module - which implements the protocol family CAN - provides several
+high efficient receive lists for this reason. If e.g. a user space
+application opens a CAN RAW socket, the raw protocol module itself
+requests the (range of) CAN-IDs from the SocketCAN core that are
+requested by the user. The subscription and unsubscription of
+CAN-IDs can be done for specific CAN interfaces or for all(!) known
+CAN interfaces with the can_rx_(un)register() functions provided to
+CAN protocol modules by the SocketCAN core (see :ref:`socketcan-core-module`).
+To optimize the CPU usage at runtime the receive lists are split up
+into several specific lists per device that match the requested
+filter complexity for a given use-case.
+
+
+.. _socketcan-local-loopback1:
+
+Local Loopback of Sent Frames
+-----------------------------
+
+As known from other networking concepts the data exchanging
+applications may run on the same or different nodes without any
+change (except for the according addressing information):
+
+.. code::
+
+	 ___   ___   ___                   _______   ___
+	| _ | | _ | | _ |                 | _   _ | | _ |
+	||A|| ||B|| ||C||                 ||A| |B|| ||C||
+	|___| |___| |___|                 |_______| |___|
+	  |     |     |                       |       |
+	-----------------(1)- CAN bus -(2)---------------
+
+To ensure that application A receives the same information in the
+example (2) as it would receive in example (1) there is need for
+some kind of local loopback of the sent CAN frames on the appropriate
+node.
+
+The Linux network devices (by default) just can handle the
+transmission and reception of media dependent frames. Due to the
+arbitration on the CAN bus the transmission of a low prio CAN-ID
+may be delayed by the reception of a high prio CAN frame. To
+reflect the correct [*]_ traffic on the node the loopback of the sent
+data has to be performed right after a successful transmission. If
+the CAN network interface is not capable of performing the loopback for
+some reason the SocketCAN core can do this task as a fallback solution.
+See :ref:`socketcan-local-loopback1` for details (recommended).
+
+The loopback functionality is enabled by default to reflect standard
+networking behaviour for CAN applications. Due to some requests from
+the RT-SocketCAN group the loopback optionally may be disabled for each
+separate socket. See sockopts from the CAN RAW sockets in :ref:`socketcan-raw-sockets`.
+
+.. [*] you really like to have this when you're running analyser
+       tools like 'candump' or 'cansniffer' on the (same) node.
+
+
+.. _socketcan-network-problem-notifications:
+
+Network Problem Notifications
+-----------------------------
+
+The use of the CAN bus may lead to several problems on the physical
+and media access control layer. Detecting and logging of these lower
+layer problems is a vital requirement for CAN users to identify
+hardware issues on the physical transceiver layer as well as
+arbitration problems and error frames caused by the different
+ECUs. The occurrence of detected errors are important for diagnosis
+and have to be logged together with the exact timestamp. For this
+reason the CAN interface driver can generate so called Error Message
+Frames that can optionally be passed to the user application in the
+same way as other CAN frames. Whenever an error on the physical layer
+or the MAC layer is detected (e.g. by the CAN controller) the driver
+creates an appropriate error message frame. Error messages frames can
+be requested by the user application using the common CAN filter
+mechanisms. Inside this filter definition the (interested) type of
+errors may be selected. The reception of error messages is disabled
+by default. The format of the CAN error message frame is briefly
+described in the Linux header file "include/uapi/linux/can/error.h".
+
+
+How to use SocketCAN
+====================
+
+Like TCP/IP, you first need to open a socket for communicating over a
+CAN network. Since SocketCAN implements a new protocol family, you
+need to pass PF_CAN as the first argument to the socket(2) system
+call. Currently, there are two CAN protocols to choose from, the raw
+socket protocol and the broadcast manager (BCM). So to open a socket,
+you would write::
+
+    s = socket(PF_CAN, SOCK_RAW, CAN_RAW);
+
+and::
+
+    s = socket(PF_CAN, SOCK_DGRAM, CAN_BCM);
+
+respectively.  After the successful creation of the socket, you would
+normally use the bind(2) system call to bind the socket to a CAN
+interface (which is different from TCP/IP due to different addressing
+- see :ref:`socketcan-concept`). After binding (CAN_RAW) or connecting (CAN_BCM)
+the socket, you can read(2) and write(2) from/to the socket or use
+send(2), sendto(2), sendmsg(2) and the recv* counterpart operations
+on the socket as usual. There are also CAN specific socket options
+described below.
+
+The basic CAN frame structure and the sockaddr structure are defined
+in include/linux/can.h:
+
+.. code-block:: C
+
+    struct can_frame {
+            canid_t can_id;  /* 32 bit CAN_ID + EFF/RTR/ERR flags */
+            __u8    can_dlc; /* frame payload length in byte (0 .. 8) */
+            __u8    __pad;   /* padding */
+            __u8    __res0;  /* reserved / padding */
+            __u8    __res1;  /* reserved / padding */
+            __u8    data[8] __attribute__((aligned(8)));
+    };
+
+The alignment of the (linear) payload data[] to a 64bit boundary
+allows the user to define their own structs and unions to easily access
+the CAN payload. There is no given byteorder on the CAN bus by
+default. A read(2) system call on a CAN_RAW socket transfers a
+struct can_frame to the user space.
+
+The sockaddr_can structure has an interface index like the
+PF_PACKET socket, that also binds to a specific interface:
+
+.. code-block:: C
+
+    struct sockaddr_can {
+            sa_family_t can_family;
+            int         can_ifindex;
+            union {
+                    /* transport protocol class address info (e.g. ISOTP) */
+                    struct { canid_t rx_id, tx_id; } tp;
+
+                    /* reserved for future CAN protocols address information */
+            } can_addr;
+    };
+
+To determine the interface index an appropriate ioctl() has to
+be used (example for CAN_RAW sockets without error checking):
+
+.. code-block:: C
+
+    int s;
+    struct sockaddr_can addr;
+    struct ifreq ifr;
+
+    s = socket(PF_CAN, SOCK_RAW, CAN_RAW);
+
+    strcpy(ifr.ifr_name, "can0" );
+    ioctl(s, SIOCGIFINDEX, &ifr);
+
+    addr.can_family = AF_CAN;
+    addr.can_ifindex = ifr.ifr_ifindex;
+
+    bind(s, (struct sockaddr *)&addr, sizeof(addr));
+
+    (..)
+
+To bind a socket to all(!) CAN interfaces the interface index must
+be 0 (zero). In this case the socket receives CAN frames from every
+enabled CAN interface. To determine the originating CAN interface
+the system call recvfrom(2) may be used instead of read(2). To send
+on a socket that is bound to 'any' interface sendto(2) is needed to
+specify the outgoing interface.
+
+Reading CAN frames from a bound CAN_RAW socket (see above) consists
+of reading a struct can_frame:
+
+.. code-block:: C
+
+    struct can_frame frame;
+
+    nbytes = read(s, &frame, sizeof(struct can_frame));
+
+    if (nbytes < 0) {
+            perror("can raw socket read");
+            return 1;
+    }
+
+    /* paranoid check ... */
+    if (nbytes < sizeof(struct can_frame)) {
+            fprintf(stderr, "read: incomplete CAN frame\n");
+            return 1;
+    }
+
+    /* do something with the received CAN frame */
+
+Writing CAN frames can be done similarly, with the write(2) system call::
+
+    nbytes = write(s, &frame, sizeof(struct can_frame));
+
+When the CAN interface is bound to 'any' existing CAN interface
+(addr.can_ifindex = 0) it is recommended to use recvfrom(2) if the
+information about the originating CAN interface is needed:
+
+.. code-block:: C
+
+    struct sockaddr_can addr;
+    struct ifreq ifr;
+    socklen_t len = sizeof(addr);
+    struct can_frame frame;
+
+    nbytes = recvfrom(s, &frame, sizeof(struct can_frame),
+                      0, (struct sockaddr*)&addr, &len);
+
+    /* get interface name of the received CAN frame */
+    ifr.ifr_ifindex = addr.can_ifindex;
+    ioctl(s, SIOCGIFNAME, &ifr);
+    printf("Received a CAN frame from interface %s", ifr.ifr_name);
+
+To write CAN frames on sockets bound to 'any' CAN interface the
+outgoing interface has to be defined certainly:
+
+.. code-block:: C
+
+    strcpy(ifr.ifr_name, "can0");
+    ioctl(s, SIOCGIFINDEX, &ifr);
+    addr.can_ifindex = ifr.ifr_ifindex;
+    addr.can_family  = AF_CAN;
+
+    nbytes = sendto(s, &frame, sizeof(struct can_frame),
+                    0, (struct sockaddr*)&addr, sizeof(addr));
+
+An accurate timestamp can be obtained with an ioctl(2) call after reading
+a message from the socket:
+
+.. code-block:: C
+
+    struct timeval tv;
+    ioctl(s, SIOCGSTAMP, &tv);
+
+The timestamp has a resolution of one microsecond and is set automatically
+at the reception of a CAN frame.
+
+Remark about CAN FD (flexible data rate) support:
+
+Generally the handling of CAN FD is very similar to the formerly described
+examples. The new CAN FD capable CAN controllers support two different
+bitrates for the arbitration phase and the payload phase of the CAN FD frame
+and up to 64 bytes of payload. This extended payload length breaks all the
+kernel interfaces (ABI) which heavily rely on the CAN frame with fixed eight
+bytes of payload (struct can_frame) like the CAN_RAW socket. Therefore e.g.
+the CAN_RAW socket supports a new socket option CAN_RAW_FD_FRAMES that
+switches the socket into a mode that allows the handling of CAN FD frames
+and (legacy) CAN frames simultaneously (see :ref:`socketcan-rawfd`).
+
+The struct canfd_frame is defined in include/linux/can.h:
+
+.. code-block:: C
+
+    struct canfd_frame {
+            canid_t can_id;  /* 32 bit CAN_ID + EFF/RTR/ERR flags */
+            __u8    len;     /* frame payload length in byte (0 .. 64) */
+            __u8    flags;   /* additional flags for CAN FD */
+            __u8    __res0;  /* reserved / padding */
+            __u8    __res1;  /* reserved / padding */
+            __u8    data[64] __attribute__((aligned(8)));
+    };
+
+The struct canfd_frame and the existing struct can_frame have the can_id,
+the payload length and the payload data at the same offset inside their
+structures. This allows to handle the different structures very similar.
+When the content of a struct can_frame is copied into a struct canfd_frame
+all structure elements can be used as-is - only the data[] becomes extended.
+
+When introducing the struct canfd_frame it turned out that the data length
+code (DLC) of the struct can_frame was used as a length information as the
+length and the DLC has a 1:1 mapping in the range of 0 .. 8. To preserve
+the easy handling of the length information the canfd_frame.len element
+contains a plain length value from 0 .. 64. So both canfd_frame.len and
+can_frame.can_dlc are equal and contain a length information and no DLC.
+For details about the distinction of CAN and CAN FD capable devices and
+the mapping to the bus-relevant data length code (DLC), see :ref:`socketcan-can-fd-driver`.
+
+The length of the two CAN(FD) frame structures define the maximum transfer
+unit (MTU) of the CAN(FD) network interface and skbuff data length. Two
+definitions are specified for CAN specific MTUs in include/linux/can.h:
+
+.. code-block:: C
+
+  #define CAN_MTU   (sizeof(struct can_frame))   == 16  => 'legacy' CAN frame
+  #define CANFD_MTU (sizeof(struct canfd_frame)) == 72  => CAN FD frame
+
+
+.. _socketcan-raw-sockets:
+
+RAW Protocol Sockets with can_filters (SOCK_RAW)
+------------------------------------------------
+
+Using CAN_RAW sockets is extensively comparable to the commonly
+known access to CAN character devices. To meet the new possibilities
+provided by the multi user SocketCAN approach, some reasonable
+defaults are set at RAW socket binding time:
+
+- The filters are set to exactly one filter receiving everything
+- The socket only receives valid data frames (=> no error message frames)
+- The loopback of sent CAN frames is enabled (see :ref:`socketcan-local-loopback2`)
+- The socket does not receive its own sent frames (in loopback mode)
+
+These default settings may be changed before or after binding the socket.
+To use the referenced definitions of the socket options for CAN_RAW
+sockets, include <linux/can/raw.h>.
+
+
+.. _socketcan-rawfilter:
+
+RAW socket option CAN_RAW_FILTER
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The reception of CAN frames using CAN_RAW sockets can be controlled
+by defining 0 .. n filters with the CAN_RAW_FILTER socket option.
+
+The CAN filter structure is defined in include/linux/can.h:
+
+.. code-block:: C
+
+    struct can_filter {
+            canid_t can_id;
+            canid_t can_mask;
+    };
+
+A filter matches, when:
+
+.. code-block:: C
+
+    <received_can_id> & mask == can_id & mask
+
+which is analogous to known CAN controllers hardware filter semantics.
+The filter can be inverted in this semantic, when the CAN_INV_FILTER
+bit is set in can_id element of the can_filter structure. In
+contrast to CAN controller hardware filters the user may set 0 .. n
+receive filters for each open socket separately:
+
+.. code-block:: C
+
+    struct can_filter rfilter[2];
+
+    rfilter[0].can_id   = 0x123;
+    rfilter[0].can_mask = CAN_SFF_MASK;
+    rfilter[1].can_id   = 0x200;
+    rfilter[1].can_mask = 0x700;
+
+    setsockopt(s, SOL_CAN_RAW, CAN_RAW_FILTER, &rfilter, sizeof(rfilter));
+
+To disable the reception of CAN frames on the selected CAN_RAW socket:
+
+.. code-block:: C
+
+    setsockopt(s, SOL_CAN_RAW, CAN_RAW_FILTER, NULL, 0);
+
+To set the filters to zero filters is quite obsolete as to not read
+data causes the raw socket to discard the received CAN frames. But
+having this 'send only' use-case we may remove the receive list in the
+Kernel to save a little (really a very little!) CPU usage.
+
+CAN Filter Usage Optimisation
+.............................
+
+The CAN filters are processed in per-device filter lists at CAN frame
+reception time. To reduce the number of checks that need to be performed
+while walking through the filter lists the CAN core provides an optimized
+filter handling when the filter subscription focusses on a single CAN ID.
+
+For the possible 2048 SFF CAN identifiers the identifier is used as an index
+to access the corresponding subscription list without any further checks.
+For the 2^29 possible EFF CAN identifiers a 10 bit XOR folding is used as
+hash function to retrieve the EFF table index.
+
+To benefit from the optimized filters for single CAN identifiers the
+CAN_SFF_MASK or CAN_EFF_MASK have to be set into can_filter.mask together
+with set CAN_EFF_FLAG and CAN_RTR_FLAG bits. A set CAN_EFF_FLAG bit in the
+can_filter.mask makes clear that it matters whether a SFF or EFF CAN ID is
+subscribed. E.g. in the example from above:
+
+.. code-block:: C
+
+    rfilter[0].can_id   = 0x123;
+    rfilter[0].can_mask = CAN_SFF_MASK;
+
+both SFF frames with CAN ID 0x123 and EFF frames with 0xXXXXX123 can pass.
+
+To filter for only 0x123 (SFF) and 0x12345678 (EFF) CAN identifiers the
+filter has to be defined in this way to benefit from the optimized filters:
+
+.. code-block:: C
+
+    struct can_filter rfilter[2];
+
+    rfilter[0].can_id   = 0x123;
+    rfilter[0].can_mask = (CAN_EFF_FLAG | CAN_RTR_FLAG | CAN_SFF_MASK);
+    rfilter[1].can_id   = 0x12345678 | CAN_EFF_FLAG;
+    rfilter[1].can_mask = (CAN_EFF_FLAG | CAN_RTR_FLAG | CAN_EFF_MASK);
+
+    setsockopt(s, SOL_CAN_RAW, CAN_RAW_FILTER, &rfilter, sizeof(rfilter));
+
+
+RAW Socket Option CAN_RAW_ERR_FILTER
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As described in :ref:`socketcan-network-problem-notifications` the CAN interface driver can generate so
+called Error Message Frames that can optionally be passed to the user
+application in the same way as other CAN frames. The possible
+errors are divided into different error classes that may be filtered
+using the appropriate error mask. To register for every possible
+error condition CAN_ERR_MASK can be used as value for the error mask.
+The values for the error mask are defined in linux/can/error.h:
+
+.. code-block:: C
+
+    can_err_mask_t err_mask = ( CAN_ERR_TX_TIMEOUT | CAN_ERR_BUSOFF );
+
+    setsockopt(s, SOL_CAN_RAW, CAN_RAW_ERR_FILTER,
+               &err_mask, sizeof(err_mask));
+
+
+RAW Socket Option CAN_RAW_LOOPBACK
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To meet multi user needs the local loopback is enabled by default
+(see :ref:`socketcan-local-loopback1` for details). But in some embedded use-cases
+(e.g. when only one application uses the CAN bus) this loopback
+functionality can be disabled (separately for each socket):
+
+.. code-block:: C
+
+    int loopback = 0; /* 0 = disabled, 1 = enabled (default) */
+
+    setsockopt(s, SOL_CAN_RAW, CAN_RAW_LOOPBACK, &loopback, sizeof(loopback));
+
+
+RAW socket option CAN_RAW_RECV_OWN_MSGS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When the local loopback is enabled, all the sent CAN frames are
+looped back to the open CAN sockets that registered for the CAN
+frames' CAN-ID on this given interface to meet the multi user
+needs. The reception of the CAN frames on the same socket that was
+sending the CAN frame is assumed to be unwanted and therefore
+disabled by default. This default behaviour may be changed on
+demand:
+
+.. code-block:: C
+
+    int recv_own_msgs = 1; /* 0 = disabled (default), 1 = enabled */
+
+    setsockopt(s, SOL_CAN_RAW, CAN_RAW_RECV_OWN_MSGS,
+               &recv_own_msgs, sizeof(recv_own_msgs));
+
+
+.. _socketcan-rawfd:
+
+RAW Socket Option CAN_RAW_FD_FRAMES
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+CAN FD support in CAN_RAW sockets can be enabled with a new socket option
+CAN_RAW_FD_FRAMES which is off by default. When the new socket option is
+not supported by the CAN_RAW socket (e.g. on older kernels), switching the
+CAN_RAW_FD_FRAMES option returns the error -ENOPROTOOPT.
+
+Once CAN_RAW_FD_FRAMES is enabled the application can send both CAN frames
+and CAN FD frames. OTOH the application has to handle CAN and CAN FD frames
+when reading from the socket:
+
+.. code-block:: C
+
+    CAN_RAW_FD_FRAMES enabled:  CAN_MTU and CANFD_MTU are allowed
+    CAN_RAW_FD_FRAMES disabled: only CAN_MTU is allowed (default)
+
+Example:
+
+.. code-block:: C
+
+    [ remember: CANFD_MTU == sizeof(struct canfd_frame) ]
+
+    struct canfd_frame cfd;
+
+    nbytes = read(s, &cfd, CANFD_MTU);
+
+    if (nbytes == CANFD_MTU) {
+            printf("got CAN FD frame with length %d\n", cfd.len);
+            /* cfd.flags contains valid data */
+    } else if (nbytes == CAN_MTU) {
+            printf("got legacy CAN frame with length %d\n", cfd.len);
+            /* cfd.flags is undefined */
+    } else {
+            fprintf(stderr, "read: invalid CAN(FD) frame\n");
+            return 1;
+    }
+
+    /* the content can be handled independently from the received MTU size */
+
+    printf("can_id: %X data length: %d data: ", cfd.can_id, cfd.len);
+    for (i = 0; i < cfd.len; i++)
+            printf("%02X ", cfd.data[i]);
+
+When reading with size CANFD_MTU only returns CAN_MTU bytes that have
+been received from the socket a legacy CAN frame has been read into the
+provided CAN FD structure. Note that the canfd_frame.flags data field is
+not specified in the struct can_frame and therefore it is only valid in
+CANFD_MTU sized CAN FD frames.
+
+Implementation hint for new CAN applications:
+
+To build a CAN FD aware application use struct canfd_frame as basic CAN
+data structure for CAN_RAW based applications. When the application is
+executed on an older Linux kernel and switching the CAN_RAW_FD_FRAMES
+socket option returns an error: No problem. You'll get legacy CAN frames
+or CAN FD frames and can process them the same way.
+
+When sending to CAN devices make sure that the device is capable to handle
+CAN FD frames by checking if the device maximum transfer unit is CANFD_MTU.
+The CAN device MTU can be retrieved e.g. with a SIOCGIFMTU ioctl() syscall.
+
+
+RAW socket option CAN_RAW_JOIN_FILTERS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The CAN_RAW socket can set multiple CAN identifier specific filters that
+lead to multiple filters in the af_can.c filter processing. These filters
+are indenpendent from each other which leads to logical OR'ed filters when
+applied (see :ref:`socketcan-rawfilter`).
+
+This socket option joines the given CAN filters in the way that only CAN
+frames are passed to user space that matched *all* given CAN filters. The
+semantic for the applied filters is therefore changed to a logical AND.
+
+This is useful especially when the filterset is a combination of filters
+where the CAN_INV_FILTER flag is set in order to notch single CAN IDs or
+CAN ID ranges from the incoming traffic.
+
+
+RAW Socket Returned Message Flags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When using recvmsg() call, the msg->msg_flags may contain following flags:
+
+MSG_DONTROUTE:
+	set when the received frame was created on the local host.
+
+MSG_CONFIRM:
+	set when the frame was sent via the socket it is received on.
+	This flag can be interpreted as a 'transmission confirmation' when the
+	CAN driver supports the echo of frames on driver level, see
+	:ref:`socketcan-local-loopback1` and :ref:`socketcan-local-loopback2`.
+	In order to receive such messages, CAN_RAW_RECV_OWN_MSGS must be set.
+
+
+Broadcast Manager Protocol Sockets (SOCK_DGRAM)
+-----------------------------------------------
+
+The Broadcast Manager protocol provides a command based configuration
+interface to filter and send (e.g. cyclic) CAN messages in kernel space.
+
+Receive filters can be used to down sample frequent messages; detect events
+such as message contents changes, packet length changes, and do time-out
+monitoring of received messages.
+
+Periodic transmission tasks of CAN frames or a sequence of CAN frames can be
+created and modified at runtime; both the message content and the two
+possible transmit intervals can be altered.
+
+A BCM socket is not intended for sending individual CAN frames using the
+struct can_frame as known from the CAN_RAW socket. Instead a special BCM
+configuration message is defined. The basic BCM configuration message used
+to communicate with the broadcast manager and the available operations are
+defined in the linux/can/bcm.h include. The BCM message consists of a
+message header with a command ('opcode') followed by zero or more CAN frames.
+The broadcast manager sends responses to user space in the same form:
+
+.. code-block:: C
+
+    struct bcm_msg_head {
+            __u32 opcode;                   /* command */
+            __u32 flags;                    /* special flags */
+            __u32 count;                    /* run 'count' times with ival1 */
+            struct timeval ival1, ival2;    /* count and subsequent interval */
+            canid_t can_id;                 /* unique can_id for task */
+            __u32 nframes;                  /* number of can_frames following */
+            struct can_frame frames[0];
+    };
+
+The aligned payload 'frames' uses the same basic CAN frame structure defined
+at the beginning of :ref:`socketcan-rawfd` and in the include/linux/can.h include. All
+messages to the broadcast manager from user space have this structure.
+
+Note a CAN_BCM socket must be connected instead of bound after socket
+creation (example without error checking):
+
+.. code-block:: C
+
+    int s;
+    struct sockaddr_can addr;
+    struct ifreq ifr;
+
+    s = socket(PF_CAN, SOCK_DGRAM, CAN_BCM);
+
+    strcpy(ifr.ifr_name, "can0");
+    ioctl(s, SIOCGIFINDEX, &ifr);
+
+    addr.can_family = AF_CAN;
+    addr.can_ifindex = ifr.ifr_ifindex;
+
+    connect(s, (struct sockaddr *)&addr, sizeof(addr));
+
+    (..)
+
+The broadcast manager socket is able to handle any number of in flight
+transmissions or receive filters concurrently. The different RX/TX jobs are
+distinguished by the unique can_id in each BCM message. However additional
+CAN_BCM sockets are recommended to communicate on multiple CAN interfaces.
+When the broadcast manager socket is bound to 'any' CAN interface (=> the
+interface index is set to zero) the configured receive filters apply to any
+CAN interface unless the sendto() syscall is used to overrule the 'any' CAN
+interface index. When using recvfrom() instead of read() to retrieve BCM
+socket messages the originating CAN interface is provided in can_ifindex.
+
+
+Broadcast Manager Operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The opcode defines the operation for the broadcast manager to carry out,
+or details the broadcast managers response to several events, including
+user requests.
+
+Transmit Operations (user space to broadcast manager):
+
+TX_SETUP:
+	Create (cyclic) transmission task.
+
+TX_DELETE:
+	Remove (cyclic) transmission task, requires only can_id.
+
+TX_READ:
+	Read properties of (cyclic) transmission task for can_id.
+
+TX_SEND:
+	Send one CAN frame.
+
+Transmit Responses (broadcast manager to user space):
+
+TX_STATUS:
+	Reply to TX_READ request (transmission task configuration).
+
+TX_EXPIRED:
+	Notification when counter finishes sending at initial interval
+	'ival1'. Requires the TX_COUNTEVT flag to be set at TX_SETUP.
+
+Receive Operations (user space to broadcast manager):
+
+RX_SETUP:
+	Create RX content filter subscription.
+
+RX_DELETE:
+	Remove RX content filter subscription, requires only can_id.
+
+RX_READ:
+	Read properties of RX content filter subscription for can_id.
+
+Receive Responses (broadcast manager to user space):
+
+RX_STATUS:
+	Reply to RX_READ request (filter task configuration).
+
+RX_TIMEOUT:
+	Cyclic message is detected to be absent (timer ival1 expired).
+
+RX_CHANGED:
+	BCM message with updated CAN frame (detected content change).
+	Sent on first message received or on receipt of revised CAN messages.
+
+
+Broadcast Manager Message Flags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When sending a message to the broadcast manager the 'flags' element may
+contain the following flag definitions which influence the behaviour:
+
+SETTIMER:
+	Set the values of ival1, ival2 and count
+
+STARTTIMER:
+	Start the timer with the actual values of ival1, ival2
+	and count. Starting the timer leads simultaneously to emit a CAN frame.
+
+TX_COUNTEVT:
+	Create the message TX_EXPIRED when count expires
+
+TX_ANNOUNCE:
+	A change of data by the process is emitted immediately.
+
+TX_CP_CAN_ID:
+	Copies the can_id from the message header to each
+	subsequent frame in frames. This is intended as usage simplification. For
+	TX tasks the unique can_id from the message header may differ from the
+	can_id(s) stored for transmission in the subsequent struct can_frame(s).
+
+RX_FILTER_ID:
+	Filter by can_id alone, no frames required (nframes=0).
+
+RX_CHECK_DLC:
+	A change of the DLC leads to an RX_CHANGED.
+
+RX_NO_AUTOTIMER:
+	Prevent automatically starting the timeout monitor.
+
+RX_ANNOUNCE_RESUME:
+	If passed at RX_SETUP and a receive timeout occurred, a
+	RX_CHANGED message will be generated when the (cyclic) receive restarts.
+
+TX_RESET_MULTI_IDX:
+	Reset the index for the multiple frame transmission.
+
+RX_RTR_FRAME:
+	Send reply for RTR-request (placed in op->frames[0]).
+
+
+Broadcast Manager Transmission Timers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Periodic transmission configurations may use up to two interval timers.
+In this case the BCM sends a number of messages ('count') at an interval
+'ival1', then continuing to send at another given interval 'ival2'. When
+only one timer is needed 'count' is set to zero and only 'ival2' is used.
+When SET_TIMER and START_TIMER flag were set the timers are activated.
+The timer values can be altered at runtime when only SET_TIMER is set.
+
+
+Broadcast Manager message sequence transmission
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Up to 256 CAN frames can be transmitted in a sequence in the case of a cyclic
+TX task configuration. The number of CAN frames is provided in the 'nframes'
+element of the BCM message head. The defined number of CAN frames are added
+as array to the TX_SETUP BCM configuration message:
+
+.. code-block:: C
+
+    /* create a struct to set up a sequence of four CAN frames */
+    struct {
+            struct bcm_msg_head msg_head;
+            struct can_frame frame[4];
+    } mytxmsg;
+
+    (..)
+    mytxmsg.msg_head.nframes = 4;
+    (..)
+
+    write(s, &mytxmsg, sizeof(mytxmsg));
+
+With every transmission the index in the array of CAN frames is increased
+and set to zero at index overflow.
+
+
+Broadcast Manager Receive Filter Timers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The timer values ival1 or ival2 may be set to non-zero values at RX_SETUP.
+When the SET_TIMER flag is set the timers are enabled:
+
+ival1:
+	Send RX_TIMEOUT when a received message is not received again within
+	the given time. When START_TIMER is set at RX_SETUP the timeout detection
+	is activated directly - even without a former CAN frame reception.
+
+ival2:
+	Throttle the received message rate down to the value of ival2. This
+	is useful to reduce messages for the application when the signal inside the
+	CAN frame is stateless as state changes within the ival2 periode may get
+	lost.
+
+Broadcast Manager Multiplex Message Receive Filter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To filter for content changes in multiplex message sequences an array of more
+than one CAN frames can be passed in a RX_SETUP configuration message. The
+data bytes of the first CAN frame contain the mask of relevant bits that
+have to match in the subsequent CAN frames with the received CAN frame.
+If one of the subsequent CAN frames is matching the bits in that frame data
+mark the relevant content to be compared with the previous received content.
+Up to 257 CAN frames (multiplex filter bit mask CAN frame plus 256 CAN
+filters) can be added as array to the TX_SETUP BCM configuration message:
+
+.. code-block:: C
+
+    /* usually used to clear CAN frame data[] - beware of endian problems! */
+    #define U64_DATA(p) (*(unsigned long long*)(p)->data)
+
+    struct {
+            struct bcm_msg_head msg_head;
+            struct can_frame frame[5];
+    } msg;
+
+    msg.msg_head.opcode  = RX_SETUP;
+    msg.msg_head.can_id  = 0x42;
+    msg.msg_head.flags   = 0;
+    msg.msg_head.nframes = 5;
+    U64_DATA(&msg.frame[0]) = 0xFF00000000000000ULL; /* MUX mask */
+    U64_DATA(&msg.frame[1]) = 0x01000000000000FFULL; /* data mask (MUX 0x01) */
+    U64_DATA(&msg.frame[2]) = 0x0200FFFF000000FFULL; /* data mask (MUX 0x02) */
+    U64_DATA(&msg.frame[3]) = 0x330000FFFFFF0003ULL; /* data mask (MUX 0x33) */
+    U64_DATA(&msg.frame[4]) = 0x4F07FC0FF0000000ULL; /* data mask (MUX 0x4F) */
+
+    write(s, &msg, sizeof(msg));
+
+
+Broadcast Manager CAN FD Support
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The programming API of the CAN_BCM depends on struct can_frame which is
+given as array directly behind the bcm_msg_head structure. To follow this
+schema for the CAN FD frames a new flag 'CAN_FD_FRAME' in the bcm_msg_head
+flags indicates that the concatenated CAN frame structures behind the
+bcm_msg_head are defined as struct canfd_frame:
+
+.. code-block:: C
+
+    struct {
+            struct bcm_msg_head msg_head;
+            struct canfd_frame frame[5];
+    } msg;
+
+    msg.msg_head.opcode  = RX_SETUP;
+    msg.msg_head.can_id  = 0x42;
+    msg.msg_head.flags   = CAN_FD_FRAME;
+    msg.msg_head.nframes = 5;
+    (..)
+
+When using CAN FD frames for multiplex filtering the MUX mask is still
+expected in the first 64 bit of the struct canfd_frame data section.
+
+
+Connected Transport Protocols (SOCK_SEQPACKET)
+----------------------------------------------
+
+(to be written)
+
+
+Unconnected Transport Protocols (SOCK_DGRAM)
+--------------------------------------------
+
+(to be written)
+
+
+.. _socketcan-core-module:
+
+SocketCAN Core Module
+=====================
+
+The SocketCAN core module implements the protocol family
+PF_CAN. CAN protocol modules are loaded by the core module at
+runtime. The core module provides an interface for CAN protocol
+modules to subscribe needed CAN IDs (see :ref:`socketcan-receive-lists`).
+
+
+can.ko Module Params
+--------------------
+
+- **stats_timer**:
+  To calculate the SocketCAN core statistics
+  (e.g. current/maximum frames per second) this 1 second timer is
+  invoked at can.ko module start time by default. This timer can be
+  disabled by using stattimer=0 on the module commandline.
+
+- **debug**:
+  (removed since SocketCAN SVN r546)
+
+
+procfs content
+--------------
+
+As described in :ref:`socketcan-receive-lists` the SocketCAN core uses several filter
+lists to deliver received CAN frames to CAN protocol modules. These
+receive lists, their filters and the count of filter matches can be
+checked in the appropriate receive list. All entries contain the
+device and a protocol module identifier::
+
+    foo@bar:~$ cat /proc/net/can/rcvlist_all
+
+    receive list 'rx_all':
+      (vcan3: no entry)
+      (vcan2: no entry)
+      (vcan1: no entry)
+      device   can_id   can_mask  function  userdata   matches  ident
+       vcan0     000    00000000  f88e6370  f6c6f400         0  raw
+      (any: no entry)
+
+In this example an application requests any CAN traffic from vcan0::
+
+    rcvlist_all - list for unfiltered entries (no filter operations)
+    rcvlist_eff - list for single extended frame (EFF) entries
+    rcvlist_err - list for error message frames masks
+    rcvlist_fil - list for mask/value filters
+    rcvlist_inv - list for mask/value filters (inverse semantic)
+    rcvlist_sff - list for single standard frame (SFF) entries
+
+Additional procfs files in /proc/net/can::
+
+    stats       - SocketCAN core statistics (rx/tx frames, match ratios, ...)
+    reset_stats - manual statistic reset
+    version     - prints the SocketCAN core version and the ABI version
+
+
+Writing Own CAN Protocol Modules
+--------------------------------
+
+To implement a new protocol in the protocol family PF_CAN a new
+protocol has to be defined in include/linux/can.h .
+The prototypes and definitions to use the SocketCAN core can be
+accessed by including include/linux/can/core.h .
+In addition to functions that register the CAN protocol and the
+CAN device notifier chain there are functions to subscribe CAN
+frames received by CAN interfaces and to send CAN frames::
+
+    can_rx_register   - subscribe CAN frames from a specific interface
+    can_rx_unregister - unsubscribe CAN frames from a specific interface
+    can_send          - transmit a CAN frame (optional with local loopback)
+
+For details see the kerneldoc documentation in net/can/af_can.c or
+the source code of net/can/raw.c or net/can/bcm.c .
+
+
+CAN Network Drivers
+===================
+
+Writing a CAN network device driver is much easier than writing a
+CAN character device driver. Similar to other known network device
+drivers you mainly have to deal with:
+
+- TX: Put the CAN frame from the socket buffer to the CAN controller.
+- RX: Put the CAN frame from the CAN controller to the socket buffer.
+
+See e.g. at Documentation/networking/netdevices.txt . The differences
+for writing CAN network device driver are described below:
+
+
+General Settings
+----------------
+
+.. code-block:: C
+
+    dev->type  = ARPHRD_CAN; /* the netdevice hardware type */
+    dev->flags = IFF_NOARP;  /* CAN has no arp */
+
+    dev->mtu = CAN_MTU; /* sizeof(struct can_frame) -> legacy CAN interface */
+
+    or alternative, when the controller supports CAN with flexible data rate:
+    dev->mtu = CANFD_MTU; /* sizeof(struct canfd_frame) -> CAN FD interface */
+
+The struct can_frame or struct canfd_frame is the payload of each socket
+buffer (skbuff) in the protocol family PF_CAN.
+
+
+.. _socketcan-local-loopback2:
+
+Local Loopback of Sent Frames
+-----------------------------
+
+As described in :ref:`socketcan-local-loopback1` the CAN network device driver should
+support a local loopback functionality similar to the local echo
+e.g. of tty devices. In this case the driver flag IFF_ECHO has to be
+set to prevent the PF_CAN core from locally echoing sent frames
+(aka loopback) as fallback solution::
+
+    dev->flags = (IFF_NOARP | IFF_ECHO);
+
+
+CAN Controller Hardware Filters
+-------------------------------
+
+To reduce the interrupt load on deep embedded systems some CAN
+controllers support the filtering of CAN IDs or ranges of CAN IDs.
+These hardware filter capabilities vary from controller to
+controller and have to be identified as not feasible in a multi-user
+networking approach. The use of the very controller specific
+hardware filters could make sense in a very dedicated use-case, as a
+filter on driver level would affect all users in the multi-user
+system. The high efficient filter sets inside the PF_CAN core allow
+to set different multiple filters for each socket separately.
+Therefore the use of hardware filters goes to the category 'handmade
+tuning on deep embedded systems'. The author is running a MPC603e
+@133MHz with four SJA1000 CAN controllers from 2002 under heavy bus
+load without any problems ...
+
+
+The Virtual CAN Driver (vcan)
+-----------------------------
+
+Similar to the network loopback devices, vcan offers a virtual local
+CAN interface. A full qualified address on CAN consists of
+
+- a unique CAN Identifier (CAN ID)
+- the CAN bus this CAN ID is transmitted on (e.g. can0)
+
+so in common use cases more than one virtual CAN interface is needed.
+
+The virtual CAN interfaces allow the transmission and reception of CAN
+frames without real CAN controller hardware. Virtual CAN network
+devices are usually named 'vcanX', like vcan0 vcan1 vcan2 ...
+When compiled as a module the virtual CAN driver module is called vcan.ko
+
+Since Linux Kernel version 2.6.24 the vcan driver supports the Kernel
+netlink interface to create vcan network devices. The creation and
+removal of vcan network devices can be managed with the ip(8) tool::
+
+  - Create a virtual CAN network interface:
+       $ ip link add type vcan
+
+  - Create a virtual CAN network interface with a specific name 'vcan42':
+       $ ip link add dev vcan42 type vcan
+
+  - Remove a (virtual CAN) network interface 'vcan42':
+       $ ip link del vcan42
+
+
+The CAN Network Device Driver Interface
+---------------------------------------
+
+The CAN network device driver interface provides a generic interface
+to setup, configure and monitor CAN network devices. The user can then
+configure the CAN device, like setting the bit-timing parameters, via
+the netlink interface using the program "ip" from the "IPROUTE2"
+utility suite. The following chapter describes briefly how to use it.
+Furthermore, the interface uses a common data structure and exports a
+set of common functions, which all real CAN network device drivers
+should use. Please have a look to the SJA1000 or MSCAN driver to
+understand how to use them. The name of the module is can-dev.ko.
+
+
+Netlink interface to set/get devices properties
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The CAN device must be configured via netlink interface. The supported
+netlink message types are defined and briefly described in
+"include/linux/can/netlink.h". CAN link support for the program "ip"
+of the IPROUTE2 utility suite is available and it can be used as shown
+below:
+
+Setting CAN device properties::
+
+    $ ip link set can0 type can help
+    Usage: ip link set DEVICE type can
+        [ bitrate BITRATE [ sample-point SAMPLE-POINT] ] |
+        [ tq TQ prop-seg PROP_SEG phase-seg1 PHASE-SEG1
+          phase-seg2 PHASE-SEG2 [ sjw SJW ] ]
+
+        [ dbitrate BITRATE [ dsample-point SAMPLE-POINT] ] |
+        [ dtq TQ dprop-seg PROP_SEG dphase-seg1 PHASE-SEG1
+          dphase-seg2 PHASE-SEG2 [ dsjw SJW ] ]
+
+        [ loopback { on | off } ]
+        [ listen-only { on | off } ]
+        [ triple-sampling { on | off } ]
+        [ one-shot { on | off } ]
+        [ berr-reporting { on | off } ]
+        [ fd { on | off } ]
+        [ fd-non-iso { on | off } ]
+        [ presume-ack { on | off } ]
+
+        [ restart-ms TIME-MS ]
+        [ restart ]
+
+        Where: BITRATE       := { 1..1000000 }
+               SAMPLE-POINT  := { 0.000..0.999 }
+               TQ            := { NUMBER }
+               PROP-SEG      := { 1..8 }
+               PHASE-SEG1    := { 1..8 }
+               PHASE-SEG2    := { 1..8 }
+               SJW           := { 1..4 }
+               RESTART-MS    := { 0 | NUMBER }
+
+Display CAN device details and statistics::
+
+    $ ip -details -statistics link show can0
+    2: can0: <NOARP,UP,LOWER_UP,ECHO> mtu 16 qdisc pfifo_fast state UP qlen 10
+      link/can
+      can <TRIPLE-SAMPLING> state ERROR-ACTIVE restart-ms 100
+      bitrate 125000 sample_point 0.875
+      tq 125 prop-seg 6 phase-seg1 7 phase-seg2 2 sjw 1
+      sja1000: tseg1 1..16 tseg2 1..8 sjw 1..4 brp 1..64 brp-inc 1
+      clock 8000000
+      re-started bus-errors arbit-lost error-warn error-pass bus-off
+      41         17457      0          41         42         41
+      RX: bytes  packets  errors  dropped overrun mcast
+      140859     17608    17457   0       0       0
+      TX: bytes  packets  errors  dropped carrier collsns
+      861        112      0       41      0       0
+
+More info to the above output:
+
+"<TRIPLE-SAMPLING>"
+	Shows the list of selected CAN controller modes: LOOPBACK,
+	LISTEN-ONLY, or TRIPLE-SAMPLING.
+
+"state ERROR-ACTIVE"
+	The current state of the CAN controller: "ERROR-ACTIVE",
+	"ERROR-WARNING", "ERROR-PASSIVE", "BUS-OFF" or "STOPPED"
+
+"restart-ms 100"
+	Automatic restart delay time. If set to a non-zero value, a
+	restart of the CAN controller will be triggered automatically
+	in case of a bus-off condition after the specified delay time
+	in milliseconds. By default it's off.
+
+"bitrate 125000 sample-point 0.875"
+	Shows the real bit-rate in bits/sec and the sample-point in the
+	range 0.000..0.999. If the calculation of bit-timing parameters
+	is enabled in the kernel (CONFIG_CAN_CALC_BITTIMING=y), the
+	bit-timing can be defined by setting the "bitrate" argument.
+	Optionally the "sample-point" can be specified. By default it's
+	0.000 assuming CIA-recommended sample-points.
+
+"tq 125 prop-seg 6 phase-seg1 7 phase-seg2 2 sjw 1"
+	Shows the time quanta in ns, propagation segment, phase buffer
+	segment 1 and 2 and the synchronisation jump width in units of
+	tq. They allow to define the CAN bit-timing in a hardware
+	independent format as proposed by the Bosch CAN 2.0 spec (see
+	chapter 8 of http://www.semiconductors.bosch.de/pdf/can2spec.pdf).
+
+"sja1000: tseg1 1..16 tseg2 1..8 sjw 1..4 brp 1..64 brp-inc 1 clock 8000000"
+	Shows the bit-timing constants of the CAN controller, here the
+	"sja1000". The minimum and maximum values of the time segment 1
+	and 2, the synchronisation jump width in units of tq, the
+	bitrate pre-scaler and the CAN system clock frequency in Hz.
+	These constants could be used for user-defined (non-standard)
+	bit-timing calculation algorithms in user-space.
+
+"re-started bus-errors arbit-lost error-warn error-pass bus-off"
+	Shows the number of restarts, bus and arbitration lost errors,
+	and the state changes to the error-warning, error-passive and
+	bus-off state. RX overrun errors are listed in the "overrun"
+	field of the standard network statistics.
+
+Setting the CAN Bit-Timing
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The CAN bit-timing parameters can always be defined in a hardware
+independent format as proposed in the Bosch CAN 2.0 specification
+specifying the arguments "tq", "prop_seg", "phase_seg1", "phase_seg2"
+and "sjw"::
+
+    $ ip link set canX type can tq 125 prop-seg 6 \
+				phase-seg1 7 phase-seg2 2 sjw 1
+
+If the kernel option CONFIG_CAN_CALC_BITTIMING is enabled, CIA
+recommended CAN bit-timing parameters will be calculated if the bit-
+rate is specified with the argument "bitrate"::
+
+    $ ip link set canX type can bitrate 125000
+
+Note that this works fine for the most common CAN controllers with
+standard bit-rates but may *fail* for exotic bit-rates or CAN system
+clock frequencies. Disabling CONFIG_CAN_CALC_BITTIMING saves some
+space and allows user-space tools to solely determine and set the
+bit-timing parameters. The CAN controller specific bit-timing
+constants can be used for that purpose. They are listed by the
+following command::
+
+    $ ip -details link show can0
+    ...
+      sja1000: clock 8000000 tseg1 1..16 tseg2 1..8 sjw 1..4 brp 1..64 brp-inc 1
+
+
+Starting and Stopping the CAN Network Device
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A CAN network device is started or stopped as usual with the command
+"ifconfig canX up/down" or "ip link set canX up/down". Be aware that
+you *must* define proper bit-timing parameters for real CAN devices
+before you can start it to avoid error-prone default settings::
+
+    $ ip link set canX up type can bitrate 125000
+
+A device may enter the "bus-off" state if too many errors occurred on
+the CAN bus. Then no more messages are received or sent. An automatic
+bus-off recovery can be enabled by setting the "restart-ms" to a
+non-zero value, e.g.::
+
+    $ ip link set canX type can restart-ms 100
+
+Alternatively, the application may realize the "bus-off" condition
+by monitoring CAN error message frames and do a restart when
+appropriate with the command::
+
+    $ ip link set canX type can restart
+
+Note that a restart will also create a CAN error message frame (see
+also :ref:`socketcan-network-problem-notifications`).
+
+
+.. _socketcan-can-fd-driver:
+
+CAN FD (Flexible Data Rate) Driver Support
+------------------------------------------
+
+CAN FD capable CAN controllers support two different bitrates for the
+arbitration phase and the payload phase of the CAN FD frame. Therefore a
+second bit timing has to be specified in order to enable the CAN FD bitrate.
+
+Additionally CAN FD capable CAN controllers support up to 64 bytes of
+payload. The representation of this length in can_frame.can_dlc and
+canfd_frame.len for userspace applications and inside the Linux network
+layer is a plain value from 0 .. 64 instead of the CAN 'data length code'.
+The data length code was a 1:1 mapping to the payload length in the legacy
+CAN frames anyway. The payload length to the bus-relevant DLC mapping is
+only performed inside the CAN drivers, preferably with the helper
+functions can_dlc2len() and can_len2dlc().
+
+The CAN netdevice driver capabilities can be distinguished by the network
+devices maximum transfer unit (MTU)::
+
+  MTU = 16 (CAN_MTU)   => sizeof(struct can_frame)   => 'legacy' CAN device
+  MTU = 72 (CANFD_MTU) => sizeof(struct canfd_frame) => CAN FD capable device
+
+The CAN device MTU can be retrieved e.g. with a SIOCGIFMTU ioctl() syscall.
+N.B. CAN FD capable devices can also handle and send legacy CAN frames.
+
+When configuring CAN FD capable CAN controllers an additional 'data' bitrate
+has to be set. This bitrate for the data phase of the CAN FD frame has to be
+at least the bitrate which was configured for the arbitration phase. This
+second bitrate is specified analogue to the first bitrate but the bitrate
+setting keywords for the 'data' bitrate start with 'd' e.g. dbitrate,
+dsample-point, dsjw or dtq and similar settings. When a data bitrate is set
+within the configuration process the controller option "fd on" can be
+specified to enable the CAN FD mode in the CAN controller. This controller
+option also switches the device MTU to 72 (CANFD_MTU).
+
+The first CAN FD specification presented as whitepaper at the International
+CAN Conference 2012 needed to be improved for data integrity reasons.
+Therefore two CAN FD implementations have to be distinguished today:
+
+- ISO compliant:     The ISO 11898-1:2015 CAN FD implementation (default)
+- non-ISO compliant: The CAN FD implementation following the 2012 whitepaper
+
+Finally there are three types of CAN FD controllers:
+
+1. ISO compliant (fixed)
+2. non-ISO compliant (fixed, like the M_CAN IP core v3.0.1 in m_can.c)
+3. ISO/non-ISO CAN FD controllers (switchable, like the PEAK PCAN-USB FD)
+
+The current ISO/non-ISO mode is announced by the CAN controller driver via
+netlink and displayed by the 'ip' tool (controller option FD-NON-ISO).
+The ISO/non-ISO-mode can be altered by setting 'fd-non-iso {on|off}' for
+switchable CAN FD controllers only.
+
+Example configuring 500 kbit/s arbitration bitrate and 4 Mbit/s data bitrate::
+
+    $ ip link set can0 up type can bitrate 500000 sample-point 0.75 \
+                                   dbitrate 4000000 dsample-point 0.8 fd on
+    $ ip -details link show can0
+    5: can0: <NOARP,UP,LOWER_UP,ECHO> mtu 72 qdisc pfifo_fast state UNKNOWN \
+             mode DEFAULT group default qlen 10
+    link/can  promiscuity 0
+    can <FD> state ERROR-ACTIVE (berr-counter tx 0 rx 0) restart-ms 0
+          bitrate 500000 sample-point 0.750
+          tq 50 prop-seg 14 phase-seg1 15 phase-seg2 10 sjw 1
+          pcan_usb_pro_fd: tseg1 1..64 tseg2 1..16 sjw 1..16 brp 1..1024 \
+          brp-inc 1
+          dbitrate 4000000 dsample-point 0.800
+          dtq 12 dprop-seg 7 dphase-seg1 8 dphase-seg2 4 dsjw 1
+          pcan_usb_pro_fd: dtseg1 1..16 dtseg2 1..8 dsjw 1..4 dbrp 1..1024 \
+          dbrp-inc 1
+          clock 80000000
+
+Example when 'fd-non-iso on' is added on this switchable CAN FD adapter::
+
+   can <FD,FD-NON-ISO> state ERROR-ACTIVE (berr-counter tx 0 rx 0) restart-ms 0
+
+
+Supported CAN Hardware
+----------------------
+
+Please check the "Kconfig" file in "drivers/net/can" to get an actual
+list of the support CAN hardware. On the SocketCAN project website
+(see :ref:`socketcan-resources`) there might be further drivers available, also for
+older kernel versions.
+
+
+.. _socketcan-resources:
+
+SocketCAN Resources
+===================
+
+The Linux CAN / SocketCAN project resources (project site / mailing list)
+are referenced in the MAINTAINERS file in the Linux source tree.
+Search for CAN NETWORK [LAYERS|DRIVERS].
+
+Credits
+=======
+
+- Oliver Hartkopp (PF_CAN core, filters, drivers, bcm, SJA1000 driver)
+- Urs Thuermann (PF_CAN core, kernel integration, socket interfaces, raw, vcan)
+- Jan Kizka (RT-SocketCAN core, Socket-API reconciliation)
+- Wolfgang Grandegger (RT-SocketCAN core & drivers, Raw Socket-API reviews, CAN device driver interface, MSCAN driver)
+- Robert Schwebel (design reviews, PTXdist integration)
+- Marc Kleine-Budde (design reviews, Kernel 2.6 cleanups, drivers)
+- Benedikt Spranger (reviews)
+- Thomas Gleixner (LKML reviews, coding style, posting hints)
+- Andrey Volkov (kernel subtree structure, ioctls, MSCAN driver)
+- Matthias Brukner (first SJA1000 CAN netdevice implementation Q2/2003)
+- Klaus Hitschler (PEAK driver integration)
+- Uwe Koppe (CAN netdevices with PF_PACKET approach)
+- Michael Schulze (driver layer loopback requirement, RT CAN drivers review)
+- Pavel Pisa (Bit-timing calculation)
+- Sascha Hauer (SJA1000 platform driver)
+- Sebastian Haas (SJA1000 EMS PCI driver)
+- Markus Plessing (SJA1000 EMS PCI driver)
+- Per Dalen (SJA1000 Kvaser PCI driver)
+- Sam Ravnborg (reviews, coding style, kbuild help)
diff --git a/Documentation/networking/can.txt b/Documentation/networking/can.txt
deleted file mode 100644
index aa15b9ee2e70..000000000000
--- a/Documentation/networking/can.txt
+++ /dev/null
@@ -1,1308 +0,0 @@
-============================================================================
-
-can.txt
-
-Readme file for the Controller Area Network Protocol Family (aka SocketCAN)
-
-This file contains
-
-  1 Overview / What is SocketCAN
-
-  2 Motivation / Why using the socket API
-
-  3 SocketCAN concept
-    3.1 receive lists
-    3.2 local loopback of sent frames
-    3.3 network problem notifications
-
-  4 How to use SocketCAN
-    4.1 RAW protocol sockets with can_filters (SOCK_RAW)
-      4.1.1 RAW socket option CAN_RAW_FILTER
-      4.1.2 RAW socket option CAN_RAW_ERR_FILTER
-      4.1.3 RAW socket option CAN_RAW_LOOPBACK
-      4.1.4 RAW socket option CAN_RAW_RECV_OWN_MSGS
-      4.1.5 RAW socket option CAN_RAW_FD_FRAMES
-      4.1.6 RAW socket option CAN_RAW_JOIN_FILTERS
-      4.1.7 RAW socket returned message flags
-    4.2 Broadcast Manager protocol sockets (SOCK_DGRAM)
-      4.2.1 Broadcast Manager operations
-      4.2.2 Broadcast Manager message flags
-      4.2.3 Broadcast Manager transmission timers
-      4.2.4 Broadcast Manager message sequence transmission
-      4.2.5 Broadcast Manager receive filter timers
-      4.2.6 Broadcast Manager multiplex message receive filter
-      4.2.7 Broadcast Manager CAN FD support
-    4.3 connected transport protocols (SOCK_SEQPACKET)
-    4.4 unconnected transport protocols (SOCK_DGRAM)
-
-  5 SocketCAN core module
-    5.1 can.ko module params
-    5.2 procfs content
-    5.3 writing own CAN protocol modules
-
-  6 CAN network drivers
-    6.1 general settings
-    6.2 local loopback of sent frames
-    6.3 CAN controller hardware filters
-    6.4 The virtual CAN driver (vcan)
-    6.5 The CAN network device driver interface
-      6.5.1 Netlink interface to set/get devices properties
-      6.5.2 Setting the CAN bit-timing
-      6.5.3 Starting and stopping the CAN network device
-    6.6 CAN FD (flexible data rate) driver support
-    6.7 supported CAN hardware
-
-  7 SocketCAN resources
-
-  8 Credits
-
-============================================================================
-
-1. Overview / What is SocketCAN
---------------------------------
-
-The socketcan package is an implementation of CAN protocols
-(Controller Area Network) for Linux.  CAN is a networking technology
-which has widespread use in automation, embedded devices, and
-automotive fields.  While there have been other CAN implementations
-for Linux based on character devices, SocketCAN uses the Berkeley
-socket API, the Linux network stack and implements the CAN device
-drivers as network interfaces.  The CAN socket API has been designed
-as similar as possible to the TCP/IP protocols to allow programmers,
-familiar with network programming, to easily learn how to use CAN
-sockets.
-
-2. Motivation / Why using the socket API
-----------------------------------------
-
-There have been CAN implementations for Linux before SocketCAN so the
-question arises, why we have started another project.  Most existing
-implementations come as a device driver for some CAN hardware, they
-are based on character devices and provide comparatively little
-functionality.  Usually, there is only a hardware-specific device
-driver which provides a character device interface to send and
-receive raw CAN frames, directly to/from the controller hardware.
-Queueing of frames and higher-level transport protocols like ISO-TP
-have to be implemented in user space applications.  Also, most
-character-device implementations support only one single process to
-open the device at a time, similar to a serial interface.  Exchanging
-the CAN controller requires employment of another device driver and
-often the need for adaption of large parts of the application to the
-new driver's API.
-
-SocketCAN was designed to overcome all of these limitations.  A new
-protocol family has been implemented which provides a socket interface
-to user space applications and which builds upon the Linux network
-layer, enabling use all of the provided queueing functionality.  A device
-driver for CAN controller hardware registers itself with the Linux
-network layer as a network device, so that CAN frames from the
-controller can be passed up to the network layer and on to the CAN
-protocol family module and also vice-versa.  Also, the protocol family
-module provides an API for transport protocol modules to register, so
-that any number of transport protocols can be loaded or unloaded
-dynamically.  In fact, the can core module alone does not provide any
-protocol and cannot be used without loading at least one additional
-protocol module.  Multiple sockets can be opened at the same time,
-on different or the same protocol module and they can listen/send
-frames on different or the same CAN IDs.  Several sockets listening on
-the same interface for frames with the same CAN ID are all passed the
-same received matching CAN frames.  An application wishing to
-communicate using a specific transport protocol, e.g. ISO-TP, just
-selects that protocol when opening the socket, and then can read and
-write application data byte streams, without having to deal with
-CAN-IDs, frames, etc.
-
-Similar functionality visible from user-space could be provided by a
-character device, too, but this would lead to a technically inelegant
-solution for a couple of reasons:
-
-* Intricate usage.  Instead of passing a protocol argument to
-  socket(2) and using bind(2) to select a CAN interface and CAN ID, an
-  application would have to do all these operations using ioctl(2)s.
-
-* Code duplication.  A character device cannot make use of the Linux
-  network queueing code, so all that code would have to be duplicated
-  for CAN networking.
-
-* Abstraction.  In most existing character-device implementations, the
-  hardware-specific device driver for a CAN controller directly
-  provides the character device for the application to work with.
-  This is at least very unusual in Unix systems for both, char and
-  block devices.  For example you don't have a character device for a
-  certain UART of a serial interface, a certain sound chip in your
-  computer, a SCSI or IDE controller providing access to your hard
-  disk or tape streamer device.  Instead, you have abstraction layers
-  which provide a unified character or block device interface to the
-  application on the one hand, and a interface for hardware-specific
-  device drivers on the other hand.  These abstractions are provided
-  by subsystems like the tty layer, the audio subsystem or the SCSI
-  and IDE subsystems for the devices mentioned above.
-
-  The easiest way to implement a CAN device driver is as a character
-  device without such a (complete) abstraction layer, as is done by most
-  existing drivers.  The right way, however, would be to add such a
-  layer with all the functionality like registering for certain CAN
-  IDs, supporting several open file descriptors and (de)multiplexing
-  CAN frames between them, (sophisticated) queueing of CAN frames, and
-  providing an API for device drivers to register with.  However, then
-  it would be no more difficult, or may be even easier, to use the
-  networking framework provided by the Linux kernel, and this is what
-  SocketCAN does.
-
-  The use of the networking framework of the Linux kernel is just the
-  natural and most appropriate way to implement CAN for Linux.
-
-3. SocketCAN concept
----------------------
-
-  As described in chapter 2 it is the main goal of SocketCAN to
-  provide a socket interface to user space applications which builds
-  upon the Linux network layer. In contrast to the commonly known
-  TCP/IP and ethernet networking, the CAN bus is a broadcast-only(!)
-  medium that has no MAC-layer addressing like ethernet. The CAN-identifier
-  (can_id) is used for arbitration on the CAN-bus. Therefore the CAN-IDs
-  have to be chosen uniquely on the bus. When designing a CAN-ECU
-  network the CAN-IDs are mapped to be sent by a specific ECU.
-  For this reason a CAN-ID can be treated best as a kind of source address.
-
-  3.1 receive lists
-
-  The network transparent access of multiple applications leads to the
-  problem that different applications may be interested in the same
-  CAN-IDs from the same CAN network interface. The SocketCAN core
-  module - which implements the protocol family CAN - provides several
-  high efficient receive lists for this reason. If e.g. a user space
-  application opens a CAN RAW socket, the raw protocol module itself
-  requests the (range of) CAN-IDs from the SocketCAN core that are
-  requested by the user. The subscription and unsubscription of
-  CAN-IDs can be done for specific CAN interfaces or for all(!) known
-  CAN interfaces with the can_rx_(un)register() functions provided to
-  CAN protocol modules by the SocketCAN core (see chapter 5).
-  To optimize the CPU usage at runtime the receive lists are split up
-  into several specific lists per device that match the requested
-  filter complexity for a given use-case.
-
-  3.2 local loopback of sent frames
-
-  As known from other networking concepts the data exchanging
-  applications may run on the same or different nodes without any
-  change (except for the according addressing information):
-
-         ___   ___   ___                   _______   ___
-        | _ | | _ | | _ |                 | _   _ | | _ |
-        ||A|| ||B|| ||C||                 ||A| |B|| ||C||
-        |___| |___| |___|                 |_______| |___|
-          |     |     |                       |       |
-        -----------------(1)- CAN bus -(2)---------------
-
-  To ensure that application A receives the same information in the
-  example (2) as it would receive in example (1) there is need for
-  some kind of local loopback of the sent CAN frames on the appropriate
-  node.
-
-  The Linux network devices (by default) just can handle the
-  transmission and reception of media dependent frames. Due to the
-  arbitration on the CAN bus the transmission of a low prio CAN-ID
-  may be delayed by the reception of a high prio CAN frame. To
-  reflect the correct* traffic on the node the loopback of the sent
-  data has to be performed right after a successful transmission. If
-  the CAN network interface is not capable of performing the loopback for
-  some reason the SocketCAN core can do this task as a fallback solution.
-  See chapter 6.2 for details (recommended).
-
-  The loopback functionality is enabled by default to reflect standard
-  networking behaviour for CAN applications. Due to some requests from
-  the RT-SocketCAN group the loopback optionally may be disabled for each
-  separate socket. See sockopts from the CAN RAW sockets in chapter 4.1.
-
-  * = you really like to have this when you're running analyser tools
-      like 'candump' or 'cansniffer' on the (same) node.
-
-  3.3 network problem notifications
-
-  The use of the CAN bus may lead to several problems on the physical
-  and media access control layer. Detecting and logging of these lower
-  layer problems is a vital requirement for CAN users to identify
-  hardware issues on the physical transceiver layer as well as
-  arbitration problems and error frames caused by the different
-  ECUs. The occurrence of detected errors are important for diagnosis
-  and have to be logged together with the exact timestamp. For this
-  reason the CAN interface driver can generate so called Error Message
-  Frames that can optionally be passed to the user application in the
-  same way as other CAN frames. Whenever an error on the physical layer
-  or the MAC layer is detected (e.g. by the CAN controller) the driver
-  creates an appropriate error message frame. Error messages frames can
-  be requested by the user application using the common CAN filter
-  mechanisms. Inside this filter definition the (interested) type of
-  errors may be selected. The reception of error messages is disabled
-  by default. The format of the CAN error message frame is briefly
-  described in the Linux header file "include/uapi/linux/can/error.h".
-
-4. How to use SocketCAN
-------------------------
-
-  Like TCP/IP, you first need to open a socket for communicating over a
-  CAN network. Since SocketCAN implements a new protocol family, you
-  need to pass PF_CAN as the first argument to the socket(2) system
-  call. Currently, there are two CAN protocols to choose from, the raw
-  socket protocol and the broadcast manager (BCM). So to open a socket,
-  you would write
-
-    s = socket(PF_CAN, SOCK_RAW, CAN_RAW);
-
-  and
-
-    s = socket(PF_CAN, SOCK_DGRAM, CAN_BCM);
-
-  respectively.  After the successful creation of the socket, you would
-  normally use the bind(2) system call to bind the socket to a CAN
-  interface (which is different from TCP/IP due to different addressing
-  - see chapter 3). After binding (CAN_RAW) or connecting (CAN_BCM)
-  the socket, you can read(2) and write(2) from/to the socket or use
-  send(2), sendto(2), sendmsg(2) and the recv* counterpart operations
-  on the socket as usual. There are also CAN specific socket options
-  described below.
-
-  The basic CAN frame structure and the sockaddr structure are defined
-  in include/linux/can.h:
-
-    struct can_frame {
-            canid_t can_id;  /* 32 bit CAN_ID + EFF/RTR/ERR flags */
-            __u8    can_dlc; /* frame payload length in byte (0 .. 8) */
-            __u8    __pad;   /* padding */
-            __u8    __res0;  /* reserved / padding */
-            __u8    __res1;  /* reserved / padding */
-            __u8    data[8] __attribute__((aligned(8)));
-    };
-
-  The alignment of the (linear) payload data[] to a 64bit boundary
-  allows the user to define their own structs and unions to easily access
-  the CAN payload. There is no given byteorder on the CAN bus by
-  default. A read(2) system call on a CAN_RAW socket transfers a
-  struct can_frame to the user space.
-
-  The sockaddr_can structure has an interface index like the
-  PF_PACKET socket, that also binds to a specific interface:
-
-    struct sockaddr_can {
-            sa_family_t can_family;
-            int         can_ifindex;
-            union {
-                    /* transport protocol class address info (e.g. ISOTP) */
-                    struct { canid_t rx_id, tx_id; } tp;
-
-                    /* reserved for future CAN protocols address information */
-            } can_addr;
-    };
-
-  To determine the interface index an appropriate ioctl() has to
-  be used (example for CAN_RAW sockets without error checking):
-
-    int s;
-    struct sockaddr_can addr;
-    struct ifreq ifr;
-
-    s = socket(PF_CAN, SOCK_RAW, CAN_RAW);
-
-    strcpy(ifr.ifr_name, "can0" );
-    ioctl(s, SIOCGIFINDEX, &ifr);
-
-    addr.can_family = AF_CAN;
-    addr.can_ifindex = ifr.ifr_ifindex;
-
-    bind(s, (struct sockaddr *)&addr, sizeof(addr));
-
-    (..)
-
-  To bind a socket to all(!) CAN interfaces the interface index must
-  be 0 (zero). In this case the socket receives CAN frames from every
-  enabled CAN interface. To determine the originating CAN interface
-  the system call recvfrom(2) may be used instead of read(2). To send
-  on a socket that is bound to 'any' interface sendto(2) is needed to
-  specify the outgoing interface.
-
-  Reading CAN frames from a bound CAN_RAW socket (see above) consists
-  of reading a struct can_frame:
-
-    struct can_frame frame;
-
-    nbytes = read(s, &frame, sizeof(struct can_frame));
-
-    if (nbytes < 0) {
-            perror("can raw socket read");
-            return 1;
-    }
-
-    /* paranoid check ... */
-    if (nbytes < sizeof(struct can_frame)) {
-            fprintf(stderr, "read: incomplete CAN frame\n");
-            return 1;
-    }
-
-    /* do something with the received CAN frame */
-
-  Writing CAN frames can be done similarly, with the write(2) system call:
-
-    nbytes = write(s, &frame, sizeof(struct can_frame));
-
-  When the CAN interface is bound to 'any' existing CAN interface
-  (addr.can_ifindex = 0) it is recommended to use recvfrom(2) if the
-  information about the originating CAN interface is needed:
-
-    struct sockaddr_can addr;
-    struct ifreq ifr;
-    socklen_t len = sizeof(addr);
-    struct can_frame frame;
-
-    nbytes = recvfrom(s, &frame, sizeof(struct can_frame),
-                      0, (struct sockaddr*)&addr, &len);
-
-    /* get interface name of the received CAN frame */
-    ifr.ifr_ifindex = addr.can_ifindex;
-    ioctl(s, SIOCGIFNAME, &ifr);
-    printf("Received a CAN frame from interface %s", ifr.ifr_name);
-
-  To write CAN frames on sockets bound to 'any' CAN interface the
-  outgoing interface has to be defined certainly.
-
-    strcpy(ifr.ifr_name, "can0");
-    ioctl(s, SIOCGIFINDEX, &ifr);
-    addr.can_ifindex = ifr.ifr_ifindex;
-    addr.can_family  = AF_CAN;
-
-    nbytes = sendto(s, &frame, sizeof(struct can_frame),
-                    0, (struct sockaddr*)&addr, sizeof(addr));
-
-  An accurate timestamp can be obtained with an ioctl(2) call after reading
-  a message from the socket:
-
-    struct timeval tv;
-    ioctl(s, SIOCGSTAMP, &tv);
-
-  The timestamp has a resolution of one microsecond and is set automatically
-  at the reception of a CAN frame.
-
-  Remark about CAN FD (flexible data rate) support:
-
-  Generally the handling of CAN FD is very similar to the formerly described
-  examples. The new CAN FD capable CAN controllers support two different
-  bitrates for the arbitration phase and the payload phase of the CAN FD frame
-  and up to 64 bytes of payload. This extended payload length breaks all the
-  kernel interfaces (ABI) which heavily rely on the CAN frame with fixed eight
-  bytes of payload (struct can_frame) like the CAN_RAW socket. Therefore e.g.
-  the CAN_RAW socket supports a new socket option CAN_RAW_FD_FRAMES that
-  switches the socket into a mode that allows the handling of CAN FD frames
-  and (legacy) CAN frames simultaneously (see section 4.1.5).
-
-  The struct canfd_frame is defined in include/linux/can.h:
-
-    struct canfd_frame {
-            canid_t can_id;  /* 32 bit CAN_ID + EFF/RTR/ERR flags */
-            __u8    len;     /* frame payload length in byte (0 .. 64) */
-            __u8    flags;   /* additional flags for CAN FD */
-            __u8    __res0;  /* reserved / padding */
-            __u8    __res1;  /* reserved / padding */
-            __u8    data[64] __attribute__((aligned(8)));
-    };
-
-  The struct canfd_frame and the existing struct can_frame have the can_id,
-  the payload length and the payload data at the same offset inside their
-  structures. This allows to handle the different structures very similar.
-  When the content of a struct can_frame is copied into a struct canfd_frame
-  all structure elements can be used as-is - only the data[] becomes extended.
-
-  When introducing the struct canfd_frame it turned out that the data length
-  code (DLC) of the struct can_frame was used as a length information as the
-  length and the DLC has a 1:1 mapping in the range of 0 .. 8. To preserve
-  the easy handling of the length information the canfd_frame.len element
-  contains a plain length value from 0 .. 64. So both canfd_frame.len and
-  can_frame.can_dlc are equal and contain a length information and no DLC.
-  For details about the distinction of CAN and CAN FD capable devices and
-  the mapping to the bus-relevant data length code (DLC), see chapter 6.6.
-
-  The length of the two CAN(FD) frame structures define the maximum transfer
-  unit (MTU) of the CAN(FD) network interface and skbuff data length. Two
-  definitions are specified for CAN specific MTUs in include/linux/can.h :
-
-  #define CAN_MTU   (sizeof(struct can_frame))   == 16  => 'legacy' CAN frame
-  #define CANFD_MTU (sizeof(struct canfd_frame)) == 72  => CAN FD frame
-
-  4.1 RAW protocol sockets with can_filters (SOCK_RAW)
-
-  Using CAN_RAW sockets is extensively comparable to the commonly
-  known access to CAN character devices. To meet the new possibilities
-  provided by the multi user SocketCAN approach, some reasonable
-  defaults are set at RAW socket binding time:
-
-  - The filters are set to exactly one filter receiving everything
-  - The socket only receives valid data frames (=> no error message frames)
-  - The loopback of sent CAN frames is enabled (see chapter 3.2)
-  - The socket does not receive its own sent frames (in loopback mode)
-
-  These default settings may be changed before or after binding the socket.
-  To use the referenced definitions of the socket options for CAN_RAW
-  sockets, include <linux/can/raw.h>.
-
-  4.1.1 RAW socket option CAN_RAW_FILTER
-
-  The reception of CAN frames using CAN_RAW sockets can be controlled
-  by defining 0 .. n filters with the CAN_RAW_FILTER socket option.
-
-  The CAN filter structure is defined in include/linux/can.h:
-
-    struct can_filter {
-            canid_t can_id;
-            canid_t can_mask;
-    };
-
-  A filter matches, when
-
-    <received_can_id> & mask == can_id & mask
-
-  which is analogous to known CAN controllers hardware filter semantics.
-  The filter can be inverted in this semantic, when the CAN_INV_FILTER
-  bit is set in can_id element of the can_filter structure. In
-  contrast to CAN controller hardware filters the user may set 0 .. n
-  receive filters for each open socket separately:
-
-    struct can_filter rfilter[2];
-
-    rfilter[0].can_id   = 0x123;
-    rfilter[0].can_mask = CAN_SFF_MASK;
-    rfilter[1].can_id   = 0x200;
-    rfilter[1].can_mask = 0x700;
-
-    setsockopt(s, SOL_CAN_RAW, CAN_RAW_FILTER, &rfilter, sizeof(rfilter));
-
-  To disable the reception of CAN frames on the selected CAN_RAW socket:
-
-    setsockopt(s, SOL_CAN_RAW, CAN_RAW_FILTER, NULL, 0);
-
-  To set the filters to zero filters is quite obsolete as to not read
-  data causes the raw socket to discard the received CAN frames. But
-  having this 'send only' use-case we may remove the receive list in the
-  Kernel to save a little (really a very little!) CPU usage.
-
-  4.1.1.1 CAN filter usage optimisation
-
-  The CAN filters are processed in per-device filter lists at CAN frame
-  reception time. To reduce the number of checks that need to be performed
-  while walking through the filter lists the CAN core provides an optimized
-  filter handling when the filter subscription focusses on a single CAN ID.
-
-  For the possible 2048 SFF CAN identifiers the identifier is used as an index
-  to access the corresponding subscription list without any further checks.
-  For the 2^29 possible EFF CAN identifiers a 10 bit XOR folding is used as
-  hash function to retrieve the EFF table index.
-
-  To benefit from the optimized filters for single CAN identifiers the
-  CAN_SFF_MASK or CAN_EFF_MASK have to be set into can_filter.mask together
-  with set CAN_EFF_FLAG and CAN_RTR_FLAG bits. A set CAN_EFF_FLAG bit in the
-  can_filter.mask makes clear that it matters whether a SFF or EFF CAN ID is
-  subscribed. E.g. in the example from above
-
-    rfilter[0].can_id   = 0x123;
-    rfilter[0].can_mask = CAN_SFF_MASK;
-
-  both SFF frames with CAN ID 0x123 and EFF frames with 0xXXXXX123 can pass.
-
-  To filter for only 0x123 (SFF) and 0x12345678 (EFF) CAN identifiers the
-  filter has to be defined in this way to benefit from the optimized filters:
-
-    struct can_filter rfilter[2];
-
-    rfilter[0].can_id   = 0x123;
-    rfilter[0].can_mask = (CAN_EFF_FLAG | CAN_RTR_FLAG | CAN_SFF_MASK);
-    rfilter[1].can_id   = 0x12345678 | CAN_EFF_FLAG;
-    rfilter[1].can_mask = (CAN_EFF_FLAG | CAN_RTR_FLAG | CAN_EFF_MASK);
-
-    setsockopt(s, SOL_CAN_RAW, CAN_RAW_FILTER, &rfilter, sizeof(rfilter));
-
-  4.1.2 RAW socket option CAN_RAW_ERR_FILTER
-
-  As described in chapter 3.3 the CAN interface driver can generate so
-  called Error Message Frames that can optionally be passed to the user
-  application in the same way as other CAN frames. The possible
-  errors are divided into different error classes that may be filtered
-  using the appropriate error mask. To register for every possible
-  error condition CAN_ERR_MASK can be used as value for the error mask.
-  The values for the error mask are defined in linux/can/error.h .
-
-    can_err_mask_t err_mask = ( CAN_ERR_TX_TIMEOUT | CAN_ERR_BUSOFF );
-
-    setsockopt(s, SOL_CAN_RAW, CAN_RAW_ERR_FILTER,
-               &err_mask, sizeof(err_mask));
-
-  4.1.3 RAW socket option CAN_RAW_LOOPBACK
-
-  To meet multi user needs the local loopback is enabled by default
-  (see chapter 3.2 for details). But in some embedded use-cases
-  (e.g. when only one application uses the CAN bus) this loopback
-  functionality can be disabled (separately for each socket):
-
-    int loopback = 0; /* 0 = disabled, 1 = enabled (default) */
-
-    setsockopt(s, SOL_CAN_RAW, CAN_RAW_LOOPBACK, &loopback, sizeof(loopback));
-
-  4.1.4 RAW socket option CAN_RAW_RECV_OWN_MSGS
-
-  When the local loopback is enabled, all the sent CAN frames are
-  looped back to the open CAN sockets that registered for the CAN
-  frames' CAN-ID on this given interface to meet the multi user
-  needs. The reception of the CAN frames on the same socket that was
-  sending the CAN frame is assumed to be unwanted and therefore
-  disabled by default. This default behaviour may be changed on
-  demand:
-
-    int recv_own_msgs = 1; /* 0 = disabled (default), 1 = enabled */
-
-    setsockopt(s, SOL_CAN_RAW, CAN_RAW_RECV_OWN_MSGS,
-               &recv_own_msgs, sizeof(recv_own_msgs));
-
-  4.1.5 RAW socket option CAN_RAW_FD_FRAMES
-
-  CAN FD support in CAN_RAW sockets can be enabled with a new socket option
-  CAN_RAW_FD_FRAMES which is off by default. When the new socket option is
-  not supported by the CAN_RAW socket (e.g. on older kernels), switching the
-  CAN_RAW_FD_FRAMES option returns the error -ENOPROTOOPT.
-
-  Once CAN_RAW_FD_FRAMES is enabled the application can send both CAN frames
-  and CAN FD frames. OTOH the application has to handle CAN and CAN FD frames
-  when reading from the socket.
-
-    CAN_RAW_FD_FRAMES enabled:  CAN_MTU and CANFD_MTU are allowed
-    CAN_RAW_FD_FRAMES disabled: only CAN_MTU is allowed (default)
-
-  Example:
-    [ remember: CANFD_MTU == sizeof(struct canfd_frame) ]
-
-    struct canfd_frame cfd;
-
-    nbytes = read(s, &cfd, CANFD_MTU);
-
-    if (nbytes == CANFD_MTU) {
-            printf("got CAN FD frame with length %d\n", cfd.len);
-	    /* cfd.flags contains valid data */
-    } else if (nbytes == CAN_MTU) {
-            printf("got legacy CAN frame with length %d\n", cfd.len);
-	    /* cfd.flags is undefined */
-    } else {
-            fprintf(stderr, "read: invalid CAN(FD) frame\n");
-            return 1;
-    }
-
-    /* the content can be handled independently from the received MTU size */
-
-    printf("can_id: %X data length: %d data: ", cfd.can_id, cfd.len);
-    for (i = 0; i < cfd.len; i++)
-            printf("%02X ", cfd.data[i]);
-
-  When reading with size CANFD_MTU only returns CAN_MTU bytes that have
-  been received from the socket a legacy CAN frame has been read into the
-  provided CAN FD structure. Note that the canfd_frame.flags data field is
-  not specified in the struct can_frame and therefore it is only valid in
-  CANFD_MTU sized CAN FD frames.
-
-  Implementation hint for new CAN applications:
-
-  To build a CAN FD aware application use struct canfd_frame as basic CAN
-  data structure for CAN_RAW based applications. When the application is
-  executed on an older Linux kernel and switching the CAN_RAW_FD_FRAMES
-  socket option returns an error: No problem. You'll get legacy CAN frames
-  or CAN FD frames and can process them the same way.
-
-  When sending to CAN devices make sure that the device is capable to handle
-  CAN FD frames by checking if the device maximum transfer unit is CANFD_MTU.
-  The CAN device MTU can be retrieved e.g. with a SIOCGIFMTU ioctl() syscall.
-
-  4.1.6 RAW socket option CAN_RAW_JOIN_FILTERS
-
-  The CAN_RAW socket can set multiple CAN identifier specific filters that
-  lead to multiple filters in the af_can.c filter processing. These filters
-  are indenpendent from each other which leads to logical OR'ed filters when
-  applied (see 4.1.1).
-
-  This socket option joines the given CAN filters in the way that only CAN
-  frames are passed to user space that matched *all* given CAN filters. The
-  semantic for the applied filters is therefore changed to a logical AND.
-
-  This is useful especially when the filterset is a combination of filters
-  where the CAN_INV_FILTER flag is set in order to notch single CAN IDs or
-  CAN ID ranges from the incoming traffic.
-
-  4.1.7 RAW socket returned message flags
-
-  When using recvmsg() call, the msg->msg_flags may contain following flags:
-
-    MSG_DONTROUTE: set when the received frame was created on the local host.
-
-    MSG_CONFIRM: set when the frame was sent via the socket it is received on.
-      This flag can be interpreted as a 'transmission confirmation' when the
-      CAN driver supports the echo of frames on driver level, see 3.2 and 6.2.
-      In order to receive such messages, CAN_RAW_RECV_OWN_MSGS must be set.
-
-  4.2 Broadcast Manager protocol sockets (SOCK_DGRAM)
-
-  The Broadcast Manager protocol provides a command based configuration
-  interface to filter and send (e.g. cyclic) CAN messages in kernel space.
-
-  Receive filters can be used to down sample frequent messages; detect events
-  such as message contents changes, packet length changes, and do time-out
-  monitoring of received messages.
-
-  Periodic transmission tasks of CAN frames or a sequence of CAN frames can be
-  created and modified at runtime; both the message content and the two
-  possible transmit intervals can be altered.
-
-  A BCM socket is not intended for sending individual CAN frames using the
-  struct can_frame as known from the CAN_RAW socket. Instead a special BCM
-  configuration message is defined. The basic BCM configuration message used
-  to communicate with the broadcast manager and the available operations are
-  defined in the linux/can/bcm.h include. The BCM message consists of a
-  message header with a command ('opcode') followed by zero or more CAN frames.
-  The broadcast manager sends responses to user space in the same form:
-
-    struct bcm_msg_head {
-            __u32 opcode;                   /* command */
-            __u32 flags;                    /* special flags */
-            __u32 count;                    /* run 'count' times with ival1 */
-            struct timeval ival1, ival2;    /* count and subsequent interval */
-            canid_t can_id;                 /* unique can_id for task */
-            __u32 nframes;                  /* number of can_frames following */
-            struct can_frame frames[0];
-    };
-
-  The aligned payload 'frames' uses the same basic CAN frame structure defined
-  at the beginning of section 4 and in the include/linux/can.h include. All
-  messages to the broadcast manager from user space have this structure.
-
-  Note a CAN_BCM socket must be connected instead of bound after socket
-  creation (example without error checking):
-
-    int s;
-    struct sockaddr_can addr;
-    struct ifreq ifr;
-
-    s = socket(PF_CAN, SOCK_DGRAM, CAN_BCM);
-
-    strcpy(ifr.ifr_name, "can0");
-    ioctl(s, SIOCGIFINDEX, &ifr);
-
-    addr.can_family = AF_CAN;
-    addr.can_ifindex = ifr.ifr_ifindex;
-
-    connect(s, (struct sockaddr *)&addr, sizeof(addr));
-
-    (..)
-
-  The broadcast manager socket is able to handle any number of in flight
-  transmissions or receive filters concurrently. The different RX/TX jobs are
-  distinguished by the unique can_id in each BCM message. However additional
-  CAN_BCM sockets are recommended to communicate on multiple CAN interfaces.
-  When the broadcast manager socket is bound to 'any' CAN interface (=> the
-  interface index is set to zero) the configured receive filters apply to any
-  CAN interface unless the sendto() syscall is used to overrule the 'any' CAN
-  interface index. When using recvfrom() instead of read() to retrieve BCM
-  socket messages the originating CAN interface is provided in can_ifindex.
-
-  4.2.1 Broadcast Manager operations
-
-  The opcode defines the operation for the broadcast manager to carry out,
-  or details the broadcast managers response to several events, including
-  user requests.
-
-  Transmit Operations (user space to broadcast manager):
-
-    TX_SETUP:   Create (cyclic) transmission task.
-
-    TX_DELETE:  Remove (cyclic) transmission task, requires only can_id.
-
-    TX_READ:    Read properties of (cyclic) transmission task for can_id.
-
-    TX_SEND:    Send one CAN frame.
-
-  Transmit Responses (broadcast manager to user space):
-
-    TX_STATUS:  Reply to TX_READ request (transmission task configuration).
-
-    TX_EXPIRED: Notification when counter finishes sending at initial interval
-      'ival1'. Requires the TX_COUNTEVT flag to be set at TX_SETUP.
-
-  Receive Operations (user space to broadcast manager):
-
-    RX_SETUP:   Create RX content filter subscription.
-
-    RX_DELETE:  Remove RX content filter subscription, requires only can_id.
-
-    RX_READ:    Read properties of RX content filter subscription for can_id.
-
-  Receive Responses (broadcast manager to user space):
-
-    RX_STATUS:  Reply to RX_READ request (filter task configuration).
-
-    RX_TIMEOUT: Cyclic message is detected to be absent (timer ival1 expired).
-
-    RX_CHANGED: BCM message with updated CAN frame (detected content change).
-      Sent on first message received or on receipt of revised CAN messages.
-
-  4.2.2 Broadcast Manager message flags
-
-  When sending a message to the broadcast manager the 'flags' element may
-  contain the following flag definitions which influence the behaviour:
-
-    SETTIMER:           Set the values of ival1, ival2 and count
-
-    STARTTIMER:         Start the timer with the actual values of ival1, ival2
-      and count. Starting the timer leads simultaneously to emit a CAN frame.
-
-    TX_COUNTEVT:        Create the message TX_EXPIRED when count expires
-
-    TX_ANNOUNCE:        A change of data by the process is emitted immediately.
-
-    TX_CP_CAN_ID:       Copies the can_id from the message header to each
-      subsequent frame in frames. This is intended as usage simplification. For
-      TX tasks the unique can_id from the message header may differ from the
-      can_id(s) stored for transmission in the subsequent struct can_frame(s).
-
-    RX_FILTER_ID:       Filter by can_id alone, no frames required (nframes=0).
-
-    RX_CHECK_DLC:       A change of the DLC leads to an RX_CHANGED.
-
-    RX_NO_AUTOTIMER:    Prevent automatically starting the timeout monitor.
-
-    RX_ANNOUNCE_RESUME: If passed at RX_SETUP and a receive timeout occurred, a
-      RX_CHANGED message will be generated when the (cyclic) receive restarts.
-
-    TX_RESET_MULTI_IDX: Reset the index for the multiple frame transmission.
-
-    RX_RTR_FRAME:       Send reply for RTR-request (placed in op->frames[0]).
-
-  4.2.3 Broadcast Manager transmission timers
-
-  Periodic transmission configurations may use up to two interval timers.
-  In this case the BCM sends a number of messages ('count') at an interval
-  'ival1', then continuing to send at another given interval 'ival2'. When
-  only one timer is needed 'count' is set to zero and only 'ival2' is used.
-  When SET_TIMER and START_TIMER flag were set the timers are activated.
-  The timer values can be altered at runtime when only SET_TIMER is set.
-
-  4.2.4 Broadcast Manager message sequence transmission
-
-  Up to 256 CAN frames can be transmitted in a sequence in the case of a cyclic
-  TX task configuration. The number of CAN frames is provided in the 'nframes'
-  element of the BCM message head. The defined number of CAN frames are added
-  as array to the TX_SETUP BCM configuration message.
-
-    /* create a struct to set up a sequence of four CAN frames */
-    struct {
-            struct bcm_msg_head msg_head;
-            struct can_frame frame[4];
-    } mytxmsg;
-
-    (..)
-    mytxmsg.msg_head.nframes = 4;
-    (..)
-
-    write(s, &mytxmsg, sizeof(mytxmsg));
-
-  With every transmission the index in the array of CAN frames is increased
-  and set to zero at index overflow.
-
-  4.2.5 Broadcast Manager receive filter timers
-
-  The timer values ival1 or ival2 may be set to non-zero values at RX_SETUP.
-  When the SET_TIMER flag is set the timers are enabled:
-
-  ival1: Send RX_TIMEOUT when a received message is not received again within
-    the given time. When START_TIMER is set at RX_SETUP the timeout detection
-    is activated directly - even without a former CAN frame reception.
-
-  ival2: Throttle the received message rate down to the value of ival2. This
-    is useful to reduce messages for the application when the signal inside the
-    CAN frame is stateless as state changes within the ival2 periode may get
-    lost.
-
-  4.2.6 Broadcast Manager multiplex message receive filter
-
-  To filter for content changes in multiplex message sequences an array of more
-  than one CAN frames can be passed in a RX_SETUP configuration message. The
-  data bytes of the first CAN frame contain the mask of relevant bits that
-  have to match in the subsequent CAN frames with the received CAN frame.
-  If one of the subsequent CAN frames is matching the bits in that frame data
-  mark the relevant content to be compared with the previous received content.
-  Up to 257 CAN frames (multiplex filter bit mask CAN frame plus 256 CAN
-  filters) can be added as array to the TX_SETUP BCM configuration message.
-
-    /* usually used to clear CAN frame data[] - beware of endian problems! */
-    #define U64_DATA(p) (*(unsigned long long*)(p)->data)
-
-    struct {
-            struct bcm_msg_head msg_head;
-            struct can_frame frame[5];
-    } msg;
-
-    msg.msg_head.opcode  = RX_SETUP;
-    msg.msg_head.can_id  = 0x42;
-    msg.msg_head.flags   = 0;
-    msg.msg_head.nframes = 5;
-    U64_DATA(&msg.frame[0]) = 0xFF00000000000000ULL; /* MUX mask */
-    U64_DATA(&msg.frame[1]) = 0x01000000000000FFULL; /* data mask (MUX 0x01) */
-    U64_DATA(&msg.frame[2]) = 0x0200FFFF000000FFULL; /* data mask (MUX 0x02) */
-    U64_DATA(&msg.frame[3]) = 0x330000FFFFFF0003ULL; /* data mask (MUX 0x33) */
-    U64_DATA(&msg.frame[4]) = 0x4F07FC0FF0000000ULL; /* data mask (MUX 0x4F) */
-
-    write(s, &msg, sizeof(msg));
-
-  4.2.7 Broadcast Manager CAN FD support
-
-  The programming API of the CAN_BCM depends on struct can_frame which is
-  given as array directly behind the bcm_msg_head structure. To follow this
-  schema for the CAN FD frames a new flag 'CAN_FD_FRAME' in the bcm_msg_head
-  flags indicates that the concatenated CAN frame structures behind the
-  bcm_msg_head are defined as struct canfd_frame.
-
-    struct {
-            struct bcm_msg_head msg_head;
-            struct canfd_frame frame[5];
-    } msg;
-
-    msg.msg_head.opcode  = RX_SETUP;
-    msg.msg_head.can_id  = 0x42;
-    msg.msg_head.flags   = CAN_FD_FRAME;
-    msg.msg_head.nframes = 5;
-    (..)
-
-  When using CAN FD frames for multiplex filtering the MUX mask is still
-  expected in the first 64 bit of the struct canfd_frame data section.
-
-  4.3 connected transport protocols (SOCK_SEQPACKET)
-  4.4 unconnected transport protocols (SOCK_DGRAM)
-
-
-5. SocketCAN core module
--------------------------
-
-  The SocketCAN core module implements the protocol family
-  PF_CAN. CAN protocol modules are loaded by the core module at
-  runtime. The core module provides an interface for CAN protocol
-  modules to subscribe needed CAN IDs (see chapter 3.1).
-
-  5.1 can.ko module params
-
-  - stats_timer: To calculate the SocketCAN core statistics
-    (e.g. current/maximum frames per second) this 1 second timer is
-    invoked at can.ko module start time by default. This timer can be
-    disabled by using stattimer=0 on the module commandline.
-
-  - debug: (removed since SocketCAN SVN r546)
-
-  5.2 procfs content
-
-  As described in chapter 3.1 the SocketCAN core uses several filter
-  lists to deliver received CAN frames to CAN protocol modules. These
-  receive lists, their filters and the count of filter matches can be
-  checked in the appropriate receive list. All entries contain the
-  device and a protocol module identifier:
-
-    foo@bar:~$ cat /proc/net/can/rcvlist_all
-
-    receive list 'rx_all':
-      (vcan3: no entry)
-      (vcan2: no entry)
-      (vcan1: no entry)
-      device   can_id   can_mask  function  userdata   matches  ident
-       vcan0     000    00000000  f88e6370  f6c6f400         0  raw
-      (any: no entry)
-
-  In this example an application requests any CAN traffic from vcan0.
-
-    rcvlist_all - list for unfiltered entries (no filter operations)
-    rcvlist_eff - list for single extended frame (EFF) entries
-    rcvlist_err - list for error message frames masks
-    rcvlist_fil - list for mask/value filters
-    rcvlist_inv - list for mask/value filters (inverse semantic)
-    rcvlist_sff - list for single standard frame (SFF) entries
-
-  Additional procfs files in /proc/net/can
-
-    stats       - SocketCAN core statistics (rx/tx frames, match ratios, ...)
-    reset_stats - manual statistic reset
-    version     - prints the SocketCAN core version and the ABI version
-
-  5.3 writing own CAN protocol modules
-
-  To implement a new protocol in the protocol family PF_CAN a new
-  protocol has to be defined in include/linux/can.h .
-  The prototypes and definitions to use the SocketCAN core can be
-  accessed by including include/linux/can/core.h .
-  In addition to functions that register the CAN protocol and the
-  CAN device notifier chain there are functions to subscribe CAN
-  frames received by CAN interfaces and to send CAN frames:
-
-    can_rx_register   - subscribe CAN frames from a specific interface
-    can_rx_unregister - unsubscribe CAN frames from a specific interface
-    can_send          - transmit a CAN frame (optional with local loopback)
-
-  For details see the kerneldoc documentation in net/can/af_can.c or
-  the source code of net/can/raw.c or net/can/bcm.c .
-
-6. CAN network drivers
-----------------------
-
-  Writing a CAN network device driver is much easier than writing a
-  CAN character device driver. Similar to other known network device
-  drivers you mainly have to deal with:
-
-  - TX: Put the CAN frame from the socket buffer to the CAN controller.
-  - RX: Put the CAN frame from the CAN controller to the socket buffer.
-
-  See e.g. at Documentation/networking/netdevices.txt . The differences
-  for writing CAN network device driver are described below:
-
-  6.1 general settings
-
-    dev->type  = ARPHRD_CAN; /* the netdevice hardware type */
-    dev->flags = IFF_NOARP;  /* CAN has no arp */
-
-    dev->mtu = CAN_MTU; /* sizeof(struct can_frame) -> legacy CAN interface */
-
-    or alternative, when the controller supports CAN with flexible data rate:
-    dev->mtu = CANFD_MTU; /* sizeof(struct canfd_frame) -> CAN FD interface */
-
-  The struct can_frame or struct canfd_frame is the payload of each socket
-  buffer (skbuff) in the protocol family PF_CAN.
-
-  6.2 local loopback of sent frames
-
-  As described in chapter 3.2 the CAN network device driver should
-  support a local loopback functionality similar to the local echo
-  e.g. of tty devices. In this case the driver flag IFF_ECHO has to be
-  set to prevent the PF_CAN core from locally echoing sent frames
-  (aka loopback) as fallback solution:
-
-    dev->flags = (IFF_NOARP | IFF_ECHO);
-
-  6.3 CAN controller hardware filters
-
-  To reduce the interrupt load on deep embedded systems some CAN
-  controllers support the filtering of CAN IDs or ranges of CAN IDs.
-  These hardware filter capabilities vary from controller to
-  controller and have to be identified as not feasible in a multi-user
-  networking approach. The use of the very controller specific
-  hardware filters could make sense in a very dedicated use-case, as a
-  filter on driver level would affect all users in the multi-user
-  system. The high efficient filter sets inside the PF_CAN core allow
-  to set different multiple filters for each socket separately.
-  Therefore the use of hardware filters goes to the category 'handmade
-  tuning on deep embedded systems'. The author is running a MPC603e
-  @133MHz with four SJA1000 CAN controllers from 2002 under heavy bus
-  load without any problems ...
-
-  6.4 The virtual CAN driver (vcan)
-
-  Similar to the network loopback devices, vcan offers a virtual local
-  CAN interface. A full qualified address on CAN consists of
-
-  - a unique CAN Identifier (CAN ID)
-  - the CAN bus this CAN ID is transmitted on (e.g. can0)
-
-  so in common use cases more than one virtual CAN interface is needed.
-
-  The virtual CAN interfaces allow the transmission and reception of CAN
-  frames without real CAN controller hardware. Virtual CAN network
-  devices are usually named 'vcanX', like vcan0 vcan1 vcan2 ...
-  When compiled as a module the virtual CAN driver module is called vcan.ko
-
-  Since Linux Kernel version 2.6.24 the vcan driver supports the Kernel
-  netlink interface to create vcan network devices. The creation and
-  removal of vcan network devices can be managed with the ip(8) tool:
-
-  - Create a virtual CAN network interface:
-       $ ip link add type vcan
-
-  - Create a virtual CAN network interface with a specific name 'vcan42':
-       $ ip link add dev vcan42 type vcan
-
-  - Remove a (virtual CAN) network interface 'vcan42':
-       $ ip link del vcan42
-
-  6.5 The CAN network device driver interface
-
-  The CAN network device driver interface provides a generic interface
-  to setup, configure and monitor CAN network devices. The user can then
-  configure the CAN device, like setting the bit-timing parameters, via
-  the netlink interface using the program "ip" from the "IPROUTE2"
-  utility suite. The following chapter describes briefly how to use it.
-  Furthermore, the interface uses a common data structure and exports a
-  set of common functions, which all real CAN network device drivers
-  should use. Please have a look to the SJA1000 or MSCAN driver to
-  understand how to use them. The name of the module is can-dev.ko.
-
-  6.5.1 Netlink interface to set/get devices properties
-
-  The CAN device must be configured via netlink interface. The supported
-  netlink message types are defined and briefly described in
-  "include/linux/can/netlink.h". CAN link support for the program "ip"
-  of the IPROUTE2 utility suite is available and it can be used as shown
-  below:
-
-  - Setting CAN device properties:
-
-    $ ip link set can0 type can help
-    Usage: ip link set DEVICE type can
-        [ bitrate BITRATE [ sample-point SAMPLE-POINT] ] |
-        [ tq TQ prop-seg PROP_SEG phase-seg1 PHASE-SEG1
-          phase-seg2 PHASE-SEG2 [ sjw SJW ] ]
-
-        [ dbitrate BITRATE [ dsample-point SAMPLE-POINT] ] |
-        [ dtq TQ dprop-seg PROP_SEG dphase-seg1 PHASE-SEG1
-          dphase-seg2 PHASE-SEG2 [ dsjw SJW ] ]
-
-        [ loopback { on | off } ]
-        [ listen-only { on | off } ]
-        [ triple-sampling { on | off } ]
-        [ one-shot { on | off } ]
-        [ berr-reporting { on | off } ]
-        [ fd { on | off } ]
-        [ fd-non-iso { on | off } ]
-        [ presume-ack { on | off } ]
-
-        [ restart-ms TIME-MS ]
-        [ restart ]
-
-        Where: BITRATE       := { 1..1000000 }
-               SAMPLE-POINT  := { 0.000..0.999 }
-               TQ            := { NUMBER }
-               PROP-SEG      := { 1..8 }
-               PHASE-SEG1    := { 1..8 }
-               PHASE-SEG2    := { 1..8 }
-               SJW           := { 1..4 }
-               RESTART-MS    := { 0 | NUMBER }
-
-  - Display CAN device details and statistics:
-
-    $ ip -details -statistics link show can0
-    2: can0: <NOARP,UP,LOWER_UP,ECHO> mtu 16 qdisc pfifo_fast state UP qlen 10
-      link/can
-      can <TRIPLE-SAMPLING> state ERROR-ACTIVE restart-ms 100
-      bitrate 125000 sample_point 0.875
-      tq 125 prop-seg 6 phase-seg1 7 phase-seg2 2 sjw 1
-      sja1000: tseg1 1..16 tseg2 1..8 sjw 1..4 brp 1..64 brp-inc 1
-      clock 8000000
-      re-started bus-errors arbit-lost error-warn error-pass bus-off
-      41         17457      0          41         42         41
-      RX: bytes  packets  errors  dropped overrun mcast
-      140859     17608    17457   0       0       0
-      TX: bytes  packets  errors  dropped carrier collsns
-      861        112      0       41      0       0
-
-  More info to the above output:
-
-    "<TRIPLE-SAMPLING>"
-	Shows the list of selected CAN controller modes: LOOPBACK,
-	LISTEN-ONLY, or TRIPLE-SAMPLING.
-
-    "state ERROR-ACTIVE"
-	The current state of the CAN controller: "ERROR-ACTIVE",
-	"ERROR-WARNING", "ERROR-PASSIVE", "BUS-OFF" or "STOPPED"
-
-    "restart-ms 100"
-	Automatic restart delay time. If set to a non-zero value, a
-	restart of the CAN controller will be triggered automatically
-	in case of a bus-off condition after the specified delay time
-	in milliseconds. By default it's off.
-
-    "bitrate 125000 sample-point 0.875"
-	Shows the real bit-rate in bits/sec and the sample-point in the
-	range 0.000..0.999. If the calculation of bit-timing parameters
-	is enabled in the kernel (CONFIG_CAN_CALC_BITTIMING=y), the
-	bit-timing can be defined by setting the "bitrate" argument.
-	Optionally the "sample-point" can be specified. By default it's
-	0.000 assuming CIA-recommended sample-points.
-
-    "tq 125 prop-seg 6 phase-seg1 7 phase-seg2 2 sjw 1"
-	Shows the time quanta in ns, propagation segment, phase buffer
-	segment 1 and 2 and the synchronisation jump width in units of
-	tq. They allow to define the CAN bit-timing in a hardware
-	independent format as proposed by the Bosch CAN 2.0 spec (see
-	chapter 8 of http://www.semiconductors.bosch.de/pdf/can2spec.pdf).
-
-    "sja1000: tseg1 1..16 tseg2 1..8 sjw 1..4 brp 1..64 brp-inc 1
-     clock 8000000"
-	Shows the bit-timing constants of the CAN controller, here the
-	"sja1000". The minimum and maximum values of the time segment 1
-	and 2, the synchronisation jump width in units of tq, the
-	bitrate pre-scaler and the CAN system clock frequency in Hz.
-	These constants could be used for user-defined (non-standard)
-	bit-timing calculation algorithms in user-space.
-
-    "re-started bus-errors arbit-lost error-warn error-pass bus-off"
-	Shows the number of restarts, bus and arbitration lost errors,
-	and the state changes to the error-warning, error-passive and
-	bus-off state. RX overrun errors are listed in the "overrun"
-	field of the standard network statistics.
-
-  6.5.2 Setting the CAN bit-timing
-
-  The CAN bit-timing parameters can always be defined in a hardware
-  independent format as proposed in the Bosch CAN 2.0 specification
-  specifying the arguments "tq", "prop_seg", "phase_seg1", "phase_seg2"
-  and "sjw":
-
-    $ ip link set canX type can tq 125 prop-seg 6 \
-				phase-seg1 7 phase-seg2 2 sjw 1
-
-  If the kernel option CONFIG_CAN_CALC_BITTIMING is enabled, CIA
-  recommended CAN bit-timing parameters will be calculated if the bit-
-  rate is specified with the argument "bitrate":
-
-    $ ip link set canX type can bitrate 125000
-
-  Note that this works fine for the most common CAN controllers with
-  standard bit-rates but may *fail* for exotic bit-rates or CAN system
-  clock frequencies. Disabling CONFIG_CAN_CALC_BITTIMING saves some
-  space and allows user-space tools to solely determine and set the
-  bit-timing parameters. The CAN controller specific bit-timing
-  constants can be used for that purpose. They are listed by the
-  following command:
-
-    $ ip -details link show can0
-    ...
-      sja1000: clock 8000000 tseg1 1..16 tseg2 1..8 sjw 1..4 brp 1..64 brp-inc 1
-
-  6.5.3 Starting and stopping the CAN network device
-
-  A CAN network device is started or stopped as usual with the command
-  "ifconfig canX up/down" or "ip link set canX up/down". Be aware that
-  you *must* define proper bit-timing parameters for real CAN devices
-  before you can start it to avoid error-prone default settings:
-
-    $ ip link set canX up type can bitrate 125000
-
-  A device may enter the "bus-off" state if too many errors occurred on
-  the CAN bus. Then no more messages are received or sent. An automatic
-  bus-off recovery can be enabled by setting the "restart-ms" to a
-  non-zero value, e.g.:
-
-    $ ip link set canX type can restart-ms 100
-
-  Alternatively, the application may realize the "bus-off" condition
-  by monitoring CAN error message frames and do a restart when
-  appropriate with the command:
-
-    $ ip link set canX type can restart
-
-  Note that a restart will also create a CAN error message frame (see
-  also chapter 3.3).
-
-  6.6 CAN FD (flexible data rate) driver support
-
-  CAN FD capable CAN controllers support two different bitrates for the
-  arbitration phase and the payload phase of the CAN FD frame. Therefore a
-  second bit timing has to be specified in order to enable the CAN FD bitrate.
-
-  Additionally CAN FD capable CAN controllers support up to 64 bytes of
-  payload. The representation of this length in can_frame.can_dlc and
-  canfd_frame.len for userspace applications and inside the Linux network
-  layer is a plain value from 0 .. 64 instead of the CAN 'data length code'.
-  The data length code was a 1:1 mapping to the payload length in the legacy
-  CAN frames anyway. The payload length to the bus-relevant DLC mapping is
-  only performed inside the CAN drivers, preferably with the helper
-  functions can_dlc2len() and can_len2dlc().
-
-  The CAN netdevice driver capabilities can be distinguished by the network
-  devices maximum transfer unit (MTU):
-
-  MTU = 16 (CAN_MTU)   => sizeof(struct can_frame)   => 'legacy' CAN device
-  MTU = 72 (CANFD_MTU) => sizeof(struct canfd_frame) => CAN FD capable device
-
-  The CAN device MTU can be retrieved e.g. with a SIOCGIFMTU ioctl() syscall.
-  N.B. CAN FD capable devices can also handle and send legacy CAN frames.
-
-  When configuring CAN FD capable CAN controllers an additional 'data' bitrate
-  has to be set. This bitrate for the data phase of the CAN FD frame has to be
-  at least the bitrate which was configured for the arbitration phase. This
-  second bitrate is specified analogue to the first bitrate but the bitrate
-  setting keywords for the 'data' bitrate start with 'd' e.g. dbitrate,
-  dsample-point, dsjw or dtq and similar settings. When a data bitrate is set
-  within the configuration process the controller option "fd on" can be
-  specified to enable the CAN FD mode in the CAN controller. This controller
-  option also switches the device MTU to 72 (CANFD_MTU).
-
-  The first CAN FD specification presented as whitepaper at the International
-  CAN Conference 2012 needed to be improved for data integrity reasons.
-  Therefore two CAN FD implementations have to be distinguished today:
-
-  - ISO compliant:     The ISO 11898-1:2015 CAN FD implementation (default)
-  - non-ISO compliant: The CAN FD implementation following the 2012 whitepaper
-
-  Finally there are three types of CAN FD controllers:
-
-  1. ISO compliant (fixed)
-  2. non-ISO compliant (fixed, like the M_CAN IP core v3.0.1 in m_can.c)
-  3. ISO/non-ISO CAN FD controllers (switchable, like the PEAK PCAN-USB FD)
-
-  The current ISO/non-ISO mode is announced by the CAN controller driver via
-  netlink and displayed by the 'ip' tool (controller option FD-NON-ISO).
-  The ISO/non-ISO-mode can be altered by setting 'fd-non-iso {on|off}' for
-  switchable CAN FD controllers only.
-
-  Example configuring 500 kbit/s arbitration bitrate and 4 Mbit/s data bitrate:
-
-    $ ip link set can0 up type can bitrate 500000 sample-point 0.75 \
-                                   dbitrate 4000000 dsample-point 0.8 fd on
-    $ ip -details link show can0
-    5: can0: <NOARP,UP,LOWER_UP,ECHO> mtu 72 qdisc pfifo_fast state UNKNOWN \
-             mode DEFAULT group default qlen 10
-    link/can  promiscuity 0
-    can <FD> state ERROR-ACTIVE (berr-counter tx 0 rx 0) restart-ms 0
-          bitrate 500000 sample-point 0.750
-          tq 50 prop-seg 14 phase-seg1 15 phase-seg2 10 sjw 1
-          pcan_usb_pro_fd: tseg1 1..64 tseg2 1..16 sjw 1..16 brp 1..1024 \
-          brp-inc 1
-          dbitrate 4000000 dsample-point 0.800
-          dtq 12 dprop-seg 7 dphase-seg1 8 dphase-seg2 4 dsjw 1
-          pcan_usb_pro_fd: dtseg1 1..16 dtseg2 1..8 dsjw 1..4 dbrp 1..1024 \
-          dbrp-inc 1
-          clock 80000000
-
-  Example when 'fd-non-iso on' is added on this switchable CAN FD adapter:
-   can <FD,FD-NON-ISO> state ERROR-ACTIVE (berr-counter tx 0 rx 0) restart-ms 0
-
-  6.7 Supported CAN hardware
-
-  Please check the "Kconfig" file in "drivers/net/can" to get an actual
-  list of the support CAN hardware. On the SocketCAN project website
-  (see chapter 7) there might be further drivers available, also for
-  older kernel versions.
-
-7. SocketCAN resources
------------------------
-
-  The Linux CAN / SocketCAN project resources (project site / mailing list)
-  are referenced in the MAINTAINERS file in the Linux source tree.
-  Search for CAN NETWORK [LAYERS|DRIVERS].
-
-8. Credits
-----------
-
-  Oliver Hartkopp (PF_CAN core, filters, drivers, bcm, SJA1000 driver)
-  Urs Thuermann (PF_CAN core, kernel integration, socket interfaces, raw, vcan)
-  Jan Kizka (RT-SocketCAN core, Socket-API reconciliation)
-  Wolfgang Grandegger (RT-SocketCAN core & drivers, Raw Socket-API reviews,
-                       CAN device driver interface, MSCAN driver)
-  Robert Schwebel (design reviews, PTXdist integration)
-  Marc Kleine-Budde (design reviews, Kernel 2.6 cleanups, drivers)
-  Benedikt Spranger (reviews)
-  Thomas Gleixner (LKML reviews, coding style, posting hints)
-  Andrey Volkov (kernel subtree structure, ioctls, MSCAN driver)
-  Matthias Brukner (first SJA1000 CAN netdevice implementation Q2/2003)
-  Klaus Hitschler (PEAK driver integration)
-  Uwe Koppe (CAN netdevices with PF_PACKET approach)
-  Michael Schulze (driver layer loopback requirement, RT CAN drivers review)
-  Pavel Pisa (Bit-timing calculation)
-  Sascha Hauer (SJA1000 platform driver)
-  Sebastian Haas (SJA1000 EMS PCI driver)
-  Markus Plessing (SJA1000 EMS PCI driver)
-  Per Dalen (SJA1000 Kvaser PCI driver)
-  Sam Ravnborg (reviews, coding style, kbuild help)
diff --git a/Documentation/networking/dsa/dsa.txt b/Documentation/networking/dsa/dsa.txt
index b8b40753133e..25170ad7d25b 100644
--- a/Documentation/networking/dsa/dsa.txt
+++ b/Documentation/networking/dsa/dsa.txt
@@ -385,11 +385,6 @@ Switch configuration
   avoid relying on what a previous software agent such as a bootloader/firmware
   may have previously configured.
 
-- set_addr: Some switches require the programming of the management interface's
-  Ethernet MAC address, switch drivers can also disable ageing of MAC addresses
-  on the management interface and "hardcode"/"force" this MAC address for the
-  CPU/management interface as an optimization
-
 PHY devices and link management
 -------------------------------
 
diff --git a/Documentation/networking/filter.txt b/Documentation/networking/filter.txt
index 87814859cfc2..a4508ec1816b 100644
--- a/Documentation/networking/filter.txt
+++ b/Documentation/networking/filter.txt
@@ -1134,7 +1134,7 @@ The verifier's knowledge about the variable offset consists of:
 mask and value; no bit should ever be 1 in both.  For example, if a byte is read
 into a register from memory, the register's top 56 bits are known zero, while
 the low 8 are unknown - which is represented as the tnum (0x0; 0xff).  If we
-then OR this with 0x40, we get (0x40; 0xcf), then if we add 1 we get (0x0;
+then OR this with 0x40, we get (0x40; 0xbf), then if we add 1 we get (0x0;
 0x1ff), because of potential carries.
 Besides arithmetic, the register state can also be updated by conditional
 branches.  For instance, if a SCALAR_VALUE is compared > 8, in the 'true' branch
diff --git a/Documentation/networking/ieee802154.txt b/Documentation/networking/ieee802154.txt
index 057e9fdbfac9..e74d8e1da0e2 100644
--- a/Documentation/networking/ieee802154.txt
+++ b/Documentation/networking/ieee802154.txt
@@ -97,6 +97,46 @@ The include/net/mac802154.h defines following functions:
  - void ieee802154_unregister_hw(struct ieee802154_hw *hw):
    freeing registered PHY
 
+ - void ieee802154_rx_irqsafe(struct ieee802154_hw *hw, struct sk_buff *skb,
+                              u8 lqi):
+   telling 802.15.4 module there is a new received frame in the skb with
+   the RF Link Quality Indicator (LQI) from the hardware device
+
+ - void ieee802154_xmit_complete(struct ieee802154_hw *hw, struct sk_buff *skb,
+                                 bool ifs_handling):
+   telling 802.15.4 module the frame in the skb is or going to be
+   transmitted through the hardware device
+
+The device driver must implement the following callbacks in the IEEE 802.15.4
+operations structure at least:
+struct ieee802154_ops {
+	...
+	int	(*start)(struct ieee802154_hw *hw);
+	void	(*stop)(struct ieee802154_hw *hw);
+	...
+	int	(*xmit_async)(struct ieee802154_hw *hw, struct sk_buff *skb);
+	int	(*ed)(struct ieee802154_hw *hw, u8 *level);
+	int	(*set_channel)(struct ieee802154_hw *hw, u8 page, u8 channel);
+	...
+};
+
+ - int start(struct ieee802154_hw *hw):
+   handler that 802.15.4 module calls for the hardware device initialization.
+
+ - void stop(struct ieee802154_hw *hw):
+   handler that 802.15.4 module calls for the hardware device cleanup.
+
+ - int xmit_async(struct ieee802154_hw *hw, struct sk_buff *skb):
+   handler that 802.15.4 module calls for each frame in the skb going to be
+   transmitted through the hardware device.
+
+ - int ed(struct ieee802154_hw *hw, u8 *level):
+   handler that 802.15.4 module calls for Energy Detection from the hardware
+   device.
+
+ - int set_channel(struct ieee802154_hw *hw, u8 page, u8 channel):
+   set radio for listening on specific channel of the hardware device.
+
 Moreover IEEE 802.15.4 device operations structure should be filled.
 
 Fake drivers
diff --git a/Documentation/networking/index.rst b/Documentation/networking/index.rst
index 7d4b15977d61..90966c2692d8 100644
--- a/Documentation/networking/index.rst
+++ b/Documentation/networking/index.rst
@@ -7,6 +7,7 @@ Contents:
    :maxdepth: 2
 
    batman-adv
+   can
    kapi
    z8530book
    msg_zerocopy
diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt
index 46c7e1085efc..a553d4e4a0fb 100644
--- a/Documentation/networking/ip-sysctl.txt
+++ b/Documentation/networking/ip-sysctl.txt
@@ -508,7 +508,7 @@ tcp_rmem - vector of 3 INTEGERs: min, default, max
 	min: Minimal size of receive buffer used by TCP sockets.
 	It is guaranteed to each TCP socket, even under moderate memory
 	pressure.
-	Default: 1 page
+	Default: 4K
 
 	default: initial size of receive buffer used by TCP sockets.
 	This value overrides net.core.rmem_default used by other protocols.
@@ -606,6 +606,7 @@ tcp_fastopen_blackhole_timeout_sec - INTEGER
 	This time period will grow exponentially when more blackhole issues
 	get detected right after Fastopen is re-enabled and will reset to
 	initial value when the blackhole issue goes away.
+	0 to disable the blackhole detection.
 	By default, it is set to 1hr.
 
 tcp_syn_retries - INTEGER
@@ -666,7 +667,7 @@ tcp_window_scaling - BOOLEAN
 tcp_wmem - vector of 3 INTEGERs: min, default, max
 	min: Amount of memory reserved for send buffers for TCP sockets.
 	Each TCP socket has rights to use it due to fact of its birth.
-	Default: 1 page
+	Default: 4K
 
 	default: initial size of send buffer used by TCP sockets.  This
 	value overrides net.core.wmem_default used by other protocols.
diff --git a/Documentation/networking/kapi.rst b/Documentation/networking/kapi.rst
index 580289f345da..f03ae64be8bc 100644
--- a/Documentation/networking/kapi.rst
+++ b/Documentation/networking/kapi.rst
@@ -145,3 +145,27 @@ PHY Support
 
 .. kernel-doc:: drivers/net/phy/mdio_bus.c
    :internal:
+
+PHYLINK
+-------
+
+  PHYLINK interfaces traditional network drivers with PHYLIB, fixed-links,
+  and SFF modules (eg, hot-pluggable SFP) that may contain PHYs.  PHYLINK
+  provides management of the link state and link modes.
+
+.. kernel-doc:: include/linux/phylink.h
+   :internal:
+
+.. kernel-doc:: drivers/net/phy/phylink.c
+
+SFP support
+-----------
+
+.. kernel-doc:: drivers/net/phy/sfp-bus.c
+   :internal:
+
+.. kernel-doc:: include/linux/sfp.h
+   :internal:
+
+.. kernel-doc:: drivers/net/phy/sfp-bus.c
+   :export:
diff --git a/Documentation/networking/netdev-features.txt b/Documentation/networking/netdev-features.txt
index 7413eb05223b..c77f9d57eb91 100644
--- a/Documentation/networking/netdev-features.txt
+++ b/Documentation/networking/netdev-features.txt
@@ -163,3 +163,12 @@ This requests that the NIC receive all possible frames, including errored
 frames (such as bad FCS, etc).  This can be helpful when sniffing a link with
 bad packets on it.  Some NICs may receive more packets if also put into normal
 PROMISC mode.
+
+*  rx-gro-hw
+
+This requests that the NIC enables Hardware GRO (generic receive offload).
+Hardware GRO is basically the exact reverse of TSO, and is generally
+stricter than Hardware LRO.  A packet stream merged by Hardware GRO must
+be re-segmentable by GSO or TSO back to the exact original packet stream.
+Hardware GRO is dependent on RXCSUM since every packet successfully merged
+by hardware must also have the checksum verified by hardware.
diff --git a/Documentation/networking/pktgen.txt b/Documentation/networking/pktgen.txt
index 2c4e3354e128..d2fd78f85aa4 100644
--- a/Documentation/networking/pktgen.txt
+++ b/Documentation/networking/pktgen.txt
@@ -12,8 +12,8 @@ suitable sample script and configure that.
 On a dual CPU:
 
 ps aux | grep pkt
-root       129  0.3  0.0     0    0 ?        SW    2003 523:20 [pktgen/0]
-root       130  0.3  0.0     0    0 ?        SW    2003 509:50 [pktgen/1]
+root       129  0.3  0.0     0    0 ?        SW    2003 523:20 [kpktgend_0]
+root       130  0.3  0.0     0    0 ?        SW    2003 509:50 [kpktgend_1]
 
 
 For monitoring and control pktgen creates:
@@ -113,9 +113,16 @@ Configuring devices
 ===================
 This is done via the /proc interface, and most easily done via pgset
 as defined in the sample scripts.
+You need to specify PGDEV environment variable to use functions from sample
+scripts, i.e.:
+export PGDEV=/proc/net/pktgen/eth4@0
+source samples/pktgen/functions.sh
 
 Examples:
 
+ pg_ctrl start           starts injection.
+ pg_ctrl stop            aborts injection. Also, ^C aborts generator.
+
  pgset "clone_skb 1"     sets the number of copies of the same packet
  pgset "clone_skb 0"     use single SKB for all transmits
  pgset "burst 8"         uses xmit_more API to queue 8 copies of the same
@@ -165,8 +172,12 @@ Examples:
                               IPSEC # IPsec encapsulation (needs CONFIG_XFRM)
                               NODE_ALLOC # node specific memory allocation
                               NO_TIMESTAMP # disable timestamping
+ pgset 'flag ![name]'    Clear a flag to determine behaviour.
+                         Note that you might need to use single quote in
+                         interactive mode, so that your shell wouldn't expand
+                         the specified flag as a history command.
 
- pgset spi SPI_VALUE     Set specific SA used to transform packet.
+ pgset "spi [SPI_VALUE]" Set specific SA used to transform packet.
 
  pgset "udp_src_min 9"   set UDP source port min, If < udp_src_max, then
                          cycle through the port range.
@@ -207,8 +218,6 @@ Examples:
  pgset "tos XX"           set former IPv4 TOS field (e.g. "tos 28" for AF11 no ECN, default 00)
  pgset "traffic_class XX" set former IPv6 TRAFFIC CLASS (e.g. "traffic_class B8" for EF no ECN, default 00)
 
- pgset stop    	          aborts injection. Also, ^C aborts generator.
-
  pgset "rate 300M"        set rate to 300 Mb/s
  pgset "ratep 1000000"    set rate to 1Mpps
 
diff --git a/Documentation/networking/segmentation-offloads.txt b/Documentation/networking/segmentation-offloads.txt
index 2f09455a993a..aca542ec125c 100644
--- a/Documentation/networking/segmentation-offloads.txt
+++ b/Documentation/networking/segmentation-offloads.txt
@@ -13,14 +13,15 @@ The following technologies are described:
  * Generic Segmentation Offload - GSO
  * Generic Receive Offload - GRO
  * Partial Generic Segmentation Offload - GSO_PARTIAL
+ * SCTP accelleration with GSO - GSO_BY_FRAGS
 
 TCP Segmentation Offload
 ========================
 
 TCP segmentation allows a device to segment a single frame into multiple
 frames with a data payload size specified in skb_shinfo()->gso_size.
-When TCP segmentation requested the bit for either SKB_GSO_TCP or
-SKB_GSO_TCP6 should be set in skb_shinfo()->gso_type and
+When TCP segmentation requested the bit for either SKB_GSO_TCPV4 or
+SKB_GSO_TCPV6 should be set in skb_shinfo()->gso_type and
 skb_shinfo()->gso_size should be set to a non-zero value.
 
 TCP segmentation is dependent on support for the use of partial checksum
@@ -49,6 +50,10 @@ datagram into multiple IPv4 fragments.  Many of the requirements for UDP
 fragmentation offload are the same as TSO.  However the IPv4 ID for
 fragments should not increment as a single IPv4 datagram is fragmented.
 
+UFO is deprecated: modern kernels will no longer generate UFO skbs, but can
+still receive them from tuntap and similar devices. Offload of UDP-based
+tunnel protocols is still supported.
+
 IPIP, SIT, GRE, UDP Tunnel, and Remote Checksum Offloads
 ========================================================
 
@@ -83,10 +88,10 @@ SKB_GSO_UDP_TUNNEL_CSUM.  These two additional tunnel types reflect the
 fact that the outer header also requests to have a non-zero checksum
 included in the outer header.
 
-Finally there is SKB_GSO_REMCSUM which indicates that a given tunnel header
-has requested a remote checksum offload.  In this case the inner headers
-will be left with a partial checksum and only the outer header checksum
-will be computed.
+Finally there is SKB_GSO_TUNNEL_REMCSUM which indicates that a given tunnel
+header has requested a remote checksum offload.  In this case the inner
+headers will be left with a partial checksum and only the outer header
+checksum will be computed.
 
 Generic Segmentation Offload
 ============================
@@ -128,3 +133,38 @@ values for if the header was simply duplicated.  The one exception to this
 is the outer IPv4 ID field.  It is up to the device drivers to guarantee
 that the IPv4 ID field is incremented in the case that a given header does
 not have the DF bit set.
+
+SCTP accelleration with GSO
+===========================
+
+SCTP - despite the lack of hardware support - can still take advantage of
+GSO to pass one large packet through the network stack, rather than
+multiple small packets.
+
+This requires a different approach to other offloads, as SCTP packets
+cannot be just segmented to (P)MTU. Rather, the chunks must be contained in
+IP segments, padding respected. So unlike regular GSO, SCTP can't just
+generate a big skb, set gso_size to the fragmentation point and deliver it
+to IP layer.
+
+Instead, the SCTP protocol layer builds an skb with the segments correctly
+padded and stored as chained skbs, and skb_segment() splits based on those.
+To signal this, gso_size is set to the special value GSO_BY_FRAGS.
+
+Therefore, any code in the core networking stack must be aware of the
+possibility that gso_size will be GSO_BY_FRAGS and handle that case
+appropriately.
+
+There are some helpers to make this easier:
+
+ - skb_is_gso(skb) && skb_is_gso_sctp(skb) is the best way to see if
+   an skb is an SCTP GSO skb.
+
+ - For size checks, the skb_gso_validate_*_len family of helpers correctly
+   considers GSO_BY_FRAGS.
+
+ - For manipulating packets, skb_increase_gso_size and skb_decrease_gso_size
+   will check for GSO_BY_FRAGS and WARN if asked to manipulate these skbs.
+
+This also affects drivers with the NETIF_F_FRAGLIST & NETIF_F_GSO_SCTP bits
+set. Note also that NETIF_F_GSO_SCTP is included in NETIF_F_GSO_SOFTWARE.
diff --git a/Documentation/networking/xfrm_device.txt b/Documentation/networking/xfrm_device.txt
new file mode 100644
index 000000000000..50c34ca65efe
--- /dev/null
+++ b/Documentation/networking/xfrm_device.txt
@@ -0,0 +1,135 @@
+
+===============================================
+XFRM device - offloading the IPsec computations
+===============================================
+Shannon Nelson <shannon.nelson@oracle.com>
+
+
+Overview
+========
+
+IPsec is a useful feature for securing network traffic, but the
+computational cost is high: a 10Gbps link can easily be brought down
+to under 1Gbps, depending on the traffic and link configuration.
+Luckily, there are NICs that offer a hardware based IPsec offload which
+can radically increase throughput and decrease CPU utilization.  The XFRM
+Device interface allows NIC drivers to offer to the stack access to the
+hardware offload.
+
+Userland access to the offload is typically through a system such as
+libreswan or KAME/raccoon, but the iproute2 'ip xfrm' command set can
+be handy when experimenting.  An example command might look something
+like this:
+
+  ip x s add proto esp dst 14.0.0.70 src 14.0.0.52 spi 0x07 mode transport \
+     reqid 0x07 replay-window 32 \
+     aead 'rfc4106(gcm(aes))' 0x44434241343332312423222114131211f4f3f2f1 128 \
+     sel src 14.0.0.52/24 dst 14.0.0.70/24 proto tcp \
+     offload dev eth4 dir in
+
+Yes, that's ugly, but that's what shell scripts and/or libreswan are for.
+
+
+
+Callbacks to implement
+======================
+
+/* from include/linux/netdevice.h */
+struct xfrmdev_ops {
+	int	(*xdo_dev_state_add) (struct xfrm_state *x);
+	void	(*xdo_dev_state_delete) (struct xfrm_state *x);
+	void	(*xdo_dev_state_free) (struct xfrm_state *x);
+	bool	(*xdo_dev_offload_ok) (struct sk_buff *skb,
+				       struct xfrm_state *x);
+	void    (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
+};
+
+The NIC driver offering ipsec offload will need to implement these
+callbacks to make the offload available to the network stack's
+XFRM subsytem.  Additionally, the feature bits NETIF_F_HW_ESP and
+NETIF_F_HW_ESP_TX_CSUM will signal the availability of the offload.
+
+
+
+Flow
+====
+
+At probe time and before the call to register_netdev(), the driver should
+set up local data structures and XFRM callbacks, and set the feature bits.
+The XFRM code's listener will finish the setup on NETDEV_REGISTER.
+
+		adapter->netdev->xfrmdev_ops = &ixgbe_xfrmdev_ops;
+		adapter->netdev->features |= NETIF_F_HW_ESP;
+		adapter->netdev->hw_enc_features |= NETIF_F_HW_ESP;
+
+When new SAs are set up with a request for "offload" feature, the
+driver's xdo_dev_state_add() will be given the new SA to be offloaded
+and an indication of whether it is for Rx or Tx.  The driver should
+	- verify the algorithm is supported for offloads
+	- store the SA information (key, salt, target-ip, protocol, etc)
+	- enable the HW offload of the SA
+
+The driver can also set an offload_handle in the SA, an opaque void pointer
+that can be used to convey context into the fast-path offload requests.
+
+		xs->xso.offload_handle = context;
+
+
+When the network stack is preparing an IPsec packet for an SA that has
+been setup for offload, it first calls into xdo_dev_offload_ok() with
+the skb and the intended offload state to ask the driver if the offload
+will serviceable.  This can check the packet information to be sure the
+offload can be supported (e.g. IPv4 or IPv6, no IPv4 options, etc) and
+return true of false to signify its support.
+
+When ready to send, the driver needs to inspect the Tx packet for the
+offload information, including the opaque context, and set up the packet
+send accordingly.
+
+		xs = xfrm_input_state(skb);
+		context = xs->xso.offload_handle;
+		set up HW for send
+
+The stack has already inserted the appropriate IPsec headers in the
+packet data, the offload just needs to do the encryption and fix up the
+header values.
+
+
+When a packet is received and the HW has indicated that it offloaded a
+decryption, the driver needs to add a reference to the decoded SA into
+the packet's skb.  At this point the data should be decrypted but the
+IPsec headers are still in the packet data; they are removed later up
+the stack in xfrm_input().
+
+	find and hold the SA that was used to the Rx skb
+		get spi, protocol, and destination IP from packet headers
+		xs = find xs from (spi, protocol, dest_IP)
+		xfrm_state_hold(xs);
+
+	store the state information into the skb
+		skb->sp = secpath_dup(skb->sp);
+		skb->sp->xvec[skb->sp->len++] = xs;
+		skb->sp->olen++;
+
+	indicate the success and/or error status of the offload
+		xo = xfrm_offload(skb);
+		xo->flags = CRYPTO_DONE;
+		xo->status = crypto_status;
+
+	hand the packet to napi_gro_receive() as usual
+
+In ESN mode, xdo_dev_state_advance_esn() is called from xfrm_replay_advance_esn().
+Driver will check packet seq number and update HW ESN state machine if needed.
+
+When the SA is removed by the user, the driver's xdo_dev_state_delete()
+is asked to disable the offload.  Later, xdo_dev_state_free() is called
+from a garbage collection routine after all reference counts to the state
+have been removed and any remaining resources can be cleared for the
+offload state.  How these are used by the driver will depend on specific
+hardware needs.
+
+As a netdev is set to DOWN the XFRM stack's netdev listener will call
+xdo_dev_state_delete() and xdo_dev_state_free() on any remaining offloaded
+states.
+
+
diff --git a/Documentation/networking/xfrm_proc.txt b/Documentation/networking/xfrm_proc.txt
index d0d8bafa9016..2eae619ab67b 100644
--- a/Documentation/networking/xfrm_proc.txt
+++ b/Documentation/networking/xfrm_proc.txt
@@ -5,13 +5,15 @@ Masahide NAKAMURA <nakam@linux-ipv6.org>
 
 Transformation Statistics
 -------------------------
-xfrm_proc is a statistics shown factor dropped by transformation
-for developer.
-It is a counter designed from current transformation source code
-and defined like linux private MIB.
 
-Inbound statistics
-~~~~~~~~~~~~~~~~~~
+The xfrm_proc code is a set of statistics showing numbers of packets
+dropped by the transformation code and why.  These counters are defined
+as part of the linux private MIB.  These counters can be viewed in
+/proc/net/xfrm_stat.
+
+
+Inbound errors
+~~~~~~~~~~~~~~
 XfrmInError:
 	All errors which is not matched others
 XfrmInBufferError:
@@ -46,6 +48,10 @@ XfrmInPolBlock:
 	Policy discards
 XfrmInPolError:
 	Policy error
+XfrmAcquireError:
+	State hasn't been fully acquired before use
+XfrmFwdHdrError:
+	Forward routing of a packet is not allowed
 
 Outbound errors
 ~~~~~~~~~~~~~~~
@@ -72,3 +78,5 @@ XfrmOutPolDead:
 	Policy is dead
 XfrmOutPolError:
 	Policy error
+XfrmOutStateInvalid:
+	State is invalid, perhaps expired
diff --git a/Documentation/perf/arm_dsu_pmu.txt b/Documentation/perf/arm_dsu_pmu.txt
new file mode 100644
index 000000000000..d611e15f5add
--- /dev/null
+++ b/Documentation/perf/arm_dsu_pmu.txt
@@ -0,0 +1,28 @@
+ARM DynamIQ Shared Unit (DSU) PMU
+==================================
+
+ARM DynamIQ Shared Unit integrates one or more cores with an L3 memory system,
+control logic and external interfaces to form a multicore cluster. The PMU
+allows counting the various events related to the L3 cache, Snoop Control Unit
+etc, using 32bit independent counters. It also provides a 64bit cycle counter.
+
+The PMU can only be accessed via CPU system registers and are common to the
+cores connected to the same DSU. Like most of the other uncore PMUs, DSU
+PMU doesn't support process specific events and cannot be used in sampling mode.
+
+The DSU provides a bitmap for a subset of implemented events via hardware
+registers. There is no way for the driver to determine if the other events
+are available or not. Hence the driver exposes only those events advertised
+by the DSU, in "events" directory under :
+
+  /sys/bus/event_sources/devices/arm_dsu_<N>/
+
+The user should refer to the TRM of the product to figure out the supported events
+and use the raw event code for the unlisted events.
+
+The driver also exposes the CPUs connected to the DSU instance in "associated_cpus".
+
+
+e.g usage :
+
+	perf stat -a -e arm_dsu_0/cycles/
diff --git a/Documentation/power/pci.txt b/Documentation/power/pci.txt
index 704cd36079b8..8eaf9ee24d43 100644
--- a/Documentation/power/pci.txt
+++ b/Documentation/power/pci.txt
@@ -994,6 +994,17 @@ into D0 going forward), but if it is in runtime suspend in pci_pm_thaw_noirq(),
 the function will set the power.direct_complete flag for it (to make the PM core
 skip the subsequent "thaw" callbacks for it) and return.
 
+Setting the DPM_FLAG_LEAVE_SUSPENDED flag means that the driver prefers the
+device to be left in suspend after system-wide transitions to the working state.
+This flag is checked by the PM core, but the PCI bus type informs the PM core
+which devices may be left in suspend from its perspective (that happens during
+the "noirq" phase of system-wide suspend and analogous transitions) and next it
+uses the dev_pm_may_skip_resume() helper to decide whether or not to return from
+pci_pm_resume_noirq() early, as the PM core will skip the remaining resume
+callbacks for the device during the transition under way and will set its
+runtime PM status to "suspended" if dev_pm_may_skip_resume() returns "true" for
+it.
+
 3.2. Device Runtime Power Management
 ------------------------------------
 In addition to providing device power management callbacks PCI device drivers
diff --git a/Documentation/power/regulator/machine.txt b/Documentation/power/regulator/machine.txt
index 757e3b53dc11..eff4dcaaa252 100644
--- a/Documentation/power/regulator/machine.txt
+++ b/Documentation/power/regulator/machine.txt
@@ -23,16 +23,12 @@ struct regulator_consumer_supply {
 e.g. for the machine above
 
 static struct regulator_consumer_supply regulator1_consumers[] = {
-{
-	.dev_name	= "dev_name(consumer B)",
-	.supply		= "Vcc",
-},};
+	REGULATOR_SUPPLY("Vcc", "consumer B"),
+};
 
 static struct regulator_consumer_supply regulator2_consumers[] = {
-{
-	.dev	= "dev_name(consumer A"),
-	.supply	= "Vcc",
-},};
+	REGULATOR_SUPPLY("Vcc", "consumer A"),
+};
 
 This maps Regulator-1 to the 'Vcc' supply for Consumer B and maps Regulator-2
 to the 'Vcc' supply for Consumer A.
@@ -78,20 +74,20 @@ static struct regulator_init_data regulator2_data = {
 Finally the regulator devices must be registered in the usual manner.
 
 static struct platform_device regulator_devices[] = {
-{
-	.name = "regulator",
-	.id = DCDC_1,
-	.dev = {
-		.platform_data = &regulator1_data,
+	{
+		.name = "regulator",
+		.id = DCDC_1,
+		.dev = {
+			.platform_data = &regulator1_data,
+		},
 	},
-},
-{
-	.name = "regulator",
-	.id = DCDC_2,
-	.dev = {
-		.platform_data = &regulator2_data,
+	{
+		.name = "regulator",
+		.id = DCDC_2,
+		.dev = {
+			.platform_data = &regulator2_data,
+		},
 	},
-},
 };
 /* register regulator 1 device */
 platform_device_register(&regulator_devices[0]);
diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst
index 560beaef5a7c..81cdb528ad46 100644
--- a/Documentation/process/changes.rst
+++ b/Documentation/process/changes.rst
@@ -32,6 +32,8 @@ you probably needn't concern yourself with isdn4k-utils.
 GNU C                  3.2              gcc --version
 GNU make               3.81             make --version
 binutils               2.20             ld -v
+flex                   2.5.35           flex --version
+bison                  2.0              bison --version
 util-linux             2.10o            fdformat --version
 module-init-tools      0.9.10           depmod -V
 e2fsprogs              1.41.4           e2fsck -V
@@ -79,6 +81,19 @@ The build system has, as of 4.13, switched to using thin archives (`ar T`)
 rather than incremental linking (`ld -r`) for built-in.o intermediate steps.
 This requires binutils 2.20 or newer.
 
+Flex
+----
+
+Since Linux 4.16, the build system generates lexical analyzers
+during build.  This requires flex 2.5.35 or later.
+
+
+Bison
+-----
+
+Since Linux 4.16, the build system generates parsers
+during build.  This requires bison 2.0 or later.
+
 Perl
 ----
 
@@ -333,6 +348,16 @@ Binutils
 
 - <https://www.kernel.org/pub/linux/devel/binutils/>
 
+Flex
+----
+
+- <https://github.com/westes/flex/releases>
+
+Bison
+-----
+
+- <ftp://ftp.gnu.org/gnu/bison/>
+
 OpenSSL
 -------
 
diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst
index a430f6eee756..1c9fe657ed01 100644
--- a/Documentation/process/index.rst
+++ b/Documentation/process/index.rst
@@ -24,6 +24,7 @@ Below are the essential guides that every developer should read.
    development-process
    submitting-patches
    coding-style
+   maintainer-pgp-guide
    email-clients
    kernel-enforcement-statement
    kernel-driver-statement
diff --git a/Documentation/process/kernel-docs.rst b/Documentation/process/kernel-docs.rst
index b8cac85a4001..3fb28de556e4 100644
--- a/Documentation/process/kernel-docs.rst
+++ b/Documentation/process/kernel-docs.rst
@@ -58,7 +58,7 @@ On-line docs
     * Title: **Linux Kernel Mailing List Glossary**
 
       :Author: various
-      :URL: http://kernelnewbies.org/glossary/
+      :URL: https://kernelnewbies.org/KernelGlossary
       :Date: rolling version
       :Keywords: glossary, terms, linux-kernel.
       :Description: From the introduction: "This glossary is intended as
diff --git a/Documentation/process/kernel-enforcement-statement.rst b/Documentation/process/kernel-enforcement-statement.rst
index b3170671a1df..6816c12d6956 100644
--- a/Documentation/process/kernel-enforcement-statement.rst
+++ b/Documentation/process/kernel-enforcement-statement.rst
@@ -68,6 +68,7 @@ we might work for today, have in the past, or will in the future.
   - Paul Burton
   - Javier Martinez Canillas
   - Rob Clark
+  - Kees Cook (Google)
   - Jonathan Corbet
   - Dennis Dalessandro
   - Vivien Didelot (Savoir-faire Linux)
@@ -118,6 +119,7 @@ we might work for today, have in the past, or will in the future.
   - Mike Marshall
   - Chris Mason
   - Paul E. McKenney
+  - Arnaldo Carvalho de Melo
   - David S. Miller
   - Ingo Molnar
   - Kuninori Morimoto
@@ -136,6 +138,7 @@ we might work for today, have in the past, or will in the future.
   - Anna Schumaker
   - Jes Sorensen
   - K.Y. Srinivasan
+  - David Sterba (SUSE)
   - Heiko Stuebner
   - Jiri Kosina (SUSE)
   - Willy Tarreau
@@ -143,6 +146,7 @@ we might work for today, have in the past, or will in the future.
   - Linus Torvalds
   - Thierry Reding
   - Rik van Riel
+  - Luis R. Rodriguez
   - Geert Uytterhoeven (Glider bvba)
   - Eduardo Valentin (Amazon.com)
   - Daniel Vetter
diff --git a/Documentation/process/license-rules.rst b/Documentation/process/license-rules.rst
new file mode 100644
index 000000000000..408f77dc6157
--- /dev/null
+++ b/Documentation/process/license-rules.rst
@@ -0,0 +1,370 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Linux kernel licensing rules
+============================
+
+The Linux Kernel is provided under the terms of the GNU General Public
+License version 2 only (GPL-2.0), as published by the Free Software
+Foundation, and provided in the COPYING file.  This documentation file is
+not meant to replace the COPYING file, but provides a description of how
+each source file should be annotated to make the licensing it is governed
+under clear and unambiguous.
+
+The license in the COPYING file applies to the kernel source as a whole,
+though individual source files can have a different license which is
+required to be compatible with the GPL-2.0::
+
+    GPL-1.0+  :  GNU General Public License v1.0 or later
+    GPL-2.0+  :  GNU General Public License v2.0 or later
+    LGPL-2.0  :  GNU Library General Public License v2 only
+    LGPL-2.0+ :  GNU Library General Public License v2 or later
+    LGPL-2.1  :  GNU Lesser General Public License v2.1 only
+    LGPL-2.1+ :  GNU Lesser General Public License v2.1 or later
+
+Aside from that, individual files can be provided under a dual license,
+e.g. one of the compatible GPL variants and alternatively under a
+permissive license like BSD, MIT etc.
+
+The User-space API (UAPI) header files, which describe the interface of
+user-space programs to the kernel are a special case.  According to the
+note in the kernel COPYING file, the syscall interface is a clear boundary,
+which does not extend the GPL requirements to any software which uses it to
+communicate with the kernel.  Because the UAPI headers must be includable
+into any source files which create an executable running on the Linux
+kernel, the exception must be documented by a special license expression.
+
+The common way of expressing the license of a source file is to add the
+matching boilerplate text into the top comment of the file.  Due to
+formatting, typos etc. these "boilerplates" are hard to validate for
+tools which are used in the context of license compliance.
+
+An alternative to boilerplate text is the use of Software Package Data
+Exchange (SPDX) license identifiers in each source file.  SPDX license
+identifiers are machine parsable and precise shorthands for the license
+under which the content of the file is contributed.  SPDX license
+identifiers are managed by the SPDX Workgroup at the Linux Foundation and
+have been agreed on by partners throughout the industry, tool vendors, and
+legal teams.  For further information see https://spdx.org/
+
+The Linux kernel requires the precise SPDX identifier in all source files.
+The valid identifiers used in the kernel are explained in the section
+`License identifiers`_ and have been retrieved from the official SPDX
+license list at https://spdx.org/licenses/ along with the license texts.
+
+License identifier syntax
+-------------------------
+
+1. Placement:
+
+   The SPDX license identifier in kernel files shall be added at the first
+   possible line in a file which can contain a comment.  For the majority
+   or files this is the first line, except for scripts which require the
+   '#!PATH_TO_INTERPRETER' in the first line.  For those scripts the SPDX
+   identifier goes into the second line.
+
+|
+
+2. Style:
+
+   The SPDX license identifier is added in form of a comment.  The comment
+   style depends on the file type::
+
+      C source:	// SPDX-License-Identifier: <SPDX License Expression>
+      C header:	/* SPDX-License-Identifier: <SPDX License Expression> */
+      ASM:	/* SPDX-License-Identifier: <SPDX License Expression> */
+      scripts:	# SPDX-License-Identifier: <SPDX License Expression>
+      .rst:	.. SPDX-License-Identifier: <SPDX License Expression>
+      .dts{i}:	// SPDX-License-Identifier: <SPDX License Expression>
+
+   If a specific tool cannot handle the standard comment style, then the
+   appropriate comment mechanism which the tool accepts shall be used. This
+   is the reason for having the "/\* \*/" style comment in C header
+   files. There was build breakage observed with generated .lds files where
+   'ld' failed to parse the C++ comment. This has been fixed by now, but
+   there are still older assembler tools which cannot handle C++ style
+   comments.
+
+|
+
+3. Syntax:
+
+   A <SPDX License Expression> is either an SPDX short form license
+   identifier found on the SPDX License List, or the combination of two
+   SPDX short form license identifiers separated by "WITH" when a license
+   exception applies. When multiple licenses apply, an expression consists
+   of keywords "AND", "OR" separating sub-expressions and surrounded by
+   "(", ")" .
+
+   License identifiers for licenses like [L]GPL with the 'or later' option
+   are constructed by using a "+" for indicating the 'or later' option.::
+
+      // SPDX-License-Identifier: GPL-2.0+
+      // SPDX-License-Identifier: LGPL-2.1+
+
+   WITH should be used when there is a modifier to a license needed.
+   For example, the linux kernel UAPI files use the expression::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+      // SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note
+
+   Other examples using WITH exceptions found in the kernel are::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH mif-exception
+      // SPDX-License-Identifier: GPL-2.0+ WITH GCC-exception-2.0
+
+   Exceptions can only be used with particular License identifiers. The
+   valid License identifiers are listed in the tags of the exception text
+   file. For details see the point `Exceptions`_ in the chapter `License
+   identifiers`_.
+
+   OR should be used if the file is dual licensed and only one license is
+   to be selected.  For example, some dtsi files are available under dual
+   licenses::
+
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+
+   Examples from the kernel for license expressions in dual licensed files::
+
+      // SPDX-License-Identifier: GPL-2.0 OR MIT
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+      // SPDX-License-Identifier: GPL-2.0 OR Apache-2.0
+      // SPDX-License-Identifier: GPL-2.0 OR MPL-1.1
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT
+      // SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause OR OpenSSL
+
+   AND should be used if the file has multiple licenses whose terms all
+   apply to use the file. For example, if code is inherited from another
+   project and permission has been given to put it in the kernel, but the
+   original license terms need to remain in effect::
+
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) AND MIT
+
+   Another other example where both sets of license terms need to be
+   adhered to is::
+
+      // SPDX-License-Identifier: GPL-1.0+ AND LGPL-2.1+
+
+License identifiers
+-------------------
+
+The licenses currently used, as well as the licenses for code added to the
+kernel, can be broken down into:
+
+1. _`Preferred licenses`:
+
+   Whenever possible these licenses should be used as they are known to be
+   fully compatible and widely used.  These licenses are available from the
+   directory::
+
+      LICENSES/preferred/
+
+   in the kernel source tree.
+
+   The files in this directory contain the full license text and
+   `Metatags`_.  The file names are identical to the SPDX license
+   identifier which shall be used for the license in source files.
+
+   Examples::
+
+      LICENSES/preferred/GPL-2.0
+
+   Contains the GPL version 2 license text and the required metatags::
+
+      LICENSES/preferred/MIT
+
+   Contains the MIT license text and the required metatags
+
+   _`Metatags`:
+
+   The following meta tags must be available in a license file:
+
+   - Valid-License-Identifier:
+
+     One or more lines which declare which License Identifiers are valid
+     inside the project to reference this particular license text.  Usually
+     this is a single valid identifier, but e.g. for licenses with the 'or
+     later' options two identifiers are valid.
+
+   - SPDX-URL:
+
+     The URL of the SPDX page which contains additional information related
+     to the license.
+
+   - Usage-Guidance:
+
+     Freeform text for usage advice. The text must include correct examples
+     for the SPDX license identifiers as they should be put into source
+     files according to the `License identifier syntax`_ guidelines.
+
+   - License-Text:
+
+     All text after this tag is treated as the original license text
+
+   File format examples::
+
+      Valid-License-Identifier: GPL-2.0
+      Valid-License-Identifier: GPL-2.0+
+      SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
+      Usage-Guide:
+        To use this license in source code, put one of the following SPDX
+	tag/value pairs into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	For 'GNU General Public License (GPL) version 2 only' use:
+	  SPDX-License-Identifier: GPL-2.0
+	For 'GNU General Public License (GPL) version 2 or any later version' use:
+	  SPDX-License-Identifier: GPL-2.0+
+      License-Text:
+        Full license text
+
+   ::
+
+      SPDX-License-Identifier: MIT
+      SPDX-URL: https://spdx.org/licenses/MIT.html
+      Usage-Guide:
+	To use this license in source code, put the following SPDX
+	tag/value pair into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	  SPDX-License-Identifier: MIT
+      License-Text:
+        Full license text
+
+|
+
+2. Not recommended licenses:
+
+   These licenses should only be used for existing code or for importing
+   code from a different project.  These licenses are available from the
+   directory::
+
+      LICENSES/other/
+
+   in the kernel source tree.
+
+   The files in this directory contain the full license text and
+   `Metatags`_.  The file names are identical to the SPDX license
+   identifier which shall be used for the license in source files.
+
+   Examples::
+
+      LICENSES/other/ISC
+
+   Contains the Internet Systems Consortium license text and the required
+   metatags::
+
+      LICENSES/other/ZLib
+
+   Contains the ZLIB license text and the required metatags.
+
+   Metatags:
+
+   The metatag requirements for 'other' licenses are identical to the
+   requirements of the `Preferred licenses`_.
+
+   File format example::
+
+      Valid-License-Identifier: ISC
+      SPDX-URL: https://spdx.org/licenses/ISC.html
+      Usage-Guide:
+        Usage of this license in the kernel for new code is discouraged
+	and it should solely be used for importing code from an already
+	existing project.
+        To use this license in source code, put the following SPDX
+	tag/value pair into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	  SPDX-License-Identifier: ISC
+      License-Text:
+        Full license text
+
+|
+
+3. _`Exceptions`:
+
+   Some licenses can be amended with exceptions which grant certain rights
+   which the original license does not.  These exceptions are available
+   from the directory::
+
+      LICENSES/exceptions/
+
+   in the kernel source tree.  The files in this directory contain the full
+   exception text and the required `Exception Metatags`_.
+
+   Examples::
+
+      LICENSES/exceptions/Linux-syscall-note
+
+   Contains the Linux syscall exception as documented in the COPYING
+   file of the Linux kernel, which is used for UAPI header files.
+   e.g. /\* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note \*/::
+
+      LICENSES/exceptions/GCC-exception-2.0
+
+   Contains the GCC 'linking exception' which allows to link any binary
+   independent of its license against the compiled version of a file marked
+   with this exception. This is required for creating runnable executables
+   from source code which is not compatible with the GPL.
+
+   _`Exception Metatags`:
+
+   The following meta tags must be available in an exception file:
+
+   - SPDX-Exception-Identifier:
+
+     One exception identifier which can be used with SPDX license
+     identifiers.
+
+   - SPDX-URL:
+
+     The URL of the SPDX page which contains additional information related
+     to the exception.
+
+   - SPDX-Licenses:
+
+     A comma separated list of SPDX license identifiers for which the
+     exception can be used.
+
+   - Usage-Guidance:
+
+     Freeform text for usage advice. The text must be followed by correct
+     examples for the SPDX license identifiers as they should be put into
+     source files according to the `License identifier syntax`_ guidelines.
+
+   - Exception-Text:
+
+     All text after this tag is treated as the original exception text
+
+   File format examples::
+
+      SPDX-Exception-Identifier: Linux-syscall-note
+      SPDX-URL: https://spdx.org/licenses/Linux-syscall-note.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+, GPL-1.0+, LGPL-2.0, LGPL-2.0+, LGPL-2.1, LGPL-2.1+
+      Usage-Guidance:
+        This exception is used together with one of the above SPDX-Licenses
+	to mark user-space API (uapi) header files so they can be included
+	into non GPL compliant user-space application code.
+        To use this exception add it with the keyword WITH to one of the
+	identifiers in the SPDX-Licenses tag:
+	  SPDX-License-Identifier: <SPDX-License> WITH Linux-syscall-note
+      Exception-Text:
+        Full exception text
+
+   ::
+
+      SPDX-Exception-Identifier: GCC-exception-2.0
+      SPDX-URL: https://spdx.org/licenses/GCC-exception-2.0.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+
+      Usage-Guidance:
+        The "GCC Runtime Library exception 2.0" is used together with one
+	of the above SPDX-Licenses for code imported from the GCC runtime
+	library.
+        To use this exception add it with the keyword WITH to one of the
+	identifiers in the SPDX-Licenses tag:
+	  SPDX-License-Identifier: <SPDX-License> WITH GCC-exception-2.0
+      Exception-Text:
+        Full exception text
+
+
+All SPDX license identifiers and exceptions must have a corresponding file
+in the LICENSE subdirectories. This is required to allow tool
+verification (e.g. checkpatch.pl) and to have the licenses ready to read
+and extract right from the source, which is recommended by various FOSS
+organizations, e.g. the `FSFE REUSE initiative <https://reuse.software/>`_.
diff --git a/Documentation/process/maintainer-pgp-guide.rst b/Documentation/process/maintainer-pgp-guide.rst
new file mode 100644
index 000000000000..b453561a7148
--- /dev/null
+++ b/Documentation/process/maintainer-pgp-guide.rst
@@ -0,0 +1,929 @@
+.. _pgpguide:
+
+===========================
+Kernel Maintainer PGP guide
+===========================
+
+:Author: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
+
+This document is aimed at Linux kernel developers, and especially at
+subsystem maintainers. It contains a subset of information discussed in
+the more general "`Protecting Code Integrity`_" guide published by the
+Linux Foundation. Please read that document for more in-depth discussion
+on some of the topics mentioned in this guide.
+
+.. _`Protecting Code Integrity`: https://github.com/lfit/itpol/blob/master/protecting-code-integrity.md
+
+The role of PGP in Linux Kernel development
+===========================================
+
+PGP helps ensure the integrity of the code that is produced by the Linux
+kernel development community and, to a lesser degree, establish trusted
+communication channels between developers via PGP-signed email exchange.
+
+The Linux kernel source code is available in two main formats:
+
+- Distributed source repositories (git)
+- Periodic release snapshots (tarballs)
+
+Both git repositories and tarballs carry PGP signatures of the kernel
+developers who create official kernel releases. These signatures offer a
+cryptographic guarantee that downloadable versions made available via
+kernel.org or any other mirrors are identical to what these developers
+have on their workstations. To this end:
+
+- git repositories provide PGP signatures on all tags
+- tarballs provide detached PGP signatures with all downloads
+
+.. _devs_not_infra:
+
+Trusting the developers, not infrastructure
+-------------------------------------------
+
+Ever since the 2011 compromise of core kernel.org systems, the main
+operating principle of the Kernel Archives project has been to assume
+that any part of the infrastructure can be compromised at any time. For
+this reason, the administrators have taken deliberate steps to emphasize
+that trust must always be placed with developers and never with the code
+hosting infrastructure, regardless of how good the security practices
+for the latter may be.
+
+The above guiding principle is the reason why this guide is needed. We
+want to make sure that by placing trust into developers we do not simply
+shift the blame for potential future security incidents to someone else.
+The goal is to provide a set of guidelines developers can use to create
+a secure working environment and safeguard the PGP keys used to
+establish the integrity of the Linux kernel itself.
+
+.. _pgp_tools:
+
+PGP tools
+=========
+
+Use GnuPG v2
+------------
+
+Your distro should already have GnuPG installed by default, you just
+need to verify that you are using version 2.x and not the legacy 1.4
+release -- many distributions still package both, with the default
+``gpg`` command invoking GnuPG v.1. To check, run::
+
+    $ gpg --version | head -n1
+
+If you see ``gpg (GnuPG) 1.4.x``, then you are using GnuPG v.1. Try the
+``gpg2`` command (if you don't have it, you may need to install the
+gnupg2 package)::
+
+    $ gpg2 --version | head -n1
+
+If you see ``gpg (GnuPG) 2.x.x``, then you are good to go. This guide
+will assume you have the version 2.2 of GnuPG (or later). If you are
+using version 2.0 of GnuPG, then some of the commands in this guide will
+not work, and you should consider installing the latest 2.2 version of
+GnuPG. Versions of gnupg-2.1.11 and later should be compatible for the
+purposes of this guide as well.
+
+If you have both ``gpg`` and ``gpg2`` commands, you should make sure you
+are always using GnuPG v2, not the legacy version. You can enforce this
+by setting the appropriate alias::
+
+    $ alias gpg=gpg2
+
+You can put that in your ``.bashrc`` to make sure it's always the case.
+
+Configure gpg-agent options
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The GnuPG agent is a helper tool that will start automatically whenever
+you use the ``gpg`` command and run in the background with the purpose
+of caching the private key passphrase. There are two options you should
+know in order to tweak when the passphrase should be expired from cache:
+
+- ``default-cache-ttl`` (seconds): If you use the same key again before
+  the time-to-live expires, the countdown will reset for another period.
+  The default is 600 (10 minutes).
+- ``max-cache-ttl`` (seconds): Regardless of how recently you've used
+  the key since initial passphrase entry, if the maximum time-to-live
+  countdown expires, you'll have to enter the passphrase again. The
+  default is 30 minutes.
+
+If you find either of these defaults too short (or too long), you can
+edit your ``~/.gnupg/gpg-agent.conf`` file to set your own values::
+
+    # set to 30 minutes for regular ttl, and 2 hours for max ttl
+    default-cache-ttl 1800
+    max-cache-ttl 7200
+
+.. note::
+
+    It is no longer necessary to start gpg-agent manually at the
+    beginning of your shell session. You may want to check your rc files
+    to remove anything you had in place for older versions of GnuPG, as
+    it may not be doing the right thing any more.
+
+Set up a refresh cronjob
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+You will need to regularly refresh your keyring in order to get the
+latest changes on other people's public keys, which is best done with a
+daily cronjob::
+
+    @daily /usr/bin/gpg2 --refresh >/dev/null 2>&1
+
+Check the full path to your ``gpg`` or ``gpg2`` command and use the
+``gpg2`` command if regular ``gpg`` for you is the legacy GnuPG v.1.
+
+.. _master_key:
+
+Protect your master PGP key
+===========================
+
+This guide assumes that you already have a PGP key that you use for Linux
+kernel development purposes. If you do not yet have one, please see the
+"`Protecting Code Integrity`_" document mentioned earlier for guidance
+on how to create a new one.
+
+You should also make a new key if your current one is weaker than 2048 bits
+(RSA).
+
+Master key vs. Subkeys
+----------------------
+
+Subkeys are fully independent PGP keypairs that are tied to the "master"
+key using certifying key signatures (certificates). It is important to
+understand the following:
+
+1. There are no technical differences between the "master key" and "subkeys."
+2. At creation time, we assign functional limitations to each key by
+   giving it specific capabilities.
+3. A PGP key can have 4 capabilities:
+
+   - **[S]** key can be used for signing
+   - **[E]** key can be used for encryption
+   - **[A]** key can be used for authentication
+   - **[C]** key can be used for certifying other keys
+
+4. A single key may have multiple capabilities.
+5. A subkey is fully independent from the master key. A message
+   encrypted to a subkey cannot be decrypted with the master key. If you
+   lose your private subkey, it cannot be recreated from the master key
+   in any way.
+
+The key carrying the **[C]** (certify) capability is considered the
+"master" key because it is the only key that can be used to indicate
+relationship with other keys. Only the **[C]** key can be used to:
+
+- add or revoke other keys (subkeys) with S/E/A capabilities
+- add, change or revoke identities (uids) associated with the key
+- add or change the expiration date on itself or any subkey
+- sign other people's keys for web of trust purposes
+
+By default, GnuPG creates the following when generating new keys:
+
+- A master key carrying both Certify and Sign capabilities (**[SC]**)
+- A separate subkey with the Encryption capability (**[E]**)
+
+If you used the default parameters when generating your key, then that
+is what you will have. You can verify by running ``gpg --list-secret-keys``,
+for example::
+
+    sec   rsa2048 2018-01-23 [SC] [expires: 2020-01-23]
+          000000000000000000000000AAAABBBBCCCCDDDD
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    ssb   rsa2048 2018-01-23 [E] [expires: 2020-01-23]
+
+Any key carrying the **[C]** capability is your master key, regardless
+of any other capabilities it may have assigned to it.
+
+The long line under the ``sec`` entry is your key fingerprint --
+whenever you see ``[fpr]`` in the examples below, that 40-character
+string is what it refers to.
+
+Ensure your passphrase is strong
+--------------------------------
+
+GnuPG uses passphrases to encrypt your private keys before storing them on
+disk. This way, even if your ``.gnupg`` directory is leaked or stolen in
+its entirety, the attackers cannot use your private keys without first
+obtaining the passphrase to decrypt them.
+
+It is absolutely essential that your private keys are protected by a
+strong passphrase. To set it or change it, use::
+
+    $ gpg --change-passphrase [fpr]
+
+Create a separate Signing subkey
+--------------------------------
+
+Our goal is to protect your master key by moving it to offline media, so
+if you only have a combined **[SC]** key, then you should create a separate
+signing subkey::
+
+    $ gpg --quick-add-key [fpr] ed25519 sign
+
+Remember to tell the keyservers about this change, so others can pull down
+your new subkey::
+
+    $ gpg --send-key [fpr]
+
+.. note:: ECC support in GnuPG
+
+    GnuPG 2.1 and later has full support for Elliptic Curve
+    Cryptography, with ability to combine ECC subkeys with traditional
+    RSA master keys. The main upside of ECC cryptography is that it is
+    much faster computationally and creates much smaller signatures when
+    compared byte for byte with 2048+ bit RSA keys. Unless you plan on
+    using a smartcard device that does not support ECC operations, we
+    recommend that you create an ECC signing subkey for your kernel
+    work.
+
+    If for some reason you prefer to stay with RSA subkeys, just replace
+    "ed25519" with "rsa2048" in the above command.
+
+
+Back up your master key for disaster recovery
+---------------------------------------------
+
+The more signatures you have on your PGP key from other developers, the
+more reasons you have to create a backup version that lives on something
+other than digital media, for disaster recovery reasons.
+
+The best way to create a printable hardcopy of your private key is by
+using the ``paperkey`` software written for this very purpose. See ``man
+paperkey`` for more details on the output format and its benefits over
+other solutions. Paperkey should already be packaged for most
+distributions.
+
+Run the following command to create a hardcopy backup of your private
+key::
+
+    $ gpg --export-secret-key [fpr] | paperkey -o /tmp/key-backup.txt
+
+Print out that file (or pipe the output straight to lpr), then take a
+pen and write your passphrase on the margin of the paper. **This is
+strongly recommended** because the key printout is still encrypted with
+that passphrase, and if you ever change it you will not remember what it
+used to be when you had created the backup -- *guaranteed*.
+
+Put the resulting printout and the hand-written passphrase into an envelope
+and store in a secure and well-protected place, preferably away from your
+home, such as your bank vault.
+
+.. note::
+
+    Your printer is probably no longer a simple dumb device connected to
+    your parallel port, but since the output is still encrypted with
+    your passphrase, printing out even to "cloud-integrated" modern
+    printers should remain a relatively safe operation. One option is to
+    change the passphrase on your master key immediately after you are
+    done with paperkey.
+
+Back up your whole GnuPG directory
+----------------------------------
+
+.. warning::
+
+    **!!!Do not skip this step!!!**
+
+It is important to have a readily available backup of your PGP keys
+should you need to recover them. This is different from the
+disaster-level preparedness we did with ``paperkey``. You will also rely
+on these external copies whenever you need to use your Certify key --
+such as when making changes to your own key or signing other people's
+keys after conferences and summits.
+
+Start by getting a small USB "thumb" drive (preferably two!) that you
+will use for backup purposes. You will need to encrypt them using LUKS
+-- refer to your distro's documentation on how to accomplish this.
+
+For the encryption passphrase, you can use the same one as on your
+master key.
+
+Once the encryption process is over, re-insert the USB drive and make
+sure it gets properly mounted. Copy your entire ``.gnupg`` directory
+over to the encrypted storage::
+
+    $ cp -a ~/.gnupg /media/disk/foo/gnupg-backup
+
+You should now test to make sure everything still works::
+
+    $ gpg --homedir=/media/disk/foo/gnupg-backup --list-key [fpr]
+
+If you don't get any errors, then you should be good to go. Unmount the
+USB drive, distinctly label it so you don't blow it away next time you
+need to use a random USB drive, and put in a safe place -- but not too
+far away, because you'll need to use it every now and again for things
+like editing identities, adding or revoking subkeys, or signing other
+people's keys.
+
+Remove the master key from  your homedir
+----------------------------------------
+
+The files in our home directory are not as well protected as we like to
+think.  They can be leaked or stolen via many different means:
+
+- by accident when making quick homedir copies to set up a new workstation
+- by systems administrator negligence or malice
+- via poorly secured backups
+- via malware in desktop apps (browsers, pdf viewers, etc)
+- via coercion when crossing international borders
+
+Protecting your key with a good passphrase greatly helps reduce the risk
+of any of the above, but passphrases can be discovered via keyloggers,
+shoulder-surfing, or any number of other means. For this reason, the
+recommended setup is to remove your master key from your home directory
+and store it on offline storage.
+
+.. warning::
+
+    Please see the previous section and make sure you have backed up
+    your GnuPG directory in its entirety. What we are about to do will
+    render your key useless if you do not have a usable backup!
+
+First, identify the keygrip of your master key::
+
+    $ gpg --with-keygrip --list-key [fpr]
+
+The output will be something like this::
+
+    pub   rsa2048 2018-01-24 [SC] [expires: 2020-01-24]
+          000000000000000000000000AAAABBBBCCCCDDDD
+          Keygrip = 1111000000000000000000000000000000000000
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    sub   rsa2048 2018-01-24 [E] [expires: 2020-01-24]
+          Keygrip = 2222000000000000000000000000000000000000
+    sub   ed25519 2018-01-24 [S]
+          Keygrip = 3333000000000000000000000000000000000000
+
+Find the keygrip entry that is beneath the ``pub`` line (right under the
+master key fingerprint). This will correspond directly to a file in your
+``~/.gnupg`` directory::
+
+    $ cd ~/.gnupg/private-keys-v1.d
+    $ ls
+    1111000000000000000000000000000000000000.key
+    2222000000000000000000000000000000000000.key
+    3333000000000000000000000000000000000000.key
+
+All you have to do is simply remove the .key file that corresponds to
+the master keygrip::
+
+    $ cd ~/.gnupg/private-keys-v1.d
+    $ rm 1111000000000000000000000000000000000000.key
+
+Now, if you issue the ``--list-secret-keys`` command, it will show that
+the master key is missing (the ``#`` indicates it is not available)::
+
+    $ gpg --list-secret-keys
+    sec#  rsa2048 2018-01-24 [SC] [expires: 2020-01-24]
+          000000000000000000000000AAAABBBBCCCCDDDD
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    ssb   rsa2048 2018-01-24 [E] [expires: 2020-01-24]
+    ssb   ed25519 2018-01-24 [S]
+
+You should also remove any ``secring.gpg`` files in the ``~/.gnupg``
+directory, which are left over from earlier versions of GnuPG.
+
+If you don't have the "private-keys-v1.d" directory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you do not have a ``~/.gnupg/private-keys-v1.d`` directory, then your
+secret keys are still stored in the legacy ``secring.gpg`` file used by
+GnuPG v1. Making any changes to your key, such as changing the
+passphrase or adding a subkey, should automatically convert the old
+``secring.gpg`` format to use ``private-keys-v1.d`` instead.
+
+Once you get that done, make sure to delete the obsolete ``secring.gpg``
+file, which still contains your private keys.
+
+.. _smartcards:
+
+Move the subkeys to a dedicated crypto device
+=============================================
+
+Even though the master key is now safe from being leaked or stolen, the
+subkeys are still in your home directory. Anyone who manages to get
+their hands on those will be able to decrypt your communication or fake
+your signatures (if they know the passphrase). Furthermore, each time a
+GnuPG operation is performed, the keys are loaded into system memory and
+can be stolen from there by sufficiently advanced malware (think
+Meltdown and Spectre).
+
+The best way to completely protect your keys is to move them to a
+specialized hardware device that is capable of smartcard operations.
+
+The benefits of smartcards
+--------------------------
+
+A smartcard contains a cryptographic chip that is capable of storing
+private keys and performing crypto operations directly on the card
+itself. Because the key contents never leave the smartcard, the
+operating system of the computer into which you plug in the hardware
+device is not able to retrieve the private keys themselves. This is very
+different from the encrypted USB storage device we used earlier for
+backup purposes -- while that USB device is plugged in and mounted, the
+operating system is able to access the private key contents.
+
+Using external encrypted USB media is not a substitute to having a
+smartcard-capable device.
+
+Available smartcard devices
+---------------------------
+
+Unless all your laptops and workstations have smartcard readers, the
+easiest is to get a specialized USB device that implements smartcard
+functionality.  There are several options available:
+
+- `Nitrokey Start`_: Open hardware and Free Software, based on FSI
+  Japan's `Gnuk`_. Offers support for ECC keys, but fewest security
+  features (such as resistance to tampering or some side-channel
+  attacks).
+- `Nitrokey Pro`_: Similar to the Nitrokey Start, but more
+  tamper-resistant and offers more security features, but no ECC
+  support.
+- `Yubikey 4`_: proprietary hardware and software, but cheaper than
+  Nitrokey Pro and comes available in the USB-C form that is more useful
+  with newer laptops. Offers additional security features such as FIDO
+  U2F, but no ECC.
+
+`LWN has a good review`_ of some of the above models, as well as several
+others. If you want to use ECC keys, your best bet among commercially
+available devices is the Nitrokey Start.
+
+.. _`Nitrokey Start`: https://shop.nitrokey.com/shop/product/nitrokey-start-6
+.. _`Nitrokey Pro`: https://shop.nitrokey.com/shop/product/nitrokey-pro-3
+.. _`Yubikey 4`: https://www.yubico.com/product/yubikey-4-series/
+.. _Gnuk: http://www.fsij.org/doc-gnuk/
+.. _`LWN has a good review`: https://lwn.net/Articles/736231/
+
+Configure your smartcard device
+-------------------------------
+
+Your smartcard device should Just Work (TM) the moment you plug it into
+any modern Linux workstation. You can verify it by running::
+
+    $ gpg --card-status
+
+If you see full smartcard details, then you are good to go.
+Unfortunately, troubleshooting all possible reasons why things may not
+be working for you is way beyond the scope of this guide. If you are
+having trouble getting the card to work with GnuPG, please seek help via
+usual support channels.
+
+To configure your smartcard, you will need to use the GnuPG menu system, as
+there are no convenient command-line switches::
+
+    $ gpg --card-edit
+    [...omitted...]
+    gpg/card> admin
+    Admin commands are allowed
+    gpg/card> passwd
+
+You should set the user PIN (1), Admin PIN (3), and the Reset Code (4).
+Please make sure to record and store these in a safe place -- especially
+the Admin PIN and the Reset Code (which allows you to completely wipe
+the smartcard).  You so rarely need to use the Admin PIN, that you will
+inevitably forget what it is if you do not record it.
+
+Getting back to the main card menu, you can also set other values (such
+as name, sex, login data, etc), but it's not necessary and will
+additionally leak information about your smartcard should you lose it.
+
+.. note::
+
+    Despite having the name "PIN", neither the user PIN nor the admin
+    PIN on the card need to be numbers.
+
+Move the subkeys to your smartcard
+----------------------------------
+
+Exit the card menu (using "q") and save all changes. Next, let's move
+your subkeys onto the smartcard. You will need both your PGP key
+passphrase and the admin PIN of the card for most operations::
+
+    $ gpg --edit-key [fpr]
+
+    Secret subkeys are available.
+
+    pub  rsa2048/AAAABBBBCCCCDDDD
+         created: 2018-01-23  expires: 2020-01-23  usage: SC
+         trust: ultimate      validity: ultimate
+    ssb  rsa2048/1111222233334444
+         created: 2018-01-23  expires: never       usage: E
+    ssb  ed25519/5555666677778888
+         created: 2017-12-07  expires: never       usage: S
+    [ultimate] (1). Alice Dev <adev@kernel.org>
+
+    gpg>
+
+Using ``--edit-key`` puts us into the menu mode again, and you will
+notice that the key listing is a little different. From here on, all
+commands are done from inside this menu mode, as indicated by ``gpg>``.
+
+First, let's select the key we'll be putting onto the card -- you do
+this by typing ``key 1`` (it's the first one in the listing, the **[E]**
+subkey)::
+
+    gpg> key 1
+
+In the output, you should now see ``ssb*`` on the **[E]** key. The ``*``
+indicates which key is currently "selected." It works as a *toggle*,
+meaning that if you type ``key 1`` again, the ``*`` will disappear and
+the key will not be selected any more.
+
+Now, let's move that key onto the smartcard::
+
+    gpg> keytocard
+    Please select where to store the key:
+       (2) Encryption key
+    Your selection? 2
+
+Since it's our **[E]** key, it makes sense to put it into the Encryption
+slot.  When you submit your selection, you will be prompted first for
+your PGP key passphrase, and then for the admin PIN. If the command
+returns without an error, your key has been moved.
+
+**Important**: Now type ``key 1`` again to unselect the first key, and
+``key 2`` to select the **[S]** key::
+
+    gpg> key 1
+    gpg> key 2
+    gpg> keytocard
+    Please select where to store the key:
+       (1) Signature key
+       (3) Authentication key
+    Your selection? 1
+
+You can use the **[S]** key both for Signature and Authentication, but
+we want to make sure it's in the Signature slot, so choose (1). Once
+again, if your command returns without an error, then the operation was
+successful::
+
+    gpg> q
+    Save changes? (y/N) y
+
+Saving the changes will delete the keys you moved to the card from your
+home directory (but it's okay, because we have them in our backups
+should we need to do this again for a replacement smartcard).
+
+Verifying that the keys were moved
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you perform ``--list-secret-keys`` now, you will see a subtle
+difference in the output::
+
+    $ gpg --list-secret-keys
+    sec#  rsa2048 2018-01-24 [SC] [expires: 2020-01-24]
+          000000000000000000000000AAAABBBBCCCCDDDD
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    ssb>  rsa2048 2018-01-24 [E] [expires: 2020-01-24]
+    ssb>  ed25519 2018-01-24 [S]
+
+The ``>`` in the ``ssb>`` output indicates that the subkey is only
+available on the smartcard. If you go back into your secret keys
+directory and look at the contents there, you will notice that the
+``.key`` files there have been replaced with stubs::
+
+    $ cd ~/.gnupg/private-keys-v1.d
+    $ strings *.key | grep 'private-key'
+
+The output should contain ``shadowed-private-key`` to indicate that
+these files are only stubs and the actual content is on the smartcard.
+
+Verifying that the smartcard is functioning
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To verify that the smartcard is working as intended, you can create a
+signature::
+
+    $ echo "Hello world" | gpg --clearsign > /tmp/test.asc
+    $ gpg --verify /tmp/test.asc
+
+This should ask for your smartcard PIN on your first command, and then
+show "Good signature" after you run ``gpg --verify``.
+
+Congratulations, you have successfully made it extremely difficult to
+steal your digital developer identity!
+
+Other common GnuPG operations
+-----------------------------
+
+Here is a quick reference for some common operations you'll need to do
+with your PGP key.
+
+Mounting your master key offline storage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You will need your master key for any of the operations below, so you
+will first need to mount your backup offline storage and tell GnuPG to
+use it::
+
+    $ export GNUPGHOME=/media/disk/foo/gnupg-backup
+    $ gpg --list-secret-keys
+
+You want to make sure that you see ``sec`` and not ``sec#`` in the
+output (the ``#`` means the key is not available and you're still using
+your regular home directory location).
+
+Extending key expiration date
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The master key has the default expiration date of 2 years from the date
+of creation. This is done both for security reasons and to make obsolete
+keys eventually disappear from keyservers.
+
+To extend the expiration on your key by a year from current date, just
+run::
+
+    $ gpg --quick-set-expire [fpr] 1y
+
+You can also use a specific date if that is easier to remember (e.g.
+your birthday, January 1st, or Canada Day)::
+
+    $ gpg --quick-set-expire [fpr] 2020-07-01
+
+Remember to send the updated key back to keyservers::
+
+    $ gpg --send-key [fpr]
+
+Updating your work directory after any changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After you make any changes to your key using the offline storage, you will
+want to import these changes back into your regular working directory::
+
+    $ gpg --export | gpg --homedir ~/.gnupg --import
+    $ unset GNUPGHOME
+
+
+Using PGP with Git
+==================
+
+One of the core features of Git is its decentralized nature -- once a
+repository is cloned to your system, you have full history of the
+project, including all of its tags, commits and branches. However, with
+hundreds of cloned repositories floating around, how does anyone verify
+that their copy of linux.git has not been tampered with by a malicious
+third party?
+
+Or what happens if a backdoor is discovered in the code and the "Author"
+line in the commit says it was done by you, while you're pretty sure you
+had `nothing to do with it`_?
+
+To address both of these issues, Git introduced PGP integration. Signed
+tags prove the repository integrity by assuring that its contents are
+exactly the same as on the workstation of the developer who created the
+tag, while signed commits make it nearly impossible for someone to
+impersonate you without having access to your PGP keys.
+
+.. _`nothing to do with it`: https://github.com/jayphelps/git-blame-someone-else
+
+Configure git to use your PGP key
+---------------------------------
+
+If you only have one secret key in your keyring, then you don't really
+need to do anything extra, as it becomes your default key.  However, if
+you happen to have multiple secret keys, you can tell git which key
+should be used (``[fpr]`` is the fingerprint of your key)::
+
+    $ git config --global user.signingKey [fpr]
+
+**IMPORTANT**: If you have a distinct ``gpg2`` command, then you should
+tell git to always use it instead of the legacy ``gpg`` from version 1::
+
+    $ git config --global gpg.program gpg2
+
+How to work with signed tags
+----------------------------
+
+To create a signed tag, simply pass the ``-s`` switch to the tag
+command::
+
+    $ git tag -s [tagname]
+
+Our recommendation is to always sign git tags, as this allows other
+developers to ensure that the git repository they are pulling from has
+not been maliciously altered.
+
+How to verify signed tags
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To verify a signed tag, simply use the ``verify-tag`` command::
+
+    $ git verify-tag [tagname]
+
+If you are pulling a tag from another fork of the project repository,
+git should automatically verify the signature at the tip you're pulling
+and show you the results during the merge operation::
+
+    $ git pull [url] tags/sometag
+
+The merge message will contain something like this::
+
+    Merge tag 'sometag' of [url]
+
+    [Tag message]
+
+    # gpg: Signature made [...]
+    # gpg: Good signature from [...]
+
+If you are verifying someone else's git tag, then you will need to
+import their PGP key. Please refer to the
+":ref:`verify_identities`" section below.
+
+Configure git to always sign annotated tags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Chances are, if you're creating an annotated tag, you'll want to sign
+it. To force git to always sign annotated tags, you can set a global
+configuration option::
+
+    $ git config --global tag.forceSignAnnotated true
+
+How to work with signed commits
+-------------------------------
+
+It is easy to create signed commits, but it is much more difficult to
+use them in Linux kernel development, since it relies on patches sent to
+the mailing list, and this workflow does not preserve PGP commit
+signatures. Furthermore, when rebasing your repository to match
+upstream, even your own PGP commit signatures will end up discarded. For
+this reason, most kernel developers don't bother signing their commits
+and will ignore signed commits in any external repositories that they
+rely upon in their work.
+
+However, if you have your working git tree publicly available at some
+git hosting service (kernel.org, infradead.org, ozlabs.org, or others),
+then the recommendation is that you sign all your git commits even if
+upstream developers do not directly benefit from this practice.
+
+We recommend this for the following reasons:
+
+1. Should there ever be a need to perform code forensics or track code
+   provenance, even externally maintained trees carrying PGP commit
+   signatures will be valuable for such purposes.
+2. If you ever need to re-clone your local repository (for example,
+   after a disk failure), this lets you easily verify the repository
+   integrity before resuming your work.
+3. If someone needs to cherry-pick your commits, this allows them to
+   quickly verify their integrity before applying them.
+
+Creating signed commits
+~~~~~~~~~~~~~~~~~~~~~~~
+
+To create a signed commit, you just need to pass the ``-S`` flag to the
+``git commit`` command (it's capital ``-S`` due to collision with
+another flag)::
+
+    $ git commit -S
+
+Configure git to always sign commits
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can tell git to always sign commits::
+
+    git config --global commit.gpgSign true
+
+.. note::
+
+    Make sure you configure ``gpg-agent`` before you turn this on.
+
+.. _verify_identities:
+
+How to verify kernel developer identities
+=========================================
+
+Signing tags and commits is easy, but how does one go about verifying
+that the key used to sign something belongs to the actual kernel
+developer and not to a malicious imposter?
+
+Configure auto-key-retrieval using WKD and DANE
+-----------------------------------------------
+
+If you are not already someone with an extensive collection of other
+developers' public keys, then you can jumpstart your keyring by relying
+on key auto-discovery and auto-retrieval. GnuPG can piggyback on other
+delegated trust technologies, namely DNSSEC and TLS, to get you going if
+the prospect of starting your own Web of Trust from scratch is too
+daunting.
+
+Add the following to your ``~/.gnupg/gpg.conf``::
+
+    auto-key-locate wkd,dane,local
+    auto-key-retrieve
+
+DNS-Based Authentication of Named Entities ("DANE") is a method for
+publishing public keys in DNS and securing them using DNSSEC signed
+zones. Web Key Directory ("WKD") is the alternative method that uses
+https lookups for the same purpose. When using either DANE or WKD for
+looking up public keys, GnuPG will validate DNSSEC or TLS certificates,
+respectively, before adding auto-retrieved public keys to your local
+keyring.
+
+Kernel.org publishes the WKD for all developers who have kernel.org
+accounts. Once you have the above changes in your ``gpg.conf``, you can
+auto-retrieve the keys for Linus Torvalds and Greg Kroah-Hartman (if you
+don't already have them)::
+
+    $ gpg --locate-keys torvalds@kernel.org gregkh@kernel.org
+
+If you have a kernel.org account, then you should `add the kernel.org
+UID to your key`_ to make WKD more useful to other kernel developers.
+
+.. _`add the kernel.org UID to your key`: https://korg.wiki.kernel.org/userdoc/mail#adding_a_kernelorg_uid_to_your_pgp_key
+
+Web of Trust (WOT) vs. Trust on First Use (TOFU)
+------------------------------------------------
+
+PGP incorporates a trust delegation mechanism known as the "Web of
+Trust." At its core, this is an attempt to replace the need for
+centralized Certification Authorities of the HTTPS/TLS world. Instead of
+various software makers dictating who should be your trusted certifying
+entity, PGP leaves this responsibility to each user.
+
+Unfortunately, very few people understand how the Web of Trust works.
+While it remains an important aspect of the OpenPGP specification,
+recent versions of GnuPG (2.2 and above) have implemented an alternative
+mechanism called "Trust on First Use" (TOFU). You can think of TOFU as
+"the SSH-like approach to trust." With SSH, the first time you connect
+to a remote system, its key fingerprint is recorded and remembered. If
+the key changes in the future, the SSH client will alert you and refuse
+to connect, forcing you to make a decision on whether you choose to
+trust the changed key or not. Similarly, the first time you import
+someone's PGP key, it is assumed to be valid. If at any point in the
+future GnuPG comes across another key with the same identity, both the
+previously imported key and the new key will be marked as invalid and
+you will need to manually figure out which one to keep.
+
+We recommend that you use the combined TOFU+PGP trust model (which is
+the new default in GnuPG v2). To set it, add (or modify) the
+``trust-model`` setting in ``~/.gnupg/gpg.conf``::
+
+    trust-model tofu+pgp
+
+How to use keyservers (more) safely
+-----------------------------------
+
+If you get a "No public key" error when trying to validate someone's
+tag, then you should attempt to lookup that key using a keyserver. It is
+important to keep in mind that there is absolutely no guarantee that the
+key you retrieve from PGP keyservers belongs to the actual person --
+that much is by design. You are supposed to use the Web of Trust to
+establish key validity.
+
+How to properly maintain the Web of Trust is beyond the scope of this
+document, simply because doing it properly requires both effort and
+dedication that tends to be beyond the caring threshold of most human
+beings. Here are some shortcuts that will help you reduce the risk of
+importing a malicious key.
+
+First, let's say you've tried to run ``git verify-tag`` but it returned
+an error saying the key is not found::
+
+    $ git verify-tag sunxi-fixes-for-4.15-2
+    gpg: Signature made Sun 07 Jan 2018 10:51:55 PM EST
+    gpg:                using RSA key DA73759BF8619E484E5A3B47389A54219C0F2430
+    gpg:                issuer "wens@...org"
+    gpg: Can't check signature: No public key
+
+Let's query the keyserver for more info about that key fingerprint (the
+fingerprint probably belongs to a subkey, so we can't use it directly
+without finding out the ID of the master key it is associated with)::
+
+    $ gpg --search DA73759BF8619E484E5A3B47389A54219C0F2430
+    gpg: data source: hkp://keys.gnupg.net
+    (1) Chen-Yu Tsai <wens@...org>
+          4096 bit RSA key C94035C21B4F2AEB, created: 2017-03-14, expires: 2019-03-15
+    Keys 1-1 of 1 for "DA73759BF8619E484E5A3B47389A54219C0F2430".  Enter number(s), N)ext, or Q)uit > q
+
+Locate the ID of the master key in the output, in our example
+``C94035C21B4F2AEB``. Now display the key of Linus Torvalds that you
+have on your keyring::
+
+    $ gpg --list-key torvalds@kernel.org
+    pub   rsa2048 2011-09-20 [SC]
+          ABAF11C65A2970B130ABE3C479BE3E4300411886
+    uid           [ unknown] Linus Torvalds <torvalds@kernel.org>
+    sub   rsa2048 2011-09-20 [E]
+
+Next, open the `PGP pathfinder`_. In the "From" field, paste the key
+fingerprint of Linus Torvalds from the output above. In the "To" field,
+paste they key-id you found via ``gpg --search`` of the unknown key, and
+check the results:
+
+- `Finding paths to Linus`_
+
+If you get a few decent trust paths, then it's a pretty good indication
+that it is a valid key. You can add it to your keyring from the
+keyserver now::
+
+    $ gpg --recv-key C94035C21B4F2AEB
+
+This process is not perfect, and you are obviously trusting the
+administrators of the PGP Pathfinder service to not be malicious (in
+fact, this goes against :ref:`devs_not_infra`). However, if you
+do not carefully maintain your own web of trust, then it is a marked
+improvement over blindly trusting keyservers.
+
+.. _`PGP pathfinder`: https://pgp.cs.uu.nl/
+.. _`Finding paths to Linus`: https://pgp.cs.uu.nl/paths/79BE3E4300411886/to/C94035C21B4F2AEB.html
diff --git a/Documentation/process/submit-checklist.rst b/Documentation/process/submit-checklist.rst
index a0d9d34bfb6d..367353c54949 100644
--- a/Documentation/process/submit-checklist.rst
+++ b/Documentation/process/submit-checklist.rst
@@ -37,7 +37,9 @@ and elsewhere regarding submitting Linux kernel patches.
    You should be able to justify all violations that remain in
    your patch.
 
-6) Any new or modified ``CONFIG`` options don't muck up the config menu.
+6) Any new or modified ``CONFIG`` options do not muck up the config menu and
+   default to off unless they meet the exception criteria documented in
+   ``Documentation/kbuild/kconfig-language.txt`` Menu attributes: default value.
 
 7) All new ``Kconfig`` options have help text.
 
diff --git a/Documentation/rtc.txt b/Documentation/rtc.txt
index c0c977445fb9..a129acf38537 100644
--- a/Documentation/rtc.txt
+++ b/Documentation/rtc.txt
@@ -136,82 +136,5 @@ a high functionality RTC is integrated into the SOC.  That system might read
 the system clock from the discrete RTC, but use the integrated one for all
 other tasks, because of its greater functionality.
 
-SYSFS interface
----------------
-
-The sysfs interface under /sys/class/rtc/rtcN provides access to various
-rtc attributes without requiring the use of ioctls. All dates and times
-are in the RTC's timezone, rather than in system time.
-
-================ ==============================================================
-date  	   	 RTC-provided date
-hctosys   	 1 if the RTC provided the system time at boot via the
-		 CONFIG_RTC_HCTOSYS kernel option, 0 otherwise
-max_user_freq	 The maximum interrupt rate an unprivileged user may request
-		 from this RTC.
-name		 The name of the RTC corresponding to this sysfs directory
-since_epoch	 The number of seconds since the epoch according to the RTC
-time		 RTC-provided time
-wakealarm	 The time at which the clock will generate a system wakeup
-		 event. This is a one shot wakeup event, so must be reset
-		 after wake if a daily wakeup is required. Format is seconds
-		 since the epoch by default, or if there's a leading +, seconds
-		 in the future, or if there is a leading +=, seconds ahead of
-		 the current alarm.
-offset		 The amount which the rtc clock has been adjusted in firmware.
-		 Visible only if the driver supports clock offset adjustment.
-		 The unit is parts per billion, i.e. The number of clock ticks
-		 which are added to or removed from the rtc's base clock per
-		 billion ticks. A positive value makes a day pass more slowly,
-		 longer, and a negative value makes a day pass more quickly.
-*/nvmem		 The non volatile storage exported as a raw file, as described
-		 in Documentation/nvmem/nvmem.txt
-================ ==============================================================
-
-IOCTL interface
----------------
-
-The ioctl() calls supported by /dev/rtc are also supported by the RTC class
-framework.  However, because the chips and systems are not standardized,
-some PC/AT functionality might not be provided.  And in the same way, some
-newer features -- including those enabled by ACPI -- are exposed by the
-RTC class framework, but can't be supported by the older driver.
-
-    *	RTC_RD_TIME, RTC_SET_TIME ... every RTC supports at least reading
-	time, returning the result as a Gregorian calendar date and 24 hour
-	wall clock time.  To be most useful, this time may also be updated.
-
-    *	RTC_AIE_ON, RTC_AIE_OFF, RTC_ALM_SET, RTC_ALM_READ ... when the RTC
-	is connected to an IRQ line, it can often issue an alarm IRQ up to
-	24 hours in the future.  (Use RTC_WKALM_* by preference.)
-
-    *	RTC_WKALM_SET, RTC_WKALM_RD ... RTCs that can issue alarms beyond
-	the next 24 hours use a slightly more powerful API, which supports
-	setting the longer alarm time and enabling its IRQ using a single
-	request (using the same model as EFI firmware).
-
-    *	RTC_UIE_ON, RTC_UIE_OFF ... if the RTC offers IRQs, the RTC framework
-	will emulate this mechanism.
-
-    *	RTC_PIE_ON, RTC_PIE_OFF, RTC_IRQP_SET, RTC_IRQP_READ ... these icotls
-	are emulated via a kernel hrtimer.
-
-In many cases, the RTC alarm can be a system wake event, used to force
-Linux out of a low power sleep state (or hibernation) back to a fully
-operational state.  For example, a system could enter a deep power saving
-state until it's time to execute some scheduled tasks.
-
-Note that many of these ioctls are handled by the common rtc-dev interface.
-Some common examples:
-
-    *	RTC_RD_TIME, RTC_SET_TIME: the read_time/set_time functions will be
-	called with appropriate values.
-
-    *	RTC_ALM_SET, RTC_ALM_READ, RTC_WKALM_SET, RTC_WKALM_RD: gets or sets
-	the alarm rtc_timer. May call the set_alarm driver function.
-
-    *	RTC_IRQP_SET, RTC_IRQP_READ: These are emulated by the generic code.
-
-    *	RTC_PIE_ON, RTC_PIE_OFF: These are also emulated by the generic code.
-
-If all else fails, check out the tools/testing/selftests/timers/rtctest.c test!
+Check out tools/testing/selftests/timers/rtctest.c for an example usage of the
+ioctl interface.
diff --git a/Documentation/security/credentials.rst b/Documentation/security/credentials.rst
index 66a2e24939d8..5bb7125faeee 100644
--- a/Documentation/security/credentials.rst
+++ b/Documentation/security/credentials.rst
@@ -451,6 +451,13 @@ checks and hooks done.  Both the current and the proposed sets of credentials
 are available for this purpose as current_cred() will return the current set
 still at this point.
 
+When replacing the group list, the new list must be sorted before it
+is added to the credential, as a binary search is used to test for
+membership.  In practice, this means :c:func:`groups_sort` should be
+called before :c:func:`set_groups` or :c:func:`set_current_groups`.
+:c:func:`groups_sort)` must not be called on a ``struct group_list`` which
+is shared as it may permute elements as part of the sorting process
+even if the array is already sorted.
 
 When the credential set is ready, it should be committed to the current process
 by calling::
diff --git a/Documentation/security/self-protection.rst b/Documentation/security/self-protection.rst
index 60c8bd8b77bf..0f53826c78b9 100644
--- a/Documentation/security/self-protection.rst
+++ b/Documentation/security/self-protection.rst
@@ -270,6 +270,21 @@ attacks, it is important to defend against exposure of both kernel memory
 addresses and kernel memory contents (since they may contain kernel
 addresses or other sensitive things like canary values).
 
+Kernel addresses
+----------------
+
+Printing kernel addresses to userspace leaks sensitive information about
+the kernel memory layout. Care should be exercised when using any printk
+specifier that prints the raw address, currently %px, %p[ad], (and %p[sSb]
+in certain circumstances [*]).  Any file written to using one of these
+specifiers should be readable only by privileged processes.
+
+Kernels 4.14 and older printed the raw address using %p. As of 4.15-rc1
+addresses printed with the specifier %p are hashed before printing.
+
+[*] If KALLSYMS is enabled and symbol lookup fails, the raw address is
+printed. If KALLSYMS is not enabled the raw address is printed.
+
 Unique identifiers
 ------------------
 
diff --git a/Documentation/sparc/oradax/dax-hv-api.txt b/Documentation/sparc/oradax/dax-hv-api.txt
new file mode 100644
index 000000000000..73e8d506cf64
--- /dev/null
+++ b/Documentation/sparc/oradax/dax-hv-api.txt
@@ -0,0 +1,1433 @@
+Excerpt from UltraSPARC Virtual Machine Specification
+Compiled from version 3.0.20+15
+Publication date 2017-09-25 08:21
+Copyright © 2008, 2015 Oracle and/or its affiliates. All rights reserved.
+Extracted via "pdftotext -f 547 -l 572 -layout sun4v_20170925.pdf"
+Authors:
+	 Charles Kunzman
+	 Sam Glidden
+	 Mark Cianchetti
+
+
+Chapter 36. Coprocessor services
+        The following APIs provide access via the Hypervisor to hardware assisted data processing functionality.
+        These APIs may only be provided by certain platforms, and may not be available to all virtual machines
+        even on supported platforms. Restrictions on the use of these APIs may be imposed in order to support
+        live-migration and other system management activities.
+
+36.1. Data Analytics Accelerator
+        The Data Analytics Accelerator (DAX) functionality is a collection of hardware coprocessors that provide
+        high speed processoring of database-centric operations. The coprocessors may support one or more of
+        the following data query operations: search, extraction, compression, decompression, and translation. The
+        functionality offered may vary by virtual machine implementation.
+
+        The DAX is a virtual device to sun4v guests, with supported data operations indicated by the virtual device
+        compatibilty property. Functionality is accessed through the submission of Command Control Blocks
+        (CCBs) via the ccb_submit API function. The operations are processed asynchronously, with the status
+        of the submitted operations reported through a Completion Area linked to each CCB. Each CCB has a
+        separate Completion Area and, unless execution order is specifically restricted through the use of serial-
+        conditional flags, the execution order of submitted CCBs is arbitrary. Likewise, the time to completion
+        for a given CCB is never guaranteed.
+
+        Guest software may implement a software timeout on CCB operations, and if the timeout is exceeded, the
+        operation may be cancelled or killed via the ccb_kill API function. It is recommended for guest software
+        to implement a software timeout to account for certain RAS errors which may result in lost CCBs. It is
+        recommended such implementation use the ccb_info API function to check the status of a CCB prior to
+        killing it in order to determine if the CCB is still in queue, or may have been lost due to a RAS error.
+
+        There is no fixed limit on the number of outstanding CCBs guest software may have queued in the virtual
+        machine, however, internal resource limitations within the virtual machine can cause CCB submissions
+        to be temporarily rejected with EWOULDBLOCK. In such cases, guests should continue to attempt
+        submissions until they succeed; waiting for an outstanding CCB to complete is not necessary, and would
+        not be a guarantee that a future submission would succeed.
+
+        The availablility of DAX coprocessor command service is indicated by the presence of the DAX virtual
+        device node in the guest MD (Section 8.24.17, “Database Analytics Accelerators (DAX) virtual-device
+        node”).
+
+36.1.1. DAX Compatibility Property
+        The query functionality may vary based on the compatibility property of the virtual device:
+
+36.1.1.1. "ORCL,sun4v-dax" Device Compatibility
+        Available CCB commands:
+
+        • No-op/Sync
+
+        • Extract
+
+        • Scan Value
+
+        • Inverted Scan Value
+
+        • Scan Range
+
+
+                                                     509
+                                             Coprocessor services
+
+
+        • Inverted Scan Range
+
+        • Translate
+
+        • Inverted Translate
+
+        • Select
+
+        See Section 36.2.1, “Query CCB Command Formats” for the corresponding CCB input and output formats.
+
+        Only version 0 CCBs are available.
+
+36.1.1.2. "ORCL,sun4v-dax-fc" Device Compatibility
+        "ORCL,sun4v-dax-fc" is compatible with the "ORCL,sun4v-dax" interface, and includes additional CCB
+        bit fields and controls.
+
+36.1.1.3. "ORCL,sun4v-dax2" Device Compatibility
+        Available CCB commands:
+
+        • No-op/Sync
+
+        • Extract
+
+        • Scan Value
+
+        • Inverted Scan Value
+
+        • Scan Range
+
+        • Inverted Scan Range
+
+        • Translate
+
+        • Inverted Translate
+
+        • Select
+
+        See Section 36.2.1, “Query CCB Command Formats” for the corresponding CCB input and output formats.
+
+        Version 0 and 1 CCBs are available. Only version 0 CCBs may use Huffman encoded data, whereas only
+        version 1 CCBs may use OZIP.
+
+36.1.2. DAX Virtual Device Interrupts
+        The DAX virtual device has multiple interrupts associated with it which may be used by the guest if
+        desired. The number of device interrupts available to the guest is indicated in the virtual device node of the
+        guest MD (Section 8.24.17, “Database Analytics Accelerators (DAX) virtual-device node”). If the device
+        node indicates N interrupts available, the guest may use any value from 0 to N - 1 (inclusive) in a CCB
+        interrupt number field. Using values outside this range will result in the CCB being rejected for an invalid
+        field value.
+
+        The interrupts may be bound and managed using the standard sun4v device interrupts API (Chapter 16,
+        Device interrupt services). Sysino interrupts are not available for DAX devices.
+
+36.2. Coprocessor Control Block (CCB)
+        CCBs are either 64 or 128 bytes long, depending on the operation type. The exact contents of the CCB
+        are command specific, but all CCBs contain at least one memory buffer address. All memory locations
+
+
+                                                      510
+                                    Coprocessor services
+
+
+referenced by a CCB must be pinned in memory until the CCB either completes execution or is killed
+via the ccb_kill API call. Changes in virtual address mappings occurring after CCB submission are not
+guaranteed to be visible, and as such all virtual address updates need to be synchronized with CCB
+execution.
+
+All CCBs begin with a common 32-bit header.
+
+Table 36.1. CCB Header Format
+Bits          Field Description
+[31:28]       CCB version. For API version 2.0: set to 1 if CCB uses OZIP encoding; set to 0 if the CCB
+              uses Huffman encoding; otherwise either 0 or 1. For API version 1.0: always set to 0.
+[27]          When API version 2.0 is negotiated, this is the Pipeline Flag [512]. It is reserved in
+              API version 1.0
+[26]          Long CCB flag [512]
+[25]          Conditional synchronization flag [512]
+[24]          Serial synchronization flag
+[23:16]       CCB operation code:
+               0x00        No Operation (No-op) or Sync
+               0x01        Extract
+               0x02        Scan Value
+               0x12        Inverted Scan Value
+               0x03        Scan Range
+               0x13        Inverted Scan Range
+               0x04        Translate
+               0x14        Inverted Translate
+               0x05        Select
+[15:13]       Reserved
+[12:11]       Table address type
+               0b'00       No address
+               0b'01       Alternate context virtual address
+               0b'10       Real address
+               0b'11       Primary context virtual address
+[10:8]        Output/Destination address type
+               0b'000      No address
+               0b'001      Alternate context virtual address
+               0b'010      Real address
+               0b'011      Primary context virtual address
+               0b'100      Reserved
+               0b'101      Reserved
+               0b'110      Reserved
+               0b'111      Reserved
+[7:5]         Secondary source address type
+
+
+                                            511
+                                    Coprocessor services
+
+
+Bits           Field Description
+                0b'000       No address
+                0b'001       Alternate context virtual address
+                0b'010       Real address
+                0b'011       Primary context virtual address
+                0b'100       Reserved
+                0b'101       Reserved
+                0b'110       Reserved
+                0b'111       Reserved
+[4:2]          Primary source address type
+                0b'000       No address
+                0b'001       Alternate context virtual address
+                0b'010       Real address
+                0b'011       Primary context virtual address
+                0b'100       Reserved
+                0b'101       Reserved
+                0b'110       Reserved
+                0b'111       Reserved
+[1:0]          Completion area address type
+                0b'00        No address
+                0b'01        Alternate context virtual address
+                0b'10        Real address
+                0b'11        Primary context virtual address
+
+The Long CCB flag indicates whether the submitted CCB is 64 or 128 bytes long; value is 0 for 64 bytes
+and 1 for 128 bytes.
+
+The Serial and Conditional flags allow simple relative ordering between CCBs. Any CCB with the Serial
+flag set will execute sequentially relative to any previous CCB that is also marked as Serial in the same
+CCB submission. CCBs without the Serial flag set execute independently, even if they are between CCBs
+with the Serial flag set. CCBs marked solely with the Serial flag will execute upon the completion of the
+previous Serial CCB, regardless of the completion status of that CCB. The Conditional flag allows CCBs
+to conditionally execute based on the successful execution of the closest CCB marked with the Serial flag.
+A CCB may only be conditional on exactly one CCB, however, a CCB may be marked both Conditional
+and Serial to allow execution chaining. The flags do NOT allow fan-out chaining, where multiple CCBs
+execute in parallel based on the completion of another CCB.
+
+The Pipeline flag is an optimization that directs the output of one CCB (the "source" CCB) directly to
+the input of the next CCB (the "target" CCB). The target CCB thus does not need to read the input from
+memory. The Pipeline flag is advisory and may be dropped.
+
+Both the Pipeline and Serial bits must be set in the source CCB. The Conditional bit must be set in the
+target CCB. Exactly one CCB must be made conditional on the source CCB; either 0 or 2 target CCBs
+is invalid. However, Pipelines can be extended beyond two CCBs: the sequence would start with a CCB
+with both the Pipeline and Serial bits set, proceed through CCBs with the Pipeline, Serial, and Conditional
+bits set, and terminate at a CCB that has the Conditional bit set, but not the Pipeline bit.
+
+
+                                             512
+                                               Coprocessor services
+
+
+          The input of the target CCB must start within 64 bytes of the output of the source CCB or the pipeline flag
+          will be ignored. All CCBs in a pipeline must be submitted in the same call to ccb_submit.
+
+          The various address type fields indicate how the various address values used in the CCB should be
+          interpreted by the virtual machine. Not all of the types specified are used by every CCB format. Types
+          which are not applicable to the given CCB command should be indicated as type 0 (No address). Virtual
+          addresses used in the CCB must have translation entries present in either the TLB or a configured TSB
+          for the submitting virtual processor. Virtual addresses which cannot be translated by the virtual machine
+          will result in the CCB submission being rejected, with the causal virtual address indicated. The CCB
+          may be resubmitted after inserting the translation, or the address may be translated by guest software and
+          resubmitted using the real address translation.
+
+36.2.1. Query CCB Command Formats
+36.2.1.1. Supported Data Formats, Elements Sizes and Offsets
+          Data for query commands may be encoded in multiple possible formats. The data query commands use a
+          common set of values to indicate the encoding formats of the data being processed. Some encoding formats
+          require multiple data streams for processing, requiring the specification of both primary data formats (the
+          encoded data) and secondary data streams (meta-data for the encoded data).
+
+36.2.1.1.1. Primary Input Format
+
+          The primary input format code is a 4-bit field when it is used. There are 10 primary input formats available.
+          The packed formats are not endian neutral. Code values not listed below are reserved.
+
+          Code        Format                              Description
+          0x0         Fixed width byte packed             Up to 16 bytes
+          0x1         Fixed width bit packed              Up to 15 bits (CCB version 0) or 23 bits (CCB version
+                                                          1); bits are read most significant bit to least significant bit
+                                                          within a byte
+          0x2         Variable width byte packed          Data stream of lengths must be provided as a secondary
+                                                          input
+          0x4         Fixed width byte packed with run Up to 16 bytes; data stream of run lengths must be
+                      length encoding                  provided as a secondary input
+          0x5         Fixed width bit packed with run Up to 15 bits (CCB version 0) or 23 bits (CCB version
+                      length encoding                 1); bits are read most significant bit to least significant bit
+                                                      within a byte; data stream of run lengths must be provided
+                                                      as a secondary input
+          0x8         Fixed width byte packed with Up to 16 bytes before the encoding; compressed stream
+                      Huffman (CCB version 0) or bits are read most significant bit to least significant bit
+                      OZIP (CCB version 1) encoding within a byte; pointer to the encoding table must be
+                                                    provided
+          0x9         Fixed width bit packed with Up to 15 bits (CCB version 0) or 23 bits (CCB version
+                      Huffman (CCB version 0) or 1); compressed stream bits are read most significant bit to
+                      OZIP (CCB version 1) encoding least significant bit within a byte; pointer to the encoding
+                                                    table must be provided
+          0xA         Variable width byte packed with Up to 16 bytes before the encoding; compressed stream
+                      Huffman (CCB version 0) or bits are read most significant bit to least significant bit
+                      OZIP (CCB version 1) encoding within a byte; data stream of lengths must be provided as
+                                                      a secondary input; pointer to the encoding table must be
+                                                      provided
+
+
+                                                        513
+                                               Coprocessor services
+
+
+          Code        Format                              Description
+          0xC         Fixed width byte packed with        Up to 16 bytes before the encoding; compressed stream
+                      run length encoding, followed by    bits are read most significant bit to least significant bit
+                      Huffman (CCB version 0) or          within a byte; data stream of run lengths must be provided
+                      OZIP (CCB version 1) encoding       as a secondary input; pointer to the encoding table must
+                                                          be provided
+          0xD         Fixed width bit packed with         Up to 15 bits (CCB version 0) or 23 bits(CCB version 1)
+                      run length encoding, followed by    before the encoding; compressed stream bits are read most
+                      Huffman (CCB version 0) or          significant bit to least significant bit within a byte; data
+                      OZIP (CCB version 1) encoding       stream of run lengths must be provided as a secondary
+                                                          input; pointer to the encoding table must be provided
+
+          If OZIP encoding is used, there must be no reserved bytes in the table.
+
+36.2.1.1.2. Primary Input Element Size
+
+          For primary input data streams with fixed size elements, the element size must be indicated in the CCB
+          command. The size is encoded as the number of bits or bytes, minus one. The valid value range for this
+          field depends on the input format selected, as listed in the table above.
+
+36.2.1.1.3. Secondary Input Format
+
+          For primary input data streams which require a secondary input stream, the secondary input stream is
+          always encoded in a fixed width, bit-packed format. The bits are read from most significant bit to least
+          significant bit within a byte. There are two encoding options for the secondary input stream data elements,
+          depending on whether the value of 0 is needed:
+
+          Secondary           Input Description
+          Format Code
+          0                          Element is stored as value minus 1 (0 evalutes to 1, 1 evalutes
+                                     to 2, etc)
+          1                          Element is stored as value
+
+36.2.1.1.4. Secondary Input Element Size
+
+          Secondary input element size is encoded as a two bit field:
+
+          Secondary Input Size Description
+          Code
+          0x0                        1 bit
+          0x1                        2 bits
+          0x2                        4 bits
+          0x3                        8 bits
+
+36.2.1.1.5. Input Element Offsets
+
+          Bit-wise input data streams may have any alignment within the base addressed byte. The offset, specified
+          from most significant bit to least significant bit, is provided as a fixed 3 bit field for each input type. A
+          value of 0 indicates that the first input element begins at the most significant bit in the first byte, and a
+          value of 7 indicates it begins with the least significant bit.
+
+          This field should be zero for any byte-wise primary input data streams.
+
+
+                                                        514
+                                              Coprocessor services
+
+
+36.2.1.1.6. Output Format
+
+          Query commands support multiple sizes and encodings for output data streams. There are four possible
+          output encodings, and up to four supported element sizes per encoding. Not all output encodings are
+          supported for every command. The format is indicated by a 4-bit field in the CCB:
+
+           Output Format Code        Description
+           0x0                       Byte aligned, 1 byte elements
+           0x1                       Byte aligned, 2 byte elements
+           0x2                       Byte aligned, 4 byte elements
+           0x3                       Byte aligned, 8 byte elements
+           0x4                       16 byte aligned, 16 byte elements
+           0x5                       Reserved
+           0x6                       Reserved
+           0x7                       Reserved
+           0x8                       Packed vector of single bit elements
+           0x9                       Reserved
+           0xA                       Reserved
+           0xB                       Reserved
+           0xC                       Reserved
+           0xD                       2 byte elements where each element is the index value of a bit,
+                                     from an bit vector, which was 1.
+           0xE                       4 byte elements where each element is the index value of a bit,
+                                     from an bit vector, which was 1.
+           0xF                       Reserved
+
+36.2.1.1.7. Application Data Integrity (ADI)
+
+          On platforms which support ADI, the ADI version number may be specified for each separate memory
+          access type used in the CCB command. ADI checking only occurs when reading data. When writing data,
+          the specified ADI version number overwrites any existing ADI value in memory.
+
+          An ADI version value of 0 or 0xF indicates the ADI checking is disabled for that data access, even if it is
+          enabled in memory. By setting the appropriate flag in CCB_SUBMIT (Section 36.3.1, “ccb_submit”) it is
+          also an option to disable ADI checking for all inputs accessed via virtual address for all CCBs submitted
+          during that hypercall invocation.
+
+          The ADI value is only guaranteed to be checked on the first 64 bytes of each data access. Mismatches on
+          subsequent data chunks may not be detected, so guest software should be careful to use page size checking
+          to protect against buffer overruns.
+
+36.2.1.1.8. Page size checking
+
+          All data accesses used in CCB commands must be bounded within a single memory page. When addresses
+          are provided using a virtual address, the page size for checking is extracted from the TTE for that virtual
+          address. When using real addresses, the guest must supply the page size in the same field as the address
+          value. The page size must be one of the sizes supported by the underlying virtual machine. Using a value
+          that is not supported may result in the CCB submission being rejected or the generation of a CCB parsing
+          error in the completion area.
+
+
+                                                       515
+                                               Coprocessor services
+
+
+36.2.1.2. Extract command
+
+        Converts an input vector in one format to an output vector in another format. All input format types are
+        supported.
+
+        The only supported output format is a padded, byte-aligned output stream, using output codes 0x0 - 0x4.
+        When the specified output element size is larger than the extracted input element size, zeros are padded to
+        the extracted input element. First, if the decompressed input size is not a whole number of bytes, 0 bits are
+        padded to the most significant bit side till the next byte boundary. Next, if the output element size is larger
+        than the byte padded input element, bytes of value 0 are added based on the Padding Direction bit in the
+        CCB. If the output element size is smaller than the byte-padded input element size, the input element is
+        truncated by dropped from the least significant byte side until the selected output size is reached.
+
+        The return value of the CCB completion area is invalid. The “number of elements processed” field in the
+        CCB completion area will be valid.
+
+        The extract CCB is a 64-byte “short format” CCB.
+
+        The extract CCB command format can be specified by the following packed C structure for a big-endian
+        machine:
+
+
+                  struct extract_ccb {
+                         uint32_t header;
+                         uint32_t control;
+                         uint64_t completion;
+                         uint64_t primary_input;
+                         uint64_t data_access_control;
+                         uint64_t secondary_input;
+                         uint64_t reserved;
+                         uint64_t output;
+                         uint64_t table;
+                  };
+
+
+        The exact field offsets, sizes, and composition are as follows:
+
+         Offset         Size            Field Description
+         0              4               CCB header (Table 36.1, “CCB Header Format”)
+         4              4               Command control
+                                        Bits        Field Description
+                                        [31:28]     Primary Input Format (see Section 36.2.1.1.1, “Primary Input
+                                                    Format”)
+                                        [27:23]     Primary Input Element Size (see Section 36.2.1.1.2, “Primary
+                                                    Input Element Size”)
+                                        [22:20]     Primary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                                                    Element Offsets”)
+                                        [19]        Secondary Input Format (see Section 36.2.1.1.3, “Secondary
+                                                    Input Format”)
+                                        [18:16]     Secondary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                                                    Element Offsets”)
+
+
+                                                       516
+                        Coprocessor services
+
+
+Offset   Size   Field Description
+                Bits         Field Description
+                [15:14]      Secondary Input Element Size (see Section 36.2.1.1.4,
+                             “Secondary Input Element Size”
+                [13:10]      Output Format (see Section 36.2.1.1.6, “Output Format”)
+                [9]          Padding Direction selector: A value of 1 causes padding bytes
+                             to be added to the left side of output elements. A value of 0
+                             causes padding bytes to be added to the right side of output
+                             elements.
+                [8:0]        Reserved
+8        8      Completion
+                Bits         Field Description
+                [63:60]      ADI version (see Section 36.2.1.1.7, “Application Data
+                             Integrity (ADI)”)
+                [59]         If set to 1, a virtual device interrupt will be generated using
+                             the device interrupt number specified in the lower bits of this
+                             completion word. If 0, the lower bits of this completion word
+                             are ignored.
+                [58:6]       Completion area address bits [58:6]. Address type is
+                             determined by CCB header.
+                [5:0]        Virtual device interrupt number for completion interrupt, if
+                             enabled.
+16       8      Primary Input
+                Bits         Field Description
+                [63:60]      ADI version (see Section 36.2.1.1.7, “Application Data
+                             Integrity (ADI)”)
+                [59:56]      If using real address, these bits should be filled in with the
+                             page size code for the page boundary checking the guest wants
+                             the virtual machine to use when accessing this data stream
+                             (checking is only guaranteed to be performed when using API
+                             version 1.1 and later). If using a virtual address, this field will
+                             be used as as primary input address bits [59:56].
+                [55:0]       Primary input address bits [55:0]. Address type is determined
+                             by CCB header.
+24       8      Data Access Control
+                Bits         Field Description
+                [63:62]      Flow Control
+                             Value      Description
+                             0b'00      Disable flow control
+                             0b'01      Enable flow control (only valid with "ORCL,sun4v-
+                                        dax-fc" compatible virtual device variants)
+                             0b'10      Reserved
+                             0b'11      Reserved
+                [61:60]      Reserved (API 1.0)
+
+
+                                517
+                       Coprocessor services
+
+
+Offset   Size   Field Description
+                Bits        Field Description
+                            Pipeline target (API 2.0)
+                            Value      Description
+                            0b'00      Connect to primary input
+                            0b'01      Connect to secondary input
+                            0b'10      Reserved
+                            0b'11      Reserved
+                [59:40]     Output buffer size given in units of 64 bytes, minus 1. Value of
+                            0 means 64 bytes, value of 1 means 128 bytes, etc. Buffer size is
+                            only enforced if flow control is enabled in Flow Control field.
+                [39:32]     Reserved
+                [31:30]     Output Data Cache Allocation
+                            Value      Description
+                            0b'00      Do not allocate cache lines for output data stream.
+                            0b'01      Force cache lines for output data stream to be
+                                       allocated in the cache that is local to the submitting
+                                       virtual cpu.
+                            0b'10      Allocate cache lines for output data stream, but allow
+                                       existing cache lines associated with the data to remain
+                                       in their current cache instance. Any memory not
+                                       already in cache will be allocated in the cache local
+                                       to the submitting virtual cpu.
+                            0b'11      Reserved
+                [29:26]     Reserved
+                [25:24]     Primary Input Length Format
+                            Value      Description
+                            0b'00      Number of primary symbols
+                            0b'01      Number of primary bytes
+                            0b'10      Number of primary bits
+                            0b'11      Reserved
+                [23:0]      Primary Input Length
+                            Format                      Field Value
+                            # of primary symbols        Number of input elements to process,
+                                                        minus 1. Command execution stops
+                                                        once count is reached.
+                            # of primary bytes          Number of input bytes to process,
+                                                        minus 1. Command execution stops
+                                                        once count is reached. The count is
+                                                        done before any decompression or
+                                                        decoding.
+                            # of primary bits           Number of input bits to process,
+                                                        minus 1. Command execution stops
+
+
+
+                               518
+                                                Coprocessor services
+
+
+        Offset          Size           Field Description
+                                        Bits         Field Description
+                                                     Format                     Field Value
+                                                                                once count is reached. The count is
+                                                                                done before any decompression or
+                                                                                decoding, and does not include any
+                                                                                bits skipped by the Primary Input
+                                                                                Offset field value of the command
+                                                                                control word.
+        32              8              Secondary Input, if used by Primary Input Format. Same fields as Primary
+                                       Input.
+        40              8              Reserved
+        48              8              Output (same fields as Primary Input)
+        56              8              Symbol Table (if used by Primary Input)
+                                        Bits         Field Description
+                                        [63:60]      ADI version (see Section 36.2.1.1.7, “Application Data
+                                                     Integrity (ADI)”)
+                                        [59:56]      If using real address, these bits should be filled in with the
+                                                     page size code for the page boundary checking the guest wants
+                                                     the virtual machine to use when accessing this data stream
+                                                     (checking is only guaranteed to be performed when using API
+                                                     version 1.1 and later). If using a virtual address, this field will
+                                                     be used as as symbol table address bits [59:56].
+                                        [55:4]       Symbol table address bits [55:4]. Address type is determined
+                                                     by CCB header.
+                                        [3:0]        Symbol table version
+                                                     Value     Description
+                                                     0         Huffman encoding. Must use 64 byte aligned table
+                                                               address. (Only available when using version 0 CCBs)
+                                                     1         OZIP encoding. Must use 16 byte aligned table
+                                                               address. (Only available when using version 1 CCBs)
+
+
+36.2.1.3. Scan commands
+
+        The scan commands search a stream of input data elements for values which match the selection criteria.
+        All the input format types are supported. There are multiple formats for the scan commands, allowing the
+        scan to search for exact matches to one value, exact matches to either of two values, or any value within
+        a specified range. The specific type of scan is indicated by the command code in the CCB header. For the
+        scan range commands, the boundary conditions can be specified as greater-than-or-equal-to a value, less-
+        than-or-equal-to a value, or both by using two boundary values.
+
+        There are two supported formats for the output stream: the bit vector and index array formats (codes 0x8,
+        0xD, and 0xE). For the standard scan command using the bit vector output, for each input element there
+        exists one bit in the vector that is set if the input element matched the scan criteria, or clear if not. The
+        inverted scan command inverts the polarity of the bits in the output. The most significant bit of the first
+        byte of the output stream corresponds to the first element in the input stream. The standard index array
+        output format contains one array entry for each input element that matched the scan criteria. Each array
+
+
+
+                                                         519
+                                       Coprocessor services
+
+
+entry is the index of an input element that matched the scan criteria. An inverted scan command produces
+a similar array, but of all the input elements which did NOT match the scan criteria.
+
+The return value of the CCB completion area contains the number of input elements found which match
+the scan criteria (or number that did not match for the inverted scans). The “number of elements processed”
+field in the CCB completion area will be valid, indicating the number of input elements processed.
+
+These commands are 128-byte “long format” CCBs.
+
+The scan CCB command format can be specified by the following packed C structure for a big-endian
+machine:
+
+
+         struct scan_ccb         {
+                uint32_t         header;
+                uint32_t         control;
+                uint64_t         completion;
+                uint64_t         primary_input;
+                uint64_t         data_access_control;
+                uint64_t         secondary_input;
+                uint64_t         match_criteria0;
+                uint64_t         output;
+                uint64_t         table;
+                uint64_t         match_criteria1;
+                uint64_t         match_criteria2;
+                uint64_t         match_criteria3;
+                uint64_t         reserved[5];
+         };
+
+
+The exact field offsets, sizes, and composition are as follows:
+
+Offset         Size            Field Description
+0              4               CCB header (Table 36.1, “CCB Header Format”)
+4              4               Command control
+                               Bits         Field Description
+                               [31:28]      Primary Input Format (see Section 36.2.1.1.1, “Primary Input
+                                            Format”)
+                               [27:23]      Primary Input Element Size (see Section 36.2.1.1.2, “Primary
+                                            Input Element Size”)
+                               [22:20]      Primary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                                            Element Offsets”)
+                               [19]         Secondary Input Format (see Section 36.2.1.1.3, “Secondary
+                                            Input Format”)
+                               [18:16]      Secondary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                                            Element Offsets”)
+                               [15:14]      Secondary Input Element Size (see Section 36.2.1.1.4,
+                                            “Secondary Input Element Size”
+                               [13:10]      Output Format (see Section 36.2.1.1.6, “Output Format”)
+                               [9:5]        Operand size for first scan criteria value. In a scan value
+                                            operation, this is one of two potential extact match values.
+                                            In a scan range operation, this is the size of the upper range
+
+
+                                               520
+                        Coprocessor services
+
+
+Offset   Size   Field Description
+                Bits         Field Description
+                             boundary. The value of this field is the number of bytes in the
+                             operand, minus 1. Values 0xF-0x1E are reserved. A value of
+                             0x1F indicates this operand is not in use for this scan operation.
+                [4:0]        Operand size for second scan criteria value. In a scan value
+                             operation, this is one of two potential extact match values.
+                             In a scan range operation, this is the size of the lower range
+                             boundary. The value of this field is the number of bytes in the
+                             operand, minus 1. Values 0xF-0x1E are reserved. A value of
+                             0x1F indicates this operand is not in use for this scan operation.
+8        8      Completion (same fields as Section 36.2.1.2, “Extract command”)
+16       8      Primary Input (same fields as Section 36.2.1.2, “Extract command”)
+24       8      Data Access Control (same fields as Section 36.2.1.2, “Extract command”)
+32       8      Secondary Input, if used by Primary Input Format. Same fields as Primary
+                Input.
+40       4      Most significant 4 bytes of first scan criteria operand. If first operand is less
+                than 4 bytes, the value is left-aligned to the lowest address bytes.
+44       4      Most significant 4 bytes of second scan criteria operand. If second operand
+                is less than 4 bytes, the value is left-aligned to the lowest address bytes.
+48       8      Output (same fields as Primary Input)
+56       8      Symbol Table (if used by Primary Input). Same fields as Section 36.2.1.2,
+                “Extract command”
+64       4      Next 4 most significant bytes of first scan criteria operand occuring after the
+                bytes specified at offset 40, if needed by the operand size. If first operand
+                is less than 8 bytes, the valid bytes are left-aligned to the lowest address.
+68       4      Next 4 most significant bytes of second scan criteria operand occuring after
+                the bytes specified at offset 44, if needed by the operand size. If second
+                operand is less than 8 bytes, the valid bytes are left-aligned to the lowest
+                address.
+72       4      Next 4 most significant bytes of first scan criteria operand occuring after the
+                bytes specified at offset 64, if needed by the operand size. If first operand
+                is less than 12 bytes, the valid bytes are left-aligned to the lowest address.
+76       4      Next 4 most significant bytes of second scan criteria operand occuring after
+                the bytes specified at offset 68, if needed by the operand size. If second
+                operand is less than 12 bytes, the valid bytes are left-aligned to the lowest
+                address.
+80       4      Next 4 most significant bytes of first scan criteria operand occuring after the
+                bytes specified at offset 72, if needed by the operand size. If first operand
+                is less than 16 bytes, the valid bytes are left-aligned to the lowest address.
+84       4      Next 4 most significant bytes of second scan criteria operand occuring after
+                the bytes specified at offset 76, if needed by the operand size. If second
+                operand is less than 16 bytes, the valid bytes are left-aligned to the lowest
+                address.
+
+
+
+
+                                521
+                                               Coprocessor services
+
+
+36.2.1.4. Translate commands
+
+        The translate commands takes an input array of indicies, and a table of single bit values indexed by those
+        indicies, and outputs a bit vector or index array created by reading the tables bit value at each index in
+        the input array. The output should therefore contain exactly one bit per index in the input data stream,
+        when outputing as a bit vector. When outputing as an index array, the number of elements depends on the
+        values read in the bit table, but will always be less than, or equal to, the number of input elements. Only
+        a restricted subset of the possible input format types are supported. No variable width or Huffman/OZIP
+        encoded input streams are allowed. The primary input data element size must be 3 bytes or less.
+
+        The maximum table index size allowed is 15 bits, however, larger input elements may be used to provide
+        additional processing of the output values. If 2 or 3 byte values are used, the least significant 15 bits are
+        used as an index into the bit table. The most significant 9 bits (when using 3-byte input elements) or single
+        bit (when using 2-byte input elements) are compared against a fixed 9-bit test value provided in the CCB.
+        If the values match, the value from the bit table is used as the output element value. If the values do not
+        match, the output data element value is forced to 0.
+
+        In the inverted translate operation, the bit value read from bit table is inverted prior to its use. The additional
+        additional processing based on any additional non-index bits remains unchanged, and still forces the output
+        element value to 0 on a mismatch. The specific type of translate command is indicated by the command
+        code in the CCB header.
+
+        There are two supported formats for the output stream: the bit vector and index array formats (codes 0x8,
+        0xD, and 0xE). The index array format is an array of indicies of bits which would have been set if the
+        output format was a bit array.
+
+        The return value of the CCB completion area contains the number of bits set in the output bit vector,
+        or number of elements in the output index array. The “number of elements processed” field in the CCB
+        completion area will be valid, indicating the number of input elements processed.
+
+        These commands are 64-byte “short format” CCBs.
+
+        The translate CCB command format can be specified by the following packed C structure for a big-endian
+        machine:
+
+
+                 struct translate_ccb {
+                        uint32_t header;
+                        uint32_t control;
+                        uint64_t completion;
+                        uint64_t primary_input;
+                        uint64_t data_access_control;
+                        uint64_t secondary_input;
+                        uint64_t reserved;
+                        uint64_t output;
+                        uint64_t table;
+                 };
+
+
+        The exact field offsets, sizes, and composition are as follows:
+
+
+        Offset          Size             Field Description
+        0               4                CCB header (Table 36.1, “CCB Header Format”)
+
+
+                                                        522
+                        Coprocessor services
+
+
+Offset   Size   Field Description
+4        4      Command control
+                Bits         Field Description
+                [31:28]      Primary Input Format (see Section 36.2.1.1.1, “Primary Input
+                             Format”)
+                [27:23]      Primary Input Element Size (see Section 36.2.1.1.2, “Primary
+                             Input Element Size”)
+                [22:20]      Primary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                             Element Offsets”)
+                [19]         Secondary Input Format (see Section 36.2.1.1.3, “Secondary
+                             Input Format”)
+                [18:16]      Secondary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                             Element Offsets”)
+                [15:14]      Secondary Input Element Size (see Section 36.2.1.1.4,
+                             “Secondary Input Element Size”
+                [13:10]      Output Format (see Section 36.2.1.1.6, “Output Format”)
+                [9]          Reserved
+                [8:0]        Test value used for comparison against the most significant bits
+                             in the input values, when using 2 or 3 byte input elements.
+8        8      Completion (same fields as Section 36.2.1.2, “Extract command”
+16       8      Primary Input (same fields as Section 36.2.1.2, “Extract command”
+24       8      Data Access Control (same fields as Section 36.2.1.2, “Extract command”,
+                except Primary Input Length Format may not use the 0x0 value)
+32       8      Secondary Input, if used by Primary Input Format. Same fields as Primary
+                Input.
+40       8      Reserved
+48       8      Output (same fields as Primary Input)
+56       8      Bit Table
+                Bits         Field Description
+                [63:60]      ADI version (see Section 36.2.1.1.7, “Application Data
+                             Integrity (ADI)”)
+                [59:56]      If using real address, these bits should be filled in with the
+                             page size code for the page boundary checking the guest wants
+                             the virtual machine to use when accessing this data stream
+                             (checking is only guaranteed to be performed when using API
+                             version 1.1 and later). If using a virtual address, this field will
+                             be used as as bit table address bits [59:56]
+                [55:4]       Bit table address bits [55:4]. Address type is determined by
+                             CCB header. Address must be 64-byte aligned (CCB version
+                             0) or 16-byte aligned (CCB version 1).
+                [3:0]        Bit table version
+                             Value      Description
+                             0          4KB table size
+                             1          8KB table size
+
+
+
+                                 523
+                                              Coprocessor services
+
+
+36.2.1.5. Select command
+        The select command filters the primary input data stream by using a secondary input bit vector to determine
+        which input elements to include in the output. For each bit set at a given index N within the bit vector,
+        the Nth input element is included in the output. If the bit is not set, the element is not included. Only a
+        restricted subset of the possible input format types are supported. No variable width or run length encoded
+        input streams are allowed, since the secondary input stream is used for the filtering bit vector.
+
+        The only supported output format is a padded, byte-aligned output stream. The stream follows the same
+        rules and restrictions as padded output stream described in Section 36.2.1.2, “Extract command”.
+
+        The return value of the CCB completion area contains the number of bits set in the input bit vector. The
+        "number of elements processed" field in the CCB completion area will be valid, indicating the number
+        of input elements processed.
+
+        The select CCB is a 64-byte “short format” CCB.
+
+        The select CCB command format can be specified by the following packed C structure for a big-endian
+        machine:
+
+
+                  struct select_ccb {
+                         uint32_t header;
+                         uint32_t control;
+                         uint64_t completion;
+                         uint64_t primary_input;
+                         uint64_t data_access_control;
+                         uint64_t secondary_input;
+                         uint64_t reserved;
+                         uint64_t output;
+                         uint64_t table;
+                  };
+
+
+        The exact field offsets, sizes, and composition are as follows:
+
+         Offset        Size            Field Description
+         0             4               CCB header (Table 36.1, “CCB Header Format”)
+         4             4               Command control
+                                       Bits        Field Description
+                                       [31:28]     Primary Input Format (see Section 36.2.1.1.1, “Primary Input
+                                                   Format”)
+                                       [27:23]     Primary Input Element Size (see Section 36.2.1.1.2, “Primary
+                                                   Input Element Size”)
+                                       [22:20]     Primary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                                                   Element Offsets”)
+                                       [19]        Secondary Input Format (see Section 36.2.1.1.3, “Secondary
+                                                   Input Format”)
+                                       [18:16]     Secondary Input Starting Offset (see Section 36.2.1.1.5, “Input
+                                                   Element Offsets”)
+                                       [15:14]     Secondary Input Element Size (see Section 36.2.1.1.4,
+                                                   “Secondary Input Element Size”
+
+
+                                                      524
+                                               Coprocessor services
+
+
+        Offset         Size            Field Description
+                                       Bits         Field Description
+                                       [13:10]      Output Format (see Section 36.2.1.1.6, “Output Format”)
+                                       [9]          Padding Direction selector: A value of 1 causes padding bytes
+                                                    to be added to the left side of output elements. A value of 0
+                                                    causes padding bytes to be added to the right side of output
+                                                    elements.
+                                       [8:0]        Reserved
+        8              8               Completion (same fields as Section 36.2.1.2, “Extract command”
+        16             8               Primary Input (same fields as Section 36.2.1.2, “Extract command”
+        24             8               Data Access Control (same fields as Section 36.2.1.2, “Extract command”)
+        32             8               Secondary Bit Vector Input. Same fields as Primary Input.
+        40             8               Reserved
+        48             8               Output (same fields as Primary Input)
+        56             8               Symbol Table (if used by Primary Input). Same fields as Section 36.2.1.2,
+                                       “Extract command”
+
+36.2.1.6. No-op and Sync commands
+        The no-op (no operation) command is a CCB which has no processing effect. The CCB, when processed
+        by the virtual machine, simply updates the completion area with its execution status. The CCB may have
+        the serial-conditional flags set in order to restrict when it executes.
+
+        The sync command is a variant of the no-op command which with restricted execution timing. A sync
+        command CCB will only execute when all previous commands submitted in the same request have
+        completed. This is stronger than the conditional flag sequencing, which is only dependent on a single
+        previous serial CCB. While the relative ordering is guaranteed, virtual machine implementations with
+        shared hardware resources may cause the sync command to wait for longer than the minimum required
+        time.
+
+        The return value of the CCB completion area is invalid for these CCBs. The “number of elements
+        processed” field is also invalid for these CCBs.
+
+        These commands are 64-byte “short format” CCBs.
+
+        The no-op CCB command format can be specified by the following packed C structure for a big-endian
+        machine:
+
+
+                 struct nop_ccb {
+                        uint32_t header;
+                        uint32_t control;
+                        uint64_t completion;
+                        uint64_t reserved[6];
+                 };
+
+
+        The exact field offsets, sizes, and composition are as follows:
+
+        Offset         Size            Field Description
+        0              4               CCB header (Table 36.1, “CCB Header Format”)
+
+
+                                                       525
+                                          Coprocessor services
+
+
+       Offset        Size          Field Description
+       4             4             Command control
+                                   Bits        Field Description
+                                   [31]        If set, this CCB functions as a Sync command. If clear, this
+                                               CCB functions as a No-op command.
+                                   [30:0]      Reserved
+       8             8             Completion (same fields as Section 36.2.1.2, “Extract command”
+       16            46            Reserved
+
+36.2.2. CCB Completion Area
+       All CCB commands use a common 128-byte Completion Area format, which can be specified by the
+       following packed C structure for a big-endian machine:
+
+
+                struct completion_area {
+                       uint8_t status_flag;
+                       uint8_t error_note;
+                       uint8_t rsvd0[2];
+                       uint32_t error_values;
+                       uint32_t output_size;
+                       uint32_t rsvd1;
+                       uint64_t run_time;
+                       uint64_t run_stats;
+                       uint32_t elements;
+                       uint8_t rsvd2[20];
+                       uint64_t return_value;
+                       uint64_t extra_return_value[8];
+                };
+
+
+       The Completion Area must be a 128-byte aligned memory location. The exact layout can be described
+       using byte offsets and sizes relative to the memory base:
+
+       Offset        Size          Field Description
+       0             1             CCB execution status
+                                   0x0                  Command not yet completed
+                                   0x1                  Command ran and succeeded
+                                   0x2                  Command ran and failed (partial results may be been
+                                                        produced)
+                                   0x3                  Command ran and was killed (partial execution may
+                                                        have occurred)
+                                   0x4                  Command was not run
+                                   0x5-0xF              Reserved
+       1             1             Error reason code
+                                   0x0                  Reserved
+                                   0x1                  Buffer overflow
+
+
+                                                  526
+                                      Coprocessor services
+
+
+Offset          Size           Field Description
+                                0x2                 CCB decoding error
+                                0x3                 Page overflow
+                                0x4-0x6             Reserved
+                                0x7                 Command was killed
+                                0x8                 Command execution timeout
+                                0x9                 ADI miscompare error
+                                0xA                 Data format error
+                                0xB-0xD             Reserved
+                                0xE                 Unexpected hardware error (Do not retry)
+                                0xF                 Unexpected hardware error (Retry is ok)
+                                0x10-0x7F           Reserved
+                                0x80                Partial Symbol Warning
+                                0x81-0xFF           Reserved
+2               2              Reserved
+4               4              If a partial symbol warning was generated, this field contains the number
+                               of remaining bits which were not decoded.
+8               4              Number of bytes of output produced
+12              4              Reserved
+16              8              Runtime of command (unspecified time units)
+24              8              Reserved
+32              4              Number of elements processed
+36              20             Reserved
+56              8              Return value
+64              64             Extended return value
+
+The CCB completion area should be treated as read-only by guest software. The CCB execution status
+byte will be cleared by the Hypervisor to reflect the pending execution status when the CCB is submitted
+successfully. All other fields are considered invalid upon CCB submission until the CCB execution status
+byte becomes non-zero.
+
+CCBs which complete with status 0x2 or 0x3 may produce partial results and/or side effects due to partial
+execution of the CCB command. Some valid data may be accessible depending on the fault type, however,
+it is recommended that guest software treat the destination buffer as being in an unknown state. If a CCB
+completes with a status byte of 0x2, the error reason code byte can be read to determine what corrective
+action should be taken.
+
+A buffer overflow indicates that the results of the operation exceeded the size of the output buffer indicated
+in the CCB. The operation can be retried by resubmitting the CCB with a larger output buffer.
+
+A CCB decoding error indicates that the CCB contained some invalid field values. It may be also be
+triggered if the CCB output is directed at a non-existent secondary input and the pipelining hint is followed.
+
+A page overflow error indicates that the operation required accessing a memory location beyond the page
+size associated with a given address. No data will have been read or written past the page boundary, but
+partial results may have been written to the destination buffer. The CCB can be resubmitted with a larger
+page size memory allocation to complete the operation.
+
+
+                                              527
+                                            Coprocessor services
+
+
+       In the case of pipelined CCBs, a page overflow error will be triggered if the output from the pipeline source
+       CCB ends before the input of the pipeline target CCB. Page boundaries are ignored when the pipeline
+       hint is followed.
+
+       Command kill indicates that the CCB execution was halted or prevented by use of the ccb_kill API call.
+
+       Command timeout indicates that the CCB execution began, but did not complete within a pre-determined
+       limit set by the virtual machine. The command may have produced some or no output. The CCB may be
+       resubmitted with no alterations.
+
+       ADI miscompare indicates that the memory buffer version specified in the CCB did not match the value
+       in memory when accessed by the virtual machine. Guest software should not attempt to resubmit the CCB
+       without determining the cause of the version mismatch.
+
+       A data format error indicates that the input data stream did not follow the specified data input formatting
+       selected in the CCB.
+
+       Some CCBs which encounter hardware errors may be resubmitted without change. Persistent hardware
+       errors may result in multiple failures until RAS software can identify and isolate the faulty component.
+
+       The output size field indicates the number of bytes of valid output in the destination buffer. This field is
+       not valid for all possible CCB commands.
+
+       The runtime field indicates the execution time of the CCB command once it leaves the internal virtual
+       machine queue. The time units are fixed, but unspecified, allowing only relative timing comparisons
+       by guest software. The time units may also vary by hardware platform, and should not be construed to
+       represent any absolute time value.
+
+       Some data query commands process data in units of elements. If applicable to the command, the number of
+       elements processed is indicated in the listed field. This field is not valid for all possible CCB commands.
+
+       The return value and extended return value fields are output locations for commands which do not use
+       a destination output buffer, or have secondary return results. The field is not valid for all possible CCB
+       commands.
+
+36.3. Hypervisor API Functions
+36.3.1. ccb_submit
+       trap#             FAST_TRAP
+       function#         CCB_SUBMIT
+       arg0              address
+       arg1              length
+       arg2              flags
+       arg3              reserved
+       ret0              status
+       ret1              length
+       ret2              status data
+       ret3              reserved
+
+       Submit one or more coprocessor control blocks (CCBs) for evaluation and processing by the virtual
+       machine. The CCBs are passed in a linear array indicated by address. length indicates the size of
+       the array in bytes.
+
+
+                                                     528
+                                      Coprocessor services
+
+
+The address should be aligned to the size indicated by length, rounded up to the nearest power of
+two. Virtual machines implementations may reject submissions which do not adhere to that alignment.
+length must be a multiple of 64 bytes. If length is zero, the maximum supported array length will be
+returned as length in ret1. In all other cases, the length value in ret1 will reflect the number of bytes
+successfully consumed from the input CCB array.
+
+      Implementation note
+      Virtual machines should never reject submissions based on the alignment of address if the
+      entire array is contained within a single memory page of the smallest page size supported by the
+      virtual machine.
+
+A guest may choose to submit addresses used in this API function, including the CCB array address,
+as either a real or virtual addresses, with the type of each address indicated in flags. Virtual addresses
+must be present in either the TLB or an active TSB to be processed. The translation context for virtual
+addresses is determined by a combination of CCB contents and the flags argument.
+
+The flags argument is divided into multiple fields defined as follows:
+
+
+Bits            Field Description
+[63:16]         Reserved
+[15]            Disable ADI for VA reads (in API 2.0)
+                Reserved (in API 1.0)
+[14]            Virtual addresses within CCBs are translated in privileged context
+[13:12]         Alternate translation context for virtual addresses within CCBs:
+                 0b'00        CCBs requesting alternate context are rejected
+                 0b'01        Reserved
+                 0b'10        CCBs requesting alternate context use secondary context
+                 0b'11        CCBs requesting alternate context use nucleus context
+[11:9]          Reserved
+[8]             Queue info flag
+[7]             All-or-nothing flag
+[6]             If address is a virtual address, treat its translation context as privileged
+[5:4]           Address type of address:
+                 0b'00        Real address
+                 0b'01        Virtual address in primary context
+                 0b'10        Virtual address in secondary context
+                 0b'11        Virtual address in nucleus context
+[3:2]           Reserved
+[1:0]           CCB command type:
+                 0b'00        Reserved
+                 0b'01        Reserved
+                 0b'10        Query command
+                 0b'11        Reserved
+
+
+
+                                              529
+                                             Coprocessor services
+
+
+         The CCB submission type and address type for the CCB array must be provided in the flags argument.
+         All other fields are optional values which change the default behavior of the CCB processing.
+
+         When set to one, the "Disable ADI for VA reads" bit will turn off ADI checking when using a virtual
+         address to load data. ADI checking will still be done when loading real-addressed memory. This bit is only
+         available when using major version 2 of the coprocessor API group; at major version 1 it is reserved. For
+         more information about using ADI and DAX, see Section 36.2.1.1.7, “Application Data Integrity (ADI)”.
+
+         By default, all virtual addresses are treated as user addresses. If the virtual address translations are
+         privileged, they must be marked as such in the appropriate flags field. The virtual addresses used within
+         the submitted CCBs must all be translated with the same privilege level.
+
+         By default, all virtual addresses used within the submitted CCBs are translated using the primary context
+         active at the time of the submission. The address type field within a CCB allows each address to request
+         translation in an alternate address context. The address context used when the alternate address context is
+         requested is selected in the flags argument.
+
+         The all-or-nothing flag specifies whether the virtual machine should allow partial submissions of the
+         input CCB array. When using CCBs with serial-conditional flags, it is strongly recommended to use
+         the all-or-nothing flag to avoid broken conditional chains. Using long CCB chains on a machine under
+         high coprocessor load may make this impractical, however, and require submitting without the flag.
+         When submitting serial-conditional CCBs without the all-or-nothing flag, guest software must manually
+         implement the serial-conditional behavior at any point where the chain was not submitted in a single API
+         call, and resubmission of the remaining CCBs should clear any conditional flag that might be set in the
+         first remaining CCB. Failure to do so will produce indeterminate CCB execution status and ordering.
+
+         When the all-or-nothing flag is not specified, callers should check the value of length in ret1 to determine
+         how many CCBs from the array were successfully submitted. Any remaining CCBs can be resubmitted
+         without modifications.
+
+         The value of length in ret1 is also valid when the API call returns an error, and callers should always
+         check its value to determine which CCBs in the array were already processed. This will additionally
+         identify which CCB encountered the processing error, and was not submitted successfully.
+
+         If the queue info flag is used during submission, and at least one CCB was successfully submitted, the
+         length value in ret1 will be a multi-field value defined as follows:
+          Bits          Field Description
+          [63:48]       DAX unit instance identifier
+          [47:32]       DAX queue instance identifier
+          [31:16]       Reserved
+          [15:0]        Number of CCB bytes successfully submitted
+
+         The value of status data depends on the status value. See error status code descriptions for details.
+         The value is undefined for status values that do not specifically list a value for the status data.
+
+         The API has a reserved input and output register which will be used in subsequent minor versions of this
+         API function. Guest software implementations should treat that register as voltile across the function call
+         in order to maintain forward compatibility.
+
+36.3.1.1. Errors
+          EOK                       One or more CCBs have been accepted and enqueued in the virtual machine
+                                    and no errors were been encountered during submission. Some submitted
+                                    CCBs may not have been enqueued due to internal virtual machine limitations,
+                                    and may be resubmitted without changes.
+
+
+                                                       530
+                        Coprocessor services
+
+
+EWOULDBLOCK    An internal resource conflict within the virtual machine has prevented it from
+               being able to complete the CCB submissions sufficiently quickly, requiring
+               it to abandon processing before it was complete. Some CCBs may have been
+               successfully enqueued prior to the block, and all remaining CCBs may be
+               resubmitted without changes.
+EBADALIGN      CCB array is not on a 64-byte boundary, or the array length is not a multiple
+               of 64 bytes.
+ENORADDR       A real address used either for the CCB array, or within one of the submitted
+               CCBs, is not valid for the guest. Some CCBs may have been enqueued prior
+               to the error being detected.
+ENOMAP         A virtual address used either for the CCB array, or within one of the submitted
+               CCBs, could not be translated by the virtual machine using either the TLB
+               or TSB contents. The submission may be retried after adding the required
+               mapping, or by converting the virtual address into a real address. Due to the
+               shared nature of address translation resources, there is no theoretical limit on
+               the number of times the translation may fail, and it is recommended all guests
+               implement some real address based backup. The virtual address which failed
+               translation is returned as status data in ret2. Some CCBs may have been
+               enqueued prior to the error being detected.
+EINVAL         The virtual machine detected an invalid CCB during submission, or invalid
+               input arguments, such as bad flag values. Note that not all invalid CCB values
+               will be detected during submission, and some may be reported as errors in the
+               completion area instead. Some CCBs may have been enqueued prior to the
+               error being detected. This error may be returned if the CCB version is invalid.
+ETOOMANY       The request was submitted with the all-or-nothing flag set, and the array size is
+               greater than the virtual machine can support in a single request. The maximum
+               supported size for the current virtual machine can be queried by submitting a
+               request with a zero length array, as described above.
+ENOACCESS      The guest does not have permission to submit CCBs, or an address used in a
+               CCBs lacks sufficient permissions to perform the required operation (no write
+               permission on the destination buffer address, for example). A virtual address
+               which fails permission checking is returned as status data in ret2. Some
+               CCBs may have been enqueued prior to the error being detected.
+EUNAVAILABLE   The requested CCB operation could not be performed at this time. The
+               restricted operation availability may apply only to the first unsuccessfully
+               submitted CCB, or may apply to a larger scope. The status should not be
+               interpreted as permanent, and the guest should attempt to submit CCBs in
+               the future which had previously been unable to be performed. The status
+               data provides additional information about scope of the retricted availability
+               as follows:
+               Value       Description
+               0           Processing for the exact CCB instance submitted was unavailable,
+                           and it is recommended the guest emulate the operation. The
+                           guest should continue to submit all other CCBs, and assume no
+                           restrictions beyond this exact CCB instance.
+               1           Processing is unavailable for all CCBs using the requested opcode,
+                           and it is recommended the guest emulate the operation. The
+                           guest should continue to submit all other CCBs that use different
+                           opcodes, but can expect continued rejections of CCBs using the
+                           same opcode in the near future.
+
+
+                                 531
+                                              Coprocessor services
+
+
+                                      Value     Description
+                                      2         Processing is unavailable for all CCBs using the requested CCB
+                                                version, and it is recommended the guest emulate the operation.
+                                                The guest should continue to submit all other CCBs that use
+                                                different CCB versions, but can expect continued rejections of
+                                                CCBs using the same CCB version in the near future.
+                                      3         Processing is unavailable for all CCBs on the submitting vcpu,
+                                                and it is recommended the guest emulate the operation or resubmit
+                                                the CCB on a different vcpu. The guest should continue to submit
+                                                CCBs on all other vcpus but can expect continued rejections of all
+                                                CCBs on this vcpu in the near future.
+                                      4         Processing is unavailable for all CCBs, and it is recommended
+                                                the guest emulate the operation. The guest should expect all CCB
+                                                submissions to be similarly rejected in the near future.
+
+
+36.3.2. ccb_info
+
+        trap#               FAST_TRAP
+        function#           CCB_INFO
+        arg0                address
+        ret0                status
+        ret1                CCB state
+        ret2                position
+        ret3                dax
+        ret4                queue
+
+       Requests status information on a previously submitted CCB. The previously submitted CCB is identified
+       by the 64-byte aligned real address of the CCBs completion area.
+
+       A CCB can be in one of 4 states:
+
+
+        State                     Value       Description
+        COMPLETED                 0           The CCB has been fetched and executed, and is no longer active in
+                                              the virtual machine.
+        ENQUEUED                  1           The requested CCB is current in a queue awaiting execution.
+        INPROGRESS                2           The CCB has been fetched and is currently being executed. It may still
+                                              be possible to stop the execution using the ccb_kill hypercall.
+        NOTFOUND                  3           The CCB could not be located in the virtual machine, and does not
+                                              appear to have been executed. This may occur if the CCB was lost
+                                              due to a hardware error, or the CCB may not have been successfully
+                                              submitted to the virtual machine in the first place.
+
+               Implementation note
+               Some platforms may not be able to report CCBs that are currently being processed, and therefore
+               guest software should invoke the ccb_kill hypercall prior to assuming the request CCB will never
+               be executed because it was in the NOTFOUND state.
+
+
+                                                       532
+                                             Coprocessor services
+
+
+         The position return value is only valid when the state is ENQUEUED. The value returned is the number
+         of other CCBs ahead of the requested CCB, to provide a relative estimate of when the CCB may execute.
+
+         The dax return value is only valid when the state is ENQUEUED. The value returned is the DAX unit
+         instance indentifier for the DAX unit processing the queue where the requested CCB is located. The value
+         matches the value that would have been, or was, returned by ccb_submit using the queue info flag.
+
+         The queue return value is only valid when the state is ENQUEUED. The value returned is the DAX
+         queue instance indentifier for the DAX unit processing the queue where the requested CCB is located. The
+         value matches the value that would have been, or was, returned by ccb_submit using the queue info flag.
+
+36.3.2.1. Errors
+
+          EOK                       The request was proccessed and the CCB state is valid.
+          EBADALIGN                 address is not on a 64-byte aligned.
+          ENORADDR                  The real address provided for address is not valid.
+          EINVAL                    The CCB completion area contents are not valid.
+          EWOULDBLOCK               Internal resource contraints prevented the CCB state from being queried at this
+                                    time. The guest should retry the request.
+          ENOACCESS                 The guest does not have permission to access the coprocessor virtual device
+                                    functionality.
+
+36.3.3. ccb_kill
+
+          trap#           FAST_TRAP
+          function#       CCB_KILL
+          arg0            address
+          ret0            status
+          ret1            result
+
+         Request to stop execution of a previously submitted CCB. The previously submitted CCB is identified by
+         the 64-byte aligned real address of the CCBs completion area.
+
+         The kill attempt can produce one of several values in the result return value, reflecting the CCB state
+         and actions taken by the Hypervisor:
+
+          Result                Value       Description
+          COMPLETED             0           The CCB has been fetched and executed, and is no longer active in
+                                            the virtual machine. It could not be killed and no action was taken.
+          DEQUEUED              1           The requested CCB was still enqueued when the kill request was
+                                            submitted, and has been removed from the queue. Since the CCB
+                                            never began execution, no memory modifications were produced by
+                                            it, and the completion area will never be updated. The same CCB may
+                                            be submitted again, if desired, with no modifications required.
+          KILLED                2           The CCB had been fetched and was being executed when the kill
+                                            request was submitted. The CCB execution was stopped, and the CCB
+                                            is no longer active in the virtual machine. The CCB completion area
+                                            will reflect the killed status, with the subsequent implications that
+                                            partial results may have been produced. Partial results may include full
+
+
+                                                      533
+                                              Coprocessor services
+
+
+          Result                 Value       Description
+                                             command execution if the command was stopped just prior to writing
+                                             to the completion area.
+          NOTFOUND               3           The CCB could not be located in the virtual machine, and does not
+                                             appear to have been executed. This may occur if the CCB was lost
+                                             due to a hardware error, or the CCB may not have been successfully
+                                             submitted to the virtual machine in the first place. CCBs in the state
+                                             are guaranteed to never execute in the future unless resubmitted.
+
+36.3.3.1. Interactions with Pipelined CCBs
+
+         If the pipeline target CCB is killed but the pipeline source CCB was skipped, the completion area of the
+         target CCB may contain status (4,0) "Command was skipped" instead of (3,7) "Command was killed".
+
+         If the pipeline source CCB is killed, the pipeline target CCB's completion status may read (1,0) "Success".
+         This does not mean the target CCB was processed; since the source CCB was killed, there was no
+         meaningful output on which the target CCB could operate.
+
+36.3.3.2. Errors
+
+          EOK                        The request was proccessed and the result is valid.
+          EBADALIGN                  address is not on a 64-byte aligned.
+          ENORADDR                   The real address provided for address is not valid.
+          EINVAL                     The CCB completion area contents are not valid.
+          EWOULDBLOCK                Internal resource contraints prevented the CCB from being killed at this time.
+                                     The guest should retry the request.
+          ENOACCESS                  The guest does not have permission to access the coprocessor virtual device
+                                     functionality.
+
+36.3.4. dax_info
+          trap#            FAST_TRAP
+          function#        DAX_INFO
+          ret0             status
+          ret1             Number of enabled DAX units
+          ret2             Number of disabled DAX units
+
+         Returns the number of DAX units that are enabled for the calling guest to submit CCBs. The number of
+         DAX units that are disabled for the calling guest are also returned. A disabled DAX unit would have been
+         available for CCB submission to the calling guest had it not been offlined.
+
+36.3.4.1. Errors
+
+          EOK                        The request was proccessed and the number of enabled/disabled DAX units
+                                     are valid.
+
+
+
+
+                                                       534
+
diff --git a/Documentation/sparc/oradax/oracle-dax.txt b/Documentation/sparc/oradax/oracle-dax.txt
new file mode 100644
index 000000000000..9d53ac93286f
--- /dev/null
+++ b/Documentation/sparc/oradax/oracle-dax.txt
@@ -0,0 +1,429 @@
+Oracle Data Analytics Accelerator (DAX)
+---------------------------------------
+
+DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8
+(DAX2) processor chips, and has direct access to the CPU's L3 caches
+as well as physical memory. It can perform several operations on data
+streams with various input and output formats.  A driver provides a
+transport mechanism and has limited knowledge of the various opcodes
+and data formats. A user space library provides high level services
+and translates these into low level commands which are then passed
+into the driver and subsequently the Hypervisor and the coprocessor.
+The library is the recommended way for applications to use the
+coprocessor, and the driver interface is not intended for general use.
+This document describes the general flow of the driver, its
+structures, and its programmatic interface. It also provides example
+code sufficient to write user or kernel applications that use DAX
+functionality.
+
+The user library is open source and available at:
+    https://oss.oracle.com/git/gitweb.cgi?p=libdax.git
+
+The Hypervisor interface to the coprocessor is described in detail in
+the accompanying document, dax-hv-api.txt, which is a plain text
+excerpt of the (Oracle internal) "UltraSPARC Virtual Machine
+Specification" version 3.0.20+15, dated 2017-09-25.
+
+
+High Level Overview
+-------------------
+
+A coprocessor request is described by a Command Control Block
+(CCB). The CCB contains an opcode and various parameters. The opcode
+specifies what operation is to be done, and the parameters specify
+options, flags, sizes, and addresses.  The CCB (or an array of CCBs)
+is passed to the Hypervisor, which handles queueing and scheduling of
+requests to the available coprocessor execution units. A status code
+returned indicates if the request was submitted successfully or if
+there was an error.  One of the addresses given in each CCB is a
+pointer to a "completion area", which is a 128 byte memory block that
+is written by the coprocessor to provide execution status. No
+interrupt is generated upon completion; the completion area must be
+polled by software to find out when a transaction has finished, but
+the M7 and later processors provide a mechanism to pause the virtual
+processor until the completion status has been updated by the
+coprocessor. This is done using the monitored load and mwait
+instructions, which are described in more detail later.  The DAX
+coprocessor was designed so that after a request is submitted, the
+kernel is no longer involved in the processing of it.  The polling is
+done at the user level, which results in almost zero latency between
+completion of a request and resumption of execution of the requesting
+thread.
+
+
+Addressing Memory
+-----------------
+
+The kernel does not have access to physical memory in the Sun4v
+architecture, as there is an additional level of memory virtualization
+present. This intermediate level is called "real" memory, and the
+kernel treats this as if it were physical.  The Hypervisor handles the
+translations between real memory and physical so that each logical
+domain (LDOM) can have a partition of physical memory that is isolated
+from that of other LDOMs.  When the kernel sets up a virtual mapping,
+it specifies a virtual address and the real address to which it should
+be mapped.
+
+The DAX coprocessor can only operate on physical memory, so before a
+request can be fed to the coprocessor, all the addresses in a CCB must
+be converted into physical addresses. The kernel cannot do this since
+it has no visibility into physical addresses. So a CCB may contain
+either the virtual or real addresses of the buffers or a combination
+of them. An "address type" field is available for each address that
+may be given in the CCB. In all cases, the Hypervisor will translate
+all the addresses to physical before dispatching to hardware. Address
+translations are performed using the context of the process initiating
+the request.
+
+
+The Driver API
+--------------
+
+An application makes requests to the driver via the write() system
+call, and gets results (if any) via read(). The completion areas are
+made accessible via mmap(), and are read-only for the application.
+
+The request may either be an immediate command or an array of CCBs to
+be submitted to the hardware.
+
+Each open instance of the device is exclusive to the thread that
+opened it, and must be used by that thread for all subsequent
+operations. The driver open function creates a new context for the
+thread and initializes it for use.  This context contains pointers and
+values used internally by the driver to keep track of submitted
+requests. The completion area buffer is also allocated, and this is
+large enough to contain the completion areas for many concurrent
+requests.  When the device is closed, any outstanding transactions are
+flushed and the context is cleaned up.
+
+On a DAX1 system (M7), the device will be called "oradax1", while on a
+DAX2 system (M8) it will be "oradax2". If an application requires one
+or the other, it should simply attempt to open the appropriate
+device. Only one of the devices will exist on any given system, so the
+name can be used to determine what the platform supports.
+
+The immediate commands are CCB_DEQUEUE, CCB_KILL, and CCB_INFO. For
+all of these, success is indicated by a return value from write()
+equal to the number of bytes given in the call. Otherwise -1 is
+returned and errno is set.
+
+CCB_DEQUEUE
+
+Tells the driver to clean up resources associated with past
+requests. Since no interrupt is generated upon the completion of a
+request, the driver must be told when it may reclaim resources.  No
+further status information is returned, so the user should not
+subsequently call read().
+
+CCB_KILL
+
+Kills a CCB during execution. The CCB is guaranteed to not continue
+executing once this call returns successfully. On success, read() must
+be called to retrieve the result of the action.
+
+CCB_INFO
+
+Retrieves information about a currently executing CCB. Note that some
+Hypervisors might return 'notfound' when the CCB is in 'inprogress'
+state. To ensure a CCB in the 'notfound' state will never be executed,
+CCB_KILL must be invoked on that CCB. Upon success, read() must be
+called to retrieve the details of the action.
+
+Submission of an array of CCBs for execution
+
+A write() whose length is a multiple of the CCB size is treated as a
+submit operation. The file offset is treated as the index of the
+completion area to use, and may be set via lseek() or using the
+pwrite() system call. If -1 is returned then errno is set to indicate
+the error. Otherwise, the return value is the length of the array that
+was actually accepted by the coprocessor. If the accepted length is
+equal to the requested length, then the submission was completely
+successful and there is no further status needed; hence, the user
+should not subsequently call read(). Partial acceptance of the CCB
+array is indicated by a return value less than the requested length,
+and read() must be called to retrieve further status information.  The
+status will reflect the error caused by the first CCB that was not
+accepted, and status_data will provide additional data in some cases.
+
+MMAP
+
+The mmap() function provides access to the completion area allocated
+in the driver.  Note that the completion area is not writeable by the
+user process, and the mmap call must not specify PROT_WRITE.
+
+
+Completion of a Request
+-----------------------
+
+The first byte in each completion area is the command status which is
+updated by the coprocessor hardware. Software may take advantage of
+new M7/M8 processor capabilities to efficiently poll this status byte.
+First, a "monitored load" is achieved via a Load from Alternate Space
+(ldxa, lduba, etc.) with ASI 0x84 (ASI_MONITOR_PRIMARY).  Second, a
+"monitored wait" is achieved via the mwait instruction (a write to
+%asr28). This instruction is like pause in that it suspends execution
+of the virtual processor for the given number of nanoseconds, but in
+addition will terminate early when one of several events occur. If the
+block of data containing the monitored location is modified, then the
+mwait terminates. This causes software to resume execution immediately
+(without a context switch or kernel to user transition) after a
+transaction completes. Thus the latency between transaction completion
+and resumption of execution may be just a few nanoseconds.
+
+
+Application Life Cycle of a DAX Submission
+------------------------------------------
+
+ - open dax device
+ - call mmap() to get the completion area address
+ - allocate a CCB and fill in the opcode, flags, parameters, addresses, etc.
+ - submit CCB via write() or pwrite()
+ - go into a loop executing monitored load + monitored wait and
+   terminate when the command status indicates the request is complete
+   (CCB_KILL or CCB_INFO may be used any time as necessary)
+ - perform a CCB_DEQUEUE
+ - call munmap() for completion area
+ - close the dax device
+
+
+Memory Constraints
+------------------
+
+The DAX hardware operates only on physical addresses. Therefore, it is
+not aware of virtual memory mappings and the discontiguities that may
+exist in the physical memory that a virtual buffer maps to. There is
+no I/O TLB or any scatter/gather mechanism. All buffers, whether input
+or output, must reside in a physically contiguous region of memory.
+
+The Hypervisor translates all addresses within a CCB to physical
+before handing off the CCB to DAX. The Hypervisor determines the
+virtual page size for each virtual address given, and uses this to
+program a size limit for each address. This prevents the coprocessor
+from reading or writing beyond the bound of the virtual page, even
+though it is accessing physical memory directly. A simpler way of
+saying this is that a DAX operation will never "cross" a virtual page
+boundary. If an 8k virtual page is used, then the data is strictly
+limited to 8k. If a user's buffer is larger than 8k, then a larger
+page size must be used, or the transaction size will be truncated to
+8k.
+
+Huge pages. A user may allocate huge pages using standard interfaces.
+Memory buffers residing on huge pages may be used to achieve much
+larger DAX transaction sizes, but the rules must still be followed,
+and no transaction will cross a page boundary, even a huge page.  A
+major caveat is that Linux on Sparc presents 8Mb as one of the huge
+page sizes. Sparc does not actually provide a 8Mb hardware page size,
+and this size is synthesized by pasting together two 4Mb pages. The
+reasons for this are historical, and it creates an issue because only
+half of this 8Mb page can actually be used for any given buffer in a
+DAX request, and it must be either the first half or the second half;
+it cannot be a 4Mb chunk in the middle, since that crosses a
+(hardware) page boundary. Note that this entire issue may be hidden by
+higher level libraries.
+
+
+CCB Structure
+-------------
+A CCB is an array of 8 64-bit words. Several of these words provide
+command opcodes, parameters, flags, etc., and the rest are addresses
+for the completion area, output buffer, and various inputs:
+
+   struct ccb {
+       u64   control;
+       u64   completion;
+       u64   input0;
+       u64   access;
+       u64   input1;
+       u64   op_data;
+       u64   output;
+       u64   table;
+   };
+
+See libdax/common/sys/dax1/dax1_ccb.h for a detailed description of
+each of these fields, and see dax-hv-api.txt for a complete description
+of the Hypervisor API available to the guest OS (ie, Linux kernel).
+
+The first word (control) is examined by the driver for the following:
+ - CCB version, which must be consistent with hardware version
+ - Opcode, which must be one of the documented allowable commands
+ - Address types, which must be set to "virtual" for all the addresses
+   given by the user, thereby ensuring that the application can
+   only access memory that it owns
+
+
+Example Code
+------------
+
+The DAX is accessible to both user and kernel code.  The kernel code
+can make hypercalls directly while the user code must use wrappers
+provided by the driver. The setup of the CCB is nearly identical for
+both; the only difference is in preparation of the completion area. An
+example of user code is given now, with kernel code afterwards.
+
+In order to program using the driver API, the file
+arch/sparc/include/uapi/asm/oradax.h must be included.
+
+First, the proper device must be opened. For M7 it will be
+/dev/oradax1 and for M8 it will be /dev/oradax2. The simplest
+procedure is to attempt to open both, as only one will succeed:
+
+	fd = open("/dev/oradax1", O_RDWR);
+	if (fd < 0)
+		fd = open("/dev/oradax2", O_RDWR);
+	if (fd < 0)
+	       /* No DAX found */
+
+Next, the completion area must be mapped:
+
+      completion_area = mmap(NULL, DAX_MMAP_LEN, PROT_READ, MAP_SHARED, fd, 0);
+
+All input and output buffers must be fully contained in one hardware
+page, since as explained above, the DAX is strictly constrained by
+virtual page boundaries.  In addition, the output buffer must be
+64-byte aligned and its size must be a multiple of 64 bytes because
+the coprocessor writes in units of cache lines.
+
+This example demonstrates the DAX Scan command, which takes as input a
+vector and a match value, and produces a bitmap as the output. For
+each input element that matches the value, the corresponding bit is
+set in the output.
+
+In this example, the input vector consists of a series of single bits,
+and the match value is 0. So each 0 bit in the input will produce a 1
+in the output, and vice versa, which produces an output bitmap which
+is the input bitmap inverted.
+
+For details of all the parameters and bits used in this CCB, please
+refer to section 36.2.1.3 of the DAX Hypervisor API document, which
+describes the Scan command in detail.
+
+	ccb->control =       /* Table 36.1, CCB Header Format */
+		  (2L << 48)     /* command = Scan Value */
+		| (3L << 40)     /* output address type = primary virtual */
+		| (3L << 34)     /* primary input address type = primary virtual */
+		             /* Section 36.2.1, Query CCB Command Formats */
+		| (1 << 28)     /* 36.2.1.1.1 primary input format = fixed width bit packed */
+		| (0 << 23)     /* 36.2.1.1.2 primary input element size = 0 (1 bit) */
+		| (8 << 10)     /* 36.2.1.1.6 output format = bit vector */
+		| (0 <<  5)	/* 36.2.1.3 First scan criteria size = 0 (1 byte) */
+		| (31 << 0);	/* 36.2.1.3 Disable second scan criteria */
+
+	ccb->completion = 0;    /* Completion area address, to be filled in by driver */
+
+	ccb->input0 = (unsigned long) input; /* primary input address */
+
+	ccb->access =       /* Section 36.2.1.2, Data Access Control */
+		  (2 << 24)    /* Primary input length format = bits */
+		| (nbits - 1); /* number of bits in primary input stream, minus 1 */
+
+	ccb->input1 = 0;       /* secondary input address, unused */
+
+	ccb->op_data = 0;      /* scan criteria (value to be matched) */
+
+	ccb->output = (unsigned long) output;	/* output address */
+
+	ccb->table = 0;	       /* table address, unused */
+
+The CCB submission is a write() or pwrite() system call to the
+driver. If the call fails, then a read() must be used to retrieve the
+status:
+
+	if (pwrite(fd, ccb, 64, 0) != 64) {
+		struct ccb_exec_result status;
+		read(fd, &status, sizeof(status));
+		/* bail out */
+	}
+
+After a successful submission of the CCB, the completion area may be
+polled to determine when the DAX is finished. Detailed information on
+the contents of the completion area can be found in section 36.2.2 of
+the DAX HV API document.
+
+	while (1) {
+		/* Monitored Load */
+		__asm__ __volatile__("lduba [%1] 0x84, %0\n"
+				     : "=r" (status)
+				     : "r"  (completion_area));
+
+		if (status)	     /* 0 indicates command in progress */
+			break;
+
+		/* MWAIT */
+		__asm__ __volatile__("wr %%g0, 1000, %%asr28\n" ::);    /* 1000 ns */
+	}
+
+A completion area status of 1 indicates successful completion of the
+CCB and validity of the output bitmap, which may be used immediately.
+All other non-zero values indicate error conditions which are
+described in section 36.2.2.
+
+	if (completion_area[0] != 1) {	/* section 36.2.2, 1 = command ran and succeeded */
+		/* completion_area[0] contains the completion status */
+		/* completion_area[1] contains an error code, see 36.2.2 */
+	}
+
+After the completion area has been processed, the driver must be
+notified that it can release any resources associated with the
+request. This is done via the dequeue operation:
+
+	struct dax_command cmd;
+	cmd.command = CCB_DEQUEUE;
+	if (write(fd, &cmd, sizeof(cmd)) != sizeof(cmd)) {
+		/* bail out */
+	}
+
+Finally, normal program cleanup should be done, i.e., unmapping
+completion area, closing the dax device, freeing memory etc.
+
+[Kernel example]
+
+The only difference in using the DAX in kernel code is the treatment
+of the completion area. Unlike user applications which mmap the
+completion area allocated by the driver, kernel code must allocate its
+own memory to use for the completion area, and this address and its
+type must be given in the CCB:
+
+	ccb->control |=      /* Table 36.1, CCB Header Format */
+	        (3L << 32);     /* completion area address type = primary virtual */
+
+	ccb->completion = (unsigned long) completion_area;   /* Completion area address */
+
+The dax submit hypercall is made directly. The flags used in the
+ccb_submit call are documented in the DAX HV API in section 36.3.1.
+
+#include <asm/hypervisor.h>
+
+	hv_rv = sun4v_ccb_submit((unsigned long)ccb, 64,
+				 HV_CCB_QUERY_CMD |
+				 HV_CCB_ARG0_PRIVILEGED | HV_CCB_ARG0_TYPE_PRIMARY |
+				 HV_CCB_VA_PRIVILEGED,
+				 0, &bytes_accepted, &status_data);
+
+	if (hv_rv != HV_EOK) {
+		/* hv_rv is an error code, status_data contains */
+		/* potential additional status, see 36.3.1.1 */
+	}
+
+After the submission, the completion area polling code is identical to
+that in user land:
+
+	while (1) {
+		/* Monitored Load */
+		__asm__ __volatile__("lduba [%1] 0x84, %0\n"
+				     : "=r" (status)
+				     : "r"  (completion_area));
+
+		if (status)	     /* 0 indicates command in progress */
+			break;
+
+		/* MWAIT */
+		__asm__ __volatile__("wr %%g0, 1000, %%asr28\n" ::);    /* 1000 ns */
+	}
+
+	if (completion_area[0] != 1) {	/* section 36.2.2, 1 = command ran and succeeded */
+		/* completion_area[0] contains the completion status */
+		/* completion_area[1] contains an error code, see 36.2.2 */
+	}
+
+The output bitmap is ready for consumption immediately after the
+completion status indicates success.
diff --git a/Documentation/speculation.txt b/Documentation/speculation.txt
new file mode 100644
index 000000000000..e9e6cbae2841
--- /dev/null
+++ b/Documentation/speculation.txt
@@ -0,0 +1,90 @@
+This document explains potential effects of speculation, and how undesirable
+effects can be mitigated portably using common APIs.
+
+===========
+Speculation
+===========
+
+To improve performance and minimize average latencies, many contemporary CPUs
+employ speculative execution techniques such as branch prediction, performing
+work which may be discarded at a later stage.
+
+Typically speculative execution cannot be observed from architectural state,
+such as the contents of registers. However, in some cases it is possible to
+observe its impact on microarchitectural state, such as the presence or
+absence of data in caches. Such state may form side-channels which can be
+observed to extract secret information.
+
+For example, in the presence of branch prediction, it is possible for bounds
+checks to be ignored by code which is speculatively executed. Consider the
+following code:
+
+	int load_array(int *array, unsigned int index)
+	{
+		if (index >= MAX_ARRAY_ELEMS)
+			return 0;
+		else
+			return array[index];
+	}
+
+Which, on arm64, may be compiled to an assembly sequence such as:
+
+	CMP	<index>, #MAX_ARRAY_ELEMS
+	B.LT	less
+	MOV	<returnval>, #0
+	RET
+  less:
+	LDR	<returnval>, [<array>, <index>]
+	RET
+
+It is possible that a CPU mis-predicts the conditional branch, and
+speculatively loads array[index], even if index >= MAX_ARRAY_ELEMS. This
+value will subsequently be discarded, but the speculated load may affect
+microarchitectural state which can be subsequently measured.
+
+More complex sequences involving multiple dependent memory accesses may
+result in sensitive information being leaked. Consider the following
+code, building on the prior example:
+
+	int load_dependent_arrays(int *arr1, int *arr2, int index)
+	{
+		int val1, val2,
+
+		val1 = load_array(arr1, index);
+		val2 = load_array(arr2, val1);
+
+		return val2;
+	}
+
+Under speculation, the first call to load_array() may return the value
+of an out-of-bounds address, while the second call will influence
+microarchitectural state dependent on this value. This may provide an
+arbitrary read primitive.
+
+====================================
+Mitigating speculation side-channels
+====================================
+
+The kernel provides a generic API to ensure that bounds checks are
+respected even under speculation. Architectures which are affected by
+speculation-based side-channels are expected to implement these
+primitives.
+
+The array_index_nospec() helper in <linux/nospec.h> can be used to
+prevent information from being leaked via side-channels.
+
+A call to array_index_nospec(index, size) returns a sanitized index
+value that is bounded to [0, size) even under cpu speculation
+conditions.
+
+This can be used to protect the earlier load_array() example:
+
+	int load_array(int *array, unsigned int index)
+	{
+		if (index >= MAX_ARRAY_ELEMS)
+			return 0;
+		else {
+			index = array_index_nospec(index, MAX_ARRAY_ELEMS);
+			return array[index];
+		}
+	}
diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py
index 39aa9e8697cc..fbedcc39460b 100644
--- a/Documentation/sphinx/kerneldoc.py
+++ b/Documentation/sphinx/kerneldoc.py
@@ -36,8 +36,7 @@ import glob
 
 from docutils import nodes, statemachine
 from docutils.statemachine import ViewList
-from docutils.parsers.rst import directives
-from sphinx.util.compat import Directive
+from docutils.parsers.rst import directives, Directive
 from sphinx.ext.autodoc import AutodocReporter
 
 __version__  = '1.0'
diff --git a/Documentation/sphinx/kfigure.py b/Documentation/sphinx/kfigure.py
index cef4ad19624c..b97228d2cc0e 100644
--- a/Documentation/sphinx/kfigure.py
+++ b/Documentation/sphinx/kfigure.py
@@ -81,7 +81,7 @@ __version__  = '1.0.0'
 # -------------
 
 def which(cmd):
-    """Searches the ``cmd`` in the ``PATH`` enviroment.
+    """Searches the ``cmd`` in the ``PATH`` environment.
 
     This *which* searches the PATH for executable ``cmd`` . First match is
     returned, if nothing is found, ``None` is returned.
@@ -419,7 +419,7 @@ def visit_kernel_render(self, node):
 
     tmp_ext = RENDER_MARKUP_EXT.get(srclang, None)
     if tmp_ext is None:
-        app.warn('kernel-render: "%s" unknow / include raw.' % (srclang))
+        app.warn('kernel-render: "%s" unknown / include raw.' % (srclang))
         return
 
     if not dot_cmd and tmp_ext == '.dot':
@@ -482,7 +482,7 @@ class KernelRender(Figure):
         srclang = self.arguments[0].strip()
         if srclang not in RENDER_MARKUP_EXT.keys():
             return [self.state_machine.reporter.warning(
-                'Unknow source language "%s", use one of: %s.' % (
+                'Unknown source language "%s", use one of: %s.' % (
                     srclang, ",".join(RENDER_MARKUP_EXT.keys())),
                 line=self.lineno)]
 
diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt
index 694968c7523c..412314eebda6 100644
--- a/Documentation/sysctl/kernel.txt
+++ b/Documentation/sysctl/kernel.txt
@@ -34,6 +34,7 @@ show up in /proc/sys/kernel:
 - hostname
 - hotplug
 - hardlockup_all_cpu_backtrace
+- hardlockup_panic
 - hung_task_panic
 - hung_task_check_count
 - hung_task_timeout_secs
@@ -313,6 +314,19 @@ will be initiated.
 1: on detection capture more debug information.
 ==============================================================
 
+hardlockup_panic:
+
+This parameter can be used to control whether the kernel panics
+when a hard lockup is detected.
+
+   0 - don't panic on hard lockup
+   1 - panic on hard lockup
+
+See Documentation/lockup-watchdogs.txt for more information.  This can
+also be set using the nmi_watchdog kernel parameter.
+
+==============================================================
+
 hotplug:
 
 Path for the hotplug policy agent.
@@ -377,7 +391,8 @@ kptr_restrict:
 This toggle indicates whether restrictions are placed on
 exposing kernel addresses via /proc and other interfaces.
 
-When kptr_restrict is set to (0), the default, there are no restrictions.
+When kptr_restrict is set to 0 (the default) the address is hashed before
+printing. (This is the equivalent to %p.)
 
 When kptr_restrict is set to (1), kernel pointers printed using the %pK
 format specifier will be replaced with 0's unless the user has CAP_SYSLOG
diff --git a/Documentation/sysctl/net.txt b/Documentation/sysctl/net.txt
index b67044a2575f..35c62f522754 100644
--- a/Documentation/sysctl/net.txt
+++ b/Documentation/sysctl/net.txt
@@ -95,7 +95,9 @@ dev_weight
 --------------
 
 The maximum number of packets that kernel can handle on a NAPI interrupt,
-it's a Per-CPU variable.
+it's a Per-CPU variable. For drivers that support LRO or GRO_HW, a hardware
+aggregated packet is counted as one packet in this context.
+
 Default: 64
 
 dev_weight_rx_bias
diff --git a/Documentation/sysctl/user.txt b/Documentation/sysctl/user.txt
index 1291c498f78f..a5882865836e 100644
--- a/Documentation/sysctl/user.txt
+++ b/Documentation/sysctl/user.txt
@@ -3,7 +3,7 @@ Documentation for /proc/sys/user/*	kernel version 4.9.0
 
 ==============================================================
 
-This file contains the documetation for the sysctl files in
+This file contains the documentation for the sysctl files in
 /proc/sys/user.
 
 The files in this directory can be used to override the default
diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt
index 5025ff9307e6..ff234d229cbb 100644
--- a/Documentation/sysctl/vm.txt
+++ b/Documentation/sysctl/vm.txt
@@ -30,7 +30,6 @@ Currently, these files are in /proc/sys/vm:
 - dirty_writeback_centisecs
 - drop_caches
 - extfrag_threshold
-- hugepages_treat_as_movable
 - hugetlb_shm_group
 - laptop_mode
 - legacy_va_layout
@@ -261,30 +260,6 @@ any throttling.
 
 ==============================================================
 
-hugepages_treat_as_movable
-
-This parameter controls whether we can allocate hugepages from ZONE_MOVABLE
-or not. If set to non-zero, hugepages can be allocated from ZONE_MOVABLE.
-ZONE_MOVABLE is created when kernel boot parameter kernelcore= is specified,
-so this parameter has no effect if used without kernelcore=.
-
-Hugepage migration is now available in some situations which depend on the
-architecture and/or the hugepage size. If a hugepage supports migration,
-allocation from ZONE_MOVABLE is always enabled for the hugepage regardless
-of the value of this parameter.
-IOW, this parameter affects only non-migratable hugepages.
-
-Assuming that hugepages are not migratable in your system, one usecase of
-this parameter is that users can make hugepage pool more extensible by
-enabling the allocation from ZONE_MOVABLE. This is because on ZONE_MOVABLE
-page reclaim/migration/compaction work more and you can get contiguous
-memory more likely. Note that using ZONE_MOVABLE for non-migratable
-hugepages can do harm to other features like memory hotremove (because
-memory hotremove expects that memory blocks on ZONE_MOVABLE are always
-removable,) so it's a trade-off responsible for the users.
-
-==============================================================
-
 hugetlb_shm_group
 
 hugetlb_shm_group contains group id that is allowed to create SysV
diff --git a/Documentation/thermal/cpu-cooling-api.txt b/Documentation/thermal/cpu-cooling-api.txt
index 71653584cd03..7df567eaea1a 100644
--- a/Documentation/thermal/cpu-cooling-api.txt
+++ b/Documentation/thermal/cpu-cooling-api.txt
@@ -26,39 +26,16 @@ the user. The registration APIs returns the cooling device pointer.
    clip_cpus: cpumask of cpus where the frequency constraints will happen.
 
 1.1.2 struct thermal_cooling_device *of_cpufreq_cooling_register(
-	struct device_node *np, const struct cpumask *clip_cpus)
+					struct cpufreq_policy *policy)
 
     This interface function registers the cpufreq cooling device with
     the name "thermal-cpufreq-%x" linking it with a device tree node, in
     order to bind it via the thermal DT code. This api can support multiple
     instances of cpufreq cooling devices.
 
-    np: pointer to the cooling device device tree node
-    clip_cpus: cpumask of cpus where the frequency constraints will happen.
+    policy: CPUFreq policy.
 
-1.1.3 struct thermal_cooling_device *cpufreq_power_cooling_register(
-    const struct cpumask *clip_cpus, u32 capacitance,
-    get_static_t plat_static_func)
-
-Similar to cpufreq_cooling_register, this function registers a cpufreq
-cooling device.  Using this function, the cooling device will
-implement the power extensions by using a simple cpu power model.  The
-cpus must have registered their OPPs using the OPP library.
-
-The additional parameters are needed for the power model (See 2. Power
-models).  "capacitance" is the dynamic power coefficient (See 2.1
-Dynamic power).  "plat_static_func" is a function to calculate the
-static power consumed by these cpus (See 2.2 Static power).
-
-1.1.4 struct thermal_cooling_device *of_cpufreq_power_cooling_register(
-    struct device_node *np, const struct cpumask *clip_cpus, u32 capacitance,
-    get_static_t plat_static_func)
-
-Similar to cpufreq_power_cooling_register, this function register a
-cpufreq cooling device with power extensions using the device tree
-information supplied by the np parameter.
-
-1.1.5 void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev)
+1.1.3 void cpufreq_cooling_unregister(struct thermal_cooling_device *cdev)
 
     This interface function unregisters the "thermal-cpufreq-%x" cooling device.
 
@@ -67,20 +44,14 @@ information supplied by the np parameter.
 2. Power models
 
 The power API registration functions provide a simple power model for
-CPUs.  The current power is calculated as dynamic + (optionally)
-static power.  This power model requires that the operating-points of
+CPUs.  The current power is calculated as dynamic power (static power isn't
+supported currently).  This power model requires that the operating-points of
 the CPUs are registered using the kernel's opp library and the
 `cpufreq_frequency_table` is assigned to the `struct device` of the
 cpu.  If you are using CONFIG_CPUFREQ_DT then the
 `cpufreq_frequency_table` should already be assigned to the cpu
 device.
 
-The `plat_static_func` parameter of `cpufreq_power_cooling_register()`
-and `of_cpufreq_power_cooling_register()` is optional.  If you don't
-provide it, only dynamic power will be considered.
-
-2.1 Dynamic power
-
 The dynamic power consumption of a processor depends on many factors.
 For a given processor implementation the primary factors are:
 
@@ -119,79 +90,3 @@ mW/MHz/uVolt^2.  Typical values for mobile CPUs might lie in range
 from 100 to 500.  For reference, the approximate values for the SoC in
 ARM's Juno Development Platform are 530 for the Cortex-A57 cluster and
 140 for the Cortex-A53 cluster.
-
-
-2.2 Static power
-
-Static leakage power consumption depends on a number of factors.  For a
-given circuit implementation the primary factors are:
-
-- Time the circuit spends in each 'power state'
-- Temperature
-- Operating voltage
-- Process grade
-
-The time the circuit spends in each 'power state' for a given
-evaluation period at first order means OFF or ON.  However,
-'retention' states can also be supported that reduce power during
-inactive periods without loss of context.
-
-Note: The visibility of state entries to the OS can vary, according to
-platform specifics, and this can then impact the accuracy of a model
-based on OS state information alone.  It might be possible in some
-cases to extract more accurate information from system resources.
-
-The temperature, operating voltage and process 'grade' (slow to fast)
-of the circuit are all significant factors in static leakage power
-consumption.  All of these have complex relationships to static power.
-
-Circuit implementation specific factors include the chosen silicon
-process as well as the type, number and size of transistors in both
-the logic gates and any RAM elements included.
-
-The static power consumption modelling must take into account the
-power managed regions that are implemented.  Taking the example of an
-ARM processor cluster, the modelling would take into account whether
-each CPU can be powered OFF separately or if only a single power
-region is implemented for the complete cluster.
-
-In one view, there are others, a static power consumption model can
-then start from a set of reference values for each power managed
-region (e.g. CPU, Cluster/L2) in each state (e.g. ON, OFF) at an
-arbitrary process grade, voltage and temperature point.  These values
-are then scaled for all of the following: the time in each state, the
-process grade, the current temperature and the operating voltage.
-However, since both implementation specific and complex relationships
-dominate the estimate, the appropriate interface to the model from the
-cpu cooling device is to provide a function callback that calculates
-the static power in this platform.  When registering the cpu cooling
-device pass a function pointer that follows the `get_static_t`
-prototype:
-
-    int plat_get_static(cpumask_t *cpumask, int interval,
-                        unsigned long voltage, u32 &power);
-
-`cpumask` is the cpumask of the cpus involved in the calculation.
-`voltage` is the voltage at which they are operating.  The function
-should calculate the average static power for the last `interval`
-milliseconds.  It returns 0 on success, -E* on error.  If it
-succeeds, it should store the static power in `power`.  Reading the
-temperature of the cpus described by `cpumask` is left for
-plat_get_static() to do as the platform knows best which thermal
-sensor is closest to the cpu.
-
-If `plat_static_func` is NULL, static power is considered to be
-negligible for this platform and only dynamic power is considered.
-
-The platform specific callback can then use any combination of tables
-and/or equations to permute the estimated value.  Process grade
-information is not passed to the model since access to such data, from
-on-chip measurement capability or manufacture time data, is platform
-specific.
-
-Note: the significance of static power for CPUs in comparison to
-dynamic power is highly dependent on implementation.  Given the
-potential complexity in implementation, the importance and accuracy of
-its inclusion when using cpu cooling devices should be assessed on a
-case by case basis.
-
diff --git a/Documentation/trace/ftrace-uses.rst b/Documentation/trace/ftrace-uses.rst
index 8494a801d341..3aed560a12ee 100644
--- a/Documentation/trace/ftrace-uses.rst
+++ b/Documentation/trace/ftrace-uses.rst
@@ -42,9 +42,9 @@ as well as what protections the callback will perform and not require
 ftrace to handle.
 
 There is only one field that is needed to be set when registering
-an ftrace_ops with ftrace::
+an ftrace_ops with ftrace:
 
-.. code-block: c
+.. code-block:: c
 
  struct ftrace_ops ops = {
        .func			= my_callback_func,
@@ -81,12 +81,12 @@ may take some time to finish.
 The callback function
 =====================
 
-The prototype of the callback function is as follows (as of v4.14)::
+The prototype of the callback function is as follows (as of v4.14):
 
-.. code-block: c
+.. code-block:: c
 
- void callback_func(unsigned long ip, unsigned long parent_ip,
-		    struct ftrace_ops *op, struct pt_regs *regs);
+   void callback_func(unsigned long ip, unsigned long parent_ip,
+                      struct ftrace_ops *op, struct pt_regs *regs);
 
 @ip
 	 This is the instruction pointer of the function that is being traced.
@@ -176,10 +176,10 @@ Filtering which functions to trace
 If a callback is only to be called from specific functions, a filter must be
 set up. The filters are added by name, or ip if it is known.
 
-.. code-block: c
+.. code-block:: c
 
- int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
-		       int len, int reset);
+   int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
+                         int len, int reset);
 
 @ops
 	The ops to set the filter with
@@ -202,9 +202,9 @@ See Filter Commands in :file:`Documentation/trace/ftrace.txt`.
 
 To just trace the schedule function::
 
-.. code-block: c
+.. code-block:: c
 
- ret = ftrace_set_filter(&ops, "schedule", strlen("schedule"), 0);
+   ret = ftrace_set_filter(&ops, "schedule", strlen("schedule"), 0);
 
 To add more functions, call the ftrace_set_filter() more than once with the
 @reset parameter set to zero. To remove the current filter set and replace it
@@ -212,17 +212,17 @@ with new functions defined by @buf, have @reset be non-zero.
 
 To remove all the filtered functions and trace all functions::
 
-.. code-block: c
+.. code-block:: c
 
-  ret = ftrace_set_filter(&ops, NULL, 0, 1);
+   ret = ftrace_set_filter(&ops, NULL, 0, 1);
 
 
 Sometimes more than one function has the same name. To trace just a specific
 function in this case, ftrace_set_filter_ip() can be used.
 
-.. code-block: c
+.. code-block:: c
 
- ret = ftrace_set_filter_ip(&ops, ip, 0, 0);
+   ret = ftrace_set_filter_ip(&ops, ip, 0, 0);
 
 Although the ip must be the address where the call to fentry or mcount is
 located in the function. This function is used by perf and kprobes that
@@ -237,10 +237,10 @@ be called by any function.
 An empty "notrace" list means to allow all functions defined by the filter
 to be traced.
 
-.. code-block: c
+.. code-block:: c
 
-  int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
-			 int len, int reset);
+   int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
+                          int len, int reset);
 
 This takes the same parameters as ftrace_set_filter() but will add the
 functions it finds to not be traced. This is a separate list from the
@@ -251,7 +251,7 @@ that match @buf to it.
 
 Clearing the "notrace" list is the same as clearing the filter list
 
-.. code-block: c
+.. code-block:: c
 
   ret = ftrace_set_notrace(&ops, NULL, 0, 1);
 
@@ -264,29 +264,29 @@ If a filter is in place, and the @reset is non-zero, and @buf contains a
 matching glob to functions, the switch will happen during the time of
 the ftrace_set_filter() call. At no time will all functions call the callback.
 
-.. code-block: c
+.. code-block:: c
 
-	ftrace_set_filter(&ops, "schedule", strlen("schedule"), 1);
+   ftrace_set_filter(&ops, "schedule", strlen("schedule"), 1);
 
-	register_ftrace_function(&ops);
+   register_ftrace_function(&ops);
 
-	msleep(10);
+   msleep(10);
 
-	ftrace_set_filter(&ops, "try_to_wake_up", strlen("try_to_wake_up"), 1);
+   ftrace_set_filter(&ops, "try_to_wake_up", strlen("try_to_wake_up"), 1);
 
 is not the same as:
 
-.. code-block: c
+.. code-block:: c
 
-	ftrace_set_filter(&ops, "schedule", strlen("schedule"), 1);
+   ftrace_set_filter(&ops, "schedule", strlen("schedule"), 1);
 
-	register_ftrace_function(&ops);
+   register_ftrace_function(&ops);
 
-	msleep(10);
+   msleep(10);
 
-	ftrace_set_filter(&ops, NULL, 0, 1);
+   ftrace_set_filter(&ops, NULL, 0, 1);
 
-	ftrace_set_filter(&ops, "try_to_wake_up", strlen("try_to_wake_up"), 0);
+   ftrace_set_filter(&ops, "try_to_wake_up", strlen("try_to_wake_up"), 0);
 
 As the latter will have a short time where all functions will call
 the callback, between the time of the reset, and the time of the
diff --git a/Documentation/translations/zh_CN/filesystems/sysfs.txt b/Documentation/translations/zh_CN/filesystems/sysfs.txt
index 7d3b05edb8ce..452271dda141 100644
--- a/Documentation/translations/zh_CN/filesystems/sysfs.txt
+++ b/Documentation/translations/zh_CN/filesystems/sysfs.txt
@@ -167,8 +167,8 @@ static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr,
         if (dev_attr->show)
                 ret = dev_attr->show(dev, dev_attr, buf);
         if (ret >= (ssize_t)PAGE_SIZE) {
-                print_symbol("dev_attr_show: %s returned bad count\n",
-                                (unsigned long)dev_attr->show);
+                printk("dev_attr_show: %pS returned bad count\n",
+                                dev_attr->show);
         }
         return ret;
 }
diff --git a/Documentation/usb/chipidea.txt b/Documentation/usb/chipidea.txt
index edf7cdfddc88..d1eedc01b00a 100644
--- a/Documentation/usb/chipidea.txt
+++ b/Documentation/usb/chipidea.txt
@@ -23,13 +23,13 @@ cat /sys/kernel/debug/ci_hdrc.0/registers
 2) Connect 2 boards with usb cable with one end is micro A plug, the other end
    is micro B plug.
 
-   The A-device(with micro A plug inserted) should enumrate B-device.
+   The A-device(with micro A plug inserted) should enumerate B-device.
 
 3) Role switch
    On B-device:
    echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
-   B-device should take host role and enumrate A-device.
+   B-device should take host role and enumerate A-device.
 
 4) A-device switch back to host.
    On B-device:
@@ -40,13 +40,13 @@ cat /sys/kernel/debug/ci_hdrc.0/registers
    side by answering the polling from B-Host, this can be done on A-device:
    echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_req
 
-   A-device should switch back to host and enumrate B-device.
+   A-device should switch back to host and enumerate B-device.
 
 5) Remove B-device(unplug micro B plug) and insert again in 10 seconds,
-   A-device should enumrate B-device again.
+   A-device should enumerate B-device again.
 
 6) Remove B-device(unplug micro B plug) and insert again after 10 seconds,
-   A-device should NOT enumrate B-device.
+   A-device should NOT enumerate B-device.
 
    if A-device wants to use bus:
    On A-device:
@@ -67,7 +67,7 @@ cat /sys/kernel/debug/ci_hdrc.0/registers
    On B-device:
    echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
-   A-device should resume usb bus and enumrate B-device.
+   A-device should resume usb bus and enumerate B-device.
 
 1.3 Reference document
 ----------------------
diff --git a/Documentation/usb/usbip_protocol.txt b/Documentation/usb/usbip_protocol.txt
index 16b6fe27284c..c7a0f4c7e7f1 100644
--- a/Documentation/usb/usbip_protocol.txt
+++ b/Documentation/usb/usbip_protocol.txt
@@ -274,7 +274,6 @@ USBIP_CMD_SUBMIT: Submit an URB
   URB_SHORT_NOT_OK        | 0x00000001 | only in | only in   | only in  | no
   URB_ISO_ASAP            | 0x00000002 | no      | no        | no       | yes
   URB_NO_TRANSFER_DMA_MAP | 0x00000004 | yes     | yes       | yes      | yes
-  URB_NO_FSBR             | 0x00000020 | yes     | no        | no       | no
   URB_ZERO_PACKET         | 0x00000040 | no      | no        | only out | no
   URB_NO_INTERRUPT        | 0x00000080 | yes     | yes       | yes      | yes
   URB_FREE_BUFFER         | 0x00000100 | yes     | yes       | yes      | yes
diff --git a/Documentation/virtual/kvm/00-INDEX b/Documentation/virtual/kvm/00-INDEX
index 69fe1a8b7ad1..3da73aabff5a 100644
--- a/Documentation/virtual/kvm/00-INDEX
+++ b/Documentation/virtual/kvm/00-INDEX
@@ -26,3 +26,6 @@ s390-diag.txt
 	- Diagnose hypercall description (for IBM S/390)
 timekeeping.txt
 	- timekeeping virtualization for x86-based architectures.
+amd-memory-encryption.txt
+	- notes on AMD Secure Encrypted Virtualization feature and SEV firmware
+	  command description
diff --git a/Documentation/virtual/kvm/amd-memory-encryption.rst b/Documentation/virtual/kvm/amd-memory-encryption.rst
new file mode 100644
index 000000000000..71d6d257074f
--- /dev/null
+++ b/Documentation/virtual/kvm/amd-memory-encryption.rst
@@ -0,0 +1,247 @@
+======================================
+Secure Encrypted Virtualization (SEV)
+======================================
+
+Overview
+========
+
+Secure Encrypted Virtualization (SEV) is a feature found on AMD processors.
+
+SEV is an extension to the AMD-V architecture which supports running
+virtual machines (VMs) under the control of a hypervisor. When enabled,
+the memory contents of a VM will be transparently encrypted with a key
+unique to that VM.
+
+The hypervisor can determine the SEV support through the CPUID
+instruction. The CPUID function 0x8000001f reports information related
+to SEV::
+
+	0x8000001f[eax]:
+			Bit[1] 	indicates support for SEV
+	    ...
+		  [ecx]:
+			Bits[31:0]  Number of encrypted guests supported simultaneously
+
+If support for SEV is present, MSR 0xc001_0010 (MSR_K8_SYSCFG) and MSR 0xc001_0015
+(MSR_K7_HWCR) can be used to determine if it can be enabled::
+
+	0xc001_0010:
+		Bit[23]	   1 = memory encryption can be enabled
+			   0 = memory encryption can not be enabled
+
+	0xc001_0015:
+		Bit[0]	   1 = memory encryption can be enabled
+			   0 = memory encryption can not be enabled
+
+When SEV support is available, it can be enabled in a specific VM by
+setting the SEV bit before executing VMRUN.::
+
+	VMCB[0x90]:
+		Bit[1]	    1 = SEV is enabled
+			    0 = SEV is disabled
+
+SEV hardware uses ASIDs to associate a memory encryption key with a VM.
+Hence, the ASID for the SEV-enabled guests must be from 1 to a maximum value
+defined in the CPUID 0x8000001f[ecx] field.
+
+SEV Key Management
+==================
+
+The SEV guest key management is handled by a separate processor called the AMD
+Secure Processor (AMD-SP). Firmware running inside the AMD-SP provides a secure
+key management interface to perform common hypervisor activities such as
+encrypting bootstrap code, snapshot, migrating and debugging the guest. For more
+information, see the SEV Key Management spec [api-spec]_
+
+KVM implements the following commands to support common lifecycle events of SEV
+guests, such as launching, running, snapshotting, migrating and decommissioning.
+
+1. KVM_SEV_INIT
+---------------
+
+The KVM_SEV_INIT command is used by the hypervisor to initialize the SEV platform
+context. In a typical workflow, this command should be the first command issued.
+
+Returns: 0 on success, -negative on error
+
+2. KVM_SEV_LAUNCH_START
+-----------------------
+
+The KVM_SEV_LAUNCH_START command is used for creating the memory encryption
+context. To create the encryption context, user must provide a guest policy,
+the owner's public Diffie-Hellman (PDH) key and session information.
+
+Parameters: struct  kvm_sev_launch_start (in/out)
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_launch_start {
+                __u32 handle;           /* if zero then firmware creates a new handle */
+                __u32 policy;           /* guest's policy */
+
+                __u64 dh_uaddr;         /* userspace address pointing to the guest owner's PDH key */
+                __u32 dh_len;
+
+                __u64 session_addr;     /* userspace address which points to the guest session information */
+                __u32 session_len;
+        };
+
+On success, the 'handle' field contains a new handle and on error, a negative value.
+
+For more details, see SEV spec Section 6.2.
+
+3. KVM_SEV_LAUNCH_UPDATE_DATA
+-----------------------------
+
+The KVM_SEV_LAUNCH_UPDATE_DATA is used for encrypting a memory region. It also
+calculates a measurement of the memory contents. The measurement is a signature
+of the memory contents that can be sent to the guest owner as an attestation
+that the memory was encrypted correctly by the firmware.
+
+Parameters (in): struct  kvm_sev_launch_update_data
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_launch_update {
+                __u64 uaddr;    /* userspace address to be encrypted (must be 16-byte aligned) */
+                __u32 len;      /* length of the data to be encrypted (must be 16-byte aligned) */
+        };
+
+For more details, see SEV spec Section 6.3.
+
+4. KVM_SEV_LAUNCH_MEASURE
+-------------------------
+
+The KVM_SEV_LAUNCH_MEASURE command is used to retrieve the measurement of the
+data encrypted by the KVM_SEV_LAUNCH_UPDATE_DATA command. The guest owner may
+wait to provide the guest with confidential information until it can verify the
+measurement. Since the guest owner knows the initial contents of the guest at
+boot, the measurement can be verified by comparing it to what the guest owner
+expects.
+
+Parameters (in): struct  kvm_sev_launch_measure
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_launch_measure {
+                __u64 uaddr;    /* where to copy the measurement */
+                __u32 len;      /* length of measurement blob */
+        };
+
+For more details on the measurement verification flow, see SEV spec Section 6.4.
+
+5. KVM_SEV_LAUNCH_FINISH
+------------------------
+
+After completion of the launch flow, the KVM_SEV_LAUNCH_FINISH command can be
+issued to make the guest ready for the execution.
+
+Returns: 0 on success, -negative on error
+
+6. KVM_SEV_GUEST_STATUS
+-----------------------
+
+The KVM_SEV_GUEST_STATUS command is used to retrieve status information about a
+SEV-enabled guest.
+
+Parameters (out): struct kvm_sev_guest_status
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_guest_status {
+                __u32 handle;   /* guest handle */
+                __u32 policy;   /* guest policy */
+                __u8 state;     /* guest state (see enum below) */
+        };
+
+SEV guest state:
+
+::
+
+        enum {
+        SEV_STATE_INVALID = 0;
+        SEV_STATE_LAUNCHING,    /* guest is currently being launched */
+        SEV_STATE_SECRET,       /* guest is being launched and ready to accept the ciphertext data */
+        SEV_STATE_RUNNING,      /* guest is fully launched and running */
+        SEV_STATE_RECEIVING,    /* guest is being migrated in from another SEV machine */
+        SEV_STATE_SENDING       /* guest is getting migrated out to another SEV machine */
+        };
+
+7. KVM_SEV_DBG_DECRYPT
+----------------------
+
+The KVM_SEV_DEBUG_DECRYPT command can be used by the hypervisor to request the
+firmware to decrypt the data at the given memory region.
+
+Parameters (in): struct kvm_sev_dbg
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_dbg {
+                __u64 src_uaddr;        /* userspace address of data to decrypt */
+                __u64 dst_uaddr;        /* userspace address of destination */
+                __u32 len;              /* length of memory region to decrypt */
+        };
+
+The command returns an error if the guest policy does not allow debugging.
+
+8. KVM_SEV_DBG_ENCRYPT
+----------------------
+
+The KVM_SEV_DEBUG_ENCRYPT command can be used by the hypervisor to request the
+firmware to encrypt the data at the given memory region.
+
+Parameters (in): struct kvm_sev_dbg
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_dbg {
+                __u64 src_uaddr;        /* userspace address of data to encrypt */
+                __u64 dst_uaddr;        /* userspace address of destination */
+                __u32 len;              /* length of memory region to encrypt */
+        };
+
+The command returns an error if the guest policy does not allow debugging.
+
+9. KVM_SEV_LAUNCH_SECRET
+------------------------
+
+The KVM_SEV_LAUNCH_SECRET command can be used by the hypervisor to inject secret
+data after the measurement has been validated by the guest owner.
+
+Parameters (in): struct kvm_sev_launch_secret
+
+Returns: 0 on success, -negative on error
+
+::
+
+        struct kvm_sev_launch_secret {
+                __u64 hdr_uaddr;        /* userspace address containing the packet header */
+                __u32 hdr_len;
+
+                __u64 guest_uaddr;      /* the guest memory region where the secret should be injected */
+                __u32 guest_len;
+
+                __u64 trans_uaddr;      /* the hypervisor memory region which contains the secret */
+                __u32 trans_len;
+        };
+
+References
+==========
+
+.. [white-paper] http://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2013/12/AMD_Memory_Encryption_Whitepaper_v7-Public.pdf
+.. [api-spec] http://support.amd.com/TechDocs/55766_SEV-KM%20API_Specification.pdf
+.. [amd-apm] http://support.amd.com/TechDocs/24593.pdf (section 15.34)
+.. [kvm-forum]  http://www.linux-kvm.org/images/7/74/02x08A-Thomas_Lendacky-AMDs_Virtualizatoin_Memory_Encryption_Technology.pdf
diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index fc3ae951bc07..d6b3ff51a14f 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -123,14 +123,15 @@ memory layout to fit in user mode), check KVM_CAP_MIPS_VZ and use the
 flag KVM_VM_MIPS_VZ.
 
 
-4.3 KVM_GET_MSR_INDEX_LIST
+4.3 KVM_GET_MSR_INDEX_LIST, KVM_GET_MSR_FEATURE_INDEX_LIST
 
-Capability: basic
+Capability: basic, KVM_CAP_GET_MSR_FEATURES for KVM_GET_MSR_FEATURE_INDEX_LIST
 Architectures: x86
-Type: system
+Type: system ioctl
 Parameters: struct kvm_msr_list (in/out)
 Returns: 0 on success; -1 on error
 Errors:
+  EFAULT:    the msr index list cannot be read from or written to
   E2BIG:     the msr index list is to be to fit in the array specified by
              the user.
 
@@ -139,16 +140,23 @@ struct kvm_msr_list {
 	__u32 indices[0];
 };
 
-This ioctl returns the guest msrs that are supported.  The list varies
-by kvm version and host processor, but does not change otherwise.  The
-user fills in the size of the indices array in nmsrs, and in return
-kvm adjusts nmsrs to reflect the actual number of msrs and fills in
-the indices array with their numbers.
+The user fills in the size of the indices array in nmsrs, and in return
+kvm adjusts nmsrs to reflect the actual number of msrs and fills in the
+indices array with their numbers.
+
+KVM_GET_MSR_INDEX_LIST returns the guest msrs that are supported.  The list
+varies by kvm version and host processor, but does not change otherwise.
 
 Note: if kvm indicates supports MCE (KVM_CAP_MCE), then the MCE bank MSRs are
 not returned in the MSR list, as different vcpus can have a different number
 of banks, as set via the KVM_X86_SETUP_MCE ioctl.
 
+KVM_GET_MSR_FEATURE_INDEX_LIST returns the list of MSRs that can be passed
+to the KVM_GET_MSRS system ioctl.  This lets userspace probe host capabilities
+and processor features that are exposed via MSRs (e.g., VMX capabilities).
+This list also varies by kvm version and host processor, but does not change
+otherwise.
+
 
 4.4 KVM_CHECK_EXTENSION
 
@@ -475,14 +483,22 @@ Support for this has been removed.  Use KVM_SET_GUEST_DEBUG instead.
 
 4.18 KVM_GET_MSRS
 
-Capability: basic
+Capability: basic (vcpu), KVM_CAP_GET_MSR_FEATURES (system)
 Architectures: x86
-Type: vcpu ioctl
+Type: system ioctl, vcpu ioctl
 Parameters: struct kvm_msrs (in/out)
-Returns: 0 on success, -1 on error
+Returns: number of msrs successfully returned;
+        -1 on error
+
+When used as a system ioctl:
+Reads the values of MSR-based features that are available for the VM.  This
+is similar to KVM_GET_SUPPORTED_CPUID, but it returns MSR indices and values.
+The list of msr-based features can be obtained using KVM_GET_MSR_FEATURE_INDEX_LIST
+in a system ioctl.
 
+When used as a vcpu ioctl:
 Reads model-specific registers from the vcpu.  Supported msr indices can
-be obtained using KVM_GET_MSR_INDEX_LIST.
+be obtained using KVM_GET_MSR_INDEX_LIST in a system ioctl.
 
 struct kvm_msrs {
 	__u32 nmsrs; /* number of msrs in entries */
@@ -1841,6 +1857,7 @@ registers, find a list below:
   PPC	| KVM_REG_PPC_DBSR              | 32
   PPC   | KVM_REG_PPC_TIDR              | 64
   PPC   | KVM_REG_PPC_PSSCR             | 64
+  PPC   | KVM_REG_PPC_DEC_EXPIRY        | 64
   PPC   | KVM_REG_PPC_TM_GPR0           | 64
           ...
   PPC   | KVM_REG_PPC_TM_GPR31          | 64
@@ -3403,7 +3420,7 @@ invalid, if invalid pages are written to (e.g. after the end of memory)
 or if no page table is present for the addresses (e.g. when using
 hugepages).
 
-4.108 KVM_PPC_GET_CPU_CHAR
+4.109 KVM_PPC_GET_CPU_CHAR
 
 Capability: KVM_CAP_PPC_GET_CPU_CHAR
 Architectures: powerpc
@@ -3449,6 +3466,57 @@ array bounds check and the array access.
 These fields use the same bit definitions as the new
 H_GET_CPU_CHARACTERISTICS hypercall.
 
+4.110 KVM_MEMORY_ENCRYPT_OP
+
+Capability: basic
+Architectures: x86
+Type: system
+Parameters: an opaque platform specific structure (in/out)
+Returns: 0 on success; -1 on error
+
+If the platform supports creating encrypted VMs then this ioctl can be used
+for issuing platform-specific memory encryption commands to manage those
+encrypted VMs.
+
+Currently, this ioctl is used for issuing Secure Encrypted Virtualization
+(SEV) commands on AMD Processors. The SEV commands are defined in
+Documentation/virtual/kvm/amd-memory-encryption.txt.
+
+4.111 KVM_MEMORY_ENCRYPT_REG_REGION
+
+Capability: basic
+Architectures: x86
+Type: system
+Parameters: struct kvm_enc_region (in)
+Returns: 0 on success; -1 on error
+
+This ioctl can be used to register a guest memory region which may
+contain encrypted data (e.g. guest RAM, SMRAM etc).
+
+It is used in the SEV-enabled guest. When encryption is enabled, a guest
+memory region may contain encrypted data. The SEV memory encryption
+engine uses a tweak such that two identical plaintext pages, each at
+different locations will have differing ciphertexts. So swapping or
+moving ciphertext of those pages will not result in plaintext being
+swapped. So relocating (or migrating) physical backing pages for the SEV
+guest will require some additional steps.
+
+Note: The current SEV key management spec does not provide commands to
+swap or migrate (move) ciphertext pages. Hence, for now we pin the guest
+memory region registered with the ioctl.
+
+4.112 KVM_MEMORY_ENCRYPT_UNREG_REGION
+
+Capability: basic
+Architectures: x86
+Type: system
+Parameters: struct kvm_enc_region (in)
+Returns: 0 on success; -1 on error
+
+This ioctl can be used to unregister the guest memory region registered
+with KVM_MEMORY_ENCRYPT_REG_REGION ioctl above.
+
+
 5. The kvm_run structure
 ------------------------
 
diff --git a/Documentation/virtual/kvm/arm/vgic-mapped-irqs.txt b/Documentation/virtual/kvm/arm/vgic-mapped-irqs.txt
deleted file mode 100644
index 38bca2835278..000000000000
--- a/Documentation/virtual/kvm/arm/vgic-mapped-irqs.txt
+++ /dev/null
@@ -1,187 +0,0 @@
-KVM/ARM VGIC Forwarded Physical Interrupts
-==========================================
-
-The KVM/ARM code implements software support for the ARM Generic
-Interrupt Controller's (GIC's) hardware support for virtualization by
-allowing software to inject virtual interrupts to a VM, which the guest
-OS sees as regular interrupts.  The code is famously known as the VGIC.
-
-Some of these virtual interrupts, however, correspond to physical
-interrupts from real physical devices.  One example could be the
-architected timer, which itself supports virtualization, and therefore
-lets a guest OS program the hardware device directly to raise an
-interrupt at some point in time.  When such an interrupt is raised, the
-host OS initially handles the interrupt and must somehow signal this
-event as a virtual interrupt to the guest.  Another example could be a
-passthrough device, where the physical interrupts are initially handled
-by the host, but the device driver for the device lives in the guest OS
-and KVM must therefore somehow inject a virtual interrupt on behalf of
-the physical one to the guest OS.
-
-These virtual interrupts corresponding to a physical interrupt on the
-host are called forwarded physical interrupts, but are also sometimes
-referred to as 'virtualized physical interrupts' and 'mapped interrupts'.
-
-Forwarded physical interrupts are handled slightly differently compared
-to virtual interrupts generated purely by a software emulated device.
-
-
-The HW bit
-----------
-Virtual interrupts are signalled to the guest by programming the List
-Registers (LRs) on the GIC before running a VCPU.  The LR is programmed
-with the virtual IRQ number and the state of the interrupt (Pending,
-Active, or Pending+Active).  When the guest ACKs and EOIs a virtual
-interrupt, the LR state moves from Pending to Active, and finally to
-inactive.
-
-The LRs include an extra bit, called the HW bit.  When this bit is set,
-KVM must also program an additional field in the LR, the physical IRQ
-number, to link the virtual with the physical IRQ.
-
-When the HW bit is set, KVM must EITHER set the Pending OR the Active
-bit, never both at the same time.
-
-Setting the HW bit causes the hardware to deactivate the physical
-interrupt on the physical distributor when the guest deactivates the
-corresponding virtual interrupt.
-
-
-Forwarded Physical Interrupts Life Cycle
-----------------------------------------
-
-The state of forwarded physical interrupts is managed in the following way:
-
-  - The physical interrupt is acked by the host, and becomes active on
-    the physical distributor (*).
-  - KVM sets the LR.Pending bit, because this is the only way the GICV
-    interface is going to present it to the guest.
-  - LR.Pending will stay set as long as the guest has not acked the interrupt.
-  - LR.Pending transitions to LR.Active on the guest read of the IAR, as
-    expected.
-  - On guest EOI, the *physical distributor* active bit gets cleared,
-    but the LR.Active is left untouched (set).
-  - KVM clears the LR on VM exits when the physical distributor
-    active state has been cleared.
-
-(*): The host handling is slightly more complicated.  For some forwarded
-interrupts (shared), KVM directly sets the active state on the physical
-distributor before entering the guest, because the interrupt is never actually
-handled on the host (see details on the timer as an example below).  For other
-forwarded interrupts (non-shared) the host does not deactivate the interrupt
-when the host ISR completes, but leaves the interrupt active until the guest
-deactivates it.  Leaving the interrupt active is allowed, because Linux
-configures the physical GIC with EOIMode=1, which causes EOI operations to
-perform a priority drop allowing the GIC to receive other interrupts of the
-default priority.
-
-
-Forwarded Edge and Level Triggered PPIs and SPIs
-------------------------------------------------
-Forwarded physical interrupts injected should always be active on the
-physical distributor when injected to a guest.
-
-Level-triggered interrupts will keep the interrupt line to the GIC
-asserted, typically until the guest programs the device to deassert the
-line.  This means that the interrupt will remain pending on the physical
-distributor until the guest has reprogrammed the device.  Since we
-always run the VM with interrupts enabled on the CPU, a pending
-interrupt will exit the guest as soon as we switch into the guest,
-preventing the guest from ever making progress as the process repeats
-over and over.  Therefore, the active state on the physical distributor
-must be set when entering the guest, preventing the GIC from forwarding
-the pending interrupt to the CPU.  As soon as the guest deactivates the
-interrupt, the physical line is sampled by the hardware again and the host
-takes a new interrupt if and only if the physical line is still asserted.
-
-Edge-triggered interrupts do not exhibit the same problem with
-preventing guest execution that level-triggered interrupts do.  One
-option is to not use HW bit at all, and inject edge-triggered interrupts
-from a physical device as pure virtual interrupts.  But that would
-potentially slow down handling of the interrupt in the guest, because a
-physical interrupt occurring in the middle of the guest ISR would
-preempt the guest for the host to handle the interrupt.  Additionally,
-if you configure the system to handle interrupts on a separate physical
-core from that running your VCPU, you still have to interrupt the VCPU
-to queue the pending state onto the LR, even though the guest won't use
-this information until the guest ISR completes.  Therefore, the HW
-bit should always be set for forwarded edge-triggered interrupts.  With
-the HW bit set, the virtual interrupt is injected and additional
-physical interrupts occurring before the guest deactivates the interrupt
-simply mark the state on the physical distributor as Pending+Active.  As
-soon as the guest deactivates the interrupt, the host takes another
-interrupt if and only if there was a physical interrupt between injecting
-the forwarded interrupt to the guest and the guest deactivating the
-interrupt.
-
-Consequently, whenever we schedule a VCPU with one or more LRs with the
-HW bit set, the interrupt must also be active on the physical
-distributor.
-
-
-Forwarded LPIs
---------------
-LPIs, introduced in GICv3, are always edge-triggered and do not have an
-active state.  They become pending when a device signal them, and as
-soon as they are acked by the CPU, they are inactive again.
-
-It therefore doesn't make sense, and is not supported, to set the HW bit
-for physical LPIs that are forwarded to a VM as virtual interrupts,
-typically virtual SPIs.
-
-For LPIs, there is no other choice than to preempt the VCPU thread if
-necessary, and queue the pending state onto the LR.
-
-
-Putting It Together: The Architected Timer
-------------------------------------------
-The architected timer is a device that signals interrupts with level
-triggered semantics.  The timer hardware is directly accessed by VCPUs
-which program the timer to fire at some point in time.  Each VCPU on a
-system programs the timer to fire at different times, and therefore the
-hardware is multiplexed between multiple VCPUs.  This is implemented by
-context-switching the timer state along with each VCPU thread.
-
-However, this means that a scenario like the following is entirely
-possible, and in fact, typical:
-
-1.  KVM runs the VCPU
-2.  The guest programs the time to fire in T+100
-3.  The guest is idle and calls WFI (wait-for-interrupts)
-4.  The hardware traps to the host
-5.  KVM stores the timer state to memory and disables the hardware timer
-6.  KVM schedules a soft timer to fire in T+(100 - time since step 2)
-7.  KVM puts the VCPU thread to sleep (on a waitqueue)
-8.  The soft timer fires, waking up the VCPU thread
-9.  KVM reprograms the timer hardware with the VCPU's values
-10. KVM marks the timer interrupt as active on the physical distributor
-11. KVM injects a forwarded physical interrupt to the guest
-12. KVM runs the VCPU
-
-Notice that KVM injects a forwarded physical interrupt in step 11 without
-the corresponding interrupt having actually fired on the host.  That is
-exactly why we mark the timer interrupt as active in step 10, because
-the active state on the physical distributor is part of the state
-belonging to the timer hardware, which is context-switched along with
-the VCPU thread.
-
-If the guest does not idle because it is busy, the flow looks like this
-instead:
-
-1.  KVM runs the VCPU
-2.  The guest programs the time to fire in T+100
-4.  At T+100 the timer fires and a physical IRQ causes the VM to exit
-    (note that this initially only traps to EL2 and does not run the host ISR
-    until KVM has returned to the host).
-5.  With interrupts still disabled on the CPU coming back from the guest, KVM
-    stores the virtual timer state to memory and disables the virtual hw timer.
-6.  KVM looks at the timer state (in memory) and injects a forwarded physical
-    interrupt because it concludes the timer has expired.
-7.  KVM marks the timer interrupt as active on the physical distributor
-7.  KVM enables the timer, enables interrupts, and runs the VCPU
-
-Notice that again the forwarded physical interrupt is injected to the
-guest without having actually been handled on the host.  In this case it
-is because the physical interrupt is never actually seen by the host because the
-timer is disabled upon guest return, and the virtual forwarded interrupt is
-injected on the KVM guest entry path.
diff --git a/Documentation/virtual/kvm/cpuid.txt b/Documentation/virtual/kvm/cpuid.txt
index 3c65feb83010..87a7506f31c2 100644
--- a/Documentation/virtual/kvm/cpuid.txt
+++ b/Documentation/virtual/kvm/cpuid.txt
@@ -54,6 +54,14 @@ KVM_FEATURE_PV_UNHALT              ||     7 || guest checks this feature bit
                                    ||       || before enabling paravirtualized
                                    ||       || spinlock support.
 ------------------------------------------------------------------------------
+KVM_FEATURE_PV_TLB_FLUSH           ||     9 || guest checks this feature bit
+                                   ||       || before enabling paravirtualized
+                                   ||       || tlb flush.
+------------------------------------------------------------------------------
+KVM_FEATURE_ASYNC_PF_VMEXIT        ||    10 || paravirtualized async PF VM exit
+                                   ||       || can be enabled by setting bit 2
+                                   ||       || when writing to msr 0x4b564d02
+------------------------------------------------------------------------------
 KVM_FEATURE_CLOCKSOURCE_STABLE_BIT ||    24 || host will warn if no guest-side
                                    ||       || per-cpu warps are expected in
                                    ||       || kvmclock.
diff --git a/Documentation/virtual/kvm/msr.txt b/Documentation/virtual/kvm/msr.txt
index 1ebecc115dc6..f3f0d57ced8e 100644
--- a/Documentation/virtual/kvm/msr.txt
+++ b/Documentation/virtual/kvm/msr.txt
@@ -170,7 +170,8 @@ MSR_KVM_ASYNC_PF_EN: 0x4b564d02
 	when asynchronous page faults are enabled on the vcpu 0 when
 	disabled. Bit 1 is 1 if asynchronous page faults can be injected
 	when vcpu is in cpl == 0. Bit 2 is 1 if asynchronous page faults
-	are delivered to L1 as #PF vmexits.
+	are delivered to L1 as #PF vmexits.  Bit 2 can be set only if
+	KVM_FEATURE_ASYNC_PF_VMEXIT is present in CPUID.
 
 	First 4 byte of 64 byte memory location will be written to by
 	the hypervisor at the time of asynchronous page fault (APF)
diff --git a/Documentation/vm/hugetlbpage.txt b/Documentation/vm/hugetlbpage.txt
index 59cbc803aad6..faf077d50d42 100644
--- a/Documentation/vm/hugetlbpage.txt
+++ b/Documentation/vm/hugetlbpage.txt
@@ -20,19 +20,20 @@ options.
 
 The /proc/meminfo file provides information about the total number of
 persistent hugetlb pages in the kernel's huge page pool.  It also displays
-information about the number of free, reserved and surplus huge pages and the
-default huge page size.  The huge page size is needed for generating the
-proper alignment and size of the arguments to system calls that map huge page
-regions.
+default huge page size and information about the number of free, reserved
+and surplus huge pages in the pool of huge pages of default size.
+The huge page size is needed for generating the proper alignment and
+size of the arguments to system calls that map huge page regions.
 
 The output of "cat /proc/meminfo" will include lines like:
 
 .....
-HugePages_Total: vvv
-HugePages_Free:  www
-HugePages_Rsvd:  xxx
-HugePages_Surp:  yyy
-Hugepagesize:    zzz kB
+HugePages_Total: uuu
+HugePages_Free:  vvv
+HugePages_Rsvd:  www
+HugePages_Surp:  xxx
+Hugepagesize:    yyy kB
+Hugetlb:         zzz kB
 
 where:
 HugePages_Total is the size of the pool of huge pages.
@@ -47,6 +48,14 @@ HugePages_Surp  is short for "surplus," and is the number of huge pages in
                 the pool above the value in /proc/sys/vm/nr_hugepages. The
                 maximum number of surplus huge pages is controlled by
                 /proc/sys/vm/nr_overcommit_hugepages.
+Hugepagesize    is the default hugepage size (in Kb).
+Hugetlb         is the total amount of memory (in kB), consumed by huge
+                pages of all sizes.
+                If huge pages of different sizes are in use, this number
+                will exceed HugePages_Total * Hugepagesize. To get more
+                detailed information, please, refer to
+                /sys/kernel/mm/hugepages (described below).
+
 
 /proc/filesystems should also show a filesystem of type "hugetlbfs" configured
 in the kernel.
diff --git a/Documentation/vm/hwpoison.txt b/Documentation/vm/hwpoison.txt
index 6ae89a9edf2a..e912d7eee769 100644
--- a/Documentation/vm/hwpoison.txt
+++ b/Documentation/vm/hwpoison.txt
@@ -104,7 +104,7 @@ madvise(MADV_HWPOISON, ....)
 
 hwpoison-inject module through debugfs
 
-/sys/debug/hwpoison/
+/sys/kernel/debug/hwpoison/
 
 corrupt-pfn
 
diff --git a/Documentation/w1/masters/w1-gpio b/Documentation/w1/masters/w1-gpio
index af5d3b4aa851..623961d9e83f 100644
--- a/Documentation/w1/masters/w1-gpio
+++ b/Documentation/w1/masters/w1-gpio
@@ -8,17 +8,27 @@ Description
 -----------
 
 GPIO 1-wire bus master driver. The driver uses the GPIO API to control the
-wire and the GPIO pin can be specified using platform data.
+wire and the GPIO pin can be specified using GPIO machine descriptor tables.
+It is also possible to define the master using device tree, see
+Documentation/devicetree/bindings/w1/w1-gpio.txt
 
 
 Example (mach-at91)
 -------------------
 
+#include <linux/gpio/machine.h>
 #include <linux/w1-gpio.h>
 
+static struct gpiod_lookup_table foo_w1_gpiod_table = {
+	.dev_id = "w1-gpio",
+	.table = {
+		GPIO_LOOKUP_IDX("at91-gpio", AT91_PIN_PB20, NULL, 0,
+			GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN),
+	},
+};
+
 static struct w1_gpio_platform_data foo_w1_gpio_pdata = {
-	.pin		= AT91_PIN_PB20,
-	.is_open_drain	= 1,
+	.ext_pullup_enable_pin	= -EINVAL,
 };
 
 static struct platform_device foo_w1_device = {
@@ -30,4 +40,5 @@ static struct platform_device foo_w1_device = {
 ...
 	at91_set_GPIO_periph(foo_w1_gpio_pdata.pin, 1);
 	at91_set_multi_drive(foo_w1_gpio_pdata.pin, 1);
+	gpiod_add_lookup_table(&foo_w1_gpiod_table);
 	platform_device_register(&foo_w1_device);
diff --git a/Documentation/w1/w1.generic b/Documentation/w1/w1.generic
index b3ffaf8cfab2..c51b1ab012d0 100644
--- a/Documentation/w1/w1.generic
+++ b/Documentation/w1/w1.generic
@@ -76,7 +76,7 @@ See struct w1_bus_master definition in w1.h for details.
 
 w1 master sysfs interface
 ------------------------------------------------------------------
-<xx-xxxxxxxxxxxxx> - A directory for a found device. The format is family-serial
+<xx-xxxxxxxxxxxx>  - A directory for a found device. The format is family-serial
 bus                - (standard) symlink to the w1 bus
 driver             - (standard) symlink to the w1 driver
 w1_master_add      - (rw) manually register a slave device
diff --git a/Documentation/watchdog/watchdog-parameters.txt b/Documentation/watchdog/watchdog-parameters.txt
index 6f9d7b418917..beea975980f6 100644
--- a/Documentation/watchdog/watchdog-parameters.txt
+++ b/Documentation/watchdog/watchdog-parameters.txt
@@ -40,11 +40,6 @@ margin: Watchdog margin in seconds (default=60)
 nowayout: Disable watchdog shutdown on close
 	(default=kernel config parameter)
 -------------------------------------------------
-at32ap700x_wdt:
-timeout: Timeout value. Limited to be 1 or 2 seconds. (default=2)
-nowayout: Watchdog cannot be stopped once started
-	(default=kernel config parameter)
--------------------------------------------------
 at91rm9200_wdt:
 wdt_time: Watchdog time in seconds. (default=5)
 nowayout: Watchdog cannot be stopped once started
@@ -162,11 +157,6 @@ testmode: Watchdog test mode (1 = no reboot), default=0
 nowayout: Watchdog cannot be stopped once started
 	(default=kernel config parameter)
 -------------------------------------------------
-ixp2000_wdt:
-heartbeat: Watchdog heartbeat in seconds (default 60s)
-nowayout: Watchdog cannot be stopped once started
-	(default=kernel config parameter)
--------------------------------------------------
 ixp4xx_wdt:
 heartbeat: Watchdog heartbeat in seconds (default 60s)
 nowayout: Watchdog cannot be stopped once started
@@ -381,19 +371,6 @@ timeout: Watchdog timeout in seconds. 1 <= timeout <= 255, default=60.
 nowayout: Watchdog cannot be stopped once started
 	(default=kernel config parameter)
 -------------------------------------------------
-w83697hf_wdt:
-wdt_io: w83697hf/hg WDT io port (default 0x2e, 0 = autodetect)
-timeout: Watchdog timeout in seconds. 1<= timeout <=255 (default=60)
-nowayout: Watchdog cannot be stopped once started
-	(default=kernel config parameter)
-early_disable: Watchdog gets disabled at boot time (default=1)
--------------------------------------------------
-w83697ug_wdt:
-wdt_io: w83697ug/uf WDT io port (default 0x2e)
-timeout: Watchdog timeout in seconds. 1<= timeout <=255 (default=60)
-nowayout: Watchdog cannot be stopped once started
-	(default=kernel config parameter)
--------------------------------------------------
 w83877f_wdt:
 timeout: Watchdog timeout in seconds. (1<=timeout<=3600, default=30)
 nowayout: Watchdog cannot be stopped once started
diff --git a/Documentation/x86/intel_rdt_ui.txt b/Documentation/x86/intel_rdt_ui.txt
index 6851854cf69d..71c30984e94d 100644
--- a/Documentation/x86/intel_rdt_ui.txt
+++ b/Documentation/x86/intel_rdt_ui.txt
@@ -7,15 +7,24 @@ Tony Luck <tony.luck@intel.com>
 Vikas Shivappa <vikas.shivappa@intel.com>
 
 This feature is enabled by the CONFIG_INTEL_RDT Kconfig and the
-X86 /proc/cpuinfo flag bits "rdt", "cqm", "cat_l3" and "cdp_l3".
+X86 /proc/cpuinfo flag bits:
+RDT (Resource Director Technology) Allocation - "rdt_a"
+CAT (Cache Allocation Technology) - "cat_l3", "cat_l2"
+CDP (Code and Data Prioritization ) - "cdp_l3", "cdp_l2"
+CQM (Cache QoS Monitoring) - "cqm_llc", "cqm_occup_llc"
+MBM (Memory Bandwidth Monitoring) - "cqm_mbm_total", "cqm_mbm_local"
+MBA (Memory Bandwidth Allocation) - "mba"
 
 To use the feature mount the file system:
 
- # mount -t resctrl resctrl [-o cdp] /sys/fs/resctrl
+ # mount -t resctrl resctrl [-o cdp[,cdpl2]] /sys/fs/resctrl
 
 mount options are:
 
 "cdp": Enable code/data prioritization in L3 cache allocations.
+"cdpl2": Enable code/data prioritization in L2 cache allocations.
+
+L2 and L3 CDP are controlled seperately.
 
 RDT features are orthogonal. A particular system may support only
 monitoring, only control, or both monitoring and control.
@@ -662,7 +671,7 @@ occupancy of the real time threads on these cores.
 # mkdir p1
 
 Move the cpus 4-7 over to p1
-# echo f0 > p0/cpus
+# echo f0 > p1/cpus
 
 View the llc occupancy snapshot
 
diff --git a/Documentation/x86/microcode.txt b/Documentation/x86/microcode.txt
index f57e1b45e628..79fdb4a8148a 100644
--- a/Documentation/x86/microcode.txt
+++ b/Documentation/x86/microcode.txt
@@ -108,12 +108,11 @@ packages already put them there.
 ====================
 
 The loader supports also loading of a builtin microcode supplied through
-the regular firmware builtin method CONFIG_FIRMWARE_IN_KERNEL. Only
-64-bit is currently supported.
+the regular builtin firmware method CONFIG_EXTRA_FIRMWARE. Only 64-bit is
+currently supported.
 
 Here's an example:
 
-CONFIG_FIRMWARE_IN_KERNEL=y
 CONFIG_EXTRA_FIRMWARE="intel-ucode/06-3a-09 amd-ucode/microcode_amd_fam15h.bin"
 CONFIG_EXTRA_FIRMWARE_DIR="/lib/firmware"
 
diff --git a/Documentation/x86/topology.txt b/Documentation/x86/topology.txt
index f3e9d7e9ed6c..2953e3ec9a02 100644
--- a/Documentation/x86/topology.txt
+++ b/Documentation/x86/topology.txt
@@ -108,7 +108,7 @@ The topology of a system is described in the units of:
 
     The number of online threads is also printed in /proc/cpuinfo "siblings."
 
-  - topology_sibling_mask():
+  - topology_sibling_cpumask():
 
     The cpumask contains all online threads in the core to which a thread
     belongs.
diff --git a/Documentation/xtensa/mmu.txt b/Documentation/xtensa/mmu.txt
index 5de8715d5bec..318114de63f3 100644
--- a/Documentation/xtensa/mmu.txt
+++ b/Documentation/xtensa/mmu.txt
@@ -69,19 +69,10 @@ Default MMUv2-compatible layout.
 | Userspace        |                           0x00000000  TASK_SIZE
 +------------------+                           0x40000000
 +------------------+
-| Page table       |                           0x80000000
-+------------------+                           0x80400000
+| Page table       |  XCHAL_PAGE_TABLE_VADDR   0x80000000  XCHAL_PAGE_TABLE_SIZE
 +------------------+
-| KMAP area        |  PKMAP_BASE                           PTRS_PER_PTE *
-|                  |                                       DCACHE_N_COLORS *
-|                  |                                       PAGE_SIZE
-|                  |                                       (4MB * DCACHE_N_COLORS)
-+------------------+
-| Atomic KMAP area |  FIXADDR_START                        KM_TYPE_NR *
-|                  |                                       NR_CPUS *
-|                  |                                       DCACHE_N_COLORS *
-|                  |                                       PAGE_SIZE
-+------------------+  FIXADDR_TOP              0xbffff000
+| KASAN shadow map |  KASAN_SHADOW_START       0x80400000  KASAN_SHADOW_SIZE
++------------------+                           0x8e400000
 +------------------+
 | VMALLOC area     |  VMALLOC_START            0xc0000000  128MB - 64KB
 +------------------+  VMALLOC_END
@@ -92,6 +83,17 @@ Default MMUv2-compatible layout.
 | remap area 2     |
 +------------------+
 +------------------+
+| KMAP area        |  PKMAP_BASE                           PTRS_PER_PTE *
+|                  |                                       DCACHE_N_COLORS *
+|                  |                                       PAGE_SIZE
+|                  |                                       (4MB * DCACHE_N_COLORS)
++------------------+
+| Atomic KMAP area |  FIXADDR_START                        KM_TYPE_NR *
+|                  |                                       NR_CPUS *
+|                  |                                       DCACHE_N_COLORS *
+|                  |                                       PAGE_SIZE
++------------------+  FIXADDR_TOP              0xcffff000
++------------------+
 | Cached KSEG      |  XCHAL_KSEG_CACHED_VADDR  0xd0000000  128MB
 +------------------+
 | Uncached KSEG    |  XCHAL_KSEG_BYPASS_VADDR  0xd8000000  128MB
@@ -109,19 +111,10 @@ Default MMUv2-compatible layout.
 | Userspace        |                           0x00000000  TASK_SIZE
 +------------------+                           0x40000000
 +------------------+
-| Page table       |                           0x80000000
-+------------------+                           0x80400000
+| Page table       |  XCHAL_PAGE_TABLE_VADDR   0x80000000  XCHAL_PAGE_TABLE_SIZE
 +------------------+
-| KMAP area        |  PKMAP_BASE                           PTRS_PER_PTE *
-|                  |                                       DCACHE_N_COLORS *
-|                  |                                       PAGE_SIZE
-|                  |                                       (4MB * DCACHE_N_COLORS)
-+------------------+
-| Atomic KMAP area |  FIXADDR_START                        KM_TYPE_NR *
-|                  |                                       NR_CPUS *
-|                  |                                       DCACHE_N_COLORS *
-|                  |                                       PAGE_SIZE
-+------------------+  FIXADDR_TOP              0x9ffff000
+| KASAN shadow map |  KASAN_SHADOW_START       0x80400000  KASAN_SHADOW_SIZE
++------------------+                           0x8e400000
 +------------------+
 | VMALLOC area     |  VMALLOC_START            0xa0000000  128MB - 64KB
 +------------------+  VMALLOC_END
@@ -132,6 +125,17 @@ Default MMUv2-compatible layout.
 | remap area 2     |
 +------------------+
 +------------------+
+| KMAP area        |  PKMAP_BASE                           PTRS_PER_PTE *
+|                  |                                       DCACHE_N_COLORS *
+|                  |                                       PAGE_SIZE
+|                  |                                       (4MB * DCACHE_N_COLORS)
++------------------+
+| Atomic KMAP area |  FIXADDR_START                        KM_TYPE_NR *
+|                  |                                       NR_CPUS *
+|                  |                                       DCACHE_N_COLORS *
+|                  |                                       PAGE_SIZE
++------------------+  FIXADDR_TOP              0xaffff000
++------------------+
 | Cached KSEG      |  XCHAL_KSEG_CACHED_VADDR  0xb0000000  256MB
 +------------------+
 | Uncached KSEG    |  XCHAL_KSEG_BYPASS_VADDR  0xc0000000  256MB
@@ -150,19 +154,10 @@ Default MMUv2-compatible layout.
 | Userspace        |                           0x00000000  TASK_SIZE
 +------------------+                           0x40000000
 +------------------+
-| Page table       |                           0x80000000
-+------------------+                           0x80400000
+| Page table       |  XCHAL_PAGE_TABLE_VADDR   0x80000000  XCHAL_PAGE_TABLE_SIZE
 +------------------+
-| KMAP area        |  PKMAP_BASE                           PTRS_PER_PTE *
-|                  |                                       DCACHE_N_COLORS *
-|                  |                                       PAGE_SIZE
-|                  |                                       (4MB * DCACHE_N_COLORS)
-+------------------+
-| Atomic KMAP area |  FIXADDR_START                        KM_TYPE_NR *
-|                  |                                       NR_CPUS *
-|                  |                                       DCACHE_N_COLORS *
-|                  |                                       PAGE_SIZE
-+------------------+  FIXADDR_TOP              0x8ffff000
+| KASAN shadow map |  KASAN_SHADOW_START       0x80400000  KASAN_SHADOW_SIZE
++------------------+                           0x8e400000
 +------------------+
 | VMALLOC area     |  VMALLOC_START            0x90000000  128MB - 64KB
 +------------------+  VMALLOC_END
@@ -173,6 +168,17 @@ Default MMUv2-compatible layout.
 | remap area 2     |
 +------------------+
 +------------------+
+| KMAP area        |  PKMAP_BASE                           PTRS_PER_PTE *
+|                  |                                       DCACHE_N_COLORS *
+|                  |                                       PAGE_SIZE
+|                  |                                       (4MB * DCACHE_N_COLORS)
++------------------+
+| Atomic KMAP area |  FIXADDR_START                        KM_TYPE_NR *
+|                  |                                       NR_CPUS *
+|                  |                                       DCACHE_N_COLORS *
+|                  |                                       PAGE_SIZE
++------------------+  FIXADDR_TOP              0x9ffff000
++------------------+
 | Cached KSEG      |  XCHAL_KSEG_CACHED_VADDR  0xa0000000  512MB
 +------------------+
 | Uncached KSEG    |  XCHAL_KSEG_BYPASS_VADDR  0xc0000000  512MB
diff --git a/LICENSES/exceptions/Linux-syscall-note b/LICENSES/exceptions/Linux-syscall-note
new file mode 100644
index 000000000000..6b60b61be4e9
--- /dev/null
+++ b/LICENSES/exceptions/Linux-syscall-note
@@ -0,0 +1,25 @@
+SPDX-Exception-Identifier: Linux-syscall-note
+SPDX-URL: https://spdx.org/licenses/Linux-syscall-note.html
+SPDX-Licenses: GPL-2.0, GPL-2.0+, GPL-1.0+, LGPL-2.0, LGPL-2.0+, LGPL-2.1, LGPL-2.1+
+Usage-Guide:
+  This exception is used together with one of the above SPDX-Licenses
+  to mark user space API (uapi) header files so they can be included
+  into non GPL compliant user space application code.
+  To use this exception add it with the keyword WITH to one of the
+  identifiers in the SPDX-Licenses tag:
+    SPDX-License-Identifier: <SPDX-License> WITH Linux-syscall-note
+License-Text:
+
+   NOTE! This copyright does *not* cover user programs that use kernel
+ services by normal system calls - this is merely considered normal use
+ of the kernel, and does *not* fall under the heading of "derived work".
+ Also note that the GPL below is copyrighted by the Free Software
+ Foundation, but the instance of code that it refers to (the Linux
+ kernel) is copyrighted by me and others who actually wrote it.
+
+ Also note that the only valid version of the GPL as far as the kernel
+ is concerned is _this_ particular version of the license (ie v2, not
+ v2.2 or v3.x or whatever), unless explicitly otherwise stated.
+
+			Linus Torvalds
+
diff --git a/LICENSES/other/GPL-1.0 b/LICENSES/other/GPL-1.0
new file mode 100644
index 000000000000..3a4fa969e4c2
--- /dev/null
+++ b/LICENSES/other/GPL-1.0
@@ -0,0 +1,260 @@
+Valid-License-Identifier: GPL-1.0+
+SPDX-URL: https://spdx.org/licenses/GPL-1.0.html
+Usage-Guide:
+  The GNU General Public License (GPL) version 1 should not be used in new
+  code. For existing kernel code the 'or any later version' option is
+  required to be compatible with the general license of the project: GPLv2.
+  To use the license in source code, put the following SPDX tag/value pair
+  into a comment according to the placement guidelines in the licensing
+  rules documentation:
+    SPDX-License-Identifier: GPL-1.0+
+License-Text:
+
+	    GNU GENERAL PUBLIC LICENSE
+	     Version 1, February 1989
+
+ Copyright (C) 1989 Free Software Foundation, Inc.
+                    675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The license agreements of most software companies try to keep users
+at the mercy of those companies.  By contrast, our General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  The
+General Public License applies to the Free Software Foundation's
+software and to any other program whose authors commit to using it.
+You can use it for your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Specifically, the General Public License is designed to make
+sure that you have the freedom to give away or sell copies of free
+software, that you receive source code or can get it if you want it,
+that you can change the software or use pieces of it in new free
+programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of a such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must tell them their rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any program or other work which
+contains a notice placed by the copyright holder saying it may be
+distributed under the terms of this General Public License.  The
+"Program", below, refers to any such program or work, and a "work based
+on the Program" means either the Program or any work containing the
+Program or a portion of it, either verbatim or with modifications.  Each
+licensee is addressed as "you".
+
+  1. You may copy and distribute verbatim copies of the Program's source
+code as you receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice and
+disclaimer of warranty; keep intact all the notices that refer to this
+General Public License and to the absence of any warranty; and give any
+other recipients of the Program a copy of this General Public License
+along with the Program.  You may charge a fee for the physical act of
+transferring a copy.
+
+  2. You may modify your copy or copies of the Program or any portion of
+it, and copy and distribute such modifications under the terms of Paragraph
+1 above, provided that you also do the following:
+
+    a) cause the modified files to carry prominent notices stating that
+    you changed the files and the date of any change; and
+
+    b) cause the whole of any work that you distribute or publish, that
+    in whole or in part contains the Program or any part thereof, either
+    with or without modifications, to be licensed at no charge to all
+    third parties under the terms of this General Public License (except
+    that you may choose to grant warranty protection to some or all
+    third parties, at your option).
+
+    c) If the modified program normally reads commands interactively when
+    run, you must cause it, when started running for such interactive use
+    in the simplest and most usual way, to print or display an
+    announcement including an appropriate copyright notice and a notice
+    that there is no warranty (or else, saying that you provide a
+    warranty) and that users may redistribute the program under these
+    conditions, and telling the user how to view a copy of this General
+    Public License.
+
+    d) You may charge a fee for the physical act of transferring a
+    copy, and you may at your option offer warranty protection in
+    exchange for a fee.
+
+Mere aggregation of another independent work with the Program (or its
+derivative) on a volume of a storage or distribution medium does not bring
+the other work under the scope of these terms.
+
+  3. You may copy and distribute the Program (or a portion or derivative of
+it, under Paragraph 2) in object code or executable form under the terms of
+Paragraphs 1 and 2 above provided that you also do one of the following:
+
+    a) accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of
+    Paragraphs 1 and 2 above; or,
+
+    b) accompany it with a written offer, valid for at least three
+    years, to give any third party free (except for a nominal charge
+    for the cost of distribution) a complete machine-readable copy of the
+    corresponding source code, to be distributed under the terms of
+    Paragraphs 1 and 2 above; or,
+
+    c) accompany it with the information you received as to where the
+    corresponding source code may be obtained.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form alone.)
+
+Source code for a work means the preferred form of the work for making
+modifications to it.  For an executable file, complete source code means
+all the source code for all modules it contains; but, as a special
+exception, it need not include source code for modules which are standard
+libraries that accompany the operating system on which the executable
+file runs, or for standard header files or definitions files that
+accompany that operating system.
+
+  4. You may not copy, modify, sublicense, distribute or transfer the
+Program except as expressly provided under this General Public License.
+Any attempt otherwise to copy, modify, sublicense, distribute or transfer
+the Program is void, and will automatically terminate your rights to use
+the Program under this License.  However, parties who have received
+copies, or rights to use copies, from you under this General Public
+License will not have their licenses terminated so long as such parties
+remain in full compliance.
+
+  5. By copying, distributing or modifying the Program (or any work based
+on the Program) you indicate your acceptance of this license to do so,
+and all its terms and conditions.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the original
+licensor to copy, distribute or modify the Program subject to these
+terms and conditions.  You may not impose any further restrictions on the
+recipients' exercise of the rights granted herein.
+
+  7. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of the license which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+the license, you may choose any version ever published by the Free Software
+Foundation.
+
+  8. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  9. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  10. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	Appendix: How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to humanity, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these
+terms.
+
+  To do so, attach the following notices to the program.  It is safest to
+attach them to the start of each source file to most effectively convey
+the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 1, or (at your option)
+    any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19xx name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the
+appropriate parts of the General Public License.  Of course, the
+commands you use may be called something other than `show w' and `show
+c'; they could even be mouse-clicks or menu items--whatever suits your
+program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  program `Gnomovision' (a program to direct compilers to make passes
+  at assemblers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/LICENSES/other/MPL-1.1 b/LICENSES/other/MPL-1.1
new file mode 100644
index 000000000000..568b6049efe6
--- /dev/null
+++ b/LICENSES/other/MPL-1.1
@@ -0,0 +1,478 @@
+Valid-License-Identifier: MPL-1.1
+SPDX-URL: https://spdx.org/licenses/MPL-1.1.html
+Usage-Guide:
+  To use the Mozilla Public License version 1.1 put the following SPDX
+  tag/value pair into a comment according to the placement guidelines in
+  the licensing rules documentation:
+    SPDX-License-Identifier: MPL-1.1
+License-Text:
+
+                          MOZILLA PUBLIC LICENSE
+                                Version 1.1
+
+                              ---------------
+
+1. Definitions.
+
+     1.0.1. "Commercial Use" means distribution or otherwise making the
+     Covered Code available to a third party.
+
+     1.1. "Contributor" means each entity that creates or contributes to
+     the creation of Modifications.
+
+     1.2. "Contributor Version" means the combination of the Original
+     Code, prior Modifications used by a Contributor, and the Modifications
+     made by that particular Contributor.
+
+     1.3. "Covered Code" means the Original Code or Modifications or the
+     combination of the Original Code and Modifications, in each case
+     including portions thereof.
+
+     1.4. "Electronic Distribution Mechanism" means a mechanism generally
+     accepted in the software development community for the electronic
+     transfer of data.
+
+     1.5. "Executable" means Covered Code in any form other than Source
+     Code.
+
+     1.6. "Initial Developer" means the individual or entity identified
+     as the Initial Developer in the Source Code notice required by Exhibit
+     A.
+
+     1.7. "Larger Work" means a work which combines Covered Code or
+     portions thereof with code not governed by the terms of this License.
+
+     1.8. "License" means this document.
+
+     1.8.1. "Licensable" means having the right to grant, to the maximum
+     extent possible, whether at the time of the initial grant or
+     subsequently acquired, any and all of the rights conveyed herein.
+
+     1.9. "Modifications" means any addition to or deletion from the
+     substance or structure of either the Original Code or any previous
+     Modifications. When Covered Code is released as a series of files, a
+     Modification is:
+          A. Any addition to or deletion from the contents of a file
+          containing Original Code or previous Modifications.
+
+          B. Any new file that contains any part of the Original Code or
+          previous Modifications.
+
+     1.10. "Original Code" means Source Code of computer software code
+     which is described in the Source Code notice required by Exhibit A as
+     Original Code, and which, at the time of its release under this
+     License is not already Covered Code governed by this License.
+
+     1.10.1. "Patent Claims" means any patent claim(s), now owned or
+     hereafter acquired, including without limitation,  method, process,
+     and apparatus claims, in any patent Licensable by grantor.
+
+     1.11. "Source Code" means the preferred form of the Covered Code for
+     making modifications to it, including all modules it contains, plus
+     any associated interface definition files, scripts used to control
+     compilation and installation of an Executable, or source code
+     differential comparisons against either the Original Code or another
+     well known, available Covered Code of the Contributor's choice. The
+     Source Code can be in a compressed or archival form, provided the
+     appropriate decompression or de-archiving software is widely available
+     for no charge.
+
+     1.12. "You" (or "Your")  means an individual or a legal entity
+     exercising rights under, and complying with all of the terms of, this
+     License or a future version of this License issued under Section 6.1.
+     For legal entities, "You" includes any entity which controls, is
+     controlled by, or is under common control with You. For purposes of
+     this definition, "control" means (a) the power, direct or indirect,
+     to cause the direction or management of such entity, whether by
+     contract or otherwise, or (b) ownership of more than fifty percent
+     (50%) of the outstanding shares or beneficial ownership of such
+     entity.
+
+2. Source Code License.
+
+     2.1. The Initial Developer Grant.
+     The Initial Developer hereby grants You a world-wide, royalty-free,
+     non-exclusive license, subject to third party intellectual property
+     claims:
+          (a)  under intellectual property rights (other than patent or
+          trademark) Licensable by Initial Developer to use, reproduce,
+          modify, display, perform, sublicense and distribute the Original
+          Code (or portions thereof) with or without Modifications, and/or
+          as part of a Larger Work; and
+
+          (b) under Patents Claims infringed by the making, using or
+          selling of Original Code, to make, have made, use, practice,
+          sell, and offer for sale, and/or otherwise dispose of the
+          Original Code (or portions thereof).
+
+          (c) the licenses granted in this Section 2.1(a) and (b) are
+          effective on the date Initial Developer first distributes
+          Original Code under the terms of this License.
+
+          (d) Notwithstanding Section 2.1(b) above, no patent license is
+          granted: 1) for code that You delete from the Original Code; 2)
+          separate from the Original Code;  or 3) for infringements caused
+          by: i) the modification of the Original Code or ii) the
+          combination of the Original Code with other software or devices.
+
+     2.2. Contributor Grant.
+     Subject to third party intellectual property claims, each Contributor
+     hereby grants You a world-wide, royalty-free, non-exclusive license
+
+          (a)  under intellectual property rights (other than patent or
+          trademark) Licensable by Contributor, to use, reproduce, modify,
+          display, perform, sublicense and distribute the Modifications
+          created by such Contributor (or portions thereof) either on an
+          unmodified basis, with other Modifications, as Covered Code
+          and/or as part of a Larger Work; and
+
+          (b) under Patent Claims infringed by the making, using, or
+          selling of  Modifications made by that Contributor either alone
+          and/or in combination with its Contributor Version (or portions
+          of such combination), to make, use, sell, offer for sale, have
+          made, and/or otherwise dispose of: 1) Modifications made by that
+          Contributor (or portions thereof); and 2) the combination of
+          Modifications made by that Contributor with its Contributor
+          Version (or portions of such combination).
+
+          (c) the licenses granted in Sections 2.2(a) and 2.2(b) are
+          effective on the date Contributor first makes Commercial Use of
+          the Covered Code.
+
+          (d)    Notwithstanding Section 2.2(b) above, no patent license is
+          granted: 1) for any code that Contributor has deleted from the
+          Contributor Version; 2)  separate from the Contributor Version;
+          3)  for infringements caused by: i) third party modifications of
+          Contributor Version or ii)  the combination of Modifications made
+          by that Contributor with other software  (except as part of the
+          Contributor Version) or other devices; or 4) under Patent Claims
+          infringed by Covered Code in the absence of Modifications made by
+          that Contributor.
+
+3. Distribution Obligations.
+
+     3.1. Application of License.
+     The Modifications which You create or to which You contribute are
+     governed by the terms of this License, including without limitation
+     Section 2.2. The Source Code version of Covered Code may be
+     distributed only under the terms of this License or a future version
+     of this License released under Section 6.1, and You must include a
+     copy of this License with every copy of the Source Code You
+     distribute. You may not offer or impose any terms on any Source Code
+     version that alters or restricts the applicable version of this
+     License or the recipients' rights hereunder. However, You may include
+     an additional document offering the additional rights described in
+     Section 3.5.
+
+     3.2. Availability of Source Code.
+     Any Modification which You create or to which You contribute must be
+     made available in Source Code form under the terms of this License
+     either on the same media as an Executable version or via an accepted
+     Electronic Distribution Mechanism to anyone to whom you made an
+     Executable version available; and if made available via Electronic
+     Distribution Mechanism, must remain available for at least twelve (12)
+     months after the date it initially became available, or at least six
+     (6) months after a subsequent version of that particular Modification
+     has been made available to such recipients. You are responsible for
+     ensuring that the Source Code version remains available even if the
+     Electronic Distribution Mechanism is maintained by a third party.
+
+     3.3. Description of Modifications.
+     You must cause all Covered Code to which You contribute to contain a
+     file documenting the changes You made to create that Covered Code and
+     the date of any change. You must include a prominent statement that
+     the Modification is derived, directly or indirectly, from Original
+     Code provided by the Initial Developer and including the name of the
+     Initial Developer in (a) the Source Code, and (b) in any notice in an
+     Executable version or related documentation in which You describe the
+     origin or ownership of the Covered Code.
+
+     3.4. Intellectual Property Matters
+          (a) Third Party Claims.
+          If Contributor has knowledge that a license under a third party's
+          intellectual property rights is required to exercise the rights
+          granted by such Contributor under Sections 2.1 or 2.2,
+          Contributor must include a text file with the Source Code
+          distribution titled "LEGAL" which describes the claim and the
+          party making the claim in sufficient detail that a recipient will
+          know whom to contact. If Contributor obtains such knowledge after
+          the Modification is made available as described in Section 3.2,
+          Contributor shall promptly modify the LEGAL file in all copies
+          Contributor makes available thereafter and shall take other steps
+          (such as notifying appropriate mailing lists or newsgroups)
+          reasonably calculated to inform those who received the Covered
+          Code that new knowledge has been obtained.
+
+          (b) Contributor APIs.
+          If Contributor's Modifications include an application programming
+          interface and Contributor has knowledge of patent licenses which
+          are reasonably necessary to implement that API, Contributor must
+          also include this information in the LEGAL file.
+
+               (c)    Representations.
+          Contributor represents that, except as disclosed pursuant to
+          Section 3.4(a) above, Contributor believes that Contributor's
+          Modifications are Contributor's original creation(s) and/or
+          Contributor has sufficient rights to grant the rights conveyed by
+          this License.
+
+     3.5. Required Notices.
+     You must duplicate the notice in Exhibit A in each file of the Source
+     Code.  If it is not possible to put such notice in a particular Source
+     Code file due to its structure, then You must include such notice in a
+     location (such as a relevant directory) where a user would be likely
+     to look for such a notice.  If You created one or more Modification(s)
+     You may add your name as a Contributor to the notice described in
+     Exhibit A.  You must also duplicate this License in any documentation
+     for the Source Code where You describe recipients' rights or ownership
+     rights relating to Covered Code.  You may choose to offer, and to
+     charge a fee for, warranty, support, indemnity or liability
+     obligations to one or more recipients of Covered Code. However, You
+     may do so only on Your own behalf, and not on behalf of the Initial
+     Developer or any Contributor. You must make it absolutely clear than
+     any such warranty, support, indemnity or liability obligation is
+     offered by You alone, and You hereby agree to indemnify the Initial
+     Developer and every Contributor for any liability incurred by the
+     Initial Developer or such Contributor as a result of warranty,
+     support, indemnity or liability terms You offer.
+
+     3.6. Distribution of Executable Versions.
+     You may distribute Covered Code in Executable form only if the
+     requirements of Section 3.1-3.5 have been met for that Covered Code,
+     and if You include a notice stating that the Source Code version of
+     the Covered Code is available under the terms of this License,
+     including a description of how and where You have fulfilled the
+     obligations of Section 3.2. The notice must be conspicuously included
+     in any notice in an Executable version, related documentation or
+     collateral in which You describe recipients' rights relating to the
+     Covered Code. You may distribute the Executable version of Covered
+     Code or ownership rights under a license of Your choice, which may
+     contain terms different from this License, provided that You are in
+     compliance with the terms of this License and that the license for the
+     Executable version does not attempt to limit or alter the recipient's
+     rights in the Source Code version from the rights set forth in this
+     License. If You distribute the Executable version under a different
+     license You must make it absolutely clear that any terms which differ
+     from this License are offered by You alone, not by the Initial
+     Developer or any Contributor. You hereby agree to indemnify the
+     Initial Developer and every Contributor for any liability incurred by
+     the Initial Developer or such Contributor as a result of any such
+     terms You offer.
+
+     3.7. Larger Works.
+     You may create a Larger Work by combining Covered Code with other code
+     not governed by the terms of this License and distribute the Larger
+     Work as a single product. In such a case, You must make sure the
+     requirements of this License are fulfilled for the Covered Code.
+
+4. Inability to Comply Due to Statute or Regulation.
+
+     If it is impossible for You to comply with any of the terms of this
+     License with respect to some or all of the Covered Code due to
+     statute, judicial order, or regulation then You must: (a) comply with
+     the terms of this License to the maximum extent possible; and (b)
+     describe the limitations and the code they affect. Such description
+     must be included in the LEGAL file described in Section 3.4 and must
+     be included with all distributions of the Source Code. Except to the
+     extent prohibited by statute or regulation, such description must be
+     sufficiently detailed for a recipient of ordinary skill to be able to
+     understand it.
+
+5. Application of this License.
+
+     This License applies to code to which the Initial Developer has
+     attached the notice in Exhibit A and to related Covered Code.
+
+6. Versions of the License.
+
+     6.1. New Versions.
+     Netscape Communications Corporation ("Netscape") may publish revised
+     and/or new versions of the License from time to time. Each version
+     will be given a distinguishing version number.
+
+     6.2. Effect of New Versions.
+     Once Covered Code has been published under a particular version of the
+     License, You may always continue to use it under the terms of that
+     version. You may also choose to use such Covered Code under the terms
+     of any subsequent version of the License published by Netscape. No one
+     other than Netscape has the right to modify the terms applicable to
+     Covered Code created under this License.
+
+     6.3. Derivative Works.
+     If You create or use a modified version of this License (which you may
+     only do in order to apply it to code which is not already Covered Code
+     governed by this License), You must (a) rename Your license so that
+     the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape",
+     "MPL", "NPL" or any confusingly similar phrase do not appear in your
+     license (except to note that your license differs from this License)
+     and (b) otherwise make it clear that Your version of the license
+     contains terms which differ from the Mozilla Public License and
+     Netscape Public License. (Filling in the name of the Initial
+     Developer, Original Code or Contributor in the notice described in
+     Exhibit A shall not of themselves be deemed to be modifications of
+     this License.)
+
+7. DISCLAIMER OF WARRANTY.
+
+     COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS,
+     WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+     WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF
+     DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING.
+     THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE
+     IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT,
+     YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE
+     COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER
+     OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF
+     ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
+
+8. TERMINATION.
+
+     8.1.  This License and the rights granted hereunder will terminate
+     automatically if You fail to comply with terms herein and fail to cure
+     such breach within 30 days of becoming aware of the breach. All
+     sublicenses to the Covered Code which are properly granted shall
+     survive any termination of this License. Provisions which, by their
+     nature, must remain in effect beyond the termination of this License
+     shall survive.
+
+     8.2.  If You initiate litigation by asserting a patent infringement
+     claim (excluding declatory judgment actions) against Initial Developer
+     or a Contributor (the Initial Developer or Contributor against whom
+     You file such action is referred to as "Participant")  alleging that:
+
+     (a)  such Participant's Contributor Version directly or indirectly
+     infringes any patent, then any and all rights granted by such
+     Participant to You under Sections 2.1 and/or 2.2 of this License
+     shall, upon 60 days notice from Participant terminate prospectively,
+     unless if within 60 days after receipt of notice You either: (i)
+     agree in writing to pay Participant a mutually agreeable reasonable
+     royalty for Your past and future use of Modifications made by such
+     Participant, or (ii) withdraw Your litigation claim with respect to
+     the Contributor Version against such Participant.  If within 60 days
+     of notice, a reasonable royalty and payment arrangement are not
+     mutually agreed upon in writing by the parties or the litigation claim
+     is not withdrawn, the rights granted by Participant to You under
+     Sections 2.1 and/or 2.2 automatically terminate at the expiration of
+     the 60 day notice period specified above.
+
+     (b)  any software, hardware, or device, other than such Participant's
+     Contributor Version, directly or indirectly infringes any patent, then
+     any rights granted to You by such Participant under Sections 2.1(b)
+     and 2.2(b) are revoked effective as of the date You first made, used,
+     sold, distributed, or had made, Modifications made by that
+     Participant.
+
+     8.3.  If You assert a patent infringement claim against Participant
+     alleging that such Participant's Contributor Version directly or
+     indirectly infringes any patent where such claim is resolved (such as
+     by license or settlement) prior to the initiation of patent
+     infringement litigation, then the reasonable value of the licenses
+     granted by such Participant under Sections 2.1 or 2.2 shall be taken
+     into account in determining the amount or value of any payment or
+     license.
+
+     8.4.  In the event of termination under Sections 8.1 or 8.2 above,
+     all end user license agreements (excluding distributors and resellers)
+     which have been validly granted by You or any distributor hereunder
+     prior to termination shall survive termination.
+
+9. LIMITATION OF LIABILITY.
+
+     UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
+     (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
+     DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE,
+     OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR
+     ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY
+     CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL,
+     WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER
+     COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN
+     INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF
+     LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY
+     RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
+     PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE
+     EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO
+     THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
+
+10. U.S. GOVERNMENT END USERS.
+
+     The Covered Code is a "commercial item," as that term is defined in
+     48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer
+     software" and "commercial computer software documentation," as such
+     terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48
+     C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995),
+     all U.S. Government End Users acquire Covered Code with only those
+     rights set forth herein.
+
+11. MISCELLANEOUS.
+
+     This License represents the complete agreement concerning subject
+     matter hereof. If any provision of this License is held to be
+     unenforceable, such provision shall be reformed only to the extent
+     necessary to make it enforceable. This License shall be governed by
+     California law provisions (except to the extent applicable law, if
+     any, provides otherwise), excluding its conflict-of-law provisions.
+     With respect to disputes in which at least one party is a citizen of,
+     or an entity chartered or registered to do business in the United
+     States of America, any litigation relating to this License shall be
+     subject to the jurisdiction of the Federal Courts of the Northern
+     District of California, with venue lying in Santa Clara County,
+     California, with the losing party responsible for costs, including
+     without limitation, court costs and reasonable attorneys' fees and
+     expenses. The application of the United Nations Convention on
+     Contracts for the International Sale of Goods is expressly excluded.
+     Any law or regulation which provides that the language of a contract
+     shall be construed against the drafter shall not apply to this
+     License.
+
+12. RESPONSIBILITY FOR CLAIMS.
+
+     As between Initial Developer and the Contributors, each party is
+     responsible for claims and damages arising, directly or indirectly,
+     out of its utilization of rights under this License and You agree to
+     work with Initial Developer and Contributors to distribute such
+     responsibility on an equitable basis. Nothing herein is intended or
+     shall be deemed to constitute any admission of liability.
+
+13. MULTIPLE-LICENSED CODE.
+
+     Initial Developer may designate portions of the Covered Code as
+     "Multiple-Licensed".  "Multiple-Licensed" means that the Initial
+     Developer permits you to utilize portions of the Covered Code under
+     Your choice of the MPL or the alternative licenses, if any, specified
+     by the Initial Developer in the file described in Exhibit A.
+
+EXHIBIT A -Mozilla Public License.
+
+     ``The contents of this file are subject to the Mozilla Public License
+     Version 1.1 (the "License"); you may not use this file except in
+     compliance with the License. You may obtain a copy of the License at
+     https://www.mozilla.org/MPL/
+
+     Software distributed under the License is distributed on an "AS IS"
+     basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+     License for the specific language governing rights and limitations
+     under the License.
+
+     The Original Code is ______________________________________.
+
+     The Initial Developer of the Original Code is ________________________.
+     Portions created by ______________________ are Copyright (C) ______
+     _______________________. All Rights Reserved.
+
+     Contributor(s): ______________________________________.
+
+     Alternatively, the contents of this file may be used under the terms
+     of the _____ license (the  "[___] License"), in which case the
+     provisions of [______] License are applicable instead of those
+     above.  If you wish to allow use of your version of this file only
+     under the terms of the [____] License and not to allow others to use
+     your version of this file under the MPL, indicate your decision by
+     deleting  the provisions above and replace  them with the notice and
+     other provisions required by the [___] License.  If you do not delete
+     the provisions above, a recipient may use your version of this file
+     under either the MPL or the [___] License."
+
+     [NOTE: The text of this Exhibit A may differ slightly from the text of
+     the notices in the Source Code files of the Original Code. You should
+     use the text of this Exhibit A rather than the text found in the
+     Original Code Source Code for Your Modifications.]
diff --git a/LICENSES/preferred/BSD-2-Clause b/LICENSES/preferred/BSD-2-Clause
new file mode 100644
index 000000000000..da366e2ce50b
--- /dev/null
+++ b/LICENSES/preferred/BSD-2-Clause
@@ -0,0 +1,32 @@
+Valid-License-Identifier: BSD-2-Clause
+SPDX-URL: https://spdx.org/licenses/BSD-2-Clause.html
+Usage-Guide:
+  To use the BSD 2-clause "Simplified" License put the following SPDX
+  tag/value pair into a comment according to the placement guidelines in
+  the licensing rules documentation:
+    SPDX-License-Identifier: BSD-2-Clause
+License-Text:
+
+Copyright (c) <year> <owner> . All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
diff --git a/LICENSES/preferred/BSD-3-Clause b/LICENSES/preferred/BSD-3-Clause
new file mode 100644
index 000000000000..34c7f057c8d5
--- /dev/null
+++ b/LICENSES/preferred/BSD-3-Clause
@@ -0,0 +1,36 @@
+Valid-License-Identifier: BSD-3-Clause
+SPDX-URL: https://spdx.org/licenses/BSD-3-Clause.html
+Usage-Guide:
+  To use the BSD 3-clause "New" or "Revised" License put the following SPDX
+  tag/value pair into a comment according to the placement guidelines in
+  the licensing rules documentation:
+    SPDX-License-Identifier: BSD-3-Clause
+License-Text:
+
+Copyright (c) <year> <owner> . All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from this
+   software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
diff --git a/LICENSES/preferred/BSD-3-Clause-Clear b/LICENSES/preferred/BSD-3-Clause-Clear
new file mode 100644
index 000000000000..e53b56092b90
--- /dev/null
+++ b/LICENSES/preferred/BSD-3-Clause-Clear
@@ -0,0 +1,41 @@
+Valid-License-Identifier: BSD-3-Clause-Clear
+SPDX-URL: https://spdx.org/licenses/BSD-3-Clause-Clear.html
+Usage-Guide:
+  To use the BSD 3-clause "Clear" License put the following SPDX
+  tag/value pair into a comment according to the placement guidelines in
+  the licensing rules documentation:
+    SPDX-License-Identifier: BSD-3-Clause-Clear
+License-Text:
+
+The Clear BSD License
+
+Copyright (c) [xxxx]-[xxxx] [Owner Organization]
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted (subject to the limitations in the disclaimer
+below) provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+ * Neither the name of [Owner Organization] nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
+THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
+NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/LICENSES/preferred/GPL-2.0 b/LICENSES/preferred/GPL-2.0
new file mode 100644
index 000000000000..b8db91d3a1cb
--- /dev/null
+++ b/LICENSES/preferred/GPL-2.0
@@ -0,0 +1,353 @@
+Valid-License-Identifier: GPL-2.0
+Valid-License-Identifier: GPL-2.0+
+SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
+Usage-Guide:
+  To use this license in source code, put one of the following SPDX
+  tag/value pairs into a comment according to the placement
+  guidelines in the licensing rules documentation.
+  For 'GNU General Public License (GPL) version 2 only' use:
+    SPDX-License-Identifier: GPL-2.0
+  For 'GNU General Public License (GPL) version 2 or any later version' use:
+    SPDX-License-Identifier: GPL-2.0+
+License-Text:
+
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                       51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
diff --git a/LICENSES/preferred/LGPL-2.0 b/LICENSES/preferred/LGPL-2.0
new file mode 100644
index 000000000000..957d798fe037
--- /dev/null
+++ b/LICENSES/preferred/LGPL-2.0
@@ -0,0 +1,487 @@
+Valid-License-Identifier: LGPL-2.0
+Valid-License-Identifier: LGPL-2.0+
+SPDX-URL: https://spdx.org/licenses/LGPL-2.0.html
+Usage-Guide:
+  To use this license in source code, put one of the following SPDX
+  tag/value pairs into a comment according to the placement
+  guidelines in the licensing rules documentation.
+  For 'GNU Library General Public License (LGPL) version 2.0 only' use:
+    SPDX-License-Identifier: LGPL-2.0
+  For 'GNU Library General Public License (LGPL) version 2.0 or any later
+  version' use:
+    SPDX-License-Identifier: LGPL-2.0+
+License-Text:
+
+GNU LIBRARY GENERAL PUBLIC LICENSE
+Version 2, June 1991
+
+Copyright (C) 1991 Free Software Foundation, Inc.
+51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+[This is the first released version of the library GPL. It is numbered 2
+because it goes with version 2 of the ordinary GPL.]
+
+Preamble
+
+The licenses for most software are designed to take away your freedom to
+share and change it. By contrast, the GNU General Public Licenses are
+intended to guarantee your freedom to share and change free software--to
+make sure the software is free for all its users.
+
+This license, the Library General Public License, applies to some specially
+designated Free Software Foundation software, and to any other libraries
+whose authors decide to use it. You can use it for your libraries, too.
+
+When we speak of free software, we are referring to freedom, not price. Our
+General Public Licenses are designed to make sure that you have the freedom
+to distribute copies of free software (and charge for this service if you
+wish), that you receive source code or can get it if you want it, that you
+can change the software or use pieces of it in new free programs; and that
+you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid anyone to
+deny you these rights or to ask you to surrender the rights. These
+restrictions translate to certain responsibilities for you if you
+distribute copies of the library, or if you modify it.
+
+For example, if you distribute copies of the library, whether gratis or for
+a fee, you must give the recipients all the rights that we gave you. You
+must make sure that they, too, receive or can get the source code. If you
+link a program with the library, you must provide complete object files to
+the recipients so that they can relink them with the library, after making
+changes to the library and recompiling it. And you must show them these
+terms so they know their rights.
+
+Our method of protecting your rights has two steps: (1) copyright the
+library, and (2) offer you this license which gives you legal permission to
+copy, distribute and/or modify the library.
+
+Also, for each distributor's protection, we want to make certain that
+everyone understands that there is no warranty for this free library. If
+the library is modified by someone else and passed on, we want its
+recipients to know that what they have is not the original version, so that
+any problems introduced by others will not reflect on the original authors'
+reputations.
+
+Finally, any free program is threatened constantly by software patents. We
+wish to avoid the danger that companies distributing free software will
+individually obtain patent licenses, thus in effect transforming the
+program into proprietary software. To prevent this, we have made it clear
+that any patent must be licensed for everyone's free use or not licensed at
+all.
+
+Most GNU software, including some libraries, is covered by the ordinary GNU
+General Public License, which was designed for utility programs. This
+license, the GNU Library General Public License, applies to certain
+designated libraries. This license is quite different from the ordinary
+one; be sure to read it in full, and don't assume that anything in it is
+the same as in the ordinary license.
+
+The reason we have a separate public license for some libraries is that
+they blur the distinction we usually make between modifying or adding to a
+program and simply using it. Linking a program with a library, without
+changing the library, is in some sense simply using the library, and is
+analogous to running a utility program or application program. However, in
+a textual and legal sense, the linked executable is a combined work, a
+derivative of the original library, and the ordinary General Public License
+treats it as such.
+
+Because of this blurred distinction, using the ordinary General Public
+License for libraries did not effectively promote software sharing, because
+most developers did not use the libraries. We concluded that weaker
+conditions might promote sharing better.
+
+However, unrestricted linking of non-free programs would deprive the users
+of those programs of all benefit from the free status of the libraries
+themselves. This Library General Public License is intended to permit
+developers of non-free programs to use free libraries, while preserving
+your freedom as a user of such programs to change the free libraries that
+are incorporated in them. (We have not seen how to achieve this as regards
+changes in header files, but we have achieved it as regards changes in the
+actual functions of the Library.) The hope is that this will lead to faster
+development of free libraries.
+
+The precise terms and conditions for copying, distribution and modification
+follow. Pay close attention to the difference between a "work based on the
+library" and a "work that uses the library". The former contains code
+derived from the library, while the latter only works together with the
+library.
+
+Note that it is possible for a library to be covered by the ordinary
+General Public License rather than by this special one.
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. This License Agreement applies to any software library which contains a
+   notice placed by the copyright holder or other authorized party saying
+   it may be distributed under the terms of this Library General Public
+   License (also called "this License"). Each licensee is addressed as
+   "you".
+
+   A "library" means a collection of software functions and/or data
+   prepared so as to be conveniently linked with application programs
+   (which use some of those functions and data) to form executables.
+
+   The "Library", below, refers to any such software library or work which
+   has been distributed under these terms. A "work based on the Library"
+   means either the Library or any derivative work under copyright law:
+   that is to say, a work containing the Library or a portion of it, either
+   verbatim or with modifications and/or translated straightforwardly into
+   another language. (Hereinafter, translation is included without
+   limitation in the term "modification".)
+
+   "Source code" for a work means the preferred form of the work for making
+   modifications to it. For a library, complete source code means all the
+   source code for all modules it contains, plus any associated interface
+   definition files, plus the scripts used to control compilation and
+   installation of the library.
+
+   Activities other than copying, distribution and modification are not
+   covered by this License; they are outside its scope. The act of running
+   a program using the Library is not restricted, and output from such a
+   program is covered only if its contents constitute a work based on the
+   Library (independent of the use of the Library in a tool for writing
+   it). Whether that is true depends on what the Library does and what the
+   program that uses the Library does.
+
+1. You may copy and distribute verbatim copies of the Library's complete
+   source code as you receive it, in any medium, provided that you
+   conspicuously and appropriately publish on each copy an appropriate
+   copyright notice and disclaimer of warranty; keep intact all the notices
+   that refer to this License and to the absence of any warranty; and
+   distribute a copy of this License along with the Library.
+
+   You may charge a fee for the physical act of transferring a copy, and
+   you may at your option offer warranty protection in exchange for a fee.
+
+2. You may modify your copy or copies of the Library or any portion of it,
+   thus forming a work based on the Library, and copy and distribute such
+   modifications or work under the terms of Section 1 above, provided that
+   you also meet all of these conditions:
+
+   a) The modified work must itself be a software library.
+
+   b) You must cause the files modified to carry prominent notices stating
+      that you changed the files and the date of any change.
+
+   c) You must cause the whole of the work to be licensed at no charge to
+      all third parties under the terms of this License.
+
+   d) If a facility in the modified Library refers to a function or a table
+      of data to be supplied by an application program that uses the
+      facility, other than as an argument passed when the facility is
+      invoked, then you must make a good faith effort to ensure that, in
+      the event an application does not supply such function or table, the
+      facility still operates, and performs whatever part of its purpose
+      remains meaningful.
+
+   (For example, a function in a library to compute square roots has a
+   purpose that is entirely well-defined independent of the
+   application. Therefore, Subsection 2d requires that any
+   application-supplied function or table used by this function must be
+   optional: if the application does not supply it, the square root
+   function must still compute square roots.)
+
+   These requirements apply to the modified work as a whole. If
+   identifiable sections of that work are not derived from the Library, and
+   can be reasonably considered independent and separate works in
+   themselves, then this License, and its terms, do not apply to those
+   sections when you distribute them as separate works. But when you
+   distribute the same sections as part of a whole which is a work based on
+   the Library, the distribution of the whole must be on the terms of this
+   License, whose permissions for other licensees extend to the entire
+   whole, and thus to each and every part regardless of who wrote it.
+
+   Thus, it is not the intent of this section to claim rights or contest
+   your rights to work written entirely by you; rather, the intent is to
+   exercise the right to control the distribution of derivative or
+   collective works based on the Library.
+
+   In addition, mere aggregation of another work not based on the Library
+   with the Library (or with a work based on the Library) on a volume of a
+   storage or distribution medium does not bring the other work under the
+   scope of this License.
+
+3. You may opt to apply the terms of the ordinary GNU General Public
+   License instead of this License to a given copy of the Library. To do
+   this, you must alter all the notices that refer to this License, so that
+   they refer to the ordinary GNU General Public License, version 2,
+   instead of to this License. (If a newer version than version 2 of the
+   ordinary GNU General Public License has appeared, then you can specify
+   that version instead if you wish.) Do not make any other change in these
+   notices.
+
+   Once this change is made in a given copy, it is irreversible for that
+   copy, so the ordinary GNU General Public License applies to all
+   subsequent copies and derivative works made from that copy.
+
+   This option is useful when you wish to copy part of the code of the
+   Library into a program that is not a library.
+
+4. You may copy and distribute the Library (or a portion or derivative of
+   it, under Section 2) in object code or executable form under the terms
+   of Sections 1 and 2 above provided that you accompany it with the
+   complete corresponding machine-readable source code, which must be
+   distributed under the terms of Sections 1 and 2 above on a medium
+   customarily used for software interchange.
+
+   If distribution of object code is made by offering access to copy from a
+   designated place, then offering equivalent access to copy the source
+   code from the same place satisfies the requirement to distribute the
+   source code, even though third parties are not compelled to copy the
+   source along with the object code.
+
+5. A program that contains no derivative of any portion of the Library, but
+   is designed to work with the Library by being compiled or linked with
+   it, is called a "work that uses the Library". Such a work, in isolation,
+   is not a derivative work of the Library, and therefore falls outside the
+   scope of this License.
+
+   However, linking a "work that uses the Library" with the Library creates
+   an executable that is a derivative of the Library (because it contains
+   portions of the Library), rather than a "work that uses the
+   library". The executable is therefore covered by this License. Section 6
+   states terms for distribution of such executables.
+
+   When a "work that uses the Library" uses material from a header file
+   that is part of the Library, the object code for the work may be a
+   derivative work of the Library even though the source code is
+   not. Whether this is true is especially significant if the work can be
+   linked without the Library, or if the work is itself a library. The
+   threshold for this to be true is not precisely defined by law.
+
+   If such an object file uses only numerical parameters, data structure
+   layouts and accessors, and small macros and small inline functions (ten
+   lines or less in length), then the use of the object file is
+   unrestricted, regardless of whether it is legally a derivative
+   work. (Executables containing this object code plus portions of the
+   Library will still fall under Section 6.)
+
+   Otherwise, if the work is a derivative of the Library, you may
+   distribute the object code for the work under the terms of Section
+   6. Any executables containing that work also fall under Section 6,
+   whether or not they are linked directly with the Library itself.
+
+6. As an exception to the Sections above, you may also compile or link a
+   "work that uses the Library" with the Library to produce a work
+   containing portions of the Library, and distribute that work under terms
+   of your choice, provided that the terms permit modification of the work
+   for the customer's own use and reverse engineering for debugging such
+   modifications.
+
+   You must give prominent notice with each copy of the work that the
+   Library is used in it and that the Library and its use are covered by
+   this License. You must supply a copy of this License. If the work during
+   execution displays copyright notices, you must include the copyright
+   notice for the Library among them, as well as a reference directing the
+   user to the copy of this License. Also, you must do one of these things:
+
+   a) Accompany the work with the complete corresponding machine-readable
+      source code for the Library including whatever changes were used in
+      the work (which must be distributed under Sections 1 and 2 above);
+      and, if the work is an executable linked with the Library, with the
+      complete machine-readable "work that uses the Library", as object
+      code and/or source code, so that the user can modify the Library and
+      then relink to produce a modified executable containing the modified
+      Library. (It is understood that the user who changes the contents of
+      definitions files in the Library will not necessarily be able to
+      recompile the application to use the modified definitions.)
+
+   b) Accompany the work with a written offer, valid for at least three
+      years, to give the same user the materials specified in Subsection
+      6a, above, for a charge no more than the cost of performing this
+      distribution.
+
+   c) If distribution of the work is made by offering access to copy from a
+      designated place, offer equivalent access to copy the above specified
+      materials from the same place.
+
+   d) Verify that the user has already received a copy of these materials
+      or that you have already sent this user a copy.
+
+   For an executable, the required form of the "work that uses the Library"
+   must include any data and utility programs needed for reproducing the
+   executable from it. However, as a special exception, the source code
+   distributed need not include anything that is normally distributed (in
+   either source or binary form) with the major components (compiler,
+   kernel, and so on) of the operating system on which the executable runs,
+   unless that component itself accompanies the executable.
+
+   It may happen that this requirement contradicts the license restrictions
+   of other proprietary libraries that do not normally accompany the
+   operating system. Such a contradiction means you cannot use both them
+   and the Library together in an executable that you distribute.
+
+7. You may place library facilities that are a work based on the Library
+   side-by-side in a single library together with other library facilities
+   not covered by this License, and distribute such a combined library,
+   provided that the separate distribution of the work based on the Library
+   and of the other library facilities is otherwise permitted, and provided
+   that you do these two things:
+
+   a) Accompany the combined library with a copy of the same work based on
+      the Library, uncombined with any other library facilities. This must
+      be distributed under the terms of the Sections above.
+
+   b) Give prominent notice with the combined library of the fact that part
+      of it is a work based on the Library, and explaining where to find
+      the accompanying uncombined form of the same work.
+
+8. You may not copy, modify, sublicense, link with, or distribute the
+   Library except as expressly provided under this License. Any attempt
+   otherwise to copy, modify, sublicense, link with, or distribute the
+   Library is void, and will automatically terminate your rights under this
+   License. However, parties who have received copies, or rights, from you
+   under this License will not have their licenses terminated so long as
+   such parties remain in full compliance.
+
+9. You are not required to accept this License, since you have not signed
+   it. However, nothing else grants you permission to modify or distribute
+   the Library or its derivative works. These actions are prohibited by law
+   if you do not accept this License. Therefore, by modifying or
+   distributing the Library (or any work based on the Library), you
+   indicate your acceptance of this License to do so, and all its terms and
+   conditions for copying, distributing or modifying the Library or works
+   based on it.
+
+10. Each time you redistribute the Library (or any work based on the
+    Library), the recipient automatically receives a license from the
+    original licensor to copy, distribute, link with or modify the Library
+    subject to these terms and conditions. You may not impose any further
+    restrictions on the recipients' exercise of the rights granted
+    herein. You are not responsible for enforcing compliance by third
+    parties to this License.
+
+11. If, as a consequence of a court judgment or allegation of patent
+    infringement or for any other reason (not limited to patent issues),
+    conditions are imposed on you (whether by court order, agreement or
+    otherwise) that contradict the conditions of this License, they do not
+    excuse you from the conditions of this License. If you cannot
+    distribute so as to satisfy simultaneously your obligations under this
+    License and any other pertinent obligations, then as a consequence you
+    may not distribute the Library at all. For example, if a patent license
+    would not permit royalty-free redistribution of the Library by all
+    those who receive copies directly or indirectly through you, then the
+    only way you could satisfy both it and this License would be to refrain
+    entirely from distribution of the Library.
+
+    If any portion of this section is held invalid or unenforceable under
+    any particular circumstance, the balance of the section is intended to
+    apply, and the section as a whole is intended to apply in other
+    circumstances.
+
+    It is not the purpose of this section to induce you to infringe any
+    patents or other property right claims or to contest validity of any
+    such claims; this section has the sole purpose of protecting the
+    integrity of the free software distribution system which is implemented
+    by public license practices. Many people have made generous
+    contributions to the wide range of software distributed through that
+    system in reliance on consistent application of that system; it is up
+    to the author/donor to decide if he or she is willing to distribute
+    software through any other system and a licensee cannot impose that
+    choice.
+
+    This section is intended to make thoroughly clear what is believed to
+    be a consequence of the rest of this License.
+
+12. If the distribution and/or use of the Library is restricted in certain
+    countries either by patents or by copyrighted interfaces, the original
+    copyright holder who places the Library under this License may add an
+    explicit geographical distribution limitation excluding those
+    countries, so that distribution is permitted only in or among countries
+    not thus excluded. In such case, this License incorporates the
+    limitation as if written in the body of this License.
+
+13. The Free Software Foundation may publish revised and/or new versions of
+    the Library General Public License from time to time. Such new versions
+    will be similar in spirit to the present version, but may differ in
+    detail to address new problems or concerns.
+
+    Each version is given a distinguishing version number. If the Library
+    specifies a version number of this License which applies to it and "any
+    later version", you have the option of following the terms and
+    conditions either of that version or of any later version published by
+    the Free Software Foundation. If the Library does not specify a license
+    version number, you may choose any version ever published by the Free
+    Software Foundation.
+
+14. If you wish to incorporate parts of the Library into other free
+    programs whose distribution conditions are incompatible with these,
+    write to the author to ask for permission. For software which is
+    copyrighted by the Free Software Foundation, write to the Free Software
+    Foundation; we sometimes make exceptions for this. Our decision will be
+    guided by the two goals of preserving the free status of all
+    derivatives of our free software and of promoting the sharing and reuse
+    of software generally.
+
+NO WARRANTY
+
+15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+    FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+    OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+    PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER
+    EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE
+    ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH
+    YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL
+    NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+    WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+    REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR
+    DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL
+    DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY
+    (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED
+    INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF
+    THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR
+    OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+END OF TERMS AND CONDITIONS
+
+How to Apply These Terms to Your New Libraries
+
+If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change. You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+To apply these terms, attach the following notices to the library. It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+one line to give the library's name and an idea of what it does.
+Copyright (C) year name of author
+
+This library is free software; you can redistribute it and/or modify it
+under the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2 of the License, or (at your
+option) any later version.
+
+This library is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
+License for more details.
+
+You should have received a copy of the GNU Library General Public License
+along with this library; if not, write to the Free Software Foundation,
+Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary. Here is a sample; alter the names:
+
+Yoyodyne, Inc., hereby disclaims all copyright interest in
+the library `Frob' (a library for tweaking knobs) written
+by James Random Hacker.
+
+signature of Ty Coon, 1 April 1990
+Ty Coon, President of Vice
+
+That's all there is to it!
diff --git a/LICENSES/preferred/LGPL-2.1 b/LICENSES/preferred/LGPL-2.1
new file mode 100644
index 000000000000..27bb4342a3e8
--- /dev/null
+++ b/LICENSES/preferred/LGPL-2.1
@@ -0,0 +1,503 @@
+Valid-License-Identifier: LGPL-2.1
+Valid-License-Identifier: LGPL-2.1+
+SPDX-URL: https://spdx.org/licenses/LGPL-2.1.html
+Usage-Guide:
+  To use this license in source code, put one of the following SPDX
+  tag/value pairs into a comment according to the placement
+  guidelines in the licensing rules documentation.
+  For 'GNU Lesser General Public License (LGPL) version 2.1 only' use:
+    SPDX-License-Identifier: LGPL-2.1
+  For 'GNU Lesser General Public License (LGPL) version 2.1 or any later
+  version' use:
+    SPDX-License-Identifier: LGPL-2.1+
+License-Text:
+
+GNU LESSER GENERAL PUBLIC LICENSE
+Version 2.1, February 1999
+
+Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL. It also counts as
+the successor of the GNU Library Public License, version 2, hence the
+version number 2.1.]
+
+Preamble
+
+The licenses for most software are designed to take away your freedom to
+share and change it. By contrast, the GNU General Public Licenses are
+intended to guarantee your freedom to share and change free software--to
+make sure the software is free for all its users.
+
+This license, the Lesser General Public License, applies to some specially
+designated software packages--typically libraries--of the Free Software
+Foundation and other authors who decide to use it. You can use it too, but
+we suggest you first think carefully about whether this license or the
+ordinary General Public License is the better strategy to use in any
+particular case, based on the explanations below.
+
+When we speak of free software, we are referring to freedom of use, not
+price. Our General Public Licenses are designed to make sure that you have
+the freedom to distribute copies of free software (and charge for this
+service if you wish); that you receive source code or can get it if you
+want it; that you can change the software and use pieces of it in new free
+programs; and that you are informed that you can do these things.
+
+To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights. These restrictions translate to certain responsibilities for you if
+you distribute copies of the library or if you modify it.
+
+For example, if you distribute copies of the library, whether gratis or for
+a fee, you must give the recipients all the rights that we gave you. You
+must make sure that they, too, receive or can get the source code. If you
+link other code with the library, you must provide complete object files to
+the recipients, so that they can relink them with the library after making
+changes to the library and recompiling it. And you must show them these
+terms so they know their rights.
+
+We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+To protect each distributor, we want to make it very clear that there is no
+warranty for the free library. Also, if the library is modified by someone
+else and passed on, the recipients should know that what they have is not
+the original version, so that the original author's reputation will not be
+affected by problems that might be introduced by others.
+
+Finally, software patents pose a constant threat to the existence of any
+free program. We wish to make sure that a company cannot effectively
+restrict the users of a free program by obtaining a restrictive license
+from a patent holder. Therefore, we insist that any patent license obtained
+for a version of the library must be consistent with the full freedom of
+use specified in this license.
+
+Most GNU software, including some libraries, is covered by the ordinary GNU
+General Public License. This license, the GNU Lesser General Public
+License, applies to certain designated libraries, and is quite different
+from the ordinary General Public License. We use this license for certain
+libraries in order to permit linking those libraries into non-free
+programs.
+
+When a program is linked with a library, whether statically or using a
+shared library, the combination of the two is legally speaking a combined
+work, a derivative of the original library. The ordinary General Public
+License therefore permits such linking only if the entire combination fits
+its criteria of freedom. The Lesser General Public License permits more lax
+criteria for linking other code with the library.
+
+We call this license the "Lesser" General Public License because it does
+Less to protect the user's freedom than the ordinary General Public
+License. It also provides other free software developers Less of an
+advantage over competing non-free programs. These disadvantages are the
+reason we use the ordinary General Public License for many
+libraries. However, the Lesser license provides advantages in certain
+special circumstances.
+
+For example, on rare occasions, there may be a special need to encourage
+the widest possible use of a certain library, so that it becomes a de-facto
+standard. To achieve this, non-free programs must be allowed to use the
+library. A more frequent case is that a free library does the same job as
+widely used non-free libraries. In this case, there is little to gain by
+limiting the free library to free software only, so we use the Lesser
+General Public License.
+
+In other cases, permission to use a particular library in non-free programs
+enables a greater number of people to use a large body of free
+software. For example, permission to use the GNU C Library in non-free
+programs enables many more people to use the whole GNU operating system, as
+well as its variant, the GNU/Linux operating system.
+
+Although the Lesser General Public License is Less protective of the users'
+freedom, it does ensure that the user of a program that is linked with the
+Library has the freedom and the wherewithal to run that program using a
+modified version of the Library.
+
+The precise terms and conditions for copying, distribution and modification
+follow. Pay close attention to the difference between a "work based on the
+library" and a "work that uses the library". The former contains code
+derived from the library, whereas the latter must be combined with the
+library in order to run.
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. This License Agreement applies to any software library or other program
+   which contains a notice placed by the copyright holder or other
+   authorized party saying it may be distributed under the terms of this
+   Lesser General Public License (also called "this License"). Each
+   licensee is addressed as "you".
+
+   A "library" means a collection of software functions and/or data
+   prepared so as to be conveniently linked with application programs
+   (which use some of those functions and data) to form executables.
+
+   The "Library", below, refers to any such software library or work which
+   has been distributed under these terms. A "work based on the Library"
+   means either the Library or any derivative work under copyright law:
+   that is to say, a work containing the Library or a portion of it, either
+   verbatim or with modifications and/or translated straightforwardly into
+   another language. (Hereinafter, translation is included without
+   limitation in the term "modification".)
+
+   "Source code" for a work means the preferred form of the work for making
+   modifications to it. For a library, complete source code means all the
+   source code for all modules it contains, plus any associated interface
+   definition files, plus the scripts used to control compilation and
+   installation of the library.
+
+    Activities other than copying, distribution and modification are not
+    covered by this License; they are outside its scope. The act of running
+    a program using the Library is not restricted, and output from such a
+    program is covered only if its contents constitute a work based on the
+    Library (independent of the use of the Library in a tool for writing
+    it). Whether that is true depends on what the Library does and what the
+    program that uses the Library does.
+
+1. You may copy and distribute verbatim copies of the Library's complete
+   source code as you receive it, in any medium, provided that you
+   conspicuously and appropriately publish on each copy an appropriate
+   copyright notice and disclaimer of warranty; keep intact all the notices
+   that refer to this License and to the absence of any warranty; and
+   distribute a copy of this License along with the Library.
+
+   You may charge a fee for the physical act of transferring a copy, and
+   you may at your option offer warranty protection in exchange for a fee.
+
+2. You may modify your copy or copies of the Library or any portion of it,
+   thus forming a work based on the Library, and copy and distribute such
+   modifications or work under the terms of Section 1 above, provided that
+   you also meet all of these conditions:
+
+   a) The modified work must itself be a software library.
+
+   b) You must cause the files modified to carry prominent notices stating
+      that you changed the files and the date of any change.
+
+   c) You must cause the whole of the work to be licensed at no charge to
+      all third parties under the terms of this License.
+
+   d) If a facility in the modified Library refers to a function or a table
+      of data to be supplied by an application program that uses the
+      facility, other than as an argument passed when the facility is
+      invoked, then you must make a good faith effort to ensure that, in
+      the event an application does not supply such function or table, the
+      facility still operates, and performs whatever part of its purpose
+      remains meaningful.
+
+   (For example, a function in a library to compute square roots has a
+    purpose that is entirely well-defined independent of the
+    application. Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must be
+    optional: if the application does not supply it, the square root
+    function must still compute square roots.)
+
+   These requirements apply to the modified work as a whole. If
+   identifiable sections of that work are not derived from the Library, and
+   can be reasonably considered independent and separate works in
+   themselves, then this License, and its terms, do not apply to those
+   sections when you distribute them as separate works. But when you
+   distribute the same sections as part of a whole which is a work based on
+   the Library, the distribution of the whole must be on the terms of this
+   License, whose permissions for other licensees extend to the entire
+   whole, and thus to each and every part regardless of who wrote it.
+
+   Thus, it is not the intent of this section to claim rights or contest
+   your rights to work written entirely by you; rather, the intent is to
+   exercise the right to control the distribution of derivative or
+   collective works based on the Library.
+
+   In addition, mere aggregation of another work not based on the Library
+   with the Library (or with a work based on the Library) on a volume of a
+   storage or distribution medium does not bring the other work under the
+   scope of this License.
+
+3. You may opt to apply the terms of the ordinary GNU General Public
+   License instead of this License to a given copy of the Library. To do
+   this, you must alter all the notices that refer to this License, so that
+   they refer to the ordinary GNU General Public License, version 2,
+   instead of to this License. (If a newer version than version 2 of the
+   ordinary GNU General Public License has appeared, then you can specify
+   that version instead if you wish.) Do not make any other change in these
+   notices.
+
+   Once this change is made in a given copy, it is irreversible for that
+   copy, so the ordinary GNU General Public License applies to all
+   subsequent copies and derivative works made from that copy.
+
+   This option is useful when you wish to copy part of the code of the
+   Library into a program that is not a library.
+
+4. You may copy and distribute the Library (or a portion or derivative of
+   it, under Section 2) in object code or executable form under the terms
+   of Sections 1 and 2 above provided that you accompany it with the
+   complete corresponding machine-readable source code, which must be
+   distributed under the terms of Sections 1 and 2 above on a medium
+   customarily used for software interchange.
+
+   If distribution of object code is made by offering access to copy from a
+   designated place, then offering equivalent access to copy the source
+   code from the same place satisfies the requirement to distribute the
+   source code, even though third parties are not compelled to copy the
+   source along with the object code.
+
+5. A program that contains no derivative of any portion of the Library, but
+   is designed to work with the Library by being compiled or linked with
+   it, is called a "work that uses the Library". Such a work, in isolation,
+   is not a derivative work of the Library, and therefore falls outside the
+   scope of this License.
+
+   However, linking a "work that uses the Library" with the Library creates
+   an executable that is a derivative of the Library (because it contains
+   portions of the Library), rather than a "work that uses the
+   library". The executable is therefore covered by this License. Section 6
+   states terms for distribution of such executables.
+
+   When a "work that uses the Library" uses material from a header file
+   that is part of the Library, the object code for the work may be a
+   derivative work of the Library even though the source code is
+   not. Whether this is true is especially significant if the work can be
+   linked without the Library, or if the work is itself a library. The
+   threshold for this to be true is not precisely defined by law.
+
+   If such an object file uses only numerical parameters, data structure
+   layouts and accessors, and small macros and small inline functions (ten
+   lines or less in length), then the use of the object file is
+   unrestricted, regardless of whether it is legally a derivative
+   work. (Executables containing this object code plus portions of the
+   Library will still fall under Section 6.)
+
+   Otherwise, if the work is a derivative of the Library, you may
+   distribute the object code for the work under the terms of Section
+   6. Any executables containing that work also fall under Section 6,
+   whether or not they are linked directly with the Library itself.
+
+6. As an exception to the Sections above, you may also combine or link a
+   "work that uses the Library" with the Library to produce a work
+   containing portions of the Library, and distribute that work under terms
+   of your choice, provided that the terms permit modification of the work
+   for the customer's own use and reverse engineering for debugging such
+   modifications.
+
+   You must give prominent notice with each copy of the work that the
+   Library is used in it and that the Library and its use are covered by
+   this License. You must supply a copy of this License. If the work during
+   execution displays copyright notices, you must include the copyright
+   notice for the Library among them, as well as a reference directing the
+   user to the copy of this License. Also, you must do one of these things:
+
+   a) Accompany the work with the complete corresponding machine-readable
+      source code for the Library including whatever changes were used in
+      the work (which must be distributed under Sections 1 and 2 above);
+      and, if the work is an executable linked with the Library, with the
+      complete machine-readable "work that uses the Library", as object
+      code and/or source code, so that the user can modify the Library and
+      then relink to produce a modified executable containing the modified
+      Library. (It is understood that the user who changes the contents of
+      definitions files in the Library will not necessarily be able to
+      recompile the application to use the modified definitions.)
+
+   b) Use a suitable shared library mechanism for linking with the
+      Library. A suitable mechanism is one that (1) uses at run time a copy
+      of the library already present on the user's computer system, rather
+      than copying library functions into the executable, and (2) will
+      operate properly with a modified version of the library, if the user
+      installs one, as long as the modified version is interface-compatible
+      with the version that the work was made with.
+
+   c) Accompany the work with a written offer, valid for at least three
+      years, to give the same user the materials specified in Subsection
+      6a, above, for a charge no more than the cost of performing this
+      distribution.
+
+   d) If distribution of the work is made by offering access to copy from a
+      designated place, offer equivalent access to copy the above specified
+      materials from the same place.
+
+   e) Verify that the user has already received a copy of these materials
+      or that you have already sent this user a copy.
+
+   For an executable, the required form of the "work that uses the Library"
+   must include any data and utility programs needed for reproducing the
+   executable from it. However, as a special exception, the materials to be
+   distributed need not include anything that is normally distributed (in
+   either source or binary form) with the major components (compiler,
+   kernel, and so on) of the operating system on which the executable runs,
+   unless that component itself accompanies the executable.
+
+   It may happen that this requirement contradicts the license restrictions
+   of other proprietary libraries that do not normally accompany the
+   operating system. Such a contradiction means you cannot use both them
+   and the Library together in an executable that you distribute.
+
+7. You may place library facilities that are a work based on the Library
+   side-by-side in a single library together with other library facilities
+   not covered by this License, and distribute such a combined library,
+   provided that the separate distribution of the work based on the Library
+   and of the other library facilities is otherwise permitted, and provided
+   that you do these two things:
+
+   a) Accompany the combined library with a copy of the same work based on
+      the Library, uncombined with any other library facilities. This must
+      be distributed under the terms of the Sections above.
+
+   b) Give prominent notice with the combined library of the fact that part
+      of it is a work based on the Library, and explaining where to find
+      the accompanying uncombined form of the same work.
+
+8. You may not copy, modify, sublicense, link with, or distribute the
+   Library except as expressly provided under this License. Any attempt
+   otherwise to copy, modify, sublicense, link with, or distribute the
+   Library is void, and will automatically terminate your rights under this
+   License. However, parties who have received copies, or rights, from you
+   under this License will not have their licenses terminated so long as
+   such parties remain in full compliance.
+
+9. You are not required to accept this License, since you have not signed
+   it. However, nothing else grants you permission to modify or distribute
+   the Library or its derivative works. These actions are prohibited by law
+   if you do not accept this License. Therefore, by modifying or
+   distributing the Library (or any work based on the Library), you
+   indicate your acceptance of this License to do so, and all its terms and
+   conditions for copying, distributing or modifying the Library or works
+   based on it.
+
+10. Each time you redistribute the Library (or any work based on the
+    Library), the recipient automatically receives a license from the
+    original licensor to copy, distribute, link with or modify the Library
+    subject to these terms and conditions. You may not impose any further
+    restrictions on the recipients' exercise of the rights granted
+    herein. You are not responsible for enforcing compliance by third
+    parties with this License.
+
+11. If, as a consequence of a court judgment or allegation of patent
+    infringement or for any other reason (not limited to patent issues),
+    conditions are imposed on you (whether by court order, agreement or
+    otherwise) that contradict the conditions of this License, they do not
+    excuse you from the conditions of this License. If you cannot
+    distribute so as to satisfy simultaneously your obligations under this
+    License and any other pertinent obligations, then as a consequence you
+    may not distribute the Library at all. For example, if a patent license
+    would not permit royalty-free redistribution of the Library by all
+    those who receive copies directly or indirectly through you, then the
+    only way you could satisfy both it and this License would be to refrain
+    entirely from distribution of the Library.
+
+    If any portion of this section is held invalid or unenforceable under
+    any particular circumstance, the balance of the section is intended to
+    apply, and the section as a whole is intended to apply in other
+    circumstances.
+
+    It is not the purpose of this section to induce you to infringe any
+    patents or other property right claims or to contest validity of any
+    such claims; this section has the sole purpose of protecting the
+    integrity of the free software distribution system which is implemented
+    by public license practices. Many people have made generous
+    contributions to the wide range of software distributed through that
+    system in reliance on consistent application of that system; it is up
+    to the author/donor to decide if he or she is willing to distribute
+    software through any other system and a licensee cannot impose that
+    choice.
+
+    This section is intended to make thoroughly clear what is believed to
+    be a consequence of the rest of this License.
+
+12. If the distribution and/or use of the Library is restricted in certain
+    countries either by patents or by copyrighted interfaces, the original
+    copyright holder who places the Library under this License may add an
+    explicit geographical distribution limitation excluding those
+    countries, so that distribution is permitted only in or among countries
+    not thus excluded. In such case, this License incorporates the
+    limitation as if written in the body of this License.
+
+13. The Free Software Foundation may publish revised and/or new versions of
+    the Lesser General Public License from time to time. Such new versions
+    will be similar in spirit to the present version, but may differ in
+    detail to address new problems or concerns.
+
+    Each version is given a distinguishing version number. If the Library
+    specifies a version number of this License which applies to it and "any
+    later version", you have the option of following the terms and
+    conditions either of that version or of any later version published by
+    the Free Software Foundation. If the Library does not specify a license
+    version number, you may choose any version ever published by the Free
+    Software Foundation.
+
+14. If you wish to incorporate parts of the Library into other free
+    programs whose distribution conditions are incompatible with these,
+    write to the author to ask for permission. For software which is
+    copyrighted by the Free Software Foundation, write to the Free Software
+    Foundation; we sometimes make exceptions for this. Our decision will be
+    guided by the two goals of preserving the free status of all
+    derivatives of our free software and of promoting the sharing and reuse
+    of software generally.
+
+NO WARRANTY
+
+15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+    FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+    OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+    PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER
+    EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE
+    ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH
+    YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL
+    NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+    WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+    REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR
+    DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL
+    DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY
+    (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED
+    INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF
+    THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR
+    OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+END OF TERMS AND CONDITIONS
+
+How to Apply These Terms to Your New Libraries
+
+If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change. You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+To apply these terms, attach the following notices to the library. It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+one line to give the library's name and an idea of what it does.
+Copyright (C) year name of author
+
+This library is free software; you can redistribute it and/or modify it
+under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or (at
+your option) any later version.
+
+This library is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
+for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this library; if not, write to the Free Software Foundation,
+Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Also add
+information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary. Here is a sample; alter the names:
+
+Yoyodyne, Inc., hereby disclaims all copyright interest in
+the library `Frob' (a library for tweaking knobs) written
+by James Random Hacker.
+
+signature of Ty Coon, 1 April 1990
+Ty Coon, President of Vice
+That's all there is to it!
diff --git a/LICENSES/preferred/MIT b/LICENSES/preferred/MIT
new file mode 100644
index 000000000000..f33a68ceb3ea
--- /dev/null
+++ b/LICENSES/preferred/MIT
@@ -0,0 +1,30 @@
+Valid-License-Identifier: MIT
+SPDX-URL: https://spdx.org/licenses/MIT.html
+Usage-Guide:
+  To use the MIT License put the following SPDX tag/value pair into a
+  comment according to the placement guidelines in the licensing rules
+  documentation:
+    SPDX-License-Identifier: MIT
+License-Text:
+
+MIT License
+
+Copyright (c) <year> <copyright holders>
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/MAINTAINERS b/MAINTAINERS
index 845fc25812f1..6e950b8b4a41 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -270,6 +270,7 @@ ACCES 104-QUAD-8 IIO DRIVER
 M:	William Breathitt Gray <vilhelm.gray@gmail.com>
 L:	linux-iio@vger.kernel.org
 S:	Maintained
+F:	Documentation/ABI/testing/sysfs-bus-iio-counter-104-quad-8
 F:	drivers/iio/counter/104-quad-8.c
 
 ACCES PCI-IDIO-16 GPIO DRIVER
@@ -278,6 +279,12 @@ L:	linux-gpio@vger.kernel.org
 S:	Maintained
 F:	drivers/gpio/gpio-pci-idio-16.c
 
+ACCES PCIe-IDIO-24 GPIO DRIVER
+M:	William Breathitt Gray <vilhelm.gray@gmail.com>
+L:	linux-gpio@vger.kernel.org
+S:	Maintained
+F:	drivers/gpio/gpio-pcie-idio-24.c
+
 ACENIC DRIVER
 M:	Jes Sorensen <jes@trained-monkey.org>
 L:	linux-acenic@sunsite.dk
@@ -329,7 +336,7 @@ F:	drivers/acpi/apei/
 
 ACPI COMPONENT ARCHITECTURE (ACPICA)
 M:	Robert Moore <robert.moore@intel.com>
-M:	Lv Zheng <lv.zheng@intel.com>
+M:	Erik Schmauss <erik.schmauss@intel.com>
 M:	"Rafael J. Wysocki" <rafael.j.wysocki@intel.com>
 L:	linux-acpi@vger.kernel.org
 L:	devel@acpica.org
@@ -853,6 +860,8 @@ M:	Michael Hennerich <Michael.Hennerich@analog.com>
 W:	http://wiki.analog.com/
 W:	http://ez.analog.com/community/linux-device-drivers
 S:	Supported
+F:	Documentation/ABI/testing/sysfs-bus-iio-frequency-ad9523
+F:	Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350
 F:	drivers/iio/*/ad*
 F:	drivers/iio/adc/ltc2497*
 X:	drivers/iio/*/adjd*
@@ -875,6 +884,12 @@ S:	Supported
 F:	drivers/android/
 F:	drivers/staging/android/
 
+ANDROID GOLDFISH PIC DRIVER
+M:	Miodrag Dinic <miodrag.dinic@mips.com>
+S:	Supported
+F:	Documentation/devicetree/bindings/interrupt-controller/google,goldfish-pic.txt
+F:	drivers/irqchip/irq-goldfish-pic.c
+
 ANDROID GOLDFISH RTC DRIVER
 M:	Miodrag Dinic <miodrag.dinic@mips.com>
 S:	Supported
@@ -888,7 +903,6 @@ L:	devel@driverdev.osuosl.org
 S:	Supported
 F:	drivers/staging/android/ion
 F:	drivers/staging/android/uapi/ion.h
-F:	drivers/staging/android/uapi/ion_test.h
 
 AOA (Apple Onboard Audio) ALSA DRIVER
 M:	Johannes Berg <johannes@sipsolutions.net>
@@ -1046,41 +1060,42 @@ ARM PORT
 M:	Russell King <linux@armlinux.org.uk>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 W:	http://www.armlinux.org.uk/
-S:	Maintained
+S:	Odd Fixes
 T:	git git://git.armlinux.org.uk/~rmk/linux-arm.git
 F:	arch/arm/
+X:	arch/arm/boot/dts/
 
 ARM PRIMECELL AACI PL041 DRIVER
 M:	Russell King <linux@armlinux.org.uk>
-S:	Maintained
+S:	Odd Fixes
 F:	sound/arm/aaci.*
 
 ARM PRIMECELL BUS SUPPORT
 M:	Russell King <linux@armlinux.org.uk>
-S:	Maintained
+S:	Odd Fixes
 F:	drivers/amba/
 F:	include/linux/amba/bus.h
 
 ARM PRIMECELL CLCD PL110 DRIVER
 M:	Russell King <linux@armlinux.org.uk>
-S:	Maintained
+S:	Odd Fixes
 F:	drivers/video/fbdev/amba-clcd.*
 
 ARM PRIMECELL KMI PL050 DRIVER
 M:	Russell King <linux@armlinux.org.uk>
-S:	Maintained
+S:	Odd Fixes
 F:	drivers/input/serio/ambakmi.*
 F:	include/linux/amba/kmi.h
 
 ARM PRIMECELL MMCI PL180/1 DRIVER
 M:	Russell King <linux@armlinux.org.uk>
-S:	Maintained
+S:	Odd Fixes
 F:	drivers/mmc/host/mmci.*
 F:	include/linux/amba/mmci.h
 
 ARM PRIMECELL UART PL010 AND PL011 DRIVERS
 M:	Russell King <linux@armlinux.org.uk>
-S:	Maintained
+S:	Odd Fixes
 F:	drivers/tty/serial/amba-pl01*.c
 F:	include/linux/amba/serial.h
 
@@ -1138,7 +1153,7 @@ S:	Maintained
 F:	drivers/clk/sunxi/
 
 ARM/Allwinner sunXi SoC support
-M:	Maxime Ripard <maxime.ripard@free-electrons.com>
+M:	Maxime Ripard <maxime.ripard@bootlin.com>
 M:	Chen-Yu Tsai <wens@csie.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
@@ -1224,7 +1239,7 @@ F:	drivers/clk/at91
 
 ARM/ATMEL AT91RM9200, AT91SAM9 AND SAMA5 SOC SUPPORT
 M:	Nicolas Ferre <nicolas.ferre@microchip.com>
-M:	Alexandre Belloni <alexandre.belloni@free-electrons.com>
+M:	Alexandre Belloni <alexandre.belloni@bootlin.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 W:	http://www.linux4sam.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/nferre/linux-at91.git
@@ -1263,6 +1278,12 @@ L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Supported
 F:	drivers/net/ethernet/cavium/thunder/
 
+ARM/CIRRUS LOGIC BK3 MACHINE SUPPORT
+M:	Lukasz Majewski <lukma@denx.de>
+L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
+S:	Maintained
+F:	arch/arm/mach-ep93xx/ts72xx.c
+
 ARM/CIRRUS LOGIC CLPS711X ARM ARCHITECTURE
 M:	Alexander Shiyan <shc_work@mail.ru>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -1287,7 +1308,6 @@ M:	Russell King <linux@armlinux.org.uk>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 T:	git git://git.armlinux.org.uk/~rmk/linux-arm.git clkdev
-F:	arch/arm/include/asm/clkdev.h
 F:	drivers/clk/clkdev.c
 
 ARM/COMPULAB CM-X270/EM-X270 and CM-X300 MACHINE SUPPORT
@@ -1321,7 +1341,8 @@ F:	tools/perf/arch/arm/util/pmu.c
 F:	tools/perf/arch/arm/util/auxtrace.c
 F:	tools/perf/arch/arm/util/cs-etm.c
 F:	tools/perf/arch/arm/util/cs-etm.h
-F:	tools/perf/util/cs-etm.h
+F:	tools/perf/util/cs-etm.*
+F:	tools/perf/util/cs-etm-decoder/*
 
 ARM/CORGI MACHINE SUPPORT
 M:	Richard Purdie <rpurdie@rpsys.net>
@@ -1335,8 +1356,10 @@ T:	git git://github.com/ulli-kroll/linux.git
 S:	Maintained
 F:	Documentation/devicetree/bindings/arm/gemini.txt
 F:	Documentation/devicetree/bindings/pinctrl/cortina,gemini-pinctrl.txt
+F:	Documentation/devicetree/bindings/net/cortina,gemini-ethernet.txt
 F:	Documentation/devicetree/bindings/rtc/faraday,ftrtc010.txt
 F:	arch/arm/mach-gemini/
+F:	drivers/net/ethernet/cortina/
 F:	drivers/pinctrl/pinctrl-gemini.c
 F:	drivers/rtc/rtc-ftrtc010.c
 
@@ -1568,7 +1591,7 @@ ARM/Marvell Dove/MV78xx0/Orion SOC support
 M:	Jason Cooper <jason@lakedaemon.net>
 M:	Andrew Lunn <andrew@lunn.ch>
 M:	Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>
-M:	Gregory Clement <gregory.clement@free-electrons.com>
+M:	Gregory Clement <gregory.clement@bootlin.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 F:	Documentation/devicetree/bindings/soc/dove/
@@ -1582,7 +1605,7 @@ F:	arch/arm/boot/dts/orion5x*
 ARM/Marvell Kirkwood and Armada 370, 375, 38x, 39x, XP, 3700, 7K/8K SOC support
 M:	Jason Cooper <jason@lakedaemon.net>
 M:	Andrew Lunn <andrew@lunn.ch>
-M:	Gregory Clement <gregory.clement@free-electrons.com>
+M:	Gregory Clement <gregory.clement@bootlin.com>
 M:	Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
@@ -1591,6 +1614,7 @@ F:	arch/arm/boot/dts/kirkwood*
 F:	arch/arm/configs/mvebu_*_defconfig
 F:	arch/arm/mach-mvebu/
 F:	arch/arm64/boot/dts/marvell/armada*
+F:	drivers/cpufreq/armada-37xx-cpufreq.c
 F:	drivers/cpufreq/mvebu-cpufreq.c
 F:	drivers/irqchip/irq-armada-370-xp.c
 F:	drivers/irqchip/irq-mvebu-*
@@ -1643,14 +1667,38 @@ ARM/NEC MOBILEPRO 900/c MACHINE SUPPORT
 M:	Michael Petchkovsky <mkpetch@internode.on.net>
 S:	Maintained
 
-ARM/NOMADIK ARCHITECTURE
-M:	Alessandro Rubini <rubini@unipv.it>
+ARM/NOMADIK/U300/Ux500 ARCHITECTURES
 M:	Linus Walleij <linus.walleij@linaro.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 F:	arch/arm/mach-nomadik/
-F:	drivers/pinctrl/nomadik/
+F:	arch/arm/mach-u300/
+F:	arch/arm/mach-ux500/
+F:	arch/arm/boot/dts/ste-*
+F:	drivers/clk/clk-nomadik.c
+F:	drivers/clk/clk-u300.c
+F:	drivers/clocksource/clksrc-dbx500-prcmu.c
+F:	drivers/clocksource/timer-u300.c
+F:	drivers/dma/coh901318*
+F:	drivers/dma/ste_dma40*
+F:	drivers/hwspinlock/u8500_hsem.c
 F:	drivers/i2c/busses/i2c-nomadik.c
+F:	drivers/i2c/busses/i2c-stu300.c
+F:	drivers/mfd/ab3100*
+F:	drivers/mfd/ab8500*
+F:	drivers/mfd/abx500*
+F:	drivers/mfd/dbx500*
+F:	drivers/mfd/db8500*
+F:	drivers/pinctrl/nomadik/
+F:	drivers/pinctrl/pinctrl-coh901*
+F:	drivers/pinctrl/pinctrl-u300.c
+F:	drivers/rtc/rtc-ab3100.c
+F:	drivers/rtc/rtc-ab8500.c
+F:	drivers/rtc/rtc-coh901331.c
+F:	drivers/rtc/rtc-pl031.c
+F:	drivers/watchdog/coh901327_wdt.c
+F:	Documentation/devicetree/bindings/arm/ste-*
+F:	Documentation/devicetree/bindings/arm/ux500/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-nomadik.git
 
 ARM/NUVOTON W90X900 ARM ARCHITECTURE
@@ -1688,9 +1736,7 @@ L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 L:	linux-oxnas@lists.tuxfamily.org (moderated for non-subscribers)
 S:	Maintained
 F:	arch/arm/mach-oxnas/
-F:	arch/arm/boot/dts/ox8*.dtsi
-F:	arch/arm/boot/dts/wd-mbwe.dts
-F:	arch/arm/boot/dts/cloudengines-pogoplug-series-3.dts
+F:	arch/arm/boot/dts/ox8*.dts*
 N:	oxnas
 
 ARM/PALM TREO SUPPORT
@@ -1698,8 +1744,7 @@ M:	Tomas Cech <sleep_walker@suse.com>
 L:	linux-arm-kernel@lists.infradead.org
 W:	http://hackndev.com
 S:	Maintained
-F:	arch/arm/mach-pxa/include/mach/palmtreo.h
-F:	arch/arm/mach-pxa/palmtreo.c
+F:	arch/arm/mach-pxa/palmtreo.*
 
 ARM/PALMTX,PALMT5,PALMLD,PALMTE2,PALMTC SUPPORT
 M:	Marek Vasut <marek.vasut@gmail.com>
@@ -1708,12 +1753,10 @@ W:	http://hackndev.com
 S:	Maintained
 F:	arch/arm/mach-pxa/include/mach/palmtx.h
 F:	arch/arm/mach-pxa/palmtx.c
-F:	arch/arm/mach-pxa/include/mach/palmt5.h
-F:	arch/arm/mach-pxa/palmt5.c
+F:	arch/arm/mach-pxa/palmt5.*
 F:	arch/arm/mach-pxa/include/mach/palmld.h
 F:	arch/arm/mach-pxa/palmld.c
-F:	arch/arm/mach-pxa/include/mach/palmte2.h
-F:	arch/arm/mach-pxa/palmte2.c
+F:	arch/arm/mach-pxa/palmte2.*
 F:	arch/arm/mach-pxa/include/mach/palmtc.h
 F:	arch/arm/mach-pxa/palmtc.c
 
@@ -1722,8 +1765,7 @@ M:	Sergey Lapin <slapin@ossfans.org>
 L:	linux-arm-kernel@lists.infradead.org
 W:	http://hackndev.com
 S:	Maintained
-F:	arch/arm/mach-pxa/include/mach/palmz72.h
-F:	arch/arm/mach-pxa/palmz72.c
+F:	arch/arm/mach-pxa/palmz72.*
 
 ARM/PLEB SUPPORT
 M:	Peter Chubb <pleb@gelato.unsw.edu.au>
@@ -1752,7 +1794,6 @@ F:	drivers/clk/qcom/
 F:	drivers/dma/qcom/
 F:	drivers/soc/qcom/
 F:	drivers/spi/spi-qup.c
-F:	drivers/tty/serial/msm_serial.h
 F:	drivers/tty/serial/msm_serial.c
 F:	drivers/*/pm8???-*
 F:	drivers/mfd/ssbi.c
@@ -1959,14 +2000,17 @@ M:	Maxime Coquelin <mcoquelin.stm32@gmail.com>
 M:	Alexandre Torgue <alexandre.torgue@st.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mcoquelin/stm32.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git stm32-next
 N:	stm32
+F:	arch/arm/boot/dts/stm32*
+F:	arch/arm/mach-stm32/
 F:	drivers/clocksource/armv7m_systick.c
 
 ARM/TANGO ARCHITECTURE
-M:	Marc Gonzalez <marc_gonzalez@sigmadesigns.com>
+M:	Marc Gonzalez <marc.w.gonzalez@free.fr>
+M:	Mans Rullgard <mans@mansr.com>
 L:	linux-arm-kernel@lists.infradead.org
-S:	Maintained
+S:	Odd Fixes
 N:	tango
 
 ARM/TECHNOLOGIC SYSTEMS TS7250 MACHINE SUPPORT
@@ -2030,27 +2074,13 @@ M:	Dmitry Eremin-Solenikov <dbaryshkov@gmail.com>
 M:	Dirk Opfer <dirk@opfer-online.de>
 S:	Maintained
 
-ARM/U300 MACHINE SUPPORT
-M:	Linus Walleij <linus.walleij@linaro.org>
-L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
-S:	Supported
-F:	arch/arm/mach-u300/
-F:	drivers/clocksource/timer-u300.c
-F:	drivers/i2c/busses/i2c-stu300.c
-F:	drivers/rtc/rtc-coh901331.c
-F:	drivers/watchdog/coh901327_wdt.c
-F:	drivers/dma/coh901318*
-F:	drivers/mfd/ab3100*
-F:	drivers/rtc/rtc-ab3100.c
-F:	drivers/rtc/rtc-coh901331.c
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-stericsson.git
-
 ARM/UNIPHIER ARCHITECTURE
 M:	Masahiro Yamada <yamada.masahiro@socionext.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-uniphier.git
 S:	Maintained
 F:	Documentation/devicetree/bindings/gpio/gpio-uniphier.txt
+F:	Documentation/devicetree/bindings/pinctrl/socionext,uniphier-pinctrl.txt
 F:	arch/arm/boot/dts/uniphier*
 F:	arch/arm/include/asm/hardware/cache-uniphier.h
 F:	arch/arm/mach-uniphier/
@@ -2066,24 +2096,6 @@ F:	drivers/reset/reset-uniphier.c
 F:	drivers/tty/serial/8250/8250_uniphier.c
 N:	uniphier
 
-ARM/Ux500 ARM ARCHITECTURE
-M:	Linus Walleij <linus.walleij@linaro.org>
-L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
-S:	Maintained
-F:	arch/arm/mach-ux500/
-F:	drivers/clocksource/clksrc-dbx500-prcmu.c
-F:	drivers/dma/ste_dma40*
-F:	drivers/hwspinlock/u8500_hsem.c
-F:	drivers/mfd/abx500*
-F:	drivers/mfd/ab8500*
-F:	drivers/mfd/dbx500*
-F:	drivers/mfd/db8500*
-F:	drivers/pinctrl/nomadik/pinctrl-ab*
-F:	drivers/pinctrl/nomadik/pinctrl-nomadik*
-F:	drivers/rtc/rtc-ab8500.c
-F:	drivers/rtc/rtc-pl031.c
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-stericsson.git
-
 ARM/Ux500 CLOCK FRAMEWORK SUPPORT
 M:	Ulf Hansson <ulf.hansson@linaro.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -2210,14 +2222,6 @@ L:	linux-leds@vger.kernel.org
 S:	Maintained
 F:	drivers/leds/leds-as3645a.c
 
-AS3645A LED FLASH CONTROLLER DRIVER
-M:	Laurent Pinchart <laurent.pinchart@ideasonboard.com>
-L:	linux-media@vger.kernel.org
-T:	git git://linuxtv.org/media_tree.git
-S:	Maintained
-F:	drivers/media/i2c/as3645a.c
-F:	include/media/i2c/as3645a.h
-
 ASAHI KASEI AK8974 DRIVER
 M:	Linus Walleij <linus.walleij@linaro.org>
 L:	linux-iio@vger.kernel.org
@@ -2270,7 +2274,9 @@ F:	include/linux/async_tx.h
 AT24 EEPROM DRIVER
 M:	Bartosz Golaszewski <brgl@bgdev.pl>
 L:	linux-i2c@vger.kernel.org
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git
 S:	Maintained
+F:	Documentation/devicetree/bindings/eeprom/at24.txt
 F:	drivers/misc/eeprom/at24.c
 F:	include/linux/platform_data/at24.h
 
@@ -2391,13 +2397,6 @@ F:	Documentation/devicetree/bindings/input/atmel,maxtouch.txt
 F:	drivers/input/touchscreen/atmel_mxt_ts.c
 F:	include/linux/platform_data/atmel_mxt_ts.h
 
-ATMEL NAND DRIVER
-M:	Wenyou Yang <wenyou.yang@atmel.com>
-M:	Josh Wu <rainyfeeling@outlook.com>
-L:	linux-mtd@lists.infradead.org
-S:	Supported
-F:	drivers/mtd/nand/atmel/*
-
 ATMEL SAMA5D2 ADC DRIVER
 M:	Ludovic Desroches <ludovic.desroches@microchip.com>
 L:	linux-iio@vger.kernel.org
@@ -2509,6 +2508,8 @@ L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 F:	Documentation/devicetree/bindings/arm/axentia.txt
 F:	arch/arm/boot/dts/at91-linea.dtsi
+F:	arch/arm/boot/dts/at91-natte.dtsi
+F:	arch/arm/boot/dts/at91-nattis-2-natte-2.dts
 F:	arch/arm/boot/dts/at91-tse850-3.dts
 
 AXENTIA ASOC DRIVERS
@@ -2572,6 +2573,7 @@ S:	Maintained
 F:	Documentation/ABI/testing/sysfs-class-net-batman-adv
 F:	Documentation/ABI/testing/sysfs-class-net-mesh
 F:	Documentation/networking/batman-adv.rst
+F:	include/uapi/linux/batadv_packet.h
 F:	include/uapi/linux/batman_adv.h
 F:	net/batman-adv/
 
@@ -2695,7 +2697,6 @@ F:	drivers/mtd/devices/block2mtd.c
 
 BLUETOOTH DRIVERS
 M:	Marcel Holtmann <marcel@holtmann.org>
-M:	Gustavo Padovan <gustavo@padovan.org>
 M:	Johan Hedberg <johan.hedberg@gmail.com>
 L:	linux-bluetooth@vger.kernel.org
 W:	http://www.bluez.org/
@@ -2706,7 +2707,6 @@ F:	drivers/bluetooth/
 
 BLUETOOTH SUBSYSTEM
 M:	Marcel Holtmann <marcel@holtmann.org>
-M:	Gustavo Padovan <gustavo@padovan.org>
 M:	Johan Hedberg <johan.hedberg@gmail.com>
 L:	linux-bluetooth@vger.kernel.org
 W:	http://www.bluez.org/
@@ -2731,12 +2731,16 @@ M:	Alexei Starovoitov <ast@kernel.org>
 M:	Daniel Borkmann <daniel@iogearbox.net>
 L:	netdev@vger.kernel.org
 L:	linux-kernel@vger.kernel.org
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git
 S:	Supported
 F:	arch/x86/net/bpf_jit*
 F:	Documentation/networking/filter.txt
 F:	Documentation/bpf/
 F:	include/linux/bpf*
 F:	include/linux/filter.h
+F:	include/trace/events/bpf.h
+F:	include/trace/events/xdp.h
 F:	include/uapi/linux/bpf*
 F:	include/uapi/linux/filter.h
 F:	kernel/bpf/
@@ -2840,6 +2844,8 @@ S:	Maintained
 F:	arch/arm/mach-bcm/*brcmstb*
 F:	arch/arm/boot/dts/bcm7*.dts*
 F:	drivers/bus/brcmstb_gisb.c
+F:	arch/arm/mm/cache-b15-rac.c
+F:	arch/arm/include/asm/hardware/cache-b15-rac.h
 N:	brcmstb
 
 BROADCOM BMIPS CPUFREQ DRIVER
@@ -3201,7 +3207,7 @@ W:	https://github.com/linux-can
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can.git
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mkl/linux-can-next.git
 S:	Maintained
-F:	Documentation/networking/can.txt
+F:	Documentation/networking/can.rst
 F:	net/can/
 F:	include/linux/can/core.h
 F:	include/uapi/linux/can.h
@@ -3406,8 +3412,8 @@ M:	Arnd Bergmann <arnd@arndb.de>
 M:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git
 S:	Supported
-F:	drivers/char/*
-F:	drivers/misc/*
+F:	drivers/char/
+F:	drivers/misc/
 F:	include/linux/miscdevice.h
 
 CHECKPATCH
@@ -3555,7 +3561,7 @@ F:	drivers/media/platform/coda/
 
 COMMON CLK FRAMEWORK
 M:	Michael Turquette <mturquette@baylibre.com>
-M:	Stephen Boyd <sboyd@codeaurora.org>
+M:	Stephen Boyd <sboyd@kernel.org>
 L:	linux-clk@vger.kernel.org
 Q:	http://patchwork.kernel.org/project/linux-clk/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git
@@ -4133,6 +4139,7 @@ DEVANTECH SRF ULTRASONIC RANGER IIO DRIVER
 M:	Andreas Klinger <ak@it-klinger.de>
 L:	linux-iio@vger.kernel.org
 S:	Maintained
+F:	Documentation/ABI/testing/sysfs-bus-iio-distance-srf08
 F:	drivers/iio/proximity/srf*.c
 
 DEVICE COREDUMP (DEV_COREDUMP)
@@ -4342,10 +4349,12 @@ T:	git git://git.infradead.org/users/hch/dma-mapping.git
 W:	http://git.infradead.org/users/hch/dma-mapping.git
 S:	Supported
 F:	lib/dma-debug.c
-F:	lib/dma-noop.c
+F:	lib/dma-direct.c
 F:	lib/dma-virt.c
 F:	drivers/base/dma-mapping.c
 F:	drivers/base/dma-coherent.c
+F:	include/asm-generic/dma-mapping.h
+F:	include/linux/dma-direct.h
 F:	include/linux/dma-mapping.h
 
 DME1737 HARDWARE MONITOR DRIVER
@@ -4468,6 +4477,12 @@ T:	git git://anongit.freedesktop.org/drm/drm-misc
 S:	Maintained
 F:	drivers/gpu/drm/tve200/
 
+DRM DRIVER FOR ILITEK ILI9225 PANELS
+M:	David Lechner <david@lechnology.com>
+S:	Maintained
+F:	drivers/gpu/drm/tinydrm/ili9225.c
+F:	Documentation/devicetree/bindings/display/ili9225.txt
+
 DRM DRIVER FOR INTEL I810 VIDEO CARDS
 S:	Orphan / Obsolete
 F:	drivers/gpu/drm/i810/
@@ -4554,6 +4569,12 @@ S:	Maintained
 F:	drivers/gpu/drm/tinydrm/st7586.c
 F:	Documentation/devicetree/bindings/display/st7586.txt
 
+DRM DRIVER FOR SITRONIX ST7735R PANELS
+M:	David Lechner <david@lechnology.com>
+S:	Maintained
+F:	drivers/gpu/drm/tinydrm/st7735r.c
+F:	Documentation/devicetree/bindings/display/st7735r.txt
+
 DRM DRIVER FOR TDFX VIDEO CARDS
 S:	Orphan / Obsolete
 F:	drivers/gpu/drm/tdfx/
@@ -4593,7 +4614,7 @@ F:	include/linux/vga*
 
 DRM DRIVERS AND MISC GPU PATCHES
 M:	Daniel Vetter <daniel.vetter@intel.com>
-M:	Jani Nikula <jani.nikula@linux.intel.com>
+M:	Gustavo Padovan <gustavo@padovan.org>
 M:	Sean Paul <seanpaul@chromium.org>
 W:	https://01.org/linuxgraphics/gfx-docs/maintainer-tools/drm-misc.html
 S:	Maintained
@@ -4606,7 +4627,7 @@ F:	include/uapi/drm/drm*
 F:	include/linux/vga*
 
 DRM DRIVERS FOR ALLWINNER A10
-M:	Maxime Ripard  <maxime.ripard@free-electrons.com>
+M:	Maxime Ripard  <maxime.ripard@bootlin.com>
 L:	dri-devel@lists.freedesktop.org
 S:	Supported
 F:	drivers/gpu/drm/sun4i/
@@ -4722,7 +4743,8 @@ F:	Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.txt
 F:	Documentation/devicetree/bindings/display/renesas,du.txt
 
 DRM DRIVERS FOR ROCKCHIP
-M:	Mark Yao <mark.yao@rock-chips.com>
+M:	Sandy Huang <hjc@rock-chips.com>
+M:	Heiko Stübner <heiko@sntech.de>
 L:	dri-devel@lists.freedesktop.org
 S:	Maintained
 F:	drivers/gpu/drm/rockchip/
@@ -4810,6 +4832,15 @@ S:	Maintained
 F:	drivers/gpu/drm/tinydrm/
 F:	include/drm/tinydrm/
 
+DRM TTM SUBSYSTEM
+M:	Christian Koenig <christian.koenig@amd.com>
+M:	Roger He <Hongbo.He@amd.com>
+T:	git git://people.freedesktop.org/~agd5f/linux
+S:	Maintained
+L:	dri-devel@lists.freedesktop.org
+F:	include/drm/ttm/
+F:	drivers/gpu/drm/ttm/
+
 DSBR100 USB FM RADIO DRIVER
 M:	Alexey Klimov <klimov.linux@gmail.com>
 L:	linux-media@vger.kernel.org
@@ -4947,6 +4978,11 @@ S:	Maintained
 F:	lib/dynamic_debug.c
 F:	include/linux/dynamic_debug.h
 
+DYNAMIC INTERRUPT MODERATION
+M:	Tal Gilboa <talgi@mellanox.com>
+S:	Maintained
+F:	include/linux/net_dim.h
+
 DZ DECSTATION DZ11 SERIAL DRIVER
 M:	"Maciej W. Rozycki" <macro@linux-mips.org>
 S:	Maintained
@@ -5147,6 +5183,12 @@ L:	linux-edac@vger.kernel.org
 S:	Maintained
 F:	drivers/edac/skx_edac.c
 
+EDAC-TI
+M:	Tero Kristo <t-kristo@ti.com>
+L:	linux-edac@vger.kernel.org
+S:	Maintained
+F:	drivers/edac/ti_edac.c
+
 EDIROL UA-101/UA-1000 DRIVER
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
@@ -5953,6 +5995,12 @@ L:	linux-input@vger.kernel.org
 S:	Maintained
 F:	drivers/input/touchscreen/goodix.c
 
+GPD POCKET FAN DRIVER
+M:	Hans de Goede <hdegoede@redhat.com>
+L:	platform-driver-x86@vger.kernel.org
+S:	Maintained
+F:	drivers/platform/x86/gpd-pocket-fan.c
+
 GPIO ACPI SUPPORT
 M:	Mika Westerberg <mika.westerberg@linux.intel.com>
 M:	Andy Shevchenko <andriy.shevchenko@linux.intel.com>
@@ -5970,6 +6018,7 @@ F:	drivers/media/rc/gpio-ir-tx.c
 
 GPIO MOCKUP DRIVER
 M:	Bamvor Jian Zhang <bamvor.zhangjian@linaro.org>
+R:	Bartosz Golaszewski <brgl@bgdev.pl>
 L:	linux-gpio@vger.kernel.org
 S:	Maintained
 F:	drivers/gpio/gpio-mockup.c
@@ -6525,6 +6574,12 @@ F:	drivers/i2c/i2c-mux.c
 F:	drivers/i2c/muxes/
 F:	include/linux/i2c-mux.h
 
+I2C MV64XXX MARVELL AND ALLWINNER DRIVER
+M:	Gregory CLEMENT <gregory.clement@free-electrons.com>
+L:	linux-i2c@vger.kernel.org
+S:	Maintained
+F:	drivers/i2c/busses/i2c-mv64xxx.c
+
 I2C OVER PARALLEL PORT
 M:	Jean Delvare <jdelvare@suse.com>
 L:	linux-i2c@vger.kernel.org
@@ -6824,6 +6879,8 @@ R:	Peter Meerwald-Stadler <pmeerw@pmeerw.net>
 L:	linux-iio@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git
 S:	Maintained
+F:	Documentation/ABI/testing/configfs-iio*
+F:	Documentation/ABI/testing/sysfs-bus-iio*
 F:	Documentation/devicetree/bindings/iio/
 F:	drivers/iio/
 F:	drivers/staging/iio/
@@ -6881,9 +6938,9 @@ INFINIBAND SUBSYSTEM
 M:	Doug Ledford <dledford@redhat.com>
 M:	Jason Gunthorpe <jgg@mellanox.com>
 L:	linux-rdma@vger.kernel.org
-W:	http://www.openfabrics.org/
+W:	https://github.com/linux-rdma/rdma-core
 Q:	http://patchwork.kernel.org/project/linux-rdma/list/
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git
 S:	Supported
 F:	Documentation/devicetree/bindings/infiniband/
 F:	Documentation/infiniband/
@@ -7024,7 +7081,7 @@ M:	Zhi Wang <zhi.a.wang@intel.com>
 L:	intel-gvt-dev@lists.freedesktop.org
 L:	intel-gfx@lists.freedesktop.org
 W:	https://01.org/igvt-g
-T:	git https://github.com/01org/gvt-linux.git
+T:	git https://github.com/intel/gvt-linux.git
 S:	Supported
 F:	drivers/gpu/drm/i915/gvt/
 
@@ -7071,6 +7128,14 @@ R:	Dan Williams <dan.j.williams@intel.com>
 S:	Odd fixes
 F:	drivers/dma/iop-adma.c
 
+INTEL IPU3 CSI-2 CIO2 DRIVER
+M:	Yong Zhi <yong.zhi@intel.com>
+M:	Sakari Ailus <sakari.ailus@linux.intel.com>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	drivers/media/pci/intel/ipu3/
+F:	Documentation/media/uapi/v4l/pixfmt-srggb10-ipu3.rst
+
 INTEL IXP4XX QMGR, NPE, ETHERNET and HSS SUPPORT
 M:	Krzysztof Halasa <khalasa@piap.pl>
 S:	Maintained
@@ -7305,17 +7370,16 @@ F:	drivers/tty/ipwireless/
 
 IPX NETWORK LAYER
 L:	netdev@vger.kernel.org
-S:	Odd fixes
-F:	include/net/ipx.h
+S:	Obsolete
 F:	include/uapi/linux/ipx.h
-F:	net/ipx/
+F:	drivers/staging/ipx/
 
 IRDA SUBSYSTEM
 M:	Samuel Ortiz <samuel@sortiz.org>
 L:	irda-users@lists.sourceforge.net (subscribers-only)
 L:	netdev@vger.kernel.org
 W:	http://irda.sourceforge.net/
-S:	Maintained
+S:	Obsolete
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/sameo/irda-2.6.git
 F:	Documentation/networking/irda.txt
 F:	drivers/staging/irda/
@@ -7539,8 +7603,10 @@ F:	mm/kasan/
 F:	scripts/Makefile.kasan
 
 KCONFIG
+M:	Masahiro Yamada <yamada.masahiro@socionext.com>
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/masahiroy/linux-kbuild.git kconfig
 L:	linux-kbuild@vger.kernel.org
-S:	Orphan
+S:	Maintained
 F:	Documentation/kbuild/kconfig-language.txt
 F:	scripts/kconfig/
 
@@ -7687,7 +7753,9 @@ F:	arch/powerpc/kernel/kvm*
 
 KERNEL VIRTUAL MACHINE for s390 (KVM/s390)
 M:	Christian Borntraeger <borntraeger@de.ibm.com>
-M:	Cornelia Huck <cohuck@redhat.com>
+M:	Janosch Frank <frankja@linux.vnet.ibm.com>
+R:	David Hildenbrand <david@redhat.com>
+R:	Cornelia Huck <cohuck@redhat.com>
 L:	linux-s390@vger.kernel.org
 W:	http://www.ibm.com/developerworks/linux/linux390/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git
@@ -7739,6 +7807,7 @@ F:	include/keys/encrypted-type.h
 F:	security/keys/encrypted-keys/
 
 KEYS-TRUSTED
+M:	James Bottomley <jejb@linux.vnet.ibm.com>
 M:	Mimi Zohar <zohar@linux.vnet.ibm.com>
 L:	linux-integrity@vger.kernel.org
 L:	keyrings@vger.kernel.org
@@ -7845,7 +7914,6 @@ S:	Maintained
 F:	scripts/leaking_addresses.pl
 
 LED SUBSYSTEM
-M:	Richard Purdie <rpurdie@rpsys.net>
 M:	Jacek Anaszewski <jacek.anaszewski@gmail.com>
 M:	Pavel Machek <pavel@ucw.cz>
 L:	linux-leds@vger.kernel.org
@@ -8190,6 +8258,7 @@ F:	arch/*/include/asm/rwsem.h
 F:	include/linux/seqlock.h
 F:	lib/locking*.[ch]
 F:	kernel/locking/
+X:	kernel/locking/locktorture.c
 
 LOGICAL DISK MANAGER SUPPORT (LDM, Windows 2000/XP/Vista Dynamic Disks)
 M:	"Richard Russon (FlatCap)" <ldm@flatcap.org>
@@ -8366,7 +8435,7 @@ S:	Orphan
 F:	drivers/net/wireless/marvell/libertas/
 
 MARVELL MACCHIATOBIN SUPPORT
-M:	Russell King <rmk@armlinux.org.uk>
+M:	Russell King <linux@armlinux.org.uk>
 L:	linux-arm-kernel@lists.infradead.org
 S:	Maintained
 F:	arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts
@@ -8379,7 +8448,7 @@ F:	drivers/net/ethernet/marvell/mv643xx_eth.*
 F:	include/linux/mv643xx.h
 
 MARVELL MV88X3310 PHY DRIVER
-M:	Russell King <rmk@armlinux.org.uk>
+M:	Russell King <linux@armlinux.org.uk>
 L:	netdev@vger.kernel.org
 S:	Maintained
 F:	drivers/net/phy/marvell10g.c
@@ -8405,6 +8474,13 @@ L:	linux-wireless@vger.kernel.org
 S:	Odd Fixes
 F:	drivers/net/wireless/marvell/mwl8k.c
 
+MARVELL NAND CONTROLLER DRIVER
+M:	Miquel Raynal <miquel.raynal@free-electrons.com>
+L:	linux-mtd@lists.infradead.org
+S:	Maintained
+F:	drivers/mtd/nand/marvell_nand.c
+F:	Documentation/devicetree/bindings/mtd/marvell-nand.txt
+
 MARVELL SOC MMC/SD/SDIO CONTROLLER DRIVER
 M:	Nicolas Pitre <nico@fluxnic.net>
 S:	Odd Fixes
@@ -8676,6 +8752,15 @@ T:	git git://linuxtv.org/media_tree.git
 S:	Maintained
 F:	drivers/media/dvb-frontends/stv6111*
 
+MEDIA DRIVERS FOR NVIDIA TEGRA - VDE
+M:	Dmitry Osipenko <digetx@gmail.com>
+L:	linux-media@vger.kernel.org
+L:	linux-tegra@vger.kernel.org
+T:	git git://linuxtv.org/media_tree.git
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/nvidia,tegra-vde.txt
+F:	drivers/staging/media/tegra-vde/
+
 MEDIA INPUT INFRASTRUCTURE (V4L/DVB)
 M:	Mauro Carvalho Chehab <mchehab@s-opensource.com>
 M:	Mauro Carvalho Chehab <mchehab@kernel.org>
@@ -8719,6 +8804,13 @@ L:	netdev@vger.kernel.org
 S:	Maintained
 F:	drivers/net/ethernet/mediatek/
 
+MEDIATEK SWITCH DRIVER
+M:	Sean Wang <sean.wang@mediatek.com>
+L:	netdev@vger.kernel.org
+S:	Maintained
+F:	drivers/net/dsa/mt7530.*
+F:	net/dsa/tag_mtk.c
+
 MEDIATEK JPEG DRIVER
 M:	Rick Chang <rick.chang@mediatek.com>
 M:	Bin Liu <bin.liu@mediatek.com>
@@ -8850,12 +8942,13 @@ W:	http://www.mellanox.com
 Q:	http://patchwork.ozlabs.org/project/netdev/list/
 F:	drivers/net/ethernet/mellanox/mlxfw/
 
-MELLANOX MLX CPLD HOTPLUG DRIVER
+MELLANOX HARDWARE PLATFORM SUPPORT
+M:	Andy Shevchenko <andy@infradead.org>
+M:	Darren Hart <dvhart@infradead.org>
 M:	Vadim Pasternak <vadimp@mellanox.com>
 L:	platform-driver-x86@vger.kernel.org
 S:	Supported
-F:	drivers/platform/x86/mlxcpld-hotplug.c
-F:	include/linux/platform_data/mlxcpld-hotplug.h
+F:	drivers/platform/mellanox/
 
 MELLANOX MLX4 core VPI driver
 M:	Tariq Toukan <tariqt@mellanox.com>
@@ -8929,6 +9022,7 @@ L:	linux-kernel@vger.kernel.org
 S:	Supported
 F:	kernel/sched/membarrier.c
 F:	include/uapi/linux/membarrier.h
+F:	arch/powerpc/include/asm/membarrier.h
 
 MEMORY MANAGEMENT
 L:	linux-mm@kvack.org
@@ -8952,7 +9046,7 @@ L:	linux-mtd@lists.infradead.org
 W:	http://www.linux-mtd.infradead.org/
 Q:	http://patchwork.ozlabs.org/project/linux-mtd/list/
 T:	git git://git.infradead.org/linux-mtd.git master
-T:	git git://git.infradead.org/l2-mtd.git master
+T:	git git://git.infradead.org/linux-mtd.git mtd/next
 S:	Maintained
 F:	Documentation/devicetree/bindings/mtd/
 F:	drivers/mtd/
@@ -9041,6 +9135,14 @@ F:	drivers/media/platform/atmel/atmel-isc.c
 F:	drivers/media/platform/atmel/atmel-isc-regs.h
 F:	devicetree/bindings/media/atmel-isc.txt
 
+MICROCHIP / ATMEL NAND DRIVER
+M:	Wenyou Yang <wenyou.yang@microchip.com>
+M:	Josh Wu <rainyfeeling@outlook.com>
+L:	linux-mtd@lists.infradead.org
+S:	Supported
+F:	drivers/mtd/nand/atmel/*
+F:	Documentation/devicetree/bindings/mtd/atmel-nand.txt
+
 MICROCHIP KSZ SERIES ETHERNET SWITCH DRIVER
 M:	Woojung Huh <Woojung.Huh@microchip.com>
 M:	Microchip Linux Driver Support <UNGLinuxDriver@microchip.com>
@@ -9092,6 +9194,7 @@ S:	Supported
 F:	Documentation/devicetree/bindings/mips/
 F:	Documentation/mips/
 F:	arch/mips/
+F:	drivers/platform/mips/
 
 MIPS BOSTON DEVELOPMENT BOARD
 M:	Paul Burton <paul.burton@mips.com>
@@ -9107,6 +9210,7 @@ MIPS GENERIC PLATFORM
 M:	Paul Burton <paul.burton@mips.com>
 L:	linux-mips@linux-mips.org
 S:	Supported
+F:	Documentation/devicetree/bindings/power/mti,mips-cpc.txt
 F:	arch/mips/generic/
 F:	arch/mips/tools/generic-board-config.sh
 
@@ -9119,6 +9223,25 @@ F:	arch/mips/include/asm/mach-loongson32/
 F:	drivers/*/*loongson1*
 F:	drivers/*/*/*loongson1*
 
+MIPS/LOONGSON2 ARCHITECTURE
+M:	Jiaxun Yang <jiaxun.yang@flygoat.com>
+L:	linux-mips@linux-mips.org
+S:	Maintained
+F:	arch/mips/loongson64/*{2e/2f}*
+F:	arch/mips/include/asm/mach-loongson64/
+F:	drivers/*/*loongson2*
+F:	drivers/*/*/*loongson2*
+
+MIPS/LOONGSON3 ARCHITECTURE
+M:	Huacai Chen <chenhc@lemote.com>
+L:	linux-mips@linux-mips.org
+S:	Maintained
+F:	arch/mips/loongson64/
+F:	arch/mips/include/asm/mach-loongson64/
+F:	drivers/platform/mips/cpu_hwmon.c
+F:	drivers/*/*loongson3*
+F:	drivers/*/*/*loongson3*
+
 MIPS RINT INSTRUCTION EMULATION
 M:	Aleksandar Markovic <aleksandar.markovic@mips.com>
 L:	linux-mips@linux-mips.org
@@ -9164,7 +9287,6 @@ F:	drivers/media/dvb-frontends/mn88473*
 
 MODULE SUPPORT
 M:	Jessica Yu <jeyu@kernel.org>
-M:	Rusty Russell <rusty@rustcorp.com.au>
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/jeyu/linux.git modules-next
 S:	Maintained
 F:	include/linux/module.h
@@ -9341,7 +9463,7 @@ L:	linux-mtd@lists.infradead.org
 W:	http://www.linux-mtd.infradead.org/
 Q:	http://patchwork.ozlabs.org/project/linux-mtd/list/
 T:	git git://git.infradead.org/linux-mtd.git nand/fixes
-T:	git git://git.infradead.org/l2-mtd.git nand/next
+T:	git git://git.infradead.org/linux-mtd.git nand/next
 S:	Maintained
 F:	drivers/mtd/nand/
 F:	include/linux/mtd/*nand*.h
@@ -9359,8 +9481,8 @@ F:	drivers/net/ethernet/natsemi/natsemi.c
 
 NCP FILESYSTEM
 M:	Petr Vandrovec <petr@vandrovec.name>
-S:	Odd Fixes
-F:	fs/ncpfs/
+S:	Obsolete
+F:	drivers/staging/ncpfs/
 
 NCR 5380 SCSI DRIVERS
 M:	Finn Thain <fthain@telegraphics.com.au>
@@ -9596,6 +9718,11 @@ NETWORKING [WIRELESS]
 L:	linux-wireless@vger.kernel.org
 Q:	http://patchwork.kernel.org/project/linux-wireless/list/
 
+NETDEVSIM
+M:	Jakub Kicinski <jakub.kicinski@netronome.com>
+S:	Maintained
+F:	drivers/net/netdevsim/*
+
 NETXEN (1/10) GbE SUPPORT
 M:	Manish Chopra <manish.chopra@cavium.com>
 M:	Rahul Verma <rahul.verma@cavium.com>
@@ -9708,7 +9835,7 @@ F:	drivers/ntb/hw/amd/
 NTB DRIVER CORE
 M:	Jon Mason <jdmason@kudzu.us>
 M:	Dave Jiang <dave.jiang@intel.com>
-M:	Allen Hubbe <Allen.Hubbe@emc.com>
+M:	Allen Hubbe <allenbh@gmail.com>
 L:	linux-ntb@googlegroups.com
 S:	Supported
 W:	https://github.com/jonmason/ntb/wiki
@@ -9742,6 +9869,15 @@ S:	Supported
 F:	Documentation/filesystems/ntfs.txt
 F:	fs/ntfs/
 
+NUBUS SUBSYSTEM
+M:	Finn Thain <fthain@telegraphics.com.au>
+L:	linux-m68k@lists.linux-m68k.org
+S:	Maintained
+F:	arch/*/include/asm/nubus.h
+F:	drivers/nubus/
+F:	include/linux/nubus.h
+F:	include/uapi/linux/nubus.h
+
 NVIDIA (rivafb and nvidiafb) FRAMEBUFFER DRIVER
 M:	Antonino Daplas <adaplas@gmail.com>
 L:	linux-fbdev@vger.kernel.org
@@ -9790,6 +9926,13 @@ F:	Documentation/ABI/stable/sysfs-bus-nvmem
 F:	include/linux/nvmem-consumer.h
 F:	include/linux/nvmem-provider.h
 
+NXP SGTL5000 DRIVER
+M:	Fabio Estevam <fabio.estevam@nxp.com>
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/sound/sgtl5000.txt
+F:	sound/soc/codecs/sgtl5000*
+
 NXP TDA998X DRM DRIVER
 M:	Russell King <linux@armlinux.org.uk>
 S:	Supported
@@ -9802,6 +9945,7 @@ NXP TFA9879 DRIVER
 M:	Peter Rosin <peda@axentia.se>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
 S:	Maintained
+F:	Documentation/devicetree/bindings/sound/tfa9879.txt
 F:	sound/soc/codecs/tfa9879*
 
 NXP-NCI NFC DRIVER
@@ -9813,9 +9957,22 @@ F:	drivers/nfc/nxp-nci
 
 OBJTOOL
 M:	Josh Poimboeuf <jpoimboe@redhat.com>
+M:	Peter Zijlstra <peterz@infradead.org>
 S:	Supported
 F:	tools/objtool/
 
+OCXL (Open Coherent Accelerator Processor Interface OpenCAPI) DRIVER
+M:	Frederic Barrat <fbarrat@linux.vnet.ibm.com>
+M:	Andrew Donnellan <andrew.donnellan@au1.ibm.com>
+L:	linuxppc-dev@lists.ozlabs.org
+S:	Supported
+F:	arch/powerpc/platforms/powernv/ocxl.c
+F:	arch/powerpc/include/asm/pnv-ocxl.h
+F:	drivers/misc/ocxl/
+F:	include/misc/ocxl*
+F:	include/uapi/misc/ocxl.h
+F:	Documentation/accelerators/ocxl.txt
+
 OMAP AUDIO SUPPORT
 M:	Peter Ujfalusi <peter.ujfalusi@ti.com>
 M:	Jarkko Nikula <jarkko.nikula@bitmer.com>
@@ -10050,6 +10207,14 @@ S:	Maintained
 F:	drivers/media/i2c/ov7670.c
 F:	Documentation/devicetree/bindings/media/i2c/ov7670.txt
 
+OMNIVISION OV7740 SENSOR DRIVER
+M:	Wenyou Yang <wenyou.yang@microchip.com>
+L:	linux-media@vger.kernel.org
+T:	git git://linuxtv.org/media_tree.git
+S:	Maintained
+F:	drivers/media/i2c/ov7740.c
+F:	Documentation/devicetree/bindings/media/i2c/ov7740.txt
+
 ONENAND FLASH DRIVER
 M:	Kyungmin Park <kyungmin.park@samsung.com>
 L:	linux-mtd@lists.infradead.org
@@ -10144,7 +10309,7 @@ F:	include/uapi/linux/openvswitch.h
 OPERATING PERFORMANCE POINTS (OPP)
 M:	Viresh Kumar <vireshk@kernel.org>
 M:	Nishanth Menon <nm@ti.com>
-M:	Stephen Boyd <sboyd@codeaurora.org>
+M:	Stephen Boyd <sboyd@kernel.org>
 L:	linux-pm@vger.kernel.org
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git
@@ -10170,7 +10335,7 @@ F:	drivers/oprofile/
 F:	include/linux/oprofile.h
 
 ORACLE CLUSTER FILESYSTEM 2 (OCFS2)
-M:	Mark Fasheh <mfasheh@versity.com>
+M:	Mark Fasheh <mark@fasheh.com>
 M:	Joel Becker <jlbec@evilplan.org>
 L:	ocfs2-devel@oss.oracle.com (moderated for non-subscribers)
 W:	http://ocfs2.wiki.kernel.org
@@ -10181,7 +10346,8 @@ F:	fs/ocfs2/
 
 ORANGEFS FILESYSTEM
 M:	Mike Marshall <hubcap@omnibond.com>
-L:	pvfs2-developers@beowulf-underground.org (subscribers-only)
+R:	Martin Brandenburg <martin@omnibond.com>
+L:	devel@lists.orangefs.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git
 S:	Supported
 F:	fs/orangefs/
@@ -10279,7 +10445,6 @@ F:	Documentation/parport*.txt
 PARAVIRT_OPS INTERFACE
 M:	Juergen Gross <jgross@suse.com>
 M:	Alok Kataria <akataria@vmware.com>
-M:	Rusty Russell <rusty@rustcorp.com.au>
 L:	virtualization@lists.linux-foundation.org
 S:	Supported
 F:	Documentation/virtual/paravirt_ops.txt
@@ -10398,6 +10563,13 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/pci/pci-armada8k.txt
 F:	drivers/pci/dwc/pcie-armada8k.c
 
+PCI DRIVER FOR CADENCE PCIE IP
+M:	Alan Douglas <adouglas@cadence.com>
+L:	linux-pci@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/pci/cdns,*.txt
+F:	drivers/pci/cadence/pcie-cadence*
+
 PCI DRIVER FOR FREESCALE LAYERSCAPE
 M:	Minghuan Lian <minghuan.Lian@freescale.com>
 M:	Mingkai Hu <mingkai.hu@freescale.com>
@@ -10548,8 +10720,12 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git
 S:	Supported
 F:	Documentation/devicetree/bindings/pci/
 F:	Documentation/PCI/
+F:	drivers/acpi/pci*
 F:	drivers/pci/
+F:	include/asm-generic/pci*
 F:	include/linux/pci*
+F:	include/uapi/linux/pci*
+F:	lib/pci*
 F:	arch/x86/pci/
 F:	arch/x86/kernel/quirks.c
 
@@ -10639,11 +10815,9 @@ S:	Maintained
 F:	drivers/pci/dwc/*spear*
 
 PCMCIA SUBSYSTEM
-P:	Linux PCMCIA Team
-L:	linux-pcmcia@lists.infradead.org
-W:	http://lists.infradead.org/mailman/listinfo/linux-pcmcia
+M:	Dominik Brodowski <linux@dominikbrodowski.net>
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/brodo/pcmcia.git
-S:	Maintained
+S:	Odd Fixes
 F:	Documentation/pcmcia/
 F:	tools/pcmcia/
 F:	drivers/pcmcia/
@@ -10671,6 +10845,7 @@ F:	drivers/platform/x86/peaq-wmi.c
 PER-CPU MEMORY ALLOCATOR
 M:	Tejun Heo <tj@kernel.org>
 M:	Christoph Lameter <cl@linux.com>
+M:	Dennis Zhou <dennisszhou@gmail.com>
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu.git
 S:	Maintained
 F:	include/linux/percpu*.h
@@ -10764,12 +10939,33 @@ L:	linux-gpio@vger.kernel.org
 S:	Supported
 F:	drivers/pinctrl/pinctrl-at91-pio4.*
 
+PIN CONTROLLER - FREESCALE
+M:	Dong Aisheng <aisheng.dong@nxp.com>
+M:	Fabio Estevam <festevam@gmail.com>
+M:	Shawn Guo <shawnguo@kernel.org>
+M:	Stefan Agner <stefan@agner.ch>
+R:	Pengutronix Kernel Team <kernel@pengutronix.de>
+L:	linux-gpio@vger.kernel.org
+S:	Maintained
+F:	drivers/pinctrl/freescale/
+F:	Documentation/devicetree/bindings/pinctrl/fsl,*
+
 PIN CONTROLLER - INTEL
 M:	Mika Westerberg <mika.westerberg@linux.intel.com>
 M:	Heikki Krogerus <heikki.krogerus@linux.intel.com>
 S:	Maintained
 F:	drivers/pinctrl/intel/
 
+PIN CONTROLLER - MEDIATEK
+M:	Sean Wang <sean.wang@mediatek.com>
+L:	linux-mediatek@lists.infradead.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
+F:	Documentation/devicetree/bindings/pinctrl/pinctrl-mt7622.txt
+F:	drivers/pinctrl/mediatek/pinctrl-mtk-common.*
+F:	drivers/pinctrl/mediatek/pinctrl-mt2701.c
+F:	drivers/pinctrl/mediatek/pinctrl-mt7622.c
+
 PIN CONTROLLER - QUALCOMM
 M:	Bjorn Andersson <bjorn.andersson@linaro.org>
 S:	Maintained
@@ -10888,6 +11084,7 @@ F:	include/linux/pm.h
 F:	include/linux/pm_*
 F:	include/linux/powercap.h
 F:	drivers/powercap/
+F:	kernel/configs/nopm.config
 
 POWER STATE COORDINATION INTERFACE (PSCI)
 M:	Mark Rutland <mark.rutland@arm.com>
@@ -11182,7 +11379,8 @@ S:	Maintained
 F:	drivers/firmware/qemu_fw_cfg.c
 
 QIB DRIVER
-M:	Mike Marciniszyn <infinipath@intel.com>
+M:	Dennis Dalessandro <dennis.dalessandro@intel.com>
+M:	Mike Marciniszyn <mike.marciniszyn@intel.com>
 L:	linux-rdma@vger.kernel.org
 S:	Supported
 F:	drivers/infiniband/hw/qib/
@@ -11209,7 +11407,6 @@ F:	include/linux/qed/
 F:	drivers/net/ethernet/qlogic/qede/
 
 QLOGIC QL4xxx RDMA DRIVER
-M:	Ram Amrani <Ram.Amrani@cavium.com>
 M:	Michal Kalderon <Michal.Kalderon@cavium.com>
 M:	Ariel Elior <Ariel.Elior@cavium.com>
 L:	linux-rdma@vger.kernel.org
@@ -11356,6 +11553,7 @@ F:	drivers/net/wireless/quantenna
 RADEON and AMDGPU DRM DRIVERS
 M:	Alex Deucher <alexander.deucher@amd.com>
 M:	Christian König <christian.koenig@amd.com>
+M:	David (ChunMing) Zhou <David1.Zhou@amd.com>
 L:	amd-gfx@lists.freedesktop.org
 T:	git git://people.freedesktop.org/~agd5f/linux
 S:	Supported
@@ -11432,6 +11630,13 @@ S:	Maintained
 F:	Documentation/blockdev/ramdisk.txt
 F:	drivers/block/brd.c
 
+RANCHU VIRTUAL BOARD FOR MIPS
+M:	Miodrag Dinic <miodrag.dinic@mips.com>
+L:	linux-mips@linux-mips.org
+S:	Supported
+F:	arch/mips/generic/board-ranchu.c
+F:	arch/mips/configs/generic/board-ranchu.config
+
 RANDOM NUMBER DRIVER
 M:	"Theodore Ts'o" <tytso@mit.edu>
 S:	Maintained
@@ -11448,15 +11653,6 @@ L:	linux-wireless@vger.kernel.org
 S:	Orphan
 F:	drivers/net/wireless/ray*
 
-RCUTORTURE MODULE
-M:	Josh Triplett <josh@joshtriplett.org>
-M:	"Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
-L:	linux-kernel@vger.kernel.org
-S:	Supported
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git
-F:	Documentation/RCU/torture.txt
-F:	kernel/rcu/rcutorture.c
-
 RCUTORTURE TEST FRAMEWORK
 M:	"Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
 M:	Josh Triplett <josh@joshtriplett.org>
@@ -11480,6 +11676,7 @@ F:	drivers/net/ethernet/rdc/r6040.c
 
 RDMAVT - RDMA verbs software
 M:	Dennis Dalessandro <dennis.dalessandro@intel.com>
+M:	Mike Marciniszyn <mike.marciniszyn@intel.com>
 L:	linux-rdma@vger.kernel.org
 S:	Supported
 F:	drivers/infiniband/sw/rdmavt
@@ -11757,15 +11954,13 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linville/wireless-testing.g
 S:	Maintained
 F:	drivers/net/wireless/realtek/rtl818x/rtl8187/
 
-RTL8192CE WIRELESS DRIVER
-M:	Larry Finger <Larry.Finger@lwfinger.net>
-M:	Chaoming Li <chaoming_li@realsil.com.cn>
+REALTEK WIRELESS DRIVER (rtlwifi family)
+M:	Ping-Ke Shih <pkshih@realtek.com>
 L:	linux-wireless@vger.kernel.org
 W:	http://wireless.kernel.org/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linville/wireless-testing.git
 S:	Maintained
 F:	drivers/net/wireless/realtek/rtlwifi/
-F:	drivers/net/wireless/realtek/rtlwifi/rtl8192ce/
 
 RTL8XXXU WIRELESS DRIVER (rtl8xxxu)
 M:	Jes Sorensen <Jes.Sorensen@gmail.com>
@@ -11858,6 +12053,7 @@ F:	drivers/pci/hotplug/s390_pci_hpc.c
 S390 VFIO-CCW DRIVER
 M:	Cornelia Huck <cohuck@redhat.com>
 M:	Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
+M:	Halil Pasic <pasic@linux.vnet.ibm.com>
 L:	linux-s390@vger.kernel.org
 L:	kvm@vger.kernel.org
 S:	Supported
@@ -11920,6 +12116,7 @@ M:	Sylwester Nawrocki <s.nawrocki@samsung.com>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
 S:	Supported
 F:	sound/soc/samsung/
+F:	Documentation/devicetree/bindings/sound/samsung*
 
 SAMSUNG EXYNOS PSEUDO RANDOM NUMBER GENERATOR (RNG) DRIVER
 M:	Krzysztof Kozlowski <krzk@kernel.org>
@@ -11929,6 +12126,13 @@ S:	Maintained
 F:	drivers/crypto/exynos-rng.c
 F:	Documentation/devicetree/bindings/crypto/samsung,exynos-rng4.txt
 
+SAMSUNG EXYNOS TRUE RANDOM NUMBER GENERATOR (TRNG) DRIVER
+M:	Łukasz Stelmach <l.stelmach@samsung.com>
+L:	linux-samsung-soc@vger.kernel.org
+S:	Maintained
+F:	drivers/char/hw_random/exynos-trng.c
+F:	Documentation/devicetree/bindings/rng/samsung,exynos5250-trng.txt
+
 SAMSUNG FRAMEBUFFER DRIVER
 M:	Jingoo Han <jingoohan1@gmail.com>
 L:	linux-fbdev@vger.kernel.org
@@ -11991,6 +12195,7 @@ F:	drivers/media/i2c/s5k5baf.c
 SAMSUNG S5P Security SubSystem (SSS) DRIVER
 M:	Krzysztof Kozlowski <krzk@kernel.org>
 M:	Vladimir Zapolskiy <vz@mleia.com>
+M:	Kamil Konieczny <k.konieczny@partner.samsung.com>
 L:	linux-crypto@vger.kernel.org
 L:	linux-samsung-soc@vger.kernel.org
 S:	Maintained
@@ -12335,6 +12540,14 @@ T:	git git://linuxtv.org/anttip/media_tree.git
 S:	Maintained
 F:	drivers/media/tuners/si2157*
 
+SI2165 MEDIA DRIVER
+M:	Matthias Schwarzott <zzam@gentoo.org>
+L:	linux-media@vger.kernel.org
+W:	https://linuxtv.org
+Q:	http://patchwork.linuxtv.org/project/linux-media/list/
+S:	Maintained
+F:	drivers/media/dvb-frontends/si2165*
+
 SI2168 MEDIA DRIVER
 M:	Antti Palosaari <crope@iki.fi>
 L:	linux-media@vger.kernel.org
@@ -12458,6 +12671,13 @@ F:	lib/siphash.c
 F:	lib/test_siphash.c
 F:	include/linux/siphash.h
 
+SIOX
+M:	Gavin Schenk <g.schenk@eckelmann.de>
+M:	Uwe Kleine-König <kernel@pengutronix.de>
+S:	Supported
+F:	drivers/siox/*
+F:	include/trace/events/siox.h
+
 SIS 190 ETHERNET DRIVER
 M:	Francois Romieu <romieu@fr.zoreil.com>
 L:	netdev@vger.kernel.org
@@ -12509,6 +12729,14 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git
 F:	include/linux/srcu.h
 F:	kernel/rcu/srcu.c
 
+SERIAL LOW-POWER INTER-CHIP MEDIA BUS (SLIMbus)
+M:	Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Maintained
+F:	drivers/slimbus/
+F:	Documentation/devicetree/bindings/slimbus/
+F:	include/linux/slimbus.h
+
 SMACK SECURITY MODULE
 M:	Casey Schaufler <casey@schaufler-ca.com>
 L:	linux-security-module@vger.kernel.org
@@ -12590,6 +12818,12 @@ F:	include/media/soc*
 F:	drivers/media/i2c/soc_camera/
 F:	drivers/media/platform/soc_camera/
 
+SOCIONEXT UNIPHIER SOUND DRIVER
+M:	Katsuhiro Suzuki <suzuki.katsuhiro@socionext.com>
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Maintained
+F:	sound/soc/uniphier/
+
 SOEKRIS NET48XX LED SUPPORT
 M:	Chris Boot <bootc@bootc.net>
 S:	Maintained
@@ -12614,6 +12848,15 @@ L:	linux-media@vger.kernel.org
 S:	Supported
 F:	drivers/media/pci/solo6x10/
 
+SOFTWARE DELEGATED EXCEPTION INTERFACE (SDEI)
+M:	James Morse <james.morse@arm.com>
+L:	linux-arm-kernel@lists.infradead.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/arm/firmware/sdei.txt
+F:	drivers/firmware/arm_sdei.c
+F:	include/linux/sdei.h
+F:	include/uapi/linux/sdei.h
+
 SOFTWARE RAID (Multiple Disks) SUPPORT
 M:	Shaohua Li <shli@kernel.org>
 L:	linux-raid@vger.kernel.org
@@ -12626,6 +12869,26 @@ F:	drivers/md/raid*
 F:	include/linux/raid/
 F:	include/uapi/linux/raid/
 
+SOCIONEXT (SNI) NETSEC NETWORK DRIVER
+M:	Jassi Brar <jaswinder.singh@linaro.org>
+L:	netdev@vger.kernel.org
+S:	Maintained
+F:	drivers/net/ethernet/socionext/netsec.c
+F:	Documentation/devicetree/bindings/net/socionext-netsec.txt
+
+SOLIDRUN CLEARFOG SUPPORT
+M:	Russell King <linux@armlinux.org.uk>
+S:	Maintained
+F:	arch/arm/boot/dts/armada-388-clearfog*
+F:	arch/arm/boot/dts/armada-38x-solidrun-*
+
+SOLIDRUN CUBOX-I/HUMMINGBOARD SUPPORT
+M:	Russell King <linux@armlinux.org.uk>
+S:	Maintained
+F:	arch/arm/boot/dts/imx6*-cubox-i*
+F:	arch/arm/boot/dts/imx6*-hummingboard*
+F:	arch/arm/boot/dts/imx6*-sr-*
+
 SONIC NETWORK DRIVER
 M:	Thomas Bogendoerfer <tsbogend@alpha.franken.de>
 L:	netdev@vger.kernel.org
@@ -12712,6 +12975,16 @@ F:	Documentation/sound/alsa/soc/
 F:	sound/soc/
 F:	include/sound/soc*
 
+SOUNDWIRE SUBSYSTEM
+M:	Vinod Koul <vinod.koul@intel.com>
+M:	Sanyog Kale <sanyog.r.kale@intel.com>
+R:	Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Supported
+F:	Documentation/driver-api/soundwire/
+F:	drivers/soundwire/
+F:	include/linux/soundwire/
+
 SP2 MEDIA DRIVER
 M:	Olli Salonen <olli.salonen@iki.fi>
 L:	linux-media@vger.kernel.org
@@ -12778,7 +13051,7 @@ L:	linux-mtd@lists.infradead.org
 W:	http://www.linux-mtd.infradead.org/
 Q:	http://patchwork.ozlabs.org/project/linux-mtd/list/
 T:	git git://git.infradead.org/linux-mtd.git spi-nor/fixes
-T:	git git://git.infradead.org/l2-mtd.git spi-nor/next
+T:	git git://git.infradead.org/linux-mtd.git spi-nor/next
 S:	Maintained
 F:	drivers/mtd/spi-nor/
 F:	include/linux/mtd/spi-nor.h
@@ -12804,7 +13077,7 @@ F:	Documentation/networking/spider_net.txt
 F:	drivers/net/ethernet/toshiba/spider_net*
 
 SPMI SUBSYSTEM
-R:	Stephen Boyd <sboyd@codeaurora.org>
+R:	Stephen Boyd <sboyd@kernel.org>
 L:	linux-arm-msm@vger.kernel.org
 F:	Documentation/devicetree/bindings/spmi/
 F:	drivers/spmi/
@@ -12865,12 +13138,6 @@ S:	Odd Fixes
 F:	Documentation/devicetree/bindings/staging/iio/
 F:	drivers/staging/iio/
 
-STAGING - LIRC (LINUX INFRARED REMOTE CONTROL) DRIVERS
-M:	Jarod Wilson <jarod@wilsonet.com>
-W:	http://www.lirc.org/
-S:	Odd Fixes
-F:	drivers/staging/media/lirc/
-
 STAGING - LUSTRE PARALLEL FILESYSTEM
 M:	Oleg Drokin <oleg.drokin@intel.com>
 M:	Andreas Dilger <andreas.dilger@intel.com>
@@ -13031,7 +13298,7 @@ F:	arch/x86/boot/video*
 
 SWIOTLB SUBSYSTEM
 M:	Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
-L:	linux-kernel@vger.kernel.org
+L:	iommu@lists.linux-foundation.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/konrad/swiotlb.git
 S:	Supported
 F:	lib/swiotlb.c
@@ -13096,6 +13363,11 @@ S:	Supported
 F:	drivers/reset/reset-axs10x.c
 F:	Documentation/devicetree/bindings/reset/snps,axs10x-reset.txt
 
+SYNOPSYS DESIGNWARE 8250 UART DRIVER
+R:	Andy Shevchenko <andriy.shevchenko@linux.intel.com>
+S:	Maintained
+F:	drivers/tty/serial/8250/8250_dw.c
+
 SYNOPSYS DESIGNWARE APB GPIO DRIVER
 M:	Hoan Tran <hotran@apm.com>
 L:	linux-gpio@vger.kernel.org
@@ -13112,7 +13384,6 @@ F:	include/linux/platform_data/dma-dw.h
 F:	drivers/dma/dw/
 
 SYNOPSYS DESIGNWARE ENTERPRISE ETHERNET DRIVER
-M:	Jie Deng <jiedeng@synopsys.com>
 M:	Jose Abreu <Jose.Abreu@synopsys.com>
 L:	netdev@vger.kernel.org
 S:	Supported
@@ -13252,6 +13523,15 @@ T:	git git://linuxtv.org/anttip/media_tree.git
 S:	Maintained
 F:	drivers/media/tuners/tda18218*
 
+TDA18250 MEDIA DRIVER
+M:	Olli Salonen <olli.salonen@iki.fi>
+L:	linux-media@vger.kernel.org
+W:	https://linuxtv.org
+Q:	http://patchwork.linuxtv.org/project/linux-media/list/
+T:	git git://linuxtv.org/media_tree.git
+S:	Maintained
+F:	drivers/media/tuners/tda18250*
+
 TDA18271 MEDIA DRIVER
 M:	Michael Krufky <mkrufky@linuxtv.org>
 L:	linux-media@vger.kernel.org
@@ -13378,7 +13658,8 @@ S:	Supported
 F:	drivers/i2c/busses/i2c-tegra.c
 
 TEGRA IOMMU DRIVERS
-M:	Hiroshi Doyu <hdoyu@nvidia.com>
+M:	Thierry Reding <thierry.reding@gmail.com>
+L:	linux-tegra@vger.kernel.org
 S:	Supported
 F:	drivers/iommu/tegra*
 
@@ -13682,7 +13963,7 @@ F:	include/linux/usb/tilegx.h
 TIMEKEEPING, CLOCKSOURCE CORE, NTP, ALARMTIMER
 M:	John Stultz <john.stultz@linaro.org>
 M:	Thomas Gleixner <tglx@linutronix.de>
-R:	Stephen Boyd <sboyd@codeaurora.org>
+R:	Stephen Boyd <sboyd@kernel.org>
 L:	linux-kernel@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/core
 S:	Supported
@@ -13765,6 +14046,18 @@ L:	platform-driver-x86@vger.kernel.org
 S:	Maintained
 F:	drivers/platform/x86/topstar-laptop.c
 
+TORTURE-TEST MODULES
+M:	Davidlohr Bueso <dave@stgolabs.net>
+M:	"Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
+M:	Josh Triplett <josh@joshtriplett.org>
+L:	linux-kernel@vger.kernel.org
+S:	Supported
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git
+F:	Documentation/RCU/torture.txt
+F:	kernel/torture.c
+F:	kernel/rcu/rcutorture.c
+F:	kernel/locking/locktorture.c
+
 TOSHIBA ACPI EXTRAS DRIVER
 M:	Azael Avalos <coproscefalo@gmail.com>
 L:	platform-driver-x86@vger.kernel.org
@@ -13807,9 +14100,10 @@ F:	drivers/platform/x86/toshiba-wmi.c
 TPM DEVICE DRIVER
 M:	Peter Huewe <peterhuewe@gmx.de>
 M:	Jarkko Sakkinen <jarkko.sakkinen@linux.intel.com>
-R:	Jason Gunthorpe <jgunthorpe@obsidianresearch.com>
+R:	Jason Gunthorpe <jgg@ziepe.ca>
 L:	linux-integrity@vger.kernel.org
 Q:	https://patchwork.kernel.org/project/linux-integrity/list/
+W:	https://kernsec.org/wiki/index.php/Linux_Kernel_Integrity
 T:	git git://git.infradead.org/users/jjs/linux-tpmdd.git
 S:	Maintained
 F:	drivers/char/tpm/
@@ -13848,6 +14142,13 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/jikos/trivial.git
 S:	Maintained
 K:	^Subject:.*(?i)trivial
 
+TEMPO SEMICONDUCTOR DRIVERS
+M:	Steven Eckhoff <steven.eckhoff.opensource@gmail.com>
+S:	Maintained
+F:	sound/soc/codecs/tscs*.c
+F:	sound/soc/codecs/tscs*.h
+F:	Documentation/devicetree/bindings/sound/tscs*.txt
+
 TTY LAYER
 M:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
 M:	Jiri Slaby <jslaby@suse.com>
@@ -14009,6 +14310,8 @@ UNISYS S-PAR DRIVERS
 M:	David Kershner <david.kershner@unisys.com>
 L:	sparmaintainer@unisys.com (Unisys internal)
 S:	Supported
+F:	include/linux/visorbus.h
+F:	drivers/visorbus/
 F:	drivers/staging/unisys/
 
 UNIVERSAL FLASH STORAGE HOST CONTROLLER DRIVER
@@ -14555,6 +14858,15 @@ S:	Maintained
 F:	drivers/virtio/virtio_input.c
 F:	include/uapi/linux/virtio_input.h
 
+VIRTUAL BOX GUEST DEVICE DRIVER
+M:	Hans de Goede <hdegoede@redhat.com>
+M:	Arnd Bergmann <arnd@arndb.de>
+M:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+S:	Maintained
+F:	include/linux/vbox_utils.h
+F:	include/uapi/linux/vbox*.h
+F:	drivers/virt/vboxguest/
+
 VIRTUAL SERIO DEVICE DRIVER
 M:	Stephen Chandler Paul <thatslyude@gmail.com>
 S:	Maintained
@@ -14646,6 +14958,7 @@ W:	http://www.slimlogic.co.uk/?p=48
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git
 S:	Supported
 F:	Documentation/devicetree/bindings/regulator/
+F:	Documentation/power/regulator/
 F:	drivers/regulator/
 F:	include/dt-bindings/regulator/
 F:	include/linux/regulator/
@@ -14718,8 +15031,8 @@ S:	Maintained
 F:	drivers/input/tablet/wacom_serial4.c
 
 WATCHDOG DEVICE DRIVERS
-M:	Wim Van Sebroeck <wim@iguana.be>
-R:	Guenter Roeck <linux@roeck-us.net>
+M:	Wim Van Sebroeck <wim@linux-watchdog.org>
+M:	Guenter Roeck <linux@roeck-us.net>
 L:	linux-watchdog@vger.kernel.org
 W:	http://www.linux-watchdog.org/
 T:	git git://www.linux-watchdog.org/linux-watchdog.git
@@ -14743,9 +15056,9 @@ S:	Maintained
 F:	drivers/hid/hid-wiimote*
 
 WILOCITY WIL6210 WIRELESS DRIVER
-M:	Maya Erez <qca_merez@qca.qualcomm.com>
+M:	Maya Erez <merez@codeaurora.org>
 L:	linux-wireless@vger.kernel.org
-L:	wil6210@qca.qualcomm.com
+L:	wil6210@qti.qualcomm.com
 S:	Supported
 W:	http://wireless.kernel.org/en/users/Drivers/wil6210
 F:	drivers/net/wireless/ath/wil6210/
@@ -14839,6 +15152,12 @@ F:	include/linux/workqueue.h
 F:	kernel/workqueue.c
 F:	Documentation/core-api/workqueue.rst
 
+X-POWERS AXP288 PMIC DRIVERS
+M:	Hans de Goede <hdegoede@redhat.com>
+S:	Maintained
+N:	axp288
+F:	drivers/acpi/pmic/intel_pmic_xpower.c
+
 X-POWERS MULTIFUNCTION PMIC DEVICE DRIVERS
 M:	Chen-Yu Tsai <wens@csie.org>
 L:	linux-kernel@vger.kernel.org
@@ -14880,7 +15199,7 @@ X86 PLATFORM DRIVERS
 M:	Darren Hart <dvhart@infradead.org>
 M:	Andy Shevchenko <andy@infradead.org>
 L:	platform-driver-x86@vger.kernel.org
-T:	git git://git.infradead.org/users/dvhart/linux-platform-drivers-x86.git
+T:	git git://git.infradead.org/linux-platform-drivers-x86.git
 S:	Maintained
 F:	drivers/platform/x86/
 F:	drivers/platform/olpc/
@@ -14966,6 +15285,7 @@ F:	include/xen/interface/io/vscsiif.h
 XEN SWIOTLB SUBSYSTEM
 M:	Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
 L:	xen-devel@lists.xenproject.org (moderated for non-subscribers)
+L:	iommu@lists.linux-foundation.org
 S:	Supported
 F:	arch/x86/xen/*swiotlb*
 F:	drivers/xen/*swiotlb*
diff --git a/Makefile b/Makefile
index c8b8e902d5a4..363dd096e46e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 VERSION = 4
-PATCHLEVEL = 15
+PATCHLEVEL = 16
 SUBLEVEL = 0
 EXTRAVERSION =
 NAME = Fearless Coyote
@@ -377,6 +377,8 @@ NM		= $(CROSS_COMPILE)nm
 STRIP		= $(CROSS_COMPILE)strip
 OBJCOPY		= $(CROSS_COMPILE)objcopy
 OBJDUMP		= $(CROSS_COMPILE)objdump
+LEX		= flex
+YACC		= bison
 AWK		= awk
 GENKSYMS	= scripts/genksyms/genksyms
 INSTALLKERNEL  := installkernel
@@ -386,7 +388,7 @@ PYTHON		= python
 CHECK		= sparse
 
 CHECKFLAGS     := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-		  -Wbitwise -Wno-return-void $(CF)
+		  -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
 NOSTDINC_FLAGS  =
 CFLAGS_MODULE   =
 AFLAGS_MODULE   =
@@ -428,11 +430,12 @@ GCC_PLUGINS_CFLAGS :=
 
 export ARCH SRCARCH CONFIG_SHELL HOSTCC HOSTCFLAGS CROSS_COMPILE AS LD CC
 export CPP AR NM STRIP OBJCOPY OBJDUMP HOSTLDFLAGS HOST_LOADLIBES
-export MAKE AWK GENKSYMS INSTALLKERNEL PERL PYTHON UTS_MACHINE
+export MAKE LEX YACC AWK GENKSYMS INSTALLKERNEL PERL PYTHON UTS_MACHINE
 export HOSTCXX HOSTCXXFLAGS LDFLAGS_MODULE CHECK CHECKFLAGS
 
 export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS LDFLAGS
-export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE CFLAGS_KASAN CFLAGS_UBSAN
+export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
+export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE CFLAGS_UBSAN
 export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE
 export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE
 export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL
@@ -484,8 +487,15 @@ CLANG_GCC_TC	:= --gcc-toolchain=$(GCC_TOOLCHAIN)
 endif
 KBUILD_CFLAGS += $(CLANG_TARGET) $(CLANG_GCC_TC)
 KBUILD_AFLAGS += $(CLANG_TARGET) $(CLANG_GCC_TC)
+KBUILD_CFLAGS += $(call cc-option, -no-integrated-as)
+KBUILD_AFLAGS += $(call cc-option, -no-integrated-as)
 endif
 
+RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern -mindirect-branch-register
+RETPOLINE_CFLAGS_CLANG := -mretpoline-external-thunk
+RETPOLINE_CFLAGS := $(call cc-option,$(RETPOLINE_CFLAGS_GCC),$(call cc-option,$(RETPOLINE_CFLAGS_CLANG)))
+export RETPOLINE_CFLAGS
+
 ifeq ($(config-targets),1)
 # ===========================================================================
 # *config targets only - make sure prerequisites are updated, and descend
@@ -576,10 +586,9 @@ ifeq ($(KBUILD_EXTMOD),)
 # To avoid any implicit rule to kick in, define an empty command
 $(KCONFIG_CONFIG) include/config/auto.conf.cmd: ;
 
-# If .config is newer than include/config/auto.conf, someone tinkered
-# with it and forgot to run make oldconfig.
-# if auto.conf.cmd is missing then we are probably in a cleaned tree so
-# we execute the config step to be sure to catch updated Kconfig files
+# The actual configuration files used during the build are stored in
+# include/generated/ and include/config/. Update them if .config is newer than
+# include/config/auto.conf (which mirrors .config).
 include/config/%.conf: $(KCONFIG_CONFIG) include/config/auto.conf.cmd
 	$(Q)$(MAKE) -f $(srctree)/Makefile silentoldconfig
 else
@@ -677,6 +686,10 @@ endif
 # This selects the stack protector compiler flag. Testing it is delayed
 # until after .config has been reprocessed, in the prepare-compiler-check
 # target.
+ifdef CONFIG_CC_STACKPROTECTOR_AUTO
+  stackp-flag := $(call cc-option,-fstack-protector-strong,$(call cc-option,-fstack-protector))
+  stackp-name := AUTO
+else
 ifdef CONFIG_CC_STACKPROTECTOR_REGULAR
   stackp-flag := -fstack-protector
   stackp-name := REGULAR
@@ -685,20 +698,43 @@ ifdef CONFIG_CC_STACKPROTECTOR_STRONG
   stackp-flag := -fstack-protector-strong
   stackp-name := STRONG
 else
+  # If either there is no stack protector for this architecture or
+  # CONFIG_CC_STACKPROTECTOR_NONE is selected, we're done, and $(stackp-name)
+  # is empty, skipping all remaining stack protector tests.
+  #
   # Force off for distro compilers that enable stack protector by default.
-  stackp-flag := $(call cc-option, -fno-stack-protector)
+  KBUILD_CFLAGS += $(call cc-option, -fno-stack-protector)
+endif
 endif
 endif
 # Find arch-specific stack protector compiler sanity-checking script.
-ifdef CONFIG_CC_STACKPROTECTOR
+ifdef stackp-name
+ifneq ($(stackp-flag),)
   stackp-path := $(srctree)/scripts/gcc-$(SRCARCH)_$(BITS)-has-stack-protector.sh
   stackp-check := $(wildcard $(stackp-path))
+  # If the wildcard test matches a test script, run it to check functionality.
+  ifdef stackp-check
+    ifneq ($(shell $(CONFIG_SHELL) $(stackp-check) $(CC) $(KBUILD_CPPFLAGS) $(biarch)),y)
+      stackp-broken := y
+    endif
+  endif
+  ifndef stackp-broken
+    # If the stack protector is functional, enable code that depends on it.
+    KBUILD_CPPFLAGS += -DCONFIG_CC_STACKPROTECTOR
+    # Either we've already detected the flag (for AUTO) or we'll fail the
+    # build in the prepare-compiler-check rule (for specific flag).
+    KBUILD_CFLAGS += $(stackp-flag)
+  else
+    # We have to make sure stack protector is unconditionally disabled if
+    # the compiler is broken (in case we're going to continue the build in
+    # AUTO mode).
+    KBUILD_CFLAGS += $(call cc-option, -fno-stack-protector)
+  endif
+endif
 endif
-KBUILD_CFLAGS += $(stackp-flag)
 
 ifeq ($(cc-name),clang)
 KBUILD_CPPFLAGS += $(call cc-option,-Qunused-arguments,)
-KBUILD_CFLAGS += $(call cc-disable-warning, unused-variable)
 KBUILD_CFLAGS += $(call cc-disable-warning, format-invalid-specifier)
 KBUILD_CFLAGS += $(call cc-disable-warning, gnu)
 KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member)
@@ -709,16 +745,14 @@ KBUILD_CFLAGS += $(call cc-disable-warning, tautological-compare)
 # See modpost pattern 2
 KBUILD_CFLAGS += $(call cc-option, -mno-global-merge,)
 KBUILD_CFLAGS += $(call cc-option, -fcatch-undefined-behavior)
-KBUILD_CFLAGS += $(call cc-option, -no-integrated-as)
-KBUILD_AFLAGS += $(call cc-option, -no-integrated-as)
 else
 
 # These warnings generated too much noise in a regular build.
 # Use make W=1 to enable them (see scripts/Makefile.extrawarn)
 KBUILD_CFLAGS += $(call cc-disable-warning, unused-but-set-variable)
-KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable)
 endif
 
+KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable)
 ifdef CONFIG_FRAME_POINTER
 KBUILD_CFLAGS	+= -fno-omit-frame-pointer -fno-optimize-sibling-calls
 else
@@ -792,6 +826,15 @@ KBUILD_CFLAGS += $(call cc-disable-warning, pointer-sign)
 # disable invalid "can't wrap" optimizations for signed / pointers
 KBUILD_CFLAGS	+= $(call cc-option,-fno-strict-overflow)
 
+# clang sets -fmerge-all-constants by default as optimization, but this
+# is non-conforming behavior for C and in fact breaks the kernel, so we
+# need to disable it here generally.
+KBUILD_CFLAGS	+= $(call cc-option,-fno-merge-all-constants)
+
+# for gcc -fno-merge-all-constants disables everything, but it is fine
+# to have actual conforming behavior enabled.
+KBUILD_CFLAGS	+= $(call cc-option,-fmerge-constants)
+
 # Make sure -fstack-check isn't enabled (like gentoo apparently did)
 KBUILD_CFLAGS  += $(call cc-option,-fno-stack-check,)
 
@@ -827,8 +870,7 @@ KBUILD_AFLAGS   += $(ARCH_AFLAGS)   $(KAFLAGS)
 KBUILD_CFLAGS   += $(ARCH_CFLAGS)   $(KCFLAGS)
 
 # Use --build-id when available.
-LDFLAGS_BUILD_ID := $(patsubst -Wl$(comma)%,%,\
-			      $(call cc-ldoption, -Wl$(comma)--build-id,))
+LDFLAGS_BUILD_ID := $(call ld-option, --build-id)
 KBUILD_LDFLAGS_MODULE += $(LDFLAGS_BUILD_ID)
 LDFLAGS_vmlinux += $(LDFLAGS_BUILD_ID)
 
@@ -1089,14 +1131,25 @@ PHONY += prepare-compiler-check
 prepare-compiler-check: FORCE
 # Make sure compiler supports requested stack protector flag.
 ifdef stackp-name
+  # Warn about CONFIG_CC_STACKPROTECTOR_AUTO having found no option.
+  ifeq ($(stackp-flag),)
+	@echo CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
+		  Compiler does not support any known stack-protector >&2
+  else
+  # Fail if specifically requested stack protector is missing.
   ifeq ($(call cc-option, $(stackp-flag)),)
 	@echo Cannot use CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
 		  $(stackp-flag) not supported by compiler >&2 && exit 1
   endif
+  endif
 endif
-# Make sure compiler does not have buggy stack-protector support.
-ifdef stackp-check
-  ifneq ($(shell $(CONFIG_SHELL) $(stackp-check) $(CC) $(KBUILD_CPPFLAGS) $(biarch)),y)
+# Make sure compiler does not have buggy stack-protector support. If a
+# specific stack-protector was requested, fail the build, otherwise warn.
+ifdef stackp-broken
+  ifeq ($(stackp-name),AUTO)
+	@echo CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
+                  $(stackp-flag) available but compiler is broken: disabling >&2
+  else
 	@echo Cannot use CONFIG_CC_STACKPROTECTOR_$(stackp-name): \
                   $(stackp-flag) available but compiler is broken >&2 && exit 1
   endif
diff --git a/arch/Kconfig b/arch/Kconfig
index 400b9e1b2f27..76c0b54443b1 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -196,6 +196,9 @@ config HAVE_OPTPROBES
 config HAVE_KPROBES_ON_FTRACE
 	bool
 
+config HAVE_FUNCTION_ERROR_INJECTION
+	bool
+
 config HAVE_NMI
 	bool
 
@@ -234,14 +237,25 @@ config ARCH_HAS_FORTIFY_SOURCE
 config ARCH_HAS_SET_MEMORY
 	bool
 
-# Select if arch init_task initializer is different to init/init_task.c
-config ARCH_INIT_TASK
+# Select if arch init_task must go in the __init_task_data section
+config ARCH_TASK_STRUCT_ON_STACK
        bool
 
 # Select if arch has its private alloc_task_struct() function
 config ARCH_TASK_STRUCT_ALLOCATOR
 	bool
 
+config HAVE_ARCH_THREAD_STRUCT_WHITELIST
+	bool
+	depends on !ARCH_TASK_STRUCT_ALLOCATOR
+	help
+	  An architecture should select this to provide hardened usercopy
+	  knowledge about what region of the thread_struct should be
+	  whitelisted for copying to userspace. Normally this is only the
+	  FPU registers. Specifically, arch_thread_struct_whitelist()
+	  should be implemented. Without this, the entire thread_struct
+	  field in task_struct will be left whitelisted.
+
 # Select if arch has its private alloc_thread_stack() function
 config ARCH_THREAD_STACK_ALLOCATOR
 	bool
@@ -524,16 +538,10 @@ config HAVE_CC_STACKPROTECTOR
 	  - its compiler supports the -fstack-protector option
 	  - it has implemented a stack canary (e.g. __stack_chk_guard)
 
-config CC_STACKPROTECTOR
-	def_bool n
-	help
-	  Set when a stack-protector mode is enabled, so that the build
-	  can enable kernel-side support for the GCC feature.
-
 choice
 	prompt "Stack Protector buffer overflow detection"
 	depends on HAVE_CC_STACKPROTECTOR
-	default CC_STACKPROTECTOR_NONE
+	default CC_STACKPROTECTOR_AUTO
 	help
 	  This option turns on the "stack-protector" GCC feature. This
 	  feature puts, at the beginning of functions, a canary value on
@@ -550,7 +558,6 @@ config CC_STACKPROTECTOR_NONE
 
 config CC_STACKPROTECTOR_REGULAR
 	bool "Regular"
-	select CC_STACKPROTECTOR
 	help
 	  Functions will have the stack-protector canary logic added if they
 	  have an 8-byte or larger character array on the stack.
@@ -564,7 +571,6 @@ config CC_STACKPROTECTOR_REGULAR
 
 config CC_STACKPROTECTOR_STRONG
 	bool "Strong"
-	select CC_STACKPROTECTOR
 	help
 	  Functions will have the stack-protector canary logic added in any
 	  of the following conditions:
@@ -582,6 +588,12 @@ config CC_STACKPROTECTOR_STRONG
 	  about 20% of all kernel functions, which increases the kernel code
 	  size by about 2%.
 
+config CC_STACKPROTECTOR_AUTO
+	bool "Automatic"
+	help
+	  If the compiler supports it, the best available stack-protector
+	  option will be chosen.
+
 endchoice
 
 config THIN_ARCHIVES
@@ -938,6 +950,10 @@ config STRICT_MODULE_RWX
 	  and non-text memory will be made non-executable. This provides
 	  protection against certain security exploits (e.g. writing to text)
 
+# select if the architecture provides an asm/dma-direct.h header
+config ARCH_HAS_PHYS_TO_DMA
+	bool
+
 config ARCH_HAS_REFCOUNT
 	bool
 	help
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index b31b974a03cb..e96adcbcab41 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -209,6 +209,7 @@ config ALPHA_EIGER
 
 config ALPHA_JENSEN
 	bool "Jensen"
+	depends on BROKEN
 	help
 	  DEC PC 150 AXP (aka Jensen): This is a very old Digital system - one
 	  of the first-generation Alpha systems. A number of these systems
diff --git a/arch/alpha/include/asm/Kbuild b/arch/alpha/include/asm/Kbuild
index 47f3fba3e41f..9b68790013e2 100644
--- a/arch/alpha/include/asm/Kbuild
+++ b/arch/alpha/include/asm/Kbuild
@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
 
-generic-y += clkdev.h
 generic-y += exec.h
 generic-y += export.h
 generic-y += fb.h
diff --git a/arch/alpha/include/asm/asm-prototypes.h b/arch/alpha/include/asm/asm-prototypes.h
index d12c68ea340b..b34cc1f06720 100644
--- a/arch/alpha/include/asm/asm-prototypes.h
+++ b/arch/alpha/include/asm/asm-prototypes.h
@@ -4,7 +4,7 @@
 #include <asm/console.h>
 #include <asm/page.h>
 #include <asm/string.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 
 #include <asm-generic/asm-prototypes.h>
 
diff --git a/arch/alpha/include/asm/cmpxchg.h b/arch/alpha/include/asm/cmpxchg.h
index 46ebf14aed4e..8a2b331e43fe 100644
--- a/arch/alpha/include/asm/cmpxchg.h
+++ b/arch/alpha/include/asm/cmpxchg.h
@@ -6,7 +6,6 @@
  * Atomic exchange routines.
  */
 
-#define __ASM__MB
 #define ____xchg(type, args...)		__xchg ## type ## _local(args)
 #define ____cmpxchg(type, args...)	__cmpxchg ## type ## _local(args)
 #include <asm/xchg.h>
@@ -33,10 +32,6 @@
 	cmpxchg_local((ptr), (o), (n));					\
 })
 
-#ifdef CONFIG_SMP
-#undef __ASM__MB
-#define __ASM__MB	"\tmb\n"
-#endif
 #undef ____xchg
 #undef ____cmpxchg
 #define ____xchg(type, args...)		__xchg ##type(args)
@@ -64,7 +59,6 @@
 	cmpxchg((ptr), (o), (n));					\
 })
 
-#undef __ASM__MB
 #undef ____cmpxchg
 
 #endif /* _ALPHA_CMPXCHG_H */
diff --git a/arch/alpha/include/asm/futex.h b/arch/alpha/include/asm/futex.h
index d2e4da93e68c..ca3322536f72 100644
--- a/arch/alpha/include/asm/futex.h
+++ b/arch/alpha/include/asm/futex.h
@@ -20,8 +20,8 @@
 	"3:	.subsection 2\n"				\
 	"4:	br	1b\n"					\
 	"	.previous\n"					\
-	EXC(1b,3b,%1,$31)					\
-	EXC(2b,3b,%1,$31)					\
+	EXC(1b,3b,$31,%1)					\
+	EXC(2b,3b,$31,%1)					\
 	:	"=&r" (oldval), "=&r"(ret)			\
 	:	"r" (uaddr), "r"(oparg)				\
 	:	"memory")
@@ -82,8 +82,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 	"3:	.subsection 2\n"
 	"4:	br	1b\n"
 	"	.previous\n"
-	EXC(1b,3b,%0,$31)
-	EXC(2b,3b,%0,$31)
+	EXC(1b,3b,$31,%0)
+	EXC(2b,3b,$31,%0)
 	:	"+r"(ret), "=&r"(prev), "=&r"(cmp)
 	:	"r"(uaddr), "r"((long)(int)oldval), "r"(newval)
 	:	"memory");
diff --git a/arch/alpha/include/asm/processor.h b/arch/alpha/include/asm/processor.h
index bfe784f2d4af..cb05d045efe3 100644
--- a/arch/alpha/include/asm/processor.h
+++ b/arch/alpha/include/asm/processor.h
@@ -40,15 +40,12 @@ typedef struct {
 struct thread_struct { };
 #define INIT_THREAD  { }
 
-/* Return saved PC of a blocked thread.  */
-struct task_struct;
-extern unsigned long thread_saved_pc(struct task_struct *);
-
 /* Do necessary setup to start up a newly executed thread.  */
 struct pt_regs;
 extern void start_thread(struct pt_regs *, unsigned long, unsigned long);
 
 /* Free all resources held by a thread. */
+struct task_struct;
 extern void release_thread(struct task_struct *);
 
 unsigned long get_wchan(struct task_struct *p);
diff --git a/arch/alpha/include/asm/thread_info.h b/arch/alpha/include/asm/thread_info.h
index 8c20c5e35432..807d7b9a1860 100644
--- a/arch/alpha/include/asm/thread_info.h
+++ b/arch/alpha/include/asm/thread_info.h
@@ -39,9 +39,6 @@ struct thread_info {
 	.preempt_count	= INIT_PREEMPT_COUNT,	\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* How to get the thread information struct from C.  */
 register struct thread_info *__current_thread_info __asm__("$8");
 #define current_thread_info()  __current_thread_info
diff --git a/arch/alpha/include/asm/xchg.h b/arch/alpha/include/asm/xchg.h
index 68dfb3cb7145..e2b59fac5257 100644
--- a/arch/alpha/include/asm/xchg.h
+++ b/arch/alpha/include/asm/xchg.h
@@ -12,6 +12,10 @@
  * Atomic exchange.
  * Since it can be used to implement critical sections
  * it must clobber "memory" (also for interrupts in UP).
+ *
+ * The leading and the trailing memory barriers guarantee that these
+ * operations are fully ordered.
+ *
  */
 
 static inline unsigned long
@@ -19,6 +23,7 @@ ____xchg(_u8, volatile char *m, unsigned long val)
 {
 	unsigned long ret, tmp, addr64;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"	andnot	%4,7,%3\n"
 	"	insbl	%1,%4,%1\n"
@@ -28,12 +33,12 @@ ____xchg(_u8, volatile char *m, unsigned long val)
 	"	or	%1,%2,%2\n"
 	"	stq_c	%2,0(%3)\n"
 	"	beq	%2,2f\n"
-		__ASM__MB
 	".subsection 2\n"
 	"2:	br	1b\n"
 	".previous"
 	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 	: "r" ((long)m), "1" (val) : "memory");
+	smp_mb();
 
 	return ret;
 }
@@ -43,6 +48,7 @@ ____xchg(_u16, volatile short *m, unsigned long val)
 {
 	unsigned long ret, tmp, addr64;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"	andnot	%4,7,%3\n"
 	"	inswl	%1,%4,%1\n"
@@ -52,12 +58,12 @@ ____xchg(_u16, volatile short *m, unsigned long val)
 	"	or	%1,%2,%2\n"
 	"	stq_c	%2,0(%3)\n"
 	"	beq	%2,2f\n"
-		__ASM__MB
 	".subsection 2\n"
 	"2:	br	1b\n"
 	".previous"
 	: "=&r" (ret), "=&r" (val), "=&r" (tmp), "=&r" (addr64)
 	: "r" ((long)m), "1" (val) : "memory");
+	smp_mb();
 
 	return ret;
 }
@@ -67,17 +73,18 @@ ____xchg(_u32, volatile int *m, unsigned long val)
 {
 	unsigned long dummy;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"1:	ldl_l %0,%4\n"
 	"	bis $31,%3,%1\n"
 	"	stl_c %1,%2\n"
 	"	beq %1,2f\n"
-		__ASM__MB
 	".subsection 2\n"
 	"2:	br 1b\n"
 	".previous"
 	: "=&r" (val), "=&r" (dummy), "=m" (*m)
 	: "rI" (val), "m" (*m) : "memory");
+	smp_mb();
 
 	return val;
 }
@@ -87,17 +94,18 @@ ____xchg(_u64, volatile long *m, unsigned long val)
 {
 	unsigned long dummy;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"1:	ldq_l %0,%4\n"
 	"	bis $31,%3,%1\n"
 	"	stq_c %1,%2\n"
 	"	beq %1,2f\n"
-		__ASM__MB
 	".subsection 2\n"
 	"2:	br 1b\n"
 	".previous"
 	: "=&r" (val), "=&r" (dummy), "=m" (*m)
 	: "rI" (val), "m" (*m) : "memory");
+	smp_mb();
 
 	return val;
 }
@@ -128,10 +136,12 @@ ____xchg(, volatile void *ptr, unsigned long x, int size)
  * store NEW in MEM.  Return the initial value in MEM.  Success is
  * indicated by comparing RETURN with OLD.
  *
- * The memory barrier should be placed in SMP only when we actually
- * make the change. If we don't change anything (so if the returned
- * prev is equal to old) then we aren't acquiring anything new and
- * we don't need any memory barrier as far I can tell.
+ * The leading and the trailing memory barriers guarantee that these
+ * operations are fully ordered.
+ *
+ * The trailing memory barrier is placed in SMP unconditionally, in
+ * order to guarantee that dependency ordering is preserved when a
+ * dependency is headed by an unsuccessful operation.
  */
 
 static inline unsigned long
@@ -139,6 +149,7 @@ ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
 {
 	unsigned long prev, tmp, cmp, addr64;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"	andnot	%5,7,%4\n"
 	"	insbl	%1,%5,%1\n"
@@ -150,13 +161,13 @@ ____cmpxchg(_u8, volatile char *m, unsigned char old, unsigned char new)
 	"	or	%1,%2,%2\n"
 	"	stq_c	%2,0(%4)\n"
 	"	beq	%2,3f\n"
-		__ASM__MB
 	"2:\n"
 	".subsection 2\n"
 	"3:	br	1b\n"
 	".previous"
 	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
+	smp_mb();
 
 	return prev;
 }
@@ -166,6 +177,7 @@ ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
 {
 	unsigned long prev, tmp, cmp, addr64;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"	andnot	%5,7,%4\n"
 	"	inswl	%1,%5,%1\n"
@@ -177,13 +189,13 @@ ____cmpxchg(_u16, volatile short *m, unsigned short old, unsigned short new)
 	"	or	%1,%2,%2\n"
 	"	stq_c	%2,0(%4)\n"
 	"	beq	%2,3f\n"
-		__ASM__MB
 	"2:\n"
 	".subsection 2\n"
 	"3:	br	1b\n"
 	".previous"
 	: "=&r" (prev), "=&r" (new), "=&r" (tmp), "=&r" (cmp), "=&r" (addr64)
 	: "r" ((long)m), "Ir" (old), "1" (new) : "memory");
+	smp_mb();
 
 	return prev;
 }
@@ -193,6 +205,7 @@ ____cmpxchg(_u32, volatile int *m, int old, int new)
 {
 	unsigned long prev, cmp;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"1:	ldl_l %0,%5\n"
 	"	cmpeq %0,%3,%1\n"
@@ -200,13 +213,13 @@ ____cmpxchg(_u32, volatile int *m, int old, int new)
 	"	mov %4,%1\n"
 	"	stl_c %1,%2\n"
 	"	beq %1,3f\n"
-		__ASM__MB
 	"2:\n"
 	".subsection 2\n"
 	"3:	br 1b\n"
 	".previous"
 	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
 	: "r"((long) old), "r"(new), "m"(*m) : "memory");
+	smp_mb();
 
 	return prev;
 }
@@ -216,6 +229,7 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
 {
 	unsigned long prev, cmp;
 
+	smp_mb();
 	__asm__ __volatile__(
 	"1:	ldq_l %0,%5\n"
 	"	cmpeq %0,%3,%1\n"
@@ -223,13 +237,13 @@ ____cmpxchg(_u64, volatile long *m, unsigned long old, unsigned long new)
 	"	mov %4,%1\n"
 	"	stq_c %1,%2\n"
 	"	beq %1,3f\n"
-		__ASM__MB
 	"2:\n"
 	".subsection 2\n"
 	"3:	br 1b\n"
 	".previous"
 	: "=&r"(prev), "=&r"(cmp), "=m"(*m)
 	: "r"((long) old), "r"(new), "m"(*m) : "memory");
+	smp_mb();
 
 	return prev;
 }
diff --git a/arch/alpha/include/uapi/asm/Kbuild b/arch/alpha/include/uapi/asm/Kbuild
index 14a2e9af97e9..9afaba5e5503 100644
--- a/arch/alpha/include/uapi/asm/Kbuild
+++ b/arch/alpha/include/uapi/asm/Kbuild
@@ -2,3 +2,4 @@
 include include/uapi/asm-generic/Kbuild.asm
 
 generic-y += bpf_perf_event.h
+generic-y += poll.h
diff --git a/arch/alpha/include/uapi/asm/poll.h b/arch/alpha/include/uapi/asm/poll.h
deleted file mode 100644
index b7132a305a47..000000000000
--- a/arch/alpha/include/uapi/asm/poll.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#include <asm-generic/poll.h>
diff --git a/arch/alpha/include/uapi/asm/termbits.h b/arch/alpha/include/uapi/asm/termbits.h
index 05e0398a83a6..de6c8360fbe3 100644
--- a/arch/alpha/include/uapi/asm/termbits.h
+++ b/arch/alpha/include/uapi/asm/termbits.h
@@ -110,7 +110,11 @@ struct ktermios {
 #define VTDLY	00200000
 #define   VT0	00000000
 #define   VT1	00200000
-#define XTABS	01000000 /* Hmm.. Linux/i386 considers this part of TABDLY.. */
+/*
+ * Should be equivalent to TAB3, see description of TAB3 in
+ * POSIX.1-2008, Ch. 11.2.3 "Output Modes"
+ */
+#define XTABS	TAB3
 
 /* c_cflag bit meaning */
 #define CBAUD	0000037
diff --git a/arch/alpha/kernel/console.c b/arch/alpha/kernel/console.c
index 8e9a41966881..5476279329a6 100644
--- a/arch/alpha/kernel/console.c
+++ b/arch/alpha/kernel/console.c
@@ -21,6 +21,7 @@
 struct pci_controller *pci_vga_hose;
 static struct resource alpha_vga = {
 	.name	= "alpha-vga+",
+	.flags	= IORESOURCE_IO,
 	.start	= 0x3C0,
 	.end	= 0x3DF
 };
diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c
index ce3a675c0c4b..fa1a392ca9a2 100644
--- a/arch/alpha/kernel/osf_sys.c
+++ b/arch/alpha/kernel/osf_sys.c
@@ -950,22 +950,31 @@ struct itimerval32
 };
 
 static inline long
-get_tv32(struct timeval *o, struct timeval32 __user *i)
+get_tv32(struct timespec64 *o, struct timeval32 __user *i)
 {
 	struct timeval32 tv;
 	if (copy_from_user(&tv, i, sizeof(struct timeval32)))
 		return -EFAULT;
 	o->tv_sec = tv.tv_sec;
-	o->tv_usec = tv.tv_usec;
+	o->tv_nsec = tv.tv_usec * NSEC_PER_USEC;
 	return 0;
 }
 
 static inline long
-put_tv32(struct timeval32 __user *o, struct timeval *i)
+put_tv32(struct timeval32 __user *o, struct timespec64 *i)
 {
 	return copy_to_user(o, &(struct timeval32){
-				.tv_sec = o->tv_sec,
-				.tv_usec = o->tv_usec},
+				.tv_sec = i->tv_sec,
+				.tv_usec = i->tv_nsec / NSEC_PER_USEC},
+			    sizeof(struct timeval32));
+}
+
+static inline long
+put_tv_to_tv32(struct timeval32 __user *o, struct timeval *i)
+{
+	return copy_to_user(o, &(struct timeval32){
+				.tv_sec = i->tv_sec,
+				.tv_usec = i->tv_usec},
 			    sizeof(struct timeval32));
 }
 
@@ -1004,9 +1013,10 @@ SYSCALL_DEFINE2(osf_gettimeofday, struct timeval32 __user *, tv,
 		struct timezone __user *, tz)
 {
 	if (tv) {
-		struct timeval ktv;
-		do_gettimeofday(&ktv);
-		if (put_tv32(tv, &ktv))
+		struct timespec64 kts;
+
+		ktime_get_real_ts64(&kts);
+		if (put_tv32(tv, &kts))
 			return -EFAULT;
 	}
 	if (tz) {
@@ -1019,22 +1029,19 @@ SYSCALL_DEFINE2(osf_gettimeofday, struct timeval32 __user *, tv,
 SYSCALL_DEFINE2(osf_settimeofday, struct timeval32 __user *, tv,
 		struct timezone __user *, tz)
 {
-	struct timespec64 kts64;
-	struct timespec kts;
+	struct timespec64 kts;
 	struct timezone ktz;
 
  	if (tv) {
-		if (get_tv32((struct timeval *)&kts, tv))
+		if (get_tv32(&kts, tv))
 			return -EFAULT;
-		kts.tv_nsec *= 1000;
-		kts64 = timespec_to_timespec64(kts);
 	}
 	if (tz) {
 		if (copy_from_user(&ktz, tz, sizeof(*tz)))
 			return -EFAULT;
 	}
 
-	return do_sys_settimeofday64(tv ? &kts64 : NULL, tz ? &ktz : NULL);
+	return do_sys_settimeofday64(tv ? &kts : NULL, tz ? &ktz : NULL);
 }
 
 asmlinkage long sys_ni_posix_timers(void);
@@ -1083,22 +1090,16 @@ SYSCALL_DEFINE3(osf_setitimer, int, which, struct itimerval32 __user *, in,
 SYSCALL_DEFINE2(osf_utimes, const char __user *, filename,
 		struct timeval32 __user *, tvs)
 {
-	struct timespec tv[2];
+	struct timespec64 tv[2];
 
 	if (tvs) {
-		struct timeval ktvs[2];
-		if (get_tv32(&ktvs[0], &tvs[0]) ||
-		    get_tv32(&ktvs[1], &tvs[1]))
+		if (get_tv32(&tv[0], &tvs[0]) ||
+		    get_tv32(&tv[1], &tvs[1]))
 			return -EFAULT;
 
-		if (ktvs[0].tv_usec < 0 || ktvs[0].tv_usec >= 1000000 ||
-		    ktvs[1].tv_usec < 0 || ktvs[1].tv_usec >= 1000000)
+		if (tv[0].tv_nsec < 0 || tv[0].tv_nsec >= 1000000000 ||
+		    tv[1].tv_nsec < 0 || tv[1].tv_nsec >= 1000000000)
 			return -EINVAL;
-
-		tv[0].tv_sec = ktvs[0].tv_sec;
-		tv[0].tv_nsec = 1000 * ktvs[0].tv_usec;
-		tv[1].tv_sec = ktvs[1].tv_sec;
-		tv[1].tv_nsec = 1000 * ktvs[1].tv_usec;
 	}
 
 	return do_utimes(AT_FDCWD, filename, tvs ? tv : NULL, 0);
@@ -1107,19 +1108,18 @@ SYSCALL_DEFINE2(osf_utimes, const char __user *, filename,
 SYSCALL_DEFINE5(osf_select, int, n, fd_set __user *, inp, fd_set __user *, outp,
 		fd_set __user *, exp, struct timeval32 __user *, tvp)
 {
-	struct timespec end_time, *to = NULL;
+	struct timespec64 end_time, *to = NULL;
 	if (tvp) {
-		struct timeval tv;
+		struct timespec64 tv;
 		to = &end_time;
 
 		if (get_tv32(&tv, tvp))
 		    	return -EFAULT;
 
-		if (tv.tv_sec < 0 || tv.tv_usec < 0)
+		if (tv.tv_sec < 0 || tv.tv_nsec < 0)
 			return -EINVAL;
 
-		if (poll_select_set_timeout(to, tv.tv_sec,
-					    tv.tv_usec * NSEC_PER_USEC))
+		if (poll_select_set_timeout(to, tv.tv_sec, tv.tv_nsec))
 			return -EINVAL;		
 
 	}
@@ -1192,9 +1192,9 @@ SYSCALL_DEFINE4(osf_wait4, pid_t, pid, int __user *, ustatus, int, options,
 		return -EFAULT;
 	if (!ur)
 		return err;
-	if (put_tv32(&ur->ru_utime, &r.ru_utime))
+	if (put_tv_to_tv32(&ur->ru_utime, &r.ru_utime))
 		return -EFAULT;
-	if (put_tv32(&ur->ru_stime, &r.ru_stime))
+	if (put_tv_to_tv32(&ur->ru_stime, &r.ru_stime))
 		return -EFAULT;
 	if (copy_to_user(&ur->ru_maxrss, &r.ru_maxrss,
 	      sizeof(struct rusage32) - offsetof(struct rusage32, ru_maxrss)))
@@ -1210,18 +1210,18 @@ SYSCALL_DEFINE4(osf_wait4, pid_t, pid, int __user *, ustatus, int, options,
 SYSCALL_DEFINE2(osf_usleep_thread, struct timeval32 __user *, sleep,
 		struct timeval32 __user *, remain)
 {
-	struct timeval tmp;
+	struct timespec64 tmp;
 	unsigned long ticks;
 
 	if (get_tv32(&tmp, sleep))
 		goto fault;
 
-	ticks = timeval_to_jiffies(&tmp);
+	ticks = timespec64_to_jiffies(&tmp);
 
 	ticks = schedule_timeout_interruptible(ticks);
 
 	if (remain) {
-		jiffies_to_timeval(ticks, &tmp);
+		jiffies_to_timespec64(ticks, &tmp);
 		if (put_tv32(remain, &tmp))
 			goto fault;
 	}
@@ -1280,7 +1280,7 @@ SYSCALL_DEFINE1(old_adjtimex, struct timex32 __user *, txc_p)
 	if (copy_to_user(txc_p, &txc, offsetof(struct timex32, time)) ||
 	    (copy_to_user(&txc_p->tick, &txc.tick, sizeof(struct timex32) - 
 			  offsetof(struct timex32, tick))) ||
-	    (put_tv32(&txc_p->time, &txc.time)))
+	    (put_tv_to_tv32(&txc_p->time, &txc.time)))
 	  return -EFAULT;
 
 	return ret;
diff --git a/arch/alpha/kernel/pci.c b/arch/alpha/kernel/pci.c
index 87da00579946..2e86ebb680ae 100644
--- a/arch/alpha/kernel/pci.c
+++ b/arch/alpha/kernel/pci.c
@@ -425,7 +425,7 @@ sys_pciconfig_iobase(long which, unsigned long bus, unsigned long dfn)
 		if (bus == 0 && dfn == 0) {
 			hose = pci_isa_hose;
 		} else {
-			dev = pci_get_bus_and_slot(bus, dfn);
+			dev = pci_get_domain_bus_and_slot(0, bus, dfn);
 			if (!dev)
 				return -ENODEV;
 			hose = dev->sysdata;
diff --git a/arch/alpha/kernel/pci_impl.h b/arch/alpha/kernel/pci_impl.h
index 2e4cb74fdc41..18043af45e2b 100644
--- a/arch/alpha/kernel/pci_impl.h
+++ b/arch/alpha/kernel/pci_impl.h
@@ -144,7 +144,8 @@ struct pci_iommu_arena
 };
 
 #if defined(CONFIG_ALPHA_SRM) && \
-    (defined(CONFIG_ALPHA_CIA) || defined(CONFIG_ALPHA_LCA))
+    (defined(CONFIG_ALPHA_CIA) || defined(CONFIG_ALPHA_LCA) || \
+     defined(CONFIG_ALPHA_AVANTI))
 # define NEED_SRM_SAVE_RESTORE
 #else
 # undef NEED_SRM_SAVE_RESTORE
diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c
index 74bfb1f2d68e..48b81d015d8a 100644
--- a/arch/alpha/kernel/process.c
+++ b/arch/alpha/kernel/process.c
@@ -269,12 +269,13 @@ copy_thread(unsigned long clone_flags, unsigned long usp,
 	   application calling fork.  */
 	if (clone_flags & CLONE_SETTLS)
 		childti->pcb.unique = regs->r20;
+	else
+		regs->r20 = 0;	/* OSF/1 has some strange fork() semantics.  */
 	childti->pcb.usp = usp ?: rdusp();
 	*childregs = *regs;
 	childregs->r0 = 0;
 	childregs->r19 = 0;
 	childregs->r20 = 1;	/* OSF/1 has some strange fork() semantics.  */
-	regs->r20 = 0;
 	stack = ((struct switch_stack *) regs) - 1;
 	*childstack = *stack;
 	childstack->r26 = (unsigned long) ret_from_fork;
@@ -361,7 +362,7 @@ EXPORT_SYMBOL(dump_elf_task_fp);
  * all.  -- r~
  */
 
-unsigned long
+static unsigned long
 thread_saved_pc(struct task_struct *t)
 {
 	unsigned long base = (unsigned long)task_stack_page(t);
diff --git a/arch/alpha/kernel/sys_nautilus.c b/arch/alpha/kernel/sys_nautilus.c
index 239dc0e601d5..ff4f54b86c7f 100644
--- a/arch/alpha/kernel/sys_nautilus.c
+++ b/arch/alpha/kernel/sys_nautilus.c
@@ -237,7 +237,7 @@ nautilus_init_pci(void)
 	bus = hose->bus = bridge->bus;
 	pcibios_claim_one_bus(bus);
 
-	irongate = pci_get_bus_and_slot(0, 0);
+	irongate = pci_get_domain_bus_and_slot(pci_domain_nr(bus), 0, 0);
 	bus->self = irongate;
 	bus->resource[0] = &irongate_io;
 	bus->resource[1] = &irongate_mem;
diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c
index 4bd99a7b1c41..f43bd05dede2 100644
--- a/arch/alpha/kernel/traps.c
+++ b/arch/alpha/kernel/traps.c
@@ -160,11 +160,16 @@ void show_stack(struct task_struct *task, unsigned long *sp)
 	for(i=0; i < kstack_depth_to_print; i++) {
 		if (((long) stack & (THREAD_SIZE-1)) == 0)
 			break;
-		if (i && ((i % 4) == 0))
-			printk("\n       ");
-		printk("%016lx ", *stack++);
+		if ((i % 4) == 0) {
+			if (i)
+				pr_cont("\n");
+			printk("       ");
+		} else {
+			pr_cont(" ");
+		}
+		pr_cont("%016lx", *stack++);
 	}
-	printk("\n");
+	pr_cont("\n");
 	dik_show_trace(sp);
 }
 
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index 9d5fd00d9e91..d76bf4a83740 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -463,9 +463,6 @@ config ARCH_PHYS_ADDR_T_64BIT
 config ARCH_DMA_ADDR_T_64BIT
 	bool
 
-config ARC_PLAT_NEEDS_PHYS_TO_DMA
-	bool
-
 config ARC_KVADDR_SIZE
 	int "Kernel Virtual Address Space size (MB)"
 	range 0 512
@@ -487,7 +484,6 @@ config ARC_CURR_IN_REG
 
 config ARC_EMUL_UNALIGNED
 	bool "Emulate unaligned memory access (userspace only)"
-	default N
 	select SYSCTL_ARCH_UNALIGN_NO_WARN
 	select SYSCTL_ARCH_UNALIGN_ALLOW
 	depends on ISA_ARCOMPACT
diff --git a/arch/arc/boot/dts/axs101.dts b/arch/arc/boot/dts/axs101.dts
index 70aec7d6ca60..626b694c7be7 100644
--- a/arch/arc/boot/dts/axs101.dts
+++ b/arch/arc/boot/dts/axs101.dts
@@ -17,6 +17,6 @@
 	compatible = "snps,axs101", "snps,arc-sdp";
 
 	chosen {
-		bootargs = "earlycon=uart8250,mmio32,0xe0022000,115200n8 console=tty0 console=ttyS3,115200n8 consoleblank=0 video=1280x720@60";
+		bootargs = "earlycon=uart8250,mmio32,0xe0022000,115200n8 console=tty0 console=ttyS3,115200n8 consoleblank=0 video=1280x720@60 print-fatal-signals=1";
 	};
 };
diff --git a/arch/arc/boot/dts/axs10x_mb.dtsi b/arch/arc/boot/dts/axs10x_mb.dtsi
index 74d070cd3c13..47b74fbc403c 100644
--- a/arch/arc/boot/dts/axs10x_mb.dtsi
+++ b/arch/arc/boot/dts/axs10x_mb.dtsi
@@ -214,13 +214,13 @@
 			};
 
 			eeprom@0x54{
-				compatible = "24c01";
+				compatible = "atmel,24c01";
 				reg = <0x54>;
 				pagesize = <0x8>;
 			};
 
 			eeprom@0x57{
-				compatible = "24c04";
+				compatible = "atmel,24c04";
 				reg = <0x57>;
 				pagesize = <0x8>;
 			};
diff --git a/arch/arc/boot/dts/haps_hs_idu.dts b/arch/arc/boot/dts/haps_hs_idu.dts
index 215cddd0b63b..0c603308aeb3 100644
--- a/arch/arc/boot/dts/haps_hs_idu.dts
+++ b/arch/arc/boot/dts/haps_hs_idu.dts
@@ -22,7 +22,7 @@
 	};
 
 	chosen {
-		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=ttyS0,115200n8 debug";
+		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=ttyS0,115200n8 debug print-fatal-signals=1";
 	};
 
 	aliases {
diff --git a/arch/arc/boot/dts/nsim_700.dts b/arch/arc/boot/dts/nsim_700.dts
index 5ee96b067c08..ff2f2c70c545 100644
--- a/arch/arc/boot/dts/nsim_700.dts
+++ b/arch/arc/boot/dts/nsim_700.dts
@@ -17,7 +17,7 @@
 	interrupt-parent = <&core_intc>;
 
 	chosen {
-		bootargs = "earlycon=arc_uart,mmio32,0xc0fc1000,115200n8 console=ttyARC0,115200n8";
+		bootargs = "earlycon=arc_uart,mmio32,0xc0fc1000,115200n8 console=ttyARC0,115200n8 print-fatal-signals=1";
 	};
 
 	aliases {
diff --git a/arch/arc/boot/dts/nsim_hs.dts b/arch/arc/boot/dts/nsim_hs.dts
index 8d787b251f73..8e2489b16b0a 100644
--- a/arch/arc/boot/dts/nsim_hs.dts
+++ b/arch/arc/boot/dts/nsim_hs.dts
@@ -24,7 +24,7 @@
 	};
 
 	chosen {
-		bootargs = "earlycon=arc_uart,mmio32,0xc0fc1000,115200n8 console=ttyARC0,115200n8";
+		bootargs = "earlycon=arc_uart,mmio32,0xc0fc1000,115200n8 console=ttyARC0,115200n8 print-fatal-signals=1";
 	};
 
 	aliases {
diff --git a/arch/arc/boot/dts/nsim_hs_idu.dts b/arch/arc/boot/dts/nsim_hs_idu.dts
index 4f98ebf71fd8..ed12f494721d 100644
--- a/arch/arc/boot/dts/nsim_hs_idu.dts
+++ b/arch/arc/boot/dts/nsim_hs_idu.dts
@@ -15,7 +15,7 @@
 	interrupt-parent = <&core_intc>;
 
 	chosen {
-		bootargs = "earlycon=arc_uart,mmio32,0xc0fc1000,115200n8 console=ttyARC0,115200n8";
+		bootargs = "earlycon=arc_uart,mmio32,0xc0fc1000,115200n8 console=ttyARC0,115200n8 print-fatal-signals=1";
 	};
 
 	aliases {
diff --git a/arch/arc/boot/dts/nsimosci.dts b/arch/arc/boot/dts/nsimosci.dts
index 3c391ba565ed..7842e5eb4ab5 100644
--- a/arch/arc/boot/dts/nsimosci.dts
+++ b/arch/arc/boot/dts/nsimosci.dts
@@ -20,7 +20,7 @@
 		/* this is for console on PGU */
 		/* bootargs = "console=tty0 consoleblank=0"; */
 		/* this is for console on serial */
-		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=tty0 console=ttyS0,115200n8 consoleblank=0 debug video=640x480-24";
+		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=tty0 console=ttyS0,115200n8 consoleblank=0 debug video=640x480-24 print-fatal-signals=1";
 	};
 
 	aliases {
diff --git a/arch/arc/boot/dts/nsimosci_hs.dts b/arch/arc/boot/dts/nsimosci_hs.dts
index 14a727cbf4c9..b8838cf2b4ec 100644
--- a/arch/arc/boot/dts/nsimosci_hs.dts
+++ b/arch/arc/boot/dts/nsimosci_hs.dts
@@ -20,7 +20,7 @@
 		/* this is for console on PGU */
 		/* bootargs = "console=tty0 consoleblank=0"; */
 		/* this is for console on serial */
-		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=tty0 console=ttyS0,115200n8 consoleblank=0 debug video=640x480-24";
+		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=tty0 console=ttyS0,115200n8 consoleblank=0 debug video=640x480-24 print-fatal-signals=1";
 	};
 
 	aliases {
diff --git a/arch/arc/boot/dts/nsimosci_hs_idu.dts b/arch/arc/boot/dts/nsimosci_hs_idu.dts
index 5052917d4a99..72a2c723f1f7 100644
--- a/arch/arc/boot/dts/nsimosci_hs_idu.dts
+++ b/arch/arc/boot/dts/nsimosci_hs_idu.dts
@@ -18,7 +18,7 @@
 
 	chosen {
 		/* this is for console on serial */
-		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=tty0 console=ttyS0,115200n8 consoleblan=0 debug video=640x480-24";
+		bootargs = "earlycon=uart8250,mmio32,0xf0000000,115200n8 console=tty0 console=ttyS0,115200n8 consoleblan=0 debug video=640x480-24 print-fatal-signals=1";
 	};
 
 	aliases {
diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig
index ec7c849a5c8e..09f85154c5a4 100644
--- a/arch/arc/configs/axs101_defconfig
+++ b/arch/arc/configs/axs101_defconfig
@@ -44,7 +44,6 @@ CONFIG_IP_PNP_RARP=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_NETDEVICES=y
diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig
index 63d3cf69e0b0..09fed3ef22b6 100644
--- a/arch/arc/configs/axs103_defconfig
+++ b/arch/arc/configs/axs103_defconfig
@@ -44,7 +44,6 @@ CONFIG_IP_PNP_RARP=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig
index f613ecac14a7..ea2f6d817d1a 100644
--- a/arch/arc/configs/axs103_smp_defconfig
+++ b/arch/arc/configs/axs103_smp_defconfig
@@ -45,7 +45,6 @@ CONFIG_IP_PNP_RARP=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig
index db04ea4dd2d9..ab231c040efe 100644
--- a/arch/arc/configs/haps_hs_defconfig
+++ b/arch/arc/configs/haps_hs_defconfig
@@ -40,7 +40,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 CONFIG_NETDEVICES=y
 # CONFIG_NET_VENDOR_ARC is not set
diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig
index 3507be2af6fe..cf449cbf440d 100644
--- a/arch/arc/configs/haps_hs_smp_defconfig
+++ b/arch/arc/configs/haps_hs_smp_defconfig
@@ -43,7 +43,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 CONFIG_NETDEVICES=y
 # CONFIG_NET_VENDOR_ARC is not set
diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig
index ac6b0ed8341e..1b54c72f4296 100644
--- a/arch/arc/configs/hsdk_defconfig
+++ b/arch/arc/configs/hsdk_defconfig
@@ -32,7 +32,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_NETDEVICES=y
diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig
index 6dff83a238b8..31c2c70b34a1 100644
--- a/arch/arc/configs/nsim_700_defconfig
+++ b/arch/arc/configs/nsim_700_defconfig
@@ -36,7 +36,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 CONFIG_NETDEVICES=y
 CONFIG_ARC_EMAC=y
diff --git a/arch/arc/configs/nsim_hs_defconfig b/arch/arc/configs/nsim_hs_defconfig
index 31ee51b987e7..a578c721d50f 100644
--- a/arch/arc/configs/nsim_hs_defconfig
+++ b/arch/arc/configs/nsim_hs_defconfig
@@ -40,7 +40,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 # CONFIG_INPUT_MOUSEDEV_PSAUX is not set
 # CONFIG_INPUT_KEYBOARD is not set
diff --git a/arch/arc/configs/nsim_hs_smp_defconfig b/arch/arc/configs/nsim_hs_smp_defconfig
index 8d3b1f67cae4..37d7395f3272 100644
--- a/arch/arc/configs/nsim_hs_smp_defconfig
+++ b/arch/arc/configs/nsim_hs_smp_defconfig
@@ -39,7 +39,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 # CONFIG_INPUT_MOUSEDEV_PSAUX is not set
 # CONFIG_INPUT_KEYBOARD is not set
diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig
index 6168ce2ac2ef..1e1470e2a7f0 100644
--- a/arch/arc/configs/nsimosci_defconfig
+++ b/arch/arc/configs/nsimosci_defconfig
@@ -35,7 +35,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 CONFIG_NETDEVICES=y
 CONFIG_EZCHIP_NPS_MANAGEMENT_ENET=y
diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig
index a70bdeb2b3fd..084a6e42685b 100644
--- a/arch/arc/configs/nsimosci_hs_defconfig
+++ b/arch/arc/configs/nsimosci_hs_defconfig
@@ -36,7 +36,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 CONFIG_NETDEVICES=y
 CONFIG_EZCHIP_NPS_MANAGEMENT_ENET=y
diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig
index ef96406c446e..f36d47990415 100644
--- a/arch/arc/configs/nsimosci_hs_smp_defconfig
+++ b/arch/arc/configs/nsimosci_hs_smp_defconfig
@@ -39,7 +39,6 @@ CONFIG_INET=y
 CONFIG_DEVTMPFS=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_BLK_DEV is not set
 CONFIG_NETDEVICES=y
 # CONFIG_NET_VENDOR_ARC is not set
diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig
index f30182549395..1aca2e8fd1ba 100644
--- a/arch/arc/configs/tb10x_defconfig
+++ b/arch/arc/configs/tb10x_defconfig
@@ -42,7 +42,6 @@ CONFIG_IP_MULTICAST=y
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_NETDEVICES=y
 # CONFIG_NET_CADENCE is not set
 # CONFIG_NET_VENDOR_BROADCOM is not set
diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig
index 4fcf4f2503f6..f629493929ea 100644
--- a/arch/arc/configs/vdk_hs38_defconfig
+++ b/arch/arc/configs/vdk_hs38_defconfig
@@ -31,7 +31,6 @@ CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig
index 7b71464f6c2f..21f0ca26a05d 100644
--- a/arch/arc/configs/vdk_hs38_smp_defconfig
+++ b/arch/arc/configs/vdk_hs38_smp_defconfig
@@ -34,7 +34,6 @@ CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/arc/include/asm/Kbuild b/arch/arc/include/asm/Kbuild
index cd8398daccee..4bd5d4369e05 100644
--- a/arch/arc/include/asm/Kbuild
+++ b/arch/arc/include/asm/Kbuild
@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += emergency-restart.h
diff --git a/arch/arc/include/asm/bug.h b/arch/arc/include/asm/bug.h
index ea022d47896c..21ec82466d62 100644
--- a/arch/arc/include/asm/bug.h
+++ b/arch/arc/include/asm/bug.h
@@ -23,7 +23,8 @@ void die(const char *str, struct pt_regs *regs, unsigned long address);
 
 #define BUG()	do {								\
 	pr_warn("BUG: failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__); \
-	dump_stack();								\
+	barrier_before_unreachable();						\
+	__builtin_trap();							\
 } while (0)
 
 #define HAVE_ARCH_BUG
diff --git a/arch/arc/include/asm/dma-mapping.h b/arch/arc/include/asm/dma-mapping.h
index 94285031c4fb..7a16824bfe98 100644
--- a/arch/arc/include/asm/dma-mapping.h
+++ b/arch/arc/include/asm/dma-mapping.h
@@ -11,13 +11,6 @@
 #ifndef ASM_ARC_DMA_MAPPING_H
 #define ASM_ARC_DMA_MAPPING_H
 
-#ifndef CONFIG_ARC_PLAT_NEEDS_PHYS_TO_DMA
-#define plat_dma_to_phys(dev, dma_handle) ((phys_addr_t)(dma_handle))
-#define plat_phys_to_dma(dev, paddr) ((dma_addr_t)(paddr))
-#else
-#include <plat/dma.h>
-#endif
-
 extern const struct dma_map_ops arc_dma_ops;
 
 static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
diff --git a/arch/arc/include/asm/entry-arcv2.h b/arch/arc/include/asm/entry-arcv2.h
index 257a68f3c2fe..309f4e6721b3 100644
--- a/arch/arc/include/asm/entry-arcv2.h
+++ b/arch/arc/include/asm/entry-arcv2.h
@@ -184,7 +184,7 @@
 .macro FAKE_RET_FROM_EXCPN
 	lr      r9, [status32]
 	bic     r9, r9, (STATUS_U_MASK|STATUS_DE_MASK|STATUS_AE_MASK)
-	or      r9, r9, (STATUS_L_MASK|STATUS_IE_MASK)
+	or      r9, r9, STATUS_IE_MASK
 	kflag   r9
 .endm
 
diff --git a/arch/arc/include/asm/hugepage.h b/arch/arc/include/asm/hugepage.h
index b18fcb606908..dc8ee011882f 100644
--- a/arch/arc/include/asm/hugepage.h
+++ b/arch/arc/include/asm/hugepage.h
@@ -74,4 +74,7 @@ extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
 extern void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
 				unsigned long end);
 
+/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/
+#define pmdp_establish generic_pmdp_establish
+
 #endif
diff --git a/arch/arc/include/asm/thread_info.h b/arch/arc/include/asm/thread_info.h
index 2d79e527fa50..c85947bac5e5 100644
--- a/arch/arc/include/asm/thread_info.h
+++ b/arch/arc/include/asm/thread_info.h
@@ -62,9 +62,6 @@ struct thread_info {
 	.addr_limit = KERNEL_DS,		\
 }
 
-#define init_thread_info    (init_thread_union.thread_info)
-#define init_stack          (init_thread_union.stack)
-
 static inline __attribute_const__ struct thread_info *current_thread_info(void)
 {
 	register unsigned long sp asm("sp");
diff --git a/arch/arc/kernel/mcip.c b/arch/arc/kernel/mcip.c
index f61a52b01625..5fe84e481654 100644
--- a/arch/arc/kernel/mcip.c
+++ b/arch/arc/kernel/mcip.c
@@ -22,10 +22,79 @@ static DEFINE_RAW_SPINLOCK(mcip_lock);
 
 static char smp_cpuinfo_buf[128];
 
+/*
+ * Set mask to halt GFRC if any online core in SMP cluster is halted.
+ * Only works for ARC HS v3.0+, on earlier versions has no effect.
+ */
+static void mcip_update_gfrc_halt_mask(int cpu)
+{
+	struct bcr_generic gfrc;
+	unsigned long flags;
+	u32 gfrc_halt_mask;
+
+	READ_BCR(ARC_REG_GFRC_BUILD, gfrc);
+
+	/*
+	 * CMD_GFRC_SET_CORE and CMD_GFRC_READ_CORE commands were added in
+	 * GFRC 0x3 version.
+	 */
+	if (gfrc.ver < 0x3)
+		return;
+
+	raw_spin_lock_irqsave(&mcip_lock, flags);
+
+	__mcip_cmd(CMD_GFRC_READ_CORE, 0);
+	gfrc_halt_mask = read_aux_reg(ARC_REG_MCIP_READBACK);
+	gfrc_halt_mask |= BIT(cpu);
+	__mcip_cmd_data(CMD_GFRC_SET_CORE, 0, gfrc_halt_mask);
+
+	raw_spin_unlock_irqrestore(&mcip_lock, flags);
+}
+
+static void mcip_update_debug_halt_mask(int cpu)
+{
+	u32 mcip_mask = 0;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&mcip_lock, flags);
+
+	/*
+	 * mcip_mask is same for CMD_DEBUG_SET_SELECT and CMD_DEBUG_SET_MASK
+	 * commands. So read it once instead of reading both CMD_DEBUG_READ_MASK
+	 * and CMD_DEBUG_READ_SELECT.
+	 */
+	__mcip_cmd(CMD_DEBUG_READ_SELECT, 0);
+	mcip_mask = read_aux_reg(ARC_REG_MCIP_READBACK);
+
+	mcip_mask |= BIT(cpu);
+
+	__mcip_cmd_data(CMD_DEBUG_SET_SELECT, 0, mcip_mask);
+	/*
+	 * Parameter specified halt cause:
+	 * STATUS32[H]/actionpoint/breakpoint/self-halt
+	 * We choose all of them (0xF).
+	 */
+	__mcip_cmd_data(CMD_DEBUG_SET_MASK, 0xF, mcip_mask);
+
+	raw_spin_unlock_irqrestore(&mcip_lock, flags);
+}
+
 static void mcip_setup_per_cpu(int cpu)
 {
+	struct mcip_bcr mp;
+
+	READ_BCR(ARC_REG_MCIP_BCR, mp);
+
 	smp_ipi_irq_setup(cpu, IPI_IRQ);
 	smp_ipi_irq_setup(cpu, SOFTIRQ_IRQ);
+
+	/* Update GFRC halt mask as new CPU came online */
+	if (mp.gfrc)
+		mcip_update_gfrc_halt_mask(cpu);
+
+	/* Update MCIP debug mask as new CPU came online */
+	if (mp.dbg)
+		mcip_update_debug_halt_mask(cpu);
 }
 
 static void mcip_ipi_send(int cpu)
@@ -101,11 +170,6 @@ static void mcip_probe_n_setup(void)
 		IS_AVAIL1(mp.gfrc, "GFRC"));
 
 	cpuinfo_arc700[0].extn.gfrc = mp.gfrc;
-
-	if (mp.dbg) {
-		__mcip_cmd_data(CMD_DEBUG_SET_SELECT, 0, 0xf);
-		__mcip_cmd_data(CMD_DEBUG_SET_MASK, 0xf, 0xf);
-	}
 }
 
 struct plat_smp_ops plat_smp_ops = {
diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c
index 9d27331fe69a..b2cae79a25d7 100644
--- a/arch/arc/kernel/setup.c
+++ b/arch/arc/kernel/setup.c
@@ -51,7 +51,7 @@ static const struct id_to_str arc_cpu_rel[] = {
 	{ 0x51, "R2.0" },
 	{ 0x52, "R2.1" },
 	{ 0x53, "R3.0" },
-	{ 0x54, "R4.0" },
+	{ 0x54, "R3.10a" },
 #endif
 	{ 0x00, NULL   }
 };
@@ -373,7 +373,7 @@ static void arc_chk_core_config(void)
 {
 	struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()];
 	int saved = 0, present = 0;
-	char *opt_nm = NULL;;
+	char *opt_nm = NULL;
 
 	if (!cpu->extn.timer0)
 		panic("Timer0 is not present!\n");
diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c
index efe8b4200a67..21d86c36692b 100644
--- a/arch/arc/kernel/smp.c
+++ b/arch/arc/kernel/smp.c
@@ -24,6 +24,7 @@
 #include <linux/reboot.h>
 #include <linux/irqdomain.h>
 #include <linux/export.h>
+#include <linux/of_fdt.h>
 
 #include <asm/processor.h>
 #include <asm/setup.h>
@@ -47,6 +48,42 @@ void __init smp_prepare_boot_cpu(void)
 {
 }
 
+static int __init arc_get_cpu_map(const char *name, struct cpumask *cpumask)
+{
+	unsigned long dt_root = of_get_flat_dt_root();
+	const char *buf;
+
+	buf = of_get_flat_dt_prop(dt_root, name, NULL);
+	if (!buf)
+		return -EINVAL;
+
+	if (cpulist_parse(buf, cpumask))
+		return -EINVAL;
+
+	return 0;
+}
+
+/*
+ * Read from DeviceTree and setup cpu possible mask. If there is no
+ * "possible-cpus" property in DeviceTree pretend all [0..NR_CPUS-1] exist.
+ */
+static void __init arc_init_cpu_possible(void)
+{
+	struct cpumask cpumask;
+
+	if (arc_get_cpu_map("possible-cpus", &cpumask)) {
+		pr_warn("Failed to get possible-cpus from dtb, pretending all %u cpus exist\n",
+			NR_CPUS);
+
+		cpumask_setall(&cpumask);
+	}
+
+	if (!cpumask_test_cpu(0, &cpumask))
+		panic("Master cpu (cpu[0]) is missed in cpu possible mask!");
+
+	init_cpu_possible(&cpumask);
+}
+
 /*
  * Called from setup_arch() before calling setup_processor()
  *
@@ -58,10 +95,7 @@ void __init smp_prepare_boot_cpu(void)
  */
 void __init smp_init_cpus(void)
 {
-	unsigned int i;
-
-	for (i = 0; i < NR_CPUS; i++)
-		set_cpu_possible(i, true);
+	arc_init_cpu_possible();
 
 	if (plat_smp_ops.init_early_smp)
 		plat_smp_ops.init_early_smp();
@@ -70,16 +104,12 @@ void __init smp_init_cpus(void)
 /* called from init ( ) =>  process 1 */
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
-	int i;
-
 	/*
 	 * if platform didn't set the present map already, do it now
 	 * boot cpu is set to present already by init/main.c
 	 */
-	if (num_present_cpus() <= 1) {
-		for (i = 0; i < max_cpus; i++)
-			set_cpu_present(i, true);
-	}
+	if (num_present_cpus() <= 1)
+		init_cpu_present(cpu_possible_mask);
 }
 
 void __init smp_cpus_done(unsigned int max_cpus)
diff --git a/arch/arc/kernel/traps.c b/arch/arc/kernel/traps.c
index 133a4dae41fe..b123558bf0bb 100644
--- a/arch/arc/kernel/traps.c
+++ b/arch/arc/kernel/traps.c
@@ -65,12 +65,14 @@ unhandled_exception(const char *str, struct pt_regs *regs, siginfo_t *info)
 #define DO_ERROR_INFO(signr, str, name, sicode) \
 int name(unsigned long address, struct pt_regs *regs) \
 {						\
-	siginfo_t info = {			\
-		.si_signo = signr,		\
-		.si_errno = 0,			\
-		.si_code  = sicode,		\
-		.si_addr = (void __user *)address,	\
-	};					\
+	siginfo_t info;				\
+						\
+	clear_siginfo(&info);			\
+	info.si_signo = signr;			\
+	info.si_errno = 0;			\
+	info.si_code  = sicode;			\
+	info.si_addr = (void __user *)address;	\
+						\
 	return unhandled_exception(str, regs, &info);\
 }
 
diff --git a/arch/arc/kernel/unwind.c b/arch/arc/kernel/unwind.c
index 333daab7def0..183391d4d33a 100644
--- a/arch/arc/kernel/unwind.c
+++ b/arch/arc/kernel/unwind.c
@@ -366,7 +366,7 @@ static void init_unwind_hdr(struct unwind_table *table,
 	return;
 
 ret_err:
-	panic("Attention !!! Dwarf FDE parsing errors\n");;
+	panic("Attention !!! Dwarf FDE parsing errors\n");
 }
 
 #ifdef CONFIG_MODULES
diff --git a/arch/arc/mm/cache.c b/arch/arc/mm/cache.c
index eee924dfffa6..2072f3451e9c 100644
--- a/arch/arc/mm/cache.c
+++ b/arch/arc/mm/cache.c
@@ -780,7 +780,10 @@ noinline static void slc_entire_op(const int op)
 
 	write_aux_reg(r, ctrl);
 
-	write_aux_reg(ARC_REG_SLC_INVALIDATE, 1);
+	if (op & OP_INV)	/* Inv or flush-n-inv use same cmd reg */
+		write_aux_reg(ARC_REG_SLC_INVALIDATE, 0x1);
+	else
+		write_aux_reg(ARC_REG_SLC_FLUSH, 0x1);
 
 	/* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */
 	read_aux_reg(r);
diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c
index e9d93604ad0f..1dcc404b5aec 100644
--- a/arch/arc/mm/dma.c
+++ b/arch/arc/mm/dma.c
@@ -60,7 +60,7 @@ static void *arc_dma_alloc(struct device *dev, size_t size,
 	/* This is linear addr (0x8000_0000 based) */
 	paddr = page_to_phys(page);
 
-	*dma_handle = plat_phys_to_dma(dev, paddr);
+	*dma_handle = paddr;
 
 	/* This is kernel Virtual address (0x7000_0000 based) */
 	if (need_kvaddr) {
@@ -92,7 +92,7 @@ static void *arc_dma_alloc(struct device *dev, size_t size,
 static void arc_dma_free(struct device *dev, size_t size, void *vaddr,
 		dma_addr_t dma_handle, unsigned long attrs)
 {
-	phys_addr_t paddr = plat_dma_to_phys(dev, dma_handle);
+	phys_addr_t paddr = dma_handle;
 	struct page *page = virt_to_page(paddr);
 	int is_non_coh = 1;
 
@@ -111,7 +111,7 @@ static int arc_dma_mmap(struct device *dev, struct vm_area_struct *vma,
 {
 	unsigned long user_count = vma_pages(vma);
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
-	unsigned long pfn = __phys_to_pfn(plat_dma_to_phys(dev, dma_addr));
+	unsigned long pfn = __phys_to_pfn(dma_addr);
 	unsigned long off = vma->vm_pgoff;
 	int ret = -ENXIO;
 
@@ -175,7 +175,7 @@ static dma_addr_t arc_dma_map_page(struct device *dev, struct page *page,
 	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 		_dma_cache_sync(paddr, size, dir);
 
-	return plat_phys_to_dma(dev, paddr);
+	return paddr;
 }
 
 /*
@@ -190,7 +190,7 @@ static void arc_dma_unmap_page(struct device *dev, dma_addr_t handle,
 			       size_t size, enum dma_data_direction dir,
 			       unsigned long attrs)
 {
-	phys_addr_t paddr = plat_dma_to_phys(dev, handle);
+	phys_addr_t paddr = handle;
 
 	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 		_dma_cache_sync(paddr, size, dir);
@@ -224,13 +224,13 @@ static void arc_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
 static void arc_dma_sync_single_for_cpu(struct device *dev,
 		dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
 {
-	_dma_cache_sync(plat_dma_to_phys(dev, dma_handle), size, DMA_FROM_DEVICE);
+	_dma_cache_sync(dma_handle, size, DMA_FROM_DEVICE);
 }
 
 static void arc_dma_sync_single_for_device(struct device *dev,
 		dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
 {
-	_dma_cache_sync(plat_dma_to_phys(dev, dma_handle), size, DMA_TO_DEVICE);
+	_dma_cache_sync(dma_handle, size, DMA_TO_DEVICE);
 }
 
 static void arc_dma_sync_sg_for_cpu(struct device *dev,
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 51c8df561077..7e3d53575486 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -3,11 +3,12 @@ config ARM
 	bool
 	default y
 	select ARCH_CLOCKSOURCE_DATA
-	select ARCH_DISCARD_MEMBLOCK if !HAVE_ARCH_PFN_VALID
-	select ARCH_HAS_DEBUG_VIRTUAL
+	select ARCH_DISCARD_MEMBLOCK if !HAVE_ARCH_PFN_VALID && !KEXEC
+	select ARCH_HAS_DEBUG_VIRTUAL if MMU
 	select ARCH_HAS_DEVMEM_IS_ALLOWED
 	select ARCH_HAS_ELF_RANDOMIZE
 	select ARCH_HAS_SET_MEMORY
+	select ARCH_HAS_PHYS_TO_DMA
 	select ARCH_HAS_STRICT_KERNEL_RWX if MMU && !XIP_KERNEL
 	select ARCH_HAS_STRICT_MODULE_RWX if MMU
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
@@ -24,7 +25,7 @@ config ARM
 	select CLONE_BACKWARDS
 	select CPU_PM if (SUSPEND || CPU_IDLE)
 	select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS
-	select DMA_NOOP_OPS if !MMU
+	select DMA_DIRECT_OPS if !MMU
 	select EDAC_SUPPORT
 	select EDAC_ATOMIC_SCRUB
 	select GENERIC_ALLOCATOR
@@ -50,6 +51,7 @@ config ARM
 	select HAVE_ARCH_KGDB if !CPU_ENDIAN_BE32 && MMU
 	select HAVE_ARCH_MMAP_RND_BITS if MMU
 	select HAVE_ARCH_SECCOMP_FILTER if (AEABI && !OABI_COMPAT)
+	select HAVE_ARCH_THREAD_STRUCT_WHITELIST
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_ARM_SMCCC if CPU_V7
 	select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32
@@ -99,6 +101,7 @@ config ARM
 	select OLD_SIGACTION
 	select OLD_SIGSUSPEND3
 	select PERF_USE_VMALLOC
+	select REFCOUNT_FULL
 	select RTC_LIB
 	select SYS_SUPPORTS_APM_EMULATION
 	# Above selects are sorted alphabetically; please add new ones
@@ -595,6 +598,7 @@ config ARCH_S3C24XX
 	select MULTI_IRQ_HANDLER
 	select NEED_MACH_IO_H
 	select SAMSUNG_ATAGS
+	select USE_OF
 	help
 	  Samsung S3C2410, S3C2412, S3C2413, S3C2416, S3C2440, S3C2442, S3C2443
 	  and S3C2450 SoCs based systems, such as the Simtec Electronics BAST
@@ -1524,12 +1528,10 @@ config THUMB2_KERNEL
 	bool "Compile the kernel in Thumb-2 mode" if !CPU_THUMBONLY
 	depends on (CPU_V7 || CPU_V7M) && !CPU_V6 && !CPU_V6K
 	default y if CPU_THUMBONLY
-	select ARM_ASM_UNIFIED
 	select ARM_UNWIND
 	help
 	  By enabling this option, the kernel will be compiled in
-	  Thumb-2 mode. A compiler/assembler that understand the unified
-	  ARM-Thumb syntax is needed.
+	  Thumb-2 mode.
 
 	  If unsure, say N.
 
@@ -1564,9 +1566,6 @@ config THUMB2_AVOID_R_ARM_THM_JUMP11
 
 	  Unless you are sure your tools don't have this problem, say Y.
 
-config ARM_ASM_UNIFIED
-	bool
-
 config ARM_PATCH_IDIV
 	bool "Runtime patch udiv/sdiv instructions into __aeabi_{u}idiv()"
 	depends on CPU_32v7 && !XIP_KERNEL
diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug
index 17685e19aed8..199ebc1c4538 100644
--- a/arch/arm/Kconfig.debug
+++ b/arch/arm/Kconfig.debug
@@ -3,10 +3,14 @@ menu "Kernel hacking"
 
 source "lib/Kconfig.debug"
 
-config ARM_PTDUMP
+config ARM_PTDUMP_CORE
+	def_bool n
+
+config ARM_PTDUMP_DEBUGFS
 	bool "Export kernel pagetable layout to userspace via debugfs"
 	depends on DEBUG_KERNEL
 	depends on MMU
+	select ARM_PTDUMP_CORE
 	select DEBUG_FS
 	---help---
 	  Say Y here if you want to show the kernel pagetable layout in a
@@ -16,6 +20,34 @@ config ARM_PTDUMP
 	  kernel.
 	  If in doubt, say "N"
 
+config DEBUG_WX
+	bool "Warn on W+X mappings at boot"
+	depends on MMU
+	select ARM_PTDUMP_CORE
+	---help---
+		Generate a warning if any W+X mappings are found at boot.
+
+		This is useful for discovering cases where the kernel is leaving
+		W+X mappings after applying NX, as such mappings are a security risk.
+
+		Look for a message in dmesg output like this:
+
+			arm/mm: Checked W+X mappings: passed, no W+X pages found.
+
+		or like this, if the check failed:
+
+			arm/mm: Checked W+X mappings: FAILED, <N> W+X pages found.
+
+		Note that even if the check fails, your kernel is possibly
+		still fine, as W+X mappings are not a security hole in
+		themselves, what they do is that they make the exploitation
+		of other unfixed kernel bugs easier.
+
+		There is no runtime or memory usage effect of this option
+		once the kernel has booted up - it's a one time check.
+
+		If in doubt, say "Y".
+
 # RMK wants arm kernels compiled with frame pointers or stack unwinding.
 # If you know what you are doing and are willing to live without stack
 # traces, you can get a slightly smaller kernel by setting this option to
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index 80351e505fd5..e83f5161fdd8 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -115,9 +115,11 @@ ifeq ($(CONFIG_ARM_UNWIND),y)
 CFLAGS_ABI	+=-funwind-tables
 endif
 
+# Accept old syntax despite ".syntax unified"
+AFLAGS_NOWARN	:=$(call as-option,-Wa$(comma)-mno-warn-deprecated,-Wa$(comma)-W)
+
 ifeq ($(CONFIG_THUMB2_KERNEL),y)
 AFLAGS_AUTOIT	:=$(call as-option,-Wa$(comma)-mimplicit-it=always,-Wa$(comma)-mauto-it)
-AFLAGS_NOWARN	:=$(call as-option,-Wa$(comma)-mno-warn-deprecated,-Wa$(comma)-W)
 CFLAGS_ISA	:=-mthumb $(AFLAGS_AUTOIT) $(AFLAGS_NOWARN)
 AFLAGS_ISA	:=$(CFLAGS_ISA) -Wa$(comma)-mthumb
 # Work around buggy relocation from gas if requested:
@@ -125,7 +127,7 @@ ifeq ($(CONFIG_THUMB2_AVOID_R_ARM_THM_JUMP11),y)
 KBUILD_CFLAGS_MODULE	+=-fno-optimize-sibling-calls
 endif
 else
-CFLAGS_ISA	:=$(call cc-option,-marm,)
+CFLAGS_ISA	:=$(call cc-option,-marm,) $(AFLAGS_NOWARN)
 AFLAGS_ISA	:=$(CFLAGS_ISA)
 endif
 
diff --git a/arch/arm/boot/compressed/string.c b/arch/arm/boot/compressed/string.c
index 309e1bbad75d..13c90abc68d6 100644
--- a/arch/arm/boot/compressed/string.c
+++ b/arch/arm/boot/compressed/string.c
@@ -130,8 +130,3 @@ void *memset(void *s, int c, size_t count)
 		*xs++ = c;
 	return s;
 }
-
-void __memzero(void *s, size_t count)
-{
-	memset(s, 0, count);
-}
diff --git a/arch/arm/boot/compressed/vmlinux.lds.S b/arch/arm/boot/compressed/vmlinux.lds.S
index e6bf6774c4bb..2b963d8e76dd 100644
--- a/arch/arm/boot/compressed/vmlinux.lds.S
+++ b/arch/arm/boot/compressed/vmlinux.lds.S
@@ -56,6 +56,7 @@ SECTIONS
   .rodata : {
     *(.rodata)
     *(.rodata.*)
+    *(.data.rel.ro)
   }
   .piggydata : {
     *(.piggydata)
@@ -101,6 +102,12 @@ SECTIONS
    * this symbol allows further debug in the near future.
    */
   .image_end (NOLOAD) : {
+    /*
+     * EFI requires that the image is aligned to 512 bytes, and appended
+     * DTB requires that we know where the end of the image is.  Ensure
+     * that both are satisfied by ensuring that there are no additional
+     * sections emitted into the decompressor image.
+     */
     _edata_real = .;
   }
 
@@ -128,3 +135,4 @@ SECTIONS
   .stab.indexstr 0	: { *(.stab.indexstr) }
   .comment 0		: { *(.comment) }
 }
+ASSERT(_edata_real == _edata, "error: zImage file size is incorrect");
diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
index 1189598a25eb..5e7d758ebdd6 100755
--- a/arch/arm/boot/deflate_xip_data.sh
+++ b/arch/arm/boot/deflate_xip_data.sh
@@ -30,7 +30,7 @@ esac
 
 sym_val() {
 	# extract hex value for symbol in $1
-	local val=$($NM "$VMLINUX" | sed -n "/ $1$/{s/ .*$//p;q}")
+	local val=$($NM "$VMLINUX" 2>/dev/null | sed -n "/ $1\$/{s/ .*$//p;q}")
 	[ "$val" ] || { echo "can't find $1 in $VMLINUX" 1>&2; exit 1; }
 	# convert from hex to decimal
 	echo $((0x$val))
@@ -48,12 +48,12 @@ data_end=$(($_edata_loc - $base_offset))
 file_end=$(stat -c "%s" "$XIPIMAGE")
 if [ "$file_end" != "$data_end" ]; then
 	printf "end of xipImage doesn't match with _edata_loc (%#x vs %#x)\n" \
-	       $(($file_end + $base_offset)) $_edata_loc 2>&1
+	       $(($file_end + $base_offset)) $_edata_loc 1>&2
 	exit 1;
 fi
 
 # be ready to clean up
-trap 'rm -f "$XIPIMAGE.tmp"' 0 1 2 3
+trap 'rm -f "$XIPIMAGE.tmp"; exit 1' 1 2 3
 
 # substitute the data section by a compressed version
 $DD if="$XIPIMAGE" count=$data_start iflag=count_bytes of="$XIPIMAGE.tmp"
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index d0381e9caf21..ade7a38543dc 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -1,6 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-ifeq ($(CONFIG_OF),y)
-
 dtb-$(CONFIG_ARCH_ALPINE) += \
 	alpine-db.dtb
 dtb-$(CONFIG_MACH_ARTPEC6) += \
@@ -47,7 +45,9 @@ dtb-$(CONFIG_SOC_AT91SAM9) += \
 	at91sam9x35ek.dtb
 dtb-$(CONFIG_SOC_SAM_V7) += \
 	at91-kizbox2.dtb \
+	at91-nattis-2-natte-2.dtb \
 	at91-sama5d27_som1_ek.dtb \
+	at91-sama5d2_ptc_ek.dtb \
 	at91-sama5d2_xplained.dtb \
 	at91-sama5d3_xplained.dtb \
 	at91-tse850-3.dtb \
@@ -192,6 +192,7 @@ dtb-$(CONFIG_ARCH_EXYNOS5) += \
 	exynos5800-peach-pi.dtb
 dtb-$(CONFIG_ARCH_GEMINI) += \
 	gemini-dlink-dir-685.dtb \
+	gemini-dlink-dns-313.dtb \
 	gemini-nas4220b.dtb \
 	gemini-rut1xx.dtb \
 	gemini-sq201.dtb \
@@ -372,6 +373,8 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6dl-aristainetos2_7.dtb \
 	imx6dl-colibri-eval-v3.dtb \
 	imx6dl-cubox-i.dtb \
+	imx6dl-cubox-i-emmc-som-v15.dtb \
+	imx6dl-cubox-i-som-v15.dtb \
 	imx6dl-dfi-fs700-m60.dtb \
 	imx6dl-gw51xx.dtb \
 	imx6dl-gw52xx.dtb \
@@ -384,6 +387,11 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6dl-gw5903.dtb \
 	imx6dl-gw5904.dtb \
 	imx6dl-hummingboard.dtb \
+	imx6dl-hummingboard-emmc-som-v15.dtb \
+	imx6dl-hummingboard-som-v15.dtb \
+	imx6dl-hummingboard2.dtb \
+	imx6dl-hummingboard2-emmc-som-v15.dtb \
+	imx6dl-hummingboard2-som-v15.dtb \
 	imx6dl-icore.dtb \
 	imx6dl-icore-rqs.dtb \
 	imx6dl-nit6xlite.dtb \
@@ -396,6 +404,7 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6dl-sabresd.dtb \
 	imx6dl-savageboard.dtb \
 	imx6dl-ts4900.dtb \
+	imx6dl-ts7970.dtb \
 	imx6dl-tx6dl-comtft.dtb \
 	imx6dl-tx6s-8034.dtb \
 	imx6dl-tx6s-8034-mb7.dtb \
@@ -421,6 +430,8 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6q-b850v3.dtb \
 	imx6q-cm-fx6.dtb \
 	imx6q-cubox-i.dtb \
+	imx6q-cubox-i-emmc-som-v15.dtb \
+	imx6q-cubox-i-som-v15.dtb \
 	imx6q-dfi-fs700-m60.dtb \
 	imx6q-display5-tianma-tm070-1280x768.dtb \
 	imx6q-dmo-edmqmx6.dtb \
@@ -439,6 +450,11 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6q-gw5904.dtb \
 	imx6q-h100.dtb \
 	imx6q-hummingboard.dtb \
+	imx6q-hummingboard-emmc-som-v15.dtb \
+	imx6q-hummingboard-som-v15.dtb \
+	imx6q-hummingboard2.dtb \
+	imx6q-hummingboard2-emmc-som-v15.dtb \
+	imx6q-hummingboard2-som-v15.dtb \
 	imx6q-icore.dtb \
 	imx6q-icore-ofcap10.dtb \
 	imx6q-icore-ofcap12.dtb \
@@ -459,6 +475,7 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6q-sbc6x.dtb \
 	imx6q-tbs2910.dtb \
 	imx6q-ts4900.dtb \
+	imx6q-ts7970.dtb \
 	imx6q-tx6q-1010.dtb \
 	imx6q-tx6q-1010-comtft.dtb \
 	imx6q-tx6q-1020.dtb \
@@ -470,6 +487,7 @@ dtb-$(CONFIG_SOC_IMX6Q) += \
 	imx6q-tx6q-11x0-mb7.dtb \
 	imx6q-udoo.dtb \
 	imx6q-utilite-pro.dtb \
+	imx6q-var-dt6customboard.dtb \
 	imx6q-wandboard.dtb \
 	imx6q-wandboard-revb1.dtb \
 	imx6q-wandboard-revd1.dtb \
@@ -511,15 +529,17 @@ dtb-$(CONFIG_SOC_IMX6UL) += \
 	imx6ull-14x14-evk.dtb
 dtb-$(CONFIG_SOC_IMX7D) += \
 	imx7d-cl-som-imx7.dtb \
+	imx7d-colibri-emmc-eval-v3.dtb \
 	imx7d-colibri-eval-v3.dtb \
 	imx7d-nitrogen7.dtb \
-	imx7d-pico.dtb \
+	imx7d-pico-pi.dtb \
 	imx7d-sbc-imx7.dtb \
 	imx7d-sdb.dtb \
 	imx7d-sdb-sht11.dtb \
 	imx7s-colibri-eval-v3.dtb \
 	imx7s-warp.dtb
 dtb-$(CONFIG_SOC_LS1021A) += \
+	ls1021a-moxa-uc-8410a.dtb \
 	ls1021a-qds.dtb \
 	ls1021a-twr.dtb
 dtb-$(CONFIG_SOC_VF610) += \
@@ -558,6 +578,7 @@ dtb-$(CONFIG_ARCH_MXS) += \
 	imx28-m28cu3.dtb \
 	imx28-m28evk.dtb \
 	imx28-sps1.dtb \
+	imx28-ts4600.dtb \
 	imx28-tx28.dtb
 dtb-$(CONFIG_ARCH_NOMADIK) += \
 	ste-nomadik-s8815.dtb \
@@ -689,6 +710,7 @@ dtb-$(CONFIG_SOC_DRA7XX) += \
 	am57xx-sbc-am57x.dtb \
 	am572x-idk.dtb \
 	am571x-idk.dtb \
+	am574x-idk.dtb \
 	dra7-evm.dtb \
 	dra72-evm.dtb \
 	dra72-evm-revc.dtb \
@@ -707,12 +729,13 @@ dtb-$(CONFIG_ARCH_ORION5X) += \
 	orion5x-rd88f5182-nas.dtb
 dtb-$(CONFIG_ARCH_ACTIONS) += \
 	owl-s500-cubieboard6.dtb \
-	owl-s500-guitar-bb-rev-b.dtb
+	owl-s500-guitar-bb-rev-b.dtb \
+	owl-s500-sparky.dtb
 dtb-$(CONFIG_ARCH_PRIMA2) += \
 	prima2-evb.dtb
 dtb-$(CONFIG_ARCH_OXNAS) += \
-	wd-mbwe.dtb \
-	cloudengines-pogoplug-series-3.dtb
+	ox810se-wd-mbwe.dtb \
+	ox820-cloudengines-pogoplug-series-3.dtb
 dtb-$(CONFIG_ARCH_QCOM) += \
 	qcom-apq8060-dragonboard.dtb \
 	qcom-apq8064-arrow-sd-600eval.dtb \
@@ -756,6 +779,7 @@ dtb-$(CONFIG_ARCH_RENESAS) += \
 	r8a7743-iwg20d-q7-dbcm-ca.dtb \
 	r8a7743-sk-rzg1m.dtb \
 	r8a7745-iwg22d-sodimm.dtb \
+	r8a7745-iwg22d-sodimm-dbhd-ca.dtb \
 	r8a7745-sk-rzg1e.dtb \
 	r8a7778-bockw.dtb \
 	r8a7779-marzen.dtb \
@@ -949,9 +973,11 @@ dtb-$(CONFIG_MACH_SUN8I) += \
 	sun8i-a83t-bananapi-m3.dtb \
 	sun8i-a83t-cubietruck-plus.dtb \
 	sun8i-a83t-tbs-a711.dtb \
+	sun8i-h2-plus-orangepi-r1.dtb \
 	sun8i-h2-plus-orangepi-zero.dtb \
 	sun8i-h3-bananapi-m2-plus.dtb \
 	sun8i-h3-beelink-x2.dtb \
+	sun8i-h3-libretech-all-h3-cc.dtb \
 	sun8i-h3-nanopi-m1.dtb	\
 	sun8i-h3-nanopi-m1-plus.dtb \
 	sun8i-h3-nanopi-neo.dtb \
@@ -1101,7 +1127,10 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += \
 	mt8127-moose.dtb \
 	mt8135-evbp1.dtb
 dtb-$(CONFIG_ARCH_ZX) += zx296702-ad1.dtb
-dtb-$(CONFIG_ARCH_ASPEED) += aspeed-bmc-opp-palmetto.dtb \
+dtb-$(CONFIG_ARCH_ASPEED) += \
+	aspeed-ast2500-evb.dtb \
+	aspeed-bmc-opp-palmetto.dtb \
 	aspeed-bmc-opp-romulus.dtb \
-	aspeed-ast2500-evb.dtb
-endif
+	aspeed-bmc-opp-witherspoon.dtb \
+	aspeed-bmc-opp-zaius.dtb \
+	aspeed-bmc-quanta-q71l.dtb
diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi
index 48a15fc641f2..e67b4d65c8d0 100644
--- a/arch/arm/boot/dts/am335x-bone-common.dtsi
+++ b/arch/arm/boot/dts/am335x-bone-common.dtsi
@@ -409,6 +409,6 @@
 };
 
 &rtc {
-	clocks = <&clk_32768_ck>, <&clkdiv32k_ick>;
+	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 	clock-names = "ext-clk", "int-clk";
 };
diff --git a/arch/arm/boot/dts/am335x-boneblue.dts b/arch/arm/boot/dts/am335x-boneblue.dts
index cdc1b2be792f..3f2480d05a3b 100644
--- a/arch/arm/boot/dts/am335x-boneblue.dts
+++ b/arch/arm/boot/dts/am335x-boneblue.dts
@@ -159,6 +159,7 @@
 		>;
 	};
 
+	/* UT0 */
 	uart0_pins: pinmux_uart0_pins {
 		pinctrl-single,pins = <
 			AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0)	/* (E15) uart0_rxd.uart0_rxd */
@@ -166,6 +167,37 @@
 		>;
 	};
 
+	/* UT1 */
+	uart1_pins: pinmux_uart1_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x980, PIN_INPUT_PULLUP | MUX_MODE0)	/* (D16) uart1_rxd.uart1_rxd */
+			AM33XX_IOPAD(0x984, PIN_OUTPUT_PULLDOWN | MUX_MODE0)	/* (D15) uart1_txd.uart1_txd */
+		>;
+	};
+
+	/* GPS */
+	uart2_pins: pinmux_uart2_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE1)	/* (A17) spi0_sclk.uart2_rxd */
+			AM33XX_IOPAD(0x954, PIN_OUTPUT_PULLDOWN | MUX_MODE1)	/* (B17) spi0_d0.uart2_txd */
+		>;
+	};
+
+	/* DSM2 */
+	uart4_pins: pinmux_uart4_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x870, PIN_INPUT_PULLUP | MUX_MODE6)	/* (T17) gpmc_wait0.uart4_rxd */
+		>;
+	};
+
+	/* UT5 */
+	uart5_pins: pinmux_uart5_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x8C4, PIN_INPUT_PULLUP | MUX_MODE4)	/* (U2) lcd_data9.uart5_rxd */
+			AM33XX_IOPAD(0x8C0, PIN_OUTPUT_PULLDOWN | MUX_MODE4)	/* (U1) lcd_data8.uart5_txd */
+		>;
+	};
+
 	mmc1_pins: pinmux_mmc1_pins {
 		pinctrl-single,pins = <
 			AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7)		/* (C15) spi0_cs1.gpio0[6] */
@@ -216,10 +248,19 @@
 	wl18xx_pins: pinmux_wl18xx_pins {
 		pinctrl-single,pins = <
 			AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7)	/* (K18) gmii1_txclk.gpio3[9] - WL_EN */
-			AM33XX_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7)	/* (H18) rmii1_refclk.gpio0[29] - WL_IRQ */
+			AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7)	/* (K16) gmii1_txd1.gpio0[21] - WL_IRQ */
 			AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7)	/* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */
 		>;
 	};
+
+	/* DCAN */
+	dcan1_pins: pinmux_dcan1_pins {
+		pinctrl-single,pins = <
+			AM33XX_IOPAD(0x96c, PIN_INPUT | MUX_MODE2)		/* (E17) uart0_rtsn.dcan1_rx */
+			AM33XX_IOPAD(0x968, PIN_OUTPUT | MUX_MODE2)		/* (E18) uart0_ctsn.dcan1_tx */
+			AM33XX_IOPAD(0x940, PIN_OUTPUT | MUX_MODE7)		/* (M16) gmii1_rxd0.gpio2[21] */
+		>;
+	};
 };
 
 &uart0 {
@@ -229,6 +270,34 @@
 	status = "okay";
 };
 
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_pins>;
+
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_pins>;
+
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart4_pins>;
+
+	status = "okay";
+};
+
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart5_pins>;
+
+	status = "okay";
+};
+
 &usb {
 	status = "okay";
 };
@@ -414,7 +483,7 @@
 		compatible = "ti,wl1835";
 		reg = <2>;
 		interrupt-parent = <&gpio0>;
-		interrupts = <29 IRQ_TYPE_EDGE_RISING>;
+		interrupts = <21 IRQ_TYPE_EDGE_RISING>;
 	};
 };
 
@@ -446,10 +515,16 @@
 
 &rtc {
 	system-power-controller;
-	clocks = <&clk_32768_ck>, <&clkdiv32k_ick>;
+	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 	clock-names = "ext-clk", "int-clk";
 };
 
+&dcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dcan1_pins>;
+	status = "okay";
+};
+
 &gpio3 {
 	ls_buf_en {
 		gpio-hog;
diff --git a/arch/arm/boot/dts/am335x-evm.dts b/arch/arm/boot/dts/am335x-evm.dts
index ddd897556e03..fee6b3ee1741 100644
--- a/arch/arm/boot/dts/am335x-evm.dts
+++ b/arch/arm/boot/dts/am335x-evm.dts
@@ -790,6 +790,6 @@
 };
 
 &rtc {
-	clocks = <&clk_32768_ck>, <&clkdiv32k_ick>;
+	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 	clock-names = "ext-clk", "int-clk";
 };
diff --git a/arch/arm/boot/dts/am335x-evmsk.dts b/arch/arm/boot/dts/am335x-evmsk.dts
index 9ba4b18c0cb2..fa608cd5dc14 100644
--- a/arch/arm/boot/dts/am335x-evmsk.dts
+++ b/arch/arm/boot/dts/am335x-evmsk.dts
@@ -722,6 +722,6 @@
 };
 
 &rtc {
-	clocks = <&clk_32768_ck>, <&clkdiv32k_ick>;
+	clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 	clock-names = "ext-clk", "int-clk";
 };
diff --git a/arch/arm/boot/dts/am335x-pepper.dts b/arch/arm/boot/dts/am335x-pepper.dts
index 03c7d77023c6..9fb7426070ce 100644
--- a/arch/arm/boot/dts/am335x-pepper.dts
+++ b/arch/arm/boot/dts/am335x-pepper.dts
@@ -139,7 +139,7 @@
 &audio_codec {
 	status = "okay";
 
-	gpio-reset = <&gpio1 16 GPIO_ACTIVE_LOW>;
+	reset-gpios = <&gpio1 16 GPIO_ACTIVE_LOW>;
 	AVDD-supply = <&ldo3_reg>;
 	IOVDD-supply = <&ldo3_reg>;
 	DRVDD-supply = <&ldo3_reg>;
diff --git a/arch/arm/boot/dts/am33xx-clocks.dtsi b/arch/arm/boot/dts/am33xx-clocks.dtsi
index 8d8319590cde..95d5c9d136c5 100644
--- a/arch/arm/boot/dts/am33xx-clocks.dtsi
+++ b/arch/arm/boot/dts/am33xx-clocks.dtsi
@@ -292,14 +292,6 @@
 		clock-div = <4>;
 	};
 
-	cefuse_fck: cefuse_fck@a20 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_clkin_ck>;
-		ti,bit-shift = <1>;
-		reg = <0x0a20>;
-	};
-
 	clk_24mhz: clk_24mhz {
 		#clock-cells = <0>;
 		compatible = "fixed-factor-clock";
@@ -316,14 +308,6 @@
 		clock-div = <732>;
 	};
 
-	clkdiv32k_ick: clkdiv32k_ick@14c {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ck>;
-		ti,bit-shift = <1>;
-		reg = <0x014c>;
-	};
-
 	l3_gclk: l3_gclk {
 		#clock-cells = <0>;
 		compatible = "fixed-factor-clock";
@@ -350,49 +334,49 @@
 	timer1_fck: timer1_fck@528 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&clkdiv32k_ick>, <&tclkin_ck>, <&clk_rc32k_ck>, <&clk_32768_ck>;
+		clocks = <&sys_clkin_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>, <&tclkin_ck>, <&clk_rc32k_ck>, <&clk_32768_ck>;
 		reg = <0x0528>;
 	};
 
 	timer2_fck: timer2_fck@508 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&clkdiv32k_ick>;
+		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x0508>;
 	};
 
 	timer3_fck: timer3_fck@50c {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&clkdiv32k_ick>;
+		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x050c>;
 	};
 
 	timer4_fck: timer4_fck@510 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&clkdiv32k_ick>;
+		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x0510>;
 	};
 
 	timer5_fck: timer5_fck@518 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&clkdiv32k_ick>;
+		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x0518>;
 	};
 
 	timer6_fck: timer6_fck@51c {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&clkdiv32k_ick>;
+		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x051c>;
 	};
 
 	timer7_fck: timer7_fck@504 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&clkdiv32k_ick>;
+		clocks = <&tclkin_ck>, <&sys_clkin_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x0504>;
 	};
 
@@ -423,7 +407,7 @@
 	wdt1_fck: wdt1_fck@538 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&clk_rc32k_ck>, <&clkdiv32k_ick>;
+		clocks = <&clk_rc32k_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x0538>;
 	};
 
@@ -493,42 +477,10 @@
 	gpio0_dbclk_mux_ck: gpio0_dbclk_mux_ck@53c {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
-		clocks = <&clk_rc32k_ck>, <&clk_32768_ck>, <&clkdiv32k_ick>;
+		clocks = <&clk_rc32k_ck>, <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 		reg = <0x053c>;
 	};
 
-	gpio0_dbclk: gpio0_dbclk@408 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&gpio0_dbclk_mux_ck>;
-		ti,bit-shift = <18>;
-		reg = <0x0408>;
-	};
-
-	gpio1_dbclk: gpio1_dbclk@ac {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <18>;
-		reg = <0x00ac>;
-	};
-
-	gpio2_dbclk: gpio2_dbclk@b0 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <18>;
-		reg = <0x00b0>;
-	};
-
-	gpio3_dbclk: gpio3_dbclk@b4 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <18>;
-		reg = <0x00b4>;
-	};
-
 	lcd_gclk: lcd_gclk@534 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
@@ -577,58 +529,6 @@
 		reg = <0x0700>;
 	};
 
-	dbg_sysclk_ck: dbg_sysclk_ck@414 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_clkin_ck>;
-		ti,bit-shift = <19>;
-		reg = <0x0414>;
-	};
-
-	dbg_clka_ck: dbg_clka_ck@414 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_core_m4_ck>;
-		ti,bit-shift = <30>;
-		reg = <0x0414>;
-	};
-
-	stm_pmd_clock_mux_ck: stm_pmd_clock_mux_ck@414 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dbg_sysclk_ck>, <&dbg_clka_ck>;
-		ti,bit-shift = <22>;
-		reg = <0x0414>;
-	};
-
-	trace_pmd_clk_mux_ck: trace_pmd_clk_mux_ck@414 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dbg_sysclk_ck>, <&dbg_clka_ck>;
-		ti,bit-shift = <20>;
-		reg = <0x0414>;
-	};
-
-	stm_clk_div_ck: stm_clk_div_ck@414 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&stm_pmd_clock_mux_ck>;
-		ti,bit-shift = <27>;
-		ti,max-div = <64>;
-		reg = <0x0414>;
-		ti,index-power-of-two;
-	};
-
-	trace_clk_div_ck: trace_clk_div_ck@414 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&trace_pmd_clk_mux_ck>;
-		ti,bit-shift = <24>;
-		ti,max-div = <64>;
-		reg = <0x0414>;
-		ti,index-power-of-two;
-	};
-
 	clkout2_ck: clkout2_ck@700 {
 		#clock-cells = <0>;
 		compatible = "ti,gate-clock";
@@ -638,9 +538,88 @@
 	};
 };
 
-&prcm_clockdomains {
-	clk_24mhz_clkdm: clk_24mhz_clkdm {
-		compatible = "ti,clockdomain";
-		clocks = <&clkdiv32k_ick>;
+&prcm {
+	l4_per_cm: l4_per_cm@0 {
+		compatible = "ti,omap4-cm";
+		reg = <0x0 0x200>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x0 0x200>;
+
+		l4_per_clkctrl: clk@14 {
+			compatible = "ti,clkctrl";
+			reg = <0x14 0x13c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4_wkup_cm: l4_wkup_cm@400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x400 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x400 0x100>;
+
+		l4_wkup_clkctrl: clk@4 {
+			compatible = "ti,clkctrl";
+			reg = <0x4 0xd4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	mpu_cm: mpu_cm@600 {
+		compatible = "ti,omap4-cm";
+		reg = <0x600 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x600 0x100>;
+
+		mpu_clkctrl: clk@4 {
+			compatible = "ti,clkctrl";
+			reg = <0x4 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4_rtc_cm: l4_rtc_cm@800 {
+		compatible = "ti,omap4-cm";
+		reg = <0x800 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x800 0x100>;
+
+		l4_rtc_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	gfx_l3_cm: gfx_l3_cm@900 {
+		compatible = "ti,omap4-cm";
+		reg = <0x900 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x900 0x100>;
+
+		gfx_l3_clkctrl: clk@4 {
+			compatible = "ti,clkctrl";
+			reg = <0x4 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4_cefuse_cm: l4_cefuse_cm@a00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xa00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xa00 0x100>;
+
+		l4_cefuse_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi
index d37f95025807..628c77b0b386 100644
--- a/arch/arm/boot/dts/am33xx.dtsi
+++ b/arch/arm/boot/dts/am33xx.dtsi
@@ -10,6 +10,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/pinctrl/am33xx.h>
+#include <dt-bindings/clock/am3.h>
 
 / {
 	compatible = "ti,am33xx";
@@ -179,8 +180,11 @@
 			};
 
 			prcm: prcm@200000 {
-				compatible = "ti,am3-prcm";
+				compatible = "ti,am3-prcm", "simple-bus";
 				reg = <0x200000 0x4000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x200000 0x4000>;
 
 				prcm_clocks: clocks {
 					#address-cells = <1>;
@@ -496,7 +500,7 @@
 			status = "disabled";
 		};
 
-		mailbox: mailbox@480C8000 {
+		mailbox: mailbox@480c8000 {
 			compatible = "ti,omap4-mailbox";
 			reg = <0x480C8000 0x200>;
 			interrupts = <77>;
@@ -517,6 +521,8 @@
 			interrupts = <67>;
 			ti,hwmods = "timer1";
 			ti,timer-alwon;
+			clocks = <&timer1_fck>;
+			clock-names = "fck";
 		};
 
 		timer2: timer@48040000 {
@@ -524,6 +530,8 @@
 			reg = <0x48040000 0x400>;
 			interrupts = <68>;
 			ti,hwmods = "timer2";
+			clocks = <&timer2_fck>;
+			clock-names = "fck";
 		};
 
 		timer3: timer@48042000 {
@@ -571,7 +579,7 @@
 			interrupts = <75
 				      76>;
 			ti,hwmods = "rtc";
-			clocks = <&clkdiv32k_ick>;
+			clocks = <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>;
 			clock-names = "int-clk";
 		};
 
@@ -991,7 +999,7 @@
 			dma-names = "tx", "rx";
 		};
 
-		mcasp1: mcasp@4803C000 {
+		mcasp1: mcasp@4803c000 {
 			compatible = "ti,am33xx-mcasp-audio";
 			ti,hwmods = "mcasp1";
 			reg = <0x4803C000 0x2000>,
@@ -1014,4 +1022,4 @@
 	};
 };
 
-/include/ "am33xx-clocks.dtsi"
+#include "am33xx-clocks.dtsi"
diff --git a/arch/arm/boot/dts/am3517.dtsi b/arch/arm/boot/dts/am3517.dtsi
index 00da3f2c4072..ca294914bbb1 100644
--- a/arch/arm/boot/dts/am3517.dtsi
+++ b/arch/arm/boot/dts/am3517.dtsi
@@ -26,7 +26,7 @@
 			interrupt-names = "mc";
 		};
 
-		davinci_emac: ethernet@0x5c000000 {
+		davinci_emac: ethernet@5c000000 {
 			compatible = "ti,am3517-emac";
 			ti,hwmods = "davinci_emac";
 			status = "disabled";
@@ -41,7 +41,7 @@
 			local-mac-address = [ 00 00 00 00 00 00 ];
 		};
 
-		davinci_mdio: ethernet@0x5c030000 {
+		davinci_mdio: ethernet@5c030000 {
 			compatible = "ti,davinci_mdio";
 			ti,hwmods = "davinci_mdio";
 			status = "disabled";
@@ -99,9 +99,5 @@
 	status = "disabled";
 };
 
-&smartreflex_mpu_iva {
-	status = "disabled";
-};
-
 /include/ "am35xx-clocks.dtsi"
 /include/ "omap36xx-am35xx-omap3430es2plus-clocks.dtsi"
diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi
index 4714a59fd86d..964f3ef79728 100644
--- a/arch/arm/boot/dts/am4372.dtsi
+++ b/arch/arm/boot/dts/am4372.dtsi
@@ -10,6 +10,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/am4.h>
 
 / {
 	compatible = "ti,am4372", "ti,am43";
@@ -163,9 +164,12 @@
 			};
 
 			prcm: prcm@1f0000 {
-				compatible = "ti,am4-prcm";
+				compatible = "ti,am4-prcm", "simple-bus";
 				reg = <0x1f0000 0x11000>;
 				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x1f0000 0x11000>;
 
 				prcm_clocks: clocks {
 					#address-cells = <1>;
@@ -325,7 +329,7 @@
 			status = "disabled";
 		};
 
-		mailbox: mailbox@480C8000 {
+		mailbox: mailbox@480c8000 {
 			compatible = "ti,omap4-mailbox";
 			reg = <0x480C8000 0x200>;
 			interrupts = <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>;
@@ -346,6 +350,8 @@
 			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
 			ti,timer-alwon;
 			ti,hwmods = "timer1";
+			clocks = <&timer1_fck>;
+			clock-names = "fck";
 		};
 
 		timer2: timer@48040000  {
@@ -353,6 +359,8 @@
 			reg = <0x48040000  0x400>;
 			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer2";
+			clocks = <&timer2_fck>;
+			clock-names = "fck";
 		};
 
 		timer3: timer@48042000 {
@@ -936,7 +944,7 @@
 			dma-names = "tx", "rx";
 		};
 
-		mcasp1: mcasp@4803C000 {
+		mcasp1: mcasp@4803c000 {
 			compatible = "ti,am33xx-mcasp-audio";
 			ti,hwmods = "mcasp1";
 			reg = <0x4803C000 0x2000>,
@@ -993,7 +1001,7 @@
 				reg = <0x483a8000 0x8000>;
 				syscon-phy-power = <&scm_conf 0x620>;
 				clocks = <&usb_phy0_always_on_clk32k>,
-					 <&usb_otg_ss0_refclk960m>;
+					 <&l4_per_clkctrl AM4_USB_OTG_SS0_CLKCTRL 8>;
 				clock-names = "wkupclk", "refclk";
 				#phy-cells = <0>;
 				status = "disabled";
@@ -1012,7 +1020,7 @@
 				reg = <0x483e8000 0x8000>;
 				syscon-phy-power = <&scm_conf 0x628>;
 				clocks = <&usb_phy1_always_on_clk32k>,
-					 <&usb_otg_ss1_refclk960m>;
+					 <&l4_per_clkctrl AM4_USB_OTG_SS1_CLKCTRL 8>;
 				clock-names = "wkupclk", "refclk";
 				#phy-cells = <0>;
 				status = "disabled";
@@ -1175,4 +1183,4 @@
 	};
 };
 
-/include/ "am43xx-clocks.dtsi"
+#include "am43xx-clocks.dtsi"
diff --git a/arch/arm/boot/dts/am437x-gp-evm.dts b/arch/arm/boot/dts/am437x-gp-evm.dts
index afb8eb0a0a16..c3b1a3fb5a2e 100644
--- a/arch/arm/boot/dts/am437x-gp-evm.dts
+++ b/arch/arm/boot/dts/am437x-gp-evm.dts
@@ -55,7 +55,7 @@
 		enable-active-high;
 	};
 
-	backlight {
+	lcd_bl: backlight {
 		compatible = "pwm-backlight";
 		pwms = <&ecap0 0 50000 PWM_POLARITY_INVERTED>;
 		brightness-levels = <0 51 53 56 62 75 101 152 255>;
@@ -86,6 +86,8 @@
 		compatible = "osddisplays,osd057T0559-34ts", "panel-dpi";
 		label = "lcd";
 
+		backlight = <&lcd_bl>;
+
 		panel-timing {
 			clock-frequency = <33000000>;
 			hactive = <800>;
diff --git a/arch/arm/boot/dts/am437x-idk-evm.dts b/arch/arm/boot/dts/am437x-idk-evm.dts
index 5e364473067f..20132477a871 100644
--- a/arch/arm/boot/dts/am437x-idk-evm.dts
+++ b/arch/arm/boot/dts/am437x-idk-evm.dts
@@ -519,3 +519,17 @@
 &cpu {
 	cpu0-supply = <&tps>;
 };
+
+&cpu0_opp_table {
+	/*
+	 * Supply voltage supervisor on board will not allow opp50 so
+	 * disable it and set opp100 as suspend OPP.
+	 */
+	opp50@300000000 {
+		status = "disabled";
+	};
+
+	opp100@600000000 {
+		opp-suspend;
+	};
+};
diff --git a/arch/arm/boot/dts/am437x-sk-evm.dts b/arch/arm/boot/dts/am437x-sk-evm.dts
index 2c6bf0684f50..3fa3b226995d 100644
--- a/arch/arm/boot/dts/am437x-sk-evm.dts
+++ b/arch/arm/boot/dts/am437x-sk-evm.dts
@@ -35,7 +35,7 @@
 		clock-frequency = <32768>;
 	};
 
-	backlight {
+	lcd_bl: backlight {
 		compatible = "pwm-backlight";
 		pwms = <&ecap0 0 50000 PWM_POLARITY_INVERTED>;
 		brightness-levels = <0 51 53 56 62 75 101 152 255>;
@@ -132,6 +132,8 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&lcd_pins>;
 
+		backlight = <&lcd_bl>;
+
 		enable-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>;
 
 		panel-timing {
diff --git a/arch/arm/boot/dts/am43x-epos-evm.dts b/arch/arm/boot/dts/am43x-epos-evm.dts
index a04d79ec212a..00c3d1de384f 100644
--- a/arch/arm/boot/dts/am43x-epos-evm.dts
+++ b/arch/arm/boot/dts/am43x-epos-evm.dts
@@ -48,6 +48,8 @@
 		compatible = "osddisplays,osd057T0559-34ts", "panel-dpi";
 		label = "lcd";
 
+		backlight = <&lcd_bl>;
+
 		panel-timing {
 			clock-frequency = <33000000>;
 			hactive = <800>;
@@ -107,7 +109,7 @@
 			0x03030069>;	/* LEFT */
 	};
 
-	backlight {
+	lcd_bl: backlight {
 		compatible = "pwm-backlight";
 		pwms = <&ecap0 0 50000 PWM_POLARITY_INVERTED>;
 		brightness-levels = <0 51 53 56 62 75 101 152 255>;
@@ -985,7 +987,11 @@
 	rx-num-evt = <32>;
 };
 
-&synctimer_32kclk {
+&mux_synctimer32k_ck {
 	assigned-clocks = <&mux_synctimer32k_ck>;
 	assigned-clock-parents = <&clkdiv32k_ick>;
 };
+
+&cpu {
+	cpu0-supply = <&dcdc2>;
+};
diff --git a/arch/arm/boot/dts/am43xx-clocks.dtsi b/arch/arm/boot/dts/am43xx-clocks.dtsi
index 430be5829f8f..a7037a4b4fd4 100644
--- a/arch/arm/boot/dts/am43xx-clocks.dtsi
+++ b/arch/arm/boot/dts/am43xx-clocks.dtsi
@@ -524,54 +524,6 @@
 		reg = <0x4240>;
 	};
 
-	gpio0_dbclk: gpio0_dbclk@2b68 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&gpio0_dbclk_mux_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x2b68>;
-	};
-
-	gpio1_dbclk: gpio1_dbclk@8c78 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <8>;
-		reg = <0x8c78>;
-	};
-
-	gpio2_dbclk: gpio2_dbclk@8c80 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <8>;
-		reg = <0x8c80>;
-	};
-
-	gpio3_dbclk: gpio3_dbclk@8c88 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <8>;
-		reg = <0x8c88>;
-	};
-
-	gpio4_dbclk: gpio4_dbclk@8c90 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <8>;
-		reg = <0x8c90>;
-	};
-
-	gpio5_dbclk: gpio5_dbclk@8c98 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkdiv32k_ick>;
-		ti,bit-shift = <8>;
-		reg = <0x8c98>;
-	};
-
 	mmc_clk: mmc_clk {
 		#clock-cells = <0>;
 		compatible = "fixed-factor-clock";
@@ -629,14 +581,6 @@
 		reg = <0x4230>;
 	};
 
-	synctimer_32kclk: synctimer_32kclk@2a30 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&mux_synctimer32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x2a30>;
-	};
-
 	timer8_fck: timer8_fck@421c {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
@@ -763,110 +707,76 @@
 		ti,bit-shift = <8>;
 		reg = <0x2a48>;
 	};
+};
 
-	usb_otg_ss0_refclk960m: usb_otg_ss0_refclk960m@8a60 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_per_clkdcoldo>;
-		ti,bit-shift = <8>;
-		reg = <0x8a60>;
-	};
-
-	usb_otg_ss1_refclk960m: usb_otg_ss1_refclk960m@8a68 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_per_clkdcoldo>;
-		ti,bit-shift = <8>;
-		reg = <0x8a68>;
-	};
-
-	clkout1_osc_div_ck: clkout1_osc_div_ck {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&sys_clkin_ck>;
-		ti,bit-shift = <20>;
-		ti,max-div = <4>;
-		reg = <0x4100>;
-	};
-
-	clkout1_src2_mux_ck: clkout1_src2_mux_ck {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&clk_rc32k_ck>, <&sysclk_div>, <&dpll_ddr_m2_ck>,
-			 <&dpll_per_m2_ck>, <&dpll_disp_m2_ck>,
-			 <&dpll_mpu_m2_ck>;
-		reg = <0x4100>;
-	};
-
-	clkout1_src2_pre_div_ck: clkout1_src2_pre_div_ck {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&clkout1_src2_mux_ck>;
-		ti,bit-shift = <4>;
-		ti,max-div = <8>;
-		reg = <0x4100>;
-	};
-
-	clkout1_src2_post_div_ck: clkout1_src2_post_div_ck {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&clkout1_src2_pre_div_ck>;
-		ti,bit-shift = <8>;
-		ti,max-div = <32>;
-		ti,index-power-of-two;
-		reg = <0x4100>;
-	};
-
-	clkout1_mux_ck: clkout1_mux_ck {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&clkout1_osc_div_ck>, <&clk_rc32k_ck>,
-			 <&clkout1_src2_post_div_ck>, <&dpll_extdev_m2_ck>;
-		ti,bit-shift = <16>;
-		reg = <0x4100>;
-	};
-
-	clkout1_ck: clkout1_ck {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkout1_mux_ck>;
-		ti,bit-shift = <23>;
-		reg = <0x4100>;
-	};
-
-	clkout2_src_mux_ck: clkout2_src_mux_ck {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&clk_rc32k_ck>, <&sysclk_div>, <&dpll_ddr_m2_ck>,
-			 <&dpll_per_m2_ck>, <&dpll_disp_m2_ck>,
-			 <&dpll_mpu_m2_ck>, <&dpll_extdev_ck>;
-		reg = <0x4108>;
-	};
-
-	clkout2_pre_div_ck: clkout2_pre_div_ck {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&clkout2_src_mux_ck>;
-		ti,bit-shift = <4>;
-		ti,max-div = <8>;
-		reg = <0x4108>;
-	};
-
-	clkout2_post_div_ck: clkout2_post_div_ck {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&clkout2_pre_div_ck>;
-		ti,bit-shift = <8>;
-		ti,max-div = <32>;
-		ti,index-power-of-two;
-		reg = <0x4108>;
-	};
-
-	clkout2_ck: clkout2_ck {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&clkout2_post_div_ck>;
-		ti,bit-shift = <16>;
-		reg = <0x4108>;
+&prcm {
+	l4_wkup_cm: l4_wkup_cm@2800 {
+		compatible = "ti,omap4-cm";
+		reg = <0x2800 0x400>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x2800 0x400>;
+
+		l4_wkup_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x34c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	mpu_cm: mpu_cm@8300 {
+		compatible = "ti,omap4-cm";
+		reg = <0x8300 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x8300 0x100>;
+
+		mpu_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	gfx_l3_cm: gfx_l3_cm@8400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x8400 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x8400 0x100>;
+
+		gfx_l3_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4_rtc_cm: l4_rtc_cm@8500 {
+		compatible = "ti,omap4-cm";
+		reg = <0x8500 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x8500 0x100>;
+
+		l4_rtc_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4_per_cm: l4_per_cm@8800 {
+		compatible = "ti,omap4-cm";
+		reg = <0x8800 0xc00>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x8800 0xc00>;
+
+		l4_per_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xb04>;
+			#clock-cells = <2>;
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/am571x-idk.dts b/arch/arm/boot/dts/am571x-idk.dts
index debf9464403e..6d3c83743156 100644
--- a/arch/arm/boot/dts/am571x-idk.dts
+++ b/arch/arm/boot/dts/am571x-idk.dts
@@ -117,3 +117,7 @@
 	pinctrl-1 = <&mmc2_pins_hs>;
 	pinctrl-2 = <&mmc2_pins_ddr_rev20 &mmc2_iodelay_ddr_conf>;
 };
+
+&cpu0 {
+	vdd-supply = <&smps12_reg>;
+};
diff --git a/arch/arm/boot/dts/am572x-idk-common.dtsi b/arch/arm/boot/dts/am572x-idk-common.dtsi
new file mode 100644
index 000000000000..c6d858b31011
--- /dev/null
+++ b/arch/arm/boot/dts/am572x-idk-common.dtsi
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "am57xx-idk-common.dtsi"
+
+/ {
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x80000000 0x0 0x80000000>;
+	};
+
+	status-leds {
+		compatible = "gpio-leds";
+		cpu0-led {
+			label = "status0:red:cpu0";
+			gpios = <&gpio4 0 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "cpu0";
+		};
+
+		usr0-led {
+			label = "status0:green:usr";
+			gpios = <&gpio3 11 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		heartbeat-led {
+			label = "status0:blue:heartbeat";
+			gpios = <&gpio3 12 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "heartbeat";
+		};
+
+		cpu1-led {
+			label = "status1:red:cpu1";
+			gpios = <&gpio3 10 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "cpu1";
+		};
+
+		usr1-led {
+			label = "status1:green:usr";
+			gpios = <&gpio7 23 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		mmc0-led {
+			label = "status1:blue:mmc0";
+			gpios = <&gpio7 22 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "mmc0";
+		};
+	};
+};
+
+&omap_dwc3_2 {
+	extcon = <&extcon_usb2>;
+};
+
+&extcon_usb2 {
+	id-gpio = <&gpio3 16 GPIO_ACTIVE_HIGH>;
+	vbus-gpio = <&gpio3 26 GPIO_ACTIVE_HIGH>;
+};
+
+&sn65hvs882 {
+	load-gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;
+};
+
+&pcie1_rc {
+	status = "okay";
+	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
+};
+
+&pcie1_ep {
+	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
+};
+
+&mailbox5 {
+	status = "okay";
+	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
+		status = "okay";
+	};
+	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
+		status = "okay";
+	};
+};
+
+&mailbox6 {
+	status = "okay";
+	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
+		status = "okay";
+	};
+	mbox_dsp2_ipc3x: mbox_dsp2_ipc3x {
+		status = "okay";
+	};
+};
diff --git a/arch/arm/boot/dts/am572x-idk.dts b/arch/arm/boot/dts/am572x-idk.dts
index a578fe97ba3b..9ab0af5017df 100644
--- a/arch/arm/boot/dts/am572x-idk.dts
+++ b/arch/arm/boot/dts/am572x-idk.dts
@@ -9,8 +9,7 @@
 /dts-v1/;
 
 #include "dra74x.dtsi"
-#include <dt-bindings/gpio/gpio.h>
-#include <dt-bindings/interrupt-controller/irq.h>
+#include "am572x-idk-common.dtsi"
 #include "am57xx-idk-common.dtsi"
 #include "dra74x-mmc-iodelay.dtsi"
 
@@ -18,54 +17,6 @@
 	model = "TI AM5728 IDK";
 	compatible = "ti,am5728-idk", "ti,am5728", "ti,dra742", "ti,dra74",
 		     "ti,dra7";
-
-	memory@0 {
-		device_type = "memory";
-		reg = <0x0 0x80000000 0x0 0x80000000>;
-	};
-
-	status-leds {
-		compatible = "gpio-leds";
-		cpu0-led {
-			label = "status0:red:cpu0";
-			gpios = <&gpio4 0 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-			linux,default-trigger = "cpu0";
-		};
-
-		usr0-led {
-			label = "status0:green:usr";
-			gpios = <&gpio3 11 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		heartbeat-led {
-			label = "status0:blue:heartbeat";
-			gpios = <&gpio3 12 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-			linux,default-trigger = "heartbeat";
-		};
-
-		cpu1-led {
-			label = "status1:red:cpu1";
-			gpios = <&gpio3 10 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-			linux,default-trigger = "cpu1";
-		};
-
-		usr1-led {
-			label = "status1:green:usr";
-			gpios = <&gpio7 23 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-		};
-
-		mmc0-led {
-			label = "status1:blue:mmc0";
-			gpios = <&gpio7 22 GPIO_ACTIVE_HIGH>;
-			default-state = "off";
-			linux,default-trigger = "mmc0";
-		};
-	};
 };
 
 &mmc1 {
@@ -86,44 +37,6 @@
 	pinctrl-2 = <&mmc2_pins_ddr_rev20>;
 };
 
-&omap_dwc3_2 {
-	extcon = <&extcon_usb2>;
-};
-
-&extcon_usb2 {
-	id-gpio = <&gpio3 16 GPIO_ACTIVE_HIGH>;
-	vbus-gpio = <&gpio3 26 GPIO_ACTIVE_HIGH>;
-};
-
-&sn65hvs882 {
-	load-gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;
-};
-
-&pcie1_rc {
-	status = "okay";
-	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
-};
-
-&pcie1_ep {
-	gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;
-};
-
-&mailbox5 {
-	status = "okay";
-	mbox_ipu1_ipc3x: mbox_ipu1_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp1_ipc3x: mbox_dsp1_ipc3x {
-		status = "okay";
-	};
-};
-
-&mailbox6 {
-	status = "okay";
-	mbox_ipu2_ipc3x: mbox_ipu2_ipc3x {
-		status = "okay";
-	};
-	mbox_dsp2_ipc3x: mbox_dsp2_ipc3x {
-		status = "okay";
-	};
+&cpu0 {
+	vdd-supply = <&smps12_reg>;
 };
diff --git a/arch/arm/boot/dts/am574x-idk.dts b/arch/arm/boot/dts/am574x-idk.dts
new file mode 100644
index 000000000000..41e12a382d2f
--- /dev/null
+++ b/arch/arm/boot/dts/am574x-idk.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/
+ */
+
+
+/dts-v1/;
+
+#include "dra76x.dtsi"
+#include "am572x-idk-common.dtsi"
+
+/ {
+	model = "TI AM5748 IDK";
+	compatible = "ti,am5728-idk", "ti,dra762", "ti,dra7";
+};
+
+&qspi {
+	spi-max-frequency = <96000000>;
+	m25p80@0 {
+		spi-max-frequency = <96000000>;
+	};
+};
diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
index 49aeecd312b4..ab60035bc50c 100644
--- a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
+++ b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi
@@ -388,7 +388,7 @@
 };
 
 &cpu0 {
-	cpu0-supply = <&smps12_reg>;
+	vdd-supply = <&smps12_reg>;
 	voltage-tolerance = <1>;
 };
 
@@ -554,7 +554,7 @@
 
 &mcasp3 {
 	#sound-dai-cells = <0>;
-	assigned-clocks = <&mcasp3_ahclkx_mux>;
+	assigned-clocks = <&l4per_clkctrl DRA7_MCASP3_CLKCTRL 24>;
 	assigned-clock-parents = <&sys_clkin2>;
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/animeo_ip.dts b/arch/arm/boot/dts/animeo_ip.dts
index 26ade8c0a960..b67a75179784 100644
--- a/arch/arm/boot/dts/animeo_ip.dts
+++ b/arch/arm/boot/dts/animeo_ip.dts
@@ -43,6 +43,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usart0: serial@fffb0000 {
 				pinctrl-0 = <&pinctrl_usart0 &pinctrl_usart0_rts>;
 				linux,rs485-enabled-at-boot-time;
diff --git a/arch/arm/boot/dts/arm-realview-eb-mp.dtsi b/arch/arm/boot/dts/arm-realview-eb-mp.dtsi
index 7b8d90b7aeea..29b636fce23f 100644
--- a/arch/arm/boot/dts/arm-realview-eb-mp.dtsi
+++ b/arch/arm/boot/dts/arm-realview-eb-mp.dtsi
@@ -150,11 +150,6 @@
 	interrupts = <0 8 IRQ_TYPE_LEVEL_HIGH>;
 };
 
-&charlcd {
-	interrupt-parent = <&intc>;
-	interrupts = <0  IRQ_TYPE_LEVEL_HIGH>;
-};
-
 &serial0 {
 	interrupt-parent = <&intc>;
 	interrupts = <0 4 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/armada-38x.dtsi b/arch/arm/boot/dts/armada-38x.dtsi
index 00ff549d4e39..a6cc568f74f7 100644
--- a/arch/arm/boot/dts/armada-38x.dtsi
+++ b/arch/arm/boot/dts/armada-38x.dtsi
@@ -279,6 +279,11 @@
 					marvell,function = "dev";
 				};
 
+				nand_rb: nand-rb {
+					marvell,pins = "mpp41";
+					marvell,function = "nand";
+				};
+
 				uart0_pins: uart-pins-0 {
 					marvell,pins = "mpp0", "mpp1";
 					marvell,function = "ua0";
diff --git a/arch/arm/boot/dts/aspeed-ast2500-evb.dts b/arch/arm/boot/dts/aspeed-ast2500-evb.dts
index 602bc10fdaf4..91a36c1f029b 100644
--- a/arch/arm/boot/dts/aspeed-ast2500-evb.dts
+++ b/arch/arm/boot/dts/aspeed-ast2500-evb.dts
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0+
 /dts-v1/;
 
 #include "aspeed-g5.dtsi"
@@ -16,7 +16,7 @@
 		bootargs = "console=ttyS4,115200 earlyprintk";
 	};
 
-	memory {
+	memory@80000000 {
 		reg = <0x80000000 0x20000000>;
 	};
 };
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-palmetto.dts b/arch/arm/boot/dts/aspeed-bmc-opp-palmetto.dts
index c786bc2f2919..4379d09a261f 100644
--- a/arch/arm/boot/dts/aspeed-bmc-opp-palmetto.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-palmetto.dts
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0+
 /dts-v1/;
 
 #include "aspeed-g4.dtsi"
@@ -12,7 +12,7 @@
 		bootargs = "console=ttyS4,115200 earlyprintk";
 	};
 
-	memory {
+	memory@40000000 {
 		reg = <0x40000000 0x20000000>;
 	};
 
@@ -34,6 +34,7 @@
 		status = "okay";
 		m25p,fast-read;
 		label = "bmc";
+#include "openbmc-flash-layout.dtsi"
 	};
 };
 
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts b/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts
index 8067793129ea..623b6ab42021 100644
--- a/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-romulus.dts
@@ -1,23 +1,19 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: GPL-2.0+
 /dts-v1/;
-
 #include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
 
 / {
 	model = "Romulus BMC";
 	compatible = "ibm,romulus-bmc", "aspeed,ast2500";
 
-	aliases {
-		serial4 = &uart5;
-	};
-
 	chosen {
 		stdout-path = &uart5;
 		bootargs = "console=ttyS4,115200 earlyprintk";
 	};
 
-	memory {
-		reg = <0x80000000 0x40000000>;
+	memory@80000000 {
+		reg = <0x80000000 0x20000000>;
 	};
 
 	reserved-memory {
@@ -29,6 +25,49 @@
 			no-map;
 			reg = <0xbf000000 0x01000000>; /* 16M */
 		};
+
+		flash_memory: region@98000000 {
+			no-map;
+			reg = <0x98000000 0x04000000>; /* 64M */
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		fault {
+			gpios = <&gpio ASPEED_GPIO(N, 2) GPIO_ACTIVE_LOW>;
+		};
+
+		identify {
+			gpios = <&gpio ASPEED_GPIO(N, 4) GPIO_ACTIVE_HIGH>;
+		};
+
+		power {
+			gpios = <&gpio ASPEED_GPIO(R, 5) GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	fsi: gpio-fsi {
+		compatible = "fsi-master-gpio", "fsi-master";
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		clock-gpios = <&gpio ASPEED_GPIO(AA, 0) GPIO_ACTIVE_HIGH>;
+		data-gpios = <&gpio ASPEED_GPIO(AA, 2) GPIO_ACTIVE_HIGH>;
+		mux-gpios = <&gpio ASPEED_GPIO(A, 6) GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio ASPEED_GPIO(D, 0) GPIO_ACTIVE_HIGH>;
+		trans-gpios = <&gpio ASPEED_GPIO(R, 2) GPIO_ACTIVE_HIGH>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		checkstop {
+			label = "checkstop";
+			gpios = <&gpio ASPEED_GPIO(J, 2) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(J, 2)>;
+		};
 	};
 };
 
@@ -38,6 +77,7 @@
 		status = "okay";
 		m25p,fast-read;
 		label = "bmc";
+#include "openbmc-flash-layout.dtsi"
 	};
 };
 
@@ -53,6 +93,12 @@
 	};
 };
 
+&lpc_ctrl {
+	status = "okay";
+	memory-region = <&flash_memory>;
+	flash = <&spi1>;
+};
+
 &uart1 {
 	/* Rear RS-232 connector */
 	status = "okay";
@@ -81,6 +127,10 @@
 	pinctrl-0 = <&pinctrl_rmii1_default>;
 };
 
+&i2c1 {
+	status = "okay";
+};
+
 &i2c2 {
 	status = "okay";
 };
@@ -133,8 +183,77 @@
 
 &i2c12 {
 	status = "okay";
+
+	max31785@52 {
+		compatible = "maxim,max31785";
+		reg = <0x52>;
+	};
+};
+
+&gpio {
+	nic_func_mode0 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(D, 3) GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "nic_func_mode0";
+	};
+	nic_func_mode1 {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(D, 4) GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "nic_func_mode1";
+	};
 };
 
 &vuart {
 	status = "okay";
 };
+
+&gfx {
+	status = "okay";
+};
+
+&pinctrl {
+	aspeed,external-nodes = <&gfx &lhc>;
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default &pinctrl_pwm1_default>;
+
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x08>;
+	};
+
+	fan@1 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x09>;
+	};
+
+	fan@2 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0a>;
+	};
+
+	fan@3 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0b>;
+	};
+
+	fan@4 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0c>;
+	};
+
+	fan@5 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0d>;
+	};
+
+	fan@6 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0e>;
+	};
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts b/arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts
new file mode 100644
index 000000000000..5f9049d2c4c3
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-witherspoon.dts
@@ -0,0 +1,548 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+#include <dt-bindings/leds/leds-pca955x.h>
+
+/ {
+	model = "Witherspoon BMC";
+	compatible = "ibm,witherspoon-bmc", "aspeed,ast2500";
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=ttyS4,115200 earlyprintk";
+	};
+
+	memory@80000000 {
+		reg = <0x80000000 0x20000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		flash_memory: region@98000000 {
+			no-map;
+			reg = <0x98000000 0x04000000>; /* 64M */
+		};
+	};
+
+	gpio-keys-polled {
+		compatible = "gpio-keys-polled";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		poll-interval = <1000>;
+
+		fan0-presence {
+			label = "fan0-presence";
+			gpios = <&pca0 4 GPIO_ACTIVE_LOW>;
+			linux,code = <4>;
+		};
+
+		fan1-presence {
+			label = "fan1-presence";
+			gpios = <&pca0 5 GPIO_ACTIVE_LOW>;
+			linux,code = <5>;
+		};
+
+		fan2-presence {
+			label = "fan2-presence";
+			gpios = <&pca0 6 GPIO_ACTIVE_LOW>;
+			linux,code = <6>;
+		};
+
+		fan3-presence {
+			label = "fan3-presence";
+			gpios = <&pca0 7 GPIO_ACTIVE_LOW>;
+			linux,code = <7>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		fan0 {
+			retain-state-shutdown;
+			default-state = "keep";
+			gpios = <&pca0 0 GPIO_ACTIVE_LOW>;
+		};
+
+		fan1 {
+			retain-state-shutdown;
+			default-state = "keep";
+			gpios = <&pca0 1 GPIO_ACTIVE_LOW>;
+		};
+
+		fan2 {
+			retain-state-shutdown;
+			default-state = "keep";
+			gpios = <&pca0 2 GPIO_ACTIVE_LOW>;
+		};
+
+		fan3 {
+			retain-state-shutdown;
+			default-state = "keep";
+			gpios = <&pca0 3 GPIO_ACTIVE_LOW>;
+		};
+
+		front-fault {
+			retain-state-shutdown;
+			default-state = "keep";
+			gpios = <&pca0 13 GPIO_ACTIVE_LOW>;
+		};
+
+		front-power {
+			retain-state-shutdown;
+			default-state = "keep";
+			gpios = <&pca0 14 GPIO_ACTIVE_LOW>;
+		};
+
+		front-id {
+			retain-state-shutdown;
+			default-state = "keep";
+			gpios = <&pca0 15 GPIO_ACTIVE_LOW>;
+		};
+
+		rear-fault {
+			gpios = <&gpio ASPEED_GPIO(N, 2) GPIO_ACTIVE_LOW>;
+		};
+
+		rear-id {
+			gpios = <&gpio ASPEED_GPIO(N, 4) GPIO_ACTIVE_LOW>;
+		};
+
+		rear-power {
+			gpios = <&gpio ASPEED_GPIO(N, 3) GPIO_ACTIVE_LOW>;
+		};
+
+		power-button {
+			gpios = <&gpio ASPEED_GPIO(R, 5) GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	fsi: gpio-fsi {
+		compatible = "fsi-master-gpio", "fsi-master";
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		clock-gpios = <&gpio ASPEED_GPIO(AA, 0) GPIO_ACTIVE_HIGH>;
+		data-gpios = <&gpio ASPEED_GPIO(E, 0) GPIO_ACTIVE_HIGH>;
+		mux-gpios = <&gpio ASPEED_GPIO(A, 6) GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio ASPEED_GPIO(D, 0) GPIO_ACTIVE_HIGH>;
+		trans-gpios = <&gpio ASPEED_GPIO(R, 2) GPIO_ACTIVE_HIGH>;
+	};
+
+	iio-hwmon-dps310 {
+		compatible = "iio-hwmon";
+		io-channels = <&dps 0>;
+	};
+
+	iio-hwmon-bmp280 {
+		compatible = "iio-hwmon";
+		io-channels = <&bmp 1>;
+	};
+
+};
+
+&fmc {
+	status = "okay";
+
+	flash@0 {
+		status = "okay";
+		label = "bmc";
+		m25p,fast-read;
+#include "openbmc-flash-layout.dtsi"
+	};
+
+	flash@1 {
+		status = "okay";
+		label = "alt";
+		m25p,fast-read;
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1_default>;
+
+	flash@0 {
+		status = "okay";
+		label = "pnor";
+		m25p,fast-read;
+	};
+};
+
+&uart1 {
+	/* Rear RS-232 connector */
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_txd1_default
+			&pinctrl_rxd1_default
+			&pinctrl_nrts1_default
+			&pinctrl_ndtr1_default
+			&pinctrl_ndsr1_default
+			&pinctrl_ncts1_default
+			&pinctrl_ndcd1_default
+			&pinctrl_nri1_default>;
+};
+
+&uart2 {
+	/* APSS */
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_txd2_default &pinctrl_rxd2_default>;
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&lpc_ctrl {
+	status = "okay";
+	memory-region = <&flash_memory>;
+	flash = <&spi1>;
+};
+
+&mac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rmii1_default>;
+	use-ncsi;
+};
+
+&i2c2 {
+	status = "okay";
+
+	/* MUX ->
+	 *    Samtec 1
+	 *    Samtec 2
+	 */
+};
+
+&i2c3 {
+	status = "okay";
+
+	bmp: bmp280@77 {
+		compatible = "bosch,bmp280";
+		reg = <0x77>;
+		#io-channel-cells = <1>;
+	};
+
+	max31785@52 {
+		compatible = "maxim,max31785a";
+		reg = <0x52>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	dps: dps310@76 {
+		compatible = "infineon,dps310";
+		reg = <0x76>;
+		#io-channel-cells = <0>;
+	};
+
+	pca0: pca9552@60 {
+		compatible = "nxp,pca9552";
+		reg = <0x60>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		gpio@0 {
+			reg = <0>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@1 {
+			reg = <1>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@2 {
+			reg = <2>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@3 {
+			reg = <3>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@4 {
+			reg = <4>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@5 {
+			reg = <5>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@6 {
+			reg = <6>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@7 {
+			reg = <7>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@8 {
+			reg = <8>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@9 {
+			reg = <9>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@10 {
+			reg = <10>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@11 {
+			reg = <11>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@12 {
+			reg = <12>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@13 {
+			reg = <13>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@14 {
+			reg = <14>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@15 {
+			reg = <15>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+	};
+
+	power-supply@68 {
+		compatible = "ibm,cffps1";
+		reg = <0x68>;
+	};
+
+	power-supply@69 {
+		compatible = "ibm,cffps1";
+		reg = <0x69>;
+	};
+};
+
+&i2c4 {
+	status = "okay";
+
+	tmp423a@4c {
+		compatible = "ti,tmp423";
+		reg = <0x4c>;
+	};
+
+	ir35221@70 {
+		compatible = "infineon,ir35221";
+		reg = <0x70>;
+	};
+
+	ir35221@71 {
+		compatible = "infineon,ir35221";
+		reg = <0x71>;
+	};
+};
+
+
+&i2c5 {
+	status = "okay";
+
+	tmp423a@4c {
+		compatible = "ti,tmp423";
+		reg = <0x4c>;
+	};
+
+	ir35221@70 {
+		compatible = "infineon,ir35221";
+		reg = <0x70>;
+	};
+
+	ir35221@71 {
+		compatible = "infineon,ir35221";
+		reg = <0x71>;
+	};
+};
+
+&i2c9 {
+	status = "okay";
+
+	tmp275@4a {
+		compatible = "ti,tmp275";
+		reg = <0x4a>;
+	};
+};
+
+&i2c10 {
+	/* MUX
+	 *   -> PCIe Slot 3
+	 *   -> PCIe Slot 4
+	 */
+	status = "okay";
+};
+
+&i2c11 {
+	status = "okay";
+
+	pca9552: pca9552@60 {
+		compatible = "nxp,pca9552";
+		reg = <0x60>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		gpio-line-names = "PS_SMBUS_RESET_N", "APSS_RESET_N",
+			"GPU0_TH_OVERT_N_BUFF",	"GPU1_TH_OVERT_N_BUFF",
+			"GPU2_TH_OVERT_N_BUFF", "GPU3_TH_OVERT_N_BUFF",
+			"GPU4_TH_OVERT_N_BUFF",	"GPU5_TH_OVERT_N_BUFF",
+			"GPU0_PWR_GOOD_BUFF", "GPU1_PWR_GOOD_BUFF",
+			"GPU2_PWR_GOOD_BUFF", "GPU3_PWR_GOOD_BUFF",
+			"GPU4_PWR_GOOD_BUFF", "GPU5_PWR_GOOD_BUFF",
+			"12V_BREAKER_FLT_N", "THROTTLE_UNLATCHED_N";
+
+		gpio@0 {
+			reg = <0>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@1 {
+			reg = <1>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@2 {
+			reg = <2>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@3 {
+			reg = <3>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@4 {
+			reg = <4>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@5 {
+			reg = <5>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@6 {
+			reg = <6>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@7 {
+			reg = <7>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@8 {
+			reg = <8>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@9 {
+			reg = <9>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@10 {
+			reg = <10>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@11 {
+			reg = <11>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@12 {
+			reg = <12>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@13 {
+			reg = <13>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@14 {
+			reg = <14>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+
+		gpio@15 {
+			reg = <15>;
+			type = <PCA955X_TYPE_GPIO>;
+		};
+	};
+
+	rtc@32 {
+		compatible = "epson,rx8900";
+		reg = <0x32>;
+	};
+
+	eeprom@51 {
+		compatible = "atmel,24c64";
+		reg = <0x51>;
+	};
+
+	ucd90160@64 {
+		compatible = "ti,ucd90160";
+		reg = <0x64>;
+	};
+};
+
+&i2c12 {
+	status = "okay";
+};
+
+&i2c13 {
+	status = "okay";
+};
+
+&vuart {
+	status = "okay";
+};
+
+&gfx {
+	status = "okay";
+};
+
+&pinctrl {
+	aspeed,external-nodes = <&gfx &lhc>;
+};
+
+&wdt1 {
+	aspeed,reset-type = "none";
+	aspeed,external-signal;
+	aspeed,ext-push-pull;
+	aspeed,ext-active-high;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdtrst1_default>;
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts b/arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts
new file mode 100644
index 000000000000..c881484a85cf
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-opp-zaius.dts
@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: GPL-2.0
+/dts-v1/;
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+
+/ {
+	model = "Zaius BMC";
+	compatible = "ingrasys,zaius-bmc", "aspeed,ast2500";
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=ttyS4,115200 earlyprintk";
+	};
+
+	memory@80000000 {
+		reg = <0x80000000 0x40000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		flash_memory: region@98000000 {
+			no-map;
+			reg = <0x98000000 0x04000000>; /* 64M */
+		};
+	};
+
+	onewire0 {
+		compatible = "w1-gpio";
+		gpios = <&gpio ASPEED_GPIO(H, 0) GPIO_ACTIVE_HIGH>;
+	};
+
+	onewire1 {
+		compatible = "w1-gpio";
+		gpios = <&gpio ASPEED_GPIO(H, 1) GPIO_ACTIVE_HIGH>;
+	};
+
+	onewire2 {
+		compatible = "w1-gpio";
+		gpios = <&gpio ASPEED_GPIO(H, 2) GPIO_ACTIVE_HIGH>;
+	};
+
+	onewire3 {
+		compatible = "w1-gpio";
+		gpios = <&gpio ASPEED_GPIO(H, 3) GPIO_ACTIVE_HIGH>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		checkstop {
+			label = "checkstop";
+			gpios = <&gpio ASPEED_GPIO(F, 7) GPIO_ACTIVE_LOW>;
+			linux,code = <ASPEED_GPIO(F, 7)>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		sys_boot_status {
+			label = "System boot status";
+			gpios = <&gpio ASPEED_GPIO(D, 5) GPIO_ACTIVE_LOW>;
+		};
+
+		attention {
+			label = "Attention";
+			gpios = <&gpio ASPEED_GPIO(D, 6) GPIO_ACTIVE_LOW>;
+		};
+
+		plt_fault {
+			label = "Platform fault";
+			gpios = <&gpio ASPEED_GPIO(D, 7) GPIO_ACTIVE_LOW>;
+		};
+
+		hdd_fault {
+			label = "Onboard drive fault";
+			gpios = <&gpio ASPEED_GPIO(AA, 2) GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	fsi: gpio-fsi {
+		compatible = "fsi-master-gpio", "fsi-master";
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		trans-gpios = <&gpio ASPEED_GPIO(O, 6) GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio ASPEED_GPIO(D, 0) GPIO_ACTIVE_HIGH>;
+		clock-gpios = <&gpio ASPEED_GPIO(G, 0) GPIO_ACTIVE_HIGH>;
+		data-gpios = <&gpio ASPEED_GPIO(G, 1) GPIO_ACTIVE_HIGH>;
+		mux-gpios = <&gpio ASPEED_GPIO(P, 6) GPIO_ACTIVE_HIGH>;
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>,
+			<&adc 4>, <&adc 5>, <&adc 6>, <&adc 7>,
+			<&adc 8>, <&adc 9>, <&adc 10>, <&adc 11>,
+			<&adc 13>, <&adc 14>, <&adc 15>;
+	};
+
+	iio-hwmon-battery {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 12>;
+	};
+
+};
+
+&fmc {
+	status = "okay";
+
+	flash@0 {
+		status = "okay";
+		label = "bmc";
+		m25p,fast-read;
+#include "openbmc-flash-layout.dtsi"
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1_default>;
+
+	flash@0 {
+		status = "okay";
+		label = "pnor";
+		m25p,fast-read;
+	};
+};
+
+&spi2 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi2ck_default
+		     &pinctrl_spi2cs0_default
+		     &pinctrl_spi2cs1_default
+		     &pinctrl_spi2miso_default
+		     &pinctrl_spi2mosi_default>;
+
+	flash@0 {
+		status = "okay";
+	};
+};
+
+&uart1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_txd1_default
+		     &pinctrl_rxd1_default>;
+};
+
+&lpc_ctrl {
+	status = "okay";
+	memory-region = <&flash_memory>;
+	flash = <&spi1>;
+};
+
+&lpc_snoop {
+	status = "okay";
+	snoop-ports = <0x80>;
+};
+
+
+&uart5 {
+	status = "okay";
+};
+
+&mac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rmii1_default>;
+	use-ncsi;
+};
+
+&mac1 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii2_default &pinctrl_mdio2_default>;
+};
+
+&i2c0 {
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+		pagesize = <32>;
+	};
+
+	rtc@68 {
+		compatible = "nxp,pcf8523";
+		reg = <0x68>;
+	};
+
+	ucd90160@64 {
+		compatible = "ti,ucd90160";
+		reg = <0x64>;
+	};
+
+	/* Power sequencer UCD90160 PMBUS @64h
+	 * FRU AT24C64D @50h
+	 * RTC PCF8523 @68h
+	 * Clock buffer 9DBL04 @6dh
+	 */
+};
+
+&i2c1 {
+	status = "okay";
+
+	i2c-switch@71 {
+		compatible = "nxp,pca9546";
+		reg = <0x71>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	/* MUX1 PCA9546A @71h
+	 *   PCIe 0
+	 *   PCIe 1
+	 *   PCIe 2
+	 *   TPM header
+	 */
+};
+
+&i2c2 {
+	status = "disabled";
+
+	/* OCP Mezz Connector A (OOB SMBUS) */
+};
+
+&i2c3 {
+	status = "disabled";
+
+	/* OCP Mezz Connector A (PCIe slot SMBUS) */
+};
+
+&i2c4 {
+	status = "okay";
+
+	i2c-switch@71 {
+		compatible = "nxp,pca9546";
+		reg = <0x71>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+
+	/* MUX1 PCA9546A @71h
+	 *   PCIe 3
+	 *   PCIe 4
+	 */
+};
+
+
+&i2c5 {
+	status = "disabled";
+
+	/* CPU0 PRM 0.7V */
+	/* CPU0 PRM 1.2V CH03 */
+	/* CPU0 PRM 0.8V */
+	/* CPU0 PRM 1.2V CH47 */
+};
+
+&i2c6 {
+	status = "disabled";
+
+	/* CPU1 PRM 0.7V */
+	/* CPU1 PRM 1.2V CH03 */
+	/* CPU1 PRM 0.8V */
+	/* CPU1 PRM 1.2V CH47 */
+};
+
+&i2c7 {
+	status = "okay";
+
+	pca9541a@70 {
+		compatible = "nxp,pca9541";
+		reg = <0x70>;
+
+		i2c-arb {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			hotswap@54 {
+				compatible = "ti,lm5066i";
+				reg = <0x54>;
+			};
+		};
+	};
+
+	/* Master selector PCA9541A @70h (other master: CPU0)
+	 *   LM5066I PMBUS @10h
+	 */
+
+	/* 12V Quarter Brick DC/DC Converter Q54SJ12050 @61h */
+	power-brick@61 {
+		compatible = "delta,dps800";
+		reg = <0x61>;
+	};
+
+	/* CPU0 VR ISL68137 0.7V, 0.96V PMBUS @64h */
+	/* CPU0 VR ISL68137 1.2V CH03 PMBUS @40h */
+	/* CPU0 VR ISL68137 0.8V PMBUS @60h */
+	/* CPU0 VR 1.0V IR38064 I2C @11h, PMBUS @41h */
+	/* CPU0 VR ISL68137 1.2V CH47 PMBUS @41h */
+};
+
+&i2c8 {
+	status = "okay";
+
+	/* CPU1 VR ISL68137 0.7V, 0.96V PMBUS @65h */
+	/* CPU1 VR ISL68137 1.2V CH03 PMBUS @44h */
+	/* CPU1 VR ISL68137 0.8V PMBUS @61h */
+	/* CPU1 VR 1.0V IR38064 I2C @12h, PMBUS @42h */
+	/* CPU0 VR ISL68137 1.2V CH47 PMBUS @45h */
+};
+
+
+&i2c9 {
+	status = "disabled";
+
+	/* Fan board */
+};
+
+&i2c10 {
+	status = "disabled";
+};
+
+&i2c11 {
+	status = "disabled";
+
+	/* GPU sideband */
+};
+
+&i2c12 {
+	status = "disabled";
+};
+
+&i2c13 {
+	status = "disabled";
+
+	/* MUX PI3USB102
+	 *   CPU0 debug
+	 *   CPU1 debug
+	 */
+};
+
+&pinctrl {
+	aspeed,external-nodes = <&gfx &lhc>;
+
+	pinctrl_gpioh_unbiased: gpioi_unbiased {
+		pins = "A8", "C7", "B7", "A7", "D7", "B6", "A6", "E7";
+		bias-disable;
+	};
+};
+
+&gpio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpioh_unbiased>;
+
+	line_iso_u146_en {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(O, 4) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "iso_u164_en";
+	};
+
+	ncsi_mux_en_n {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(P, 0) GPIO_ACTIVE_HIGH>;
+		output-low;
+		line-name = "ncsi_mux_en_n";
+	};
+
+	line_bmc_i2c2_sw_rst_n {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(P, 1) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "bmc_i2c2_sw_rst_n";
+	};
+
+	line_bmc_i2c5_sw_rst_n {
+		gpio-hog;
+		gpios = <ASPEED_GPIO(P, 3) GPIO_ACTIVE_HIGH>;
+		output-high;
+		line-name = "bmc_i2c5_sw_rst_n";
+	};
+};
+
+&vuart {
+	status = "okay";
+};
+
+&gfx {
+	status = "okay";
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default &pinctrl_pwm1_default
+		&pinctrl_pwm2_default &pinctrl_pwm3_default>;
+
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x00>;
+	};
+
+	fan@1 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x01>;
+	};
+
+	fan@2 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x02>;
+	};
+
+	fan@3 {
+		reg = <0x03>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x03>;
+	};
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-quanta-q71l.dts b/arch/arm/boot/dts/aspeed-bmc-quanta-q71l.dts
new file mode 100644
index 000000000000..76aa6ea1f988
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-quanta-q71l.dts
@@ -0,0 +1,458 @@
+// SPDX-License-Identifier: GPL-2.0
+/dts-v1/;
+#include "aspeed-g4.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+
+/ {
+	model = "Quanta Q71L BMC";
+	compatible = "quanta,q71l-bmc", "aspeed,ast2400";
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=ttyS4,115200 earlyprintk";
+	};
+
+	memory@40000000 {
+		reg = <0x40000000 0x8000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		vga_memory: framebuffer@47800000 {
+			no-map;
+			reg = <0x47800000 0x00800000>; /* 8MB */
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		heartbeat {
+			gpios = <&gpio ASPEED_GPIO(B, 0) GPIO_ACTIVE_LOW>;
+		};
+
+		power {
+			gpios = <&gpio ASPEED_GPIO(B, 2) GPIO_ACTIVE_LOW>;
+		};
+
+		identify {
+			gpios = <&gpio ASPEED_GPIO(B, 3) GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>,
+			<&adc 4>, <&adc 5>, <&adc 6>, <&adc 7>,
+			<&adc 8>, <&adc 9>, <&adc 10>;
+	};
+
+	iio-hwmon-battery {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 11>;
+	};
+
+	i2c1mux: i2cmux {
+		compatible = "i2c-mux-gpio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* mux-gpios = <&sgpio 10 GPIO_ACTIVE_HIGH> */
+		i2c-parent = <&i2c1>;
+	};
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		label = "bmc";
+		m25p,fast-read;
+#include "openbmc-flash-layout.dtsi"
+	};
+};
+
+&spi {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1_default>;
+
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "pnor";
+	};
+};
+
+&pinctrl {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_vgahs_default &pinctrl_vgavs_default
+			&pinctrl_ddcclk_default &pinctrl_ddcdat_default>;
+};
+
+&lpc_snoop {
+	status = "okay";
+	snoop-ports = <0x80>;
+};
+
+&mac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rmii1_default>;
+	use-ncsi;
+};
+
+&mac1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii2_default &pinctrl_mdio2_default>;
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+
+	/* temp2 inlet */
+	tmp75@4c {
+		compatible = "ti,tmp75";
+		reg = <0x4c>;
+	};
+
+	/* temp3 */
+	tmp75@4e {
+		compatible = "ti,tmp75";
+		reg = <0x4e>;
+	};
+
+	/* temp1 */
+	tmp75@4f {
+		compatible = "ti,tmp75";
+		reg = <0x4f>;
+	};
+
+	/* Baseboard FRU */
+	eeprom@54 {
+		compatible = "atmel,24c64";
+		reg = <0x54>;
+	};
+
+	/* FP FRU */
+	eeprom@57 {
+		compatible = "atmel,24c64";
+		reg = <0x57>;
+	};
+};
+
+&i2c2 {
+	status = "okay";
+
+	/* 0: PCIe Slot 2,
+	 *    Slot 3,
+	 *    Slot 6,
+	 *    Slot 7
+	 */
+	i2c-switch@74 {
+		compatible = "nxp,pca9546";
+		reg = <0x74>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		i2c-mux-idle-disconnect;  /* may use mux@77 next. */
+
+		i2c_pcie2: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+		};
+
+		i2c_pcie3: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+		};
+
+		i2c_pcie6: i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+		};
+
+		i2c_pcie7: i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+		};
+	};
+
+	/* 0: PCIe Slot 1,
+	 *    Slot 4,
+	 *    Slot 5,
+	 *    Slot 8,
+	 *    Slot 9,
+	 *    Slot 10,
+	 *    SSD 1,
+	 *    SSD 2
+	 */
+	i2c-switch@77 {
+		compatible = "nxp,pca9548";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x77>;
+		i2c-mux-idle-disconnect;  /* may use mux@74 next. */
+
+		i2c_pcie1: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+		};
+
+		i2c_pcie4: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+		};
+
+		i2c_pcie5: i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+		};
+
+		i2c_pcie8: i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+		};
+
+		i2c_pcie9: i2c@4 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <4>;
+		};
+
+		i2c_pcie10: i2c@5 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <5>;
+		};
+
+		i2c_ssd1: i2c@6 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <6>;
+		};
+
+		i2c_ssd2: i2c@7 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <7>;
+		};
+	};
+};
+
+&i2c3 {
+	status = "okay";
+
+	/* BIOS FRU */
+	eeprom@56 {
+		compatible = "atmel,24c64";
+		reg = <0x56>;
+	};
+};
+
+&i2c4 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&i2c6 {
+	status = "okay";
+};
+
+&i2c7 {
+	status = "okay";
+
+	/* 0: PSU4
+	 *    PSU1
+	 *    PSU3
+	 *    PSU2
+	 */
+	i2c-switch@70 {
+		compatible = "nxp,pca9546";
+		reg = <0x70>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c_psu4: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+		};
+
+		i2c_psu1: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+		};
+
+		i2c_psu3: i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+		};
+
+		i2c_psu2: i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+		};
+	};
+
+	/* PDB FRU */
+	eeprom@52 {
+		compatible = "atmel,24c64";
+		reg = <0x52>;
+	};
+};
+
+&i2c8 {
+	status = "okay";
+
+	/* BMC FRU */
+	eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+	};
+};
+
+&vuart {
+	status = "okay";
+};
+
+&wdt2 {
+	status = "okay";
+};
+
+&pwm_tacho {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default
+		&pinctrl_pwm1_default
+		&pinctrl_pwm2_default
+		&pinctrl_pwm3_default>;
+
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x00>;
+	};
+
+	fan@1 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x01>;
+	};
+
+	fan@2 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x02>;
+	};
+
+	fan@3 {
+		reg = <0x03>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x03>;
+	};
+
+	fan@4 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x04>;
+	};
+
+	fan@5 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x05>;
+	};
+
+	fan@6 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x06>;
+	};
+
+	fan@7 {
+		reg = <0x03>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x07>;
+	};
+};
+
+&i2c1mux {
+	i2c@0 {
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Memory Riser 1 FRU */
+		eeprom@50 {
+			compatible = "atmel,24c02";
+			reg = <0x50>;
+		};
+
+		/* Memory Riser 2 FRU */
+		eeprom@51 {
+			compatible = "atmel,24c02";
+			reg = <0x51>;
+		};
+
+		/* Memory Riser 3 FRU */
+		eeprom@52 {
+			compatible = "atmel,24c02";
+			reg = <0x52>;
+		};
+
+		/* Memory Riser 4 FRU */
+		eeprom@53 {
+			compatible = "atmel,24c02";
+			reg = <0x53>;
+		};
+	};
+
+	i2c@1 {
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Memory Riser 5 FRU */
+		eeprom@50 {
+			compatible = "atmel,24c02";
+			reg = <0x50>;
+		};
+
+		/* Memory Riser 6 FRU */
+		eeprom@51 {
+			compatible = "atmel,24c02";
+			reg = <0x51>;
+		};
+
+		/* Memory Riser 7 FRU */
+		eeprom@52 {
+			compatible = "atmel,24c02";
+			reg = <0x52>;
+		};
+
+		/* Memory Riser 8 FRU */
+		eeprom@53 {
+			compatible = "atmel,24c02";
+			reg = <0x53>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/aspeed-g4.dtsi b/arch/arm/boot/dts/aspeed-g4.dtsi
index de08d9045cb8..ae2b8c952e80 100644
--- a/arch/arm/boot/dts/aspeed-g4.dtsi
+++ b/arch/arm/boot/dts/aspeed-g4.dtsi
@@ -1,5 +1,5 @@
-// SPDX-License-Identifier: GPL-2.0
-#include "skeleton.dtsi"
+// SPDX-License-Identifier: GPL-2.0+
+#include <dt-bindings/clock/aspeed-clock.h>
 
 / {
 	model = "Aspeed BMC";
@@ -42,6 +42,11 @@
 		};
 	};
 
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x40000000 0>;
+	};
+
 	ahb {
 		compatible = "simple-bus";
 		#address-cells = <1>;
@@ -54,6 +59,7 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "aspeed,ast2400-fmc";
+			clocks = <&syscon ASPEED_CLK_AHB>;
 			status = "disabled";
 			interrupts = <19>;
 			flash@0 {
@@ -69,6 +75,7 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "aspeed,ast2400-spi";
+			clocks = <&syscon ASPEED_CLK_AHB>;
 			status = "disabled";
 			flash@0 {
 				reg = < 0 >;
@@ -89,6 +96,7 @@
 			compatible = "aspeed,ast2400-mac", "faraday,ftgmac100";
 			reg = <0x1e660000 0x180>;
 			interrupts = <2>;
+			clocks = <&syscon ASPEED_CLK_GATE_MAC1CLK>;
 			status = "disabled";
 		};
 
@@ -96,6 +104,7 @@
 			compatible = "aspeed,ast2400-mac", "faraday,ftgmac100";
 			reg = <0x1e680000 0x180>;
 			interrupts = <3>;
+			clocks = <&syscon ASPEED_CLK_GATE_MAC2CLK>;
 			status = "disabled";
 		};
 
@@ -106,47 +115,12 @@
 			ranges;
 
 			syscon: syscon@1e6e2000 {
-				compatible = "aspeed,g4-scu", "syscon", "simple-mfd";
+				compatible = "aspeed,ast2400-scu", "syscon", "simple-mfd";
 				reg = <0x1e6e2000 0x1a8>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-
-                                clk_clkin: clk_clkin {
-                                        #clock-cells = <0>;
-                                        compatible = "fixed-clock";
-                                        clock-frequency = <48000000>;
-                                };
-
-                                clk_hpll: clk_hpll@70 {
-                                        #clock-cells = <0>;
-                                        compatible = "aspeed,g4-hpll-clock", "fixed-clock";
-                                        reg = <0x70>;
-                                        clocks = <&clk_clkin>;
-                                        clock-frequency = <384000000>;
-                                };
-
-                                clk_ahb: clk_ahb@70 {
-                                        #clock-cells = <0>;
-                                        compatible = "aspeed,g4-ahb-clock", "fixed-clock";
-                                        reg = <0x70>;
-                                        clocks = <&clk_hpll>;
-                                        clock-frequency = <192000000>;
-                                };
-
-                                clk_apb: clk_apb@8 {
-                                        #clock-cells = <0>;
-                                        compatible = "aspeed,g4-apb-clock", "fixed-clock";
-                                        reg = <0x08>;
-                                        clocks = <&clk_hpll>;
-                                        clock-frequency = <48000000>;
-                                };
-
-                                clk_uart: clk_uart@2c{
-                                        #clock-cells = <0>;
-                                        compatible = "aspeed,g4-uart-clock", "fixed-clock";
-                                        reg = <0x2c>;
-                                        clock-frequency = <24000000>;
-                                };
+				#clock-cells = <1>;
+				#reset-cells = <1>;
 
 				pinctrl: pinctrl {
 					compatible = "aspeed,g4-pinctrl";
@@ -156,7 +130,8 @@
 			adc: adc@1e6e9000 {
 				compatible = "aspeed,ast2400-adc";
 				reg = <0x1e6e9000 0xb0>;
-				clocks = <&clk_apb>;
+				clocks = <&syscon ASPEED_CLK_APB>;
+				resets = <&syscon ASPEED_RESET_ADC>;
 				#io-channel-cells = <1>;
 				status = "disabled";
 			};
@@ -173,6 +148,7 @@
 				reg = <0x1e780000 0x1000>;
 				interrupts = <20>;
 				gpio-ranges = <&pinctrl 0 0 220>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 				interrupt-controller;
 			};
 
@@ -181,7 +157,7 @@
 				compatible = "aspeed,ast2400-timer";
 				reg = <0x1e782000 0x90>;
 				interrupts = <16 17 18 35 36 37 38 39>;
-				clocks = <&clk_apb>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 				clock-names = "PCLK";
 			};
 
@@ -190,7 +166,7 @@
 				reg = <0x1e783000 0x20>;
 				reg-shift = <2>;
 				interrupts = <9>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART1CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -200,7 +176,7 @@
 				reg = <0x1e784000 0x20>;
 				reg-shift = <2>;
 				interrupts = <10>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART5CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -208,11 +184,23 @@
 			wdt1: watchdog@1e785000 {
 				compatible = "aspeed,ast2400-wdt";
 				reg = <0x1e785000 0x1c>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 			};
 
 			wdt2: watchdog@1e785020 {
 				compatible = "aspeed,ast2400-wdt";
 				reg = <0x1e785020 0x1c>;
+				clocks = <&syscon ASPEED_CLK_APB>;
+			};
+
+			pwm_tacho: pwm-tacho-controller@1e786000 {
+				compatible = "aspeed,ast2400-pwm-tacho";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x1e786000 0x1000>;
+				clocks = <&syscon ASPEED_CLK_APB>;
+				resets = <&syscon ASPEED_RESET_PWM>;
+				status = "disabled";
 			};
 
 			vuart: serial@1e787000 {
@@ -220,17 +208,59 @@
 				reg = <0x1e787000 0x40>;
 				reg-shift = <2>;
 				interrupts = <8>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 				no-loopback-test;
 				status = "disabled";
 			};
 
+			lpc: lpc@1e789000 {
+				compatible = "aspeed,ast2400-lpc", "simple-mfd";
+				reg = <0x1e789000 0x1000>;
+
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0x0 0x1e789000 0x1000>;
+
+				lpc_bmc: lpc-bmc@0 {
+					compatible = "aspeed,ast2400-lpc-bmc";
+					reg = <0x0 0x80>;
+				};
+
+				lpc_host: lpc-host@80 {
+					compatible = "aspeed,ast2400-lpc-host", "simple-mfd", "syscon";
+					reg = <0x80 0x1e0>;
+					reg-io-width = <4>;
+
+					#address-cells = <1>;
+					#size-cells = <1>;
+					ranges = <0x0 0x80 0x1e0>;
+
+					lpc_ctrl: lpc-ctrl@0 {
+						compatible = "aspeed,ast2400-lpc-ctrl";
+						reg = <0x0 0x80>;
+						status = "disabled";
+					};
+
+					lpc_snoop: lpc-snoop@0 {
+						compatible = "aspeed,ast2400-lpc-snoop";
+						reg = <0x0 0x80>;
+						interrupts = <8>;
+						status = "disabled";
+					};
+
+					lhc: lhc@20 {
+						compatible = "aspeed,ast2400-lhc";
+						reg = <0x20 0x24 0x48 0x8>;
+					};
+				};
+			};
+
 			uart2: serial@1e78d000 {
 				compatible = "ns16550a";
 				reg = <0x1e78d000 0x20>;
 				reg-shift = <2>;
 				interrupts = <32>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART2CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -240,7 +270,7 @@
 				reg = <0x1e78e000 0x20>;
 				reg-shift = <2>;
 				interrupts = <33>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART3CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -250,7 +280,7 @@
 				reg = <0x1e78f000 0x20>;
 				reg-shift = <2>;
 				interrupts = <34>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART4CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -281,7 +311,8 @@
 
 		reg = <0x40 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <0>;
 		interrupt-parent = <&i2c_ic>;
@@ -296,7 +327,8 @@
 
 		reg = <0x80 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <1>;
 		interrupt-parent = <&i2c_ic>;
@@ -311,7 +343,8 @@
 
 		reg = <0xc0 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <2>;
 		interrupt-parent = <&i2c_ic>;
@@ -327,7 +360,8 @@
 
 		reg = <0x100 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <3>;
 		interrupt-parent = <&i2c_ic>;
@@ -343,7 +377,8 @@
 
 		reg = <0x140 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <4>;
 		interrupt-parent = <&i2c_ic>;
@@ -359,7 +394,8 @@
 
 		reg = <0x180 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <5>;
 		interrupt-parent = <&i2c_ic>;
@@ -375,7 +411,8 @@
 
 		reg = <0x1c0 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <6>;
 		interrupt-parent = <&i2c_ic>;
@@ -391,7 +428,8 @@
 
 		reg = <0x300 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <7>;
 		interrupt-parent = <&i2c_ic>;
@@ -407,7 +445,8 @@
 
 		reg = <0x340 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <8>;
 		interrupt-parent = <&i2c_ic>;
@@ -423,7 +462,8 @@
 
 		reg = <0x380 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <9>;
 		interrupt-parent = <&i2c_ic>;
@@ -439,7 +479,8 @@
 
 		reg = <0x3c0 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <10>;
 		interrupt-parent = <&i2c_ic>;
@@ -455,7 +496,8 @@
 
 		reg = <0x400 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <11>;
 		interrupt-parent = <&i2c_ic>;
@@ -471,7 +513,8 @@
 
 		reg = <0x440 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <12>;
 		interrupt-parent = <&i2c_ic>;
@@ -487,7 +530,8 @@
 
 		reg = <0x480 0x40>;
 		compatible = "aspeed,ast2400-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <13>;
 		interrupt-parent = <&i2c_ic>;
diff --git a/arch/arm/boot/dts/aspeed-g5.dtsi b/arch/arm/boot/dts/aspeed-g5.dtsi
index 5c4ecdba3a6b..2477ebc11d9d 100644
--- a/arch/arm/boot/dts/aspeed-g5.dtsi
+++ b/arch/arm/boot/dts/aspeed-g5.dtsi
@@ -1,5 +1,5 @@
-// SPDX-License-Identifier: GPL-2.0
-#include "skeleton.dtsi"
+// SPDX-License-Identifier: GPL-2.0+
+#include <dt-bindings/clock/aspeed-clock.h>
 
 / {
 	model = "Aspeed BMC";
@@ -42,6 +42,11 @@
 		};
 	};
 
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0>;
+	};
+
 	ahb {
 		compatible = "simple-bus";
 		#address-cells = <1>;
@@ -54,6 +59,7 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "aspeed,ast2500-fmc";
+			clocks = <&syscon ASPEED_CLK_AHB>;
 			status = "disabled";
 			interrupts = <19>;
 			flash@0 {
@@ -79,6 +85,7 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "aspeed,ast2500-spi";
+			clocks = <&syscon ASPEED_CLK_AHB>;
 			status = "disabled";
 			flash@0 {
 				reg = < 0 >;
@@ -98,6 +105,7 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "aspeed,ast2500-spi";
+			clocks = <&syscon ASPEED_CLK_AHB>;
 			status = "disabled";
 			flash@0 {
 				reg = < 0 >;
@@ -123,6 +131,7 @@
 			compatible = "aspeed,ast2500-mac", "faraday,ftgmac100";
 			reg = <0x1e660000 0x180>;
 			interrupts = <2>;
+			clocks = <&syscon ASPEED_CLK_GATE_MAC1CLK>;
 			status = "disabled";
 		};
 
@@ -130,6 +139,7 @@
 			compatible = "aspeed,ast2500-mac", "faraday,ftgmac100";
 			reg = <0x1e680000 0x180>;
 			interrupts = <3>;
+			clocks = <&syscon ASPEED_CLK_GATE_MAC2CLK>;
 			status = "disabled";
 		};
 
@@ -140,55 +150,18 @@
 			ranges;
 
 			syscon: syscon@1e6e2000 {
-				compatible = "aspeed,g5-scu", "syscon", "simple-mfd";
+				compatible = "aspeed,ast2500-scu", "syscon", "simple-mfd";
 				reg = <0x1e6e2000 0x1a8>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-
-				clk_clkin: clk_clkin@70 {
-					#clock-cells = <0>;
-					compatible = "aspeed,g5-clkin-clock", "fixed-clock";
-					reg = <0x70>;
-					clock-frequency = <24000000>;
-				};
-
-				clk_hpll: clk_hpll@24 {
-					#clock-cells = <0>;
-					compatible = "aspeed,g5-hpll-clock", "fixed-clock";
-					reg = <0x24>;
-					clocks = <&clk_clkin>;
-					clock-frequency = <792000000>;
-				};
-
-				clk_ahb: clk_ahb@70 {
-					#clock-cells = <0>;
-					compatible = "aspeed,g5-ahb-clock", "fixed-clock";
-					reg = <0x70>;
-					clocks = <&clk_hpll>;
-					clock-frequency = <198000000>;
-				};
-
-				clk_apb: clk_apb@8 {
-					#clock-cells = <0>;
-					compatible = "aspeed,g5-apb-clock", "fixed-clock";
-					reg = <0x08>;
-					clocks = <&clk_hpll>;
-					clock-frequency = <24750000>;
-				};
-
-				clk_uart: clk_uart@2c {
-					#clock-cells = <0>;
-					compatible = "aspeed,uart-clock", "fixed-clock";
-					reg = <0x2c>;
-					clock-frequency = <24000000>;
-				};
+				#clock-cells = <1>;
+				#reset-cells = <1>;
 
 				pinctrl: pinctrl {
 					compatible = "aspeed,g5-pinctrl";
 					aspeed,external-nodes = <&gfx &lhc>;
 
 				};
-
 			};
 
 			gfx: display@1e6e6000 {
@@ -200,7 +173,8 @@
 			adc: adc@1e6e9000 {
 				compatible = "aspeed,ast2500-adc";
 				reg = <0x1e6e9000 0xb0>;
-				clocks = <&clk_apb>;
+				clocks = <&syscon ASPEED_CLK_APB>;
+				resets = <&syscon ASPEED_RESET_ADC>;
 				#io-channel-cells = <1>;
 				status = "disabled";
 			};
@@ -217,6 +191,7 @@
 				reg = <0x1e780000 0x1000>;
 				interrupts = <20>;
 				gpio-ranges = <&pinctrl 0 0 220>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 				interrupt-controller;
 			};
 
@@ -225,7 +200,7 @@
 				compatible = "aspeed,ast2400-timer";
 				reg = <0x1e782000 0x90>;
 				interrupts = <16 17 18 35 36 37 38 39>;
-				clocks = <&clk_apb>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 				clock-names = "PCLK";
 			};
 
@@ -234,7 +209,7 @@
 				reg = <0x1e783000 0x20>;
 				reg-shift = <2>;
 				interrupts = <9>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART1CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -244,7 +219,7 @@
 				reg = <0x1e784000 0x20>;
 				reg-shift = <2>;
 				interrupts = <10>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART5CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -252,16 +227,39 @@
 			wdt1: watchdog@1e785000 {
 				compatible = "aspeed,ast2500-wdt";
 				reg = <0x1e785000 0x20>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 			};
 
 			wdt2: watchdog@1e785020 {
 				compatible = "aspeed,ast2500-wdt";
 				reg = <0x1e785020 0x20>;
+				clocks = <&syscon ASPEED_CLK_APB>;
 			};
 
 			wdt3: watchdog@1e785040 {
 				compatible = "aspeed,ast2500-wdt";
 				reg = <0x1e785040 0x20>;
+				clocks = <&syscon ASPEED_CLK_APB>;
+				status = "disabled";
+			};
+
+			pwm_tacho: pwm-tacho-controller@1e786000 {
+				compatible = "aspeed,ast2500-pwm-tacho";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0x1e786000 0x1000>;
+				clocks = <&syscon ASPEED_CLK_APB>;
+				resets = <&syscon ASPEED_RESET_PWM>;
+				status = "disabled";
+			};
+
+			vuart: serial@1e787000 {
+				compatible = "aspeed,ast2500-vuart";
+				reg = <0x1e787000 0x40>;
+				reg-shift = <2>;
+				interrupts = <8>;
+				clocks = <&syscon ASPEED_CLK_APB>;
+				no-loopback-test;
 				status = "disabled";
 			};
 
@@ -288,6 +286,19 @@
 
 					reg-io-width = <4>;
 
+					lpc_ctrl: lpc-ctrl@0 {
+						compatible = "aspeed,ast2500-lpc-ctrl";
+						reg = <0x0 0x80>;
+						status = "disabled";
+					};
+
+					lpc_snoop: lpc-snoop@0 {
+						compatible = "aspeed,ast2500-lpc-snoop";
+						reg = <0x0 0x80>;
+						interrupts = <8>;
+						status = "disabled";
+					};
+
 					lhc: lhc@20 {
 						compatible = "aspeed,ast2500-lhc";
 						reg = <0x20 0x24 0x48 0x8>;
@@ -295,22 +306,12 @@
 				};
 			};
 
-			vuart: serial@1e787000 {
-				compatible = "aspeed,ast2500-vuart";
-				reg = <0x1e787000 0x40>;
-				reg-shift = <2>;
-				interrupts = <10>;
-				clocks = <&clk_uart>;
-				no-loopback-test;
-				status = "disabled";
-			};
-
 			uart2: serial@1e78d000 {
 				compatible = "ns16550a";
 				reg = <0x1e78d000 0x20>;
 				reg-shift = <2>;
 				interrupts = <32>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART2CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -320,7 +321,7 @@
 				reg = <0x1e78e000 0x20>;
 				reg-shift = <2>;
 				interrupts = <33>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART3CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -330,7 +331,7 @@
 				reg = <0x1e78f000 0x20>;
 				reg-shift = <2>;
 				interrupts = <34>;
-				clocks = <&clk_uart>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART4CLK>;
 				no-loopback-test;
 				status = "disabled";
 			};
@@ -361,7 +362,8 @@
 
 		reg = <0x40 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <0>;
 		interrupt-parent = <&i2c_ic>;
@@ -376,7 +378,8 @@
 
 		reg = <0x80 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <1>;
 		interrupt-parent = <&i2c_ic>;
@@ -391,7 +394,8 @@
 
 		reg = <0xc0 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <2>;
 		interrupt-parent = <&i2c_ic>;
@@ -407,7 +411,8 @@
 
 		reg = <0x100 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <3>;
 		interrupt-parent = <&i2c_ic>;
@@ -423,7 +428,8 @@
 
 		reg = <0x140 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <4>;
 		interrupt-parent = <&i2c_ic>;
@@ -439,7 +445,8 @@
 
 		reg = <0x180 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <5>;
 		interrupt-parent = <&i2c_ic>;
@@ -455,7 +462,8 @@
 
 		reg = <0x1c0 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <6>;
 		interrupt-parent = <&i2c_ic>;
@@ -471,7 +479,8 @@
 
 		reg = <0x300 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <7>;
 		interrupt-parent = <&i2c_ic>;
@@ -487,7 +496,8 @@
 
 		reg = <0x340 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <8>;
 		interrupt-parent = <&i2c_ic>;
@@ -503,7 +513,8 @@
 
 		reg = <0x380 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <9>;
 		interrupt-parent = <&i2c_ic>;
@@ -519,7 +530,8 @@
 
 		reg = <0x3c0 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <10>;
 		interrupt-parent = <&i2c_ic>;
@@ -535,7 +547,8 @@
 
 		reg = <0x400 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <11>;
 		interrupt-parent = <&i2c_ic>;
@@ -551,7 +564,8 @@
 
 		reg = <0x440 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <12>;
 		interrupt-parent = <&i2c_ic>;
@@ -567,7 +581,8 @@
 
 		reg = <0x480 0x40>;
 		compatible = "aspeed,ast2500-i2c-bus";
-		clocks = <&clk_apb>;
+		clocks = <&syscon ASPEED_CLK_APB>;
+		resets = <&syscon ASPEED_RESET_I2C>;
 		bus-frequency = <100000>;
 		interrupts = <13>;
 		interrupt-parent = <&i2c_ic>;
diff --git a/arch/arm/boot/dts/at91-ariag25.dts b/arch/arm/boot/dts/at91-ariag25.dts
index 1c86537a42a0..e3e14b7e25aa 100644
--- a/arch/arm/boot/dts/at91-ariag25.dts
+++ b/arch/arm/boot/dts/at91-ariag25.dts
@@ -58,6 +58,18 @@
 				};
 			};
 
+			tcb0: timer@f8008000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			i2c0: i2c@f8010000 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/at91-ariettag25.dts b/arch/arm/boot/dts/at91-ariettag25.dts
index f877f3430bcc..7a34c4dc05d2 100644
--- a/arch/arm/boot/dts/at91-ariettag25.dts
+++ b/arch/arm/boot/dts/at91-ariettag25.dts
@@ -46,6 +46,18 @@
 				};
 			};
 
+			tcb0: timer@f8008000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			usb2: gadget@f803c000 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/at91-cosino.dtsi b/arch/arm/boot/dts/at91-cosino.dtsi
index 89cde175154e..295a5a43fe01 100644
--- a/arch/arm/boot/dts/at91-cosino.dtsi
+++ b/arch/arm/boot/dts/at91-cosino.dtsi
@@ -37,6 +37,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@f8008000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			mmc0: mmc@f0008000 {
 				pinctrl-0 = <
 					&pinctrl_board_mmc0
diff --git a/arch/arm/boot/dts/at91-foxg20.dts b/arch/arm/boot/dts/at91-foxg20.dts
index 50d5e719b451..d413e2d9abdf 100644
--- a/arch/arm/boot/dts/at91-foxg20.dts
+++ b/arch/arm/boot/dts/at91-foxg20.dts
@@ -34,6 +34,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usb1: gadget@fffa4000 {
 				atmel,vbus-gpio = <&pioC 6 GPIO_ACTIVE_HIGH>;
 				status = "okay";
diff --git a/arch/arm/boot/dts/at91-kizbox.dts b/arch/arm/boot/dts/at91-kizbox.dts
index 1f31df846d61..1003846679c1 100644
--- a/arch/arm/boot/dts/at91-kizbox.dts
+++ b/arch/arm/boot/dts/at91-kizbox.dts
@@ -31,6 +31,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			macb0: ethernet@fffc4000 {
 				phy-mode = "mii";
 				pinctrl-0 = <&pinctrl_macb_rmii
diff --git a/arch/arm/boot/dts/at91-kizbox2.dts b/arch/arm/boot/dts/at91-kizbox2.dts
index ec6c28c521a5..30041e8f12d5 100644
--- a/arch/arm/boot/dts/at91-kizbox2.dts
+++ b/arch/arm/boot/dts/at91-kizbox2.dts
@@ -94,6 +94,18 @@
 				};
 			};
 
+			tcb0: timer@f0010000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			usart0: serial@f001c000 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/at91-kizboxmini.dts b/arch/arm/boot/dts/at91-kizboxmini.dts
index fe1bc0a59a98..15b0746e44e2 100644
--- a/arch/arm/boot/dts/at91-kizboxmini.dts
+++ b/arch/arm/boot/dts/at91-kizboxmini.dts
@@ -34,6 +34,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@f8008000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			usart0: serial@f801c000 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/at91-linea.dtsi b/arch/arm/boot/dts/at91-linea.dtsi
index 87e5090fb4c4..c7b964e25321 100644
--- a/arch/arm/boot/dts/at91-linea.dtsi
+++ b/arch/arm/boot/dts/at91-linea.dtsi
@@ -27,6 +27,18 @@
 	clock-frequency = <12000000>;
 };
 
+&tcb0 {
+	timer@0 {
+		compatible = "atmel,tcb-timer";
+		reg = <0>;
+	};
+
+	timer@1 {
+		compatible = "atmel,tcb-timer";
+		reg = <1>;
+	};
+};
+
 &i2c0 {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/at91-natte.dtsi b/arch/arm/boot/dts/at91-natte.dtsi
new file mode 100644
index 000000000000..49f0a0c46cde
--- /dev/null
+++ b/arch/arm/boot/dts/at91-natte.dtsi
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * at91-natte.dts - Device Tree include file for the Natte board
+ *
+ * Copyright (C) 2017 Axentia Technologies AB
+ *
+ * Author: Peter Rosin <peda@axentia.se>
+ */
+
+/ {
+	mux: mux-controller {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+
+		mux-gpios = <&ioexp 0 GPIO_ACTIVE_HIGH>,
+			    <&ioexp 1 GPIO_ACTIVE_HIGH>,
+			    <&ioexp 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	batntc-mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc 5>;
+		io-channel-names = "parent";
+		mux-controls = <&mux>;
+
+		channels =
+			"batntc0", "batntc1", "batntc2", "batntc3",
+			"batntc4", "batntc5", "batntc6", "batntc7";
+	};
+
+	batv-mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc 6>;
+		io-channel-names = "parent";
+		mux-controls = <&mux>;
+
+		channels =
+			"batv0", "batv1", "batv2", "batv3",
+			"batv4", "batv5", "batv6", "batv7";
+	};
+
+	iout-mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc 7>;
+		io-channel-names = "parent";
+		mux-controls = <&mux>;
+
+		channels =
+			"iout0", "iout1", "iout2", "iout3",
+			"iout4", "iout5", "iout6", "iout7";
+	};
+
+	i2c-mux {
+		compatible = "i2c-mux";
+		mux-locked;
+		i2c-parent = <&i2c0>;
+		mux-controls = <&mux>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+
+		i2c@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+
+		i2c@2 {
+			reg = <2>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+
+		i2c@3 {
+			reg = <3>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+
+		i2c@4 {
+			reg = <4>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+
+		i2c@5 {
+			reg = <5>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+
+		i2c@6 {
+			reg = <6>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+
+		i2c@7 {
+			reg = <7>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			charger@9 {
+				compatible = "ti,bq24735";
+				reg = <0x9>;
+
+				ti,charge-current = <2000>;
+				ti,charge-voltage = <16800>;
+
+				poll-interval = <20000>;
+			};
+		};
+	};
+};
+
+&i2c0 {
+	status = "okay";
+
+	ioexp: ioexp@20 {
+		#gpio-cells = <2>;
+		compatible = "semtech,sx1502q";
+		reg = <0x20>;
+
+		gpio-controller;
+		ngpios = <8>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio3_cfg_pins>;
+
+		gpio3_cfg_pins: gpio3_cfg {
+			pins = "gpio3";
+			bias-pull-up;
+		};
+	};
+
+	adc: adc@48 {
+		compatible = "ti,ads1015";
+		reg = <0x48>;
+		#io-channel-cells = <1>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		channel@4 {
+			reg = <4>;
+			ti,gain = <2>;
+			ti,datarate = <4>;
+		};
+
+		channel@5 {
+			reg = <5>;
+			ti,gain = <2>;
+			ti,datarate = <4>;
+		};
+
+		channel@6 {
+			reg = <6>;
+			ti,gain = <1>;
+			ti,datarate = <4>;
+		};
+
+		channel@7 {
+			reg = <7>;
+			ti,gain = <3>;
+			ti,datarate = <4>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/at91-nattis-2-natte-2.dts b/arch/arm/boot/dts/at91-nattis-2-natte-2.dts
new file mode 100644
index 000000000000..3ea1d26e1c68
--- /dev/null
+++ b/arch/arm/boot/dts/at91-nattis-2-natte-2.dts
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * at91-nattis-2-natte-2.dts - Device Tree file for the Linea/Nattis board
+ *
+ * Copyright (C) 2017 Axentia Technologies AB
+ *
+ * Author: Peter Rosin <peda@axentia.se>
+ */
+/dts-v1/;
+#include "at91-linea.dtsi"
+#include "sama5d3_lcd.dtsi"
+#include "at91-natte.dtsi"
+
+/ {
+	model = "Axentia Linea-Nattis v2 Natte v2";
+	compatible = "axentia,nattis-2", "axentia,natte-2", "axentia,linea",
+		     "atmel,sama5d31", "atmel,sama5d3", "atmel,sama5";
+
+	ahb {
+		apb {
+			pinctrl@fffff200 {
+				nattis {
+					pinctrl_usba_vbus: usba_vbus {
+						atmel,pins =
+							<AT91_PIOD 28
+							 AT91_PERIPH_GPIO
+							 AT91_PINCTRL_DEGLITCH>;
+					};
+
+					pinctrl_mmc0_cd: mmc0_cd {
+						atmel,pins =
+							<AT91_PIOD 5
+							 AT91_PERIPH_GPIO
+							 AT91_PINCTRL_PULL_UP_DEGLITCH>;
+					};
+
+					pinctrl_lcd_prlud0: lcd_prlud0 {
+						atmel,pins =
+							<AT91_PIOA 21
+							 AT91_PERIPH_GPIO
+							 AT91_PINCTRL_OUTPUT_VAL(0)>;
+					};
+
+					pinctrl_lcd_hipow0: lcd_hipow0 {
+						atmel,pins =
+							<AT91_PIOA 23
+							 AT91_PERIPH_GPIO
+							 AT91_PINCTRL_OUTPUT_VAL(0)>;
+					};
+				};
+			};
+
+			watchdog@fffffe40 {
+				status = "okay";
+			};
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		wakeup {
+			label = "Wakeup";
+			linux,code = <10>;
+			gpio-key,wakeup;
+			gpios = <&pioB 27 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	panel_reg: panel-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "panel-VCC";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	bl_reg: backlight-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "panel-VDD";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+
+	panel_bl: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&hlcdc_pwm 0 100000 0>;
+
+		brightness-levels = < 0  1  2  3  4  5  6  7  8  9
+				     10 11 12 13 14 15 16 17 18 19
+				     20 21 22 23 24 25 26 27 28 29
+				     30 31 32 33 34 35 36 37 38 39
+				     40 41 42 43 44 45 46 47 48 49
+				     50 51 52 53 54 55 56 57 58 59
+				     60 61 62 63 64 65 66 67 68 69
+				     70 71 72 73 74 75 76 77 78 79
+				     80 81 82 83 84 85 86 87 88 89
+				     90 91 92 93 94 95 96 97 98 99
+				    100>;
+		default-brightness-level = <40>;
+
+		power-supply = <&bl_reg>;
+		enable-gpios = <&pioA 20 GPIO_ACTIVE_HIGH>;
+	};
+
+	panel: panel {
+		compatible = "sharp,lq150x1lg11";
+		backlight = <&panel_bl>;
+		power-supply = <&panel_reg>;
+
+		port {
+			panel_input: endpoint {
+				remote-endpoint = <&hlcdc_panel_output>;
+			};
+		};
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+
+		simple-audio-card,name = "nattis-tfa9879";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&cpu_dai>;
+		simple-audio-card,frame-master = <&cpu_dai>;
+		simple-audio-card,widgets = "Line", "Line Out Jack";
+		simple-audio-card,routing = "Line Out Jack", "LINEOUT";
+
+		cpu_dai: simple-audio-card,cpu {
+			sound-dai = <&ssc0>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&amp>;
+		};
+	};
+};
+
+&i2c0 {
+	status = "okay";
+
+	clock-frequency = <100000>;
+
+	temp@18 {
+		compatible = "nxp,se97b", "jedec,jc-42.4-temp";
+		reg = <0x18>;
+		smbus-timeout-disable;
+	};
+
+	eeprom@50 {
+		compatible = "nxp,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+
+	amp: amplifier@6c {
+		 compatible = "nxp,tfa9879";
+		 reg = <0x6c>;
+		 #sound-dai-cells = <0>;
+	};
+};
+
+&ssc0 {
+	status = "okay";
+
+	atmel,clk-from-rk-pin;
+	#sound-dai-cells = <0>;
+};
+
+&hlcdc {
+	status = "okay";
+
+	hlcdc-display-controller {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_lcd_base
+			     &pinctrl_lcd_rgb565
+			     &pinctrl_lcd_prlud0
+			     &pinctrl_lcd_hipow0>;
+
+		port@0 {
+			hlcdc_panel_output: endpoint {
+				remote-endpoint = <&panel_input>;
+			};
+		};
+	};
+};
+
+&mmc0 {
+	status = "okay";
+
+	pinctrl-0 = <&pinctrl_mmc0_clk_cmd_dat0
+		     &pinctrl_mmc0_dat1_3
+		     &pinctrl_mmc0_cd>;
+
+	slot@0 {
+		reg = <0>;
+		bus-width = <4>;
+		cd-gpios = <&pioD 5 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&usart0 {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usart0>;
+	atmel,use-dma-rx;
+};
+
+&nand {
+	partitions {
+		compatible = "fixed-partitions";
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		at91bootstrap@0 {
+			label = "at91bootstrap";
+			reg = <0x0 0x40000>;
+		};
+
+		bootloader@40000 {
+			label = "bootloader";
+			reg = <0x40000 0x80000>;
+		};
+
+		bootloaderenv@c0000 {
+			label = "bootloader env";
+			reg = <0xc0000 0xc0000>;
+		};
+
+		dtb@180000 {
+			label = "device tree";
+			reg = <0x180000 0x80000>;
+		};
+
+		kernel@200000 {
+			label = "kernel";
+			reg = <0x200000 0x600000>;
+		};
+
+		rootfs@800000 {
+			label = "rootfs";
+			reg = <0x800000 0x0f800000>;
+		};
+	};
+};
+
+&dbgu {
+	status = "okay";
+
+	atmel,use-dma-rx;
+};
+
+&usb0 {
+	status = "okay";
+
+	atmel,vbus-gpio = <&pioD 28 GPIO_ACTIVE_HIGH>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usba_vbus>;
+};
diff --git a/arch/arm/boot/dts/at91-qil_a9260.dts b/arch/arm/boot/dts/at91-qil_a9260.dts
index f46352790a41..72e9148ca096 100644
--- a/arch/arm/boot/dts/at91-qil_a9260.dts
+++ b/arch/arm/boot/dts/at91-qil_a9260.dts
@@ -31,6 +31,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usb1: gadget@fffa4000 {
 				atmel,vbus-gpio = <&pioC 5 GPIO_ACTIVE_HIGH>;
 				status = "okay";
diff --git a/arch/arm/boot/dts/at91-sam9_l9260.dts b/arch/arm/boot/dts/at91-sam9_l9260.dts
index 171243ca4f2f..70cb36f7a9d7 100644
--- a/arch/arm/boot/dts/at91-sam9_l9260.dts
+++ b/arch/arm/boot/dts/at91-sam9_l9260.dts
@@ -32,6 +32,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			mmc0: mmc@fffa8000 {
 				pinctrl-0 = <
 					&pinctrl_board_mmc0
diff --git a/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts b/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts
index 6d87b4eb6c41..e86e0c00eb6b 100644
--- a/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts
+++ b/arch/arm/boot/dts/at91-sama5d27_som1_ek.dts
@@ -119,6 +119,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@f800c000 {
+				timer0: timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer1: timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			uart1: serial@f8020000 {
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_uart1_default>;
diff --git a/arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts b/arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts
new file mode 100644
index 000000000000..e603a267bdf1
--- /dev/null
+++ b/arch/arm/boot/dts/at91-sama5d2_ptc_ek.dts
@@ -0,0 +1,413 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR X11)
+/*
+ * at91-sama5d2_ptc_ek.dts - Device Tree file for SAMA5D2 PTC EK board
+ *
+ *  Copyright (C) 2017 Microchip/Atmel,
+ *		  2017 Wenyou Yang <wenyou.yang@microchip.com>
+ *		  2017 Ludovic Desroches <ludovic.desroches@microchip.com>
+ */
+/dts-v1/;
+#include "sama5d2.dtsi"
+#include "sama5d2-pinfunc.h"
+#include <dt-bindings/mfd/atmel-flexcom.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Atmel SAMA5D2 PTC EK";
+	compatible = "atmel,sama5d2-ptc_ek", "atmel,sama5d2", "atmel,sama5";
+
+	aliases {
+		serial0 = &uart0;
+		i2c0	= &i2c0;
+		i2c1	= &i2c1;
+		i2c2	= &i2c2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	clocks {
+		slow_xtal {
+			clock-frequency = <32768>;
+		};
+
+		main_xtal {
+			clock-frequency = <24000000>;
+		};
+	};
+
+	ahb {
+		usb0: gadget@300000 {
+			atmel,vbus-gpio = <&pioA PIN_PA27 GPIO_ACTIVE_HIGH>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usba_vbus>;
+			status = "okay";
+		};
+
+		usb1: ohci@400000 {
+			num-ports = <3>;
+			atmel,vbus-gpio = <0
+					   &pioA PIN_PB12 GPIO_ACTIVE_HIGH
+					   0
+					  >;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_usb_default>;
+			status = "okay";
+		};
+
+		usb2: ehci@500000 {
+			status = "okay";
+		};
+
+		ebi: ebi@10000000 {
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_nand_default>;
+			status = "okay"; /* conflicts with sdmmc1 and qspi0 */
+
+			nand_controller: nand-controller {
+				status = "okay";
+
+				nand@3 {
+					reg = <0x3 0x0 0x2>;
+					atmel,rb = <0>;
+					nand-bus-width = <8>;
+					nand-ecc-mode = "hw";
+					nand-on-flash-bbt;
+					label = "atmel_nand";
+
+					partitions {
+						compatible = "fixed-partitions";
+						#address-cells = <1>;
+						#size-cells = <1>;
+
+						at91bootstrap@0 {
+							label = "bootstrap";
+							reg = <0x0 0x40000>;
+						};
+
+						bootloader@40000 {
+							label = "bootloader";
+							reg = <0x40000 0xc0000>;
+						};
+
+						bootloaderenv@0x100000 {
+							label = "bootloader env";
+							reg = <0x100000 0x40000>;
+						};
+
+						bootloaderenvred@0x140000 {
+							label = "bootloader env redundant";
+							reg = <0x140000 0x40000>;
+						};
+
+						dtb@180000 {
+							label = "device tree";
+							reg = <0x180000 0x80000>;
+						};
+
+						kernel@200000 {
+							label = "kernel";
+							reg = <0x200000 0x600000>;
+						};
+
+						rootfs@800000 {
+							label = "rootfs";
+							reg = <0x800000 0x1f800000>;
+						};
+					};
+				};
+			};
+		};
+
+		sdmmc0: sdio-host@a0000000 {
+			bus-width = <8>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_sdmmc0_default>;
+			non-removable;
+			mmc-ddr-1_8v;
+			status = "okay";
+		};
+
+		apb {
+			spi0: spi@f8000000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_spi0_default>;
+				status = "okay";
+			};
+
+			macb0: ethernet@f8008000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_macb0_default &pinctrl_macb0_phy_irq>;
+				phy-mode = "rmii";
+				status = "okay";
+
+				ethernet-phy@1 {
+					reg = <0x1>;
+					interrupt-parent = <&pioA>;
+					interrupts = <56 IRQ_TYPE_LEVEL_LOW>;
+				};
+			};
+
+			tcb0: timer@f800c000 {
+				timer0: timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer1: timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
+			uart0: serial@f801c000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_uart0_default>;
+				atmel,use-dma-rx;
+				atmel,use-dma-tx;
+				status = "okay";
+			};
+
+			uart2: serial@f8024000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_uart2_default>;
+				atmel,use-dma-rx;
+				atmel,use-dma-tx;
+				status = "okay";
+			};
+
+			i2c0: i2c@f8028000 {
+				dmas = <0>, <0>;
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_i2c0_default>;
+				status = "okay";
+			};
+
+			flx0: flexcom@f8034000 {
+				atmel,flexcom-mode = <ATMEL_FLEXCOM_MODE_TWI>;
+				status = "okay";
+
+				i2c2: i2c@600 {
+					compatible = "atmel,sama5d2-i2c";
+					reg = <0x600 0x200>;
+					interrupts = <19 IRQ_TYPE_LEVEL_HIGH 7>;
+					dmas = <0>, <0>;
+					dma-names = "tx", "rx";
+					#address-cells = <1>;
+					#size-cells = <0>;
+					clocks = <&flx0_clk>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_flx0_default>;
+					atmel,fifo-size = <16>;
+					status = "okay";
+				};
+			};
+
+			shdwc@f8048010 {
+				atmel,shdwc-debouncer = <976>;
+
+				input@0 {
+					reg = <0>;
+					atmel,wakeup-type = "low";
+				};
+			};
+
+			watchdog@f8048040 {
+				status = "okay";
+			};
+
+			spi1: spi@fc000000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_spi1_default>;
+				status = "okay";
+			};
+
+			i2c1: i2c@fc028000 {
+				dmas = <0>, <0>;
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_i2c1_default>;
+				status = "okay";
+
+				at24@50 {
+					compatible = "24c02";
+					reg = <0x50>;
+					pagesize = <8>;
+				};
+			};
+
+			pinctrl@fc038000 {
+				pinctrl_flx0_default: flx0_default {
+					pinmux = <PIN_PB28__FLEXCOM0_IO0>,
+						 <PIN_PB29__FLEXCOM0_IO1>;
+					bias-disable;
+				};
+
+				pinctrl_i2c0_default: i2c0_default {
+					pinmux = <PIN_PD21__TWD0>,
+						 <PIN_PD22__TWCK0>;
+					bias-disable;
+				};
+
+				pinctrl_i2c1_default: i2c1_default {
+					pinmux = <PIN_PC6__TWD1>,
+						 <PIN_PC7__TWCK1>;
+					bias-disable;
+				};
+
+				pinctrl_key_gpio_default: key_gpio_default {
+					pinmux = <PIN_PA10__GPIO>;
+					bias-pull-up;
+				};
+
+				pinctrl_led_gpio_default: led_gpio_default {
+					pinmux = <PIN_PB6__GPIO>,
+						 <PIN_PB8__GPIO>,
+						 <PIN_PB10__GPIO>;
+					bias-pull-up;
+				};
+
+				pinctrl_macb0_default: macb0_default {
+					pinmux = <PIN_PB14__GTXCK>,
+						 <PIN_PB15__GTXEN>,
+						 <PIN_PB16__GRXDV>,
+						 <PIN_PB17__GRXER>,
+						 <PIN_PB18__GRX0>,
+						 <PIN_PB19__GRX1>,
+						 <PIN_PB20__GTX0>,
+						 <PIN_PB21__GTX1>,
+						 <PIN_PB22__GMDC>,
+						 <PIN_PB23__GMDIO>;
+					bias-disable;
+				};
+
+				pinctrl_macb0_phy_irq: macb0_phy_irq {
+					pinmux = <PIN_PB24__GPIO>;
+					bias-disable;
+				};
+
+				pinctrl_nand_default: nand_default {
+					re_we_data {
+						pinmux = <PIN_PA22__D0>,
+							 <PIN_PA23__D1>,
+							 <PIN_PA24__D2>,
+							 <PIN_PA25__D3>,
+							 <PIN_PA26__D4>,
+							 <PIN_PA27__D5>,
+							 <PIN_PA28__D6>,
+							 <PIN_PA29__D7>,
+							 <PIN_PA30__NWE_NANDWE>,
+							 <PIN_PB2__NRD_NANDOE>;
+						bias-pull-up;
+					};
+
+					ale_cle_rdy_cs {
+						pinmux = <PIN_PB0__A21_NANDALE>,
+							 <PIN_PB1__A22_NANDCLE>,
+							 <PIN_PC8__NANDRDY>,
+							 <PIN_PA31__NCS3>;
+						bias-pull-up;
+					};
+				};
+
+				pinctrl_sdmmc0_default: sdmmc0_default {
+					cmd_data {
+						pinmux = <PIN_PA1__SDMMC0_CMD>,
+							 <PIN_PA2__SDMMC0_DAT0>,
+							 <PIN_PA3__SDMMC0_DAT1>,
+							 <PIN_PA4__SDMMC0_DAT2>,
+							 <PIN_PA5__SDMMC0_DAT3>,
+							 <PIN_PA6__SDMMC0_DAT4>,
+							 <PIN_PA7__SDMMC0_DAT5>,
+							 <PIN_PA8__SDMMC0_DAT6>,
+							 <PIN_PA9__SDMMC0_DAT7>;
+						bias-pull-up;
+					};
+
+					ck_cd_vddsel {
+						pinmux = <PIN_PA0__SDMMC0_CK>,
+							 <PIN_PA11__SDMMC0_VDDSEL>,
+							 <PIN_PA13__SDMMC0_CD>;
+						bias-disable;
+					};
+				};
+
+				pinctrl_spi0_default: spi0_default {
+					pinmux = <PIN_PA14__SPI0_SPCK>,
+						 <PIN_PA15__SPI0_MOSI>,
+						 <PIN_PA16__SPI0_MISO>,
+						 <PIN_PA17__SPI0_NPCS0>;
+					bias-disable;
+				};
+
+				pinctrl_spi1_default: spi1_default {
+					pinmux = <PIN_PC1__SPI1_SPCK>,
+						 <PIN_PC2__SPI1_MOSI>,
+						 <PIN_PC3__SPI1_MISO>,
+						 <PIN_PC4__SPI1_NPCS0>;
+					bias-disable;
+				};
+
+				pinctrl_uart0_default: uart0_default {
+					pinmux = <PIN_PB26__URXD0>,
+						 <PIN_PB27__UTXD0>;
+					bias-disable;
+				};
+
+				pinctrl_uart2_default: uart2_default {
+					pinmux = <PIN_PD23__URXD2>,
+						 <PIN_PD24__UTXD2>;
+					bias-disable;
+				};
+
+				pinctrl_usb_default: usb_default {
+					pinmux = <PIN_PB12__GPIO>;
+					bias-disable;
+				};
+
+				pinctrl_usba_vbus: usba_vbus {
+					pinmux = <PIN_PB11__GPIO>;
+					bias-disable;
+				};
+
+			};
+
+		};
+	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_key_gpio_default>;
+
+		bp1 {
+			label = "PB_USER";
+			gpios = <&pioA PIN_PA10 GPIO_ACTIVE_LOW>;
+			linux,code = <0x104>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_led_gpio_default>;
+		status = "okay";
+
+		red {
+			label = "red";
+			gpios = <&pioA PIN_PB10 GPIO_ACTIVE_HIGH>;
+		};
+
+		green {
+			label = "green";
+			gpios = <&pioA PIN_PB8 GPIO_ACTIVE_HIGH>;
+		};
+
+		blue {
+			label = "blue";
+			gpios = <&pioA PIN_PB6 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/at91-sama5d2_xplained.dts b/arch/arm/boot/dts/at91-sama5d2_xplained.dts
index 56de21de2779..e4bbb7e0f793 100644
--- a/arch/arm/boot/dts/at91-sama5d2_xplained.dts
+++ b/arch/arm/boot/dts/at91-sama5d2_xplained.dts
@@ -133,6 +133,18 @@
 				};
 			};
 
+			tcb0: timer@f800c000 {
+				timer0: timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer1: timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			pdmic@f8018000 {
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_pdmic_default>;
diff --git a/arch/arm/boot/dts/at91-sama5d3_xplained.dts b/arch/arm/boot/dts/at91-sama5d3_xplained.dts
index 40879aded680..02c1d2958d78 100644
--- a/arch/arm/boot/dts/at91-sama5d3_xplained.dts
+++ b/arch/arm/boot/dts/at91-sama5d3_xplained.dts
@@ -65,6 +65,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@f0010000 {
+				timer0: timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer1: timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			i2c0: i2c@f0014000 {
 				pinctrl-0 = <&pinctrl_i2c0_pu>;
 				status = "okay";
diff --git a/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi b/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi
index b813fdfa2842..d3e79fbf1944 100644
--- a/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi
+++ b/arch/arm/boot/dts/at91-sama5d4_ma5d4.dtsi
@@ -89,6 +89,18 @@
 				};
 			};
 
+			tcb2: timer@fc024000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			adc0: adc@fc034000 {
 				pinctrl-names = "default";
 				pinctrl-0 = <
diff --git a/arch/arm/boot/dts/at91-sama5d4_xplained.dts b/arch/arm/boot/dts/at91-sama5d4_xplained.dts
index 29ab17a97f9a..4b7c762d5f22 100644
--- a/arch/arm/boot/dts/at91-sama5d4_xplained.dts
+++ b/arch/arm/boot/dts/at91-sama5d4_xplained.dts
@@ -130,6 +130,18 @@
 				status = "okay";
 			};
 
+			tcb2: timer@fc024000 {
+				timer0: timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer1: timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			adc0: adc@fc034000 {
 				pinctrl-names = "default";
 				pinctrl-0 = <
diff --git a/arch/arm/boot/dts/at91-sama5d4ek.dts b/arch/arm/boot/dts/at91-sama5d4ek.dts
index 5b7ee92e32a7..7887a7160a54 100644
--- a/arch/arm/boot/dts/at91-sama5d4ek.dts
+++ b/arch/arm/boot/dts/at91-sama5d4ek.dts
@@ -174,6 +174,18 @@
 				status = "okay";
 			};
 
+			tcb2: timer@fc024000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			watchdog@fc068640 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/at91-vinco.dts b/arch/arm/boot/dts/at91-vinco.dts
index 9f6005708ea8..1be9889a2b3a 100644
--- a/arch/arm/boot/dts/at91-vinco.dts
+++ b/arch/arm/boot/dts/at91-vinco.dts
@@ -151,6 +151,18 @@
 				status = "okay";
 			};
 
+			tcb2: timer@fc024000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			macb1: ethernet@fc028000 {
 				phy-mode = "rmii";
 				status = "okay";
diff --git a/arch/arm/boot/dts/at91rm9200.dtsi b/arch/arm/boot/dts/at91rm9200.dtsi
index da622bf45b4a..ba61893a02a0 100644
--- a/arch/arm/boot/dts/at91rm9200.dtsi
+++ b/arch/arm/boot/dts/at91rm9200.dtsi
@@ -375,7 +375,9 @@
 			};
 
 			tcb0: timer@fffa0000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfffa0000 0x100>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 0
 					      18 IRQ_TYPE_LEVEL_HIGH 0
@@ -385,7 +387,9 @@
 			};
 
 			tcb1: timer@fffa4000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfffa4000 0x100>;
 				interrupts = <20 IRQ_TYPE_LEVEL_HIGH 0
 					      21 IRQ_TYPE_LEVEL_HIGH 0
diff --git a/arch/arm/boot/dts/at91rm9200ek.dts b/arch/arm/boot/dts/at91rm9200ek.dts
index 33192d0cefee..81aaf8151c76 100644
--- a/arch/arm/boot/dts/at91rm9200ek.dts
+++ b/arch/arm/boot/dts/at91rm9200ek.dts
@@ -32,6 +32,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usb1: gadget@fffb0000 {
 				atmel,vbus-gpio = <&pioD 4 GPIO_ACTIVE_HIGH>;
 				atmel,pullup-gpio = <&pioD 5 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/at91sam9260.dtsi b/arch/arm/boot/dts/at91sam9260.dtsi
index bc655e7332d6..655f06cd716a 100644
--- a/arch/arm/boot/dts/at91sam9260.dtsi
+++ b/arch/arm/boot/dts/at91sam9260.dtsi
@@ -386,7 +386,9 @@
 			};
 
 			tcb0: timer@fffa0000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfffa0000 0x100>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 0
 					      18 IRQ_TYPE_LEVEL_HIGH 0
@@ -396,7 +398,9 @@
 			};
 
 			tcb1: timer@fffdc000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfffdc000 0x100>;
 				interrupts = <26 IRQ_TYPE_LEVEL_HIGH 0
 					      27 IRQ_TYPE_LEVEL_HIGH 0
diff --git a/arch/arm/boot/dts/at91sam9260ek.dts b/arch/arm/boot/dts/at91sam9260ek.dts
index b2578feceb08..e16c706d91ef 100644
--- a/arch/arm/boot/dts/at91sam9260ek.dts
+++ b/arch/arm/boot/dts/at91sam9260ek.dts
@@ -69,6 +69,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usb1: gadget@fffa4000 {
 				atmel,vbus-gpio = <&pioC 5 GPIO_ACTIVE_HIGH>;
 				status = "okay";
diff --git a/arch/arm/boot/dts/at91sam9261.dtsi b/arch/arm/boot/dts/at91sam9261.dtsi
index 66876019101d..ddfc63b8fd4e 100644
--- a/arch/arm/boot/dts/at91sam9261.dtsi
+++ b/arch/arm/boot/dts/at91sam9261.dtsi
@@ -80,7 +80,7 @@
 			status = "disabled";
 		};
 
-		fb0: fb@0x00600000 {
+		fb0: fb@600000 {
 			compatible = "atmel,at91sam9261-lcdc";
 			reg = <0x00600000 0x1000>;
 			interrupts = <21 IRQ_TYPE_LEVEL_HIGH 3>;
@@ -125,7 +125,9 @@
 			ranges;
 
 			tcb0: timer@fffa0000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfffa0000 0x100>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 0>,
 					     <18 IRQ_TYPE_LEVEL_HIGH 0>,
diff --git a/arch/arm/boot/dts/at91sam9261ek.dts b/arch/arm/boot/dts/at91sam9261ek.dts
index 960d6940ebf6..a29fc0494076 100644
--- a/arch/arm/boot/dts/at91sam9261ek.dts
+++ b/arch/arm/boot/dts/at91sam9261ek.dts
@@ -15,6 +15,14 @@
 	chosen {
 		bootargs = "rootfstype=ubifs ubi.mtd=5 root=ubi0:rootfs rw";
 		stdout-path = "serial0:115200n8";
+
+		clocksource {
+			timer = <&timer0>;
+		};
+
+		clockevent {
+			timer = <&timer1>;
+		};
 	};
 
 	memory {
@@ -36,7 +44,7 @@
 			status = "okay";
 		};
 
-		fb0: fb@0x00600000 {
+		fb0: fb@600000 {
 			display = <&display0>;
 			atmel,power-control-gpio = <&pioA 12 GPIO_ACTIVE_LOW>;
 			status = "okay";
@@ -125,6 +133,18 @@
 		};
 
 		apb {
+			tcb0: timer@fffa0000 {
+				timer0: timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer1: timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usb1: gadget@fffa4000 {
 				atmel,vbus-gpio = <&pioB 29 GPIO_ACTIVE_HIGH>;
 				status = "okay";
diff --git a/arch/arm/boot/dts/at91sam9263.dtsi b/arch/arm/boot/dts/at91sam9263.dtsi
index e54f14d36b6f..f2405671e3bd 100644
--- a/arch/arm/boot/dts/at91sam9263.dtsi
+++ b/arch/arm/boot/dts/at91sam9263.dtsi
@@ -389,7 +389,9 @@
 			};
 
 			tcb0: timer@fff7c000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfff7c000 0x100>;
 				interrupts = <19 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb_clk>, <&slow_xtal>;
@@ -999,7 +1001,7 @@
 			};
 		};
 
-		fb0: fb@0x00700000 {
+		fb0: fb@700000 {
 			compatible = "atmel,at91sam9263-lcdc";
 			reg = <0x00700000 0x1000>;
 			interrupts = <26 IRQ_TYPE_LEVEL_HIGH 3>;
diff --git a/arch/arm/boot/dts/at91sam9263ek.dts b/arch/arm/boot/dts/at91sam9263ek.dts
index 5a2e1af793f5..e9a7c70830a8 100644
--- a/arch/arm/boot/dts/at91sam9263ek.dts
+++ b/arch/arm/boot/dts/at91sam9263ek.dts
@@ -37,6 +37,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fff7c000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usart0: serial@fff8c000 {
 				pinctrl-0 = <
 					&pinctrl_usart0
@@ -95,7 +107,7 @@
 			};
 		};
 
-		fb0: fb@0x00700000 {
+		fb0: fb@700000 {
 			display = <&display0>;
 			status = "okay";
 
diff --git a/arch/arm/boot/dts/at91sam9g20ek_common.dtsi b/arch/arm/boot/dts/at91sam9g20ek_common.dtsi
index ac9a1511e239..50561b7b7939 100644
--- a/arch/arm/boot/dts/at91sam9g20ek_common.dtsi
+++ b/arch/arm/boot/dts/at91sam9g20ek_common.dtsi
@@ -51,6 +51,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usart0: serial@fffb0000 {
 				pinctrl-0 =
 					<&pinctrl_usart0
diff --git a/arch/arm/boot/dts/at91sam9g25ek.dts b/arch/arm/boot/dts/at91sam9g25ek.dts
index 91a71774472e..31fecc2cdaf9 100644
--- a/arch/arm/boot/dts/at91sam9g25ek.dts
+++ b/arch/arm/boot/dts/at91sam9g25ek.dts
@@ -25,7 +25,7 @@
 			};
 
 			i2c0: i2c@f8010000 {
-				ov2640: camera@0x30 {
+				ov2640: camera@30 {
 					compatible = "ovti,ov2640";
 					reg = <0x30>;
 					pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/at91sam9g45.dtsi b/arch/arm/boot/dts/at91sam9g45.dtsi
index 2b127ca7aaa0..3a30eec7f508 100644
--- a/arch/arm/boot/dts/at91sam9g45.dtsi
+++ b/arch/arm/boot/dts/at91sam9g45.dtsi
@@ -415,7 +415,9 @@
 			};
 
 			tcb0: timer@fff7c000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfff7c000 0x100>;
 				interrupts = <18 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb0_clk>, <&tcb0_clk>, <&tcb0_clk>, <&clk32k>;
@@ -423,7 +425,9 @@
 			};
 
 			tcb1: timer@fffd4000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfffd4000 0x100>;
 				interrupts = <18 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb0_clk>, <&tcb0_clk>, <&tcb0_clk>, <&clk32k>;
@@ -1302,7 +1306,7 @@
 			};
 		};
 
-		fb0: fb@0x00500000 {
+		fb0: fb@500000 {
 			compatible = "atmel,at91sam9g45-lcdc";
 			reg = <0x00500000 0x1000>;
 			interrupts = <23 IRQ_TYPE_LEVEL_HIGH 3>;
diff --git a/arch/arm/boot/dts/at91sam9m10g45ek.dts b/arch/arm/boot/dts/at91sam9m10g45ek.dts
index e922552a04cb..25888339d8de 100644
--- a/arch/arm/boot/dts/at91sam9m10g45ek.dts
+++ b/arch/arm/boot/dts/at91sam9m10g45ek.dts
@@ -39,6 +39,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fff7c000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usart1: serial@fff90000 {
 				pinctrl-0 =
 					<&pinctrl_usart1
@@ -220,7 +232,7 @@
 			};
 		};
 
-		fb0: fb@0x00500000 {
+		fb0: fb@500000 {
 			display = <&display0>;
 			status = "okay";
 
diff --git a/arch/arm/boot/dts/at91sam9n12.dtsi b/arch/arm/boot/dts/at91sam9n12.dtsi
index e0ac824e0785..4b62f4f963f6 100644
--- a/arch/arm/boot/dts/at91sam9n12.dtsi
+++ b/arch/arm/boot/dts/at91sam9n12.dtsi
@@ -446,7 +446,9 @@
 			};
 
 			tcb0: timer@f8008000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf8008000 0x100>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb_clk>, <&clk32k>;
@@ -454,7 +456,9 @@
 			};
 
 			tcb1: timer@f800c000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf800c000 0x100>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb_clk>, <&clk32k>;
diff --git a/arch/arm/boot/dts/at91sam9n12ek.dts b/arch/arm/boot/dts/at91sam9n12ek.dts
index 212562aedf5e..c43095c5cb5c 100644
--- a/arch/arm/boot/dts/at91sam9n12ek.dts
+++ b/arch/arm/boot/dts/at91sam9n12ek.dts
@@ -42,6 +42,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@f8008000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			i2c0: i2c@f8010000 {
 				status = "okay";
 
diff --git a/arch/arm/boot/dts/at91sam9rl.dtsi b/arch/arm/boot/dts/at91sam9rl.dtsi
index 52f0e9ef8f67..3cae687dccbd 100644
--- a/arch/arm/boot/dts/at91sam9rl.dtsi
+++ b/arch/arm/boot/dts/at91sam9rl.dtsi
@@ -124,7 +124,9 @@
 			ranges;
 
 			tcb0: timer@fffa0000 {
-				compatible = "atmel,at91rm9200-tcb";
+				compatible = "atmel,at91rm9200-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfffa0000 0x100>;
 				interrupts = <16 IRQ_TYPE_LEVEL_HIGH 0>,
 					     <17 IRQ_TYPE_LEVEL_HIGH 0>,
diff --git a/arch/arm/boot/dts/at91sam9rlek.dts b/arch/arm/boot/dts/at91sam9rlek.dts
index ea6ed98960c9..4bde9f245e61 100644
--- a/arch/arm/boot/dts/at91sam9rlek.dts
+++ b/arch/arm/boot/dts/at91sam9rlek.dts
@@ -124,6 +124,18 @@
 		};
 
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			mmc0: mmc@fffa4000 {
 				pinctrl-0 = <
 					&pinctrl_board_mmc0
diff --git a/arch/arm/boot/dts/at91sam9x5.dtsi b/arch/arm/boot/dts/at91sam9x5.dtsi
index ad779a7dfefd..fee4fe51a97e 100644
--- a/arch/arm/boot/dts/at91sam9x5.dtsi
+++ b/arch/arm/boot/dts/at91sam9x5.dtsi
@@ -433,7 +433,9 @@
 			};
 
 			tcb0: timer@f8008000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf8008000 0x100>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb0_clk>, <&clk32k>;
@@ -441,7 +443,9 @@
 			};
 
 			tcb1: timer@f800c000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf800c000 0x100>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb0_clk>, <&clk32k>;
diff --git a/arch/arm/boot/dts/at91sam9x5cm.dtsi b/arch/arm/boot/dts/at91sam9x5cm.dtsi
index bdeaa0b64a5b..4908ee07e628 100644
--- a/arch/arm/boot/dts/at91sam9x5cm.dtsi
+++ b/arch/arm/boot/dts/at91sam9x5cm.dtsi
@@ -24,6 +24,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@f8008000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			pinctrl@fffff400 {
 				1wire_cm {
 					pinctrl_1wire_cm: 1wire_cm-0 {
diff --git a/arch/arm/boot/dts/axp81x.dtsi b/arch/arm/boot/dts/axp81x.dtsi
index 73b761f850c5..fd55b896afa1 100644
--- a/arch/arm/boot/dts/axp81x.dtsi
+++ b/arch/arm/boot/dts/axp81x.dtsi
@@ -48,6 +48,22 @@
 	interrupt-controller;
 	#interrupt-cells = <1>;
 
+	axp_gpio: axp-gpio {
+		compatible = "x-powers,axp813-gpio";
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		gpio0_ldo: gpio0-ldo {
+			pins = "GPIO0";
+			function = "ldo";
+		};
+
+		gpio1_ldo: gpio1-ldo {
+			pins = "GPIO1";
+			function = "ldo";
+		};
+	};
+
 	regulators {
 		/* Default work frequency for buck regulators */
 		x-powers,dcdc-freq = <3000>;
@@ -113,11 +129,15 @@
 		};
 
 		reg_ldo_io0: ldo-io0 {
+			pinctrl-names = "default";
+			pinctrl-0 = <&gpio0_ldo>;
 			/* Disable by default to avoid conflicts with GPIO */
 			status = "disabled";
 		};
 
 		reg_ldo_io1: ldo-io1 {
+			pinctrl-names = "default";
+			pinctrl-0 = <&gpio1_ldo>;
 			/* Disable by default to avoid conflicts with GPIO */
 			status = "disabled";
 		};
diff --git a/arch/arm/boot/dts/bcm11351.dtsi b/arch/arm/boot/dts/bcm11351.dtsi
index 18045c38bcf1..db7cded1b7ad 100644
--- a/arch/arm/boot/dts/bcm11351.dtsi
+++ b/arch/arm/boot/dts/bcm11351.dtsi
@@ -55,7 +55,7 @@
 		      <0x3ff00100 0x100>;
 	};
 
-	smc@0x3404c000 {
+	smc@3404c000 {
 		compatible = "brcm,bcm11351-smc", "brcm,kona-smc";
 		reg = <0x3404c000 0x400>; /* 1 KiB in SRAM */
 	};
diff --git a/arch/arm/boot/dts/bcm21664.dtsi b/arch/arm/boot/dts/bcm21664.dtsi
index 6dde95f21cef..266f2611dc22 100644
--- a/arch/arm/boot/dts/bcm21664.dtsi
+++ b/arch/arm/boot/dts/bcm21664.dtsi
@@ -55,7 +55,7 @@
 		      <0x3ff00100 0x100>;
 	};
 
-	smc@0x3404e000 {
+	smc@3404e000 {
 		compatible = "brcm,bcm21664-smc", "brcm,kona-smc";
 		reg = <0x3404e000 0x400>; /* 1 KiB in SRAM */
 	};
diff --git a/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts b/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
index f81ae0a10310..aa1fc7babfea 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-a-plus.dts
@@ -10,12 +10,12 @@
 
 	leds {
 		act {
-			gpios = <&gpio 47 0>;
+			gpios = <&gpio 47 GPIO_ACTIVE_HIGH>;
 		};
 
 		pwr {
 			label = "PWR";
-			gpios = <&gpio 35 0>;
+			gpios = <&gpio 35 GPIO_ACTIVE_HIGH>;
 			default-state = "keep";
 			linux,default-trigger = "default-on";
 		};
diff --git a/arch/arm/boot/dts/bcm2835-rpi-a.dts b/arch/arm/boot/dts/bcm2835-rpi-a.dts
index 7a960a048204..425f6b0a5ef8 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-a.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-a.dts
@@ -10,7 +10,7 @@
 
 	leds {
 		act {
-			gpios = <&gpio 16 1>;
+			gpios = <&gpio 16 GPIO_ACTIVE_LOW>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
index 0161a8483e39..effa195e7895 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts
@@ -11,12 +11,12 @@
 
 	leds {
 		act {
-			gpios = <&gpio 47 0>;
+			gpios = <&gpio 47 GPIO_ACTIVE_HIGH>;
 		};
 
 		pwr {
 			label = "PWR";
-			gpios = <&gpio 35 0>;
+			gpios = <&gpio 35 GPIO_ACTIVE_HIGH>;
 			default-state = "keep";
 			linux,default-trigger = "default-on";
 		};
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
index 4bc70efe43d6..772ec3b48231 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts
@@ -11,7 +11,7 @@
 
 	leds {
 		act {
-			gpios = <&gpio 16 1>;
+			gpios = <&gpio 16 GPIO_ACTIVE_LOW>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/bcm2835-rpi-b.dts b/arch/arm/boot/dts/bcm2835-rpi-b.dts
index cca4a75a5651..434483d6fc14 100644
--- a/arch/arm/boot/dts/bcm2835-rpi-b.dts
+++ b/arch/arm/boot/dts/bcm2835-rpi-b.dts
@@ -11,7 +11,7 @@
 
 	leds {
 		act {
-			gpios = <&gpio 16 1>;
+			gpios = <&gpio 16 GPIO_ACTIVE_LOW>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/bcm2835.dtsi b/arch/arm/boot/dts/bcm2835.dtsi
index 0e3d2a5ff208..a5c3824c8056 100644
--- a/arch/arm/boot/dts/bcm2835.dtsi
+++ b/arch/arm/boot/dts/bcm2835.dtsi
@@ -18,10 +18,10 @@
 	soc {
 		ranges = <0x7e000000 0x20000000 0x02000000>;
 		dma-ranges = <0x40000000 0x00000000 0x20000000>;
+	};
 
-		arm-pmu {
-			compatible = "arm,arm1176-pmu";
-		};
+	arm-pmu {
+		compatible = "arm,arm1176-pmu";
 	};
 };
 
diff --git a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
index 6669355fd655..5c339adabdf0 100644
--- a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
+++ b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts
@@ -15,12 +15,12 @@
 
 	leds {
 		act {
-			gpios = <&gpio 47 0>;
+			gpios = <&gpio 47 GPIO_ACTIVE_HIGH>;
 		};
 
 		pwr {
 			label = "PWR";
-			gpios = <&gpio 35 0>;
+			gpios = <&gpio 35 GPIO_ACTIVE_HIGH>;
 			default-state = "keep";
 			linux,default-trigger = "default-on";
 		};
diff --git a/arch/arm/boot/dts/bcm2836.dtsi b/arch/arm/boot/dts/bcm2836.dtsi
index 61e158003509..c933e8413884 100644
--- a/arch/arm/boot/dts/bcm2836.dtsi
+++ b/arch/arm/boot/dts/bcm2836.dtsi
@@ -9,28 +9,28 @@
 			 <0x40000000 0x40000000 0x00001000>;
 		dma-ranges = <0xc0000000 0x00000000 0x3f000000>;
 
-		local_intc: local_intc {
+		local_intc: local_intc@40000000 {
 			compatible = "brcm,bcm2836-l1-intc";
 			reg = <0x40000000 0x100>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
 			interrupt-parent = <&local_intc>;
 		};
+	};
 
-		arm-pmu {
-			compatible = "arm,cortex-a7-pmu";
-			interrupt-parent = <&local_intc>;
-			interrupts = <9>;
-		};
+	arm-pmu {
+		compatible = "arm,cortex-a7-pmu";
+		interrupt-parent = <&local_intc>;
+		interrupts = <9 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	timer {
 		compatible = "arm,armv7-timer";
 		interrupt-parent = <&local_intc>;
-		interrupts = <0>, // PHYS_SECURE_PPI
-			     <1>, // PHYS_NONSECURE_PPI
-			     <3>, // VIRT_PPI
-			     <2>; // HYP_PPI
+		interrupts = <0 IRQ_TYPE_LEVEL_HIGH>, // PHYS_SECURE_PPI
+			     <1 IRQ_TYPE_LEVEL_HIGH>, // PHYS_NONSECURE_PPI
+			     <3 IRQ_TYPE_LEVEL_HIGH>, // VIRT_PPI
+			     <2 IRQ_TYPE_LEVEL_HIGH>; // HYP_PPI
 		always-on;
 	};
 
@@ -76,7 +76,7 @@
 	compatible = "brcm,bcm2836-armctrl-ic";
 	reg = <0x7e00b200 0x200>;
 	interrupt-parent = <&local_intc>;
-	interrupts = <8>;
+	interrupts = <8 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &cpu_thermal {
diff --git a/arch/arm/boot/dts/bcm2837-rpi-3-b.dts b/arch/arm/boot/dts/bcm2837-rpi-3-b.dts
index a8844d033b3f..3e4ed7c5b0b3 100644
--- a/arch/arm/boot/dts/bcm2837-rpi-3-b.dts
+++ b/arch/arm/boot/dts/bcm2837-rpi-3-b.dts
@@ -20,7 +20,7 @@
 
 	leds {
 		act {
-			gpios = <&gpio 47 0>;
+			gpios = <&gpio 47 GPIO_ACTIVE_HIGH>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/bcm2837.dtsi b/arch/arm/boot/dts/bcm2837.dtsi
index bc1cca5cf43c..7704bb029605 100644
--- a/arch/arm/boot/dts/bcm2837.dtsi
+++ b/arch/arm/boot/dts/bcm2837.dtsi
@@ -8,11 +8,11 @@
 			 <0x40000000 0x40000000 0x00001000>;
 		dma-ranges = <0xc0000000 0x00000000 0x3f000000>;
 
-		local_intc: local_intc {
+		local_intc: local_intc@40000000 {
 			compatible = "brcm,bcm2836-l1-intc";
 			reg = <0x40000000 0x100>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
 			interrupt-parent = <&local_intc>;
 		};
 	};
@@ -20,10 +20,10 @@
 	timer {
 		compatible = "arm,armv7-timer";
 		interrupt-parent = <&local_intc>;
-		interrupts = <0>, // PHYS_SECURE_PPI
-			     <1>, // PHYS_NONSECURE_PPI
-			     <3>, // VIRT_PPI
-			     <2>; // HYP_PPI
+		interrupts = <0 IRQ_TYPE_LEVEL_HIGH>, // PHYS_SECURE_PPI
+			     <1 IRQ_TYPE_LEVEL_HIGH>, // PHYS_NONSECURE_PPI
+			     <3 IRQ_TYPE_LEVEL_HIGH>, // VIRT_PPI
+			     <2 IRQ_TYPE_LEVEL_HIGH>; // HYP_PPI
 		always-on;
 	};
 
@@ -73,7 +73,7 @@
 	compatible = "brcm,bcm2836-armctrl-ic";
 	reg = <0x7e00b200 0x200>;
 	interrupt-parent = <&local_intc>;
-	interrupts = <8>;
+	interrupts = <8 IRQ_TYPE_LEVEL_HIGH>;
 };
 
 &cpu_thermal {
diff --git a/arch/arm/boot/dts/bcm283x.dtsi b/arch/arm/boot/dts/bcm283x.dtsi
index dcde93c85c2d..9d293decf8d3 100644
--- a/arch/arm/boot/dts/bcm283x.dtsi
+++ b/arch/arm/boot/dts/bcm283x.dtsi
@@ -2,6 +2,7 @@
 #include <dt-bindings/clock/bcm2835.h>
 #include <dt-bindings/clock/bcm2835-aux.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 
 /* firmware-provided startup stubs live here, where the secondary CPUs are
  * spinning.
@@ -464,7 +465,7 @@
 			status = "disabled";
 		};
 
-		aux: aux@0x7e215000 {
+		aux: aux@7e215000 {
 			compatible = "brcm,bcm2835-aux";
 			#clock-cells = <1>;
 			reg = <0x7e215000 0x8>;
diff --git a/arch/arm/boot/dts/bcm958625hr.dts b/arch/arm/boot/dts/bcm958625hr.dts
index 6a44b8021702..f0e2008f7490 100644
--- a/arch/arm/boot/dts/bcm958625hr.dts
+++ b/arch/arm/boot/dts/bcm958625hr.dts
@@ -49,7 +49,7 @@
 
 	memory {
 		device_type = "memory";
-		reg = <0x60000000 0x80000000>;
+		reg = <0x60000000 0x20000000>;
 	};
 
 	gpio-restart {
diff --git a/arch/arm/boot/dts/dm814x-clocks.dtsi b/arch/arm/boot/dts/dm814x-clocks.dtsi
index c4671af0a28d..f80525a290bb 100644
--- a/arch/arm/boot/dts/dm814x-clocks.dtsi
+++ b/arch/arm/boot/dts/dm814x-clocks.dtsi
@@ -337,3 +337,33 @@
 		clock-frequency = <20000000>;
 	};
 };
+
+&prcm {
+	default_cm: default_cm@500 {
+		compatible = "ti,omap4-cm";
+		reg = <0x500 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x500 0x100>;
+
+		default_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x5c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	alwon_cm: alwon_cm@1400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1400 0x300>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1400 0x300>;
+
+		alwon_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x228>;
+			#clock-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/dm814x.dtsi b/arch/arm/boot/dts/dm814x.dtsi
index 681f5487406e..601c57afd4fe 100644
--- a/arch/arm/boot/dts/dm814x.dtsi
+++ b/arch/arm/boot/dts/dm814x.dtsi
@@ -250,6 +250,8 @@
 				interrupts = <67>;
 				ti,hwmods = "timer1";
 				ti,timer-alwon;
+				clocks = <&timer1_fck>;
+				clock-names = "fck";
 			};
 
 			uart1: uart@20000 {
@@ -287,6 +289,8 @@
 				reg = <0x40000 0x2000>;
 				interrupts = <68>;
 				ti,hwmods = "timer2";
+				clocks = <&timer2_fck>;
+				clock-names = "fck";
 			};
 
 			timer3: timer@42000 {
diff --git a/arch/arm/boot/dts/dm816x-clocks.dtsi b/arch/arm/boot/dts/dm816x-clocks.dtsi
index 51865eb84a80..1efd4e23e50d 100644
--- a/arch/arm/boot/dts/dm816x-clocks.dtsi
+++ b/arch/arm/boot/dts/dm816x-clocks.dtsi
@@ -248,3 +248,33 @@
 		reg = <0x03a8>;
 	};
 };
+
+&prcm {
+	default_cm: default_cm@500 {
+		compatible = "ti,omap4-cm";
+		reg = <0x500 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x500 0x100>;
+
+		default_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x5c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	alwon_cm: alwon_cm@1400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1400 0x300>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1400 0x300>;
+
+		alwon_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x208>;
+			#clock-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/dm816x.dtsi b/arch/arm/boot/dts/dm816x.dtsi
index 566b2a8c8b96..1edc2b48b254 100644
--- a/arch/arm/boot/dts/dm816x.dtsi
+++ b/arch/arm/boot/dts/dm816x.dtsi
@@ -67,8 +67,11 @@
 		ranges;
 
 		prcm: prcm@48180000 {
-			compatible = "ti,dm816-prcm";
+			compatible = "ti,dm816-prcm", "simple-bus";
 			reg = <0x48180000 0x4000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x48180000 0x4000>;
 
 			prcm_clocks: clocks {
 				#address-cells = <1>;
@@ -331,6 +334,8 @@
 			interrupts = <67>;
 			ti,hwmods = "timer1";
 			ti,timer-alwon;
+			clocks = <&timer1_fck>;
+			clock-names = "fck";
 		};
 
 		timer2: timer@48040000 {
@@ -338,6 +343,8 @@
 			reg = <0x48040000 0x2000>;
 			interrupts = <68>;
 			ti,hwmods = "timer2";
+			clocks = <&timer2_fck>;
+			clock-names = "fck";
 		};
 
 		timer3: timer@48042000 {
diff --git a/arch/arm/boot/dts/dra62x-j5eco-evm.dts b/arch/arm/boot/dts/dra62x-j5eco-evm.dts
index 155eb32ee213..fee0547f7302 100644
--- a/arch/arm/boot/dts/dra62x-j5eco-evm.dts
+++ b/arch/arm/boot/dts/dra62x-j5eco-evm.dts
@@ -74,19 +74,19 @@
 			label = "X-Loader";
 			reg = <0 0x80000>;
 		};
-		partition@0x80000 {
+		partition@80000 {
 			label = "U-Boot";
 			reg = <0x80000 0x1c0000>;
 		};
-		partition@0x1c0000 {
+		partition@1c0000 {
 			label = "Environment";
 			reg = <0x240000 0x40000>;
 		};
-		partition@0x280000 {
+		partition@280000 {
 			label = "Kernel";
 			reg = <0x280000 0x500000>;
 		};
-		partition@0x780000 {
+		partition@780000 {
 			label = "Filesystem";
 			reg = <0x780000 0xf880000>;
 		};
diff --git a/arch/arm/boot/dts/dra7-evm-common.dtsi b/arch/arm/boot/dts/dra7-evm-common.dtsi
index e088bb93636a..05a7b1a01bc3 100644
--- a/arch/arm/boot/dts/dra7-evm-common.dtsi
+++ b/arch/arm/boot/dts/dra7-evm-common.dtsi
@@ -204,7 +204,7 @@
 
 &atl {
 	assigned-clocks = <&abe_dpll_sys_clk_mux>,
-			  <&atl_gfclk_mux>,
+			  <&atl_clkctrl DRA7_ATL_CLKCTRL 26>,
 			  <&dpll_abe_ck>,
 			  <&dpll_abe_m2x2_ck>,
 			  <&atl_clkin2_ck>;
@@ -222,7 +222,7 @@
 &mcasp3 {
 	#sound-dai-cells = <0>;
 
-	assigned-clocks = <&mcasp3_ahclkx_mux>;
+	assigned-clocks = <&l4per_clkctrl DRA7_MCASP3_CLKCTRL 24>;
 	assigned-clock-parents = <&atl_clkin2_ck>;
 
 	status = "okay";
diff --git a/arch/arm/boot/dts/dra7-evm.dts b/arch/arm/boot/dts/dra7-evm.dts
index ef9c90daa74b..a7385c338ee9 100644
--- a/arch/arm/boot/dts/dra7-evm.dts
+++ b/arch/arm/boot/dts/dra7-evm.dts
@@ -361,7 +361,7 @@
 };
 
 &cpu0 {
-	cpu0-supply = <&smps123_reg>;
+	vdd-supply = <&smps123_reg>;
 };
 
 &omap_dwc3_2 {
diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi
index ac9216293b7c..e4a420f16800 100644
--- a/arch/arm/boot/dts/dra7.dtsi
+++ b/arch/arm/boot/dts/dra7.dtsi
@@ -7,8 +7,11 @@
  * Based on "omap4.dtsi"
  */
 
+#include <dt-bindings/bus/ti-sysc.h>
+#include <dt-bindings/clock/dra7.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/pinctrl/dra.h>
+#include <dt-bindings/clock/dra7.h>
 
 #define MAX_SOURCES 400
 
@@ -92,6 +95,8 @@
 			cooling-min-level = <0>;
 			cooling-max-level = <2>;
 			#cooling-cells = <2>; /* min followed by max */
+
+			vbb-supply = <&abb_mpu>;
 		};
 	};
 
@@ -101,16 +106,26 @@
 
 		opp_nom-1000000000 {
 			opp-hz = /bits/ 64 <1000000000>;
-			opp-microvolt = <1060000 850000 1150000>;
+			opp-microvolt = <1060000 850000 1150000>,
+					<1060000 850000 1150000>;
 			opp-supported-hw = <0xFF 0x01>;
 			opp-suspend;
 		};
 
 		opp_od-1176000000 {
 			opp-hz = /bits/ 64 <1176000000>;
-			opp-microvolt = <1160000 885000 1160000>;
+			opp-microvolt = <1160000 885000 1160000>,
+					<1160000 885000 1160000>;
+
 			opp-supported-hw = <0xFF 0x02>;
 		};
+
+		opp_high@1500000000 {
+			opp-hz = /bits/ 64 <1500000000>;
+			opp-microvolt = <1210000 950000 1250000>,
+					<1210000 950000 1250000>;
+			opp-supported-hw = <0xFF 0x04>;
+		};
 	};
 
 	/*
@@ -224,8 +239,12 @@
 			};
 
 			cm_core_aon: cm_core_aon@5000 {
-				compatible = "ti,dra7-cm-core-aon";
+				compatible = "ti,dra7-cm-core-aon",
+					      "simple-bus";
+				#address-cells = <1>;
+				#size-cells = <1>;
 				reg = <0x5000 0x2000>;
+				ranges = <0 0x5000 0x2000>;
 
 				cm_core_aon_clocks: clocks {
 					#address-cells = <1>;
@@ -237,8 +256,11 @@
 			};
 
 			cm_core: cm_core@8000 {
-				compatible = "ti,dra7-cm-core";
+				compatible = "ti,dra7-cm-core", "simple-bus";
+				#address-cells = <1>;
+				#size-cells = <1>;
 				reg = <0x8000 0x3000>;
+				ranges = <0 0x8000 0x3000>;
 
 				cm_core_clocks: clocks {
 					#address-cells = <1>;
@@ -263,9 +285,12 @@
 			};
 
 			prm: prm@6000 {
-				compatible = "ti,dra7-prm";
+				compatible = "ti,dra7-prm", "simple-bus";
 				reg = <0x6000 0x3000>;
 				interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x6000 0x3000>;
 
 				prm_clocks: clocks {
 					#address-cells = <1>;
@@ -293,7 +318,6 @@
 			 * node and enable pcie1_ep mode.
 			 */
 			pcie1_rc: pcie@51000000 {
-				compatible = "ti,dra7-pcie";
 				reg = <0x51000000 0x2000>, <0x51002000 0x14c>, <0x1000 0x2000>;
 				reg-names = "rc_dbics", "ti_conf", "config";
 				interrupts = <0 232 0x4>, <0 233 0x4>;
@@ -323,7 +347,6 @@
 			};
 
 			pcie1_ep: pcie_ep@51000000 {
-				compatible = "ti,dra7-pcie-ep";
 				reg = <0x51000000 0x28>, <0x51002000 0x14c>, <0x51001000 0x28>, <0x1000 0x10000000>;
 				reg-names = "ep_dbics", "ti_conf", "ep_dbics2", "addr_space";
 				interrupts = <0 232 0x4>;
@@ -345,8 +368,7 @@
 			ranges = <0x51800000 0x51800000 0x3000
 				  0x0	     0x30000000 0x10000000>;
 			status = "disabled";
-			pcie@51800000 {
-				compatible = "ti,dra7-pcie";
+			pcie2_rc: pcie@51800000 {
 				reg = <0x51800000 0x2000>, <0x51802000 0x14c>, <0x1000 0x2000>;
 				reg-names = "rc_dbics", "ti_conf", "config";
 				interrupts = <0 355 0x4>, <0 356 0x4>;
@@ -876,6 +898,8 @@
 			interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer1";
 			ti,timer-alwon;
+			clock-names = "fck";
+			clocks = <&wkupaon_clkctrl DRA7_TIMER1_CLKCTRL 24>;
 		};
 
 		timer2: timer@48032000 {
@@ -1358,7 +1382,7 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			ti,hwmods = "qspi";
-			clocks = <&qspi_gfclk_div>;
+			clocks = <&l4per_clkctrl DRA7_QSPI_CLKCTRL 25>;
 			clock-names = "fck";
 			num-cs = <4>;
 			interrupts = <GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH>;
@@ -1373,14 +1397,15 @@
 			ranges;
 			reg = <0x4a090000 0x20>;
 			ti,hwmods = "ocp2scp3";
-			sata_phy: phy@4A096000 {
+			sata_phy: phy@4a096000 {
 				compatible = "ti,phy-pipe3-sata";
 				reg = <0x4A096000 0x80>, /* phy_rx */
 				      <0x4A096400 0x64>, /* phy_tx */
 				      <0x4A096800 0x40>; /* pll_ctrl */
 				reg-names = "phy_rx", "phy_tx", "pll_ctrl";
 				syscon-phy-power = <&scm_conf 0x374>;
-				clocks = <&sys_clkin1>, <&sata_ref_clk>;
+				clocks = <&sys_clkin1>,
+					 <&l3init_clkctrl DRA7_SATA_CLKCTRL 8>;
 				clock-names = "sysclk", "refclk";
 				syscon-pllreset = <&scm_conf 0x3fc>;
 				#phy-cells = <0>;
@@ -1395,9 +1420,9 @@
 				syscon-pcs = <&scm_conf_pcie 0x10>;
 				clocks = <&dpll_pcie_ref_ck>,
 					 <&dpll_pcie_ref_m2ldo_ck>,
-					 <&optfclk_pciephy1_32khz>,
-					 <&optfclk_pciephy1_clk>,
-					 <&optfclk_pciephy1_div_clk>,
+					 <&l3init_clkctrl DRA7_PCIE1_CLKCTRL 8>,
+					 <&l3init_clkctrl DRA7_PCIE1_CLKCTRL 9>,
+					 <&l3init_clkctrl DRA7_PCIE1_CLKCTRL 10>,
 					 <&optfclk_pciephy_div>,
 					 <&sys_clkin1>;
 				clock-names = "dpll_ref", "dpll_ref_m2",
@@ -1415,9 +1440,9 @@
 				syscon-pcs = <&scm_conf_pcie 0x10>;
 				clocks = <&dpll_pcie_ref_ck>,
 					 <&dpll_pcie_ref_m2ldo_ck>,
-					 <&optfclk_pciephy2_32khz>,
-					 <&optfclk_pciephy2_clk>,
-					 <&optfclk_pciephy2_div_clk>,
+					 <&l3init_clkctrl DRA7_PCIE2_CLKCTRL 8>,
+					 <&l3init_clkctrl DRA7_PCIE2_CLKCTRL 9>,
+					 <&l3init_clkctrl DRA7_PCIE2_CLKCTRL 10>,
 					 <&optfclk_pciephy_div>,
 					 <&sys_clkin1>;
 				clock-names = "dpll_ref", "dpll_ref_m2",
@@ -1434,7 +1459,7 @@
 			interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
 			phys = <&sata_phy>;
 			phy-names = "sata-phy";
-			clocks = <&sata_ref_clk>;
+			clocks = <&l3init_clkctrl DRA7_SATA_CLKCTRL 8>;
 			ti,hwmods = "sata";
 			ports-implemented = <0x1>;
 		};
@@ -1462,7 +1487,7 @@
 				reg = <0x4a084000 0x400>;
 				syscon-phy-power = <&scm_conf 0x300>;
 				clocks = <&usb_phy1_always_on_clk32k>,
-					 <&usb_otg_ss1_refclk960m>;
+					 <&l3init_clkctrl DRA7_USB_OTG_SS1_CLKCTRL 8>;
 				clock-names =	"wkupclk",
 						"refclk";
 				#phy-cells = <0>;
@@ -1474,7 +1499,7 @@
 				reg = <0x4a085000 0x400>;
 				syscon-phy-power = <&scm_conf 0xe74>;
 				clocks = <&usb_phy2_always_on_clk32k>,
-					 <&usb_otg_ss2_refclk960m>;
+					 <&l3init_clkctrl DRA7_USB_OTG_SS2_CLKCTRL 8>;
 				clock-names =	"wkupclk",
 						"refclk";
 				#phy-cells = <0>;
@@ -1489,7 +1514,7 @@
 				syscon-phy-power = <&scm_conf 0x370>;
 				clocks = <&usb_phy3_always_on_clk32k>,
 					 <&sys_clkin1>,
-					 <&usb_otg_ss1_refclk960m>;
+					 <&l3init_clkctrl DRA7_USB_OTG_SS1_CLKCTRL 8>;
 				clock-names =	"wkupclk",
 						"sysclk",
 						"refclk";
@@ -1498,11 +1523,17 @@
 		};
 
 		target-module@4a0dd000 {
-			compatible = "ti,sysc-omap4-sr";
+			compatible = "ti,sysc-omap4-sr", "ti,sysc";
 			ti,hwmods = "smartreflex_core";
-			reg = <0x4a0dd000 0x4>,
-			      <0x4a0dd008 0x4>;
-			reg-names = "rev", "sysc";
+			reg = <0x4a0dd038 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP3_SR_ENAWAKEUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&coreaon_clkctrl DRA7_SMARTREFLEX_CORE_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x4a0dd000 0x001000>;
@@ -1511,11 +1542,17 @@
 		};
 
 		target-module@4a0d9000 {
-			compatible = "ti,sysc-omap4-sr";
+			compatible = "ti,sysc-omap4-sr", "ti,sysc";
 			ti,hwmods = "smartreflex_mpu";
-			reg = <0x4a0d9000 0x4>,
-			      <0x4a0d9008 0x4>;
-			reg-names = "rev", "sysc";
+			reg = <0x4a0d9038 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP3_SR_ENAWAKEUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&coreaon_clkctrl DRA7_SMARTREFLEX_MPU_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x4a0d9000 0x001000>;
@@ -1547,6 +1584,7 @@
 				dr_mode = "otg";
 				snps,dis_u3_susphy_quirk;
 				snps,dis_u2_susphy_quirk;
+				snps,dis_metastability_quirk;
 			};
 		};
 
@@ -1636,7 +1674,7 @@
 			ti,hwmods = "atl";
 			ti,provided-clocks = <&atl_clkin0_ck>, <&atl_clkin1_ck>,
 					     <&atl_clkin2_ck>, <&atl_clkin3_ck>;
-			clocks = <&atl_gfclk_mux>;
+			clocks = <&atl_clkctrl DRA7_ATL_CLKCTRL 26>;
 			clock-names = "fck";
 			status = "disabled";
 		};
@@ -1652,8 +1690,8 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 129 1>, <&edma_xbar 128 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp1_aux_gfclk_mux>, <&mcasp1_ahclkx_mux>,
-				 <&mcasp1_ahclkr_mux>;
+			clocks = <&ipu_clkctrl DRA7_MCASP1_CLKCTRL 22>, <&ipu_clkctrl DRA7_MCASP1_CLKCTRL 24>,
+				 <&ipu_clkctrl DRA7_MCASP1_CLKCTRL 28>;
 			clock-names = "fck", "ahclkx", "ahclkr";
 			status = "disabled";
 		};
@@ -1669,8 +1707,9 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 131 1>, <&edma_xbar 130 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp2_aux_gfclk_mux>, <&mcasp2_ahclkx_mux>,
-				 <&mcasp2_ahclkr_mux>;
+			clocks = <&l4per_clkctrl DRA7_MCASP2_CLKCTRL 22>,
+				 <&l4per_clkctrl DRA7_MCASP2_CLKCTRL 24>,
+				 <&l4per_clkctrl DRA7_MCASP2_CLKCTRL 28>;
 			clock-names = "fck", "ahclkx", "ahclkr";
 			status = "disabled";
 		};
@@ -1686,7 +1725,8 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 133 1>, <&edma_xbar 132 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp3_aux_gfclk_mux>, <&mcasp3_ahclkx_mux>;
+			clocks = <&l4per_clkctrl DRA7_MCASP3_CLKCTRL 22>,
+				 <&l4per_clkctrl DRA7_MCASP3_CLKCTRL 24>;
 			clock-names = "fck", "ahclkx";
 			status = "disabled";
 		};
@@ -1702,7 +1742,8 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 135 1>, <&edma_xbar 134 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp4_aux_gfclk_mux>, <&mcasp4_ahclkx_mux>;
+			clocks = <&l4per_clkctrl DRA7_MCASP4_CLKCTRL 22>,
+				 <&l4per_clkctrl DRA7_MCASP4_CLKCTRL 24>;
 			clock-names = "fck", "ahclkx";
 			status = "disabled";
 		};
@@ -1718,7 +1759,8 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 137 1>, <&edma_xbar 136 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp5_aux_gfclk_mux>, <&mcasp5_ahclkx_mux>;
+			clocks = <&l4per_clkctrl DRA7_MCASP5_CLKCTRL 22>,
+				 <&l4per_clkctrl DRA7_MCASP5_CLKCTRL 24>;
 			clock-names = "fck", "ahclkx";
 			status = "disabled";
 		};
@@ -1734,7 +1776,8 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 139 1>, <&edma_xbar 138 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp6_aux_gfclk_mux>, <&mcasp6_ahclkx_mux>;
+			clocks = <&l4per_clkctrl DRA7_MCASP6_CLKCTRL 22>,
+				 <&l4per_clkctrl DRA7_MCASP6_CLKCTRL 24>;
 			clock-names = "fck", "ahclkx";
 			status = "disabled";
 		};
@@ -1750,7 +1793,8 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 141 1>, <&edma_xbar 140 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp7_aux_gfclk_mux>, <&mcasp7_ahclkx_mux>;
+			clocks = <&l4per_clkctrl DRA7_MCASP7_CLKCTRL 22>,
+				 <&l4per_clkctrl DRA7_MCASP7_CLKCTRL 24>;
 			clock-names = "fck", "ahclkx";
 			status = "disabled";
 		};
@@ -1766,7 +1810,8 @@
 			interrupt-names = "tx", "rx";
 			dmas = <&edma_xbar 143 1>, <&edma_xbar 142 1>;
 			dma-names = "tx", "rx";
-			clocks = <&mcasp8_aux_gfclk_mux>, <&mcasp8_ahclkx_mux>;
+			clocks = <&l4per_clkctrl DRA7_MCASP8_CLKCTRL 22>,
+				 <&l4per_clkctrl DRA7_MCASP8_CLKCTRL 24>;
 			clock-names = "fck", "ahclkx";
 			status = "disabled";
 		};
@@ -1788,7 +1833,7 @@
 		mac: ethernet@48484000 {
 			compatible = "ti,dra7-cpsw","ti,cpsw";
 			ti,hwmods = "gmac";
-			clocks = <&gmac_main_clk>, <&gmac_rft_clk_mux>;
+			clocks = <&gmac_main_clk>, <&l3init_clkctrl DRA7_GMAC_CLKCTRL 25>;
 			clock-names = "fck", "cpts";
 			cpdma_channels = <8>;
 			ale_entries = <1024>;
@@ -1858,7 +1903,7 @@
 			reg = <0x4ae3c000 0x2000>;
 			syscon-raminit = <&scm_conf 0x558 0>;
 			interrupts = <GIC_SPI 222 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dcan1_sys_clk_mux>;
+			clocks = <&wkupaon_clkctrl DRA7_DCAN1_CLKCTRL 24>;
 			status = "disabled";
 		};
 
@@ -1889,7 +1934,7 @@
 				reg = <0x58001000 0x1000>;
 				interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
 				ti,hwmods = "dss_dispc";
-				clocks = <&dss_dss_clk>;
+				clocks = <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 8>;
 				clock-names = "fck";
 				/* CTRL_CORE_SMA_SW_1 */
 				syscon-pol = <&scm_conf 0x534>;
@@ -1905,8 +1950,11 @@
 				interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
 				status = "disabled";
 				ti,hwmods = "dss_hdmi";
-				clocks = <&dss_48mhz_clk>, <&dss_hdmi_clk>;
+				clocks = <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 9>,
+					 <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 10>;
 				clock-names = "fck", "sys_clk";
+				dmas = <&sdma_xbar 76>;
+				dma-names = "audio_tx";
 			};
 		};
 
@@ -2052,6 +2100,19 @@
 			clocks = <&l3_iclk_div>;
 			clock-names = "fck";
 		};
+
+		opp_supply_mpu: opp-supply@4a003b20 {
+			compatible = "ti,omap5-opp-supply";
+			reg = <0x4a003b20 0xc>;
+			ti,efuse-settings = <
+			/* uV   offset */
+			1060000 0x0
+			1160000 0x4
+			1210000 0x8
+			>;
+			ti,absolute-max-voltage-uv = <1500000>;
+		};
+
 	};
 
 	thermal_zones: thermal-zones {
@@ -2089,4 +2150,20 @@
 	temperature = <120000>; /* milli Celsius */
 };
 
-/include/ "dra7xx-clocks.dtsi"
+#include "dra7xx-clocks.dtsi"
+
+&core_crit {
+	temperature = <120000>; /* milli Celsius */
+};
+
+&gpu_crit {
+	temperature = <120000>; /* milli Celsius */
+};
+
+&dspeve_crit {
+	temperature = <120000>; /* milli Celsius */
+};
+
+&iva_crit {
+	temperature = <120000>; /* milli Celsius */
+};
diff --git a/arch/arm/boot/dts/dra72-evm-common.dtsi b/arch/arm/boot/dts/dra72-evm-common.dtsi
index 2e485a13dfd7..e85f560a2f78 100644
--- a/arch/arm/boot/dts/dra72-evm-common.dtsi
+++ b/arch/arm/boot/dts/dra72-evm-common.dtsi
@@ -514,7 +514,7 @@
 
 &atl {
 	assigned-clocks = <&abe_dpll_sys_clk_mux>,
-			  <&atl_gfclk_mux>,
+			  <&atl_clkctrl DRA7_ATL_CLKCTRL 26>,
 			  <&dpll_abe_ck>,
 			  <&dpll_abe_m2x2_ck>,
 			  <&atl_clkin2_ck>;
@@ -532,7 +532,7 @@
 &mcasp3 {
 	#sound-dai-cells = <0>;
 
-	assigned-clocks = <&mcasp3_ahclkx_mux>;
+	assigned-clocks = <&l4per_clkctrl DRA7_MCASP3_CLKCTRL 24>;
 	assigned-clock-parents = <&atl_clkin2_ck>;
 
 	status = "okay";
diff --git a/arch/arm/boot/dts/dra72-evm-tps65917.dtsi b/arch/arm/boot/dts/dra72-evm-tps65917.dtsi
index 57bfe5caf5e4..47bb90f41f3f 100644
--- a/arch/arm/boot/dts/dra72-evm-tps65917.dtsi
+++ b/arch/arm/boot/dts/dra72-evm-tps65917.dtsi
@@ -148,3 +148,7 @@
 &mmc1 {
 	vqmmc-supply = <&ldo1_reg>;
 };
+
+&cpu0 {
+	vdd-supply = <&smps1_reg>;
+};
diff --git a/arch/arm/boot/dts/dra72x.dtsi b/arch/arm/boot/dts/dra72x.dtsi
index 67107605fb4c..c011d2e64fef 100644
--- a/arch/arm/boot/dts/dra72x.dtsi
+++ b/arch/arm/boot/dts/dra72x.dtsi
@@ -25,8 +25,8 @@
 	      <0x58004300 0x20>;
 	reg-names = "dss", "pll1_clkctrl", "pll1";
 
-	clocks = <&dss_dss_clk>,
-		 <&dss_video1_clk>;
+	clocks = <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 8>,
+		 <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 12>;
 	clock-names = "fck", "video1_clk";
 };
 
@@ -50,3 +50,15 @@
 		status = "disabled";
 	};
 };
+
+&pcie1_rc {
+	compatible = "ti,dra726-pcie-rc", "ti,dra7-pcie";
+};
+
+&pcie1_ep {
+	compatible = "ti,dra726-pcie-ep", "ti,dra7-pcie-ep";
+};
+
+&pcie2_rc {
+	compatible = "ti,dra726-pcie-rc", "ti,dra7-pcie";
+};
diff --git a/arch/arm/boot/dts/dra74x.dtsi b/arch/arm/boot/dts/dra74x.dtsi
index 24e6746c5b26..91e93ab588ca 100644
--- a/arch/arm/boot/dts/dra74x.dtsi
+++ b/arch/arm/boot/dts/dra74x.dtsi
@@ -93,9 +93,9 @@
 	reg-names = "dss", "pll1_clkctrl", "pll1",
 		    "pll2_clkctrl", "pll2";
 
-	clocks = <&dss_dss_clk>,
-		 <&dss_video1_clk>,
-		 <&dss_video2_clk>;
+	clocks = <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 8>,
+		 <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 12>,
+		 <&dss_clkctrl DRA7_DSS_CORE_CLKCTRL 13>;
 	clock-names = "fck", "video1_clk", "video2_clk";
 };
 
@@ -124,3 +124,15 @@
 		status = "disabled";
 	};
 };
+
+&pcie1_rc {
+	compatible = "ti,dra746-pcie-rc", "ti,dra7-pcie";
+};
+
+&pcie1_ep {
+	compatible = "ti,dra746-pcie-ep", "ti,dra7-pcie-ep";
+};
+
+&pcie2_rc {
+	compatible = "ti,dra746-pcie-rc", "ti,dra7-pcie";
+};
diff --git a/arch/arm/boot/dts/dra76-evm.dts b/arch/arm/boot/dts/dra76-evm.dts
index b024a65c6e27..c4fe7f8ef72a 100644
--- a/arch/arm/boot/dts/dra76-evm.dts
+++ b/arch/arm/boot/dts/dra76-evm.dts
@@ -148,6 +148,7 @@
 		compatible = "ti,tps65917";
 		reg = <0x58>;
 		ti,system-power-controller;
+		ti,palmas-override-powerhold;
 		interrupt-controller;
 		#interrupt-cells = <2>;
 
@@ -421,3 +422,19 @@
 		spi-max-frequency = <96000000>;
 	};
 };
+
+&pcie2_phy {
+	status = "okay";
+};
+
+&pcie1_rc {
+	num-lanes = <2>;
+	phys = <&pcie1_phy>, <&pcie2_phy>;
+	phy-names = "pcie-phy0", "pcie-phy1";
+};
+
+&pcie1_ep {
+	num-lanes = <2>;
+	phys = <&pcie1_phy>, <&pcie2_phy>;
+	phy-names = "pcie-phy0", "pcie-phy1";
+};
diff --git a/arch/arm/boot/dts/dra7xx-clocks.dtsi b/arch/arm/boot/dts/dra7xx-clocks.dtsi
index e62b62875cba..69562cdbeada 100644
--- a/arch/arm/boot/dts/dra7xx-clocks.dtsi
+++ b/arch/arm/boot/dts/dra7xx-clocks.dtsi
@@ -11,25 +11,25 @@
 	atl_clkin0_ck: atl_clkin0_ck {
 		#clock-cells = <0>;
 		compatible = "ti,dra7-atl-clock";
-		clocks = <&atl_gfclk_mux>;
+		clocks = <&atl_clkctrl DRA7_ATL_CLKCTRL 26>;
 	};
 
 	atl_clkin1_ck: atl_clkin1_ck {
 		#clock-cells = <0>;
 		compatible = "ti,dra7-atl-clock";
-		clocks = <&atl_gfclk_mux>;
+		clocks = <&atl_clkctrl DRA7_ATL_CLKCTRL 26>;
 	};
 
 	atl_clkin2_ck: atl_clkin2_ck {
 		#clock-cells = <0>;
 		compatible = "ti,dra7-atl-clock";
-		clocks = <&atl_gfclk_mux>;
+		clocks = <&atl_clkctrl DRA7_ATL_CLKCTRL 26>;
 	};
 
 	atl_clkin3_ck: atl_clkin3_ck {
 		#clock-cells = <0>;
 		compatible = "ti,dra7-atl-clock";
-		clocks = <&atl_gfclk_mux>;
+		clocks = <&atl_clkctrl DRA7_ATL_CLKCTRL 26>;
 	};
 
 	hdmi_clkin_ck: hdmi_clkin_ck {
@@ -809,70 +809,6 @@
 		assigned-clock-parents = <&dpll_core_h22x2_ck>;
 	};
 
-	mcasp1_ahclkr_mux: mcasp1_ahclkr_mux@550 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <28>;
-		reg = <0x0550>;
-	};
-
-	mcasp1_ahclkx_mux: mcasp1_ahclkx_mux@550 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0550>;
-	};
-
-	mcasp1_aux_gfclk_mux: mcasp1_aux_gfclk_mux@550 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <22>;
-		reg = <0x0550>;
-	};
-
-	timer5_gfclk_mux: timer5_gfclk_mux@558 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>, <&clkoutmux0_clk_mux>;
-		ti,bit-shift = <24>;
-		reg = <0x0558>;
-	};
-
-	timer6_gfclk_mux: timer6_gfclk_mux@560 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>, <&clkoutmux0_clk_mux>;
-		ti,bit-shift = <24>;
-		reg = <0x0560>;
-	};
-
-	timer7_gfclk_mux: timer7_gfclk_mux@568 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>, <&clkoutmux0_clk_mux>;
-		ti,bit-shift = <24>;
-		reg = <0x0568>;
-	};
-
-	timer8_gfclk_mux: timer8_gfclk_mux@570 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>, <&clkoutmux0_clk_mux>;
-		ti,bit-shift = <24>;
-		reg = <0x0570>;
-	};
-
-	uart6_gfclk_mux: uart6_gfclk_mux@580 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0580>;
-	};
-
 	dummy_ck: dummy_ck {
 		#clock-cells = <0>;
 		compatible = "fixed-clock";
@@ -1188,39 +1124,8 @@
 		clocks = <&sys_clkin1>, <&abe_lp_clk_div>;
 		reg = <0x0108>;
 	};
-
-	gpio1_dbclk: gpio1_dbclk@1838 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1838>;
-	};
-
-	dcan1_sys_clk_mux: dcan1_sys_clk_mux@1888 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin1>, <&sys_clkin2>;
-		ti,bit-shift = <24>;
-		reg = <0x1888>;
-	};
-
-	timer1_gfclk_mux: timer1_gfclk_mux@1840 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1840>;
-	};
-
-	uart10_gfclk_mux: uart10_gfclk_mux@1880 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1880>;
-	};
 };
+
 &cm_core_clocks {
 	dpll_pcie_ref_ck: dpll_pcie_ref_ck@200 {
 		#clock-cells = <0>;
@@ -1255,22 +1160,6 @@
 		reg = <0x021c>, <0x0220>;
 	};
 
-	optfclk_pciephy1_32khz: optfclk_pciephy1_32khz@4a0093b0 {
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		#clock-cells = <0>;
-		reg = <0x13b0>;
-		ti,bit-shift = <8>;
-	};
-
-	optfclk_pciephy2_32khz: optfclk_pciephy2_32khz@4a0093b8 {
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		#clock-cells = <0>;
-		reg = <0x13b8>;
-		ti,bit-shift = <8>;
-	};
-
 	optfclk_pciephy_div: optfclk_pciephy_div@4a00821c {
 		compatible = "ti,divider-clock";
 		clocks = <&apll_pcie_ck>;
@@ -1281,38 +1170,6 @@
 		ti,max-div = <2>;
 	};
 
-	optfclk_pciephy1_clk: optfclk_pciephy1_clk@4a0093b0 {
-		compatible = "ti,gate-clock";
-		clocks = <&apll_pcie_ck>;
-		#clock-cells = <0>;
-		reg = <0x13b0>;
-		ti,bit-shift = <9>;
-	};
-
-	optfclk_pciephy2_clk: optfclk_pciephy2_clk@4a0093b8 {
-		compatible = "ti,gate-clock";
-		clocks = <&apll_pcie_ck>;
-		#clock-cells = <0>;
-		reg = <0x13b8>;
-		ti,bit-shift = <9>;
-	};
-
-	optfclk_pciephy1_div_clk: optfclk_pciephy1_div_clk@4a0093b0 {
-		compatible = "ti,gate-clock";
-		clocks = <&optfclk_pciephy_div>;
-		#clock-cells = <0>;
-		reg = <0x13b0>;
-		ti,bit-shift = <10>;
-	};
-
-	optfclk_pciephy2_div_clk: optfclk_pciephy2_div_clk@4a0093b8 {
-		compatible = "ti,gate-clock";
-		clocks = <&optfclk_pciephy_div>;
-		#clock-cells = <0>;
-		reg = <0x13b8>;
-		ti,bit-shift = <10>;
-	};
-
 	apll_pcie_clkvcoldo: apll_pcie_clkvcoldo {
 		#clock-cells = <0>;
 		compatible = "fixed-factor-clock";
@@ -1541,167 +1398,6 @@
 		reg = <0x06c0>;
 	};
 
-	dss_32khz_clk: dss_32khz_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <11>;
-		reg = <0x1120>;
-	};
-
-	dss_48mhz_clk: dss_48mhz_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_48m_fclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1120>;
-	};
-
-	dss_dss_clk: dss_dss_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_per_h12x2_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1120>;
-		ti,set-rate-parent;
-	};
-
-	dss_hdmi_clk: dss_hdmi_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&hdmi_dpll_clk_mux>;
-		ti,bit-shift = <10>;
-		reg = <0x1120>;
-	};
-
-	dss_video1_clk: dss_video1_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&video1_dpll_clk_mux>;
-		ti,bit-shift = <12>;
-		reg = <0x1120>;
-	};
-
-	dss_video2_clk: dss_video2_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&video2_dpll_clk_mux>;
-		ti,bit-shift = <13>;
-		reg = <0x1120>;
-	};
-
-	gpio2_dbclk: gpio2_dbclk@1760 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1760>;
-	};
-
-	gpio3_dbclk: gpio3_dbclk@1768 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1768>;
-	};
-
-	gpio4_dbclk: gpio4_dbclk@1770 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1770>;
-	};
-
-	gpio5_dbclk: gpio5_dbclk@1778 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1778>;
-	};
-
-	gpio6_dbclk: gpio6_dbclk@1780 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1780>;
-	};
-
-	gpio7_dbclk: gpio7_dbclk@1810 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1810>;
-	};
-
-	gpio8_dbclk: gpio8_dbclk@1818 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1818>;
-	};
-
-	mmc1_clk32k: mmc1_clk32k@1328 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1328>;
-	};
-
-	mmc2_clk32k: mmc2_clk32k@1330 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1330>;
-	};
-
-	mmc3_clk32k: mmc3_clk32k@1820 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1820>;
-	};
-
-	mmc4_clk32k: mmc4_clk32k@1828 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1828>;
-	};
-
-	sata_ref_clk: sata_ref_clk@1388 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_clkin1>;
-		ti,bit-shift = <8>;
-		reg = <0x1388>;
-	};
-
-	usb_otg_ss1_refclk960m: usb_otg_ss1_refclk960m@13f0 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_960m_gfclk>;
-		ti,bit-shift = <8>;
-		reg = <0x13f0>;
-	};
-
-	usb_otg_ss2_refclk960m: usb_otg_ss2_refclk960m@1340 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_960m_gfclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1340>;
-	};
-
 	usb_phy1_always_on_clk32k: usb_phy1_always_on_clk32k@640 {
 		#clock-cells = <0>;
 		compatible = "ti,gate-clock";
@@ -1726,38 +1422,6 @@
 		reg = <0x0698>;
 	};
 
-	atl_dpll_clk_mux: atl_dpll_clk_mux@c00 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_32k_ck>, <&video1_clkin_ck>, <&video2_clkin_ck>, <&hdmi_clkin_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0c00>;
-	};
-
-	atl_gfclk_mux: atl_gfclk_mux@c00 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&l3_iclk_div>, <&dpll_abe_m2_ck>, <&atl_dpll_clk_mux>;
-		ti,bit-shift = <26>;
-		reg = <0x0c00>;
-	};
-
-	rmii_50mhz_clk_mux: rmii_50mhz_clk_mux@13d0 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dpll_gmac_h11x2_ck>, <&rmii_clk_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x13d0>;
-	};
-
-	gmac_rft_clk_mux: gmac_rft_clk_mux@13d0 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&video1_clkin_ck>, <&video2_clkin_ck>, <&dpll_abe_m2_ck>, <&hdmi_clkin_ck>, <&l3_iclk_div>;
-		ti,bit-shift = <25>;
-		reg = <0x13d0>;
-	};
-
 	gpu_core_gclk_mux: gpu_core_gclk_mux@1220 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
@@ -1787,362 +1451,6 @@
 		ti,dividers = <8>, <16>, <32>;
 	};
 
-	mcasp2_ahclkr_mux: mcasp2_ahclkr_mux@1860 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <28>;
-		reg = <0x1860>;
-	};
-
-	mcasp2_ahclkx_mux: mcasp2_ahclkx_mux@1860 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1860>;
-	};
-
-	mcasp2_aux_gfclk_mux: mcasp2_aux_gfclk_mux@1860 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <22>;
-		reg = <0x1860>;
-	};
-
-	mcasp3_ahclkx_mux: mcasp3_ahclkx_mux@1868 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1868>;
-		assigned-clocks = <&mcasp3_ahclkx_mux>;
-		assigned-clock-parents = <&abe_24m_fclk>;
-	};
-
-	mcasp3_aux_gfclk_mux: mcasp3_aux_gfclk_mux@1868 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <22>;
-		reg = <0x1868>;
-	};
-
-	mcasp4_ahclkx_mux: mcasp4_ahclkx_mux@1898 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1898>;
-	};
-
-	mcasp4_aux_gfclk_mux: mcasp4_aux_gfclk_mux@1898 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <22>;
-		reg = <0x1898>;
-	};
-
-	mcasp5_ahclkx_mux: mcasp5_ahclkx_mux@1878 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1878>;
-	};
-
-	mcasp5_aux_gfclk_mux: mcasp5_aux_gfclk_mux@1878 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <22>;
-		reg = <0x1878>;
-	};
-
-	mcasp6_ahclkx_mux: mcasp6_ahclkx_mux@1904 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1904>;
-	};
-
-	mcasp6_aux_gfclk_mux: mcasp6_aux_gfclk_mux@1904 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <22>;
-		reg = <0x1904>;
-	};
-
-	mcasp7_ahclkx_mux: mcasp7_ahclkx_mux@1908 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1908>;
-	};
-
-	mcasp7_aux_gfclk_mux: mcasp7_aux_gfclk_mux@1908 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <22>;
-		reg = <0x1908>;
-	};
-
-	mcasp8_ahclkx_mux: mcasp8_ahclkx_mux@1890 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&abe_sys_clk_div>, <&func_24m_clk>, <&atl_clkin3_ck>, <&atl_clkin2_ck>, <&atl_clkin1_ck>, <&atl_clkin0_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&mlb_clk>, <&mlbp_clk>;
-		ti,bit-shift = <22>;
-		reg = <0x1890>;
-	};
-
-	mcasp8_aux_gfclk_mux: mcasp8_aux_gfclk_mux@1890 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&per_abe_x1_gfclk2_div>, <&video1_clk2_div>, <&video2_clk2_div>, <&hdmi_clk2_div>;
-		ti,bit-shift = <24>;
-		reg = <0x1890>;
-	};
-
-	mmc1_fclk_mux: mmc1_fclk_mux@1328 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_128m_clk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1328>;
-	};
-
-	mmc1_fclk_div: mmc1_fclk_div@1328 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&mmc1_fclk_mux>;
-		ti,bit-shift = <25>;
-		ti,max-div = <4>;
-		reg = <0x1328>;
-		ti,index-power-of-two;
-	};
-
-	mmc2_fclk_mux: mmc2_fclk_mux@1330 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_128m_clk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1330>;
-	};
-
-	mmc2_fclk_div: mmc2_fclk_div@1330 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&mmc2_fclk_mux>;
-		ti,bit-shift = <25>;
-		ti,max-div = <4>;
-		reg = <0x1330>;
-		ti,index-power-of-two;
-	};
-
-	mmc3_gfclk_mux: mmc3_gfclk_mux@1820 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1820>;
-	};
-
-	mmc3_gfclk_div: mmc3_gfclk_div@1820 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&mmc3_gfclk_mux>;
-		ti,bit-shift = <25>;
-		ti,max-div = <4>;
-		reg = <0x1820>;
-		ti,index-power-of-two;
-	};
-
-	mmc4_gfclk_mux: mmc4_gfclk_mux@1828 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1828>;
-	};
-
-	mmc4_gfclk_div: mmc4_gfclk_div@1828 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&mmc4_gfclk_mux>;
-		ti,bit-shift = <25>;
-		ti,max-div = <4>;
-		reg = <0x1828>;
-		ti,index-power-of-two;
-	};
-
-	qspi_gfclk_mux: qspi_gfclk_mux@1838 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_128m_clk>, <&dpll_per_h13x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1838>;
-	};
-
-	qspi_gfclk_div: qspi_gfclk_div@1838 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&qspi_gfclk_mux>;
-		ti,bit-shift = <25>;
-		ti,max-div = <4>;
-		reg = <0x1838>;
-		ti,index-power-of-two;
-	};
-
-	timer10_gfclk_mux: timer10_gfclk_mux@1728 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1728>;
-	};
-
-	timer11_gfclk_mux: timer11_gfclk_mux@1730 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1730>;
-	};
-
-	timer13_gfclk_mux: timer13_gfclk_mux@17c8 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x17c8>;
-	};
-
-	timer14_gfclk_mux: timer14_gfclk_mux@17d0 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x17d0>;
-	};
-
-	timer15_gfclk_mux: timer15_gfclk_mux@17d8 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x17d8>;
-	};
-
-	timer16_gfclk_mux: timer16_gfclk_mux@1830 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1830>;
-	};
-
-	timer2_gfclk_mux: timer2_gfclk_mux@1738 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1738>;
-	};
-
-	timer3_gfclk_mux: timer3_gfclk_mux@1740 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1740>;
-	};
-
-	timer4_gfclk_mux: timer4_gfclk_mux@1748 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1748>;
-	};
-
-	timer9_gfclk_mux: timer9_gfclk_mux@1750 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&timer_sys_clk_div>, <&sys_32k_ck>, <&sys_clkin2>, <&ref_clkin0_ck>, <&ref_clkin1_ck>, <&ref_clkin2_ck>, <&ref_clkin3_ck>, <&abe_giclk_div>, <&video1_div_clk>, <&video2_div_clk>, <&hdmi_div_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x1750>;
-	};
-
-	uart1_gfclk_mux: uart1_gfclk_mux@1840 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1840>;
-	};
-
-	uart2_gfclk_mux: uart2_gfclk_mux@1848 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1848>;
-	};
-
-	uart3_gfclk_mux: uart3_gfclk_mux@1850 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1850>;
-	};
-
-	uart4_gfclk_mux: uart4_gfclk_mux@1858 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1858>;
-	};
-
-	uart5_gfclk_mux: uart5_gfclk_mux@1870 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1870>;
-	};
-
-	uart7_gfclk_mux: uart7_gfclk_mux@18d0 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x18d0>;
-	};
-
-	uart8_gfclk_mux: uart8_gfclk_mux@18e0 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x18e0>;
-	};
-
-	uart9_gfclk_mux: uart9_gfclk_mux@18e8 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_48m_fclk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x18e8>;
-	};
-
 	vip1_gclk_mux: vip1_gclk_mux@1020 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
@@ -2216,3 +1524,210 @@
 		reg = <0x6c4>;
 	};
 };
+
+&cm_core_aon {
+	mpu_cm: mpu_cm@300 {
+		compatible = "ti,omap4-cm";
+		reg = <0x300 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x300 0x100>;
+
+		mpu_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	ipu_cm: ipu_cm@500 {
+		compatible = "ti,omap4-cm";
+		reg = <0x500 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x500 0x100>;
+
+		ipu_clkctrl: clk@40 {
+			compatible = "ti,clkctrl";
+			reg = <0x40 0x44>;
+			#clock-cells = <2>;
+		};
+	};
+
+	rtc_cm: rtc_cm@700 {
+		compatible = "ti,omap4-cm";
+		reg = <0x700 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x700 0x100>;
+
+		rtc_clkctrl: clk@40 {
+			compatible = "ti,clkctrl";
+			reg = <0x40 0x8>;
+			#clock-cells = <2>;
+		};
+	};
+
+};
+
+&cm_core {
+	coreaon_cm: coreaon_cm@600 {
+		compatible = "ti,omap4-cm";
+		reg = <0x600 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x600 0x100>;
+
+		coreaon_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x1c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3main1_cm: l3main1_cm@700 {
+		compatible = "ti,omap4-cm";
+		reg = <0x700 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x700 0x100>;
+
+		l3main1_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x74>;
+			#clock-cells = <2>;
+		};
+	};
+
+	dma_cm: dma_cm@a00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xa00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xa00 0x100>;
+
+		dma_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	emif_cm: emif_cm@b00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xb00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xb00 0x100>;
+
+		emif_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	atl_cm: atl_cm@c00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xc00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xc00 0x100>;
+
+		atl_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4cfg_cm: l4cfg_cm@d00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xd00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xd00 0x100>;
+
+		l4cfg_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x84>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3instr_cm: l3instr_cm@e00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xe00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xe00 0x100>;
+
+		l3instr_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xc>;
+			#clock-cells = <2>;
+		};
+	};
+
+	dss_cm: dss_cm@1100 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1100 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1100 0x100>;
+
+		dss_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x14>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3init_cm: l3init_cm@1300 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1300 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1300 0x100>;
+
+		l3init_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xd4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4per_cm: l4per_cm@1700 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1700 0x300>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1700 0x300>;
+
+		l4per_clkctrl: clk@0 {
+			compatible = "ti,clkctrl";
+			reg = <0x0 0x20c>;
+			#clock-cells = <2>;
+
+			assigned-clocks = <&l4per_clkctrl DRA7_MCASP3_CLKCTRL 24>;
+			assigned-clock-parents = <&abe_24m_fclk>;
+		};
+	};
+
+};
+
+&prm {
+	wkupaon_cm: wkupaon_cm@1800 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1800 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1800 0x100>;
+
+		wkupaon_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x6c>;
+			#clock-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/efm32gg-dk3750.dts b/arch/arm/boot/dts/efm32gg-dk3750.dts
index c9f3ea29cf9e..adfa559a488b 100644
--- a/arch/arm/boot/dts/efm32gg-dk3750.dts
+++ b/arch/arm/boot/dts/efm32gg-dk3750.dts
@@ -37,7 +37,7 @@
 			};
 
 			eeprom@50 {
-				compatible = "microchip,24c02";
+				compatible = "microchip,24c02", "atmel,24c02";
 				reg = <0x50>;
 				pagesize = <16>;
 			};
diff --git a/arch/arm/boot/dts/ethernut5.dts b/arch/arm/boot/dts/ethernut5.dts
index 123b203be06c..b554f85add41 100644
--- a/arch/arm/boot/dts/ethernut5.dts
+++ b/arch/arm/boot/dts/ethernut5.dts
@@ -36,6 +36,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usart0: serial@fffb0000 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/exynos3250-artik5-eval.dts b/arch/arm/boot/dts/exynos3250-artik5-eval.dts
index 4cbfa09c6c4e..20446a846a98 100644
--- a/arch/arm/boot/dts/exynos3250-artik5-eval.dts
+++ b/arch/arm/boot/dts/exynos3250-artik5-eval.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos3250 based ARTIK5 evaluation board device tree source
  *
@@ -6,10 +7,6 @@
  *
  * Device tree source file for Samsung's ARTIK5 evaluation board
  * which is based on Samsung Exynos3250 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos3250-artik5.dtsi b/arch/arm/boot/dts/exynos3250-artik5.dtsi
index 152e0291d0da..0aa577fe9f95 100644
--- a/arch/arm/boot/dts/exynos3250-artik5.dtsi
+++ b/arch/arm/boot/dts/exynos3250-artik5.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos3250 based ARTIK5 module device tree source
  *
@@ -6,10 +7,6 @@
  *
  * Device tree source file for Samsung's ARTIK5 module which is based on
  * Samsung Exynos3250 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "exynos3250.dtsi"
diff --git a/arch/arm/boot/dts/exynos3250-monk.dts b/arch/arm/boot/dts/exynos3250-monk.dts
index 029eb18590cf..6ffedf4ed9f2 100644
--- a/arch/arm/boot/dts/exynos3250-monk.dts
+++ b/arch/arm/boot/dts/exynos3250-monk.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos3250 based Monk board device tree source
  *
@@ -6,10 +7,6 @@
  *
  * Device tree source file for Samsung's Monk board which is based on
  * Samsung Exynos3250 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos3250-pinctrl.dtsi b/arch/arm/boot/dts/exynos3250-pinctrl.dtsi
index a149f148e659..dff3c6e3aa1f 100644
--- a/arch/arm/boot/dts/exynos3250-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos3250-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos3250 SoCs pin-mux and pin-config device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Samsung's Exynos3250 SoCs pin-mux and pin-config optiosn are listed as device
  * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/pinctrl/samsung.h>
 
diff --git a/arch/arm/boot/dts/exynos3250-rinato.dts b/arch/arm/boot/dts/exynos3250-rinato.dts
index 3743df4de390..2c3460781cc6 100644
--- a/arch/arm/boot/dts/exynos3250-rinato.dts
+++ b/arch/arm/boot/dts/exynos3250-rinato.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos3250 based Rinato board device tree source
  *
@@ -6,10 +7,6 @@
  *
  * Device tree source file for Samsung's Rinato board which is based on
  * Samsung Exynos3250 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos3250.dtsi b/arch/arm/boot/dts/exynos3250.dtsi
index 2bd3872221a1..b8fb94f5daa8 100644
--- a/arch/arm/boot/dts/exynos3250.dtsi
+++ b/arch/arm/boot/dts/exynos3250.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos3250 SoC device tree source
  *
@@ -11,10 +12,6 @@
  * Note: This file does not include device nodes for all the controllers in
  * Exynos3250 SoC. As device tree coverage for Exynos3250 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "exynos4-cpu-thermal.dtsi"
@@ -164,31 +161,31 @@
 			syscon = <&pmu_system_controller>;
 		};
 
-		pd_cam: cam-power-domain@10023C00 {
+		pd_cam: cam-power-domain@10023c00 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10023C00 0x20>;
 			#power-domain-cells = <0>;
 		};
 
-		pd_mfc: mfc-power-domain@10023C40 {
+		pd_mfc: mfc-power-domain@10023c40 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10023C40 0x20>;
 			#power-domain-cells = <0>;
 		};
 
-		pd_g3d: g3d-power-domain@10023C60 {
+		pd_g3d: g3d-power-domain@10023c60 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10023C60 0x20>;
 			#power-domain-cells = <0>;
 		};
 
-		pd_lcd0: lcd0-power-domain@10023C80 {
+		pd_lcd0: lcd0-power-domain@10023c80 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10023C80 0x20>;
 			#power-domain-cells = <0>;
 		};
 
-		pd_isp: isp-power-domain@10023CA0 {
+		pd_isp: isp-power-domain@10023ca0 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10023CA0 0x20>;
 			#power-domain-cells = <0>;
@@ -204,7 +201,7 @@
 						 <&cmu CLK_FIN_PLL>;
 		};
 
-		cmu_dmc: clock-controller@105C0000 {
+		cmu_dmc: clock-controller@105c0000 {
 			compatible = "samsung,exynos3250-cmu-dmc";
 			reg = <0x105C0000 0x2000>;
 			#clock-cells = <1>;
@@ -219,7 +216,7 @@
 			status = "disabled";
 		};
 
-		tmu: tmu@100C0000 {
+		tmu: tmu@100c0000 {
 			compatible = "samsung,exynos3250-tmu";
 			reg = <0x100C0000 0x100>;
 			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
@@ -287,7 +284,7 @@
 			status = "disabled";
 		};
 
-		sysmmu_jpeg: sysmmu@11A60000 {
+		sysmmu_jpeg: sysmmu@11a60000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11a60000 0x1000>;
 			interrupts = <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>,
@@ -313,7 +310,7 @@
 			status = "disabled";
 		};
 
-		dsi_0: dsi@11C80000 {
+		dsi_0: dsi@11c80000 {
 			compatible = "samsung,exynos3250-mipi-dsi";
 			reg = <0x11C80000 0x10000>;
 			interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
@@ -328,7 +325,7 @@
 			status = "disabled";
 		};
 
-		sysmmu_fimd0: sysmmu@11E20000 {
+		sysmmu_fimd0: sysmmu@11e20000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11e20000 0x1000>;
 			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>,
@@ -386,7 +383,7 @@
 			status = "disabled";
 		};
 
-		exynos_usbphy: exynos-usbphy@125B0000 {
+		exynos_usbphy: exynos-usbphy@125b0000 {
 			compatible = "samsung,exynos3250-usb2-phy";
 			reg = <0x125B0000 0x100>;
 			samsung,pmureg-phandle = <&pmu_system_controller>;
@@ -425,7 +422,7 @@
 			};
 		};
 
-		adc: adc@126C0000 {
+		adc: adc@126c0000 {
 			compatible = "samsung,exynos3250-adc",
 				     "samsung,exynos-adc-v2";
 			reg = <0x126C0000 0x100>;
@@ -544,7 +541,7 @@
 			status = "disabled";
 		};
 
-		i2c_4: i2c@138A0000 {
+		i2c_4: i2c@138a0000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
@@ -557,7 +554,7 @@
 			status = "disabled";
 		};
 
-		i2c_5: i2c@138B0000 {
+		i2c_5: i2c@138b0000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
@@ -570,7 +567,7 @@
 			status = "disabled";
 		};
 
-		i2c_6: i2c@138C0000 {
+		i2c_6: i2c@138c0000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
@@ -583,7 +580,7 @@
 			status = "disabled";
 		};
 
-		i2c_7: i2c@138D0000 {
+		i2c_7: i2c@138d0000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "samsung,s3c2440-i2c";
@@ -641,7 +638,7 @@
 			status = "disabled";
 		};
 
-		pwm: pwm@139D0000 {
+		pwm: pwm@139d0000 {
 			compatible = "samsung,exynos4210-pwm";
 			reg = <0x139D0000 0x1000>;
 			interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
diff --git a/arch/arm/boot/dts/exynos4-cpu-thermal.dtsi b/arch/arm/boot/dts/exynos4-cpu-thermal.dtsi
index 735cb2f10817..021d9fc1b492 100644
--- a/arch/arm/boot/dts/exynos4-cpu-thermal.dtsi
+++ b/arch/arm/boot/dts/exynos4-cpu-thermal.dtsi
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos4 thermal zone
  *
  * Copyright (c) 2014 Lukasz Majewski <l.majewski@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 #include <dt-bindings/thermal/thermal.h>
diff --git a/arch/arm/boot/dts/exynos4.dtsi b/arch/arm/boot/dts/exynos4.dtsi
index 4768b086ed67..6d59cc827649 100644
--- a/arch/arm/boot/dts/exynos4.dtsi
+++ b/arch/arm/boot/dts/exynos4.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4 SoC series common device tree source
  *
@@ -13,10 +14,6 @@
  * Note: This file does not include device nodes for all the controllers in
  * Exynos4 SoCs. As device tree coverage for Exynos4 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/clock/exynos4.h>
@@ -101,28 +98,28 @@
 		syscon = <&pmu_system_controller>;
 	};
 
-	pd_mfc: mfc-power-domain@10023C40 {
+	pd_mfc: mfc-power-domain@10023c40 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023C40 0x20>;
 		#power-domain-cells = <0>;
 		label = "MFC";
 	};
 
-	pd_g3d: g3d-power-domain@10023C60 {
+	pd_g3d: g3d-power-domain@10023c60 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023C60 0x20>;
 		#power-domain-cells = <0>;
 		label = "G3D";
 	};
 
-	pd_lcd0: lcd0-power-domain@10023C80 {
+	pd_lcd0: lcd0-power-domain@10023c80 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023C80 0x20>;
 		#power-domain-cells = <0>;
 		label = "LCD0";
 	};
 
-	pd_tv: tv-power-domain@10023C20 {
+	pd_tv: tv-power-domain@10023c20 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023C20 0x20>;
 		#power-domain-cells = <0>;
@@ -130,21 +127,21 @@
 		label = "TV";
 	};
 
-	pd_cam: cam-power-domain@10023C00 {
+	pd_cam: cam-power-domain@10023c00 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023C00 0x20>;
 		#power-domain-cells = <0>;
 		label = "CAM";
 	};
 
-	pd_gps: gps-power-domain@10023CE0 {
+	pd_gps: gps-power-domain@10023ce0 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023CE0 0x20>;
 		#power-domain-cells = <0>;
 		label = "GPS";
 	};
 
-	pd_gps_alive: gps-alive-power-domain@10023D00 {
+	pd_gps_alive: gps-alive-power-domain@10023d00 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023D00 0x20>;
 		#power-domain-cells = <0>;
@@ -184,7 +181,7 @@
 		interrupt-parent = <&gic>;
 	};
 
-	dsi_0: dsi@11C80000 {
+	dsi_0: dsi@11c80000 {
 		compatible = "samsung,exynos4210-mipi-dsi";
 		reg = <0x11C80000 0x10000>;
 		interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
@@ -297,7 +294,7 @@
 		status = "disabled";
 	};
 
-	keypad: keypad@100A0000 {
+	keypad: keypad@100a0000 {
 		compatible = "samsung,s5pv210-keypad";
 		reg = <0x100A0000 0x100>;
 		interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
@@ -342,7 +339,7 @@
 		status = "disabled";
 	};
 
-	exynos_usbphy: exynos-usbphy@125B0000 {
+	exynos_usbphy: exynos-usbphy@125b0000 {
 		compatible = "samsung,exynos4210-usb2-phy";
 		reg = <0x125B0000 0x100>;
 		samsung,pmureg-phandle = <&pmu_system_controller>;
@@ -538,7 +535,7 @@
 		status = "disabled";
 	};
 
-	i2c_4: i2c@138A0000 {
+	i2c_4: i2c@138a0000 {
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
@@ -551,7 +548,7 @@
 		status = "disabled";
 	};
 
-	i2c_5: i2c@138B0000 {
+	i2c_5: i2c@138b0000 {
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
@@ -564,7 +561,7 @@
 		status = "disabled";
 	};
 
-	i2c_6: i2c@138C0000 {
+	i2c_6: i2c@138c0000 {
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
@@ -577,7 +574,7 @@
 		status = "disabled";
 	};
 
-	i2c_7: i2c@138D0000 {
+	i2c_7: i2c@138d0000 {
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-i2c";
@@ -590,7 +587,7 @@
 		status = "disabled";
 	};
 
-	i2c_8: i2c@138E0000 {
+	i2c_8: i2c@138e0000 {
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "samsung,s3c2440-hdmiphy-i2c";
@@ -651,7 +648,7 @@
 		status = "disabled";
 	};
 
-	pwm: pwm@139D0000 {
+	pwm: pwm@139d0000 {
 		compatible = "samsung,exynos4210-pwm";
 		reg = <0x139D0000 0x1000>;
 		interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>,
@@ -720,7 +717,7 @@
 		status = "disabled";
 	};
 
-	tmu: tmu@100C0000 {
+	tmu: tmu@100c0000 {
 		#include "exynos4412-tmu-sensor-conf.dtsi"
 	};
 
@@ -743,7 +740,7 @@
 		iommus = <&sysmmu_rotator>;
 	};
 
-	hdmi: hdmi@12D00000 {
+	hdmi: hdmi@12d00000 {
 		compatible = "samsung,exynos4210-hdmi";
 		reg = <0x12D00000 0x70000>;
 		interrupts = <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>;
@@ -755,10 +752,11 @@
 		phy = <&hdmi_i2c_phy>;
 		power-domains = <&pd_tv>;
 		samsung,syscon-phandle = <&pmu_system_controller>;
+		#sound-dai-cells = <0>;
 		status = "disabled";
 	};
 
-	hdmicec: cec@100B0000 {
+	hdmicec: cec@100b0000 {
 		compatible = "samsung,s5p-cec";
 		reg = <0x100B0000 0x200>;
 		interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
@@ -771,7 +769,7 @@
 		status = "disabled";
 	};
 
-	mixer: mixer@12C10000 {
+	mixer: mixer@12c10000 {
 		compatible = "samsung,exynos4210-mixer";
 		interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
 		reg = <0x12C10000 0x2100>, <0x12c00000 0x300>;
@@ -910,7 +908,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_tv: sysmmu@12E20000 {
+	sysmmu_tv: sysmmu@12e20000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x12E20000 0x1000>;
 		interrupt-parent = <&combiner>;
@@ -921,7 +919,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc0: sysmmu@11A20000 {
+	sysmmu_fimc0: sysmmu@11a20000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x11A20000 0x1000>;
 		interrupt-parent = <&combiner>;
@@ -932,7 +930,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc1: sysmmu@11A30000 {
+	sysmmu_fimc1: sysmmu@11a30000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x11A30000 0x1000>;
 		interrupt-parent = <&combiner>;
@@ -943,7 +941,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc2: sysmmu@11A40000 {
+	sysmmu_fimc2: sysmmu@11a40000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x11A40000 0x1000>;
 		interrupt-parent = <&combiner>;
@@ -954,7 +952,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc3: sysmmu@11A50000 {
+	sysmmu_fimc3: sysmmu@11a50000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x11A50000 0x1000>;
 		interrupt-parent = <&combiner>;
@@ -965,7 +963,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_jpeg: sysmmu@11A60000 {
+	sysmmu_jpeg: sysmmu@11a60000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x11A60000 0x1000>;
 		interrupt-parent = <&combiner>;
@@ -976,7 +974,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_rotator: sysmmu@12A30000 {
+	sysmmu_rotator: sysmmu@12a30000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x12A30000 0x1000>;
 		interrupt-parent = <&combiner>;
@@ -986,7 +984,7 @@
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimd0: sysmmu@11E20000 {
+	sysmmu_fimd0: sysmmu@11e20000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x11E20000 0x1000>;
 		interrupt-parent = <&combiner>;
diff --git a/arch/arm/boot/dts/exynos4210-origen.dts b/arch/arm/boot/dts/exynos4210-origen.dts
index 084fcc5574ef..9a310e841d5d 100644
--- a/arch/arm/boot/dts/exynos4210-origen.dts
+++ b/arch/arm/boot/dts/exynos4210-origen.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4210 based Origen board device tree source
  *
@@ -8,11 +9,7 @@
  *
  * Device tree source file for Insignal's Origen board which is based on
  * Samsung's Exynos4210 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4210.dtsi"
diff --git a/arch/arm/boot/dts/exynos4210-pinctrl.dtsi b/arch/arm/boot/dts/exynos4210-pinctrl.dtsi
index 82c32d4d83d8..dbe6c052d8c1 100644
--- a/arch/arm/boot/dts/exynos4210-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos4210-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4210 SoC pin-mux and pin-config device tree source
  *
@@ -8,11 +9,7 @@
  *
  * Samsung's Exynos4210 SoC pin-mux and pin-config optiosn are listed as device
  * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/pinctrl/samsung.h>
 
diff --git a/arch/arm/boot/dts/exynos4210-smdkv310.dts b/arch/arm/boot/dts/exynos4210-smdkv310.dts
index 9c98a3724396..7a3e621edede 100644
--- a/arch/arm/boot/dts/exynos4210-smdkv310.dts
+++ b/arch/arm/boot/dts/exynos4210-smdkv310.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4210 based SMDKV310 board device tree source
  *
@@ -8,11 +9,7 @@
  *
  * Device tree source file for Samsung's SMDKV310 board which is based on
  * Samsung's Exynos4210 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4210.dtsi"
diff --git a/arch/arm/boot/dts/exynos4210-trats.dts b/arch/arm/boot/dts/exynos4210-trats.dts
index 7b6ab7265110..aaade17b140e 100644
--- a/arch/arm/boot/dts/exynos4210-trats.dts
+++ b/arch/arm/boot/dts/exynos4210-trats.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4210 based Trats board device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Device tree source file for Samsung's Trats board which is based on
  * Samsung's Exynos4210 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4210.dtsi"
diff --git a/arch/arm/boot/dts/exynos4210-universal_c210.dts b/arch/arm/boot/dts/exynos4210-universal_c210.dts
index acd2b2286ccb..21fff7cd3aa4 100644
--- a/arch/arm/boot/dts/exynos4210-universal_c210.dts
+++ b/arch/arm/boot/dts/exynos4210-universal_c210.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4210 based Universal C210 board device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Device tree source file for Samsung's Universal C210 board which is based on
  * Samsung's Exynos4210 rev0 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4210.dtsi"
diff --git a/arch/arm/boot/dts/exynos4210.dtsi b/arch/arm/boot/dts/exynos4210.dtsi
index 03dd61f64809..cc978cf28267 100644
--- a/arch/arm/boot/dts/exynos4210.dtsi
+++ b/arch/arm/boot/dts/exynos4210.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4210 SoC device tree source
  *
@@ -13,11 +14,7 @@
  * Note: This file does not include device nodes for all the controllers in
  * Exynos4210 SoC. As device tree coverage for Exynos4210 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include "exynos4.dtsi"
 #include "exynos4210-pinctrl.dtsi"
@@ -82,7 +79,7 @@
 		};
 	};
 
-	pd_lcd1: lcd1-power-domain@10023CA0 {
+	pd_lcd1: lcd1-power-domain@10023ca0 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023CA0 0x20>;
 		#power-domain-cells = <0>;
@@ -156,7 +153,7 @@
 		reg = <0x03860000 0x1000>;
 	};
 
-	tmu: tmu@100C0000 {
+	tmu: tmu@100c0000 {
 		compatible = "samsung,exynos4210-tmu";
 		interrupt-parent = <&combiner>;
 		reg = <0x100C0000 0x100>;
@@ -229,7 +226,7 @@
 		};
 	};
 
-	mixer: mixer@12C10000 {
+	mixer: mixer@12c10000 {
 		clock-names = "mixer", "hdmi", "sclk_hdmi", "vp", "mout_mixer",
 			"sclk_mixer";
 		clocks = <&clock CLK_MIXER>, <&clock CLK_HDMI>,
@@ -245,7 +242,7 @@
 		status = "disabled";
 	};
 
-	sysmmu_g2d: sysmmu@12A20000 {
+	sysmmu_g2d: sysmmu@12a20000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x12A20000 0x1000>;
 		interrupt-parent = <&combiner>;
diff --git a/arch/arm/boot/dts/exynos4412-itop-elite.dts b/arch/arm/boot/dts/exynos4412-itop-elite.dts
index d66093084dbb..a4cd4939fe9a 100644
--- a/arch/arm/boot/dts/exynos4412-itop-elite.dts
+++ b/arch/arm/boot/dts/exynos4412-itop-elite.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * TOPEET's Exynos4412 based itop board device tree source
  *
@@ -7,10 +8,6 @@
  *
  * Device tree source file for TOPEET iTop Exynos 4412 core board
  * which is based on Samsung's Exynos4412 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi b/arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi
index bda49b232f7b..ab7affab7f1c 100644
--- a/arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi
+++ b/arch/arm/boot/dts/exynos4412-itop-scp-core.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * TOPEET's Exynos4412 based itop board device tree source
  *
@@ -7,10 +8,6 @@
  *
  * Device tree source file for TOPEET iTop Exynos 4412 SCP package core
  * board which is based on Samsung's Exynos4412 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/clock/samsung,s2mps11.h>
diff --git a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi
index a21be71000c1..556ea78b8e32 100644
--- a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi
+++ b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi
@@ -1,10 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Common definition for Hardkernel's Exynos4412 based ODROID-X/X2/U2/U3 boards
  * device tree source
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
 */
 
 #include <dt-bindings/sound/samsung-i2s.h>
diff --git a/arch/arm/boot/dts/exynos4412-odroidu3.dts b/arch/arm/boot/dts/exynos4412-odroidu3.dts
index 44a4de08466b..bdcd4523cc1c 100644
--- a/arch/arm/boot/dts/exynos4412-odroidu3.dts
+++ b/arch/arm/boot/dts/exynos4412-odroidu3.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel's Exynos4412 based ODROID-U3 board device tree source
  *
@@ -5,11 +6,7 @@
  *
  * Device tree source file for Hardkernel's ODROID-U3 board which is based
  * on Samsung's Exynos4412 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4412-odroid-common.dtsi"
diff --git a/arch/arm/boot/dts/exynos4412-odroidx.dts b/arch/arm/boot/dts/exynos4412-odroidx.dts
index acf48a018e5e..2dff129bc2ad 100644
--- a/arch/arm/boot/dts/exynos4412-odroidx.dts
+++ b/arch/arm/boot/dts/exynos4412-odroidx.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel's Exynos4412 based ODROID-X board device tree source
  *
@@ -5,11 +6,7 @@
  *
  * Device tree source file for Hardkernel's ODROID-X board which is based
  * on Samsung's Exynos4412 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4412-odroid-common.dtsi"
diff --git a/arch/arm/boot/dts/exynos4412-odroidx2.dts b/arch/arm/boot/dts/exynos4412-odroidx2.dts
index d867b2ee95ca..f4b68c75c962 100644
--- a/arch/arm/boot/dts/exynos4412-odroidx2.dts
+++ b/arch/arm/boot/dts/exynos4412-odroidx2.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel's Exynos4412 based ODROID-X2 board device tree source
  *
@@ -5,11 +6,7 @@
  *
  * Device tree source file for Hardkernel's ODROID-X2 board which is based
  * on Samsung's Exynos4412 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include "exynos4412-odroidx.dts"
 #include "exynos4412-prime.dtsi"
diff --git a/arch/arm/boot/dts/exynos4412-origen.dts b/arch/arm/boot/dts/exynos4412-origen.dts
index b0b5ec7903a5..1514f0dbaff8 100644
--- a/arch/arm/boot/dts/exynos4412-origen.dts
+++ b/arch/arm/boot/dts/exynos4412-origen.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Insignal's Exynos4412 based Origen board device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Device tree source file for Insignal's Origen board which is based on
  * Samsung's Exynos4412 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4412.dtsi"
diff --git a/arch/arm/boot/dts/exynos4412-pinctrl.dtsi b/arch/arm/boot/dts/exynos4412-pinctrl.dtsi
index 4eebd4721a5f..e8dd5f2d976f 100644
--- a/arch/arm/boot/dts/exynos4412-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos4412-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4412 SoCs pin-mux and pin-config device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Samsung's Exynos4412 SoCs pin-mux and pin-config optiosn are listed as device
  * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/pinctrl/samsung.h>
 
@@ -925,7 +922,7 @@
 		};
 	};
 
-	pinctrl_3: pinctrl@106E0000 {
+	pinctrl_3: pinctrl@106e0000 {
 		gpv0: gpv0 {
 			gpio-controller;
 			#gpio-cells = <2>;
diff --git a/arch/arm/boot/dts/exynos4412-ppmu-common.dtsi b/arch/arm/boot/dts/exynos4412-ppmu-common.dtsi
index 16e4b77d8cb1..3a3b2fafefdd 100644
--- a/arch/arm/boot/dts/exynos4412-ppmu-common.dtsi
+++ b/arch/arm/boot/dts/exynos4412-ppmu-common.dtsi
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos4412 PPMU common device tree
  *
  * Copyright (C) 2015 Samsung Electronics
  * Author: Chanwoo Choi <cw00.choi@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 &ppmu_dmc0 {
diff --git a/arch/arm/boot/dts/exynos4412-prime.dtsi b/arch/arm/boot/dts/exynos4412-prime.dtsi
index a67bd953d754..8e7a7fb98124 100644
--- a/arch/arm/boot/dts/exynos4412-prime.dtsi
+++ b/arch/arm/boot/dts/exynos4412-prime.dtsi
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4412 Prime SoC device tree source
  *
  * Copyright (c) 2016 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /*
diff --git a/arch/arm/boot/dts/exynos4412-smdk4412.dts b/arch/arm/boot/dts/exynos4412-smdk4412.dts
index 7fcb43431b59..5c5c2887c14f 100644
--- a/arch/arm/boot/dts/exynos4412-smdk4412.dts
+++ b/arch/arm/boot/dts/exynos4412-smdk4412.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4412 based SMDK board device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Device tree source file for Samsung's SMDK4412 board which is based on
  * Samsung's Exynos4412 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos4412.dtsi"
diff --git a/arch/arm/boot/dts/exynos4412-trats2.dts b/arch/arm/boot/dts/exynos4412-trats2.dts
index 220cdf109405..f285790e8e04 100644
--- a/arch/arm/boot/dts/exynos4412-trats2.dts
+++ b/arch/arm/boot/dts/exynos4412-trats2.dts
@@ -300,6 +300,13 @@
 
 	};
 
+	wlan_pwrseq: sdhci3-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpj0 0 GPIO_ACTIVE_LOW>;
+		clocks = <&max77686 MAX77686_CLK_PMIC>;
+		clock-names = "ext_clock";
+	};
+
 	sound {
 		compatible = "samsung,trats2-audio";
 		samsung,i2s-controller = <&i2s0>;
@@ -454,7 +461,7 @@
 		reg = <0>;
 		vdd3-supply = <&lcd_vdd3_reg>;
 		vci-supply = <&ldo25_reg>;
-		reset-gpios = <&gpy4 5 GPIO_ACTIVE_HIGH>;
+		reset-gpios = <&gpf2 1 GPIO_ACTIVE_HIGH>;
 		power-on-delay= <50>;
 		reset-delay = <100>;
 		init-delay = <100>;
@@ -1350,6 +1357,26 @@
 	status = "okay";
 };
 
+&sdhci_3 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	non-removable;
+	bus-width = <4>;
+
+	mmc-pwrseq = <&wlan_pwrseq>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sd3_clk &sd3_cmd &sd3_bus4>;
+	status = "okay";
+
+	brcmf: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+		interrupt-parent = <&gpx2>;
+		interrupts = <5 IRQ_TYPE_NONE>;
+		interrupt-names = "host-wake";
+	};
+};
+
 &serial_0 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/exynos4412.dtsi b/arch/arm/boot/dts/exynos4412.dtsi
index b255ac55b1c1..e4ad2fc0329e 100644
--- a/arch/arm/boot/dts/exynos4412.dtsi
+++ b/arch/arm/boot/dts/exynos4412.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos4412 SoC device tree source
  *
@@ -11,11 +12,7 @@
  * Note: This file does not include device nodes for all the controllers in
  * Exynos4412 SoC. As device tree coverage for Exynos4412 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include "exynos4.dtsi"
 #include "exynos4412-pinctrl.dtsi"
@@ -38,7 +35,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu0: cpu@A00 {
+		cpu0: cpu@a00 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0xA00>;
@@ -50,21 +47,21 @@
 			#cooling-cells = <2>; /* min followed by max */
 		};
 
-		cpu@A01 {
+		cpu@a01 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0xA01>;
 			operating-points-v2 = <&cpu0_opp_table>;
 		};
 
-		cpu@A02 {
+		cpu@a02 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0xA02>;
 			operating-points-v2 = <&cpu0_opp_table>;
 		};
 
-		cpu@A03 {
+		cpu@a03 {
 			device_type = "cpu";
 			compatible = "arm,cortex-a9";
 			reg = <0xA03>;
@@ -168,7 +165,7 @@
 		};
 	};
 
-	pd_isp: isp-power-domain@10023CA0 {
+	pd_isp: isp-power-domain@10023ca0 {
 		compatible = "samsung,exynos4210-pd";
 		reg = <0x10023CA0 0x20>;
 		#power-domain-cells = <0>;
@@ -191,10 +188,19 @@
 
 	clock: clock-controller@10030000 {
 		compatible = "samsung,exynos4412-clock";
-		reg = <0x10030000 0x20000>;
+		reg = <0x10030000 0x18000>;
 		#clock-cells = <1>;
 	};
 
+	isp_clock: clock-controller@10048000 {
+		compatible = "samsung,exynos4412-isp-clock";
+		reg = <0x10048000 0x1000>;
+		#clock-cells = <1>;
+		power-domains = <&pd_isp>;
+		clocks = <&clock CLK_ACLK200>, <&clock CLK_ACLK400_MCUISP>;
+		clock-names = "aclk200", "aclk400_mcuisp";
+	};
+
 	mct@10050000 {
 		compatible = "samsung,exynos4412-mct";
 		reg = <0x10050000 0x800>;
@@ -224,7 +230,7 @@
 		samsung,syscon-phandle = <&pmu_system_controller>;
 	};
 
-	adc: adc@126C0000 {
+	adc: adc@126c0000 {
 		compatible = "samsung,exynos-adc-v1";
 		reg = <0x126C0000 0x100>;
 		interrupt-parent = <&combiner>;
@@ -257,18 +263,18 @@
 			reg = <0x12390000 0x1000>;
 			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_isp>;
-			clocks = <&clock CLK_FIMC_LITE0>;
+			clocks = <&isp_clock CLK_ISP_FIMC_LITE0>;
 			clock-names = "flite";
 			iommus = <&sysmmu_fimc_lite0>;
 			status = "disabled";
 		};
 
-		fimc_lite_1: fimc-lite@123A0000 {
+		fimc_lite_1: fimc-lite@123a0000 {
 			compatible = "samsung,exynos4212-fimc-lite";
 			reg = <0x123A0000 0x1000>;
 			interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_isp>;
-			clocks = <&clock CLK_FIMC_LITE1>;
+			clocks = <&isp_clock CLK_ISP_FIMC_LITE1>;
 			clock-names = "flite";
 			iommus = <&sysmmu_fimc_lite1>;
 			status = "disabled";
@@ -280,29 +286,35 @@
 			interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
 			power-domains = <&pd_isp>;
-			clocks = <&clock CLK_FIMC_LITE0>,
-				 <&clock CLK_FIMC_LITE1>, <&clock CLK_PPMUISPX>,
-				 <&clock CLK_PPMUISPMX>,
+			clocks = <&isp_clock CLK_ISP_FIMC_LITE0>,
+				 <&isp_clock CLK_ISP_FIMC_LITE1>,
+				 <&isp_clock CLK_ISP_PPMUISPX>,
+				 <&isp_clock CLK_ISP_PPMUISPMX>,
+				 <&isp_clock CLK_ISP_FIMC_ISP>,
+				 <&isp_clock CLK_ISP_FIMC_DRC>,
+				 <&isp_clock CLK_ISP_FIMC_FD>,
+				 <&isp_clock CLK_ISP_MCUISP>,
+				 <&isp_clock CLK_ISP_GICISP>,
+				 <&isp_clock CLK_ISP_MCUCTL_ISP>,
+				 <&isp_clock CLK_ISP_PWM_ISP>,
+				 <&isp_clock CLK_ISP_DIV_ISP0>,
+				 <&isp_clock CLK_ISP_DIV_ISP1>,
+				 <&isp_clock CLK_ISP_DIV_MCUISP0>,
+				 <&isp_clock CLK_ISP_DIV_MCUISP1>,
 				 <&clock CLK_MOUT_MPLL_USER_T>,
-				 <&clock CLK_FIMC_ISP>, <&clock CLK_FIMC_DRC>,
-				 <&clock CLK_FIMC_FD>, <&clock CLK_MCUISP>,
-				 <&clock CLK_GICISP>, <&clock CLK_MCUCTL_ISP>,
-				 <&clock CLK_PWM_ISP>,
-				 <&clock CLK_DIV_ISP0>, <&clock CLK_DIV_ISP1>,
-				 <&clock CLK_DIV_MCUISP0>,
-				 <&clock CLK_DIV_MCUISP1>,
-				 <&clock CLK_UART_ISP_SCLK>,
-				 <&clock CLK_ACLK200>, <&clock CLK_DIV_ACLK200>,
+				 <&clock CLK_ACLK200>,
 				 <&clock CLK_ACLK400_MCUISP>,
-				 <&clock CLK_DIV_ACLK400_MCUISP>;
+				 <&clock CLK_DIV_ACLK200>,
+				 <&clock CLK_DIV_ACLK400_MCUISP>,
+				 <&clock CLK_UART_ISP_SCLK>;
 			clock-names = "lite0", "lite1", "ppmuispx",
-				      "ppmuispmx", "mpll", "isp",
+				      "ppmuispmx", "isp",
 				      "drc", "fd", "mcuisp",
 				      "gicisp", "mcuctl_isp", "pwm_isp",
 				      "ispdiv0", "ispdiv1", "mcuispdiv0",
-				      "mcuispdiv1", "uart", "aclk200",
-				      "div_aclk200", "aclk400mcuisp",
-				      "div_aclk400mcuisp";
+				      "mcuispdiv1", "mpll", "aclk200",
+				      "aclk400mcuisp", "div_aclk200",
+				      "div_aclk400mcuisp", "uart";
 			iommus = <&sysmmu_fimc_isp>, <&sysmmu_fimc_drc>,
 				 <&sysmmu_fimc_fd>, <&sysmmu_fimc_mcuctl>;
 			iommu-names = "isp", "drc", "fd", "mcuctl";
@@ -318,7 +330,7 @@
 			i2c1_isp: i2c-isp@12140000 {
 				compatible = "samsung,exynos4212-i2c-isp";
 				reg = <0x12140000 0x100>;
-				clocks = <&clock CLK_I2C1_ISP>;
+				clocks = <&isp_clock CLK_ISP_I2C1_ISP>;
 				clock-names = "i2c_isp";
 				#address-cells = <1>;
 				#size-cells = <0>;
@@ -355,7 +367,7 @@
 		interrupts = <16 2>;
 		power-domains = <&pd_isp>;
 		clock-names = "sysmmu";
-		clocks = <&clock CLK_SMMU_ISP>;
+		clocks = <&isp_clock CLK_ISP_SMMU_ISP>;
 		#iommu-cells = <0>;
 	};
 
@@ -366,51 +378,53 @@
 		interrupts = <16 3>;
 		power-domains = <&pd_isp>;
 		clock-names = "sysmmu";
-		clocks = <&clock CLK_SMMU_DRC>;
+		clocks = <&isp_clock CLK_ISP_SMMU_DRC>;
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc_fd: sysmmu@122A0000 {
+	sysmmu_fimc_fd: sysmmu@122a0000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x122A0000 0x1000>;
 		interrupt-parent = <&combiner>;
 		interrupts = <16 4>;
 		power-domains = <&pd_isp>;
 		clock-names = "sysmmu";
-		clocks = <&clock CLK_SMMU_FD>;
+		clocks = <&isp_clock CLK_ISP_SMMU_FD>;
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc_mcuctl: sysmmu@122B0000 {
+	sysmmu_fimc_mcuctl: sysmmu@122b0000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x122B0000 0x1000>;
 		interrupt-parent = <&combiner>;
 		interrupts = <16 5>;
 		power-domains = <&pd_isp>;
 		clock-names = "sysmmu";
-		clocks = <&clock CLK_SMMU_ISPCX>;
+		clocks = <&isp_clock CLK_ISP_SMMU_ISPCX>;
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc_lite0: sysmmu@123B0000 {
+	sysmmu_fimc_lite0: sysmmu@123b0000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x123B0000 0x1000>;
 		interrupt-parent = <&combiner>;
 		interrupts = <16 0>;
 		power-domains = <&pd_isp>;
 		clock-names = "sysmmu", "master";
-		clocks = <&clock CLK_SMMU_LITE0>, <&clock CLK_FIMC_LITE0>;
+		clocks = <&isp_clock CLK_ISP_SMMU_LITE0>,
+			 <&isp_clock CLK_ISP_FIMC_LITE0>;
 		#iommu-cells = <0>;
 	};
 
-	sysmmu_fimc_lite1: sysmmu@123C0000 {
+	sysmmu_fimc_lite1: sysmmu@123c0000 {
 		compatible = "samsung,exynos-sysmmu";
 		reg = <0x123C0000 0x1000>;
 		interrupt-parent = <&combiner>;
 		interrupts = <16 1>;
 		power-domains = <&pd_isp>;
 		clock-names = "sysmmu", "master";
-		clocks = <&clock CLK_SMMU_LITE1>, <&clock CLK_FIMC_LITE1>;
+		clocks = <&isp_clock CLK_ISP_SMMU_LITE1>,
+			 <&isp_clock CLK_ISP_FIMC_LITE1>;
 		#iommu-cells = <0>;
 	};
 
diff --git a/arch/arm/boot/dts/exynos5.dtsi b/arch/arm/boot/dts/exynos5.dtsi
index 66d22521c976..f8d7de1144f1 100644
--- a/arch/arm/boot/dts/exynos5.dtsi
+++ b/arch/arm/boot/dts/exynos5.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5 SoC series common device tree source
  *
@@ -7,10 +8,6 @@
  * Samsung's Exynos5 SoC series device nodes are listed in this file. Particular
  * SoCs from Exynos5 series can include this file and provide values for SoCs
  * specfic bindings.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
@@ -106,31 +103,31 @@
 			reg = <0x10050000 0x5000>;
 		};
 
-		serial_0: serial@12C00000 {
+		serial_0: serial@12c00000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C00000 0x100>;
 			interrupts = <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		serial_1: serial@12C10000 {
+		serial_1: serial@12c10000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C10000 0x100>;
 			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		serial_2: serial@12C20000 {
+		serial_2: serial@12c20000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C20000 0x100>;
 			interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		serial_3: serial@12C30000 {
+		serial_3: serial@12c30000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C30000 0x100>;
 			interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		i2c_0: i2c@12C60000 {
+		i2c_0: i2c@12c60000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C60000 0x100>;
 			interrupts = <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>;
@@ -140,7 +137,7 @@
 			status = "disabled";
 		};
 
-		i2c_1: i2c@12C70000 {
+		i2c_1: i2c@12c70000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C70000 0x100>;
 			interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
@@ -150,7 +147,7 @@
 			status = "disabled";
 		};
 
-		i2c_2: i2c@12C80000 {
+		i2c_2: i2c@12c80000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C80000 0x100>;
 			interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
@@ -160,7 +157,7 @@
 			status = "disabled";
 		};
 
-		i2c_3: i2c@12C90000 {
+		i2c_3: i2c@12c90000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12C90000 0x100>;
 			interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
@@ -170,14 +167,14 @@
 			status = "disabled";
 		};
 
-		pwm: pwm@12DD0000 {
+		pwm: pwm@12dd0000 {
 			compatible = "samsung,exynos4210-pwm";
 			reg = <0x12DD0000 0x100>;
 			samsung,pwm-outputs = <0>, <1>, <2>, <3>;
 			#pwm-cells = <3>;
 		};
 
-		rtc: rtc@101E0000 {
+		rtc: rtc@101e0000 {
 			compatible = "samsung,s3c6410-rtc";
 			reg = <0x101E0000 0x100>;
 			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
@@ -195,7 +192,7 @@
 			status = "disabled";
 		};
 
-		dp: dp-controller@145B0000 {
+		dp: dp-controller@145b0000 {
 			compatible = "samsung,exynos5-dp";
 			reg = <0x145B0000 0x1000>;
 			interrupts = <10 3>;
@@ -204,5 +201,28 @@
 			#size-cells = <0>;
 			status = "disabled";
 		};
+
+		sss: sss@10830000 {
+			compatible = "samsung,exynos4210-secss";
+			reg = <0x10830000 0x300>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		prng: rng@10830400 {
+		      compatible = "samsung,exynos5250-prng";
+		      reg = <0x10830400 0x200>;
+		};
+
+		trng: rng@10830600 {
+		      compatible = "samsung,exynos5250-trng";
+		      reg = <0x10830600 0x100>;
+		};
+
+		g2d: g2d@10850000 {
+			compatible = "samsung,exynos5250-g2d";
+			reg = <0x10850000 0x1000>;
+			interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/exynos5250-arndale.dts b/arch/arm/boot/dts/exynos5250-arndale.dts
index 0efd678b8251..7a8a5c55701a 100644
--- a/arch/arm/boot/dts/exynos5250-arndale.dts
+++ b/arch/arm/boot/dts/exynos5250-arndale.dts
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5250 based Arndale board device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos5250-pinctrl.dtsi b/arch/arm/boot/dts/exynos5250-pinctrl.dtsi
index 1fd122db18e6..6ff6dea29d44 100644
--- a/arch/arm/boot/dts/exynos5250-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos5250-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5250 SoC pin-mux and pin-config device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Samsung's Exynos5250 SoC pin-mux and pin-config optiosn are listed as device
  * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/pinctrl/samsung.h>
 
diff --git a/arch/arm/boot/dts/exynos5250-smdk5250.dts b/arch/arm/boot/dts/exynos5250-smdk5250.dts
index 1e3f9627766c..d5e66189ed2a 100644
--- a/arch/arm/boot/dts/exynos5250-smdk5250.dts
+++ b/arch/arm/boot/dts/exynos5250-smdk5250.dts
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG SMDK5250 board device tree source
  *
  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos5250-snow-common.dtsi b/arch/arm/boot/dts/exynos5250-snow-common.dtsi
index 2e7175d2b1b8..59cf1b202849 100644
--- a/arch/arm/boot/dts/exynos5250-snow-common.dtsi
+++ b/arch/arm/boot/dts/exynos5250-snow-common.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Google Snow board device tree source
  *
  * Copyright (c) 2012 Google, Inc
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/gpio/gpio.h>
diff --git a/arch/arm/boot/dts/exynos5250-snow-rev5.dts b/arch/arm/boot/dts/exynos5250-snow-rev5.dts
index 90560c316f64..0348b1c49a69 100644
--- a/arch/arm/boot/dts/exynos5250-snow-rev5.dts
+++ b/arch/arm/boot/dts/exynos5250-snow-rev5.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Google Snow Rev 5+ board device tree source
  *
  * Copyright (c) 2012 Google, Inc
  * Copyright (c) 2015 Samsung Electronics Co., Ltd.
  *             http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos5250-snow.dts b/arch/arm/boot/dts/exynos5250-snow.dts
index df48f2cc96f7..4827cb506fa3 100644
--- a/arch/arm/boot/dts/exynos5250-snow.dts
+++ b/arch/arm/boot/dts/exynos5250-snow.dts
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Google Snow board device tree source
  *
  * Copyright (c) 2012 Google, Inc
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos5250-spring.dts b/arch/arm/boot/dts/exynos5250-spring.dts
index 47dbc50546c1..3d501926c227 100644
--- a/arch/arm/boot/dts/exynos5250-spring.dts
+++ b/arch/arm/boot/dts/exynos5250-spring.dts
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Google Spring board device tree source
  *
  * Copyright (c) 2013 Google, Inc
  * Copyright (c) 2014 SUSE LINUX Products GmbH
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos5250.dtsi b/arch/arm/boot/dts/exynos5250.dtsi
index 5286084e1032..56626d1a4235 100644
--- a/arch/arm/boot/dts/exynos5250.dtsi
+++ b/arch/arm/boot/dts/exynos5250.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS5250 SoC device tree source
  *
@@ -11,11 +12,7 @@
  * Note: This file does not include device nodes for all the controllers in
  * EXYNOS5250 SoC. As device tree coverage for EXYNOS5250 increases,
  * additional nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/clock/exynos5250.h>
 #include "exynos5.dtsi"
@@ -111,21 +108,28 @@
 			};
 		};
 
-		pd_gsc: gsc-power-domain@10044000 {
+		pd_gsc: power-domain@10044000 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10044000 0x20>;
 			#power-domain-cells = <0>;
 			label = "GSC";
 		};
 
-		pd_mfc: mfc-power-domain@10044040 {
+		pd_mfc: power-domain@10044040 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x10044040 0x20>;
 			#power-domain-cells = <0>;
 			label = "MFC";
 		};
 
-		pd_disp1: disp1-power-domain@100440A0 {
+		pd_g3d: power-domain@10044060 {
+			compatible = "samsung,exynos4210-pd";
+			reg = <0x10044060 0x20>;
+			#power-domain-cells = <0>;
+			label = "G3D";
+		};
+
+		pd_disp1: power-domain@100440a0 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x100440A0 0x20>;
 			#power-domain-cells = <0>;
@@ -136,6 +140,13 @@
 			clock-names = "oscclk", "clk0", "clk1";
 		};
 
+		pd_mau: power-domain@100440c0 {
+			compatible = "samsung,exynos4210-pd";
+			reg = <0x100440C0 0x20>;
+			#power-domain-cells = <0>;
+			label = "MAU";
+		};
+
 		clock: clock-controller@10010000 {
 			compatible = "samsung,exynos5250-clock";
 			reg = <0x10010000 0x30000>;
@@ -149,6 +160,7 @@
 			clocks = <&clock CLK_FIN_PLL>, <&clock CLK_FOUT_EPLL>,
 				 <&clock CLK_SCLK_AUDIO0>, <&clock CLK_DIV_PCM0>;
 			clock-names = "pll_ref", "pll_in", "sclk_audio", "sclk_pcm_in";
+			power-domains = <&pd_mau>;
 		};
 
 		timer {
@@ -165,7 +177,7 @@
 			clock-frequency = <24000000>;
 		};
 
-		mct@101C0000 {
+		mct@101c0000 {
 			compatible = "samsung,exynos4210-mct";
 			reg = <0x101C0000 0x800>;
 			interrupt-controller;
@@ -223,6 +235,7 @@
 			compatible = "samsung,exynos5250-pinctrl";
 			reg = <0x03860000 0x1000>;
 			interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&pd_mau>;
 		};
 
 		pmu_system_controller: system-controller@10040000 {
@@ -236,7 +249,7 @@
 			interrupt-parent = <&gic>;
 		};
 
-		watchdog@101D0000 {
+		watchdog@101d0000 {
 			compatible = "samsung,exynos5250-wdt";
 			reg = <0x101D0000 0x100>;
 			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
@@ -245,15 +258,6 @@
 			samsung,syscon-phandle = <&pmu_system_controller>;
 		};
 
-		g2d@10850000 {
-			compatible = "samsung,exynos5250-g2d";
-			reg = <0x10850000 0x1000>;
-			interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_G2D>;
-			clock-names = "fimg2d";
-			iommus = <&sysmmu_g2d>;
-		};
-
 		mfc: codec@11000000 {
 			compatible = "samsung,mfc-v6";
 			reg = <0x11000000 0x10000>;
@@ -265,7 +269,7 @@
 			iommu-names = "left", "right";
 		};
 
-		rotator: rotator@11C00000 {
+		rotator: rotator@11c00000 {
 			compatible = "samsung,exynos5250-rotator";
 			reg = <0x11C00000 0x64>;
 			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
@@ -283,7 +287,7 @@
 			#include "exynos4412-tmu-sensor-conf.dtsi"
 		};
 
-		sata: sata@122F0000 {
+		sata: sata@122f0000 {
 			compatible = "snps,dwc-ahci";
 			samsung,sata-freq = <66>;
 			reg = <0x122F0000 0x1ff>;
@@ -306,7 +310,7 @@
 		};
 
 		/* i2c_0-3 are defined in exynos5.dtsi */
-		i2c_4: i2c@12CA0000 {
+		i2c_4: i2c@12ca0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CA0000 0x100>;
 			interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
@@ -319,7 +323,7 @@
 			status = "disabled";
 		};
 
-		i2c_5: i2c@12CB0000 {
+		i2c_5: i2c@12cb0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CB0000 0x100>;
 			interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
@@ -332,7 +336,7 @@
 			status = "disabled";
 		};
 
-		i2c_6: i2c@12CC0000 {
+		i2c_6: i2c@12cc0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CC0000 0x100>;
 			interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
@@ -345,7 +349,7 @@
 			status = "disabled";
 		};
 
-		i2c_7: i2c@12CD0000 {
+		i2c_7: i2c@12cd0000 {
 			compatible = "samsung,s3c2440-i2c";
 			reg = <0x12CD0000 0x100>;
 			interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
@@ -358,7 +362,7 @@
 			status = "disabled";
 		};
 
-		i2c_8: i2c@12CE0000 {
+		i2c_8: i2c@12ce0000 {
 			compatible = "samsung,s3c2440-hdmiphy-i2c";
 			reg = <0x12CE0000 0x1000>;
 			interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
@@ -374,7 +378,7 @@
 			};
 		};
 
-		i2c_9: i2c@121D0000 {
+		i2c_9: i2c@121d0000 {
 			compatible = "samsung,exynos5-sata-phy-i2c";
 			reg = <0x121D0000 0x100>;
 			#address-cells = <1>;
@@ -495,9 +499,10 @@
 			samsung,idma-addr = <0x03000000>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&i2s0_bus>;
+			power-domains = <&pd_mau>;
 		};
 
-		i2s1: i2s@12D60000 {
+		i2s1: i2s@12d60000 {
 			compatible = "samsung,s3c6410-i2s";
 			status = "disabled";
 			reg = <0x12D60000 0x100>;
@@ -508,9 +513,10 @@
 			clock-names = "iis", "i2s_opclk0";
 			pinctrl-names = "default";
 			pinctrl-0 = <&i2s1_bus>;
+			power-domains = <&pd_mau>;
 		};
 
-		i2s2: i2s@12D70000 {
+		i2s2: i2s@12d70000 {
 			compatible = "samsung,s3c6410-i2s";
 			status = "disabled";
 			reg = <0x12D70000 0x100>;
@@ -521,6 +527,7 @@
 			clock-names = "iis", "i2s_opclk0";
 			pinctrl-names = "default";
 			pinctrl-0 = <&i2s2_bus>;
+			power-domains = <&pd_mau>;
 		};
 
 		usb_dwc3 {
@@ -596,7 +603,7 @@
 			interrupt-parent = <&gic>;
 			ranges;
 
-			pdma0: pdma@121A0000 {
+			pdma0: pdma@121a0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121A0000 0x1000>;
 				interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
@@ -607,7 +614,7 @@
 				#dma-requests = <32>;
 			};
 
-			pdma1: pdma@121B0000 {
+			pdma1: pdma@121b0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121B0000 0x1000>;
 				interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
@@ -629,7 +636,7 @@
 				#dma-requests = <1>;
 			};
 
-			mdma1: mdma@11C10000 {
+			mdma1: mdma@11c10000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x11C10000 0x1000>;
 				interrupts = <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>;
@@ -696,7 +703,7 @@
 			status = "disabled";
 		};
 
-		hdmicec: cec@101B0000 {
+		hdmicec: cec@101b0000 {
 			compatible = "samsung,s5p-cec";
 			reg = <0x101B0000 0x200>;
 			interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
@@ -727,7 +734,7 @@
 			#phy-cells = <0>;
 		};
 
-		adc: adc@12D10000 {
+		adc: adc@12d10000 {
 			compatible = "samsung,exynos-adc-v1";
 			reg = <0x12D10000 0x100>;
 			interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
@@ -739,15 +746,7 @@
 			status = "disabled";
 		};
 
-		sss@10830000 {
-			compatible = "samsung,exynos4210-secss";
-			reg = <0x10830000 0x300>;
-			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clock CLK_SSS>;
-			clock-names = "secss";
-		};
-
-		sysmmu_g2d: sysmmu@10A60000 {
+		sysmmu_g2d: sysmmu@10a60000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x10A60000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -779,7 +778,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_rotator: sysmmu@11D40000 {
+		sysmmu_rotator: sysmmu@11d40000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11D40000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -789,7 +788,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_jpeg: sysmmu@11F20000 {
+		sysmmu_jpeg: sysmmu@11f20000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x11F20000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -820,7 +819,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_fd: sysmmu@132A0000 {
+		sysmmu_fimc_fd: sysmmu@132a0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x132A0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -850,7 +849,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_mcuctl: sysmmu@132B0000 {
+		sysmmu_fimc_mcuctl: sysmmu@132b0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x132B0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -860,7 +859,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_odc: sysmmu@132C0000 {
+		sysmmu_fimc_odc: sysmmu@132c0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x132C0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -870,7 +869,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_dis0: sysmmu@132D0000 {
+		sysmmu_fimc_dis0: sysmmu@132d0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x132D0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -890,7 +889,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_3dnr: sysmmu@132F0000 {
+		sysmmu_fimc_3dnr: sysmmu@132f0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x132F0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -900,7 +899,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_lite0: sysmmu@13C40000 {
+		sysmmu_fimc_lite0: sysmmu@13c40000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13C40000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -911,7 +910,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_fimc_lite1: sysmmu@13C50000 {
+		sysmmu_fimc_lite1: sysmmu@13c50000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13C50000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -922,7 +921,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_gsc0: sysmmu@13E80000 {
+		sysmmu_gsc0: sysmmu@13e80000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13E80000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -933,7 +932,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_gsc1: sysmmu@13E90000 {
+		sysmmu_gsc1: sysmmu@13e90000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13E90000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -944,7 +943,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_gsc2: sysmmu@13EA0000 {
+		sysmmu_gsc2: sysmmu@13ea0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13EA0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -955,7 +954,7 @@
 			#iommu-cells = <0>;
 		};
 
-		sysmmu_gsc3: sysmmu@13EB0000 {
+		sysmmu_gsc3: sysmmu@13eb0000 {
 			compatible = "samsung,exynos-sysmmu";
 			reg = <0x13EB0000 0x1000>;
 			interrupt-parent = <&combiner>;
@@ -1024,6 +1023,13 @@
 	iommus = <&sysmmu_fimd1>;
 };
 
+&g2d {
+	iommus = <&sysmmu_g2d>;
+	clocks = <&clock CLK_G2D>;
+	clock-names = "fimg2d";
+	status = "okay";
+};
+
 &i2c_0 {
 	clocks = <&clock CLK_I2C0>;
 	clock-names = "i2c";
@@ -1052,6 +1058,11 @@
 	pinctrl-0 = <&i2c3_bus>;
 };
 
+&prng {
+	clocks = <&clock CLK_SSS>;
+	clock-names = "secss";
+};
+
 &pwm {
 	clocks = <&clock CLK_PWM>;
 	clock-names = "timers";
@@ -1092,4 +1103,14 @@
 	dma-names = "rx", "tx";
 };
 
+&sss {
+	clocks = <&clock CLK_SSS>;
+	clock-names = "secss";
+};
+
+&trng {
+	clocks = <&clock CLK_SSS>;
+	clock-names = "secss";
+};
+
 #include "exynos5250-pinctrl.dtsi"
diff --git a/arch/arm/boot/dts/exynos5260-pinctrl.dtsi b/arch/arm/boot/dts/exynos5260-pinctrl.dtsi
index 1b911a219a27..b1edb20b789e 100644
--- a/arch/arm/boot/dts/exynos5260-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos5260-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5260 SoC pin-mux and pin-config device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Samsung's Exynos5260 SoC pin-mux and pin-config options are listed as device
  * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/pinctrl/samsung.h>
 
diff --git a/arch/arm/boot/dts/exynos5260-xyref5260.dts b/arch/arm/boot/dts/exynos5260-xyref5260.dts
index 73b7cdd5f522..442eb0353f29 100644
--- a/arch/arm/boot/dts/exynos5260-xyref5260.dts
+++ b/arch/arm/boot/dts/exynos5260-xyref5260.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG XYREF5260 board device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5260.dtsi"
diff --git a/arch/arm/boot/dts/exynos5260.dtsi b/arch/arm/boot/dts/exynos5260.dtsi
index 5e88c9645975..55167850619c 100644
--- a/arch/arm/boot/dts/exynos5260.dtsi
+++ b/arch/arm/boot/dts/exynos5260.dtsi
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS5260 SoC device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/clock/exynos5260-clk.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
@@ -106,13 +103,13 @@
 			#clock-cells = <1>;
 		};
 
-		clock_g2d: clock-controller@10A00000 {
+		clock_g2d: clock-controller@10a00000 {
 			compatible = "samsung,exynos5260-clock-g2d";
 			reg = <0x10A00000 0x10000>;
 			#clock-cells = <1>;
 		};
 
-		clock_mif: clock-controller@10CE0000 {
+		clock_mif: clock-controller@10ce0000 {
 			compatible = "samsung,exynos5260-clock-mif";
 			reg = <0x10CE0000 0x10000>;
 			#clock-cells = <1>;
@@ -130,25 +127,25 @@
 			#clock-cells = <1>;
 		};
 
-		clock_fsys: clock-controller@122E0000 {
+		clock_fsys: clock-controller@122e0000 {
 			compatible = "samsung,exynos5260-clock-fsys";
 			reg = <0x122E0000 0x10000>;
 			#clock-cells = <1>;
 		};
 
-		clock_aud: clock-controller@128C0000 {
+		clock_aud: clock-controller@128c0000 {
 			compatible = "samsung,exynos5260-clock-aud";
 			reg = <0x128C0000 0x10000>;
 			#clock-cells = <1>;
 		};
 
-		clock_isp: clock-controller@133C0000 {
+		clock_isp: clock-controller@133c0000 {
 			compatible = "samsung,exynos5260-clock-isp";
 			reg = <0x133C0000 0x10000>;
 			#clock-cells = <1>;
 		};
 
-		clock_gscl: clock-controller@13F00000 {
+		clock_gscl: clock-controller@13f00000 {
 			compatible = "samsung,exynos5260-clock-gscl";
 			reg = <0x13F00000 0x10000>;
 			#clock-cells = <1>;
@@ -179,7 +176,7 @@
 			reg = <0x10000000 0x100>;
 		};
 
-		mct: mct@100B0000 {
+		mct: mct@100b0000 {
 			compatible = "samsung,exynos4210-mct";
 			reg = <0x100B0000 0x1000>;
 			clocks = <&fin_pll>, <&clock_peri PERI_CLK_MCT>;
@@ -198,7 +195,7 @@
 				     <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		cci: cci@10F00000 {
+		cci: cci@10f00000 {
 			compatible = "arm,cci-400";
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -236,18 +233,18 @@
 			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		pinctrl_2: pinctrl@128B0000 {
+		pinctrl_2: pinctrl@128b0000 {
 			compatible = "samsung,exynos5260-pinctrl";
 			reg = <0x128B0000 0x1000>;
 			interrupts = <GIC_SPI 243 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		pmu_system_controller: system-controller@10D50000 {
+		pmu_system_controller: system-controller@10d50000 {
 			compatible = "samsung,exynos5260-pmu", "syscon";
 			reg = <0x10D50000 0x10000>;
 		};
 
-		uart0: serial@12C00000 {
+		uart0: serial@12c00000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C00000 0x100>;
 			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>;
@@ -256,7 +253,7 @@
 			status = "disabled";
 		};
 
-		uart1: serial@12C10000 {
+		uart1: serial@12c10000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C10000 0x100>;
 			interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>;
@@ -265,7 +262,7 @@
 			status = "disabled";
 		};
 
-		uart2: serial@12C20000 {
+		uart2: serial@12c20000 {
 			compatible = "samsung,exynos4210-uart";
 			reg = <0x12C20000 0x100>;
 			interrupts = <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/exynos5410-odroidxu.dts b/arch/arm/boot/dts/exynos5410-odroidxu.dts
index a45eaae33f8f..a2046f5f998c 100644
--- a/arch/arm/boot/dts/exynos5410-odroidxu.dts
+++ b/arch/arm/boot/dts/exynos5410-odroidxu.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid XU board device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  * Copyright (c) 2016 Krzysztof Kozlowski
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos5410-pinctrl.dtsi b/arch/arm/boot/dts/exynos5410-pinctrl.dtsi
index ff46a1c27182..369a8a7f2105 100644
--- a/arch/arm/boot/dts/exynos5410-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos5410-pinctrl.dtsi
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Exynos5410 SoC pin-mux and pin-config device tree source
  *
  * Copyright (c) 2013 Hardkernel Co., Ltd.
  *              http://www.hardkernel.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/pinctrl/samsung.h>
diff --git a/arch/arm/boot/dts/exynos5410-smdk5410.dts b/arch/arm/boot/dts/exynos5410-smdk5410.dts
index 25f21e9e7d58..8fc8c841d34b 100644
--- a/arch/arm/boot/dts/exynos5410-smdk5410.dts
+++ b/arch/arm/boot/dts/exynos5410-smdk5410.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG SMDK5410 board device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5410.dtsi"
diff --git a/arch/arm/boot/dts/exynos5410.dtsi b/arch/arm/boot/dts/exynos5410.dtsi
index 06713ec86f0d..1886aa00b2db 100644
--- a/arch/arm/boot/dts/exynos5410.dtsi
+++ b/arch/arm/boot/dts/exynos5410.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS5410 SoC device tree source
  *
@@ -7,10 +8,6 @@
  * SAMSUNG EXYNOS5410 SoC device nodes are listed in this file.
  * EXYNOS5410 based board files can include this file and provide
  * values for board specfic bindings.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "exynos54xx.dtsi"
@@ -264,6 +261,11 @@
 	};
 };
 
+&arm_a15_pmu {
+	interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	status = "okay";
+};
+
 &i2c_0 {
 	clocks = <&clock CLK_I2C0>;
 	clock-names = "i2c";
@@ -325,6 +327,11 @@
 	clock-names = "fin_pll", "mct";
 };
 
+&prng {
+	clocks = <&clock CLK_SSS>;
+	clock-names = "secss";
+};
+
 &pwm {
 	clocks = <&clock CLK_PWM>;
 	clock-names = "timers";
@@ -333,7 +340,6 @@
 &rtc {
 	clocks = <&clock CLK_RTC>;
 	clock-names = "rtc";
-	interrupt-parent = <&pmu_system_controller>;
 	status = "disabled";
 };
 
@@ -379,6 +385,11 @@
 		  3 0 0x07000000 0x20000>;
 };
 
+&trng {
+	clocks = <&clock CLK_SSS>;
+	clock-names = "secss";
+};
+
 &usbdrd3_0 {
 	clocks = <&clock CLK_USBD300>;
 	clock-names = "usbdrd30";
diff --git a/arch/arm/boot/dts/exynos5420-arndale-octa.dts b/arch/arm/boot/dts/exynos5420-arndale-octa.dts
index bc78575d8a4d..cdda614e417e 100644
--- a/arch/arm/boot/dts/exynos5420-arndale-octa.dts
+++ b/arch/arm/boot/dts/exynos5420-arndale-octa.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5420 based Arndale Octa board device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5420.dtsi"
diff --git a/arch/arm/boot/dts/exynos5420-cpus.dtsi b/arch/arm/boot/dts/exynos5420-cpus.dtsi
index d7d703aa1699..123f0cef658d 100644
--- a/arch/arm/boot/dts/exynos5420-cpus.dtsi
+++ b/arch/arm/boot/dts/exynos5420-cpus.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS5420 SoC cpu device tree source
  *
@@ -14,10 +15,6 @@
  * booting cluster (big or LITTLE) is chosen by IROM code by reading
  * the gpg2-1 GPIO. By default all Exynos5422 based boards choose booting
  * from the LITTLE: Cortex-A7.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 / {
@@ -132,3 +129,13 @@
 		};
 	};
 };
+
+&arm_a7_pmu {
+	interrupt-affinity = <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
+	status = "okay";
+};
+
+&arm_a15_pmu {
+	interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/exynos5420-peach-pit.dts b/arch/arm/boot/dts/exynos5420-peach-pit.dts
index 38af8769711c..5a76ed77dda1 100644
--- a/arch/arm/boot/dts/exynos5420-peach-pit.dts
+++ b/arch/arm/boot/dts/exynos5420-peach-pit.dts
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Google Peach Pit Rev 6+ board device tree source
  *
  * Copyright (c) 2014 Google, Inc
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/exynos5420-pinctrl.dtsi b/arch/arm/boot/dts/exynos5420-pinctrl.dtsi
index 65aa0e300c23..dda8ca2d2324 100644
--- a/arch/arm/boot/dts/exynos5420-pinctrl.dtsi
+++ b/arch/arm/boot/dts/exynos5420-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5420 SoC pin-mux and pin-config device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Samsung's Exynos5420 SoC pin-mux and pin-config options are listed as device
  * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/pinctrl/samsung.h>
 
diff --git a/arch/arm/boot/dts/exynos5420-smdk5420.dts b/arch/arm/boot/dts/exynos5420-smdk5420.dts
index 310d8637ce9f..831c7336f237 100644
--- a/arch/arm/boot/dts/exynos5420-smdk5420.dts
+++ b/arch/arm/boot/dts/exynos5420-smdk5420.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG SMDK5420 board device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5420.dtsi"
diff --git a/arch/arm/boot/dts/exynos5420-tmu-sensor-conf.dtsi b/arch/arm/boot/dts/exynos5420-tmu-sensor-conf.dtsi
index c8771c660550..fbc77cb58473 100644
--- a/arch/arm/boot/dts/exynos5420-tmu-sensor-conf.dtsi
+++ b/arch/arm/boot/dts/exynos5420-tmu-sensor-conf.dtsi
@@ -1,13 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos5420 TMU sensor configuration
  *
  * Copyright (c) 2014 Lukasz Majewski <l.majewski@samsung.com>
  * Copyright (c) 2017 Krzysztof Kozlowski <krzk@kernel.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 #include <dt-bindings/thermal/thermal_exynos.h>
diff --git a/arch/arm/boot/dts/exynos5420-trip-points.dtsi b/arch/arm/boot/dts/exynos5420-trip-points.dtsi
index 2180a0152c9b..a67a380717ec 100644
--- a/arch/arm/boot/dts/exynos5420-trip-points.dtsi
+++ b/arch/arm/boot/dts/exynos5420-trip-points.dtsi
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for default Exynos5420 thermal zone definition
  *
  * Copyright (c) 2014 Lukasz Majewski <l.majewski@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 polling-delay-passive = <0>;
diff --git a/arch/arm/boot/dts/exynos5420.dtsi b/arch/arm/boot/dts/exynos5420.dtsi
index 8aa2cc7aa125..2f3cb2a97f71 100644
--- a/arch/arm/boot/dts/exynos5420.dtsi
+++ b/arch/arm/boot/dts/exynos5420.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS5420 SoC device tree source
  *
@@ -7,10 +8,6 @@
  * SAMSUNG EXYNOS54200 SoC device nodes are listed in this file.
  * EXYNOS5420 based board files can include this file and provide
  * values for board specfic bindings.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "exynos54xx.dtsi"
@@ -188,6 +185,7 @@
 			clocks = <&clock CLK_FIN_PLL>, <&clock CLK_MAU_EPLL>,
 				 <&clock CLK_SCLK_MAUDIO0>, <&clock CLK_SCLK_MAUPCM0>;
 			clock-names = "pll_ref", "pll_in", "sclk_audio", "sclk_pcm_in";
+			power-domains = <&mau_pd>;
 		};
 
 		mfc: codec@11000000 {
@@ -237,37 +235,37 @@
 			status = "disabled";
 		};
 
-		nocp_mem0_0: nocp@10CA1000 {
+		nocp_mem0_0: nocp@10ca1000 {
 			compatible = "samsung,exynos5420-nocp";
 			reg = <0x10CA1000 0x200>;
 			status = "disabled";
 		};
 
-		nocp_mem0_1: nocp@10CA1400 {
+		nocp_mem0_1: nocp@10ca1400 {
 			compatible = "samsung,exynos5420-nocp";
 			reg = <0x10CA1400 0x200>;
 			status = "disabled";
 		};
 
-		nocp_mem1_0: nocp@10CA1800 {
+		nocp_mem1_0: nocp@10ca1800 {
 			compatible = "samsung,exynos5420-nocp";
 			reg = <0x10CA1800 0x200>;
 			status = "disabled";
 		};
 
-		nocp_mem1_1: nocp@10CA1C00 {
+		nocp_mem1_1: nocp@10ca1c00 {
 			compatible = "samsung,exynos5420-nocp";
 			reg = <0x10CA1C00 0x200>;
 			status = "disabled";
 		};
 
-		nocp_g3d_0: nocp@11A51000 {
+		nocp_g3d_0: nocp@11a51000 {
 			compatible = "samsung,exynos5420-nocp";
 			reg = <0x11A51000 0x200>;
 			status = "disabled";
 		};
 
-		nocp_g3d_1: nocp@11A51400 {
+		nocp_g3d_1: nocp@11a51400 {
 			compatible = "samsung,exynos5420-nocp";
 			reg = <0x11A51400 0x200>;
 			status = "disabled";
@@ -309,7 +307,7 @@
 			label = "MSC";
 		};
 
-		disp_pd: power-domain@100440C0 {
+		disp_pd: power-domain@100440c0 {
 			compatible = "samsung,exynos4210-pd";
 			reg = <0x100440C0 0x20>;
 			#power-domain-cells = <0>;
@@ -322,6 +320,13 @@
 			clock-names = "oscclk", "clk0", "clk1", "clk2", "asb0", "asb1";
 		};
 
+		mau_pd: power-domain@100440e0 {
+			compatible = "samsung,exynos4210-pd";
+			reg = <0x100440E0 0x20>;
+			#power-domain-cells = <0>;
+			label = "MAU";
+		};
+
 		pinctrl_0: pinctrl@13400000 {
 			compatible = "samsung,exynos5420-pinctrl";
 			reg = <0x13400000 0x1000>;
@@ -356,6 +361,7 @@
 			compatible = "samsung,exynos5420-pinctrl";
 			reg = <0x03860000 0x1000>;
 			interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&mau_pd>;
 		};
 
 		amba {
@@ -374,9 +380,10 @@
 				#dma-cells = <1>;
 				#dma-channels = <6>;
 				#dma-requests = <16>;
+				power-domains = <&mau_pd>;
 			};
 
-			pdma0: pdma@121A0000 {
+			pdma0: pdma@121a0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121A0000 0x1000>;
 				interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
@@ -387,7 +394,7 @@
 				#dma-requests = <32>;
 			};
 
-			pdma1: pdma@121B0000 {
+			pdma1: pdma@121b0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x121B0000 0x1000>;
 				interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
@@ -409,7 +416,7 @@
 				#dma-requests = <1>;
 			};
 
-			mdma1: mdma@11C10000 {
+			mdma1: mdma@11c10000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x11C10000 0x1000>;
 				interrupts = <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>;
@@ -446,10 +453,11 @@
 			samsung,idma-addr = <0x03000000>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&i2s0_bus>;
+			power-domains = <&mau_pd>;
 			status = "disabled";
 		};
 
-		i2s1: i2s@12D60000 {
+		i2s1: i2s@12d60000 {
 			compatible = "samsung,exynos5420-i2s";
 			reg = <0x12D60000 0x100>;
 			dmas = <&pdma1 12
@@ -465,7 +473,7 @@
 			status = "disabled";
 		};
 
-		i2s2: i2s@12D70000 {
+		i2s2: i2s@12d70000 {
 			compatible = "samsung,exynos5420-i2s";
 			reg = <0x12D70000 0x100>;
 			dmas = <&pdma0 12
@@ -554,7 +562,7 @@
 			status = "disabled";
 		};
 
-		adc: adc@12D10000 {
+		adc: adc@12d10000 {
 			compatible = "samsung,exynos-adc-v2";
 			reg = <0x12D10000 0x100>;
 			interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
@@ -566,7 +574,7 @@
 			status = "disabled";
 		};
 
-		hsi2c_8: i2c@12E00000 {
+		hsi2c_8: i2c@12e00000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12E00000 0x1000>;
 			interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
@@ -579,7 +587,7 @@
 			status = "disabled";
 		};
 
-		hsi2c_9: i2c@12E10000 {
+		hsi2c_9: i2c@12e10000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12E10000 0x1000>;
 			interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>;
@@ -592,7 +600,7 @@
 			status = "disabled";
 		};
 
-		hsi2c_10: i2c@12E20000 {
+		hsi2c_10: i2c@12e20000 {
 			compatible = "samsung,exynos5250-hsi2c";
 			reg = <0x12E20000 0x1000>;
 			interrupts = <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>;
@@ -618,13 +626,14 @@
 			samsung,syscon-phandle = <&pmu_system_controller>;
 			status = "disabled";
 			power-domains = <&disp_pd>;
+			#sound-dai-cells = <0>;
 		};
 
-		hdmiphy: hdmiphy@145D0000 {
+		hdmiphy: hdmiphy@145d0000 {
 			reg = <0x145D0000 0x20>;
 		};
 
-		hdmicec: cec@101B0000 {
+		hdmicec: cec@101b0000 {
 			compatible = "samsung,s5p-cec";
 			reg = <0x101B0000 0x200>;
 			interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
@@ -649,7 +658,7 @@
 			status = "disabled";
 		};
 
-		rotator: rotator@11C00000 {
+		rotator: rotator@11c00000 {
 			compatible = "samsung,exynos5250-rotator";
 			reg = <0x11C00000 0x64>;
 			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
@@ -678,7 +687,7 @@
 			iommus = <&sysmmu_gscl1>;
 		};
 
-		jpeg_0: jpeg@11F50000 {
+		jpeg_0: jpeg@11f50000 {
 			compatible = "samsung,exynos5420-jpeg";
 			reg = <0x11F50000 0x1000>;
 			interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
@@ -687,7 +696,7 @@
 			iommus = <&sysmmu_jpeg0>;
 		};
 
-		jpeg_1: jpeg@11F60000 {
+		jpeg_1: jpeg@11f60000 {
 			compatible = "samsung,exynos5420-jpeg";
 			reg = <0x11F60000 0x1000>;
 			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
@@ -1349,6 +1358,13 @@
 	iommu-names = "m0", "m1";
 };
 
+&g2d {
+	iommus = <&sysmmu_g2dr>, <&sysmmu_g2dw>;
+	clocks = <&clock CLK_G2D>;
+	clock-names = "fimg2d";
+	status = "okay";
+};
+
 &i2c_0 {
 	clocks = <&clock CLK_I2C0>;
 	clock-names = "i2c";
@@ -1410,6 +1426,11 @@
 	clock-names = "fin_pll", "mct";
 };
 
+&prng {
+	clocks = <&clock CLK_SSS>;
+	clock-names = "secss";
+};
+
 &pwm {
 	clocks = <&clock CLK_PWM>;
 	clock-names = "timers";
@@ -1455,6 +1476,11 @@
 	clock-names = "secss";
 };
 
+&trng {
+	clocks = <&clock CLK_SSS>;
+	clock-names = "secss";
+};
+
 &usbdrd3_0 {
 	clocks = <&clock CLK_USBD300>;
 	clock-names = "usbdrd30";
diff --git a/arch/arm/boot/dts/exynos5422-cpus.dtsi b/arch/arm/boot/dts/exynos5422-cpus.dtsi
index ec01d8020c2d..c593809c7f08 100644
--- a/arch/arm/boot/dts/exynos5422-cpus.dtsi
+++ b/arch/arm/boot/dts/exynos5422-cpus.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS5422 SoC cpu device tree source
  *
@@ -13,10 +14,6 @@
  * booting cluster (big or LITTLE) is chosen by IROM code by reading
  * the gpg2-1 GPIO. By default all Exynos5422 based boards choose booting
  * from the LITTLE: Cortex-A7.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 / {
@@ -131,3 +128,13 @@
 		};
 	};
 };
+
+&arm_a7_pmu {
+	interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	status = "okay";
+};
+
+&arm_a15_pmu {
+	interrupt-affinity = <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/exynos5422-odroid-core.dtsi b/arch/arm/boot/dts/exynos5422-odroid-core.dtsi
index a5b8d0f0877e..d31249f25ccf 100644
--- a/arch/arm/boot/dts/exynos5422-odroid-core.dtsi
+++ b/arch/arm/boot/dts/exynos5422-odroid-core.dtsi
@@ -1,14 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid XU3/XU4/HC1 boards core device tree source
  *
  * Copyright (c) 2017 Marek Szyprowski
  * Copyright (c) 2013-2017 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/clock/samsung,s2mps11.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -233,8 +230,8 @@
 
 			ldo15_reg: LDO15 {
 				regulator-name = "vdd_ldo15";
-				regulator-min-microvolt = <3100000>;
-				regulator-max-microvolt = <3100000>;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
 				regulator-always-on;
 			};
 
@@ -246,7 +243,7 @@
 			};
 
 			ldo17_reg: LDO17 {
-				regulator-name = "tsp_avdd";
+				regulator-name = "vdd_ldo17";
 				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-always-on;
diff --git a/arch/arm/boot/dts/exynos5422-odroidhc1.dts b/arch/arm/boot/dts/exynos5422-odroidhc1.dts
index fb8e8ae776e9..8f332be143f7 100644
--- a/arch/arm/boot/dts/exynos5422-odroidhc1.dts
+++ b/arch/arm/boot/dts/exynos5422-odroidhc1.dts
@@ -1,14 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid HC1 board device tree source
  *
  * Copyright (c) 2017 Marek Szyprowski
  * Copyright (c) 2017 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5422-odroid-core.dtsi"
diff --git a/arch/arm/boot/dts/exynos5422-odroidxu3-audio.dtsi b/arch/arm/boot/dts/exynos5422-odroidxu3-audio.dtsi
index da3141a307d5..03611d50c5a9 100644
--- a/arch/arm/boot/dts/exynos5422-odroidxu3-audio.dtsi
+++ b/arch/arm/boot/dts/exynos5422-odroidxu3-audio.dtsi
@@ -1,27 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Hardkernel Odroid XU3 Audio Codec device tree source
+ * Hardkernel Odroid XU3 audio subsystem device tree source
  *
  * Copyright (c) 2015 Krzysztof Kozlowski
  * Copyright (c) 2014 Collabora Ltd.
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/sound/samsung-i2s.h>
 
 / {
 	sound: sound {
-		compatible = "simple-audio-card";
+		compatible = "samsung,odroid-xu3-audio";
+		model = "Odroid-XU3";
 
-		simple-audio-card,name = "Odroid-XU3";
-		simple-audio-card,widgets =
+		samsung,audio-widgets =
 			"Headphone", "Headphone Jack",
 			"Speakers", "Speakers";
-		simple-audio-card,routing =
+		samsung,audio-routing =
 			"Headphone Jack", "HPL",
 			"Headphone Jack", "HPR",
 			"Headphone Jack", "MICBIAS",
@@ -29,31 +26,47 @@
 			"Speakers", "SPKL",
 			"Speakers", "SPKR";
 
-		simple-audio-card,format = "i2s";
-		simple-audio-card,bitclock-master = <&link0_codec>;
-		simple-audio-card,frame-master = <&link0_codec>;
+		assigned-clocks = <&i2s0 CLK_I2S_RCLK_SRC>,
+				<&clock CLK_MOUT_EPLL>,
+				<&clock CLK_MOUT_MAU_EPLL>,
+				<&clock CLK_MOUT_USER_MAU_EPLL>,
+				<&clock_audss EXYNOS_MOUT_AUDSS>,
+				<&clock_audss EXYNOS_MOUT_I2S>,
+				<&clock_audss EXYNOS_DOUT_SRP>,
+				<&clock_audss EXYNOS_DOUT_AUD_BUS>,
+				<&clock_audss EXYNOS_DOUT_I2S>;
+
+		assigned-clock-parents = <&clock_audss EXYNOS_SCLK_I2S>,
+				<&clock CLK_FOUT_EPLL>,
+				<&clock CLK_MOUT_EPLL>,
+				<&clock CLK_MOUT_MAU_EPLL>,
+				<&clock CLK_MAU_EPLL>,
+				<&clock_audss EXYNOS_MOUT_AUDSS>;
 
-		simple-audio-card,cpu {
+		assigned-clock-rates = <0>,
+				<0>,
+				<0>,
+				<0>,
+				<0>,
+				<0>,
+				<196608001>,
+				<(196608002 / 2)>,
+				<196608000>;
+
+		cpu {
 			sound-dai = <&i2s0 0>;
-			system-clock-frequency = <19200000>;
 		};
-
-		link0_codec: simple-audio-card,codec {
-			sound-dai = <&max98090>;
-			clocks = <&i2s0 CLK_I2S_CDCLK>;
+		codec {
+			sound-dai = <&hdmi>, <&max98090>;
 		};
 	};
 };
 
 &clock_audss {
-	assigned-clocks = <&clock_audss EXYNOS_MOUT_AUDSS>,
-			<&clock_audss EXYNOS_MOUT_I2S>,
-			<&clock_audss EXYNOS_DOUT_AUD_BUS>;
-	assigned-clock-parents = <&clock CLK_FIN_PLL>,
-			<&clock_audss EXYNOS_MOUT_AUDSS>;
-	assigned-clock-rates = <0>,
-			<0>,
-			<19200000>;
+	assigned-clocks = <&clock_audss EXYNOS_DOUT_SRP>,
+			  <&clock CLK_FOUT_EPLL>;
+	assigned-clock-rates = <(196608000 / 256)>,
+			       <196608000>;
 };
 
 &hsi2c_5 {
diff --git a/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi b/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi
index 445c6c5a1300..96e281c0a118 100644
--- a/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi
+++ b/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid XU3 board device tree source
  *
@@ -6,11 +7,7 @@
  * Copyright (c) 2014 Collabora Ltd.
  * Copyright (c) 2015 Lukasz Majewski <l.majewski@samsung.com>
  *                    Anand Moon <linux.amoon@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/input/input.h>
 #include "exynos5422-odroid-core.dtsi"
diff --git a/arch/arm/boot/dts/exynos5422-odroidxu3-lite.dts b/arch/arm/boot/dts/exynos5422-odroidxu3-lite.dts
index 03fa88c45426..0db935f2b836 100644
--- a/arch/arm/boot/dts/exynos5422-odroidxu3-lite.dts
+++ b/arch/arm/boot/dts/exynos5422-odroidxu3-lite.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid XU3-Lite board device tree source
  *
@@ -5,11 +6,7 @@
  * Copyright (c) 2014 Collabora Ltd.
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5422-odroidxu3-common.dtsi"
diff --git a/arch/arm/boot/dts/exynos5422-odroidxu3.dts b/arch/arm/boot/dts/exynos5422-odroidxu3.dts
index 9ed6564acfb0..0322f281912c 100644
--- a/arch/arm/boot/dts/exynos5422-odroidxu3.dts
+++ b/arch/arm/boot/dts/exynos5422-odroidxu3.dts
@@ -1,14 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid XU3 board device tree source
  *
  * Copyright (c) 2014 Collabora Ltd.
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5422-odroidxu3-common.dtsi"
diff --git a/arch/arm/boot/dts/exynos5422-odroidxu4.dts b/arch/arm/boot/dts/exynos5422-odroidxu4.dts
index 2faf88627a48..4a30cc849b00 100644
--- a/arch/arm/boot/dts/exynos5422-odroidxu4.dts
+++ b/arch/arm/boot/dts/exynos5422-odroidxu4.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid XU4 board device tree source
  *
@@ -5,13 +6,10 @@
  * Copyright (c) 2014 Collabora Ltd.
  * Copyright (c) 2013-2015 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
+#include <dt-bindings/sound/samsung-i2s.h>
 #include "exynos5422-odroidxu3-common.dtsi"
 
 / {
@@ -30,6 +28,57 @@
 			linux,default-trigger = "heartbeat";
 		};
 	};
+
+	sound: sound {
+		compatible = "samsung,odroid-xu3-audio";
+		model = "Odroid-XU4";
+
+		assigned-clocks = <&i2s0 CLK_I2S_RCLK_SRC>,
+				<&clock CLK_MOUT_EPLL>,
+				<&clock CLK_MOUT_MAU_EPLL>,
+				<&clock CLK_MOUT_USER_MAU_EPLL>,
+				<&clock_audss EXYNOS_MOUT_AUDSS>,
+				<&clock_audss EXYNOS_MOUT_I2S>,
+				<&clock_audss EXYNOS_DOUT_SRP>,
+				<&clock_audss EXYNOS_DOUT_AUD_BUS>,
+				<&clock_audss EXYNOS_DOUT_I2S>;
+
+		assigned-clock-parents = <&clock_audss EXYNOS_SCLK_I2S>,
+				<&clock CLK_FOUT_EPLL>,
+				<&clock CLK_MOUT_EPLL>,
+				<&clock CLK_MOUT_MAU_EPLL>,
+				<&clock CLK_MAU_EPLL>,
+				<&clock_audss EXYNOS_MOUT_AUDSS>;
+
+		assigned-clock-rates = <0>,
+				<0>,
+				<0>,
+				<0>,
+				<0>,
+				<0>,
+				<196608001>,
+				<(196608002 / 2)>,
+				<196608000>;
+
+		cpu {
+			sound-dai = <&i2s0 0>;
+		};
+
+		codec {
+			sound-dai = <&hdmi>;
+		};
+	};
+};
+
+&clock_audss {
+	assigned-clocks = <&clock_audss EXYNOS_DOUT_SRP>,
+			  <&clock CLK_FOUT_EPLL>;
+	assigned-clock-rates = <(196608000 / 256)>,
+			       <196608000>;
+};
+
+&i2s0 {
+	status = "okay";
 };
 
 &pwm {
diff --git a/arch/arm/boot/dts/exynos5440-sd5v1.dts b/arch/arm/boot/dts/exynos5440-sd5v1.dts
index ad6f533b3f40..c4b8392d1ae1 100644
--- a/arch/arm/boot/dts/exynos5440-sd5v1.dts
+++ b/arch/arm/boot/dts/exynos5440-sd5v1.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG SD5v1 board device tree source
  *
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5440.dtsi"
diff --git a/arch/arm/boot/dts/exynos5440-ssdk5440.dts b/arch/arm/boot/dts/exynos5440-ssdk5440.dts
index 7eafad333bdb..a33c4fc29ae5 100644
--- a/arch/arm/boot/dts/exynos5440-ssdk5440.dts
+++ b/arch/arm/boot/dts/exynos5440-ssdk5440.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG SSDK5440 board device tree source
  *
  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos5440.dtsi"
diff --git a/arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi b/arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi
index 7b2fba0ae92b..0421c3d42905 100644
--- a/arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi
+++ b/arch/arm/boot/dts/exynos5440-tmu-sensor-conf.dtsi
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos5440 TMU sensor configuration
  *
  * Copyright (c) 2014 Lukasz Majewski <l.majewski@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 #include <dt-bindings/thermal/thermal_exynos.h>
diff --git a/arch/arm/boot/dts/exynos5440-trip-points.dtsi b/arch/arm/boot/dts/exynos5440-trip-points.dtsi
index 356e963edf11..a2b04fed7d0b 100644
--- a/arch/arm/boot/dts/exynos5440-trip-points.dtsi
+++ b/arch/arm/boot/dts/exynos5440-trip-points.dtsi
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for default Exynos5440 thermal zone definition
  *
  * Copyright (c) 2014 Lukasz Majewski <l.majewski@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 polling-delay-passive = <0>;
diff --git a/arch/arm/boot/dts/exynos5440.dtsi b/arch/arm/boot/dts/exynos5440.dtsi
index 9c3c75ae5e48..fce9e26b5930 100644
--- a/arch/arm/boot/dts/exynos5440.dtsi
+++ b/arch/arm/boot/dts/exynos5440.dtsi
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS5440 SoC device tree source
  *
  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/clock/exynos5440.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
@@ -35,7 +32,7 @@
 		#clock-cells = <1>;
 	};
 
-	gic: interrupt-controller@2E0000 {
+	gic: interrupt-controller@2e0000 {
 		compatible = "arm,cortex-a15-gic";
 		#interrupt-cells = <3>;
 		interrupt-controller;
@@ -108,7 +105,7 @@
 		>;
 	};
 
-	serial_0: serial@B0000 {
+	serial_0: serial@b0000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0xB0000 0x1000>;
 		interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
@@ -116,7 +113,7 @@
 		clock-names = "uart", "clk_uart_baud0";
 	};
 
-	serial_1: serial@C0000 {
+	serial_1: serial@c0000 {
 		compatible = "samsung,exynos4210-uart";
 		reg = <0xC0000 0x1000>;
 		interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
@@ -124,7 +121,7 @@
 		clock-names = "uart", "clk_uart_baud0";
 	};
 
-	spi_0: spi@D0000 {
+	spi_0: spi@d0000 {
 		compatible = "samsung,exynos5440-spi";
 		reg = <0xD0000 0x100>;
 		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
@@ -136,7 +133,7 @@
 		clock-names = "spi", "spi_busclk0";
 	};
 
-	pin_ctrl: pinctrl@E0000 {
+	pin_ctrl: pinctrl@e0000 {
 		compatible = "samsung,exynos5440-pinctrl";
 		reg = <0xE0000 0x1000>;
 		interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>,
@@ -168,7 +165,7 @@
 		};
 	};
 
-	i2c@F0000 {
+	i2c@f0000 {
 		compatible = "samsung,exynos5440-i2c";
 		reg = <0xF0000 0x1000>;
 		interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
@@ -233,7 +230,7 @@
 		#include "exynos5440-tmu-sensor-conf.dtsi"
 	};
 
-	tmuctrl_1: tmuctrl@16011C {
+	tmuctrl_1: tmuctrl@16011c {
 		compatible = "samsung,exynos5440-tmu";
 		reg = <0x16011C 0x230>, <0x160368 0x10>;
 		interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/exynos54xx-odroidxu-leds.dtsi b/arch/arm/boot/dts/exynos54xx-odroidxu-leds.dtsi
index 0ed30206625c..56acd832f0b3 100644
--- a/arch/arm/boot/dts/exynos54xx-odroidxu-leds.dtsi
+++ b/arch/arm/boot/dts/exynos54xx-odroidxu-leds.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Hardkernel Odroid XU/XU3 LED device tree source
  *
@@ -5,11 +6,7 @@
  * Copyright (c) 2014 Collabora Ltd.
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/gpio/gpio.h>
 
diff --git a/arch/arm/boot/dts/exynos54xx.dtsi b/arch/arm/boot/dts/exynos54xx.dtsi
index a5007f182bc4..de26e5ee0d2d 100644
--- a/arch/arm/boot/dts/exynos54xx.dtsi
+++ b/arch/arm/boot/dts/exynos54xx.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos54xx SoC series common device tree source
  *
@@ -8,10 +9,6 @@
  * Device nodes common for Samsung Exynos5410/5420/5422/5800. Specific
  * Exynos 54xx SoCs should include this file and customize it further
  * (e.g. with clocks).
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "exynos5.dtsi"
@@ -29,6 +26,26 @@
 	};
 
 	soc: soc {
+		arm_a7_pmu: arm-a7-pmu {
+			compatible = "arm,cortex-a7-pmu";
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		arm_a15_pmu: arm-a15-pmu {
+			compatible = "arm,cortex-a15-pmu";
+			interrupt-parent = <&combiner>;
+			interrupts = <1 2>,
+				     <7 0>,
+				     <16 6>,
+				     <19 2>;
+			status = "disabled";
+		};
+
 		sysram@2020000 {
 			compatible = "mmio-sram";
 			reg = <0x02020000 0x54000>;
@@ -79,12 +96,6 @@
 			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
-		sss: sss@10830000 {
-			compatible = "samsung,exynos4210-secss";
-			reg = <0x10830000 0x300>;
-			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
 		/* i2c_0-3 are defined in exynos5.dtsi */
 		hsi2c_4: i2c@12ca0000 {
 			compatible = "samsung,exynos5250-hsi2c";
diff --git a/arch/arm/boot/dts/ge863-pro3.dtsi b/arch/arm/boot/dts/ge863-pro3.dtsi
index 6a9fdc0760f0..a7adbdc2f0c0 100644
--- a/arch/arm/boot/dts/ge863-pro3.dtsi
+++ b/arch/arm/boot/dts/ge863-pro3.dtsi
@@ -18,6 +18,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			dbgu: serial@fffff200 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/gemini-dlink-dir-685.dts b/arch/arm/boot/dts/gemini-dlink-dir-685.dts
index e75e2d44371c..cadde92bc6b5 100644
--- a/arch/arm/boot/dts/gemini-dlink-dir-685.dts
+++ b/arch/arm/boot/dts/gemini-dlink-dir-685.dts
@@ -45,6 +45,47 @@
 		};
 	};
 
+	vdisp: regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "display-power";
+		regulator-min-microvolt = <3600000>;
+		regulator-max-microvolt = <3600000>;
+		/* Collides with LCD E */
+		gpio = <&gpio0 16 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	spi {
+		compatible = "spi-gpio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* Collides with IDE pins, that's cool (we do not use them) */
+		gpio-sck = <&gpio1 5 GPIO_ACTIVE_HIGH>;
+		gpio-miso = <&gpio1 8 GPIO_ACTIVE_HIGH>;
+		gpio-mosi = <&gpio1 7 GPIO_ACTIVE_HIGH>;
+		/* Collides with pflash CE1, not so cool */
+		cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>;
+		num-chipselects = <1>;
+
+		panel: display@0 {
+			compatible = "dlink,dir-685-panel", "ilitek,ili9322";
+			reg = <0>;
+			/* 50 ns min period = 20 MHz */
+			spi-max-frequency = <20000000>;
+			spi-cpol; /* Clock active low */
+			vcc-supply = <&vdisp>;
+			iovcc-supply = <&vdisp>;
+			vci-supply = <&vdisp>;
+
+			port {
+				panel_in: endpoint {
+					remote-endpoint = <&display_out>;
+				};
+			};
+		};
+	};
+
 	leds {
 		compatible = "gpio-leds";
 		led-wps {
@@ -99,8 +140,8 @@
 	gpio-i2c {
 		compatible = "i2c-gpio";
 		/* Collides with ICE */
-		gpios = <&gpio0 5 0>, /* SDA */
-			<&gpio0 6 0>; /* SCL */
+		sda-gpios = <&gpio0 5 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio0 6 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 
@@ -115,7 +156,16 @@
 
 	soc {
 		flash@30000000 {
-			status = "okay";
+			/*
+			 * Flash access is by default disabled, because it
+			 * collides with the Chip Enable signal for the display
+			 * panel, that reuse the parallel flash Chip Select 1
+			 * (CS1). Enabling flash makes graphics stop working.
+			 *
+			 * We might be able to hack around this by letting
+			 * GPIO poke around in the flash controller registers.
+			 */
+			/* status = "okay"; */
 			/* 32MB of flash */
 			reg = <0x30000000 0x02000000>;
 
@@ -242,5 +292,16 @@
 		ata@63000000 {
 			status = "okay";
 		};
+
+		display-controller@6a000000 {
+			status = "okay";
+
+			port@0 {
+				reg = <0>;
+				display_out: endpoint {
+					remote-endpoint = <&panel_in>;
+				};
+			};
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/gemini-dlink-dns-313.dts b/arch/arm/boot/dts/gemini-dlink-dns-313.dts
new file mode 100644
index 000000000000..da8bb9d60f99
--- /dev/null
+++ b/arch/arm/boot/dts/gemini-dlink-dns-313.dts
@@ -0,0 +1,303 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree file for D-Link DNS-313 1-Bay Network Storage Enclosure
+ */
+
+/dts-v1/;
+
+#include "gemini.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/thermal/thermal.h>
+
+/ {
+	model = "D-Link DNS-313 1-Bay Network Storage Enclosure";
+	compatible = "dlink,dir-313", "cortina,gemini";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	memory {
+		/* 64 MB SDRAM in a Nanya NT5DS32M16BS-6K package */
+		device_type = "memory";
+		reg = <0x00000000 0x4000000>;
+	};
+
+	aliases {
+		mdio-gpio0 = &mdio0;
+	};
+
+	chosen {
+		stdout-path = "uart0:19200n8";
+	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		button-esc {
+			debounce_interval = <50>;
+			wakeup-source;
+			linux,code = <KEY_ESC>;
+			label = "reset";
+			gpios = <&gpio1 31 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		led-power {
+			label = "dns313:blue:power";
+			gpios = <&gpio0 1 GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+			linux,default-trigger = "heartbeat";
+		};
+		led-disk-blue {
+			label = "dns313:blue:disk";
+			gpios = <&gpio0 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+		led-disk-green {
+			label = "dns313:green:disk";
+			gpios = <&gpio0 3 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "ide-disk";
+			/* Ideally should activate while reading */
+		};
+		led-disk-red {
+			label = "dns313:red:disk";
+			gpios = <&gpio0 4 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			/* Ideally should activate while writing */
+		};
+	};
+
+	/*
+	 * This is a ADDA AD0405GB-G73 fan @3000 and 6000 RPM.
+	 */
+	fan0: gpio-fan {
+		compatible = "gpio-fan";
+		gpios = <&gpio0 11 GPIO_ACTIVE_HIGH>,
+			<&gpio0 12 GPIO_ACTIVE_HIGH>;
+		gpio-fan,speed-map = <0 0>, <3000 1>, <6000 2>;
+		cooling-min-level = <0>;
+		cooling-max-level = <2>;
+		#cooling-cells = <2>;
+	};
+
+
+	/* Global Mixed-Mode Technology G751 mounted on GPIO I2C */
+	gpio-i2c {
+		compatible = "i2c-gpio";
+		sda-gpios = <&gpio0 15 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio0 16 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		g751: temperature-sensor@48 {
+			compatible = "gmt,g751";
+			reg = <0x48>;
+			#thermal-sensor-cells = <0>;
+		};
+	};
+
+	thermal-zones {
+		chassis-thermal {
+			/* Poll every 20 seconds */
+			polling-delay = <20000>;
+			/* Poll every 2nd second when cooling */
+			polling-delay-passive = <2000>;
+
+			thermal-sensors = <&g751>;
+
+			/* Tripping points from the fan.script in the rootfs */
+			trips {
+				chassis_alert0: chassis-alert0 {
+					/* At 43 degrees turn on low speed */
+					temperature = <43000>;
+					hysteresis = <3000>;
+					type = "active";
+				};
+				chassis_alert1: chassis-alert1 {
+					/* At 47 degrees turn on high speed */
+					temperature = <47000>;
+					hysteresis = <3000>;
+					type = "active";
+				};
+				chassis_crit: chassis-crit {
+					/* Just shut down at 60 degrees */
+					temperature = <60000>;
+					hysteresis = <2000>;
+					type = "critical";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					trip = <&chassis_alert0>;
+					cooling-device = <&fan0 1 1>;
+				};
+				map1 {
+					trip = <&chassis_alert1>;
+					cooling-device = <&fan0 2 2>;
+				};
+			};
+		};
+	};
+
+	mdio0: ethernet-phy {
+		compatible = "virtual,mdio-gpio";
+		/* Uses MDC and MDIO */
+		gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>, /* MDC */
+			<&gpio0 21 GPIO_ACTIVE_HIGH>; /* MDIO */
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* This is a Realtek RTL8211B Gigabit ethernet transceiver */
+		phy0: ethernet-phy@1 {
+			reg = <1>;
+			device_type = "ethernet-phy";
+		};
+	};
+
+	soc {
+		flash@30000000 {
+			status = "okay";
+			/* 512KB of flash */
+			reg = <0x30000000 0x00080000>;
+
+			/*
+			 * This "RedBoot" is the Storlink derivative.
+			 */
+			partition@0 {
+				label = "RedBoot";
+				reg = <0x00000000 0x00040000>;
+				read-only;
+			};
+			partition@40000 {
+				label = "MTD1";
+				reg = <0x00040000 0x00020000>;
+				read-only;
+			};
+			partition@60000 {
+				label = "MTD2";
+				reg = <0x00060000 0x00020000>;
+				read-only;
+			};
+		};
+
+		syscon: syscon@40000000 {
+			pinctrl {
+				/*
+				 */
+				gpio0_default_pins: pinctrl-gpio0 {
+					mux {
+						function = "gpio0";
+						groups =
+						/* Used by LEDs conflicts ICE */
+						"gpio0bgrp",
+						/* Used by ? conflicts ICE */
+						"gpio0cgrp",
+						/*
+						 * Used by fan & G751, conflicts LPC,
+						 * UART modem lines, SSP
+						 */
+						"gpio0egrp",
+						/* Used by G751 */
+						"gpio0fgrp",
+						/* Used by MDIO */
+						"gpio0igrp";
+					};
+				};
+				gpio1_default_pins: pinctrl-gpio1 {
+					mux {
+						function = "gpio1";
+						/* Used by "reset" button */
+						groups = "gpio1dgrp";
+					};
+				};
+				pinctrl-gmii {
+					mux {
+						function = "gmii";
+						groups = "gmii_gmac0_grp";
+					};
+					/*
+					 * In the vendor Linux tree, these values are set for the C3
+					 * version of the SL3512 ASIC with the comment "benson suggest"
+					 */
+					conf0 {
+						pins = "R8 GMAC0 RXDV", "U11 GMAC1 RXDV";
+						skew-delay = <0>;
+					};
+					conf1 {
+						pins = "T8 GMAC0 RXC";
+						skew-delay = <10>;
+					};
+					conf2 {
+						pins = "T11 GMAC1 RXC";
+						skew-delay = <15>;
+					};
+					conf3 {
+						pins = "P8 GMAC0 TXEN", "V11 GMAC1 TXEN";
+						skew-delay = <7>;
+					};
+					conf4 {
+						pins = "V7 GMAC0 TXC", "P10 GMAC1 TXC";
+						skew-delay = <10>;
+					};
+					conf5 {
+						/* The data lines all have default skew */
+						pins = "U8 GMAC0 RXD0", "V8 GMAC0 RXD1",
+						       "P9 GMAC0 RXD2", "R9 GMAC0 RXD3",
+						       "R11 GMAC1 RXD0", "P11 GMAC1 RXD1",
+						       "V12 GMAC1 RXD2", "U12 GMAC1 RXD3",
+						       "R10 GMAC1 TXD0", "T10 GMAC1 TXD1",
+						       "U10 GMAC1 TXD2", "V10 GMAC1 TXD3";
+						skew-delay = <7>;
+					};
+					conf6 {
+						pins = "U7 GMAC0 TXD0", "T7 GMAC0 TXD1",
+						       "R7 GMAC0 TXD2", "P7 GMAC0 TXD3";
+						skew-delay = <5>;
+					};
+					/* Set up drive strength on GMAC0 to 16 mA */
+					conf7 {
+						groups = "gmii_gmac0_grp";
+						drive-strength = <16>;
+					};
+				};
+			};
+		};
+
+		sata: sata@46000000 {
+			/* The ROM uses this muxmode */
+			cortina,gemini-ata-muxmode = <0>;
+			cortina,gemini-enable-sata-bridge;
+			status = "okay";
+		};
+
+		gpio0: gpio@4d000000 {
+			pinctrl-names = "default";
+			pinctrl-0 = <&gpio0_default_pins>;
+		};
+
+		gpio1: gpio@4e000000 {
+			pinctrl-names = "default";
+			pinctrl-0 = <&gpio1_default_pins>;
+		};
+
+		ethernet@60000000 {
+			status = "okay";
+
+			ethernet-port@0 {
+				phy-mode = "rgmii";
+				phy-handle = <&phy0>;
+			};
+			ethernet-port@1 {
+				/* Not used in this platform */
+			};
+		};
+
+		ata@63000000 {
+			status = "okay";
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/gemini-nas4220b.dts b/arch/arm/boot/dts/gemini-nas4220b.dts
index d6a22e677c7a..8bbb6f85d161 100644
--- a/arch/arm/boot/dts/gemini-nas4220b.dts
+++ b/arch/arm/boot/dts/gemini-nas4220b.dts
@@ -64,6 +64,19 @@
 		};
 	};
 
+	mdio0: ethernet-phy {
+		compatible = "virtual,mdio-gpio";
+		gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>, /* MDC */
+			<&gpio0 21 GPIO_ACTIVE_HIGH>; /* MDIO */
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@1 {
+			reg = <1>;
+			device_type = "ethernet-phy";
+		};
+	};
+
 	soc {
 		flash@30000000 {
 			status = "okay";
@@ -116,6 +129,50 @@
 						groups = "gpio1dgrp";
 					};
 				};
+				pinctrl-gmii {
+					mux {
+						function = "gmii";
+						groups = "gmii_gmac0_grp";
+					};
+					/* Settings come from OpenWRT */
+					conf0 {
+						pins = "R8 GMAC0 RXDV", "U11 GMAC1 RXDV";
+						skew-delay = <0>;
+					};
+					conf1 {
+						pins = "T8 GMAC0 RXC", "T11 GMAC1 RXC";
+						skew-delay = <15>;
+					};
+					conf2 {
+						pins = "P8 GMAC0 TXEN", "V11 GMAC1 TXEN";
+						skew-delay = <7>;
+					};
+					conf3 {
+						pins = "V7 GMAC0 TXC";
+						skew-delay = <11>;
+					};
+					conf4 {
+						pins = "P10 GMAC1 TXC";
+						skew-delay = <10>;
+					};
+					conf5 {
+						/* The data lines all have default skew */
+						pins = "U8 GMAC0 RXD0", "V8 GMAC0 RXD1",
+						       "P9 GMAC0 RXD2", "R9 GMAC0 RXD3",
+						       "U7 GMAC0 TXD0", "T7 GMAC0 TXD1",
+						       "R7 GMAC0 TXD2", "P7 GMAC0 TXD3",
+						       "R11 GMAC1 RXD0", "P11 GMAC1 RXD1",
+						       "V12 GMAC1 RXD2", "U12 GMAC1 RXD3",
+						       "R10 GMAC1 TXD0", "T10 GMAC1 TXD1",
+						       "U10 GMAC1 TXD2", "V10 GMAC1 TXD3";
+						skew-delay = <7>;
+					};
+					/* Set up drive strength on GMAC0 to 16 mA */
+					conf6 {
+						groups = "gmii_gmac0_grp";
+						drive-strength = <16>;
+					};
+				};
 			};
 		};
 
@@ -130,6 +187,18 @@
 			pinctrl-0 = <&gpio1_default_pins>;
 		};
 
+		ethernet@60000000 {
+			status = "okay";
+
+			ethernet-port@0 {
+				phy-mode = "rgmii";
+				phy-handle = <&phy0>;
+			};
+			ethernet-port@1 {
+				/* Not used in this platform */
+			};
+		};
+
 		ata@63000000 {
 			status = "okay";
 		};
diff --git a/arch/arm/boot/dts/gemini-rut1xx.dts b/arch/arm/boot/dts/gemini-rut1xx.dts
index 500057b6570e..15f20178642c 100644
--- a/arch/arm/boot/dts/gemini-rut1xx.dts
+++ b/arch/arm/boot/dts/gemini-rut1xx.dts
@@ -58,6 +58,19 @@
 		};
 	};
 
+	mdio0: ethernet-phy {
+		compatible = "virtual,mdio-gpio";
+		gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>, /* MDC */
+			<&gpio0 21 GPIO_ACTIVE_HIGH>; /* MDIO */
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@1 {
+			reg = <1>;
+			device_type = "ethernet-phy";
+		};
+	};
+
 	soc {
 		flash@30000000 {
 			status = "okay";
@@ -101,5 +114,17 @@
 			pinctrl-names = "default";
 			pinctrl-0 = <&gpio1_default_pins>;
 		};
+
+		ethernet@60000000 {
+			status = "okay";
+
+			ethernet-port@0 {
+				phy-mode = "rgmii";
+				phy-handle = <&phy0>;
+			};
+			ethernet-port@1 {
+				/* Not used in this platform */
+			};
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/gemini-wbd111.dts b/arch/arm/boot/dts/gemini-wbd111.dts
index b413fd12c5ba..b4ec9ad85d72 100644
--- a/arch/arm/boot/dts/gemini-wbd111.dts
+++ b/arch/arm/boot/dts/gemini-wbd111.dts
@@ -69,6 +69,19 @@
 		};
 	};
 
+	mdio0: ethernet-phy {
+		compatible = "virtual,mdio-gpio";
+		gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>, /* MDC */
+			<&gpio0 21 GPIO_ACTIVE_HIGH>; /* MDIO */
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@1 {
+			reg = <1>;
+			device_type = "ethernet-phy";
+		};
+	};
+
 	soc {
 		flash@30000000 {
 			status = "okay";
@@ -125,5 +138,39 @@
 			pinctrl-names = "default";
 			pinctrl-0 = <&gpio0_default_pins>;
 		};
+
+		pci@50000000 {
+			status = "okay";
+			interrupt-map-mask = <0xf800 0 0 7>;
+			interrupt-map =
+				<0x4800 0 0 1 &pci_intc 0>, /* Slot 9 */
+				<0x4800 0 0 2 &pci_intc 1>,
+				<0x4800 0 0 3 &pci_intc 2>,
+				<0x4800 0 0 4 &pci_intc 3>,
+				<0x5000 0 0 1 &pci_intc 1>, /* Slot 10 */
+				<0x5000 0 0 2 &pci_intc 2>,
+				<0x5000 0 0 3 &pci_intc 3>,
+				<0x5000 0 0 4 &pci_intc 0>,
+				<0x5800 0 0 1 &pci_intc 2>, /* Slot 11 */
+				<0x5800 0 0 2 &pci_intc 3>,
+				<0x5800 0 0 3 &pci_intc 0>,
+				<0x5800 0 0 4 &pci_intc 1>,
+				<0x6000 0 0 1 &pci_intc 3>, /* Slot 12 */
+				<0x6000 0 0 2 &pci_intc 0>,
+				<0x6000 0 0 3 &pci_intc 1>,
+				<0x6000 0 0 4 &pci_intc 2>;
+		};
+
+		ethernet@60000000 {
+			status = "okay";
+
+			ethernet-port@0 {
+				phy-mode = "rgmii";
+				phy-handle = <&phy0>;
+			};
+			ethernet-port@1 {
+				/* Not used in this platform */
+			};
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/gemini-wbd222.dts b/arch/arm/boot/dts/gemini-wbd222.dts
index 3ba710538662..6d25bcc046e7 100644
--- a/arch/arm/boot/dts/gemini-wbd222.dts
+++ b/arch/arm/boot/dts/gemini-wbd222.dts
@@ -69,6 +69,24 @@
 		};
 	};
 
+	mdio0: ethernet-phy {
+		compatible = "virtual,mdio-gpio";
+		gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>, /* MDC */
+			<&gpio0 21 GPIO_ACTIVE_HIGH>; /* MDIO */
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@1 {
+			reg = <1>;
+			device_type = "ethernet-phy";
+		};
+
+		phy1: ethernet-phy@3 {
+			reg = <3>;
+			device_type = "ethernet-phy";
+		};
+	};
+
 	soc {
 		flash@30000000 {
 			status = "okay";
@@ -118,6 +136,13 @@
 						"gpio0bgrp";
 					};
 				};
+				pinctrl-gmii {
+					/* This platform use both the ethernet ports */
+					mux {
+						function = "gmii";
+						groups = "gmii_gmac0_grp", "gmii_gmac1_grp";
+					};
+				};
 			};
 		};
 
@@ -125,5 +150,40 @@
 			pinctrl-names = "default";
 			pinctrl-0 = <&gpio0_default_pins>;
 		};
+
+		pci@50000000 {
+			status = "okay";
+			interrupt-map-mask = <0xf800 0 0 7>;
+			interrupt-map =
+				<0x4800 0 0 1 &pci_intc 0>, /* Slot 9 */
+				<0x4800 0 0 2 &pci_intc 1>,
+				<0x4800 0 0 3 &pci_intc 2>,
+				<0x4800 0 0 4 &pci_intc 3>,
+				<0x5000 0 0 1 &pci_intc 1>, /* Slot 10 */
+				<0x5000 0 0 2 &pci_intc 2>,
+				<0x5000 0 0 3 &pci_intc 3>,
+				<0x5000 0 0 4 &pci_intc 0>,
+				<0x5800 0 0 1 &pci_intc 2>, /* Slot 11 */
+				<0x5800 0 0 2 &pci_intc 3>,
+				<0x5800 0 0 3 &pci_intc 0>,
+				<0x5800 0 0 4 &pci_intc 1>,
+				<0x6000 0 0 1 &pci_intc 3>, /* Slot 12 */
+				<0x6000 0 0 2 &pci_intc 0>,
+				<0x6000 0 0 3 &pci_intc 1>,
+				<0x6000 0 0 4 &pci_intc 2>;
+		};
+
+		ethernet@60000000 {
+			status = "okay";
+
+			ethernet-port@0 {
+				phy-mode = "rgmii";
+				phy-handle = <&phy0>;
+			};
+			ethernet-port@1 {
+				phy-mode = "rgmii";
+				phy-handle = <&phy1>;
+			};
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/gemini.dtsi b/arch/arm/boot/dts/gemini.dtsi
index cb5c925bd597..0568baca500a 100644
--- a/arch/arm/boot/dts/gemini.dtsi
+++ b/arch/arm/boot/dts/gemini.dtsi
@@ -114,9 +114,16 @@
 					};
 				};
 				gmii_default_pins: pinctrl-gmii {
+					/*
+					 * Only activate GMAC0 by default since
+					 * GMAC1 will overlap with 8 GPIO lines
+					 * gpio2a, gpio2b. Overlay groups with
+					 * "gmii_gmac0_grp", "gmii_gmac1_grp" for
+					 * both ethernet interfaces.
+					 */
 					mux {
 						function = "gmii";
-						groups = "gmiigrp";
+						groups = "gmii_gmac0_grp";
 					};
 				};
 				pci_default_pins: pinctrl-pci {
@@ -316,6 +323,41 @@
 			};
 		};
 
+		ethernet@60000000 {
+			compatible = "cortina,gemini-ethernet";
+			reg = <0x60000000 0x4000>, /* Global registers, queue */
+			      <0x60004000 0x2000>, /* V-bit */
+			      <0x60006000 0x2000>; /* A-bit */
+			pinctrl-names = "default";
+			pinctrl-0 = <&gmii_default_pins>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			gmac0: ethernet-port@0 {
+				compatible = "cortina,gemini-ethernet-port";
+				reg = <0x60008000 0x2000>, /* Port 0 DMA/TOE */
+				      <0x6000a000 0x2000>; /* Port 0 GMAC */
+				interrupt-parent = <&intcon>;
+				interrupts = <1 IRQ_TYPE_LEVEL_HIGH>;
+				resets = <&syscon GEMINI_RESET_GMAC0>;
+				clocks = <&syscon GEMINI_CLK_GATE_GMAC0>;
+				clock-names = "PCLK";
+			};
+
+			gmac1: ethernet-port@1 {
+				compatible = "cortina,gemini-ethernet-port";
+				reg = <0x6000c000 0x2000>, /* Port 1 DMA/TOE */
+				      <0x6000e000 0x2000>; /* Port 1 GMAC */
+				interrupt-parent = <&intcon>;
+				interrupts = <2 IRQ_TYPE_LEVEL_HIGH>;
+				resets = <&syscon GEMINI_RESET_GMAC1>;
+				clocks = <&syscon GEMINI_CLK_GATE_GMAC1>;
+				clock-names = "PCLK";
+			};
+		};
+
 		ata@63000000 {
 			compatible = "cortina,gemini-pata", "faraday,ftide010";
 			reg = <0x63000000 0x1000>;
diff --git a/arch/arm/boot/dts/imx25.dtsi b/arch/arm/boot/dts/imx25.dtsi
index 09ce8b81fafa..9445f8e1473c 100644
--- a/arch/arm/boot/dts/imx25.dtsi
+++ b/arch/arm/boot/dts/imx25.dtsi
@@ -122,7 +122,7 @@
 			};
 
 			can1: can@43f88000 {
-				compatible = "fsl,imx25-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx25-flexcan";
 				reg = <0x43f88000 0x4000>;
 				interrupts = <43>;
 				clocks = <&clks 75>, <&clks 75>;
@@ -131,7 +131,7 @@
 			};
 
 			can2: can@43f8c000 {
-				compatible = "fsl,imx25-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx25-flexcan";
 				reg = <0x43f8c000 0x4000>;
 				interrupts = <44>;
 				clocks = <&clks 76>, <&clks 76>;
@@ -628,11 +628,13 @@
 		usbphy0: usb-phy@0 {
 			reg = <0>;
 			compatible = "usb-nop-xceiv";
+			#phy-cells = <0>;
 		};
 
 		usbphy1: usb-phy@1 {
 			reg = <1>;
 			compatible = "usb-nop-xceiv";
+			#phy-cells = <0>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx27-pdk.dts b/arch/arm/boot/dts/imx27-pdk.dts
index 96f442ba6d22..2a140c8ae6d2 100644
--- a/arch/arm/boot/dts/imx27-pdk.dts
+++ b/arch/arm/boot/dts/imx27-pdk.dts
@@ -30,6 +30,7 @@
 			reg = <0>;
 			clocks = <&clks IMX27_CLK_DUMMY>;
 			clock-names = "main_clk";
+			#phy-cells = <0>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx27-phytec-phycore-rdk.dts b/arch/arm/boot/dts/imx27-phytec-phycore-rdk.dts
index 2a9198f99a8d..2ed2d73b087e 100644
--- a/arch/arm/boot/dts/imx27-phytec-phycore-rdk.dts
+++ b/arch/arm/boot/dts/imx27-phytec-phycore-rdk.dts
@@ -63,6 +63,7 @@
 			vcc-supply = <&reg_5v0>;
 			clocks = <&clks IMX27_CLK_DUMMY>;
 			clock-names = "main_clk";
+			#phy-cells = <0>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi b/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi
index c973c5d91875..c9095b7654c6 100644
--- a/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi
+++ b/arch/arm/boot/dts/imx27-phytec-phycore-som.dtsi
@@ -53,6 +53,7 @@
 			vcc-supply = <&sw3_reg>;
 			clocks = <&clks IMX27_CLK_DUMMY>;
 			clock-names = "main_clk";
+			#phy-cells = <0>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx28-ts4600.dts b/arch/arm/boot/dts/imx28-ts4600.dts
new file mode 100644
index 000000000000..1e391c9f1b7a
--- /dev/null
+++ b/arch/arm/boot/dts/imx28-ts4600.dts
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2016 Savoir-Faire Linux
+ * Author: Sebastien Bourdelin <sebastien.bourdelin@savoirfairelinux.com>
+ *
+ * The code contained herein is licensed under the GNU General Public
+ * License. You may obtain a copy of the GNU General Public License
+ * Version 2 or later at the following locations:
+ *
+ * http://www.opensource.org/licenses/gpl-license.html
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+/dts-v1/;
+#include "imx28.dtsi"
+#include "dt-bindings/gpio/gpio.h"
+
+/ {
+
+	model = "Technologic Systems i.MX28 TS-4600";
+	compatible = "technologic,imx28-ts4600", "fsl,imx28";
+
+	memory {
+		reg = <0x40000000 0x10000000>;   /* 256MB */
+	};
+
+	apb@80000000 {
+		apbh@80000000 {
+			ssp0: ssp@80010000 {
+				compatible = "fsl,imx28-mmc";
+				pinctrl-names = "default";
+				pinctrl-0 = <&mmc0_4bit_pins_a
+					     &mmc0_sck_cfg
+					     &en_sd_pwr>;
+				broken-cd = <1>;
+				bus-width = <4>;
+				vmmc-supply = <&reg_vddio_sd0>;
+				status = "okay";
+			};
+
+			pinctrl@80018000 {
+
+				en_sd_pwr: en-sd-pwr@0 {
+					reg = <0>;
+					fsl,pinmux-ids = <
+						MX28_PAD_PWM3__GPIO_3_28
+					>;
+					fsl,drive-strength = <MXS_DRIVE_4mA>;
+					fsl,voltage = <MXS_VOLTAGE_HIGH>;
+					fsl,pull-up = <MXS_PULL_DISABLE>;
+				};
+
+			};
+		};
+
+		apbx@80040000 {
+			pwm: pwm@80064000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&pwm2_pins_a>;
+				status = "okay";
+			};
+
+			duart: serial@80074000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&duart_pins_a>;
+				status = "okay";
+			};
+		};
+	};
+
+	reg_vddio_sd0: regulator-vddio-sd0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vddio-sd0";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		gpio = <&gpio3 28 GPIO_ACTIVE_LOW>;
+	};
+
+};
diff --git a/arch/arm/boot/dts/imx28.dtsi b/arch/arm/boot/dts/imx28.dtsi
index 2f4ebe0318d3..e52e05c0fe56 100644
--- a/arch/arm/boot/dts/imx28.dtsi
+++ b/arch/arm/boot/dts/imx28.dtsi
@@ -1038,7 +1038,7 @@
 			};
 
 			can0: can@80032000 {
-				compatible = "fsl,imx28-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx28-flexcan";
 				reg = <0x80032000 0x2000>;
 				interrupts = <8>;
 				clocks = <&clks 58>, <&clks 58>;
@@ -1047,7 +1047,7 @@
 			};
 
 			can1: can@80034000 {
-				compatible = "fsl,imx28-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx28-flexcan";
 				reg = <0x80034000 0x2000>;
 				interrupts = <9>;
 				clocks = <&clks 59>, <&clks 59>;
diff --git a/arch/arm/boot/dts/imx35.dtsi b/arch/arm/boot/dts/imx35.dtsi
index 6d5e6a60bee7..e08c0c193767 100644
--- a/arch/arm/boot/dts/imx35.dtsi
+++ b/arch/arm/boot/dts/imx35.dtsi
@@ -303,7 +303,7 @@
 			};
 
 			can1: can@53fe4000 {
-				compatible = "fsl,imx35-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx35-flexcan";
 				reg = <0x53fe4000 0x1000>;
 				clocks = <&clks 33>, <&clks 33>;
 				clock-names = "ipg", "per";
@@ -312,7 +312,7 @@
 			};
 
 			can2: can@53fe8000 {
-				compatible = "fsl,imx35-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx35-flexcan";
 				reg = <0x53fe8000 0x1000>;
 				clocks = <&clks 34>, <&clks 34>;
 				clock-names = "ipg", "per";
@@ -402,11 +402,13 @@
 		usbphy0: usb-phy@0 {
 			reg = <0>;
 			compatible = "usb-nop-xceiv";
+			#phy-cells = <0>;
 		};
 
 		usbphy1: usb-phy@1 {
 			reg = <1>;
 			compatible = "usb-nop-xceiv";
+			#phy-cells = <0>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx51-babbage.dts b/arch/arm/boot/dts/imx51-babbage.dts
index 2a694c5cc8ae..4ac5ab614a7f 100644
--- a/arch/arm/boot/dts/imx51-babbage.dts
+++ b/arch/arm/boot/dts/imx51-babbage.dts
@@ -25,18 +25,41 @@
 		reg = <0x90000000 0x20000000>;
 	};
 
-	clocks {
-		ckih1 {
-			clock-frequency = <22579200>;
-		};
+	ckih1 {
+		clock-frequency = <22579200>;
+	};
 
-		clk_26M: codec_clock {
-			compatible = "fixed-clock";
-			reg=<0>;
-			#clock-cells = <0>;
-			clock-frequency = <26000000>;
-			gpios = <&gpio4 26 GPIO_ACTIVE_LOW>;
-		};
+	clk_osc: clk-osc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
+	};
+
+	clk_osc_gate: clk-osc-gate {
+		compatible = "gpio-gate-clock";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_clk26mhz_osc>;
+		clocks = <&clk_osc>;
+		#clock-cells = <0>;
+		enable-gpios = <&gpio3 1 GPIO_ACTIVE_HIGH>;
+	};
+
+	clk_audio: clk-audio {
+		compatible = "gpio-gate-clock";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_clk26mhz_audio>;
+		clocks = <&clk_osc_gate>;
+		#clock-cells = <0>;
+		enable-gpios = <&gpio4 26 GPIO_ACTIVE_LOW>;
+	};
+
+	clk_usb: clk-usb {
+		compatible = "gpio-gate-clock";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_clk26mhz_usb>;
+		clocks = <&clk_osc_gate>;
+		#clock-cells = <0>;
+		enable-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>;
 	};
 
 	display1: disp1 {
@@ -162,9 +185,11 @@
 		usbh1phy: usbh1phy@0 {
 			compatible = "usb-nop-xceiv";
 			reg = <0>;
-			clocks = <&clks IMX5_CLK_DUMMY>;
+			clocks = <&clk_usb>;
 			clock-names = "main_clk";
 			reset-gpios = <&gpio2 5 GPIO_ACTIVE_LOW>;
+			vcc-supply = <&vusb_reg>;
+			#phy-cells = <0>;
 		};
 	};
 };
@@ -240,6 +265,10 @@
 				regulator-max-microvolt = <3150000>;
 			};
 
+			vusb_reg: vusb {
+				regulator-boot-on;
+			};
+
 			vusb2_reg: vusb2 {
 				regulator-min-microvolt = <2400000>;
 				regulator-max-microvolt = <2775000>;
@@ -339,10 +368,8 @@
 
 	sgtl5000: codec@a {
 		compatible = "fsl,sgtl5000";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_clkcodec>;
 		reg = <0x0a>;
-		clocks = <&clk_26M>;
+		clocks = <&clk_audio>;
 		VDDA-supply = <&vdig_reg>;
 		VDDIO-supply = <&vvideo_reg>;
 	};
@@ -413,6 +440,10 @@
 	status = "okay";
 };
 
+&usbphy0 {
+	vcc-supply = <&vusb_reg>;
+};
+
 &usbotg {
 	dr_mode = "otg";
 	disable-over-current;
@@ -431,9 +462,21 @@
 			>;
 		};
 
-		pinctrl_clkcodec: clkcodecgrp {
+		pinctrl_clk26mhz_audio: clk26mhzaudiocgrp {
+			fsl,pins = <
+				MX51_PAD_CSPI1_RDY__GPIO4_26		0x85
+			>;
+		};
+
+		pinctrl_clk26mhz_osc: clk26mhzoscgrp {
+			fsl,pins = <
+				MX51_PAD_DI1_PIN12__GPIO3_1		0x85
+			>;
+		};
+
+		pinctrl_clk26mhz_usb: clk26mhzusbgrp {
 			fsl,pins = <
-				MX51_PAD_CSPI1_RDY__GPIO4_26		0x80000000
+				MX51_PAD_EIM_D17__GPIO2_1		0x85
 			>;
 		};
 
diff --git a/arch/arm/boot/dts/imx51-eukrea-mbimxsd51-baseboard.dts b/arch/arm/boot/dts/imx51-eukrea-mbimxsd51-baseboard.dts
index 1305b05c7ed9..b3d952f37cdc 100644
--- a/arch/arm/boot/dts/imx51-eukrea-mbimxsd51-baseboard.dts
+++ b/arch/arm/boot/dts/imx51-eukrea-mbimxsd51-baseboard.dts
@@ -94,6 +94,7 @@
 			clocks = <&clks IMX5_CLK_USB_PHY_GATE>;
 			clock-names = "main_clk";
 			clock-frequency = <19200000>;
+			#phy-cells = <0>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx51-ts4800.dts b/arch/arm/boot/dts/imx51-ts4800.dts
index 564233e97412..f59b02bae68d 100644
--- a/arch/arm/boot/dts/imx51-ts4800.dts
+++ b/arch/arm/boot/dts/imx51-ts4800.dts
@@ -149,18 +149,18 @@
 		#size-cells = <1>;
 		ranges = <0 0 0 0x1d000>;
 
-		syscon: syscon@b0010000 {
+		syscon: syscon@10000 {
 			compatible = "syscon", "simple-mfd";
 			reg = <0x10000 0x3d>;
 			reg-io-width = <2>;
 
-			wdt@e {
+			wdt {
 				compatible = "technologic,ts4800-wdt";
 				syscon = <&syscon 0xe>;
 			};
 		};
 
-		touchscreen {
+		touchscreen@12000 {
 			compatible = "technologic,ts4800-ts";
 			reg = <0x12000 0x1000>;
 			syscon = <&syscon 0x10 6>;
diff --git a/arch/arm/boot/dts/imx51-zii-rdu1.dts b/arch/arm/boot/dts/imx51-zii-rdu1.dts
index 49be0e1c812d..5306b78de0ca 100644
--- a/arch/arm/boot/dts/imx51-zii-rdu1.dts
+++ b/arch/arm/boot/dts/imx51-zii-rdu1.dts
@@ -242,7 +242,7 @@
 
 	sound {
 		compatible = "simple-audio-card";
-		simple-audio-card,name = "RDU1 audio";
+		simple-audio-card,name = "Front";
 		simple-audio-card,format = "i2s";
 		simple-audio-card,bitclock-master = <&sound_codec>;
 		simple-audio-card,frame-master = <&sound_codec>;
@@ -251,7 +251,7 @@
 		simple-audio-card,routing =
 			"Headphone Jack", "HPLEFT",
 			"Headphone Jack", "HPRIGHT";
-		simple-audio-card,aux-devs = <&tpa6130a2>;
+		simple-audio-card,aux-devs = <&hpa1>;
 
 		sound_cpu: simple-audio-card,cpu {
 			sound-dai = <&ssi2>;
@@ -271,6 +271,7 @@
 		clock-names = "main_clk";
 		reset-gpios = <&gpio4 8 GPIO_ACTIVE_LOW>;
 		vcc-supply = <&vusb_reg>;
+		#phy-cells = <0>;
 	};
 
 	usbh2phy: usbphy2 {
@@ -281,6 +282,7 @@
 		clock-names = "main_clk";
 		reset-gpios = <&gpio4 7 GPIO_ACTIVE_LOW>;
 		vcc-supply = <&vusb_reg>;
+		#phy-cells = <0>;
 	};
 };
 
@@ -428,13 +430,13 @@
 			#size-cells = <0>;
 			led-control = <0x0 0x0 0x3f83f8 0x0>;
 
-			sysled0 {
+			sysled0@3 {
 				reg = <3>;
 				label = "system:green:status";
 				linux,default-trigger = "default-on";
 			};
 
-			sysled1 {
+			sysled1@4 {
 				reg = <4>;
 				label = "system:green:act";
 				linux,default-trigger = "heartbeat";
@@ -479,7 +481,7 @@
 		reg = <0x50>;
 	};
 
-	tpa6130a2: amp@60 {
+	hpa1: amp@60 {
 		compatible = "ti,tpa6130a2";
 		reg = <0x60>;
 		pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index 378be720b3c7..00d30bd70068 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -113,6 +113,7 @@
 			reg = <0>;
 			clocks = <&clks IMX5_CLK_USB_PHY_GATE>;
 			clock-names = "main_clk";
+			#phy-cells = <0>;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/imx53-cx9020.dts b/arch/arm/boot/dts/imx53-cx9020.dts
index 4f54fd4418a3..5e67e43004e7 100644
--- a/arch/arm/boot/dts/imx53-cx9020.dts
+++ b/arch/arm/boot/dts/imx53-cx9020.dts
@@ -152,7 +152,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_fec>;
 	phy-mode = "rmii";
-	phy-reset-gpios = <&gpio7 6 GPIO_ACTIVE_HIGH>;
+	phy-reset-gpios = <&gpio7 6 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/imx53-m53.dtsi b/arch/arm/boot/dts/imx53-m53.dtsi
index ec390aa562c3..7ce69c63510c 100644
--- a/arch/arm/boot/dts/imx53-m53.dtsi
+++ b/arch/arm/boot/dts/imx53-m53.dtsi
@@ -64,7 +64,6 @@
 
 		stmpe_touchscreen {
 			compatible = "st,stmpe-ts";
-			reg = <0>;
 			st,sample-time = <4>;
 			st,mod-12b = <1>;
 			st,ref-sel = <0>;
diff --git a/arch/arm/boot/dts/imx53-tx53-x03x.dts b/arch/arm/boot/dts/imx53-tx53-x03x.dts
index 7eb53e48c2f4..fe15c9555d6e 100644
--- a/arch/arm/boot/dts/imx53-tx53-x03x.dts
+++ b/arch/arm/boot/dts/imx53-tx53-x03x.dts
@@ -53,132 +53,130 @@
 		display = &display;
 	};
 
-	soc {
-		display: disp0 {
-			compatible = "fsl,imx-parallel-display";
-			interface-pix-fmt = "rgb24";
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_rgb24_vga1>;
-			status = "okay";
+	display: disp0 {
+		compatible = "fsl,imx-parallel-display";
+		interface-pix-fmt = "rgb24";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rgb24_vga1>;
+		status = "okay";
 
-			port {
-				display0_in: endpoint {
-					remote-endpoint = <&ipu_di0_disp0>;
-				};
+		port {
+			display0_in: endpoint {
+				remote-endpoint = <&ipu_di0_disp0>;
 			};
+		};
 
-			display-timings {
-				VGA {
-					clock-frequency = <25200000>;
-					hactive = <640>;
-					vactive = <480>;
-					hback-porch = <48>;
-					hsync-len = <96>;
-					hfront-porch = <16>;
-					vback-porch = <31>;
-					vsync-len = <2>;
-					vfront-porch = <12>;
-					hsync-active = <0>;
-					vsync-active = <0>;
-					de-active = <1>;
-					pixelclk-active = <0>;
-				};
+		display-timings {
+			VGA {
+				clock-frequency = <25200000>;
+				hactive = <640>;
+				vactive = <480>;
+				hback-porch = <48>;
+				hsync-len = <96>;
+				hfront-porch = <16>;
+				vback-porch = <31>;
+				vsync-len = <2>;
+				vfront-porch = <12>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				de-active = <1>;
+				pixelclk-active = <0>;
+			};
 
-				ETV570 {
-					clock-frequency = <25200000>;
-					hactive = <640>;
-					vactive = <480>;
-					hback-porch = <114>;
-					hsync-len = <30>;
-					hfront-porch = <16>;
-					vback-porch = <32>;
-					vsync-len = <3>;
-					vfront-porch = <10>;
-					hsync-active = <0>;
-					vsync-active = <0>;
-					de-active = <1>;
-					pixelclk-active = <0>;
-				};
+			ETV570 {
+				clock-frequency = <25200000>;
+				hactive = <640>;
+				vactive = <480>;
+				hback-porch = <114>;
+				hsync-len = <30>;
+				hfront-porch = <16>;
+				vback-porch = <32>;
+				vsync-len = <3>;
+				vfront-porch = <10>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				de-active = <1>;
+				pixelclk-active = <0>;
+			};
 
-				ET0350 {
-					clock-frequency = <6413760>;
-					hactive = <320>;
-					vactive = <240>;
-					hback-porch = <34>;
-					hsync-len = <34>;
-					hfront-porch = <20>;
-					vback-porch = <15>;
-					vsync-len = <3>;
-					vfront-porch = <4>;
-					hsync-active = <0>;
-					vsync-active = <0>;
-					de-active = <1>;
-					pixelclk-active = <0>;
-				};
+			ET0350 {
+				clock-frequency = <6413760>;
+				hactive = <320>;
+				vactive = <240>;
+				hback-porch = <34>;
+				hsync-len = <34>;
+				hfront-porch = <20>;
+				vback-porch = <15>;
+				vsync-len = <3>;
+				vfront-porch = <4>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				de-active = <1>;
+				pixelclk-active = <0>;
+			};
 
-				ET0430 {
-					clock-frequency = <9009000>;
-					hactive = <480>;
-					vactive = <272>;
-					hback-porch = <2>;
-					hsync-len = <41>;
-					hfront-porch = <2>;
-					vback-porch = <2>;
-					vsync-len = <10>;
-					vfront-porch = <2>;
-					hsync-active = <0>;
-					vsync-active = <0>;
-					de-active = <1>;
-					pixelclk-active = <1>;
-				};
+			ET0430 {
+				clock-frequency = <9009000>;
+				hactive = <480>;
+				vactive = <272>;
+				hback-porch = <2>;
+				hsync-len = <41>;
+				hfront-porch = <2>;
+				vback-porch = <2>;
+				vsync-len = <10>;
+				vfront-porch = <2>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				de-active = <1>;
+				pixelclk-active = <1>;
+			};
 
-				ET0500 {
-					clock-frequency = <33264000>;
-					hactive = <800>;
-					vactive = <480>;
-					hback-porch = <88>;
-					hsync-len = <128>;
-					hfront-porch = <40>;
-					vback-porch = <33>;
-					vsync-len = <2>;
-					vfront-porch = <10>;
-					hsync-active = <0>;
-					vsync-active = <0>;
-					de-active = <1>;
-					pixelclk-active = <0>;
-				};
+			ET0500 {
+				clock-frequency = <33264000>;
+				hactive = <800>;
+				vactive = <480>;
+				hback-porch = <88>;
+				hsync-len = <128>;
+				hfront-porch = <40>;
+				vback-porch = <33>;
+				vsync-len = <2>;
+				vfront-porch = <10>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				de-active = <1>;
+				pixelclk-active = <0>;
+			};
 
-				ET0700 { /* same as ET0500 */
-					clock-frequency = <33264000>;
-					hactive = <800>;
-					vactive = <480>;
-					hback-porch = <88>;
-					hsync-len = <128>;
-					hfront-porch = <40>;
-					vback-porch = <33>;
-					vsync-len = <2>;
-					vfront-porch = <10>;
-					hsync-active = <0>;
-					vsync-active = <0>;
-					de-active = <1>;
-					pixelclk-active = <0>;
-				};
+			ET0700 { /* same as ET0500 */
+				clock-frequency = <33264000>;
+				hactive = <800>;
+				vactive = <480>;
+				hback-porch = <88>;
+				hsync-len = <128>;
+				hfront-porch = <40>;
+				vback-porch = <33>;
+				vsync-len = <2>;
+				vfront-porch = <10>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				de-active = <1>;
+				pixelclk-active = <0>;
+			};
 
-				ETQ570 {
-					clock-frequency = <6596040>;
-					hactive = <320>;
-					vactive = <240>;
-					hback-porch = <38>;
-					hsync-len = <30>;
-					hfront-porch = <30>;
-					vback-porch = <16>;
-					vsync-len = <3>;
-					vfront-porch = <4>;
-					hsync-active = <0>;
-					vsync-active = <0>;
-					de-active = <1>;
-					pixelclk-active = <0>;
-				};
+			ETQ570 {
+				clock-frequency = <6596040>;
+				hactive = <320>;
+				vactive = <240>;
+				hback-porch = <38>;
+				hsync-len = <30>;
+				hfront-porch = <30>;
+				vback-porch = <16>;
+				vsync-len = <3>;
+				vfront-porch = <4>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				de-active = <1>;
+				pixelclk-active = <0>;
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/imx53-tx53.dtsi b/arch/arm/boot/dts/imx53-tx53.dtsi
index 71b58b6933e1..a22e461fc168 100644
--- a/arch/arm/boot/dts/imx53-tx53.dtsi
+++ b/arch/arm/boot/dts/imx53-tx53.dtsi
@@ -225,10 +225,16 @@
 	mac-address = [000000000000]; /* placeholder; will be overwritten by bootloader */
 	status = "okay";
 
-	phy0: ethernet-phy@0 {
-		interrupt-parent = <&gpio2>;
-		interrupts = <4 IRQ_TYPE_EDGE_FALLING>;
-		device_type = "ethernet-phy";
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+			interrupt-parent = <&gpio2>;
+			interrupts = <4 IRQ_TYPE_EDGE_FALLING>;
+			device_type = "ethernet-phy";
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi
index 84f17f7abb71..1040251f2951 100644
--- a/arch/arm/boot/dts/imx53.dtsi
+++ b/arch/arm/boot/dts/imx53.dtsi
@@ -116,6 +116,28 @@
 		};
 	};
 
+	pmu {
+		compatible = "arm,cortex-a8-pmu";
+		interrupt-parent = <&tzic>;
+		interrupts = <77>;
+	};
+
+	usbphy0: usbphy-0 {
+		compatible = "usb-nop-xceiv";
+		clocks = <&clks IMX5_CLK_USB_PHY1_GATE>;
+		clock-names = "main_clk";
+		#phy-cells = <0>;
+		status = "okay";
+	};
+
+	usbphy1: usbphy-1 {
+		compatible = "usb-nop-xceiv";
+		clocks = <&clks IMX5_CLK_USB_PHY2_GATE>;
+		clock-names = "main_clk";
+		#phy-cells = <0>;
+		status = "okay";
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -299,20 +321,6 @@
 				reg = <0x53f00000 0x60>;
 			};
 
-			usbphy0: usbphy-0 {
-				compatible = "usb-nop-xceiv";
-				clocks = <&clks IMX5_CLK_USB_PHY1_GATE>;
-				clock-names = "main_clk";
-				status = "okay";
-			};
-
-			usbphy1: usbphy-1 {
-				compatible = "usb-nop-xceiv";
-				clocks = <&clks IMX5_CLK_USB_PHY2_GATE>;
-				clock-names = "main_clk";
-				status = "okay";
-			};
-
 			usbotg: usb@53f80000 {
 				compatible = "fsl,imx53-usb", "fsl,imx27-usb";
 				reg = <0x53f80000 0x0200>;
@@ -433,6 +441,13 @@
 				clock-names = "ipg", "per";
 			};
 
+			srtc: rtc@53fa4000 {
+				compatible = "fsl,imx53-rtc";
+				reg = <0x53fa4000 0x4000>;
+				interrupts = <24>;
+				clocks = <&clks IMX5_CLK_SRTC_GATE>;
+			};
+
 			iomuxc: iomuxc@53fa8000 {
 				compatible = "fsl,imx53-iomuxc";
 				reg = <0x53fa8000 0x4000>;
@@ -536,7 +551,7 @@
 			};
 
 			can1: can@53fc8000 {
-				compatible = "fsl,imx53-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx53-flexcan";
 				reg = <0x53fc8000 0x4000>;
 				interrupts = <82>;
 				clocks = <&clks IMX5_CLK_CAN1_IPG_GATE>,
@@ -546,7 +561,7 @@
 			};
 
 			can2: can@53fcc000 {
-				compatible = "fsl,imx53-flexcan", "fsl,p1010-flexcan";
+				compatible = "fsl,imx53-flexcan";
 				reg = <0x53fcc000 0x4000>;
 				interrupts = <83>;
 				clocks = <&clks IMX5_CLK_CAN2_IPG_GATE>,
@@ -813,10 +828,5 @@
 			reg = <0xf8000000 0x20000>;
 			clocks = <&clks IMX5_CLK_OCRAM>;
 		};
-
-		pmu {
-			compatible = "arm,cortex-a8-pmu";
-			interrupts = <77>;
-		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx6dl-aristainetos_4.dts b/arch/arm/boot/dts/imx6dl-aristainetos_4.dts
index cc418cecabdb..3c9f4af9e9ff 100644
--- a/arch/arm/boot/dts/imx6dl-aristainetos_4.dts
+++ b/arch/arm/boot/dts/imx6dl-aristainetos_4.dts
@@ -31,35 +31,33 @@
 		reg = <0x10000000 0x40000000>;
 	};
 
-	soc {
-		display0: disp0 {
-			compatible = "fsl,imx-parallel-display";
-			interface-pix-fmt = "rgb24";
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_ipu_disp>;
-			status = "okay";
+	display0: disp0 {
+		compatible = "fsl,imx-parallel-display";
+		interface-pix-fmt = "rgb24";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ipu_disp>;
+		status = "okay";
 
-			display-timings {
-				480x800p60 {
-					native-mode;
-					clock-frequency = <30000000>;
-					hactive = <480>;
-					vactive = <800>;
-					hfront-porch = <59>;
-					hback-porch = <10>;
-					hsync-len = <10>;
-					vback-porch = <15>;
-					vfront-porch = <15>;
-					vsync-len = <15>;
-					hsync-active = <1>;
-					vsync-active = <1>;
-				};
+		display-timings {
+			480x800p60 {
+				native-mode;
+				clock-frequency = <30000000>;
+				hactive = <480>;
+				vactive = <800>;
+				hfront-porch = <59>;
+				hback-porch = <10>;
+				hsync-len = <10>;
+				vback-porch = <15>;
+				vfront-porch = <15>;
+				vsync-len = <15>;
+				hsync-active = <1>;
+				vsync-active = <1>;
 			};
+		};
 
-			port {
-				display0_in: endpoint {
-					remote-endpoint = <&ipu1_di0_disp0>;
-				};
+		port {
+			display0_in: endpoint {
+				remote-endpoint = <&ipu1_di0_disp0>;
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/imx6dl-aristainetos_7.dts b/arch/arm/boot/dts/imx6dl-aristainetos_7.dts
index 126ff964eded..96cd835ccbf6 100644
--- a/arch/arm/boot/dts/imx6dl-aristainetos_7.dts
+++ b/arch/arm/boot/dts/imx6dl-aristainetos_7.dts
@@ -20,34 +20,32 @@
 		reg = <0x10000000 0x40000000>;
 	};
 
-	soc {
-		display0: disp0 {
-			compatible = "fsl,imx-parallel-display";
-			interface-pix-fmt = "rgb24";
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_ipu_disp>;
-			status = "okay";
+	display0: disp0 {
+		compatible = "fsl,imx-parallel-display";
+		interface-pix-fmt = "rgb24";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ipu_disp>;
+		status = "okay";
 
-			display-timings {
-				800x480p60 {
-					native-mode;
-					clock-frequency = <33246000>;
-					hactive = <800>;
-					vactive = <480>;
-					hfront-porch = <88>;
-					hback-porch = <88>;
-					hsync-len = <80>;
-					vback-porch = <10>;
-					vfront-porch = <10>;
-					vsync-len = <25>;
-					vsync-active = <1>;
-				};
+		display-timings {
+			800x480p60 {
+				native-mode;
+				clock-frequency = <33246000>;
+				hactive = <800>;
+				vactive = <480>;
+				hfront-porch = <88>;
+				hback-porch = <88>;
+				hsync-len = <80>;
+				vback-porch = <10>;
+				vfront-porch = <10>;
+				vsync-len = <25>;
+				vsync-active = <1>;
 			};
+		};
 
-			port {
-				display0_in: endpoint {
-					remote-endpoint = <&ipu1_di0_disp0>;
-				};
+		port {
+			display0_in: endpoint {
+				remote-endpoint = <&ipu1_di0_disp0>;
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts b/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
index 5705ebee0595..dcf9206f3e0d 100644
--- a/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
+++ b/arch/arm/boot/dts/imx6dl-colibri-eval-v3.dts
@@ -163,7 +163,7 @@
 
 	/* M41T0M6 real time clock on carrier board */
 	rtc_i2c: rtc@68 {
-		compatible = "st,m41t00";
+		compatible = "st,m41t0";
 		reg = <0x68>;
 	};
 };
diff --git a/arch/arm/boot/dts/imx6dl-cubox-i-emmc-som-v15.dts b/arch/arm/boot/dts/imx6dl-cubox-i-emmc-som-v15.dts
new file mode 100644
index 000000000000..2b2fc360b865
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-cubox-i-emmc-som-v15.dts
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2014 Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-sr-som-emmc.dtsi"
+#include "imx6qdl-cubox-i.dtsi"
+
+/ {
+	model = "SolidRun Cubox-i Solo/DualLite (1.5som+emmc)";
+	compatible = "solidrun,cubox-i/dl", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl-cubox-i-som-v15.dts b/arch/arm/boot/dts/imx6dl-cubox-i-som-v15.dts
new file mode 100644
index 000000000000..e09c565d1d1f
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-cubox-i-som-v15.dts
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2014 Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-cubox-i.dtsi"
+
+/ {
+	model = "SolidRun Cubox-i Solo/DualLite (1.5som)";
+	compatible = "solidrun,cubox-i/dl", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl-cubox-i.dts b/arch/arm/boot/dts/imx6dl-cubox-i.dts
index f10a36b8647d..2b1b3e193f53 100644
--- a/arch/arm/boot/dts/imx6dl-cubox-i.dts
+++ b/arch/arm/boot/dts/imx6dl-cubox-i.dts
@@ -41,6 +41,8 @@
 /dts-v1/;
 
 #include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-brcm.dtsi"
 #include "imx6qdl-cubox-i.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/imx6dl-hummingboard-emmc-som-v15.dts b/arch/arm/boot/dts/imx6dl-hummingboard-emmc-som-v15.dts
new file mode 100644
index 000000000000..a63f742f20d9
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-hummingboard-emmc-som-v15.dts
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2014 Rabeeh Khoury (rabeeh@solid-run.com)
+ * Based on dt work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-sr-som-emmc.dtsi"
+#include "imx6qdl-hummingboard.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard Solo/DualLite (1.5som+emmc)";
+	compatible = "solidrun,hummingboard/dl", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl-hummingboard-som-v15.dts b/arch/arm/boot/dts/imx6dl-hummingboard-som-v15.dts
new file mode 100644
index 000000000000..66a06cf3cdf3
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-hummingboard-som-v15.dts
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2014 Rabeeh Khoury (rabeeh@solid-run.com)
+ * Based on dt work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-hummingboard.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard Solo/DualLite (1.5som)";
+	compatible = "solidrun,hummingboard/dl", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl-hummingboard.dts b/arch/arm/boot/dts/imx6dl-hummingboard.dts
index 39c2602fa87c..cbd02eb486e1 100644
--- a/arch/arm/boot/dts/imx6dl-hummingboard.dts
+++ b/arch/arm/boot/dts/imx6dl-hummingboard.dts
@@ -42,6 +42,8 @@
 /dts-v1/;
 
 #include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-brcm.dtsi"
 #include "imx6qdl-hummingboard.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/imx6dl-hummingboard2-emmc-som-v15.dts b/arch/arm/boot/dts/imx6dl-hummingboard2-emmc-som-v15.dts
new file mode 100644
index 000000000000..80313c13bcdb
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-hummingboard2-emmc-som-v15.dts
@@ -0,0 +1,55 @@
+/*
+ * Device Tree file for SolidRun HummingBoard2
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ * Based on work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-emmc.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-hummingboard2.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard2 Solo/DualLite (1.5som+emmc)";
+	compatible = "solidrun,hummingboard2/dl", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl-hummingboard2-som-v15.dts b/arch/arm/boot/dts/imx6dl-hummingboard2-som-v15.dts
new file mode 100644
index 000000000000..e61ef1156f8b
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-hummingboard2-som-v15.dts
@@ -0,0 +1,54 @@
+/*
+ * Device Tree file for SolidRun HummingBoard2
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ * Based on work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-hummingboard2.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard2 Solo/DualLite (1.5som)";
+	compatible = "solidrun,hummingboard2/dl", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl-hummingboard2.dts b/arch/arm/boot/dts/imx6dl-hummingboard2.dts
new file mode 100644
index 000000000000..b12cd87f3f94
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-hummingboard2.dts
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ * Based on dt work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6dl.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-brcm.dtsi"
+#include "imx6qdl-hummingboard2.dtsi"
+#include "imx6qdl-hummingboard2-emmc.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard2 Solo/DualLite";
+	compatible = "solidrun,hummingboard2/dl", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl-icore-rqs.dts b/arch/arm/boot/dts/imx6dl-icore-rqs.dts
index cf42c2f5cdc7..1281bc39b7ab 100644
--- a/arch/arm/boot/dts/imx6dl-icore-rqs.dts
+++ b/arch/arm/boot/dts/imx6dl-icore-rqs.dts
@@ -42,7 +42,7 @@
 
 /dts-v1/;
 
-#include "imx6q.dtsi"
+#include "imx6dl.dtsi"
 #include "imx6qdl-icore-rqs.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/imx6dl-pinfunc.h b/arch/arm/boot/dts/imx6dl-pinfunc.h
index 0ead323fdbd2..37e430a261de 100644
--- a/arch/arm/boot/dts/imx6dl-pinfunc.h
+++ b/arch/arm/boot/dts/imx6dl-pinfunc.h
@@ -668,6 +668,7 @@
 #define MX6QDL_PAD_ENET_RX_ER__SPDIF_IN             0x1f4 0x5c4 0x8f0 0x3 0x1
 #define MX6QDL_PAD_ENET_RX_ER__ENET_1588_EVENT2_OUT 0x1f4 0x5c4 0x000 0x4 0x0
 #define MX6QDL_PAD_ENET_RX_ER__GPIO1_IO24           0x1f4 0x5c4 0x000 0x5 0x0
+#define MX6QDL_PAD_ENET_RXD0__OSC32K_32K_OUT        0x1f8 0x5c8 0x000 0x0 0x0
 #define MX6QDL_PAD_ENET_RXD0__ENET_RX_DATA0         0x1f8 0x5c8 0x818 0x1 0x0
 #define MX6QDL_PAD_ENET_RXD0__ESAI_TX_HF_CLK        0x1f8 0x5c8 0x838 0x2 0x0
 #define MX6QDL_PAD_ENET_RXD0__SPDIF_OUT             0x1f8 0x5c8 0x000 0x3 0x0
diff --git a/arch/arm/boot/dts/imx6dl-ts7970.dts b/arch/arm/boot/dts/imx6dl-ts7970.dts
new file mode 100644
index 000000000000..d104daf305d9
--- /dev/null
+++ b/arch/arm/boot/dts/imx6dl-ts7970.dts
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2015 Technologic Systems
+ * Copyright 2017 Savoir-faire Linux
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include "imx6dl.dtsi"
+#include "imx6qdl-ts7970.dtsi"
+
+/ {
+	model = "Technologic Systems i.MX6 Solo/DualLite TS-7970 (Default Device Tree)";
+	compatible = "technologic,imx6dl-ts7970", "fsl,imx6dl";
+};
diff --git a/arch/arm/boot/dts/imx6dl.dtsi b/arch/arm/boot/dts/imx6dl.dtsi
index 4d693a75ce98..c01674fa098a 100644
--- a/arch/arm/boot/dts/imx6dl.dtsi
+++ b/arch/arm/boot/dts/imx6dl.dtsi
@@ -170,7 +170,7 @@
 };
 
 &gpr {
-	ipu1_csi0_mux: ipu1_csi0_mux@34 {
+	ipu1_csi0_mux {
 		compatible = "video-mux";
 		mux-controls = <&mux 0>;
 		#address-cells = <1>;
@@ -224,7 +224,7 @@
 		};
 	};
 
-	ipu1_csi1_mux: ipu1_csi1_mux@34 {
+	ipu1_csi1_mux {
 		compatible = "video-mux";
 		mux-controls = <&mux 1>;
 		#address-cells = <1>;
@@ -309,10 +309,12 @@
 		#size-cells = <0>;
 
 		mipi_vc0_to_ipu1_csi0_mux: endpoint@0 {
+			reg = <0>;
 			remote-endpoint = <&ipu1_csi0_mux_from_mipi_vc0>;
 		};
 
 		mipi_vc0_to_ipu1_csi1_mux: endpoint@1 {
+			reg = <1>;
 			remote-endpoint = <&ipu1_csi1_mux_from_mipi_vc0>;
 		};
 	};
@@ -323,10 +325,12 @@
 		#size-cells = <0>;
 
 		mipi_vc1_to_ipu1_csi0_mux: endpoint@0 {
+			reg = <0>;
 			remote-endpoint = <&ipu1_csi0_mux_from_mipi_vc1>;
 		};
 
 		mipi_vc1_to_ipu1_csi1_mux: endpoint@1 {
+			reg = <1>;
 			remote-endpoint = <&ipu1_csi1_mux_from_mipi_vc1>;
 		};
 	};
@@ -337,10 +341,12 @@
 		#size-cells = <0>;
 
 		mipi_vc2_to_ipu1_csi0_mux: endpoint@0 {
+			reg = <0>;
 			remote-endpoint = <&ipu1_csi0_mux_from_mipi_vc2>;
 		};
 
 		mipi_vc2_to_ipu1_csi1_mux: endpoint@1 {
+			reg = <1>;
 			remote-endpoint = <&ipu1_csi1_mux_from_mipi_vc2>;
 		};
 	};
@@ -351,10 +357,12 @@
 		#size-cells = <0>;
 
 		mipi_vc3_to_ipu1_csi0_mux: endpoint@0 {
+			reg = <0>;
 			remote-endpoint = <&ipu1_csi0_mux_from_mipi_vc3>;
 		};
 
 		mipi_vc3_to_ipu1_csi1_mux: endpoint@1 {
+			reg = <1>;
 			remote-endpoint = <&ipu1_csi1_mux_from_mipi_vc3>;
 		};
 	};
diff --git a/arch/arm/boot/dts/imx6q-apalis-eval.dts b/arch/arm/boot/dts/imx6q-apalis-eval.dts
index 8b56656e53da..aa0e631f3c0a 100644
--- a/arch/arm/boot/dts/imx6q-apalis-eval.dts
+++ b/arch/arm/boot/dts/imx6q-apalis-eval.dts
@@ -158,7 +158,7 @@
 
 	/* M41T0M6 real time clock on carrier board */
 	rtc_i2c: rtc@68 {
-		compatible = "st,m41t00";
+		compatible = "st,m41t0";
 		reg = <0x68>;
 	};
 };
diff --git a/arch/arm/boot/dts/imx6q-apalis-ixora-v1.1.dts b/arch/arm/boot/dts/imx6q-apalis-ixora-v1.1.dts
index 27dc0fc686a9..e8dccf552122 100644
--- a/arch/arm/boot/dts/imx6q-apalis-ixora-v1.1.dts
+++ b/arch/arm/boot/dts/imx6q-apalis-ixora-v1.1.dts
@@ -170,7 +170,7 @@
 
 	/* M41T0M6 real time clock on carrier board */
 	rtc_i2c: rtc@68 {
-		compatible = "st,m41t00";
+		compatible = "st,m41t0";
 		reg = <0x68>;
 	};
 };
diff --git a/arch/arm/boot/dts/imx6q-apalis-ixora.dts b/arch/arm/boot/dts/imx6q-apalis-ixora.dts
index 40b2c67fe7af..6831dfd24cc1 100644
--- a/arch/arm/boot/dts/imx6q-apalis-ixora.dts
+++ b/arch/arm/boot/dts/imx6q-apalis-ixora.dts
@@ -174,7 +174,7 @@
 
 	/* M41T0M6 real time clock on carrier board */
 	rtc_i2c: rtc@68 {
-		compatible = "st,m41t00";
+		compatible = "st,m41t0";
 		reg = <0x68>;
 	};
 };
diff --git a/arch/arm/boot/dts/imx6q-b450v3.dts b/arch/arm/boot/dts/imx6q-b450v3.dts
index 404a93d9596b..3ec58500e9c2 100644
--- a/arch/arm/boot/dts/imx6q-b450v3.dts
+++ b/arch/arm/boot/dts/imx6q-b450v3.dts
@@ -112,3 +112,55 @@
                 line-name = "PCA9539-P07";
         };
 };
+
+&pci_root {
+	/* Intel Corporation I210 Gigabit Network Connection */
+	switch_nic: ethernet@3,0 {
+		compatible = "pci8086,1533";
+		reg = <0x00010000 0 0 0 0>;
+	};
+};
+
+&switch_ports {
+	port@0 {
+		reg = <0>;
+		label = "enacq";
+		phy-handle = <&switchphy0>;
+	};
+
+	port@1 {
+		reg = <1>;
+		label = "eneport1";
+		phy-handle = <&switchphy1>;
+	};
+
+	port@2 {
+		reg = <2>;
+		label = "enix";
+		phy-handle = <&switchphy2>;
+	};
+
+	port@3 {
+		reg = <3>;
+		label = "enid";
+		phy-handle = <&switchphy3>;
+	};
+
+	port@4 {
+		reg = <4>;
+		label = "cpu";
+		ethernet = <&switch_nic>;
+		phy-handle = <&switchphy4>;
+	};
+
+	port@5 {
+		reg = <5>;
+		label = "enembc";
+
+		/* connected to Ethernet MAC of AT91RM9200 in MII mode */
+		fixed-link {
+			speed = <100>;
+			full-duplex;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/imx6q-b650v3.dts b/arch/arm/boot/dts/imx6q-b650v3.dts
index 7f9f176901d4..5650a9b11091 100644
--- a/arch/arm/boot/dts/imx6q-b650v3.dts
+++ b/arch/arm/boot/dts/imx6q-b650v3.dts
@@ -111,3 +111,55 @@
 	fsl,tx-cal-45-dp-ohms = <55>;
 	fsl,tx-d-cal = <100>;
 };
+
+&pci_root {
+	/* Intel Corporation I210 Gigabit Network Connection */
+	switch_nic: ethernet@3,0 {
+		compatible = "pci8086,1533";
+		reg = <0x00010000 0 0 0 0>;
+	};
+};
+
+&switch_ports {
+	port@0 {
+		reg = <0>;
+		label = "enacq";
+		phy-handle = <&switchphy0>;
+	};
+
+	port@1 {
+		reg = <1>;
+		label = "eneport1";
+		phy-handle = <&switchphy1>;
+	};
+
+	port@2 {
+		reg = <2>;
+		label = "enix";
+		phy-handle = <&switchphy2>;
+	};
+
+	port@3 {
+		reg = <3>;
+		label = "enid";
+		phy-handle = <&switchphy3>;
+	};
+
+	port@4 {
+		reg = <4>;
+		label = "cpu";
+		ethernet = <&switch_nic>;
+		phy-handle = <&switchphy4>;
+	};
+
+	port@5 {
+		reg = <5>;
+		label = "enembc";
+
+		/* connected to Ethernet MAC of AT91RM9200 in MII mode */
+		fixed-link {
+			speed = <100>;
+			full-duplex;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/imx6q-b850v3.dts b/arch/arm/boot/dts/imx6q-b850v3.dts
index 46bdc6722715..35edbdc7bcd1 100644
--- a/arch/arm/boot/dts/imx6q-b850v3.dts
+++ b/arch/arm/boot/dts/imx6q-b850v3.dts
@@ -212,3 +212,78 @@
 		};
 	};
 };
+
+&pci_root {
+	/* PLX Technology, Inc. PEX 8605 PCI Express 4-port Gen2 Switch */
+	bridge@1,0 {
+		compatible = "pci10b5,8605";
+		reg = <0x00010000 0 0 0 0>;
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		#interrupt-cells = <1>;
+
+		bridge@2,1 {
+			compatible = "pci10b5,8605";
+			reg = <0x00020800 0 0 0 0>;
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			#interrupt-cells = <1>;
+
+			/* Intel Corporation I210 Gigabit Network Connection */
+			ethernet@3,0 {
+				compatible = "pci8086,1533";
+				reg = <0x00030000 0 0 0 0>;
+			};
+		};
+
+		bridge@2,2 {
+			compatible = "pci10b5,8605";
+			reg = <0x00021000 0 0 0 0>;
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			#interrupt-cells = <1>;
+
+			/* Intel Corporation I210 Gigabit Network Connection */
+			switch_nic: ethernet@4,0 {
+				compatible = "pci8086,1533";
+				reg = <0x00040000 0 0 0 0>;
+			};
+		};
+	};
+};
+
+&switch_ports {
+	port@0 {
+		reg = <0>;
+		label = "eneport1";
+		phy-handle = <&switchphy0>;
+	};
+
+	port@1 {
+		reg = <1>;
+		label = "eneport2";
+		phy-handle = <&switchphy1>;
+	};
+
+	port@2 {
+		reg = <2>;
+		label = "enix";
+		phy-handle = <&switchphy2>;
+	};
+
+	port@3 {
+		reg = <3>;
+		label = "enid";
+		phy-handle = <&switchphy3>;
+	};
+
+	port@4 {
+		reg = <4>;
+		label = "cpu";
+		ethernet = <&switch_nic>;
+		phy-handle = <&switchphy4>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6q-bx50v3.dtsi b/arch/arm/boot/dts/imx6q-bx50v3.dtsi
index b915837bbb5f..916ea94d75ca 100644
--- a/arch/arm/boot/dts/imx6q-bx50v3.dtsi
+++ b/arch/arm/boot/dts/imx6q-bx50v3.dtsi
@@ -92,6 +92,56 @@
 		mux-int-port = <1>;
 		mux-ext-port = <4>;
 	};
+
+	aliases {
+		mdio-gpio0 = &mdio0;
+	};
+
+	mdio0: mdio-gpio {
+		compatible = "virtual,mdio-gpio";
+		gpios = <&gpio2 5 GPIO_ACTIVE_HIGH>, /* mdc */
+			<&gpio2 7 GPIO_ACTIVE_HIGH>; /* mdio */
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		switch@0 {
+			compatible = "marvell,mv88e6085"; /* 88e6240*/
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+
+			switch_ports: ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			mdio {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				switchphy0: switchphy@0 {
+					reg = <0>;
+				};
+
+				switchphy1: switchphy@1 {
+					reg = <1>;
+				};
+
+				switchphy2: switchphy@2 {
+					reg = <2>;
+				};
+
+				switchphy3: switchphy@3 {
+					reg = <3>;
+				};
+
+				switchphy4: switchphy@4 {
+					reg = <4>;
+				};
+			};
+		};
+	};
 };
 
 &ecspi5 {
@@ -326,3 +376,15 @@
 		tcxo-clock-frequency = <26000000>;
 	};
 };
+
+&pcie {
+	/* Synopsys, Inc. Device */
+	pci_root: root@0,0 {
+		compatible = "pci16c3,abcd";
+		reg = <0x00000000 0 0 0 0>;
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		#interrupt-cells = <1>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6q-cubox-i-emmc-som-v15.dts b/arch/arm/boot/dts/imx6q-cubox-i-emmc-som-v15.dts
new file mode 100644
index 000000000000..3e59ebbb3608
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-cubox-i-emmc-som-v15.dts
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2014 Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-sr-som-emmc.dtsi"
+#include "imx6qdl-cubox-i.dtsi"
+
+/ {
+	model = "SolidRun Cubox-i Dual/Quad (1.5som+emmc)";
+	compatible = "solidrun,cubox-i/q", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+	fsl,transmit-level-mV = <1104>;
+	fsl,transmit-boost-mdB = <0>;
+	fsl,transmit-atten-16ths = <9>;
+	fsl,no-spread-spectrum;
+};
diff --git a/arch/arm/boot/dts/imx6q-cubox-i-som-v15.dts b/arch/arm/boot/dts/imx6q-cubox-i-som-v15.dts
new file mode 100644
index 000000000000..dab70d1230a2
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-cubox-i-som-v15.dts
@@ -0,0 +1,59 @@
+/*
+ * Copyright (C) 2014 Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-cubox-i.dtsi"
+
+/ {
+	model = "SolidRun Cubox-i Dual/Quad (1.5som)";
+	compatible = "solidrun,cubox-i/q", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+	fsl,transmit-level-mV = <1104>;
+	fsl,transmit-boost-mdB = <0>;
+	fsl,transmit-atten-16ths = <9>;
+	fsl,no-spread-spectrum;
+};
diff --git a/arch/arm/boot/dts/imx6q-cubox-i.dts b/arch/arm/boot/dts/imx6q-cubox-i.dts
index b68aa0e57f20..1c7b262e3709 100644
--- a/arch/arm/boot/dts/imx6q-cubox-i.dts
+++ b/arch/arm/boot/dts/imx6q-cubox-i.dts
@@ -41,6 +41,8 @@
 /dts-v1/;
 
 #include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-brcm.dtsi"
 #include "imx6qdl-cubox-i.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/imx6q-display5.dtsi b/arch/arm/boot/dts/imx6q-display5.dtsi
index 4084de43d4d9..09085fde3341 100644
--- a/arch/arm/boot/dts/imx6q-display5.dtsi
+++ b/arch/arm/boot/dts/imx6q-display5.dtsi
@@ -255,7 +255,7 @@
 	pinctrl-0 = <&pinctrl_i2c1>;
 	status = "okay";
 
-	codec: tfa9879@6C {
+	codec: tfa9879@6c {
 		#sound-dai-cells = <0>;
 		compatible = "nxp,tfa9879";
 		reg = <0x6C>;
diff --git a/arch/arm/boot/dts/imx6q-h100.dts b/arch/arm/boot/dts/imx6q-h100.dts
index a3269f57df2b..8a2ea6c58902 100644
--- a/arch/arm/boot/dts/imx6q-h100.dts
+++ b/arch/arm/boot/dts/imx6q-h100.dts
@@ -42,8 +42,8 @@
 /dts-v1/;
 
 #include "imx6q.dtsi"
-#include "imx6qdl-microsom.dtsi"
-#include "imx6qdl-microsom-ar8035.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-brcm.dtsi"
 
 / {
 	model = "Auvidea H100";
@@ -108,21 +108,6 @@
 		regulator-always-on;
 	};
 
-	reg_nvcc_sd2: regulator-nvcc-sd2 {
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_h100_reg_nvcc_sd2>;
-		compatible = "regulator-gpio";
-		regulator-name = "NVCC_SD2";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-type = "voltage";
-		regulator-boot-on;
-		regulator-always-on;
-		gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>;
-		states = <1800000 0x1
-			  3300000 0x0>;
-	};
-
 	reg_usbh1_vbus: regulator-usb-h1-vbus {
 		compatible = "regulator-fixed";
 		enable-active-high;
@@ -205,7 +190,7 @@
 		reset-gpios = <&gpio6 15 GPIO_ACTIVE_LOW>;
 		/* IRQ has a wrong pull resistor which renders it useless  */
 
-		port@0 {
+		port {
 			tc358743_out: endpoint {
 				remote-endpoint = <&mipi_csi2_in>;
 				data-lanes = <1 2 3 4>;
@@ -260,12 +245,6 @@
 			>;
 		};
 
-		pinctrl_h100_reg_nvcc_sd2: h100-reg-nvcc-sd2 {
-			fsl,pins = <
-				MX6QDL_PAD_KEY_ROW1__GPIO4_IO09		0x1b0b0
-			>;
-		};
-
 		pinctrl_h100_sgtl5000: h100-sgtl5000 {
 			fsl,pins = <
 				MX6QDL_PAD_DISP0_DAT19__AUD5_RXD	0x130b0
@@ -316,6 +295,7 @@
 				MX6QDL_PAD_SD2_DAT1__SD2_DATA1		0x17059
 				MX6QDL_PAD_SD2_DAT2__SD2_DATA2		0x17059
 				MX6QDL_PAD_SD2_DAT3__SD2_DATA3		0x13059
+				MX6QDL_PAD_KEY_ROW1__SD2_VSELECT	0x1b0b0
 			>;
 		};
 
@@ -328,6 +308,7 @@
 				MX6QDL_PAD_SD2_DAT1__SD2_DATA1		0x170b9
 				MX6QDL_PAD_SD2_DAT2__SD2_DATA2		0x170b9
 				MX6QDL_PAD_SD2_DAT3__SD2_DATA3		0x170b9
+				MX6QDL_PAD_KEY_ROW1__SD2_VSELECT	0x1b0b0
 			>;
 		};
 
@@ -340,6 +321,7 @@
 				MX6QDL_PAD_SD2_DAT1__SD2_DATA1		0x170f9
 				MX6QDL_PAD_SD2_DAT2__SD2_DATA2		0x170f9
 				MX6QDL_PAD_SD2_DAT3__SD2_DATA3		0x170f9
+				MX6QDL_PAD_KEY_ROW1__SD2_VSELECT	0x1b0b0
 			>;
 		};
 	};
@@ -348,7 +330,7 @@
 &mipi_csi {
 	status = "okay";
 
-	port@0 {
+	port {
 		mipi_csi2_in: endpoint {
 			remote-endpoint = <&tc358743_out>;
 			data-lanes = <1 2 3 4>;
@@ -389,7 +371,6 @@
 	pinctrl-1 = <&pinctrl_h100_usdhc2_100mhz>;
 	pinctrl-2 = <&pinctrl_h100_usdhc2_200mhz>;
 	vmmc-supply = <&reg_3p3v>;
-	vqmmc-supply = <&reg_nvcc_sd2>;
 	cd-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/imx6q-hummingboard-emmc-som-v15.dts b/arch/arm/boot/dts/imx6q-hummingboard-emmc-som-v15.dts
new file mode 100644
index 000000000000..c51b4e4fd71e
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-hummingboard-emmc-som-v15.dts
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2014 Rabeeh Khoury (rabeeh@solid-run.com)
+ * Based on dt work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-sr-som-emmc.dtsi"
+#include "imx6qdl-hummingboard.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard Dual/Quad (1.5som+emmc)";
+	compatible = "solidrun,hummingboard/q", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+	fsl,transmit-level-mV = <1025>;
+	fsl,transmit-boost-mdB = <3330>;
+	fsl,transmit-atten-16ths = <9>;
+	fsl,receive-eq-mdB = <3000>;
+};
diff --git a/arch/arm/boot/dts/imx6q-hummingboard-som-v15.dts b/arch/arm/boot/dts/imx6q-hummingboard-som-v15.dts
new file mode 100644
index 000000000000..e4132d62ffa2
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-hummingboard-som-v15.dts
@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2014 Rabeeh Khoury (rabeeh@solid-run.com)
+ * Based on dt work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-hummingboard.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard Dual/Quad (1.5som)";
+	compatible = "solidrun,hummingboard/q", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+	fsl,transmit-level-mV = <1025>;
+	fsl,transmit-boost-mdB = <3330>;
+	fsl,transmit-atten-16ths = <9>;
+	fsl,receive-eq-mdB = <3000>;
+};
diff --git a/arch/arm/boot/dts/imx6q-hummingboard.dts b/arch/arm/boot/dts/imx6q-hummingboard.dts
index 69a7a0a1cb21..8c9e94e648a7 100644
--- a/arch/arm/boot/dts/imx6q-hummingboard.dts
+++ b/arch/arm/boot/dts/imx6q-hummingboard.dts
@@ -42,6 +42,8 @@
 /dts-v1/;
 
 #include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-brcm.dtsi"
 #include "imx6qdl-hummingboard.dtsi"
 
 / {
diff --git a/arch/arm/boot/dts/imx6q-hummingboard2-emmc-som-v15.dts b/arch/arm/boot/dts/imx6q-hummingboard2-emmc-som-v15.dts
new file mode 100644
index 000000000000..1998ebfa0fe0
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-hummingboard2-emmc-som-v15.dts
@@ -0,0 +1,63 @@
+/*
+ * Device Tree file for SolidRun HummingBoard2
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ * Based on work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-emmc.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-hummingboard2.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard2 Dual/Quad (1.5som+emmc)";
+	compatible = "solidrun,hummingboard2/q", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+	fsl,transmit-level-mV = <1104>;
+	fsl,transmit-boost-mdB = <0>;
+	fsl,transmit-atten-16ths = <9>;
+	fsl,no-spread-spectrum;
+};
diff --git a/arch/arm/boot/dts/imx6q-hummingboard2-som-v15.dts b/arch/arm/boot/dts/imx6q-hummingboard2-som-v15.dts
new file mode 100644
index 000000000000..d3ad7329cd6d
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-hummingboard2-som-v15.dts
@@ -0,0 +1,62 @@
+/*
+ * Device Tree file for SolidRun HummingBoard2
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ * Based on work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License.
+ *
+ *     This file is distributed in the hope that it will be useful
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED , WITHOUT WARRANTY OF ANY KIND
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-ti.dtsi"
+#include "imx6qdl-hummingboard2.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard2 Dual/Quad (1.5som)";
+	compatible = "solidrun,hummingboard2/q", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+	fsl,transmit-level-mV = <1104>;
+	fsl,transmit-boost-mdB = <0>;
+	fsl,transmit-atten-16ths = <9>;
+	fsl,no-spread-spectrum;
+};
diff --git a/arch/arm/boot/dts/imx6q-hummingboard2.dts b/arch/arm/boot/dts/imx6q-hummingboard2.dts
new file mode 100644
index 000000000000..5249f53dcdbc
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-hummingboard2.dts
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ * Based on dt work by Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-sr-som.dtsi"
+#include "imx6qdl-sr-som-brcm.dtsi"
+#include "imx6qdl-hummingboard2.dtsi"
+#include "imx6qdl-hummingboard2-emmc.dtsi"
+
+/ {
+	model = "SolidRun HummingBoard2 Dual/Quad";
+	compatible = "solidrun,hummingboard2/q", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+	fsl,transmit-level-mV = <1104>;
+	fsl,transmit-boost-mdB = <0>;
+	fsl,transmit-atten-16ths = <9>;
+	fsl,no-spread-spectrum;
+};
diff --git a/arch/arm/boot/dts/imx6q-pinfunc.h b/arch/arm/boot/dts/imx6q-pinfunc.h
index 9fc6120a1853..cfb11d3e739c 100644
--- a/arch/arm/boot/dts/imx6q-pinfunc.h
+++ b/arch/arm/boot/dts/imx6q-pinfunc.h
@@ -551,6 +551,7 @@
 #define MX6QDL_PAD_ENET_RXD1__ESAI_TX_FS            0x1e0 0x4f4 0x860 0x2 0x0
 #define MX6QDL_PAD_ENET_RXD1__ENET_1588_EVENT3_OUT  0x1e0 0x4f4 0x000 0x4 0x0
 #define MX6QDL_PAD_ENET_RXD1__GPIO1_IO26            0x1e0 0x4f4 0x000 0x5 0x0
+#define MX6QDL_PAD_ENET_RXD0__OSC32K_32K_OUT        0x1e4 0x4f8 0x000 0x0 0x0
 #define MX6QDL_PAD_ENET_RXD0__ENET_RX_DATA0         0x1e4 0x4f8 0x848 0x1 0x1
 #define MX6QDL_PAD_ENET_RXD0__ESAI_TX_HF_CLK        0x1e4 0x4f8 0x868 0x2 0x0
 #define MX6QDL_PAD_ENET_RXD0__SPDIF_OUT             0x1e4 0x4f8 0x000 0x3 0x0
diff --git a/arch/arm/boot/dts/imx6q-ts7970.dts b/arch/arm/boot/dts/imx6q-ts7970.dts
new file mode 100644
index 000000000000..f19e18995e68
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-ts7970.dts
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2015 Technologic Systems
+ * Copyright 2017 Savoir-faire Linux
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+#include "imx6q.dtsi"
+#include "imx6qdl-ts7970.dtsi"
+
+/ {
+	model = "Technologic Systems i.MX6 Quad TS-7970 (Default Device Tree)";
+	compatible = "technologic,imx6q-ts7970", "fsl,imx6q";
+};
+
+&sata {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6q-var-dt6customboard.dts b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
new file mode 100644
index 000000000000..e0728d475f6f
--- /dev/null
+++ b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Support for Variscite DART-MX6 Carrier-board
+ *
+ * Copyright 2017 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+/dts-v1/;
+
+#include "imx6q.dtsi"
+#include "imx6qdl-var-dart.dtsi"
+#include <dt-bindings/input/linux-event-codes.h>
+
+/ {
+	model = "Variscite DART-MX6 Carrier-board";
+	compatible = "variscite,dt6customboard", "fsl,imx6q";
+
+	backlight_lvds: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm2 0 50000>;
+		brightness-levels = <0 4 8 16 32 64 128 248>;
+		default-brightness-level = <7>;
+		status = "okay";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		autorepeat;
+
+		back {
+			gpios = <&gpio4 26 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_BACK>;
+			label = "Key Back";
+			linux,input-type = <1>;
+			debounce-interval = <100>;
+			wakeup-source;
+		};
+
+		home {
+			gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_HOME>;
+			label = "Key Home";
+			linux,input-type = <1>;
+			debounce-interval = <100>;
+			wakeup-source;
+		};
+
+		menu {
+			gpios = <&gpio4 25 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_MENU>;
+			label = "Key Menu";
+			linux,input-type = <1>;
+			debounce-interval = <100>;
+			wakeup-source;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+
+		led1 {
+			gpios = <&gpio4 27 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+
+		led2 {
+			gpios = <&gpio4 28 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "default-on";
+		};
+	};
+
+	panel1: lvds-panel {
+		compatible = "sgd,gktw70sdae4se", "panel-lvds";
+		backlight = <&backlight_lvds>;
+		width-mm = <153>;
+		height-mm = <86>;
+		label = "gktw70sdae4se";
+		data-mapping = "jeida-18";
+
+		panel-timing {
+			clock-frequency = <32000000>;
+			hactive = <800>;
+			vactive = <480>;
+			hback-porch = <39>;
+			hfront-porch = <39>;
+			vback-porch = <29>;
+			vfront-porch = <13>;
+			hsync-len = <47>;
+			vsync-len = <2>;
+		};
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&lvds1_out>;
+			};
+		};
+	};
+
+	reg_usb_h1_vbus: regulator-usbh1vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio1 28 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usb_otg_vbus: regulator-usbotgvbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio4 15 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "dt6-customboard-audio";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&sound_codec>;
+		simple-audio-card,frame-master = <&sound_codec>;
+		simple-audio-card,widgets = "Headphone", "Headphone Jack",
+					    "Line", "Line In";
+		simple-audio-card,routing = "Headphone Jack", "HPLOUT",
+					    "Headphone Jack", "HPROUT",
+					    "LINE1L", "Line In",
+					    "LINE1R", "Line In";
+
+		sound_cpu: simple-audio-card,cpu {
+			sound-dai = <&ssi2>;
+		};
+
+		sound_codec: simple-audio-card,codec {
+			sound-dai = <&tlv320aic3106>;
+			clocks = <&clks IMX6QDL_CLK_CKO>;
+		};
+	};
+};
+
+&can1 {
+	status = "okay";
+};
+
+&ecspi1 {
+	cs-gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>,
+		   <&gpio4 10 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&fec {
+	status = "okay";
+	phy-mode = "rgmii";
+	phy-reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>;
+};
+
+&hdmi {
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+	status = "okay";
+
+	touchscreen@38 {
+		compatible = "edt,edt-ft5x06";
+		reg = <0x38>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <4 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <480>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+	};
+
+	rtc@68 {
+		compatible = "isil,isl12057";
+		reg = <0x68>;
+	};
+};
+
+&ldb {
+	status = "okay";
+
+	lvds-channel@1 {
+		status = "okay";
+
+		port@4 {
+			reg = <4>;
+
+			lvds1_out: endpoint {
+				remote-endpoint = <&panel_in>;
+			};
+		};
+	};
+};
+
+&pwm2 {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&usbh1 {
+	vbus-supply = <&reg_usb_h1_vbus>;
+	status = "okay";
+};
+
+&usbotg {
+	vbus-supply = <&reg_usb_otg_vbus>;
+	dr_mode = "otg";
+	srp-disable;
+	hnp-disable;
+	adp-disable;
+	status = "okay";
+};
+
+&usdhc2 {
+	cd-gpios = <&gpio1 6 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6qdl-apalis.dtsi b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
index e80fdca585f8..4e776e036cbc 100644
--- a/arch/arm/boot/dts/imx6qdl-apalis.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-apalis.dtsi
@@ -338,7 +338,6 @@
 
 		stmpe_touchscreen {
 			compatible = "st,stmpe-ts";
-			reg = <0>;
 			/* 3.25 MHz ADC clock speed */
 			st,adc-freq = <1>;
 			/* 8 sample average control */
diff --git a/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi b/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi
index 829a47938179..9fc1fa449f64 100644
--- a/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-apf6dev.dtsi
@@ -111,32 +111,28 @@
 		};
 	};
 
-	regulators {
-		compatible = "simple-bus";
-
-		reg_3p3v: 3p3v {
-			compatible = "regulator-fixed";
-			regulator-name = "3P3V";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
 
-		reg_usbh1_vbus: usb-h1-vbus {
-			compatible = "regulator-fixed";
-			regulator-name = "usb_h1_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-			regulator-always-on;
-		};
+	reg_usbh1_vbus: regulator-usb-h1-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
 
-		reg_usb_otg_vbus: usb-otg-vbus {
-			compatible = "regulator-fixed";
-			regulator-name = "usb_otg_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-			regulator-always-on;
-		};
+	reg_usb_otg_vbus: regulator-usb-otg-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
 	};
 
 	sound {
diff --git a/arch/arm/boot/dts/imx6qdl-aristainetos.dtsi b/arch/arm/boot/dts/imx6qdl-aristainetos.dtsi
index 7d64075204ae..ee4d0f84eeb2 100644
--- a/arch/arm/boot/dts/imx6qdl-aristainetos.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-aristainetos.dtsi
@@ -12,48 +12,43 @@
 #include <dt-bindings/gpio/gpio.h>
 
 / {
-	regulators {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		reg_2p5v: regulator@0 {
-			compatible = "regulator-fixed";
-			regulator-name = "2P5V";
-			regulator-min-microvolt = <2500000>;
-			regulator-max-microvolt = <2500000>;
-			regulator-always-on;
-		};
 
-		reg_3p3v: regulator@1 {
-			compatible = "regulator-fixed";
-			regulator-name = "3P3V";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
+	reg_2p5v: regulator-2p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "2P5V";
+		regulator-min-microvolt = <2500000>;
+		regulator-max-microvolt = <2500000>;
+		regulator-always-on;
+	};
 
-		reg_usbh1_vbus: regulator@2 {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio3 31 GPIO_ACTIVE_HIGH>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_aristainetos_usbh1_vbus>;
-			regulator-name = "usb_h1_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
 
-		reg_usbotg_vbus: regulator@3 {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio4 15 GPIO_ACTIVE_HIGH>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_aristainetos_usbotg_vbus>;
-			regulator-name = "usb_otg_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
+	reg_usbh1_vbus: regulator-usbh1-vbus {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 31 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_aristainetos_usbh1_vbus>;
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	reg_usbotg_vbus: regulator-usbotg-vbus {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio4 15 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_aristainetos_usbotg_vbus>;
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/imx6qdl-aristainetos2.dtsi b/arch/arm/boot/dts/imx6qdl-aristainetos2.dtsi
index 1b1872873207..376750882ed3 100644
--- a/arch/arm/boot/dts/imx6qdl-aristainetos2.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-aristainetos2.dtsi
@@ -52,46 +52,42 @@
 		enable-gpios = <&gpio6 31 GPIO_ACTIVE_HIGH>;
 	};
 
-	regulators {
-		compatible = "simple-bus";
-
-		reg_2p5v: 2p5v {
-			compatible = "regulator-fixed";
-			regulator-name = "2P5V";
-			regulator-min-microvolt = <2500000>;
-			regulator-max-microvolt = <2500000>;
-			regulator-always-on;
-		};
-
-		reg_3p3v: 3p3v {
-			compatible = "regulator-fixed";
-			regulator-name = "3P3V";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
-
-		reg_usbh1_vbus: usb-h1-vbus {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_aristainetos2_usbh1_vbus>;
-			regulator-name = "usb_h1_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
-
-		reg_usbotg_vbus: usb-otg-vbus {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio4 15 GPIO_ACTIVE_HIGH>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_aristainetos2_usbotg_vbus>;
-			regulator-name = "usb_otg_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
+	reg_2p5v: regulator-2p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "2P5V";
+		regulator-min-microvolt = <2500000>;
+		regulator-max-microvolt = <2500000>;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_usbh1_vbus: regulator-usbh1-vbus {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_aristainetos2_usbh1_vbus>;
+		regulator-name = "usb_h1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	reg_usbotg_vbus: regulator-usbotg-vbus {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio4 15 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_aristainetos2_usbotg_vbus>;
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/imx6qdl-colibri.dtsi b/arch/arm/boot/dts/imx6qdl-colibri.dtsi
index fc66bbfd6796..e4eb300549d4 100644
--- a/arch/arm/boot/dts/imx6qdl-colibri.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-colibri.dtsi
@@ -274,7 +274,6 @@
 
 		stmpe_touchscreen {
 			compatible = "st,stmpe-ts";
-			reg = <0>;
 			/* 3.25 MHz ADC clock speed */
 			st,adc-freq = <1>;
 			/* 8 sample average control */
diff --git a/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi b/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi
index 14fff4ee6516..d1cfdc264126 100644
--- a/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-cubox-i.dtsi
@@ -38,8 +38,6 @@
  *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
-#include "imx6qdl-microsom.dtsi"
-#include "imx6qdl-microsom-ar8035.dtsi"
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/gpio/gpio.h>
 
@@ -64,38 +62,36 @@
 		};
 	};
 
-	regulators {
-		compatible = "simple-bus";
-
-		reg_3p3v: 3p3v {
-			compatible = "regulator-fixed";
-			regulator-name = "3P3V";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
+	v_5v0: regulator-v-5v0 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_5v0";
+	};
 
-		reg_usbh1_vbus: usb-h1-vbus {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio1 0 0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_cubox_i_usbh1_vbus>;
-			regulator-name = "usb_h1_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
+	v_usb2: regulator-v-usb2 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_cubox_i_usbh1_vbus>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb2";
+		vin-supply = <&v_5v0>;
+	};
 
-		reg_usbotg_vbus: usb-otg-vbus {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio3 22 0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_cubox_i_usbotg_vbus>;
-			regulator-name = "usb_otg_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
+	v_usb1: regulator-v-usb1 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_cubox_i_usbotg_vbus>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb1";
+		vin-supply = <&v_5v0>;
 	};
 
 	sound-spdif {
@@ -139,7 +135,7 @@
 
 	status = "okay";
 
-	rtc: pcf8523@68 {
+	rtc@68 {
 		compatible = "nxp,pcf8523";
 		reg = <0x68>;
 	};
@@ -243,21 +239,25 @@
 &usbh1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_cubox_i_usbh1>;
-	vbus-supply = <&reg_usbh1_vbus>;
+	vbus-supply = <&v_usb2>;
 	status = "okay";
 };
 
 &usbotg {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_cubox_i_usbotg>;
-	vbus-supply = <&reg_usbotg_vbus>;
+	vbus-supply = <&v_usb1>;
 	status = "okay";
 };
 
 &usdhc2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_cubox_i_usdhc2_aux &pinctrl_cubox_i_usdhc2>;
-	vmmc-supply = <&reg_3p3v>;
+	vmmc-supply = <&vcc_3v3>;
 	cd-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
+
+&vcc_3v3 {
+	vin-supply = <&v_5v0>;
+};
diff --git a/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi b/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi
index 37c07c0748aa..92583238ca4a 100644
--- a/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-hummingboard.dtsi
@@ -38,8 +38,6 @@
  *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
-#include "imx6qdl-microsom.dtsi"
-#include "imx6qdl-microsom-ar8035.dtsi"
 
 / {
 	chosen {
@@ -53,38 +51,58 @@
 		pinctrl-0 = <&pinctrl_hummingboard_gpio3_5>;
 	};
 
-	regulators {
-		compatible = "simple-bus";
+	v_3v2: regulator-v-3v2 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "v_3v2";
+		vin-supply = <&v_5v0>;
+	};
 
-		reg_3p3v: 3p3v {
-			compatible = "regulator-fixed";
-			regulator-name = "3P3V";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
+	v_5v0: regulator-v-5v0 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_5v0";
+	};
 
-		reg_usbh1_vbus: usb-h1-vbus {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_hummingboard_usbh1_vbus>;
-			regulator-name = "usb_h1_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
+	v_sd: regulator-v-sd {
+		compatible = "regulator-fixed";
+		gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard_vmmc>;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "v_sd";
+		startup-delay-us = <1000>;
+		vin-supply = <&v_3v2>;
+	};
 
-		reg_usbotg_vbus: usb-otg-vbus {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio3 22 GPIO_ACTIVE_HIGH>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_hummingboard_usbotg_vbus>;
-			regulator-name = "usb_otg_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
+	v_usb2: regulator-v-usb2 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard_usbh1_vbus>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb2";
+		vin-supply = <&v_5v0>;
+	};
+
+	v_usb1: regulator-v-usb1 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard_usbotg_vbus>;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb1";
+		vin-supply = <&v_5v0>;
 	};
 
 	sound-sgtl5000 {
@@ -132,20 +150,20 @@
 	status = "okay";
 
 	/* Pro baseboard model */
-	rtc: pcf8523@68 {
+	rtc@68 {
 		compatible = "nxp,pcf8523";
 		reg = <0x68>;
 	};
 
 	/* Pro baseboard model */
-	sgtl5000: sgtl5000@a {
+	sgtl5000: codec@a {
 		clocks = <&clks IMX6QDL_CLK_CKO>;
 		compatible = "fsl,sgtl5000";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_hummingboard_sgtl5000>;
 		reg = <0x0a>;
-		VDDA-supply = <&reg_3p3v>;
-		VDDIO-supply = <&reg_3p3v>;
+		VDDA-supply = <&v_3v2>;
+		VDDIO-supply = <&v_3v2>;
 	};
 };
 
@@ -247,6 +265,11 @@
 				MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x13059
 			>;
 		};
+		pinctrl_hummingboard_vmmc: hummingboard-vmmc {
+			fsl,pins = <
+				MX6QDL_PAD_DISP0_DAT9__GPIO4_IO30 0x1b0b0
+			>;
+		};
 	};
 };
 
@@ -280,7 +303,7 @@
 
 &usbh1 {
 	disable-over-current;
-	vbus-supply = <&reg_usbh1_vbus>;
+	vbus-supply = <&v_usb2>;
 	status = "okay";
 };
 
@@ -288,7 +311,7 @@
 	disable-over-current;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_hummingboard_usbotg_id>;
-	vbus-supply = <&reg_usbotg_vbus>;
+	vbus-supply = <&v_usb1>;
 	status = "okay";
 };
 
@@ -298,7 +321,11 @@
 		&pinctrl_hummingboard_usdhc2_aux
 		&pinctrl_hummingboard_usdhc2
 	>;
-	vmmc-supply = <&reg_3p3v>;
+	vmmc-supply = <&v_sd>;
 	cd-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
+
+&vcc_3v3 {
+	vin-supply = <&v_3v2>;
+};
diff --git a/arch/arm/boot/dts/imx6qdl-hummingboard2-emmc.dtsi b/arch/arm/boot/dts/imx6qdl-hummingboard2-emmc.dtsi
new file mode 100644
index 000000000000..f400405381a7
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-hummingboard2-emmc.dtsi
@@ -0,0 +1,72 @@
+/*
+ * Device Tree file for SolidRun HummingBoard2
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+&iomuxc {
+	hummingboard2 {
+		pinctrl_hummingboard2_usdhc3: hummingboard2-usdhc3 {
+			fsl,pins = <
+				MX6QDL_PAD_SD3_CMD__SD3_CMD    0x17059
+				MX6QDL_PAD_SD3_CLK__SD3_CLK    0x10059
+				MX6QDL_PAD_SD3_DAT0__SD3_DATA0 0x17059
+				MX6QDL_PAD_SD3_DAT1__SD3_DATA1 0x17059
+				MX6QDL_PAD_SD3_DAT2__SD3_DATA2 0x17059
+				MX6QDL_PAD_SD3_DAT3__SD3_DATA3 0x17059
+				MX6QDL_PAD_SD3_DAT4__SD3_DATA4 0x17059
+				MX6QDL_PAD_SD3_DAT5__SD3_DATA5 0x17059
+				MX6QDL_PAD_SD3_DAT6__SD3_DATA6 0x17059
+				MX6QDL_PAD_SD3_DAT7__SD3_DATA7 0x17059
+				MX6QDL_PAD_SD3_RST__SD3_RESET  0x17059
+			>;
+		};
+	};
+};
+
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_usdhc3>;
+	vmmc-supply = <&v_3v2>;
+	vqmmc-supply = <&v_3v2>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi b/arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi
new file mode 100644
index 000000000000..dffbc92e0023
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-hummingboard2.dtsi
@@ -0,0 +1,540 @@
+/*
+ * Copyright (C) 2015 Rabeeh Khoury <rabeeh@solid-run.com>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/ {
+	chosen {
+		stdout-path = &uart1;
+	};
+
+	ir_recv: ir-receiver {
+		compatible = "gpio-ir-receiver";
+		gpios = <&gpio7 9 GPIO_ACTIVE_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard2_gpio7_9>;
+		linux,rc-map-name = "rc-rc6-mce";
+	};
+
+	v_3v2: regulator-v-3v2 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "v_3v2";
+	};
+
+	v_5v0: regulator-v-5v0 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_5v0";
+	};
+
+	vcc_1p8: regulator-vcc-1p8 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+		regulator-name = "vcc_1p8";
+		vin-supply = <&v_3v2>;
+	};
+
+	v_sd: regulator-v-sd {
+		compatible = "regulator-fixed";
+		gpio = <&gpio4 30 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard2_vmmc>;
+		regulator-boot-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-name = "v_sd";
+		startup-delay-us = <1000>;
+		vin-supply = <&v_3v2>;
+	};
+
+	v_usb1: regulator-v-usb1 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard2_usbotg_vbus>;
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb1";
+		vin-supply = <&v_5v0>;
+	};
+
+	v_usb2: regulator-v-usb2 {
+		/* USB hub port 1 */
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard2_usbh1_vbus>;
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb2";
+		vin-supply = <&v_5v0>;
+	};
+
+	v_usb3: regulator-v-usb3 {
+		/* USB hub port 3 */
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio2 13 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard2_usbh2_vbus>;
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb3";
+		vin-supply = <&v_5v0>;
+	};
+
+	v_usb4: regulator-v-usb4 {
+		/* USB hub port 4 */
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio7 10 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard2_usbh3_vbus>;
+		regulator-always-on;
+		regulator-max-microvolt = <5000000>;
+		regulator-min-microvolt = <5000000>;
+		regulator-name = "v_usb4";
+		vin-supply = <&v_5v0>;
+	};
+
+	sound-sgtl5000 {
+		audio-codec = <&sgtl5000>;
+		audio-routing =
+			"MIC_IN", "Mic Jack",
+			"Mic Jack", "Mic Bias",
+			"Headphone Jack", "HP_OUT";
+		compatible = "fsl,imx-audio-sgtl5000";
+		model = "On-board Codec";
+		mux-ext-port = <5>;
+		mux-int-port = <1>;
+		ssi-controller = <&ssi1>;
+	};
+};
+
+&audmux {
+	status = "okay";
+};
+
+&ecspi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_ecspi2>;
+	cs-gpios = <&gpio2 26 0>;
+	status = "okay";
+};
+
+&hdmi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_hdmi>;
+	ddc-i2c-bus = <&i2c2>;
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_i2c1>;
+	status = "okay";
+
+	pcf8523: rtc@68 {
+		compatible = "nxp,pcf8523";
+		reg = <0x68>;
+	};
+
+	sgtl5000: codec@a {
+		clocks = <&clks IMX6QDL_CLK_CKO>;
+		compatible = "fsl,sgtl5000";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_hummingboard2_sgtl5000>;
+		reg = <0x0a>;
+		VDDA-supply = <&v_3v2>;
+		VDDD-supply = <&vcc_1p8>;
+		VDDIO-supply = <&v_3v2>;
+	};
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_i2c2>;
+	status = "okay";
+};
+
+&i2c3 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_i2c3>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog>;
+
+	hummingboard2 {
+		pinctrl_hog: hoggrp {
+		fsl,pins = <
+				/*
+				 * 36 pin headers GPIO description. The pins
+				 * numbering as following -
+				 *
+				 * 	3.2v	5v	74	75
+				 *	73	72	71	70
+				 *	69	68	67	66
+				 *
+				 *	77	78	79	76
+				 *	65	64	61	60
+				 *	53	52	51	50
+				 *	49	48	166	132
+				 *	95	94	90	91
+				 *	GND	54	24	204
+				 *
+				 * The GPIO numbers can be extracted using
+				 * signal name from below.
+				 * Example -
+				 * MX6QDL_PAD_EIM_DA10__GPIO3_IO10 is
+				 * GPIO(3,10) which is (3-1)*32+10 = gpio 74
+				 *
+				 * i.e. The mapping of GPIO(X,Y) to Linux gpio
+				 * number is : gpio number = (X-1) * 32 + Y
+				 */
+				/* DI1_PIN15 */
+				MX6QDL_PAD_EIM_DA10__GPIO3_IO10 0x400130b1
+				/* DI1_PIN02 */
+				MX6QDL_PAD_EIM_DA11__GPIO3_IO11 0x400130b1
+				/* DISP1_DATA00 */
+				MX6QDL_PAD_EIM_DA9__GPIO3_IO09 0x400130b1
+				/* DISP1_DATA01 */
+				MX6QDL_PAD_EIM_DA8__GPIO3_IO08 0x400130b1
+				/* DISP1_DATA02 */
+				MX6QDL_PAD_EIM_DA7__GPIO3_IO07 0x400130b1
+				/* DISP1_DATA03 */
+				MX6QDL_PAD_EIM_DA6__GPIO3_IO06 0x400130b1
+				/* DISP1_DATA04 */
+				MX6QDL_PAD_EIM_DA5__GPIO3_IO05 0x400130b1
+				/* DISP1_DATA05 */
+				MX6QDL_PAD_EIM_DA4__GPIO3_IO04 0x400130b1
+				/* DISP1_DATA06 */
+				MX6QDL_PAD_EIM_DA3__GPIO3_IO03 0x400130b1
+				/* DISP1_DATA07 */
+				MX6QDL_PAD_EIM_DA2__GPIO3_IO02 0x400130b1
+				/* DI1_D0_CS */
+				MX6QDL_PAD_EIM_DA13__GPIO3_IO13 0x400130b1
+				/* DI1_D1_CS */
+				MX6QDL_PAD_EIM_DA14__GPIO3_IO14 0x400130b1
+				/* DI1_PIN01 */
+				MX6QDL_PAD_EIM_DA15__GPIO3_IO15 0x400130b1
+				/* DI1_PIN03 */
+				MX6QDL_PAD_EIM_DA12__GPIO3_IO12 0x400130b1
+				/* DISP1_DATA08 */
+				MX6QDL_PAD_EIM_DA1__GPIO3_IO01 0x400130b1
+				/* DISP1_DATA09 */
+				MX6QDL_PAD_EIM_DA0__GPIO3_IO00 0x400130b1
+				/* DISP1_DATA10 */
+				MX6QDL_PAD_EIM_EB1__GPIO2_IO29 0x400130b1
+				/* DISP1_DATA11 */
+				MX6QDL_PAD_EIM_EB0__GPIO2_IO28 0x400130b1
+				/* DISP1_DATA12 */
+				MX6QDL_PAD_EIM_A17__GPIO2_IO21 0x400130b1
+				/* DISP1_DATA13 */
+				MX6QDL_PAD_EIM_A18__GPIO2_IO20 0x400130b1
+				/* DISP1_DATA14 */
+				MX6QDL_PAD_EIM_A19__GPIO2_IO19 0x400130b1
+				/* DISP1_DATA15 */
+				MX6QDL_PAD_EIM_A20__GPIO2_IO18 0x400130b1
+				/* DISP1_DATA16 */
+				MX6QDL_PAD_EIM_A21__GPIO2_IO17 0x400130b1
+				/* DISP1_DATA17 */
+				MX6QDL_PAD_EIM_A22__GPIO2_IO16 0x400130b1
+				/* DISP1_DATA18 */
+				MX6QDL_PAD_EIM_A23__GPIO6_IO06 0x400130b1
+				/* DISP1_DATA19 */
+				MX6QDL_PAD_EIM_A24__GPIO5_IO04 0x400130b1
+				/* DISP1_DATA20 */
+				MX6QDL_PAD_EIM_D31__GPIO3_IO31 0x400130b1
+				/* DISP1_DATA21 */
+				MX6QDL_PAD_EIM_D30__GPIO3_IO30 0x400130b1
+				/* DISP1_DATA22 */
+				MX6QDL_PAD_EIM_D26__GPIO3_IO26 0x400130b1
+				/* DISP1_DATA23 */
+				MX6QDL_PAD_EIM_D27__GPIO3_IO27 0x400130b1
+				/* DI1_DISP_CLK */
+				MX6QDL_PAD_EIM_A16__GPIO2_IO22 0x400130b1
+				/* SPDIF_IN */
+				MX6QDL_PAD_ENET_RX_ER__GPIO1_IO24 0x400130b1
+				/* SPDIF_OUT */
+				MX6QDL_PAD_GPIO_17__GPIO7_IO12 0x400130b1
+
+				/* MikroBUS GPIO pin number 10 */
+				MX6QDL_PAD_EIM_LBA__GPIO2_IO27 0x400130b1
+			>;
+		};
+
+		pinctrl_hummingboard2_ecspi2: hummingboard2-ecspi2grp {
+			fsl,pins = <
+				MX6QDL_PAD_EIM_OE__ECSPI2_MISO	0x100b1
+				MX6QDL_PAD_EIM_CS1__ECSPI2_MOSI	0x100b1
+				MX6QDL_PAD_EIM_CS0__ECSPI2_SCLK	0x100b1
+				MX6QDL_PAD_EIM_RW__GPIO2_IO26	0x000b1 /* CS */
+			>;
+		};
+
+		pinctrl_hummingboard2_gpio7_9: hummingboard2-gpio7_9 {
+			fsl,pins = <
+				MX6QDL_PAD_SD4_CMD__GPIO7_IO09 0x80000000
+			>;
+		};
+
+		pinctrl_hummingboard2_hdmi: hummingboard2-hdmi {
+			fsl,pins = <
+				MX6QDL_PAD_KEY_ROW2__HDMI_TX_CEC_LINE 0x1f8b0
+			>;
+		};
+
+		pinctrl_hummingboard2_i2c1: hummingboard2-i2c1 {
+			fsl,pins = <
+				MX6QDL_PAD_EIM_D21__I2C1_SCL 0x4001b8b1
+				MX6QDL_PAD_EIM_D28__I2C1_SDA 0x4001b8b1
+			>;
+		};
+
+		pinctrl_hummingboard2_i2c2: hummingboard2-i2c2 {
+			fsl,pins = <
+				MX6QDL_PAD_KEY_COL3__I2C2_SCL 0x4001b8b1
+				MX6QDL_PAD_KEY_ROW3__I2C2_SDA 0x4001b8b1
+			>;
+		};
+
+		pinctrl_hummingboard2_i2c3: hummingboard2-i2c3 {
+			fsl,pins = <
+				MX6QDL_PAD_EIM_D17__I2C3_SCL 0x4001b8b1
+				MX6QDL_PAD_EIM_D18__I2C3_SDA 0x4001b8b1
+			>;
+		};
+
+		pinctrl_hummingboard2_mipi: hummingboard2_mipi {
+			fsl,pins = <
+				MX6QDL_PAD_SD4_DAT2__GPIO2_IO10 0x4001b8b1
+				MX6QDL_PAD_KEY_COL4__GPIO4_IO14 0x4001b8b1
+				MX6QDL_PAD_NANDF_CS2__CCM_CLKO2 0x130b0
+			>;
+		};
+
+		pinctrl_hummingboard2_pcie_reset: hummingboard2-pcie-reset {
+			fsl,pins = <
+				MX6QDL_PAD_SD4_DAT3__GPIO2_IO11 0x1b0b1
+			>;
+		};
+
+		pinctrl_hummingboard2_pwm1: pwm1grp {
+			fsl,pins = <
+				MX6QDL_PAD_DISP0_DAT8__PWM1_OUT 0x1b0b1
+			>;
+		};
+
+		pinctrl_hummingboard2_pwm3: pwm3grp {
+			fsl,pins = <
+				MX6QDL_PAD_SD4_DAT1__PWM3_OUT 0x1b0b1
+			>;
+		};
+
+		pinctrl_hummingboard2_sgtl5000: hummingboard2-sgtl5000 {
+			fsl,pins = <
+				MX6QDL_PAD_DISP0_DAT19__AUD5_RXD 0x130b0
+				MX6QDL_PAD_KEY_COL0__AUD5_TXC 0x130b0
+				MX6QDL_PAD_KEY_ROW0__AUD5_TXD 0x110b0
+				MX6QDL_PAD_KEY_COL1__AUD5_TXFS 0x130b0
+				MX6QDL_PAD_GPIO_5__CCM_CLKO1 0x130b0
+			>;
+		};
+
+		pinctrl_hummingboard2_usbh1_vbus: hummingboard2-usbh1-vbus {
+			fsl,pins = <MX6QDL_PAD_GPIO_0__GPIO1_IO00 0x1b0b0>;
+		};
+
+		pinctrl_hummingboard2_usbh2_vbus: hummingboard2-usbh2-vbus {
+			fsl,pins = <MX6QDL_PAD_SD4_DAT5__GPIO2_IO13 0x1b0b0>;
+		};
+
+		pinctrl_hummingboard2_usbh3_vbus: hummingboard2-usbh3-vbus {
+			fsl,pins = <MX6QDL_PAD_SD4_CLK__GPIO7_IO10 0x1b0b0>;
+		};
+
+		pinctrl_hummingboard2_usbotg_id: hummingboard2-usbotg-id {
+			/*
+			 * Similar to pinctrl_usbotg_2, but we want it
+			 * pulled down for a fixed host connection.
+			 */
+			fsl,pins = <MX6QDL_PAD_GPIO_1__USB_OTG_ID 0x13059>;
+		};
+
+		pinctrl_hummingboard2_usbotg_vbus: hummingboard2-usbotg-vbus {
+			fsl,pins = <MX6QDL_PAD_EIM_D22__GPIO3_IO22 0x1b0b0>;
+		};
+
+		pinctrl_hummingboard2_usdhc2_aux: hummingboard2-usdhc2-aux {
+			fsl,pins = <
+				MX6QDL_PAD_GPIO_4__GPIO1_IO04    0x1f071
+				MX6QDL_PAD_KEY_ROW1__SD2_VSELECT 0x1b071
+			>;
+		};
+
+		pinctrl_hummingboard2_usdhc2: hummingboard2-usdhc2 {
+			fsl,pins = <
+				MX6QDL_PAD_SD2_CMD__SD2_CMD    0x17059
+				MX6QDL_PAD_SD2_CLK__SD2_CLK    0x10059
+				MX6QDL_PAD_SD2_DAT0__SD2_DATA0 0x17059
+				MX6QDL_PAD_SD2_DAT1__SD2_DATA1 0x17059
+				MX6QDL_PAD_SD2_DAT2__SD2_DATA2 0x17059
+				MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x13059
+			>;
+		};
+
+		pinctrl_hummingboard2_usdhc2_100mhz: hummingboard2-usdhc2-100mhz {
+			fsl,pins = <
+				MX6QDL_PAD_SD2_CMD__SD2_CMD    0x170b9
+				MX6QDL_PAD_SD2_CLK__SD2_CLK    0x100b9
+				MX6QDL_PAD_SD2_DAT0__SD2_DATA0 0x170b9
+				MX6QDL_PAD_SD2_DAT1__SD2_DATA1 0x170b9
+				MX6QDL_PAD_SD2_DAT2__SD2_DATA2 0x170b9
+				MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x130b9
+			>;
+		};
+
+		pinctrl_hummingboard2_usdhc2_200mhz: hummingboard2-usdhc2-200mhz {
+			fsl,pins = <
+				MX6QDL_PAD_SD2_CMD__SD2_CMD    0x170f9
+				MX6QDL_PAD_SD2_CLK__SD2_CLK    0x100f9
+				MX6QDL_PAD_SD2_DAT0__SD2_DATA0 0x170f9
+				MX6QDL_PAD_SD2_DAT1__SD2_DATA1 0x170f9
+				MX6QDL_PAD_SD2_DAT2__SD2_DATA2 0x170f9
+				MX6QDL_PAD_SD2_DAT3__SD2_DATA3 0x130f9
+			>;
+		};
+
+		pinctrl_hummingboard2_vmmc: hummingboard2-vmmc {
+			fsl,pins = <
+				MX6QDL_PAD_DISP0_DAT9__GPIO4_IO30 0x1b0b0
+			>;
+		};
+
+		pinctrl_hummingboard2_uart3: hummingboard2-uart3 {
+			fsl,pins = <
+				MX6QDL_PAD_EIM_D25__UART3_TX_DATA	0x1b0b1
+				MX6QDL_PAD_EIM_D24__UART3_RX_DATA	0x40013000
+			>;
+		};
+	};
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_pcie_reset>;
+	reset-gpio = <&gpio2 11 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_pwm1>;
+	status = "okay";
+};
+
+&pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_pwm3>;
+	status = "okay";
+};
+
+&ssi1 {
+	status = "okay";
+};
+
+&usbh1 {
+	disable-over-current;
+	status = "okay";
+};
+
+&usbotg {
+	disable-over-current;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_usbotg_id>;
+	vbus-supply = <&v_usb1>;
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <
+		&pinctrl_hummingboard2_usdhc2_aux
+		&pinctrl_hummingboard2_usdhc2
+	>;
+	pinctrl-1 = <
+		&pinctrl_hummingboard2_usdhc2_aux
+		&pinctrl_hummingboard2_usdhc2_100mhz
+	>;
+	pinctrl-2 = <
+		&pinctrl_hummingboard2_usdhc2_aux
+		&pinctrl_hummingboard2_usdhc2_200mhz
+	>;
+	vmmc-supply = <&v_sd>;
+	cd-gpios = <&gpio1 4 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hummingboard2_uart3>;
+	status = "okay";
+};
+
+&vcc_3v3 {
+	vin-supply = <&v_3v2>;
+};
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
index 3a77f0fedfce..fd05f7caa472 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
@@ -172,7 +172,7 @@
 		};
 	};
 
-	i2cmux@2 {
+	i2c2mux {
 		compatible = "i2c-mux-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_i2c2mux>;
@@ -183,20 +183,20 @@
 		i2c-parent = <&i2c2>;
 		idle-state = <0>;
 
-		i2c2@1 {
+		i2c2mux@1 {
 			reg = <1>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 		};
 
-		i2c2@2 {
+		i2c2mux@2 {
 			reg = <2>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 		};
 	};
 
-	i2cmux@3 {
+	i2c3mux {
 		compatible = "i2c-mux-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_i2c3mux>;
@@ -206,7 +206,7 @@
 		i2c-parent = <&i2c3>;
 		idle-state = <0>;
 
-		i2c3@1 {
+		i2c3mux@1 {
 			reg = <1>;
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
index 4bdf29169d2a..919b6b7619a4 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
@@ -276,6 +276,23 @@
 		compatible = "sst,sst25vf016b", "jedec,spi-nor";
 		spi-max-frequency = <20000000>;
 		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		partition@0 {
+			label = "bootloader";
+			reg = <0x0 0xc0000>;
+		};
+
+		partition@c0000 {
+			label = "env";
+			reg = <0xc0000 0x2000>;
+		};
+
+		partition@c2000 {
+			label = "splash";
+			reg = <0xc2000 0x13e000>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
index 4fa2fac3877b..82d6ccb46982 100644
--- a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
@@ -235,6 +235,9 @@
 };
 
 &hdmi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hdmi_cec>;
+	ddc-i2c-bus = <&i2c2>;
 	status = "okay";
 };
 
@@ -453,6 +456,12 @@
 			>;
 		};
 
+		pinctrl_hdmi_cec: hdmicecgrp {
+			fsl,pins = <
+				MX6QDL_PAD_EIM_A25__HDMI_TX_CEC_LINE	0x1f8b0
+			>;
+		};
+
 		pinctrl_i2c2: i2c2grp {
 			fsl,pins = <
 				MX6QDL_PAD_EIM_EB2__I2C2_SCL	0x4001b8b1
diff --git a/arch/arm/boot/dts/imx6qdl-microsom.dtsi b/arch/arm/boot/dts/imx6qdl-sr-som-brcm.dtsi
similarity index 85%
rename from arch/arm/boot/dts/imx6qdl-microsom.dtsi
rename to arch/arm/boot/dts/imx6qdl-sr-som-brcm.dtsi
index 6a410160c9ee..b55af61dfeca 100644
--- a/arch/arm/boot/dts/imx6qdl-microsom.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sr-som-brcm.dtsi
@@ -40,7 +40,7 @@
  */
 #include <dt-bindings/gpio/gpio.h>
 / {
-	clk_sdio: sdio-clock {
+	clk_brcm: brcm-clock {
 		compatible = "gpio-gate-clock";
 		#clock-cells = <0>;
 		pinctrl-names = "default";
@@ -48,27 +48,23 @@
 		enable-gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>;
 	};
 
-	regulators {
-		compatible = "simple-bus";
-
-		reg_brcm: brcm-reg {
-			compatible = "regulator-fixed";
-			enable-active-high;
-			gpio = <&gpio3 19 0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_microsom_brcm_reg>;
-			regulator-name = "brcm_reg";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			startup-delay-us = <200000>;
-		};
+	reg_brcm: brcm-reg {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 19 0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_microsom_brcm_reg>;
+		regulator-name = "brcm_reg";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		startup-delay-us = <200000>;
 	};
 
 	usdhc1_pwrseq: usdhc1_pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		reset-gpios = <&gpio5 26 GPIO_ACTIVE_LOW>,
 			      <&gpio6 0 GPIO_ACTIVE_LOW>;
-		clocks = <&clk_sdio>;
+		clocks = <&clk_brcm>;
 		clock-names = "ext_clock";
 	};
 };
@@ -104,13 +100,6 @@
 			>;
 		};
 
-		pinctrl_microsom_uart1: microsom-uart1 {
-			fsl,pins = <
-				MX6QDL_PAD_CSI0_DAT10__UART1_TX_DATA	0x1b0b1
-				MX6QDL_PAD_CSI0_DAT11__UART1_RX_DATA	0x1b0b1
-			>;
-		};
-
 		pinctrl_microsom_uart4: microsom-uart4 {
 			fsl,pins = <
 				MX6QDL_PAD_CSI0_DAT12__UART4_TX_DATA 0x1b0b1
@@ -133,12 +122,6 @@
 	};
 };
 
-&uart1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_microsom_uart1>;
-	status = "okay";
-};
-
 /* UART4 - Connected to optional BRCM Wifi/BT/FM */
 &uart4 {
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/imx6qdl-sr-som-emmc.dtsi b/arch/arm/boot/dts/imx6qdl-sr-som-emmc.dtsi
new file mode 100644
index 000000000000..5f3b8baab20f
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-sr-som-emmc.dtsi
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2013,2014 Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+&iomuxc {
+	microsom {
+		pinctrl_microsom_usdhc3: microsom-usdhc3 {
+			fsl,pins = <
+				MX6QDL_PAD_SD3_CMD__SD3_CMD    0x17059
+				MX6QDL_PAD_SD3_CLK__SD3_CLK    0x10059
+				MX6QDL_PAD_SD3_DAT0__SD3_DATA0 0x17059
+				MX6QDL_PAD_SD3_DAT1__SD3_DATA1 0x17059
+				MX6QDL_PAD_SD3_DAT2__SD3_DATA2 0x17059
+				MX6QDL_PAD_SD3_DAT3__SD3_DATA3 0x17059
+				MX6QDL_PAD_SD3_DAT4__SD3_DATA4 0x17059
+				MX6QDL_PAD_SD3_DAT5__SD3_DATA5 0x17059
+				MX6QDL_PAD_SD3_DAT6__SD3_DATA6 0x17059
+				MX6QDL_PAD_SD3_DAT7__SD3_DATA7 0x17059
+				MX6QDL_PAD_SD3_RST__SD3_RESET  0x17059
+			>;
+		};
+	};
+};
+
+/* USDHC3 - eMMC */
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_microsom_usdhc3>;
+	bus-width = <8>;
+	non-removable;
+	vmmc-supply = <&vcc_3v3>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6qdl-sr-som-ti.dtsi b/arch/arm/boot/dts/imx6qdl-sr-som-ti.dtsi
new file mode 100644
index 000000000000..44a97ba93a95
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-sr-som-ti.dtsi
@@ -0,0 +1,170 @@
+/*
+ * Copyright (C) 2013,2014 Russell King
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	nvcc_sd1: regulator-nvcc-sd1 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-name = "nvcc_sd1";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc_3v3>;
+	};
+
+	clk_ti_wifi: ti-wifi-clock {
+		/* This is a hack around the kernel - using "fixed clock"
+		 * results in the "pinctrl" properties being ignored, and
+		 * the clock not being output.  Instead, use a gated clock
+		 * and the unrouted WL_XTAL_PU gpio.
+		 */
+		compatible = "gpio-gate-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_microsom_ti_clk>;
+		enable-gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>;
+	};
+
+	pwrseq_ti_wifi: ti-wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_microsom_ti_wifi_en>;
+		reset-gpios = <&gpio5 26 GPIO_ACTIVE_LOW>;
+		post-power-on-delay-ms = <200>;
+		clocks = <&clk_ti_wifi>;
+		clock-names = "ext_clock";
+	};
+};
+
+&iomuxc {
+	microsom {
+		pinctrl_microsom_ti_bt: microsom-ti-bt {
+			fsl,pins = <
+				/* BT_EN_SOC */
+				MX6QDL_PAD_CSI0_DAT14__GPIO6_IO00	0x40013070
+			>;
+		};
+
+		pinctrl_microsom_ti_clk: microsom-ti-clk {
+			fsl,pins = <
+				/* EXT_32K */
+				MX6QDL_PAD_GPIO_8__XTALOSC_REF_CLK_32K	0x1b0b0
+				/* WL_XTAL_PU (unrouted) */
+				MX6QDL_PAD_DISP0_DAT11__GPIO5_IO05 0x40013070
+			>;
+		};
+
+		pinctrl_microsom_ti_wifi_en: microsom-ti-wifi-en {
+			fsl,pins = <
+				/* WLAN_EN_SOC */
+				MX6QDL_PAD_CSI0_DAT8__GPIO5_IO26	0x40013070
+			>;
+		};
+
+		pinctrl_microsom_ti_wifi_irq: microsom-ti-wifi-irq {
+			fsl,pins = <
+				/* WLAN_IRQ */
+				MX6QDL_PAD_CSI0_DAT18__GPIO6_IO04	0x40013070
+			>;
+		};
+
+		pinctrl_microsom_uart4: microsom-uart4 {
+			fsl,pins = <
+				MX6QDL_PAD_CSI0_DAT12__UART4_TX_DATA 0x1b0b1
+				MX6QDL_PAD_CSI0_DAT13__UART4_RX_DATA 0x1b0b1
+				MX6QDL_PAD_CSI0_DAT16__UART4_RTS_B 0x1b0b1
+				MX6QDL_PAD_CSI0_DAT17__UART4_CTS_B 0x1b0b1
+			>;
+		};
+
+		pinctrl_microsom_usdhc1: microsom-usdhc1 {
+			fsl,pins = <
+				MX6QDL_PAD_SD1_CMD__SD1_CMD    0x17059
+				MX6QDL_PAD_SD1_CLK__SD1_CLK    0x10059
+				MX6QDL_PAD_SD1_DAT0__SD1_DATA0 0x17059
+				MX6QDL_PAD_SD1_DAT1__SD1_DATA1 0x17059
+				MX6QDL_PAD_SD1_DAT2__SD1_DATA2 0x17059
+				MX6QDL_PAD_SD1_DAT3__SD1_DATA3 0x17059
+			>;
+		};
+	};
+};
+
+/* UART4 - Connected to optional TI Wi-Fi/BT/FM */
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_microsom_uart4>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "ti,wl1837-st";
+		clocks = <&clk_ti_wifi>;
+		clock-names = "ext_clock";
+		enable-gpios = <&gpio6 0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_microsom_ti_bt>;
+	};
+};
+
+/* USDHC1 - Connected to optional TI Wi-Fi/BT/FM */
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_microsom_usdhc1>;
+	bus-width = <4>;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&pwrseq_ti_wifi>;
+	non-removable;
+	vmmc-supply = <&vcc_3v3>;
+	/* vqmmc-supply = <&nvcc_sd1>; - MMC layer doesn't like it! */
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	wlcore@2 {
+		compatible = "ti,wl1837";
+		reg = <2>;
+		interrupts-extended = <&gpio6 4 IRQ_TYPE_LEVEL_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_microsom_ti_wifi_irq>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx6qdl-microsom-ar8035.dtsi b/arch/arm/boot/dts/imx6qdl-sr-som.dtsi
similarity index 84%
rename from arch/arm/boot/dts/imx6qdl-microsom-ar8035.dtsi
rename to arch/arm/boot/dts/imx6qdl-sr-som.dtsi
index 900e8c781f91..4ccb7afc4b35 100644
--- a/arch/arm/boot/dts/imx6qdl-microsom-ar8035.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sr-som.dtsi
@@ -1,9 +1,6 @@
 /*
  * Copyright (C) 2013,2014 Russell King
  *
- * This describes the hookup for an AR8035 to the iMX6 on the SolidRun
- * MicroSOM.
- *
  * This file is dual-licensed: you can use it either under the terms
  * of the GPL or the X11 license, at your option. Note that this dual
  * licensing only applies to this file, and not this project as a
@@ -41,6 +38,18 @@
  *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	vcc_3v3: regulator-vcc-3v3 {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-name = "vcc_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+};
+
 &fec {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_microsom_enet_ar8035>;
@@ -51,7 +60,7 @@
 };
 
 &iomuxc {
-	enet {
+	microsom {
 		pinctrl_microsom_enet_ar8035: microsom-enet-ar8035 {
 			fsl,pins = <
 				MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b8b0
@@ -59,10 +68,10 @@
 				/* AR8035 reset */
 				MX6QDL_PAD_KEY_ROW4__GPIO4_IO15		0x130b0
 				/* AR8035 interrupt */
-				MX6QDL_PAD_DI0_PIN2__GPIO4_IO18		0x80000000
+				MX6QDL_PAD_DI0_PIN2__GPIO4_IO18		0x1b0b0
 				/* GPIO16 -> AR8035 25MHz */
-				MX6QDL_PAD_GPIO_16__ENET_REF_CLK	0xc0000000
-				MX6QDL_PAD_RGMII_TXC__RGMII_TXC		0x80000000
+				MX6QDL_PAD_GPIO_16__ENET_REF_CLK	0x4001b0b0
+				MX6QDL_PAD_RGMII_TXC__RGMII_TXC		0x13030
 				MX6QDL_PAD_RGMII_TD0__RGMII_TD0		0x1b030
 				MX6QDL_PAD_RGMII_TD1__RGMII_TD1		0x1b030
 				MX6QDL_PAD_RGMII_TD2__RGMII_TD2		0x1b030
@@ -95,5 +104,18 @@
 				MX6QDL_PAD_ENET_RXD1__GPIO1_IO26	0x03000
 			>;
 		};
+
+		pinctrl_microsom_uart1: microsom-uart1 {
+			fsl,pins = <
+				MX6QDL_PAD_CSI0_DAT10__UART1_TX_DATA	0x1b0b1
+				MX6QDL_PAD_CSI0_DAT11__UART1_RX_DATA	0x1b0b1
+			>;
+		};
 	};
 };
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_microsom_uart1>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6qdl-ts7970.dtsi b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi
new file mode 100644
index 000000000000..f0be516dc28e
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi
@@ -0,0 +1,594 @@
+/*
+ * Copyright 2015 Technologic Systems
+ * Copyright 2017 Savoir-Faire Linux
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License
+ *     version 2 as published by the Free Software Foundation.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	leds {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_leds1>;
+		compatible = "gpio-leds";
+
+		green-led {
+			label = "green-led";
+			gpios = <&gpio3 27 GPIO_ACTIVE_LOW>;
+			default-state = "on";
+		};
+
+		red-led {
+			label = "red-led";
+			gpios = <&gpio1 2 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		yel-led {
+			label = "yellow-led";
+			gpios = <&gpio1 9 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		blue-led {
+			label = "blue-led";
+			gpios = <&gpio4 25 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		en-usb-5v {
+			label = "en-usb-5v";
+			gpios = <&gpio2 22 GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+		};
+
+		sel_dc_usb {
+			label = "sel_dc_usb";
+			gpios = <&gpio5 17 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3p3v";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_can1_3v3: reg_can1_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "reg_can1_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio4 21 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_can2_3v3: en-reg_can2_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "reg_can2_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio6 31 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usb_otg_vbus: regulator-usb-otg-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_wlan_vmmc: regulator_wlan_vmmc {
+		compatible = "regulator-fixed";
+		regulator-name = "wlan_vmmc";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpio8 14 GPIO_ACTIVE_HIGH>;
+		startup-delay-us = <70000>;
+		enable-active-high;
+	};
+
+	sound-sgtl5000 {
+		audio-codec = <&sgtl5000>;
+		audio-routing =
+			"MIC_IN", "Mic Jack",
+			"Mic Jack", "Mic Bias",
+			"Headphone Jack", "HP_OUT";
+		compatible = "fsl,imx-audio-sgtl5000";
+		model = "On-board Codec";
+		mux-ext-port = <3>;
+		mux-int-port = <1>;
+		ssi-controller = <&ssi1>;
+	};
+};
+
+&audmux {
+	status = "okay";
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	xceiver-supply = <&reg_can1_3v3>;
+	status = "okay";
+};
+
+&can2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	xceiver-supply = <&reg_can2_3v3>;
+	status = "okay";
+};
+
+&ecspi1 {
+	cs-gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	status = "okay";
+
+	n25q064: flash@0 {
+		compatible = "micron,n25q064", "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&ecspi2 {
+	cs-gpios = <
+		&gpio5 31 GPIO_ACTIVE_HIGH
+		&gpio7 12 GPIO_ACTIVE_HIGH
+		&gpio5 18 GPIO_ACTIVE_HIGH
+	>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	status = "okay";
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-mode = "rgmii";
+	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
+			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
+	fsl,err006687-workaround-present;
+	status = "okay";
+};
+
+&hdmi {
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	pinctrl-1 = <&pinctrl_i2c1_gpio>;
+	scl-gpios = <&gpio3 21 GPIO_ACTIVE_HIGH>;
+	sda-gpios = <&gpio3 28 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+
+	m41t00s: rtc@68 {
+		compatible = "m41t00";
+		reg = <0x68>;
+	};
+
+	isl12022: rtc@6f {
+		compatible = "isl,isl12022";
+		reg = <0x6f>;
+	};
+
+	gpio8: gpio@28 {
+		compatible = "technologic,ts7970-gpio";
+		reg = <0x28>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		ngpio = <32>;
+	};
+
+	sgtl5000: codec@a {
+		compatible = "fsl,sgtl5000";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_sgtl5000>;
+		reg = <0x0a>;
+		clocks = <&clks IMX6QDL_CLK_CKO>;
+		VDDA-supply = <&reg_3p3v>;
+		VDDIO-supply = <&reg_3p3v>;
+	};
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	pinctrl-1 = <&pinctrl_i2c2_gpio>;
+	scl-gpios = <&gpio4 12 GPIO_ACTIVE_HIGH>;
+	sda-gpios = <&gpio4 13 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog>;
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D17__ECSPI1_MISO		0x100b1
+			MX6QDL_PAD_EIM_D18__ECSPI1_MOSI		0x100b1
+			MX6QDL_PAD_EIM_D16__ECSPI1_SCLK		0x100b1
+			MX6QDL_PAD_EIM_D19__GPIO3_IO19		0x100b1 /* Onboard Flash CS */
+		>;
+	};
+
+	pinctrl_ecspi2: ecspi2 {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT8__ECSPI2_SCLK      0x100b1
+			MX6QDL_PAD_CSI0_DAT9__ECSPI2_MOSI      0x100b1
+			MX6QDL_PAD_CSI0_DAT10__ECSPI2_MISO     0x100b1
+			MX6QDL_PAD_CSI0_DAT13__GPIO5_IO31      0x100b1 /* FPGA_SPI_CS0 */
+			MX6QDL_PAD_GPIO_17__GPIO7_IO12         0x100b1 /* FPGA_SPI_CS1 */
+			MX6QDL_PAD_CSI0_PIXCLK__GPIO5_IO18     0x100b1 /* HD1_SPI_CS */
+			MX6QDL_PAD_CSI0_VSYNC__GPIO5_IO21      0x1b088 /* FPGA_RESET */
+			MX6QDL_PAD_GPIO_3__XTALOSC_REF_CLK_24M 0x10    /* FPGA 24MHZ */
+			MX6QDL_PAD_CSI0_DATA_EN__GPIO5_IO20    0x1b088 /* FPGA_IRQ_0 */
+			MX6QDL_PAD_GPIO_4__GPIO1_IO04          0x1b088 /* FPGA_IRQ_1 */
+		>;
+	};
+
+	pinctrl_enet: enet {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x1b0b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x1b0b0
+			MX6QDL_PAD_RGMII_TXC__RGMII_TXC		0x1b0b0
+			MX6QDL_PAD_RGMII_TD0__RGMII_TD0		0x1b0b0
+			MX6QDL_PAD_RGMII_TD1__RGMII_TD1		0x1b0b0
+			MX6QDL_PAD_RGMII_TD2__RGMII_TD2		0x1b0b0
+			MX6QDL_PAD_RGMII_TD3__RGMII_TD3		0x1b0b0
+			MX6QDL_PAD_RGMII_TX_CTL__RGMII_TX_CTL	0x1b0b0
+			MX6QDL_PAD_RGMII_RXC__RGMII_RXC		0x1b0b0
+			MX6QDL_PAD_RGMII_RD0__RGMII_RD0		0x1b0b0
+			MX6QDL_PAD_RGMII_RD1__RGMII_RD1		0x1b0b0
+			MX6QDL_PAD_RGMII_RD2__RGMII_RD2		0x1b0b0
+			MX6QDL_PAD_RGMII_RD3__RGMII_RD3		0x1b0b0
+			MX6QDL_PAD_RGMII_RX_CTL__RGMII_RX_CTL	0x1b0b0
+			MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK	0x1b0b0
+			MX6QDL_PAD_ENET_TX_EN__GPIO1_IO28	0x1b088
+			MX6QDL_PAD_DI0_PIN4__GPIO4_IO20		0x1b088 /* ETH_PHY_RESET */
+			MX6QDL_PAD_GPIO_6__ENET_IRQ		0x000b1
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX	0x1b088
+			MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX	0x1b088
+			MX6QDL_PAD_DISP0_DAT0__GPIO4_IO21	0x1b088 /* EN_CAN_1 */
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL4__FLEXCAN2_TX	0x1b088
+			MX6QDL_PAD_KEY_ROW4__FLEXCAN2_RX	0x1b088
+			MX6QDL_PAD_EIM_BCLK__GPIO6_IO31		0x1b088 /* EN_CAN_2 */
+		>;
+	};
+
+	pinctrl_hog: hoggrp {
+		fsl,pins = <
+			/* Onboard */
+			MX6QDL_PAD_SD4_DAT3__GPIO2_IO11		0x1b088 /* USB_HUB_RESET */
+			MX6QDL_PAD_DISP0_DAT23__GPIO5_IO17	0x1b088 /* SEL_DC_USB */
+			MX6QDL_PAD_EIM_A16__GPIO2_IO22		0x1b088 /* EN_USB_5V */
+			MX6QDL_PAD_DISP0_DAT14__GPIO5_IO08	0x1b088 /* JTAG_FPGA_TMS */
+			MX6QDL_PAD_DISP0_DAT17__GPIO5_IO11	0x1b088 /* JTAG_FPGA_TCK */
+			MX6QDL_PAD_DISP0_DAT18__GPIO5_IO12	0x1b088 /* JTAG_FPGA_TDO */
+			MX6QDL_PAD_DISP0_DAT22__GPIO5_IO16	0x1b088 /* JTAG_FPGA_TDI */
+			MX6QDL_PAD_DISP0_DAT20__GPIO5_IO14	0x1b088 /* GYRO_INT */
+			MX6QDL_PAD_EIM_OE__GPIO2_IO25		0x1b088 /* MODBUS_FAULT */
+			MX6QDL_PAD_EIM_RW__GPIO2_IO26		0x1b088 /* BUS_DIR/JP_SD_BOOT */
+			MX6QDL_PAD_EIM_A19__GPIO2_IO19		0x1b088 /* EN_MODBUS_24V */
+			MX6QDL_PAD_DISP0_DAT5__GPIO4_IO26	0x1b088 /* EN_MODBUS_3V */
+			MX6QDL_PAD_EIM_A17__GPIO2_IO21		0x1b088 /* I210_RESET */
+			MX6QDL_PAD_EIM_D23__GPIO3_IO23		0x1b088 /* EN_RTC_PWR */
+			MX6QDL_PAD_SD3_DAT4__GPIO7_IO01		0x1b088 /* REVSTRAP1 */
+
+			/* Offboard */
+			MX6QDL_PAD_DISP0_DAT7__GPIO4_IO28	0x1b088 /* LCD_D09 */
+			MX6QDL_PAD_DISP0_DAT9__GPIO4_IO30	0x1b088 /* HD1_IRQ */
+			MX6QDL_PAD_DISP0_DAT10__GPIO4_IO31	0x1b088 /* LCD_D10 */
+			MX6QDL_PAD_DISP0_DAT11__GPIO5_IO05	0x1b088 /* LCD_D11 */
+			MX6QDL_PAD_EIM_EB1__GPIO2_IO29		0x1b088 /* BUS_BHE */
+			MX6QDL_PAD_EIM_LBA__GPIO2_IO27		0x1b088 /* BUS_ALE */
+			MX6QDL_PAD_EIM_CS0__GPIO2_IO23		0x1b088 /* BUS_CS */
+			MX6QDL_PAD_EIM_A24__GPIO5_IO04		0x1b088 /* DIO_20 */
+			MX6QDL_PAD_EIM_WAIT__GPIO5_IO00		0x1b088 /* BUS_WAIT */
+			MX6QDL_PAD_EIM_DA0__GPIO3_IO00		0x1b088 /* MUX_AD_00 */
+			MX6QDL_PAD_EIM_DA1__GPIO3_IO01		0x1b088 /* MUX_AD_01 */
+			MX6QDL_PAD_EIM_DA2__GPIO3_IO02		0x1b088 /* MUX_AD_02 */
+			MX6QDL_PAD_EIM_DA3__GPIO3_IO03		0x1b088 /* MUX_AD_03 */
+			MX6QDL_PAD_EIM_DA4__GPIO3_IO04		0x1b088 /* MUX_AD_04 */
+			MX6QDL_PAD_EIM_DA5__GPIO3_IO05		0x1b088 /* MUX_AD_05 */
+			MX6QDL_PAD_EIM_DA6__GPIO3_IO06		0x1b088 /* MUX_AD_06 */
+			MX6QDL_PAD_EIM_DA7__GPIO3_IO07		0x1b088 /* MUX_AD_07 */
+			MX6QDL_PAD_EIM_DA8__GPIO3_IO08		0x1b088 /* MUX_AD_08 */
+			MX6QDL_PAD_EIM_DA9__GPIO3_IO09		0x1b088 /* MUX_AD_09 */
+			MX6QDL_PAD_EIM_DA10__GPIO3_IO10		0x1b088 /* MUX_AD_10 */
+			MX6QDL_PAD_EIM_DA11__GPIO3_IO11		0x1b088 /* MUX_AD_11 */
+			MX6QDL_PAD_EIM_DA12__GPIO3_IO12		0x1b088 /* MUX_AD_12 */
+			MX6QDL_PAD_EIM_DA13__GPIO3_IO13		0x1b088 /* MUX_AD_13 */
+			MX6QDL_PAD_EIM_DA14__GPIO3_IO14		0x1b088 /* MUX_AD_14 */
+			MX6QDL_PAD_EIM_DA15__GPIO3_IO15		0x1b088 /* MUX_AD_15 */
+
+			/* Strapping only */
+			MX6QDL_PAD_EIM_A18__GPIO2_IO20		0x1b088
+			MX6QDL_PAD_EIM_A21__GPIO2_IO17		0x1b088
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D21__I2C1_SCL		0x4001b8b1
+			MX6QDL_PAD_EIM_D28__I2C1_SDA		0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c1_gpio: i2c1gpiogrp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D21__GPIO3_IO21		0x4001b8b1
+			MX6QDL_PAD_EIM_D28__GPIO3_IO28		0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL3__I2C2_SCL		0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__I2C2_SDA		0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2_gpio: i2c2gpiogrp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL3__GPIO4_IO12		0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__GPIO4_IO13		0x4001b8b1
+		>;
+	};
+
+	pinctrl_leds1: leds1grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D27__GPIO3_IO27		0x1b088 /* GREEN_LED */
+			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x1b088 /* RED_LED */
+			MX6QDL_PAD_GPIO_9__GPIO1_IO09		0x1b088 /* YEL_LED */
+			MX6QDL_PAD_DISP0_DAT4__GPIO4_IO25	0x1b088 /* IMX6_BLUE_LED */
+		>;
+	};
+
+	pinctrl_sgtl5000: sgtl5000grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT7__AUD3_RXD		0x130b0
+			MX6QDL_PAD_CSI0_DAT4__AUD3_TXC		0x130b0
+			MX6QDL_PAD_CSI0_DAT5__AUD3_TXD		0x110b0
+			MX6QDL_PAD_CSI0_DAT6__AUD3_TXFS		0x130b0
+			MX6QDL_PAD_GPIO_0__CCM_CLKO1		0x130b0 /* Audio CLK */
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA	0x1b088
+			MX6QDL_PAD_SD3_DAT6__UART1_RX_DATA	0x1b088
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_7__UART2_TX_DATA	0x1b088
+			MX6QDL_PAD_GPIO_8__UART2_RX_DATA	0x1b088
+			MX6QDL_PAD_SD4_DAT6__UART2_CTS_B	0x1b088
+			MX6QDL_PAD_SD4_DAT5__UART2_RTS_B	0x1b088
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D24__UART3_TX_DATA	0x1b088
+			MX6QDL_PAD_EIM_D25__UART3_RX_DATA	0x1b088
+			MX6QDL_PAD_EIM_D30__UART3_RTS_B		0x1b088
+			MX6QDL_PAD_EIM_D31__UART3_CTS_B		0x1b088
+		>;
+	};
+
+	pinctrl_uart4: uart4grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL0__UART4_TX_DATA	0x1b088
+			MX6QDL_PAD_KEY_ROW0__UART4_RX_DATA	0x1b088
+		>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL1__UART5_TX_DATA	0x1b088
+			MX6QDL_PAD_KEY_ROW1__UART5_RX_DATA	0x1b088
+		>;
+	};
+
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_1__USB_OTG_ID		0x17059
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_CMD__SD1_CMD		0x17059
+			MX6QDL_PAD_SD1_CLK__SD1_CLK		0x10059
+			MX6QDL_PAD_SD1_DAT0__SD1_DATA0		0x17059
+			MX6QDL_PAD_SD1_DAT1__SD1_DATA1		0x17059
+			MX6QDL_PAD_SD1_DAT2__SD1_DATA2		0x17059
+			MX6QDL_PAD_SD1_DAT3__SD1_DATA3		0x17059
+			MX6QDL_PAD_ENET_RXD1__GPIO1_IO26	0x17059 /* WIFI IRQ */
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD2_CMD__SD2_CMD		0x17059
+			MX6QDL_PAD_SD2_CLK__SD2_CLK		0x10059
+			MX6QDL_PAD_SD2_DAT0__SD2_DATA0		0x17059
+			MX6QDL_PAD_SD2_DAT1__SD2_DATA1		0x17059
+			MX6QDL_PAD_SD2_DAT2__SD2_DATA2		0x17059
+			MX6QDL_PAD_SD2_DAT3__SD2_DATA3		0x17059
+			MX6QDL_PAD_EIM_EB0__GPIO2_IO28		0x1b088 /* EN_SD_POWER */
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD		0x17059
+			MX6QDL_PAD_SD3_CLK__SD3_CLK		0x10059
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0		0x17059
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1		0x17059
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2		0x17059
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3		0x17059
+		>;
+	};
+};
+
+&pcie {
+	status = "okay";
+};
+
+&snvs_rtc {
+	status = "disabled";
+};
+
+&ssi1 {
+	fsl,mode = "i2s-slave";
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>;
+	status = "okay";
+};
+
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	status = "okay";
+};
+
+&usbh1 {
+	status = "okay";
+};
+
+&usbotg {
+	vbus-supply = <&reg_usb_otg_vbus>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbotg>;
+	disable-over-current;
+	status = "okay";
+};
+
+/* WIFI */
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	vmmc-supply = <&reg_wlan_vmmc>;
+	bus-width = <4>;
+	non-removable;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1271";
+		reg = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <26 IRQ_TYPE_LEVEL_HIGH>;
+		ref-clock-frequency = <38400000>;
+	};
+};
+
+/* SD */
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	vmmc-supply = <&reg_3p3v>;
+	bus-width = <4>;
+	fsl,wp-controller;
+	status = "okay";
+};
+
+/* eMMC */
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	vmmc-supply = <&reg_3p3v>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6qdl-udoo.dtsi b/arch/arm/boot/dts/imx6qdl-udoo.dtsi
index c96c91d83678..4161b7d4323a 100644
--- a/arch/arm/boot/dts/imx6qdl-udoo.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-udoo.dtsi
@@ -28,6 +28,13 @@
 		status = "disabled";
 	};
 
+	gpio-poweroff {
+		compatible = "gpio-poweroff";
+		gpios = <&gpio2 4 0>;
+		pinctrl-0 = <&pinctrl_power_off>;
+		pinctrl-names = "default";
+	};
+
 	memory {
 		reg = <0x10000000 0x40000000>;
 	};
@@ -172,6 +179,12 @@
 			>;
 		};
 
+		pinctrl_power_off: poweroffgrp {
+			fsl,pins = <
+				MX6QDL_PAD_NANDF_D4__GPIO2_IO04		0x30
+			>;
+		};
+
 		pinctrl_touchscreenp7: touchscreenp7grp {
 			fsl,pins = <
 				MX6QDL_PAD_SD2_DAT0__GPIO1_IO15		0x70
@@ -208,8 +221,8 @@
 			fsl,pins = <
 				MX6QDL_PAD_DI0_PIN2__AUD6_TXD		0x1b0b0
 				MX6QDL_PAD_DI0_PIN3__AUD6_TXFS		0x1b0b0
-				MX6QDL_PAD_DI0_PIN4__AUD6_RXD		0x1b0b0
-				MX6QDL_PAD_DI0_PIN15__AUD6_TXC		0x1b0b0
+				MX6QDL_PAD_DI0_PIN4__AUD6_RXD		0x13080
+				MX6QDL_PAD_DI0_PIN15__AUD6_TXC		0x13080
 				MX6QDL_PAD_EIM_EB2__GPIO2_IO30		0x1b0b0
 			>;
 		};
@@ -218,8 +231,8 @@
 			fsl,pins = <
 				MX6QDL_PAD_DI0_PIN2__AUD6_TXD		0x1b0b0
 				MX6QDL_PAD_DI0_PIN3__GPIO4_IO19		0x1b0b0
-				MX6QDL_PAD_DI0_PIN4__AUD6_RXD		0x1b0b0
-				MX6QDL_PAD_DI0_PIN15__AUD6_TXC		0x1b0b0
+				MX6QDL_PAD_DI0_PIN4__AUD6_RXD		0x13080
+				MX6QDL_PAD_DI0_PIN15__AUD6_TXC		0x13080
 				MX6QDL_PAD_EIM_EB2__GPIO2_IO30		0x1b0b0
 			>;
 		};
@@ -228,8 +241,8 @@
 			fsl,pins = <
 				MX6QDL_PAD_DI0_PIN2__GPIO4_IO18		0x1b0b0
 				MX6QDL_PAD_DI0_PIN3__GPIO4_IO19		0x1b0b0
-				MX6QDL_PAD_DI0_PIN4__AUD6_RXD		0x1b0b0
-				MX6QDL_PAD_DI0_PIN15__AUD6_TXC		0x1b0b0
+				MX6QDL_PAD_DI0_PIN4__AUD6_RXD		0x13080
+				MX6QDL_PAD_DI0_PIN15__AUD6_TXC		0x13080
 				MX6QDL_PAD_EIM_EB2__GPIO2_IO30		0x1b0b0
 			>;
 		};
diff --git a/arch/arm/boot/dts/imx6qdl-var-dart.dtsi b/arch/arm/boot/dts/imx6qdl-var-dart.dtsi
new file mode 100644
index 000000000000..421d6f527609
--- /dev/null
+++ b/arch/arm/boot/dts/imx6qdl-var-dart.dtsi
@@ -0,0 +1,503 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Support for Variscite DART-MX6 Module
+ *
+ * Copyright 2017 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/sound/fsl-imx-audmux.h>
+
+/ {
+	memory {
+		reg = <0x10000000 0x40000000>;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_wl18xx_vmmc: regulator-wl18xx {
+		compatible = "regulator-fixed";
+		regulator-name = "vwl1807";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpio7 8 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <70000>;
+	};
+};
+
+&audmux {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_audmux>;
+	status = "okay";
+
+	ssi2 {
+		fsl,audmux-port = <1>;
+		fsl,port-config = <
+			(IMX_AUDMUX_V2_PTCR_SYN |
+			IMX_AUDMUX_V2_PTCR_TFSDIR |
+			IMX_AUDMUX_V2_PTCR_TFSEL(2) |
+			IMX_AUDMUX_V2_PTCR_TCLKDIR |
+			IMX_AUDMUX_V2_PTCR_TCSEL(2))
+			IMX_AUDMUX_V2_PDCR_RXDSEL(2)
+		>;
+	};
+
+	aud3 {
+		fsl,audmux-port = <2>;
+		fsl,port-config = <
+			IMX_AUDMUX_V2_PTCR_SYN
+			IMX_AUDMUX_V2_PDCR_RXDSEL(1)
+		>;
+	};
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "disabled";
+};
+
+&can2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	status = "disabled";
+};
+
+&ecspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	status = "disabled";
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-mode = "rgmii";
+	status = "disabled";
+};
+
+&hdmi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hdmicec>;
+	ddc-i2c-bus = <&i2c1>;
+	status = "disabled";
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "disabled";
+};
+
+&i2c2 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+
+	pmic@8 {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pmic>;
+		compatible = "fsl,pfuze100";
+		reg = <0x08>;
+
+		regulators {
+			sw1a_reg: sw1ab {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw1c_reg: sw1c {
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1875000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <6250>;
+			};
+
+			sw2_reg: sw2 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw3a_reg: sw3a {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3950000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw3b_reg: sw3b {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3950000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			sw4_reg: sw4 {
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3950000>;
+			};
+
+			snvs_reg: vsnvs {
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vref_reg: vrefddr {
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			vgen6_reg: vgen6 {
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-always-on;
+				regulator-boot-on;
+			};
+		};
+	};
+
+	tlv320aic3106: codec@1b {
+		compatible = "ti,tlv320aic3106";
+		reg = <0x1b>;
+		#sound-dai-cells = <0>;
+		DRVDD-supply = <&reg_3p3v>;
+		AVDD-supply = <&reg_3p3v>;
+		IOVDD-supply = <&reg_3p3v>;
+		DVDD-supply = <&reg_3p3v>;
+		ai3x-ocmv = <0>;
+		gpio-reset = <&gpio5 5 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	status = "disabled";
+};
+
+&iomuxc {
+	pinctrl_audmux: audmux {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT7__AUD3_RXD		0x130b0
+			MX6QDL_PAD_CSI0_DAT4__AUD3_TXC		0x130b0
+			MX6QDL_PAD_CSI0_DAT5__AUD3_TXD		0x110b0
+			MX6QDL_PAD_CSI0_DAT6__AUD3_TXFS		0x130b0
+			/* Audio Clock */
+			MX6QDL_PAD_GPIO_0__CCM_CLKO1		0x130b0
+		>;
+	};
+
+	pinctrl_bt: bt {
+		fsl,pins = <
+			/* Bluetooth enable */
+			MX6QDL_PAD_SD3_DAT6__GPIO6_IO18		0x1b0b1
+			/* Bluetooth Slow Clock */
+			MX6QDL_PAD_ENET_RXD0__OSC32K_32K_OUT	0x000b0
+		>;
+	};
+
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL1__ECSPI1_MISO	0x100b1
+			MX6QDL_PAD_KEY_ROW0__ECSPI1_MOSI	0x100b1
+			MX6QDL_PAD_KEY_COL0__ECSPI1_SCLK	0x100b1
+			/* SPI1 CS0 */
+			MX6QDL_PAD_KEY_ROW1__GPIO4_IO09		0x1b0b0
+			/* SPI1 CS1 */
+			MX6QDL_PAD_KEY_COL2__GPIO4_IO10		0x1b0b0
+		>;
+	};
+
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_MDIO__ENET_MDIO		0x100b0
+			MX6QDL_PAD_ENET_MDC__ENET_MDC		0x100b0
+			MX6QDL_PAD_RGMII_TXC__RGMII_TXC		0x10030
+			MX6QDL_PAD_RGMII_TD0__RGMII_TD0		0x10030
+			MX6QDL_PAD_RGMII_TD1__RGMII_TD1		0x10030
+			MX6QDL_PAD_RGMII_TD2__RGMII_TD2		0x10030
+			MX6QDL_PAD_RGMII_TD3__RGMII_TD3		0x10030
+			MX6QDL_PAD_RGMII_TX_CTL__RGMII_TX_CTL	0x10030
+			MX6QDL_PAD_ENET_REF_CLK__ENET_TX_CLK	0x100b0
+			MX6QDL_PAD_RGMII_RXC__RGMII_RXC		0x1b030
+			MX6QDL_PAD_RGMII_RD0__RGMII_RD0		0x1b030
+			MX6QDL_PAD_RGMII_RD1__RGMII_RD1		0x1b030
+			MX6QDL_PAD_RGMII_RD2__RGMII_RD2		0x1b030
+			MX6QDL_PAD_RGMII_RD3__RGMII_RD3		0x1b030
+			MX6QDL_PAD_RGMII_RX_CTL__RGMII_RX_CTL	0x1b030
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_7__FLEXCAN1_TX	0x1b0b0
+			MX6QDL_PAD_GPIO_8__FLEXCAN1_RX	0x1b0b0
+		>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL4__FLEXCAN2_TX	0x1b0b0
+			MX6QDL_PAD_KEY_ROW4__FLEXCAN2_RX	0x1b0b0
+		>;
+	};
+
+	pinctrl_hdmicec: hdmicecgrp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_ROW2__HDMI_TX_CEC_LINE	0x1f8b0
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT8__I2C1_SDA	0x4001b8b1
+			MX6QDL_PAD_CSI0_DAT9__I2C1_SCL	0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX6QDL_PAD_KEY_COL3__I2C2_SCL	0x4001b8b1
+			MX6QDL_PAD_KEY_ROW3__I2C2_SDA	0x4001b8b1
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_5__I2C3_SCL	0x4001b8b1
+			MX6QDL_PAD_GPIO_16__I2C3_SDA	0x4001b8b1
+		>;
+	};
+
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			/* PMIC INT */
+			MX6QDL_PAD_GPIO_17__GPIO7_IO12	0x1b0b1
+		>;
+	};
+
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <
+			MX6QDL_PAD_DISP0_DAT9__PWM2_OUT	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX6QDL_PAD_CSI0_DAT11__UART1_RX_DATA 0x1b0b1
+			MX6QDL_PAD_CSI0_DAT10__UART1_TX_DATA 0x1b0b1
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_DAT4__UART2_RX_DATA	0x1b0b1
+			MX6QDL_PAD_SD3_DAT5__UART2_TX_DATA	0x1b0b1
+			MX6QDL_PAD_SD4_DAT6__UART2_CTS_B	0x1b0b1
+			MX6QDL_PAD_SD4_DAT5__UART2_RTS_B	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX6QDL_PAD_EIM_D25__UART3_RX_DATA 0x1b0b1
+			MX6QDL_PAD_EIM_D24__UART3_TX_DATA 0x1b0b1
+			MX6QDL_PAD_EIM_D23__UART3_CTS_B   0x1b0b1
+			MX6QDL_PAD_EIM_EB3__UART3_RTS_B   0x1b0b1
+		>;
+	};
+
+	pinctrl_usbotg: usbotggrp {
+		fsl,pins = <
+			MX6QDL_PAD_ENET_RX_ER__USB_OTG_ID 0x17059
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_CMD__SD1_CMD	0x17059
+			MX6QDL_PAD_SD1_CLK__SD1_CLK	0x10059
+			MX6QDL_PAD_SD1_DAT0__SD1_DATA0	0x17059
+			MX6QDL_PAD_SD1_DAT1__SD1_DATA1	0x17059
+			MX6QDL_PAD_SD1_DAT2__SD1_DATA2	0x17059
+			MX6QDL_PAD_SD1_DAT3__SD1_DATA3	0x17059
+			/* WL_EN */
+			MX6QDL_PAD_SD3_DAT7__GPIO6_IO17 0x17071
+			/* WL_IRQ */
+			MX6QDL_PAD_SD3_RST__GPIO7_IO08  0x17071
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1grp100mhz {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_CMD__SD1_CMD	0x170B9
+			MX6QDL_PAD_SD1_CLK__SD1_CLK	0x100B9
+			MX6QDL_PAD_SD1_DAT0__SD1_DATA0	0x170B9
+			MX6QDL_PAD_SD1_DAT1__SD1_DATA1	0x170B9
+			MX6QDL_PAD_SD1_DAT2__SD1_DATA2	0x170B9
+			MX6QDL_PAD_SD1_DAT3__SD1_DATA3	0x170B9
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1grp200mhz {
+		fsl,pins = <
+			MX6QDL_PAD_SD1_CMD__SD1_CMD	0x170F9
+			MX6QDL_PAD_SD1_CLK__SD1_CLK	0x100F9
+			MX6QDL_PAD_SD1_DAT0__SD1_DATA0	0x170F9
+			MX6QDL_PAD_SD1_DAT1__SD1_DATA1	0x170F9
+			MX6QDL_PAD_SD1_DAT2__SD1_DATA2	0x170F9
+			MX6QDL_PAD_SD1_DAT3__SD1_DATA3	0x170F9
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD2_CMD__SD2_CMD	0x17059
+			MX6QDL_PAD_SD2_CLK__SD2_CLK	0x10059
+			MX6QDL_PAD_SD2_DAT0__SD2_DATA0	0x17059
+			MX6QDL_PAD_SD2_DAT1__SD2_DATA1	0x17059
+			MX6QDL_PAD_SD2_DAT2__SD2_DATA2	0x17059
+			MX6QDL_PAD_SD2_DAT3__SD2_DATA3	0x17059
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX6QDL_PAD_SD3_CMD__SD3_CMD	0x17059
+			MX6QDL_PAD_SD3_CLK__SD3_CLK	0x10059
+			MX6QDL_PAD_SD3_DAT0__SD3_DATA0	0x17059
+			MX6QDL_PAD_SD3_DAT1__SD3_DATA1	0x17059
+			MX6QDL_PAD_SD3_DAT2__SD3_DATA2	0x17059
+			MX6QDL_PAD_SD3_DAT3__SD3_DATA3	0x17059
+		>;
+	};
+};
+
+&pcie {
+	fsl,tx-swing-full = <103>;
+	fsl,tx-swing-low = <103>;
+	reset-gpio = <&gpio4 11 GPIO_ACTIVE_LOW>;
+	status = "disabled";
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+	status = "disabled";
+};
+
+&reg_arm {
+	vin-supply = <&sw1a_reg>;
+};
+
+&reg_pu {
+	vin-supply = <&sw1c_reg>;
+};
+
+&reg_soc {
+	vin-supply = <&sw1c_reg>;
+};
+
+&snvs_poweroff {
+	status = "okay";
+};
+
+&ssi2 {
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "disabled";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2 &pinctrl_bt>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "ti,wl1835-st";
+		enable-gpios = <&gpio6 18 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	uart-has-rtscts;
+	status = "disabled";
+};
+
+&usbh1 {
+	status = "disabled";
+};
+
+&usbotg {
+	vbus-supply = <&reg_usb_otg_vbus>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbotg>;
+	disable-over-current;
+	status = "disabled";
+};
+
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	bus-width = <4>;
+	vmmc-supply = <&reg_wl18xx_vmmc>;
+	non-removable;
+	wakeup-source;
+	keep-power-in-suspend;
+	cap-power-off-card;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1835";
+		reg = <2>;
+		interrupt-parent = <&gpio6>;
+		interrupts = <17 IRQ_TYPE_LEVEL_HIGH>;
+		ref-clock-frequency = <38400000>;
+	};
+};
+
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	no-1-8-v;
+	keep-power-in-suspend;
+	wakeup-source;
+	status = "disabled";
+};
+
+&usdhc3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	non-removable;
+	keep-power-in-suspend;
+	wakeup-source;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi b/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi
index 6d8d9ca96646..3a8a4952d45e 100644
--- a/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-wandboard-revd1.dtsi
@@ -21,6 +21,11 @@
 	};
 };
 
+&hdmi {
+	ddc-i2c-bus = <&i2c2>;
+	status = "okay";
+};
+
 &i2c3 {
 	clock-frequency = <100000>;
 	pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi b/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi
index 7812fbac963c..72f52fcecee1 100644
--- a/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-zii-rdu2.dtsi
@@ -49,6 +49,7 @@
 
 	aliases {
 		mdio-gpio0 = &mdio1;
+		rtc0 = &ds1341;
 	};
 
 	mdio1: mdio {
@@ -501,7 +502,7 @@
 		reg = <0x54>;
 	};
 
-	rtc@68 {
+	ds1341: rtc@68 {
 		compatible = "dallas,ds1341";
 		reg = <0x68>;
 	};
@@ -580,6 +581,17 @@
 	pinctrl-0 = <&pinctrl_pcie>;
 	reset-gpio = <&gpio7 12 GPIO_ACTIVE_LOW>;
 	status = "okay";
+
+	host@0 {
+		reg = <0 0 0 0 0>;
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		i210: i210@0 {
+			reg = <0 0 0 0 0>;
+		};
+	};
 };
 
 &usdhc2 {
diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi
index 1ce4eabf0590..59ff86695a14 100644
--- a/arch/arm/boot/dts/imx6qdl.dtsi
+++ b/arch/arm/boot/dts/imx6qdl.dtsi
@@ -80,6 +80,75 @@
 		};
 	};
 
+	tempmon: tempmon {
+		compatible = "fsl,imx6q-tempmon";
+		interrupt-parent = <&gpc>;
+		interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>;
+		fsl,tempmon = <&anatop>;
+		fsl,tempmon-data = <&ocotp>;
+		clocks = <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
+	};
+
+	ldb: ldb {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "fsl,imx6q-ldb", "fsl,imx53-ldb";
+		gpr = <&gpr>;
+		status = "disabled";
+
+		lvds-channel@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+			status = "disabled";
+
+			port@0 {
+				reg = <0>;
+
+				lvds0_mux_0: endpoint {
+					remote-endpoint = <&ipu1_di0_lvds0>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				lvds0_mux_1: endpoint {
+					remote-endpoint = <&ipu1_di1_lvds0>;
+				};
+			};
+		};
+
+		lvds-channel@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+			status = "disabled";
+
+			port@0 {
+				reg = <0>;
+
+				lvds1_mux_0: endpoint {
+					remote-endpoint = <&ipu1_di0_lvds1>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				lvds1_mux_1: endpoint {
+					remote-endpoint = <&ipu1_di1_lvds1>;
+				};
+			};
+		};
+	};
+
+	pmu {
+		compatible = "arm,cortex-a9-pmu";
+		interrupt-parent = <&gpc>;
+		interrupts = <0 94 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -224,11 +293,6 @@
 			status = "disabled";
 		};
 
-		pmu {
-			compatible = "arm,cortex-a9-pmu";
-			interrupts = <0 94 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
 		aips-bus@2000000 { /* AIPS1 */
 			compatible = "fsl,aips-bus", "simple-bus";
 			#address-cells = <1>;
@@ -631,8 +695,11 @@
 				interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 127 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-				regulator-1p1 {
+				regulator-1p1@20c8110 {
+					reg = <0x20c8110>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p1";
 					regulator-min-microvolt = <1000000>;
@@ -647,7 +714,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-3p0 {
+				regulator-3p0@20c8120 {
+					reg = <0x20c8120>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2800000>;
@@ -662,7 +730,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-2p5 {
+				regulator-2p5@20c8130 {
+					reg = <0x20c8130>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd2p5";
 					regulator-min-microvolt = <2250000>;
@@ -677,7 +746,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore {
+				reg_arm: regulator-vddcore@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddarm";
 					regulator-min-microvolt = <725000>;
@@ -694,7 +764,8 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_pu: regulator-vddpu {
+				reg_pu: regulator-vddpu@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddpu";
 					regulator-min-microvolt = <725000>;
@@ -711,7 +782,8 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc {
+				reg_soc: regulator-vddsoc@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
@@ -729,14 +801,6 @@
 				};
 			};
 
-			tempmon: tempmon {
-				compatible = "fsl,imx6q-tempmon";
-				interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>;
-				fsl,tempmon = <&anatop>;
-				fsl,tempmon-data = <&ocotp>;
-				clocks = <&clks IMX6QDL_CLK_PLL3_USB_OTG>;
-			};
-
 			usbphy1: usbphy@20c9000 {
 				compatible = "fsl,imx6q-usbphy", "fsl,imx23-usbphy";
 				reg = <0x020c9000 0x1000>;
@@ -773,6 +837,10 @@
 					mask = <0x60>;
 					status = "disabled";
 				};
+
+				snvs_lpgpr: snvs-lpgpr {
+					compatible = "fsl,imx6q-snvs-lpgpr";
+				};
 			};
 
 			epit1: epit@20d0000 { /* EPIT1 */
@@ -841,60 +909,6 @@
 				reg = <0x20e0000 0x4000>;
 			};
 
-			ldb: ldb {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				compatible = "fsl,imx6q-ldb", "fsl,imx53-ldb";
-				gpr = <&gpr>;
-				status = "disabled";
-
-				lvds-channel@0 {
-					#address-cells = <1>;
-					#size-cells = <0>;
-					reg = <0>;
-					status = "disabled";
-
-					port@0 {
-						reg = <0>;
-
-						lvds0_mux_0: endpoint {
-							remote-endpoint = <&ipu1_di0_lvds0>;
-						};
-					};
-
-					port@1 {
-						reg = <1>;
-
-						lvds0_mux_1: endpoint {
-							remote-endpoint = <&ipu1_di1_lvds0>;
-						};
-					};
-				};
-
-				lvds-channel@1 {
-					#address-cells = <1>;
-					#size-cells = <0>;
-					reg = <1>;
-					status = "disabled";
-
-					port@0 {
-						reg = <0>;
-
-						lvds1_mux_0: endpoint {
-							remote-endpoint = <&ipu1_di0_lvds1>;
-						};
-					};
-
-					port@1 {
-						reg = <1>;
-
-						lvds1_mux_1: endpoint {
-							remote-endpoint = <&ipu1_di1_lvds1>;
-						};
-					};
-				};
-			};
-
 			dcic1: dcic@20e4000 {
 				reg = <0x020e4000 0x4000>;
 				interrupts = <0 124 IRQ_TYPE_LEVEL_HIGH>;
@@ -1017,6 +1031,7 @@
 			fec: ethernet@2188000 {
 				compatible = "fsl,imx6q-fec";
 				reg = <0x02188000 0x4000>;
+				interrupt-names = "int0", "pps";
 				interrupts-extended =
 					<&intc 0 118 IRQ_TYPE_LEVEL_HIGH>,
 					<&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/imx6sl-evk.dts b/arch/arm/boot/dts/imx6sl-evk.dts
index 60600b4cf5fe..2844ab541759 100644
--- a/arch/arm/boot/dts/imx6sl-evk.dts
+++ b/arch/arm/boot/dts/imx6sl-evk.dts
@@ -20,7 +20,7 @@
 		reg = <0x80000000 0x40000000>;
 	};
 
-	backlight {
+	backlight_display: backlight_display {
 		compatible = "pwm-backlight";
 		pwms = <&pwm1 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -39,58 +39,54 @@
 		};
 	};
 
-	regulators {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		reg_usb_otg1_vbus: regulator@0 {
-			compatible = "regulator-fixed";
-			reg = <0>;
-			regulator-name = "usb_otg1_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-			gpio = <&gpio4 0 0>;
-			enable-active-high;
-			vin-supply = <&swbst_reg>;
-		};
+	reg_usb_otg1_vbus: regulator-usb-otg1-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio4 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		vin-supply = <&swbst_reg>;
+	};
 
-		reg_usb_otg2_vbus: regulator@1 {
-			compatible = "regulator-fixed";
-			reg = <1>;
-			regulator-name = "usb_otg2_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-			gpio = <&gpio4 2 0>;
-			enable-active-high;
-			vin-supply = <&swbst_reg>;
-		};
+	reg_usb_otg2_vbus: regulator-usb-otg2-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb_otg2_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio4 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		vin-supply = <&swbst_reg>;
+	};
 
-		reg_aud3v: regulator@2 {
-			compatible = "regulator-fixed";
-			reg = <2>;
-			regulator-name = "wm8962-supply-3v15";
-			regulator-min-microvolt = <3150000>;
-			regulator-max-microvolt = <3150000>;
-			regulator-boot-on;
-		};
+	reg_aud3v: regulator-aud3v {
+		compatible = "regulator-fixed";
+		regulator-name = "wm8962-supply-3v15";
+		regulator-min-microvolt = <3150000>;
+		regulator-max-microvolt = <3150000>;
+		regulator-boot-on;
+	};
 
-		reg_aud4v: regulator@3 {
-			compatible = "regulator-fixed";
-			reg = <3>;
-			regulator-name = "wm8962-supply-4v2";
-			regulator-min-microvolt = <4325000>;
-			regulator-max-microvolt = <4325000>;
-			regulator-boot-on;
-		};
+	reg_aud4v: regulator-aud4v {
+		compatible = "regulator-fixed";
+		regulator-name = "wm8962-supply-4v2";
+		regulator-min-microvolt = <4325000>;
+		regulator-max-microvolt = <4325000>;
+		regulator-boot-on;
+	};
 
-		reg_lcd_3v3: regulator@4 {
-			compatible = "regulator-fixed";
-			reg = <4>;
-			regulator-name = "lcd-3v3";
-			gpio = <&gpio4 3 0>;
-			enable-active-high;
-		};
+	reg_lcd_3v3: regulator-lcd-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "lcd-3v3";
+		gpio = <&gpio4 3 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_lcd_5v: regulator-lcd-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "lcd-5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
 	};
 
 	sound {
@@ -108,6 +104,19 @@
 		mux-int-port = <2>;
 		mux-ext-port = <3>;
 	};
+
+	panel {
+		compatible = "sii,43wvf1g";
+		backlight = <&backlight_display>;
+		dvdd-supply = <&reg_lcd_3v3>;
+		avdd-supply = <&reg_lcd_5v>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&display_out>;
+			};
+		};
+	};
 };
 
 &audmux {
@@ -546,31 +555,11 @@
 &lcdif {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_lcd>;
-	lcd-supply = <&reg_lcd_3v3>;
-	display = <&display0>;
 	status = "okay";
 
-	display0: display0 {
-		bits-per-pixel = <32>;
-		bus-width = <24>;
-
-		display-timings {
-			native-mode = <&timing0>;
-			timing0: timing0 {
-				clock-frequency = <33500000>;
-				hactive = <800>;
-				vactive = <480>;
-				hback-porch = <89>;
-				hfront-porch = <164>;
-				vback-porch = <23>;
-				vfront-porch = <10>;
-				hsync-len = <10>;
-				vsync-len = <10>;
-				hsync-active = <0>;
-				vsync-active = <0>;
-				de-active = <1>;
-				pixelclk-active = <0>;
-			};
+	port {
+		display_out: endpoint {
+			remote-endpoint = <&panel_in>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/imx6sl.dtsi b/arch/arm/boot/dts/imx6sl.dtsi
index 3ea1a41893c8..ae8df3cf687e 100644
--- a/arch/arm/boot/dts/imx6sl.dtsi
+++ b/arch/arm/boot/dts/imx6sl.dtsi
@@ -102,6 +102,21 @@
 		};
 	};
 
+	tempmon: tempmon {
+		compatible = "fsl,imx6q-tempmon";
+		interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-parent = <&gpc>;
+		fsl,tempmon = <&anatop>;
+		fsl,tempmon-data = <&ocotp>;
+		clocks = <&clks IMX6SL_CLK_PLL3_USB_OTG>;
+	};
+
+	pmu {
+		compatible = "arm,cortex-a9-pmu";
+		interrupt-parent = <&gpc>;
+		interrupts = <0 94 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -125,11 +140,6 @@
 			arm,data-latency = <4 2 3>;
 		};
 
-		pmu {
-			compatible = "arm,cortex-a9-pmu";
-			interrupts = <0 94 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
 		aips1: aips-bus@2000000 {
 			compatible = "fsl,aips-bus", "simple-bus";
 			#address-cells = <1>;
@@ -517,8 +527,11 @@
 				interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <0 127 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-				regulator-1p1 {
+				regulator-1p1@20c8110 {
+					reg = <0x20c8110>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p1";
 					regulator-min-microvolt = <800000>;
@@ -533,7 +546,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-3p0 {
+				regulator-3p0@20c8120 {
+					reg = <0x20c8120>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2800000>;
@@ -548,7 +562,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-2p5 {
+				regulator-2p5@20c8130 {
+					reg = <0x20c8130>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd2p5";
 					regulator-min-microvolt = <2100000>;
@@ -563,7 +578,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore {
+				reg_arm: regulator-vddcore@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddarm";
 					regulator-min-microvolt = <725000>;
@@ -580,7 +596,8 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_pu: regulator-vddpu {
+				reg_pu: regulator-vddpu@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddpu";
 					regulator-min-microvolt = <725000>;
@@ -597,7 +614,8 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc {
+				reg_soc: regulator-vddsoc@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
@@ -615,14 +633,6 @@
 				};
 			};
 
-			tempmon: tempmon {
-				compatible = "fsl,imx6q-tempmon";
-				interrupts = <0 49 IRQ_TYPE_LEVEL_HIGH>;
-				fsl,tempmon = <&anatop>;
-				fsl,tempmon-data = <&ocotp>;
-				clocks = <&clks IMX6SL_CLK_PLL3_USB_OTG>;
-			};
-
 			usbphy1: usbphy@20c9000 {
 				compatible = "fsl,imx6sl-usbphy", "fsl,imx23-usbphy";
 				reg = <0x020c9000 0x1000>;
diff --git a/arch/arm/boot/dts/imx6sx-sdb.dtsi b/arch/arm/boot/dts/imx6sx-sdb.dtsi
index da815527a7f8..d35aa858f9db 100644
--- a/arch/arm/boot/dts/imx6sx-sdb.dtsi
+++ b/arch/arm/boot/dts/imx6sx-sdb.dtsi
@@ -24,7 +24,7 @@
 		reg = <0x80000000 0x40000000>;
 	};
 
-	backlight {
+	backlight_display: backlight-display {
 		compatible = "pwm-backlight";
 		pwms = <&pwm3 0 5000000>;
 		brightness-levels = <0 4 8 16 32 64 128 255>;
@@ -49,86 +49,91 @@
 		};
 	};
 
-	regulators {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
+	vcc_sd3: regulator-vcc-sd3 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_vcc_sd3>;
+		regulator-name = "VCC_SD3";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		gpio = <&gpio2 11 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 
-		vcc_sd3: regulator@0 {
-			compatible = "regulator-fixed";
-			reg = <0>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_vcc_sd3>;
-			regulator-name = "VCC_SD3";
-			regulator-min-microvolt = <3000000>;
-			regulator-max-microvolt = <3000000>;
-			gpio = <&gpio2 11 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	reg_usb_otg1_vbus: regulator-usb-otg1-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb_otg1>;
+		regulator-name = "usb_otg1_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio1 9 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 
-		reg_usb_otg1_vbus: regulator@1 {
-			compatible = "regulator-fixed";
-			reg = <1>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_usb_otg1>;
-			regulator-name = "usb_otg1_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-			gpio = <&gpio1 9 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	reg_usb_otg2_vbus: regulator-usb-otg2-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb_otg2>;
+		regulator-name = "usb_otg2_vbus";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
 
-		reg_usb_otg2_vbus: regulator@2 {
-			compatible = "regulator-fixed";
-			reg = <2>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_usb_otg2>;
-			regulator-name = "usb_otg2_vbus";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-			gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
-
-		reg_psu_5v: regulator@3 {
-			compatible = "regulator-fixed";
-			reg = <3>;
-			regulator-name = "PSU-5V0";
-			regulator-min-microvolt = <5000000>;
-			regulator-max-microvolt = <5000000>;
-		};
-
-		reg_lcd_3v3: regulator@4 {
-			compatible = "regulator-fixed";
-			reg = <4>;
-			regulator-name = "lcd-3v3";
-			gpio = <&gpio3 27 0>;
-			enable-active-high;
-		};
-
-		reg_peri_3v3: regulator@5 {
-			compatible = "regulator-fixed";
-			reg = <5>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_peri_3v3>;
-			regulator-name = "peri_3v3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			gpio = <&gpio4 16 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-			regulator-always-on;
-		};
-
-		reg_enet_3v3: regulator@6 {
-			compatible = "regulator-fixed";
-			reg = <6>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&pinctrl_enet_3v3>;
-			regulator-name = "enet_3v3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			gpios = <&gpio2 6 GPIO_ACTIVE_LOW>;
-		};
+	reg_psu_5v: regulator-psu-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "PSU-5V0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	reg_lcd_3v3: regulator-lcd-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "lcd-3v3";
+		gpio = <&gpio3 27 0>;
+		enable-active-high;
+	};
+
+	reg_peri_3v3: regulator-peri-3v3 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_peri_3v3>;
+		regulator-name = "peri_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio4 16 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	reg_enet_3v3: regulator-enet-3v3 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_enet_3v3>;
+		regulator-name = "enet_3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpios = <&gpio2 6 GPIO_ACTIVE_LOW>;
+	};
+
+	reg_pcie_gpio: regulator-pcie-gpio {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pcie_reg>;
+		regulator-name = "MPCIE_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio2 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_lcd_5v: regulator-lcd-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "lcd-5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
 	};
 
 	sound {
@@ -146,6 +151,19 @@
 		mux-int-port = <2>;
 		mux-ext-port = <6>;
 	};
+
+	panel {
+		compatible = "sii,43wvf1g";
+		backlight = <&backlight_display>;
+		dvdd-supply = <&reg_lcd_3v3>;
+		avdd-supply = <&reg_lcd_5v>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&display_out>;
+			};
+		};
+	};
 };
 
 &audmux {
@@ -212,34 +230,22 @@
 	};
 };
 
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie>;
+	reset-gpio = <&gpio2 0 GPIO_ACTIVE_LOW>;
+	vpcie-supply = <&reg_pcie_gpio>;
+	status = "okay";
+};
+
 &lcdif1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_lcd>;
-	lcd-supply = <&reg_lcd_3v3>;
-	display = <&display0>;
 	status = "okay";
 
-	display0: display0 {
-		bits-per-pixel = <16>;
-		bus-width = <24>;
-
-		display-timings {
-			native-mode = <&timing0>;
-			timing0: timing0 {
-				clock-frequency = <33500000>;
-				hactive = <800>;
-				vactive = <480>;
-				hback-porch = <89>;
-				hfront-porch = <164>;
-				vback-porch = <23>;
-				vfront-porch = <10>;
-				hsync-len = <10>;
-				vsync-len = <10>;
-				hsync-active = <0>;
-				vsync-active = <0>;
-				de-active = <1>;
-				pixelclk-active = <0>;
-			};
+	port {
+		display_out: endpoint {
+			remote-endpoint = <&panel_in>;
 		};
 	};
 };
@@ -453,6 +459,18 @@
 			>;
 		};
 
+		pinctrl_pcie: pciegrp {
+			fsl,pins = <
+				MX6SX_PAD_ENET1_COL__GPIO2_IO_0 0x10b0
+			>;
+		};
+
+		pinctrl_pcie_reg: pciereggrp {
+			fsl,pins = <
+				MX6SX_PAD_ENET1_CRS__GPIO2_IO_1	0x10b0
+			>;
+		};
+
 		pinctrl_peri_3v3: peri3v3grp {
 			fsl,pins = <
 				MX6SX_PAD_QSPI1A_DATA0__GPIO4_IO_16	0x80000000
diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi
index 5b03ba3beda9..fd7879342d0d 100644
--- a/arch/arm/boot/dts/imx6sx.dtsi
+++ b/arch/arm/boot/dts/imx6sx.dtsi
@@ -141,6 +141,22 @@
 		};
 	};
 
+	tempmon: tempmon {
+		compatible = "fsl,imx6sx-tempmon", "fsl,imx6q-tempmon";
+		interrupt-parent = <&gpc>;
+		interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+		fsl,tempmon = <&anatop>;
+		nvmem-cells = <&tempmon_calib>, <&tempmon_temp_grade>;
+		nvmem-cell-names = "calib", "temp_grade";
+		clocks = <&clks IMX6SX_CLK_PLL3_USB_OTG>;
+	};
+
+	pmu {
+		compatible = "arm,cortex-a9-pmu";
+		interrupt-parent = <&gpc>;
+		interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -148,11 +164,6 @@
 		interrupt-parent = <&gpc>;
 		ranges;
 
-		pmu {
-			compatible = "arm,cortex-a9-pmu";
-			interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
-		};
-
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
 			reg = <0x00900000 0x20000>;
@@ -574,8 +585,11 @@
 				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-				regulator-1p1 {
+				regulator-1p1@20c8110 {
+					reg = <0x20c8110>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p1";
 					regulator-min-microvolt = <800000>;
@@ -590,7 +604,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-3p0 {
+				regulator-3p0@20c8120 {
+					reg = <0x20c8120>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2800000>;
@@ -605,7 +620,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				regulator-2p5 {
+				regulator-2p5@20c8130 {
+					reg = <0x20c8130>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd2p5";
 					regulator-min-microvolt = <2100000>;
@@ -620,7 +636,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore {
+				reg_arm: regulator-vddcore@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddarm";
 					regulator-min-microvolt = <725000>;
@@ -637,7 +654,8 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_pcie: regulator-vddpcie {
+				reg_pcie: regulator-vddpcie@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddpcie";
 					regulator-min-microvolt = <725000>;
@@ -653,7 +671,8 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc {
+				reg_soc: regulator-vddsoc@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
@@ -671,15 +690,6 @@
 				};
 			};
 
-			tempmon: tempmon {
-				compatible = "fsl,imx6sx-tempmon", "fsl,imx6q-tempmon";
-				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
-				fsl,tempmon = <&anatop>;
-				nvmem-cells = <&tempmon_calib>, <&tempmon_temp_grade>;
-				nvmem-cell-names = "calib", "temp_grade";
-				clocks = <&clks IMX6SX_CLK_PLL3_USB_OTG>;
-			};
-
 			usbphy1: usbphy@20c9000 {
 				compatible = "fsl,imx6sx-usbphy", "fsl,imx23-usbphy";
 				reg = <0x020c9000 0x1000>;
@@ -750,6 +760,19 @@
 				#interrupt-cells = <3>;
 				interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
 				interrupt-parent = <&intc>;
+				clocks = <&clks IMX6SX_CLK_IPG>;
+				clock-names = "ipg";
+
+				pgc {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					pd_pci: power-domain@3 {
+						reg = <3>;
+						#power-domain-cells = <0>;
+						power-supply = <&reg_pcie>;
+					};
+				};
 			};
 
 			iomuxc: iomuxc@20e0000 {
@@ -862,6 +885,7 @@
 			fec1: ethernet@2188000 {
 				compatible = "fsl,imx6sx-fec", "fsl,imx6q-fec";
 				reg = <0x02188000 0x4000>;
+				interrupt-names = "int0", "pps";
 				interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX6SX_CLK_ENET>,
@@ -971,6 +995,7 @@
 			fec2: ethernet@21b4000 {
 				compatible = "fsl,imx6sx-fec", "fsl,imx6q-fec";
 				reg = <0x021b4000 0x4000>;
+				interrupt-names = "int0", "pps";
 				interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX6SX_CLK_ENET>,
@@ -1138,7 +1163,7 @@
 			reg = <0x02200000 0x100000>;
 			ranges;
 
-			spba-bus@2200000 {
+			spba-bus@2240000 {
 				compatible = "fsl,spba-bus", "simple-bus";
 				#address-cells = <1>;
 				#size-cells = <1>;
@@ -1304,25 +1329,29 @@
 
 		pcie: pcie@8ffc000 {
 			compatible = "fsl,imx6sx-pcie", "snps,dw-pcie";
-			reg = <0x08ffc000 0x4000>; /* DBI */
+			reg = <0x08ffc000 0x04000>, <0x08f00000 0x80000>;
+			reg-names = "dbi", "config";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
-				  /* configuration space */
-			ranges = <0x00000800 0 0x08f00000 0x08f00000 0 0x00080000
-				  /* downstream I/O */
-				  0x81000000 0 0          0x08f80000 0 0x00010000
-				  /* non-prefetchable memory */
-				  0x82000000 0 0x08000000 0x08000000 0 0x00f00000>;
 			bus-range = <0x00 0xff>;
+			ranges = <0x81000000 0 0          0x08f80000 0 0x00010000 /* downstream I/O */
+				  0x82000000 0 0x08000000 0x08000000 0 0x00f00000>; /* non-prefetchable memory */
 			num-lanes = <1>;
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clks IMX6SX_CLK_PCIE_REF_125M>,
-				 <&clks IMX6SX_CLK_PCIE_AXI>,
+			interrupt-names = "msi";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &gpc GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gpc GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gpc GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gpc GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clks IMX6SX_CLK_PCIE_AXI>,
 				 <&clks IMX6SX_CLK_LVDS1_OUT>,
+				 <&clks IMX6SX_CLK_PCIE_REF_125M>,
 				 <&clks IMX6SX_CLK_DISPLAY_AXI>;
-			clock-names = "pcie_ref_125m", "pcie_axi",
-				      "lvds_gate", "display_axi";
+			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_inbound_axi";
+			power-domains = <&pd_pci>;
 			status = "disabled";
 		};
 	};
diff --git a/arch/arm/boot/dts/imx6ul-14x14-evk.dts b/arch/arm/boot/dts/imx6ul-14x14-evk.dts
index e5d3ef88be60..18fdb088ba1e 100644
--- a/arch/arm/boot/dts/imx6ul-14x14-evk.dts
+++ b/arch/arm/boot/dts/imx6ul-14x14-evk.dts
@@ -30,19 +30,14 @@
 		status = "okay";
 	};
 
-	regulators {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
 
-		reg_sd1_vmmc: sd1_regulator {
-			compatible = "regulator-fixed";
-			regulator-name = "VSD_3V3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			gpio = <&gpio1 9 GPIO_ACTIVE_HIGH>;
-			enable-active-high;
-		};
+	reg_sd1_vmmc: regulator-sd1-vmmc {
+		compatible = "regulator-fixed";
+		regulator-name = "VSD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio1 9 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
 	};
 
 	sound {
diff --git a/arch/arm/boot/dts/imx6ul.dtsi b/arch/arm/boot/dts/imx6ul.dtsi
index d5181f85ca9c..5d6c3ba36cd1 100644
--- a/arch/arm/boot/dts/imx6ul.dtsi
+++ b/arch/arm/boot/dts/imx6ul.dtsi
@@ -68,12 +68,14 @@
 			clock-latency = <61036>; /* two CLK32 periods */
 			operating-points = <
 				/* kHz	uV */
+				696000	1275000
 				528000	1175000
 				396000	1025000
 				198000	950000
 			>;
 			fsl,soc-operating-points = <
 				/* KHz	uV */
+				696000	1275000
 				528000	1175000
 				396000	1175000
 				198000	1175000
@@ -136,6 +138,23 @@
 		clock-output-names = "ipp_di1";
 	};
 
+	tempmon: tempmon {
+		compatible = "fsl,imx6ul-tempmon", "fsl,imx6sx-tempmon";
+		interrupt-parent = <&gpc>;
+		interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+		fsl,tempmon = <&anatop>;
+		nvmem-cells = <&tempmon_calib>, <&tempmon_temp_grade>;
+		nvmem-cell-names = "calib", "temp_grade";
+		clocks = <&clks IMX6UL_CLK_PLL3_USB_OTG>;
+	};
+
+	pmu {
+		compatible = "arm,cortex-a7-pmu";
+		interrupt-parent = <&gpc>;
+		interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+		status = "disabled";
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -143,12 +162,6 @@
 		interrupt-parent = <&gpc>;
 		ranges;
 
-		pmu {
-			compatible = "arm,cortex-a7-pmu";
-			interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
-			status = "disabled";
-		};
-
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
 			reg = <0x00900000 0x20000>;
@@ -476,6 +489,7 @@
 			fec2: ethernet@20b4000 {
 				compatible = "fsl,imx6ul-fec", "fsl,imx6q-fec";
 				reg = <0x020b4000 0x4000>;
+				interrupt-names = "int0", "pps";
 				interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX6UL_CLK_ENET>,
@@ -530,8 +544,11 @@
 				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-				reg_3p0: regulator-3p0 {
+				reg_3p0: regulator-3p0@20c8110 {
+					reg = <0x20c8110>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd3p0";
 					regulator-min-microvolt = <2625000>;
@@ -545,7 +562,8 @@
 					anatop-enable-bit = <0>;
 				};
 
-				reg_arm: regulator-vddcore {
+				reg_arm: regulator-vddcore@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "cpu";
 					regulator-min-microvolt = <725000>;
@@ -562,7 +580,8 @@
 					anatop-max-voltage = <1450000>;
 				};
 
-				reg_soc: regulator-vddsoc {
+				reg_soc: regulator-vddsoc@20c8140 {
+					reg = <0x20c8140>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vddsoc";
 					regulator-min-microvolt = <725000>;
@@ -598,15 +617,6 @@
 				fsl,anatop = <&anatop>;
 			};
 
-			tempmon: tempmon {
-				compatible = "fsl,imx6ul-tempmon", "fsl,imx6sx-tempmon";
-				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
-				fsl,tempmon = <&anatop>;
-				nvmem-cells = <&tempmon_calib>, <&tempmon_temp_grade>;
-				nvmem-cell-names = "calib", "temp_grade";
-				clocks = <&clks IMX6UL_CLK_PLL3_USB_OTG>;
-			};
-
 			snvs: snvs@20cc000 {
 				compatible = "fsl,sec-v4.0-mon", "syscon", "simple-mfd";
 				reg = <0x020cc000 0x4000>;
@@ -635,6 +645,10 @@
 					linux,keycode = <KEY_POWER>;
 					wakeup-source;
 				};
+
+				snvs_lpgpr: snvs-lpgpr {
+					compatible = "fsl,imx6ul-snvs-lpgpr";
+				};
 			};
 
 			epit1: epit@20d0000 {
@@ -784,6 +798,7 @@
 			fec1: ethernet@2188000 {
 				compatible = "fsl,imx6ul-fec", "fsl,imx6q-fec";
 				reg = <0x02188000 0x4000>;
+				interrupt-names = "int0", "pps";
 				interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
 					     <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX6UL_CLK_ENET>,
diff --git a/arch/arm/boot/dts/imx7-colibri-eval-v3.dtsi b/arch/arm/boot/dts/imx7-colibri-eval-v3.dtsi
index 18bebd6d8d47..3f2746169181 100644
--- a/arch/arm/boot/dts/imx7-colibri-eval-v3.dtsi
+++ b/arch/arm/boot/dts/imx7-colibri-eval-v3.dtsi
@@ -45,6 +45,13 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	/* fixed crystal dedicated to mpc258x */
+	clk16m: clk16m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <16000000>;
+	};
+
 	panel: panel {
 		compatible = "edt,et057090dhu";
 		backlight = <&bl>;
@@ -70,6 +77,17 @@
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
 	};
+
+	reg_usbh_vbus: regulator-usbh-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbh_reg>;
+		regulator-name = "VCC_USB[1-4]";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio4 7 GPIO_ACTIVE_LOW>;
+		vin-supply = <&reg_5v0>;
+	};
 };
 
 &bl {
@@ -88,6 +106,24 @@
 	status = "okay";
 };
 
+&ecspi3 {
+	status = "okay";
+
+	mcp2515: can@0 {
+		compatible = "microchip,mcp2515";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_can_int>;
+		reg = <0>;
+		clocks = <&clk16m>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+		spi-max-frequency = <10000000>;
+		vdd-supply = <&reg_3v3>;
+		xceiver-supply = <&reg_5v0>;
+		status = "okay";
+	};
+};
+
 &fec1 {
 	status = "okay";
 };
@@ -97,7 +133,7 @@
 
 	/* M41T0M6 real time clock on carrier board */
 	rtc: m41t0m6@68 {
-		compatible = "st,m41t00";
+		compatible = "st,m41t0";
 		reg = <0x68>;
 	};
 };
@@ -147,5 +183,6 @@
 &usdhc1 {
 	keep-power-in-suspend;
 	wakeup-source;
+	vmmc-supply = <&reg_3v3>;
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/imx7-colibri.dtsi b/arch/arm/boot/dts/imx7-colibri.dtsi
index bb5bf94f1a32..895fbde4d433 100644
--- a/arch/arm/boot/dts/imx7-colibri.dtsi
+++ b/arch/arm/boot/dts/imx7-colibri.dtsi
@@ -43,7 +43,10 @@
 / {
 	bl: backlight {
 		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_bl_on>;
 		pwms = <&pwm1 0 5000000 0>;
+		enable-gpios = <&gpio5 1 GPIO_ACTIVE_HIGH>;
 	};
 
 	reg_module_3v3: regulator-module-3v3 {
@@ -86,7 +89,13 @@
 };
 
 &cpu0 {
-	arm-supply = <&reg_DCDC2>;
+	cpu-supply = <&reg_DCDC2>;
+};
+
+&ecspi3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi3 &pinctrl_ecspi3_cs>;
+	cs-gpios = <&gpio4 11 GPIO_ACTIVE_HIGH>;
 };
 
 &fec1 {
@@ -112,7 +121,6 @@
 	fsl,use-minimum-ecc;
 	nand-on-flash-bbt;
 	nand-ecc-mode = "hw";
-	status = "okay";
 };
 
 &i2c1 {
@@ -299,6 +307,22 @@
 	no-1-8-v;
 	cd-gpios = <&gpio1 0 GPIO_ACTIVE_LOW>;
 	disable-wp;
+	vqmmc-supply = <&reg_LDO2>;
+};
+
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	assigned-clocks = <&clks IMX7D_USDHC3_ROOT_CLK>;
+	assigned-clock-rates = <400000000>;
+	bus-width = <8>;
+	fsl,tuning-step = <2>;
+	max-frequency = <100000000>;
+	vmmc-supply = <&reg_module_3v3>;
+	vqmmc-supply = <&reg_DCDC3>;
+	non-removable;
 };
 
 &iomuxc {
@@ -307,17 +331,16 @@
 
 	pinctrl_gpio1: gpio1-grp {
 		fsl,pins = <
-			MX7D_PAD_ENET1_RGMII_RD3__GPIO7_IO3	0x14 /* SODIMM 55 */
-			MX7D_PAD_ENET1_RGMII_RD2__GPIO7_IO2	0x14 /* SODIMM 63 */
-			MX7D_PAD_SD1_RESET_B__GPIO5_IO2		0X14 /* SODIMM 73 */
-			MX7D_PAD_SAI1_RX_SYNC__GPIO6_IO16	0X14 /* SODIMM 77 */
+			MX7D_PAD_ENET1_RGMII_RD3__GPIO7_IO3	0x74 /* SODIMM 55 */
+			MX7D_PAD_ENET1_RGMII_RD2__GPIO7_IO2	0x74 /* SODIMM 63 */
+			MX7D_PAD_SAI1_RX_SYNC__GPIO6_IO16	0x14 /* SODIMM 77 */
 			MX7D_PAD_EPDC_DATA09__GPIO2_IO9		0x14 /* SODIMM 89 */
-			MX7D_PAD_EPDC_DATA08__GPIO2_IO8		0x14 /* SODIMM 91 */
+			MX7D_PAD_EPDC_DATA08__GPIO2_IO8		0x74 /* SODIMM 91 */
 			MX7D_PAD_LCD_RESET__GPIO3_IO4		0x14 /* SODIMM 93 */
 			MX7D_PAD_EPDC_DATA13__GPIO2_IO13	0x14 /* SODIMM 95 */
 			MX7D_PAD_ENET1_RGMII_TXC__GPIO7_IO11	0x14 /* SODIMM 99 */
-			MX7D_PAD_EPDC_DATA10__GPIO2_IO10	0x14 /* SODIMM 105 */
-			MX7D_PAD_EPDC_DATA15__GPIO2_IO15	0x14 /* SODIMM 107 */
+			MX7D_PAD_EPDC_DATA10__GPIO2_IO10	0x74 /* SODIMM 105 */
+			MX7D_PAD_EPDC_DATA15__GPIO2_IO15	0x74 /* SODIMM 107 */
 			MX7D_PAD_EPDC_DATA00__GPIO2_IO0		0x14 /* SODIMM 111 */
 			MX7D_PAD_EPDC_DATA01__GPIO2_IO1		0x14 /* SODIMM 113 */
 			MX7D_PAD_EPDC_DATA02__GPIO2_IO2		0x14 /* SODIMM 115 */
@@ -329,11 +352,12 @@
 			MX7D_PAD_EPDC_SDCE2__GPIO2_IO22		0x14 /* SODIMM 127 */
 			MX7D_PAD_UART3_RTS_B__GPIO4_IO6		0x14 /* SODIMM 131 */
 			MX7D_PAD_EPDC_GDRL__GPIO2_IO26		0x14 /* SODIMM 133 */
+			MX7D_PAD_SAI1_RX_DATA__GPIO6_IO12	0x14 /* SODIMM 169 */
 			MX7D_PAD_SAI1_RX_BCLK__GPIO6_IO17	0x14 /* SODIMM 24 */
 			MX7D_PAD_SD2_DATA2__GPIO5_IO16		0x14 /* SODIMM 100 */
 			MX7D_PAD_SD2_DATA3__GPIO5_IO17		0x14 /* SODIMM 102 */
 			MX7D_PAD_EPDC_GDSP__GPIO2_IO27		0x14 /* SODIMM 104 */
-			MX7D_PAD_EPDC_BDR0__GPIO2_IO28		0x14 /* SODIMM 106 */
+			MX7D_PAD_EPDC_BDR0__GPIO2_IO28		0x74 /* SODIMM 106 */
 			MX7D_PAD_EPDC_BDR1__GPIO2_IO29		0x14 /* SODIMM 110 */
 			MX7D_PAD_EPDC_PWR_COM__GPIO2_IO30	0x14 /* SODIMM 112 */
 			MX7D_PAD_EPDC_SDCLK__GPIO2_IO16		0x14 /* SODIMM 114 */
@@ -357,8 +381,7 @@
 	pinctrl_gpio2: gpio2-grp { /* On X22 Camera interface */
 		fsl,pins = <
 			MX7D_PAD_ECSPI2_SS0__GPIO4_IO23		0x14 /* SODIMM 65 */
-			MX7D_PAD_SD1_CD_B__GPIO5_IO0		0x14 /* SODIMM 69 */
-			MX7D_PAD_SD1_WP__GPIO5_IO1		0x14 /* SODIMM 71 */
+			MX7D_PAD_SD1_CD_B__GPIO5_IO0		0x74 /* SODIMM 69 */
 			MX7D_PAD_I2C4_SDA__GPIO4_IO15		0x14 /* SODIMM 75 */
 			MX7D_PAD_ECSPI1_MISO__GPIO4_IO18	0x14 /* SODIMM 79 */
 			MX7D_PAD_I2C3_SCL__GPIO4_IO12		0x14 /* SODIMM 81 */
@@ -378,8 +401,8 @@
 			MX7D_PAD_LCD_DATA19__GPIO3_IO24		0x14 /* SODIMM 138 */
 			MX7D_PAD_LCD_DATA20__GPIO3_IO25		0x14 /* SODIMM 140 */
 			MX7D_PAD_LCD_DATA21__GPIO3_IO26		0x14 /* SODIMM 142 */
-			MX7D_PAD_LCD_DATA22__GPIO3_IO27		0x14 /* SODIMM 146 */
-			MX7D_PAD_LCD_DATA23__GPIO3_IO28		0x14 /* SODIMM 148 */
+			MX7D_PAD_LCD_DATA22__GPIO3_IO27		0x74 /* SODIMM 144 */
+			MX7D_PAD_LCD_DATA23__GPIO3_IO28		0x74 /* SODIMM 146 */
 		>;
 	};
 
@@ -396,6 +419,12 @@
 		>;
 	};
 
+	pinctrl_can_int: can-int-grp {
+		fsl,pins = <
+			MX7D_PAD_SD1_RESET_B__GPIO5_IO2		0X14 /* SODIMM 73 */
+		>;
+	};
+
 	pinctrl_enet1: enet1grp {
 		fsl,pins = <
 			MX7D_PAD_ENET1_CRS__GPIO7_IO14			0x14
@@ -434,12 +463,17 @@
 		>;
 	};
 
+	pinctrl_gpio_bl_on: gpio-bl-on {
+		fsl,pins = <
+			MX7D_PAD_SD1_WP__GPIO5_IO1		0x14 /* SODIMM 71 */
+		>;
+	};
+
 	pinctrl_gpmi_nand: gpmi-nand-grp {
 		fsl,pins = <
 			MX7D_PAD_SD3_CLK__NAND_CLE		0x71
 			MX7D_PAD_SD3_CMD__NAND_ALE		0x71
 			MX7D_PAD_SAI1_TX_BCLK__NAND_CE0_B	0x71
-			MX7D_PAD_SAI1_RX_DATA__NAND_CE1_B	0x71
 			MX7D_PAD_SAI1_TX_DATA__NAND_READY_B	0x74
 			MX7D_PAD_SD3_STROBE__NAND_RE_B		0x71
 			MX7D_PAD_SD3_RESET_B__NAND_WE_B		0x71
@@ -507,6 +541,7 @@
 	pinctrl_pwm1: pwm1-grp {
 		fsl,pins = <
 			MX7D_PAD_GPIO1_IO08__PWM1_OUT		0x79
+			MX7D_PAD_ECSPI2_MOSI__GPIO4_IO21	0x4
 		>;
 	};
 
@@ -525,6 +560,7 @@
 	pinctrl_pwm4: pwm4-grp {
 		fsl,pins = <
 			MX7D_PAD_GPIO1_IO11__PWM4_OUT		0x79
+			MX7D_PAD_ECSPI2_SCLK__GPIO4_IO20	0x4
 		>;
 	};
 
@@ -559,7 +595,7 @@
 		>;
 	};
 
-	pinctrl_usbotg2_reg: gpio-usbotg2-vbus {
+	pinctrl_usbh_reg: gpio-usbh-vbus {
 		fsl,pins = <
 			MX7D_PAD_UART3_CTS_B__GPIO4_IO7	0x14 /* SODIMM 129 USBH PEN */
 		>;
@@ -576,6 +612,54 @@
 		>;
 	};
 
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX7D_PAD_SD3_CMD__SD3_CMD		0x59
+			MX7D_PAD_SD3_CLK__SD3_CLK		0x19
+			MX7D_PAD_SD3_DATA0__SD3_DATA0		0x59
+			MX7D_PAD_SD3_DATA1__SD3_DATA1		0x59
+			MX7D_PAD_SD3_DATA2__SD3_DATA2		0x59
+			MX7D_PAD_SD3_DATA3__SD3_DATA3		0x59
+			MX7D_PAD_SD3_DATA4__SD3_DATA4		0x59
+			MX7D_PAD_SD3_DATA5__SD3_DATA5		0x59
+			MX7D_PAD_SD3_DATA6__SD3_DATA6		0x59
+			MX7D_PAD_SD3_DATA7__SD3_DATA7		0x59
+			MX7D_PAD_SD3_STROBE__SD3_STROBE         0x19
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3grp_100mhz {
+		fsl,pins = <
+			MX7D_PAD_SD3_CMD__SD3_CMD		0x5a
+			MX7D_PAD_SD3_CLK__SD3_CLK		0x1a
+			MX7D_PAD_SD3_DATA0__SD3_DATA0		0x5a
+			MX7D_PAD_SD3_DATA1__SD3_DATA1		0x5a
+			MX7D_PAD_SD3_DATA2__SD3_DATA2		0x5a
+			MX7D_PAD_SD3_DATA3__SD3_DATA3		0x5a
+			MX7D_PAD_SD3_DATA4__SD3_DATA4		0x5a
+			MX7D_PAD_SD3_DATA5__SD3_DATA5		0x5a
+			MX7D_PAD_SD3_DATA6__SD3_DATA6		0x5a
+			MX7D_PAD_SD3_DATA7__SD3_DATA7		0x5a
+			MX7D_PAD_SD3_STROBE__SD3_STROBE         0x1a
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3grp_200mhz {
+		fsl,pins = <
+			MX7D_PAD_SD3_CMD__SD3_CMD		0x5b
+			MX7D_PAD_SD3_CLK__SD3_CLK		0x1b
+			MX7D_PAD_SD3_DATA0__SD3_DATA0		0x5b
+			MX7D_PAD_SD3_DATA1__SD3_DATA1		0x5b
+			MX7D_PAD_SD3_DATA2__SD3_DATA2		0x5b
+			MX7D_PAD_SD3_DATA3__SD3_DATA3		0x5b
+			MX7D_PAD_SD3_DATA4__SD3_DATA4		0x5b
+			MX7D_PAD_SD3_DATA5__SD3_DATA5		0x5b
+			MX7D_PAD_SD3_DATA6__SD3_DATA6		0x5b
+			MX7D_PAD_SD3_DATA7__SD3_DATA7		0x5b
+			MX7D_PAD_SD3_STROBE__SD3_STROBE         0x1b
+		>;
+	};
+
 	pinctrl_sai1: sai1-grp {
 		fsl,pins = <
 			MX7D_PAD_ENET1_RX_CLK__SAI1_TX_BCLK     0x1f
diff --git a/arch/arm/boot/dts/imx7d-colibri-emmc-eval-v3.dts b/arch/arm/boot/dts/imx7d-colibri-emmc-eval-v3.dts
new file mode 100644
index 000000000000..8ee73c870b12
--- /dev/null
+++ b/arch/arm/boot/dts/imx7d-colibri-emmc-eval-v3.dts
@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright 2017 Toradex AG
+ */
+
+/dts-v1/;
+#include "imx7d-colibri-emmc.dtsi"
+#include "imx7-colibri-eval-v3.dtsi"
+
+/ {
+	model = "Toradex Colibri iMX7D 1GB (eMMC) on Colibri Evaluation Board V3";
+	compatible = "toradex,colibri-imx7d-emmc-eval-v3",
+		     "toradex,colibri-imx7d-emmc", "fsl,imx7d";
+};
+
+&usbotg2 {
+	vbus-supply = <&reg_usbh_vbus>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx7d-colibri-emmc.dtsi b/arch/arm/boot/dts/imx7d-colibri-emmc.dtsi
new file mode 100644
index 000000000000..9b63b9c89e4b
--- /dev/null
+++ b/arch/arm/boot/dts/imx7d-colibri-emmc.dtsi
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright 2017 Toradex AG
+ */
+
+#include "imx7d.dtsi"
+#include "imx7-colibri.dtsi"
+
+/ {
+	memory {
+		reg = <0x80000000 0x40000000>;
+	};
+};
+
+&usbotg2 {
+	dr_mode = "host";
+};
+
+&usdhc3 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/imx7d-colibri-eval-v3.dts b/arch/arm/boot/dts/imx7d-colibri-eval-v3.dts
index a608a14d8c85..136e11ab4893 100644
--- a/arch/arm/boot/dts/imx7d-colibri-eval-v3.dts
+++ b/arch/arm/boot/dts/imx7d-colibri-eval-v3.dts
@@ -48,20 +48,9 @@
 	model = "Toradex Colibri iMX7D on Colibri Evaluation Board V3";
 	compatible = "toradex,colibri-imx7d-eval-v3", "toradex,colibri-imx7d",
 		     "fsl,imx7d";
-
-	reg_usb_otg2_vbus: regulator-usb-otg2-vbus {
-		compatible = "regulator-fixed";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_usbotg2_reg>;
-		regulator-name = "VCC_USB[1-4]";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		gpio = <&gpio4 7 GPIO_ACTIVE_LOW>;
-		vin-supply = <&reg_5v0>;
-	};
 };
 
 &usbotg2 {
-	vbus-supply = <&reg_usb_otg2_vbus>;
+	vbus-supply = <&reg_usbh_vbus>;
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/imx7d-colibri.dtsi b/arch/arm/boot/dts/imx7d-colibri.dtsi
index 3c2cb502b388..6f2bb70c1fbd 100644
--- a/arch/arm/boot/dts/imx7d-colibri.dtsi
+++ b/arch/arm/boot/dts/imx7d-colibri.dtsi
@@ -49,6 +49,10 @@
 	};
 };
 
+&gpmi {
+	status = "okay";
+};
+
 &usbotg2 {
 	dr_mode = "host";
 };
diff --git a/arch/arm/boot/dts/imx7d-pico-pi.dts b/arch/arm/boot/dts/imx7d-pico-pi.dts
new file mode 100644
index 000000000000..ee02d931cf49
--- /dev/null
+++ b/arch/arm/boot/dts/imx7d-pico-pi.dts
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2017 NXP
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "imx7d-pico.dtsi"
+
+/ {
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "imx7-sgtl5000";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&dailink_master>;
+		simple-audio-card,frame-master = <&dailink_master>;
+		simple-audio-card,cpu {
+			sound-dai = <&sai1>;
+		};
+
+		dailink_master: simple-audio-card,codec {
+			sound-dai = <&codec>;
+			clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
+		};
+	};
+};
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet1>;
+	assigned-clocks = <&clks IMX7D_ENET1_TIME_ROOT_SRC>,
+			  <&clks IMX7D_ENET1_TIME_ROOT_CLK>;
+	assigned-clock-parents = <&clks IMX7D_PLL_ENET_MAIN_100M_CLK>;
+	assigned-clock-rates = <0>, <100000000>;
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy0>;
+	fsl,magic-packet;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			status = "okay";
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+
+	codec: sgtl5000@a {
+		#sound-dai-cells = <0>;
+		reg = <0x0a>;
+		compatible = "fsl,sgtl5000";
+		clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
+		VDDA-supply = <&reg_2p5v>;
+		VDDIO-supply = <&reg_vref_1v8>;
+	};
+};
+
+
+&sai1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai1>;
+	assigned-clocks = <&clks IMX7D_SAI1_ROOT_SRC>,
+			  <&clks IMX7D_SAI1_ROOT_CLK>;
+	assigned-clock-parents = <&clks IMX7D_PLL_AUDIO_POST_DIV>;
+	assigned-clock-rates = <0>, <24576000>;
+	status = "okay";
+};
+
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	assigned-clocks = <&clks IMX7D_UART5_ROOT_SRC>;
+	assigned-clock-parents = <&clks IMX7D_PLL_SYS_MAIN_240M_CLK>;
+	status = "okay";
+};
+
+&usbotg1 {
+	vbus-supply = <&reg_usb_otg1_vbus>;
+	status = "okay";
+};
+
+&usbotg2 {
+	vbus-supply = <&reg_usb_otg2_vbus>;
+	dr_mode = "host";
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_enet1: enet1grp {
+		fsl,pins = <
+			MX7D_PAD_SD2_CD_B__ENET1_MDIO			0x3
+			MX7D_PAD_SD2_WP__ENET1_MDC			0x3
+			MX7D_PAD_ENET1_RGMII_TXC__ENET1_RGMII_TXC	0x1
+			MX7D_PAD_ENET1_RGMII_TD0__ENET1_RGMII_TD0	0x1
+			MX7D_PAD_ENET1_RGMII_TD1__ENET1_RGMII_TD1	0x1
+			MX7D_PAD_ENET1_RGMII_TD2__ENET1_RGMII_TD2	0x1
+			MX7D_PAD_ENET1_RGMII_TD3__ENET1_RGMII_TD3	0x1
+			MX7D_PAD_ENET1_RGMII_TX_CTL__ENET1_RGMII_TX_CTL	0x1
+			MX7D_PAD_ENET1_RGMII_RXC__ENET1_RGMII_RXC	0x1
+			MX7D_PAD_ENET1_RGMII_RD0__ENET1_RGMII_RD0	0x1
+			MX7D_PAD_ENET1_RGMII_RD1__ENET1_RGMII_RD1	0x1
+			MX7D_PAD_ENET1_RGMII_RD2__ENET1_RGMII_RD2	0x1
+			MX7D_PAD_ENET1_RGMII_RD3__ENET1_RGMII_RD3	0x1
+			MX7D_PAD_ENET1_RGMII_RX_CTL__ENET1_RGMII_RX_CTL	0x1
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX7D_PAD_UART1_TX_DATA__I2C1_SDA	0x4000007f
+			MX7D_PAD_UART1_RX_DATA__I2C1_SCL	0x4000007f
+		>;
+	};
+
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <
+			MX7D_PAD_ENET1_RX_CLK__SAI1_TX_BCLK	0x1f
+			MX7D_PAD_ENET1_CRS__SAI1_TX_SYNC	0x1f
+			MX7D_PAD_ENET1_COL__SAI1_TX_DATA0	0x30
+			MX7D_PAD_ENET1_TX_CLK__SAI1_RX_DATA0	0x1f
+		>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			MX7D_PAD_I2C4_SDA__UART5_DCE_TX		0x79
+			MX7D_PAD_I2C4_SCL__UART5_DCE_RX		0x79
+		>;
+	};
+
+	pinctrl_usbotg1_pwr: usbotg_pwr {
+		fsl,pins = <
+			MX7D_PAD_UART3_TX_DATA__GPIO4_IO5	0x14
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/imx7d-pico.dts b/arch/arm/boot/dts/imx7d-pico.dtsi
similarity index 71%
rename from arch/arm/boot/dts/imx7d-pico.dts
rename to arch/arm/boot/dts/imx7d-pico.dtsi
index 508328b2a6bf..e307462a48ec 100644
--- a/arch/arm/boot/dts/imx7d-pico.dts
+++ b/arch/arm/boot/dts/imx7d-pico.dtsi
@@ -100,62 +100,6 @@
 		regulator-min-microvolt = <1800000>;
 		regulator-max-microvolt = <1800000>;
 	};
-
-	sound {
-		compatible = "simple-audio-card";
-		simple-audio-card,name = "imx7-sgtl5000";
-		simple-audio-card,format = "i2s";
-		simple-audio-card,bitclock-master = <&dailink_master>;
-		simple-audio-card,frame-master = <&dailink_master>;
-		simple-audio-card,cpu {
-			sound-dai = <&sai1>;
-		};
-
-		dailink_master: simple-audio-card,codec {
-			sound-dai = <&codec>;
-			clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
-		};
-	};
-};
-
-&fec1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_enet1>;
-	assigned-clocks = <&clks IMX7D_ENET1_TIME_ROOT_SRC>,
-			  <&clks IMX7D_ENET1_TIME_ROOT_CLK>;
-	assigned-clock-parents = <&clks IMX7D_PLL_ENET_MAIN_100M_CLK>;
-	assigned-clock-rates = <0>, <100000000>;
-	phy-mode = "rgmii";
-	phy-handle = <&ethphy0>;
-	fsl,magic-packet;
-	status = "okay";
-
-	mdio {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		ethphy0: ethernet-phy@1 {
-			compatible = "ethernet-phy-ieee802.3-c22";
-			reg = <1>;
-			status = "okay";
-		};
-	};
-};
-
-&i2c1 {
-	clock-frequency = <100000>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_i2c1>;
-	status = "okay";
-
-	codec: sgtl5000@a {
-		#sound-dai-cells = <0>;
-		reg = <0x0a>;
-		compatible = "fsl,sgtl5000";
-		clocks = <&clks IMX7D_AUDIO_MCLK_ROOT_CLK>;
-		VDDA-supply = <&reg_2p5v>;
-		VDDIO-supply = <&reg_vref_1v8>;
-	};
 };
 
 &i2c4 {
@@ -253,35 +197,6 @@
 	};
 };
 
-&sai1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_sai1>;
-	assigned-clocks = <&clks IMX7D_SAI1_ROOT_SRC>,
-			  <&clks IMX7D_SAI1_ROOT_CLK>;
-	assigned-clock-parents = <&clks IMX7D_PLL_AUDIO_POST_DIV>;
-	assigned-clock-rates = <0>, <24576000>;
-	status = "okay";
-};
-
-&uart5 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart5>;
-	assigned-clocks = <&clks IMX7D_UART5_ROOT_SRC>;
-	assigned-clock-parents = <&clks IMX7D_PLL_SYS_MAIN_240M_CLK>;
-	status = "okay";
-};
-
-&usbotg1 {
-	vbus-supply = <&reg_usb_otg1_vbus>;
-	status = "okay";
-};
-
-&usbotg2 {
-	vbus-supply = <&reg_usb_otg2_vbus>;
-	dr_mode = "host";
-	status = "okay";
-};
-
 &usdhc2 { /* Wifi SDIO */
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_usdhc2>;
@@ -315,32 +230,6 @@
 };
 
 &iomuxc {
-	pinctrl_enet1: enet1grp {
-		fsl,pins = <
-			MX7D_PAD_SD2_CD_B__ENET1_MDIO			0x3
-			MX7D_PAD_SD2_WP__ENET1_MDC			0x3
-			MX7D_PAD_ENET1_RGMII_TXC__ENET1_RGMII_TXC	0x1
-			MX7D_PAD_ENET1_RGMII_TD0__ENET1_RGMII_TD0	0x1
-			MX7D_PAD_ENET1_RGMII_TD1__ENET1_RGMII_TD1	0x1
-			MX7D_PAD_ENET1_RGMII_TD2__ENET1_RGMII_TD2	0x1
-			MX7D_PAD_ENET1_RGMII_TD3__ENET1_RGMII_TD3	0x1
-			MX7D_PAD_ENET1_RGMII_TX_CTL__ENET1_RGMII_TX_CTL	0x1
-			MX7D_PAD_ENET1_RGMII_RXC__ENET1_RGMII_RXC	0x1
-			MX7D_PAD_ENET1_RGMII_RD0__ENET1_RGMII_RD0	0x1
-			MX7D_PAD_ENET1_RGMII_RD1__ENET1_RGMII_RD1	0x1
-			MX7D_PAD_ENET1_RGMII_RD2__ENET1_RGMII_RD2	0x1
-			MX7D_PAD_ENET1_RGMII_RD3__ENET1_RGMII_RD3	0x1
-			MX7D_PAD_ENET1_RGMII_RX_CTL__ENET1_RGMII_RX_CTL	0x1
-		>;
-	};
-
-	pinctrl_i2c1: i2c1grp {
-		fsl,pins = <
-			MX7D_PAD_UART1_TX_DATA__I2C1_SDA	0x4000007f
-			MX7D_PAD_UART1_RX_DATA__I2C1_SCL	0x4000007f
-		>;
-	};
-
 	pinctrl_i2c4: i2c4grp {
 		fsl,pins = <
 			MX7D_PAD_SAI1_RX_BCLK__I2C4_SDA		0x4000007f
@@ -354,28 +243,6 @@
 		>;
 	};
 
-	pinctrl_sai1: sai1grp {
-		fsl,pins = <
-			MX7D_PAD_ENET1_RX_CLK__SAI1_TX_BCLK	0x1f
-			MX7D_PAD_ENET1_CRS__SAI1_TX_SYNC	0x1f
-			MX7D_PAD_ENET1_COL__SAI1_TX_DATA0	0x30
-			MX7D_PAD_ENET1_TX_CLK__SAI1_RX_DATA0	0x1f
-		>;
-	};
-
-	pinctrl_uart5: uart5grp {
-		fsl,pins = <
-			MX7D_PAD_I2C4_SDA__UART5_DCE_TX		0x79
-			MX7D_PAD_I2C4_SCL__UART5_DCE_RX		0x79
-		>;
-	};
-
-	pinctrl_usbotg1_pwr: usbotg_pwr {
-		fsl,pins = <
-			MX7D_PAD_UART3_TX_DATA__GPIO4_IO5	0x14
-		>;
-	};
-
 	pinctrl_usdhc2: usdhc2grp {
 		fsl,pins = <
 			MX7D_PAD_SD2_CMD__SD2_CMD		0x59
diff --git a/arch/arm/boot/dts/imx7d-sdb.dts b/arch/arm/boot/dts/imx7d-sdb.dts
index a7a5dc7b2700..e7d2db839d70 100644
--- a/arch/arm/boot/dts/imx7d-sdb.dts
+++ b/arch/arm/boot/dts/imx7d-sdb.dts
@@ -82,7 +82,7 @@
 		enable-active-high;
 	};
 
-	reg_usb_otg2_vbus: regulator-usb-otg1-vbus {
+	reg_usb_otg2_vbus: regulator-usb-otg2-vbus {
 		compatible = "regulator-fixed";
 		regulator-name = "usb_otg2_vbus";
 		regulator-min-microvolt = <5000000>;
diff --git a/arch/arm/boot/dts/imx7d.dtsi b/arch/arm/boot/dts/imx7d.dtsi
index 4d308d17f040..200714e3feea 100644
--- a/arch/arm/boot/dts/imx7d.dtsi
+++ b/arch/arm/boot/dts/imx7d.dtsi
@@ -63,6 +63,13 @@
 		};
 	};
 
+	usbphynop2: usbphynop2 {
+		compatible = "usb-nop-xceiv";
+		clocks = <&clks IMX7D_USB_PHY2_CLK>;
+		clock-names = "main_clk";
+		#phy-cells = <0>;
+	};
+
 	soc {
 		etm@3007d000 {
 			compatible = "arm,coresight-etm3x", "arm,primecell";
@@ -105,18 +112,14 @@
 		reg = <0x30b20200 0x200>;
 	};
 
-	usbphynop2: usbphynop2 {
-		compatible = "usb-nop-xceiv";
-		clocks = <&clks IMX7D_USB_PHY2_CLK>;
-		clock-names = "main_clk";
-	};
-
 	fec2: ethernet@30bf0000 {
 		compatible = "fsl,imx7d-fec", "fsl,imx6sx-fec";
 		reg = <0x30bf0000 0x10000>;
-		interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+		interrupt-names = "int0", "int1", "int2", "pps";
+		interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+			<GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
 			<GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
-			<GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
+			<GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&clks IMX7D_ENET_AXI_ROOT_CLK>,
 			<&clks IMX7D_ENET_AXI_ROOT_CLK>,
 			<&clks IMX7D_ENET2_TIME_ROOT_CLK>,
@@ -129,7 +132,7 @@
 		status = "disabled";
 	};
 
-	pcie: pcie@0x33800000 {
+	pcie: pcie@33800000 {
 		compatible = "fsl,imx7d-pcie", "snps,dw-pcie";
 		reg = <0x33800000 0x4000>,
 		      <0x4ff00000 0x80000>;
@@ -137,6 +140,7 @@
 		#address-cells = <3>;
 		#size-cells = <2>;
 		device_type = "pci";
+		bus-range = <0x00 0xff>;
 		ranges = <0x81000000 0 0          0x4ff80000 0 0x00010000   /* downstream I/O */
 			  0x82000000 0 0x40000000 0x40000000 0 0x0ff00000>; /* non-prefetchable memory */
 		num-lanes = <1>;
diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi
index 82ad26e766eb..9aa2bb998552 100644
--- a/arch/arm/boot/dts/imx7s.dtsi
+++ b/arch/arm/boot/dts/imx7s.dtsi
@@ -116,6 +116,66 @@
 		clock-output-names = "osc";
 	};
 
+	usbphynop1: usbphynop1 {
+		compatible = "usb-nop-xceiv";
+		clocks = <&clks IMX7D_USB_PHY1_CLK>;
+		clock-names = "main_clk";
+		#phy-cells = <0>;
+	};
+
+	usbphynop3: usbphynop3 {
+		compatible = "usb-nop-xceiv";
+		clocks = <&clks IMX7D_USB_HSIC_ROOT_CLK>;
+		clock-names = "main_clk";
+		#phy-cells = <0>;
+	};
+
+
+	replicator {
+		/*
+		 * non-configurable replicators don't show up on the
+		 * AMBA bus.  As such no need to add "arm,primecell"
+		 */
+		compatible = "arm,coresight-replicator";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+				/* replicator output ports */
+			port@0 {
+				reg = <0>;
+				replicator_out_port0: endpoint {
+					remote-endpoint = <&tpiu_in_port>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				replicator_out_port1: endpoint {
+					remote-endpoint = <&etr_in_port>;
+				};
+			};
+
+			/* replicator input port */
+			port@2 {
+				reg = <0>;
+				replicator_in_port0: endpoint {
+					slave-mode;
+					remote-endpoint = <&etf_out_port>;
+				};
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupt-parent = <&intc>;
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -260,43 +320,6 @@
 			};
 		};
 
-		replicator {
-			/*
-			 * non-configurable replicators don't show up on the
-			 * AMBA bus.  As such no need to add "arm,primecell"
-			 */
-			compatible = "arm,coresight-replicator";
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				/* replicator output ports */
-				port@0 {
-					reg = <0>;
-					replicator_out_port0: endpoint {
-						remote-endpoint = <&tpiu_in_port>;
-					};
-				};
-
-				port@1 {
-					reg = <1>;
-					replicator_out_port1: endpoint {
-						remote-endpoint = <&etr_in_port>;
-					};
-				};
-
-				/* replicator input port */
-				port@2 {
-					reg = <0>;
-					replicator_in_port0: endpoint {
-						slave-mode;
-						remote-endpoint = <&etf_out_port>;
-					};
-				};
-			};
-		};
-
 		intc: interrupt-controller@31001000 {
 			compatible = "arm,cortex-a7-gic";
 			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
@@ -309,15 +332,6 @@
 			      <0x31006000 0x2000>;
 		};
 
-		timer {
-			compatible = "arm,armv7-timer";
-			interrupt-parent = <&intc>;
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
-		};
-
 		aips1: aips-bus@30000000 {
 			compatible = "fsl,aips-bus", "simple-bus";
 			#address-cells = <1>;
@@ -508,8 +522,11 @@
 				reg = <0x30360000 0x10000>;
 				interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <0>;
 
-				reg_1p0d: regulator-vdd1p0d {
+				reg_1p0d: regulator-vdd1p0d@30360210 {
+					reg = <0x30360210>;
 					compatible = "fsl,anatop-regulator";
 					regulator-name = "vdd1p0d";
 					regulator-min-microvolt = <800000>;
@@ -583,9 +600,9 @@
 					#address-cells = <1>;
 					#size-cells = <0>;
 
-					pgc_pcie_phy: pgc-power-domain@IMX7_POWER_DOMAIN_PCIE_PHY {
+					pgc_pcie_phy: pgc-power-domain@1 {
 						#power-domain-cells = <0>;
-						reg = <IMX7_POWER_DOMAIN_PCIE_PHY>;
+						reg = <1>;
 						power-supply = <&reg_1p0d>;
 					};
 				};
@@ -945,18 +962,6 @@
 				reg = <0x30b30200 0x200>;
 			};
 
-			usbphynop1: usbphynop1 {
-				compatible = "usb-nop-xceiv";
-				clocks = <&clks IMX7D_USB_PHY1_CLK>;
-				clock-names = "main_clk";
-			};
-
-			usbphynop3: usbphynop3 {
-				compatible = "usb-nop-xceiv";
-				clocks = <&clks IMX7D_USB_HSIC_ROOT_CLK>;
-				clock-names = "main_clk";
-			};
-
 			usdhc1: usdhc@30b40000 {
 				compatible = "fsl,imx7d-usdhc", "fsl,imx6sl-usdhc";
 				reg = <0x30b40000 0x10000>;
@@ -1007,9 +1012,11 @@
 			fec1: ethernet@30be0000 {
 				compatible = "fsl,imx7d-fec", "fsl,imx6sx-fec";
 				reg = <0x30be0000 0x10000>;
-				interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
+				interrupt-names = "int0", "int1", "int2", "pps";
+				interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
+					<GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>,
-					<GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+					<GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX7D_ENET_AXI_ROOT_CLK>,
 					<&clks IMX7D_ENET_AXI_ROOT_CLK>,
 					<&clks IMX7D_ENET1_TIME_ROOT_CLK>,
diff --git a/arch/arm/boot/dts/iwg20d-q7-common.dtsi b/arch/arm/boot/dts/iwg20d-q7-common.dtsi
index efd8af9242d1..66954aaf2c47 100644
--- a/arch/arm/boot/dts/iwg20d-q7-common.dtsi
+++ b/arch/arm/boot/dts/iwg20d-q7-common.dtsi
@@ -8,9 +8,33 @@
  * kind, whether express or implied.
  */
 
+/*
+ * SSI-SGTL5000
+ *
+ * This command is required when Playback/Capture
+ *
+ *      amixer set "DVC Out" 100%
+ *      amixer set "DVC In" 100%
+ *
+ * You can use Mute
+ *
+ *      amixer set "DVC Out Mute" on
+ *      amixer set "DVC In Mute" on
+ *
+ * You can use Volume Ramp
+ *
+ *      amixer set "DVC Out Ramp Up Rate"   "0.125 dB/64 steps"
+ *      amixer set "DVC Out Ramp Down Rate" "0.125 dB/512 steps"
+ *      amixer set "DVC Out Ramp" on
+ *      aplay xxx.wav &
+ *      amixer set "DVC Out"  80%  // Volume Down
+ *      amixer set "DVC Out" 100%  // Volume Up
+ */
+
 / {
 	aliases {
 		serial0 = &scif0;
+		serial3 = &scifb1;
 		ethernet0 = &avb;
 	};
 
@@ -19,6 +43,36 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	audio_clock: audio_clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
+	};
+
+	reg_1p5v: 1p5v {
+		compatible = "regulator-fixed";
+		regulator-name = "1P5V";
+		regulator-min-microvolt = <1500000>;
+		regulator-max-microvolt = <1500000>;
+		regulator-always-on;
+	};
+
+	rsnd_sgtl5000: sound {
+		compatible = "simple-audio-card";
+
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&sndcodec>;
+		simple-audio-card,frame-master = <&sndcodec>;
+
+		sndcpu: simple-audio-card,cpu {
+			sound-dai = <&rcar_sound>;
+		};
+
+		sndcodec: simple-audio-card,codec {
+			sound-dai = <&sgtl5000>;
+		};
+	};
+
 	vcc_sdhi1: regulator-vcc-sdhi1 {
 		compatible = "regulator-fixed";
 
@@ -58,6 +112,13 @@
 	};
 };
 
+&can0 {
+	pinctrl-0 = <&can0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
 &hsusb {
 	status = "okay";
 	pinctrl-0 = <&usb0_pins>;
@@ -75,6 +136,16 @@
 		compatible = "ti,bq32000";
 		reg = <0x68>;
 	};
+
+	sgtl5000: codec@a {
+		compatible = "fsl,sgtl5000";
+		#sound-dai-cells = <0>;
+		reg = <0x0a>;
+		clocks = <&audio_clock>;
+		VDDA-supply = <&reg_3p3v>;
+		VDDIO-supply = <&reg_3p3v>;
+		VDDD-supply = <&reg_1p5v>;
+	};
 };
 
 &pci0 {
@@ -88,7 +159,20 @@
 	pinctrl-names = "default";
 };
 
+&pcie_bus_clk {
+	clock-frequency = <100000000>;
+};
+
+&pciec {
+	status = "okay";
+};
+
 &pfc {
+	can0_pins: can0 {
+		groups = "can0_data_d";
+		function = "can0";
+	};
+
 	avb_pins: avb {
 		groups = "avb_mdio", "avb_gmii";
 		function = "avb";
@@ -104,6 +188,11 @@
 		function = "scif0";
 	};
 
+	scifb1_pins: scifb1 {
+		groups = "scifb1_data_d", "scifb1_ctrl";
+		function = "scifb1";
+	};
+
 	sdhi1_pins: sd1 {
 		groups = "sdhi1_data4", "sdhi1_ctrl";
 		function = "sdhi1";
@@ -116,6 +205,11 @@
 		power-source = <1800>;
 	};
 
+	sound_pins: sound {
+		groups = "ssi0129_ctrl", "ssi0_data", "ssi1_data";
+		function = "ssi";
+	};
+
 	usb0_pins: usb0 {
 		groups = "usb0";
 		function = "usb0";
@@ -127,6 +221,22 @@
 	};
 };
 
+&rcar_sound {
+	pinctrl-0 = <&sound_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	/* Single DAI */
+	#sound-dai-cells = <0>;
+
+	rcar_sound,dai {
+		dai0 {
+			playback = <&ssi1 &src3 &dvc1>;
+			capture = <&ssi0 &src2 &dvc0>;
+		};
+	};
+};
+
 &scif0 {
 	pinctrl-0 = <&scif0_pins>;
 	pinctrl-names = "default";
@@ -134,6 +244,14 @@
 	status = "okay";
 };
 
+&scifb1 {
+	pinctrl-0 = <&scifb1_pins>;
+	pinctrl-names = "default";
+
+	uart-has-rtscts;
+	status = "okay";
+};
+
 &sdhi1 {
 	pinctrl-0 = <&sdhi1_pins>;
 	pinctrl-1 = <&sdhi1_pins_uhs>;
@@ -147,6 +265,10 @@
 	status = "okay";
 };
 
+&ssi1 {
+	shared-pin;
+};
+
 &usbphy {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/iwg20d-q7-dbcm-ca.dtsi b/arch/arm/boot/dts/iwg20d-q7-dbcm-ca.dtsi
index 31fab5f183a9..476273b3f994 100644
--- a/arch/arm/boot/dts/iwg20d-q7-dbcm-ca.dtsi
+++ b/arch/arm/boot/dts/iwg20d-q7-dbcm-ca.dtsi
@@ -13,6 +13,44 @@
 		serial1 = &scif1;
 		serial4 = &hscif1;
 	};
+
+	cec_clock: cec-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <12000000>;
+	};
+
+	hdmi-out {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_out: endpoint {
+				remote-endpoint = <&adv7511_out>;
+			};
+		};
+	};
+};
+
+&can1 {
+	pinctrl-0 = <&can1_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&du {
+	pinctrl-0 = <&du_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ports {
+		port@0 {
+			endpoint {
+				remote-endpoint = <&adv7511_in>;
+			};
+		};
+	};
 };
 
 &hscif1 {
@@ -23,7 +61,56 @@
 	status = "okay";
 };
 
+&i2c5 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	hdmi@39 {
+		compatible = "adi,adv7511w";
+		reg = <0x39>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <13 IRQ_TYPE_LEVEL_LOW>;
+		clocks = <&cec_clock>;
+		clock-names = "cec";
+
+		adi,input-depth = <8>;
+		adi,input-colorspace = "rgb";
+		adi,input-clock = "1x";
+		adi,input-style = <1>;
+		adi,input-justification = "evenly";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7511_in: endpoint {
+					remote-endpoint = <&du_out_rgb>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7511_out: endpoint {
+					remote-endpoint = <&hdmi_con_out>;
+				};
+			};
+		};
+	};
+};
+
 &pfc {
+	can1_pins: can1 {
+		groups = "can1_data_d";
+		function = "can1";
+	};
+
+	du_pins: du {
+		groups = "du_rgb888", "du_sync", "du_oddf", "du_clk_out_0";
+		function = "du";
+	};
+
 	hscif1_pins: hscif1 {
 		groups = "hscif1_data_c", "hscif1_ctrl_c";
 		function = "hscif1";
diff --git a/arch/arm/boot/dts/keystone-clocks.dtsi b/arch/arm/boot/dts/keystone-clocks.dtsi
index 0c334b25781e..457515b0736a 100644
--- a/arch/arm/boot/dts/keystone-clocks.dtsi
+++ b/arch/arm/boot/dts/keystone-clocks.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree Source for Keystone 2 clock tree
  *
- * Copyright (C) 2013 Texas Instruments, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 clocks {
@@ -51,7 +48,7 @@ clocks {
 		clock-output-names = "gemtraceclk";
 	};
 
-	chipstmxptclk: chipstmxptclk {
+	chipstmxptclk: chipstmxptclk@2310164 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,pll-divider-clock";
 		clocks = <&mainmuxclk>;
@@ -160,7 +157,7 @@ clocks {
 		clock-output-names = "chipclk1rstiso112";
 	};
 
-	clkmodrst0: clkmodrst0 {
+	clkmodrst0: clkmodrst0@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk16>;
@@ -171,7 +168,7 @@ clocks {
 	};
 
 
-	clkusb: clkusb {
+	clkusb: clkusb@2350008 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk16>;
@@ -181,7 +178,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkaemifspi: clkaemifspi {
+	clkaemifspi: clkaemifspi@235000c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk16>;
@@ -192,7 +189,7 @@ clocks {
 	};
 
 
-	clkdebugsstrc: clkdebugsstrc {
+	clkdebugsstrc: clkdebugsstrc@2350014 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -202,7 +199,7 @@ clocks {
 		domain-id = <1>;
 	};
 
-	clktetbtrc: clktetbtrc {
+	clktetbtrc: clktetbtrc@2350018 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -212,7 +209,7 @@ clocks {
 		domain-id = <1>;
 	};
 
-	clkpa: clkpa {
+	clkpa: clkpa@235001c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&paclk13>;
@@ -222,7 +219,7 @@ clocks {
 		domain-id = <2>;
 	};
 
-	clkcpgmac: clkcpgmac {
+	clkcpgmac: clkcpgmac@2350020 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkpa>;
@@ -232,7 +229,7 @@ clocks {
 		domain-id = <2>;
 	};
 
-	clksa: clksa {
+	clksa: clksa@2350024 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkpa>;
@@ -242,7 +239,7 @@ clocks {
 		domain-id = <2>;
 	};
 
-	clkpcie: clkpcie {
+	clkpcie: clkpcie@2350028 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -252,7 +249,7 @@ clocks {
 		domain-id = <3>;
 	};
 
-	clksr: clksr {
+	clksr: clksr@2350034 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1rstiso112>;
@@ -262,7 +259,7 @@ clocks {
 		domain-id = <6>;
 	};
 
-	clkgem0: clkgem0 {
+	clkgem0: clkgem0@235003c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -272,7 +269,7 @@ clocks {
 		domain-id = <8>;
 	};
 
-	clkddr30: clkddr30 {
+	clkddr30: clkddr30@235005c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -282,7 +279,7 @@ clocks {
 		domain-id = <16>;
 	};
 
-	clkwdtimer0: clkwdtimer0 {
+	clkwdtimer0: clkwdtimer0@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -292,7 +289,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkwdtimer1: clkwdtimer1 {
+	clkwdtimer1: clkwdtimer1@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -302,7 +299,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkwdtimer2: clkwdtimer2 {
+	clkwdtimer2: clkwdtimer2@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -312,7 +309,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkwdtimer3: clkwdtimer3 {
+	clkwdtimer3: clkwdtimer3@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -322,7 +319,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clktimer15: clktimer15 {
+	clktimer15: clktimer15@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -332,7 +329,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkuart0: clkuart0 {
+	clkuart0: clkuart0@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -342,7 +339,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkuart1: clkuart1 {
+	clkuart1: clkuart1@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -352,7 +349,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkaemif: clkaemif {
+	clkaemif: clkaemif@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkaemifspi>;
@@ -362,7 +359,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkusim: clkusim {
+	clkusim: clkusim@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -372,7 +369,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clki2c: clki2c {
+	clki2c: clki2c@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -382,7 +379,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkspi: clkspi {
+	clkspi: clkspi@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkaemifspi>;
@@ -392,7 +389,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkgpio: clkgpio {
+	clkgpio: clkgpio@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -402,7 +399,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkkeymgr: clkkeymgr {
+	clkkeymgr: clkkeymgr@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
diff --git a/arch/arm/boot/dts/keystone-k2e-clocks.dtsi b/arch/arm/boot/dts/keystone-k2e-clocks.dtsi
index d56d68fe7ffc..5e0e7d232161 100644
--- a/arch/arm/boot/dts/keystone-k2e-clocks.dtsi
+++ b/arch/arm/boot/dts/keystone-k2e-clocks.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2014 Texas Instruments, Inc.
- *
  * Keystone 2 Edison SoC specific device tree
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2014-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 clocks {
@@ -35,7 +32,7 @@ clocks {
 		reg-names = "control";
 	};
 
-	clkusb1: clkusb1 {
+	clkusb1: clkusb1@2350004 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk16>;
@@ -45,7 +42,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkhyperlink0: clkhyperlink0 {
+	clkhyperlink0: clkhyperlink02350030 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -55,7 +52,7 @@ clocks {
 		domain-id = <5>;
 	};
 
-	clkpcie1: clkpcie1 {
+	clkpcie1: clkpcie1@235006c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -65,7 +62,7 @@ clocks {
 		domain-id = <18>;
 	};
 
-	clkxge: clkxge {
+	clkxge: clkxge@23500c8 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
diff --git a/arch/arm/boot/dts/keystone-k2e-evm.dts b/arch/arm/boot/dts/keystone-k2e-evm.dts
index f1f32c54e72f..66fec5f5d081 100644
--- a/arch/arm/boot/dts/keystone-k2e-evm.dts
+++ b/arch/arm/boot/dts/keystone-k2e-evm.dts
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2013-2014 Texas Instruments, Inc.
- *
  * Keystone 2 Edison EVM device tree
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 /dts-v1/;
 
@@ -28,8 +25,9 @@
 			status = "okay";
 		};
 	};
+};
 
-	soc {
+&soc0 {
 
 		clocks {
 			refclksys: refclksys {
@@ -53,7 +51,6 @@
 				clock-output-names = "refclk-ddr3a";
 			};
 		};
-	};
 };
 
 &usb_phy {
@@ -82,7 +79,7 @@
 
 &i2c0 {
 	dtt@50 {
-		compatible = "at,24c1024";
+		compatible = "atmel,24c1024";
 		reg = <0x50>;
 	};
 };
diff --git a/arch/arm/boot/dts/keystone-k2e-netcp.dtsi b/arch/arm/boot/dts/keystone-k2e-netcp.dtsi
index ba828cb59587..a17311c602aa 100644
--- a/arch/arm/boot/dts/keystone-k2e-netcp.dtsi
+++ b/arch/arm/boot/dts/keystone-k2e-netcp.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree Source for Keystone 2 Edison Netcp driver
  *
- * Copyright 2015 Texas Instruments, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2015-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 qmss: qmss@2a40000 {
diff --git a/arch/arm/boot/dts/keystone-k2e.dtsi b/arch/arm/boot/dts/keystone-k2e.dtsi
index 6b796b52ff4f..0bcd3f8a9c45 100644
--- a/arch/arm/boot/dts/keystone-k2e.dtsi
+++ b/arch/arm/boot/dts/keystone-k2e.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2013-2014 Texas Instruments, Inc.
- *
  * Keystone 2 Edison soc device tree
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 #include <dt-bindings/reset/ti-syscon.h>
@@ -48,8 +45,9 @@
 	aliases {
 		rproc0 = &dsp0;
 	};
+};
 
-	soc {
+&soc0 {
 		/include/ "keystone-k2e-clocks.dtsi"
 
 		usb: usb@2680000 {
@@ -194,5 +192,4 @@
 			bus_freq	= <2500000>;
 		};
 		/include/ "keystone-k2e-netcp.dtsi"
-	};
 };
diff --git a/arch/arm/boot/dts/keystone-k2g-evm.dts b/arch/arm/boot/dts/keystone-k2g-evm.dts
index 656af194a518..6a4657799b99 100644
--- a/arch/arm/boot/dts/keystone-k2g-evm.dts
+++ b/arch/arm/boot/dts/keystone-k2g-evm.dts
@@ -1,16 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree Source for K2G EVM
  *
- * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+ * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 /dts-v1/;
 
@@ -45,22 +37,6 @@
 		regulator-max-microvolt = <3300000>;
 		regulator-always-on;
 	};
-
-	ecap0_pins: ecap0_pins {
-		pinctrl-single,pins = <
-			K2G_CORE_IOPAD(0x1374) (BUFFER_CLASS_B | MUX_MODE4)	/* pr1_mdio_data.ecap0_in_apwm0_out */
-		>;
-	};
-
-	spi1_pins: pinmux_spi1_pins {
-		pinctrl-single,pins = <
-			K2G_CORE_IOPAD(0x11a4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_scs0.spi1_scs0 */
-			K2G_CORE_IOPAD(0x11ac) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_clk.spi1_clk */
-			K2G_CORE_IOPAD(0x11b0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_miso.spi1_miso */
-			K2G_CORE_IOPAD(0x11b4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_mosi.spi1_mosi */
-		>;
-	};
-
 };
 
 &k2g_pinctrl {
@@ -105,6 +81,39 @@
 		>;
 	};
 
+	ecap0_pins: ecap0_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1374) (BUFFER_CLASS_B | MUX_MODE4)	/* pr1_mdio_data.ecap0_in_apwm0_out */
+		>;
+	};
+
+	spi1_pins: pinmux_spi1_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x11a4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_scs0.spi1_scs0 */
+			K2G_CORE_IOPAD(0x11ac) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_clk.spi1_clk */
+			K2G_CORE_IOPAD(0x11b0) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_miso.spi1_miso */
+			K2G_CORE_IOPAD(0x11b4) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)	/* spi1_mosi.spi1_mosi */
+		>;
+	};
+
+	qspi_pins: pinmux_qspi_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1204) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_clk.qspi_clk */
+			K2G_CORE_IOPAD(0x1208) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_rclk.qspi_rclk */
+			K2G_CORE_IOPAD(0x120c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d0.qspi_d0 */
+			K2G_CORE_IOPAD(0x1210) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d1.qspi_d1 */
+			K2G_CORE_IOPAD(0x1214) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d2.qspi_d2 */
+			K2G_CORE_IOPAD(0x1218) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d3.qspi_d3 */
+			K2G_CORE_IOPAD(0x121c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_csn0.qspi_csn0 */
+		>;
+	};
+
+	uart2_pins: pinmux_uart2_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x11ec) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0)      /* uart2_rxd.uart2_rxd */
+			K2G_CORE_IOPAD(0x11f0) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE0)      /* uart2_txd.uart2_txd */
+		>;
+	};
 };
 
 &uart0 {
@@ -129,6 +138,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc1_pins>;
 	vmmc-supply = <&vcc3v3_dcin_reg>; /* VCC3V3_EMMC is connected to VCC3V3_DCIN */
+	ti,non-removable;
 	status = "okay";
 };
 
@@ -205,3 +215,56 @@
 		};
 	};
 };
+
+&qspi {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_pins>;
+	cdns,rclk-en;
+
+	flash0: m25p80@0 {
+		compatible = "s25fl512s", "jedec,spi-nor";
+		reg = <0>;
+		spi-tx-bus-width = <1>;
+		spi-rx-bus-width = <4>;
+		spi-max-frequency = <96000000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		cdns,read-delay = <5>;
+		cdns,tshsl-ns = <500>;
+		cdns,tsd2d-ns = <500>;
+		cdns,tchsh-ns = <119>;
+		cdns,tslch-ns = <119>;
+
+		partition@0 {
+			label = "QSPI.u-boot-spl-os";
+			reg = <0x00000000 0x00100000>;
+		};
+		partition@1 {
+			label = "QSPI.u-boot-env";
+			reg = <0x00100000 0x00040000>;
+		};
+		partition@2 {
+			label = "QSPI.skern";
+			reg = <0x00140000 0x0040000>;
+		};
+		partition@3 {
+			label = "QSPI.pmmc-firmware";
+			reg = <0x00180000 0x0040000>;
+		};
+		partition@4 {
+			label = "QSPI.kernel";
+			reg = <0x001C0000 0x0800000>;
+		};
+		partition@5 {
+			label = "QSPI.file-system";
+			reg = <0x009C0000 0x3640000>;
+		};
+	};
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart2_pins>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/keystone-k2g-ice.dts b/arch/arm/boot/dts/keystone-k2g-ice.dts
index 78692745e0af..d820ed2474bb 100644
--- a/arch/arm/boot/dts/keystone-k2g-ice.dts
+++ b/arch/arm/boot/dts/keystone-k2g-ice.dts
@@ -1,9 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree Source for K2G Industrial Communication Engine EVM
  *
  * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/
- *
- * SPDX-License-Identifier: GPL-2.0
  */
 /dts-v1/;
 
@@ -30,6 +29,191 @@
 			status = "okay";
 		};
 	};
+
+	vmain: fixedregulator-vmain {
+		compatible = "regulator-fixed";
+		regulator-name = "vmain_fixed";
+		regulator-min-microvolt = <24000000>;
+		regulator-max-microvolt = <24000000>;
+		regulator-always-on;
+	};
+
+	v5_0: fixedregulator-v5_0 {
+		/* TPS54531 */
+		compatible = "regulator-fixed";
+		regulator-name = "v5_0_fixed";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vmain>;
+		regulator-always-on;
+	};
+
+	vdd_3v3: fixedregulator-vdd_3v3 {
+		/* TLV62084 */
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_3v3_fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&v5_0>;
+		regulator-always-on;
+	};
+
+	vdd_1v8: fixedregulator-vdd_1v8 {
+		/* TLV62084 */
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_1v8_fixed";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&v5_0>;
+		regulator-always-on;
+	};
+
+	vdds_ddr: fixedregulator-vdds_ddr {
+		/* TLV62080 */
+		compatible = "regulator-fixed";
+		regulator-name = "vdds_ddr_fixed";
+		regulator-min-microvolt = <1350000>;
+		regulator-max-microvolt = <1350000>;
+		vin-supply = <&v5_0>;
+		regulator-always-on;
+	};
+
+	vref_ddr: fixedregulator-vref_ddr {
+		/* LP2996A */
+		compatible = "regulator-fixed";
+		regulator-name = "vref_ddr_fixed";
+		regulator-min-microvolt = <675000>;
+		regulator-max-microvolt = <675000>;
+		vin-supply = <&vdd_3v3>;
+		regulator-always-on;
+	};
+
+	vtt_ddr: fixedregulator-vtt_ddr {
+		/* LP2996A */
+		compatible = "regulator-fixed";
+		regulator-name = "vtt_ddr_fixed";
+		regulator-min-microvolt = <675000>;
+		regulator-max-microvolt = <675000>;
+		vin-supply = <&vdd_3v3>;
+		regulator-always-on;
+	};
+
+	vdd_0v9: fixedregulator-vdd_0v9 {
+		/* TPS62180 */
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_0v9_fixed";
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		vin-supply = <&v5_0>;
+		regulator-always-on;
+	};
+
+	vddb: fixedregulator-vddb {
+		/* TPS22945 */
+		compatible = "regulator-fixed";
+		regulator-name = "vddb_fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&gpio1 53 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	gpio-decoder {
+		compatible = "gpio-decoder";
+		gpios = <&pca9536 3 GPIO_ACTIVE_HIGH>,
+			<&pca9536 2 GPIO_ACTIVE_HIGH>,
+			<&pca9536 1 GPIO_ACTIVE_HIGH>,
+			<&pca9536 0 GPIO_ACTIVE_HIGH>;
+		linux,axis = <0>; /* ABS_X */
+		decoder-max-value = <9>;
+	};
+
+	leds1 {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&user_leds>;
+
+		led0 {
+			label = "status0:red:cpu0";
+			gpios = <&gpio0 11 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "cpu0";
+		};
+
+		led1 {
+			label = "status0:green:usr";
+			gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led2 {
+			label = "status0:yellow:usr";
+			gpios = <&gpio0 13 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led3 {
+			label = "status1:red:mmc0";
+			gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "mmc0";
+		};
+
+		led4 {
+			label = "status1:green:usr";
+			gpios = <&gpio0 15 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led5 {
+			label = "status1:yellow:usr";
+			gpios = <&gpio0 16 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led6 {
+			label = "status2:red:usr";
+			gpios = <&gpio0 44 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led7 {
+			label = "status2:green:usr";
+			gpios = <&gpio0 43 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led8 {
+			label = "status2:yellow:usr";
+			gpios = <&gpio0 42 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led9 {
+			label = "status3:red:usr";
+			gpios = <&gpio0 41 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led10 {
+			label = "status3:green:usr";
+			gpios = <&gpio0 101 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led11 {
+			label = "status3:yellow:usr";
+			gpios = <&gpio0 102 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led12 {
+			label = "status4:green:heartbeat";
+			gpios = <&gpio0 19 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
 };
 
 &k2g_pinctrl {
@@ -39,6 +223,64 @@
 			K2G_CORE_IOPAD(0x11d0) (BUFFER_CLASS_B | PIN_PULLDOWN | MUX_MODE0)	/* uart0_txd.uart0_txd */
 		>;
 	};
+
+	qspi_pins: pinmux_qspi_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1204) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_clk.qspi_clk */
+			K2G_CORE_IOPAD(0x1208) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_rclk.qspi_rclk */
+			K2G_CORE_IOPAD(0x120c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d0.qspi_d0 */
+			K2G_CORE_IOPAD(0x1210) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d1.qspi_d1 */
+			K2G_CORE_IOPAD(0x1214) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d2.qspi_d2 */
+			K2G_CORE_IOPAD(0x1218) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_d3.qspi_d3 */
+			K2G_CORE_IOPAD(0x121c) (BUFFER_CLASS_B | PULL_DISABLE | MUX_MODE0) /* qspi_csn0.qspi_csn0 */
+		>;
+	};
+
+	mmc1_pins: pinmux_mmc1_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x10FC) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_dat3.mmc1_dat3 */
+			K2G_CORE_IOPAD(0x1100) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_dat2.mmc1_dat2 */
+			K2G_CORE_IOPAD(0x1104) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_dat1.mmc1_dat1 */
+			K2G_CORE_IOPAD(0x1108) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_dat0.mmc1_dat0 */
+			K2G_CORE_IOPAD(0x110C) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_clk.mmc1_clk */
+			K2G_CORE_IOPAD(0x1110) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_cmd.mmc1_cmd */
+			K2G_CORE_IOPAD(0x1114) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* mmc1_sdcd.gpio0_69 */
+			K2G_CORE_IOPAD(0x1118) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_sdwp.mmc1_sdwp */
+			K2G_CORE_IOPAD(0x111C) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* mmc1_pow.mmc1_pow */
+		>;
+	};
+
+	i2c0_pins: pinmux_i2c0_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x137c) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* i2c0_scl.i2c0_scl */
+			K2G_CORE_IOPAD(0x1380) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* i2c0_sda.i2c0_sda */
+		>;
+	};
+
+	i2c1_pins: pinmux_i2c1_pins {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x1384) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* i2c1_scl.i2c1_scl */
+			K2G_CORE_IOPAD(0x1388) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE0)	/* i2c1_sda.i2c1_sda */
+		>;
+	};
+
+	user_leds: pinmux_user_leds {
+		pinctrl-single,pins = <
+			K2G_CORE_IOPAD(0x102c) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_ad11.gpio0_11 */
+			K2G_CORE_IOPAD(0x1030) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_ad12.gpio0_12 */
+			K2G_CORE_IOPAD(0x1034) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_ad13.gpio0_13 */
+			K2G_CORE_IOPAD(0x1038) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_ad14.gpio0_14 */
+			K2G_CORE_IOPAD(0x103c) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_ad15.gpio0_15 */
+			K2G_CORE_IOPAD(0x1040) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_clk.gpio0_16 */
+			K2G_CORE_IOPAD(0x104c) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* gpmc_wen.gpio0_19 */
+			K2G_CORE_IOPAD(0x10b0) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* dss_data9.gpio0_44 */
+			K2G_CORE_IOPAD(0x10ac) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* dss_data10.gpio0_43 */
+			K2G_CORE_IOPAD(0x10a8) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* dss_data11.gpio0_42 */
+			K2G_CORE_IOPAD(0x10a4) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* dss_data12.gpio0_41 */
+			K2G_CORE_IOPAD(0x11b8) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* spi2_scsn0.gpio0_101 */
+			K2G_CORE_IOPAD(0x11bc) (BUFFER_CLASS_B | PIN_PULLUP | MUX_MODE3)	/* spi2_scsn1.gpio0_102 */
+		>;
+	};
 };
 
 &uart0 {
@@ -51,3 +293,96 @@
 	memory-region = <&dsp_common_memory>;
 	status = "okay";
 };
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_pins>;
+	cdns,rclk-en;
+	status = "okay";
+
+	flash0: m25p80@0 {
+		compatible = "s25fl256s1", "jedec,spi-nor";
+		reg = <0>;
+		spi-tx-bus-width = <1>;
+		spi-rx-bus-width = <4>;
+		spi-max-frequency = <96000000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		cdns,read-delay = <5>;
+		cdns,tshsl-ns = <500>;
+		cdns,tsd2d-ns = <500>;
+		cdns,tchsh-ns = <119>;
+		cdns,tslch-ns = <119>;
+
+		partition@0 {
+			label = "QSPI.u-boot";
+			reg = <0x00000000 0x00100000>;
+		};
+		partition@1 {
+			label = "QSPI.u-boot-env";
+			reg = <0x00100000 0x00040000>;
+		};
+		partition@2 {
+			label = "QSPI.skern";
+			reg = <0x00140000 0x0040000>;
+		};
+		partition@3 {
+			label = "QSPI.pmmc-firmware";
+			reg = <0x00180000 0x0040000>;
+		};
+		partition@4 {
+			label = "QSPI.kernel";
+			reg = <0x001c0000 0x0800000>;
+		};
+		partition@5 {
+			label = "QSPI.u-boot-spl-os";
+			reg = <0x009c0000 0x0040000>;
+		};
+		partition@6 {
+			label = "QSPI.file-system";
+			reg = <0x00a00000 0x1600000>;
+		};
+	};
+};
+
+&gpio0 {
+	status = "okay";
+};
+
+&gpio1 {
+	status = "okay";
+};
+
+&mmc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>;
+	vmmc-supply = <&vdd_3v3>;
+	cd-gpios = <&gpio0 69 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins>;
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c256";
+		reg = <0x50>;
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+	status = "okay";
+	clock-frequency = <400000>;
+
+	pca9536: gpio@41 {
+		compatible = "ti,pca9536";
+		reg = <0x41>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		vcc-supply = <&vdd_3v3>;
+	};
+};
diff --git a/arch/arm/boot/dts/keystone-k2g.dtsi b/arch/arm/boot/dts/keystone-k2g.dtsi
index 8f313ff406b9..fd061718dc0a 100644
--- a/arch/arm/boot/dts/keystone-k2g.dtsi
+++ b/arch/arm/boot/dts/keystone-k2g.dtsi
@@ -1,16 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree Source for K2G SOC
  *
- * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+ * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
@@ -28,6 +20,8 @@
 
 	aliases {
 		serial0 = &uart0;
+		serial1 = &uart1;
+		serial2 = &uart2;
 		i2c0 = &i2c0;
 		i2c1 = &i2c1;
 		i2c2 = &i2c2;
@@ -75,7 +69,7 @@
 		interrupts = <GIC_SPI 4 IRQ_TYPE_EDGE_RISING>;
 	};
 
-	soc {
+	soc0: soc@0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
 		#pinctrl-cells = <1>;
@@ -114,7 +108,32 @@
 			reg-io-width = <4>;
 			reg = <0x02530c00 0x100>;
 			interrupts = <GIC_SPI 164 IRQ_TYPE_EDGE_RISING>;
-			clock-frequency = <200000000>;
+			clocks = <&k2g_clks 0x2c 0>;
+			power-domains = <&k2g_pds 0x2c>;
+			status = "disabled";
+		};
+
+		uart1: serial@02531000 {
+			compatible = "ti,da830-uart", "ns16550a";
+			current-speed = <115200>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			reg = <0x02531000 0x100>;
+			interrupts = <GIC_SPI 165 IRQ_TYPE_EDGE_RISING>;
+			clocks = <&k2g_clks 0x2d 0>;
+			power-domains = <&k2g_pds 0x2d>;
+			status = "disabled";
+		};
+
+		uart2: serial@02531400 {
+			compatible = "ti,da830-uart", "ns16550a";
+			current-speed = <115200>;
+			reg-shift = <2>;
+			reg-io-width = <4>;
+			reg = <0x02531400 0x100>;
+			interrupts = <GIC_SPI 166 IRQ_TYPE_EDGE_RISING>;
+			clocks = <&k2g_clks 0x2e 0>;
+			power-domains = <&k2g_pds 0x2e>;
 			status = "disabled";
 		};
 
@@ -372,11 +391,24 @@
 			dma-names = "tx", "rx";
 			bus-width = <8>;
 			ti,needs-special-reset;
-			ti,non-removable;
 			max-frequency = <96000000>;
 			power-domains = <&k2g_pds 0xc>;
 			clocks = <&k2g_clks 0xc 1>, <&k2g_clks 0xc 2>;
 			clock-names = "fck", "mmchsdb_fck";
+		};
+
+		qspi: qspi@2940000 {
+			compatible = "ti,k2g-qspi", "cdns,qspi-nor";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x02940000 0x1000>,
+			      <0x24000000 0x4000000>;
+			interrupts = <GIC_SPI 198 IRQ_TYPE_EDGE_RISING>;
+			cdns,fifo-depth = <256>;
+			cdns,fifo-width = <4>;
+			cdns,trigger-address = <0x24000000>;
+			clocks = <&k2g_clks 0x43 0x0>;
+			power-domains = <&k2g_pds 0x43>;
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/keystone-k2hk-clocks.dtsi b/arch/arm/boot/dts/keystone-k2hk-clocks.dtsi
index af9b7190533a..4ba6912176ef 100644
--- a/arch/arm/boot/dts/keystone-k2hk-clocks.dtsi
+++ b/arch/arm/boot/dts/keystone-k2hk-clocks.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2013-2014 Texas Instruments, Inc.
- *
  * Keystone 2 Kepler/Hawking SoC clock nodes
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 clocks {
@@ -53,7 +50,7 @@ clocks {
 		reg-names = "control";
 	};
 
-	clktsip: clktsip {
+	clktsip: clktsip@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk16>;
@@ -63,7 +60,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clksrio: clksrio {
+	clksrio: clksrio@235002c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1rstiso13>;
@@ -73,7 +70,7 @@ clocks {
 		domain-id = <4>;
 	};
 
-	clkhyperlink0: clkhyperlink0 {
+	clkhyperlink0: clkhyperlink0@2350030 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -83,7 +80,7 @@ clocks {
 		domain-id = <5>;
 	};
 
-	clkgem1: clkgem1 {
+	clkgem1: clkgem1@2350040 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -93,7 +90,7 @@ clocks {
 		domain-id = <9>;
 	};
 
-	clkgem2: clkgem2 {
+	clkgem2: clkgem2@2350044 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -103,7 +100,7 @@ clocks {
 		domain-id = <10>;
 	};
 
-	clkgem3: clkgem3 {
+	clkgem3: clkgem3@2350048 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -113,7 +110,7 @@ clocks {
 		domain-id = <11>;
 	};
 
-	clkgem4: clkgem4 {
+	clkgem4: clkgem4@235004c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -123,7 +120,7 @@ clocks {
 		domain-id = <12>;
 	};
 
-	clkgem5: clkgem5 {
+	clkgem5: clkgem5@2350050 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -133,7 +130,7 @@ clocks {
 		domain-id = <13>;
 	};
 
-	clkgem6: clkgem6 {
+	clkgem6: clkgem6@2350054 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -143,7 +140,7 @@ clocks {
 		domain-id = <14>;
 	};
 
-	clkgem7: clkgem7 {
+	clkgem7: clkgem7@2350058 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -153,7 +150,7 @@ clocks {
 		domain-id = <15>;
 	};
 
-	clkddr31: clkddr31 {
+	clkddr31: clkddr31@2350060 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -163,7 +160,7 @@ clocks {
 		domain-id = <16>;
 	};
 
-	clktac: clktac {
+	clktac: clktac@2350064 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -173,7 +170,7 @@ clocks {
 		domain-id = <17>;
 	};
 
-	clkrac01: clkrac01 {
+	clkrac01: clkrac01@2350068 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -183,7 +180,7 @@ clocks {
 		domain-id = <17>;
 	};
 
-	clkrac23: clkrac23 {
+	clkrac23: clkrac23@235006c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -193,7 +190,7 @@ clocks {
 		domain-id = <18>;
 	};
 
-	clkfftc0: clkfftc0 {
+	clkfftc0: clkfftc0@2350070 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -203,7 +200,7 @@ clocks {
 		domain-id = <19>;
 	};
 
-	clkfftc1: clkfftc1 {
+	clkfftc1: clkfftc1@2350074 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -213,7 +210,7 @@ clocks {
 		domain-id = <19>;
 	};
 
-	clkfftc2: clkfftc2 {
+	clkfftc2: clkfftc2@2350078 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -223,7 +220,7 @@ clocks {
 		domain-id = <20>;
 	};
 
-	clkfftc3: clkfftc3 {
+	clkfftc3: clkfftc3@235007c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -233,7 +230,7 @@ clocks {
 		domain-id = <20>;
 	};
 
-	clkfftc4: clkfftc4 {
+	clkfftc4: clkfftc4@2350080 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -243,7 +240,7 @@ clocks {
 		domain-id = <20>;
 	};
 
-	clkfftc5: clkfftc5 {
+	clkfftc5: clkfftc5@2350084 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -253,7 +250,7 @@ clocks {
 		domain-id = <20>;
 	};
 
-	clkaif: clkaif {
+	clkaif: clkaif@2350088 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -263,7 +260,7 @@ clocks {
 		domain-id = <21>;
 	};
 
-	clktcp3d0: clktcp3d0 {
+	clktcp3d0: clktcp3d0@235008c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -273,7 +270,7 @@ clocks {
 		domain-id = <22>;
 	};
 
-	clktcp3d1: clktcp3d1 {
+	clktcp3d1: clktcp3d1@2350090 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -283,7 +280,7 @@ clocks {
 		domain-id = <22>;
 	};
 
-	clktcp3d2: clktcp3d2 {
+	clktcp3d2: clktcp3d2@2350094 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -293,7 +290,7 @@ clocks {
 		domain-id = <23>;
 	};
 
-	clktcp3d3: clktcp3d3 {
+	clktcp3d3: clktcp3d3@2350098 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -303,7 +300,7 @@ clocks {
 		domain-id = <23>;
 	};
 
-	clkvcp0: clkvcp0 {
+	clkvcp0: clkvcp0@235009c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -313,7 +310,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkvcp1: clkvcp1 {
+	clkvcp1: clkvcp1@23500a0 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -323,7 +320,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkvcp2: clkvcp2 {
+	clkvcp2: clkvcp2@23500a4 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -333,7 +330,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkvcp3: clkvcp3 {
+	clkvcp3: clkvcp3@23500a8 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -343,7 +340,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkvcp4: clkvcp4 {
+	clkvcp4: clkvcp4@23500ac {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -353,7 +350,7 @@ clocks {
 		domain-id = <25>;
 	};
 
-	clkvcp5: clkvcp5 {
+	clkvcp5: clkvcp5@23500b0 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -363,7 +360,7 @@ clocks {
 		domain-id = <25>;
 	};
 
-	clkvcp6: clkvcp6 {
+	clkvcp6: clkvcp6@23500b4 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -373,7 +370,7 @@ clocks {
 		domain-id = <25>;
 	};
 
-	clkvcp7: clkvcp7 {
+	clkvcp7: clkvcp7@23500b8 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -383,7 +380,7 @@ clocks {
 		domain-id = <25>;
 	};
 
-	clkbcp: clkbcp {
+	clkbcp: clkbcp@23500bc {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -393,7 +390,7 @@ clocks {
 		domain-id = <26>;
 	};
 
-	clkdxb: clkdxb {
+	clkdxb: clkdxb@23500c0 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -403,7 +400,7 @@ clocks {
 		domain-id = <27>;
 	};
 
-	clkhyperlink1: clkhyperlink1 {
+	clkhyperlink1: clkhyperlink1@23500c4 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -413,7 +410,7 @@ clocks {
 		domain-id = <28>;
 	};
 
-	clkxge: clkxge {
+	clkxge: clkxge@23500c8 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
diff --git a/arch/arm/boot/dts/keystone-k2hk-evm.dts b/arch/arm/boot/dts/keystone-k2hk-evm.dts
index 6dd13b98aaba..ad4e22afe133 100644
--- a/arch/arm/boot/dts/keystone-k2hk-evm.dts
+++ b/arch/arm/boot/dts/keystone-k2hk-evm.dts
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2013-2014 Texas Instruments, Inc.
- *
  * Keystone 2 Kepler/Hawking EVM device tree
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 /dts-v1/;
 
@@ -29,7 +26,31 @@
 		};
 	};
 
-	soc {
+	leds {
+		compatible = "gpio-leds";
+		debug1_1 {
+			label = "keystone:green:debug1";
+			gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>; /* 12 */
+		};
+
+		debug1_2 {
+			label = "keystone:red:debug1";
+			gpios = <&gpio0 13 GPIO_ACTIVE_HIGH>; /* 13 */
+		};
+
+		debug2 {
+			label = "keystone:blue:debug2";
+			gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>; /* 14 */
+		};
+
+		debug3 {
+			label = "keystone:blue:debug3";
+			gpios = <&gpio0 15 GPIO_ACTIVE_HIGH>; /* 15 */
+		};
+	};
+};
+
+&soc0 {
 		clocks {
 			refclksys: refclksys {
 				#clock-cells = <0>;
@@ -66,30 +87,6 @@
 				clock-output-names = "refclk-ddr3b";
 			};
 		};
-	};
-
-	leds {
-		compatible = "gpio-leds";
-		debug1_1 {
-			label = "keystone:green:debug1";
-			gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>; /* 12 */
-		};
-
-		debug1_2 {
-			label = "keystone:red:debug1";
-			gpios = <&gpio0 13 GPIO_ACTIVE_HIGH>; /* 13 */
-		};
-
-		debug2 {
-			label = "keystone:blue:debug2";
-			gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>; /* 14 */
-		};
-
-		debug3 {
-			label = "keystone:blue:debug3";
-			gpios = <&gpio0 15 GPIO_ACTIVE_HIGH>; /* 15 */
-		};
-	};
 };
 
 &usb_phy {
@@ -158,7 +155,7 @@
 
 &i2c0 {
 	dtt@50 {
-		compatible = "at,24c1024";
+		compatible = "atmel,24c1024";
 		reg = <0x50>;
 	};
 };
diff --git a/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi b/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi
index a5ac845464bf..b88c0689c285 100644
--- a/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi
+++ b/arch/arm/boot/dts/keystone-k2hk-netcp.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree Source for Keystone 2 Hawking Netcp driver
  *
- * Copyright 2015 Texas Instruments, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2015-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 qmss: qmss@2a40000 {
diff --git a/arch/arm/boot/dts/keystone-k2hk.dtsi b/arch/arm/boot/dts/keystone-k2hk.dtsi
index 7c486d9dc90e..ed59474522cb 100644
--- a/arch/arm/boot/dts/keystone-k2hk.dtsi
+++ b/arch/arm/boot/dts/keystone-k2hk.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2013-2014 Texas Instruments, Inc.
- *
  * Keystone 2 Kepler/Hawking soc specific device tree
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 #include <dt-bindings/reset/ti-syscon.h>
@@ -55,8 +52,9 @@
 		rproc6 = &dsp6;
 		rproc7 = &dsp7;
 	};
+};
 
-	soc {
+&soc0 {
 		/include/ "keystone-k2hk-clocks.dtsi"
 
 		msm_ram: msmram@c000000 {
@@ -284,5 +282,4 @@
 			bus_freq	= <2500000>;
 		};
 		/include/ "keystone-k2hk-netcp.dtsi"
-	};
 };
diff --git a/arch/arm/boot/dts/keystone-k2l-clocks.dtsi b/arch/arm/boot/dts/keystone-k2l-clocks.dtsi
index ef8464bb11ff..635528064dea 100644
--- a/arch/arm/boot/dts/keystone-k2l-clocks.dtsi
+++ b/arch/arm/boot/dts/keystone-k2l-clocks.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2013-2014 Texas Instruments, Inc.
- *
  * Keystone 2 lamarr SoC clock nodes
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 clocks {
@@ -44,7 +41,7 @@ clocks {
 		reg-names = "control";
 	};
 
-	clkdfeiqnsys: clkdfeiqnsys {
+	clkdfeiqnsys: clkdfeiqnsys@2350004 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -54,7 +51,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkpcie1: clkpcie1 {
+	clkpcie1: clkpcie1@235002c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk12>;
@@ -64,7 +61,7 @@ clocks {
 		domain-id = <4>;
 	};
 
-	clkgem1: clkgem1 {
+	clkgem1: clkgem1@2350040 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -74,7 +71,7 @@ clocks {
 		domain-id = <9>;
 	};
 
-	clkgem2: clkgem2 {
+	clkgem2: clkgem2@2350044 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -84,7 +81,7 @@ clocks {
 		domain-id = <10>;
 	};
 
-	clkgem3: clkgem3 {
+	clkgem3: clkgem3@2350048 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk1>;
@@ -94,7 +91,7 @@ clocks {
 		domain-id = <11>;
 	};
 
-	clktac: clktac {
+	clktac: clktac@2350064 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -104,7 +101,7 @@ clocks {
 		domain-id = <17>;
 	};
 
-	clkrac: clkrac {
+	clkrac: clkrac@2350068 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -114,7 +111,7 @@ clocks {
 		domain-id = <17>;
 	};
 
-	clkdfepd0: clkdfepd0 {
+	clkdfepd0: clkdfepd0@235006c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -124,7 +121,7 @@ clocks {
 		domain-id = <18>;
 	};
 
-	clkfftc0: clkfftc0 {
+	clkfftc0: clkfftc0@2350070 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -134,7 +131,7 @@ clocks {
 		domain-id = <19>;
 	};
 
-	clkosr: clkosr {
+	clkosr: clkosr@2350088 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -144,7 +141,7 @@ clocks {
 		domain-id = <21>;
 	};
 
-	clktcp3d0: clktcp3d0 {
+	clktcp3d0: clktcp3d0@235008c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -154,7 +151,7 @@ clocks {
 		domain-id = <22>;
 	};
 
-	clktcp3d1: clktcp3d1 {
+	clktcp3d1: clktcp3d1@2350094 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -164,7 +161,7 @@ clocks {
 		domain-id = <23>;
 	};
 
-	clkvcp0: clkvcp0 {
+	clkvcp0: clkvcp0@235009c {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -174,7 +171,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkvcp1: clkvcp1 {
+	clkvcp1: clkvcp1@23500a0 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -184,7 +181,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkvcp2: clkvcp2 {
+	clkvcp2: clkvcp2@23500a4 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -194,7 +191,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkvcp3: clkvcp3 {
+	clkvcp3: clkvcp3@23500a8 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -204,7 +201,7 @@ clocks {
 		domain-id = <24>;
 	};
 
-	clkbcp: clkbcp {
+	clkbcp: clkbcp@23500bc {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -214,7 +211,7 @@ clocks {
 		domain-id = <26>;
 	};
 
-	clkdfepd1: clkdfepd1 {
+	clkdfepd1: clkdfepd1@23500c0 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -224,7 +221,7 @@ clocks {
 		domain-id = <27>;
 	};
 
-	clkfftc1: clkfftc1 {
+	clkfftc1: clkfftc1@23500c4 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -234,7 +231,7 @@ clocks {
 		domain-id = <28>;
 	};
 
-	clkiqnail: clkiqnail {
+	clkiqnail: clkiqnail@23500c8 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&chipclk13>;
@@ -244,7 +241,7 @@ clocks {
 		domain-id = <29>;
 	};
 
-	clkuart2: clkuart2 {
+	clkuart2: clkuart2@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
@@ -254,7 +251,7 @@ clocks {
 		domain-id = <0>;
 	};
 
-	clkuart3: clkuart3 {
+	clkuart3: clkuart3@2350000 {
 		#clock-cells = <0>;
 		compatible = "ti,keystone,psc-clock";
 		clocks = <&clkmodrst0>;
diff --git a/arch/arm/boot/dts/keystone-k2l-evm.dts b/arch/arm/boot/dts/keystone-k2l-evm.dts
index 528667618db4..e200533d26a4 100644
--- a/arch/arm/boot/dts/keystone-k2l-evm.dts
+++ b/arch/arm/boot/dts/keystone-k2l-evm.dts
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2014 Texas Instruments, Inc.
- *
  * Keystone 2 Lamarr EVM device tree
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2014-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 /dts-v1/;
 
@@ -28,8 +25,9 @@
 			status = "okay";
 		};
 	};
+};
 
-	soc {
+&soc0 {
 		clocks {
 			refclksys: refclksys {
 				#clock-cells = <0>;
@@ -38,7 +36,6 @@
 				clock-output-names = "refclk-sys";
 			};
 		};
-	};
 };
 
 &usb_phy {
@@ -55,7 +52,7 @@
 
 &i2c0 {
 	dtt@50 {
-		compatible = "at,24c1024";
+		compatible = "atmel,24c1024";
 		reg = <0x50>;
 	};
 };
diff --git a/arch/arm/boot/dts/keystone-k2l-netcp.dtsi b/arch/arm/boot/dts/keystone-k2l-netcp.dtsi
index 66f615a74118..9ec84228bc16 100644
--- a/arch/arm/boot/dts/keystone-k2l-netcp.dtsi
+++ b/arch/arm/boot/dts/keystone-k2l-netcp.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device Tree Source for Keystone 2 Lamarr Netcp driver
  *
- * Copyright 2015 Texas Instruments, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2015-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 qmss: qmss@2a40000 {
diff --git a/arch/arm/boot/dts/keystone-k2l.dtsi b/arch/arm/boot/dts/keystone-k2l.dtsi
index 4370e6513aa4..b61a830f4a4d 100644
--- a/arch/arm/boot/dts/keystone-k2l.dtsi
+++ b/arch/arm/boot/dts/keystone-k2l.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2014 Texas Instruments, Inc.
- *
  * Keystone 2 Lamarr SoC specific device tree
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2014-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 #include <dt-bindings/reset/ti-syscon.h>
@@ -39,8 +36,9 @@
 		rproc2 = &dsp2;
 		rproc3 = &dsp3;
 	};
+};
 
-	soc {
+&soc0 {
 		/include/ "keystone-k2l-clocks.dtsi"
 
 		uart2: serial@2348400 {
@@ -63,6 +61,50 @@
 			interrupts = <GIC_SPI 435 IRQ_TYPE_EDGE_RISING>;
 		};
 
+		gpio1: gpio@2348000 {
+			compatible = "ti,keystone-gpio";
+			reg = <0x02348000 0x100>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			/* HW Interrupts mapped to GPIO pins */
+			interrupts = <GIC_SPI 152 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 153 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 154 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 155 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 156 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 157 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 158 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 159 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 160 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 161 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 162 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 163 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 164 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 165 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 166 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 167 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 168 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 169 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 170 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 171 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 172 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 173 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 174 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 175 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 176 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 401 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 402 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 403 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 404 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 405 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 406 IRQ_TYPE_EDGE_RISING>,
+					<GIC_SPI 407 IRQ_TYPE_EDGE_RISING>;
+			clocks = <&clkgpio>;
+			clock-names = "gpio";
+			ti,ngpio = <32>;
+			ti,davinci-gpio-unbanked = <32>;
+		};
+
 		k2l_pmx: pinmux@2620690 {
 			compatible = "pinctrl-single";
 			reg = <0x02620690 0xc>;
@@ -350,7 +392,6 @@
 			bus_freq	= <2500000>;
 		};
 		/include/ "keystone-k2l-netcp.dtsi"
-	};
 };
 
 &spi0 {
diff --git a/arch/arm/boot/dts/keystone.dtsi b/arch/arm/boot/dts/keystone.dtsi
index 06e10544f9b1..93ea5c69ea77 100644
--- a/arch/arm/boot/dts/keystone.dtsi
+++ b/arch/arm/boot/dts/keystone.dtsi
@@ -1,16 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright 2013 Texas Instruments, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2013-2017 Texas Instruments Incorporated - http://www.ti.com/
  */
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/gpio/gpio.h>
 
-#include "skeleton.dtsi"
-
 / {
 	compatible = "ti,keystone";
 	model = "Texas Instruments Keystone 2 SoC";
@@ -25,11 +20,14 @@
 		spi2 = &spi2;
 	};
 
-	memory {
+	chosen { };
+
+	memory: memory@80000000 {
+		device_type = "memory";
 		reg = <0x00000000 0x80000000 0x00000000 0x40000000>;
 	};
 
-	gic: interrupt-controller {
+	gic: interrupt-controller@2561000 {
 		compatible = "arm,gic-400", "arm,cortex-a15-gic";
 		#interrupt-cells = <3>;
 		interrupt-controller;
@@ -70,7 +68,7 @@
 		cpu_on		= <0x84000003>;
 	};
 
-	soc {
+	soc0: soc@0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
 		compatible = "ti,keystone","simple-bus";
diff --git a/arch/arm/boot/dts/kirkwood-linksys-viper.dts b/arch/arm/boot/dts/kirkwood-linksys-viper.dts
index df7851820507..f21a50dd9869 100644
--- a/arch/arm/boot/dts/kirkwood-linksys-viper.dts
+++ b/arch/arm/boot/dts/kirkwood-linksys-viper.dts
@@ -157,7 +157,7 @@
 			reg = <0x80000 0x20000>;
 		};
 
-		partition@A0000 {
+		partition@a0000 {
 			label = "s_env";
 			reg = <0xA0000 0x20000>;
 		};
@@ -167,17 +167,17 @@
 			reg = <0x200000 0x2A0000>;
 		};
 
-		partition@4A0000 {
+		partition@4a0000 {
 			label = "rootfs";
 			reg = <0x4A0000 0x1760000>;
 		};
 
-		partition@1C00000 {
+		partition@1c00000 {
 			label = "alt_kernel";
 			reg = <0x1C00000 0x2A0000>;
 		};
 
-		partition@1EA0000 {
+		partition@1ea0000 {
 			label = "alt_rootfs";
 			reg = <0x1EA0000 0x1760000>;
 		};
@@ -187,7 +187,7 @@
 			reg = <0x3600000 0x4A00000>;
 		};
 
-		partition@C0000 {
+		partition@c0000 {
 			label = "unused";
 			reg = <0xC0000 0x140000>;
 		};
diff --git a/arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts b/arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts
new file mode 100644
index 000000000000..32d0dc371fc3
--- /dev/null
+++ b/arch/arm/boot/dts/logicpd-som-lv-35xx-devkit.dts
@@ -0,0 +1,17 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/dts-v1/;
+
+#include "omap34xx.dtsi"
+#include "logicpd-som-lv.dtsi"
+#include "logicpd-som-lv-baseboard.dtsi"
+#include "omap-gpmc-smsc9221.dtsi"
+
+/ {
+	model = "LogicPD Zoom OMAP35xx SOM-LV Development Kit";
+	compatible = "logicpd,dm3730-som-lv-devkit", "ti,omap3";
+};
diff --git a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
index 2fa5eb4bd402..24283739526c 100644
--- a/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
+++ b/arch/arm/boot/dts/logicpd-som-lv-37xx-devkit.dts
@@ -8,257 +8,10 @@
 
 #include "omap36xx.dtsi"
 #include "logicpd-som-lv.dtsi"
+#include "logicpd-som-lv-baseboard.dtsi"
 #include "omap-gpmc-smsc9221.dtsi"
 
 / {
 	model = "LogicPD Zoom DM3730 SOM-LV Development Kit";
 	compatible = "logicpd,dm3730-som-lv-devkit", "ti,omap3630", "ti,omap3";
-
-	gpio_keys {
-		compatible = "gpio-keys";
-		pinctrl-names = "default";
-		pinctrl-0 = <&gpio_key_pins>;
-
-		sysboot2 {
-			label = "gpio3";
-			gpios = <&gpio4 15 GPIO_ACTIVE_LOW>;	/* gpio_111 / uP_GPIO_3 */
-			linux,code = <BTN_0>;
-			wakeup-source;
-		};
-	};
-
-	sound {
-		compatible = "ti,omap-twl4030";
-		ti,model = "omap3logic";
-		ti,mcbsp = <&mcbsp2>;
-	};
-
-	leds {
-		compatible = "gpio-leds";
-		pinctrl-names = "default";
-		pinctrl-0 = <&led_pins &led_pins_wkup>;
-
-		led1 {
-			label = "led1";
-			gpios = <&gpio5 5 GPIO_ACTIVE_LOW>;	/* gpio133 */
-			linux,default-trigger = "cpu0";
-		};
-
-		led2 {
-			label = "led2";
-			gpios = <&gpio1 11 GPIO_ACTIVE_LOW>;	/* gpio11 */
-			linux,default-trigger = "none";
-		};
-	};
-};
-
-&vaux1 {
-	regulator-min-microvolt = <3000000>;
-	regulator-max-microvolt = <3000000>;
-};
-
-&vaux4 {
-	regulator-min-microvolt = <1800000>;
-	regulator-max-microvolt = <1800000>;
-};
-
-&mcbsp2 {
-	status = "okay";
-};
-
-&charger {
-	ti,bb-uvolt = <3200000>;
-	ti,bb-uamp = <150>;
-};
-
-&gpmc {
-	ranges = <0 0 0x30000000 0x1000000	/* CS0: 16MB for NAND */
-		  1 0 0x2c000000 0x1000000>;	/* CS1: 16MB for LAN9221 */
-
-	ethernet@gpmc {
-		pinctrl-names = "default";
-		pinctrl-0 = <&lan9221_pins>;
-		interrupt-parent = <&gpio5>;
-		interrupts = <24 IRQ_TYPE_LEVEL_LOW>;		/* gpio_152 */
-		reg = <1 0 0xff>;
-	};
-};
-
-&vpll2 {
-	regulator-always-on;
-};
-
-&dss {
-	status = "ok";
-	vdds_dsi-supply = <&vpll2>;
-	vdda_video-supply = <&video_reg>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&dss_dpi_pins1>;
-	port {
-		dpi_out: endpoint {
-			remote-endpoint = <&lcd_in>;
-			data-lines = <16>;
-		};
-	};
-};
-
-/ {
-	aliases {
-		display0 = &lcd0;
-	};
-
-	video_reg: video_reg {
-		compatible = "regulator-fixed";
-		regulator-name = "fixed-supply";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-	};
-
-	lcd0: display@0 {
-		compatible = "panel-dpi";
-		label = "28";
-		status = "okay";
-		/* default-on; */
-		pinctrl-names = "default";
-		pinctrl-0 = <&lcd_enable_pin>;
-		enable-gpios = <&gpio5 27 GPIO_ACTIVE_HIGH>;	/* gpio155, lcd INI */
-		port {
-			lcd_in: endpoint {
-				remote-endpoint = <&dpi_out>;
-			};
-		};
-
-		panel-timing {
-			clock-frequency = <9000000>;
-			hactive = <480>;
-			vactive = <272>;
-			hfront-porch = <3>;
-			hback-porch = <2>;
-			hsync-len = <42>;
-			vback-porch = <3>;
-			vfront-porch = <2>;
-			vsync-len = <11>;
-			hsync-active = <1>;
-			vsync-active = <1>;
-			de-active = <1>;
-			pixelclk-active = <0>;
-		};
-	};
-
-	bl: backlight {
-		compatible = "pwm-backlight";
-		pinctrl-names = "default";
-		pinctrl-0 = <&backlight_pins>;
-		pwms = <&twl_pwm 0 5000000>;
-		brightness-levels = <0 10 20 30 40 50 60 70 80 90 100>;
-		default-brightness-level = <7>;
-		enable-gpios = <&gpio1 8 GPIO_ACTIVE_HIGH>; /* gpio_8 */
-	};
-};
-
-&mmc1 {
-	interrupts-extended = <&intc 83 &omap3_pmx_core 0x11a>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&mmc1_pins>;
-	wp-gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>;		/* gpio_126 */
-	cd-gpios = <&gpio4 14 IRQ_TYPE_LEVEL_LOW>;		/* gpio_110 */
-	vmmc-supply = <&vmmc1>;
-	bus-width = <4>;
-	cap-power-off-card;
-};
-
-&omap3_pmx_core {
-	gpio_key_pins: pinmux_gpio_key_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x212e, PIN_INPUT_PULLUP | MUX_MODE4)	/* cam_xclkb.gpio_111 / uP_GPIO_3*/
-		>;
-	};
-
-	led_pins: pinmux_led_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x215e, PIN_OUTPUT_PULLUP | MUX_MODE4)	/* sdmmc2_dat1.gpio_133 / uP_GPIO_0 */
-		>;
-	};
-
-	lan9221_pins: pinmux_lan9221_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x2184, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcbsp4_clkx.gpio_152 */
-		>;
-	};
-
-	mmc1_pins: pinmux_mmc1_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x2144, PIN_OUTPUT | MUX_MODE0)	/* sdmmc1_clk.sdmmc1_clk */
-			OMAP3_CORE1_IOPAD(0x2146, PIN_INPUT | MUX_MODE0)	/* sdmmc1_cmd.sdmmc1_cmd */
-			OMAP3_CORE1_IOPAD(0x2148, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat0.sdmmc1_dat0 */
-			OMAP3_CORE1_IOPAD(0x214a, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat1.sdmmc1_dat1 */
-			OMAP3_CORE1_IOPAD(0x214c, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat2.sdmmc1_dat2 */
-			OMAP3_CORE1_IOPAD(0x214e, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat3.sdmmc1_dat3 */
-			OMAP3_CORE1_IOPAD(0x2132, PIN_INPUT_PULLUP | MUX_MODE4)	/* cam_strobe.gpio_126 */
-			OMAP3_CORE1_IOPAD(0x212c, PIN_INPUT_PULLUP | MUX_MODE4)	/* cam_d11.gpio_110 */
-		>;
-	};
-
-	lcd_enable_pin: pinmux_lcd_enable_pin {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x218a, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* mcbsp4_fs.gpio_155 */
-		>;
-	};
-
-	dss_dpi_pins1: pinmux_dss_dpi_pins1 {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x20d4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_pclk.dss_pclk */
-			OMAP3_CORE1_IOPAD(0x20d6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_hsync.dss_hsync */
-			OMAP3_CORE1_IOPAD(0x20d8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_vsync.dss_vsync */
-			OMAP3_CORE1_IOPAD(0x20da, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_acbias.dss_acbias */
-
-			OMAP3_CORE1_IOPAD(0x20dc, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data0.dss_data0 */
-			OMAP3_CORE1_IOPAD(0x20de, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data1.dss_data1 */
-			OMAP3_CORE1_IOPAD(0x20e0, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data2.dss_data2 */
-			OMAP3_CORE1_IOPAD(0x20e2, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data3.dss_data3 */
-			OMAP3_CORE1_IOPAD(0x20e4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data4.dss_data4 */
-			OMAP3_CORE1_IOPAD(0x20e6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data5.dss_data5 */
-			OMAP3_CORE1_IOPAD(0x20e8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data6.dss_data6 */
-			OMAP3_CORE1_IOPAD(0x20ea, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data7.dss_data7 */
-			OMAP3_CORE1_IOPAD(0x20ec, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data8.dss_data8 */
-			OMAP3_CORE1_IOPAD(0x20ee, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data9.dss_data9 */
-			OMAP3_CORE1_IOPAD(0x20f0, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data10.dss_data10 */
-			OMAP3_CORE1_IOPAD(0x20f2, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data11.dss_data11 */
-			OMAP3_CORE1_IOPAD(0x20f4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data12.dss_data12 */
-			OMAP3_CORE1_IOPAD(0x20f6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data13.dss_data13 */
-			OMAP3_CORE1_IOPAD(0x20f8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data14.dss_data14 */
-			OMAP3_CORE1_IOPAD(0x20fa, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data15.dss_data15 */
-		>;
-	};
-};
-
-&omap3_pmx_wkup {
-	led_pins_wkup: pinmux_led_pins_wkup {
-		pinctrl-single,pins = <
-			OMAP3_WKUP_IOPAD(0x2a24, PIN_OUTPUT_PULLUP | MUX_MODE4)	/* jtag_emu0.gpio_11 / uP_GPIO_1 */
-		>;
-	};
-
-	backlight_pins: pinmux_backlight_pins {
-		pinctrl-single,pins = <
-			OMAP3_WKUP_IOPAD(0x2a16, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* sys_boot6.gpio_8 */
-		>;
-	};
-};
-
-
-&uart1 {
-	interrupts-extended = <&intc 72 &omap3_pmx_core OMAP3_UART1_RX>;
-};
-
-/* Wired to the tps65950 on the SOM, only the USB connector is on the devkit */
-&usb_otg_hs {
-	pinctrl-names = "default";
-	pinctrl-0 = <&hsusb_otg_pins>;
-	interface-type = <0>;
-	usb-phy = <&usb2_phy>;
-	phys = <&usb2_phy>;
-	phy-names = "usb2-phy";
-	mode = <3>;
-	power = <50>;
 };
diff --git a/arch/arm/boot/dts/logicpd-som-lv-baseboard.dtsi b/arch/arm/boot/dts/logicpd-som-lv-baseboard.dtsi
new file mode 100644
index 000000000000..4990ed90dcea
--- /dev/null
+++ b/arch/arm/boot/dts/logicpd-som-lv-baseboard.dtsi
@@ -0,0 +1,256 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/ {
+	gpio_keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_key_pins>;
+
+		sysboot2 {
+			label = "gpio3";
+			gpios = <&gpio4 15 GPIO_ACTIVE_LOW>;	/* gpio_111 / uP_GPIO_3 */
+			linux,code = <BTN_0>;
+			wakeup-source;
+		};
+	};
+
+	sound {
+		compatible = "ti,omap-twl4030";
+		ti,model = "omap3logic";
+		ti,mcbsp = <&mcbsp2>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&led_pins &led_pins_wkup>;
+
+		led1 {
+			label = "led1";
+			gpios = <&gpio5 5 GPIO_ACTIVE_LOW>;	/* gpio133 */
+			linux,default-trigger = "cpu0";
+		};
+
+		led2 {
+			label = "led2";
+			gpios = <&gpio1 11 GPIO_ACTIVE_LOW>;	/* gpio11 */
+			linux,default-trigger = "none";
+		};
+	};
+};
+
+&vaux1 {
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+};
+
+&vaux4 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+};
+
+&mcbsp2 {
+	status = "okay";
+};
+
+&charger {
+	ti,bb-uvolt = <3200000>;
+	ti,bb-uamp = <150>;
+};
+
+&gpmc {
+	ranges = <0 0 0x30000000 0x1000000	/* CS0: 16MB for NAND */
+		  1 0 0x2c000000 0x1000000	/* CS1: 16MB for LAN9221 */
+		  2 0 0x10000000 0x2000000>;    /* CS2: 32MB for NOR */
+
+	ethernet@gpmc {
+		pinctrl-names = "default";
+		pinctrl-0 = <&lan9221_pins>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <24 IRQ_TYPE_LEVEL_LOW>;		/* gpio_152 */
+		reg = <1 0 0xff>;
+	};
+};
+
+&vpll2 {
+	regulator-always-on;
+};
+
+&dss {
+	status = "ok";
+	vdds_dsi-supply = <&vpll2>;
+	vdda_video-supply = <&video_reg>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_dpi_pins1>;
+	port {
+		dpi_out: endpoint {
+			remote-endpoint = <&lcd_in>;
+			data-lines = <16>;
+		};
+	};
+};
+
+/ {
+	aliases {
+		display0 = &lcd0;
+	};
+
+	video_reg: video_reg {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	lcd0: display@0 {
+		compatible = "panel-dpi";
+		label = "28";
+		status = "okay";
+		/* default-on; */
+		pinctrl-names = "default";
+		pinctrl-0 = <&lcd_enable_pin>;
+		enable-gpios = <&gpio5 27 GPIO_ACTIVE_HIGH>;	/* gpio155, lcd INI */
+		port {
+			lcd_in: endpoint {
+				remote-endpoint = <&dpi_out>;
+			};
+		};
+
+		panel-timing {
+			clock-frequency = <9000000>;
+			hactive = <480>;
+			vactive = <272>;
+			hfront-porch = <3>;
+			hback-porch = <2>;
+			hsync-len = <42>;
+			vback-porch = <3>;
+			vfront-porch = <2>;
+			vsync-len = <11>;
+			hsync-active = <1>;
+			vsync-active = <1>;
+			de-active = <1>;
+			pixelclk-active = <0>;
+		};
+	};
+
+	bl: backlight {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&backlight_pins>;
+		pwms = <&twl_pwm 0 5000000>;
+		brightness-levels = <0 10 20 30 40 50 60 70 80 90 100>;
+		default-brightness-level = <7>;
+		enable-gpios = <&gpio1 8 GPIO_ACTIVE_HIGH>; /* gpio_8 */
+	};
+};
+
+&mmc1 {
+	interrupts-extended = <&intc 83 &omap3_pmx_core 0x11a>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>;
+	wp-gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>;		/* gpio_126 */
+	cd-gpios = <&gpio4 14 IRQ_TYPE_LEVEL_LOW>;		/* gpio_110 */
+	vmmc-supply = <&vmmc1>;
+	bus-width = <4>;
+	cap-power-off-card;
+};
+
+&omap3_pmx_core {
+	gpio_key_pins: pinmux_gpio_key_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x212e, PIN_INPUT_PULLUP | MUX_MODE4)	/* cam_xclkb.gpio_111 / uP_GPIO_3*/
+		>;
+	};
+
+	led_pins: pinmux_led_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x215e, PIN_OUTPUT_PULLUP | MUX_MODE4)	/* sdmmc2_dat1.gpio_133 / uP_GPIO_0 */
+		>;
+	};
+
+	lan9221_pins: pinmux_lan9221_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x2184, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcbsp4_clkx.gpio_152 */
+		>;
+	};
+
+	mmc1_pins: pinmux_mmc1_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x2144, PIN_OUTPUT | MUX_MODE0)	/* sdmmc1_clk.sdmmc1_clk */
+			OMAP3_CORE1_IOPAD(0x2146, PIN_INPUT | MUX_MODE0)	/* sdmmc1_cmd.sdmmc1_cmd */
+			OMAP3_CORE1_IOPAD(0x2148, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat0.sdmmc1_dat0 */
+			OMAP3_CORE1_IOPAD(0x214a, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat1.sdmmc1_dat1 */
+			OMAP3_CORE1_IOPAD(0x214c, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat2.sdmmc1_dat2 */
+			OMAP3_CORE1_IOPAD(0x214e, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat3.sdmmc1_dat3 */
+			OMAP3_CORE1_IOPAD(0x2132, PIN_INPUT_PULLUP | MUX_MODE4)	/* cam_strobe.gpio_126 */
+			OMAP3_CORE1_IOPAD(0x212c, PIN_INPUT_PULLUP | MUX_MODE4)	/* cam_d11.gpio_110 */
+		>;
+	};
+
+	lcd_enable_pin: pinmux_lcd_enable_pin {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x218a, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* mcbsp4_fs.gpio_155 */
+		>;
+	};
+
+	dss_dpi_pins1: pinmux_dss_dpi_pins1 {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x20d4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_pclk.dss_pclk */
+			OMAP3_CORE1_IOPAD(0x20d6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_hsync.dss_hsync */
+			OMAP3_CORE1_IOPAD(0x20d8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_vsync.dss_vsync */
+			OMAP3_CORE1_IOPAD(0x20da, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_acbias.dss_acbias */
+
+			OMAP3_CORE1_IOPAD(0x20dc, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data0.dss_data0 */
+			OMAP3_CORE1_IOPAD(0x20de, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data1.dss_data1 */
+			OMAP3_CORE1_IOPAD(0x20e0, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data2.dss_data2 */
+			OMAP3_CORE1_IOPAD(0x20e2, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data3.dss_data3 */
+			OMAP3_CORE1_IOPAD(0x20e4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data4.dss_data4 */
+			OMAP3_CORE1_IOPAD(0x20e6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data5.dss_data5 */
+			OMAP3_CORE1_IOPAD(0x20e8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data6.dss_data6 */
+			OMAP3_CORE1_IOPAD(0x20ea, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data7.dss_data7 */
+			OMAP3_CORE1_IOPAD(0x20ec, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data8.dss_data8 */
+			OMAP3_CORE1_IOPAD(0x20ee, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data9.dss_data9 */
+			OMAP3_CORE1_IOPAD(0x20f0, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data10.dss_data10 */
+			OMAP3_CORE1_IOPAD(0x20f2, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data11.dss_data11 */
+			OMAP3_CORE1_IOPAD(0x20f4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data12.dss_data12 */
+			OMAP3_CORE1_IOPAD(0x20f6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data13.dss_data13 */
+			OMAP3_CORE1_IOPAD(0x20f8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data14.dss_data14 */
+			OMAP3_CORE1_IOPAD(0x20fa, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data15.dss_data15 */
+		>;
+	};
+};
+
+&omap3_pmx_wkup {
+	led_pins_wkup: pinmux_led_pins_wkup {
+		pinctrl-single,pins = <
+			OMAP3_WKUP_IOPAD(0x2a24, PIN_OUTPUT_PULLUP | MUX_MODE4)	/* jtag_emu0.gpio_11 / uP_GPIO_1 */
+		>;
+	};
+
+	backlight_pins: pinmux_backlight_pins {
+		pinctrl-single,pins = <
+			OMAP3_WKUP_IOPAD(0x2a16, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* sys_boot6.gpio_8 */
+		>;
+	};
+};
+
+
+&uart1 {
+	interrupts-extended = <&intc 72 &omap3_pmx_core OMAP3_UART1_RX>;
+};
+
+/* Wired to the tps65950 on the SOM, only the USB connector is on the devkit */
+&usb_otg_hs {
+	pinctrl-names = "default";
+	pinctrl-0 = <&hsusb_otg_pins>;
+	interface-type = <0>;
+	usb-phy = <&usb2_phy>;
+	phys = <&usb2_phy>;
+	phy-names = "usb2-phy";
+	mode = <3>;
+	power = <50>;
+};
diff --git a/arch/arm/boot/dts/logicpd-som-lv.dtsi b/arch/arm/boot/dts/logicpd-som-lv.dtsi
index 29cb804d10cc..a30ee9fcb3ae 100644
--- a/arch/arm/boot/dts/logicpd-som-lv.dtsi
+++ b/arch/arm/boot/dts/logicpd-som-lv.dtsi
@@ -67,37 +67,12 @@
 		gpmc,device-width = <2>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-
-		/* u-boot uses mtdparts=omap2-nand.0:512k(x-loader),1920k(u-boot),128k(u-boot-env),4m(kernel),-(fs) */
-
-		x-loader@0 {
-			label = "x-loader";
-			reg = <0 0x80000>;
-		};
-
-		bootloaders@80000 {
-			label = "u-boot";
-			reg = <0x80000 0x1e0000>;
-		};
-
-		bootloaders_env@260000 {
-			label = "u-boot-env";
-			reg = <0x260000 0x20000>;
-		};
-
-		kernel@280000 {
-			label = "kernel";
-			reg = <0x280000 0x400000>;
-		};
-
-		filesystem@680000 {
-			label = "fs";
-			reg = <0x680000 0>;	/* 0 = MTDPART_SIZ_FULL */
-		};
 	};
 };
 
 &i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
 	clock-frequency = <2600000>;
 
 	twl: twl@48 {
@@ -216,7 +191,12 @@
 		>;
 	};
 
-
+	i2c1_pins: pinmux_i2c1_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x21ba, PIN_INPUT | MUX_MODE0)        /* i2c1_scl.i2c1_scl */
+			OMAP3_CORE1_IOPAD(0x21bc, PIN_INPUT | MUX_MODE0)        /* i2c1_sda.i2c1_sda */
+		>;
+	};
 };
 
 &omap3_pmx_wkup {
diff --git a/arch/arm/boot/dts/logicpd-torpedo-35xx-devkit.dts b/arch/arm/boot/dts/logicpd-torpedo-35xx-devkit.dts
new file mode 100644
index 000000000000..d7cb659656ce
--- /dev/null
+++ b/arch/arm/boot/dts/logicpd-torpedo-35xx-devkit.dts
@@ -0,0 +1,17 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/dts-v1/;
+
+#include "omap34xx.dtsi"
+#include "logicpd-torpedo-som.dtsi"
+#include "logicpd-torpedo-baseboard.dtsi"
+#include "omap-gpmc-smsc9221.dtsi"
+
+/ {
+	model = "LogicPD Zoom OMAP35xx Torpedo Development Kit";
+	compatible = "logicpd,dm3730-torpedo-devkit", "ti,omap3";
+};
diff --git a/arch/arm/boot/dts/logicpd-torpedo-37xx-devkit.dts b/arch/arm/boot/dts/logicpd-torpedo-37xx-devkit.dts
index b4575bbaf085..234afd6d60ec 100644
--- a/arch/arm/boot/dts/logicpd-torpedo-37xx-devkit.dts
+++ b/arch/arm/boot/dts/logicpd-torpedo-37xx-devkit.dts
@@ -9,396 +9,69 @@
 #include "omap36xx.dtsi"
 #include "logicpd-torpedo-som.dtsi"
 #include "omap-gpmc-smsc9221.dtsi"
+#include "logicpd-torpedo-baseboard.dtsi"
 
 / {
 	model = "LogicPD Zoom DM3730 Torpedo + Wireless Development Kit";
 	compatible = "logicpd,dm3730-torpedo-devkit", "ti,omap3630", "ti,omap3";
 
-	gpio_keys {
-		compatible = "gpio-keys";
-		pinctrl-names = "default";
-		pinctrl-0 = <&gpio_key_pins &gpio_key_pins_wkup>;
-
-		sysboot2 {
-			label = "sysboot2";
-			gpios = <&gpio1 2 GPIO_ACTIVE_LOW>;	/* gpio2 */
-			linux,code = <BTN_0>;
-			wakeup-source;
-		};
-
-		sysboot5 {
-			label = "sysboot5";
-			gpios = <&gpio1 7 GPIO_ACTIVE_LOW>;	/* gpio7 */
-			linux,code = <BTN_1>;
-			wakeup-source;
-		};
-
-		gpio1 {
-			label = "gpio1";
-			gpios = <&gpio6 21 GPIO_ACTIVE_LOW>;	/* gpio181 */
-			linux,code = <BTN_2>;
-			wakeup-source;
-		};
-
-		gpio2 {
-			label = "gpio2";
-			gpios = <&gpio6 18 GPIO_ACTIVE_LOW>;	/* gpio178 */
-			linux,code = <BTN_3>;
-			wakeup-source;
-		};
-	};
-
-	sound {
-		compatible = "ti,omap-twl4030";
-		ti,model = "omap3logic";
-		ti,mcbsp = <&mcbsp2>;
-	};
-
-	leds {
-		compatible = "gpio-leds";
-		pinctrl-names = "default";
-		pinctrl-0 = <&led_pins>;
-
-		led1 {
-			label = "led1";
-			gpios = <&gpio6 20 GPIO_ACTIVE_HIGH>;	/* gpio180 */
-			linux,default-trigger = "cpu0";
-		};
-
-		led2 {
-			label = "led2";
-			gpios = <&gpio6 19 GPIO_ACTIVE_HIGH>;	/* gpio179 */
-			linux,default-trigger = "none";
-		};
-	};
-
-	pwm10: dmtimer-pwm {
-		compatible = "ti,omap-dmtimer-pwm";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pwm_pins>;
-		ti,timers = <&timer10>;
-		#pwm-cells = <3>;
-	};
-
-};
-
-&vaux1 {
-	regulator-min-microvolt = <3000000>;
-	regulator-max-microvolt = <3000000>;
-};
-
-&vaux4 {
-	regulator-min-microvolt = <1800000>;
-	regulator-max-microvolt = <1800000>;
-};
-
-&mcbsp2 {
-	status = "okay";
-};
-
-&charger {
-	ti,bb-uvolt = <3200000>;
-	ti,bb-uamp = <150>;
-};
-
-&gpmc {
-	ranges = <0 0 0x30000000 0x1000000	/* CS0: 16MB for NAND */
-		  1 0 0x2c000000 0x1000000>;	/* CS1: 16MB for LAN9221 */
-
-	ethernet@gpmc {
-		pinctrl-names = "default";
-		pinctrl-0 = <&lan9221_pins>;
-		interrupt-parent = <&gpio5>;
-		interrupts = <1 IRQ_TYPE_LEVEL_LOW>;		/* gpio129 */
-		reg = <1 0 0xff>;
-	};
-};
-
-&vpll2 {
-	regulator-always-on;
-};
-
-&dss {
-	status = "ok";
-	vdds_dsi-supply = <&vpll2>;
-	vdda_video-supply = <&video_reg>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&dss_dpi_pins1>;
-	port {
-		dpi_out: endpoint {
-			remote-endpoint = <&lcd_in>;
-			data-lines = <16>;
-		};
-	};
-};
-
-/ {
-	aliases {
-		display0 = &lcd0;
-	};
-
-	video_reg: video_reg {
-		pinctrl-names = "default";
-		pinctrl-0 = <&panel_pwr_pins>;
+	wl12xx_vmmc: wl12xx_vmmc {
 		compatible = "regulator-fixed";
-		regulator-name = "fixed-supply";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		gpio = <&gpio5 27 GPIO_ACTIVE_HIGH>;	/* gpio155, lcd INI */
-	};
-
-	lcd0: display {
-		compatible = "panel-dpi";
-		label = "15";
-		status = "okay";
-		/* default-on; */
-		pinctrl-names = "default";
-
-		port {
-			lcd_in: endpoint {
-				remote-endpoint = <&dpi_out>;
-			};
-		};
-
-		panel-timing {
-			clock-frequency = <9000000>;
-			hactive = <480>;
-			vactive = <272>;
-			hfront-porch = <3>;
-			hback-porch = <2>;
-			hsync-len = <42>;
-			vback-porch = <3>;
-			vfront-porch = <4>;
-			vsync-len = <11>;
-			hsync-active = <0>;
-			vsync-active = <0>;
-			de-active = <1>;
-			pixelclk-active = <1>;
-		};
-	};
-
-	bl: backlight {
-		compatible = "pwm-backlight";
-		pinctrl-names = "default";
-		pinctrl-0 = <&backlight_pins>;
-		pwms = <&pwm10 0 5000000 0>;
-		brightness-levels = <0 10 20 30 40 50 60 70 80 90 100>;
-		default-brightness-level = <7>;
-		enable-gpios = <&gpio5 26 GPIO_ACTIVE_HIGH>; /* gpio_154 */
+		regulator-name = "vwl1271";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&gpio5 29 0>;   /* gpio157 */
+		startup-delay-us = <70000>;
+		enable-active-high;
+		vin-supply = <&vmmc2>;
 	};
 };
 
-&mmc1 {
-	interrupts-extended = <&intc 83 &omap3_pmx_core 0x11a>;
+/*
+ * Only found on the wireless SOM. For the SOM without wireless, the pins for
+ * MMC3 can be routed with jumpers to the second MMC slot on the devkit and
+ * gpio157 is not connected. So this should be OK to keep common for now,
+ * probably device tree overlays is the way to go with the various SOM and
+ * jumpering combinations for the long run.
+ */
+&mmc3 {
+	interrupts-extended = <&intc 94 &omap3_pmx_core2 0x46>;
+	pinctrl-0 = <&mmc3_pins &mmc3_core2_pins>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&mmc1_pins &mmc1_cd>;
-	cd-gpios = <&gpio4 31 GPIO_ACTIVE_LOW>;		/* gpio127 */
-	vmmc-supply = <&vmmc1>;
+	vmmc-supply = <&wl12xx_vmmc>;
+	non-removable;
 	bus-width = <4>;
 	cap-power-off-card;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	wlcore: wlcore@2 {
+		compatible = "ti,wl1283";
+		reg = <2>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <24 IRQ_TYPE_LEVEL_HIGH>; /* gpio 152 */
+		ref-clock-frequency = <26000000>;
+		tcxo-clock-frequency = <26000000>;
+	};
 };
 
 &omap3_pmx_core {
-	gpio_key_pins: pinmux_gpio_key_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x21d6, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcspi2_clk.gpio_178 */
-			OMAP3_CORE1_IOPAD(0x21dc, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcspi2_cs0.gpio_181 */
-		>;
-	};
-
-	pwm_pins: pinmux_pwm_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x20B8, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE3)       /* gpmc_ncs5.gpt_10_pwm_evt */
-		>;
-	};
-
-	led_pins: pinmux_led_pins {
+	mmc3_pins: pinmux_mm3_pins {
 		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x21d8, PIN_OUTPUT | MUX_MODE4)	/* gpio_179 */
-			OMAP3_CORE1_IOPAD(0x21da, PIN_OUTPUT | MUX_MODE4)	/* gpio_180 */
-		>;
-	};
-
-	mmc1_pins: pinmux_mmc1_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x2144, PIN_OUTPUT | MUX_MODE0)	/* sdmmc1_clk.sdmmc1_clk */
-			OMAP3_CORE1_IOPAD(0x2146, PIN_INPUT | MUX_MODE0)	/* sdmmc1_cmd.sdmmc1_cmd */
-			OMAP3_CORE1_IOPAD(0x2148, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat0.sdmmc1_dat0 */
-			OMAP3_CORE1_IOPAD(0x214a, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat1.sdmmc1_dat1 */
-			OMAP3_CORE1_IOPAD(0x214c, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat2.sdmmc1_dat2 */
-			OMAP3_CORE1_IOPAD(0x214e, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat3.sdmmc1_dat3 */
-		>;
-	};
-
-	tsc2004_pins: pinmux_tsc2004_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x2186, PIN_INPUT | MUX_MODE4)	/* mcbsp4_dr.gpio_153 */
-		>;
-	};
-
-	backlight_pins: pinmux_backlight_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x2188, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* mcbsp4_dx.gpio_154 */
-		>;
-	};
-
-	isp_pins: pinmux_isp_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x210c, PIN_INPUT | MUX_MODE0)   /* cam_hs.cam_hs */
-			OMAP3_CORE1_IOPAD(0x210e, PIN_INPUT | MUX_MODE0)   /* cam_vs.cam_vs */
-			OMAP3_CORE1_IOPAD(0x2110, PIN_INPUT | MUX_MODE0)   /* cam_xclka.cam_xclka */
-			OMAP3_CORE1_IOPAD(0x2112, PIN_INPUT | MUX_MODE0)   /* cam_pclk.cam_pclk */
-
-			OMAP3_CORE1_IOPAD(0x2116, PIN_INPUT | MUX_MODE0)   /* cam_d0.cam_d0 */
-			OMAP3_CORE1_IOPAD(0x2118, PIN_INPUT | MUX_MODE0)   /* cam_d1.cam_d1 */
-			OMAP3_CORE1_IOPAD(0x211a, PIN_INPUT | MUX_MODE0)   /* cam_d2.cam_d2 */
-			OMAP3_CORE1_IOPAD(0x211c, PIN_INPUT | MUX_MODE0)   /* cam_d3.cam_d3 */
-			OMAP3_CORE1_IOPAD(0x211e, PIN_INPUT | MUX_MODE0)   /* cam_d4.cam_d4 */
-			OMAP3_CORE1_IOPAD(0x2120, PIN_INPUT | MUX_MODE0)   /* cam_d5.cam_d5 */
-			OMAP3_CORE1_IOPAD(0x2122, PIN_INPUT | MUX_MODE0)   /* cam_d6.cam_d6 */
-			OMAP3_CORE1_IOPAD(0x2124, PIN_INPUT | MUX_MODE0)   /* cam_d7.cam_d7 */
-		>;
-	};
-
-	panel_pwr_pins: pinmux_panel_pwr_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x218a, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* mcbsp4_fs.gpio_155 */
-		>;
-	};
-
-	dss_dpi_pins1: pinmux_dss_dpi_pins1 {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x20d4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_pclk.dss_pclk */
-			OMAP3_CORE1_IOPAD(0x20d6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_hsync.dss_hsync */
-			OMAP3_CORE1_IOPAD(0x20d8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_vsync.dss_vsync */
-			OMAP3_CORE1_IOPAD(0x20da, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_acbias.dss_acbias */
-
-			OMAP3_CORE1_IOPAD(0x20e8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data6.dss_data6 */
-			OMAP3_CORE1_IOPAD(0x20ea, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data7.dss_data7 */
-			OMAP3_CORE1_IOPAD(0x20ec, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data8.dss_data8 */
-			OMAP3_CORE1_IOPAD(0x20ee, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data9.dss_data9 */
-			OMAP3_CORE1_IOPAD(0x20f0, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data10.dss_data10 */
-			OMAP3_CORE1_IOPAD(0x20f2, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data11.dss_data11 */
-			OMAP3_CORE1_IOPAD(0x20f4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data12.dss_data12 */
-			OMAP3_CORE1_IOPAD(0x20f6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data13.dss_data13 */
-			OMAP3_CORE1_IOPAD(0x20f8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data14.dss_data14 */
-			OMAP3_CORE1_IOPAD(0x20fa, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data15.dss_data15 */
-			OMAP3_CORE1_IOPAD(0x20fc, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data16.dss_data16 */
-			OMAP3_CORE1_IOPAD(0x20fe, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data17.dss_data17 */
-
-			OMAP3_CORE1_IOPAD(0x2100, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data18.dss_data0 */
-			OMAP3_CORE1_IOPAD(0x2102, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data19.dss_data1 */
-			OMAP3_CORE1_IOPAD(0x2104, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data20.dss_data2 */
-			OMAP3_CORE1_IOPAD(0x2106, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data21.dss_data3 */
-			OMAP3_CORE1_IOPAD(0x2108, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data22.dss_data4 */
-			OMAP3_CORE1_IOPAD(0x210a, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data23.dss_data5 */
+			OMAP3_CORE1_IOPAD(0x2164, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat4.sdmmc3_dat0 */
+			OMAP3_CORE1_IOPAD(0x2166, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat5.sdmmc3_dat1 */
+			OMAP3_CORE1_IOPAD(0x2168, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat6.sdmmc3_dat2 */
+			OMAP3_CORE1_IOPAD(0x216a, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat6.sdmmc3_dat3 */
+			OMAP3_CORE1_IOPAD(0x2184, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcbsp4_clkx.gpio_152 */
+			OMAP3_CORE1_IOPAD(0x218e, PIN_OUTPUT | MUX_MODE4)	/* mcbsp1_fsr.gpio_157 */
 		>;
 	};
 };
 
-&omap3_pmx_wkup {
-	gpio_key_pins_wkup: pinmux_gpio_key_pins_wkup {
-		pinctrl-single,pins = <
-			OMAP3_WKUP_IOPAD(0x2a0a, PIN_INPUT_PULLUP | MUX_MODE4)	/* sys_boot0.gpio_2 */
-			OMAP3_WKUP_IOPAD(0x2a14, PIN_INPUT_PULLUP | MUX_MODE4)	/* sys_boot5.gpio_7 */
-		>;
-	};
-
-	lan9221_pins: pinmux_lan9221_pins {
-		pinctrl-single,pins = <
-			OMAP3_WKUP_IOPAD(0x2a5a, PIN_INPUT | MUX_MODE4)		/* reserved.gpio_129 */
-		>;
-	};
-
-	mmc1_cd: pinmux_mmc1_cd {
+&omap3_pmx_core2 {
+	mmc3_core2_pins: pinmux_mmc3_core2_pins {
 		pinctrl-single,pins = <
-			OMAP3_WKUP_IOPAD(0x2a54, PIN_INPUT_PULLUP | MUX_MODE4)	/* reserved.gpio_127 */
+			OMAP3630_CORE2_IOPAD(0x25d8, PIN_INPUT_PULLUP | MUX_MODE2)   /* etk_clk.sdmmc3_clk */
+			OMAP3630_CORE2_IOPAD(0x25da, PIN_INPUT_PULLUP | MUX_MODE2)   /* etk_ctl.sdmmc3_cmd */
 		>;
 	};
 };
-
-&i2c2 {
-	mt9p031@48 {
-		compatible = "aptina,mt9p031";
-		reg = <0x48>;
-		clocks = <&isp 0>;
-		vaa-supply = <&vaux4>;
-		vdd-supply = <&vaux4>;
-		vdd_io-supply = <&vaux4>;
-		port {
-			mt9p031_out: endpoint {
-				input-clock-frequency = <24000000>;
-				pixel-clock-frequency = <72000000>;
-				remote-endpoint = <&ccdc_ep>;
-			};
-		};
-	};
-};
-
-&i2c3 {
-	touchscreen: tsc2004@48 {
-		compatible = "ti,tsc2004";
-		reg = <0x48>;
-		vio-supply = <&vaux1>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&tsc2004_pins>;
-		interrupts-extended = <&gpio5 25 IRQ_TYPE_EDGE_RISING>; /* gpio 153 */
-
-		touchscreen-fuzz-x = <4>;
-		touchscreen-fuzz-y = <7>;
-		touchscreen-fuzz-pressure = <2>;
-		touchscreen-size-x = <4096>;
-		touchscreen-size-y = <4096>;
-		touchscreen-max-pressure = <2048>;
-
-		ti,x-plate-ohms = <280>;
-		ti,esd-recovery-timeout-ms = <8000>;
-	};
-};
-
-&mcspi1 {
-	at25@0 {
-		compatible = "atmel,at25";
-		reg = <0>;
-		spi-max-frequency = <5000000>;
-		spi-cpha;
-		spi-cpol;
-
-		pagesize = <64>;
-		size = <32768>;
-		address-width = <16>;
-	};
-};
-
-&isp {
-	pinctrl-names = "default";
-	pinctrl-0 = <&isp_pins>;
-	ports {
-		port@0 {
-			reg = <0>;
-			ccdc_ep: endpoint {
-				remote-endpoint = <&mt9p031_out>;
-				bus-width = <8>;
-				hsync-active = <1>;
-				vsync-active = <1>;
-				pclk-sample = <0>;
-			};
-		};
-	};
-};
-
-&uart1 {
-	interrupts-extended = <&intc 72 &omap3_pmx_core OMAP3_UART1_RX>;
-};
-
-/* Wired to the tps65950 on the SOM, only the USB connector is on the devkit */
-&usb_otg_hs {
-	pinctrl-names = "default";
-	pinctrl-0 = <&hsusb_otg_pins>;
-	interface-type = <0>;
-	usb-phy = <&usb2_phy>;
-	phys = <&usb2_phy>;
-	phy-names = "usb2-phy";
-	mode = <3>;
-	power = <50>;
-};
diff --git a/arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi b/arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi
new file mode 100644
index 000000000000..86c5644f558c
--- /dev/null
+++ b/arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi
@@ -0,0 +1,395 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/ {
+	gpio_keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_key_pins &gpio_key_pins_wkup>;
+
+		sysboot2 {
+			label = "sysboot2";
+			gpios = <&gpio1 2 GPIO_ACTIVE_LOW>;	/* gpio2 */
+			linux,code = <BTN_0>;
+			wakeup-source;
+		};
+
+		sysboot5 {
+			label = "sysboot5";
+			gpios = <&gpio1 7 GPIO_ACTIVE_LOW>;	/* gpio7 */
+			linux,code = <BTN_1>;
+			wakeup-source;
+		};
+
+		gpio1 {
+			label = "gpio1";
+			gpios = <&gpio6 21 GPIO_ACTIVE_LOW>;	/* gpio181 */
+			linux,code = <BTN_2>;
+			wakeup-source;
+		};
+
+		gpio2 {
+			label = "gpio2";
+			gpios = <&gpio6 18 GPIO_ACTIVE_LOW>;	/* gpio178 */
+			linux,code = <BTN_3>;
+			wakeup-source;
+		};
+	};
+
+	sound {
+		compatible = "ti,omap-twl4030";
+		ti,model = "omap3logic";
+		ti,mcbsp = <&mcbsp2>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&led_pins>;
+
+		led1 {
+			label = "led1";
+			gpios = <&gpio6 20 GPIO_ACTIVE_HIGH>;	/* gpio180 */
+			linux,default-trigger = "cpu0";
+		};
+
+		led2 {
+			label = "led2";
+			gpios = <&gpio6 19 GPIO_ACTIVE_HIGH>;	/* gpio179 */
+			linux,default-trigger = "none";
+		};
+	};
+
+	pwm10: dmtimer-pwm {
+		compatible = "ti,omap-dmtimer-pwm";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pwm_pins>;
+		ti,timers = <&timer10>;
+		#pwm-cells = <3>;
+	};
+
+};
+
+&vaux1 {
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+};
+
+&vaux4 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+};
+
+&mcbsp2 {
+	status = "okay";
+};
+
+&charger {
+	ti,bb-uvolt = <3200000>;
+	ti,bb-uamp = <150>;
+};
+
+&gpmc {
+	ranges = <0 0 0x30000000 0x1000000	/* CS0: 16MB for NAND */
+		  1 0 0x2c000000 0x1000000>;	/* CS1: 16MB for LAN9221 */
+
+	ethernet@gpmc {
+		pinctrl-names = "default";
+		pinctrl-0 = <&lan9221_pins>;
+		interrupt-parent = <&gpio5>;
+		interrupts = <1 IRQ_TYPE_LEVEL_LOW>;		/* gpio129 */
+		reg = <1 0 0xff>;
+	};
+};
+
+&vpll2 {
+	regulator-always-on;
+};
+
+&dss {
+	status = "ok";
+	vdds_dsi-supply = <&vpll2>;
+	vdda_video-supply = <&video_reg>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_dpi_pins1>;
+	port {
+		dpi_out: endpoint {
+			remote-endpoint = <&lcd_in>;
+			data-lines = <16>;
+		};
+	};
+};
+
+/ {
+	aliases {
+		display0 = &lcd0;
+	};
+
+	video_reg: video_reg {
+		pinctrl-names = "default";
+		pinctrl-0 = <&panel_pwr_pins>;
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio5 27 GPIO_ACTIVE_HIGH>;	/* gpio155, lcd INI */
+	};
+
+	lcd0: display {
+		compatible = "panel-dpi";
+		label = "15";
+		status = "okay";
+		/* default-on; */
+		pinctrl-names = "default";
+
+		port {
+			lcd_in: endpoint {
+				remote-endpoint = <&dpi_out>;
+			};
+		};
+
+		panel-timing {
+			clock-frequency = <9000000>;
+			hactive = <480>;
+			vactive = <272>;
+			hfront-porch = <3>;
+			hback-porch = <2>;
+			hsync-len = <42>;
+			vback-porch = <3>;
+			vfront-porch = <4>;
+			vsync-len = <11>;
+			hsync-active = <0>;
+			vsync-active = <0>;
+			de-active = <1>;
+			pixelclk-active = <1>;
+		};
+	};
+
+	bl: backlight {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&backlight_pins>;
+		pwms = <&pwm10 0 5000000 0>;
+		brightness-levels = <0 10 20 30 40 50 60 70 80 90 100>;
+		default-brightness-level = <7>;
+		enable-gpios = <&gpio5 26 GPIO_ACTIVE_HIGH>; /* gpio_154 */
+	};
+};
+
+&mmc1 {
+	interrupts-extended = <&intc 83 &omap3_pmx_core 0x11a>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins &mmc1_cd>;
+	cd-gpios = <&gpio4 31 GPIO_ACTIVE_LOW>;		/* gpio127 */
+	vmmc-supply = <&vmmc1>;
+	bus-width = <4>;
+	cap-power-off-card;
+};
+
+&omap3_pmx_core {
+	gpio_key_pins: pinmux_gpio_key_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x21d6, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcspi2_clk.gpio_178 */
+			OMAP3_CORE1_IOPAD(0x21dc, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcspi2_cs0.gpio_181 */
+		>;
+	};
+
+	pwm_pins: pinmux_pwm_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x20B8, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE3)       /* gpmc_ncs5.gpt_10_pwm_evt */
+		>;
+	};
+
+	led_pins: pinmux_led_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x21d8, PIN_OUTPUT | MUX_MODE4)	/* gpio_179 */
+			OMAP3_CORE1_IOPAD(0x21da, PIN_OUTPUT | MUX_MODE4)	/* gpio_180 */
+		>;
+	};
+
+	mmc1_pins: pinmux_mmc1_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x2144, PIN_OUTPUT | MUX_MODE0)	/* sdmmc1_clk.sdmmc1_clk */
+			OMAP3_CORE1_IOPAD(0x2146, PIN_INPUT | MUX_MODE0)	/* sdmmc1_cmd.sdmmc1_cmd */
+			OMAP3_CORE1_IOPAD(0x2148, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat0.sdmmc1_dat0 */
+			OMAP3_CORE1_IOPAD(0x214a, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat1.sdmmc1_dat1 */
+			OMAP3_CORE1_IOPAD(0x214c, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat2.sdmmc1_dat2 */
+			OMAP3_CORE1_IOPAD(0x214e, PIN_INPUT | MUX_MODE0)	/* sdmmc1_dat3.sdmmc1_dat3 */
+		>;
+	};
+
+	tsc2004_pins: pinmux_tsc2004_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x2186, PIN_INPUT | MUX_MODE4)	/* mcbsp4_dr.gpio_153 */
+		>;
+	};
+
+	backlight_pins: pinmux_backlight_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x2188, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* mcbsp4_dx.gpio_154 */
+		>;
+	};
+
+	isp_pins: pinmux_isp_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x210c, PIN_INPUT | MUX_MODE0)   /* cam_hs.cam_hs */
+			OMAP3_CORE1_IOPAD(0x210e, PIN_INPUT | MUX_MODE0)   /* cam_vs.cam_vs */
+			OMAP3_CORE1_IOPAD(0x2110, PIN_INPUT | MUX_MODE0)   /* cam_xclka.cam_xclka */
+			OMAP3_CORE1_IOPAD(0x2112, PIN_INPUT | MUX_MODE0)   /* cam_pclk.cam_pclk */
+
+			OMAP3_CORE1_IOPAD(0x2116, PIN_INPUT | MUX_MODE0)   /* cam_d0.cam_d0 */
+			OMAP3_CORE1_IOPAD(0x2118, PIN_INPUT | MUX_MODE0)   /* cam_d1.cam_d1 */
+			OMAP3_CORE1_IOPAD(0x211a, PIN_INPUT | MUX_MODE0)   /* cam_d2.cam_d2 */
+			OMAP3_CORE1_IOPAD(0x211c, PIN_INPUT | MUX_MODE0)   /* cam_d3.cam_d3 */
+			OMAP3_CORE1_IOPAD(0x211e, PIN_INPUT | MUX_MODE0)   /* cam_d4.cam_d4 */
+			OMAP3_CORE1_IOPAD(0x2120, PIN_INPUT | MUX_MODE0)   /* cam_d5.cam_d5 */
+			OMAP3_CORE1_IOPAD(0x2122, PIN_INPUT | MUX_MODE0)   /* cam_d6.cam_d6 */
+			OMAP3_CORE1_IOPAD(0x2124, PIN_INPUT | MUX_MODE0)   /* cam_d7.cam_d7 */
+		>;
+	};
+
+	panel_pwr_pins: pinmux_panel_pwr_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x218a, PIN_OUTPUT | PIN_OFF_OUTPUT_LOW | MUX_MODE4)       /* mcbsp4_fs.gpio_155 */
+		>;
+	};
+
+	dss_dpi_pins1: pinmux_dss_dpi_pins1 {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x20d4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_pclk.dss_pclk */
+			OMAP3_CORE1_IOPAD(0x20d6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_hsync.dss_hsync */
+			OMAP3_CORE1_IOPAD(0x20d8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_vsync.dss_vsync */
+			OMAP3_CORE1_IOPAD(0x20da, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_acbias.dss_acbias */
+
+			OMAP3_CORE1_IOPAD(0x20e8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data6.dss_data6 */
+			OMAP3_CORE1_IOPAD(0x20ea, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data7.dss_data7 */
+			OMAP3_CORE1_IOPAD(0x20ec, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data8.dss_data8 */
+			OMAP3_CORE1_IOPAD(0x20ee, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data9.dss_data9 */
+			OMAP3_CORE1_IOPAD(0x20f0, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data10.dss_data10 */
+			OMAP3_CORE1_IOPAD(0x20f2, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data11.dss_data11 */
+			OMAP3_CORE1_IOPAD(0x20f4, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data12.dss_data12 */
+			OMAP3_CORE1_IOPAD(0x20f6, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data13.dss_data13 */
+			OMAP3_CORE1_IOPAD(0x20f8, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data14.dss_data14 */
+			OMAP3_CORE1_IOPAD(0x20fa, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data15.dss_data15 */
+			OMAP3_CORE1_IOPAD(0x20fc, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data16.dss_data16 */
+			OMAP3_CORE1_IOPAD(0x20fe, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE0)   /* dss_data17.dss_data17 */
+
+			OMAP3_CORE1_IOPAD(0x2100, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data18.dss_data0 */
+			OMAP3_CORE1_IOPAD(0x2102, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data19.dss_data1 */
+			OMAP3_CORE1_IOPAD(0x2104, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data20.dss_data2 */
+			OMAP3_CORE1_IOPAD(0x2106, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data21.dss_data3 */
+			OMAP3_CORE1_IOPAD(0x2108, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data22.dss_data4 */
+			OMAP3_CORE1_IOPAD(0x210a, PIN_OUTPUT_PULLDOWN | PIN_OFF_OUTPUT_LOW | MUX_MODE3)   /* dss_data23.dss_data5 */
+		>;
+	};
+};
+
+&omap3_pmx_wkup {
+	gpio_key_pins_wkup: pinmux_gpio_key_pins_wkup {
+		pinctrl-single,pins = <
+			OMAP3_WKUP_IOPAD(0x2a0a, PIN_INPUT_PULLUP | MUX_MODE4)	/* sys_boot0.gpio_2 */
+			OMAP3_WKUP_IOPAD(0x2a14, PIN_INPUT_PULLUP | MUX_MODE4)	/* sys_boot5.gpio_7 */
+		>;
+	};
+
+	lan9221_pins: pinmux_lan9221_pins {
+		pinctrl-single,pins = <
+			OMAP3_WKUP_IOPAD(0x2a5a, PIN_INPUT | MUX_MODE4)		/* reserved.gpio_129 */
+		>;
+	};
+
+	mmc1_cd: pinmux_mmc1_cd {
+		pinctrl-single,pins = <
+			OMAP3_WKUP_IOPAD(0x2a54, PIN_INPUT_PULLUP | MUX_MODE4)	/* reserved.gpio_127 */
+		>;
+	};
+};
+
+&i2c2 {
+	mt9p031@48 {
+		compatible = "aptina,mt9p031";
+		reg = <0x48>;
+		clocks = <&isp 0>;
+		vaa-supply = <&vaux4>;
+		vdd-supply = <&vaux4>;
+		vdd_io-supply = <&vaux4>;
+		port {
+			mt9p031_out: endpoint {
+				input-clock-frequency = <24000000>;
+				pixel-clock-frequency = <72000000>;
+				remote-endpoint = <&ccdc_ep>;
+			};
+		};
+	};
+};
+
+&i2c3 {
+	touchscreen: tsc2004@48 {
+		compatible = "ti,tsc2004";
+		reg = <0x48>;
+		vio-supply = <&vaux1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&tsc2004_pins>;
+		interrupts-extended = <&gpio5 25 IRQ_TYPE_EDGE_RISING>; /* gpio 153 */
+
+		touchscreen-fuzz-x = <4>;
+		touchscreen-fuzz-y = <7>;
+		touchscreen-fuzz-pressure = <2>;
+		touchscreen-size-x = <4096>;
+		touchscreen-size-y = <4096>;
+		touchscreen-max-pressure = <2048>;
+
+		ti,x-plate-ohms = <280>;
+		ti,esd-recovery-timeout-ms = <8000>;
+	};
+};
+
+&mcspi1 {
+	at25@0 {
+		compatible = "atmel,at25";
+		reg = <0>;
+		spi-max-frequency = <5000000>;
+		spi-cpha;
+		spi-cpol;
+
+		pagesize = <64>;
+		size = <32768>;
+		address-width = <16>;
+	};
+};
+
+&isp {
+	pinctrl-names = "default";
+	pinctrl-0 = <&isp_pins>;
+	ports {
+		port@0 {
+			reg = <0>;
+			ccdc_ep: endpoint {
+				remote-endpoint = <&mt9p031_out>;
+				bus-width = <8>;
+				hsync-active = <1>;
+				vsync-active = <1>;
+				pclk-sample = <0>;
+			};
+		};
+	};
+};
+
+&uart1 {
+	interrupts-extended = <&intc 72 &omap3_pmx_core OMAP3_UART1_RX>;
+};
+
+/* Wired to the tps65950 on the SOM, only the USB connector is on the devkit */
+&usb_otg_hs {
+	pinctrl-names = "default";
+	pinctrl-0 = <&hsusb_otg_pins>;
+	interface-type = <0>;
+	usb-phy = <&usb2_phy>;
+	phys = <&usb2_phy>;
+	phy-names = "usb2-phy";
+	mode = <3>;
+	power = <50>;
+};
diff --git a/arch/arm/boot/dts/logicpd-torpedo-som.dtsi b/arch/arm/boot/dts/logicpd-torpedo-som.dtsi
index 6d89736c7b44..47915447a826 100644
--- a/arch/arm/boot/dts/logicpd-torpedo-som.dtsi
+++ b/arch/arm/boot/dts/logicpd-torpedo-som.dtsi
@@ -30,17 +30,6 @@
 			linux,default-trigger = "none";
 		};
 	};
-
-	wl12xx_vmmc: wl12xx_vmmc {
-		compatible = "regulator-fixed";
-		regulator-name = "vwl1271";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		gpio = <&gpio5 29 0>;   /* gpio157 */
-		startup-delay-us = <70000>;
-		enable-active-high;
-		vin-supply = <&vmmc2>;
-	};
 };
 
 &gpmc {
@@ -73,37 +62,12 @@
 		gpmc,device-width = <2>;
 		#address-cells = <1>;
 		#size-cells = <1>;
-
-		/* u-boot uses mtdparts=omap2-nand.0:512k(x-loader),1920k(u-boot),128k(u-boot-env),4m(kernel),-(fs) */
-
-		x-loader@0 {
-			label = "x-loader";
-			reg = <0 0x80000>;
-		};
-
-		bootloaders@80000 {
-			label = "u-boot";
-			reg = <0x80000 0x1e0000>;
-		};
-
-		bootloaders_env@260000 {
-			label = "u-boot-env";
-			reg = <0x260000 0x20000>;
-		};
-
-		kernel@280000 {
-			label = "kernel";
-			reg = <0x280000 0x400000>;
-		};
-
-		filesystem@680000 {
-			label = "fs";
-			reg = <0x680000 0>;	/* 0 = MTDPART_SIZ_FULL */
-		};
 	};
 };
 
 &i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
 	clock-frequency = <2600000>;
 
 	twl: twl@48 {
@@ -131,44 +95,7 @@
 	};
 };
 
-/*
- * Only found on the wireless SOM. For the SOM without wireless, the pins for
- * MMC3 can be routed with jumpers to the second MMC slot on the devkit and
- * gpio157 is not connected. So this should be OK to keep common for now,
- * probably device tree overlays is the way to go with the various SOM and
- * jumpering combinations for the long run.
- */
-&mmc3 {
-	interrupts-extended = <&intc 94 &omap3_pmx_core2 0x46>;
-	pinctrl-0 = <&mmc3_pins &mmc3_core2_pins>;
-	pinctrl-names = "default";
-	vmmc-supply = <&wl12xx_vmmc>;
-	non-removable;
-	bus-width = <4>;
-	cap-power-off-card;
-	#address-cells = <1>;
-	#size-cells = <0>;
-	wlcore: wlcore@2 {
-		compatible = "ti,wl1283";
-		reg = <2>;
-		interrupt-parent = <&gpio5>;
-		interrupts = <24 IRQ_TYPE_LEVEL_HIGH>; /* gpio 152 */
-		ref-clock-frequency = <26000000>;
-		tcxo-clock-frequency = <26000000>;
-	};
-};
-
 &omap3_pmx_core {
-	mmc3_pins: pinmux_mm3_pins {
-		pinctrl-single,pins = <
-			OMAP3_CORE1_IOPAD(0x2164, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat4.sdmmc3_dat0 */
-			OMAP3_CORE1_IOPAD(0x2166, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat5.sdmmc3_dat1 */
-			OMAP3_CORE1_IOPAD(0x2168, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat6.sdmmc3_dat2 */
-			OMAP3_CORE1_IOPAD(0x216a, PIN_INPUT_PULLUP | MUX_MODE3)	/* sdmmc2_dat6.sdmmc3_dat3 */
-			OMAP3_CORE1_IOPAD(0x2184, PIN_INPUT_PULLUP | MUX_MODE4)	/* mcbsp4_clkx.gpio_152 */
-			OMAP3_CORE1_IOPAD(0x218e, PIN_OUTPUT | MUX_MODE4)	/* mcbsp1_fsr.gpio_157 */
-		>;
-	};
 	mcbsp2_pins: pinmux_mcbsp2_pins {
 		pinctrl-single,pins = <
 			OMAP3_CORE1_IOPAD(0x213c, PIN_INPUT | MUX_MODE0)        /* mcbsp2_fsx */
@@ -211,6 +138,12 @@
 			OMAP3_CORE1_IOPAD(0x21b8, PIN_INPUT | MUX_MODE0)	/* hsusb0_data7.hsusb0_data7 */
 		>;
 	};
+	i2c1_pins: pinmux_i2c1_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x21ba, PIN_INPUT | MUX_MODE0)        /* i2c1_scl.i2c1_scl */
+			OMAP3_CORE1_IOPAD(0x21bc, PIN_INPUT | MUX_MODE0)        /* i2c1_sda.i2c1_sda */
+		>;
+	};
 };
 
 &uart2 {
@@ -224,15 +157,6 @@
 	pinctrl-0 = <&mcspi1_pins>;
 };
 
-&omap3_pmx_core2 {
-	mmc3_core2_pins: pinmux_mmc3_core2_pins {
-		pinctrl-single,pins = <
-			OMAP3630_CORE2_IOPAD(0x25d8, PIN_INPUT_PULLUP | MUX_MODE2)   /* etk_clk.sdmmc3_clk */
-			OMAP3630_CORE2_IOPAD(0x25da, PIN_INPUT_PULLUP | MUX_MODE2)   /* etk_ctl.sdmmc3_cmd */
-		>;
-	};
-};
-
 #include "twl4030.dtsi"
 #include "twl4030_omap3.dtsi"
 
diff --git a/arch/arm/boot/dts/lpc3250-ea3250.dts b/arch/arm/boot/dts/lpc3250-ea3250.dts
index c43adb7b4d7c..58ea0a4e7afa 100644
--- a/arch/arm/boot/dts/lpc3250-ea3250.dts
+++ b/arch/arm/boot/dts/lpc3250-ea3250.dts
@@ -156,8 +156,8 @@
 	uda1380: uda1380@18 {
 		compatible = "nxp,uda1380";
 		reg = <0x18>;
-		power-gpio = <&gpio 0x59 0>;
-		reset-gpio = <&gpio 0x51 0>;
+		power-gpio = <&gpio 3 10 0>;
+		reset-gpio = <&gpio 3 2 0>;
 		dac-clk = "wspll";
 	};
 
diff --git a/arch/arm/boot/dts/lpc3250-phy3250.dts b/arch/arm/boot/dts/lpc3250-phy3250.dts
index c72eb9845603..1e1c2f517a82 100644
--- a/arch/arm/boot/dts/lpc3250-phy3250.dts
+++ b/arch/arm/boot/dts/lpc3250-phy3250.dts
@@ -81,8 +81,8 @@
 	uda1380: uda1380@18 {
 		compatible = "nxp,uda1380";
 		reg = <0x18>;
-		power-gpio = <&gpio 0x59 0>;
-		reset-gpio = <&gpio 0x51 0>;
+		power-gpio = <&gpio 3 10 0>;
+		reset-gpio = <&gpio 3 2 0>;
 		dac-clk = "wspll";
 	};
 
diff --git a/arch/arm/boot/dts/lpc4337-ciaa.dts b/arch/arm/boot/dts/lpc4337-ciaa.dts
index 7c16d639a1b4..beddaba85393 100644
--- a/arch/arm/boot/dts/lpc4337-ciaa.dts
+++ b/arch/arm/boot/dts/lpc4337-ciaa.dts
@@ -174,17 +174,17 @@
 	clock-frequency = <400000>;
 
 	eeprom@50 {
-		compatible = "microchip,24c512";
+		compatible = "microchip,24c512", "atmel,24c512";
 		reg = <0x50>;
 	};
 
 	eeprom@51 {
-		compatible = "microchip,24c02";
+		compatible = "microchip,24c02", "atmel,24c02";
 		reg = <0x51>;
 	};
 
 	eeprom@54 {
-		compatible = "microchip,24c512";
+		compatible = "microchip,24c512", "atmel,24c512";
 		reg = <0x54>;
 	};
 };
diff --git a/arch/arm/boot/dts/lpc4350-hitex-eval.dts b/arch/arm/boot/dts/lpc4350-hitex-eval.dts
index 874c75d44013..8b973f537d3a 100644
--- a/arch/arm/boot/dts/lpc4350-hitex-eval.dts
+++ b/arch/arm/boot/dts/lpc4350-hitex-eval.dts
@@ -429,7 +429,7 @@
 	};
 
 	eeprom@50 {
-		compatible = "nxp,24c02";
+		compatible = "nxp,24c02", "atmel,24c02";
 		reg = <0x50>;
 	};
 
diff --git a/arch/arm/boot/dts/lpc4357-ea4357-devkit.dts b/arch/arm/boot/dts/lpc4357-ea4357-devkit.dts
index 9b5fad622522..02b23fa29d75 100644
--- a/arch/arm/boot/dts/lpc4357-ea4357-devkit.dts
+++ b/arch/arm/boot/dts/lpc4357-ea4357-devkit.dts
@@ -490,7 +490,7 @@
 	};
 
 	eeprom@57 {
-		compatible = "microchip,24c64";
+		compatible = "microchip,24c64", "atmel,24c64";
 		reg = <0x57>;
 	};
 };
diff --git a/arch/arm/boot/dts/ls1021a-moxa-uc-8410a.dts b/arch/arm/boot/dts/ls1021a-moxa-uc-8410a.dts
new file mode 100644
index 000000000000..d01f64b252b1
--- /dev/null
+++ b/arch/arm/boot/dts/ls1021a-moxa-uc-8410a.dts
@@ -0,0 +1,241 @@
+/*
+ * Copyright (C) 2017 Moxa Inc. - https://www.moxa.com/
+ *
+ * Author: Harry YJ Jhou (周亞諄) <harryyj.jhou@moxa.com>
+ *         Jimmy Chen (陳永達)    <jimmy.chen@moxa.com>
+ *         SZ Lin (林上智)        <sz.lin@moxa.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include "ls1021a.dtsi"
+
+/ {
+	model = "Moxa UC-8410A";
+
+	aliases {
+		enet0_rgmii_phy = &rgmii_phy0;
+		enet1_rgmii_phy = &rgmii_phy1;
+		enet2_rgmii_phy = &rgmii_phy2;
+	};
+
+	sys_mclk: clock-mclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24576000>;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		  compatible = "regulator-fixed";
+		  regulator-name = "3P3V";
+		  regulator-min-microvolt = <3300000>;
+		  regulator-max-microvolt = <3300000>;
+		  regulator-always-on;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		cel-pwr {
+			label = "UC8410A:CEL-PWR";
+			gpios = <&gpio3 27 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		cel-reset {
+			label = "UC8410A:CEL-RESET";
+			gpios = <&gpio3 28 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		str-led {
+			label = "UC8410A:RED:PROG";
+			gpios = <&gpio0 16 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "mmc0";
+		};
+
+		sw-ready {
+			label = "UC8410A:GREEN:SWRDY";
+			gpios = <&gpio0 18 GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+		};
+
+		beeper {
+			label = "UC8410A:BEEP";
+			gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		prog-led0 {
+			label = "UC8410A:GREEN:PROG2";
+			gpios = <&gpio3 14 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		prog-led1 {
+			label = "UC8410A:GREEN:PROG1";
+			gpios = <&gpio3 15 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		prog-led2 {
+			label = "UC8410A:GREEN:PROG0";
+			gpios = <&gpio3 16 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		wifi-signal0 {
+			label = "UC8410A:GREEN:CEL2";
+			gpios = <&gpio3 17 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		wifi-signal1 {
+			label = "UC8410A:GREEN:CEL1";
+			gpios = <&gpio3 18 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		wifi-signal2 {
+			label = "UC8410A:GREEN:CEL0";
+			gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		cpu-diag-red {
+			label = "UC8410A:RED:DIA";
+			gpios = <&gpio3 20 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		cpu-diag-green {
+			label = "UC8410A:GREEN:DIA";
+			gpios = <&gpio3 21 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		cpu-diag-yellow {
+			label = "UC8410A:YELLOW:DIA";
+			gpios = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pushbtn-key {
+			label = "push button key";
+			gpios = <&gpio1 21 GPIO_ACTIVE_LOW>;
+			linux,code = <BTN_MISC>;
+			default-state = "on";
+		};
+	};
+};
+
+&enet0 {
+	phy-handle = <&rgmii_phy0>;
+	phy-connection-type = "rgmii-id";
+	status = "okay";
+};
+
+&enet1 {
+	phy-handle = <&rgmii_phy1>;
+	phy-connection-type = "rgmii-id";
+	status = "okay";
+};
+
+&enet2 {
+	phy-handle = <&rgmii_phy2>;
+	phy-connection-type = "rgmii-id";
+	status = "okay";
+};
+
+&i2c0 {
+	clock-frequency = <100000>;
+	status = "okay";
+
+	tpm@20 {
+		compatible = "infineon,slb9635tt";
+		reg = <0x20>;
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1374";
+		reg = <0x68>;
+	};
+};
+
+&lpuart0 {
+	status = "okay";
+};
+
+&mdio0 {
+	rgmii_phy0: ethernet-phy@0 {
+		compatible = "marvell,88e1118";
+		reg = <0x0>;
+		marvell,reg-init =
+			<3 0x11 0 0x4415>, /* Reg 3,17 */
+			<3 0x10 0 0x77>; /* Reg 3,16 */
+	};
+
+	rgmii_phy1: ethernet-phy@1 {
+		compatible = "marvell,88e1118";
+		reg = <0x1>;
+		marvell,reg-init =
+			<3 0x11 0 0x4415>, /* Reg 3,17 */
+			<3 0x10 0 0x77>; /* Reg 3,16 */
+	};
+
+	rgmii_phy2: ethernet-phy@2 {
+		compatible = "marvell,88e1118";
+		reg = <0x2>;
+		marvell,reg-init =
+			<3 0x11 0 0x4415>, /* Reg 3,17 */
+			<3 0x10 0 0x77>; /* Reg 3,16 */
+	};
+};
+
+&qspi {
+	bus-num = <0>;
+	fsl,spi-num-chipselects = <2>;
+	fsl,spi-flash-chipselects = <0>;
+	fsl,qspi-has-second-chip;
+	status = "okay";
+
+	flash: flash@0 {
+		compatible = "spansion,s25fl064l", "spansion,s25fl164k";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <20000000>;
+		reg = <0>;
+
+		partitions@0 {
+			label = "U-Boot";
+			reg = <0x0 0x180000>;
+		};
+
+		partitions@180000 {
+			label = "U-Boot Env";
+			reg = <0x180000 0x680000>;
+		};
+	};
+};
+
+&sata {
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&uart1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/ls1021a-qds.dts b/arch/arm/boot/dts/ls1021a-qds.dts
index 67b4de0e3439..499f41a2c6f0 100644
--- a/arch/arm/boot/dts/ls1021a-qds.dts
+++ b/arch/arm/boot/dts/ls1021a-qds.dts
@@ -239,6 +239,11 @@
 		device-width = <1>;
 	};
 
+	nand@2,0 {
+		compatible = "fsl,ifc-nand";
+		reg = <0x2 0x0 0x10000>;
+	};
+
 	fpga: board-control@3,0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -331,3 +336,19 @@
 &uart1 {
 	status = "okay";
 };
+
+&can0 {
+	status = "okay";
+};
+
+&can1 {
+	status = "okay";
+};
+
+&can2 {
+	status = "disabled";
+};
+
+&can3 {
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/ls1021a-twr.dts b/arch/arm/boot/dts/ls1021a-twr.dts
index 44715c8ef756..f0c949d74833 100644
--- a/arch/arm/boot/dts/ls1021a-twr.dts
+++ b/arch/arm/boot/dts/ls1021a-twr.dts
@@ -228,6 +228,10 @@
 	};
 };
 
+&esdhc {
+        status = "okay";
+};
+
 &sai1 {
 	status = "okay";
 };
@@ -243,3 +247,19 @@
 &uart1 {
 	status = "okay";
 };
+
+&can0 {
+	status = "okay";
+};
+
+&can1 {
+	status = "okay";
+};
+
+&can2 {
+	status = "disabled";
+};
+
+&can3 {
+	status = "disabled";
+};
diff --git a/arch/arm/boot/dts/ls1021a.dtsi b/arch/arm/boot/dts/ls1021a.dtsi
index 9319e1f0f1d8..c31dad98f989 100644
--- a/arch/arm/boot/dts/ls1021a.dtsi
+++ b/arch/arm/boot/dts/ls1021a.dtsi
@@ -106,6 +106,14 @@
 		compatible = "arm,cortex-a7-pmu";
 		interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>;
+	};
+
+	reboot {
+		compatible = "syscon-reboot";
+		regmap = <&dcfg>;
+		offset = <0xb0>;
+		mask = <0x02>;
 	};
 
 	soc {
@@ -154,8 +162,22 @@
 			big-endian;
 		};
 
+		qspi: quadspi@1550000 {
+			compatible = "fsl,ls1021a-qspi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0 0x1550000 0x0 0x10000>,
+			      <0x0 0x40000000 0x0 0x40000000>;
+			reg-names = "QuadSPI", "QuadSPI-memory";
+			interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "qspi_en", "qspi";
+			clocks = <&clockgen 4 1>, <&clockgen 4 1>;
+			big-endian;
+			status = "disabled";
+		};
+
 		esdhc: esdhc@1560000 {
-			compatible = "fsl,esdhc";
+			compatible = "fsl,ls1021a-esdhc", "fsl,esdhc";
 			reg = <0x0 0x1560000 0x0 0x10000>;
 			interrupts = <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
 			clock-frequency = <0>;
@@ -575,7 +597,7 @@
 			fsl,tclk-period = <5>;
 			fsl,tmr-prsc    = <2>;
 			fsl,tmr-add     = <0xaaaaaaab>;
-			fsl,tmr-fiper1  = <999999990>;
+			fsl,tmr-fiper1  = <999999995>;
 			fsl,tmr-fiper2  = <99990>;
 			fsl,max-adj     = <499999999>;
 		};
@@ -668,7 +690,7 @@
 			};
 		};
 
-		usb@8600000 {
+		usb2: usb@8600000 {
 			compatible = "fsl-usb2-dr-v2.5", "fsl-usb2-dr";
 			reg = <0x0 0x8600000 0x0 0x1000>;
 			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>;
@@ -676,7 +698,7 @@
 			phy_type = "ulpi";
 		};
 
-		usb3@3100000 {
+		usb3: usb3@3100000 {
 			compatible = "snps,dwc3";
 			reg = <0x0 0x3100000 0x0 0x10000>;
 			interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>;
@@ -730,5 +752,41 @@
 					<0000 0 0 3 &gic GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>,
 					<0000 0 0 4 &gic GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>;
 		};
+
+		can0: can@2a70000 {
+			compatible = "fsl,ls1021ar2-flexcan";
+			reg = <0x0 0x2a70000 0x0 0x1000>;
+			interrupts = <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>, <&clockgen 4 1>;
+			clock-names = "ipg", "per";
+			big-endian;
+		};
+
+		can1: can@2a80000 {
+			compatible = "fsl,ls1021ar2-flexcan";
+			reg = <0x0 0x2a80000 0x0 0x1000>;
+			interrupts = <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>, <&clockgen 4 1>;
+			clock-names = "ipg", "per";
+			big-endian;
+		};
+
+		can2: can@2a90000 {
+			compatible = "fsl,ls1021ar2-flexcan";
+			reg = <0x0 0x2a90000 0x0 0x1000>;
+			interrupts = <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>, <&clockgen 4 1>;
+			clock-names = "ipg", "per";
+			big-endian;
+		};
+
+		can3: can@2aa0000 {
+			compatible = "fsl,ls1021ar2-flexcan";
+			reg = <0x0 0x2aa0000 0x0 0x1000>;
+			interrupts = <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&clockgen 4 1>, <&clockgen 4 1>;
+			clock-names = "ipg", "per";
+			big-endian;
+		};
 	};
 };
diff --git a/arch/arm/boot/dts/meson8.dtsi b/arch/arm/boot/dts/meson8.dtsi
index 2d7a0752a460..d2e3eeaa1a5f 100644
--- a/arch/arm/boot/dts/meson8.dtsi
+++ b/arch/arm/boot/dts/meson8.dtsi
@@ -286,6 +286,11 @@
 	clock-names = "stmmaceth";
 };
 
+&gpio_intc {
+	compatible = "amlogic,meson8-gpio-intc", "amlogic,meson-gpio-intc";
+	status = "okay";
+};
+
 &hwrng {
 	compatible = "amlogic,meson8-rng", "amlogic,meson-rng";
 	clocks = <&clkc CLKID_RNG0>;
@@ -308,6 +313,9 @@
 	arm,data-latency = <3 3 3>;
 	arm,tag-latency = <2 2 2>;
 	arm,filter-ranges = <0x100000 0xc0000000>;
+	prefetch-data = <1>;
+	prefetch-instr = <1>;
+	arm,shared-override;
 };
 
 &pwm_ab {
@@ -321,9 +329,8 @@
 &saradc {
 	compatible = "amlogic,meson8-saradc", "amlogic,meson-saradc";
 	clocks = <&clkc CLKID_XTAL>,
-		<&clkc CLKID_SAR_ADC>,
-		<&clkc CLKID_SANA>;
-	clock-names = "clkin", "core", "sana";
+		<&clkc CLKID_SAR_ADC>;
+	clock-names = "clkin", "core";
 };
 
 &sdio {
@@ -337,19 +344,27 @@
 };
 
 &uart_AO {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_CLK81>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &uart_A {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_UART0>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &uart_B {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_UART1>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &uart_C {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_UART2>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &usb0 {
diff --git a/arch/arm/boot/dts/meson8b.dtsi b/arch/arm/boot/dts/meson8b.dtsi
index d75e0ceda8bb..7cd03ed3742e 100644
--- a/arch/arm/boot/dts/meson8b.dtsi
+++ b/arch/arm/boot/dts/meson8b.dtsi
@@ -223,6 +223,9 @@
 	arm,data-latency = <3 3 3>;
 	arm,tag-latency = <2 2 2>;
 	arm,filter-ranges = <0x100000 0xc0000000>;
+	prefetch-data = <1>;
+	prefetch-instr = <1>;
+	arm,shared-override;
 };
 
 &pwm_ab {
@@ -236,9 +239,8 @@
 &saradc {
 	compatible = "amlogic,meson8b-saradc", "amlogic,meson-saradc";
 	clocks = <&clkc CLKID_XTAL>,
-		<&clkc CLKID_SAR_ADC>,
-		<&clkc CLKID_SANA>;
-	clock-names = "clkin", "core", "sana";
+		<&clkc CLKID_SAR_ADC>;
+	clock-names = "clkin", "core";
 };
 
 &sdio {
@@ -248,19 +250,27 @@
 };
 
 &uart_AO {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_CLK81>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &uart_A {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_UART0>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &uart_B {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_UART1>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &uart_C {
-	clocks = <&clkc CLKID_CLK81>;
+	compatible = "amlogic,meson8b-uart", "amlogic,meson-uart";
+	clocks = <&clkc CLKID_CLK81>, <&clkc CLKID_XTAL>, <&clkc CLKID_UART2>;
+	clock-names = "baud", "xtal", "pclk";
 };
 
 &usb0 {
diff --git a/arch/arm/boot/dts/mpa1600.dts b/arch/arm/boot/dts/mpa1600.dts
index 36cfa215620d..43aaa67fcd87 100644
--- a/arch/arm/boot/dts/mpa1600.dts
+++ b/arch/arm/boot/dts/mpa1600.dts
@@ -32,6 +32,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			macb0: ethernet@fffbc000 {
 				phy-mode = "rmii";
 				status = "okay";
diff --git a/arch/arm/boot/dts/mt2701.dtsi b/arch/arm/boot/dts/mt2701.dtsi
index 965ddfbc9953..05557fce0f1d 100644
--- a/arch/arm/boot/dts/mt2701.dtsi
+++ b/arch/arm/boot/dts/mt2701.dtsi
@@ -604,6 +604,7 @@
 		compatible = "mediatek,mt2701-hifsys", "syscon";
 		reg = <0 0x1a000000 0 0x1000>;
 		#clock-cells = <1>;
+		#reset-cells = <1>;
 	};
 
 	usb0: usb@1a1c0000 {
@@ -688,6 +689,7 @@
 		compatible = "mediatek,mt2701-ethsys", "syscon";
 		reg = <0 0x1b000000 0 0x1000>;
 		#clock-cells = <1>;
+		#reset-cells = <1>;
 	};
 
 	eth: ethernet@1b100000 {
diff --git a/arch/arm/boot/dts/mt7623.dtsi b/arch/arm/boot/dts/mt7623.dtsi
index 0640fb75bf59..b750da5362f7 100644
--- a/arch/arm/boot/dts/mt7623.dtsi
+++ b/arch/arm/boot/dts/mt7623.dtsi
@@ -641,7 +641,7 @@
 
 	mmc0: mmc@11230000 {
 		compatible = "mediatek,mt7623-mmc",
-			     "mediatek,mt8135-mmc";
+			     "mediatek,mt2701-mmc";
 		reg = <0 0x11230000 0 0x1000>;
 		interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_LOW>;
 		clocks = <&pericfg CLK_PERI_MSDC30_0>,
@@ -652,7 +652,7 @@
 
 	mmc1: mmc@11240000 {
 		compatible = "mediatek,mt7623-mmc",
-			     "mediatek,mt8135-mmc";
+			     "mediatek,mt2701-mmc";
 		reg = <0 0x11240000 0 0x1000>;
 		interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_LOW>;
 		clocks = <&pericfg CLK_PERI_MSDC30_1>,
@@ -758,6 +758,7 @@
 			     "syscon";
 		reg = <0 0x1b000000 0 0x1000>;
 		#clock-cells = <1>;
+		#reset-cells = <1>;
 	};
 
 	eth: ethernet@1b100000 {
diff --git a/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts b/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts
index 688a86378cee..7bf5aa2237c9 100644
--- a/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts
+++ b/arch/arm/boot/dts/mt7623n-bananapi-bpi-r2.dts
@@ -204,7 +204,7 @@
 	bus-width = <4>;
 	max-frequency = <50000000>;
 	cap-sd-highspeed;
-	cd-gpios = <&pio 261 0>;
+	cd-gpios = <&pio 261 GPIO_ACTIVE_LOW>;
 	vmmc-supply = <&mt6323_vmch_reg>;
 	vqmmc-supply = <&mt6323_vio18_reg>;
 };
diff --git a/arch/arm/boot/dts/mt7623n-rfb-nand.dts b/arch/arm/boot/dts/mt7623n-rfb-nand.dts
index 17c578f0d261..e66de8611650 100644
--- a/arch/arm/boot/dts/mt7623n-rfb-nand.dts
+++ b/arch/arm/boot/dts/mt7623n-rfb-nand.dts
@@ -51,7 +51,7 @@
 				reg = <0x40000 0x80000>;
 			};
 
-			partition@C0000 {
+			partition@c0000 {
 				label = "uboot-env";
 				reg = <0xC0000 0x40000>;
 			};
diff --git a/arch/arm/boot/dts/omap2420-n8x0-common.dtsi b/arch/arm/boot/dts/omap2420-n8x0-common.dtsi
index 1df3ace3af92..63b0b4921e4e 100644
--- a/arch/arm/boot/dts/omap2420-n8x0-common.dtsi
+++ b/arch/arm/boot/dts/omap2420-n8x0-common.dtsi
@@ -52,6 +52,7 @@
 	onenand@0,0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
+		compatible = "ti,omap2-onenand";
 		reg = <0 0 0x20000>;	/* CS0, offset 0, IO size 128K */
 
 		gpmc,sync-read;
diff --git a/arch/arm/boot/dts/omap3-cm-t3x.dtsi b/arch/arm/boot/dts/omap3-cm-t3x.dtsi
index ab6003fe5a43..9dcb18d22cde 100644
--- a/arch/arm/boot/dts/omap3-cm-t3x.dtsi
+++ b/arch/arm/boot/dts/omap3-cm-t3x.dtsi
@@ -306,19 +306,19 @@
 			label = "xloader";
 			reg = <0 0x80000>;
 		};
-		partition@0x80000 {
+		partition@80000 {
 			label = "uboot";
 			reg = <0x80000 0x1e0000>;
 		};
-		partition@0x260000 {
+		partition@260000 {
 			label = "uboot environment";
 			reg = <0x260000 0x40000>;
 		};
-		partition@0x2a0000 {
+		partition@2a0000 {
 			label = "linux";
 			reg = <0x2a0000 0x400000>;
 		};
-		partition@0x6a0000 {
+		partition@6a0000 {
 			label = "rootfs";
 			reg = <0x6a0000 0x1f880000>;
 		};
diff --git a/arch/arm/boot/dts/omap3-evm-37xx.dts b/arch/arm/boot/dts/omap3-evm-37xx.dts
index 5a4ba0aea447..a14303b09ae2 100644
--- a/arch/arm/boot/dts/omap3-evm-37xx.dts
+++ b/arch/arm/boot/dts/omap3-evm-37xx.dts
@@ -90,19 +90,19 @@
 			label = "X-Loader";
 			reg = <0 0x80000>;
 		};
-		partition@0x80000 {
+		partition@80000 {
 			label = "U-Boot";
 			reg = <0x80000 0x1c0000>;
 		};
-		partition@0x1c0000 {
+		partition@1c0000 {
 			label = "Environment";
 			reg = <0x240000 0x40000>;
 		};
-		partition@0x280000 {
+		partition@280000 {
 			label = "Kernel";
 			reg = <0x280000 0x500000>;
 		};
-		partition@0x780000 {
+		partition@780000 {
 			label = "Filesystem";
 			reg = <0x780000 0x1f880000>;
 		};
diff --git a/arch/arm/boot/dts/omap3-gta04.dtsi b/arch/arm/boot/dts/omap3-gta04.dtsi
index 3dc56fb156b7..4170be70460e 100644
--- a/arch/arm/boot/dts/omap3-gta04.dtsi
+++ b/arch/arm/boot/dts/omap3-gta04.dtsi
@@ -86,7 +86,7 @@
 
 		/* lcd panel */
 		lcd: td028ttec1@0 {
-			compatible = "toppoly,td028ttec1";
+			compatible = "tpo,td028ttec1";
 			reg = <0>;
 			spi-max-frequency = <100000>;
 			spi-cpol;
diff --git a/arch/arm/boot/dts/omap3-igep.dtsi b/arch/arm/boot/dts/omap3-igep.dtsi
index 4ad7d5565906..f33cc80c9dbc 100644
--- a/arch/arm/boot/dts/omap3-igep.dtsi
+++ b/arch/arm/boot/dts/omap3-igep.dtsi
@@ -147,32 +147,32 @@
 		gpmc,sync-read;
 		gpmc,sync-write;
 		gpmc,burst-length = <16>;
-		gpmc,burst-read;
 		gpmc,burst-wrap;
+		gpmc,burst-read;
 		gpmc,burst-write;
 		gpmc,device-width = <2>; /* GPMC_DEVWIDTH_16BIT */
 		gpmc,mux-add-data = <2>; /* GPMC_MUX_AD */
 		gpmc,cs-on-ns = <0>;
-		gpmc,cs-rd-off-ns = <87>;
-		gpmc,cs-wr-off-ns = <87>;
+		gpmc,cs-rd-off-ns = <96>;
+		gpmc,cs-wr-off-ns = <96>;
 		gpmc,adv-on-ns = <0>;
-		gpmc,adv-rd-off-ns = <10>;
-		gpmc,adv-wr-off-ns = <10>;
-		gpmc,oe-on-ns = <15>;
-		gpmc,oe-off-ns = <87>;
+		gpmc,adv-rd-off-ns = <12>;
+		gpmc,adv-wr-off-ns = <12>;
+		gpmc,oe-on-ns = <18>;
+		gpmc,oe-off-ns = <96>;
 		gpmc,we-on-ns = <0>;
-		gpmc,we-off-ns = <87>;
-		gpmc,rd-cycle-ns = <112>;
-		gpmc,wr-cycle-ns = <112>;
-		gpmc,access-ns = <81>;
-		gpmc,page-burst-access-ns = <15>;
+		gpmc,we-off-ns = <96>;
+		gpmc,rd-cycle-ns = <114>;
+		gpmc,wr-cycle-ns = <114>;
+		gpmc,access-ns = <90>;
+		gpmc,page-burst-access-ns = <12>;
 		gpmc,bus-turnaround-ns = <0>;
 		gpmc,cycle2cycle-delay-ns = <0>;
 		gpmc,wait-monitoring-ns = <0>;
-		gpmc,clk-activation-ns = <5>;
+		gpmc,clk-activation-ns = <6>;
 		gpmc,wr-data-mux-bus-ns = <30>;
-		gpmc,wr-access-ns = <81>;
-		gpmc,sync-clk-ps = <15000>;
+		gpmc,wr-access-ns = <90>;
+		gpmc,sync-clk-ps = <12000>;
 
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/omap3-lilly-a83x.dtsi b/arch/arm/boot/dts/omap3-lilly-a83x.dtsi
index 7ada1e93e166..cf7a2a72348d 100644
--- a/arch/arm/boot/dts/omap3-lilly-a83x.dtsi
+++ b/arch/arm/boot/dts/omap3-lilly-a83x.dtsi
@@ -405,22 +405,22 @@
 			reg = <0 0x80000>;
 		};
 
-		partition@0x80000 {
+		partition@80000 {
 			label = "u-boot";
 			reg = <0x80000 0x1e0000>;
 		};
 
-		partition@0x260000 {
+		partition@260000 {
 			label = "u-boot-environment";
 			reg = <0x260000 0x20000>;
 		};
 
-		partition@0x280000 {
+		partition@280000 {
 			label = "kernel";
 			reg = <0x280000 0x500000>;
 		};
 
-		partition@0x780000 {
+		partition@780000 {
 			label = "filesystem";
 			reg = <0x780000 0xf880000>;
 		};
diff --git a/arch/arm/boot/dts/omap3-n9.dts b/arch/arm/boot/dts/omap3-n9.dts
index 39e35f8b8206..e44d93fc644c 100644
--- a/arch/arm/boot/dts/omap3-n9.dts
+++ b/arch/arm/boot/dts/omap3-n9.dts
@@ -11,6 +11,7 @@
 /dts-v1/;
 
 #include "omap3-n950-n9.dtsi"
+#include <dt-bindings/input/input.h>
 
 / {
 	model = "Nokia N9";
@@ -72,3 +73,9 @@
 	st,max-limit-y = <46>;
 	st,max-limit-z = <46>;
 };
+
+&twl_keypad {
+	linux,keymap = < MATRIX_KEY(6, 8, KEY_VOLUMEUP)
+			 MATRIX_KEY(7, 8, KEY_VOLUMEDOWN)
+			 >;
+};
diff --git a/arch/arm/boot/dts/omap3-n900.dts b/arch/arm/boot/dts/omap3-n900.dts
index 669c51c00c00..ab930581fc7a 100644
--- a/arch/arm/boot/dts/omap3-n900.dts
+++ b/arch/arm/boot/dts/omap3-n900.dts
@@ -35,6 +35,8 @@
 		i2c1 = &i2c1;
 		i2c2 = &i2c2;
 		i2c3 = &i2c3;
+		display0 = &lcd;
+		display1 = &tv;
 	};
 
 	cpus {
@@ -558,7 +560,7 @@
 	tlv320aic3x: tlv320aic3x@18 {
 		compatible = "ti,tlv320aic3x";
 		reg = <0x18>;
-		gpio-reset = <&gpio2 28 GPIO_ACTIVE_HIGH>; /* 60 */
+		reset-gpios = <&gpio2 28 GPIO_ACTIVE_LOW>; /* 60 */
 		ai3x-gpio-func = <
 			0 /* AIC3X_GPIO1_FUNC_DISABLED */
 			5 /* AIC3X_GPIO2_FUNC_DIGITAL_MIC_INPUT */
@@ -575,7 +577,7 @@
 	tlv320aic3x_aux: tlv320aic3x@19 {
 		compatible = "ti,tlv320aic3x";
 		reg = <0x19>;
-		gpio-reset = <&gpio2 28 GPIO_ACTIVE_HIGH>; /* 60 */
+		reset-gpios = <&gpio2 28 GPIO_ACTIVE_LOW>; /* 60 */
 
 		AVDD-supply = <&vmmc2>;
 		DRVDD-supply = <&vmmc2>;
@@ -778,6 +780,8 @@
 
 		reset-gpio = <&gpio4 6 GPIO_ACTIVE_HIGH>; /* 102 */
 
+		lens-focus = <&ad5820>;
+
 		port {
 			csi_cam1: endpoint {
 				bus-type = <3>; /* CCP2 */
@@ -838,6 +842,7 @@
 	onenand@0,0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
+		compatible = "ti,omap2-onenand";
 		reg = <0 0 0x20000>;	/* CS0, offset 0, IO size 128K */
 
 		gpmc,sync-read;
@@ -965,7 +970,7 @@
 		ti,esd-recovery-timeout-ms = <8000>;
 	};
 
-	acx565akm@2 {
+	lcd: acx565akm@2 {
 		compatible = "sony,acx565akm";
 		spi-max-frequency = <6000000>;
 		reg = <2>;
diff --git a/arch/arm/boot/dts/omap3-n950-n9.dtsi b/arch/arm/boot/dts/omap3-n950-n9.dtsi
index 12fbb3da5fce..0d9b85317529 100644
--- a/arch/arm/boot/dts/omap3-n950-n9.dtsi
+++ b/arch/arm/boot/dts/omap3-n950-n9.dtsi
@@ -367,6 +367,7 @@
 	onenand@0,0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
+		compatible = "ti,omap2-onenand";
 		reg = <0 0 0x20000>;	/* CS0, offset 0, IO size 128K */
 
 		gpmc,sync-read;
diff --git a/arch/arm/boot/dts/omap3-n950.dts b/arch/arm/boot/dts/omap3-n950.dts
index c354a1ed1e70..2354e09343f3 100644
--- a/arch/arm/boot/dts/omap3-n950.dts
+++ b/arch/arm/boot/dts/omap3-n950.dts
@@ -51,6 +51,26 @@
 	};
 };
 
+&omap3_pmx_core {
+	dsi_pins: pinmux_dsi_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x20dc, PIN_OUTPUT | MUX_MODE1) /* dsi_dx0 - data0+ */
+			OMAP3_CORE1_IOPAD(0x20de, PIN_OUTPUT | MUX_MODE1) /* dsi_dy0 - data0- */
+			OMAP3_CORE1_IOPAD(0x20e0, PIN_OUTPUT | MUX_MODE1) /* dsi_dx1 - clk+   */
+			OMAP3_CORE1_IOPAD(0x20e2, PIN_OUTPUT | MUX_MODE1) /* dsi_dy1 - clk-   */
+			OMAP3_CORE1_IOPAD(0x20e4, PIN_OUTPUT | MUX_MODE1) /* dsi_dx2 - data1+ */
+			OMAP3_CORE1_IOPAD(0x20e6, PIN_OUTPUT | MUX_MODE1) /* dsi_dy2 - data1- */
+		>;
+	};
+
+	display_pins: pinmux_display_pins {
+		pinctrl-single,pins = <
+			OMAP3_CORE1_IOPAD(0x20ca, PIN_INPUT | MUX_MODE4) /* gpio 62 - display te */
+			OMAP3_CORE1_IOPAD(0x20fe, PIN_OUTPUT | MUX_MODE4) /* gpio 87 - display reset */
+		>;
+	};
+};
+
 &i2c2 {
 	smia_1: camera@10 {
 		compatible = "nokia,smia";
@@ -186,3 +206,71 @@
 	st,max-limit-y = <32>;
 	st,max-limit-z = <32>;
 };
+
+&dss {
+	status = "ok";
+
+	vdda_video-supply = <&vdac>;
+};
+
+&dsi {
+	status = "ok";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&dsi_pins>;
+
+	vdd-supply = <&vpll2>;
+
+	port {
+		dsi_out_ep: endpoint {
+			remote-endpoint = <&lcd0_in>;
+			lanes = <2 3 0 1 4 5>;
+		};
+	};
+
+	lcd0: display {
+		compatible = "nokia,himalaya", "panel-dsi-cm";
+		label = "lcd0";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&display_pins>;
+
+		vpnl-supply = <&vmmc2>;
+		vddi-supply = <&vio>;
+
+		reset-gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>;	/* 87 */
+		te-gpios = <&gpio2 30 GPIO_ACTIVE_HIGH>;	/* 62 */
+
+		width-mm = <49>; /* 48.960 mm */
+		height-mm = <88>; /* 88.128 mm */
+
+		/* TODO:
+		 * - panel is upside-down
+		 * - top + bottom 5px are not visible
+		 */
+		panel-timing {
+			clock-frequency = <0>;          /* Calculated by dsi */
+
+			hback-porch = <2>;
+			hactive = <480>;
+			hfront-porch = <0>;
+			hsync-len = <2>;
+
+			vback-porch = <1>;
+			vactive = <864>;
+			vfront-porch = <0>;
+			vsync-len = <1>;
+
+			hsync-active = <0>;
+			vsync-active = <0>;
+			de-active = <1>;
+			pixelclk-active = <1>;
+		};
+
+		port {
+			lcd0_in: endpoint {
+				remote-endpoint = <&dsi_out_ep>;
+			};
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/omap3-pandora-common.dtsi b/arch/arm/boot/dts/omap3-pandora-common.dtsi
index cd53dc6c0051..f83b1029b3b7 100644
--- a/arch/arm/boot/dts/omap3-pandora-common.dtsi
+++ b/arch/arm/boot/dts/omap3-pandora-common.dtsi
@@ -627,7 +627,7 @@
 
 	lcd: lcd@1 {
 		reg = <1>;	/* CS1 */
-		compatible =	"omapdss,tpo,td043mtea1";
+		compatible =	"tpo,td043mtea1";
 		spi-max-frequency = <100000>;
 		spi-cpol;
 		spi-cpha;
diff --git a/arch/arm/boot/dts/omap3.dtsi b/arch/arm/boot/dts/omap3.dtsi
index bb33935df7b0..a005802cd52b 100644
--- a/arch/arm/boot/dts/omap3.dtsi
+++ b/arch/arm/boot/dts/omap3.dtsi
@@ -587,20 +587,6 @@
 			dma-names = "rx";
 		};
 
-		smartreflex_core: smartreflex@480cb000 {
-			compatible = "ti,omap3-smartreflex-core";
-			ti,hwmods = "smartreflex_core";
-			reg = <0x480cb000 0x400>;
-			interrupts = <19>;
-		};
-
-		smartreflex_mpu_iva: smartreflex@480c9000 {
-			compatible = "ti,omap3-smartreflex-iva";
-			ti,hwmods = "smartreflex_mpu_iva";
-			reg = <0x480c9000 0x400>;
-			interrupts = <18>;
-		};
-
 		timer1: timer@48318000 {
 			compatible = "ti,omap3430-timer";
 			reg = <0x48318000 0x400>;
diff --git a/arch/arm/boot/dts/omap3430-sdp.dts b/arch/arm/boot/dts/omap3430-sdp.dts
index 908951eb5943..d652708f6bef 100644
--- a/arch/arm/boot/dts/omap3430-sdp.dts
+++ b/arch/arm/boot/dts/omap3430-sdp.dts
@@ -154,6 +154,7 @@
 		linux,mtd-name= "samsung,kfm2g16q2m-deb8";
 		#address-cells = <1>;
 		#size-cells = <1>;
+		compatible = "ti,omap2-onenand";
 		reg = <2 0 0x20000>;	/* CS2, offset 0, IO size 4 */
 
 		gpmc,device-width = <2>;
diff --git a/arch/arm/boot/dts/omap34xx.dtsi b/arch/arm/boot/dts/omap34xx.dtsi
index ac4f8795b756..f572a477f74c 100644
--- a/arch/arm/boot/dts/omap34xx.dtsi
+++ b/arch/arm/boot/dts/omap34xx.dtsi
@@ -8,6 +8,7 @@
  * kind, whether express or implied.
  */
 
+#include <dt-bindings/bus/ti-sysc.h>
 #include <dt-bindings/media/omap3-isp.h>
 
 #include "omap3.dtsi"
@@ -61,6 +62,44 @@
 			compatible = "ti,omap34xx-bandgap";
 			#thermal-sensor-cells = <0>;
 		};
+
+		target-module@480cb000 {
+			compatible = "ti,sysc-omap3430-sr", "ti,sysc";
+			ti,hwmods = "smartreflex_core";
+			reg = <0x480cb024 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP2_CLOCKACTIVITY>;
+			clocks = <&sr2_fck>;
+			clock-names = "fck";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x480cb000 0x001000>;
+
+			smartreflex_core: smartreflex@0 {
+				compatible = "ti,omap3-smartreflex-core";
+				reg = <0 0x400>;
+				interrupts = <19>;
+			};
+		};
+
+		target-module@480c9000 {
+			compatible = "ti,sysc-omap3430-sr", "ti,sysc";
+			ti,hwmods = "smartreflex_mpu_iva";
+			reg = <0x480c9024 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP2_CLOCKACTIVITY>;
+			clocks = <&sr1_fck>;
+			clock-names = "fck";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x480c9000 0x001000>;
+
+			smartreflex_mpu_iva: smartreflex@480c9000 {
+				compatible = "ti,omap3-smartreflex-mpu-iva";
+				reg = <0 0x400>;
+				interrupts = <18>;
+			};
+		};
 	};
 
 	thermal_zones: thermal-zones {
diff --git a/arch/arm/boot/dts/omap36xx.dtsi b/arch/arm/boot/dts/omap36xx.dtsi
index ade31d74c70c..6fb23ada1f64 100644
--- a/arch/arm/boot/dts/omap36xx.dtsi
+++ b/arch/arm/boot/dts/omap36xx.dtsi
@@ -8,6 +8,7 @@
  * kind, whether express or implied.
  */
 
+#include <dt-bindings/bus/ti-sysc.h>
 #include <dt-bindings/media/omap3-isp.h>
 
 #include "omap3.dtsi"
@@ -93,6 +94,51 @@
 			compatible = "ti,omap36xx-bandgap";
 			#thermal-sensor-cells = <0>;
 		};
+
+		target-module@480cb000 {
+			compatible = "ti,sysc-omap3630-sr", "ti,sysc";
+			ti,hwmods = "smartreflex_core";
+			reg = <0x480cb038 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP3_SR_ENAWAKEUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>;
+			clocks = <&sr2_fck>;
+			clock-names = "fck";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x480cb000 0x001000>;
+
+			smartreflex_core: smartreflex@0 {
+				compatible = "ti,omap3-smartreflex-core";
+				reg = <0 0x400>;
+				interrupts = <19>;
+			};
+		};
+
+		target-module@480c9000 {
+			compatible = "ti,sysc-omap3630-sr", "ti,sysc";
+			ti,hwmods = "smartreflex_mpu_iva";
+			reg = <0x480c9038 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP3_SR_ENAWAKEUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>;
+			clocks = <&sr1_fck>;
+			clock-names = "fck";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x480c9000 0x001000>;
+
+
+			smartreflex_mpu_iva: smartreflex@480c9000 {
+				compatible = "ti,omap3-smartreflex-mpu-iva";
+				reg = <0 0x400>;
+				interrupts = <18>;
+			};
+		};
 	};
 
 	thermal_zones: thermal-zones {
diff --git a/arch/arm/boot/dts/omap4-droid4-xt894.dts b/arch/arm/boot/dts/omap4-droid4-xt894.dts
index 24a463f8641f..b21084da490b 100644
--- a/arch/arm/boot/dts/omap4-droid4-xt894.dts
+++ b/arch/arm/boot/dts/omap4-droid4-xt894.dts
@@ -177,6 +177,10 @@
 		vddi-supply = <&lcd_regulator>;
 		reset-gpios = <&gpio4 5 GPIO_ACTIVE_HIGH>;	/* gpio101 */
 
+		width-mm = <50>;
+		height-mm = <89>;
+		backlight = <&lcd_backlight>;
+
 		panel-timing {
 			clock-frequency = <0>;		/* Calculated by dsi */
 
@@ -346,7 +350,7 @@
 
 		enable-gpios = <&gpio6 12 GPIO_ACTIVE_HIGH>;
 
-		backlight {
+		lcd_backlight: backlight {
 			compatible = "ti,lm3532-backlight";
 
 			lcd {
diff --git a/arch/arm/boot/dts/omap4.dtsi b/arch/arm/boot/dts/omap4.dtsi
index cc1a07a3620f..475904894b86 100644
--- a/arch/arm/boot/dts/omap4.dtsi
+++ b/arch/arm/boot/dts/omap4.dtsi
@@ -6,9 +6,12 @@
  * published by the Free Software Foundation.
  */
 
+#include <dt-bindings/bus/ti-sysc.h>
+#include <dt-bindings/clock/omap4.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/pinctrl/omap.h>
+#include <dt-bindings/clock/omap4.h>
 
 / {
 	compatible = "ti,omap4430", "ti,omap4";
@@ -143,8 +146,11 @@
 			ranges = <0 0x4a000000 0x1000000>;
 
 			cm1: cm1@4000 {
-				compatible = "ti,omap4-cm1";
+				compatible = "ti,omap4-cm1", "simple-bus";
 				reg = <0x4000 0x2000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x4000 0x2000>;
 
 				cm1_clocks: clocks {
 					#address-cells = <1>;
@@ -156,8 +162,11 @@
 			};
 
 			cm2: cm2@8000 {
-				compatible = "ti,omap4-cm2";
+				compatible = "ti,omap4-cm2", "simple-bus";
 				reg = <0x8000 0x3000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x8000 0x3000>;
 
 				cm2_clocks: clocks {
 					#address-cells = <1>;
@@ -243,6 +252,9 @@
 					compatible = "ti,omap4-prm";
 					reg = <0x6000 0x3000>;
 					interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+					#address-cells = <1>;
+					#size-cells = <1>;
+					ranges = <0 0x6000 0x3000>;
 
 					prm_clocks: clocks {
 						#address-cells = <1>;
@@ -383,11 +395,18 @@
 		};
 
 		target-module@48076000 {
-			compatible = "ti,sysc-omap4";
+			compatible = "ti,sysc-omap4", "ti,sysc";
 			ti,hwmods = "slimbus2";
 			reg = <0x48076000 0x4>,
 			      <0x48076010 0x4>;
 			reg-names = "rev", "sysc";
+			ti,sysc-mask = <SYSC_OMAP4_SOFTRESET>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&l4_per_clkctrl OMAP4_SLIMBUS2_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x48076000 0x001000>;
@@ -456,11 +475,17 @@
 		};
 
 		target-module@4a0db000 {
-			compatible = "ti,sysc-sr";
+			compatible = "ti,sysc-omap4-sr", "ti,sysc";
 			ti,hwmods = "smartreflex_iva";
-			reg = <0x4a0db000 0x4>,
-			      <0x4a0db008 0x4>;
-			reg-names = "rev", "sysc";
+			reg = <0x4a0db038 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP3_SR_ENAWAKEUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&l4_ao_clkctrl OMAP4_SMARTREFLEX_IVA_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x4a0db000 0x001000>;
@@ -473,11 +498,17 @@
 		};
 
 		target-module@4a0dd000 {
-			compatible = "ti,sysc-sr";
+			compatible = "ti,sysc-omap4-sr", "ti,sysc";
 			ti,hwmods = "smartreflex_core";
-			reg = <0x4a0dd000 0x4>,
-			      <0x4a0dd008 0x4>;
-			reg-names = "rev", "sysc";
+			reg = <0x4a0dd038 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP3_SR_ENAWAKEUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&l4_ao_clkctrl OMAP4_SMARTREFLEX_CORE_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x4a0dd000 0x001000>;
@@ -490,11 +521,17 @@
 		};
 
 		target-module@4a0d9000 {
-			compatible = "ti,sysc-sr";
+			compatible = "ti,sysc-omap4-sr", "ti,sysc";
 			ti,hwmods = "smartreflex_mpu";
-			reg = <0x4a0d9000 0x4>,
-			      <0x4a0d9008 0x4>;
-			reg-names = "rev", "sysc";
+			reg = <0x4a0d9038 0x4>;
+			reg-names = "sysc";
+			ti,sysc-mask = <SYSC_OMAP3_SR_ENAWAKEUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&l4_ao_clkctrl OMAP4_SMARTREFLEX_MPU_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x4a0d9000 0x001000>;
@@ -674,7 +711,7 @@
 			reg-names = "sys", "gdd";
 			ti,hwmods = "hsi";
 
-			clocks = <&hsi_fck>;
+			clocks = <&l3_init_clkctrl OMAP4_HSI_CLKCTRL 0>;
 			clock-names = "hsi_fck";
 
 			interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
@@ -710,11 +747,23 @@
 		};
 
 		target-module@52000000 {
-			compatible = "ti,sysc-omap4";
+			compatible = "ti,sysc-omap4", "ti,sysc";
 			ti,hwmods = "iss";
 			reg = <0x52000000 0x4>,
 			      <0x52000010 0x4>;
 			reg-names = "rev", "sysc";
+			ti,sysc-mask = <SYSC_OMAP4_SOFTRESET>;
+			ti,sysc-midle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			ti,sysc-delay-us = <2>;
+			clocks = <&iss_clkctrl OMAP4_ISS_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x52000000 0x1000000>;
@@ -817,10 +866,17 @@
 		};
 
 		target-module@40128000 {
-			compatible = "ti,sysc-mcasp";
+			compatible = "ti,sysc-mcasp", "ti,sysc";
 			ti,hwmods = "mcasp";
-			reg = <0x40128004 0x4>;
-			reg-names = "sysc";
+			reg = <0x40128000 0x4>,
+			      <0x40128004 0x4>;
+			reg-names = "rev", "sysc";
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&abe_clkctrl OMAP4_MCASP_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0x00000000 0x40128000 0x1000>, /* MPU */
@@ -835,11 +891,18 @@
 		};
 
 		target-module@4012c000 {
-			compatible = "ti,sysc-omap4";
+			compatible = "ti,sysc-omap4", "ti,sysc";
 			ti,hwmods = "slimbus1";
 			reg = <0x4012c000 0x4>,
 			      <0x4012c010 0x4>;
 			reg-names = "rev", "sysc";
+			ti,sysc-mask = <SYSC_OMAP4_SOFTRESET>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&abe_clkctrl OMAP4_SLIMBUS1_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0x00000000 0x4012c000 0x1000>, /* MPU */
@@ -849,11 +912,20 @@
 		};
 
 		target-module@401f1000 {
-			compatible = "ti,sysc-omap4";
+			compatible = "ti,sysc-omap4", "ti,sysc";
 			ti,hwmods = "aess";
 			reg = <0x401f1000 0x4>,
 			      <0x401f1010 0x4>;
 			reg-names = "rev", "sysc";
+			ti,sysc-midle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>;
+			clocks = <&abe_clkctrl OMAP4_AESS_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0x00000000 0x401f1000 0x1000>, /* MPU */
@@ -955,11 +1027,21 @@
 		};
 
 		target-module@4a10a000 {
-			compatible = "ti,sysc-omap4";
+			compatible = "ti,sysc-omap4", "ti,sysc";
 			ti,hwmods = "fdif";
 			reg = <0x4a10a000 0x4>,
 			      <0x4a10a010 0x4>;
 			reg-names = "rev", "sysc";
+			ti,sysc-mask = <SYSC_OMAP4_SOFTRESET>;
+			ti,sysc-midle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>;
+			ti,sysc-delay-us = <2>;
+			clocks = <&iss_clkctrl OMAP4_FDIF_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x4a10a000 0x1000>;
@@ -973,6 +1055,8 @@
 			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer1";
 			ti,timer-alwon;
+			clocks = <&l4_wkup_clkctrl OMAP4_TIMER1_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer2: timer@48032000 {
@@ -1182,11 +1266,21 @@
 		};
 
 		target-module@56000000 {
-			compatible = "ti,sysc-omap4";
+			compatible = "ti,sysc-omap4", "ti,sysc";
 			ti,hwmods = "gpu";
 			reg = <0x5601fc00 0x4>,
 			      <0x5601fc10 0x4>;
 			reg-names = "rev", "sysc";
+			ti,sysc-midle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			ti,sysc-sidle = <SYSC_IDLE_FORCE>,
+					<SYSC_IDLE_NO>,
+					<SYSC_IDLE_SMART>,
+					<SYSC_IDLE_SMART_WKUP>;
+			clocks = <&l3_gfx_clkctrl OMAP4_GPU_CLKCTRL 0>;
+			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges = <0 0x56000000 0x2000000>;
@@ -1202,7 +1296,7 @@
 			reg = <0x58000000 0x80>;
 			status = "disabled";
 			ti,hwmods = "dss_core";
-			clocks = <&dss_dss_clk>;
+			clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 8>;
 			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -1213,7 +1307,7 @@
 				reg = <0x58001000 0x1000>;
 				interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 				ti,hwmods = "dss_dispc";
-				clocks = <&dss_dss_clk>;
+				clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 8>;
 				clock-names = "fck";
 			};
 
@@ -1222,7 +1316,7 @@
 				reg = <0x58002000 0x1000>;
 				status = "disabled";
 				ti,hwmods = "dss_rfbi";
-				clocks = <&dss_dss_clk>, <&l3_div_ck>;
+				clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 8>, <&l3_div_ck>;
 				clock-names = "fck", "ick";
 			};
 
@@ -1231,7 +1325,7 @@
 				reg = <0x58003000 0x1000>;
 				status = "disabled";
 				ti,hwmods = "dss_venc";
-				clocks = <&dss_tv_clk>;
+				clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 11>;
 				clock-names = "fck";
 			};
 
@@ -1244,7 +1338,8 @@
 				interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
 				status = "disabled";
 				ti,hwmods = "dss_dsi1";
-				clocks = <&dss_dss_clk>, <&dss_sys_clk>;
+				clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 8>,
+					 <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 10>;
 				clock-names = "fck", "sys_clk";
 			};
 
@@ -1257,7 +1352,8 @@
 				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 				status = "disabled";
 				ti,hwmods = "dss_dsi2";
-				clocks = <&dss_dss_clk>, <&dss_sys_clk>;
+				clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 8>,
+					 <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 10>;
 				clock-names = "fck", "sys_clk";
 			};
 
@@ -1271,7 +1367,8 @@
 				interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
 				status = "disabled";
 				ti,hwmods = "dss_hdmi";
-				clocks = <&dss_48mhz_clk>, <&dss_sys_clk>;
+				clocks = <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 9>,
+					 <&l3_dss_clkctrl OMAP4_DSS_CORE_CLKCTRL 10>;
 				clock-names = "fck", "sys_clk";
 				dmas = <&sdma 76>;
 				dma-names = "audio_tx";
@@ -1280,4 +1377,4 @@
 	};
 };
 
-/include/ "omap44xx-clocks.dtsi"
+#include "omap44xx-clocks.dtsi"
diff --git a/arch/arm/boot/dts/omap44xx-clocks.dtsi b/arch/arm/boot/dts/omap44xx-clocks.dtsi
index 05732ed4f50f..279ff2f419df 100644
--- a/arch/arm/boot/dts/omap44xx-clocks.dtsi
+++ b/arch/arm/boot/dts/omap44xx-clocks.dtsi
@@ -174,14 +174,6 @@
 		ti,index-power-of-two;
 	};
 
-	aess_fclk: aess_fclk@528 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&abe_clk>;
-		ti,bit-shift = <24>;
-		ti,max-div = <2>;
-		reg = <0x0528>;
-	};
 
 	dpll_abe_m3x2_ck: dpll_abe_m3x2_ck@1f4 {
 		#clock-cells = <0>;
@@ -464,7 +456,7 @@
 	ocp_abe_iclk: ocp_abe_iclk@528 {
 		#clock-cells = <0>;
 		compatible = "ti,divider-clock";
-		clocks = <&aess_fclk>;
+		clocks = <&abe_clkctrl OMAP4_AESS_CLKCTRL 24>;
 		ti,bit-shift = <24>;
 		reg = <0x0528>;
 		ti,dividers = <2>, <1>;
@@ -478,156 +470,13 @@
 		clock-div = <4>;
 	};
 
-	dmic_sync_mux_ck: dmic_sync_mux_ck@538 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&syc_clk_div_ck>, <&func_24m_clk>;
-		ti,bit-shift = <25>;
-		reg = <0x0538>;
-	};
-
-	func_dmic_abe_gfclk: func_dmic_abe_gfclk@538 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dmic_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0538>;
-	};
-
-	mcasp_sync_mux_ck: mcasp_sync_mux_ck@540 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&syc_clk_div_ck>, <&func_24m_clk>;
-		ti,bit-shift = <25>;
-		reg = <0x0540>;
-	};
-
-	func_mcasp_abe_gfclk: func_mcasp_abe_gfclk@540 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcasp_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0540>;
-	};
-
-	mcbsp1_sync_mux_ck: mcbsp1_sync_mux_ck@548 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&syc_clk_div_ck>, <&func_24m_clk>;
-		ti,bit-shift = <25>;
-		reg = <0x0548>;
-	};
-
-	func_mcbsp1_gfclk: func_mcbsp1_gfclk@548 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcbsp1_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0548>;
-	};
-
-	mcbsp2_sync_mux_ck: mcbsp2_sync_mux_ck@550 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&syc_clk_div_ck>, <&func_24m_clk>;
-		ti,bit-shift = <25>;
-		reg = <0x0550>;
-	};
-
-	func_mcbsp2_gfclk: func_mcbsp2_gfclk@550 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcbsp2_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0550>;
-	};
-
-	mcbsp3_sync_mux_ck: mcbsp3_sync_mux_ck@558 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&syc_clk_div_ck>, <&func_24m_clk>;
-		ti,bit-shift = <25>;
-		reg = <0x0558>;
-	};
-
-	func_mcbsp3_gfclk: func_mcbsp3_gfclk@558 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcbsp3_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0558>;
-	};
-
-	slimbus1_fclk_1: slimbus1_fclk_1@560 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_24m_clk>;
-		ti,bit-shift = <9>;
-		reg = <0x0560>;
-	};
-
-	slimbus1_fclk_0: slimbus1_fclk_0@560 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&abe_24m_fclk>;
-		ti,bit-shift = <8>;
-		reg = <0x0560>;
-	};
-
-	slimbus1_fclk_2: slimbus1_fclk_2@560 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&pad_clks_ck>;
-		ti,bit-shift = <10>;
-		reg = <0x0560>;
-	};
-
-	slimbus1_slimbus_clk: slimbus1_slimbus_clk@560 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&slimbus_clk>;
-		ti,bit-shift = <11>;
-		reg = <0x0560>;
-	};
-
-	timer5_sync_mux: timer5_sync_mux@568 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&syc_clk_div_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0568>;
-	};
-
-	timer6_sync_mux: timer6_sync_mux@570 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&syc_clk_div_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0570>;
-	};
-
-	timer7_sync_mux: timer7_sync_mux@578 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&syc_clk_div_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0578>;
-	};
-
-	timer8_sync_mux: timer8_sync_mux@580 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&syc_clk_div_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0580>;
-	};
-
 	dummy_ck: dummy_ck {
 		#clock-cells = <0>;
 		compatible = "fixed-clock";
 		clock-frequency = <0>;
 	};
 };
+
 &prm_clocks {
 	sys_clkin_ck: sys_clkin_ck@110 {
 		#clock-cells = <0>;
@@ -675,22 +524,6 @@
 		ti,max-div = <2>;
 	};
 
-	gpio1_dbclk: gpio1_dbclk@1838 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1838>;
-	};
-
-	dmt1_clk_mux: dmt1_clk_mux@1840 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1840>;
-	};
-
 	usim_ck: usim_ck@1858 {
 		#clock-cells = <0>;
 		compatible = "ti,divider-clock";
@@ -708,45 +541,10 @@
 		reg = <0x1858>;
 	};
 
-	pmd_stm_clock_mux_ck: pmd_stm_clock_mux_ck@1a20 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&dpll_core_m6x2_ck>, <&tie_low_clock_ck>;
-		ti,bit-shift = <20>;
-		reg = <0x1a20>;
-	};
-
-	pmd_trace_clk_mux_ck: pmd_trace_clk_mux_ck@1a20 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&dpll_core_m6x2_ck>, <&tie_low_clock_ck>;
-		ti,bit-shift = <22>;
-		reg = <0x1a20>;
-	};
-
-	stm_clk_div_ck: stm_clk_div_ck@1a20 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&pmd_stm_clock_mux_ck>;
-		ti,bit-shift = <27>;
-		ti,max-div = <64>;
-		reg = <0x1a20>;
-		ti,index-power-of-two;
-	};
-
-	trace_clk_div_div_ck: trace_clk_div_div_ck@1a20 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&pmd_trace_clk_mux_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1a20>;
-		ti,dividers = <0>, <1>, <2>, <0>, <4>;
-	};
-
 	trace_clk_div_ck: trace_clk_div_ck {
 		#clock-cells = <0>;
 		compatible = "ti,clkdm-gate-clock";
-		clocks = <&trace_clk_div_div_ck>;
+		clocks = <&emu_sys_clkctrl OMAP4_DEBUGSS_CLKCTRL 24>;
 	};
 };
 
@@ -975,155 +773,6 @@
 		ti,max-div = <2>;
 	};
 
-	dss_sys_clk: dss_sys_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&syc_clk_div_ck>;
-		ti,bit-shift = <10>;
-		reg = <0x1120>;
-	};
-
-	dss_tv_clk: dss_tv_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&extalt_clkin_ck>;
-		ti,bit-shift = <11>;
-		reg = <0x1120>;
-	};
-
-	dss_dss_clk: dss_dss_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_per_m5x2_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1120>;
-		ti,set-rate-parent;
-	};
-
-	dss_48mhz_clk: dss_48mhz_clk@1120 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_48mc_fclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1120>;
-	};
-
-	fdif_fck: fdif_fck@1028 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&dpll_per_m4x2_ck>;
-		ti,bit-shift = <24>;
-		ti,max-div = <4>;
-		reg = <0x1028>;
-		ti,index-power-of-two;
-	};
-
-	gpio2_dbclk: gpio2_dbclk@1460 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1460>;
-	};
-
-	gpio3_dbclk: gpio3_dbclk@1468 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1468>;
-	};
-
-	gpio4_dbclk: gpio4_dbclk@1470 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1470>;
-	};
-
-	gpio5_dbclk: gpio5_dbclk@1478 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1478>;
-	};
-
-	gpio6_dbclk: gpio6_dbclk@1480 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1480>;
-	};
-
-	sgx_clk_mux: sgx_clk_mux@1220 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dpll_core_m7x2_ck>, <&dpll_per_m7x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1220>;
-	};
-
-	hsi_fck: hsi_fck@1338 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		ti,max-div = <4>;
-		reg = <0x1338>;
-		ti,index-power-of-two;
-	};
-
-	iss_ctrlclk: iss_ctrlclk@1020 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_96m_fclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1020>;
-	};
-
-	mcbsp4_sync_mux_ck: mcbsp4_sync_mux_ck@14e0 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_96m_fclk>, <&per_abe_nc_fclk>;
-		ti,bit-shift = <25>;
-		reg = <0x14e0>;
-	};
-
-	per_mcbsp4_gfclk: per_mcbsp4_gfclk@14e0 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcbsp4_sync_mux_ck>, <&pad_clks_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x14e0>;
-	};
-
-	hsmmc1_fclk: hsmmc1_fclk@1328 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_64m_fclk>, <&func_96m_fclk>;
-		ti,bit-shift = <24>;
-		reg = <0x1328>;
-	};
-
-	hsmmc2_fclk: hsmmc2_fclk@1330 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_64m_fclk>, <&func_96m_fclk>;
-		ti,bit-shift = <24>;
-		reg = <0x1330>;
-	};
-
-	ocp2scp_usb_phy_phy_48m: ocp2scp_usb_phy_phy_48m@13e0 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_48m_fclk>;
-		ti,bit-shift = <8>;
-		reg = <0x13e0>;
-	};
-
 	sha2md5_fck: sha2md5_fck@15c8 {
 		#clock-cells = <0>;
 		compatible = "ti,gate-clock";
@@ -1132,222 +781,6 @@
 		reg = <0x15c8>;
 	};
 
-	slimbus2_fclk_1: slimbus2_fclk_1@1538 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&per_abe_24m_fclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1538>;
-	};
-
-	slimbus2_fclk_0: slimbus2_fclk_0@1538 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_24mc_fclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1538>;
-	};
-
-	slimbus2_slimbus_clk: slimbus2_slimbus_clk@1538 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&pad_slimbus_core_clks_ck>;
-		ti,bit-shift = <10>;
-		reg = <0x1538>;
-	};
-
-	smartreflex_core_fck: smartreflex_core_fck@638 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l4_wkup_clk_mux_ck>;
-		ti,bit-shift = <1>;
-		reg = <0x0638>;
-	};
-
-	smartreflex_iva_fck: smartreflex_iva_fck@630 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l4_wkup_clk_mux_ck>;
-		ti,bit-shift = <1>;
-		reg = <0x0630>;
-	};
-
-	smartreflex_mpu_fck: smartreflex_mpu_fck@628 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l4_wkup_clk_mux_ck>;
-		ti,bit-shift = <1>;
-		reg = <0x0628>;
-	};
-
-	cm2_dm10_mux: cm2_dm10_mux@1428 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1428>;
-	};
-
-	cm2_dm11_mux: cm2_dm11_mux@1430 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1430>;
-	};
-
-	cm2_dm2_mux: cm2_dm2_mux@1438 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1438>;
-	};
-
-	cm2_dm3_mux: cm2_dm3_mux@1440 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1440>;
-	};
-
-	cm2_dm4_mux: cm2_dm4_mux@1448 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1448>;
-	};
-
-	cm2_dm9_mux: cm2_dm9_mux@1450 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin_ck>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1450>;
-	};
-
-	usb_host_fs_fck: usb_host_fs_fck@13d0 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_48mc_fclk>;
-		ti,bit-shift = <1>;
-		reg = <0x13d0>;
-	};
-
-	utmi_p1_gfclk: utmi_p1_gfclk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&init_60m_fclk>, <&xclk60mhsp1_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_utmi_p1_clk: usb_host_hs_utmi_p1_clk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&utmi_p1_gfclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1358>;
-	};
-
-	utmi_p2_gfclk: utmi_p2_gfclk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&init_60m_fclk>, <&xclk60mhsp2_ck>;
-		ti,bit-shift = <25>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_utmi_p2_clk: usb_host_hs_utmi_p2_clk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&utmi_p2_gfclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_utmi_p3_clk: usb_host_hs_utmi_p3_clk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&init_60m_fclk>;
-		ti,bit-shift = <10>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_hsic480m_p1_clk: usb_host_hs_hsic480m_p1_clk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_usb_m2_ck>;
-		ti,bit-shift = <13>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_hsic60m_p1_clk: usb_host_hs_hsic60m_p1_clk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&init_60m_fclk>;
-		ti,bit-shift = <11>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_hsic60m_p2_clk: usb_host_hs_hsic60m_p2_clk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&init_60m_fclk>;
-		ti,bit-shift = <12>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_hsic480m_p2_clk: usb_host_hs_hsic480m_p2_clk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_usb_m2_ck>;
-		ti,bit-shift = <14>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_func48mclk: usb_host_hs_func48mclk@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_48mc_fclk>;
-		ti,bit-shift = <15>;
-		reg = <0x1358>;
-	};
-
-	usb_host_hs_fck: usb_host_hs_fck@1358 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&init_60m_fclk>;
-		ti,bit-shift = <1>;
-		reg = <0x1358>;
-	};
-
-	otg_60m_gfclk: otg_60m_gfclk@1360 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&utmi_phy_clkout_ck>, <&xclk60motg_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1360>;
-	};
-
-	usb_otg_hs_xclk: usb_otg_hs_xclk@1360 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&otg_60m_gfclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1360>;
-	};
-
-	usb_otg_hs_ick: usb_otg_hs_ick@1360 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3_div_ck>;
-		ti,bit-shift = <0>;
-		reg = <0x1360>;
-	};
-
 	usb_phy_cm_clk32k: usb_phy_cm_clk32k@640 {
 		#clock-cells = <0>;
 		compatible = "ti,gate-clock";
@@ -1355,44 +788,12 @@
 		ti,bit-shift = <8>;
 		reg = <0x0640>;
 	};
-
-	usb_tll_hs_usb_ch2_clk: usb_tll_hs_usb_ch2_clk@1368 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&init_60m_fclk>;
-		ti,bit-shift = <10>;
-		reg = <0x1368>;
-	};
-
-	usb_tll_hs_usb_ch0_clk: usb_tll_hs_usb_ch0_clk@1368 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&init_60m_fclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1368>;
-	};
-
-	usb_tll_hs_usb_ch1_clk: usb_tll_hs_usb_ch1_clk@1368 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&init_60m_fclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1368>;
-	};
-
-	usb_tll_hs_ick: usb_tll_hs_ick@1368 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l4_div_ck>;
-		ti,bit-shift = <0>;
-		reg = <0x1368>;
-	};
 };
 
 &cm2_clockdomains {
 	l3_init_clkdm: l3_init_clkdm {
 		compatible = "ti,clockdomain";
-		clocks = <&dpll_usb_ck>, <&usb_host_fs_fck>;
+		clocks = <&dpll_usb_ck>;
 	};
 };
 
@@ -1631,3 +1032,291 @@
 		reg = <0x0224>;
 	};
 };
+
+&cm1 {
+	mpuss_cm: mpuss_cm@300 {
+		compatible = "ti,omap4-cm";
+		reg = <0x300 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x300 0x100>;
+
+		mpuss_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	tesla_cm: tesla_cm@400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x400 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x400 0x100>;
+
+		tesla_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	abe_cm: abe_cm@500 {
+		compatible = "ti,omap4-cm";
+		reg = <0x500 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x500 0x100>;
+
+		abe_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x6c>;
+			#clock-cells = <2>;
+		};
+	};
+
+};
+
+&cm2 {
+	l4_ao_cm: l4_ao_cm@600 {
+		compatible = "ti,omap4-cm";
+		reg = <0x600 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x600 0x100>;
+
+		l4_ao_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x1c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_1_cm: l3_1_cm@700 {
+		compatible = "ti,omap4-cm";
+		reg = <0x700 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x700 0x100>;
+
+		l3_1_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_2_cm: l3_2_cm@800 {
+		compatible = "ti,omap4-cm";
+		reg = <0x800 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x800 0x100>;
+
+		l3_2_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x14>;
+			#clock-cells = <2>;
+		};
+	};
+
+	ducati_cm: ducati_cm@900 {
+		compatible = "ti,omap4-cm";
+		reg = <0x900 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x900 0x100>;
+
+		ducati_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_dma_cm: l3_dma_cm@a00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xa00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xa00 0x100>;
+
+		l3_dma_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_emif_cm: l3_emif_cm@b00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xb00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xb00 0x100>;
+
+		l3_emif_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x1c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	d2d_cm: d2d_cm@c00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xc00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xc00 0x100>;
+
+		d2d_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4_cfg_cm: l4_cfg_cm@d00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xd00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xd00 0x100>;
+
+		l4_cfg_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x14>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_instr_cm: l3_instr_cm@e00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xe00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xe00 0x100>;
+
+		l3_instr_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x24>;
+			#clock-cells = <2>;
+		};
+	};
+
+	ivahd_cm: ivahd_cm@f00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xf00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xf00 0x100>;
+
+		ivahd_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xc>;
+			#clock-cells = <2>;
+		};
+	};
+
+	iss_cm: iss_cm@1000 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1000 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1000 0x100>;
+
+		iss_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xc>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_dss_cm: l3_dss_cm@1100 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1100 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1100 0x100>;
+
+		l3_dss_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_gfx_cm: l3_gfx_cm@1200 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1200 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1200 0x100>;
+
+		l3_gfx_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3_init_cm: l3_init_cm@1300 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1300 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1300 0x100>;
+
+		l3_init_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xc4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4_per_cm: l4_per_cm@1400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1400 0x200>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1400 0x200>;
+
+		l4_per_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x144>;
+			#clock-cells = <2>;
+		};
+	};
+
+};
+
+&prm {
+	l4_wkup_cm: l4_wkup_cm@1800 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1800 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1800 0x100>;
+
+		l4_wkup_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x5c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	emu_sys_cm: emu_sys_cm@1a00 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1a00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1a00 0x100>;
+
+		emu_sys_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/omap5-uevm.dts b/arch/arm/boot/dts/omap5-uevm.dts
index ec2c8baef62a..592e17fd4eeb 100644
--- a/arch/arm/boot/dts/omap5-uevm.dts
+++ b/arch/arm/boot/dts/omap5-uevm.dts
@@ -47,7 +47,7 @@
 			gpios = <&gpio3 19 GPIO_ACTIVE_LOW>;	/* gpio3_83 */
 			wakeup-source;
 			autorepeat;
-			debounce_interval = <50>;
+			debounce-interval = <50>;
 		};
 	};
 
diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi
index 51a7fb3d7b9a..35d4298da83d 100644
--- a/arch/arm/boot/dts/omap5.dtsi
+++ b/arch/arm/boot/dts/omap5.dtsi
@@ -10,6 +10,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/pinctrl/omap.h>
+#include <dt-bindings/clock/omap5.h>
 
 / {
 	#address-cells = <2>;
@@ -201,8 +202,12 @@
 			};
 
 			cm_core_aon: cm_core_aon@4000 {
-				compatible = "ti,omap5-cm-core-aon";
+				compatible = "ti,omap5-cm-core-aon",
+					     "simple-bus";
 				reg = <0x4000 0x2000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x4000 0x2000>;
 
 				cm_core_aon_clocks: clocks {
 					#address-cells = <1>;
@@ -214,8 +219,11 @@
 			};
 
 			cm_core: cm_core@8000 {
-				compatible = "ti,omap5-cm-core";
+				compatible = "ti,omap5-cm-core", "simple-bus";
 				reg = <0x8000 0x3000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x8000 0x3000>;
 
 				cm_core_clocks: clocks {
 					#address-cells = <1>;
@@ -240,9 +248,12 @@
 			};
 
 			prm: prm@6000 {
-				compatible = "ti,omap5-prm";
+				compatible = "ti,omap5-prm", "simple-bus";
 				reg = <0x6000 0x3000>;
 				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x6000 0x3000>;
 
 				prm_clocks: clocks {
 					#address-cells = <1>;
@@ -734,6 +745,8 @@
 			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			ti,hwmods = "timer1";
 			ti,timer-alwon;
+			clocks = <&wkupaon_clkctrl OMAP5_TIMER1_CLKCTRL 24>;
+			clock-names = "fck";
 		};
 
 		timer2: timer@48032000 {
@@ -893,7 +906,8 @@
 				compatible = "ti,omap-usb2";
 				reg = <0x4a084000 0x7c>;
 				syscon-phy-power = <&scm_conf 0x300>;
-				clocks = <&usb_phy_cm_clk32k>, <&usb_otg_ss_refclk960m>;
+				clocks = <&usb_phy_cm_clk32k>,
+					 <&l3init_clkctrl OMAP5_USB_OTG_SS_CLKCTRL 8>;
 				clock-names = "wkupclk", "refclk";
 				#phy-cells = <0>;
 			};
@@ -907,7 +921,7 @@
 				syscon-phy-power = <&scm_conf 0x370>;
 				clocks = <&usb_phy_cm_clk32k>,
 					 <&sys_clkin>,
-					 <&usb_otg_ss_refclk960m>;
+					 <&l3init_clkctrl OMAP5_USB_OTG_SS_CLKCTRL 8>;
 				clock-names =	"wkupclk",
 						"sysclk",
 						"refclk";
@@ -976,7 +990,8 @@
 				      <0x4A096800 0x40>; /* pll_ctrl */
 				reg-names = "phy_rx", "phy_tx", "pll_ctrl";
 				syscon-phy-power = <&scm_conf 0x374>;
-				clocks = <&sys_clkin>, <&sata_ref_clk>;
+				clocks = <&sys_clkin>,
+					 <&l3init_clkctrl OMAP5_SATA_CLKCTRL 8>;
 				clock-names = "sysclk", "refclk";
 				#phy-cells = <0>;
 			};
@@ -988,7 +1003,7 @@
 			interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
 			phys = <&sata_phy>;
 			phy-names = "sata-phy";
-			clocks = <&sata_ref_clk>;
+			clocks = <&l3init_clkctrl OMAP5_SATA_CLKCTRL 8>;
 			ti,hwmods = "sata";
 			ports-implemented = <0x1>;
 		};
@@ -998,7 +1013,7 @@
 			reg = <0x58000000 0x80>;
 			status = "disabled";
 			ti,hwmods = "dss_core";
-			clocks = <&dss_dss_clk>;
+			clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>;
 			clock-names = "fck";
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -1009,7 +1024,7 @@
 				reg = <0x58001000 0x1000>;
 				interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 				ti,hwmods = "dss_dispc";
-				clocks = <&dss_dss_clk>;
+				clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>;
 				clock-names = "fck";
 			};
 
@@ -1018,7 +1033,7 @@
 				reg = <0x58002000 0x100>;
 				status = "disabled";
 				ti,hwmods = "dss_rfbi";
-				clocks = <&dss_dss_clk>, <&l3_iclk_div>;
+				clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>, <&l3_iclk_div>;
 				clock-names = "fck", "ick";
 			};
 
@@ -1031,7 +1046,8 @@
 				interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
 				status = "disabled";
 				ti,hwmods = "dss_dsi1";
-				clocks = <&dss_dss_clk>, <&dss_sys_clk>;
+				clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>,
+					 <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 10>;
 				clock-names = "fck", "sys_clk";
 			};
 
@@ -1044,7 +1060,8 @@
 				interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
 				status = "disabled";
 				ti,hwmods = "dss_dsi2";
-				clocks = <&dss_dss_clk>, <&dss_sys_clk>;
+				clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>,
+					 <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 10>;
 				clock-names = "fck", "sys_clk";
 			};
 
@@ -1058,7 +1075,8 @@
 				interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
 				status = "disabled";
 				ti,hwmods = "dss_hdmi";
-				clocks = <&dss_48mhz_clk>, <&dss_sys_clk>;
+				clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 9>,
+					 <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 10>;
 				clock-names = "fck", "sys_clk";
 				dmas = <&sdma 76>;
 				dma-names = "audio_tx";
@@ -1132,7 +1150,7 @@
 	coefficients = <65 (-1791)>;
 };
 
-/include/ "omap54xx-clocks.dtsi"
+#include "omap54xx-clocks.dtsi"
 
 &gpu_thermal {
 	coefficients = <117 (-2992)>;
diff --git a/arch/arm/boot/dts/omap54xx-clocks.dtsi b/arch/arm/boot/dts/omap54xx-clocks.dtsi
index 529193442620..9619a746d657 100644
--- a/arch/arm/boot/dts/omap54xx-clocks.dtsi
+++ b/arch/arm/boot/dts/omap54xx-clocks.dtsi
@@ -432,22 +432,6 @@
 		reg = <0x0528>;
 	};
 
-	dmic_sync_mux_ck: dmic_sync_mux_ck@538 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&dss_syc_gfclk_div>, <&func_24m_clk>;
-		ti,bit-shift = <26>;
-		reg = <0x0538>;
-	};
-
-	dmic_gfclk: dmic_gfclk@538 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dmic_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0538>;
-	};
-
 	mcasp_sync_mux_ck: mcasp_sync_mux_ck@540 {
 		#clock-cells = <0>;
 		compatible = "ti,mux-clock";
@@ -464,86 +448,6 @@
 		reg = <0x0540>;
 	};
 
-	mcbsp1_sync_mux_ck: mcbsp1_sync_mux_ck@548 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&dss_syc_gfclk_div>, <&func_24m_clk>;
-		ti,bit-shift = <26>;
-		reg = <0x0548>;
-	};
-
-	mcbsp1_gfclk: mcbsp1_gfclk@548 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcbsp1_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0548>;
-	};
-
-	mcbsp2_sync_mux_ck: mcbsp2_sync_mux_ck@550 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&dss_syc_gfclk_div>, <&func_24m_clk>;
-		ti,bit-shift = <26>;
-		reg = <0x0550>;
-	};
-
-	mcbsp2_gfclk: mcbsp2_gfclk@550 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcbsp2_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0550>;
-	};
-
-	mcbsp3_sync_mux_ck: mcbsp3_sync_mux_ck@558 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&abe_24m_fclk>, <&dss_syc_gfclk_div>, <&func_24m_clk>;
-		ti,bit-shift = <26>;
-		reg = <0x0558>;
-	};
-
-	mcbsp3_gfclk: mcbsp3_gfclk@558 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&mcbsp3_sync_mux_ck>, <&pad_clks_ck>, <&slimbus_clk>;
-		ti,bit-shift = <24>;
-		reg = <0x0558>;
-	};
-
-	timer5_gfclk_mux: timer5_gfclk_mux@568 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dss_syc_gfclk_div>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0568>;
-	};
-
-	timer6_gfclk_mux: timer6_gfclk_mux@570 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dss_syc_gfclk_div>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0570>;
-	};
-
-	timer7_gfclk_mux: timer7_gfclk_mux@578 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dss_syc_gfclk_div>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0578>;
-	};
-
-	timer8_gfclk_mux: timer8_gfclk_mux@580 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&dss_syc_gfclk_div>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x0580>;
-	};
-
 	dummy_ck: dummy_ck {
 		#clock-cells = <0>;
 		compatible = "fixed-clock";
@@ -603,23 +507,8 @@
 		clock-mult = <1>;
 		clock-div = <1>;
 	};
-
-	gpio1_dbclk: gpio1_dbclk@1938 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1938>;
-	};
-
-	timer1_gfclk_mux: timer1_gfclk_mux@1940 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1940>;
-	};
 };
+
 &cm_core_clocks {
 
 	dpll_per_byp_mux: dpll_per_byp_mux@14c {
@@ -825,95 +714,6 @@
 		ti,dividers = <1>, <8>;
 	};
 
-	dss_32khz_clk: dss_32khz_clk@1420 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <11>;
-		reg = <0x1420>;
-	};
-
-	dss_48mhz_clk: dss_48mhz_clk@1420 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&func_48m_fclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1420>;
-	};
-
-	dss_dss_clk: dss_dss_clk@1420 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_per_h12x2_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1420>;
-		ti,set-rate-parent;
-	};
-
-	dss_sys_clk: dss_sys_clk@1420 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dss_syc_gfclk_div>;
-		ti,bit-shift = <10>;
-		reg = <0x1420>;
-	};
-
-	gpio2_dbclk: gpio2_dbclk@1060 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1060>;
-	};
-
-	gpio3_dbclk: gpio3_dbclk@1068 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1068>;
-	};
-
-	gpio4_dbclk: gpio4_dbclk@1070 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1070>;
-	};
-
-	gpio5_dbclk: gpio5_dbclk@1078 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1078>;
-	};
-
-	gpio6_dbclk: gpio6_dbclk@1080 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1080>;
-	};
-
-	gpio7_dbclk: gpio7_dbclk@1110 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1110>;
-	};
-
-	gpio8_dbclk: gpio8_dbclk@1118 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1118>;
-	};
-
 	iss_ctrlclk: iss_ctrlclk@1320 {
 		#clock-cells = <0>;
 		compatible = "ti,gate-clock";
@@ -938,118 +738,6 @@
 		reg = <0x0f20>;
 	};
 
-	mmc1_32khz_clk: mmc1_32khz_clk@1628 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_32k_ck>;
-		ti,bit-shift = <8>;
-		reg = <0x1628>;
-	};
-
-	sata_ref_clk: sata_ref_clk@1688 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&sys_clkin>;
-		ti,bit-shift = <8>;
-		reg = <0x1688>;
-	};
-
-	usb_host_hs_hsic480m_p1_clk: usb_host_hs_hsic480m_p1_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_usb_m2_ck>;
-		ti,bit-shift = <13>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_hsic480m_p2_clk: usb_host_hs_hsic480m_p2_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_usb_m2_ck>;
-		ti,bit-shift = <14>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_hsic480m_p3_clk: usb_host_hs_hsic480m_p3_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_usb_m2_ck>;
-		ti,bit-shift = <7>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_hsic60m_p1_clk: usb_host_hs_hsic60m_p1_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_60m_fclk>;
-		ti,bit-shift = <11>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_hsic60m_p2_clk: usb_host_hs_hsic60m_p2_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_60m_fclk>;
-		ti,bit-shift = <12>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_hsic60m_p3_clk: usb_host_hs_hsic60m_p3_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_60m_fclk>;
-		ti,bit-shift = <6>;
-		reg = <0x1658>;
-	};
-
-	utmi_p1_gfclk: utmi_p1_gfclk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&l3init_60m_fclk>, <&xclk60mhsp1_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_utmi_p1_clk: usb_host_hs_utmi_p1_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&utmi_p1_gfclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1658>;
-	};
-
-	utmi_p2_gfclk: utmi_p2_gfclk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&l3init_60m_fclk>, <&xclk60mhsp2_ck>;
-		ti,bit-shift = <25>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_utmi_p2_clk: usb_host_hs_utmi_p2_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&utmi_p2_gfclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1658>;
-	};
-
-	usb_host_hs_utmi_p3_clk: usb_host_hs_utmi_p3_clk@1658 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_60m_fclk>;
-		ti,bit-shift = <10>;
-		reg = <0x1658>;
-	};
-
-	usb_otg_ss_refclk960m: usb_otg_ss_refclk960m@16f0 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&dpll_usb_clkdcoldo>;
-		ti,bit-shift = <8>;
-		reg = <0x16f0>;
-	};
-
 	usb_phy_cm_clk32k: usb_phy_cm_clk32k@640 {
 		#clock-cells = <0>;
 		compatible = "ti,gate-clock";
@@ -1058,30 +746,6 @@
 		reg = <0x0640>;
 	};
 
-	usb_tll_hs_usb_ch0_clk: usb_tll_hs_usb_ch0_clk@1668 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_60m_fclk>;
-		ti,bit-shift = <8>;
-		reg = <0x1668>;
-	};
-
-	usb_tll_hs_usb_ch1_clk: usb_tll_hs_usb_ch1_clk@1668 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_60m_fclk>;
-		ti,bit-shift = <9>;
-		reg = <0x1668>;
-	};
-
-	usb_tll_hs_usb_ch2_clk: usb_tll_hs_usb_ch2_clk@1668 {
-		#clock-cells = <0>;
-		compatible = "ti,gate-clock";
-		clocks = <&l3init_60m_fclk>;
-		ti,bit-shift = <10>;
-		reg = <0x1668>;
-	};
-
 	fdif_fclk: fdif_fclk@1328 {
 		#clock-cells = <0>;
 		compatible = "ti,divider-clock";
@@ -1115,88 +779,6 @@
 		ti,max-div = <2>;
 		reg = <0x1638>;
 	};
-
-	mmc1_fclk_mux: mmc1_fclk_mux@1628 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_128m_clk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1628>;
-	};
-
-	mmc1_fclk: mmc1_fclk@1628 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&mmc1_fclk_mux>;
-		ti,bit-shift = <25>;
-		ti,max-div = <2>;
-		reg = <0x1628>;
-	};
-
-	mmc2_fclk_mux: mmc2_fclk_mux@1630 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&func_128m_clk>, <&dpll_per_m2x2_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1630>;
-	};
-
-	mmc2_fclk: mmc2_fclk@1630 {
-		#clock-cells = <0>;
-		compatible = "ti,divider-clock";
-		clocks = <&mmc2_fclk_mux>;
-		ti,bit-shift = <25>;
-		ti,max-div = <2>;
-		reg = <0x1630>;
-	};
-
-	timer10_gfclk_mux: timer10_gfclk_mux@1028 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1028>;
-	};
-
-	timer11_gfclk_mux: timer11_gfclk_mux@1030 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1030>;
-	};
-
-	timer2_gfclk_mux: timer2_gfclk_mux@1038 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1038>;
-	};
-
-	timer3_gfclk_mux: timer3_gfclk_mux@1040 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1040>;
-	};
-
-	timer4_gfclk_mux: timer4_gfclk_mux@1048 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1048>;
-	};
-
-	timer9_gfclk_mux: timer9_gfclk_mux@1050 {
-		#clock-cells = <0>;
-		compatible = "ti,mux-clock";
-		clocks = <&sys_clkin>, <&sys_32k_ck>;
-		ti,bit-shift = <24>;
-		reg = <0x1050>;
-	};
 };
 
 &cm_core_clockdomains {
@@ -1394,3 +976,206 @@
 		reg = <0x021c>;
 	};
 };
+
+&cm_core_aon {
+	mpu_cm: mpu_cm@300 {
+		compatible = "ti,omap4-cm";
+		reg = <0x300 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x300 0x100>;
+
+		mpu_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	dsp_cm: dsp_cm@400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x400 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x400 0x100>;
+
+		dsp_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	abe_cm: abe_cm@500 {
+		compatible = "ti,omap4-cm";
+		reg = <0x500 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x500 0x100>;
+
+		abe_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x64>;
+			#clock-cells = <2>;
+		};
+	};
+
+};
+
+&cm_core {
+	l3main1_cm: l3main1_cm@700 {
+		compatible = "ti,omap4-cm";
+		reg = <0x700 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x700 0x100>;
+
+		l3main1_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3main2_cm: l3main2_cm@800 {
+		compatible = "ti,omap4-cm";
+		reg = <0x800 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x800 0x100>;
+
+		l3main2_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	ipu_cm: ipu_cm@900 {
+		compatible = "ti,omap4-cm";
+		reg = <0x900 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x900 0x100>;
+
+		ipu_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	dma_cm: dma_cm@a00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xa00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xa00 0x100>;
+
+		dma_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	emif_cm: emif_cm@b00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xb00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xb00 0x100>;
+
+		emif_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x1c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4cfg_cm: l4cfg_cm@d00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xd00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xd00 0x100>;
+
+		l4cfg_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x14>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3instr_cm: l3instr_cm@e00 {
+		compatible = "ti,omap4-cm";
+		reg = <0xe00 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0xe00 0x100>;
+
+		l3instr_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xc>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l4per_cm: l4per_cm@1000 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1000 0x200>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1000 0x200>;
+
+		l4per_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x15c>;
+			#clock-cells = <2>;
+		};
+	};
+
+	dss_cm: dss_cm@1400 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1400 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1400 0x100>;
+
+		dss_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x4>;
+			#clock-cells = <2>;
+		};
+	};
+
+	l3init_cm: l3init_cm@1600 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1600 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1600 0x100>;
+
+		l3init_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0xd4>;
+			#clock-cells = <2>;
+		};
+	};
+};
+
+&prm {
+	wkupaon_cm: wkupaon_cm@1900 {
+		compatible = "ti,omap4-cm";
+		reg = <0x1900 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x1900 0x100>;
+
+		wkupaon_clkctrl: clk@20 {
+			compatible = "ti,clkctrl";
+			reg = <0x20 0x5c>;
+			#clock-cells = <2>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/openbmc-flash-layout.dtsi b/arch/arm/boot/dts/openbmc-flash-layout.dtsi
new file mode 100644
index 000000000000..6c26524e93e1
--- /dev/null
+++ b/arch/arm/boot/dts/openbmc-flash-layout.dtsi
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+partitions {
+	compatible = "fixed-partitions";
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	u-boot@0 {
+		reg = <0x0 0x60000>;
+		label = "u-boot";
+	};
+
+	u-boot-env@60000 {
+		reg = <0x60000 0x20000>;
+		label = "u-boot-env";
+	};
+
+	kernel@80000 {
+		reg = <0x80000 0x440000>;
+		label = "kernel";
+	};
+
+	rofs@c0000 {
+		reg = <0x4c0000 0x1740000>;
+		label = "rofs";
+	};
+
+	rwfs@1c00000 {
+		reg = <0x1c00000 0x400000>;
+		label = "rwfs";
+	};
+};
diff --git a/arch/arm/boot/dts/owl-s500-sparky.dts b/arch/arm/boot/dts/owl-s500-sparky.dts
new file mode 100644
index 000000000000..c665ce8b88b4
--- /dev/null
+++ b/arch/arm/boot/dts/owl-s500-sparky.dts
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Allo.com Sparky
+ *
+ * Copyright (c) 2017 Andreas Färber
+ */
+
+/dts-v1/;
+
+#include "owl-s500.dtsi"
+
+/ {
+	compatible = "allo,sparky", "actions,s500";
+	model = "Allo.com Sparky";
+
+	aliases {
+		serial3 = &uart3;
+	};
+
+	chosen {
+		stdout-path = "serial3:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x40000000>; /* 1 or 2 GiB */
+	};
+
+	uart3_clk: uart3-clk {
+		compatible = "fixed-clock";
+		clock-frequency = <921600>;
+		#clock-cells = <0>;
+	};
+};
+
+&timer {
+	clocks = <&hosc>;
+};
+
+&uart3 {
+	status = "okay";
+	clocks = <&uart3_clk>;
+};
diff --git a/arch/arm/boot/dts/wd-mbwe.dts b/arch/arm/boot/dts/ox810se-wd-mbwe.dts
similarity index 100%
rename from arch/arm/boot/dts/wd-mbwe.dts
rename to arch/arm/boot/dts/ox810se-wd-mbwe.dts
diff --git a/arch/arm/boot/dts/cloudengines-pogoplug-series-3.dts b/arch/arm/boot/dts/ox820-cloudengines-pogoplug-series-3.dts
similarity index 100%
rename from arch/arm/boot/dts/cloudengines-pogoplug-series-3.dts
rename to arch/arm/boot/dts/ox820-cloudengines-pogoplug-series-3.dts
diff --git a/arch/arm/boot/dts/pm9g45.dts b/arch/arm/boot/dts/pm9g45.dts
index be5177221cbb..927d00cab014 100644
--- a/arch/arm/boot/dts/pm9g45.dts
+++ b/arch/arm/boot/dts/pm9g45.dts
@@ -52,6 +52,18 @@
 				};
 			};
 
+			tcb0: timer@fff7c000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			mmc0: mmc@fff80000 {
 				pinctrl-0 = <
 					&pinctrl_board_mmc
diff --git a/arch/arm/boot/dts/qcom-msm8974-fairphone-fp2.dts b/arch/arm/boot/dts/qcom-msm8974-fairphone-fp2.dts
index d0a5df90b543..643c57f84818 100644
--- a/arch/arm/boot/dts/qcom-msm8974-fairphone-fp2.dts
+++ b/arch/arm/boot/dts/qcom-msm8974-fairphone-fp2.dts
@@ -265,6 +265,28 @@
 				bias-pull-up;
 			};
 		};
+
+		sdhc2_cd_pin_a: sdhc2-cd-pin-active {
+			pins = "gpio62";
+			function = "gpio";
+
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		sdhc2_pin_a: sdhc2-pin-active {
+			clk {
+				pins = "sdc2_clk";
+				drive-strength = <10>;
+				bias-disable;
+			};
+
+			cmd-data {
+				pins = "sdc2_cmd", "sdc2_data";
+				drive-strength = <6>;
+				bias-pull-up;
+			};
+		};
 	};
 
 	sdhci@f9824900 {
@@ -280,6 +302,18 @@
 		pinctrl-0 = <&sdhc1_pin_a>;
 	};
 
+	sdhci@f98a4900 {
+		status = "ok";
+
+		vmmc-supply = <&pm8941_l21>;
+		vqmmc-supply = <&pm8941_l13>;
+
+		bus-width = <4>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&sdhc2_pin_a>, <&sdhc2_cd_pin_a>;
+	};
+
 	usb@f9a55000 {
 		status = "ok";
 
diff --git a/arch/arm/boot/dts/r7s72100-genmai.dts b/arch/arm/boot/dts/r7s72100-genmai.dts
index cd4d5ff7749e..5af2a0116ff2 100644
--- a/arch/arm/boot/dts/r7s72100-genmai.dts
+++ b/arch/arm/boot/dts/r7s72100-genmai.dts
@@ -123,7 +123,7 @@
 	pinctrl-0 = <&i2c2_pins>;
 
 	eeprom@50 {
-		compatible = "renesas,24c128", "atmel,24c128";
+		compatible = "renesas,r1ex24128", "atmel,24c128";
 		reg = <0x50>;
 		pagesize = <64>;
 	};
diff --git a/arch/arm/boot/dts/r8a73a4.dtsi b/arch/arm/boot/dts/r8a73a4.dtsi
index dd4d09712a2a..8e48090e4fdc 100644
--- a/arch/arm/boot/dts/r8a73a4.dtsi
+++ b/arch/arm/boot/dts/r8a73a4.dtsi
@@ -132,15 +132,12 @@
 	};
 
 	cmt1: timer@e6130000 {
-		compatible = "renesas,cmt-48-r8a73a4", "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a73a4-cmt1", "renesas,rcar-gen2-cmt1";
 		reg = <0 0xe6130000 0 0x1004>;
 		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A73A4_CLK_CMT1>;
 		clock-names = "fck";
 		power-domains = <&pd_c5>;
-
-		renesas,channels-mask = <0xff>;
-
 		status = "disabled";
 	};
 
diff --git a/arch/arm/boot/dts/r8a7740-armadillo800eva.dts b/arch/arm/boot/dts/r8a7740-armadillo800eva.dts
index 1788e186a512..03b00d87b39b 100644
--- a/arch/arm/boot/dts/r8a7740-armadillo800eva.dts
+++ b/arch/arm/boot/dts/r8a7740-armadillo800eva.dts
@@ -131,9 +131,8 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "i2c-gpio";
-		gpios = <&pfc 208 GPIO_ACTIVE_HIGH /* sda */
-			 &pfc 91 GPIO_ACTIVE_HIGH /* scl */
-			>;
+		sda-gpios = <&pfc 208 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&pfc 91 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 		i2c-gpio,delay-us = <5>;
 	};
 
diff --git a/arch/arm/boot/dts/r8a7740.dtsi b/arch/arm/boot/dts/r8a7740.dtsi
index d37d22682a63..afd3bc5e6cf2 100644
--- a/arch/arm/boot/dts/r8a7740.dtsi
+++ b/arch/arm/boot/dts/r8a7740.dtsi
@@ -74,9 +74,6 @@
 		clocks = <&mstp3_clks R8A7740_CLK_CMT1>;
 		clock-names = "fck";
 		power-domains = <&pd_c5>;
-
-		renesas,channels-mask = <0x3f>;
-
 		status = "disabled";
 	};
 
@@ -320,7 +317,7 @@
 
 	tpu: pwm@e6600000 {
 		compatible = "renesas,tpu-r8a7740", "renesas,tpu";
-		reg = <0xe6600000 0x100>;
+		reg = <0xe6600000 0x148>;
 		clocks = <&mstp3_clks R8A7740_CLK_TPU0>;
 		power-domains = <&pd_a3sp>;
 		status = "disabled";
diff --git a/arch/arm/boot/dts/r8a7743.dtsi b/arch/arm/boot/dts/r8a7743.dtsi
index 7bbba4a36f31..0b74c6c7d21d 100644
--- a/arch/arm/boot/dts/r8a7743.dtsi
+++ b/arch/arm/boot/dts/r8a7743.dtsi
@@ -32,6 +32,40 @@
 		spi1 = &msiof0;
 		spi2 = &msiof1;
 		spi3 = &msiof2;
+		vin0 = &vin0;
+		vin1 = &vin1;
+		vin2 = &vin2;
+	};
+
+	/*
+	 * The external audio clocks are configured as 0 Hz fixed frequency
+	 * clocks by default.
+	 * Boards that provide audio clocks should override them.
+	 */
+	audio_clk_a: audio_clk_a {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_b: audio_clk_b {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_c: audio_clk_c {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	/* External CAN clock */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
 	};
 
 	cpus {
@@ -76,6 +110,29 @@
 		};
 	};
 
+	/* External root clock */
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
+	/* External PCIe clock - can be overridden by the board */
+	pcie_bus_clk: pcie_bus {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	/* External SCIF clock */
+	scif_clk: scif {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
 	soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gic>;
@@ -247,16 +304,48 @@
 			resets = <&cpg 407>;
 		};
 
-		timer {
-			compatible = "arm,armv7-timer";
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>;
+		thermal: thermal@e61f0000 {
+			compatible = "renesas,thermal-r8a7743",
+				     "renesas,rcar-gen2-thermal",
+				     "renesas,rcar-thermal";
+			reg = <0 0xe61f0000 0 0x10>, <0 0xe61f0100 0 0x38>;
+			interrupts = <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 522>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 522>;
+			#thermal-sensor-cells = <0>;
+		};
+
+		cmt0: timer@ffca0000 {
+			compatible = "renesas,r8a7743-cmt0",
+				     "renesas,rcar-gen2-cmt0";
+			reg = <0 0xffca0000 0 0x1004>;
+			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 124>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 124>;
+			status = "disabled";
+		};
+
+		cmt1: timer@e6130000 {
+			compatible = "renesas,r8a7743-cmt1",
+				     "renesas,rcar-gen2-cmt1";
+			reg = <0 0xe6130000 0 0x1004>;
+			interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 329>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 329>;
+			status = "disabled";
 		};
 
 		cpg: clock-controller@e6150000 {
@@ -356,6 +445,68 @@
 			dma-channels = <15>;
 		};
 
+		audma0: dma-controller@ec700000 {
+			compatible = "renesas,dmac-r8a7743",
+				     "renesas,rcar-dmac";
+			reg = <0 0xec700000 0 0x10000>;
+			interrupts = <GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12";
+			clocks = <&cpg CPG_MOD 502>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 502>;
+			#dma-cells = <1>;
+			dma-channels = <13>;
+		};
+
+		audma1: dma-controller@ec720000 {
+			compatible = "renesas,dmac-r8a7743",
+				     "renesas,rcar-dmac";
+			reg = <0 0xec720000 0 0x10000>;
+			interrupts = <GIC_SPI 347 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 333 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 334 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12";
+			clocks = <&cpg CPG_MOD 501>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 501>;
+			#dma-cells = <1>;
+			dma-channels = <13>;
+		};
+
 		usb_dmac0: dma-controller@e65a0000 {
 			compatible = "renesas,r8a7743-usb-dmac",
 				     "renesas,usb-dmac";
@@ -827,7 +978,8 @@
 		};
 
 		ether: ethernet@ee700000 {
-			compatible = "renesas,ether-r8a7743";
+			compatible = "renesas,ether-r8a7743",
+				     "renesas,rcar-gen2-ether";
 			reg = <0 0xee700000 0 0x400>;
 			interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 813>;
@@ -952,8 +1104,89 @@
 			status = "disabled";
 		};
 
+		pwm0: pwm@e6e30000 {
+			compatible = "renesas,pwm-r8a7743", "renesas,pwm-rcar";
+			reg = <0 0xe6e30000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm1: pwm@e6e31000 {
+			compatible = "renesas,pwm-r8a7743", "renesas,pwm-rcar";
+			reg = <0 0xe6e31000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm2: pwm@e6e32000 {
+			compatible = "renesas,pwm-r8a7743", "renesas,pwm-rcar";
+			reg = <0 0xe6e32000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm3: pwm@e6e33000 {
+			compatible = "renesas,pwm-r8a7743", "renesas,pwm-rcar";
+			reg = <0 0xe6e33000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm4: pwm@e6e34000 {
+			compatible = "renesas,pwm-r8a7743", "renesas,pwm-rcar";
+			reg = <0 0xe6e34000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm5: pwm@e6e35000 {
+			compatible = "renesas,pwm-r8a7743", "renesas,pwm-rcar";
+			reg = <0 0xe6e35000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm6: pwm@e6e36000 {
+			compatible = "renesas,pwm-r8a7743", "renesas,pwm-rcar";
+			reg = <0 0xe6e36000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		tpu: pwm@e60f0000 {
+			compatible = "renesas,tpu-r8a7743", "renesas,tpu";
+			reg = <0 0xe60f0000 0 0x148>;
+			clocks = <&cpg CPG_MOD 304>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 304>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		sdhi0: sd@ee100000 {
-			compatible = "renesas,sdhi-r8a7743";
+			compatible = "renesas,sdhi-r8a7743",
+				     "renesas,rcar-gen2-sdhi";
 			reg = <0 0xee100000 0 0x328>;
 			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 314>;
@@ -967,7 +1200,8 @@
 		};
 
 		sdhi1: sd@ee140000 {
-			compatible = "renesas,sdhi-r8a7743";
+			compatible = "renesas,sdhi-r8a7743",
+				     "renesas,rcar-gen2-sdhi";
 			reg = <0 0xee140000 0 0x100>;
 			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 312>;
@@ -981,7 +1215,8 @@
 		};
 
 		sdhi2: sd@ee160000 {
-			compatible = "renesas,sdhi-r8a7743";
+			compatible = "renesas,sdhi-r8a7743",
+				     "renesas,rcar-gen2-sdhi";
 			reg = <0 0xee160000 0 0x100>;
 			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 311>;
@@ -1033,6 +1268,97 @@
 			};
 		};
 
+		vin0: video@e6ef0000 {
+			compatible = "renesas,vin-r8a7743",
+				     "renesas,rcar-gen2-vin";
+			reg = <0 0xe6ef0000 0 0x1000>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 811>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 811>;
+			status = "disabled";
+		};
+
+		vin1: video@e6ef1000 {
+			compatible = "renesas,vin-r8a7743",
+				     "renesas,rcar-gen2-vin";
+			reg = <0 0xe6ef1000 0 0x1000>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 810>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 810>;
+			status = "disabled";
+		};
+
+		vin2: video@e6ef2000 {
+			compatible = "renesas,vin-r8a7743",
+				     "renesas,rcar-gen2-vin";
+			reg = <0 0xe6ef2000 0 0x1000>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 809>;
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 809>;
+			status = "disabled";
+		};
+
+		du: display@feb00000 {
+			compatible = "renesas,du-r8a7743";
+			reg = <0 0xfeb00000 0 0x40000>,
+			      <0 0xfeb90000 0 0x1c>;
+			reg-names = "du", "lvds.0";
+			interrupts = <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 724>,
+				 <&cpg CPG_MOD 723>,
+				 <&cpg CPG_MOD 726>;
+			clock-names = "du.0", "du.1", "lvds.0";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					du_out_rgb: endpoint {
+					};
+				};
+				port@1 {
+					reg = <1>;
+					du_out_lvds0: endpoint {
+					};
+				};
+			};
+		};
+
+		can0: can@e6e80000 {
+			compatible = "renesas,can-r8a7743",
+				     "renesas,rcar-gen2-can";
+			reg = <0 0xe6e80000 0 0x1000>;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>,
+				 <&cpg CPG_CORE R8A7743_CLK_RCAN>,
+				 <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 916>;
+			status = "disabled";
+		};
+
+		can1: can@e6e88000 {
+			compatible = "renesas,can-r8a7743",
+				     "renesas,rcar-gen2-can";
+			reg = <0 0xe6e88000 0 0x1000>;
+			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 915>,
+				 <&cpg CPG_CORE R8A7743_CLK_RCAN>,
+				 <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 915>;
+			status = "disabled";
+		};
+
 		pci0: pci@ee090000 {
 			compatible = "renesas,pci-r8a7743",
 				     "renesas,pci-rcar-gen2";
@@ -1102,14 +1428,247 @@
 				phy-names = "usb";
 			};
 		};
+
+		pciec: pcie@fe000000 {
+			compatible = "renesas,pcie-r8a7743",
+				     "renesas,pcie-rcar-gen2";
+			reg = <0 0xfe000000 0 0x80000>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			bus-range = <0x00 0xff>;
+			device_type = "pci";
+			ranges = <0x01000000 0 0x00000000 0 0xfe100000 0 0x00100000
+				  0x02000000 0 0xfe200000 0 0xfe200000 0 0x00200000
+				  0x02000000 0 0x30000000 0 0x30000000 0 0x08000000
+				  0x42000000 0 0x38000000 0 0x38000000 0 0x08000000>;
+			/* Map all possible DDR as inbound ranges */
+			dma-ranges = <0x42000000 0 0x40000000 0 0x40000000 0 0x80000000
+				      0x43000000 2 0x00000000 2 0x00000000 1 0x00000000>;
+			interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>;
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0>;
+			interrupt-map = <0 0 0 0 &gic GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 319>, <&pcie_bus_clk>;
+			clock-names = "pcie", "pcie_bus";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 319>;
+			status = "disabled";
+		};
+
+		rcar_sound: sound@ec500000 {
+			/*
+			 * #sound-dai-cells is required
+			 *
+			 * Single DAI : #sound-dai-cells = <0>;         <&rcar_sound>;
+			 * Multi  DAI : #sound-dai-cells = <1>;         <&rcar_sound N>;
+			 */
+			compatible = "renesas,rcar_sound-r8a7743",
+				     "renesas,rcar_sound-gen2";
+			reg = <0 0xec500000 0 0x1000>, /* SCU */
+			      <0 0xec5a0000 0 0x100>,  /* ADG */
+			      <0 0xec540000 0 0x1000>, /* SSIU */
+			      <0 0xec541000 0 0x280>,  /* SSI */
+			      <0 0xec740000 0 0x200>;  /* Audio DMAC peri peri*/
+			reg-names = "scu", "adg", "ssiu", "ssi", "audmapp";
+
+			clocks = <&cpg CPG_MOD 1005>,
+				 <&cpg CPG_MOD 1006>, <&cpg CPG_MOD 1007>,
+				 <&cpg CPG_MOD 1008>, <&cpg CPG_MOD 1009>,
+				 <&cpg CPG_MOD 1010>, <&cpg CPG_MOD 1011>,
+				 <&cpg CPG_MOD 1012>, <&cpg CPG_MOD 1013>,
+				 <&cpg CPG_MOD 1014>, <&cpg CPG_MOD 1015>,
+				 <&cpg CPG_MOD 1022>, <&cpg CPG_MOD 1023>,
+				 <&cpg CPG_MOD 1024>, <&cpg CPG_MOD 1025>,
+				 <&cpg CPG_MOD 1026>, <&cpg CPG_MOD 1027>,
+				 <&cpg CPG_MOD 1028>, <&cpg CPG_MOD 1029>,
+				 <&cpg CPG_MOD 1030>, <&cpg CPG_MOD 1031>,
+				 <&cpg CPG_MOD 1021>, <&cpg CPG_MOD 1020>,
+				 <&cpg CPG_MOD 1021>, <&cpg CPG_MOD 1020>,
+				 <&cpg CPG_MOD 1019>, <&cpg CPG_MOD 1018>,
+				 <&audio_clk_a>, <&audio_clk_b>, <&audio_clk_c>,
+				 <&cpg CPG_CORE R8A7743_CLK_M2>;
+			clock-names = "ssi-all",
+				      "ssi.9", "ssi.8", "ssi.7", "ssi.6", "ssi.5",
+				      "ssi.4", "ssi.3", "ssi.2", "ssi.1", "ssi.0",
+				      "src.9", "src.8", "src.7", "src.6", "src.5",
+				      "src.4", "src.3", "src.2", "src.1", "src.0",
+				      "ctu.0", "ctu.1",
+				      "mix.0", "mix.1",
+				      "dvc.0", "dvc.1",
+				      "clk_a", "clk_b", "clk_c", "clk_i";
+			power-domains = <&sysc R8A7743_PD_ALWAYS_ON>;
+			resets = <&cpg 1005>,
+				 <&cpg 1006>, <&cpg 1007>, <&cpg 1008>, <&cpg 1009>,
+				 <&cpg 1010>, <&cpg 1011>, <&cpg 1012>, <&cpg 1013>,
+				 <&cpg 1014>, <&cpg 1015>;
+			reset-names = "ssi-all",
+				      "ssi.9", "ssi.8", "ssi.7", "ssi.6", "ssi.5",
+				      "ssi.4", "ssi.3", "ssi.2", "ssi.1", "ssi.0";
+			status = "disabled";
+
+			rcar_sound,dvc {
+				dvc0: dvc-0 {
+					dmas = <&audma1 0xbc>;
+					dma-names = "tx";
+				};
+				dvc1: dvc-1 {
+					dmas = <&audma1 0xbe>;
+					dma-names = "tx";
+				};
+			};
+
+			rcar_sound,mix {
+				mix0: mix-0 { };
+				mix1: mix-1 { };
+			};
+
+			rcar_sound,ctu {
+				ctu00: ctu-0 { };
+				ctu01: ctu-1 { };
+				ctu02: ctu-2 { };
+				ctu03: ctu-3 { };
+				ctu10: ctu-4 { };
+				ctu11: ctu-5 { };
+				ctu12: ctu-6 { };
+				ctu13: ctu-7 { };
+			};
+
+			rcar_sound,src {
+				src0: src-0 {
+					interrupts = <GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x85>, <&audma1 0x9a>;
+					dma-names = "rx", "tx";
+				};
+				src1: src-1 {
+					interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x87>, <&audma1 0x9c>;
+					dma-names = "rx", "tx";
+				};
+				src2: src-2 {
+					interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x89>, <&audma1 0x9e>;
+					dma-names = "rx", "tx";
+				};
+				src3: src-3 {
+					interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x8b>, <&audma1 0xa0>;
+					dma-names = "rx", "tx";
+				};
+				src4: src-4 {
+					interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x8d>, <&audma1 0xb0>;
+					dma-names = "rx", "tx";
+				};
+				src5: src-5 {
+					interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x8f>, <&audma1 0xb2>;
+					dma-names = "rx", "tx";
+				};
+				src6: src-6 {
+					interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x91>, <&audma1 0xb4>;
+					dma-names = "rx", "tx";
+				};
+				src7: src-7 {
+					interrupts = <GIC_SPI 359 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x93>, <&audma1 0xb6>;
+					dma-names = "rx", "tx";
+				};
+				src8: src-8 {
+					interrupts = <GIC_SPI 360 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x95>, <&audma1 0xb8>;
+					dma-names = "rx", "tx";
+				};
+				src9: src-9 {
+					interrupts = <GIC_SPI 361 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x97>, <&audma1 0xba>;
+					dma-names = "rx", "tx";
+				};
+			};
+
+			rcar_sound,ssi {
+				ssi0: ssi-0 {
+					interrupts = <GIC_SPI 370 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x01>, <&audma1 0x02>, <&audma0 0x15>, <&audma1 0x16>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi1: ssi-1 {
+					interrupts = <GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x03>, <&audma1 0x04>, <&audma0 0x49>, <&audma1 0x4a>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi2: ssi-2 {
+					interrupts = <GIC_SPI 372 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x05>, <&audma1 0x06>, <&audma0 0x63>, <&audma1 0x64>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi3: ssi-3 {
+					interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x07>, <&audma1 0x08>, <&audma0 0x6f>, <&audma1 0x70>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi4: ssi-4 {
+					interrupts = <GIC_SPI 374 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x09>, <&audma1 0x0a>, <&audma0 0x71>, <&audma1 0x72>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi5: ssi-5 {
+					interrupts = <GIC_SPI 375 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x0b>, <&audma1 0x0c>, <&audma0 0x73>, <&audma1 0x74>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi6: ssi-6 {
+					interrupts = <GIC_SPI 376 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x0d>, <&audma1 0x0e>, <&audma0 0x75>, <&audma1 0x76>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi7: ssi-7 {
+					interrupts = <GIC_SPI 377 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x0f>, <&audma1 0x10>, <&audma0 0x79>, <&audma1 0x7a>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi8: ssi-8 {
+					interrupts = <GIC_SPI 378 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x11>, <&audma1 0x12>, <&audma0 0x7b>, <&audma1 0x7c>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi9: ssi-9 {
+					interrupts = <GIC_SPI 379 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x13>, <&audma1 0x14>, <&audma0 0x7d>, <&audma1 0x7e>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+			};
+		};
 	};
 
-	/* External root clock */
-	extal_clk: extal {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		/* This value must be overridden by the board. */
-		clock-frequency = <0>;
+	thermal-zones {
+		cpu_thermal: cpu-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&thermal>;
+
+			trips {
+				cpu-crit {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+
+			cooling-maps {
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 
 	/* External USB clock - can be overridden by the board */
@@ -1118,12 +1677,4 @@
 		#clock-cells = <0>;
 		clock-frequency = <48000000>;
 	};
-
-	/* External SCIF clock */
-	scif_clk: scif {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		/* This value must be overridden by the board. */
-		clock-frequency = <0>;
-	};
 };
diff --git a/arch/arm/boot/dts/r8a7745-iwg22d-sodimm-dbhd-ca.dts b/arch/arm/boot/dts/r8a7745-iwg22d-sodimm-dbhd-ca.dts
new file mode 100644
index 000000000000..d34de8266ccd
--- /dev/null
+++ b/arch/arm/boot/dts/r8a7745-iwg22d-sodimm-dbhd-ca.dts
@@ -0,0 +1,158 @@
+/*
+ * Device Tree Source for the iWave-RZG1E SODIMM carrier board + HDMI daughter
+ * board
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2.  This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+#include "r8a7745-iwg22d-sodimm.dts"
+
+/ {
+	model = "iWave RainboW-G22D-SODIMM RZ/G1E based board with HDMI add-on";
+	compatible = "iwave,g22d", "iwave,g22m", "renesas,r8a7745";
+
+	aliases {
+		serial0 = &scif1;
+		serial4 = &scif5;
+		serial6 = &hscif2;
+	};
+
+	cec_clock: cec-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <12000000>;
+	};
+
+	hdmi-out {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&adv7511_out>;
+			};
+		};
+	};
+};
+
+&du {
+	pinctrl-0 = <&du0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	ports {
+		port@0 {
+			endpoint {
+				remote-endpoint = <&adv7511_in>;
+			};
+		};
+	};
+};
+
+&can1 {
+	pinctrl-0 = <&can1_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&hscif2 {
+	pinctrl-0 = <&hscif2_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&i2c1 {
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	hdmi@39 {
+		compatible = "adi,adv7511w";
+		reg = <0x39>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		clocks = <&cec_clock>;
+		clock-names = "cec";
+		pd-gpios = <&gpio2 24 GPIO_ACTIVE_HIGH>;
+
+		adi,input-depth = <8>;
+		adi,input-colorspace = "rgb";
+		adi,input-clock = "1x";
+		adi,input-style = <1>;
+		adi,input-justification = "evenly";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7511_in: endpoint {
+					remote-endpoint = <&du_out_rgb0>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7511_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+};
+
+&pfc {
+	can1_pins: can1 {
+		groups = "can1_data_b";
+		function = "can1";
+	};
+
+	du0_pins: du0 {
+		groups = "du0_rgb888", "du0_sync", "du0_disp", "du0_clk0_out";
+		function = "du0";
+	};
+
+	hscif2_pins: hscif2 {
+		groups = "hscif2_data";
+		function = "hscif2";
+	};
+
+	i2c1_pins: i2c1 {
+		groups = "i2c1_d";
+		function = "i2c1";
+	};
+
+	scif1_pins: scif1 {
+		groups = "scif1_data";
+		function = "scif1";
+	};
+
+	scif5_pins: scif5 {
+		groups = "scif5_data_d";
+		function = "scif5";
+	};
+};
+
+&scif1 {
+	pinctrl-0 = <&scif1_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&scif5 {
+	pinctrl-0 = <&scif5_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/r8a7745-iwg22d-sodimm.dts b/arch/arm/boot/dts/r8a7745-iwg22d-sodimm.dts
index 52153ec3638c..a4058f4cfbcd 100644
--- a/arch/arm/boot/dts/r8a7745-iwg22d-sodimm.dts
+++ b/arch/arm/boot/dts/r8a7745-iwg22d-sodimm.dts
@@ -8,6 +8,29 @@
  * kind, whether express or implied.
  */
 
+/*
+ * SSI-SGTL5000
+ *
+ * This command is required when Playback/Capture
+ *
+ *      amixer set "DVC Out" 100%
+ *      amixer set "DVC In" 100%
+ *
+ * You can use Mute
+ *
+ *      amixer set "DVC Out Mute" on
+ *      amixer set "DVC In Mute" on
+ *
+ * You can use Volume Ramp
+ *
+ *      amixer set "DVC Out Ramp Up Rate"   "0.125 dB/64 steps"
+ *      amixer set "DVC Out Ramp Down Rate" "0.125 dB/512 steps"
+ *      amixer set "DVC Out Ramp" on
+ *      aplay xxx.wav &
+ *      amixer set "DVC Out"  80%  // Volume Down
+ *      amixer set "DVC Out" 100%  // Volume Up
+ */
+
 /dts-v1/;
 #include "r8a7745-iwg22m.dtsi"
 
@@ -16,13 +39,35 @@
 	compatible = "iwave,g22d", "iwave,g22m", "renesas,r8a7745";
 
 	aliases {
-		serial0 = &scif4;
 		ethernet0 = &avb;
+		serial3 = &scif4;
+		serial5 = &hscif1;
 	};
 
 	chosen {
 		bootargs = "ignore_loglevel rw root=/dev/nfs ip=dhcp";
-		stdout-path = "serial0:115200n8";
+		stdout-path = "serial3:115200n8";
+	};
+
+	audio_clock: audio_clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
+	};
+
+	rsnd_sgtl5000: sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&sndcodec>;
+		simple-audio-card,frame-master = <&sndcodec>;
+
+		sndcpu: simple-audio-card,cpu {
+			sound-dai = <&rcar_sound>;
+		};
+
+		sndcodec: simple-audio-card,codec {
+			sound-dai = <&sgtl5000>;
+		};
 	};
 
 	vccq_sdhi0: regulator-vccq-sdhi0 {
@@ -39,53 +84,139 @@
 	};
 };
 
-&pfc {
-	scif4_pins: scif4 {
-		groups = "scif4_data_b";
-		function = "scif4";
+&avb {
+	pinctrl-0 = <&avb_pins>;
+	pinctrl-names = "default";
+
+	phy-handle = <&phy3>;
+	phy-mode = "gmii";
+	renesas,no-ether-link;
+	status = "okay";
+
+	phy3: ethernet-phy@3 {
+	/*
+	 * On some older versions of the platform (before R4.0) the phy address
+	 * may be 1 or 3. The address is fixed to 3 for R4.0 onwards.
+	 */
+		reg = <3>;
+		micrel,led-mode = <1>;
 	};
+};
+
+&can0 {
+	pinctrl-0 = <&can0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&hscif1 {
+	pinctrl-0 = <&hscif1_pins>;
+	pinctrl-names = "default";
 
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&hsusb {
+	status = "okay";
+	pinctrl-0 = <&usb0_pins>;
+	pinctrl-names = "default";
+};
+
+&i2c5 {
+	pinctrl-0 = <&i2c5_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	sgtl5000: codec@a {
+		compatible = "fsl,sgtl5000";
+		#sound-dai-cells = <0>;
+		reg = <0x0a>;
+		clocks = <&audio_clock>;
+		VDDA-supply = <&reg_3p3v>;
+		VDDIO-supply = <&reg_3p3v>;
+	};
+};
+
+&pci1 {
+	status = "okay";
+	pinctrl-0 = <&usb1_pins>;
+	pinctrl-names = "default";
+};
+
+&pfc {
 	avb_pins: avb {
 		groups = "avb_mdio", "avb_gmii";
 		function = "avb";
 	};
 
+	can0_pins: can0 {
+		groups = "can0_data";
+		function = "can0";
+	};
+
+	hscif1_pins: hscif1 {
+		groups = "hscif1_data", "hscif1_ctrl";
+		function = "hscif1";
+	};
+
+	i2c5_pins: i2c5 {
+		groups = "i2c5_b";
+		function = "i2c5";
+	};
+
+	scif4_pins: scif4 {
+		groups = "scif4_data_b";
+		function = "scif4";
+	};
+
 	sdhi0_pins: sd0 {
 		groups = "sdhi0_data4", "sdhi0_ctrl";
 		function = "sdhi0";
 		power-source = <3300>;
 	};
 
+	sound_pins: sound {
+		groups = "ssi34_ctrl", "ssi3_data", "ssi4_data";
+		function = "ssi";
+	};
+
+	usb0_pins: usb0 {
+		groups = "usb0";
+		function = "usb0";
+	};
+
 	usb1_pins: usb1 {
 		groups = "usb1";
 		function = "usb1";
 	};
 };
 
-&scif4 {
-	pinctrl-0 = <&scif4_pins>;
+&rcar_sound {
+	pinctrl-0 = <&sound_pins>;
 	pinctrl-names = "default";
-
 	status = "okay";
+
+	/* Single DAI */
+
+	#sound-dai-cells = <0>;
+
+	rcar_sound,dai {
+		dai0 {
+			playback = <&ssi3 &src3 &dvc0>;
+			capture = <&ssi4 &src4 &dvc1>;
+		};
+	};
 };
 
-&avb {
-	pinctrl-0 = <&avb_pins>;
+&scif4 {
+	pinctrl-0 = <&scif4_pins>;
 	pinctrl-names = "default";
 
-	phy-handle = <&phy3>;
-	phy-mode = "gmii";
-	renesas,no-ether-link;
 	status = "okay";
-
-	phy3: ethernet-phy@3 {
-	/*
-	 * On some older versions of the platform (before R4.0) the phy address
-	 * may be 1 or 3. The address is fixed to 3 for R4.0 onwards.
-	 */
-		reg = <3>;
-		micrel,led-mode = <1>;
-	};
 };
 
 &sdhi0 {
@@ -98,10 +229,8 @@
 	status = "okay";
 };
 
-&pci1 {
-	status = "okay";
-	pinctrl-0 = <&usb1_pins>;
-	pinctrl-names = "default";
+&ssi4 {
+	shared-pin;
 };
 
 &usbphy {
diff --git a/arch/arm/boot/dts/r8a7745.dtsi b/arch/arm/boot/dts/r8a7745.dtsi
index 3a50f703601c..ae918e9cce21 100644
--- a/arch/arm/boot/dts/r8a7745.dtsi
+++ b/arch/arm/boot/dts/r8a7745.dtsi
@@ -25,15 +25,49 @@
 		i2c3 = &i2c3;
 		i2c4 = &i2c4;
 		i2c5 = &i2c5;
+		i2c6 = &iic0;
+		i2c7 = &iic1;
 		spi0 = &qspi;
 		spi1 = &msiof0;
 		spi2 = &msiof1;
 		spi3 = &msiof2;
+		vin0 = &vin0;
+		vin1 = &vin1;
+	};
+
+	/*
+	 * The external audio clocks are configured  as 0 Hz fixed
+	 * frequency clocks by default.  Boards that provide audio
+	 * clocks should override them.
+	 */
+	audio_clka: audio_clka {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+	audio_clkb: audio_clkb {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+	audio_clkc: audio_clkc {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	/* External CAN clock */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
 	};
 
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
+		enable-method = "renesas,apmu";
 
 		cpu0: cpu@0 {
 			device_type = "cpu";
@@ -45,6 +79,16 @@
 			next-level-cache = <&L2_CA7>;
 		};
 
+		cpu1: cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a7";
+			reg = <1>;
+			clock-frequency = <1000000000>;
+			clocks = <&cpg CPG_CORE R8A7745_CLK_Z2>;
+			power-domains = <&sysc R8A7745_PD_CA7_CPU1>;
+			next-level-cache = <&L2_CA7>;
+		};
+
 		L2_CA7: cache-controller-0 {
 			compatible = "cache";
 			cache-unified;
@@ -53,6 +97,22 @@
 		};
 	};
 
+	/* External root clock */
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
+	/* External SCIF clock */
+	scif_clk: scif {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
 	soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gic>;
@@ -61,6 +121,12 @@
 		#size-cells = <2>;
 		ranges;
 
+		apmu@e6151000 {
+			compatible = "renesas,r8a7745-apmu", "renesas,apmu";
+			reg = <0 0xe6151000 0 0x188>;
+			cpus = <&cpu0 &cpu1>;
+		};
+
 		gic: interrupt-controller@f1001000 {
 			compatible = "arm,gic-400";
 			#interrupt-cells = <3>;
@@ -203,16 +269,36 @@
 			resets = <&cpg 407>;
 		};
 
-		timer {
-			compatible = "arm,armv7-timer";
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) |
-						  IRQ_TYPE_LEVEL_LOW)>;
+		cmt0: timer@ffca0000 {
+			compatible = "renesas,r8a7745-cmt0",
+				     "renesas,rcar-gen2-cmt0";
+			reg = <0 0xffca0000 0 0x1004>;
+			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 124>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 124>;
+			status = "disabled";
+		};
+
+		cmt1: timer@e6130000 {
+			compatible = "renesas,r8a7745-cmt1",
+				     "renesas,rcar-gen2-cmt1";
+			reg = <0 0xe6130000 0 0x1004>;
+			interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 329>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 329>;
+			status = "disabled";
 		};
 
 		cpg: clock-controller@e6150000 {
@@ -312,6 +398,65 @@
 			dma-channels = <15>;
 		};
 
+		audma0: dma-controller@ec700000 {
+			compatible = "renesas,dmac-r8a7745",
+				     "renesas,rcar-dmac";
+			reg = <0 0xec700000 0 0x10000>;
+			interrupts = <GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 320 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 321 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 322 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 323 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 324 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 325 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 326 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 327 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 328 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 329 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 331 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 332 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					  "ch0", "ch1", "ch2", "ch3",
+					  "ch4", "ch5", "ch6", "ch7",
+					  "ch8", "ch9", "ch10", "ch11",
+					  "ch12";
+			clocks = <&cpg CPG_MOD 502>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 502>;
+			#dma-cells = <1>;
+			dma-channels = <13>;
+		};
+
+		usb_dmac0: dma-controller@e65a0000 {
+			compatible = "renesas,r8a7745-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65a0000 0 0x100>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 330>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 330>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
+		usb_dmac1: dma-controller@e65b0000 {
+			compatible = "renesas,r8a7745-usb-dmac",
+				     "renesas,usb-dmac";
+			reg = <0 0xe65b0000 0 0x100>;
+			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1";
+			clocks = <&cpg CPG_MOD 331>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 331>;
+			#dma-cells = <1>;
+			dma-channels = <2>;
+		};
+
 		scifa0: serial@e6c40000 {
 			compatible = "renesas,scifa-r8a7745",
 				     "renesas,rcar-gen2-scifa", "renesas,scifa";
@@ -615,7 +760,8 @@
 		};
 
 		ether: ethernet@ee700000 {
-			compatible = "renesas,ether-r8a7745";
+			compatible = "renesas,ether-r8a7745",
+				     "renesas,rcar-gen2-ether";
 			reg = <0 0xee700000 0 0x400>;
 			interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 813>;
@@ -724,6 +870,40 @@
 			status = "disabled";
 		};
 
+		iic0: i2c@e6500000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,iic-r8a7745",
+				     "renesas,rcar-gen2-iic",
+				     "renesas,rmobile-iic";
+			reg = <0 0xe6500000 0 0x425>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 318>;
+			dmas = <&dmac0 0x61>, <&dmac0 0x62>,
+			       <&dmac1 0x61>, <&dmac1 0x62>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 318>;
+			status = "disabled";
+		};
+
+		iic1: i2c@e6510000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,iic-r8a7745",
+				     "renesas,rcar-gen2-iic",
+				     "renesas,rmobile-iic";
+			reg = <0 0xe6510000 0 0x425>;
+			interrupts = <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 323>;
+			dmas = <&dmac0 0x65>, <&dmac0 0x66>,
+			       <&dmac1 0x65>, <&dmac1 0x66>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 323>;
+			status = "disabled";
+		};
+
 		mmcif0: mmc@ee200000 {
 			compatible = "renesas,mmcif-r8a7745",
 				     "renesas,sh-mmcif";
@@ -756,6 +936,55 @@
 			status = "disabled";
 		};
 
+		vin0: video@e6ef0000 {
+			compatible = "renesas,vin-r8a7745",
+				     "renesas,rcar-gen2-vin";
+			reg = <0 0xe6ef0000 0 0x1000>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 811>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 811>;
+			status = "disabled";
+		};
+
+		vin1: video@e6ef1000 {
+			compatible = "renesas,vin-r8a7745",
+				     "renesas,rcar-gen2-vin";
+			reg = <0 0xe6ef1000 0 0x1000>;
+			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 810>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 810>;
+			status = "disabled";
+		};
+
+		du: display@feb00000 {
+			compatible = "renesas,du-r8a7745";
+			reg = <0 0xfeb00000 0 0x40000>;
+			reg-names = "du";
+			interrupts = <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 724>, <&cpg CPG_MOD 723>;
+			clock-names = "du.0", "du.1";
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					du_out_rgb0: endpoint {
+					};
+				};
+				port@1 {
+					reg = <1>;
+					du_out_rgb1: endpoint {
+					};
+				};
+			};
+		};
+
 		msiof0: spi@e6e20000 {
 			compatible = "renesas,msiof-r8a7745",
 				     "renesas,rcar-gen2-msiof";
@@ -804,8 +1033,89 @@
 			status = "disabled";
 		};
 
+		pwm0: pwm@e6e30000 {
+			compatible = "renesas,pwm-r8a7745", "renesas,pwm-rcar";
+			reg = <0 0xe6e30000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm1: pwm@e6e31000 {
+			compatible = "renesas,pwm-r8a7745", "renesas,pwm-rcar";
+			reg = <0 0xe6e31000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm2: pwm@e6e32000 {
+			compatible = "renesas,pwm-r8a7745", "renesas,pwm-rcar";
+			reg = <0 0xe6e32000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm3: pwm@e6e33000 {
+			compatible = "renesas,pwm-r8a7745", "renesas,pwm-rcar";
+			reg = <0 0xe6e33000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm4: pwm@e6e34000 {
+			compatible = "renesas,pwm-r8a7745", "renesas,pwm-rcar";
+			reg = <0 0xe6e34000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm5: pwm@e6e35000 {
+			compatible = "renesas,pwm-r8a7745", "renesas,pwm-rcar";
+			reg = <0 0xe6e35000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		pwm6: pwm@e6e36000 {
+			compatible = "renesas,pwm-r8a7745", "renesas,pwm-rcar";
+			reg = <0 0xe6e36000 0 0x8>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			#pwm-cells = <2>;
+			status = "disabled";
+		};
+
+		tpu: pwm@e60f0000 {
+			compatible = "renesas,tpu-r8a7745", "renesas,tpu";
+			reg = <0 0xe60f0000 0 0x148>;
+			clocks = <&cpg CPG_MOD 304>;
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 304>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		sdhi0: sd@ee100000 {
-			compatible = "renesas,sdhi-r8a7745";
+			compatible = "renesas,sdhi-r8a7745",
+				     "renesas,rcar-gen2-sdhi";
 			reg = <0 0xee100000 0 0x328>;
 			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 314>;
@@ -819,7 +1129,8 @@
 		};
 
 		sdhi1: sd@ee140000 {
-			compatible = "renesas,sdhi-r8a7745";
+			compatible = "renesas,sdhi-r8a7745",
+				     "renesas,rcar-gen2-sdhi";
 			reg = <0 0xee140000 0 0x100>;
 			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 312>;
@@ -833,7 +1144,8 @@
 		};
 
 		sdhi2: sd@ee160000 {
-			compatible = "renesas,sdhi-r8a7745";
+			compatible = "renesas,sdhi-r8a7745",
+				     "renesas,rcar-gen2-sdhi";
 			reg = <0 0xee160000 0 0x100>;
 			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 311>;
@@ -916,6 +1228,23 @@
 			};
 		};
 
+		hsusb: usb@e6590000 {
+			compatible = "renesas,usbhs-r8a7745",
+				     "renesas,rcar-gen2-usbhs";
+			reg = <0 0xe6590000 0 0x100>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 704>;
+			dmas = <&usb_dmac0 0>, <&usb_dmac0 1>,
+			       <&usb_dmac1 0>, <&usb_dmac1 1>;
+			dma-names = "ch0", "ch1", "ch2", "ch3";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 704>;
+			renesas,buswait = <4>;
+			phys = <&usb0 1>;
+			phy-names = "usb";
+			status = "disabled";
+		};
+
 		usbphy: usb-phy@e6590100 {
 			compatible = "renesas,usb-phy-r8a7745",
 				     "renesas,rcar-gen2-usb-phy";
@@ -937,14 +1266,222 @@
 				#phy-cells = <1>;
 			};
 		};
+
+		can0: can@e6e80000 {
+			compatible = "renesas,can-r8a7745",
+				     "renesas,rcar-gen2-can";
+			reg = <0 0xe6e80000 0 0x1000>;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>,
+				 <&cpg CPG_CORE R8A7745_CLK_RCAN>,
+				 <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 916>;
+			status = "disabled";
+		};
+
+		can1: can@e6e88000 {
+			compatible = "renesas,can-r8a7745",
+				     "renesas,rcar-gen2-can";
+			reg = <0 0xe6e88000 0 0x1000>;
+			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 915>,
+				 <&cpg CPG_CORE R8A7745_CLK_RCAN>,
+				 <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 915>;
+			status = "disabled";
+		};
+
+		rcar_sound: sound@ec500000 {
+			/*
+			 * #sound-dai-cells is required
+			 *
+			 * Single DAI : #sound-dai-cells = <0>; <&rcar_sound>;
+			 * Multi  DAI : #sound-dai-cells = <1>; <&rcar_sound N>;
+			 */
+			compatible = "renesas,rcar_sound-r8a7745",
+				     "renesas,rcar_sound-gen2";
+			reg = <0 0xec500000 0 0x1000>, /* SCU */
+			      <0 0xec5a0000 0 0x100>,  /* ADG */
+			      <0 0xec540000 0 0x1000>, /* SSIU */
+			      <0 0xec541000 0 0x280>,  /* SSI */
+			      <0 0xec740000 0 0x200>;  /* Audio DMAC peri peri */
+			reg-names = "scu", "adg", "ssiu", "ssi", "audmapp";
+
+			clocks = <&cpg CPG_MOD 1005>,
+				 <&cpg CPG_MOD 1006>, <&cpg CPG_MOD 1007>,
+				 <&cpg CPG_MOD 1008>, <&cpg CPG_MOD 1009>,
+				 <&cpg CPG_MOD 1010>, <&cpg CPG_MOD 1011>,
+				 <&cpg CPG_MOD 1012>, <&cpg CPG_MOD 1013>,
+				 <&cpg CPG_MOD 1014>, <&cpg CPG_MOD 1015>,
+				 <&cpg CPG_MOD 1025>, <&cpg CPG_MOD 1026>,
+				 <&cpg CPG_MOD 1027>, <&cpg CPG_MOD 1028>,
+				 <&cpg CPG_MOD 1029>, <&cpg CPG_MOD 1030>,
+				 <&cpg CPG_MOD 1021>, <&cpg CPG_MOD 1020>,
+				 <&cpg CPG_MOD 1021>, <&cpg CPG_MOD 1020>,
+				 <&cpg CPG_MOD 1019>, <&cpg CPG_MOD 1018>,
+				 <&audio_clka>, <&audio_clkb>, <&audio_clkc>,
+				 <&cpg CPG_CORE R8A7745_CLK_M2>;
+			clock-names = "ssi-all",
+				      "ssi.9", "ssi.8", "ssi.7", "ssi.6",
+				      "ssi.5", "ssi.4", "ssi.3", "ssi.2",
+				      "ssi.1", "ssi.0",
+				      "src.6", "src.5", "src.4", "src.3",
+				      "src.2", "src.1",
+				      "ctu.0", "ctu.1",
+				      "mix.0", "mix.1",
+				      "dvc.0", "dvc.1",
+				      "clk_a", "clk_b", "clk_c", "clk_i";
+			power-domains = <&sysc R8A7745_PD_ALWAYS_ON>;
+			resets = <&cpg 1005>,
+				 <&cpg 1006>, <&cpg 1007>, <&cpg 1008>,
+				 <&cpg 1009>, <&cpg 1010>, <&cpg 1011>,
+				 <&cpg 1012>, <&cpg 1013>, <&cpg 1014>,
+				 <&cpg 1015>;
+			reset-names = "ssi-all",
+				      "ssi.9", "ssi.8", "ssi.7", "ssi.6",
+				      "ssi.5", "ssi.4", "ssi.3", "ssi.2",
+				      "ssi.1", "ssi.0";
+
+			status = "disabled";
+
+			rcar_sound,dvc {
+				dvc0: dvc-0 {
+					dmas = <&audma0 0xbc>;
+					dma-names = "tx";
+				};
+				dvc1: dvc-1 {
+					dmas = <&audma0 0xbe>;
+					dma-names = "tx";
+				};
+			};
+
+			rcar_sound,mix {
+				mix0: mix-0 { };
+				mix1: mix-1 { };
+			};
+
+			rcar_sound,ctu {
+				ctu00: ctu-0 { };
+				ctu01: ctu-1 { };
+				ctu02: ctu-2 { };
+				ctu03: ctu-3 { };
+				ctu10: ctu-4 { };
+				ctu11: ctu-5 { };
+				ctu12: ctu-6 { };
+				ctu13: ctu-7 { };
+			};
+
+			rcar_sound,src {
+				src-0 {
+					status = "disabled";
+				};
+				src1: src-1 {
+					interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x87>, <&audma0 0x9c>;
+					dma-names = "rx", "tx";
+				};
+				src2: src-2 {
+					interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x89>, <&audma0 0x9e>;
+					dma-names = "rx", "tx";
+				};
+				src3: src-3 {
+					interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x8b>, <&audma0 0xa0>;
+					dma-names = "rx", "tx";
+				};
+				src4: src-4 {
+					interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x8d>, <&audma0 0xb0>;
+					dma-names = "rx", "tx";
+				};
+				src5: src-5 {
+					interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x8f>, <&audma0 0xb2>;
+					dma-names = "rx", "tx";
+				};
+				src6: src-6 {
+					interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x91>, <&audma0 0xb4>;
+					dma-names = "rx", "tx";
+				};
+			};
+
+			rcar_sound,ssi {
+				ssi0: ssi-0 {
+					interrupts = <GIC_SPI 370 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x01>, <&audma0 0x02>,
+					       <&audma0 0x15>, <&audma0 0x16>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi1: ssi-1 {
+					interrupts = <GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x03>, <&audma0 0x04>,
+					       <&audma0 0x49>, <&audma0 0x4a>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi2: ssi-2 {
+					interrupts = <GIC_SPI 372 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x05>, <&audma0 0x06>,
+					       <&audma0 0x63>, <&audma0 0x64>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi3: ssi-3 {
+					interrupts = <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x07>, <&audma0 0x08>,
+					       <&audma0 0x6f>, <&audma0 0x70>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi4: ssi-4 {
+					interrupts = <GIC_SPI 374 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x09>, <&audma0 0x0a>,
+					       <&audma0 0x71>, <&audma0 0x72>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi5: ssi-5 {
+					interrupts = <GIC_SPI 375 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x0b>, <&audma0 0x0c>,
+					       <&audma0 0x73>, <&audma0 0x74>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi6: ssi-6 {
+					interrupts = <GIC_SPI 376 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x0d>, <&audma0 0x0e>,
+					       <&audma0 0x75>, <&audma0 0x76>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi7: ssi-7 {
+					interrupts = <GIC_SPI 377 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x0f>, <&audma0 0x10>,
+					       <&audma0 0x79>, <&audma0 0x7a>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi8: ssi-8 {
+					interrupts = <GIC_SPI 378 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x11>, <&audma0 0x12>,
+					       <&audma0 0x7b>, <&audma0 0x7c>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+				ssi9: ssi-9 {
+					interrupts = <GIC_SPI 379 IRQ_TYPE_LEVEL_HIGH>;
+					dmas = <&audma0 0x13>, <&audma0 0x14>,
+					       <&audma0 0x7d>, <&audma0 0x7e>;
+					dma-names = "rx", "tx", "rxu", "txu";
+				};
+			};
+		};
 	};
 
-	/* External root clock */
-	extal_clk: extal {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		/* This value must be overridden by the board. */
-		clock-frequency = <0>;
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 
 	/* External USB clock - can be overridden by the board */
@@ -953,12 +1490,4 @@
 		#clock-cells = <0>;
 		clock-frequency = <48000000>;
 	};
-
-	/* External SCIF clock */
-	scif_clk: scif {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		/* This value must be overridden by the board. */
-		clock-frequency = <0>;
-	};
 };
diff --git a/arch/arm/boot/dts/r8a7778.dtsi b/arch/arm/boot/dts/r8a7778.dtsi
index a39472aab867..3b49f9ed2e2b 100644
--- a/arch/arm/boot/dts/r8a7778.dtsi
+++ b/arch/arm/boot/dts/r8a7778.dtsi
@@ -51,7 +51,8 @@
 	};
 
 	ether: ethernet@fde00000 {
-		compatible = "renesas,ether-r8a7778";
+		compatible = "renesas,ether-r8a7778",
+			     "renesas,rcar-gen1-ether";
 		reg = <0xfde00000 0x400>;
 		interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp1_clks R8A7778_CLK_ETHER>;
@@ -379,7 +380,8 @@
 	};
 
 	sdhi0: sd@ffe4c000 {
-		compatible = "renesas,sdhi-r8a7778";
+		compatible = "renesas,sdhi-r8a7778",
+			     "renesas,rcar-gen1-sdhi";
 		reg = <0xffe4c000 0x100>;
 		interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7778_CLK_SDHI0>;
@@ -388,7 +390,8 @@
 	};
 
 	sdhi1: sd@ffe4d000 {
-		compatible = "renesas,sdhi-r8a7778";
+		compatible = "renesas,sdhi-r8a7778",
+			     "renesas,rcar-gen1-sdhi";
 		reg = <0xffe4d000 0x100>;
 		interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7778_CLK_SDHI1>;
@@ -397,7 +400,8 @@
 	};
 
 	sdhi2: sd@ffe4f000 {
-		compatible = "renesas,sdhi-r8a7778";
+		compatible = "renesas,sdhi-r8a7778",
+			     "renesas,rcar-gen1-sdhi";
 		reg = <0xffe4f000 0x100>;
 		interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7778_CLK_SDHI2>;
diff --git a/arch/arm/boot/dts/r8a7779.dtsi b/arch/arm/boot/dts/r8a7779.dtsi
index e8eb94748b27..e79ae306eedd 100644
--- a/arch/arm/boot/dts/r8a7779.dtsi
+++ b/arch/arm/boot/dts/r8a7779.dtsi
@@ -355,7 +355,8 @@
 	};
 
 	sdhi0: sd@ffe4c000 {
-		compatible = "renesas,sdhi-r8a7779";
+		compatible = "renesas,sdhi-r8a7779",
+			     "renesas,rcar-gen1-sdhi";
 		reg = <0xffe4c000 0x100>;
 		interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7779_CLK_SDHI0>;
@@ -364,7 +365,8 @@
 	};
 
 	sdhi1: sd@ffe4d000 {
-		compatible = "renesas,sdhi-r8a7779";
+		compatible = "renesas,sdhi-r8a7779",
+			     "renesas,rcar-gen1-sdhi";
 		reg = <0xffe4d000 0x100>;
 		interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7779_CLK_SDHI1>;
@@ -373,7 +375,8 @@
 	};
 
 	sdhi2: sd@ffe4e000 {
-		compatible = "renesas,sdhi-r8a7779";
+		compatible = "renesas,sdhi-r8a7779",
+			     "renesas,rcar-gen1-sdhi";
 		reg = <0xffe4e000 0x100>;
 		interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7779_CLK_SDHI2>;
@@ -382,7 +385,8 @@
 	};
 
 	sdhi3: sd@ffe4f000 {
-		compatible = "renesas,sdhi-r8a7779";
+		compatible = "renesas,sdhi-r8a7779",
+			     "renesas,rcar-gen1-sdhi";
 		reg = <0xffe4f000 0x100>;
 		interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&mstp3_clks R8A7779_CLK_SDHI3>;
diff --git a/arch/arm/boot/dts/r8a7790-lager.dts b/arch/arm/boot/dts/r8a7790-lager.dts
index e3d27783b6b5..f2ea632381e7 100644
--- a/arch/arm/boot/dts/r8a7790-lager.dts
+++ b/arch/arm/boot/dts/r8a7790-lager.dts
@@ -272,9 +272,8 @@
 		#size-cells = <0>;
 		compatible = "i2c-gpio";
 		status = "disabled";
-		gpios = <&gpio1 17 GPIO_ACTIVE_HIGH /* sda */
-			 &gpio1 16 GPIO_ACTIVE_HIGH /* scl */
-			>;
+		sda-gpios = <&gpio1 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio1 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 		i2c-gpio,delay-us = <5>;
 	};
 
diff --git a/arch/arm/boot/dts/r8a7790.dtsi b/arch/arm/boot/dts/r8a7790.dtsi
index 62baabd757b6..ed9a68538a55 100644
--- a/arch/arm/boot/dts/r8a7790.dtsi
+++ b/arch/arm/boot/dts/r8a7790.dtsi
@@ -168,7 +168,7 @@
 
 			trips {
 				cpu-crit {
-					temperature	= <115000>;
+					temperature	= <95000>;
 					hysteresis	= <0>;
 					type		= "critical";
 				};
@@ -311,7 +311,7 @@
 	};
 
 	cmt0: timer@ffca0000 {
-		compatible = "renesas,cmt-48-r8a7790", "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7790-cmt0", "renesas,rcar-gen2-cmt0";
 		reg = <0 0xffca0000 0 0x1004>;
 		interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
@@ -320,13 +320,11 @@
 		power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
 		resets = <&cpg 124>;
 
-		renesas,channels-mask = <0x60>;
-
 		status = "disabled";
 	};
 
 	cmt1: timer@e6130000 {
-		compatible = "renesas,cmt-48-r8a7790", "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7790-cmt1", "renesas,rcar-gen2-cmt1";
 		reg = <0 0xe6130000 0 0x1004>;
 		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
@@ -341,8 +339,6 @@
 		power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
 		resets = <&cpg 329>;
 
-		renesas,channels-mask = <0xff>;
-
 		status = "disabled";
 	};
 
@@ -662,7 +658,8 @@
 	};
 
 	sdhi0: sd@ee100000 {
-		compatible = "renesas,sdhi-r8a7790";
+		compatible = "renesas,sdhi-r8a7790",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee100000 0 0x328>;
 		interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 314>;
@@ -676,7 +673,8 @@
 	};
 
 	sdhi1: sd@ee120000 {
-		compatible = "renesas,sdhi-r8a7790";
+		compatible = "renesas,sdhi-r8a7790",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee120000 0 0x328>;
 		interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 313>;
@@ -690,7 +688,8 @@
 	};
 
 	sdhi2: sd@ee140000 {
-		compatible = "renesas,sdhi-r8a7790";
+		compatible = "renesas,sdhi-r8a7790",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee140000 0 0x100>;
 		interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 312>;
@@ -704,7 +703,8 @@
 	};
 
 	sdhi3: sd@ee160000 {
-		compatible = "renesas,sdhi-r8a7790";
+		compatible = "renesas,sdhi-r8a7790",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee160000 0 0x100>;
 		interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 311>;
@@ -906,7 +906,8 @@
 	};
 
 	ether: ethernet@ee700000 {
-		compatible = "renesas,ether-r8a7790";
+		compatible = "renesas,ether-r8a7790",
+			     "renesas,rcar-gen2-ether";
 		reg = <0 0xee700000 0 0x400>;
 		interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 813>;
diff --git a/arch/arm/boot/dts/r8a7791-koelsch.dts b/arch/arm/boot/dts/r8a7791-koelsch.dts
index e164eda69baf..a50924d12b6f 100644
--- a/arch/arm/boot/dts/r8a7791-koelsch.dts
+++ b/arch/arm/boot/dts/r8a7791-koelsch.dts
@@ -278,6 +278,12 @@
 		};
 	};
 
+	cec_clock: cec-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <12000000>;
+	};
+
 	hdmi-out {
 		compatible = "hdmi-connector";
 		type = "a";
@@ -306,9 +312,8 @@
 		#size-cells = <0>;
 		compatible = "i2c-gpio";
 		status = "disabled";
-		gpios = <&gpio7 16 GPIO_ACTIVE_HIGH /* sda */
-			 &gpio7 15 GPIO_ACTIVE_HIGH /* scl */
-			>;
+		sda-gpios = <&gpio7 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio7 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 		i2c-gpio,delay-us = <5>;
 	};
 
@@ -640,12 +645,6 @@
 		};
 	};
 
-	cec_clock: cec-clock {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <12000000>;
-	};
-
 	hdmi@39 {
 		compatible = "adi,adv7511w";
 		reg = <0x39>;
@@ -708,7 +707,7 @@
 	};
 
 	eeprom@50 {
-		compatible = "renesas,24c02", "atmel,24c02";
+		compatible = "renesas,r1ex24002", "atmel,24c02";
 		reg = <0x50>;
 		pagesize = <16>;
 	};
diff --git a/arch/arm/boot/dts/r8a7791.dtsi b/arch/arm/boot/dts/r8a7791.dtsi
index 67831d0405f3..008a260f86a5 100644
--- a/arch/arm/boot/dts/r8a7791.dtsi
+++ b/arch/arm/boot/dts/r8a7791.dtsi
@@ -92,7 +92,7 @@
 
 			trips {
 				cpu-crit {
-					temperature	= <115000>;
+					temperature	= <95000>;
 					hysteresis	= <0>;
 					type		= "critical";
 				};
@@ -257,7 +257,7 @@
 	};
 
 	cmt0: timer@ffca0000 {
-		compatible = "renesas,cmt-48-r8a7791", "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7791-cmt0", "renesas,rcar-gen2-cmt0";
 		reg = <0 0xffca0000 0 0x1004>;
 		interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
@@ -266,13 +266,11 @@
 		power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
 		resets = <&cpg 124>;
 
-		renesas,channels-mask = <0x60>;
-
 		status = "disabled";
 	};
 
 	cmt1: timer@e6130000 {
-		compatible = "renesas,cmt-48-r8a7791", "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7791-cmt1", "renesas,rcar-gen2-cmt1";
 		reg = <0 0xe6130000 0 0x1004>;
 		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
@@ -287,8 +285,6 @@
 		power-domains = <&sysc R8A7791_PD_ALWAYS_ON>;
 		resets = <&cpg 329>;
 
-		renesas,channels-mask = <0xff>;
-
 		status = "disabled";
 	};
 
@@ -612,7 +608,8 @@
 	};
 
 	sdhi0: sd@ee100000 {
-		compatible = "renesas,sdhi-r8a7791";
+		compatible = "renesas,sdhi-r8a7791",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee100000 0 0x328>;
 		interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 314>;
@@ -626,7 +623,8 @@
 	};
 
 	sdhi1: sd@ee140000 {
-		compatible = "renesas,sdhi-r8a7791";
+		compatible = "renesas,sdhi-r8a7791",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee140000 0 0x100>;
 		interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 312>;
@@ -640,7 +638,8 @@
 	};
 
 	sdhi2: sd@ee160000 {
-		compatible = "renesas,sdhi-r8a7791";
+		compatible = "renesas,sdhi-r8a7791",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee160000 0 0x100>;
 		interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 311>;
@@ -961,7 +960,8 @@
 	};
 
 	ether: ethernet@ee700000 {
-		compatible = "renesas,ether-r8a7791";
+		compatible = "renesas,ether-r8a7791",
+			     "renesas,rcar-gen2-ether";
 		reg = <0 0xee700000 0 0x400>;
 		interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 813>;
diff --git a/arch/arm/boot/dts/r8a7792.dtsi b/arch/arm/boot/dts/r8a7792.dtsi
index 3d080e07374c..3be15a158bad 100644
--- a/arch/arm/boot/dts/r8a7792.dtsi
+++ b/arch/arm/boot/dts/r8a7792.dtsi
@@ -36,6 +36,14 @@
 		vin5 = &vin5;
 	};
 
+	/* External CAN clock */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -69,6 +77,22 @@
 		};
 	};
 
+	/* External root clock */
+	extal_clk: extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
+	/* External SCIF clock */
+	scif_clk: scif {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		/* This value must be overridden by the board. */
+		clock-frequency = <0>;
+	};
+
 	soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gic>;
@@ -113,18 +137,6 @@
 			resets = <&cpg 407>;
 		};
 
-		timer {
-			compatible = "arm,armv7-timer";
-			interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) |
-				      IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) |
-				      IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) |
-				      IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) |
-				      IRQ_TYPE_LEVEL_LOW)>;
-		};
-
 		rst: reset-controller@e6160000 {
 			compatible = "renesas,r8a7792-rst";
 			reg = <0 0xe6160000 0 0x0100>;
@@ -507,7 +519,8 @@
 		};
 
 		sdhi0: sd@ee100000 {
-			compatible = "renesas,sdhi-r8a7792";
+			compatible = "renesas,sdhi-r8a7792",
+				     "renesas,rcar-gen2-sdhi";
 			reg = <0 0xee100000 0 0x328>;
 			interrupts = <0 165 IRQ_TYPE_LEVEL_HIGH>;
 			dmas = <&dmac0 0xcd>, <&dmac0 0xce>,
@@ -833,27 +846,11 @@
 		};
 	};
 
-	/* External root clock */
-	extal_clk: extal {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		/* This value must be overridden by the board. */
-		clock-frequency = <0>;
-	};
-
-	/* External SCIF clock */
-	scif_clk: scif {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		/* This value must be overridden by the board. */
-		clock-frequency = <0>;
-	};
-
-	/* External CAN clock */
-	can_clk: can {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		/* This value must be overridden by the board. */
-		clock-frequency = <0>;
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 };
diff --git a/arch/arm/boot/dts/r8a7793.dtsi b/arch/arm/boot/dts/r8a7793.dtsi
index 0cd1035de1a4..039b22517526 100644
--- a/arch/arm/boot/dts/r8a7793.dtsi
+++ b/arch/arm/boot/dts/r8a7793.dtsi
@@ -89,7 +89,7 @@
 
 			trips {
 				cpu-crit {
-					temperature	= <115000>;
+					temperature	= <95000>;
 					hysteresis	= <0>;
 					type		= "critical";
 				};
@@ -248,7 +248,7 @@
 	};
 
 	cmt0: timer@ffca0000 {
-		compatible = "renesas,cmt-48-r8a7793", "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7793-cmt0", "renesas,rcar-gen2-cmt0";
 		reg = <0 0xffca0000 0 0x1004>;
 		interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
@@ -257,13 +257,11 @@
 		power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
 		resets = <&cpg 124>;
 
-		renesas,channels-mask = <0x60>;
-
 		status = "disabled";
 	};
 
 	cmt1: timer@e6130000 {
-		compatible = "renesas,cmt-48-r8a7793", "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7793-cmt1", "renesas,rcar-gen2-cmt1";
 		reg = <0 0xe6130000 0 0x1004>;
 		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
@@ -278,8 +276,6 @@
 		power-domains = <&sysc R8A7793_PD_ALWAYS_ON>;
 		resets = <&cpg 329>;
 
-		renesas,channels-mask = <0xff>;
-
 		status = "disabled";
 	};
 
@@ -562,7 +558,8 @@
 	};
 
 	sdhi0: sd@ee100000 {
-		compatible = "renesas,sdhi-r8a7793";
+		compatible = "renesas,sdhi-r8a7793",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee100000 0 0x328>;
 		interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 314>;
@@ -576,7 +573,8 @@
 	};
 
 	sdhi1: sd@ee140000 {
-		compatible = "renesas,sdhi-r8a7793";
+		compatible = "renesas,sdhi-r8a7793",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee140000 0 0x100>;
 		interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 312>;
@@ -590,7 +588,8 @@
 	};
 
 	sdhi2: sd@ee160000 {
-		compatible = "renesas,sdhi-r8a7793";
+		compatible = "renesas,sdhi-r8a7793",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee160000 0 0x100>;
 		interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 311>;
@@ -916,7 +915,8 @@
 	};
 
 	ether: ethernet@ee700000 {
-		compatible = "renesas,ether-r8a7793";
+		compatible = "renesas,ether-r8a7793",
+			     "renesas,rcar-gen2-ether";
 		reg = <0 0xee700000 0 0x400>;
 		interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 813>;
diff --git a/arch/arm/boot/dts/r8a7794-alt.dts b/arch/arm/boot/dts/r8a7794-alt.dts
index bd98790d964e..60c6515c4996 100644
--- a/arch/arm/boot/dts/r8a7794-alt.dts
+++ b/arch/arm/boot/dts/r8a7794-alt.dts
@@ -143,9 +143,8 @@
 		#size-cells = <0>;
 		compatible = "i2c-gpio";
 		status = "disabled";
-		gpios = <&gpio4 9 GPIO_ACTIVE_HIGH /* sda */
-			 &gpio4 8 GPIO_ACTIVE_HIGH /* scl */
-			>;
+		sda-gpios = <&gpio4 9 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+		scl-gpios = <&gpio4 8 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 		i2c-gpio,delay-us = <5>;
 	};
 
diff --git a/arch/arm/boot/dts/r8a7794.dtsi b/arch/arm/boot/dts/r8a7794.dtsi
index 5643976c1356..106b4e1649ff 100644
--- a/arch/arm/boot/dts/r8a7794.dtsi
+++ b/arch/arm/boot/dts/r8a7794.dtsi
@@ -37,6 +37,7 @@
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
+		enable-method = "renesas,apmu";
 
 		cpu0: cpu@0 {
 			device_type = "cpu";
@@ -66,6 +67,12 @@
 		};
 	};
 
+	apmu@e6151000 {
+		compatible = "renesas,r8a7794-apmu", "renesas,apmu";
+		reg = <0 0xe6151000 0 0x188>;
+		cpus = <&cpu0 &cpu1>;
+	};
+
 	gic: interrupt-controller@f1001000 {
 		compatible = "arm,gic-400";
 		#interrupt-cells = <3>;
@@ -181,7 +188,7 @@
 	};
 
 	cmt0: timer@ffca0000 {
-		compatible = "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7794-cmt0", "renesas,rcar-gen2-cmt0";
 		reg = <0 0xffca0000 0 0x1004>;
 		interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
@@ -190,13 +197,11 @@
 		power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
 		resets = <&cpg 124>;
 
-		renesas,channels-mask = <0x60>;
-
 		status = "disabled";
 	};
 
 	cmt1: timer@e6130000 {
-		compatible = "renesas,cmt-48-gen2";
+		compatible = "renesas,r8a7794-cmt1", "renesas,rcar-gen2-cmt1";
 		reg = <0 0xe6130000 0 0x1004>;
 		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>,
@@ -211,8 +216,6 @@
 		power-domains = <&sysc R8A7794_PD_ALWAYS_ON>;
 		resets = <&cpg 329>;
 
-		renesas,channels-mask = <0xff>;
-
 		status = "disabled";
 	};
 
@@ -640,7 +643,8 @@
 	};
 
 	ether: ethernet@ee700000 {
-		compatible = "renesas,ether-r8a7794";
+		compatible = "renesas,ether-r8a7794",
+			     "renesas,rcar-gen2-ether";
 		reg = <0 0xee700000 0 0x400>;
 		interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 813>;
@@ -791,7 +795,8 @@
 	};
 
 	sdhi0: sd@ee100000 {
-		compatible = "renesas,sdhi-r8a7794";
+		compatible = "renesas,sdhi-r8a7794",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee100000 0 0x328>;
 		interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 314>;
@@ -805,7 +810,8 @@
 	};
 
 	sdhi1: sd@ee140000 {
-		compatible = "renesas,sdhi-r8a7794";
+		compatible = "renesas,sdhi-r8a7794",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee140000 0 0x100>;
 		interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 312>;
@@ -819,7 +825,8 @@
 	};
 
 	sdhi2: sd@ee160000 {
-		compatible = "renesas,sdhi-r8a7794";
+		compatible = "renesas,sdhi-r8a7794",
+			     "renesas,rcar-gen2-sdhi";
 		reg = <0 0xee160000 0 0x100>;
 		interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cpg CPG_MOD 311>;
diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi
index 3b704cfed69a..a97458112ff6 100644
--- a/arch/arm/boot/dts/rk3036.dtsi
+++ b/arch/arm/boot/dts/rk3036.dtsi
@@ -280,7 +280,7 @@
 		max-frequency = <37500000>;
 		clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>,
 			 <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
 		interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>;
 		resets = <&cru SRST_SDIO>;
@@ -298,7 +298,7 @@
 		max-frequency = <37500000>;
 		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
 			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		default-sample-phase = <158>;
 		disable-wp;
 		dmas = <&pdma 12>;
diff --git a/arch/arm/boot/dts/rk3066a-rayeager.dts b/arch/arm/boot/dts/rk3066a-rayeager.dts
index cdf301f5778b..4d7057a10a4c 100644
--- a/arch/arm/boot/dts/rk3066a-rayeager.dts
+++ b/arch/arm/boot/dts/rk3066a-rayeager.dts
@@ -177,6 +177,7 @@
 
 	phy0: ethernet-phy@0 {
 		reg = <0>;
+		reset-gpios = <&gpio1 RK_PD6 GPIO_ACTIVE_LOW>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/rk322x.dtsi b/arch/arm/boot/dts/rk322x.dtsi
index 780ec3a99b21..341deaf62ff6 100644
--- a/arch/arm/boot/dts/rk322x.dtsi
+++ b/arch/arm/boot/dts/rk322x.dtsi
@@ -621,7 +621,7 @@
 		interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>,
 			 <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_bus4>;
@@ -634,7 +634,7 @@
 		interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>,
 			 <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&sdio_clk &sdio_cmd &sdio_bus4>;
@@ -649,7 +649,7 @@
 		max-frequency = <37500000>;
 		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
 			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		bus-width = <8>;
 		default-sample-phase = <158>;
 		fifo-depth = <0x100>;
diff --git a/arch/arm/boot/dts/rk3288-phycore-som.dtsi b/arch/arm/boot/dts/rk3288-phycore-som.dtsi
index 99cfae875e12..5eae4776ffde 100644
--- a/arch/arm/boot/dts/rk3288-phycore-som.dtsi
+++ b/arch/arm/boot/dts/rk3288-phycore-som.dtsi
@@ -110,26 +110,6 @@
 	};
 };
 
-&cpu0 {
-	cpu0-supply = <&vdd_cpu>;
-	operating-points = <
-		/* KHz    uV */
-		1800000	1400000
-		1608000	1350000
-		1512000 1300000
-		1416000 1200000
-		1200000 1100000
-		1008000 1050000
-		 816000 1000000
-		 696000  950000
-		 600000  900000
-		 408000  900000
-		 312000  900000
-		 216000  900000
-		 126000  900000
-	>;
-};
-
 &emmc {
 	status = "okay";
 	bus-width = <8>;
diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi
index 6102e4e7f35c..354aff45c1af 100644
--- a/arch/arm/boot/dts/rk3288.dtsi
+++ b/arch/arm/boot/dts/rk3288.dtsi
@@ -927,6 +927,7 @@
 	i2s: i2s@ff890000 {
 		compatible = "rockchip,rk3288-i2s", "rockchip,rk3066-i2s";
 		reg = <0x0 0xff890000 0x0 0x10000>;
+		#sound-dai-cells = <0>;
 		interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -1176,6 +1177,7 @@
 		compatible = "rockchip,rk3288-dw-hdmi";
 		reg = <0x0 0xff980000 0x0 0x20000>;
 		reg-io-width = <4>;
+		#sound-dai-cells = <0>;
 		rockchip,grf = <&grf>;
 		interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cru  PCLK_HDMI_CTRL>, <&cru SCLK_HDMI_HDCP>, <&cru SCLK_HDMI_CEC>;
diff --git a/arch/arm/boot/dts/s3c2416-pinctrl.dtsi b/arch/arm/boot/dts/s3c2416-pinctrl.dtsi
index 6274359fb323..92439ee5d7de 100644
--- a/arch/arm/boot/dts/s3c2416-pinctrl.dtsi
+++ b/arch/arm/boot/dts/s3c2416-pinctrl.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung S3C2416 pinctrl settings
  *
  * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/pinctrl/samsung.h>
diff --git a/arch/arm/boot/dts/s3c2416-smdk2416.dts b/arch/arm/boot/dts/s3c2416-smdk2416.dts
index f257926c13b7..a1c9d8c695cc 100644
--- a/arch/arm/boot/dts/s3c2416-smdk2416.dts
+++ b/arch/arm/boot/dts/s3c2416-smdk2416.dts
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG SMDK2416 board device tree source
  *
  * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/s3c2416.dtsi b/arch/arm/boot/dts/s3c2416.dtsi
index 80f007550324..3c7385cab248 100644
--- a/arch/arm/boot/dts/s3c2416.dtsi
+++ b/arch/arm/boot/dts/s3c2416.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S3C2416 SoC device tree source
  *
  * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/clock/s3c2443.h>
diff --git a/arch/arm/boot/dts/s3c24xx.dtsi b/arch/arm/boot/dts/s3c24xx.dtsi
index 5ed43b857cc4..34c7fe6751cf 100644
--- a/arch/arm/boot/dts/s3c24xx.dtsi
+++ b/arch/arm/boot/dts/s3c24xx.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S3C24XX family device tree source
  *
  * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "skeleton.dtsi"
diff --git a/arch/arm/boot/dts/s3c6400.dtsi b/arch/arm/boot/dts/s3c6400.dtsi
index a7d1c8ec150d..8c28e8a0c824 100644
--- a/arch/arm/boot/dts/s3c6400.dtsi
+++ b/arch/arm/boot/dts/s3c6400.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S3C6400 SoC device tree source
  *
@@ -10,11 +11,7 @@
  * Note: This file does not include device nodes for all the controllers in
  * S3C6400 SoC. As device tree coverage for S3C6400 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include "s3c64xx.dtsi"
 
diff --git a/arch/arm/boot/dts/s3c6410-mini6410.dts b/arch/arm/boot/dts/s3c6410-mini6410.dts
index f4afda3594f8..f68601bd9c91 100644
--- a/arch/arm/boot/dts/s3c6410-mini6410.dts
+++ b/arch/arm/boot/dts/s3c6410-mini6410.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S3C6410 based Mini6410 board device tree source
  *
@@ -5,11 +6,7 @@
  *
  * Device tree source file for FriendlyARM Mini6410 board which is based on
  * Samsung's S3C6410 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/s3c6410-smdk6410.dts b/arch/arm/boot/dts/s3c6410-smdk6410.dts
index ecf35ec466f7..b6b5afcd7602 100644
--- a/arch/arm/boot/dts/s3c6410-smdk6410.dts
+++ b/arch/arm/boot/dts/s3c6410-smdk6410.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung S3C6410 based SMDK6410 board device tree source.
  *
@@ -5,11 +6,7 @@
  *
  * Device tree source file for SAMSUNG SMDK6410 board which is based on
  * Samsung's S3C6410 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 
diff --git a/arch/arm/boot/dts/s3c6410.dtsi b/arch/arm/boot/dts/s3c6410.dtsi
index eb4226b3407c..a766d6de696c 100644
--- a/arch/arm/boot/dts/s3c6410.dtsi
+++ b/arch/arm/boot/dts/s3c6410.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S3C6410 SoC device tree source
  *
@@ -10,11 +11,7 @@
  * Note: This file does not include device nodes for all the controllers in
  * S3C6410 SoC. As device tree coverage for S3C6410 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include "s3c64xx.dtsi"
 
diff --git a/arch/arm/boot/dts/s3c64xx-pinctrl.dtsi b/arch/arm/boot/dts/s3c64xx-pinctrl.dtsi
index 4e8e802b4ee1..8e9594d64b57 100644
--- a/arch/arm/boot/dts/s3c64xx-pinctrl.dtsi
+++ b/arch/arm/boot/dts/s3c64xx-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S3C64xx SoC series common device tree source
  * - pin control-related definitions
@@ -6,10 +7,6 @@
  *
  * Samsung's S3C64xx SoCs pin banks, pin-mux and pin-config options are
  * listed as device tree nodes in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/pinctrl/samsung.h>
diff --git a/arch/arm/boot/dts/s3c64xx.dtsi b/arch/arm/boot/dts/s3c64xx.dtsi
index c55cbb3af2c0..e2be3fbdd3f3 100644
--- a/arch/arm/boot/dts/s3c64xx.dtsi
+++ b/arch/arm/boot/dts/s3c64xx.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S3C64xx SoC series common device tree source
  *
@@ -10,10 +11,6 @@
  * Note: This file does not include device nodes for all the controllers in
  * S3C64xx SoCs. As device tree coverage for S3C64xx increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "skeleton.dtsi"
diff --git a/arch/arm/boot/dts/s5pv210-aquila.dts b/arch/arm/boot/dts/s5pv210-aquila.dts
index 40139923eef0..14969b6529e8 100644
--- a/arch/arm/boot/dts/s5pv210-aquila.dts
+++ b/arch/arm/boot/dts/s5pv210-aquila.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S5PV210 SoC device tree source
  *
@@ -7,10 +8,6 @@
  * Tomasz Figa <t.figa@samsung.com>
  *
  * Board device tree source for Samsung Aquila board.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/s5pv210-goni.dts b/arch/arm/boot/dts/s5pv210-goni.dts
index c56f51ee7897..eb6d1926c0d6 100644
--- a/arch/arm/boot/dts/s5pv210-goni.dts
+++ b/arch/arm/boot/dts/s5pv210-goni.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S5PV210 SoC device tree source
  *
@@ -7,10 +8,6 @@
  * Tomasz Figa <t.figa@samsung.com>
  *
  * Board device tree source for Samsung Goni board.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/s5pv210-pinctrl.dtsi b/arch/arm/boot/dts/s5pv210-pinctrl.dtsi
index 9a3e851e2e22..3a79feab11c3 100644
--- a/arch/arm/boot/dts/s5pv210-pinctrl.dtsi
+++ b/arch/arm/boot/dts/s5pv210-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S5PV210 SoC device tree source
  *
@@ -13,10 +14,6 @@
  * Note: This file does not include device nodes for all the controllers in
  * S5PV210 SoC. As device tree coverage for S5PV210 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/pinctrl/samsung.h>
diff --git a/arch/arm/boot/dts/s5pv210-smdkc110.dts b/arch/arm/boot/dts/s5pv210-smdkc110.dts
index 5d14da911aa5..e5aec6c526fb 100644
--- a/arch/arm/boot/dts/s5pv210-smdkc110.dts
+++ b/arch/arm/boot/dts/s5pv210-smdkc110.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S5PV210 SoC device tree source
  *
@@ -11,10 +12,6 @@
  * NOTE: This file is completely based on original board file for mach-smdkc110
  * available in Linux 3.15 and intends to provide equivalent level of hardware
  * support. Due to lack of hardware, _no_ testing has been performed.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/s5pv210-smdkv210.dts b/arch/arm/boot/dts/s5pv210-smdkv210.dts
index 75398318ed57..84b38f185199 100644
--- a/arch/arm/boot/dts/s5pv210-smdkv210.dts
+++ b/arch/arm/boot/dts/s5pv210-smdkv210.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S5PV210 SoC device tree source
  *
@@ -11,10 +12,6 @@
  * NOTE: This file is completely based on original board file for mach-smdkv210
  * available in Linux 3.15 and intends to provide equivalent level of hardware
  * support. Due to lack of hardware, _no_ testing has been performed.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/s5pv210-torbreck.dts b/arch/arm/boot/dts/s5pv210-torbreck.dts
index 7cb50bcee888..cd25e72ccd84 100644
--- a/arch/arm/boot/dts/s5pv210-torbreck.dts
+++ b/arch/arm/boot/dts/s5pv210-torbreck.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S5PV210 SoC device tree source
  *
@@ -11,10 +12,6 @@
  * NOTE: This file is completely based on original board file for mach-torbreck
  * available in Linux 3.15 and intends to provide equivalent level of hardware
  * support. Due to lack of hardware, _no_ testing has been performed.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/s5pv210.dtsi b/arch/arm/boot/dts/s5pv210.dtsi
index 726c5d0dbd5b..67358562a6ea 100644
--- a/arch/arm/boot/dts/s5pv210.dtsi
+++ b/arch/arm/boot/dts/s5pv210.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's S5PV210 SoC device tree source
  *
@@ -13,11 +14,7 @@
  * Note: This file does not include device nodes for all the controllers in
  * S5PV210 SoC. As device tree coverage for S5PV210 increases, additional
  * nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/clock/s5pv210.h>
 #include <dt-bindings/clock/s5pv210-audss.h>
@@ -463,6 +460,7 @@
 			compatible = "samsung,exynos4210-ohci";
 			reg = <0xec300000 0x100>;
 			interrupts = <23>;
+			interrupt-parent = <&vic1>;
 			clocks = <&clocks CLK_USB_HOST>;
 			clock-names = "usbhost";
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/sama5d2.dtsi b/arch/arm/boot/dts/sama5d2.dtsi
index b44e63995583..61f68e5c48e9 100644
--- a/arch/arm/boot/dts/sama5d2.dtsi
+++ b/arch/arm/boot/dts/sama5d2.dtsi
@@ -1094,7 +1094,9 @@
 			};
 
 			tcb0: timer@f800c000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf800c000 0x100>;
 				interrupts = <35 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb0_clk>, <&clk32k>;
@@ -1102,7 +1104,9 @@
 			};
 
 			tcb1: timer@f8010000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf8010000 0x100>;
 				interrupts = <36 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb1_clk>, <&clk32k>;
@@ -1427,6 +1431,8 @@
 				interrupts = <40 IRQ_TYPE_LEVEL_HIGH 7>;
 				clocks = <&adc_clk>;
 				clock-names = "adc_clk";
+				dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(25))>;
+				dma-names = "rx";
 				atmel,min-sample-rate-hz = <200000>;
 				atmel,max-sample-rate-hz = <20000000>;
 				atmel,startup-time-ms = <4>;
diff --git a/arch/arm/boot/dts/sama5d3.dtsi b/arch/arm/boot/dts/sama5d3.dtsi
index 1889b4dea066..b9c05b57735e 100644
--- a/arch/arm/boot/dts/sama5d3.dtsi
+++ b/arch/arm/boot/dts/sama5d3.dtsi
@@ -142,7 +142,9 @@
 			};
 
 			tcb0: timer@f0010000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf0010000 0x100>;
 				interrupts = <26 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb0_clk>, <&clk32k>;
diff --git a/arch/arm/boot/dts/sama5d3_tcb1.dtsi b/arch/arm/boot/dts/sama5d3_tcb1.dtsi
index 801f9745e82f..cb30bdb1a9ca 100644
--- a/arch/arm/boot/dts/sama5d3_tcb1.dtsi
+++ b/arch/arm/boot/dts/sama5d3_tcb1.dtsi
@@ -28,7 +28,9 @@
 			};
 
 			tcb1: timer@f8014000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf8014000 0x100>;
 				interrupts = <27 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb1_clk>, <&clk32k>;
diff --git a/arch/arm/boot/dts/sama5d3xcm.dtsi b/arch/arm/boot/dts/sama5d3xcm.dtsi
index 9506daf5efb6..3311a882458b 100644
--- a/arch/arm/boot/dts/sama5d3xcm.dtsi
+++ b/arch/arm/boot/dts/sama5d3xcm.dtsi
@@ -34,6 +34,18 @@
 			spi0: spi@f0004000 {
 				cs-gpios = <&pioD 13 0>, <0>, <0>, <0>;
 			};
+
+			tcb0: timer@f0010000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
 		};
 
 		ebi@10000000 {
diff --git a/arch/arm/boot/dts/sama5d3xcm_cmp.dtsi b/arch/arm/boot/dts/sama5d3xcm_cmp.dtsi
index 75cbf4d4ab1a..a02f59021364 100644
--- a/arch/arm/boot/dts/sama5d3xcm_cmp.dtsi
+++ b/arch/arm/boot/dts/sama5d3xcm_cmp.dtsi
@@ -69,6 +69,18 @@
 				cs-gpios = <&pioD 13 0>, <0>, <0>, <0>;
 			};
 
+			tcb0: timer@f0010000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>;
+				};
+
+				timer@1 {
+					compatible = "atmel,tcb-timer";
+					reg = <1>;
+				};
+			};
+
 			macb0: ethernet@f0028000 {
 				phy-mode = "rgmii";
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/sama5d3xmb.dtsi b/arch/arm/boot/dts/sama5d3xmb.dtsi
index 7f55050dd405..ef0f2d049e15 100644
--- a/arch/arm/boot/dts/sama5d3xmb.dtsi
+++ b/arch/arm/boot/dts/sama5d3xmb.dtsi
@@ -53,7 +53,7 @@
 			};
 
 			i2c1: i2c@f0018000 {
-				ov2640: camera@0x30 {
+				ov2640: camera@30 {
 					compatible = "ovti,ov2640";
 					reg = <0x30>;
 					pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/sama5d3xmb_cmp.dtsi b/arch/arm/boot/dts/sama5d3xmb_cmp.dtsi
index 83e3d3e08fd4..97e171db5970 100644
--- a/arch/arm/boot/dts/sama5d3xmb_cmp.dtsi
+++ b/arch/arm/boot/dts/sama5d3xmb_cmp.dtsi
@@ -88,7 +88,7 @@
 			};
 
 			i2c1: i2c@f0018000 {
-				ov2640: camera@0x30 {
+				ov2640: camera@30 {
 					compatible = "ovti,ov2640";
 					reg = <0x30>;
 					pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/sama5d4.dtsi b/arch/arm/boot/dts/sama5d4.dtsi
index b069644ed238..373b3621b536 100644
--- a/arch/arm/boot/dts/sama5d4.dtsi
+++ b/arch/arm/boot/dts/sama5d4.dtsi
@@ -960,7 +960,9 @@
 			};
 
 			tcb0: timer@f801c000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xf801c000 0x100>;
 				interrupts = <40 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb0_clk>, <&clk32k>;
@@ -1188,13 +1190,25 @@
 			};
 
 			tcb1: timer@fc020000 {
-				compatible = "atmel,at91sam9x5-tcb";
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
 				reg = <0xfc020000 0x100>;
 				interrupts = <41 IRQ_TYPE_LEVEL_HIGH 0>;
 				clocks = <&tcb1_clk>, <&clk32k>;
 				clock-names = "t0_clk", "slow_clk";
 			};
 
+			tcb2: timer@fc024000 {
+				compatible = "atmel,at91sam9x5-tcb", "simple-mfd", "syscon";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0xfc024000 0x100>;
+				interrupts = <42 IRQ_TYPE_LEVEL_HIGH 0>;
+				clocks = <&tcb2_clk>, <&clk32k>;
+				clock-names = "t0_clk", "slow_clk";
+			};
+
 			macb1: ethernet@fc028000 {
 				compatible = "atmel,sama5d4-gem";
 				reg = <0xfc028000 0x100>;
diff --git a/arch/arm/boot/dts/sh73a0.dtsi b/arch/arm/boot/dts/sh73a0.dtsi
index 88d7e5631d34..914a7c2a584f 100644
--- a/arch/arm/boot/dts/sh73a0.dtsi
+++ b/arch/arm/boot/dts/sh73a0.dtsi
@@ -100,9 +100,6 @@
 		clocks = <&mstp3_clks SH73A0_CLK_CMT1>;
 		clock-names = "fck";
 		power-domains = <&pd_c5>;
-
-		renesas,channels-mask = <0x3f>;
-
 		status = "disabled";
 	};
 
diff --git a/arch/arm/boot/dts/socfpga.dtsi b/arch/arm/boot/dts/socfpga.dtsi
index 7e24dc8e82d4..c42ca7022e8c 100644
--- a/arch/arm/boot/dts/socfpga.dtsi
+++ b/arch/arm/boot/dts/socfpga.dtsi
@@ -651,6 +651,7 @@
 			#size-cells = <0>;
 			compatible = "snps,designware-i2c";
 			reg = <0xffc04000 0x1000>;
+			resets = <&rst I2C0_RESET>;
 			clocks = <&l4_sp_clk>;
 			interrupts = <0 158 0x4>;
 			status = "disabled";
@@ -661,6 +662,7 @@
 			#size-cells = <0>;
 			compatible = "snps,designware-i2c";
 			reg = <0xffc05000 0x1000>;
+			resets = <&rst I2C1_RESET>;
 			clocks = <&l4_sp_clk>;
 			interrupts = <0 159 0x4>;
 			status = "disabled";
@@ -671,6 +673,7 @@
 			#size-cells = <0>;
 			compatible = "snps,designware-i2c";
 			reg = <0xffc06000 0x1000>;
+			resets = <&rst I2C2_RESET>;
 			clocks = <&l4_sp_clk>;
 			interrupts = <0 160 0x4>;
 			status = "disabled";
@@ -681,6 +684,7 @@
 			#size-cells = <0>;
 			compatible = "snps,designware-i2c";
 			reg = <0xffc07000 0x1000>;
+			resets = <&rst I2C3_RESET>;
 			clocks = <&l4_sp_clk>;
 			interrupts = <0 161 0x4>;
 			status = "disabled";
diff --git a/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi b/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi
index 3a32de9ded3b..64cc86a98771 100644
--- a/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi
+++ b/arch/arm/boot/dts/socfpga_arria10_socdk.dtsi
@@ -163,6 +163,7 @@
 
 &usb0 {
 	status = "okay";
+	disable-over-current;
 };
 
 &watchdog1 {
diff --git a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
index 655fe87e272d..2459d133f1be 100644
--- a/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
+++ b/arch/arm/boot/dts/socfpga_cyclone5_vining_fpga.dts
@@ -237,7 +237,7 @@
 	clock-frequency = <100000>;
 
 	at24@50 {
-		compatible = "at24,24c02";
+		compatible = "atmel,24c02";
 		pagesize = <8>;
 		reg = <0x50>;
 	};
diff --git a/arch/arm/boot/dts/spear1310-evb.dts b/arch/arm/boot/dts/spear1310-evb.dts
index 84101e4eebbf..0f5f379323a8 100644
--- a/arch/arm/boot/dts/spear1310-evb.dts
+++ b/arch/arm/boot/dts/spear1310-evb.dts
@@ -349,7 +349,7 @@
 			spi0: spi@e0100000 {
 				status = "okay";
 				num-cs = <3>;
-				cs-gpios = <&gpio1 7 0>, <&spics 0>, <&spics 1>;
+				cs-gpios = <&gpio1 7 0>, <&spics 0 0>, <&spics 1 0>;
 
 				stmpe610@0 {
 					compatible = "st,stmpe610";
diff --git a/arch/arm/boot/dts/spear1340.dtsi b/arch/arm/boot/dts/spear1340.dtsi
index 5f347054527d..d4dbc4098653 100644
--- a/arch/arm/boot/dts/spear1340.dtsi
+++ b/arch/arm/boot/dts/spear1340.dtsi
@@ -142,8 +142,8 @@
 				reg = <0xb4100000 0x1000>;
 				interrupts = <0 105 0x4>;
 				status = "disabled";
-				dmas = <&dwdma0 0x600 0 0 1>, /* 0xC << 11 */
-					<&dwdma0 0x680 0 1 0>; /* 0xD << 7 */
+				dmas = <&dwdma0 12 0 1>,
+					<&dwdma0 13 1 0>;
 				dma-names = "tx", "rx";
 			};
 
diff --git a/arch/arm/boot/dts/spear13xx.dtsi b/arch/arm/boot/dts/spear13xx.dtsi
index 17ea0abcdbd7..086b4b333249 100644
--- a/arch/arm/boot/dts/spear13xx.dtsi
+++ b/arch/arm/boot/dts/spear13xx.dtsi
@@ -100,7 +100,7 @@
 			reg = <0xb2800000 0x1000>;
 			interrupts = <0 29 0x4>;
 			status = "disabled";
-			dmas = <&dwdma0 0 0 0 0>;
+			dmas = <&dwdma0 0 0 0>;
 			dma-names = "data";
 		};
 
@@ -290,8 +290,8 @@
 				#size-cells = <0>;
 				interrupts = <0 31 0x4>;
 				status = "disabled";
-				dmas = <&dwdma0 0x2000 0 0 0>, /* 0x4 << 11 */
-					<&dwdma0 0x0280 0 0 0>;  /* 0x5 << 7 */
+				dmas = <&dwdma0 4 0 0>,
+					<&dwdma0 5 0 0>;
 				dma-names = "tx", "rx";
 			};
 
diff --git a/arch/arm/boot/dts/spear600.dtsi b/arch/arm/boot/dts/spear600.dtsi
index 6b32d20acc9f..00166eb9be86 100644
--- a/arch/arm/boot/dts/spear600.dtsi
+++ b/arch/arm/boot/dts/spear600.dtsi
@@ -194,6 +194,7 @@
 			rtc: rtc@fc900000 {
 				compatible = "st,spear600-rtc";
 				reg = <0xfc900000 0x1000>;
+				interrupt-parent = <&vic0>;
 				interrupts = <10>;
 				status = "disabled";
 			};
diff --git a/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi b/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi
index 68aab50a73ab..733678b75b88 100644
--- a/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi
+++ b/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi
@@ -750,6 +750,7 @@
 			reg = <0x10120000 0x1000>;
 			interrupt-names = "combined";
 			interrupts = <14>;
+			interrupt-parent = <&vica>;
 			clocks = <&clcdclk>, <&hclkclcd>;
 			clock-names = "clcdclk", "apb_pclk";
 			status = "disabled";
diff --git a/arch/arm/boot/dts/stih407.dtsi b/arch/arm/boot/dts/stih407.dtsi
index fa149837df14..11fdecd9312e 100644
--- a/arch/arm/boot/dts/stih407.dtsi
+++ b/arch/arm/boot/dts/stih407.dtsi
@@ -8,6 +8,7 @@
  */
 #include "stih407-clock.dtsi"
 #include "stih407-family.dtsi"
+#include <dt-bindings/gpio/gpio.h>
 / {
 	soc {
 		sti-display-subsystem {
@@ -122,7 +123,7 @@
 					 <&clk_s_d2_quadfs 0>,
 					 <&clk_s_d2_quadfs 1>;
 
-				hdmi,hpd-gpio = <&pio5 3>;
+				hdmi,hpd-gpio = <&pio5 3 GPIO_ACTIVE_LOW>;
 				reset-names = "hdmi";
 				resets = <&softreset STIH407_HDMI_TX_PHY_SOFTRESET>;
 				ddc = <&hdmiddc>;
diff --git a/arch/arm/boot/dts/stih410.dtsi b/arch/arm/boot/dts/stih410.dtsi
index cffa50db5d72..68b5ff91d6a7 100644
--- a/arch/arm/boot/dts/stih410.dtsi
+++ b/arch/arm/boot/dts/stih410.dtsi
@@ -9,6 +9,7 @@
 #include "stih410-clock.dtsi"
 #include "stih407-family.dtsi"
 #include "stih410-pinctrl.dtsi"
+#include <dt-bindings/gpio/gpio.h>
 / {
 	aliases {
 		bdisp0 = &bdisp0;
@@ -213,7 +214,7 @@
 					 <&clk_s_d2_quadfs 0>,
 					 <&clk_s_d2_quadfs 1>;
 
-				hdmi,hpd-gpio = <&pio5 3>;
+				hdmi,hpd-gpio = <&pio5 3 GPIO_ACTIVE_LOW>;
 				reset-names = "hdmi";
 				resets = <&softreset STIH407_HDMI_TX_PHY_SOFTRESET>;
 				ddc = <&hdmiddc>;
diff --git a/arch/arm/boot/dts/sun5i-a13-olinuxino.dts b/arch/arm/boot/dts/sun5i-a13-olinuxino.dts
index 38072c7e10e2..4b9af423c6d5 100644
--- a/arch/arm/boot/dts/sun5i-a13-olinuxino.dts
+++ b/arch/arm/boot/dts/sun5i-a13-olinuxino.dts
@@ -114,6 +114,10 @@
 	status = "okay";
 };
 
+&codec {
+	status = "okay";
+};
+
 &ehci0 {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/sun6i-a31s-sinovoip-bpi-m2.dts b/arch/arm/boot/dts/sun6i-a31s-sinovoip-bpi-m2.dts
index 51e6f1d21c32..b2758dd8ce43 100644
--- a/arch/arm/boot/dts/sun6i-a31s-sinovoip-bpi-m2.dts
+++ b/arch/arm/boot/dts/sun6i-a31s-sinovoip-bpi-m2.dts
@@ -42,7 +42,6 @@
 
 /dts-v1/;
 #include "sun6i-a31s.dtsi"
-#include "sunxi-common-regulators.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 
 / {
@@ -99,6 +98,7 @@
 	pinctrl-0 = <&gmac_pins_rgmii_a>, <&gmac_phy_reset_pin_bpi_m2>;
 	phy = <&phy1>;
 	phy-mode = "rgmii";
+	phy-supply = <&reg_dldo1>;
 	snps,reset-gpio = <&pio 0 21 GPIO_ACTIVE_HIGH>; /* PA21 */
 	snps,reset-active-low;
 	snps,reset-delays-us = <0 10000 30000>;
@@ -118,7 +118,7 @@
 &mmc0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin_bpi_m2>;
-	vmmc-supply = <&reg_vcc3v0>;
+	vmmc-supply = <&reg_dcdc1>;
 	bus-width = <4>;
 	cd-gpios = <&pio 0 4 GPIO_ACTIVE_HIGH>; /* PA4 */
 	cd-inverted;
@@ -132,7 +132,7 @@
 &mmc2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc2_pins_a>;
-	vmmc-supply = <&reg_vcc3v0>;
+	vmmc-supply = <&reg_aldo1>;
 	mmc-pwrseq = <&mmc2_pwrseq>;
 	bus-width = <4>;
 	non-removable;
@@ -163,6 +163,8 @@
 		reg = <0x68>;
 		interrupt-parent = <&nmi_intc>;
 		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		eldoin-supply = <&reg_dcdc1>;
+		x-powers,drive-vbus-en;
 	};
 };
 
@@ -193,7 +195,28 @@
 
 #include "axp22x.dtsi"
 
+&reg_aldo1 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi";
+};
+
+&reg_aldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <2500000>;
+	regulator-max-microvolt = <2500000>;
+	regulator-name = "vcc-gmac";
+};
+
+&reg_aldo3 {
+	regulator-always-on;
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+	regulator-name = "avcc";
+};
+
 &reg_dc5ldo {
+	regulator-always-on;
 	regulator-min-microvolt = <700000>;
 	regulator-max-microvolt = <1320000>;
 	regulator-name = "vdd-cpus";
@@ -233,6 +256,40 @@
 	regulator-name = "vcc-dram";
 };
 
+&reg_dldo1 {
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+	regulator-name = "vcc-mac";
+};
+
+&reg_dldo2 {
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-name = "avdd-csi";
+};
+
+&reg_dldo3 {
+	regulator-always-on;
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-pb";
+};
+
+&reg_eldo1 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vdd-csi";
+	status = "okay";
+};
+
+&reg_ldo_io1 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "vcc-pm-cpus";
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pins_a>;
diff --git a/arch/arm/boot/dts/sun7i-a20-pcduino3-nano.dts b/arch/arm/boot/dts/sun7i-a20-pcduino3-nano.dts
index 39bc73db72e5..fb591f32252c 100644
--- a/arch/arm/boot/dts/sun7i-a20-pcduino3-nano.dts
+++ b/arch/arm/boot/dts/sun7i-a20-pcduino3-nano.dts
@@ -58,6 +58,17 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
 	leds {
 		compatible = "gpio-leds";
 		pinctrl-names = "default";
@@ -90,6 +101,10 @@
 	cpu-supply = <&reg_dcdc2>;
 };
 
+&de {
+	status = "okay";
+};
+
 &ehci0 {
 	status = "okay";
 };
@@ -110,6 +125,16 @@
 	};
 };
 
+&hdmi {
+	status = "okay";
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
 &i2c0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c0_pins_a>;
diff --git a/arch/arm/boot/dts/sun8i-a83t-bananapi-m3.dts b/arch/arm/boot/dts/sun8i-a83t-bananapi-m3.dts
index c606af3dbfed..6550bf0e594b 100644
--- a/arch/arm/boot/dts/sun8i-a83t-bananapi-m3.dts
+++ b/arch/arm/boot/dts/sun8i-a83t-bananapi-m3.dts
@@ -52,6 +52,7 @@
 	compatible = "sinovoip,bpi-m3", "allwinner,sun8i-a83t";
 
 	aliases {
+		ethernet0 = &emac;
 		serial0 = &uart0;
 	};
 
@@ -88,6 +89,24 @@
 	/* TODO GL830 USB-to-SATA bridge downstream w/ GPIO power controls */
 };
 
+&emac {
+	pinctrl-names = "default";
+	pinctrl-0 = <&emac_rgmii_pins>;
+	phy-supply = <&reg_sw>;
+	phy-handle = <&rgmii_phy>;
+	phy-mode = "rgmii";
+	allwinner,rx-delay-ps = <700>;
+	allwinner,tx-delay-ps = <700>;
+	status = "okay";
+};
+
+&mdio {
+	rgmii_phy: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+	};
+};
+
 &mmc0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc0_pins>;
diff --git a/arch/arm/boot/dts/sun8i-a83t-cubietruck-plus.dts b/arch/arm/boot/dts/sun8i-a83t-cubietruck-plus.dts
index 7f0a3f6d0cf2..6da08cd0e107 100644
--- a/arch/arm/boot/dts/sun8i-a83t-cubietruck-plus.dts
+++ b/arch/arm/boot/dts/sun8i-a83t-cubietruck-plus.dts
@@ -52,6 +52,7 @@
 	compatible = "cubietech,cubietruck-plus", "allwinner,sun8i-a83t";
 
 	aliases {
+		ethernet0 = &emac;
 		serial0 = &uart0;
 	};
 
@@ -154,6 +155,22 @@
 	status = "okay";
 };
 
+&emac {
+	pinctrl-names = "default";
+	pinctrl-0 = <&emac_rgmii_pins>;
+	phy-supply = <&reg_dldo4>;
+	phy-handle = <&rgmii_phy>;
+	phy-mode = "rgmii";
+	status = "okay";
+};
+
+&mdio {
+	rgmii_phy: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+	};
+};
+
 &mmc0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc0_pins>;
diff --git a/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts b/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts
index a021ee6da396..511fca491fe8 100644
--- a/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts
+++ b/arch/arm/boot/dts/sun8i-a83t-tbs-a711.dts
@@ -45,6 +45,7 @@
 #include "sun8i-a83t.dtsi"
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/pwm/pwm.h>
 
 / {
 	model = "TBS A711 Tablet";
@@ -59,6 +60,44 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm 0 50000 PWM_POLARITY_INVERTED>;
+		enable-gpios = <&pio 3 29 GPIO_ACTIVE_HIGH>;
+
+		brightness-levels = <0 1 2 4 8 16 32 64 128 255>;
+		default-brightness-level = <9>;
+	};
+
+	panel {
+		compatible = "tbs,a711-panel", "panel-lvds";
+		backlight = <&backlight>;
+		power-supply = <&reg_sw>;
+
+		width-mm = <153>;
+		height-mm = <90>;
+		data-mapping = "vesa-24";
+
+		panel-timing {
+			/* 1024x600 @60Hz */
+			clock-frequency = <52000000>;
+			hactive = <1024>;
+			vactive = <600>;
+			hsync-len = <20>;
+			hfront-porch = <180>;
+			hback-porch = <160>;
+			vfront-porch = <12>;
+			vback-porch = <23>;
+			vsync-len = <5>;
+		};
+
+		port {
+			panel_input: endpoint {
+				remote-endpoint = <&tcon0_out_lcd>;
+			};
+		};
+	};
+
 	reg_vbat: reg-vbat {
 		compatible = "regulator-fixed";
 		regulator-name = "vbat";
@@ -89,6 +128,10 @@
 	};
 };
 
+&de {
+	status = "okay";
+};
+
 /*
  * An USB-2 hub is connected here, which also means we don't need to
  * enable the OHCI controller.
@@ -142,6 +185,12 @@
 	status = "okay";
 };
 
+&pwm {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pwm_pin>;
+	status = "okay";
+};
+
 &r_rsb {
 	status = "okay";
 
@@ -323,6 +372,18 @@
 	regulator-name = "vcc-lcd";
 };
 
+&tcon0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&lcd_lvds_pins>;
+};
+
+&tcon0_out {
+	tcon0_out_lcd: endpoint@0 {
+		reg = <0>;
+		remote-endpoint = <&panel_input>;
+	};
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pb_pins>;
diff --git a/arch/arm/boot/dts/sun8i-a83t.dtsi b/arch/arm/boot/dts/sun8i-a83t.dtsi
index 19acae1b4089..7f4955a5fab7 100644
--- a/arch/arm/boot/dts/sun8i-a83t.dtsi
+++ b/arch/arm/boot/dts/sun8i-a83t.dtsi
@@ -45,8 +45,10 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 
 #include <dt-bindings/clock/sun8i-a83t-ccu.h>
+#include <dt-bindings/clock/sun8i-de2.h>
 #include <dt-bindings/clock/sun8i-r-ccu.h>
 #include <dt-bindings/reset/sun8i-a83t-ccu.h>
+#include <dt-bindings/reset/sun8i-de2.h>
 #include <dt-bindings/reset/sun8i-r-ccu.h>
 
 / {
@@ -151,6 +153,12 @@
 		};
 	};
 
+	de: display-engine {
+		compatible = "allwinner,sun8i-a83t-display-engine";
+		allwinner,pipelines = <&mixer0>;
+		status = "disabled";
+	};
+
 	memory {
 		reg = <0x40000000 0x80000000>;
 		device_type = "memory";
@@ -162,6 +170,44 @@
 		#size-cells = <1>;
 		ranges;
 
+		display_clocks: clock@1000000 {
+			compatible = "allwinner,sun8i-a83t-de2-clk";
+			reg = <0x01000000 0x100000>;
+			clocks = <&ccu CLK_PLL_DE>,
+				 <&ccu CLK_BUS_DE>;
+			clock-names = "mod",
+				      "bus";
+			resets = <&ccu RST_BUS_DE>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		mixer0: mixer@1100000 {
+			compatible = "allwinner,sun8i-a83t-de2-mixer-0";
+			reg = <0x01100000 0x100000>;
+			clocks = <&display_clocks CLK_BUS_MIXER0>,
+				 <&display_clocks CLK_MIXER0>;
+			clock-names = "bus",
+				      "mod";
+			resets = <&display_clocks RST_MIXER0>;
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				mixer0_out: port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <1>;
+
+					mixer0_out_tcon0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&tcon0_in_mixer0>;
+					};
+				};
+			};
+		};
+
 		syscon: syscon@1c00000 {
 			compatible = "allwinner,sun8i-a83t-system-controller",
 				"syscon";
@@ -177,6 +223,39 @@
 			#dma-cells = <1>;
 		};
 
+		tcon0: lcd-controller@1c0c000 {
+			compatible = "allwinner,sun8i-a83t-tcon-lcd";
+			reg = <0x01c0c000 0x1000>;
+			interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_TCON0>, <&ccu CLK_TCON0>;
+			clock-names = "ahb", "tcon-ch0";
+			clock-output-names = "tcon-pixel-clock";
+			resets = <&ccu RST_BUS_TCON0>, <&ccu RST_BUS_LVDS>;
+			reset-names = "lcd", "lvds";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				tcon0_in: port@0 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0>;
+
+					tcon0_in_mixer0: endpoint@0 {
+						reg = <0>;
+						remote-endpoint = <&mixer0_out_tcon0>;
+					};
+				};
+
+				tcon0_out: port@1 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <1>;
+				};
+			};
+		};
+
 		mmc0: mmc@1c0f000 {
 			compatible = "allwinner,sun8i-a83t-mmc",
 				     "allwinner,sun7i-a20-mmc";
@@ -336,6 +415,45 @@
 			#interrupt-cells = <3>;
 			#gpio-cells = <3>;
 
+			emac_rgmii_pins: emac-rgmii-pins {
+				pins = "PD2", "PD3", "PD4", "PD5", "PD6", "PD7",
+				       "PD11", "PD12", "PD13", "PD14", "PD18",
+				       "PD19", "PD21", "PD22", "PD23";
+				function = "gmac";
+				/*
+				 * data lines in RGMII mode use DDR mode
+				 * and need a higher signal drive strength
+				 */
+				drive-strength = <40>;
+			};
+
+			i2c0_pins: i2c0-pins {
+				pins = "PH0", "PH1";
+				function = "i2c0";
+			};
+
+			i2c1_pins: i2c1-pins {
+				pins = "PH2", "PH3";
+				function = "i2c1";
+			};
+
+			i2c2_ph_pins: i2c2-ph-pins {
+				pins = "PH4", "PH5";
+				function = "i2c2";
+			};
+
+			i2s1_pins: i2s1-pins {
+				/* I2S1 does not have external MCLK pin */
+				pins = "PG10", "PG11", "PG12", "PG13";
+				function = "i2s1";
+			};
+
+			lcd_lvds_pins: lcd-lvds-pins {
+				pins = "PD18", "PD19", "PD20", "PD21", "PD22",
+				       "PD23", "PD24", "PD25", "PD26", "PD27";
+				function = "lvds0";
+			};
+
 			mmc0_pins: mmc0-pins {
 				pins = "PF0", "PF1", "PF2",
 				       "PF3", "PF4", "PF5";
@@ -361,6 +479,11 @@
 				bias-pull-up;
 			};
 
+			pwm_pin: pwm-pin {
+				pins = "PD28";
+				function = "pwm";
+			};
+
 			spdif_tx_pin: spdif-tx-pin {
 				pins = "PE18";
 				function = "spdif";
@@ -418,6 +541,56 @@
 			status = "disabled";
 		};
 
+		i2s0: i2s@1c22000 {
+			#sound-dai-cells = <0>;
+			compatible = "allwinner,sun8i-a83t-i2s";
+			reg = <0x01c22000 0x400>;
+			interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_I2S0>, <&ccu CLK_I2S0>;
+			clock-names = "apb", "mod";
+			dmas = <&dma 3>, <&dma 3>;
+			resets = <&ccu RST_BUS_I2S0>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		i2s1: i2s@1c22400 {
+			#sound-dai-cells = <0>;
+			compatible = "allwinner,sun8i-a83t-i2s";
+			reg = <0x01c22400 0x400>;
+			interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_I2S1>, <&ccu CLK_I2S1>;
+			clock-names = "apb", "mod";
+			dmas = <&dma 4>, <&dma 4>;
+			resets = <&ccu RST_BUS_I2S1>;
+			dma-names = "rx", "tx";
+			pinctrl-names = "default";
+			pinctrl-0 = <&i2s1_pins>;
+			status = "disabled";
+		};
+
+		i2s2: i2s@1c22800 {
+			#sound-dai-cells = <0>;
+			compatible = "allwinner,sun8i-a83t-i2s";
+			reg = <0x01c22800 0x400>;
+			interrupts = <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_I2S2>, <&ccu CLK_I2S2>;
+			clock-names = "apb", "mod";
+			dmas = <&dma 27>;
+			resets = <&ccu RST_BUS_I2S2>;
+			dma-names = "tx";
+			status = "disabled";
+		};
+
+		pwm: pwm@1c21400 {
+			compatible = "allwinner,sun8i-a83t-pwm",
+				     "allwinner,sun8i-h3-pwm";
+			reg = <0x01c21400 0x400>;
+			clocks = <&osc24M>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		uart0: serial@1c28000 {
 			compatible = "snps,dw-apb-uart";
 			reg = <0x01c28000 0x400>;
@@ -440,6 +613,67 @@
 			status = "disabled";
 		};
 
+		i2c0: i2c@1c2ac00 {
+			compatible = "allwinner,sun8i-a83t-i2c",
+				     "allwinner,sun6i-a31-i2c";
+			reg = <0x01c2ac00 0x400>;
+			interrupts = <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_I2C0>;
+			resets = <&ccu RST_BUS_I2C0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&i2c0_pins>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c1: i2c@1c2b000 {
+			compatible = "allwinner,sun8i-a83t-i2c",
+				     "allwinner,sun6i-a31-i2c";
+			reg = <0x01c2b000 0x400>;
+			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_I2C1>;
+			resets = <&ccu RST_BUS_I2C1>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&i2c1_pins>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		i2c2: i2c@1c2b400 {
+			compatible = "allwinner,sun8i-a83t-i2c",
+				     "allwinner,sun6i-a31-i2c";
+			reg = <0x01c2b400 0x400>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_I2C2>;
+			resets = <&ccu RST_BUS_I2C2>;
+			status = "disabled";
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		emac: ethernet@1c30000 {
+			compatible = "allwinner,sun8i-a83t-emac";
+			syscon = <&syscon>;
+			reg = <0x01c30000 0x104>;
+			interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq";
+			resets = <&ccu 13>;
+			reset-names = "stmmaceth";
+			clocks = <&ccu 27>;
+			clock-names = "stmmaceth";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			mdio: mdio {
+				compatible = "snps,dwmac-mdio";
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
 		gic: interrupt-controller@1c81000 {
 			compatible = "arm,cortex-a7-gic", "arm,cortex-a15-gic";
 			reg = <0x01c81000 0x1000>,
diff --git a/arch/arm/boot/dts/sun8i-h2-plus-orangepi-r1.dts b/arch/arm/boot/dts/sun8i-h2-plus-orangepi-r1.dts
new file mode 100644
index 000000000000..112f09c67d67
--- /dev/null
+++ b/arch/arm/boot/dts/sun8i-h2-plus-orangepi-r1.dts
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2017 Icenowy Zheng <icenowy@aosc.xyz>
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This file is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This file is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/* Orange Pi R1 is based on Orange Pi Zero design */
+#include "sun8i-h2-plus-orangepi-zero.dts"
+
+/ {
+	model = "Xunlong Orange Pi R1";
+	compatible = "xunlong,orangepi-r1", "allwinner,sun8i-h2-plus";
+
+	/delete-node/ reg_vcc_wifi;
+
+	/*
+	 * Ths pin of this regulator is the same with the Wi-Fi extra
+	 * regulator on the original Zero. However it's used for USB
+	 * Ethernet rather than the Wi-Fi now.
+	 */
+	reg_vcc_usb_eth: reg-vcc-usb-ethernet {
+		compatible = "regulator-fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "vcc-usb-ethernet";
+		enable-active-high;
+		gpio = <&pio 0 20 GPIO_ACTIVE_HIGH>;
+	};
+
+	aliases {
+		ethernet1 = &rtl8189etv;
+	};
+};
+
+&ohci1 {
+	/*
+	 * RTL8152B USB-Ethernet adapter is connected to USB1,
+	 * and it's a USB 2.0 device. So the OHCI1 controller
+	 * can be left disabled.
+	 */
+	status = "disabled";
+};
+
+&mmc1 {
+	vmmc-supply = <&reg_vcc3v3>;
+	vqmmc-supply = <&reg_vcc3v3>;
+
+	rtl8189etv: sdio_wifi@1 {
+		reg = <1>;
+	};
+};
+
+&usbphy {
+	usb1_vbus-supply = <&reg_vcc_usb_eth>;
+};
diff --git a/arch/arm/boot/dts/sun8i-h3-bananapi-m2-plus.dts b/arch/arm/boot/dts/sun8i-h3-bananapi-m2-plus.dts
index f2292deaa590..f1c3f1cc4d97 100644
--- a/arch/arm/boot/dts/sun8i-h3-bananapi-m2-plus.dts
+++ b/arch/arm/boot/dts/sun8i-h3-bananapi-m2-plus.dts
@@ -119,7 +119,6 @@
 	phy-handle = <&ext_rgmii_phy>;
 	phy-mode = "rgmii";
 
-	allwinner,leds-active-low;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts b/arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts
new file mode 100644
index 000000000000..d406571a0dd6
--- /dev/null
+++ b/arch/arm/boot/dts/sun8i-h3-libretech-all-h3-cc.dts
@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2017 Chen-Yu Tsai <wens@csie.org>
+ *
+ * SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+ */
+
+/dts-v1/;
+#include "sun8i-h3.dtsi"
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+
+/ {
+	model = "Libre Computer Board ALL-H3-CC H3";
+	compatible = "libretech,all-h3-cc-h3", "allwinner,sun8i-h3";
+
+	aliases {
+		ethernet0 = &emac;
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		pwr_led {
+			label = "librecomputer:green:pwr";
+			gpios = <&r_pio 0 10 GPIO_ACTIVE_HIGH>; /* PL10 */
+			default-state = "on";
+		};
+
+		status_led {
+			label = "librecomputer:blue:status";
+			gpios = <&pio 0 7 GPIO_ACTIVE_HIGH>; /* PA7 */
+		};
+	};
+
+	gpio_keys {
+		compatible = "gpio-keys";
+
+		power {
+			label = "power";
+			linux,code = <KEY_POWER>;
+			gpios = <&r_pio 0 2 GPIO_ACTIVE_LOW>; /* PL2 */
+		};
+	};
+
+	reg_vcc1v2: vcc1v2 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v2";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc5v0>;
+		gpio = <&r_pio 0 8 GPIO_ACTIVE_HIGH>; /* PL8 */
+		enable-active-high;
+	};
+
+	reg_vcc3v3: vcc3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_vcc5v0>;
+	};
+
+	/* This represents the board's 5V input */
+	reg_vcc5v0: vcc5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	reg_vcc_dram: vcc-dram {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc-dram";
+		regulator-min-microvolt = <1500000>;
+		regulator-max-microvolt = <1500000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc5v0>;
+		gpio = <&r_pio 0 9 GPIO_ACTIVE_HIGH>; /* PL9 */
+		enable-active-high;
+	};
+
+	reg_vcc_io: vcc-io {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc-io";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc3v3>;
+		gpio = <&r_pio 0 5 GPIO_ACTIVE_LOW>; /* PL5 */
+	};
+
+	reg_vdd_cpux: vdd-cpux {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd-cpux";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vcc5v0>;
+		gpio = <&r_pio 0 8 GPIO_ACTIVE_HIGH>; /* PL8 */
+		enable-active-high;
+	};
+};
+
+&codec {
+	allwinner,audio-routing =
+		"Line Out", "LINEOUT",
+		"MIC1", "Mic",
+		"Mic",  "MBIAS";
+	status = "okay";
+};
+
+&ehci0 {
+	status = "okay";
+};
+
+&ehci1 {
+	status = "okay";
+};
+
+&ehci2 {
+	status = "okay";
+};
+
+&ehci3 {
+	status = "okay";
+};
+
+&emac {
+	phy-handle = <&int_mii_phy>;
+	phy-mode = "mii";
+	allwinner,leds-active-low;
+	status = "okay";
+};
+
+&ir {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ir_pins_a>;
+	status = "okay";
+};
+
+&mmc0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc0_pins_a>;
+	vmmc-supply = <&reg_vcc_io>;
+	bus-width = <4>;
+	cd-gpios = <&pio 5 6 GPIO_ACTIVE_HIGH>; /* PF6 */
+	cd-inverted;
+	status = "okay";
+};
+
+&ohci0 {
+	status = "okay";
+};
+
+&ohci1 {
+	status = "okay";
+};
+
+&ohci2 {
+	status = "okay";
+};
+
+&ohci3 {
+	status = "okay";
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_pins_a>;
+	status = "okay";
+};
+
+&usb_otg {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usbphy {
+	/* VBUS on USB ports are always on */
+	usb0_vbus-supply = <&reg_vcc5v0>;
+	usb1_vbus-supply = <&reg_vcc5v0>;
+	usb2_vbus-supply = <&reg_vcc5v0>;
+	usb3_vbus-supply = <&reg_vcc5v0>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/sun8i-h3-nanopi-m1-plus.dts b/arch/arm/boot/dts/sun8i-h3-nanopi-m1-plus.dts
index 0a8b79cf5954..a6e61915d648 100644
--- a/arch/arm/boot/dts/sun8i-h3-nanopi-m1-plus.dts
+++ b/arch/arm/boot/dts/sun8i-h3-nanopi-m1-plus.dts
@@ -48,6 +48,7 @@
 
 	aliases {
 		serial1 = &uart3;
+		ethernet0 = &emac;
 		ethernet1 = &sdio_wifi;
 	};
 
@@ -83,8 +84,6 @@
 	phy-handle = <&ext_rgmii_phy>;
 	phy-mode = "rgmii";
 
-	allwinner,leds-active-low;
-
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts b/arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts
index 3a2ccdb28afd..c77fbca4f227 100644
--- a/arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts
+++ b/arch/arm/boot/dts/sun8i-h3-nanopi-m1.dts
@@ -45,6 +45,10 @@
 / {
 	model = "FriendlyArm NanoPi M1";
 	compatible = "friendlyarm,nanopi-m1", "allwinner,sun8i-h3";
+
+	aliases {
+		ethernet0 = &emac;
+	};
 };
 
 &ehci1 {
@@ -55,6 +59,13 @@
 	status = "okay";
 };
 
+&emac {
+	phy-handle = <&int_mii_phy>;
+	phy-mode = "mii";
+	allwinner,leds-active-low;
+	status = "okay";
+};
+
 &ir {
 	pinctrl-names = "default";
 	pinctrl-0 = <&ir_pins_a>;
diff --git a/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts b/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts
index 78f6c24952dd..9f33f6fae595 100644
--- a/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts
+++ b/arch/arm/boot/dts/sun8i-h3-nanopi-neo.dts
@@ -47,9 +47,26 @@
 	compatible = "friendlyarm,nanopi-neo", "allwinner,sun8i-h3";
 };
 
+&ehci0 {
+	status = "okay";
+};
+
 &emac {
 	phy-handle = <&int_mii_phy>;
 	phy-mode = "mii";
 	allwinner,leds-active-low;
 	status = "okay";
 };
+
+&ohci0 {
+	status = "okay";
+};
+
+&usb_otg {
+	status = "okay";
+	dr_mode = "peripheral";
+};
+
+&usbphy {
+	usb0_id_det-gpios = <&pio 6 12 GPIO_ACTIVE_HIGH>; /* PG12 */
+};
diff --git a/arch/arm/boot/dts/sun8i-h3-orangepi-plus.dts b/arch/arm/boot/dts/sun8i-h3-orangepi-plus.dts
index cbc499b04de4..b403e5d787cb 100644
--- a/arch/arm/boot/dts/sun8i-h3-orangepi-plus.dts
+++ b/arch/arm/boot/dts/sun8i-h3-orangepi-plus.dts
@@ -85,7 +85,6 @@
 	phy-handle = <&ext_rgmii_phy>;
 	phy-mode = "rgmii";
 
-	allwinner,leds-active-low;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/sun8i-h3.dtsi b/arch/arm/boot/dts/sun8i-h3.dtsi
index b36f9f423c39..8495deecedad 100644
--- a/arch/arm/boot/dts/sun8i-h3.dtsi
+++ b/arch/arm/boot/dts/sun8i-h3.dtsi
@@ -85,6 +85,10 @@
 	compatible = "allwinner,sun8i-h3-ccu";
 };
 
+&display_clocks {
+	compatible = "allwinner,sun8i-h3-de2-clk";
+};
+
 &mmc0 {
 	compatible = "allwinner,sun7i-a20-mmc";
 	clocks = <&ccu CLK_BUS_MMC0>,
diff --git a/arch/arm/boot/dts/sunxi-h3-h5.dtsi b/arch/arm/boot/dts/sunxi-h3-h5.dtsi
index 8d40c00d64bb..7a83b15225c7 100644
--- a/arch/arm/boot/dts/sunxi-h3-h5.dtsi
+++ b/arch/arm/boot/dts/sunxi-h3-h5.dtsi
@@ -40,9 +40,11 @@
  *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <dt-bindings/clock/sun8i-de2.h>
 #include <dt-bindings/clock/sun8i-h3-ccu.h>
 #include <dt-bindings/clock/sun8i-r-ccu.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/reset/sun8i-de2.h>
 #include <dt-bindings/reset/sun8i-h3-ccu.h>
 #include <dt-bindings/reset/sun8i-r-ccu.h>
 
@@ -51,6 +53,30 @@
 	#address-cells = <1>;
 	#size-cells = <1>;
 
+	chosen {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		framebuffer-hdmi {
+			compatible = "allwinner,simple-framebuffer",
+				     "simple-framebuffer";
+			allwinner,pipeline = "mixer0-lcd0-hdmi";
+			clocks = <&display_clocks CLK_MIXER0>,
+				 <&ccu CLK_TCON0>, <&ccu CLK_HDMI>;
+			status = "disabled";
+		};
+
+		framebuffer-tve {
+			compatible = "allwinner,simple-framebuffer",
+				     "simple-framebuffer";
+			allwinner,pipeline = "mixer1-lcd1-tve";
+			clocks = <&display_clocks CLK_MIXER1>,
+				 <&ccu CLK_TVE>;
+			status = "disabled";
+		};
+	};
+
 	clocks {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -85,6 +111,18 @@
 		#size-cells = <1>;
 		ranges;
 
+		display_clocks: clock@1000000 {
+			/* compatible is in per SoC .dtsi file */
+			reg = <0x01000000 0x100000>;
+			clocks = <&ccu CLK_DE>,
+				 <&ccu CLK_BUS_DE>;
+			clock-names = "mod",
+				      "bus";
+			resets = <&ccu RST_BUS_DE>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
 		syscon: syscon@1c00000 {
 			compatible = "allwinner,sun8i-h3-system-controller",
 				"syscon";
diff --git a/arch/arm/boot/dts/tegra20.dtsi b/arch/arm/boot/dts/tegra20.dtsi
index 914f59166a99..864a95872b8d 100644
--- a/arch/arm/boot/dts/tegra20.dtsi
+++ b/arch/arm/boot/dts/tegra20.dtsi
@@ -10,6 +10,19 @@
 	compatible = "nvidia,tegra20";
 	interrupt-parent = <&lic>;
 
+	iram@40000000 {
+		compatible = "mmio-sram";
+		reg = <0x40000000 0x40000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x40000000 0x40000>;
+
+		vde_pool: vde {
+			reg = <0x400 0x3fc00>;
+			pool;
+		};
+	};
+
 	host1x@50000000 {
 		compatible = "nvidia,tegra20-host1x", "simple-bus";
 		reg = <0x50000000 0x00024000>;
@@ -250,6 +263,28 @@
 		*/
 	};
 
+	vde@6001a000 {
+		compatible = "nvidia,tegra20-vde";
+		reg = <0x6001a000 0x1000   /* Syntax Engine */
+		       0x6001b000 0x1000   /* Video Bitstream Engine */
+		       0x6001c000  0x100   /* Macroblock Engine */
+		       0x6001c200  0x100   /* Post-processing Engine */
+		       0x6001c400  0x100   /* Motion Compensation Engine */
+		       0x6001c600  0x100   /* Transform Engine */
+		       0x6001c800  0x100   /* Pixel prediction block */
+		       0x6001ca00  0x100   /* Video DMA */
+		       0x6001d800  0x300>; /* Video frame controls */
+		reg-names = "sxe", "bsev", "mbe", "ppe", "mce",
+			    "tfe", "ppb", "vdma", "frameid";
+		iram = <&vde_pool>; /* IRAM region */
+		interrupts = <GIC_SPI  9 IRQ_TYPE_LEVEL_HIGH>, /* Sync token interrupt */
+			     <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>, /* BSE-V interrupt */
+			     <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>; /* SXE interrupt */
+		interrupt-names = "sync-token", "bsev", "sxe";
+		clocks = <&tegra_car TEGRA20_CLK_VDE>;
+		resets = <&tegra_car 61>;
+	};
+
 	apbmisc@70000800 {
 		compatible = "nvidia,tegra20-apbmisc";
 		reg = <0x70000800 0x64   /* Chip revision */
diff --git a/arch/arm/boot/dts/tny_a9260_common.dtsi b/arch/arm/boot/dts/tny_a9260_common.dtsi
index 5d83df4675a1..8e3cf5109f98 100644
--- a/arch/arm/boot/dts/tny_a9260_common.dtsi
+++ b/arch/arm/boot/dts/tny_a9260_common.dtsi
@@ -27,6 +27,18 @@
 
 	ahb {
 		apb {
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			dbgu: serial@fffff200 {
 				status = "okay";
 			};
diff --git a/arch/arm/boot/dts/tny_a9263.dts b/arch/arm/boot/dts/tny_a9263.dts
index 8cf0a9e08571..7b403db4cd34 100644
--- a/arch/arm/boot/dts/tny_a9263.dts
+++ b/arch/arm/boot/dts/tny_a9263.dts
@@ -36,6 +36,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fff7c000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			usb1: gadget@fff78000 {
 				atmel,vbus-gpio = <&pioB 11 GPIO_ACTIVE_HIGH>;
 				status = "okay";
diff --git a/arch/arm/boot/dts/uniphier-ld4-ref.dts b/arch/arm/boot/dts/uniphier-ld4-ref.dts
index 0056852c4fb0..a3afd0cda42f 100644
--- a/arch/arm/boot/dts/uniphier-ld4-ref.dts
+++ b/arch/arm/boot/dts/uniphier-ld4-ref.dts
@@ -56,7 +56,7 @@
 &gpio {
 	xirq1 {
 		gpio-hog;
-		gpios = <121 0>;
+		gpios = <UNIPHIER_GPIO_IRQ(1) 0>;
 		input;
 	};
 };
diff --git a/arch/arm/boot/dts/uniphier-ld4.dtsi b/arch/arm/boot/dts/uniphier-ld4.dtsi
index 01fc3e16e2bd..0459e84d4d8e 100644
--- a/arch/arm/boot/dts/uniphier-ld4.dtsi
+++ b/arch/arm/boot/dts/uniphier-ld4.dtsi
@@ -7,6 +7,8 @@
  * SPDX-License-Identifier: (GPL-2.0+ OR MIT)
  */
 
+#include <dt-bindings/gpio/uniphier-gpio.h>
+
 / {
 	compatible = "socionext,uniphier-ld4";
 	#address-cells = <1>;
@@ -235,6 +237,7 @@
 				 <&mio_clk 12>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
 				 <&mio_rst 12>;
+			has-transaction-translator;
 		};
 
 		usb1: usb@5a810100 {
@@ -248,6 +251,7 @@
 				 <&mio_clk 13>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
 				 <&mio_rst 13>;
+			has-transaction-translator;
 		};
 
 		usb2: usb@5a820100 {
@@ -261,6 +265,7 @@
 				 <&mio_clk 14>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 10>,
 				 <&mio_rst 14>;
+			has-transaction-translator;
 		};
 
 		soc-glue@5f800000 {
@@ -273,6 +278,24 @@
 			};
 		};
 
+		soc-glue@5f900000 {
+			compatible = "socionext,uniphier-ld4-soc-glue-debug",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x5f900000 0x2000>;
+
+			efuse@100 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x100 0x28>;
+			};
+
+			efuse@130 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x130 0x8>;
+			};
+		};
+
 		timer@60000200 {
 			compatible = "arm,cortex-a9-global-timer";
 			reg = <0x60000200 0x20>;
diff --git a/arch/arm/boot/dts/uniphier-ld6b-ref.dts b/arch/arm/boot/dts/uniphier-ld6b-ref.dts
index 0e510a725976..811b999800ed 100644
--- a/arch/arm/boot/dts/uniphier-ld6b-ref.dts
+++ b/arch/arm/boot/dts/uniphier-ld6b-ref.dts
@@ -58,7 +58,7 @@
 &gpio {
 	xirq4 {
 		gpio-hog;
-		gpios = <124 0>;
+		gpios = <UNIPHIER_GPIO_IRQ(4) 0>;
 		input;
 	};
 };
diff --git a/arch/arm/boot/dts/uniphier-pro4-ref.dts b/arch/arm/boot/dts/uniphier-pro4-ref.dts
index be99467ac6bb..6a004e5cf786 100644
--- a/arch/arm/boot/dts/uniphier-pro4-ref.dts
+++ b/arch/arm/boot/dts/uniphier-pro4-ref.dts
@@ -58,7 +58,7 @@
 &gpio {
 	xirq2 {
 		gpio-hog;
-		gpios = <122 0>;
+		gpios = <UNIPHIER_GPIO_IRQ(2) 0>;
 		input;
 	};
 };
diff --git a/arch/arm/boot/dts/uniphier-pro4.dtsi b/arch/arm/boot/dts/uniphier-pro4.dtsi
index 7955c3a49e65..1a29a8619856 100644
--- a/arch/arm/boot/dts/uniphier-pro4.dtsi
+++ b/arch/arm/boot/dts/uniphier-pro4.dtsi
@@ -7,6 +7,8 @@
  * SPDX-License-Identifier: (GPL-2.0+ OR MIT)
  */
 
+#include <dt-bindings/gpio/uniphier-gpio.h>
+
 / {
 	compatible = "socionext,uniphier-pro4";
 	#address-cells = <1>;
@@ -269,6 +271,7 @@
 				 <&mio_clk 12>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
 				 <&mio_rst 12>;
+			has-transaction-translator;
 		};
 
 		usb3: usb@5a810100 {
@@ -282,6 +285,7 @@
 				 <&mio_clk 13>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
 				 <&mio_rst 13>;
+			has-transaction-translator;
 		};
 
 		soc-glue@5f800000 {
@@ -294,6 +298,29 @@
 			};
 		};
 
+		soc-glue@5f900000 {
+			compatible = "socionext,uniphier-pro4-soc-glue-debug",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x5f900000 0x2000>;
+
+			efuse@100 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x100 0x28>;
+			};
+
+			efuse@130 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x130 0x8>;
+			};
+
+			efuse@200 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x200 0x14>;
+			};
+		};
+
 		aidet: aidet@5fc20000 {
 			compatible = "socionext,uniphier-pro4-aidet";
 			reg = <0x5fc20000 0x200>;
diff --git a/arch/arm/boot/dts/uniphier-pro5.dtsi b/arch/arm/boot/dts/uniphier-pro5.dtsi
index 6589b8a2c65c..f291dd63de9c 100644
--- a/arch/arm/boot/dts/uniphier-pro5.dtsi
+++ b/arch/arm/boot/dts/uniphier-pro5.dtsi
@@ -355,6 +355,39 @@
 			};
 		};
 
+		soc-glue@5f900000 {
+			compatible = "socionext,uniphier-pro5-soc-glue-debug",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x5f900000 0x2000>;
+
+			efuse@100 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x100 0x28>;
+			};
+
+			efuse@130 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x130 0x8>;
+			};
+
+			efuse@200 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x200 0x28>;
+			};
+
+			efuse@300 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x300 0x14>;
+			};
+
+			efuse@400 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x400 0x8>;
+			};
+		};
+
 		aidet: aidet@5fc20000 {
 			compatible = "socionext,uniphier-pro5-aidet";
 			reg = <0x5fc20000 0x200>;
diff --git a/arch/arm/boot/dts/uniphier-pxs2.dtsi b/arch/arm/boot/dts/uniphier-pxs2.dtsi
index d82d6d872131..c083468c17db 100644
--- a/arch/arm/boot/dts/uniphier-pxs2.dtsi
+++ b/arch/arm/boot/dts/uniphier-pxs2.dtsi
@@ -7,6 +7,7 @@
  * SPDX-License-Identifier: (GPL-2.0+ OR MIT)
  */
 
+#include <dt-bindings/gpio/uniphier-gpio.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -375,6 +376,24 @@
 			};
 		};
 
+		soc-glue@5f900000 {
+			compatible = "socionext,uniphier-pxs2-soc-glue-debug",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x5f900000 0x2000>;
+
+			efuse@100 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x100 0x28>;
+			};
+
+			efuse@200 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x200 0x58>;
+			};
+		};
+
 		aidet: aidet@5fc20000 {
 			compatible = "socionext,uniphier-pxs2-aidet";
 			reg = <0x5fc20000 0x200>;
diff --git a/arch/arm/boot/dts/uniphier-sld8-ref.dts b/arch/arm/boot/dts/uniphier-sld8-ref.dts
index 1c0e7077a560..e052ea3b4020 100644
--- a/arch/arm/boot/dts/uniphier-sld8-ref.dts
+++ b/arch/arm/boot/dts/uniphier-sld8-ref.dts
@@ -56,7 +56,7 @@
 &gpio {
 	xirq0 {
 		gpio-hog;
-		gpios = <120 0>;
+		gpios = <UNIPHIER_GPIO_IRQ(0) 0>;
 		input;
 	};
 };
diff --git a/arch/arm/boot/dts/uniphier-sld8.dtsi b/arch/arm/boot/dts/uniphier-sld8.dtsi
index 71885366cd23..bc8c24078faa 100644
--- a/arch/arm/boot/dts/uniphier-sld8.dtsi
+++ b/arch/arm/boot/dts/uniphier-sld8.dtsi
@@ -7,6 +7,8 @@
  * SPDX-License-Identifier: (GPL-2.0+ OR MIT)
  */
 
+#include <dt-bindings/gpio/uniphier-gpio.h>
+
 / {
 	compatible = "socionext,uniphier-sld8";
 	#address-cells = <1>;
@@ -239,6 +241,7 @@
 				 <&mio_clk 12>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
 				 <&mio_rst 12>;
+			has-transaction-translator;
 		};
 
 		usb1: usb@5a810100 {
@@ -252,6 +255,7 @@
 				 <&mio_clk 13>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
 				 <&mio_rst 13>;
+			has-transaction-translator;
 		};
 
 		usb2: usb@5a820100 {
@@ -265,6 +269,7 @@
 				 <&mio_clk 14>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 10>,
 				 <&mio_rst 14>;
+			has-transaction-translator;
 		};
 
 		soc-glue@5f800000 {
@@ -277,6 +282,24 @@
 			};
 		};
 
+		soc-glue@5f900000 {
+			compatible = "socionext,uniphier-sld8-soc-glue-debug",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x5f900000 0x2000>;
+
+			efuse@100 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x100 0x28>;
+			};
+
+			efuse@200 {
+				compatible = "socionext,uniphier-efuse";
+				reg = <0x200 0x14>;
+			};
+		};
+
 		timer@60000200 {
 			compatible = "arm,cortex-a9-global-timer";
 			reg = <0x60000200 0x20>;
diff --git a/arch/arm/boot/dts/usb_a9260_common.dtsi b/arch/arm/boot/dts/usb_a9260_common.dtsi
index 34a49995eda3..de3fa3ab34d6 100644
--- a/arch/arm/boot/dts/usb_a9260_common.dtsi
+++ b/arch/arm/boot/dts/usb_a9260_common.dtsi
@@ -23,6 +23,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fffa0000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			macb0: ethernet@fffc4000 {
 				phy-mode = "rmii";
 				status = "okay";
diff --git a/arch/arm/boot/dts/usb_a9263.dts b/arch/arm/boot/dts/usb_a9263.dts
index 7b1125be99c4..0a753b4c74e7 100644
--- a/arch/arm/boot/dts/usb_a9263.dts
+++ b/arch/arm/boot/dts/usb_a9263.dts
@@ -36,6 +36,18 @@
 				status = "okay";
 			};
 
+			tcb0: timer@fff7c000 {
+				timer@0 {
+					compatible = "atmel,tcb-timer";
+					reg = <0>, <1>;
+				};
+
+				timer@2 {
+					compatible = "atmel,tcb-timer";
+					reg = <2>;
+				};
+			};
+
 			macb0: ethernet@fffbc000 {
 				phy-mode = "rmii";
 				status = "okay";
diff --git a/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi b/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi
index 091b738041a0..d8b2972527eb 100644
--- a/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi
+++ b/arch/arm/boot/dts/vf-colibri-eval-v3.dtsi
@@ -140,7 +140,7 @@
 
 	/* M41T0M6 real time clock on carrier board */
 	rtc: m41t0m6@68 {
-		compatible = "st,m41t00";
+		compatible = "st,m41t0";
 		reg = <0x68>;
 	};
 };
diff --git a/arch/arm/boot/dts/vf500.dtsi b/arch/arm/boot/dts/vf500.dtsi
index 3d9896171bfc..348bcd30c0f7 100644
--- a/arch/arm/boot/dts/vf500.dtsi
+++ b/arch/arm/boot/dts/vf500.dtsi
@@ -58,7 +58,7 @@
 	soc {
 		aips-bus@40000000 {
 
-			intc: interrupt-controller@40002000 {
+			intc: interrupt-controller@40003000 {
 				compatible = "arm,cortex-a9-gic";
 				#interrupt-cells = <3>;
 				interrupt-controller;
diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
index acdf12ad0622..782b69a3acdf 100644
--- a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
+++ b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
@@ -255,16 +255,19 @@
 					port@0 {
 						reg = <0>;
 						label = "lan6";
+						phy-handle = <&switch2phy0>;
 					};
 
 					port@1 {
 						reg = <1>;
 						label = "lan7";
+						phy-handle = <&switch2phy1>;
 					};
 
 					port@2 {
 						reg = <2>;
 						label = "lan8";
+						phy-handle = <&switch2phy2>;
 					};
 
 					port@3 {
@@ -304,6 +307,20 @@
 						};
 					};
 				};
+				mdio {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					switch2phy0: phy@0 {
+						reg = <0>;
+					};
+					switch2phy1: phy@1 {
+						reg = <1>;
+					};
+					switch2phy2: phy@2 {
+						reg = <2>;
+					};
+				};
 			};
 		};
 
@@ -371,7 +388,8 @@
 		reg = <0x22>;
 		gpio-controller;
 		#gpio-cells = <2>;
-		interrupt-parent = <&gpio2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio3>;
 		interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 	};
 };
diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts
index 4b8edc8982cf..c6f134c78303 100644
--- a/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts
+++ b/arch/arm/boot/dts/vf610-zii-dev-rev-c.dts
@@ -73,7 +73,7 @@
 				#size-cells = <0>;
 				reg = <0>;
 				dsa,member = <0 0>;
-				eeprom-length = <512>;
+				eeprom-length = <65536>;
 				interrupt-parent = <&gpio0>;
 				interrupts = <27 IRQ_TYPE_LEVEL_LOW>;
 				interrupt-controller;
@@ -170,7 +170,7 @@
 				#size-cells = <0>;
 				reg = <0>;
 				dsa,member = <0 1>;
-				eeprom-length = <512>;
+				eeprom-length = <65536>;
 				interrupt-parent = <&gpio0>;
 				interrupts = <26 IRQ_TYPE_LEVEL_LOW>;
 				interrupt-controller;
diff --git a/arch/arm/boot/dts/vf610-zii-dev.dtsi b/arch/arm/boot/dts/vf610-zii-dev.dtsi
index 6b58d3a97992..aadd36db0092 100644
--- a/arch/arm/boot/dts/vf610-zii-dev.dtsi
+++ b/arch/arm/boot/dts/vf610-zii-dev.dtsi
@@ -96,6 +96,10 @@
 	status = "okay";
 };
 
+&edma1 {
+	status = "okay";
+};
+
 &esdhc1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_esdhc1>;
diff --git a/arch/arm/boot/dts/vfxxx.dtsi b/arch/arm/boot/dts/vfxxx.dtsi
index 5d654b5b4ce6..c3f09b737924 100644
--- a/arch/arm/boot/dts/vfxxx.dtsi
+++ b/arch/arm/boot/dts/vfxxx.dtsi
@@ -84,6 +84,11 @@
 		mask = <0x1000>;
 	};
 
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc0 16>, <&adc1 16>;
+	};
+
 	soc {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -762,10 +767,5 @@
 				status = "disabled";
 			};
 		};
-
-		iio-hwmon {
-			compatible = "iio-hwmon";
-			io-channels = <&adc0 16>, <&adc1 16>;
-		};
 	};
 };
diff --git a/arch/arm/boot/dts/zx296702.dtsi b/arch/arm/boot/dts/zx296702.dtsi
index 8a74efdb6360..240e7a23d81f 100644
--- a/arch/arm/boot/dts/zx296702.dtsi
+++ b/arch/arm/boot/dts/zx296702.dtsi
@@ -56,7 +56,7 @@
 			clocks = <&topclk ZX296702_A9_PERIPHCLK>;
 		};
 
-		l2cc: l2-cache-controller@0x00c00000 {
+		l2cc: l2-cache-controller@c00000 {
 			compatible = "arm,pl310-cache";
 			reg = <0x00c00000 0x1000>;
 			cache-unified;
@@ -67,30 +67,30 @@
 			arm,double-linefill-incr = <0>;
 		};
 
-		pcu: pcu@0xa0008000 {
+		pcu: pcu@a0008000 {
 			compatible = "zte,zx296702-pcu";
 			reg = <0xa0008000 0x1000>;
 		};
 
-		topclk: topclk@0x09800000 {
+		topclk: topclk@9800000 {
 			compatible = "zte,zx296702-topcrm-clk";
 			reg = <0x09800000 0x1000>;
 			#clock-cells = <1>;
 		};
 
-		lsp1clk: lsp1clk@0x09400000 {
+		lsp1clk: lsp1clk@9400000 {
 			compatible = "zte,zx296702-lsp1crpm-clk";
 			reg = <0x09400000 0x1000>;
 			#clock-cells = <1>;
 		};
 
-		lsp0clk: lsp0clk@0x0b000000 {
+		lsp0clk: lsp0clk@b000000 {
 			compatible = "zte,zx296702-lsp0crpm-clk";
 			reg = <0x0b000000 0x1000>;
 			#clock-cells = <1>;
 		};
 
-		uart0: serial@0x09405000 {
+		uart0: serial@9405000 {
 			compatible = "zte,zx296702-uart";
 			reg = <0x09405000 0x1000>;
 			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
@@ -98,7 +98,7 @@
 			status = "disabled";
 		};
 
-		uart1: serial@0x09406000 {
+		uart1: serial@9406000 {
 			compatible = "zte,zx296702-uart";
 			reg = <0x09406000 0x1000>;
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
@@ -106,7 +106,7 @@
 			status = "disabled";
 		};
 
-		mmc0: mmc@0x09408000 {
+		mmc0: mmc@9408000 {
 			compatible = "snps,dw-mshc";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -119,7 +119,7 @@
 			status = "disabled";
 		};
 
-		mmc1: mmc@0x0b003000 {
+		mmc1: mmc@b003000 {
 			compatible = "snps,dw-mshc";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -132,7 +132,7 @@
 			status = "disabled";
 		};
 
-		sysctrl: sysctrl@0xa0007000 {
+		sysctrl: sysctrl@a0007000 {
 			compatible = "zte,sysctrl", "syscon";
 			reg = <0xa0007000 0x1000>;
 		};
diff --git a/arch/arm/common/bL_switcher_dummy_if.c b/arch/arm/common/bL_switcher_dummy_if.c
index 4c10c6452678..f4dc1714a79e 100644
--- a/arch/arm/common/bL_switcher_dummy_if.c
+++ b/arch/arm/common/bL_switcher_dummy_if.c
@@ -57,3 +57,7 @@ static struct miscdevice bL_switcher_device = {
 	&bL_switcher_fops
 };
 module_misc_device(bL_switcher_device);
+
+MODULE_AUTHOR("Nicolas Pitre <nico@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("big.LITTLE switcher dummy user interface");
diff --git a/arch/arm/common/sa1111.c b/arch/arm/common/sa1111.c
index 4ecd5120fce7..a2c878769eaf 100644
--- a/arch/arm/common/sa1111.c
+++ b/arch/arm/common/sa1111.c
@@ -108,6 +108,7 @@ struct sa1111 {
 	spinlock_t	lock;
 	void __iomem	*base;
 	struct sa1111_platform_data *pdata;
+	struct irq_domain *irqdomain;
 	struct gpio_chip gc;
 #ifdef CONFIG_PM
 	void		*saved_state;
@@ -125,7 +126,7 @@ struct sa1111_dev_info {
 	unsigned long	skpcr_mask;
 	bool		dma;
 	unsigned int	devid;
-	unsigned int	irq[6];
+	unsigned int	hwirq[6];
 };
 
 static struct sa1111_dev_info sa1111_devices[] = {
@@ -134,7 +135,7 @@ static struct sa1111_dev_info sa1111_devices[] = {
 		.skpcr_mask	= SKPCR_UCLKEN,
 		.dma		= true,
 		.devid		= SA1111_DEVID_USB,
-		.irq = {
+		.hwirq = {
 			IRQ_USBPWR,
 			IRQ_HCIM,
 			IRQ_HCIBUFFACC,
@@ -148,7 +149,7 @@ static struct sa1111_dev_info sa1111_devices[] = {
 		.skpcr_mask	= SKPCR_I2SCLKEN | SKPCR_L3CLKEN,
 		.dma		= true,
 		.devid		= SA1111_DEVID_SAC,
-		.irq = {
+		.hwirq = {
 			AUDXMTDMADONEA,
 			AUDXMTDMADONEB,
 			AUDRCVDMADONEA,
@@ -164,7 +165,7 @@ static struct sa1111_dev_info sa1111_devices[] = {
 		.offset		= SA1111_KBD,
 		.skpcr_mask	= SKPCR_PTCLKEN,
 		.devid		= SA1111_DEVID_PS2_KBD,
-		.irq = {
+		.hwirq = {
 			IRQ_TPRXINT,
 			IRQ_TPTXINT
 		},
@@ -173,7 +174,7 @@ static struct sa1111_dev_info sa1111_devices[] = {
 		.offset		= SA1111_MSE,
 		.skpcr_mask	= SKPCR_PMCLKEN,
 		.devid		= SA1111_DEVID_PS2_MSE,
-		.irq = {
+		.hwirq = {
 			IRQ_MSRXINT,
 			IRQ_MSTXINT
 		},
@@ -182,7 +183,7 @@ static struct sa1111_dev_info sa1111_devices[] = {
 		.offset		= 0x1800,
 		.skpcr_mask	= 0,
 		.devid		= SA1111_DEVID_PCMCIA,
-		.irq = {
+		.hwirq = {
 			IRQ_S0_READY_NINT,
 			IRQ_S0_CD_VALID,
 			IRQ_S0_BVD1_STSCHG,
@@ -193,6 +194,19 @@ static struct sa1111_dev_info sa1111_devices[] = {
 	},
 };
 
+static int sa1111_map_irq(struct sa1111 *sachip, irq_hw_number_t hwirq)
+{
+	return irq_create_mapping(sachip->irqdomain, hwirq);
+}
+
+static void sa1111_handle_irqdomain(struct irq_domain *irqdomain, int irq)
+{
+	struct irq_desc *d = irq_to_desc(irq_linear_revmap(irqdomain, irq));
+
+	if (d)
+		generic_handle_irq_desc(d);
+}
+
 /*
  * SA1111 interrupt support.  Since clearing an IRQ while there are
  * active IRQs causes the interrupt output to pulse, the upper levels
@@ -202,49 +216,45 @@ static void sa1111_irq_handler(struct irq_desc *desc)
 {
 	unsigned int stat0, stat1, i;
 	struct sa1111 *sachip = irq_desc_get_handler_data(desc);
+	struct irq_domain *irqdomain;
 	void __iomem *mapbase = sachip->base + SA1111_INTC;
 
-	stat0 = sa1111_readl(mapbase + SA1111_INTSTATCLR0);
-	stat1 = sa1111_readl(mapbase + SA1111_INTSTATCLR1);
+	stat0 = readl_relaxed(mapbase + SA1111_INTSTATCLR0);
+	stat1 = readl_relaxed(mapbase + SA1111_INTSTATCLR1);
 
-	sa1111_writel(stat0, mapbase + SA1111_INTSTATCLR0);
+	writel_relaxed(stat0, mapbase + SA1111_INTSTATCLR0);
 
 	desc->irq_data.chip->irq_ack(&desc->irq_data);
 
-	sa1111_writel(stat1, mapbase + SA1111_INTSTATCLR1);
+	writel_relaxed(stat1, mapbase + SA1111_INTSTATCLR1);
 
 	if (stat0 == 0 && stat1 == 0) {
 		do_bad_IRQ(desc);
 		return;
 	}
 
+	irqdomain = sachip->irqdomain;
+
 	for (i = 0; stat0; i++, stat0 >>= 1)
 		if (stat0 & 1)
-			generic_handle_irq(i + sachip->irq_base);
+			sa1111_handle_irqdomain(irqdomain, i);
 
 	for (i = 32; stat1; i++, stat1 >>= 1)
 		if (stat1 & 1)
-			generic_handle_irq(i + sachip->irq_base);
+			sa1111_handle_irqdomain(irqdomain, i);
 
 	/* For level-based interrupts */
 	desc->irq_data.chip->irq_unmask(&desc->irq_data);
 }
 
-#define SA1111_IRQMASK_LO(x)	(1 << (x - sachip->irq_base))
-#define SA1111_IRQMASK_HI(x)	(1 << (x - sachip->irq_base - 32))
-
 static u32 sa1111_irqmask(struct irq_data *d)
 {
-	struct sa1111 *sachip = irq_data_get_irq_chip_data(d);
-
-	return BIT((d->irq - sachip->irq_base) & 31);
+	return BIT(irqd_to_hwirq(d) & 31);
 }
 
 static int sa1111_irqbank(struct irq_data *d)
 {
-	struct sa1111 *sachip = irq_data_get_irq_chip_data(d);
-
-	return ((d->irq - sachip->irq_base) / 32) * 4;
+	return (irqd_to_hwirq(d) / 32) * 4;
 }
 
 static void sa1111_ack_irq(struct irq_data *d)
@@ -257,9 +267,9 @@ static void sa1111_mask_irq(struct irq_data *d)
 	void __iomem *mapbase = sachip->base + SA1111_INTC + sa1111_irqbank(d);
 	u32 ie;
 
-	ie = sa1111_readl(mapbase + SA1111_INTEN0);
+	ie = readl_relaxed(mapbase + SA1111_INTEN0);
 	ie &= ~sa1111_irqmask(d);
-	sa1111_writel(ie, mapbase + SA1111_INTEN0);
+	writel(ie, mapbase + SA1111_INTEN0);
 }
 
 static void sa1111_unmask_irq(struct irq_data *d)
@@ -268,9 +278,9 @@ static void sa1111_unmask_irq(struct irq_data *d)
 	void __iomem *mapbase = sachip->base + SA1111_INTC + sa1111_irqbank(d);
 	u32 ie;
 
-	ie = sa1111_readl(mapbase + SA1111_INTEN0);
+	ie = readl_relaxed(mapbase + SA1111_INTEN0);
 	ie |= sa1111_irqmask(d);
-	sa1111_writel(ie, mapbase + SA1111_INTEN0);
+	writel_relaxed(ie, mapbase + SA1111_INTEN0);
 }
 
 /*
@@ -287,11 +297,11 @@ static int sa1111_retrigger_irq(struct irq_data *d)
 	u32 ip, mask = sa1111_irqmask(d);
 	int i;
 
-	ip = sa1111_readl(mapbase + SA1111_INTPOL0);
+	ip = readl_relaxed(mapbase + SA1111_INTPOL0);
 	for (i = 0; i < 8; i++) {
-		sa1111_writel(ip ^ mask, mapbase + SA1111_INTPOL0);
-		sa1111_writel(ip, mapbase + SA1111_INTPOL0);
-		if (sa1111_readl(mapbase + SA1111_INTSTATCLR0) & mask)
+		writel_relaxed(ip ^ mask, mapbase + SA1111_INTPOL0);
+		writel_relaxed(ip, mapbase + SA1111_INTPOL0);
+		if (readl_relaxed(mapbase + SA1111_INTSTATCLR0) & mask)
 			break;
 	}
 
@@ -313,13 +323,13 @@ static int sa1111_type_irq(struct irq_data *d, unsigned int flags)
 	if ((!(flags & IRQ_TYPE_EDGE_RISING) ^ !(flags & IRQ_TYPE_EDGE_FALLING)) == 0)
 		return -EINVAL;
 
-	ip = sa1111_readl(mapbase + SA1111_INTPOL0);
+	ip = readl_relaxed(mapbase + SA1111_INTPOL0);
 	if (flags & IRQ_TYPE_EDGE_RISING)
 		ip &= ~mask;
 	else
 		ip |= mask;
-	sa1111_writel(ip, mapbase + SA1111_INTPOL0);
-	sa1111_writel(ip, mapbase + SA1111_WAKEPOL0);
+	writel_relaxed(ip, mapbase + SA1111_INTPOL0);
+	writel_relaxed(ip, mapbase + SA1111_WAKEPOL0);
 
 	return 0;
 }
@@ -330,12 +340,12 @@ static int sa1111_wake_irq(struct irq_data *d, unsigned int on)
 	void __iomem *mapbase = sachip->base + SA1111_INTC + sa1111_irqbank(d);
 	u32 we, mask = sa1111_irqmask(d);
 
-	we = sa1111_readl(mapbase + SA1111_WAKEEN0);
+	we = readl_relaxed(mapbase + SA1111_WAKEEN0);
 	if (on)
 		we |= mask;
 	else
 		we &= ~mask;
-	sa1111_writel(we, mapbase + SA1111_WAKEEN0);
+	writel_relaxed(we, mapbase + SA1111_WAKEEN0);
 
 	return 0;
 }
@@ -350,10 +360,30 @@ static struct irq_chip sa1111_irq_chip = {
 	.irq_set_wake	= sa1111_wake_irq,
 };
 
+static int sa1111_irqdomain_map(struct irq_domain *d, unsigned int irq,
+	irq_hw_number_t hwirq)
+{
+	struct sa1111 *sachip = d->host_data;
+
+	/* Disallow unavailable interrupts */
+	if (hwirq > SSPROR && hwirq < AUDXMTDMADONEA)
+		return -EINVAL;
+
+	irq_set_chip_data(irq, sachip);
+	irq_set_chip_and_handler(irq, &sa1111_irq_chip, handle_edge_irq);
+	irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
+
+	return 0;
+}
+
+static const struct irq_domain_ops sa1111_irqdomain_ops = {
+	.map = sa1111_irqdomain_map,
+	.xlate = irq_domain_xlate_twocell,
+};
+
 static int sa1111_setup_irq(struct sa1111 *sachip, unsigned irq_base)
 {
 	void __iomem *irqbase = sachip->base + SA1111_INTC;
-	unsigned i, irq;
 	int ret;
 
 	/*
@@ -373,38 +403,40 @@ static int sa1111_setup_irq(struct sa1111 *sachip, unsigned irq_base)
 	sachip->irq_base = ret;
 
 	/* disable all IRQs */
-	sa1111_writel(0, irqbase + SA1111_INTEN0);
-	sa1111_writel(0, irqbase + SA1111_INTEN1);
-	sa1111_writel(0, irqbase + SA1111_WAKEEN0);
-	sa1111_writel(0, irqbase + SA1111_WAKEEN1);
+	writel_relaxed(0, irqbase + SA1111_INTEN0);
+	writel_relaxed(0, irqbase + SA1111_INTEN1);
+	writel_relaxed(0, irqbase + SA1111_WAKEEN0);
+	writel_relaxed(0, irqbase + SA1111_WAKEEN1);
 
 	/*
 	 * detect on rising edge.  Note: Feb 2001 Errata for SA1111
 	 * specifies that S0ReadyInt and S1ReadyInt should be '1'.
 	 */
-	sa1111_writel(0, irqbase + SA1111_INTPOL0);
-	sa1111_writel(BIT(IRQ_S0_READY_NINT & 31) |
-		      BIT(IRQ_S1_READY_NINT & 31),
-		      irqbase + SA1111_INTPOL1);
+	writel_relaxed(0, irqbase + SA1111_INTPOL0);
+	writel_relaxed(BIT(IRQ_S0_READY_NINT & 31) |
+		       BIT(IRQ_S1_READY_NINT & 31),
+		       irqbase + SA1111_INTPOL1);
 
 	/* clear all IRQs */
-	sa1111_writel(~0, irqbase + SA1111_INTSTATCLR0);
-	sa1111_writel(~0, irqbase + SA1111_INTSTATCLR1);
-
-	for (i = IRQ_GPAIN0; i <= SSPROR; i++) {
-		irq = sachip->irq_base + i;
-		irq_set_chip_and_handler(irq, &sa1111_irq_chip, handle_edge_irq);
-		irq_set_chip_data(irq, sachip);
-		irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
-	}
+	writel_relaxed(~0, irqbase + SA1111_INTSTATCLR0);
+	writel_relaxed(~0, irqbase + SA1111_INTSTATCLR1);
 
-	for (i = AUDXMTDMADONEA; i <= IRQ_S1_BVD1_STSCHG; i++) {
-		irq = sachip->irq_base + i;
-		irq_set_chip_and_handler(irq, &sa1111_irq_chip, handle_edge_irq);
-		irq_set_chip_data(irq, sachip);
-		irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE);
+	sachip->irqdomain = irq_domain_add_linear(NULL, SA1111_IRQ_NR,
+						  &sa1111_irqdomain_ops,
+						  sachip);
+	if (!sachip->irqdomain) {
+		irq_free_descs(sachip->irq_base, SA1111_IRQ_NR);
+		return -ENOMEM;
 	}
 
+	irq_domain_associate_many(sachip->irqdomain,
+				  sachip->irq_base + IRQ_GPAIN0,
+				  IRQ_GPAIN0, SSPROR + 1 - IRQ_GPAIN0);
+	irq_domain_associate_many(sachip->irqdomain,
+				  sachip->irq_base + AUDXMTDMADONEA,
+				  AUDXMTDMADONEA,
+				  IRQ_S1_BVD1_STSCHG + 1 - AUDXMTDMADONEA);
+
 	/*
 	 * Register SA1111 interrupt
 	 */
@@ -420,20 +452,22 @@ static int sa1111_setup_irq(struct sa1111 *sachip, unsigned irq_base)
 
 static void sa1111_remove_irq(struct sa1111 *sachip)
 {
+	struct irq_domain *domain = sachip->irqdomain;
 	void __iomem *irqbase = sachip->base + SA1111_INTC;
+	int i;
 
 	/* disable all IRQs */
-	sa1111_writel(0, irqbase + SA1111_INTEN0);
-	sa1111_writel(0, irqbase + SA1111_INTEN1);
-	sa1111_writel(0, irqbase + SA1111_WAKEEN0);
-	sa1111_writel(0, irqbase + SA1111_WAKEEN1);
+	writel_relaxed(0, irqbase + SA1111_INTEN0);
+	writel_relaxed(0, irqbase + SA1111_INTEN1);
+	writel_relaxed(0, irqbase + SA1111_WAKEEN0);
+	writel_relaxed(0, irqbase + SA1111_WAKEEN1);
 
-	if (sachip->irq != NO_IRQ) {
-		irq_set_chained_handler_and_data(sachip->irq, NULL, NULL);
-		irq_free_descs(sachip->irq_base, SA1111_IRQ_NR);
+	irq_set_chained_handler_and_data(sachip->irq, NULL, NULL);
+	for (i = 0; i < SA1111_IRQ_NR; i++)
+		irq_dispose_mapping(irq_find_mapping(domain, i));
+	irq_domain_remove(domain);
 
-		release_mem_region(sachip->phys + SA1111_INTC, 512);
-	}
+	release_mem_region(sachip->phys + SA1111_INTC, 512);
 }
 
 enum {
@@ -572,7 +606,7 @@ static int sa1111_gpio_to_irq(struct gpio_chip *gc, unsigned offset)
 {
 	struct sa1111 *sachip = gc_to_sa1111(gc);
 
-	return sachip->irq_base + offset;
+	return sa1111_map_irq(sachip, offset);
 }
 
 static int sa1111_setup_gpios(struct sa1111 *sachip)
@@ -618,11 +652,11 @@ static void sa1111_wake(struct sa1111 *sachip)
 	/*
 	 * Turn VCO on, and disable PLL Bypass.
 	 */
-	r = sa1111_readl(sachip->base + SA1111_SKCR);
+	r = readl_relaxed(sachip->base + SA1111_SKCR);
 	r &= ~SKCR_VCO_OFF;
-	sa1111_writel(r, sachip->base + SA1111_SKCR);
+	writel_relaxed(r, sachip->base + SA1111_SKCR);
 	r |= SKCR_PLL_BYPASS | SKCR_OE_EN;
-	sa1111_writel(r, sachip->base + SA1111_SKCR);
+	writel_relaxed(r, sachip->base + SA1111_SKCR);
 
 	/*
 	 * Wait lock time.  SA1111 manual _doesn't_
@@ -634,7 +668,7 @@ static void sa1111_wake(struct sa1111 *sachip)
 	 * Enable RCLK.  We also ensure that RDYEN is set.
 	 */
 	r |= SKCR_RCLKEN | SKCR_RDYEN;
-	sa1111_writel(r, sachip->base + SA1111_SKCR);
+	writel_relaxed(r, sachip->base + SA1111_SKCR);
 
 	/*
 	 * Wait 14 RCLK cycles for the chip to finish coming out
@@ -645,7 +679,7 @@ static void sa1111_wake(struct sa1111 *sachip)
 	/*
 	 * Ensure all clocks are initially off.
 	 */
-	sa1111_writel(0, sachip->base + SA1111_SKPCR);
+	writel_relaxed(0, sachip->base + SA1111_SKPCR);
 
 	spin_unlock_irqrestore(&sachip->lock, flags);
 }
@@ -675,7 +709,7 @@ sa1111_configure_smc(struct sa1111 *sachip, int sdram, unsigned int drac,
 	if (cas_latency == 3)
 		smcr |= SMCR_CLAT;
 
-	sa1111_writel(smcr, sachip->base + SA1111_SMCR);
+	writel_relaxed(smcr, sachip->base + SA1111_SMCR);
 
 	/*
 	 * Now clear the bits in the DMA mask to work around the SA1111
@@ -723,8 +757,8 @@ sa1111_init_one_child(struct sa1111 *sachip, struct resource *parent,
 	dev->mapbase     = sachip->base + info->offset;
 	dev->skpcr_mask  = info->skpcr_mask;
 
-	for (i = 0; i < ARRAY_SIZE(info->irq); i++)
-		dev->irq[i] = sachip->irq_base + info->irq[i];
+	for (i = 0; i < ARRAY_SIZE(info->hwirq); i++)
+		dev->hwirq[i] = info->hwirq[i];
 
 	/*
 	 * If the parent device has a DMA mask associated with it, and
@@ -814,7 +848,7 @@ static int __sa1111_probe(struct device *me, struct resource *mem, int irq)
 	/*
 	 * Probe for the chip.  Only touch the SBI registers.
 	 */
-	id = sa1111_readl(sachip->base + SA1111_SKID);
+	id = readl_relaxed(sachip->base + SA1111_SKID);
 	if ((id & SKID_ID_MASK) != SKID_SA1111_ID) {
 		printk(KERN_DEBUG "SA1111 not detected: ID = %08lx\n", id);
 		ret = -ENODEV;
@@ -833,11 +867,9 @@ static int __sa1111_probe(struct device *me, struct resource *mem, int irq)
 	 * The interrupt controller must be initialised before any
 	 * other device to ensure that the interrupts are available.
 	 */
-	if (sachip->irq != NO_IRQ) {
-		ret = sa1111_setup_irq(sachip, pd->irq_base);
-		if (ret)
-			goto err_clk;
-	}
+	ret = sa1111_setup_irq(sachip, pd->irq_base);
+	if (ret)
+		goto err_clk;
 
 	/* Setup the GPIOs - should really be done after the IRQ setup */
 	ret = sa1111_setup_gpios(sachip);
@@ -864,8 +896,8 @@ static int __sa1111_probe(struct device *me, struct resource *mem, int irq)
 	 * DMA.  It can otherwise be held firmly in the off position.
 	 * (currently, we always enable it.)
 	 */
-	val = sa1111_readl(sachip->base + SA1111_SKPCR);
-	sa1111_writel(val | SKPCR_DCLKEN, sachip->base + SA1111_SKPCR);
+	val = readl_relaxed(sachip->base + SA1111_SKPCR);
+	writel_relaxed(val | SKPCR_DCLKEN, sachip->base + SA1111_SKPCR);
 
 	/*
 	 * Enable the SA1110 memory bus request and grant signals.
@@ -962,31 +994,31 @@ static int sa1111_suspend_noirq(struct device *dev)
 	 * Save state.
 	 */
 	base = sachip->base;
-	save->skcr     = sa1111_readl(base + SA1111_SKCR);
-	save->skpcr    = sa1111_readl(base + SA1111_SKPCR);
-	save->skcdr    = sa1111_readl(base + SA1111_SKCDR);
-	save->skaud    = sa1111_readl(base + SA1111_SKAUD);
-	save->skpwm0   = sa1111_readl(base + SA1111_SKPWM0);
-	save->skpwm1   = sa1111_readl(base + SA1111_SKPWM1);
+	save->skcr     = readl_relaxed(base + SA1111_SKCR);
+	save->skpcr    = readl_relaxed(base + SA1111_SKPCR);
+	save->skcdr    = readl_relaxed(base + SA1111_SKCDR);
+	save->skaud    = readl_relaxed(base + SA1111_SKAUD);
+	save->skpwm0   = readl_relaxed(base + SA1111_SKPWM0);
+	save->skpwm1   = readl_relaxed(base + SA1111_SKPWM1);
 
-	sa1111_writel(0, sachip->base + SA1111_SKPWM0);
-	sa1111_writel(0, sachip->base + SA1111_SKPWM1);
+	writel_relaxed(0, sachip->base + SA1111_SKPWM0);
+	writel_relaxed(0, sachip->base + SA1111_SKPWM1);
 
 	base = sachip->base + SA1111_INTC;
-	save->intpol0  = sa1111_readl(base + SA1111_INTPOL0);
-	save->intpol1  = sa1111_readl(base + SA1111_INTPOL1);
-	save->inten0   = sa1111_readl(base + SA1111_INTEN0);
-	save->inten1   = sa1111_readl(base + SA1111_INTEN1);
-	save->wakepol0 = sa1111_readl(base + SA1111_WAKEPOL0);
-	save->wakepol1 = sa1111_readl(base + SA1111_WAKEPOL1);
-	save->wakeen0  = sa1111_readl(base + SA1111_WAKEEN0);
-	save->wakeen1  = sa1111_readl(base + SA1111_WAKEEN1);
+	save->intpol0  = readl_relaxed(base + SA1111_INTPOL0);
+	save->intpol1  = readl_relaxed(base + SA1111_INTPOL1);
+	save->inten0   = readl_relaxed(base + SA1111_INTEN0);
+	save->inten1   = readl_relaxed(base + SA1111_INTEN1);
+	save->wakepol0 = readl_relaxed(base + SA1111_WAKEPOL0);
+	save->wakepol1 = readl_relaxed(base + SA1111_WAKEPOL1);
+	save->wakeen0  = readl_relaxed(base + SA1111_WAKEEN0);
+	save->wakeen1  = readl_relaxed(base + SA1111_WAKEEN1);
 
 	/*
 	 * Disable.
 	 */
-	val = sa1111_readl(sachip->base + SA1111_SKCR);
-	sa1111_writel(val | SKCR_SLEEP, sachip->base + SA1111_SKCR);
+	val = readl_relaxed(sachip->base + SA1111_SKCR);
+	writel_relaxed(val | SKCR_SLEEP, sachip->base + SA1111_SKCR);
 
 	clk_disable(sachip->clk);
 
@@ -1023,7 +1055,7 @@ static int sa1111_resume_noirq(struct device *dev)
 	 * Ensure that the SA1111 is still here.
 	 * FIXME: shouldn't do this here.
 	 */
-	id = sa1111_readl(sachip->base + SA1111_SKID);
+	id = readl_relaxed(sachip->base + SA1111_SKID);
 	if ((id & SKID_ID_MASK) != SKID_SA1111_ID) {
 		__sa1111_remove(sachip);
 		dev_set_drvdata(dev, NULL);
@@ -1047,26 +1079,26 @@ static int sa1111_resume_noirq(struct device *dev)
 	 */
 	spin_lock_irqsave(&sachip->lock, flags);
 
-	sa1111_writel(0, sachip->base + SA1111_INTC + SA1111_INTEN0);
-	sa1111_writel(0, sachip->base + SA1111_INTC + SA1111_INTEN1);
+	writel_relaxed(0, sachip->base + SA1111_INTC + SA1111_INTEN0);
+	writel_relaxed(0, sachip->base + SA1111_INTC + SA1111_INTEN1);
 
 	base = sachip->base;
-	sa1111_writel(save->skcr,     base + SA1111_SKCR);
-	sa1111_writel(save->skpcr,    base + SA1111_SKPCR);
-	sa1111_writel(save->skcdr,    base + SA1111_SKCDR);
-	sa1111_writel(save->skaud,    base + SA1111_SKAUD);
-	sa1111_writel(save->skpwm0,   base + SA1111_SKPWM0);
-	sa1111_writel(save->skpwm1,   base + SA1111_SKPWM1);
+	writel_relaxed(save->skcr,     base + SA1111_SKCR);
+	writel_relaxed(save->skpcr,    base + SA1111_SKPCR);
+	writel_relaxed(save->skcdr,    base + SA1111_SKCDR);
+	writel_relaxed(save->skaud,    base + SA1111_SKAUD);
+	writel_relaxed(save->skpwm0,   base + SA1111_SKPWM0);
+	writel_relaxed(save->skpwm1,   base + SA1111_SKPWM1);
 
 	base = sachip->base + SA1111_INTC;
-	sa1111_writel(save->intpol0,  base + SA1111_INTPOL0);
-	sa1111_writel(save->intpol1,  base + SA1111_INTPOL1);
-	sa1111_writel(save->inten0,   base + SA1111_INTEN0);
-	sa1111_writel(save->inten1,   base + SA1111_INTEN1);
-	sa1111_writel(save->wakepol0, base + SA1111_WAKEPOL0);
-	sa1111_writel(save->wakepol1, base + SA1111_WAKEPOL1);
-	sa1111_writel(save->wakeen0,  base + SA1111_WAKEEN0);
-	sa1111_writel(save->wakeen1,  base + SA1111_WAKEEN1);
+	writel_relaxed(save->intpol0,  base + SA1111_INTPOL0);
+	writel_relaxed(save->intpol1,  base + SA1111_INTPOL1);
+	writel_relaxed(save->inten0,   base + SA1111_INTEN0);
+	writel_relaxed(save->inten1,   base + SA1111_INTEN1);
+	writel_relaxed(save->wakepol0, base + SA1111_WAKEPOL0);
+	writel_relaxed(save->wakepol1, base + SA1111_WAKEPOL1);
+	writel_relaxed(save->wakeen0,  base + SA1111_WAKEEN0);
+	writel_relaxed(save->wakeen1,  base + SA1111_WAKEEN1);
 
 	spin_unlock_irqrestore(&sachip->lock, flags);
 
@@ -1153,7 +1185,7 @@ static unsigned int __sa1111_pll_clock(struct sa1111 *sachip)
 {
 	unsigned int skcdr, fbdiv, ipdiv, opdiv;
 
-	skcdr = sa1111_readl(sachip->base + SA1111_SKCDR);
+	skcdr = readl_relaxed(sachip->base + SA1111_SKCDR);
 
 	fbdiv = (skcdr & 0x007f) + 2;
 	ipdiv = ((skcdr & 0x0f80) >> 7) + 2;
@@ -1195,13 +1227,13 @@ void sa1111_select_audio_mode(struct sa1111_dev *sadev, int mode)
 
 	spin_lock_irqsave(&sachip->lock, flags);
 
-	val = sa1111_readl(sachip->base + SA1111_SKCR);
+	val = readl_relaxed(sachip->base + SA1111_SKCR);
 	if (mode == SA1111_AUDIO_I2S) {
 		val &= ~SKCR_SELAC;
 	} else {
 		val |= SKCR_SELAC;
 	}
-	sa1111_writel(val, sachip->base + SA1111_SKCR);
+	writel_relaxed(val, sachip->base + SA1111_SKCR);
 
 	spin_unlock_irqrestore(&sachip->lock, flags);
 }
@@ -1226,7 +1258,7 @@ int sa1111_set_audio_rate(struct sa1111_dev *sadev, int rate)
 	if (div > 128)
 		div = 128;
 
-	sa1111_writel(div - 1, sachip->base + SA1111_SKAUD);
+	writel_relaxed(div - 1, sachip->base + SA1111_SKAUD);
 
 	return 0;
 }
@@ -1244,7 +1276,7 @@ int sa1111_get_audio_rate(struct sa1111_dev *sadev)
 	if (sadev->devid != SA1111_DEVID_SAC)
 		return -EINVAL;
 
-	div = sa1111_readl(sachip->base + SA1111_SKAUD) + 1;
+	div = readl_relaxed(sachip->base + SA1111_SKAUD) + 1;
 
 	return __sa1111_pll_clock(sachip) / (256 * div);
 }
@@ -1261,10 +1293,10 @@ void sa1111_set_io_dir(struct sa1111_dev *sadev,
 
 #define MODIFY_BITS(port, mask, dir)		\
 	if (mask) {				\
-		val = sa1111_readl(port);	\
+		val = readl_relaxed(port);	\
 		val &= ~(mask);			\
 		val |= (dir) & (mask);		\
-		sa1111_writel(val, port);	\
+		writel_relaxed(val, port);	\
 	}
 
 	spin_lock_irqsave(&sachip->lock, flags);
@@ -1329,8 +1361,8 @@ int sa1111_enable_device(struct sa1111_dev *sadev)
 
 	if (ret == 0) {
 		spin_lock_irqsave(&sachip->lock, flags);
-		val = sa1111_readl(sachip->base + SA1111_SKPCR);
-		sa1111_writel(val | sadev->skpcr_mask, sachip->base + SA1111_SKPCR);
+		val = readl_relaxed(sachip->base + SA1111_SKPCR);
+		writel_relaxed(val | sadev->skpcr_mask, sachip->base + SA1111_SKPCR);
 		spin_unlock_irqrestore(&sachip->lock, flags);
 	}
 	return ret;
@@ -1348,8 +1380,8 @@ void sa1111_disable_device(struct sa1111_dev *sadev)
 	unsigned int val;
 
 	spin_lock_irqsave(&sachip->lock, flags);
-	val = sa1111_readl(sachip->base + SA1111_SKPCR);
-	sa1111_writel(val & ~sadev->skpcr_mask, sachip->base + SA1111_SKPCR);
+	val = readl_relaxed(sachip->base + SA1111_SKPCR);
+	writel_relaxed(val & ~sadev->skpcr_mask, sachip->base + SA1111_SKPCR);
 	spin_unlock_irqrestore(&sachip->lock, flags);
 
 	if (sachip->pdata && sachip->pdata->disable)
@@ -1359,9 +1391,10 @@ EXPORT_SYMBOL(sa1111_disable_device);
 
 int sa1111_get_irq(struct sa1111_dev *sadev, unsigned num)
 {
-	if (num >= ARRAY_SIZE(sadev->irq))
+	struct sa1111 *sachip = sa1111_chip_driver(sadev);
+	if (num >= ARRAY_SIZE(sadev->hwirq))
 		return -EINVAL;
-	return sadev->irq[num];
+	return sa1111_map_irq(sachip, sadev->hwirq[num]);
 }
 EXPORT_SYMBOL_GPL(sa1111_get_irq);
 
@@ -1379,36 +1412,6 @@ static int sa1111_match(struct device *_dev, struct device_driver *_drv)
 	return !!(dev->devid & drv->devid);
 }
 
-static int sa1111_bus_suspend(struct device *dev, pm_message_t state)
-{
-	struct sa1111_dev *sadev = to_sa1111_device(dev);
-	struct sa1111_driver *drv = SA1111_DRV(dev->driver);
-	int ret = 0;
-
-	if (drv && drv->suspend)
-		ret = drv->suspend(sadev, state);
-	return ret;
-}
-
-static int sa1111_bus_resume(struct device *dev)
-{
-	struct sa1111_dev *sadev = to_sa1111_device(dev);
-	struct sa1111_driver *drv = SA1111_DRV(dev->driver);
-	int ret = 0;
-
-	if (drv && drv->resume)
-		ret = drv->resume(sadev);
-	return ret;
-}
-
-static void sa1111_bus_shutdown(struct device *dev)
-{
-	struct sa1111_driver *drv = SA1111_DRV(dev->driver);
-
-	if (drv && drv->shutdown)
-		drv->shutdown(to_sa1111_device(dev));
-}
-
 static int sa1111_bus_probe(struct device *dev)
 {
 	struct sa1111_dev *sadev = to_sa1111_device(dev);
@@ -1436,9 +1439,6 @@ struct bus_type sa1111_bus_type = {
 	.match		= sa1111_match,
 	.probe		= sa1111_bus_probe,
 	.remove		= sa1111_bus_remove,
-	.suspend	= sa1111_bus_suspend,
-	.resume		= sa1111_bus_resume,
-	.shutdown	= sa1111_bus_shutdown,
 };
 EXPORT_SYMBOL(sa1111_bus_type);
 
diff --git a/arch/arm/configs/aspeed_g4_defconfig b/arch/arm/configs/aspeed_g4_defconfig
index d23b9d56a88b..95946dee9c77 100644
--- a/arch/arm/configs/aspeed_g4_defconfig
+++ b/arch/arm/configs/aspeed_g4_defconfig
@@ -1,7 +1,6 @@
 CONFIG_KERNEL_XZ=y
 # CONFIG_SWAP is not set
 CONFIG_SYSVIPC=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_LOG_BUF_SHIFT=14
diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig
index c0ad7b82086b..8c7ea033cdc2 100644
--- a/arch/arm/configs/aspeed_g5_defconfig
+++ b/arch/arm/configs/aspeed_g5_defconfig
@@ -1,7 +1,6 @@
 CONFIG_KERNEL_XZ=y
 # CONFIG_SWAP is not set
 CONFIG_SYSVIPC=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_LOG_BUF_SHIFT=14
diff --git a/arch/arm/configs/cns3420vb_defconfig b/arch/arm/configs/cns3420vb_defconfig
index 63a953d855a6..c6dcd6e4f4e6 100644
--- a/arch/arm/configs/cns3420vb_defconfig
+++ b/arch/arm/configs/cns3420vb_defconfig
@@ -28,7 +28,6 @@ CONFIG_ZBOOT_ROM_TEXT=0x0
 CONFIG_ZBOOT_ROM_BSS=0x0
 CONFIG_CMDLINE="console=ttyS0,38400 mem=128M root=/dev/mmcblk0p1 ro rootwait"
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig
index bd0cf22f9ceb..026154c1d55a 100644
--- a/arch/arm/configs/davinci_all_defconfig
+++ b/arch/arm/configs/davinci_all_defconfig
@@ -94,6 +94,7 @@ CONFIG_PPP=m
 CONFIG_PPP_DEFLATE=m
 CONFIG_PPP_ASYNC=m
 CONFIG_PPP_SYNC_TTY=m
+CONFIG_USB_USBNET=m
 CONFIG_INPUT_EVDEV=m
 CONFIG_INPUT_EVBUG=m
 CONFIG_KEYBOARD_ATKBD=m
diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig
index f1d7834990ec..629189c62fd1 100644
--- a/arch/arm/configs/exynos_defconfig
+++ b/arch/arm/configs/exynos_defconfig
@@ -3,6 +3,7 @@ CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_CGROUPS=y
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_PERF_EVENTS=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_PARTITION_ADVANCED=y
@@ -179,6 +180,7 @@ CONFIG_REGULATOR_TPS65090=y
 CONFIG_REGULATOR_WM8994=y
 CONFIG_MEDIA_SUPPORT=m
 CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_CEC_SUPPORT=y
 CONFIG_MEDIA_CONTROLLER=y
 CONFIG_VIDEO_V4L2_SUBDEV_API=y
 CONFIG_MEDIA_USB_SUPPORT=y
@@ -195,6 +197,8 @@ CONFIG_VIDEO_SAMSUNG_S5P_MFC=m
 CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m
 CONFIG_V4L_TEST_DRIVERS=y
 CONFIG_VIDEO_VIVID=m
+CONFIG_CEC_PLATFORM_DRIVERS=y
+CONFIG_VIDEO_SAMSUNG_S5P_CEC=m
 CONFIG_DRM=y
 CONFIG_DRM_EXYNOS=y
 CONFIG_DRM_EXYNOS_FIMD=y
@@ -207,6 +211,7 @@ CONFIG_DRM_PANEL_SAMSUNG_LD9040=y
 CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=y
 CONFIG_DRM_NXP_PTN3460=y
 CONFIG_DRM_PARADE_PS8622=y
+CONFIG_DRM_SII9234=y
 CONFIG_LCD_CLASS_DEVICE=y
 CONFIG_LCD_PLATFORM=y
 CONFIG_BACKLIGHT_PWM=y
@@ -218,6 +223,7 @@ CONFIG_SND_SOC_SAMSUNG=y
 CONFIG_SND_SOC_SAMSUNG_SMDK_WM8994=y
 CONFIG_SND_SOC_SMDK_WM8994_PCM=y
 CONFIG_SND_SOC_SNOW=y
+CONFIG_SND_SOC_ODROID=y
 CONFIG_SND_SIMPLE_CARD=y
 CONFIG_USB=y
 CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
@@ -281,6 +287,7 @@ CONFIG_DEVFREQ_GOV_POWERSAVE=y
 CONFIG_DEVFREQ_GOV_USERSPACE=y
 CONFIG_ARM_EXYNOS_BUS_DEVFREQ=y
 CONFIG_DEVFREQ_EVENT_EXYNOS_NOCP=y
+CONFIG_EXYNOS_IOMMU=y
 CONFIG_EXTCON=y
 CONFIG_EXTCON_MAX14577=y
 CONFIG_EXTCON_MAX77693=y
diff --git a/arch/arm/configs/hisi_defconfig b/arch/arm/configs/hisi_defconfig
index b2e340b272ee..74d611e41e02 100644
--- a/arch/arm/configs/hisi_defconfig
+++ b/arch/arm/configs/hisi_defconfig
@@ -1,4 +1,3 @@
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BLK_DEV_INITRD=y
diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig
index 0d4494922561..4cb9829fccd1 100644
--- a/arch/arm/configs/imx_v6_v7_defconfig
+++ b/arch/arm/configs/imx_v6_v7_defconfig
@@ -55,6 +55,7 @@ CONFIG_FORCE_MAX_ZONEORDER=14
 CONFIG_CMDLINE="noinitrd console=ttymxc0,115200"
 CONFIG_KEXEC=y
 CONFIG_CPU_FREQ=y
+CONFIG_CPU_FREQ_STAT=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_POWERSAVE=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
@@ -80,6 +81,7 @@ CONFIG_CAN=y
 CONFIG_CAN_FLEXCAN=y
 CONFIG_BT=y
 CONFIG_BT_HCIUART=y
+CONFIG_BT_HCIUART_SERDEV=y
 CONFIG_BT_HCIUART_H4=y
 CONFIG_BT_HCIUART_LL=y
 CONFIG_CFG80211=y
@@ -155,6 +157,7 @@ CONFIG_USB_USBNET=y
 CONFIG_USB_NET_CDC_EEM=m
 CONFIG_BRCMFMAC=m
 CONFIG_WL12XX=m
+CONFIG_WL18XX=m
 CONFIG_WLCORE_SDIO=m
 # CONFIG_WILINK_PLATFORM_DATA is not set
 CONFIG_INPUT_EVDEV=y
@@ -185,6 +188,8 @@ CONFIG_SERIAL_IMX=y
 CONFIG_SERIAL_IMX_CONSOLE=y
 CONFIG_SERIAL_FSL_LPUART=y
 CONFIG_SERIAL_FSL_LPUART_CONSOLE=y
+CONFIG_SERIAL_DEV_BUS=y
+CONFIG_SERIAL_DEV_CTRL_TTYPORT=y
 # CONFIG_I2C_COMPAT is not set
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_MUX=y
@@ -248,6 +253,7 @@ CONFIG_VIDEO_OV5640=m
 CONFIG_SOC_CAMERA_OV2640=y
 CONFIG_IMX_IPUV3_CORE=y
 CONFIG_DRM=y
+CONFIG_DRM_PANEL_LVDS=y
 CONFIG_DRM_PANEL_SIMPLE=y
 CONFIG_DRM_DW_HDMI_AHB_AUDIO=m
 CONFIG_DRM_DW_HDMI_CEC=y
@@ -340,6 +346,7 @@ CONFIG_LEDS_TRIGGER_ONESHOT=y
 CONFIG_LEDS_TRIGGER_HEARTBEAT=y
 CONFIG_LEDS_TRIGGER_BACKLIGHT=y
 CONFIG_LEDS_TRIGGER_GPIO=y
+CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_INTF_DEV_UIE_EMUL=y
 CONFIG_RTC_DRV_DS1307=y
@@ -349,6 +356,7 @@ CONFIG_RTC_DRV_PCF8563=y
 CONFIG_RTC_DRV_M41T80=y
 CONFIG_RTC_DRV_MC13XXX=y
 CONFIG_RTC_DRV_MXC=y
+CONFIG_RTC_DRV_MXC_V2=y
 CONFIG_RTC_DRV_SNVS=y
 CONFIG_DMADEVICES=y
 CONFIG_FSL_EDMA=y
diff --git a/arch/arm/configs/keystone_defconfig b/arch/arm/configs/keystone_defconfig
index f710c192b33a..2536c231eea1 100644
--- a/arch/arm/configs/keystone_defconfig
+++ b/arch/arm/configs/keystone_defconfig
@@ -228,3 +228,10 @@ CONFIG_CRYPTO_DES=y
 CONFIG_CRYPTO_ANSI_CPRNG=y
 CONFIG_CRYPTO_USER_API_HASH=y
 CONFIG_CRYPTO_USER_API_SKCIPHER=y
+CONFIG_SPI_CADENCE_QUADSPI=y
+CONFIG_INPUT_MISC=y
+CONFIG_INPUT_EVDEV=m
+CONFIG_INPUT_GPIO_DECODER=m
+CONFIG_GPIO_PCA953X=m
+CONFIG_LEDS_TRIGGER_ACTIVITY=y
+CONFIG_LEDS_TRIGGER_CPU=y
diff --git a/arch/arm/configs/magician_defconfig b/arch/arm/configs/magician_defconfig
index ec5674c229a3..de5be2fc7306 100644
--- a/arch/arm/configs/magician_defconfig
+++ b/arch/arm/configs/magician_defconfig
@@ -54,7 +54,6 @@ CONFIG_BT_BNEP_PROTO_FILTER=y
 CONFIG_BT_HIDP=m
 CONFIG_BT_HCIBTUSB=m
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/arm/configs/mini2440_defconfig b/arch/arm/configs/mini2440_defconfig
index cf7dcb2c86e6..88ea02e7ba19 100644
--- a/arch/arm/configs/mini2440_defconfig
+++ b/arch/arm/configs/mini2440_defconfig
@@ -77,7 +77,6 @@ CONFIG_MAC80211=m
 CONFIG_MAC80211_MESH=y
 CONFIG_MAC80211_LEDS=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_CONNECTOR=m
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index 61509c4b769f..da7387689b88 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -1,6 +1,5 @@
 CONFIG_SYSVIPC=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_CGROUPS=y
@@ -116,6 +115,8 @@ CONFIG_ARCH_ZYNQ=y
 CONFIG_TRUSTED_FOUNDATIONS=y
 CONFIG_PCI=y
 CONFIG_PCI_HOST_GENERIC=y
+CONFIG_PCI_DRA7XX=y
+CONFIG_PCI_DRA7XX_EP=y
 CONFIG_PCI_KEYSTONE=y
 CONFIG_PCI_MSI=y
 CONFIG_PCI_MVEBU=y
@@ -123,6 +124,9 @@ CONFIG_PCI_TEGRA=y
 CONFIG_PCI_RCAR_GEN2=y
 CONFIG_PCIE_RCAR=y
 CONFIG_PCIEPORTBUS=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_PCI_ENDPOINT_CONFIGFS=y
+CONFIG_PCI_EPF_TEST=m
 CONFIG_SMP=y
 CONFIG_NR_CPUS=16
 CONFIG_HIGHPTE=y
@@ -222,6 +226,7 @@ CONFIG_ATMEL_SSC=m
 CONFIG_QCOM_COINCELL=m
 CONFIG_APDS9802ALS=y
 CONFIG_ISL29003=y
+CONFIG_PCI_ENDPOINT_TEST=m
 CONFIG_EEPROM_AT24=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_BLK_DEV_SR=y
@@ -557,6 +562,7 @@ CONFIG_REGULATOR_VEXPRESS=y
 CONFIG_REGULATOR_WM8994=m
 CONFIG_MEDIA_SUPPORT=m
 CONFIG_MEDIA_CAMERA_SUPPORT=y
+CONFIG_MEDIA_CEC_SUPPORT=y
 CONFIG_MEDIA_CONTROLLER=y
 CONFIG_VIDEO_V4L2_SUBDEV_API=y
 CONFIG_MEDIA_USB_SUPPORT=y
@@ -582,6 +588,8 @@ CONFIG_VIDEO_STI_DELTA=m
 CONFIG_VIDEO_RENESAS_JPU=m
 CONFIG_VIDEO_RENESAS_VSP1=m
 CONFIG_V4L_TEST_DRIVERS=y
+CONFIG_CEC_PLATFORM_DRIVERS=y
+CONFIG_VIDEO_SAMSUNG_S5P_CEC=m
 # CONFIG_MEDIA_SUBDRV_AUTOSELECT is not set
 CONFIG_VIDEO_ADV7180=m
 CONFIG_VIDEO_ML86V7667=m
@@ -613,6 +621,7 @@ CONFIG_DRM_TEGRA=y
 CONFIG_DRM_PANEL_SAMSUNG_LD9040=m
 CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=m
 CONFIG_DRM_PANEL_SIMPLE=y
+CONFIG_DRM_SII9234=m
 CONFIG_DRM_STI=m
 CONFIG_DRM_VC4=y
 CONFIG_FB_ARMCLCD=y
@@ -651,6 +660,7 @@ CONFIG_SND_SOC_SAMSUNG=m
 CONFIG_SND_SOC_SAMSUNG_SMDK_WM8994=m
 CONFIG_SND_SOC_SMDK_WM8994_PCM=m
 CONFIG_SND_SOC_SNOW=m
+CONFIG_SND_SOC_ODROID=m
 CONFIG_SND_SOC_SH4_FSI=m
 CONFIG_SND_SOC_RCAR=m
 CONFIG_SND_SIMPLE_SCU_CARD=m
@@ -879,6 +889,7 @@ CONFIG_PWM_ATMEL=m
 CONFIG_PWM_ATMEL_HLCDC_PWM=m
 CONFIG_PWM_ATMEL_TCB=m
 CONFIG_PWM_FSL_FTM=m
+CONFIG_PWM_RCAR=m
 CONFIG_PWM_RENESAS_TPU=y
 CONFIG_PWM_ROCKCHIP=m
 CONFIG_PWM_SAMSUNG=m
diff --git a/arch/arm/configs/mv78xx0_defconfig b/arch/arm/configs/mv78xx0_defconfig
index 752e2e74de5b..0448bd8075ac 100644
--- a/arch/arm/configs/mv78xx0_defconfig
+++ b/arch/arm/configs/mv78xx0_defconfig
@@ -37,7 +37,6 @@ CONFIG_IP_PNP_BOOTP=y
 # CONFIG_IPV6 is not set
 CONFIG_NET_PKTGEN=m
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/arm/configs/mvebu_v7_defconfig b/arch/arm/configs/mvebu_v7_defconfig
index 69553704f2dc..ddaeda4f2e82 100644
--- a/arch/arm/configs/mvebu_v7_defconfig
+++ b/arch/arm/configs/mvebu_v7_defconfig
@@ -1,6 +1,5 @@
 CONFIG_SYSVIPC=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
@@ -57,7 +56,7 @@ CONFIG_MTD_CFI_STAA=y
 CONFIG_MTD_PHYSMAP_OF=y
 CONFIG_MTD_M25P80=y
 CONFIG_MTD_NAND=y
-CONFIG_MTD_NAND_PXA3xx=y
+CONFIG_MTD_NAND_MARVELL=y
 CONFIG_MTD_SPI_NOR=y
 CONFIG_SRAM=y
 CONFIG_MTD_UBI=y
diff --git a/arch/arm/configs/mxs_defconfig b/arch/arm/configs/mxs_defconfig
index e5822ab01b7d..bbfb6759447b 100644
--- a/arch/arm/configs/mxs_defconfig
+++ b/arch/arm/configs/mxs_defconfig
@@ -46,7 +46,6 @@ CONFIG_CAN_FLEXCAN=m
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
index 7b97200c1d64..92674f247a12 100644
--- a/arch/arm/configs/omap2plus_defconfig
+++ b/arch/arm/configs/omap2plus_defconfig
@@ -48,6 +48,13 @@ CONFIG_SOC_AM43XX=y
 CONFIG_SOC_DRA7XX=y
 CONFIG_ARM_THUMBEE=y
 CONFIG_ARM_ERRATA_411920=y
+CONFIG_PCI=y
+CONFIG_PCI_MSI=y
+CONFIG_PCI_DRA7XX=y
+CONFIG_PCI_DRA7XX_EP=y
+CONFIG_PCI_ENDPOINT=y
+CONFIG_PCI_ENDPOINT_CONFIGFS=y
+CONFIG_PCI_EPF_TEST=m
 CONFIG_SMP=y
 CONFIG_NR_CPUS=2
 CONFIG_CMA=y
@@ -137,6 +144,7 @@ CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=16384
 CONFIG_SENSORS_TSL2550=m
 CONFIG_SRAM=y
+CONFIG_PCI_ENDPOINT_TEST=m
 CONFIG_EEPROM_AT24=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_SCSI_SCAN_ASYNC=y
@@ -311,7 +319,7 @@ CONFIG_MEDIA_CAMERA_SUPPORT=y
 CONFIG_RC_CORE=m
 CONFIG_MEDIA_CONTROLLER=y
 CONFIG_VIDEO_V4L2_SUBDEV_API=y
-CONFIG_LIRC=m
+CONFIG_LIRC=y
 CONFIG_RC_DEVICES=y
 CONFIG_IR_RX51=m
 CONFIG_V4L_PLATFORM_DRIVERS=y
@@ -468,6 +476,7 @@ CONFIG_PWM_TIECAP=m
 CONFIG_PWM_TIEHRPWM=m
 CONFIG_PWM_TWL=m
 CONFIG_PWM_TWL_LED=m
+CONFIG_PHY_CPCAP_USB=m
 CONFIG_PHY_DM816X_USB=m
 CONFIG_OMAP_USB2=m
 CONFIG_TI_PIPE3=y
@@ -516,3 +525,13 @@ CONFIG_LIBCRC32C=y
 CONFIG_FONTS=y
 CONFIG_FONT_8x8=y
 CONFIG_FONT_8x16=y
+CONFIG_KERNEL_MODE_NEON=y
+CONFIG_ARM_CRYPTO=y
+CONFIG_CRYPTO_SHA1_ARM=m
+CONFIG_CRYPTO_SHA1_ARM_NEON=m
+CONFIG_CRYPTO_SHA256_ARM=m
+CONFIG_CRYPTO_SHA512_ARM=m
+CONFIG_CRYPTO_AES_ARM=m
+CONFIG_CRYPTO_AES_ARM_BS=m
+CONFIG_CRYPTO_CHACHA20_NEON=m
+CONFIG_CRYPTO_GHASH_ARM_CE=m
diff --git a/arch/arm/configs/orion5x_defconfig b/arch/arm/configs/orion5x_defconfig
index b831baddae02..bf9046331f6e 100644
--- a/arch/arm/configs/orion5x_defconfig
+++ b/arch/arm/configs/orion5x_defconfig
@@ -60,7 +60,6 @@ CONFIG_IP_PNP_BOOTP=y
 CONFIG_NET_DSA=y
 CONFIG_NET_PKTGEN=m
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
index 830e817a028a..837d0c9c8b0e 100644
--- a/arch/arm/configs/pxa_defconfig
+++ b/arch/arm/configs/pxa_defconfig
@@ -1,7 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BSD_PROCESS_ACCT=y
diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig
index c784d04e2ab7..dd2a089f939f 100644
--- a/arch/arm/configs/qcom_defconfig
+++ b/arch/arm/configs/qcom_defconfig
@@ -136,7 +136,9 @@ CONFIG_REGULATOR_QCOM_RPM=y
 CONFIG_REGULATOR_QCOM_SMD_RPM=y
 CONFIG_REGULATOR_QCOM_SPMI=y
 CONFIG_MEDIA_SUPPORT=y
+CONFIG_DRM=y
 CONFIG_FB=y
+CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_BACKLIGHT_LCD_SUPPORT=y
 # CONFIG_LCD_CLASS_DEVICE is not set
 CONFIG_BACKLIGHT_CLASS_DEVICE=y
@@ -200,6 +202,7 @@ CONFIG_MSM_LCC_8960=y
 CONFIG_MDM_LCC_9615=y
 CONFIG_MSM_MMCC_8960=y
 CONFIG_MSM_MMCC_8974=y
+CONFIG_MSM_IOMMU=y
 CONFIG_HWSPINLOCK=y
 CONFIG_HWSPINLOCK_QCOM=y
 CONFIG_REMOTEPROC=y
diff --git a/arch/arm/configs/sama5_defconfig b/arch/arm/configs/sama5_defconfig
index 6529cb43e0fd..2080025556b5 100644
--- a/arch/arm/configs/sama5_defconfig
+++ b/arch/arm/configs/sama5_defconfig
@@ -2,7 +2,6 @@
 # CONFIG_SWAP is not set
 CONFIG_SYSVIPC=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_LOG_BUF_SHIFT=14
diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig
index 7b4fc0143148..578434cfd1a0 100644
--- a/arch/arm/configs/shmobile_defconfig
+++ b/arch/arm/configs/shmobile_defconfig
@@ -173,6 +173,7 @@ CONFIG_SND_SOC=y
 CONFIG_SND_SOC_SH4_FSI=y
 CONFIG_SND_SOC_RCAR=y
 CONFIG_SND_SOC_AK4642=y
+CONFIG_SND_SOC_SGTL5000=y
 CONFIG_SND_SOC_WM8978=y
 CONFIG_SND_SIMPLE_SCU_CARD=y
 CONFIG_USB=y
@@ -207,6 +208,7 @@ CONFIG_STAGING_BOARD=y
 CONFIG_IIO=y
 CONFIG_AK8975=y
 CONFIG_PWM=y
+CONFIG_PWM_RCAR=y
 CONFIG_PWM_RENESAS_TPU=y
 CONFIG_GENERIC_PHY=y
 CONFIG_PHY_RCAR_GEN2=y
diff --git a/arch/arm/configs/tegra_defconfig b/arch/arm/configs/tegra_defconfig
index 6678f2929356..c7b99ebf5fcf 100644
--- a/arch/arm/configs/tegra_defconfig
+++ b/arch/arm/configs/tegra_defconfig
@@ -1,5 +1,4 @@
 CONFIG_SYSVIPC=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_IKCONFIG=y
@@ -76,7 +75,6 @@ CONFIG_RFKILL_INPUT=y
 CONFIG_RFKILL_GPIO=y
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_DMA_CMA=y
 CONFIG_CMA_SIZE_MBYTES=64
 CONFIG_TEGRA_GMI=y
diff --git a/arch/arm/configs/vf610m4_defconfig b/arch/arm/configs/vf610m4_defconfig
index b7ecb83a95b6..a89f035c3b01 100644
--- a/arch/arm/configs/vf610m4_defconfig
+++ b/arch/arm/configs/vf610m4_defconfig
@@ -23,7 +23,6 @@ CONFIG_BINFMT_SHARED_FLAT=y
 # CONFIG_UEVENT_HELPER is not set
 # CONFIG_STANDALONE is not set
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_ALLOW_DEV_COREDUMP is not set
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=4
diff --git a/arch/arm/configs/vt8500_v6_v7_defconfig b/arch/arm/configs/vt8500_v6_v7_defconfig
index 1bfaa7bfc392..9b85326ba287 100644
--- a/arch/arm/configs/vt8500_v6_v7_defconfig
+++ b/arch/arm/configs/vt8500_v6_v7_defconfig
@@ -1,4 +1,3 @@
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BLK_DEV_INITRD=y
diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
index 18768f330449..07e31941dc67 100644
--- a/arch/arm/crypto/aes-neonbs-glue.c
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -181,9 +181,8 @@ static int cbc_init(struct crypto_tfm *tfm)
 	struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
 
 	ctx->enc_tfm = crypto_alloc_cipher("aes", 0, 0);
-	if (IS_ERR(ctx->enc_tfm))
-		return PTR_ERR(ctx->enc_tfm);
-	return 0;
+
+	return PTR_ERR_OR_ZERO(ctx->enc_tfm);
 }
 
 static void cbc_exit(struct crypto_tfm *tfm)
@@ -258,9 +257,8 @@ static int xts_init(struct crypto_tfm *tfm)
 	struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 
 	ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0);
-	if (IS_ERR(ctx->tweak_tfm))
-		return PTR_ERR(ctx->tweak_tfm);
-	return 0;
+
+	return PTR_ERR_OR_ZERO(ctx->tweak_tfm);
 }
 
 static void xts_exit(struct crypto_tfm *tfm)
diff --git a/arch/arm/crypto/crc32-ce-glue.c b/arch/arm/crypto/crc32-ce-glue.c
index 1b0e0e86ee9c..96e62ec105d0 100644
--- a/arch/arm/crypto/crc32-ce-glue.c
+++ b/arch/arm/crypto/crc32-ce-glue.c
@@ -188,6 +188,7 @@ static struct shash_alg crc32_pmull_algs[] = { {
 	.base.cra_name		= "crc32",
 	.base.cra_driver_name	= "crc32-arm-ce",
 	.base.cra_priority	= 200,
+	.base.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 	.base.cra_blocksize	= 1,
 	.base.cra_module	= THIS_MODULE,
 }, {
@@ -203,6 +204,7 @@ static struct shash_alg crc32_pmull_algs[] = { {
 	.base.cra_name		= "crc32c",
 	.base.cra_driver_name	= "crc32c-arm-ce",
 	.base.cra_priority	= 200,
+	.base.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 	.base.cra_blocksize	= 1,
 	.base.cra_module	= THIS_MODULE,
 } };
diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild
index 0f2c8a2a8131..873e3c189279 100644
--- a/arch/arm/include/asm/Kbuild
+++ b/arch/arm/include/asm/Kbuild
@@ -1,4 +1,3 @@
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += early_ioremap.h
 generic-y += emergency-restart.h
diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h
index ce5ee762ed66..4cab9bb823fb 100644
--- a/arch/arm/include/asm/bitops.h
+++ b/arch/arm/include/asm/bitops.h
@@ -338,6 +338,7 @@ static inline int find_next_bit_le(const void *p, int size, int offset)
 
 #endif
 
+#include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/le.h>
 
 /*
diff --git a/arch/arm/include/asm/dma-direct.h b/arch/arm/include/asm/dma-direct.h
new file mode 100644
index 000000000000..5b0a8a421894
--- /dev/null
+++ b/arch/arm/include/asm/dma-direct.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef ASM_ARM_DMA_DIRECT_H
+#define ASM_ARM_DMA_DIRECT_H 1
+
+static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
+{
+	unsigned int offset = paddr & ~PAGE_MASK;
+	return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset;
+}
+
+static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr)
+{
+	unsigned int offset = dev_addr & ~PAGE_MASK;
+	return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset;
+}
+
+static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
+{
+	u64 limit, mask;
+
+	if (!dev->dma_mask)
+		return 0;
+
+	mask = *dev->dma_mask;
+
+	limit = (mask + 1) & ~mask;
+	if (limit && size > limit)
+		return 0;
+
+	if ((addr | (addr + size - 1)) & ~mask)
+		return 0;
+
+	return 1;
+}
+
+#endif /* ASM_ARM_DMA_DIRECT_H */
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index daf837423a76..8436f6ade57d 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -18,7 +18,7 @@ extern const struct dma_map_ops arm_coherent_dma_ops;
 
 static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 {
-	return IS_ENABLED(CONFIG_MMU) ? &arm_dma_ops : &dma_noop_ops;
+	return IS_ENABLED(CONFIG_MMU) ? &arm_dma_ops : &dma_direct_ops;
 }
 
 #ifdef __arch_page_to_dma
@@ -109,39 +109,6 @@ static inline bool is_device_dma_coherent(struct device *dev)
 	return dev->archdata.dma_coherent;
 }
 
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	unsigned int offset = paddr & ~PAGE_MASK;
-	return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset;
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr)
-{
-	unsigned int offset = dev_addr & ~PAGE_MASK;
-	return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset;
-}
-
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	u64 limit, mask;
-
-	if (!dev->dma_mask)
-		return 0;
-
-	mask = *dev->dma_mask;
-
-	limit = (mask + 1) & ~mask;
-	if (limit && size > limit)
-		return 0;
-
-	if ((addr | (addr + size - 1)) & ~mask)
-		return 0;
-
-	return 1;
-}
-
-static inline void dma_mark_clean(void *addr, size_t size) { }
-
 /**
  * arm_dma_alloc - allocate consistent memory for DMA
  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
diff --git a/arch/arm/include/asm/exception.h b/arch/arm/include/asm/exception.h
index a7273ad9587a..58e039a851af 100644
--- a/arch/arm/include/asm/exception.h
+++ b/arch/arm/include/asm/exception.h
@@ -10,11 +10,10 @@
 
 #include <linux/interrupt.h>
 
-#define __exception	__attribute__((section(".exception.text")))
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 #define __exception_irq_entry	__irq_entry
 #else
-#define __exception_irq_entry	__exception
+#define __exception_irq_entry
 #endif
 
 #endif /* __ASM_ARM_EXCEPTION_H */
diff --git a/arch/arm/include/asm/glue-cache.h b/arch/arm/include/asm/glue-cache.h
index 01c3d92624e5..8d1f498e5dd8 100644
--- a/arch/arm/include/asm/glue-cache.h
+++ b/arch/arm/include/asm/glue-cache.h
@@ -117,6 +117,10 @@
 # endif
 #endif
 
+#if defined(CONFIG_CACHE_B15_RAC)
+# define MULTI_CACHE 1
+#endif
+
 #if defined(CONFIG_CPU_V7M)
 #  define MULTI_CACHE 1
 #endif
diff --git a/arch/arm/include/asm/hardware/cache-b15-rac.h b/arch/arm/include/asm/hardware/cache-b15-rac.h
new file mode 100644
index 000000000000..3d43ec06fd35
--- /dev/null
+++ b/arch/arm/include/asm/hardware/cache-b15-rac.h
@@ -0,0 +1,10 @@
+#ifndef __ASM_ARM_HARDWARE_CACHE_B15_RAC_H
+#define __ASM_ARM_HARDWARE_CACHE_B15_RAC_H
+
+#ifndef __ASSEMBLY__
+
+void b15_flush_kern_cache_all(void);
+
+#endif
+
+#endif
diff --git a/arch/arm/include/asm/hardware/sa1111.h b/arch/arm/include/asm/hardware/sa1111.h
index 0bbf163d1ed3..798e520e8a49 100644
--- a/arch/arm/include/asm/hardware/sa1111.h
+++ b/arch/arm/include/asm/hardware/sa1111.h
@@ -15,33 +15,6 @@
 
 #include <mach/bitfield.h>
 
-/*
- * The SA1111 is always located at virtual 0xf4000000, and is always
- * "native" endian.
- */
-
-#define SA1111_VBASE		0xf4000000
-
-/* Don't use these! */
-#define SA1111_p2v( x )         ((x) - SA1111_BASE + SA1111_VBASE)
-#define SA1111_v2p( x )         ((x) - SA1111_VBASE + SA1111_BASE)
-
-#ifndef __ASSEMBLY__
-#define _SA1111(x)	((x) + sa1111->resource.start)
-#endif
-
-#define sa1111_writel(val,addr)	__raw_writel(val, addr)
-#define sa1111_readl(addr)	__raw_readl(addr)
-
-/*
- * 26 bits of the SA-1110 address bus are available to the SA-1111.
- * Use these when feeding target addresses to the DMA engines.
- */
-
-#define SA1111_ADDR_WIDTH	(26)
-#define SA1111_ADDR_MASK	((1<<SA1111_ADDR_WIDTH)-1)
-#define SA1111_DMA_ADDR(x)	((x)&SA1111_ADDR_MASK)
-
 /*
  * Don't ask the (SAC) DMA engines to move less than this amount.
  */
@@ -417,7 +390,7 @@ struct sa1111_dev {
 	struct resource	res;
 	void __iomem	*mapbase;
 	unsigned int	skpcr_mask;
-	unsigned int	irq[6];
+	unsigned int	hwirq[6];
 	u64		dma_mask;
 };
 
@@ -431,9 +404,6 @@ struct sa1111_driver {
 	unsigned int		devid;
 	int (*probe)(struct sa1111_dev *);
 	int (*remove)(struct sa1111_dev *);
-	int (*suspend)(struct sa1111_dev *, pm_message_t);
-	int (*resume)(struct sa1111_dev *);
-	void (*shutdown)(struct sa1111_dev *);
 };
 
 #define SA1111_DRV(_d)	container_of((_d), struct sa1111_driver, drv)
diff --git a/arch/arm/include/asm/kvm_emulate.h b/arch/arm/include/asm/kvm_emulate.h
index 3d22eb87f919..9003bd19cb70 100644
--- a/arch/arm/include/asm/kvm_emulate.h
+++ b/arch/arm/include/asm/kvm_emulate.h
@@ -131,7 +131,7 @@ static inline bool mode_has_spsr(struct kvm_vcpu *vcpu)
 static inline bool vcpu_mode_priv(struct kvm_vcpu *vcpu)
 {
 	unsigned long cpsr_mode = vcpu->arch.ctxt.gp_regs.usr_regs.ARM_cpsr & MODE_MASK;
-	return cpsr_mode > USR_MODE;;
+	return cpsr_mode > USR_MODE;
 }
 
 static inline u32 kvm_vcpu_get_hsr(const struct kvm_vcpu *vcpu)
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index a9f7d3f47134..248b930563e5 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -48,6 +48,8 @@
 	KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_IRQ_PENDING	KVM_ARCH_REQ(1)
 
+DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
+
 u32 *kvm_vcpu_reg(struct kvm_vcpu *vcpu, u8 reg_num, u32 mode);
 int __attribute_const__ kvm_target_cpu(void);
 int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
@@ -238,6 +240,9 @@ int kvm_arm_coproc_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *);
 int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 		int exception_index);
 
+static inline void handle_exit_early(struct kvm_vcpu *vcpu, struct kvm_run *run,
+				     int exception_index) {}
+
 static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
 				       unsigned long hyp_stack_ptr,
 				       unsigned long vector_ptr)
@@ -301,4 +306,13 @@ int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu,
 /* All host FP/SIMD state is restored on guest exit, so nothing to save: */
 static inline void kvm_fpsimd_flush_cpu_state(void) {}
 
+static inline void kvm_arm_vhe_guest_enter(void) {}
+static inline void kvm_arm_vhe_guest_exit(void) {}
+
+static inline bool kvm_arm_harden_branch_predictor(void)
+{
+	/* No way to detect it yet, pretend it is not there. */
+	return false;
+}
+
 #endif /* __ARM_KVM_HOST_H__ */
diff --git a/arch/arm/include/asm/kvm_hyp.h b/arch/arm/include/asm/kvm_hyp.h
index ab20ffa8b9e7..1ab8329e9ff7 100644
--- a/arch/arm/include/asm/kvm_hyp.h
+++ b/arch/arm/include/asm/kvm_hyp.h
@@ -21,7 +21,6 @@
 #include <linux/compiler.h>
 #include <linux/kvm_host.h>
 #include <asm/cp15.h>
-#include <asm/kvm_mmu.h>
 #include <asm/vfp.h>
 
 #define __hyp_text __section(.hyp.text) notrace
@@ -69,6 +68,8 @@
 #define HIFAR		__ACCESS_CP15(c6, 4, c0, 2)
 #define HPFAR		__ACCESS_CP15(c6, 4, c0, 4)
 #define ICIALLUIS	__ACCESS_CP15(c7, 0, c1, 0)
+#define BPIALLIS	__ACCESS_CP15(c7, 0, c1, 6)
+#define ICIMVAU		__ACCESS_CP15(c7, 0, c5, 1)
 #define ATS1CPR		__ACCESS_CP15(c7, 0, c8, 0)
 #define TLBIALLIS	__ACCESS_CP15(c8, 0, c3, 0)
 #define TLBIALL		__ACCESS_CP15(c8, 0, c7, 0)
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index fa6f2174276b..de1b919404e4 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -37,6 +37,8 @@
 
 #include <linux/highmem.h>
 #include <asm/cacheflush.h>
+#include <asm/cputype.h>
+#include <asm/kvm_hyp.h>
 #include <asm/pgalloc.h>
 #include <asm/stage2_pgtable.h>
 
@@ -83,6 +85,18 @@ static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
 	return pmd;
 }
 
+static inline pte_t kvm_s2pte_mkexec(pte_t pte)
+{
+	pte_val(pte) &= ~L_PTE_XN;
+	return pte;
+}
+
+static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd)
+{
+	pmd_val(pmd) &= ~PMD_SECT_XN;
+	return pmd;
+}
+
 static inline void kvm_set_s2pte_readonly(pte_t *pte)
 {
 	pte_val(*pte) = (pte_val(*pte) & ~L_PTE_S2_RDWR) | L_PTE_S2_RDONLY;
@@ -93,6 +107,11 @@ static inline bool kvm_s2pte_readonly(pte_t *pte)
 	return (pte_val(*pte) & L_PTE_S2_RDWR) == L_PTE_S2_RDONLY;
 }
 
+static inline bool kvm_s2pte_exec(pte_t *pte)
+{
+	return !(pte_val(*pte) & L_PTE_XN);
+}
+
 static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
 {
 	pmd_val(*pmd) = (pmd_val(*pmd) & ~L_PMD_S2_RDWR) | L_PMD_S2_RDONLY;
@@ -103,6 +122,11 @@ static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
 	return (pmd_val(*pmd) & L_PMD_S2_RDWR) == L_PMD_S2_RDONLY;
 }
 
+static inline bool kvm_s2pmd_exec(pmd_t *pmd)
+{
+	return !(pmd_val(*pmd) & PMD_SECT_XN);
+}
+
 static inline bool kvm_page_empty(void *ptr)
 {
 	struct page *ptr_page = virt_to_page(ptr);
@@ -126,10 +150,36 @@ static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
 	return (vcpu_cp15(vcpu, c1_SCTLR) & 0b101) == 0b101;
 }
 
-static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
-					       kvm_pfn_t pfn,
-					       unsigned long size)
+static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size)
+{
+	/*
+	 * Clean the dcache to the Point of Coherency.
+	 *
+	 * We need to do this through a kernel mapping (using the
+	 * user-space mapping has proved to be the wrong
+	 * solution). For that, we need to kmap one page at a time,
+	 * and iterate over the range.
+	 */
+
+	VM_BUG_ON(size & ~PAGE_MASK);
+
+	while (size) {
+		void *va = kmap_atomic_pfn(pfn);
+
+		kvm_flush_dcache_to_poc(va, PAGE_SIZE);
+
+		size -= PAGE_SIZE;
+		pfn++;
+
+		kunmap_atomic(va);
+	}
+}
+
+static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn,
+						  unsigned long size)
 {
+	u32 iclsz;
+
 	/*
 	 * If we are going to insert an instruction page and the icache is
 	 * either VIPT or PIPT, there is a potential problem where the host
@@ -141,23 +191,40 @@ static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
 	 *
 	 * VIVT caches are tagged using both the ASID and the VMID and doesn't
 	 * need any kind of flushing (DDI 0406C.b - Page B3-1392).
-	 *
-	 * We need to do this through a kernel mapping (using the
-	 * user-space mapping has proved to be the wrong
-	 * solution). For that, we need to kmap one page at a time,
-	 * and iterate over the range.
 	 */
 
 	VM_BUG_ON(size & ~PAGE_MASK);
 
+	if (icache_is_vivt_asid_tagged())
+		return;
+
+	if (!icache_is_pipt()) {
+		/* any kind of VIPT cache */
+		__flush_icache_all();
+		return;
+	}
+
+	/*
+	 * CTR IminLine contains Log2 of the number of words in the
+	 * cache line, so we can get the number of words as
+	 * 2 << (IminLine - 1).  To get the number of bytes, we
+	 * multiply by 4 (the number of bytes in a 32-bit word), and
+	 * get 4 << (IminLine).
+	 */
+	iclsz = 4 << (read_cpuid(CPUID_CACHETYPE) & 0xf);
+
 	while (size) {
 		void *va = kmap_atomic_pfn(pfn);
+		void *end = va + PAGE_SIZE;
+		void *addr = va;
 
-		kvm_flush_dcache_to_poc(va, PAGE_SIZE);
+		do {
+			write_sysreg(addr, ICIMVAU);
+			addr += iclsz;
+		} while (addr < end);
 
-		if (icache_is_pipt())
-			__cpuc_coherent_user_range((unsigned long)va,
-						   (unsigned long)va + PAGE_SIZE);
+		dsb(ishst);
+		isb();
 
 		size -= PAGE_SIZE;
 		pfn++;
@@ -165,9 +232,11 @@ static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
 		kunmap_atomic(va);
 	}
 
-	if (!icache_is_pipt() && !icache_is_vivt_asid_tagged()) {
-		/* any kind of VIPT cache */
-		__flush_icache_all();
+	/* Check if we need to invalidate the BTB */
+	if ((read_cpuid_ext(CPUID_EXT_MMFR1) >> 28) != 4) {
+		write_sysreg(0, BPIALLIS);
+		dsb(ishst);
+		isb();
 	}
 }
 
@@ -211,6 +280,11 @@ static inline bool __kvm_cpu_uses_extended_idmap(void)
 	return false;
 }
 
+static inline unsigned long __kvm_idmap_ptrs_per_pgd(void)
+{
+	return PTRS_PER_PGD;
+}
+
 static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
 				       pgd_t *hyp_pgd,
 				       pgd_t *merged_hyp_pgd,
@@ -221,6 +295,18 @@ static inline unsigned int kvm_get_vmid_bits(void)
 	return 8;
 }
 
+static inline void *kvm_get_hyp_vector(void)
+{
+	return kvm_ksym_ref(__kvm_hyp_vector);
+}
+
+static inline int kvm_map_vectors(void)
+{
+	return 0;
+}
+
+#define kvm_phys_to_vttbr(addr)		(addr)
+
 #endif	/* !__ASSEMBLY__ */
 
 #endif /* __ARM_KVM_MMU_H__ */
diff --git a/arch/arm/include/asm/kvm_psci.h b/arch/arm/include/asm/kvm_psci.h
deleted file mode 100644
index 6bda945d31fa..000000000000
--- a/arch/arm/include/asm/kvm_psci.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (C) 2012 - ARM Ltd
- * Author: Marc Zyngier <marc.zyngier@arm.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __ARM_KVM_PSCI_H__
-#define __ARM_KVM_PSCI_H__
-
-#define KVM_ARM_PSCI_0_1	1
-#define KVM_ARM_PSCI_0_2	2
-
-int kvm_psci_version(struct kvm_vcpu *vcpu);
-int kvm_psci_call(struct kvm_vcpu *vcpu);
-
-#endif /* __ARM_KVM_PSCI_H__ */
diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h
index 1f54e4e98c1e..496667703693 100644
--- a/arch/arm/include/asm/memory.h
+++ b/arch/arm/include/asm/memory.h
@@ -88,6 +88,7 @@
 #else /* CONFIG_MMU */
 
 #ifndef __ASSEMBLY__
+extern unsigned long setup_vectors_base(void);
 extern unsigned long vectors_base;
 #define VECTORS_BASE		vectors_base
 #endif
diff --git a/arch/arm/include/asm/pci.h b/arch/arm/include/asm/pci.h
index 960d9dc4f380..1f0de808d111 100644
--- a/arch/arm/include/asm/pci.h
+++ b/arch/arm/include/asm/pci.h
@@ -10,10 +10,7 @@ extern unsigned long pcibios_min_io;
 extern unsigned long pcibios_min_mem;
 #define PCIBIOS_MIN_MEM pcibios_min_mem
 
-static inline int pcibios_assign_all_busses(void)
-{
-	return pci_has_flag(PCI_REASSIGN_ALL_RSRC);
-}
+#define pcibios_assign_all_busses()	pci_has_flag(PCI_REASSIGN_ALL_BUS)
 
 #ifdef CONFIG_PCI_DOMAINS
 static inline int pci_proc_domain(struct pci_bus *bus)
diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
index 1a7a17b2a1ba..2a4836087358 100644
--- a/arch/arm/include/asm/pgtable-3level.h
+++ b/arch/arm/include/asm/pgtable-3level.h
@@ -249,6 +249,9 @@ PMD_BIT_FUNC(mkyoung,   |= PMD_SECT_AF);
 #define pfn_pmd(pfn,prot)	(__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
 #define mk_pmd(page,prot)	pfn_pmd(page_to_pfn(page),prot)
 
+/* No hardware dirty/accessed bits -- generic_pmdp_establish() fits */
+#define pmdp_establish generic_pmdp_establish
+
 /* represent a notpresent pmd by faulting entry, this is used by pmdp_invalidate */
 static inline pmd_t pmd_mknotpresent(pmd_t pmd)
 {
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index 150ece66ddf3..a757401129f9 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -102,8 +102,8 @@ extern pgprot_t		pgprot_s2_device;
 #define PAGE_HYP_EXEC		_MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_RDONLY)
 #define PAGE_HYP_RO		_MOD_PROT(pgprot_kernel, L_PTE_HYP | L_PTE_RDONLY | L_PTE_XN)
 #define PAGE_HYP_DEVICE		_MOD_PROT(pgprot_hyp_device, L_PTE_HYP)
-#define PAGE_S2			_MOD_PROT(pgprot_s2, L_PTE_S2_RDONLY)
-#define PAGE_S2_DEVICE		_MOD_PROT(pgprot_s2_device, L_PTE_S2_RDONLY)
+#define PAGE_S2			_MOD_PROT(pgprot_s2, L_PTE_S2_RDONLY | L_PTE_XN)
+#define PAGE_S2_DEVICE		_MOD_PROT(pgprot_s2_device, L_PTE_S2_RDONLY | L_PTE_XN)
 
 #define __PAGE_NONE		__pgprot(_L_PTE_DEFAULT | L_PTE_RDONLY | L_PTE_XN | L_PTE_NONE)
 #define __PAGE_SHARED		__pgprot(_L_PTE_DEFAULT | L_PTE_USER | L_PTE_XN)
diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h
index 338cbe0a18ef..1bf65b47808a 100644
--- a/arch/arm/include/asm/processor.h
+++ b/arch/arm/include/asm/processor.h
@@ -45,6 +45,16 @@ struct thread_struct {
 	struct debug_info	debug;
 };
 
+/*
+ * Everything usercopied to/from thread_struct is statically-sized, so
+ * no hardened usercopy whitelist is needed.
+ */
+static inline void arch_thread_struct_whitelist(unsigned long *offset,
+						unsigned long *size)
+{
+	*offset = *size = 0;
+}
+
 #define INIT_THREAD  {	}
 
 #define start_thread(regs,pc,sp)					\
diff --git a/arch/arm/include/asm/ptdump.h b/arch/arm/include/asm/ptdump.h
new file mode 100644
index 000000000000..3ebf9718288d
--- /dev/null
+++ b/arch/arm/include/asm/ptdump.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2014 ARM Ltd. */
+#ifndef __ASM_PTDUMP_H
+#define __ASM_PTDUMP_H
+
+#ifdef CONFIG_ARM_PTDUMP_CORE
+
+#include <linux/mm_types.h>
+#include <linux/seq_file.h>
+
+struct addr_marker {
+	unsigned long start_address;
+	char *name;
+};
+
+struct ptdump_info {
+	struct mm_struct		*mm;
+	const struct addr_marker	*markers;
+	unsigned long			base_addr;
+};
+
+void ptdump_walk_pgd(struct seq_file *s, struct ptdump_info *info);
+#ifdef CONFIG_ARM_PTDUMP_DEBUGFS
+int ptdump_debugfs_register(struct ptdump_info *info, const char *name);
+#else
+static inline int ptdump_debugfs_register(struct ptdump_info *info,
+					const char *name)
+{
+	return 0;
+}
+#endif /* CONFIG_ARM_PTDUMP_DEBUGFS */
+
+void ptdump_check_wx(void);
+
+#endif /* CONFIG_ARM_PTDUMP_CORE */
+
+#ifdef CONFIG_DEBUG_WX
+#define debug_checkwx() ptdump_check_wx()
+#else
+#define debug_checkwx() do { } while (0)
+#endif
+
+#endif /* __ASM_PTDUMP_H */
diff --git a/arch/arm/include/asm/sections.h b/arch/arm/include/asm/sections.h
index 63dfe1f10335..4ceb4f757d4d 100644
--- a/arch/arm/include/asm/sections.h
+++ b/arch/arm/include/asm/sections.h
@@ -6,4 +6,25 @@
 
 extern char _exiprom[];
 
+extern char __idmap_text_start[];
+extern char __idmap_text_end[];
+extern char __entry_text_start[];
+extern char __entry_text_end[];
+extern char __hyp_idmap_text_start[];
+extern char __hyp_idmap_text_end[];
+
+static inline bool in_entry_text(unsigned long addr)
+{
+	return memory_contains(__entry_text_start, __entry_text_end,
+			       (void *)addr, 1);
+}
+
+static inline bool in_idmap_text(unsigned long addr)
+{
+	void *a = (void *)addr;
+	return memory_contains(__idmap_text_start, __idmap_text_end, a, 1) ||
+	       memory_contains(__hyp_idmap_text_start, __hyp_idmap_text_end,
+			       a, 1);
+}
+
 #endif	/* _ASM_ARM_SECTIONS_H */
diff --git a/arch/arm/include/asm/string.h b/arch/arm/include/asm/string.h
index f54a3136aac6..111a1d8a41dd 100644
--- a/arch/arm/include/asm/string.h
+++ b/arch/arm/include/asm/string.h
@@ -39,18 +39,4 @@ static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n)
 	return __memset64(p, v, n * 8, v >> 32);
 }
 
-extern void __memzero(void *ptr, __kernel_size_t n);
-
-#define memset(p,v,n)							\
-	({								\
-	 	void *__p = (p); size_t __n = n;			\
-		if ((__n) != 0) {					\
-			if (__builtin_constant_p((v)) && (v) == 0)	\
-				__memzero((__p),(__n));			\
-			else						\
-				memset((__p),(v),(__n));		\
-		}							\
-		(__p);							\
-	})
-
 #endif
diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h
index 776757d1604a..e71cc35de163 100644
--- a/arch/arm/include/asm/thread_info.h
+++ b/arch/arm/include/asm/thread_info.h
@@ -75,9 +75,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,					\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /*
  * how to get the current stack pointer in C
  */
diff --git a/arch/arm/include/asm/traps.h b/arch/arm/include/asm/traps.h
index f9a6c5fc3fd1..a00288d75ee6 100644
--- a/arch/arm/include/asm/traps.h
+++ b/arch/arm/include/asm/traps.h
@@ -28,18 +28,6 @@ static inline int __in_irqentry_text(unsigned long ptr)
 	       ptr < (unsigned long)&__irqentry_text_end;
 }
 
-static inline int in_exception_text(unsigned long ptr)
-{
-	extern char __exception_text_start[];
-	extern char __exception_text_end[];
-	int in;
-
-	in = ptr >= (unsigned long)&__exception_text_start &&
-	     ptr < (unsigned long)&__exception_text_end;
-
-	return in ? : __in_irqentry_text(ptr);
-}
-
 extern void __init early_trap_init(void *);
 extern void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame);
 extern void ptrace_break(struct task_struct *tsk, struct pt_regs *regs);
diff --git a/arch/arm/include/asm/unified.h b/arch/arm/include/asm/unified.h
index a91ae499614c..2c3b952be63e 100644
--- a/arch/arm/include/asm/unified.h
+++ b/arch/arm/include/asm/unified.h
@@ -20,8 +20,10 @@
 #ifndef __ASM_UNIFIED_H
 #define __ASM_UNIFIED_H
 
-#if defined(__ASSEMBLY__) && defined(CONFIG_ARM_ASM_UNIFIED)
+#if defined(__ASSEMBLY__)
 	.syntax unified
+#else
+__asm__(".syntax unified");
 #endif
 
 #ifdef CONFIG_CPU_V7M
@@ -64,77 +66,4 @@
 
 #endif	/* CONFIG_THUMB2_KERNEL */
 
-#ifndef CONFIG_ARM_ASM_UNIFIED
-
-/*
- * If the unified assembly syntax isn't used (in ARM mode), these
- * macros expand to an empty string
- */
-#ifdef __ASSEMBLY__
-	.macro	it, cond
-	.endm
-	.macro	itt, cond
-	.endm
-	.macro	ite, cond
-	.endm
-	.macro	ittt, cond
-	.endm
-	.macro	itte, cond
-	.endm
-	.macro	itet, cond
-	.endm
-	.macro	itee, cond
-	.endm
-	.macro	itttt, cond
-	.endm
-	.macro	ittte, cond
-	.endm
-	.macro	ittet, cond
-	.endm
-	.macro	ittee, cond
-	.endm
-	.macro	itett, cond
-	.endm
-	.macro	itete, cond
-	.endm
-	.macro	iteet, cond
-	.endm
-	.macro	iteee, cond
-	.endm
-#else	/* !__ASSEMBLY__ */
-__asm__(
-"	.macro	it, cond\n"
-"	.endm\n"
-"	.macro	itt, cond\n"
-"	.endm\n"
-"	.macro	ite, cond\n"
-"	.endm\n"
-"	.macro	ittt, cond\n"
-"	.endm\n"
-"	.macro	itte, cond\n"
-"	.endm\n"
-"	.macro	itet, cond\n"
-"	.endm\n"
-"	.macro	itee, cond\n"
-"	.endm\n"
-"	.macro	itttt, cond\n"
-"	.endm\n"
-"	.macro	ittte, cond\n"
-"	.endm\n"
-"	.macro	ittet, cond\n"
-"	.endm\n"
-"	.macro	ittee, cond\n"
-"	.endm\n"
-"	.macro	itett, cond\n"
-"	.endm\n"
-"	.macro	itete, cond\n"
-"	.endm\n"
-"	.macro	iteet, cond\n"
-"	.endm\n"
-"	.macro	iteee, cond\n"
-"	.endm\n");
-#endif	/* __ASSEMBLY__ */
-
-#endif	/* CONFIG_ARM_ASM_UNIFIED */
-
 #endif	/* !__ASM_UNIFIED_H */
diff --git a/arch/arm/include/asm/vdso.h b/arch/arm/include/asm/vdso.h
index 9c99e817535e..5b85889f82ee 100644
--- a/arch/arm/include/asm/vdso.h
+++ b/arch/arm/include/asm/vdso.h
@@ -12,8 +12,6 @@ struct mm_struct;
 
 void arm_install_vdso(struct mm_struct *mm, unsigned long addr);
 
-extern char vdso_start, vdso_end;
-
 extern unsigned int vdso_total_pages;
 
 #else /* CONFIG_VDSO */
diff --git a/arch/arm/include/uapi/asm/siginfo.h b/arch/arm/include/uapi/asm/siginfo.h
new file mode 100644
index 000000000000..d0513880be21
--- /dev/null
+++ b/arch/arm/include/uapi/asm/siginfo.h
@@ -0,0 +1,13 @@
+#ifndef __ASM_SIGINFO_H
+#define __ASM_SIGINFO_H
+
+#include <asm-generic/siginfo.h>
+
+/*
+ * SIGFPE si_codes
+ */
+#ifdef __KERNEL__
+#define FPE_FIXME	0	/* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
+#endif
diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c
index 5266fd9ad6b4..783fbb4de5f9 100644
--- a/arch/arm/kernel/armksyms.c
+++ b/arch/arm/kernel/armksyms.c
@@ -92,7 +92,6 @@ EXPORT_SYMBOL(__memset64);
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memmove);
 EXPORT_SYMBOL(memchr);
-EXPORT_SYMBOL(__memzero);
 
 EXPORT_SYMBOL(mmioset);
 EXPORT_SYMBOL(mmiocpy);
diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
index 0cd0aefb3a8f..ed46ca69813d 100644
--- a/arch/arm/kernel/bios32.c
+++ b/arch/arm/kernel/bios32.c
@@ -527,7 +527,7 @@ void pci_common_init_dev(struct device *parent, struct hw_pci *hw)
 	struct pci_sys_data *sys;
 	LIST_HEAD(head);
 
-	pci_add_flags(PCI_REASSIGN_ALL_RSRC);
+	pci_add_flags(PCI_REASSIGN_ALL_BUS);
 	if (hw->preinit)
 		hw->preinit();
 	pcibios_init_hw(parent, hw, &head);
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index fbc707626b3e..1752033b0070 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -82,11 +82,7 @@
 #endif
 	.endm
 
-#ifdef CONFIG_KPROBES
-	.section	.kprobes.text,"ax",%progbits
-#else
-	.text
-#endif
+	.section	.entry.text,"ax",%progbits
 
 /*
  * Invalid mode handlers
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index e655dcd0a933..3c4f88701f22 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -37,6 +37,7 @@ saved_pc	.req	lr
 #define TRACE(x...)
 #endif
 
+	.section .entry.text,"ax",%progbits
 	.align	5
 #if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING))
 /*
diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S
index 21dde771a7dd..6e0375e7db05 100644
--- a/arch/arm/kernel/head-common.S
+++ b/arch/arm/kernel/head-common.S
@@ -105,8 +105,9 @@ __mmap_switched:
    ARM(	ldmia	r4!, {r0, r1, sp} )
  THUMB(	ldmia	r4!, {r0, r1, r3} )
  THUMB(	mov	sp, r3 )
-	sub	r1, r1, r0
-	bl	__memzero			@ clear .bss
+	sub	r2, r1, r0
+	mov	r1, #0
+	bl	memset				@ clear .bss
 
 	ldmia	r4, {r0, r1, r2, r3}
 	str	r9, [r0]			@ Save processor ID
diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index af2a7f1e3103..629e25152c0d 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -44,17 +44,17 @@ static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
 static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
 
 /* Number of BRP/WRP registers on this CPU. */
-static int core_num_brps;
-static int core_num_wrps;
+static int core_num_brps __ro_after_init;
+static int core_num_wrps __ro_after_init;
 
 /* Debug architecture version. */
-static u8 debug_arch;
+static u8 debug_arch __ro_after_init;
 
 /* Does debug architecture support OS Save and Restore? */
-static bool has_ossr;
+static bool has_ossr __ro_after_init;
 
 /* Maximum supported watchpoint length. */
-static u8 max_watchpoint_len;
+static u8 max_watchpoint_len __ro_after_init;
 
 #define READ_WB_REG_CASE(OP2, M, VAL)			\
 	case ((OP2 << 4) + M):				\
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index d96714e1858c..1523cb18b109 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -21,7 +21,6 @@
 #include <linux/unistd.h>
 #include <linux/user.h>
 #include <linux/interrupt.h>
-#include <linux/kallsyms.h>
 #include <linux/init.h>
 #include <linux/elfcore.h>
 #include <linux/pm.h>
@@ -121,8 +120,8 @@ void __show_regs(struct pt_regs *regs)
 
 	show_regs_print_info(KERN_DEFAULT);
 
-	print_symbol("PC is at %s\n", instruction_pointer(regs));
-	print_symbol("LR is at %s\n", regs->ARM_lr);
+	printk("PC is at %pS\n", (void *)instruction_pointer(regs));
+	printk("LR is at %pS\n", (void *)regs->ARM_lr);
 	printk("pc : [<%08lx>]    lr : [<%08lx>]    psr: %08lx\n",
 	       regs->ARM_pc, regs->ARM_lr, regs->ARM_cpsr);
 	printk("sp : %08lx  ip : %08lx  fp : %08lx\n",
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index 58e3771e4c5b..7724b0f661b3 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -390,7 +390,6 @@ static void ptrace_hbptriggered(struct perf_event *bp,
 	struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
 	long num;
 	int i;
-	siginfo_t info;
 
 	for (i = 0; i < ARM_MAX_HBP_SLOTS; ++i)
 		if (current->thread.debug.hbp[i] == bp)
@@ -398,12 +397,7 @@ static void ptrace_hbptriggered(struct perf_event *bp,
 
 	num = (i == ARM_MAX_HBP_SLOTS) ? 0 : ptrace_hbp_idx_to_num(i);
 
-	info.si_signo	= SIGTRAP;
-	info.si_errno	= (int)num;
-	info.si_code	= TRAP_HWBKPT;
-	info.si_addr	= (void __user *)(bkpt->trigger);
-
-	force_sig_info(SIGTRAP, &info, current);
+	force_sig_ptrace_errno_trap((int)num, (void __user *)(bkpt->trigger));
 }
 
 /*
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index b4fbf00ee4ad..2da087926ebe 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -379,6 +379,9 @@ asmlinkage void secondary_start_kernel(void)
 
 	cpu_init();
 
+#ifndef CONFIG_MMU
+	setup_vectors_base();
+#endif
 	pr_debug("CPU%u: Booted secondary processor\n", cpu);
 
 	preempt_disable();
diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c
index 65228bf4c6df..a56e7c856ab5 100644
--- a/arch/arm/kernel/stacktrace.c
+++ b/arch/arm/kernel/stacktrace.c
@@ -3,6 +3,7 @@
 #include <linux/sched/debug.h>
 #include <linux/stacktrace.h>
 
+#include <asm/sections.h>
 #include <asm/stacktrace.h>
 #include <asm/traps.h>
 
@@ -63,7 +64,6 @@ EXPORT_SYMBOL(walk_stackframe);
 #ifdef CONFIG_STACKTRACE
 struct stack_trace_data {
 	struct stack_trace *trace;
-	unsigned long last_pc;
 	unsigned int no_sched_functions;
 	unsigned int skip;
 };
@@ -87,16 +87,7 @@ static int save_trace(struct stackframe *frame, void *d)
 	if (trace->nr_entries >= trace->max_entries)
 		return 1;
 
-	/*
-	 * in_exception_text() is designed to test if the PC is one of
-	 * the functions which has an exception stack above it, but
-	 * unfortunately what is in frame->pc is the return LR value,
-	 * not the saved PC value.  So, we need to track the previous
-	 * frame PC value when doing this.
-	 */
-	addr = data->last_pc;
-	data->last_pc = frame->pc;
-	if (!in_exception_text(addr))
+	if (!in_entry_text(frame->pc))
 		return 0;
 
 	regs = (struct pt_regs *)frame->sp;
@@ -114,7 +105,6 @@ static noinline void __save_stack_trace(struct task_struct *tsk,
 	struct stackframe frame;
 
 	data.trace = trace;
-	data.last_pc = ULONG_MAX;
 	data.skip = trace->skip;
 	data.no_sched_functions = nosched;
 
diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c
index 629f8e9981f1..cf2701cb0de8 100644
--- a/arch/arm/kernel/time.c
+++ b/arch/arm/kernel/time.c
@@ -83,7 +83,7 @@ static void dummy_clock_access(struct timespec64 *ts)
 }
 
 static clock_access_fn __read_persistent_clock = dummy_clock_access;
-static clock_access_fn __read_boot_clock = dummy_clock_access;;
+static clock_access_fn __read_boot_clock = dummy_clock_access;
 
 void read_persistent_clock64(struct timespec64 *ts)
 {
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index 3e26c6f7a191..5e3633c24e63 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -72,7 +72,7 @@ void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long
 	printk("Function entered at [<%08lx>] from [<%08lx>]\n", where, from);
 #endif
 
-	if (in_exception_text(where))
+	if (in_entry_text(from))
 		dump_mem("", "Exception stack", frame + 4, frame + 4 + sizeof(struct pt_regs));
 }
 
@@ -433,7 +433,7 @@ static int call_undef_hook(struct pt_regs *regs, unsigned int instr)
 	return fn ? fn(regs, instr) : 1;
 }
 
-asmlinkage void __exception do_undefinstr(struct pt_regs *regs)
+asmlinkage void do_undefinstr(struct pt_regs *regs)
 {
 	unsigned int instr;
 	siginfo_t info;
diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c
index a4d6dc0f2427..f4dd7f9663c1 100644
--- a/arch/arm/kernel/vdso.c
+++ b/arch/arm/kernel/vdso.c
@@ -39,6 +39,8 @@
 
 static struct page **vdso_text_pagelist;
 
+extern char vdso_start[], vdso_end[];
+
 /* Total number of pages needed for the data and text portions of the VDSO. */
 unsigned int vdso_total_pages __ro_after_init;
 
@@ -197,13 +199,13 @@ static int __init vdso_init(void)
 	unsigned int text_pages;
 	int i;
 
-	if (memcmp(&vdso_start, "\177ELF", 4)) {
+	if (memcmp(vdso_start, "\177ELF", 4)) {
 		pr_err("VDSO is not a valid ELF object!\n");
 		return -ENOEXEC;
 	}
 
-	text_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT;
-	pr_debug("vdso: %i text pages at base %p\n", text_pages, &vdso_start);
+	text_pages = (vdso_end - vdso_start) >> PAGE_SHIFT;
+	pr_debug("vdso: %i text pages at base %p\n", text_pages, vdso_start);
 
 	/* Allocate the VDSO text pagelist */
 	vdso_text_pagelist = kcalloc(text_pages, sizeof(struct page *),
@@ -218,7 +220,7 @@ static int __init vdso_init(void)
 	for (i = 0; i < text_pages; i++) {
 		struct page *page;
 
-		page = virt_to_page(&vdso_start + i * PAGE_SIZE);
+		page = virt_to_page(vdso_start + i * PAGE_SIZE);
 		vdso_text_pagelist[i] = page;
 	}
 
@@ -229,7 +231,7 @@ static int __init vdso_init(void)
 
 	cntvct_ok = cntvct_functional();
 
-	patch_vdso(&vdso_start);
+	patch_vdso(vdso_start);
 
 	return 0;
 }
diff --git a/arch/arm/kernel/vmlinux-xip.lds.S b/arch/arm/kernel/vmlinux-xip.lds.S
index ec4b3f94ad80..12b87591eb7c 100644
--- a/arch/arm/kernel/vmlinux-xip.lds.S
+++ b/arch/arm/kernel/vmlinux-xip.lds.S
@@ -96,9 +96,9 @@ SECTIONS
 	.text : {			/* Real text segment		*/
 		_stext = .;		/* Text and read-only data	*/
 			IDMAP_TEXT
-			__exception_text_start = .;
-			*(.exception.text)
-			__exception_text_end = .;
+			__entry_text_start = .;
+			*(.entry.text)
+			__entry_text_end = .;
 			IRQENTRY_TEXT
 			TEXT_TEXT
 			SCHED_TEXT
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index ee53f6518872..84a1ae3ce46e 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -105,9 +105,9 @@ SECTIONS
 	.text : {			/* Real text segment		*/
 		_stext = .;		/* Text and read-only data	*/
 			IDMAP_TEXT
-			__exception_text_start = .;
-			*(.exception.text)
-			__exception_text_end = .;
+			__entry_text_start = .;
+			*(.entry.text)
+			__entry_text_end = .;
 			IRQENTRY_TEXT
 			SOFTIRQENTRY_TEXT
 			TEXT_TEXT
diff --git a/arch/arm/kvm/handle_exit.c b/arch/arm/kvm/handle_exit.c
index cf8bf6bf87c4..910bd8dabb3c 100644
--- a/arch/arm/kvm/handle_exit.c
+++ b/arch/arm/kvm/handle_exit.c
@@ -21,7 +21,7 @@
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_mmu.h>
-#include <asm/kvm_psci.h>
+#include <kvm/arm_psci.h>
 #include <trace/events/kvm.h>
 
 #include "trace.h"
@@ -36,9 +36,9 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		      kvm_vcpu_hvc_get_imm(vcpu));
 	vcpu->stat.hvc_exit_stat++;
 
-	ret = kvm_psci_call(vcpu);
+	ret = kvm_hvc_call_handler(vcpu);
 	if (ret < 0) {
-		kvm_inject_undefined(vcpu);
+		vcpu_set_reg(vcpu, 0, ~0UL);
 		return 1;
 	}
 
@@ -47,7 +47,16 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
-	kvm_inject_undefined(vcpu);
+	/*
+	 * "If an SMC instruction executed at Non-secure EL1 is
+	 * trapped to EL2 because HCR_EL2.TSC is 1, the exception is a
+	 * Trap exception, not a Secure Monitor Call exception [...]"
+	 *
+	 * We need to advance the PC after the trap, as it would
+	 * otherwise return to the same address...
+	 */
+	vcpu_set_reg(vcpu, 0, ~0UL);
+	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 	return 1;
 }
 
diff --git a/arch/arm/kvm/hyp/Makefile b/arch/arm/kvm/hyp/Makefile
index 5638ce0c9524..63d6b404d88e 100644
--- a/arch/arm/kvm/hyp/Makefile
+++ b/arch/arm/kvm/hyp/Makefile
@@ -7,6 +7,8 @@ ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING
 
 KVM=../../../../virt/kvm
 
+CFLAGS_ARMV7VE		   :=$(call cc-option, -march=armv7ve)
+
 obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/vgic-v2-sr.o
 obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/vgic-v3-sr.o
 obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/timer-sr.o
@@ -15,7 +17,10 @@ obj-$(CONFIG_KVM_ARM_HOST) += tlb.o
 obj-$(CONFIG_KVM_ARM_HOST) += cp15-sr.o
 obj-$(CONFIG_KVM_ARM_HOST) += vfp.o
 obj-$(CONFIG_KVM_ARM_HOST) += banked-sr.o
+CFLAGS_banked-sr.o	   += $(CFLAGS_ARMV7VE)
+
 obj-$(CONFIG_KVM_ARM_HOST) += entry.o
 obj-$(CONFIG_KVM_ARM_HOST) += hyp-entry.o
 obj-$(CONFIG_KVM_ARM_HOST) += switch.o
+CFLAGS_switch.o		   += $(CFLAGS_ARMV7VE)
 obj-$(CONFIG_KVM_ARM_HOST) += s2-setup.o
diff --git a/arch/arm/kvm/hyp/banked-sr.c b/arch/arm/kvm/hyp/banked-sr.c
index 111bda8cdebd..be4b8b0a40ad 100644
--- a/arch/arm/kvm/hyp/banked-sr.c
+++ b/arch/arm/kvm/hyp/banked-sr.c
@@ -20,6 +20,10 @@
 
 #include <asm/kvm_hyp.h>
 
+/*
+ * gcc before 4.9 doesn't understand -march=armv7ve, so we have to
+ * trick the assembler.
+ */
 __asm__(".arch_extension     virt");
 
 void __hyp_text __banked_save_state(struct kvm_cpu_context *ctxt)
diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index 330c9ce34ba5..ae45ae96aac2 100644
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -18,6 +18,7 @@
 
 #include <asm/kvm_asm.h>
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
 
 __asm__(".arch_extension     virt");
 
diff --git a/arch/arm/kvm/hyp/tlb.c b/arch/arm/kvm/hyp/tlb.c
index 6d810af2d9fd..c0edd450e104 100644
--- a/arch/arm/kvm/hyp/tlb.c
+++ b/arch/arm/kvm/hyp/tlb.c
@@ -19,6 +19,7 @@
  */
 
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
 
 /**
  * Flush per-VMID TLBs
diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
index 4cb0b9624d8f..ad25fd1872c7 100644
--- a/arch/arm/lib/Makefile
+++ b/arch/arm/lib/Makefile
@@ -8,7 +8,7 @@
 lib-y		:= backtrace.o changebit.o csumipv6.o csumpartial.o   \
 		   csumpartialcopy.o csumpartialcopyuser.o clearbit.o \
 		   delay.o delay-loop.o findbit.o memchr.o memcpy.o   \
-		   memmove.o memset.o memzero.o setbit.o              \
+		   memmove.o memset.o setbit.o                        \
 		   strchr.o strrchr.o                                 \
 		   testchangebit.o testclearbit.o testsetbit.o        \
 		   ashldi3.o ashrdi3.o lshrdi3.o muldi3.o             \
diff --git a/arch/arm/lib/memzero.S b/arch/arm/lib/memzero.S
deleted file mode 100644
index 0eded952e089..000000000000
--- a/arch/arm/lib/memzero.S
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- *  linux/arch/arm/lib/memzero.S
- *
- *  Copyright (C) 1995-2000 Russell King
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-#include <asm/unwind.h>
-
-	.text
-	.align	5
-	.word	0
-/*
- * Align the pointer in r0.  r3 contains the number of bytes that we are
- * mis-aligned by, and r1 is the number of bytes.  If r1 < 4, then we
- * don't bother; we use byte stores instead.
- */
-UNWIND(	.fnstart			)
-1:	subs	r1, r1, #4		@ 1 do we have enough
-	blt	5f			@ 1 bytes to align with?
-	cmp	r3, #2			@ 1
-	strltb	r2, [r0], #1		@ 1
-	strleb	r2, [r0], #1		@ 1
-	strb	r2, [r0], #1		@ 1
-	add	r1, r1, r3		@ 1 (r1 = r1 - (4 - r3))
-/*
- * The pointer is now aligned and the length is adjusted.  Try doing the
- * memzero again.
- */
-
-ENTRY(__memzero)
-	mov	r2, #0			@ 1
-	ands	r3, r0, #3		@ 1 unaligned?
-	bne	1b			@ 1
-/*
- * r3 = 0, and we know that the pointer in r0 is aligned to a word boundary.
- */
-	cmp	r1, #16			@ 1 we can skip this chunk if we
-	blt	4f			@ 1 have < 16 bytes
-
-#if ! CALGN(1)+0
-
-/*
- * We need an extra register for this loop - save the return address and
- * use the LR
- */
-	str	lr, [sp, #-4]!		@ 1
-UNWIND(	.fnend				)
-UNWIND(	.fnstart			)
-UNWIND(	.save 	{lr}			)
-	mov	ip, r2			@ 1
-	mov	lr, r2			@ 1
-
-3:	subs	r1, r1, #64		@ 1 write 32 bytes out per loop
-	stmgeia	r0!, {r2, r3, ip, lr}	@ 4
-	stmgeia	r0!, {r2, r3, ip, lr}	@ 4
-	stmgeia	r0!, {r2, r3, ip, lr}	@ 4
-	stmgeia	r0!, {r2, r3, ip, lr}	@ 4
-	bgt	3b			@ 1
-	ldmeqfd	sp!, {pc}		@ 1/2 quick exit
-/*
- * No need to correct the count; we're only testing bits from now on
- */
-	tst	r1, #32			@ 1
-	stmneia	r0!, {r2, r3, ip, lr}	@ 4
-	stmneia	r0!, {r2, r3, ip, lr}	@ 4
-	tst	r1, #16			@ 1 16 bytes or more?
-	stmneia	r0!, {r2, r3, ip, lr}	@ 4
-	ldr	lr, [sp], #4		@ 1
-UNWIND(	.fnend				)
-
-#else
-
-/*
- * This version aligns the destination pointer in order to write
- * whole cache lines at once.
- */
-
-	stmfd	sp!, {r4-r7, lr}
-UNWIND(	.fnend		       )
-UNWIND(	.fnstart	       )
-UNWIND(	.save 	{r4-r7, lr}    )
-	mov	r4, r2
-	mov	r5, r2
-	mov	r6, r2
-	mov	r7, r2
-	mov	ip, r2
-	mov	lr, r2
-
-	cmp	r1, #96
-	andgts	ip, r0, #31
-	ble	3f
-
-	rsb	ip, ip, #32
-	sub	r1, r1, ip
-	movs	ip, ip, lsl #(32 - 4)
-	stmcsia	r0!, {r4, r5, r6, r7}
-	stmmiia	r0!, {r4, r5}
-	movs	ip, ip, lsl #2
-	strcs	r2, [r0], #4
-
-3:	subs	r1, r1, #64
-	stmgeia	r0!, {r2-r7, ip, lr}
-	stmgeia	r0!, {r2-r7, ip, lr}
-	bgt	3b
-	ldmeqfd	sp!, {r4-r7, pc}
-
-	tst	r1, #32
-	stmneia	r0!, {r2-r7, ip, lr}
-	tst	r1, #16
-	stmneia	r0!, {r4-r7}
-	ldmfd	sp!, {r4-r7, lr}
-UNWIND(	.fnend		       )
-
-#endif
-
-UNWIND(	.fnstart			)
-4:	tst	r1, #8			@ 1 8 bytes or more?
-	stmneia	r0!, {r2, r3}		@ 2
-	tst	r1, #4			@ 1 4 bytes or more?
-	strne	r2, [r0], #4		@ 1
-/*
- * When we get here, we've got less than 4 bytes to zero.  We
- * may have an unaligned pointer as well.
- */
-5:	tst	r1, #2			@ 1 2 bytes or more?
-	strneb	r2, [r0], #1		@ 1
-	strneb	r2, [r0], #1		@ 1
-	tst	r1, #1			@ 1 a byte left over
-	strneb	r2, [r0], #1		@ 1
-	ret	lr			@ 1
-UNWIND(	.fnend				)
-ENDPROC(__memzero)
diff --git a/arch/arm/mach-bcm/brcmstb.c b/arch/arm/mach-bcm/brcmstb.c
index 07e3a86c6466..5f127d5f1045 100644
--- a/arch/arm/mach-bcm/brcmstb.c
+++ b/arch/arm/mach-bcm/brcmstb.c
@@ -14,7 +14,6 @@
 #include <linux/init.h>
 #include <linux/irqchip.h>
 #include <linux/of_platform.h>
-#include <linux/soc/brcmstb/brcmstb.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
@@ -38,7 +37,6 @@ u32 brcmstb_uart_config[3] = {
 static void __init brcmstb_init_irq(void)
 {
 	irqchip_init();
-	brcmstb_biuctrl_init();
 }
 
 static const char *const brcmstb_match[] __initconst = {
diff --git a/arch/arm/mach-clps711x/board-dt.c b/arch/arm/mach-clps711x/board-dt.c
index ee1f83b1a332..4c89a8e9a2e3 100644
--- a/arch/arm/mach-clps711x/board-dt.c
+++ b/arch/arm/mach-clps711x/board-dt.c
@@ -69,7 +69,7 @@ static void clps711x_restart(enum reboot_mode mode, const char *cmd)
 	soft_restart(0);
 }
 
-static const char *clps711x_compat[] __initconst = {
+static const char *const clps711x_compat[] __initconst = {
 	"cirrus,ep7209",
 	NULL
 };
diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c
index cbde0030c092..d898a94f6eae 100644
--- a/arch/arm/mach-davinci/board-da850-evm.c
+++ b/arch/arm/mach-davinci/board-da850-evm.c
@@ -798,11 +798,11 @@ static int da850_lcd_hw_init(void)
 {
 	int status;
 
-	status = gpio_request(DA850_LCD_BL_PIN, "lcd bl\n");
+	status = gpio_request(DA850_LCD_BL_PIN, "lcd bl");
 	if (status < 0)
 		return status;
 
-	status = gpio_request(DA850_LCD_PWR_PIN, "lcd pwr\n");
+	status = gpio_request(DA850_LCD_PWR_PIN, "lcd pwr");
 	if (status < 0) {
 		gpio_free(DA850_LCD_BL_PIN);
 		return status;
diff --git a/arch/arm/mach-davinci/board-dm355-evm.c b/arch/arm/mach-davinci/board-dm355-evm.c
index 62e7bc3018f0..d6b11907380c 100644
--- a/arch/arm/mach-davinci/board-dm355-evm.c
+++ b/arch/arm/mach-davinci/board-dm355-evm.c
@@ -17,6 +17,7 @@
 #include <linux/mtd/rawnand.h>
 #include <linux/i2c.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/clk.h>
 #include <linux/videodev2.h>
 #include <media/i2c/tvp514x.h>
@@ -108,11 +109,20 @@ static struct platform_device davinci_nand_device = {
 	},
 };
 
+static struct gpiod_lookup_table i2c_recovery_gpiod_table = {
+	.dev_id = "i2c_davinci",
+	.table = {
+		GPIO_LOOKUP("davinci_gpio", 15, "sda",
+			    GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN),
+		GPIO_LOOKUP("davinci_gpio", 14, "scl",
+			    GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN),
+	},
+};
+
 static struct davinci_i2c_platform_data i2c_pdata = {
 	.bus_freq	= 400	/* kHz */,
 	.bus_delay	= 0	/* usec */,
-	.sda_pin        = 15,
-	.scl_pin        = 14,
+	.gpio_recovery	= true,
 };
 
 static int dm355evm_mmc_gpios = -EINVAL;
@@ -141,6 +151,7 @@ static struct i2c_board_info dm355evm_i2c_info[] = {
 
 static void __init evm_init_i2c(void)
 {
+	gpiod_add_lookup_table(&i2c_recovery_gpiod_table);
 	davinci_init_i2c(&i2c_pdata);
 
 	gpio_request(5, "dm355evm_msp");
@@ -357,7 +368,7 @@ static struct spi_eeprom at25640a = {
 	.flags		= EE_ADDR2,
 };
 
-static struct spi_board_info dm355_evm_spi_info[] __initconst = {
+static const struct spi_board_info dm355_evm_spi_info[] __initconst = {
 	{
 		.modalias	= "at25",
 		.platform_data	= &at25640a,
diff --git a/arch/arm/mach-davinci/board-dm355-leopard.c b/arch/arm/mach-davinci/board-dm355-leopard.c
index be997243447b..fad9a5611a5d 100644
--- a/arch/arm/mach-davinci/board-dm355-leopard.c
+++ b/arch/arm/mach-davinci/board-dm355-leopard.c
@@ -217,7 +217,7 @@ static struct spi_eeprom at25640a = {
 	.flags		= EE_ADDR2,
 };
 
-static struct spi_board_info dm355_leopard_spi_info[] __initconst = {
+static const struct spi_board_info dm355_leopard_spi_info[] __initconst = {
 	{
 		.modalias	= "at25",
 		.platform_data	= &at25640a,
diff --git a/arch/arm/mach-davinci/board-dm365-evm.c b/arch/arm/mach-davinci/board-dm365-evm.c
index e75741fb2c1d..e3780986d2a3 100644
--- a/arch/arm/mach-davinci/board-dm365-evm.c
+++ b/arch/arm/mach-davinci/board-dm365-evm.c
@@ -726,7 +726,7 @@ static struct spi_eeprom at25640 = {
 	.flags		= EE_ADDR2,
 };
 
-static struct spi_board_info dm365_evm_spi_info[] __initconst = {
+static const struct spi_board_info dm365_evm_spi_info[] __initconst = {
 	{
 		.modalias	= "at25",
 		.platform_data	= &at25640,
diff --git a/arch/arm/mach-davinci/board-dm644x-evm.c b/arch/arm/mach-davinci/board-dm644x-evm.c
index b07c9b18d427..85e6fb33b1ee 100644
--- a/arch/arm/mach-davinci/board-dm644x-evm.c
+++ b/arch/arm/mach-davinci/board-dm644x-evm.c
@@ -13,6 +13,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/platform_device.h>
 #include <linux/gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/i2c.h>
 #include <linux/platform_data/pcf857x.h>
 #include <linux/platform_data/at24.h>
@@ -595,18 +596,28 @@ static struct i2c_board_info __initdata i2c_info[] =  {
 	},
 };
 
+static struct gpiod_lookup_table i2c_recovery_gpiod_table = {
+	.dev_id = "i2c_davinci",
+	.table = {
+		GPIO_LOOKUP("davinci_gpio", 44, "sda",
+			    GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN),
+		GPIO_LOOKUP("davinci_gpio", 43, "scl",
+			    GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN),
+	},
+};
+
 /* The msp430 uses a slow bitbanged I2C implementation (ergo 20 KHz),
  * which requires 100 usec of idle bus after i2c writes sent to it.
  */
 static struct davinci_i2c_platform_data i2c_pdata = {
 	.bus_freq	= 20 /* kHz */,
 	.bus_delay	= 100 /* usec */,
-	.sda_pin        = 44,
-	.scl_pin        = 43,
+	.gpio_recovery	= true,
 };
 
 static void __init evm_init_i2c(void)
 {
+	gpiod_add_lookup_table(&i2c_recovery_gpiod_table);
 	davinci_init_i2c(&i2c_pdata);
 	i2c_add_driver(&dm6446evm_msp_driver);
 	i2c_register_board_info(1, i2c_info, ARRAY_SIZE(i2c_info));
diff --git a/arch/arm/mach-davinci/board-neuros-osd2.c b/arch/arm/mach-davinci/board-neuros-osd2.c
index 0c02aaad0539..4da210a1a110 100644
--- a/arch/arm/mach-davinci/board-neuros-osd2.c
+++ b/arch/arm/mach-davinci/board-neuros-osd2.c
@@ -128,7 +128,7 @@ static struct platform_device davinci_fb_device = {
 	.num_resources = 0,
 };
 
-static struct gpio_led ntosd2_leds[] = {
+static const struct gpio_led ntosd2_leds[] = {
 	{ .name = "led1_green", .gpio = GPIO(10), },
 	{ .name = "led1_red",   .gpio = GPIO(11), },
 	{ .name = "led2_green", .gpio = GPIO(12), },
diff --git a/arch/arm/mach-davinci/board-omapl138-hawk.c b/arch/arm/mach-davinci/board-omapl138-hawk.c
index a3e78074be70..62eb7d668890 100644
--- a/arch/arm/mach-davinci/board-omapl138-hawk.c
+++ b/arch/arm/mach-davinci/board-omapl138-hawk.c
@@ -127,8 +127,8 @@ static struct gpiod_lookup_table mmc_gpios_table = {
 	.dev_id = "da830-mmc.0",
 	.table = {
 		/* CD: gpio3_12: gpio60: chip 1 contains gpio range 32-63*/
-		GPIO_LOOKUP("davinci_gpio.1", 28, "cd", GPIO_ACTIVE_LOW),
-		GPIO_LOOKUP("davinci_gpio.1", 29, "wp", GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP("davinci_gpio.0", 28, "cd", GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP("davinci_gpio.0", 29, "wp", GPIO_ACTIVE_LOW),
 	},
 };
 
diff --git a/arch/arm/mach-davinci/common.c b/arch/arm/mach-davinci/common.c
index 9f9fbfa6da0d..bcb6a7ba84e9 100644
--- a/arch/arm/mach-davinci/common.c
+++ b/arch/arm/mach-davinci/common.c
@@ -77,7 +77,7 @@ static int __init davinci_init_id(struct davinci_soc_info *soc_info)
 	return -EINVAL;
 }
 
-void __init davinci_common_init(struct davinci_soc_info *soc_info)
+void __init davinci_common_init(const struct davinci_soc_info *soc_info)
 {
 	int ret;
 
diff --git a/arch/arm/mach-davinci/da830.c b/arch/arm/mach-davinci/da830.c
index bd88470f3e5c..57ab18cf2a89 100644
--- a/arch/arm/mach-davinci/da830.c
+++ b/arch/arm/mach-davinci/da830.c
@@ -417,7 +417,7 @@ static struct clk_lookup da830_clks[] = {
 	CLK("davinci-mcasp.0",	NULL,		&mcasp0_clk),
 	CLK("davinci-mcasp.1",	NULL,		&mcasp1_clk),
 	CLK("davinci-mcasp.2",	NULL,		&mcasp2_clk),
-	CLK("musb-da8xx",	"usb20",	&usb20_clk),
+	CLK("musb-da8xx",	NULL,		&usb20_clk),
 	CLK("cppi41-dmaengine",	NULL,		&cppi41_clk),
 	CLK(NULL,		"aemif",	&aemif_clk),
 	CLK(NULL,		"aintc",	&aintc_clk),
@@ -426,7 +426,7 @@ static struct clk_lookup da830_clks[] = {
 	CLK("davinci_mdio.0",   "fck",          &emac_clk),
 	CLK(NULL,		"gpio",		&gpio_clk),
 	CLK("i2c_davinci.2",	NULL,		&i2c1_clk),
-	CLK("ohci-da8xx",	"usb11",	&usb11_clk),
+	CLK("ohci-da8xx",	NULL,	&usb11_clk),
 	CLK(NULL,		"emif3",	&emif3_clk),
 	CLK(NULL,		"arm",		&arm_clk),
 	CLK(NULL,		"rmii",		&rmii_clk),
@@ -1194,7 +1194,7 @@ static struct davinci_timer_info da830_timer_info = {
 	.clocksource_id	= T0_BOT,
 };
 
-static struct davinci_soc_info davinci_soc_info_da830 = {
+static const struct davinci_soc_info davinci_soc_info_da830 = {
 	.io_desc		= da830_io_desc,
 	.io_desc_num		= ARRAY_SIZE(da830_io_desc),
 	.jtag_id_reg		= DA8XX_SYSCFG0_BASE + DA8XX_JTAG_ID_REG,
diff --git a/arch/arm/mach-davinci/da850.c b/arch/arm/mach-davinci/da850.c
index 07d6f0eb8c82..aa37cbdf7d4d 100644
--- a/arch/arm/mach-davinci/da850.c
+++ b/arch/arm/mach-davinci/da850.c
@@ -563,8 +563,8 @@ static struct clk_lookup da850_clks[] = {
 	CLK("da830-mmc.1",	NULL,		&mmcsd1_clk),
 	CLK("ti-aemif",		NULL,		&aemif_clk),
 	CLK("davinci-nand.0",	"aemif",	&aemif_nand_clk),
-	CLK("ohci-da8xx",	"usb11",	&usb11_clk),
-	CLK("musb-da8xx",	"usb20",	&usb20_clk),
+	CLK("ohci-da8xx",	NULL,		&usb11_clk),
+	CLK("musb-da8xx",	NULL,		&usb20_clk),
 	CLK("cppi41-dmaengine",	NULL,		&cppi41_clk),
 	CLK("spi_davinci.0",	NULL,		&spi0_clk),
 	CLK("spi_davinci.1",	NULL,		&spi1_clk),
@@ -1347,7 +1347,7 @@ int __init da850_register_gpio(void)
 	return da8xx_register_gpio(&da850_gpio_platform_data);
 }
 
-static struct davinci_soc_info davinci_soc_info_da850 = {
+static const struct davinci_soc_info davinci_soc_info_da850 = {
 	.io_desc		= da850_io_desc,
 	.io_desc_num		= ARRAY_SIZE(da850_io_desc),
 	.jtag_id_reg		= DA8XX_SYSCFG0_BASE + DA8XX_JTAG_ID_REG,
diff --git a/arch/arm/mach-davinci/devices-da8xx.c b/arch/arm/mach-davinci/devices-da8xx.c
index 22440c05d66a..e1c40e73d30a 100644
--- a/arch/arm/mach-davinci/devices-da8xx.c
+++ b/arch/arm/mach-davinci/devices-da8xx.c
@@ -252,7 +252,7 @@ int __init da830_register_edma(struct edma_rsv_info *rsv)
 	da8xx_edma0_pdata.slavecnt = ARRAY_SIZE(da830_edma_map);
 
 	edma_pdev = platform_device_register_full(&da8xx_edma0_device);
-	return IS_ERR(edma_pdev) ? PTR_ERR(edma_pdev) : 0;
+	return PTR_ERR_OR_ZERO(edma_pdev);
 }
 
 static const struct dma_slave_map da850_edma0_map[] = {
@@ -297,7 +297,7 @@ int __init da850_register_edma(struct edma_rsv_info *rsv[2])
 	da850_edma1_pdata.slavecnt = ARRAY_SIZE(da850_edma1_map);
 
 	edma_pdev = platform_device_register_full(&da850_edma1_device);
-	return IS_ERR(edma_pdev) ? PTR_ERR(edma_pdev) : 0;
+	return PTR_ERR_OR_ZERO(edma_pdev);
 }
 
 static struct resource da8xx_i2c_resources0[] = {
diff --git a/arch/arm/mach-davinci/dm355.c b/arch/arm/mach-davinci/dm355.c
index bd50367f654e..938747f20c22 100644
--- a/arch/arm/mach-davinci/dm355.c
+++ b/arch/arm/mach-davinci/dm355.c
@@ -1006,7 +1006,7 @@ struct platform_device dm355_serial_device[] = {
 	}
 };
 
-static struct davinci_soc_info davinci_soc_info_dm355 = {
+static const struct davinci_soc_info davinci_soc_info_dm355 = {
 	.io_desc		= dm355_io_desc,
 	.io_desc_num		= ARRAY_SIZE(dm355_io_desc),
 	.jtag_id_reg		= 0x01c40028,
diff --git a/arch/arm/mach-davinci/dm365.c b/arch/arm/mach-davinci/dm365.c
index 5ace9380626a..5d9f96df08e9 100644
--- a/arch/arm/mach-davinci/dm365.c
+++ b/arch/arm/mach-davinci/dm365.c
@@ -1110,7 +1110,7 @@ struct platform_device dm365_serial_device[] = {
 	}
 };
 
-static struct davinci_soc_info davinci_soc_info_dm365 = {
+static const struct davinci_soc_info davinci_soc_info_dm365 = {
 	.io_desc		= dm365_io_desc,
 	.io_desc_num		= ARRAY_SIZE(dm365_io_desc),
 	.jtag_id_reg		= 0x01c40028,
diff --git a/arch/arm/mach-davinci/dm644x.c b/arch/arm/mach-davinci/dm644x.c
index b437c3730f65..6b41e1ca511e 100644
--- a/arch/arm/mach-davinci/dm644x.c
+++ b/arch/arm/mach-davinci/dm644x.c
@@ -899,7 +899,7 @@ struct platform_device dm644x_serial_device[] = {
 	}
 };
 
-static struct davinci_soc_info davinci_soc_info_dm644x = {
+static const struct davinci_soc_info davinci_soc_info_dm644x = {
 	.io_desc		= dm644x_io_desc,
 	.io_desc_num		= ARRAY_SIZE(dm644x_io_desc),
 	.jtag_id_reg		= 0x01c40028,
diff --git a/arch/arm/mach-davinci/dm646x.c b/arch/arm/mach-davinci/dm646x.c
index da21353cac45..6fc06a6ad4f8 100644
--- a/arch/arm/mach-davinci/dm646x.c
+++ b/arch/arm/mach-davinci/dm646x.c
@@ -882,7 +882,7 @@ struct platform_device dm646x_serial_device[] = {
 	}
 };
 
-static struct davinci_soc_info davinci_soc_info_dm646x = {
+static const struct davinci_soc_info davinci_soc_info_dm646x = {
 	.io_desc		= dm646x_io_desc,
 	.io_desc_num		= ARRAY_SIZE(dm646x_io_desc),
 	.jtag_id_reg		= 0x01c40028,
@@ -949,7 +949,7 @@ int __init dm646x_init_edma(struct edma_rsv_info *rsv)
 	dm646x_edma_pdata.rsv = rsv;
 
 	edma_pdev = platform_device_register_full(&dm646x_edma_device);
-	return IS_ERR(edma_pdev) ? PTR_ERR(edma_pdev) : 0;
+	return PTR_ERR_OR_ZERO(edma_pdev);
 }
 
 void __init dm646x_init(void)
diff --git a/arch/arm/mach-davinci/include/mach/common.h b/arch/arm/mach-davinci/include/mach/common.h
index 037aa66bcac1..433a008ff796 100644
--- a/arch/arm/mach-davinci/include/mach/common.h
+++ b/arch/arm/mach-davinci/include/mach/common.h
@@ -79,7 +79,7 @@ struct davinci_soc_info {
 
 extern struct davinci_soc_info davinci_soc_info;
 
-extern void davinci_common_init(struct davinci_soc_info *soc_info);
+extern void davinci_common_init(const struct davinci_soc_info *soc_info);
 extern void davinci_init_ide(void);
 void davinci_restart(enum reboot_mode mode, const char *cmd);
 void davinci_init_late(void);
diff --git a/arch/arm/mach-ep93xx/Kconfig b/arch/arm/mach-ep93xx/Kconfig
index 61a75ca3684e..c095236d7ff8 100644
--- a/arch/arm/mach-ep93xx/Kconfig
+++ b/arch/arm/mach-ep93xx/Kconfig
@@ -21,6 +21,13 @@ config MACH_ADSSPHERE
 	  Say 'Y' here if you want your kernel to support the ADS
 	  Sphere board.
 
+config MACH_BK3
+	bool "Support Liebherr BK3.1"
+	select MACH_TS72XX
+	help
+	  Say 'Y' here if you want your kernel to support the
+	  Liebherr controller BK3.1.
+
 config MACH_EDB93XX
 	bool
 
diff --git a/arch/arm/mach-ep93xx/ts72xx.c b/arch/arm/mach-ep93xx/ts72xx.c
index f386ebae0163..c089a2a4fe30 100644
--- a/arch/arm/mach-ep93xx/ts72xx.c
+++ b/arch/arm/mach-ep93xx/ts72xx.c
@@ -19,10 +19,15 @@
 #include <linux/mtd/rawnand.h>
 #include <linux/mtd/partitions.h>
 #include <linux/spi/spi.h>
+#include <linux/spi/flash.h>
+#include <linux/spi/mmc_spi.h>
+#include <linux/mmc/host.h>
 #include <linux/platform_data/spi-ep93xx.h>
 
 #include <mach/gpio-ep93xx.h>
 #include <mach/hardware.h>
+#include <mach/irqs.h>
+#include <mach/gpio-ep93xx.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/map.h>
@@ -31,6 +36,9 @@
 #include "soc.h"
 #include "ts72xx.h"
 
+/*************************************************************************
+ * IO map
+ *************************************************************************/
 static struct map_desc ts72xx_io_desc[] __initdata = {
 	{
 		.virtual	= (unsigned long)TS72XX_MODEL_VIRT_BASE,
@@ -47,6 +55,11 @@ static struct map_desc ts72xx_io_desc[] __initdata = {
 		.pfn		= __phys_to_pfn(TS72XX_OPTIONS2_PHYS_BASE),
 		.length		= TS72XX_OPTIONS2_SIZE,
 		.type		= MT_DEVICE,
+	}, {
+		.virtual	= (unsigned long)TS72XX_CPLDVER_VIRT_BASE,
+		.pfn		= __phys_to_pfn(TS72XX_CPLDVER_PHYS_BASE),
+		.length		= TS72XX_CPLDVER_SIZE,
+		.type		= MT_DEVICE,
 	}
 };
 
@@ -123,8 +136,6 @@ static struct platform_nand_data ts72xx_nand_data = {
 		.nr_chips	= 1,
 		.chip_offset	= 0,
 		.chip_delay	= 15,
-		.partitions	= ts72xx_nand_parts,
-		.nr_partitions	= ARRAY_SIZE(ts72xx_nand_parts),
 	},
 	.ctrl = {
 		.cmd_ctrl	= ts72xx_nand_hwcontrol,
@@ -148,8 +159,8 @@ static struct platform_device ts72xx_nand_flash = {
 	.num_resources		= ARRAY_SIZE(ts72xx_nand_resource),
 };
 
-
-static void __init ts72xx_register_flash(void)
+void __init ts72xx_register_flash(struct mtd_partition *parts, int n,
+				  resource_size_t start)
 {
 	/*
 	 * TS7200 has NOR flash all other TS72xx board have NAND flash.
@@ -157,16 +168,12 @@ static void __init ts72xx_register_flash(void)
 	if (board_is_ts7200()) {
 		ep93xx_register_flash(2, EP93XX_CS6_PHYS_BASE, SZ_16M);
 	} else {
-		resource_size_t start;
-
-		if (is_ts9420_installed())
-			start = EP93XX_CS7_PHYS_BASE;
-		else
-			start = EP93XX_CS6_PHYS_BASE;
-
 		ts72xx_nand_resource[0].start = start;
 		ts72xx_nand_resource[0].end = start + SZ_16M - 1;
 
+		ts72xx_nand_data.chip.partitions = parts;
+		ts72xx_nand_data.chip.nr_partitions = n;
+
 		platform_device_register(&ts72xx_nand_flash);
 	}
 }
@@ -207,10 +214,79 @@ static struct platform_device ts72xx_wdt_device = {
 	.num_resources	= ARRAY_SIZE(ts72xx_wdt_resources),
 };
 
+/*************************************************************************
+ * ETH
+ *************************************************************************/
 static struct ep93xx_eth_data __initdata ts72xx_eth_data = {
 	.phy_id		= 1,
 };
 
+/*************************************************************************
+ * SPI SD/MMC host
+ *************************************************************************/
+#define BK3_EN_SDCARD_PHYS_BASE         0x12400000
+#define BK3_EN_SDCARD_PWR 0x0
+#define BK3_DIS_SDCARD_PWR 0x0C
+static void bk3_mmc_spi_setpower(struct device *dev, unsigned int vdd)
+{
+	void __iomem *pwr_sd = ioremap(BK3_EN_SDCARD_PHYS_BASE, SZ_4K);
+
+	if (!pwr_sd) {
+		pr_err("Failed to enable SD card power!");
+		return;
+	}
+
+	pr_debug("%s: SD card pwr %s VDD:0x%x\n", __func__,
+		 !!vdd ? "ON" : "OFF", vdd);
+
+	if (!!vdd)
+		__raw_writeb(BK3_EN_SDCARD_PWR, pwr_sd);
+	else
+		__raw_writeb(BK3_DIS_SDCARD_PWR, pwr_sd);
+
+	iounmap(pwr_sd);
+}
+
+static struct mmc_spi_platform_data bk3_spi_mmc_data = {
+	.detect_delay	= 500,
+	.powerup_msecs	= 100,
+	.ocr_mask	= MMC_VDD_32_33 | MMC_VDD_33_34,
+	.caps		= MMC_CAP_NONREMOVABLE,
+	.setpower       = bk3_mmc_spi_setpower,
+};
+
+/*************************************************************************
+ * SPI Bus - SD card access
+ *************************************************************************/
+static struct spi_board_info bk3_spi_board_info[] __initdata = {
+	{
+		.modalias		= "mmc_spi",
+		.platform_data		= &bk3_spi_mmc_data,
+		.max_speed_hz		= 7.4E6,
+		.bus_num		= 0,
+		.chip_select		= 0,
+		.mode			= SPI_MODE_0,
+	},
+};
+
+/*
+ * This is a stub -> the FGPIO[3] pin is not connected on the schematic
+ * The all work is performed automatically by !SPI_FRAME (SFRM1) and
+ * goes through CPLD
+ */
+static int bk3_spi_chipselects[] __initdata = {
+	EP93XX_GPIO_LINE_F(3),
+};
+
+static struct ep93xx_spi_info bk3_spi_master __initdata = {
+	.chipselect	= bk3_spi_chipselects,
+	.num_chipselect = ARRAY_SIZE(bk3_spi_chipselects),
+	.use_dma	= 1,
+};
+
+/*************************************************************************
+ * TS72XX support code
+ *************************************************************************/
 #if IS_ENABLED(CONFIG_FPGA_MGR_TS73XX)
 
 /* Relative to EP93XX_CS1_PHYS_BASE */
@@ -257,7 +333,9 @@ static struct ep93xx_spi_info ts72xx_spi_info __initdata = {
 static void __init ts72xx_init_machine(void)
 {
 	ep93xx_init_devices();
-	ts72xx_register_flash();
+	ts72xx_register_flash(ts72xx_nand_parts, ARRAY_SIZE(ts72xx_nand_parts),
+			      is_ts9420_installed() ?
+			      EP93XX_CS7_PHYS_BASE : EP93XX_CS6_PHYS_BASE);
 	platform_device_register(&ts72xx_rtc_device);
 	platform_device_register(&ts72xx_wdt_device);
 
@@ -280,3 +358,66 @@ MACHINE_START(TS72XX, "Technologic Systems TS-72xx SBC")
 	.init_late	= ep93xx_init_late,
 	.restart	= ep93xx_restart,
 MACHINE_END
+
+/*************************************************************************
+ * EP93xx I2S audio peripheral handling
+ *************************************************************************/
+static struct resource ep93xx_i2s_resource[] = {
+	DEFINE_RES_MEM(EP93XX_I2S_PHYS_BASE, 0x100),
+	DEFINE_RES_IRQ_NAMED(IRQ_EP93XX_SAI, "spilink i2s slave"),
+};
+
+static struct platform_device ep93xx_i2s_device = {
+	.name		= "ep93xx-spilink-i2s",
+	.id		= -1,
+	.num_resources	= ARRAY_SIZE(ep93xx_i2s_resource),
+	.resource	= ep93xx_i2s_resource,
+};
+
+/*************************************************************************
+ * BK3 support code
+ *************************************************************************/
+static struct mtd_partition bk3_nand_parts[] = {
+	{
+		.name		= "System",
+		.offset	= 0x00000000,
+		.size		= 0x01e00000,
+	}, {
+		.name		= "Data",
+		.offset	= 0x01e00000,
+		.size		= 0x05f20000
+	}, {
+		.name		= "RedBoot",
+		.offset	= 0x07d20000,
+		.size		= 0x002e0000,
+		.mask_flags	= MTD_WRITEABLE,	/* force RO */
+	},
+};
+
+static void __init bk3_init_machine(void)
+{
+	ep93xx_init_devices();
+
+	ts72xx_register_flash(bk3_nand_parts, ARRAY_SIZE(bk3_nand_parts),
+			      EP93XX_CS6_PHYS_BASE);
+
+	ep93xx_register_eth(&ts72xx_eth_data, 1);
+
+	ep93xx_register_spi(&bk3_spi_master, bk3_spi_board_info,
+			    ARRAY_SIZE(bk3_spi_board_info));
+
+	/* Configure ep93xx's I2S to use AC97 pins */
+	ep93xx_devcfg_set_bits(EP93XX_SYSCON_DEVCFG_I2SONAC97);
+	platform_device_register(&ep93xx_i2s_device);
+}
+
+MACHINE_START(BK3, "Liebherr controller BK3.1")
+	/* Maintainer: Lukasz Majewski <lukma@denx.de> */
+	.atag_offset	= 0x100,
+	.map_io		= ts72xx_map_io,
+	.init_irq	= ep93xx_init_irq,
+	.init_time	= ep93xx_timer_init,
+	.init_machine	= bk3_init_machine,
+	.init_late	= ep93xx_init_late,
+	.restart	= ep93xx_restart,
+MACHINE_END
diff --git a/arch/arm/mach-ep93xx/ts72xx.h b/arch/arm/mach-ep93xx/ts72xx.h
index 8a3206a54b39..00b4941d29c9 100644
--- a/arch/arm/mach-ep93xx/ts72xx.h
+++ b/arch/arm/mach-ep93xx/ts72xx.h
@@ -10,8 +10,12 @@
  * febff000	22000000	4K	model number register (bits 0-2)
  * febfe000	22400000	4K	options register
  * febfd000	22800000	4K	options register #2
+ * febfc000     23400000        4K      CPLD version register
  */
 
+#ifndef __TS72XX_H_
+#define __TS72XX_H_
+
 #define TS72XX_MODEL_PHYS_BASE		0x22000000
 #define TS72XX_MODEL_VIRT_BASE		IOMEM(0xfebff000)
 #define TS72XX_MODEL_SIZE		0x00001000
@@ -39,6 +43,10 @@
 #define TS72XX_OPTIONS2_TS9420		0x04
 #define TS72XX_OPTIONS2_TS9420_BOOT	0x02
 
+#define TS72XX_CPLDVER_PHYS_BASE	0x23400000
+#define TS72XX_CPLDVER_VIRT_BASE	IOMEM(0xfebfc000)
+#define TS72XX_CPLDVER_SIZE		0x00001000
+
 #ifndef __ASSEMBLY__
 
 static inline int ts72xx_model(void)
@@ -83,3 +91,4 @@ static inline int is_ts9420_installed(void)
 					TS72XX_OPTIONS2_TS9420);
 }
 #endif
+#endif /* __TS72XX_H_ */
diff --git a/arch/arm/mach-exynos/Kconfig b/arch/arm/mach-exynos/Kconfig
index 44fa753bd79c..647c319f9f5f 100644
--- a/arch/arm/mach-exynos/Kconfig
+++ b/arch/arm/mach-exynos/Kconfig
@@ -1,9 +1,7 @@
-# arch/arm/mach-exynos/Kconfig
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
 #		http://www.samsung.com/
-#
-# Licensed under GPLv2
 
 # Configuration options for the EXYNOS
 
diff --git a/arch/arm/mach-exynos/Makefile b/arch/arm/mach-exynos/Makefile
index 9ea6c54645ad..cd00c82a1add 100644
--- a/arch/arm/mach-exynos/Makefile
+++ b/arch/arm/mach-exynos/Makefile
@@ -1,9 +1,7 @@
-# arch/arm/mach-exynos/Makefile
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
 #		http://www.samsung.com/
-#
-# Licensed under GPLv2
 
 ccflags-$(CONFIG_ARCH_MULTIPLATFORM) += -I$(srctree)/$(src)/include -I$(srctree)/arch/arm/plat-samsung/include
 
diff --git a/arch/arm/mach-exynos/common.h b/arch/arm/mach-exynos/common.h
index 3f715524c9d6..098f84a149a3 100644
--- a/arch/arm/mach-exynos/common.h
+++ b/arch/arm/mach-exynos/common.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
  * Common Header for EXYNOS machines
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ARCH_ARM_MACH_EXYNOS_COMMON_H
diff --git a/arch/arm/mach-exynos/exynos-smc.S b/arch/arm/mach-exynos/exynos-smc.S
index 2e27aa3813fd..d259532ba937 100644
--- a/arch/arm/mach-exynos/exynos-smc.S
+++ b/arch/arm/mach-exynos/exynos-smc.S
@@ -1,11 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2012 Samsung Electronics.
  *
  * Copied from omap-smc.S Copyright (C) 2010 Texas Instruments, Inc.
- *
- * This program is free software,you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/linkage.h>
diff --git a/arch/arm/mach-exynos/exynos.c b/arch/arm/mach-exynos/exynos.c
index 9a9caac1125a..fbd108ce8745 100644
--- a/arch/arm/mach-exynos/exynos.c
+++ b/arch/arm/mach-exynos/exynos.c
@@ -1,13 +1,9 @@
-/*
- * SAMSUNG EXYNOS Flattened Device Tree enabled machine
- *
- * Copyright (c) 2010-2014 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// SAMSUNG EXYNOS Flattened Device Tree enabled machine
+//
+// Copyright (c) 2010-2014 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
 
 #include <linux/init.h>
 #include <linux/io.h>
diff --git a/arch/arm/mach-exynos/firmware.c b/arch/arm/mach-exynos/firmware.c
index 2a51e4603a6f..be1f20fe28f4 100644
--- a/arch/arm/mach-exynos/firmware.c
+++ b/arch/arm/mach-exynos/firmware.c
@@ -1,12 +1,8 @@
-/*
- * Copyright (C) 2012 Samsung Electronics.
- * Kyungmin Park <kyungmin.park@samsung.com>
- * Tomasz Figa <t.figa@samsung.com>
- *
- * This program is free software,you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2012 Samsung Electronics.
+// Kyungmin Park <kyungmin.park@samsung.com>
+// Tomasz Figa <t.figa@samsung.com>
 
 #include <linux/kernel.h>
 #include <linux/io.h>
diff --git a/arch/arm/mach-exynos/headsmp.S b/arch/arm/mach-exynos/headsmp.S
index d3d24ab351ae..005695c9bf40 100644
--- a/arch/arm/mach-exynos/headsmp.S
+++ b/arch/arm/mach-exynos/headsmp.S
@@ -1,13 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- *
  *  Cloned from linux/arch/arm/mach-realview/headsmp.S
  *
  *  Copyright (c) 2003 ARM Limited
  *  All Rights Reserved
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 #include <linux/linkage.h>
 #include <linux/init.h>
diff --git a/arch/arm/mach-exynos/include/mach/map.h b/arch/arm/mach-exynos/include/mach/map.h
index 5fb0040cc6d3..37a5ea5e2602 100644
--- a/arch/arm/mach-exynos/include/mach/map.h
+++ b/arch/arm/mach-exynos/include/mach/map.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com/
  *
  * EXYNOS - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_MAP_H
 #define __ASM_ARCH_MAP_H __FILE__
diff --git a/arch/arm/mach-exynos/mcpm-exynos.c b/arch/arm/mach-exynos/mcpm-exynos.c
index b42622562ea7..72bc035bedbe 100644
--- a/arch/arm/mach-exynos/mcpm-exynos.c
+++ b/arch/arm/mach-exynos/mcpm-exynos.c
@@ -1,15 +1,8 @@
-/*
- * Copyright (c) 2014 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * arch/arm/mach-exynos/mcpm-exynos.c
- *
- * Based on arch/arm/mach-vexpress/dcscb.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2014 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// Based on arch/arm/mach-vexpress/dcscb.c
 
 #include <linux/arm-cci.h>
 #include <linux/delay.h>
diff --git a/arch/arm/mach-exynos/platsmp.c b/arch/arm/mach-exynos/platsmp.c
index 5a03bffe7226..5156fe70e030 100644
--- a/arch/arm/mach-exynos/platsmp.c
+++ b/arch/arm/mach-exynos/platsmp.c
@@ -1,16 +1,11 @@
- /*
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * Cloned from linux/arch/arm/mach-vexpress/platsmp.c
- *
- *  Copyright (C) 2002 ARM Ltd.
- *  All Rights Reserved
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// Cloned from linux/arch/arm/mach-vexpress/platsmp.c
+//
+//  Copyright (C) 2002 ARM Ltd.
+//  All Rights Reserved
 
 #include <linux/init.h>
 #include <linux/errno.h>
diff --git a/arch/arm/mach-exynos/pm.c b/arch/arm/mach-exynos/pm.c
index c9740d96db9e..dc4346ecf16d 100644
--- a/arch/arm/mach-exynos/pm.c
+++ b/arch/arm/mach-exynos/pm.c
@@ -1,17 +1,13 @@
-/*
- * Copyright (c) 2011-2014 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * EXYNOS - Power Management support
- *
- * Based on arch/arm/mach-s3c2410/pm.c
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2011-2014 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// EXYNOS - Power Management support
+//
+// Based on arch/arm/mach-s3c2410/pm.c
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/init.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/mach-exynos/sleep.S b/arch/arm/mach-exynos/sleep.S
index 4292cae43f3c..2783c3a0c06a 100644
--- a/arch/arm/mach-exynos/sleep.S
+++ b/arch/arm/mach-exynos/sleep.S
@@ -1,18 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright (c) 2013 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
  * Exynos low-level resume code
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/linkage.h>
diff --git a/arch/arm/mach-exynos/smc.h b/arch/arm/mach-exynos/smc.h
index c2845717bc8f..f355185d4239 100644
--- a/arch/arm/mach-exynos/smc.h
+++ b/arch/arm/mach-exynos/smc.h
@@ -1,11 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *  Copyright (c) 2012 Samsung Electronics.
  *
  * EXYNOS - SMC Call
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ASM_ARCH_EXYNOS_SMC_H
diff --git a/arch/arm/mach-exynos/suspend.c b/arch/arm/mach-exynos/suspend.c
index 370d37ded7e7..c2ed997fedef 100644
--- a/arch/arm/mach-exynos/suspend.c
+++ b/arch/arm/mach-exynos/suspend.c
@@ -1,17 +1,13 @@
-/*
- * Copyright (c) 2011-2014 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * EXYNOS - Suspend support
- *
- * Based on arch/arm/mach-s3c2410/pm.c
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2011-2014 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// EXYNOS - Suspend support
+//
+// Based on arch/arm/mach-s3c2410/pm.c
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/init.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/mach-imx/common.h b/arch/arm/mach-imx/common.h
index b09a2ec19267..c8d68e918b2f 100644
--- a/arch/arm/mach-imx/common.h
+++ b/arch/arm/mach-imx/common.h
@@ -71,15 +71,6 @@ enum mxc_cpu_pwr_mode {
 	STOP_POWER_OFF,		/* STOP + SRPG */
 };
 
-enum mx3_cpu_pwr_mode {
-	MX3_RUN,
-	MX3_WAIT,
-	MX3_DOZE,
-	MX3_SLEEP,
-};
-
-void mx3_cpu_lp_set(enum mx3_cpu_pwr_mode mode);
-
 void imx_enable_cpu(int cpu, bool enable);
 void imx_set_cpu_jump(int cpu, void *jump_addr);
 u32 imx_get_cpu_arg(int cpu);
diff --git a/arch/arm/mach-imx/mmdc.c b/arch/arm/mach-imx/mmdc.c
index 78262899a590..5fb1d2254b5e 100644
--- a/arch/arm/mach-imx/mmdc.c
+++ b/arch/arm/mach-imx/mmdc.c
@@ -547,7 +547,6 @@ static int imx_mmdc_probe(struct platform_device *pdev)
 	struct device_node *np = pdev->dev.of_node;
 	void __iomem *mmdc_base, *reg;
 	u32 val;
-	int timeout = 0x400;
 
 	mmdc_base = of_iomap(np, 0);
 	WARN_ON(!mmdc_base);
@@ -565,16 +564,6 @@ static int imx_mmdc_probe(struct platform_device *pdev)
 	val &= ~(1 << BP_MMDC_MAPSR_PSD);
 	writel_relaxed(val, reg);
 
-	/* Ensure it's successfully enabled */
-	while (!(readl_relaxed(reg) & 1 << BP_MMDC_MAPSR_PSS) && --timeout)
-		cpu_relax();
-
-	if (unlikely(!timeout)) {
-		pr_warn("%s: failed to enable automatic power saving\n",
-			__func__);
-		return -EBUSY;
-	}
-
 	return imx_mmdc_perf_init(pdev, mmdc_base);
 }
 
diff --git a/arch/arm/mach-ixp4xx/vulcan-setup.c b/arch/arm/mach-ixp4xx/vulcan-setup.c
index 731fb2019ecb..2c03d2f6b647 100644
--- a/arch/arm/mach-ixp4xx/vulcan-setup.c
+++ b/arch/arm/mach-ixp4xx/vulcan-setup.c
@@ -16,6 +16,7 @@
 #include <linux/serial_8250.h>
 #include <linux/io.h>
 #include <linux/w1-gpio.h>
+#include <linux/gpio/machine.h>
 #include <linux/mtd/plat-ram.h>
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
@@ -162,9 +163,16 @@ static struct platform_device vulcan_max6369 = {
 	.num_resources		= 1,
 };
 
+static struct gpiod_lookup_table vulcan_w1_gpiod_table = {
+	.dev_id = "w1-gpio",
+	.table = {
+		GPIO_LOOKUP_IDX("IXP4XX_GPIO_CHIP", 14, NULL, 0,
+				GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN),
+	},
+};
+
 static struct w1_gpio_platform_data vulcan_w1_gpio_pdata = {
-	.pin			= 14,
-	.ext_pullup_enable_pin	= -EINVAL,
+	/* Intentionally left blank */
 };
 
 static struct platform_device vulcan_w1_gpio = {
@@ -233,6 +241,7 @@ static void __init vulcan_init(void)
 			  IXP4XX_EXP_BUS_WR_EN		|
 			  IXP4XX_EXP_BUS_BYTE_EN;
 
+	gpiod_add_lookup_table(&vulcan_w1_gpiod_table);
 	platform_add_devices(vulcan_devices, ARRAY_SIZE(vulcan_devices));
 }
 
diff --git a/arch/arm/mach-mediatek/Kconfig b/arch/arm/mach-mediatek/Kconfig
index 70e49d54434e..91cc461f7b04 100644
--- a/arch/arm/mach-mediatek/Kconfig
+++ b/arch/arm/mach-mediatek/Kconfig
@@ -1,5 +1,5 @@
 menuconfig ARCH_MEDIATEK
-	bool "Mediatek MT65xx & MT81xx SoC"
+	bool "MediaTek SoC Support"
 	depends on ARCH_MULTI_V7
 	select ARM_GIC
 	select PINCTRL
diff --git a/arch/arm/mach-meson/Kconfig b/arch/arm/mach-meson/Kconfig
index aff6164b2083..d90f61e6254f 100644
--- a/arch/arm/mach-meson/Kconfig
+++ b/arch/arm/mach-meson/Kconfig
@@ -23,6 +23,7 @@ config MACH_MESON8
 	default ARCH_MESON
 	select MESON6_TIMER
 	select COMMON_CLK_MESON8B
+	select MESON_IRQ_GPIO
 
 config MACH_MESON8B
 	bool "Amlogic Meson8b SoCs support"
diff --git a/arch/arm/mach-mmp/mmp2.h b/arch/arm/mach-mmp/mmp2.h
index a4b82f719de1..adafc4fba8f4 100644
--- a/arch/arm/mach-mmp/mmp2.h
+++ b/arch/arm/mach-mmp/mmp2.h
@@ -10,7 +10,7 @@ extern void __init mmp2_init_irq(void);
 extern void mmp2_clear_pmic_int(void);
 
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/platform_data/dma-mmp_tdma.h>
 
 #include "devices.h"
diff --git a/arch/arm/mach-mmp/pxa168.h b/arch/arm/mach-mmp/pxa168.h
index b39bff37ff11..0331c58b07a2 100644
--- a/arch/arm/mach-mmp/pxa168.h
+++ b/arch/arm/mach-mmp/pxa168.h
@@ -11,7 +11,7 @@ extern void pxa168_restart(enum reboot_mode, const char *);
 extern void pxa168_clear_keypad_wakeup(void);
 
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/platform_data/mtd-nand-pxa3xx.h>
 #include <video/pxa168fb.h>
 #include <linux/platform_data/keypad-pxa27x.h>
diff --git a/arch/arm/mach-mmp/pxa910.h b/arch/arm/mach-mmp/pxa910.h
index cb3923dcf580..42009c349eae 100644
--- a/arch/arm/mach-mmp/pxa910.h
+++ b/arch/arm/mach-mmp/pxa910.h
@@ -7,7 +7,7 @@ extern void __init icu_init_irq(void);
 extern void __init pxa910_init_irq(void);
 
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/platform_data/mtd-nand-pxa3xx.h>
 #include <video/mmp_disp.h>
 
diff --git a/arch/arm/mach-mvebu/Kconfig b/arch/arm/mach-mvebu/Kconfig
index 9b49867154bf..2c20599cc350 100644
--- a/arch/arm/mach-mvebu/Kconfig
+++ b/arch/arm/mach-mvebu/Kconfig
@@ -10,7 +10,6 @@ menuconfig ARCH_MVEBU
 	select ZONE_DMA if ARM_LPAE
 	select GPIOLIB
 	select PCI_QUIRKS if PCI
-	select OF_ADDRESS_PCI
 
 if ARCH_MVEBU
 
@@ -42,7 +41,7 @@ config MACH_ARMADA_375
 	depends on ARCH_MULTI_V7
 	select ARMADA_370_XP_IRQ
 	select ARM_ERRATA_720789
-	select ARM_ERRATA_753970
+	select PL310_ERRATA_753970
 	select ARM_GIC
 	select ARMADA_375_CLK
 	select HAVE_ARM_SCU
@@ -58,7 +57,7 @@ config MACH_ARMADA_38X
 	bool "Marvell Armada 380/385 boards"
 	depends on ARCH_MULTI_V7
 	select ARM_ERRATA_720789
-	select ARM_ERRATA_753970
+	select PL310_ERRATA_753970
 	select ARM_GIC
 	select ARM_GLOBAL_TIMER
 	select CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK
diff --git a/arch/arm/mach-omap1/clock.c b/arch/arm/mach-omap1/clock.c
index 43e3e188f521..fa512413a471 100644
--- a/arch/arm/mach-omap1/clock.c
+++ b/arch/arm/mach-omap1/clock.c
@@ -1011,17 +1011,17 @@ static int clk_debugfs_register_one(struct clk *c)
 		return -ENOMEM;
 	c->dent = d;
 
-	d = debugfs_create_u8("usecount", S_IRUGO, c->dent, (u8 *)&c->usecount);
+	d = debugfs_create_u8("usecount", S_IRUGO, c->dent, &c->usecount);
 	if (!d) {
 		err = -ENOMEM;
 		goto err_out;
 	}
-	d = debugfs_create_u32("rate", S_IRUGO, c->dent, (u32 *)&c->rate);
+	d = debugfs_create_ulong("rate", S_IRUGO, c->dent, &c->rate);
 	if (!d) {
 		err = -ENOMEM;
 		goto err_out;
 	}
-	d = debugfs_create_x32("flags", S_IRUGO, c->dent, (u32 *)&c->flags);
+	d = debugfs_create_x8("flags", S_IRUGO, c->dent, &c->flags);
 	if (!d) {
 		err = -ENOMEM;
 		goto err_out;
diff --git a/arch/arm/mach-omap2/Makefile b/arch/arm/mach-omap2/Makefile
index 2f722a805948..c15bbcad5f67 100644
--- a/arch/arm/mach-omap2/Makefile
+++ b/arch/arm/mach-omap2/Makefile
@@ -232,6 +232,3 @@ obj-y					+= $(omap-hsmmc-m) $(omap-hsmmc-y)
 obj-y					+= omap_phy_internal.o
 
 obj-$(CONFIG_MACH_OMAP2_TUSB6010)	+= usb-tusb6010.o
-
-onenand-$(CONFIG_MTD_ONENAND_OMAP2)	:= gpmc-onenand.o
-obj-y					+= $(onenand-m) $(onenand-y)
diff --git a/arch/arm/mach-omap2/clockdomain.c b/arch/arm/mach-omap2/clockdomain.c
index 518926410b62..b79b1ca9aee9 100644
--- a/arch/arm/mach-omap2/clockdomain.c
+++ b/arch/arm/mach-omap2/clockdomain.c
@@ -1224,14 +1224,6 @@ int clkdm_clk_disable(struct clockdomain *clkdm, struct clk *clk)
 	return 0;
 }
 
-u32 clkdm_xlate_address(struct clockdomain *clkdm)
-{
-	if (arch_clkdm->clkdm_xlate_address)
-		return arch_clkdm->clkdm_xlate_address(clkdm);
-
-	return 0;
-}
-
 /**
  * clkdm_hwmod_enable - add an enabled downstream hwmod to this clkdm
  * @clkdm: struct clockdomain *
diff --git a/arch/arm/mach-omap2/clockdomain.h b/arch/arm/mach-omap2/clockdomain.h
index 827f01e2d0af..24667a5a9dc0 100644
--- a/arch/arm/mach-omap2/clockdomain.h
+++ b/arch/arm/mach-omap2/clockdomain.h
@@ -175,7 +175,6 @@ struct clkdm_ops {
 	void	(*clkdm_deny_idle)(struct clockdomain *clkdm);
 	int	(*clkdm_clk_enable)(struct clockdomain *clkdm);
 	int	(*clkdm_clk_disable)(struct clockdomain *clkdm);
-	u32	(*clkdm_xlate_address)(struct clockdomain *clkdm);
 };
 
 int clkdm_register_platform_funcs(struct clkdm_ops *co);
@@ -214,7 +213,6 @@ int clkdm_clk_enable(struct clockdomain *clkdm, struct clk *clk);
 int clkdm_clk_disable(struct clockdomain *clkdm, struct clk *clk);
 int clkdm_hwmod_enable(struct clockdomain *clkdm, struct omap_hwmod *oh);
 int clkdm_hwmod_disable(struct clockdomain *clkdm, struct omap_hwmod *oh);
-u32 clkdm_xlate_address(struct clockdomain *clkdm);
 
 extern void __init omap242x_clockdomains_init(void);
 extern void __init omap243x_clockdomains_init(void);
diff --git a/arch/arm/mach-omap2/cm-regbits-24xx.h b/arch/arm/mach-omap2/cm-regbits-24xx.h
index d7a5d11cbcbf..9ff0fc70f152 100644
--- a/arch/arm/mach-omap2/cm-regbits-24xx.h
+++ b/arch/arm/mach-omap2/cm-regbits-24xx.h
@@ -14,38 +14,8 @@
  * published by the Free Software Foundation.
  */
 
-#define OMAP24XX_EN_CAM_SHIFT				31
-#define OMAP24XX_EN_WDT4_SHIFT				29
-#define OMAP2420_EN_WDT3_SHIFT				28
-#define OMAP24XX_EN_MSPRO_SHIFT				27
-#define OMAP24XX_EN_FAC_SHIFT				25
-#define OMAP2420_EN_EAC_SHIFT				24
-#define OMAP24XX_EN_HDQ_SHIFT				23
-#define OMAP2420_EN_I2C2_SHIFT				20
-#define OMAP2420_EN_I2C1_SHIFT				19
-#define OMAP2430_EN_MCBSP5_SHIFT			5
-#define OMAP2430_EN_MCBSP4_SHIFT			4
-#define OMAP2430_EN_MCBSP3_SHIFT			3
-#define OMAP24XX_EN_SSI_SHIFT				1
-#define OMAP24XX_EN_MPU_WDT_SHIFT			3
-#define OMAP24XX_CLKSEL_MPU_SHIFT			0
-#define OMAP24XX_CLKSEL_MPU_WIDTH			5
 #define OMAP24XX_AUTOSTATE_MPU_MASK			(1 << 0)
-#define OMAP24XX_EN_TV_SHIFT				2
-#define OMAP24XX_EN_DSS2_SHIFT				1
-#define OMAP24XX_EN_DSS1_SHIFT				0
 #define OMAP24XX_EN_DSS1_MASK				(1 << 0)
-#define OMAP2430_EN_I2CHS2_SHIFT			20
-#define OMAP2430_EN_I2CHS1_SHIFT			19
-#define OMAP2430_EN_MMCHSDB2_SHIFT			17
-#define OMAP2430_EN_MMCHSDB1_SHIFT			16
-#define OMAP24XX_EN_MAILBOXES_SHIFT			30
-#define OMAP2430_EN_SDRC_SHIFT				2
-#define OMAP24XX_EN_PKA_SHIFT				4
-#define OMAP24XX_EN_AES_SHIFT				3
-#define OMAP24XX_EN_RNG_SHIFT				2
-#define OMAP24XX_EN_SHA_SHIFT				1
-#define OMAP24XX_EN_DES_SHIFT				0
 #define OMAP24XX_ST_MAILBOXES_SHIFT			30
 #define OMAP24XX_ST_HDQ_SHIFT				23
 #define OMAP2420_ST_I2C2_SHIFT				20
@@ -54,81 +24,30 @@
 #define OMAP2430_ST_I2CHS2_SHIFT			20
 #define OMAP24XX_ST_MCBSP2_SHIFT			16
 #define OMAP24XX_ST_MCBSP1_SHIFT			15
-#define OMAP24XX_ST_DSS_SHIFT				0
 #define OMAP2430_ST_MCBSP5_SHIFT			5
 #define OMAP2430_ST_MCBSP4_SHIFT			4
 #define OMAP2430_ST_MCBSP3_SHIFT			3
 #define OMAP24XX_ST_AES_SHIFT				3
 #define OMAP24XX_ST_RNG_SHIFT				2
 #define OMAP24XX_ST_SHA_SHIFT				1
-#define OMAP24XX_AUTO_SDRC_SHIFT			2
-#define OMAP24XX_AUTO_GPMC_SHIFT			1
-#define OMAP24XX_AUTO_SDMA_SHIFT			0
-#define OMAP24XX_CLKSEL_USB_MASK			(0x7 << 25)
-#define OMAP24XX_CLKSEL_SSI_MASK			(0x1f << 20)
-#define OMAP2420_CLKSEL_VLYNQ_MASK			(0x1f << 15)
 #define OMAP24XX_CLKSEL_DSS2_MASK			(0x1 << 13)
-#define OMAP24XX_CLKSEL_DSS1_MASK			(0x1f << 8)
-#define OMAP24XX_CLKSEL_L4_SHIFT			5
-#define OMAP24XX_CLKSEL_L4_WIDTH			2
-#define OMAP24XX_CLKSEL_L3_SHIFT			0
-#define OMAP24XX_CLKSEL_L3_WIDTH			5
-#define OMAP24XX_CLKSEL_GPT12_MASK			(0x3 << 22)
-#define OMAP24XX_CLKSEL_GPT11_MASK			(0x3 << 20)
-#define OMAP24XX_CLKSEL_GPT10_MASK			(0x3 << 18)
-#define OMAP24XX_CLKSEL_GPT9_MASK			(0x3 << 16)
-#define OMAP24XX_CLKSEL_GPT8_MASK			(0x3 << 14)
-#define OMAP24XX_CLKSEL_GPT7_MASK			(0x3 << 12)
-#define OMAP24XX_CLKSEL_GPT6_MASK			(0x3 << 10)
-#define OMAP24XX_CLKSEL_GPT5_MASK			(0x3 << 8)
-#define OMAP24XX_CLKSEL_GPT4_MASK			(0x3 << 6)
-#define OMAP24XX_CLKSEL_GPT3_MASK			(0x3 << 4)
-#define OMAP24XX_CLKSEL_GPT2_MASK			(0x3 << 2)
 #define OMAP24XX_AUTOSTATE_DSS_MASK			(1 << 2)
 #define OMAP24XX_AUTOSTATE_L4_MASK			(1 << 1)
 #define OMAP24XX_AUTOSTATE_L3_MASK			(1 << 0)
-#define OMAP24XX_EN_3D_SHIFT				2
-#define OMAP24XX_EN_2D_SHIFT				1
 #define OMAP24XX_AUTOSTATE_GFX_MASK			(1 << 0)
-#define OMAP2430_EN_ICR_SHIFT				6
-#define OMAP24XX_EN_OMAPCTRL_SHIFT			5
-#define OMAP24XX_EN_WDT1_SHIFT				4
-#define OMAP24XX_EN_32KSYNC_SHIFT			1
 #define OMAP24XX_ST_MPU_WDT_SHIFT			3
 #define OMAP24XX_ST_32KSYNC_SHIFT			1
-#define OMAP24XX_CLKSEL_GPT1_MASK			(0x3 << 0)
 #define OMAP24XX_EN_54M_PLL_SHIFT			6
 #define OMAP24XX_EN_96M_PLL_SHIFT			2
-#define OMAP24XX_EN_DPLL_MASK				(0x3 << 0)
 #define OMAP24XX_ST_54M_APLL_SHIFT			9
 #define OMAP24XX_ST_96M_APLL_SHIFT			8
 #define OMAP24XX_AUTO_54M_MASK				(0x3 << 6)
 #define OMAP24XX_AUTO_96M_MASK				(0x3 << 2)
 #define OMAP24XX_AUTO_DPLL_SHIFT			0
 #define OMAP24XX_AUTO_DPLL_MASK				(0x3 << 0)
-#define OMAP24XX_APLLS_CLKIN_SHIFT			23
-#define OMAP24XX_APLLS_CLKIN_WIDTH			3
-#define OMAP24XX_APLLS_CLKIN_MASK			(0x7 << 23)
-#define OMAP24XX_DPLL_MULT_MASK				(0x3ff << 12)
-#define OMAP24XX_DPLL_DIV_MASK				(0xf << 8)
-#define OMAP24XX_54M_SOURCE_SHIFT			5
-#define OMAP24XX_54M_SOURCE_WIDTH			1
-#define OMAP2430_96M_SOURCE_SHIFT			4
-#define OMAP2430_96M_SOURCE_WIDTH			1
-#define OMAP24XX_48M_SOURCE_MASK			(1 << 3)
 #define OMAP24XX_CORE_CLK_SRC_MASK			(0x3 << 0)
-#define OMAP2420_EN_IVA_COP_SHIFT			10
-#define OMAP2420_EN_IVA_MPU_SHIFT			8
-#define OMAP24XX_CM_FCLKEN_DSP_EN_DSP_SHIFT		0
-#define OMAP2420_EN_DSP_IPI_SHIFT			1
-#define OMAP2420_CLKSEL_IVA_MASK			(0x1f << 8)
-#define OMAP24XX_CLKSEL_DSP_IF_MASK			(0x3 << 5)
-#define OMAP24XX_CLKSEL_DSP_MASK			(0x1f << 0)
 #define OMAP2420_AUTOSTATE_IVA_MASK			(1 << 8)
 #define OMAP24XX_AUTOSTATE_DSP_MASK			(1 << 0)
-#define OMAP2430_EN_OSC_SHIFT				1
-#define OMAP2430_CM_ICLKEN_MDM_EN_MDM_SHIFT		0
-#define OMAP2430_CLKSEL_MDM_MASK			(0xf << 0)
 #define OMAP2430_AUTOSTATE_MDM_MASK			(1 << 0)
 #define OMAP24XX_CLKSTCTRL_DISABLE_AUTO		0x0
 #define OMAP24XX_CLKSTCTRL_ENABLE_AUTO		0x1
diff --git a/arch/arm/mach-omap2/cm-regbits-34xx.h b/arch/arm/mach-omap2/cm-regbits-34xx.h
index ee6c784cd6b7..38656ce2432c 100644
--- a/arch/arm/mach-omap2/cm-regbits-34xx.h
+++ b/arch/arm/mach-omap2/cm-regbits-34xx.h
@@ -14,68 +14,11 @@
  * published by the Free Software Foundation.
  */
 
-#define OMAP3430ES2_EN_MMC3_SHIFT			30
-#define OMAP3430_EN_MSPRO_SHIFT				23
-#define OMAP3430_EN_HDQ_SHIFT				22
-#define OMAP3430ES1_EN_FSHOSTUSB_SHIFT			5
-#define OMAP3430ES1_EN_D2D_SHIFT			3
-#define OMAP3430_EN_SSI_SHIFT				0
-#define OMAP3430ES2_EN_USBTLL_SHIFT			2
-#define OMAP3430_EN_WDT2_SHIFT				5
-#define OMAP3430_EN_CAM_SHIFT				0
-#define OMAP3430_EN_WDT3_SHIFT				12
 #define OMAP3430_CM_FCLKEN_IVA2_EN_IVA2_MASK		(1 << 0)
-#define OMAP3430_CM_FCLKEN_IVA2_EN_IVA2_SHIFT		0
-#define OMAP3430_IVA2_DPLL_FREQSEL_SHIFT		4
-#define OMAP3430_IVA2_DPLL_FREQSEL_MASK			(0xf << 4)
-#define OMAP3430_EN_IVA2_DPLL_DRIFTGUARD_SHIFT		3
-#define OMAP3430_EN_IVA2_DPLL_SHIFT			0
-#define OMAP3430_EN_IVA2_DPLL_MASK			(0x7 << 0)
 #define OMAP3430_ST_IVA2_SHIFT				0
-#define OMAP3430_ST_IVA2_CLK_MASK			(1 << 0)
-#define OMAP3430_AUTO_IVA2_DPLL_SHIFT			0
-#define OMAP3430_AUTO_IVA2_DPLL_MASK			(0x7 << 0)
-#define OMAP3430_IVA2_CLK_SRC_SHIFT			19
-#define OMAP3430_IVA2_CLK_SRC_WIDTH			3
-#define OMAP3430_IVA2_DPLL_MULT_MASK			(0x7ff << 8)
-#define OMAP3430_IVA2_DPLL_DIV_MASK			(0x7f << 0)
-#define OMAP3430_IVA2_DPLL_CLKOUT_DIV_SHIFT		0
-#define OMAP3430_IVA2_DPLL_CLKOUT_DIV_WIDTH		5
 #define OMAP3430_CLKTRCTRL_IVA2_MASK			(0x3 << 0)
 #define OMAP3430_CLKACTIVITY_IVA2_MASK			(1 << 0)
-#define OMAP3430_MPU_DPLL_FREQSEL_MASK			(0xf << 4)
-#define OMAP3430_EN_MPU_DPLL_DRIFTGUARD_SHIFT		3
-#define OMAP3430_EN_MPU_DPLL_MASK			(0x7 << 0)
-#define OMAP3430_ST_MPU_CLK_SHIFT			0
-#define OMAP3430_ST_MPU_CLK_MASK			(1 << 0)
-#define OMAP3430_ST_MPU_CLK_WIDTH			1
-#define OMAP3430_AUTO_MPU_DPLL_MASK			(0x7 << 0)
-#define OMAP3430_MPU_CLK_SRC_SHIFT			19
-#define OMAP3430_MPU_CLK_SRC_WIDTH			3
-#define OMAP3430_MPU_DPLL_MULT_MASK			(0x7ff << 8)
-#define OMAP3430_MPU_DPLL_DIV_MASK			(0x7f << 0)
-#define OMAP3430_MPU_DPLL_CLKOUT_DIV_SHIFT		0
-#define OMAP3430_MPU_DPLL_CLKOUT_DIV_WIDTH		5
 #define OMAP3430_CLKTRCTRL_MPU_MASK			(0x3 << 0)
-#define OMAP3430_EN_MODEM_SHIFT				31
-#define OMAP3430_EN_ICR_SHIFT				29
-#define OMAP3430_EN_AES2_SHIFT				28
-#define OMAP3430_EN_SHA12_SHIFT				27
-#define OMAP3430_EN_DES2_SHIFT				26
-#define OMAP3430ES1_EN_FAC_SHIFT			8
-#define OMAP3430_EN_MAILBOXES_SHIFT			7
-#define OMAP3430_EN_OMAPCTRL_SHIFT			6
-#define OMAP3430_EN_SAD2D_SHIFT				3
-#define OMAP3430_EN_SDRC_SHIFT				1
-#define AM35XX_EN_IPSS_SHIFT				4
-#define OMAP3430_EN_PKA_SHIFT				4
-#define OMAP3430_EN_AES1_SHIFT				3
-#define OMAP3430_EN_RNG_SHIFT				2
-#define OMAP3430_EN_SHA11_SHIFT				1
-#define OMAP3430_EN_DES1_SHIFT				0
-#define OMAP3430_EN_MAD2D_SHIFT				3
-#define OMAP3430ES2_EN_TS_SHIFT				1
-#define OMAP3430ES2_EN_CPEFUSE_SHIFT			0
 #define OMAP3430_ST_AES2_SHIFT				28
 #define OMAP3430_ST_SHA12_SHIFT				27
 #define AM35XX_ST_UART4_SHIFT				23
@@ -84,131 +27,26 @@
 #define OMAP3430_ST_MAILBOXES_SHIFT			7
 #define OMAP3430_ST_SAD2D_SHIFT				3
 #define OMAP3430_ST_SDMA_SHIFT				2
-#define AM35XX_ST_IPSS_SHIFT				5
 #define OMAP3430ES2_ST_USBTLL_SHIFT			2
-#define OMAP3430_CLKSEL_SSI_MASK			(0xf << 8)
-#define OMAP3430_CLKSEL_GPT11_MASK			(1 << 7)
-#define OMAP3430_CLKSEL_GPT10_MASK			(1 << 6)
-#define OMAP3430ES1_CLKSEL_FSHOSTUSB_MASK		(0x3 << 4)
-#define OMAP3430_CLKSEL_L4_SHIFT			2
-#define OMAP3430_CLKSEL_L4_WIDTH			2
-#define OMAP3430_CLKSEL_L3_SHIFT			0
-#define OMAP3430_CLKSEL_L3_WIDTH			2
-#define OMAP3630_CLKSEL_96M_MASK			(0x3 << 12)
 #define OMAP3430ES1_CLKTRCTRL_D2D_MASK			(0x3 << 4)
 #define OMAP3430_CLKTRCTRL_L4_MASK			(0x3 << 2)
 #define OMAP3430_CLKTRCTRL_L3_MASK			(0x3 << 0)
-#define OMAP3430ES1_EN_3D_SHIFT				2
-#define OMAP3430ES1_EN_2D_SHIFT				1
 #define OMAP3430ES1_CLKTRCTRL_GFX_MASK			(0x3 << 0)
-#define OMAP3430ES2_CM_FCLKEN_SGX_EN_SGX_SHIFT		1
-#define OMAP3430ES2_CM_ICLKEN_SGX_EN_SGX_SHIFT		0
-#define OMAP3430ES2_CLKSEL_SGX_MASK			(0x7 << 0)
 #define OMAP3430ES2_CLKTRCTRL_SGX_MASK			(0x3 << 0)
-#define OMAP3430ES2_EN_USIMOCP_SHIFT			9
-#define OMAP3430_EN_WDT1_SHIFT				4
-#define OMAP3430_EN_32KSYNC_SHIFT			2
 #define OMAP3430_ST_WDT2_SHIFT				5
 #define OMAP3430_ST_32KSYNC_SHIFT			2
-#define OMAP3430ES2_CLKSEL_USIMOCP_MASK			(0xf << 3)
-#define OMAP3430_CLKSEL_RM_SHIFT			1
-#define OMAP3430_CLKSEL_RM_WIDTH			2
-#define OMAP3430_CLKSEL_GPT1_MASK			(1 << 0)
-#define OMAP3430_PWRDN_EMU_PERIPH_SHIFT			31
-#define OMAP3430_PWRDN_CAM_SHIFT			30
-#define OMAP3430_PWRDN_DSS1_SHIFT			29
-#define OMAP3430_PWRDN_TV_SHIFT				28
-#define OMAP3430_PWRDN_96M_SHIFT			27
-#define OMAP3430_PERIPH_DPLL_FREQSEL_MASK		(0xf << 20)
-#define OMAP3430_EN_PERIPH_DPLL_DRIFTGUARD_SHIFT	19
-#define OMAP3430_EN_PERIPH_DPLL_MASK			(0x7 << 16)
-#define OMAP3430_PWRDN_EMU_CORE_SHIFT			12
-#define OMAP3430_CORE_DPLL_FREQSEL_MASK			(0xf << 4)
-#define OMAP3430_EN_CORE_DPLL_DRIFTGUARD_SHIFT		3
-#define OMAP3430_EN_CORE_DPLL_MASK			(0x7 << 0)
-#define OMAP3430ES2_PERIPH2_DPLL_FREQSEL_MASK		(0xf << 4)
-#define OMAP3430ES2_EN_PERIPH2_DPLL_DRIFTGUARD_SHIFT	3
-#define OMAP3430ES2_EN_PERIPH2_DPLL_MASK		(0x7 << 0)
-#define OMAP3430_ST_PERIPH_CLK_MASK			(1 << 1)
-#define OMAP3430_ST_CORE_CLK_MASK			(1 << 0)
-#define OMAP3430ES2_ST_PERIPH2_CLK_MASK			(1 << 0)
 #define OMAP3430_AUTO_PERIPH_DPLL_MASK			(0x7 << 3)
-#define OMAP3430_AUTO_CORE_DPLL_MASK			(0x7 << 0)
-#define OMAP3430ES2_AUTO_PERIPH2_DPLL_MASK		(0x7 << 0)
-#define OMAP3430_CORE_DPLL_CLKOUT_DIV_SHIFT		27
-#define OMAP3430_CORE_DPLL_CLKOUT_DIV_WIDTH		5
-#define OMAP3430_CORE_DPLL_MULT_MASK			(0x7ff << 16)
-#define OMAP3430_CORE_DPLL_DIV_MASK			(0x7f << 8)
-#define OMAP3430_SOURCE_96M_SHIFT			6
-#define OMAP3430_SOURCE_96M_WIDTH			1
-#define OMAP3430_SOURCE_54M_SHIFT			5
-#define OMAP3430_SOURCE_54M_WIDTH			1
-#define OMAP3430_SOURCE_48M_MASK			(1 << 3)
-#define OMAP3430_PERIPH_DPLL_MULT_MASK			(0x7ff << 8)
-#define OMAP3630_PERIPH_DPLL_MULT_MASK			(0xfff << 8)
-#define OMAP3430_PERIPH_DPLL_DIV_MASK			(0x7f << 0)
-#define OMAP3630_PERIPH_DPLL_DCO_SEL_MASK		(0x7 << 21)
-#define OMAP3630_PERIPH_DPLL_SD_DIV_MASK		(0xff << 24)
-#define OMAP3430_DIV_96M_SHIFT				0
-#define OMAP3630_DIV_96M_WIDTH				6
-#define OMAP3430ES2_PERIPH2_DPLL_MULT_MASK		(0x7ff << 8)
-#define OMAP3430ES2_PERIPH2_DPLL_DIV_MASK		(0x7f << 0)
-#define OMAP3430ES2_DIV_120M_SHIFT			0
-#define OMAP3430ES2_DIV_120M_WIDTH			5
-#define OMAP3430_CLKOUT2_EN_SHIFT			7
-#define OMAP3430_CLKOUT2_DIV_SHIFT			3
-#define OMAP3430_CLKOUT2_DIV_WIDTH			3
-#define OMAP3430_CLKOUT2SOURCE_MASK			(0x3 << 0)
-#define OMAP3430_EN_TV_SHIFT				2
-#define OMAP3430_EN_DSS2_SHIFT				1
-#define OMAP3430_EN_DSS1_SHIFT				0
-#define OMAP3430_CM_ICLKEN_DSS_EN_DSS_SHIFT		0
 #define OMAP3430ES2_ST_DSS_IDLE_SHIFT			1
-#define OMAP3430ES2_ST_DSS_STDBY_SHIFT			0
-#define OMAP3430ES1_ST_DSS_SHIFT			0
-#define OMAP3430_CLKSEL_TV_SHIFT			8
-#define OMAP3630_CLKSEL_TV_WIDTH			6
-#define OMAP3430_CLKSEL_DSS1_SHIFT			0
-#define OMAP3630_CLKSEL_DSS1_WIDTH			6
 #define OMAP3430_CLKTRCTRL_DSS_MASK			(0x3 << 0)
-#define OMAP3430_EN_CSI2_SHIFT				1
-#define OMAP3430_CLKSEL_CAM_SHIFT			0
-#define OMAP3630_CLKSEL_CAM_WIDTH			6
 #define OMAP3430_CLKTRCTRL_CAM_MASK			(0x3 << 0)
 #define OMAP3430_ST_MCBSP4_SHIFT			2
 #define OMAP3430_ST_MCBSP3_SHIFT			1
 #define OMAP3430_ST_MCBSP2_SHIFT			0
-#define OMAP3430_CLKSEL_GPT9_MASK			(1 << 7)
-#define OMAP3430_CLKSEL_GPT8_MASK			(1 << 6)
-#define OMAP3430_CLKSEL_GPT7_MASK			(1 << 5)
-#define OMAP3430_CLKSEL_GPT6_MASK			(1 << 4)
-#define OMAP3430_CLKSEL_GPT5_MASK			(1 << 3)
-#define OMAP3430_CLKSEL_GPT4_MASK			(1 << 2)
-#define OMAP3430_CLKSEL_GPT3_MASK			(1 << 1)
-#define OMAP3430_CLKSEL_GPT2_MASK			(1 << 0)
 #define OMAP3430_CLKTRCTRL_PER_MASK			(0x3 << 0)
-#define OMAP3430_DIV_DPLL4_SHIFT			24
-#define OMAP3630_DIV_DPLL4_WIDTH			6
-#define OMAP3430_DIV_DPLL3_SHIFT			16
-#define OMAP3430_DIV_DPLL3_WIDTH			5
-#define OMAP3430_CLKSEL_TRACECLK_SHIFT			11
-#define OMAP3430_CLKSEL_TRACECLK_WIDTH			3
-#define OMAP3430_CLKSEL_PCLK_SHIFT			8
-#define OMAP3430_CLKSEL_PCLK_WIDTH			3
-#define OMAP3430_CLKSEL_PCLKX2_SHIFT			6
-#define OMAP3430_CLKSEL_PCLKX2_WIDTH			2
-#define OMAP3430_CLKSEL_ATCLK_SHIFT			4
-#define OMAP3430_CLKSEL_ATCLK_WIDTH			2
-#define OMAP3430_TRACE_MUX_CTRL_SHIFT			2
-#define OMAP3430_TRACE_MUX_CTRL_WIDTH			2
-#define OMAP3430_MUX_CTRL_MASK				(0x3 << 0)
 #define OMAP3430_CLKTRCTRL_EMU_MASK			(0x3 << 0)
 #define OMAP3430_CLKTRCTRL_NEON_MASK			(0x3 << 0)
 #define OMAP3430ES2_EN_USBHOST2_SHIFT			1
-#define OMAP3430ES2_EN_USBHOST1_SHIFT			0
-#define OMAP3430ES2_EN_USBHOST_SHIFT			0
 #define OMAP3430ES2_ST_USBHOST_IDLE_SHIFT		1
-#define OMAP3430ES2_ST_USBHOST_STDBY_SHIFT		0
 #define OMAP3430ES2_CLKTRCTRL_USBHOST_MASK		(3 << 0)
 #define OMAP34XX_CLKSTCTRL_DISABLE_AUTO		0x0
 #define OMAP34XX_CLKSTCTRL_FORCE_SLEEP		0x1
diff --git a/arch/arm/mach-omap2/cm.h b/arch/arm/mach-omap2/cm.h
index e833984cc85e..b19e83d53501 100644
--- a/arch/arm/mach-omap2/cm.h
+++ b/arch/arm/mach-omap2/cm.h
@@ -52,6 +52,7 @@ extern void omap2_set_globals_cm(void __iomem *cm, void __iomem *cm2);
  * @wait_module_idle: ptr to the SoC CM-specific wait_module_idle impl
  * @module_enable: ptr to the SoC CM-specific module_enable impl
  * @module_disable: ptr to the SoC CM-specific module_disable impl
+ * @xlate_clkctrl: ptr to the SoC CM-specific clkctrl xlate addr impl
  */
 struct cm_ll_data {
 	int (*split_idlest_reg)(struct clk_omap_reg *idlest_reg, s16 *prcm_inst,
@@ -62,6 +63,7 @@ struct cm_ll_data {
 				u8 idlest_shift);
 	void (*module_enable)(u8 mode, u8 part, u16 inst, u16 clkctrl_offs);
 	void (*module_disable)(u8 part, u16 inst, u16 clkctrl_offs);
+	u32 (*xlate_clkctrl)(u8 part, u16 inst, u16 clkctrl_offs);
 };
 
 extern int cm_split_idlest_reg(struct clk_omap_reg *idlest_reg, s16 *prcm_inst,
@@ -72,8 +74,9 @@ int omap_cm_wait_module_idle(u8 part, s16 prcm_mod, u16 idlest_reg,
 			     u8 idlest_shift);
 int omap_cm_module_enable(u8 mode, u8 part, u16 inst, u16 clkctrl_offs);
 int omap_cm_module_disable(u8 part, u16 inst, u16 clkctrl_offs);
-extern int cm_register(struct cm_ll_data *cld);
-extern int cm_unregister(struct cm_ll_data *cld);
+u32 omap_cm_xlate_clkctrl(u8 part, u16 inst, u16 clkctrl_offs);
+extern int cm_register(const struct cm_ll_data *cld);
+extern int cm_unregister(const struct cm_ll_data *cld);
 int omap_cm_init(void);
 int omap2_cm_base_init(void);
 
diff --git a/arch/arm/mach-omap2/cm2xxx.c b/arch/arm/mach-omap2/cm2xxx.c
index cd90b4c6a06b..d5b87f42a96e 100644
--- a/arch/arm/mach-omap2/cm2xxx.c
+++ b/arch/arm/mach-omap2/cm2xxx.c
@@ -385,7 +385,7 @@ void omap2xxx_cm_set_mod_dividers(u32 mpu, u32 dsp, u32 gfx, u32 core, u32 mdm)
  *
  */
 
-static struct cm_ll_data omap2xxx_cm_ll_data = {
+static const struct cm_ll_data omap2xxx_cm_ll_data = {
 	.split_idlest_reg	= &omap2xxx_cm_split_idlest_reg,
 	.wait_module_ready	= &omap2xxx_cm_wait_module_ready,
 };
diff --git a/arch/arm/mach-omap2/cm33xx.c b/arch/arm/mach-omap2/cm33xx.c
index a9e08d89104e..1cc0247a2cb5 100644
--- a/arch/arm/mach-omap2/cm33xx.c
+++ b/arch/arm/mach-omap2/cm33xx.c
@@ -333,6 +333,11 @@ static int am33xx_clkdm_clk_disable(struct clockdomain *clkdm)
 	return 0;
 }
 
+static u32 am33xx_cm_xlate_clkctrl(u8 part, u16 inst, u16 offset)
+{
+	return cm_base.pa + inst + offset;
+}
+
 struct clkdm_ops am33xx_clkdm_operations = {
 	.clkdm_sleep		= am33xx_clkdm_sleep,
 	.clkdm_wakeup		= am33xx_clkdm_wakeup,
@@ -342,11 +347,12 @@ struct clkdm_ops am33xx_clkdm_operations = {
 	.clkdm_clk_disable	= am33xx_clkdm_clk_disable,
 };
 
-static struct cm_ll_data am33xx_cm_ll_data = {
+static const struct cm_ll_data am33xx_cm_ll_data = {
 	.wait_module_ready	= &am33xx_cm_wait_module_ready,
 	.wait_module_idle	= &am33xx_cm_wait_module_idle,
 	.module_enable		= &am33xx_cm_module_enable,
 	.module_disable		= &am33xx_cm_module_disable,
+	.xlate_clkctrl		= &am33xx_cm_xlate_clkctrl,
 };
 
 int __init am33xx_cm_init(const struct omap_prcm_init_data *data)
diff --git a/arch/arm/mach-omap2/cm3xxx.c b/arch/arm/mach-omap2/cm3xxx.c
index 961bc478b9de..ec580fd094a6 100644
--- a/arch/arm/mach-omap2/cm3xxx.c
+++ b/arch/arm/mach-omap2/cm3xxx.c
@@ -662,7 +662,7 @@ void omap3_cm_save_scratchpad_contents(u32 *ptr)
  *
  */
 
-static struct cm_ll_data omap3xxx_cm_ll_data = {
+static const struct cm_ll_data omap3xxx_cm_ll_data = {
 	.split_idlest_reg	= &omap3xxx_cm_split_idlest_reg,
 	.wait_module_ready	= &omap3xxx_cm_wait_module_ready,
 };
diff --git a/arch/arm/mach-omap2/cm_common.c b/arch/arm/mach-omap2/cm_common.c
index 83c6fa74cc31..aff747ecad51 100644
--- a/arch/arm/mach-omap2/cm_common.c
+++ b/arch/arm/mach-omap2/cm_common.c
@@ -29,7 +29,7 @@
  * common CM functions
  */
 static struct cm_ll_data null_cm_ll_data;
-static struct cm_ll_data *cm_ll_data = &null_cm_ll_data;
+static const struct cm_ll_data *cm_ll_data = &null_cm_ll_data;
 
 /* cm_base: base virtual address of the CM IP block */
 struct omap_domain_base cm_base;
@@ -178,6 +178,16 @@ int omap_cm_module_disable(u8 part, u16 inst, u16 clkctrl_offs)
 	return 0;
 }
 
+u32 omap_cm_xlate_clkctrl(u8 part, u16 inst, u16 clkctrl_offs)
+{
+	if (!cm_ll_data->xlate_clkctrl) {
+		WARN_ONCE(1, "cm: %s: no low-level function defined\n",
+			  __func__);
+		return 0;
+	}
+	return cm_ll_data->xlate_clkctrl(part, inst, clkctrl_offs);
+}
+
 /**
  * cm_register - register per-SoC low-level data with the CM
  * @cld: low-level per-SoC OMAP CM data & function pointers to register
@@ -189,7 +199,7 @@ int omap_cm_module_disable(u8 part, u16 inst, u16 clkctrl_offs)
  * is NULL, or -EEXIST if cm_register() has already been called
  * without an intervening cm_unregister().
  */
-int cm_register(struct cm_ll_data *cld)
+int cm_register(const struct cm_ll_data *cld)
 {
 	if (!cld)
 		return -EINVAL;
@@ -213,7 +223,7 @@ int cm_register(struct cm_ll_data *cld)
  * -EINVAL if @cld is NULL or if @cld does not match the struct
  * cm_ll_data * previously registered by cm_register().
  */
-int cm_unregister(struct cm_ll_data *cld)
+int cm_unregister(const struct cm_ll_data *cld)
 {
 	if (!cld || cm_ll_data != cld)
 		return -EINVAL;
diff --git a/arch/arm/mach-omap2/cminst44xx.c b/arch/arm/mach-omap2/cminst44xx.c
index 8774e983bea1..7deefee49fc3 100644
--- a/arch/arm/mach-omap2/cminst44xx.c
+++ b/arch/arm/mach-omap2/cminst44xx.c
@@ -476,12 +476,9 @@ static int omap4_clkdm_clk_disable(struct clockdomain *clkdm)
 	return 0;
 }
 
-static u32 omap4_clkdm_xlate_address(struct clockdomain *clkdm)
+static u32 omap4_cminst_xlate_clkctrl(u8 part, u16 inst, u16 offset)
 {
-	u32 addr = _cm_bases[clkdm->prcm_partition].pa + clkdm->cm_inst +
-		clkdm->clkdm_offs;
-
-	return addr;
+	return _cm_bases[part].pa + inst + offset;
 }
 
 struct clkdm_ops omap4_clkdm_operations = {
@@ -499,7 +496,6 @@ struct clkdm_ops omap4_clkdm_operations = {
 	.clkdm_deny_idle	= omap4_clkdm_deny_idle,
 	.clkdm_clk_enable	= omap4_clkdm_clk_enable,
 	.clkdm_clk_disable	= omap4_clkdm_clk_disable,
-	.clkdm_xlate_address	= omap4_clkdm_xlate_address,
 };
 
 struct clkdm_ops am43xx_clkdm_operations = {
@@ -509,14 +505,14 @@ struct clkdm_ops am43xx_clkdm_operations = {
 	.clkdm_deny_idle	= omap4_clkdm_deny_idle,
 	.clkdm_clk_enable	= omap4_clkdm_clk_enable,
 	.clkdm_clk_disable	= omap4_clkdm_clk_disable,
-	.clkdm_xlate_address	= omap4_clkdm_xlate_address,
 };
 
-static struct cm_ll_data omap4xxx_cm_ll_data = {
+static const struct cm_ll_data omap4xxx_cm_ll_data = {
 	.wait_module_ready	= &omap4_cminst_wait_module_ready,
 	.wait_module_idle	= &omap4_cminst_wait_module_idle,
 	.module_enable		= &omap4_cminst_module_enable,
 	.module_disable		= &omap4_cminst_module_disable,
+	.xlate_clkctrl		= &omap4_cminst_xlate_clkctrl,
 };
 
 int __init omap4_cm_init(const struct omap_prcm_init_data *data)
diff --git a/arch/arm/mach-omap2/gpmc-onenand.c b/arch/arm/mach-omap2/gpmc-onenand.c
deleted file mode 100644
index 2944af820558..000000000000
--- a/arch/arm/mach-omap2/gpmc-onenand.c
+++ /dev/null
@@ -1,409 +0,0 @@
-/*
- * linux/arch/arm/mach-omap2/gpmc-onenand.c
- *
- * Copyright (C) 2006 - 2009 Nokia Corporation
- * Contacts:	Juha Yrjola
- *		Tony Lindgren
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/string.h>
-#include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/mtd/onenand_regs.h>
-#include <linux/io.h>
-#include <linux/omap-gpmc.h>
-#include <linux/platform_data/mtd-onenand-omap2.h>
-#include <linux/err.h>
-
-#include <asm/mach/flash.h>
-
-#include "soc.h"
-
-#define	ONENAND_IO_SIZE	SZ_128K
-
-#define	ONENAND_FLAG_SYNCREAD	(1 << 0)
-#define	ONENAND_FLAG_SYNCWRITE	(1 << 1)
-#define	ONENAND_FLAG_HF		(1 << 2)
-#define	ONENAND_FLAG_VHF	(1 << 3)
-
-static unsigned onenand_flags;
-static unsigned latency;
-
-static struct omap_onenand_platform_data *gpmc_onenand_data;
-
-static struct resource gpmc_onenand_resource = {
-	.flags		= IORESOURCE_MEM,
-};
-
-static struct platform_device gpmc_onenand_device = {
-	.name		= "omap2-onenand",
-	.id		= -1,
-	.num_resources	= 1,
-	.resource	= &gpmc_onenand_resource,
-};
-
-static struct gpmc_settings onenand_async = {
-	.device_width	= GPMC_DEVWIDTH_16BIT,
-	.mux_add_data	= GPMC_MUX_AD,
-};
-
-static struct gpmc_settings onenand_sync = {
-	.burst_read	= true,
-	.burst_wrap	= true,
-	.burst_len	= GPMC_BURST_16,
-	.device_width	= GPMC_DEVWIDTH_16BIT,
-	.mux_add_data	= GPMC_MUX_AD,
-	.wait_pin	= 0,
-};
-
-static void omap2_onenand_calc_async_timings(struct gpmc_timings *t)
-{
-	struct gpmc_device_timings dev_t;
-	const int t_cer = 15;
-	const int t_avdp = 12;
-	const int t_aavdh = 7;
-	const int t_ce = 76;
-	const int t_aa = 76;
-	const int t_oe = 20;
-	const int t_cez = 20; /* max of t_cez, t_oez */
-	const int t_wpl = 40;
-	const int t_wph = 30;
-
-	memset(&dev_t, 0, sizeof(dev_t));
-
-	dev_t.t_avdp_r = max_t(int, t_avdp, t_cer) * 1000;
-	dev_t.t_avdp_w = dev_t.t_avdp_r;
-	dev_t.t_aavdh = t_aavdh * 1000;
-	dev_t.t_aa = t_aa * 1000;
-	dev_t.t_ce = t_ce * 1000;
-	dev_t.t_oe = t_oe * 1000;
-	dev_t.t_cez_r = t_cez * 1000;
-	dev_t.t_cez_w = dev_t.t_cez_r;
-	dev_t.t_wpl = t_wpl * 1000;
-	dev_t.t_wph = t_wph * 1000;
-
-	gpmc_calc_timings(t, &onenand_async, &dev_t);
-}
-
-static void omap2_onenand_set_async_mode(void __iomem *onenand_base)
-{
-	u32 reg;
-
-	/* Ensure sync read and sync write are disabled */
-	reg = readw(onenand_base + ONENAND_REG_SYS_CFG1);
-	reg &= ~ONENAND_SYS_CFG1_SYNC_READ & ~ONENAND_SYS_CFG1_SYNC_WRITE;
-	writew(reg, onenand_base + ONENAND_REG_SYS_CFG1);
-}
-
-static void set_onenand_cfg(void __iomem *onenand_base)
-{
-	u32 reg = ONENAND_SYS_CFG1_RDY | ONENAND_SYS_CFG1_INT;
-
-	reg |=	(latency << ONENAND_SYS_CFG1_BRL_SHIFT) |
-		ONENAND_SYS_CFG1_BL_16;
-	if (onenand_flags & ONENAND_FLAG_SYNCREAD)
-		reg |= ONENAND_SYS_CFG1_SYNC_READ;
-	else
-		reg &= ~ONENAND_SYS_CFG1_SYNC_READ;
-	if (onenand_flags & ONENAND_FLAG_SYNCWRITE)
-		reg |= ONENAND_SYS_CFG1_SYNC_WRITE;
-	else
-		reg &= ~ONENAND_SYS_CFG1_SYNC_WRITE;
-	if (onenand_flags & ONENAND_FLAG_HF)
-		reg |= ONENAND_SYS_CFG1_HF;
-	else
-		reg &= ~ONENAND_SYS_CFG1_HF;
-	if (onenand_flags & ONENAND_FLAG_VHF)
-		reg |= ONENAND_SYS_CFG1_VHF;
-	else
-		reg &= ~ONENAND_SYS_CFG1_VHF;
-
-	writew(reg, onenand_base + ONENAND_REG_SYS_CFG1);
-}
-
-static int omap2_onenand_get_freq(struct omap_onenand_platform_data *cfg,
-				  void __iomem *onenand_base)
-{
-	u16 ver = readw(onenand_base + ONENAND_REG_VERSION_ID);
-	int freq;
-
-	switch ((ver >> 4) & 0xf) {
-	case 0:
-		freq = 40;
-		break;
-	case 1:
-		freq = 54;
-		break;
-	case 2:
-		freq = 66;
-		break;
-	case 3:
-		freq = 83;
-		break;
-	case 4:
-		freq = 104;
-		break;
-	default:
-		pr_err("onenand rate not detected, bad GPMC async timings?\n");
-		freq = 0;
-	}
-
-	return freq;
-}
-
-static void omap2_onenand_calc_sync_timings(struct gpmc_timings *t,
-					    unsigned int flags,
-					    int freq)
-{
-	struct gpmc_device_timings dev_t;
-	const int t_cer  = 15;
-	const int t_avdp = 12;
-	const int t_cez  = 20; /* max of t_cez, t_oez */
-	const int t_wpl  = 40;
-	const int t_wph  = 30;
-	int min_gpmc_clk_period, t_ces, t_avds, t_avdh, t_ach, t_aavdh, t_rdyo;
-	int div, gpmc_clk_ns;
-
-	if (flags & ONENAND_SYNC_READ)
-		onenand_flags = ONENAND_FLAG_SYNCREAD;
-	else if (flags & ONENAND_SYNC_READWRITE)
-		onenand_flags = ONENAND_FLAG_SYNCREAD | ONENAND_FLAG_SYNCWRITE;
-
-	switch (freq) {
-	case 104:
-		min_gpmc_clk_period = 9600; /* 104 MHz */
-		t_ces   = 3;
-		t_avds  = 4;
-		t_avdh  = 2;
-		t_ach   = 3;
-		t_aavdh = 6;
-		t_rdyo  = 6;
-		break;
-	case 83:
-		min_gpmc_clk_period = 12000; /* 83 MHz */
-		t_ces   = 5;
-		t_avds  = 4;
-		t_avdh  = 2;
-		t_ach   = 6;
-		t_aavdh = 6;
-		t_rdyo  = 9;
-		break;
-	case 66:
-		min_gpmc_clk_period = 15000; /* 66 MHz */
-		t_ces   = 6;
-		t_avds  = 5;
-		t_avdh  = 2;
-		t_ach   = 6;
-		t_aavdh = 6;
-		t_rdyo  = 11;
-		break;
-	default:
-		min_gpmc_clk_period = 18500; /* 54 MHz */
-		t_ces   = 7;
-		t_avds  = 7;
-		t_avdh  = 7;
-		t_ach   = 9;
-		t_aavdh = 7;
-		t_rdyo  = 15;
-		onenand_flags &= ~ONENAND_FLAG_SYNCWRITE;
-		break;
-	}
-
-	div = gpmc_calc_divider(min_gpmc_clk_period);
-	gpmc_clk_ns = gpmc_ticks_to_ns(div);
-	if (gpmc_clk_ns < 15) /* >66MHz */
-		onenand_flags |= ONENAND_FLAG_HF;
-	else
-		onenand_flags &= ~ONENAND_FLAG_HF;
-	if (gpmc_clk_ns < 12) /* >83MHz */
-		onenand_flags |= ONENAND_FLAG_VHF;
-	else
-		onenand_flags &= ~ONENAND_FLAG_VHF;
-	if (onenand_flags & ONENAND_FLAG_VHF)
-		latency = 8;
-	else if (onenand_flags & ONENAND_FLAG_HF)
-		latency = 6;
-	else if (gpmc_clk_ns >= 25) /* 40 MHz*/
-		latency = 3;
-	else
-		latency = 4;
-
-	/* Set synchronous read timings */
-	memset(&dev_t, 0, sizeof(dev_t));
-
-	if (onenand_flags & ONENAND_FLAG_SYNCREAD)
-		onenand_sync.sync_read = true;
-	if (onenand_flags & ONENAND_FLAG_SYNCWRITE) {
-		onenand_sync.sync_write = true;
-		onenand_sync.burst_write = true;
-	} else {
-		dev_t.t_avdp_w = max(t_avdp, t_cer) * 1000;
-		dev_t.t_wpl = t_wpl * 1000;
-		dev_t.t_wph = t_wph * 1000;
-		dev_t.t_aavdh = t_aavdh * 1000;
-	}
-	dev_t.ce_xdelay = true;
-	dev_t.avd_xdelay = true;
-	dev_t.oe_xdelay = true;
-	dev_t.we_xdelay = true;
-	dev_t.clk = min_gpmc_clk_period;
-	dev_t.t_bacc = dev_t.clk;
-	dev_t.t_ces = t_ces * 1000;
-	dev_t.t_avds = t_avds * 1000;
-	dev_t.t_avdh = t_avdh * 1000;
-	dev_t.t_ach = t_ach * 1000;
-	dev_t.cyc_iaa = (latency + 1);
-	dev_t.t_cez_r = t_cez * 1000;
-	dev_t.t_cez_w = dev_t.t_cez_r;
-	dev_t.cyc_aavdh_oe = 1;
-	dev_t.t_rdyo = t_rdyo * 1000 + min_gpmc_clk_period;
-
-	gpmc_calc_timings(t, &onenand_sync, &dev_t);
-}
-
-static int omap2_onenand_setup_async(void __iomem *onenand_base)
-{
-	struct gpmc_timings t;
-	int ret;
-
-	/*
-	 * Note that we need to keep sync_write set for the call to
-	 * omap2_onenand_set_async_mode() to work to detect the onenand
-	 * supported clock rate for the sync timings.
-	 */
-	if (gpmc_onenand_data->of_node) {
-		gpmc_read_settings_dt(gpmc_onenand_data->of_node,
-				      &onenand_async);
-		if (onenand_async.sync_read || onenand_async.sync_write) {
-			if (onenand_async.sync_write)
-				gpmc_onenand_data->flags |=
-					ONENAND_SYNC_READWRITE;
-			else
-				gpmc_onenand_data->flags |= ONENAND_SYNC_READ;
-			onenand_async.sync_read = false;
-		}
-	}
-
-	onenand_async.sync_write = true;
-	omap2_onenand_calc_async_timings(&t);
-
-	ret = gpmc_cs_program_settings(gpmc_onenand_data->cs, &onenand_async);
-	if (ret < 0)
-		return ret;
-
-	ret = gpmc_cs_set_timings(gpmc_onenand_data->cs, &t, &onenand_async);
-	if (ret < 0)
-		return ret;
-
-	omap2_onenand_set_async_mode(onenand_base);
-
-	return 0;
-}
-
-static int omap2_onenand_setup_sync(void __iomem *onenand_base, int *freq_ptr)
-{
-	int ret, freq = *freq_ptr;
-	struct gpmc_timings t;
-
-	if (!freq) {
-		/* Very first call freq is not known */
-		freq = omap2_onenand_get_freq(gpmc_onenand_data, onenand_base);
-		if (!freq)
-			return -ENODEV;
-		set_onenand_cfg(onenand_base);
-	}
-
-	if (gpmc_onenand_data->of_node) {
-		gpmc_read_settings_dt(gpmc_onenand_data->of_node,
-				      &onenand_sync);
-	} else {
-		/*
-		 * FIXME: Appears to be legacy code from initial ONENAND commit.
-		 * Unclear what boards this is for and if this can be removed.
-		 */
-		if (!cpu_is_omap34xx())
-			onenand_sync.wait_on_read = true;
-	}
-
-	omap2_onenand_calc_sync_timings(&t, gpmc_onenand_data->flags, freq);
-
-	ret = gpmc_cs_program_settings(gpmc_onenand_data->cs, &onenand_sync);
-	if (ret < 0)
-		return ret;
-
-	ret = gpmc_cs_set_timings(gpmc_onenand_data->cs, &t, &onenand_sync);
-	if (ret < 0)
-		return ret;
-
-	set_onenand_cfg(onenand_base);
-
-	*freq_ptr = freq;
-
-	return 0;
-}
-
-static int gpmc_onenand_setup(void __iomem *onenand_base, int *freq_ptr)
-{
-	struct device *dev = &gpmc_onenand_device.dev;
-	unsigned l = ONENAND_SYNC_READ | ONENAND_SYNC_READWRITE;
-	int ret;
-
-	ret = omap2_onenand_setup_async(onenand_base);
-	if (ret) {
-		dev_err(dev, "unable to set to async mode\n");
-		return ret;
-	}
-
-	if (!(gpmc_onenand_data->flags & l))
-		return 0;
-
-	ret = omap2_onenand_setup_sync(onenand_base, freq_ptr);
-	if (ret)
-		dev_err(dev, "unable to set to sync mode\n");
-	return ret;
-}
-
-int gpmc_onenand_init(struct omap_onenand_platform_data *_onenand_data)
-{
-	int err;
-	struct device *dev = &gpmc_onenand_device.dev;
-
-	gpmc_onenand_data = _onenand_data;
-	gpmc_onenand_data->onenand_setup = gpmc_onenand_setup;
-	gpmc_onenand_device.dev.platform_data = gpmc_onenand_data;
-
-	if (cpu_is_omap24xx() &&
-			(gpmc_onenand_data->flags & ONENAND_SYNC_READWRITE)) {
-		dev_warn(dev, "OneNAND using only SYNC_READ on 24xx\n");
-		gpmc_onenand_data->flags &= ~ONENAND_SYNC_READWRITE;
-		gpmc_onenand_data->flags |= ONENAND_SYNC_READ;
-	}
-
-	if (cpu_is_omap34xx())
-		gpmc_onenand_data->flags |= ONENAND_IN_OMAP34XX;
-	else
-		gpmc_onenand_data->flags &= ~ONENAND_IN_OMAP34XX;
-
-	err = gpmc_cs_request(gpmc_onenand_data->cs, ONENAND_IO_SIZE,
-				(unsigned long *)&gpmc_onenand_resource.start);
-	if (err < 0) {
-		dev_err(dev, "Cannot request GPMC CS %d, error %d\n",
-			gpmc_onenand_data->cs, err);
-		return err;
-	}
-
-	gpmc_onenand_resource.end = gpmc_onenand_resource.start +
-							ONENAND_IO_SIZE - 1;
-
-	err = platform_device_register(&gpmc_onenand_device);
-	if (err) {
-		dev_err(dev, "Unable to register OneNAND device\n");
-		gpmc_cs_free(gpmc_onenand_data->cs);
-	}
-
-	return err;
-}
diff --git a/arch/arm/mach-omap2/id.c b/arch/arm/mach-omap2/id.c
index df2c29edbbcd..68ba5f472f6b 100644
--- a/arch/arm/mach-omap2/id.c
+++ b/arch/arm/mach-omap2/id.c
@@ -657,8 +657,11 @@ void __init dra7xxx_check_revision(void)
 {
 	u32 idcode;
 	u16 hawkeye;
-	u8 rev;
+	u8 rev, package;
+	struct omap_die_id odi;
 
+	omap_get_die_id(&odi);
+	package = (odi.id_2 >> 16) & 0x3;
 	idcode = read_tap_reg(OMAP_TAP_IDCODE);
 	hawkeye = (idcode >> 12) & 0xffff;
 	rev = (idcode >> 28) & 0xff;
@@ -667,7 +670,17 @@ void __init dra7xxx_check_revision(void)
 		switch (rev) {
 		case 0:
 		default:
-			omap_revision = DRA762_REV_ES1_0;
+			switch (package) {
+			case 0x2:
+				omap_revision = DRA762_ABZ_REV_ES1_0;
+				break;
+			case 0x3:
+				omap_revision = DRA762_ACD_REV_ES1_0;
+				break;
+			default:
+				omap_revision = DRA762_REV_ES1_0;
+				break;
+			}
 			break;
 		}
 		break;
diff --git a/arch/arm/mach-omap2/omap-wakeupgen.c b/arch/arm/mach-omap2/omap-wakeupgen.c
index 4bb6751864a5..fc5fb776a710 100644
--- a/arch/arm/mach-omap2/omap-wakeupgen.c
+++ b/arch/arm/mach-omap2/omap-wakeupgen.c
@@ -299,8 +299,6 @@ static void irq_save_context(void)
 	if (soc_is_dra7xx())
 		return;
 
-	if (!sar_base)
-		sar_base = omap4_get_sar_ram_base();
 	if (wakeupgen_ops && wakeupgen_ops->save_context)
 		wakeupgen_ops->save_context();
 }
@@ -598,6 +596,8 @@ static int __init wakeupgen_init(struct device_node *node,
 	irq_hotplug_init();
 	irq_pm_init();
 
+	sar_base = omap4_get_sar_ram_base();
+
 	return 0;
 }
 IRQCHIP_DECLARE(ti_wakeupgen, "ti,omap4-wugen-mpu", wakeupgen_init);
diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c
index 104256a5f0f7..34156eca8e23 100644
--- a/arch/arm/mach-omap2/omap_hwmod.c
+++ b/arch/arm/mach-omap2/omap_hwmod.c
@@ -143,6 +143,8 @@
 #include <linux/of_address.h>
 #include <linux/bootmem.h>
 
+#include <linux/platform_data/ti-sysc.h>
+
 #include <asm/system_misc.h>
 
 #include "clock.h"
@@ -185,15 +187,15 @@
 /**
  * struct clkctrl_provider - clkctrl provider mapping data
  * @addr: base address for the provider
- * @offset: base offset for the provider
- * @clkdm: base clockdomain for provider
+ * @size: size of the provider address space
+ * @offset: offset of the provider from PRCM instance base
  * @node: device node associated with the provider
  * @link: list link
  */
 struct clkctrl_provider {
 	u32			addr;
+	u32			size;
 	u16			offset;
-	struct clockdomain	*clkdm;
 	struct device_node	*node;
 	struct list_head	link;
 };
@@ -223,8 +225,7 @@ struct omap_hwmod_soc_ops {
 	void (*update_context_lost)(struct omap_hwmod *oh);
 	int (*get_context_lost)(struct omap_hwmod *oh);
 	int (*disable_direct_prcm)(struct omap_hwmod *oh);
-	u32 (*xlate_clkctrl)(struct omap_hwmod *oh,
-			     struct clkctrl_provider *provider);
+	u32 (*xlate_clkctrl)(struct omap_hwmod *oh);
 };
 
 /* soc_ops: adapts the omap_hwmod code to the currently-booted SoC */
@@ -716,52 +717,35 @@ static const struct of_device_id ti_clkctrl_match_table[] __initconst = {
 	{ }
 };
 
-static int _match_clkdm(struct clockdomain *clkdm, void *user)
-{
-	struct clkctrl_provider *provider = user;
-
-	if (clkdm_xlate_address(clkdm) == provider->addr) {
-		pr_debug("%s: Matched clkdm %s for addr %x (%s)\n", __func__,
-			 clkdm->name, provider->addr,
-			 provider->node->parent->name);
-		provider->clkdm = clkdm;
-
-		return -1;
-	}
-
-	return 0;
-}
-
-static int _setup_clkctrl_provider(struct device_node *np)
+static int __init _setup_clkctrl_provider(struct device_node *np)
 {
 	const __be32 *addrp;
 	struct clkctrl_provider *provider;
+	u64 size;
 
 	provider = memblock_virt_alloc(sizeof(*provider), 0);
 	if (!provider)
 		return -ENOMEM;
 
-	addrp = of_get_address(np, 0, NULL, NULL);
+	addrp = of_get_address(np, 0, &size, NULL);
 	provider->addr = (u32)of_translate_address(np, addrp);
-	provider->offset = provider->addr & 0xff;
+	addrp = of_get_address(np->parent, 0, NULL, NULL);
+	provider->offset = provider->addr -
+			   (u32)of_translate_address(np->parent, addrp);
 	provider->addr &= ~0xff;
+	provider->size = size | 0xff;
 	provider->node = np;
 
-	clkdm_for_each(_match_clkdm, provider);
-
-	if (!provider->clkdm) {
-		pr_err("%s: nothing matched for node %s (%x)\n",
-		       __func__, np->parent->name, provider->addr);
-		memblock_free_early(__pa(provider), sizeof(*provider));
-		return -EINVAL;
-	}
+	pr_debug("%s: %s: %x...%x [+%x]\n", __func__, np->parent->name,
+		 provider->addr, provider->addr + provider->size,
+		 provider->offset);
 
 	list_add(&provider->link, &clkctrl_providers);
 
 	return 0;
 }
 
-static int _init_clkctrl_providers(void)
+static int __init _init_clkctrl_providers(void)
 {
 	struct device_node *np;
 	int ret = 0;
@@ -775,32 +759,48 @@ static int _init_clkctrl_providers(void)
 	return ret;
 }
 
-static u32 _omap4_xlate_clkctrl(struct omap_hwmod *oh,
-				struct clkctrl_provider *provider)
+static u32 _omap4_xlate_clkctrl(struct omap_hwmod *oh)
 {
-	return oh->prcm.omap4.clkctrl_offs -
-	       provider->offset - provider->clkdm->clkdm_offs;
+	if (!oh->prcm.omap4.modulemode)
+		return 0;
+
+	return omap_cm_xlate_clkctrl(oh->clkdm->prcm_partition,
+				     oh->clkdm->cm_inst,
+				     oh->prcm.omap4.clkctrl_offs);
 }
 
 static struct clk *_lookup_clkctrl_clk(struct omap_hwmod *oh)
 {
 	struct clkctrl_provider *provider;
 	struct clk *clk;
+	u32 addr;
 
 	if (!soc_ops.xlate_clkctrl)
 		return NULL;
 
+	addr = soc_ops.xlate_clkctrl(oh);
+	if (!addr)
+		return NULL;
+
+	pr_debug("%s: %s: addr=%x\n", __func__, oh->name, addr);
+
 	list_for_each_entry(provider, &clkctrl_providers, link) {
-		if (provider->clkdm == oh->clkdm) {
+		if (provider->addr <= addr &&
+		    provider->addr + provider->size >= addr) {
 			struct of_phandle_args clkspec;
 
 			clkspec.np = provider->node;
 			clkspec.args_count = 2;
-			clkspec.args[0] = soc_ops.xlate_clkctrl(oh, provider);
+			clkspec.args[0] = addr - provider->addr -
+					  provider->offset;
 			clkspec.args[1] = 0;
 
 			clk = of_clk_get_from_provider(&clkspec);
 
+			pr_debug("%s: %s got %p (offset=%x, provider=%s)\n",
+				 __func__, oh->name, clk, clkspec.args[0],
+				 provider->node->parent->name);
+
 			return clk;
 		}
 	}
@@ -977,6 +977,9 @@ static int _enable_clocks(struct omap_hwmod *oh)
 
 	pr_debug("omap_hwmod: %s: enabling clocks\n", oh->name);
 
+	if (oh->flags & HWMOD_OPT_CLKS_NEEDED)
+		_enable_optional_clocks(oh);
+
 	if (oh->_clk)
 		clk_enable(oh->_clk);
 
@@ -985,9 +988,6 @@ static int _enable_clocks(struct omap_hwmod *oh)
 			clk_enable(os->_clk);
 	}
 
-	if (oh->flags & HWMOD_OPT_CLKS_NEEDED)
-		_enable_optional_clocks(oh);
-
 	/* The opt clocks are controlled by the device driver. */
 
 	return 0;
@@ -3521,6 +3521,7 @@ void __init omap_hwmod_init(void)
 		soc_ops.is_hardreset_asserted = _omap4_is_hardreset_asserted;
 		soc_ops.init_clkdm = _init_clkdm;
 		soc_ops.disable_direct_prcm = _omap4_disable_direct_prcm;
+		soc_ops.xlate_clkctrl = _omap4_xlate_clkctrl;
 	} else {
 		WARN(1, "omap_hwmod: unknown SoC type\n");
 	}
diff --git a/arch/arm/mach-omap2/omap_hwmod.h b/arch/arm/mach-omap2/omap_hwmod.h
index df2239a58555..0b8e19f40402 100644
--- a/arch/arm/mach-omap2/omap_hwmod.h
+++ b/arch/arm/mach-omap2/omap_hwmod.h
@@ -37,9 +37,15 @@
 
 struct omap_device;
 
-extern struct omap_hwmod_sysc_fields omap_hwmod_sysc_type1;
-extern struct omap_hwmod_sysc_fields omap_hwmod_sysc_type2;
-extern struct omap_hwmod_sysc_fields omap_hwmod_sysc_type3;
+extern struct sysc_regbits omap_hwmod_sysc_type1;
+extern struct sysc_regbits omap_hwmod_sysc_type2;
+extern struct sysc_regbits omap_hwmod_sysc_type3;
+extern struct sysc_regbits omap34xx_sr_sysc_fields;
+extern struct sysc_regbits omap36xx_sr_sysc_fields;
+extern struct sysc_regbits omap3_sham_sysc_fields;
+extern struct sysc_regbits omap3xxx_aes_sysc_fields;
+extern struct sysc_regbits omap_hwmod_sysc_type_mcasp;
+extern struct sysc_regbits omap_hwmod_sysc_type_usb_host_fs;
 
 /*
  * OCP SYSCONFIG bit shifts/masks TYPE1. These are for IPs compliant
@@ -284,26 +290,6 @@ struct omap_hwmod_ocp_if {
 #define CLOCKACT_TEST_ICLK	0x2
 #define CLOCKACT_TEST_NONE	0x3
 
-/**
- * struct omap_hwmod_sysc_fields - hwmod OCP_SYSCONFIG register field offsets.
- * @midle_shift: Offset of the midle bit
- * @clkact_shift: Offset of the clockactivity bit
- * @sidle_shift: Offset of the sidle bit
- * @enwkup_shift: Offset of the enawakeup bit
- * @srst_shift: Offset of the softreset bit
- * @autoidle_shift: Offset of the autoidle bit
- * @dmadisable_shift: Offset of the dmadisable bit
- */
-struct omap_hwmod_sysc_fields {
-	u8 midle_shift;
-	u8 clkact_shift;
-	u8 sidle_shift;
-	u8 enwkup_shift;
-	u8 srst_shift;
-	u8 autoidle_shift;
-	u8 dmadisable_shift;
-};
-
 /**
  * struct omap_hwmod_class_sysconfig - hwmod class OCP_SYS* data
  * @rev_offs: IP block revision register offset (from module base addr)
@@ -335,7 +321,7 @@ struct omap_hwmod_class_sysconfig {
 	u32 sysc_offs;
 	u32 syss_offs;
 	u16 sysc_flags;
-	struct omap_hwmod_sysc_fields *sysc_fields;
+	struct sysc_regbits *sysc_fields;
 	u8 srst_udelay;
 	u8 idlemodes;
 };
@@ -343,11 +329,8 @@ struct omap_hwmod_class_sysconfig {
 /**
  * struct omap_hwmod_omap2_prcm - OMAP2/3-specific PRCM data
  * @module_offs: PRCM submodule offset from the start of the PRM/CM
- * @prcm_reg_id: PRCM register ID (e.g., 3 for CM_AUTOIDLE3)
- * @module_bit: register bit shift for AUTOIDLE, WKST, WKEN, GRPSEL regs
  * @idlest_reg_id: IDLEST register ID (e.g., 3 for CM_IDLEST3)
  * @idlest_idle_bit: register bit shift for CM_IDLEST slave idle bit
- * @idlest_stdby_bit: register bit shift for CM_IDLEST master standby bit
  *
  * @prcm_reg_id and @module_bit are specific to the AUTOIDLE, WKST,
  * WKEN, GRPSEL registers.  In an ideal world, no extra information
@@ -357,11 +340,8 @@ struct omap_hwmod_class_sysconfig {
  */
 struct omap_hwmod_omap2_prcm {
 	s16 module_offs;
-	u8 prcm_reg_id;
-	u8 module_bit;
 	u8 idlest_reg_id;
 	u8 idlest_idle_bit;
-	u8 idlest_stdby_bit;
 };
 
 /*
diff --git a/arch/arm/mach-omap2/omap_hwmod_2420_data.c b/arch/arm/mach-omap2/omap_hwmod_2420_data.c
index 1a15a347945a..0afb014b211f 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2420_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2420_data.c
@@ -111,8 +111,6 @@ static struct omap_hwmod omap2420_i2c1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2420_EN_I2C1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP2420_ST_I2C1_SHIFT,
 		},
@@ -134,8 +132,6 @@ static struct omap_hwmod omap2420_i2c2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2420_EN_I2C2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP2420_ST_I2C2_SHIFT,
 		},
@@ -167,8 +163,6 @@ static struct omap_hwmod omap2420_mailbox_hwmod = {
 	.main_clk	= "mailboxes_ick",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MAILBOXES_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MAILBOXES_SHIFT,
@@ -197,8 +191,6 @@ static struct omap_hwmod omap2420_mcbsp1_hwmod = {
 	.main_clk	= "mcbsp1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MCBSP1_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MCBSP1_SHIFT,
@@ -215,8 +207,6 @@ static struct omap_hwmod omap2420_mcbsp2_hwmod = {
 	.main_clk	= "mcbsp2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MCBSP2_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MCBSP2_SHIFT,
@@ -247,8 +237,6 @@ static struct omap_hwmod omap2420_msdi1_hwmod = {
 	.main_clk	= "mmc_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2420_EN_MMC_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP2420_ST_MMC_SHIFT,
@@ -264,8 +252,6 @@ static struct omap_hwmod omap2420_hdq1w_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_HDQ_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_HDQ_SHIFT,
 		},
diff --git a/arch/arm/mach-omap2/omap_hwmod_2430_data.c b/arch/arm/mach-omap2/omap_hwmod_2430_data.c
index 3801850bccec..013b26b305d2 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2430_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2430_data.c
@@ -97,8 +97,6 @@ static struct omap_hwmod omap2430_i2c1_hwmod = {
 			 * to hwmod framework.
 			 */
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2430_EN_I2CHS1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP2430_ST_I2CHS1_SHIFT,
 		},
@@ -115,8 +113,6 @@ static struct omap_hwmod omap2430_i2c2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2430_EN_I2CHS2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP2430_ST_I2CHS2_SHIFT,
 		},
@@ -132,8 +128,6 @@ static struct omap_hwmod omap2430_gpio5_hwmod = {
 	.main_clk	= "gpio5_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 2,
-			.module_bit = OMAP2430_EN_GPIO5_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP2430_ST_GPIO5_SHIFT,
@@ -165,8 +159,6 @@ static struct omap_hwmod omap2430_mailbox_hwmod = {
 	.main_clk	= "mailboxes_ick",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MAILBOXES_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MAILBOXES_SHIFT,
@@ -185,8 +177,6 @@ static struct omap_hwmod omap2430_mcspi3_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 2,
-			.module_bit = OMAP2430_EN_MCSPI3_SHIFT,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP2430_ST_MCSPI3_SHIFT,
 		},
@@ -219,8 +209,6 @@ static struct omap_hwmod omap2430_usbhsotg_hwmod = {
 	.main_clk	= "usbhs_ick",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2430_EN_USBHS_MASK,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP2430_ST_USBHS_SHIFT,
@@ -266,8 +254,6 @@ static struct omap_hwmod omap2430_mcbsp1_hwmod = {
 	.main_clk	= "mcbsp1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MCBSP1_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MCBSP1_SHIFT,
@@ -284,8 +270,6 @@ static struct omap_hwmod omap2430_mcbsp2_hwmod = {
 	.main_clk	= "mcbsp2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MCBSP2_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MCBSP2_SHIFT,
@@ -302,8 +286,6 @@ static struct omap_hwmod omap2430_mcbsp3_hwmod = {
 	.main_clk	= "mcbsp3_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2430_EN_MCBSP3_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP2430_ST_MCBSP3_SHIFT,
@@ -320,8 +302,6 @@ static struct omap_hwmod omap2430_mcbsp4_hwmod = {
 	.main_clk	= "mcbsp4_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2430_EN_MCBSP4_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP2430_ST_MCBSP4_SHIFT,
@@ -338,8 +318,6 @@ static struct omap_hwmod omap2430_mcbsp5_hwmod = {
 	.main_clk	= "mcbsp5_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP2430_EN_MCBSP5_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP2430_ST_MCBSP5_SHIFT,
@@ -384,8 +362,6 @@ static struct omap_hwmod omap2430_mmc1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 2,
-			.module_bit  = OMAP2430_EN_MMCHS1_SHIFT,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP2430_ST_MMCHS1_SHIFT,
 		},
@@ -408,8 +384,6 @@ static struct omap_hwmod omap2430_mmc2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 2,
-			.module_bit  = OMAP2430_EN_MMCHS2_SHIFT,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP2430_ST_MMCHS2_SHIFT,
 		},
@@ -424,8 +398,6 @@ static struct omap_hwmod omap2430_hdq1w_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_HDQ_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_HDQ_SHIFT,
 		},
diff --git a/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c b/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c
index beec4cd617b1..4b094cb384cb 100644
--- a/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_2xxx_ipblock_data.c
@@ -242,8 +242,6 @@ struct omap_hwmod omap2xxx_timer1_hwmod = {
 	.main_clk	= "gpt1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT1_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT1_SHIFT,
@@ -261,8 +259,6 @@ struct omap_hwmod omap2xxx_timer2_hwmod = {
 	.main_clk	= "gpt2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT2_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT2_SHIFT,
@@ -279,8 +275,6 @@ struct omap_hwmod omap2xxx_timer3_hwmod = {
 	.main_clk	= "gpt3_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT3_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT3_SHIFT,
@@ -297,8 +291,6 @@ struct omap_hwmod omap2xxx_timer4_hwmod = {
 	.main_clk	= "gpt4_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT4_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT4_SHIFT,
@@ -315,8 +307,6 @@ struct omap_hwmod omap2xxx_timer5_hwmod = {
 	.main_clk	= "gpt5_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT5_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT5_SHIFT,
@@ -334,8 +324,6 @@ struct omap_hwmod omap2xxx_timer6_hwmod = {
 	.main_clk	= "gpt6_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT6_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT6_SHIFT,
@@ -353,8 +341,6 @@ struct omap_hwmod omap2xxx_timer7_hwmod = {
 	.main_clk	= "gpt7_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT7_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT7_SHIFT,
@@ -372,8 +358,6 @@ struct omap_hwmod omap2xxx_timer8_hwmod = {
 	.main_clk	= "gpt8_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT8_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT8_SHIFT,
@@ -391,8 +375,6 @@ struct omap_hwmod omap2xxx_timer9_hwmod = {
 	.main_clk	= "gpt9_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT9_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT9_SHIFT,
@@ -410,8 +392,6 @@ struct omap_hwmod omap2xxx_timer10_hwmod = {
 	.main_clk	= "gpt10_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT10_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT10_SHIFT,
@@ -429,8 +409,6 @@ struct omap_hwmod omap2xxx_timer11_hwmod = {
 	.main_clk	= "gpt11_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT11_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT11_SHIFT,
@@ -448,8 +426,6 @@ struct omap_hwmod omap2xxx_timer12_hwmod = {
 	.main_clk	= "gpt12_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPT12_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPT12_SHIFT,
@@ -467,8 +443,6 @@ struct omap_hwmod omap2xxx_wd_timer2_hwmod = {
 	.main_clk	= "mpu_wdt_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MPU_WDT_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MPU_WDT_SHIFT,
@@ -485,8 +459,6 @@ struct omap_hwmod omap2xxx_uart1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_UART1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_EN_UART1_SHIFT,
 		},
@@ -503,8 +475,6 @@ struct omap_hwmod omap2xxx_uart2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_UART2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_EN_UART2_SHIFT,
 		},
@@ -521,8 +491,6 @@ struct omap_hwmod omap2xxx_uart3_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 2,
-			.module_bit = OMAP24XX_EN_UART3_SHIFT,
 			.idlest_reg_id = 2,
 			.idlest_idle_bit = OMAP24XX_EN_UART3_SHIFT,
 		},
@@ -547,11 +515,8 @@ struct omap_hwmod omap2xxx_dss_core_hwmod = {
 	.main_clk	= "dss1_fck", /* instead of dss_fck */
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_DSS1_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
-			.idlest_stdby_bit = OMAP24XX_ST_DSS_SHIFT,
 		},
 	},
 	.opt_clks	= dss_opt_clks,
@@ -565,11 +530,8 @@ struct omap_hwmod omap2xxx_dss_dispc_hwmod = {
 	.main_clk	= "dss1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_DSS1_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
-			.idlest_stdby_bit = OMAP24XX_ST_DSS_SHIFT,
 		},
 	},
 	.flags		= HWMOD_NO_IDLEST,
@@ -586,8 +548,6 @@ struct omap_hwmod omap2xxx_dss_rfbi_hwmod = {
 	.main_clk	= "dss1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_DSS1_SHIFT,
 			.module_offs = CORE_MOD,
 		},
 	},
@@ -602,8 +562,6 @@ struct omap_hwmod omap2xxx_dss_venc_hwmod = {
 	.main_clk	= "dss_54m_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_DSS1_SHIFT,
 			.module_offs = CORE_MOD,
 		},
 	},
@@ -623,8 +581,6 @@ struct omap_hwmod omap2xxx_gpio1_hwmod = {
 	.main_clk	= "gpios_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPIOS_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPIOS_SHIFT,
@@ -641,8 +597,6 @@ struct omap_hwmod omap2xxx_gpio2_hwmod = {
 	.main_clk	= "gpios_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPIOS_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPIOS_SHIFT,
@@ -659,8 +613,6 @@ struct omap_hwmod omap2xxx_gpio3_hwmod = {
 	.main_clk	= "gpios_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPIOS_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPIOS_SHIFT,
@@ -677,8 +629,6 @@ struct omap_hwmod omap2xxx_gpio4_hwmod = {
 	.main_clk	= "gpios_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_GPIOS_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_GPIOS_SHIFT,
@@ -699,8 +649,6 @@ struct omap_hwmod omap2xxx_mcspi1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MCSPI1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MCSPI1_SHIFT,
 		},
@@ -720,8 +668,6 @@ struct omap_hwmod omap2xxx_mcspi2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_EN_MCSPI2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_MCSPI2_SHIFT,
 		},
@@ -740,8 +686,6 @@ struct omap_hwmod omap2xxx_counter_32k_hwmod = {
 	.prcm		= {
 		.omap2	= {
 			.module_offs = WKUP_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP24XX_ST_32KSYNC_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP24XX_ST_32KSYNC_SHIFT,
 		},
@@ -758,8 +702,6 @@ struct omap_hwmod omap2xxx_gpmc_hwmod = {
 	.flags		= HWMOD_NO_IDLEST | DEBUG_OMAP_GPMC_HWMOD_FLAGS,
 	.prcm		= {
 		.omap2	= {
-			.prcm_reg_id = 3,
-			.module_bit = OMAP24XX_EN_GPMC_MASK,
 			.module_offs = CORE_MOD,
 		},
 	},
@@ -787,8 +729,6 @@ struct omap_hwmod omap2xxx_rng_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 4,
-			.module_bit = OMAP24XX_EN_RNG_SHIFT,
 			.idlest_reg_id = 4,
 			.idlest_idle_bit = OMAP24XX_ST_RNG_SHIFT,
 		},
@@ -825,8 +765,6 @@ struct omap_hwmod omap2xxx_sham_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 4,
-			.module_bit = OMAP24XX_EN_SHA_SHIFT,
 			.idlest_reg_id = 4,
 			.idlest_idle_bit = OMAP24XX_ST_SHA_SHIFT,
 		},
@@ -856,8 +794,6 @@ struct omap_hwmod omap2xxx_aes_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 4,
-			.module_bit = OMAP24XX_EN_AES_SHIFT,
 			.idlest_reg_id = 4,
 			.idlest_idle_bit = OMAP24XX_ST_AES_SHIFT,
 		},
diff --git a/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c b/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
index 52c9d585b44d..1a2f2242e31b 100644
--- a/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_3xxx_data.c
@@ -113,8 +113,6 @@ static struct omap_hwmod omap3xxx_iva_hwmod = {
 	.prcm = {
 		.omap2 = {
 			.module_offs = OMAP3430_IVA2_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_CM_FCLKEN_IVA2_EN_IVA2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_IVA2_SHIFT,
 		},
@@ -188,8 +186,6 @@ static struct omap_hwmod omap3xxx_timer1_hwmod = {
 	.main_clk	= "gpt1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT1_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT1_SHIFT,
@@ -206,8 +202,6 @@ static struct omap_hwmod omap3xxx_timer2_hwmod = {
 	.main_clk	= "gpt2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT2_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT2_SHIFT,
@@ -223,8 +217,6 @@ static struct omap_hwmod omap3xxx_timer3_hwmod = {
 	.main_clk	= "gpt3_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT3_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT3_SHIFT,
@@ -240,8 +232,6 @@ static struct omap_hwmod omap3xxx_timer4_hwmod = {
 	.main_clk	= "gpt4_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT4_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT4_SHIFT,
@@ -257,8 +247,6 @@ static struct omap_hwmod omap3xxx_timer5_hwmod = {
 	.main_clk	= "gpt5_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT5_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT5_SHIFT,
@@ -275,8 +263,6 @@ static struct omap_hwmod omap3xxx_timer6_hwmod = {
 	.main_clk	= "gpt6_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT6_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT6_SHIFT,
@@ -293,8 +279,6 @@ static struct omap_hwmod omap3xxx_timer7_hwmod = {
 	.main_clk	= "gpt7_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT7_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT7_SHIFT,
@@ -311,8 +295,6 @@ static struct omap_hwmod omap3xxx_timer8_hwmod = {
 	.main_clk	= "gpt8_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT8_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT8_SHIFT,
@@ -329,8 +311,6 @@ static struct omap_hwmod omap3xxx_timer9_hwmod = {
 	.main_clk	= "gpt9_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT9_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT9_SHIFT,
@@ -347,8 +327,6 @@ static struct omap_hwmod omap3xxx_timer10_hwmod = {
 	.main_clk	= "gpt10_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT10_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT10_SHIFT,
@@ -365,8 +343,6 @@ static struct omap_hwmod omap3xxx_timer11_hwmod = {
 	.main_clk	= "gpt11_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT11_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT11_SHIFT,
@@ -384,8 +360,6 @@ static struct omap_hwmod omap3xxx_timer12_hwmod = {
 	.main_clk	= "gpt12_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPT12_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPT12_SHIFT,
@@ -439,8 +413,6 @@ static struct omap_hwmod omap3xxx_wd_timer2_hwmod = {
 	.main_clk	= "wdt2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_WDT2_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_WDT2_SHIFT,
@@ -461,8 +433,6 @@ static struct omap_hwmod omap3xxx_uart1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_UART1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_EN_UART1_SHIFT,
 		},
@@ -478,8 +448,6 @@ static struct omap_hwmod omap3xxx_uart2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_UART2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_EN_UART2_SHIFT,
 		},
@@ -496,8 +464,6 @@ static struct omap_hwmod omap3xxx_uart3_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = OMAP3430_PER_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_UART3_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_EN_UART3_SHIFT,
 		},
@@ -515,8 +481,6 @@ static struct omap_hwmod omap36xx_uart4_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = OMAP3430_PER_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3630_EN_UART4_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3630_EN_UART4_SHIFT,
 		},
@@ -546,8 +510,6 @@ static struct omap_hwmod am35xx_uart4_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = AM35XX_EN_UART4_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = AM35XX_ST_UART4_SHIFT,
 		},
@@ -583,11 +545,8 @@ static struct omap_hwmod omap3430es1_dss_core_hwmod = {
 	.main_clk	= "dss1_alwon_fck", /* instead of dss_fck */
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_DSS1_SHIFT,
 			.module_offs = OMAP3430_DSS_MOD,
 			.idlest_reg_id = 1,
-			.idlest_stdby_bit = OMAP3430ES1_ST_DSS_SHIFT,
 		},
 	},
 	.opt_clks	= dss_opt_clks,
@@ -602,12 +561,9 @@ static struct omap_hwmod omap3xxx_dss_core_hwmod = {
 	.main_clk	= "dss1_alwon_fck", /* instead of dss_fck */
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_DSS1_SHIFT,
 			.module_offs = OMAP3430_DSS_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430ES2_ST_DSS_IDLE_SHIFT,
-			.idlest_stdby_bit = OMAP3430ES2_ST_DSS_STDBY_SHIFT,
 		},
 	},
 	.opt_clks	= dss_opt_clks,
@@ -642,8 +598,6 @@ static struct omap_hwmod omap3xxx_dss_dispc_hwmod = {
 	.main_clk	= "dss1_alwon_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_DSS1_SHIFT,
 			.module_offs = OMAP3430_DSS_MOD,
 		},
 	},
@@ -683,8 +637,6 @@ static struct omap_hwmod omap3xxx_dss_dsi1_hwmod = {
 	.main_clk	= "dss1_alwon_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_DSS1_SHIFT,
 			.module_offs = OMAP3430_DSS_MOD,
 		},
 	},
@@ -703,8 +655,6 @@ static struct omap_hwmod omap3xxx_dss_rfbi_hwmod = {
 	.main_clk	= "dss1_alwon_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_DSS1_SHIFT,
 			.module_offs = OMAP3430_DSS_MOD,
 		},
 	},
@@ -724,8 +674,6 @@ static struct omap_hwmod omap3xxx_dss_venc_hwmod = {
 	.main_clk	= "dss_tv_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_DSS1_SHIFT,
 			.module_offs = OMAP3430_DSS_MOD,
 		},
 	},
@@ -747,8 +695,6 @@ static struct omap_hwmod omap3xxx_i2c1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_I2C1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_I2C1_SHIFT,
 		},
@@ -770,8 +716,6 @@ static struct omap_hwmod omap3xxx_i2c2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_I2C2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_I2C2_SHIFT,
 		},
@@ -795,8 +739,6 @@ static struct omap_hwmod omap3xxx_i2c3_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_I2C3_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_I2C3_SHIFT,
 		},
@@ -846,8 +788,6 @@ static struct omap_hwmod omap3xxx_gpio1_hwmod = {
 	.opt_clks_cnt	= ARRAY_SIZE(gpio1_opt_clks),
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPIO1_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPIO1_SHIFT,
@@ -870,8 +810,6 @@ static struct omap_hwmod omap3xxx_gpio2_hwmod = {
 	.opt_clks_cnt	= ARRAY_SIZE(gpio2_opt_clks),
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPIO2_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPIO2_SHIFT,
@@ -894,8 +832,6 @@ static struct omap_hwmod omap3xxx_gpio3_hwmod = {
 	.opt_clks_cnt	= ARRAY_SIZE(gpio3_opt_clks),
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPIO3_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPIO3_SHIFT,
@@ -918,8 +854,6 @@ static struct omap_hwmod omap3xxx_gpio4_hwmod = {
 	.opt_clks_cnt	= ARRAY_SIZE(gpio4_opt_clks),
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPIO4_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPIO4_SHIFT,
@@ -943,8 +877,6 @@ static struct omap_hwmod omap3xxx_gpio5_hwmod = {
 	.opt_clks_cnt	= ARRAY_SIZE(gpio5_opt_clks),
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPIO5_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPIO5_SHIFT,
@@ -968,8 +900,6 @@ static struct omap_hwmod omap3xxx_gpio6_hwmod = {
 	.opt_clks_cnt	= ARRAY_SIZE(gpio6_opt_clks),
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_GPIO6_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_GPIO6_SHIFT,
@@ -1012,8 +942,6 @@ static struct omap_hwmod omap3xxx_dma_system_hwmod = {
 	.prcm = {
 		.omap2 = {
 			.module_offs		= CORE_MOD,
-			.prcm_reg_id		= 1,
-			.module_bit		= OMAP3430_ST_SDMA_SHIFT,
 			.idlest_reg_id		= 1,
 			.idlest_idle_bit	= OMAP3430_ST_SDMA_SHIFT,
 		},
@@ -1060,8 +988,6 @@ static struct omap_hwmod omap3xxx_mcbsp1_hwmod = {
 	.main_clk	= "mcbsp1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCBSP1_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCBSP1_SHIFT,
@@ -1083,8 +1009,6 @@ static struct omap_hwmod omap3xxx_mcbsp2_hwmod = {
 	.main_clk	= "mcbsp2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCBSP2_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCBSP2_SHIFT,
@@ -1107,8 +1031,6 @@ static struct omap_hwmod omap3xxx_mcbsp3_hwmod = {
 	.main_clk	= "mcbsp3_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCBSP3_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCBSP3_SHIFT,
@@ -1128,8 +1050,6 @@ static struct omap_hwmod omap3xxx_mcbsp4_hwmod = {
 	.main_clk	= "mcbsp4_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCBSP4_SHIFT,
 			.module_offs = OMAP3430_PER_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCBSP4_SHIFT,
@@ -1148,8 +1068,6 @@ static struct omap_hwmod omap3xxx_mcbsp5_hwmod = {
 	.main_clk	= "mcbsp5_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCBSP5_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCBSP5_SHIFT,
@@ -1190,10 +1108,6 @@ static struct omap_hwmod omap3xxx_mcbsp3_sidetone_hwmod = {
 };
 
 /* SR common */
-static struct omap_hwmod_sysc_fields omap34xx_sr_sysc_fields = {
-	.clkact_shift	= 20,
-};
-
 static struct omap_hwmod_class_sysconfig omap34xx_sr_sysc = {
 	.sysc_offs	= 0x24,
 	.sysc_flags	= (SYSC_HAS_CLOCKACTIVITY | SYSC_NO_CACHE),
@@ -1206,11 +1120,6 @@ static struct omap_hwmod_class omap34xx_smartreflex_hwmod_class = {
 	.rev  = 1,
 };
 
-static struct omap_hwmod_sysc_fields omap36xx_sr_sysc_fields = {
-	.sidle_shift	= 24,
-	.enwkup_shift	= 26,
-};
-
 static struct omap_hwmod_class_sysconfig omap36xx_sr_sysc = {
 	.sysc_offs	= 0x38,
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART),
@@ -1237,8 +1146,6 @@ static struct omap_hwmod omap34xx_sr1_hwmod = {
 	.main_clk	= "sr1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_SR1_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_EN_SR1_SHIFT,
@@ -1254,8 +1161,6 @@ static struct omap_hwmod omap36xx_sr1_hwmod = {
 	.main_clk	= "sr1_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_SR1_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_EN_SR1_SHIFT,
@@ -1276,8 +1181,6 @@ static struct omap_hwmod omap34xx_sr2_hwmod = {
 	.main_clk	= "sr2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_SR2_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_EN_SR2_SHIFT,
@@ -1293,8 +1196,6 @@ static struct omap_hwmod omap36xx_sr2_hwmod = {
 	.main_clk	= "sr2_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_SR2_SHIFT,
 			.module_offs = WKUP_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_EN_SR2_SHIFT,
@@ -1330,8 +1231,6 @@ static struct omap_hwmod omap3xxx_mailbox_hwmod = {
 	.main_clk	= "mailboxes_ick",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MAILBOXES_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MAILBOXES_SHIFT,
@@ -1373,8 +1272,6 @@ static struct omap_hwmod omap34xx_mcspi1 = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCSPI1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCSPI1_SHIFT,
 		},
@@ -1394,8 +1291,6 @@ static struct omap_hwmod omap34xx_mcspi2 = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCSPI2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCSPI2_SHIFT,
 		},
@@ -1417,8 +1312,6 @@ static struct omap_hwmod omap34xx_mcspi3 = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCSPI3_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCSPI3_SHIFT,
 		},
@@ -1440,8 +1333,6 @@ static struct omap_hwmod omap34xx_mcspi4 = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MCSPI4_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MCSPI4_SHIFT,
 		},
@@ -1475,12 +1366,9 @@ static struct omap_hwmod omap3xxx_usbhsotg_hwmod = {
 	.main_clk	= "hsotgusb_ick",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_HSOTGUSB_SHIFT,
 			.module_offs = CORE_MOD,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430ES2_ST_HSOTGUSB_IDLE_SHIFT,
-			.idlest_stdby_bit = OMAP3430ES2_ST_HSOTGUSB_STDBY_SHIFT,
 		},
 	},
 	.class		= &usbotg_class,
@@ -1555,8 +1443,6 @@ static struct omap_hwmod omap3xxx_pre_es3_mmc1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MMC1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MMC1_SHIFT,
 		},
@@ -1573,8 +1459,6 @@ static struct omap_hwmod omap3xxx_es3plus_mmc1_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MMC1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MMC1_SHIFT,
 		},
@@ -1604,8 +1488,6 @@ static struct omap_hwmod omap3xxx_pre_es3_mmc2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MMC2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MMC2_SHIFT,
 		},
@@ -1622,8 +1504,6 @@ static struct omap_hwmod omap3xxx_es3plus_mmc2_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MMC2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MMC2_SHIFT,
 		},
@@ -1647,8 +1527,6 @@ static struct omap_hwmod omap3xxx_mmc3_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_MMC3_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_MMC3_SHIFT,
 		},
@@ -1688,11 +1566,8 @@ static struct omap_hwmod omap3xxx_usb_host_hs_hwmod = {
 	.prcm = {
 		.omap2 = {
 			.module_offs = OMAP3430ES2_USBHOST_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430ES2_EN_USBHOST1_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430ES2_ST_USBHOST_IDLE_SHIFT,
-			.idlest_stdby_bit = OMAP3430ES2_ST_USBHOST_STDBY_SHIFT,
 		},
 	},
 
@@ -1766,8 +1641,6 @@ static struct omap_hwmod omap3xxx_usb_tll_hs_hwmod = {
 	.prcm = {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 3,
-			.module_bit = OMAP3430ES2_EN_USBTLL_SHIFT,
 			.idlest_reg_id = 3,
 			.idlest_idle_bit = OMAP3430ES2_ST_USBTLL_SHIFT,
 		},
@@ -1780,8 +1653,6 @@ static struct omap_hwmod omap3xxx_hdq1w_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_HDQ_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_HDQ_SHIFT,
 		},
@@ -1807,8 +1678,6 @@ static struct omap_hwmod omap3xxx_sad2d_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_SAD2D_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_SAD2D_SHIFT,
 		},
@@ -1842,8 +1711,6 @@ static struct omap_hwmod omap3xxx_counter_32k_hwmod = {
 	.prcm		= {
 		.omap2	= {
 			.module_offs = WKUP_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_ST_32KSYNC_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_32KSYNC_SHIFT,
 		},
@@ -2454,7 +2321,6 @@ static struct omap_hwmod omap3xxx_mmu_iva_hwmod = {
 	.prcm = {
 		.omap2 = {
 			.module_offs = OMAP3430_IVA2_MOD,
-			.module_bit = OMAP3430_CM_FCLKEN_IVA2_EN_IVA2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_IVA2_SHIFT,
 		},
@@ -2732,12 +2598,6 @@ static struct omap_hwmod_ocp_if omap3xxx_l3_main__gpmc = {
 };
 
 /* l4_core -> SHAM2 (SHA1/MD5) (similar to omap24xx) */
-static struct omap_hwmod_sysc_fields omap3_sham_sysc_fields = {
-	.sidle_shift	= 4,
-	.srst_shift	= 1,
-	.autoidle_shift	= 0,
-};
-
 static struct omap_hwmod_class_sysconfig omap3_sham_sysc = {
 	.rev_offs	= 0x5c,
 	.sysc_offs	= 0x60,
@@ -2760,8 +2620,6 @@ static struct omap_hwmod omap3xxx_sham_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_SHA12_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_SHA12_SHIFT,
 		},
@@ -2778,12 +2636,6 @@ static struct omap_hwmod_ocp_if omap3xxx_l4_core__sham = {
 };
 
 /* l4_core -> AES */
-static struct omap_hwmod_sysc_fields omap3xxx_aes_sysc_fields = {
-	.sidle_shift	= 6,
-	.srst_shift	= 1,
-	.autoidle_shift	= 0,
-};
-
 static struct omap_hwmod_class_sysconfig omap3_aes_sysc = {
 	.rev_offs	= 0x44,
 	.sysc_offs	= 0x48,
@@ -2806,8 +2658,6 @@ static struct omap_hwmod omap3xxx_aes_hwmod = {
 	.prcm		= {
 		.omap2 = {
 			.module_offs = CORE_MOD,
-			.prcm_reg_id = 1,
-			.module_bit = OMAP3430_EN_AES2_SHIFT,
 			.idlest_reg_id = 1,
 			.idlest_idle_bit = OMAP3430_ST_AES2_SHIFT,
 		},
@@ -2850,8 +2700,6 @@ static struct omap_hwmod omap3xxx_ssi_hwmod = {
 	.main_clk	= "ssi_ssr_fck",
 	.prcm		= {
 		.omap2 = {
-			.prcm_reg_id		= 1,
-			.module_bit		= OMAP3430_EN_SSI_SHIFT,
 			.module_offs		= CORE_MOD,
 			.idlest_reg_id		= 1,
 			.idlest_idle_bit	= OMAP3430ES2_ST_SSI_IDLE_SHIFT,
diff --git a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
index c47709659a54..a1901c22a0f0 100644
--- a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
@@ -1658,10 +1658,6 @@ static struct omap_hwmod omap44xx_mailbox_hwmod = {
  */
 
 /* The IP is not compliant to type1 / type2 scheme */
-static struct omap_hwmod_sysc_fields omap_hwmod_sysc_type_mcasp = {
-	.sidle_shift	= 0,
-};
-
 static struct omap_hwmod_class_sysconfig omap44xx_mcasp_sysc = {
 	.sysc_offs	= 0x0004,
 	.sysc_flags	= SYSC_HAS_SIDLEMODE,
@@ -2403,17 +2399,12 @@ static struct omap_hwmod omap44xx_slimbus2_hwmod = {
  */
 
 /* The IP is not compliant to type1 / type2 scheme */
-static struct omap_hwmod_sysc_fields omap_hwmod_sysc_type_smartreflex = {
-	.sidle_shift	= 24,
-	.enwkup_shift	= 26,
-};
-
 static struct omap_hwmod_class_sysconfig omap44xx_smartreflex_sysc = {
 	.sysc_offs	= 0x0038,
 	.sysc_flags	= (SYSC_HAS_ENAWAKEUP | SYSC_HAS_SIDLEMODE),
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
 			   SIDLE_SMART_WKUP),
-	.sysc_fields	= &omap_hwmod_sysc_type_smartreflex,
+	.sysc_fields	= &omap36xx_sr_sysc_fields,
 };
 
 static struct omap_hwmod_class omap44xx_smartreflex_hwmod_class = {
@@ -2844,12 +2835,6 @@ static struct omap_hwmod omap44xx_uart4_hwmod = {
  */
 
 /* The IP is not compliant to type1 / type2 scheme */
-static struct omap_hwmod_sysc_fields omap_hwmod_sysc_type_usb_host_fs = {
-	.midle_shift	= 4,
-	.sidle_shift	= 2,
-	.srst_shift	= 1,
-};
-
 static struct omap_hwmod_class_sysconfig omap44xx_usb_host_fs_sysc = {
 	.rev_offs	= 0x0000,
 	.sysc_offs	= 0x0210,
diff --git a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
index d05e553d6346..4c2a05b1bd19 100644
--- a/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_7xx_data.c
@@ -2055,17 +2055,12 @@ static struct omap_hwmod dra7xx_sata_hwmod = {
  */
 
 /* The IP is not compliant to type1 / type2 scheme */
-static struct omap_hwmod_sysc_fields omap_hwmod_sysc_type_smartreflex = {
-	.sidle_shift	= 24,
-	.enwkup_shift	= 26,
-};
-
 static struct omap_hwmod_class_sysconfig dra7xx_smartreflex_sysc = {
 	.sysc_offs	= 0x0038,
 	.sysc_flags	= (SYSC_HAS_ENAWAKEUP | SYSC_HAS_SIDLEMODE),
 	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
 			   SIDLE_SMART_WKUP),
-	.sysc_fields	= &omap_hwmod_sysc_type_smartreflex,
+	.sysc_fields	= &omap36xx_sr_sysc_fields,
 };
 
 static struct omap_hwmod_class dra7xx_smartreflex_hwmod_class = {
@@ -4019,6 +4014,10 @@ static struct omap_hwmod_ocp_if *dra76x_hwmod_ocp_ifs[] __initdata = {
 	NULL,
 };
 
+static struct omap_hwmod_ocp_if *acd_76x_hwmod_ocp_ifs[] __initdata = {
+	NULL,
+};
+
 static struct omap_hwmod_ocp_if *dra74x_hwmod_ocp_ifs[] __initdata = {
 	&dra7xx_l4_per3__usb_otg_ss4,
 	NULL,
@@ -4028,7 +4027,7 @@ static struct omap_hwmod_ocp_if *dra72x_hwmod_ocp_ifs[] __initdata = {
 	NULL,
 };
 
-static struct omap_hwmod_ocp_if *dra74x_dra72x_hwmod_ocp_ifs[] __initdata = {
+static struct omap_hwmod_ocp_if *rtc_hwmod_ocp_ifs[] __initdata = {
 	&dra7xx_l4_per3__rtcss,
 	NULL,
 };
@@ -4040,19 +4039,26 @@ int __init dra7xx_hwmod_init(void)
 	omap_hwmod_init();
 	ret = omap_hwmod_register_links(dra7xx_hwmod_ocp_ifs);
 
-	if (!ret && soc_is_dra74x())
+	if (!ret && soc_is_dra74x()) {
 		ret = omap_hwmod_register_links(dra74x_hwmod_ocp_ifs);
-	else if (!ret && soc_is_dra72x())
+		if (!ret)
+			ret = omap_hwmod_register_links(rtc_hwmod_ocp_ifs);
+	} else if (!ret && soc_is_dra72x()) {
 		ret = omap_hwmod_register_links(dra72x_hwmod_ocp_ifs);
-	else if (!ret && soc_is_dra76x())
+		if (!ret && !of_machine_is_compatible("ti,dra718"))
+			ret = omap_hwmod_register_links(rtc_hwmod_ocp_ifs);
+	} else if (!ret && soc_is_dra76x()) {
 		ret = omap_hwmod_register_links(dra76x_hwmod_ocp_ifs);
 
+		if (!ret && soc_is_dra76x_acd()) {
+			ret = omap_hwmod_register_links(acd_76x_hwmod_ocp_ifs);
+		} else if (!ret && soc_is_dra76x_abz()) {
+			ret = omap_hwmod_register_links(rtc_hwmod_ocp_ifs);
+		}
+	}
+
 	if (!ret && omap_type() == OMAP2_DEVICE_TYPE_GP)
 		ret = omap_hwmod_register_links(dra7xx_gp_hwmod_ocp_ifs);
 
-	/* now for the IPs available only in dra74 and dra72 */
-	if (!ret && !of_machine_is_compatible("ti,dra718") && !soc_is_dra76x())
-		ret = omap_hwmod_register_links(dra74x_dra72x_hwmod_ocp_ifs);
-
 	return ret;
 }
diff --git a/arch/arm/mach-omap2/omap_hwmod_81xx_data.c b/arch/arm/mach-omap2/omap_hwmod_81xx_data.c
index 77a515b11ec2..84f118280a0e 100644
--- a/arch/arm/mach-omap2/omap_hwmod_81xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_81xx_data.c
@@ -988,7 +988,7 @@ static struct omap_hwmod_class dm81xx_sata_hwmod_class = {
 
 static struct omap_hwmod dm81xx_sata_hwmod = {
 	.name		= "sata",
-	.clkdm_name	= "default_sata_clkdm",
+	.clkdm_name	= "default_clkdm",
 	.flags		= HWMOD_NO_IDLEST,
 	.prcm = {
 		.omap4 = {
diff --git a/arch/arm/mach-omap2/omap_hwmod_common_data.c b/arch/arm/mach-omap2/omap_hwmod_common_data.c
index 79d623b83e49..77c0b7618ea2 100644
--- a/arch/arm/mach-omap2/omap_hwmod_common_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_common_data.c
@@ -16,6 +16,9 @@
  * data and their integration with other OMAP modules and Linux.
  */
 
+#include <linux/types.h>
+#include <linux/platform_data/ti-sysc.h>
+
 #include "omap_hwmod.h"
 
 #include "omap_hwmod_common_data.h"
@@ -27,7 +30,7 @@
  * if the device ip is compliant with the original PRCM protocol
  * defined for OMAP2420.
  */
-struct omap_hwmod_sysc_fields omap_hwmod_sysc_type1 = {
+struct sysc_regbits omap_hwmod_sysc_type1 = {
 	.midle_shift	= SYSC_TYPE1_MIDLEMODE_SHIFT,
 	.clkact_shift	= SYSC_TYPE1_CLOCKACTIVITY_SHIFT,
 	.sidle_shift	= SYSC_TYPE1_SIDLEMODE_SHIFT,
@@ -43,7 +46,7 @@ struct omap_hwmod_sysc_fields omap_hwmod_sysc_type1 = {
  * device ip is compliant with the new PRCM protocol defined for new
  * OMAP4 IPs.
  */
-struct omap_hwmod_sysc_fields omap_hwmod_sysc_type2 = {
+struct sysc_regbits omap_hwmod_sysc_type2 = {
 	.midle_shift	= SYSC_TYPE2_MIDLEMODE_SHIFT,
 	.sidle_shift	= SYSC_TYPE2_SIDLEMODE_SHIFT,
 	.srst_shift	= SYSC_TYPE2_SOFTRESET_SHIFT,
@@ -54,7 +57,7 @@ struct omap_hwmod_sysc_fields omap_hwmod_sysc_type2 = {
  * struct omap_hwmod_sysc_type3 - TYPE3 sysconfig scheme.
  * Used by some IPs on AM33xx
  */
-struct omap_hwmod_sysc_fields omap_hwmod_sysc_type3 = {
+struct sysc_regbits omap_hwmod_sysc_type3 = {
 	.midle_shift	= SYSC_TYPE3_MIDLEMODE_SHIFT,
 	.sidle_shift	= SYSC_TYPE3_SIDLEMODE_SHIFT,
 };
@@ -63,3 +66,34 @@ struct omap_dss_dispc_dev_attr omap2_3_dss_dispc_dev_attr = {
 	.manager_count		= 2,
 	.has_framedonetv_irq	= 0
 };
+
+struct sysc_regbits omap34xx_sr_sysc_fields = {
+	.clkact_shift	= 20,
+};
+
+struct sysc_regbits omap36xx_sr_sysc_fields = {
+	.sidle_shift	= 24,
+	.enwkup_shift	= 26,
+};
+
+struct sysc_regbits omap3_sham_sysc_fields = {
+	.sidle_shift	= 4,
+	.srst_shift	= 1,
+	.autoidle_shift	= 0,
+};
+
+struct sysc_regbits omap3xxx_aes_sysc_fields = {
+	.sidle_shift	= 6,
+	.srst_shift	= 1,
+	.autoidle_shift	= 0,
+};
+
+struct sysc_regbits omap_hwmod_sysc_type_mcasp = {
+	.sidle_shift	= 0,
+};
+
+struct sysc_regbits omap_hwmod_sysc_type_usb_host_fs = {
+	.midle_shift	= 4,
+	.sidle_shift	= 2,
+	.srst_shift	= 1,
+};
diff --git a/arch/arm/mach-omap2/pm.c b/arch/arm/mach-omap2/pm.c
index 366158a54fcd..6f68576e5695 100644
--- a/arch/arm/mach-omap2/pm.c
+++ b/arch/arm/mach-omap2/pm.c
@@ -186,7 +186,7 @@ static void omap_pm_end(void)
 	cpu_idle_poll_ctrl(false);
 }
 
-static void omap_pm_finish(void)
+static void omap_pm_wake(void)
 {
 	if (soc_is_omap34xx())
 		omap_prcm_irq_complete();
@@ -196,7 +196,7 @@ static const struct platform_suspend_ops omap_pm_ops = {
 	.begin		= omap_pm_begin,
 	.end		= omap_pm_end,
 	.enter		= omap_pm_enter,
-	.finish		= omap_pm_finish,
+	.wake		= omap_pm_wake,
 	.valid		= suspend_valid_only_mem,
 };
 
diff --git a/arch/arm/mach-omap2/soc.h b/arch/arm/mach-omap2/soc.h
index 28fa1f8d8363..050891e055a4 100644
--- a/arch/arm/mach-omap2/soc.h
+++ b/arch/arm/mach-omap2/soc.h
@@ -143,6 +143,14 @@ static inline int is_dra ##subclass (void)		\
 	return (GET_OMAP_SUBCLASS == (id)) ? 1 : 0;	\
 }
 
+#define GET_DRA_PACKAGE		(omap_rev() & 0xff)
+
+#define IS_DRA_SUBCLASS_PACKAGE(subclass, package, id)			\
+static inline int is_dra ##subclass ##_ ##package (void)		\
+{									\
+	return (is_dra ##subclass () && GET_DRA_PACKAGE == id) ? 1 : 0;	\
+}
+
 IS_OMAP_CLASS(24xx, 0x24)
 IS_OMAP_CLASS(34xx, 0x34)
 IS_OMAP_CLASS(44xx, 0x44)
@@ -168,6 +176,8 @@ IS_TI_SUBCLASS(814x, 0x814)
 IS_AM_SUBCLASS(335x, 0x335)
 IS_AM_SUBCLASS(437x, 0x437)
 IS_DRA_SUBCLASS(76x, 0x76)
+IS_DRA_SUBCLASS_PACKAGE(76x, abz, 2)
+IS_DRA_SUBCLASS_PACKAGE(76x, acd, 3)
 IS_DRA_SUBCLASS(75x, 0x75)
 IS_DRA_SUBCLASS(72x, 0x72)
 
@@ -317,10 +327,14 @@ IS_OMAP_TYPE(3430, 0x3430)
 #if defined(CONFIG_SOC_DRA7XX)
 #undef soc_is_dra7xx
 #undef soc_is_dra76x
+#undef soc_is_dra76x_abz
+#undef soc_is_dra76x_acd
 #undef soc_is_dra74x
 #undef soc_is_dra72x
 #define soc_is_dra7xx()	is_dra7xx()
 #define soc_is_dra76x()	is_dra76x()
+#define soc_is_dra76x_abz()	is_dra76x_abz()
+#define soc_is_dra76x_acd()	is_dra76x_acd()
 #define soc_is_dra74x()	is_dra75x()
 #define soc_is_dra72x()	is_dra72x()
 #endif
@@ -391,6 +405,8 @@ IS_OMAP_TYPE(3430, 0x3430)
 
 #define DRA7XX_CLASS		0x07000000
 #define DRA762_REV_ES1_0	(DRA7XX_CLASS | (0x62 << 16) | (0x10 << 8))
+#define DRA762_ABZ_REV_ES1_0	(DRA762_REV_ES1_0 | (2 << 0))
+#define DRA762_ACD_REV_ES1_0	(DRA762_REV_ES1_0 | (3 << 0))
 #define DRA752_REV_ES1_0	(DRA7XX_CLASS | (0x52 << 16) | (0x10 << 8))
 #define DRA752_REV_ES1_1	(DRA7XX_CLASS | (0x52 << 16) | (0x11 << 8))
 #define DRA752_REV_ES2_0	(DRA7XX_CLASS | (0x52 << 16) | (0x20 << 8))
diff --git a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c
index ece09c9461f7..d61fbd7a2840 100644
--- a/arch/arm/mach-omap2/timer.c
+++ b/arch/arm/mach-omap2/timer.c
@@ -156,12 +156,6 @@ static struct clock_event_device clockevent_gpt = {
 	.tick_resume		= omap2_gp_timer_shutdown,
 };
 
-static struct property device_disabled = {
-	.name = "status",
-	.length = sizeof("disabled"),
-	.value = "disabled",
-};
-
 static const struct of_device_id omap_timer_match[] __initconst = {
 	{ .compatible = "ti,omap2420-timer", },
 	{ .compatible = "ti,omap3430-timer", },
@@ -203,8 +197,17 @@ static struct device_node * __init omap_get_timer_dt(const struct of_device_id *
 				  of_get_property(np, "ti,timer-secure", NULL)))
 			continue;
 
-		if (!of_device_is_compatible(np, "ti,omap-counter32k"))
-			of_add_property(np, &device_disabled);
+		if (!of_device_is_compatible(np, "ti,omap-counter32k")) {
+			struct property *prop;
+
+			prop = kzalloc(sizeof(*prop), GFP_KERNEL);
+			if (!prop)
+				return NULL;
+			prop->name = "status";
+			prop->value = "disabled";
+			prop->length = strlen(prop->value);
+			of_add_property(np, prop);
+		}
 		return np;
 	}
 
diff --git a/arch/arm/mach-orion5x/Kconfig b/arch/arm/mach-orion5x/Kconfig
index 2a7bb6ccdcb7..a810f4dd34b1 100644
--- a/arch/arm/mach-orion5x/Kconfig
+++ b/arch/arm/mach-orion5x/Kconfig
@@ -58,7 +58,6 @@ config MACH_KUROBOX_PRO
 
 config MACH_DNS323
 	bool "D-Link DNS-323"
-	select GENERIC_NET_UTILS
 	select I2C_BOARDINFO if I2C
 	help
 	  Say 'Y' here if you want your kernel to support the
@@ -66,7 +65,6 @@ config MACH_DNS323
 
 config MACH_TS209
 	bool "QNAP TS-109/TS-209"
-	select GENERIC_NET_UTILS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  QNAP TS-109/TS-209 platform.
@@ -101,7 +99,6 @@ config MACH_LINKSTATION_LS_HGL
 
 config MACH_TS409
 	bool "QNAP TS-409"
-	select GENERIC_NET_UTILS
 	help
 	  Say 'Y' here if you want your kernel to support the
 	  QNAP TS-409 platform.
diff --git a/arch/arm/mach-orion5x/dns323-setup.c b/arch/arm/mach-orion5x/dns323-setup.c
index cd483bfb5ca8..d13344b2ddcd 100644
--- a/arch/arm/mach-orion5x/dns323-setup.c
+++ b/arch/arm/mach-orion5x/dns323-setup.c
@@ -173,10 +173,42 @@ static struct mv643xx_eth_platform_data dns323_eth_data = {
 	.phy_addr = MV643XX_ETH_PHY_ADDR(8),
 };
 
+/* dns323_parse_hex_*() taken from tsx09-common.c; should a common copy of these
+ * functions be kept somewhere?
+ */
+static int __init dns323_parse_hex_nibble(char n)
+{
+	if (n >= '0' && n <= '9')
+		return n - '0';
+
+	if (n >= 'A' && n <= 'F')
+		return n - 'A' + 10;
+
+	if (n >= 'a' && n <= 'f')
+		return n - 'a' + 10;
+
+	return -1;
+}
+
+static int __init dns323_parse_hex_byte(const char *b)
+{
+	int hi;
+	int lo;
+
+	hi = dns323_parse_hex_nibble(b[0]);
+	lo = dns323_parse_hex_nibble(b[1]);
+
+	if (hi < 0 || lo < 0)
+		return -1;
+
+	return (hi << 4) | lo;
+}
+
 static int __init dns323_read_mac_addr(void)
 {
 	u_int8_t addr[6];
-	void __iomem *mac_page;
+	int i;
+	char *mac_page;
 
 	/* MAC address is stored as a regular ol' string in /dev/mtdblock4
 	 * (0x007d0000-0x00800000) starting at offset 196480 (0x2ff80).
@@ -185,8 +217,23 @@ static int __init dns323_read_mac_addr(void)
 	if (!mac_page)
 		return -ENOMEM;
 
-	if (!mac_pton((__force const char *) mac_page, addr))
-		goto error_fail;
+	/* Sanity check the string we're looking at */
+	for (i = 0; i < 5; i++) {
+		if (*(mac_page + (i * 3) + 2) != ':') {
+			goto error_fail;
+		}
+	}
+
+	for (i = 0; i < 6; i++)	{
+		int byte;
+
+		byte = dns323_parse_hex_byte(mac_page + (i * 3));
+		if (byte < 0) {
+			goto error_fail;
+		}
+
+		addr[i] = byte;
+	}
 
 	iounmap(mac_page);
 	printk("DNS-323: Found ethernet MAC address: %pM\n", addr);
diff --git a/arch/arm/mach-orion5x/tsx09-common.c b/arch/arm/mach-orion5x/tsx09-common.c
index 89774985d380..905d4f2dd0b8 100644
--- a/arch/arm/mach-orion5x/tsx09-common.c
+++ b/arch/arm/mach-orion5x/tsx09-common.c
@@ -53,12 +53,53 @@ struct mv643xx_eth_platform_data qnap_tsx09_eth_data = {
 	.phy_addr	= MV643XX_ETH_PHY_ADDR(8),
 };
 
+static int __init qnap_tsx09_parse_hex_nibble(char n)
+{
+	if (n >= '0' && n <= '9')
+		return n - '0';
+
+	if (n >= 'A' && n <= 'F')
+		return n - 'A' + 10;
+
+	if (n >= 'a' && n <= 'f')
+		return n - 'a' + 10;
+
+	return -1;
+}
+
+static int __init qnap_tsx09_parse_hex_byte(const char *b)
+{
+	int hi;
+	int lo;
+
+	hi = qnap_tsx09_parse_hex_nibble(b[0]);
+	lo = qnap_tsx09_parse_hex_nibble(b[1]);
+
+	if (hi < 0 || lo < 0)
+		return -1;
+
+	return (hi << 4) | lo;
+}
+
 static int __init qnap_tsx09_check_mac_addr(const char *addr_str)
 {
 	u_int8_t addr[6];
+	int i;
 
-	if (!mac_pton(addr_str, addr))
-		return -1;
+	for (i = 0; i < 6; i++) {
+		int byte;
+
+		/*
+		 * Enforce "xx:xx:xx:xx:xx:xx\n" format.
+		 */
+		if (addr_str[(i * 3) + 2] != ((i < 5) ? ':' : '\n'))
+			return -1;
+
+		byte = qnap_tsx09_parse_hex_byte(addr_str + (i * 3));
+		if (byte < 0)
+			return -1;
+		addr[i] = byte;
+	}
 
 	printk(KERN_INFO "tsx09: found ethernet mac address %pM\n", addr);
 
@@ -77,12 +118,12 @@ void __init qnap_tsx09_find_mac_addr(u32 mem_base, u32 size)
 	unsigned long addr;
 
 	for (addr = mem_base; addr < (mem_base + size); addr += 1024) {
-		void __iomem *nor_page;
+		char *nor_page;
 		int ret = 0;
 
 		nor_page = ioremap(addr, 1024);
 		if (nor_page != NULL) {
-			ret = qnap_tsx09_check_mac_addr((__force const char *)nor_page);
+			ret = qnap_tsx09_check_mac_addr(nor_page);
 			iounmap(nor_page);
 		}
 
diff --git a/arch/arm/mach-pxa/Kconfig b/arch/arm/mach-pxa/Kconfig
index ce7d97babb0f..a68b34183107 100644
--- a/arch/arm/mach-pxa/Kconfig
+++ b/arch/arm/mach-pxa/Kconfig
@@ -40,6 +40,7 @@ config MACH_PXA3XX_DT
 
 config ARCH_LUBBOCK
 	bool "Intel DBPXA250 Development Platform (aka Lubbock)"
+	select GPIO_REG
 	select PXA25x
 	select SA1111
 
diff --git a/arch/arm/mach-pxa/balloon3.c b/arch/arm/mach-pxa/balloon3.c
index d6d92f388f14..f4f8f23bda8c 100644
--- a/arch/arm/mach-pxa/balloon3.c
+++ b/arch/arm/mach-pxa/balloon3.c
@@ -28,7 +28,7 @@
 #include <linux/mtd/partitions.h>
 #include <linux/types.h>
 #include <linux/platform_data/pcf857x.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/mtd/rawnand.h>
 #include <linux/mtd/physmap.h>
 #include <linux/regulator/max1586.h>
diff --git a/arch/arm/mach-pxa/cm-x300.c b/arch/arm/mach-pxa/cm-x300.c
index 868448d2cd82..c487401b6fdb 100644
--- a/arch/arm/mach-pxa/cm-x300.c
+++ b/arch/arm/mach-pxa/cm-x300.c
@@ -31,7 +31,7 @@
 
 #include <linux/i2c.h>
 #include <linux/platform_data/pca953x.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <linux/mfd/da903x.h>
 #include <linux/regulator/machine.h>
diff --git a/arch/arm/mach-pxa/colibri-evalboard.c b/arch/arm/mach-pxa/colibri-evalboard.c
index dc44fbbe5073..10e2278b7a28 100644
--- a/arch/arm/mach-pxa/colibri-evalboard.c
+++ b/arch/arm/mach-pxa/colibri-evalboard.c
@@ -19,7 +19,7 @@
 #include <mach/hardware.h>
 #include <asm/mach/arch.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <asm/io.h>
 
 #include "pxa27x.h"
diff --git a/arch/arm/mach-pxa/colibri-pxa270-income.c b/arch/arm/mach-pxa/colibri-pxa270-income.c
index d7cf47d03618..3ccf2a95569b 100644
--- a/arch/arm/mach-pxa/colibri-pxa270-income.c
+++ b/arch/arm/mach-pxa/colibri-pxa270-income.c
@@ -23,7 +23,7 @@
 #include <linux/platform_device.h>
 #include <linux/pwm.h>
 #include <linux/pwm_backlight.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/irq.h>
 #include <asm/mach-types.h>
diff --git a/arch/arm/mach-pxa/corgi.c b/arch/arm/mach-pxa/corgi.c
index 7270f0db3432..9a5a35e90769 100644
--- a/arch/arm/mach-pxa/corgi.c
+++ b/arch/arm/mach-pxa/corgi.c
@@ -26,7 +26,7 @@
 #include <linux/gpio.h>
 #include <linux/backlight.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/io.h>
 #include <linux/regulator/machine.h>
 #include <linux/spi/spi.h>
@@ -606,24 +606,6 @@ static void __init corgi_init_spi(void)
 static inline void corgi_init_spi(void) {}
 #endif
 
-static struct mtd_partition sharpsl_nand_partitions[] = {
-	{
-		.name = "System Area",
-		.offset = 0,
-		.size = 7 * 1024 * 1024,
-	},
-	{
-		.name = "Root Filesystem",
-		.offset = 7 * 1024 * 1024,
-		.size = 25 * 1024 * 1024,
-	},
-	{
-		.name = "Home Filesystem",
-		.offset = MTDPART_OFS_APPEND,
-		.size = MTDPART_SIZ_FULL,
-	},
-};
-
 static uint8_t scan_ff_pattern[] = { 0xff, 0xff };
 
 static struct nand_bbt_descr sharpsl_bbt = {
@@ -633,10 +615,16 @@ static struct nand_bbt_descr sharpsl_bbt = {
 	.pattern = scan_ff_pattern
 };
 
+static const char * const probes[] = {
+	"cmdlinepart",
+	"ofpart",
+	"sharpslpart",
+	NULL,
+};
+
 static struct sharpsl_nand_platform_data sharpsl_nand_platform_data = {
 	.badblock_pattern	= &sharpsl_bbt,
-	.partitions		= sharpsl_nand_partitions,
-	.nr_partitions		= ARRAY_SIZE(sharpsl_nand_partitions),
+	.part_parsers		= probes,
 };
 
 static struct resource sharpsl_nand_resources[] = {
@@ -750,9 +738,6 @@ static void __init corgi_init(void)
 
 	platform_scoop_config = &corgi_pcmcia_config;
 
-	if (machine_is_husky())
-		sharpsl_nand_partitions[1].size = 53 * 1024 * 1024;
-
 	platform_add_devices(devices, ARRAY_SIZE(devices));
 
 	regulator_has_full_constraints();
diff --git a/arch/arm/mach-pxa/csb726.c b/arch/arm/mach-pxa/csb726.c
index bf19b8426d2c..271aedae7542 100644
--- a/arch/arm/mach-pxa/csb726.c
+++ b/arch/arm/mach-pxa/csb726.c
@@ -17,7 +17,7 @@
 #include <linux/mtd/partitions.h>
 #include <linux/sm501.h>
 #include <linux/smsc911x.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
diff --git a/arch/arm/mach-pxa/devices.c b/arch/arm/mach-pxa/devices.c
index 5a72456a19ce..d7c9a8476d57 100644
--- a/arch/arm/mach-pxa/devices.c
+++ b/arch/arm/mach-pxa/devices.c
@@ -5,7 +5,7 @@
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
 #include <linux/spi/pxa2xx_spi.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include "udc.h"
 #include <linux/platform_data/usb-pxa3xx-ulpi.h>
diff --git a/arch/arm/mach-pxa/devices.h b/arch/arm/mach-pxa/devices.h
index 905628dfbbbb..11263f7c455b 100644
--- a/arch/arm/mach-pxa/devices.h
+++ b/arch/arm/mach-pxa/devices.h
@@ -56,3 +56,12 @@ extern struct platform_device pxa93x_device_gpio;
 
 void __init pxa_register_device(struct platform_device *dev, void *data);
 void __init pxa2xx_set_dmac_info(int nb_channels, int nb_requestors);
+
+struct i2c_pxa_platform_data;
+extern void pxa_set_i2c_info(struct i2c_pxa_platform_data *info);
+#ifdef CONFIG_PXA27x
+extern void pxa27x_set_i2c_power_info(struct i2c_pxa_platform_data *info);
+#endif
+#ifdef CONFIG_PXA3xx
+extern void pxa3xx_set_i2c_power_info(struct i2c_pxa_platform_data *info);
+#endif
diff --git a/arch/arm/mach-pxa/em-x270.c b/arch/arm/mach-pxa/em-x270.c
index 6d28035ebba5..49022ad338e9 100644
--- a/arch/arm/mach-pxa/em-x270.c
+++ b/arch/arm/mach-pxa/em-x270.c
@@ -32,7 +32,7 @@
 #include <linux/apm-emulation.h>
 #include <linux/i2c.h>
 #include <linux/platform_data/pca953x.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/regulator/userspace-consumer.h>
 
 #include <asm/mach-types.h>
diff --git a/arch/arm/mach-pxa/ezx.c b/arch/arm/mach-pxa/ezx.c
index a057cf9c0e7b..2c90b58f347d 100644
--- a/arch/arm/mach-pxa/ezx.c
+++ b/arch/arm/mach-pxa/ezx.c
@@ -23,7 +23,7 @@
 #include <linux/gpio.h>
 #include <linux/gpio_keys.h>
 #include <linux/leds-lp3944.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/setup.h>
 #include <asm/mach-types.h>
diff --git a/arch/arm/mach-pxa/hx4700.c b/arch/arm/mach-pxa/hx4700.c
index 66184f5cbe40..e2e7f247a645 100644
--- a/arch/arm/mach-pxa/hx4700.c
+++ b/arch/arm/mach-pxa/hx4700.c
@@ -38,7 +38,7 @@
 #include <linux/spi/spi.h>
 #include <linux/spi/pxa2xx_spi.h>
 #include <linux/usb/gpio_vbus.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <mach/hardware.h>
 #include <asm/mach-types.h>
diff --git a/arch/arm/mach-pxa/littleton.c b/arch/arm/mach-pxa/littleton.c
index fae38fdc8d8e..4105614cc38e 100644
--- a/arch/arm/mach-pxa/littleton.c
+++ b/arch/arm/mach-pxa/littleton.c
@@ -28,7 +28,7 @@
 #include <linux/leds.h>
 #include <linux/mfd/da903x.h>
 #include <linux/platform_data/max732x.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/types.h>
 #include <asm/setup.h>
@@ -42,6 +42,7 @@
 #include <asm/mach/irq.h>
 
 #include "pxa300.h"
+#include "devices.h"
 #include <linux/platform_data/video-pxafb.h>
 #include <linux/platform_data/mmc-pxamci.h>
 #include <linux/platform_data/keypad-pxa27x.h>
diff --git a/arch/arm/mach-pxa/lubbock.c b/arch/arm/mach-pxa/lubbock.c
index df45682e99a5..fe2ef9b78602 100644
--- a/arch/arm/mach-pxa/lubbock.c
+++ b/arch/arm/mach-pxa/lubbock.c
@@ -13,6 +13,7 @@
  */
 #include <linux/clkdev.h>
 #include <linux/gpio.h>
+#include <linux/gpio/gpio-reg.h>
 #include <linux/gpio/machine.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -110,20 +111,18 @@ static unsigned long lubbock_pin_config[] __initdata = {
 };
 
 #define LUB_HEXLED		__LUB_REG(LUBBOCK_FPGA_PHYS + 0x010)
-#define LUB_MISC_WR		__LUB_REG(LUBBOCK_FPGA_PHYS + 0x080)
 
 void lubbock_set_hexled(uint32_t value)
 {
 	LUB_HEXLED = value;
 }
 
+static struct gpio_chip *lubbock_misc_wr_gc;
+
 void lubbock_set_misc_wr(unsigned int mask, unsigned int set)
 {
-	unsigned long flags;
-
-	local_irq_save(flags);
-	LUB_MISC_WR = (LUB_MISC_WR & ~mask) | (set & mask);
-	local_irq_restore(flags);
+	unsigned long m = mask, v = set;
+	lubbock_misc_wr_gc->set_multiple(lubbock_misc_wr_gc, &m, &v);
 }
 EXPORT_SYMBOL(lubbock_set_misc_wr);
 
@@ -452,9 +451,9 @@ static void lubbock_irda_transceiver_mode(struct device *dev, int mode)
 
 	local_irq_save(flags);
 	if (mode & IR_SIRMODE) {
-		LUB_MISC_WR &= ~(1 << 4);
+		lubbock_set_misc_wr(BIT(4), 0);
 	} else if (mode & IR_FIRMODE) {
-		LUB_MISC_WR |= 1 << 4;
+		lubbock_set_misc_wr(BIT(4), BIT(4));
 	}
 	pxa2xx_transceiver_mode(dev, mode);
 	local_irq_restore(flags);
@@ -472,6 +471,15 @@ static void __init lubbock_init(void)
 
 	pxa2xx_mfp_config(ARRAY_AND_SIZE(lubbock_pin_config));
 
+	lubbock_misc_wr_gc = gpio_reg_init(NULL, (void *)&LUB_MISC_WR,
+					   -1, 16, "lubbock", 0, LUB_MISC_WR,
+					   NULL, NULL, NULL);
+	if (IS_ERR(lubbock_misc_wr_gc)) {
+		pr_err("Lubbock: unable to register lubbock GPIOs: %ld\n",
+		       PTR_ERR(lubbock_misc_wr_gc));
+		lubbock_misc_wr_gc = NULL;
+	}
+
 	pxa_set_ffuart_info(NULL);
 	pxa_set_btuart_info(NULL);
 	pxa_set_stuart_info(NULL);
diff --git a/arch/arm/mach-pxa/magician.c b/arch/arm/mach-pxa/magician.c
index 7f3566c93733..c5325d1ae77b 100644
--- a/arch/arm/mach-pxa/magician.c
+++ b/arch/arm/mach-pxa/magician.c
@@ -31,7 +31,7 @@
 #include <linux/regulator/gpio-regulator.h>
 #include <linux/regulator/machine.h>
 #include <linux/usb/gpio_vbus.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <mach/hardware.h>
 #include <asm/mach-types.h>
diff --git a/arch/arm/mach-pxa/mainstone.c b/arch/arm/mach-pxa/mainstone.c
index a2d851a3a546..afd62a94fdbf 100644
--- a/arch/arm/mach-pxa/mainstone.c
+++ b/arch/arm/mach-pxa/mainstone.c
@@ -29,7 +29,7 @@
 #include <linux/pwm.h>
 #include <linux/pwm_backlight.h>
 #include <linux/smc91x.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/slab.h>
 #include <linux/leds.h>
 
diff --git a/arch/arm/mach-pxa/mioa701.c b/arch/arm/mach-pxa/mioa701.c
index 8a5d0491e73c..9b6c7ea45a40 100644
--- a/arch/arm/mach-pxa/mioa701.c
+++ b/arch/arm/mach-pxa/mioa701.c
@@ -42,7 +42,7 @@
 #include <linux/regulator/fixed.h>
 #include <linux/regulator/max1586.h>
 #include <linux/slab.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
diff --git a/arch/arm/mach-pxa/mxm8x10.c b/arch/arm/mach-pxa/mxm8x10.c
index 9a22ae0ad8c9..f9e3d41a4609 100644
--- a/arch/arm/mach-pxa/mxm8x10.c
+++ b/arch/arm/mach-pxa/mxm8x10.c
@@ -22,7 +22,7 @@
 #include <linux/serial_8250.h>
 #include <linux/dm9000.h>
 #include <linux/gpio.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <linux/platform_data/mtd-nand-pxa3xx.h>
 
diff --git a/arch/arm/mach-pxa/palm27x.c b/arch/arm/mach-pxa/palm27x.c
index e5ae99db1de4..1efe9bcf07fa 100644
--- a/arch/arm/mach-pxa/palm27x.c
+++ b/arch/arm/mach-pxa/palm27x.c
@@ -22,7 +22,7 @@
 #include <linux/power_supply.h>
 #include <linux/usb/gpio_vbus.h>
 #include <linux/regulator/max1586.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
diff --git a/arch/arm/mach-pxa/pcm990-baseboard.c b/arch/arm/mach-pxa/pcm990-baseboard.c
index 0bd5959ef7d5..973568d4b9ec 100644
--- a/arch/arm/mach-pxa/pcm990-baseboard.c
+++ b/arch/arm/mach-pxa/pcm990-baseboard.c
@@ -23,7 +23,7 @@
 #include <linux/irq.h>
 #include <linux/platform_device.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/pwm.h>
 #include <linux/pwm_backlight.h>
 
diff --git a/arch/arm/mach-pxa/poodle.c b/arch/arm/mach-pxa/poodle.c
index 62a119137be7..1adde1251e2b 100644
--- a/arch/arm/mach-pxa/poodle.c
+++ b/arch/arm/mach-pxa/poodle.c
@@ -24,7 +24,7 @@
 #include <linux/mtd/physmap.h>
 #include <linux/gpio.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/regulator/machine.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/ads7846.h>
@@ -333,24 +333,6 @@ static struct pxafb_mach_info poodle_fb_info = {
 	.lcd_conn	= LCD_COLOR_TFT_16BPP,
 };
 
-static struct mtd_partition sharpsl_nand_partitions[] = {
-	{
-		.name = "System Area",
-		.offset = 0,
-		.size = 7 * 1024 * 1024,
-	},
-	{
-		.name = "Root Filesystem",
-		.offset = 7 * 1024 * 1024,
-		.size = 22 * 1024 * 1024,
-	},
-	{
-		.name = "Home Filesystem",
-		.offset = MTDPART_OFS_APPEND,
-		.size = MTDPART_SIZ_FULL,
-	},
-};
-
 static uint8_t scan_ff_pattern[] = { 0xff, 0xff };
 
 static struct nand_bbt_descr sharpsl_bbt = {
@@ -360,10 +342,16 @@ static struct nand_bbt_descr sharpsl_bbt = {
 	.pattern = scan_ff_pattern
 };
 
+static const char * const probes[] = {
+	"cmdlinepart",
+	"ofpart",
+	"sharpslpart",
+	NULL,
+};
+
 static struct sharpsl_nand_platform_data sharpsl_nand_platform_data = {
 	.badblock_pattern	= &sharpsl_bbt,
-	.partitions		= sharpsl_nand_partitions,
-	.nr_partitions		= ARRAY_SIZE(sharpsl_nand_partitions),
+	.part_parsers		= probes,
 };
 
 static struct resource sharpsl_nand_resources[] = {
diff --git a/arch/arm/mach-pxa/pxa27x.c b/arch/arm/mach-pxa/pxa27x.c
index 9b69be4e9fe3..0c06f383ad52 100644
--- a/arch/arm/mach-pxa/pxa27x.c
+++ b/arch/arm/mach-pxa/pxa27x.c
@@ -22,7 +22,7 @@
 #include <linux/syscore_ops.h>
 #include <linux/io.h>
 #include <linux/irq.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/mach/map.h>
 #include <mach/hardware.h>
diff --git a/arch/arm/mach-pxa/pxa3xx.c b/arch/arm/mach-pxa/pxa3xx.c
index 0cc9f124c9ac..4b8a0df8ea57 100644
--- a/arch/arm/mach-pxa/pxa3xx.c
+++ b/arch/arm/mach-pxa/pxa3xx.c
@@ -23,7 +23,7 @@
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/syscore_ops.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/mach/map.h>
 #include <asm/suspend.h>
diff --git a/arch/arm/mach-pxa/raumfeld.c b/arch/arm/mach-pxa/raumfeld.c
index 9d662fed03ec..4d5d05cf87d6 100644
--- a/arch/arm/mach-pxa/raumfeld.c
+++ b/arch/arm/mach-pxa/raumfeld.c
@@ -32,7 +32,7 @@
 #include <linux/pwm.h>
 #include <linux/pwm_backlight.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/spi_gpio.h>
 #include <linux/lis3lv02d.h>
@@ -506,11 +506,16 @@ static void w1_enable_external_pullup(int enable)
 	msleep(100);
 }
 
+static struct gpiod_lookup_table raumfeld_w1_gpiod_table = {
+	.dev_id = "w1-gpio",
+	.table = {
+		GPIO_LOOKUP_IDX("gpio-pxa", GPIO_ONE_WIRE, NULL, 0,
+				GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN),
+	},
+};
+
 static struct w1_gpio_platform_data w1_gpio_platform_data = {
-	.pin			= GPIO_ONE_WIRE,
-	.is_open_drain		= 0,
-	.enable_external_pullup	= w1_enable_external_pullup,
-	.ext_pullup_enable_pin	= -EINVAL,
+	.enable_external_pullup = w1_enable_external_pullup,
 };
 
 static struct platform_device raumfeld_w1_gpio_device = {
@@ -523,13 +528,14 @@ static struct platform_device raumfeld_w1_gpio_device = {
 static void __init raumfeld_w1_init(void)
 {
 	int ret = gpio_request(GPIO_W1_PULLUP_ENABLE,
-				"W1 external pullup enable");
+			        "W1 external pullup enable");
 
 	if (ret < 0)
 		pr_warn("Unable to request GPIO_W1_PULLUP_ENABLE\n");
 	else
 		gpio_direction_output(GPIO_W1_PULLUP_ENABLE, 0);
 
+	gpiod_add_lookup_table(&raumfeld_w1_gpiod_table);
 	platform_device_register(&raumfeld_w1_gpio_device);
 }
 
diff --git a/arch/arm/mach-pxa/saar.c b/arch/arm/mach-pxa/saar.c
index 1414b5f29114..834991034f30 100644
--- a/arch/arm/mach-pxa/saar.c
+++ b/arch/arm/mach-pxa/saar.c
@@ -20,7 +20,7 @@
 #include <linux/delay.h>
 #include <linux/fb.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/smc91x.h>
 #include <linux/mfd/da903x.h>
 #include <linux/mtd/mtd.h>
diff --git a/arch/arm/mach-pxa/sharpsl_pm.c b/arch/arm/mach-pxa/sharpsl_pm.c
index 398ba9ba2632..ef9fd9b759cb 100644
--- a/arch/arm/mach-pxa/sharpsl_pm.c
+++ b/arch/arm/mach-pxa/sharpsl_pm.c
@@ -802,8 +802,8 @@ static ssize_t battery_voltage_show(struct device *dev, struct device_attribute
 	return sprintf(buf, "%d\n", sharpsl_pm.battstat.mainbat_voltage);
 }
 
-static DEVICE_ATTR(battery_percentage, 0444, battery_percentage_show, NULL);
-static DEVICE_ATTR(battery_voltage, 0444, battery_voltage_show, NULL);
+static DEVICE_ATTR_RO(battery_percentage);
+static DEVICE_ATTR_RO(battery_voltage);
 
 extern void (*apm_get_power_status)(struct apm_power_info *);
 
diff --git a/arch/arm/mach-pxa/spitz.c b/arch/arm/mach-pxa/spitz.c
index 67d66c702574..5d50025492b7 100644
--- a/arch/arm/mach-pxa/spitz.c
+++ b/arch/arm/mach-pxa/spitz.c
@@ -20,7 +20,7 @@
 #include <linux/gpio.h>
 #include <linux/leds.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/platform_data/pca953x.h>
 #include <linux/spi/spi.h>
 #include <linux/spi/ads7846.h>
@@ -739,21 +739,6 @@ static inline void spitz_lcd_init(void) {}
  * NAND Flash
  ******************************************************************************/
 #if defined(CONFIG_MTD_NAND_SHARPSL) || defined(CONFIG_MTD_NAND_SHARPSL_MODULE)
-static struct mtd_partition spitz_nand_partitions[] = {
-	{
-		.name = "System Area",
-		.offset = 0,
-		.size = 7 * 1024 * 1024,
-	}, {
-		.name = "Root Filesystem",
-		.offset = 7 * 1024 * 1024,
-	}, {
-		.name = "Home Filesystem",
-		.offset = MTDPART_OFS_APPEND,
-		.size = MTDPART_SIZ_FULL,
-	},
-};
-
 static uint8_t scan_ff_pattern[] = { 0xff, 0xff };
 
 static struct nand_bbt_descr spitz_nand_bbt = {
@@ -808,10 +793,16 @@ static const struct mtd_ooblayout_ops akita_ooblayout_ops = {
 	.free = akita_ooblayout_free,
 };
 
+static const char * const probes[] = {
+	"cmdlinepart",
+	"ofpart",
+	"sharpslpart",
+	NULL,
+};
+
 static struct sharpsl_nand_platform_data spitz_nand_pdata = {
 	.badblock_pattern	= &spitz_nand_bbt,
-	.partitions		= spitz_nand_partitions,
-	.nr_partitions		= ARRAY_SIZE(spitz_nand_partitions),
+	.part_parsers		= probes,
 };
 
 static struct resource spitz_nand_resources[] = {
@@ -834,14 +825,7 @@ static struct platform_device spitz_nand_device = {
 
 static void __init spitz_nand_init(void)
 {
-	if (machine_is_spitz()) {
-		spitz_nand_partitions[1].size = 5 * 1024 * 1024;
-	} else if (machine_is_akita()) {
-		spitz_nand_partitions[1].size = 58 * 1024 * 1024;
-		spitz_nand_bbt.len = 1;
-		spitz_nand_pdata.ecc_layout = &akita_ooblayout_ops;
-	} else if (machine_is_borzoi()) {
-		spitz_nand_partitions[1].size = 32 * 1024 * 1024;
+	if (machine_is_akita() || machine_is_borzoi()) {
 		spitz_nand_bbt.len = 1;
 		spitz_nand_pdata.ecc_layout = &akita_ooblayout_ops;
 	}
diff --git a/arch/arm/mach-pxa/stargate2.c b/arch/arm/mach-pxa/stargate2.c
index 6b7df6fd2448..df62bb23dbee 100644
--- a/arch/arm/mach-pxa/stargate2.c
+++ b/arch/arm/mach-pxa/stargate2.c
@@ -25,7 +25,7 @@
 #include <linux/mtd/plat-ram.h>
 #include <linux/mtd/partitions.h>
 
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/platform_data/pcf857x.h>
 #include <linux/platform_data/at24.h>
 #include <linux/smc91x.h>
diff --git a/arch/arm/mach-pxa/tosa-bt.c b/arch/arm/mach-pxa/tosa-bt.c
index 107f37210fb9..83606087edc7 100644
--- a/arch/arm/mach-pxa/tosa-bt.c
+++ b/arch/arm/mach-pxa/tosa-bt.c
@@ -132,3 +132,7 @@ static struct platform_driver tosa_bt_driver = {
 	},
 };
 module_platform_driver(tosa_bt_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Dmitry Baryshkov");
+MODULE_DESCRIPTION("Bluetooth built-in chip control");
diff --git a/arch/arm/mach-pxa/tosa.c b/arch/arm/mach-pxa/tosa.c
index 6a386fd6363e..cb5cd8e78c94 100644
--- a/arch/arm/mach-pxa/tosa.c
+++ b/arch/arm/mach-pxa/tosa.c
@@ -35,7 +35,7 @@
 #include <linux/spi/spi.h>
 #include <linux/spi/pxa2xx_spi.h>
 #include <linux/input/matrix_keypad.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/usb/gpio_vbus.h>
 #include <linux/reboot.h>
 #include <linux/memblock.h>
@@ -673,24 +673,6 @@ static int tosa_tc6393xb_suspend(struct platform_device *dev)
 	return 0;
 }
 
-static struct mtd_partition tosa_nand_partition[] = {
-	{
-		.name	= "smf",
-		.offset	= 0,
-		.size	= 7 * 1024 * 1024,
-	},
-	{
-		.name	= "root",
-		.offset	= MTDPART_OFS_APPEND,
-		.size	= 28 * 1024 * 1024,
-	},
-	{
-		.name	= "home",
-		.offset	= MTDPART_OFS_APPEND,
-		.size	= MTDPART_SIZ_FULL,
-	},
-};
-
 static uint8_t scan_ff_pattern[] = { 0xff, 0xff };
 
 static struct nand_bbt_descr tosa_tc6393xb_nand_bbt = {
@@ -700,10 +682,16 @@ static struct nand_bbt_descr tosa_tc6393xb_nand_bbt = {
 	.pattern	= scan_ff_pattern
 };
 
+static const char * const probes[] = {
+	"cmdlinepart",
+	"ofpart",
+	"sharpslpart",
+	NULL,
+};
+
 static struct tmio_nand_data tosa_tc6393xb_nand_config = {
-	.num_partitions	= ARRAY_SIZE(tosa_nand_partition),
-	.partition	= tosa_nand_partition,
 	.badblock_pattern = &tosa_tc6393xb_nand_bbt,
+	.part_parsers = probes,
 };
 
 static int tosa_tc6393xb_setup(struct platform_device *dev)
diff --git a/arch/arm/mach-pxa/trizeps4.c b/arch/arm/mach-pxa/trizeps4.c
index 3dd13b44c311..55b8c501b6fc 100644
--- a/arch/arm/mach-pxa/trizeps4.c
+++ b/arch/arm/mach-pxa/trizeps4.c
@@ -28,7 +28,7 @@
 #include <linux/mtd/physmap.h>
 #include <linux/mtd/partitions.h>
 #include <linux/regulator/machine.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/types.h>
 #include <asm/setup.h>
diff --git a/arch/arm/mach-pxa/viper.c b/arch/arm/mach-pxa/viper.c
index 4185e7ff073f..90d0f277de55 100644
--- a/arch/arm/mach-pxa/viper.c
+++ b/arch/arm/mach-pxa/viper.c
@@ -37,7 +37,7 @@
 #include <linux/jiffies.h>
 #include <linux/i2c-gpio.h>
 #include <linux/gpio/machine.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/serial_8250.h>
 #include <linux/smc91x.h>
 #include <linux/pwm.h>
diff --git a/arch/arm/mach-pxa/vpac270.c b/arch/arm/mach-pxa/vpac270.c
index 70ab3ad28237..f65dfb6e20e2 100644
--- a/arch/arm/mach-pxa/vpac270.c
+++ b/arch/arm/mach-pxa/vpac270.c
@@ -27,7 +27,7 @@
 #include <linux/ata_platform.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/max1586.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
diff --git a/arch/arm/mach-pxa/xcep.c b/arch/arm/mach-pxa/xcep.c
index 056369ef250e..c368c98584c0 100644
--- a/arch/arm/mach-pxa/xcep.c
+++ b/arch/arm/mach-pxa/xcep.c
@@ -16,7 +16,7 @@
 
 #include <linux/platform_device.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/smc91x.h>
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/partitions.h>
@@ -32,6 +32,7 @@
 #include <mach/smemc.h>
 
 #include "generic.h"
+#include "devices.h"
 
 #define XCEP_ETH_PHYS		(PXA_CS3_PHYS + 0x00000300)
 #define XCEP_ETH_PHYS_END	(PXA_CS3_PHYS + 0x000fffff)
diff --git a/arch/arm/mach-pxa/z2.c b/arch/arm/mach-pxa/z2.c
index 510e533871f3..6fffcfc4621e 100644
--- a/arch/arm/mach-pxa/z2.c
+++ b/arch/arm/mach-pxa/z2.c
@@ -30,7 +30,7 @@
 #include <linux/gpio_keys.h>
 #include <linux/delay.h>
 #include <linux/regulator/machine.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/mach-types.h>
 #include <asm/mach/arch.h>
diff --git a/arch/arm/mach-pxa/zeus.c b/arch/arm/mach-pxa/zeus.c
index ecbcaee5a2d5..e3851795d6d7 100644
--- a/arch/arm/mach-pxa/zeus.c
+++ b/arch/arm/mach-pxa/zeus.c
@@ -26,7 +26,7 @@
 #include <linux/mtd/partitions.h>
 #include <linux/mtd/physmap.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/platform_data/pca953x.h>
 #include <linux/apm-emulation.h>
 #include <linux/can/platform/mcp251x.h>
@@ -40,6 +40,7 @@
 #include <asm/mach/map.h>
 
 #include "pxa27x.h"
+#include "devices.h"
 #include <mach/regs-uart.h>
 #include <linux/platform_data/usb-ohci-pxa27x.h>
 #include <linux/platform_data/mmc-pxamci.h>
diff --git a/arch/arm/mach-pxa/zylonite_pxa300.c b/arch/arm/mach-pxa/zylonite_pxa300.c
index e247acf1400a..0ff4e218080f 100644
--- a/arch/arm/mach-pxa/zylonite_pxa300.c
+++ b/arch/arm/mach-pxa/zylonite_pxa300.c
@@ -17,11 +17,12 @@
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/i2c.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/platform_data/pca953x.h>
 #include <linux/gpio.h>
 
 #include "pxa300.h"
+#include "devices.h"
 #include "zylonite.h"
 
 #include "generic.h"
diff --git a/arch/arm/mach-s3c24xx/Kconfig b/arch/arm/mach-s3c24xx/Kconfig
index b198be7d32b6..686f0bbde998 100644
--- a/arch/arm/mach-s3c24xx/Kconfig
+++ b/arch/arm/mach-s3c24xx/Kconfig
@@ -1,11 +1,9 @@
-# arch/arm/mach-s3c24xx/Kconfig
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (c) 2012 Samsung Electronics Co., Ltd.
 #		http://www.samsung.com/
 #
 # Copyright 2007 Simtec Electronics
-#
-# Licensed under GPLv2
 
 if ARCH_S3C24XX
 
diff --git a/arch/arm/mach-s3c24xx/Makefile b/arch/arm/mach-s3c24xx/Makefile
index 8ac2f58a3480..6692f2de71b2 100644
--- a/arch/arm/mach-s3c24xx/Makefile
+++ b/arch/arm/mach-s3c24xx/Makefile
@@ -1,11 +1,9 @@
-# arch/arm/mach-s3c24xx/Makefile
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (c) 2012 Samsung Electronics Co., Ltd.
 #		http://www.samsung.com/
 #
 # Copyright 2007 Simtec Electronics
-#
-# Licensed under GPLv2
 
 # core
 
diff --git a/arch/arm/mach-s3c24xx/Makefile.boot b/arch/arm/mach-s3c24xx/Makefile.boot
index 4457605ba04a..7f19e226035e 100644
--- a/arch/arm/mach-s3c24xx/Makefile.boot
+++ b/arch/arm/mach-s3c24xx/Makefile.boot
@@ -1,3 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
 ifeq ($(CONFIG_PM_H1940),y)
 	zreladdr-y	+= 0x30108000
 	params_phys-y	:= 0x30100100
diff --git a/arch/arm/mach-s3c24xx/anubis.h b/arch/arm/mach-s3c24xx/anubis.h
index 2691665f27d9..13847292e6c7 100644
--- a/arch/arm/mach-s3c24xx/anubis.h
+++ b/arch/arm/mach-s3c24xx/anubis.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2005 Simtec Electronics
  *	http://www.simtec.co.uk/products/
@@ -6,11 +7,7 @@
  * ANUBIS - CPLD control constants
  * ANUBIS - IRQ Number definitions
  * ANUBIS - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C24XX_ANUBIS_H
 #define __MACH_S3C24XX_ANUBIS_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/bast-ide.c b/arch/arm/mach-s3c24xx/bast-ide.c
index 3f0288f2f542..067944398f46 100644
--- a/arch/arm/mach-s3c24xx/bast-ide.c
+++ b/arch/arm/mach-s3c24xx/bast-ide.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2410/bast-ide.c
- *
- * Copyright 2007 Simtec Electronics
- *	http://www.simtec.co.uk/products/EB2410ITX/
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2007 Simtec Electronics
+//	http://www.simtec.co.uk/products/EB2410ITX/
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/bast-irq.c b/arch/arm/mach-s3c24xx/bast-irq.c
index ad8f4cd7c327..03728058d58d 100644
--- a/arch/arm/mach-s3c24xx/bast-irq.c
+++ b/arch/arm/mach-s3c24xx/bast-irq.c
@@ -1,25 +1,9 @@
-/* linux/arch/arm/mach-s3c2410/bast-irq.c
- *
- * Copyright 2003-2005 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.simtec.co.uk/products/EB2410ITX/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
-
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2003-2005 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.simtec.co.uk/products/EB2410ITX/
 
 #include <linux/init.h>
 #include <linux/module.h>
diff --git a/arch/arm/mach-s3c24xx/bast.h b/arch/arm/mach-s3c24xx/bast.h
index 5c7534bae92d..a7726f93f5eb 100644
--- a/arch/arm/mach-s3c24xx/bast.h
+++ b/arch/arm/mach-s3c24xx/bast.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2003-2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
@@ -5,11 +6,7 @@
  * BAST - CPLD control constants
  * BAST - IRQ Number definitions
  * BAST - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C24XX_BAST_H
 #define __MACH_S3C24XX_BAST_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/common-smdk.c b/arch/arm/mach-s3c24xx/common-smdk.c
index 0e116c92bf01..58e30cad386c 100644
--- a/arch/arm/mach-s3c24xx/common-smdk.c
+++ b/arch/arm/mach-s3c24xx/common-smdk.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c24xx/common-smdk.c
- *
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Common code for SMDK2410 and SMDK2440 boards
- *
- * http://www.fluff.org/ben/smdk2440/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Common code for SMDK2410 and SMDK2440 boards
+//
+// http://www.fluff.org/ben/smdk2440/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/common-smdk.h b/arch/arm/mach-s3c24xx/common-smdk.h
index 98f733e1cb42..c0352b06e435 100644
--- a/arch/arm/mach-s3c24xx/common-smdk.h
+++ b/arch/arm/mach-s3c24xx/common-smdk.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2006 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
@@ -5,10 +6,6 @@
  * Common code for SMDK2410 and SMDK2440 boards
  *
  * http://www.fluff.org/ben/smdk2440/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 extern void smdk_machine_init(void);
diff --git a/arch/arm/mach-s3c24xx/common.c b/arch/arm/mach-s3c24xx/common.c
index 5b6b94ef41e2..3dc029c2d2cb 100644
--- a/arch/arm/mach-s3c24xx/common.c
+++ b/arch/arm/mach-s3c24xx/common.c
@@ -1,25 +1,10 @@
-/* linux/arch/arm/plat-s3c24xx/cpu.c
- *
- * Copyright (c) 2004-2005 Simtec Electronics
- *	http://www.simtec.co.uk/products/SWLINUX/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Common code for S3C24XX machines
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2004-2005 Simtec Electronics
+//	http://www.simtec.co.uk/products/SWLINUX/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Common code for S3C24XX machines
 
 #include <linux/dma-mapping.h>
 #include <linux/init.h>
diff --git a/arch/arm/mach-s3c24xx/common.h b/arch/arm/mach-s3c24xx/common.h
index c7ac7e61a22e..d087b20e8857 100644
--- a/arch/arm/mach-s3c24xx/common.h
+++ b/arch/arm/mach-s3c24xx/common.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
  * Common Header for S3C24XX SoCs
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ARCH_ARM_MACH_S3C24XX_COMMON_H
diff --git a/arch/arm/mach-s3c24xx/cpufreq-utils.c b/arch/arm/mach-s3c24xx/cpufreq-utils.c
index d4d9514335f4..1a7f38d085dd 100644
--- a/arch/arm/mach-s3c24xx/cpufreq-utils.c
+++ b/arch/arm/mach-s3c24xx/cpufreq-utils.c
@@ -1,14 +1,10 @@
-/*
- * Copyright (c) 2009 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C24XX CPU Frequency scaling - utils for S3C2410/S3C2440/S3C2442
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2009 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C24XX CPU Frequency scaling - utils for S3C2410/S3C2440/S3C2442
 
 #include <linux/kernel.h>
 #include <linux/errno.h>
diff --git a/arch/arm/mach-s3c24xx/fb-core.h b/arch/arm/mach-s3c24xx/fb-core.h
index 103bdbaddd55..1821e820262c 100644
--- a/arch/arm/mach-s3c24xx/fb-core.h
+++ b/arch/arm/mach-s3c24xx/fb-core.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2010 Samsung Electronics Co., Ltd.
  *	Pawel Osciak <p.osciak@samsung.com>
  *
  * Samsung framebuffer driver core functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 #ifndef __ASM_PLAT_FB_CORE_H
 #define __ASM_PLAT_FB_CORE_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/gta02.h b/arch/arm/mach-s3c24xx/gta02.h
index 9430a71e9184..d5610ba829a4 100644
--- a/arch/arm/mach-s3c24xx/gta02.h
+++ b/arch/arm/mach-s3c24xx/gta02.h
@@ -1,10 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * GTA02 header
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C24XX_GTA02_H
 #define __MACH_S3C24XX_GTA02_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/h1940-bluetooth.c b/arch/arm/mach-s3c24xx/h1940-bluetooth.c
index 9c8b1279a4ba..46ad20ea87d1 100644
--- a/arch/arm/mach-s3c24xx/h1940-bluetooth.c
+++ b/arch/arm/mach-s3c24xx/h1940-bluetooth.c
@@ -1,14 +1,8 @@
-/*
- * arch/arm/mach-s3c2410/h1940-bluetooth.c
- * Copyright (c) Arnaud Patard <arnaud.patard@rtp-net.org>
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file COPYING in the main directory of this archive for
- * more details.
- *
- *	    S3C2410 bluetooth "driver"
- *
- */
+// SPDX-License-Identifier: GPL-1.0
+//
+// Copyright (c) Arnaud Patard <arnaud.patard@rtp-net.org>
+//
+//	    S3C2410 bluetooth "driver"
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
diff --git a/arch/arm/mach-s3c24xx/h1940.h b/arch/arm/mach-s3c24xx/h1940.h
index 596d9f64c5b6..5dfe9d10cd15 100644
--- a/arch/arm/mach-s3c24xx/h1940.h
+++ b/arch/arm/mach-s3c24xx/h1940.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2006 Ben Dooks <ben-linux@fluff.org>
  *
@@ -6,11 +7,7 @@
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * iPAQ H1940 series definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C24XX_H1940_H
 #define __MACH_S3C24XX_H1940_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/include/mach/dma.h b/arch/arm/mach-s3c24xx/include/mach/dma.h
index 9e8117198e0c..25fc9c258fc1 100644
--- a/arch/arm/mach-s3c24xx/include/mach/dma.h
+++ b/arch/arm/mach-s3c24xx/include/mach/dma.h
@@ -1,14 +1,10 @@
-/* arch/arm/mach-s3c2410/include/mach/dma.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (C) 2003-2006 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * Samsung S3C24XX DMA support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_DMA_H
 #define __ASM_ARCH_DMA_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/include/mach/fb.h b/arch/arm/mach-s3c24xx/include/mach/fb.h
index a957bc8ed44f..4e539cb8b884 100644
--- a/arch/arm/mach-s3c24xx/include/mach/fb.h
+++ b/arch/arm/mach-s3c24xx/include/mach/fb.h
@@ -1 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 #include <plat/fb-s3c2410.h>
diff --git a/arch/arm/mach-s3c24xx/include/mach/gpio-samsung.h b/arch/arm/mach-s3c24xx/include/mach/gpio-samsung.h
index 528fcdc4f63e..2ad22b2d459b 100644
--- a/arch/arm/mach-s3c24xx/include/mach/gpio-samsung.h
+++ b/arch/arm/mach-s3c24xx/include/mach/gpio-samsung.h
@@ -1,14 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C2410 - GPIO lib support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /* some boards require extra gpio capacity to support external
  * devices that need GPIO.
diff --git a/arch/arm/mach-s3c24xx/include/mach/hardware.h b/arch/arm/mach-s3c24xx/include/mach/hardware.h
index dedd3837c193..1b2975708e3f 100644
--- a/arch/arm/mach-s3c24xx/include/mach/hardware.h
+++ b/arch/arm/mach-s3c24xx/include/mach/hardware.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2003 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C2410 - hardware
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_HARDWARE_H
 #define __ASM_ARCH_HARDWARE_H
diff --git a/arch/arm/mach-s3c24xx/include/mach/irqs.h b/arch/arm/mach-s3c24xx/include/mach/irqs.h
index b6dd4cb5a2ec..aaf3bae08b52 100644
--- a/arch/arm/mach-s3c24xx/include/mach/irqs.h
+++ b/arch/arm/mach-s3c24xx/include/mach/irqs.h
@@ -1,12 +1,8 @@
-/* arch/arm/mach-s3c2410/include/mach/irqs.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2003-2005 Simtec Electronics
  *   Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 
 #ifndef __ASM_ARCH_IRQS_H
diff --git a/arch/arm/mach-s3c24xx/include/mach/map.h b/arch/arm/mach-s3c24xx/include/mach/map.h
index adc39043aa21..bca93112f57d 100644
--- a/arch/arm/mach-s3c24xx/include/mach/map.h
+++ b/arch/arm/mach-s3c24xx/include/mach/map.h
@@ -1,14 +1,10 @@
-/* arch/arm/mach-s3c2410/include/mach/map.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2003 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C2410 - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_MAP_H
 #define __ASM_ARCH_MAP_H
diff --git a/arch/arm/mach-s3c24xx/include/mach/pm-core.h b/arch/arm/mach-s3c24xx/include/mach/pm-core.h
index 712333fec589..5e4ce89d0158 100644
--- a/arch/arm/mach-s3c24xx/include/mach/pm-core.h
+++ b/arch/arm/mach-s3c24xx/include/mach/pm-core.h
@@ -1,15 +1,12 @@
-/* linux/arch/arm/mach-s3c2410/include/pm-core.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Simtec Electronics
  *      Ben Dooks <ben@simtec.co.uk>
  *      http://armlinux.simtec.co.uk/
  *
  * S3C24xx - PM core support for arch/arm/plat-s3c/pm.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
+
 #include <linux/delay.h>
 #include <linux/io.h>
 
diff --git a/arch/arm/mach-s3c24xx/include/mach/regs-clock.h b/arch/arm/mach-s3c24xx/include/mach/regs-clock.h
index ae4a3e0f3ba2..7ca3dd4f13c0 100644
--- a/arch/arm/mach-s3c24xx/include/mach/regs-clock.h
+++ b/arch/arm/mach-s3c24xx/include/mach/regs-clock.h
@@ -1,14 +1,10 @@
-/* arch/arm/mach-s3c2410/include/mach/regs-clock.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2003-2006 Simtec Electronics <linux@simtec.co.uk>
  *	http://armlinux.simtec.co.uk/
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2410 clock register definitions
-*/
+ */
 
 #ifndef __ASM_ARM_REGS_CLOCK
 #define __ASM_ARM_REGS_CLOCK
diff --git a/arch/arm/mach-s3c24xx/include/mach/regs-gpio.h b/arch/arm/mach-s3c24xx/include/mach/regs-gpio.h
index 0d622f3b57a5..594e967c0673 100644
--- a/arch/arm/mach-s3c24xx/include/mach/regs-gpio.h
+++ b/arch/arm/mach-s3c24xx/include/mach/regs-gpio.h
@@ -1,14 +1,10 @@
-/* arch/arm/mach-s3c2410/include/mach/regs-gpio.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2003-2004 Simtec Electronics <linux@simtec.co.uk>
  *	http://www.simtec.co.uk/products/SWLINUX/
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2410 GPIO register definitions
-*/
+ */
 
 
 #ifndef __ASM_ARCH_REGS_GPIO_H
diff --git a/arch/arm/mach-s3c24xx/include/mach/regs-irq.h b/arch/arm/mach-s3c24xx/include/mach/regs-irq.h
index 0f07ba30b1fb..8d8e669e3903 100644
--- a/arch/arm/mach-s3c24xx/include/mach/regs-irq.h
+++ b/arch/arm/mach-s3c24xx/include/mach/regs-irq.h
@@ -1,12 +1,8 @@
-/* arch/arm/mach-s3c2410/include/mach/regs-irq.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2003 Simtec Electronics <linux@simtec.co.uk>
  *		      http://www.simtec.co.uk/products/SWLINUX/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 
 #ifndef ___ASM_ARCH_REGS_IRQ_H
diff --git a/arch/arm/mach-s3c24xx/include/mach/regs-lcd.h b/arch/arm/mach-s3c24xx/include/mach/regs-lcd.h
index ee8f040aff5f..4c3434f261bb 100644
--- a/arch/arm/mach-s3c24xx/include/mach/regs-lcd.h
+++ b/arch/arm/mach-s3c24xx/include/mach/regs-lcd.h
@@ -1,13 +1,8 @@
-/* arch/arm/mach-s3c2410/include/mach/regs-lcd.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2003 Simtec Electronics <linux@simtec.co.uk>
  *		      http://www.simtec.co.uk/products/SWLINUX/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
-
+ */
 
 #ifndef ___ASM_ARCH_REGS_LCD_H
 #define ___ASM_ARCH_REGS_LCD_H
diff --git a/arch/arm/mach-s3c24xx/include/mach/regs-s3c2443-clock.h b/arch/arm/mach-s3c24xx/include/mach/regs-s3c2443-clock.h
index ffe37bdb9f59..6bf924612b06 100644
--- a/arch/arm/mach-s3c24xx/include/mach/regs-s3c2443-clock.h
+++ b/arch/arm/mach-s3c24xx/include/mach/regs-s3c2443-clock.h
@@ -1,15 +1,11 @@
-/* arch/arm/mach-s3c2410/include/mach/regs-s3c2443-clock.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2007 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *	http://armlinux.simtec.co.uk/
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2443 clock register definitions
-*/
+ */
 
 #ifndef __ASM_ARM_REGS_S3C2443_CLOCK
 #define __ASM_ARM_REGS_S3C2443_CLOCK
diff --git a/arch/arm/mach-s3c24xx/include/mach/rtc-core.h b/arch/arm/mach-s3c24xx/include/mach/rtc-core.h
index 4d5f5768f700..88510333b96b 100644
--- a/arch/arm/mach-s3c24xx/include/mach/rtc-core.h
+++ b/arch/arm/mach-s3c24xx/include/mach/rtc-core.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2011 Heiko Stuebner <heiko@sntech.de>
  *
  * Samsung RTC Controller core functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __RTC_CORE_H
 #define __RTC_CORE_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/include/mach/s3c2412.h b/arch/arm/mach-s3c24xx/include/mach/s3c2412.h
index 548ced42cbb7..b6b32724ace8 100644
--- a/arch/arm/mach-s3c24xx/include/mach/s3c2412.h
+++ b/arch/arm/mach-s3c24xx/include/mach/s3c2412.h
@@ -1,11 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2008 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ARCH_ARM_MACH_S3C24XX_S3C2412_H
diff --git a/arch/arm/mach-s3c24xx/iotiming-s3c2410.c b/arch/arm/mach-s3c24xx/iotiming-s3c2410.c
index d5f1f06e4811..9f90aaf70bf3 100644
--- a/arch/arm/mach-s3c24xx/iotiming-s3c2410.c
+++ b/arch/arm/mach-s3c24xx/iotiming-s3c2410.c
@@ -1,14 +1,10 @@
-/*
- * Copyright (c) 2006-2009 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C24XX CPU Frequency scaling - IO timing for S3C2410/S3C2440/S3C2442
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006-2009 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C24XX CPU Frequency scaling - IO timing for S3C2410/S3C2440/S3C2442
 
 #include <linux/init.h>
 #include <linux/kernel.h>
diff --git a/arch/arm/mach-s3c24xx/iotiming-s3c2412.c b/arch/arm/mach-s3c24xx/iotiming-s3c2412.c
index c5b12f6b02b5..59356d10fbcf 100644
--- a/arch/arm/mach-s3c24xx/iotiming-s3c2412.c
+++ b/arch/arm/mach-s3c24xx/iotiming-s3c2412.c
@@ -1,14 +1,10 @@
-/*
- * Copyright (c) 2006-2008 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C2412/S3C2443 (PL093 based) IO timing support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006-2008 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C2412/S3C2443 (PL093 based) IO timing support
 
 #include <linux/init.h>
 #include <linux/module.h>
diff --git a/arch/arm/mach-s3c24xx/irq-pm.c b/arch/arm/mach-s3c24xx/irq-pm.c
index 417b7a20c2d1..e0131b16a4af 100644
--- a/arch/arm/mach-s3c24xx/irq-pm.c
+++ b/arch/arm/mach-s3c24xx/irq-pm.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/plat-s3c24xx/irq-om.c
- *
- * Copyright (c) 2003-2004 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * S3C24XX - IRQ PM code
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2003-2004 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// S3C24XX - IRQ PM code
 
 #include <linux/init.h>
 #include <linux/module.h>
diff --git a/arch/arm/mach-s3c24xx/mach-amlm5900.c b/arch/arm/mach-s3c24xx/mach-amlm5900.c
index 3e63777a109f..9a9daf526d0c 100644
--- a/arch/arm/mach-s3c24xx/mach-amlm5900.c
+++ b/arch/arm/mach-s3c24xx/mach-amlm5900.c
@@ -1,30 +1,11 @@
-/* linux/arch/arm/mach-s3c2410/mach-amlm5900.c
- *
- * linux/arch/arm/mach-s3c2410/mach-amlm5900.c
- *
- * Copyright (c) 2006 American Microsystems Limited
- *	David Anders <danders@amltd.com>
-
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
- * MA 02111-1307 USA
- *
- * @History:
- * derived from linux/arch/arm/mach-s3c2410/mach-bast.c, written by
- * Ben Dooks <ben@simtec.co.uk>
- *
- ***********************************************************************/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2006 American Microsystems Limited
+//	David Anders <danders@amltd.com>
+//
+// @History:
+// derived from linux/arch/arm/mach-s3c2410/mach-bast.c, written by
+// Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-anubis.c b/arch/arm/mach-s3c24xx/mach-anubis.c
index c14cab361922..072966dcad78 100644
--- a/arch/arm/mach-s3c24xx/mach-anubis.c
+++ b/arch/arm/mach-s3c24xx/mach-anubis.c
@@ -1,13 +1,8 @@
-/* linux/arch/arm/mach-s3c2440/mach-anubis.c
- *
- * Copyright 2003-2009 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2003-2009 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-at2440evb.c b/arch/arm/mach-s3c24xx/mach-at2440evb.c
index ebdbafb9382a..68a4fa94257a 100644
--- a/arch/arm/mach-s3c24xx/mach-at2440evb.c
+++ b/arch/arm/mach-s3c24xx/mach-at2440evb.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/mach-s3c2440/mach-at2440evb.c
- *
- * Copyright (c) 2008 Ramax Lo <ramaxlo@gmail.com>
- *      Based on mach-anubis.c by Ben Dooks <ben@simtec.co.uk>
- *      and modifications by SBZ <sbz@spgui.org> and
- *      Weibing <http://weibing.blogbus.com>
- *
- * For product information, visit http://www.arm.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2008 Ramax Lo <ramaxlo@gmail.com>
+//      Based on mach-anubis.c by Ben Dooks <ben@simtec.co.uk>
+//      and modifications by SBZ <sbz@spgui.org> and
+//      Weibing <http://weibing.blogbus.com>
+//
+// For product information, visit http://www.arm.com/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-bast.c b/arch/arm/mach-s3c24xx/mach-bast.c
index 704dc84b3480..a7c3955ae8f6 100644
--- a/arch/arm/mach-s3c24xx/mach-bast.c
+++ b/arch/arm/mach-s3c24xx/mach-bast.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2410/mach-bast.c
- *
- * Copyright 2003-2008 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.simtec.co.uk/products/EB2410ITX/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2003-2008 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.simtec.co.uk/products/EB2410ITX/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-gta02.c b/arch/arm/mach-s3c24xx/mach-gta02.c
index afe18baf0c84..9d5595c4ad99 100644
--- a/arch/arm/mach-s3c24xx/mach-gta02.c
+++ b/arch/arm/mach-s3c24xx/mach-gta02.c
@@ -1,27 +1,12 @@
-/*
- * S3C2442 Machine Support for Openmoko GTA02 / FreeRunner.
- *
- * Copyright (C) 2006-2009 by Openmoko, Inc.
- * Authors: Harald Welte <laforge@openmoko.org>
- *          Andy Green <andy@openmoko.org>
- *          Werner Almesberger <werner@openmoko.org>
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
- * MA 02111-1307 USA
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// S3C2442 Machine Support for Openmoko GTA02 / FreeRunner.
+//
+// Copyright (C) 2006-2009 by Openmoko, Inc.
+// Authors: Harald Welte <laforge@openmoko.org>
+//          Andy Green <andy@openmoko.org>
+//          Werner Almesberger <werner@openmoko.org>
+// All rights reserved.
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-h1940.c b/arch/arm/mach-s3c24xx/mach-h1940.c
index 7ed78619217c..e064c73a57d3 100644
--- a/arch/arm/mach-s3c24xx/mach-h1940.c
+++ b/arch/arm/mach-s3c24xx/mach-h1940.c
@@ -1,14 +1,9 @@
-/*
- * Copyright (c) 2003-2005 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.handhelds.org/projects/h1940.html
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2003-2005 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.handhelds.org/projects/h1940.html
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-jive.c b/arch/arm/mach-s3c24xx/mach-jive.c
index 17821976f769..a3ddbbbd6d92 100644
--- a/arch/arm/mach-s3c24xx/mach-jive.c
+++ b/arch/arm/mach-s3c24xx/mach-jive.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2410/mach-jive.c
- *
- * Copyright 2007 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2007 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://armlinux.simtec.co.uk/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-mini2440.c b/arch/arm/mach-s3c24xx/mach-mini2440.c
index 04c9f488c498..95753e0bc073 100644
--- a/arch/arm/mach-s3c24xx/mach-mini2440.c
+++ b/arch/arm/mach-s3c24xx/mach-mini2440.c
@@ -1,17 +1,12 @@
-/* linux/arch/arm/mach-s3c2440/mach-mini2440.c
- *
- * Copyright (c) 2008 Ramax Lo <ramaxlo@gmail.com>
- *      Based on mach-anubis.c by Ben Dooks <ben@simtec.co.uk>
- *      and modifications by SBZ <sbz@spgui.org> and
- *      Weibing <http://weibing.blogbus.com> and
- *      Michel Pollet <buserror@gmail.com>
- *
- * For product information, visit http://code.google.com/p/mini2440/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2008 Ramax Lo <ramaxlo@gmail.com>
+//      Based on mach-anubis.c by Ben Dooks <ben@simtec.co.uk>
+//      and modifications by SBZ <sbz@spgui.org> and
+//      Weibing <http://weibing.blogbus.com> and
+//      Michel Pollet <buserror@gmail.com>
+//
+// For product information, visit http://code.google.com/p/mini2440/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-n30.c b/arch/arm/mach-s3c24xx/mach-n30.c
index 070a0d0714a5..eec51fadb14a 100644
--- a/arch/arm/mach-s3c24xx/mach-n30.c
+++ b/arch/arm/mach-s3c24xx/mach-n30.c
@@ -1,18 +1,15 @@
-/* Machine specific code for the Acer n30, Acer N35, Navman PiN 570,
- * Yakumo AlphaX and Airis NC05 PDAs.
- *
- * Copyright (c) 2003-2005 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Copyright (c) 2005-2008 Christer Weinigel <christer@weinigel.se>
- *
- * There is a wiki with more information about the n30 port at
- * http://handhelds.org/moin/moin.cgi/AcerN30Documentation .
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Machine specific code for the Acer n30, Acer N35, Navman PiN 570,
+// Yakumo AlphaX and Airis NC05 PDAs.
+//
+// Copyright (c) 2003-2005 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Copyright (c) 2005-2008 Christer Weinigel <christer@weinigel.se>
+//
+// There is a wiki with more information about the n30 port at
+// http://handhelds.org/moin/moin.cgi/AcerN30Documentation .
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-nexcoder.c b/arch/arm/mach-s3c24xx/mach-nexcoder.c
index 2a61d13dcd6c..c2f34758ccb6 100644
--- a/arch/arm/mach-s3c24xx/mach-nexcoder.c
+++ b/arch/arm/mach-s3c24xx/mach-nexcoder.c
@@ -1,16 +1,12 @@
-/* linux/arch/arm/mach-s3c2440/mach-nexcoder.c
- *
- * Copyright (c) 2004 Nex Vision
- *   Guillaume GOURAT <guillaume.gourat@nexvision.tv>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * Modifications:
- *     15-10-2004 GG  Created initial version
- *     12-03-2005 BJD Updated for release
- */
+// SPDX-License-Identifier: GPL-2.0
+// linux/arch/arm/mach-s3c2440/mach-nexcoder.c
+//
+// Copyright (c) 2004 Nex Vision
+//   Guillaume GOURAT <guillaume.gourat@nexvision.tv>
+//
+// Modifications:
+//     15-10-2004 GG  Created initial version
+//     12-03-2005 BJD Updated for release
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-osiris-dvs.c b/arch/arm/mach-s3c24xx/mach-osiris-dvs.c
index 6cac7da15e2b..058ce73137e8 100644
--- a/arch/arm/mach-s3c24xx/mach-osiris-dvs.c
+++ b/arch/arm/mach-s3c24xx/mach-osiris-dvs.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/mach-s3c2440/mach-osiris-dvs.c
- *
- * Copyright (c) 2009 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Simtec Osiris Dynamic Voltage Scaling support.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2009 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Simtec Osiris Dynamic Voltage Scaling support.
 
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/arch/arm/mach-s3c24xx/mach-osiris.c b/arch/arm/mach-s3c24xx/mach-osiris.c
index ed3b22ceef06..ee3630cb236a 100644
--- a/arch/arm/mach-s3c24xx/mach-osiris.c
+++ b/arch/arm/mach-s3c24xx/mach-osiris.c
@@ -1,12 +1,8 @@
-/*
- * Copyright (c) 2005-2008 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2005-2008 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-otom.c b/arch/arm/mach-s3c24xx/mach-otom.c
index 345a484b93cc..4e24d89e870b 100644
--- a/arch/arm/mach-s3c24xx/mach-otom.c
+++ b/arch/arm/mach-s3c24xx/mach-otom.c
@@ -1,12 +1,7 @@
-/*
- *
- * Copyright (c) 2004 Nex Vision
- *   Guillaume GOURAT <guillaume.gourat@nexvision.fr>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2004 Nex Vision
+//   Guillaume GOURAT <guillaume.gourat@nexvision.fr>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-qt2410.c b/arch/arm/mach-s3c24xx/mach-qt2410.c
index 84e3a9c53184..9c8373b8d9c3 100644
--- a/arch/arm/mach-s3c24xx/mach-qt2410.c
+++ b/arch/arm/mach-s3c24xx/mach-qt2410.c
@@ -1,25 +1,8 @@
-/* linux/arch/arm/mach-s3c2410/mach-qt2410.c
- *
- * Copyright (C) 2006 by OpenMoko, Inc.
- * Author: Harald Welte <laforge@openmoko.org>
- * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
- * MA 02111-1307 USA
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2006 by OpenMoko, Inc.
+// Author: Harald Welte <laforge@openmoko.org>
+// All rights reserved.
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-rx1950.c b/arch/arm/mach-s3c24xx/mach-rx1950.c
index e86ad6a68a0b..7f5a18fa305b 100644
--- a/arch/arm/mach-s3c24xx/mach-rx1950.c
+++ b/arch/arm/mach-s3c24xx/mach-rx1950.c
@@ -1,14 +1,9 @@
-/*
- * Copyright (c) 2006-2009 Victor Chukhantsev, Denis Grigoriev,
- * Copyright (c) 2007-2010 Vasily Khoruzhick
- *
- * based on smdk2440 written by Ben Dooks
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006-2009 Victor Chukhantsev, Denis Grigoriev,
+// Copyright (c) 2007-2010 Vasily Khoruzhick
+//
+// based on smdk2440 written by Ben Dooks
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-rx3715.c b/arch/arm/mach-s3c24xx/mach-rx3715.c
index b5ba615cf9dd..529c6faf862f 100644
--- a/arch/arm/mach-s3c24xx/mach-rx3715.c
+++ b/arch/arm/mach-s3c24xx/mach-rx3715.c
@@ -1,15 +1,9 @@
-/* linux/arch/arm/mach-s3c2440/mach-rx3715.c
- *
- * Copyright (c) 2003-2004 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.handhelds.org/projects/rx3715.html
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2003-2004 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.handhelds.org/projects/rx3715.html
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-s3c2416-dt.c b/arch/arm/mach-s3c24xx/mach-s3c2416-dt.c
index c83c076578dd..aa7102713b37 100644
--- a/arch/arm/mach-s3c24xx/mach-s3c2416-dt.c
+++ b/arch/arm/mach-s3c24xx/mach-s3c2416-dt.c
@@ -1,19 +1,15 @@
-/*
- * Samsung's S3C2416 flattened device tree enabled machine
- *
- * Copyright (c) 2012 Heiko Stuebner <heiko@sntech.de>
- *
- * based on mach-exynos/mach-exynos4-dt.c
- *
- * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- * Copyright (c) 2010-2011 Linaro Ltd.
- *		www.linaro.org
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Samsung's S3C2416 flattened device tree enabled machine
+//
+// Copyright (c) 2012 Heiko Stuebner <heiko@sntech.de>
+//
+// based on mach-exynos/mach-exynos4-dt.c
+//
+// Copyright (c) 2010-2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+// Copyright (c) 2010-2011 Linaro Ltd.
+//		www.linaro.org
 
 #include <linux/clocksource.h>
 #include <linux/irqchip.h>
diff --git a/arch/arm/mach-s3c24xx/mach-smdk2410.c b/arch/arm/mach-s3c24xx/mach-smdk2410.c
index 27dd6605e395..18dfef52c8bf 100644
--- a/arch/arm/mach-s3c24xx/mach-smdk2410.c
+++ b/arch/arm/mach-s3c24xx/mach-smdk2410.c
@@ -1,32 +1,13 @@
-/* linux/arch/arm/mach-s3c2410/mach-smdk2410.c
- *
- * linux/arch/arm/mach-s3c2410/mach-smdk2410.c
- *
- * Copyright (C) 2004 by FS Forth-Systeme GmbH
- * All rights reserved.
- *
- * @Author: Jonas Dietsche
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
- * MA 02111-1307 USA
- *
- * @History:
- * derived from linux/arch/arm/mach-s3c2410/mach-bast.c, written by
- * Ben Dooks <ben@simtec.co.uk>
- *
- ***********************************************************************/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2004 by FS Forth-Systeme GmbH
+// All rights reserved.
+//
+// @Author: Jonas Dietsche
+//
+// @History:
+// derived from linux/arch/arm/mach-s3c2410/mach-bast.c, written by
+// Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-smdk2413.c b/arch/arm/mach-s3c24xx/mach-smdk2413.c
index 586e4a3b8d5d..ca80167f268d 100644
--- a/arch/arm/mach-s3c24xx/mach-smdk2413.c
+++ b/arch/arm/mach-s3c24xx/mach-smdk2413.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/mach-s3c2412/mach-smdk2413.c
- *
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Thanks to Dimity Andric (TomTom) and Steven Ryu (Samsung) for the
- * loans of SMDK2413 to work with.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Thanks to Dimity Andric (TomTom) and Steven Ryu (Samsung) for the
+// loans of SMDK2413 to work with.
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-smdk2416.c b/arch/arm/mach-s3c24xx/mach-smdk2416.c
index 86394f72d29e..61c3e45898d3 100644
--- a/arch/arm/mach-s3c24xx/mach-smdk2416.c
+++ b/arch/arm/mach-s3c24xx/mach-smdk2416.c
@@ -1,15 +1,9 @@
-/* linux/arch/arm/mach-s3c2416/mach-hanlin_v3c.c
- *
- * Copyright (c) 2009 Yauhen Kharuzhy <jekhor@gmail.com>,
- *	as part of OpenInkpot project
- * Copyright (c) 2009 Promwad Innovation Company
- *	Yauhen Kharuzhy <yauhen.kharuzhy@promwad.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2009 Yauhen Kharuzhy <jekhor@gmail.com>,
+//	as part of OpenInkpot project
+// Copyright (c) 2009 Promwad Innovation Company
+//	Yauhen Kharuzhy <yauhen.kharuzhy@promwad.com>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-smdk2440.c b/arch/arm/mach-s3c24xx/mach-smdk2440.c
index 9bb96bfbb420..7bafcd8ea104 100644
--- a/arch/arm/mach-s3c24xx/mach-smdk2440.c
+++ b/arch/arm/mach-s3c24xx/mach-smdk2440.c
@@ -1,17 +1,12 @@
-/* linux/arch/arm/mach-s3c2440/mach-smdk2440.c
- *
- * Copyright (c) 2004-2005 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.fluff.org/ben/smdk2440/
- *
- * Thanks to Dimity Andric and TomTom for the loan of an SMDK2440.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+// linux/arch/arm/mach-s3c2440/mach-smdk2440.c
+//
+// Copyright (c) 2004-2005 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.fluff.org/ben/smdk2440/
+//
+// Thanks to Dimity Andric and TomTom for the loan of an SMDK2440.
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-smdk2443.c b/arch/arm/mach-s3c24xx/mach-smdk2443.c
index 474cd81aa8ad..2358ed5ed7be 100644
--- a/arch/arm/mach-s3c24xx/mach-smdk2443.c
+++ b/arch/arm/mach-s3c24xx/mach-smdk2443.c
@@ -1,17 +1,11 @@
-/* linux/arch/arm/mach-s3c2443/mach-smdk2443.c
- *
- * Copyright (c) 2007 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.fluff.org/ben/smdk2443/
- *
- * Thanks to Samsung for the loan of an SMDK2443
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2007 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.fluff.org/ben/smdk2443/
+//
+// Thanks to Samsung for the loan of an SMDK2443
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-tct_hammer.c b/arch/arm/mach-s3c24xx/mach-tct_hammer.c
index 2deb62f92fb2..8d8ddd6ea305 100644
--- a/arch/arm/mach-s3c24xx/mach-tct_hammer.c
+++ b/arch/arm/mach-s3c24xx/mach-tct_hammer.c
@@ -1,28 +1,11 @@
-/* linux/arch/arm/mach-s3c2410/mach-tct_hammer.c
- *
- * Copyright (c) 2007 TinCanTools
- *	David Anders <danders@amltd.com>
-
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston,
- * MA 02111-1307 USA
- *
- * @History:
- * derived from linux/arch/arm/mach-s3c2410/mach-bast.c, written by
- * Ben Dooks <ben@simtec.co.uk>
- *
- ***********************************************************************/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2007 TinCanTools
+//	David Anders <danders@amltd.com>
+//
+// @History:
+// derived from linux/arch/arm/mach-s3c2410/mach-bast.c, written by
+// Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-vr1000.c b/arch/arm/mach-s3c24xx/mach-vr1000.c
index 89f32bd3f01b..853e74f9b8b5 100644
--- a/arch/arm/mach-s3c24xx/mach-vr1000.c
+++ b/arch/arm/mach-s3c24xx/mach-vr1000.c
@@ -1,15 +1,10 @@
-/*
- * Copyright (c) 2003-2008 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * Machine support for Thorcom VR1000 board. Designed for Thorcom by
- * Simtec Electronics, http://www.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2003-2008 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// Machine support for Thorcom VR1000 board. Designed for Thorcom by
+// Simtec Electronics, http://www.simtec.co.uk/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/mach-vstms.c b/arch/arm/mach-s3c24xx/mach-vstms.c
index 1adc957edf0f..d76b28b65e65 100644
--- a/arch/arm/mach-s3c24xx/mach-vstms.c
+++ b/arch/arm/mach-s3c24xx/mach-vstms.c
@@ -1,13 +1,8 @@
-/* linux/arch/arm/mach-s3c2412/mach-vstms.c
- *
- * (C) 2006 Thomas Gleixner <tglx@linutronix.de>
- *
- * Derived from mach-smdk2413.c - (C) 2006 Simtec Electronics
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// (C) 2006 Thomas Gleixner <tglx@linutronix.de>
+//
+// Derived from mach-smdk2413.c - (C) 2006 Simtec Electronics
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/nand-core.h b/arch/arm/mach-s3c24xx/nand-core.h
index 7e811fe1cf41..8de633d416ae 100644
--- a/arch/arm/mach-s3c24xx/nand-core.h
+++ b/arch/arm/mach-s3c24xx/nand-core.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2010 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com/
  *
  * S3C -  Nand Controller core functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_NAND_CORE_H
 #define __ASM_ARCH_NAND_CORE_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/osiris.h b/arch/arm/mach-s3c24xx/osiris.h
index b8d56074abac..b6c9c5ed2ba7 100644
--- a/arch/arm/mach-s3c24xx/osiris.h
+++ b/arch/arm/mach-s3c24xx/osiris.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2005 Simtec Electronics
  *	http://www.simtec.co.uk/products/
@@ -5,11 +6,7 @@
  *
  * OSIRIS - CPLD control constants
  * OSIRIS - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C24XX_OSIRIS_H
 #define __MACH_S3C24XX_OSIRIS_H __FILE__
diff --git a/arch/arm/mach-s3c24xx/otom.h b/arch/arm/mach-s3c24xx/otom.h
index 321b7be1c0f7..c800f67d03d4 100644
--- a/arch/arm/mach-s3c24xx/otom.h
+++ b/arch/arm/mach-s3c24xx/otom.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * (c) 2005 Guillaume GOURAT / NexVision
  *          guillaume.gourat@nexvision.fr
  *
  * NexVision OTOM board memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /*
  * ok, we've used up to 0x01300000, now we need to find space for the
diff --git a/arch/arm/mach-s3c24xx/pll-s3c2410.c b/arch/arm/mach-s3c24xx/pll-s3c2410.c
index 7ee4924a543d..0561f79ddce8 100644
--- a/arch/arm/mach-s3c24xx/pll-s3c2410.c
+++ b/arch/arm/mach-s3c24xx/pll-s3c2410.c
@@ -1,25 +1,11 @@
-/*
- * Copyright (c) 2006-2007 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *	Vincent Sanders <vince@arm.linux.org.uk>
- *
- * S3C2410 CPU PLL tables
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2006-2007 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//	Vincent Sanders <vince@arm.linux.org.uk>
+//
+// S3C2410 CPU PLL tables
 
 #include <linux/types.h>
 #include <linux/kernel.h>
diff --git a/arch/arm/mach-s3c24xx/pll-s3c2440-12000000.c b/arch/arm/mach-s3c24xx/pll-s3c2440-12000000.c
index a3fbfed75e28..2ec3a2f9a6a5 100644
--- a/arch/arm/mach-s3c24xx/pll-s3c2440-12000000.c
+++ b/arch/arm/mach-s3c24xx/pll-s3c2440-12000000.c
@@ -1,15 +1,11 @@
-/*
- * Copyright (c) 2006-2007 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *	Vincent Sanders <vince@arm.linux.org.uk>
- *
- * S3C2440/S3C2442 CPU PLL tables (12MHz Crystal)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006-2007 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//	Vincent Sanders <vince@arm.linux.org.uk>
+//
+// S3C2440/S3C2442 CPU PLL tables (12MHz Crystal)
 
 #include <linux/types.h>
 #include <linux/kernel.h>
diff --git a/arch/arm/mach-s3c24xx/pll-s3c2440-16934400.c b/arch/arm/mach-s3c24xx/pll-s3c2440-16934400.c
index bcff89fd9871..4b3d9e36c6bb 100644
--- a/arch/arm/mach-s3c24xx/pll-s3c2440-16934400.c
+++ b/arch/arm/mach-s3c24xx/pll-s3c2440-16934400.c
@@ -1,15 +1,11 @@
-/*
- * Copyright (c) 2006-2008 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *	Vincent Sanders <vince@arm.linux.org.uk>
- *
- * S3C2440/S3C2442 CPU PLL tables (16.93444MHz Crystal)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006-2008 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//	Vincent Sanders <vince@arm.linux.org.uk>
+//
+// S3C2440/S3C2442 CPU PLL tables (16.93444MHz Crystal)
 
 #include <linux/types.h>
 #include <linux/kernel.h>
diff --git a/arch/arm/mach-s3c24xx/pm-h1940.S b/arch/arm/mach-s3c24xx/pm-h1940.S
index 6183a688012b..a7bbe336ac6b 100644
--- a/arch/arm/mach-s3c24xx/pm-h1940.S
+++ b/arch/arm/mach-s3c24xx/pm-h1940.S
@@ -1,22 +1,9 @@
-/* linux/arch/arm/mach-s3c2410/pm-h1940.S
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * Copyright (c) 2006 Ben Dooks <ben-linux@fluff.org>
  *
  * H1940 Suspend to RAM
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+ */
 
 #include <linux/linkage.h>
 #include <asm/assembler.h>
diff --git a/arch/arm/mach-s3c24xx/pm-s3c2410.c b/arch/arm/mach-s3c24xx/pm-s3c2410.c
index a4588daeddb0..2d8ea701380a 100644
--- a/arch/arm/mach-s3c24xx/pm-s3c2410.c
+++ b/arch/arm/mach-s3c24xx/pm-s3c2410.c
@@ -1,24 +1,9 @@
-/* linux/arch/arm/mach-s3c2410/pm.c
- *
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C2410 (and compatible) Power Manager (Suspend-To-RAM) support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C2410 (and compatible) Power Manager (Suspend-To-RAM) support
 
 #include <linux/init.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/mach-s3c24xx/pm-s3c2412.c b/arch/arm/mach-s3c24xx/pm-s3c2412.c
index 0ae4d47a4663..2dfdaab0aa1f 100644
--- a/arch/arm/mach-s3c24xx/pm-s3c2412.c
+++ b/arch/arm/mach-s3c24xx/pm-s3c2412.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2412/pm.c
- *
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://armlinux.simtec.co.uk/.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://armlinux.simtec.co.uk/.
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/pm-s3c2416.c b/arch/arm/mach-s3c24xx/pm-s3c2416.c
index b5bbf0d5985c..9a2f05e279d4 100644
--- a/arch/arm/mach-s3c24xx/pm-s3c2416.c
+++ b/arch/arm/mach-s3c24xx/pm-s3c2416.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2416/pm.c
- *
- * Copyright (c) 2010 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * S3C2416 - PM support (Based on Ben Dooks' S3C2412 PM support)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2010 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// S3C2416 - PM support (Based on Ben Dooks' S3C2412 PM support)
 
 #include <linux/device.h>
 #include <linux/syscore_ops.h>
diff --git a/arch/arm/mach-s3c24xx/pm.c b/arch/arm/mach-s3c24xx/pm.c
index 5d510bca0844..adcb90645460 100644
--- a/arch/arm/mach-s3c24xx/pm.c
+++ b/arch/arm/mach-s3c24xx/pm.c
@@ -1,30 +1,15 @@
-/* linux/arch/arm/plat-s3c24xx/pm.c
- *
- * Copyright (c) 2004-2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C24XX Power Manager (Suspend-To-RAM) support
- *
- * See Documentation/arm/Samsung-S3C24XX/Suspend.txt for more information
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- *
- * Parts based on arch/arm/mach-pxa/pm.c
- *
- * Thanks to Dimitry Andric for debugging
-*/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2004-2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C24XX Power Manager (Suspend-To-RAM) support
+//
+// See Documentation/arm/Samsung-S3C24XX/Suspend.txt for more information
+//
+// Parts based on arch/arm/mach-pxa/pm.c
+//
+// Thanks to Dimitry Andric for debugging
 
 #include <linux/init.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/mach-s3c24xx/regs-dsc.h b/arch/arm/mach-s3c24xx/regs-dsc.h
index 61b3d1387d76..b500636276f2 100644
--- a/arch/arm/mach-s3c24xx/regs-dsc.h
+++ b/arch/arm/mach-s3c24xx/regs-dsc.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2004 Simtec Electronics <linux@simtec.co.uk>
  *		      http://www.simtec.co.uk/products/SWLINUX/
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2440/S3C2412 Signal Drive Strength Control
-*/
+ */
 
 
 #ifndef __ASM_ARCH_REGS_DSC_H
diff --git a/arch/arm/mach-s3c24xx/regs-mem.h b/arch/arm/mach-s3c24xx/regs-mem.h
index 86b1258368c2..2f3bc48b5890 100644
--- a/arch/arm/mach-s3c24xx/regs-mem.h
+++ b/arch/arm/mach-s3c24xx/regs-mem.h
@@ -1,11 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2004 Simtec Electronics <linux@simtec.co.uk>
  *		http://www.simtec.co.uk/products/SWLINUX/
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2410 Memory Control register definitions
  */
 
diff --git a/arch/arm/mach-s3c24xx/s3c2410.c b/arch/arm/mach-s3c24xx/s3c2410.c
index 5061d66ca10c..21fd5404bc98 100644
--- a/arch/arm/mach-s3c24xx/s3c2410.c
+++ b/arch/arm/mach-s3c24xx/s3c2410.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2410/s3c2410.c
- *
- * Copyright (c) 2003-2005 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.simtec.co.uk/products/EB2410ITX/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2003-2005 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.simtec.co.uk/products/EB2410ITX/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/s3c2412-power.h b/arch/arm/mach-s3c24xx/s3c2412-power.h
index 1b02c5ddb31b..0031cfaa1d76 100644
--- a/arch/arm/mach-s3c24xx/s3c2412-power.h
+++ b/arch/arm/mach-s3c24xx/s3c2412-power.h
@@ -1,10 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2003-2006 Simtec Electronics <linux@simtec.co.uk>
  *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ARCH_ARM_MACH_S3C24XX_S3C2412_POWER_H
diff --git a/arch/arm/mach-s3c24xx/s3c2412.c b/arch/arm/mach-s3c24xx/s3c2412.c
index fb5ee8d38913..8fe4d4670dcb 100644
--- a/arch/arm/mach-s3c24xx/s3c2412.c
+++ b/arch/arm/mach-s3c24xx/s3c2412.c
@@ -1,13 +1,9 @@
-/*
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://armlinux.simtec.co.uk/.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://armlinux.simtec.co.uk/.
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/s3c2416.c b/arch/arm/mach-s3c24xx/s3c2416.c
index 621b8648a7ef..1cdb7bd3e713 100644
--- a/arch/arm/mach-s3c24xx/s3c2416.c
+++ b/arch/arm/mach-s3c24xx/s3c2416.c
@@ -1,26 +1,11 @@
-/* linux/arch/arm/mach-s3c2416/s3c2416.c
- *
- * Copyright (c) 2009 Yauhen Kharuzhy <jekhor@gmail.com>,
- *	as part of OpenInkpot project
- * Copyright (c) 2009 Promwad Innovation Company
- *	Yauhen Kharuzhy <yauhen.kharuzhy@promwad.com>
- *
- * Samsung S3C2416 Mobile CPU support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2009 Yauhen Kharuzhy <jekhor@gmail.com>,
+//	as part of OpenInkpot project
+// Copyright (c) 2009 Promwad Innovation Company
+//	Yauhen Kharuzhy <yauhen.kharuzhy@promwad.com>
+//
+// Samsung S3C2416 Mobile CPU support
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/s3c2440.c b/arch/arm/mach-s3c24xx/s3c2440.c
index eb733555fab5..451d9851b0a7 100644
--- a/arch/arm/mach-s3c24xx/s3c2440.c
+++ b/arch/arm/mach-s3c24xx/s3c2440.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2440/s3c2440.c
- *
- * Copyright (c) 2004-2006 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * Samsung S3C2440 Mobile CPU support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2004-2006 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// Samsung S3C2440 Mobile CPU support
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/s3c2442.c b/arch/arm/mach-s3c24xx/s3c2442.c
index 893998ede022..432d68325c9d 100644
--- a/arch/arm/mach-s3c24xx/s3c2442.c
+++ b/arch/arm/mach-s3c24xx/s3c2442.c
@@ -1,25 +1,10 @@
-/* linux/arch/arm/mach-s3c2442/s3c2442.c
- *
- * Copyright (c) 2004-2005 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C2442 core and lock support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2004-2005 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C2442 core and lock support
 
 #include <linux/init.h>
 #include <linux/module.h>
diff --git a/arch/arm/mach-s3c24xx/s3c2443.c b/arch/arm/mach-s3c24xx/s3c2443.c
index b559d378cf43..313e369c3ddd 100644
--- a/arch/arm/mach-s3c24xx/s3c2443.c
+++ b/arch/arm/mach-s3c24xx/s3c2443.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c2443/s3c2443.c
- *
- * Copyright (c) 2007 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * Samsung S3C2443 Mobile CPU support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2007 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// Samsung S3C2443 Mobile CPU support
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/s3c244x.c b/arch/arm/mach-s3c24xx/s3c244x.c
index 31fd273269c2..a75f588b9d45 100644
--- a/arch/arm/mach-s3c24xx/s3c244x.c
+++ b/arch/arm/mach-s3c24xx/s3c244x.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/plat-s3c24xx/s3c244x.c
- *
- * Copyright (c) 2004-2006 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * Samsung S3C2440 and S3C2442 Mobile CPU support (not S3C2443)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2004-2006 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// Samsung S3C2440 and S3C2442 Mobile CPU support (not S3C2443)
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/setup-camif.c b/arch/arm/mach-s3c24xx/setup-camif.c
index 72d8edb8927a..2b262fae3f61 100644
--- a/arch/arm/mach-s3c24xx/setup-camif.c
+++ b/arch/arm/mach-s3c24xx/setup-camif.c
@@ -1,12 +1,8 @@
-/*
- * Copyright (C) 2012 Sylwester Nawrocki <sylvester.nawrocki@gmail.com>
- *
- * Helper functions for S3C24XX/S3C64XX SoC series CAMIF driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2012 Sylwester Nawrocki <sylvester.nawrocki@gmail.com>
+//
+// Helper functions for S3C24XX/S3C64XX SoC series CAMIF driver
 
 #include <linux/gpio.h>
 #include <plat/gpio-cfg.h>
diff --git a/arch/arm/mach-s3c24xx/setup-i2c.c b/arch/arm/mach-s3c24xx/setup-i2c.c
index 1852696ca16e..1a01d44b5910 100644
--- a/arch/arm/mach-s3c24xx/setup-i2c.c
+++ b/arch/arm/mach-s3c24xx/setup-i2c.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/plat-s3c24xx/setup-i2c.c
- *
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C24XX Base setup for i2c device
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C24XX Base setup for i2c device
 
 #include <linux/kernel.h>
 #include <linux/gpio.h>
diff --git a/arch/arm/mach-s3c24xx/setup-sdhci-gpio.c b/arch/arm/mach-s3c24xx/setup-sdhci-gpio.c
index c99b0f664db7..218346a36d1e 100644
--- a/arch/arm/mach-s3c24xx/setup-sdhci-gpio.c
+++ b/arch/arm/mach-s3c24xx/setup-sdhci-gpio.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c2416/setup-sdhci-gpio.c
- *
- * Copyright 2010 Promwad Innovation Company
- *	Yauhen Kharuzhy <yauhen.kharuzhy@promwad.com>
- *
- * S3C2416 - Helper functions for setting up SDHCI device(s) GPIO (HSMMC)
- *
- * Based on mach-s3c64xx/setup-sdhci-gpio.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2010 Promwad Innovation Company
+//	Yauhen Kharuzhy <yauhen.kharuzhy@promwad.com>
+//
+// S3C2416 - Helper functions for setting up SDHCI device(s) GPIO (HSMMC)
+//
+// Based on mach-s3c64xx/setup-sdhci-gpio.c
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/setup-spi.c b/arch/arm/mach-s3c24xx/setup-spi.c
index 3d47e023ce94..6c2b96a82da5 100644
--- a/arch/arm/mach-s3c24xx/setup-spi.c
+++ b/arch/arm/mach-s3c24xx/setup-spi.c
@@ -1,13 +1,9 @@
-/*
- * HS-SPI device setup for S3C2443/S3C2416
- *
- * Copyright (C) 2011 Samsung Electronics Ltd.
- *		http://www.samsung.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// HS-SPI device setup for S3C2443/S3C2416
+//
+// Copyright (C) 2011 Samsung Electronics Ltd.
+//		http://www.samsung.com/
 
 #include <linux/gpio.h>
 #include <linux/platform_device.h>
diff --git a/arch/arm/mach-s3c24xx/setup-ts.c b/arch/arm/mach-s3c24xx/setup-ts.c
index 46466d20257e..53a14d4f4852 100644
--- a/arch/arm/mach-s3c24xx/setup-ts.c
+++ b/arch/arm/mach-s3c24xx/setup-ts.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/plat-s3c24xx/setup-ts.c
- *
- * Copyright (c) 2010 Samsung Electronics Co., Ltd.
- *                     http://www.samsung.com/
- *
- * Based on S3C24XX setup for i2c device
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2010 Samsung Electronics Co., Ltd.
+//                     http://www.samsung.com/
+//
+// Based on S3C24XX setup for i2c device
 
 #include <linux/kernel.h>
 #include <linux/gpio.h>
diff --git a/arch/arm/mach-s3c24xx/simtec-audio.c b/arch/arm/mach-s3c24xx/simtec-audio.c
index 67cb5120dfeb..12e17f82dae3 100644
--- a/arch/arm/mach-s3c24xx/simtec-audio.c
+++ b/arch/arm/mach-s3c24xx/simtec-audio.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/plat-s3c24xx/simtec-audio.c
- *
- * Copyright (c) 2009 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Audio setup for various Simtec S3C24XX implementations
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2009 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Audio setup for various Simtec S3C24XX implementations
 
 #include <linux/kernel.h>
 #include <linux/interrupt.h>
diff --git a/arch/arm/mach-s3c24xx/simtec-nor.c b/arch/arm/mach-s3c24xx/simtec-nor.c
index 8884bffa619a..26b18497e959 100644
--- a/arch/arm/mach-s3c24xx/simtec-nor.c
+++ b/arch/arm/mach-s3c24xx/simtec-nor.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/mach-s3c2410/nor-simtec.c
- *
- * Copyright (c) 2008 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Simtec NOR mapping
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2008 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Simtec NOR mapping
 
 #include <linux/module.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/simtec-pm.c b/arch/arm/mach-s3c24xx/simtec-pm.c
index 38a2f1fdebab..c19074d81389 100644
--- a/arch/arm/mach-s3c24xx/simtec-pm.c
+++ b/arch/arm/mach-s3c24xx/simtec-pm.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c24xx/pm-simtec.c
- *
- * Copyright 2004 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * http://armlinux.simtec.co.uk/
- *
- * Power Management helpers for Simtec S3C24XX implementations
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2004 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// http://armlinux.simtec.co.uk/
+//
+// Power Management helpers for Simtec S3C24XX implementations
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c24xx/simtec-usb.c b/arch/arm/mach-s3c24xx/simtec-usb.c
index b70aa66efebe..dc1016ffed94 100644
--- a/arch/arm/mach-s3c24xx/simtec-usb.c
+++ b/arch/arm/mach-s3c24xx/simtec-usb.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/mach-s3c2410/usb-simtec.c
- *
- * Copyright 2004-2005 Simtec Electronics
- *   Ben Dooks <ben@simtec.co.uk>
- *
- * http://www.simtec.co.uk/products/EB2410ITX/
- *
- * Simtec BAST and Thorcom VR1000 USB port support functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2004-2005 Simtec Electronics
+//   Ben Dooks <ben@simtec.co.uk>
+//
+// http://www.simtec.co.uk/products/EB2410ITX/
+//
+// Simtec BAST and Thorcom VR1000 USB port support functions
 
 #define DEBUG
 
diff --git a/arch/arm/mach-s3c24xx/simtec.h b/arch/arm/mach-s3c24xx/simtec.h
index ae8f4f9ad2ee..d96bd60872b8 100644
--- a/arch/arm/mach-s3c24xx/simtec.h
+++ b/arch/arm/mach-s3c24xx/simtec.h
@@ -1,15 +1,11 @@
-/* linux/arch/arm/mach-s3c2410/nor-simtec.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * Simtec common functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 struct s3c24xx_audio_simtec_pdata;
 
diff --git a/arch/arm/mach-s3c24xx/sleep-s3c2410.S b/arch/arm/mach-s3c24xx/sleep-s3c2410.S
index 875ba8911127..659f9eff9de2 100644
--- a/arch/arm/mach-s3c24xx/sleep-s3c2410.S
+++ b/arch/arm/mach-s3c24xx/sleep-s3c2410.S
@@ -1,5 +1,5 @@
-/* linux/arch/arm/mach-s3c2410/sleep.S
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * Copyright (c) 2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
@@ -8,21 +8,7 @@
  * Based on PXA/SA1100 sleep code by:
  *	Nicolas Pitre, (c) 2002 Monta Vista Software Inc
  *	Cliff Brake, (c) 2001
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+ */
 
 #include <linux/linkage.h>
 #include <linux/serial_s3c.h>
diff --git a/arch/arm/mach-s3c24xx/sleep-s3c2412.S b/arch/arm/mach-s3c24xx/sleep-s3c2412.S
index 6bf5b4d8743c..c373f1ca862b 100644
--- a/arch/arm/mach-s3c24xx/sleep-s3c2412.S
+++ b/arch/arm/mach-s3c24xx/sleep-s3c2412.S
@@ -1,24 +1,10 @@
-/* linux/arch/arm/mach-s3c2412/sleep.S
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * Copyright (c) 2007 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C2412 Power Manager low-level sleep support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+ */
 
 #include <linux/linkage.h>
 #include <asm/assembler.h>
diff --git a/arch/arm/mach-s3c24xx/sleep.S b/arch/arm/mach-s3c24xx/sleep.S
index b859268fa8da..f0f11ad60c52 100644
--- a/arch/arm/mach-s3c24xx/sleep.S
+++ b/arch/arm/mach-s3c24xx/sleep.S
@@ -1,5 +1,5 @@
-/* linux/arch/arm/plat-s3c24xx/sleep.S
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * Copyright (c) 2004 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
@@ -8,21 +8,7 @@
  * Based on PXA/SA1100 sleep code by:
  *	Nicolas Pitre, (c) 2002 Monta Vista Software Inc
  *	Cliff Brake, (c) 2001
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+ */
 
 #include <linux/linkage.h>
 #include <linux/serial_s3c.h>
diff --git a/arch/arm/mach-s3c24xx/spi-core.h b/arch/arm/mach-s3c24xx/spi-core.h
index 0b9428ab3fc3..bb555ccbe057 100644
--- a/arch/arm/mach-s3c24xx/spi-core.h
+++ b/arch/arm/mach-s3c24xx/spi-core.h
@@ -1,9 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2012 Heiko Stuebner <heiko@sntech.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __PLAT_S3C_SPI_CORE_H
diff --git a/arch/arm/mach-s3c24xx/vr1000.h b/arch/arm/mach-s3c24xx/vr1000.h
index 7fcd2c2f183c..3cfa296bec2a 100644
--- a/arch/arm/mach-s3c24xx/vr1000.h
+++ b/arch/arm/mach-s3c24xx/vr1000.h
@@ -1,17 +1,12 @@
-
-/* arch/arm/mach-s3c2410/include/mach/vr1000-cpld.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2003 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * VR1000 - CPLD control constants
  * Machine VR1000 - IRQ Number definitions
  * Machine VR1000 - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C24XX_VR1000_H
 #define __MACH_S3C24XX_VR1000_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/Kconfig b/arch/arm/mach-s3c64xx/Kconfig
index afd1f20be49e..5700822e3c74 100644
--- a/arch/arm/mach-s3c64xx/Kconfig
+++ b/arch/arm/mach-s3c64xx/Kconfig
@@ -1,7 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
 # Copyright 2008 Openmoko, Inc.
 #	Simtec Electronics, Ben Dooks <ben@simtec.co.uk>
-#
-# Licensed under GPLv2
+
 menuconfig ARCH_S3C64XX
 	bool "Samsung S3C64XX"
 	depends on ARCH_MULTI_V6
diff --git a/arch/arm/mach-s3c64xx/Makefile b/arch/arm/mach-s3c64xx/Makefile
index 256cd5b40c60..8caeb4ad17e9 100644
--- a/arch/arm/mach-s3c64xx/Makefile
+++ b/arch/arm/mach-s3c64xx/Makefile
@@ -1,9 +1,7 @@
-# arch/arm/mach-s3c64xx/Makefile
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright 2008 Openmoko, Inc.
 # Copyright 2008 Simtec Electronics
-#
-# Licensed under GPLv2
 
 ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/$(src)/include -I$(srctree)/arch/arm/plat-samsung/include
 asflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/$(src)/include -I$(srctree)/arch/arm/plat-samsung/include
diff --git a/arch/arm/mach-s3c64xx/ata-core.h b/arch/arm/mach-s3c64xx/ata-core.h
index 5951f24a9ec8..6d9a81f759e6 100644
--- a/arch/arm/mach-s3c64xx/ata-core.h
+++ b/arch/arm/mach-s3c64xx/ata-core.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2010 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
  * Samsung CF-ATA Controller core functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_PLAT_ATA_CORE_H
 #define __ASM_PLAT_ATA_CORE_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/backlight.h b/arch/arm/mach-s3c64xx/backlight.h
index 8dcacac523a2..028663f1cacc 100644
--- a/arch/arm/mach-s3c64xx/backlight.h
+++ b/arch/arm/mach-s3c64xx/backlight.h
@@ -1,10 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *              http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ASM_PLAT_BACKLIGHT_H
diff --git a/arch/arm/mach-s3c64xx/common.c b/arch/arm/mach-s3c64xx/common.c
index 9843eb4dd04e..13e91074308a 100644
--- a/arch/arm/mach-s3c64xx/common.c
+++ b/arch/arm/mach-s3c64xx/common.c
@@ -1,18 +1,14 @@
-/*
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * Common Codes for S3C64XX machines
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// Common Codes for S3C64XX machines
 
 /*
  * NOTE: Code in this file is not used when booting with Device Tree support.
diff --git a/arch/arm/mach-s3c64xx/common.h b/arch/arm/mach-s3c64xx/common.h
index 4f204668f00e..03670887a764 100644
--- a/arch/arm/mach-s3c64xx/common.h
+++ b/arch/arm/mach-s3c64xx/common.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
@@ -8,10 +9,6 @@
  *	http://armlinux.simtec.co.uk/
  *
  * Common Header for S3C64XX machines
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ARCH_ARM_MACH_S3C64XX_COMMON_H
diff --git a/arch/arm/mach-s3c64xx/cpuidle.c b/arch/arm/mach-s3c64xx/cpuidle.c
index 5322db51150e..0bac6f6413b0 100644
--- a/arch/arm/mach-s3c64xx/cpuidle.c
+++ b/arch/arm/mach-s3c64xx/cpuidle.c
@@ -1,13 +1,8 @@
-/* linux/arch/arm/mach-s3c64xx/cpuidle.c
- *
- * Copyright (c) 2011 Wolfson Microelectronics, plc
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2011 Wolfson Microelectronics, plc
+// Copyright (c) 2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
 
 #include <linux/kernel.h>
 #include <linux/init.h>
diff --git a/arch/arm/mach-s3c64xx/crag6410.h b/arch/arm/mach-s3c64xx/crag6410.h
index dcbe17f5e5f8..00d9aa114aa7 100644
--- a/arch/arm/mach-s3c64xx/crag6410.h
+++ b/arch/arm/mach-s3c64xx/crag6410.h
@@ -1,11 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* Cragganmore 6410 shared definitions
  *
  * Copyright 2011 Wolfson Microelectronics plc
  *	Mark Brown <broonie@opensource.wolfsonmicro.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef MACH_CRAG6410_H
diff --git a/arch/arm/mach-s3c64xx/dev-audio.c b/arch/arm/mach-s3c64xx/dev-audio.c
index cb953e238b2a..e3c49b5d1355 100644
--- a/arch/arm/mach-s3c64xx/dev-audio.c
+++ b/arch/arm/mach-s3c64xx/dev-audio.c
@@ -1,12 +1,7 @@
-/* linux/arch/arm/plat-s3c/dev-audio.c
- *
- * Copyright 2009 Wolfson Microelectronics
- *      Mark Brown <broonie@opensource.wolfsonmicro.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2009 Wolfson Microelectronics
+//      Mark Brown <broonie@opensource.wolfsonmicro.com>
 
 #include <linux/kernel.h>
 #include <linux/string.h>
diff --git a/arch/arm/mach-s3c64xx/dev-backlight.c b/arch/arm/mach-s3c64xx/dev-backlight.c
index 7ef8b9019344..799cfdf0606b 100644
--- a/arch/arm/mach-s3c64xx/dev-backlight.c
+++ b/arch/arm/mach-s3c64xx/dev-backlight.c
@@ -1,13 +1,9 @@
-/*
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- *              http://www.samsung.com
- *
- * Common infrastructure for PWM Backlight for Samsung boards
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2011 Samsung Electronics Co., Ltd.
+//              http://www.samsung.com
+//
+// Common infrastructure for PWM Backlight for Samsung boards
 
 #include <linux/gpio.h>
 #include <linux/platform_device.h>
diff --git a/arch/arm/mach-s3c64xx/dev-uart.c b/arch/arm/mach-s3c64xx/dev-uart.c
index a0b4f0329811..5fb59ad30008 100644
--- a/arch/arm/mach-s3c64xx/dev-uart.c
+++ b/arch/arm/mach-s3c64xx/dev-uart.c
@@ -1,17 +1,11 @@
-/* linux/arch/arm/plat-s3c64xx/dev-uart.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * Base S3C64XX UART resource and device definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// Base S3C64XX UART resource and device definitions
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/include/mach/gpio-samsung.h b/arch/arm/mach-s3c64xx/include/mach/gpio-samsung.h
index 1d3636512e33..8ed144a0d474 100644
--- a/arch/arm/mach-s3c64xx/include/mach/gpio-samsung.h
+++ b/arch/arm/mach-s3c64xx/include/mach/gpio-samsung.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
@@ -5,11 +6,7 @@
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C6400 - GPIO lib support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef GPIO_SAMSUNG_S3C64XX_H
 #define GPIO_SAMSUNG_S3C64XX_H
diff --git a/arch/arm/mach-s3c64xx/include/mach/hardware.h b/arch/arm/mach-s3c64xx/include/mach/hardware.h
index 862d033e57a4..c4ed359474de 100644
--- a/arch/arm/mach-s3c64xx/include/mach/hardware.h
+++ b/arch/arm/mach-s3c64xx/include/mach/hardware.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* linux/arch/arm/mach-s3c6400/include/mach/hardware.h
  *
  * Copyright 2008 Openmoko, Inc.
diff --git a/arch/arm/mach-s3c64xx/include/mach/map.h b/arch/arm/mach-s3c64xx/include/mach/map.h
index d51873e8f63f..9372a535b7ba 100644
--- a/arch/arm/mach-s3c64xx/include/mach/map.h
+++ b/arch/arm/mach-s3c64xx/include/mach/map.h
@@ -1,16 +1,12 @@
-/* linux/arch/arm/mach-s3c6400/include/mach/map.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C64XX - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_MAP_H
 #define __ASM_ARCH_MAP_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/include/mach/pm-core.h b/arch/arm/mach-s3c64xx/include/mach/pm-core.h
index 4a285e97afff..bbf79ed28583 100644
--- a/arch/arm/mach-s3c64xx/include/mach/pm-core.h
+++ b/arch/arm/mach-s3c64xx/include/mach/pm-core.h
@@ -1,15 +1,11 @@
-/* linux/arch/arm/mach-s3c64xx/include/mach/pm-core.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *      Ben Dooks <ben@simtec.co.uk>
  *      http://armlinux.simtec.co.uk/
  *
  * S3C64XX - PM core support for arch/arm/plat-s3c/pm.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __MACH_S3C64XX_PM_CORE_H
diff --git a/arch/arm/mach-s3c64xx/include/mach/regs-clock.h b/arch/arm/mach-s3c64xx/include/mach/regs-clock.h
index 4f44aac77092..35a68767b318 100644
--- a/arch/arm/mach-s3c64xx/include/mach/regs-clock.h
+++ b/arch/arm/mach-s3c64xx/include/mach/regs-clock.h
@@ -1,16 +1,12 @@
-/* arch/arm/plat-s3c64xx/include/plat/regs-clock.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *	http://armlinux.simtec.co.uk/
  *
  * S3C64XX clock register definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __PLAT_REGS_CLOCK_H
 #define __PLAT_REGS_CLOCK_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/include/mach/regs-irq.h b/arch/arm/mach-s3c64xx/include/mach/regs-irq.h
index 6a1127891c87..b18c7bcb61c5 100644
--- a/arch/arm/mach-s3c64xx/include/mach/regs-irq.h
+++ b/arch/arm/mach-s3c64xx/include/mach/regs-irq.h
@@ -1,16 +1,12 @@
-/* linux/arch/arm/mach-s3c6400/include/mach/regs-irq.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C64XX - IRQ register definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_REGS_IRQ_H
 #define __ASM_ARCH_REGS_IRQ_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/irq-pm.c b/arch/arm/mach-s3c64xx/irq-pm.c
index 0bbf1faaee42..31b221190479 100644
--- a/arch/arm/mach-s3c64xx/irq-pm.c
+++ b/arch/arm/mach-s3c64xx/irq-pm.c
@@ -1,16 +1,11 @@
-/* arch/arm/plat-s3c64xx/irq-pm.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *      Ben Dooks <ben@simtec.co.uk>
- *      http://armlinux.simtec.co.uk/
- *
- * S3C64XX - Interrupt handling Power Management
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//      Ben Dooks <ben@simtec.co.uk>
+//      http://armlinux.simtec.co.uk/
+//
+// S3C64XX - Interrupt handling Power Management
 
 /*
  * NOTE: Code in this file is not used when booting with Device Tree support.
diff --git a/arch/arm/mach-s3c64xx/irq-uart.h b/arch/arm/mach-s3c64xx/irq-uart.h
index 4b296132962f..78eccdce95a7 100644
--- a/arch/arm/mach-s3c64xx/irq-uart.h
+++ b/arch/arm/mach-s3c64xx/irq-uart.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2010 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * Header file for Samsung SoC UART IRQ demux for S3C64XX and later
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 struct s3c_uart_irq {
 	void __iomem	*regs;
diff --git a/arch/arm/mach-s3c64xx/mach-anw6410.c b/arch/arm/mach-s3c64xx/mach-anw6410.c
index 347ce6009a8c..0d3d5befb806 100644
--- a/arch/arm/mach-s3c64xx/mach-anw6410.c
+++ b/arch/arm/mach-s3c64xx/mach-anw6410.c
@@ -1,17 +1,11 @@
-/* linux/arch/arm/mach-s3c64xx/mach-anw6410.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- * Copyright 2009 Kwangwoo Lee
- * 	Kwangwoo Lee <kwangwoo.lee@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+// Copyright 2009 Kwangwoo Lee
+//	Kwangwoo Lee <kwangwoo.lee@gmail.com>
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/mach-crag6410-module.c b/arch/arm/mach-s3c64xx/mach-crag6410-module.c
index ea5f2169c850..f00988705408 100644
--- a/arch/arm/mach-s3c64xx/mach-crag6410-module.c
+++ b/arch/arm/mach-s3c64xx/mach-crag6410-module.c
@@ -1,12 +1,9 @@
-/* Speyside modules for Cragganmore - board data probing
- *
- * Copyright 2011 Wolfson Microelectronics plc
- *	Mark Brown <broonie@opensource.wolfsonmicro.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Speyside modules for Cragganmore - board data probing
+//
+// Copyright 2011 Wolfson Microelectronics plc
+//	Mark Brown <broonie@opensource.wolfsonmicro.com>
 
 #include <linux/export.h>
 #include <linux/interrupt.h>
diff --git a/arch/arm/mach-s3c64xx/mach-crag6410.c b/arch/arm/mach-s3c64xx/mach-crag6410.c
index d9d0440aed78..f04650297487 100644
--- a/arch/arm/mach-s3c64xx/mach-crag6410.c
+++ b/arch/arm/mach-s3c64xx/mach-crag6410.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/mach-s3c64xx/mach-crag6410.c
- *
- * Copyright 2011 Wolfson Microelectronics plc
- *	Mark Brown <broonie@opensource.wolfsonmicro.com>
- *
- * Copyright 2011 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2011 Wolfson Microelectronics plc
+//	Mark Brown <broonie@opensource.wolfsonmicro.com>
+//
+// Copyright 2011 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/kernel.h>
 #include <linux/list.h>
diff --git a/arch/arm/mach-s3c64xx/mach-hmt.c b/arch/arm/mach-s3c64xx/mach-hmt.c
index 59b5531f1987..bfe9881d12cc 100644
--- a/arch/arm/mach-s3c64xx/mach-hmt.c
+++ b/arch/arm/mach-s3c64xx/mach-hmt.c
@@ -1,12 +1,8 @@
-/* mach-hmt.c - Platform code for Airgoo HMT
- *
- * Copyright 2009 Peter Korsgaard <jacmet@sunsite.dk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// mach-hmt.c - Platform code for Airgoo HMT
+//
+// Copyright 2009 Peter Korsgaard <jacmet@sunsite.dk>
 
 #include <linux/kernel.h>
 #include <linux/init.h>
diff --git a/arch/arm/mach-s3c64xx/mach-mini6410.c b/arch/arm/mach-s3c64xx/mach-mini6410.c
index a3e3e25728b4..0dd36ae49e6a 100644
--- a/arch/arm/mach-s3c64xx/mach-mini6410.c
+++ b/arch/arm/mach-s3c64xx/mach-mini6410.c
@@ -1,16 +1,10 @@
-/* linux/arch/arm/mach-s3c64xx/mach-mini6410.c
- *
- * Copyright 2010 Darius Augulis <augulis.darius@gmail.com>
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2010 Darius Augulis <augulis.darius@gmail.com>
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
 
 #include <linux/init.h>
 #include <linux/interrupt.h>
diff --git a/arch/arm/mach-s3c64xx/mach-ncp.c b/arch/arm/mach-s3c64xx/mach-ncp.c
index 23baaa04318c..13fea5c86ca3 100644
--- a/arch/arm/mach-s3c64xx/mach-ncp.c
+++ b/arch/arm/mach-s3c64xx/mach-ncp.c
@@ -1,13 +1,6 @@
-/*
- * linux/arch/arm/mach-s3c64xx/mach-ncp.c
- *
- * Copyright (C) 2008-2009 Samsung Electronics
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2008-2009 Samsung Electronics
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/mach-real6410.c b/arch/arm/mach-s3c64xx/mach-real6410.c
index d6b3ffd7704b..0ff88b6859c4 100644
--- a/arch/arm/mach-s3c64xx/mach-real6410.c
+++ b/arch/arm/mach-s3c64xx/mach-real6410.c
@@ -1,16 +1,10 @@
-/* linux/arch/arm/mach-s3c64xx/mach-real6410.c
- *
- * Copyright 2010 Darius Augulis <augulis.darius@gmail.com>
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2010 Darius Augulis <augulis.darius@gmail.com>
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
 
 #include <linux/init.h>
 #include <linux/interrupt.h>
diff --git a/arch/arm/mach-s3c64xx/mach-s3c64xx-dt.c b/arch/arm/mach-s3c64xx/mach-s3c64xx-dt.c
index 5bf9afae752d..1724f5ea5c46 100644
--- a/arch/arm/mach-s3c64xx/mach-s3c64xx-dt.c
+++ b/arch/arm/mach-s3c64xx/mach-s3c64xx-dt.c
@@ -1,12 +1,8 @@
-/*
- * Samsung's S3C64XX flattened device tree enabled machine
- *
- * Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Samsung's S3C64XX flattened device tree enabled machine
+//
+// Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
 
 #include <asm/mach/arch.h>
 #include <asm/mach/map.h>
diff --git a/arch/arm/mach-s3c64xx/mach-smartq.c b/arch/arm/mach-s3c64xx/mach-smartq.c
index e0e1a729ef98..5655fe968b1f 100644
--- a/arch/arm/mach-s3c64xx/mach-smartq.c
+++ b/arch/arm/mach-s3c64xx/mach-smartq.c
@@ -1,13 +1,6 @@
-/*
- * linux/arch/arm/mach-s3c64xx/mach-smartq.c
- *
- * Copyright (C) 2010 Maurus Cuelenaere
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2010 Maurus Cuelenaere
 
 #include <linux/delay.h>
 #include <linux/fb.h>
diff --git a/arch/arm/mach-s3c64xx/mach-smartq.h b/arch/arm/mach-s3c64xx/mach-smartq.h
index 8e8b693db3af..f98132f4f430 100644
--- a/arch/arm/mach-s3c64xx/mach-smartq.h
+++ b/arch/arm/mach-s3c64xx/mach-smartq.h
@@ -1,12 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * linux/arch/arm/mach-s3c64xx/mach-smartq.h
  *
  * Copyright (C) 2010 Maurus Cuelenaere
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 #ifndef __MACH_SMARTQ_H
diff --git a/arch/arm/mach-s3c64xx/mach-smartq5.c b/arch/arm/mach-s3c64xx/mach-smartq5.c
index 0972b6ce0ef6..44e9edb144fa 100644
--- a/arch/arm/mach-s3c64xx/mach-smartq5.c
+++ b/arch/arm/mach-s3c64xx/mach-smartq5.c
@@ -1,13 +1,6 @@
-/*
- * linux/arch/arm/mach-s3c64xx/mach-smartq5.c
- *
- * Copyright (C) 2010 Maurus Cuelenaere
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2010 Maurus Cuelenaere
 
 #include <linux/fb.h>
 #include <linux/gpio.h>
diff --git a/arch/arm/mach-s3c64xx/mach-smartq7.c b/arch/arm/mach-s3c64xx/mach-smartq7.c
index 51ac1c6c654a..815ee7d0b5e3 100644
--- a/arch/arm/mach-s3c64xx/mach-smartq7.c
+++ b/arch/arm/mach-s3c64xx/mach-smartq7.c
@@ -1,13 +1,6 @@
-/*
- * linux/arch/arm/mach-s3c64xx/mach-smartq7.c
- *
- * Copyright (C) 2010 Maurus Cuelenaere
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2010 Maurus Cuelenaere
 
 #include <linux/fb.h>
 #include <linux/gpio.h>
diff --git a/arch/arm/mach-s3c64xx/mach-smdk6400.c b/arch/arm/mach-s3c64xx/mach-smdk6400.c
index 7d8a74fd8915..cbd16843c7d1 100644
--- a/arch/arm/mach-s3c64xx/mach-smdk6400.c
+++ b/arch/arm/mach-s3c64xx/mach-smdk6400.c
@@ -1,13 +1,8 @@
-/* linux/arch/arm/mach-s3c64xx/mach-smdk6400.c
- *
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/mach-smdk6410.c b/arch/arm/mach-s3c64xx/mach-smdk6410.c
index 92ec8c3b42b9..c46fa5dfd2e0 100644
--- a/arch/arm/mach-s3c64xx/mach-smdk6410.c
+++ b/arch/arm/mach-s3c64xx/mach-smdk6410.c
@@ -1,15 +1,9 @@
-/* linux/arch/arm/mach-s3c64xx/mach-smdk6410.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/onenand-core.h b/arch/arm/mach-s3c64xx/onenand-core.h
index 925eb13bbb60..0cf6b5e76b24 100644
--- a/arch/arm/mach-s3c64xx/onenand-core.h
+++ b/arch/arm/mach-s3c64xx/onenand-core.h
@@ -1,14 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *  Copyright (c) 2010 Samsung Electronics
  *  Kyungmin Park <kyungmin.park@samsung.com>
  *  Marek Szyprowski <m.szyprowski@samsung.com>
  *
  * Samsung OneNAD Controller core functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_ONENAND_CORE_H
 #define __ASM_ARCH_ONENAND_CORE_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/pl080.c b/arch/arm/mach-s3c64xx/pl080.c
index 66fc774b70ec..152edbeea0c7 100644
--- a/arch/arm/mach-s3c64xx/pl080.c
+++ b/arch/arm/mach-s3c64xx/pl080.c
@@ -1,12 +1,8 @@
-/*
- * Samsung's S3C64XX generic DMA support using amba-pl08x driver.
- *
- * Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Samsung's S3C64XX generic DMA support using amba-pl08x driver.
+//
+// Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
 
 #include <linux/kernel.h>
 #include <linux/amba/bus.h>
diff --git a/arch/arm/mach-s3c64xx/pm.c b/arch/arm/mach-s3c64xx/pm.c
index 2f579be8fe67..fd6dbb263ed5 100644
--- a/arch/arm/mach-s3c64xx/pm.c
+++ b/arch/arm/mach-s3c64xx/pm.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c64xx/pm.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * S3C64XX CPU PM support.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// S3C64XX CPU PM support.
 
 #include <linux/init.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/mach-s3c64xx/regs-modem.h b/arch/arm/mach-s3c64xx/regs-modem.h
index 073cdd3a03be..136ad44291bf 100644
--- a/arch/arm/mach-s3c64xx/regs-modem.h
+++ b/arch/arm/mach-s3c64xx/regs-modem.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
@@ -5,11 +6,7 @@
  *      Ben Dooks <ben@simtec.co.uk>
  *
  * S3C64XX - modem block registers
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C64XX_REGS_MODEM_H
 #define __MACH_S3C64XX_REGS_MODEM_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/regs-srom.h b/arch/arm/mach-s3c64xx/regs-srom.h
index d56f3386eb00..2b37988bdf94 100644
--- a/arch/arm/mach-s3c64xx/regs-srom.h
+++ b/arch/arm/mach-s3c64xx/regs-srom.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2009 Andy Green <andy@warmcat.com>
  *
  * S3C64XX SROM definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __MACH_S3C64XX_REGS_SROM_H
 #define __MACH_S3C64XX_REGS_SROM_H __FILE__
diff --git a/arch/arm/mach-s3c64xx/s3c6400.c b/arch/arm/mach-s3c64xx/s3c6400.c
index 5ea82accc773..545eea716db8 100644
--- a/arch/arm/mach-s3c64xx/s3c6400.c
+++ b/arch/arm/mach-s3c64xx/s3c6400.c
@@ -1,13 +1,8 @@
-/* linux/arch/arm/mach-s3c64xx/cpu.c
- *
- * Copyright 2009 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2009 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
 
 /*
  * NOTE: Code in this file is not used when booting with Device Tree support.
diff --git a/arch/arm/mach-s3c64xx/s3c6410.c b/arch/arm/mach-s3c64xx/s3c6410.c
index 92bb927c4478..47e04e019624 100644
--- a/arch/arm/mach-s3c64xx/s3c6410.c
+++ b/arch/arm/mach-s3c64xx/s3c6410.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c64xx/s3c6410.c
- *
- * Copyright 2008 Simtec Electronics
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Simtec Electronics
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
 
 /*
  * NOTE: Code in this file is not used when booting with Device Tree support.
diff --git a/arch/arm/mach-s3c64xx/setup-fb-24bpp.c b/arch/arm/mach-s3c64xx/setup-fb-24bpp.c
index 9d17bff12d4d..2c7178b26ebb 100644
--- a/arch/arm/mach-s3c64xx/setup-fb-24bpp.c
+++ b/arch/arm/mach-s3c64xx/setup-fb-24bpp.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c64xx/setup-fb-24bpp.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * Base S3C64XX setup information for 24bpp LCD framebuffer
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// Base S3C64XX setup information for 24bpp LCD framebuffer
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/setup-i2c0.c b/arch/arm/mach-s3c64xx/setup-i2c0.c
index 4b8c1cfdd1fc..552eb50da38c 100644
--- a/arch/arm/mach-s3c64xx/setup-i2c0.c
+++ b/arch/arm/mach-s3c64xx/setup-i2c0.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c64xx/setup-i2c0.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * Base S3C64XX I2C bus 0 gpio configuration
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// Base S3C64XX I2C bus 0 gpio configuration
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/setup-i2c1.c b/arch/arm/mach-s3c64xx/setup-i2c1.c
index cd1df71ee13b..d231f0fc508d 100644
--- a/arch/arm/mach-s3c64xx/setup-i2c1.c
+++ b/arch/arm/mach-s3c64xx/setup-i2c1.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c64xx/setup-i2c1.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * Base S3C64XX I2C bus 1 gpio configuration
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// Base S3C64XX I2C bus 1 gpio configuration
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/setup-ide.c b/arch/arm/mach-s3c64xx/setup-ide.c
index 689fb72e715c..810139a807ce 100644
--- a/arch/arm/mach-s3c64xx/setup-ide.c
+++ b/arch/arm/mach-s3c64xx/setup-ide.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c64xx/setup-ide.c
- *
- * Copyright (c) 2010 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com/
- *
- * S3C64XX setup information for IDE
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2010 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com/
+//
+// S3C64XX setup information for IDE
 
 #include <linux/kernel.h>
 #include <linux/gpio.h>
diff --git a/arch/arm/mach-s3c64xx/setup-keypad.c b/arch/arm/mach-s3c64xx/setup-keypad.c
index 6ad9a89dfddf..351961025273 100644
--- a/arch/arm/mach-s3c64xx/setup-keypad.c
+++ b/arch/arm/mach-s3c64xx/setup-keypad.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/mach-s3c64xx/setup-keypad.c
- *
- * Copyright (c) 2010 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com/
- *
- * GPIO configuration for S3C64XX KeyPad device
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2010 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com/
+//
+// GPIO configuration for S3C64XX KeyPad device
 
 #include <linux/gpio.h>
 #include <plat/gpio-cfg.h>
diff --git a/arch/arm/mach-s3c64xx/setup-sdhci-gpio.c b/arch/arm/mach-s3c64xx/setup-sdhci-gpio.c
index f426b7a16c16..138455af4937 100644
--- a/arch/arm/mach-s3c64xx/setup-sdhci-gpio.c
+++ b/arch/arm/mach-s3c64xx/setup-sdhci-gpio.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/plat-s3c64xx/setup-sdhci-gpio.c
- *
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * S3C64XX - Helper functions for setting up SDHCI device(s) GPIO (HSMMC)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// S3C64XX - Helper functions for setting up SDHCI device(s) GPIO (HSMMC)
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/mach-s3c64xx/setup-spi.c b/arch/arm/mach-s3c64xx/setup-spi.c
index 5fd1a315c901..39dfae1f46e7 100644
--- a/arch/arm/mach-s3c64xx/setup-spi.c
+++ b/arch/arm/mach-s3c64xx/setup-spi.c
@@ -1,12 +1,7 @@
-/* linux/arch/arm/mach-s3c64xx/setup-spi.c
- *
- * Copyright (C) 2011 Samsung Electronics Ltd.
- *		http://www.samsung.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2011 Samsung Electronics Ltd.
+//		http://www.samsung.com/
 
 #include <linux/gpio.h>
 #include <plat/gpio-cfg.h>
diff --git a/arch/arm/mach-s3c64xx/setup-usb-phy.c b/arch/arm/mach-s3c64xx/setup-usb-phy.c
index 2b17b7f5152f..46a9e955607f 100644
--- a/arch/arm/mach-s3c64xx/setup-usb-phy.c
+++ b/arch/arm/mach-s3c64xx/setup-usb-phy.c
@@ -1,13 +1,7 @@
-/*
- * Copyright (C) 2011 Samsung Electronics Co.Ltd
- * Author: Joonyoung Shim <jy0922.shim@samsung.com>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2011 Samsung Electronics Co.Ltd
+// Author: Joonyoung Shim <jy0922.shim@samsung.com>
 
 #include <linux/clk.h>
 #include <linux/delay.h>
diff --git a/arch/arm/mach-s3c64xx/sleep.S b/arch/arm/mach-s3c64xx/sleep.S
index 34313f9c8792..39e16a07a5e4 100644
--- a/arch/arm/mach-s3c64xx/sleep.S
+++ b/arch/arm/mach-s3c64xx/sleep.S
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* linux/arch/arm/plat-s3c64xx/sleep.S
  *
  * Copyright 2008 Openmoko, Inc.
@@ -6,11 +7,7 @@
  *	http://armlinux.simtec.co.uk/
  *
  * S3C64XX CPU sleep code
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <linux/linkage.h>
 #include <asm/assembler.h>
diff --git a/arch/arm/mach-s3c64xx/watchdog-reset.h b/arch/arm/mach-s3c64xx/watchdog-reset.h
index 42707dfbda9c..1042d6c463dc 100644
--- a/arch/arm/mach-s3c64xx/watchdog-reset.h
+++ b/arch/arm/mach-s3c64xx/watchdog-reset.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2008 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C2410 - System define for arch_reset() function
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __PLAT_SAMSUNG_WATCHDOG_RESET_H
 #define __PLAT_SAMSUNG_WATCHDOG_RESET_H
diff --git a/arch/arm/mach-s5pv210/Kconfig b/arch/arm/mach-s5pv210/Kconfig
index 4cec11cf5e6f..03984a791879 100644
--- a/arch/arm/mach-s5pv210/Kconfig
+++ b/arch/arm/mach-s5pv210/Kconfig
@@ -1,9 +1,7 @@
-# arch/arm/mach-s5pv210/Kconfig
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (c) 2010 Samsung Electronics Co., Ltd.
 #		http://www.samsung.com/
-#
-# Licensed under GPLv2
 
 # Configuration options for the S5PV210/S5PC110
 
diff --git a/arch/arm/mach-s5pv210/Makefile b/arch/arm/mach-s5pv210/Makefile
index fa7fb716e388..e7b551e18e5c 100644
--- a/arch/arm/mach-s5pv210/Makefile
+++ b/arch/arm/mach-s5pv210/Makefile
@@ -1,9 +1,7 @@
-# arch/arm/mach-s5pv210/Makefile
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright (c) 2010 Samsung Electronics Co., Ltd.
 # 		http://www.samsung.com/
-#
-# Licensed under GPLv2
 
 ccflags-$(CONFIG_ARCH_MULTIPLATFORM) += -I$(srctree)/arch/arm/plat-samsung/include
 
diff --git a/arch/arm/mach-s5pv210/common.h b/arch/arm/mach-s5pv210/common.h
index 2ad387c1ecf0..0a188134deae 100644
--- a/arch/arm/mach-s5pv210/common.h
+++ b/arch/arm/mach-s5pv210/common.h
@@ -1,12 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
  * Common Header for S5PV210 machines
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ARCH_ARM_MACH_S5PV210_COMMON_H
diff --git a/arch/arm/mach-s5pv210/pm.c b/arch/arm/mach-s5pv210/pm.c
index 07cee14a363b..f491249ab658 100644
--- a/arch/arm/mach-s5pv210/pm.c
+++ b/arch/arm/mach-s5pv210/pm.c
@@ -1,18 +1,13 @@
-/* linux/arch/arm/mach-s5pv210/pm.c
- *
- * Copyright (c) 2010-2014 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * S5PV210 - Power Management support
- *
- * Based on arch/arm/mach-s3c2410/pm.c
- * Copyright (c) 2006 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2010-2014 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// S5PV210 - Power Management support
+//
+// Based on arch/arm/mach-s3c2410/pm.c
+// Copyright (c) 2006 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/init.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/mach-s5pv210/regs-clock.h b/arch/arm/mach-s5pv210/regs-clock.h
index fb3eb77412db..2a35c831a9b0 100644
--- a/arch/arm/mach-s5pv210/regs-clock.h
+++ b/arch/arm/mach-s5pv210/regs-clock.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2010 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com/
  *
  * S5PV210 - Clock register definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_REGS_CLOCK_H
 #define __ASM_ARCH_REGS_CLOCK_H __FILE__
diff --git a/arch/arm/mach-s5pv210/s5pv210.c b/arch/arm/mach-s5pv210/s5pv210.c
index 83e656ea95ae..868f9c20419d 100644
--- a/arch/arm/mach-s5pv210/s5pv210.c
+++ b/arch/arm/mach-s5pv210/s5pv210.c
@@ -1,14 +1,10 @@
-/*
- * Samsung's S5PC110/S5PV210 flattened device tree enabled machine.
- *
- * Copyright (c) 2013-2014 Samsung Electronics Co., Ltd.
- * Mateusz Krawczuk <m.krawczuk@partner.samsung.com>
- * Tomasz Figa <t.figa@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Samsung's S5PC110/S5PV210 flattened device tree enabled machine.
+//
+// Copyright (c) 2013-2014 Samsung Electronics Co., Ltd.
+// Mateusz Krawczuk <m.krawczuk@partner.samsung.com>
+// Tomasz Figa <t.figa@samsung.com>
 
 #include <linux/of_fdt.h>
 #include <linux/of_platform.h>
diff --git a/arch/arm/mach-s5pv210/sleep.S b/arch/arm/mach-s5pv210/sleep.S
index dfbfc0f7f8b8..81568767f30a 100644
--- a/arch/arm/mach-s5pv210/sleep.S
+++ b/arch/arm/mach-s5pv210/sleep.S
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright (c) 2011-2014 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
@@ -5,11 +6,6 @@
  * S5PV210 Sleep Code
  * Based on S3C64XX sleep code by:
  *	Ben Dooks, (c) 2008 Simtec Electronics
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #include <linux/linkage.h>
diff --git a/arch/arm/mach-sa1100/Kconfig b/arch/arm/mach-sa1100/Kconfig
index 36e3c79f4973..07df3a59b13f 100644
--- a/arch/arm/mach-sa1100/Kconfig
+++ b/arch/arm/mach-sa1100/Kconfig
@@ -5,6 +5,7 @@ menu "SA11x0 Implementations"
 config SA1100_ASSABET
 	bool "Assabet"
 	select ARM_SA1110_CPUFREQ
+	select GPIO_REG
 	help
 	  Say Y here if you are using the Intel(R) StrongARM(R) SA-1110
 	  Microprocessor Development Board (also known as the Assabet).
diff --git a/arch/arm/mach-sa1100/assabet.c b/arch/arm/mach-sa1100/assabet.c
index d28ecb9ef172..f68241d995f2 100644
--- a/arch/arm/mach-sa1100/assabet.c
+++ b/arch/arm/mach-sa1100/assabet.c
@@ -13,6 +13,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/errno.h>
+#include <linux/gpio/gpio-reg.h>
 #include <linux/ioport.h>
 #include <linux/platform_data/sa11x0-serial.h>
 #include <linux/serial_core.h>
@@ -61,20 +62,45 @@
 unsigned long SCR_value = ASSABET_SCR_INIT;
 EXPORT_SYMBOL(SCR_value);
 
-static unsigned long BCR_value = ASSABET_BCR_DB1110;
+static struct gpio_chip *assabet_bcr_gc;
+
+static const char *assabet_names[] = {
+	"cf_pwr", "cf_gfx_reset", "nsoft_reset", "irda_fsel",
+	"irda_md0", "irda_md1", "stereo_loopback", "ncf_bus_on",
+	"audio_pwr_on", "light_pwr_on", "lcd16data", "lcd_pwr_on",
+	"rs232_on", "nred_led", "ngreen_led", "vib_on",
+	"com_dtr", "com_rts", "radio_wake_mod", "i2c_enab",
+	"tvir_enab", "qmute", "radio_pwr_on", "spkr_off",
+	"rs232_valid", "com_dcd", "com_cts", "com_dsr",
+	"radio_cts", "radio_dsr", "radio_dcd", "radio_ri",
+};
 
+/* The old deprecated interface */
 void ASSABET_BCR_frob(unsigned int mask, unsigned int val)
 {
-	unsigned long flags;
+	unsigned long m = mask, v = val;
 
-	local_irq_save(flags);
-	BCR_value = (BCR_value & ~mask) | val;
-	ASSABET_BCR = BCR_value;
-	local_irq_restore(flags);
+	assabet_bcr_gc->set_multiple(assabet_bcr_gc, &m, &v);
 }
-
 EXPORT_SYMBOL(ASSABET_BCR_frob);
 
+static int __init assabet_init_gpio(void __iomem *reg, u32 def_val)
+{
+	struct gpio_chip *gc;
+
+	writel_relaxed(def_val, reg);
+
+	gc = gpio_reg_init(NULL, reg, -1, 32, "assabet", 0xff000000, def_val,
+			   assabet_names, NULL, NULL);
+
+	if (IS_ERR(gc))
+		return PTR_ERR(gc);
+
+	assabet_bcr_gc = gc;
+
+	return gc->base;
+}
+
 /*
  * The codec reset goes to three devices, so we need to release
  * the rest when any one of these requests it.  However, that
@@ -146,7 +172,7 @@ static void adv7171_write(unsigned reg, unsigned val)
 	unsigned gpdr = GPDR;
 	unsigned gplr = GPLR;
 
-	ASSABET_BCR = BCR_value | ASSABET_BCR_AUDIO_ON;
+	ASSABET_BCR_frob(ASSABET_BCR_AUDIO_ON, ASSABET_BCR_AUDIO_ON);
 	udelay(100);
 
 	GPCR = SDA | SCK | MOD; /* clear L3 mode to ensure UDA1341 doesn't respond */
@@ -457,14 +483,6 @@ static void __init assabet_init(void)
 	sa11x0_ppc_configure_mcp();
 
 	if (machine_has_neponset()) {
-		/*
-		 * Angel sets this, but other bootloaders may not.
-		 *
-		 * This must precede any driver calls to BCR_set()
-		 * or BCR_clear().
-		 */
-		ASSABET_BCR = BCR_value = ASSABET_BCR_DB1111;
-
 #ifndef CONFIG_ASSABET_NEPONSET
 		printk( "Warning: Neponset detected but full support "
 			"hasn't been configured in the kernel\n" );
@@ -748,12 +766,31 @@ static int __init assabet_leds_init(void)
 fs_initcall(assabet_leds_init);
 #endif
 
+void __init assabet_init_irq(void)
+{
+	u32 def_val;
+
+	sa1100_init_irq();
+
+	if (machine_has_neponset())
+		def_val = ASSABET_BCR_DB1111;
+	else
+		def_val = ASSABET_BCR_DB1110;
+
+	/*
+	 * Angel sets this, but other bootloaders may not.
+	 *
+	 * This must precede any driver calls to BCR_set() or BCR_clear().
+	 */
+	assabet_init_gpio((void *)&ASSABET_BCR, def_val);
+}
+
 MACHINE_START(ASSABET, "Intel-Assabet")
 	.atag_offset	= 0x100,
 	.fixup		= fixup_assabet,
 	.map_io		= assabet_map_io,
 	.nr_irqs	= SA1100_NR_IRQS,
-	.init_irq	= sa1100_init_irq,
+	.init_irq	= assabet_init_irq,
 	.init_time	= sa1100_timer_init,
 	.init_machine	= assabet_init,
 	.init_late	= sa11x0_init_late,
diff --git a/arch/arm/mach-sa1100/neponset.c b/arch/arm/mach-sa1100/neponset.c
index a61a2432766b..b1823f445358 100644
--- a/arch/arm/mach-sa1100/neponset.c
+++ b/arch/arm/mach-sa1100/neponset.c
@@ -3,6 +3,8 @@
  * linux/arch/arm/mach-sa1100/neponset.c
  */
 #include <linux/err.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/gpio-reg.h>
 #include <linux/init.h>
 #include <linux/ioport.h>
 #include <linux/irq.h>
@@ -45,10 +47,13 @@
 #define IRR_USAR	(1 << 1)
 #define IRR_SA1111	(1 << 2)
 
+#define NCR_NGPIO	7
+
 #define MDM_CTL0_RTS1	(1 << 0)
 #define MDM_CTL0_DTR1	(1 << 1)
 #define MDM_CTL0_RTS2	(1 << 2)
 #define MDM_CTL0_DTR2	(1 << 3)
+#define MDM_CTL0_NGPIO	4
 
 #define MDM_CTL1_CTS1	(1 << 0)
 #define MDM_CTL1_DSR1	(1 << 1)
@@ -56,80 +61,87 @@
 #define MDM_CTL1_CTS2	(1 << 3)
 #define MDM_CTL1_DSR2	(1 << 4)
 #define MDM_CTL1_DCD2	(1 << 5)
+#define MDM_CTL1_NGPIO	6
 
 #define AUD_SEL_1341	(1 << 0)
 #define AUD_MUTE_1341	(1 << 1)
+#define AUD_NGPIO	2
 
 extern void sa1110_mb_disable(void);
 
+#define to_neponset_gpio_chip(x) container_of(x, struct neponset_gpio_chip, gc)
+
+static const char *neponset_ncr_names[] = {
+	"gp01_off", "tp_power", "ms_power", "enet_osc",
+	"spi_kb_wk_up", "a0vpp", "a1vpp"
+};
+
+static const char *neponset_mdmctl0_names[] = {
+	"rts3", "dtr3", "rts1", "dtr1",
+};
+
+static const char *neponset_mdmctl1_names[] = {
+	"cts3", "dsr3", "dcd3", "cts1", "dsr1", "dcd1"
+};
+
+static const char *neponset_aud_names[] = {
+	"sel_1341", "mute_1341",
+};
+
 struct neponset_drvdata {
 	void __iomem *base;
 	struct platform_device *sa1111;
 	struct platform_device *smc91x;
 	unsigned irq_base;
-#ifdef CONFIG_PM_SLEEP
-	u32 ncr0;
-	u32 mdm_ctl_0;
-#endif
+	struct gpio_chip *gpio[4];
 };
 
-static void __iomem *nep_base;
+static struct neponset_drvdata *nep;
 
 void neponset_ncr_frob(unsigned int mask, unsigned int val)
 {
-	void __iomem *base = nep_base;
-
-	if (base) {
-		unsigned long flags;
-		unsigned v;
-
-		local_irq_save(flags);
-		v = readb_relaxed(base + NCR_0);
-		writeb_relaxed((v & ~mask) | val, base + NCR_0);
-		local_irq_restore(flags);
-	} else {
-		WARN(1, "nep_base unset\n");
-	}
+	struct neponset_drvdata *n = nep;
+	unsigned long m = mask, v = val;
+
+	if (nep)
+		n->gpio[0]->set_multiple(n->gpio[0], &m, &v);
+	else
+		WARN(1, "nep unset\n");
 }
 EXPORT_SYMBOL(neponset_ncr_frob);
 
 static void neponset_set_mctrl(struct uart_port *port, u_int mctrl)
 {
-	void __iomem *base = nep_base;
-	u_int mdm_ctl0;
+	struct neponset_drvdata *n = nep;
+	unsigned long mask, val = 0;
 
-	if (!base)
+	if (!n)
 		return;
 
-	mdm_ctl0 = readb_relaxed(base + MDM_CTL_0);
 	if (port->mapbase == _Ser1UTCR0) {
-		if (mctrl & TIOCM_RTS)
-			mdm_ctl0 &= ~MDM_CTL0_RTS2;
-		else
-			mdm_ctl0 |= MDM_CTL0_RTS2;
-
-		if (mctrl & TIOCM_DTR)
-			mdm_ctl0 &= ~MDM_CTL0_DTR2;
-		else
-			mdm_ctl0 |= MDM_CTL0_DTR2;
+		mask = MDM_CTL0_RTS2 | MDM_CTL0_DTR2;
+
+		if (!(mctrl & TIOCM_RTS))
+			val |= MDM_CTL0_RTS2;
+
+		if (!(mctrl & TIOCM_DTR))
+			val |= MDM_CTL0_DTR2;
 	} else if (port->mapbase == _Ser3UTCR0) {
-		if (mctrl & TIOCM_RTS)
-			mdm_ctl0 &= ~MDM_CTL0_RTS1;
-		else
-			mdm_ctl0 |= MDM_CTL0_RTS1;
-
-		if (mctrl & TIOCM_DTR)
-			mdm_ctl0 &= ~MDM_CTL0_DTR1;
-		else
-			mdm_ctl0 |= MDM_CTL0_DTR1;
+		mask = MDM_CTL0_RTS1 | MDM_CTL0_DTR1;
+
+		if (!(mctrl & TIOCM_RTS))
+			val |= MDM_CTL0_RTS1;
+
+		if (!(mctrl & TIOCM_DTR))
+			val |= MDM_CTL0_DTR1;
 	}
 
-	writeb_relaxed(mdm_ctl0, base + MDM_CTL_0);
+	n->gpio[1]->set_multiple(n->gpio[1], &mask, &val);
 }
 
 static u_int neponset_get_mctrl(struct uart_port *port)
 {
-	void __iomem *base = nep_base;
+	void __iomem *base = nep->base;
 	u_int ret = TIOCM_CD | TIOCM_CTS | TIOCM_DSR;
 	u_int mdm_ctl1;
 
@@ -231,6 +243,22 @@ static struct irq_chip nochip = {
 	.irq_unmask = nochip_noop,
 };
 
+static int neponset_init_gpio(struct gpio_chip **gcp,
+	struct device *dev, const char *label, void __iomem *reg,
+	unsigned num, bool in, const char *const * names)
+{
+	struct gpio_chip *gc;
+
+	gc = gpio_reg_init(dev, reg, -1, num, label, in ? 0xffffffff : 0,
+			   readl_relaxed(reg), names, NULL, NULL);
+	if (IS_ERR(gc))
+		return PTR_ERR(gc);
+
+	*gcp = gc;
+
+	return 0;
+}
+
 static struct sa1111_platform_data sa1111_info = {
 	.disable_devs	= SA1111_DEVID_PS2_MSE,
 };
@@ -274,7 +302,7 @@ static int neponset_probe(struct platform_device *dev)
 	};
 	int ret, irq;
 
-	if (nep_base)
+	if (nep)
 		return -EBUSY;
 
 	irq = ret = platform_get_irq(dev, 0);
@@ -330,6 +358,22 @@ static int neponset_probe(struct platform_device *dev)
 	irq_set_irq_type(irq, IRQ_TYPE_EDGE_RISING);
 	irq_set_chained_handler_and_data(irq, neponset_irq_handler, d);
 
+	/* Disable GPIO 0/1 drivers so the buttons work on the Assabet */
+	writeb_relaxed(NCR_GP01_OFF, d->base + NCR_0);
+
+	neponset_init_gpio(&d->gpio[0], &dev->dev, "neponset-ncr",
+			   d->base + NCR_0, NCR_NGPIO, false,
+			   neponset_ncr_names);
+	neponset_init_gpio(&d->gpio[1], &dev->dev, "neponset-mdm-ctl0",
+			   d->base + MDM_CTL_0, MDM_CTL0_NGPIO, false,
+			   neponset_mdmctl0_names);
+	neponset_init_gpio(&d->gpio[2], &dev->dev, "neponset-mdm-ctl1",
+			   d->base + MDM_CTL_1, MDM_CTL1_NGPIO, true,
+			   neponset_mdmctl1_names);
+	neponset_init_gpio(&d->gpio[3], &dev->dev, "neponset-aud-ctl",
+			   d->base + AUD_CTL, AUD_NGPIO, false,
+			   neponset_aud_names);
+
 	/*
 	 * We would set IRQ_GPIO25 to be a wake-up IRQ, but unfortunately
 	 * something on the Neponset activates this IRQ on sleep (eth?)
@@ -340,16 +384,13 @@ static int neponset_probe(struct platform_device *dev)
 
 	dev_info(&dev->dev, "Neponset daughter board, providing IRQ%u-%u\n",
 		 d->irq_base, d->irq_base + NEP_IRQ_NR - 1);
-	nep_base = d->base;
+	nep = d;
 
 	sa1100_register_uart_fns(&neponset_port_fns);
 
 	/* Ensure that the memory bus request/grant signals are setup */
 	sa1110_mb_disable();
 
-	/* Disable GPIO 0/1 drivers so the buttons work on the Assabet */
-	writeb_relaxed(NCR_GP01_OFF, d->base + NCR_0);
-
 	sa1111_resources[0].parent = sa1111_res;
 	sa1111_resources[1].start = d->irq_base + NEP_IRQ_SA1111;
 	sa1111_resources[1].end = d->irq_base + NEP_IRQ_SA1111;
@@ -385,7 +426,7 @@ static int neponset_remove(struct platform_device *dev)
 		platform_device_unregister(d->smc91x);
 	irq_set_chained_handler(irq, NULL);
 	irq_free_descs(d->irq_base, NEP_IRQ_NR);
-	nep_base = NULL;
+	nep = NULL;
 	iounmap(d->base);
 	kfree(d);
 
@@ -393,30 +434,22 @@ static int neponset_remove(struct platform_device *dev)
 }
 
 #ifdef CONFIG_PM_SLEEP
-static int neponset_suspend(struct device *dev)
-{
-	struct neponset_drvdata *d = dev_get_drvdata(dev);
-
-	d->ncr0 = readb_relaxed(d->base + NCR_0);
-	d->mdm_ctl_0 = readb_relaxed(d->base + MDM_CTL_0);
-
-	return 0;
-}
-
 static int neponset_resume(struct device *dev)
 {
 	struct neponset_drvdata *d = dev_get_drvdata(dev);
+	int i, ret = 0;
 
-	writeb_relaxed(d->ncr0, d->base + NCR_0);
-	writeb_relaxed(d->mdm_ctl_0, d->base + MDM_CTL_0);
+	for (i = 0; i < ARRAY_SIZE(d->gpio); i++) {
+		ret = gpio_reg_resume(d->gpio[i]);
+		if (ret)
+			break;
+	}
 
-	return 0;
+	return ret;
 }
 
 static const struct dev_pm_ops neponset_pm_ops = {
-	.suspend_noirq = neponset_suspend,
 	.resume_noirq = neponset_resume,
-	.freeze_noirq = neponset_suspend,
 	.restore_noirq = neponset_resume,
 };
 #define PM_OPS &neponset_pm_ops
diff --git a/arch/arm/mach-ux500/cpu-db8500.c b/arch/arm/mach-ux500/cpu-db8500.c
index 57058ac46f49..36cd23c8be9b 100644
--- a/arch/arm/mach-ux500/cpu-db8500.c
+++ b/arch/arm/mach-ux500/cpu-db8500.c
@@ -23,7 +23,6 @@
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
-#include <linux/perf/arm_pmu.h>
 #include <linux/regulator/machine.h>
 
 #include <asm/outercache.h>
@@ -112,37 +111,6 @@ static void ux500_restart(enum reboot_mode mode, const char *cmd)
 	prcmu_system_reset(0);
 }
 
-/*
- * The PMU IRQ lines of two cores are wired together into a single interrupt.
- * Bounce the interrupt to the other core if it's not ours.
- */
-static irqreturn_t db8500_pmu_handler(int irq, void *dev, irq_handler_t handler)
-{
-	irqreturn_t ret = handler(irq, dev);
-	int other = !smp_processor_id();
-
-	if (ret == IRQ_NONE && cpu_online(other))
-		irq_set_affinity(irq, cpumask_of(other));
-
-	/*
-	 * We should be able to get away with the amount of IRQ_NONEs we give,
-	 * while still having the spurious IRQ detection code kick in if the
-	 * interrupt really starts hitting spuriously.
-	 */
-	return ret;
-}
-
-static struct arm_pmu_platdata db8500_pmu_platdata = {
-	.handle_irq		= db8500_pmu_handler,
-	.irq_flags		= IRQF_NOBALANCING | IRQF_NO_THREAD,
-};
-
-static struct of_dev_auxdata u8500_auxdata_lookup[] __initdata = {
-	/* Requires call-back bindings. */
-	OF_DEV_AUXDATA("arm,cortex-a9-pmu", 0, "arm-pmu", &db8500_pmu_platdata),
-	{},
-};
-
 static struct of_dev_auxdata u8540_auxdata_lookup[] __initdata = {
 	OF_DEV_AUXDATA("stericsson,db8500-prcmu", 0x80157000, "db8500-prcmu", NULL),
 	{},
@@ -167,7 +135,7 @@ static void __init u8500_init_machine(void)
 				     u8540_auxdata_lookup, NULL);
 	else
 		of_platform_populate(NULL, u8500_local_bus_nodes,
-				     u8500_auxdata_lookup, NULL);
+				     NULL, NULL);
 }
 
 static const char * stericsson_dt_platform_compat[] = {
diff --git a/arch/arm/mach-vt8500/Kconfig b/arch/arm/mach-vt8500/Kconfig
index 1156a585dafc..8841199058ea 100644
--- a/arch/arm/mach-vt8500/Kconfig
+++ b/arch/arm/mach-vt8500/Kconfig
@@ -13,7 +13,6 @@ config ARCH_WM8505
  	depends on ARCH_MULTI_V5
  	select ARCH_VT8500
  	select CPU_ARM926T
- 	help
 
 config ARCH_WM8750
 	bool "WonderMedia WM8750"
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index fd9077a74fce..7f14acf67caf 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -909,6 +909,14 @@ config OUTER_CACHE_SYNC
 	  The outer cache has a outer_cache_fns.sync function pointer
 	  that can be used to drain the write buffer of the outer cache.
 
+config CACHE_B15_RAC
+	bool "Enable the Broadcom Brahma-B15 read-ahead cache controller"
+	depends on ARCH_BRCMSTB
+	default y
+	help
+	  This option enables the Broadcom Brahma-B15 read-ahead cache
+	  controller. If disabled, the read-ahead cache remains off.
+
 config CACHE_FEROCEON_L2
 	bool "Enable the Feroceon L2 cache controller"
 	depends on ARCH_MV78XX0 || ARCH_MVEBU
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 01bcc33f59e3..9dbb84923e12 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -13,7 +13,8 @@ obj-y				+= nommu.o
 obj-$(CONFIG_ARM_MPU)		+= pmsa-v7.o
 endif
 
-obj-$(CONFIG_ARM_PTDUMP)	+= dump.o
+obj-$(CONFIG_ARM_PTDUMP_CORE)	+= dump.o
+obj-$(CONFIG_ARM_PTDUMP_DEBUGFS)	+= ptdump_debugfs.o
 obj-$(CONFIG_MODULES)		+= proc-syms.o
 obj-$(CONFIG_DEBUG_VIRTUAL)	+= physaddr.o
 
@@ -103,6 +104,7 @@ AFLAGS_proc-v6.o	:=-Wa,-march=armv6
 AFLAGS_proc-v7.o	:=-Wa,-march=armv7-a
 
 obj-$(CONFIG_OUTER_CACHE)	+= l2c-common.o
+obj-$(CONFIG_CACHE_B15_RAC)	+= cache-b15-rac.o
 obj-$(CONFIG_CACHE_FEROCEON_L2)	+= cache-feroceon-l2.o
 obj-$(CONFIG_CACHE_L2X0)	+= cache-l2x0.o l2c-l2x0-resume.o
 obj-$(CONFIG_CACHE_L2X0_PMU)	+= cache-l2x0-pmu.o
diff --git a/arch/arm/mm/cache-b15-rac.c b/arch/arm/mm/cache-b15-rac.c
new file mode 100644
index 000000000000..d9586ba2e63c
--- /dev/null
+++ b/arch/arm/mm/cache-b15-rac.c
@@ -0,0 +1,356 @@
+/*
+ * Broadcom Brahma-B15 CPU read-ahead cache management functions
+ *
+ * Copyright (C) 2015-2016 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/err.h>
+#include <linux/spinlock.h>
+#include <linux/io.h>
+#include <linux/bitops.h>
+#include <linux/of_address.h>
+#include <linux/notifier.h>
+#include <linux/cpu.h>
+#include <linux/syscore_ops.h>
+#include <linux/reboot.h>
+
+#include <asm/cacheflush.h>
+#include <asm/hardware/cache-b15-rac.h>
+
+extern void v7_flush_kern_cache_all(void);
+
+/* RAC register offsets, relative to the HIF_CPU_BIUCTRL register base */
+#define RAC_CONFIG0_REG			(0x78)
+#define  RACENPREF_MASK			(0x3)
+#define  RACPREFINST_SHIFT		(0)
+#define  RACENINST_SHIFT		(2)
+#define  RACPREFDATA_SHIFT		(4)
+#define  RACENDATA_SHIFT		(6)
+#define  RAC_CPU_SHIFT			(8)
+#define  RACCFG_MASK			(0xff)
+#define RAC_CONFIG1_REG			(0x7c)
+#define RAC_FLUSH_REG			(0x80)
+#define  FLUSH_RAC			(1 << 0)
+
+/* Bitmask to enable instruction and data prefetching with a 256-bytes stride */
+#define RAC_DATA_INST_EN_MASK		(1 << RACPREFINST_SHIFT | \
+					 RACENPREF_MASK << RACENINST_SHIFT | \
+					 1 << RACPREFDATA_SHIFT | \
+					 RACENPREF_MASK << RACENDATA_SHIFT)
+
+#define RAC_ENABLED			0
+/* Special state where we want to bypass the spinlock and call directly
+ * into the v7 cache maintenance operations during suspend/resume
+ */
+#define RAC_SUSPENDED			1
+
+static void __iomem *b15_rac_base;
+static DEFINE_SPINLOCK(rac_lock);
+
+static u32 rac_config0_reg;
+
+/* Initialization flag to avoid checking for b15_rac_base, and to prevent
+ * multi-platform kernels from crashing here as well.
+ */
+static unsigned long b15_rac_flags;
+
+static inline u32 __b15_rac_disable(void)
+{
+	u32 val = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
+	__raw_writel(0, b15_rac_base + RAC_CONFIG0_REG);
+	dmb();
+	return val;
+}
+
+static inline void __b15_rac_flush(void)
+{
+	u32 reg;
+
+	__raw_writel(FLUSH_RAC, b15_rac_base + RAC_FLUSH_REG);
+	do {
+		/* This dmb() is required to force the Bus Interface Unit
+		 * to clean oustanding writes, and forces an idle cycle
+		 * to be inserted.
+		 */
+		dmb();
+		reg = __raw_readl(b15_rac_base + RAC_FLUSH_REG);
+	} while (reg & FLUSH_RAC);
+}
+
+static inline u32 b15_rac_disable_and_flush(void)
+{
+	u32 reg;
+
+	reg = __b15_rac_disable();
+	__b15_rac_flush();
+	return reg;
+}
+
+static inline void __b15_rac_enable(u32 val)
+{
+	__raw_writel(val, b15_rac_base + RAC_CONFIG0_REG);
+	/* dsb() is required here to be consistent with __flush_icache_all() */
+	dsb();
+}
+
+#define BUILD_RAC_CACHE_OP(name, bar)				\
+void b15_flush_##name(void)					\
+{								\
+	unsigned int do_flush;					\
+	u32 val = 0;						\
+								\
+	if (test_bit(RAC_SUSPENDED, &b15_rac_flags)) {		\
+		v7_flush_##name();				\
+		bar;						\
+		return;						\
+	}							\
+								\
+	spin_lock(&rac_lock);					\
+	do_flush = test_bit(RAC_ENABLED, &b15_rac_flags);	\
+	if (do_flush)						\
+		val = b15_rac_disable_and_flush();		\
+	v7_flush_##name();					\
+	if (!do_flush)						\
+		bar;						\
+	else							\
+		__b15_rac_enable(val);				\
+	spin_unlock(&rac_lock);					\
+}
+
+#define nobarrier
+
+/* The readahead cache present in the Brahma-B15 CPU is a special piece of
+ * hardware after the integrated L2 cache of the B15 CPU complex whose purpose
+ * is to prefetch instruction and/or data with a line size of either 64 bytes
+ * or 256 bytes. The rationale is that the data-bus of the CPU interface is
+ * optimized for 256-bytes transactions, and enabling the readahead cache
+ * provides a significant performance boost we want it enabled (typically
+ * twice the performance for a memcpy benchmark application).
+ *
+ * The readahead cache is transparent for Modified Virtual Addresses
+ * cache maintenance operations: ICIMVAU, DCIMVAC, DCCMVAC, DCCMVAU and
+ * DCCIMVAC.
+ *
+ * It is however not transparent for the following cache maintenance
+ * operations: DCISW, DCCSW, DCCISW, ICIALLUIS and ICIALLU which is precisely
+ * what we are patching here with our BUILD_RAC_CACHE_OP here.
+ */
+BUILD_RAC_CACHE_OP(kern_cache_all, nobarrier);
+
+static void b15_rac_enable(void)
+{
+	unsigned int cpu;
+	u32 enable = 0;
+
+	for_each_possible_cpu(cpu)
+		enable |= (RAC_DATA_INST_EN_MASK << (cpu * RAC_CPU_SHIFT));
+
+	b15_rac_disable_and_flush();
+	__b15_rac_enable(enable);
+}
+
+static int b15_rac_reboot_notifier(struct notifier_block *nb,
+				   unsigned long action,
+				   void *data)
+{
+	/* During kexec, we are not yet migrated on the boot CPU, so we need to
+	 * make sure we are SMP safe here. Once the RAC is disabled, flag it as
+	 * suspended such that the hotplug notifier returns early.
+	 */
+	if (action == SYS_RESTART) {
+		spin_lock(&rac_lock);
+		b15_rac_disable_and_flush();
+		clear_bit(RAC_ENABLED, &b15_rac_flags);
+		set_bit(RAC_SUSPENDED, &b15_rac_flags);
+		spin_unlock(&rac_lock);
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block b15_rac_reboot_nb = {
+	.notifier_call	= b15_rac_reboot_notifier,
+};
+
+/* The CPU hotplug case is the most interesting one, we basically need to make
+ * sure that the RAC is disabled for the entire system prior to having a CPU
+ * die, in particular prior to this dying CPU having exited the coherency
+ * domain.
+ *
+ * Once this CPU is marked dead, we can safely re-enable the RAC for the
+ * remaining CPUs in the system which are still online.
+ *
+ * Offlining a CPU is the problematic case, onlining a CPU is not much of an
+ * issue since the CPU and its cache-level hierarchy will start filling with
+ * the RAC disabled, so L1 and L2 only.
+ *
+ * In this function, we should NOT have to verify any unsafe setting/condition
+ * b15_rac_base:
+ *
+ *   It is protected by the RAC_ENABLED flag which is cleared by default, and
+ *   being cleared when initial procedure is done. b15_rac_base had been set at
+ *   that time.
+ *
+ * RAC_ENABLED:
+ *   There is a small timing windows, in b15_rac_init(), between
+ *      cpuhp_setup_state_*()
+ *      ...
+ *      set RAC_ENABLED
+ *   However, there is no hotplug activity based on the Linux booting procedure.
+ *
+ * Since we have to disable RAC for all cores, we keep RAC on as long as as
+ * possible (disable it as late as possible) to gain the cache benefit.
+ *
+ * Thus, dying/dead states are chosen here
+ *
+ * We are choosing not do disable the RAC on a per-CPU basis, here, if we did
+ * we would want to consider disabling it as early as possible to benefit the
+ * other active CPUs.
+ */
+
+/* Running on the dying CPU */
+static int b15_rac_dying_cpu(unsigned int cpu)
+{
+	/* During kexec/reboot, the RAC is disabled via the reboot notifier
+	 * return early here.
+	 */
+	if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
+		return 0;
+
+	spin_lock(&rac_lock);
+
+	/* Indicate that we are starting a hotplug procedure */
+	__clear_bit(RAC_ENABLED, &b15_rac_flags);
+
+	/* Disable the readahead cache and save its value to a global */
+	rac_config0_reg = b15_rac_disable_and_flush();
+
+	spin_unlock(&rac_lock);
+
+	return 0;
+}
+
+/* Running on a non-dying CPU */
+static int b15_rac_dead_cpu(unsigned int cpu)
+{
+	/* During kexec/reboot, the RAC is disabled via the reboot notifier
+	 * return early here.
+	 */
+	if (test_bit(RAC_SUSPENDED, &b15_rac_flags))
+		return 0;
+
+	spin_lock(&rac_lock);
+
+	/* And enable it */
+	__b15_rac_enable(rac_config0_reg);
+	__set_bit(RAC_ENABLED, &b15_rac_flags);
+
+	spin_unlock(&rac_lock);
+
+	return 0;
+}
+
+static int b15_rac_suspend(void)
+{
+	/* Suspend the read-ahead cache oeprations, forcing our cache
+	 * implementation to fallback to the regular ARMv7 calls.
+	 *
+	 * We are guaranteed to be running on the boot CPU at this point and
+	 * with every other CPU quiesced, so setting RAC_SUSPENDED is not racy
+	 * here.
+	 */
+	rac_config0_reg = b15_rac_disable_and_flush();
+	set_bit(RAC_SUSPENDED, &b15_rac_flags);
+
+	return 0;
+}
+
+static void b15_rac_resume(void)
+{
+	/* Coming out of a S3 suspend/resume cycle, the read-ahead cache
+	 * register RAC_CONFIG0_REG will be restored to its default value, make
+	 * sure we re-enable it and set the enable flag, we are also guaranteed
+	 * to run on the boot CPU, so not racy again.
+	 */
+	__b15_rac_enable(rac_config0_reg);
+	clear_bit(RAC_SUSPENDED, &b15_rac_flags);
+}
+
+static struct syscore_ops b15_rac_syscore_ops = {
+	.suspend	= b15_rac_suspend,
+	.resume		= b15_rac_resume,
+};
+
+static int __init b15_rac_init(void)
+{
+	struct device_node *dn;
+	int ret = 0, cpu;
+	u32 reg, en_mask = 0;
+
+	dn = of_find_compatible_node(NULL, NULL, "brcm,brcmstb-cpu-biu-ctrl");
+	if (!dn)
+		return -ENODEV;
+
+	if (WARN(num_possible_cpus() > 4, "RAC only supports 4 CPUs\n"))
+		goto out;
+
+	b15_rac_base = of_iomap(dn, 0);
+	if (!b15_rac_base) {
+		pr_err("failed to remap BIU control base\n");
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = register_reboot_notifier(&b15_rac_reboot_nb);
+	if (ret) {
+		pr_err("failed to register reboot notifier\n");
+		iounmap(b15_rac_base);
+		goto out;
+	}
+
+	if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
+		ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DEAD,
+					"arm/cache-b15-rac:dead",
+					NULL, b15_rac_dead_cpu);
+		if (ret)
+			goto out_unmap;
+
+		ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING,
+					"arm/cache-b15-rac:dying",
+					NULL, b15_rac_dying_cpu);
+		if (ret)
+			goto out_cpu_dead;
+	}
+
+	if (IS_ENABLED(CONFIG_PM_SLEEP))
+		register_syscore_ops(&b15_rac_syscore_ops);
+
+	spin_lock(&rac_lock);
+	reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG);
+	for_each_possible_cpu(cpu)
+		en_mask |= ((1 << RACPREFDATA_SHIFT) << (cpu * RAC_CPU_SHIFT));
+	WARN(reg & en_mask, "Read-ahead cache not previously disabled\n");
+
+	b15_rac_enable();
+	set_bit(RAC_ENABLED, &b15_rac_flags);
+	spin_unlock(&rac_lock);
+
+	pr_info("Broadcom Brahma-B15 readahead cache at: 0x%p\n",
+		b15_rac_base + RAC_CONFIG0_REG);
+
+	goto out;
+
+out_cpu_dead:
+	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CACHE_B15_RAC_DYING);
+out_unmap:
+	unregister_reboot_notifier(&b15_rac_reboot_nb);
+	iounmap(b15_rac_base);
+out:
+	of_node_put(dn);
+	return ret;
+}
+arch_initcall(b15_rac_init);
diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S
index de78109d002d..215df435bfb9 100644
--- a/arch/arm/mm/cache-v7.S
+++ b/arch/arm/mm/cache-v7.S
@@ -15,6 +15,7 @@
 #include <asm/assembler.h>
 #include <asm/errno.h>
 #include <asm/unwind.h>
+#include <asm/hardware/cache-b15-rac.h>
 
 #include "proc-macros.S"
 
@@ -446,3 +447,23 @@ ENDPROC(v7_dma_unmap_area)
 
 	@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
 	define_cache_functions v7
+
+	/* The Broadcom Brahma-B15 read-ahead cache requires some modifications
+	 * to the v7_cache_fns, we only override the ones we need
+	 */
+#ifndef CONFIG_CACHE_B15_RAC
+	globl_equ	b15_flush_kern_cache_all,	v7_flush_kern_cache_all
+#endif
+	globl_equ	b15_flush_icache_all,		v7_flush_icache_all
+	globl_equ	b15_flush_kern_cache_louis,	v7_flush_kern_cache_louis
+	globl_equ	b15_flush_user_cache_all,	v7_flush_user_cache_all
+	globl_equ	b15_flush_user_cache_range,	v7_flush_user_cache_range
+	globl_equ	b15_coherent_kern_range,	v7_coherent_kern_range
+	globl_equ	b15_coherent_user_range,	v7_coherent_user_range
+	globl_equ	b15_flush_kern_dcache_area,	v7_flush_kern_dcache_area
+
+	globl_equ	b15_dma_map_area,		v7_dma_map_area
+	globl_equ	b15_dma_unmap_area,		v7_dma_unmap_area
+	globl_equ	b15_dma_flush_range,		v7_dma_flush_range
+
+	define_cache_functions b15
diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c
index 6db5fc26d154..619f24a42d09 100644
--- a/arch/arm/mm/dma-mapping-nommu.c
+++ b/arch/arm/mm/dma-mapping-nommu.c
@@ -11,7 +11,7 @@
 
 #include <linux/export.h>
 #include <linux/mm.h>
-#include <linux/dma-mapping.h>
+#include <linux/dma-direct.h>
 #include <linux/scatterlist.h>
 
 #include <asm/cachetype.h>
@@ -22,7 +22,7 @@
 #include "dma.h"
 
 /*
- *  dma_noop_ops is used if
+ *  dma_direct_ops is used if
  *   - MMU/MPU is off
  *   - cpu is v7m w/o cache support
  *   - device is coherent
@@ -39,7 +39,6 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
 				 unsigned long attrs)
 
 {
-	const struct dma_map_ops *ops = &dma_noop_ops;
 	void *ret;
 
 	/*
@@ -48,7 +47,7 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
 	 */
 
 	if (attrs & DMA_ATTR_NON_CONSISTENT)
-		return ops->alloc(dev, size, dma_handle, gfp, attrs);
+		return dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
 
 	ret = dma_alloc_from_global_coherent(size, dma_handle);
 
@@ -70,10 +69,8 @@ static void arm_nommu_dma_free(struct device *dev, size_t size,
 			       void *cpu_addr, dma_addr_t dma_addr,
 			       unsigned long attrs)
 {
-	const struct dma_map_ops *ops = &dma_noop_ops;
-
 	if (attrs & DMA_ATTR_NON_CONSISTENT) {
-		ops->free(dev, size, cpu_addr, dma_addr, attrs);
+		dma_direct_free(dev, size, cpu_addr, dma_addr, attrs);
 	} else {
 		int ret = dma_release_from_global_coherent(get_order(size),
 							   cpu_addr);
@@ -213,7 +210,7 @@ EXPORT_SYMBOL(arm_nommu_dma_ops);
 
 static const struct dma_map_ops *arm_nommu_get_dma_map_ops(bool coherent)
 {
-	return coherent ? &dma_noop_ops : &arm_nommu_dma_ops;
+	return coherent ? &dma_direct_ops : &arm_nommu_dma_ops;
 }
 
 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c
index fc3b44028cfb..084779c5c893 100644
--- a/arch/arm/mm/dump.c
+++ b/arch/arm/mm/dump.c
@@ -21,11 +21,7 @@
 #include <asm/fixmap.h>
 #include <asm/memory.h>
 #include <asm/pgtable.h>
-
-struct addr_marker {
-	unsigned long start_address;
-	const char *name;
-};
+#include <asm/ptdump.h>
 
 static struct addr_marker address_markers[] = {
 	{ MODULES_VADDR,	"Modules" },
@@ -38,12 +34,26 @@ static struct addr_marker address_markers[] = {
 	{ -1,			NULL },
 };
 
+#define pt_dump_seq_printf(m, fmt, args...) \
+({                      \
+	if (m)					\
+		seq_printf(m, fmt, ##args);	\
+})
+
+#define pt_dump_seq_puts(m, fmt)    \
+({						\
+	if (m)					\
+		seq_printf(m, fmt);	\
+})
+
 struct pg_state {
 	struct seq_file *seq;
 	const struct addr_marker *marker;
 	unsigned long start_address;
 	unsigned level;
 	u64 current_prot;
+	bool check_wx;
+	unsigned long wx_pages;
 	const char *current_domain;
 };
 
@@ -52,6 +62,8 @@ struct prot_bits {
 	u64		val;
 	const char	*set;
 	const char	*clear;
+	bool		ro_bit;
+	bool		nx_bit;
 };
 
 static const struct prot_bits pte_bits[] = {
@@ -65,11 +77,13 @@ static const struct prot_bits pte_bits[] = {
 		.val	= L_PTE_RDONLY,
 		.set	= "ro",
 		.clear	= "RW",
+		.ro_bit	= true,
 	}, {
 		.mask	= L_PTE_XN,
 		.val	= L_PTE_XN,
 		.set	= "NX",
 		.clear	= "x ",
+		.nx_bit	= true,
 	}, {
 		.mask	= L_PTE_SHARED,
 		.val	= L_PTE_SHARED,
@@ -133,11 +147,13 @@ static const struct prot_bits section_bits[] = {
 		.val	= L_PMD_SECT_RDONLY | PMD_SECT_AP2,
 		.set	= "ro",
 		.clear	= "RW",
+		.ro_bit	= true,
 #elif __LINUX_ARM_ARCH__ >= 6
 	{
 		.mask	= PMD_SECT_APX | PMD_SECT_AP_READ | PMD_SECT_AP_WRITE,
 		.val	= PMD_SECT_APX | PMD_SECT_AP_WRITE,
 		.set	= "    ro",
+		.ro_bit	= true,
 	}, {
 		.mask	= PMD_SECT_APX | PMD_SECT_AP_READ | PMD_SECT_AP_WRITE,
 		.val	= PMD_SECT_AP_WRITE,
@@ -156,6 +172,7 @@ static const struct prot_bits section_bits[] = {
 		.mask   = PMD_SECT_AP_READ | PMD_SECT_AP_WRITE,
 		.val    = 0,
 		.set    = "    ro",
+		.ro_bit	= true,
 	}, {
 		.mask   = PMD_SECT_AP_READ | PMD_SECT_AP_WRITE,
 		.val    = PMD_SECT_AP_WRITE,
@@ -174,6 +191,7 @@ static const struct prot_bits section_bits[] = {
 		.val	= PMD_SECT_XN,
 		.set	= "NX",
 		.clear	= "x ",
+		.nx_bit	= true,
 	}, {
 		.mask	= PMD_SECT_S,
 		.val	= PMD_SECT_S,
@@ -186,6 +204,8 @@ struct pg_level {
 	const struct prot_bits *bits;
 	size_t num;
 	u64 mask;
+	const struct prot_bits *ro_bit;
+	const struct prot_bits *nx_bit;
 };
 
 static struct pg_level pg_level[] = {
@@ -214,10 +234,27 @@ static void dump_prot(struct pg_state *st, const struct prot_bits *bits, size_t
 			s = bits->clear;
 
 		if (s)
-			seq_printf(st->seq, " %s", s);
+			pt_dump_seq_printf(st->seq, " %s", s);
 	}
 }
 
+static void note_prot_wx(struct pg_state *st, unsigned long addr)
+{
+	if (!st->check_wx)
+		return;
+	if ((st->current_prot & pg_level[st->level].ro_bit->mask) ==
+				pg_level[st->level].ro_bit->val)
+		return;
+	if ((st->current_prot & pg_level[st->level].nx_bit->mask) ==
+				pg_level[st->level].nx_bit->val)
+		return;
+
+	WARN_ONCE(1, "arm/mm: Found insecure W+X mapping at address %pS\n",
+			(void *)st->start_address);
+
+	st->wx_pages += (addr - st->start_address) / PAGE_SIZE;
+}
+
 static void note_page(struct pg_state *st, unsigned long addr,
 		      unsigned int level, u64 val, const char *domain)
 {
@@ -228,7 +265,7 @@ static void note_page(struct pg_state *st, unsigned long addr,
 		st->level = level;
 		st->current_prot = prot;
 		st->current_domain = domain;
-		seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+		pt_dump_seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
 	} else if (prot != st->current_prot || level != st->level ||
 		   domain != st->current_domain ||
 		   addr >= st->marker[1].start_address) {
@@ -236,7 +273,8 @@ static void note_page(struct pg_state *st, unsigned long addr,
 		unsigned long delta;
 
 		if (st->current_prot) {
-			seq_printf(st->seq, "0x%08lx-0x%08lx   ",
+			note_prot_wx(st, addr);
+			pt_dump_seq_printf(st->seq, "0x%08lx-0x%08lx   ",
 				   st->start_address, addr);
 
 			delta = (addr - st->start_address) >> 10;
@@ -244,17 +282,19 @@ static void note_page(struct pg_state *st, unsigned long addr,
 				delta >>= 10;
 				unit++;
 			}
-			seq_printf(st->seq, "%9lu%c", delta, *unit);
+			pt_dump_seq_printf(st->seq, "%9lu%c", delta, *unit);
 			if (st->current_domain)
-				seq_printf(st->seq, " %s", st->current_domain);
+				pt_dump_seq_printf(st->seq, " %s",
+							st->current_domain);
 			if (pg_level[st->level].bits)
 				dump_prot(st, pg_level[st->level].bits, pg_level[st->level].num);
-			seq_printf(st->seq, "\n");
+			pt_dump_seq_printf(st->seq, "\n");
 		}
 
 		if (addr >= st->marker[1].start_address) {
 			st->marker++;
-			seq_printf(st->seq, "---[ %s ]---\n", st->marker->name);
+			pt_dump_seq_printf(st->seq, "---[ %s ]---\n",
+							st->marker->name);
 		}
 		st->start_address = addr;
 		st->current_prot = prot;
@@ -335,61 +375,82 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
 	}
 }
 
-static void walk_pgd(struct seq_file *m)
+static void walk_pgd(struct pg_state *st, struct mm_struct *mm,
+			unsigned long start)
 {
-	pgd_t *pgd = swapper_pg_dir;
-	struct pg_state st;
-	unsigned long addr;
+	pgd_t *pgd = pgd_offset(mm, 0UL);
 	unsigned i;
-
-	memset(&st, 0, sizeof(st));
-	st.seq = m;
-	st.marker = address_markers;
+	unsigned long addr;
 
 	for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
-		addr = i * PGDIR_SIZE;
+		addr = start + i * PGDIR_SIZE;
 		if (!pgd_none(*pgd)) {
-			walk_pud(&st, pgd, addr);
+			walk_pud(st, pgd, addr);
 		} else {
-			note_page(&st, addr, 1, pgd_val(*pgd), NULL);
+			note_page(st, addr, 1, pgd_val(*pgd), NULL);
 		}
 	}
-
-	note_page(&st, 0, 0, 0, NULL);
 }
 
-static int ptdump_show(struct seq_file *m, void *v)
+void ptdump_walk_pgd(struct seq_file *m, struct ptdump_info *info)
 {
-	walk_pgd(m);
-	return 0;
-}
+	struct pg_state st = {
+		.seq = m,
+		.marker = info->markers,
+		.check_wx = false,
+	};
 
-static int ptdump_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ptdump_show, NULL);
+	walk_pgd(&st, info->mm, info->base_addr);
+	note_page(&st, 0, 0, 0, NULL);
 }
 
-static const struct file_operations ptdump_fops = {
-	.open		= ptdump_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-
-static int ptdump_init(void)
+static void ptdump_initialize(void)
 {
-	struct dentry *pe;
 	unsigned i, j;
 
 	for (i = 0; i < ARRAY_SIZE(pg_level); i++)
 		if (pg_level[i].bits)
-			for (j = 0; j < pg_level[i].num; j++)
+			for (j = 0; j < pg_level[i].num; j++) {
 				pg_level[i].mask |= pg_level[i].bits[j].mask;
+				if (pg_level[i].bits[j].ro_bit)
+					pg_level[i].ro_bit = &pg_level[i].bits[j];
+				if (pg_level[i].bits[j].nx_bit)
+					pg_level[i].nx_bit = &pg_level[i].bits[j];
+			}
 
 	address_markers[2].start_address = VMALLOC_START;
+}
+
+static struct ptdump_info kernel_ptdump_info = {
+	.mm = &init_mm,
+	.markers = address_markers,
+	.base_addr = 0,
+};
 
-	pe = debugfs_create_file("kernel_page_tables", 0400, NULL, NULL,
-				 &ptdump_fops);
-	return pe ? 0 : -ENOMEM;
+void ptdump_check_wx(void)
+{
+	struct pg_state st = {
+		.seq = NULL,
+		.marker = (struct addr_marker[]) {
+			{ 0, NULL},
+			{ -1, NULL},
+		},
+		.check_wx = true,
+	};
+
+	walk_pgd(&st, &init_mm, 0);
+	note_page(&st, 0, 0, 0, NULL);
+	if (st.wx_pages)
+		pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found\n",
+			st.wx_pages);
+	else
+		pr_info("Checked W+X mappings: passed, no W+X pages found\n");
+}
+
+static int ptdump_init(void)
+{
+	ptdump_initialize();
+	return ptdump_debugfs_register(&kernel_ptdump_info,
+					"kernel_page_tables");
 }
 __initcall(ptdump_init);
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index 42f585379e19..b75eada23d0a 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -21,7 +21,6 @@
 #include <linux/highmem.h>
 #include <linux/perf_event.h>
 
-#include <asm/exception.h>
 #include <asm/pgtable.h>
 #include <asm/system_misc.h>
 #include <asm/system_info.h>
@@ -545,7 +544,7 @@ hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *)
 /*
  * Dispatch a data abort to the relevant handler.
  */
-asmlinkage void __exception
+asmlinkage void
 do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
 {
 	const struct fsr_info *inf = fsr_info + fsr_fs(fsr);
@@ -578,7 +577,7 @@ hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *
 	ifsr_info[nr].name = name;
 }
 
-asmlinkage void __exception
+asmlinkage void
 do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
 {
 	const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr);
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index 10bfba85eb96..1d1edd064199 100644
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -16,8 +16,8 @@
  * are not supported on any CPU using the idmap tables as its current
  * page tables.
  */
-pgd_t *idmap_pgd;
-long long arch_phys_to_idmap_offset;
+pgd_t *idmap_pgd __ro_after_init;
+long long arch_phys_to_idmap_offset __ro_after_init;
 
 #ifdef CONFIG_ARM_LPAE
 static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index a1f11a7ee81b..bd6f4513539a 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -36,6 +36,7 @@
 #include <asm/system_info.h>
 #include <asm/tlb.h>
 #include <asm/fixmap.h>
+#include <asm/ptdump.h>
 
 #include <asm/mach/arch.h>
 #include <asm/mach/map.h>
@@ -738,6 +739,7 @@ static int __mark_rodata_ro(void *unused)
 void mark_rodata_ro(void)
 {
 	stop_machine(__mark_rodata_ro, NULL, NULL);
+	debug_checkwx();
 }
 
 void set_kernel_text_rw(void)
diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c
index e4370810f4f1..7c087961b7ce 100644
--- a/arch/arm/mm/nommu.c
+++ b/arch/arm/mm/nommu.c
@@ -31,7 +31,7 @@ struct mpu_rgn_info mpu_rgn_info;
 
 #ifdef CONFIG_CPU_CP15
 #ifdef CONFIG_CPU_HIGH_VECTOR
-static unsigned long __init setup_vectors_base(void)
+unsigned long setup_vectors_base(void)
 {
 	unsigned long reg = get_cr();
 
@@ -57,7 +57,7 @@ static inline bool security_extensions_enabled(void)
 	return 0;
 }
 
-static unsigned long __init setup_vectors_base(void)
+unsigned long setup_vectors_base(void)
 {
 	unsigned long base = 0, reg = get_cr();
 
diff --git a/arch/arm/mm/pmsa-v7.c b/arch/arm/mm/pmsa-v7.c
index 976df60ac426..e2853bfff74e 100644
--- a/arch/arm/mm/pmsa-v7.c
+++ b/arch/arm/mm/pmsa-v7.c
@@ -6,6 +6,7 @@
 
 #include <linux/bitops.h>
 #include <linux/memblock.h>
+#include <linux/string.h>
 
 #include <asm/cacheflush.h>
 #include <asm/cp15.h>
@@ -296,6 +297,7 @@ void __init adjust_lowmem_bounds_mpu(void)
 		}
 	}
 
+	memset(mem, 0, sizeof(mem));
 	num = allocate_region(mem_start, specified_mem_size, mem_max_regions, mem);
 
 	for (i = 0; i < num; i++) {
@@ -433,7 +435,7 @@ void __init mpu_setup(void)
 
 	/* Background */
 	err |= mpu_setup_region(region++, 0, 32,
-				MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0NA,
+				MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0RW,
 				0, false);
 
 #ifdef CONFIG_XIP_KERNEL
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 01d64c0b2563..d55d493f9a1e 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -567,7 +567,7 @@ __v7_setup_stack:
 	/*
 	 * Standard v7 proc info content
 	 */
-.macro __v7_proc name, initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0, proc_fns = v7_processor_functions
+.macro __v7_proc name, initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0, proc_fns = v7_processor_functions, cache_fns = v7_cache_fns
 	ALT_SMP(.long	PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
 			PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags)
 	ALT_UP(.long	PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
@@ -583,7 +583,7 @@ __v7_setup_stack:
 	.long	\proc_fns
 	.long	v7wbi_tlb_fns
 	.long	v6_user_fns
-	.long	v7_cache_fns
+	.long	\cache_fns
 .endm
 
 #ifndef CONFIG_ARM_LPAE
@@ -678,7 +678,7 @@ __v7_ca15mp_proc_info:
 __v7_b15mp_proc_info:
 	.long	0x420f00f0
 	.long	0xff0ffff0
-	__v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup
+	__v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup, cache_fns = b15_cache_fns
 	.size	__v7_b15mp_proc_info, . - __v7_b15mp_proc_info
 
 	/*
diff --git a/arch/arm/mm/ptdump_debugfs.c b/arch/arm/mm/ptdump_debugfs.c
new file mode 100644
index 000000000000..be8d87be4b93
--- /dev/null
+++ b/arch/arm/mm/ptdump_debugfs.c
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
+#include <asm/ptdump.h>
+
+static int ptdump_show(struct seq_file *m, void *v)
+{
+	struct ptdump_info *info = m->private;
+
+	ptdump_walk_pgd(m, info);
+	return 0;
+}
+
+static int ptdump_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ptdump_show, inode->i_private);
+}
+
+static const struct file_operations ptdump_fops = {
+	.open		= ptdump_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+int ptdump_debugfs_register(struct ptdump_info *info, const char *name)
+{
+	struct dentry *pe;
+
+	pe = debugfs_create_file(name, 0400, NULL, info, &ptdump_fops);
+	return pe ? 0 : -ENOMEM;
+
+}
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 323a4df59a6c..b5030e1a41d8 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -25,8 +25,6 @@
 
 #include "bpf_jit_32.h"
 
-int bpf_jit_enable __read_mostly;
-
 /*
  * eBPF prog stack layout:
  *
@@ -365,15 +363,7 @@ static inline int epilogue_offset(const struct jit_ctx *ctx)
 static inline void emit_udivmod(u8 rd, u8 rm, u8 rn, struct jit_ctx *ctx, u8 op)
 {
 	const u8 *tmp = bpf2a32[TMP_REG_1];
-	s32 jmp_offset;
 
-	/* checks if divisor is zero or not. If it is, then
-	 * exit directly.
-	 */
-	emit(ARM_CMP_I(rn, 0), ctx);
-	_emit(ARM_COND_EQ, ARM_MOV_I(ARM_R0, 0), ctx);
-	jmp_offset = epilogue_offset(ctx);
-	_emit(ARM_COND_EQ, ARM_B(jmp_offset), ctx);
 #if __LINUX_ARM_ARCH__ == 7
 	if (elf_hwcap & HWCAP_IDIVA) {
 		if (op == BPF_DIV)
@@ -1821,7 +1811,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	/* If BPF JIT was not enabled then we must fall back to
 	 * the interpreter.
 	 */
-	if (!bpf_jit_enable)
+	if (!prog->jit_requested)
 		return orig_prog;
 
 	/* If constant blinding was enabled and we failed during blinding
diff --git a/arch/arm/plat-omap/dmtimer.c b/arch/arm/plat-omap/dmtimer.c
index d443e481c3e9..8805a59bae53 100644
--- a/arch/arm/plat-omap/dmtimer.c
+++ b/arch/arm/plat-omap/dmtimer.c
@@ -888,11 +888,8 @@ static int omap_dm_timer_probe(struct platform_device *pdev)
 	timer->irq = irq->start;
 	timer->pdev = pdev;
 
-	/* Skip pm_runtime_enable for OMAP1 */
-	if (!(timer->capability & OMAP_TIMER_NEEDS_RESET)) {
-		pm_runtime_enable(dev);
-		pm_runtime_irq_safe(dev);
-	}
+	pm_runtime_enable(dev);
+	pm_runtime_irq_safe(dev);
 
 	if (!timer->reserved) {
 		ret = pm_runtime_get_sync(dev);
diff --git a/arch/arm/plat-omap/include/plat/sram.h b/arch/arm/plat-omap/include/plat/sram.h
index fb061cf0d736..30a07730807a 100644
--- a/arch/arm/plat-omap/include/plat/sram.h
+++ b/arch/arm/plat-omap/include/plat/sram.h
@@ -5,13 +5,4 @@ void omap_map_sram(unsigned long start, unsigned long size,
 			unsigned long skip, int cached);
 void omap_sram_reset(void);
 
-extern void *omap_sram_push_address(unsigned long size);
-
-/* Macro to push a function to the internal SRAM, using the fncpy API */
-#define omap_sram_push(funcp, size) ({				\
-	typeof(&(funcp)) _res = NULL;				\
-	void *_sram_address = omap_sram_push_address(size);	\
-	if (_sram_address)					\
-		_res = fncpy(_sram_address, &(funcp), size);	\
-	_res;							\
-})
+extern void *omap_sram_push(void *funcp, unsigned long size);
diff --git a/arch/arm/plat-omap/sram.c b/arch/arm/plat-omap/sram.c
index a5bc92d7e476..921840acf65c 100644
--- a/arch/arm/plat-omap/sram.c
+++ b/arch/arm/plat-omap/sram.c
@@ -23,6 +23,7 @@
 #include <asm/fncpy.h>
 #include <asm/tlb.h>
 #include <asm/cacheflush.h>
+#include <asm/set_memory.h>
 
 #include <asm/mach/map.h>
 
@@ -42,7 +43,7 @@ static void __iomem *omap_sram_ceil;
  * Note that fncpy requires the returned address to be aligned
  * to an 8-byte boundary.
  */
-void *omap_sram_push_address(unsigned long size)
+static void *omap_sram_push_address(unsigned long size)
 {
 	unsigned long available, new_ceil = (unsigned long)omap_sram_ceil;
 
@@ -60,6 +61,30 @@ void *omap_sram_push_address(unsigned long size)
 	return (void *)omap_sram_ceil;
 }
 
+void *omap_sram_push(void *funcp, unsigned long size)
+{
+	void *sram;
+	unsigned long base;
+	int pages;
+	void *dst = NULL;
+
+	sram = omap_sram_push_address(size);
+	if (!sram)
+		return NULL;
+
+	base = (unsigned long)sram & PAGE_MASK;
+	pages = PAGE_ALIGN(size) / PAGE_SIZE;
+
+	set_memory_rw(base, pages);
+
+	dst = fncpy(sram, funcp, size);
+
+	set_memory_ro(base, pages);
+	set_memory_x(base, pages);
+
+	return dst;
+}
+
 /*
  * The SRAM context is lost during off-idle and stack
  * needs to be reset.
@@ -75,6 +100,9 @@ void omap_sram_reset(void)
 void __init omap_map_sram(unsigned long start, unsigned long size,
 				 unsigned long skip, int cached)
 {
+	unsigned long base;
+	int pages;
+
 	if (size == 0)
 		return;
 
@@ -95,4 +123,10 @@ void __init omap_map_sram(unsigned long start, unsigned long size,
 	 */
 	memset_io(omap_sram_base + omap_sram_skip, 0,
 		  omap_sram_size - omap_sram_skip);
+
+	base = (unsigned long)omap_sram_base;
+	pages = PAGE_ALIGN(omap_sram_size) / PAGE_SIZE;
+
+	set_memory_ro(base, pages);
+	set_memory_x(base, pages);
 }
diff --git a/arch/arm/plat-orion/common.c b/arch/arm/plat-orion/common.c
index aff6994950ba..a2399fd66e97 100644
--- a/arch/arm/plat-orion/common.c
+++ b/arch/arm/plat-orion/common.c
@@ -472,28 +472,27 @@ void __init orion_ge11_init(struct mv643xx_eth_platform_data *eth_data,
 /*****************************************************************************
  * Ethernet switch
  ****************************************************************************/
-static __initconst const char *orion_ge00_mvmdio_bus_name = "orion-mii";
-static __initdata struct mdio_board_info
-		  orion_ge00_switch_board_info;
+static __initdata struct mdio_board_info orion_ge00_switch_board_info = {
+	.bus_id   = "orion-mii",
+	.modalias = "mv88e6085",
+};
 
 void __init orion_ge00_switch_init(struct dsa_chip_data *d)
 {
-	struct mdio_board_info *bd;
 	unsigned int i;
 
 	if (!IS_BUILTIN(CONFIG_PHYLIB))
 		return;
 
-	for (i = 0; i < ARRAY_SIZE(d->port_names); i++)
-		if (!strcmp(d->port_names[i], "cpu"))
+	for (i = 0; i < ARRAY_SIZE(d->port_names); i++) {
+		if (!strcmp(d->port_names[i], "cpu")) {
+			d->netdev[i] = &orion_ge00.dev;
 			break;
+		}
+	}
 
-	bd = &orion_ge00_switch_board_info;
-	bd->bus_id = orion_ge00_mvmdio_bus_name;
-	bd->mdio_addr = d->sw_addr;
-	d->netdev[i] = &orion_ge00.dev;
-	strcpy(bd->modalias, "mv88e6085");
-	bd->platform_data = d;
+	orion_ge00_switch_board_info.mdio_addr = d->sw_addr;
+	orion_ge00_switch_board_info.platform_data = d;
 
 	mdiobus_register_board_info(&orion_ge00_switch_board_info, 1);
 }
diff --git a/arch/arm/plat-samsung/Kconfig b/arch/arm/plat-samsung/Kconfig
index 8d4a64cc644c..b600e38364eb 100644
--- a/arch/arm/plat-samsung/Kconfig
+++ b/arch/arm/plat-samsung/Kconfig
@@ -1,8 +1,6 @@
-# arch/arm/plat-samsung/Kconfig
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright 2009 Simtec Electronics
-#
-# Licensed under GPLv2
 
 config PLAT_SAMSUNG
 	bool
diff --git a/arch/arm/plat-samsung/Makefile b/arch/arm/plat-samsung/Makefile
index be172efec15c..3db9d2c38258 100644
--- a/arch/arm/plat-samsung/Makefile
+++ b/arch/arm/plat-samsung/Makefile
@@ -1,8 +1,6 @@
-# arch/arm/plat-samsung/Makefile
+# SPDX-License-Identifier: GPL-2.0
 #
 # Copyright 2009 Simtec Electronics
-#
-# Licensed under GPLv2
 
 ccflags-$(CONFIG_ARCH_S3C64XX) := -I$(srctree)/arch/arm/mach-s3c64xx/include
 ccflags-$(CONFIG_ARCH_MULTIPLATFORM) += -I$(srctree)/$(src)/include
diff --git a/arch/arm/plat-samsung/adc.c b/arch/arm/plat-samsung/adc.c
index e9de9e92ce01..42bac8d5ab5d 100644
--- a/arch/arm/plat-samsung/adc.c
+++ b/arch/arm/plat-samsung/adc.c
@@ -1,15 +1,10 @@
-/* arch/arm/plat-samsung/adc.c
- *
- * Copyright (c) 2008 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>, <ben-linux@fluff.org>
- *
- * Samsung ADC device core
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License.
-*/
+// SPDX-License-Identifier: GPL-1.0+
+//
+// Copyright (c) 2008 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>, <ben-linux@fluff.org>
+//
+// Samsung ADC device core
 
 #include <linux/module.h>
 #include <linux/kernel.h>
diff --git a/arch/arm/plat-samsung/cpu.c b/arch/arm/plat-samsung/cpu.c
index a107b3a0b095..e1ba88ba31d8 100644
--- a/arch/arm/plat-samsung/cpu.c
+++ b/arch/arm/plat-samsung/cpu.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/plat-samsung/cpu.c
- *
- * Copyright (c) 2009-2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * Samsung CPU Support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2009-2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// Samsung CPU Support
 
 #include <linux/module.h>
 #include <linux/kernel.h>
diff --git a/arch/arm/plat-samsung/dev-uart.c b/arch/arm/plat-samsung/dev-uart.c
index 5928105490fa..7476a5dbae77 100644
--- a/arch/arm/plat-samsung/dev-uart.c
+++ b/arch/arm/plat-samsung/dev-uart.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-samsung/dev-uart.c
- *	originally from arch/arm/plat-s3c24xx/devs.c
- *x
- * Copyright (c) 2004 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Base S3C24XX platform device definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+//	originally from arch/arm/plat-s3c24xx/devs.c
+//
+// Copyright (c) 2004 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Base S3C24XX platform device definitions
 
 #include <linux/kernel.h>
 #include <linux/platform_device.h>
diff --git a/arch/arm/plat-samsung/devs.c b/arch/arm/plat-samsung/devs.c
index 5668e4eb03df..1d1fa068d228 100644
--- a/arch/arm/plat-samsung/devs.c
+++ b/arch/arm/plat-samsung/devs.c
@@ -1,14 +1,9 @@
-/* linux/arch/arm/plat-samsung/devs.c
- *
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * Base SAMSUNG platform device definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+//
+// Base SAMSUNG platform device definitions
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/arch/arm/plat-samsung/gpio-samsung.c b/arch/arm/plat-samsung/gpio-samsung.c
index 7861488f7ca0..f66c820cd82b 100644
--- a/arch/arm/plat-samsung/gpio-samsung.c
+++ b/arch/arm/plat-samsung/gpio-samsung.c
@@ -1,18 +1,14 @@
-/*
- * Copyright (c) 2009-2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com/
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *      Ben Dooks <ben@simtec.co.uk>
- *      http://armlinux.simtec.co.uk/
- *
- * SAMSUNG - GPIOlib support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2009-2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com/
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//      Ben Dooks <ben@simtec.co.uk>
+//      http://armlinux.simtec.co.uk/
+//
+// SAMSUNG - GPIOlib support
 
 #include <linux/kernel.h>
 #include <linux/irq.h>
diff --git a/arch/arm/plat-samsung/include/plat/adc-core.h b/arch/arm/plat-samsung/include/plat/adc-core.h
index a927bee55359..039f6862b6a7 100644
--- a/arch/arm/plat-samsung/include/plat/adc-core.h
+++ b/arch/arm/plat-samsung/include/plat/adc-core.h
@@ -1,14 +1,10 @@
-/* linux/arch/arm/plat-samsung/include/plat/adc-core.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2010 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com/
  *
  * Samsung ADC Controller core functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_PLAT_ADC_CORE_H
 #define __ASM_PLAT_ADC_CORE_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/adc.h b/arch/arm/plat-samsung/include/plat/adc.h
index 2fc89315553f..74d1a46408c1 100644
--- a/arch/arm/plat-samsung/include/plat/adc.h
+++ b/arch/arm/plat-samsung/include/plat/adc.h
@@ -1,15 +1,11 @@
-/* arch/arm/plat-samsung/include/plat/adc.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/	
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C ADC driver information
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_PLAT_ADC_H
 #define __ASM_PLAT_ADC_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/cpu-freq-core.h b/arch/arm/plat-samsung/include/plat/cpu-freq-core.h
index 37cf20e04aff..2c7cf2665634 100644
--- a/arch/arm/plat-samsung/include/plat/cpu-freq-core.h
+++ b/arch/arm/plat-samsung/include/plat/cpu-freq-core.h
@@ -1,15 +1,11 @@
-/* arch/arm/plat-samsung/include/plat/cpu-freq-core.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2006-2009 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C CPU frequency scaling support - core support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <plat/cpu-freq.h>
 
diff --git a/arch/arm/plat-samsung/include/plat/cpu-freq.h b/arch/arm/plat-samsung/include/plat/cpu-freq.h
index 85517ab962ae..558892bcf9b6 100644
--- a/arch/arm/plat-samsung/include/plat/cpu-freq.h
+++ b/arch/arm/plat-samsung/include/plat/cpu-freq.h
@@ -1,15 +1,11 @@
-/* arch/arm/plat-samsung/include/plat/cpu-freq.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2006-2007 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C CPU frequency scaling support - driver and board
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <linux/cpufreq.h>
 
diff --git a/arch/arm/plat-samsung/include/plat/cpu.h b/arch/arm/plat-samsung/include/plat/cpu.h
index b7b702a72cac..fadcddbea064 100644
--- a/arch/arm/plat-samsung/include/plat/cpu.h
+++ b/arch/arm/plat-samsung/include/plat/cpu.h
@@ -1,5 +1,5 @@
-/* linux/arch/arm/plat-samsung/include/plat/cpu.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com/
  *
@@ -7,11 +7,7 @@
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * Header file for Samsung CPU support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /* todo - fix when rmk changes iodescs to use `void __iomem *` */
 
diff --git a/arch/arm/plat-samsung/include/plat/devs.h b/arch/arm/plat-samsung/include/plat/devs.h
index e23fed311e5f..02b0c5750572 100644
--- a/arch/arm/plat-samsung/include/plat/devs.h
+++ b/arch/arm/plat-samsung/include/plat/devs.h
@@ -1,5 +1,5 @@
-/* arch/arm/plat-samsung/include/plat/devs.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
@@ -7,11 +7,7 @@
  * Ben Dooks <ben@simtec.co.uk>
  *
  * Header file for s3c2410 standard platform devices
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __PLAT_DEVS_H
 #define __PLAT_DEVS_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/fb-s3c2410.h b/arch/arm/plat-samsung/include/plat/fb-s3c2410.h
index 1f2972a74e9f..614240d768b4 100644
--- a/arch/arm/plat-samsung/include/plat/fb-s3c2410.h
+++ b/arch/arm/plat-samsung/include/plat/fb-s3c2410.h
@@ -1,12 +1,8 @@
-/* arch/arm/plat-samsung/include/plat/fb-s3c2410.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2004 Arnaud Patard <arnaud.patard@rtp-net.org>
  *
  * Inspired by pxafb.h
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
 */
 
 #ifndef __ASM_PLAT_FB_S3C2410_H
diff --git a/arch/arm/plat-samsung/include/plat/fb.h b/arch/arm/plat-samsung/include/plat/fb.h
index b89f8f208515..615d381ae32e 100644
--- a/arch/arm/plat-samsung/include/plat/fb.h
+++ b/arch/arm/plat-samsung/include/plat/fb.h
@@ -1,16 +1,12 @@
-/* arch/arm/plat-samsung/include/plat/fb.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C - FB platform data definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __PLAT_S3C_FB_H
 #define __PLAT_S3C_FB_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/gpio-cfg-helpers.h b/arch/arm/plat-samsung/include/plat/gpio-cfg-helpers.h
index a181d7ce81cf..db0c56f5ca15 100644
--- a/arch/arm/plat-samsung/include/plat/gpio-cfg-helpers.h
+++ b/arch/arm/plat-samsung/include/plat/gpio-cfg-helpers.h
@@ -1,16 +1,12 @@
-/* linux/arch/arm/plat-samsung/include/plat/gpio-cfg-helper.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * Samsung Platform - GPIO pin configuration helper definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /* This is meant for core cpu support, machine or other driver files
  * should not be including this header.
diff --git a/arch/arm/plat-samsung/include/plat/gpio-cfg.h b/arch/arm/plat-samsung/include/plat/gpio-cfg.h
index e55d1f597db8..469c220e092b 100644
--- a/arch/arm/plat-samsung/include/plat/gpio-cfg.h
+++ b/arch/arm/plat-samsung/include/plat/gpio-cfg.h
@@ -1,16 +1,12 @@
-/* linux/arch/arm/plat-s3c/include/plat/gpio-cfg.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C Platform - GPIO pin configuration
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /* This file contains the necessary definitions to get the basic gpio
  * pin configuration done such as setting a pin to input or output or
diff --git a/arch/arm/plat-samsung/include/plat/gpio-core.h b/arch/arm/plat-samsung/include/plat/gpio-core.h
index 6ce11bfdc37e..51e721f5e491 100644
--- a/arch/arm/plat-samsung/include/plat/gpio-core.h
+++ b/arch/arm/plat-samsung/include/plat/gpio-core.h
@@ -1,15 +1,11 @@
-/* linux/arch/arm/plat-s3c/include/plat/gpio-core.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C Platform - GPIO core
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __PLAT_SAMSUNG_GPIO_CORE_H
 #define __PLAT_SAMSUNG_GPIO_CORE_H
diff --git a/arch/arm/plat-samsung/include/plat/iic-core.h b/arch/arm/plat-samsung/include/plat/iic-core.h
index f182669b8e8e..c5cfd5af3874 100644
--- a/arch/arm/plat-samsung/include/plat/iic-core.h
+++ b/arch/arm/plat-samsung/include/plat/iic-core.h
@@ -1,15 +1,11 @@
-/* arch/arm/mach-s3c2410/include/mach/iic-core.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Openmoko, Inc.
  * Copyright 2008 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C - I2C Controller core functions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_ARCH_IIC_CORE_H
 #define __ASM_ARCH_IIC_CORE_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/keypad.h b/arch/arm/plat-samsung/include/plat/keypad.h
index c81ace332a1e..9754b9a29945 100644
--- a/arch/arm/plat-samsung/include/plat/keypad.h
+++ b/arch/arm/plat-samsung/include/plat/keypad.h
@@ -1,13 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Samsung Platform - Keypad platform data definitions
  *
  * Copyright (C) 2010 Samsung Electronics Co.Ltd
  * Author: Joonyoung Shim <jy0922.shim@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
  */
 
 #ifndef __PLAT_SAMSUNG_KEYPAD_H
diff --git a/arch/arm/plat-samsung/include/plat/map-base.h b/arch/arm/plat-samsung/include/plat/map-base.h
index 3ffac4d2e4f0..34b39ded0e2e 100644
--- a/arch/arm/plat-samsung/include/plat/map-base.h
+++ b/arch/arm/plat-samsung/include/plat/map-base.h
@@ -1,15 +1,11 @@
-/* linux/include/asm-arm/plat-s3c/map.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2003, 2007 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C - Memory map definitions (virtual addresses)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_PLAT_MAP_H
 #define __ASM_PLAT_MAP_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/map-s3c.h b/arch/arm/plat-samsung/include/plat/map-s3c.h
index 33104911862e..4244acbf4b65 100644
--- a/arch/arm/plat-samsung/include/plat/map-s3c.h
+++ b/arch/arm/plat-samsung/include/plat/map-s3c.h
@@ -1,14 +1,10 @@
-/* linux/arch/arm/plat-samsung/include/plat/map-s3c.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2008 Simtec Electronics
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C24XX - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_PLAT_MAP_S3C_H
 #define __ASM_PLAT_MAP_S3C_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/map-s5p.h b/arch/arm/plat-samsung/include/plat/map-s5p.h
index 0fe2828f9354..f5769e93544a 100644
--- a/arch/arm/plat-samsung/include/plat/map-s5p.h
+++ b/arch/arm/plat-samsung/include/plat/map-s5p.h
@@ -1,14 +1,10 @@
-/* linux/arch/arm/plat-samsung/include/plat/map-s5p.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2010 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com/
  *
  * S5P - Memory map definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_PLAT_MAP_S5P_H
 #define __ASM_PLAT_MAP_S5P_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/pm-common.h b/arch/arm/plat-samsung/include/plat/pm-common.h
index 8705f9e0e288..1268bae04234 100644
--- a/arch/arm/plat-samsung/include/plat/pm-common.h
+++ b/arch/arm/plat-samsung/include/plat/pm-common.h
@@ -1,14 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2013 Samsung Electronics Co., Ltd.
  *	Tomasz Figa <t.figa@samsung.com>
  * Copyright (c) 2004 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Written by Ben Dooks, <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __PLAT_SAMSUNG_PM_COMMON_H
 #define __PLAT_SAMSUNG_PM_COMMON_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/pm.h b/arch/arm/plat-samsung/include/plat/pm.h
index 9dd562ab0841..2746137f9794 100644
--- a/arch/arm/plat-samsung/include/plat/pm.h
+++ b/arch/arm/plat-samsung/include/plat/pm.h
@@ -1,13 +1,9 @@
-/* arch/arm/plat-samsung/include/plat/pm.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2004 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Written by Ben Dooks, <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /* s3c_pm_init
  *
diff --git a/arch/arm/plat-samsung/include/plat/pwm-core.h b/arch/arm/plat-samsung/include/plat/pwm-core.h
index 5bff1facb672..05e3448642a1 100644
--- a/arch/arm/plat-samsung/include/plat/pwm-core.h
+++ b/arch/arm/plat-samsung/include/plat/pwm-core.h
@@ -1,11 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2013 Tomasz Figa <tomasz.figa@gmail.com>
  *
  * Samsung PWM controller platform data helpers.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __ASM_ARCH_PWM_CORE_H
diff --git a/arch/arm/plat-samsung/include/plat/regs-adc.h b/arch/arm/plat-samsung/include/plat/regs-adc.h
index 70612100120f..58953c7381dd 100644
--- a/arch/arm/plat-samsung/include/plat/regs-adc.h
+++ b/arch/arm/plat-samsung/include/plat/regs-adc.h
@@ -1,13 +1,9 @@
-/* arch/arm/mach-s3c2410/include/mach/regs-adc.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2004 Shannon Holland <holland@loser.net>
  *
- * This program is free software; yosu can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2410 ADC registers
-*/
+ */
 
 #ifndef __ASM_ARCH_REGS_ADC_H
 #define __ASM_ARCH_REGS_ADC_H "regs-adc.h"
diff --git a/arch/arm/plat-samsung/include/plat/regs-irqtype.h b/arch/arm/plat-samsung/include/plat/regs-irqtype.h
index c63cd3fc5ad3..ec5c4c5fdd8f 100644
--- a/arch/arm/plat-samsung/include/plat/regs-irqtype.h
+++ b/arch/arm/plat-samsung/include/plat/regs-irqtype.h
@@ -1,14 +1,10 @@
-/* arch/arm/plat-s3c/include/plat/regs-irqtype.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2008 Simtec Electronics
  *      Ben Dooks <ben@simtec.co.uk>
  *      http://armlinux.simtec.co.uk/
  *
  * S3C - IRQ detection types.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /* values for S3C2410_EXTINT0/1/2 and other cpus in the series, including
diff --git a/arch/arm/plat-samsung/include/plat/regs-spi.h b/arch/arm/plat-samsung/include/plat/regs-spi.h
index 552fe7cfe281..607844311566 100644
--- a/arch/arm/plat-samsung/include/plat/regs-spi.h
+++ b/arch/arm/plat-samsung/include/plat/regs-spi.h
@@ -1,13 +1,9 @@
-/* arch/arm/plat-samsung/include/plat/regs-spi.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2004 Fetron GmbH
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2410 SPI register definition
-*/
+ */
 
 #ifndef __ASM_ARCH_REGS_SPI_H
 #define __ASM_ARCH_REGS_SPI_H
diff --git a/arch/arm/plat-samsung/include/plat/regs-udc.h b/arch/arm/plat-samsung/include/plat/regs-udc.h
index 4003d3dab4e7..d8d2eeaca088 100644
--- a/arch/arm/plat-samsung/include/plat/regs-udc.h
+++ b/arch/arm/plat-samsung/include/plat/regs-udc.h
@@ -1,12 +1,7 @@
-/* arch/arm/plat-samsung/include/plat/regs-udc.h
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * Copyright (C) 2004 Herbert Poetzl <herbert@13thfloor.at>
- *
- * This include file is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
-*/
+ */
 
 #ifndef __ASM_ARCH_REGS_UDC_H
 #define __ASM_ARCH_REGS_UDC_H
diff --git a/arch/arm/plat-samsung/include/plat/samsung-time.h b/arch/arm/plat-samsung/include/plat/samsung-time.h
index 209464adef97..d16eefe9ae78 100644
--- a/arch/arm/plat-samsung/include/plat/samsung-time.h
+++ b/arch/arm/plat-samsung/include/plat/samsung-time.h
@@ -1,14 +1,10 @@
-/* linux/arch/arm/plat-samsung/include/plat/samsung-time.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com/
  *
  * Header file for samsung s3c and s5p time support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __ASM_PLAT_SAMSUNG_TIME_H
 #define __ASM_PLAT_SAMSUNG_TIME_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/sdhci.h b/arch/arm/plat-samsung/include/plat/sdhci.h
index 2787553c3ae2..5731e42ea208 100644
--- a/arch/arm/plat-samsung/include/plat/sdhci.h
+++ b/arch/arm/plat-samsung/include/plat/sdhci.h
@@ -1,5 +1,5 @@
-/* linux/arch/arm/plat-samsung/include/plat/sdhci.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
@@ -9,11 +9,7 @@
  *	Ben Dooks <ben@simtec.co.uk>
  *
  * S3C Platform - SDHCI (HSMMC) platform data definitions
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #ifndef __PLAT_S3C_SDHCI_H
 #define __PLAT_S3C_SDHCI_H __FILE__
diff --git a/arch/arm/plat-samsung/include/plat/usb-phy.h b/arch/arm/plat-samsung/include/plat/usb-phy.h
index ab34dfadb7f9..6d0c788beb9d 100644
--- a/arch/arm/plat-samsung/include/plat/usb-phy.h
+++ b/arch/arm/plat-samsung/include/plat/usb-phy.h
@@ -1,11 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright (C) 2011 Samsung Electronics Co.Ltd
  * Author: Joonyoung Shim <jy0922.shim@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
  */
 
 #ifndef __PLAT_SAMSUNG_USB_PHY_H
diff --git a/arch/arm/plat-samsung/include/plat/wakeup-mask.h b/arch/arm/plat-samsung/include/plat/wakeup-mask.h
index bbfa84b0505a..630909e6630b 100644
--- a/arch/arm/plat-samsung/include/plat/wakeup-mask.h
+++ b/arch/arm/plat-samsung/include/plat/wakeup-mask.h
@@ -1,14 +1,9 @@
-/* arch/arm/plat-samsung/include/plat/wakeup-mask.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2010 Ben Dooks <ben-linux@fluff.org>
  *
  * Support for wakeup mask interrupts on newer SoCs
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
-*/
+ */
 
 #ifndef __PLAT_WAKEUP_MASK_H
 #define __PLAT_WAKEUP_MASK_H __file__
diff --git a/arch/arm/plat-samsung/init.c b/arch/arm/plat-samsung/init.c
index 3776f7e752f0..e9acf02ef3c3 100644
--- a/arch/arm/plat-samsung/init.c
+++ b/arch/arm/plat-samsung/init.c
@@ -1,15 +1,10 @@
-/* linux/arch/arm/plat-s3c/init.c
- *
- * Copyright (c) 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * S3C series CPU initialisation
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// S3C series CPU initialisation
 
 /*
  * NOTE: Code in this file is not used on S3C64xx when booting with
diff --git a/arch/arm/plat-samsung/platformdata.c b/arch/arm/plat-samsung/platformdata.c
index 6cf52ee7eeec..cbc3b4b45c74 100644
--- a/arch/arm/plat-samsung/platformdata.c
+++ b/arch/arm/plat-samsung/platformdata.c
@@ -1,13 +1,8 @@
-/* linux/arch/arm/plat-samsung/platformdata.c
- *
- * Copyright 2010 Ben Dooks <ben-linux <at> fluff.org>
- *
- * Helper for platform data setting
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2010 Ben Dooks <ben-linux <at> fluff.org>
+//
+// Helper for platform data setting
 
 #include <linux/kernel.h>
 #include <linux/slab.h>
diff --git a/arch/arm/plat-samsung/pm-check.c b/arch/arm/plat-samsung/pm-check.c
index d63516374f7c..cd2c02c68bc3 100644
--- a/arch/arm/plat-samsung/pm-check.c
+++ b/arch/arm/plat-samsung/pm-check.c
@@ -1,16 +1,12 @@
-/* linux/arch/arm/plat-s3c/pm-check.c
- *  originally in linux/arch/arm/plat-s3c24xx/pm.c
- *
- * Copyright (c) 2004-2008 Simtec Electronics
- *	http://armlinux.simtec.co.uk
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * S3C Power Mangament - suspend/resume memory corruption check.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// originally in linux/arch/arm/plat-s3c24xx/pm.c
+//
+// Copyright (c) 2004-2008 Simtec Electronics
+//	http://armlinux.simtec.co.uk
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// S3C Power Mangament - suspend/resume memory corruption check.
 
 #include <linux/kernel.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/plat-samsung/pm-common.c b/arch/arm/plat-samsung/pm-common.c
index 6534c3ff9fe2..59a10c6dcba1 100644
--- a/arch/arm/plat-samsung/pm-common.c
+++ b/arch/arm/plat-samsung/pm-common.c
@@ -1,17 +1,13 @@
-/*
- * Copyright (C) 2013 Samsung Electronics Co., Ltd.
- *	Tomasz Figa <t.figa@samsung.com>
- * Copyright (C) 2008 Openmoko, Inc.
- * Copyright (C) 2004-2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * Samsung common power management helper functions.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2013 Samsung Electronics Co., Ltd.
+//	Tomasz Figa <t.figa@samsung.com>
+// Copyright (C) 2008 Openmoko, Inc.
+// Copyright (C) 2004-2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// Samsung common power management helper functions.
 
 #include <linux/io.h>
 #include <linux/kernel.h>
diff --git a/arch/arm/plat-samsung/pm-debug.c b/arch/arm/plat-samsung/pm-debug.c
index 64e15da33b42..b76b1e9ba4ae 100644
--- a/arch/arm/plat-samsung/pm-debug.c
+++ b/arch/arm/plat-samsung/pm-debug.c
@@ -1,17 +1,13 @@
-/*
- * Copyright (C) 2013 Samsung Electronics Co., Ltd.
- *	Tomasz Figa <t.figa@samsung.com>
- * Copyright (C) 2008 Openmoko, Inc.
- * Copyright (C) 2004-2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * Samsung common power management (suspend to RAM) debug support
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2013 Samsung Electronics Co., Ltd.
+//	Tomasz Figa <t.figa@samsung.com>
+// Copyright (C) 2008 Openmoko, Inc.
+// Copyright (C) 2004-2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// Samsung common power management (suspend to RAM) debug support
 
 #include <linux/serial_core.h>
 #include <linux/serial_s3c.h>
diff --git a/arch/arm/plat-samsung/pm-gpio.c b/arch/arm/plat-samsung/pm-gpio.c
index f9a09262f2fa..cb2e3bc79336 100644
--- a/arch/arm/plat-samsung/pm-gpio.c
+++ b/arch/arm/plat-samsung/pm-gpio.c
@@ -1,17 +1,11 @@
-
-/* linux/arch/arm/plat-s3c/pm-gpio.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * S3C series GPIO PM code
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// S3C series GPIO PM code
 
 #include <linux/kernel.h>
 #include <linux/device.h>
diff --git a/arch/arm/plat-samsung/pm.c b/arch/arm/plat-samsung/pm.c
index d7803b434732..d6bfd66592b0 100644
--- a/arch/arm/plat-samsung/pm.c
+++ b/arch/arm/plat-samsung/pm.c
@@ -1,16 +1,11 @@
-/* linux/arch/arm/plat-s3c/pm.c
- *
- * Copyright 2008 Openmoko, Inc.
- * Copyright 2004-2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *	http://armlinux.simtec.co.uk/
- *
- * S3C common power management (suspend to ram) support.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2008 Openmoko, Inc.
+// Copyright 2004-2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//	http://armlinux.simtec.co.uk/
+//
+// S3C common power management (suspend to ram) support.
 
 #include <linux/init.h>
 #include <linux/suspend.h>
diff --git a/arch/arm/plat-samsung/wakeup-mask.c b/arch/arm/plat-samsung/wakeup-mask.c
index b9de6b543330..24f96fb80738 100644
--- a/arch/arm/plat-samsung/wakeup-mask.c
+++ b/arch/arm/plat-samsung/wakeup-mask.c
@@ -1,13 +1,8 @@
-/* arch/arm/plat-samsung/wakeup-mask.c
- *
- * Copyright 2010 Ben Dooks <ben-linux@fluff.org>
- *
- * Support for wakeup mask interrupts on newer SoCs
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2010 Ben Dooks <ben-linux@fluff.org>
+//
+// Support for wakeup mask interrupts on newer SoCs
 
 #include <linux/kernel.h>
 #include <linux/spinlock.h>
diff --git a/arch/arm/plat-samsung/watchdog-reset.c b/arch/arm/plat-samsung/watchdog-reset.c
index 307d8ad96a78..ce42cc640a61 100644
--- a/arch/arm/plat-samsung/watchdog-reset.c
+++ b/arch/arm/plat-samsung/watchdog-reset.c
@@ -1,16 +1,11 @@
-/* arch/arm/plat-samsung/watchdog-reset.c
- *
- * Copyright (c) 2008 Simtec Electronics
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
- *
- * Watchdog reset support for Samsung SoCs.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2008 Simtec Electronics
+//	Ben Dooks <ben@simtec.co.uk>
+//
+// Copyright (c) 2013 Tomasz Figa <tomasz.figa@gmail.com>
+//
+// Watchdog reset support for Samsung SoCs.
 
 #include <linux/clk.h>
 #include <linux/err.h>
diff --git a/arch/arm/probes/kprobes/core.c b/arch/arm/probes/kprobes/core.c
index 52d1cd14fda4..e90cc8a08186 100644
--- a/arch/arm/probes/kprobes/core.c
+++ b/arch/arm/probes/kprobes/core.c
@@ -32,6 +32,7 @@
 #include <linux/percpu.h>
 #include <linux/bug.h>
 #include <asm/patch.h>
+#include <asm/sections.h>
 
 #include "../decode-arm.h"
 #include "../decode-thumb.h"
@@ -64,9 +65,6 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p)
 	int is;
 	const struct decode_checker **checkers;
 
-	if (in_exception_text(addr))
-		return -EINVAL;
-
 #ifdef CONFIG_THUMB2_KERNEL
 	thumb = true;
 	addr &= ~1; /* Bit 0 would normally be set to indicate Thumb code */
@@ -680,3 +678,13 @@ int __init arch_init_kprobes()
 #endif
 	return 0;
 }
+
+bool arch_within_kprobe_blacklist(unsigned long addr)
+{
+	void *a = (void *)addr;
+
+	return __in_irqentry_text(addr) ||
+	       in_entry_text(addr) ||
+	       in_idmap_text(addr) ||
+	       memory_contains(__kprobes_text_start, __kprobes_text_end, a, 1);
+}
diff --git a/arch/arm/tools/mach-types b/arch/arm/tools/mach-types
index a9313b66f770..4eac94c1eb6f 100644
--- a/arch/arm/tools/mach-types
+++ b/arch/arm/tools/mach-types
@@ -345,6 +345,7 @@ mxlads			MACH_MXLADS		MXLADS			1851
 linkstation_mini	MACH_LINKSTATION_MINI	LINKSTATION_MINI	1858
 afeb9260		MACH_AFEB9260		AFEB9260		1859
 imx27ipcam		MACH_IMX27IPCAM		IMX27IPCAM		1871
+bk3			MACH_BK3		BK3			1880
 rd88f6183ap_ge		MACH_RD88F6183AP_GE	RD88F6183AP_GE		1894
 realview_pba8		MACH_REALVIEW_PBA8	REALVIEW_PBA8		1897
 realview_pbx		MACH_REALVIEW_PBX	REALVIEW_PBX		1901
diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c
index a71a48e71fff..4c375e11ae95 100644
--- a/arch/arm/vfp/vfpmodule.c
+++ b/arch/arm/vfp/vfpmodule.c
@@ -257,7 +257,7 @@ static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_
 
 	if (exceptions == VFP_EXCEPTION_ERROR) {
 		vfp_panic("unhandled bounce", inst);
-		vfp_raise_sigfpe(0, regs);
+		vfp_raise_sigfpe(FPE_FIXME, regs);
 		return;
 	}
 
@@ -648,7 +648,7 @@ int vfp_restore_user_hwstate(struct user_vfp __user *ufp,
  */
 static int vfp_dying_cpu(unsigned int cpu)
 {
-	vfp_force_reload(cpu, current_thread_info());
+	vfp_current_hw_state[cpu] = NULL;
 	return 0;
 }
 
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index c9a7e9e1414f..7381eeb7ef8e 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -16,6 +16,7 @@ config ARM64
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_HAS_GIGANTIC_PAGE if (MEMORY_ISOLATION && COMPACTION) || CMA
 	select ARCH_HAS_KCOV
+	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_SET_MEMORY
 	select ARCH_HAS_SG_CHAIN
 	select ARCH_HAS_STRICT_KERNEL_RWX
@@ -59,6 +60,7 @@ config ARM64
 	select COMMON_CLK
 	select CPU_PM if (SUSPEND || CPU_IDLE)
 	select DCACHE_WORD_ACCESS
+	select DMA_DIRECT_OPS
 	select EDAC_SUPPORT
 	select FRAME_POINTER
 	select GENERIC_ALLOCATOR
@@ -90,6 +92,7 @@ config ARM64
 	select HAVE_ARCH_MMAP_RND_BITS
 	select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT
 	select HAVE_ARCH_SECCOMP_FILTER
+	select HAVE_ARCH_THREAD_STRUCT_WHITELIST
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
 	select HAVE_ARCH_VMAP_STACK
@@ -227,7 +230,7 @@ config GENERIC_CSUM
 config GENERIC_CALIBRATE_DELAY
 	def_bool y
 
-config ZONE_DMA
+config ZONE_DMA32
 	def_bool y
 
 config HAVE_GENERIC_GUP
@@ -522,20 +525,13 @@ config CAVIUM_ERRATUM_30115
 config QCOM_FALKOR_ERRATUM_1003
 	bool "Falkor E1003: Incorrect translation due to ASID change"
 	default y
-	select ARM64_PAN if ARM64_SW_TTBR0_PAN
 	help
 	  On Falkor v1, an incorrect ASID may be cached in the TLB when ASID
-	  and BADDR are changed together in TTBRx_EL1. The workaround for this
-	  issue is to use a reserved ASID in cpu_do_switch_mm() before
-	  switching to the new ASID. Saying Y here selects ARM64_PAN if
-	  ARM64_SW_TTBR0_PAN is selected. This is done because implementing and
-	  maintaining the E1003 workaround in the software PAN emulation code
-	  would be an unnecessary complication. The affected Falkor v1 CPU
-	  implements ARMv8.1 hardware PAN support and using hardware PAN
-	  support versus software PAN emulation is mutually exclusive at
-	  runtime.
-
-	  If unsure, say Y.
+	  and BADDR are changed together in TTBRx_EL1. Since we keep the ASID
+	  in TTBR1_EL1, this situation only occurs in the entry trampoline and
+	  then only for entries in the walk cache, since the leaf translation
+	  is unchanged. Work around the erratum by invalidating the walk cache
+	  entries for the trampoline before entering the kernel proper.
 
 config QCOM_FALKOR_ERRATUM_1009
 	bool "Falkor E1009: Prematurely complete a DSB after a TLBI"
@@ -656,6 +652,35 @@ config ARM64_VA_BITS
 	default 47 if ARM64_VA_BITS_47
 	default 48 if ARM64_VA_BITS_48
 
+choice
+	prompt "Physical address space size"
+	default ARM64_PA_BITS_48
+	help
+	  Choose the maximum physical address range that the kernel will
+	  support.
+
+config ARM64_PA_BITS_48
+	bool "48-bit"
+
+config ARM64_PA_BITS_52
+	bool "52-bit (ARMv8.2)"
+	depends on ARM64_64K_PAGES
+	depends on ARM64_PAN || !ARM64_SW_TTBR0_PAN
+	help
+	  Enable support for a 52-bit physical address space, introduced as
+	  part of the ARMv8.2-LPA extension.
+
+	  With this enabled, the kernel will also continue to work on CPUs that
+	  do not support ARMv8.2-LPA, but with some added memory overhead (and
+	  minor performance overhead).
+
+endchoice
+
+config ARM64_PA_BITS
+	int
+	default 48 if ARM64_PA_BITS_48
+	default 52 if ARM64_PA_BITS_52
+
 config CPU_BIG_ENDIAN
        bool "Build big-endian kernel"
        help
@@ -850,6 +875,35 @@ config FORCE_MAX_ZONEORDER
 	  However for 4K, we choose a higher default value, 11 as opposed to 10, giving us
 	  4M allocations matching the default size used by generic code.
 
+config UNMAP_KERNEL_AT_EL0
+	bool "Unmap kernel when running in userspace (aka \"KAISER\")" if EXPERT
+	default y
+	help
+	  Speculation attacks against some high-performance processors can
+	  be used to bypass MMU permission checks and leak kernel data to
+	  userspace. This can be defended against by unmapping the kernel
+	  when running in userspace, mapping it back in on exception entry
+	  via a trampoline page in the vector table.
+
+	  If unsure, say Y.
+
+config HARDEN_BRANCH_PREDICTOR
+	bool "Harden the branch predictor against aliasing attacks" if EXPERT
+	default y
+	help
+	  Speculation attacks against some high-performance processors rely on
+	  being able to manipulate the branch predictor for a victim context by
+	  executing aliasing branches in the attacker context.  Such attacks
+	  can be partially mitigated against by clearing internal branch
+	  predictor state and limiting the prediction logic in some situations.
+
+	  This config option will take CPU-specific actions to harden the
+	  branch predictor against aliasing attacks and may rely on specific
+	  instruction sequences or control bits being set by the system
+	  firmware.
+
+	  If unsure, say Y.
+
 menuconfig ARMV8_DEPRECATED
 	bool "Emulate deprecated/obsolete ARMv8 instructions"
 	depends on COMPAT
@@ -1021,6 +1075,22 @@ config ARM64_PMEM
 	  operations if DC CVAP is not supported (following the behaviour of
 	  DC CVAP itself if the system does not define a point of persistence).
 
+config ARM64_RAS_EXTN
+	bool "Enable support for RAS CPU Extensions"
+	default y
+	help
+	  CPUs that support the Reliability, Availability and Serviceability
+	  (RAS) Extensions, part of ARMv8.2 are able to track faults and
+	  errors, classify them and report them to software.
+
+	  On CPUs with these extensions system software can use additional
+	  barriers to determine if faults are pending and read the
+	  classification from a new set of registers.
+
+	  Selecting this feature will allow the kernel to use these barriers
+	  and access the new registers if the system supports the extension.
+	  Platform RAS features may additionally depend on firmware support.
+
 endmenu
 
 config ARM64_SVE
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index 2401373565ff..fbedbd8f619a 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -105,6 +105,7 @@ config ARCH_MESON
 	select PINCTRL_MESON
 	select COMMON_CLK_AMLOGIC
 	select COMMON_CLK_GXBB
+	select COMMON_CLK_AXG
 	select MESON_IRQ_GPIO
 	help
 	  This enables support for the Amlogic S905 SoCs.
diff --git a/arch/arm64/boot/dts/actions/Makefile b/arch/arm64/boot/dts/actions/Makefile
index cc4661256356..d8b923480f5a 100644
--- a/arch/arm64/boot/dts/actions/Makefile
+++ b/arch/arm64/boot/dts/actions/Makefile
@@ -1 +1,3 @@
+dtb-$(CONFIG_ARCH_ACTIONS) += s700-cubieboard7.dtb
+
 dtb-$(CONFIG_ARCH_ACTIONS) += s900-bubblegum-96.dtb
diff --git a/arch/arm64/boot/dts/actions/s700-cubieboard7.dts b/arch/arm64/boot/dts/actions/s700-cubieboard7.dts
new file mode 100644
index 000000000000..ef79d7905f44
--- /dev/null
+++ b/arch/arm64/boot/dts/actions/s700-cubieboard7.dts
@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2017 Andreas Färber
+ */
+
+/dts-v1/;
+
+#include "s700.dtsi"
+
+/ {
+	compatible = "cubietech,cubieboard7", "actions,s700";
+	model = "CubieBoard7";
+
+	aliases {
+		serial3 = &uart3;
+	};
+
+	chosen {
+		stdout-path = "serial3:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x80000000>;
+	};
+
+	memory@1,e0000000 {
+		device_type = "memory";
+		reg = <0x1 0xe0000000 0x0 0x0>;
+	};
+
+	uart3_clk: uart3-clk {
+		compatible = "fixed-clock";
+		clock-frequency = <921600>;
+		#clock-cells = <0>;
+	};
+};
+
+&timer {
+	clocks = <&hosc>;
+};
+
+&uart3 {
+	status = "okay";
+	clocks = <&uart3_clk>;
+};
diff --git a/arch/arm64/boot/dts/actions/s700.dtsi b/arch/arm64/boot/dts/actions/s700.dtsi
new file mode 100644
index 000000000000..66dd5309f0a2
--- /dev/null
+++ b/arch/arm64/boot/dts/actions/s700.dtsi
@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2017 Andreas Färber
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	compatible = "actions,s700";
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+		};
+
+		cpu1: cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x1>;
+			enable-method = "psci";
+		};
+
+		cpu2: cpu@2 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x2>;
+			enable-method = "psci";
+		};
+
+		cpu3: cpu@3 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53", "arm,armv8";
+			reg = <0x0 0x3>;
+			enable-method = "psci";
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		secmon@1f000000 {
+			reg = <0x0 0x1f000000 0x0 0x1000000>;
+			no-map;
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
+	arm-pmu {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+		             <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+		             <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+		             <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 14
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 11
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>,
+			     <GIC_PPI 10
+			(GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	hosc: hosc {
+		compatible = "fixed-clock";
+		clock-frequency = <24000000>;
+		#clock-cells = <0>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		gic: interrupt-controller@e00f1000 {
+			compatible = "arm,gic-400";
+			reg = <0x0 0xe00f1000 0x0 0x1000>,
+			      <0x0 0xe00f2000 0x0 0x2000>,
+			      <0x0 0xe00f4000 0x0 0x2000>,
+			      <0x0 0xe00f6000 0x0 0x2000>;
+			interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>;
+			interrupt-controller;
+			#interrupt-cells = <3>;
+		};
+
+		uart0: serial@e0120000 {
+			compatible = "actions,s900-uart", "actions,owl-uart";
+			reg = <0x0 0xe0120000 0x0 0x2000>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart1: serial@e0122000 {
+			compatible = "actions,s900-uart", "actions,owl-uart";
+			reg = <0x0 0xe0122000 0x0 0x2000>;
+			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart2: serial@e0124000 {
+			compatible = "actions,s900-uart", "actions,owl-uart";
+			reg = <0x0 0xe0124000 0x0 0x2000>;
+			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart3: serial@e0126000 {
+			compatible = "actions,s900-uart", "actions,owl-uart";
+			reg = <0x0 0xe0126000 0x0 0x2000>;
+			interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart4: serial@e0128000 {
+			compatible = "actions,s900-uart", "actions,owl-uart";
+			reg = <0x0 0xe0128000 0x0 0x2000>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart5: serial@e012a000 {
+			compatible = "actions,s900-uart", "actions,owl-uart";
+			reg = <0x0 0xe012a000 0x0 0x2000>;
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		uart6: serial@e012c000 {
+			compatible = "actions,s900-uart", "actions,owl-uart";
+			reg = <0x0 0xe012c000 0x0 0x2000>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled";
+		};
+
+		sps: power-controller@e01b0100 {
+			compatible = "actions,s700-sps";
+			reg = <0x0 0xe01b0100 0x0 0x100>;
+			#power-domain-cells = <1>;
+		};
+
+		timer: timer@e024c000 {
+			compatible = "actions,s700-timer";
+			reg = <0x0 0xe024c000 0x0 0x4000>;
+			interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "timer1";
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts
index 4a8d3f83a36e..a6975670cd1c 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-bananapi-m64.dts
@@ -60,6 +60,26 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	leds {
+		compatible = "gpio-leds";
+
+		pwr-led {
+			label = "bananapi-m64:red:pwr";
+			gpios = <&pio 3 24 GPIO_ACTIVE_HIGH>; /* PD24 */
+			default-state = "on";
+		};
+
+		green {
+			label = "bananapi-m64:green:user";
+			gpios = <&pio 4 14 GPIO_ACTIVE_HIGH>; /* PE14 */
+		};
+
+		blue {
+			label = "bananapi-m64:blue:user";
+			gpios = <&pio 4 15 GPIO_ACTIVE_HIGH>; /* PE15 */
+		};
+	};
+
 	wifi_pwrseq: wifi_pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		reset-gpios = <&r_pio 0 2 GPIO_ACTIVE_LOW>; /* PL2 */
@@ -153,6 +173,17 @@
 
 #include "axp803.dtsi"
 
+&reg_aldo1 {
+	/*
+	 * This regulator also drives the PE pingroup GPIOs,
+	 * which also controls two LEDs.
+	 */
+	regulator-always-on;
+	regulator-min-microvolt = <2800000>;
+	regulator-max-microvolt = <2800000>;
+	regulator-name = "afvcc-csi";
+};
+
 &reg_aldo2 {
 	regulator-always-on;
 	regulator-min-microvolt = <1800000>;
@@ -168,6 +199,11 @@
 };
 
 &reg_dc1sw {
+	/*
+	 * This regulator also indirectly drives the PD pingroup GPIOs,
+	 * which also controls the power LED.
+	 */
+	regulator-always-on;
 	regulator-name = "vcc-phy";
 };
 
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-olinuxino.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-olinuxino.dts
index 338e786155b1..8807664f363a 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-a64-olinuxino.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-olinuxino.dts
@@ -57,6 +57,11 @@
 	chosen {
 		stdout-path = "serial0:115200n8";
 	};
+
+	wifi_pwrseq: wifi_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&r_pio 0 2 GPIO_ACTIVE_LOW>; /* PL2 */
+	};
 };
 
 &mmc0 {
@@ -70,6 +75,24 @@
 	status = "okay";
 };
 
+&mmc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins>;
+	vmmc-supply = <&reg_aldo2>;
+	vqmmc-supply = <&reg_dldo4>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+
+	rtl8723bs: wifi@1 {
+		reg = <1>;
+		interrupt-parent = <&r_pio>;
+		interrupts = <0 3 IRQ_TYPE_LEVEL_LOW>; /* PL3 */
+		interrupt-names = "host-wake";
+	};
+};
+
 &r_rsb {
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts b/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts
index 5f8ff4017d45..240d35731d10 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-orangepi-win.dts
@@ -57,13 +57,6 @@
 	chosen {
 		stdout-path = "serial0:115200n8";
 	};
-
-	reg_vcc3v3: vcc3v3 {
-		compatible = "regulator-fixed";
-		regulator-name = "vcc3v3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-	};
 };
 
 &ehci1 {
@@ -73,7 +66,7 @@
 &mmc0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc0_pins>;
-	vmmc-supply = <&reg_vcc3v3>;
+	vmmc-supply = <&reg_dcdc1>;
 	cd-gpios = <&pio 5 6 GPIO_ACTIVE_HIGH>;
 	cd-inverted;
 	status = "okay";
@@ -83,6 +76,122 @@
 	status = "okay";
 };
 
+&r_rsb {
+	status = "okay";
+
+	axp803: pmic@3a3 {
+		compatible = "x-powers,axp803";
+		reg = <0x3a3>;
+		interrupt-parent = <&r_intc>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+#include "axp803.dtsi"
+
+&reg_aldo1 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "afvcc-csi";
+};
+
+&reg_aldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-pl";
+};
+
+&reg_aldo3 {
+	regulator-always-on;
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+	regulator-name = "vcc-pll-avcc";
+};
+
+&reg_dcdc1 {
+	regulator-always-on;
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-3v3";
+};
+
+&reg_dcdc2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1040000>;
+	regulator-max-microvolt = <1300000>;
+	regulator-name = "vdd-cpux";
+};
+
+/* DCDC3 is polyphased with DCDC2 */
+
+&reg_dcdc5 {
+	regulator-always-on;
+	regulator-min-microvolt = <1500000>;
+	regulator-max-microvolt = <1500000>;
+	regulator-name = "vcc-dram";
+};
+
+&reg_dcdc6 {
+	regulator-always-on;
+	regulator-min-microvolt = <1100000>;
+	regulator-max-microvolt = <1100000>;
+	regulator-name = "vdd-sys";
+};
+
+&reg_dldo1 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-hdmi-dsi";
+};
+
+&reg_dldo2 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi";
+};
+
+&reg_dldo3 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "avdd-csi";
+};
+
+&reg_dldo4 {
+	regulator-min-microvolt = <3300000>;
+	regulator-max-microvolt = <3300000>;
+	regulator-name = "vcc-wifi-io";
+};
+
+&reg_eldo1 {
+	regulator-min-microvolt = <1800000>;
+	regulator-max-microvolt = <1800000>;
+	regulator-name = "cpvdd";
+};
+
+&reg_fldo1 {
+	regulator-min-microvolt = <1200000>;
+	regulator-max-microvolt = <1200000>;
+	regulator-name = "vcc-1v2-hsic";
+};
+
+/*
+ * The A64 chip cannot work without this regulator off, although
+ * it seems to be only driving the AR100 core.
+ * Maybe we don't still know well about CPUs domain.
+ */
+&reg_fldo2 {
+	regulator-always-on;
+	regulator-min-microvolt = <1100000>;
+	regulator-max-microvolt = <1100000>;
+	regulator-name = "vdd-cpus";
+};
+
+&reg_rtc_ldo {
+	regulator-name = "vcc-rtc";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_pins_a>;
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo-plus2.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo-plus2.dts
index 7c028af58f47..1ed9f219deaf 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo-plus2.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo-plus2.dts
@@ -53,6 +53,7 @@
 	compatible = "friendlyarm,nanopi-neo-plus2", "allwinner,sun50i-h5";
 
 	aliases {
+		ethernet0 = &emac;
 		serial0 = &uart0;
 	};
 
@@ -133,9 +134,25 @@
 	status = "okay";
 };
 
+&emac {
+	pinctrl-names = "default";
+	pinctrl-0 = <&emac_rgmii_pins>;
+	phy-supply = <&reg_gmac_3v3>;
+	phy-handle = <&ext_rgmii_phy>;
+	phy-mode = "rgmii";
+	status = "okay";
+};
+
+&external_mdio {
+	ext_rgmii_phy: ethernet-phy@7 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <7>;
+	};
+};
+
 &mmc0 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin>;
+	pinctrl-0 = <&mmc0_pins_a>;
 	vmmc-supply = <&reg_vcc3v3>;
 	bus-width = <4>;
 	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo2.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo2.dts
index 6eb8092d8e57..f1447003ea3c 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo2.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-nanopi-neo2.dts
@@ -127,7 +127,7 @@
 
 &mmc0 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin>;
+	pinctrl-0 = <&mmc0_pins_a>;
 	vmmc-supply = <&reg_vcc3v3>;
 	bus-width = <4>;
 	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts
index a0ca925175aa..9e51d3a5f4e6 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-pc2.dts
@@ -161,7 +161,7 @@
 
 &mmc0 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin>;
+	pinctrl-0 = <&mmc0_pins_a>;
 	vmmc-supply = <&reg_vcc3v3>;
 	bus-width = <4>;
 	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-prime.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-prime.dts
index b47790650144..0f25c4a6f15d 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-prime.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-prime.dts
@@ -168,7 +168,7 @@
 
 &mmc0 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin>;
+	pinctrl-0 = <&mmc0_pins_a>;
 	vmmc-supply = <&reg_vcc3v3>;
 	bus-width = <4>;
 	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>; /* PF6 */
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus2.dts b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus2.dts
index a42fd79a62a3..af43533c7134 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus2.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5-orangepi-zero-plus2.dts
@@ -64,17 +64,43 @@
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 	};
+
+	wifi_pwrseq: wifi_pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		reset-gpios = <&pio 0 9 GPIO_ACTIVE_LOW>; /* PA9 */
+		post-power-on-delay-ms = <200>;
+	};
 };
 
 &mmc0 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&mmc0_pins_a>, <&mmc0_cd_pin>;
+	pinctrl-0 = <&mmc0_pins_a>;
 	vmmc-supply = <&reg_vcc3v3>;
 	bus-width = <4>;
 	cd-gpios = <&pio 5 6 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
 
+&mmc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mmc1_pins_a>;
+	vmmc-supply = <&reg_vcc3v3>;
+	vqmmc-supply = <&reg_vcc3v3>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	bus-width = <4>;
+	non-removable;
+	status = "okay";
+
+	brcmf: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+		interrupt-parent = <&r_pio>;
+		interrupts = <0 7 IRQ_TYPE_LEVEL_LOW>;	/* PL7 */
+		interrupt-names = "host-wake";
+	};
+};
+
 &mmc2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&mmc2_8bit_pins>;
@@ -90,3 +116,9 @@
 	pinctrl-0 = <&uart0_pins_a>;
 	status = "okay";
 };
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart1_pins>, <&uart1_rts_cts_pins>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
index d9a720bff05d..e237c05cfdb4 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h5.dtsi
@@ -98,6 +98,10 @@
 	compatible = "allwinner,sun50i-h5-ccu";
 };
 
+&display_clocks {
+	compatible = "allwinner,sun50i-h5-de2-clk";
+};
+
 &mmc0 {
 	compatible = "allwinner,sun50i-h5-mmc",
 		     "allwinner,sun50i-a64-mmc";
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
index 9db19314c60c..c89d0c307f8d 100644
--- a/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10.dtsi
@@ -249,7 +249,9 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0xffda4000 0x1000>;
-			interrupts = <0 101 4>;
+			interrupts = <0 99 4>;
+			resets = <&rst SPIM0_RESET>;
+			reg-io-width = <4>;
 			num-chipselect = <4>;
 			bus-num = <0>;
 			status = "disabled";
@@ -260,7 +262,9 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0xffda5000 0x1000>;
-			interrupts = <0 102 4>;
+			interrupts = <0 100 4>;
+			resets = <&rst SPIM1_RESET>;
+			reg-io-width = <4>;
 			num-chipselect = <4>;
 			bus-num = <0>;
 			status = "disabled";
@@ -336,8 +340,8 @@
 			interrupts = <0 93 4>;
 			phys = <&usbphy0>;
 			phy-names = "usb2-phy";
-			resets = <&rst USB0_RESET>;
-			reset-names = "dwc2";
+			resets = <&rst USB0_RESET>, <&rst USB0_OCP_RESET>;
+			reset-names = "dwc2", "dwc2-ecc";
 			status = "disabled";
 		};
 
@@ -347,8 +351,8 @@
 			interrupts = <0 94 4>;
 			phys = <&usbphy0>;
 			phy-names = "usb2-phy";
-			resets = <&rst USB1_RESET>;
-			reset-names = "dwc2";
+			resets = <&rst USB1_RESET>, <&rst USB1_OCP_RESET>;
+			reset-names = "dwc2", "dwc2-ecc";
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
index a37c46112876..000756429b77 100644
--- a/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
+++ b/arch/arm64/boot/dts/altera/socfpga_stratix10_socdk.dts
@@ -97,3 +97,7 @@
 &uart0 {
 	status = "okay";
 };
+
+&usb0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts
index 70eca1f8736a..447b98d30921 100644
--- a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts
@@ -17,6 +17,19 @@
 	};
 };
 
+&ethmac {
+	status = "okay";
+	phy-mode = "rgmii";
+	pinctrl-0 = <&eth_rgmii_y_pins>;
+	pinctrl-names = "default";
+};
+
 &uart_AO {
 	status = "okay";
 };
+
+&ir {
+	status = "okay";
+	pinctrl-0 = <&remote_input_ao_pins>;
+	pinctrl-names = "default";
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi
index b932a784b02a..70c776ef7aa7 100644
--- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi
@@ -7,6 +7,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/axg-clkc.h>
 
 / {
 	compatible = "amlogic,meson-axg";
@@ -113,28 +114,83 @@
 		#size-cells = <2>;
 		ranges;
 
-		cbus: cbus@ffd00000 {
+		cbus: bus@ffd00000 {
 			compatible = "simple-bus";
 			reg = <0x0 0xffd00000 0x0 0x25000>;
 			#address-cells = <2>;
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xffd00000 0x0 0x25000>;
 
+			pwm_ab: pwm@1b000 {
+				compatible = "amlogic,meson-axg-ee-pwm";
+				reg = <0x0 0x1b000 0x0 0x20>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_cd: pwm@1a000 {
+				compatible = "amlogic,meson-axg-ee-pwm";
+				reg = <0x0 0x1a000 0x0 0x20>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			reset: reset-controller@1004 {
+				compatible = "amlogic,meson-axg-reset";
+				reg = <0x0 0x01004 0x0 0x9c>;
+				#reset-cells = <1>;
+			};
+
+			spicc0: spi@13000 {
+				compatible = "amlogic,meson-axg-spicc";
+				reg = <0x0 0x13000 0x0 0x3c>;
+				interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clkc CLKID_SPICC0>;
+				clock-names = "core";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+
+			spicc1: spi@15000 {
+				compatible = "amlogic,meson-axg-spicc";
+				reg = <0x0 0x15000 0x0 0x3c>;
+				interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clkc CLKID_SPICC1>;
+				clock-names = "core";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+
 			uart_A: serial@24000 {
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-uart";
-				reg = <0x0 0x24000 0x0 0x14>;
+				reg = <0x0 0x24000 0x0 0x18>;
 				interrupts = <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>;
 				status = "disabled";
 			};
 
 			uart_B: serial@23000 {
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-uart";
-				reg = <0x0 0x23000 0x0 0x14>;
+				reg = <0x0 0x23000 0x0 0x18>;
 				interrupts = <GIC_SPI 75 IRQ_TYPE_EDGE_RISING>;
 				status = "disabled";
 			};
 		};
 
+		ethmac: ethernet@ff3f0000 {
+			compatible = "amlogic,meson-gxbb-dwmac", "snps,dwmac";
+			reg = <0x0 0xff3f0000 0x0 0x10000
+				0x0 0xff634540 0x0 0x8>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "macirq";
+			clocks = <&clkc CLKID_ETH>,
+				 <&clkc CLKID_FCLK_DIV2>,
+				 <&clkc CLKID_MPLL2>;
+			clock-names = "stmmaceth", "clkin0", "clkin1";
+			status = "disabled";
+		};
+
 		gic: interrupt-controller@ffc01000 {
 			compatible = "arm,gic-400";
 			reg = <0x0 0xffc01000 0 0x1000>,
@@ -148,6 +204,20 @@
 			#address-cells = <0>;
 		};
 
+		hiubus: bus@ff63c000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xff63c000 0x0 0x1c00>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xff63c000 0x0 0x1c00>;
+
+			clkc: clock-controller@0 {
+				compatible = "amlogic,axg-clkc";
+				#clock-cells = <1>;
+				reg = <0x0 0x0 0x0 0x320>;
+			};
+		};
+
 		mailbox: mailbox@ff63dc00 {
 			compatible = "amlogic,meson-gx-mhu", "amlogic,meson-gxbb-mhu";
 			reg = <0 0xff63dc00 0 0x400>;
@@ -157,6 +227,226 @@
 			#mbox-cells = <1>;
 		};
 
+		periphs: periphs@ff634000 {
+			compatible = "simple-bus";
+			reg = <0x0 0xff634000 0x0 0x2000>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x0 0x0 0x0 0xff634000 0x0 0x2000>;
+
+			pinctrl_periphs: pinctrl@480 {
+				compatible = "amlogic,meson-axg-periphs-pinctrl";
+				#address-cells = <2>;
+				#size-cells = <2>;
+				ranges;
+
+				gpio: bank@480 {
+					reg = <0x0 0x00480 0x0 0x40>,
+						<0x0 0x004e8 0x0 0x14>,
+						<0x0 0x00520 0x0 0x14>,
+						<0x0 0x00430 0x0 0x3c>;
+					reg-names = "mux", "pull", "pull-enable", "gpio";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&pinctrl_periphs 0 0 86>;
+				};
+
+				eth_rgmii_x_pins: eth-x-rgmii {
+					mux {
+						groups = "eth_mdio_x",
+						       "eth_mdc_x",
+						       "eth_rgmii_rx_clk_x",
+						       "eth_rx_dv_x",
+						       "eth_rxd0_x",
+						       "eth_rxd1_x",
+						       "eth_rxd2_rgmii",
+						       "eth_rxd3_rgmii",
+						       "eth_rgmii_tx_clk",
+						       "eth_txen_x",
+						       "eth_txd0_x",
+						       "eth_txd1_x",
+						       "eth_txd2_rgmii",
+						       "eth_txd3_rgmii";
+						function = "eth";
+					};
+				};
+
+				eth_rgmii_y_pins: eth-y-rgmii {
+					mux {
+						groups = "eth_mdio_y",
+						       "eth_mdc_y",
+						       "eth_rgmii_rx_clk_y",
+						       "eth_rx_dv_y",
+						       "eth_rxd0_y",
+						       "eth_rxd1_y",
+						       "eth_rxd2_rgmii",
+						       "eth_rxd3_rgmii",
+						       "eth_rgmii_tx_clk",
+						       "eth_txen_y",
+						       "eth_txd0_y",
+						       "eth_txd1_y",
+						       "eth_txd2_rgmii",
+						       "eth_txd3_rgmii";
+						function = "eth";
+					};
+				};
+
+				pwm_a_a_pins: pwm_a_a {
+					mux {
+						groups = "pwm_a_a";
+						function = "pwm_a";
+					};
+				};
+
+				pwm_a_x18_pins: pwm_a_x18 {
+					mux {
+						groups = "pwm_a_x18";
+						function = "pwm_a";
+					};
+				};
+
+				pwm_a_x20_pins: pwm_a_x20 {
+					mux {
+						groups = "pwm_a_x20";
+						function = "pwm_a";
+					};
+				};
+
+				pwm_a_z_pins: pwm_a_z {
+					mux {
+						groups = "pwm_a_z";
+						function = "pwm_a";
+					};
+				};
+
+				pwm_b_a_pins: pwm_b_a {
+					mux {
+						groups = "pwm_b_a";
+						function = "pwm_b";
+					};
+				};
+
+				pwm_b_x_pins: pwm_b_x {
+					mux {
+						groups = "pwm_b_x";
+						function = "pwm_b";
+					};
+				};
+
+				pwm_b_z_pins: pwm_b_z {
+					mux {
+						groups = "pwm_b_z";
+						function = "pwm_b";
+					};
+				};
+
+				pwm_c_a_pins: pwm_c_a {
+					mux {
+						groups = "pwm_c_a";
+						function = "pwm_c";
+					};
+				};
+
+				pwm_c_x10_pins: pwm_c_x10 {
+					mux {
+						groups = "pwm_c_x10";
+						function = "pwm_c";
+					};
+				};
+
+				pwm_c_x17_pins: pwm_c_x17 {
+					mux {
+						groups = "pwm_c_x17";
+						function = "pwm_c";
+					};
+				};
+
+				pwm_d_x11_pins: pwm_d_x11 {
+					mux {
+						groups = "pwm_d_x11";
+						function = "pwm_d";
+					};
+				};
+
+				pwm_d_x16_pins: pwm_d_x16 {
+					mux {
+						groups = "pwm_d_x16";
+						function = "pwm_d";
+					};
+				};
+
+				spi0_pins: spi0 {
+					mux {
+						groups = "spi0_miso",
+							"spi0_mosi",
+							"spi0_clk";
+						function = "spi0";
+					};
+				};
+
+				spi0_ss0_pins: spi0_ss0 {
+					mux {
+						groups = "spi0_ss0";
+						function = "spi0";
+					};
+				};
+
+				spi0_ss1_pins: spi0_ss1 {
+					mux {
+						groups = "spi0_ss1";
+						function = "spi0";
+					};
+				};
+
+				spi0_ss2_pins: spi0_ss2 {
+					mux {
+						groups = "spi0_ss2";
+						function = "spi0";
+					};
+				};
+
+
+				spi1_a_pins: spi1_a {
+					mux {
+						groups = "spi1_miso_a",
+							"spi1_mosi_a",
+							"spi1_clk_a";
+						function = "spi1";
+					};
+				};
+
+				spi1_ss0_a_pins: spi1_ss0_a {
+					mux {
+						groups = "spi1_ss0_a";
+						function = "spi1";
+					};
+				};
+
+				spi1_ss1_pins: spi1_ss1 {
+					mux {
+						groups = "spi1_ss1";
+						function = "spi1";
+					};
+				};
+
+				spi1_x_pins: spi1_x {
+					mux {
+						groups = "spi1_miso_x",
+							"spi1_mosi_x",
+							"spi1_clk_x";
+						function = "spi1";
+					};
+				};
+
+				spi1_ss0_x_pins: spi1_ss0_x {
+					mux {
+						groups = "spi1_ss0_x";
+						function = "spi1";
+					};
+				};
+			};
+		};
+
 		sram: sram@fffc0000 {
 			compatible = "amlogic,meson-axg-sram", "mmio-sram";
 			reg = <0x0 0xfffc0000 0x0 0x20000>;
@@ -175,18 +465,56 @@
 			};
 		};
 
-		aobus: aobus@ff800000 {
+		aobus: bus@ff800000 {
 			compatible = "simple-bus";
 			reg = <0x0 0xff800000 0x0 0x100000>;
 			#address-cells = <2>;
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xff800000 0x0 0x100000>;
 
+			pinctrl_aobus: pinctrl@14 {
+				compatible = "amlogic,meson-axg-aobus-pinctrl";
+				#address-cells = <2>;
+				#size-cells = <2>;
+				ranges;
+
+				gpio_ao: bank@14 {
+					reg = <0x0 0x00014 0x0 0x8>,
+						<0x0 0x0002c 0x0 0x4>,
+						<0x0 0x00024 0x0 0x8>;
+					reg-names = "mux", "pull", "gpio";
+					gpio-controller;
+					#gpio-cells = <2>;
+					gpio-ranges = <&pinctrl_aobus 0 0 15>;
+				};
+
+				remote_input_ao_pins: remote_input_ao {
+					mux {
+						groups = "remote_input_ao";
+						function = "remote_input_ao";
+					};
+				};
+			};
+
+			pwm_AO_ab: pwm@7000 {
+				compatible = "amlogic,meson-axg-ao-pwm";
+				reg = <0x0 0x07000 0x0 0x20>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_AO_cd: pwm@2000 {
+				compatible = "amlogic,axg-ao-pwm";
+				reg = <0x0 0x02000  0x0 0x20>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
 			uart_AO: serial@3000 {
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-ao-uart";
 				reg = <0x0 0x3000 0x0 0x18>;
 				interrupts = <GIC_SPI 193 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>, <&xtal>, <&xtal>;
+				clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
 				clock-names = "xtal", "pclk", "baud";
 				status = "disabled";
 			};
@@ -195,10 +523,17 @@
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-ao-uart";
 				reg = <0x0 0x4000 0x0 0x18>;
 				interrupts = <GIC_SPI 197 IRQ_TYPE_EDGE_RISING>;
-				clocks = <&xtal>, <&xtal>, <&xtal>;
+				clocks = <&xtal>, <&clkc CLKID_CLK81>, <&xtal>;
 				clock-names = "xtal", "pclk", "baud";
 				status = "disabled";
 			};
+
+			ir: ir@8000 {
+				compatible = "amlogic,meson-gxbb-ir";
+				reg = <0x0 0x8000 0x0 0x20>;
+				interrupts = <GIC_SPI 196 IRQ_TYPE_EDGE_RISING>;
+				status = "disabled";
+			};
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi b/arch/arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi
index 7d4b95e49993..aeb6d21a3bec 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gx-p23x-q20x.dtsi
@@ -59,6 +59,18 @@
 		reg = <0x0 0x0 0x0 0x80000000>;
 	};
 
+	hdmi_5v: regulator-hdmi-5v {
+		compatible = "regulator-fixed";
+
+		regulator-name = "HDMI_5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&gpio GPIOH_3 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
 	vddio_boot: regulator-vddio_boot {
 		compatible = "regulator-fixed";
 		regulator-name = "VDDIO_BOOT";
diff --git a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
index ab7ce1644cdc..4ee2e7951482 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gx.dtsi
@@ -211,7 +211,7 @@
 		#size-cells = <2>;
 		ranges;
 
-		cbus: cbus@c1100000 {
+		cbus: bus@c1100000 {
 			compatible = "simple-bus";
 			reg = <0x0 0xc1100000 0x0 0x100000>;
 			#address-cells = <2>;
@@ -229,20 +229,20 @@
 
 			reset: reset-controller@4404 {
 				compatible = "amlogic,meson-gx-reset", "amlogic,meson-gxbb-reset";
-				reg = <0x0 0x04404 0x0 0x20>;
+				reg = <0x0 0x04404 0x0 0x9c>;
 				#reset-cells = <1>;
 			};
 
 			uart_A: serial@84c0 {
 				compatible = "amlogic,meson-gx-uart";
-				reg = <0x0 0x84c0 0x0 0x14>;
+				reg = <0x0 0x84c0 0x0 0x18>;
 				interrupts = <GIC_SPI 26 IRQ_TYPE_EDGE_RISING>;
 				status = "disabled";
 			};
 
 			uart_B: serial@84dc {
 				compatible = "amlogic,meson-gx-uart";
-				reg = <0x0 0x84dc 0x0 0x14>;
+				reg = <0x0 0x84dc 0x0 0x18>;
 				interrupts = <GIC_SPI 75 IRQ_TYPE_EDGE_RISING>;
 				status = "disabled";
 			};
@@ -287,7 +287,7 @@
 
 			uart_C: serial@8700 {
 				compatible = "amlogic,meson-gx-uart";
-				reg = <0x0 0x8700 0x0 0x14>;
+				reg = <0x0 0x8700 0x0 0x18>;
 				interrupts = <GIC_SPI 93 IRQ_TYPE_EDGE_RISING>;
 				status = "disabled";
 			};
@@ -366,7 +366,7 @@
 			};
 		};
 
-		aobus: aobus@c8100000 {
+		aobus: bus@c8100000 {
 			compatible = "simple-bus";
 			reg = <0x0 0xc8100000 0x0 0x100000>;
 			#address-cells = <2>;
@@ -377,6 +377,12 @@
 				compatible = "amlogic,meson-gx-ao-sysctrl", "syscon", "simple-mfd";
 				reg =  <0x0 0x0 0x0 0x100>;
 
+				pwrc_vpu: power-controller-vpu {
+					compatible = "amlogic,meson-gx-pwrc-vpu";
+					#power-domain-cells = <0>;
+					amlogic,hhi-sysctrl = <&sysctrl>;
+				};
+
 				clkc_AO: clock-controller {
 					compatible = "amlogic,meson-gx-aoclkc";
 					#clock-cells = <1>;
@@ -398,14 +404,14 @@
 
 			uart_AO: serial@4c0 {
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-ao-uart";
-				reg = <0x0 0x004c0 0x0 0x14>;
+				reg = <0x0 0x004c0 0x0 0x18>;
 				interrupts = <GIC_SPI 193 IRQ_TYPE_EDGE_RISING>;
 				status = "disabled";
 			};
 
 			uart_AO_B: serial@4e0 {
 				compatible = "amlogic,meson-gx-uart", "amlogic,meson-ao-uart";
-				reg = <0x0 0x004e0 0x0 0x14>;
+				reg = <0x0 0x004e0 0x0 0x18>;
 				interrupts = <GIC_SPI 197 IRQ_TYPE_EDGE_RISING>;
 				status = "disabled";
 			};
@@ -447,13 +453,18 @@
 			};
 		};
 
-		hiubus: hiubus@c883c000 {
+		hiubus: bus@c883c000 {
 			compatible = "simple-bus";
 			reg = <0x0 0xc883c000 0x0 0x2000>;
 			#address-cells = <2>;
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xc883c000 0x0 0x2000>;
 
+			sysctrl: system-controller@0 {
+				compatible = "amlogic,meson-gx-hhi-sysctrl", "syscon", "simple-mfd";
+				reg = <0 0 0 0x400>;
+			};
+
 			mailbox: mailbox@404 {
 				compatible = "amlogic,meson-gx-mhu", "amlogic,meson-gxbb-mhu";
 				reg = <0 0x404 0 0x4c>;
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts b/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts
index 4a4251001bfd..011e8e08e429 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-nanopi-k2.dts
@@ -169,6 +169,7 @@
 			/* Realtek RTL8211F (0x001cc916) */
 			reg = <0>;
 			interrupt-parent = <&gpio_intc>;
+			/* MAC_INTR on GPIOZ_15 */
 			interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
 		};
 	};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts b/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts
index f8d221463c60..ee4ada61c59c 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-odroidc2.dts
@@ -135,6 +135,24 @@
 		compatible = "mmc-pwrseq-emmc";
 		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
 	};
+
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&hdmi_tx_tmds_out>;
+			};
+		};
+	};
+};
+
+&cec_AO {
+	status = "okay";
+	pinctrl-0 = <&ao_cec_pins>;
+	pinctrl-names = "default";
+	hdmi-phandle = <&hdmi_tx>;
 };
 
 &ethmac {
@@ -156,8 +174,10 @@
 		#size-cells = <0>;
 
 		eth_phy0: ethernet-phy@0 {
+			/* Realtek RTL8211F (0x001cc916) */
 			reg = <0>;
 			interrupt-parent = <&gpio_intc>;
+			/* MAC_INTR on GPIOZ_15 */
 			interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
 			eee-broken-1000t;
 		};
@@ -179,6 +199,18 @@
 	};
 };
 
+&hdmi_tx {
+	status = "okay";
+	pinctrl-0 = <&hdmi_hpd_pins>, <&hdmi_i2c_pins>;
+	pinctrl-names = "default";
+};
+
+&hdmi_tx_tmds_port {
+	hdmi_tx_tmds_out: endpoint {
+		remote-endpoint = <&hdmi_connector_in>;
+	};
+};
+
 &i2c_A {
 	status = "okay";
 	pinctrl-0 = <&i2c_a_pins>;
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb-p200.dts b/arch/arm64/boot/dts/amlogic/meson-gxbb-p200.dts
index 9bf16bb7c491..09f34f7ef084 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb-p200.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb-p200.dts
@@ -118,6 +118,7 @@
 			/* Micrel KSZ9031 (0x00221620) */
 			reg = <3>;
 			interrupt-parent = <&gpio_intc>;
+			/* MAC_INTR on GPIOZ_15 */
 			interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
 		};
 	};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
index 1fb8b9d6cb4e..3290a4dc3522 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxbb.dtsi
@@ -694,14 +694,55 @@
 	};
 };
 
+&pwrc_vpu {
+	resets = <&reset RESET_VIU>,
+		 <&reset RESET_VENC>,
+		 <&reset RESET_VCBUS>,
+		 <&reset RESET_BT656>,
+		 <&reset RESET_DVIN_RESET>,
+		 <&reset RESET_RDMA>,
+		 <&reset RESET_VENCI>,
+		 <&reset RESET_VENCP>,
+		 <&reset RESET_VDAC>,
+		 <&reset RESET_VDI6>,
+		 <&reset RESET_VENCL>,
+		 <&reset RESET_VID_LOCK>;
+	clocks = <&clkc CLKID_VPU>,
+	         <&clkc CLKID_VAPB>;
+	clock-names = "vpu", "vapb";
+	/*
+	 * VPU clocking is provided by two identical clock paths
+	 * VPU_0 and VPU_1 muxed to a single clock by a glitch
+	 * free mux to safely change frequency while running.
+	 * Same for VAPB but with a final gate after the glitch free mux.
+	 */
+	assigned-clocks = <&clkc CLKID_VPU_0_SEL>,
+			  <&clkc CLKID_VPU_0>,
+			  <&clkc CLKID_VPU>, /* Glitch free mux */
+			  <&clkc CLKID_VAPB_0_SEL>,
+			  <&clkc CLKID_VAPB_0>,
+			  <&clkc CLKID_VAPB_SEL>; /* Glitch free mux */
+	assigned-clock-parents = <&clkc CLKID_FCLK_DIV3>,
+				 <0>, /* Do Nothing */
+				 <&clkc CLKID_VPU_0>,
+				 <&clkc CLKID_FCLK_DIV4>,
+				 <0>, /* Do Nothing */
+				 <&clkc CLKID_VAPB_0>;
+	assigned-clock-rates = <0>, /* Do Nothing */
+			       <666666666>,
+			       <0>, /* Do Nothing */
+			       <0>, /* Do Nothing */
+			       <250000000>,
+			       <0>; /* Do Nothing */
+};
+
 &saradc {
 	compatible = "amlogic,meson-gxbb-saradc", "amlogic,meson-saradc";
 	clocks = <&xtal>,
 		 <&clkc CLKID_SAR_ADC>,
-		 <&clkc CLKID_SANA>,
 		 <&clkc CLKID_SAR_ADC_CLK>,
 		 <&clkc CLKID_SAR_ADC_SEL>;
-	clock-names = "clkin", "core", "sana", "adc_clk", "adc_sel";
+	clock-names = "clkin", "core", "adc_clk", "adc_sel";
 };
 
 &sd_emmc_a {
@@ -763,4 +804,5 @@
 
 &vpu {
 	compatible = "amlogic,meson-gxbb-vpu", "amlogic,meson-gx-vpu";
+	power-domains = <&pwrc_vpu>;
 };
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts
index dc9c3b8216c2..9671f1e3c74a 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-libretech-cc.dts
@@ -72,6 +72,18 @@
 		reg = <0x0 0x0 0x0 0x80000000>;
 	};
 
+	hdmi_5v: regulator-hdmi-5v {
+		compatible = "regulator-fixed";
+
+		regulator-name = "HDMI_5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&gpio GPIOH_3 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
 	vcc_3v3: regulator-vcc_3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "VCC_3V3";
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-p212.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-p212.dtsi
index ff09df1fd5a3..7005068346a0 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-p212.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905x-p212.dtsi
@@ -28,6 +28,18 @@
 		reg = <0x0 0x0 0x0 0x80000000>;
 	};
 
+	hdmi_5v: regulator-hdmi-5v {
+		compatible = "regulator-fixed";
+
+		regulator-name = "HDMI_5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&gpio GPIOH_3 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
 	vddio_boot: regulator-vddio_boot {
 		compatible = "regulator-fixed";
 		regulator-name = "VDDIO_BOOT";
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
index 6524b89e7115..c8514110b9da 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl.dtsi
@@ -631,6 +631,7 @@
 
 			internal_phy: ethernet-phy@8 {
 				compatible = "ethernet-phy-id0181.4400", "ethernet-phy-ieee802.3-c22";
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
 				reg = <8>;
 				max-speed = <100>;
 			};
@@ -644,14 +645,55 @@
 	};
 };
 
+&pwrc_vpu {
+	resets = <&reset RESET_VIU>,
+		 <&reset RESET_VENC>,
+		 <&reset RESET_VCBUS>,
+		 <&reset RESET_BT656>,
+		 <&reset RESET_DVIN_RESET>,
+		 <&reset RESET_RDMA>,
+		 <&reset RESET_VENCI>,
+		 <&reset RESET_VENCP>,
+		 <&reset RESET_VDAC>,
+		 <&reset RESET_VDI6>,
+		 <&reset RESET_VENCL>,
+		 <&reset RESET_VID_LOCK>;
+	clocks = <&clkc CLKID_VPU>,
+	         <&clkc CLKID_VAPB>;
+	clock-names = "vpu", "vapb";
+	/*
+	 * VPU clocking is provided by two identical clock paths
+	 * VPU_0 and VPU_1 muxed to a single clock by a glitch
+	 * free mux to safely change frequency while running.
+	 * Same for VAPB but with a final gate after the glitch free mux.
+	 */
+	assigned-clocks = <&clkc CLKID_VPU_0_SEL>,
+			  <&clkc CLKID_VPU_0>,
+			  <&clkc CLKID_VPU>, /* Glitch free mux */
+			  <&clkc CLKID_VAPB_0_SEL>,
+			  <&clkc CLKID_VAPB_0>,
+			  <&clkc CLKID_VAPB_SEL>; /* Glitch free mux */
+	assigned-clock-parents = <&clkc CLKID_FCLK_DIV3>,
+				 <0>, /* Do Nothing */
+				 <&clkc CLKID_VPU_0>,
+				 <&clkc CLKID_FCLK_DIV4>,
+				 <0>, /* Do Nothing */
+				 <&clkc CLKID_VAPB_0>;
+	assigned-clock-rates = <0>, /* Do Nothing */
+			       <666666666>,
+			       <0>, /* Do Nothing */
+			       <0>, /* Do Nothing */
+			       <250000000>,
+			       <0>; /* Do Nothing */
+};
+
 &saradc {
 	compatible = "amlogic,meson-gxl-saradc", "amlogic,meson-saradc";
 	clocks = <&xtal>,
 		 <&clkc CLKID_SAR_ADC>,
-		 <&clkc CLKID_SANA>,
 		 <&clkc CLKID_SAR_ADC_CLK>,
 		 <&clkc CLKID_SAR_ADC_SEL>;
-	clock-names = "clkin", "core", "sana", "adc_clk", "adc_sel";
+	clock-names = "clkin", "core", "adc_clk", "adc_sel";
 };
 
 &sd_emmc_a {
@@ -713,4 +755,5 @@
 
 &vpu {
 	compatible = "amlogic,meson-gxl-vpu", "amlogic,meson-gx-vpu";
+	power-domains = <&pwrc_vpu>;
 };
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-khadas-vim2.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-khadas-vim2.dts
index 34a41b26a4ed..1448c3dba08e 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxm-khadas-vim2.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxm-khadas-vim2.dts
@@ -153,6 +153,18 @@
 		};
 	};
 
+	hdmi_5v: regulator-hdmi-5v {
+		compatible = "regulator-fixed";
+
+		regulator-name = "HDMI_5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+
+		gpio = <&gpio GPIOH_3 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
 	vcc_3v3: regulator-vcc_3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "VCC_3V3";
@@ -232,6 +244,9 @@
 	external_phy: ethernet-phy@0 {
 		/* Realtek RTL8211F (0x001cc916) */
 		reg = <0>;
+		interrupt-parent = <&gpio_intc>;
+		/* MAC_INTR on GPIOZ_15 */
+		interrupts = <25 IRQ_TYPE_LEVEL_LOW>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-q200.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-q200.dts
index 66c6da7e112c..388fac4f2d97 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxm-q200.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxm-q200.dts
@@ -111,7 +111,8 @@
 		reg = <0>;
 		max-speed = <1000>;
 		interrupt-parent = <&gpio_intc>;
-		interrupts = <29 IRQ_TYPE_LEVEL_LOW>;
+		/* MAC_INTR on GPIOZ_15 */
+		interrupts = <25 IRQ_TYPE_LEVEL_LOW>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/cavium/thunder2-99xx.dtsi b/arch/arm64/boot/dts/cavium/thunder2-99xx.dtsi
index 4220fbdcb24a..ff5c4c47b22b 100644
--- a/arch/arm64/boot/dts/cavium/thunder2-99xx.dtsi
+++ b/arch/arm64/boot/dts/cavium/thunder2-99xx.dtsi
@@ -98,7 +98,7 @@
 		clock-output-names = "clk125mhz";
 	};
 
-	pci {
+	pcie@30000000 {
 		compatible = "pci-host-ecam-generic";
 		device_type = "pci";
 		#interrupt-cells = <1>;
@@ -118,6 +118,7 @@
 		ranges =
 		  <0x02000000    0 0x40000000    0 0x40000000    0 0x20000000
 		   0x43000000 0x40 0x00000000 0x40 0x00000000 0x20 0x00000000>;
+		bus-range = <0 0xff>;
 		interrupt-map-mask = <0 0 0 7>;
 		interrupt-map =
 		      /* addr  pin  ic   icaddr  icintr */
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi
index ec11343dc528..d77b88af9582 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-bus.dtsi
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5433 SoC Memory interface and AMBA bus device tree source
  *
  * Copyright (c) 2016 Samsung Electronics Co., Ltd.
  * Chanwoo Choi <cw00.choi@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 &soc {
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi
index 50403700274b..9df7c65593a1 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5433 SoC pin-mux and pin-config device tree source
  *
@@ -6,10 +7,6 @@
  *
  * Samsung's Exynos5433 SoC pin-mux and pin-config options are listed as device
  * tree nodes are listed in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/pinctrl/samsung.h>
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
index 297597442c44..a77462da4a36 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG Exynos5433 TM2 board device tree source
  *
@@ -5,10 +6,6 @@
  *
  * Common device tree source file for Samsung's TM2 and TM2E boards
  * which are based on Samsung Exynos5433 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 /dts-v1/;
@@ -741,6 +738,19 @@
 	};
 };
 
+&hsi2c_4 {
+	status = "okay";
+
+	s3fwrn5: nfc@27 {
+		compatible = "samsung,s3fwrn5-i2c";
+		reg = <0x27>;
+		interrupt-parent = <&gpa1>;
+		interrupts = <3 IRQ_TYPE_LEVEL_HIGH>;
+		s3fwrn5,en-gpios = <&gpf1 4 GPIO_ACTIVE_HIGH>;
+		s3fwrn5,fw-gpios = <&gpj0 2 GPIO_ACTIVE_HIGH>;
+	};
+};
+
 &hsi2c_5 {
 	status = "okay";
 
@@ -756,6 +766,7 @@
 
 &hsi2c_7 {
 	status = "okay";
+	clock-frequency = <1000000>;
 
 	sii8620@39 {
 		reg = <0x39>;
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts b/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts
index 23191eb9397c..3d7e0a782243 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG Exynos5433 TM2 board device tree source
  *
@@ -5,10 +6,6 @@
  *
  * Device tree source file for Samsung's TM2 board which is based on
  * Samsung Exynos5433 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "exynos5433-tm2-common.dtsi"
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts b/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts
index b73e1231a86f..1e207ce8b97b 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2e.dts
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG Exynos5433 TM2E board device tree source
  *
@@ -5,10 +6,6 @@
  *
  * Device tree source file for Samsung's TM2E(TM2 EDGE) board which is based on
  * Samsung Exynos5433 SoC.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include "exynos5433-tm2-common.dtsi"
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi
index 9be2978f1b9a..f0803575fd9f 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tmu-g3d-sensor-conf.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos5433 TMU sensor configuration
  *
  * Copyright (c) 2016 Jonghwa Lee <jonghwa3.lee@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/thermal/thermal_exynos.h>
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tmu-sensor-conf.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tmu-sensor-conf.dtsi
index 125fe58d77ce..cccae662228a 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tmu-sensor-conf.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tmu-sensor-conf.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos5433 TMU sensor configuration
  *
  * Copyright (c) 2016 Chanwoo Choi <cw00.choi@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/thermal/thermal_exynos.h>
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi
index ceaa05145b8a..fe3a0b14bee6 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tmu.dtsi
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos5433 thermal zone
  *
  * Copyright (c) 2016 Chanwoo Choi <cw00.choi@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/thermal/thermal.h>
diff --git a/arch/arm64/boot/dts/exynos/exynos5433.dtsi b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
index 7fe994b750da..62f276970174 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos5433 SoC device tree source
  *
@@ -10,10 +11,6 @@
  * Note: This file does not include device nodes for all the controllers in
  * Exynos5433 SoC. As device tree coverage for Exynos5433 increases,
  * additional nodes can be added to this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/clock/exynos5433.h>
@@ -247,6 +244,24 @@
 		#size-cells = <1>;
 		ranges = <0x0 0x0 0x0 0x18000000>;
 
+		arm_a53_pmu {
+			compatible = "arm,cortex-a53-pmu", "arm,armv8-pmuv3";
+			interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-affinity = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>;
+		};
+
+		arm_a57_pmu {
+			compatible = "arm,cortex-a57-pmu", "arm,armv8-pmuv3";
+			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-affinity = <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
+		};
+
 		chipid@10000000 {
 			compatible = "samsung,exynos4210-chipid";
 			reg = <0x10000000 0x100>;
@@ -343,6 +358,7 @@
 			clocks = <&xxti>,
 				<&cmu_top CLK_ACLK_G2D_266>,
 				<&cmu_top CLK_ACLK_G2D_400>;
+			power-domains = <&pd_g2d>;
 		};
 
 		cmu_disp: clock-controller@13b90000 {
@@ -368,6 +384,7 @@
 				<&cmu_mif CLK_SCLK_DECON_ECLK_DISP>,
 				<&cmu_mif CLK_SCLK_DECON_TV_VCLK_DISP>,
 				<&cmu_mif CLK_ACLK_DISP_333>;
+			power-domains = <&pd_disp>;
 		};
 
 		cmu_aud: clock-controller@114c0000 {
@@ -376,6 +393,7 @@
 			#clock-cells = <1>;
 			clock-names = "oscclk", "fout_aud_pll";
 			clocks = <&xxti>, <&cmu_top CLK_FOUT_AUD_PLL>;
+			power-domains = <&pd_aud>;
 		};
 
 		cmu_bus0: clock-controller@13600000 {
@@ -412,6 +430,7 @@
 
 			clock-names = "oscclk", "aclk_g3d_400";
 			clocks = <&xxti>, <&cmu_top CLK_ACLK_G3D_400>;
+			power-domains = <&pd_g3d>;
 		};
 
 		cmu_gscl: clock-controller@13cf0000 {
@@ -425,6 +444,7 @@
 			clocks = <&xxti>,
 				<&cmu_top CLK_ACLK_GSCL_111>,
 				<&cmu_top CLK_ACLK_GSCL_333>;
+			power-domains = <&pd_gscl>;
 		};
 
 		cmu_apollo: clock-controller@11900000 {
@@ -445,7 +465,7 @@
 			clocks = <&xxti>, <&cmu_mif CLK_SCLK_BUS_PLL_ATLAS>;
 		};
 
-		cmu_mscl: clock-controller@105d0000 {
+		cmu_mscl: clock-controller@150d0000 {
 			compatible = "samsung,exynos5433-cmu-mscl";
 			reg = <0x150d0000 0x1000>;
 			#clock-cells = <1>;
@@ -456,6 +476,7 @@
 			clocks = <&xxti>,
 				<&cmu_top CLK_SCLK_JPEG_MSCL>,
 				<&cmu_top CLK_ACLK_MSCL_400>;
+			power-domains = <&pd_mscl>;
 		};
 
 		cmu_mfc: clock-controller@15280000 {
@@ -465,6 +486,7 @@
 
 			clock-names = "oscclk", "aclk_mfc_400";
 			clocks = <&xxti>, <&cmu_top CLK_ACLK_MFC_400>;
+			power-domains = <&pd_mfc>;
 		};
 
 		cmu_hevc: clock-controller@14f80000 {
@@ -474,6 +496,7 @@
 
 			clock-names = "oscclk", "aclk_hevc_400";
 			clocks = <&xxti>, <&cmu_top CLK_ACLK_HEVC_400>;
+			power-domains = <&pd_hevc>;
 		};
 
 		cmu_isp: clock-controller@146d0000 {
@@ -487,6 +510,7 @@
 			clocks = <&xxti>,
 				<&cmu_top CLK_ACLK_ISP_DIS_400>,
 				<&cmu_top CLK_ACLK_ISP_400>;
+			power-domains = <&pd_isp>;
 		};
 
 		cmu_cam0: clock-controller@120d0000 {
@@ -502,6 +526,7 @@
 				<&cmu_top CLK_ACLK_CAM0_333>,
 				<&cmu_top CLK_ACLK_CAM0_400>,
 				<&cmu_top CLK_ACLK_CAM0_552>;
+			power-domains = <&pd_cam0>;
 		};
 
 		cmu_cam1: clock-controller@145d0000 {
@@ -523,6 +548,86 @@
 				<&cmu_top CLK_ACLK_CAM1_333>,
 				<&cmu_top CLK_ACLK_CAM1_400>,
 				<&cmu_top CLK_ACLK_CAM1_552>;
+			power-domains = <&pd_cam1>;
+		};
+
+		pd_gscl: power-domain@105c4000 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4000 0x20>;
+			#power-domain-cells = <0>;
+			label = "GSCL";
+		};
+
+		pd_cam0: power-domain@105c4020 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4020 0x20>;
+			#power-domain-cells = <0>;
+			power-domains = <&pd_cam1>;
+			label = "CAM0";
+		};
+
+		pd_mscl: power-domain@105c4040 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4040 0x20>;
+			#power-domain-cells = <0>;
+			label = "MSCL";
+		};
+
+		pd_g3d: power-domain@105c4060 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4060 0x20>;
+			#power-domain-cells = <0>;
+			label = "G3D";
+		};
+
+		pd_disp: power-domain@105c4080 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4080 0x20>;
+			#power-domain-cells = <0>;
+			label = "DISP";
+		};
+
+		pd_cam1: power-domain@105c40a0 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c40a0 0x20>;
+			#power-domain-cells = <0>;
+			label = "CAM1";
+		};
+
+		pd_aud: power-domain@105c40c0 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c40c0 0x20>;
+			#power-domain-cells = <0>;
+			label = "AUD";
+		};
+
+		pd_g2d: power-domain@105c4120 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4120 0x20>;
+			#power-domain-cells = <0>;
+			label = "G2D";
+		};
+
+		pd_isp: power-domain@105c4140 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4140 0x20>;
+			#power-domain-cells = <0>;
+			power-domains = <&pd_cam0>;
+			label = "ISP";
+		};
+
+		pd_mfc: power-domain@105c4180 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c4180 0x20>;
+			#power-domain-cells = <0>;
+			label = "MFC";
+		};
+
+		pd_hevc: power-domain@105c41c0 {
+			compatible = "samsung,exynos5433-pd";
+			reg = <0x105c41c0 0x20>;
+			#power-domain-cells = <0>;
+			label = "HEVC";
 		};
 
 		tmu_atlas0: tmu@10060000 {
@@ -637,6 +742,7 @@
 			compatible = "samsung,exynos5433-pinctrl";
 			reg = <0x114b0000 0x1000>;
 			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&pd_aud>;
 		};
 
 		pinctrl_cpif: pinctrl@10fe0000 {
@@ -728,6 +834,7 @@
 			clock-names = "pclk", "aclk_decon", "aclk_smmu_decon0x",
 				"aclk_xiu_decon0x", "pclk_smmu_decon0x",
 				"sclk_decon_vclk", "sclk_decon_eclk";
+			power-domains = <&pd_disp>;
 			interrupt-names = "fifo", "vsync", "lcd_sys";
 			interrupts = <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
@@ -765,6 +872,7 @@
 				      "aclk_xiu_decon0x", "pclk_smmu_decon0x",
 				      "sclk_decon_vclk", "sclk_decon_eclk";
 			samsung,disp-sysreg = <&syscon_disp>;
+			power-domains = <&pd_disp>;
 			interrupt-names = "fifo", "vsync", "lcd_sys";
 			interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>,
@@ -790,6 +898,7 @@
 					"phyclk_mipidphy0_rxclkesc0",
 					"sclk_rgb_vclk_to_dsim0",
 					"sclk_mipi";
+			power-domains = <&pd_disp>;
 			status = "disabled";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -813,6 +922,7 @@
 			clocks = <&cmu_disp CLK_PCLK_MIC0>,
 				<&cmu_disp CLK_SCLK_RGB_VCLK_TO_MIC0>;
 			clock-names = "pclk_mic0", "sclk_rgb_vclk_to_mic0";
+			power-domains = <&pd_disp>;
 			samsung,disp-syscon = <&syscon_disp>;
 			status = "disabled";
 
@@ -881,7 +991,7 @@
 			reg = <0x145f0000 0x1038>;
 		};
 
-		gsc_0: video-scaler@13C00000 {
+		gsc_0: video-scaler@13c00000 {
 			compatible = "samsung,exynos5433-gsc";
 			reg = <0x13c00000 0x1000>;
 			interrupts = <GIC_SPI 297 IRQ_TYPE_LEVEL_HIGH>;
@@ -892,9 +1002,10 @@
 				 <&cmu_gscl CLK_ACLK_XIU_GSCLX>,
 				 <&cmu_gscl CLK_ACLK_GSCLBEND_333>;
 			iommus = <&sysmmu_gscl0>;
+			power-domains = <&pd_gscl>;
 		};
 
-		gsc_1: video-scaler@13C10000 {
+		gsc_1: video-scaler@13c10000 {
 			compatible = "samsung,exynos5433-gsc";
 			reg = <0x13c10000 0x1000>;
 			interrupts = <GIC_SPI 298 IRQ_TYPE_LEVEL_HIGH>;
@@ -905,9 +1016,10 @@
 				 <&cmu_gscl CLK_ACLK_XIU_GSCLX>,
 				 <&cmu_gscl CLK_ACLK_GSCLBEND_333>;
 			iommus = <&sysmmu_gscl1>;
+			power-domains = <&pd_gscl>;
 		};
 
-		gsc_2: video-scaler@13C20000 {
+		gsc_2: video-scaler@13c20000 {
 			compatible = "samsung,exynos5433-gsc";
 			reg = <0x13c20000 0x1000>;
 			interrupts = <GIC_SPI 299 IRQ_TYPE_LEVEL_HIGH>;
@@ -918,6 +1030,7 @@
 				 <&cmu_gscl CLK_ACLK_XIU_GSCLX>,
 				 <&cmu_gscl CLK_ACLK_GSCLBEND_333>;
 			iommus = <&sysmmu_gscl2>;
+			power-domains = <&pd_gscl>;
 		};
 
 		jpeg: codec@15020000 {
@@ -930,9 +1043,10 @@
 				 <&cmu_mscl CLK_ACLK_XIU_MSCLX>,
 				 <&cmu_mscl CLK_SCLK_JPEG>;
 			iommus = <&sysmmu_jpeg>;
+			power-domains = <&pd_mscl>;
 		};
 
-		mfc: codec@152E0000 {
+		mfc: codec@152e0000 {
 			compatible = "samsung,exynos5433-mfc";
 			reg = <0x152E0000 0x10000>;
 			interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
@@ -942,6 +1056,7 @@
 				 <&cmu_mfc CLK_ACLK_XIU_MFCX>;
 			iommus = <&sysmmu_mfc_0>, <&sysmmu_mfc_1>;
 			iommu-names = "left", "right";
+			power-domains = <&pd_mfc>;
 		};
 
 		sysmmu_decon0x: sysmmu@13a00000 {
@@ -951,6 +1066,7 @@
 			clock-names = "pclk", "aclk";
 			clocks = <&cmu_disp CLK_PCLK_SMMU_DECON0X>,
 				<&cmu_disp CLK_ACLK_SMMU_DECON0X>;
+			power-domains = <&pd_disp>;
 			#iommu-cells = <0>;
 		};
 
@@ -962,6 +1078,7 @@
 			clocks = <&cmu_disp CLK_PCLK_SMMU_DECON1X>,
 				<&cmu_disp CLK_ACLK_SMMU_DECON1X>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_disp>;
 		};
 
 		sysmmu_tv0x: sysmmu@13a20000 {
@@ -972,6 +1089,7 @@
 			clocks = <&cmu_disp CLK_PCLK_SMMU_TV0X>,
 				<&cmu_disp CLK_ACLK_SMMU_TV0X>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_disp>;
 		};
 
 		sysmmu_tv1x: sysmmu@13a30000 {
@@ -982,6 +1100,7 @@
 			clocks = <&cmu_disp CLK_PCLK_SMMU_TV1X>,
 				<&cmu_disp CLK_ACLK_SMMU_TV1X>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_disp>;
 		};
 
 		sysmmu_gscl0: sysmmu@13c80000 {
@@ -992,6 +1111,7 @@
 			clocks = <&cmu_gscl CLK_ACLK_SMMU_GSCL0>,
 				 <&cmu_gscl CLK_PCLK_SMMU_GSCL0>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_gscl>;
 		};
 
 		sysmmu_gscl1: sysmmu@13c90000 {
@@ -1002,6 +1122,7 @@
 			clocks = <&cmu_gscl CLK_ACLK_SMMU_GSCL1>,
 				 <&cmu_gscl CLK_PCLK_SMMU_GSCL1>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_gscl>;
 		};
 
 		sysmmu_gscl2: sysmmu@13ca0000 {
@@ -1012,6 +1133,7 @@
 			clocks = <&cmu_gscl CLK_ACLK_SMMU_GSCL2>,
 				 <&cmu_gscl CLK_PCLK_SMMU_GSCL2>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_gscl>;
 		};
 
 		sysmmu_jpeg: sysmmu@15060000 {
@@ -1022,6 +1144,7 @@
 			clocks = <&cmu_mscl CLK_PCLK_SMMU_JPEG>,
 				 <&cmu_mscl CLK_ACLK_SMMU_JPEG>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_mscl>;
 		};
 
 		sysmmu_mfc_0: sysmmu@15200000 {
@@ -1032,6 +1155,7 @@
 			clocks = <&cmu_mfc CLK_PCLK_SMMU_MFC_0>,
 				 <&cmu_mfc CLK_ACLK_SMMU_MFC_0>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_mfc>;
 		};
 
 		sysmmu_mfc_1: sysmmu@15210000 {
@@ -1042,6 +1166,7 @@
 			clocks = <&cmu_mfc CLK_PCLK_SMMU_MFC_1>,
 				 <&cmu_mfc CLK_ACLK_SMMU_MFC_1>;
 			#iommu-cells = <0>;
+			power-domains = <&pd_mfc>;
 		};
 
 		serial_0: serial@14c10000 {
@@ -1497,6 +1622,7 @@
 			clocks = <&cmu_aud CLK_PCLK_SFR0_CTRL>;
 			clock-names = "sfr0_ctrl";
 			samsung,pmu-syscon = <&pmu_system_controller>;
+			power-domains = <&pd_aud>;
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges;
@@ -1510,6 +1636,7 @@
 				#dma-cells = <1>;
 				#dma-channels = <8>;
 				#dma-requests = <32>;
+				power-domains = <&pd_aud>;
 			};
 
 			i2s0: i2s0@11440000 {
@@ -1526,6 +1653,7 @@
 				clock-names = "iis", "i2s_opclk0", "i2s_opclk1";
 				pinctrl-names = "default";
 				pinctrl-0 = <&i2s0_bus>;
+				power-domains = <&pd_aud>;
 				status = "disabled";
 			};
 
@@ -1538,6 +1666,7 @@
 				clock-names = "uart", "clk_uart_baud0";
 				pinctrl-names = "default";
 				pinctrl-0 = <&uart_aud_bus>;
+				power-domains = <&pd_aud>;
 				status = "disabled";
 			};
 		};
diff --git a/arch/arm64/boot/dts/exynos/exynos7-espresso.dts b/arch/arm64/boot/dts/exynos/exynos7-espresso.dts
index 4a8b1fb51243..22723527e626 100644
--- a/arch/arm64/boot/dts/exynos/exynos7-espresso.dts
+++ b/arch/arm64/boot/dts/exynos/exynos7-espresso.dts
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG Exynos7 Espresso board device tree source
  *
  * Copyright (c) 2014 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 /dts-v1/;
 #include "exynos7.dtsi"
diff --git a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
index 8f58850cd28c..472dd649aa7e 100644
--- a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Samsung's Exynos7 SoC pin-mux and pin-config device tree source
  *
@@ -6,11 +7,7 @@
  *
  * Samsung's Exynos7 SoC pin-mux and pin-config options are listed as
  * device tree nodes in this file.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <dt-bindings/pinctrl/samsung.h>
 
diff --git a/arch/arm64/boot/dts/exynos/exynos7-tmu-sensor-conf.dtsi b/arch/arm64/boot/dts/exynos/exynos7-tmu-sensor-conf.dtsi
index 1d6dcf2aadba..48494710b7b2 100644
--- a/arch/arm64/boot/dts/exynos/exynos7-tmu-sensor-conf.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7-tmu-sensor-conf.dtsi
@@ -1,13 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for Exynos7 TMU sensor configuration
  *
  * Copyright (c) 2016 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 #include <dt-bindings/thermal/thermal_exynos.h>
diff --git a/arch/arm64/boot/dts/exynos/exynos7-trip-points.dtsi b/arch/arm64/boot/dts/exynos/exynos7-trip-points.dtsi
index 062358355a53..d3301b8bd364 100644
--- a/arch/arm64/boot/dts/exynos/exynos7-trip-points.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7-trip-points.dtsi
@@ -1,13 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Device tree sources for default Exynos7 thermal zone definition
  *
  * Copyright (c) 2016 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 trips {
diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi
index 9a3fbed1765a..ad9dce6894ce 100644
--- a/arch/arm64/boot/dts/exynos/exynos7.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * SAMSUNG EXYNOS7 SoC device tree source
  *
  * Copyright (c) 2014 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <dt-bindings/clock/exynos7-clk.h>
@@ -103,7 +100,7 @@
 			#size-cells = <1>;
 			ranges;
 
-			pdma0: pdma@10E10000 {
+			pdma0: pdma@10e10000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x10E10000 0x1000>;
 				interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
@@ -114,7 +111,7 @@
 				#dma-requests = <32>;
 			};
 
-			pdma1: pdma@10EB0000 {
+			pdma1: pdma@10eb0000 {
 				compatible = "arm,pl330", "arm,primecell";
 				reg = <0x10EB0000 0x1000>;
 				interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1088a-rdb.dts
index 0f6fcda36b9e..4f17601b919c 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1088a-rdb.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a-rdb.dts
@@ -125,3 +125,11 @@
 &sata {
 	status = "okay";
 };
+
+&usb0 {
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
index bd80e9a2e67c..4fc150cd4ca5 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi
@@ -166,6 +166,77 @@
 			     <1 10 IRQ_TYPE_LEVEL_LOW>;/* Hypervisor PPI */
 	};
 
+	fsl_mc: fsl-mc@80c000000 {
+		compatible = "fsl,qoriq-mc";
+		reg = <0x00000008 0x0c000000 0 0x40>,	 /* MC portal base */
+		      <0x00000000 0x08340000 0 0x40000>; /* MC control reg */
+		msi-parent = <&its>;
+		#address-cells = <3>;
+		#size-cells = <1>;
+
+		/*
+		 * Region type 0x0 - MC portals
+		 * Region type 0x1 - QBMAN portals
+		 */
+		ranges = <0x0 0x0 0x0 0x8 0x0c000000 0x4000000
+			  0x1 0x0 0x0 0x8 0x18000000 0x8000000>;
+
+		dpmacs {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dpmac1: dpmac@1 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <1>;
+			};
+
+			dpmac2: dpmac@2 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <2>;
+			};
+
+			dpmac3: dpmac@3 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <3>;
+			};
+
+			dpmac4: dpmac@4 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <4>;
+			};
+
+			dpmac5: dpmac@5 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <5>;
+			};
+
+			dpmac6: dpmac@6 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <6>;
+			};
+
+			dpmac7: dpmac@7 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <7>;
+			};
+
+			dpmac8: dpmac@8 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <8>;
+			};
+
+			dpmac9: dpmac@9 {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <9>;
+			};
+
+			dpmac10: dpmac@a {
+				compatible = "fsl,qoriq-mc-dpmac";
+				reg = <0xa>;
+			};
+		};
+	};
+
 	psci {
 		compatible = "arm,psci-0.2";
 		method = "smc";
@@ -191,6 +262,12 @@
 			clocks = <&sysclk>;
 		};
 
+		dcfg: dcfg@1e00000 {
+			compatible = "fsl,ls1088a-dcfg", "syscon";
+			reg = <0x0 0x1e00000 0x0 0x10000>;
+			little-endian;
+		};
+
 		tmu: tmu@1f80000 {
 			compatible = "fsl,qoriq-tmu";
 			reg = <0x0 0x1f80000 0x0 0x10000>;
@@ -394,6 +471,26 @@
 			status = "disabled";
 		};
 
+		usb0: usb3@3100000 {
+			compatible = "snps,dwc3";
+			reg = <0x0 0x3100000 0x0 0x10000>;
+			interrupts = <0 80 IRQ_TYPE_LEVEL_HIGH>;
+			dr_mode = "host";
+			snps,quirk-frame-length-adjustment = <0x20>;
+			snps,dis_rxdet_inp3_quirk;
+			status = "disabled";
+		};
+
+		usb1: usb3@3110000 {
+			compatible = "snps,dwc3";
+			reg = <0x0 0x3110000 0x0 0x10000>;
+			interrupts = <0 81 IRQ_TYPE_LEVEL_HIGH>;
+			dr_mode = "host";
+			snps,quirk-frame-length-adjustment = <0x20>;
+			snps,dis_rxdet_inp3_quirk;
+			status = "disabled";
+		};
+
 		sata: sata@3200000 {
 			compatible = "fsl,ls1088a-ahci";
 			reg = <0x0 0x3200000 0x0 0x10000>,
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
index 9a1d0d2ab1c3..5498c705ae6a 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
@@ -95,6 +95,18 @@
 			};
 		};
 
+		i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x02>;
+
+			ina220@40 {
+				compatible = "ti,ina220";
+				reg = <0x40>;
+				shunt-resistor = <500>;
+			};
+		};
+
 		i2c@3 {
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts b/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts
index 97d768730952..c6999624ed8a 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts
+++ b/arch/arm64/boot/dts/hisilicon/hi3660-hikey960.dts
@@ -109,6 +109,7 @@
 			label = "user_led4";
 			/* gpio_190_user_led4 */
 			gpios = <&gpio23 6 0>;
+			panic-indicator;
 			linux,default-trigger = "cpu0";
 		};
 
diff --git a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
index ab0b95ba5ae5..63d4f9dca77f 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hi3660.dtsi
@@ -61,6 +61,7 @@
 			enable-method = "psci";
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <592>;
 		};
 
 		cpu1: cpu@1 {
@@ -70,6 +71,7 @@
 			enable-method = "psci";
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <592>;
 		};
 
 		cpu2: cpu@2 {
@@ -79,6 +81,7 @@
 			enable-method = "psci";
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <592>;
 		};
 
 		cpu3: cpu@3 {
@@ -88,6 +91,7 @@
 			enable-method = "psci";
 			next-level-cache = <&A53_L2>;
 			cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP_0>;
+			capacity-dmips-mhz = <592>;
 		};
 
 		cpu4: cpu@100 {
@@ -101,6 +105,7 @@
 					&CPU_SLEEP
 					&CLUSTER_SLEEP_1
 			>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		cpu5: cpu@101 {
@@ -114,6 +119,7 @@
 					&CPU_SLEEP
 					&CLUSTER_SLEEP_1
 			>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		cpu6: cpu@102 {
@@ -127,6 +133,7 @@
 					&CPU_SLEEP
 					&CLUSTER_SLEEP_1
 			>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		cpu7: cpu@103 {
@@ -140,6 +147,7 @@
 					&CPU_SLEEP
 					&CLUSTER_SLEEP_1
 			>;
+			capacity-dmips-mhz = <1024>;
 		};
 
 		idle-states {
@@ -203,21 +211,25 @@
 					 IRQ_TYPE_LEVEL_HIGH)>;
 	};
 
-	pmu {
-		compatible = "arm,armv8-pmuv3";
+	a53-pmu {
+		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+			     <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
 		interrupt-affinity = <&cpu0>,
 				     <&cpu1>,
 				     <&cpu2>,
-				     <&cpu3>,
-				     <&cpu4>,
+				     <&cpu3>;
+	};
+
+	a73-pmu {
+		compatible = "arm,cortex-a73-pmu";
+		interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&cpu4>,
 				     <&cpu5>,
 				     <&cpu6>,
 				     <&cpu7>;
diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts
index a6fd13389f8d..4d5d644abb12 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts
+++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200-poplar.dts
@@ -146,6 +146,12 @@
 	status = "okay";
 };
 
+&sd0 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	status = "okay";
+};
+
 &spi0 {
 	status = "okay";
 	label = "LS-SPI0";
diff --git a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
index 75865f8a862a..962bd79139e4 100644
--- a/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
+++ b/arch/arm64/boot/dts/hisilicon/hi3798cv200.dtsi
@@ -192,6 +192,18 @@
 			status = "disabled";
 		};
 
+		sd0: mmc@9820000 {
+			compatible = "snps,dw-mshc";
+			reg = <0x9820000 0x10000>;
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&crg HISTB_SDIO0_CIU_CLK>,
+				 <&crg HISTB_SDIO0_BIU_CLK>;
+			clock-names = "ciu", "biu";
+			resets = <&crg 0x9c 4>;
+			reset-names = "reset";
+			status = "disabled";
+		};
+
 		emmc: mmc@9830000 {
 			compatible = "snps,dw-mshc";
 			reg = <0x9830000 0x10000>;
diff --git a/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts b/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts
index 3aee6123d161..047641fe294c 100644
--- a/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts
+++ b/arch/arm64/boot/dts/hisilicon/hi6220-hikey.dts
@@ -51,7 +51,7 @@
 		#size-cells = <2>;
 		ranges;
 
-		ramoops@0x21f00000 {
+		ramoops@21f00000 {
 			compatible = "ramoops";
 			reg = <0x0 0x21f00000 0x0 0x00100000>;
 			record-size	= <0x00020000>;
@@ -359,6 +359,7 @@
 		user_led1 {
 			label = "user_led1";
 			gpios = <&gpio4 3 0>; /* <&gpio_user_led_4>; */
+			panic-indicator;
 			linux,default-trigger = "cpu0";
 		};
 
diff --git a/arch/arm64/boot/dts/marvell/armada-372x.dtsi b/arch/arm64/boot/dts/marvell/armada-372x.dtsi
index 59d7557d3b1b..2554e0baea6b 100644
--- a/arch/arm64/boot/dts/marvell/armada-372x.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-372x.dtsi
@@ -56,6 +56,7 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a53","arm,armv8";
 			reg = <0x1>;
+			clocks = <&nb_periph_clk 16>;
 			enable-method = "psci";
 		};
 	};
diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
index 90c26d616a54..375026867342 100644
--- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi
@@ -65,6 +65,7 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0>;
+			clocks = <&nb_periph_clk 16>;
 			enable-method = "psci";
 		};
 	};
@@ -234,6 +235,12 @@
 				};
 			};
 
+			nb_pm: syscon@14000 {
+				compatible = "marvell,armada-3700-nb-pm",
+					     "syscon";
+				reg = <0x14000 0x60>;
+			};
+
 			pinctrl_sb: pinctrl@18800 {
 				compatible = "marvell,armada3710-sb-pinctrl",
 					     "syscon", "simple-mfd";
@@ -309,6 +316,20 @@
 				};
 			};
 
+			crypto: crypto@90000 {
+				compatible = "inside-secure,safexcel-eip97";
+				reg = <0x90000 0x20000>;
+				interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>,
+					     <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "mem", "ring0", "ring1",
+						  "ring2", "ring3", "eip";
+				clocks = <&nb_periph_clk 15>;
+			};
+
 			sdhci1: sdhci@d0000 {
 				compatible = "marvell,armada-3700-sdhci",
 					     "marvell,sdhci-xenon";
diff --git a/arch/arm64/boot/dts/marvell/armada-7040-db.dts b/arch/arm64/boot/dts/marvell/armada-7040-db.dts
index 52b5341cb270..3ae05eee2c9a 100644
--- a/arch/arm64/boot/dts/marvell/armada-7040-db.dts
+++ b/arch/arm64/boot/dts/marvell/armada-7040-db.dts
@@ -61,7 +61,13 @@
 		reg = <0x0 0x0 0x0 0x80000000>;
 	};
 
-	cpm_reg_usb3_0_vbus: cpm-usb3-0-vbus {
+	aliases {
+		ethernet0 = &cp0_eth0;
+		ethernet1 = &cp0_eth1;
+		ethernet2 = &cp0_eth2;
+	};
+
+	cp0_reg_usb3_0_vbus: cp0-usb3-0-vbus {
 		compatible = "regulator-fixed";
 		regulator-name = "usb3h0-vbus";
 		regulator-min-microvolt = <5000000>;
@@ -70,7 +76,7 @@
 		gpio = <&expander0 0 GPIO_ACTIVE_HIGH>;
 	};
 
-	cpm_reg_usb3_1_vbus: cpm-usb3-1-vbus {
+	cp0_reg_usb3_1_vbus: cp0-usb3-1-vbus {
 		compatible = "regulator-fixed";
 		regulator-name = "usb3h1-vbus";
 		regulator-min-microvolt = <5000000>;
@@ -79,14 +85,14 @@
 		gpio = <&expander0 1 GPIO_ACTIVE_HIGH>;
 	};
 
-	cpm_usb3_0_phy: cpm-usb3-0-phy {
+	cp0_usb3_0_phy: cp0-usb3-0-phy {
 		compatible = "usb-nop-xceiv";
-		vcc-supply = <&cpm_reg_usb3_0_vbus>;
+		vcc-supply = <&cp0_reg_usb3_0_vbus>;
 	};
 
-	cpm_usb3_1_phy: cpm-usb3-1-phy {
+	cp0_usb3_1_phy: cp0-usb3-1-phy {
 		compatible = "usb-nop-xceiv";
-		vcc-supply = <&cpm_reg_usb3_1_vbus>;
+		vcc-supply = <&cp0_reg_usb3_1_vbus>;
 	};
 };
 
@@ -129,11 +135,11 @@
 };
 
 
-&cpm_pcie2 {
+&cp0_pcie2 {
 	status = "okay";
 };
 
-&cpm_i2c0 {
+&cp0_i2c0 {
 	status = "okay";
 	clock-frequency = <100000>;
 
@@ -156,7 +162,7 @@
 	};
 };
 
-&cpm_nand {
+&cp0_nand {
 	/*
 	 * SPI on CPM and NAND have common pins on this board. We can
 	 * use only one at a time. To enable the NAND (whihch will
@@ -186,7 +192,7 @@
 };
 
 
-&cpm_spi1 {
+&cp0_spi1 {
 	status = "okay";
 
 	spi-flash@0 {
@@ -214,17 +220,17 @@
 	};
 };
 
-&cpm_sata0 {
+&cp0_sata0 {
 	status = "okay";
 };
 
-&cpm_usb3_0 {
-	usb-phy = <&cpm_usb3_0_phy>;
+&cp0_usb3_0 {
+	usb-phy = <&cp0_usb3_0_phy>;
 	status = "okay";
 };
 
-&cpm_usb3_1 {
-	usb-phy = <&cpm_usb3_1_phy>;
+&cp0_usb3_1 {
+	usb-phy = <&cp0_usb3_1_phy>;
 	status = "okay";
 };
 
@@ -235,14 +241,14 @@
 	non-removable;
 };
 
-&cpm_sdhci0 {
+&cp0_sdhci0 {
 	status = "okay";
 	bus-width = <4>;
 	no-1-8-v;
 	cd-gpios = <&expander0 12 GPIO_ACTIVE_LOW>;
 };
 
-&cpm_mdio {
+&cp0_mdio {
 	status = "okay";
 
 	phy0: ethernet-phy@0 {
@@ -253,28 +259,28 @@
 	};
 };
 
-&cpm_ethernet {
+&cp0_ethernet {
 	status = "okay";
 };
 
-&cpm_eth0 {
+&cp0_eth0 {
 	status = "okay";
 	/* Network PHY */
 	phy-mode = "10gbase-kr";
 	/* Generic PHY, providing serdes lanes */
-	phys = <&cpm_comphy2 0>;
+	phys = <&cp0_comphy2 0>;
 };
 
-&cpm_eth1 {
+&cp0_eth1 {
 	status = "okay";
 	/* Network PHY */
 	phy = <&phy0>;
 	phy-mode = "sgmii";
 	/* Generic PHY, providing serdes lanes */
-	phys = <&cpm_comphy0 1>;
+	phys = <&cp0_comphy0 1>;
 };
 
-&cpm_eth2 {
+&cp0_eth2 {
 	status = "okay";
 	phy = <&phy1>;
 	phy-mode = "rgmii-id";
diff --git a/arch/arm64/boot/dts/marvell/armada-70x0.dtsi b/arch/arm64/boot/dts/marvell/armada-70x0.dtsi
index 0e1a1e5be399..f63b4fbd642b 100644
--- a/arch/arm64/boot/dts/marvell/armada-70x0.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-70x0.dtsi
@@ -44,25 +44,46 @@
  * Device Tree file for the Armada 70x0 SoC
  */
 
-#include "armada-cp110-master.dtsi"
-
 / {
 	aliases {
-		gpio1 = &cpm_gpio1;
-		gpio2 = &cpm_gpio2;
+		gpio1 = &cp0_gpio1;
+		gpio2 = &cp0_gpio2;
+		spi1 = &cp0_spi0;
+		spi2 = &cp0_spi1;
 	};
 };
 
-&cpm_gpio1 {
+/*
+ * Instantiate the CP110
+ */
+#define CP110_NAME		cp0
+#define CP110_BASE		f2000000
+#define CP110_PCIE_IO_BASE	0xf9000000
+#define CP110_PCIE_MEM_BASE	0xf6000000
+#define CP110_PCIE0_BASE	f2600000
+#define CP110_PCIE1_BASE	f2620000
+#define CP110_PCIE2_BASE	f2640000
+
+#include "armada-cp110.dtsi"
+
+#undef CP110_NAME
+#undef CP110_BASE
+#undef CP110_PCIE_IO_BASE
+#undef CP110_PCIE_MEM_BASE
+#undef CP110_PCIE0_BASE
+#undef CP110_PCIE1_BASE
+#undef CP110_PCIE2_BASE
+
+&cp0_gpio1 {
 	status = "okay";
 };
 
-&cpm_gpio2 {
+&cp0_gpio2 {
 	status = "okay";
 };
 
-&cpm_syscon0 {
-	cpm_pinctrl: pinctrl {
+&cp0_syscon0 {
+	cp0_pinctrl: pinctrl {
 		compatible = "marvell,armada-7k-pinctrl";
 
 		nand_pins: nand-pins {
diff --git a/arch/arm64/boot/dts/marvell/armada-8020.dtsi b/arch/arm64/boot/dts/marvell/armada-8020.dtsi
index 0ba0bc942598..3318d6b0214b 100644
--- a/arch/arm64/boot/dts/marvell/armada-8020.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-8020.dtsi
@@ -60,6 +60,6 @@
  * oscillator so this one is let enabled.
  */
 
-&cpm_rtc {
+&cp0_rtc {
 	status = "disabled";
 };
diff --git a/arch/arm64/boot/dts/marvell/armada-8040-db.dts b/arch/arm64/boot/dts/marvell/armada-8040-db.dts
index d97b72bed662..dba55baff20f 100644
--- a/arch/arm64/boot/dts/marvell/armada-8040-db.dts
+++ b/arch/arm64/boot/dts/marvell/armada-8040-db.dts
@@ -61,46 +61,53 @@
 		reg = <0x0 0x0 0x0 0x80000000>;
 	};
 
-	cpm_reg_usb3_0_vbus: cpm-usb3-0-vbus {
+	aliases {
+		ethernet0 = &cp0_eth0;
+		ethernet1 = &cp0_eth2;
+		ethernet2 = &cp1_eth0;
+		ethernet3 = &cp1_eth1;
+	};
+
+	cp0_reg_usb3_0_vbus: cp0-usb3-0-vbus {
 		compatible = "regulator-fixed";
-		regulator-name = "cpm-usb3h0-vbus";
+		regulator-name = "cp0-usb3h0-vbus";
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
 		enable-active-high;
 		gpio = <&expander0 0 GPIO_ACTIVE_HIGH>;
 	};
 
-	cpm_reg_usb3_1_vbus: cpm-usb3-1-vbus {
+	cp0_reg_usb3_1_vbus: cp0-usb3-1-vbus {
 		compatible = "regulator-fixed";
-		regulator-name = "cpm-usb3h1-vbus";
+		regulator-name = "cp0-usb3h1-vbus";
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
 		enable-active-high;
 		gpio = <&expander0 1 GPIO_ACTIVE_HIGH>;
 	};
 
-	cpm_usb3_0_phy: cpm-usb3-0-phy {
+	cp0_usb3_0_phy: cp0-usb3-0-phy {
 		compatible = "usb-nop-xceiv";
-		vcc-supply = <&cpm_reg_usb3_0_vbus>;
+		vcc-supply = <&cp0_reg_usb3_0_vbus>;
 	};
 
-	cpm_usb3_1_phy: cpm-usb3-1-phy {
+	cp0_usb3_1_phy: cp0-usb3-1-phy {
 		compatible = "usb-nop-xceiv";
-		vcc-supply = <&cpm_reg_usb3_1_vbus>;
+		vcc-supply = <&cp0_reg_usb3_1_vbus>;
 	};
 
-	cps_reg_usb3_0_vbus: cps-usb3-0-vbus {
+	cp1_reg_usb3_0_vbus: cp1-usb3-0-vbus {
 		compatible = "regulator-fixed";
-		regulator-name = "cps-usb3h0-vbus";
+		regulator-name = "cp1-usb3h0-vbus";
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
 		enable-active-high;
 		gpio = <&expander1 0 GPIO_ACTIVE_HIGH>;
 	};
 
-	cps_usb3_0_phy: cps-usb3-0-phy {
+	cp1_usb3_0_phy: cp1-usb3-0-phy {
 		compatible = "usb-nop-xceiv";
-		vcc-supply = <&cps_reg_usb3_0_vbus>;
+		vcc-supply = <&cp1_reg_usb3_0_vbus>;
 	};
 };
 
@@ -144,16 +151,16 @@
 };
 
 /* CON6 on CP0 expansion */
-&cpm_pcie0 {
+&cp0_pcie0 {
 	status = "okay";
 };
 
 /* CON5 on CP0 expansion */
-&cpm_pcie2 {
+&cp0_pcie2 {
 	status = "okay";
 };
 
-&cpm_i2c0 {
+&cp0_i2c0 {
 	status = "okay";
 	clock-frequency = <100000>;
 
@@ -178,23 +185,23 @@
 };
 
 /* CON4 on CP0 expansion */
-&cpm_sata0 {
+&cp0_sata0 {
 	status = "okay";
 };
 
 /* CON9 on CP0 expansion */
-&cpm_usb3_0 {
-	usb-phy = <&cpm_usb3_0_phy>;
+&cp0_usb3_0 {
+	usb-phy = <&cp0_usb3_0_phy>;
 	status = "okay";
 };
 
 /* CON10 on CP0 expansion */
-&cpm_usb3_1 {
-	usb-phy = <&cpm_usb3_1_phy>;
+&cp0_usb3_1 {
+	usb-phy = <&cp0_usb3_1_phy>;
 	status = "okay";
 };
 
-&cpm_mdio {
+&cp0_mdio {
 	status = "okay";
 
 	phy1: ethernet-phy@1 {
@@ -202,42 +209,42 @@
 	};
 };
 
-&cpm_ethernet {
+&cp0_ethernet {
 	status = "okay";
 };
 
-&cpm_eth0 {
+&cp0_eth0 {
 	status = "okay";
 	phy-mode = "10gbase-kr";
 };
 
-&cpm_eth2 {
+&cp0_eth2 {
 	status = "okay";
 	phy = <&phy1>;
 	phy-mode = "rgmii-id";
 };
 
 /* CON6 on CP1 expansion */
-&cps_pcie0 {
+&cp1_pcie0 {
 	status = "okay";
 };
 
 /* CON7 on CP1 expansion */
-&cps_pcie1 {
+&cp1_pcie1 {
 	status = "okay";
 };
 
 /* CON5 on CP1 expansion */
-&cps_pcie2 {
+&cp1_pcie2 {
 	status = "okay";
 };
 
-&cps_i2c0 {
+&cp1_i2c0 {
 	status = "okay";
 	clock-frequency = <100000>;
 };
 
-&cps_spi1 {
+&cp1_spi1 {
 	status = "okay";
 
 	spi-flash@0 {
@@ -268,23 +275,51 @@
 	};
 };
 
+/*
+ * Proper NAND usage will require DPR-76 to be in position 1-2, which disables
+ * MDIO signal of CP1.
+ */
+&cp1_nand {
+	num-cs = <1>;
+	pinctrl-0 = <&nand_pins>, <&nand_rb>;
+	pinctrl-names = "default";
+	nand-ecc-strength = <4>;
+	nand-ecc-step-size = <512>;
+	marvell,nand-enable-arbiter;
+	marvell,system-controller = <&cp1_syscon0>;
+	nand-on-flash-bbt;
+
+	partition@0 {
+		label = "U-Boot";
+		reg = <0 0x200000>;
+	};
+	partition@200000 {
+		label = "Linux";
+		reg = <0x200000 0xe00000>;
+	};
+	partition@1000000 {
+		label = "Filesystem";
+		reg = <0x1000000 0x3f000000>;
+	};
+};
+
 /* CON4 on CP1 expansion */
-&cps_sata0 {
+&cp1_sata0 {
 	status = "okay";
 };
 
 /* CON9 on CP1 expansion */
-&cps_usb3_0 {
-	usb-phy = <&cps_usb3_0_phy>;
+&cp1_usb3_0 {
+	usb-phy = <&cp1_usb3_0_phy>;
 	status = "okay";
 };
 
 /* CON10 on CP1 expansion */
-&cps_usb3_1 {
+&cp1_usb3_1 {
 	status = "okay";
 };
 
-&cps_mdio {
+&cp1_mdio {
 	status = "okay";
 
 	phy0: ethernet-phy@0 {
@@ -292,16 +327,16 @@
 	};
 };
 
-&cps_ethernet {
+&cp1_ethernet {
 	status = "okay";
 };
 
-&cps_eth0 {
+&cp1_eth0 {
 	status = "okay";
 	phy-mode = "10gbase-kr";
 };
 
-&cps_eth1 {
+&cp1_eth1 {
 	status = "okay";
 	phy = <&phy0>;
 	phy-mode = "rgmii-id";
@@ -313,7 +348,7 @@
 	non-removable;
 };
 
-&cpm_sdhci0 {
+&cp0_sdhci0 {
 	status = "okay";
 	bus-width = <8>;
 	non-removable;
diff --git a/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts b/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts
index b3350827ee55..626e9d0462c3 100644
--- a/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts
+++ b/arch/arm64/boot/dts/marvell/armada-8040-mcbin.dts
@@ -62,6 +62,12 @@
 		reg = <0x0 0x0 0x0 0x80000000>;
 	};
 
+	aliases {
+		ethernet0 = &cp0_eth0;
+		ethernet1 = &cp1_eth0;
+		ethernet2 = &cp1_eth1;
+	};
+
 	/* Regulator labels correspond with schematics */
 	v_3_3: regulator-3-3v {
 		compatible = "regulator-fixed";
@@ -84,9 +90,9 @@
 	v_5v0_usb3_hst_vbus: regulator-usb3-vbus0 {
 		compatible = "regulator-fixed";
 		enable-active-high;
-		gpio = <&cpm_gpio2 15 GPIO_ACTIVE_HIGH>;
+		gpio = <&cp0_gpio2 15 GPIO_ACTIVE_HIGH>;
 		pinctrl-names = "default";
-		pinctrl-0 = <&cpm_xhci_vbus_pins>;
+		pinctrl-0 = <&cp0_xhci_vbus_pins>;
 		regulator-name = "v_5v0_usb3_hst_vbus";
 		regulator-min-microvolt = <5000000>;
 		regulator-max-microvolt = <5000000>;
@@ -120,17 +126,17 @@
 	vqmmc-supply = <&v_vddo_h>;
 };
 
-&cpm_i2c0 {
+&cp0_i2c0 {
 	clock-frequency = <100000>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&cpm_i2c0_pins>;
+	pinctrl-0 = <&cp0_i2c0_pins>;
 	status = "okay";
 };
 
-&cpm_i2c1 {
+&cp0_i2c1 {
 	clock-frequency = <100000>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&cpm_i2c1_pins>;
+	pinctrl-0 = <&cp0_i2c1_pins>;
 	status = "okay";
 
 	i2c-switch@70 {
@@ -157,9 +163,9 @@
 	};
 };
 
-&cpm_mdio {
+&cp0_mdio {
 	pinctrl-names = "default";
-	pinctrl-0 = <&cpm_ge_mdio_pins>;
+	pinctrl-0 = <&cp0_ge_mdio_pins>;
 	status = "okay";
 
 	ge_phy: ethernet-phy@0 {
@@ -167,44 +173,44 @@
 	};
 };
 
-&cpm_pcie0 {
+&cp0_pcie0 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&cpm_pcie_pins>;
+	pinctrl-0 = <&cp0_pcie_pins>;
 	num-lanes = <4>;
 	num-viewport = <8>;
-	reset-gpio = <&cpm_gpio1 20 GPIO_ACTIVE_LOW>;
+	reset-gpio = <&cp0_gpio1 20 GPIO_ACTIVE_LOW>;
 	status = "okay";
 };
 
-&cpm_pinctrl {
-	cpm_ge_mdio_pins: ge-mdio-pins {
+&cp0_pinctrl {
+	cp0_ge_mdio_pins: ge-mdio-pins {
 		marvell,pins = "mpp32", "mpp34";
 		marvell,function = "ge";
 	};
-	cpm_i2c1_pins: i2c1-pins {
+	cp0_i2c1_pins: i2c1-pins {
 		marvell,pins = "mpp35", "mpp36";
 		marvell,function = "i2c1";
 	};
-	cpm_i2c0_pins: i2c0-pins {
+	cp0_i2c0_pins: i2c0-pins {
 		marvell,pins = "mpp37", "mpp38";
 		marvell,function = "i2c0";
 	};
-	cpm_xhci_vbus_pins: xhci0-vbus-pins {
+	cp0_xhci_vbus_pins: xhci0-vbus-pins {
 		marvell,pins = "mpp47";
 		marvell,function = "gpio";
 	};
-	cpm_pcie_pins: pcie-pins {
+	cp0_pcie_pins: pcie-pins {
 		marvell,pins = "mpp52";
 		marvell,function = "gpio";
 	};
-	cpm_sdhci_pins: sdhci-pins {
+	cp0_sdhci_pins: sdhci-pins {
 		marvell,pins = "mpp55", "mpp56", "mpp57", "mpp58", "mpp59",
 			       "mpp60", "mpp61";
 		marvell,function = "sdio";
 	};
 };
 
-&cpm_xmdio {
+&cp0_xmdio {
 	status = "okay";
 
 	phy0: ethernet-phy@0 {
@@ -218,83 +224,83 @@
 	};
 };
 
-&cpm_ethernet {
+&cp0_ethernet {
 	status = "okay";
 };
 
-&cpm_eth0 {
+&cp0_eth0 {
 	status = "okay";
 	/* Network PHY */
 	phy = <&phy0>;
 	phy-mode = "10gbase-kr";
 	/* Generic PHY, providing serdes lanes */
-	phys = <&cpm_comphy4 0>;
+	phys = <&cp0_comphy4 0>;
 };
 
-&cpm_sata0 {
+&cp0_sata0 {
 	/* CPM Lane 0 - U29 */
 	status = "okay";
 };
 
-&cpm_sdhci0 {
+&cp0_sdhci0 {
 	/* U6 */
 	broken-cd;
 	bus-width = <4>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&cpm_sdhci_pins>;
+	pinctrl-0 = <&cp0_sdhci_pins>;
 	status = "okay";
 	vqmmc-supply = <&v_3_3>;
 };
 
-&cpm_usb3_0 {
+&cp0_usb3_0 {
 	/* J38? - USB2.0 only */
 	status = "okay";
 };
 
-&cpm_usb3_1 {
+&cp0_usb3_1 {
 	/* J38? - USB2.0 only */
 	status = "okay";
 };
 
-&cps_ethernet {
+&cp1_ethernet {
 	status = "okay";
 };
 
-&cps_eth0 {
+&cp1_eth0 {
 	status = "okay";
 	/* Network PHY */
 	phy = <&phy8>;
 	phy-mode = "10gbase-kr";
 	/* Generic PHY, providing serdes lanes */
-	phys = <&cps_comphy4 0>;
+	phys = <&cp1_comphy4 0>;
 };
 
-&cps_eth1 {
+&cp1_eth1 {
 	/* CPS Lane 0 - J5 (Gigabit RJ45) */
 	status = "okay";
 	/* Network PHY */
 	phy = <&ge_phy>;
 	phy-mode = "sgmii";
 	/* Generic PHY, providing serdes lanes */
-	phys = <&cps_comphy0 1>;
+	phys = <&cp1_comphy0 1>;
 };
 
-&cps_pinctrl {
-	cps_spi1_pins: spi1-pins {
+&cp1_pinctrl {
+	cp1_spi1_pins: spi1-pins {
 		marvell,pins = "mpp12", "mpp13", "mpp14", "mpp15", "mpp16";
 		marvell,function = "spi1";
 	};
 };
 
-&cps_sata0 {
+&cp1_sata0 {
 	/* CPS Lane 1 - U32 */
 	/* CPS Lane 3 - U31 */
 	status = "okay";
 };
 
-&cps_spi1 {
+&cp1_spi1 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&cps_spi1_pins>;
+	pinctrl-0 = <&cp1_spi1_pins>;
 	status = "okay";
 
 	spi-flash@0 {
@@ -304,7 +310,7 @@
 	};
 };
 
-&cps_usb3_0 {
+&cp1_usb3_0 {
 	/* CPS Lane 2 - CON7 */
 	usb-phy = <&usb3h0_phy>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/marvell/armada-8040.dtsi b/arch/arm64/boot/dts/marvell/armada-8040.dtsi
index 60fe84f5cbcc..83d2b40e5981 100644
--- a/arch/arm64/boot/dts/marvell/armada-8040.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-8040.dtsi
@@ -59,6 +59,6 @@
  * disable it. However, the RTC clock in CP slave is connected to the
  * oscillator so this one is let enabled.
  */
-&cpm_rtc {
+&cp0_rtc {
 	status = "disabled";
 };
diff --git a/arch/arm64/boot/dts/marvell/armada-80x0.dtsi b/arch/arm64/boot/dts/marvell/armada-80x0.dtsi
index 666ebe96ba0d..e9c84a1d3c4d 100644
--- a/arch/arm64/boot/dts/marvell/armada-80x0.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-80x0.dtsi
@@ -44,33 +44,104 @@
  * Device Tree file for the Armada 80x0 SoC family
  */
 
-#include "armada-cp110-master.dtsi"
-#include "armada-cp110-slave.dtsi"
-
 / {
 	aliases {
-		gpio1 = &cps_gpio1;
-		gpio2 = &cpm_gpio2;
+		gpio1 = &cp1_gpio1;
+		gpio2 = &cp0_gpio2;
+		spi1 = &cp0_spi0;
+		spi2 = &cp0_spi1;
+		spi3 = &cp1_spi0;
+		spi4 = &cp1_spi1;
 	};
 };
 
+/*
+ * Instantiate the master CP110
+ */
+#define CP110_NAME		cp0
+#define CP110_BASE		f2000000
+#define CP110_PCIE_IO_BASE	0xf9000000
+#define CP110_PCIE_MEM_BASE	0xf6000000
+#define CP110_PCIE0_BASE	f2600000
+#define CP110_PCIE1_BASE	f2620000
+#define CP110_PCIE2_BASE	f2640000
+
+#include "armada-cp110.dtsi"
+
+#undef CP110_NAME
+#undef CP110_BASE
+#undef CP110_PCIE_IO_BASE
+#undef CP110_PCIE_MEM_BASE
+#undef CP110_PCIE0_BASE
+#undef CP110_PCIE1_BASE
+#undef CP110_PCIE2_BASE
+
+/*
+ * Instantiate the slave CP110
+ */
+#define CP110_NAME		cp1
+#define CP110_BASE		f4000000
+#define CP110_PCIE_IO_BASE	0xfd000000
+#define CP110_PCIE_MEM_BASE	0xfa000000
+#define CP110_PCIE0_BASE	f4600000
+#define CP110_PCIE1_BASE	f4620000
+#define CP110_PCIE2_BASE	f4640000
+
+#include "armada-cp110.dtsi"
+
+#undef CP110_NAME
+#undef CP110_BASE
+#undef CP110_PCIE_IO_BASE
+#undef CP110_PCIE_MEM_BASE
+#undef CP110_PCIE0_BASE
+#undef CP110_PCIE1_BASE
+#undef CP110_PCIE2_BASE
+
 /* The 80x0 has two CP blocks, but uses only one block from each. */
-&cps_gpio1 {
+&cp1_gpio1 {
 	status = "okay";
 };
 
-&cpm_gpio2 {
+&cp0_gpio2 {
 	status = "okay";
 };
 
-&cpm_syscon0 {
-	cpm_pinctrl: pinctrl {
+&cp0_syscon0 {
+	cp0_pinctrl: pinctrl {
 		compatible = "marvell,armada-8k-cpm-pinctrl";
 	};
 };
 
-&cps_syscon0 {
-	cps_pinctrl: pinctrl {
+&cp1_syscon0 {
+	cp1_pinctrl: pinctrl {
 		compatible = "marvell,armada-8k-cps-pinctrl";
+
+		nand_pins: nand-pins {
+			marvell,pins =
+			"mpp0", "mpp1", "mpp2", "mpp3",
+			"mpp4", "mpp5", "mpp6", "mpp7",
+			"mpp8", "mpp9", "mpp10", "mpp11",
+			"mpp15", "mpp16", "mpp17", "mpp18",
+			"mpp19", "mpp20", "mpp21", "mpp22",
+			"mpp23", "mpp24", "mpp25", "mpp26",
+			"mpp27";
+			marvell,function = "dev";
+		};
+
+		nand_rb: nand-rb {
+			marvell,pins = "mpp13", "mpp12";
+			marvell,function = "nf";
+		};
 	};
 };
+
+&cp1_crypto {
+	/*
+	 * The cryptographic engine found on the cp110
+	 * master is enabled by default at the SoC
+	 * level. Because it is not possible as of now
+	 * to enable two cryptographic engines in
+	 * parallel, disable this one by default.
+	 */
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/marvell/armada-ap806.dtsi b/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
index 1c4dd8ab9ad5..f9b66b81f9fc 100644
--- a/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
+++ b/arch/arm64/boot/dts/marvell/armada-ap806.dtsi
@@ -58,6 +58,7 @@
 		serial0 = &uart0;
 		serial1 = &uart1;
 		gpio0 = &ap_gpio;
+		spi0 = &spi0;
 	};
 
 	psci {
@@ -203,7 +204,6 @@
 				reg = <0x510600 0x50>;
 				#address-cells = <1>;
 				#size-cells = <0>;
-				cell-index = <0>;
 				interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&ap_clk 3>;
 				status = "disabled";
@@ -241,7 +241,7 @@
 
 			};
 
-			watchdog: watchdog@600000 {
+			watchdog: watchdog@610000 {
 				compatible = "arm,sbsa-gwdt";
 				reg = <0x610000 0x1000>, <0x600000 0x1000>;
 				interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
@@ -285,6 +285,12 @@
 					gpio-ranges = <&ap_pinctrl 0 0 20>;
 				};
 			};
+
+			ap_thermal: thermal@6f808c {
+				compatible = "marvell,armada-ap806-thermal";
+				reg = <0x6f808c 0x4>,
+				      <0x6f8084 0x8>;
+			};
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/marvell/armada-common.dtsi b/arch/arm64/boot/dts/marvell/armada-common.dtsi
new file mode 100644
index 000000000000..c6dd1d81c68d
--- /dev/null
+++ b/arch/arm64/boot/dts/marvell/armada-common.dtsi
@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR X11)
+/*
+ * Copyright (C) 2016 Marvell Technology Group Ltd.
+ */
+
+/* Common definitions used by Armada 7K/8K DTs */
+#define PASTER(x, y) x ## y
+#define EVALUATOR(x, y) PASTER(x, y)
+#define CP110_LABEL(name) EVALUATOR(CP110_NAME, EVALUATOR(_, name))
+#define ADDRESSIFY(addr) EVALUATOR(0x, addr)
diff --git a/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi b/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi
deleted file mode 100644
index 9c7724e82aff..000000000000
--- a/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi
+++ /dev/null
@@ -1,446 +0,0 @@
-/*
- * Copyright (C) 2016 Marvell Technology Group Ltd.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPLv2 or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/*
- * Device Tree file for Marvell Armada CP110 Master.
- */
-
-#define ICU_GRP_NSR 0x0
-
-/ {
-	cp110-master {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		compatible = "simple-bus";
-		interrupt-parent = <&cpm_icu>;
-		ranges;
-
-		config-space@f2000000 {
-			#address-cells = <1>;
-			#size-cells = <1>;
-			compatible = "simple-bus";
-			ranges = <0x0 0x0 0xf2000000 0x2000000>;
-
-			cpm_ethernet: ethernet@0 {
-				compatible = "marvell,armada-7k-pp22";
-				reg = <0x0 0x100000>, <0x129000 0xb000>;
-				clocks = <&cpm_clk 1 3>, <&cpm_clk 1 9>,
-					 <&cpm_clk 1 5>, <&cpm_clk 1 18>;
-				clock-names = "pp_clk", "gop_clk",
-					      "mg_clk","axi_clk";
-				marvell,system-controller = <&cpm_syscon0>;
-				status = "disabled";
-				dma-coherent;
-
-				cpm_eth0: eth0 {
-					interrupts = <ICU_GRP_NSR 39 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 43 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 47 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 51 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 55 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 129 IRQ_TYPE_LEVEL_HIGH>;
-					interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
-							  "tx-cpu3", "rx-shared", "link";
-					port-id = <0>;
-					gop-port-id = <0>;
-					status = "disabled";
-				};
-
-				cpm_eth1: eth1 {
-					interrupts = <ICU_GRP_NSR 40 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 44 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 48 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 52 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 56 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 128 IRQ_TYPE_LEVEL_HIGH>;
-					interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
-							  "tx-cpu3", "rx-shared", "link";
-					port-id = <1>;
-					gop-port-id = <2>;
-					status = "disabled";
-				};
-
-				cpm_eth2: eth2 {
-					interrupts = <ICU_GRP_NSR 41 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 45 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 49 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 53 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 57 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 127 IRQ_TYPE_LEVEL_HIGH>;
-					interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
-							  "tx-cpu3", "rx-shared", "link";
-					port-id = <2>;
-					gop-port-id = <3>;
-					status = "disabled";
-				};
-			};
-
-			cpm_comphy: phy@120000 {
-				compatible = "marvell,comphy-cp110";
-				reg = <0x120000 0x6000>;
-				marvell,system-controller = <&cpm_syscon0>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				cpm_comphy0: phy@0 {
-					reg = <0>;
-					#phy-cells = <1>;
-				};
-
-				cpm_comphy1: phy@1 {
-					reg = <1>;
-					#phy-cells = <1>;
-				};
-
-				cpm_comphy2: phy@2 {
-					reg = <2>;
-					#phy-cells = <1>;
-				};
-
-				cpm_comphy3: phy@3 {
-					reg = <3>;
-					#phy-cells = <1>;
-				};
-
-				cpm_comphy4: phy@4 {
-					reg = <4>;
-					#phy-cells = <1>;
-				};
-
-				cpm_comphy5: phy@5 {
-					reg = <5>;
-					#phy-cells = <1>;
-				};
-			};
-
-			cpm_mdio: mdio@12a200 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				compatible = "marvell,orion-mdio";
-				reg = <0x12a200 0x10>;
-				clocks = <&cpm_clk 1 9>, <&cpm_clk 1 5>,
-					 <&cpm_clk 1 6>, <&cpm_clk 1 18>;
-				status = "disabled";
-			};
-
-			cpm_xmdio: mdio@12a600 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				compatible = "marvell,xmdio";
-				reg = <0x12a600 0x10>;
-				status = "disabled";
-			};
-
-			cpm_icu: interrupt-controller@1e0000 {
-				compatible = "marvell,cp110-icu";
-				reg = <0x1e0000 0x10>;
-				#interrupt-cells = <3>;
-				interrupt-controller;
-				msi-parent = <&gicp>;
-			};
-
-			cpm_rtc: rtc@284000 {
-				compatible = "marvell,armada-8k-rtc";
-				reg = <0x284000 0x20>, <0x284080 0x24>;
-				reg-names = "rtc", "rtc-soc";
-				interrupts = <ICU_GRP_NSR 77 IRQ_TYPE_LEVEL_HIGH>;
-			};
-
-			cpm_syscon0: system-controller@440000 {
-				compatible = "syscon", "simple-mfd";
-				reg = <0x440000 0x2000>;
-
-				cpm_clk: clock {
-					compatible = "marvell,cp110-clock";
-					#clock-cells = <2>;
-				};
-
-				cpm_gpio1: gpio@100 {
-					compatible = "marvell,armada-8k-gpio";
-					offset = <0x100>;
-					ngpios = <32>;
-					gpio-controller;
-					#gpio-cells = <2>;
-					gpio-ranges = <&cpm_pinctrl 0 0 32>;
-					interrupt-controller;
-					interrupts = <ICU_GRP_NSR 86 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 85 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 84 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 83 IRQ_TYPE_LEVEL_HIGH>;
-					status = "disabled";
-				};
-
-				cpm_gpio2: gpio@140 {
-					compatible = "marvell,armada-8k-gpio";
-					offset = <0x140>;
-					ngpios = <31>;
-					gpio-controller;
-					#gpio-cells = <2>;
-					gpio-ranges = <&cpm_pinctrl 0 32 31>;
-					interrupt-controller;
-					interrupts = <ICU_GRP_NSR 82 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 81 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 80 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 79 IRQ_TYPE_LEVEL_HIGH>;
-					status = "disabled";
-				};
-			};
-
-			cpm_usb3_0: usb3@500000 {
-				compatible = "marvell,armada-8k-xhci",
-					     "generic-xhci";
-				reg = <0x500000 0x4000>;
-				dma-coherent;
-				interrupts = <ICU_GRP_NSR 106 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cpm_clk 1 22>;
-				status = "disabled";
-			};
-
-			cpm_usb3_1: usb3@510000 {
-				compatible = "marvell,armada-8k-xhci",
-					     "generic-xhci";
-				reg = <0x510000 0x4000>;
-				dma-coherent;
-				interrupts = <ICU_GRP_NSR 105 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cpm_clk 1 23>;
-				status = "disabled";
-			};
-
-			cpm_sata0: sata@540000 {
-				compatible = "marvell,armada-8k-ahci",
-					     "generic-ahci";
-				reg = <0x540000 0x30000>;
-				interrupts = <ICU_GRP_NSR 107 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cpm_clk 1 15>;
-				status = "disabled";
-			};
-
-			cpm_xor0: xor@6a0000 {
-				compatible = "marvell,armada-7k-xor", "marvell,xor-v2";
-				reg = <0x6a0000 0x1000>,
-				      <0x6b0000 0x1000>;
-				dma-coherent;
-				msi-parent = <&gic_v2m0>;
-				clocks = <&cpm_clk 1 8>;
-			};
-
-			cpm_xor1: xor@6c0000 {
-				compatible = "marvell,armada-7k-xor", "marvell,xor-v2";
-				reg = <0x6c0000 0x1000>,
-				      <0x6d0000 0x1000>;
-				dma-coherent;
-				msi-parent = <&gic_v2m0>;
-				clocks = <&cpm_clk 1 7>;
-			};
-
-			cpm_spi0: spi@700600 {
-				compatible = "marvell,armada-380-spi";
-				reg = <0x700600 0x50>;
-				#address-cells = <0x1>;
-				#size-cells = <0x0>;
-				cell-index = <1>;
-				clocks = <&cpm_clk 1 21>;
-				status = "disabled";
-			};
-
-			cpm_spi1: spi@700680 {
-				compatible = "marvell,armada-380-spi";
-				reg = <0x700680 0x50>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				cell-index = <2>;
-				clocks = <&cpm_clk 1 21>;
-				status = "disabled";
-			};
-
-			cpm_i2c0: i2c@701000 {
-				compatible = "marvell,mv78230-i2c";
-				reg = <0x701000 0x20>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				interrupts = <ICU_GRP_NSR 120 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cpm_clk 1 21>;
-				status = "disabled";
-			};
-
-			cpm_i2c1: i2c@701100 {
-				compatible = "marvell,mv78230-i2c";
-				reg = <0x701100 0x20>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				interrupts = <ICU_GRP_NSR 121 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cpm_clk 1 21>;
-				status = "disabled";
-			};
-
-			cpm_nand: nand@720000 {
-				/*
-				 * Due to the limiation of the pin available
-				 * this controller is only usable on the CPM
-				 * for A7K and on the CPS for A8K.
-				 */
-				compatible = "marvell,armada-8k-nand",
-					     "marvell,armada370-nand";
-				reg = <0x720000 0x54>;
-				#address-cells = <1>;
-				#size-cells = <1>;
-				interrupts = <ICU_GRP_NSR 115 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cpm_clk 1 2>;
-				marvell,system-controller = <&cpm_syscon0>;
-				status = "disabled";
-			};
-
-			cpm_trng: trng@760000 {
-				compatible = "marvell,armada-8k-rng", "inside-secure,safexcel-eip76";
-				reg = <0x760000 0x7d>;
-				interrupts = <ICU_GRP_NSR 95 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cpm_clk 1 25>;
-				status = "okay";
-			};
-
-			cpm_sdhci0: sdhci@780000 {
-				compatible = "marvell,armada-cp110-sdhci";
-				reg = <0x780000 0x300>;
-				interrupts = <ICU_GRP_NSR 27 IRQ_TYPE_LEVEL_HIGH>;
-				clock-names = "core","axi";
-				clocks = <&cpm_clk 1 4>, <&cpm_clk 1 18>;
-				dma-coherent;
-				status = "disabled";
-			};
-
-			cpm_crypto: crypto@800000 {
-				compatible = "inside-secure,safexcel-eip197";
-				reg = <0x800000 0x200000>;
-				interrupts = <ICU_GRP_NSR 87 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 88 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 89 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 90 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 91 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 92 IRQ_TYPE_LEVEL_HIGH>;
-				interrupt-names = "mem", "ring0", "ring1",
-				"ring2", "ring3", "eip";
-				clocks = <&cpm_clk 1 26>;
-				dma-coherent;
-			};
-		};
-
-		cpm_pcie0: pcie@f2600000 {
-			compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
-			reg = <0 0xf2600000 0 0x10000>,
-			      <0 0xf6f00000 0 0x80000>;
-			reg-names = "ctrl", "config";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			#interrupt-cells = <1>;
-			device_type = "pci";
-			dma-coherent;
-			msi-parent = <&gic_v2m0>;
-
-			bus-range = <0 0xff>;
-			ranges =
-				/* downstream I/O */
-				<0x81000000 0 0xf9000000 0  0xf9000000 0 0x10000
-				/* non-prefetchable memory */
-				0x82000000 0 0xf6000000 0  0xf6000000 0 0xf00000>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0 0 0 0 &cpm_icu ICU_GRP_NSR 22 IRQ_TYPE_LEVEL_HIGH>;
-			interrupts = <ICU_GRP_NSR 22 IRQ_TYPE_LEVEL_HIGH>;
-			num-lanes = <1>;
-			clocks = <&cpm_clk 1 13>;
-			status = "disabled";
-		};
-
-		cpm_pcie1: pcie@f2620000 {
-			compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
-			reg = <0 0xf2620000 0 0x10000>,
-			      <0 0xf7f00000 0 0x80000>;
-			reg-names = "ctrl", "config";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			#interrupt-cells = <1>;
-			device_type = "pci";
-			dma-coherent;
-			msi-parent = <&gic_v2m0>;
-
-			bus-range = <0 0xff>;
-			ranges =
-				/* downstream I/O */
-				<0x81000000 0 0xf9010000 0  0xf9010000 0 0x10000
-				/* non-prefetchable memory */
-				0x82000000 0 0xf7000000 0  0xf7000000 0 0xf00000>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0 0 0 0 &cpm_icu ICU_GRP_NSR 24 IRQ_TYPE_LEVEL_HIGH>;
-			interrupts = <ICU_GRP_NSR 24 IRQ_TYPE_LEVEL_HIGH>;
-
-			num-lanes = <1>;
-			clocks = <&cpm_clk 1 11>;
-			status = "disabled";
-		};
-
-		cpm_pcie2: pcie@f2640000 {
-			compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
-			reg = <0 0xf2640000 0 0x10000>,
-			      <0 0xf8f00000 0 0x80000>;
-			reg-names = "ctrl", "config";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			#interrupt-cells = <1>;
-			device_type = "pci";
-			dma-coherent;
-			msi-parent = <&gic_v2m0>;
-
-			bus-range = <0 0xff>;
-			ranges =
-				/* downstream I/O */
-				<0x81000000 0 0xf9020000 0  0xf9020000 0 0x10000
-				/* non-prefetchable memory */
-				0x82000000 0 0xf8000000 0  0xf8000000 0 0xf00000>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0 0 0 0 &cpm_icu ICU_GRP_NSR 23 IRQ_TYPE_LEVEL_HIGH>;
-			interrupts = <ICU_GRP_NSR 23 IRQ_TYPE_LEVEL_HIGH>;
-
-			num-lanes = <1>;
-			clocks = <&cpm_clk 1 12>;
-			status = "disabled";
-		};
-	};
-};
diff --git a/arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi b/arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi
deleted file mode 100644
index 87ac68b2cf37..000000000000
--- a/arch/arm64/boot/dts/marvell/armada-cp110-slave.dtsi
+++ /dev/null
@@ -1,444 +0,0 @@
-/*
- * Copyright (C) 2016 Marvell Technology Group Ltd.
- *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPLv2 or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This library is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License as
- *     published by the Free Software Foundation; either version 2 of the
- *     License, or (at your option) any later version.
- *
- *     This library is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
- */
-
-/*
- * Device Tree file for Marvell Armada CP110 Slave.
- */
-
-#define ICU_GRP_NSR 0x0
-
-/ {
-	cp110-slave {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		compatible = "simple-bus";
-		interrupt-parent = <&cps_icu>;
-		ranges;
-
-		config-space@f4000000 {
-			#address-cells = <1>;
-			#size-cells = <1>;
-			compatible = "simple-bus";
-			ranges = <0x0 0x0 0xf4000000 0x2000000>;
-
-			cps_ethernet: ethernet@0 {
-				compatible = "marvell,armada-7k-pp22";
-				reg = <0x0 0x100000>, <0x129000 0xb000>;
-				clocks = <&cps_clk 1 3>, <&cps_clk 1 9>,
-					 <&cps_clk 1 5>, <&cps_clk 1 18>;
-				clock-names = "pp_clk", "gop_clk",
-					      "mg_clk", "axi_clk";
-				marvell,system-controller = <&cps_syscon0>;
-				status = "disabled";
-				dma-coherent;
-
-				cps_eth0: eth0 {
-					interrupts = <ICU_GRP_NSR 39 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 43 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 47 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 51 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 55 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 129 IRQ_TYPE_LEVEL_HIGH>;
-					interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
-							  "tx-cpu3", "rx-shared", "link";
-					port-id = <0>;
-					gop-port-id = <0>;
-					status = "disabled";
-				};
-
-				cps_eth1: eth1 {
-					interrupts = <ICU_GRP_NSR 40 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 44 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 48 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 52 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 56 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 128 IRQ_TYPE_LEVEL_HIGH>;
-					interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
-							  "tx-cpu3", "rx-shared", "link";
-					port-id = <1>;
-					gop-port-id = <2>;
-					status = "disabled";
-				};
-
-				cps_eth2: eth2 {
-					interrupts = <ICU_GRP_NSR 41 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 45 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 49 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 53 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 57 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 127 IRQ_TYPE_LEVEL_HIGH>;
-					interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
-							  "tx-cpu3", "rx-shared", "link";
-					port-id = <2>;
-					gop-port-id = <3>;
-					status = "disabled";
-				};
-			};
-
-			cps_comphy: phy@120000 {
-				compatible = "marvell,comphy-cp110";
-				reg = <0x120000 0x6000>;
-				marvell,system-controller = <&cps_syscon0>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				cps_comphy0: phy@0 {
-					reg = <0>;
-					#phy-cells = <1>;
-				};
-
-				cps_comphy1: phy@1 {
-					reg = <1>;
-					#phy-cells = <1>;
-				};
-
-				cps_comphy2: phy@2 {
-					reg = <2>;
-					#phy-cells = <1>;
-				};
-
-				cps_comphy3: phy@3 {
-					reg = <3>;
-					#phy-cells = <1>;
-				};
-
-				cps_comphy4: phy@4 {
-					reg = <4>;
-					#phy-cells = <1>;
-				};
-
-				cps_comphy5: phy@5 {
-					reg = <5>;
-					#phy-cells = <1>;
-				};
-			};
-
-			cps_mdio: mdio@12a200 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				compatible = "marvell,orion-mdio";
-				reg = <0x12a200 0x10>;
-				clocks = <&cps_clk 1 9>, <&cps_clk 1 5>,
-					 <&cps_clk 1 6>, <&cps_clk 1 18>;
-				status = "disabled";
-			};
-
-			cps_xmdio: mdio@12a600 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				compatible = "marvell,xmdio";
-				reg = <0x12a600 0x10>;
-				status = "disabled";
-			};
-
-			cps_icu: interrupt-controller@1e0000 {
-				compatible = "marvell,cp110-icu";
-				reg = <0x1e0000 0x10>;
-				#interrupt-cells = <3>;
-				interrupt-controller;
-				msi-parent = <&gicp>;
-			};
-
-			cps_rtc: rtc@284000 {
-				compatible = "marvell,armada-8k-rtc";
-				reg = <0x284000 0x20>, <0x284080 0x24>;
-				reg-names = "rtc", "rtc-soc";
-				interrupts = <ICU_GRP_NSR 77 IRQ_TYPE_LEVEL_HIGH>;
-			};
-
-			cps_syscon0: system-controller@440000 {
-				compatible = "syscon", "simple-mfd";
-				reg = <0x440000 0x2000>;
-
-				cps_clk: clock {
-					compatible = "marvell,cp110-clock";
-					#clock-cells = <2>;
-				};
-
-				cps_gpio1: gpio@100 {
-					compatible = "marvell,armada-8k-gpio";
-					offset = <0x100>;
-					ngpios = <32>;
-					gpio-controller;
-					#gpio-cells = <2>;
-					gpio-ranges = <&cps_pinctrl 0 0 32>;
-					interrupt-controller;
-					interrupts = <ICU_GRP_NSR 86 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 85 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 84 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 83 IRQ_TYPE_LEVEL_HIGH>;
-					status = "disabled";
-				};
-
-				cps_gpio2: gpio@140 {
-					compatible = "marvell,armada-8k-gpio";
-					offset = <0x140>;
-					ngpios = <31>;
-					gpio-controller;
-					#gpio-cells = <2>;
-					gpio-ranges = <&cps_pinctrl 0 32 31>;
-					interrupt-controller;
-					interrupts = <ICU_GRP_NSR 82 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 81 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 80 IRQ_TYPE_LEVEL_HIGH>,
-						     <ICU_GRP_NSR 79 IRQ_TYPE_LEVEL_HIGH>;
-					status = "disabled";
-				};
-
-			};
-
-			cps_usb3_0: usb3@500000 {
-				compatible = "marvell,armada-8k-xhci",
-					     "generic-xhci";
-				reg = <0x500000 0x4000>;
-				dma-coherent;
-				interrupts = <ICU_GRP_NSR 106 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cps_clk 1 22>;
-				status = "disabled";
-			};
-
-			cps_usb3_1: usb3@510000 {
-				compatible = "marvell,armada-8k-xhci",
-					     "generic-xhci";
-				reg = <0x510000 0x4000>;
-				dma-coherent;
-				interrupts = <ICU_GRP_NSR 105 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cps_clk 1 23>;
-				status = "disabled";
-			};
-
-			cps_sata0: sata@540000 {
-				compatible = "marvell,armada-8k-ahci",
-					     "generic-ahci";
-				reg = <0x540000 0x30000>;
-				interrupts = <ICU_GRP_NSR 107 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cps_clk 1 15>;
-				status = "disabled";
-			};
-
-			cps_xor0: xor@6a0000 {
-				compatible = "marvell,armada-7k-xor", "marvell,xor-v2";
-				reg = <0x6a0000 0x1000>,
-				      <0x6b0000 0x1000>;
-				dma-coherent;
-				msi-parent = <&gic_v2m0>;
-				clocks = <&cps_clk 1 8>;
-			};
-
-			cps_xor1: xor@6c0000 {
-				compatible = "marvell,armada-7k-xor", "marvell,xor-v2";
-				reg = <0x6c0000 0x1000>,
-				      <0x6d0000 0x1000>;
-				dma-coherent;
-				msi-parent = <&gic_v2m0>;
-				clocks = <&cps_clk 1 7>;
-			};
-
-			cps_spi0: spi@700600 {
-				compatible = "marvell,armada-380-spi";
-				reg = <0x700600 0x50>;
-				#address-cells = <0x1>;
-				#size-cells = <0x0>;
-				cell-index = <3>;
-				clocks = <&cps_clk 1 21>;
-				status = "disabled";
-			};
-
-			cps_spi1: spi@700680 {
-				compatible = "marvell,armada-380-spi";
-				reg = <0x700680 0x50>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				cell-index = <4>;
-				clocks = <&cps_clk 1 21>;
-				status = "disabled";
-			};
-
-			cps_i2c0: i2c@701000 {
-				compatible = "marvell,mv78230-i2c";
-				reg = <0x701000 0x20>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				interrupts = <ICU_GRP_NSR 120 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cps_clk 1 21>;
-				status = "disabled";
-			};
-
-			cps_i2c1: i2c@701100 {
-				compatible = "marvell,mv78230-i2c";
-				reg = <0x701100 0x20>;
-				#address-cells = <1>;
-				#size-cells = <0>;
-				interrupts = <ICU_GRP_NSR 121 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cps_clk 1 21>;
-				status = "disabled";
-			};
-
-			cps_nand: nand@720000 {
-				/*
-				 * Due to the limiation of the pin available
-				 * this controller is only usable on the CPM
-				 * for A7K and on the CPS for A8K.
-				 */
-				compatible = "marvell,armada370-nand",
-					     "marvell,armada370-nand";
-				reg = <0x720000 0x54>;
-				#address-cells = <1>;
-				#size-cells = <1>;
-				interrupts = <ICU_GRP_NSR 115 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cps_clk 1 2>;
-				status = "disabled";
-			};
-
-			cps_trng: trng@760000 {
-				compatible = "marvell,armada-8k-rng", "inside-secure,safexcel-eip76";
-				reg = <0x760000 0x7d>;
-				interrupts = <ICU_GRP_NSR 95 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&cps_clk 1 25>;
-				status = "okay";
-			};
-
-			cps_crypto: crypto@800000 {
-				compatible = "inside-secure,safexcel-eip197";
-				reg = <0x800000 0x200000>;
-				interrupts = <ICU_GRP_NSR 87 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 88 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 89 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 90 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 91 IRQ_TYPE_LEVEL_HIGH>,
-					     <ICU_GRP_NSR 92 IRQ_TYPE_LEVEL_HIGH>;
-				interrupt-names = "mem", "ring0", "ring1",
-						  "ring2", "ring3", "eip";
-				clocks = <&cps_clk 1 26>;
-				dma-coherent;
-				/*
-				 * The cryptographic engine found on the cp110
-				 * master is enabled by default at the SoC
-				 * level. Because it is not possible as of now
-				 * to enable two cryptographic engines in
-				 * parallel, disable this one by default.
-				 */
-				status = "disabled";
-			};
-		};
-
-		cps_pcie0: pcie@f4600000 {
-			compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
-			reg = <0 0xf4600000 0 0x10000>,
-			      <0 0xfaf00000 0 0x80000>;
-			reg-names = "ctrl", "config";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			#interrupt-cells = <1>;
-			device_type = "pci";
-			dma-coherent;
-			msi-parent = <&gic_v2m0>;
-
-			bus-range = <0 0xff>;
-			ranges =
-				/* downstream I/O */
-				<0x81000000 0 0xfd000000 0  0xfd000000 0 0x10000
-				/* non-prefetchable memory */
-				0x82000000 0 0xfa000000 0  0xfa000000 0 0xf00000>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0 0 0 0 &cps_icu ICU_GRP_NSR 22 IRQ_TYPE_LEVEL_HIGH>;
-			interrupts = <ICU_GRP_NSR 22 IRQ_TYPE_LEVEL_HIGH>;
-			num-lanes = <1>;
-			clocks = <&cps_clk 1 13>;
-			status = "disabled";
-		};
-
-		cps_pcie1: pcie@f4620000 {
-			compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
-			reg = <0 0xf4620000 0 0x10000>,
-			      <0 0xfbf00000 0 0x80000>;
-			reg-names = "ctrl", "config";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			#interrupt-cells = <1>;
-			device_type = "pci";
-			dma-coherent;
-			msi-parent = <&gic_v2m0>;
-
-			bus-range = <0 0xff>;
-			ranges =
-				/* downstream I/O */
-				<0x81000000 0 0xfd010000 0  0xfd010000 0 0x10000
-				/* non-prefetchable memory */
-				0x82000000 0 0xfb000000 0  0xfb000000 0 0xf00000>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0 0 0 0 &cps_icu ICU_GRP_NSR 24 IRQ_TYPE_LEVEL_HIGH>;
-			interrupts = <ICU_GRP_NSR 24 IRQ_TYPE_LEVEL_HIGH>;
-
-			num-lanes = <1>;
-			clocks = <&cps_clk 1 11>;
-			status = "disabled";
-		};
-
-		cps_pcie2: pcie@f4640000 {
-			compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
-			reg = <0 0xf4640000 0 0x10000>,
-			      <0 0xfcf00000 0 0x80000>;
-			reg-names = "ctrl", "config";
-			#address-cells = <3>;
-			#size-cells = <2>;
-			#interrupt-cells = <1>;
-			device_type = "pci";
-			dma-coherent;
-			msi-parent = <&gic_v2m0>;
-
-			bus-range = <0 0xff>;
-			ranges =
-				/* downstream I/O */
-				<0x81000000 0 0xfd020000 0  0xfd020000 0 0x10000
-				/* non-prefetchable memory */
-				0x82000000 0 0xfc000000 0  0xfc000000 0 0xf00000>;
-			interrupt-map-mask = <0 0 0 0>;
-			interrupt-map = <0 0 0 0 &cps_icu ICU_GRP_NSR 23 IRQ_TYPE_LEVEL_HIGH>;
-			interrupts = <ICU_GRP_NSR 23 IRQ_TYPE_LEVEL_HIGH>;
-
-			num-lanes = <1>;
-			clocks = <&cps_clk 1 12>;
-			status = "disabled";
-		};
-	};
-};
diff --git a/arch/arm64/boot/dts/marvell/armada-cp110.dtsi b/arch/arm64/boot/dts/marvell/armada-cp110.dtsi
new file mode 100644
index 000000000000..a8af4136dbe7
--- /dev/null
+++ b/arch/arm64/boot/dts/marvell/armada-cp110.dtsi
@@ -0,0 +1,424 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR X11)
+/*
+ * Copyright (C) 2016 Marvell Technology Group Ltd.
+ */
+
+/*
+ * Device Tree file for Marvell Armada CP110.
+ */
+
+#include <dt-bindings/interrupt-controller/mvebu-icu.h>
+
+#include "armada-common.dtsi"
+
+#define CP110_PCIEx_IO_BASE(iface)	(CP110_PCIE_IO_BASE + (iface *  0x10000))
+#define CP110_PCIEx_MEM_BASE(iface)	(CP110_PCIE_MEM_BASE + (iface *  0x1000000))
+#define CP110_PCIEx_CONF_BASE(iface)	(CP110_PCIEx_MEM_BASE(iface) + 0xf00000)
+
+/ {
+	/*
+	 * The contents of the node are defined below, in order to
+	 * save one indentation level
+	 */
+	CP110_NAME: CP110_NAME { };
+};
+
+&CP110_NAME {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	compatible = "simple-bus";
+	interrupt-parent = <&CP110_LABEL(icu)>;
+	ranges;
+
+	config-space@CP110_BASE {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		ranges = <0x0 0x0 ADDRESSIFY(CP110_BASE) 0x2000000>;
+
+		CP110_LABEL(ethernet): ethernet@0 {
+			compatible = "marvell,armada-7k-pp22";
+			reg = <0x0 0x100000>, <0x129000 0xb000>;
+			clocks = <&CP110_LABEL(clk) 1 3>, <&CP110_LABEL(clk) 1 9>,
+				 <&CP110_LABEL(clk) 1 5>, <&CP110_LABEL(clk) 1 18>;
+			clock-names = "pp_clk", "gop_clk",
+				      "mg_clk", "axi_clk";
+			marvell,system-controller = <&CP110_LABEL(syscon0)>;
+			status = "disabled";
+			dma-coherent;
+
+			CP110_LABEL(eth0): eth0 {
+				interrupts = <ICU_GRP_NSR 39 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 43 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 47 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 51 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 55 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 129 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
+					"tx-cpu3", "rx-shared", "link";
+				port-id = <0>;
+				gop-port-id = <0>;
+				status = "disabled";
+			};
+
+			CP110_LABEL(eth1): eth1 {
+				interrupts = <ICU_GRP_NSR 40 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 44 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 48 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 52 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 56 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 128 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
+					"tx-cpu3", "rx-shared", "link";
+				port-id = <1>;
+				gop-port-id = <2>;
+				status = "disabled";
+			};
+
+			CP110_LABEL(eth2): eth2 {
+				interrupts = <ICU_GRP_NSR 41 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 45 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 49 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 53 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 57 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 127 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "tx-cpu0", "tx-cpu1", "tx-cpu2",
+					"tx-cpu3", "rx-shared", "link";
+				port-id = <2>;
+				gop-port-id = <3>;
+				status = "disabled";
+			};
+		};
+
+		CP110_LABEL(comphy): phy@120000 {
+			compatible = "marvell,comphy-cp110";
+			reg = <0x120000 0x6000>;
+			marvell,system-controller = <&CP110_LABEL(syscon0)>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			CP110_LABEL(comphy0): phy@0 {
+				reg = <0>;
+				#phy-cells = <1>;
+			};
+
+			CP110_LABEL(comphy1): phy@1 {
+				reg = <1>;
+				#phy-cells = <1>;
+			};
+
+			CP110_LABEL(comphy2): phy@2 {
+				reg = <2>;
+				#phy-cells = <1>;
+			};
+
+			CP110_LABEL(comphy3): phy@3 {
+				reg = <3>;
+				#phy-cells = <1>;
+			};
+
+			CP110_LABEL(comphy4): phy@4 {
+				reg = <4>;
+				#phy-cells = <1>;
+			};
+
+			CP110_LABEL(comphy5): phy@5 {
+				reg = <5>;
+				#phy-cells = <1>;
+			};
+		};
+
+		CP110_LABEL(mdio): mdio@12a200 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "marvell,orion-mdio";
+			reg = <0x12a200 0x10>;
+			clocks = <&CP110_LABEL(clk) 1 9>, <&CP110_LABEL(clk) 1 5>,
+				 <&CP110_LABEL(clk) 1 6>, <&CP110_LABEL(clk) 1 18>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(xmdio): mdio@12a600 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "marvell,xmdio";
+			reg = <0x12a600 0x10>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(icu): interrupt-controller@1e0000 {
+			compatible = "marvell,cp110-icu";
+			reg = <0x1e0000 0x10>;
+			#interrupt-cells = <3>;
+			interrupt-controller;
+			msi-parent = <&gicp>;
+		};
+
+		CP110_LABEL(rtc): rtc@284000 {
+			compatible = "marvell,armada-8k-rtc";
+			reg = <0x284000 0x20>, <0x284080 0x24>;
+			reg-names = "rtc", "rtc-soc";
+			interrupts = <ICU_GRP_NSR 77 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		CP110_LABEL(thermal): thermal@400078 {
+			compatible = "marvell,armada-cp110-thermal";
+			reg = <0x400078 0x4>,
+			<0x400070 0x8>;
+		};
+
+		CP110_LABEL(syscon0): system-controller@440000 {
+			compatible = "syscon", "simple-mfd";
+			reg = <0x440000 0x2000>;
+
+			CP110_LABEL(clk): clock {
+				compatible = "marvell,cp110-clock";
+				#clock-cells = <2>;
+			};
+
+			CP110_LABEL(gpio1): gpio@100 {
+				compatible = "marvell,armada-8k-gpio";
+				offset = <0x100>;
+				ngpios = <32>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&CP110_LABEL(pinctrl) 0 0 32>;
+				interrupt-controller;
+				interrupts = <ICU_GRP_NSR 86 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 85 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 84 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 83 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+
+			CP110_LABEL(gpio2): gpio@140 {
+				compatible = "marvell,armada-8k-gpio";
+				offset = <0x140>;
+				ngpios = <31>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				gpio-ranges = <&CP110_LABEL(pinctrl) 0 32 31>;
+				interrupt-controller;
+				interrupts = <ICU_GRP_NSR 82 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 81 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 80 IRQ_TYPE_LEVEL_HIGH>,
+					<ICU_GRP_NSR 79 IRQ_TYPE_LEVEL_HIGH>;
+				status = "disabled";
+			};
+		};
+
+		CP110_LABEL(usb3_0): usb3@500000 {
+			compatible = "marvell,armada-8k-xhci",
+			"generic-xhci";
+			reg = <0x500000 0x4000>;
+			dma-coherent;
+			interrupts = <ICU_GRP_NSR 106 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&CP110_LABEL(clk) 1 22>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(usb3_1): usb3@510000 {
+			compatible = "marvell,armada-8k-xhci",
+			"generic-xhci";
+			reg = <0x510000 0x4000>;
+			dma-coherent;
+			interrupts = <ICU_GRP_NSR 105 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&CP110_LABEL(clk) 1 23>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(sata0): sata@540000 {
+			compatible = "marvell,armada-8k-ahci",
+			"generic-ahci";
+			reg = <0x540000 0x30000>;
+			interrupts = <ICU_GRP_NSR 107 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&CP110_LABEL(clk) 1 15>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(xor0): xor@6a0000 {
+			compatible = "marvell,armada-7k-xor", "marvell,xor-v2";
+			reg = <0x6a0000 0x1000>, <0x6b0000 0x1000>;
+			dma-coherent;
+			msi-parent = <&gic_v2m0>;
+			clocks = <&CP110_LABEL(clk) 1 8>;
+		};
+
+		CP110_LABEL(xor1): xor@6c0000 {
+			compatible = "marvell,armada-7k-xor", "marvell,xor-v2";
+			reg = <0x6c0000 0x1000>, <0x6d0000 0x1000>;
+			dma-coherent;
+			msi-parent = <&gic_v2m0>;
+			clocks = <&CP110_LABEL(clk) 1 7>;
+		};
+
+		CP110_LABEL(spi0): spi@700600 {
+			compatible = "marvell,armada-380-spi";
+			reg = <0x700600 0x50>;
+			#address-cells = <0x1>;
+			#size-cells = <0x0>;
+			clocks = <&CP110_LABEL(clk) 1 21>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(spi1): spi@700680 {
+			compatible = "marvell,armada-380-spi";
+			reg = <0x700680 0x50>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			clocks = <&CP110_LABEL(clk) 1 21>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(i2c0): i2c@701000 {
+			compatible = "marvell,mv78230-i2c";
+			reg = <0x701000 0x20>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <ICU_GRP_NSR 120 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&CP110_LABEL(clk) 1 21>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(i2c1): i2c@701100 {
+			compatible = "marvell,mv78230-i2c";
+			reg = <0x701100 0x20>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			interrupts = <ICU_GRP_NSR 121 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&CP110_LABEL(clk) 1 21>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(nand): nand@720000 {
+			/*
+			* Due to the limitation of the pins available
+			* this controller is only usable on the CPM
+			* for A7K and on the CPS for A8K.
+			*/
+			compatible = "marvell,armada-8k-nand",
+			"marvell,armada370-nand";
+			reg = <0x720000 0x54>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			interrupts = <ICU_GRP_NSR 115 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&CP110_LABEL(clk) 1 2>;
+			marvell,system-controller = <&CP110_LABEL(syscon0)>;
+			status = "disabled";
+		};
+
+		CP110_LABEL(trng): trng@760000 {
+			compatible = "marvell,armada-8k-rng",
+			"inside-secure,safexcel-eip76";
+			reg = <0x760000 0x7d>;
+			interrupts = <ICU_GRP_NSR 95 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&CP110_LABEL(clk) 1 25>;
+			status = "okay";
+		};
+
+		CP110_LABEL(sdhci0): sdhci@780000 {
+			compatible = "marvell,armada-cp110-sdhci";
+			reg = <0x780000 0x300>;
+			interrupts = <ICU_GRP_NSR 27 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "core", "axi";
+			clocks = <&CP110_LABEL(clk) 1 4>, <&CP110_LABEL(clk) 1 18>;
+			dma-coherent;
+			status = "disabled";
+		};
+
+		CP110_LABEL(crypto): crypto@800000 {
+			compatible = "inside-secure,safexcel-eip197";
+			reg = <0x800000 0x200000>;
+			interrupts = <ICU_GRP_NSR 87 IRQ_TYPE_LEVEL_HIGH>,
+				<ICU_GRP_NSR 88 IRQ_TYPE_LEVEL_HIGH>,
+				<ICU_GRP_NSR 89 IRQ_TYPE_LEVEL_HIGH>,
+				<ICU_GRP_NSR 90 IRQ_TYPE_LEVEL_HIGH>,
+				<ICU_GRP_NSR 91 IRQ_TYPE_LEVEL_HIGH>,
+				<ICU_GRP_NSR 92 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "mem", "ring0", "ring1",
+				"ring2", "ring3", "eip";
+			clocks = <&CP110_LABEL(clk) 1 26>;
+			dma-coherent;
+		};
+	};
+
+	CP110_LABEL(pcie0): pcie@CP110_PCIE0_BASE {
+		compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
+		reg = <0 ADDRESSIFY(CP110_PCIE0_BASE) 0 0x10000>,
+		      <0 CP110_PCIEx_CONF_BASE(0) 0 0x80000>;
+		reg-names = "ctrl", "config";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		#interrupt-cells = <1>;
+		device_type = "pci";
+		dma-coherent;
+		msi-parent = <&gic_v2m0>;
+
+		bus-range = <0 0xff>;
+		ranges =
+		/* downstream I/O */
+		<0x81000000 0 CP110_PCIEx_IO_BASE(0) 0  CP110_PCIEx_IO_BASE(0) 0 0x10000
+		/* non-prefetchable memory */
+		0x82000000 0 CP110_PCIEx_MEM_BASE(0) 0  CP110_PCIEx_MEM_BASE(0) 0 0xf00000>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &CP110_LABEL(icu) ICU_GRP_NSR 22 IRQ_TYPE_LEVEL_HIGH>;
+		interrupts = <ICU_GRP_NSR 22 IRQ_TYPE_LEVEL_HIGH>;
+		num-lanes = <1>;
+		clocks = <&CP110_LABEL(clk) 1 13>;
+		status = "disabled";
+	};
+
+	CP110_LABEL(pcie1): pcie@CP110_PCIE1_BASE {
+		compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
+		reg = <0 ADDRESSIFY(CP110_PCIE1_BASE) 0 0x10000>,
+		      <0 CP110_PCIEx_CONF_BASE(1) 0 0x80000>;
+		reg-names = "ctrl", "config";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		#interrupt-cells = <1>;
+		device_type = "pci";
+		dma-coherent;
+		msi-parent = <&gic_v2m0>;
+
+		bus-range = <0 0xff>;
+		ranges =
+		/* downstream I/O */
+		<0x81000000 0 CP110_PCIEx_IO_BASE(1) 0  CP110_PCIEx_IO_BASE(1) 0 0x10000
+		/* non-prefetchable memory */
+		0x82000000 0 CP110_PCIEx_MEM_BASE(1) 0  CP110_PCIEx_MEM_BASE(1) 0 0xf00000>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &CP110_LABEL(icu) ICU_GRP_NSR 24 IRQ_TYPE_LEVEL_HIGH>;
+		interrupts = <ICU_GRP_NSR 24 IRQ_TYPE_LEVEL_HIGH>;
+
+		num-lanes = <1>;
+		clocks = <&CP110_LABEL(clk) 1 11>;
+		status = "disabled";
+	};
+
+	CP110_LABEL(pcie2): pcie@CP110_PCIE2_BASE {
+		compatible = "marvell,armada8k-pcie", "snps,dw-pcie";
+		reg = <0 ADDRESSIFY(CP110_PCIE2_BASE) 0 0x10000>,
+		      <0 CP110_PCIEx_CONF_BASE(2) 0 0x80000>;
+		reg-names = "ctrl", "config";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		#interrupt-cells = <1>;
+		device_type = "pci";
+		dma-coherent;
+		msi-parent = <&gic_v2m0>;
+
+		bus-range = <0 0xff>;
+		ranges =
+		/* downstream I/O */
+		<0x81000000 0 CP110_PCIEx_IO_BASE(2) 0  CP110_PCIEx_IO_BASE(2) 0 0x10000
+		/* non-prefetchable memory */
+		0x82000000 0 CP110_PCIEx_MEM_BASE(2) 0  CP110_PCIEx_MEM_BASE(2) 0 0xf00000>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &CP110_LABEL(icu) ICU_GRP_NSR 23 IRQ_TYPE_LEVEL_HIGH>;
+		interrupts = <ICU_GRP_NSR 23 IRQ_TYPE_LEVEL_HIGH>;
+
+		num-lanes = <1>;
+		clocks = <&CP110_LABEL(clk) 1 12>;
+		status = "disabled";
+	};
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt2712-evb.dts b/arch/arm64/boot/dts/mediatek/mt2712-evb.dts
index 8c804df3da4e..10f9c76cd105 100644
--- a/arch/arm64/boot/dts/mediatek/mt2712-evb.dts
+++ b/arch/arm64/boot/dts/mediatek/mt2712-evb.dts
@@ -24,6 +24,33 @@
 	chosen {
 		stdout-path = "serial0:921600n8";
 	};
+
+	cpus_fixed_vproc0: fixedregulator@0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vproc_buck0";
+		regulator-min-microvolt = <1000000>;
+		regulator-max-microvolt = <1000000>;
+	};
+
+	cpus_fixed_vproc1: fixedregulator@1 {
+		compatible = "regulator-fixed";
+		regulator-name = "vproc_buck1";
+		regulator-min-microvolt = <1000000>;
+		regulator-max-microvolt = <1000000>;
+	};
+
+};
+
+&cpu0 {
+	proc-supply = <&cpus_fixed_vproc0>;
+};
+
+&cpu1 {
+	proc-supply = <&cpus_fixed_vproc0>;
+};
+
+&cpu2 {
+	proc-supply = <&cpus_fixed_vproc1>;
 };
 
 &uart0 {
diff --git a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
index 5d4e406bb35d..fdf66f4fe7c3 100644
--- a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
@@ -5,8 +5,10 @@
  * SPDX-License-Identifier: (GPL-2.0 OR MIT)
  */
 
+#include <dt-bindings/clock/mt2712-clk.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/power/mt2712-power.h>
 
 / {
 	compatible = "mediatek,mt2712";
@@ -14,6 +16,48 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	cluster0_opp: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+		opp00 {
+			opp-hz = /bits/ 64 <598000000>;
+			opp-microvolt = <1000000>;
+		};
+		opp01 {
+			opp-hz = /bits/ 64 <702000000>;
+			opp-microvolt = <1000000>;
+		};
+		opp02 {
+			opp-hz = /bits/ 64 <793000000>;
+			opp-microvolt = <1000000>;
+		};
+	};
+
+	cluster1_opp: opp_table1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+		opp00 {
+			opp-hz = /bits/ 64 <598000000>;
+			opp-microvolt = <1000000>;
+		};
+		opp01 {
+			opp-hz = /bits/ 64 <702000000>;
+			opp-microvolt = <1000000>;
+		};
+		opp02 {
+			opp-hz = /bits/ 64 <793000000>;
+			opp-microvolt = <1000000>;
+		};
+		opp03 {
+			opp-hz = /bits/ 64 <897000000>;
+			opp-microvolt = <1000000>;
+		};
+		opp04 {
+			opp-hz = /bits/ 64 <1001000000>;
+			opp-microvolt = <1000000>;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -39,6 +83,11 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a35";
 			reg = <0x000>;
+			clocks = <&mcucfg CLK_MCU_MP0_SEL>,
+				<&topckgen CLK_TOP_F_MP0_PLL1>;
+			clock-names = "cpu", "intermediate";
+			proc-supply = <&cpus_fixed_vproc0>;
+			operating-points-v2 = <&cluster0_opp>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
 		};
 
@@ -47,6 +96,11 @@
 			compatible = "arm,cortex-a35";
 			reg = <0x001>;
 			enable-method = "psci";
+			clocks = <&mcucfg CLK_MCU_MP0_SEL>,
+				<&topckgen CLK_TOP_F_MP0_PLL1>;
+			clock-names = "cpu", "intermediate";
+			proc-supply = <&cpus_fixed_vproc0>;
+			operating-points-v2 = <&cluster0_opp>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
 		};
 
@@ -55,6 +109,11 @@
 			compatible = "arm,cortex-a72";
 			reg = <0x200>;
 			enable-method = "psci";
+			clocks = <&mcucfg CLK_MCU_MP2_SEL>,
+				<&topckgen CLK_TOP_F_BIG_PLL1>;
+			clock-names = "cpu", "intermediate";
+			proc-supply = <&cpus_fixed_vproc1>;
+			operating-points-v2 = <&cluster1_opp>;
 			cpu-idle-states = <&CPU_SLEEP_0 &CLUSTER_SLEEP_0>;
 		};
 
@@ -98,6 +157,48 @@
 		#clock-cells = <0>;
 	};
 
+	clk26m: oscillator@0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
+		clock-output-names = "clk26m";
+	};
+
+	clk32k: oscillator@1 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		clock-output-names = "clk32k";
+	};
+
+	clkfpc: oscillator@2 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <50000000>;
+		clock-output-names = "clkfpc";
+	};
+
+	clkaud_ext_i_0: oscillator@3 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <6500000>;
+		clock-output-names = "clkaud_ext_i_0";
+	};
+
+	clkaud_ext_i_1: oscillator@4 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <196608000>;
+		clock-output-names = "clkaud_ext_i_1";
+	};
+
+	clkaud_ext_i_2: oscillator@5 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <180633600>;
+		clock-output-names = "clkaud_ext_i_2";
+	};
+
 	timer {
 		compatible = "arm,armv8-timer";
 		interrupt-parent = <&gic>;
@@ -111,6 +212,39 @@
 			      (GIC_CPU_MASK_RAW(0x13) | IRQ_TYPE_LEVEL_LOW)>;
 	};
 
+	topckgen: syscon@10000000 {
+		compatible = "mediatek,mt2712-topckgen", "syscon";
+		reg = <0 0x10000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	infracfg: syscon@10001000 {
+		compatible = "mediatek,mt2712-infracfg", "syscon";
+		reg = <0 0x10001000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	pericfg: syscon@10003000 {
+		compatible = "mediatek,mt2712-pericfg", "syscon";
+		reg = <0 0x10003000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	scpsys: scpsys@10006000 {
+		compatible = "mediatek,mt2712-scpsys", "syscon";
+		#power-domain-cells = <1>;
+		reg = <0 0x10006000 0 0x1000>;
+		clocks = <&topckgen CLK_TOP_MM_SEL>,
+			 <&topckgen CLK_TOP_MFG_SEL>,
+			 <&topckgen CLK_TOP_VENC_SEL>,
+			 <&topckgen CLK_TOP_JPGDEC_SEL>,
+			 <&topckgen CLK_TOP_A1SYS_HP_SEL>,
+			 <&topckgen CLK_TOP_VDEC_SEL>;
+		clock-names = "mm", "mfg", "venc",
+			"jpgdec", "audio", "vdec";
+		infracfg = <&infracfg>;
+	};
+
 	uart5: serial@1000f000 {
 		compatible = "mediatek,mt2712-uart",
 			     "mediatek,mt6577-uart";
@@ -121,6 +255,18 @@
 		status = "disabled";
 	};
 
+	apmixedsys: syscon@10209000 {
+		compatible = "mediatek,mt2712-apmixedsys", "syscon";
+		reg = <0 0x10209000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	mcucfg: syscon@10220000 {
+		compatible = "mediatek,mt2712-mcucfg", "syscon";
+		reg = <0 0x10220000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
 	sysirq: interrupt-controller@10220a80 {
 		compatible = "mediatek,mt2712-sysirq",
 			     "mediatek,mt6577-sysirq";
@@ -192,5 +338,47 @@
 		clock-names = "baud", "bus";
 		status = "disabled";
 	};
+
+	mfgcfg: syscon@13000000 {
+		compatible = "mediatek,mt2712-mfgcfg", "syscon";
+		reg = <0 0x13000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	mmsys: syscon@14000000 {
+		compatible = "mediatek,mt2712-mmsys", "syscon";
+		reg = <0 0x14000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	imgsys: syscon@15000000 {
+		compatible = "mediatek,mt2712-imgsys", "syscon";
+		reg = <0 0x15000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	bdpsys: syscon@15010000 {
+		compatible = "mediatek,mt2712-bdpsys", "syscon";
+		reg = <0 0x15010000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	vdecsys: syscon@16000000 {
+		compatible = "mediatek,mt2712-vdecsys", "syscon";
+		reg = <0 0x16000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	vencsys: syscon@18000000 {
+		compatible = "mediatek,mt2712-vencsys", "syscon";
+		reg = <0 0x18000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
+
+	jpgdecsys: syscon@19000000 {
+		compatible = "mediatek,mt2712-jpgdecsys", "syscon";
+		reg = <0 0x19000000 0 0x1000>;
+		#clock-cells = <1>;
+	};
 };
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
index 1c3634fa94bf..8e6ada20e6db 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts
@@ -74,6 +74,24 @@
 	status = "okay";
 };
 
+&cpu0 {
+	proc-supply = <&mt6397_vpca15_reg>;
+};
+
+&cpu1 {
+	proc-supply = <&mt6397_vpca15_reg>;
+};
+
+&cpu2 {
+	proc-supply = <&da9211_vcpu_reg>;
+	sram-supply = <&mt6397_vsramca7_reg>;
+};
+
+&cpu3 {
+	proc-supply = <&da9211_vcpu_reg>;
+	sram-supply = <&mt6397_vsramca7_reg>;
+};
+
 &dpi0 {
 	status = "okay";
 };
@@ -505,7 +523,7 @@
 	vbus-supply = <&usb_p0_vbus>;
 	extcon = <&extcon_usb>;
 	dr_mode = "otg";
-	mediatek,enable-wakeup;
+	wakeup-source;
 	pinctrl-names = "default", "id_float", "id_ground";
 	pinctrl-0 = <&usb_id_pins_float>;
 	pinctrl-1 = <&usb_id_pins_float>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
index 26396ef53bde..94597e33c806 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
@@ -51,6 +51,80 @@
 		mdp_wrot1 = &mdp_wrot1;
 	};
 
+	cluster0_opp: opp_table0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+		opp-507000000 {
+			opp-hz = /bits/ 64 <507000000>;
+			opp-microvolt = <859000>;
+		};
+		opp-702000000 {
+			opp-hz = /bits/ 64 <702000000>;
+			opp-microvolt = <908000>;
+		};
+		opp-1001000000 {
+			opp-hz = /bits/ 64 <1001000000>;
+			opp-microvolt = <983000>;
+		};
+		opp-1105000000 {
+			opp-hz = /bits/ 64 <1105000000>;
+			opp-microvolt = <1009000>;
+		};
+		opp-1209000000 {
+			opp-hz = /bits/ 64 <1209000000>;
+			opp-microvolt = <1034000>;
+		};
+		opp-1300000000 {
+			opp-hz = /bits/ 64 <1300000000>;
+			opp-microvolt = <1057000>;
+		};
+		opp-1508000000 {
+			opp-hz = /bits/ 64 <1508000000>;
+			opp-microvolt = <1109000>;
+		};
+		opp-1703000000 {
+			opp-hz = /bits/ 64 <1703000000>;
+			opp-microvolt = <1125000>;
+		};
+	};
+
+	cluster1_opp: opp_table1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+		opp-507000000 {
+			opp-hz = /bits/ 64 <507000000>;
+			opp-microvolt = <828000>;
+		};
+		opp-702000000 {
+			opp-hz = /bits/ 64 <702000000>;
+			opp-microvolt = <867000>;
+		};
+		opp-1001000000 {
+			opp-hz = /bits/ 64 <1001000000>;
+			opp-microvolt = <927000>;
+		};
+		opp-1209000000 {
+			opp-hz = /bits/ 64 <1209000000>;
+			opp-microvolt = <968000>;
+		};
+		opp-1404000000 {
+			opp-hz = /bits/ 64 <1404000000>;
+			opp-microvolt = <1007000>;
+		};
+		opp-1612000000 {
+			opp-hz = /bits/ 64 <1612000000>;
+			opp-microvolt = <1049000>;
+		};
+		opp-1807000000 {
+			opp-hz = /bits/ 64 <1807000000>;
+			opp-microvolt = <1089000>;
+		};
+		opp-2106000000 {
+			opp-hz = /bits/ 64 <2106000000>;
+			opp-microvolt = <1125000>;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -81,6 +155,11 @@
 			reg = <0x000>;
 			enable-method = "psci";
 			cpu-idle-states = <&CPU_SLEEP_0>;
+			#cooling-cells = <2>;
+			clocks = <&infracfg CLK_INFRA_CA53SEL>,
+				 <&apmixedsys CLK_APMIXED_MAINPLL>;
+			clock-names = "cpu", "intermediate";
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu1: cpu@1 {
@@ -89,6 +168,10 @@
 			reg = <0x001>;
 			enable-method = "psci";
 			cpu-idle-states = <&CPU_SLEEP_0>;
+			clocks = <&infracfg CLK_INFRA_CA53SEL>,
+				 <&apmixedsys CLK_APMIXED_MAINPLL>;
+			clock-names = "cpu", "intermediate";
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu2: cpu@100 {
@@ -97,6 +180,11 @@
 			reg = <0x100>;
 			enable-method = "psci";
 			cpu-idle-states = <&CPU_SLEEP_0>;
+			#cooling-cells = <2>;
+			clocks = <&infracfg CLK_INFRA_CA57SEL>,
+				 <&apmixedsys CLK_APMIXED_MAINPLL>;
+			clock-names = "cpu", "intermediate";
+			operating-points-v2 = <&cluster1_opp>;
 		};
 
 		cpu3: cpu@101 {
@@ -105,6 +193,10 @@
 			reg = <0x101>;
 			enable-method = "psci";
 			cpu-idle-states = <&CPU_SLEEP_0>;
+			clocks = <&infracfg CLK_INFRA_CA57SEL>,
+				 <&apmixedsys CLK_APMIXED_MAINPLL>;
+			clock-names = "cpu", "intermediate";
+			operating-points-v2 = <&cluster1_opp>;
 		};
 
 		idle-states {
@@ -249,7 +341,7 @@
 			reg = <0 0x10005000 0 0x1000>;
 		};
 
-		pio: pinctrl@0x10005000 {
+		pio: pinctrl@10005000 {
 			compatible = "mediatek,mt8173-pinctrl";
 			reg = <0 0x1000b000 0 0x1000>;
 			mediatek,pctl-regmap = <&syscfg_pctl_a>;
@@ -731,15 +823,9 @@
 			       <&u3port0 PHY_TYPE_USB3>,
 			       <&u2port1 PHY_TYPE_USB2>;
 			power-domains = <&scpsys MT8173_POWER_DOMAIN_USB>;
-			clocks = <&topckgen CLK_TOP_USB30_SEL>,
-				 <&clk26m>,
-				 <&pericfg CLK_PERI_USB0>,
-				 <&pericfg CLK_PERI_USB1>;
-			clock-names = "sys_ck",
-				      "ref_ck",
-				      "wakeup_deb_p0",
-				      "wakeup_deb_p1";
-			mediatek,syscon-wakeup = <&pericfg>;
+			clocks = <&topckgen CLK_TOP_USB30_SEL>, <&clk26m>;
+			clock-names = "sys_ck", "ref_ck";
+			mediatek,syscon-wakeup = <&pericfg 0x400 1>;
 			#address-cells = <2>;
 			#size-cells = <2>;
 			ranges;
diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts b/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts
index 42a23997dcdb..bd5305a634b1 100644
--- a/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra186-p2771-0000.dts
@@ -74,6 +74,43 @@
 		};
 	};
 
+	host1x@13e00000 {
+		status = "okay";
+
+		dpaux@15040000 {
+			status = "okay";
+		};
+
+		display-hub@15200000 {
+			status = "okay";
+		};
+
+		dsi@15300000 {
+			status = "disabled";
+		};
+
+		sor@15540000 {
+			status = "disabled";
+
+			nvidia,dpaux = <&dpaux1>;
+		};
+
+		sor@15580000 {
+			status = "okay";
+
+			avdd-io-supply = <&vdd_hdmi_1v05>;
+			vdd-pll-supply = <&vdd_1v8_ap>;
+			hdmi-supply = <&vdd_hdmi>;
+
+			nvidia,ddc-i2c-bus = <&ddc>;
+			nvidia,hpd-gpio = <&gpio TEGRA_MAIN_GPIO(P, 1) GPIO_ACTIVE_LOW>;
+		};
+
+		dpaux@155c0000 {
+			status = "okay";
+		};
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 
@@ -120,5 +157,19 @@
 
 			vin-supply = <&vdd_3v3_sys>;
 		};
+
+		vdd_hdmi: regulator@101 {
+			compatible = "regulator-fixed";
+			reg = <101>;
+
+			regulator-name = "VDD_HDMI_5V0";
+			regulator-min-microvolt = <5000000>;
+			regulator-max-microvolt = <5000000>;
+
+			gpio = <&exp1 14 GPIO_ACTIVE_HIGH>;
+			enable-active-high;
+
+			vin-supply = <&vdd_5v0_sys>;
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi b/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi
index 54f418d05e15..a8baad7b80df 100644
--- a/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra186-p3310.dtsi
@@ -51,6 +51,10 @@
 		};
 	};
 
+	memory-controller@2c00000 {
+		status = "okay";
+	};
+
 	serial@3100000 {
 		status = "okay";
 	};
@@ -73,7 +77,7 @@
 		status = "okay";
 	};
 
-	i2c@3190000 {
+	ddc: i2c@3190000 {
 		status = "okay";
 	};
 
@@ -88,7 +92,7 @@
 	/* SDMMC1 (SD/MMC) */
 	sdhci@3400000 {
 		cd-gpios = <&gpio TEGRA_MAIN_GPIO(P, 5) GPIO_ACTIVE_LOW>;
-		wp-gpios = <&gpio TEGRA_MAIN_GPIO(P, 4) GPIO_ACTIVE_LOW>;
+		wp-gpios = <&gpio TEGRA_MAIN_GPIO(P, 4) GPIO_ACTIVE_HIGH>;
 
 		vqmmc-supply = <&vddio_sdmmc1>;
 	};
@@ -317,7 +321,7 @@
 						regulator-max-microvolt = <2800000>;
 					};
 
-					avdd_1v05: ldo7 {
+					vdd_hdmi_1v05: ldo7 {
 						regulator-name = "VDD_HDMI_1V05";
 						regulator-min-microvolt = <1050000>;
 						regulator-max-microvolt = <1050000>;
diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
index 46d1f287fb0f..b762227f6aa1 100644
--- a/arch/arm64/boot/dts/nvidia/tegra186.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
@@ -3,6 +3,7 @@
 #include <dt-bindings/gpio/tegra186-gpio.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/tegra186-hsp.h>
+#include <dt-bindings/memory/tegra186-mc.h>
 #include <dt-bindings/power/tegra186-powergate.h>
 #include <dt-bindings/reset/tegra186-reset.h>
 #include <dt-bindings/thermal/tegra186-bpmp-thermal.h>
@@ -13,6 +14,12 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	misc@100000 {
+		compatible = "nvidia,tegra186-misc";
+		reg = <0x0 0x00100000 0x0 0xf000>,
+		      <0x0 0x0010f000 0x0 0x1000>;
+	};
+
 	gpio: gpio@2200000 {
 		compatible = "nvidia,tegra186-gpio";
 		reg-names = "security", "gpio";
@@ -61,6 +68,12 @@
 		snps,rxpbl = <8>;
 	};
 
+	memory-controller@2c00000 {
+		compatible = "nvidia,tegra186-mc";
+		reg = <0x0 0x02c00000 0x0 0xb0000>;
+		status = "disabled";
+	};
+
 	uarta: serial@3100000 {
 		compatible = "nvidia,tegra186-uart", "nvidia,tegra20-uart";
 		reg = <0x0 0x03100000 0x0 0x40>;
@@ -259,6 +272,13 @@
 		status = "disabled";
 	};
 
+	fuse@3820000 {
+		compatible = "nvidia,tegra186-efuse";
+		reg = <0x0 0x03820000 0x0 0x10000>;
+		clocks = <&bpmp TEGRA186_CLK_FUSE>;
+		clock-names = "fuse";
+	};
+
 	gic: interrupt-controller@3881000 {
 		compatible = "arm,gic-400";
 		#interrupt-cells = <3>;
@@ -437,6 +457,79 @@
 		};
 	};
 
+	smmu: iommu@12000000 {
+		compatible = "arm,mmu-500";
+		reg = <0 0x12000000 0 0x800000>;
+		interrupts = <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>;
+		stream-match-mask = <0x7f80>;
+		#global-interrupts = <1>;
+		#iommu-cells = <1>;
+	};
+
 	host1x@13e00000 {
 		compatible = "nvidia,tegra186-host1x", "simple-bus";
 		reg = <0x0 0x13e00000 0x0 0x10000>,
@@ -453,6 +546,129 @@
 		#size-cells = <1>;
 
 		ranges = <0x15000000 0x0 0x15000000 0x01000000>;
+		iommus = <&smmu TEGRA186_SID_HOST1X>;
+
+		dpaux1: dpaux@15040000 {
+			compatible = "nvidia,tegra186-dpaux";
+			reg = <0x15040000 0x10000>;
+			interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_DPAUX1>,
+				 <&bpmp TEGRA186_CLK_PLLDP>;
+			clock-names = "dpaux", "parent";
+			resets = <&bpmp TEGRA186_RESET_DPAUX1>;
+			reset-names = "dpaux";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+
+			state_dpaux1_aux: pinmux-aux {
+				groups = "dpaux-io";
+				function = "aux";
+			};
+
+			state_dpaux1_i2c: pinmux-i2c {
+				groups = "dpaux-io";
+				function = "i2c";
+			};
+
+			state_dpaux1_off: pinmux-off {
+				groups = "dpaux-io";
+				function = "off";
+			};
+
+			i2c-bus {
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		display-hub@15200000 {
+			compatible = "nvidia,tegra186-display", "simple-bus";
+			resets = <&bpmp TEGRA186_RESET_NVDISPLAY0_MISC>,
+				 <&bpmp TEGRA186_RESET_NVDISPLAY0_WGRP0>,
+				 <&bpmp TEGRA186_RESET_NVDISPLAY0_WGRP1>,
+				 <&bpmp TEGRA186_RESET_NVDISPLAY0_WGRP2>,
+				 <&bpmp TEGRA186_RESET_NVDISPLAY0_WGRP3>,
+				 <&bpmp TEGRA186_RESET_NVDISPLAY0_WGRP4>,
+				 <&bpmp TEGRA186_RESET_NVDISPLAY0_WGRP5>;
+			reset-names = "misc", "wgrp0", "wgrp1", "wgrp2",
+				      "wgrp3", "wgrp4", "wgrp5";
+			clocks = <&bpmp TEGRA186_CLK_NVDISPLAY_DISP>,
+				 <&bpmp TEGRA186_CLK_NVDISPLAY_DSC>,
+				 <&bpmp TEGRA186_CLK_NVDISPLAYHUB>;
+			clock-names = "disp", "dsc", "hub";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			ranges = <0x15200000 0x15200000 0x40000>;
+
+			display@15200000 {
+				compatible = "nvidia,tegra186-dc";
+				reg = <0x15200000 0x10000>;
+				interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&bpmp TEGRA186_CLK_NVDISPLAY_P0>;
+				clock-names = "dc";
+				resets = <&bpmp TEGRA186_RESET_NVDISPLAY0_HEAD0>;
+				reset-names = "dc";
+
+				power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+				iommus = <&smmu TEGRA186_SID_NVDISPLAY>;
+
+				nvidia,outputs = <&dsia &dsib &sor0 &sor1>;
+				nvidia,head = <0>;
+			};
+
+			display@15210000 {
+				compatible = "nvidia,tegra186-dc";
+				reg = <0x15210000 0x10000>;
+				interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&bpmp TEGRA186_CLK_NVDISPLAY_P1>;
+				clock-names = "dc";
+				resets = <&bpmp TEGRA186_RESET_NVDISPLAY0_HEAD1>;
+				reset-names = "dc";
+
+				power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISPB>;
+				iommus = <&smmu TEGRA186_SID_NVDISPLAY>;
+
+				nvidia,outputs = <&dsia &dsib &sor0 &sor1>;
+				nvidia,head = <1>;
+			};
+
+			display@15220000 {
+				compatible = "nvidia,tegra186-dc";
+				reg = <0x15220000 0x10000>;
+				interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&bpmp TEGRA186_CLK_NVDISPLAY_P2>;
+				clock-names = "dc";
+				resets = <&bpmp TEGRA186_RESET_NVDISPLAY0_HEAD2>;
+				reset-names = "dc";
+
+				power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISPC>;
+				iommus = <&smmu TEGRA186_SID_NVDISPLAY>;
+
+				nvidia,outputs = <&sor0 &sor1>;
+				nvidia,head = <2>;
+			};
+		};
+
+		dsia: dsi@15300000 {
+			compatible = "nvidia,tegra186-dsi";
+			reg = <0x15300000 0x10000>;
+			interrupts = <GIC_SPI 20 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_DSI>,
+				 <&bpmp TEGRA186_CLK_DSIA_LP>,
+				 <&bpmp TEGRA186_CLK_PLLD>;
+			clock-names = "dsi", "lp", "parent";
+			resets = <&bpmp TEGRA186_RESET_DSI>;
+			reset-names = "dsi";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+		};
 
 		vic@15340000 {
 			compatible = "nvidia,tegra186-vic";
@@ -465,6 +681,141 @@
 
 			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_VIC>;
 		};
+
+		dsib: dsi@15400000 {
+			compatible = "nvidia,tegra186-dsi";
+			reg = <0x15400000 0x10000>;
+			interrupts = <GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_DSIB>,
+				 <&bpmp TEGRA186_CLK_DSIB_LP>,
+				 <&bpmp TEGRA186_CLK_PLLD>;
+			clock-names = "dsi", "lp", "parent";
+			resets = <&bpmp TEGRA186_RESET_DSIB>;
+			reset-names = "dsi";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+		};
+
+		sor0: sor@15540000 {
+			compatible = "nvidia,tegra186-sor";
+			reg = <0x15540000 0x10000>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_SOR0>,
+				 <&bpmp TEGRA186_CLK_SOR0_OUT>,
+				 <&bpmp TEGRA186_CLK_PLLD2>,
+				 <&bpmp TEGRA186_CLK_PLLDP>,
+				 <&bpmp TEGRA186_CLK_SOR_SAFE>,
+				 <&bpmp TEGRA186_CLK_SOR0_PAD_CLKOUT>;
+			clock-names = "sor", "out", "parent", "dp", "safe",
+				      "pad";
+			resets = <&bpmp TEGRA186_RESET_SOR0>;
+			reset-names = "sor";
+			pinctrl-0 = <&state_dpaux_aux>;
+			pinctrl-1 = <&state_dpaux_i2c>;
+			pinctrl-2 = <&state_dpaux_off>;
+			pinctrl-names = "aux", "i2c", "off";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+			nvidia,interface = <0>;
+		};
+
+		sor1: sor@15580000 {
+			compatible = "nvidia,tegra186-sor1";
+			reg = <0x15580000 0x10000>;
+			interrupts = <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_SOR1>,
+				 <&bpmp TEGRA186_CLK_SOR1_OUT>,
+				 <&bpmp TEGRA186_CLK_PLLD3>,
+				 <&bpmp TEGRA186_CLK_PLLDP>,
+				 <&bpmp TEGRA186_CLK_SOR_SAFE>,
+				 <&bpmp TEGRA186_CLK_SOR1_PAD_CLKOUT>;
+			clock-names = "sor", "out", "parent", "dp", "safe",
+				      "pad";
+			resets = <&bpmp TEGRA186_RESET_SOR1>;
+			reset-names = "sor";
+			pinctrl-0 = <&state_dpaux1_aux>;
+			pinctrl-1 = <&state_dpaux1_i2c>;
+			pinctrl-2 = <&state_dpaux1_off>;
+			pinctrl-names = "aux", "i2c", "off";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+			nvidia,interface = <1>;
+		};
+
+		dpaux: dpaux@155c0000 {
+			compatible = "nvidia,tegra186-dpaux";
+			reg = <0x155c0000 0x10000>;
+			interrupts = <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_DPAUX>,
+				 <&bpmp TEGRA186_CLK_PLLDP>;
+			clock-names = "dpaux", "parent";
+			resets = <&bpmp TEGRA186_RESET_DPAUX>;
+			reset-names = "dpaux";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+
+			state_dpaux_aux: pinmux-aux {
+				groups = "dpaux-io";
+				function = "aux";
+			};
+
+			state_dpaux_i2c: pinmux-i2c {
+				groups = "dpaux-io";
+				function = "i2c";
+			};
+
+			state_dpaux_off: pinmux-off {
+				groups = "dpaux-io";
+				function = "off";
+			};
+
+			i2c-bus {
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		padctl@15880000 {
+			compatible = "nvidia,tegra186-dsi-padctl";
+			reg = <0x15880000 0x10000>;
+			resets = <&bpmp TEGRA186_RESET_DSI>;
+			reset-names = "dsi";
+			status = "disabled";
+		};
+
+		dsic: dsi@15900000 {
+			compatible = "nvidia,tegra186-dsi";
+			reg = <0x15900000 0x10000>;
+			interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_DSIC>,
+				 <&bpmp TEGRA186_CLK_DSIC_LP>,
+				 <&bpmp TEGRA186_CLK_PLLD>;
+			clock-names = "dsi", "lp", "parent";
+			resets = <&bpmp TEGRA186_RESET_DSIC>;
+			reset-names = "dsi";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+		};
+
+		dsid: dsi@15940000 {
+			compatible = "nvidia,tegra186-dsi";
+			reg = <0x15940000 0x10000>;
+			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&bpmp TEGRA186_CLK_DSID>,
+				 <&bpmp TEGRA186_CLK_DSID_LP>,
+				 <&bpmp TEGRA186_CLK_PLLD>;
+			clock-names = "dsi", "lp", "parent";
+			resets = <&bpmp TEGRA186_RESET_DSID>;
+			reset-names = "dsi";
+			status = "disabled";
+
+			power-domains = <&bpmp TEGRA186_POWER_DOMAIN_DISP>;
+		};
 	};
 
 	gpu@17000000 {
diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi
index d10d4430537a..212e6634c9ba 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210-p2180.dtsi
@@ -297,6 +297,29 @@
 		};
 	};
 
+	cpus {
+		cpu@0 {
+			enable-method = "psci";
+		};
+
+		cpu@1 {
+			enable-method = "psci";
+		};
+
+		cpu@2 {
+			enable-method = "psci";
+		};
+
+		cpu@3 {
+			enable-method = "psci";
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
 	regulators {
 		vdd_gpu: regulator@100 {
 			compatible = "pwm-regulator";
diff --git a/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi b/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi
index be6066ff97c9..d0dc03923723 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210-p2530.dtsi
@@ -52,4 +52,27 @@
 			clock-frequency = <32768>;
 		};
 	};
+
+	cpus {
+		cpu@0 {
+			enable-method = "psci";
+		};
+
+		cpu@1 {
+			enable-method = "psci";
+		};
+
+		cpu@2 {
+			enable-method = "psci";
+		};
+
+		cpu@3 {
+			enable-method = "psci";
+		};
+	};
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra210.dtsi b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
index 9bdf19f2cca7..9c2402108772 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra210.dtsi
@@ -266,11 +266,11 @@
 			reg = <0x0 0x54580000 0x0 0x00040000>;
 			interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&tegra_car TEGRA210_CLK_SOR1>,
-				 <&tegra_car TEGRA210_CLK_SOR1_SRC>,
+				 <&tegra_car TEGRA210_CLK_SOR1_OUT>,
 				 <&tegra_car TEGRA210_CLK_PLL_D2_OUT0>,
 				 <&tegra_car TEGRA210_CLK_PLL_DP>,
 				 <&tegra_car TEGRA210_CLK_SOR_SAFE>;
-			clock-names = "sor", "source", "parent", "dp", "safe";
+			clock-names = "sor", "out", "parent", "dp", "safe";
 			resets = <&tegra_car 183>;
 			reset-names = "sor";
 			pinctrl-0 = <&state_dpaux1_aux>;
diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
index 33a3297eb284..9ff848792712 100644
--- a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi
@@ -324,6 +324,7 @@
 				label = "apq8016-sbc:green:user4";
 				gpios = <&pm8916_gpios 2 GPIO_ACTIVE_HIGH>;
 				linux,default-trigger = "none";
+				panic-indicator;
 				default-state = "off";
 			};
 
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
index 492a011f14f6..1c8f1b86472d 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dtsi
@@ -140,16 +140,16 @@
 		};
 
 		agnoc@0 {
-			qcom,pcie@00600000 {
+			qcom,pcie@600000 {
 				perst-gpio = <&msmgpio 35 GPIO_ACTIVE_LOW>;
 			};
 
-			qcom,pcie@00608000 {
+			qcom,pcie@608000 {
 				status = "okay";
 				perst-gpio = <&msmgpio 130 GPIO_ACTIVE_LOW>;
 			};
 
-			qcom,pcie@00610000 {
+			qcom,pcie@610000 {
 				status = "okay";
 				perst-gpio = <&msmgpio 114 GPIO_ACTIVE_LOW>;
 			};
diff --git a/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi b/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi
index 4cb0b5834143..390a2fa28514 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi
@@ -278,7 +278,7 @@
 		pinconf {
 			pins = "gpio6", "gpio7";
 			drive-strength = <16>;
-			bias-disable = <0>;
+			bias-disable;
 		};
 	};
 
@@ -290,7 +290,7 @@
 		pinconf {
 			pins = "gpio6", "gpio7";
 			drive-strength = <2>;
-			bias-disable = <0>;
+			bias-disable;
 		};
 	};
 
@@ -302,7 +302,7 @@
 		pinconf {
 			pins = "gpio14", "gpio15";
 			drive-strength = <16>;
-			bias-disable = <0>;
+			bias-disable;
 		};
 	};
 
@@ -314,7 +314,7 @@
 		pinconf {
 			pins = "gpio14", "gpio15";
 			drive-strength = <2>;
-			bias-disable = <0>;
+			bias-disable;
 		};
 	};
 
@@ -326,7 +326,7 @@
 		pinconf {
 			pins = "gpio22", "gpio23";
 			drive-strength = <16>;
-			bias-disable = <0>;
+			bias-disable;
 		};
 	};
 
@@ -338,32 +338,7 @@
 		pinconf {
 			pins = "gpio22", "gpio23";
 			drive-strength = <2>;
-			bias-disable = <0>;
-		};
-	};
-
-	sdhc2_cd_pin {
-		sdc2_cd_on: cd_on {
-			pinmux {
-				function = "gpio";
-				pins = "gpio38";
-			};
-			pinconf {
-				pins = "gpio38";
-				drive-strength = <2>;
-				bias-pull-up;
-			};
-		};
-		sdc2_cd_off: cd_off {
-			pinmux {
-				function = "gpio";
-				pins = "gpio38";
-			};
-			pinconf {
-				pins = "gpio38";
-				drive-strength = <2>;
-				bias-disable;
-			};
+			bias-disable;
 		};
 	};
 
@@ -505,26 +480,25 @@
 		};
 	};
 
-	ext-codec-lines {
-		ext_codec_lines_act: lines_on {
+	pmx_sdc2_cd_pin {
+		sdc2_cd_on: cd_on {
 			pinmux {
 				function = "gpio";
-				pins = "gpio67";
+				pins = "gpio38";
 			};
 			pinconf {
-				pins = "gpio67";
-				drive-strength = <8>;
-				bias-disable;
-				output-high;
+				pins = "gpio38";
+				drive-strength = <2>;
+				bias-pull-up;
 			};
 		};
-		ext_codec_lines_sus: lines_off {
+		sdc2_cd_off: cd_off {
 			pinmux {
 				function = "gpio";
-				pins = "gpio67";
+				pins = "gpio38";
 			};
 			pinconf {
-				pins = "gpio67";
+				pins = "gpio38";
 				drive-strength = <2>;
 				bias-disable;
 			};
@@ -687,34 +661,16 @@
 			};
 		};
 		cdc_dmic_lines_sus: dmic_lines_off {
-			pinconf {
-				pins = "gpio0", "gpio1";
-				drive-strength = <2>;
-				bias-disable;
-			};
-		};
-	};
-
-	cross-conn-det {
-		cross_conn_det_act: lines_on {
-			pinmux {
-				function = "gpio";
-				pins = "gpio120";
-			};
-			pinconf {
-				pins = "gpio120";
-				drive-strength = <8>;
-				output-low;
-				bias-pull-down;
+			pinmux_dmic0_clk {
+				function = "dmic0_clk";
+				pins = "gpio0";
 			};
-		};
-		cross_conn_det_sus: lines_off {
-			pinmux {
-				function = "gpio";
-				pins = "gpio120";
+			pinmux_dmic0_data {
+				function = "dmic0_data";
+				pins = "gpio1";
 			};
 			pinconf {
-				pins = "gpio120";
+				pins = "gpio0", "gpio1";
 				drive-strength = <2>;
 				bias-disable;
 			};
diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi
index 6b2127a6ced1..e51b04900726 100644
--- a/arch/arm64/boot/dts/qcom/msm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi
@@ -355,7 +355,7 @@
 
 		blsp_spi1: spi@78b5000 {
 			compatible = "qcom,spi-qup-v2.2.1";
-			reg = <0x078b5000 0x600>;
+			reg = <0x078b5000 0x500>;
 			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_QUP1_SPI_APPS_CLK>,
 				 <&gcc GCC_BLSP1_AHB_CLK>;
@@ -372,7 +372,7 @@
 
 		blsp_spi2: spi@78b6000 {
 			compatible = "qcom,spi-qup-v2.2.1";
-			reg = <0x078b6000 0x600>;
+			reg = <0x078b6000 0x500>;
 			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_QUP2_SPI_APPS_CLK>,
 				 <&gcc GCC_BLSP1_AHB_CLK>;
@@ -389,7 +389,7 @@
 
 		blsp_spi3: spi@78b7000 {
 			compatible = "qcom,spi-qup-v2.2.1";
-			reg = <0x078b7000 0x600>;
+			reg = <0x078b7000 0x500>;
 			interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_QUP3_SPI_APPS_CLK>,
 				 <&gcc GCC_BLSP1_AHB_CLK>;
@@ -406,7 +406,7 @@
 
 		blsp_spi4: spi@78b8000 {
 			compatible = "qcom,spi-qup-v2.2.1";
-			reg = <0x078b8000 0x600>;
+			reg = <0x078b8000 0x500>;
 			interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_QUP4_SPI_APPS_CLK>,
 				 <&gcc GCC_BLSP1_AHB_CLK>;
@@ -423,7 +423,7 @@
 
 		blsp_spi5: spi@78b9000 {
 			compatible = "qcom,spi-qup-v2.2.1";
-			reg = <0x078b9000 0x600>;
+			reg = <0x078b9000 0x500>;
 			interrupts = <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_QUP5_SPI_APPS_CLK>,
 				 <&gcc GCC_BLSP1_AHB_CLK>;
@@ -440,7 +440,7 @@
 
 		blsp_spi6: spi@78ba000 {
 			compatible = "qcom,spi-qup-v2.2.1";
-			reg = <0x078ba000 0x600>;
+			reg = <0x078ba000 0x500>;
 			interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&gcc GCC_BLSP1_QUP6_SPI_APPS_CLK>,
 				 <&gcc GCC_BLSP1_AHB_CLK>;
@@ -457,10 +457,10 @@
 
 		blsp_i2c2: i2c@78b6000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
-			reg = <0x78b6000 0x1000>;
+			reg = <0x078b6000 0x500>;
 			interrupts = <GIC_SPI 96 0>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
-				<&gcc GCC_BLSP1_QUP2_I2C_APPS_CLK>;
+				 <&gcc GCC_BLSP1_QUP2_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
 			pinctrl-names = "default", "sleep";
 			pinctrl-0 = <&i2c2_default>;
@@ -472,10 +472,10 @@
 
 		blsp_i2c4: i2c@78b8000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
-			reg = <0x78b8000 0x1000>;
+			reg = <0x078b8000 0x500>;
 			interrupts = <GIC_SPI 98 0>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
-				<&gcc GCC_BLSP1_QUP4_I2C_APPS_CLK>;
+				 <&gcc GCC_BLSP1_QUP4_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
 			pinctrl-names = "default", "sleep";
 			pinctrl-0 = <&i2c4_default>;
@@ -487,10 +487,10 @@
 
 		blsp_i2c6: i2c@78ba000 {
 			compatible = "qcom,i2c-qup-v2.2.1";
-			reg = <0x78ba000 0x1000>;
+			reg = <0x078ba000 0x500>;
 			interrupts = <GIC_SPI 100 0>;
 			clocks = <&gcc GCC_BLSP1_AHB_CLK>,
-				<&gcc GCC_BLSP1_QUP6_I2C_APPS_CLK>;
+				 <&gcc GCC_BLSP1_QUP6_I2C_APPS_CLK>;
 			clock-names = "iface", "core";
 			pinctrl-names = "default", "sleep";
 			pinctrl-0 = <&i2c6_default>;
@@ -906,6 +906,7 @@
 					    "dsi_phy_regulator";
 
 				#clock-cells = <1>;
+				#phy-cells = <0>;
 
 				clocks = <&gcc GCC_MDSS_AHB_CLK>;
 				clock-names = "iface_clk";
@@ -1435,8 +1436,8 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		qcom,ipc-1 = <&apcs 0 13>;
-		qcom,ipc-6 = <&apcs 0 19>;
+		qcom,ipc-1 = <&apcs 8 13>;
+		qcom,ipc-3 = <&apcs 8 19>;
 
 		apps_smsm: apps@0 {
 			reg = <0>;
diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi
index 4b2afcc4fdf4..0a6f7952bbb1 100644
--- a/arch/arm64/boot/dts/qcom/msm8996.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi
@@ -840,7 +840,7 @@
 			#size-cells = <1>;
 			ranges;
 
-			pcie0: qcom,pcie@00600000 {
+			pcie0: qcom,pcie@600000 {
 				compatible = "qcom,pcie-msm8996", "snps,dw-pcie";
 				status = "disabled";
 				power-domains = <&gcc PCIE0_GDSC>;
@@ -893,7 +893,7 @@
 
 			};
 
-			pcie1: qcom,pcie@00608000 {
+			pcie1: qcom,pcie@608000 {
 				compatible = "qcom,pcie-msm8996", "snps,dw-pcie";
 				power-domains = <&gcc PCIE1_GDSC>;
 				bus-range = <0x00 0xff>;
@@ -946,7 +946,7 @@
 						"bus_slave";
 			};
 
-			pcie2: qcom,pcie@00610000 {
+			pcie2: qcom,pcie@610000 {
 				compatible = "qcom,pcie-msm8996", "snps,dw-pcie";
 				power-domains = <&gcc PCIE2_GDSC>;
 				bus-range = <0x00 0xff>;
diff --git a/arch/arm64/boot/dts/qcom/pm8916.dtsi b/arch/arm64/boot/dts/qcom/pm8916.dtsi
index 0223e60d8b6a..196b1c0ceb9b 100644
--- a/arch/arm64/boot/dts/qcom/pm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8916.dtsi
@@ -97,47 +97,45 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-                wcd_codec: codec@f000 {
-                       compatible = "qcom,pm8916-wcd-analog-codec";
-	               reg = <0xf000 0x200>;
-	               reg-names = "pmic-codec-core";
-	               clocks = <&gcc GCC_CODEC_DIGCODEC_CLK>;
-	               clock-names = "mclk";
-	               interrupt-parent = <&spmi_bus>;
-	               interrupts = <0x1 0xf0 0x0 IRQ_TYPE_NONE>,
-	                            <0x1 0xf0 0x1 IRQ_TYPE_NONE>,
-	                            <0x1 0xf0 0x2 IRQ_TYPE_NONE>,
-	                            <0x1 0xf0 0x3 IRQ_TYPE_NONE>,
-	                            <0x1 0xf0 0x4 IRQ_TYPE_NONE>,
-	                            <0x1 0xf0 0x5 IRQ_TYPE_NONE>,
-	                            <0x1 0xf0 0x6 IRQ_TYPE_NONE>,
-	                            <0x1 0xf0 0x7 IRQ_TYPE_NONE>,
-	                            <0x1 0xf1 0x0 IRQ_TYPE_NONE>,
-	                            <0x1 0xf1 0x1 IRQ_TYPE_NONE>,
-	                            <0x1 0xf1 0x2 IRQ_TYPE_NONE>,
-	                            <0x1 0xf1 0x3 IRQ_TYPE_NONE>,
-	                            <0x1 0xf1 0x4 IRQ_TYPE_NONE>,
-	                            <0x1 0xf1 0x5 IRQ_TYPE_NONE>;
-	               interrupt-names = "cdc_spk_cnp_int",
-	                                 "cdc_spk_clip_int",
-	                                 "cdc_spk_ocp_int",
-	                                 "mbhc_ins_rem_det1",
-	                                 "mbhc_but_rel_det",
-	                                 "mbhc_but_press_det",
-	                                 "mbhc_ins_rem_det",
-	                                 "mbhc_switch_int",
-	                                 "cdc_ear_ocp_int",
-	                                 "cdc_hphr_ocp_int",
-	                                 "cdc_hphl_ocp_det",
-	                                 "cdc_ear_cnp_int",
-	                                 "cdc_hphr_cnp_int",
-	                                 "cdc_hphl_cnp_int";
-	               vdd-cdc-io-supply = <&pm8916_l5>;
-	               vdd-cdc-tx-rx-cx-supply = <&pm8916_l5>;
-	               vdd-micbias-supply = <&pm8916_l13>;
-	               #sound-dai-cells = <1>;
-
-                };
-
+		wcd_codec: codec@f000 {
+			compatible = "qcom,pm8916-wcd-analog-codec";
+			reg = <0xf000 0x200>;
+			reg-names = "pmic-codec-core";
+			clocks = <&gcc GCC_CODEC_DIGCODEC_CLK>;
+			clock-names = "mclk";
+			interrupt-parent = <&spmi_bus>;
+			interrupts = <0x1 0xf0 0x0 IRQ_TYPE_NONE>,
+				     <0x1 0xf0 0x1 IRQ_TYPE_NONE>,
+				     <0x1 0xf0 0x2 IRQ_TYPE_NONE>,
+				     <0x1 0xf0 0x3 IRQ_TYPE_NONE>,
+				     <0x1 0xf0 0x4 IRQ_TYPE_NONE>,
+				     <0x1 0xf0 0x5 IRQ_TYPE_NONE>,
+				     <0x1 0xf0 0x6 IRQ_TYPE_NONE>,
+				     <0x1 0xf0 0x7 IRQ_TYPE_NONE>,
+				     <0x1 0xf1 0x0 IRQ_TYPE_NONE>,
+				     <0x1 0xf1 0x1 IRQ_TYPE_NONE>,
+				     <0x1 0xf1 0x2 IRQ_TYPE_NONE>,
+				     <0x1 0xf1 0x3 IRQ_TYPE_NONE>,
+				     <0x1 0xf1 0x4 IRQ_TYPE_NONE>,
+				     <0x1 0xf1 0x5 IRQ_TYPE_NONE>;
+			interrupt-names = "cdc_spk_cnp_int",
+					  "cdc_spk_clip_int",
+					  "cdc_spk_ocp_int",
+					  "mbhc_ins_rem_det1",
+					  "mbhc_but_rel_det",
+					  "mbhc_but_press_det",
+					  "mbhc_ins_rem_det",
+					  "mbhc_switch_int",
+					  "cdc_ear_ocp_int",
+					  "cdc_hphr_ocp_int",
+					  "cdc_hphl_ocp_det",
+					  "cdc_ear_cnp_int",
+					  "cdc_hphr_cnp_int",
+					  "cdc_hphl_cnp_int";
+			vdd-cdc-io-supply = <&pm8916_l5>;
+			vdd-cdc-tx-rx-cx-supply = <&pm8916_l5>;
+			vdd-micbias-supply = <&pm8916_l13>;
+			#sound-dai-cells = <1>;
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile
index 646198d82903..2186d0193b73 100644
--- a/arch/arm64/boot/dts/renesas/Makefile
+++ b/arch/arm64/boot/dts/renesas/Makefile
@@ -6,5 +6,6 @@ dtb-$(CONFIG_ARCH_R8A7795) += r8a7795-es1-salvator-x.dtb r8a7795-es1-h3ulcb.dtb
 dtb-$(CONFIG_ARCH_R8A7795) += r8a7795-es1-h3ulcb-kf.dtb
 dtb-$(CONFIG_ARCH_R8A7796) += r8a7796-salvator-x.dtb r8a7796-m3ulcb.dtb
 dtb-$(CONFIG_ARCH_R8A7796) += r8a7796-m3ulcb-kf.dtb
-dtb-$(CONFIG_ARCH_R8A77970) += r8a77970-eagle.dtb
+dtb-$(CONFIG_ARCH_R8A7796) += r8a7796-salvator-xs.dtb
+dtb-$(CONFIG_ARCH_R8A77970) += r8a77970-eagle.dtb r8a77970-v3msk.dtb
 dtb-$(CONFIG_ARCH_R8A77995) += r8a77995-draak.dtb
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts
index 3f7d5f51e428..7f2a3d923f21 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-es1-salvator-x.dts
@@ -1,5 +1,5 @@
 /*
- * Device Tree Source for the Salvator-X board
+ * Device Tree Source for the Salvator-X board with R-Car H3 ES1.x
  *
  * Copyright (C) 2015 Renesas Electronics Corp.
  *
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi b/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi
index 655dd30639c5..26769a11a190 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7795-es1.dtsi
@@ -21,6 +21,26 @@
 		status = "disabled";
 	};
 
+	/delete-node/ mmu@febe0000;
+	/delete-node/ mmu@fe980000;
+	/delete-node/ mmu@fd960000;
+	/delete-node/ mmu@fd970000;
+
+	ipmmu_mp1: mmu@ec680000 {
+		compatible = "renesas,ipmmu-r8a7795";
+		reg = <0 0xec680000 0 0x1000>;
+		renesas,ipmmu-main = <&ipmmu_mm 5>;
+		#iommu-cells = <1>;
+	};
+
+	ipmmu_sy: mmu@e7730000 {
+		compatible = "renesas,ipmmu-r8a7795";
+		reg = <0 0xe7730000 0 0x1000>;
+		renesas,ipmmu-main = <&ipmmu_mm 8>;
+		#iommu-cells = <1>;
+		status = "disabled";
+	};
+
 	/delete-node/ usb-phy@ee0e0200;
 	/delete-node/ usb@ee0e0100;
 	/delete-node/ usb@ee0e0000;
@@ -35,6 +55,7 @@
 		clocks = <&cpg CPG_MOD 613>;
 		power-domains = <&sysc R8A7795_PD_A3VP>;
 		resets = <&cpg 613>;
+		iommus = <&ipmmu_vp0 2>;
 	};
 
 	vspi2: vsp@fe9c0000 {
@@ -54,6 +75,7 @@
 		clocks = <&cpg CPG_MOD 609>;
 		power-domains = <&sysc R8A7795_PD_A3VP>;
 		resets = <&cpg 609>;
+		iommus = <&ipmmu_vp0 10>;
 	};
 
 	vspd3: vsp@fea38000 {
@@ -73,6 +95,7 @@
 		clocks = <&cpg CPG_MOD 600>;
 		power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 		resets = <&cpg 600>;
+		iommus = <&ipmmu_vi0 11>;
 	};
 
 	fdp1@fe948000 {
@@ -86,6 +109,68 @@
 	};
 };
 
+&gpio1 {
+	gpio-ranges = <&pfc 0 32 28>;
+};
+
+&ipmmu_vi0 {
+	renesas,ipmmu-main = <&ipmmu_mm 11>;
+};
+
+&ipmmu_vp0 {
+	renesas,ipmmu-main = <&ipmmu_mm 12>;
+};
+
+&ipmmu_vc0 {
+	renesas,ipmmu-main = <&ipmmu_mm 9>;
+};
+
+&ipmmu_vc1 {
+	renesas,ipmmu-main = <&ipmmu_mm 10>;
+};
+
+&ipmmu_rt {
+	renesas,ipmmu-main = <&ipmmu_mm 7>;
+};
+
+&audma0 {
+	iommus = <&ipmmu_mp1 0>, <&ipmmu_mp1 1>,
+	       <&ipmmu_mp1 2>, <&ipmmu_mp1 3>,
+	       <&ipmmu_mp1 4>, <&ipmmu_mp1 5>,
+	       <&ipmmu_mp1 6>, <&ipmmu_mp1 7>,
+	       <&ipmmu_mp1 8>, <&ipmmu_mp1 9>,
+	       <&ipmmu_mp1 10>, <&ipmmu_mp1 11>,
+	       <&ipmmu_mp1 12>, <&ipmmu_mp1 13>,
+	       <&ipmmu_mp1 14>, <&ipmmu_mp1 15>;
+};
+
+&audma1 {
+	iommus = <&ipmmu_mp1 16>, <&ipmmu_mp1 17>,
+	       <&ipmmu_mp1 18>, <&ipmmu_mp1 19>,
+	       <&ipmmu_mp1 20>, <&ipmmu_mp1 21>,
+	       <&ipmmu_mp1 22>, <&ipmmu_mp1 23>,
+	       <&ipmmu_mp1 24>, <&ipmmu_mp1 25>,
+	       <&ipmmu_mp1 26>, <&ipmmu_mp1 27>,
+	       <&ipmmu_mp1 28>, <&ipmmu_mp1 29>,
+	       <&ipmmu_mp1 30>, <&ipmmu_mp1 31>;
+};
+
+&fcpvb1 {
+	iommus = <&ipmmu_vp0 7>;
+};
+
+&fcpf1 {
+	iommus = <&ipmmu_vp0 1>;
+};
+
+&fcpvi1 {
+	iommus = <&ipmmu_vp0 9>;
+};
+
+&fcpvd2 {
+	iommus = <&ipmmu_vi0 10>;
+};
+
 &du {
 	vsps = <&vspd0 &vspd1 &vspd2 &vspd3>;
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
index 17953070f38d..af467419266a 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-salvator-x.dts
@@ -1,5 +1,5 @@
 /*
- * Device Tree Source for the Salvator-X board
+ * Device Tree Source for the Salvator-X board with R-Car H3 ES2.0
  *
  * Copyright (C) 2015 Renesas Electronics Corp.
  *
diff --git a/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts
index 7675de5d4f2c..8b50ceb746e8 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7795-salvator-xs.dts
@@ -1,5 +1,5 @@
 /*
- * Device Tree Source for the Salvator-X 2nd version board
+ * Device Tree Source for the Salvator-X 2nd version board with R-Car H3 ES2.0
  *
  * Copyright (C) 2015-2017 Renesas Electronics Corp.
  *
diff --git a/arch/arm64/boot/dts/renesas/r8a7795.dtsi b/arch/arm64/boot/dts/renesas/r8a7795.dtsi
index 15ef292a8d9f..d12df6f2ff09 100644
--- a/arch/arm64/boot/dts/renesas/r8a7795.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7795.dtsi
@@ -30,11 +30,6 @@
 		i2c7 = &i2c_dvfs;
 	};
 
-	psci {
-		compatible = "arm,psci-1.0", "arm,psci-0.2";
-		method = "smc";
-	};
-
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -184,6 +179,35 @@
 		clock-frequency = <0>;
 	};
 
+	pmu_a57 {
+		compatible = "arm,cortex-a57-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+		                      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a57_0>,
+				     <&a57_1>,
+				     <&a57_2>,
+				     <&a57_3>;
+	};
+
+	pmu_a53 {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts-extended = <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a53_0>,
+				     <&a53_1>,
+				     <&a53_2>,
+				     <&a53_3>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
+	};
+
 	soc: soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gic>;
@@ -240,7 +264,7 @@
 			interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
 			#gpio-cells = <2>;
 			gpio-controller;
-			gpio-ranges = <&pfc 0 32 28>;
+			gpio-ranges = <&pfc 0 32 29>;
 			#interrupt-cells = <2>;
 			interrupt-controller;
 			clocks = <&cpg CPG_MOD 911>;
@@ -338,42 +362,6 @@
 			resets = <&cpg 905>;
 		};
 
-		pmu_a57 {
-			compatible = "arm,cortex-a57-pmu";
-			interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-affinity = <&a57_0>,
-					     <&a57_1>,
-					     <&a57_2>,
-					     <&a57_3>;
-		};
-
-		pmu_a53 {
-			compatible = "arm,cortex-a53-pmu";
-			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-affinity = <&a53_0>,
-					     <&a53_1>,
-					     <&a53_2>,
-					     <&a53_3>;
-		};
-
-		timer {
-			compatible = "arm,armv8-timer";
-			interrupts = <GIC_PPI 13
-					(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 14
-					(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 11
-					(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 10
-					(GIC_CPU_MASK_SIMPLE(8) | IRQ_TYPE_LEVEL_LOW)>;
-		};
-
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a7795-cpg-mssr";
 			reg = <0 0xe6150000 0 0x1000>;
@@ -421,6 +409,146 @@
 			resets = <&cpg 407>;
 		};
 
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 14>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vi1: mmu@febe0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfebe0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 15>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_vp0: mmu@fe990000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe990000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 16>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_vp1: mmu@fe980000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe980000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 17>;
+			power-domains = <&sysc R8A7795_PD_A3VP>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vc0: mmu@fe6b0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe6b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 12>;
+			power-domains = <&sysc R8A7795_PD_A3VC>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_vc1: mmu@fe6f0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfe6f0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 13>;
+			power-domains = <&sysc R8A7795_PD_A3VC>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_pv0: mmu@fd800000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfd800000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 6>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_pv2: mmu@fd960000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfd960000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 8>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_pv3: mmu@fd970000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xfd970000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 9>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_ir: mmu@ff8b0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xff8b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 3>;
+			power-domains = <&sysc R8A7795_PD_A3IR>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_hc: mmu@e6570000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe6570000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 2>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 10>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_mp0: mmu@ec670000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xec670000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 4>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_ds0: mmu@e6740000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe6740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 0>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 1>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a7795";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
 		dmac0: dma-controller@e6700000 {
 			compatible = "renesas,dmac-r8a7795",
 				     "renesas,rcar-dmac";
@@ -453,6 +581,14 @@
 			resets = <&cpg 219>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_ds0 0>, <&ipmmu_ds0 1>,
+			       <&ipmmu_ds0 2>, <&ipmmu_ds0 3>,
+			       <&ipmmu_ds0 4>, <&ipmmu_ds0 5>,
+			       <&ipmmu_ds0 6>, <&ipmmu_ds0 7>,
+			       <&ipmmu_ds0 8>, <&ipmmu_ds0 9>,
+			       <&ipmmu_ds0 10>, <&ipmmu_ds0 11>,
+			       <&ipmmu_ds0 12>, <&ipmmu_ds0 13>,
+			       <&ipmmu_ds0 14>, <&ipmmu_ds0 15>;
 		};
 
 		dmac1: dma-controller@e7300000 {
@@ -487,6 +623,14 @@
 			resets = <&cpg 218>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_ds1 0>, <&ipmmu_ds1 1>,
+			       <&ipmmu_ds1 2>, <&ipmmu_ds1 3>,
+			       <&ipmmu_ds1 4>, <&ipmmu_ds1 5>,
+			       <&ipmmu_ds1 6>, <&ipmmu_ds1 7>,
+			       <&ipmmu_ds1 8>, <&ipmmu_ds1 9>,
+			       <&ipmmu_ds1 10>, <&ipmmu_ds1 11>,
+			       <&ipmmu_ds1 12>, <&ipmmu_ds1 13>,
+			       <&ipmmu_ds1 14>, <&ipmmu_ds1 15>;
 		};
 
 		dmac2: dma-controller@e7310000 {
@@ -521,6 +665,14 @@
 			resets = <&cpg 217>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_ds1 16>, <&ipmmu_ds1 17>,
+			       <&ipmmu_ds1 18>, <&ipmmu_ds1 19>,
+			       <&ipmmu_ds1 20>, <&ipmmu_ds1 21>,
+			       <&ipmmu_ds1 22>, <&ipmmu_ds1 23>,
+			       <&ipmmu_ds1 24>, <&ipmmu_ds1 25>,
+			       <&ipmmu_ds1 26>, <&ipmmu_ds1 27>,
+			       <&ipmmu_ds1 28>, <&ipmmu_ds1 29>,
+			       <&ipmmu_ds1 30>, <&ipmmu_ds1 31>;
 		};
 
 		audma0: dma-controller@ec700000 {
@@ -555,6 +707,14 @@
 			resets = <&cpg 502>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_mp0 0>, <&ipmmu_mp0 1>,
+			       <&ipmmu_mp0 2>, <&ipmmu_mp0 3>,
+			       <&ipmmu_mp0 4>, <&ipmmu_mp0 5>,
+			       <&ipmmu_mp0 6>, <&ipmmu_mp0 7>,
+			       <&ipmmu_mp0 8>, <&ipmmu_mp0 9>,
+			       <&ipmmu_mp0 10>, <&ipmmu_mp0 11>,
+			       <&ipmmu_mp0 12>, <&ipmmu_mp0 13>,
+			       <&ipmmu_mp0 14>, <&ipmmu_mp0 15>;
 		};
 
 		audma1: dma-controller@ec720000 {
@@ -589,6 +749,14 @@
 			resets = <&cpg 501>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_mp0 16>, <&ipmmu_mp0 17>,
+			       <&ipmmu_mp0 18>, <&ipmmu_mp0 19>,
+			       <&ipmmu_mp0 20>, <&ipmmu_mp0 21>,
+			       <&ipmmu_mp0 22>, <&ipmmu_mp0 23>,
+			       <&ipmmu_mp0 24>, <&ipmmu_mp0 25>,
+			       <&ipmmu_mp0 26>, <&ipmmu_mp0 27>,
+			       <&ipmmu_mp0 28>, <&ipmmu_mp0 29>,
+			       <&ipmmu_mp0 30>, <&ipmmu_mp0 31>;
 		};
 
 		avb: ethernet@e6800000 {
@@ -631,6 +799,7 @@
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 812>;
 			phy-mode = "rgmii-txid";
+			iommus = <&ipmmu_ds0 16>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
@@ -1459,6 +1628,20 @@
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 815>;
 			status = "disabled";
+			iommus = <&ipmmu_hc 2>;
+		};
+
+		usb3_phy0: usb-phy@e65ee000 {
+			compatible = "renesas,r8a7795-usb3-phy",
+				     "renesas,rcar-gen3-usb3-phy";
+			reg = <0 0xe65ee000 0 0x90>;
+			clocks = <&cpg CPG_MOD 328>, <&usb3s0_clk>,
+				 <&usb_extal_clk>;
+			clock-names = "usb3-if", "usb3s_clk", "usb_extal";
+			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			#phy-cells = <0>;
+			status = "disabled";
 		};
 
 		xhci0: usb@ee000000 {
@@ -1539,7 +1722,8 @@
 		};
 
 		sdhi0: sd@ee100000 {
-			compatible = "renesas,sdhi-r8a7795";
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee100000 0 0x2000>;
 			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 314>;
@@ -1550,7 +1734,8 @@
 		};
 
 		sdhi1: sd@ee120000 {
-			compatible = "renesas,sdhi-r8a7795";
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee120000 0 0x2000>;
 			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 313>;
@@ -1561,7 +1746,8 @@
 		};
 
 		sdhi2: sd@ee140000 {
-			compatible = "renesas,sdhi-r8a7795";
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee140000 0 0x2000>;
 			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 312>;
@@ -1572,7 +1758,8 @@
 		};
 
 		sdhi3: sd@ee160000 {
-			compatible = "renesas,sdhi-r8a7795";
+			compatible = "renesas,sdhi-r8a7795",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee160000 0 0x2000>;
 			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 311>;
@@ -1873,6 +2060,7 @@
 			clocks = <&cpg CPG_MOD 606>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 			resets = <&cpg 606>;
+			iommus = <&ipmmu_vp1 7>;
 		};
 
 		fcpf0: fcp@fe950000 {
@@ -1881,6 +2069,7 @@
 			clocks = <&cpg CPG_MOD 615>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 			resets = <&cpg 615>;
+			iommus = <&ipmmu_vp0 0>;
 		};
 
 		fcpf1: fcp@fe951000 {
@@ -1889,6 +2078,7 @@
 			clocks = <&cpg CPG_MOD 614>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 			resets = <&cpg 614>;
+			iommus = <&ipmmu_vp1 1>;
 		};
 
 		vspbd: vsp@fe960000 {
@@ -1908,6 +2098,7 @@
 			clocks = <&cpg CPG_MOD 607>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 			resets = <&cpg 607>;
+			iommus = <&ipmmu_vp0 5>;
 		};
 
 		vspi0: vsp@fe9a0000 {
@@ -1927,6 +2118,7 @@
 			clocks = <&cpg CPG_MOD 611>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 			resets = <&cpg 611>;
+			iommus = <&ipmmu_vp0 8>;
 		};
 
 		vspi1: vsp@fe9b0000 {
@@ -1946,6 +2138,7 @@
 			clocks = <&cpg CPG_MOD 610>;
 			power-domains = <&sysc R8A7795_PD_A3VP>;
 			resets = <&cpg 610>;
+			iommus = <&ipmmu_vp1 9>;
 		};
 
 		vspd0: vsp@fea20000 {
@@ -1965,6 +2158,7 @@
 			clocks = <&cpg CPG_MOD 603>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 603>;
+			iommus = <&ipmmu_vi0 8>;
 		};
 
 		vspd1: vsp@fea28000 {
@@ -1984,6 +2178,7 @@
 			clocks = <&cpg CPG_MOD 602>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 602>;
+			iommus = <&ipmmu_vi0 9>;
 		};
 
 		vspd2: vsp@fea30000 {
@@ -2003,6 +2198,7 @@
 			clocks = <&cpg CPG_MOD 601>;
 			power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
 			resets = <&cpg 601>;
+			iommus = <&ipmmu_vi1 10>;
 		};
 
 		fdp1@fe940000 {
@@ -2136,49 +2332,78 @@
 			#thermal-sensor-cells = <1>;
 			status = "okay";
 		};
+	};
 
-		thermal-zones {
-			sensor_thermal1: sensor-thermal1 {
-				polling-delay-passive = <250>;
-				polling-delay = <1000>;
-				thermal-sensors = <&tsc 0>;
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts-extended = <&gic GIC_PPI 13
+				       (GIC_CPU_MASK_SIMPLE(8) |
+				       IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14
+				       (GIC_CPU_MASK_SIMPLE(8) |
+				       IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11
+				       (GIC_CPU_MASK_SIMPLE(8) |
+				       IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10
+				       (GIC_CPU_MASK_SIMPLE(8) |
+				       IRQ_TYPE_LEVEL_LOW)>;
+	};
 
-				trips {
-					sensor1_crit: sensor1-crit {
-						temperature = <120000>;
-						hysteresis = <2000>;
-						type = "critical";
-					};
+	thermal-zones {
+		sensor_thermal1: sensor-thermal1 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 0>;
+
+			trips {
+				sensor1_crit: sensor1-crit {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "critical";
 				};
 			};
+		};
 
-			sensor_thermal2: sensor-thermal2 {
-				polling-delay-passive = <250>;
-				polling-delay = <1000>;
-				thermal-sensors = <&tsc 1>;
+		sensor_thermal2: sensor-thermal2 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 1>;
 
-				trips {
-					sensor2_crit: sensor2-crit {
-						temperature = <120000>;
-						hysteresis = <2000>;
-						type = "critical";
-					};
+			trips {
+				sensor2_crit: sensor2-crit {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "critical";
 				};
 			};
+		};
 
-			sensor_thermal3: sensor-thermal3 {
-				polling-delay-passive = <250>;
-				polling-delay = <1000>;
-				thermal-sensors = <&tsc 2>;
+		sensor_thermal3: sensor-thermal3 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 2>;
 
-				trips {
-					sensor3_crit: sensor3-crit {
-						temperature = <120000>;
-						hysteresis = <2000>;
-						type = "critical";
-					};
+			trips {
+				sensor3_crit: sensor3-crit {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "critical";
 				};
 			};
 		};
 	};
+
+	/* External USB clocks - can be overridden by the board */
+	usb3s0_clk: usb3s0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	usb_extal_clk: usb_extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
index b317be03306e..498c9e807dc4 100644
--- a/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a7796-salvator-x.dts
@@ -1,5 +1,5 @@
 /*
- * Device Tree Source for the Salvator-X board
+ * Device Tree Source for the Salvator-X board with R-Car M3-W
  *
  * Copyright (C) 2016 Renesas Electronics Corp.
  *
diff --git a/arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts
new file mode 100644
index 000000000000..2c37055efa94
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a7796-salvator-xs.dts
@@ -0,0 +1,58 @@
+/*
+ * Device Tree Source for the Salvator-X 2nd version board with R-Car M3-W
+ *
+ * Copyright (C) 2015-2017 Renesas Electronics Corp.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2.  This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+/dts-v1/;
+#include "r8a7796.dtsi"
+#include "salvator-xs.dtsi"
+
+/ {
+	model = "Renesas Salvator-X 2nd version board based on r8a7796";
+	compatible = "renesas,salvator-xs", "renesas,r8a7796";
+
+	memory@48000000 {
+		device_type = "memory";
+		/* first 128MB is reserved for secure area. */
+		reg = <0x0 0x48000000 0x0 0x78000000>;
+	};
+
+	memory@600000000 {
+		device_type = "memory";
+		reg = <0x6 0x00000000 0x0 0x80000000>;
+	};
+};
+
+&du {
+	clocks = <&cpg CPG_MOD 724>,
+		 <&cpg CPG_MOD 723>,
+		 <&cpg CPG_MOD 722>,
+		 <&cpg CPG_MOD 727>,
+		 <&versaclock6 1>,
+		 <&x21_clk>,
+		 <&versaclock6 2>;
+	clock-names = "du.0", "du.1", "du.2", "lvds.0",
+		      "dclkin.0", "dclkin.1", "dclkin.2";
+};
+
+&hdmi0 {
+	status = "okay";
+
+	ports {
+		port@1 {
+			reg = <1>;
+			rcar_dw_hdmi0_out: endpoint {
+				remote-endpoint = <&hdmi0_con>;
+			};
+		};
+	};
+};
+
+&hdmi0_con {
+	remote-endpoint = <&rcar_dw_hdmi0_out>;
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a7796.dtsi b/arch/arm64/boot/dts/renesas/r8a7796.dtsi
index f2b2e40c655e..c5192d513d7d 100644
--- a/arch/arm64/boot/dts/renesas/r8a7796.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a7796.dtsi
@@ -30,9 +30,34 @@
 		i2c7 = &i2c_dvfs;
 	};
 
-	psci {
-		compatible = "arm,psci-1.0", "arm,psci-0.2";
-		method = "smc";
+	/*
+	 * The external audio clocks are configured as 0 Hz fixed frequency
+	 * clocks by default.
+	 * Boards that provide audio clocks should override them.
+	 */
+	audio_clk_a: audio_clk_a {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_b: audio_clk_b {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	audio_clk_c: audio_clk_c {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	/* External CAN clock - to be overridden by boards that provide it */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
 	};
 
 	cpus {
@@ -122,34 +147,32 @@
 		clock-frequency = <0>;
 	};
 
-	/*
-	 * The external audio clocks are configured as 0 Hz fixed frequency
-	 * clocks by default.
-	 * Boards that provide audio clocks should override them.
-	 */
-	audio_clk_a: audio_clk_a {
+	/* External PCIe clock - can be overridden by the board */
+	pcie_bus_clk: pcie_bus {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <0>;
 	};
 
-	audio_clk_b: audio_clk_b {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
+	pmu_a57 {
+		compatible = "arm,cortex-a57-pmu";
+		interrupts-extended = <&gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a57_0>, <&a57_1>;
 	};
 
-	audio_clk_c: audio_clk_c {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
+	pmu_a53 {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts-extended = <&gic GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>,
+				      <&gic GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-affinity = <&a53_0>, <&a53_1>, <&a53_2>, <&a53_3>;
 	};
 
-	/* External CAN clock - to be overridden by boards that provide it */
-	can_clk: can {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
+	psci {
+		compatible = "arm,psci-1.0", "arm,psci-0.2";
+		method = "smc";
 	};
 
 	/* External SCIF clock - to be overridden by boards that provide it */
@@ -159,13 +182,6 @@
 		clock-frequency = <0>;
 	};
 
-	/* External PCIe clock - can be overridden by the board */
-	pcie_bus_clk: pcie_bus {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <0>;
-	};
-
 	soc {
 		compatible = "simple-bus";
 		interrupt-parent = <&gic>;
@@ -190,18 +206,6 @@
 			resets = <&cpg 408>;
 		};
 
-		timer {
-			compatible = "arm,armv8-timer";
-			interrupts = <GIC_PPI 13
-					(GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 14
-					(GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 11
-					(GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
-				     <GIC_PPI 10
-					(GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>;
-		};
-
 		wdt0: watchdog@e6020000 {
 			compatible = "renesas,r8a7796-wdt",
 				     "renesas,rcar-gen3-wdt";
@@ -337,24 +341,98 @@
 			reg = <0 0xe6060000 0 0x50c>;
 		};
 
-		pmu_a57 {
-			compatible = "arm,cortex-a57-pmu";
-			interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-affinity = <&a57_0>,
-					     <&a57_1>;
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 9>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_vc0: mmu@fe6b0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfe6b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 8>;
+			power-domains = <&sysc R8A7796_PD_A3VC>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_pv0: mmu@fd800000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfd800000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 5>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_pv1: mmu@fd950000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xfd950000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 6>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_ir: mmu@ff8b0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xff8b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 3>;
+			power-domains = <&sysc R8A7796_PD_A3IR>;
+			#iommu-cells = <1>;
+			status = "disabled";
 		};
 
-		pmu_a53 {
-			compatible = "arm,cortex-a53-pmu";
-			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-affinity = <&a53_0>,
-					     <&a53_1>,
-					     <&a53_2>,
-					     <&a53_3>;
+		ipmmu_hc: mmu@e6570000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe6570000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 2>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 7>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_mp: mmu@ec670000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xec670000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 4>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_ds0: mmu@e6740000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe6740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 0>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 1>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a7796";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
 		};
 
 		cpg: clock-controller@e6150000 {
@@ -817,6 +895,7 @@
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
 			resets = <&cpg 812>;
 			phy-mode = "rgmii-txid";
+			iommus = <&ipmmu_ds0 16>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
@@ -1101,6 +1180,14 @@
 			resets = <&cpg 219>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_ds0 0>, <&ipmmu_ds0 1>,
+			       <&ipmmu_ds0 2>, <&ipmmu_ds0 3>,
+			       <&ipmmu_ds0 4>, <&ipmmu_ds0 5>,
+			       <&ipmmu_ds0 6>, <&ipmmu_ds0 7>,
+			       <&ipmmu_ds0 8>, <&ipmmu_ds0 9>,
+			       <&ipmmu_ds0 10>, <&ipmmu_ds0 11>,
+			       <&ipmmu_ds0 12>, <&ipmmu_ds0 13>,
+			       <&ipmmu_ds0 14>, <&ipmmu_ds0 15>;
 		};
 
 		dmac1: dma-controller@e7300000 {
@@ -1135,6 +1222,14 @@
 			resets = <&cpg 218>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_ds1 0>, <&ipmmu_ds1 1>,
+			       <&ipmmu_ds1 2>, <&ipmmu_ds1 3>,
+			       <&ipmmu_ds1 4>, <&ipmmu_ds1 5>,
+			       <&ipmmu_ds1 6>, <&ipmmu_ds1 7>,
+			       <&ipmmu_ds1 8>, <&ipmmu_ds1 9>,
+			       <&ipmmu_ds1 10>, <&ipmmu_ds1 11>,
+			       <&ipmmu_ds1 12>, <&ipmmu_ds1 13>,
+			       <&ipmmu_ds1 14>, <&ipmmu_ds1 15>;
 		};
 
 		dmac2: dma-controller@e7310000 {
@@ -1169,6 +1264,14 @@
 			resets = <&cpg 217>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_ds1 16>, <&ipmmu_ds1 17>,
+			       <&ipmmu_ds1 18>, <&ipmmu_ds1 19>,
+			       <&ipmmu_ds1 20>, <&ipmmu_ds1 21>,
+			       <&ipmmu_ds1 22>, <&ipmmu_ds1 23>,
+			       <&ipmmu_ds1 24>, <&ipmmu_ds1 25>,
+			       <&ipmmu_ds1 26>, <&ipmmu_ds1 27>,
+			       <&ipmmu_ds1 28>, <&ipmmu_ds1 29>,
+			       <&ipmmu_ds1 30>, <&ipmmu_ds1 31>;
 		};
 
 		audma0: dma-controller@ec700000 {
@@ -1203,6 +1306,14 @@
 			resets = <&cpg 502>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_mp 0>, <&ipmmu_mp 1>,
+			       <&ipmmu_mp 2>, <&ipmmu_mp 3>,
+			       <&ipmmu_mp 4>, <&ipmmu_mp 5>,
+			       <&ipmmu_mp 6>, <&ipmmu_mp 7>,
+			       <&ipmmu_mp 8>, <&ipmmu_mp 9>,
+			       <&ipmmu_mp 10>, <&ipmmu_mp 11>,
+			       <&ipmmu_mp 12>, <&ipmmu_mp 13>,
+			       <&ipmmu_mp 14>, <&ipmmu_mp 15>;
 		};
 
 		audma1: dma-controller@ec720000 {
@@ -1237,6 +1348,14 @@
 			resets = <&cpg 501>;
 			#dma-cells = <1>;
 			dma-channels = <16>;
+			iommus = <&ipmmu_mp 16>, <&ipmmu_mp 17>,
+			       <&ipmmu_mp 18>, <&ipmmu_mp 19>,
+			       <&ipmmu_mp 20>, <&ipmmu_mp 21>,
+			       <&ipmmu_mp 22>, <&ipmmu_mp 23>,
+			       <&ipmmu_mp 24>, <&ipmmu_mp 25>,
+			       <&ipmmu_mp 26>, <&ipmmu_mp 27>,
+			       <&ipmmu_mp 28>, <&ipmmu_mp 29>,
+			       <&ipmmu_mp 30>, <&ipmmu_mp 31>;
 		};
 
 		usb_dmac0: dma-controller@e65a0000 {
@@ -1284,6 +1403,19 @@
 			status = "disabled";
 		};
 
+		usb3_phy0: usb-phy@e65ee000 {
+			compatible = "renesas,r8a7796-usb3-phy",
+				     "renesas,rcar-gen3-usb3-phy";
+			reg = <0 0xe65ee000 0 0x90>;
+			clocks = <&cpg CPG_MOD 328>, <&usb3s0_clk>,
+				 <&usb_extal_clk>;
+			clock-names = "usb3-if", "usb3s_clk", "usb_extal";
+			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
+			resets = <&cpg 328>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
 		xhci0: usb@ee000000 {
 			compatible = "renesas,xhci-r8a7796",
 				     "renesas,rcar-gen3-xhci";
@@ -1380,7 +1512,8 @@
 		};
 
 		sdhi0: sd@ee100000 {
-			compatible = "renesas,sdhi-r8a7796";
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee100000 0 0x2000>;
 			interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 314>;
@@ -1391,7 +1524,8 @@
 		};
 
 		sdhi1: sd@ee120000 {
-			compatible = "renesas,sdhi-r8a7796";
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee120000 0 0x2000>;
 			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 313>;
@@ -1402,7 +1536,8 @@
 		};
 
 		sdhi2: sd@ee140000 {
-			compatible = "renesas,sdhi-r8a7796";
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee140000 0 0x2000>;
 			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 312>;
@@ -1413,7 +1548,8 @@
 		};
 
 		sdhi3: sd@ee160000 {
-			compatible = "renesas,sdhi-r8a7796";
+			compatible = "renesas,sdhi-r8a7796",
+				     "renesas,rcar-gen3-sdhi";
 			reg = <0 0xee160000 0 0x2000>;
 			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 311>;
@@ -1438,50 +1574,6 @@
 			status = "okay";
 		};
 
-		thermal-zones {
-			sensor_thermal1: sensor-thermal1 {
-				polling-delay-passive = <250>;
-				polling-delay = <1000>;
-				thermal-sensors = <&tsc 0>;
-
-				trips {
-					sensor1_crit: sensor1-crit {
-						temperature = <120000>;
-						hysteresis = <2000>;
-						type = "critical";
-					};
-				};
-			};
-
-			sensor_thermal2: sensor-thermal2 {
-				polling-delay-passive = <250>;
-				polling-delay = <1000>;
-				thermal-sensors = <&tsc 1>;
-
-				trips {
-					sensor2_crit: sensor2-crit {
-						temperature = <120000>;
-						hysteresis = <2000>;
-						type = "critical";
-					};
-				};
-			};
-
-			sensor_thermal3: sensor-thermal3 {
-				polling-delay-passive = <250>;
-				polling-delay = <1000>;
-				thermal-sensors = <&tsc 2>;
-
-				trips {
-					sensor3_crit: sensor3-crit {
-						temperature = <120000>;
-						hysteresis = <2000>;
-						type = "critical";
-					};
-				};
-			};
-		};
-
 		rcar_sound: sound@ec500000 {
 			/*
 			 * #sound-dai-cells is required
@@ -1679,10 +1771,12 @@
 		};
 
 		pciec0: pcie@fe000000 {
+			reg = <0 0xfe000000 0 0x80000>;
 			/* placeholder */
 		};
 
 		pciec1: pcie@ee800000 {
+			reg = <0 0xee800000 0 0x80000>;
 			/* placeholder */
 		};
 
@@ -1740,6 +1834,7 @@
 			clocks = <&cpg CPG_MOD 611>;
 			power-domains = <&sysc R8A7796_PD_A3VC>;
 			resets = <&cpg 611>;
+			iommus = <&ipmmu_vc0 19>;
 		};
 
 		vspd0: vsp@fea20000 {
@@ -1759,6 +1854,7 @@
 			clocks = <&cpg CPG_MOD 603>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
 			resets = <&cpg 603>;
+			iommus = <&ipmmu_vi0 8>;
 		};
 
 		vspd1: vsp@fea28000 {
@@ -1778,6 +1874,7 @@
 			clocks = <&cpg CPG_MOD 602>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
 			resets = <&cpg 602>;
+			iommus = <&ipmmu_vi0 9>;
 		};
 
 		vspd2: vsp@fea30000 {
@@ -1797,6 +1894,7 @@
 			clocks = <&cpg CPG_MOD 601>;
 			power-domains = <&sysc R8A7796_PD_ALWAYS_ON>;
 			resets = <&cpg 601>;
+			iommus = <&ipmmu_vi0 10>;
 		};
 
 		hdmi0: hdmi@fead0000 {
@@ -1884,4 +1982,69 @@
 			resets = <&cpg 822>;
 		};
 	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts-extended = <&gic GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>,
+				      <&gic GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>;
+	};
+
+	thermal-zones {
+		sensor_thermal1: sensor-thermal1 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 0>;
+
+			trips {
+				sensor1_crit: sensor1-crit {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "critical";
+				};
+			};
+		};
+
+		sensor_thermal2: sensor-thermal2 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 1>;
+
+			trips {
+				sensor2_crit: sensor2-crit {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "critical";
+				};
+			};
+		};
+
+		sensor_thermal3: sensor-thermal3 {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&tsc 2>;
+
+			trips {
+				sensor3_crit: sensor3-crit {
+					temperature = <120000>;
+					hysteresis = <2000>;
+					type = "critical";
+				};
+			};
+		};
+	};
+
+	/* External USB clocks - can be overridden by the board */
+	usb3s0_clk: usb3s0 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
+	usb_extal_clk: usb_extal {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
index a711e77cc6a5..8fe5c193e049 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
@@ -33,6 +33,17 @@
 	};
 };
 
+&avb {
+	renesas,no-ether-link;
+	phy-handle = <&phy0>;
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		rxc-skew-ps = <1500>;
+		reg = <0>;
+	};
+};
+
 &extal_clk {
 	clock-frequency = <16666666>;
 };
@@ -41,17 +52,11 @@
 	clock-frequency = <32768>;
 };
 
-&scif0 {
+&rwdt {
+	timeout-sec = <60>;
 	status = "okay";
 };
 
-&avb {
-	renesas,no-ether-link;
-	phy-handle = <&phy0>;
+&scif0 {
 	status = "okay";
-
-	phy0: ethernet-phy@0 {
-		rxc-skew-ps = <1500>;
-		reg = <0>;
-	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
new file mode 100644
index 000000000000..8624ca87d6b2
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
@@ -0,0 +1,55 @@
+/*
+ * Device Tree Source for the V3M Starter Kit board
+ *
+ * Copyright (C) 2017 Renesas Electronics Corp.
+ * Copyright (C) 2017 Cogent Embedded, Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2.  This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+/dts-v1/;
+#include "r8a77970.dtsi"
+
+/ {
+	model = "Renesas V3M Starter Kit board";
+	compatible = "renesas,v3msk", "renesas,r8a77970";
+
+	aliases {
+		serial0 = &scif0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@48000000 {
+		device_type = "memory";
+		/* first 128MB is reserved for secure area. */
+		reg = <0x0 0x48000000 0x0 0x38000000>;
+	};
+};
+
+&avb {
+	renesas,no-ether-link;
+	phy-handle = <&phy0>;
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		rxc-skew-ps = <1500>;
+		reg = <0>;
+	};
+};
+
+&extal_clk {
+	clock-frequency = <16666666>;
+};
+
+&extalr_clk {
+	clock-frequency = <32768>;
+};
+
+&scif0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77970.dtsi b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
index 97e6981938e7..c35a117fc447 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
@@ -9,9 +9,10 @@
  * kind, whether express or implied.
  */
 
-#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/clock/r8a77970-cpg-mssr.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
-#include <dt-bindings/clock/renesas-cpg-mssr.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/power/r8a77970-sysc.h>
 
 / {
 	compatible = "renesas,r8a77970";
@@ -31,15 +32,15 @@
 			device_type = "cpu";
 			compatible = "arm,cortex-a53", "arm,armv8";
 			reg = <0>;
-			clocks = <&cpg CPG_CORE 0>;
-			power-domains = <&sysc 5>;
+			clocks = <&cpg CPG_CORE R8A77970_CLK_Z2>;
+			power-domains = <&sysc R8A77970_PD_CA53_CPU0>;
 			next-level-cache = <&L2_CA53>;
 			enable-method = "psci";
 		};
 
 		L2_CA53: cache-controller {
 			compatible = "cache";
-			power-domains = <&sysc 21>;
+			power-domains = <&sysc R8A77970_PD_CA53_SCU>;
 			cache-unified;
 			cache-level = <2>;
 		};
@@ -87,7 +88,7 @@
 				      IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&cpg CPG_MOD 408>;
 			clock-names = "clk";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 408>;
 		};
 
@@ -103,6 +104,16 @@
 						  IRQ_TYPE_LEVEL_LOW)>;
 		};
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a77970-wdt",
+				     "renesas,rcar-gen3-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			clocks = <&cpg CPG_MOD 402>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			resets = <&cpg 402>;
+			status = "disabled";
+		};
+
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a77970-cpg-mssr";
 			reg = <0 0xe6150000 0 0x1000>;
@@ -124,6 +135,49 @@
 			#power-domain-cells = <1>;
 		};
 
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 9>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_ir: mmu@ff8b0000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xff8b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 3>;
+			power-domains = <&sysc R8A77970_PD_A3IR>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 7>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 1>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a77970";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
+			#iommu-cells = <1>;
+		};
+
 		intc_ex: interrupt-controller@e61c0000 {
 			compatible = "renesas,intc-ex-r8a77970", "renesas,irqc";
 			#interrupt-cells = <2>;
@@ -136,7 +190,7 @@
 				      GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH
 				      GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 407>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 407>;
 		};
 
@@ -163,10 +217,14 @@
 					  "ch4", "ch5", "ch6", "ch7";
 			clocks = <&cpg CPG_MOD 218>;
 			clock-names = "fck";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 218>;
 			#dma-cells = <1>;
 			dma-channels = <8>;
+			iommus = <&ipmmu_ds1 0>, <&ipmmu_ds1 1>,
+			       <&ipmmu_ds1 2>, <&ipmmu_ds1 3>,
+			       <&ipmmu_ds1 4>, <&ipmmu_ds1 5>,
+			       <&ipmmu_ds1 6>, <&ipmmu_ds1 7>;
 		};
 
 		dmac2: dma-controller@e7310000 {
@@ -187,10 +245,14 @@
 					  "ch4", "ch5", "ch6", "ch7";
 			clocks = <&cpg CPG_MOD 217>;
 			clock-names = "fck";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 217>;
 			#dma-cells = <1>;
 			dma-channels = <8>;
+			iommus = <&ipmmu_ds1 16>, <&ipmmu_ds1 17>,
+			       <&ipmmu_ds1 18>, <&ipmmu_ds1 19>,
+			       <&ipmmu_ds1 20>, <&ipmmu_ds1 21>,
+			       <&ipmmu_ds1 22>, <&ipmmu_ds1 23>;
 		};
 
 		hscif0: serial@e6540000 {
@@ -200,13 +262,13 @@
 			reg = <0 0xe6540000 0 96>;
 			interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 520>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x31>, <&dmac1 0x30>,
 			       <&dmac2 0x31>, <&dmac2 0x30>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 520>;
 			status = "disabled";
 		};
@@ -218,13 +280,13 @@
 			reg = <0 0xe6550000 0 96>;
 			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 519>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x33>, <&dmac1 0x32>,
 			       <&dmac2 0x33>, <&dmac2 0x32>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 519>;
 			status = "disabled";
 		};
@@ -236,13 +298,13 @@
 			reg = <0 0xe6560000 0 96>;
 			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 518>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x35>, <&dmac1 0x34>,
 			       <&dmac2 0x35>, <&dmac2 0x34>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 518>;
 			status = "disabled";
 		};
@@ -253,13 +315,13 @@
 			reg = <0 0xe66a0000 0 96>;
 			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 517>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x37>, <&dmac1 0x36>,
 			       <&dmac2 0x37>, <&dmac2 0x36>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 517>;
 			status = "disabled";
 		};
@@ -271,13 +333,13 @@
 			reg = <0 0xe6e60000 0 64>;
 			interrupts = <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 207>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x51>, <&dmac1 0x50>,
 			       <&dmac2 0x51>, <&dmac2 0x50>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 207>;
 			status = "disabled";
 		};
@@ -289,13 +351,13 @@
 			reg = <0 0xe6e68000 0 64>;
 			interrupts = <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 206>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x53>, <&dmac1 0x52>,
 			       <&dmac2 0x53>, <&dmac2 0x52>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 206>;
 			status = "disabled";
 		};
@@ -307,13 +369,13 @@
 			reg = <0 0xe6c50000 0 64>;
 			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 204>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x57>, <&dmac1 0x56>,
 			       <&dmac2 0x57>, <&dmac2 0x56>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 204>;
 			status = "disabled";
 		};
@@ -324,13 +386,13 @@
 			reg = <0 0xe6c40000 0 64>;
 			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 203>,
-				 <&cpg CPG_CORE 9>,
+				 <&cpg CPG_CORE R8A77970_CLK_S2D1>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			dmas = <&dmac1 0x59>, <&dmac1 0x58>,
 			       <&dmac2 0x59>, <&dmac2 0x58>;
 			dma-names = "tx", "rx", "tx", "rx";
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 203>;
 			status = "disabled";
 		};
@@ -372,9 +434,10 @@
 					  "ch20", "ch21", "ch22", "ch23",
 					  "ch24";
 			clocks = <&cpg CPG_MOD 812>;
-			power-domains = <&sysc 32>;
+			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 812>;
 			phy-mode = "rgmii-id";
+			iommus = <&ipmmu_rt 3>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 		};
diff --git a/arch/arm64/boot/dts/renesas/r8a77995.dtsi b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
index 788e3afae6e3..cff42cd1a6c8 100644
--- a/arch/arm64/boot/dts/renesas/r8a77995.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77995.dtsi
@@ -51,6 +51,13 @@
 		clock-frequency = <0>;
 	};
 
+	/* External CAN clock - to be overridden by boards that provide it */
+	can_clk: can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <0>;
+	};
+
 	scif_clk: scif {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -108,6 +115,88 @@
 			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		ipmmu_vi0: mmu@febd0000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfebd0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 14>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_vp0: mmu@fe990000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfe990000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 16>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_vc0: mmu@fe6b0000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfe6b0000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 12>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_pv0: mmu@fd800000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xfd800000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 6>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_hc: mmu@e6570000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe6570000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 2>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_rt: mmu@ffc80000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xffc80000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 10>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_mp: mmu@ec670000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xec670000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 4>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_ds0: mmu@e6740000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe6740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 0>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_ds1: mmu@e7740000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe7740000 0 0x1000>;
+			renesas,ipmmu-main = <&ipmmu_mm 1>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+		ipmmu_mm: mmu@e67b0000 {
+			compatible = "renesas,ipmmu-r8a77995";
+			reg = <0 0xe67b0000 0 0x1000>;
+			interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+			#iommu-cells = <1>;
+			status = "disabled";
+		};
+
+
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a77995-cpg-mssr";
 			reg = <0 0xe6150000 0 0x1000>;
@@ -155,6 +244,78 @@
 			resets = <&cpg 407>;
 		};
 
+		dmac0: dma-controller@e6700000 {
+			compatible = "renesas,dmac-r8a77995",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe6700000 0 0x10000>;
+			interrupts = <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7";
+			clocks = <&cpg CPG_MOD 219>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 219>;
+			#dma-cells = <1>;
+			dma-channels = <8>;
+		};
+
+		dmac1: dma-controller@e7300000 {
+			compatible = "renesas,dmac-r8a77995",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7300000 0 0x10000>;
+			interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7";
+			clocks = <&cpg CPG_MOD 218>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 218>;
+			#dma-cells = <1>;
+			dma-channels = <8>;
+		};
+
+		dmac2: dma-controller@e7310000 {
+			compatible = "renesas,dmac-r8a77995",
+				     "renesas,rcar-dmac";
+			reg = <0 0xe7310000 0 0x10000>;
+			interrupts = <GIC_SPI 416 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 417 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 418 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 419 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 420 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 421 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 422 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 423 IRQ_TYPE_LEVEL_HIGH
+				      GIC_SPI 424 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "error",
+					"ch0", "ch1", "ch2", "ch3",
+					"ch4", "ch5", "ch6", "ch7";
+			clocks = <&cpg CPG_MOD 217>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 217>;
+			#dma-cells = <1>;
+			dma-channels = <8>;
+		};
+
 		gpio0: gpio@e6050000 {
 			compatible = "renesas,gpio-r8a77995",
 				     "renesas,rcar-gen3-gpio",
@@ -267,6 +428,63 @@
 			resets = <&cpg 906>;
 		};
 
+		can0: can@e6c30000 {
+			compatible = "renesas,can-r8a77995",
+				     "renesas,rcar-gen3-can";
+			reg = <0 0xe6c30000 0 0x1000>;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>,
+			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 916>;
+			status = "disabled";
+		};
+
+		can1: can@e6c38000 {
+			compatible = "renesas,can-r8a77995",
+				     "renesas,rcar-gen3-can";
+			reg = <0 0xe6c38000 0 0x1000>;
+			interrupts = <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 915>,
+			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "clkp1", "clkp2", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 915>;
+			status = "disabled";
+		};
+
+		canfd: can@e66c0000 {
+			compatible = "renesas,r8a77995-canfd",
+				     "renesas,rcar-gen3-canfd";
+			reg = <0 0xe66c0000 0 0x8000>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>,
+				   <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 914>,
+			       <&cpg CPG_CORE R8A77995_CLK_CANFD>,
+			       <&can_clk>;
+			clock-names = "fck", "canfd", "can_clk";
+			assigned-clocks = <&cpg CPG_CORE R8A77995_CLK_CANFD>;
+			assigned-clock-rates = <40000000>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 914>;
+			status = "disabled";
+
+			channel0 {
+				status = "disabled";
+			};
+
+			channel1 {
+				status = "disabled";
+			};
+		};
+
 		avb: ethernet@e6800000 {
 			compatible = "renesas,etheravb-r8a77995",
 				     "renesas,etheravb-rcar-gen3";
@@ -307,6 +525,7 @@
 			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
 			resets = <&cpg 812>;
 			phy-mode = "rgmii-txid";
+			iommus = <&ipmmu_ds0 16>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
@@ -321,6 +540,9 @@
 				 <&cpg CPG_CORE R8A77995_CLK_S3D1C>,
 				 <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
+			dmas = <&dmac1 0x13>, <&dmac1 0x12>,
+			       <&dmac2 0x13>, <&dmac2 0x12>;
+			dma-names = "tx", "rx", "tx", "rx";
 			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
 			resets = <&cpg 310>;
 			status = "disabled";
@@ -366,6 +588,18 @@
 			status = "disabled";
 		};
 
+		sdhi2: sd@ee140000 {
+			compatible = "renesas,sdhi-r8a77995",
+				     "renesas,rcar-gen3-sdhi";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 312>;
+			max-frequency = <200000000>;
+			power-domains = <&sysc R8A77995_PD_ALWAYS_ON>;
+			resets = <&cpg 312>;
+			status = "disabled";
+		};
+
 		ehci0: usb@ee080100 {
 			compatible = "generic-ehci";
 			reg = <0 0xee080100 0 0x100>;
diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
index dbe2648649db..c3fafb6025b3 100644
--- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
@@ -263,6 +263,7 @@
 		reg = <0>;
 		interrupt-parent = <&gpio2>;
 		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>;
 	};
 };
 
@@ -354,6 +355,30 @@
 
 &i2c_dvfs {
 	status = "okay";
+
+	pmic: pmic@30 {
+		pinctrl-0 = <&irq0_pins>;
+		pinctrl-names = "default";
+
+		compatible = "rohm,bd9571mwv";
+		reg = <0x30>;
+		interrupt-parent = <&intc_ex>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		regulators {
+			dvfs: dvfs {
+				regulator-name = "dvfs";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <1030000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
 };
 
 &ohci0 {
@@ -409,6 +434,11 @@
 		function = "i2c2";
 	};
 
+	irq0_pins: irq0 {
+		groups = "intc_ex_irq0";
+		function = "intc_ex";
+	};
+
 	pwm1_pins: pwm1 {
 		groups = "pwm1_a";
 		function = "pwm1";
@@ -595,6 +625,7 @@
 	bus-width = <8>;
 	mmc-hs200-1_8v;
 	non-removable;
+	fixed-emmc-driver-type = <1>;
 	status = "okay";
 };
 
@@ -616,6 +647,10 @@
 	shared-pin;
 };
 
+&usb_extal_clk {
+	clock-frequency = <50000000>;
+};
+
 &usb2_phy0 {
 	pinctrl-0 = <&usb0_pins>;
 	pinctrl-names = "default";
@@ -631,6 +666,21 @@
 	status = "okay";
 };
 
+&usb3_peri0 {
+	phys = <&usb3_phy0>;
+	phy-names = "usb";
+
+	status = "okay";
+};
+
+&usb3_phy0 {
+	status = "okay";
+};
+
+&usb3s0_clk {
+	clock-frequency = <100000000>;
+};
+
 &wdt0 {
 	timeout-sec = <60>;
 	status = "okay";
diff --git a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
index 657ad1041965..a4e715cbde87 100644
--- a/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb-kf.dtsi
@@ -29,6 +29,7 @@
 };
 
 &ehci0 {
+	dr_mode = "otg";
 	status = "okay";
 };
 
@@ -41,6 +42,7 @@
 };
 
 &hsusb {
+	dr_mode = "otg";
 	status = "okay";
 };
 
@@ -67,6 +69,20 @@
 			output-high;
 			line-name = "HUB rst";
 		};
+
+		otg_offvbusn {
+			gpio-hog;
+			gpios = <8 GPIO_ACTIVE_HIGH>;
+			output-low;
+			line-name = "OTG OFFVBUSn";
+		};
+
+		otg_extlpn {
+			gpio-hog;
+			gpios = <9 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "OTG EXTLPn";
+		};
 	};
 
 	gpio_exp_75: gpio@75 {
@@ -119,6 +135,7 @@
 };
 
 &ohci0 {
+	dr_mode = "otg";
 	status = "okay";
 };
 
@@ -154,6 +171,11 @@
 		groups = "scif1_data_b", "scif1_ctrl";
 		function = "scif1";
 	};
+
+	usb0_pins: usb0 {
+		groups = "usb0";
+		function = "usb0";
+	};
 };
 
 &scif1 {
@@ -164,6 +186,13 @@
 	status = "okay";
 };
 
+&usb2_phy0 {
+	pinctrl-0 = <&usb0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
 &xhci0 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi
index 73439cf48659..3e7a6b94e9f8 100644
--- a/arch/arm64/boot/dts/renesas/ulcb.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi
@@ -153,6 +153,7 @@
 		reg = <0>;
 		interrupt-parent = <&gpio2>;
 		interrupts = <11 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&gpio2 10 GPIO_ACTIVE_LOW>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts
index 3890468678ce..28257724a56e 100644
--- a/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3328-rock64.dts
@@ -132,17 +132,16 @@
 	assigned-clocks = <&cru SCLK_MAC2IO>, <&cru SCLK_MAC2IO_EXT>;
 	assigned-clock-parents = <&gmac_clkin>, <&gmac_clkin>;
 	clock_in_out = "input";
-	/* shows instability at 1GBit right now */
-	max-speed = <100>;
 	phy-supply = <&vcc_io>;
 	phy-mode = "rgmii";
 	pinctrl-names = "default";
 	pinctrl-0 = <&rgmiim1_pins>;
+	snps,force_thresh_dma_mode;
 	snps,reset-gpio = <&gpio1 RK_PC2 GPIO_ACTIVE_LOW>;
 	snps,reset-active-low;
 	snps,reset-delays-us = <0 10000 50000>;
-	tx_delay = <0x26>;
-	rx_delay = <0x11>;
+	tx_delay = <0x24>;
+	rx_delay = <0x18>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3328.dtsi b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
index 2426da631938..cae341554486 100644
--- a/arch/arm64/boot/dts/rockchip/rk3328.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3328.dtsi
@@ -531,6 +531,31 @@
 		status = "disabled";
 	};
 
+	efuse: efuse@ff260000 {
+		compatible = "rockchip,rk3328-efuse";
+		reg = <0x0 0xff260000 0x0 0x50>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		clocks = <&cru SCLK_EFUSE>;
+		clock-names = "pclk_efuse";
+		rockchip,efuse-size = <0x20>;
+
+		/* Data cells */
+		efuse_id: id@7 {
+			reg = <0x07 0x10>;
+		};
+		cpu_leakage: cpu-leakage@17 {
+			reg = <0x17 0x1>;
+		};
+		logic_leakage: logic-leakage@19 {
+			reg = <0x19 0x1>;
+		};
+		efuse_cpu_version: cpu-version@1a {
+			reg = <0x1a 0x1>;
+			bits = <3 3>;
+		};
+	};
+
 	saradc: adc@ff280000 {
 		compatible = "rockchip,rk3328-saradc", "rockchip,rk3399-saradc";
 		reg = <0x0 0xff280000 0x0 0x100>;
@@ -543,6 +568,28 @@
 		status = "disabled";
 	};
 
+	gpu: gpu@ff300000 {
+		compatible = "rockchip,rk3328-mali", "arm,mali-450";
+		reg = <0x0 0xff300000 0x0 0x40000>;
+		interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "gp",
+				  "gpmmu",
+				  "pp",
+				  "pp0",
+				  "ppmmu0",
+				  "pp1",
+				  "ppmmu1";
+		clocks = <&cru ACLK_GPU>, <&cru ACLK_GPU>;
+		clock-names = "bus", "core";
+		resets = <&cru SRST_GPU_A>;
+	};
+
 	h265e_mmu: iommu@ff330200 {
 		compatible = "rockchip,iommu";
 		reg = <0x0 0xff330200 0 0x100>;
@@ -683,7 +730,7 @@
 		interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>,
 			 <&cru SCLK_SDMMC_DRV>, <&cru SCLK_SDMMC_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
 		status = "disabled";
 	};
@@ -694,7 +741,7 @@
 		interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cru HCLK_SDIO>, <&cru SCLK_SDIO>,
 			 <&cru SCLK_SDIO_DRV>, <&cru SCLK_SDIO_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
 		status = "disabled";
 	};
@@ -705,7 +752,7 @@
 		interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
 		clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>,
 			 <&cru SCLK_EMMC_DRV>, <&cru SCLK_EMMC_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
 		status = "disabled";
 	};
diff --git a/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/arch/arm64/boot/dts/rockchip/rk3368.dtsi
index aa4d07046a7b..03458ac44201 100644
--- a/arch/arm64/boot/dts/rockchip/rk3368.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3368.dtsi
@@ -257,7 +257,7 @@
 		max-frequency = <150000000>;
 		clocks = <&cru HCLK_SDIO0>, <&cru SCLK_SDIO0>,
 			 <&cru SCLK_SDIO0_DRV>, <&cru SCLK_SDIO0_SAMPLE>;
-		clock-names = "biu", "ciu", "ciu_drv", "ciu_sample";
+		clock-names = "biu", "ciu", "ciu-drive", "ciu-sample";
 		fifo-depth = <0x100>;
 		interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 		resets = <&cru SRST_SDIO0>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts
index 0384e3121f18..191a6bcb1704 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-kevin.dts
@@ -93,6 +93,18 @@
 		pwm-delay-us = <10000>;
 	};
 
+	edp_panel: edp-panel {
+		compatible = "sharp,lq123p1jx31", "simple-panel";
+		backlight = <&backlight>;
+		power-supply = <&pp3300_disp>;
+
+		ports {
+			panel_in_edp: endpoint {
+				remote-endpoint = <&edp_out_panel>;
+			};
+		};
+	};
+
 	thermistor_ppvar_bigcpu: thermistor-ppvar-bigcpu {
 		compatible = "murata,ncp15wb473";
 		pullup-uv = <1800000>;
@@ -268,6 +280,23 @@ ap_i2c_dig: &i2c2 {
 	};
 };
 
+&edp {
+	status = "okay";
+
+	ports {
+		edp_out: port@1 {
+			reg = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			edp_out_panel: endpoint@0 {
+				reg = <0>;
+				remote-endpoint = <&panel_in_edp>;
+			};
+		};
+	};
+};
+
 &ppvar_bigcpu_pwm {
 	regulator-min-microvolt = <798674>;
 	regulator-max-microvolt = <1302172>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
index 5772c52fbfd3..204bdb9857b9 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
@@ -406,8 +406,9 @@
 	wlan_pd_n: wlan-pd-n {
 		compatible = "regulator-fixed";
 		regulator-name = "wlan_pd_n";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wlan_module_reset_l>;
 
-		/* Note the wlan_module_reset_l pinctrl */
 		enable-active-high;
 		gpio = <&gpio1 11 GPIO_ACTIVE_HIGH>;
 
@@ -855,6 +856,20 @@ ap_i2c_audio: &i2c8 {
 			compatible = "google,cros-ec-pwm";
 			#pwm-cells = <1>;
 		};
+
+		usbc_extcon0: extcon@0 {
+			compatible = "google,extcon-usbc-cros-ec";
+			google,usb-port-id = <0>;
+
+			#extcon-cells = <0>;
+		};
+
+		usbc_extcon1: extcon@1 {
+			compatible = "google,extcon-usbc-cros-ec";
+			google,usb-port-id = <1>;
+
+			#extcon-cells = <0>;
+		};
 	};
 };
 
@@ -865,6 +880,16 @@ ap_i2c_audio: &i2c8 {
 	rockchip,hw-tshut-polarity = <1>; /* tshut polarity 0:LOW 1:HIGH */
 };
 
+&tcphy0 {
+	status = "okay";
+	extcon = <&usbc_extcon0>;
+};
+
+&tcphy1 {
+	status = "okay";
+	extcon = <&usbc_extcon1>;
+};
+
 &u2phy0 {
 	status = "okay";
 };
@@ -911,6 +936,7 @@ ap_i2c_audio: &i2c8 {
 
 &usbdrd3_0 {
 	status = "okay";
+	extcon = <&usbc_extcon0>;
 };
 
 &usbdrd_dwc3_0 {
@@ -920,6 +946,7 @@ ap_i2c_audio: &i2c8 {
 
 &usbdrd3_1 {
 	status = "okay";
+	extcon = <&usbc_extcon1>;
 };
 
 &usbdrd_dwc3_1 {
@@ -927,6 +954,22 @@ ap_i2c_audio: &i2c8 {
 	dr_mode = "host";
 };
 
+&vopb {
+	status = "okay";
+};
+
+&vopb_mmu {
+	status = "okay";
+};
+
+&vopl {
+	status = "okay";
+};
+
+&vopl_mmu {
+	status = "okay";
+};
+
 #include <arm/cros-ec-keyboard.dtsi>
 #include <arm/cros-ec-sbs.dtsi>
 
@@ -941,12 +984,6 @@ ap_i2c_audio: &i2c8 {
 	pinctrl-0 = <
 		&ap_pwroff	/* AP will auto-assert this when in S3 */
 		&clk_32k	/* This pin is always 32k on gru boards */
-
-		/*
-		 * We want this driven low ASAP; firmware should help us, but
-		 * we can help ourselves too.
-		 */
-		&wlan_module_reset_l
 	>;
 
 	pcfg_output_low: pcfg-output-low {
@@ -1126,12 +1163,7 @@ ap_i2c_audio: &i2c8 {
 		};
 
 		wlan_module_reset_l: wlan-module-reset-l {
-			/*
-			 * We want this driven low ASAP (As {Soon,Strongly} As
-			 * Possible), to avoid leakage through the powered-down
-			 * WiFi.
-			 */
-			rockchip,pins = <1 11 RK_FUNC_GPIO &pcfg_output_low>;
+			rockchip,pins = <1 11 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 
 		bt_host_wake_l: bt-host-wake-l {
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi
index 0f873c897d0d..ce592a4c0c4c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-sapphire.dtsi
@@ -457,7 +457,7 @@
 	assigned-clocks = <&cru SCLK_PCIEPHY_REF>;
 	assigned-clock-parents = <&cru SCLK_PCIEPHY_REF100M>;
 	assigned-clock-rates = <100000000>;
-	ep-gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_HIGH>;
+	ep-gpios = <&gpio2 RK_PA4 GPIO_ACTIVE_HIGH>;
 	num-lanes = <4>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie_clkreqn_cpm>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
index d340b58ab184..0b81ca1d07e7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
@@ -397,9 +397,13 @@
 		#size-cells = <2>;
 		ranges;
 		clocks = <&cru SCLK_USB3OTG0_REF>, <&cru SCLK_USB3OTG0_SUSPEND>,
-			 <&cru ACLK_USB3OTG0>, <&cru ACLK_USB3_GRF>;
+			 <&cru ACLK_USB3OTG0>, <&cru ACLK_USB3_RKSOC_AXI_PERF>,
+			 <&cru ACLK_USB3>, <&cru ACLK_USB3_GRF>;
 		clock-names = "ref_clk", "suspend_clk",
-			      "bus_clk", "grf_clk";
+			      "bus_clk", "aclk_usb3_rksoc_axi_perf",
+			      "aclk_usb3", "grf_clk";
+		resets = <&cru SRST_A_USB3_OTG0>;
+		reset-names = "usb3-otg";
 		status = "disabled";
 
 		usbdrd_dwc3_0: dwc3 {
@@ -415,6 +419,7 @@
 			snps,dis_u2_susphy_quirk;
 			snps,dis-del-phy-power-chg-quirk;
 			snps,dis-tx-ipgap-linecheck-quirk;
+			power-domains = <&power RK3399_PD_USB3>;
 			status = "disabled";
 		};
 	};
@@ -425,9 +430,13 @@
 		#size-cells = <2>;
 		ranges;
 		clocks = <&cru SCLK_USB3OTG1_REF>, <&cru SCLK_USB3OTG1_SUSPEND>,
-			 <&cru ACLK_USB3OTG1>, <&cru ACLK_USB3_GRF>;
+			 <&cru ACLK_USB3OTG1>, <&cru ACLK_USB3_RKSOC_AXI_PERF>,
+			 <&cru ACLK_USB3>, <&cru ACLK_USB3_GRF>;
 		clock-names = "ref_clk", "suspend_clk",
-			      "bus_clk", "grf_clk";
+			      "bus_clk", "aclk_usb3_rksoc_axi_perf",
+			      "aclk_usb3", "grf_clk";
+		resets = <&cru SRST_A_USB3_OTG1>;
+		reset-names = "usb3-otg";
 		status = "disabled";
 
 		usbdrd_dwc3_1: dwc3 {
@@ -443,6 +452,7 @@
 			snps,dis_u2_susphy_quirk;
 			snps,dis-del-phy-power-chg-quirk;
 			snps,dis-tx-ipgap-linecheck-quirk;
+			power-domains = <&power RK3399_PD_USB3>;
 			status = "disabled";
 		};
 	};
@@ -991,6 +1001,12 @@
 				clocks = <&cru HCLK_SDIO>;
 				pm_qos = <&qos_sdioaudio>;
 			};
+			pd_usb3@RK3399_PD_USB3 {
+				reg = <RK3399_PD_USB3>;
+				clocks = <&cru ACLK_USB3>;
+				pm_qos = <&qos_usb_otg0>,
+					 <&qos_usb_otg1>;
+			};
 			pd_vio@RK3399_PD_VIO {
 				reg = <RK3399_PD_VIO>;
 				#address-cells = <1>;
@@ -1526,6 +1542,11 @@
 				reg = <2>;
 				remote-endpoint = <&hdmi_in_vopl>;
 			};
+
+			vopl_out_mipi1: endpoint@3 {
+				reg = <3>;
+				remote-endpoint = <&mipi1_in_vopl>;
+			};
 		};
 	};
 
@@ -1573,6 +1594,11 @@
 				reg = <2>;
 				remote-endpoint = <&hdmi_in_vopb>;
 			};
+
+			vopb_out_mipi1: endpoint@3 {
+				reg = <3>;
+				remote-endpoint = <&mipi1_in_vopb>;
+			};
 		};
 	};
 
@@ -1648,11 +1674,17 @@
 			 <&cru SCLK_DPHY_TX0_CFG>, <&cru PCLK_VIO_GRF>;
 		clock-names = "ref", "pclk", "phy_cfg", "grf";
 		power-domains = <&power RK3399_PD_VIO>;
+		resets = <&cru SRST_P_MIPI_DSI0>;
+		reset-names = "apb";
 		rockchip,grf = <&grf>;
 		status = "disabled";
 
 		ports {
-			mipi_in: port {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi_in: port@0 {
+				reg = <0>;
 				#address-cells = <1>;
 				#size-cells = <0>;
 
@@ -1668,12 +1700,47 @@
 		};
 	};
 
+	mipi_dsi1: mipi@ff968000 {
+		compatible = "rockchip,rk3399-mipi-dsi", "snps,dw-mipi-dsi";
+		reg = <0x0 0xff968000 0x0 0x8000>;
+		interrupts = <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH 0>;
+		clocks = <&cru SCLK_DPHY_PLL>, <&cru PCLK_MIPI_DSI1>,
+			 <&cru SCLK_DPHY_TX1RX1_CFG>, <&cru PCLK_VIO_GRF>;
+		clock-names = "ref", "pclk", "phy_cfg", "grf";
+		power-domains = <&power RK3399_PD_VIO>;
+		resets = <&cru SRST_P_MIPI_DSI1>;
+		reset-names = "apb";
+		rockchip,grf = <&grf>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			mipi1_in: port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				mipi1_in_vopb: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&vopb_out_mipi1>;
+				};
+
+				mipi1_in_vopl: endpoint@1 {
+					reg = <1>;
+					remote-endpoint = <&vopl_out_mipi1>;
+				};
+			};
+		};
+	};
+
 	edp: edp@ff970000 {
 		compatible = "rockchip,rk3399-edp";
 		reg = <0x0 0xff970000 0x0 0x8000>;
 		interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH 0>;
-		clocks = <&cru PCLK_EDP>, <&cru PCLK_EDP_CTRL>;
-		clock-names = "dp", "pclk";
+		clocks = <&cru PCLK_EDP>, <&cru PCLK_EDP_CTRL>, <&cru PCLK_VIO_GRF>;
+		clock-names = "dp", "pclk", "grf";
 		pinctrl-names = "default";
 		pinctrl-0 = <&edp_hpd>;
 		power-domains = <&power RK3399_PD_EDP>;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts
index 6bdefb26b329..54c53170699a 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts
@@ -50,7 +50,7 @@
 &gpio {
 	xirq0 {
 		gpio-hog;
-		gpios = <120 0>;
+		gpios = <UNIPHIER_GPIO_IRQ(0) 0>;
 		input;
 	};
 };
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
index 1c63d0ab8a58..cd7c2d0a1f64 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
@@ -8,6 +8,7 @@
  */
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/gpio/uniphier-gpio.h>
 
 /memreserve/ 0x80000000 0x02000000;
 
@@ -100,7 +101,7 @@
 
 	emmc_pwrseq: emmc-pwrseq {
 		compatible = "mmc-pwrseq-emmc";
-		reset-gpios = <&gpio 26 GPIO_ACTIVE_LOW>;
+		reset-gpios = <&gpio UNIPHIER_GPIO_PORT(3, 2) GPIO_ACTIVE_LOW>;
 	};
 
 	timer {
@@ -347,6 +348,7 @@
 				 <&mio_clk 12>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 8>,
 				 <&mio_rst 12>;
+			has-transaction-translator;
 		};
 
 		usb1: usb@5a810100 {
@@ -360,6 +362,7 @@
 				 <&mio_clk 13>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 9>,
 				 <&mio_rst 13>;
+			has-transaction-translator;
 		};
 
 		usb2: usb@5a820100 {
@@ -373,6 +376,7 @@
 				 <&mio_clk 14>;
 			resets = <&sys_rst 8>, <&mio_rst 7>, <&mio_rst 10>,
 				 <&mio_rst 14>;
+			has-transaction-translator;
 		};
 
 		mioctrl@5b3e0000 {
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts
index 254d6795c67e..693371033c90 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts
@@ -50,7 +50,7 @@
 &gpio {
 	xirq0 {
 		gpio-hog;
-		gpios = <120 0>;
+		gpios = <UNIPHIER_GPIO_IRQ(0) 0>;
 		input;
 	};
 };
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
index 5c81070944cc..8a3276ba2da1 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
@@ -8,6 +8,7 @@
  */
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/gpio/uniphier-gpio.h>
 #include <dt-bindings/thermal/thermal.h>
 
 /memreserve/ 0x80000000 0x02000000;
@@ -172,7 +173,7 @@
 
 	emmc_pwrseq: emmc-pwrseq {
 		compatible = "mmc-pwrseq-emmc";
-		reset-gpios = <&gpio 26 GPIO_ACTIVE_LOW>;
+		reset-gpios = <&gpio UNIPHIER_GPIO_PORT(3, 2) GPIO_ACTIVE_LOW>;
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts
index f9f06fcfb94a..3c7108729827 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts
+++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts
@@ -45,6 +45,22 @@
 	status = "okay";
 };
 
+&serial2 {
+	status = "okay";
+};
+
+&serial3 {
+	status = "okay";
+};
+
+&gpio {
+	xirq4 {
+		gpio-hog;
+		gpios = <UNIPHIER_GPIO_IRQ(4) 0>;
+		input;
+	};
+};
+
 &i2c0 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
index 0ac2ace82435..234fc58cc599 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
@@ -8,6 +8,7 @@
  */
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/gpio/uniphier-gpio.h>
 
 /memreserve/ 0x80000000 0x02000000;
 
@@ -128,7 +129,7 @@
 
 	emmc_pwrseq: emmc-pwrseq {
 		compatible = "mmc-pwrseq-emmc";
-		reset-gpios = <&gpio 47 GPIO_ACTIVE_LOW>;
+		reset-gpios = <&gpio UNIPHIER_GPIO_PORT(5, 7) GPIO_ACTIVE_LOW>;
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/sprd/sc9860.dtsi b/arch/arm64/boot/dts/sprd/sc9860.dtsi
index 7b7d8cedacda..5dbfb796d9f9 100644
--- a/arch/arm64/boot/dts/sprd/sc9860.dtsi
+++ b/arch/arm64/boot/dts/sprd/sc9860.dtsi
@@ -183,6 +183,120 @@
 	};
 
 	soc {
+		pmu_gate: pmu-gate {
+			compatible = "sprd,sc9860-pmu-gate";
+			sprd,syscon = <&pmu_regs>; /* 0x402b0000 */
+			clocks = <&ext_26m>;
+			#clock-cells = <1>;
+		};
+
+		pll: pll {
+			compatible = "sprd,sc9860-pll";
+			sprd,syscon = <&ana_regs>; /* 0x40400000 */
+			clocks = <&pmu_gate 0>;
+			#clock-cells = <1>;
+		};
+
+		ap_clk: clock-controller@20000000 {
+			compatible = "sprd,sc9860-ap-clk";
+			reg = <0 0x20000000 0 0x400>;
+			clocks = <&ext_26m>, <&pll 0>,
+				 <&pmu_gate 0>;
+			#clock-cells = <1>;
+		};
+
+		aon_prediv: aon-prediv {
+			compatible = "sprd,sc9860-aon-prediv";
+			reg = <0 0x402d0000 0 0x400>;
+			clocks = <&ext_26m>, <&pll 0>,
+				 <&pmu_gate 0>;
+			#clock-cells = <1>;
+		};
+
+		apahb_gate: apahb-gate {
+			compatible = "sprd,sc9860-apahb-gate";
+			sprd,syscon = <&ap_ahb_regs>; /* 0x20210000 */
+			clocks = <&aon_prediv 0>;
+			#clock-cells = <1>;
+		};
+
+		aon_gate: aon-gate {
+			compatible = "sprd,sc9860-aon-gate";
+			sprd,syscon = <&aon_regs>; /* 0x402e0000 */
+			clocks = <&aon_prediv 0>;
+			#clock-cells = <1>;
+		};
+
+		aonsecure_clk: clock-controller@40880000 {
+			compatible = "sprd,sc9860-aonsecure-clk";
+			reg = <0 0x40880000 0 0x400>;
+			clocks = <&ext_26m>, <&pll 0>;
+			#clock-cells = <1>;
+		};
+
+		agcp_gate: agcp-gate {
+			compatible = "sprd,sc9860-agcp-gate";
+			sprd,syscon = <&agcp_regs>; /* 0x415e0000 */
+			clocks = <&aon_prediv 0>;
+			#clock-cells = <1>;
+		};
+
+		gpu_clk: clock-controller@60200000 {
+			compatible = "sprd,sc9860-gpu-clk";
+			reg = <0 0x60200000 0 0x400>;
+			clocks = <&pll 0>;
+			#clock-cells = <1>;
+		};
+
+		vsp_clk: clock-controller@61000000 {
+			compatible = "sprd,sc9860-vsp-clk";
+			reg = <0 0x61000000 0 0x400>;
+			clocks = <&ext_26m>, <&pll 0>;
+			#clock-cells = <1>;
+		};
+
+		vsp_gate: vsp-gate {
+			compatible = "sprd,sc9860-vsp-gate";
+			sprd,syscon = <&vsp_regs>; /* 0x61100000 */
+			clocks = <&vsp_clk 0>;
+			#clock-cells = <1>;
+		};
+
+		cam_clk: clock-controller@62000000 {
+			compatible = "sprd,sc9860-cam-clk";
+			reg = <0 0x62000000 0 0x4000>;
+			clocks = <&ext_26m>, <&pll 0>;
+			#clock-cells = <1>;
+		};
+
+		cam_gate: cam-gate {
+			compatible = "sprd,sc9860-cam-gate";
+			sprd,syscon = <&cam_regs>; /* 0x62100000 */
+			clocks = <&cam_clk 0>;
+			#clock-cells = <1>;
+		};
+
+		disp_clk: clock-controller@63000000 {
+			compatible = "sprd,sc9860-disp-clk";
+			reg = <0 0x63000000 0 0x400>;
+			clocks = <&ext_26m>, <&pll 0>;
+			#clock-cells = <1>;
+		};
+
+		disp_gate: disp-gate {
+			compatible = "sprd,sc9860-disp-gate";
+			sprd,syscon = <&disp_regs>; /* 0x63100000 */
+			clocks = <&disp_clk 0>;
+			#clock-cells = <1>;
+		};
+
+		apapb_gate: apapb-gate {
+			compatible = "sprd,sc9860-apapb-gate";
+			sprd,syscon = <&ap_apb_regs>; /* 0x70b00000 */
+			clocks = <&ap_clk 0>;
+			#clock-cells = <1>;
+		};
+
 		funnel@10001000 { /* SoC Funnel */
 			compatible = "arm,coresight-funnel", "arm,primecell";
 			reg = <0 0x10001000 0 0x1000>;
diff --git a/arch/arm64/boot/dts/sprd/whale2.dtsi b/arch/arm64/boot/dts/sprd/whale2.dtsi
index 7c217c547f85..328009c4638c 100644
--- a/arch/arm64/boot/dts/sprd/whale2.dtsi
+++ b/arch/arm64/boot/dts/sprd/whale2.dtsi
@@ -17,6 +17,51 @@
 		#size-cells = <2>;
 		ranges;
 
+		ap_ahb_regs: syscon@20210000 {
+			compatible = "syscon";
+			reg = <0 0x20210000 0 0x10000>;
+		};
+
+		pmu_regs: syscon@402b0000 {
+			compatible = "syscon";
+			reg = <0 0x402b0000 0 0x10000>;
+		};
+
+		aon_regs: syscon@402e0000 {
+			compatible = "syscon";
+			reg = <0 0x402e0000 0 0x10000>;
+		};
+
+		ana_regs: syscon@40400000 {
+			compatible = "syscon";
+			reg = <0 0x40400000 0 0x10000>;
+		};
+
+		agcp_regs: syscon@415e0000 {
+			compatible = "syscon";
+			reg = <0 0x415e0000 0 0x1000000>;
+		};
+
+		vsp_regs: syscon@61100000 {
+			compatible = "syscon";
+			reg = <0 0x61100000 0 0x10000>;
+		};
+
+		cam_regs: syscon@62100000 {
+			compatible = "syscon";
+			reg = <0 0x62100000 0 0x10000>;
+		};
+
+		disp_regs: syscon@63100000 {
+			compatible = "syscon";
+			reg = <0 0x63100000 0 0x10000>;
+		};
+
+		ap_apb_regs: syscon@70b00000 {
+			compatible = "syscon";
+			reg = <0 0x70b00000 0 0x40000>;
+		};
+
 		ap-apb {
 			compatible = "simple-bus";
 			#address-cells = <1>;
@@ -59,13 +104,26 @@
 				status = "disabled";
 			};
 		};
+	};
 
+	ext_32k: ext_32k {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		clock-output-names = "ext-32k";
 	};
 
-	ext_26m: ext-26m {
+	ext_26m: ext_26m {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		clock-frequency = <26000000>;
-		clock-output-names = "ext_26m";
+		clock-output-names = "ext-26m";
+	};
+
+	ext_rco_100m: ext_rco_100m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+		clock-output-names = "ext-rco-100m";
 	};
 };
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 6356c6da34ea..634b373785c4 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -86,6 +86,7 @@ CONFIG_SCHED_MC=y
 CONFIG_NUMA=y
 CONFIG_PREEMPT=y
 CONFIG_KSM=y
+CONFIG_MEMORY_FAILURE=y
 CONFIG_TRANSPARENT_HUGEPAGE=y
 CONFIG_CMA=y
 CONFIG_SECCOMP=y
@@ -99,6 +100,7 @@ CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y
 CONFIG_ARM_CPUIDLE=y
 CONFIG_CPU_FREQ=y
 CONFIG_CPUFREQ_DT=y
+CONFIG_ARM_ARMADA_37XX_CPUFREQ=y
 CONFIG_ARM_BIG_LITTLE_CPUFREQ=y
 CONFIG_ARM_SCPI_CPUFREQ=y
 CONFIG_ACPI_CPPC_CPUFREQ=m
@@ -161,7 +163,7 @@ CONFIG_MTD_BLOCK=y
 CONFIG_MTD_M25P80=y
 CONFIG_MTD_NAND=y
 CONFIG_MTD_NAND_DENALI_DT=y
-CONFIG_MTD_NAND_PXA3xx=y
+CONFIG_MTD_NAND_MARVELL=y
 CONFIG_MTD_SPI_NOR=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=m
@@ -421,7 +423,6 @@ CONFIG_USB_OTG=y
 CONFIG_USB_XHCI_HCD=y
 CONFIG_USB_XHCI_TEGRA=y
 CONFIG_USB_EHCI_HCD=y
-CONFIG_USB_EHCI_MSM=y
 CONFIG_USB_EHCI_EXYNOS=y
 CONFIG_USB_EHCI_HCD_PLATFORM=y
 CONFIG_USB_OHCI_HCD=y
@@ -429,6 +430,8 @@ CONFIG_USB_OHCI_EXYNOS=y
 CONFIG_USB_OHCI_HCD_PLATFORM=y
 CONFIG_USB_RENESAS_USBHS=m
 CONFIG_USB_STORAGE=y
+CONFIG_USB_MUSB_HDRC=y
+CONFIG_USB_MUSB_SUNXI=y
 CONFIG_USB_DWC3=y
 CONFIG_USB_DWC2=y
 CONFIG_USB_CHIPIDEA=y
@@ -438,8 +441,6 @@ CONFIG_USB_CHIPIDEA_ULPI=y
 CONFIG_USB_ISP1760=y
 CONFIG_USB_HSIC_USB3503=y
 CONFIG_NOP_USB_XCEIV=y
-CONFIG_USB_MSM_OTG=y
-CONFIG_USB_QCOM_8X16_PHY=y
 CONFIG_USB_ULPI=y
 CONFIG_USB_GADGET=y
 CONFIG_USB_RENESAS_USBHS_UDC=m
@@ -473,7 +474,10 @@ CONFIG_LEDS_SYSCON=y
 CONFIG_LEDS_TRIGGER_HEARTBEAT=y
 CONFIG_LEDS_TRIGGER_CPU=y
 CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
+CONFIG_LEDS_TRIGGER_PANIC=y
+CONFIG_LEDS_TRIGGER_DISK=y
 CONFIG_EDAC=y
+CONFIG_EDAC_GHES=y
 CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_MAX77686=y
 CONFIG_RTC_DRV_RK808=m
@@ -558,6 +562,7 @@ CONFIG_PHY_XGENE=y
 CONFIG_PHY_TEGRA_XUSB=y
 CONFIG_QCOM_L2_PMU=y
 CONFIG_QCOM_L3_PMU=y
+CONFIG_UNIPHIER_EFUSE=y
 CONFIG_TEE=y
 CONFIG_OPTEE=y
 CONFIG_ARM_SCPI_PROTOCOL=y
@@ -567,6 +572,8 @@ CONFIG_ACPI=y
 CONFIG_ACPI_APEI=y
 CONFIG_ACPI_APEI_GHES=y
 CONFIG_ACPI_APEI_PCIEAER=y
+CONFIG_ACPI_APEI_MEMORY_FAILURE=y
+CONFIG_ACPI_APEI_EINJ=y
 CONFIG_EXT2_FS=y
 CONFIG_EXT3_FS=y
 CONFIG_EXT4_FS_POSIX_ACL=y
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 70c517aa4501..285c36c7b408 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -29,6 +29,24 @@ config CRYPTO_SHA2_ARM64_CE
 	select CRYPTO_HASH
 	select CRYPTO_SHA256_ARM64
 
+config CRYPTO_SHA512_ARM64_CE
+	tristate "SHA-384/SHA-512 digest algorithm (ARMv8 Crypto Extensions)"
+	depends on KERNEL_MODE_NEON
+	select CRYPTO_HASH
+	select CRYPTO_SHA512_ARM64
+
+config CRYPTO_SHA3_ARM64
+	tristate "SHA3 digest algorithm (ARMv8.2 Crypto Extensions)"
+	depends on KERNEL_MODE_NEON
+	select CRYPTO_HASH
+	select CRYPTO_SHA3
+
+config CRYPTO_SM3_ARM64_CE
+	tristate "SM3 digest algorithm (ARMv8.2 Crypto Extensions)"
+	depends on KERNEL_MODE_NEON
+	select CRYPTO_HASH
+	select CRYPTO_SM3
+
 config CRYPTO_GHASH_ARM64_CE
 	tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions"
 	depends on KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index b5edc5918c28..cee9b8d9830b 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -14,6 +14,15 @@ sha1-ce-y := sha1-ce-glue.o sha1-ce-core.o
 obj-$(CONFIG_CRYPTO_SHA2_ARM64_CE) += sha2-ce.o
 sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o
 
+obj-$(CONFIG_CRYPTO_SHA512_ARM64_CE) += sha512-ce.o
+sha512-ce-y := sha512-ce-glue.o sha512-ce-core.o
+
+obj-$(CONFIG_CRYPTO_SHA3_ARM64) += sha3-ce.o
+sha3-ce-y := sha3-ce-glue.o sha3-ce-core.o
+
+obj-$(CONFIG_CRYPTO_SM3_ARM64_CE) += sm3-ce.o
+sm3-ce-y := sm3-ce-glue.o sm3-ce-core.o
+
 obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
 ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
 
@@ -24,7 +33,7 @@ obj-$(CONFIG_CRYPTO_CRC32_ARM64_CE) += crc32-ce.o
 crc32-ce-y:= crc32-ce-core.o crc32-ce-glue.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
-CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
+aes-ce-cipher-y := aes-ce-core.o aes-ce-glue.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o
 aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o
diff --git a/arch/arm64/crypto/aes-ce-core.S b/arch/arm64/crypto/aes-ce-core.S
new file mode 100644
index 000000000000..8efdfdade393
--- /dev/null
+++ b/arch/arm64/crypto/aes-ce-core.S
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.arch		armv8-a+crypto
+
+ENTRY(__aes_ce_encrypt)
+	sub		w3, w3, #2
+	ld1		{v0.16b}, [x2]
+	ld1		{v1.4s}, [x0], #16
+	cmp		w3, #10
+	bmi		0f
+	bne		3f
+	mov		v3.16b, v1.16b
+	b		2f
+0:	mov		v2.16b, v1.16b
+	ld1		{v3.4s}, [x0], #16
+1:	aese		v0.16b, v2.16b
+	aesmc		v0.16b, v0.16b
+2:	ld1		{v1.4s}, [x0], #16
+	aese		v0.16b, v3.16b
+	aesmc		v0.16b, v0.16b
+3:	ld1		{v2.4s}, [x0], #16
+	subs		w3, w3, #3
+	aese		v0.16b, v1.16b
+	aesmc		v0.16b, v0.16b
+	ld1		{v3.4s}, [x0], #16
+	bpl		1b
+	aese		v0.16b, v2.16b
+	eor		v0.16b, v0.16b, v3.16b
+	st1		{v0.16b}, [x1]
+	ret
+ENDPROC(__aes_ce_encrypt)
+
+ENTRY(__aes_ce_decrypt)
+	sub		w3, w3, #2
+	ld1		{v0.16b}, [x2]
+	ld1		{v1.4s}, [x0], #16
+	cmp		w3, #10
+	bmi		0f
+	bne		3f
+	mov		v3.16b, v1.16b
+	b		2f
+0:	mov		v2.16b, v1.16b
+	ld1		{v3.4s}, [x0], #16
+1:	aesd		v0.16b, v2.16b
+	aesimc		v0.16b, v0.16b
+2:	ld1		{v1.4s}, [x0], #16
+	aesd		v0.16b, v3.16b
+	aesimc		v0.16b, v0.16b
+3:	ld1		{v2.4s}, [x0], #16
+	subs		w3, w3, #3
+	aesd		v0.16b, v1.16b
+	aesimc		v0.16b, v0.16b
+	ld1		{v3.4s}, [x0], #16
+	bpl		1b
+	aesd		v0.16b, v2.16b
+	eor		v0.16b, v0.16b, v3.16b
+	st1		{v0.16b}, [x1]
+	ret
+ENDPROC(__aes_ce_decrypt)
+
+/*
+ * __aes_ce_sub() - use the aese instruction to perform the AES sbox
+ *                  substitution on each byte in 'input'
+ */
+ENTRY(__aes_ce_sub)
+	dup		v1.4s, w0
+	movi		v0.16b, #0
+	aese		v0.16b, v1.16b
+	umov		w0, v0.s[0]
+	ret
+ENDPROC(__aes_ce_sub)
+
+ENTRY(__aes_ce_invert)
+	ld1		{v0.4s}, [x1]
+	aesimc		v1.16b, v0.16b
+	st1		{v1.4s}, [x0]
+	ret
+ENDPROC(__aes_ce_invert)
diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-glue.c
similarity index 62%
rename from arch/arm64/crypto/aes-ce-cipher.c
rename to arch/arm64/crypto/aes-ce-glue.c
index 6a75cd75ed11..e6b3227bbf57 100644
--- a/arch/arm64/crypto/aes-ce-cipher.c
+++ b/arch/arm64/crypto/aes-ce-glue.c
@@ -29,6 +29,13 @@ struct aes_block {
 	u8 b[AES_BLOCK_SIZE];
 };
 
+asmlinkage void __aes_ce_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+asmlinkage void __aes_ce_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+
+asmlinkage u32 __aes_ce_sub(u32 l);
+asmlinkage void __aes_ce_invert(struct aes_block *out,
+				const struct aes_block *in);
+
 static int num_rounds(struct crypto_aes_ctx *ctx)
 {
 	/*
@@ -44,10 +51,6 @@ static int num_rounds(struct crypto_aes_ctx *ctx)
 static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
 {
 	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-	struct aes_block *out = (struct aes_block *)dst;
-	struct aes_block const *in = (struct aes_block *)src;
-	void *dummy0;
-	int dummy1;
 
 	if (!may_use_simd()) {
 		__aes_arm64_encrypt(ctx->key_enc, dst, src, num_rounds(ctx));
@@ -55,49 +58,13 @@ static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
 	}
 
 	kernel_neon_begin();
-
-	__asm__("	ld1	{v0.16b}, %[in]			;"
-		"	ld1	{v1.4s}, [%[key]], #16		;"
-		"	cmp	%w[rounds], #10			;"
-		"	bmi	0f				;"
-		"	bne	3f				;"
-		"	mov	v3.16b, v1.16b			;"
-		"	b	2f				;"
-		"0:	mov	v2.16b, v1.16b			;"
-		"	ld1	{v3.4s}, [%[key]], #16		;"
-		"1:	aese	v0.16b, v2.16b			;"
-		"	aesmc	v0.16b, v0.16b			;"
-		"2:	ld1	{v1.4s}, [%[key]], #16		;"
-		"	aese	v0.16b, v3.16b			;"
-		"	aesmc	v0.16b, v0.16b			;"
-		"3:	ld1	{v2.4s}, [%[key]], #16		;"
-		"	subs	%w[rounds], %w[rounds], #3	;"
-		"	aese	v0.16b, v1.16b			;"
-		"	aesmc	v0.16b, v0.16b			;"
-		"	ld1	{v3.4s}, [%[key]], #16		;"
-		"	bpl	1b				;"
-		"	aese	v0.16b, v2.16b			;"
-		"	eor	v0.16b, v0.16b, v3.16b		;"
-		"	st1	{v0.16b}, %[out]		;"
-
-	:	[out]		"=Q"(*out),
-		[key]		"=r"(dummy0),
-		[rounds]	"=r"(dummy1)
-	:	[in]		"Q"(*in),
-				"1"(ctx->key_enc),
-				"2"(num_rounds(ctx) - 2)
-	:	"cc");
-
+	__aes_ce_encrypt(ctx->key_enc, dst, src, num_rounds(ctx));
 	kernel_neon_end();
 }
 
 static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
 {
 	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-	struct aes_block *out = (struct aes_block *)dst;
-	struct aes_block const *in = (struct aes_block *)src;
-	void *dummy0;
-	int dummy1;
 
 	if (!may_use_simd()) {
 		__aes_arm64_decrypt(ctx->key_dec, dst, src, num_rounds(ctx));
@@ -105,62 +72,10 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
 	}
 
 	kernel_neon_begin();
-
-	__asm__("	ld1	{v0.16b}, %[in]			;"
-		"	ld1	{v1.4s}, [%[key]], #16		;"
-		"	cmp	%w[rounds], #10			;"
-		"	bmi	0f				;"
-		"	bne	3f				;"
-		"	mov	v3.16b, v1.16b			;"
-		"	b	2f				;"
-		"0:	mov	v2.16b, v1.16b			;"
-		"	ld1	{v3.4s}, [%[key]], #16		;"
-		"1:	aesd	v0.16b, v2.16b			;"
-		"	aesimc	v0.16b, v0.16b			;"
-		"2:	ld1	{v1.4s}, [%[key]], #16		;"
-		"	aesd	v0.16b, v3.16b			;"
-		"	aesimc	v0.16b, v0.16b			;"
-		"3:	ld1	{v2.4s}, [%[key]], #16		;"
-		"	subs	%w[rounds], %w[rounds], #3	;"
-		"	aesd	v0.16b, v1.16b			;"
-		"	aesimc	v0.16b, v0.16b			;"
-		"	ld1	{v3.4s}, [%[key]], #16		;"
-		"	bpl	1b				;"
-		"	aesd	v0.16b, v2.16b			;"
-		"	eor	v0.16b, v0.16b, v3.16b		;"
-		"	st1	{v0.16b}, %[out]		;"
-
-	:	[out]		"=Q"(*out),
-		[key]		"=r"(dummy0),
-		[rounds]	"=r"(dummy1)
-	:	[in]		"Q"(*in),
-				"1"(ctx->key_dec),
-				"2"(num_rounds(ctx) - 2)
-	:	"cc");
-
+	__aes_ce_decrypt(ctx->key_dec, dst, src, num_rounds(ctx));
 	kernel_neon_end();
 }
 
-/*
- * aes_sub() - use the aese instruction to perform the AES sbox substitution
- *             on each byte in 'input'
- */
-static u32 aes_sub(u32 input)
-{
-	u32 ret;
-
-	__asm__("dup	v1.4s, %w[in]		;"
-		"movi	v0.16b, #0		;"
-		"aese	v0.16b, v1.16b		;"
-		"umov	%w[out], v0.4s[0]	;"
-
-	:	[out]	"=r"(ret)
-	:	[in]	"r"(input)
-	:		"v0","v1");
-
-	return ret;
-}
-
 int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		     unsigned int key_len)
 {
@@ -189,7 +104,7 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		u32 *rki = ctx->key_enc + (i * kwords);
 		u32 *rko = rki + kwords;
 
-		rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
+		rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0];
 		rko[1] = rko[0] ^ rki[1];
 		rko[2] = rko[1] ^ rki[2];
 		rko[3] = rko[2] ^ rki[3];
@@ -202,7 +117,7 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 		} else if (key_len == AES_KEYSIZE_256) {
 			if (i >= 6)
 				break;
-			rko[4] = aes_sub(rko[3]) ^ rki[4];
+			rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
 			rko[5] = rko[4] ^ rki[5];
 			rko[6] = rko[5] ^ rki[6];
 			rko[7] = rko[6] ^ rki[7];
@@ -221,13 +136,7 @@ int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
 
 	key_dec[0] = key_enc[j];
 	for (i = 1, j--; j > 0; i++, j--)
-		__asm__("ld1	{v0.4s}, %[in]		;"
-			"aesimc	v1.16b, v0.16b		;"
-			"st1	{v1.4s}, %[out]	;"
-
-		:	[out]	"=Q"(key_dec[i])
-		:	[in]	"Q"(key_enc[j])
-		:		"v0","v1");
+		__aes_ce_invert(key_dec + i, key_enc + j);
 	key_dec[i] = key_enc[0];
 
 	kernel_neon_end();
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
index 6d2445d603cc..3a44eada2347 100644
--- a/arch/arm64/crypto/aes-cipher-core.S
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -125,6 +125,16 @@ CPU_BE(	rev		w7, w7		)
 	ret
 	.endm
 
+ENTRY(__aes_arm64_encrypt)
+	do_crypt	fround, crypto_ft_tab, crypto_ft_tab + 1, 2
+ENDPROC(__aes_arm64_encrypt)
+
+	.align		5
+ENTRY(__aes_arm64_decrypt)
+	do_crypt	iround, crypto_it_tab, __aes_arm64_inverse_sbox, 0
+ENDPROC(__aes_arm64_decrypt)
+
+	.section	".rodata", "a"
 	.align		L1_CACHE_SHIFT
 	.type		__aes_arm64_inverse_sbox, %object
 __aes_arm64_inverse_sbox:
@@ -161,12 +171,3 @@ __aes_arm64_inverse_sbox:
 	.byte		0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
 	.byte		0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
 	.size		__aes_arm64_inverse_sbox, . - __aes_arm64_inverse_sbox
-
-ENTRY(__aes_arm64_encrypt)
-	do_crypt	fround, crypto_ft_tab, crypto_ft_tab + 1, 2
-ENDPROC(__aes_arm64_encrypt)
-
-	.align		5
-ENTRY(__aes_arm64_decrypt)
-	do_crypt	iround, crypto_it_tab, __aes_arm64_inverse_sbox, 0
-ENDPROC(__aes_arm64_decrypt)
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 998ba519a026..2fa850e86aa8 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -665,6 +665,7 @@ static int __init aes_init(void)
 
 unregister_simds:
 	aes_exit();
+	return err;
 unregister_ciphers:
 	crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 	return err;
diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S
index f1e3aa2732f9..1c7b45b7268e 100644
--- a/arch/arm64/crypto/aes-neon.S
+++ b/arch/arm64/crypto/aes-neon.S
@@ -32,10 +32,10 @@
 
 	/* preload the entire Sbox */
 	.macro		prepare, sbox, shiftrows, temp
-	adr		\temp, \sbox
 	movi		v12.16b, #0x1b
-	ldr		q13, \shiftrows
-	ldr		q14, .Lror32by8
+	ldr_l		q13, \shiftrows, \temp
+	ldr_l		q14, .Lror32by8, \temp
+	adr_l		\temp, \sbox
 	ld1		{v16.16b-v19.16b}, [\temp], #64
 	ld1		{v20.16b-v23.16b}, [\temp], #64
 	ld1		{v24.16b-v27.16b}, [\temp], #64
@@ -272,7 +272,7 @@
 
 #include "aes-modes.S"
 
-	.text
+	.section	".rodata", "a"
 	.align		6
 .LForward_Sbox:
 	.byte		0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
diff --git a/arch/arm64/crypto/crc32-ce-core.S b/arch/arm64/crypto/crc32-ce-core.S
index 18f5a8442276..16ed3c7ebd37 100644
--- a/arch/arm64/crypto/crc32-ce-core.S
+++ b/arch/arm64/crypto/crc32-ce-core.S
@@ -50,7 +50,7 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 
-	.text
+	.section	".rodata", "a"
 	.align		6
 	.cpu		generic+crypto+crc
 
@@ -115,12 +115,13 @@
 	 * uint crc32_pmull_le(unsigned char const *buffer,
 	 *                     size_t len, uint crc32)
 	 */
+	.text
 ENTRY(crc32_pmull_le)
-	adr		x3, .Lcrc32_constants
+	adr_l		x3, .Lcrc32_constants
 	b		0f
 
 ENTRY(crc32c_pmull_le)
-	adr		x3, .Lcrc32c_constants
+	adr_l		x3, .Lcrc32c_constants
 
 0:	bic		LEN, LEN, #15
 	ld1		{v1.16b-v4.16b}, [BUF], #0x40
diff --git a/arch/arm64/crypto/crc32-ce-glue.c b/arch/arm64/crypto/crc32-ce-glue.c
index 624f4137918c..34b4e3d46aab 100644
--- a/arch/arm64/crypto/crc32-ce-glue.c
+++ b/arch/arm64/crypto/crc32-ce-glue.c
@@ -185,6 +185,7 @@ static struct shash_alg crc32_pmull_algs[] = { {
 	.base.cra_name		= "crc32",
 	.base.cra_driver_name	= "crc32-arm64-ce",
 	.base.cra_priority	= 200,
+	.base.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 	.base.cra_blocksize	= 1,
 	.base.cra_module	= THIS_MODULE,
 }, {
@@ -200,6 +201,7 @@ static struct shash_alg crc32_pmull_algs[] = { {
 	.base.cra_name		= "crc32c",
 	.base.cra_driver_name	= "crc32c-arm64-ce",
 	.base.cra_priority	= 200,
+	.base.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 	.base.cra_blocksize	= 1,
 	.base.cra_module	= THIS_MODULE,
 } };
diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S
index d5b5a8c038c8..f179c01bd55c 100644
--- a/arch/arm64/crypto/crct10dif-ce-core.S
+++ b/arch/arm64/crypto/crct10dif-ce-core.S
@@ -128,7 +128,7 @@ CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
 	// XOR the initial_crc value
 	eor		v0.16b, v0.16b, v10.16b
 
-	ldr		q10, rk3	// xmm10 has rk3 and rk4
+	ldr_l		q10, rk3, x8	// xmm10 has rk3 and rk4
 					// type of pmull instruction
 					// will determine which constant to use
 
@@ -184,13 +184,13 @@ CPU_LE(	ext		v12.16b, v12.16b, v12.16b, #8	)
 	// fold the 8 vector registers to 1 vector register with different
 	// constants
 
-	ldr		q10, rk9
+	ldr_l		q10, rk9, x8
 
 	.macro		fold16, reg, rk
 	pmull		v8.1q, \reg\().1d, v10.1d
 	pmull2		\reg\().1q, \reg\().2d, v10.2d
 	.ifnb		\rk
-	ldr		q10, \rk
+	ldr_l		q10, \rk, x8
 	.endif
 	eor		v7.16b, v7.16b, v8.16b
 	eor		v7.16b, v7.16b, \reg\().16b
@@ -251,7 +251,7 @@ CPU_LE(	ext		v1.16b, v1.16b, v1.16b, #8	)
 
 	// get rid of the extra data that was loaded before
 	// load the shift constant
-	adr		x4, tbl_shf_table + 16
+	adr_l		x4, tbl_shf_table + 16
 	sub		x4, x4, arg3
 	ld1		{v0.16b}, [x4]
 
@@ -275,7 +275,7 @@ CPU_LE(	ext		v1.16b, v1.16b, v1.16b, #8	)
 
 _128_done:
 	// compute crc of a 128-bit value
-	ldr		q10, rk5		// rk5 and rk6 in xmm10
+	ldr_l		q10, rk5, x8		// rk5 and rk6 in xmm10
 
 	// 64b fold
 	ext		v0.16b, vzr.16b, v7.16b, #8
@@ -291,7 +291,7 @@ _128_done:
 
 	// barrett reduction
 _barrett:
-	ldr		q10, rk7
+	ldr_l		q10, rk7, x8
 	mov		v0.d[0], v7.d[1]
 
 	pmull		v0.1q, v0.1d, v10.1d
@@ -321,7 +321,7 @@ CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
 	b.eq		_128_done		// exactly 16 left
 	b.lt		_less_than_16_left
 
-	ldr		q10, rk1		// rk1 and rk2 in xmm10
+	ldr_l		q10, rk1, x8		// rk1 and rk2 in xmm10
 
 	// update the counter. subtract 32 instead of 16 to save one
 	// instruction from the loop
@@ -333,7 +333,7 @@ CPU_LE(	ext		v7.16b, v7.16b, v7.16b, #8	)
 
 _less_than_16_left:
 	// shl r9, 4
-	adr		x0, tbl_shf_table + 16
+	adr_l		x0, tbl_shf_table + 16
 	sub		x0, x0, arg3
 	ld1		{v0.16b}, [x0]
 	movi		v9.16b, #0x80
@@ -345,6 +345,7 @@ ENDPROC(crc_t10dif_pmull)
 // precomputed constants
 // these constants are precomputed from the poly:
 // 0x8bb70000 (0x8bb7 scaled to 32 bits)
+	.section	".rodata", "a"
 	.align		4
 // Q = 0x18BB70000
 // rk1 = 2^(32*3) mod Q << 32
diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S
index 8550408735a0..46049850727d 100644
--- a/arch/arm64/crypto/sha1-ce-core.S
+++ b/arch/arm64/crypto/sha1-ce-core.S
@@ -58,12 +58,11 @@
 	sha1su1		v\s0\().4s, v\s3\().4s
 	.endm
 
-	/*
-	 * The SHA1 round constants
-	 */
-	.align		4
-.Lsha1_rcon:
-	.word		0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6
+	.macro		loadrc, k, val, tmp
+	movz		\tmp, :abs_g0_nc:\val
+	movk		\tmp, :abs_g1:\val
+	dup		\k, \tmp
+	.endm
 
 	/*
 	 * void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src,
@@ -71,11 +70,10 @@
 	 */
 ENTRY(sha1_ce_transform)
 	/* load round constants */
-	adr		x6, .Lsha1_rcon
-	ld1r		{k0.4s}, [x6], #4
-	ld1r		{k1.4s}, [x6], #4
-	ld1r		{k2.4s}, [x6], #4
-	ld1r		{k3.4s}, [x6]
+	loadrc		k0.4s, 0x5a827999, w6
+	loadrc		k1.4s, 0x6ed9eba1, w6
+	loadrc		k2.4s, 0x8f1bbcdc, w6
+	loadrc		k3.4s, 0xca62c1d6, w6
 
 	/* load state */
 	ld1		{dgav.4s}, [x0]
diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S
index 679c6c002f4f..4c3c89b812ce 100644
--- a/arch/arm64/crypto/sha2-ce-core.S
+++ b/arch/arm64/crypto/sha2-ce-core.S
@@ -53,6 +53,7 @@
 	/*
 	 * The SHA-256 round constants
 	 */
+	.section	".rodata", "a"
 	.align		4
 .Lsha2_rcon:
 	.word		0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5
@@ -76,9 +77,10 @@
 	 * void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src,
 	 *			  int blocks)
 	 */
+	.text
 ENTRY(sha2_ce_transform)
 	/* load round constants */
-	adr		x8, .Lsha2_rcon
+	adr_l		x8, .Lsha2_rcon
 	ld1		{ v0.4s- v3.4s}, [x8], #64
 	ld1		{ v4.4s- v7.4s}, [x8], #64
 	ld1		{ v8.4s-v11.4s}, [x8], #64
diff --git a/arch/arm64/crypto/sha3-ce-core.S b/arch/arm64/crypto/sha3-ce-core.S
new file mode 100644
index 000000000000..332ad7530690
--- /dev/null
+++ b/arch/arm64/crypto/sha3-ce-core.S
@@ -0,0 +1,210 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * sha3-ce-core.S - core SHA-3 transform using v8.2 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.irp	b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
+	.set	.Lv\b\().2d, \b
+	.set	.Lv\b\().16b, \b
+	.endr
+
+	/*
+	 * ARMv8.2 Crypto Extensions instructions
+	 */
+	.macro	eor3, rd, rn, rm, ra
+	.inst	0xce000000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
+	.endm
+
+	.macro	rax1, rd, rn, rm
+	.inst	0xce608c00 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro	bcax, rd, rn, rm, ra
+	.inst	0xce200000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
+	.endm
+
+	.macro	xar, rd, rn, rm, imm6
+	.inst	0xce800000 | .L\rd | (.L\rn << 5) | ((\imm6) << 10) | (.L\rm << 16)
+	.endm
+
+	/*
+	 * sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size)
+	 */
+	.text
+ENTRY(sha3_ce_transform)
+	/* load state */
+	add	x8, x0, #32
+	ld1	{ v0.1d- v3.1d}, [x0]
+	ld1	{ v4.1d- v7.1d}, [x8], #32
+	ld1	{ v8.1d-v11.1d}, [x8], #32
+	ld1	{v12.1d-v15.1d}, [x8], #32
+	ld1	{v16.1d-v19.1d}, [x8], #32
+	ld1	{v20.1d-v23.1d}, [x8], #32
+	ld1	{v24.1d}, [x8]
+
+0:	sub	w2, w2, #1
+	mov	w8, #24
+	adr_l	x9, .Lsha3_rcon
+
+	/* load input */
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	ld1	{v29.8b-v31.8b}, [x1], #24
+	eor	v0.8b, v0.8b, v25.8b
+	eor	v1.8b, v1.8b, v26.8b
+	eor	v2.8b, v2.8b, v27.8b
+	eor	v3.8b, v3.8b, v28.8b
+	eor	v4.8b, v4.8b, v29.8b
+	eor	v5.8b, v5.8b, v30.8b
+	eor	v6.8b, v6.8b, v31.8b
+
+	tbnz	x3, #6, 2f		// SHA3-512
+
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	ld1	{v29.8b-v30.8b}, [x1], #16
+	eor	 v7.8b,  v7.8b, v25.8b
+	eor	 v8.8b,  v8.8b, v26.8b
+	eor	 v9.8b,  v9.8b, v27.8b
+	eor	v10.8b, v10.8b, v28.8b
+	eor	v11.8b, v11.8b, v29.8b
+	eor	v12.8b, v12.8b, v30.8b
+
+	tbnz	x3, #4, 1f		// SHA3-384 or SHA3-224
+
+	// SHA3-256
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	eor	v13.8b, v13.8b, v25.8b
+	eor	v14.8b, v14.8b, v26.8b
+	eor	v15.8b, v15.8b, v27.8b
+	eor	v16.8b, v16.8b, v28.8b
+	b	3f
+
+1:	tbz	x3, #2, 3f		// bit 2 cleared? SHA-384
+
+	// SHA3-224
+	ld1	{v25.8b-v28.8b}, [x1], #32
+	ld1	{v29.8b}, [x1], #8
+	eor	v13.8b, v13.8b, v25.8b
+	eor	v14.8b, v14.8b, v26.8b
+	eor	v15.8b, v15.8b, v27.8b
+	eor	v16.8b, v16.8b, v28.8b
+	eor	v17.8b, v17.8b, v29.8b
+	b	3f
+
+	// SHA3-512
+2:	ld1	{v25.8b-v26.8b}, [x1], #16
+	eor	 v7.8b,  v7.8b, v25.8b
+	eor	 v8.8b,  v8.8b, v26.8b
+
+3:	sub	w8, w8, #1
+
+	eor3	v29.16b,  v4.16b,  v9.16b, v14.16b
+	eor3	v26.16b,  v1.16b,  v6.16b, v11.16b
+	eor3	v28.16b,  v3.16b,  v8.16b, v13.16b
+	eor3	v25.16b,  v0.16b,  v5.16b, v10.16b
+	eor3	v27.16b,  v2.16b,  v7.16b, v12.16b
+	eor3	v29.16b, v29.16b, v19.16b, v24.16b
+	eor3	v26.16b, v26.16b, v16.16b, v21.16b
+	eor3	v28.16b, v28.16b, v18.16b, v23.16b
+	eor3	v25.16b, v25.16b, v15.16b, v20.16b
+	eor3	v27.16b, v27.16b, v17.16b, v22.16b
+
+	rax1	v30.2d, v29.2d, v26.2d	// bc[0]
+	rax1	v26.2d, v26.2d, v28.2d	// bc[2]
+	rax1	v28.2d, v28.2d, v25.2d	// bc[4]
+	rax1	v25.2d, v25.2d, v27.2d	// bc[1]
+	rax1	v27.2d, v27.2d, v29.2d	// bc[3]
+
+	eor	 v0.16b,  v0.16b, v30.16b
+	xar	 v29.2d,   v1.2d,  v25.2d, (64 - 1)
+	xar	  v1.2d,   v6.2d,  v25.2d, (64 - 44)
+	xar	  v6.2d,   v9.2d,  v28.2d, (64 - 20)
+	xar	  v9.2d,  v22.2d,  v26.2d, (64 - 61)
+	xar	 v22.2d,  v14.2d,  v28.2d, (64 - 39)
+	xar	 v14.2d,  v20.2d,  v30.2d, (64 - 18)
+	xar	 v31.2d,   v2.2d,  v26.2d, (64 - 62)
+	xar	  v2.2d,  v12.2d,  v26.2d, (64 - 43)
+	xar	 v12.2d,  v13.2d,  v27.2d, (64 - 25)
+	xar	 v13.2d,  v19.2d,  v28.2d, (64 - 8)
+	xar	 v19.2d,  v23.2d,  v27.2d, (64 - 56)
+	xar	 v23.2d,  v15.2d,  v30.2d, (64 - 41)
+	xar	 v15.2d,   v4.2d,  v28.2d, (64 - 27)
+	xar	 v28.2d,  v24.2d,  v28.2d, (64 - 14)
+	xar	 v24.2d,  v21.2d,  v25.2d, (64 - 2)
+	xar	  v8.2d,   v8.2d,  v27.2d, (64 - 55)
+	xar	  v4.2d,  v16.2d,  v25.2d, (64 - 45)
+	xar	 v16.2d,   v5.2d,  v30.2d, (64 - 36)
+	xar	  v5.2d,   v3.2d,  v27.2d, (64 - 28)
+	xar	 v27.2d,  v18.2d,  v27.2d, (64 - 21)
+	xar	  v3.2d,  v17.2d,  v26.2d, (64 - 15)
+	xar	 v25.2d,  v11.2d,  v25.2d, (64 - 10)
+	xar	 v26.2d,   v7.2d,  v26.2d, (64 - 6)
+	xar	 v30.2d,  v10.2d,  v30.2d, (64 - 3)
+
+	bcax	v20.16b, v31.16b, v22.16b,  v8.16b
+	bcax	v21.16b,  v8.16b, v23.16b, v22.16b
+	bcax	v22.16b, v22.16b, v24.16b, v23.16b
+	bcax	v23.16b, v23.16b, v31.16b, v24.16b
+	bcax	v24.16b, v24.16b,  v8.16b, v31.16b
+
+	ld1r	{v31.2d}, [x9], #8
+
+	bcax	v17.16b, v25.16b, v19.16b,  v3.16b
+	bcax	v18.16b,  v3.16b, v15.16b, v19.16b
+	bcax	v19.16b, v19.16b, v16.16b, v15.16b
+	bcax	v15.16b, v15.16b, v25.16b, v16.16b
+	bcax	v16.16b, v16.16b,  v3.16b, v25.16b
+
+	bcax	v10.16b, v29.16b, v12.16b, v26.16b
+	bcax	v11.16b, v26.16b, v13.16b, v12.16b
+	bcax	v12.16b, v12.16b, v14.16b, v13.16b
+	bcax	v13.16b, v13.16b, v29.16b, v14.16b
+	bcax	v14.16b, v14.16b, v26.16b, v29.16b
+
+	bcax	 v7.16b, v30.16b,  v9.16b,  v4.16b
+	bcax	 v8.16b,  v4.16b,  v5.16b,  v9.16b
+	bcax	 v9.16b,  v9.16b,  v6.16b,  v5.16b
+	bcax	 v5.16b,  v5.16b, v30.16b,  v6.16b
+	bcax	 v6.16b,  v6.16b,  v4.16b, v30.16b
+
+	bcax	 v3.16b, v27.16b,  v0.16b, v28.16b
+	bcax	 v4.16b, v28.16b,  v1.16b,  v0.16b
+	bcax	 v0.16b,  v0.16b,  v2.16b,  v1.16b
+	bcax	 v1.16b,  v1.16b, v27.16b,  v2.16b
+	bcax	 v2.16b,  v2.16b, v28.16b, v27.16b
+
+	eor	 v0.16b,  v0.16b, v31.16b
+
+	cbnz	w8, 3b
+	cbnz	w2, 0b
+
+	/* save state */
+	st1	{ v0.1d- v3.1d}, [x0], #32
+	st1	{ v4.1d- v7.1d}, [x0], #32
+	st1	{ v8.1d-v11.1d}, [x0], #32
+	st1	{v12.1d-v15.1d}, [x0], #32
+	st1	{v16.1d-v19.1d}, [x0], #32
+	st1	{v20.1d-v23.1d}, [x0], #32
+	st1	{v24.1d}, [x0]
+	ret
+ENDPROC(sha3_ce_transform)
+
+	.section	".rodata", "a"
+	.align		8
+.Lsha3_rcon:
+	.quad	0x0000000000000001, 0x0000000000008082, 0x800000000000808a
+	.quad	0x8000000080008000, 0x000000000000808b, 0x0000000080000001
+	.quad	0x8000000080008081, 0x8000000000008009, 0x000000000000008a
+	.quad	0x0000000000000088, 0x0000000080008009, 0x000000008000000a
+	.quad	0x000000008000808b, 0x800000000000008b, 0x8000000000008089
+	.quad	0x8000000000008003, 0x8000000000008002, 0x8000000000000080
+	.quad	0x000000000000800a, 0x800000008000000a, 0x8000000080008081
+	.quad	0x8000000000008080, 0x0000000080000001, 0x8000000080008008
diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c
new file mode 100644
index 000000000000..da8222e528bd
--- /dev/null
+++ b/arch/arm64/crypto/sha3-ce-glue.c
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * sha3-ce-glue.c - core SHA-3 transform using v8.2 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <asm/unaligned.h>
+#include <crypto/internal/hash.h>
+#include <crypto/sha3.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+asmlinkage void sha3_ce_transform(u64 *st, const u8 *data, int blocks,
+				  int md_len);
+
+static int sha3_update(struct shash_desc *desc, const u8 *data,
+		       unsigned int len)
+{
+	struct sha3_state *sctx = shash_desc_ctx(desc);
+	unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
+
+	if (!may_use_simd())
+		return crypto_sha3_update(desc, data, len);
+
+	if ((sctx->partial + len) >= sctx->rsiz) {
+		int blocks;
+
+		if (sctx->partial) {
+			int p = sctx->rsiz - sctx->partial;
+
+			memcpy(sctx->buf + sctx->partial, data, p);
+			kernel_neon_begin();
+			sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size);
+			kernel_neon_end();
+
+			data += p;
+			len -= p;
+			sctx->partial = 0;
+		}
+
+		blocks = len / sctx->rsiz;
+		len %= sctx->rsiz;
+
+		if (blocks) {
+			kernel_neon_begin();
+			sha3_ce_transform(sctx->st, data, blocks, digest_size);
+			kernel_neon_end();
+			data += blocks * sctx->rsiz;
+		}
+	}
+
+	if (len) {
+		memcpy(sctx->buf + sctx->partial, data, len);
+		sctx->partial += len;
+	}
+	return 0;
+}
+
+static int sha3_final(struct shash_desc *desc, u8 *out)
+{
+	struct sha3_state *sctx = shash_desc_ctx(desc);
+	unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
+	__le64 *digest = (__le64 *)out;
+	int i;
+
+	if (!may_use_simd())
+		return crypto_sha3_final(desc, out);
+
+	sctx->buf[sctx->partial++] = 0x06;
+	memset(sctx->buf + sctx->partial, 0, sctx->rsiz - sctx->partial);
+	sctx->buf[sctx->rsiz - 1] |= 0x80;
+
+	kernel_neon_begin();
+	sha3_ce_transform(sctx->st, sctx->buf, 1, digest_size);
+	kernel_neon_end();
+
+	for (i = 0; i < digest_size / 8; i++)
+		put_unaligned_le64(sctx->st[i], digest++);
+
+	if (digest_size & 4)
+		put_unaligned_le32(sctx->st[i], (__le32 *)digest);
+
+	*sctx = (struct sha3_state){};
+	return 0;
+}
+
+static struct shash_alg algs[] = { {
+	.digestsize		= SHA3_224_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= sha3_update,
+	.final			= sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-224",
+	.base.cra_driver_name	= "sha3-224-ce",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_224_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+	.base.cra_priority	= 200,
+}, {
+	.digestsize		= SHA3_256_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= sha3_update,
+	.final			= sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-256",
+	.base.cra_driver_name	= "sha3-256-ce",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_256_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+	.base.cra_priority	= 200,
+}, {
+	.digestsize		= SHA3_384_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= sha3_update,
+	.final			= sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-384",
+	.base.cra_driver_name	= "sha3-384-ce",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_384_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+	.base.cra_priority	= 200,
+}, {
+	.digestsize		= SHA3_512_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= sha3_update,
+	.final			= sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-512",
+	.base.cra_driver_name	= "sha3-512-ce",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_512_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+	.base.cra_priority	= 200,
+} };
+
+static int __init sha3_neon_mod_init(void)
+{
+	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
+}
+
+static void __exit sha3_neon_mod_fini(void)
+{
+	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+}
+
+module_cpu_feature_match(SHA3, sha3_neon_mod_init);
+module_exit(sha3_neon_mod_fini);
diff --git a/arch/arm64/crypto/sha512-ce-core.S b/arch/arm64/crypto/sha512-ce-core.S
new file mode 100644
index 000000000000..7f3bca5c59a2
--- /dev/null
+++ b/arch/arm64/crypto/sha512-ce-core.S
@@ -0,0 +1,204 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * sha512-ce-core.S - core SHA-384/SHA-512 transform using v8 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.irp		b,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19
+	.set		.Lq\b, \b
+	.set		.Lv\b\().2d, \b
+	.endr
+
+	.macro		sha512h, rd, rn, rm
+	.inst		0xce608000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro		sha512h2, rd, rn, rm
+	.inst		0xce608400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro		sha512su0, rd, rn
+	.inst		0xcec08000 | .L\rd | (.L\rn << 5)
+	.endm
+
+	.macro		sha512su1, rd, rn, rm
+	.inst		0xce608800 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	/*
+	 * The SHA-512 round constants
+	 */
+	.section	".rodata", "a"
+	.align		4
+.Lsha512_rcon:
+	.quad		0x428a2f98d728ae22, 0x7137449123ef65cd
+	.quad		0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc
+	.quad		0x3956c25bf348b538, 0x59f111f1b605d019
+	.quad		0x923f82a4af194f9b, 0xab1c5ed5da6d8118
+	.quad		0xd807aa98a3030242, 0x12835b0145706fbe
+	.quad		0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2
+	.quad		0x72be5d74f27b896f, 0x80deb1fe3b1696b1
+	.quad		0x9bdc06a725c71235, 0xc19bf174cf692694
+	.quad		0xe49b69c19ef14ad2, 0xefbe4786384f25e3
+	.quad		0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65
+	.quad		0x2de92c6f592b0275, 0x4a7484aa6ea6e483
+	.quad		0x5cb0a9dcbd41fbd4, 0x76f988da831153b5
+	.quad		0x983e5152ee66dfab, 0xa831c66d2db43210
+	.quad		0xb00327c898fb213f, 0xbf597fc7beef0ee4
+	.quad		0xc6e00bf33da88fc2, 0xd5a79147930aa725
+	.quad		0x06ca6351e003826f, 0x142929670a0e6e70
+	.quad		0x27b70a8546d22ffc, 0x2e1b21385c26c926
+	.quad		0x4d2c6dfc5ac42aed, 0x53380d139d95b3df
+	.quad		0x650a73548baf63de, 0x766a0abb3c77b2a8
+	.quad		0x81c2c92e47edaee6, 0x92722c851482353b
+	.quad		0xa2bfe8a14cf10364, 0xa81a664bbc423001
+	.quad		0xc24b8b70d0f89791, 0xc76c51a30654be30
+	.quad		0xd192e819d6ef5218, 0xd69906245565a910
+	.quad		0xf40e35855771202a, 0x106aa07032bbd1b8
+	.quad		0x19a4c116b8d2d0c8, 0x1e376c085141ab53
+	.quad		0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8
+	.quad		0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb
+	.quad		0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3
+	.quad		0x748f82ee5defb2fc, 0x78a5636f43172f60
+	.quad		0x84c87814a1f0ab72, 0x8cc702081a6439ec
+	.quad		0x90befffa23631e28, 0xa4506cebde82bde9
+	.quad		0xbef9a3f7b2c67915, 0xc67178f2e372532b
+	.quad		0xca273eceea26619c, 0xd186b8c721c0c207
+	.quad		0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178
+	.quad		0x06f067aa72176fba, 0x0a637dc5a2c898a6
+	.quad		0x113f9804bef90dae, 0x1b710b35131c471b
+	.quad		0x28db77f523047d84, 0x32caab7b40c72493
+	.quad		0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c
+	.quad		0x4cc5d4becb3e42b6, 0x597f299cfc657e2a
+	.quad		0x5fcb6fab3ad6faec, 0x6c44198c4a475817
+
+	.macro		dround, i0, i1, i2, i3, i4, rc0, rc1, in0, in1, in2, in3, in4
+	.ifnb		\rc1
+	ld1		{v\rc1\().2d}, [x4], #16
+	.endif
+	add		v5.2d, v\rc0\().2d, v\in0\().2d
+	ext		v6.16b, v\i2\().16b, v\i3\().16b, #8
+	ext		v5.16b, v5.16b, v5.16b, #8
+	ext		v7.16b, v\i1\().16b, v\i2\().16b, #8
+	add		v\i3\().2d, v\i3\().2d, v5.2d
+	.ifnb		\in1
+	ext		v5.16b, v\in3\().16b, v\in4\().16b, #8
+	sha512su0	v\in0\().2d, v\in1\().2d
+	.endif
+	sha512h		q\i3, q6, v7.2d
+	.ifnb		\in1
+	sha512su1	v\in0\().2d, v\in2\().2d, v5.2d
+	.endif
+	add		v\i4\().2d, v\i1\().2d, v\i3\().2d
+	sha512h2	q\i3, q\i1, v\i0\().2d
+	.endm
+
+	/*
+	 * void sha512_ce_transform(struct sha512_state *sst, u8 const *src,
+	 *			  int blocks)
+	 */
+	.text
+ENTRY(sha512_ce_transform)
+	/* load state */
+	ld1		{v8.2d-v11.2d}, [x0]
+
+	/* load first 4 round constants */
+	adr_l		x3, .Lsha512_rcon
+	ld1		{v20.2d-v23.2d}, [x3], #64
+
+	/* load input */
+0:	ld1		{v12.2d-v15.2d}, [x1], #64
+	ld1		{v16.2d-v19.2d}, [x1], #64
+	sub		w2, w2, #1
+
+CPU_LE(	rev64		v12.16b, v12.16b	)
+CPU_LE(	rev64		v13.16b, v13.16b	)
+CPU_LE(	rev64		v14.16b, v14.16b	)
+CPU_LE(	rev64		v15.16b, v15.16b	)
+CPU_LE(	rev64		v16.16b, v16.16b	)
+CPU_LE(	rev64		v17.16b, v17.16b	)
+CPU_LE(	rev64		v18.16b, v18.16b	)
+CPU_LE(	rev64		v19.16b, v19.16b	)
+
+	mov		x4, x3				// rc pointer
+
+	mov		v0.16b, v8.16b
+	mov		v1.16b, v9.16b
+	mov		v2.16b, v10.16b
+	mov		v3.16b, v11.16b
+
+	// v0  ab  cd  --  ef  gh  ab
+	// v1  cd  --  ef  gh  ab  cd
+	// v2  ef  gh  ab  cd  --  ef
+	// v3  gh  ab  cd  --  ef  gh
+	// v4  --  ef  gh  ab  cd  --
+
+	dround		0, 1, 2, 3, 4, 20, 24, 12, 13, 19, 16, 17
+	dround		3, 0, 4, 2, 1, 21, 25, 13, 14, 12, 17, 18
+	dround		2, 3, 1, 4, 0, 22, 26, 14, 15, 13, 18, 19
+	dround		4, 2, 0, 1, 3, 23, 27, 15, 16, 14, 19, 12
+	dround		1, 4, 3, 0, 2, 24, 28, 16, 17, 15, 12, 13
+
+	dround		0, 1, 2, 3, 4, 25, 29, 17, 18, 16, 13, 14
+	dround		3, 0, 4, 2, 1, 26, 30, 18, 19, 17, 14, 15
+	dround		2, 3, 1, 4, 0, 27, 31, 19, 12, 18, 15, 16
+	dround		4, 2, 0, 1, 3, 28, 24, 12, 13, 19, 16, 17
+	dround		1, 4, 3, 0, 2, 29, 25, 13, 14, 12, 17, 18
+
+	dround		0, 1, 2, 3, 4, 30, 26, 14, 15, 13, 18, 19
+	dround		3, 0, 4, 2, 1, 31, 27, 15, 16, 14, 19, 12
+	dround		2, 3, 1, 4, 0, 24, 28, 16, 17, 15, 12, 13
+	dround		4, 2, 0, 1, 3, 25, 29, 17, 18, 16, 13, 14
+	dround		1, 4, 3, 0, 2, 26, 30, 18, 19, 17, 14, 15
+
+	dround		0, 1, 2, 3, 4, 27, 31, 19, 12, 18, 15, 16
+	dround		3, 0, 4, 2, 1, 28, 24, 12, 13, 19, 16, 17
+	dround		2, 3, 1, 4, 0, 29, 25, 13, 14, 12, 17, 18
+	dround		4, 2, 0, 1, 3, 30, 26, 14, 15, 13, 18, 19
+	dround		1, 4, 3, 0, 2, 31, 27, 15, 16, 14, 19, 12
+
+	dround		0, 1, 2, 3, 4, 24, 28, 16, 17, 15, 12, 13
+	dround		3, 0, 4, 2, 1, 25, 29, 17, 18, 16, 13, 14
+	dround		2, 3, 1, 4, 0, 26, 30, 18, 19, 17, 14, 15
+	dround		4, 2, 0, 1, 3, 27, 31, 19, 12, 18, 15, 16
+	dround		1, 4, 3, 0, 2, 28, 24, 12, 13, 19, 16, 17
+
+	dround		0, 1, 2, 3, 4, 29, 25, 13, 14, 12, 17, 18
+	dround		3, 0, 4, 2, 1, 30, 26, 14, 15, 13, 18, 19
+	dround		2, 3, 1, 4, 0, 31, 27, 15, 16, 14, 19, 12
+	dround		4, 2, 0, 1, 3, 24, 28, 16, 17, 15, 12, 13
+	dround		1, 4, 3, 0, 2, 25, 29, 17, 18, 16, 13, 14
+
+	dround		0, 1, 2, 3, 4, 26, 30, 18, 19, 17, 14, 15
+	dround		3, 0, 4, 2, 1, 27, 31, 19, 12, 18, 15, 16
+	dround		2, 3, 1, 4, 0, 28, 24, 12
+	dround		4, 2, 0, 1, 3, 29, 25, 13
+	dround		1, 4, 3, 0, 2, 30, 26, 14
+
+	dround		0, 1, 2, 3, 4, 31, 27, 15
+	dround		3, 0, 4, 2, 1, 24,   , 16
+	dround		2, 3, 1, 4, 0, 25,   , 17
+	dround		4, 2, 0, 1, 3, 26,   , 18
+	dround		1, 4, 3, 0, 2, 27,   , 19
+
+	/* update state */
+	add		v8.2d, v8.2d, v0.2d
+	add		v9.2d, v9.2d, v1.2d
+	add		v10.2d, v10.2d, v2.2d
+	add		v11.2d, v11.2d, v3.2d
+
+	/* handled all input blocks? */
+	cbnz		w2, 0b
+
+	/* store new state */
+3:	st1		{v8.2d-v11.2d}, [x0]
+	ret
+ENDPROC(sha512_ce_transform)
diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c
new file mode 100644
index 000000000000..a77c8632a589
--- /dev/null
+++ b/arch/arm64/crypto/sha512-ce-glue.c
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * sha512-ce-glue.c - SHA-384/SHA-512 using ARMv8 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <asm/unaligned.h>
+#include <crypto/internal/hash.h>
+#include <crypto/sha.h>
+#include <crypto/sha512_base.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+asmlinkage void sha512_ce_transform(struct sha512_state *sst, u8 const *src,
+				    int blocks);
+
+asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks);
+
+static int sha512_ce_update(struct shash_desc *desc, const u8 *data,
+			    unsigned int len)
+{
+	if (!may_use_simd())
+		return sha512_base_do_update(desc, data, len,
+				(sha512_block_fn *)sha512_block_data_order);
+
+	kernel_neon_begin();
+	sha512_base_do_update(desc, data, len,
+			      (sha512_block_fn *)sha512_ce_transform);
+	kernel_neon_end();
+
+	return 0;
+}
+
+static int sha512_ce_finup(struct shash_desc *desc, const u8 *data,
+			   unsigned int len, u8 *out)
+{
+	if (!may_use_simd()) {
+		if (len)
+			sha512_base_do_update(desc, data, len,
+				(sha512_block_fn *)sha512_block_data_order);
+		sha512_base_do_finalize(desc,
+				(sha512_block_fn *)sha512_block_data_order);
+		return sha512_base_finish(desc, out);
+	}
+
+	kernel_neon_begin();
+	sha512_base_do_update(desc, data, len,
+			      (sha512_block_fn *)sha512_ce_transform);
+	sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_ce_transform);
+	kernel_neon_end();
+	return sha512_base_finish(desc, out);
+}
+
+static int sha512_ce_final(struct shash_desc *desc, u8 *out)
+{
+	if (!may_use_simd()) {
+		sha512_base_do_finalize(desc,
+				(sha512_block_fn *)sha512_block_data_order);
+		return sha512_base_finish(desc, out);
+	}
+
+	kernel_neon_begin();
+	sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_ce_transform);
+	kernel_neon_end();
+	return sha512_base_finish(desc, out);
+}
+
+static struct shash_alg algs[] = { {
+	.init			= sha384_base_init,
+	.update			= sha512_ce_update,
+	.final			= sha512_ce_final,
+	.finup			= sha512_ce_finup,
+	.descsize		= sizeof(struct sha512_state),
+	.digestsize		= SHA384_DIGEST_SIZE,
+	.base.cra_name		= "sha384",
+	.base.cra_driver_name	= "sha384-ce",
+	.base.cra_priority	= 200,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA512_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.init			= sha512_base_init,
+	.update			= sha512_ce_update,
+	.final			= sha512_ce_final,
+	.finup			= sha512_ce_finup,
+	.descsize		= sizeof(struct sha512_state),
+	.digestsize		= SHA512_DIGEST_SIZE,
+	.base.cra_name		= "sha512",
+	.base.cra_driver_name	= "sha512-ce",
+	.base.cra_priority	= 200,
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA512_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+} };
+
+static int __init sha512_ce_mod_init(void)
+{
+	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
+}
+
+static void __exit sha512_ce_mod_fini(void)
+{
+	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
+}
+
+module_cpu_feature_match(SHA512, sha512_ce_mod_init);
+module_exit(sha512_ce_mod_fini);
diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c
index aff35c9992a4..27db4851e380 100644
--- a/arch/arm64/crypto/sha512-glue.c
+++ b/arch/arm64/crypto/sha512-glue.c
@@ -27,6 +27,7 @@ MODULE_ALIAS_CRYPTO("sha512");
 
 asmlinkage void sha512_block_data_order(u32 *digest, const void *data,
 					unsigned int num_blks);
+EXPORT_SYMBOL(sha512_block_data_order);
 
 static int sha512_update(struct shash_desc *desc, const u8 *data,
 			 unsigned int len)
diff --git a/arch/arm64/crypto/sm3-ce-core.S b/arch/arm64/crypto/sm3-ce-core.S
new file mode 100644
index 000000000000..27169fe07a68
--- /dev/null
+++ b/arch/arm64/crypto/sm3-ce-core.S
@@ -0,0 +1,141 @@
+/*
+ * sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+	.irp		b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
+	.set		.Lv\b\().4s, \b
+	.endr
+
+	.macro		sm3partw1, rd, rn, rm
+	.inst		0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3partw2, rd, rn, rm
+	.inst		0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3ss1, rd, rn, rm, ra
+	.inst		0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt1a, rd, rn, rm, imm2
+	.inst		0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt1b, rd, rn, rm, imm2
+	.inst		0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt2a, rd, rn, rm, imm2
+	.inst		0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		sm3tt2b, rd, rn, rm, imm2
+	.inst		0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
+	.endm
+
+	.macro		round, ab, s0, t0, t1, i
+	sm3ss1		v5.4s, v8.4s, \t0\().4s, v9.4s
+	shl		\t1\().4s, \t0\().4s, #1
+	sri		\t1\().4s, \t0\().4s, #31
+	sm3tt1\ab	v8.4s, v5.4s, v10.4s, \i
+	sm3tt2\ab	v9.4s, v5.4s, \s0\().4s, \i
+	.endm
+
+	.macro		qround, ab, s0, s1, s2, s3, s4
+	.ifnb		\s4
+	ext		\s4\().16b, \s1\().16b, \s2\().16b, #12
+	ext		v6.16b, \s0\().16b, \s1\().16b, #12
+	ext		v7.16b, \s2\().16b, \s3\().16b, #8
+	sm3partw1	\s4\().4s, \s0\().4s, \s3\().4s
+	.endif
+
+	eor		v10.16b, \s0\().16b, \s1\().16b
+
+	round		\ab, \s0, v11, v12, 0
+	round		\ab, \s0, v12, v11, 1
+	round		\ab, \s0, v11, v12, 2
+	round		\ab, \s0, v12, v11, 3
+
+	.ifnb		\s4
+	sm3partw2	\s4\().4s, v7.4s, v6.4s
+	.endif
+	.endm
+
+	/*
+	 * void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
+	 *                       int blocks)
+	 */
+	.text
+ENTRY(sm3_ce_transform)
+	/* load state */
+	ld1		{v8.4s-v9.4s}, [x0]
+	rev64		v8.4s, v8.4s
+	rev64		v9.4s, v9.4s
+	ext		v8.16b, v8.16b, v8.16b, #8
+	ext		v9.16b, v9.16b, v9.16b, #8
+
+	adr_l		x8, .Lt
+	ldp		s13, s14, [x8]
+
+	/* load input */
+0:	ld1		{v0.16b-v3.16b}, [x1], #64
+	sub		w2, w2, #1
+
+	mov		v15.16b, v8.16b
+	mov		v16.16b, v9.16b
+
+CPU_LE(	rev32		v0.16b, v0.16b		)
+CPU_LE(	rev32		v1.16b, v1.16b		)
+CPU_LE(	rev32		v2.16b, v2.16b		)
+CPU_LE(	rev32		v3.16b, v3.16b		)
+
+	ext		v11.16b, v13.16b, v13.16b, #4
+
+	qround		a, v0, v1, v2, v3, v4
+	qround		a, v1, v2, v3, v4, v0
+	qround		a, v2, v3, v4, v0, v1
+	qround		a, v3, v4, v0, v1, v2
+
+	ext		v11.16b, v14.16b, v14.16b, #4
+
+	qround		b, v4, v0, v1, v2, v3
+	qround		b, v0, v1, v2, v3, v4
+	qround		b, v1, v2, v3, v4, v0
+	qround		b, v2, v3, v4, v0, v1
+	qround		b, v3, v4, v0, v1, v2
+	qround		b, v4, v0, v1, v2, v3
+	qround		b, v0, v1, v2, v3, v4
+	qround		b, v1, v2, v3, v4, v0
+	qround		b, v2, v3, v4, v0, v1
+	qround		b, v3, v4
+	qround		b, v4, v0
+	qround		b, v0, v1
+
+	eor		v8.16b, v8.16b, v15.16b
+	eor		v9.16b, v9.16b, v16.16b
+
+	/* handled all input blocks? */
+	cbnz		w2, 0b
+
+	/* save state */
+	rev64		v8.4s, v8.4s
+	rev64		v9.4s, v9.4s
+	ext		v8.16b, v8.16b, v8.16b, #8
+	ext		v9.16b, v9.16b, v9.16b, #8
+	st1		{v8.4s-v9.4s}, [x0]
+	ret
+ENDPROC(sm3_ce_transform)
+
+	.section	".rodata", "a"
+	.align		3
+.Lt:	.word		0x79cc4519, 0x9d8a7a87
diff --git a/arch/arm64/crypto/sm3-ce-glue.c b/arch/arm64/crypto/sm3-ce-glue.c
new file mode 100644
index 000000000000..3b4948f7e26f
--- /dev/null
+++ b/arch/arm64/crypto/sm3-ce-glue.c
@@ -0,0 +1,92 @@
+/*
+ * sm3-ce-glue.c - SM3 secure hash using ARMv8.2 Crypto Extensions
+ *
+ * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <asm/simd.h>
+#include <asm/unaligned.h>
+#include <crypto/internal/hash.h>
+#include <crypto/sm3.h>
+#include <crypto/sm3_base.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+MODULE_DESCRIPTION("SM3 secure hash using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+asmlinkage void sm3_ce_transform(struct sm3_state *sst, u8 const *src,
+				 int blocks);
+
+static int sm3_ce_update(struct shash_desc *desc, const u8 *data,
+			 unsigned int len)
+{
+	if (!may_use_simd())
+		return crypto_sm3_update(desc, data, len);
+
+	kernel_neon_begin();
+	sm3_base_do_update(desc, data, len, sm3_ce_transform);
+	kernel_neon_end();
+
+	return 0;
+}
+
+static int sm3_ce_final(struct shash_desc *desc, u8 *out)
+{
+	if (!may_use_simd())
+		return crypto_sm3_finup(desc, NULL, 0, out);
+
+	kernel_neon_begin();
+	sm3_base_do_finalize(desc, sm3_ce_transform);
+	kernel_neon_end();
+
+	return sm3_base_finish(desc, out);
+}
+
+static int sm3_ce_finup(struct shash_desc *desc, const u8 *data,
+			unsigned int len, u8 *out)
+{
+	if (!may_use_simd())
+		return crypto_sm3_finup(desc, data, len, out);
+
+	kernel_neon_begin();
+	sm3_base_do_update(desc, data, len, sm3_ce_transform);
+	kernel_neon_end();
+
+	return sm3_ce_final(desc, out);
+}
+
+static struct shash_alg sm3_alg = {
+	.digestsize		= SM3_DIGEST_SIZE,
+	.init			= sm3_base_init,
+	.update			= sm3_ce_update,
+	.final			= sm3_ce_final,
+	.finup			= sm3_ce_finup,
+	.descsize		= sizeof(struct sm3_state),
+	.base.cra_name		= "sm3",
+	.base.cra_driver_name	= "sm3-ce",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SM3_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+	.base.cra_priority	= 200,
+};
+
+static int __init sm3_ce_mod_init(void)
+{
+	return crypto_register_shash(&sm3_alg);
+}
+
+static void __exit sm3_ce_mod_fini(void)
+{
+	crypto_unregister_shash(&sm3_alg);
+}
+
+module_cpu_feature_match(SM3, sm3_ce_mod_init);
+module_exit(sm3_ce_mod_fini);
diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild
index e63d0a8312de..3a9b84d39d71 100644
--- a/arch/arm64/include/asm/Kbuild
+++ b/arch/arm64/include/asm/Kbuild
@@ -1,5 +1,4 @@
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += delay.h
 generic-y += div64.h
 generic-y += dma.h
diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h
index 4a85c6952a22..669028172fd6 100644
--- a/arch/arm64/include/asm/alternative.h
+++ b/arch/arm64/include/asm/alternative.h
@@ -12,6 +12,8 @@
 #include <linux/stddef.h>
 #include <linux/stringify.h>
 
+extern int alternatives_applied;
+
 struct alt_instr {
 	s32 orig_offset;	/* offset to original instruction */
 	s32 alt_offset;		/* offset to replacement instruction */
diff --git a/arch/arm64/include/asm/arm_dsu_pmu.h b/arch/arm64/include/asm/arm_dsu_pmu.h
new file mode 100644
index 000000000000..82e5cc3356bf
--- /dev/null
+++ b/arch/arm64/include/asm/arm_dsu_pmu.h
@@ -0,0 +1,129 @@
+/*
+ * ARM DynamIQ Shared Unit (DSU) PMU Low level register access routines.
+ *
+ * Copyright (C) ARM Limited, 2017.
+ *
+ * Author: Suzuki K Poulose <suzuki.poulose@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2, as published by the Free Software Foundation.
+ */
+
+#include <linux/bitops.h>
+#include <linux/build_bug.h>
+#include <linux/compiler.h>
+#include <linux/types.h>
+#include <asm/barrier.h>
+#include <asm/sysreg.h>
+
+
+#define CLUSTERPMCR_EL1			sys_reg(3, 0, 15, 5, 0)
+#define CLUSTERPMCNTENSET_EL1		sys_reg(3, 0, 15, 5, 1)
+#define CLUSTERPMCNTENCLR_EL1		sys_reg(3, 0, 15, 5, 2)
+#define CLUSTERPMOVSSET_EL1		sys_reg(3, 0, 15, 5, 3)
+#define CLUSTERPMOVSCLR_EL1		sys_reg(3, 0, 15, 5, 4)
+#define CLUSTERPMSELR_EL1		sys_reg(3, 0, 15, 5, 5)
+#define CLUSTERPMINTENSET_EL1		sys_reg(3, 0, 15, 5, 6)
+#define CLUSTERPMINTENCLR_EL1		sys_reg(3, 0, 15, 5, 7)
+#define CLUSTERPMCCNTR_EL1		sys_reg(3, 0, 15, 6, 0)
+#define CLUSTERPMXEVTYPER_EL1		sys_reg(3, 0, 15, 6, 1)
+#define CLUSTERPMXEVCNTR_EL1		sys_reg(3, 0, 15, 6, 2)
+#define CLUSTERPMMDCR_EL1		sys_reg(3, 0, 15, 6, 3)
+#define CLUSTERPMCEID0_EL1		sys_reg(3, 0, 15, 6, 4)
+#define CLUSTERPMCEID1_EL1		sys_reg(3, 0, 15, 6, 5)
+
+static inline u32 __dsu_pmu_read_pmcr(void)
+{
+	return read_sysreg_s(CLUSTERPMCR_EL1);
+}
+
+static inline void __dsu_pmu_write_pmcr(u32 val)
+{
+	write_sysreg_s(val, CLUSTERPMCR_EL1);
+	isb();
+}
+
+static inline u32 __dsu_pmu_get_reset_overflow(void)
+{
+	u32 val = read_sysreg_s(CLUSTERPMOVSCLR_EL1);
+	/* Clear the bit */
+	write_sysreg_s(val, CLUSTERPMOVSCLR_EL1);
+	isb();
+	return val;
+}
+
+static inline void __dsu_pmu_select_counter(int counter)
+{
+	write_sysreg_s(counter, CLUSTERPMSELR_EL1);
+	isb();
+}
+
+static inline u64 __dsu_pmu_read_counter(int counter)
+{
+	__dsu_pmu_select_counter(counter);
+	return read_sysreg_s(CLUSTERPMXEVCNTR_EL1);
+}
+
+static inline void __dsu_pmu_write_counter(int counter, u64 val)
+{
+	__dsu_pmu_select_counter(counter);
+	write_sysreg_s(val, CLUSTERPMXEVCNTR_EL1);
+	isb();
+}
+
+static inline void __dsu_pmu_set_event(int counter, u32 event)
+{
+	__dsu_pmu_select_counter(counter);
+	write_sysreg_s(event, CLUSTERPMXEVTYPER_EL1);
+	isb();
+}
+
+static inline u64 __dsu_pmu_read_pmccntr(void)
+{
+	return read_sysreg_s(CLUSTERPMCCNTR_EL1);
+}
+
+static inline void __dsu_pmu_write_pmccntr(u64 val)
+{
+	write_sysreg_s(val, CLUSTERPMCCNTR_EL1);
+	isb();
+}
+
+static inline void __dsu_pmu_disable_counter(int counter)
+{
+	write_sysreg_s(BIT(counter), CLUSTERPMCNTENCLR_EL1);
+	isb();
+}
+
+static inline void __dsu_pmu_enable_counter(int counter)
+{
+	write_sysreg_s(BIT(counter), CLUSTERPMCNTENSET_EL1);
+	isb();
+}
+
+static inline void __dsu_pmu_counter_interrupt_enable(int counter)
+{
+	write_sysreg_s(BIT(counter), CLUSTERPMINTENSET_EL1);
+	isb();
+}
+
+static inline void __dsu_pmu_counter_interrupt_disable(int counter)
+{
+	write_sysreg_s(BIT(counter), CLUSTERPMINTENCLR_EL1);
+	isb();
+}
+
+
+static inline u32 __dsu_pmu_read_pmceid(int n)
+{
+	switch (n) {
+	case 0:
+		return read_sysreg_s(CLUSTERPMCEID0_EL1);
+	case 1:
+		return read_sysreg_s(CLUSTERPMCEID1_EL1);
+	default:
+		BUILD_BUG();
+		return 0;
+	}
+}
diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
index b3da6c886835..4128bec033f6 100644
--- a/arch/arm64/include/asm/asm-uaccess.h
+++ b/arch/arm64/include/asm/asm-uaccess.h
@@ -4,6 +4,7 @@
 
 #include <asm/alternative.h>
 #include <asm/kernel-pgtable.h>
+#include <asm/mmu.h>
 #include <asm/sysreg.h>
 #include <asm/assembler.h>
 
@@ -12,52 +13,63 @@
  */
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	.macro	__uaccess_ttbr0_disable, tmp1
-	mrs	\tmp1, ttbr1_el1		// swapper_pg_dir
-	add	\tmp1, \tmp1, #SWAPPER_DIR_SIZE	// reserved_ttbr0 at the end of swapper_pg_dir
-	msr	ttbr0_el1, \tmp1		// set reserved TTBR0_EL1
+	mrs	\tmp1, ttbr1_el1			// swapper_pg_dir
+	bic	\tmp1, \tmp1, #TTBR_ASID_MASK
+	sub	\tmp1, \tmp1, #RESERVED_TTBR0_SIZE	// reserved_ttbr0 just before swapper_pg_dir
+	msr	ttbr0_el1, \tmp1			// set reserved TTBR0_EL1
+	isb
+	add	\tmp1, \tmp1, #RESERVED_TTBR0_SIZE
+	msr	ttbr1_el1, \tmp1		// set reserved ASID
 	isb
 	.endm
 
-	.macro	__uaccess_ttbr0_enable, tmp1
+	.macro	__uaccess_ttbr0_enable, tmp1, tmp2
 	get_thread_info \tmp1
 	ldr	\tmp1, [\tmp1, #TSK_TI_TTBR0]	// load saved TTBR0_EL1
+	mrs	\tmp2, ttbr1_el1
+	extr    \tmp2, \tmp2, \tmp1, #48
+	ror     \tmp2, \tmp2, #16
+	msr	ttbr1_el1, \tmp2		// set the active ASID
+	isb
 	msr	ttbr0_el1, \tmp1		// set the non-PAN TTBR0_EL1
 	isb
 	.endm
 
-	.macro	uaccess_ttbr0_disable, tmp1
+	.macro	uaccess_ttbr0_disable, tmp1, tmp2
 alternative_if_not ARM64_HAS_PAN
+	save_and_disable_irq \tmp2		// avoid preemption
 	__uaccess_ttbr0_disable \tmp1
+	restore_irq \tmp2
 alternative_else_nop_endif
 	.endm
 
-	.macro	uaccess_ttbr0_enable, tmp1, tmp2
+	.macro	uaccess_ttbr0_enable, tmp1, tmp2, tmp3
 alternative_if_not ARM64_HAS_PAN
-	save_and_disable_irq \tmp2		// avoid preemption
-	__uaccess_ttbr0_enable \tmp1
-	restore_irq \tmp2
+	save_and_disable_irq \tmp3		// avoid preemption
+	__uaccess_ttbr0_enable \tmp1, \tmp2
+	restore_irq \tmp3
 alternative_else_nop_endif
 	.endm
 #else
-	.macro	uaccess_ttbr0_disable, tmp1
+	.macro	uaccess_ttbr0_disable, tmp1, tmp2
 	.endm
 
-	.macro	uaccess_ttbr0_enable, tmp1, tmp2
+	.macro	uaccess_ttbr0_enable, tmp1, tmp2, tmp3
 	.endm
 #endif
 
 /*
  * These macros are no-ops when UAO is present.
  */
-	.macro	uaccess_disable_not_uao, tmp1
-	uaccess_ttbr0_disable \tmp1
+	.macro	uaccess_disable_not_uao, tmp1, tmp2
+	uaccess_ttbr0_disable \tmp1, \tmp2
 alternative_if ARM64_ALT_PAN_NOT_UAO
 	SET_PSTATE_PAN(1)
 alternative_else_nop_endif
 	.endm
 
-	.macro	uaccess_enable_not_uao, tmp1, tmp2
-	uaccess_ttbr0_enable \tmp1, \tmp2
+	.macro	uaccess_enable_not_uao, tmp1, tmp2, tmp3
+	uaccess_ttbr0_enable \tmp1, \tmp2, \tmp3
 alternative_if ARM64_ALT_PAN_NOT_UAO
 	SET_PSTATE_PAN(0)
 alternative_else_nop_endif
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 8b168280976f..3c78835bba94 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -26,7 +26,6 @@
 #include <asm/asm-offsets.h>
 #include <asm/cpufeature.h>
 #include <asm/debug-monitors.h>
-#include <asm/mmu_context.h>
 #include <asm/page.h>
 #include <asm/pgtable-hwdef.h>
 #include <asm/ptrace.h>
@@ -109,6 +108,31 @@
 	dmb	\opt
 	.endm
 
+/*
+ * RAS Error Synchronization barrier
+ */
+	.macro  esb
+	hint    #16
+	.endm
+
+/*
+ * Value prediction barrier
+ */
+	.macro	csdb
+	hint	#20
+	.endm
+
+/*
+ * Sanitise a 64-bit bounded index wrt speculation, returning zero if out
+ * of bounds.
+ */
+	.macro	mask_nospec64, idx, limit, tmp
+	sub	\tmp, \idx, \limit
+	bic	\tmp, \tmp, \idx
+	and	\idx, \idx, \tmp, asr #63
+	csdb
+	.endm
+
 /*
  * NOP sequence
  */
@@ -255,7 +279,11 @@ lr	.req	x30		// link register
 #else
 	adr_l	\dst, \sym
 #endif
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 	mrs	\tmp, tpidr_el1
+alternative_else
+	mrs	\tmp, tpidr_el2
+alternative_endif
 	add	\dst, \dst, \tmp
 	.endm
 
@@ -266,7 +294,11 @@ lr	.req	x30		// link register
 	 */
 	.macro ldr_this_cpu dst, sym, tmp
 	adr_l	\dst, \sym
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 	mrs	\tmp, tpidr_el1
+alternative_else
+	mrs	\tmp, tpidr_el2
+alternative_endif
 	ldr	\dst, [\dst, \tmp]
 	.endm
 
@@ -344,10 +376,26 @@ alternative_endif
  * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map
  */
 	.macro	tcr_set_idmap_t0sz, valreg, tmpreg
-#ifndef CONFIG_ARM64_VA_BITS_48
 	ldr_l	\tmpreg, idmap_t0sz
 	bfi	\valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
-#endif
+	.endm
+
+/*
+ * tcr_compute_pa_size - set TCR.(I)PS to the highest supported
+ * ID_AA64MMFR0_EL1.PARange value
+ *
+ *	tcr:		register with the TCR_ELx value to be updated
+ *	pos:		IPS or PS bitfield position
+ *	tmp{0,1}:	temporary registers
+ */
+	.macro	tcr_compute_pa_size, tcr, pos, tmp0, tmp1
+	mrs	\tmp0, ID_AA64MMFR0_EL1
+	// Narrow PARange to fit the PS field in TCR_ELx
+	ubfx	\tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
+	mov	\tmp1, #ID_AA64MMFR0_PARANGE_MAX
+	cmp	\tmp0, \tmp1
+	csel	\tmp0, \tmp1, \tmp0, hi
+	bfi	\tcr, \tmp0, \pos, #3
 	.endm
 
 /*
@@ -387,6 +435,27 @@ alternative_endif
 	dsb	\domain
 	.endm
 
+/*
+ * Macro to perform an instruction cache maintenance for the interval
+ * [start, end)
+ *
+ * 	start, end:	virtual addresses describing the region
+ *	label:		A label to branch to on user fault.
+ * 	Corrupts:	tmp1, tmp2
+ */
+	.macro invalidate_icache_by_line start, end, tmp1, tmp2, label
+	icache_line_size \tmp1, \tmp2
+	sub	\tmp2, \tmp1, #1
+	bic	\tmp2, \start, \tmp2
+9997:
+USER(\label, ic	ivau, \tmp2)			// invalidate I line PoU
+	add	\tmp2, \tmp2, \tmp1
+	cmp	\tmp2, \end
+	b.lo	9997b
+	dsb	ish
+	isb
+	.endm
+
 /*
  * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
  */
@@ -478,37 +547,41 @@ alternative_endif
 	.endm
 
 /*
- * Errata workaround prior to TTBR0_EL1 update
+ * Arrange a physical address in a TTBR register, taking care of 52-bit
+ * addresses.
  *
- * 	val:	TTBR value with new BADDR, preserved
- * 	tmp0:	temporary register, clobbered
- * 	tmp1:	other temporary register, clobbered
- */
-	.macro	pre_ttbr0_update_workaround, val, tmp0, tmp1
-#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
-alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003
-	mrs	\tmp0, ttbr0_el1
-	mov	\tmp1, #FALKOR_RESERVED_ASID
-	bfi	\tmp0, \tmp1, #48, #16		// reserved ASID + old BADDR
-	msr	ttbr0_el1, \tmp0
-	isb
-	bfi	\tmp0, \val, #0, #48		// reserved ASID + new BADDR
-	msr	ttbr0_el1, \tmp0
-	isb
-alternative_else_nop_endif
+ * 	phys:	physical address, preserved
+ * 	ttbr:	returns the TTBR value
+ */
+	.macro	phys_to_ttbr, ttbr, phys
+#ifdef CONFIG_ARM64_PA_BITS_52
+	orr	\ttbr, \phys, \phys, lsr #46
+	and	\ttbr, \ttbr, #TTBR_BADDR_MASK_52
+#else
+	mov	\ttbr, \phys
 #endif
 	.endm
 
-/*
- * Errata workaround post TTBR0_EL1 update.
- */
-	.macro	post_ttbr0_update_workaround
-#ifdef CONFIG_CAVIUM_ERRATUM_27456
-alternative_if ARM64_WORKAROUND_CAVIUM_27456
-	ic	iallu
-	dsb	nsh
-	isb
-alternative_else_nop_endif
+	.macro	phys_to_pte, pte, phys
+#ifdef CONFIG_ARM64_PA_BITS_52
+	/*
+	 * We assume \phys is 64K aligned and this is guaranteed by only
+	 * supporting this configuration with 64K pages.
+	 */
+	orr	\pte, \phys, \phys, lsr #36
+	and	\pte, \pte, #PTE_ADDR_MASK
+#else
+	mov	\pte, \phys
+#endif
+	.endm
+
+	.macro	pte_to_phys, phys, pte
+#ifdef CONFIG_ARM64_PA_BITS_52
+	ubfiz	\phys, \pte, #(48 - 16 - 12), #16
+	bfxil	\phys, \pte, #16, #32
+	lsl	\phys, \phys, #16
+#else
+	and	\phys, \pte, #PTE_ADDR_MASK
 #endif
 	.endm
 
diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h
index 77651c49ef44..f11518af96a9 100644
--- a/arch/arm64/include/asm/barrier.h
+++ b/arch/arm64/include/asm/barrier.h
@@ -32,6 +32,7 @@
 #define dsb(opt)	asm volatile("dsb " #opt : : : "memory")
 
 #define psb_csync()	asm volatile("hint #17" : : : "memory")
+#define csdb()		asm volatile("hint #20" : : : "memory")
 
 #define mb()		dsb(sy)
 #define rmb()		dsb(ld)
@@ -40,6 +41,27 @@
 #define dma_rmb()	dmb(oshld)
 #define dma_wmb()	dmb(oshst)
 
+/*
+ * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
+ * and 0 otherwise.
+ */
+#define array_index_mask_nospec array_index_mask_nospec
+static inline unsigned long array_index_mask_nospec(unsigned long idx,
+						    unsigned long sz)
+{
+	unsigned long mask;
+
+	asm volatile(
+	"	cmp	%1, %2\n"
+	"	sbc	%0, xzr, xzr\n"
+	: "=r" (mask)
+	: "r" (idx), "Ir" (sz)
+	: "cc");
+
+	csdb();
+	return mask;
+}
+
 #define __smp_mb()	dmb(ish)
 #define __smp_rmb()	dmb(ishld)
 #define __smp_wmb()	dmb(ishst)
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 955130762a3c..bef9f418f089 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -52,6 +52,12 @@
  *		- start  - virtual start address
  *		- end    - virtual end address
  *
+ *	invalidate_icache_range(start, end)
+ *
+ *		Invalidate the I-cache in the region described by start, end.
+ *		- start  - virtual start address
+ *		- end    - virtual end address
+ *
  *	__flush_cache_user_range(start, end)
  *
  *		Ensure coherency between the I-cache and the D-cache in the
@@ -66,6 +72,7 @@
  *		- size   - region size
  */
 extern void flush_icache_range(unsigned long start, unsigned long end);
+extern int  invalidate_icache_range(unsigned long start, unsigned long end);
 extern void __flush_dcache_area(void *addr, size_t len);
 extern void __inval_dcache_area(void *addr, size_t len);
 extern void __clean_dcache_area_poc(void *addr, size_t len);
diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h
index a3c7f271ad4c..c00c62e1a4a3 100644
--- a/arch/arm64/include/asm/compat.h
+++ b/arch/arm64/include/asm/compat.h
@@ -150,70 +150,6 @@ typedef u32		compat_old_sigset_t;
 
 typedef u32		compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[128/sizeof(int) - 3];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			__compat_uid32_t _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;       /* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			__compat_uid32_t _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;	/* which child */
-			__compat_uid32_t _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			compat_uptr_t _addr; /* faulting insn/memory ref. */
-			short _addr_lsb; /* LSB of the reported address */
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			compat_long_t _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		/* SIGSYS */
-		struct {
-			compat_uptr_t _call_addr; /* calling user insn */
-			int _syscall;	/* triggering system call number */
-			compat_uint_t _arch;	/* AUDIT_ARCH_* of syscall */
-		} _sigsys;
-	} _sifields;
-} compat_siginfo_t;
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 /*
diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 2ff7c5e8efab..bb263820de13 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -41,7 +41,11 @@
 #define ARM64_WORKAROUND_CAVIUM_30115		20
 #define ARM64_HAS_DCPOP				21
 #define ARM64_SVE				22
+#define ARM64_UNMAP_KERNEL_AT_EL0		23
+#define ARM64_HARDEN_BRANCH_PREDICTOR		24
+#define ARM64_HARDEN_BP_POST_GUEST_EXIT		25
+#define ARM64_HAS_RAS_EXTN			26
 
-#define ARM64_NCAPS				23
+#define ARM64_NCAPS				27
 
 #endif /* __ASM_CPUCAPS_H */
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index cbf08d7cbf30..350c76a1d15b 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -20,7 +20,7 @@
 
 #define MPIDR_UP_BITMASK	(0x1 << 30)
 #define MPIDR_MT_BITMASK	(0x1 << 24)
-#define MPIDR_HWID_BITMASK	0xff00ffffff
+#define MPIDR_HWID_BITMASK	UL(0xff00ffffff)
 
 #define MPIDR_LEVEL_BITS_SHIFT	3
 #define MPIDR_LEVEL_BITS	(1 << MPIDR_LEVEL_BITS_SHIFT)
@@ -79,28 +79,37 @@
 #define ARM_CPU_PART_AEM_V8		0xD0F
 #define ARM_CPU_PART_FOUNDATION		0xD00
 #define ARM_CPU_PART_CORTEX_A57		0xD07
+#define ARM_CPU_PART_CORTEX_A72		0xD08
 #define ARM_CPU_PART_CORTEX_A53		0xD03
 #define ARM_CPU_PART_CORTEX_A73		0xD09
+#define ARM_CPU_PART_CORTEX_A75		0xD0A
 
 #define APM_CPU_PART_POTENZA		0x000
 
 #define CAVIUM_CPU_PART_THUNDERX	0x0A1
 #define CAVIUM_CPU_PART_THUNDERX_81XX	0x0A2
 #define CAVIUM_CPU_PART_THUNDERX_83XX	0x0A3
+#define CAVIUM_CPU_PART_THUNDERX2	0x0AF
 
 #define BRCM_CPU_PART_VULCAN		0x516
 
 #define QCOM_CPU_PART_FALKOR_V1		0x800
 #define QCOM_CPU_PART_FALKOR		0xC00
+#define QCOM_CPU_PART_KRYO		0x200
 
 #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
 #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72)
 #define MIDR_CORTEX_A73 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A73)
+#define MIDR_CORTEX_A75 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A75)
 #define MIDR_THUNDERX	MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
 #define MIDR_THUNDERX_81XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_81XX)
 #define MIDR_THUNDERX_83XX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX_83XX)
+#define MIDR_CAVIUM_THUNDERX2 MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX2)
+#define MIDR_BRCM_VULCAN MIDR_CPU_MODEL(ARM_CPU_IMP_BRCM, BRCM_CPU_PART_VULCAN)
 #define MIDR_QCOM_FALKOR_V1 MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR_V1)
 #define MIDR_QCOM_FALKOR MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_FALKOR)
+#define MIDR_QCOM_KRYO MIDR_CPU_MODEL(ARM_CPU_IMP_QCOM, QCOM_CPU_PART_KRYO)
 
 #ifndef __ASSEMBLY__
 
diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h
index 0df756b24863..b7847eb8a7bb 100644
--- a/arch/arm64/include/asm/dma-mapping.h
+++ b/arch/arm64/include/asm/dma-mapping.h
@@ -50,40 +50,5 @@ static inline bool is_device_dma_coherent(struct device *dev)
 	return dev->archdata.dma_coherent;
 }
 
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	dma_addr_t dev_addr = (dma_addr_t)paddr;
-
-	return dev_addr - ((dma_addr_t)dev->dma_pfn_offset << PAGE_SHIFT);
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr)
-{
-	phys_addr_t paddr = (phys_addr_t)dev_addr;
-
-	return paddr + ((phys_addr_t)dev->dma_pfn_offset << PAGE_SHIFT);
-}
-
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return false;
-
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
-static inline void dma_mark_clean(void *addr, size_t size)
-{
-}
-
-/* Override for dma_max_pfn() */
-static inline unsigned long dma_max_pfn(struct device *dev)
-{
-	dma_addr_t dma_max = (dma_addr_t)*dev->dma_mask;
-
-	return (ulong)dma_to_phys(dev, dma_max) >> PAGE_SHIFT;
-}
-#define dma_max_pfn(dev) dma_max_pfn(dev)
-
 #endif	/* __KERNEL__ */
 #endif	/* __ASM_DMA_MAPPING_H */
diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h
index c4cd5081d78b..8389050328bb 100644
--- a/arch/arm64/include/asm/efi.h
+++ b/arch/arm64/include/asm/efi.h
@@ -121,19 +121,21 @@ static inline void efi_set_pgd(struct mm_struct *mm)
 		if (mm != current->active_mm) {
 			/*
 			 * Update the current thread's saved ttbr0 since it is
-			 * restored as part of a return from exception. Set
-			 * the hardware TTBR0_EL1 using cpu_switch_mm()
-			 * directly to enable potential errata workarounds.
+			 * restored as part of a return from exception. Enable
+			 * access to the valid TTBR0_EL1 and invoke the errata
+			 * workaround directly since there is no return from
+			 * exception when invoking the EFI run-time services.
 			 */
 			update_saved_ttbr0(current, mm);
-			cpu_switch_mm(mm->pgd, mm);
+			uaccess_ttbr0_enable();
+			post_ttbr_update_workaround();
 		} else {
 			/*
 			 * Defer the switch to the current thread's TTBR0_EL1
 			 * until uaccess_enable(). Restore the current
 			 * thread's saved ttbr0 corresponding to its active_mm
 			 */
-			cpu_set_reserved_ttbr0();
+			uaccess_ttbr0_disable();
 			update_saved_ttbr0(current, current->active_mm);
 		}
 	}
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 014d7d8edcf9..803443d74926 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -86,6 +86,18 @@
 #define ESR_ELx_WNR_SHIFT	(6)
 #define ESR_ELx_WNR		(UL(1) << ESR_ELx_WNR_SHIFT)
 
+/* Asynchronous Error Type */
+#define ESR_ELx_IDS_SHIFT	(24)
+#define ESR_ELx_IDS		(UL(1) << ESR_ELx_IDS_SHIFT)
+#define ESR_ELx_AET_SHIFT	(10)
+#define ESR_ELx_AET		(UL(0x7) << ESR_ELx_AET_SHIFT)
+
+#define ESR_ELx_AET_UC		(UL(0) << ESR_ELx_AET_SHIFT)
+#define ESR_ELx_AET_UEU		(UL(1) << ESR_ELx_AET_SHIFT)
+#define ESR_ELx_AET_UEO		(UL(2) << ESR_ELx_AET_SHIFT)
+#define ESR_ELx_AET_UER		(UL(3) << ESR_ELx_AET_SHIFT)
+#define ESR_ELx_AET_CE		(UL(6) << ESR_ELx_AET_SHIFT)
+
 /* Shared ISS field definitions for Data/Instruction aborts */
 #define ESR_ELx_SET_SHIFT	(11)
 #define ESR_ELx_SET_MASK	(UL(3) << ESR_ELx_SET_SHIFT)
@@ -100,6 +112,7 @@
 #define ESR_ELx_FSC		(0x3F)
 #define ESR_ELx_FSC_TYPE	(0x3C)
 #define ESR_ELx_FSC_EXTABT	(0x10)
+#define ESR_ELx_FSC_SERROR	(0x11)
 #define ESR_ELx_FSC_ACCESS	(0x08)
 #define ESR_ELx_FSC_FAULT	(0x04)
 #define ESR_ELx_FSC_PERM	(0x0C)
@@ -127,6 +140,13 @@
 #define ESR_ELx_WFx_ISS_WFE	(UL(1) << 0)
 #define ESR_ELx_xVC_IMM_MASK	((1UL << 16) - 1)
 
+#define DISR_EL1_IDS		(UL(1) << 24)
+/*
+ * DISR_EL1 and ESR_ELx share the bottom 13 bits, but the RES0 bits may mean
+ * different things in the future...
+ */
+#define DISR_EL1_ESR_MASK	(ESR_ELx_AET | ESR_ELx_EA | ESR_ELx_FSC)
+
 /* ESR value templates for specific events */
 
 /* BRK instruction trap from AArch64 state */
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index 0c2eec490abf..bc30429d8e91 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -18,6 +18,8 @@
 #ifndef __ASM_EXCEPTION_H
 #define __ASM_EXCEPTION_H
 
+#include <asm/esr.h>
+
 #include <linux/interrupt.h>
 
 #define __exception	__attribute__((section(".exception.text")))
@@ -27,4 +29,16 @@
 #define __exception_irq_entry	__exception
 #endif
 
+static inline u32 disr_to_esr(u64 disr)
+{
+	unsigned int esr = ESR_ELx_EC_SERROR << ESR_ELx_EC_SHIFT;
+
+	if ((disr & DISR_EL1_IDS) == 0)
+		esr |= (disr & DISR_EL1_ESR_MASK);
+	else
+		esr |= (disr & ESR_ELx_ISS_MASK);
+
+	return esr;
+}
+
 #endif	/* __ASM_EXCEPTION_H */
diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 4052ec39e8db..ec1e6d6fa14c 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -58,6 +58,11 @@ enum fixed_addresses {
 	FIX_APEI_GHES_NMI,
 #endif /* CONFIG_ACPI_APEI_GHES */
 
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	FIX_ENTRY_TRAMP_DATA,
+	FIX_ENTRY_TRAMP_TEXT,
+#define TRAMP_VALIAS		(__fix_to_virt(FIX_ENTRY_TRAMP_TEXT))
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 	__end_of_permanent_fixed_addresses,
 
 	/*
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 74f34392a531..8857a0f0d0f7 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -71,7 +71,7 @@ extern void fpsimd_flush_thread(void);
 extern void fpsimd_signal_preserve_current_state(void);
 extern void fpsimd_preserve_current_state(void);
 extern void fpsimd_restore_current_state(void);
-extern void fpsimd_update_current_state(struct fpsimd_state *state);
+extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
 
 extern void fpsimd_flush_task_state(struct task_struct *target);
 extern void sve_flush_cpu_state(void);
diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h
index 5bb2fd4674e7..07fe2479d310 100644
--- a/arch/arm64/include/asm/futex.h
+++ b/arch/arm64/include/asm/futex.h
@@ -48,9 +48,10 @@ do {									\
 } while (0)
 
 static inline int
-arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
+arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
 {
 	int oldval = 0, ret, tmp;
+	u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
 
 	pagefault_disable();
 
@@ -88,15 +89,17 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
 }
 
 static inline int
-futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
+futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
 			      u32 oldval, u32 newval)
 {
 	int ret = 0;
 	u32 val, tmp;
+	u32 __user *uaddr;
 
-	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
+	if (!access_ok(VERIFY_WRITE, _uaddr, sizeof(u32)))
 		return -EFAULT;
 
+	uaddr = __uaccess_mask_ptr(_uaddr);
 	uaccess_enable();
 	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
 "	prfm	pstl1strm, %2\n"
diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h
index 1dca41bea16a..e73f68569624 100644
--- a/arch/arm64/include/asm/hugetlb.h
+++ b/arch/arm64/include/asm/hugetlb.h
@@ -22,7 +22,7 @@
 
 static inline pte_t huge_ptep_get(pte_t *ptep)
 {
-	return *ptep;
+	return READ_ONCE(*ptep);
 }
 
 
diff --git a/arch/arm64/include/asm/kasan.h b/arch/arm64/include/asm/kasan.h
index e266f80e45b7..8758bb008436 100644
--- a/arch/arm64/include/asm/kasan.h
+++ b/arch/arm64/include/asm/kasan.h
@@ -12,7 +12,8 @@
 
 /*
  * KASAN_SHADOW_START: beginning of the kernel virtual addresses.
- * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/8 of kernel virtual addresses.
+ * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/N of kernel virtual addresses,
+ * where N = (1 << KASAN_SHADOW_SCALE_SHIFT).
  */
 #define KASAN_SHADOW_START      (VA_START)
 #define KASAN_SHADOW_END        (KASAN_SHADOW_START + KASAN_SHADOW_SIZE)
@@ -20,14 +21,16 @@
 /*
  * This value is used to map an address to the corresponding shadow
  * address by the following formula:
- *     shadow_addr = (address >> 3) + KASAN_SHADOW_OFFSET;
+ *     shadow_addr = (address >> KASAN_SHADOW_SCALE_SHIFT) + KASAN_SHADOW_OFFSET
  *
- * (1 << 61) shadow addresses - [KASAN_SHADOW_OFFSET,KASAN_SHADOW_END]
- * cover all 64-bits of virtual addresses. So KASAN_SHADOW_OFFSET
- * should satisfy the following equation:
- *      KASAN_SHADOW_OFFSET = KASAN_SHADOW_END - (1ULL << 61)
+ * (1 << (64 - KASAN_SHADOW_SCALE_SHIFT)) shadow addresses that lie in range
+ * [KASAN_SHADOW_OFFSET, KASAN_SHADOW_END) cover all 64-bits of virtual
+ * addresses. So KASAN_SHADOW_OFFSET should satisfy the following equation:
+ *      KASAN_SHADOW_OFFSET = KASAN_SHADOW_END -
+ *				(1ULL << (64 - KASAN_SHADOW_SCALE_SHIFT))
  */
-#define KASAN_SHADOW_OFFSET     (KASAN_SHADOW_END - (1ULL << (64 - 3)))
+#define KASAN_SHADOW_OFFSET     (KASAN_SHADOW_END - (1ULL << \
+					(64 - KASAN_SHADOW_SCALE_SHIFT)))
 
 void kasan_init(void);
 void kasan_copy_shadow(pgd_t *pgdir);
diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
index 7803343e5881..a780f6714b44 100644
--- a/arch/arm64/include/asm/kernel-pgtable.h
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -52,7 +52,52 @@
 #define IDMAP_PGTABLE_LEVELS	(ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT))
 #endif
 
-#define SWAPPER_DIR_SIZE	(SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
+
+/*
+ * If KASLR is enabled, then an offset K is added to the kernel address
+ * space. The bottom 21 bits of this offset are zero to guarantee 2MB
+ * alignment for PA and VA.
+ *
+ * For each pagetable level of the swapper, we know that the shift will
+ * be larger than 21 (for the 4KB granule case we use section maps thus
+ * the smallest shift is actually 30) thus there is the possibility that
+ * KASLR can increase the number of pagetable entries by 1, so we make
+ * room for this extra entry.
+ *
+ * Note KASLR cannot increase the number of required entries for a level
+ * by more than one because it increments both the virtual start and end
+ * addresses equally (the extra entry comes from the case where the end
+ * address is just pushed over a boundary and the start address isn't).
+ */
+
+#ifdef CONFIG_RANDOMIZE_BASE
+#define EARLY_KASLR	(1)
+#else
+#define EARLY_KASLR	(0)
+#endif
+
+#define EARLY_ENTRIES(vstart, vend, shift) (((vend) >> (shift)) \
+					- ((vstart) >> (shift)) + 1 + EARLY_KASLR)
+
+#define EARLY_PGDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PGDIR_SHIFT))
+
+#if SWAPPER_PGTABLE_LEVELS > 3
+#define EARLY_PUDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PUD_SHIFT))
+#else
+#define EARLY_PUDS(vstart, vend) (0)
+#endif
+
+#if SWAPPER_PGTABLE_LEVELS > 2
+#define EARLY_PMDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, SWAPPER_TABLE_SHIFT))
+#else
+#define EARLY_PMDS(vstart, vend) (0)
+#endif
+
+#define EARLY_PAGES(vstart, vend) ( 1 			/* PGDIR page */				\
+			+ EARLY_PGDS((vstart), (vend)) 	/* each PGDIR needs a next level page table */	\
+			+ EARLY_PUDS((vstart), (vend))	/* each PUD needs a next level page table */	\
+			+ EARLY_PMDS((vstart), (vend)))	/* each PMD needs a next level page table */
+#define SWAPPER_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR + TEXT_OFFSET, _end))
 #define IDMAP_DIR_SIZE		(IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
 
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 715d395ef45b..b0c84171e6a3 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -23,6 +23,8 @@
 #include <asm/types.h>
 
 /* Hyp Configuration Register (HCR) bits */
+#define HCR_TEA		(UL(1) << 37)
+#define HCR_TERR	(UL(1) << 36)
 #define HCR_E2H		(UL(1) << 34)
 #define HCR_ID		(UL(1) << 33)
 #define HCR_CD		(UL(1) << 32)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index ab4d0a926043..24961b732e65 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -68,6 +68,8 @@ extern u32 __kvm_get_mdcr_el2(void);
 
 extern u32 __init_stage2_translation(void);
 
+extern void __qcom_hyp_sanitize_btac_predictors(void);
+
 #endif
 
 #endif /* __ARM_KVM_ASM_H__ */
diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h
index 5f28dfa14cee..413dc82b1e89 100644
--- a/arch/arm64/include/asm/kvm_emulate.h
+++ b/arch/arm64/include/asm/kvm_emulate.h
@@ -50,6 +50,13 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
 	vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
 	if (is_kernel_in_hyp_mode())
 		vcpu->arch.hcr_el2 |= HCR_E2H;
+	if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) {
+		/* route synchronous external abort exceptions to EL2 */
+		vcpu->arch.hcr_el2 |= HCR_TEA;
+		/* trap error record accesses */
+		vcpu->arch.hcr_el2 |= HCR_TERR;
+	}
+
 	if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
 		vcpu->arch.hcr_el2 &= ~HCR_RW;
 }
@@ -64,6 +71,11 @@ static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr)
 	vcpu->arch.hcr_el2 = hcr;
 }
 
+static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
+{
+	vcpu->arch.vsesr_el2 = vsesr;
+}
+
 static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
 {
 	return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc;
@@ -171,6 +183,11 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
 	return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
 }
 
+static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
+{
+	return vcpu->arch.fault.disr_el1;
+}
+
 static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
 {
 	return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_xVC_IMM_MASK;
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index ea6cb5b24258..596f8e414a4c 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -25,6 +25,7 @@
 #include <linux/types.h>
 #include <linux/kvm_types.h>
 #include <asm/cpufeature.h>
+#include <asm/daifflags.h>
 #include <asm/fpsimd.h>
 #include <asm/kvm.h>
 #include <asm/kvm_asm.h>
@@ -47,6 +48,8 @@
 	KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
 #define KVM_REQ_IRQ_PENDING	KVM_ARCH_REQ(1)
 
+DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
+
 int __attribute_const__ kvm_target_cpu(void);
 int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
 int kvm_arch_dev_ioctl_check_extension(struct kvm *kvm, long ext);
@@ -89,6 +92,7 @@ struct kvm_vcpu_fault_info {
 	u32 esr_el2;		/* Hyp Syndrom Register */
 	u64 far_el2;		/* Hyp Fault Address Register */
 	u64 hpfar_el2;		/* Hyp IPA Fault Address Register */
+	u64 disr_el1;		/* Deferred [SError] Status Register */
 };
 
 /*
@@ -120,6 +124,7 @@ enum vcpu_sysreg {
 	PAR_EL1,	/* Physical Address Register */
 	MDSCR_EL1,	/* Monitor Debug System Control Register */
 	MDCCINT_EL1,	/* Monitor Debug Comms Channel Interrupt Enable Reg */
+	DISR_EL1,	/* Deferred Interrupt Status Register */
 
 	/* Performance Monitors Registers */
 	PMCR_EL0,	/* Control Register */
@@ -192,6 +197,8 @@ struct kvm_cpu_context {
 		u64 sys_regs[NR_SYS_REGS];
 		u32 copro[NR_COPRO_REGS];
 	};
+
+	struct kvm_vcpu *__hyp_running_vcpu;
 };
 
 typedef struct kvm_cpu_context kvm_cpu_context_t;
@@ -277,6 +284,9 @@ struct kvm_vcpu_arch {
 
 	/* Detect first run of a vcpu */
 	bool has_run_once;
+
+	/* Virtual SError ESR to restore when HCR_EL2.VSE is set */
+	u64 vsesr_el2;
 };
 
 #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.gp_regs)
@@ -340,6 +350,8 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
 
 int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 		int exception_index);
+void handle_exit_early(struct kvm_vcpu *vcpu, struct kvm_run *run,
+		       int exception_index);
 
 int kvm_perf_init(void);
 int kvm_perf_teardown(void);
@@ -396,4 +408,19 @@ static inline void kvm_fpsimd_flush_cpu_state(void)
 		sve_flush_cpu_state();
 }
 
+static inline void kvm_arm_vhe_guest_enter(void)
+{
+	local_daif_mask();
+}
+
+static inline void kvm_arm_vhe_guest_exit(void)
+{
+	local_daif_restore(DAIF_PROCCTX_NOIRQ);
+}
+
+static inline bool kvm_arm_harden_branch_predictor(void)
+{
+	return cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR);
+}
+
 #endif /* __ARM64_KVM_HOST_H__ */
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 08d3bb66c8b7..f26f9cd70c72 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -20,7 +20,6 @@
 
 #include <linux/compiler.h>
 #include <linux/kvm_host.h>
-#include <asm/kvm_mmu.h>
 #include <asm/sysreg.h>
 
 #define __hyp_text __section(.hyp.text) notrace
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 672c8684d5c2..7faed6e48b46 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -173,32 +173,54 @@ static inline pmd_t kvm_s2pmd_mkwrite(pmd_t pmd)
 	return pmd;
 }
 
-static inline void kvm_set_s2pte_readonly(pte_t *pte)
+static inline pte_t kvm_s2pte_mkexec(pte_t pte)
+{
+	pte_val(pte) &= ~PTE_S2_XN;
+	return pte;
+}
+
+static inline pmd_t kvm_s2pmd_mkexec(pmd_t pmd)
+{
+	pmd_val(pmd) &= ~PMD_S2_XN;
+	return pmd;
+}
+
+static inline void kvm_set_s2pte_readonly(pte_t *ptep)
 {
 	pteval_t old_pteval, pteval;
 
-	pteval = READ_ONCE(pte_val(*pte));
+	pteval = READ_ONCE(pte_val(*ptep));
 	do {
 		old_pteval = pteval;
 		pteval &= ~PTE_S2_RDWR;
 		pteval |= PTE_S2_RDONLY;
-		pteval = cmpxchg_relaxed(&pte_val(*pte), old_pteval, pteval);
+		pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
 	} while (pteval != old_pteval);
 }
 
-static inline bool kvm_s2pte_readonly(pte_t *pte)
+static inline bool kvm_s2pte_readonly(pte_t *ptep)
+{
+	return (READ_ONCE(pte_val(*ptep)) & PTE_S2_RDWR) == PTE_S2_RDONLY;
+}
+
+static inline bool kvm_s2pte_exec(pte_t *ptep)
 {
-	return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY;
+	return !(READ_ONCE(pte_val(*ptep)) & PTE_S2_XN);
 }
 
-static inline void kvm_set_s2pmd_readonly(pmd_t *pmd)
+static inline void kvm_set_s2pmd_readonly(pmd_t *pmdp)
 {
-	kvm_set_s2pte_readonly((pte_t *)pmd);
+	kvm_set_s2pte_readonly((pte_t *)pmdp);
 }
 
-static inline bool kvm_s2pmd_readonly(pmd_t *pmd)
+static inline bool kvm_s2pmd_readonly(pmd_t *pmdp)
 {
-	return kvm_s2pte_readonly((pte_t *)pmd);
+	return kvm_s2pte_readonly((pte_t *)pmdp);
+}
+
+static inline bool kvm_s2pmd_exec(pmd_t *pmdp)
+{
+	return !(READ_ONCE(pmd_val(*pmdp)) & PMD_S2_XN);
 }
 
 static inline bool kvm_page_empty(void *ptr)
@@ -230,21 +252,25 @@ static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
 	return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101;
 }
 
-static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu,
-					       kvm_pfn_t pfn,
-					       unsigned long size)
+static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size)
 {
 	void *va = page_address(pfn_to_page(pfn));
 
 	kvm_flush_dcache_to_poc(va, size);
+}
 
+static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn,
+						  unsigned long size)
+{
 	if (icache_is_aliasing()) {
 		/* any kind of VIPT cache */
 		__flush_icache_all();
 	} else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
 		/* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
-		flush_icache_range((unsigned long)va,
-				   (unsigned long)va + size);
+		void *va = page_address(pfn_to_page(pfn));
+
+		invalidate_icache_range((unsigned long)va,
+					(unsigned long)va + size);
 	}
 }
 
@@ -273,15 +299,26 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled);
 
 static inline bool __kvm_cpu_uses_extended_idmap(void)
 {
-	return __cpu_uses_extended_idmap();
+	return __cpu_uses_extended_idmap_level();
 }
 
+static inline unsigned long __kvm_idmap_ptrs_per_pgd(void)
+{
+	return idmap_ptrs_per_pgd;
+}
+
+/*
+ * Can't use pgd_populate here, because the extended idmap adds an extra level
+ * above CONFIG_PGTABLE_LEVELS (which is 2 or 3 if we're using the extended
+ * idmap), and pgd_populate is only available if CONFIG_PGTABLE_LEVELS = 4.
+ */
 static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
 				       pgd_t *hyp_pgd,
 				       pgd_t *merged_hyp_pgd,
 				       unsigned long hyp_idmap_start)
 {
 	int idmap_idx;
+	u64 pgd_addr;
 
 	/*
 	 * Use the first entry to access the HYP mappings. It is
@@ -289,7 +326,8 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
 	 * extended idmap.
 	 */
 	VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
-	merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE);
+	pgd_addr = __phys_to_pgd_val(__pa(hyp_pgd));
+	merged_hyp_pgd[0] = __pgd(pgd_addr | PMD_TYPE_TABLE);
 
 	/*
 	 * Create another extended level entry that points to the boot HYP map,
@@ -299,7 +337,8 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
 	 */
 	idmap_idx = hyp_idmap_start >> VA_BITS;
 	VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
-	merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE);
+	pgd_addr = __phys_to_pgd_val(__pa(boot_hyp_pgd));
+	merged_hyp_pgd[idmap_idx] = __pgd(pgd_addr | PMD_TYPE_TABLE);
 }
 
 static inline unsigned int kvm_get_vmid_bits(void)
@@ -309,5 +348,45 @@ static inline unsigned int kvm_get_vmid_bits(void)
 	return (cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR1_VMIDBITS_SHIFT) == 2) ? 16 : 8;
 }
 
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+#include <asm/mmu.h>
+
+static inline void *kvm_get_hyp_vector(void)
+{
+	struct bp_hardening_data *data = arm64_get_bp_hardening_data();
+	void *vect = kvm_ksym_ref(__kvm_hyp_vector);
+
+	if (data->fn) {
+		vect = __bp_harden_hyp_vecs_start +
+		       data->hyp_vectors_slot * SZ_2K;
+
+		if (!has_vhe())
+			vect = lm_alias(vect);
+	}
+
+	return vect;
+}
+
+static inline int kvm_map_vectors(void)
+{
+	return create_hyp_mappings(kvm_ksym_ref(__bp_harden_hyp_vecs_start),
+				   kvm_ksym_ref(__bp_harden_hyp_vecs_end),
+				   PAGE_HYP_EXEC);
+}
+
+#else
+static inline void *kvm_get_hyp_vector(void)
+{
+	return kvm_ksym_ref(__kvm_hyp_vector);
+}
+
+static inline int kvm_map_vectors(void)
+{
+	return 0;
+}
+#endif
+
+#define kvm_phys_to_vttbr(addr)		phys_to_ttbr(addr)
+
 #endif /* __ASSEMBLY__ */
 #endif /* __ARM64_KVM_MMU_H__ */
diff --git a/arch/arm64/include/asm/kvm_psci.h b/arch/arm64/include/asm/kvm_psci.h
deleted file mode 100644
index bc39e557c56c..000000000000
--- a/arch/arm64/include/asm/kvm_psci.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (C) 2012,2013 - ARM Ltd
- * Author: Marc Zyngier <marc.zyngier@arm.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef __ARM64_KVM_PSCI_H__
-#define __ARM64_KVM_PSCI_H__
-
-#define KVM_ARM_PSCI_0_1	1
-#define KVM_ARM_PSCI_0_2	2
-
-int kvm_psci_version(struct kvm_vcpu *vcpu);
-int kvm_psci_call(struct kvm_vcpu *vcpu);
-
-#endif /* __ARM64_KVM_PSCI_H__ */
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index d4bae7d6e0d8..50fa96a49792 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -85,7 +85,8 @@
  * stack size when KASAN is in use.
  */
 #ifdef CONFIG_KASAN
-#define KASAN_SHADOW_SIZE	(UL(1) << (VA_BITS - 3))
+#define KASAN_SHADOW_SCALE_SHIFT 3
+#define KASAN_SHADOW_SIZE	(UL(1) << (VA_BITS - KASAN_SHADOW_SCALE_SHIFT))
 #define KASAN_THREAD_SHIFT	1
 #else
 #define KASAN_SHADOW_SIZE	(0)
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 0d34bf0a89c7..a050d4f3615d 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -17,6 +17,11 @@
 #define __ASM_MMU_H
 
 #define MMCF_AARCH32	0x1	/* mm context flag for AArch32 executables */
+#define USER_ASID_BIT	48
+#define USER_ASID_FLAG	(UL(1) << USER_ASID_BIT)
+#define TTBR_ASID_MASK	(UL(0xffff) << 48)
+
+#ifndef __ASSEMBLY__
 
 typedef struct {
 	atomic64_t	id;
@@ -31,6 +36,49 @@ typedef struct {
  */
 #define ASID(mm)	((mm)->context.id.counter & 0xffff)
 
+static inline bool arm64_kernel_unmapped_at_el0(void)
+{
+	return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) &&
+	       cpus_have_const_cap(ARM64_UNMAP_KERNEL_AT_EL0);
+}
+
+typedef void (*bp_hardening_cb_t)(void);
+
+struct bp_hardening_data {
+	int			hyp_vectors_slot;
+	bp_hardening_cb_t	fn;
+};
+
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+extern char __bp_harden_hyp_vecs_start[], __bp_harden_hyp_vecs_end[];
+
+DECLARE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
+
+static inline struct bp_hardening_data *arm64_get_bp_hardening_data(void)
+{
+	return this_cpu_ptr(&bp_hardening_data);
+}
+
+static inline void arm64_apply_bp_hardening(void)
+{
+	struct bp_hardening_data *d;
+
+	if (!cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR))
+		return;
+
+	d = arm64_get_bp_hardening_data();
+	if (d->fn)
+		d->fn();
+}
+#else
+static inline struct bp_hardening_data *arm64_get_bp_hardening_data(void)
+{
+	return NULL;
+}
+
+static inline void arm64_apply_bp_hardening(void)	{ }
+#endif	/* CONFIG_HARDEN_BRANCH_PREDICTOR */
+
 extern void paging_init(void);
 extern void bootmem_init(void);
 extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
@@ -41,4 +89,5 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
 extern void *fixmap_remap_fdt(phys_addr_t dt_phys);
 extern void mark_linear_text_alias_ro(void);
 
+#endif	/* !__ASSEMBLY__ */
 #endif
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 9d155fa9a507..39ec0b8a689e 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -19,8 +19,6 @@
 #ifndef __ASM_MMU_CONTEXT_H
 #define __ASM_MMU_CONTEXT_H
 
-#define FALKOR_RESERVED_ASID	1
-
 #ifndef __ASSEMBLY__
 
 #include <linux/compiler.h>
@@ -51,23 +49,39 @@ static inline void contextidr_thread_switch(struct task_struct *next)
  */
 static inline void cpu_set_reserved_ttbr0(void)
 {
-	unsigned long ttbr = __pa_symbol(empty_zero_page);
+	unsigned long ttbr = phys_to_ttbr(__pa_symbol(empty_zero_page));
 
 	write_sysreg(ttbr, ttbr0_el1);
 	isb();
 }
 
+static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm)
+{
+	BUG_ON(pgd == swapper_pg_dir);
+	cpu_set_reserved_ttbr0();
+	cpu_do_switch_mm(virt_to_phys(pgd),mm);
+}
+
 /*
  * TCR.T0SZ value to use when the ID map is active. Usually equals
  * TCR_T0SZ(VA_BITS), unless system RAM is positioned very high in
  * physical memory, in which case it will be smaller.
  */
 extern u64 idmap_t0sz;
+extern u64 idmap_ptrs_per_pgd;
 
 static inline bool __cpu_uses_extended_idmap(void)
 {
-	return (!IS_ENABLED(CONFIG_ARM64_VA_BITS_48) &&
-		unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS)));
+	return unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS));
+}
+
+/*
+ * True if the extended ID map requires an extra level of translation table
+ * to be configured.
+ */
+static inline bool __cpu_uses_extended_idmap_level(void)
+{
+	return ARM64_HW_PGTABLE_LEVELS(64 - idmap_t0sz) > CONFIG_PGTABLE_LEVELS;
 }
 
 /*
@@ -127,13 +141,13 @@ static inline void cpu_install_idmap(void)
  * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
  * avoiding the possibility of conflicting TLB entries being allocated.
  */
-static inline void cpu_replace_ttbr1(pgd_t *pgd)
+static inline void cpu_replace_ttbr1(pgd_t *pgdp)
 {
 	typedef void (ttbr_replace_func)(phys_addr_t);
 	extern ttbr_replace_func idmap_cpu_replace_ttbr1;
 	ttbr_replace_func *replace_phys;
 
-	phys_addr_t pgd_phys = virt_to_phys(pgd);
+	phys_addr_t pgd_phys = virt_to_phys(pgdp);
 
 	replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
 
@@ -170,7 +184,7 @@ static inline void update_saved_ttbr0(struct task_struct *tsk,
 	else
 		ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48;
 
-	task_thread_info(tsk)->ttbr0 = ttbr;
+	WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);
 }
 #else
 static inline void update_saved_ttbr0(struct task_struct *tsk,
@@ -225,6 +239,7 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 #define activate_mm(prev,next)	switch_mm(prev, next, current)
 
 void verify_cpu_asid_bits(void);
+void post_ttbr_update_workaround(void);
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 3bd498e4de4c..43393208229e 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -16,11 +16,15 @@
 #ifndef __ASM_PERCPU_H
 #define __ASM_PERCPU_H
 
+#include <asm/alternative.h>
 #include <asm/stack_pointer.h>
 
 static inline void set_my_cpu_offset(unsigned long off)
 {
-	asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
+	asm volatile(ALTERNATIVE("msr tpidr_el1, %0",
+				 "msr tpidr_el2, %0",
+				 ARM64_HAS_VIRT_HOST_EXTN)
+			:: "r" (off) : "memory");
 }
 
 static inline unsigned long __my_cpu_offset(void)
@@ -31,7 +35,10 @@ static inline unsigned long __my_cpu_offset(void)
 	 * We want to allow caching the value, so avoid using volatile and
 	 * instead use a fake stack read to hazard against barrier().
 	 */
-	asm("mrs %0, tpidr_el1" : "=r" (off) :
+	asm(ALTERNATIVE("mrs %0, tpidr_el1",
+			"mrs %0, tpidr_el2",
+			ARM64_HAS_VIRT_HOST_EXTN)
+		: "=r" (off) :
 		"Q" (*(const unsigned long *)current_stack_pointer));
 
 	return off;
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 5ca6a573a701..2e05bcd944c8 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -36,23 +36,23 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 	return (pmd_t *)__get_free_page(PGALLOC_GFP);
 }
 
-static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
+static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp)
 {
-	BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
-	free_page((unsigned long)pmd);
+	BUG_ON((unsigned long)pmdp & (PAGE_SIZE-1));
+	free_page((unsigned long)pmdp);
 }
 
-static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot)
+static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 {
-	set_pud(pud, __pud(pmd | prot));
+	set_pud(pudp, __pud(__phys_to_pud_val(pmdp) | prot));
 }
 
-static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
+static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
 {
-	__pud_populate(pud, __pa(pmd), PMD_TYPE_TABLE);
+	__pud_populate(pudp, __pa(pmdp), PMD_TYPE_TABLE);
 }
 #else
-static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot)
+static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 {
 	BUILD_BUG();
 }
@@ -65,30 +65,30 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
 	return (pud_t *)__get_free_page(PGALLOC_GFP);
 }
 
-static inline void pud_free(struct mm_struct *mm, pud_t *pud)
+static inline void pud_free(struct mm_struct *mm, pud_t *pudp)
 {
-	BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
-	free_page((unsigned long)pud);
+	BUG_ON((unsigned long)pudp & (PAGE_SIZE-1));
+	free_page((unsigned long)pudp);
 }
 
-static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot)
+static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot)
 {
-	set_pgd(pgdp, __pgd(pud | prot));
+	set_pgd(pgdp, __pgd(__phys_to_pgd_val(pudp) | prot));
 }
 
-static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
+static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, pud_t *pudp)
 {
-	__pgd_populate(pgd, __pa(pud), PUD_TYPE_TABLE);
+	__pgd_populate(pgdp, __pa(pudp), PUD_TYPE_TABLE);
 }
 #else
-static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot)
+static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot)
 {
 	BUILD_BUG();
 }
 #endif	/* CONFIG_PGTABLE_LEVELS > 3 */
 
 extern pgd_t *pgd_alloc(struct mm_struct *mm);
-extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
+extern void pgd_free(struct mm_struct *mm, pgd_t *pgdp);
 
 static inline pte_t *
 pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr)
@@ -114,10 +114,10 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
 /*
  * Free a PTE table.
  */
-static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+static inline void pte_free_kernel(struct mm_struct *mm, pte_t *ptep)
 {
-	if (pte)
-		free_page((unsigned long)pte);
+	if (ptep)
+		free_page((unsigned long)ptep);
 }
 
 static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
@@ -126,10 +126,10 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
 	__free_page(pte);
 }
 
-static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
+static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t ptep,
 				  pmdval_t prot)
 {
-	set_pmd(pmdp, __pmd(pte | prot));
+	set_pmd(pmdp, __pmd(__phys_to_pmd_val(ptep) | prot));
 }
 
 /*
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index eb0c2bd90de9..cdfe3e657a9e 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -16,6 +16,8 @@
 #ifndef __ASM_PGTABLE_HWDEF_H
 #define __ASM_PGTABLE_HWDEF_H
 
+#include <asm/memory.h>
+
 /*
  * Number of page-table levels required to address 'va_bits' wide
  * address, without section mapping. We resolve the top (va_bits - PAGE_SHIFT)
@@ -116,9 +118,9 @@
  * Level 1 descriptor (PUD).
  */
 #define PUD_TYPE_TABLE		(_AT(pudval_t, 3) << 0)
-#define PUD_TABLE_BIT		(_AT(pgdval_t, 1) << 1)
-#define PUD_TYPE_MASK		(_AT(pgdval_t, 3) << 0)
-#define PUD_TYPE_SECT		(_AT(pgdval_t, 1) << 0)
+#define PUD_TABLE_BIT		(_AT(pudval_t, 1) << 1)
+#define PUD_TYPE_MASK		(_AT(pudval_t, 3) << 0)
+#define PUD_TYPE_SECT		(_AT(pudval_t, 1) << 0)
 
 /*
  * Level 2 descriptor (PMD).
@@ -166,6 +168,14 @@
 #define PTE_UXN			(_AT(pteval_t, 1) << 54)	/* User XN */
 #define PTE_HYP_XN		(_AT(pteval_t, 1) << 54)	/* HYP XN */
 
+#define PTE_ADDR_LOW		(((_AT(pteval_t, 1) << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)
+#ifdef CONFIG_ARM64_PA_BITS_52
+#define PTE_ADDR_HIGH		(_AT(pteval_t, 0xf) << 12)
+#define PTE_ADDR_MASK		(PTE_ADDR_LOW | PTE_ADDR_HIGH)
+#else
+#define PTE_ADDR_MASK		PTE_ADDR_LOW
+#endif
+
 /*
  * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
  */
@@ -177,9 +187,11 @@
  */
 #define PTE_S2_RDONLY		(_AT(pteval_t, 1) << 6)   /* HAP[2:1] */
 #define PTE_S2_RDWR		(_AT(pteval_t, 3) << 6)   /* HAP[2:1] */
+#define PTE_S2_XN		(_AT(pteval_t, 2) << 53)  /* XN[1:0] */
 
 #define PMD_S2_RDONLY		(_AT(pmdval_t, 1) << 6)   /* HAP[2:1] */
 #define PMD_S2_RDWR		(_AT(pmdval_t, 3) << 6)   /* HAP[2:1] */
+#define PMD_S2_XN		(_AT(pmdval_t, 2) << 53)  /* XN[1:0] */
 
 /*
  * Memory Attribute override for Stage-2 (MemAttr[3:0])
@@ -196,7 +208,7 @@
 /*
  * Highest possible physical address supported.
  */
-#define PHYS_MASK_SHIFT		(48)
+#define PHYS_MASK_SHIFT		(CONFIG_ARM64_PA_BITS)
 #define PHYS_MASK		((UL(1) << PHYS_MASK_SHIFT) - 1)
 
 /*
@@ -272,9 +284,23 @@
 #define TCR_TG1_4K		(UL(2) << TCR_TG1_SHIFT)
 #define TCR_TG1_64K		(UL(3) << TCR_TG1_SHIFT)
 
+#define TCR_IPS_SHIFT		32
+#define TCR_IPS_MASK		(UL(7) << TCR_IPS_SHIFT)
+#define TCR_A1			(UL(1) << 22)
 #define TCR_ASID16		(UL(1) << 36)
 #define TCR_TBI0		(UL(1) << 37)
 #define TCR_HA			(UL(1) << 39)
 #define TCR_HD			(UL(1) << 40)
 
+/*
+ * TTBR.
+ */
+#ifdef CONFIG_ARM64_PA_BITS_52
+/*
+ * This should be GENMASK_ULL(47, 2).
+ * TTBR_ELx[1] is RES0 in this configuration.
+ */
+#define TTBR_BADDR_MASK_52	(((UL(1) << 46) - 1) << 2)
+#endif
+
 #endif
diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h
index 0a5635fb0ef9..108ecad7acc5 100644
--- a/arch/arm64/include/asm/pgtable-prot.h
+++ b/arch/arm64/include/asm/pgtable-prot.h
@@ -34,8 +34,14 @@
 
 #include <asm/pgtable-types.h>
 
-#define PROT_DEFAULT		(PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
-#define PROT_SECT_DEFAULT	(PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+#define _PROT_DEFAULT		(PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
+#define _PROT_SECT_DEFAULT	(PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+
+#define PTE_MAYBE_NG		(arm64_kernel_unmapped_at_el0() ? PTE_NG : 0)
+#define PMD_MAYBE_NG		(arm64_kernel_unmapped_at_el0() ? PMD_SECT_NG : 0)
+
+#define PROT_DEFAULT		(_PROT_DEFAULT | PTE_MAYBE_NG)
+#define PROT_SECT_DEFAULT	(_PROT_SECT_DEFAULT | PMD_MAYBE_NG)
 
 #define PROT_DEVICE_nGnRnE	(PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE))
 #define PROT_DEVICE_nGnRE	(PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE))
@@ -47,23 +53,24 @@
 #define PROT_SECT_NORMAL	(PROT_SECT_DEFAULT | PMD_SECT_PXN | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
 #define PROT_SECT_NORMAL_EXEC	(PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
 
-#define _PAGE_DEFAULT		(PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
+#define _PAGE_DEFAULT		(_PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
+#define _HYP_PAGE_DEFAULT	_PAGE_DEFAULT
 
-#define PAGE_KERNEL		__pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE)
-#define PAGE_KERNEL_RO		__pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
-#define PAGE_KERNEL_ROX		__pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
-#define PAGE_KERNEL_EXEC	__pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE)
-#define PAGE_KERNEL_EXEC_CONT	__pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT)
+#define PAGE_KERNEL		__pgprot(PROT_NORMAL)
+#define PAGE_KERNEL_RO		__pgprot((PROT_NORMAL & ~PTE_WRITE) | PTE_RDONLY)
+#define PAGE_KERNEL_ROX		__pgprot((PROT_NORMAL & ~(PTE_WRITE | PTE_PXN)) | PTE_RDONLY)
+#define PAGE_KERNEL_EXEC	__pgprot(PROT_NORMAL & ~PTE_PXN)
+#define PAGE_KERNEL_EXEC_CONT	__pgprot((PROT_NORMAL & ~PTE_PXN) | PTE_CONT)
 
-#define PAGE_HYP		__pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_HYP_XN)
-#define PAGE_HYP_EXEC		__pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY)
-#define PAGE_HYP_RO		__pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY | PTE_HYP_XN)
+#define PAGE_HYP		__pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_HYP_XN)
+#define PAGE_HYP_EXEC		__pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY)
+#define PAGE_HYP_RO		__pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY | PTE_HYP_XN)
 #define PAGE_HYP_DEVICE		__pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
 
-#define PAGE_S2			__pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
-#define PAGE_S2_DEVICE		__pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
+#define PAGE_S2			__pgprot(_PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY | PTE_S2_XN)
+#define PAGE_S2_DEVICE		__pgprot(_PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_S2_XN)
 
-#define PAGE_NONE		__pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN)
+#define PAGE_NONE		__pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
 #define PAGE_SHARED		__pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
 #define PAGE_SHARED_EXEC	__pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
 #define PAGE_READONLY		__pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index bdcc7f1c9d06..7e2c27e63cd8 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -59,9 +59,22 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 
 #define pte_ERROR(pte)		__pte_error(__FILE__, __LINE__, pte_val(pte))
 
-#define pte_pfn(pte)		((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT)
+/*
+ * Macros to convert between a physical address and its placement in a
+ * page table entry, taking care of 52-bit addresses.
+ */
+#ifdef CONFIG_ARM64_PA_BITS_52
+#define __pte_to_phys(pte)	\
+	((pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << 36))
+#define __phys_to_pte_val(phys)	(((phys) | ((phys) >> 36)) & PTE_ADDR_MASK)
+#else
+#define __pte_to_phys(pte)	(pte_val(pte) & PTE_ADDR_MASK)
+#define __phys_to_pte_val(phys)	(phys)
+#endif
 
-#define pfn_pte(pfn,prot)	(__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
+#define pte_pfn(pte)		(__pte_to_phys(pte) >> PAGE_SHIFT)
+#define pfn_pte(pfn,prot)	\
+	__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 
 #define pte_none(pte)		(!pte_val(pte))
 #define pte_clear(mm,addr,ptep)	set_pte(ptep, __pte(0))
@@ -205,7 +218,7 @@ static inline pmd_t pmd_mkcont(pmd_t pmd)
 
 static inline void set_pte(pte_t *ptep, pte_t pte)
 {
-	*ptep = pte;
+	WRITE_ONCE(*ptep, pte);
 
 	/*
 	 * Only if the new pte is valid and kernel, otherwise TLB maintenance
@@ -237,6 +250,8 @@ extern void __sync_icache_dcache(pte_t pteval, unsigned long addr);
 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 			      pte_t *ptep, pte_t pte)
 {
+	pte_t old_pte;
+
 	if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
 		__sync_icache_dcache(pte, addr);
 
@@ -245,14 +260,15 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 	 * hardware updates of the pte (ptep_set_access_flags safely changes
 	 * valid ptes without going through an invalid entry).
 	 */
-	if (IS_ENABLED(CONFIG_DEBUG_VM) && pte_valid(*ptep) && pte_valid(pte) &&
+	old_pte = READ_ONCE(*ptep);
+	if (IS_ENABLED(CONFIG_DEBUG_VM) && pte_valid(old_pte) && pte_valid(pte) &&
 	   (mm == current->active_mm || atomic_read(&mm->mm_users) > 1)) {
 		VM_WARN_ONCE(!pte_young(pte),
 			     "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
-			     __func__, pte_val(*ptep), pte_val(pte));
-		VM_WARN_ONCE(pte_write(*ptep) && !pte_dirty(pte),
+			     __func__, pte_val(old_pte), pte_val(pte));
+		VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
 			     "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
-			     __func__, pte_val(*ptep), pte_val(pte));
+			     __func__, pte_val(old_pte), pte_val(pte));
 	}
 
 	set_pte(ptep, pte);
@@ -292,6 +308,11 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b)
 
 #define __HAVE_ARCH_PTE_SPECIAL
 
+static inline pte_t pgd_pte(pgd_t pgd)
+{
+	return __pte(pgd_val(pgd));
+}
+
 static inline pte_t pud_pte(pud_t pud)
 {
 	return __pte(pud_val(pud));
@@ -357,15 +378,24 @@ static inline int pmd_protnone(pmd_t pmd)
 
 #define pmd_mkhuge(pmd)		(__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
 
-#define pmd_pfn(pmd)		(((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
-#define pfn_pmd(pfn,prot)	(__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
+#define __pmd_to_phys(pmd)	__pte_to_phys(pmd_pte(pmd))
+#define __phys_to_pmd_val(phys)	__phys_to_pte_val(phys)
+#define pmd_pfn(pmd)		((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
+#define pfn_pmd(pfn,prot)	__pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 #define mk_pmd(page,prot)	pfn_pmd(page_to_pfn(page),prot)
 
 #define pud_write(pud)		pte_write(pud_pte(pud))
-#define pud_pfn(pud)		(((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
+
+#define __pud_to_phys(pud)	__pte_to_phys(pud_pte(pud))
+#define __phys_to_pud_val(phys)	__phys_to_pte_val(phys)
+#define pud_pfn(pud)		((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
+#define pfn_pud(pfn,prot)	__pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 
 #define set_pmd_at(mm, addr, pmdp, pmd)	set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))
 
+#define __pgd_to_phys(pgd)	__pte_to_phys(pgd_pte(pgd))
+#define __phys_to_pgd_val(phys)	__phys_to_pte_val(phys)
+
 #define __pgprot_modify(prot,mask,bits) \
 	__pgprot((pgprot_val(prot) & ~(mask)) | (bits))
 
@@ -404,7 +434,7 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 
 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 {
-	*pmdp = pmd;
+	WRITE_ONCE(*pmdp, pmd);
 	dsb(ishst);
 	isb();
 }
@@ -416,7 +446,7 @@ static inline void pmd_clear(pmd_t *pmdp)
 
 static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
 {
-	return pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK;
+	return __pmd_to_phys(pmd);
 }
 
 /* Find an entry in the third-level page table. */
@@ -434,7 +464,7 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
 #define pte_set_fixmap_offset(pmd, addr)	pte_set_fixmap(pte_offset_phys(pmd, addr))
 #define pte_clear_fixmap()		clear_fixmap(FIX_PTE)
 
-#define pmd_page(pmd)		pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
+#define pmd_page(pmd)		pfn_to_page(__phys_to_pfn(__pmd_to_phys(pmd)))
 
 /* use ONLY for statically allocated translation tables */
 #define pte_offset_kimg(dir,addr)	((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
@@ -455,7 +485,7 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
 
 static inline void set_pud(pud_t *pudp, pud_t pud)
 {
-	*pudp = pud;
+	WRITE_ONCE(*pudp, pud);
 	dsb(ishst);
 	isb();
 }
@@ -467,20 +497,20 @@ static inline void pud_clear(pud_t *pudp)
 
 static inline phys_addr_t pud_page_paddr(pud_t pud)
 {
-	return pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK;
+	return __pud_to_phys(pud);
 }
 
 /* Find an entry in the second-level page table. */
 #define pmd_index(addr)		(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
 
-#define pmd_offset_phys(dir, addr)	(pud_page_paddr(*(dir)) + pmd_index(addr) * sizeof(pmd_t))
+#define pmd_offset_phys(dir, addr)	(pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
 #define pmd_offset(dir, addr)		((pmd_t *)__va(pmd_offset_phys((dir), (addr))))
 
 #define pmd_set_fixmap(addr)		((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
 #define pmd_set_fixmap_offset(pud, addr)	pmd_set_fixmap(pmd_offset_phys(pud, addr))
 #define pmd_clear_fixmap()		clear_fixmap(FIX_PMD)
 
-#define pud_page(pud)		pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
+#define pud_page(pud)		pfn_to_page(__phys_to_pfn(__pud_to_phys(pud)))
 
 /* use ONLY for statically allocated translation tables */
 #define pmd_offset_kimg(dir,addr)	((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
@@ -508,7 +538,7 @@ static inline phys_addr_t pud_page_paddr(pud_t pud)
 
 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
 {
-	*pgdp = pgd;
+	WRITE_ONCE(*pgdp, pgd);
 	dsb(ishst);
 }
 
@@ -519,20 +549,20 @@ static inline void pgd_clear(pgd_t *pgdp)
 
 static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
 {
-	return pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK;
+	return __pgd_to_phys(pgd);
 }
 
 /* Find an entry in the frst-level page table. */
 #define pud_index(addr)		(((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
 
-#define pud_offset_phys(dir, addr)	(pgd_page_paddr(*(dir)) + pud_index(addr) * sizeof(pud_t))
+#define pud_offset_phys(dir, addr)	(pgd_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
 #define pud_offset(dir, addr)		((pud_t *)__va(pud_offset_phys((dir), (addr))))
 
 #define pud_set_fixmap(addr)		((pud_t *)set_fixmap_offset(FIX_PUD, addr))
 #define pud_set_fixmap_offset(pgd, addr)	pud_set_fixmap(pud_offset_phys(pgd, addr))
 #define pud_clear_fixmap()		clear_fixmap(FIX_PUD)
 
-#define pgd_page(pgd)		pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK))
+#define pgd_page(pgd)		pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
 
 /* use ONLY for statically allocated translation tables */
 #define pud_offset_kimg(dir,addr)	((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))
@@ -679,10 +709,19 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,
 {
 	ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
 }
+
+#define pmdp_establish pmdp_establish
+static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
+		unsigned long address, pmd_t *pmdp, pmd_t pmd)
+{
+	return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
+}
 #endif
 
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
+extern pgd_t swapper_pg_end[];
 extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
+extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
 
 /*
  * Encode and decode a swap entry:
@@ -736,6 +775,12 @@ static inline void update_mmu_cache(struct vm_area_struct *vma,
 #define kc_vaddr_to_offset(v)	((v) & ~VA_START)
 #define kc_offset_to_vaddr(o)	((o) | VA_START)
 
+#ifdef CONFIG_ARM64_PA_BITS_52
+#define phys_to_ttbr(addr)	(((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
+#else
+#define phys_to_ttbr(addr)	(addr)
+#endif
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* __ASM_PGTABLE_H */
diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h
index 14ad6e4e87d1..16cef2e8449e 100644
--- a/arch/arm64/include/asm/proc-fns.h
+++ b/arch/arm64/include/asm/proc-fns.h
@@ -35,12 +35,6 @@ extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr);
 
 #include <asm/memory.h>
 
-#define cpu_switch_mm(pgd,mm)				\
-do {							\
-	BUG_ON(pgd == swapper_pg_dir);			\
-	cpu_do_switch_mm(virt_to_phys(pgd),mm);		\
-} while (0)
-
 #endif /* __ASSEMBLY__ */
 #endif /* __KERNEL__ */
 #endif /* __ASM_PROCFNS_H */
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 023cacb946c3..fce604e3e599 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -21,6 +21,9 @@
 
 #define TASK_SIZE_64		(UL(1) << VA_BITS)
 
+#define KERNEL_DS	UL(-1)
+#define USER_DS		(TASK_SIZE_64 - 1)
+
 #ifndef __ASSEMBLY__
 
 /*
@@ -113,6 +116,16 @@ struct thread_struct {
 	struct debug_info	debug;		/* debugging */
 };
 
+/*
+ * Everything usercopied to/from thread_struct is statically-sized, so
+ * no hardened usercopy whitelist is needed.
+ */
+static inline void arch_thread_struct_whitelist(unsigned long *offset,
+						unsigned long *size)
+{
+	*offset = *size = 0;
+}
+
 #ifdef CONFIG_COMPAT
 #define task_user_tls(t)						\
 ({									\
@@ -216,6 +229,7 @@ static inline void spin_lock_prefetch(const void *ptr)
 
 int cpu_enable_pan(void *__unused);
 int cpu_enable_cache_maint_trap(void *__unused);
+int cpu_clear_disr(void *__unused);
 
 /* Userspace interface for PR_SVE_{SET,GET}_VL prctl()s: */
 #define SVE_SET_VL(arg)	sve_set_current_vl(arg)
diff --git a/arch/arm64/include/asm/sdei.h b/arch/arm64/include/asm/sdei.h
new file mode 100644
index 000000000000..e073e6886685
--- /dev/null
+++ b/arch/arm64/include/asm/sdei.h
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2017 Arm Ltd.
+#ifndef __ASM_SDEI_H
+#define __ASM_SDEI_H
+
+/* Values for sdei_exit_mode */
+#define SDEI_EXIT_HVC  0
+#define SDEI_EXIT_SMC  1
+
+#define SDEI_STACK_SIZE		IRQ_STACK_SIZE
+
+#ifndef __ASSEMBLY__
+
+#include <linux/linkage.h>
+#include <linux/preempt.h>
+#include <linux/types.h>
+
+#include <asm/virt.h>
+
+extern unsigned long sdei_exit_mode;
+
+/* Software Delegated Exception entry point from firmware*/
+asmlinkage void __sdei_asm_handler(unsigned long event_num, unsigned long arg,
+				   unsigned long pc, unsigned long pstate);
+
+/* and its CONFIG_UNMAP_KERNEL_AT_EL0 trampoline */
+asmlinkage void __sdei_asm_entry_trampoline(unsigned long event_num,
+						   unsigned long arg,
+						   unsigned long pc,
+						   unsigned long pstate);
+
+/*
+ * The above entry point does the minimum to call C code. This function does
+ * anything else, before calling the driver.
+ */
+struct sdei_registered_event;
+asmlinkage unsigned long __sdei_handler(struct pt_regs *regs,
+					struct sdei_registered_event *arg);
+
+unsigned long sdei_arch_get_entry_point(int conduit);
+#define sdei_arch_get_entry_point(x)	sdei_arch_get_entry_point(x)
+
+bool _on_sdei_stack(unsigned long sp);
+static inline bool on_sdei_stack(unsigned long sp)
+{
+	if (!IS_ENABLED(CONFIG_VMAP_STACK))
+		return false;
+	if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
+		return false;
+	if (in_nmi())
+		return _on_sdei_stack(sp);
+
+	return false;
+}
+
+#endif /* __ASSEMBLY__ */
+#endif	/* __ASM_SDEI_H */
diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
index 941267caa39c..caab039d6305 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -28,5 +28,6 @@ extern char __initdata_begin[], __initdata_end[];
 extern char __inittext_begin[], __inittext_end[];
 extern char __irqentry_text_start[], __irqentry_text_end[];
 extern char __mmuoff_data_start[], __mmuoff_data_end[];
+extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
 
 #endif /* __ASM_SECTIONS_H */
diff --git a/arch/arm64/include/asm/sparsemem.h b/arch/arm64/include/asm/sparsemem.h
index 74a9d301819f..b299929fe56c 100644
--- a/arch/arm64/include/asm/sparsemem.h
+++ b/arch/arm64/include/asm/sparsemem.h
@@ -17,7 +17,7 @@
 #define __ASM_SPARSEMEM_H
 
 #ifdef CONFIG_SPARSEMEM
-#define MAX_PHYSMEM_BITS	48
+#define MAX_PHYSMEM_BITS	CONFIG_ARM64_PA_BITS
 #define SECTION_SIZE_BITS	30
 #endif
 
diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h
index fdb827c7832f..ebdae15d665d 100644
--- a/arch/arm64/include/asm/spinlock.h
+++ b/arch/arm64/include/asm/spinlock.h
@@ -87,8 +87,8 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock)
 	"	cbnz	%w1, 1f\n"
 	"	add	%w1, %w0, %3\n"
 	"	casa	%w0, %w1, %2\n"
-	"	and	%w1, %w1, #0xffff\n"
-	"	eor	%w1, %w1, %w0, lsr #16\n"
+	"	sub	%w1, %w1, %3\n"
+	"	eor	%w1, %w1, %w0\n"
 	"1:")
 	: "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
 	: "I" (1 << TICKET_SHIFT)
diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h
index 6ad30776e984..902f9edacbea 100644
--- a/arch/arm64/include/asm/stacktrace.h
+++ b/arch/arm64/include/asm/stacktrace.h
@@ -22,12 +22,13 @@
 
 #include <asm/memory.h>
 #include <asm/ptrace.h>
+#include <asm/sdei.h>
 
 struct stackframe {
 	unsigned long fp;
 	unsigned long pc;
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
-	unsigned int graph;
+	int graph;
 #endif
 };
 
@@ -85,6 +86,8 @@ static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp
 		return true;
 	if (on_overflow_stack(sp))
 		return true;
+	if (on_sdei_stack(sp))
+		return true;
 
 	return false;
 }
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 08cc88574659..0e1960c59197 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -20,6 +20,7 @@
 #ifndef __ASM_SYSREG_H
 #define __ASM_SYSREG_H
 
+#include <asm/compiler.h>
 #include <linux/stringify.h>
 
 /*
@@ -175,6 +176,16 @@
 #define SYS_AFSR0_EL1			sys_reg(3, 0, 5, 1, 0)
 #define SYS_AFSR1_EL1			sys_reg(3, 0, 5, 1, 1)
 #define SYS_ESR_EL1			sys_reg(3, 0, 5, 2, 0)
+
+#define SYS_ERRIDR_EL1			sys_reg(3, 0, 5, 3, 0)
+#define SYS_ERRSELR_EL1			sys_reg(3, 0, 5, 3, 1)
+#define SYS_ERXFR_EL1			sys_reg(3, 0, 5, 4, 0)
+#define SYS_ERXCTLR_EL1			sys_reg(3, 0, 5, 4, 1)
+#define SYS_ERXSTATUS_EL1		sys_reg(3, 0, 5, 4, 2)
+#define SYS_ERXADDR_EL1			sys_reg(3, 0, 5, 4, 3)
+#define SYS_ERXMISC0_EL1		sys_reg(3, 0, 5, 5, 0)
+#define SYS_ERXMISC1_EL1		sys_reg(3, 0, 5, 5, 1)
+
 #define SYS_FAR_EL1			sys_reg(3, 0, 6, 0, 0)
 #define SYS_PAR_EL1			sys_reg(3, 0, 7, 4, 0)
 
@@ -278,6 +289,7 @@
 #define SYS_AMAIR_EL1			sys_reg(3, 0, 10, 3, 0)
 
 #define SYS_VBAR_EL1			sys_reg(3, 0, 12, 0, 0)
+#define SYS_DISR_EL1			sys_reg(3, 0, 12, 1, 1)
 
 #define SYS_ICC_IAR0_EL1		sys_reg(3, 0, 12, 8, 0)
 #define SYS_ICC_EOIR0_EL1		sys_reg(3, 0, 12, 8, 1)
@@ -353,8 +365,10 @@
 
 #define SYS_DACR32_EL2			sys_reg(3, 4, 3, 0, 0)
 #define SYS_IFSR32_EL2			sys_reg(3, 4, 5, 0, 1)
+#define SYS_VSESR_EL2			sys_reg(3, 4, 5, 2, 3)
 #define SYS_FPEXC32_EL2			sys_reg(3, 4, 5, 3, 0)
 
+#define SYS_VDISR_EL2			sys_reg(3, 4, 12, 1,  1)
 #define __SYS__AP0Rx_EL2(x)		sys_reg(3, 4, 12, 8, x)
 #define SYS_ICH_AP0R0_EL2		__SYS__AP0Rx_EL2(0)
 #define SYS_ICH_AP0R1_EL2		__SYS__AP0Rx_EL2(1)
@@ -398,27 +412,85 @@
 
 /* Common SCTLR_ELx flags. */
 #define SCTLR_ELx_EE    (1 << 25)
+#define SCTLR_ELx_IESB	(1 << 21)
+#define SCTLR_ELx_WXN	(1 << 19)
 #define SCTLR_ELx_I	(1 << 12)
 #define SCTLR_ELx_SA	(1 << 3)
 #define SCTLR_ELx_C	(1 << 2)
 #define SCTLR_ELx_A	(1 << 1)
 #define SCTLR_ELx_M	1
 
+#define SCTLR_ELx_FLAGS	(SCTLR_ELx_M  | SCTLR_ELx_A | SCTLR_ELx_C | \
+			 SCTLR_ELx_SA | SCTLR_ELx_I | SCTLR_ELx_IESB)
+
+/* SCTLR_EL2 specific flags. */
 #define SCTLR_EL2_RES1	((1 << 4)  | (1 << 5)  | (1 << 11) | (1 << 16) | \
 			 (1 << 18) | (1 << 22) | (1 << 23) | (1 << 28) | \
 			 (1 << 29))
+#define SCTLR_EL2_RES0	((1 << 6)  | (1 << 7)  | (1 << 8)  | (1 << 9)  | \
+			 (1 << 10) | (1 << 13) | (1 << 14) | (1 << 15) | \
+			 (1 << 17) | (1 << 20) | (1 << 24) | (1 << 26) | \
+			 (1 << 27) | (1 << 30) | (1 << 31))
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+#define ENDIAN_SET_EL2		SCTLR_ELx_EE
+#define ENDIAN_CLEAR_EL2	0
+#else
+#define ENDIAN_SET_EL2		0
+#define ENDIAN_CLEAR_EL2	SCTLR_ELx_EE
+#endif
+
+/* SCTLR_EL2 value used for the hyp-stub */
+#define SCTLR_EL2_SET	(SCTLR_ELx_IESB   | ENDIAN_SET_EL2   | SCTLR_EL2_RES1)
+#define SCTLR_EL2_CLEAR	(SCTLR_ELx_M      | SCTLR_ELx_A    | SCTLR_ELx_C   | \
+			 SCTLR_ELx_SA     | SCTLR_ELx_I    | SCTLR_ELx_WXN | \
+			 ENDIAN_CLEAR_EL2 | SCTLR_EL2_RES0)
+
+/* Check all the bits are accounted for */
+#define SCTLR_EL2_BUILD_BUG_ON_MISSING_BITS	BUILD_BUG_ON((SCTLR_EL2_SET ^ SCTLR_EL2_CLEAR) != ~0)
 
-#define SCTLR_ELx_FLAGS	(SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \
-			 SCTLR_ELx_SA | SCTLR_ELx_I)
 
 /* SCTLR_EL1 specific flags. */
 #define SCTLR_EL1_UCI		(1 << 26)
+#define SCTLR_EL1_E0E		(1 << 24)
 #define SCTLR_EL1_SPAN		(1 << 23)
+#define SCTLR_EL1_NTWE		(1 << 18)
+#define SCTLR_EL1_NTWI		(1 << 16)
 #define SCTLR_EL1_UCT		(1 << 15)
+#define SCTLR_EL1_DZE		(1 << 14)
+#define SCTLR_EL1_UMA		(1 << 9)
 #define SCTLR_EL1_SED		(1 << 8)
+#define SCTLR_EL1_ITD		(1 << 7)
 #define SCTLR_EL1_CP15BEN	(1 << 5)
+#define SCTLR_EL1_SA0		(1 << 4)
+
+#define SCTLR_EL1_RES1	((1 << 11) | (1 << 20) | (1 << 22) | (1 << 28) | \
+			 (1 << 29))
+#define SCTLR_EL1_RES0  ((1 << 6)  | (1 << 10) | (1 << 13) | (1 << 17) | \
+			 (1 << 27) | (1 << 30) | (1 << 31))
+
+#ifdef CONFIG_CPU_BIG_ENDIAN
+#define ENDIAN_SET_EL1		(SCTLR_EL1_E0E | SCTLR_ELx_EE)
+#define ENDIAN_CLEAR_EL1	0
+#else
+#define ENDIAN_SET_EL1		0
+#define ENDIAN_CLEAR_EL1	(SCTLR_EL1_E0E | SCTLR_ELx_EE)
+#endif
+
+#define SCTLR_EL1_SET	(SCTLR_ELx_M    | SCTLR_ELx_C    | SCTLR_ELx_SA   |\
+			 SCTLR_EL1_SA0  | SCTLR_EL1_SED  | SCTLR_ELx_I    |\
+			 SCTLR_EL1_DZE  | SCTLR_EL1_UCT  | SCTLR_EL1_NTWI |\
+			 SCTLR_EL1_NTWE | SCTLR_ELx_IESB | SCTLR_EL1_SPAN |\
+			 ENDIAN_SET_EL1 | SCTLR_EL1_UCI  | SCTLR_EL1_RES1)
+#define SCTLR_EL1_CLEAR	(SCTLR_ELx_A   | SCTLR_EL1_CP15BEN | SCTLR_EL1_ITD    |\
+			 SCTLR_EL1_UMA | SCTLR_ELx_WXN     | ENDIAN_CLEAR_EL1 |\
+			 SCTLR_EL1_RES0)
+
+/* Check all the bits are accounted for */
+#define SCTLR_EL1_BUILD_BUG_ON_MISSING_BITS	BUILD_BUG_ON((SCTLR_EL1_SET ^ SCTLR_EL1_CLEAR) != ~0)
 
 /* id_aa64isar0 */
+#define ID_AA64ISAR0_FHM_SHIFT		48
 #define ID_AA64ISAR0_DP_SHIFT		44
 #define ID_AA64ISAR0_SM4_SHIFT		40
 #define ID_AA64ISAR0_SM3_SHIFT		36
@@ -437,7 +509,10 @@
 #define ID_AA64ISAR1_DPB_SHIFT		0
 
 /* id_aa64pfr0 */
+#define ID_AA64PFR0_CSV3_SHIFT		60
+#define ID_AA64PFR0_CSV2_SHIFT		56
 #define ID_AA64PFR0_SVE_SHIFT		32
+#define ID_AA64PFR0_RAS_SHIFT		28
 #define ID_AA64PFR0_GIC_SHIFT		24
 #define ID_AA64PFR0_ASIMD_SHIFT		20
 #define ID_AA64PFR0_FP_SHIFT		16
@@ -447,6 +522,7 @@
 #define ID_AA64PFR0_EL0_SHIFT		0
 
 #define ID_AA64PFR0_SVE			0x1
+#define ID_AA64PFR0_RAS_V1		0x1
 #define ID_AA64PFR0_FP_NI		0xf
 #define ID_AA64PFR0_FP_SUPPORTED	0x0
 #define ID_AA64PFR0_ASIMD_NI		0xf
@@ -471,6 +547,14 @@
 #define ID_AA64MMFR0_TGRAN64_SUPPORTED	0x0
 #define ID_AA64MMFR0_TGRAN16_NI		0x0
 #define ID_AA64MMFR0_TGRAN16_SUPPORTED	0x1
+#define ID_AA64MMFR0_PARANGE_48		0x5
+#define ID_AA64MMFR0_PARANGE_52		0x6
+
+#ifdef CONFIG_ARM64_PA_BITS_52
+#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_52
+#else
+#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_48
+#endif
 
 /* id_aa64mmfr1 */
 #define ID_AA64MMFR1_PAN_SHIFT		20
@@ -582,6 +666,7 @@
 
 #else
 
+#include <linux/build_bug.h>
 #include <linux/types.h>
 
 asm(
@@ -638,6 +723,9 @@ static inline void config_sctlr_el1(u32 clear, u32 set)
 {
 	u32 val;
 
+	SCTLR_EL2_BUILD_BUG_ON_MISSING_BITS;
+	SCTLR_EL1_BUILD_BUG_ON_MISSING_BITS;
+
 	val = read_sysreg(sctlr_el1);
 	val &= ~clear;
 	val |= set;
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index eb431286bacd..740aa03c5f0d 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -51,8 +51,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,					\
 }
 
-#define init_stack		(init_thread_union.stack)
-
 #define thread_saved_pc(tsk)	\
 	((unsigned long)(tsk->thread.cpu_context.pc))
 #define thread_saved_sp(tsk)	\
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index af1c76981911..9e82dd79c7db 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -23,6 +23,7 @@
 
 #include <linux/sched.h>
 #include <asm/cputype.h>
+#include <asm/mmu.h>
 
 /*
  * Raw TLBI operations.
@@ -54,6 +55,11 @@
 
 #define __tlbi(op, ...)		__TLBI_N(op, ##__VA_ARGS__, 1, 0)
 
+#define __tlbi_user(op, arg) do {						\
+	if (arm64_kernel_unmapped_at_el0())					\
+		__tlbi(op, (arg) | USER_ASID_FLAG);				\
+} while (0)
+
 /*
  *	TLB Management
  *	==============
@@ -115,6 +121,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
 
 	dsb(ishst);
 	__tlbi(aside1is, asid);
+	__tlbi_user(aside1is, asid);
 	dsb(ish);
 }
 
@@ -125,6 +132,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
 
 	dsb(ishst);
 	__tlbi(vale1is, addr);
+	__tlbi_user(vale1is, addr);
 	dsb(ish);
 }
 
@@ -151,10 +159,13 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
 
 	dsb(ishst);
 	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) {
-		if (last_level)
+		if (last_level) {
 			__tlbi(vale1is, addr);
-		else
+			__tlbi_user(vale1is, addr);
+		} else {
 			__tlbi(vae1is, addr);
+			__tlbi_user(vae1is, addr);
+		}
 	}
 	dsb(ish);
 }
@@ -194,6 +205,7 @@ static inline void __flush_tlb_pgtable(struct mm_struct *mm,
 	unsigned long addr = uaddr >> 12 | (ASID(mm) << 48);
 
 	__tlbi(vae1is, addr);
+	__tlbi_user(vae1is, addr);
 	dsb(ish);
 }
 
diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h
index 1696f9de9359..178e338d2889 100644
--- a/arch/arm64/include/asm/traps.h
+++ b/arch/arm64/include/asm/traps.h
@@ -19,6 +19,7 @@
 #define __ASM_TRAP_H
 
 #include <linux/list.h>
+#include <asm/esr.h>
 #include <asm/sections.h>
 
 struct pt_regs;
@@ -66,4 +67,57 @@ static inline int in_entry_text(unsigned long ptr)
 	return ptr >= (unsigned long)&__entry_text_start &&
 	       ptr < (unsigned long)&__entry_text_end;
 }
+
+/*
+ * CPUs with the RAS extensions have an Implementation-Defined-Syndrome bit
+ * to indicate whether this ESR has a RAS encoding. CPUs without this feature
+ * have a ISS-Valid bit in the same position.
+ * If this bit is set, we know its not a RAS SError.
+ * If its clear, we need to know if the CPU supports RAS. Uncategorized RAS
+ * errors share the same encoding as an all-zeros encoding from a CPU that
+ * doesn't support RAS.
+ */
+static inline bool arm64_is_ras_serror(u32 esr)
+{
+	WARN_ON(preemptible());
+
+	if (esr & ESR_ELx_IDS)
+		return false;
+
+	if (this_cpu_has_cap(ARM64_HAS_RAS_EXTN))
+		return true;
+	else
+		return false;
+}
+
+/*
+ * Return the AET bits from a RAS SError's ESR.
+ *
+ * It is implementation defined whether Uncategorized errors are containable.
+ * We treat them as Uncontainable.
+ * Non-RAS SError's are reported as Uncontained/Uncategorized.
+ */
+static inline u32 arm64_ras_serror_get_severity(u32 esr)
+{
+	u32 aet = esr & ESR_ELx_AET;
+
+	if (!arm64_is_ras_serror(esr)) {
+		/* Not a RAS error, we can't interpret the ESR. */
+		return ESR_ELx_AET_UC;
+	}
+
+	/*
+	 * AET is RES0 if 'the value returned in the DFSC field is not
+	 * [ESR_ELx_FSC_SERROR]'
+	 */
+	if ((esr & ESR_ELx_FSC) != ESR_ELx_FSC_SERROR) {
+		/* No severity information : Uncategorized */
+		return ESR_ELx_AET_UC;
+	}
+
+	return aet;
+}
+
+bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr);
+void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr);
 #endif
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index fc0f9eb66039..e66b0fca99c2 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -35,16 +35,20 @@
 #include <asm/compiler.h>
 #include <asm/extable.h>
 
-#define KERNEL_DS	(-1UL)
 #define get_ds()	(KERNEL_DS)
-
-#define USER_DS		TASK_SIZE_64
 #define get_fs()	(current_thread_info()->addr_limit)
 
 static inline void set_fs(mm_segment_t fs)
 {
 	current_thread_info()->addr_limit = fs;
 
+	/*
+	 * Prevent a mispredicted conditional call to set_fs from forwarding
+	 * the wrong address limit to access_ok under speculation.
+	 */
+	dsb(nsh);
+	isb();
+
 	/* On user-mode return, check fs is correct */
 	set_thread_flag(TIF_FSCHECK);
 
@@ -66,22 +70,32 @@ static inline void set_fs(mm_segment_t fs)
  * Returns 1 if the range is valid, 0 otherwise.
  *
  * This is equivalent to the following test:
- * (u65)addr + (u65)size <= current->addr_limit
- *
- * This needs 65-bit arithmetic.
+ * (u65)addr + (u65)size <= (u65)current->addr_limit + 1
  */
-#define __range_ok(addr, size)						\
-({									\
-	unsigned long __addr = (unsigned long)(addr);			\
-	unsigned long flag, roksum;					\
-	__chk_user_ptr(addr);						\
-	asm("adds %1, %1, %3; ccmp %1, %4, #2, cc; cset %0, ls"		\
-		: "=&r" (flag), "=&r" (roksum)				\
-		: "1" (__addr), "Ir" (size),				\
-		  "r" (current_thread_info()->addr_limit)		\
-		: "cc");						\
-	flag;								\
-})
+static inline unsigned long __range_ok(const void __user *addr, unsigned long size)
+{
+	unsigned long ret, limit = current_thread_info()->addr_limit;
+
+	__chk_user_ptr(addr);
+	asm volatile(
+	// A + B <= C + 1 for all A,B,C, in four easy steps:
+	// 1: X = A + B; X' = X % 2^64
+	"	adds	%0, %3, %2\n"
+	// 2: Set C = 0 if X > 2^64, to guarantee X' > C in step 4
+	"	csel	%1, xzr, %1, hi\n"
+	// 3: Set X' = ~0 if X >= 2^64. For X == 2^64, this decrements X'
+	//    to compensate for the carry flag being set in step 4. For
+	//    X > 2^64, X' merely has to remain nonzero, which it does.
+	"	csinv	%0, %0, xzr, cc\n"
+	// 4: For X < 2^64, this gives us X' - C - 1 <= 0, where the -1
+	//    comes from the carry in being clear. Otherwise, we are
+	//    testing X' - C == 0, subject to the previous adjustments.
+	"	sbcs	xzr, %0, %1\n"
+	"	cset	%0, ls\n"
+	: "=&r" (ret), "+r" (limit) : "Ir" (size), "0" (addr) : "cc");
+
+	return ret;
+}
 
 /*
  * When dealing with data aborts, watchpoints, or instruction traps we may end
@@ -105,17 +119,23 @@ static inline void set_fs(mm_segment_t fs)
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 static inline void __uaccess_ttbr0_disable(void)
 {
-	unsigned long ttbr;
+	unsigned long flags, ttbr;
 
-	/* reserved_ttbr0 placed at the end of swapper_pg_dir */
-	ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
-	write_sysreg(ttbr, ttbr0_el1);
+	local_irq_save(flags);
+	ttbr = read_sysreg(ttbr1_el1);
+	ttbr &= ~TTBR_ASID_MASK;
+	/* reserved_ttbr0 placed before swapper_pg_dir */
+	write_sysreg(ttbr - RESERVED_TTBR0_SIZE, ttbr0_el1);
+	isb();
+	/* Set reserved ASID */
+	write_sysreg(ttbr, ttbr1_el1);
 	isb();
+	local_irq_restore(flags);
 }
 
 static inline void __uaccess_ttbr0_enable(void)
 {
-	unsigned long flags;
+	unsigned long flags, ttbr0, ttbr1;
 
 	/*
 	 * Disable interrupts to avoid preemption between reading the 'ttbr0'
@@ -123,7 +143,17 @@ static inline void __uaccess_ttbr0_enable(void)
 	 * roll-over and an update of 'ttbr0'.
 	 */
 	local_irq_save(flags);
-	write_sysreg(current_thread_info()->ttbr0, ttbr0_el1);
+	ttbr0 = READ_ONCE(current_thread_info()->ttbr0);
+
+	/* Restore active ASID */
+	ttbr1 = read_sysreg(ttbr1_el1);
+	ttbr1 &= ~TTBR_ASID_MASK;		/* safety measure */
+	ttbr1 |= ttbr0 & TTBR_ASID_MASK;
+	write_sysreg(ttbr1, ttbr1_el1);
+	isb();
+
+	/* Restore user page table */
+	write_sysreg(ttbr0, ttbr0_el1);
 	isb();
 	local_irq_restore(flags);
 }
@@ -155,6 +185,18 @@ static inline bool uaccess_ttbr0_enable(void)
 }
 #endif
 
+static inline void __uaccess_disable_hw_pan(void)
+{
+	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN,
+			CONFIG_ARM64_PAN));
+}
+
+static inline void __uaccess_enable_hw_pan(void)
+{
+	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
+			CONFIG_ARM64_PAN));
+}
+
 #define __uaccess_disable(alt)						\
 do {									\
 	if (!uaccess_ttbr0_disable())					\
@@ -192,6 +234,26 @@ static inline void uaccess_enable_not_uao(void)
 	__uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
 }
 
+/*
+ * Sanitise a uaccess pointer such that it becomes NULL if above the
+ * current addr_limit.
+ */
+#define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
+static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
+{
+	void __user *safe_ptr;
+
+	asm volatile(
+	"	bics	xzr, %1, %2\n"
+	"	csel	%0, %1, xzr, eq\n"
+	: "=&r" (safe_ptr)
+	: "r" (ptr), "r" (current_thread_info()->addr_limit)
+	: "cc");
+
+	csdb();
+	return safe_ptr;
+}
+
 /*
  * The "__xxx" versions of the user access functions do not verify the address
  * space - it must have been done previously with a separate "access_ok()"
@@ -244,28 +306,33 @@ do {									\
 	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
 } while (0)
 
-#define __get_user(x, ptr)						\
+#define __get_user_check(x, ptr, err)					\
 ({									\
-	int __gu_err = 0;						\
-	__get_user_err((x), (ptr), __gu_err);				\
-	__gu_err;							\
+	__typeof__(*(ptr)) __user *__p = (ptr);				\
+	might_fault();							\
+	if (access_ok(VERIFY_READ, __p, sizeof(*__p))) {		\
+		__p = uaccess_mask_ptr(__p);				\
+		__get_user_err((x), __p, (err));			\
+	} else {							\
+		(x) = 0; (err) = -EFAULT;				\
+	}								\
 })
 
 #define __get_user_error(x, ptr, err)					\
 ({									\
-	__get_user_err((x), (ptr), (err));				\
+	__get_user_check((x), (ptr), (err));				\
 	(void)0;							\
 })
 
-#define get_user(x, ptr)						\
+#define __get_user(x, ptr)						\
 ({									\
-	__typeof__(*(ptr)) __user *__p = (ptr);				\
-	might_fault();							\
-	access_ok(VERIFY_READ, __p, sizeof(*__p)) ?			\
-		__get_user((x), __p) :					\
-		((x) = 0, -EFAULT);					\
+	int __gu_err = 0;						\
+	__get_user_check((x), (ptr), __gu_err);				\
+	__gu_err;							\
 })
 
+#define get_user	__get_user
+
 #define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature)	\
 	asm volatile(							\
 	"1:"ALTERNATIVE(instr "     " reg "1, [%2]\n",			\
@@ -308,43 +375,63 @@ do {									\
 	uaccess_disable_not_uao();					\
 } while (0)
 
-#define __put_user(x, ptr)						\
+#define __put_user_check(x, ptr, err)					\
 ({									\
-	int __pu_err = 0;						\
-	__put_user_err((x), (ptr), __pu_err);				\
-	__pu_err;							\
+	__typeof__(*(ptr)) __user *__p = (ptr);				\
+	might_fault();							\
+	if (access_ok(VERIFY_WRITE, __p, sizeof(*__p))) {		\
+		__p = uaccess_mask_ptr(__p);				\
+		__put_user_err((x), __p, (err));			\
+	} else	{							\
+		(err) = -EFAULT;					\
+	}								\
 })
 
 #define __put_user_error(x, ptr, err)					\
 ({									\
-	__put_user_err((x), (ptr), (err));				\
+	__put_user_check((x), (ptr), (err));				\
 	(void)0;							\
 })
 
-#define put_user(x, ptr)						\
+#define __put_user(x, ptr)						\
 ({									\
-	__typeof__(*(ptr)) __user *__p = (ptr);				\
-	might_fault();							\
-	access_ok(VERIFY_WRITE, __p, sizeof(*__p)) ?			\
-		__put_user((x), __p) :					\
-		-EFAULT;						\
+	int __pu_err = 0;						\
+	__put_user_check((x), (ptr), __pu_err);				\
+	__pu_err;							\
 })
 
+#define put_user	__put_user
+
 extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
-#define raw_copy_from_user __arch_copy_from_user
+#define raw_copy_from_user(to, from, n)					\
+({									\
+	__arch_copy_from_user((to), __uaccess_mask_ptr(from), (n));	\
+})
+
 extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
-#define raw_copy_to_user __arch_copy_to_user
-extern unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, unsigned long n);
-extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n);
+#define raw_copy_to_user(to, from, n)					\
+({									\
+	__arch_copy_to_user(__uaccess_mask_ptr(to), (from), (n));	\
+})
+
+extern unsigned long __must_check __arch_copy_in_user(void __user *to, const void __user *from, unsigned long n);
+#define raw_copy_in_user(to, from, n)					\
+({									\
+	__arch_copy_in_user(__uaccess_mask_ptr(to),			\
+			    __uaccess_mask_ptr(from), (n));		\
+})
+
 #define INLINE_COPY_TO_USER
 #define INLINE_COPY_FROM_USER
 
-static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
+extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
+static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
 {
 	if (access_ok(VERIFY_WRITE, to, n))
-		n = __clear_user(to, n);
+		n = __arch_clear_user(__uaccess_mask_ptr(to), n);
 	return n;
 }
+#define clear_user	__clear_user
 
 extern long strncpy_from_user(char *dest, const char __user *src, long count);
 
@@ -358,7 +445,7 @@ extern unsigned long __must_check __copy_user_flushcache(void *to, const void __
 static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
 {
 	kasan_check_write(dst, size);
-	return __copy_user_flushcache(dst, src, size);
+	return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size);
 }
 #endif
 
diff --git a/arch/arm64/include/asm/vmap_stack.h b/arch/arm64/include/asm/vmap_stack.h
new file mode 100644
index 000000000000..0b5ec6e08c10
--- /dev/null
+++ b/arch/arm64/include/asm/vmap_stack.h
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2017 Arm Ltd.
+#ifndef __ASM_VMAP_STACK_H
+#define __ASM_VMAP_STACK_H
+
+#include <linux/bug.h>
+#include <linux/gfp.h>
+#include <linux/kconfig.h>
+#include <linux/vmalloc.h>
+#include <asm/memory.h>
+#include <asm/pgtable.h>
+#include <asm/thread_info.h>
+
+/*
+ * To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd
+ * stacks need to have the same alignment.
+ */
+static inline unsigned long *arch_alloc_vmap_stack(size_t stack_size, int node)
+{
+	BUILD_BUG_ON(!IS_ENABLED(CONFIG_VMAP_STACK));
+
+	return __vmalloc_node_range(stack_size, THREAD_ALIGN,
+				    VMALLOC_START, VMALLOC_END,
+				    THREADINFO_GFP, PAGE_KERNEL, 0, node,
+				    __builtin_return_address(0));
+}
+
+#endif /* __ASM_VMAP_STACK_H */
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index cda76fa8b9b2..f018c3deea3b 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -43,5 +43,6 @@
 #define HWCAP_ASIMDDP		(1 << 20)
 #define HWCAP_SHA512		(1 << 21)
 #define HWCAP_SVE		(1 << 22)
+#define HWCAP_ASIMDFHM		(1 << 23)
 
 #endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/include/uapi/asm/siginfo.h b/arch/arm64/include/uapi/asm/siginfo.h
index 574d12f86039..9b4d91277742 100644
--- a/arch/arm64/include/uapi/asm/siginfo.h
+++ b/arch/arm64/include/uapi/asm/siginfo.h
@@ -21,4 +21,25 @@
 
 #include <asm-generic/siginfo.h>
 
+/*
+ * SIGFPE si_codes
+ */
+#ifdef __KERNEL__
+#define FPE_FIXME	0	/* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
+/*
+ * SIGBUS si_codes
+ */
+#ifdef __KERNEL__
+#define BUS_FIXME	0	/* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
+/*
+ * SIGTRAP si_codes
+ */
+#ifdef __KERNEL__
+#define TRAP_FIXME	0	/* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
 #endif
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 067baace74a0..b87541360f43 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -52,6 +52,11 @@ arm64-obj-$(CONFIG_KEXEC)		+= machine_kexec.o relocate_kernel.o	\
 arm64-obj-$(CONFIG_ARM64_RELOC_TEST)	+= arm64-reloc-test.o
 arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o
 arm64-obj-$(CONFIG_CRASH_DUMP)		+= crash_dump.o
+arm64-obj-$(CONFIG_ARM_SDE_INTERFACE)	+= sdei.o
+
+ifeq ($(CONFIG_KVM),y)
+arm64-obj-$(CONFIG_HARDEN_BRANCH_PREDICTOR)	+= bpi.o
+endif
 
 obj-y					+= $(arm64-obj-y) vdso/ probes/
 obj-m					+= $(arm64-obj-m)
diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c
index b3162715ed78..7b09487ff8fb 100644
--- a/arch/arm64/kernel/acpi.c
+++ b/arch/arm64/kernel/acpi.c
@@ -117,7 +117,7 @@ bool __init acpi_psci_present(void)
 }
 
 /* Whether HVC must be used instead of SMC as the PSCI conduit */
-bool __init acpi_psci_use_hvc(void)
+bool acpi_psci_use_hvc(void)
 {
 	return acpi_gbl_FADT.arm_boot_flags & ACPI_FADT_PSCI_USE_HVC;
 }
@@ -230,10 +230,10 @@ void __init acpi_boot_table_init(void)
 
 done:
 	if (acpi_disabled) {
-		if (earlycon_init_is_deferred)
+		if (earlycon_acpi_spcr_enable)
 			early_init_dt_scan_chosen_stdout();
 	} else {
-		parse_spcr(earlycon_init_is_deferred);
+		acpi_parse_spcr(earlycon_acpi_spcr_enable, true);
 		if (IS_ENABLED(CONFIG_ACPI_BGRT))
 			acpi_table_parse(ACPI_SIG_BGRT, acpi_parse_bgrt);
 	}
diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 6dd0a3a3e5c9..414288a558c8 100644
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -32,6 +32,8 @@
 #define ALT_ORIG_PTR(a)		__ALT_PTR(a, orig_offset)
 #define ALT_REPL_PTR(a)		__ALT_PTR(a, alt_offset)
 
+int alternatives_applied;
+
 struct alt_region {
 	struct alt_instr *begin;
 	struct alt_instr *end;
@@ -143,7 +145,6 @@ static void __apply_alternatives(void *alt_region, bool use_linear_alias)
  */
 static int __apply_alternatives_multi_stop(void *unused)
 {
-	static int patched = 0;
 	struct alt_region region = {
 		.begin	= (struct alt_instr *)__alt_instructions,
 		.end	= (struct alt_instr *)__alt_instructions_end,
@@ -151,14 +152,14 @@ static int __apply_alternatives_multi_stop(void *unused)
 
 	/* We always have a CPU 0 at this point (__init) */
 	if (smp_processor_id()) {
-		while (!READ_ONCE(patched))
+		while (!READ_ONCE(alternatives_applied))
 			cpu_relax();
 		isb();
 	} else {
-		BUG_ON(patched);
+		BUG_ON(alternatives_applied);
 		__apply_alternatives(&region, true);
 		/* Barriers provided by the cache flushing */
-		WRITE_ONCE(patched, 1);
+		WRITE_ONCE(alternatives_applied, 1);
 	}
 
 	return 0;
diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
index 67368c7329c0..66be504edb6c 100644
--- a/arch/arm64/kernel/arm64ksyms.c
+++ b/arch/arm64/kernel/arm64ksyms.c
@@ -37,8 +37,8 @@ EXPORT_SYMBOL(clear_page);
 	/* user mem (segment) */
 EXPORT_SYMBOL(__arch_copy_from_user);
 EXPORT_SYMBOL(__arch_copy_to_user);
-EXPORT_SYMBOL(__clear_user);
-EXPORT_SYMBOL(raw_copy_in_user);
+EXPORT_SYMBOL(__arch_clear_user);
+EXPORT_SYMBOL(__arch_copy_in_user);
 
 	/* physical memory */
 EXPORT_SYMBOL(memstart_addr);
diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c
index c33b5e4010ab..68450e954d47 100644
--- a/arch/arm64/kernel/armv8_deprecated.c
+++ b/arch/arm64/kernel/armv8_deprecated.c
@@ -370,6 +370,7 @@ static unsigned int __kprobes aarch32_check_condition(u32 opcode, u32 psr)
 static int swp_handler(struct pt_regs *regs, u32 instr)
 {
 	u32 destreg, data, type, address = 0;
+	const void __user *user_ptr;
 	int rn, rt2, res = 0;
 
 	perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
@@ -401,7 +402,8 @@ static int swp_handler(struct pt_regs *regs, u32 instr)
 		aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
 
 	/* Check access in reasonable access range for both SWP and SWPB */
-	if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) {
+	user_ptr = (const void __user *)(unsigned long)(address & ~3);
+	if (!access_ok(VERIFY_WRITE, user_ptr, 4)) {
 		pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
 			address);
 		goto fault;
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 71bf088f1e4b..1303e04110cd 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -18,12 +18,14 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/arm_sdei.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
 #include <linux/kvm_host.h>
 #include <linux/suspend.h>
 #include <asm/cpufeature.h>
+#include <asm/fixmap.h>
 #include <asm/thread_info.h>
 #include <asm/memory.h>
 #include <asm/smp_plat.h>
@@ -130,6 +132,7 @@ int main(void)
   BLANK();
 #ifdef CONFIG_KVM_ARM_HOST
   DEFINE(VCPU_CONTEXT,		offsetof(struct kvm_vcpu, arch.ctxt));
+  DEFINE(VCPU_FAULT_DISR,	offsetof(struct kvm_vcpu, arch.fault.disr_el1));
   DEFINE(CPU_GP_REGS,		offsetof(struct kvm_cpu_context, gp_regs));
   DEFINE(CPU_USER_PT_REGS,	offsetof(struct kvm_regs, regs));
   DEFINE(CPU_FP_REGS,		offsetof(struct kvm_regs, fp_regs));
@@ -148,11 +151,18 @@ int main(void)
   DEFINE(ARM_SMCCC_RES_X2_OFFS,		offsetof(struct arm_smccc_res, a2));
   DEFINE(ARM_SMCCC_QUIRK_ID_OFFS,	offsetof(struct arm_smccc_quirk, id));
   DEFINE(ARM_SMCCC_QUIRK_STATE_OFFS,	offsetof(struct arm_smccc_quirk, state));
-
   BLANK();
   DEFINE(HIBERN_PBE_ORIG,	offsetof(struct pbe, orig_address));
   DEFINE(HIBERN_PBE_ADDR,	offsetof(struct pbe, address));
   DEFINE(HIBERN_PBE_NEXT,	offsetof(struct pbe, next));
   DEFINE(ARM64_FTR_SYSVAL,	offsetof(struct arm64_ftr_reg, sys_val));
+  BLANK();
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+  DEFINE(TRAMP_VALIAS,		TRAMP_VALIAS);
+#endif
+#ifdef CONFIG_ARM_SDE_INTERFACE
+  DEFINE(SDEI_EVENT_INTREGS,	offsetof(struct sdei_registered_event, interrupted_regs));
+  DEFINE(SDEI_EVENT_PRIORITY,	offsetof(struct sdei_registered_event, priority));
+#endif
   return 0;
 }
diff --git a/arch/arm64/kernel/bpi.S b/arch/arm64/kernel/bpi.S
new file mode 100644
index 000000000000..e5de33513b5d
--- /dev/null
+++ b/arch/arm64/kernel/bpi.S
@@ -0,0 +1,83 @@
+/*
+ * Contains CPU specific branch predictor invalidation sequences
+ *
+ * Copyright (C) 2018 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/linkage.h>
+#include <linux/arm-smccc.h>
+
+.macro ventry target
+	.rept 31
+	nop
+	.endr
+	b	\target
+.endm
+
+.macro vectors target
+	ventry \target + 0x000
+	ventry \target + 0x080
+	ventry \target + 0x100
+	ventry \target + 0x180
+
+	ventry \target + 0x200
+	ventry \target + 0x280
+	ventry \target + 0x300
+	ventry \target + 0x380
+
+	ventry \target + 0x400
+	ventry \target + 0x480
+	ventry \target + 0x500
+	ventry \target + 0x580
+
+	ventry \target + 0x600
+	ventry \target + 0x680
+	ventry \target + 0x700
+	ventry \target + 0x780
+.endm
+
+	.align	11
+ENTRY(__bp_harden_hyp_vecs_start)
+	.rept 4
+	vectors __kvm_hyp_vector
+	.endr
+ENTRY(__bp_harden_hyp_vecs_end)
+
+ENTRY(__qcom_hyp_sanitize_link_stack_start)
+	stp     x29, x30, [sp, #-16]!
+	.rept	16
+	bl	. + 4
+	.endr
+	ldp	x29, x30, [sp], #16
+ENTRY(__qcom_hyp_sanitize_link_stack_end)
+
+.macro smccc_workaround_1 inst
+	sub	sp, sp, #(8 * 4)
+	stp	x2, x3, [sp, #(8 * 0)]
+	stp	x0, x1, [sp, #(8 * 2)]
+	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_1
+	\inst	#0
+	ldp	x2, x3, [sp, #(8 * 0)]
+	ldp	x0, x1, [sp, #(8 * 2)]
+	add	sp, sp, #(8 * 4)
+.endm
+
+ENTRY(__smccc_workaround_1_smc_start)
+	smccc_workaround_1	smc
+ENTRY(__smccc_workaround_1_smc_end)
+
+ENTRY(__smccc_workaround_1_hvc_start)
+	smccc_workaround_1	hvc
+ENTRY(__smccc_workaround_1_hvc_end)
diff --git a/arch/arm64/kernel/cpu-reset.S b/arch/arm64/kernel/cpu-reset.S
index 2a752cb2a0f3..8021b46c9743 100644
--- a/arch/arm64/kernel/cpu-reset.S
+++ b/arch/arm64/kernel/cpu-reset.S
@@ -16,7 +16,7 @@
 #include <asm/virt.h>
 
 .text
-.pushsection    .idmap.text, "ax"
+.pushsection    .idmap.text, "awx"
 
 /*
  * __cpu_soft_restart(el2_switch, entry, arg0, arg1, arg2) - Helper for
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 0e27f86ee709..b5a28336c077 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -30,6 +30,20 @@ is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
 				       entry->midr_range_max);
 }
 
+static bool __maybe_unused
+is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u32 model;
+
+	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
+
+	model = read_cpuid_id();
+	model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
+		 MIDR_ARCHITECTURE_MASK;
+
+	return model == entry->midr_model;
+}
+
 static bool
 has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
 				int scope)
@@ -46,6 +60,174 @@ static int cpu_enable_trap_ctr_access(void *__unused)
 	return 0;
 }
 
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+#include <asm/mmu_context.h>
+#include <asm/cacheflush.h>
+
+DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
+
+#ifdef CONFIG_KVM
+extern char __qcom_hyp_sanitize_link_stack_start[];
+extern char __qcom_hyp_sanitize_link_stack_end[];
+extern char __smccc_workaround_1_smc_start[];
+extern char __smccc_workaround_1_smc_end[];
+extern char __smccc_workaround_1_hvc_start[];
+extern char __smccc_workaround_1_hvc_end[];
+
+static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
+				const char *hyp_vecs_end)
+{
+	void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
+	int i;
+
+	for (i = 0; i < SZ_2K; i += 0x80)
+		memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
+
+	flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
+}
+
+static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
+				      const char *hyp_vecs_start,
+				      const char *hyp_vecs_end)
+{
+	static int last_slot = -1;
+	static DEFINE_SPINLOCK(bp_lock);
+	int cpu, slot = -1;
+
+	spin_lock(&bp_lock);
+	for_each_possible_cpu(cpu) {
+		if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
+			slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
+			break;
+		}
+	}
+
+	if (slot == -1) {
+		last_slot++;
+		BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start)
+			/ SZ_2K) <= last_slot);
+		slot = last_slot;
+		__copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
+	}
+
+	__this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
+	__this_cpu_write(bp_hardening_data.fn, fn);
+	spin_unlock(&bp_lock);
+}
+#else
+#define __qcom_hyp_sanitize_link_stack_start	NULL
+#define __qcom_hyp_sanitize_link_stack_end	NULL
+#define __smccc_workaround_1_smc_start		NULL
+#define __smccc_workaround_1_smc_end		NULL
+#define __smccc_workaround_1_hvc_start		NULL
+#define __smccc_workaround_1_hvc_end		NULL
+
+static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
+				      const char *hyp_vecs_start,
+				      const char *hyp_vecs_end)
+{
+	__this_cpu_write(bp_hardening_data.fn, fn);
+}
+#endif	/* CONFIG_KVM */
+
+static void  install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
+				     bp_hardening_cb_t fn,
+				     const char *hyp_vecs_start,
+				     const char *hyp_vecs_end)
+{
+	u64 pfr0;
+
+	if (!entry->matches(entry, SCOPE_LOCAL_CPU))
+		return;
+
+	pfr0 = read_cpuid(ID_AA64PFR0_EL1);
+	if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
+		return;
+
+	__install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
+}
+
+#include <uapi/linux/psci.h>
+#include <linux/arm-smccc.h>
+#include <linux/psci.h>
+
+static void call_smc_arch_workaround_1(void)
+{
+	arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static void call_hvc_arch_workaround_1(void)
+{
+	arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
+}
+
+static int enable_smccc_arch_workaround_1(void *data)
+{
+	const struct arm64_cpu_capabilities *entry = data;
+	bp_hardening_cb_t cb;
+	void *smccc_start, *smccc_end;
+	struct arm_smccc_res res;
+
+	if (!entry->matches(entry, SCOPE_LOCAL_CPU))
+		return 0;
+
+	if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
+		return 0;
+
+	switch (psci_ops.conduit) {
+	case PSCI_CONDUIT_HVC:
+		arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+				  ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+		if ((int)res.a0 < 0)
+			return 0;
+		cb = call_hvc_arch_workaround_1;
+		smccc_start = __smccc_workaround_1_hvc_start;
+		smccc_end = __smccc_workaround_1_hvc_end;
+		break;
+
+	case PSCI_CONDUIT_SMC:
+		arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
+				  ARM_SMCCC_ARCH_WORKAROUND_1, &res);
+		if ((int)res.a0 < 0)
+			return 0;
+		cb = call_smc_arch_workaround_1;
+		smccc_start = __smccc_workaround_1_smc_start;
+		smccc_end = __smccc_workaround_1_smc_end;
+		break;
+
+	default:
+		return 0;
+	}
+
+	install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
+
+	return 0;
+}
+
+static void qcom_link_stack_sanitization(void)
+{
+	u64 tmp;
+
+	asm volatile("mov	%0, x30		\n"
+		     ".rept	16		\n"
+		     "bl	. + 4		\n"
+		     ".endr			\n"
+		     "mov	x30, %0		\n"
+		     : "=&r" (tmp));
+}
+
+static int qcom_enable_link_stack_sanitization(void *data)
+{
+	const struct arm64_cpu_capabilities *entry = data;
+
+	install_bp_hardening_cb(entry, qcom_link_stack_sanitization,
+				__qcom_hyp_sanitize_link_stack_start,
+				__qcom_hyp_sanitize_link_stack_end);
+
+	return 0;
+}
+#endif	/* CONFIG_HARDEN_BRANCH_PREDICTOR */
+
 #define MIDR_RANGE(model, min, max) \
 	.def_scope = SCOPE_LOCAL_CPU, \
 	.matches = is_affected_midr_range, \
@@ -169,6 +351,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 			   MIDR_CPU_VAR_REV(0, 0),
 			   MIDR_CPU_VAR_REV(0, 0)),
 	},
+	{
+		.desc = "Qualcomm Technologies Kryo erratum 1003",
+		.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
+		.def_scope = SCOPE_LOCAL_CPU,
+		.midr_model = MIDR_QCOM_KRYO,
+		.matches = is_kryo_midr,
+	},
 #endif
 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
 	{
@@ -186,6 +375,56 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.capability = ARM64_WORKAROUND_858921,
 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
 	},
+#endif
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
+		.enable = enable_smccc_arch_workaround_1,
+	},
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
+		.enable = enable_smccc_arch_workaround_1,
+	},
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
+		.enable = enable_smccc_arch_workaround_1,
+	},
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
+		.enable = enable_smccc_arch_workaround_1,
+	},
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
+		.enable = qcom_enable_link_stack_sanitization,
+	},
+	{
+		.capability = ARM64_HARDEN_BP_POST_GUEST_EXIT,
+		MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
+	},
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR),
+		.enable = qcom_enable_link_stack_sanitization,
+	},
+	{
+		.capability = ARM64_HARDEN_BP_POST_GUEST_EXIT,
+		MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR),
+	},
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
+		.enable = enable_smccc_arch_workaround_1,
+	},
+	{
+		.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
+		MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
+		.enable = enable_smccc_arch_workaround_1,
+	},
 #endif
 	{
 	}
@@ -200,15 +439,18 @@ void verify_local_cpu_errata_workarounds(void)
 {
 	const struct arm64_cpu_capabilities *caps = arm64_errata;
 
-	for (; caps->matches; caps++)
-		if (!cpus_have_cap(caps->capability) &&
-			caps->matches(caps, SCOPE_LOCAL_CPU)) {
+	for (; caps->matches; caps++) {
+		if (cpus_have_cap(caps->capability)) {
+			if (caps->enable)
+				caps->enable((void *)caps);
+		} else if (caps->matches(caps, SCOPE_LOCAL_CPU)) {
 			pr_crit("CPU%d: Requires work around for %s, not detected"
 					" at boot time\n",
 				smp_processor_id(),
 				caps->desc ? : "an erratum");
 			cpu_die_early();
 		}
+	}
 }
 
 void update_cpu_errata_workarounds(void)
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index a73a5928f09b..2985a067fc13 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -123,6 +123,7 @@ cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
  * sync with the documentation of the CPU feature register ABI.
  */
 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_FHM_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM4_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM3_SHIFT, 4, 0),
@@ -145,8 +146,11 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
 				   FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
@@ -195,9 +199,11 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
 };
 
 static const struct arm64_ftr_bits ftr_ctr[] = {
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1),	/* RAO */
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1),		/* RES1 */
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 29, 1, 1),	/* DIC */
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 28, 1, 1),	/* IDC */
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0),	/* CWG */
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),	/* ERG */
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, 20, 4, 0),	/* ERG */
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1),	/* DminLine */
 	/*
 	 * Linux can handle differing I-cache policies. Userspace JITs will
@@ -846,6 +852,102 @@ static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unus
 					ID_AA64PFR0_FP_SHIFT) < 0;
 }
 
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
+
+static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
+				int __unused)
+{
+	char const *str = "command line option";
+	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
+
+	/*
+	 * For reasons that aren't entirely clear, enabling KPTI on Cavium
+	 * ThunderX leads to apparent I-cache corruption of kernel text, which
+	 * ends as well as you might imagine. Don't even try.
+	 */
+	if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
+		str = "ARM64_WORKAROUND_CAVIUM_27456";
+		__kpti_forced = -1;
+	}
+
+	/* Forced? */
+	if (__kpti_forced) {
+		pr_info_once("kernel page table isolation forced %s by %s\n",
+			     __kpti_forced > 0 ? "ON" : "OFF", str);
+		return __kpti_forced > 0;
+	}
+
+	/* Useful for KASLR robustness */
+	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
+		return true;
+
+	/* Don't force KPTI for CPUs that are not vulnerable */
+	switch (read_cpuid_id() & MIDR_CPU_MODEL_MASK) {
+	case MIDR_CAVIUM_THUNDERX2:
+	case MIDR_BRCM_VULCAN:
+		return false;
+	}
+
+	/* Defer to CPU feature registers */
+	return !cpuid_feature_extract_unsigned_field(pfr0,
+						     ID_AA64PFR0_CSV3_SHIFT);
+}
+
+static int kpti_install_ng_mappings(void *__unused)
+{
+	typedef void (kpti_remap_fn)(int, int, phys_addr_t);
+	extern kpti_remap_fn idmap_kpti_install_ng_mappings;
+	kpti_remap_fn *remap_fn;
+
+	static bool kpti_applied = false;
+	int cpu = smp_processor_id();
+
+	if (kpti_applied)
+		return 0;
+
+	remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
+
+	cpu_install_idmap();
+	remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
+	cpu_uninstall_idmap();
+
+	if (!cpu)
+		kpti_applied = true;
+
+	return 0;
+}
+
+static int __init parse_kpti(char *str)
+{
+	bool enabled;
+	int ret = strtobool(str, &enabled);
+
+	if (ret)
+		return ret;
+
+	__kpti_forced = enabled ? 1 : -1;
+	return 0;
+}
+__setup("kpti=", parse_kpti);
+#endif	/* CONFIG_UNMAP_KERNEL_AT_EL0 */
+
+static int cpu_copy_el2regs(void *__unused)
+{
+	/*
+	 * Copy register values that aren't redirected by hardware.
+	 *
+	 * Before code patching, we only set tpidr_el1, all CPUs need to copy
+	 * this value to tpidr_el2 before we patch the code. Once we've done
+	 * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
+	 * do anything here.
+	 */
+	if (!alternatives_applied)
+		write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
+
+	return 0;
+}
+
 static const struct arm64_cpu_capabilities arm64_features[] = {
 	{
 		.desc = "GIC system register CPU interface",
@@ -915,6 +1017,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.capability = ARM64_HAS_VIRT_HOST_EXTN,
 		.def_scope = SCOPE_SYSTEM,
 		.matches = runs_at_el2,
+		.enable = cpu_copy_el2regs,
 	},
 	{
 		.desc = "32-bit EL0 Support",
@@ -932,6 +1035,15 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.def_scope = SCOPE_SYSTEM,
 		.matches = hyp_offset_low,
 	},
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	{
+		.desc = "Kernel page table isolation (KPTI)",
+		.capability = ARM64_UNMAP_KERNEL_AT_EL0,
+		.def_scope = SCOPE_SYSTEM,
+		.matches = unmap_kernel_at_el0,
+		.enable = kpti_install_ng_mappings,
+	},
+#endif
 	{
 		/* FP/SIMD is not implemented */
 		.capability = ARM64_HAS_NO_FPSIMD,
@@ -963,6 +1075,19 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.enable = sve_kernel_enable,
 	},
 #endif /* CONFIG_ARM64_SVE */
+#ifdef CONFIG_ARM64_RAS_EXTN
+	{
+		.desc = "RAS Extension Support",
+		.capability = ARM64_HAS_RAS_EXTN,
+		.def_scope = SCOPE_SYSTEM,
+		.matches = has_cpuid_feature,
+		.sys_reg = SYS_ID_AA64PFR0_EL1,
+		.sign = FTR_UNSIGNED,
+		.field_pos = ID_AA64PFR0_RAS_SHIFT,
+		.min_field_value = ID_AA64PFR0_RAS_V1,
+		.enable = cpu_clear_disr,
+	},
+#endif /* CONFIG_ARM64_RAS_EXTN */
 	{},
 };
 
@@ -992,6 +1117,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM3),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM4),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDDP),
+	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_FHM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDFHM),
 	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP),
 	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
 	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
@@ -1071,6 +1197,25 @@ static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
 			cap_set_elf_hwcap(hwcaps);
 }
 
+/*
+ * Check if the current CPU has a given feature capability.
+ * Should be called from non-preemptible context.
+ */
+static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array,
+			       unsigned int cap)
+{
+	const struct arm64_cpu_capabilities *caps;
+
+	if (WARN_ON(preemptible()))
+		return false;
+
+	for (caps = cap_array; caps->matches; caps++)
+		if (caps->capability == cap &&
+		    caps->matches(caps, SCOPE_LOCAL_CPU))
+			return true;
+	return false;
+}
+
 void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
 			    const char *info)
 {
@@ -1106,7 +1251,7 @@ void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
 			 * uses an IPI, giving us a PSTATE that disappears when
 			 * we return.
 			 */
-			stop_machine(caps->enable, NULL, cpu_online_mask);
+			stop_machine(caps->enable, (void *)caps, cpu_online_mask);
 		}
 	}
 }
@@ -1134,8 +1279,9 @@ verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
 }
 
 static void
-verify_local_cpu_features(const struct arm64_cpu_capabilities *caps)
+verify_local_cpu_features(const struct arm64_cpu_capabilities *caps_list)
 {
+	const struct arm64_cpu_capabilities *caps = caps_list;
 	for (; caps->matches; caps++) {
 		if (!cpus_have_cap(caps->capability))
 			continue;
@@ -1143,13 +1289,13 @@ verify_local_cpu_features(const struct arm64_cpu_capabilities *caps)
 		 * If the new CPU misses an advertised feature, we cannot proceed
 		 * further, park the cpu.
 		 */
-		if (!caps->matches(caps, SCOPE_LOCAL_CPU)) {
+		if (!__this_cpu_has_cap(caps_list, caps->capability)) {
 			pr_crit("CPU%d: missing feature: %s\n",
 					smp_processor_id(), caps->desc);
 			cpu_die_early();
 		}
 		if (caps->enable)
-			caps->enable(NULL);
+			caps->enable((void *)caps);
 	}
 }
 
@@ -1189,6 +1335,9 @@ static void verify_local_cpu_capabilities(void)
 
 	if (system_supports_sve())
 		verify_sve_features();
+
+	if (system_uses_ttbr0_pan())
+		pr_info("Emulating Privileged Access Never (PAN) using TTBR0_EL1 switching\n");
 }
 
 void check_local_cpu_capabilities(void)
@@ -1225,25 +1374,6 @@ static void __init mark_const_caps_ready(void)
 	static_branch_enable(&arm64_const_caps_ready);
 }
 
-/*
- * Check if the current CPU has a given feature capability.
- * Should be called from non-preemptible context.
- */
-static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array,
-			       unsigned int cap)
-{
-	const struct arm64_cpu_capabilities *caps;
-
-	if (WARN_ON(preemptible()))
-		return false;
-
-	for (caps = cap_array; caps->desc; caps++)
-		if (caps->capability == cap && caps->matches)
-			return caps->matches(caps, SCOPE_LOCAL_CPU);
-
-	return false;
-}
-
 extern const struct arm64_cpu_capabilities arm64_errata[];
 
 bool this_cpu_has_cap(unsigned int cap)
@@ -1387,3 +1517,11 @@ static int __init enable_mrs_emulation(void)
 }
 
 core_initcall(enable_mrs_emulation);
+
+int cpu_clear_disr(void *__unused)
+{
+	/* Firmware may have left a deferred SError in this register. */
+	write_sysreg_s(0, SYS_DISR_EL1);
+
+	return 0;
+}
diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c
index fd691087dc9a..f2d13810daa8 100644
--- a/arch/arm64/kernel/cpuidle.c
+++ b/arch/arm64/kernel/cpuidle.c
@@ -47,6 +47,8 @@ int arm_cpuidle_suspend(int index)
 
 #include <acpi/processor.h>
 
+#define ARM64_LPI_IS_RETENTION_STATE(arch_flags) (!(arch_flags))
+
 int acpi_processor_ffh_lpi_probe(unsigned int cpu)
 {
 	return arm_cpuidle_init(cpu);
@@ -54,6 +56,10 @@ int acpi_processor_ffh_lpi_probe(unsigned int cpu)
 
 int acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi)
 {
-	return CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, lpi->index);
+	if (ARM64_LPI_IS_RETENTION_STATE(lpi->arch_flags))
+		return CPU_PM_CPU_IDLE_ENTER_RETENTION(arm_cpuidle_suspend,
+						lpi->index);
+	else
+		return CPU_PM_CPU_IDLE_ENTER(arm_cpuidle_suspend, lpi->index);
 }
 #endif
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 1e2554543506..7f94623df8a5 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -76,6 +76,7 @@ static const char *const hwcap_str[] = {
 	"asimddp",
 	"sha512",
 	"sve",
+	"asimdfhm",
 	NULL
 };
 
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index a88b6ccebbb4..53781f5687c5 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -209,12 +209,13 @@ NOKPROBE_SYMBOL(call_step_hook);
 static void send_user_sigtrap(int si_code)
 {
 	struct pt_regs *regs = current_pt_regs();
-	siginfo_t info = {
-		.si_signo	= SIGTRAP,
-		.si_errno	= 0,
-		.si_code	= si_code,
-		.si_addr	= (void __user *)instruction_pointer(regs),
-	};
+	siginfo_t info;
+
+	clear_siginfo(&info);
+	info.si_signo	= SIGTRAP;
+	info.si_errno	= 0;
+	info.si_code	= si_code;
+	info.si_addr	= (void __user *)instruction_pointer(regs);
 
 	if (WARN_ON(!user_mode(regs)))
 		return;
diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c
index 82cd07592519..a8bf1c892b90 100644
--- a/arch/arm64/kernel/efi.c
+++ b/arch/arm64/kernel/efi.c
@@ -48,7 +48,9 @@ static __init pteval_t create_mapping_protection(efi_memory_desc_t *md)
 		return pgprot_val(PAGE_KERNEL_ROX);
 
 	/* RW- */
-	if (attr & EFI_MEMORY_XP || type != EFI_RUNTIME_SERVICES_CODE)
+	if (((attr & (EFI_MEMORY_RP | EFI_MEMORY_WP | EFI_MEMORY_XP)) ==
+	     EFI_MEMORY_XP) ||
+	    type != EFI_RUNTIME_SERVICES_CODE)
 		return pgprot_val(PAGE_KERNEL);
 
 	/* RWX */
@@ -88,7 +90,7 @@ static int __init set_permissions(pte_t *ptep, pgtable_t token,
 				  unsigned long addr, void *data)
 {
 	efi_memory_desc_t *md = data;
-	pte_t pte = *ptep;
+	pte_t pte = READ_ONCE(*ptep);
 
 	if (md->attribute & EFI_MEMORY_RO)
 		pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 6d14b8f29b5f..ec2ee720e33e 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -28,6 +28,8 @@
 #include <asm/errno.h>
 #include <asm/esr.h>
 #include <asm/irq.h>
+#include <asm/memory.h>
+#include <asm/mmu.h>
 #include <asm/processor.h>
 #include <asm/ptrace.h>
 #include <asm/thread_info.h>
@@ -69,8 +71,21 @@
 #define BAD_FIQ		2
 #define BAD_ERROR	3
 
-	.macro kernel_ventry	label
+	.macro kernel_ventry, el, label, regsize = 64
 	.align 7
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+alternative_if ARM64_UNMAP_KERNEL_AT_EL0
+	.if	\el == 0
+	.if	\regsize == 64
+	mrs	x30, tpidrro_el0
+	msr	tpidrro_el0, xzr
+	.else
+	mov	x30, xzr
+	.endif
+	.endif
+alternative_else_nop_endif
+#endif
+
 	sub	sp, sp, #S_FRAME_SIZE
 #ifdef CONFIG_VMAP_STACK
 	/*
@@ -82,7 +97,7 @@
 	tbnz	x0, #THREAD_SHIFT, 0f
 	sub	x0, sp, x0			// x0'' = sp' - x0' = (sp + x0) - sp = x0
 	sub	sp, sp, x0			// sp'' = sp' - x0 = (sp + x0) - x0 = sp
-	b	\label
+	b	el\()\el\()_\label
 
 0:
 	/*
@@ -114,7 +129,12 @@
 	sub	sp, sp, x0
 	mrs	x0, tpidrro_el0
 #endif
-	b	\label
+	b	el\()\el\()_\label
+	.endm
+
+	.macro tramp_alias, dst, sym
+	mov_q	\dst, TRAMP_VALIAS
+	add	\dst, \dst, #(\sym - .entry.tramp.text)
 	.endm
 
 	.macro	kernel_entry, el, regsize = 64
@@ -147,10 +167,10 @@
 	.else
 	add	x21, sp, #S_FRAME_SIZE
 	get_thread_info tsk
-	/* Save the task's original addr_limit and set USER_DS (TASK_SIZE_64) */
+	/* Save the task's original addr_limit and set USER_DS */
 	ldr	x20, [tsk, #TSK_TI_ADDR_LIMIT]
 	str	x20, [sp, #S_ORIG_ADDR_LIMIT]
-	mov	x20, #TASK_SIZE_64
+	mov	x20, #USER_DS
 	str	x20, [tsk, #TSK_TI_ADDR_LIMIT]
 	/* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
 	.endif /* \el == 0 */
@@ -185,7 +205,7 @@ alternative_else_nop_endif
 
 	.if	\el != 0
 	mrs	x21, ttbr0_el1
-	tst	x21, #0xffff << 48		// Check for the reserved ASID
+	tst	x21, #TTBR_ASID_MASK		// Check for the reserved ASID
 	orr	x23, x23, #PSR_PAN_BIT		// Set the emulated PAN in the saved SPSR
 	b.eq	1f				// TTBR0 access already disabled
 	and	x23, x23, #~PSR_PAN_BIT		// Clear the emulated PAN in the saved SPSR
@@ -248,7 +268,7 @@ alternative_else_nop_endif
 	tbnz	x22, #22, 1f			// Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
 	.endif
 
-	__uaccess_ttbr0_enable x0
+	__uaccess_ttbr0_enable x0, x1
 
 	.if	\el == 0
 	/*
@@ -257,7 +277,7 @@ alternative_else_nop_endif
 	 * Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
 	 * corruption).
 	 */
-	post_ttbr0_update_workaround
+	bl	post_ttbr_update_workaround
 	.endif
 1:
 	.if	\el != 0
@@ -269,18 +289,20 @@ alternative_else_nop_endif
 	.if	\el == 0
 	ldr	x23, [sp, #S_SP]		// load return stack pointer
 	msr	sp_el0, x23
+	tst	x22, #PSR_MODE32_BIT		// native task?
+	b.eq	3f
+
 #ifdef CONFIG_ARM64_ERRATUM_845719
 alternative_if ARM64_WORKAROUND_845719
-	tbz	x22, #4, 1f
 #ifdef CONFIG_PID_IN_CONTEXTIDR
 	mrs	x29, contextidr_el1
 	msr	contextidr_el1, x29
 #else
 	msr contextidr_el1, xzr
 #endif
-1:
 alternative_else_nop_endif
 #endif
+3:
 	.endif
 
 	msr	elr_el1, x21			// set up the return data
@@ -302,7 +324,25 @@ alternative_else_nop_endif
 	ldp	x28, x29, [sp, #16 * 14]
 	ldr	lr, [sp, #S_LR]
 	add	sp, sp, #S_FRAME_SIZE		// restore sp
-	eret					// return to kernel
+	/*
+	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on eret context synchronization
+	 * when returning from IPI handler, and when returning to user-space.
+	 */
+
+	.if	\el == 0
+alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	bne	4f
+	msr	far_el1, x30
+	tramp_alias	x30, tramp_exit_native
+	br	x30
+4:
+	tramp_alias	x30, tramp_exit_compat
+	br	x30
+#endif
+	.else
+	eret
+	.endif
 	.endm
 
 	.macro	irq_stack_entry
@@ -342,6 +382,7 @@ alternative_else_nop_endif
  * x7 is reserved for the system call number in 32-bit mode.
  */
 wsc_nr	.req	w25		// number of system calls
+xsc_nr	.req	x25		// number of system calls (zero-extended)
 wscno	.req	w26		// syscall number
 xscno	.req	x26		// syscall number (zero-extended)
 stbl	.req	x27		// syscall table pointer
@@ -367,31 +408,31 @@ tsk	.req	x28		// current thread_info
 
 	.align	11
 ENTRY(vectors)
-	kernel_ventry	el1_sync_invalid		// Synchronous EL1t
-	kernel_ventry	el1_irq_invalid			// IRQ EL1t
-	kernel_ventry	el1_fiq_invalid			// FIQ EL1t
-	kernel_ventry	el1_error_invalid		// Error EL1t
+	kernel_ventry	1, sync_invalid			// Synchronous EL1t
+	kernel_ventry	1, irq_invalid			// IRQ EL1t
+	kernel_ventry	1, fiq_invalid			// FIQ EL1t
+	kernel_ventry	1, error_invalid		// Error EL1t
 
-	kernel_ventry	el1_sync			// Synchronous EL1h
-	kernel_ventry	el1_irq				// IRQ EL1h
-	kernel_ventry	el1_fiq_invalid			// FIQ EL1h
-	kernel_ventry	el1_error			// Error EL1h
+	kernel_ventry	1, sync				// Synchronous EL1h
+	kernel_ventry	1, irq				// IRQ EL1h
+	kernel_ventry	1, fiq_invalid			// FIQ EL1h
+	kernel_ventry	1, error			// Error EL1h
 
-	kernel_ventry	el0_sync			// Synchronous 64-bit EL0
-	kernel_ventry	el0_irq				// IRQ 64-bit EL0
-	kernel_ventry	el0_fiq_invalid			// FIQ 64-bit EL0
-	kernel_ventry	el0_error			// Error 64-bit EL0
+	kernel_ventry	0, sync				// Synchronous 64-bit EL0
+	kernel_ventry	0, irq				// IRQ 64-bit EL0
+	kernel_ventry	0, fiq_invalid			// FIQ 64-bit EL0
+	kernel_ventry	0, error			// Error 64-bit EL0
 
 #ifdef CONFIG_COMPAT
-	kernel_ventry	el0_sync_compat			// Synchronous 32-bit EL0
-	kernel_ventry	el0_irq_compat			// IRQ 32-bit EL0
-	kernel_ventry	el0_fiq_invalid_compat		// FIQ 32-bit EL0
-	kernel_ventry	el0_error_compat		// Error 32-bit EL0
+	kernel_ventry	0, sync_compat, 32		// Synchronous 32-bit EL0
+	kernel_ventry	0, irq_compat, 32		// IRQ 32-bit EL0
+	kernel_ventry	0, fiq_invalid_compat, 32	// FIQ 32-bit EL0
+	kernel_ventry	0, error_compat, 32		// Error 32-bit EL0
 #else
-	kernel_ventry	el0_sync_invalid		// Synchronous 32-bit EL0
-	kernel_ventry	el0_irq_invalid			// IRQ 32-bit EL0
-	kernel_ventry	el0_fiq_invalid			// FIQ 32-bit EL0
-	kernel_ventry	el0_error_invalid		// Error 32-bit EL0
+	kernel_ventry	0, sync_invalid, 32		// Synchronous 32-bit EL0
+	kernel_ventry	0, irq_invalid, 32		// IRQ 32-bit EL0
+	kernel_ventry	0, fiq_invalid, 32		// FIQ 32-bit EL0
+	kernel_ventry	0, error_invalid, 32		// Error 32-bit EL0
 #endif
 END(vectors)
 
@@ -685,12 +726,15 @@ el0_ia:
 	 * Instruction abort handling
 	 */
 	mrs	x26, far_el1
-	enable_daif
+	enable_da_f
+#ifdef CONFIG_TRACE_IRQFLAGS
+	bl	trace_hardirqs_off
+#endif
 	ct_user_exit
 	mov	x0, x26
 	mov	x1, x25
 	mov	x2, sp
-	bl	do_mem_abort
+	bl	do_el0_ia_bp_hardening
 	b	ret_to_user
 el0_fpsimd_acc:
 	/*
@@ -727,7 +771,10 @@ el0_sp_pc:
 	 * Stack or PC alignment exception handling
 	 */
 	mrs	x26, far_el1
-	enable_daif
+	enable_da_f
+#ifdef CONFIG_TRACE_IRQFLAGS
+	bl	trace_hardirqs_off
+#endif
 	ct_user_exit
 	mov	x0, x26
 	mov	x1, x25
@@ -785,6 +832,11 @@ el0_irq_naked:
 #endif
 
 	ct_user_exit
+#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
+	tbz	x22, #55, 1f
+	bl	do_el0_irq_bp_hardening
+1:
+#endif
 	irq_handler
 
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -896,6 +948,7 @@ el0_svc_naked:					// compat entry point
 	b.ne	__sys_trace
 	cmp     wscno, wsc_nr			// check upper syscall limit
 	b.hs	ni_sys
+	mask_nospec64 xscno, xsc_nr, x19	// enforce bounds for syscall number
 	ldr	x16, [stbl, xscno, lsl #3]	// address in the syscall table
 	blr	x16				// call sys_* routine
 	b	ret_fast_syscall
@@ -943,6 +996,117 @@ __ni_sys_trace:
 
 	.popsection				// .entry.text
 
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+/*
+ * Exception vectors trampoline.
+ */
+	.pushsection ".entry.tramp.text", "ax"
+
+	.macro tramp_map_kernel, tmp
+	mrs	\tmp, ttbr1_el1
+	add	\tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
+	bic	\tmp, \tmp, #USER_ASID_FLAG
+	msr	ttbr1_el1, \tmp
+#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
+alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003
+	/* ASID already in \tmp[63:48] */
+	movk	\tmp, #:abs_g2_nc:(TRAMP_VALIAS >> 12)
+	movk	\tmp, #:abs_g1_nc:(TRAMP_VALIAS >> 12)
+	/* 2MB boundary containing the vectors, so we nobble the walk cache */
+	movk	\tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12)
+	isb
+	tlbi	vae1, \tmp
+	dsb	nsh
+alternative_else_nop_endif
+#endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */
+	.endm
+
+	.macro tramp_unmap_kernel, tmp
+	mrs	\tmp, ttbr1_el1
+	sub	\tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
+	orr	\tmp, \tmp, #USER_ASID_FLAG
+	msr	ttbr1_el1, \tmp
+	/*
+	 * We avoid running the post_ttbr_update_workaround here because
+	 * it's only needed by Cavium ThunderX, which requires KPTI to be
+	 * disabled.
+	 */
+	.endm
+
+	.macro tramp_ventry, regsize = 64
+	.align	7
+1:
+	.if	\regsize == 64
+	msr	tpidrro_el0, x30	// Restored in kernel_ventry
+	.endif
+	/*
+	 * Defend against branch aliasing attacks by pushing a dummy
+	 * entry onto the return stack and using a RET instruction to
+	 * enter the full-fat kernel vectors.
+	 */
+	bl	2f
+	b	.
+2:
+	tramp_map_kernel	x30
+#ifdef CONFIG_RANDOMIZE_BASE
+	adr	x30, tramp_vectors + PAGE_SIZE
+alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
+	ldr	x30, [x30]
+#else
+	ldr	x30, =vectors
+#endif
+	prfm	plil1strm, [x30, #(1b - tramp_vectors)]
+	msr	vbar_el1, x30
+	add	x30, x30, #(1b - tramp_vectors)
+	isb
+	ret
+	.endm
+
+	.macro tramp_exit, regsize = 64
+	adr	x30, tramp_vectors
+	msr	vbar_el1, x30
+	tramp_unmap_kernel	x30
+	.if	\regsize == 64
+	mrs	x30, far_el1
+	.endif
+	eret
+	.endm
+
+	.align	11
+ENTRY(tramp_vectors)
+	.space	0x400
+
+	tramp_ventry
+	tramp_ventry
+	tramp_ventry
+	tramp_ventry
+
+	tramp_ventry	32
+	tramp_ventry	32
+	tramp_ventry	32
+	tramp_ventry	32
+END(tramp_vectors)
+
+ENTRY(tramp_exit_native)
+	tramp_exit
+END(tramp_exit_native)
+
+ENTRY(tramp_exit_compat)
+	tramp_exit	32
+END(tramp_exit_compat)
+
+	.ltorg
+	.popsection				// .entry.tramp.text
+#ifdef CONFIG_RANDOMIZE_BASE
+	.pushsection ".rodata", "a"
+	.align PAGE_SHIFT
+	.globl	__entry_tramp_data_start
+__entry_tramp_data_start:
+	.quad	vectors
+	.popsection				// .rodata
+#endif /* CONFIG_RANDOMIZE_BASE */
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+
 /*
  * Special system call wrappers.
  */
@@ -996,3 +1160,180 @@ ENTRY(ret_from_fork)
 	b	ret_to_user
 ENDPROC(ret_from_fork)
 NOKPROBE(ret_from_fork)
+
+#ifdef CONFIG_ARM_SDE_INTERFACE
+
+#include <asm/sdei.h>
+#include <uapi/linux/arm_sdei.h>
+
+.macro sdei_handler_exit exit_mode
+	/* On success, this call never returns... */
+	cmp	\exit_mode, #SDEI_EXIT_SMC
+	b.ne	99f
+	smc	#0
+	b	.
+99:	hvc	#0
+	b	.
+.endm
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+/*
+ * The regular SDEI entry point may have been unmapped along with the rest of
+ * the kernel. This trampoline restores the kernel mapping to make the x1 memory
+ * argument accessible.
+ *
+ * This clobbers x4, __sdei_handler() will restore this from firmware's
+ * copy.
+ */
+.ltorg
+.pushsection ".entry.tramp.text", "ax"
+ENTRY(__sdei_asm_entry_trampoline)
+	mrs	x4, ttbr1_el1
+	tbz	x4, #USER_ASID_BIT, 1f
+
+	tramp_map_kernel tmp=x4
+	isb
+	mov	x4, xzr
+
+	/*
+	 * Use reg->interrupted_regs.addr_limit to remember whether to unmap
+	 * the kernel on exit.
+	 */
+1:	str	x4, [x1, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
+
+#ifdef CONFIG_RANDOMIZE_BASE
+	adr	x4, tramp_vectors + PAGE_SIZE
+	add	x4, x4, #:lo12:__sdei_asm_trampoline_next_handler
+	ldr	x4, [x4]
+#else
+	ldr	x4, =__sdei_asm_handler
+#endif
+	br	x4
+ENDPROC(__sdei_asm_entry_trampoline)
+NOKPROBE(__sdei_asm_entry_trampoline)
+
+/*
+ * Make the exit call and restore the original ttbr1_el1
+ *
+ * x0 & x1: setup for the exit API call
+ * x2: exit_mode
+ * x4: struct sdei_registered_event argument from registration time.
+ */
+ENTRY(__sdei_asm_exit_trampoline)
+	ldr	x4, [x4, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
+	cbnz	x4, 1f
+
+	tramp_unmap_kernel	tmp=x4
+
+1:	sdei_handler_exit exit_mode=x2
+ENDPROC(__sdei_asm_exit_trampoline)
+NOKPROBE(__sdei_asm_exit_trampoline)
+	.ltorg
+.popsection		// .entry.tramp.text
+#ifdef CONFIG_RANDOMIZE_BASE
+.pushsection ".rodata", "a"
+__sdei_asm_trampoline_next_handler:
+	.quad	__sdei_asm_handler
+.popsection		// .rodata
+#endif /* CONFIG_RANDOMIZE_BASE */
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+
+/*
+ * Software Delegated Exception entry point.
+ *
+ * x0: Event number
+ * x1: struct sdei_registered_event argument from registration time.
+ * x2: interrupted PC
+ * x3: interrupted PSTATE
+ * x4: maybe clobbered by the trampoline
+ *
+ * Firmware has preserved x0->x17 for us, we must save/restore the rest to
+ * follow SMC-CC. We save (or retrieve) all the registers as the handler may
+ * want them.
+ */
+ENTRY(__sdei_asm_handler)
+	stp     x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC]
+	stp     x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2]
+	stp     x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3]
+	stp     x8, x9, [x1, #SDEI_EVENT_INTREGS + 16 * 4]
+	stp     x10, x11, [x1, #SDEI_EVENT_INTREGS + 16 * 5]
+	stp     x12, x13, [x1, #SDEI_EVENT_INTREGS + 16 * 6]
+	stp     x14, x15, [x1, #SDEI_EVENT_INTREGS + 16 * 7]
+	stp     x16, x17, [x1, #SDEI_EVENT_INTREGS + 16 * 8]
+	stp     x18, x19, [x1, #SDEI_EVENT_INTREGS + 16 * 9]
+	stp     x20, x21, [x1, #SDEI_EVENT_INTREGS + 16 * 10]
+	stp     x22, x23, [x1, #SDEI_EVENT_INTREGS + 16 * 11]
+	stp     x24, x25, [x1, #SDEI_EVENT_INTREGS + 16 * 12]
+	stp     x26, x27, [x1, #SDEI_EVENT_INTREGS + 16 * 13]
+	stp     x28, x29, [x1, #SDEI_EVENT_INTREGS + 16 * 14]
+	mov	x4, sp
+	stp     lr, x4, [x1, #SDEI_EVENT_INTREGS + S_LR]
+
+	mov	x19, x1
+
+#ifdef CONFIG_VMAP_STACK
+	/*
+	 * entry.S may have been using sp as a scratch register, find whether
+	 * this is a normal or critical event and switch to the appropriate
+	 * stack for this CPU.
+	 */
+	ldrb	w4, [x19, #SDEI_EVENT_PRIORITY]
+	cbnz	w4, 1f
+	ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
+	b	2f
+1:	ldr_this_cpu dst=x5, sym=sdei_stack_critical_ptr, tmp=x6
+2:	mov	x6, #SDEI_STACK_SIZE
+	add	x5, x5, x6
+	mov	sp, x5
+#endif
+
+	/*
+	 * We may have interrupted userspace, or a guest, or exit-from or
+	 * return-to either of these. We can't trust sp_el0, restore it.
+	 */
+	mrs	x28, sp_el0
+	ldr_this_cpu	dst=x0, sym=__entry_task, tmp=x1
+	msr	sp_el0, x0
+
+	/* If we interrupted the kernel point to the previous stack/frame. */
+	and     x0, x3, #0xc
+	mrs     x1, CurrentEL
+	cmp     x0, x1
+	csel	x29, x29, xzr, eq	// fp, or zero
+	csel	x4, x2, xzr, eq		// elr, or zero
+
+	stp	x29, x4, [sp, #-16]!
+	mov	x29, sp
+
+	add	x0, x19, #SDEI_EVENT_INTREGS
+	mov	x1, x19
+	bl	__sdei_handler
+
+	msr	sp_el0, x28
+	/* restore regs >x17 that we clobbered */
+	mov	x4, x19         // keep x4 for __sdei_asm_exit_trampoline
+	ldp	x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14]
+	ldp	x18, x19, [x4, #SDEI_EVENT_INTREGS + 16 * 9]
+	ldp	lr, x1, [x4, #SDEI_EVENT_INTREGS + S_LR]
+	mov	sp, x1
+
+	mov	x1, x0			// address to complete_and_resume
+	/* x0 = (x0 <= 1) ? EVENT_COMPLETE:EVENT_COMPLETE_AND_RESUME */
+	cmp	x0, #1
+	mov_q	x2, SDEI_1_0_FN_SDEI_EVENT_COMPLETE
+	mov_q	x3, SDEI_1_0_FN_SDEI_EVENT_COMPLETE_AND_RESUME
+	csel	x0, x2, x3, ls
+
+	ldr_l	x2, sdei_exit_mode
+
+alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
+	sdei_handler_exit exit_mode=x2
+alternative_else_nop_endif
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	tramp_alias	dst=x5, sym=__sdei_asm_exit_trampoline
+	br	x5
+#endif
+ENDPROC(__sdei_asm_handler)
+NOKPROBE(__sdei_asm_handler)
+#endif /* CONFIG_ARM_SDE_INTERFACE */
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index fae81f7964b4..e7226c4c7493 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -867,7 +867,7 @@ asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
 asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 {
 	siginfo_t info;
-	unsigned int si_code = 0;
+	unsigned int si_code = FPE_FIXME;
 
 	if (esr & FPEXC_IOF)
 		si_code = FPE_FLTINV;
@@ -1036,14 +1036,14 @@ void fpsimd_restore_current_state(void)
  * flag that indicates that the FPSIMD register contents are the most recent
  * FPSIMD state of 'current'
  */
-void fpsimd_update_current_state(struct fpsimd_state *state)
+void fpsimd_update_current_state(struct user_fpsimd_state const *state)
 {
 	if (!system_supports_fpsimd())
 		return;
 
 	local_bh_disable();
 
-	current->thread.fpsimd_state.user_fpsimd = state->user_fpsimd;
+	current->thread.fpsimd_state.user_fpsimd = *state;
 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
 		fpsimd_to_sve(current);
 
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index e3cb9fbf96b6..2b6b8b24e5ab 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -156,54 +156,124 @@ ENDPROC(preserve_boot_args)
  *	ptrs:	#imm pointers per table page
  *
  * Preserves:	virt
- * Corrupts:	tmp1, tmp2
+ * Corrupts:	ptrs, tmp1, tmp2
  * Returns:	tbl -> next level table page address
  */
 	.macro	create_table_entry, tbl, virt, shift, ptrs, tmp1, tmp2
-	lsr	\tmp1, \virt, #\shift
-	and	\tmp1, \tmp1, #\ptrs - 1	// table index
-	add	\tmp2, \tbl, #PAGE_SIZE
+	add	\tmp1, \tbl, #PAGE_SIZE
+	phys_to_pte \tmp2, \tmp1
 	orr	\tmp2, \tmp2, #PMD_TYPE_TABLE	// address of next table and entry type
+	lsr	\tmp1, \virt, #\shift
+	sub	\ptrs, \ptrs, #1
+	and	\tmp1, \tmp1, \ptrs		// table index
 	str	\tmp2, [\tbl, \tmp1, lsl #3]
 	add	\tbl, \tbl, #PAGE_SIZE		// next level table page
 	.endm
 
 /*
- * Macro to populate the PGD (and possibily PUD) for the corresponding
- * block entry in the next level (tbl) for the given virtual address.
+ * Macro to populate page table entries, these entries can be pointers to the next level
+ * or last level entries pointing to physical memory.
+ *
+ *	tbl:	page table address
+ *	rtbl:	pointer to page table or physical memory
+ *	index:	start index to write
+ *	eindex:	end index to write - [index, eindex] written to
+ *	flags:	flags for pagetable entry to or in
+ *	inc:	increment to rtbl between each entry
+ *	tmp1:	temporary variable
  *
- * Preserves:	tbl, next, virt
- * Corrupts:	tmp1, tmp2
+ * Preserves:	tbl, eindex, flags, inc
+ * Corrupts:	index, tmp1
+ * Returns:	rtbl
  */
-	.macro	create_pgd_entry, tbl, virt, tmp1, tmp2
-	create_table_entry \tbl, \virt, PGDIR_SHIFT, PTRS_PER_PGD, \tmp1, \tmp2
-#if SWAPPER_PGTABLE_LEVELS > 3
-	create_table_entry \tbl, \virt, PUD_SHIFT, PTRS_PER_PUD, \tmp1, \tmp2
-#endif
-#if SWAPPER_PGTABLE_LEVELS > 2
-	create_table_entry \tbl, \virt, SWAPPER_TABLE_SHIFT, PTRS_PER_PTE, \tmp1, \tmp2
-#endif
+	.macro populate_entries, tbl, rtbl, index, eindex, flags, inc, tmp1
+.Lpe\@:	phys_to_pte \tmp1, \rtbl
+	orr	\tmp1, \tmp1, \flags	// tmp1 = table entry
+	str	\tmp1, [\tbl, \index, lsl #3]
+	add	\rtbl, \rtbl, \inc	// rtbl = pa next level
+	add	\index, \index, #1
+	cmp	\index, \eindex
+	b.ls	.Lpe\@
 	.endm
 
 /*
- * Macro to populate block entries in the page table for the start..end
- * virtual range (inclusive).
+ * Compute indices of table entries from virtual address range. If multiple entries
+ * were needed in the previous page table level then the next page table level is assumed
+ * to be composed of multiple pages. (This effectively scales the end index).
+ *
+ *	vstart:	virtual address of start of range
+ *	vend:	virtual address of end of range
+ *	shift:	shift used to transform virtual address into index
+ *	ptrs:	number of entries in page table
+ *	istart:	index in table corresponding to vstart
+ *	iend:	index in table corresponding to vend
+ *	count:	On entry: how many extra entries were required in previous level, scales
+ *			  our end index.
+ *		On exit: returns how many extra entries required for next page table level
  *
- * Preserves:	tbl, flags
- * Corrupts:	phys, start, end, pstate
+ * Preserves:	vstart, vend, shift, ptrs
+ * Returns:	istart, iend, count
  */
-	.macro	create_block_map, tbl, flags, phys, start, end
-	lsr	\phys, \phys, #SWAPPER_BLOCK_SHIFT
-	lsr	\start, \start, #SWAPPER_BLOCK_SHIFT
-	and	\start, \start, #PTRS_PER_PTE - 1	// table index
-	orr	\phys, \flags, \phys, lsl #SWAPPER_BLOCK_SHIFT	// table entry
-	lsr	\end, \end, #SWAPPER_BLOCK_SHIFT
-	and	\end, \end, #PTRS_PER_PTE - 1		// table end index
-9999:	str	\phys, [\tbl, \start, lsl #3]		// store the entry
-	add	\start, \start, #1			// next entry
-	add	\phys, \phys, #SWAPPER_BLOCK_SIZE		// next block
-	cmp	\start, \end
-	b.ls	9999b
+	.macro compute_indices, vstart, vend, shift, ptrs, istart, iend, count
+	lsr	\iend, \vend, \shift
+	mov	\istart, \ptrs
+	sub	\istart, \istart, #1
+	and	\iend, \iend, \istart	// iend = (vend >> shift) & (ptrs - 1)
+	mov	\istart, \ptrs
+	mul	\istart, \istart, \count
+	add	\iend, \iend, \istart	// iend += (count - 1) * ptrs
+					// our entries span multiple tables
+
+	lsr	\istart, \vstart, \shift
+	mov	\count, \ptrs
+	sub	\count, \count, #1
+	and	\istart, \istart, \count
+
+	sub	\count, \iend, \istart
+	.endm
+
+/*
+ * Map memory for specified virtual address range. Each level of page table needed supports
+ * multiple entries. If a level requires n entries the next page table level is assumed to be
+ * formed from n pages.
+ *
+ *	tbl:	location of page table
+ *	rtbl:	address to be used for first level page table entry (typically tbl + PAGE_SIZE)
+ *	vstart:	start address to map
+ *	vend:	end address to map - we map [vstart, vend]
+ *	flags:	flags to use to map last level entries
+ *	phys:	physical address corresponding to vstart - physical memory is contiguous
+ *	pgds:	the number of pgd entries
+ *
+ * Temporaries:	istart, iend, tmp, count, sv - these need to be different registers
+ * Preserves:	vstart, vend, flags
+ * Corrupts:	tbl, rtbl, istart, iend, tmp, count, sv
+ */
+	.macro map_memory, tbl, rtbl, vstart, vend, flags, phys, pgds, istart, iend, tmp, count, sv
+	add \rtbl, \tbl, #PAGE_SIZE
+	mov \sv, \rtbl
+	mov \count, #0
+	compute_indices \vstart, \vend, #PGDIR_SHIFT, \pgds, \istart, \iend, \count
+	populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
+	mov \tbl, \sv
+	mov \sv, \rtbl
+
+#if SWAPPER_PGTABLE_LEVELS > 3
+	compute_indices \vstart, \vend, #PUD_SHIFT, #PTRS_PER_PUD, \istart, \iend, \count
+	populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
+	mov \tbl, \sv
+	mov \sv, \rtbl
+#endif
+
+#if SWAPPER_PGTABLE_LEVELS > 2
+	compute_indices \vstart, \vend, #SWAPPER_TABLE_SHIFT, #PTRS_PER_PMD, \istart, \iend, \count
+	populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
+	mov \tbl, \sv
+#endif
+
+	compute_indices \vstart, \vend, #SWAPPER_BLOCK_SHIFT, #PTRS_PER_PTE, \istart, \iend, \count
+	bic \count, \phys, #SWAPPER_BLOCK_SIZE - 1
+	populate_entries \tbl, \count, \istart, \iend, \flags, #SWAPPER_BLOCK_SIZE, \tmp
 	.endm
 
 /*
@@ -221,14 +291,16 @@ __create_page_tables:
 	 * dirty cache lines being evicted.
 	 */
 	adrp	x0, idmap_pg_dir
-	ldr	x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+	adrp	x1, swapper_pg_end
+	sub	x1, x1, x0
 	bl	__inval_dcache_area
 
 	/*
 	 * Clear the idmap and swapper page tables.
 	 */
 	adrp	x0, idmap_pg_dir
-	ldr	x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+	adrp	x1, swapper_pg_end
+	sub	x1, x1, x0
 1:	stp	xzr, xzr, [x0], #16
 	stp	xzr, xzr, [x0], #16
 	stp	xzr, xzr, [x0], #16
@@ -244,26 +316,13 @@ __create_page_tables:
 	adrp	x0, idmap_pg_dir
 	adrp	x3, __idmap_text_start		// __pa(__idmap_text_start)
 
-#ifndef CONFIG_ARM64_VA_BITS_48
-#define EXTRA_SHIFT	(PGDIR_SHIFT + PAGE_SHIFT - 3)
-#define EXTRA_PTRS	(1 << (48 - EXTRA_SHIFT))
-
-	/*
-	 * If VA_BITS < 48, it may be too small to allow for an ID mapping to be
-	 * created that covers system RAM if that is located sufficiently high
-	 * in the physical address space. So for the ID map, use an extended
-	 * virtual range in that case, by configuring an additional translation
-	 * level.
-	 * First, we have to verify our assumption that the current value of
-	 * VA_BITS was chosen such that all translation levels are fully
-	 * utilised, and that lowering T0SZ will always result in an additional
-	 * translation level to be configured.
-	 */
-#if VA_BITS != EXTRA_SHIFT
-#error "Mismatch between VA_BITS and page size/number of translation levels"
-#endif
-
 	/*
+	 * VA_BITS may be too small to allow for an ID mapping to be created
+	 * that covers system RAM if that is located sufficiently high in the
+	 * physical address space. So for the ID map, use an extended virtual
+	 * range in that case, and configure an additional translation level
+	 * if needed.
+	 *
 	 * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the
 	 * entire ID map region can be mapped. As T0SZ == (64 - #bits used),
 	 * this number conveniently equals the number of leading zeroes in
@@ -272,21 +331,44 @@ __create_page_tables:
 	adrp	x5, __idmap_text_end
 	clz	x5, x5
 	cmp	x5, TCR_T0SZ(VA_BITS)	// default T0SZ small enough?
-	b.ge	1f			// .. then skip additional level
+	b.ge	1f			// .. then skip VA range extension
 
 	adr_l	x6, idmap_t0sz
 	str	x5, [x6]
 	dmb	sy
 	dc	ivac, x6		// Invalidate potentially stale cache line
 
-	create_table_entry x0, x3, EXTRA_SHIFT, EXTRA_PTRS, x5, x6
-1:
+#if (VA_BITS < 48)
+#define EXTRA_SHIFT	(PGDIR_SHIFT + PAGE_SHIFT - 3)
+#define EXTRA_PTRS	(1 << (PHYS_MASK_SHIFT - EXTRA_SHIFT))
+
+	/*
+	 * If VA_BITS < 48, we have to configure an additional table level.
+	 * First, we have to verify our assumption that the current value of
+	 * VA_BITS was chosen such that all translation levels are fully
+	 * utilised, and that lowering T0SZ will always result in an additional
+	 * translation level to be configured.
+	 */
+#if VA_BITS != EXTRA_SHIFT
+#error "Mismatch between VA_BITS and page size/number of translation levels"
 #endif
 
-	create_pgd_entry x0, x3, x5, x6
+	mov	x4, EXTRA_PTRS
+	create_table_entry x0, x3, EXTRA_SHIFT, x4, x5, x6
+#else
+	/*
+	 * If VA_BITS == 48, we don't have to configure an additional
+	 * translation level, but the top-level table has more entries.
+	 */
+	mov	x4, #1 << (PHYS_MASK_SHIFT - PGDIR_SHIFT)
+	str_l	x4, idmap_ptrs_per_pgd, x5
+#endif
+1:
+	ldr_l	x4, idmap_ptrs_per_pgd
 	mov	x5, x3				// __pa(__idmap_text_start)
 	adr_l	x6, __idmap_text_end		// __pa(__idmap_text_end)
-	create_block_map x0, x7, x3, x5, x6
+
+	map_memory x0, x1, x3, x6, x7, x3, x4, x10, x11, x12, x13, x14
 
 	/*
 	 * Map the kernel image (starting with PHYS_OFFSET).
@@ -294,12 +376,13 @@ __create_page_tables:
 	adrp	x0, swapper_pg_dir
 	mov_q	x5, KIMAGE_VADDR + TEXT_OFFSET	// compile time __va(_text)
 	add	x5, x5, x23			// add KASLR displacement
-	create_pgd_entry x0, x5, x3, x6
+	mov	x4, PTRS_PER_PGD
 	adrp	x6, _end			// runtime __pa(_end)
 	adrp	x3, _text			// runtime __pa(_text)
 	sub	x6, x6, x3			// _end - _text
 	add	x6, x6, x5			// runtime __va(_end)
-	create_block_map x0, x7, x3, x5, x6
+
+	map_memory x0, x1, x5, x6, x7, x3, x4, x10, x11, x12, x13, x14
 
 	/*
 	 * Since the page tables have been populated with non-cacheable
@@ -307,7 +390,8 @@ __create_page_tables:
 	 * tables again to remove any speculatively loaded cache lines.
 	 */
 	adrp	x0, idmap_pg_dir
-	ldr	x1, =(IDMAP_DIR_SIZE + SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+	adrp	x1, swapper_pg_end
+	sub	x1, x1, x0
 	dmb	sy
 	bl	__inval_dcache_area
 
@@ -371,7 +455,7 @@ ENDPROC(__primary_switched)
  * end early head section, begin head code that is also used for
  * hotplug and needs to have the same protections as the text region
  */
-	.section ".idmap.text","ax"
+	.section ".idmap.text","awx"
 
 ENTRY(kimage_vaddr)
 	.quad		_text - TEXT_OFFSET
@@ -388,17 +472,13 @@ ENTRY(el2_setup)
 	mrs	x0, CurrentEL
 	cmp	x0, #CurrentEL_EL2
 	b.eq	1f
-	mrs	x0, sctlr_el1
-CPU_BE(	orr	x0, x0, #(3 << 24)	)	// Set the EE and E0E bits for EL1
-CPU_LE(	bic	x0, x0, #(3 << 24)	)	// Clear the EE and E0E bits for EL1
+	mov_q	x0, (SCTLR_EL1_RES1 | ENDIAN_SET_EL1)
 	msr	sctlr_el1, x0
 	mov	w0, #BOOT_CPU_MODE_EL1		// This cpu booted in EL1
 	isb
 	ret
 
-1:	mrs	x0, sctlr_el2
-CPU_BE(	orr	x0, x0, #(1 << 25)	)	// Set the EE bit for EL2
-CPU_LE(	bic	x0, x0, #(1 << 25)	)	// Clear the EE bit for EL2
+1:	mov_q	x0, (SCTLR_EL2_RES1 | ENDIAN_SET_EL2)
 	msr	sctlr_el2, x0
 
 #ifdef CONFIG_ARM64_VHE
@@ -514,10 +594,7 @@ install_el2_stub:
 	 * requires no configuration, and all non-hyp-specific EL2 setup
 	 * will be done via the _EL1 system register aliases in __cpu_setup.
 	 */
-	/* sctlr_el1 */
-	mov	x0, #0x0800			// Set/clear RES{1,0} bits
-CPU_BE(	movk	x0, #0x33d0, lsl #16	)	// Set EE and E0E on BE systems
-CPU_LE(	movk	x0, #0x30d0, lsl #16	)	// Clear EE and E0E on LE systems
+	mov_q	x0, (SCTLR_EL1_RES1 | ENDIAN_SET_EL1)
 	msr	sctlr_el1, x0
 
 	/* Coprocessor traps. */
@@ -679,8 +756,10 @@ ENTRY(__enable_mmu)
 	update_early_cpu_boot_status 0, x1, x2
 	adrp	x1, idmap_pg_dir
 	adrp	x2, swapper_pg_dir
-	msr	ttbr0_el1, x1			// load TTBR0
-	msr	ttbr1_el1, x2			// load TTBR1
+	phys_to_ttbr x3, x1
+	phys_to_ttbr x4, x2
+	msr	ttbr0_el1, x3			// load TTBR0
+	msr	ttbr1_el1, x4			// load TTBR1
 	isb
 	msr	sctlr_el1, x0
 	isb
diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S
index e56d848b6466..dd14ab8c9f72 100644
--- a/arch/arm64/kernel/hibernate-asm.S
+++ b/arch/arm64/kernel/hibernate-asm.S
@@ -33,12 +33,14 @@
  * Even switching to our copied tables will cause a changed output address at
  * each stage of the walk.
  */
-.macro break_before_make_ttbr_switch zero_page, page_table
-	msr	ttbr1_el1, \zero_page
+.macro break_before_make_ttbr_switch zero_page, page_table, tmp
+	phys_to_ttbr \tmp, \zero_page
+	msr	ttbr1_el1, \tmp
 	isb
 	tlbi	vmalle1
 	dsb	nsh
-	msr	ttbr1_el1, \page_table
+	phys_to_ttbr \tmp, \page_table
+	msr	ttbr1_el1, \tmp
 	isb
 .endm
 
@@ -78,7 +80,7 @@ ENTRY(swsusp_arch_suspend_exit)
 	 * We execute from ttbr0, change ttbr1 to our copied linear map tables
 	 * with a break-before-make via the zero page
 	 */
-	break_before_make_ttbr_switch	x5, x0
+	break_before_make_ttbr_switch	x5, x0, x6
 
 	mov	x21, x1
 	mov	x30, x2
@@ -109,7 +111,7 @@ ENTRY(swsusp_arch_suspend_exit)
 	dsb	ish		/* wait for PoU cleaning to finish */
 
 	/* switch to the restored kernels page tables */
-	break_before_make_ttbr_switch	x25, x21
+	break_before_make_ttbr_switch	x25, x21, x6
 
 	ic	ialluis
 	dsb	ish
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 3009b8b80f08..1ec5f28c39fc 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -202,10 +202,10 @@ static int create_safe_exec_page(void *src_start, size_t length,
 				 gfp_t mask)
 {
 	int rc = 0;
-	pgd_t *pgd;
-	pud_t *pud;
-	pmd_t *pmd;
-	pte_t *pte;
+	pgd_t *pgdp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep;
 	unsigned long dst = (unsigned long)allocator(mask);
 
 	if (!dst) {
@@ -216,39 +216,38 @@ static int create_safe_exec_page(void *src_start, size_t length,
 	memcpy((void *)dst, src_start, length);
 	flush_icache_range(dst, dst + length);
 
-	pgd = pgd_offset_raw(allocator(mask), dst_addr);
-	if (pgd_none(*pgd)) {
-		pud = allocator(mask);
-		if (!pud) {
+	pgdp = pgd_offset_raw(allocator(mask), dst_addr);
+	if (pgd_none(READ_ONCE(*pgdp))) {
+		pudp = allocator(mask);
+		if (!pudp) {
 			rc = -ENOMEM;
 			goto out;
 		}
-		pgd_populate(&init_mm, pgd, pud);
+		pgd_populate(&init_mm, pgdp, pudp);
 	}
 
-	pud = pud_offset(pgd, dst_addr);
-	if (pud_none(*pud)) {
-		pmd = allocator(mask);
-		if (!pmd) {
+	pudp = pud_offset(pgdp, dst_addr);
+	if (pud_none(READ_ONCE(*pudp))) {
+		pmdp = allocator(mask);
+		if (!pmdp) {
 			rc = -ENOMEM;
 			goto out;
 		}
-		pud_populate(&init_mm, pud, pmd);
+		pud_populate(&init_mm, pudp, pmdp);
 	}
 
-	pmd = pmd_offset(pud, dst_addr);
-	if (pmd_none(*pmd)) {
-		pte = allocator(mask);
-		if (!pte) {
+	pmdp = pmd_offset(pudp, dst_addr);
+	if (pmd_none(READ_ONCE(*pmdp))) {
+		ptep = allocator(mask);
+		if (!ptep) {
 			rc = -ENOMEM;
 			goto out;
 		}
-		pmd_populate_kernel(&init_mm, pmd, pte);
+		pmd_populate_kernel(&init_mm, pmdp, ptep);
 	}
 
-	pte = pte_offset_kernel(pmd, dst_addr);
-	set_pte(pte, __pte(virt_to_phys((void *)dst) |
-			 pgprot_val(PAGE_KERNEL_EXEC)));
+	ptep = pte_offset_kernel(pmdp, dst_addr);
+	set_pte(ptep, pfn_pte(virt_to_pfn(dst), PAGE_KERNEL_EXEC));
 
 	/*
 	 * Load our new page tables. A strict BBM approach requires that we
@@ -264,7 +263,7 @@ static int create_safe_exec_page(void *src_start, size_t length,
 	 */
 	cpu_set_reserved_ttbr0();
 	local_flush_tlb_all();
-	write_sysreg(virt_to_phys(pgd), ttbr0_el1);
+	write_sysreg(phys_to_ttbr(virt_to_phys(pgdp)), ttbr0_el1);
 	isb();
 
 	*phys_dst_addr = virt_to_phys((void *)dst);
@@ -321,9 +320,9 @@ int swsusp_arch_suspend(void)
 	return ret;
 }
 
-static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr)
+static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr)
 {
-	pte_t pte = *src_pte;
+	pte_t pte = READ_ONCE(*src_ptep);
 
 	if (pte_valid(pte)) {
 		/*
@@ -331,7 +330,7 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr)
 		 * read only (code, rodata). Clear the RDONLY bit from
 		 * the temporary mappings we use during restore.
 		 */
-		set_pte(dst_pte, pte_mkwrite(pte));
+		set_pte(dst_ptep, pte_mkwrite(pte));
 	} else if (debug_pagealloc_enabled() && !pte_none(pte)) {
 		/*
 		 * debug_pagealloc will removed the PTE_VALID bit if
@@ -344,112 +343,116 @@ static void _copy_pte(pte_t *dst_pte, pte_t *src_pte, unsigned long addr)
 		 */
 		BUG_ON(!pfn_valid(pte_pfn(pte)));
 
-		set_pte(dst_pte, pte_mkpresent(pte_mkwrite(pte)));
+		set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte)));
 	}
 }
 
-static int copy_pte(pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long start,
+static int copy_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start,
 		    unsigned long end)
 {
-	pte_t *src_pte;
-	pte_t *dst_pte;
+	pte_t *src_ptep;
+	pte_t *dst_ptep;
 	unsigned long addr = start;
 
-	dst_pte = (pte_t *)get_safe_page(GFP_ATOMIC);
-	if (!dst_pte)
+	dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC);
+	if (!dst_ptep)
 		return -ENOMEM;
-	pmd_populate_kernel(&init_mm, dst_pmd, dst_pte);
-	dst_pte = pte_offset_kernel(dst_pmd, start);
+	pmd_populate_kernel(&init_mm, dst_pmdp, dst_ptep);
+	dst_ptep = pte_offset_kernel(dst_pmdp, start);
 
-	src_pte = pte_offset_kernel(src_pmd, start);
+	src_ptep = pte_offset_kernel(src_pmdp, start);
 	do {
-		_copy_pte(dst_pte, src_pte, addr);
-	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
+		_copy_pte(dst_ptep, src_ptep, addr);
+	} while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end);
 
 	return 0;
 }
 
-static int copy_pmd(pud_t *dst_pud, pud_t *src_pud, unsigned long start,
+static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start,
 		    unsigned long end)
 {
-	pmd_t *src_pmd;
-	pmd_t *dst_pmd;
+	pmd_t *src_pmdp;
+	pmd_t *dst_pmdp;
 	unsigned long next;
 	unsigned long addr = start;
 
-	if (pud_none(*dst_pud)) {
-		dst_pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
-		if (!dst_pmd)
+	if (pud_none(READ_ONCE(*dst_pudp))) {
+		dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC);
+		if (!dst_pmdp)
 			return -ENOMEM;
-		pud_populate(&init_mm, dst_pud, dst_pmd);
+		pud_populate(&init_mm, dst_pudp, dst_pmdp);
 	}
-	dst_pmd = pmd_offset(dst_pud, start);
+	dst_pmdp = pmd_offset(dst_pudp, start);
 
-	src_pmd = pmd_offset(src_pud, start);
+	src_pmdp = pmd_offset(src_pudp, start);
 	do {
+		pmd_t pmd = READ_ONCE(*src_pmdp);
+
 		next = pmd_addr_end(addr, end);
-		if (pmd_none(*src_pmd))
+		if (pmd_none(pmd))
 			continue;
-		if (pmd_table(*src_pmd)) {
-			if (copy_pte(dst_pmd, src_pmd, addr, next))
+		if (pmd_table(pmd)) {
+			if (copy_pte(dst_pmdp, src_pmdp, addr, next))
 				return -ENOMEM;
 		} else {
-			set_pmd(dst_pmd,
-				__pmd(pmd_val(*src_pmd) & ~PMD_SECT_RDONLY));
+			set_pmd(dst_pmdp,
+				__pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY));
 		}
-	} while (dst_pmd++, src_pmd++, addr = next, addr != end);
+	} while (dst_pmdp++, src_pmdp++, addr = next, addr != end);
 
 	return 0;
 }
 
-static int copy_pud(pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long start,
+static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
 		    unsigned long end)
 {
-	pud_t *dst_pud;
-	pud_t *src_pud;
+	pud_t *dst_pudp;
+	pud_t *src_pudp;
 	unsigned long next;
 	unsigned long addr = start;
 
-	if (pgd_none(*dst_pgd)) {
-		dst_pud = (pud_t *)get_safe_page(GFP_ATOMIC);
-		if (!dst_pud)
+	if (pgd_none(READ_ONCE(*dst_pgdp))) {
+		dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC);
+		if (!dst_pudp)
 			return -ENOMEM;
-		pgd_populate(&init_mm, dst_pgd, dst_pud);
+		pgd_populate(&init_mm, dst_pgdp, dst_pudp);
 	}
-	dst_pud = pud_offset(dst_pgd, start);
+	dst_pudp = pud_offset(dst_pgdp, start);
 
-	src_pud = pud_offset(src_pgd, start);
+	src_pudp = pud_offset(src_pgdp, start);
 	do {
+		pud_t pud = READ_ONCE(*src_pudp);
+
 		next = pud_addr_end(addr, end);
-		if (pud_none(*src_pud))
+		if (pud_none(pud))
 			continue;
-		if (pud_table(*(src_pud))) {
-			if (copy_pmd(dst_pud, src_pud, addr, next))
+		if (pud_table(pud)) {
+			if (copy_pmd(dst_pudp, src_pudp, addr, next))
 				return -ENOMEM;
 		} else {
-			set_pud(dst_pud,
-				__pud(pud_val(*src_pud) & ~PMD_SECT_RDONLY));
+			set_pud(dst_pudp,
+				__pud(pud_val(pud) & ~PMD_SECT_RDONLY));
 		}
-	} while (dst_pud++, src_pud++, addr = next, addr != end);
+	} while (dst_pudp++, src_pudp++, addr = next, addr != end);
 
 	return 0;
 }
 
-static int copy_page_tables(pgd_t *dst_pgd, unsigned long start,
+static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start,
 			    unsigned long end)
 {
 	unsigned long next;
 	unsigned long addr = start;
-	pgd_t *src_pgd = pgd_offset_k(start);
+	pgd_t *src_pgdp = pgd_offset_k(start);
 
-	dst_pgd = pgd_offset_raw(dst_pgd, start);
+	dst_pgdp = pgd_offset_raw(dst_pgdp, start);
 	do {
 		next = pgd_addr_end(addr, end);
-		if (pgd_none(*src_pgd))
+		if (pgd_none(READ_ONCE(*src_pgdp)))
 			continue;
-		if (copy_pud(dst_pgd, src_pgd, addr, next))
+		if (copy_pud(dst_pgdp, src_pgdp, addr, next))
 			return -ENOMEM;
-	} while (dst_pgd++, src_pgd++, addr = next, addr != end);
+	} while (dst_pgdp++, src_pgdp++, addr = next, addr != end);
 
 	return 0;
 }
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 713561e5bcab..60e5fc661f74 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -29,6 +29,7 @@
 #include <linux/irqchip.h>
 #include <linux/seq_file.h>
 #include <linux/vmalloc.h>
+#include <asm/vmap_stack.h>
 
 unsigned long irq_err_count;
 
@@ -58,17 +59,7 @@ static void init_irq_stacks(void)
 	unsigned long *p;
 
 	for_each_possible_cpu(cpu) {
-		/*
-		* To ensure that VMAP'd stack overflow detection works
-		* correctly, the IRQ stacks need to have the same
-		* alignment as other stacks.
-		*/
-		p = __vmalloc_node_range(IRQ_STACK_SIZE, THREAD_ALIGN,
-					 VMALLOC_START, VMALLOC_END,
-					 THREADINFO_GFP, PAGE_KERNEL,
-					 0, cpu_to_node(cpu),
-					 __builtin_return_address(0));
-
+		p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, cpu_to_node(cpu));
 		per_cpu(irq_stack_ptr, cpu) = p;
 	}
 }
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index 3affca3dd96a..85a251b6dfa8 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -908,9 +908,9 @@ static void __armv8pmu_probe_pmu(void *info)
 	int pmuver;
 
 	dfr0 = read_sysreg(id_aa64dfr0_el1);
-	pmuver = cpuid_feature_extract_signed_field(dfr0,
+	pmuver = cpuid_feature_extract_unsigned_field(dfr0,
 			ID_AA64DFR0_PMUVER_SHIFT);
-	if (pmuver < 1)
+	if (pmuver == 0xf || pmuver == 0)
 		return;
 
 	probe->present = true;
@@ -925,9 +925,8 @@ static void __armv8pmu_probe_pmu(void *info)
 	pmceid[0] = read_sysreg(pmceid0_el0);
 	pmceid[1] = read_sysreg(pmceid1_el0);
 
-	bitmap_from_u32array(cpu_pmu->pmceid_bitmap,
-			     ARMV8_PMUV3_MAX_COMMON_EVENTS, pmceid,
-			     ARRAY_SIZE(pmceid));
+	bitmap_from_arr32(cpu_pmu->pmceid_bitmap,
+			     pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
 }
 
 static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 6b7dcf4310ac..c0da6efe5465 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -35,7 +35,6 @@
 #include <linux/delay.h>
 #include <linux/reboot.h>
 #include <linux/interrupt.h>
-#include <linux/kallsyms.h>
 #include <linux/init.h>
 #include <linux/cpu.h>
 #include <linux/elfcore.h>
@@ -221,8 +220,15 @@ void __show_regs(struct pt_regs *regs)
 
 	show_regs_print_info(KERN_DEFAULT);
 	print_pstate(regs);
-	print_symbol("pc : %s\n", regs->pc);
-	print_symbol("lr : %s\n", lr);
+
+	if (!user_mode(regs)) {
+		printk("pc : %pS\n", (void *)regs->pc);
+		printk("lr : %pS\n", (void *)lr);
+	} else {
+		printk("pc : %016llx\n", regs->pc);
+		printk("lr : %016llx\n", lr);
+	}
+
 	printk("sp : %016llx\n", sp);
 
 	i = top_reg;
@@ -370,16 +376,14 @@ void tls_preserve_current_state(void)
 
 static void tls_thread_switch(struct task_struct *next)
 {
-	unsigned long tpidr, tpidrro;
-
 	tls_preserve_current_state();
 
-	tpidr = *task_user_tls(next);
-	tpidrro = is_compat_thread(task_thread_info(next)) ?
-		  next->thread.tp_value : 0;
+	if (is_compat_thread(task_thread_info(next)))
+		write_sysreg(next->thread.tp_value, tpidrro_el0);
+	else if (!arm64_kernel_unmapped_at_el0())
+		write_sysreg(0, tpidrro_el0);
 
-	write_sysreg(tpidr, tpidr_el0);
-	write_sysreg(tpidrro, tpidrro_el0);
+	write_sysreg(*task_user_tls(next), tpidr_el0);
 }
 
 /* Restore the UAO state depending on next's addr_limit */
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 7c44658b316d..9ae31f7e2243 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -180,34 +180,34 @@ static void ptrace_hbptriggered(struct perf_event *bp,
 				struct pt_regs *regs)
 {
 	struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
-	siginfo_t info = {
-		.si_signo	= SIGTRAP,
-		.si_errno	= 0,
-		.si_code	= TRAP_HWBKPT,
-		.si_addr	= (void __user *)(bkpt->trigger),
-	};
+	siginfo_t info;
 
-#ifdef CONFIG_COMPAT
-	int i;
+	clear_siginfo(&info);
+	info.si_signo	= SIGTRAP;
+	info.si_errno	= 0;
+	info.si_code	= TRAP_HWBKPT;
+	info.si_addr	= (void __user *)(bkpt->trigger);
 
-	if (!is_compat_task())
-		goto send_sig;
+#ifdef CONFIG_COMPAT
+	if (is_compat_task()) {
+		int si_errno = 0;
+		int i;
 
-	for (i = 0; i < ARM_MAX_BRP; ++i) {
-		if (current->thread.debug.hbp_break[i] == bp) {
-			info.si_errno = (i << 1) + 1;
-			break;
+		for (i = 0; i < ARM_MAX_BRP; ++i) {
+			if (current->thread.debug.hbp_break[i] == bp) {
+				si_errno = (i << 1) + 1;
+				break;
+			}
 		}
-	}
 
-	for (i = 0; i < ARM_MAX_WRP; ++i) {
-		if (current->thread.debug.hbp_watch[i] == bp) {
-			info.si_errno = -((i << 1) + 1);
-			break;
+		for (i = 0; i < ARM_MAX_WRP; ++i) {
+			if (current->thread.debug.hbp_watch[i] == bp) {
+				si_errno = -((i << 1) + 1);
+				break;
+			}
 		}
+		force_sig_ptrace_errno_trap(si_errno, (void __user *)bkpt->trigger);
 	}
-
-send_sig:
 #endif
 	force_sig_info(SIGTRAP, &info, current);
 }
@@ -1419,7 +1419,7 @@ static int compat_ptrace_hbp_get(unsigned int note_type,
 	u64 addr = 0;
 	u32 ctrl = 0;
 
-	int err, idx = compat_ptrace_hbp_num_to_idx(num);;
+	int err, idx = compat_ptrace_hbp_num_to_idx(num);
 
 	if (num & 1) {
 		err = ptrace_hbp_get_addr(note_type, tsk, idx, &addr);
diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c
new file mode 100644
index 000000000000..6b8d90d5ceae
--- /dev/null
+++ b/arch/arm64/kernel/sdei.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2017 Arm Ltd.
+#define pr_fmt(fmt) "sdei: " fmt
+
+#include <linux/arm_sdei.h>
+#include <linux/hardirq.h>
+#include <linux/irqflags.h>
+#include <linux/sched/task_stack.h>
+#include <linux/uaccess.h>
+
+#include <asm/alternative.h>
+#include <asm/kprobes.h>
+#include <asm/mmu.h>
+#include <asm/ptrace.h>
+#include <asm/sections.h>
+#include <asm/sysreg.h>
+#include <asm/vmap_stack.h>
+
+unsigned long sdei_exit_mode;
+
+/*
+ * VMAP'd stacks checking for stack overflow on exception using sp as a scratch
+ * register, meaning SDEI has to switch to its own stack. We need two stacks as
+ * a critical event may interrupt a normal event that has just taken a
+ * synchronous exception, and is using sp as scratch register. For a critical
+ * event interrupting a normal event, we can't reliably tell if we were on the
+ * sdei stack.
+ * For now, we allocate stacks when the driver is probed.
+ */
+DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
+DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
+
+#ifdef CONFIG_VMAP_STACK
+DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
+DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
+#endif
+
+static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
+{
+	unsigned long *p;
+
+	p = per_cpu(*ptr, cpu);
+	if (p) {
+		per_cpu(*ptr, cpu) = NULL;
+		vfree(p);
+	}
+}
+
+static void free_sdei_stacks(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
+		_free_sdei_stack(&sdei_stack_critical_ptr, cpu);
+	}
+}
+
+static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
+{
+	unsigned long *p;
+
+	p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
+	if (!p)
+		return -ENOMEM;
+	per_cpu(*ptr, cpu) = p;
+
+	return 0;
+}
+
+static int init_sdei_stacks(void)
+{
+	int cpu;
+	int err = 0;
+
+	for_each_possible_cpu(cpu) {
+		err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
+		if (err)
+			break;
+		err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
+		if (err)
+			break;
+	}
+
+	if (err)
+		free_sdei_stacks();
+
+	return err;
+}
+
+bool _on_sdei_stack(unsigned long sp)
+{
+	unsigned long low, high;
+
+	if (!IS_ENABLED(CONFIG_VMAP_STACK))
+		return false;
+
+	low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
+	high = low + SDEI_STACK_SIZE;
+
+	if (low <= sp && sp < high)
+		return true;
+
+	low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
+	high = low + SDEI_STACK_SIZE;
+
+	return (low <= sp && sp < high);
+}
+
+unsigned long sdei_arch_get_entry_point(int conduit)
+{
+	/*
+	 * SDEI works between adjacent exception levels. If we booted at EL1 we
+	 * assume a hypervisor is marshalling events. If we booted at EL2 and
+	 * dropped to EL1 because we don't support VHE, then we can't support
+	 * SDEI.
+	 */
+	if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
+		pr_err("Not supported on this hardware/boot configuration\n");
+		return 0;
+	}
+
+	if (IS_ENABLED(CONFIG_VMAP_STACK)) {
+		if (init_sdei_stacks())
+			return 0;
+	}
+
+	sdei_exit_mode = (conduit == CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	if (arm64_kernel_unmapped_at_el0()) {
+		unsigned long offset;
+
+		offset = (unsigned long)__sdei_asm_entry_trampoline -
+			 (unsigned long)__entry_tramp_text_start;
+		return TRAMP_VALIAS + offset;
+	} else
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+		return (unsigned long)__sdei_asm_handler;
+
+}
+
+/*
+ * __sdei_handler() returns one of:
+ *  SDEI_EV_HANDLED -  success, return to the interrupted context.
+ *  SDEI_EV_FAILED  -  failure, return this error code to firmare.
+ *  virtual-address -  success, return to this address.
+ */
+static __kprobes unsigned long _sdei_handler(struct pt_regs *regs,
+					     struct sdei_registered_event *arg)
+{
+	u32 mode;
+	int i, err = 0;
+	int clobbered_registers = 4;
+	u64 elr = read_sysreg(elr_el1);
+	u32 kernel_mode = read_sysreg(CurrentEL) | 1;	/* +SPSel */
+	unsigned long vbar = read_sysreg(vbar_el1);
+
+	if (arm64_kernel_unmapped_at_el0())
+		clobbered_registers++;
+
+	/* Retrieve the missing registers values */
+	for (i = 0; i < clobbered_registers; i++) {
+		/* from within the handler, this call always succeeds */
+		sdei_api_event_context(i, &regs->regs[i]);
+	}
+
+	/*
+	 * We didn't take an exception to get here, set PAN. UAO will be cleared
+	 * by sdei_event_handler()s set_fs(USER_DS) call.
+	 */
+	__uaccess_enable_hw_pan();
+
+	err = sdei_event_handler(regs, arg);
+	if (err)
+		return SDEI_EV_FAILED;
+
+	if (elr != read_sysreg(elr_el1)) {
+		/*
+		 * We took a synchronous exception from the SDEI handler.
+		 * This could deadlock, and if you interrupt KVM it will
+		 * hyp-panic instead.
+		 */
+		pr_warn("unsafe: exception during handler\n");
+	}
+
+	mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
+
+	/*
+	 * If we interrupted the kernel with interrupts masked, we always go
+	 * back to wherever we came from.
+	 */
+	if (mode == kernel_mode && !interrupts_enabled(regs))
+		return SDEI_EV_HANDLED;
+
+	/*
+	 * Otherwise, we pretend this was an IRQ. This lets user space tasks
+	 * receive signals before we return to them, and KVM to invoke it's
+	 * world switch to do the same.
+	 *
+	 * See DDI0487B.a Table D1-7 'Vector offsets from vector table base
+	 * address'.
+	 */
+	if (mode == kernel_mode)
+		return vbar + 0x280;
+	else if (mode & PSR_MODE32_BIT)
+		return vbar + 0x680;
+
+	return vbar + 0x480;
+}
+
+
+asmlinkage __kprobes notrace unsigned long
+__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
+{
+	unsigned long ret;
+	bool do_nmi_exit = false;
+
+	/*
+	 * nmi_enter() deals with printk() re-entrance and use of RCU when
+	 * RCU believed this CPU was idle. Because critical events can
+	 * interrupt normal events, we may already be in_nmi().
+	 */
+	if (!in_nmi()) {
+		nmi_enter();
+		do_nmi_exit = true;
+	}
+
+	ret = _sdei_handler(regs, arg);
+
+	if (do_nmi_exit)
+		nmi_exit();
+
+	return ret;
+}
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index b120111a46be..f60c052e8d1c 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -178,7 +178,8 @@ static void __user *apply_user_offset(
 
 static int preserve_fpsimd_context(struct fpsimd_context __user *ctx)
 {
-	struct fpsimd_state *fpsimd = &current->thread.fpsimd_state;
+	struct user_fpsimd_state const *fpsimd =
+		&current->thread.fpsimd_state.user_fpsimd;
 	int err;
 
 	/* copy the FP and status/control registers */
@@ -195,7 +196,7 @@ static int preserve_fpsimd_context(struct fpsimd_context __user *ctx)
 
 static int restore_fpsimd_context(struct fpsimd_context __user *ctx)
 {
-	struct fpsimd_state fpsimd;
+	struct user_fpsimd_state fpsimd;
 	__u32 magic, size;
 	int err = 0;
 
@@ -266,7 +267,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user)
 {
 	int err;
 	unsigned int vq;
-	struct fpsimd_state fpsimd;
+	struct user_fpsimd_state fpsimd;
 	struct sve_context sve;
 
 	if (__copy_from_user(&sve, user->sve, sizeof(sve)))
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index 22711ee8e36c..79feb861929b 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -125,86 +125,6 @@ static inline int get_sigset_t(sigset_t *set,
 	return 0;
 }
 
-int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from)
-{
-	int err;
-
-	if (!access_ok(VERIFY_WRITE, to, sizeof(*to)))
-		return -EFAULT;
-
-	/* If you change siginfo_t structure, please be sure
-	 * this code is fixed accordingly.
-	 * It should never copy any pad contained in the structure
-	 * to avoid security leaks, but must copy the generic
-	 * 3 ints plus the relevant union member.
-	 * This routine must convert siginfo from 64bit to 32bit as well
-	 * at the same time.
-	 */
-	err = __put_user(from->si_signo, &to->si_signo);
-	err |= __put_user(from->si_errno, &to->si_errno);
-	err |= __put_user(from->si_code, &to->si_code);
-	if (from->si_code < 0)
-		err |= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad,
-				      SI_PAD_SIZE);
-	else switch (siginfo_layout(from->si_signo, from->si_code)) {
-	case SIL_KILL:
-		err |= __put_user(from->si_pid, &to->si_pid);
-		err |= __put_user(from->si_uid, &to->si_uid);
-		break;
-	case SIL_TIMER:
-		 err |= __put_user(from->si_tid, &to->si_tid);
-		 err |= __put_user(from->si_overrun, &to->si_overrun);
-		 err |= __put_user(from->si_int, &to->si_int);
-		break;
-	case SIL_POLL:
-		err |= __put_user(from->si_band, &to->si_band);
-		err |= __put_user(from->si_fd, &to->si_fd);
-		break;
-	case SIL_FAULT:
-		err |= __put_user((compat_uptr_t)(unsigned long)from->si_addr,
-				  &to->si_addr);
-#ifdef BUS_MCEERR_AO
-		/*
-		 * Other callers might not initialize the si_lsb field,
-		 * so check explicitly for the right codes here.
-		 */
-		if (from->si_signo == SIGBUS &&
-		    (from->si_code == BUS_MCEERR_AR || from->si_code == BUS_MCEERR_AO))
-			err |= __put_user(from->si_addr_lsb, &to->si_addr_lsb);
-#endif
-		break;
-	case SIL_CHLD:
-		err |= __put_user(from->si_pid, &to->si_pid);
-		err |= __put_user(from->si_uid, &to->si_uid);
-		err |= __put_user(from->si_status, &to->si_status);
-		err |= __put_user(from->si_utime, &to->si_utime);
-		err |= __put_user(from->si_stime, &to->si_stime);
-		break;
-	case SIL_RT:
-		err |= __put_user(from->si_pid, &to->si_pid);
-		err |= __put_user(from->si_uid, &to->si_uid);
-		err |= __put_user(from->si_int, &to->si_int);
-		break;
-	case SIL_SYS:
-		err |= __put_user((compat_uptr_t)(unsigned long)
-				from->si_call_addr, &to->si_call_addr);
-		err |= __put_user(from->si_syscall, &to->si_syscall);
-		err |= __put_user(from->si_arch, &to->si_arch);
-		break;
-	}
-	return err;
-}
-
-int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from)
-{
-	if (copy_from_user(to, from, __ARCH_SI_PREAMBLE_SIZE) ||
-	    copy_from_user(to->_sifields._pad,
-			   from->_sifields._pad, SI_PAD_SIZE))
-		return -EFAULT;
-
-	return 0;
-}
-
 /*
  * VFP save/restore code.
  *
@@ -228,7 +148,8 @@ union __fpsimd_vreg {
 
 static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)
 {
-	struct fpsimd_state *fpsimd = &current->thread.fpsimd_state;
+	struct user_fpsimd_state const *fpsimd =
+		&current->thread.fpsimd_state.user_fpsimd;
 	compat_ulong_t magic = VFP_MAGIC;
 	compat_ulong_t size = VFP_STORAGE_SIZE;
 	compat_ulong_t fpscr, fpexc;
@@ -277,7 +198,7 @@ static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)
 
 static int compat_restore_vfp_context(struct compat_vfp_sigframe __user *frame)
 {
-	struct fpsimd_state fpsimd;
+	struct user_fpsimd_state fpsimd;
 	compat_ulong_t magic = VFP_MAGIC;
 	compat_ulong_t size = VFP_STORAGE_SIZE;
 	compat_ulong_t fpscr;
diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index 10dd16d7902d..bebec8ef9372 100644
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -96,7 +96,7 @@ ENTRY(__cpu_suspend_enter)
 	ret
 ENDPROC(__cpu_suspend_enter)
 
-	.pushsection ".idmap.text", "ax"
+	.pushsection ".idmap.text", "awx"
 ENTRY(cpu_resume)
 	bl	el2_setup		// if in EL2 drop to EL1 cleanly
 	bl	__cpu_setup
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 551eb07c53b6..3b8ad7be9c33 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/arm_sdei.h>
 #include <linux/delay.h>
 #include <linux/init.h>
 #include <linux/spinlock.h>
@@ -836,6 +837,7 @@ static void ipi_cpu_stop(unsigned int cpu)
 	set_cpu_online(cpu, false);
 
 	local_daif_mask();
+	sdei_mask_local_cpu();
 
 	while (1)
 		cpu_relax();
@@ -853,6 +855,7 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
 	atomic_dec(&waiting_for_crash_ipi);
 
 	local_irq_disable();
+	sdei_mask_local_cpu();
 
 #ifdef CONFIG_HOTPLUG_CPU
 	if (cpu_ops[cpu]->cpu_die)
@@ -972,6 +975,8 @@ void smp_send_stop(void)
 	if (num_online_cpus() > 1)
 		pr_warning("SMP: failed to stop secondary CPUs %*pbl\n",
 			   cpumask_pr_args(cpu_online_mask));
+
+	sdei_mask_local_cpu();
 }
 
 #ifdef CONFIG_KEXEC_CORE
@@ -990,8 +995,10 @@ void crash_smp_send_stop(void)
 
 	cpus_stopped = 1;
 
-	if (num_online_cpus() == 1)
+	if (num_online_cpus() == 1) {
+		sdei_mask_local_cpu();
 		return;
+	}
 
 	cpumask_copy(&mask, cpu_online_mask);
 	cpumask_clear_cpu(smp_processor_id(), &mask);
@@ -1009,6 +1016,8 @@ void crash_smp_send_stop(void)
 	if (atomic_read(&waiting_for_crash_ipi) > 0)
 		pr_warning("SMP: failed to stop secondary CPUs %*pbl\n",
 			   cpumask_pr_args(&mask));
+
+	sdei_mask_local_cpu();
 }
 
 bool smp_crash_stop_failed(void)
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index 76809ccd309c..d5718a060672 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -59,6 +59,11 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame)
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	if (tsk->ret_stack &&
 			(frame->pc == (unsigned long)return_to_handler)) {
+		if (WARN_ON_ONCE(frame->graph == -1))
+			return -EINVAL;
+		if (frame->graph < -1)
+			frame->graph += FTRACE_NOTRACE_DEPTH;
+
 		/*
 		 * This is a case where function graph tracer has
 		 * modified a return address (LR) in a stack frame
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 3fe5ad884418..a307b9e13392 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -2,6 +2,7 @@
 #include <linux/ftrace.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
+#include <linux/uaccess.h>
 #include <asm/alternative.h>
 #include <asm/cacheflush.h>
 #include <asm/cpufeature.h>
@@ -51,8 +52,7 @@ void notrace __cpu_suspend_exit(void)
 	 * PSTATE was not saved over suspend/resume, re-enable any detected
 	 * features that might not have been set correctly.
 	 */
-	asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN,
-			CONFIG_ARM64_PAN));
+	__uaccess_enable_hw_pan();
 	uao_thread_switch(current);
 
 	/*
diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c
index 8b8bbd3eaa52..a382b2a1b84e 100644
--- a/arch/arm64/kernel/sys_compat.c
+++ b/arch/arm64/kernel/sys_compat.c
@@ -57,7 +57,7 @@ do_compat_cache_op(unsigned long start, unsigned long end, int flags)
 	if (end < start || flags)
 		return -EINVAL;
 
-	if (!access_ok(VERIFY_READ, start, end - start))
+	if (!access_ok(VERIFY_READ, (const void __user *)start, end - start))
 		return -EFAULT;
 
 	return __do_compat_cache_op(start, end);
diff --git a/arch/arm64/kernel/time.c b/arch/arm64/kernel/time.c
index a4391280fba9..f258636273c9 100644
--- a/arch/arm64/kernel/time.c
+++ b/arch/arm64/kernel/time.c
@@ -52,7 +52,7 @@ unsigned long profile_pc(struct pt_regs *regs)
 	frame.fp = regs->regs[29];
 	frame.pc = regs->pc;
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
-	frame.graph = -1; /* no task info */
+	frame.graph = current->curr_ret_stack;
 #endif
 	do {
 		int ret = unwind_frame(NULL, &frame);
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index 8d48b233e6ce..21868530018e 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -37,18 +37,14 @@ static int __init get_cpu_for_node(struct device_node *node)
 	if (!cpu_node)
 		return -1;
 
-	for_each_possible_cpu(cpu) {
-		if (of_get_cpu_node(cpu, NULL) == cpu_node) {
-			topology_parse_cpu_capacity(cpu_node, cpu);
-			of_node_put(cpu_node);
-			return cpu;
-		}
-	}
-
-	pr_crit("Unable to find CPU node for %pOF\n", cpu_node);
+	cpu = of_cpu_node_to_id(cpu_node);
+	if (cpu >= 0)
+		topology_parse_cpu_capacity(cpu_node, cpu);
+	else
+		pr_crit("Unable to find CPU node for %pOF\n", cpu_node);
 
 	of_node_put(cpu_node);
-	return -1;
+	return cpu;
 }
 
 static int __init parse_core(struct device_node *core, int cluster_id,
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index 3d3588fcd1c7..eb2d15147e8d 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -57,7 +57,7 @@ static const char *handler[]= {
 	"Error"
 };
 
-int show_unhandled_signals = 1;
+int show_unhandled_signals = 0;
 
 static void dump_backtrace_entry(unsigned long where)
 {
@@ -526,14 +526,6 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs)
 	}
 #endif
 
-	if (show_unhandled_signals_ratelimited()) {
-		pr_info("%s[%d]: syscall %d\n", current->comm,
-			task_pid_nr(current), regs->syscallno);
-		dump_instr("", regs);
-		if (user_mode(regs))
-			__show_regs(regs);
-	}
-
 	return sys_ni_syscall();
 }
 
@@ -662,17 +654,58 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs)
 }
 #endif
 
-asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
+void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
 {
-	nmi_enter();
-
 	console_verbose();
 
 	pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
 		smp_processor_id(), esr, esr_get_class_string(esr));
-	__show_regs(regs);
+	if (regs)
+		__show_regs(regs);
+
+	nmi_panic(regs, "Asynchronous SError Interrupt");
+
+	cpu_park_loop();
+	unreachable();
+}
+
+bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr)
+{
+	u32 aet = arm64_ras_serror_get_severity(esr);
+
+	switch (aet) {
+	case ESR_ELx_AET_CE:	/* corrected error */
+	case ESR_ELx_AET_UEO:	/* restartable, not yet consumed */
+		/*
+		 * The CPU can make progress. We may take UEO again as
+		 * a more severe error.
+		 */
+		return false;
+
+	case ESR_ELx_AET_UEU:	/* Uncorrected Unrecoverable */
+	case ESR_ELx_AET_UER:	/* Uncorrected Recoverable */
+		/*
+		 * The CPU can't make progress. The exception may have
+		 * been imprecise.
+		 */
+		return true;
+
+	case ESR_ELx_AET_UC:	/* Uncontainable or Uncategorized error */
+	default:
+		/* Error has been silently propagated */
+		arm64_serror_panic(regs, esr);
+	}
+}
+
+asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
+{
+	nmi_enter();
+
+	/* non-RAS errors are not containable */
+	if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
+		arm64_serror_panic(regs, esr);
 
-	panic("Asynchronous SError Interrupt");
+	nmi_exit();
 }
 
 void __pte_error(const char *file, int line, unsigned long val)
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 7da3e5c366a0..0221aca6493d 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -57,6 +57,17 @@ jiffies = jiffies_64;
 #define HIBERNATE_TEXT
 #endif
 
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define TRAMP_TEXT					\
+	. = ALIGN(PAGE_SIZE);				\
+	VMLINUX_SYMBOL(__entry_tramp_text_start) = .;	\
+	*(.entry.tramp.text)				\
+	. = ALIGN(PAGE_SIZE);				\
+	VMLINUX_SYMBOL(__entry_tramp_text_end) = .;
+#else
+#define TRAMP_TEXT
+#endif
+
 /*
  * The size of the PE/COFF section that covers the kernel image, which
  * runs from stext to _edata, must be a round multiple of the PE/COFF
@@ -113,6 +124,7 @@ SECTIONS
 			HYPERVISOR_TEXT
 			IDMAP_TEXT
 			HIBERNATE_TEXT
+			TRAMP_TEXT
 			*(.fixup)
 			*(.gnu.warning)
 		. = ALIGN(16);
@@ -206,13 +218,19 @@ SECTIONS
 	. = ALIGN(PAGE_SIZE);
 	idmap_pg_dir = .;
 	. += IDMAP_DIR_SIZE;
-	swapper_pg_dir = .;
-	. += SWAPPER_DIR_SIZE;
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	tramp_pg_dir = .;
+	. += PAGE_SIZE;
+#endif
 
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	reserved_ttbr0 = .;
 	. += RESERVED_TTBR0_SIZE;
 #endif
+	swapper_pg_dir = .;
+	. += SWAPPER_DIR_SIZE;
+	swapper_pg_end = .;
 
 	__pecoff_data_size = ABSOLUTE(. - __initdata_begin);
 	_end = .;
@@ -234,7 +252,10 @@ ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
 ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1))
 	<= SZ_4K, "Hibernate exit text too big or misaligned")
 #endif
-
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE,
+	"Entry trampoline text too big")
+#endif
 /*
  * If padding is applied before .head.text, virt<->phys conversions will fail.
  */
diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c
index 5c7f657dd207..959e50d2588c 100644
--- a/arch/arm64/kvm/guest.c
+++ b/arch/arm64/kvm/guest.c
@@ -361,10 +361,14 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
+	int ret = 0;
+
 	trace_kvm_set_guest_debug(vcpu, dbg->control);
 
-	if (dbg->control & ~KVM_GUESTDBG_VALID_MASK)
-		return -EINVAL;
+	if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) {
+		ret = -EINVAL;
+		goto out;
+	}
 
 	if (dbg->control & KVM_GUESTDBG_ENABLE) {
 		vcpu->guest_debug = dbg->control;
@@ -378,7 +382,9 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 		/* If not enabled clear all flags */
 		vcpu->guest_debug = 0;
 	}
-	return 0;
+
+out:
+	return ret;
 }
 
 int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index e60494f1eef9..e5e741bfffe1 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -22,19 +22,28 @@
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
 
+#include <kvm/arm_psci.h>
+
 #include <asm/esr.h>
+#include <asm/exception.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_coproc.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_mmu.h>
-#include <asm/kvm_psci.h>
 #include <asm/debug-monitors.h>
+#include <asm/traps.h>
 
 #define CREATE_TRACE_POINTS
 #include "trace.h"
 
 typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *);
 
+static void kvm_handle_guest_serror(struct kvm_vcpu *vcpu, u32 esr)
+{
+	if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(NULL, esr))
+		kvm_inject_vabt(vcpu);
+}
+
 static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	int ret;
@@ -43,7 +52,7 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 			    kvm_vcpu_hvc_get_imm(vcpu));
 	vcpu->stat.hvc_exit_stat++;
 
-	ret = kvm_psci_call(vcpu);
+	ret = kvm_hvc_call_handler(vcpu);
 	if (ret < 0) {
 		vcpu_set_reg(vcpu, 0, ~0UL);
 		return 1;
@@ -54,7 +63,16 @@ static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
+	/*
+	 * "If an SMC instruction executed at Non-secure EL1 is
+	 * trapped to EL2 because HCR_EL2.TSC is 1, the exception is a
+	 * Trap exception, not a Secure Monitor Call exception [...]"
+	 *
+	 * We need to advance the PC after the trap, as it would
+	 * otherwise return to the same address...
+	 */
 	vcpu_set_reg(vcpu, 0, ~0UL);
+	kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 	return 1;
 }
 
@@ -242,7 +260,6 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 			*vcpu_pc(vcpu) -= adj;
 		}
 
-		kvm_inject_vabt(vcpu);
 		return 1;
 	}
 
@@ -252,7 +269,6 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 	case ARM_EXCEPTION_IRQ:
 		return 1;
 	case ARM_EXCEPTION_EL1_SERROR:
-		kvm_inject_vabt(vcpu);
 		/* We may still need to return for single-step */
 		if (!(*vcpu_cpsr(vcpu) & DBG_SPSR_SS)
 			&& kvm_arm_handle_step_debug(vcpu, run))
@@ -275,3 +291,25 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run,
 		return 0;
 	}
 }
+
+/* For exit types that need handling before we can be preempted */
+void handle_exit_early(struct kvm_vcpu *vcpu, struct kvm_run *run,
+		       int exception_index)
+{
+	if (ARM_SERROR_PENDING(exception_index)) {
+		if (this_cpu_has_cap(ARM64_HAS_RAS_EXTN)) {
+			u64 disr = kvm_vcpu_get_disr(vcpu);
+
+			kvm_handle_guest_serror(vcpu, disr_to_esr(disr));
+		} else {
+			kvm_inject_vabt(vcpu);
+		}
+
+		return;
+	}
+
+	exception_index = ARM_EXCEPTION_CODE(exception_index);
+
+	if (exception_index == ARM_EXCEPTION_EL1_SERROR)
+		kvm_handle_guest_serror(vcpu, kvm_vcpu_get_hsr(vcpu));
+}
diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S
index 870828c364c5..5aa9ccf6db99 100644
--- a/arch/arm64/kvm/hyp-init.S
+++ b/arch/arm64/kvm/hyp-init.S
@@ -63,7 +63,8 @@ __do_hyp_init:
 	cmp	x0, #HVC_STUB_HCALL_NR
 	b.lo	__kvm_handle_stub_hvc
 
-	msr	ttbr0_el2, x0
+	phys_to_ttbr x4, x0
+	msr	ttbr0_el2, x4
 
 	mrs	x4, tcr_el1
 	ldr	x5, =TCR_EL2_MASK
@@ -71,30 +72,27 @@ __do_hyp_init:
 	mov	x5, #TCR_EL2_RES1
 	orr	x4, x4, x5
 
-#ifndef CONFIG_ARM64_VA_BITS_48
 	/*
-	 * If we are running with VA_BITS < 48, we may be running with an extra
-	 * level of translation in the ID map. This is only the case if system
-	 * RAM is out of range for the currently configured page size and number
-	 * of translation levels, in which case we will also need the extra
-	 * level for the HYP ID map, or we won't be able to enable the EL2 MMU.
+	 * The ID map may be configured to use an extended virtual address
+	 * range. This is only the case if system RAM is out of range for the
+	 * currently configured page size and VA_BITS, in which case we will
+	 * also need the extended virtual range for the HYP ID map, or we won't
+	 * be able to enable the EL2 MMU.
 	 *
 	 * However, at EL2, there is only one TTBR register, and we can't switch
 	 * between translation tables *and* update TCR_EL2.T0SZ at the same
-	 * time. Bottom line: we need the extra level in *both* our translation
-	 * tables.
+	 * time. Bottom line: we need to use the extended range with *both* our
+	 * translation tables.
 	 *
 	 * So use the same T0SZ value we use for the ID map.
 	 */
 	ldr_l	x5, idmap_t0sz
 	bfi	x4, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
-#endif
+
 	/*
-	 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the PS bits in
-	 * TCR_EL2.
+	 * Set the PS bits in TCR_EL2.
 	 */
-	mrs	x5, ID_AA64MMFR0_EL1
-	bfi	x4, x5, #16, #3
+	tcr_compute_pa_size x4, #TCR_EL2_PS_SHIFT, x5, x6
 
 	msr	tcr_el2, x4
 
@@ -122,6 +120,10 @@ CPU_BE(	orr	x4, x4, #SCTLR_ELx_EE)
 	kern_hyp_va	x2
 	msr	vbar_el2, x2
 
+	/* copy tpidr_el1 into tpidr_el2 for use by HYP */
+	mrs	x1, tpidr_el1
+	msr	tpidr_el2, x1
+
 	/* Hello, World! */
 	eret
 ENDPROC(__kvm_hyp_init)
diff --git a/arch/arm64/kvm/hyp/debug-sr.c b/arch/arm64/kvm/hyp/debug-sr.c
index f4363d40e2cd..dabb5cc7b087 100644
--- a/arch/arm64/kvm/hyp/debug-sr.c
+++ b/arch/arm64/kvm/hyp/debug-sr.c
@@ -21,6 +21,7 @@
 #include <asm/debug-monitors.h>
 #include <asm/kvm_asm.h>
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
 
 #define read_debug(r,n)		read_sysreg(r##n##_el1)
 #define write_debug(v,r,n)	write_sysreg(v, r##n##_el1)
diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S
index 12ee62d6d410..fdd1068ee3a5 100644
--- a/arch/arm64/kvm/hyp/entry.S
+++ b/arch/arm64/kvm/hyp/entry.S
@@ -62,8 +62,8 @@ ENTRY(__guest_enter)
 	// Store the host regs
 	save_callee_saved_regs x1
 
-	// Store the host_ctxt for use at exit time
-	str	x1, [sp, #-16]!
+	// Store host_ctxt and vcpu for use at exit time
+	stp	x1, x0, [sp, #-16]!
 
 	add	x18, x0, #VCPU_CONTEXT
 
@@ -124,6 +124,17 @@ ENTRY(__guest_exit)
 	// Now restore the host regs
 	restore_callee_saved_regs x2
 
+alternative_if ARM64_HAS_RAS_EXTN
+	// If we have the RAS extensions we can consume a pending error
+	// without an unmask-SError and isb.
+	esb
+	mrs_s	x2, SYS_DISR_EL1
+	str	x2, [x1, #(VCPU_FAULT_DISR - VCPU_CONTEXT)]
+	cbz	x2, 1f
+	msr_s	SYS_DISR_EL1, xzr
+	orr	x0, x0, #(1<<ARM_EXIT_WITH_SERROR_BIT)
+1:	ret
+alternative_else
 	// If we have a pending asynchronous abort, now is the
 	// time to find out. From your VAXorcist book, page 666:
 	// "Threaten me not, oh Evil one!  For I speak with
@@ -134,7 +145,9 @@ ENTRY(__guest_exit)
 	mov	x5, x0
 
 	dsb	sy		// Synchronize against in-flight ld/st
+	nop
 	msr	daifclr, #4	// Unmask aborts
+alternative_endif
 
 	// This is our single instruction exception window. A pending
 	// SError is guaranteed to occur at the earliest when we unmask
@@ -159,6 +172,10 @@ abort_guest_exit_end:
 ENDPROC(__guest_exit)
 
 ENTRY(__fpsimd_guest_restore)
+	// x0: esr
+	// x1: vcpu
+	// x2-x29,lr: vcpu regs
+	// vcpu x0-x1 on the stack
 	stp	x2, x3, [sp, #-16]!
 	stp	x4, lr, [sp, #-16]!
 
@@ -173,7 +190,7 @@ alternative_else
 alternative_endif
 	isb
 
-	mrs	x3, tpidr_el2
+	mov	x3, x1
 
 	ldr	x0, [x3, #VCPU_HOST_CONTEXT]
 	kern_hyp_va x0
@@ -196,3 +213,15 @@ alternative_endif
 
 	eret
 ENDPROC(__fpsimd_guest_restore)
+
+ENTRY(__qcom_hyp_sanitize_btac_predictors)
+	/**
+	 * Call SMC64 with Silicon provider serviceID 23<<8 (0xc2001700)
+	 * 0xC2000000-0xC200FFFF: assigned to SiP Service Calls
+	 * b15-b0: contains SiP functionID
+	 */
+	movz    x0, #0x1700
+	movk    x0, #0xc200, lsl #16
+	smc     #0
+	ret
+ENDPROC(__qcom_hyp_sanitize_btac_predictors)
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 5170ce1021da..f36464bd57c5 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -15,6 +15,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/arm-smccc.h>
 #include <linux/linkage.h>
 
 #include <asm/alternative.h>
@@ -64,10 +65,11 @@ alternative_endif
 	lsr	x0, x1, #ESR_ELx_EC_SHIFT
 
 	cmp	x0, #ESR_ELx_EC_HVC64
+	ccmp	x0, #ESR_ELx_EC_HVC32, #4, ne
 	b.ne	el1_trap
 
-	mrs	x1, vttbr_el2		// If vttbr is valid, the 64bit guest
-	cbnz	x1, el1_trap		// called HVC
+	mrs	x1, vttbr_el2		// If vttbr is valid, the guest
+	cbnz	x1, el1_hvc_guest	// called HVC
 
 	/* Here, we're pretty sure the host called HVC. */
 	ldp	x0, x1, [sp], #16
@@ -100,10 +102,25 @@ alternative_endif
 
 	eret
 
+el1_hvc_guest:
+	/*
+	 * Fastest possible path for ARM_SMCCC_ARCH_WORKAROUND_1.
+	 * The workaround has already been applied on the host,
+	 * so let's quickly get back to the guest. We don't bother
+	 * restoring x1, as it can be clobbered anyway.
+	 */
+	ldr	x1, [sp]				// Guest's x0
+	eor	w1, w1, #ARM_SMCCC_ARCH_WORKAROUND_1
+	cbnz	w1, el1_trap
+	mov	x0, x1
+	add	sp, sp, #16
+	eret
+
 el1_trap:
 	/*
 	 * x0: ESR_EC
 	 */
+	ldr	x1, [sp, #16 + 8]	// vcpu stored by __guest_enter
 
 	/*
 	 * We trap the first access to the FP/SIMD to save the host context
@@ -116,19 +133,18 @@ alternative_if_not ARM64_HAS_NO_FPSIMD
 	b.eq	__fpsimd_guest_restore
 alternative_else_nop_endif
 
-	mrs	x1, tpidr_el2
 	mov	x0, #ARM_EXCEPTION_TRAP
 	b	__guest_exit
 
 el1_irq:
 	stp     x0, x1, [sp, #-16]!
-	mrs	x1, tpidr_el2
+	ldr	x1, [sp, #16 + 8]
 	mov	x0, #ARM_EXCEPTION_IRQ
 	b	__guest_exit
 
 el1_error:
 	stp     x0, x1, [sp, #-16]!
-	mrs	x1, tpidr_el2
+	ldr	x1, [sp, #16 + 8]
 	mov	x0, #ARM_EXCEPTION_EL1_SERROR
 	b	__guest_exit
 
@@ -163,6 +179,18 @@ ENTRY(__hyp_do_panic)
 	eret
 ENDPROC(__hyp_do_panic)
 
+ENTRY(__hyp_panic)
+	/*
+	 * '=kvm_host_cpu_state' is a host VA from the constant pool, it may
+	 * not be accessible by this address from EL2, hyp_panic() converts
+	 * it with kern_hyp_va() before use.
+	 */
+	ldr	x0, =kvm_host_cpu_state
+	mrs	x1, tpidr_el2
+	add	x0, x0, x1
+	b	hyp_panic
+ENDPROC(__hyp_panic)
+
 .macro invalid_vector	label, target = __hyp_panic
 	.align	2
 \label:
diff --git a/arch/arm64/kvm/hyp/s2-setup.c b/arch/arm64/kvm/hyp/s2-setup.c
index a81f5e10fc8c..603e1ee83e89 100644
--- a/arch/arm64/kvm/hyp/s2-setup.c
+++ b/arch/arm64/kvm/hyp/s2-setup.c
@@ -32,6 +32,8 @@ u32 __hyp_text __init_stage2_translation(void)
 	 * PS is only 3. Fortunately, bit 19 is RES0 in VTCR_EL2...
 	 */
 	parange = read_sysreg(id_aa64mmfr0_el1) & 7;
+	if (parange > ID_AA64MMFR0_PARANGE_MAX)
+		parange = ID_AA64MMFR0_PARANGE_MAX;
 	val |= parange << 16;
 
 	/* Compute the actual PARange... */
diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index f7c651f3a8c0..870f4b1587f9 100644
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -17,10 +17,14 @@
 
 #include <linux/types.h>
 #include <linux/jump_label.h>
+#include <uapi/linux/psci.h>
+
+#include <kvm/arm_psci.h>
 
 #include <asm/kvm_asm.h>
 #include <asm/kvm_emulate.h>
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
 #include <asm/fpsimd.h>
 #include <asm/debug-monitors.h>
 
@@ -52,7 +56,7 @@ static void __hyp_text __activate_traps_vhe(void)
 	val &= ~(CPACR_EL1_FPEN | CPACR_EL1_ZEN);
 	write_sysreg(val, cpacr_el1);
 
-	write_sysreg(__kvm_hyp_vector, vbar_el1);
+	write_sysreg(kvm_get_hyp_vector(), vbar_el1);
 }
 
 static void __hyp_text __activate_traps_nvhe(void)
@@ -93,6 +97,9 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
 
 	write_sysreg(val, hcr_el2);
 
+	if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (val & HCR_VSE))
+		write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
+
 	/* Trap on AArch32 cp15 c15 accesses (EL1 or EL0) */
 	write_sysreg(1 << 15, hstr_el2);
 	/*
@@ -235,11 +242,12 @@ static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
 
 static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
 {
-	u64 esr = read_sysreg_el2(esr);
-	u8 ec = ESR_ELx_EC(esr);
+	u8 ec;
+	u64 esr;
 	u64 hpfar, far;
 
-	vcpu->arch.fault.esr_el2 = esr;
+	esr = vcpu->arch.fault.esr_el2;
+	ec = ESR_ELx_EC(esr);
 
 	if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
 		return true;
@@ -305,9 +313,9 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
 	u64 exit_code;
 
 	vcpu = kern_hyp_va(vcpu);
-	write_sysreg(vcpu, tpidr_el2);
 
 	host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+	host_ctxt->__hyp_running_vcpu = vcpu;
 	guest_ctxt = &vcpu->arch.ctxt;
 
 	__sysreg_save_host_state(host_ctxt);
@@ -332,6 +340,8 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
 	exit_code = __guest_enter(vcpu, host_ctxt);
 	/* And we're baaack! */
 
+	if (ARM_EXCEPTION_CODE(exit_code) != ARM_EXCEPTION_IRQ)
+		vcpu->arch.fault.esr_el2 = read_sysreg_el2(esr);
 	/*
 	 * We're using the raw exception code in order to only process
 	 * the trap if no SError is pending. We will come back to the
@@ -393,6 +403,16 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
 		/* 0 falls through to be handled out of EL2 */
 	}
 
+	if (cpus_have_const_cap(ARM64_HARDEN_BP_POST_GUEST_EXIT)) {
+		u32 midr = read_cpuid_id();
+
+		/* Apply BTAC predictors mitigation to all Falkor chips */
+		if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
+		    ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1)) {
+			__qcom_hyp_sanitize_btac_predictors();
+		}
+	}
+
 	fp_enabled = __fpsimd_enabled();
 
 	__sysreg_save_guest_state(guest_ctxt);
@@ -422,7 +442,8 @@ int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
 
 static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
 
-static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
+static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par,
+					     struct kvm_vcpu *vcpu)
 {
 	unsigned long str_va;
 
@@ -436,35 +457,35 @@ static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par)
 	__hyp_do_panic(str_va,
 		       spsr,  elr,
 		       read_sysreg(esr_el2),   read_sysreg_el2(far),
-		       read_sysreg(hpfar_el2), par,
-		       (void *)read_sysreg(tpidr_el2));
+		       read_sysreg(hpfar_el2), par, vcpu);
 }
 
-static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par)
+static void __hyp_text __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par,
+					    struct kvm_vcpu *vcpu)
 {
 	panic(__hyp_panic_string,
 	      spsr,  elr,
 	      read_sysreg_el2(esr),   read_sysreg_el2(far),
-	      read_sysreg(hpfar_el2), par,
-	      (void *)read_sysreg(tpidr_el2));
+	      read_sysreg(hpfar_el2), par, vcpu);
 }
 
 static hyp_alternate_select(__hyp_call_panic,
 			    __hyp_call_panic_nvhe, __hyp_call_panic_vhe,
 			    ARM64_HAS_VIRT_HOST_EXTN);
 
-void __hyp_text __noreturn __hyp_panic(void)
+void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *__host_ctxt)
 {
+	struct kvm_vcpu *vcpu = NULL;
+
 	u64 spsr = read_sysreg_el2(spsr);
 	u64 elr = read_sysreg_el2(elr);
 	u64 par = read_sysreg(par_el1);
 
 	if (read_sysreg(vttbr_el2)) {
-		struct kvm_vcpu *vcpu;
 		struct kvm_cpu_context *host_ctxt;
 
-		vcpu = (struct kvm_vcpu *)read_sysreg(tpidr_el2);
-		host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+		host_ctxt = kern_hyp_va(__host_ctxt);
+		vcpu = host_ctxt->__hyp_running_vcpu;
 		__timer_disable_traps(vcpu);
 		__deactivate_traps(vcpu);
 		__deactivate_vm(vcpu);
@@ -472,7 +493,7 @@ void __hyp_text __noreturn __hyp_panic(void)
 	}
 
 	/* Call panic for real */
-	__hyp_call_panic()(spsr, elr, par);
+	__hyp_call_panic()(spsr, elr, par, vcpu);
 
 	unreachable();
 }
diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c
index 934137647837..2c17afd2be96 100644
--- a/arch/arm64/kvm/hyp/sysreg-sr.c
+++ b/arch/arm64/kvm/hyp/sysreg-sr.c
@@ -27,8 +27,8 @@ static void __hyp_text __sysreg_do_nothing(struct kvm_cpu_context *ctxt) { }
 /*
  * Non-VHE: Both host and guest must save everything.
  *
- * VHE: Host must save tpidr*_el[01], actlr_el1, mdscr_el1, sp0, pc,
- * pstate, and guest must save everything.
+ * VHE: Host must save tpidr*_el0, actlr_el1, mdscr_el1, sp_el0,
+ * and guest must save everything.
  */
 
 static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
@@ -36,11 +36,8 @@ static void __hyp_text __sysreg_save_common_state(struct kvm_cpu_context *ctxt)
 	ctxt->sys_regs[ACTLR_EL1]	= read_sysreg(actlr_el1);
 	ctxt->sys_regs[TPIDR_EL0]	= read_sysreg(tpidr_el0);
 	ctxt->sys_regs[TPIDRRO_EL0]	= read_sysreg(tpidrro_el0);
-	ctxt->sys_regs[TPIDR_EL1]	= read_sysreg(tpidr_el1);
 	ctxt->sys_regs[MDSCR_EL1]	= read_sysreg(mdscr_el1);
 	ctxt->gp_regs.regs.sp		= read_sysreg(sp_el0);
-	ctxt->gp_regs.regs.pc		= read_sysreg_el2(elr);
-	ctxt->gp_regs.regs.pstate	= read_sysreg_el2(spsr);
 }
 
 static void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt)
@@ -62,10 +59,16 @@ static void __hyp_text __sysreg_save_state(struct kvm_cpu_context *ctxt)
 	ctxt->sys_regs[AMAIR_EL1]	= read_sysreg_el1(amair);
 	ctxt->sys_regs[CNTKCTL_EL1]	= read_sysreg_el1(cntkctl);
 	ctxt->sys_regs[PAR_EL1]		= read_sysreg(par_el1);
+	ctxt->sys_regs[TPIDR_EL1]	= read_sysreg(tpidr_el1);
 
 	ctxt->gp_regs.sp_el1		= read_sysreg(sp_el1);
 	ctxt->gp_regs.elr_el1		= read_sysreg_el1(elr);
 	ctxt->gp_regs.spsr[KVM_SPSR_EL1]= read_sysreg_el1(spsr);
+	ctxt->gp_regs.regs.pc		= read_sysreg_el2(elr);
+	ctxt->gp_regs.regs.pstate	= read_sysreg_el2(spsr);
+
+	if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN))
+		ctxt->sys_regs[DISR_EL1] = read_sysreg_s(SYS_VDISR_EL2);
 }
 
 static hyp_alternate_select(__sysreg_call_save_host_state,
@@ -89,11 +92,8 @@ static void __hyp_text __sysreg_restore_common_state(struct kvm_cpu_context *ctx
 	write_sysreg(ctxt->sys_regs[ACTLR_EL1],	  actlr_el1);
 	write_sysreg(ctxt->sys_regs[TPIDR_EL0],	  tpidr_el0);
 	write_sysreg(ctxt->sys_regs[TPIDRRO_EL0], tpidrro_el0);
-	write_sysreg(ctxt->sys_regs[TPIDR_EL1],	  tpidr_el1);
 	write_sysreg(ctxt->sys_regs[MDSCR_EL1],	  mdscr_el1);
 	write_sysreg(ctxt->gp_regs.regs.sp,	  sp_el0);
-	write_sysreg_el2(ctxt->gp_regs.regs.pc,	  elr);
-	write_sysreg_el2(ctxt->gp_regs.regs.pstate, spsr);
 }
 
 static void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt)
@@ -115,10 +115,16 @@ static void __hyp_text __sysreg_restore_state(struct kvm_cpu_context *ctxt)
 	write_sysreg_el1(ctxt->sys_regs[AMAIR_EL1],	amair);
 	write_sysreg_el1(ctxt->sys_regs[CNTKCTL_EL1], 	cntkctl);
 	write_sysreg(ctxt->sys_regs[PAR_EL1],		par_el1);
+	write_sysreg(ctxt->sys_regs[TPIDR_EL1],		tpidr_el1);
 
 	write_sysreg(ctxt->gp_regs.sp_el1,		sp_el1);
 	write_sysreg_el1(ctxt->gp_regs.elr_el1,		elr);
 	write_sysreg_el1(ctxt->gp_regs.spsr[KVM_SPSR_EL1],spsr);
+	write_sysreg_el2(ctxt->gp_regs.regs.pc,		elr);
+	write_sysreg_el2(ctxt->gp_regs.regs.pstate,	spsr);
+
+	if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN))
+		write_sysreg_s(ctxt->sys_regs[DISR_EL1], SYS_VDISR_EL2);
 }
 
 static hyp_alternate_select(__sysreg_call_restore_host_state,
diff --git a/arch/arm64/kvm/hyp/tlb.c b/arch/arm64/kvm/hyp/tlb.c
index 73464a96c365..131c7772703c 100644
--- a/arch/arm64/kvm/hyp/tlb.c
+++ b/arch/arm64/kvm/hyp/tlb.c
@@ -16,6 +16,7 @@
  */
 
 #include <asm/kvm_hyp.h>
+#include <asm/kvm_mmu.h>
 #include <asm/tlbflush.h>
 
 static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm)
diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c
index 8ecbcb40e317..60666a056944 100644
--- a/arch/arm64/kvm/inject_fault.c
+++ b/arch/arm64/kvm/inject_fault.c
@@ -164,14 +164,25 @@ void kvm_inject_undefined(struct kvm_vcpu *vcpu)
 		inject_undef64(vcpu);
 }
 
+static void pend_guest_serror(struct kvm_vcpu *vcpu, u64 esr)
+{
+	vcpu_set_vsesr(vcpu, esr);
+	vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) | HCR_VSE);
+}
+
 /**
  * kvm_inject_vabt - inject an async abort / SError into the guest
  * @vcpu: The VCPU to receive the exception
  *
  * It is assumed that this code is called from the VCPU thread and that the
  * VCPU therefore is not currently executing guest code.
+ *
+ * Systems with the RAS Extensions specify an imp-def ESR (ISV/IDS = 1) with
+ * the remaining ISS all-zeros so that this error is not interpreted as an
+ * uncategorized RAS error. Without the RAS Extensions we can't specify an ESR
+ * value, so the CPU generates an imp-def value.
  */
 void kvm_inject_vabt(struct kvm_vcpu *vcpu)
 {
-	vcpu_set_hcr(vcpu, vcpu_get_hcr(vcpu) | HCR_VSE);
+	pend_guest_serror(vcpu, ESR_ELx_ISV);
 }
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 1830ebc227d1..50a43c7b97ca 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -1159,6 +1159,16 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	{ SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
 	{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
 	{ SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
+
+	{ SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
+	{ SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
+	{ SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
+	{ SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
+	{ SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
+	{ SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
+	{ SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
+	{ SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
+
 	{ SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
 	{ SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
 
@@ -1169,6 +1179,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
 	{ SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
 
 	{ SYS_DESC(SYS_VBAR_EL1), NULL, reset_val, VBAR_EL1, 0 },
+	{ SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
 
 	{ SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
 	{ SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S
index e88fb99c1561..21ba0b29621b 100644
--- a/arch/arm64/lib/clear_user.S
+++ b/arch/arm64/lib/clear_user.S
@@ -21,7 +21,7 @@
 
 	.text
 
-/* Prototype: int __clear_user(void *addr, size_t sz)
+/* Prototype: int __arch_clear_user(void *addr, size_t sz)
  * Purpose  : clear some user memory
  * Params   : addr - user memory address to clear
  *          : sz   - number of bytes to clear
@@ -29,8 +29,8 @@
  *
  * Alignment fixed up by hardware.
  */
-ENTRY(__clear_user)
-	uaccess_enable_not_uao x2, x3
+ENTRY(__arch_clear_user)
+	uaccess_enable_not_uao x2, x3, x4
 	mov	x2, x1			// save the size for fixup return
 	subs	x1, x1, #8
 	b.mi	2f
@@ -50,9 +50,9 @@ uao_user_alternative 9f, strh, sttrh, wzr, x0, 2
 	b.mi	5f
 uao_user_alternative 9f, strb, sttrb, wzr, x0, 0
 5:	mov	x0, #0
-	uaccess_disable_not_uao x2
+	uaccess_disable_not_uao x2, x3
 	ret
-ENDPROC(__clear_user)
+ENDPROC(__arch_clear_user)
 
 	.section .fixup,"ax"
 	.align	2
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 4b5d826895ff..20305d485046 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -64,10 +64,10 @@
 
 end	.req	x5
 ENTRY(__arch_copy_from_user)
-	uaccess_enable_not_uao x3, x4
+	uaccess_enable_not_uao x3, x4, x5
 	add	end, x0, x2
 #include "copy_template.S"
-	uaccess_disable_not_uao x3
+	uaccess_disable_not_uao x3, x4
 	mov	x0, #0				// Nothing to copy
 	ret
 ENDPROC(__arch_copy_from_user)
diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S
index b24a830419ad..54b75deb1d16 100644
--- a/arch/arm64/lib/copy_in_user.S
+++ b/arch/arm64/lib/copy_in_user.S
@@ -64,14 +64,15 @@
 	.endm
 
 end	.req	x5
-ENTRY(raw_copy_in_user)
-	uaccess_enable_not_uao x3, x4
+
+ENTRY(__arch_copy_in_user)
+	uaccess_enable_not_uao x3, x4, x5
 	add	end, x0, x2
 #include "copy_template.S"
-	uaccess_disable_not_uao x3
+	uaccess_disable_not_uao x3, x4
 	mov	x0, #0
 	ret
-ENDPROC(raw_copy_in_user)
+ENDPROC(__arch_copy_in_user)
 
 	.section .fixup,"ax"
 	.align	2
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 351f0766f7a6..fda6172d6b88 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -63,10 +63,10 @@
 
 end	.req	x5
 ENTRY(__arch_copy_to_user)
-	uaccess_enable_not_uao x3, x4
+	uaccess_enable_not_uao x3, x4, x5
 	add	end, x0, x2
 #include "copy_template.S"
-	uaccess_disable_not_uao x3
+	uaccess_disable_not_uao x3, x4
 	mov	x0, #0
 	ret
 ENDPROC(__arch_copy_to_user)
diff --git a/arch/arm64/lib/tishift.S b/arch/arm64/lib/tishift.S
index 0179a43cc045..d3db9b2cd479 100644
--- a/arch/arm64/lib/tishift.S
+++ b/arch/arm64/lib/tishift.S
@@ -38,19 +38,19 @@ ENTRY(__ashlti3)
 ENDPROC(__ashlti3)
 
 ENTRY(__ashrti3)
-	cbz	x2, 3f
+	cbz	x2, 1f
 	mov	x3, #64
 	sub	x3, x3, x2
 	cmp	x3, #0
-	b.le	4f
+	b.le	2f
 	lsr	x0, x0, x2
 	lsl	x3, x1, x3
 	asr	x2, x1, x2
 	orr	x0, x0, x3
 	mov	x1, x2
-3:
+1:
 	ret
-4:
+2:
 	neg	w0, w3
 	asr	x2, x1, #63
 	asr	x0, x1, x0
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 7f1dbe962cf5..758bde7e2fa6 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -49,7 +49,7 @@ ENTRY(flush_icache_range)
  *	- end     - virtual end address of region
  */
 ENTRY(__flush_cache_user_range)
-	uaccess_ttbr0_enable x2, x3
+	uaccess_ttbr0_enable x2, x3, x4
 	dcache_line_size x2, x3
 	sub	x3, x2, #1
 	bic	x4, x0, x3
@@ -60,19 +60,10 @@ user_alt 9f, "dc cvau, x4",  "dc civac, x4",  ARM64_WORKAROUND_CLEAN_CACHE
 	b.lo	1b
 	dsb	ish
 
-	icache_line_size x2, x3
-	sub	x3, x2, #1
-	bic	x4, x0, x3
-1:
-USER(9f, ic	ivau, x4	)		// invalidate I line PoU
-	add	x4, x4, x2
-	cmp	x4, x1
-	b.lo	1b
-	dsb	ish
-	isb
+	invalidate_icache_by_line x0, x1, x2, x3, 9f
 	mov	x0, #0
 1:
-	uaccess_ttbr0_disable x1
+	uaccess_ttbr0_disable x1, x2
 	ret
 9:
 	mov	x0, #-EFAULT
@@ -80,6 +71,27 @@ USER(9f, ic	ivau, x4	)		// invalidate I line PoU
 ENDPROC(flush_icache_range)
 ENDPROC(__flush_cache_user_range)
 
+/*
+ *	invalidate_icache_range(start,end)
+ *
+ *	Ensure that the I cache is invalid within specified region.
+ *
+ *	- start   - virtual start address of region
+ *	- end     - virtual end address of region
+ */
+ENTRY(invalidate_icache_range)
+	uaccess_ttbr0_enable x2, x3, x4
+
+	invalidate_icache_by_line x0, x1, x2, x3, 2f
+	mov	x0, xzr
+1:
+	uaccess_ttbr0_disable x1, x2
+	ret
+2:
+	mov	x0, #-EFAULT
+	b	1b
+ENDPROC(invalidate_icache_range)
+
 /*
  *	__flush_dcache_area(kaddr, size)
  *
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index 6f4017046323..301417ae2ba8 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -39,7 +39,16 @@ static cpumask_t tlb_flush_pending;
 
 #define ASID_MASK		(~GENMASK(asid_bits - 1, 0))
 #define ASID_FIRST_VERSION	(1UL << asid_bits)
-#define NUM_USER_ASIDS		ASID_FIRST_VERSION
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define NUM_USER_ASIDS		(ASID_FIRST_VERSION >> 1)
+#define asid2idx(asid)		(((asid) & ~ASID_MASK) >> 1)
+#define idx2asid(idx)		(((idx) << 1) & ~ASID_MASK)
+#else
+#define NUM_USER_ASIDS		(ASID_FIRST_VERSION)
+#define asid2idx(asid)		((asid) & ~ASID_MASK)
+#define idx2asid(idx)		asid2idx(idx)
+#endif
 
 /* Get the ASIDBits supported by the current CPU */
 static u32 get_cpu_asid_bits(void)
@@ -79,13 +88,6 @@ void verify_cpu_asid_bits(void)
 	}
 }
 
-static void set_reserved_asid_bits(void)
-{
-	if (IS_ENABLED(CONFIG_QCOM_FALKOR_ERRATUM_1003) &&
-	    cpus_have_const_cap(ARM64_WORKAROUND_QCOM_FALKOR_E1003))
-		__set_bit(FALKOR_RESERVED_ASID, asid_map);
-}
-
 static void flush_context(unsigned int cpu)
 {
 	int i;
@@ -94,8 +96,6 @@ static void flush_context(unsigned int cpu)
 	/* Update the list of reserved ASIDs and the ASID bitmap. */
 	bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
 
-	set_reserved_asid_bits();
-
 	for_each_possible_cpu(i) {
 		asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0);
 		/*
@@ -107,7 +107,7 @@ static void flush_context(unsigned int cpu)
 		 */
 		if (asid == 0)
 			asid = per_cpu(reserved_asids, i);
-		__set_bit(asid & ~ASID_MASK, asid_map);
+		__set_bit(asid2idx(asid), asid_map);
 		per_cpu(reserved_asids, i) = asid;
 	}
 
@@ -162,16 +162,16 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
 		 * We had a valid ASID in a previous life, so try to re-use
 		 * it if possible.
 		 */
-		asid &= ~ASID_MASK;
-		if (!__test_and_set_bit(asid, asid_map))
+		if (!__test_and_set_bit(asid2idx(asid), asid_map))
 			return newasid;
 	}
 
 	/*
 	 * Allocate a free ASID. If we can't find one, take a note of the
-	 * currently active ASIDs and mark the TLBs as requiring flushes.
-	 * We always count from ASID #1, as we use ASID #0 when setting a
-	 * reserved TTBR0 for the init_mm.
+	 * currently active ASIDs and mark the TLBs as requiring flushes.  We
+	 * always count from ASID #2 (index 1), as we use ASID #0 when setting
+	 * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd
+	 * pairs.
 	 */
 	asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
 	if (asid != NUM_USER_ASIDS)
@@ -188,32 +188,35 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
 set_asid:
 	__set_bit(asid, asid_map);
 	cur_idx = asid;
-	return asid | generation;
+	return idx2asid(asid) | generation;
 }
 
 void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
 {
 	unsigned long flags;
-	u64 asid;
+	u64 asid, old_active_asid;
 
 	asid = atomic64_read(&mm->context.id);
 
 	/*
 	 * The memory ordering here is subtle.
-	 * If our ASID matches the current generation, then we update
-	 * our active_asids entry with a relaxed xchg. Racing with a
-	 * concurrent rollover means that either:
+	 * If our active_asids is non-zero and the ASID matches the current
+	 * generation, then we update the active_asids entry with a relaxed
+	 * cmpxchg. Racing with a concurrent rollover means that either:
 	 *
-	 * - We get a zero back from the xchg and end up waiting on the
+	 * - We get a zero back from the cmpxchg and end up waiting on the
 	 *   lock. Taking the lock synchronises with the rollover and so
 	 *   we are forced to see the updated generation.
 	 *
-	 * - We get a valid ASID back from the xchg, which means the
+	 * - We get a valid ASID back from the cmpxchg, which means the
 	 *   relaxed xchg in flush_context will treat us as reserved
 	 *   because atomic RmWs are totally ordered for a given location.
 	 */
-	if (!((asid ^ atomic64_read(&asid_generation)) >> asid_bits)
-	    && atomic64_xchg_relaxed(&per_cpu(active_asids, cpu), asid))
+	old_active_asid = atomic64_read(&per_cpu(active_asids, cpu));
+	if (old_active_asid &&
+	    !((asid ^ atomic64_read(&asid_generation)) >> asid_bits) &&
+	    atomic64_cmpxchg_relaxed(&per_cpu(active_asids, cpu),
+				     old_active_asid, asid))
 		goto switch_mm_fastpath;
 
 	raw_spin_lock_irqsave(&cpu_asid_lock, flags);
@@ -231,6 +234,9 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
 	raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
 
 switch_mm_fastpath:
+
+	arm64_apply_bp_hardening();
+
 	/*
 	 * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when
 	 * emulating PAN.
@@ -239,6 +245,15 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
 		cpu_switch_mm(mm->pgd, mm);
 }
 
+/* Errata workaround post TTBRx_EL1 update. */
+asmlinkage void post_ttbr_update_workaround(void)
+{
+	asm(ALTERNATIVE("nop; nop; nop",
+			"ic iallu; dsb nsh; isb",
+			ARM64_WORKAROUND_CAVIUM_27456,
+			CONFIG_CAVIUM_ERRATUM_27456));
+}
+
 static int asids_init(void)
 {
 	asid_bits = get_cpu_asid_bits();
@@ -254,8 +269,6 @@ static int asids_init(void)
 		panic("Failed to allocate bitmap for %lu ASIDs\n",
 		      NUM_USER_ASIDS);
 
-	set_reserved_asid_bits();
-
 	pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
 	return 0;
 }
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index b45c5bcaeccb..a96ec0181818 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -24,7 +24,7 @@
 #include <linux/export.h>
 #include <linux/slab.h>
 #include <linux/genalloc.h>
-#include <linux/dma-mapping.h>
+#include <linux/dma-direct.h>
 #include <linux/dma-contiguous.h>
 #include <linux/vmalloc.h>
 #include <linux/swiotlb.h>
@@ -91,46 +91,6 @@ static int __free_from_pool(void *start, size_t size)
 	return 1;
 }
 
-static void *__dma_alloc_coherent(struct device *dev, size_t size,
-				  dma_addr_t *dma_handle, gfp_t flags,
-				  unsigned long attrs)
-{
-	if (IS_ENABLED(CONFIG_ZONE_DMA) &&
-	    dev->coherent_dma_mask <= DMA_BIT_MASK(32))
-		flags |= GFP_DMA;
-	if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) {
-		struct page *page;
-		void *addr;
-
-		page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
-						 get_order(size), flags);
-		if (!page)
-			return NULL;
-
-		*dma_handle = phys_to_dma(dev, page_to_phys(page));
-		addr = page_address(page);
-		memset(addr, 0, size);
-		return addr;
-	} else {
-		return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
-	}
-}
-
-static void __dma_free_coherent(struct device *dev, size_t size,
-				void *vaddr, dma_addr_t dma_handle,
-				unsigned long attrs)
-{
-	bool freed;
-	phys_addr_t paddr = dma_to_phys(dev, dma_handle);
-
-
-	freed = dma_release_from_contiguous(dev,
-					phys_to_page(paddr),
-					size >> PAGE_SHIFT);
-	if (!freed)
-		swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
 static void *__dma_alloc(struct device *dev, size_t size,
 			 dma_addr_t *dma_handle, gfp_t flags,
 			 unsigned long attrs)
@@ -152,7 +112,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
 		return addr;
 	}
 
-	ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
+	ptr = swiotlb_alloc(dev, size, dma_handle, flags, attrs);
 	if (!ptr)
 		goto no_mem;
 
@@ -173,7 +133,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
 	return coherent_ptr;
 
 no_map:
-	__dma_free_coherent(dev, size, ptr, *dma_handle, attrs);
+	swiotlb_free(dev, size, ptr, *dma_handle, attrs);
 no_mem:
 	return NULL;
 }
@@ -191,7 +151,7 @@ static void __dma_free(struct device *dev, size_t size,
 			return;
 		vunmap(vaddr);
 	}
-	__dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
+	swiotlb_free(dev, size, swiotlb_addr, dma_handle, attrs);
 }
 
 static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
@@ -368,7 +328,7 @@ static int __swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t addr)
 	return 0;
 }
 
-static const struct dma_map_ops swiotlb_dma_ops = {
+static const struct dma_map_ops arm64_swiotlb_dma_ops = {
 	.alloc = __dma_alloc,
 	.free = __dma_free,
 	.mmap = __swiotlb_mmap,
@@ -397,7 +357,7 @@ static int __init atomic_pool_init(void)
 		page = dma_alloc_from_contiguous(NULL, nr_pages,
 						 pool_size_order, GFP_KERNEL);
 	else
-		page = alloc_pages(GFP_DMA, pool_size_order);
+		page = alloc_pages(GFP_DMA32, pool_size_order);
 
 	if (page) {
 		int ret;
@@ -923,7 +883,7 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
 			const struct iommu_ops *iommu, bool coherent)
 {
 	if (!dev->dma_ops)
-		dev->dma_ops = &swiotlb_dma_ops;
+		dev->dma_ops = &arm64_swiotlb_dma_ops;
 
 	dev->archdata.dma_coherent = coherent;
 	__iommu_setup_dma_ops(dev, dma_base, size, iommu);
diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index 7b60d62ac593..65dfc8571bf8 100644
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -286,48 +286,52 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level,
 
 }
 
-static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start)
+static void walk_pte(struct pg_state *st, pmd_t *pmdp, unsigned long start)
 {
-	pte_t *pte = pte_offset_kernel(pmd, 0UL);
+	pte_t *ptep = pte_offset_kernel(pmdp, 0UL);
 	unsigned long addr;
 	unsigned i;
 
-	for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
+	for (i = 0; i < PTRS_PER_PTE; i++, ptep++) {
 		addr = start + i * PAGE_SIZE;
-		note_page(st, addr, 4, pte_val(*pte));
+		note_page(st, addr, 4, READ_ONCE(pte_val(*ptep)));
 	}
 }
 
-static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
+static void walk_pmd(struct pg_state *st, pud_t *pudp, unsigned long start)
 {
-	pmd_t *pmd = pmd_offset(pud, 0UL);
+	pmd_t *pmdp = pmd_offset(pudp, 0UL);
 	unsigned long addr;
 	unsigned i;
 
-	for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
+	for (i = 0; i < PTRS_PER_PMD; i++, pmdp++) {
+		pmd_t pmd = READ_ONCE(*pmdp);
+
 		addr = start + i * PMD_SIZE;
-		if (pmd_none(*pmd) || pmd_sect(*pmd)) {
-			note_page(st, addr, 3, pmd_val(*pmd));
+		if (pmd_none(pmd) || pmd_sect(pmd)) {
+			note_page(st, addr, 3, pmd_val(pmd));
 		} else {
-			BUG_ON(pmd_bad(*pmd));
-			walk_pte(st, pmd, addr);
+			BUG_ON(pmd_bad(pmd));
+			walk_pte(st, pmdp, addr);
 		}
 	}
 }
 
-static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
+static void walk_pud(struct pg_state *st, pgd_t *pgdp, unsigned long start)
 {
-	pud_t *pud = pud_offset(pgd, 0UL);
+	pud_t *pudp = pud_offset(pgdp, 0UL);
 	unsigned long addr;
 	unsigned i;
 
-	for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
+	for (i = 0; i < PTRS_PER_PUD; i++, pudp++) {
+		pud_t pud = READ_ONCE(*pudp);
+
 		addr = start + i * PUD_SIZE;
-		if (pud_none(*pud) || pud_sect(*pud)) {
-			note_page(st, addr, 2, pud_val(*pud));
+		if (pud_none(pud) || pud_sect(pud)) {
+			note_page(st, addr, 2, pud_val(pud));
 		} else {
-			BUG_ON(pud_bad(*pud));
-			walk_pmd(st, pud, addr);
+			BUG_ON(pud_bad(pud));
+			walk_pmd(st, pudp, addr);
 		}
 	}
 }
@@ -335,17 +339,19 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
 static void walk_pgd(struct pg_state *st, struct mm_struct *mm,
 		     unsigned long start)
 {
-	pgd_t *pgd = pgd_offset(mm, 0UL);
+	pgd_t *pgdp = pgd_offset(mm, 0UL);
 	unsigned i;
 	unsigned long addr;
 
-	for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
+	for (i = 0; i < PTRS_PER_PGD; i++, pgdp++) {
+		pgd_t pgd = READ_ONCE(*pgdp);
+
 		addr = start + i * PGDIR_SIZE;
-		if (pgd_none(*pgd)) {
-			note_page(st, addr, 1, pgd_val(*pgd));
+		if (pgd_none(pgd)) {
+			note_page(st, addr, 1, pgd_val(pgd));
 		} else {
-			BUG_ON(pgd_bad(*pgd));
-			walk_pud(st, pgd, addr);
+			BUG_ON(pgd_bad(pgd));
+			walk_pud(st, pgdp, addr);
 		}
 	}
 }
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 9b7f89df49db..bff11553eb05 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -130,7 +130,8 @@ static void mem_abort_decode(unsigned int esr)
 void show_pte(unsigned long addr)
 {
 	struct mm_struct *mm;
-	pgd_t *pgd;
+	pgd_t *pgdp;
+	pgd_t pgd;
 
 	if (addr < TASK_SIZE) {
 		/* TTBR0 */
@@ -149,33 +150,37 @@ void show_pte(unsigned long addr)
 		return;
 	}
 
-	pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgd = %p\n",
+	pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgdp = %p\n",
 		 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
 		 VA_BITS, mm->pgd);
-	pgd = pgd_offset(mm, addr);
-	pr_alert("[%016lx] *pgd=%016llx", addr, pgd_val(*pgd));
+	pgdp = pgd_offset(mm, addr);
+	pgd = READ_ONCE(*pgdp);
+	pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
 
 	do {
-		pud_t *pud;
-		pmd_t *pmd;
-		pte_t *pte;
+		pud_t *pudp, pud;
+		pmd_t *pmdp, pmd;
+		pte_t *ptep, pte;
 
-		if (pgd_none(*pgd) || pgd_bad(*pgd))
+		if (pgd_none(pgd) || pgd_bad(pgd))
 			break;
 
-		pud = pud_offset(pgd, addr);
-		pr_cont(", *pud=%016llx", pud_val(*pud));
-		if (pud_none(*pud) || pud_bad(*pud))
+		pudp = pud_offset(pgdp, addr);
+		pud = READ_ONCE(*pudp);
+		pr_cont(", pud=%016llx", pud_val(pud));
+		if (pud_none(pud) || pud_bad(pud))
 			break;
 
-		pmd = pmd_offset(pud, addr);
-		pr_cont(", *pmd=%016llx", pmd_val(*pmd));
-		if (pmd_none(*pmd) || pmd_bad(*pmd))
+		pmdp = pmd_offset(pudp, addr);
+		pmd = READ_ONCE(*pmdp);
+		pr_cont(", pmd=%016llx", pmd_val(pmd));
+		if (pmd_none(pmd) || pmd_bad(pmd))
 			break;
 
-		pte = pte_offset_map(pmd, addr);
-		pr_cont(", *pte=%016llx", pte_val(*pte));
-		pte_unmap(pte);
+		ptep = pte_offset_map(pmdp, addr);
+		pte = READ_ONCE(*ptep);
+		pr_cont(", pte=%016llx", pte_val(pte));
+		pte_unmap(ptep);
 	} while(0);
 
 	pr_cont("\n");
@@ -196,8 +201,9 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
 			  pte_t entry, int dirty)
 {
 	pteval_t old_pteval, pteval;
+	pte_t pte = READ_ONCE(*ptep);
 
-	if (pte_same(*ptep, entry))
+	if (pte_same(pte, entry))
 		return 0;
 
 	/* only preserve the access flags and write permission */
@@ -210,7 +216,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
 	 * (calculated as: a & b == ~(~a | ~b)).
 	 */
 	pte_val(entry) ^= PTE_RDONLY;
-	pteval = READ_ONCE(pte_val(*ptep));
+	pteval = pte_val(pte);
 	do {
 		old_pteval = pteval;
 		pteval ^= PTE_RDONLY;
@@ -240,7 +246,7 @@ static inline bool is_permission_fault(unsigned int esr, struct pt_regs *regs,
 	if (fsc_type == ESR_ELx_FSC_PERM)
 		return true;
 
-	if (addr < USER_DS && system_uses_ttbr0_pan())
+	if (addr < TASK_SIZE && system_uses_ttbr0_pan())
 		return fsc_type == ESR_ELx_FSC_FAULT &&
 			(regs->pstate & PSR_PAN_BIT);
 
@@ -414,7 +420,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
 		mm_flags |= FAULT_FLAG_WRITE;
 	}
 
-	if (addr < USER_DS && is_permission_fault(esr, regs, addr)) {
+	if (addr < TASK_SIZE && is_permission_fault(esr, regs, addr)) {
 		/* regs->orig_addr_limit may be 0 if we entered from EL0 */
 		if (regs->orig_addr_limit == KERNEL_DS)
 			die("Accessing user space memory with fs=KERNEL_DS", regs, esr);
@@ -596,7 +602,7 @@ static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 
 	info.si_signo = SIGBUS;
 	info.si_errno = 0;
-	info.si_code  = 0;
+	info.si_code  = BUS_FIXME;
 	if (esr & ESR_ELx_FnV)
 		info.si_addr = NULL;
 	else
@@ -607,70 +613,70 @@ static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
 }
 
 static const struct fault_info fault_info[] = {
-	{ do_bad,		SIGBUS,  0,		"ttbr address size fault"	},
-	{ do_bad,		SIGBUS,  0,		"level 1 address size fault"	},
-	{ do_bad,		SIGBUS,  0,		"level 2 address size fault"	},
-	{ do_bad,		SIGBUS,  0,		"level 3 address size fault"	},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"ttbr address size fault"	},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"level 1 address size fault"	},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"level 2 address size fault"	},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"level 3 address size fault"	},
 	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 0 translation fault"	},
 	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 1 translation fault"	},
 	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 2 translation fault"	},
 	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 3 translation fault"	},
-	{ do_bad,		SIGBUS,  0,		"unknown 8"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 8"			},
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 access flag fault"	},
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 access flag fault"	},
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 access flag fault"	},
-	{ do_bad,		SIGBUS,  0,		"unknown 12"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 12"			},
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 permission fault"	},
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 permission fault"	},
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 permission fault"	},
-	{ do_sea,		SIGBUS,  0,		"synchronous external abort"	},
-	{ do_bad,		SIGBUS,  0,		"unknown 17"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 18"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 19"			},
-	{ do_sea,		SIGBUS,  0,		"level 0 (translation table walk)"	},
-	{ do_sea,		SIGBUS,  0,		"level 1 (translation table walk)"	},
-	{ do_sea,		SIGBUS,  0,		"level 2 (translation table walk)"	},
-	{ do_sea,		SIGBUS,  0,		"level 3 (translation table walk)"	},
-	{ do_sea,		SIGBUS,  0,		"synchronous parity or ECC error" },	// Reserved when RAS is implemented
-	{ do_bad,		SIGBUS,  0,		"unknown 25"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 26"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 27"			},
-	{ do_sea,		SIGBUS,  0,		"level 0 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
-	{ do_sea,		SIGBUS,  0,		"level 1 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
-	{ do_sea,		SIGBUS,  0,		"level 2 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
-	{ do_sea,		SIGBUS,  0,		"level 3 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
-	{ do_bad,		SIGBUS,  0,		"unknown 32"			},
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"synchronous external abort"	},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 17"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 18"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 19"			},
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 0 (translation table walk)"	},
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 1 (translation table walk)"	},
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 2 (translation table walk)"	},
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 3 (translation table walk)"	},
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"synchronous parity or ECC error" },	// Reserved when RAS is implemented
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 25"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 26"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 27"			},
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 0 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 1 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 2 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
+	{ do_sea,		SIGBUS,  BUS_FIXME,	"level 3 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 32"			},
 	{ do_alignment_fault,	SIGBUS,  BUS_ADRALN,	"alignment fault"		},
-	{ do_bad,		SIGBUS,  0,		"unknown 34"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 35"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 36"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 37"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 38"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 39"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 40"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 41"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 42"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 43"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 44"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 45"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 46"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 47"			},
-	{ do_bad,		SIGBUS,  0,		"TLB conflict abort"		},
-	{ do_bad,		SIGBUS,  0,		"Unsupported atomic hardware update fault"	},
-	{ do_bad,		SIGBUS,  0,		"unknown 50"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 51"			},
-	{ do_bad,		SIGBUS,  0,		"implementation fault (lockdown abort)" },
-	{ do_bad,		SIGBUS,  0,		"implementation fault (unsupported exclusive)" },
-	{ do_bad,		SIGBUS,  0,		"unknown 54"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 55"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 56"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 57"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 58" 			},
-	{ do_bad,		SIGBUS,  0,		"unknown 59"			},
-	{ do_bad,		SIGBUS,  0,		"unknown 60"			},
-	{ do_bad,		SIGBUS,  0,		"section domain fault"		},
-	{ do_bad,		SIGBUS,  0,		"page domain fault"		},
-	{ do_bad,		SIGBUS,  0,		"unknown 63"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 34"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 35"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 36"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 37"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 38"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 39"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 40"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 41"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 42"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 43"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 44"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 45"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 46"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 47"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"TLB conflict abort"		},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"Unsupported atomic hardware update fault"	},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 50"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 51"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"implementation fault (lockdown abort)" },
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"implementation fault (unsupported exclusive)" },
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 54"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 55"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 56"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 57"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 58" 			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 59"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 60"			},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"section domain fault"		},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"page domain fault"		},
+	{ do_bad,		SIGBUS,  BUS_FIXME,	"unknown 63"			},
 };
 
 int handle_guest_sea(phys_addr_t addr, unsigned int esr)
@@ -707,6 +713,29 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
 	arm64_notify_die("", regs, &info, esr);
 }
 
+asmlinkage void __exception do_el0_irq_bp_hardening(void)
+{
+	/* PC has already been checked in entry.S */
+	arm64_apply_bp_hardening();
+}
+
+asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr,
+						   unsigned int esr,
+						   struct pt_regs *regs)
+{
+	/*
+	 * We've taken an instruction abort from userspace and not yet
+	 * re-enabled IRQs. If the address is a kernel address, apply
+	 * BP hardening prior to enabling IRQs and pre-emption.
+	 */
+	if (addr > TASK_SIZE)
+		arm64_apply_bp_hardening();
+
+	local_irq_enable();
+	do_mem_abort(addr, esr, regs);
+}
+
+
 asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
 					   unsigned int esr,
 					   struct pt_regs *regs)
@@ -714,6 +743,12 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
 	struct siginfo info;
 	struct task_struct *tsk = current;
 
+	if (user_mode(regs)) {
+		if (instruction_pointer(regs) > TASK_SIZE)
+			arm64_apply_bp_hardening();
+		local_irq_enable();
+	}
+
 	if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS))
 		pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n",
 				    tsk->comm, task_pid_nr(tsk),
@@ -739,11 +774,11 @@ static struct fault_info __refdata debug_fault_info[] = {
 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware breakpoint"	},
 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware single-step"	},
 	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware watchpoint"	},
-	{ do_bad,	SIGBUS,		0,		"unknown 3"		},
+	{ do_bad,	SIGBUS,		BUS_FIXME,	"unknown 3"		},
 	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
-	{ do_bad,	SIGTRAP,	0,		"aarch32 vector catch"	},
+	{ do_bad,	SIGTRAP,	TRAP_FIXME,	"aarch32 vector catch"	},
 	{ early_brk64,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
-	{ do_bad,	SIGBUS,		0,		"unknown 7"		},
+	{ do_bad,	SIGBUS,		BUS_FIXME,	"unknown 7"		},
 };
 
 void __init hook_debug_fault_code(int nr,
@@ -773,6 +808,9 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
 	if (interrupts_enabled(regs))
 		trace_hardirqs_off();
 
+	if (user_mode(regs) && instruction_pointer(regs) > TASK_SIZE)
+		arm64_apply_bp_hardening();
+
 	if (!inf->fn(addr, esr, regs)) {
 		rv = 1;
 	} else {
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index 6cb0fa92a651..ecc6818191df 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -54,14 +54,14 @@ static inline pgprot_t pte_pgprot(pte_t pte)
 static int find_num_contig(struct mm_struct *mm, unsigned long addr,
 			   pte_t *ptep, size_t *pgsize)
 {
-	pgd_t *pgd = pgd_offset(mm, addr);
-	pud_t *pud;
-	pmd_t *pmd;
+	pgd_t *pgdp = pgd_offset(mm, addr);
+	pud_t *pudp;
+	pmd_t *pmdp;
 
 	*pgsize = PAGE_SIZE;
-	pud = pud_offset(pgd, addr);
-	pmd = pmd_offset(pud, addr);
-	if ((pte_t *)pmd == ptep) {
+	pudp = pud_offset(pgdp, addr);
+	pmdp = pmd_offset(pudp, addr);
+	if ((pte_t *)pmdp == ptep) {
 		*pgsize = PMD_SIZE;
 		return CONT_PMDS;
 	}
@@ -181,11 +181,8 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
 
 	clear_flush(mm, addr, ptep, pgsize, ncontig);
 
-	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn) {
-		pr_debug("%s: set pte %p to 0x%llx\n", __func__, ptep,
-			 pte_val(pfn_pte(pfn, hugeprot)));
+	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
 		set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
-	}
 }
 
 void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
@@ -203,20 +200,20 @@ void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
 pte_t *huge_pte_alloc(struct mm_struct *mm,
 		      unsigned long addr, unsigned long sz)
 {
-	pgd_t *pgd;
-	pud_t *pud;
-	pte_t *pte = NULL;
-
-	pr_debug("%s: addr:0x%lx sz:0x%lx\n", __func__, addr, sz);
-	pgd = pgd_offset(mm, addr);
-	pud = pud_alloc(mm, pgd, addr);
-	if (!pud)
+	pgd_t *pgdp;
+	pud_t *pudp;
+	pmd_t *pmdp;
+	pte_t *ptep = NULL;
+
+	pgdp = pgd_offset(mm, addr);
+	pudp = pud_alloc(mm, pgdp, addr);
+	if (!pudp)
 		return NULL;
 
 	if (sz == PUD_SIZE) {
-		pte = (pte_t *)pud;
+		ptep = (pte_t *)pudp;
 	} else if (sz == (PAGE_SIZE * CONT_PTES)) {
-		pmd_t *pmd = pmd_alloc(mm, pud, addr);
+		pmdp = pmd_alloc(mm, pudp, addr);
 
 		WARN_ON(addr & (sz - 1));
 		/*
@@ -226,60 +223,55 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 		 * will be no pte_unmap() to correspond with this
 		 * pte_alloc_map().
 		 */
-		pte = pte_alloc_map(mm, pmd, addr);
+		ptep = pte_alloc_map(mm, pmdp, addr);
 	} else if (sz == PMD_SIZE) {
 		if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
-		    pud_none(*pud))
-			pte = huge_pmd_share(mm, addr, pud);
+		    pud_none(READ_ONCE(*pudp)))
+			ptep = huge_pmd_share(mm, addr, pudp);
 		else
-			pte = (pte_t *)pmd_alloc(mm, pud, addr);
+			ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
 	} else if (sz == (PMD_SIZE * CONT_PMDS)) {
-		pmd_t *pmd;
-
-		pmd = pmd_alloc(mm, pud, addr);
+		pmdp = pmd_alloc(mm, pudp, addr);
 		WARN_ON(addr & (sz - 1));
-		return (pte_t *)pmd;
+		return (pte_t *)pmdp;
 	}
 
-	pr_debug("%s: addr:0x%lx sz:0x%lx ret pte=%p/0x%llx\n", __func__, addr,
-	       sz, pte, pte_val(*pte));
-	return pte;
+	return ptep;
 }
 
 pte_t *huge_pte_offset(struct mm_struct *mm,
 		       unsigned long addr, unsigned long sz)
 {
-	pgd_t *pgd;
-	pud_t *pud;
-	pmd_t *pmd;
+	pgd_t *pgdp;
+	pud_t *pudp, pud;
+	pmd_t *pmdp, pmd;
 
-	pgd = pgd_offset(mm, addr);
-	pr_debug("%s: addr:0x%lx pgd:%p\n", __func__, addr, pgd);
-	if (!pgd_present(*pgd))
+	pgdp = pgd_offset(mm, addr);
+	if (!pgd_present(READ_ONCE(*pgdp)))
 		return NULL;
 
-	pud = pud_offset(pgd, addr);
-	if (sz != PUD_SIZE && pud_none(*pud))
+	pudp = pud_offset(pgdp, addr);
+	pud = READ_ONCE(*pudp);
+	if (sz != PUD_SIZE && pud_none(pud))
 		return NULL;
 	/* hugepage or swap? */
-	if (pud_huge(*pud) || !pud_present(*pud))
-		return (pte_t *)pud;
+	if (pud_huge(pud) || !pud_present(pud))
+		return (pte_t *)pudp;
 	/* table; check the next level */
 
 	if (sz == CONT_PMD_SIZE)
 		addr &= CONT_PMD_MASK;
 
-	pmd = pmd_offset(pud, addr);
+	pmdp = pmd_offset(pudp, addr);
+	pmd = READ_ONCE(*pmdp);
 	if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
-	    pmd_none(*pmd))
+	    pmd_none(pmd))
 		return NULL;
-	if (pmd_huge(*pmd) || !pmd_present(*pmd))
-		return (pte_t *)pmd;
+	if (pmd_huge(pmd) || !pmd_present(pmd))
+		return (pte_t *)pmdp;
 
-	if (sz == CONT_PTE_SIZE) {
-		pte_t *pte = pte_offset_kernel(pmd, (addr & CONT_PTE_MASK));
-		return pte;
-	}
+	if (sz == CONT_PTE_SIZE)
+		return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK));
 
 	return NULL;
 }
@@ -367,7 +359,7 @@ void huge_ptep_set_wrprotect(struct mm_struct *mm,
 	size_t pgsize;
 	pte_t pte;
 
-	if (!pte_cont(*ptep)) {
+	if (!pte_cont(READ_ONCE(*ptep))) {
 		ptep_set_wrprotect(mm, addr, ptep);
 		return;
 	}
@@ -391,7 +383,7 @@ void huge_ptep_clear_flush(struct vm_area_struct *vma,
 	size_t pgsize;
 	int ncontig;
 
-	if (!pte_cont(*ptep)) {
+	if (!pte_cont(READ_ONCE(*ptep))) {
 		ptep_clear_flush(vma, addr, ptep);
 		return;
 	}
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 00e7b900ca41..9f3c47acf8ff 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -217,7 +217,7 @@ static void __init reserve_elfcorehdr(void)
 }
 #endif /* CONFIG_CRASH_DUMP */
 /*
- * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It
+ * Return the maximum physical address for ZONE_DMA32 (DMA_BIT_MASK(32)). It
  * currently assumes that for memory starting above 4G, 32-bit devices will
  * use a DMA offset.
  */
@@ -233,8 +233,8 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
 {
 	unsigned long max_zone_pfns[MAX_NR_ZONES]  = {0};
 
-	if (IS_ENABLED(CONFIG_ZONE_DMA))
-		max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
+	if (IS_ENABLED(CONFIG_ZONE_DMA32))
+		max_zone_pfns[ZONE_DMA32] = PFN_DOWN(max_zone_dma_phys());
 	max_zone_pfns[ZONE_NORMAL] = max;
 
 	free_area_init_nodes(max_zone_pfns);
@@ -251,9 +251,9 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
 	memset(zone_size, 0, sizeof(zone_size));
 
 	/* 4GB maximum for 32-bit only capable devices */
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
 	max_dma = PFN_DOWN(arm64_dma_phys_limit);
-	zone_size[ZONE_DMA] = max_dma - min;
+	zone_size[ZONE_DMA32] = max_dma - min;
 #endif
 	zone_size[ZONE_NORMAL] = max - max_dma;
 
@@ -266,10 +266,10 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
 		if (start >= max)
 			continue;
 
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
 		if (start < max_dma) {
 			unsigned long dma_end = min(end, max_dma);
-			zhole_size[ZONE_DMA] -= dma_end - start;
+			zhole_size[ZONE_DMA32] -= dma_end - start;
 		}
 #endif
 		if (end > max_dma) {
@@ -366,6 +366,9 @@ void __init arm64_memblock_init(void)
 	/* Handle linux,usable-memory-range property */
 	fdt_enforce_memory_region();
 
+	/* Remove memory above our supported physical address size */
+	memblock_remove(1ULL << PHYS_MASK_SHIFT, ULLONG_MAX);
+
 	/*
 	 * Ensure that the linear region takes up exactly half of the kernel
 	 * virtual address space. This way, we can distinguish a linear address
@@ -467,7 +470,7 @@ void __init arm64_memblock_init(void)
 	early_init_fdt_scan_reserved_mem();
 
 	/* 4GB maximum for 32-bit only capable devices */
-	if (IS_ENABLED(CONFIG_ZONE_DMA))
+	if (IS_ENABLED(CONFIG_ZONE_DMA32))
 		arm64_dma_phys_limit = max_zone_dma_phys();
 	else
 		arm64_dma_phys_limit = PHYS_MASK + 1;
@@ -600,49 +603,6 @@ void __init mem_init(void)
 
 	mem_init_print_info(NULL);
 
-#define MLK(b, t) b, t, ((t) - (b)) >> 10
-#define MLM(b, t) b, t, ((t) - (b)) >> 20
-#define MLG(b, t) b, t, ((t) - (b)) >> 30
-#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
-
-	pr_notice("Virtual kernel memory layout:\n");
-#ifdef CONFIG_KASAN
-	pr_notice("    kasan   : 0x%16lx - 0x%16lx   (%6ld GB)\n",
-		MLG(KASAN_SHADOW_START, KASAN_SHADOW_END));
-#endif
-	pr_notice("    modules : 0x%16lx - 0x%16lx   (%6ld MB)\n",
-		MLM(MODULES_VADDR, MODULES_END));
-	pr_notice("    vmalloc : 0x%16lx - 0x%16lx   (%6ld GB)\n",
-		MLG(VMALLOC_START, VMALLOC_END));
-	pr_notice("      .text : 0x%p" " - 0x%p" "   (%6ld KB)\n",
-		MLK_ROUNDUP(_text, _etext));
-	pr_notice("    .rodata : 0x%p" " - 0x%p" "   (%6ld KB)\n",
-		MLK_ROUNDUP(__start_rodata, __init_begin));
-	pr_notice("      .init : 0x%p" " - 0x%p" "   (%6ld KB)\n",
-		MLK_ROUNDUP(__init_begin, __init_end));
-	pr_notice("      .data : 0x%p" " - 0x%p" "   (%6ld KB)\n",
-		MLK_ROUNDUP(_sdata, _edata));
-	pr_notice("       .bss : 0x%p" " - 0x%p" "   (%6ld KB)\n",
-		MLK_ROUNDUP(__bss_start, __bss_stop));
-	pr_notice("    fixed   : 0x%16lx - 0x%16lx   (%6ld KB)\n",
-		MLK(FIXADDR_START, FIXADDR_TOP));
-	pr_notice("    PCI I/O : 0x%16lx - 0x%16lx   (%6ld MB)\n",
-		MLM(PCI_IO_START, PCI_IO_END));
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-	pr_notice("    vmemmap : 0x%16lx - 0x%16lx   (%6ld GB maximum)\n",
-		MLG(VMEMMAP_START, VMEMMAP_START + VMEMMAP_SIZE));
-	pr_notice("              0x%16lx - 0x%16lx   (%6ld MB actual)\n",
-		MLM((unsigned long)phys_to_page(memblock_start_of_DRAM()),
-		    (unsigned long)virt_to_page(high_memory)));
-#endif
-	pr_notice("    memory  : 0x%16lx - 0x%16lx   (%6ld MB)\n",
-		MLM(__phys_to_virt(memblock_start_of_DRAM()),
-		    (unsigned long)high_memory));
-
-#undef MLK
-#undef MLM
-#undef MLK_ROUNDUP
-
 	/*
 	 * Check boundaries twice: Some fundamental inconsistencies can be
 	 * detected at build time already.
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index acba49fb5aac..dabfc1ecda3d 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -44,98 +44,99 @@ static phys_addr_t __init kasan_alloc_zeroed_page(int node)
 	return __pa(p);
 }
 
-static pte_t *__init kasan_pte_offset(pmd_t *pmd, unsigned long addr, int node,
+static pte_t *__init kasan_pte_offset(pmd_t *pmdp, unsigned long addr, int node,
 				      bool early)
 {
-	if (pmd_none(*pmd)) {
+	if (pmd_none(READ_ONCE(*pmdp))) {
 		phys_addr_t pte_phys = early ? __pa_symbol(kasan_zero_pte)
 					     : kasan_alloc_zeroed_page(node);
-		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
+		__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
 	}
 
-	return early ? pte_offset_kimg(pmd, addr)
-		     : pte_offset_kernel(pmd, addr);
+	return early ? pte_offset_kimg(pmdp, addr)
+		     : pte_offset_kernel(pmdp, addr);
 }
 
-static pmd_t *__init kasan_pmd_offset(pud_t *pud, unsigned long addr, int node,
+static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,
 				      bool early)
 {
-	if (pud_none(*pud)) {
+	if (pud_none(READ_ONCE(*pudp))) {
 		phys_addr_t pmd_phys = early ? __pa_symbol(kasan_zero_pmd)
 					     : kasan_alloc_zeroed_page(node);
-		__pud_populate(pud, pmd_phys, PMD_TYPE_TABLE);
+		__pud_populate(pudp, pmd_phys, PMD_TYPE_TABLE);
 	}
 
-	return early ? pmd_offset_kimg(pud, addr) : pmd_offset(pud, addr);
+	return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);
 }
 
-static pud_t *__init kasan_pud_offset(pgd_t *pgd, unsigned long addr, int node,
+static pud_t *__init kasan_pud_offset(pgd_t *pgdp, unsigned long addr, int node,
 				      bool early)
 {
-	if (pgd_none(*pgd)) {
+	if (pgd_none(READ_ONCE(*pgdp))) {
 		phys_addr_t pud_phys = early ? __pa_symbol(kasan_zero_pud)
 					     : kasan_alloc_zeroed_page(node);
-		__pgd_populate(pgd, pud_phys, PMD_TYPE_TABLE);
+		__pgd_populate(pgdp, pud_phys, PMD_TYPE_TABLE);
 	}
 
-	return early ? pud_offset_kimg(pgd, addr) : pud_offset(pgd, addr);
+	return early ? pud_offset_kimg(pgdp, addr) : pud_offset(pgdp, addr);
 }
 
-static void __init kasan_pte_populate(pmd_t *pmd, unsigned long addr,
+static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
 				      unsigned long end, int node, bool early)
 {
 	unsigned long next;
-	pte_t *pte = kasan_pte_offset(pmd, addr, node, early);
+	pte_t *ptep = kasan_pte_offset(pmdp, addr, node, early);
 
 	do {
 		phys_addr_t page_phys = early ? __pa_symbol(kasan_zero_page)
 					      : kasan_alloc_zeroed_page(node);
 		next = addr + PAGE_SIZE;
-		set_pte(pte, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
-	} while (pte++, addr = next, addr != end && pte_none(*pte));
+		set_pte(ptep, pfn_pte(__phys_to_pfn(page_phys), PAGE_KERNEL));
+	} while (ptep++, addr = next, addr != end && pte_none(READ_ONCE(*ptep)));
 }
 
-static void __init kasan_pmd_populate(pud_t *pud, unsigned long addr,
+static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
 				      unsigned long end, int node, bool early)
 {
 	unsigned long next;
-	pmd_t *pmd = kasan_pmd_offset(pud, addr, node, early);
+	pmd_t *pmdp = kasan_pmd_offset(pudp, addr, node, early);
 
 	do {
 		next = pmd_addr_end(addr, end);
-		kasan_pte_populate(pmd, addr, next, node, early);
-	} while (pmd++, addr = next, addr != end && pmd_none(*pmd));
+		kasan_pte_populate(pmdp, addr, next, node, early);
+	} while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp)));
 }
 
-static void __init kasan_pud_populate(pgd_t *pgd, unsigned long addr,
+static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr,
 				      unsigned long end, int node, bool early)
 {
 	unsigned long next;
-	pud_t *pud = kasan_pud_offset(pgd, addr, node, early);
+	pud_t *pudp = kasan_pud_offset(pgdp, addr, node, early);
 
 	do {
 		next = pud_addr_end(addr, end);
-		kasan_pmd_populate(pud, addr, next, node, early);
-	} while (pud++, addr = next, addr != end && pud_none(*pud));
+		kasan_pmd_populate(pudp, addr, next, node, early);
+	} while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp)));
 }
 
 static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
 				      int node, bool early)
 {
 	unsigned long next;
-	pgd_t *pgd;
+	pgd_t *pgdp;
 
-	pgd = pgd_offset_k(addr);
+	pgdp = pgd_offset_k(addr);
 	do {
 		next = pgd_addr_end(addr, end);
-		kasan_pud_populate(pgd, addr, next, node, early);
-	} while (pgd++, addr = next, addr != end);
+		kasan_pud_populate(pgdp, addr, next, node, early);
+	} while (pgdp++, addr = next, addr != end);
 }
 
 /* The early shadow maps everything to a single page of zeroes */
 asmlinkage void __init kasan_early_init(void)
 {
-	BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61));
+	BUILD_BUG_ON(KASAN_SHADOW_OFFSET !=
+		KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT)));
 	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE));
 	BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
 	kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, NUMA_NO_NODE,
@@ -154,14 +155,14 @@ static void __init kasan_map_populate(unsigned long start, unsigned long end,
  */
 void __init kasan_copy_shadow(pgd_t *pgdir)
 {
-	pgd_t *pgd, *pgd_new, *pgd_end;
+	pgd_t *pgdp, *pgdp_new, *pgdp_end;
 
-	pgd = pgd_offset_k(KASAN_SHADOW_START);
-	pgd_end = pgd_offset_k(KASAN_SHADOW_END);
-	pgd_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START);
+	pgdp = pgd_offset_k(KASAN_SHADOW_START);
+	pgdp_end = pgd_offset_k(KASAN_SHADOW_END);
+	pgdp_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START);
 	do {
-		set_pgd(pgd_new, *pgd);
-	} while (pgd++, pgd_new++, pgd != pgd_end);
+		set_pgd(pgdp_new, READ_ONCE(*pgdp));
+	} while (pgdp++, pgdp_new++, pgdp != pgdp_end);
 }
 
 static void __init clear_pgds(unsigned long start,
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 267d2b79d52d..2dbb2c9f1ec1 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -50,6 +50,7 @@
 #define NO_CONT_MAPPINGS	BIT(1)
 
 u64 idmap_t0sz = TCR_T0SZ(VA_BITS);
+u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
 
 u64 kimage_voffset __ro_after_init;
 EXPORT_SYMBOL(kimage_voffset);
@@ -107,7 +108,7 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
 	 * The following mapping attributes may be updated in live
 	 * kernel mappings without the need for break-before-make.
 	 */
-	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE;
+	static const pteval_t mask = PTE_PXN | PTE_RDONLY | PTE_WRITE | PTE_NG;
 
 	/* creating or taking down mappings is always safe */
 	if (old == 0 || new == 0)
@@ -117,48 +118,55 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
 	if ((old | new) & PTE_CONT)
 		return false;
 
+	/* Transitioning from Non-Global to Global is unsafe */
+	if (old & ~new & PTE_NG)
+		return false;
+
 	return ((old ^ new) & ~mask) == 0;
 }
 
-static void init_pte(pmd_t *pmd, unsigned long addr, unsigned long end,
+static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
 		     phys_addr_t phys, pgprot_t prot)
 {
-	pte_t *pte;
+	pte_t *ptep;
 
-	pte = pte_set_fixmap_offset(pmd, addr);
+	ptep = pte_set_fixmap_offset(pmdp, addr);
 	do {
-		pte_t old_pte = *pte;
+		pte_t old_pte = READ_ONCE(*ptep);
 
-		set_pte(pte, pfn_pte(__phys_to_pfn(phys), prot));
+		set_pte(ptep, pfn_pte(__phys_to_pfn(phys), prot));
 
 		/*
 		 * After the PTE entry has been populated once, we
 		 * only allow updates to the permission attributes.
 		 */
-		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte), pte_val(*pte)));
+		BUG_ON(!pgattr_change_is_safe(pte_val(old_pte),
+					      READ_ONCE(pte_val(*ptep))));
 
 		phys += PAGE_SIZE;
-	} while (pte++, addr += PAGE_SIZE, addr != end);
+	} while (ptep++, addr += PAGE_SIZE, addr != end);
 
 	pte_clear_fixmap();
 }
 
-static void alloc_init_cont_pte(pmd_t *pmd, unsigned long addr,
+static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
 				unsigned long end, phys_addr_t phys,
 				pgprot_t prot,
 				phys_addr_t (*pgtable_alloc)(void),
 				int flags)
 {
 	unsigned long next;
+	pmd_t pmd = READ_ONCE(*pmdp);
 
-	BUG_ON(pmd_sect(*pmd));
-	if (pmd_none(*pmd)) {
+	BUG_ON(pmd_sect(pmd));
+	if (pmd_none(pmd)) {
 		phys_addr_t pte_phys;
 		BUG_ON(!pgtable_alloc);
 		pte_phys = pgtable_alloc();
-		__pmd_populate(pmd, pte_phys, PMD_TYPE_TABLE);
+		__pmd_populate(pmdp, pte_phys, PMD_TYPE_TABLE);
+		pmd = READ_ONCE(*pmdp);
 	}
-	BUG_ON(pmd_bad(*pmd));
+	BUG_ON(pmd_bad(pmd));
 
 	do {
 		pgprot_t __prot = prot;
@@ -170,67 +178,69 @@ static void alloc_init_cont_pte(pmd_t *pmd, unsigned long addr,
 		    (flags & NO_CONT_MAPPINGS) == 0)
 			__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
 
-		init_pte(pmd, addr, next, phys, __prot);
+		init_pte(pmdp, addr, next, phys, __prot);
 
 		phys += next - addr;
 	} while (addr = next, addr != end);
 }
 
-static void init_pmd(pud_t *pud, unsigned long addr, unsigned long end,
+static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
 		     phys_addr_t phys, pgprot_t prot,
 		     phys_addr_t (*pgtable_alloc)(void), int flags)
 {
 	unsigned long next;
-	pmd_t *pmd;
+	pmd_t *pmdp;
 
-	pmd = pmd_set_fixmap_offset(pud, addr);
+	pmdp = pmd_set_fixmap_offset(pudp, addr);
 	do {
-		pmd_t old_pmd = *pmd;
+		pmd_t old_pmd = READ_ONCE(*pmdp);
 
 		next = pmd_addr_end(addr, end);
 
 		/* try section mapping first */
 		if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
 		    (flags & NO_BLOCK_MAPPINGS) == 0) {
-			pmd_set_huge(pmd, phys, prot);
+			pmd_set_huge(pmdp, phys, prot);
 
 			/*
 			 * After the PMD entry has been populated once, we
 			 * only allow updates to the permission attributes.
 			 */
 			BUG_ON(!pgattr_change_is_safe(pmd_val(old_pmd),
-						      pmd_val(*pmd)));
+						      READ_ONCE(pmd_val(*pmdp))));
 		} else {
-			alloc_init_cont_pte(pmd, addr, next, phys, prot,
+			alloc_init_cont_pte(pmdp, addr, next, phys, prot,
 					    pgtable_alloc, flags);
 
 			BUG_ON(pmd_val(old_pmd) != 0 &&
-			       pmd_val(old_pmd) != pmd_val(*pmd));
+			       pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
 		}
 		phys += next - addr;
-	} while (pmd++, addr = next, addr != end);
+	} while (pmdp++, addr = next, addr != end);
 
 	pmd_clear_fixmap();
 }
 
-static void alloc_init_cont_pmd(pud_t *pud, unsigned long addr,
+static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
 				unsigned long end, phys_addr_t phys,
 				pgprot_t prot,
 				phys_addr_t (*pgtable_alloc)(void), int flags)
 {
 	unsigned long next;
+	pud_t pud = READ_ONCE(*pudp);
 
 	/*
 	 * Check for initial section mappings in the pgd/pud.
 	 */
-	BUG_ON(pud_sect(*pud));
-	if (pud_none(*pud)) {
+	BUG_ON(pud_sect(pud));
+	if (pud_none(pud)) {
 		phys_addr_t pmd_phys;
 		BUG_ON(!pgtable_alloc);
 		pmd_phys = pgtable_alloc();
-		__pud_populate(pud, pmd_phys, PUD_TYPE_TABLE);
+		__pud_populate(pudp, pmd_phys, PUD_TYPE_TABLE);
+		pud = READ_ONCE(*pudp);
 	}
-	BUG_ON(pud_bad(*pud));
+	BUG_ON(pud_bad(pud));
 
 	do {
 		pgprot_t __prot = prot;
@@ -242,7 +252,7 @@ static void alloc_init_cont_pmd(pud_t *pud, unsigned long addr,
 		    (flags & NO_CONT_MAPPINGS) == 0)
 			__prot = __pgprot(pgprot_val(prot) | PTE_CONT);
 
-		init_pmd(pud, addr, next, phys, __prot, pgtable_alloc, flags);
+		init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags);
 
 		phys += next - addr;
 	} while (addr = next, addr != end);
@@ -260,25 +270,27 @@ static inline bool use_1G_block(unsigned long addr, unsigned long next,
 	return true;
 }
 
-static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
-				  phys_addr_t phys, pgprot_t prot,
-				  phys_addr_t (*pgtable_alloc)(void),
-				  int flags)
+static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
+			   phys_addr_t phys, pgprot_t prot,
+			   phys_addr_t (*pgtable_alloc)(void),
+			   int flags)
 {
-	pud_t *pud;
 	unsigned long next;
+	pud_t *pudp;
+	pgd_t pgd = READ_ONCE(*pgdp);
 
-	if (pgd_none(*pgd)) {
+	if (pgd_none(pgd)) {
 		phys_addr_t pud_phys;
 		BUG_ON(!pgtable_alloc);
 		pud_phys = pgtable_alloc();
-		__pgd_populate(pgd, pud_phys, PUD_TYPE_TABLE);
+		__pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE);
+		pgd = READ_ONCE(*pgdp);
 	}
-	BUG_ON(pgd_bad(*pgd));
+	BUG_ON(pgd_bad(pgd));
 
-	pud = pud_set_fixmap_offset(pgd, addr);
+	pudp = pud_set_fixmap_offset(pgdp, addr);
 	do {
-		pud_t old_pud = *pud;
+		pud_t old_pud = READ_ONCE(*pudp);
 
 		next = pud_addr_end(addr, end);
 
@@ -287,23 +299,23 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
 		 */
 		if (use_1G_block(addr, next, phys) &&
 		    (flags & NO_BLOCK_MAPPINGS) == 0) {
-			pud_set_huge(pud, phys, prot);
+			pud_set_huge(pudp, phys, prot);
 
 			/*
 			 * After the PUD entry has been populated once, we
 			 * only allow updates to the permission attributes.
 			 */
 			BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
-						      pud_val(*pud)));
+						      READ_ONCE(pud_val(*pudp))));
 		} else {
-			alloc_init_cont_pmd(pud, addr, next, phys, prot,
+			alloc_init_cont_pmd(pudp, addr, next, phys, prot,
 					    pgtable_alloc, flags);
 
 			BUG_ON(pud_val(old_pud) != 0 &&
-			       pud_val(old_pud) != pud_val(*pud));
+			       pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
 		}
 		phys += next - addr;
-	} while (pud++, addr = next, addr != end);
+	} while (pudp++, addr = next, addr != end);
 
 	pud_clear_fixmap();
 }
@@ -315,7 +327,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
 				 int flags)
 {
 	unsigned long addr, length, end, next;
-	pgd_t *pgd = pgd_offset_raw(pgdir, virt);
+	pgd_t *pgdp = pgd_offset_raw(pgdir, virt);
 
 	/*
 	 * If the virtual and physical address don't have the same offset
@@ -331,10 +343,10 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys,
 	end = addr + length;
 	do {
 		next = pgd_addr_end(addr, end);
-		alloc_init_pud(pgd, addr, next, phys, prot, pgtable_alloc,
+		alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,
 			       flags);
 		phys += next - addr;
-	} while (pgd++, addr = next, addr != end);
+	} while (pgdp++, addr = next, addr != end);
 }
 
 static phys_addr_t pgd_pgtable_alloc(void)
@@ -396,10 +408,10 @@ static void update_mapping_prot(phys_addr_t phys, unsigned long virt,
 	flush_tlb_kernel_range(virt, virt + size);
 }
 
-static void __init __map_memblock(pgd_t *pgd, phys_addr_t start,
+static void __init __map_memblock(pgd_t *pgdp, phys_addr_t start,
 				  phys_addr_t end, pgprot_t prot, int flags)
 {
-	__create_pgd_mapping(pgd, start, __phys_to_virt(start), end - start,
+	__create_pgd_mapping(pgdp, start, __phys_to_virt(start), end - start,
 			     prot, early_pgtable_alloc, flags);
 }
 
@@ -413,7 +425,7 @@ void __init mark_linear_text_alias_ro(void)
 			    PAGE_KERNEL_RO);
 }
 
-static void __init map_mem(pgd_t *pgd)
+static void __init map_mem(pgd_t *pgdp)
 {
 	phys_addr_t kernel_start = __pa_symbol(_text);
 	phys_addr_t kernel_end = __pa_symbol(__init_begin);
@@ -446,7 +458,7 @@ static void __init map_mem(pgd_t *pgd)
 		if (memblock_is_nomap(reg))
 			continue;
 
-		__map_memblock(pgd, start, end, PAGE_KERNEL, flags);
+		__map_memblock(pgdp, start, end, PAGE_KERNEL, flags);
 	}
 
 	/*
@@ -459,7 +471,7 @@ static void __init map_mem(pgd_t *pgd)
 	 * Note that contiguous mappings cannot be remapped in this way,
 	 * so we should avoid them here.
 	 */
-	__map_memblock(pgd, kernel_start, kernel_end,
+	__map_memblock(pgdp, kernel_start, kernel_end,
 		       PAGE_KERNEL, NO_CONT_MAPPINGS);
 	memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
 
@@ -470,7 +482,7 @@ static void __init map_mem(pgd_t *pgd)
 	 * through /sys/kernel/kexec_crash_size interface.
 	 */
 	if (crashk_res.end) {
-		__map_memblock(pgd, crashk_res.start, crashk_res.end + 1,
+		__map_memblock(pgdp, crashk_res.start, crashk_res.end + 1,
 			       PAGE_KERNEL,
 			       NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
 		memblock_clear_nomap(crashk_res.start,
@@ -494,7 +506,7 @@ void mark_rodata_ro(void)
 	debug_checkwx();
 }
 
-static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
+static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end,
 				      pgprot_t prot, struct vm_struct *vma,
 				      int flags, unsigned long vm_flags)
 {
@@ -504,7 +516,7 @@ static void __init map_kernel_segment(pgd_t *pgd, void *va_start, void *va_end,
 	BUG_ON(!PAGE_ALIGNED(pa_start));
 	BUG_ON(!PAGE_ALIGNED(size));
 
-	__create_pgd_mapping(pgd, pa_start, (unsigned long)va_start, size, prot,
+	__create_pgd_mapping(pgdp, pa_start, (unsigned long)va_start, size, prot,
 			     early_pgtable_alloc, flags);
 
 	if (!(vm_flags & VM_NO_GUARD))
@@ -525,10 +537,39 @@ static int __init parse_rodata(char *arg)
 }
 early_param("rodata", parse_rodata);
 
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+static int __init map_entry_trampoline(void)
+{
+	pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
+	phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
+
+	/* The trampoline is always mapped and can therefore be global */
+	pgprot_val(prot) &= ~PTE_NG;
+
+	/* Map only the text into the trampoline page table */
+	memset(tramp_pg_dir, 0, PGD_SIZE);
+	__create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
+			     prot, pgd_pgtable_alloc, 0);
+
+	/* Map both the text and data into the kernel page table */
+	__set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
+	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
+		extern char __entry_tramp_data_start[];
+
+		__set_fixmap(FIX_ENTRY_TRAMP_DATA,
+			     __pa_symbol(__entry_tramp_data_start),
+			     PAGE_KERNEL_RO);
+	}
+
+	return 0;
+}
+core_initcall(map_entry_trampoline);
+#endif
+
 /*
  * Create fine-grained mappings for the kernel.
  */
-static void __init map_kernel(pgd_t *pgd)
+static void __init map_kernel(pgd_t *pgdp)
 {
 	static struct vm_struct vmlinux_text, vmlinux_rodata, vmlinux_inittext,
 				vmlinux_initdata, vmlinux_data;
@@ -544,24 +585,24 @@ static void __init map_kernel(pgd_t *pgd)
 	 * Only rodata will be remapped with different permissions later on,
 	 * all other segments are allowed to use contiguous mappings.
 	 */
-	map_kernel_segment(pgd, _text, _etext, text_prot, &vmlinux_text, 0,
+	map_kernel_segment(pgdp, _text, _etext, text_prot, &vmlinux_text, 0,
 			   VM_NO_GUARD);
-	map_kernel_segment(pgd, __start_rodata, __inittext_begin, PAGE_KERNEL,
+	map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL,
 			   &vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD);
-	map_kernel_segment(pgd, __inittext_begin, __inittext_end, text_prot,
+	map_kernel_segment(pgdp, __inittext_begin, __inittext_end, text_prot,
 			   &vmlinux_inittext, 0, VM_NO_GUARD);
-	map_kernel_segment(pgd, __initdata_begin, __initdata_end, PAGE_KERNEL,
+	map_kernel_segment(pgdp, __initdata_begin, __initdata_end, PAGE_KERNEL,
 			   &vmlinux_initdata, 0, VM_NO_GUARD);
-	map_kernel_segment(pgd, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0);
+	map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0);
 
-	if (!pgd_val(*pgd_offset_raw(pgd, FIXADDR_START))) {
+	if (!READ_ONCE(pgd_val(*pgd_offset_raw(pgdp, FIXADDR_START)))) {
 		/*
 		 * The fixmap falls in a separate pgd to the kernel, and doesn't
 		 * live in the carveout for the swapper_pg_dir. We can simply
 		 * re-use the existing dir for the fixmap.
 		 */
-		set_pgd(pgd_offset_raw(pgd, FIXADDR_START),
-			*pgd_offset_k(FIXADDR_START));
+		set_pgd(pgd_offset_raw(pgdp, FIXADDR_START),
+			READ_ONCE(*pgd_offset_k(FIXADDR_START)));
 	} else if (CONFIG_PGTABLE_LEVELS > 3) {
 		/*
 		 * The fixmap shares its top level pgd entry with the kernel
@@ -570,14 +611,15 @@ static void __init map_kernel(pgd_t *pgd)
 		 * entry instead.
 		 */
 		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
-		set_pud(pud_set_fixmap_offset(pgd, FIXADDR_START),
-			__pud(__pa_symbol(bm_pmd) | PUD_TYPE_TABLE));
+		pud_populate(&init_mm,
+			     pud_set_fixmap_offset(pgdp, FIXADDR_START),
+			     lm_alias(bm_pmd));
 		pud_clear_fixmap();
 	} else {
 		BUG();
 	}
 
-	kasan_copy_shadow(pgd);
+	kasan_copy_shadow(pgdp);
 }
 
 /*
@@ -587,10 +629,10 @@ static void __init map_kernel(pgd_t *pgd)
 void __init paging_init(void)
 {
 	phys_addr_t pgd_phys = early_pgtable_alloc();
-	pgd_t *pgd = pgd_set_fixmap(pgd_phys);
+	pgd_t *pgdp = pgd_set_fixmap(pgd_phys);
 
-	map_kernel(pgd);
-	map_mem(pgd);
+	map_kernel(pgdp);
+	map_mem(pgdp);
 
 	/*
 	 * We want to reuse the original swapper_pg_dir so we don't have to
@@ -601,7 +643,7 @@ void __init paging_init(void)
 	 * To do this we need to go via a temporary pgd.
 	 */
 	cpu_replace_ttbr1(__va(pgd_phys));
-	memcpy(swapper_pg_dir, pgd, PGD_SIZE);
+	memcpy(swapper_pg_dir, pgdp, PGD_SIZE);
 	cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
 
 	pgd_clear_fixmap();
@@ -612,7 +654,8 @@ void __init paging_init(void)
 	 * allocated with it.
 	 */
 	memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE,
-		      SWAPPER_DIR_SIZE - PAGE_SIZE);
+		      __pa_symbol(swapper_pg_end) - __pa_symbol(swapper_pg_dir)
+		      - PAGE_SIZE);
 }
 
 /*
@@ -620,101 +663,109 @@ void __init paging_init(void)
  */
 int kern_addr_valid(unsigned long addr)
 {
-	pgd_t *pgd;
-	pud_t *pud;
-	pmd_t *pmd;
-	pte_t *pte;
+	pgd_t *pgdp;
+	pud_t *pudp, pud;
+	pmd_t *pmdp, pmd;
+	pte_t *ptep, pte;
 
 	if ((((long)addr) >> VA_BITS) != -1UL)
 		return 0;
 
-	pgd = pgd_offset_k(addr);
-	if (pgd_none(*pgd))
+	pgdp = pgd_offset_k(addr);
+	if (pgd_none(READ_ONCE(*pgdp)))
 		return 0;
 
-	pud = pud_offset(pgd, addr);
-	if (pud_none(*pud))
+	pudp = pud_offset(pgdp, addr);
+	pud = READ_ONCE(*pudp);
+	if (pud_none(pud))
 		return 0;
 
-	if (pud_sect(*pud))
-		return pfn_valid(pud_pfn(*pud));
+	if (pud_sect(pud))
+		return pfn_valid(pud_pfn(pud));
 
-	pmd = pmd_offset(pud, addr);
-	if (pmd_none(*pmd))
+	pmdp = pmd_offset(pudp, addr);
+	pmd = READ_ONCE(*pmdp);
+	if (pmd_none(pmd))
 		return 0;
 
-	if (pmd_sect(*pmd))
-		return pfn_valid(pmd_pfn(*pmd));
+	if (pmd_sect(pmd))
+		return pfn_valid(pmd_pfn(pmd));
 
-	pte = pte_offset_kernel(pmd, addr);
-	if (pte_none(*pte))
+	ptep = pte_offset_kernel(pmdp, addr);
+	pte = READ_ONCE(*ptep);
+	if (pte_none(pte))
 		return 0;
 
-	return pfn_valid(pte_pfn(*pte));
+	return pfn_valid(pte_pfn(pte));
 }
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 #if !ARM64_SWAPPER_USES_SECTION_MAPS
-int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
+int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
+		struct vmem_altmap *altmap)
 {
 	return vmemmap_populate_basepages(start, end, node);
 }
 #else	/* !ARM64_SWAPPER_USES_SECTION_MAPS */
-int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
+int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
+		struct vmem_altmap *altmap)
 {
 	unsigned long addr = start;
 	unsigned long next;
-	pgd_t *pgd;
-	pud_t *pud;
-	pmd_t *pmd;
+	pgd_t *pgdp;
+	pud_t *pudp;
+	pmd_t *pmdp;
 
 	do {
 		next = pmd_addr_end(addr, end);
 
-		pgd = vmemmap_pgd_populate(addr, node);
-		if (!pgd)
+		pgdp = vmemmap_pgd_populate(addr, node);
+		if (!pgdp)
 			return -ENOMEM;
 
-		pud = vmemmap_pud_populate(pgd, addr, node);
-		if (!pud)
+		pudp = vmemmap_pud_populate(pgdp, addr, node);
+		if (!pudp)
 			return -ENOMEM;
 
-		pmd = pmd_offset(pud, addr);
-		if (pmd_none(*pmd)) {
+		pmdp = pmd_offset(pudp, addr);
+		if (pmd_none(READ_ONCE(*pmdp))) {
 			void *p = NULL;
 
 			p = vmemmap_alloc_block_buf(PMD_SIZE, node);
 			if (!p)
 				return -ENOMEM;
 
-			set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL));
+			pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));
 		} else
-			vmemmap_verify((pte_t *)pmd, node, addr, next);
+			vmemmap_verify((pte_t *)pmdp, node, addr, next);
 	} while (addr = next, addr != end);
 
 	return 0;
 }
 #endif	/* CONFIG_ARM64_64K_PAGES */
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+		struct vmem_altmap *altmap)
 {
 }
 #endif	/* CONFIG_SPARSEMEM_VMEMMAP */
 
 static inline pud_t * fixmap_pud(unsigned long addr)
 {
-	pgd_t *pgd = pgd_offset_k(addr);
+	pgd_t *pgdp = pgd_offset_k(addr);
+	pgd_t pgd = READ_ONCE(*pgdp);
 
-	BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd));
+	BUG_ON(pgd_none(pgd) || pgd_bad(pgd));
 
-	return pud_offset_kimg(pgd, addr);
+	return pud_offset_kimg(pgdp, addr);
 }
 
 static inline pmd_t * fixmap_pmd(unsigned long addr)
 {
-	pud_t *pud = fixmap_pud(addr);
+	pud_t *pudp = fixmap_pud(addr);
+	pud_t pud = READ_ONCE(*pudp);
 
-	BUG_ON(pud_none(*pud) || pud_bad(*pud));
+	BUG_ON(pud_none(pud) || pud_bad(pud));
 
-	return pmd_offset_kimg(pud, addr);
+	return pmd_offset_kimg(pudp, addr);
 }
 
 static inline pte_t * fixmap_pte(unsigned long addr)
@@ -730,30 +781,31 @@ static inline pte_t * fixmap_pte(unsigned long addr)
  */
 void __init early_fixmap_init(void)
 {
-	pgd_t *pgd;
-	pud_t *pud;
-	pmd_t *pmd;
+	pgd_t *pgdp, pgd;
+	pud_t *pudp;
+	pmd_t *pmdp;
 	unsigned long addr = FIXADDR_START;
 
-	pgd = pgd_offset_k(addr);
+	pgdp = pgd_offset_k(addr);
+	pgd = READ_ONCE(*pgdp);
 	if (CONFIG_PGTABLE_LEVELS > 3 &&
-	    !(pgd_none(*pgd) || pgd_page_paddr(*pgd) == __pa_symbol(bm_pud))) {
+	    !(pgd_none(pgd) || pgd_page_paddr(pgd) == __pa_symbol(bm_pud))) {
 		/*
 		 * We only end up here if the kernel mapping and the fixmap
 		 * share the top level pgd entry, which should only happen on
 		 * 16k/4 levels configurations.
 		 */
 		BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
-		pud = pud_offset_kimg(pgd, addr);
+		pudp = pud_offset_kimg(pgdp, addr);
 	} else {
-		if (pgd_none(*pgd))
-			__pgd_populate(pgd, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
-		pud = fixmap_pud(addr);
+		if (pgd_none(pgd))
+			__pgd_populate(pgdp, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
+		pudp = fixmap_pud(addr);
 	}
-	if (pud_none(*pud))
-		__pud_populate(pud, __pa_symbol(bm_pmd), PMD_TYPE_TABLE);
-	pmd = fixmap_pmd(addr);
-	__pmd_populate(pmd, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
+	if (pud_none(READ_ONCE(*pudp)))
+		__pud_populate(pudp, __pa_symbol(bm_pmd), PMD_TYPE_TABLE);
+	pmdp = fixmap_pmd(addr);
+	__pmd_populate(pmdp, __pa_symbol(bm_pte), PMD_TYPE_TABLE);
 
 	/*
 	 * The boot-ioremap range spans multiple pmds, for which
@@ -762,11 +814,11 @@ void __init early_fixmap_init(void)
 	BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
 		     != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
 
-	if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
-	     || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
+	if ((pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)))
+	     || pmdp != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) {
 		WARN_ON(1);
-		pr_warn("pmd %p != %p, %p\n",
-			pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
+		pr_warn("pmdp %p != %p, %p\n",
+			pmdp, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)),
 			fixmap_pmd(fix_to_virt(FIX_BTMAP_END)));
 		pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
 			fix_to_virt(FIX_BTMAP_BEGIN));
@@ -786,16 +838,16 @@ void __set_fixmap(enum fixed_addresses idx,
 			       phys_addr_t phys, pgprot_t flags)
 {
 	unsigned long addr = __fix_to_virt(idx);
-	pte_t *pte;
+	pte_t *ptep;
 
 	BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses);
 
-	pte = fixmap_pte(addr);
+	ptep = fixmap_pte(addr);
 
 	if (pgprot_val(flags)) {
-		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
+		set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags));
 	} else {
-		pte_clear(&init_mm, addr, pte);
+		pte_clear(&init_mm, addr, ptep);
 		flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
 	}
 }
@@ -877,32 +929,56 @@ int __init arch_ioremap_pmd_supported(void)
 	return 1;
 }
 
-int pud_set_huge(pud_t *pud, phys_addr_t phys, pgprot_t prot)
+int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
 {
+	pgprot_t sect_prot = __pgprot(PUD_TYPE_SECT |
+					pgprot_val(mk_sect_prot(prot)));
+
+	/* ioremap_page_range doesn't honour BBM */
+	if (pud_present(READ_ONCE(*pudp)))
+		return 0;
+
 	BUG_ON(phys & ~PUD_MASK);
-	set_pud(pud, __pud(phys | PUD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
+	set_pud(pudp, pfn_pud(__phys_to_pfn(phys), sect_prot));
 	return 1;
 }
 
-int pmd_set_huge(pmd_t *pmd, phys_addr_t phys, pgprot_t prot)
+int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
 {
+	pgprot_t sect_prot = __pgprot(PMD_TYPE_SECT |
+					pgprot_val(mk_sect_prot(prot)));
+
+	/* ioremap_page_range doesn't honour BBM */
+	if (pmd_present(READ_ONCE(*pmdp)))
+		return 0;
+
 	BUG_ON(phys & ~PMD_MASK);
-	set_pmd(pmd, __pmd(phys | PMD_TYPE_SECT | pgprot_val(mk_sect_prot(prot))));
+	set_pmd(pmdp, pfn_pmd(__phys_to_pfn(phys), sect_prot));
 	return 1;
 }
 
-int pud_clear_huge(pud_t *pud)
+int pud_clear_huge(pud_t *pudp)
 {
-	if (!pud_sect(*pud))
+	if (!pud_sect(READ_ONCE(*pudp)))
 		return 0;
-	pud_clear(pud);
+	pud_clear(pudp);
 	return 1;
 }
 
-int pmd_clear_huge(pmd_t *pmd)
+int pmd_clear_huge(pmd_t *pmdp)
 {
-	if (!pmd_sect(*pmd))
+	if (!pmd_sect(READ_ONCE(*pmdp)))
 		return 0;
-	pmd_clear(pmd);
+	pmd_clear(pmdp);
 	return 1;
 }
+
+int pud_free_pmd_page(pud_t *pud)
+{
+	return pud_none(*pud);
+}
+
+int pmd_free_pte_page(pmd_t *pmd)
+{
+	return pmd_none(*pmd);
+}
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index a682a0a2a0fa..a56359373d8b 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -29,7 +29,7 @@ static int change_page_range(pte_t *ptep, pgtable_t token, unsigned long addr,
 			void *data)
 {
 	struct page_change_data *cdata = data;
-	pte_t pte = *ptep;
+	pte_t pte = READ_ONCE(*ptep);
 
 	pte = clear_pte_bit(pte, cdata->clear_mask);
 	pte = set_pte_bit(pte, cdata->set_mask);
@@ -156,30 +156,32 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)
  */
 bool kernel_page_present(struct page *page)
 {
-	pgd_t *pgd;
-	pud_t *pud;
-	pmd_t *pmd;
-	pte_t *pte;
+	pgd_t *pgdp;
+	pud_t *pudp, pud;
+	pmd_t *pmdp, pmd;
+	pte_t *ptep;
 	unsigned long addr = (unsigned long)page_address(page);
 
-	pgd = pgd_offset_k(addr);
-	if (pgd_none(*pgd))
+	pgdp = pgd_offset_k(addr);
+	if (pgd_none(READ_ONCE(*pgdp)))
 		return false;
 
-	pud = pud_offset(pgd, addr);
-	if (pud_none(*pud))
+	pudp = pud_offset(pgdp, addr);
+	pud = READ_ONCE(*pudp);
+	if (pud_none(pud))
 		return false;
-	if (pud_sect(*pud))
+	if (pud_sect(pud))
 		return true;
 
-	pmd = pmd_offset(pud, addr);
-	if (pmd_none(*pmd))
+	pmdp = pmd_offset(pudp, addr);
+	pmd = READ_ONCE(*pmdp);
+	if (pmd_none(pmd))
 		return false;
-	if (pmd_sect(*pmd))
+	if (pmd_sect(pmd))
 		return true;
 
-	pte = pte_offset_kernel(pmd, addr);
-	return pte_valid(*pte);
+	ptep = pte_offset_kernel(pmdp, addr);
+	return pte_valid(READ_ONCE(*ptep));
 }
 #endif /* CONFIG_HIBERNATION */
 #endif /* CONFIG_DEBUG_PAGEALLOC */
diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 051e71ec3335..289f9113a27a 100644
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -49,6 +49,14 @@ void __init pgd_cache_init(void)
 	if (PGD_SIZE == PAGE_SIZE)
 		return;
 
+#ifdef CONFIG_ARM64_PA_BITS_52
+	/*
+	 * With 52-bit physical addresses, the architecture requires the
+	 * top-level table to be aligned to at least 64 bytes.
+	 */
+	BUILD_BUG_ON(PGD_SIZE < 64);
+#endif
+
 	/*
 	 * Naturally aligned pgds required by the architecture.
 	 */
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 95233dfc4c39..c0af47617299 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -70,7 +70,11 @@ ENTRY(cpu_do_suspend)
 	mrs	x8, mdscr_el1
 	mrs	x9, oslsr_el1
 	mrs	x10, sctlr_el1
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 	mrs	x11, tpidr_el1
+alternative_else
+	mrs	x11, tpidr_el2
+alternative_endif
 	mrs	x12, sp_el0
 	stp	x2, x3, [x0]
 	stp	x4, xzr, [x0, #16]
@@ -86,7 +90,7 @@ ENDPROC(cpu_do_suspend)
  *
  * x0: Address of context pointer
  */
-	.pushsection ".idmap.text", "ax"
+	.pushsection ".idmap.text", "awx"
 ENTRY(cpu_do_resume)
 	ldp	x2, x3, [x0]
 	ldp	x4, x5, [x0, #16]
@@ -116,7 +120,11 @@ ENTRY(cpu_do_resume)
 	msr	mdscr_el1, x10
 
 	msr	sctlr_el1, x12
+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
 	msr	tpidr_el1, x13
+alternative_else
+	msr	tpidr_el2, x13
+alternative_endif
 	msr	sp_el0, x14
 	/*
 	 * Restore oslsr_el1 by writing oslar_el1
@@ -124,6 +132,11 @@ ENTRY(cpu_do_resume)
 	ubfx	x11, x11, #1, #1
 	msr	oslar_el1, x11
 	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
+
+alternative_if ARM64_HAS_RAS_EXTN
+	msr_s	SYS_DISR_EL1, xzr
+alternative_else_nop_endif
+
 	isb
 	ret
 ENDPROC(cpu_do_resume)
@@ -138,16 +151,32 @@ ENDPROC(cpu_do_resume)
  *	- pgd_phys - physical address of new TTB
  */
 ENTRY(cpu_do_switch_mm)
-	pre_ttbr0_update_workaround x0, x2, x3
+	mrs	x2, ttbr1_el1
 	mmid	x1, x1				// get mm->context.id
-	bfi	x0, x1, #48, #16		// set the ASID
-	msr	ttbr0_el1, x0			// set TTBR0
+	phys_to_ttbr x3, x0
+#ifdef CONFIG_ARM64_SW_TTBR0_PAN
+	bfi	x3, x1, #48, #16		// set the ASID field in TTBR0
+#endif
+	bfi	x2, x1, #48, #16		// set the ASID
+	msr	ttbr1_el1, x2			// in TTBR1 (since TCR.A1 is set)
 	isb
-	post_ttbr0_update_workaround
-	ret
+	msr	ttbr0_el1, x3			// now update TTBR0
+	isb
+	b	post_ttbr_update_workaround	// Back to C code...
 ENDPROC(cpu_do_switch_mm)
 
-	.pushsection ".idmap.text", "ax"
+	.pushsection ".idmap.text", "awx"
+
+.macro	__idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
+	adrp	\tmp1, empty_zero_page
+	phys_to_ttbr \tmp2, \tmp1
+	msr	ttbr1_el1, \tmp2
+	isb
+	tlbi	vmalle1
+	dsb	nsh
+	isb
+.endm
+
 /*
  * void idmap_cpu_replace_ttbr1(phys_addr_t new_pgd)
  *
@@ -157,22 +186,205 @@ ENDPROC(cpu_do_switch_mm)
 ENTRY(idmap_cpu_replace_ttbr1)
 	save_and_disable_daif flags=x2
 
-	adrp	x1, empty_zero_page
-	msr	ttbr1_el1, x1
+	__idmap_cpu_set_reserved_ttbr1 x1, x3
+
+	phys_to_ttbr x3, x0
+	msr	ttbr1_el1, x3
 	isb
 
-	tlbi	vmalle1
-	dsb	nsh
+	restore_daif x2
+
+	ret
+ENDPROC(idmap_cpu_replace_ttbr1)
+	.popsection
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+	.pushsection ".idmap.text", "awx"
+
+	.macro	__idmap_kpti_get_pgtable_ent, type
+	dc	cvac, cur_\()\type\()p		// Ensure any existing dirty
+	dmb	sy				// lines are written back before
+	ldr	\type, [cur_\()\type\()p]	// loading the entry
+	tbz	\type, #0, skip_\()\type	// Skip invalid and
+	tbnz	\type, #11, skip_\()\type	// non-global entries
+	.endm
+
+	.macro __idmap_kpti_put_pgtable_ent_ng, type
+	orr	\type, \type, #PTE_NG		// Same bit for blocks and pages
+	str	\type, [cur_\()\type\()p]	// Update the entry and ensure it
+	dc	civac, cur_\()\type\()p		// is visible to all CPUs.
+	.endm
+
+/*
+ * void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper)
+ *
+ * Called exactly once from stop_machine context by each CPU found during boot.
+ */
+__idmap_kpti_flag:
+	.long	1
+ENTRY(idmap_kpti_install_ng_mappings)
+	cpu		.req	w0
+	num_cpus	.req	w1
+	swapper_pa	.req	x2
+	swapper_ttb	.req	x3
+	flag_ptr	.req	x4
+	cur_pgdp	.req	x5
+	end_pgdp	.req	x6
+	pgd		.req	x7
+	cur_pudp	.req	x8
+	end_pudp	.req	x9
+	pud		.req	x10
+	cur_pmdp	.req	x11
+	end_pmdp	.req	x12
+	pmd		.req	x13
+	cur_ptep	.req	x14
+	end_ptep	.req	x15
+	pte		.req	x16
+
+	mrs	swapper_ttb, ttbr1_el1
+	adr	flag_ptr, __idmap_kpti_flag
+
+	cbnz	cpu, __idmap_kpti_secondary
+
+	/* We're the boot CPU. Wait for the others to catch up */
+	sevl
+1:	wfe
+	ldaxr	w18, [flag_ptr]
+	eor	w18, w18, num_cpus
+	cbnz	w18, 1b
+
+	/* We need to walk swapper, so turn off the MMU. */
+	pre_disable_mmu_workaround
+	mrs	x18, sctlr_el1
+	bic	x18, x18, #SCTLR_ELx_M
+	msr	sctlr_el1, x18
 	isb
 
-	msr	ttbr1_el1, x0
+	/* Everybody is enjoying the idmap, so we can rewrite swapper. */
+	/* PGD */
+	mov	cur_pgdp, swapper_pa
+	add	end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
+do_pgd:	__idmap_kpti_get_pgtable_ent	pgd
+	tbnz	pgd, #1, walk_puds
+next_pgd:
+	__idmap_kpti_put_pgtable_ent_ng	pgd
+skip_pgd:
+	add	cur_pgdp, cur_pgdp, #8
+	cmp	cur_pgdp, end_pgdp
+	b.ne	do_pgd
+
+	/* Publish the updated tables and nuke all the TLBs */
+	dsb	sy
+	tlbi	vmalle1is
+	dsb	ish
 	isb
 
-	restore_daif x2
+	/* We're done: fire up the MMU again */
+	mrs	x18, sctlr_el1
+	orr	x18, x18, #SCTLR_ELx_M
+	msr	sctlr_el1, x18
+	isb
 
+	/* Set the flag to zero to indicate that we're all done */
+	str	wzr, [flag_ptr]
 	ret
-ENDPROC(idmap_cpu_replace_ttbr1)
+
+	/* PUD */
+walk_puds:
+	.if CONFIG_PGTABLE_LEVELS > 3
+	pte_to_phys	cur_pudp, pgd
+	add	end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
+do_pud:	__idmap_kpti_get_pgtable_ent	pud
+	tbnz	pud, #1, walk_pmds
+next_pud:
+	__idmap_kpti_put_pgtable_ent_ng	pud
+skip_pud:
+	add	cur_pudp, cur_pudp, 8
+	cmp	cur_pudp, end_pudp
+	b.ne	do_pud
+	b	next_pgd
+	.else /* CONFIG_PGTABLE_LEVELS <= 3 */
+	mov	pud, pgd
+	b	walk_pmds
+next_pud:
+	b	next_pgd
+	.endif
+
+	/* PMD */
+walk_pmds:
+	.if CONFIG_PGTABLE_LEVELS > 2
+	pte_to_phys	cur_pmdp, pud
+	add	end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
+do_pmd:	__idmap_kpti_get_pgtable_ent	pmd
+	tbnz	pmd, #1, walk_ptes
+next_pmd:
+	__idmap_kpti_put_pgtable_ent_ng	pmd
+skip_pmd:
+	add	cur_pmdp, cur_pmdp, #8
+	cmp	cur_pmdp, end_pmdp
+	b.ne	do_pmd
+	b	next_pud
+	.else /* CONFIG_PGTABLE_LEVELS <= 2 */
+	mov	pmd, pud
+	b	walk_ptes
+next_pmd:
+	b	next_pud
+	.endif
+
+	/* PTE */
+walk_ptes:
+	pte_to_phys	cur_ptep, pmd
+	add	end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
+do_pte:	__idmap_kpti_get_pgtable_ent	pte
+	__idmap_kpti_put_pgtable_ent_ng	pte
+skip_pte:
+	add	cur_ptep, cur_ptep, #8
+	cmp	cur_ptep, end_ptep
+	b.ne	do_pte
+	b	next_pmd
+
+	/* Secondary CPUs end up here */
+__idmap_kpti_secondary:
+	/* Uninstall swapper before surgery begins */
+	__idmap_cpu_set_reserved_ttbr1 x18, x17
+
+	/* Increment the flag to let the boot CPU we're ready */
+1:	ldxr	w18, [flag_ptr]
+	add	w18, w18, #1
+	stxr	w17, w18, [flag_ptr]
+	cbnz	w17, 1b
+
+	/* Wait for the boot CPU to finish messing around with swapper */
+	sevl
+1:	wfe
+	ldxr	w18, [flag_ptr]
+	cbnz	w18, 1b
+
+	/* All done, act like nothing happened */
+	msr	ttbr1_el1, swapper_ttb
+	isb
+	ret
+
+	.unreq	cpu
+	.unreq	num_cpus
+	.unreq	swapper_pa
+	.unreq	swapper_ttb
+	.unreq	flag_ptr
+	.unreq	cur_pgdp
+	.unreq	end_pgdp
+	.unreq	pgd
+	.unreq	cur_pudp
+	.unreq	end_pudp
+	.unreq	pud
+	.unreq	cur_pmdp
+	.unreq	end_pmdp
+	.unreq	pmd
+	.unreq	cur_ptep
+	.unreq	end_ptep
+	.unreq	pte
+ENDPROC(idmap_kpti_install_ng_mappings)
 	.popsection
+#endif
 
 /*
  *	__cpu_setup
@@ -180,7 +392,7 @@ ENDPROC(idmap_cpu_replace_ttbr1)
  *	Initialise the processor for turning the MMU on.  Return in x0 the
  *	value of the SCTLR_EL1 register.
  */
-	.pushsection ".idmap.text", "ax"
+	.pushsection ".idmap.text", "awx"
 ENTRY(__cpu_setup)
 	tlbi	vmalle1				// Invalidate local TLB
 	dsb	nsh
@@ -214,25 +426,19 @@ ENTRY(__cpu_setup)
 	/*
 	 * Prepare SCTLR
 	 */
-	adr	x5, crval
-	ldp	w5, w6, [x5]
-	mrs	x0, sctlr_el1
-	bic	x0, x0, x5			// clear bits
-	orr	x0, x0, x6			// set bits
+	mov_q	x0, SCTLR_EL1_SET
 	/*
 	 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
 	 * both user and kernel.
 	 */
 	ldr	x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
-			TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
+			TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 | TCR_A1
 	tcr_set_idmap_t0sz	x10, x9
 
 	/*
-	 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
-	 * TCR_EL1.
+	 * Set the IPS bits in TCR_EL1.
 	 */
-	mrs	x9, ID_AA64MMFR0_EL1
-	bfi	x10, x9, #32, #3
+	tcr_compute_pa_size x10, #TCR_IPS_SHIFT, x5, x6
 #ifdef CONFIG_ARM64_HW_AFDBM
 	/*
 	 * Hardware update of the Access and Dirty bits.
@@ -249,21 +455,3 @@ ENTRY(__cpu_setup)
 	msr	tcr_el1, x10
 	ret					// return to head.S
 ENDPROC(__cpu_setup)
-
-	/*
-	 * We set the desired value explicitly, including those of the
-	 * reserved bits. The values of bits EE & E0E were set early in
-	 * el2_setup, which are left untouched below.
-	 *
-	 *                 n n            T
-	 *       U E      WT T UD     US IHBS
-	 *       CE0      XWHW CZ     ME TEEA S
-	 * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
-	 * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
-	 * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
-	 */
-	.type	crval, #object
-crval:
-	.word	0xfcffffff			// clear
-	.word	0x34d5d91d			// set
-	.popsection
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index bb32f7f6dd0f..a93350451e8e 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -31,8 +31,6 @@
 
 #include "bpf_jit.h"
 
-int bpf_jit_enable __read_mostly;
-
 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
 #define TCALL_CNT (MAX_BPF_JIT_REG + 2)
@@ -99,6 +97,20 @@ static inline void emit_a64_mov_i64(const int reg, const u64 val,
 	}
 }
 
+static inline void emit_addr_mov_i64(const int reg, const u64 val,
+				     struct jit_ctx *ctx)
+{
+	u64 tmp = val;
+	int shift = 0;
+
+	emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx);
+	for (;shift < 48;) {
+		tmp >>= 16;
+		shift += 16;
+		emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx);
+	}
+}
+
 static inline void emit_a64_mov_i(const int is64, const int reg,
 				  const s32 val, struct jit_ctx *ctx)
 {
@@ -238,8 +250,9 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
 	off = offsetof(struct bpf_array, map.max_entries);
 	emit_a64_mov_i64(tmp, off, ctx);
 	emit(A64_LDR32(tmp, r2, tmp), ctx);
+	emit(A64_MOV(0, r3, r3), ctx);
 	emit(A64_CMP(0, r3, tmp), ctx);
-	emit(A64_B_(A64_COND_GE, jmp_offset), ctx);
+	emit(A64_B_(A64_COND_CS, jmp_offset), ctx);
 
 	/* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
 	 *     goto out;
@@ -247,7 +260,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
 	 */
 	emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx);
 	emit(A64_CMP(1, tcc, tmp), ctx);
-	emit(A64_B_(A64_COND_GT, jmp_offset), ctx);
+	emit(A64_B_(A64_COND_HI, jmp_offset), ctx);
 	emit(A64_ADD_I(1, tcc, tcc, 1), ctx);
 
 	/* prog = array->ptrs[index];
@@ -378,18 +391,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 	case BPF_ALU64 | BPF_DIV | BPF_X:
 	case BPF_ALU | BPF_MOD | BPF_X:
 	case BPF_ALU64 | BPF_MOD | BPF_X:
-	{
-		const u8 r0 = bpf2a64[BPF_REG_0];
-
-		/* if (src == 0) return 0 */
-		jmp_offset = 3; /* skip ahead to else path */
-		check_imm19(jmp_offset);
-		emit(A64_CBNZ(is64, src, jmp_offset), ctx);
-		emit(A64_MOVZ(1, r0, 0, 0), ctx);
-		jmp_offset = epilogue_offset(ctx);
-		check_imm26(jmp_offset);
-		emit(A64_B(jmp_offset), ctx);
-		/* else */
 		switch (BPF_OP(code)) {
 		case BPF_DIV:
 			emit(A64_UDIV(is64, dst, dst, src), ctx);
@@ -401,7 +402,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 			break;
 		}
 		break;
-	}
 	case BPF_ALU | BPF_LSH | BPF_X:
 	case BPF_ALU64 | BPF_LSH | BPF_X:
 		emit(A64_LSLV(is64, dst, dst, src), ctx);
@@ -605,7 +605,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 		const u8 r0 = bpf2a64[BPF_REG_0];
 		const u64 func = (u64)__bpf_call_base + imm;
 
-		emit_a64_mov_i64(tmp, func, ctx);
+		if (ctx->prog->is_func)
+			emit_addr_mov_i64(tmp, func, ctx);
+		else
+			emit_a64_mov_i64(tmp, func, ctx);
 		emit(A64_BLR(tmp), ctx);
 		emit(A64_MOV(1, r0, A64_R(0)), ctx);
 		break;
@@ -837,16 +840,24 @@ static inline void bpf_flush_icache(void *start, void *end)
 	flush_icache_range((unsigned long)start, (unsigned long)end);
 }
 
+struct arm64_jit_data {
+	struct bpf_binary_header *header;
+	u8 *image;
+	struct jit_ctx ctx;
+};
+
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 {
 	struct bpf_prog *tmp, *orig_prog = prog;
 	struct bpf_binary_header *header;
+	struct arm64_jit_data *jit_data;
 	bool tmp_blinded = false;
+	bool extra_pass = false;
 	struct jit_ctx ctx;
 	int image_size;
 	u8 *image_ptr;
 
-	if (!bpf_jit_enable)
+	if (!prog->jit_requested)
 		return orig_prog;
 
 	tmp = bpf_jit_blind_constants(prog);
@@ -860,13 +871,30 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		prog = tmp;
 	}
 
+	jit_data = prog->aux->jit_data;
+	if (!jit_data) {
+		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+		if (!jit_data) {
+			prog = orig_prog;
+			goto out;
+		}
+		prog->aux->jit_data = jit_data;
+	}
+	if (jit_data->ctx.offset) {
+		ctx = jit_data->ctx;
+		image_ptr = jit_data->image;
+		header = jit_data->header;
+		extra_pass = true;
+		image_size = sizeof(u32) * ctx.idx;
+		goto skip_init_ctx;
+	}
 	memset(&ctx, 0, sizeof(ctx));
 	ctx.prog = prog;
 
 	ctx.offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
 	if (ctx.offset == NULL) {
 		prog = orig_prog;
-		goto out;
+		goto out_off;
 	}
 
 	/* 1. Initial fake pass to compute ctx->idx. */
@@ -897,6 +925,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	/* 2. Now, the actual pass. */
 
 	ctx.image = (__le32 *)image_ptr;
+skip_init_ctx:
 	ctx.idx = 0;
 
 	build_prologue(&ctx);
@@ -922,13 +951,31 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 
 	bpf_flush_icache(header, ctx.image + ctx.idx);
 
-	bpf_jit_binary_lock_ro(header);
+	if (!prog->is_func || extra_pass) {
+		if (extra_pass && ctx.idx != jit_data->ctx.idx) {
+			pr_err_once("multi-func JIT bug %d != %d\n",
+				    ctx.idx, jit_data->ctx.idx);
+			bpf_jit_binary_free(header);
+			prog->bpf_func = NULL;
+			prog->jited = 0;
+			goto out_off;
+		}
+		bpf_jit_binary_lock_ro(header);
+	} else {
+		jit_data->ctx = ctx;
+		jit_data->image = image_ptr;
+		jit_data->header = header;
+	}
 	prog->bpf_func = (void *)ctx.image;
 	prog->jited = 1;
 	prog->jited_len = image_size;
 
+	if (!prog->is_func || extra_pass) {
 out_off:
-	kfree(ctx.offset);
+		kfree(ctx.offset);
+		kfree(jit_data);
+		prog->aux->jit_data = NULL;
+	}
 out:
 	if (tmp_blinded)
 		bpf_jit_prog_release_other(prog, prog == orig_prog ?
diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S
index 401ceb71540c..c5f05c4a4d00 100644
--- a/arch/arm64/xen/hypercall.S
+++ b/arch/arm64/xen/hypercall.S
@@ -101,12 +101,12 @@ ENTRY(privcmd_call)
 	 * need the explicit uaccess_enable/disable if the TTBR0 PAN emulation
 	 * is enabled (it implies that hardware UAO and PAN disabled).
 	 */
-	uaccess_ttbr0_enable x6, x7
+	uaccess_ttbr0_enable x6, x7, x8
 	hvc XEN_IMM
 
 	/*
 	 * Disable userspace access from kernel once the hyp call completed.
 	 */
-	uaccess_ttbr0_disable x6
+	uaccess_ttbr0_disable x6, x7
 	ret
 ENDPROC(privcmd_call);
diff --git a/arch/blackfin/include/asm/clkdev.h b/arch/blackfin/include/asm/clkdev.h
deleted file mode 100644
index ded0209348c7..000000000000
--- a/arch/blackfin/include/asm/clkdev.h
+++ /dev/null
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __ASM_CLKDEV__H_
-#define __ASM_CLKDEV__H_
-
-#include <linux/slab.h>
-
-static inline struct clk_lookup_alloc *__clkdev_alloc(size_t size)
-{
-	return kzalloc(size, GFP_KERNEL);
-}
-
-#ifndef CONFIG_COMMON_CLK
-#define __clk_put(clk)
-#define __clk_get(clk) ({ 1; })
-#endif
-
-#endif
diff --git a/arch/blackfin/include/asm/thread_info.h b/arch/blackfin/include/asm/thread_info.h
index 2966b93850a1..a5aeab4e5f2d 100644
--- a/arch/blackfin/include/asm/thread_info.h
+++ b/arch/blackfin/include/asm/thread_info.h
@@ -56,8 +56,6 @@ struct thread_info {
 	.cpu		= 0,			\
 	.preempt_count	= INIT_PREEMPT_COUNT,	\
 }
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
 
 /* Given a task stack pointer, you can find its corresponding
  * thread_info structure just by masking it to the THREAD_SIZE
diff --git a/arch/blackfin/include/uapi/asm/poll.h b/arch/blackfin/include/uapi/asm/poll.h
index 8b094d43e9b7..cd2f1a78aba5 100644
--- a/arch/blackfin/include/uapi/asm/poll.h
+++ b/arch/blackfin/include/uapi/asm/poll.h
@@ -9,7 +9,7 @@
 #ifndef _UAPI__BFIN_POLL_H
 #define _UAPI__BFIN_POLL_H
 
-#define POLLWRNORM	4 /* POLLOUT */
+#define POLLWRNORM	POLLOUT
 #define POLLWRBAND	256
 
 #include <asm-generic/poll.h>
diff --git a/arch/blackfin/include/uapi/asm/siginfo.h b/arch/blackfin/include/uapi/asm/siginfo.h
index b1db506c8d2e..2dd8c9c39248 100644
--- a/arch/blackfin/include/uapi/asm/siginfo.h
+++ b/arch/blackfin/include/uapi/asm/siginfo.h
@@ -11,40 +11,6 @@
 #include <linux/types.h>
 #include <asm-generic/siginfo.h>
 
-#define UID16_SIGINFO_COMPAT_NEEDED
-
 #define si_uid16	_sifields._kill._uid
 
-#define ILL_ILLPARAOP	2	/* illegal opcode combine ********** */
-#define ILL_ILLEXCPT	4	/* unrecoverable exception ********** */
-#define ILL_CPLB_VI	9	/* D/I CPLB protect violation ******** */
-#define ILL_CPLB_MISS	10	/* D/I CPLB miss ******** */
-#define ILL_CPLB_MULHIT	11	/* D/I CPLB multiple hit ******** */
-#undef NSIGILL
-#define NSIGILL         11
-
-/*
- * SIGBUS si_codes
- */
-#define BUS_OPFETCH	4	/* error from instruction fetch ******** */
-#undef NSIGBUS
-#define NSIGBUS		4
-
-/*
- * SIGTRAP si_codes
- */
-#define TRAP_STEP	1	/* single-step breakpoint************* */
-#define TRAP_TRACEFLOW	2	/* trace buffer overflow ************* */
-#define TRAP_WATCHPT	3	/* watchpoint match      ************* */
-#define TRAP_ILLTRAP	4	/* illegal trap          ************* */
-#undef NSIGTRAP
-#define NSIGTRAP	4
-
-/*
- * SIGSEGV si_codes
- */
-#define SEGV_STACKFLOW	3	/* stack overflow */
-#undef NSIGSEGV
-#define NSIGSEGV	3
-
 #endif /* _UAPI_BFIN_SIGINFO_H */
diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild
index d717329c8cf9..fd4c840de837 100644
--- a/arch/c6x/include/asm/Kbuild
+++ b/arch/c6x/include/asm/Kbuild
@@ -1,7 +1,6 @@
 generic-y += atomic.h
 generic-y += barrier.h
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/c6x/include/asm/thread_info.h b/arch/c6x/include/asm/thread_info.h
index acc70c135ab8..59a5697fe0f3 100644
--- a/arch/c6x/include/asm/thread_info.h
+++ b/arch/c6x/include/asm/thread_info.h
@@ -60,9 +60,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* get the thread information struct of current task */
 static inline __attribute__((const))
 struct thread_info *current_thread_info(void)
diff --git a/arch/c6x/kernel/traps.c b/arch/c6x/kernel/traps.c
index 09b8a40d5680..4c1d4b84dd2b 100644
--- a/arch/c6x/kernel/traps.c
+++ b/arch/c6x/kernel/traps.c
@@ -11,7 +11,6 @@
 #include <linux/module.h>
 #include <linux/ptrace.h>
 #include <linux/sched/debug.h>
-#include <linux/kallsyms.h>
 #include <linux/bug.h>
 
 #include <asm/soc.h>
@@ -375,8 +374,7 @@ static void show_trace(unsigned long *stack, unsigned long *endstack)
 			if (i % 5 == 0)
 				pr_debug("\n	    ");
 #endif
-			pr_debug(" [<%08lx>]", addr);
-			print_symbol(" %s\n", addr);
+			pr_debug(" [<%08lx>] %pS\n", addr, (void *)addr);
 			i++;
 		}
 	}
diff --git a/arch/cris/Kconfig b/arch/cris/Kconfig
index 54d3f426763b..cd5a0865c97f 100644
--- a/arch/cris/Kconfig
+++ b/arch/cris/Kconfig
@@ -33,6 +33,9 @@ config GENERIC_CALIBRATE_DELAY
 config NO_IOPORT_MAP
 	def_bool y if !PCI
 
+config NO_DMA
+	def_bool y if !PCI
+
 config FORCE_MAX_ZONEORDER
 	int
 	default 6
@@ -72,6 +75,7 @@ config CRIS
 	select GENERIC_SCHED_CLOCK if ETRAX_ARCH_V32
 	select HAVE_DEBUG_BUGVERBOSE if ETRAX_ARCH_V32
 	select HAVE_NMI
+	select DMA_DIRECT_OPS if PCI
 
 config HZ
 	int
diff --git a/arch/cris/arch-v10/drivers/gpio.c b/arch/cris/arch-v10/drivers/gpio.c
index 68dbe261dc57..cd0e05d89d42 100644
--- a/arch/cris/arch-v10/drivers/gpio.c
+++ b/arch/cris/arch-v10/drivers/gpio.c
@@ -50,7 +50,7 @@ static ssize_t gpio_write(struct file *file, const char __user *buf,
 	size_t count, loff_t *off);
 static int gpio_open(struct inode *inode, struct file *filp);
 static int gpio_release(struct inode *inode, struct file *filp);
-static unsigned int gpio_poll(struct file *filp, struct poll_table_struct *wait);
+static __poll_t gpio_poll(struct file *filp, struct poll_table_struct *wait);
 
 /* private data per open() of this driver */
 
@@ -141,9 +141,9 @@ static unsigned long dir_g_shadow; /* 1=output */
 #define USE_PORTS(priv) ((priv)->minor <= GPIO_MINOR_B)
 
 
-static unsigned int gpio_poll(struct file *file, poll_table *wait)
+static __poll_t gpio_poll(struct file *file, poll_table *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	struct gpio_private *priv = file->private_data;
 	unsigned long data;
 	unsigned long flags;
@@ -173,7 +173,7 @@ static unsigned int gpio_poll(struct file *file, poll_table *wait)
 
 	if ((data & priv->highalarm) ||
 	    (~data & priv->lowalarm)) {
-		mask = POLLIN|POLLRDNORM;
+		mask = EPOLLIN|EPOLLRDNORM;
 	}
 
 out:
diff --git a/arch/cris/arch-v10/drivers/sync_serial.c b/arch/cris/arch-v10/drivers/sync_serial.c
index cfe9176f2205..ed1a568a7217 100644
--- a/arch/cris/arch-v10/drivers/sync_serial.c
+++ b/arch/cris/arch-v10/drivers/sync_serial.c
@@ -157,7 +157,7 @@ static inline int sync_data_avail(struct sync_port *port);
 
 static int sync_serial_open(struct inode *inode, struct file *file);
 static int sync_serial_release(struct inode *inode, struct file *file);
-static unsigned int sync_serial_poll(struct file *filp, poll_table *wait);
+static __poll_t sync_serial_poll(struct file *filp, poll_table *wait);
 
 static long sync_serial_ioctl(struct file *file,
 	unsigned int cmd, unsigned long arg);
@@ -654,28 +654,28 @@ static int sync_serial_release(struct inode *inode, struct file *file)
 
 
 
-static unsigned int sync_serial_poll(struct file *file, poll_table *wait)
+static __poll_t sync_serial_poll(struct file *file, poll_table *wait)
 {
 	int dev = MINOR(file_inode(file)->i_rdev);
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	struct sync_port *port;
-	DEBUGPOLL(static unsigned int prev_mask = 0);
+	DEBUGPOLL(static __poll_t prev_mask = 0);
 
 	port = &ports[dev];
 	poll_wait(file, &port->out_wait_q, wait);
 	poll_wait(file, &port->in_wait_q, wait);
 	/* Some room to write */
 	if (port->out_count < OUT_BUFFER_SIZE)
-		mask |=  POLLOUT | POLLWRNORM;
+		mask |=  EPOLLOUT | EPOLLWRNORM;
 	/* At least an inbufchunk of data */
 	if (sync_data_avail(port) >= port->inbufchunk)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	DEBUGPOLL(if (mask != prev_mask)
 		printk(KERN_DEBUG "sync_serial_poll: mask 0x%08X %s %s\n",
 			mask,
-			mask & POLLOUT ? "POLLOUT" : "",
-			mask & POLLIN ? "POLLIN" : "");
+			mask & EPOLLOUT ? "POLLOUT" : "",
+			mask & EPOLLIN ? "POLLIN" : "");
 		prev_mask = mask;
 	);
 	return mask;
diff --git a/arch/cris/arch-v32/drivers/cryptocop.c b/arch/cris/arch-v32/drivers/cryptocop.c
index d688fe117dca..a3c353472a8c 100644
--- a/arch/cris/arch-v32/drivers/cryptocop.c
+++ b/arch/cris/arch-v32/drivers/cryptocop.c
@@ -2717,37 +2717,28 @@ static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsig
 		}
 	}
 
-	/* Acquire the mm page semaphore. */
-	down_read(&current->mm->mmap_sem);
-
-	err = get_user_pages((unsigned long int)(oper.indata + prev_ix),
+	err = get_user_pages_fast((unsigned long)(oper.indata + prev_ix),
 			     noinpages,
-			     0,  /* read access only for in data */
-			     inpages,
-			     NULL);
+			     false,  /* read access only for in data */
+			     inpages);
 
 	if (err < 0) {
-		up_read(&current->mm->mmap_sem);
 		nooutpages = noinpages = 0;
 		DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
 		goto error_cleanup;
 	}
 	noinpages = err;
-	if (oper.do_cipher){
-		err = get_user_pages((unsigned long int)oper.cipher_outdata,
+	if (oper.do_cipher) {
+		err = get_user_pages_fast((unsigned long)oper.cipher_outdata,
 				     nooutpages,
-				     FOLL_WRITE, /* write access for out data */
-				     outpages,
-				     NULL);
-		up_read(&current->mm->mmap_sem);
+				     true, /* write access for out data */
+				     outpages);
 		if (err < 0) {
 			nooutpages = 0;
 			DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
 			goto error_cleanup;
 		}
 		nooutpages = err;
-	} else {
-		up_read(&current->mm->mmap_sem);
 	}
 
 	/* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
diff --git a/arch/cris/arch-v32/drivers/pci/Makefile b/arch/cris/arch-v32/drivers/pci/Makefile
index bff7482f2444..93c8be6170b1 100644
--- a/arch/cris/arch-v32/drivers/pci/Makefile
+++ b/arch/cris/arch-v32/drivers/pci/Makefile
@@ -2,4 +2,4 @@
 # Makefile for Etrax cardbus driver
 #
 
-obj-$(CONFIG_ETRAX_CARDBUS)        += bios.o dma.o
+obj-$(CONFIG_ETRAX_CARDBUS)        += bios.o
diff --git a/arch/cris/arch-v32/drivers/pci/dma.c b/arch/cris/arch-v32/drivers/pci/dma.c
deleted file mode 100644
index dbbd3816cc0b..000000000000
--- a/arch/cris/arch-v32/drivers/pci/dma.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Dynamic DMA mapping support.
- *
- * On cris there is no hardware dynamic DMA address translation,
- * so consistent alloc/free are merely page allocation/freeing.
- * The rest of the dynamic DMA mapping interface is implemented
- * in asm/pci.h.
- *
- * Borrowed from i386.
- */
-
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/string.h>
-#include <linux/pci.h>
-#include <linux/gfp.h>
-#include <asm/io.h>
-
-static void *v32_dma_alloc(struct device *dev, size_t size,
-		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
-{
-	void *ret;
-
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
-
-	if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
-		gfp |= GFP_DMA;
-
-	ret = (void *)__get_free_pages(gfp,  get_order(size));
-
-	if (ret != NULL) {
-		memset(ret, 0, size);
-		*dma_handle = virt_to_phys(ret);
-	}
-	return ret;
-}
-
-static void v32_dma_free(struct device *dev, size_t size, void *vaddr,
-		dma_addr_t dma_handle, unsigned long attrs)
-{
-	free_pages((unsigned long)vaddr, get_order(size));
-}
-
-static inline dma_addr_t v32_dma_map_page(struct device *dev,
-		struct page *page, unsigned long offset, size_t size,
-		enum dma_data_direction direction, unsigned long attrs)
-{
-	return page_to_phys(page) + offset;
-}
-
-static inline int v32_dma_map_sg(struct device *dev, struct scatterlist *sg,
-		int nents, enum dma_data_direction direction,
-		unsigned long attrs)
-{
-	printk("Map sg\n");
-	return nents;
-}
-
-static inline int v32_dma_supported(struct device *dev, u64 mask)
-{
-        /*
-         * we fall back to GFP_DMA when the mask isn't all 1s,
-         * so we can't guarantee allocations that must be
-         * within a tighter range than GFP_DMA..
-         */
-        if (mask < 0x00ffffff)
-                return 0;
-	return 1;
-}
-
-const struct dma_map_ops v32_dma_ops = {
-	.alloc			= v32_dma_alloc,
-	.free			= v32_dma_free,
-	.map_page		= v32_dma_map_page,
-	.map_sg                 = v32_dma_map_sg,
-	.dma_supported		= v32_dma_supported,
-};
-EXPORT_SYMBOL(v32_dma_ops);
diff --git a/arch/cris/arch-v32/drivers/sync_serial.c b/arch/cris/arch-v32/drivers/sync_serial.c
index 8efcc1a899a8..1b0ce8a8af16 100644
--- a/arch/cris/arch-v32/drivers/sync_serial.c
+++ b/arch/cris/arch-v32/drivers/sync_serial.c
@@ -178,7 +178,7 @@ static inline int sync_data_avail(struct sync_port *port);
 
 static int sync_serial_open(struct inode *, struct file *);
 static int sync_serial_release(struct inode *, struct file *);
-static unsigned int sync_serial_poll(struct file *filp, poll_table *wait);
+static __poll_t sync_serial_poll(struct file *filp, poll_table *wait);
 
 static long sync_serial_ioctl(struct file *file,
 			      unsigned int cmd, unsigned long arg);
@@ -555,13 +555,13 @@ static int sync_serial_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int sync_serial_poll(struct file *file, poll_table *wait)
+static __poll_t sync_serial_poll(struct file *file, poll_table *wait)
 {
 	int dev = iminor(file_inode(file));
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	struct sync_port *port;
 	DEBUGPOLL(
-	static unsigned int prev_mask;
+	static __poll_t prev_mask;
 	);
 
 	port = &ports[dev];
@@ -574,24 +574,24 @@ static unsigned int sync_serial_poll(struct file *file, poll_table *wait)
 
 	/* No active transfer, descriptors are available */
 	if (port->output && !port->tr_running)
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 
 	/* Descriptor and buffer space available. */
 	if (port->output &&
 	    port->active_tr_descr != port->catch_tr_descr &&
 	    port->out_buf_count < OUT_BUFFER_SIZE)
-		mask |=  POLLOUT | POLLWRNORM;
+		mask |=  EPOLLOUT | EPOLLWRNORM;
 
 	/* At least an inbufchunk of data */
 	if (port->input && sync_data_avail(port) >= port->inbufchunk)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	DEBUGPOLL(
 	if (mask != prev_mask)
 		pr_info("sync_serial_poll: mask 0x%08X %s %s\n",
 			mask,
-			mask & POLLOUT ? "POLLOUT" : "",
-			mask & POLLIN ? "POLLIN" : "");
+			mask & EPOLLOUT ? "POLLOUT" : "",
+			mask & EPOLLIN ? "POLLIN" : "");
 		prev_mask = mask;
 	);
 	return mask;
diff --git a/arch/cris/include/arch-v10/arch/bug.h b/arch/cris/include/arch-v10/arch/bug.h
index 905afeacfedf..06da9d49152a 100644
--- a/arch/cris/include/arch-v10/arch/bug.h
+++ b/arch/cris/include/arch-v10/arch/bug.h
@@ -44,18 +44,25 @@ struct bug_frame {
  * not be used like this with newer versions of gcc.
  */
 #define BUG()								\
+do {									\
 	__asm__ __volatile__ ("clear.d [" __stringify(BUG_MAGIC) "]\n\t"\
 			      "movu.w " __stringify(__LINE__) ",$r0\n\t"\
 			      "jump 0f\n\t"				\
 			      ".section .rodata\n"			\
 			      "0:\t.string \"" __FILE__ "\"\n\t"	\
-			      ".previous")
+			      ".previous");				\
+	unreachable();							\
+} while (0)
 #endif
 
 #else
 
 /* This just causes an oops. */
-#define BUG() (*(int *)0 = 0)
+#define BUG()								\
+do {									\
+	barrier_before_unreachable();					\
+	__builtin_trap();						\
+} while (0)
 
 #endif
 
diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild
index 460349cb147f..6a547fe8752b 100644
--- a/arch/cris/include/asm/Kbuild
+++ b/arch/cris/include/asm/Kbuild
@@ -1,10 +1,10 @@
 generic-y += atomic.h
 generic-y += barrier.h
-generic-y += clkdev.h
 generic-y += cmpxchg.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
+generic-y += dma-mapping.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += extable.h
diff --git a/arch/cris/include/asm/dma-mapping.h b/arch/cris/include/asm/dma-mapping.h
deleted file mode 100644
index 1553bdb30a0c..000000000000
--- a/arch/cris/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,20 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_CRIS_DMA_MAPPING_H
-#define _ASM_CRIS_DMA_MAPPING_H
-
-#ifdef CONFIG_PCI
-extern const struct dma_map_ops v32_dma_ops;
-
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &v32_dma_ops;
-}
-#else
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	BUG();
-	return NULL;
-}
-#endif
-
-#endif
diff --git a/arch/cris/include/asm/processor.h b/arch/cris/include/asm/processor.h
index 124dd5ec7f65..ee4d8b03d048 100644
--- a/arch/cris/include/asm/processor.h
+++ b/arch/cris/include/asm/processor.h
@@ -26,13 +26,6 @@ struct task_struct;
  */
 #define TASK_UNMAPPED_BASE      (PAGE_ALIGN(TASK_SIZE / 3))
 
-/* THREAD_SIZE is the size of the thread_info/kernel_stack combo.
- * normally, the stack is found by doing something like p + THREAD_SIZE
- * in CRIS, a page is 8192 bytes, which seems like a sane size
- */
-#define THREAD_SIZE       PAGE_SIZE
-#define THREAD_SIZE_ORDER (0)
-
 /*
  * At user->kernel entry, the pt_regs struct is stacked on the top of the kernel-stack.
  * This macro allows us to find those regs for a task.
@@ -59,8 +52,6 @@ static inline void release_thread(struct task_struct *dead_task)
         /* Nothing needs to be done.  */
 }
 
-#define init_stack      (init_thread_union.stack)
-
 #define cpu_relax()     barrier()
 
 void default_idle(void);
diff --git a/arch/cris/include/asm/thread_info.h b/arch/cris/include/asm/thread_info.h
index 472830c90997..996fef3be1d5 100644
--- a/arch/cris/include/asm/thread_info.h
+++ b/arch/cris/include/asm/thread_info.h
@@ -20,6 +20,13 @@
 #endif
 
 
+/* THREAD_SIZE is the size of the thread_info/kernel_stack combo.
+ * normally, the stack is found by doing something like p + THREAD_SIZE
+ * in CRIS, a page is 8192 bytes, which seems like a sane size
+ */
+#define THREAD_SIZE       PAGE_SIZE
+#define THREAD_SIZE_ORDER (0)
+
 /*
  * low level task data that entry.S needs immediate access to
  * - this struct should fit entirely inside of one cache line
@@ -56,8 +63,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,			\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-
 #endif /* !__ASSEMBLY__ */
 
 /*
diff --git a/arch/cris/kernel/Makefile b/arch/cris/kernel/Makefile
index af075a5fb9aa..f6bfee6c8c1b 100644
--- a/arch/cris/kernel/Makefile
+++ b/arch/cris/kernel/Makefile
@@ -1,19 +1,17 @@
 # SPDX-License-Identifier: GPL-2.0
-# $Id: Makefile,v 1.12 2004/10/19 13:07:43 starvik Exp $
 #
-# Makefile for the linux kernel.
+# Makefile for the CRIS port.
 #
 
 CPPFLAGS_vmlinux.lds := -DDRAM_VIRTUAL_BASE=0x$(CONFIG_ETRAX_DRAM_VIRTUAL_BASE)
-extra-y	:= vmlinux.lds
+extra-y := vmlinux.lds
 
-obj-y   := process.o traps.o irq.o ptrace.o setup.o time.o sys_cris.o
-obj-y += devicetree.o
+obj-y := process.o traps.o irq.o ptrace.o setup.o time.o sys_cris.o
 obj-y += stacktrace.o
 
-obj-$(CONFIG_MODULES)    += crisksyms.o
-obj-$(CONFIG_MODULES)	 += module.o
-obj-$(CONFIG_SYSTEM_PROFILER)   += profile.o
+obj-$(CONFIG_MODULES)		+= crisksyms.o
+obj-$(CONFIG_MODULES)		+= module.o
+obj-$(CONFIG_SYSTEM_PROFILER)	+= profile.o
 
 clean:
 
diff --git a/arch/cris/kernel/devicetree.c b/arch/cris/kernel/devicetree.c
deleted file mode 100644
index 36e1c658229f..000000000000
--- a/arch/cris/kernel/devicetree.c
+++ /dev/null
@@ -1,15 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/init.h>
-#include <linux/bootmem.h>
-#include <linux/printk.h>
-
-void __init early_init_dt_add_memory_arch(u64 base, u64 size)
-{
-	pr_err("%s(%llx, %llx)\n",
-	       __func__, base, size);
-}
-
-void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
-{
-	return alloc_bootmem_align(size, align);
-}
diff --git a/arch/cris/kernel/setup.c b/arch/cris/kernel/setup.c
index 524d47501a23..1b61a7207afb 100644
--- a/arch/cris/kernel/setup.c
+++ b/arch/cris/kernel/setup.c
@@ -24,6 +24,7 @@
 #include <linux/of_fdt.h>
 #include <asm/setup.h>
 #include <arch/system.h>
+#include <asm/sections.h>
 
 /*
  * Setup options
@@ -31,7 +32,6 @@
 struct screen_info screen_info;
 
 extern int root_mountflags;
-extern char _etext, _edata, _end;
 
 char __initdata cris_command_line[COMMAND_LINE_SIZE] = { 0, };
 
diff --git a/arch/cris/kernel/vmlinux.lds.S b/arch/cris/kernel/vmlinux.lds.S
index 6d1dbc1ba767..9b232e0f673e 100644
--- a/arch/cris/kernel/vmlinux.lds.S
+++ b/arch/cris/kernel/vmlinux.lds.S
@@ -11,6 +11,7 @@
 
 #include <asm-generic/vmlinux.lds.h>
 #include <asm/page.h>
+#include <asm/thread_info.h>
 
 #ifdef CONFIG_ETRAX_VMEM_SIZE
 #define __CONFIG_ETRAX_VMEM_SIZE CONFIG_ETRAX_VMEM_SIZE
diff --git a/arch/frv/include/asm/Kbuild b/arch/frv/include/asm/Kbuild
index 2cf7648787b2..b16b9c48ea09 100644
--- a/arch/frv/include/asm/Kbuild
+++ b/arch/frv/include/asm/Kbuild
@@ -1,5 +1,4 @@
 
-generic-y += clkdev.h
 generic-y += device.h
 generic-y += exec.h
 generic-y += extable.h
diff --git a/arch/frv/include/asm/thread_info.h b/arch/frv/include/asm/thread_info.h
index ccba3b6ce918..0f950845fad9 100644
--- a/arch/frv/include/asm/thread_info.h
+++ b/arch/frv/include/asm/thread_info.h
@@ -64,9 +64,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 register struct thread_info *__current_thread_info asm("gr15");
 
diff --git a/arch/frv/include/uapi/asm/Kbuild b/arch/frv/include/uapi/asm/Kbuild
index 14a2e9af97e9..5354b0f84d41 100644
--- a/arch/frv/include/uapi/asm/Kbuild
+++ b/arch/frv/include/uapi/asm/Kbuild
@@ -1,4 +1,5 @@
 # UAPI Header export list
 include include/uapi/asm-generic/Kbuild.asm
 
+generic-y += siginfo.h
 generic-y += bpf_perf_event.h
diff --git a/arch/frv/include/uapi/asm/poll.h b/arch/frv/include/uapi/asm/poll.h
index 887b67288340..f55b45f475ec 100644
--- a/arch/frv/include/uapi/asm/poll.h
+++ b/arch/frv/include/uapi/asm/poll.h
@@ -6,8 +6,6 @@
 #define POLLWRBAND	256
 
 #include <asm-generic/poll.h>
-
 #undef POLLREMOVE
 
 #endif
-
diff --git a/arch/frv/include/uapi/asm/siginfo.h b/arch/frv/include/uapi/asm/siginfo.h
deleted file mode 100644
index 4c8c975747ac..000000000000
--- a/arch/frv/include/uapi/asm/siginfo.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ASM_SIGINFO_H
-#define _ASM_SIGINFO_H
-
-#include <linux/types.h>
-#include <asm-generic/siginfo.h>
-
-#define FPE_MDAOVF	9	/* media overflow */
-#undef NSIGFPE
-#define NSIGFPE		9
-
-#endif
-
diff --git a/arch/h8300/Kconfig b/arch/h8300/Kconfig
index f8d3fde08190..091d6d04b5e5 100644
--- a/arch/h8300/Kconfig
+++ b/arch/h8300/Kconfig
@@ -23,6 +23,7 @@ config H8300
 	select HAVE_ARCH_KGDB
 	select HAVE_ARCH_HASH
 	select CPU_NO_EFFICIENT_FFS
+	select DMA_DIRECT_OPS
 
 config CPU_BIG_ENDIAN
 	def_bool y
diff --git a/arch/h8300/include/asm/Kbuild b/arch/h8300/include/asm/Kbuild
index bc077491d299..14bac06b7116 100644
--- a/arch/h8300/include/asm/Kbuild
+++ b/arch/h8300/include/asm/Kbuild
@@ -3,12 +3,12 @@ generic-y += barrier.h
 generic-y += bugs.h
 generic-y += cacheflush.h
 generic-y += checksum.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += delay.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += dma.h
+generic-y += dma-mapping.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += extable.h
diff --git a/arch/h8300/include/asm/byteorder.h b/arch/h8300/include/asm/byteorder.h
index ecff2d1ca5a3..6eaa7ad5fc2c 100644
--- a/arch/h8300/include/asm/byteorder.h
+++ b/arch/h8300/include/asm/byteorder.h
@@ -2,7 +2,6 @@
 #ifndef __H8300_BYTEORDER_H__
 #define __H8300_BYTEORDER_H__
 
-#define __BIG_ENDIAN __ORDER_BIG_ENDIAN__
 #include <linux/byteorder/big_endian.h>
 
 #endif
diff --git a/arch/h8300/include/asm/dma-mapping.h b/arch/h8300/include/asm/dma-mapping.h
deleted file mode 100644
index 21bb1fc3a6f1..000000000000
--- a/arch/h8300/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _H8300_DMA_MAPPING_H
-#define _H8300_DMA_MAPPING_H
-
-extern const struct dma_map_ops h8300_dma_map_ops;
-
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &h8300_dma_map_ops;
-}
-
-#endif
diff --git a/arch/h8300/include/asm/thread_info.h b/arch/h8300/include/asm/thread_info.h
index 072b92c0d8b5..0cdaa302d3d2 100644
--- a/arch/h8300/include/asm/thread_info.h
+++ b/arch/h8300/include/asm/thread_info.h
@@ -46,9 +46,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 static inline struct thread_info *current_thread_info(void)
 {
diff --git a/arch/h8300/kernel/Makefile b/arch/h8300/kernel/Makefile
index b62e830525c6..307aa51576dd 100644
--- a/arch/h8300/kernel/Makefile
+++ b/arch/h8300/kernel/Makefile
@@ -7,7 +7,7 @@ extra-y := vmlinux.lds
 
 obj-y := process.o traps.o ptrace.o \
 	 signal.o setup.o syscalls.o \
-	 irq.o entry.o dma.o
+	 irq.o entry.o
 
 obj-$(CONFIG_ROMKERNEL) += head_rom.o
 obj-$(CONFIG_RAMKERNEL) += head_ram.o
diff --git a/arch/h8300/kernel/dma.c b/arch/h8300/kernel/dma.c
deleted file mode 100644
index 225dd0a188dc..000000000000
--- a/arch/h8300/kernel/dma.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file COPYING in the main directory of this archive
- * for more details.
- */
-
-#include <linux/dma-mapping.h>
-#include <linux/kernel.h>
-#include <linux/scatterlist.h>
-#include <linux/module.h>
-#include <asm/pgalloc.h>
-
-static void *dma_alloc(struct device *dev, size_t size,
-		       dma_addr_t *dma_handle, gfp_t gfp,
-		       unsigned long attrs)
-{
-	void *ret;
-
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
-
-	if (dev == NULL || (*dev->dma_mask < 0xffffffff))
-		gfp |= GFP_DMA;
-	ret = (void *)__get_free_pages(gfp, get_order(size));
-
-	if (ret != NULL) {
-		memset(ret, 0, size);
-		*dma_handle = virt_to_phys(ret);
-	}
-	return ret;
-}
-
-static void dma_free(struct device *dev, size_t size,
-		     void *vaddr, dma_addr_t dma_handle,
-		     unsigned long attrs)
-
-{
-	free_pages((unsigned long)vaddr, get_order(size));
-}
-
-static dma_addr_t map_page(struct device *dev, struct page *page,
-				  unsigned long offset, size_t size,
-				  enum dma_data_direction direction,
-				  unsigned long attrs)
-{
-	return page_to_phys(page) + offset;
-}
-
-static int map_sg(struct device *dev, struct scatterlist *sgl,
-		  int nents, enum dma_data_direction direction,
-		  unsigned long attrs)
-{
-	struct scatterlist *sg;
-	int i;
-
-	for_each_sg(sgl, sg, nents, i) {
-		sg->dma_address = sg_phys(sg);
-	}
-
-	return nents;
-}
-
-const struct dma_map_ops h8300_dma_map_ops = {
-	.alloc = dma_alloc,
-	.free = dma_free,
-	.map_page = map_page,
-	.map_sg = map_sg,
-};
-EXPORT_SYMBOL(h8300_dma_map_ops);
diff --git a/arch/hexagon/include/asm/Kbuild b/arch/hexagon/include/asm/Kbuild
index 4a23e527d651..e9743f689fb8 100644
--- a/arch/hexagon/include/asm/Kbuild
+++ b/arch/hexagon/include/asm/Kbuild
@@ -2,7 +2,6 @@
 generic-y += barrier.h
 generic-y += bug.h
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/hexagon/include/asm/dma-mapping.h b/arch/hexagon/include/asm/dma-mapping.h
index 5208de242e79..263f6acbfb0f 100644
--- a/arch/hexagon/include/asm/dma-mapping.h
+++ b/arch/hexagon/include/asm/dma-mapping.h
@@ -37,11 +37,4 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 	return dma_ops;
 }
 
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return 0;
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
 #endif
diff --git a/arch/hexagon/include/asm/io.h b/arch/hexagon/include/asm/io.h
index 66f5e9a61efc..9e8621d94ee9 100644
--- a/arch/hexagon/include/asm/io.h
+++ b/arch/hexagon/include/asm/io.h
@@ -330,8 +330,6 @@ static inline void outsl(unsigned long port, const void *buffer, int count)
 	}
 }
 
-#define flush_write_buffers() do { } while (0)
-
 #endif /* __KERNEL__ */
 
 #endif
diff --git a/arch/hexagon/include/asm/thread_info.h b/arch/hexagon/include/asm/thread_info.h
index b80fe1db7b64..f41f9c6f0e31 100644
--- a/arch/hexagon/include/asm/thread_info.h
+++ b/arch/hexagon/include/asm/thread_info.h
@@ -84,9 +84,6 @@ struct thread_info {
 	.regs = NULL,			\
 }
 
-#define init_thread_info        (init_thread_union.thread_info)
-#define init_stack              (init_thread_union.stack)
-
 /* Tacky preprocessor trickery */
 #define	qqstr(s) qstr(s)
 #define qstr(s) #s
diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c
index 546792d176a4..ad8347c29dcf 100644
--- a/arch/hexagon/kernel/dma.c
+++ b/arch/hexagon/kernel/dma.c
@@ -19,6 +19,7 @@
  */
 
 #include <linux/dma-mapping.h>
+#include <linux/dma-direct.h>
 #include <linux/bootmem.h>
 #include <linux/genalloc.h>
 #include <asm/dma-mapping.h>
diff --git a/arch/hexagon/kernel/vmlinux.lds.S b/arch/hexagon/kernel/vmlinux.lds.S
index ec87e67feb19..ad69d181c939 100644
--- a/arch/hexagon/kernel/vmlinux.lds.S
+++ b/arch/hexagon/kernel/vmlinux.lds.S
@@ -22,6 +22,8 @@
 #include <asm/asm-offsets.h>	/*  Most of the kernel defines are here  */
 #include <asm/mem-layout.h>	/*  except for page_offset  */
 #include <asm/cache.h>		/*  and now we're pulling cache line size  */
+#include <asm/thread_info.h>	/*  and we need THREAD_SIZE too */
+
 OUTPUT_ARCH(hexagon)
 ENTRY(stext)
 
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 49583c5a5d44..bbe12a038d21 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -33,6 +33,7 @@ config IA64
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_VIRT_CPU_ACCOUNTING
+	select ARCH_HAS_DMA_MARK_CLEAN
 	select ARCH_HAS_SG_CHAIN
 	select VIRT_TO_BUS
 	select ARCH_DISCARD_MEMBLOCK
@@ -43,7 +44,7 @@ config IA64
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select GENERIC_IOMAP
 	select GENERIC_SMP_IDLE_THREAD
-	select ARCH_INIT_TASK
+	select ARCH_TASK_STRUCT_ON_STACK
 	select ARCH_TASK_STRUCT_ALLOCATOR
 	select ARCH_THREAD_STACK_ALLOCATOR
 	select ARCH_CLOCKSOURCE_DATA
@@ -65,7 +66,7 @@ config 64BIT
 	select ATA_NONSTANDARD if ATA
 	default y
 
-config ZONE_DMA
+config ZONE_DMA32
 	def_bool y
 	depends on !IA64_SGI_SN2
 
@@ -145,6 +146,7 @@ config IA64_GENERIC
 	bool "generic"
 	select NUMA
 	select ACPI_NUMA
+	select DMA_DIRECT_OPS
 	select SWIOTLB
 	select PCI_MSI
 	help
@@ -165,6 +167,7 @@ config IA64_GENERIC
 
 config IA64_DIG
 	bool "DIG-compliant"
+	select DMA_DIRECT_OPS
 	select SWIOTLB
 
 config IA64_DIG_VTD
@@ -180,6 +183,7 @@ config IA64_HP_ZX1
 
 config IA64_HP_ZX1_SWIOTLB
 	bool "HP-zx1/sx1000 with software I/O TLB"
+	select DMA_DIRECT_OPS
 	select SWIOTLB
 	help
 	  Build a kernel that runs on HP zx1 and sx1000 systems even when they
@@ -203,6 +207,7 @@ config IA64_SGI_UV
 	bool "SGI-UV"
 	select NUMA
 	select ACPI_NUMA
+	select DMA_DIRECT_OPS
 	select SWIOTLB
 	help
 	  Selecting this option will optimize the kernel for use on UV based
@@ -213,6 +218,7 @@ config IA64_SGI_UV
 
 config IA64_HP_SIM
 	bool "Ski-simulator"
+	select DMA_DIRECT_OPS
 	select SWIOTLB
 	depends on !PM
 
diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile
index c100d780f1eb..2dd7f519ad0b 100644
--- a/arch/ia64/Makefile
+++ b/arch/ia64/Makefile
@@ -42,7 +42,7 @@ $(error Sorry, you need a newer version of the assember, one that is built from
 endif
 
 KBUILD_CFLAGS += $(cflags-y)
-head-y := arch/ia64/kernel/head.o arch/ia64/kernel/init_task.o
+head-y := arch/ia64/kernel/head.o
 
 libs-y				+= arch/ia64/lib/
 core-y				+= arch/ia64/kernel/ arch/ia64/mm/
diff --git a/arch/ia64/hp/common/hwsw_iommu.c b/arch/ia64/hp/common/hwsw_iommu.c
index 63d8e1d2477f..58969039bed2 100644
--- a/arch/ia64/hp/common/hwsw_iommu.c
+++ b/arch/ia64/hp/common/hwsw_iommu.c
@@ -19,7 +19,7 @@
 #include <linux/export.h>
 #include <asm/machvec.h>
 
-extern const struct dma_map_ops sba_dma_ops, swiotlb_dma_ops;
+extern const struct dma_map_ops sba_dma_ops;
 
 /* swiotlb declarations & definitions: */
 extern int swiotlb_late_init_with_default_size (size_t size);
diff --git a/arch/ia64/include/asm/Kbuild b/arch/ia64/include/asm/Kbuild
index 1d7641f891e1..6dd867873364 100644
--- a/arch/ia64/include/asm/Kbuild
+++ b/arch/ia64/include/asm/Kbuild
@@ -1,4 +1,3 @@
-generic-y += clkdev.h
 generic-y += exec.h
 generic-y += irq_work.h
 generic-y += mcs_spinlock.h
diff --git a/arch/ia64/include/asm/asm-prototypes.h b/arch/ia64/include/asm/asm-prototypes.h
index c711536674e3..a96689447a74 100644
--- a/arch/ia64/include/asm/asm-prototypes.h
+++ b/arch/ia64/include/asm/asm-prototypes.h
@@ -9,7 +9,7 @@
 #include <asm/page.h>
 #include <asm/pal.h>
 #include <asm/string.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <asm/unwind.h>
 #include <asm/xor.h>
 
diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h
index 762eeb0fcc1d..2524fb60fbc2 100644
--- a/arch/ia64/include/asm/atomic.h
+++ b/arch/ia64/include/asm/atomic.h
@@ -66,38 +66,35 @@ ATOMIC_OPS(add, +)
 ATOMIC_OPS(sub, -)
 
 #ifdef __OPTIMIZE__
-#define __ia64_atomic_const(i)	__builtin_constant_p(i) ?		\
+#define __ia64_atomic_const(i)						\
+	static const int __ia64_atomic_p = __builtin_constant_p(i) ?	\
 		((i) == 1 || (i) == 4 || (i) == 8 || (i) == 16 ||	\
-		 (i) == -1 || (i) == -4 || (i) == -8 || (i) == -16) : 0
+		 (i) == -1 || (i) == -4 || (i) == -8 || (i) == -16) : 0;\
+	__ia64_atomic_p
+#else
+#define __ia64_atomic_const(i)	0
+#endif
 
-#define atomic_add_return(i, v)						\
+#define atomic_add_return(i,v)						\
 ({									\
-	int __i = (i);							\
-	static const int __ia64_atomic_p = __ia64_atomic_const(i);	\
-	__ia64_atomic_p ? ia64_fetch_and_add(__i, &(v)->counter) :	\
-				ia64_atomic_add(__i, v);		\
+	int __ia64_aar_i = (i);						\
+	__ia64_atomic_const(i)						\
+		? ia64_fetch_and_add(__ia64_aar_i, &(v)->counter)	\
+		: ia64_atomic_add(__ia64_aar_i, v);			\
 })
 
-#define atomic_sub_return(i, v)						\
+#define atomic_sub_return(i,v)						\
 ({									\
-	int __i = (i);							\
-	static const int __ia64_atomic_p = __ia64_atomic_const(i);	\
-	__ia64_atomic_p ? ia64_fetch_and_add(-__i, &(v)->counter) :	\
-				ia64_atomic_sub(__i, v);		\
+	int __ia64_asr_i = (i);						\
+	__ia64_atomic_const(i)						\
+		? ia64_fetch_and_add(-__ia64_asr_i, &(v)->counter)	\
+		: ia64_atomic_sub(__ia64_asr_i, v);			\
 })
-#else
-#define atomic_add_return(i, v)	ia64_atomic_add(i, v)
-#define atomic_sub_return(i, v)	ia64_atomic_sub(i, v)
-#endif
 
 #define atomic_fetch_add(i,v)						\
 ({									\
 	int __ia64_aar_i = (i);						\
-	(__builtin_constant_p(i)					\
-	 && (   (__ia64_aar_i ==  1) || (__ia64_aar_i ==   4)		\
-	     || (__ia64_aar_i ==  8) || (__ia64_aar_i ==  16)		\
-	     || (__ia64_aar_i == -1) || (__ia64_aar_i ==  -4)		\
-	     || (__ia64_aar_i == -8) || (__ia64_aar_i == -16)))		\
+	__ia64_atomic_const(i)						\
 		? ia64_fetchadd(__ia64_aar_i, &(v)->counter, acq)	\
 		: ia64_atomic_fetch_add(__ia64_aar_i, v);		\
 })
@@ -105,11 +102,7 @@ ATOMIC_OPS(sub, -)
 #define atomic_fetch_sub(i,v)						\
 ({									\
 	int __ia64_asr_i = (i);						\
-	(__builtin_constant_p(i)					\
-	 && (   (__ia64_asr_i ==   1) || (__ia64_asr_i ==   4)		\
-	     || (__ia64_asr_i ==   8) || (__ia64_asr_i ==  16)		\
-	     || (__ia64_asr_i ==  -1) || (__ia64_asr_i ==  -4)		\
-	     || (__ia64_asr_i ==  -8) || (__ia64_asr_i == -16)))	\
+	__ia64_atomic_const(i)						\
 		? ia64_fetchadd(-__ia64_asr_i, &(v)->counter, acq)	\
 		: ia64_atomic_fetch_sub(__ia64_asr_i, v);		\
 })
@@ -170,11 +163,7 @@ ATOMIC64_OPS(sub, -)
 #define atomic64_add_return(i,v)					\
 ({									\
 	long __ia64_aar_i = (i);					\
-	(__builtin_constant_p(i)					\
-	 && (   (__ia64_aar_i ==  1) || (__ia64_aar_i ==   4)		\
-	     || (__ia64_aar_i ==  8) || (__ia64_aar_i ==  16)		\
-	     || (__ia64_aar_i == -1) || (__ia64_aar_i ==  -4)		\
-	     || (__ia64_aar_i == -8) || (__ia64_aar_i == -16)))		\
+	__ia64_atomic_const(i)						\
 		? ia64_fetch_and_add(__ia64_aar_i, &(v)->counter)	\
 		: ia64_atomic64_add(__ia64_aar_i, v);			\
 })
@@ -182,11 +171,7 @@ ATOMIC64_OPS(sub, -)
 #define atomic64_sub_return(i,v)					\
 ({									\
 	long __ia64_asr_i = (i);					\
-	(__builtin_constant_p(i)					\
-	 && (   (__ia64_asr_i ==   1) || (__ia64_asr_i ==   4)		\
-	     || (__ia64_asr_i ==   8) || (__ia64_asr_i ==  16)		\
-	     || (__ia64_asr_i ==  -1) || (__ia64_asr_i ==  -4)		\
-	     || (__ia64_asr_i ==  -8) || (__ia64_asr_i == -16)))	\
+	__ia64_atomic_const(i)						\
 		? ia64_fetch_and_add(-__ia64_asr_i, &(v)->counter)	\
 		: ia64_atomic64_sub(__ia64_asr_i, v);			\
 })
@@ -194,11 +179,7 @@ ATOMIC64_OPS(sub, -)
 #define atomic64_fetch_add(i,v)						\
 ({									\
 	long __ia64_aar_i = (i);					\
-	(__builtin_constant_p(i)					\
-	 && (   (__ia64_aar_i ==  1) || (__ia64_aar_i ==   4)		\
-	     || (__ia64_aar_i ==  8) || (__ia64_aar_i ==  16)		\
-	     || (__ia64_aar_i == -1) || (__ia64_aar_i ==  -4)		\
-	     || (__ia64_aar_i == -8) || (__ia64_aar_i == -16)))		\
+	__ia64_atomic_const(i)						\
 		? ia64_fetchadd(__ia64_aar_i, &(v)->counter, acq)	\
 		: ia64_atomic64_fetch_add(__ia64_aar_i, v);		\
 })
@@ -206,11 +187,7 @@ ATOMIC64_OPS(sub, -)
 #define atomic64_fetch_sub(i,v)						\
 ({									\
 	long __ia64_asr_i = (i);					\
-	(__builtin_constant_p(i)					\
-	 && (   (__ia64_asr_i ==   1) || (__ia64_asr_i ==   4)		\
-	     || (__ia64_asr_i ==   8) || (__ia64_asr_i ==  16)		\
-	     || (__ia64_asr_i ==  -1) || (__ia64_asr_i ==  -4)		\
-	     || (__ia64_asr_i ==  -8) || (__ia64_asr_i == -16)))	\
+	__ia64_atomic_const(i)						\
 		? ia64_fetchadd(-__ia64_asr_i, &(v)->counter, acq)	\
 		: ia64_atomic64_fetch_sub(__ia64_asr_i, v);		\
 })
diff --git a/arch/ia64/include/asm/bug.h b/arch/ia64/include/asm/bug.h
index bd3eeb8d1cfa..66b37a532765 100644
--- a/arch/ia64/include/asm/bug.h
+++ b/arch/ia64/include/asm/bug.h
@@ -4,7 +4,11 @@
 
 #ifdef CONFIG_BUG
 #define ia64_abort()	__builtin_trap()
-#define BUG() do { printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__); ia64_abort(); } while (0)
+#define BUG() do {						\
+	printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__);	\
+	barrier_before_unreachable();				\
+	ia64_abort();						\
+} while (0)
 
 /* should this BUG be made generic? */
 #define HAVE_ARCH_BUG
diff --git a/arch/ia64/include/asm/dma-mapping.h b/arch/ia64/include/asm/dma-mapping.h
index c1bab526a046..76e4d6632d68 100644
--- a/arch/ia64/include/asm/dma-mapping.h
+++ b/arch/ia64/include/asm/dma-mapping.h
@@ -8,7 +8,6 @@
  */
 #include <asm/machvec.h>
 #include <linux/scatterlist.h>
-#include <asm/swiotlb.h>
 #include <linux/dma-debug.h>
 
 #define ARCH_HAS_DMA_GET_REQUIRED_MASK
@@ -27,22 +26,4 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 	return platform_dma_get_ops(NULL);
 }
 
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return 0;
-
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return paddr;
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
-{
-	return daddr;
-}
-
 #endif /* _ASM_IA64_DMA_MAPPING_H */
diff --git a/arch/ia64/include/asm/dma.h b/arch/ia64/include/asm/dma.h
index 186850eec934..23604d6a2cb2 100644
--- a/arch/ia64/include/asm/dma.h
+++ b/arch/ia64/include/asm/dma.h
@@ -20,6 +20,4 @@ extern unsigned long MAX_DMA_ADDRESS;
 
 #define free_dma(x)
 
-void dma_mark_clean(void *addr, size_t size);
-
 #endif /* _ASM_IA64_DMA_H */
diff --git a/arch/ia64/include/asm/sections.h b/arch/ia64/include/asm/sections.h
index f3481408594e..cea15f2dd38d 100644
--- a/arch/ia64/include/asm/sections.h
+++ b/arch/ia64/include/asm/sections.h
@@ -27,6 +27,8 @@ extern char __start_gate_brl_fsys_bubble_down_patchlist[], __end_gate_brl_fsys_b
 extern char __start_unwind[], __end_unwind[];
 extern char __start_ivt_text[], __end_ivt_text[];
 
+#define HAVE_DEREFERENCE_FUNCTION_DESCRIPTOR 1
+
 #undef dereference_function_descriptor
 static inline void *dereference_function_descriptor(void *ptr)
 {
@@ -38,6 +40,12 @@ static inline void *dereference_function_descriptor(void *ptr)
 	return ptr;
 }
 
+#undef dereference_kernel_function_descriptor
+static inline void *dereference_kernel_function_descriptor(void *ptr)
+{
+	if (ptr < (void *)__start_opd || ptr >= (void *)__end_opd)
+		return ptr;
+	return dereference_function_descriptor(ptr);
+}
 
 #endif /* _ASM_IA64_SECTIONS_H */
-
diff --git a/arch/ia64/include/asm/swiotlb.h b/arch/ia64/include/asm/swiotlb.h
deleted file mode 100644
index 841e2c7d0b21..000000000000
--- a/arch/ia64/include/asm/swiotlb.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef ASM_IA64__SWIOTLB_H
-#define ASM_IA64__SWIOTLB_H
-
-#include <linux/dma-mapping.h>
-#include <linux/swiotlb.h>
-
-#ifdef CONFIG_SWIOTLB
-extern int swiotlb;
-extern void pci_swiotlb_init(void);
-#else
-#define swiotlb 0
-static inline void pci_swiotlb_init(void)
-{
-}
-#endif
-
-#endif /* ASM_IA64__SWIOTLB_H */
diff --git a/arch/ia64/include/asm/thread_info.h b/arch/ia64/include/asm/thread_info.h
index 1d172a4119a7..64a1011f6812 100644
--- a/arch/ia64/include/asm/thread_info.h
+++ b/arch/ia64/include/asm/thread_info.h
@@ -12,6 +12,8 @@
 #include <asm/processor.h>
 #include <asm/ptrace.h>
 
+#define THREAD_SIZE			KERNEL_STACK_SIZE
+
 #ifndef __ASSEMBLY__
 
 /*
@@ -41,8 +43,6 @@ struct thread_info {
 #endif
 };
 
-#define THREAD_SIZE			KERNEL_STACK_SIZE
-
 #define INIT_THREAD_INFO(tsk)			\
 {						\
 	.task		= &tsk,			\
diff --git a/arch/ia64/include/uapi/asm/Kbuild b/arch/ia64/include/uapi/asm/Kbuild
index f5c6967a93bb..c0527cfc48f0 100644
--- a/arch/ia64/include/uapi/asm/Kbuild
+++ b/arch/ia64/include/uapi/asm/Kbuild
@@ -3,3 +3,4 @@ include include/uapi/asm-generic/Kbuild.asm
 
 generic-y += bpf_perf_event.h
 generic-y += kvm_para.h
+generic-y += poll.h
diff --git a/arch/ia64/include/uapi/asm/poll.h b/arch/ia64/include/uapi/asm/poll.h
deleted file mode 100644
index b7132a305a47..000000000000
--- a/arch/ia64/include/uapi/asm/poll.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#include <asm-generic/poll.h>
diff --git a/arch/ia64/include/uapi/asm/siginfo.h b/arch/ia64/include/uapi/asm/siginfo.h
index f3a02a10c3a3..5aa454ed89db 100644
--- a/arch/ia64/include/uapi/asm/siginfo.h
+++ b/arch/ia64/include/uapi/asm/siginfo.h
@@ -11,77 +11,8 @@
 
 #define __ARCH_SI_PREAMBLE_SIZE	(4 * sizeof(int))
 
-#define HAVE_ARCH_SIGINFO_T
-#define HAVE_ARCH_COPY_SIGINFO_TO_USER
-
 #include <asm-generic/siginfo.h>
 
-typedef struct siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-	int __pad0;
-
-	union {
-		int _pad[SI_PAD_SIZE];
-
-		/* kill() */
-		struct {
-			pid_t _pid;		/* sender's pid */
-			uid_t _uid;		/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			timer_t _tid;		/* timer id */
-			int _overrun;		/* overrun count */
-			char _pad[sizeof(__ARCH_SI_UID_T) - sizeof(int)];
-			sigval_t _sigval;	/* must overlay ._rt._sigval! */
-			int _sys_private;	/* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			pid_t _pid;		/* sender's pid */
-			uid_t _uid;		/* sender's uid */
-			sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			pid_t _pid;		/* which child */
-			uid_t _uid;		/* sender's uid */
-			int _status;		/* exit code */
-			clock_t _utime;
-			clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			void __user *_addr;	/* faulting insn/memory ref. */
-			int _imm;		/* immediate value for "break" */
-			unsigned int _flags;	/* see below */
-			unsigned long _isr;	/* isr */
-			short _addr_lsb;	/* lsb of faulting address */
-			union {
-				/* used when si_code=SEGV_BNDERR */
-				struct {
-					void __user *_lower;
-					void __user *_upper;
-				} _addr_bnd;
-				/* used when si_code=SEGV_PKUERR */
-				__u32 _pkey;
-			};
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			long _band;	/* POLL_IN, POLL_OUT, POLL_MSG (XPG requires a "long") */
-			int _fd;
-		} _sigpoll;
-	} _sifields;
-} siginfo_t;
-
 #define si_imm		_sifields._sigfault._imm	/* as per UNIX SysV ABI spec */
 #define si_flags	_sifields._sigfault._flags
 /*
@@ -96,38 +27,11 @@ typedef struct siginfo {
 #define __ISR_VALID_BIT	0
 #define __ISR_VALID	(1 << __ISR_VALID_BIT)
 
-/*
- * SIGILL si_codes
- */
-#define ILL_BADIADDR	9	/* unimplemented instruction address */
-#define __ILL_BREAK	10	/* illegal break */
-#define __ILL_BNDMOD	11	/* bundle-update (modification) in progress */
-#undef NSIGILL
-#define NSIGILL		11
-
 /*
  * SIGFPE si_codes
  */
 #ifdef __KERNEL__
 #define FPE_FIXME	0	/* Broken dup of SI_USER */
 #endif /* __KERNEL__ */
-#define __FPE_DECOVF	9	/* decimal overflow */
-#define __FPE_DECDIV	10	/* decimal division by zero */
-#define __FPE_DECERR	11	/* packed decimal error */
-#define __FPE_INVASC	12	/* invalid ASCII digit */
-#define __FPE_INVDEC	13	/* invalid decimal digit */
-#undef NSIGFPE
-#define NSIGFPE		13
-
-/*
- * SIGSEGV si_codes
- */
-#define __SEGV_PSTKOVF	4	/* paragraph stack overflow */
-#undef NSIGSEGV
-#define NSIGSEGV	4
-
-#undef NSIGTRAP
-#define NSIGTRAP	4
-
 
 #endif /* _UAPI_ASM_IA64_SIGINFO_H */
diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile
index 14ad79f394e5..498f3da3f225 100644
--- a/arch/ia64/kernel/Makefile
+++ b/arch/ia64/kernel/Makefile
@@ -7,7 +7,7 @@ ifdef CONFIG_DYNAMIC_FTRACE
 CFLAGS_REMOVE_ftrace.o = -pg
 endif
 
-extra-y	:= head.o init_task.o vmlinux.lds
+extra-y	:= head.o vmlinux.lds
 
 obj-y := entry.o efi.o efi_stub.o gate-data.o fsys.o ia64_ksyms.o irq.o irq_ia64.o	\
 	 irq_lsapic.o ivt.o machvec.o pal.o patch.o process.o perfmon.o ptrace.o sal.o		\
@@ -41,7 +41,6 @@ ifneq ($(CONFIG_IA64_ESI),)
 obj-y				+= esi_stub.o	# must be in kernel proper
 endif
 obj-$(CONFIG_INTEL_IOMMU)	+= pci-dma.o
-obj-$(CONFIG_SWIOTLB)		+= pci-swiotlb.o
 
 obj-$(CONFIG_BINFMT_ELF)	+= elfcore.o
 
diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c
index 1d29b2f8726b..1dacbf5e9e09 100644
--- a/arch/ia64/kernel/acpi.c
+++ b/arch/ia64/kernel/acpi.c
@@ -504,6 +504,11 @@ acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
 	if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
 		return -1;
 
+	if (num_node_memblks >= NR_NODE_MEMBLKS) {
+		pr_err("NUMA: too many memblk ranges\n");
+		return -EINVAL;
+	}
+
 	/* record this node in proximity bitmap */
 	pxm_bit_set(pxm);
 
diff --git a/arch/ia64/kernel/dma-mapping.c b/arch/ia64/kernel/dma-mapping.c
index 7a82c9259609..f2d57e66fd86 100644
--- a/arch/ia64/kernel/dma-mapping.c
+++ b/arch/ia64/kernel/dma-mapping.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/dma-mapping.h>
+#include <linux/swiotlb.h>
 #include <linux/export.h>
 
 /* Set this to 1 if there is a HW IOMMU in the system */
@@ -23,3 +24,11 @@ const struct dma_map_ops *dma_get_ops(struct device *dev)
 	return dma_ops;
 }
 EXPORT_SYMBOL(dma_get_ops);
+
+#ifdef CONFIG_SWIOTLB
+void __init swiotlb_dma_init(void)
+{
+	dma_ops = &swiotlb_dma_ops;
+	swiotlb_init(1);
+}
+#endif
diff --git a/arch/ia64/kernel/err_inject.c b/arch/ia64/kernel/err_inject.c
index 85bba43e7d5d..8b5b8e6bc9d9 100644
--- a/arch/ia64/kernel/err_inject.c
+++ b/arch/ia64/kernel/err_inject.c
@@ -117,7 +117,7 @@ store_call_start(struct device *dev, struct device_attribute *attr,
 
 #ifdef ERR_INJ_DEBUG
 	printk(KERN_DEBUG "Returns: status=%d,\n", (int)status[cpu]);
-	printk(KERN_DEBUG "capapbilities=%lx,\n", capabilities[cpu]);
+	printk(KERN_DEBUG "capabilities=%lx,\n", capabilities[cpu]);
 	printk(KERN_DEBUG "resources=%lx\n", resources[cpu]);
 #endif
 	return size;
@@ -142,7 +142,7 @@ store_virtual_to_phys(struct device *dev, struct device_attribute *attr,
 	u64 virt_addr=simple_strtoull(buf, NULL, 16);
 	int ret;
 
-	ret = get_user_pages(virt_addr, 1, FOLL_WRITE, NULL, NULL);
+	ret = get_user_pages_fast(virt_addr, 1, FOLL_WRITE, NULL);
 	if (ret<=0) {
 #ifdef ERR_INJ_DEBUG
 		printk("Virtual address %lx is not existing.\n",virt_addr);
diff --git a/arch/ia64/kernel/init_task.c b/arch/ia64/kernel/init_task.c
deleted file mode 100644
index 8df9245e29d9..000000000000
--- a/arch/ia64/kernel/init_task.c
+++ /dev/null
@@ -1,44 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * This is where we statically allocate and initialize the initial
- * task.
- *
- * Copyright (C) 1999, 2002-2003 Hewlett-Packard Co
- *	David Mosberger-Tang <davidm@hpl.hp.com>
- */
-
-#include <linux/init.h>
-#include <linux/mm.h>
-#include <linux/fs.h>
-#include <linux/module.h>
-#include <linux/sched.h>
-#include <linux/init_task.h>
-#include <linux/mqueue.h>
-
-#include <linux/uaccess.h>
-#include <asm/pgtable.h>
-
-static struct signal_struct init_signals = INIT_SIGNALS(init_signals);
-static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand);
-/*
- * Initial task structure.
- *
- * We need to make sure that this is properly aligned due to the way process stacks are
- * handled. This is done by having a special ".data..init_task" section...
- */
-#define init_thread_info	init_task_mem.s.thread_info
-#define init_stack		init_task_mem.stack
-
-union {
-	struct {
-		struct task_struct task;
-		struct thread_info thread_info;
-	} s;
-	unsigned long stack[KERNEL_STACK_SIZE/sizeof (unsigned long)];
-} init_task_mem asm ("init_task") __init_task_data =
-	{{
-	.task =		INIT_TASK(init_task_mem.s.task),
-	.thread_info =	INIT_THREAD_INFO(init_task_mem.s.task)
-}};
-
-EXPORT_SYMBOL(init_task);
diff --git a/arch/ia64/kernel/module.c b/arch/ia64/kernel/module.c
index 853b5611a894..326448f9df16 100644
--- a/arch/ia64/kernel/module.c
+++ b/arch/ia64/kernel/module.c
@@ -36,6 +36,7 @@
 
 #include <asm/patch.h>
 #include <asm/unaligned.h>
+#include <asm/sections.h>
 
 #define ARCH_MODULE_DEBUG 0
 
@@ -918,3 +919,14 @@ module_arch_cleanup (struct module *mod)
 	if (mod->arch.core_unw_table)
 		unw_remove_unwind_table(mod->arch.core_unw_table);
 }
+
+void *dereference_module_function_descriptor(struct module *mod, void *ptr)
+{
+	Elf64_Shdr *opd = mod->arch.opd;
+
+	if (ptr < (void *)opd->sh_addr ||
+			ptr >= (void *)(opd->sh_addr + opd->sh_size))
+		return ptr;
+
+	return dereference_function_descriptor(ptr);
+}
diff --git a/arch/ia64/kernel/pci-dma.c b/arch/ia64/kernel/pci-dma.c
index 3ba87c22dfbc..b5df084c0af4 100644
--- a/arch/ia64/kernel/pci-dma.c
+++ b/arch/ia64/kernel/pci-dma.c
@@ -12,12 +12,7 @@
 #include <asm/iommu.h>
 #include <asm/machvec.h>
 #include <linux/dma-mapping.h>
-
-
-#ifdef CONFIG_INTEL_IOMMU
-
 #include <linux/kernel.h>
-
 #include <asm/page.h>
 
 dma_addr_t bad_dma_address __read_mostly;
@@ -104,8 +99,14 @@ void __init pci_iommu_alloc(void)
 	detect_intel_iommu();
 
 #ifdef CONFIG_SWIOTLB
-	pci_swiotlb_init();
-#endif
+	if (!iommu_detected) {
+#ifdef CONFIG_IA64_GENERIC
+		printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
+		machvec_init("dig");
+		swiotlb_dma_init();
+#else
+		panic("Unable to find Intel IOMMU");
+#endif /* CONFIG_IA64_GENERIC */
+	}
+#endif /* CONFIG_SWIOTLB */
 }
-
-#endif
diff --git a/arch/ia64/kernel/pci-swiotlb.c b/arch/ia64/kernel/pci-swiotlb.c
deleted file mode 100644
index 5e50939aa03e..000000000000
--- a/arch/ia64/kernel/pci-swiotlb.c
+++ /dev/null
@@ -1,68 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Glue code to lib/swiotlb.c */
-
-#include <linux/pci.h>
-#include <linux/gfp.h>
-#include <linux/cache.h>
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-
-#include <asm/swiotlb.h>
-#include <asm/dma.h>
-#include <asm/iommu.h>
-#include <asm/machvec.h>
-
-int swiotlb __read_mostly;
-EXPORT_SYMBOL(swiotlb);
-
-static void *ia64_swiotlb_alloc_coherent(struct device *dev, size_t size,
-					 dma_addr_t *dma_handle, gfp_t gfp,
-					 unsigned long attrs)
-{
-	if (dev->coherent_dma_mask != DMA_BIT_MASK(64))
-		gfp |= GFP_DMA;
-	return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
-}
-
-static void ia64_swiotlb_free_coherent(struct device *dev, size_t size,
-				       void *vaddr, dma_addr_t dma_addr,
-				       unsigned long attrs)
-{
-	swiotlb_free_coherent(dev, size, vaddr, dma_addr);
-}
-
-const struct dma_map_ops swiotlb_dma_ops = {
-	.alloc = ia64_swiotlb_alloc_coherent,
-	.free = ia64_swiotlb_free_coherent,
-	.map_page = swiotlb_map_page,
-	.unmap_page = swiotlb_unmap_page,
-	.map_sg = swiotlb_map_sg_attrs,
-	.unmap_sg = swiotlb_unmap_sg_attrs,
-	.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
-	.sync_single_for_device = swiotlb_sync_single_for_device,
-	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
-	.sync_sg_for_device = swiotlb_sync_sg_for_device,
-	.dma_supported = swiotlb_dma_supported,
-	.mapping_error = swiotlb_dma_mapping_error,
-};
-
-void __init swiotlb_dma_init(void)
-{
-	dma_ops = &swiotlb_dma_ops;
-	swiotlb_init(1);
-}
-
-void __init pci_swiotlb_init(void)
-{
-	if (!iommu_detected) {
-#ifdef CONFIG_IA64_GENERIC
-		swiotlb = 1;
-		printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
-		machvec_init("dig");
-		swiotlb_init(1);
-		dma_ops = &swiotlb_dma_ops;
-#else
-		panic("Unable to find Intel IOMMU");
-#endif
-	}
-}
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index 09f86ebfcc7b..8fb280e33114 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -1644,12 +1644,12 @@ pfm_write(struct file *file, const char __user *ubuf,
 	return -EINVAL;
 }
 
-static unsigned int
+static __poll_t
 pfm_poll(struct file *filp, poll_table * wait)
 {
 	pfm_context_t *ctx;
 	unsigned long flags;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	if (PFM_IS_FILE(filp) == 0) {
 		printk(KERN_ERR "perfmon: pfm_poll: bad magic [%d]\n", task_pid_nr(current));
@@ -1670,7 +1670,7 @@ pfm_poll(struct file *filp, poll_table * wait)
 	PROTECT_CTX(ctx, flags);
 
 	if (PFM_CTXQ_EMPTY(ctx) == 0)
-		mask =  POLLIN | POLLRDNORM;
+		mask =  EPOLLIN | EPOLLRDNORM;
 
 	UNPROTECT_CTX(ctx, flags);
 
@@ -2610,17 +2610,10 @@ pfm_get_task(pfm_context_t *ctx, pid_t pid, struct task_struct **task)
 	if (pid < 2) return -EPERM;
 
 	if (pid != task_pid_vnr(current)) {
-
-		read_lock(&tasklist_lock);
-
-		p = find_task_by_vpid(pid);
-
 		/* make sure task cannot go away while we operate on it */
-		if (p) get_task_struct(p);
-
-		read_unlock(&tasklist_lock);
-
-		if (p == NULL) return -ESRCH;
+		p = find_get_task_by_vpid(pid);
+		if (!p)
+			return -ESRCH;
 	}
 
 	ret = pfm_task_incompatible(ctx, p);
diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c
index dda0082056b3..968b5f33e725 100644
--- a/arch/ia64/kernel/process.c
+++ b/arch/ia64/kernel/process.c
@@ -13,7 +13,6 @@
 #include <linux/pm.h>
 #include <linux/elf.h>
 #include <linux/errno.h>
-#include <linux/kallsyms.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
@@ -69,7 +68,6 @@ void
 ia64_do_show_stack (struct unw_frame_info *info, void *arg)
 {
 	unsigned long ip, sp, bsp;
-	char buf[128];			/* don't make it so big that it overflows the stack! */
 
 	printk("\nCall Trace:\n");
 	do {
@@ -79,11 +77,9 @@ ia64_do_show_stack (struct unw_frame_info *info, void *arg)
 
 		unw_get_sp(info, &sp);
 		unw_get_bsp(info, &bsp);
-		snprintf(buf, sizeof(buf),
-			 " [<%016lx>] %%s\n"
+		printk(" [<%016lx>] %pS\n"
 			 "                                sp=%016lx bsp=%016lx\n",
-			 ip, sp, bsp);
-		print_symbol(buf, ip);
+			 ip, (void *)ip, sp, bsp);
 	} while (unw_unwind(info) >= 0);
 }
 
@@ -111,7 +107,7 @@ show_regs (struct pt_regs *regs)
 	printk("psr : %016lx ifs : %016lx ip  : [<%016lx>]    %s (%s)\n",
 	       regs->cr_ipsr, regs->cr_ifs, ip, print_tainted(),
 	       init_utsname()->release);
-	print_symbol("ip is at %s\n", ip);
+	printk("ip is at %pS\n", (void *)ip);
 	printk("unat: %016lx pfs : %016lx rsc : %016lx\n",
 	       regs->ar_unat, regs->ar_pfs, regs->ar_rsc);
 	printk("rnat: %016lx bsps: %016lx pr  : %016lx\n",
diff --git a/arch/ia64/kernel/signal.c b/arch/ia64/kernel/signal.c
index a254cc98f95c..54547c7cf8a2 100644
--- a/arch/ia64/kernel/signal.c
+++ b/arch/ia64/kernel/signal.c
@@ -105,58 +105,6 @@ restore_sigcontext (struct sigcontext __user *sc, struct sigscratch *scr)
 	return err;
 }
 
-int
-copy_siginfo_to_user (siginfo_t __user *to, const siginfo_t *from)
-{
-	if (!access_ok(VERIFY_WRITE, to, sizeof(siginfo_t)))
-		return -EFAULT;
-	if (from->si_code < 0) {
-		if (__copy_to_user(to, from, sizeof(siginfo_t)))
-			return -EFAULT;
-		return 0;
-	} else {
-		int err;
-
-		/*
-		 * If you change siginfo_t structure, please be sure this code is fixed
-		 * accordingly.  It should never copy any pad contained in the structure
-		 * to avoid security leaks, but must copy the generic 3 ints plus the
-		 * relevant union member.
-		 */
-		err = __put_user(from->si_signo, &to->si_signo);
-		err |= __put_user(from->si_errno, &to->si_errno);
-		err |= __put_user(from->si_code, &to->si_code);
-		switch (siginfo_layout(from->si_signo, from->si_code)) {
-		      case SIL_FAULT:
-			err |= __put_user(from->si_flags, &to->si_flags);
-			err |= __put_user(from->si_isr, &to->si_isr);
-		      case SIL_POLL:
-			err |= __put_user(from->si_addr, &to->si_addr);
-			err |= __put_user(from->si_imm, &to->si_imm);
-			break;
-		      case SIL_TIMER:
-			err |= __put_user(from->si_tid, &to->si_tid);
-			err |= __put_user(from->si_overrun, &to->si_overrun);
-			err |= __put_user(from->si_ptr, &to->si_ptr);
-			break;
-		      case SIL_RT:
-			err |= __put_user(from->si_uid, &to->si_uid);
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_ptr, &to->si_ptr);
-			break;
-		      case SIL_CHLD:
-			err |= __put_user(from->si_utime, &to->si_utime);
-			err |= __put_user(from->si_stime, &to->si_stime);
-			err |= __put_user(from->si_status, &to->si_status);
-		      case SIL_KILL:
-			err |= __put_user(from->si_uid, &to->si_uid);
-			err |= __put_user(from->si_pid, &to->si_pid);
-			break;
-		}
-		return err;
-	}
-}
-
 long
 ia64_rt_sigreturn (struct sigscratch *scr)
 {
diff --git a/arch/ia64/kernel/vmlinux.lds.S b/arch/ia64/kernel/vmlinux.lds.S
index 58db59da0bd8..0da58cf8e213 100644
--- a/arch/ia64/kernel/vmlinux.lds.S
+++ b/arch/ia64/kernel/vmlinux.lds.S
@@ -3,6 +3,7 @@
 #include <asm/cache.h>
 #include <asm/ptrace.h>
 #include <asm/pgtable.h>
+#include <asm/thread_info.h>
 
 #include <asm-generic/vmlinux.lds.h>
 
@@ -108,7 +109,9 @@ SECTIONS {
 	RODATA
 
 	.opd : AT(ADDR(.opd) - LOAD_OFFSET) {
+		__start_opd = .;
 		*(.opd)
+		__end_opd = .;
 	}
 
 	/*
diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c
index 52715a71aede..7d64b30913d1 100644
--- a/arch/ia64/mm/contig.c
+++ b/arch/ia64/mm/contig.c
@@ -237,9 +237,9 @@ paging_init (void)
 	unsigned long max_zone_pfns[MAX_NR_ZONES];
 
 	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
 	max_dma = virt_to_phys((void *) MAX_DMA_ADDRESS) >> PAGE_SHIFT;
-	max_zone_pfns[ZONE_DMA] = max_dma;
+	max_zone_pfns[ZONE_DMA32] = max_dma;
 #endif
 	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
 
diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index 9b2d994cddf6..7d9bd20319ff 100644
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -38,7 +38,7 @@ struct early_node_data {
 	struct ia64_node_data *node_data;
 	unsigned long pernode_addr;
 	unsigned long pernode_size;
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
 	unsigned long num_dma_physpages;
 #endif
 	unsigned long min_pfn;
@@ -669,7 +669,7 @@ static __init int count_node_pages(unsigned long start, unsigned long len, int n
 {
 	unsigned long end = start + len;
 
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
 	if (start <= __pa(MAX_DMA_ADDRESS))
 		mem_data[node].num_dma_physpages +=
 			(min(end, __pa(MAX_DMA_ADDRESS)) - start) >>PAGE_SHIFT;
@@ -724,8 +724,8 @@ void __init paging_init(void)
 	}
 
 	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
-#ifdef CONFIG_ZONE_DMA
-	max_zone_pfns[ZONE_DMA] = max_dma;
+#ifdef CONFIG_ZONE_DMA32
+	max_zone_pfns[ZONE_DMA32] = max_dma;
 #endif
 	max_zone_pfns[ZONE_NORMAL] = max_pfn;
 	free_area_init_nodes(max_zone_pfns);
@@ -754,12 +754,14 @@ void arch_refresh_nodedata(int update_node, pg_data_t *update_pgdat)
 #endif
 
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
-int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
+int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
+		struct vmem_altmap *altmap)
 {
 	return vmemmap_populate_basepages(start, end, node);
 }
 
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+		struct vmem_altmap *altmap)
 {
 }
 #endif
diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index 7af4e05bb61e..18278b448530 100644
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -501,7 +501,7 @@ virtual_memmap_init(u64 start, u64 end, void *arg)
 	if (map_start < map_end)
 		memmap_init_zone((unsigned long)(map_end - map_start),
 				 args->nid, args->zone, page_to_pfn(map_start),
-				 MEMMAP_EARLY);
+				 MEMMAP_EARLY, NULL);
 	return 0;
 }
 
@@ -509,9 +509,10 @@ void __meminit
 memmap_init (unsigned long size, int nid, unsigned long zone,
 	     unsigned long start_pfn)
 {
-	if (!vmem_map)
-		memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY);
-	else {
+	if (!vmem_map) {
+		memmap_init_zone(size, nid, zone, start_pfn, MEMMAP_EARLY,
+				NULL);
+	} else {
 		struct page *start;
 		struct memmap_init_callback_data args;
 
@@ -647,13 +648,14 @@ mem_init (void)
 }
 
 #ifdef CONFIG_MEMORY_HOTPLUG
-int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
+int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
+		bool want_memblock)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 	int ret;
 
-	ret = __add_pages(nid, start_pfn, nr_pages, want_memblock);
+	ret = __add_pages(nid, start_pfn, nr_pages, altmap, want_memblock);
 	if (ret)
 		printk("%s: Problem encountered in __add_pages() as ret=%d\n",
 		       __func__,  ret);
@@ -662,7 +664,7 @@ int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
-int arch_remove_memory(u64 start, u64 size)
+int arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
@@ -670,7 +672,7 @@ int arch_remove_memory(u64 start, u64 size)
 	int ret;
 
 	zone = page_zone(pfn_to_page(start_pfn));
-	ret = __remove_pages(zone, start_pfn, nr_pages);
+	ret = __remove_pages(zone, start_pfn, nr_pages, altmap);
 	if (ret)
 		pr_warn("%s: Problem encountered in __remove_pages() as"
 			" ret=%d\n", __func__,  ret);
diff --git a/arch/ia64/scripts/unwcheck.py b/arch/ia64/scripts/unwcheck.py
index 89f3a1480a63..c55276e31b6b 100644
--- a/arch/ia64/scripts/unwcheck.py
+++ b/arch/ia64/scripts/unwcheck.py
@@ -16,7 +16,7 @@ import re
 import sys
 
 if len(sys.argv) != 2:
-    print "Usage: %s FILE" % sys.argv[0]
+    print("Usage: %s FILE" % sys.argv[0])
     sys.exit(2)
 
 readelf = os.getenv("READELF", "readelf")
@@ -29,7 +29,7 @@ def check_func (func, slots, rlen_sum):
         global num_errors
         num_errors += 1
         if not func: func = "[%#x-%#x]" % (start, end)
-        print "ERROR: %s: %lu slots, total region length = %lu" % (func, slots, rlen_sum)
+        print("ERROR: %s: %lu slots, total region length = %lu" % (func, slots, rlen_sum))
     return
 
 num_funcs = 0
@@ -43,23 +43,23 @@ for line in os.popen("%s -u %s" % (readelf, sys.argv[1])):
         check_func(func, slots, rlen_sum)
 
         func  = m.group(1)
-        start = long(m.group(2), 16)
-        end   = long(m.group(3), 16)
+        start = int(m.group(2), 16)
+        end   = int(m.group(3), 16)
         slots = 3 * (end - start) / 16
-        rlen_sum = 0L
+        rlen_sum = 0
         num_funcs += 1
     else:
         m = rlen_pattern.match(line)
         if m:
-            rlen_sum += long(m.group(1))
+            rlen_sum += int(m.group(1))
 check_func(func, slots, rlen_sum)
 
 if num_errors == 0:
-    print "No errors detected in %u functions." % num_funcs
+    print("No errors detected in %u functions." % num_funcs)
 else:
     if num_errors > 1:
         err="errors"
     else:
         err="error"
-    print "%u %s detected in %u functions." % (num_errors, err, num_funcs)
+    print("%u %s detected in %u functions." % (num_errors, err, num_funcs))
     sys.exit(1)
diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig
index 498398d915c1..dd84ee194579 100644
--- a/arch/m32r/Kconfig
+++ b/arch/m32r/Kconfig
@@ -19,7 +19,7 @@ config M32R
 	select MODULES_USE_ELF_RELA
 	select HAVE_DEBUG_STACKOVERFLOW
 	select CPU_NO_EFFICIENT_FFS
-	select DMA_NOOP_OPS
+	select DMA_DIRECT_OPS
 	select ARCH_NO_COHERENT_DMA_MMAP if !MMU
 
 config SBUS
diff --git a/arch/m32r/include/asm/Kbuild b/arch/m32r/include/asm/Kbuild
index 7e11b125c35e..985ef1d9f556 100644
--- a/arch/m32r/include/asm/Kbuild
+++ b/arch/m32r/include/asm/Kbuild
@@ -1,5 +1,5 @@
-generic-y += clkdev.h
 generic-y += current.h
+generic-y += dma-mapping.h
 generic-y += exec.h
 generic-y += extable.h
 generic-y += irq_work.h
diff --git a/arch/m32r/include/asm/dma-mapping.h b/arch/m32r/include/asm/dma-mapping.h
deleted file mode 100644
index 336ffe60814b..000000000000
--- a/arch/m32r/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_M32R_DMA_MAPPING_H
-#define _ASM_M32R_DMA_MAPPING_H
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/scatterlist.h>
-#include <linux/dma-debug.h>
-#include <linux/io.h>
-
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &dma_noop_ops;
-}
-
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return false;
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
-#endif /* _ASM_M32R_DMA_MAPPING_H */
diff --git a/arch/m32r/include/asm/io.h b/arch/m32r/include/asm/io.h
index 1b653bb16f9a..a4272d8f0d9c 100644
--- a/arch/m32r/include/asm/io.h
+++ b/arch/m32r/include/asm/io.h
@@ -191,8 +191,6 @@ static inline void _writel(unsigned long l, unsigned long addr)
 
 #define mmiowb()
 
-#define flush_write_buffers() do { } while (0)  /* M32R_FIXME */
-
 static inline void
 memset_io(volatile void __iomem *addr, unsigned char val, int count)
 {
diff --git a/arch/m32r/include/asm/thread_info.h b/arch/m32r/include/asm/thread_info.h
index b3a215b0ce0a..ba00f1032587 100644
--- a/arch/m32r/include/asm/thread_info.h
+++ b/arch/m32r/include/asm/thread_info.h
@@ -56,9 +56,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 static inline struct thread_info *current_thread_info(void)
 {
diff --git a/arch/m32r/include/uapi/asm/Kbuild b/arch/m32r/include/uapi/asm/Kbuild
index 451bf6071c6e..c3df55aeefe7 100644
--- a/arch/m32r/include/uapi/asm/Kbuild
+++ b/arch/m32r/include/uapi/asm/Kbuild
@@ -3,4 +3,5 @@ include include/uapi/asm-generic/Kbuild.asm
 
 generic-y += bpf_perf_event.h
 generic-y += kvm_para.h
+generic-y += poll.h
 generic-y += siginfo.h
diff --git a/arch/m32r/include/uapi/asm/poll.h b/arch/m32r/include/uapi/asm/poll.h
deleted file mode 100644
index b7132a305a47..000000000000
--- a/arch/m32r/include/uapi/asm/poll.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#include <asm-generic/poll.h>
diff --git a/arch/m32r/kernel/traps.c b/arch/m32r/kernel/traps.c
index b88a8dd14933..a6f300a208bd 100644
--- a/arch/m32r/kernel/traps.c
+++ b/arch/m32r/kernel/traps.c
@@ -115,14 +115,6 @@ static void set_eit_vector_entries(void)
 	_flush_cache_copyback_all();
 }
 
-void abort(void)
-{
-	BUG();
-
-	/* if that doesn't kill us, halt */
-	panic("Oops failed to kill thread");
-}
-
 void __init trap_init(void)
 {
 	set_eit_vector_entries();
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index 5b5fa9831b4d..52fa7fbdefa2 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -313,7 +313,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_PARPORT=m
@@ -454,7 +453,6 @@ CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PPS_CLIENT_PARPORT=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FB_CIRRUS=y
 CONFIG_FB_AMIGA=y
@@ -595,6 +593,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -624,6 +623,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -653,3 +653,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index 72a7764b74ed..b00fe19e1c04 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -311,7 +311,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_LOOP=y
@@ -422,7 +421,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_LOGO=y
@@ -554,6 +552,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -583,6 +582,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -612,3 +612,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index 884b43a2f0d9..2871d75b912f 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -311,7 +311,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_PARPORT=m
@@ -437,7 +436,6 @@ CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PPS_CLIENT_PARPORT=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FB_ATARI=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
@@ -576,6 +574,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -605,6 +604,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -634,3 +634,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index fcfa60d31499..2c90328433bf 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -309,7 +309,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_LOOP=y
@@ -420,7 +419,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_HID=m
 CONFIG_HIDRAW=y
 CONFIG_UHID=m
@@ -546,6 +544,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -575,6 +574,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -604,3 +604,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 9d597bbbbbfe..177c91d63826 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -311,7 +311,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_LOOP=y
@@ -425,7 +424,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_LOGO=y
@@ -556,6 +554,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -585,6 +584,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -614,3 +614,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index 45da20d1286c..efbcaffa30ed 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -313,7 +313,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_SWIM=m
@@ -447,7 +446,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FB_VALKYRIE=y
 CONFIG_FB_MAC=y
@@ -578,6 +576,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -607,6 +606,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -636,3 +636,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index fda880c10861..e78a205d266a 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -323,7 +323,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_PARPORT=m
@@ -504,7 +503,6 @@ CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PPS_CLIENT_PARPORT=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FB_CIRRUS=y
 CONFIG_FB_AMIGA=y
@@ -658,6 +656,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -687,6 +686,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -716,3 +716,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index 7d5e4863efec..0d42ecacfd7a 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -308,7 +308,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_LOOP=y
@@ -420,7 +419,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_HID=m
 CONFIG_HIDRAW=y
 CONFIG_UHID=m
@@ -546,6 +544,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -575,6 +574,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -604,3 +604,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 7763b71a9c49..8dc609c5b8aa 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -309,7 +309,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_LOOP=y
@@ -420,7 +419,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_HID=m
 CONFIG_HIDRAW=y
 CONFIG_UHID=m
@@ -546,6 +544,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -575,6 +574,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -604,3 +604,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index 17eaebfa3e19..11c96087fc5d 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -309,7 +309,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_PARPORT=m
@@ -437,7 +436,6 @@ CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PPS_CLIENT_PARPORT=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_LOGO=y
@@ -569,6 +567,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -598,6 +597,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -627,3 +627,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index d1cb7a04ae1d..ddba205ee1aa 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -306,7 +306,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_LOOP=y
@@ -419,7 +418,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_LOGO=y
@@ -548,6 +546,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -576,6 +575,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -605,3 +605,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index ea3a331c62d5..88d09a14c684 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -306,7 +306,6 @@ CONFIG_NET_DEVLINK=m
 # CONFIG_UEVENT_HELPER is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_TEST_ASYNC_DRIVER_PROBE=m
 CONFIG_CONNECTOR=m
 CONFIG_BLK_DEV_LOOP=y
@@ -419,7 +418,6 @@ CONFIG_NTP_PPS=y
 CONFIG_PPS_CLIENT_LDISC=m
 CONFIG_PTP_1588_CLOCK=m
 # CONFIG_HWMON is not set
-# CONFIG_RC_CORE is not set
 CONFIG_FB=y
 CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_LOGO=y
@@ -548,6 +546,7 @@ CONFIG_TEST_RHASHTABLE=m
 CONFIG_TEST_HASH=m
 CONFIG_TEST_USER_COPY=m
 CONFIG_TEST_BPF=m
+CONFIG_TEST_FIND_BIT=m
 CONFIG_TEST_FIRMWARE=m
 CONFIG_TEST_SYSCTL=m
 CONFIG_TEST_UDELAY=m
@@ -577,6 +576,7 @@ CONFIG_CRYPTO_RMD256=m
 CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
 CONFIG_CRYPTO_SHA3=m
+CONFIG_CRYPTO_SM3=m
 CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_AES_TI=m
@@ -606,3 +606,4 @@ CONFIG_CRYPTO_USER_API_AEAD=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC32_SELFTEST=m
 CONFIG_XZ_DEC_TEST=m
+CONFIG_STRING_SELFTEST=m
diff --git a/arch/m68k/include/asm/Kbuild b/arch/m68k/include/asm/Kbuild
index 59d6d0d38f67..88a9d27df1ac 100644
--- a/arch/m68k/include/asm/Kbuild
+++ b/arch/m68k/include/asm/Kbuild
@@ -1,5 +1,4 @@
 generic-y += barrier.h
-generic-y += clkdev.h
 generic-y += device.h
 generic-y += emergency-restart.h
 generic-y += exec.h
diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h
index dda58cfe8c22..93b47b1f6fb4 100644
--- a/arch/m68k/include/asm/bitops.h
+++ b/arch/m68k/include/asm/bitops.h
@@ -311,7 +311,6 @@ static inline int bfchg_mem_test_and_change_bit(int nr,
  *	functions.
  */
 #if defined(CONFIG_CPU_HAS_NO_BITFIELDS)
-#include <asm-generic/bitops/find.h>
 #include <asm-generic/bitops/ffz.h>
 #else
 
@@ -441,6 +440,8 @@ static inline unsigned long ffz(unsigned long word)
 
 #endif
 
+#include <asm-generic/bitops/find.h>
+
 #ifdef __KERNEL__
 
 #if defined(CONFIG_CPU_HAS_NO_BITFIELDS)
diff --git a/arch/m68k/include/asm/bug.h b/arch/m68k/include/asm/bug.h
index b7e2bf1ba4a6..275dca1435bf 100644
--- a/arch/m68k/include/asm/bug.h
+++ b/arch/m68k/include/asm/bug.h
@@ -8,16 +8,19 @@
 #ifndef CONFIG_SUN3
 #define BUG() do { \
 	pr_crit("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	barrier_before_unreachable(); \
 	__builtin_trap(); \
 } while (0)
 #else
 #define BUG() do { \
 	pr_crit("kernel BUG at %s:%d!\n", __FILE__, __LINE__); \
+	barrier_before_unreachable(); \
 	panic("BUG!"); \
 } while (0)
 #endif
 #else
 #define BUG() do { \
+	barrier_before_unreachable(); \
 	__builtin_trap(); \
 } while (0)
 #endif
diff --git a/arch/m68k/include/asm/macintosh.h b/arch/m68k/include/asm/macintosh.h
index f42c27400dbc..9b840c03ebb7 100644
--- a/arch/m68k/include/asm/macintosh.h
+++ b/arch/m68k/include/asm/macintosh.h
@@ -33,7 +33,7 @@ struct mac_model
 	char ide_type;
 	char scc_type;
 	char ether_type;
-	char nubus_type;
+	char expansion_type;
 	char floppy_type;
 };
 
@@ -73,8 +73,11 @@ struct mac_model
 #define MAC_ETHER_SONIC		1
 #define MAC_ETHER_MACE		2
 
-#define MAC_NO_NUBUS		0
-#define MAC_NUBUS		1
+#define MAC_EXP_NONE		0
+#define MAC_EXP_PDS		1 /* Accepts only a PDS card */
+#define MAC_EXP_NUBUS		2 /* Accepts only NuBus card(s) */
+#define MAC_EXP_PDS_NUBUS	3 /* Accepts PDS card and/or NuBus card(s) */
+#define MAC_EXP_PDS_COMM	4 /* Accepts PDS card or Comm Slot card */
 
 #define MAC_FLOPPY_IWM		0
 #define MAC_FLOPPY_SWIM_ADDR1	1
diff --git a/arch/m68k/include/asm/thread_info.h b/arch/m68k/include/asm/thread_info.h
index 928035591f2e..015f1ca38305 100644
--- a/arch/m68k/include/asm/thread_info.h
+++ b/arch/m68k/include/asm/thread_info.h
@@ -41,8 +41,6 @@ struct thread_info {
 	.preempt_count	= INIT_PREEMPT_COUNT,	\
 }
 
-#define init_stack		(init_thread_union.stack)
-
 #ifndef __ASSEMBLY__
 /* how to get the thread information struct from C */
 static inline struct thread_info *current_thread_info(void)
@@ -58,8 +56,6 @@ static inline struct thread_info *current_thread_info(void)
 }
 #endif
 
-#define init_thread_info	(init_thread_union.thread_info)
-
 /* entry.S relies on these definitions!
  * bits 0-7 are tested at every exception exit
  * bits 8-15 are also tested at syscall exit
diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c
index 87ef73a93856..c01b9b8f97bf 100644
--- a/arch/m68k/kernel/dma.c
+++ b/arch/m68k/kernel/dma.c
@@ -76,8 +76,6 @@ static void *m68k_dma_alloc(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
 	void *ret;
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
 
 	if (dev == NULL || (*dev->dma_mask < 0xffffffff))
 		gfp |= GFP_DMA;
diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c
index 16cd5cea5207..d3d435248a24 100644
--- a/arch/m68k/mac/config.c
+++ b/arch/m68k/mac/config.c
@@ -212,7 +212,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_II,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_IWM,
 	},
 
@@ -227,7 +227,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_II,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_IWM,
 	}, {
 		.ident		= MAC_MODEL_IIX,
@@ -236,7 +236,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_II,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_IICX,
@@ -245,7 +245,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_II,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_SE30,
@@ -254,7 +254,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_II,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 
@@ -272,7 +272,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_IIFX,
@@ -281,7 +281,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_IIFX,
 		.scc_type	= MAC_SCC_IOP,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_IOP,
 	}, {
 		.ident		= MAC_MODEL_IISI,
@@ -290,7 +290,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_IIVI,
@@ -299,7 +299,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_IIVX,
@@ -308,7 +308,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 
@@ -323,7 +323,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_CCL,
@@ -332,7 +331,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_CCLII,
@@ -341,7 +340,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 
@@ -356,7 +355,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_LCII,
@@ -365,7 +364,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_LCIII,
@@ -374,7 +373,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 
@@ -395,7 +394,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_Q605_ACC,
@@ -404,7 +403,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_Q610,
@@ -414,7 +413,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_Q630,
@@ -424,8 +423,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.ide_type	= MAC_IDE_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
-		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_COMM,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_Q650,
@@ -435,7 +433,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	},
 	/* The Q700 does have a NS Sonic */
@@ -447,7 +445,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA2,
 		.scc_type	= MAC_SCC_QUADRA,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_Q800,
@@ -457,7 +455,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_Q840,
@@ -467,7 +465,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA3,
 		.scc_type	= MAC_SCC_PSC,
 		.ether_type	= MAC_ETHER_MACE,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_AV,
 	}, {
 		.ident		= MAC_MODEL_Q900,
@@ -477,7 +475,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA2,
 		.scc_type	= MAC_SCC_IOP,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_IOP,
 	}, {
 		.ident		= MAC_MODEL_Q950,
@@ -487,7 +485,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA2,
 		.scc_type	= MAC_SCC_IOP,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_IOP,
 	},
 
@@ -502,7 +500,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_P475,
@@ -511,7 +509,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_P475F,
@@ -520,7 +518,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_P520,
@@ -529,7 +527,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_P550,
@@ -538,7 +536,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 	/* These have the comm slot, and therefore possibly SONIC ethernet */
@@ -549,8 +547,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_II,
-		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_COMM,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_P588,
@@ -560,8 +557,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.ide_type	= MAC_IDE_QUADRA,
 		.scc_type	= MAC_SCC_II,
-		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_COMM,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_TV,
@@ -570,7 +566,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_P600,
@@ -579,7 +574,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_LC,
 		.scc_type	= MAC_SCC_II,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 
@@ -596,7 +591,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_C650,
@@ -606,7 +601,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA,
 		.scc_type	= MAC_SCC_QUADRA,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR1,
 	}, {
 		.ident		= MAC_MODEL_C660,
@@ -616,7 +611,7 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_QUADRA3,
 		.scc_type	= MAC_SCC_PSC,
 		.ether_type	= MAC_ETHER_MACE,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_PDS_NUBUS,
 		.floppy_type	= MAC_FLOPPY_AV,
 	},
 
@@ -633,7 +628,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB145,
@@ -642,7 +636,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB150,
@@ -652,7 +645,6 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_OLD,
 		.ide_type	= MAC_IDE_PB,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB160,
@@ -661,7 +653,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB165,
@@ -670,7 +661,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB165C,
@@ -679,7 +669,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB170,
@@ -688,7 +677,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB180,
@@ -697,7 +685,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB180C,
@@ -706,7 +693,6 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_QUADRA,
 		.scsi_type	= MAC_SCSI_OLD,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB190,
@@ -716,7 +702,6 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_LATE,
 		.ide_type	= MAC_IDE_BABOON,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB520,
@@ -726,7 +711,6 @@ static struct mac_model mac_data_table[] = {
 		.scsi_type	= MAC_SCSI_LATE,
 		.scc_type	= MAC_SCC_QUADRA,
 		.ether_type	= MAC_ETHER_SONIC,
-		.nubus_type	= MAC_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 
@@ -743,7 +727,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_DUO,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB230,
@@ -752,7 +736,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_DUO,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB250,
@@ -761,7 +745,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_DUO,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB270C,
@@ -770,7 +754,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_DUO,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB280,
@@ -779,7 +763,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_DUO,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	}, {
 		.ident		= MAC_MODEL_PB280C,
@@ -788,7 +772,7 @@ static struct mac_model mac_data_table[] = {
 		.via_type	= MAC_VIA_IICI,
 		.scsi_type	= MAC_SCSI_DUO,
 		.scc_type	= MAC_SCC_QUADRA,
-		.nubus_type	= MAC_NUBUS,
+		.expansion_type	= MAC_EXP_NUBUS,
 		.floppy_type	= MAC_FLOPPY_SWIM_ADDR2,
 	},
 
@@ -1100,14 +1084,12 @@ int __init mac_platform_init(void)
 	 * Ethernet device
 	 */
 
-	switch (macintosh_config->ether_type) {
-	case MAC_ETHER_SONIC:
+	if (macintosh_config->ether_type == MAC_ETHER_SONIC ||
+	    macintosh_config->expansion_type == MAC_EXP_PDS_COMM)
 		platform_device_register_simple("macsonic", -1, NULL, 0);
-		break;
-	case MAC_ETHER_MACE:
+
+	if (macintosh_config->ether_type == MAC_ETHER_MACE)
 		platform_device_register_simple("macmace", -1, NULL, 0);
-		break;
-	}
 
 	return 0;
 }
diff --git a/arch/m68k/mac/oss.c b/arch/m68k/mac/oss.c
index 3f81892527ad..921e6c092f2c 100644
--- a/arch/m68k/mac/oss.c
+++ b/arch/m68k/mac/oss.c
@@ -53,56 +53,41 @@ void __init oss_init(void)
 }
 
 /*
- * Handle miscellaneous OSS interrupts.
+ * Handle OSS interrupts.
+ * XXX how do you clear a pending IRQ? is it even necessary?
  */
 
-static void oss_irq(struct irq_desc *desc)
+static void oss_iopism_irq(struct irq_desc *desc)
 {
-	int events = oss->irq_pending &
-		(OSS_IP_IOPSCC | OSS_IP_SCSI | OSS_IP_IOPISM);
-
-	if (events & OSS_IP_IOPSCC) {
-		oss->irq_pending &= ~OSS_IP_IOPSCC;
-		generic_handle_irq(IRQ_MAC_SCC);
-	}
-
-	if (events & OSS_IP_SCSI) {
-		oss->irq_pending &= ~OSS_IP_SCSI;
-		generic_handle_irq(IRQ_MAC_SCSI);
-	}
-
-	if (events & OSS_IP_IOPISM) {
-		oss->irq_pending &= ~OSS_IP_IOPISM;
-		generic_handle_irq(IRQ_MAC_ADB);
-	}
+	generic_handle_irq(IRQ_MAC_ADB);
 }
 
-/*
- * Nubus IRQ handler, OSS style
- *
- * Unlike the VIA/RBV this is on its own autovector interrupt level.
- */
+static void oss_scsi_irq(struct irq_desc *desc)
+{
+	generic_handle_irq(IRQ_MAC_SCSI);
+}
 
 static void oss_nubus_irq(struct irq_desc *desc)
 {
-	int events, irq_bit, i;
+	u16 events, irq_bit;
+	int irq_num;
 
 	events = oss->irq_pending & OSS_IP_NUBUS;
-	if (!events)
-		return;
-
-	/* There are only six slots on the OSS, not seven */
-
-	i = 6;
-	irq_bit = 0x40;
+	irq_num = NUBUS_SOURCE_BASE + 5;
+	irq_bit = OSS_IP_NUBUS5;
 	do {
-		--i;
-		irq_bit >>= 1;
 		if (events & irq_bit) {
-			oss->irq_pending &= ~irq_bit;
-			generic_handle_irq(NUBUS_SOURCE_BASE + i);
+			events &= ~irq_bit;
+			generic_handle_irq(irq_num);
 		}
-	} while(events & (irq_bit - 1));
+		--irq_num;
+		irq_bit >>= 1;
+	} while (events);
+}
+
+static void oss_iopscc_irq(struct irq_desc *desc)
+{
+	generic_handle_irq(IRQ_MAC_SCC);
 }
 
 /*
@@ -122,14 +107,14 @@ static void oss_nubus_irq(struct irq_desc *desc)
 
 void __init oss_register_interrupts(void)
 {
-	irq_set_chained_handler(OSS_IRQLEV_IOPISM, oss_irq);
-	irq_set_chained_handler(OSS_IRQLEV_SCSI,   oss_irq);
+	irq_set_chained_handler(OSS_IRQLEV_IOPISM, oss_iopism_irq);
+	irq_set_chained_handler(OSS_IRQLEV_SCSI,   oss_scsi_irq);
 	irq_set_chained_handler(OSS_IRQLEV_NUBUS,  oss_nubus_irq);
-	irq_set_chained_handler(OSS_IRQLEV_IOPSCC, oss_irq);
+	irq_set_chained_handler(OSS_IRQLEV_IOPSCC, oss_iopscc_irq);
 	irq_set_chained_handler(OSS_IRQLEV_VIA1,   via1_irq);
 
 	/* OSS_VIA1 gets enabled here because it has no machspec interrupt. */
-	oss->irq_level[OSS_VIA1] = IRQ_AUTO_6;
+	oss->irq_level[OSS_VIA1] = OSS_IRQLEV_VIA1;
 }
 
 /*
diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c
index 127d7c1f2090..03253c4f8e6a 100644
--- a/arch/m68k/mm/fault.c
+++ b/arch/m68k/mm/fault.c
@@ -21,8 +21,9 @@ extern void die_if_kernel(char *, struct pt_regs *, long);
 
 int send_fault_sig(struct pt_regs *regs)
 {
-	siginfo_t siginfo = { 0, 0, 0, };
+	siginfo_t siginfo;
 
+	clear_siginfo(&siginfo);
 	siginfo.si_signo = current->thread.signo;
 	siginfo.si_code = current->thread.code;
 	siginfo.si_addr = (void *)current->thread.faddr;
diff --git a/arch/metag/include/asm/Kbuild b/arch/metag/include/asm/Kbuild
index 3fba97ed9bb2..913c779979df 100644
--- a/arch/metag/include/asm/Kbuild
+++ b/arch/metag/include/asm/Kbuild
@@ -1,5 +1,4 @@
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += device.h
 generic-y += dma.h
diff --git a/arch/metag/include/asm/thread_info.h b/arch/metag/include/asm/thread_info.h
index 554f73a77e6e..a1a9c7f5ca8c 100644
--- a/arch/metag/include/asm/thread_info.h
+++ b/arch/metag/include/asm/thread_info.h
@@ -74,9 +74,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the current stack pointer from C */
 register unsigned long current_stack_pointer asm("A0StP") __used;
 
diff --git a/arch/metag/include/uapi/asm/siginfo.h b/arch/metag/include/uapi/asm/siginfo.h
index b54ef7186ca3..9a3f6cde9487 100644
--- a/arch/metag/include/uapi/asm/siginfo.h
+++ b/arch/metag/include/uapi/asm/siginfo.h
@@ -6,4 +6,11 @@
 
 #include <asm-generic/siginfo.h>
 
+/*
+ * SIGFPE si_codes
+ */
+#ifdef __KERNEL__
+#define FPE_FIXME      0       /* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
 #endif
diff --git a/arch/metag/kernel/devtree.c b/arch/metag/kernel/devtree.c
index 18dd7aea9fdc..6af749a64438 100644
--- a/arch/metag/kernel/devtree.c
+++ b/arch/metag/kernel/devtree.c
@@ -14,26 +14,12 @@
 #include <linux/init.h>
 #include <linux/export.h>
 #include <linux/types.h>
-#include <linux/bootmem.h>
-#include <linux/memblock.h>
-#include <linux/of.h>
 #include <linux/of_fdt.h>
 
 #include <asm/setup.h>
 #include <asm/page.h>
 #include <asm/mach/arch.h>
 
-void __init early_init_dt_add_memory_arch(u64 base, u64 size)
-{
-	pr_err("%s(%llx, %llx)\n",
-	       __func__, base, size);
-}
-
-void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
-{
-	return alloc_bootmem_align(size, align);
-}
-
 static const void * __init arch_get_next_mach(const char *const **match)
 {
 	static const struct machine_desc *mdesc = __arch_info_begin;
diff --git a/arch/metag/kernel/traps.c b/arch/metag/kernel/traps.c
index 444851e510d5..3b62b1b0c0b5 100644
--- a/arch/metag/kernel/traps.c
+++ b/arch/metag/kernel/traps.c
@@ -735,7 +735,7 @@ TBIRES fpe_handler(TBIRES State, int SigNum, int Triggers, int Inst, PTBI pTBI)
 	else if (error_state & TXSTAT_FPE_INEXACT_BIT)
 		info.si_code = FPE_FLTRES;
 	else
-		info.si_code = 0;
+		info.si_code = FPE_FIXME;
 	info.si_errno = 0;
 	info.si_addr = (__force void __user *)regs->ctx.CurrPC;
 	force_sig_info(SIGFPE, &info, current);
diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig
index 4f798aa671dd..3817a3e2146c 100644
--- a/arch/microblaze/Kconfig
+++ b/arch/microblaze/Kconfig
@@ -24,6 +24,7 @@ config MICROBLAZE
 	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_FUNCTION_GRAPH_TRACER
 	select HAVE_FUNCTION_TRACER
+	select NO_BOOTMEM
 	select HAVE_MEMBLOCK
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_OPROFILE
diff --git a/arch/microblaze/Kconfig.platform b/arch/microblaze/Kconfig.platform
index 1b3d8c849101..f7f1739c11b9 100644
--- a/arch/microblaze/Kconfig.platform
+++ b/arch/microblaze/Kconfig.platform
@@ -20,6 +20,7 @@ config OPT_LIB_FUNCTION
 config OPT_LIB_ASM
 	bool "Optimalized lib function ASM"
 	depends on OPT_LIB_FUNCTION && (XILINX_MICROBLAZE0_USE_BARREL = 1)
+	depends on CPU_BIG_ENDIAN
 	default n
 	help
 	  Allows turn on optimalized library function (memcpy and memmove).
diff --git a/arch/microblaze/Makefile b/arch/microblaze/Makefile
index 830ee7d42fa0..d269dd4b8279 100644
--- a/arch/microblaze/Makefile
+++ b/arch/microblaze/Makefile
@@ -36,16 +36,21 @@ endif
 CPUFLAGS-$(CONFIG_XILINX_MICROBLAZE0_USE_DIV) += -mno-xl-soft-div
 CPUFLAGS-$(CONFIG_XILINX_MICROBLAZE0_USE_BARREL) += -mxl-barrel-shift
 CPUFLAGS-$(CONFIG_XILINX_MICROBLAZE0_USE_PCMP_INSTR) += -mxl-pattern-compare
-CPUFLAGS-$(CONFIG_BIG_ENDIAN) += -mbig-endian
-CPUFLAGS-$(CONFIG_LITTLE_ENDIAN) += -mlittle-endian
+
+ifdef CONFIG_CPU_BIG_ENDIAN
+KBUILD_CFLAGS += -mbig-endian
+KBUILD_AFLAGS += -mbig-endian
+LD += -EB
+else
+KBUILD_CFLAGS += -mlittle-endian
+KBUILD_AFLAGS += -mlittle-endian
+LD += -EL
+endif
 
 CPUFLAGS-1 += $(call cc-option,-mcpu=v$(CPU_VER))
 
 # r31 holds current when in kernel mode
-KBUILD_CFLAGS += -ffixed-r31 $(CPUFLAGS-1) $(CPUFLAGS-2)
-
-LDFLAGS		:=
-LDFLAGS_vmlinux	:=
+KBUILD_CFLAGS += -ffixed-r31 $(CPUFLAGS-y) $(CPUFLAGS-1) $(CPUFLAGS-2)
 
 head-y := arch/microblaze/kernel/head.o
 libs-y += arch/microblaze/lib/
diff --git a/arch/microblaze/include/asm/Kbuild b/arch/microblaze/include/asm/Kbuild
index 9d66f7793841..3c80a5a308ed 100644
--- a/arch/microblaze/include/asm/Kbuild
+++ b/arch/microblaze/include/asm/Kbuild
@@ -2,7 +2,6 @@ generic-y += barrier.h
 generic-y += bitops.h
 generic-y += bug.h
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += emergency-restart.h
diff --git a/arch/microblaze/include/asm/dma-mapping.h b/arch/microblaze/include/asm/dma-mapping.h
index 6b9ea39405b8..add50c1373bf 100644
--- a/arch/microblaze/include/asm/dma-mapping.h
+++ b/arch/microblaze/include/asm/dma-mapping.h
@@ -18,11 +18,11 @@
 /*
  * Available generic sets of operations
  */
-extern const struct dma_map_ops dma_direct_ops;
+extern const struct dma_map_ops dma_nommu_ops;
 
 static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 {
-	return &dma_direct_ops;
+	return &dma_nommu_ops;
 }
 
 #endif	/* _ASM_MICROBLAZE_DMA_MAPPING_H */
diff --git a/arch/microblaze/include/asm/io.h b/arch/microblaze/include/asm/io.h
index 39b6315db82e..c7968139486f 100644
--- a/arch/microblaze/include/asm/io.h
+++ b/arch/microblaze/include/asm/io.h
@@ -36,7 +36,7 @@ extern resource_size_t isa_mem_base;
 #ifdef CONFIG_MMU
 #define page_to_bus(page)	(page_to_phys(page))
 
-extern void iounmap(void __iomem *addr);
+extern void iounmap(volatile void __iomem *addr);
 
 extern void __iomem *ioremap(phys_addr_t address, unsigned long size);
 #define ioremap_nocache(addr, size)		ioremap((addr), (size))
diff --git a/arch/microblaze/include/asm/setup.h b/arch/microblaze/include/asm/setup.h
index be84a4d3917f..7c968c1d1729 100644
--- a/arch/microblaze/include/asm/setup.h
+++ b/arch/microblaze/include/asm/setup.h
@@ -44,7 +44,6 @@ void machine_shutdown(void);
 void machine_halt(void);
 void machine_power_off(void);
 
-extern void *alloc_maybe_bootmem(size_t size, gfp_t mask);
 extern void *zalloc_maybe_bootmem(size_t size, gfp_t mask);
 
 # endif /* __ASSEMBLY__ */
diff --git a/arch/microblaze/include/asm/thread_info.h b/arch/microblaze/include/asm/thread_info.h
index e7e8954e9815..9afe4b5bd6c8 100644
--- a/arch/microblaze/include/asm/thread_info.h
+++ b/arch/microblaze/include/asm/thread_info.h
@@ -86,9 +86,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 static inline struct thread_info *current_thread_info(void)
 {
diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c
index 990bf9ea0ec6..c91e8cef98dd 100644
--- a/arch/microblaze/kernel/dma.c
+++ b/arch/microblaze/kernel/dma.c
@@ -15,42 +15,18 @@
 #include <linux/bug.h>
 #include <asm/cacheflush.h>
 
-#define NOT_COHERENT_CACHE
-
-static void *dma_direct_alloc_coherent(struct device *dev, size_t size,
+static void *dma_nommu_alloc_coherent(struct device *dev, size_t size,
 				       dma_addr_t *dma_handle, gfp_t flag,
 				       unsigned long attrs)
 {
-#ifdef NOT_COHERENT_CACHE
 	return consistent_alloc(flag, size, dma_handle);
-#else
-	void *ret;
-	struct page *page;
-	int node = dev_to_node(dev);
-
-	/* ignore region specifiers */
-	flag  &= ~(__GFP_HIGHMEM);
-
-	page = alloc_pages_node(node, flag, get_order(size));
-	if (page == NULL)
-		return NULL;
-	ret = page_address(page);
-	memset(ret, 0, size);
-	*dma_handle = virt_to_phys(ret);
-
-	return ret;
-#endif
 }
 
-static void dma_direct_free_coherent(struct device *dev, size_t size,
+static void dma_nommu_free_coherent(struct device *dev, size_t size,
 				     void *vaddr, dma_addr_t dma_handle,
 				     unsigned long attrs)
 {
-#ifdef NOT_COHERENT_CACHE
 	consistent_free(size, vaddr);
-#else
-	free_pages((unsigned long)vaddr, get_order(size));
-#endif
 }
 
 static inline void __dma_sync(unsigned long paddr,
@@ -69,7 +45,7 @@ static inline void __dma_sync(unsigned long paddr,
 	}
 }
 
-static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
+static int dma_nommu_map_sg(struct device *dev, struct scatterlist *sgl,
 			     int nents, enum dma_data_direction direction,
 			     unsigned long attrs)
 {
@@ -89,12 +65,7 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
 	return nents;
 }
 
-static int dma_direct_dma_supported(struct device *dev, u64 mask)
-{
-	return 1;
-}
-
-static inline dma_addr_t dma_direct_map_page(struct device *dev,
+static inline dma_addr_t dma_nommu_map_page(struct device *dev,
 					     struct page *page,
 					     unsigned long offset,
 					     size_t size,
@@ -106,7 +77,7 @@ static inline dma_addr_t dma_direct_map_page(struct device *dev,
 	return page_to_phys(page) + offset;
 }
 
-static inline void dma_direct_unmap_page(struct device *dev,
+static inline void dma_nommu_unmap_page(struct device *dev,
 					 dma_addr_t dma_address,
 					 size_t size,
 					 enum dma_data_direction direction,
@@ -122,7 +93,7 @@ static inline void dma_direct_unmap_page(struct device *dev,
 }
 
 static inline void
-dma_direct_sync_single_for_cpu(struct device *dev,
+dma_nommu_sync_single_for_cpu(struct device *dev,
 			       dma_addr_t dma_handle, size_t size,
 			       enum dma_data_direction direction)
 {
@@ -136,7 +107,7 @@ dma_direct_sync_single_for_cpu(struct device *dev,
 }
 
 static inline void
-dma_direct_sync_single_for_device(struct device *dev,
+dma_nommu_sync_single_for_device(struct device *dev,
 				  dma_addr_t dma_handle, size_t size,
 				  enum dma_data_direction direction)
 {
@@ -150,7 +121,7 @@ dma_direct_sync_single_for_device(struct device *dev,
 }
 
 static inline void
-dma_direct_sync_sg_for_cpu(struct device *dev,
+dma_nommu_sync_sg_for_cpu(struct device *dev,
 			   struct scatterlist *sgl, int nents,
 			   enum dma_data_direction direction)
 {
@@ -164,7 +135,7 @@ dma_direct_sync_sg_for_cpu(struct device *dev,
 }
 
 static inline void
-dma_direct_sync_sg_for_device(struct device *dev,
+dma_nommu_sync_sg_for_device(struct device *dev,
 			      struct scatterlist *sgl, int nents,
 			      enum dma_data_direction direction)
 {
@@ -178,7 +149,7 @@ dma_direct_sync_sg_for_device(struct device *dev,
 }
 
 static
-int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
+int dma_nommu_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
 			     void *cpu_addr, dma_addr_t handle, size_t size,
 			     unsigned long attrs)
 {
@@ -191,12 +162,8 @@ int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
 	if (off >= count || user_count > (count - off))
 		return -ENXIO;
 
-#ifdef NOT_COHERENT_CACHE
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	pfn = consistent_virt_to_pfn(cpu_addr);
-#else
-	pfn = virt_to_pfn(cpu_addr);
-#endif
 	return remap_pfn_range(vma, vma->vm_start, pfn + off,
 			       vma->vm_end - vma->vm_start, vma->vm_page_prot);
 #else
@@ -204,20 +171,19 @@ int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
 #endif
 }
 
-const struct dma_map_ops dma_direct_ops = {
-	.alloc		= dma_direct_alloc_coherent,
-	.free		= dma_direct_free_coherent,
-	.mmap		= dma_direct_mmap_coherent,
-	.map_sg		= dma_direct_map_sg,
-	.dma_supported	= dma_direct_dma_supported,
-	.map_page	= dma_direct_map_page,
-	.unmap_page	= dma_direct_unmap_page,
-	.sync_single_for_cpu		= dma_direct_sync_single_for_cpu,
-	.sync_single_for_device		= dma_direct_sync_single_for_device,
-	.sync_sg_for_cpu		= dma_direct_sync_sg_for_cpu,
-	.sync_sg_for_device		= dma_direct_sync_sg_for_device,
+const struct dma_map_ops dma_nommu_ops = {
+	.alloc			= dma_nommu_alloc_coherent,
+	.free			= dma_nommu_free_coherent,
+	.mmap			= dma_nommu_mmap_coherent,
+	.map_sg			= dma_nommu_map_sg,
+	.map_page		= dma_nommu_map_page,
+	.unmap_page		= dma_nommu_unmap_page,
+	.sync_single_for_cpu	= dma_nommu_sync_single_for_cpu,
+	.sync_single_for_device	= dma_nommu_sync_single_for_device,
+	.sync_sg_for_cpu	= dma_nommu_sync_sg_for_cpu,
+	.sync_sg_for_device	= dma_nommu_sync_sg_for_device,
 };
-EXPORT_SYMBOL(dma_direct_ops);
+EXPORT_SYMBOL(dma_nommu_ops);
 
 /* Number of entries preallocated for DMA-API debugging */
 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
diff --git a/arch/microblaze/lib/fastcopy.S b/arch/microblaze/lib/fastcopy.S
index 62021d7e249e..fdc48bb065d8 100644
--- a/arch/microblaze/lib/fastcopy.S
+++ b/arch/microblaze/lib/fastcopy.S
@@ -29,10 +29,6 @@
  *	between mem locations with size of xfer spec'd in bytes
  */
 
-#ifdef __MICROBLAZEEL__
-#error Microblaze LE not support ASM optimized lib func. Disable OPT_LIB_ASM.
-#endif
-
 #include <linux/linkage.h>
 	.text
 	.globl	memcpy
diff --git a/arch/microblaze/mm/init.c b/arch/microblaze/mm/init.c
index 434639f9a3a6..df6de7ccdc2e 100644
--- a/arch/microblaze/mm/init.c
+++ b/arch/microblaze/mm/init.c
@@ -32,9 +32,6 @@ int mem_init_done;
 #ifndef CONFIG_MMU
 unsigned int __page_offset;
 EXPORT_SYMBOL(__page_offset);
-
-#else
-static int init_bootmem_done;
 #endif /* CONFIG_MMU */
 
 char *klimit = _end;
@@ -117,7 +114,6 @@ static void __init paging_init(void)
 
 void __init setup_memory(void)
 {
-	unsigned long map_size;
 	struct memblock_region *reg;
 
 #ifndef CONFIG_MMU
@@ -174,17 +170,6 @@ void __init setup_memory(void)
 	pr_info("%s: max_low_pfn: %#lx\n", __func__, max_low_pfn);
 	pr_info("%s: max_pfn: %#lx\n", __func__, max_pfn);
 
-	/*
-	 * Find an area to use for the bootmem bitmap.
-	 * We look for the first area which is at least
-	 * 128kB in length (128kB is enough for a bitmap
-	 * for 4GB of memory, using 4kB pages), plus 1 page
-	 * (in case the address isn't page-aligned).
-	 */
-	map_size = init_bootmem_node(NODE_DATA(0),
-		PFN_UP(TOPHYS((u32)klimit)), min_low_pfn, max_low_pfn);
-	memblock_reserve(PFN_UP(TOPHYS((u32)klimit)) << PAGE_SHIFT, map_size);
-
 	/* Add active regions with valid PFNs */
 	for_each_memblock(memory, reg) {
 		unsigned long start_pfn, end_pfn;
@@ -196,32 +181,9 @@ void __init setup_memory(void)
 				  &memblock.memory, 0);
 	}
 
-	/* free bootmem is whole main memory */
-	free_bootmem_with_active_regions(0, max_low_pfn);
-
-	/* reserve allocate blocks */
-	for_each_memblock(reserved, reg) {
-		unsigned long top = reg->base + reg->size - 1;
-
-		pr_debug("reserved - 0x%08x-0x%08x, %lx, %lx\n",
-			 (u32) reg->base, (u32) reg->size, top,
-						memory_start + lowmem_size - 1);
-
-		if (top <= (memory_start + lowmem_size - 1)) {
-			reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
-		} else if (reg->base < (memory_start + lowmem_size - 1)) {
-			unsigned long trunc_size = memory_start + lowmem_size -
-								reg->base;
-			reserve_bootmem(reg->base, trunc_size, BOOTMEM_DEFAULT);
-		}
-	}
-
 	/* XXX need to clip this if using highmem? */
 	sparse_memory_present_with_active_regions(0);
 
-#ifdef CONFIG_MMU
-	init_bootmem_done = 1;
-#endif
 	paging_init();
 }
 
@@ -398,30 +360,16 @@ asmlinkage void __init mmu_init(void)
 /* This is only called until mem_init is done. */
 void __init *early_get_page(void)
 {
-	void *p;
-	if (init_bootmem_done) {
-		p = alloc_bootmem_pages(PAGE_SIZE);
-	} else {
-		/*
-		 * Mem start + kernel_tlb -> here is limit
-		 * because of mem mapping from head.S
-		 */
-		p = __va(memblock_alloc_base(PAGE_SIZE, PAGE_SIZE,
-					memory_start + kernel_tlb));
-	}
-	return p;
+	/*
+	 * Mem start + kernel_tlb -> here is limit
+	 * because of mem mapping from head.S
+	 */
+	return __va(memblock_alloc_base(PAGE_SIZE, PAGE_SIZE,
+				memory_start + kernel_tlb));
 }
 
 #endif /* CONFIG_MMU */
 
-void * __ref alloc_maybe_bootmem(size_t size, gfp_t mask)
-{
-	if (mem_init_done)
-		return kmalloc(size, mask);
-	else
-		return alloc_bootmem(size);
-}
-
 void * __ref zalloc_maybe_bootmem(size_t size, gfp_t mask)
 {
 	void *p;
diff --git a/arch/microblaze/mm/pgtable.c b/arch/microblaze/mm/pgtable.c
index 4c0599239915..7f525962cdfa 100644
--- a/arch/microblaze/mm/pgtable.c
+++ b/arch/microblaze/mm/pgtable.c
@@ -127,7 +127,7 @@ void __iomem *ioremap(phys_addr_t addr, unsigned long size)
 }
 EXPORT_SYMBOL(ioremap);
 
-void iounmap(void __iomem *addr)
+void iounmap(volatile void __iomem *addr)
 {
 	if ((__force void *)addr > high_memory &&
 					(unsigned long) addr < ioremap_bot)
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 8e0b3702f1c0..8128c3b68d6b 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -7,8 +7,6 @@ config MIPS
 	select ARCH_DISCARD_MEMBLOCK
 	select ARCH_HAS_ELF_RANDOMIZE
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
-	select ARCH_MIGHT_HAVE_PC_PARPORT
-	select ARCH_MIGHT_HAVE_PC_SERIO
 	select ARCH_SUPPORTS_UPROBES
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_CMPXCHG_LOCKREF if 64BIT
@@ -119,12 +117,12 @@ config MIPS_GENERIC
 	select SYS_SUPPORTS_MULTITHREADING
 	select SYS_SUPPORTS_RELOCATABLE
 	select SYS_SUPPORTS_SMARTMIPS
-	select USB_EHCI_BIG_ENDIAN_DESC if BIG_ENDIAN
-	select USB_EHCI_BIG_ENDIAN_MMIO if BIG_ENDIAN
-	select USB_OHCI_BIG_ENDIAN_DESC if BIG_ENDIAN
-	select USB_OHCI_BIG_ENDIAN_MMIO if BIG_ENDIAN
-	select USB_UHCI_BIG_ENDIAN_DESC if BIG_ENDIAN
-	select USB_UHCI_BIG_ENDIAN_MMIO if BIG_ENDIAN
+	select USB_EHCI_BIG_ENDIAN_DESC if CPU_BIG_ENDIAN
+	select USB_EHCI_BIG_ENDIAN_MMIO if CPU_BIG_ENDIAN
+	select USB_OHCI_BIG_ENDIAN_DESC if CPU_BIG_ENDIAN
+	select USB_OHCI_BIG_ENDIAN_MMIO if CPU_BIG_ENDIAN
+	select USB_UHCI_BIG_ENDIAN_DESC if CPU_BIG_ENDIAN
+	select USB_UHCI_BIG_ENDIAN_MMIO if CPU_BIG_ENDIAN
 	select USE_OF
 	help
 	  Select this to build a kernel which aims to support multiple boards,
@@ -253,6 +251,7 @@ config BCM47XX
 	select SYS_SUPPORTS_32BIT_KERNEL
 	select SYS_SUPPORTS_LITTLE_ENDIAN
 	select SYS_SUPPORTS_MIPS16
+	select SYS_SUPPORTS_ZBOOT
 	select SYS_HAS_EARLY_PRINTK
 	select USE_GENERIC_EARLY_PRINTK_8250
 	select GPIOLIB
@@ -341,6 +340,8 @@ config MACH_DECSTATION
 
 config MACH_JAZZ
 	bool "Jazz family of machines"
+	select ARCH_MIGHT_HAVE_PC_PARPORT
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select FW_ARC
 	select FW_ARC32
 	select ARCH_MAY_HAVE_PC_FDC
@@ -431,6 +432,7 @@ config MACH_LOONGSON32
 
 config MACH_LOONGSON64
 	bool "Loongson-2/3 family of machines"
+	select ARCH_HAS_PHYS_TO_DMA
 	select SYS_SUPPORTS_ZBOOT
 	help
 	  This enables the support of Loongson-2/3 family of machines.
@@ -475,6 +477,8 @@ config MACH_PISTACHIO
 config MIPS_MALTA
 	bool "MIPS Malta board"
 	select ARCH_MAY_HAVE_PC_FDC
+	select ARCH_MIGHT_HAVE_PC_PARPORT
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select BOOT_ELF32
 	select BOOT_RAW
 	select BUILTIN_DTB
@@ -612,6 +616,7 @@ config SGI_IP22
 	bool "SGI IP22 (Indy/Indigo2)"
 	select FW_ARC
 	select FW_ARC32
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select BOOT_ELF32
 	select CEVT_R4K
 	select CSRC_R4K
@@ -674,6 +679,7 @@ config SGI_IP28
 	bool "SGI IP28 (Indigo2 R10k)"
 	select FW_ARC
 	select FW_ARC64
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select BOOT_ELF64
 	select CEVT_R4K
 	select CSRC_R4K
@@ -823,6 +829,8 @@ config SNI_RM
 	select FW_ARC32 if CPU_LITTLE_ENDIAN
 	select FW_SNIPROM if CPU_BIG_ENDIAN
 	select ARCH_MAY_HAVE_PC_FDC
+	select ARCH_MIGHT_HAVE_PC_PARPORT
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select BOOT_ELF32
 	select CEVT_R4K
 	select CSRC_R4K
@@ -880,6 +888,7 @@ config MIKROTIK_RB532
 config CAVIUM_OCTEON_SOC
 	bool "Cavium Networks Octeon SoC based boards"
 	select CEVT_R4K
+	select ARCH_HAS_PHYS_TO_DMA
 	select ARCH_PHYS_ADDR_T_64BIT
 	select DMA_COHERENT
 	select SYS_SUPPORTS_64BIT_KERNEL
@@ -2324,7 +2333,6 @@ config MIPS_VPE_LOADER_TOM
 config MIPS_VPE_APSP_API
 	bool "Enable support for AP/SP API (RTLX)"
 	depends on MIPS_VPE_LOADER
-	help
 
 config MIPS_VPE_APSP_API_CMP
 	bool
diff --git a/arch/mips/Makefile b/arch/mips/Makefile
index 9f6a26d72f9f..d1ca839c3981 100644
--- a/arch/mips/Makefile
+++ b/arch/mips/Makefile
@@ -216,6 +216,12 @@ cflags-$(toolchain-msa)			+= -DTOOLCHAIN_SUPPORTS_MSA
 endif
 toolchain-virt				:= $(call cc-option-yn,$(mips-cflags) -mvirt)
 cflags-$(toolchain-virt)		+= -DTOOLCHAIN_SUPPORTS_VIRT
+# For -mmicromips, use -Wa,-fatal-warnings to catch unsupported -mxpa which
+# only warns
+xpa-cflags-y				:= $(mips-cflags)
+xpa-cflags-$(micromips-ase)		+= -mmicromips -Wa$(comma)-fatal-warnings
+toolchain-xpa				:= $(call cc-option-yn,$(xpa-cflags-y) -mxpa)
+cflags-$(toolchain-xpa)			+= -DTOOLCHAIN_SUPPORTS_XPA
 
 #
 # Firmware support
@@ -228,7 +234,7 @@ libs-y				+= arch/mips/fw/lib/
 #
 # Kernel compression
 #
-ifdef SYS_SUPPORTS_ZBOOT
+ifdef CONFIG_SYS_SUPPORTS_ZBOOT
 COMPRESSION_FNAME		= vmlinuz
 else
 COMPRESSION_FNAME		= vmlinux
diff --git a/arch/mips/ath25/board.c b/arch/mips/ath25/board.c
index 9ab48ff80c1c..6d11ae581ea7 100644
--- a/arch/mips/ath25/board.c
+++ b/arch/mips/ath25/board.c
@@ -135,6 +135,8 @@ int __init ath25_find_config(phys_addr_t base, unsigned long size)
 	}
 
 	board_data = kzalloc(BOARD_CONFIG_BUFSZ, GFP_KERNEL);
+	if (!board_data)
+		goto error;
 	ath25_board.config = (struct ath25_boarddata *)board_data;
 	memcpy_fromio(board_data, bcfg, 0x100);
 	if (broken_boarddata) {
diff --git a/arch/mips/bcm47xx/Platform b/arch/mips/bcm47xx/Platform
index 874b7ca4cd11..70783b75fd9d 100644
--- a/arch/mips/bcm47xx/Platform
+++ b/arch/mips/bcm47xx/Platform
@@ -5,3 +5,4 @@ platform-$(CONFIG_BCM47XX)	+= bcm47xx/
 cflags-$(CONFIG_BCM47XX)	+=					\
 		-I$(srctree)/arch/mips/include/asm/mach-bcm47xx
 load-$(CONFIG_BCM47XX)		:= 0xffffffff80001000
+zload-$(CONFIG_BCM47XX)		+= 0xffffffff80400000
diff --git a/arch/mips/bcm63xx/boards/Kconfig b/arch/mips/bcm63xx/boards/Kconfig
index 6ff0a7481081..f60d96610ace 100644
--- a/arch/mips/bcm63xx/boards/Kconfig
+++ b/arch/mips/bcm63xx/boards/Kconfig
@@ -7,6 +7,5 @@ choice
 config BOARD_BCM963XX
        bool "Generic Broadcom 963xx boards"
 	select SSB
-       help
 
 endchoice
diff --git a/arch/mips/bcm63xx/dev-enet.c b/arch/mips/bcm63xx/dev-enet.c
index e8284771d620..07b4c65a88a4 100644
--- a/arch/mips/bcm63xx/dev-enet.c
+++ b/arch/mips/bcm63xx/dev-enet.c
@@ -265,6 +265,14 @@ int __init bcm63xx_enet_register(int unit,
 		dpd->dma_chan_width = ENETDMA_CHAN_WIDTH;
 	}
 
+	if (unit == 0) {
+		dpd->rx_chan = 0;
+		dpd->tx_chan = 1;
+	} else {
+		dpd->rx_chan = 2;
+		dpd->tx_chan = 3;
+	}
+
 	ret = platform_device_register(pdev);
 	if (ret)
 		return ret;
diff --git a/arch/mips/boot/Makefile b/arch/mips/boot/Makefile
index 1bd5c4f00d19..c22da16d67b8 100644
--- a/arch/mips/boot/Makefile
+++ b/arch/mips/boot/Makefile
@@ -126,6 +126,7 @@ $(obj)/vmlinux.its.S: $(addprefix $(srctree)/arch/mips/$(PLATFORM)/,$(ITS_INPUTS
 
 quiet_cmd_cpp_its_S = ITS     $@
       cmd_cpp_its_S = $(CPP) $(cpp_flags) -P -C -o $@ $< \
+			-D__ASSEMBLY__ \
 		        -DKERNEL_NAME="\"Linux $(KERNELRELEASE)\"" \
 			-DVMLINUX_BINARY="\"$(3)\"" \
 			-DVMLINUX_COMPRESSION="\"$(2)\"" \
diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile
index c675eece389a..adce180f3ee4 100644
--- a/arch/mips/boot/compressed/Makefile
+++ b/arch/mips/boot/compressed/Makefile
@@ -133,4 +133,8 @@ vmlinuz.srec: vmlinuz
 uzImage.bin: vmlinuz.bin FORCE
 	$(call if_changed,uimage,none)
 
-clean-files := $(objtree)/vmlinuz $(objtree)/vmlinuz.{32,ecoff,bin,srec}
+clean-files += $(objtree)/vmlinuz
+clean-files += $(objtree)/vmlinuz.32
+clean-files += $(objtree)/vmlinuz.ecoff
+clean-files += $(objtree)/vmlinuz.bin
+clean-files += $(objtree)/vmlinuz.srec
diff --git a/arch/mips/boot/dts/ingenic/Makefile b/arch/mips/boot/dts/ingenic/Makefile
index 6a31759839b4..5b1361a89e02 100644
--- a/arch/mips/boot/dts/ingenic/Makefile
+++ b/arch/mips/boot/dts/ingenic/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_JZ4740_QI_LB60)	+= qi_lb60.dtb
+dtb-$(CONFIG_JZ4770_GCW0)	+= gcw0.dtb
 dtb-$(CONFIG_JZ4780_CI20)	+= ci20.dtb
 
 obj-y				+= $(patsubst %.dtb, %.dtb.o, $(dtb-y))
diff --git a/arch/mips/boot/dts/ingenic/gcw0.dts b/arch/mips/boot/dts/ingenic/gcw0.dts
new file mode 100644
index 000000000000..35f0291e8d38
--- /dev/null
+++ b/arch/mips/boot/dts/ingenic/gcw0.dts
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0
+/dts-v1/;
+
+#include "jz4770.dtsi"
+
+/ {
+	compatible = "gcw,zero", "ingenic,jz4770";
+	model = "GCW Zero";
+
+	aliases {
+		serial0 = &uart0;
+		serial1 = &uart1;
+		serial2 = &uart2;
+		serial3 = &uart3;
+	};
+
+	chosen {
+		stdout-path = "serial2:57600n8";
+	};
+
+	board {
+		compatible = "simple-bus";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		otg_phy: otg-phy {
+			compatible = "usb-nop-xceiv";
+			clocks = <&cgu JZ4770_CLK_OTG_PHY>;
+			clock-names = "main_clk";
+		};
+	};
+};
+
+&ext {
+	clock-frequency = <12000000>;
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&cgu {
+	/* Put high-speed peripherals under PLL1, such that we can change the
+	 * PLL0 frequency on demand without having to suspend peripherals.
+	 * We use a rate of 432 MHz, which is the least common multiple of
+	 * 27 MHz (required by TV encoder) and 48 MHz (required by USB host).
+	 */
+	assigned-clocks =
+		<&cgu JZ4770_CLK_PLL1>,
+		<&cgu JZ4770_CLK_UHC>;
+	assigned-clock-parents =
+		<0>,
+		<&cgu JZ4770_CLK_PLL1>;
+	assigned-clock-rates =
+		<432000000>;
+};
+
+&uhc {
+	/* The WiFi module is connected to the UHC. */
+	status = "okay";
+};
diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi
new file mode 100644
index 000000000000..7c2804f3f5f1
--- /dev/null
+++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <dt-bindings/clock/jz4770-cgu.h>
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+	compatible = "ingenic,jz4770";
+
+	cpuintc: interrupt-controller {
+		#address-cells = <0>;
+		#interrupt-cells = <1>;
+		interrupt-controller;
+		compatible = "mti,cpu-interrupt-controller";
+	};
+
+	intc: interrupt-controller@10001000 {
+		compatible = "ingenic,jz4770-intc";
+		reg = <0x10001000 0x40>;
+
+		interrupt-controller;
+		#interrupt-cells = <1>;
+
+		interrupt-parent = <&cpuintc>;
+		interrupts = <2>;
+	};
+
+	ext: ext {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+	};
+
+	osc32k: osc32k {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+
+	cgu: jz4770-cgu@10000000 {
+		compatible = "ingenic,jz4770-cgu";
+		reg = <0x10000000 0x100>;
+
+		clocks = <&ext>, <&osc32k>;
+		clock-names = "ext", "osc32k";
+
+		#clock-cells = <1>;
+	};
+
+	pinctrl: pin-controller@10010000 {
+		compatible = "ingenic,jz4770-pinctrl";
+		reg = <0x10010000 0x600>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		gpa: gpio@0 {
+			compatible = "ingenic,jz4770-gpio";
+			reg = <0>;
+
+			gpio-controller;
+			gpio-ranges = <&pinctrl 0 0 32>;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			interrupt-parent = <&intc>;
+			interrupts = <17>;
+		};
+
+		gpb: gpio@1 {
+			compatible = "ingenic,jz4770-gpio";
+			reg = <1>;
+
+			gpio-controller;
+			gpio-ranges = <&pinctrl 0 32 32>;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			interrupt-parent = <&intc>;
+			interrupts = <16>;
+		};
+
+		gpc: gpio@2 {
+			compatible = "ingenic,jz4770-gpio";
+			reg = <2>;
+
+			gpio-controller;
+			gpio-ranges = <&pinctrl 0 64 32>;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			interrupt-parent = <&intc>;
+			interrupts = <15>;
+		};
+
+		gpd: gpio@3 {
+			compatible = "ingenic,jz4770-gpio";
+			reg = <3>;
+
+			gpio-controller;
+			gpio-ranges = <&pinctrl 0 96 32>;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			interrupt-parent = <&intc>;
+			interrupts = <14>;
+		};
+
+		gpe: gpio@4 {
+			compatible = "ingenic,jz4770-gpio";
+			reg = <4>;
+
+			gpio-controller;
+			gpio-ranges = <&pinctrl 0 128 32>;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			interrupt-parent = <&intc>;
+			interrupts = <13>;
+		};
+
+		gpf: gpio@5 {
+			compatible = "ingenic,jz4770-gpio";
+			reg = <5>;
+
+			gpio-controller;
+			gpio-ranges = <&pinctrl 0 160 32>;
+			#gpio-cells = <2>;
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+
+			interrupt-parent = <&intc>;
+			interrupts = <12>;
+		};
+	};
+
+	uart0: serial@10030000 {
+		compatible = "ingenic,jz4770-uart";
+		reg = <0x10030000 0x100>;
+
+		clocks = <&ext>, <&cgu JZ4770_CLK_UART0>;
+		clock-names = "baud", "module";
+
+		interrupt-parent = <&intc>;
+		interrupts = <5>;
+
+		status = "disabled";
+	};
+
+	uart1: serial@10031000 {
+		compatible = "ingenic,jz4770-uart";
+		reg = <0x10031000 0x100>;
+
+		clocks = <&ext>, <&cgu JZ4770_CLK_UART1>;
+		clock-names = "baud", "module";
+
+		interrupt-parent = <&intc>;
+		interrupts = <4>;
+
+		status = "disabled";
+	};
+
+	uart2: serial@10032000 {
+		compatible = "ingenic,jz4770-uart";
+		reg = <0x10032000 0x100>;
+
+		clocks = <&ext>, <&cgu JZ4770_CLK_UART2>;
+		clock-names = "baud", "module";
+
+		interrupt-parent = <&intc>;
+		interrupts = <3>;
+
+		status = "disabled";
+	};
+
+	uart3: serial@10033000 {
+		compatible = "ingenic,jz4770-uart";
+		reg = <0x10033000 0x100>;
+
+		clocks = <&ext>, <&cgu JZ4770_CLK_UART3>;
+		clock-names = "baud", "module";
+
+		interrupt-parent = <&intc>;
+		interrupts = <2>;
+
+		status = "disabled";
+	};
+
+	uhc: uhc@13430000 {
+		compatible = "generic-ohci";
+		reg = <0x13430000 0x1000>;
+
+		clocks = <&cgu JZ4770_CLK_UHC>, <&cgu JZ4770_CLK_UHC_PHY>;
+		assigned-clocks = <&cgu JZ4770_CLK_UHC>;
+		assigned-clock-rates = <48000000>;
+
+		interrupt-parent = <&intc>;
+		interrupts = <20>;
+
+		status = "disabled";
+	};
+};
diff --git a/arch/mips/cavium-octeon/Kconfig b/arch/mips/cavium-octeon/Kconfig
index 204a1670fd9b..b5eee1a57d6c 100644
--- a/arch/mips/cavium-octeon/Kconfig
+++ b/arch/mips/cavium-octeon/Kconfig
@@ -75,6 +75,7 @@ config NEED_SG_DMA_LENGTH
 
 config SWIOTLB
 	def_bool y
+	select DMA_DIRECT_OPS
 	select IOMMU_HELPER
 	select NEED_SG_DMA_LENGTH
 
diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c
index c64bd87f0b6e..c7bb8a407041 100644
--- a/arch/mips/cavium-octeon/dma-octeon.c
+++ b/arch/mips/cavium-octeon/dma-octeon.c
@@ -159,36 +159,13 @@ static void octeon_dma_sync_sg_for_device(struct device *dev,
 static void *octeon_dma_alloc_coherent(struct device *dev, size_t size,
 	dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
-	void *ret;
-
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
-
-	if (IS_ENABLED(CONFIG_ZONE_DMA) && dev == NULL)
-		gfp |= __GFP_DMA;
-	else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
-		 dev->coherent_dma_mask <= DMA_BIT_MASK(24))
-		gfp |= __GFP_DMA;
-	else if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
-		 dev->coherent_dma_mask <= DMA_BIT_MASK(32))
-		gfp |= __GFP_DMA32;
-
-	/* Don't invoke OOM killer */
-	gfp |= __GFP_NORETRY;
-
-	ret = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
+	void *ret = swiotlb_alloc(dev, size, dma_handle, gfp, attrs);
 
 	mb();
 
 	return ret;
 }
 
-static void octeon_dma_free_coherent(struct device *dev, size_t size,
-	void *vaddr, dma_addr_t dma_handle, unsigned long attrs)
-{
-	swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
 static dma_addr_t octeon_unity_phys_to_dma(struct device *dev, phys_addr_t paddr)
 {
 	return paddr;
@@ -228,7 +205,7 @@ EXPORT_SYMBOL(dma_to_phys);
 static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
 	.dma_map_ops = {
 		.alloc = octeon_dma_alloc_coherent,
-		.free = octeon_dma_free_coherent,
+		.free = swiotlb_free,
 		.map_page = octeon_dma_map_page,
 		.unmap_page = swiotlb_unmap_page,
 		.map_sg = octeon_dma_map_sg,
@@ -314,7 +291,7 @@ void __init plat_swiotlb_setup(void)
 static struct octeon_dma_map_ops _octeon_pci_dma_map_ops = {
 	.dma_map_ops = {
 		.alloc = octeon_dma_alloc_coherent,
-		.free = octeon_dma_free_coherent,
+		.free = swiotlb_free,
 		.map_page = octeon_dma_map_page,
 		.unmap_page = swiotlb_unmap_page,
 		.map_sg = octeon_dma_map_sg,
diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c
index 5b3a3f6a9ad3..d99f5242169e 100644
--- a/arch/mips/cavium-octeon/octeon-irq.c
+++ b/arch/mips/cavium-octeon/octeon-irq.c
@@ -2277,6 +2277,8 @@ static int __init octeon_irq_init_cib(struct device_node *ciu_node,
 	}
 
 	host_data = kzalloc(sizeof(*host_data), GFP_KERNEL);
+	if (!host_data)
+		return -ENOMEM;
 	raw_spin_lock_init(&host_data->lock);
 
 	addr = of_get_address(ciu_node, 0, NULL, NULL);
diff --git a/arch/mips/configs/ar7_defconfig b/arch/mips/configs/ar7_defconfig
index 92fca3c42eac..5651f4d8f45c 100644
--- a/arch/mips/configs/ar7_defconfig
+++ b/arch/mips/configs/ar7_defconfig
@@ -82,7 +82,6 @@ CONFIG_MAC80211=m
 CONFIG_MAC80211_RC_PID=y
 CONFIG_MAC80211_RC_DEFAULT_PID=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_BLOCK=y
 CONFIG_MTD_CFI=y
diff --git a/arch/mips/configs/ath25_defconfig b/arch/mips/configs/ath25_defconfig
index 2c829950be17..b8d48038e74f 100644
--- a/arch/mips/configs/ath25_defconfig
+++ b/arch/mips/configs/ath25_defconfig
@@ -38,7 +38,6 @@ CONFIG_CFG80211=m
 CONFIG_MAC80211=m
 CONFIG_MAC80211_DEBUGFS=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_REDBOOT_PARTS=y
 CONFIG_MTD_REDBOOT_DIRECTORY_BLOCK=-2
diff --git a/arch/mips/configs/ath79_defconfig b/arch/mips/configs/ath79_defconfig
index 25ed914933e5..951c4231bdb8 100644
--- a/arch/mips/configs/ath79_defconfig
+++ b/arch/mips/configs/ath79_defconfig
@@ -39,7 +39,6 @@ CONFIG_CFG80211=m
 CONFIG_MAC80211=m
 CONFIG_MAC80211_DEBUGFS=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_REDBOOT_PARTS=y
 CONFIG_MTD_REDBOOT_DIRECTORY_BLOCK=-2
diff --git a/arch/mips/configs/bigsur_defconfig b/arch/mips/configs/bigsur_defconfig
index a55009edbb29..5e73fe755be6 100644
--- a/arch/mips/configs/bigsur_defconfig
+++ b/arch/mips/configs/bigsur_defconfig
@@ -153,7 +153,6 @@ CONFIG_SLIP_COMPRESSED=y
 CONFIG_SLIP_SMART=y
 CONFIG_SLIP_MODE_SLIP6=y
 # CONFIG_INPUT is not set
-# CONFIG_SERIO_I8042 is not set
 CONFIG_SERIO_RAW=m
 # CONFIG_VT is not set
 CONFIG_SERIAL_NONSTANDARD=y
diff --git a/arch/mips/configs/gcw0_defconfig b/arch/mips/configs/gcw0_defconfig
new file mode 100644
index 000000000000..99ac1fa3b35f
--- /dev/null
+++ b/arch/mips/configs/gcw0_defconfig
@@ -0,0 +1,27 @@
+CONFIG_MACH_INGENIC=y
+CONFIG_JZ4770_GCW0=y
+CONFIG_HIGHMEM=y
+# CONFIG_BOUNCE is not set
+CONFIG_PREEMPT_VOLUNTARY=y
+# CONFIG_SECCOMP is not set
+CONFIG_NO_HZ_IDLE=y
+CONFIG_HIGH_RES_TIMERS=y
+CONFIG_EMBEDDED=y
+# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_SUSPEND is not set
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+CONFIG_DEVTMPFS=y
+CONFIG_DEVTMPFS_MOUNT=y
+CONFIG_NETDEVICES=y
+CONFIG_SERIAL_8250=y
+# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
+CONFIG_SERIAL_8250_CONSOLE=y
+CONFIG_SERIAL_8250_INGENIC=y
+CONFIG_USB=y
+CONFIG_USB_OHCI_HCD=y
+CONFIG_USB_OHCI_HCD_PLATFORM=y
+CONFIG_NOP_USB_XCEIV=y
+CONFIG_TMPFS=y
diff --git a/arch/mips/configs/generic/board-ranchu.config b/arch/mips/configs/generic/board-ranchu.config
new file mode 100644
index 000000000000..fee9ad4c5598
--- /dev/null
+++ b/arch/mips/configs/generic/board-ranchu.config
@@ -0,0 +1,30 @@
+CONFIG_VIRT_BOARD_RANCHU=y
+
+CONFIG_BATTERY_GOLDFISH=y
+CONFIG_FB=y
+CONFIG_FB_GOLDFISH=y
+CONFIG_GOLDFISH=y
+CONFIG_STAGING=y
+CONFIG_GOLDFISH_AUDIO=y
+CONFIG_GOLDFISH_PIC=y
+CONFIG_GOLDFISH_PIPE=y
+CONFIG_GOLDFISH_TTY=y
+CONFIG_RTC_CLASS=y
+CONFIG_RTC_DRV_GOLDFISH=y
+
+CONFIG_INPUT_EVDEV=y
+CONFIG_INPUT_KEYBOARD=y
+CONFIG_KEYBOARD_GOLDFISH_EVENTS=y
+
+CONFIG_MAGIC_SYSRQ=y
+CONFIG_POWER_SUPPLY=y
+CONFIG_POWER_RESET=y
+CONFIG_POWER_RESET_SYSCON=y
+CONFIG_POWER_RESET_SYSCON_POWEROFF=y
+
+CONFIG_VIRTIO_BLK=y
+CONFIG_VIRTIO_CONSOLE=y
+CONFIG_VIRTIO_MMIO=y
+CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y
+CONFIG_NETDEVICES=y
+CONFIG_VIRTIO_NET=y
diff --git a/arch/mips/configs/ip27_defconfig b/arch/mips/configs/ip27_defconfig
index a0d593248668..91a9c13e2c82 100644
--- a/arch/mips/configs/ip27_defconfig
+++ b/arch/mips/configs/ip27_defconfig
@@ -252,7 +252,6 @@ CONFIG_RT2800PCI=m
 CONFIG_WL12XX=m
 CONFIG_WL1251=m
 # CONFIG_INPUT is not set
-# CONFIG_SERIO_I8042 is not set
 CONFIG_SERIO_LIBPS2=m
 CONFIG_SERIO_RAW=m
 CONFIG_SERIO_ALTERA_PS2=m
diff --git a/arch/mips/configs/ip32_defconfig b/arch/mips/configs/ip32_defconfig
index 1e26e58b9dc3..ebff297328ae 100644
--- a/arch/mips/configs/ip32_defconfig
+++ b/arch/mips/configs/ip32_defconfig
@@ -75,7 +75,6 @@ CONFIG_DE2104X=m
 CONFIG_TULIP=m
 CONFIG_TULIP_MMIO=y
 CONFIG_INPUT_EVDEV=m
-# CONFIG_SERIO_I8042 is not set
 CONFIG_SERIO_MACEPS2=y
 CONFIG_SERIO_RAW=y
 # CONFIG_CONSOLE_TRANSLATIONS is not set
diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig
index 396408404487..df8a9a15ca83 100644
--- a/arch/mips/configs/malta_defconfig
+++ b/arch/mips/configs/malta_defconfig
@@ -312,9 +312,8 @@ CONFIG_HOSTAP_PCI=m
 CONFIG_IPW2100=m
 CONFIG_IPW2100_MONITOR=y
 CONFIG_LIBERTAS=m
-# CONFIG_INPUT_KEYBOARD is not set
-# CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO_I8042 is not set
+CONFIG_INPUT_MOUSEDEV=y
+CONFIG_MOUSE_PS2_ELANTECH=y
 CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
 CONFIG_POWER_RESET=y
diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig
index 5691673a3327..14df9ef15d40 100644
--- a/arch/mips/configs/malta_kvm_defconfig
+++ b/arch/mips/configs/malta_kvm_defconfig
@@ -324,9 +324,7 @@ CONFIG_HOSTAP_PCI=m
 CONFIG_IPW2100=m
 CONFIG_IPW2100_MONITOR=y
 CONFIG_LIBERTAS=m
-# CONFIG_INPUT_KEYBOARD is not set
-# CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO_I8042 is not set
+CONFIG_INPUT_MOUSEDEV=y
 CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
 CONFIG_POWER_RESET=y
diff --git a/arch/mips/configs/malta_kvm_guest_defconfig b/arch/mips/configs/malta_kvm_guest_defconfig
index e9cadb37d684..25092e344574 100644
--- a/arch/mips/configs/malta_kvm_guest_defconfig
+++ b/arch/mips/configs/malta_kvm_guest_defconfig
@@ -326,9 +326,7 @@ CONFIG_HOSTAP_PCI=m
 CONFIG_IPW2100=m
 CONFIG_IPW2100_MONITOR=y
 CONFIG_LIBERTAS=m
-# CONFIG_INPUT_KEYBOARD is not set
-# CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO_I8042 is not set
+CONFIG_INPUT_MOUSEDEV=y
 CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
 CONFIG_POWER_RESET=y
diff --git a/arch/mips/configs/malta_qemu_32r6_defconfig b/arch/mips/configs/malta_qemu_32r6_defconfig
index 77145ecaa23b..210bf609f785 100644
--- a/arch/mips/configs/malta_qemu_32r6_defconfig
+++ b/arch/mips/configs/malta_qemu_32r6_defconfig
@@ -126,6 +126,7 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_VIA is not set
 # CONFIG_NET_VENDOR_WIZNET is not set
 # CONFIG_WLAN is not set
+CONFIG_INPUT_MOUSEDEV=y
 # CONFIG_VT is not set
 CONFIG_LEGACY_PTY_COUNT=4
 CONFIG_SERIAL_8250=y
diff --git a/arch/mips/configs/maltaaprp_defconfig b/arch/mips/configs/maltaaprp_defconfig
index cc2687cfdc13..e5934aa98397 100644
--- a/arch/mips/configs/maltaaprp_defconfig
+++ b/arch/mips/configs/maltaaprp_defconfig
@@ -126,6 +126,7 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_TOSHIBA is not set
 # CONFIG_NET_VENDOR_VIA is not set
 # CONFIG_WLAN is not set
+CONFIG_INPUT_MOUSEDEV=y
 # CONFIG_VT is not set
 CONFIG_LEGACY_PTY_COUNT=16
 CONFIG_SERIAL_8250=y
diff --git a/arch/mips/configs/maltasmvp_defconfig b/arch/mips/configs/maltasmvp_defconfig
index d8c8f5fb8918..cb2ca11c1789 100644
--- a/arch/mips/configs/maltasmvp_defconfig
+++ b/arch/mips/configs/maltasmvp_defconfig
@@ -127,6 +127,7 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_VIA is not set
 # CONFIG_NET_VENDOR_WIZNET is not set
 # CONFIG_WLAN is not set
+CONFIG_INPUT_MOUSEDEV=y
 # CONFIG_VT is not set
 CONFIG_LEGACY_PTY_COUNT=4
 CONFIG_SERIAL_8250=y
diff --git a/arch/mips/configs/maltasmvp_eva_defconfig b/arch/mips/configs/maltasmvp_eva_defconfig
index 04827bc9f87f..be29fcec69fc 100644
--- a/arch/mips/configs/maltasmvp_eva_defconfig
+++ b/arch/mips/configs/maltasmvp_eva_defconfig
@@ -130,6 +130,7 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_VIA is not set
 # CONFIG_NET_VENDOR_WIZNET is not set
 # CONFIG_WLAN is not set
+CONFIG_INPUT_MOUSEDEV=y
 # CONFIG_VT is not set
 CONFIG_LEGACY_PTY_COUNT=4
 CONFIG_SERIAL_8250=y
diff --git a/arch/mips/configs/maltaup_defconfig b/arch/mips/configs/maltaup_defconfig
index 7ea7c0ba2666..40462d4c90a0 100644
--- a/arch/mips/configs/maltaup_defconfig
+++ b/arch/mips/configs/maltaup_defconfig
@@ -125,6 +125,7 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_TOSHIBA is not set
 # CONFIG_NET_VENDOR_VIA is not set
 # CONFIG_WLAN is not set
+CONFIG_INPUT_MOUSEDEV=y
 # CONFIG_VT is not set
 CONFIG_LEGACY_PTY_COUNT=16
 CONFIG_SERIAL_8250=y
diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig
index 2942610e4082..4e50176cb3df 100644
--- a/arch/mips/configs/maltaup_xpa_defconfig
+++ b/arch/mips/configs/maltaup_xpa_defconfig
@@ -321,9 +321,8 @@ CONFIG_HOSTAP_PCI=m
 CONFIG_IPW2100=m
 CONFIG_IPW2100_MONITOR=y
 CONFIG_LIBERTAS=m
-# CONFIG_INPUT_KEYBOARD is not set
-# CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO_I8042 is not set
+CONFIG_INPUT_MOUSEDEV=y
+CONFIG_MOUSE_PS2_ELANTECH=y
 CONFIG_SERIAL_8250=y
 CONFIG_SERIAL_8250_CONSOLE=y
 CONFIG_POWER_RESET=y
diff --git a/arch/mips/configs/nlm_xlp_defconfig b/arch/mips/configs/nlm_xlp_defconfig
index 7357248b3d7a..e8e1dd8e0e99 100644
--- a/arch/mips/configs/nlm_xlp_defconfig
+++ b/arch/mips/configs/nlm_xlp_defconfig
@@ -399,7 +399,6 @@ CONFIG_INPUT_EVDEV=y
 CONFIG_INPUT_EVBUG=m
 # CONFIG_INPUT_KEYBOARD is not set
 # CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO_I8042 is not set
 CONFIG_SERIO_SERPORT=m
 CONFIG_SERIO_LIBPS2=y
 CONFIG_SERIO_RAW=m
diff --git a/arch/mips/configs/nlm_xlr_defconfig b/arch/mips/configs/nlm_xlr_defconfig
index 1e18fd7de209..c4477a4d40c1 100644
--- a/arch/mips/configs/nlm_xlr_defconfig
+++ b/arch/mips/configs/nlm_xlr_defconfig
@@ -332,7 +332,6 @@ CONFIG_INPUT_EVDEV=y
 CONFIG_INPUT_EVBUG=m
 # CONFIG_INPUT_KEYBOARD is not set
 # CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO_I8042 is not set
 CONFIG_SERIO_SERPORT=m
 CONFIG_SERIO_LIBPS2=y
 CONFIG_SERIO_RAW=m
diff --git a/arch/mips/configs/pic32mzda_defconfig b/arch/mips/configs/pic32mzda_defconfig
index 52192c632ae8..41190c2036e6 100644
--- a/arch/mips/configs/pic32mzda_defconfig
+++ b/arch/mips/configs/pic32mzda_defconfig
@@ -26,7 +26,6 @@ CONFIG_BINFMT_MISC=m
 # CONFIG_SUSPEND is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 # CONFIG_ALLOW_DEV_COREDUMP is not set
 CONFIG_BLK_DEV_LOOP=m
 CONFIG_SCSI=y
diff --git a/arch/mips/configs/pnx8335_stb225_defconfig b/arch/mips/configs/pnx8335_stb225_defconfig
index 81b5eb89446c..e73cdb08fc6e 100644
--- a/arch/mips/configs/pnx8335_stb225_defconfig
+++ b/arch/mips/configs/pnx8335_stb225_defconfig
@@ -49,7 +49,6 @@ CONFIG_INPUT_EVDEV=m
 CONFIG_INPUT_EVBUG=m
 # CONFIG_INPUT_KEYBOARD is not set
 # CONFIG_INPUT_MOUSE is not set
-# CONFIG_SERIO_I8042 is not set
 # CONFIG_VT_CONSOLE is not set
 CONFIG_SERIAL_PNX8XXX=y
 CONFIG_SERIAL_PNX8XXX_CONSOLE=y
diff --git a/arch/mips/configs/qi_lb60_defconfig b/arch/mips/configs/qi_lb60_defconfig
index 3f1333517405..3b02ff9a7c64 100644
--- a/arch/mips/configs/qi_lb60_defconfig
+++ b/arch/mips/configs/qi_lb60_defconfig
@@ -42,7 +42,6 @@ CONFIG_TCP_CONG_WESTWOOD=y
 # CONFIG_TCP_CONG_HTCP is not set
 # CONFIG_IPV6 is not set
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_BLOCK=y
 CONFIG_MTD_NAND=y
diff --git a/arch/mips/configs/rm200_defconfig b/arch/mips/configs/rm200_defconfig
index 99679e514042..5f71aa598b06 100644
--- a/arch/mips/configs/rm200_defconfig
+++ b/arch/mips/configs/rm200_defconfig
@@ -325,15 +325,6 @@ CONFIG_USB_SERIAL_EDGEPORT=m
 CONFIG_USB_SERIAL_EDGEPORT_TI=m
 CONFIG_USB_SERIAL_KEYSPAN_PDA=m
 CONFIG_USB_SERIAL_KEYSPAN=m
-CONFIG_USB_SERIAL_KEYSPAN_MPR=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XA=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XB=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QW=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QI=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49WLC=y
 CONFIG_USB_SERIAL_KLSI=m
 CONFIG_USB_SERIAL_KOBIL_SCT=m
 CONFIG_USB_SERIAL_MCT_U232=m
diff --git a/arch/mips/configs/rt305x_defconfig b/arch/mips/configs/rt305x_defconfig
index c695b7b1c4ae..dbe6a4639d05 100644
--- a/arch/mips/configs/rt305x_defconfig
+++ b/arch/mips/configs/rt305x_defconfig
@@ -76,7 +76,6 @@ CONFIG_VLAN_8021Q=y
 CONFIG_NET_SCHED=y
 CONFIG_HAMRADIO=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/mips/configs/sb1250_swarm_defconfig b/arch/mips/configs/sb1250_swarm_defconfig
index c724bdd6a7e6..1edd8430ad61 100644
--- a/arch/mips/configs/sb1250_swarm_defconfig
+++ b/arch/mips/configs/sb1250_swarm_defconfig
@@ -65,7 +65,6 @@ CONFIG_NET_ETHERNET=y
 CONFIG_MII=y
 CONFIG_SB1250_MAC=y
 # CONFIG_INPUT is not set
-# CONFIG_SERIO_I8042 is not set
 CONFIG_SERIO_RAW=m
 # CONFIG_VT is not set
 # CONFIG_HW_RANDOM is not set
diff --git a/arch/mips/configs/xway_defconfig b/arch/mips/configs/xway_defconfig
index 4365108bef77..fa750d501c11 100644
--- a/arch/mips/configs/xway_defconfig
+++ b/arch/mips/configs/xway_defconfig
@@ -75,7 +75,6 @@ CONFIG_VLAN_8021Q=y
 CONFIG_NET_SCHED=y
 CONFIG_HAMRADIO=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/mips/generic/Kconfig b/arch/mips/generic/Kconfig
index 52e0286a1612..2ff3b17bfab1 100644
--- a/arch/mips/generic/Kconfig
+++ b/arch/mips/generic/Kconfig
@@ -49,4 +49,14 @@ config FIT_IMAGE_FDT_XILFPGA
 	  Enable this to include the FDT for the MIPSfpga platform
 	  from Imagination Technologies in the FIT kernel image.
 
+config VIRT_BOARD_RANCHU
+	bool "Support Ranchu platform for Android emulator"
+	help
+	  This enables support for the platform used by Android emulator.
+
+	  Ranchu platform consists of a set of virtual devices. This platform
+	  enables emulation of variety of virtual configurations while using
+	  Android emulator. Android emulator is based on Qemu, and contains
+	  the support for the same set of virtual devices.
+
 endif
diff --git a/arch/mips/generic/Makefile b/arch/mips/generic/Makefile
index 874967363dbb..5c31e0c4697d 100644
--- a/arch/mips/generic/Makefile
+++ b/arch/mips/generic/Makefile
@@ -15,3 +15,4 @@ obj-y += proc.o
 obj-$(CONFIG_YAMON_DT_SHIM)		+= yamon-dt.o
 obj-$(CONFIG_LEGACY_BOARD_SEAD3)	+= board-sead3.o
 obj-$(CONFIG_KEXEC)			+= kexec.o
+obj-$(CONFIG_VIRT_BOARD_RANCHU)		+= board-ranchu.o
diff --git a/arch/mips/generic/board-ranchu.c b/arch/mips/generic/board-ranchu.c
new file mode 100644
index 000000000000..59a8c18fa2cc
--- /dev/null
+++ b/arch/mips/generic/board-ranchu.c
@@ -0,0 +1,93 @@
+/*
+ * Support code for virtual Ranchu board for MIPS.
+ *
+ * Author: Miodrag Dinic <miodrag.dinic@mips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation;  either version 2 of the  License, or (at your
+ * option) any later version.
+ */
+
+#include <linux/of_address.h>
+#include <linux/types.h>
+
+#include <asm/machine.h>
+#include <asm/mipsregs.h>
+#include <asm/time.h>
+
+#define GOLDFISH_TIMER_LOW		0x00
+#define GOLDFISH_TIMER_HIGH		0x04
+
+static __init u64 read_rtc_time(void __iomem *base)
+{
+	u32 time_low;
+	u32 time_high;
+
+	/*
+	 * Reading the low address latches the high value
+	 * as well so there is no fear that we may read
+	 * inaccurate high value.
+	 */
+	time_low = readl(base + GOLDFISH_TIMER_LOW);
+	time_high = readl(base + GOLDFISH_TIMER_HIGH);
+
+	return ((u64)time_high << 32) | time_low;
+}
+
+static __init unsigned int ranchu_measure_hpt_freq(void)
+{
+	u64 rtc_start, rtc_current, rtc_delta;
+	unsigned int start, count;
+	struct device_node *np;
+	void __iomem *rtc_base;
+
+	np = of_find_compatible_node(NULL, NULL, "google,goldfish-rtc");
+	if (!np)
+		panic("%s(): Failed to find 'google,goldfish-rtc' dt node!",
+		      __func__);
+
+	rtc_base = of_iomap(np, 0);
+	if (!rtc_base)
+		panic("%s(): Failed to ioremap Goldfish RTC base!", __func__);
+
+	/*
+	 * Poll the nanosecond resolution RTC for one
+	 * second to calibrate the CPU frequency.
+	 */
+	rtc_start = read_rtc_time(rtc_base);
+	start = read_c0_count();
+
+	do {
+		rtc_current = read_rtc_time(rtc_base);
+		rtc_delta = rtc_current - rtc_start;
+	} while (rtc_delta < NSEC_PER_SEC);
+
+	count = read_c0_count() - start;
+
+	/*
+	 * Make sure the frequency will be a round number.
+	 * Without this correction, the returned value may vary
+	 * between subsequent emulation executions.
+	 *
+	 * TODO: Set this value using device tree.
+	 */
+	count += 5000;
+	count -= count % 10000;
+
+	iounmap(rtc_base);
+
+	return count;
+}
+
+static const struct of_device_id ranchu_of_match[] __initconst = {
+	{
+		.compatible = "mti,ranchu",
+	},
+	{}
+};
+
+MIPS_MACHINE(ranchu) = {
+	.matches = ranchu_of_match,
+	.measure_hpt_freq = ranchu_measure_hpt_freq,
+};
diff --git a/arch/mips/generic/irq.c b/arch/mips/generic/irq.c
index 394f8161e462..cb7fdaeef426 100644
--- a/arch/mips/generic/irq.c
+++ b/arch/mips/generic/irq.c
@@ -22,10 +22,10 @@ int get_c0_fdc_int(void)
 {
 	int mips_cpu_fdc_irq;
 
-	if (cpu_has_veic)
-		panic("Unimplemented!");
-	else if (mips_gic_present())
+	if (mips_gic_present())
 		mips_cpu_fdc_irq = gic_get_c0_fdc_int();
+	else if (cpu_has_veic)
+		panic("Unimplemented!");
 	else if (cp0_fdc_irq >= 0)
 		mips_cpu_fdc_irq = MIPS_CPU_IRQ_BASE + cp0_fdc_irq;
 	else
@@ -38,10 +38,10 @@ int get_c0_perfcount_int(void)
 {
 	int mips_cpu_perf_irq;
 
-	if (cpu_has_veic)
-		panic("Unimplemented!");
-	else if (mips_gic_present())
+	if (mips_gic_present())
 		mips_cpu_perf_irq = gic_get_c0_perfcount_int();
+	else if (cpu_has_veic)
+		panic("Unimplemented!");
 	else if (cp0_perfcount_irq >= 0)
 		mips_cpu_perf_irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
 	else
@@ -54,10 +54,10 @@ unsigned int get_c0_compare_int(void)
 {
 	int mips_cpu_timer_irq;
 
-	if (cpu_has_veic)
-		panic("Unimplemented!");
-	else if (mips_gic_present())
+	if (mips_gic_present())
 		mips_cpu_timer_irq = gic_get_c0_compare_int();
+	else if (cpu_has_veic)
+		panic("Unimplemented!");
 	else
 		mips_cpu_timer_irq = MIPS_CPU_IRQ_BASE + cp0_compare_irq;
 
diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild
index b1f66699677d..45d541baf359 100644
--- a/arch/mips/include/asm/Kbuild
+++ b/arch/mips/include/asm/Kbuild
@@ -1,6 +1,5 @@
 # MIPS headers
 generic-(CONFIG_GENERIC_CSUM) += checksum.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += dma-contiguous.h
 generic-y += emergency-restart.h
diff --git a/arch/mips/include/asm/asm-prototypes.h b/arch/mips/include/asm/asm-prototypes.h
index d60b57f34e92..576f1a62dea9 100644
--- a/arch/mips/include/asm/asm-prototypes.h
+++ b/arch/mips/include/asm/asm-prototypes.h
@@ -3,5 +3,5 @@
 #include <asm/page.h>
 #include <asm/fpu.h>
 #include <asm-generic/asm-prototypes.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <asm/ftrace.h>
diff --git a/arch/mips/include/asm/bootinfo.h b/arch/mips/include/asm/bootinfo.h
index e26a093bb17a..a301a8f4bc66 100644
--- a/arch/mips/include/asm/bootinfo.h
+++ b/arch/mips/include/asm/bootinfo.h
@@ -79,6 +79,8 @@ enum loongson_machine_type {
  */
 #define  MACH_INGENIC_JZ4730	0	/* JZ4730 SOC		*/
 #define  MACH_INGENIC_JZ4740	1	/* JZ4740 SOC		*/
+#define  MACH_INGENIC_JZ4770	2	/* JZ4770 SOC		*/
+#define  MACH_INGENIC_JZ4780	3	/* JZ4780 SOC		*/
 
 extern char *system_type;
 const char *get_system_type(void);
diff --git a/arch/mips/include/asm/checksum.h b/arch/mips/include/asm/checksum.h
index 77cad232a1c6..e8161e4dfde7 100644
--- a/arch/mips/include/asm/checksum.h
+++ b/arch/mips/include/asm/checksum.h
@@ -110,7 +110,7 @@ __wsum csum_partial_copy_nocheck(const void *src, void *dst,
  */
 static inline __sum16 csum_fold(__wsum csum)
 {
-	u32 sum = (__force u32)csum;;
+	u32 sum = (__force u32)csum;
 
 	sum += (sum << 16);
 	csum = (sum < csum);
diff --git a/arch/mips/include/asm/compat.h b/arch/mips/include/asm/compat.h
index 49691331ada4..9a0fa66b81ac 100644
--- a/arch/mips/include/asm/compat.h
+++ b/arch/mips/include/asm/compat.h
@@ -86,7 +86,6 @@ struct compat_flock {
 	compat_off_t	l_len;
 	s32		l_sysid;
 	compat_pid_t	l_pid;
-	short		__unused;
 	s32		pad[4];
 };
 
@@ -126,79 +125,6 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32		compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-/* Can't use the generic version because si_code and si_errno are swapped */
-
-#define SI_PAD_SIZE32	(128/sizeof(int) - 3)
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_code;
-	int si_errno;
-
-	union {
-		int _pad[128 / sizeof(int) - 3];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			__compat_uid32_t _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;	/* sender's pid */
-			__compat_uid32_t _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;	/* which child */
-			__compat_uid32_t _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			compat_uptr_t _addr;	/* faulting insn/memory ref. */
-#ifdef __ARCH_SI_TRAPNO
-			int _trapno;	/* TRAP # which caused the signal */
-#endif
-			short _addr_lsb; /* LSB of the reported address */
-			struct {
-				compat_uptr_t _lower;
-				compat_uptr_t _upper;
-			} _addr_bnd;
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			compat_long_t _band; /* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		struct {
-			compat_uptr_t _call_addr; /* calling insn */
-			int _syscall;	/* triggering system call number */
-			compat_uint_t _arch;	/* AUDIT_ARCH_* of syscall */
-		} _sigsys;
-	} _sifields;
-} compat_siginfo_t;
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 /*
diff --git a/arch/mips/include/asm/dma-direct.h b/arch/mips/include/asm/dma-direct.h
new file mode 100644
index 000000000000..f32f15530aba
--- /dev/null
+++ b/arch/mips/include/asm/dma-direct.h
@@ -0,0 +1 @@
+#include <asm/dma-coherence.h>
diff --git a/arch/mips/include/asm/dma-mapping.h b/arch/mips/include/asm/dma-mapping.h
index 0d9418d264f9..886e75a383f2 100644
--- a/arch/mips/include/asm/dma-mapping.h
+++ b/arch/mips/include/asm/dma-mapping.h
@@ -17,16 +17,6 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 	return mips_dma_map_ops;
 }
 
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return false;
-
-	return addr + size <= *dev->dma_mask;
-}
-
-static inline void dma_mark_clean(void *addr, size_t size) {}
-
 #define arch_setup_dma_ops arch_setup_dma_ops
 static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
 				      u64 size, const struct iommu_ops *iommu,
diff --git a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_dev_enet.h b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_dev_enet.h
index c0bd47444cff..da39e4d326ba 100644
--- a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_dev_enet.h
+++ b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_dev_enet.h
@@ -55,6 +55,10 @@ struct bcm63xx_enet_platform_data {
 
 	/* DMA descriptor shift */
 	unsigned int dma_desc_shift;
+
+	/* dma channel ids */
+	int rx_chan;
+	int tx_chan;
 };
 
 /*
diff --git a/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h b/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h
index 9110988b92a1..138edf6b5b48 100644
--- a/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h
+++ b/arch/mips/include/asm/mach-cavium-octeon/dma-coherence.h
@@ -61,6 +61,14 @@ static inline void plat_post_dma_flush(struct device *dev)
 {
 }
 
+static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
+{
+	if (!dev->dma_mask)
+		return false;
+
+	return addr + size - 1 <= *dev->dma_mask;
+}
+
 dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
 phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr);
 
diff --git a/arch/mips/include/asm/mach-generic/dma-coherence.h b/arch/mips/include/asm/mach-generic/dma-coherence.h
index 61addb1677e9..8ad7a40ca786 100644
--- a/arch/mips/include/asm/mach-generic/dma-coherence.h
+++ b/arch/mips/include/asm/mach-generic/dma-coherence.h
@@ -70,16 +70,4 @@ static inline void plat_post_dma_flush(struct device *dev)
 }
 #endif
 
-#ifdef CONFIG_SWIOTLB
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return paddr;
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
-{
-	return daddr;
-}
-#endif
-
 #endif /* __ASM_MACH_GENERIC_DMA_COHERENCE_H */
diff --git a/arch/mips/include/asm/mach-loongson64/boot_param.h b/arch/mips/include/asm/mach-loongson64/boot_param.h
index 4f69f08717f6..8c286bedff3e 100644
--- a/arch/mips/include/asm/mach-loongson64/boot_param.h
+++ b/arch/mips/include/asm/mach-loongson64/boot_param.h
@@ -4,7 +4,7 @@
 
 #define SYSTEM_RAM_LOW		1
 #define SYSTEM_RAM_HIGH		2
-#define MEM_RESERVED		3
+#define SYSTEM_RAM_RESERVED	3
 #define PCI_IO			4
 #define PCI_MEM			5
 #define LOONGSON_CFG_REG	6
diff --git a/arch/mips/include/asm/mach-loongson64/dma-coherence.h b/arch/mips/include/asm/mach-loongson64/dma-coherence.h
index 1602a9e9e8c2..b1b575f5c6c1 100644
--- a/arch/mips/include/asm/mach-loongson64/dma-coherence.h
+++ b/arch/mips/include/asm/mach-loongson64/dma-coherence.h
@@ -17,6 +17,14 @@
 
 struct device;
 
+static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
+{
+	if (!dev->dma_mask)
+		return false;
+
+	return addr + size - 1 <= *dev->dma_mask;
+}
+
 extern dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
 extern phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr);
 static inline dma_addr_t plat_map_dma_mem(struct device *dev, void *addr,
diff --git a/arch/mips/include/asm/machine.h b/arch/mips/include/asm/machine.h
index e0d9b373d415..f83879dadd1e 100644
--- a/arch/mips/include/asm/machine.h
+++ b/arch/mips/include/asm/machine.h
@@ -52,7 +52,7 @@ mips_machine_is_compatible(const struct mips_machine *mach, const void *fdt)
 	if (!mach->matches)
 		return NULL;
 
-	for (match = mach->matches; match->compatible; match++) {
+	for (match = mach->matches; match->compatible[0]; match++) {
 		if (fdt_node_check_compatible(fdt, 0, match->compatible) == 0)
 			return match;
 	}
diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h
index 6b1f1ad0542c..858752dac337 100644
--- a/arch/mips/include/asm/mipsregs.h
+++ b/arch/mips/include/asm/mipsregs.h
@@ -1180,6 +1180,89 @@ static inline int mm_insn_16bit(u16 insn)
 #define _ASM_INSN_IF_MIPS(_enc)
 #endif
 
+/*
+ * parse_r var, r - Helper assembler macro for parsing register names.
+ *
+ * This converts the register name in $n form provided in \r to the
+ * corresponding register number, which is assigned to the variable \var. It is
+ * needed to allow explicit encoding of instructions in inline assembly where
+ * registers are chosen by the compiler in $n form, allowing us to avoid using
+ * fixed register numbers.
+ *
+ * It also allows newer instructions (not implemented by the assembler) to be
+ * transparently implemented using assembler macros, instead of needing separate
+ * cases depending on toolchain support.
+ *
+ * Simple usage example:
+ * __asm__ __volatile__("parse_r __rt, %0\n\t"
+ *			".insn\n\t"
+ *			"# di    %0\n\t"
+ *			".word   (0x41606000 | (__rt << 16))"
+ *			: "=r" (status);
+ */
+
+/* Match an individual register number and assign to \var */
+#define _IFC_REG(n)				\
+	".ifc	\\r, $" #n "\n\t"		\
+	"\\var	= " #n "\n\t"			\
+	".endif\n\t"
+
+__asm__(".macro	parse_r var r\n\t"
+	"\\var	= -1\n\t"
+	_IFC_REG(0)  _IFC_REG(1)  _IFC_REG(2)  _IFC_REG(3)
+	_IFC_REG(4)  _IFC_REG(5)  _IFC_REG(6)  _IFC_REG(7)
+	_IFC_REG(8)  _IFC_REG(9)  _IFC_REG(10) _IFC_REG(11)
+	_IFC_REG(12) _IFC_REG(13) _IFC_REG(14) _IFC_REG(15)
+	_IFC_REG(16) _IFC_REG(17) _IFC_REG(18) _IFC_REG(19)
+	_IFC_REG(20) _IFC_REG(21) _IFC_REG(22) _IFC_REG(23)
+	_IFC_REG(24) _IFC_REG(25) _IFC_REG(26) _IFC_REG(27)
+	_IFC_REG(28) _IFC_REG(29) _IFC_REG(30) _IFC_REG(31)
+	".iflt	\\var\n\t"
+	".error	\"Unable to parse register name \\r\"\n\t"
+	".endif\n\t"
+	".endm");
+
+#undef _IFC_REG
+
+/*
+ * C macros for generating assembler macros for common instruction formats.
+ *
+ * The names of the operands can be chosen by the caller, and the encoding of
+ * register operand \<Rn> is assigned to __<Rn> where it can be accessed from
+ * the ENC encodings.
+ */
+
+/* Instructions with no operands */
+#define _ASM_MACRO_0(OP, ENC)						\
+	__asm__(".macro	" #OP "\n\t"					\
+		ENC							\
+		".endm")
+
+/* Instructions with 2 register operands */
+#define _ASM_MACRO_2R(OP, R1, R2, ENC)					\
+	__asm__(".macro	" #OP " " #R1 ", " #R2 "\n\t"			\
+		"parse_r __" #R1 ", \\" #R1 "\n\t"			\
+		"parse_r __" #R2 ", \\" #R2 "\n\t"			\
+		ENC							\
+		".endm")
+
+/* Instructions with 3 register operands */
+#define _ASM_MACRO_3R(OP, R1, R2, R3, ENC)				\
+	__asm__(".macro	" #OP " " #R1 ", " #R2 ", " #R3 "\n\t"		\
+		"parse_r __" #R1 ", \\" #R1 "\n\t"			\
+		"parse_r __" #R2 ", \\" #R2 "\n\t"			\
+		"parse_r __" #R3 ", \\" #R3 "\n\t"			\
+		ENC							\
+		".endm")
+
+/* Instructions with 2 register operands and 1 optional select operand */
+#define _ASM_MACRO_2R_1S(OP, R1, R2, SEL3, ENC)				\
+	__asm__(".macro	" #OP " " #R1 ", " #R2 ", " #SEL3 " = 0\n\t"	\
+		"parse_r __" #R1 ", \\" #R1 "\n\t"			\
+		"parse_r __" #R2 ", \\" #R2 "\n\t"			\
+		ENC							\
+		".endm")
+
 /*
  * TLB Invalidate Flush
  */
@@ -1245,14 +1328,14 @@ do {								\
  * Macros to access the system control coprocessor
  */
 
-#define __read_32bit_c0_register(source, sel)				\
+#define ___read_32bit_c0_register(source, sel, vol)			\
 ({ unsigned int __res;							\
 	if (sel == 0)							\
-		__asm__ __volatile__(					\
+		__asm__ vol(						\
 			"mfc0\t%0, " #source "\n\t"			\
 			: "=r" (__res));				\
 	else								\
-		__asm__ __volatile__(					\
+		__asm__ vol(						\
 			".set\tmips32\n\t"				\
 			"mfc0\t%0, " #source ", " #sel "\n\t"		\
 			".set\tmips0\n\t"				\
@@ -1260,18 +1343,18 @@ do {								\
 	__res;								\
 })
 
-#define __read_64bit_c0_register(source, sel)				\
+#define ___read_64bit_c0_register(source, sel, vol)			\
 ({ unsigned long long __res;						\
 	if (sizeof(unsigned long) == 4)					\
-		__res = __read_64bit_c0_split(source, sel);		\
+		__res = __read_64bit_c0_split(source, sel, vol);	\
 	else if (sel == 0)						\
-		__asm__ __volatile__(					\
+		__asm__ vol(						\
 			".set\tmips3\n\t"				\
 			"dmfc0\t%0, " #source "\n\t"			\
 			".set\tmips0"					\
 			: "=r" (__res));				\
 	else								\
-		__asm__ __volatile__(					\
+		__asm__ vol(						\
 			".set\tmips64\n\t"				\
 			"dmfc0\t%0, " #source ", " #sel "\n\t"		\
 			".set\tmips0"					\
@@ -1279,6 +1362,18 @@ do {								\
 	__res;								\
 })
 
+#define __read_32bit_c0_register(source, sel)				\
+	___read_32bit_c0_register(source, sel, __volatile__)
+
+#define __read_const_32bit_c0_register(source, sel)			\
+	___read_32bit_c0_register(source, sel,)
+
+#define __read_64bit_c0_register(source, sel)				\
+	___read_64bit_c0_register(source, sel, __volatile__)
+
+#define __read_const_64bit_c0_register(source, sel)			\
+	___read_64bit_c0_register(source, sel,)
+
 #define __write_32bit_c0_register(register, sel, value)			\
 do {									\
 	if (sel == 0)							\
@@ -1316,6 +1411,11 @@ do {									\
 	(unsigned long) __read_32bit_c0_register(reg, sel) :		\
 	(unsigned long) __read_64bit_c0_register(reg, sel))
 
+#define __read_const_ulong_c0_register(reg, sel)			\
+	((sizeof(unsigned long) == 4) ?					\
+	(unsigned long) __read_const_32bit_c0_register(reg, sel) :	\
+	(unsigned long) __read_const_64bit_c0_register(reg, sel))
+
 #define __write_ulong_c0_register(reg, sel, val)			\
 do {									\
 	if (sizeof(unsigned long) == 4)					\
@@ -1346,14 +1446,14 @@ do {									\
  * These versions are only needed for systems with more than 38 bits of
  * physical address space running the 32-bit kernel.  That's none atm :-)
  */
-#define __read_64bit_c0_split(source, sel)				\
+#define __read_64bit_c0_split(source, sel, vol)				\
 ({									\
 	unsigned long long __val;					\
 	unsigned long __flags;						\
 									\
 	local_irq_save(__flags);					\
 	if (sel == 0)							\
-		__asm__ __volatile__(					\
+		__asm__ vol(						\
 			".set\tmips64\n\t"				\
 			"dmfc0\t%L0, " #source "\n\t"			\
 			"dsra\t%M0, %L0, 32\n\t"			\
@@ -1361,7 +1461,7 @@ do {									\
 			".set\tmips0"					\
 			: "=r" (__val));				\
 	else								\
-		__asm__ __volatile__(					\
+		__asm__ vol(						\
 			".set\tmips64\n\t"				\
 			"dmfc0\t%L0, " #source ", " #sel "\n\t"		\
 			"dsra\t%M0, %L0, 32\n\t"			\
@@ -1404,37 +1504,43 @@ do {									\
 	local_irq_restore(__flags);					\
 } while (0)
 
-#define __readx_32bit_c0_register(source)				\
+#ifndef TOOLCHAIN_SUPPORTS_XPA
+_ASM_MACRO_2R_1S(mfhc0, rt, rs, sel,
+	_ASM_INSN_IF_MIPS(0x40400000 | __rt << 16 | __rs << 11 | \\sel)
+	_ASM_INSN32_IF_MM(0x000000f4 | __rt << 21 | __rs << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(mthc0, rt, rd, sel,
+	_ASM_INSN_IF_MIPS(0x40c00000 | __rt << 16 | __rd << 11 | \\sel)
+	_ASM_INSN32_IF_MM(0x000002f4 | __rt << 21 | __rd << 16 | \\sel << 11));
+#define _ASM_SET_XPA ""
+#else	/* !TOOLCHAIN_SUPPORTS_XPA */
+#define _ASM_SET_XPA ".set\txpa\n\t"
+#endif
+
+#define __readx_32bit_c0_register(source, sel)				\
 ({									\
 	unsigned int __res;						\
 									\
 	__asm__ __volatile__(						\
 	"	.set	push					\n"	\
-	"	.set	noat					\n"	\
 	"	.set	mips32r2				\n"	\
-	"	# mfhc0 $1, %1					\n"	\
-	_ASM_INSN_IF_MIPS(0x40410000 | ((%1 & 0x1f) << 11))		\
-	_ASM_INSN32_IF_MM(0x002000f4 | ((%1 & 0x1f) << 16))		\
-	"	move	%0, $1					\n"	\
+	_ASM_SET_XPA							\
+	"	mfhc0	%0, " #source ", %1			\n"	\
 	"	.set	pop					\n"	\
 	: "=r" (__res)							\
-	: "i" (source));						\
+	: "i" (sel));							\
 	__res;								\
 })
 
-#define __writex_32bit_c0_register(register, value)			\
+#define __writex_32bit_c0_register(register, sel, value)		\
 do {									\
 	__asm__ __volatile__(						\
 	"	.set	push					\n"	\
-	"	.set	noat					\n"	\
 	"	.set	mips32r2				\n"	\
-	"	move	$1, %0					\n"	\
-	"	# mthc0 $1, %1					\n"	\
-	_ASM_INSN_IF_MIPS(0x40c10000 | ((%1 & 0x1f) << 11))		\
-	_ASM_INSN32_IF_MM(0x002002f4 | ((%1 & 0x1f) << 16))		\
+	_ASM_SET_XPA							\
+	"	mthc0	%z0, " #register ", %1			\n"	\
 	"	.set	pop					\n"	\
 	:								\
-	: "r" (value), "i" (register));					\
+	: "Jr" (value), "i" (sel));					\
 } while (0)
 
 #define read_c0_index()		__read_32bit_c0_register($0, 0)
@@ -1446,14 +1552,14 @@ do {									\
 #define read_c0_entrylo0()	__read_ulong_c0_register($2, 0)
 #define write_c0_entrylo0(val)	__write_ulong_c0_register($2, 0, val)
 
-#define readx_c0_entrylo0()	__readx_32bit_c0_register(2)
-#define writex_c0_entrylo0(val)	__writex_32bit_c0_register(2, val)
+#define readx_c0_entrylo0()	__readx_32bit_c0_register($2, 0)
+#define writex_c0_entrylo0(val)	__writex_32bit_c0_register($2, 0, val)
 
 #define read_c0_entrylo1()	__read_ulong_c0_register($3, 0)
 #define write_c0_entrylo1(val)	__write_ulong_c0_register($3, 0, val)
 
-#define readx_c0_entrylo1()	__readx_32bit_c0_register(3)
-#define writex_c0_entrylo1(val)	__writex_32bit_c0_register(3, val)
+#define readx_c0_entrylo1()	__readx_32bit_c0_register($3, 0)
+#define writex_c0_entrylo1(val)	__writex_32bit_c0_register($3, 0, val)
 
 #define read_c0_conf()		__read_32bit_c0_register($3, 0)
 #define write_c0_conf(val)	__write_32bit_c0_register($3, 0, val)
@@ -1541,7 +1647,7 @@ do {									\
 #define read_c0_epc()		__read_ulong_c0_register($14, 0)
 #define write_c0_epc(val)	__write_ulong_c0_register($14, 0, val)
 
-#define read_c0_prid()		__read_32bit_c0_register($15, 0)
+#define read_c0_prid()		__read_const_32bit_c0_register($15, 0)
 
 #define read_c0_cmgcrbase()	__read_ulong_c0_register($15, 3)
 
@@ -1830,18 +1936,44 @@ do {									\
  * Macros to access the guest system control coprocessor
  */
 
-#ifdef TOOLCHAIN_SUPPORTS_VIRT
+#ifndef TOOLCHAIN_SUPPORTS_VIRT
+_ASM_MACRO_2R_1S(mfgc0, rt, rs, sel,
+	_ASM_INSN_IF_MIPS(0x40600000 | __rt << 16 | __rs << 11 | \\sel)
+	_ASM_INSN32_IF_MM(0x000004fc | __rt << 21 | __rs << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(dmfgc0, rt, rs, sel,
+	_ASM_INSN_IF_MIPS(0x40600100 | __rt << 16 | __rs << 11 | \\sel)
+	_ASM_INSN32_IF_MM(0x580004fc | __rt << 21 | __rs << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(mtgc0, rt, rd, sel,
+	_ASM_INSN_IF_MIPS(0x40600200 | __rt << 16 | __rd << 11 | \\sel)
+	_ASM_INSN32_IF_MM(0x000006fc | __rt << 21 | __rd << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(dmtgc0, rt, rd, sel,
+	_ASM_INSN_IF_MIPS(0x40600300 | __rt << 16 | __rd << 11 | \\sel)
+	_ASM_INSN32_IF_MM(0x580006fc | __rt << 21 | __rd << 16 | \\sel << 11));
+_ASM_MACRO_0(tlbgp,    _ASM_INSN_IF_MIPS(0x42000010)
+		       _ASM_INSN32_IF_MM(0x0000017c));
+_ASM_MACRO_0(tlbgr,    _ASM_INSN_IF_MIPS(0x42000009)
+		       _ASM_INSN32_IF_MM(0x0000117c));
+_ASM_MACRO_0(tlbgwi,   _ASM_INSN_IF_MIPS(0x4200000a)
+		       _ASM_INSN32_IF_MM(0x0000217c));
+_ASM_MACRO_0(tlbgwr,   _ASM_INSN_IF_MIPS(0x4200000e)
+		       _ASM_INSN32_IF_MM(0x0000317c));
+_ASM_MACRO_0(tlbginvf, _ASM_INSN_IF_MIPS(0x4200000c)
+		       _ASM_INSN32_IF_MM(0x0000517c));
+#define _ASM_SET_VIRT ""
+#else	/* !TOOLCHAIN_SUPPORTS_VIRT */
+#define _ASM_SET_VIRT ".set\tvirt\n\t"
+#endif
 
 #define __read_32bit_gc0_register(source, sel)				\
 ({ int __res;								\
 	__asm__ __volatile__(						\
 		".set\tpush\n\t"					\
 		".set\tmips32r2\n\t"					\
-		".set\tvirt\n\t"					\
-		"mfgc0\t%0, $%1, %2\n\t"				\
+		_ASM_SET_VIRT						\
+		"mfgc0\t%0, " #source ", %1\n\t"			\
 		".set\tpop"						\
 		: "=r" (__res)						\
-		: "i" (source), "i" (sel));				\
+		: "i" (sel));						\
 	__res;								\
 })
 
@@ -1850,11 +1982,11 @@ do {									\
 	__asm__ __volatile__(						\
 		".set\tpush\n\t"					\
 		".set\tmips64r2\n\t"					\
-		".set\tvirt\n\t"					\
-		"dmfgc0\t%0, $%1, %2\n\t"			\
+		_ASM_SET_VIRT						\
+		"dmfgc0\t%0, " #source ", %1\n\t"			\
 		".set\tpop"						\
 		: "=r" (__res)						\
-		: "i" (source), "i" (sel));				\
+		: "i" (sel));						\
 	__res;								\
 })
 
@@ -1863,11 +1995,11 @@ do {									\
 	__asm__ __volatile__(						\
 		".set\tpush\n\t"					\
 		".set\tmips32r2\n\t"					\
-		".set\tvirt\n\t"					\
-		"mtgc0\t%z0, $%1, %2\n\t"				\
+		_ASM_SET_VIRT						\
+		"mtgc0\t%z0, " #register ", %1\n\t"			\
 		".set\tpop"						\
 		: : "Jr" ((unsigned int)(value)),			\
-		    "i" (register), "i" (sel));				\
+		    "i" (sel));						\
 } while (0)
 
 #define __write_64bit_gc0_register(register, sel, value)		\
@@ -1875,75 +2007,13 @@ do {									\
 	__asm__ __volatile__(						\
 		".set\tpush\n\t"					\
 		".set\tmips64r2\n\t"					\
-		".set\tvirt\n\t"					\
-		"dmtgc0\t%z0, $%1, %2\n\t"				\
+		_ASM_SET_VIRT						\
+		"dmtgc0\t%z0, " #register ", %1\n\t"			\
 		".set\tpop"						\
 		: : "Jr" (value),					\
-		    "i" (register), "i" (sel));				\
+		    "i" (sel));						\
 } while (0)
 
-#else	/* TOOLCHAIN_SUPPORTS_VIRT */
-
-#define __read_32bit_gc0_register(source, sel)				\
-({ int __res;								\
-	__asm__ __volatile__(						\
-		".set\tpush\n\t"					\
-		".set\tnoat\n\t"					\
-		"# mfgc0\t$1, $%1, %2\n\t"				\
-		_ASM_INSN_IF_MIPS(0x40610000 | %1 << 11 | %2)		\
-		_ASM_INSN32_IF_MM(0x002004fc | %1 << 16 | %2 << 11)	\
-		"move\t%0, $1\n\t"					\
-		".set\tpop"						\
-		: "=r" (__res)						\
-		: "i" (source), "i" (sel));				\
-	__res;								\
-})
-
-#define __read_64bit_gc0_register(source, sel)				\
-({ unsigned long long __res;						\
-	__asm__ __volatile__(						\
-		".set\tpush\n\t"					\
-		".set\tnoat\n\t"					\
-		"# dmfgc0\t$1, $%1, %2\n\t"				\
-		_ASM_INSN_IF_MIPS(0x40610100 | %1 << 11 | %2)		\
-		_ASM_INSN32_IF_MM(0x582004fc | %1 << 16 | %2 << 11)	\
-		"move\t%0, $1\n\t"					\
-		".set\tpop"						\
-		: "=r" (__res)						\
-		: "i" (source), "i" (sel));				\
-	__res;								\
-})
-
-#define __write_32bit_gc0_register(register, sel, value)		\
-do {									\
-	__asm__ __volatile__(						\
-		".set\tpush\n\t"					\
-		".set\tnoat\n\t"					\
-		"move\t$1, %z0\n\t"					\
-		"# mtgc0\t$1, $%1, %2\n\t"				\
-		_ASM_INSN_IF_MIPS(0x40610200 | %1 << 11 | %2)		\
-		_ASM_INSN32_IF_MM(0x002006fc | %1 << 16 | %2 << 11)	\
-		".set\tpop"						\
-		: : "Jr" ((unsigned int)(value)),			\
-		    "i" (register), "i" (sel));				\
-} while (0)
-
-#define __write_64bit_gc0_register(register, sel, value)		\
-do {									\
-	__asm__ __volatile__(						\
-		".set\tpush\n\t"					\
-		".set\tnoat\n\t"					\
-		"move\t$1, %z0\n\t"					\
-		"# dmtgc0\t$1, $%1, %2\n\t"				\
-		_ASM_INSN_IF_MIPS(0x40610300 | %1 << 11 | %2)		\
-		_ASM_INSN32_IF_MM(0x582006fc | %1 << 16 | %2 << 11)	\
-		".set\tpop"						\
-		: : "Jr" (value),					\
-		    "i" (register), "i" (sel));				\
-} while (0)
-
-#endif	/* !TOOLCHAIN_SUPPORTS_VIRT */
-
 #define __read_ulong_gc0_register(reg, sel)				\
 	((sizeof(unsigned long) == 4) ?					\
 	(unsigned long) __read_32bit_gc0_register(reg, sel) :		\
@@ -1957,207 +2027,207 @@ do {									\
 		__write_64bit_gc0_register(reg, sel, val);		\
 } while (0)
 
-#define read_gc0_index()		__read_32bit_gc0_register(0, 0)
-#define write_gc0_index(val)		__write_32bit_gc0_register(0, 0, val)
+#define read_gc0_index()		__read_32bit_gc0_register($0, 0)
+#define write_gc0_index(val)		__write_32bit_gc0_register($0, 0, val)
 
-#define read_gc0_entrylo0()		__read_ulong_gc0_register(2, 0)
-#define write_gc0_entrylo0(val)		__write_ulong_gc0_register(2, 0, val)
+#define read_gc0_entrylo0()		__read_ulong_gc0_register($2, 0)
+#define write_gc0_entrylo0(val)		__write_ulong_gc0_register($2, 0, val)
 
-#define read_gc0_entrylo1()		__read_ulong_gc0_register(3, 0)
-#define write_gc0_entrylo1(val)		__write_ulong_gc0_register(3, 0, val)
+#define read_gc0_entrylo1()		__read_ulong_gc0_register($3, 0)
+#define write_gc0_entrylo1(val)		__write_ulong_gc0_register($3, 0, val)
 
-#define read_gc0_context()		__read_ulong_gc0_register(4, 0)
-#define write_gc0_context(val)		__write_ulong_gc0_register(4, 0, val)
+#define read_gc0_context()		__read_ulong_gc0_register($4, 0)
+#define write_gc0_context(val)		__write_ulong_gc0_register($4, 0, val)
 
-#define read_gc0_contextconfig()	__read_32bit_gc0_register(4, 1)
-#define write_gc0_contextconfig(val)	__write_32bit_gc0_register(4, 1, val)
+#define read_gc0_contextconfig()	__read_32bit_gc0_register($4, 1)
+#define write_gc0_contextconfig(val)	__write_32bit_gc0_register($4, 1, val)
 
-#define read_gc0_userlocal()		__read_ulong_gc0_register(4, 2)
-#define write_gc0_userlocal(val)	__write_ulong_gc0_register(4, 2, val)
+#define read_gc0_userlocal()		__read_ulong_gc0_register($4, 2)
+#define write_gc0_userlocal(val)	__write_ulong_gc0_register($4, 2, val)
 
-#define read_gc0_xcontextconfig()	__read_ulong_gc0_register(4, 3)
-#define write_gc0_xcontextconfig(val)	__write_ulong_gc0_register(4, 3, val)
+#define read_gc0_xcontextconfig()	__read_ulong_gc0_register($4, 3)
+#define write_gc0_xcontextconfig(val)	__write_ulong_gc0_register($4, 3, val)
 
-#define read_gc0_pagemask()		__read_32bit_gc0_register(5, 0)
-#define write_gc0_pagemask(val)		__write_32bit_gc0_register(5, 0, val)
+#define read_gc0_pagemask()		__read_32bit_gc0_register($5, 0)
+#define write_gc0_pagemask(val)		__write_32bit_gc0_register($5, 0, val)
 
-#define read_gc0_pagegrain()		__read_32bit_gc0_register(5, 1)
-#define write_gc0_pagegrain(val)	__write_32bit_gc0_register(5, 1, val)
+#define read_gc0_pagegrain()		__read_32bit_gc0_register($5, 1)
+#define write_gc0_pagegrain(val)	__write_32bit_gc0_register($5, 1, val)
 
-#define read_gc0_segctl0()		__read_ulong_gc0_register(5, 2)
-#define write_gc0_segctl0(val)		__write_ulong_gc0_register(5, 2, val)
+#define read_gc0_segctl0()		__read_ulong_gc0_register($5, 2)
+#define write_gc0_segctl0(val)		__write_ulong_gc0_register($5, 2, val)
 
-#define read_gc0_segctl1()		__read_ulong_gc0_register(5, 3)
-#define write_gc0_segctl1(val)		__write_ulong_gc0_register(5, 3, val)
+#define read_gc0_segctl1()		__read_ulong_gc0_register($5, 3)
+#define write_gc0_segctl1(val)		__write_ulong_gc0_register($5, 3, val)
 
-#define read_gc0_segctl2()		__read_ulong_gc0_register(5, 4)
-#define write_gc0_segctl2(val)		__write_ulong_gc0_register(5, 4, val)
+#define read_gc0_segctl2()		__read_ulong_gc0_register($5, 4)
+#define write_gc0_segctl2(val)		__write_ulong_gc0_register($5, 4, val)
 
-#define read_gc0_pwbase()		__read_ulong_gc0_register(5, 5)
-#define write_gc0_pwbase(val)		__write_ulong_gc0_register(5, 5, val)
+#define read_gc0_pwbase()		__read_ulong_gc0_register($5, 5)
+#define write_gc0_pwbase(val)		__write_ulong_gc0_register($5, 5, val)
 
-#define read_gc0_pwfield()		__read_ulong_gc0_register(5, 6)
-#define write_gc0_pwfield(val)		__write_ulong_gc0_register(5, 6, val)
+#define read_gc0_pwfield()		__read_ulong_gc0_register($5, 6)
+#define write_gc0_pwfield(val)		__write_ulong_gc0_register($5, 6, val)
 
-#define read_gc0_pwsize()		__read_ulong_gc0_register(5, 7)
-#define write_gc0_pwsize(val)		__write_ulong_gc0_register(5, 7, val)
+#define read_gc0_pwsize()		__read_ulong_gc0_register($5, 7)
+#define write_gc0_pwsize(val)		__write_ulong_gc0_register($5, 7, val)
 
-#define read_gc0_wired()		__read_32bit_gc0_register(6, 0)
-#define write_gc0_wired(val)		__write_32bit_gc0_register(6, 0, val)
+#define read_gc0_wired()		__read_32bit_gc0_register($6, 0)
+#define write_gc0_wired(val)		__write_32bit_gc0_register($6, 0, val)
 
-#define read_gc0_pwctl()		__read_32bit_gc0_register(6, 6)
-#define write_gc0_pwctl(val)		__write_32bit_gc0_register(6, 6, val)
-
-#define read_gc0_hwrena()		__read_32bit_gc0_register(7, 0)
-#define write_gc0_hwrena(val)		__write_32bit_gc0_register(7, 0, val)
-
-#define read_gc0_badvaddr()		__read_ulong_gc0_register(8, 0)
-#define write_gc0_badvaddr(val)		__write_ulong_gc0_register(8, 0, val)
-
-#define read_gc0_badinstr()		__read_32bit_gc0_register(8, 1)
-#define write_gc0_badinstr(val)		__write_32bit_gc0_register(8, 1, val)
-
-#define read_gc0_badinstrp()		__read_32bit_gc0_register(8, 2)
-#define write_gc0_badinstrp(val)	__write_32bit_gc0_register(8, 2, val)
-
-#define read_gc0_count()		__read_32bit_gc0_register(9, 0)
-
-#define read_gc0_entryhi()		__read_ulong_gc0_register(10, 0)
-#define write_gc0_entryhi(val)		__write_ulong_gc0_register(10, 0, val)
-
-#define read_gc0_compare()		__read_32bit_gc0_register(11, 0)
-#define write_gc0_compare(val)		__write_32bit_gc0_register(11, 0, val)
-
-#define read_gc0_status()		__read_32bit_gc0_register(12, 0)
-#define write_gc0_status(val)		__write_32bit_gc0_register(12, 0, val)
-
-#define read_gc0_intctl()		__read_32bit_gc0_register(12, 1)
-#define write_gc0_intctl(val)		__write_32bit_gc0_register(12, 1, val)
-
-#define read_gc0_cause()		__read_32bit_gc0_register(13, 0)
-#define write_gc0_cause(val)		__write_32bit_gc0_register(13, 0, val)
-
-#define read_gc0_epc()			__read_ulong_gc0_register(14, 0)
-#define write_gc0_epc(val)		__write_ulong_gc0_register(14, 0, val)
-
-#define read_gc0_prid()			__read_32bit_gc0_register(15, 0)
-
-#define read_gc0_ebase()		__read_32bit_gc0_register(15, 1)
-#define write_gc0_ebase(val)		__write_32bit_gc0_register(15, 1, val)
-
-#define read_gc0_ebase_64()		__read_64bit_gc0_register(15, 1)
-#define write_gc0_ebase_64(val)		__write_64bit_gc0_register(15, 1, val)
-
-#define read_gc0_config()		__read_32bit_gc0_register(16, 0)
-#define read_gc0_config1()		__read_32bit_gc0_register(16, 1)
-#define read_gc0_config2()		__read_32bit_gc0_register(16, 2)
-#define read_gc0_config3()		__read_32bit_gc0_register(16, 3)
-#define read_gc0_config4()		__read_32bit_gc0_register(16, 4)
-#define read_gc0_config5()		__read_32bit_gc0_register(16, 5)
-#define read_gc0_config6()		__read_32bit_gc0_register(16, 6)
-#define read_gc0_config7()		__read_32bit_gc0_register(16, 7)
-#define write_gc0_config(val)		__write_32bit_gc0_register(16, 0, val)
-#define write_gc0_config1(val)		__write_32bit_gc0_register(16, 1, val)
-#define write_gc0_config2(val)		__write_32bit_gc0_register(16, 2, val)
-#define write_gc0_config3(val)		__write_32bit_gc0_register(16, 3, val)
-#define write_gc0_config4(val)		__write_32bit_gc0_register(16, 4, val)
-#define write_gc0_config5(val)		__write_32bit_gc0_register(16, 5, val)
-#define write_gc0_config6(val)		__write_32bit_gc0_register(16, 6, val)
-#define write_gc0_config7(val)		__write_32bit_gc0_register(16, 7, val)
-
-#define read_gc0_lladdr()		__read_ulong_gc0_register(17, 0)
-#define write_gc0_lladdr(val)		__write_ulong_gc0_register(17, 0, val)
-
-#define read_gc0_watchlo0()		__read_ulong_gc0_register(18, 0)
-#define read_gc0_watchlo1()		__read_ulong_gc0_register(18, 1)
-#define read_gc0_watchlo2()		__read_ulong_gc0_register(18, 2)
-#define read_gc0_watchlo3()		__read_ulong_gc0_register(18, 3)
-#define read_gc0_watchlo4()		__read_ulong_gc0_register(18, 4)
-#define read_gc0_watchlo5()		__read_ulong_gc0_register(18, 5)
-#define read_gc0_watchlo6()		__read_ulong_gc0_register(18, 6)
-#define read_gc0_watchlo7()		__read_ulong_gc0_register(18, 7)
-#define write_gc0_watchlo0(val)		__write_ulong_gc0_register(18, 0, val)
-#define write_gc0_watchlo1(val)		__write_ulong_gc0_register(18, 1, val)
-#define write_gc0_watchlo2(val)		__write_ulong_gc0_register(18, 2, val)
-#define write_gc0_watchlo3(val)		__write_ulong_gc0_register(18, 3, val)
-#define write_gc0_watchlo4(val)		__write_ulong_gc0_register(18, 4, val)
-#define write_gc0_watchlo5(val)		__write_ulong_gc0_register(18, 5, val)
-#define write_gc0_watchlo6(val)		__write_ulong_gc0_register(18, 6, val)
-#define write_gc0_watchlo7(val)		__write_ulong_gc0_register(18, 7, val)
-
-#define read_gc0_watchhi0()		__read_32bit_gc0_register(19, 0)
-#define read_gc0_watchhi1()		__read_32bit_gc0_register(19, 1)
-#define read_gc0_watchhi2()		__read_32bit_gc0_register(19, 2)
-#define read_gc0_watchhi3()		__read_32bit_gc0_register(19, 3)
-#define read_gc0_watchhi4()		__read_32bit_gc0_register(19, 4)
-#define read_gc0_watchhi5()		__read_32bit_gc0_register(19, 5)
-#define read_gc0_watchhi6()		__read_32bit_gc0_register(19, 6)
-#define read_gc0_watchhi7()		__read_32bit_gc0_register(19, 7)
-#define write_gc0_watchhi0(val)		__write_32bit_gc0_register(19, 0, val)
-#define write_gc0_watchhi1(val)		__write_32bit_gc0_register(19, 1, val)
-#define write_gc0_watchhi2(val)		__write_32bit_gc0_register(19, 2, val)
-#define write_gc0_watchhi3(val)		__write_32bit_gc0_register(19, 3, val)
-#define write_gc0_watchhi4(val)		__write_32bit_gc0_register(19, 4, val)
-#define write_gc0_watchhi5(val)		__write_32bit_gc0_register(19, 5, val)
-#define write_gc0_watchhi6(val)		__write_32bit_gc0_register(19, 6, val)
-#define write_gc0_watchhi7(val)		__write_32bit_gc0_register(19, 7, val)
-
-#define read_gc0_xcontext()		__read_ulong_gc0_register(20, 0)
-#define write_gc0_xcontext(val)		__write_ulong_gc0_register(20, 0, val)
-
-#define read_gc0_perfctrl0()		__read_32bit_gc0_register(25, 0)
-#define write_gc0_perfctrl0(val)	__write_32bit_gc0_register(25, 0, val)
-#define read_gc0_perfcntr0()		__read_32bit_gc0_register(25, 1)
-#define write_gc0_perfcntr0(val)	__write_32bit_gc0_register(25, 1, val)
-#define read_gc0_perfcntr0_64()		__read_64bit_gc0_register(25, 1)
-#define write_gc0_perfcntr0_64(val)	__write_64bit_gc0_register(25, 1, val)
-#define read_gc0_perfctrl1()		__read_32bit_gc0_register(25, 2)
-#define write_gc0_perfctrl1(val)	__write_32bit_gc0_register(25, 2, val)
-#define read_gc0_perfcntr1()		__read_32bit_gc0_register(25, 3)
-#define write_gc0_perfcntr1(val)	__write_32bit_gc0_register(25, 3, val)
-#define read_gc0_perfcntr1_64()		__read_64bit_gc0_register(25, 3)
-#define write_gc0_perfcntr1_64(val)	__write_64bit_gc0_register(25, 3, val)
-#define read_gc0_perfctrl2()		__read_32bit_gc0_register(25, 4)
-#define write_gc0_perfctrl2(val)	__write_32bit_gc0_register(25, 4, val)
-#define read_gc0_perfcntr2()		__read_32bit_gc0_register(25, 5)
-#define write_gc0_perfcntr2(val)	__write_32bit_gc0_register(25, 5, val)
-#define read_gc0_perfcntr2_64()		__read_64bit_gc0_register(25, 5)
-#define write_gc0_perfcntr2_64(val)	__write_64bit_gc0_register(25, 5, val)
-#define read_gc0_perfctrl3()		__read_32bit_gc0_register(25, 6)
-#define write_gc0_perfctrl3(val)	__write_32bit_gc0_register(25, 6, val)
-#define read_gc0_perfcntr3()		__read_32bit_gc0_register(25, 7)
-#define write_gc0_perfcntr3(val)	__write_32bit_gc0_register(25, 7, val)
-#define read_gc0_perfcntr3_64()		__read_64bit_gc0_register(25, 7)
-#define write_gc0_perfcntr3_64(val)	__write_64bit_gc0_register(25, 7, val)
-
-#define read_gc0_errorepc()		__read_ulong_gc0_register(30, 0)
-#define write_gc0_errorepc(val)		__write_ulong_gc0_register(30, 0, val)
-
-#define read_gc0_kscratch1()		__read_ulong_gc0_register(31, 2)
-#define read_gc0_kscratch2()		__read_ulong_gc0_register(31, 3)
-#define read_gc0_kscratch3()		__read_ulong_gc0_register(31, 4)
-#define read_gc0_kscratch4()		__read_ulong_gc0_register(31, 5)
-#define read_gc0_kscratch5()		__read_ulong_gc0_register(31, 6)
-#define read_gc0_kscratch6()		__read_ulong_gc0_register(31, 7)
-#define write_gc0_kscratch1(val)	__write_ulong_gc0_register(31, 2, val)
-#define write_gc0_kscratch2(val)	__write_ulong_gc0_register(31, 3, val)
-#define write_gc0_kscratch3(val)	__write_ulong_gc0_register(31, 4, val)
-#define write_gc0_kscratch4(val)	__write_ulong_gc0_register(31, 5, val)
-#define write_gc0_kscratch5(val)	__write_ulong_gc0_register(31, 6, val)
-#define write_gc0_kscratch6(val)	__write_ulong_gc0_register(31, 7, val)
+#define read_gc0_pwctl()		__read_32bit_gc0_register($6, 6)
+#define write_gc0_pwctl(val)		__write_32bit_gc0_register($6, 6, val)
+
+#define read_gc0_hwrena()		__read_32bit_gc0_register($7, 0)
+#define write_gc0_hwrena(val)		__write_32bit_gc0_register($7, 0, val)
+
+#define read_gc0_badvaddr()		__read_ulong_gc0_register($8, 0)
+#define write_gc0_badvaddr(val)		__write_ulong_gc0_register($8, 0, val)
+
+#define read_gc0_badinstr()		__read_32bit_gc0_register($8, 1)
+#define write_gc0_badinstr(val)		__write_32bit_gc0_register($8, 1, val)
+
+#define read_gc0_badinstrp()		__read_32bit_gc0_register($8, 2)
+#define write_gc0_badinstrp(val)	__write_32bit_gc0_register($8, 2, val)
+
+#define read_gc0_count()		__read_32bit_gc0_register($9, 0)
+
+#define read_gc0_entryhi()		__read_ulong_gc0_register($10, 0)
+#define write_gc0_entryhi(val)		__write_ulong_gc0_register($10, 0, val)
+
+#define read_gc0_compare()		__read_32bit_gc0_register($11, 0)
+#define write_gc0_compare(val)		__write_32bit_gc0_register($11, 0, val)
+
+#define read_gc0_status()		__read_32bit_gc0_register($12, 0)
+#define write_gc0_status(val)		__write_32bit_gc0_register($12, 0, val)
+
+#define read_gc0_intctl()		__read_32bit_gc0_register($12, 1)
+#define write_gc0_intctl(val)		__write_32bit_gc0_register($12, 1, val)
+
+#define read_gc0_cause()		__read_32bit_gc0_register($13, 0)
+#define write_gc0_cause(val)		__write_32bit_gc0_register($13, 0, val)
+
+#define read_gc0_epc()			__read_ulong_gc0_register($14, 0)
+#define write_gc0_epc(val)		__write_ulong_gc0_register($14, 0, val)
+
+#define read_gc0_prid()			__read_32bit_gc0_register($15, 0)
+
+#define read_gc0_ebase()		__read_32bit_gc0_register($15, 1)
+#define write_gc0_ebase(val)		__write_32bit_gc0_register($15, 1, val)
+
+#define read_gc0_ebase_64()		__read_64bit_gc0_register($15, 1)
+#define write_gc0_ebase_64(val)		__write_64bit_gc0_register($15, 1, val)
+
+#define read_gc0_config()		__read_32bit_gc0_register($16, 0)
+#define read_gc0_config1()		__read_32bit_gc0_register($16, 1)
+#define read_gc0_config2()		__read_32bit_gc0_register($16, 2)
+#define read_gc0_config3()		__read_32bit_gc0_register($16, 3)
+#define read_gc0_config4()		__read_32bit_gc0_register($16, 4)
+#define read_gc0_config5()		__read_32bit_gc0_register($16, 5)
+#define read_gc0_config6()		__read_32bit_gc0_register($16, 6)
+#define read_gc0_config7()		__read_32bit_gc0_register($16, 7)
+#define write_gc0_config(val)		__write_32bit_gc0_register($16, 0, val)
+#define write_gc0_config1(val)		__write_32bit_gc0_register($16, 1, val)
+#define write_gc0_config2(val)		__write_32bit_gc0_register($16, 2, val)
+#define write_gc0_config3(val)		__write_32bit_gc0_register($16, 3, val)
+#define write_gc0_config4(val)		__write_32bit_gc0_register($16, 4, val)
+#define write_gc0_config5(val)		__write_32bit_gc0_register($16, 5, val)
+#define write_gc0_config6(val)		__write_32bit_gc0_register($16, 6, val)
+#define write_gc0_config7(val)		__write_32bit_gc0_register($16, 7, val)
+
+#define read_gc0_lladdr()		__read_ulong_gc0_register($17, 0)
+#define write_gc0_lladdr(val)		__write_ulong_gc0_register($17, 0, val)
+
+#define read_gc0_watchlo0()		__read_ulong_gc0_register($18, 0)
+#define read_gc0_watchlo1()		__read_ulong_gc0_register($18, 1)
+#define read_gc0_watchlo2()		__read_ulong_gc0_register($18, 2)
+#define read_gc0_watchlo3()		__read_ulong_gc0_register($18, 3)
+#define read_gc0_watchlo4()		__read_ulong_gc0_register($18, 4)
+#define read_gc0_watchlo5()		__read_ulong_gc0_register($18, 5)
+#define read_gc0_watchlo6()		__read_ulong_gc0_register($18, 6)
+#define read_gc0_watchlo7()		__read_ulong_gc0_register($18, 7)
+#define write_gc0_watchlo0(val)		__write_ulong_gc0_register($18, 0, val)
+#define write_gc0_watchlo1(val)		__write_ulong_gc0_register($18, 1, val)
+#define write_gc0_watchlo2(val)		__write_ulong_gc0_register($18, 2, val)
+#define write_gc0_watchlo3(val)		__write_ulong_gc0_register($18, 3, val)
+#define write_gc0_watchlo4(val)		__write_ulong_gc0_register($18, 4, val)
+#define write_gc0_watchlo5(val)		__write_ulong_gc0_register($18, 5, val)
+#define write_gc0_watchlo6(val)		__write_ulong_gc0_register($18, 6, val)
+#define write_gc0_watchlo7(val)		__write_ulong_gc0_register($18, 7, val)
+
+#define read_gc0_watchhi0()		__read_32bit_gc0_register($19, 0)
+#define read_gc0_watchhi1()		__read_32bit_gc0_register($19, 1)
+#define read_gc0_watchhi2()		__read_32bit_gc0_register($19, 2)
+#define read_gc0_watchhi3()		__read_32bit_gc0_register($19, 3)
+#define read_gc0_watchhi4()		__read_32bit_gc0_register($19, 4)
+#define read_gc0_watchhi5()		__read_32bit_gc0_register($19, 5)
+#define read_gc0_watchhi6()		__read_32bit_gc0_register($19, 6)
+#define read_gc0_watchhi7()		__read_32bit_gc0_register($19, 7)
+#define write_gc0_watchhi0(val)		__write_32bit_gc0_register($19, 0, val)
+#define write_gc0_watchhi1(val)		__write_32bit_gc0_register($19, 1, val)
+#define write_gc0_watchhi2(val)		__write_32bit_gc0_register($19, 2, val)
+#define write_gc0_watchhi3(val)		__write_32bit_gc0_register($19, 3, val)
+#define write_gc0_watchhi4(val)		__write_32bit_gc0_register($19, 4, val)
+#define write_gc0_watchhi5(val)		__write_32bit_gc0_register($19, 5, val)
+#define write_gc0_watchhi6(val)		__write_32bit_gc0_register($19, 6, val)
+#define write_gc0_watchhi7(val)		__write_32bit_gc0_register($19, 7, val)
+
+#define read_gc0_xcontext()		__read_ulong_gc0_register($20, 0)
+#define write_gc0_xcontext(val)		__write_ulong_gc0_register($20, 0, val)
+
+#define read_gc0_perfctrl0()		__read_32bit_gc0_register($25, 0)
+#define write_gc0_perfctrl0(val)	__write_32bit_gc0_register($25, 0, val)
+#define read_gc0_perfcntr0()		__read_32bit_gc0_register($25, 1)
+#define write_gc0_perfcntr0(val)	__write_32bit_gc0_register($25, 1, val)
+#define read_gc0_perfcntr0_64()		__read_64bit_gc0_register($25, 1)
+#define write_gc0_perfcntr0_64(val)	__write_64bit_gc0_register($25, 1, val)
+#define read_gc0_perfctrl1()		__read_32bit_gc0_register($25, 2)
+#define write_gc0_perfctrl1(val)	__write_32bit_gc0_register($25, 2, val)
+#define read_gc0_perfcntr1()		__read_32bit_gc0_register($25, 3)
+#define write_gc0_perfcntr1(val)	__write_32bit_gc0_register($25, 3, val)
+#define read_gc0_perfcntr1_64()		__read_64bit_gc0_register($25, 3)
+#define write_gc0_perfcntr1_64(val)	__write_64bit_gc0_register($25, 3, val)
+#define read_gc0_perfctrl2()		__read_32bit_gc0_register($25, 4)
+#define write_gc0_perfctrl2(val)	__write_32bit_gc0_register($25, 4, val)
+#define read_gc0_perfcntr2()		__read_32bit_gc0_register($25, 5)
+#define write_gc0_perfcntr2(val)	__write_32bit_gc0_register($25, 5, val)
+#define read_gc0_perfcntr2_64()		__read_64bit_gc0_register($25, 5)
+#define write_gc0_perfcntr2_64(val)	__write_64bit_gc0_register($25, 5, val)
+#define read_gc0_perfctrl3()		__read_32bit_gc0_register($25, 6)
+#define write_gc0_perfctrl3(val)	__write_32bit_gc0_register($25, 6, val)
+#define read_gc0_perfcntr3()		__read_32bit_gc0_register($25, 7)
+#define write_gc0_perfcntr3(val)	__write_32bit_gc0_register($25, 7, val)
+#define read_gc0_perfcntr3_64()		__read_64bit_gc0_register($25, 7)
+#define write_gc0_perfcntr3_64(val)	__write_64bit_gc0_register($25, 7, val)
+
+#define read_gc0_errorepc()		__read_ulong_gc0_register($30, 0)
+#define write_gc0_errorepc(val)		__write_ulong_gc0_register($30, 0, val)
+
+#define read_gc0_kscratch1()		__read_ulong_gc0_register($31, 2)
+#define read_gc0_kscratch2()		__read_ulong_gc0_register($31, 3)
+#define read_gc0_kscratch3()		__read_ulong_gc0_register($31, 4)
+#define read_gc0_kscratch4()		__read_ulong_gc0_register($31, 5)
+#define read_gc0_kscratch5()		__read_ulong_gc0_register($31, 6)
+#define read_gc0_kscratch6()		__read_ulong_gc0_register($31, 7)
+#define write_gc0_kscratch1(val)	__write_ulong_gc0_register($31, 2, val)
+#define write_gc0_kscratch2(val)	__write_ulong_gc0_register($31, 3, val)
+#define write_gc0_kscratch3(val)	__write_ulong_gc0_register($31, 4, val)
+#define write_gc0_kscratch4(val)	__write_ulong_gc0_register($31, 5, val)
+#define write_gc0_kscratch5(val)	__write_ulong_gc0_register($31, 6, val)
+#define write_gc0_kscratch6(val)	__write_ulong_gc0_register($31, 7, val)
 
 /* Cavium OCTEON (cnMIPS) */
-#define read_gc0_cvmcount()		__read_ulong_gc0_register(9, 6)
-#define write_gc0_cvmcount(val)		__write_ulong_gc0_register(9, 6, val)
+#define read_gc0_cvmcount()		__read_ulong_gc0_register($9, 6)
+#define write_gc0_cvmcount(val)		__write_ulong_gc0_register($9, 6, val)
 
-#define read_gc0_cvmctl()		__read_64bit_gc0_register(9, 7)
-#define write_gc0_cvmctl(val)		__write_64bit_gc0_register(9, 7, val)
+#define read_gc0_cvmctl()		__read_64bit_gc0_register($9, 7)
+#define write_gc0_cvmctl(val)		__write_64bit_gc0_register($9, 7, val)
 
-#define read_gc0_cvmmemctl()		__read_64bit_gc0_register(11, 7)
-#define write_gc0_cvmmemctl(val)	__write_64bit_gc0_register(11, 7, val)
+#define read_gc0_cvmmemctl()		__read_64bit_gc0_register($11, 7)
+#define write_gc0_cvmmemctl(val)	__write_64bit_gc0_register($11, 7, val)
 
-#define read_gc0_cvmmemctl2()		__read_64bit_gc0_register(16, 6)
-#define write_gc0_cvmmemctl2(val)	__write_64bit_gc0_register(16, 6, val)
+#define read_gc0_cvmmemctl2()		__read_64bit_gc0_register($16, 6)
+#define write_gc0_cvmmemctl2(val)	__write_64bit_gc0_register($16, 6, val)
 
 /*
  * Macros to access the floating point coprocessor control registers
@@ -2581,8 +2651,6 @@ static inline void tlb_write_random(void)
 		".set reorder");
 }
 
-#ifdef TOOLCHAIN_SUPPORTS_VIRT
-
 /*
  * Guest TLB operations.
  *
@@ -2593,7 +2661,7 @@ static inline void guest_tlb_probe(void)
 	__asm__ __volatile__(
 		".set push\n\t"
 		".set noreorder\n\t"
-		".set virt\n\t"
+		_ASM_SET_VIRT
 		"tlbgp\n\t"
 		".set pop");
 }
@@ -2603,7 +2671,7 @@ static inline void guest_tlb_read(void)
 	__asm__ __volatile__(
 		".set push\n\t"
 		".set noreorder\n\t"
-		".set virt\n\t"
+		_ASM_SET_VIRT
 		"tlbgr\n\t"
 		".set pop");
 }
@@ -2613,7 +2681,7 @@ static inline void guest_tlb_write_indexed(void)
 	__asm__ __volatile__(
 		".set push\n\t"
 		".set noreorder\n\t"
-		".set virt\n\t"
+		_ASM_SET_VIRT
 		"tlbgwi\n\t"
 		".set pop");
 }
@@ -2623,7 +2691,7 @@ static inline void guest_tlb_write_random(void)
 	__asm__ __volatile__(
 		".set push\n\t"
 		".set noreorder\n\t"
-		".set virt\n\t"
+		_ASM_SET_VIRT
 		"tlbgwr\n\t"
 		".set pop");
 }
@@ -2636,63 +2704,11 @@ static inline void guest_tlbinvf(void)
 	__asm__ __volatile__(
 		".set push\n\t"
 		".set noreorder\n\t"
-		".set virt\n\t"
+		_ASM_SET_VIRT
 		"tlbginvf\n\t"
 		".set pop");
 }
 
-#else	/* TOOLCHAIN_SUPPORTS_VIRT */
-
-/*
- * Guest TLB operations.
- *
- * It is responsibility of the caller to take care of any TLB hazards.
- */
-static inline void guest_tlb_probe(void)
-{
-	__asm__ __volatile__(
-		"# tlbgp\n\t"
-		_ASM_INSN_IF_MIPS(0x42000010)
-		_ASM_INSN32_IF_MM(0x0000017c));
-}
-
-static inline void guest_tlb_read(void)
-{
-	__asm__ __volatile__(
-		"# tlbgr\n\t"
-		_ASM_INSN_IF_MIPS(0x42000009)
-		_ASM_INSN32_IF_MM(0x0000117c));
-}
-
-static inline void guest_tlb_write_indexed(void)
-{
-	__asm__ __volatile__(
-		"# tlbgwi\n\t"
-		_ASM_INSN_IF_MIPS(0x4200000a)
-		_ASM_INSN32_IF_MM(0x0000217c));
-}
-
-static inline void guest_tlb_write_random(void)
-{
-	__asm__ __volatile__(
-		"# tlbgwr\n\t"
-		_ASM_INSN_IF_MIPS(0x4200000e)
-		_ASM_INSN32_IF_MM(0x0000317c));
-}
-
-/*
- * Guest TLB Invalidate Flush
- */
-static inline void guest_tlbinvf(void)
-{
-	__asm__ __volatile__(
-		"# tlbginvf\n\t"
-		_ASM_INSN_IF_MIPS(0x4200000c)
-		_ASM_INSN32_IF_MM(0x0000517c));
-}
-
-#endif	/* !TOOLCHAIN_SUPPORTS_VIRT */
-
 /*
  * Manipulate bits in a register.
  */
diff --git a/arch/mips/include/asm/msa.h b/arch/mips/include/asm/msa.h
index b1845102f8f9..b4f9577ed96a 100644
--- a/arch/mips/include/asm/msa.h
+++ b/arch/mips/include/asm/msa.h
@@ -160,7 +160,23 @@ static inline void init_msa_upper(void)
 	_init_msa_upper();
 }
 
-#ifdef TOOLCHAIN_SUPPORTS_MSA
+#ifndef TOOLCHAIN_SUPPORTS_MSA
+/*
+ * Define assembler macros using .word for the c[ft]cmsa instructions in order
+ * to allow compilation with toolchains that do not support MSA. Once all
+ * toolchains in use support MSA these can be removed.
+ */
+_ASM_MACRO_2R(cfcmsa, rd, cs,
+	_ASM_INSN_IF_MIPS(0x787e0019 | __cs << 11 | __rd << 6)
+	_ASM_INSN32_IF_MM(0x587e0016 | __cs << 11 | __rd << 6));
+_ASM_MACRO_2R(ctcmsa, cd, rs,
+	_ASM_INSN_IF_MIPS(0x783e0019 | __rs << 11 | __cd << 6)
+	_ASM_INSN32_IF_MM(0x583e0016 | __rs << 11 | __cd << 6));
+#define _ASM_SET_MSA ""
+#else /* TOOLCHAIN_SUPPORTS_MSA */
+#define _ASM_SET_MSA ".set\tfp=64\n\t"				\
+		     ".set\tmsa\n\t"
+#endif
 
 #define __BUILD_MSA_CTL_REG(name, cs)				\
 static inline unsigned int read_msa_##name(void)		\
@@ -168,8 +184,7 @@ static inline unsigned int read_msa_##name(void)		\
 	unsigned int reg;					\
 	__asm__ __volatile__(					\
 	"	.set	push\n"					\
-	"	.set	fp=64\n"				\
-	"	.set	msa\n"					\
+	_ASM_SET_MSA						\
 	"	cfcmsa	%0, $" #cs "\n"				\
 	"	.set	pop\n"					\
 	: "=r"(reg));						\
@@ -180,52 +195,12 @@ static inline void write_msa_##name(unsigned int val)		\
 {								\
 	__asm__ __volatile__(					\
 	"	.set	push\n"					\
-	"	.set	fp=64\n"				\
-	"	.set	msa\n"					\
+	_ASM_SET_MSA						\
 	"	ctcmsa	$" #cs ", %0\n"				\
 	"	.set	pop\n"					\
 	: : "r"(val));						\
 }
 
-#else /* !TOOLCHAIN_SUPPORTS_MSA */
-
-/*
- * Define functions using .word for the c[ft]cmsa instructions in order to
- * allow compilation with toolchains that do not support MSA. Once all
- * toolchains in use support MSA these can be removed.
- */
-
-#define __BUILD_MSA_CTL_REG(name, cs)				\
-static inline unsigned int read_msa_##name(void)		\
-{								\
-	unsigned int reg;					\
-	__asm__ __volatile__(					\
-	"	.set	push\n"					\
-	"	.set	noat\n"					\
-	"	# cfcmsa $1, $%1\n"				\
-	_ASM_INSN_IF_MIPS(0x787e0059 | %1 << 11)		\
-	_ASM_INSN32_IF_MM(0x587e0056 | %1 << 11)		\
-	"	move	%0, $1\n"				\
-	"	.set	pop\n"					\
-	: "=r"(reg) : "i"(cs));					\
-	return reg;						\
-}								\
-								\
-static inline void write_msa_##name(unsigned int val)		\
-{								\
-	__asm__ __volatile__(					\
-	"	.set	push\n"					\
-	"	.set	noat\n"					\
-	"	move	$1, %0\n"				\
-	"	# ctcmsa $%1, $1\n"				\
-	_ASM_INSN_IF_MIPS(0x783e0819 | %1 << 6)			\
-	_ASM_INSN32_IF_MM(0x583e0816 | %1 << 6)			\
-	"	.set	pop\n"					\
-	: : "r"(val), "i"(cs));					\
-}
-
-#endif /* !TOOLCHAIN_SUPPORTS_MSA */
-
 __BUILD_MSA_CTL_REG(ir, 0)
 __BUILD_MSA_CTL_REG(csr, 1)
 __BUILD_MSA_CTL_REG(access, 2)
diff --git a/arch/mips/include/asm/netlogic/common.h b/arch/mips/include/asm/netlogic/common.h
index a6e6cbebe046..57616649b4f3 100644
--- a/arch/mips/include/asm/netlogic/common.h
+++ b/arch/mips/include/asm/netlogic/common.h
@@ -87,9 +87,6 @@ unsigned int nlm_get_cpu_frequency(void);
 extern const struct plat_smp_ops nlm_smp_ops;
 extern char nlm_reset_entry[], nlm_reset_entry_end[];
 
-/* SWIOTLB */
-extern const struct dma_map_ops nlm_swiotlb_dma_ops;
-
 extern unsigned int nlm_threads_per_core;
 extern cpumask_t nlm_cpumask;
 
diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
index 1a508a74d48d..129e0328367f 100644
--- a/arch/mips/include/asm/pgtable.h
+++ b/arch/mips/include/asm/pgtable.h
@@ -534,6 +534,9 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma,
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 
+/* We don't have hardware dirty/accessed bits, generic_pmdp_establish is fine.*/
+#define pmdp_establish generic_pmdp_establish
+
 #define has_transparent_hugepage has_transparent_hugepage
 extern int has_transparent_hugepage(void);
 
diff --git a/arch/mips/include/asm/thread_info.h b/arch/mips/include/asm/thread_info.h
index 5e8927f99a76..4993db40482c 100644
--- a/arch/mips/include/asm/thread_info.h
+++ b/arch/mips/include/asm/thread_info.h
@@ -49,9 +49,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* How to get the thread information struct from C.  */
 register struct thread_info *__current_thread_info __asm__("$28");
 
diff --git a/arch/mips/include/uapi/asm/siginfo.h b/arch/mips/include/uapi/asm/siginfo.h
index f17d8163dec6..262504bd59a5 100644
--- a/arch/mips/include/uapi/asm/siginfo.h
+++ b/arch/mips/include/uapi/asm/siginfo.h
@@ -14,8 +14,6 @@
 #define __ARCH_SIGEV_PREAMBLE_SIZE (sizeof(long) + 2*sizeof(int))
 #undef __ARCH_SI_TRAPNO /* exception code needs to fill this ...  */
 
-#define HAVE_ARCH_SIGINFO_T
-
 /*
  * Careful to keep union _sifields from shifting ...
  */
@@ -27,92 +25,10 @@
 #error _MIPS_SZLONG neither 32 nor 64
 #endif
 
-#define __ARCH_SIGSYS
+#define __ARCH_HAS_SWAPPED_SIGINFO
 
 #include <asm-generic/siginfo.h>
 
-/* We can't use generic siginfo_t, because our si_code and si_errno are swapped */
-typedef struct siginfo {
-	int si_signo;
-	int si_code;
-	int si_errno;
-	int __pad0[SI_MAX_SIZE / sizeof(int) - SI_PAD_SIZE - 3];
-
-	union {
-		int _pad[SI_PAD_SIZE];
-
-		/* kill() */
-		struct {
-			__kernel_pid_t _pid;	/* sender's pid */
-			__ARCH_SI_UID_T _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			__kernel_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			char _pad[sizeof( __ARCH_SI_UID_T) - sizeof(int)];
-			sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			__kernel_pid_t _pid;	/* sender's pid */
-			__ARCH_SI_UID_T _uid;	/* sender's uid */
-			sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			__kernel_pid_t _pid;	/* which child */
-			__ARCH_SI_UID_T _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			__kernel_clock_t _utime;
-			__kernel_clock_t _stime;
-		} _sigchld;
-
-		/* IRIX SIGCHLD */
-		struct {
-			__kernel_pid_t _pid;	/* which child */
-			__kernel_clock_t _utime;
-			int _status;		/* exit code */
-			__kernel_clock_t _stime;
-		} _irix_sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			void __user *_addr; /* faulting insn/memory ref. */
-#ifdef __ARCH_SI_TRAPNO
-			int _trapno;	/* TRAP # which caused the signal */
-#endif
-			short _addr_lsb;
-			union {
-				/* used when si_code=SEGV_BNDERR */
-				struct {
-					void __user *_lower;
-					void __user *_upper;
-				} _addr_bnd;
-				/* used when si_code=SEGV_PKUERR */
-				__u32 _pkey;
-			};
-		} _sigfault;
-
-		/* SIGPOLL, SIGXFSZ (To do ...)	 */
-		struct {
-			__ARCH_SI_BAND_T _band; /* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		/* SIGSYS */
-		struct {
-			void __user *_call_addr; /* calling user insn */
-			int _syscall;	/* triggering system call number */
-			unsigned int _arch;	/* AUDIT_ARCH_* of syscall */
-		} _sigsys;
-	} _sifields;
-} siginfo_t;
-
 /*
  * si_code values
  * Again these have been chosen to be IRIX compatible.
diff --git a/arch/mips/jazz/setup.c b/arch/mips/jazz/setup.c
index e4374a5651ce..448fd41792e4 100644
--- a/arch/mips/jazz/setup.c
+++ b/arch/mips/jazz/setup.c
@@ -32,22 +32,22 @@ static struct resource jazz_io_resources[] = {
 		.start	= 0x00,
 		.end	= 0x1f,
 		.name	= "dma1",
-		.flags	= IORESOURCE_BUSY
+		.flags	= IORESOURCE_IO | IORESOURCE_BUSY
 	}, {
 		.start	= 0x40,
 		.end	= 0x5f,
 		.name	= "timer",
-		.flags	= IORESOURCE_BUSY
+		.flags	= IORESOURCE_IO | IORESOURCE_BUSY
 	}, {
 		.start	= 0x80,
 		.end	= 0x8f,
 		.name	= "dma page reg",
-		.flags	= IORESOURCE_BUSY
+		.flags	= IORESOURCE_IO | IORESOURCE_BUSY
 	}, {
 		.start	= 0xc0,
 		.end	= 0xdf,
 		.name	= "dma2",
-		.flags	= IORESOURCE_BUSY
+		.flags	= IORESOURCE_IO | IORESOURCE_BUSY
 	}
 };
 
diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig
index 643af2012e14..4dd0c446ecec 100644
--- a/arch/mips/jz4740/Kconfig
+++ b/arch/mips/jz4740/Kconfig
@@ -8,6 +8,10 @@ config JZ4740_QI_LB60
 	bool "Qi Hardware Ben NanoNote"
 	select MACH_JZ4740
 
+config JZ4770_GCW0
+	bool "Game Consoles Worldwide GCW Zero"
+	select MACH_JZ4770
+
 config JZ4780_CI20
 	bool "MIPS Creator CI20"
 	select MACH_JZ4780
@@ -18,6 +22,12 @@ config MACH_JZ4740
 	bool
 	select SYS_HAS_CPU_MIPS32_R1
 
+config MACH_JZ4770
+	bool
+	select MIPS_CPU_SCACHE
+	select SYS_HAS_CPU_MIPS32_R2
+	select SYS_SUPPORTS_HIGHMEM
+
 config MACH_JZ4780
 	bool
 	select MIPS_CPU_SCACHE
diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c
index 47e857194ce6..eb9f2f97bedb 100644
--- a/arch/mips/jz4740/prom.c
+++ b/arch/mips/jz4740/prom.c
@@ -20,33 +20,12 @@
 #include <linux/serial_reg.h>
 
 #include <asm/bootinfo.h>
+#include <asm/fw/fw.h>
 #include <asm/mach-jz4740/base.h>
 
-static __init void jz4740_init_cmdline(int argc, char *argv[])
-{
-	unsigned int count = COMMAND_LINE_SIZE - 1;
-	int i;
-	char *dst = &(arcs_cmdline[0]);
-	char *src;
-
-	for (i = 1; i < argc && count; ++i) {
-		src = argv[i];
-		while (*src && count) {
-			*dst++ = *src++;
-			--count;
-		}
-		*dst++ = ' ';
-	}
-	if (i > 1)
-		--dst;
-
-	*dst = 0;
-}
-
 void __init prom_init(void)
 {
-	jz4740_init_cmdline((int)fw_arg0, (char **)fw_arg1);
-	mips_machtype = MACH_INGENIC_JZ4740;
+	fw_init_cmdline();
 }
 
 void __init prom_free_prom_memory(void)
diff --git a/arch/mips/jz4740/setup.c b/arch/mips/jz4740/setup.c
index 6d0152321819..afb40f8bce96 100644
--- a/arch/mips/jz4740/setup.c
+++ b/arch/mips/jz4740/setup.c
@@ -53,6 +53,16 @@ static void __init jz4740_detect_mem(void)
 	add_memory_region(0, size, BOOT_MEM_RAM);
 }
 
+static unsigned long __init get_board_mach_type(const void *fdt)
+{
+	if (!fdt_node_check_compatible(fdt, 0, "ingenic,jz4780"))
+		return MACH_INGENIC_JZ4780;
+	if (!fdt_node_check_compatible(fdt, 0, "ingenic,jz4770"))
+		return MACH_INGENIC_JZ4770;
+
+	return MACH_INGENIC_JZ4740;
+}
+
 void __init plat_mem_setup(void)
 {
 	int offset;
@@ -63,6 +73,8 @@ void __init plat_mem_setup(void)
 	offset = fdt_path_offset(__dtb_start, "/memory");
 	if (offset < 0)
 		jz4740_detect_mem();
+
+	mips_machtype = get_board_mach_type(__dtb_start);
 }
 
 void __init device_tree_init(void)
@@ -75,10 +87,14 @@ void __init device_tree_init(void)
 
 const char *get_system_type(void)
 {
-	if (IS_ENABLED(CONFIG_MACH_JZ4780))
+	switch (mips_machtype) {
+	case MACH_INGENIC_JZ4780:
 		return "JZ4780";
-
-	return "JZ4740";
+	case MACH_INGENIC_JZ4770:
+		return "JZ4770";
+	default:
+		return "JZ4740";
+	}
 }
 
 void __init arch_init_irq(void)
diff --git a/arch/mips/jz4740/time.c b/arch/mips/jz4740/time.c
index bb1ad5119da4..2ca9160f642a 100644
--- a/arch/mips/jz4740/time.c
+++ b/arch/mips/jz4740/time.c
@@ -113,7 +113,7 @@ static struct clock_event_device jz4740_clockevent = {
 #ifdef CONFIG_MACH_JZ4740
 	.irq = JZ4740_IRQ_TCU0,
 #endif
-#ifdef CONFIG_MACH_JZ4780
+#if defined(CONFIG_MACH_JZ4770) || defined(CONFIG_MACH_JZ4780)
 	.irq = JZ4780_IRQ_TCU2,
 #endif
 };
diff --git a/arch/mips/kernel/branch.c b/arch/mips/kernel/branch.c
index b79ed9af9886..e48f6c0a9e4a 100644
--- a/arch/mips/kernel/branch.c
+++ b/arch/mips/kernel/branch.c
@@ -399,7 +399,7 @@ int __MIPS16e_compute_return_epc(struct pt_regs *regs)
  *
  * @regs:	Pointer to pt_regs
  * @insn:	branch instruction to decode
- * @returns:	-EFAULT on error and forces SIGILL, and on success
+ * Return:	-EFAULT on error and forces SIGILL, and on success
  *		returns 0 or BRANCH_LIKELY_TAKEN as appropriate after
  *		evaluating the branch.
  *
diff --git a/arch/mips/kernel/cps-vec.S b/arch/mips/kernel/cps-vec.S
index e68e6e04063a..1025f937ab0e 100644
--- a/arch/mips/kernel/cps-vec.S
+++ b/arch/mips/kernel/cps-vec.S
@@ -388,15 +388,16 @@ LEAF(mips_cps_boot_vpes)
 
 #elif defined(CONFIG_MIPS_MT)
 
-	.set	push
-	.set	MIPS_ISA_LEVEL_RAW
-	.set	mt
-
 	/* If the core doesn't support MT then return */
 	has_mt	t0, 5f
 
 	/* Enter VPE configuration state */
+	.set	push
+	.set	MIPS_ISA_LEVEL_RAW
+	.set	mt
 	dvpe
+	.set	pop
+
 	PTR_LA	t1, 1f
 	jr.hb	t1
 	 nop
@@ -422,6 +423,10 @@ LEAF(mips_cps_boot_vpes)
 	mtc0	t0, CP0_VPECONTROL
 	ehb
 
+	.set	push
+	.set	MIPS_ISA_LEVEL_RAW
+	.set	mt
+
 	/* Skip the VPE if its TC is not halted */
 	mftc0	t0, CP0_TCHALT
 	beqz	t0, 2f
@@ -495,6 +500,8 @@ LEAF(mips_cps_boot_vpes)
 	ehb
 	evpe
 
+	.set	pop
+
 	/* Check whether this VPE is meant to be running */
 	li	t0, 1
 	sll	t0, t0, a1
@@ -509,7 +516,7 @@ LEAF(mips_cps_boot_vpes)
 1:	jr.hb	t0
 	 nop
 
-2:	.set	pop
+2:
 
 #endif /* CONFIG_MIPS_MT_SMP */
 
diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c
index 99285be0e088..7f3dfdbc3657 100644
--- a/arch/mips/kernel/ftrace.c
+++ b/arch/mips/kernel/ftrace.c
@@ -361,7 +361,7 @@ void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra,
 	 * If fails when getting the stack address of the non-leaf function's
 	 * ra, stop function graph tracer and return
 	 */
-	if (parent_ra_addr == 0)
+	if (parent_ra_addr == NULL)
 		goto out;
 #endif
 	/* *parent_ra_addr = return_hooker; */
diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c
index 19c88d770054..fcf9af492d60 100644
--- a/arch/mips/kernel/mips-cpc.c
+++ b/arch/mips/kernel/mips-cpc.c
@@ -10,6 +10,8 @@
 
 #include <linux/errno.h>
 #include <linux/percpu.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/spinlock.h>
 
 #include <asm/mips-cps.h>
@@ -22,6 +24,17 @@ static DEFINE_PER_CPU_ALIGNED(unsigned long, cpc_core_lock_flags);
 
 phys_addr_t __weak mips_cpc_default_phys_base(void)
 {
+	struct device_node *cpc_node;
+	struct resource res;
+	int err;
+
+	cpc_node = of_find_compatible_node(of_root, NULL, "mti,mips-cpc");
+	if (cpc_node) {
+		err = of_address_to_resource(cpc_node, 0, &res);
+		if (!err)
+			return res.start;
+	}
+
 	return 0;
 }
 
diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c
index 0dbcd152a1a9..89950b7bf536 100644
--- a/arch/mips/kernel/prom.c
+++ b/arch/mips/kernel/prom.c
@@ -44,11 +44,6 @@ void __init early_init_dt_add_memory_arch(u64 base, u64 size)
 	return add_memory_region(base, size, BOOT_MEM_RAM);
 }
 
-void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
-{
-	return __alloc_bootmem(size, align, __pa(MAX_DMA_ADDRESS));
-}
-
 int __init early_init_dt_reserve_memory_arch(phys_addr_t base,
 					phys_addr_t size, bool nomap)
 {
diff --git a/arch/mips/kernel/rtlx.c b/arch/mips/kernel/rtlx.c
index b80dd8b17a76..18c509c59f33 100644
--- a/arch/mips/kernel/rtlx.c
+++ b/arch/mips/kernel/rtlx.c
@@ -336,10 +336,10 @@ static int file_release(struct inode *inode, struct file *filp)
 	return rtlx_release(iminor(inode));
 }
 
-static unsigned int file_poll(struct file *file, poll_table *wait)
+static __poll_t file_poll(struct file *file, poll_table *wait)
 {
 	int minor = iminor(file_inode(file));
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &channel_wqs[minor].rt_queue, wait);
 	poll_wait(file, &channel_wqs[minor].lx_queue, wait);
@@ -349,11 +349,11 @@ static unsigned int file_poll(struct file *file, poll_table *wait)
 
 	/* data available to read? */
 	if (rtlx_read_poll(minor, 0))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	/* space to write */
 	if (rtlx_write_poll(minor))
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 
 	return mask;
 }
diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 702c678de116..5f8b0a9e30b3 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -375,6 +375,7 @@ static void __init bootmem_init(void)
 	unsigned long reserved_end;
 	unsigned long mapstart = ~0UL;
 	unsigned long bootmap_size;
+	phys_addr_t ramstart = (phys_addr_t)ULLONG_MAX;
 	bool bootmap_valid = false;
 	int i;
 
@@ -395,7 +396,8 @@ static void __init bootmem_init(void)
 	max_low_pfn = 0;
 
 	/*
-	 * Find the highest page frame number we have available.
+	 * Find the highest page frame number we have available
+	 * and the lowest used RAM address
 	 */
 	for (i = 0; i < boot_mem_map.nr_map; i++) {
 		unsigned long start, end;
@@ -407,6 +409,8 @@ static void __init bootmem_init(void)
 		end = PFN_DOWN(boot_mem_map.map[i].addr
 				+ boot_mem_map.map[i].size);
 
+		ramstart = min(ramstart, boot_mem_map.map[i].addr);
+
 #ifndef CONFIG_HIGHMEM
 		/*
 		 * Skip highmem here so we get an accurate max_low_pfn if low
@@ -436,6 +440,13 @@ static void __init bootmem_init(void)
 		mapstart = max(reserved_end, start);
 	}
 
+	/*
+	 * Reserve any memory between the start of RAM and PHYS_OFFSET
+	 */
+	if (ramstart > PHYS_OFFSET)
+		add_memory_region(PHYS_OFFSET, ramstart - PHYS_OFFSET,
+				  BOOT_MEM_RESERVED);
+
 	if (min_low_pfn >= max_low_pfn)
 		panic("Incorrect memory mapping !!!");
 	if (min_low_pfn > ARCH_PFN_OFFSET) {
@@ -664,9 +675,6 @@ static int __init early_parse_mem(char *p)
 
 	add_memory_region(start, size, BOOT_MEM_RAM);
 
-	if (start && start > PHYS_OFFSET)
-		add_memory_region(PHYS_OFFSET, start - PHYS_OFFSET,
-				BOOT_MEM_RESERVED);
 	return 0;
 }
 early_param("mem", early_parse_mem);
@@ -826,25 +834,6 @@ static void __init arch_mem_init(char **cmdline_p)
 	struct memblock_region *reg;
 	extern void plat_mem_setup(void);
 
-	/* call board setup routine */
-	plat_mem_setup();
-
-	/*
-	 * Make sure all kernel memory is in the maps.  The "UP" and
-	 * "DOWN" are opposite for initdata since if it crosses over
-	 * into another memory section you don't want that to be
-	 * freed when the initdata is freed.
-	 */
-	arch_mem_addpart(PFN_DOWN(__pa_symbol(&_text)) << PAGE_SHIFT,
-			 PFN_UP(__pa_symbol(&_edata)) << PAGE_SHIFT,
-			 BOOT_MEM_RAM);
-	arch_mem_addpart(PFN_UP(__pa_symbol(&__init_begin)) << PAGE_SHIFT,
-			 PFN_DOWN(__pa_symbol(&__init_end)) << PAGE_SHIFT,
-			 BOOT_MEM_INIT_RAM);
-
-	pr_info("Determined physical RAM map:\n");
-	print_memory_map();
-
 #if defined(CONFIG_CMDLINE_BOOL) && defined(CONFIG_CMDLINE_OVERRIDE)
 	strlcpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
 #else
@@ -872,6 +861,26 @@ static void __init arch_mem_init(char **cmdline_p)
 	}
 #endif
 #endif
+
+	/* call board setup routine */
+	plat_mem_setup();
+
+	/*
+	 * Make sure all kernel memory is in the maps.  The "UP" and
+	 * "DOWN" are opposite for initdata since if it crosses over
+	 * into another memory section you don't want that to be
+	 * freed when the initdata is freed.
+	 */
+	arch_mem_addpart(PFN_DOWN(__pa_symbol(&_text)) << PAGE_SHIFT,
+			 PFN_UP(__pa_symbol(&_edata)) << PAGE_SHIFT,
+			 BOOT_MEM_RAM);
+	arch_mem_addpart(PFN_UP(__pa_symbol(&__init_begin)) << PAGE_SHIFT,
+			 PFN_DOWN(__pa_symbol(&__init_end)) << PAGE_SHIFT,
+			 BOOT_MEM_INIT_RAM);
+
+	pr_info("Determined physical RAM map:\n");
+	print_memory_map();
+
 	strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE);
 
 	*cmdline_p = command_line;
diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c
index cf5c7c05e5a3..c4db910a8794 100644
--- a/arch/mips/kernel/signal32.c
+++ b/arch/mips/kernel/signal32.c
@@ -76,70 +76,3 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct compat_sigaction __user *,
 
 	return ret;
 }
-
-int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from)
-{
-	int err;
-
-	if (!access_ok (VERIFY_WRITE, to, sizeof(compat_siginfo_t)))
-		return -EFAULT;
-
-	/* If you change siginfo_t structure, please be sure
-	   this code is fixed accordingly.
-	   It should never copy any pad contained in the structure
-	   to avoid security leaks, but must copy the generic
-	   3 ints plus the relevant union member.
-	   This routine must convert siginfo from 64bit to 32bit as well
-	   at the same time.  */
-	err = __put_user(from->si_signo, &to->si_signo);
-	err |= __put_user(from->si_errno, &to->si_errno);
-	err |= __put_user(from->si_code, &to->si_code);
-	if (from->si_code < 0)
-		err |= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE);
-	else {
-		switch (siginfo_layout(from->si_signo, from->si_code)) {
-		case SIL_TIMER:
-			err |= __put_user(from->si_tid, &to->si_tid);
-			err |= __put_user(from->si_overrun, &to->si_overrun);
-			err |= __put_user(from->si_int, &to->si_int);
-			break;
-		case SIL_CHLD:
-			err |= __put_user(from->si_utime, &to->si_utime);
-			err |= __put_user(from->si_stime, &to->si_stime);
-			err |= __put_user(from->si_status, &to->si_status);
-		case SIL_KILL:
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_uid, &to->si_uid);
-			break;
-		case SIL_FAULT:
-			err |= __put_user((unsigned long)from->si_addr, &to->si_addr);
-			break;
-		case SIL_POLL:
-			err |= __put_user(from->si_band, &to->si_band);
-			err |= __put_user(from->si_fd, &to->si_fd);
-			break;
-		case SIL_RT:
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_uid, &to->si_uid);
-			err |= __put_user(from->si_int, &to->si_int);
-			break;
-		case SIL_SYS:
-			err |= __copy_to_user(&to->si_call_addr, &from->si_call_addr,
-					      sizeof(compat_uptr_t));
-			err |= __put_user(from->si_syscall, &to->si_syscall);
-			err |= __put_user(from->si_arch, &to->si_arch);
-			break;
-		}
-	}
-	return err;
-}
-
-int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from)
-{
-	if (copy_from_user(to, from, 3*sizeof(int)) ||
-	    copy_from_user(to->_sifields._pad,
-			   from->_sifields._pad, SI_PAD_SIZE32))
-		return -EFAULT;
-
-	return 0;
-}
diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c
index 87dcac2447c8..159e83add4bb 100644
--- a/arch/mips/kernel/smp-bmips.c
+++ b/arch/mips/kernel/smp-bmips.c
@@ -168,11 +168,11 @@ static void bmips_prepare_cpus(unsigned int max_cpus)
 		return;
 	}
 
-	if (request_irq(IPI0_IRQ, bmips_ipi_interrupt, IRQF_PERCPU,
-			"smp_ipi0", NULL))
+	if (request_irq(IPI0_IRQ, bmips_ipi_interrupt,
+			IRQF_PERCPU | IRQF_NO_SUSPEND, "smp_ipi0", NULL))
 		panic("Can't request IPI0 interrupt");
-	if (request_irq(IPI1_IRQ, bmips_ipi_interrupt, IRQF_PERCPU,
-			"smp_ipi1", NULL))
+	if (request_irq(IPI1_IRQ, bmips_ipi_interrupt,
+			IRQF_PERCPU | IRQF_NO_SUSPEND, "smp_ipi1", NULL))
 		panic("Can't request IPI1 interrupt");
 }
 
@@ -572,7 +572,7 @@ asmlinkage void __weak plat_wired_tlb_setup(void)
 	 */
 }
 
-void __init bmips_cpu_setup(void)
+void bmips_cpu_setup(void)
 {
 	void __iomem __maybe_unused *cbr = BMIPS_GET_CBR();
 	u32 __maybe_unused cfg;
diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c
index ecc1a853f48d..03f1026ad148 100644
--- a/arch/mips/kernel/smp-cps.c
+++ b/arch/mips/kernel/smp-cps.c
@@ -439,8 +439,6 @@ void play_dead(void)
 	pr_debug("CPU%d going offline\n", cpu);
 
 	if (cpu_has_mipsmt || cpu_has_vp) {
-		core = cpu_core(&cpu_data[cpu]);
-
 		/* Look for another online VPE within the core */
 		for_each_online_cpu(cpu_death_sibling) {
 			if (!cpus_are_siblings(cpu, cpu_death_sibling))
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 5d19ed07e99d..0ae4a731cc12 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -699,11 +699,12 @@ static int simulate_sync(struct pt_regs *regs, unsigned int opcode)
 asmlinkage void do_ov(struct pt_regs *regs)
 {
 	enum ctx_state prev_state;
-	siginfo_t info = {
-		.si_signo = SIGFPE,
-		.si_code = FPE_INTOVF,
-		.si_addr = (void __user *)regs->cp0_epc,
-	};
+	siginfo_t info;
+
+	clear_siginfo(&info);
+	info.si_signo = SIGFPE;
+	info.si_code = FPE_INTOVF;
+	info.si_addr = (void __user *)regs->cp0_epc;
 
 	prev_state = exception_enter();
 	die_if_kernel("Integer overflow", regs);
@@ -721,7 +722,11 @@ asmlinkage void do_ov(struct pt_regs *regs)
 void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr,
 		     struct task_struct *tsk)
 {
-	struct siginfo si = { .si_addr = fault_addr, .si_signo = SIGFPE };
+	struct siginfo si;
+
+	clear_siginfo(&si);
+	si.si_addr = fault_addr;
+	si.si_signo = SIGFPE;
 
 	if (fcr31 & FPU_CSR_INV_X)
 		si.si_code = FPE_FLTINV;
@@ -739,9 +744,10 @@ void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr,
 
 int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31)
 {
-	struct siginfo si = { 0 };
+	struct siginfo si;
 	struct vm_area_struct *vma;
 
+	clear_siginfo(&si);
 	switch (sig) {
 	case 0:
 		return 0;
@@ -890,9 +896,10 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
 void do_trap_or_bp(struct pt_regs *regs, unsigned int code, int si_code,
 	const char *str)
 {
-	siginfo_t info = { 0 };
+	siginfo_t info;
 	char b[40];
 
+	clear_siginfo(&info);
 #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
 	if (kgdb_ll_trap(DIE_TRAP, str, regs, code, current->thread.trap_nr,
 			 SIGTRAP) == NOTIFY_STOP)
@@ -1499,9 +1506,13 @@ asmlinkage void do_mdmx(struct pt_regs *regs)
  */
 asmlinkage void do_watch(struct pt_regs *regs)
 {
-	siginfo_t info = { .si_signo = SIGTRAP, .si_code = TRAP_HWBKPT };
+	siginfo_t info;
 	enum ctx_state prev_state;
 
+	clear_siginfo(&info);
+	info.si_signo = SIGTRAP;
+	info.si_code = TRAP_HWBKPT;
+
 	prev_state = exception_enter();
 	/*
 	 * Clear WP (bit 22) bit of cause register so we don't loop
diff --git a/arch/mips/kernel/watch.c b/arch/mips/kernel/watch.c
index 19fcab7348b1..0e61a5b7647f 100644
--- a/arch/mips/kernel/watch.c
+++ b/arch/mips/kernel/watch.c
@@ -18,27 +18,24 @@
 void mips_install_watch_registers(struct task_struct *t)
 {
 	struct mips3264_watch_reg_state *watches = &t->thread.watch.mips3264;
+	unsigned int watchhi = MIPS_WATCHHI_G |		/* Trap all ASIDs */
+			       MIPS_WATCHHI_IRW;	/* Clear result bits */
+
 	switch (current_cpu_data.watch_reg_use_cnt) {
 	default:
 		BUG();
 	case 4:
 		write_c0_watchlo3(watches->watchlo[3]);
-		/* Write 1 to the I, R, and W bits to clear them, and
-		   1 to G so all ASIDs are trapped. */
-		write_c0_watchhi3(MIPS_WATCHHI_G | MIPS_WATCHHI_IRW |
-				  watches->watchhi[3]);
+		write_c0_watchhi3(watchhi | watches->watchhi[3]);
 	case 3:
 		write_c0_watchlo2(watches->watchlo[2]);
-		write_c0_watchhi2(MIPS_WATCHHI_G | MIPS_WATCHHI_IRW |
-				  watches->watchhi[2]);
+		write_c0_watchhi2(watchhi | watches->watchhi[2]);
 	case 2:
 		write_c0_watchlo1(watches->watchlo[1]);
-		write_c0_watchhi1(MIPS_WATCHHI_G | MIPS_WATCHHI_IRW |
-				  watches->watchhi[1]);
+		write_c0_watchhi1(watchhi | watches->watchhi[1]);
 	case 1:
 		write_c0_watchlo0(watches->watchlo[0]);
-		write_c0_watchhi0(MIPS_WATCHHI_G | MIPS_WATCHHI_IRW |
-				  watches->watchhi[0]);
+		write_c0_watchhi0(watchhi | watches->watchhi[0]);
 	}
 }
 
@@ -51,21 +48,19 @@ void mips_read_watch_registers(void)
 {
 	struct mips3264_watch_reg_state *watches =
 		&current->thread.watch.mips3264;
+	unsigned int watchhi_mask = MIPS_WATCHHI_MASK | MIPS_WATCHHI_IRW;
+
 	switch (current_cpu_data.watch_reg_use_cnt) {
 	default:
 		BUG();
 	case 4:
-		watches->watchhi[3] = (read_c0_watchhi3() &
-				       (MIPS_WATCHHI_MASK | MIPS_WATCHHI_IRW));
+		watches->watchhi[3] = (read_c0_watchhi3() & watchhi_mask);
 	case 3:
-		watches->watchhi[2] = (read_c0_watchhi2() &
-				       (MIPS_WATCHHI_MASK | MIPS_WATCHHI_IRW));
+		watches->watchhi[2] = (read_c0_watchhi2() & watchhi_mask);
 	case 2:
-		watches->watchhi[1] = (read_c0_watchhi1() &
-				       (MIPS_WATCHHI_MASK | MIPS_WATCHHI_IRW));
+		watches->watchhi[1] = (read_c0_watchhi1() & watchhi_mask);
 	case 1:
-		watches->watchhi[0] = (read_c0_watchhi0() &
-				       (MIPS_WATCHHI_MASK | MIPS_WATCHHI_IRW));
+		watches->watchhi[0] = (read_c0_watchhi0() & watchhi_mask);
 	}
 	if (current_cpu_data.watch_reg_use_cnt == 1 &&
 	    (watches->watchhi[0] & MIPS_WATCHHI_IRW) == 0) {
diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig
index b17447ce8873..76b93a9c8c9b 100644
--- a/arch/mips/kvm/Kconfig
+++ b/arch/mips/kvm/Kconfig
@@ -22,6 +22,7 @@ config KVM
 	select PREEMPT_NOTIFIERS
 	select ANON_INODES
 	select KVM_GENERIC_DIRTYLOG_READ_PROTECT
+	select HAVE_KVM_VCPU_ASYNC_IOCTL
 	select KVM_MMIO
 	select MMU_NOTIFIER
 	select SRCU
diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index 75fdeaa8c62f..2549fdd27ee1 100644
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -446,6 +446,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	int r = -EINTR;
 
+	vcpu_load(vcpu);
+
 	kvm_sigset_activate(vcpu);
 
 	if (vcpu->mmio_needed) {
@@ -480,6 +482,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 out:
 	kvm_sigset_deactivate(vcpu);
 
+	vcpu_put(vcpu);
 	return r;
 }
 
@@ -900,6 +903,26 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
 	return r;
 }
 
+long kvm_arch_vcpu_async_ioctl(struct file *filp, unsigned int ioctl,
+			       unsigned long arg)
+{
+	struct kvm_vcpu *vcpu = filp->private_data;
+	void __user *argp = (void __user *)arg;
+
+	if (ioctl == KVM_INTERRUPT) {
+		struct kvm_mips_interrupt irq;
+
+		if (copy_from_user(&irq, argp, sizeof(irq)))
+			return -EFAULT;
+		kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__,
+			  irq.irq);
+
+		return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
+	}
+
+	return -ENOIOCTLCMD;
+}
+
 long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
 			 unsigned long arg)
 {
@@ -907,56 +930,54 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
 	void __user *argp = (void __user *)arg;
 	long r;
 
+	vcpu_load(vcpu);
+
 	switch (ioctl) {
 	case KVM_SET_ONE_REG:
 	case KVM_GET_ONE_REG: {
 		struct kvm_one_reg reg;
 
+		r = -EFAULT;
 		if (copy_from_user(&reg, argp, sizeof(reg)))
-			return -EFAULT;
+			break;
 		if (ioctl == KVM_SET_ONE_REG)
-			return kvm_mips_set_reg(vcpu, &reg);
+			r = kvm_mips_set_reg(vcpu, &reg);
 		else
-			return kvm_mips_get_reg(vcpu, &reg);
+			r = kvm_mips_get_reg(vcpu, &reg);
+		break;
 	}
 	case KVM_GET_REG_LIST: {
 		struct kvm_reg_list __user *user_list = argp;
 		struct kvm_reg_list reg_list;
 		unsigned n;
 
+		r = -EFAULT;
 		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
-			return -EFAULT;
+			break;
 		n = reg_list.n;
 		reg_list.n = kvm_mips_num_regs(vcpu);
 		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
-			return -EFAULT;
+			break;
+		r = -E2BIG;
 		if (n < reg_list.n)
-			return -E2BIG;
-		return kvm_mips_copy_reg_indices(vcpu, user_list->reg);
-	}
-	case KVM_INTERRUPT:
-		{
-			struct kvm_mips_interrupt irq;
-
-			if (copy_from_user(&irq, argp, sizeof(irq)))
-				return -EFAULT;
-			kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__,
-				  irq.irq);
-
-			r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
 			break;
-		}
+		r = kvm_mips_copy_reg_indices(vcpu, user_list->reg);
+		break;
+	}
 	case KVM_ENABLE_CAP: {
 		struct kvm_enable_cap cap;
 
+		r = -EFAULT;
 		if (copy_from_user(&cap, argp, sizeof(cap)))
-			return -EFAULT;
+			break;
 		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
 		break;
 	}
 	default:
 		r = -ENOIOCTLCMD;
 	}
+
+	vcpu_put(vcpu);
 	return r;
 }
 
@@ -1145,6 +1166,8 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
 
+	vcpu_load(vcpu);
+
 	for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
 		vcpu->arch.gprs[i] = regs->gpr[i];
 	vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */
@@ -1152,6 +1175,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	vcpu->arch.lo = regs->lo;
 	vcpu->arch.pc = regs->pc;
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -1159,6 +1183,8 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
 
+	vcpu_load(vcpu);
+
 	for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
 		regs->gpr[i] = vcpu->arch.gprs[i];
 
@@ -1166,6 +1192,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	regs->lo = vcpu->arch.lo;
 	regs->pc = vcpu->arch.pc;
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
diff --git a/arch/mips/lantiq/Kconfig b/arch/mips/lantiq/Kconfig
index 692ae85a3e3d..8e3a1fc2bc39 100644
--- a/arch/mips/lantiq/Kconfig
+++ b/arch/mips/lantiq/Kconfig
@@ -13,6 +13,8 @@ choice
 config SOC_AMAZON_SE
 	bool "Amazon SE"
 	select SOC_TYPE_XWAY
+	select MFD_SYSCON
+	select MFD_CORE
 
 config SOC_XWAY
 	bool "XWAY"
diff --git a/arch/mips/lantiq/xway/sysctrl.c b/arch/mips/lantiq/xway/sysctrl.c
index 52500d3b7004..e0af39b33e28 100644
--- a/arch/mips/lantiq/xway/sysctrl.c
+++ b/arch/mips/lantiq/xway/sysctrl.c
@@ -549,9 +549,9 @@ void __init ltq_soc_init(void)
 		clkdev_add_static(ltq_ar9_cpu_hz(), ltq_ar9_fpi_hz(),
 				ltq_ar9_fpi_hz(), CLOCK_250M);
 		clkdev_add_pmu("1f203018.usb2-phy", "phy", 1, 0, PMU_USB0_P);
-		clkdev_add_pmu("1e101000.usb", "otg", 1, 0, PMU_USB0);
+		clkdev_add_pmu("1e101000.usb", "otg", 1, 0, PMU_USB0 | PMU_AHBM);
 		clkdev_add_pmu("1f203034.usb2-phy", "phy", 1, 0, PMU_USB1_P);
-		clkdev_add_pmu("1e106000.usb", "otg", 1, 0, PMU_USB1);
+		clkdev_add_pmu("1e106000.usb", "otg", 1, 0, PMU_USB1 | PMU_AHBM);
 		clkdev_add_pmu("1e180000.etop", "switch", 1, 0, PMU_SWITCH);
 		clkdev_add_pmu("1e103000.sdio", NULL, 1, 0, PMU_SDIO);
 		clkdev_add_pmu("1e103100.deu", NULL, 1, 0, PMU_DEU);
@@ -560,7 +560,7 @@ void __init ltq_soc_init(void)
 	} else {
 		clkdev_add_static(ltq_danube_cpu_hz(), ltq_danube_fpi_hz(),
 				ltq_danube_fpi_hz(), ltq_danube_pp32_hz());
-		clkdev_add_pmu("1f203018.usb2-phy", "ctrl", 1, 0, PMU_USB0);
+		clkdev_add_pmu("1e101000.usb", "otg", 1, 0, PMU_USB0 | PMU_AHBM);
 		clkdev_add_pmu("1f203018.usb2-phy", "phy", 1, 0, PMU_USB0_P);
 		clkdev_add_pmu("1e103000.sdio", NULL, 1, 0, PMU_SDIO);
 		clkdev_add_pmu("1e103100.deu", NULL, 1, 0, PMU_DEU);
diff --git a/arch/mips/loongson64/Kconfig b/arch/mips/loongson64/Kconfig
index 0d249fc3cfe9..72af0c183969 100644
--- a/arch/mips/loongson64/Kconfig
+++ b/arch/mips/loongson64/Kconfig
@@ -7,6 +7,8 @@ choice
 config LEMOTE_FULOONG2E
 	bool "Lemote Fuloong(2e) mini-PC"
 	select ARCH_SPARSEMEM_ENABLE
+	select ARCH_MIGHT_HAVE_PC_PARPORT
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select CEVT_R4K
 	select CSRC_R4K
 	select SYS_HAS_CPU_LOONGSON2E
@@ -17,7 +19,6 @@ config LEMOTE_FULOONG2E
 	select I8259
 	select ISA
 	select IRQ_MIPS_CPU
-	select SYS_SUPPORTS_32BIT_KERNEL
 	select SYS_SUPPORTS_64BIT_KERNEL
 	select SYS_SUPPORTS_LITTLE_ENDIAN
 	select SYS_SUPPORTS_HIGHMEM
@@ -34,6 +35,8 @@ config LEMOTE_FULOONG2E
 config LEMOTE_MACH2F
 	bool "Lemote Loongson 2F family machines"
 	select ARCH_SPARSEMEM_ENABLE
+	select ARCH_MIGHT_HAVE_PC_PARPORT
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select BOARD_SCACHE
 	select BOOT_ELF32
 	select CEVT_R4K if ! MIPS_EXTERNAL_TIMER
@@ -49,7 +52,6 @@ config LEMOTE_MACH2F
 	select ISA
 	select SYS_HAS_CPU_LOONGSON2F
 	select SYS_HAS_EARLY_PRINTK
-	select SYS_SUPPORTS_32BIT_KERNEL
 	select SYS_SUPPORTS_64BIT_KERNEL
 	select SYS_SUPPORTS_HIGHMEM
 	select SYS_SUPPORTS_LITTLE_ENDIAN
@@ -64,6 +66,8 @@ config LEMOTE_MACH2F
 config LOONGSON_MACH3X
 	bool "Generic Loongson 3 family machines"
 	select ARCH_SPARSEMEM_ENABLE
+	select ARCH_MIGHT_HAVE_PC_PARPORT
+	select ARCH_MIGHT_HAVE_PC_SERIO
 	select GENERIC_ISA_DMA_SUPPORT_BROKEN
 	select BOOT_ELF32
 	select BOARD_SCACHE
@@ -136,6 +140,7 @@ config SWIOTLB
 	bool "Soft IOMMU Support for All-Memory DMA"
 	default y
 	depends on CPU_LOONGSON3
+	select DMA_DIRECT_OPS
 	select IOMMU_HELPER
 	select NEED_SG_DMA_LENGTH
 	select NEED_DMA_MAP_STATE
diff --git a/arch/mips/loongson64/common/dma-swiotlb.c b/arch/mips/loongson64/common/dma-swiotlb.c
index ef07740cee61..7bbcf89475f3 100644
--- a/arch/mips/loongson64/common/dma-swiotlb.c
+++ b/arch/mips/loongson64/common/dma-swiotlb.c
@@ -13,32 +13,12 @@
 static void *loongson_dma_alloc_coherent(struct device *dev, size_t size,
 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
 {
-	void *ret;
+	void *ret = swiotlb_alloc(dev, size, dma_handle, gfp, attrs);
 
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
-
-	if ((IS_ENABLED(CONFIG_ISA) && dev == NULL) ||
-	    (IS_ENABLED(CONFIG_ZONE_DMA) &&
-	     dev->coherent_dma_mask < DMA_BIT_MASK(32)))
-		gfp |= __GFP_DMA;
-	else if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
-		 dev->coherent_dma_mask < DMA_BIT_MASK(40))
-		gfp |= __GFP_DMA32;
-
-	gfp |= __GFP_NORETRY;
-
-	ret = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
 	mb();
 	return ret;
 }
 
-static void loongson_dma_free_coherent(struct device *dev, size_t size,
-		void *vaddr, dma_addr_t dma_handle, unsigned long attrs)
-{
-	swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
 static dma_addr_t loongson_dma_map_page(struct device *dev, struct page *page,
 				unsigned long offset, size_t size,
 				enum dma_data_direction dir,
@@ -109,7 +89,7 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
 
 static const struct dma_map_ops loongson_dma_map_ops = {
 	.alloc = loongson_dma_alloc_coherent,
-	.free = loongson_dma_free_coherent,
+	.free = swiotlb_free,
 	.map_page = loongson_dma_map_page,
 	.unmap_page = swiotlb_unmap_page,
 	.map_sg = loongson_dma_map_sg,
diff --git a/arch/mips/loongson64/common/mem.c b/arch/mips/loongson64/common/mem.c
index b01d52473da8..c549e525fc11 100644
--- a/arch/mips/loongson64/common/mem.c
+++ b/arch/mips/loongson64/common/mem.c
@@ -79,7 +79,7 @@ void __init prom_init_memory(void)
 					(u64)loongson_memmap->map[i].mem_size << 20,
 					BOOT_MEM_RAM);
 				break;
-			case MEM_RESERVED:
+			case SYSTEM_RAM_RESERVED:
 				add_memory_region(loongson_memmap->map[i].mem_start,
 					(u64)loongson_memmap->map[i].mem_size << 20,
 					BOOT_MEM_RESERVED);
diff --git a/arch/mips/loongson64/loongson-3/numa.c b/arch/mips/loongson64/loongson-3/numa.c
index f17ef520799a..9717106de4a5 100644
--- a/arch/mips/loongson64/loongson-3/numa.c
+++ b/arch/mips/loongson64/loongson-3/numa.c
@@ -166,7 +166,7 @@ static void __init szmem(unsigned int node)
 			memblock_add_node(PFN_PHYS(start_pfn),
 				PFN_PHYS(end_pfn - start_pfn), node);
 			break;
-		case MEM_RESERVED:
+		case SYSTEM_RAM_RESERVED:
 			pr_info("Node%d: mem_type:%d, mem_start:0x%llx, mem_size:0x%llx MB\n",
 				(u32)node_id, mem_type, mem_start, mem_size);
 			add_memory_region((node_id << 44) + mem_start,
diff --git a/arch/mips/math-emu/cp1emu.c b/arch/mips/math-emu/cp1emu.c
index da6c1c0c30c1..62deb025970b 100644
--- a/arch/mips/math-emu/cp1emu.c
+++ b/arch/mips/math-emu/cp1emu.c
@@ -451,7 +451,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 					regs->cp0_epc + dec_insn.pc_inc +
 					dec_insn.next_pc_inc;
 			}
-			/* Fall through */
+			/* fall through */
 		case jr_op:
 			/* For R6, JR already emulated in jalr_op */
 			if (NO_R6EMU && insn.r_format.func == jr_op)
@@ -471,10 +471,11 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 			regs->regs[31] = regs->cp0_epc +
 				dec_insn.pc_inc +
 				dec_insn.next_pc_inc;
-			/* Fall through */
+			/* fall through */
 		case bltzl_op:
 			if (NO_R6EMU)
 				break;
+			/* fall through */
 		case bltz_op:
 			if ((long)regs->regs[insn.i_format.rs] < 0)
 				*contpc = regs->cp0_epc +
@@ -494,10 +495,11 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 			regs->regs[31] = regs->cp0_epc +
 				dec_insn.pc_inc +
 				dec_insn.next_pc_inc;
-			/* Fall through */
+			/* fall through */
 		case bgezl_op:
 			if (NO_R6EMU)
 				break;
+			/* fall through */
 		case bgez_op:
 			if ((long)regs->regs[insn.i_format.rs] >= 0)
 				*contpc = regs->cp0_epc +
@@ -512,11 +514,12 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 		break;
 	case jalx_op:
 		set_isa16_mode(bit);
+		/* fall through */
 	case jal_op:
 		regs->regs[31] = regs->cp0_epc +
 			dec_insn.pc_inc +
 			dec_insn.next_pc_inc;
-		/* Fall through */
+		/* fall through */
 	case j_op:
 		*contpc = regs->cp0_epc + dec_insn.pc_inc;
 		*contpc >>= 28;
@@ -528,6 +531,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 	case beql_op:
 		if (NO_R6EMU)
 			break;
+		/* fall through */
 	case beq_op:
 		if (regs->regs[insn.i_format.rs] ==
 		    regs->regs[insn.i_format.rt])
@@ -542,6 +546,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 	case bnel_op:
 		if (NO_R6EMU)
 			break;
+		/* fall through */
 	case bne_op:
 		if (regs->regs[insn.i_format.rs] !=
 		    regs->regs[insn.i_format.rt])
@@ -556,6 +561,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 	case blezl_op:
 		if (!insn.i_format.rt && NO_R6EMU)
 			break;
+		/* fall through */
 	case blez_op:
 
 		/*
@@ -593,6 +599,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 	case bgtzl_op:
 		if (!insn.i_format.rt && NO_R6EMU)
 			break;
+		/* fall through */
 	case bgtz_op:
 		/*
 		 * Compact branches for R6 for the
@@ -729,7 +736,8 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn,
 
 			return 1;
 		}
-		/* R2/R6 compatible cop1 instruction. Fall through */
+		/* R2/R6 compatible cop1 instruction */
+		/* fall through */
 	case cop2_op:
 	case cop1x_op:
 		if (insn.i_format.rs == bc_op) {
@@ -1190,7 +1198,8 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx,
 			if (!cpu_has_mips_r6 || delay_slot(xcp))
 				return SIGILL;
 
-			cond = likely = 0;
+			likely = 0;
+			cond = 0;
 			fpr = &current->thread.fpu.fpr[MIPSInst_RT(ir)];
 			bit0 = get_fpr32(fpr, 0) & 0x1;
 			switch (MIPSInst_RS(ir)) {
@@ -1220,14 +1229,14 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx,
 			case bcfl_op:
 				if (cpu_has_mips_2_3_4_5_r)
 					likely = 1;
-				/* Fall through */
+				/* fall through */
 			case bcf_op:
 				cond = !cond;
 				break;
 			case bctl_op:
 				if (cpu_has_mips_2_3_4_5_r)
 					likely = 1;
-				/* Fall through */
+				/* fall through */
 			case bct_op:
 				break;
 			}
@@ -1353,7 +1362,8 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx,
 				return SIGILL;
 
 			/* a real fpu computation instruction */
-			if ((sig = fpu_emu(xcp, ctx, ir)))
+			sig = fpu_emu(xcp, ctx, ir);
+			if (sig)
 				return sig;
 		}
 		break;
diff --git a/arch/mips/math-emu/dp_add.c b/arch/mips/math-emu/dp_add.c
index 8954ef031f84..678de20e4cb1 100644
--- a/arch/mips/math-emu/dp_add.c
+++ b/arch/mips/math-emu/dp_add.c
@@ -104,8 +104,7 @@ union ieee754dp ieee754dp_add(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
-
-		/* FALL THROUGH */
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		DPDNORMY;
diff --git a/arch/mips/math-emu/dp_div.c b/arch/mips/math-emu/dp_div.c
index f4746f7c5f63..3063ae3ab3b9 100644
--- a/arch/mips/math-emu/dp_div.c
+++ b/arch/mips/math-emu/dp_div.c
@@ -103,6 +103,7 @@ union ieee754dp ieee754dp_div(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		DPDNORMY;
diff --git a/arch/mips/math-emu/dp_fmax.c b/arch/mips/math-emu/dp_fmax.c
index 5bec64f2884e..d1f984b40344 100644
--- a/arch/mips/math-emu/dp_fmax.c
+++ b/arch/mips/math-emu/dp_fmax.c
@@ -96,6 +96,7 @@ union ieee754dp ieee754dp_fmax(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		DPDNORMY;
@@ -224,6 +225,7 @@ union ieee754dp ieee754dp_fmaxa(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		DPDNORMY;
diff --git a/arch/mips/math-emu/dp_fmin.c b/arch/mips/math-emu/dp_fmin.c
index a287b23818d8..f98b96135c8d 100644
--- a/arch/mips/math-emu/dp_fmin.c
+++ b/arch/mips/math-emu/dp_fmin.c
@@ -96,6 +96,7 @@ union ieee754dp ieee754dp_fmin(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		DPDNORMY;
@@ -224,6 +225,7 @@ union ieee754dp ieee754dp_fmina(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		DPDNORMY;
diff --git a/arch/mips/math-emu/dp_maddf.c b/arch/mips/math-emu/dp_maddf.c
index 7ad79ed411f5..7ea2f8222026 100644
--- a/arch/mips/math-emu/dp_maddf.c
+++ b/arch/mips/math-emu/dp_maddf.c
@@ -16,7 +16,7 @@
 
 
 /* 128 bits shift right logical with rounding. */
-void srl128(u64 *hptr, u64 *lptr, int count)
+static void srl128(u64 *hptr, u64 *lptr, int count)
 {
 	u64 low;
 
@@ -157,6 +157,7 @@ static union ieee754dp _dp_maddf(union ieee754dp z, union ieee754dp x,
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		if (zc == IEEE754_CLASS_INF)
@@ -173,7 +174,7 @@ static union ieee754dp _dp_maddf(union ieee754dp z, union ieee754dp x,
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_NORM):
 		if (zc == IEEE754_CLASS_INF)
 			return ieee754dp_inf(zs);
-		/* fall through to real computations */
+		/* continue to real computations */
 	}
 
 	/* Finally get to do some computation */
@@ -201,9 +202,6 @@ static union ieee754dp _dp_maddf(union ieee754dp z, union ieee754dp x,
 	 * Multiply 64 bits xm and ym to give 128 bits result in hrm:lrm.
 	 */
 
-	/* 32 * 32 => 64 */
-#define DPXMULT(x, y)	((u64)(x) * (u64)y)
-
 	lxm = xm;
 	hxm = xm >> 32;
 	lym = ym;
diff --git a/arch/mips/math-emu/dp_mul.c b/arch/mips/math-emu/dp_mul.c
index 60c8bfe40947..c34a6cdf1b25 100644
--- a/arch/mips/math-emu/dp_mul.c
+++ b/arch/mips/math-emu/dp_mul.c
@@ -101,6 +101,7 @@ union ieee754dp ieee754dp_mul(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		DPDNORMY;
@@ -128,9 +129,6 @@ union ieee754dp ieee754dp_mul(union ieee754dp x, union ieee754dp y)
 	 * Multiply 64 bits xm, ym to give high 64 bits rm with stickness.
 	 */
 
-	/* 32 * 32 => 64 */
-#define DPXMULT(x, y)	((u64)(x) * (u64)y)
-
 	lxm = xm;
 	hxm = xm >> 32;
 	lym = ym;
diff --git a/arch/mips/math-emu/dp_sqrt.c b/arch/mips/math-emu/dp_sqrt.c
index cea907b83146..1d26c92e5295 100644
--- a/arch/mips/math-emu/dp_sqrt.c
+++ b/arch/mips/math-emu/dp_sqrt.c
@@ -91,7 +91,8 @@ union ieee754dp ieee754dp_sqrt(union ieee754dp x)
 		scalx -= 256;
 	}
 
-	y = x = builddp(0, xe + DP_EBIAS, xm & ~DP_HIDDEN_BIT);
+	x = builddp(0, xe + DP_EBIAS, xm & ~DP_HIDDEN_BIT);
+	y = x;
 
 	/* magic initial approximation to almost 8 sig. bits */
 	yh = y.bits >> 32;
@@ -108,7 +109,8 @@ union ieee754dp ieee754dp_sqrt(union ieee754dp x)
 
 	/* triple to almost 56 sig. bits: y ~= sqrt(x) to within 1 ulp */
 	/* t=y*y; z=t;	pt[n0]+=0x00100000; t+=z; z=(x-z)*y; */
-	z = t = ieee754dp_mul(y, y);
+	t = ieee754dp_mul(y, y);
+	z = t;
 	t.bexp += 0x001;
 	t = ieee754dp_add(t, z);
 	z = ieee754dp_mul(ieee754dp_sub(x, z), y);
@@ -140,7 +142,7 @@ union ieee754dp ieee754dp_sqrt(union ieee754dp x)
 		switch (oldcsr.rm) {
 		case FPU_CSR_RU:
 			y.bits += 1;
-			/* drop through */
+			/* fall through */
 		case FPU_CSR_RN:
 			t.bits += 1;
 			break;
diff --git a/arch/mips/math-emu/dp_sub.c b/arch/mips/math-emu/dp_sub.c
index fc17a781b9ae..3cc48b86519b 100644
--- a/arch/mips/math-emu/dp_sub.c
+++ b/arch/mips/math-emu/dp_sub.c
@@ -106,7 +106,7 @@ union ieee754dp ieee754dp_sub(union ieee754dp x, union ieee754dp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		DPDNORMX;
-		/* FALL THROUGH */
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		/* normalize ym,ye */
diff --git a/arch/mips/math-emu/ieee754dp.h b/arch/mips/math-emu/ieee754dp.h
index 9ba023004eb6..a56707b75282 100644
--- a/arch/mips/math-emu/ieee754dp.h
+++ b/arch/mips/math-emu/ieee754dp.h
@@ -55,6 +55,9 @@ static inline int ieee754dp_finite(union ieee754dp x)
 #define XDPSRS1(v)	\
 	(((v) >> 1) | ((v) & 1))
 
+/* 32bit * 32bit => 64bit unsigned integer multiplication */
+#define DPXMULT(x, y)	((u64)(x) * (u64)y)
+
 /* convert denormal to normalized with extended exponent */
 #define DPDNORMx(m,e) \
 	while ((m >> DP_FBITS) == 0) { m <<= 1; e--; }
diff --git a/arch/mips/math-emu/sp_add.c b/arch/mips/math-emu/sp_add.c
index c55c0c00bca8..51dced9fbdaf 100644
--- a/arch/mips/math-emu/sp_add.c
+++ b/arch/mips/math-emu/sp_add.c
@@ -104,8 +104,7 @@ union ieee754sp ieee754sp_add(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
-
-		/* FALL THROUGH */
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
diff --git a/arch/mips/math-emu/sp_div.c b/arch/mips/math-emu/sp_div.c
index 23587b31ca87..5d2904960eb8 100644
--- a/arch/mips/math-emu/sp_div.c
+++ b/arch/mips/math-emu/sp_div.c
@@ -103,6 +103,7 @@ union ieee754sp ieee754sp_div(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
diff --git a/arch/mips/math-emu/sp_fdp.c b/arch/mips/math-emu/sp_fdp.c
index 5060e8fdcb0b..36a50f9082d1 100644
--- a/arch/mips/math-emu/sp_fdp.c
+++ b/arch/mips/math-emu/sp_fdp.c
@@ -46,7 +46,8 @@ union ieee754sp ieee754sp_fdp(union ieee754dp x)
 	case IEEE754_CLASS_SNAN:
 		x = ieee754dp_nanxcpt(x);
 		EXPLODEXDP;
-		/* Fall through.  */
+		/* fall through */
+
 	case IEEE754_CLASS_QNAN:
 		y = ieee754sp_nan_fdp(xs, xm);
 		if (!ieee754_csr.nan2008) {
diff --git a/arch/mips/math-emu/sp_fmax.c b/arch/mips/math-emu/sp_fmax.c
index 74a5a00d2f22..22019ed691df 100644
--- a/arch/mips/math-emu/sp_fmax.c
+++ b/arch/mips/math-emu/sp_fmax.c
@@ -96,6 +96,7 @@ union ieee754sp ieee754sp_fmax(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
@@ -224,6 +225,7 @@ union ieee754sp ieee754sp_fmaxa(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
diff --git a/arch/mips/math-emu/sp_fmin.c b/arch/mips/math-emu/sp_fmin.c
index c51385f46b09..feaec3985cca 100644
--- a/arch/mips/math-emu/sp_fmin.c
+++ b/arch/mips/math-emu/sp_fmin.c
@@ -96,6 +96,7 @@ union ieee754sp ieee754sp_fmin(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
@@ -224,6 +225,7 @@ union ieee754sp ieee754sp_fmina(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
diff --git a/arch/mips/math-emu/sp_maddf.c b/arch/mips/math-emu/sp_maddf.c
index f823338dbb65..07ba675401e2 100644
--- a/arch/mips/math-emu/sp_maddf.c
+++ b/arch/mips/math-emu/sp_maddf.c
@@ -126,6 +126,7 @@ static union ieee754sp _sp_maddf(union ieee754sp z, union ieee754sp x,
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		if (zc == IEEE754_CLASS_INF)
@@ -142,7 +143,7 @@ static union ieee754sp _sp_maddf(union ieee754sp z, union ieee754sp x,
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_NORM):
 		if (zc == IEEE754_CLASS_INF)
 			return ieee754sp_inf(zs);
-		/* fall through to real computations */
+		/* continue to real computations */
 	}
 
 	/* Finally get to do some computation */
diff --git a/arch/mips/math-emu/sp_mul.c b/arch/mips/math-emu/sp_mul.c
index 4015101fbc37..fde71e293ec4 100644
--- a/arch/mips/math-emu/sp_mul.c
+++ b/arch/mips/math-emu/sp_mul.c
@@ -101,6 +101,7 @@ union ieee754sp ieee754sp_mul(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
diff --git a/arch/mips/math-emu/sp_sqrt.c b/arch/mips/math-emu/sp_sqrt.c
index 67059c33a250..9cc83f012342 100644
--- a/arch/mips/math-emu/sp_sqrt.c
+++ b/arch/mips/math-emu/sp_sqrt.c
@@ -82,7 +82,8 @@ union ieee754sp ieee754sp_sqrt(union ieee754sp x)
 
 	/* generate sqrt(x) bit by bit */
 	ix += ix;
-	q = s = 0;		/* q = sqrt(x) */
+	s = 0;
+	q = 0;			/* q = sqrt(x) */
 	r = 0x01000000;		/* r = moving bit from right to left */
 
 	while (r != 0) {
diff --git a/arch/mips/math-emu/sp_sub.c b/arch/mips/math-emu/sp_sub.c
index dc998ed47295..9f2ff72c3d6b 100644
--- a/arch/mips/math-emu/sp_sub.c
+++ b/arch/mips/math-emu/sp_sub.c
@@ -106,6 +106,7 @@ union ieee754sp ieee754sp_sub(union ieee754sp x, union ieee754sp y)
 
 	case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM):
 		SPDNORMX;
+		/* fall through */
 
 	case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM):
 		SPDNORMY;
diff --git a/arch/mips/math-emu/sp_tlong.c b/arch/mips/math-emu/sp_tlong.c
index a2450c7e452a..bca5ac995801 100644
--- a/arch/mips/math-emu/sp_tlong.c
+++ b/arch/mips/math-emu/sp_tlong.c
@@ -20,7 +20,6 @@
  */
 
 #include "ieee754sp.h"
-#include "ieee754dp.h"
 
 s64 ieee754sp_tlong(union ieee754sp x)
 {
diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c
index e3e94d05f0fd..dcafa43613b6 100644
--- a/arch/mips/mm/dma-default.c
+++ b/arch/mips/mm/dma-default.c
@@ -93,9 +93,6 @@ static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
 {
 	gfp_t dma_flag;
 
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
-
 #ifdef CONFIG_ISA
 	if (dev == NULL)
 		dma_flag = __GFP_DMA;
@@ -373,11 +370,6 @@ static void mips_dma_sync_sg_for_device(struct device *dev,
 	}
 }
 
-static int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
-{
-	return 0;
-}
-
 static int mips_dma_supported(struct device *dev, u64 mask)
 {
 	return plat_dma_supported(dev, mask);
@@ -404,7 +396,6 @@ static const struct dma_map_ops mips_default_dma_map_ops = {
 	.sync_single_for_device = mips_dma_sync_single_for_device,
 	.sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
 	.sync_sg_for_device = mips_dma_sync_sg_for_device,
-	.mapping_error = mips_dma_mapping_error,
 	.dma_supported = mips_dma_supported,
 	.cache_sync = mips_dma_cache_sync,
 };
diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
index 548acb7f8557..394673991bab 100644
--- a/arch/mips/mm/sc-mips.c
+++ b/arch/mips/mm/sc-mips.c
@@ -16,6 +16,7 @@
 #include <asm/mmu_context.h>
 #include <asm/r4kcache.h>
 #include <asm/mips-cps.h>
+#include <asm/bootinfo.h>
 
 /*
  * MIPS32/MIPS64 L2 cache handling
@@ -220,6 +221,14 @@ static inline int __init mips_sc_probe(void)
 	else
 		return 0;
 
+	/*
+	 * According to config2 it would be 5-ways, but that is contradicted
+	 * by all documentation.
+	 */
+	if (current_cpu_type() == CPU_JZRISC &&
+				mips_machtype == MACH_INGENIC_JZ4770)
+		c->scache.ways = 4;
+
 	c->scache.waysize = c->scache.sets * c->scache.linesz;
 	c->scache.waybit = __ffs(c->scache.waysize);
 
diff --git a/arch/mips/mti-malta/malta-setup.c b/arch/mips/mti-malta/malta-setup.c
index de34adb76157..7b63914d2e58 100644
--- a/arch/mips/mti-malta/malta-setup.c
+++ b/arch/mips/mti-malta/malta-setup.c
@@ -47,31 +47,31 @@ static struct resource standard_io_resources[] = {
 		.name = "dma1",
 		.start = 0x00,
 		.end = 0x1f,
-		.flags = IORESOURCE_BUSY
+		.flags = IORESOURCE_IO | IORESOURCE_BUSY
 	},
 	{
 		.name = "timer",
 		.start = 0x40,
 		.end = 0x5f,
-		.flags = IORESOURCE_BUSY
+		.flags = IORESOURCE_IO | IORESOURCE_BUSY
 	},
 	{
 		.name = "keyboard",
 		.start = 0x60,
 		.end = 0x6f,
-		.flags = IORESOURCE_BUSY
+		.flags = IORESOURCE_IO | IORESOURCE_BUSY
 	},
 	{
 		.name = "dma page reg",
 		.start = 0x80,
 		.end = 0x8f,
-		.flags = IORESOURCE_BUSY
+		.flags = IORESOURCE_IO | IORESOURCE_BUSY
 	},
 	{
 		.name = "dma2",
 		.start = 0xc0,
 		.end = 0xdf,
-		.flags = IORESOURCE_BUSY
+		.flags = IORESOURCE_IO | IORESOURCE_BUSY
 	},
 };
 
diff --git a/arch/mips/net/bpf_jit.c b/arch/mips/net/bpf_jit.c
index 44b925005dd3..4d8cb9bb8365 100644
--- a/arch/mips/net/bpf_jit.c
+++ b/arch/mips/net/bpf_jit.c
@@ -1207,8 +1207,6 @@ static int build_body(struct jit_ctx *ctx)
 	return 0;
 }
 
-int bpf_jit_enable __read_mostly;
-
 void bpf_jit_compile(struct bpf_prog *fp)
 {
 	struct jit_ctx ctx;
diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c
index 962b0259b4b6..3e2798bfea4f 100644
--- a/arch/mips/net/ebpf_jit.c
+++ b/arch/mips/net/ebpf_jit.c
@@ -177,8 +177,6 @@ static u32 b_imm(unsigned int tgt, struct jit_ctx *ctx)
 		(ctx->idx * 4) - 4;
 }
 
-int bpf_jit_enable __read_mostly;
-
 enum which_ebpf_reg {
 	src_reg,
 	src_reg_no_fp,
@@ -743,16 +741,11 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 		break;
 	case BPF_ALU | BPF_DIV | BPF_K: /* ALU_IMM */
 	case BPF_ALU | BPF_MOD | BPF_K: /* ALU_IMM */
+		if (insn->imm == 0)
+			return -EINVAL;
 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
 		if (dst < 0)
 			return dst;
-		if (insn->imm == 0) { /* Div by zero */
-			b_off = b_imm(exit_idx, ctx);
-			if (is_bad_offset(b_off))
-				return -E2BIG;
-			emit_instr(ctx, beq, MIPS_R_ZERO, MIPS_R_ZERO, b_off);
-			emit_instr(ctx, addu, MIPS_R_V0, MIPS_R_ZERO, MIPS_R_ZERO);
-		}
 		td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
 		if (td == REG_64BIT || td == REG_32BIT_ZERO_EX)
 			/* sign extend */
@@ -772,19 +765,13 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 		break;
 	case BPF_ALU64 | BPF_DIV | BPF_K: /* ALU_IMM */
 	case BPF_ALU64 | BPF_MOD | BPF_K: /* ALU_IMM */
+		if (insn->imm == 0)
+			return -EINVAL;
 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
 		if (dst < 0)
 			return dst;
-		if (insn->imm == 0) { /* Div by zero */
-			b_off = b_imm(exit_idx, ctx);
-			if (is_bad_offset(b_off))
-				return -E2BIG;
-			emit_instr(ctx, beq, MIPS_R_ZERO, MIPS_R_ZERO, b_off);
-			emit_instr(ctx, addu, MIPS_R_V0, MIPS_R_ZERO, MIPS_R_ZERO);
-		}
 		if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
 			emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
-
 		if (insn->imm == 1) {
 			/* div by 1 is a nop, mod by 1 is zero */
 			if (bpf_op == BPF_MOD)
@@ -862,11 +849,6 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			break;
 		case BPF_DIV:
 		case BPF_MOD:
-			b_off = b_imm(exit_idx, ctx);
-			if (is_bad_offset(b_off))
-				return -E2BIG;
-			emit_instr(ctx, beq, src, MIPS_R_ZERO, b_off);
-			emit_instr(ctx, movz, MIPS_R_V0, MIPS_R_ZERO, src);
 			emit_instr(ctx, ddivu, dst, src);
 			if (bpf_op == BPF_DIV)
 				emit_instr(ctx, mflo, dst);
@@ -945,11 +927,6 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			break;
 		case BPF_DIV:
 		case BPF_MOD:
-			b_off = b_imm(exit_idx, ctx);
-			if (is_bad_offset(b_off))
-				return -E2BIG;
-			emit_instr(ctx, beq, src, MIPS_R_ZERO, b_off);
-			emit_instr(ctx, movz, MIPS_R_V0, MIPS_R_ZERO, src);
 			emit_instr(ctx, divu, dst, src);
 			if (bpf_op == BPF_DIV)
 				emit_instr(ctx, mflo, dst);
@@ -1869,7 +1846,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	unsigned int image_size;
 	u8 *image_ptr;
 
-	if (!bpf_jit_enable || !cpu_has_mips64r2)
+	if (!prog->jit_requested || !cpu_has_mips64r2)
 		return prog;
 
 	tmp = bpf_jit_blind_constants(prog);
diff --git a/arch/mips/netlogic/Kconfig b/arch/mips/netlogic/Kconfig
index 8296b13affd2..7fcfc7fe9f14 100644
--- a/arch/mips/netlogic/Kconfig
+++ b/arch/mips/netlogic/Kconfig
@@ -89,9 +89,4 @@ config IOMMU_HELPER
 config NEED_SG_DMA_LENGTH
 	bool
 
-config SWIOTLB
-	def_bool y
-	select NEED_SG_DMA_LENGTH
-	select IOMMU_HELPER
-
 endif
diff --git a/arch/mips/netlogic/common/Makefile b/arch/mips/netlogic/common/Makefile
index 60d00b5d748e..89f6e3f39fed 100644
--- a/arch/mips/netlogic/common/Makefile
+++ b/arch/mips/netlogic/common/Makefile
@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-y				+= irq.o time.o
-obj-y				+= nlm-dma.o
 obj-y				+= reset.o
 obj-$(CONFIG_SMP)		+= smp.o smpboot.o
 obj-$(CONFIG_EARLY_PRINTK)	+= earlycons.o
diff --git a/arch/mips/netlogic/common/nlm-dma.c b/arch/mips/netlogic/common/nlm-dma.c
deleted file mode 100644
index 0ec9d9da6d51..000000000000
--- a/arch/mips/netlogic/common/nlm-dma.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
-*  Copyright (C) 2003-2013 Broadcom Corporation
-*  All Rights Reserved
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the Broadcom
- * license below:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
- * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/dma-mapping.h>
-#include <linux/scatterlist.h>
-#include <linux/bootmem.h>
-#include <linux/export.h>
-#include <linux/swiotlb.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/mm.h>
-
-#include <asm/bootinfo.h>
-
-static char *nlm_swiotlb;
-
-static void *nlm_dma_alloc_coherent(struct device *dev, size_t size,
-	dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
-{
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
-
-#ifdef CONFIG_ZONE_DMA32
-	if (dev->coherent_dma_mask <= DMA_BIT_MASK(32))
-		gfp |= __GFP_DMA32;
-#endif
-
-	/* Don't invoke OOM killer */
-	gfp |= __GFP_NORETRY;
-
-	return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
-}
-
-static void nlm_dma_free_coherent(struct device *dev, size_t size,
-	void *vaddr, dma_addr_t dma_handle, unsigned long attrs)
-{
-	swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
-const struct dma_map_ops nlm_swiotlb_dma_ops = {
-	.alloc = nlm_dma_alloc_coherent,
-	.free = nlm_dma_free_coherent,
-	.map_page = swiotlb_map_page,
-	.unmap_page = swiotlb_unmap_page,
-	.map_sg = swiotlb_map_sg_attrs,
-	.unmap_sg = swiotlb_unmap_sg_attrs,
-	.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
-	.sync_single_for_device = swiotlb_sync_single_for_device,
-	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
-	.sync_sg_for_device = swiotlb_sync_sg_for_device,
-	.mapping_error = swiotlb_dma_mapping_error,
-	.dma_supported = swiotlb_dma_supported
-};
-
-void __init plat_swiotlb_setup(void)
-{
-	size_t swiotlbsize;
-	unsigned long swiotlb_nslabs;
-
-	swiotlbsize = 1 << 20; /* 1 MB for now */
-	swiotlb_nslabs = swiotlbsize >> IO_TLB_SHIFT;
-	swiotlb_nslabs = ALIGN(swiotlb_nslabs, IO_TLB_SEGSIZE);
-	swiotlbsize = swiotlb_nslabs << IO_TLB_SHIFT;
-
-	nlm_swiotlb = alloc_bootmem_low_pages(swiotlbsize);
-	swiotlb_init_with_tbl(nlm_swiotlb, swiotlb_nslabs, 1);
-}
diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c
index 1b274742077d..d2718de60b9b 100644
--- a/arch/mips/ralink/mt7621.c
+++ b/arch/mips/ralink/mt7621.c
@@ -170,6 +170,28 @@ void prom_soc_init(struct ralink_soc_info *soc_info)
 	u32 n1;
 	u32 rev;
 
+	/* Early detection of CMP support */
+	mips_cm_probe();
+	mips_cpc_probe();
+
+	if (mips_cps_numiocu(0)) {
+		/*
+		 * mips_cm_probe() wipes out bootloader
+		 * config for CM regions and we have to configure them
+		 * again. This SoC cannot talk to pamlbus devices
+		 * witout proper iocu region set up.
+		 *
+		 * FIXME: it would be better to do this with values
+		 * from DT, but we need this very early because
+		 * without this we cannot talk to pretty much anything
+		 * including serial.
+		 */
+		write_gcr_reg0_base(MT7621_PALMBUS_BASE);
+		write_gcr_reg0_mask(~MT7621_PALMBUS_SIZE |
+				    CM_GCR_REGn_MASK_CMTGT_IOCU0);
+		__sync();
+	}
+
 	n0 = __raw_readl(sysc + SYSC_REG_CHIP_NAME0);
 	n1 = __raw_readl(sysc + SYSC_REG_CHIP_NAME1);
 
@@ -194,26 +216,6 @@ void prom_soc_init(struct ralink_soc_info *soc_info)
 
 	rt2880_pinmux_data = mt7621_pinmux_data;
 
-	/* Early detection of CMP support */
-	mips_cm_probe();
-	mips_cpc_probe();
-
-	if (mips_cps_numiocu(0)) {
-		/*
-		 * mips_cm_probe() wipes out bootloader
-		 * config for CM regions and we have to configure them
-		 * again. This SoC cannot talk to pamlbus devices
-		 * witout proper iocu region set up.
-		 *
-		 * FIXME: it would be better to do this with values
-		 * from DT, but we need this very early because
-		 * without this we cannot talk to pretty much anything
-		 * including serial.
-		 */
-		write_gcr_reg0_base(MT7621_PALMBUS_BASE);
-		write_gcr_reg0_mask(~MT7621_PALMBUS_SIZE |
-				    CM_GCR_REGn_MASK_CMTGT_IOCU0);
-	}
 
 	if (!register_cps_smp_ops())
 		return;
diff --git a/arch/mips/ralink/reset.c b/arch/mips/ralink/reset.c
index 64543d66e76b..e9531fea23a2 100644
--- a/arch/mips/ralink/reset.c
+++ b/arch/mips/ralink/reset.c
@@ -96,16 +96,9 @@ static void ralink_restart(char *command)
 	unreachable();
 }
 
-static void ralink_halt(void)
-{
-	local_irq_disable();
-	unreachable();
-}
-
 static int __init mips_reboot_setup(void)
 {
 	_machine_restart = ralink_restart;
-	_machine_halt = ralink_halt;
 
 	return 0;
 }
diff --git a/arch/mips/txx9/rbtx4939/setup.c b/arch/mips/txx9/rbtx4939/setup.c
index 8b937300fb7f..fd26fadc8617 100644
--- a/arch/mips/txx9/rbtx4939/setup.c
+++ b/arch/mips/txx9/rbtx4939/setup.c
@@ -186,7 +186,7 @@ static void __init rbtx4939_update_ioc_pen(void)
 
 #define RBTX4939_MAX_7SEGLEDS	8
 
-#if IS_ENABLED(CONFIG_LEDS_CLASS)
+#if IS_BUILTIN(CONFIG_LEDS_CLASS)
 static u8 led_val[RBTX4939_MAX_7SEGLEDS];
 struct rbtx4939_led_data {
 	struct led_classdev cdev;
@@ -261,7 +261,7 @@ static inline void rbtx4939_led_setup(void)
 
 static void __rbtx4939_7segled_putc(unsigned int pos, unsigned char val)
 {
-#if IS_ENABLED(CONFIG_LEDS_CLASS)
+#if IS_BUILTIN(CONFIG_LEDS_CLASS)
 	unsigned long flags;
 	local_irq_save(flags);
 	/* bit7: reserved for LED class */
diff --git a/arch/mn10300/configs/asb2364_defconfig b/arch/mn10300/configs/asb2364_defconfig
index b1d80cee97ee..a84c3153f22a 100644
--- a/arch/mn10300/configs/asb2364_defconfig
+++ b/arch/mn10300/configs/asb2364_defconfig
@@ -44,7 +44,6 @@ CONFIG_IPV6=y
 # CONFIG_INET6_XFRM_MODE_TRANSPORT is not set
 # CONFIG_INET6_XFRM_MODE_TUNNEL is not set
 # CONFIG_INET6_XFRM_MODE_BEET is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_CONNECTOR=y
 CONFIG_MTD=y
 CONFIG_MTD_DEBUG=y
diff --git a/arch/mn10300/include/asm/Kbuild b/arch/mn10300/include/asm/Kbuild
index db5b57829a81..509c45a75d1f 100644
--- a/arch/mn10300/include/asm/Kbuild
+++ b/arch/mn10300/include/asm/Kbuild
@@ -1,6 +1,5 @@
 
 generic-y += barrier.h
-generic-y += clkdev.h
 generic-y += device.h
 generic-y += exec.h
 generic-y += extable.h
diff --git a/arch/mn10300/include/asm/thread_info.h b/arch/mn10300/include/asm/thread_info.h
index f5f90bbf019d..1748a7b25bf8 100644
--- a/arch/mn10300/include/asm/thread_info.h
+++ b/arch/mn10300/include/asm/thread_info.h
@@ -79,8 +79,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
 #define init_uregs							\
 	((struct pt_regs *)						\
 	 ((unsigned long) init_stack + THREAD_SIZE - sizeof(struct pt_regs)))
diff --git a/arch/mn10300/include/uapi/asm/Kbuild b/arch/mn10300/include/uapi/asm/Kbuild
index 81271d3af47c..b04fd1632051 100644
--- a/arch/mn10300/include/uapi/asm/Kbuild
+++ b/arch/mn10300/include/uapi/asm/Kbuild
@@ -2,4 +2,5 @@
 include include/uapi/asm-generic/Kbuild.asm
 
 generic-y	+= bpf_perf_event.h
+generic-y	+= poll.h
 generic-y	+= siginfo.h
diff --git a/arch/mn10300/include/uapi/asm/poll.h b/arch/mn10300/include/uapi/asm/poll.h
deleted file mode 100644
index b7132a305a47..000000000000
--- a/arch/mn10300/include/uapi/asm/poll.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#include <asm-generic/poll.h>
diff --git a/arch/mn10300/kernel/mn10300-serial.c b/arch/mn10300/kernel/mn10300-serial.c
index d7ef1232a82a..4994b570dfd9 100644
--- a/arch/mn10300/kernel/mn10300-serial.c
+++ b/arch/mn10300/kernel/mn10300-serial.c
@@ -550,7 +550,7 @@ static void mn10300_serial_receive_interrupt(struct mn10300_serial_port *port)
 		return;
 	}
 
-	smp_read_barrier_depends();
+	/* READ_ONCE() enforces dependency, but dangerous through integer!!! */
 	ch = port->rx_buffer[ix++];
 	st = port->rx_buffer[ix++];
 	smp_mb();
@@ -1728,7 +1728,10 @@ static int mn10300_serial_poll_get_char(struct uart_port *_port)
 			if (CIRC_CNT(port->rx_inp, ix, MNSC_BUFFER_SIZE) == 0)
 				return NO_POLL_CHAR;
 
-			smp_read_barrier_depends();
+			/*
+			 * READ_ONCE() enforces dependency, but dangerous
+			 * through integer!!!
+			 */
 			ch = port->rx_buffer[ix++];
 			st = port->rx_buffer[ix++];
 			smp_mb();
diff --git a/arch/mn10300/kernel/traps.c b/arch/mn10300/kernel/traps.c
index 800fd0801969..72d1015b2ae7 100644
--- a/arch/mn10300/kernel/traps.c
+++ b/arch/mn10300/kernel/traps.c
@@ -22,7 +22,6 @@
 #include <linux/delay.h>
 #include <linux/spinlock.h>
 #include <linux/interrupt.h>
-#include <linux/kallsyms.h>
 #include <linux/pci.h>
 #include <linux/kdebug.h>
 #include <linux/bug.h>
@@ -262,8 +261,7 @@ void show_trace(unsigned long *sp)
 				raslot = ULONG_MAX;
 			else
 				printk(" ?");
-			print_symbol(" %s", addr);
-			printk("\n");
+			printk(" %pS\n", (void *)addr);
 		}
 	}
 
diff --git a/arch/mn10300/mm/dma-alloc.c b/arch/mn10300/mm/dma-alloc.c
index 86108d2496b3..e3910d4db102 100644
--- a/arch/mn10300/mm/dma-alloc.c
+++ b/arch/mn10300/mm/dma-alloc.c
@@ -37,9 +37,6 @@ static void *mn10300_dma_alloc(struct device *dev, size_t size,
 		goto done;
 	}
 
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
-
 	if (dev == NULL || dev->coherent_dma_mask < 0xffffffff)
 		gfp |= GFP_DMA;
 
diff --git a/arch/mn10300/mm/misalignment.c b/arch/mn10300/mm/misalignment.c
index b39a388825ae..8ace89617c1c 100644
--- a/arch/mn10300/mm/misalignment.c
+++ b/arch/mn10300/mm/misalignment.c
@@ -437,7 +437,7 @@ asmlinkage void misalignment(struct pt_regs *regs, enum exception_code code)
 
 	info.si_signo	= SIGSEGV;
 	info.si_errno	= 0;
-	info.si_code	= 0;
+	info.si_code	= SEGV_MAPERR;
 	info.si_addr	= (void *) regs->pc;
 	force_sig_info(SIGSEGV, &info, current);
 	return;
diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig
index 60fae03dac79..3d4ec88f1db1 100644
--- a/arch/nios2/Kconfig
+++ b/arch/nios2/Kconfig
@@ -152,7 +152,6 @@ menu "Advanced setup"
 
 config ADVANCED_OPTIONS
 	bool "Prompt for advanced kernel configuration options"
-	help
 
 comment "Default settings for advanced configuration options are used"
 	depends on !ADVANCED_OPTIONS
diff --git a/arch/nios2/boot/dts/3c120_devboard.dts b/arch/nios2/boot/dts/3c120_devboard.dts
index 36ccdf05837d..56f4b5df6d65 100644
--- a/arch/nios2/boot/dts/3c120_devboard.dts
+++ b/arch/nios2/boot/dts/3c120_devboard.dts
@@ -29,7 +29,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		cpu: cpu@0x0 {
+		cpu: cpu@0 {
 			device_type = "cpu";
 			compatible = "altr,nios2-1.0";
 			reg = <0x00000000>;
@@ -69,7 +69,7 @@
 		compatible = "altr,avalon", "simple-bus";
 		bus-frequency = <125000000>;
 
-		pb_cpu_to_io: bridge@0x8000000 {
+		pb_cpu_to_io: bridge@8000000 {
 			compatible = "simple-bus";
 			reg = <0x08000000 0x00800000>;
 			#address-cells = <1>;
@@ -83,7 +83,7 @@
 				<0x00008000 0x08008000 0x00000020>,
 				<0x00400000 0x08400000 0x00000020>;
 
-			timer_1ms: timer@0x400000 {
+			timer_1ms: timer@400000 {
 				compatible = "altr,timer-1.0";
 				reg = <0x00400000 0x00000020>;
 				interrupt-parent = <&cpu>;
@@ -91,7 +91,7 @@
 				clock-frequency = <125000000>;
 			};
 
-			timer_0: timer@0x8000 {
+			timer_0: timer@8000 {
 				compatible = "altr,timer-1.0";
 				reg = < 0x00008000 0x00000020 >;
 				interrupt-parent = < &cpu >;
@@ -99,14 +99,14 @@
 				clock-frequency = < 125000000 >;
 			};
 
-			jtag_uart: serial@0x4d50 {
+			jtag_uart: serial@4d50 {
 				compatible = "altr,juart-1.0";
 				reg = <0x00004d50 0x00000008>;
 				interrupt-parent = <&cpu>;
 				interrupts = <1>;
 			};
 
-			tse_mac: ethernet@0x4000 {
+			tse_mac: ethernet@4000 {
 				compatible = "altr,tse-1.0";
 				reg = <0x00004000 0x00000400>,
 					<0x00004400 0x00000040>,
@@ -133,7 +133,7 @@
 				};
 			};
 
-			uart: serial@0x4c80 {
+			uart: serial@4c80 {
 				compatible = "altr,uart-1.0";
 				reg = <0x00004c80 0x00000020>;
 				interrupt-parent = <&cpu>;
@@ -143,7 +143,7 @@
 			};
 		};
 
-		cfi_flash_64m: flash@0x0 {
+		cfi_flash_64m: flash@0 {
 			compatible = "cfi-flash";
 			reg = <0x00000000 0x04000000>;
 			bank-width = <2>;
diff --git a/arch/nios2/configs/10m50_defconfig b/arch/nios2/configs/10m50_defconfig
index 8b2a30b3b34f..c601c8ff1ae6 100644
--- a/arch/nios2/configs/10m50_defconfig
+++ b/arch/nios2/configs/10m50_defconfig
@@ -33,7 +33,6 @@ CONFIG_IP_PNP_RARP=y
 # CONFIG_INET_XFRM_MODE_TRANSPORT is not set
 # CONFIG_INET_XFRM_MODE_TUNNEL is not set
 # CONFIG_INET_XFRM_MODE_BEET is not set
-# CONFIG_INET_LRO is not set
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
diff --git a/arch/nios2/configs/3c120_defconfig b/arch/nios2/configs/3c120_defconfig
index 9451940678a0..fce33588d55c 100644
--- a/arch/nios2/configs/3c120_defconfig
+++ b/arch/nios2/configs/3c120_defconfig
@@ -35,7 +35,6 @@ CONFIG_IP_PNP_RARP=y
 # CONFIG_INET_XFRM_MODE_TRANSPORT is not set
 # CONFIG_INET_XFRM_MODE_TUNNEL is not set
 # CONFIG_INET_XFRM_MODE_BEET is not set
-# CONFIG_INET_LRO is not set
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
diff --git a/arch/nios2/include/asm/Kbuild b/arch/nios2/include/asm/Kbuild
index 896c26ae0da9..d232da2cbb38 100644
--- a/arch/nios2/include/asm/Kbuild
+++ b/arch/nios2/include/asm/Kbuild
@@ -3,7 +3,6 @@ generic-y += barrier.h
 generic-y += bitops.h
 generic-y += bug.h
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += cmpxchg.h
 generic-y += current.h
 generic-y += device.h
diff --git a/arch/nios2/include/asm/thread_info.h b/arch/nios2/include/asm/thread_info.h
index d69c338bd19c..7349a4fa635b 100644
--- a/arch/nios2/include/asm/thread_info.h
+++ b/arch/nios2/include/asm/thread_info.h
@@ -63,9 +63,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 static inline struct thread_info *current_thread_info(void)
 {
diff --git a/arch/nios2/kernel/prom.c b/arch/nios2/kernel/prom.c
index 6688576b3a47..8d7446a4b475 100644
--- a/arch/nios2/kernel/prom.c
+++ b/arch/nios2/kernel/prom.c
@@ -42,11 +42,6 @@ void __init early_init_dt_add_memory_arch(u64 base, u64 size)
 
 }
 
-void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
-{
-	return alloc_bootmem_align(size, align);
-}
-
 int __init early_init_dt_reserve_memory_arch(phys_addr_t base, phys_addr_t size,
 					     bool nomap)
 {
diff --git a/arch/nios2/mm/dma-mapping.c b/arch/nios2/mm/dma-mapping.c
index 7040c1adbb5e..4be815519dd4 100644
--- a/arch/nios2/mm/dma-mapping.c
+++ b/arch/nios2/mm/dma-mapping.c
@@ -63,9 +63,6 @@ static void *nios2_dma_alloc(struct device *dev, size_t size,
 {
 	void *ret;
 
-	/* ignore region specifiers */
-	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
-
 	/* optimized page clearing */
 	gfp |= __GFP_ZERO;
 
diff --git a/arch/openrisc/include/asm/Kbuild b/arch/openrisc/include/asm/Kbuild
index 6eb16719549e..f05c722a21f8 100644
--- a/arch/openrisc/include/asm/Kbuild
+++ b/arch/openrisc/include/asm/Kbuild
@@ -2,7 +2,6 @@ generic-y += barrier.h
 generic-y += bug.h
 generic-y += bugs.h
 generic-y += checksum.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/openrisc/include/asm/processor.h b/arch/openrisc/include/asm/processor.h
index 396d8f306c21..af31a9fe736a 100644
--- a/arch/openrisc/include/asm/processor.h
+++ b/arch/openrisc/include/asm/processor.h
@@ -84,8 +84,6 @@ void start_thread(struct pt_regs *regs, unsigned long nip, unsigned long sp);
 void release_thread(struct task_struct *);
 unsigned long get_wchan(struct task_struct *p);
 
-#define init_stack      (init_thread_union.stack)
-
 #define cpu_relax()     barrier()
 
 #endif /* __ASSEMBLY__ */
diff --git a/arch/openrisc/include/asm/thread_info.h b/arch/openrisc/include/asm/thread_info.h
index c229aa6bb502..5c15dfa2fd4f 100644
--- a/arch/openrisc/include/asm/thread_info.h
+++ b/arch/openrisc/include/asm/thread_info.h
@@ -79,8 +79,6 @@ struct thread_info {
 	.ksp            = 0,                            \
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-
 /* how to get the thread information struct from C */
 register struct thread_info *current_thread_info_reg asm("r10");
 #define current_thread_info()   (current_thread_info_reg)
diff --git a/arch/openrisc/kernel/traps.c b/arch/openrisc/kernel/traps.c
index 4085d72fa5ae..113c175fe469 100644
--- a/arch/openrisc/kernel/traps.c
+++ b/arch/openrisc/kernel/traps.c
@@ -39,8 +39,7 @@
 #include <asm/io.h>
 #include <asm/pgtable.h>
 #include <asm/unwinder.h>
-
-extern char _etext, _stext;
+#include <asm/sections.h>
 
 int kstack_depth_to_print = 0x180;
 int lwa_flag;
@@ -266,12 +265,12 @@ asmlinkage void do_unaligned_access(struct pt_regs *regs, unsigned long address)
 	siginfo_t info;
 
 	if (user_mode(regs)) {
-		/* Send a SIGSEGV */
-		info.si_signo = SIGSEGV;
+		/* Send a SIGBUS */
+		info.si_signo = SIGBUS;
 		info.si_errno = 0;
-		/* info.si_code has been set above */
-		info.si_addr = (void *)address;
-		force_sig_info(SIGSEGV, &info, current);
+		info.si_code = BUS_ADRALN;
+		info.si_addr = (void __user *)address;
+		force_sig_info(SIGBUS, &info, current);
 	} else {
 		printk("KERNEL: Unaligned Access 0x%.8lx\n", address);
 		show_registers(regs);
diff --git a/arch/openrisc/kernel/vmlinux.lds.S b/arch/openrisc/kernel/vmlinux.lds.S
index 00ddb7804be4..953bdcd54efe 100644
--- a/arch/openrisc/kernel/vmlinux.lds.S
+++ b/arch/openrisc/kernel/vmlinux.lds.S
@@ -28,6 +28,7 @@
 
 #include <asm/page.h>
 #include <asm/cache.h>
+#include <asm/thread_info.h>
 #include <asm-generic/vmlinux.lds.h>
 
 #ifdef __OR1K__
diff --git a/arch/parisc/boot/compressed/vmlinux.lds.S b/arch/parisc/boot/compressed/vmlinux.lds.S
index a4ce3314e78e..4ebd4e65524c 100644
--- a/arch/parisc/boot/compressed/vmlinux.lds.S
+++ b/arch/parisc/boot/compressed/vmlinux.lds.S
@@ -29,7 +29,9 @@ SECTIONS
 	. = ALIGN(16);
 	/* Linkage tables */
 	.opd : {
+		__start_opd = .;
 		*(.opd)
+		__end_opd = .;
 	} PROVIDE (__gp = .);
 	.plt : {
 		*(.plt)
diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild
index a41139575ab4..2013d639e735 100644
--- a/arch/parisc/include/asm/Kbuild
+++ b/arch/parisc/include/asm/Kbuild
@@ -1,5 +1,4 @@
 generic-y += barrier.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h
index 3742508cc534..bd5ce31936f5 100644
--- a/arch/parisc/include/asm/cacheflush.h
+++ b/arch/parisc/include/asm/cacheflush.h
@@ -26,6 +26,7 @@ void flush_user_icache_range_asm(unsigned long, unsigned long);
 void flush_kernel_icache_range_asm(unsigned long, unsigned long);
 void flush_user_dcache_range_asm(unsigned long, unsigned long);
 void flush_kernel_dcache_range_asm(unsigned long, unsigned long);
+void purge_kernel_dcache_range_asm(unsigned long, unsigned long);
 void flush_kernel_dcache_page_asm(void *);
 void flush_kernel_icache_page(void *);
 
diff --git a/arch/parisc/include/asm/compat.h b/arch/parisc/include/asm/compat.h
index acf8aa07cbe0..c22db5323244 100644
--- a/arch/parisc/include/asm/compat.h
+++ b/arch/parisc/include/asm/compat.h
@@ -130,70 +130,6 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32		compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[128/sizeof(int) - 3];
-
-		/* kill() */
-		struct {
-			unsigned int _pid;      /* sender's pid */
-			unsigned int _uid;      /* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;            /* timer id */
-			int _overrun;           /* overrun count */
-			char _pad[sizeof(unsigned int) - sizeof(int)];
-			compat_sigval_t _sigval;        /* same as below */
-			int _sys_private;       /* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			unsigned int _pid;      /* sender's pid */
-			unsigned int _uid;      /* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			unsigned int _pid;      /* which child */
-			unsigned int _uid;      /* sender's uid */
-			int _status;            /* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			unsigned int _addr;     /* faulting insn/memory ref. */
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;      /* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		/* SIGSYS */
-		struct {
-			compat_uptr_t _call_addr; /* calling user insn */
-			int _syscall;	/* triggering system call number */
-			compat_uint_t _arch;	/* AUDIT_ARCH_* of syscall */
-		} _sigsys;
-	} _sifields;
-} compat_siginfo_t;
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 struct compat_ipc64_perm {
diff --git a/arch/parisc/include/asm/processor.h b/arch/parisc/include/asm/processor.h
index 0e6ab6e4a4e9..2dbe5580a1a4 100644
--- a/arch/parisc/include/asm/processor.h
+++ b/arch/parisc/include/asm/processor.h
@@ -316,6 +316,8 @@ extern int _parisc_requires_coherency;
 #define parisc_requires_coherency()	(0)
 #endif
 
+extern int running_on_qemu;
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* __ASM_PARISC_PROCESSOR_H */
diff --git a/arch/parisc/include/asm/sections.h b/arch/parisc/include/asm/sections.h
index accdf40aa5b7..5a40b51df80c 100644
--- a/arch/parisc/include/asm/sections.h
+++ b/arch/parisc/include/asm/sections.h
@@ -6,8 +6,14 @@
 #include <asm-generic/sections.h>
 
 #ifdef CONFIG_64BIT
+
+#define HAVE_DEREFERENCE_FUNCTION_DESCRIPTOR 1
+
 #undef dereference_function_descriptor
 void *dereference_function_descriptor(void *);
+
+#undef dereference_kernel_function_descriptor
+void *dereference_kernel_function_descriptor(void *);
 #endif
 
 #endif
diff --git a/arch/parisc/include/asm/thread_info.h b/arch/parisc/include/asm/thread_info.h
index 598c8d60fa5e..285757544cca 100644
--- a/arch/parisc/include/asm/thread_info.h
+++ b/arch/parisc/include/asm/thread_info.h
@@ -25,9 +25,6 @@ struct thread_info {
 	.preempt_count	= INIT_PREEMPT_COUNT,	\
 }
 
-#define init_thread_info        (init_thread_union.thread_info)
-#define init_stack              (init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 #define current_thread_info()	((struct thread_info *)mfctl(30))
 
diff --git a/arch/parisc/include/uapi/asm/siginfo.h b/arch/parisc/include/uapi/asm/siginfo.h
index 4a1062e05aaf..be40331f757d 100644
--- a/arch/parisc/include/uapi/asm/siginfo.h
+++ b/arch/parisc/include/uapi/asm/siginfo.h
@@ -8,4 +8,11 @@
 
 #include <asm-generic/siginfo.h>
 
+/*
+ * SIGFPE si_codes
+ */
+#ifdef __KERNEL__
+#define FPE_FIXME	0	/* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
 #endif
diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c
index 19c0c141bc3f..e3b45546d589 100644
--- a/arch/parisc/kernel/cache.c
+++ b/arch/parisc/kernel/cache.c
@@ -465,10 +465,10 @@ EXPORT_SYMBOL(copy_user_page);
 int __flush_tlb_range(unsigned long sid, unsigned long start,
 		      unsigned long end)
 {
-	unsigned long flags, size;
+	unsigned long flags;
 
-	size = (end - start);
-	if (size >= parisc_tlb_flush_threshold) {
+	if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
+	    end - start >= parisc_tlb_flush_threshold) {
 		flush_tlb_all();
 		return 1;
 	}
@@ -539,13 +539,12 @@ void flush_cache_mm(struct mm_struct *mm)
 	struct vm_area_struct *vma;
 	pgd_t *pgd;
 
-	/* Flush the TLB to avoid speculation if coherency is required. */
-	if (parisc_requires_coherency())
-		flush_tlb_all();
-
 	/* Flushing the whole cache on each cpu takes forever on
 	   rp3440, etc.  So, avoid it if the mm isn't too big.  */
-	if (mm_total_size(mm) >= parisc_cache_flush_threshold) {
+	if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
+	    mm_total_size(mm) >= parisc_cache_flush_threshold) {
+		if (mm->context)
+			flush_tlb_all();
 		flush_cache_all();
 		return;
 	}
@@ -553,9 +552,9 @@ void flush_cache_mm(struct mm_struct *mm)
 	if (mm->context == mfsp(3)) {
 		for (vma = mm->mmap; vma; vma = vma->vm_next) {
 			flush_user_dcache_range_asm(vma->vm_start, vma->vm_end);
-			if ((vma->vm_flags & VM_EXEC) == 0)
-				continue;
-			flush_user_icache_range_asm(vma->vm_start, vma->vm_end);
+			if (vma->vm_flags & VM_EXEC)
+				flush_user_icache_range_asm(vma->vm_start, vma->vm_end);
+			flush_tlb_range(vma, vma->vm_start, vma->vm_end);
 		}
 		return;
 	}
@@ -573,6 +572,8 @@ void flush_cache_mm(struct mm_struct *mm)
 			pfn = pte_pfn(*ptep);
 			if (!pfn_valid(pfn))
 				continue;
+			if (unlikely(mm->context))
+				flush_tlb_page(vma, addr);
 			__flush_cache_page(vma, addr, PFN_PHYS(pfn));
 		}
 	}
@@ -581,30 +582,45 @@ void flush_cache_mm(struct mm_struct *mm)
 void flush_cache_range(struct vm_area_struct *vma,
 		unsigned long start, unsigned long end)
 {
-	BUG_ON(!vma->vm_mm->context);
-
-	/* Flush the TLB to avoid speculation if coherency is required. */
-	if (parisc_requires_coherency())
-		flush_tlb_range(vma, start, end);
+	pgd_t *pgd;
+	unsigned long addr;
 
-	if ((end - start) >= parisc_cache_flush_threshold
-	    || vma->vm_mm->context != mfsp(3)) {
+	if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
+	    end - start >= parisc_cache_flush_threshold) {
+		if (vma->vm_mm->context)
+			flush_tlb_range(vma, start, end);
 		flush_cache_all();
 		return;
 	}
 
-	flush_user_dcache_range_asm(start, end);
-	if (vma->vm_flags & VM_EXEC)
-		flush_user_icache_range_asm(start, end);
+	if (vma->vm_mm->context == mfsp(3)) {
+		flush_user_dcache_range_asm(start, end);
+		if (vma->vm_flags & VM_EXEC)
+			flush_user_icache_range_asm(start, end);
+		flush_tlb_range(vma, start, end);
+		return;
+	}
+
+	pgd = vma->vm_mm->pgd;
+	for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
+		unsigned long pfn;
+		pte_t *ptep = get_ptep(pgd, addr);
+		if (!ptep)
+			continue;
+		pfn = pte_pfn(*ptep);
+		if (pfn_valid(pfn)) {
+			if (unlikely(vma->vm_mm->context))
+				flush_tlb_page(vma, addr);
+			__flush_cache_page(vma, addr, PFN_PHYS(pfn));
+		}
+	}
 }
 
 void
 flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn)
 {
-	BUG_ON(!vma->vm_mm->context);
-
 	if (pfn_valid(pfn)) {
-		if (parisc_requires_coherency())
+		if (likely(vma->vm_mm->context))
 			flush_tlb_page(vma, vmaddr);
 		__flush_cache_page(vma, vmaddr, PFN_PHYS(pfn));
 	}
@@ -613,21 +629,33 @@ flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long
 void flush_kernel_vmap_range(void *vaddr, int size)
 {
 	unsigned long start = (unsigned long)vaddr;
+	unsigned long end = start + size;
 
-	if ((unsigned long)size > parisc_cache_flush_threshold)
+	if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
+	    (unsigned long)size >= parisc_cache_flush_threshold) {
+		flush_tlb_kernel_range(start, end);
 		flush_data_cache();
-	else
-		flush_kernel_dcache_range_asm(start, start + size);
+		return;
+	}
+
+	flush_kernel_dcache_range_asm(start, end);
+	flush_tlb_kernel_range(start, end);
 }
 EXPORT_SYMBOL(flush_kernel_vmap_range);
 
 void invalidate_kernel_vmap_range(void *vaddr, int size)
 {
 	unsigned long start = (unsigned long)vaddr;
+	unsigned long end = start + size;
 
-	if ((unsigned long)size > parisc_cache_flush_threshold)
+	if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) &&
+	    (unsigned long)size >= parisc_cache_flush_threshold) {
+		flush_tlb_kernel_range(start, end);
 		flush_data_cache();
-	else
-		flush_kernel_dcache_range_asm(start, start + size);
+		return;
+	}
+
+	purge_kernel_dcache_range_asm(start, end);
+	flush_tlb_kernel_range(start, end);
 }
 EXPORT_SYMBOL(invalidate_kernel_vmap_range);
diff --git a/arch/parisc/kernel/head.S b/arch/parisc/kernel/head.S
index bbbe360b458f..fbb4e43fda05 100644
--- a/arch/parisc/kernel/head.S
+++ b/arch/parisc/kernel/head.S
@@ -138,6 +138,16 @@ $pgt_fill_loop:
 	std		%dp,0x18(%r10)
 #endif
 
+#ifdef CONFIG_64BIT
+	/* Get PDCE_PROC for monarch CPU. */
+#define MEM_PDC_LO 0x388
+#define MEM_PDC_HI 0x35C
+	ldw             MEM_PDC_LO(%r0),%r3
+	ldw             MEM_PDC_HI(%r0),%r10
+	depd            %r10, 31, 32, %r3        /* move to upper word */
+#endif
+
+
 #ifdef CONFIG_SMP
 	/* Set the smp rendezvous address into page zero.
 	** It would be safer to do this in init_smp_config() but
@@ -196,12 +206,6 @@ common_stext:
         ** Someday, palo might not do this for the Monarch either.
         */
 2:
-#define MEM_PDC_LO 0x388
-#define MEM_PDC_HI 0x35C
-	ldw             MEM_PDC_LO(%r0),%r3
-	ldw             MEM_PDC_HI(%r0),%r6
-	depd            %r6, 31, 32, %r3        /* move to upper word */
-
 	mfctl		%cr30,%r6		/* PCX-W2 firmware bug */
 
 	ldo             PDC_PSW(%r0),%arg0              /* 21 */
@@ -268,6 +272,8 @@ $install_iva:
 aligned_rfi:
 	pcxt_ssm_bug
 
+	copy		%r3, %arg0	/* PDCE_PROC for smp_callin() */
+
 	rsm		PSW_SM_QUIET,%r0	/* off troublesome PSW bits */
 	/* Don't need NOPs, have 8 compliant insn before rfi */
 
diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c
index f1a76935a314..b5b3cb00f1fb 100644
--- a/arch/parisc/kernel/module.c
+++ b/arch/parisc/kernel/module.c
@@ -66,6 +66,7 @@
 
 #include <asm/pgtable.h>
 #include <asm/unwind.h>
+#include <asm/sections.h>
 
 #if 0
 #define DEBUGP printk
@@ -954,3 +955,18 @@ void module_arch_cleanup(struct module *mod)
 {
 	deregister_unwind_table(mod);
 }
+
+#ifdef CONFIG_64BIT
+void *dereference_module_function_descriptor(struct module *mod, void *ptr)
+{
+	unsigned long start_opd = (Elf64_Addr)mod->core_layout.base +
+				   mod->arch.fdesc_offset;
+	unsigned long end_opd = start_opd +
+				mod->arch.fdesc_count * sizeof(Elf64_Fdesc);
+
+	if (ptr < (void *)start_opd || ptr >= (void *)end_opd)
+		return ptr;
+
+	return dereference_function_descriptor(ptr);
+}
+#endif
diff --git a/arch/parisc/kernel/pacache.S b/arch/parisc/kernel/pacache.S
index 2d40c4ff3f69..67b0f7532e83 100644
--- a/arch/parisc/kernel/pacache.S
+++ b/arch/parisc/kernel/pacache.S
@@ -1110,6 +1110,28 @@ ENTRY_CFI(flush_kernel_dcache_range_asm)
 	.procend
 ENDPROC_CFI(flush_kernel_dcache_range_asm)
 
+ENTRY_CFI(purge_kernel_dcache_range_asm)
+	.proc
+	.callinfo NO_CALLS
+	.entry
+
+	ldil		L%dcache_stride, %r1
+	ldw		R%dcache_stride(%r1), %r23
+	ldo		-1(%r23), %r21
+	ANDCM		%r26, %r21, %r26
+
+1:      cmpb,COND(<<),n	%r26, %r25,1b
+	pdc,m		%r23(%r26)
+
+	sync
+	syncdma
+	bv		%r0(%r2)
+	nop
+	.exit
+
+	.procend
+ENDPROC_CFI(purge_kernel_dcache_range_asm)
+
 ENTRY_CFI(flush_user_icache_range_asm)
 	.proc
 	.callinfo NO_CALLS
diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c
index c0dfd892f70c..91bc0cac03a1 100644
--- a/arch/parisc/kernel/pci-dma.c
+++ b/arch/parisc/kernel/pci-dma.c
@@ -75,11 +75,6 @@ void dump_resmap(void)
 static inline void dump_resmap(void) {;}
 #endif
 
-static int pa11_dma_supported( struct device *dev, u64 mask)
-{
-	return 1;
-}
-
 static inline int map_pte_uncached(pte_t * pte,
 		unsigned long vaddr,
 		unsigned long size, unsigned long *paddr_ptr)
@@ -579,7 +574,6 @@ static void pa11_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 }
 
 const struct dma_map_ops pcxl_dma_ops = {
-	.dma_supported =	pa11_dma_supported,
 	.alloc =		pa11_dma_alloc,
 	.free =			pa11_dma_free,
 	.map_page =		pa11_dma_map_page,
@@ -616,7 +610,6 @@ static void pcx_dma_free(struct device *dev, size_t size, void *vaddr,
 }
 
 const struct dma_map_ops pcx_dma_ops = {
-	.dma_supported =	pa11_dma_supported,
 	.alloc =		pcx_dma_alloc,
 	.free =			pcx_dma_free,
 	.map_page =		pa11_dma_map_page,
diff --git a/arch/parisc/kernel/pdt.c b/arch/parisc/kernel/pdt.c
index e07eb34c8750..36434d4da381 100644
--- a/arch/parisc/kernel/pdt.c
+++ b/arch/parisc/kernel/pdt.c
@@ -325,7 +325,7 @@ static int pdt_mainloop(void *unused)
 #ifdef CONFIG_MEMORY_FAILURE
 			if ((pde & PDT_ADDR_PERM_ERR) ||
 			    ((pde & PDT_ADDR_SINGLE_ERR) == 0))
-				memory_failure(pde >> PAGE_SHIFT, 0, 0);
+				memory_failure(pde >> PAGE_SHIFT, 0);
 			else
 				soft_offline_page(
 					pfn_to_page(pde >> PAGE_SHIFT), 0);
diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c
index cad3e8661cd6..6975a0627078 100644
--- a/arch/parisc/kernel/process.c
+++ b/arch/parisc/kernel/process.c
@@ -315,6 +315,15 @@ void *dereference_function_descriptor(void *ptr)
 		ptr = p;
 	return ptr;
 }
+
+void *dereference_kernel_function_descriptor(void *ptr)
+{
+	if (ptr < (void *)__start_opd ||
+			ptr >= (void *)__end_opd)
+		return ptr;
+
+	return dereference_function_descriptor(ptr);
+}
 #endif
 
 static inline unsigned long brk_rnd(void)
diff --git a/arch/parisc/kernel/signal32.c b/arch/parisc/kernel/signal32.c
index 41afa9cd1f55..e8ef3eb69449 100644
--- a/arch/parisc/kernel/signal32.c
+++ b/arch/parisc/kernel/signal32.c
@@ -260,109 +260,3 @@ setup_sigcontext32(struct compat_sigcontext __user *sc, struct compat_regfile __
 
 	return err;
 }
-
-int
-copy_siginfo_from_user32 (siginfo_t *to, compat_siginfo_t __user *from)
-{
-	compat_uptr_t addr;
-	int err;
-
-	if (!access_ok(VERIFY_READ, from, sizeof(compat_siginfo_t)))
-		return -EFAULT;
-
-	err = __get_user(to->si_signo, &from->si_signo);
-	err |= __get_user(to->si_errno, &from->si_errno);
-	err |= __get_user(to->si_code, &from->si_code);
-
-	if (to->si_code < 0)
-		err |= __copy_from_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE);
-	else {
-		switch (siginfo_layout(to->si_signo, to->si_code)) {
-		      case SIL_CHLD:
-			err |= __get_user(to->si_utime, &from->si_utime);
-			err |= __get_user(to->si_stime, &from->si_stime);
-			err |= __get_user(to->si_status, &from->si_status);
-		      default:
-		      case SIL_KILL:
-			err |= __get_user(to->si_pid, &from->si_pid);
-			err |= __get_user(to->si_uid, &from->si_uid);
-			break;
-		      case SIL_FAULT:
-			err |= __get_user(addr, &from->si_addr);
-			to->si_addr = compat_ptr(addr);
-			break;
-		      case SIL_POLL:
-			err |= __get_user(to->si_band, &from->si_band);
-			err |= __get_user(to->si_fd, &from->si_fd);
-			break;
-		      case SIL_RT:
-			err |= __get_user(to->si_pid, &from->si_pid);
-			err |= __get_user(to->si_uid, &from->si_uid);
-			err |= __get_user(to->si_int, &from->si_int);
-			break;
-		}
-	}
-	return err;
-}
-
-int
-copy_siginfo_to_user32 (compat_siginfo_t __user *to, const siginfo_t *from)
-{
-	compat_uptr_t addr;
-	compat_int_t val;
-	int err;
-
-	if (!access_ok(VERIFY_WRITE, to, sizeof(compat_siginfo_t)))
-		return -EFAULT;
-
-	/* If you change siginfo_t structure, please be sure
-	   this code is fixed accordingly.
-	   It should never copy any pad contained in the structure
-	   to avoid security leaks, but must copy the generic
-	   3 ints plus the relevant union member.
-	   This routine must convert siginfo from 64bit to 32bit as well
-	   at the same time.  */
-	err = __put_user(from->si_signo, &to->si_signo);
-	err |= __put_user(from->si_errno, &to->si_errno);
-	err |= __put_user(from->si_code, &to->si_code);
-	if (from->si_code < 0)
-		err |= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE);
-	else {
-		switch (siginfo_layout(from->si_signo, from->si_code)) {
-		case SIL_CHLD:
-			err |= __put_user(from->si_utime, &to->si_utime);
-			err |= __put_user(from->si_stime, &to->si_stime);
-			err |= __put_user(from->si_status, &to->si_status);
-		case SIL_KILL:
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_uid, &to->si_uid);
-			break;
-		case SIL_FAULT:
-			addr = ptr_to_compat(from->si_addr);
-			err |= __put_user(addr, &to->si_addr);
-			break;
-		case SIL_POLL:
-			err |= __put_user(from->si_band, &to->si_band);
-			err |= __put_user(from->si_fd, &to->si_fd);
-			break;
-		case SIL_TIMER:
-			err |= __put_user(from->si_tid, &to->si_tid);
-			err |= __put_user(from->si_overrun, &to->si_overrun);
-			val = (compat_int_t)from->si_int;
-			err |= __put_user(val, &to->si_int);
-			break;
-		case SIL_RT:
-			err |= __put_user(from->si_uid, &to->si_uid);
-			err |= __put_user(from->si_pid, &to->si_pid);
-			val = (compat_int_t)from->si_int;
-			err |= __put_user(val, &to->si_int);
-			break;
-		case SIL_SYS:
-			err |= __put_user(ptr_to_compat(from->si_call_addr), &to->si_call_addr);
-			err |= __put_user(from->si_syscall, &to->si_syscall);
-			err |= __put_user(from->si_arch, &to->si_arch);
-			break;
-		}
-	}
-	return err;
-}
diff --git a/arch/parisc/kernel/signal32.h b/arch/parisc/kernel/signal32.h
index 719e7417732c..a271dc0976ce 100644
--- a/arch/parisc/kernel/signal32.h
+++ b/arch/parisc/kernel/signal32.h
@@ -34,9 +34,6 @@ struct compat_ucontext {
 
 /* ELF32 signal handling */
 
-int copy_siginfo_to_user32 (compat_siginfo_t __user *to, const siginfo_t *from);
-int copy_siginfo_from_user32 (siginfo_t *to, compat_siginfo_t __user *from);
-
 /* In a deft move of uber-hackery, we decide to carry the top half of all
  * 64-bit registers in a non-portable, non-ABI, hidden structure.
  * Userspace can read the hidden structure if it *wants* but is never
diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c
index 30c28ab14540..4065b5e48c9d 100644
--- a/arch/parisc/kernel/smp.c
+++ b/arch/parisc/kernel/smp.c
@@ -292,10 +292,15 @@ smp_cpu_init(int cpunum)
  * Slaves start using C here. Indirectly called from smp_slave_stext.
  * Do what start_kernel() and main() do for boot strap processor (aka monarch)
  */
-void __init smp_callin(void)
+void __init smp_callin(unsigned long pdce_proc)
 {
 	int slave_id = cpu_now_booting;
 
+#ifdef CONFIG_64BIT
+	WARN_ON(((unsigned long)(PAGE0->mem_pdc_hi) << 32
+			| PAGE0->mem_pdc) != pdce_proc);
+#endif
+
 	smp_cpu_init(slave_id);
 	preempt_disable();
 
diff --git a/arch/parisc/kernel/time.c b/arch/parisc/kernel/time.c
index 4b8fd6dc22da..f7e684560186 100644
--- a/arch/parisc/kernel/time.c
+++ b/arch/parisc/kernel/time.c
@@ -76,10 +76,10 @@ irqreturn_t __irq_entry timer_interrupt(int irq, void *dev_id)
 	next_tick = cpuinfo->it_value;
 
 	/* Calculate how many ticks have elapsed. */
+	now = mfctl(16);
 	do {
 		++ticks_elapsed;
 		next_tick += cpt;
-		now = mfctl(16);
 	} while (next_tick - now > cpt);
 
 	/* Store (in CR16 cycles) up to when we are accounting right now. */
@@ -103,16 +103,17 @@ irqreturn_t __irq_entry timer_interrupt(int irq, void *dev_id)
 	 * if one or the other wrapped. If "now" is "bigger" we'll end up
 	 * with a very large unsigned number.
 	 */
-	while (next_tick - mfctl(16) > cpt)
+	now = mfctl(16);
+	while (next_tick - now > cpt)
 		next_tick += cpt;
 
 	/* Program the IT when to deliver the next interrupt.
 	 * Only bottom 32-bits of next_tick are writable in CR16!
 	 * Timer interrupt will be delivered at least a few hundred cycles
-	 * after the IT fires, so if we are too close (<= 500 cycles) to the
+	 * after the IT fires, so if we are too close (<= 8000 cycles) to the
 	 * next cycle, simply skip it.
 	 */
-	if (next_tick - mfctl(16) <= 500)
+	if (next_tick - now <= 8000)
 		next_tick += cpt;
 	mtctl(next_tick, 16);
 
@@ -248,7 +249,7 @@ static int __init init_cr16_clocksource(void)
 	 * different sockets, so mark them unstable and lower rating on
 	 * multi-socket SMP systems.
 	 */
-	if (num_online_cpus() > 1) {
+	if (num_online_cpus() > 1 && !running_on_qemu) {
 		int cpu;
 		unsigned long cpu0_loc;
 		cpu0_loc = per_cpu(cpu_data, 0).cpu_loc;
diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c
index 8453724b8009..c919e6c0a687 100644
--- a/arch/parisc/kernel/traps.c
+++ b/arch/parisc/kernel/traps.c
@@ -629,7 +629,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
 			si.si_signo = SIGFPE;
 			/* Set to zero, and let the userspace app figure it out from
 			   the insn pointed to by si_addr */
-			si.si_code = 0;
+			si.si_code = FPE_FIXME;
 			si.si_addr = (void __user *) regs->iaoq[0];
 			force_sig_info(SIGFPE, &si, current);
 			return;
diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S
index 159a2ec0b4e0..da2e31190efa 100644
--- a/arch/parisc/kernel/vmlinux.lds.S
+++ b/arch/parisc/kernel/vmlinux.lds.S
@@ -100,7 +100,9 @@ SECTIONS
 	. = ALIGN(16);
 	/* Linkage tables */
 	.opd : {
+		__start_opd = .;
 		*(.opd)
+		__end_opd = .;
 	} PROVIDE (__gp = .);
 	.plt : {
 		*(.plt)
diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c
index 48f41399fc0b..cab32ee824d2 100644
--- a/arch/parisc/mm/init.c
+++ b/arch/parisc/mm/init.c
@@ -629,7 +629,12 @@ void __init mem_init(void)
 #endif
 
 	mem_init_print_info(NULL);
-#ifdef CONFIG_DEBUG_KERNEL /* double-sanity-check paranoia */
+
+#if 0
+	/*
+	 * Do not expose the virtual kernel memory layout to userspace.
+	 * But keep code for debugging purposes.
+	 */
 	printk("virtual kernel memory layout:\n"
 	       "    vmalloc : 0x%px - 0x%px   (%4ld MB)\n"
 	       "    memory  : 0x%px - 0x%px   (%4ld MB)\n"
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 2ed525a44734..73ce5dd07642 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -139,9 +139,12 @@ config PPC
 	select ARCH_HAS_ELF_RANDOMIZE
 	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_GCOV_PROFILE_ALL
+	select ARCH_HAS_PHYS_TO_DMA
 	select ARCH_HAS_PMEM_API                if PPC64
+	select ARCH_HAS_MEMBARRIER_CALLBACKS
 	select ARCH_HAS_SCALED_CPUTIME		if VIRT_CPU_ACCOUNTING_NATIVE
 	select ARCH_HAS_SG_CHAIN
+	select ARCH_HAS_STRICT_KERNEL_RWX	if ((PPC_BOOK3S_64 || PPC32) && !RELOCATABLE && !HIBERNATION)
 	select ARCH_HAS_TICK_BROADCAST		if GENERIC_CLOCKEVENTS_BROADCAST
 	select ARCH_HAS_UACCESS_FLUSHCACHE	if PPC64
 	select ARCH_HAS_UBSAN_SANITIZE_ALL
@@ -149,8 +152,8 @@ config PPC
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
 	select ARCH_MIGHT_HAVE_PC_PARPORT
 	select ARCH_MIGHT_HAVE_PC_SERIO
+	select ARCH_OPTIONAL_KERNEL_RWX		if ARCH_HAS_STRICT_KERNEL_RWX
 	select ARCH_SUPPORTS_ATOMIC_RMW
-	select ARCH_SUPPORTS_DEFERRED_STRUCT_PAGE_INIT
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_CMPXCHG_LOCKREF		if PPC64
 	select ARCH_WANT_IPC_PARSE_VERSION
@@ -180,8 +183,6 @@ config PPC
 	select HAVE_ARCH_MMAP_RND_COMPAT_BITS	if COMPAT
 	select HAVE_ARCH_SECCOMP_FILTER
 	select HAVE_ARCH_TRACEHOOK
-	select ARCH_HAS_STRICT_KERNEL_RWX	if ((PPC_BOOK3S_64 || PPC32) && !RELOCATABLE && !HIBERNATION)
-	select ARCH_OPTIONAL_KERNEL_RWX		if ARCH_HAS_STRICT_KERNEL_RWX
 	select HAVE_CBPF_JIT			if !PPC64
 	select HAVE_CONTEXT_TRACKING		if PPC64
 	select HAVE_DEBUG_KMEMLEAK
@@ -868,6 +869,21 @@ config SECCOMP
 
 	  If unsure, say Y. Only embedded should say N here.
 
+config PPC_MEM_KEYS
+	prompt "PowerPC Memory Protection Keys"
+	def_bool y
+	depends on PPC_BOOK3S_64
+	select ARCH_USES_HIGH_VMA_FLAGS
+	select ARCH_HAS_PKEYS
+	help
+	  Memory Protection Keys provides a mechanism for enforcing
+	  page-based protections, but without requiring modification of the
+	  page tables when an application changes protection domains.
+
+	  For details, see Documentation/vm/protection-keys.txt
+
+	  If unsure, say y.
+
 endmenu
 
 config ISA_DMA_API
diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug
index 657c33cd4eee..c45424c64e19 100644
--- a/arch/powerpc/Kconfig.debug
+++ b/arch/powerpc/Kconfig.debug
@@ -90,6 +90,10 @@ config MSI_BITMAP_SELFTEST
 	depends on DEBUG_KERNEL
 	default n
 
+config PPC_IRQ_SOFT_MASK_DEBUG
+	bool "Include extra checks for powerpc irq soft masking"
+	default n
+
 config XMON
 	bool "Include xmon kernel debugger"
 	depends on DEBUG_KERNEL
@@ -368,7 +372,7 @@ config PPC_PTDUMP
 
 config PPC_HTDUMP
 	def_bool y
-	depends on PPC_PTDUMP && PPC_BOOK3S
+	depends on PPC_PTDUMP && PPC_BOOK3S_64
 
 config PPC_FAST_ENDIAN_SWITCH
 	bool "Deprecated fast endian-switch syscall"
diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
index 1381693a4a51..ccd2556bdb53 100644
--- a/arch/powerpc/Makefile
+++ b/arch/powerpc/Makefile
@@ -63,6 +63,7 @@ UTS_MACHINE := $(subst $(space),,$(machine-y))
 ifdef CONFIG_PPC32
 KBUILD_LDFLAGS_MODULE += arch/powerpc/lib/crtsavres.o
 else
+KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/powerpc/kernel/module.lds
 ifeq ($(call ld-ifversion, -ge, 225000000, y),y)
 # Have the linker provide sfpr if possible.
 # There is a corresponding test in arch/powerpc/lib/Makefile
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index 08782f55b89f..26d5d2a5b8e9 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -101,17 +101,18 @@ $(addprefix $(obj)/,$(zlib-y)): \
 libfdt       := fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c
 libfdtheader := fdt.h libfdt.h libfdt_internal.h
 
-$(addprefix $(obj)/,$(libfdt) libfdt-wrapper.o simpleboot.o epapr.o opal.o): \
+$(addprefix $(obj)/,$(libfdt) libfdt-wrapper.o simpleboot.o epapr.o opal.o \
+	treeboot-akebono.o treeboot-currituck.o treeboot-iss4xx.o): \
 	$(addprefix $(obj)/,$(libfdtheader))
 
 src-wlib-y := string.S crt0.S stdio.c decompress.c main.c \
 		$(libfdt) libfdt-wrapper.c \
 		ns16550.c serial.c simple_alloc.c div64.S util.S \
 		elf_util.c $(zlib-y) devtree.c stdlib.c \
-		oflib.c ofconsole.c cuboot.c cpm-serial.c \
-		uartlite.c opal.c
+		oflib.c ofconsole.c cuboot.c
+
 src-wlib-$(CONFIG_PPC_MPC52XX) += mpc52xx-psc.c
-src-wlib-$(CONFIG_PPC64_BOOT_WRAPPER) +=  opal-calls.S
+src-wlib-$(CONFIG_PPC64_BOOT_WRAPPER) += opal-calls.S opal.c
 ifndef CONFIG_PPC64_BOOT_WRAPPER
 src-wlib-y += crtsavres.S
 endif
@@ -120,6 +121,8 @@ src-wlib-$(CONFIG_44x) += 4xx.c ebony.c bamboo.c
 src-wlib-$(CONFIG_PPC_8xx) += mpc8xx.c planetcore.c fsl-soc.c
 src-wlib-$(CONFIG_PPC_82xx) += pq2.c fsl-soc.c planetcore.c
 src-wlib-$(CONFIG_EMBEDDED6xx) += mpsc.c mv64x60.c mv64x60_i2c.c ugecon.c fsl-soc.c
+src-wlib-$(CONFIG_XILINX_VIRTEX) += uartlite.c
+src-wlib-$(CONFIG_CPM) += cpm-serial.c
 
 src-plat-y := of.c epapr.c
 src-plat-$(CONFIG_40x) += fixed-head.S ep405.c cuboot-hotfoot.c \
diff --git a/arch/powerpc/boot/dts/a3m071.dts b/arch/powerpc/boot/dts/a3m071.dts
index bf81b8f9704c..187ce458d03a 100644
--- a/arch/powerpc/boot/dts/a3m071.dts
+++ b/arch/powerpc/boot/dts/a3m071.dts
@@ -105,24 +105,24 @@
 			reg = <0 0x0 0x02000000>;
 			compatible = "cfi-flash";
 			bank-width = <2>;
-			partition@0x0 {
+			partition@0 {
 				label = "u-boot";
 				reg = <0x00000000 0x00040000>;
 				read-only;
 			};
-			partition@0x00040000 {
+			partition@40000 {
 				label = "env";
 				reg = <0x00040000 0x00020000>;
 			};
-			partition@0x00060000 {
+			partition@60000 {
 				label = "dtb";
 				reg = <0x00060000 0x00020000>;
 			};
-			partition@0x00080000 {
+			partition@80000 {
 				label = "kernel";
 				reg = <0x00080000 0x00500000>;
 			};
-			partition@0x00580000 {
+			partition@580000 {
 				label = "root";
 				reg = <0x00580000 0x00A80000>;
 			};
diff --git a/arch/powerpc/boot/dts/akebono.dts b/arch/powerpc/boot/dts/akebono.dts
index e61d5dc598c1..746779202a12 100644
--- a/arch/powerpc/boot/dts/akebono.dts
+++ b/arch/powerpc/boot/dts/akebono.dts
@@ -216,7 +216,7 @@
 				interrupts = <39 2>;
 			};
 
-			IIC0: i2c@00000000 {
+			IIC0: i2c@0 {
 				compatible = "ibm,iic-476gtr", "ibm,iic";
 				reg = <0x0 0x00000020>;
 				interrupt-parent = <&MPIC>;
@@ -229,7 +229,7 @@
 				};
 			};
 
-			IIC1: i2c@00000100 {
+			IIC1: i2c@100 {
 				compatible = "ibm,iic-476gtr", "ibm,iic";
 				reg = <0x100 0x00000020>;
 				interrupt-parent = <&MPIC>;
diff --git a/arch/powerpc/boot/dts/c2k.dts b/arch/powerpc/boot/dts/c2k.dts
index 1e32903cb0a8..27f169e3ade9 100644
--- a/arch/powerpc/boot/dts/c2k.dts
+++ b/arch/powerpc/boot/dts/c2k.dts
@@ -276,14 +276,14 @@
 			>;
 		};
 
-		cpu-error@0070 {
+		cpu-error@70 {
 			compatible = "marvell,mv64360-cpu-error";
 			reg = <0x0070 0x10 0x0128 0x28>;
 			interrupts = <3>;
 			interrupt-parent = <&PIC>;
 		};
 
-		sram-ctrl@0380 {
+		sram-ctrl@380 {
 			compatible = "marvell,mv64360-sram-ctrl";
 			reg = <0x0380 0x80>;
 			interrupts = <13>;
@@ -311,7 +311,7 @@
 			interrupt-parent = <&PIC>;
 		};
 		/* Devices attached to the device controller */
-		devicebus@045c {
+		devicebus@45c {
 			#address-cells = <2>;
 			#size-cells = <1>;
 			compatible = "marvell,mv64306-devctrl";
diff --git a/arch/powerpc/boot/dts/currituck.dts b/arch/powerpc/boot/dts/currituck.dts
index 4191e1850ea1..f2ad5815f08d 100644
--- a/arch/powerpc/boot/dts/currituck.dts
+++ b/arch/powerpc/boot/dts/currituck.dts
@@ -108,7 +108,7 @@
 				reg = <0x50000000 0x4>;
 			};
 
-			IIC0: i2c@00000000 {
+			IIC0: i2c@0 {
 				compatible = "ibm,iic-currituck", "ibm,iic";
 				reg = <0x0 0x00000014>;
 				interrupt-parent = <&MPIC>;
diff --git a/arch/powerpc/boot/dts/fsl/mpc8568mds.dts b/arch/powerpc/boot/dts/fsl/mpc8568mds.dts
index 01706a339603..bc3e8039bdc7 100644
--- a/arch/powerpc/boot/dts/fsl/mpc8568mds.dts
+++ b/arch/powerpc/boot/dts/fsl/mpc8568mds.dts
@@ -126,7 +126,7 @@
 		par_io@e0100 {
 			num-ports = <7>;
 
-			pio1: ucc_pin@01 {
+			pio1: ucc_pin@1 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x4  0xa  0x1  0x0  0x2  0x0 	/* TxD0 */
@@ -154,7 +154,7 @@
 					0x1  0x1f  0x2  0x0  0x3  0x0>;	/* GTX125 */
 			};
 
-			pio2: ucc_pin@02 {
+			pio2: ucc_pin@2 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x5  0xa 0x1  0x0  0x2  0x0   /* TxD0 */
@@ -228,22 +228,22 @@
 
 			/* These are the same PHYs as on
 			 * gianfar's MDIO bus */
-			qe_phy0: ethernet-phy@07 {
+			qe_phy0: ethernet-phy@7 {
 				interrupt-parent = <&mpic>;
 				interrupts = <1 1 0 0>;
 				reg = <0x7>;
 			};
-			qe_phy1: ethernet-phy@01 {
+			qe_phy1: ethernet-phy@1 {
 				interrupt-parent = <&mpic>;
 				interrupts = <2 1 0 0>;
 				reg = <0x1>;
 			};
-			qe_phy2: ethernet-phy@02 {
+			qe_phy2: ethernet-phy@2 {
 				interrupt-parent = <&mpic>;
 				interrupts = <1 1 0 0>;
 				reg = <0x2>;
 			};
-			qe_phy3: ethernet-phy@03 {
+			qe_phy3: ethernet-phy@3 {
 				interrupt-parent = <&mpic>;
 				interrupts = <2 1 0 0>;
 				reg = <0x3>;
diff --git a/arch/powerpc/boot/dts/fsl/mpc8569mds.dts b/arch/powerpc/boot/dts/fsl/mpc8569mds.dts
index 76b2bd6f7742..d8367ceddea6 100644
--- a/arch/powerpc/boot/dts/fsl/mpc8569mds.dts
+++ b/arch/powerpc/boot/dts/fsl/mpc8569mds.dts
@@ -141,7 +141,7 @@
 				gpio-controller;
 			};
 
-			pio1: ucc_pin@01 {
+			pio1: ucc_pin@1 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x2  0x1f 0x1  0x0  0x1  0x0	/* QE_MUX_MDC */
@@ -161,7 +161,7 @@
 					0x2  0x14 0x1  0x0  0x2  0x0>;	/* ENET1_GTXCLK	*/
 			};
 
-			pio2: ucc_pin@02 {
+			pio2: ucc_pin@2 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x2  0x1f 0x1  0x0  0x1  0x0	/* QE_MUX_MDC */
@@ -181,7 +181,7 @@
 					0x2  0x2 0x1  0x0  0x2  0x0>;	/* ENET2_GTXCLK	*/
 			};
 
-			pio3: ucc_pin@03 {
+			pio3: ucc_pin@3 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x2  0x1f 0x1  0x0  0x1  0x0	/* QE_MUX_MDC */
@@ -201,7 +201,7 @@
 					0x2  0x19 0x1  0x0  0x2  0x0>;	/* ENET3_GTXCLK	*/
 			};
 
-			pio4: ucc_pin@04 {
+			pio4: ucc_pin@4 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x2  0x1f 0x1  0x0  0x1  0x0	/* QE_MUX_MDC */
@@ -272,30 +272,30 @@
 			reg = <0x2120 0x18>;
 			compatible = "fsl,ucc-mdio";
 
-			qe_phy0: ethernet-phy@07 {
+			qe_phy0: ethernet-phy@7 {
 				interrupt-parent = <&mpic>;
 				interrupts = <1 1 0 0>;
 				reg = <0x7>;
 			};
-			qe_phy1: ethernet-phy@01 {
+			qe_phy1: ethernet-phy@1 {
 				interrupt-parent = <&mpic>;
 				interrupts = <2 1 0 0>;
 				reg = <0x1>;
 			};
-			qe_phy2: ethernet-phy@02 {
+			qe_phy2: ethernet-phy@2 {
 				interrupt-parent = <&mpic>;
 				interrupts = <3 1 0 0>;
 				reg = <0x2>;
 			};
-			qe_phy3: ethernet-phy@03 {
+			qe_phy3: ethernet-phy@3 {
 				interrupt-parent = <&mpic>;
 				interrupts = <4 1 0 0>;
 				reg = <0x3>;
 			};
-			qe_phy5: ethernet-phy@04 {
+			qe_phy5: ethernet-phy@4 {
 				reg = <0x04>;
 			};
-			qe_phy7: ethernet-phy@06 {
+			qe_phy7: ethernet-phy@6 {
 				reg = <0x6>;
 			};
 			tbi1: tbi-phy@11 {
diff --git a/arch/powerpc/boot/dts/fsl/p1010si-post.dtsi b/arch/powerpc/boot/dts/fsl/p1010si-post.dtsi
index af12ead88c5f..1b4aafc1f6a2 100644
--- a/arch/powerpc/boot/dts/fsl/p1010si-post.dtsi
+++ b/arch/powerpc/boot/dts/fsl/p1010si-post.dtsi
@@ -137,12 +137,14 @@
 		compatible = "fsl,p1010-flexcan";
 		reg = <0x1c000 0x1000>;
 		interrupts = <48 0x2 0 0>;
+		big-endian;
 	};
 
 	can1: can@1d000 {
 		compatible = "fsl,p1010-flexcan";
 		reg = <0x1d000 0x1000>;
 		interrupts = <61 0x2 0 0>;
+		big-endian;
 	};
 
 	L2: l2-cache-controller@20000 {
diff --git a/arch/powerpc/boot/dts/fsl/p1021mds.dts b/arch/powerpc/boot/dts/fsl/p1021mds.dts
index 291454c75dda..1047802f4d2a 100644
--- a/arch/powerpc/boot/dts/fsl/p1021mds.dts
+++ b/arch/powerpc/boot/dts/fsl/p1021mds.dts
@@ -202,7 +202,7 @@
 			ranges = <0x0 0xe0100 0x60>;
 			device_type = "par_io";
 			num-ports = <3>;
-			pio1: ucc_pin@01 {
+			pio1: ucc_pin@1 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x1  0x13 0x1  0x0  0x1  0x0    /* QE_MUX_MDC */
@@ -225,7 +225,7 @@
 					0x0  0x10 0x2  0x0  0x2  0x0>;    /* ENET1_COL */
 			};
 
-			pio2: ucc_pin@02 {
+			pio2: ucc_pin@2 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0x1  0x13 0x1  0x0  0x1  0x0    /* QE_MUX_MDC */
@@ -296,7 +296,7 @@
 				interrupts = <4 1 0 0>;
 				reg = <0x0>;
 			};
-			qe_phy1: ethernet-phy@03 {
+			qe_phy1: ethernet-phy@3 {
 				interrupt-parent = <&mpic>;
 				interrupts = <5 1 0 0>;
 				reg = <0x3>;
diff --git a/arch/powerpc/boot/dts/fsl/p1025rdb.dtsi b/arch/powerpc/boot/dts/fsl/p1025rdb.dtsi
index d44bb12debb0..0a5434a631c3 100644
--- a/arch/powerpc/boot/dts/fsl/p1025rdb.dtsi
+++ b/arch/powerpc/boot/dts/fsl/p1025rdb.dtsi
@@ -245,7 +245,7 @@
 		ranges = <0x0 0xe0100 0x60>;
 		device_type = "par_io";
 		num-ports = <3>;
-		pio1: ucc_pin@01 {
+		pio1: ucc_pin@1 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x1  0x13 0x1  0x0  0x1  0x0    /* QE_MUX_MDC */
@@ -268,7 +268,7 @@
 				0x0  0x10 0x2  0x0  0x2  0x0>;    /* ENET1_COL */
 		};
 
-		pio2: ucc_pin@02 {
+		pio2: ucc_pin@2 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x1  0x13 0x1  0x0  0x1  0x0    /* QE_MUX_MDC */
@@ -283,7 +283,7 @@
 				0x1  0x8  0x2  0x0  0x2  0x0>;    /* ENET5_RX_ER_SER5_CD_B */
 		};
 
-		pio3: ucc_pin@03 {
+		pio3: ucc_pin@3 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x0  0x16 0x2  0x0  0x2  0x0    /* SER7_CD_B*/
@@ -293,7 +293,7 @@
 				0x0  0x15 0x1  0x0  0x2  0x0>;    /* SER7_TXD0*/
 		};
 
-		pio4: ucc_pin@04 {
+		pio4: ucc_pin@4 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x1  0x0  0x2  0x0  0x2  0x0    /* SER3_CD_B*/
diff --git a/arch/powerpc/boot/dts/fsl/p1025rdb_32b.dts b/arch/powerpc/boot/dts/fsl/p1025rdb_32b.dts
index b15acbaea34b..ea33b57f8774 100644
--- a/arch/powerpc/boot/dts/fsl/p1025rdb_32b.dts
+++ b/arch/powerpc/boot/dts/fsl/p1025rdb_32b.dts
@@ -106,7 +106,7 @@
 				interrupts = <4 1 0 0>;
 				reg = <0x6>;
 			};
-			qe_phy1: ethernet-phy@03 {
+			qe_phy1: ethernet-phy@3 {
 				interrupt-parent = <&mpic>;
 				interrupts = <5 1 0 0>;
 				reg = <0x3>;
diff --git a/arch/powerpc/boot/dts/fsl/p1025twr.dtsi b/arch/powerpc/boot/dts/fsl/p1025twr.dtsi
index 08816fb474f5..ab75b8f29ae2 100644
--- a/arch/powerpc/boot/dts/fsl/p1025twr.dtsi
+++ b/arch/powerpc/boot/dts/fsl/p1025twr.dtsi
@@ -172,7 +172,7 @@
 		ranges = <0x0 0xe0100 0x60>;
 		device_type = "par_io";
 		num-ports = <3>;
-		pio1: ucc_pin@01 {
+		pio1: ucc_pin@1 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x1  0x13 0x1  0x0  0x1  0x0    /* QE_MUX_MDC */
@@ -195,7 +195,7 @@
 				0x0  0x10 0x2  0x0  0x2  0x0>;    /* ENET1_COL */
 		};
 
-		pio2: ucc_pin@02 {
+		pio2: ucc_pin@2 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x1  0x13 0x1  0x0  0x1  0x0    /* QE_MUX_MDC */
@@ -210,7 +210,7 @@
 				0x1  0x8  0x2  0x0  0x2  0x0>;    /* ENET5_RX_ER_SER5_CD_B */
 		};
 
-		pio3: ucc_pin@03 {
+		pio3: ucc_pin@3 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x0  0x16 0x2  0x0  0x2  0x0    /* SER7_CD_B*/
@@ -220,7 +220,7 @@
 				0x0  0x15 0x1  0x0  0x2  0x0>;    /* SER7_TXD0*/
 		};
 
-		pio4: ucc_pin@04 {
+		pio4: ucc_pin@4 {
 			pio-map = <
 		/* port  pin  dir  open_drain  assignment  has_irq */
 				0x1  0x0  0x2  0x0  0x2  0x0    /* SER3_CD_B*/
diff --git a/arch/powerpc/boot/dts/fsl/t1040rdb.dts b/arch/powerpc/boot/dts/fsl/t1040rdb.dts
index 621f2c6ee6ad..65ff34c49025 100644
--- a/arch/powerpc/boot/dts/fsl/t1040rdb.dts
+++ b/arch/powerpc/boot/dts/fsl/t1040rdb.dts
@@ -61,7 +61,7 @@
 			};
 
 			mdio@fc000 {
-				phy_sgmii_2: ethernet-phy@03 {
+				phy_sgmii_2: ethernet-phy@3 {
 					reg = <0x03>;
 				};
 			};
diff --git a/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts b/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts
index fcd2aeb5b8ac..4fa15f48a4c3 100644
--- a/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts
+++ b/arch/powerpc/boot/dts/fsl/t1042d4rdb.dts
@@ -77,23 +77,23 @@
 			};
 
 			mdio0: mdio@fc000 {
-				phy_sgmii_0: ethernet-phy@02 {
+				phy_sgmii_0: ethernet-phy@2 {
 					reg = <0x02>;
 				};
 
-				phy_sgmii_1: ethernet-phy@03 {
+				phy_sgmii_1: ethernet-phy@3 {
 					reg = <0x03>;
 				};
 
-				phy_sgmii_2: ethernet-phy@01 {
+				phy_sgmii_2: ethernet-phy@1 {
 					reg = <0x01>;
 				};
 
-				phy_rgmii_0: ethernet-phy@04 {
+				phy_rgmii_0: ethernet-phy@4 {
 					reg = <0x04>;
 				};
 
-				phy_rgmii_1: ethernet-phy@05 {
+				phy_rgmii_1: ethernet-phy@5 {
 					reg = <0x05>;
 				};
 			};
diff --git a/arch/powerpc/boot/dts/fsl/t1042rdb.dts b/arch/powerpc/boot/dts/fsl/t1042rdb.dts
index 2c138627b1b4..3ebb712224cb 100644
--- a/arch/powerpc/boot/dts/fsl/t1042rdb.dts
+++ b/arch/powerpc/boot/dts/fsl/t1042rdb.dts
@@ -59,7 +59,7 @@
 			};
 
 			mdio@fc000 {
-				phy_sgmii_2: ethernet-phy@03 {
+				phy_sgmii_2: ethernet-phy@3 {
 					reg = <0x03>;
 				};
 			};
diff --git a/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi b/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi
index 5fdddbd2a62b..099a598c74c0 100644
--- a/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi
+++ b/arch/powerpc/boot/dts/fsl/t104xrdb.dtsi
@@ -148,15 +148,15 @@
 			};
 
 			mdio0: mdio@fc000 {
-				phy_sgmii_2: ethernet-phy@03 {
+				phy_sgmii_2: ethernet-phy@3 {
 					reg = <0x03>;
 				};
 
-				phy_rgmii_0: ethernet-phy@01 {
+				phy_rgmii_0: ethernet-phy@1 {
 					reg = <0x01>;
 				};
 
-				phy_rgmii_1: ethernet-phy@02 {
+				phy_rgmii_1: ethernet-phy@2 {
 					reg = <0x02>;
 				};
 			};
diff --git a/arch/powerpc/boot/dts/fsp2.dts b/arch/powerpc/boot/dts/fsp2.dts
index f10a64aeb83b..6560283c5aec 100644
--- a/arch/powerpc/boot/dts/fsp2.dts
+++ b/arch/powerpc/boot/dts/fsp2.dts
@@ -583,21 +583,21 @@
 			};
 		};
 
-		OHCI1: ohci@02040000 {
+		OHCI1: ohci@2040000 {
 			compatible = "ohci-le";
 			reg = <0x02040000 0xa0>;
 			interrupt-parent = <&UIC1_3>;
 			interrupts = <28 0x8 29 0x8>;
 		};
 
-		OHCI2: ohci@02080000 {
+		OHCI2: ohci@2080000 {
 			compatible = "ohci-le";
 			reg = <0x02080000 0xa0>;
 			interrupt-parent = <&UIC1_3>;
 			interrupts = <30 0x8 31 0x8>;
 		};
 
-		EHCI: ehci@02000000 {
+		EHCI: ehci@2000000 {
 			compatible = "usb-ehci";
 			reg = <0x02000000 0xa4>;
 			interrupt-parent = <&UIC1_3>;
diff --git a/arch/powerpc/boot/dts/gamecube.dts b/arch/powerpc/boot/dts/gamecube.dts
index ef3be0e58b02..58d06c9ee08b 100644
--- a/arch/powerpc/boot/dts/gamecube.dts
+++ b/arch/powerpc/boot/dts/gamecube.dts
@@ -54,13 +54,13 @@
 		ranges = <0x0c000000 0x0c000000 0x00010000>;
 		interrupt-parent = <&PIC>;
 
-		video@0c002000 {
+		video@c002000 {
 			compatible = "nintendo,flipper-vi";
 			reg = <0x0c002000 0x100>;
 			interrupts = <8>;
 		};
 
-		processor-interface@0c003000 {
+		processor-interface@c003000 {
 			compatible = "nintendo,flipper-pi";
 			reg = <0x0c003000 0x100>;
 
@@ -71,7 +71,7 @@
 			};
 		};
 
-		dsp@0c005000 {
+		dsp@c005000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "nintendo,flipper-dsp";
@@ -84,26 +84,26 @@
 			};
 		};
 
-		disk@0c006000 {
+		disk@c006000 {
 			compatible = "nintendo,flipper-di";
 			reg = <0x0c006000 0x40>;
 			interrupts = <2>;
 		};
 
-		audio@0c006c00 {
+		audio@c006c00 {
 			compatible = "nintendo,flipper-ai";
 			reg = <0x0c006c00 0x20>;
 			interrupts = <6>;
 		};
 
-		gamepad-controller@0c006400 {
+		gamepad-controller@c006400 {
 			compatible = "nintendo,flipper-si";
 			reg = <0x0c006400 0x100>;
 			interrupts = <3>;
 		};
 
 		/* External Interface bus */
-		exi@0c006800 {
+		exi@c006800 {
 			compatible = "nintendo,flipper-exi";
 			reg = <0x0c006800 0x40>;
 			virtual-reg = <0x0c006800>;
diff --git a/arch/powerpc/boot/dts/haleakala.dts b/arch/powerpc/boot/dts/haleakala.dts
index 2b256694eca6..cb16dad43c92 100644
--- a/arch/powerpc/boot/dts/haleakala.dts
+++ b/arch/powerpc/boot/dts/haleakala.dts
@@ -237,7 +237,7 @@
 			};
 		};
 
-		PCIE0: pciex@0a0000000 {
+		PCIE0: pciex@a0000000 {
 			device_type = "pci";
 			#interrupt-cells = <1>;
 			#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/kilauea.dts b/arch/powerpc/boot/dts/kilauea.dts
index 5ba7f01e2a29..2a3413221cc1 100644
--- a/arch/powerpc/boot/dts/kilauea.dts
+++ b/arch/powerpc/boot/dts/kilauea.dts
@@ -322,7 +322,7 @@
 			};
 		};
 
-		PCIE0: pciex@0a0000000 {
+		PCIE0: pciex@a0000000 {
 			device_type = "pci";
 			#interrupt-cells = <1>;
 			#size-cells = <2>;
@@ -363,7 +363,7 @@
 				0x0 0x0 0x0 0x4 &UIC2 0x3 0x4 /* swizzled int D */>;
 		};
 
-		PCIE1: pciex@0c0000000 {
+		PCIE1: pciex@c0000000 {
 			device_type = "pci";
 			#interrupt-cells = <1>;
 			#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/kmeter1.dts b/arch/powerpc/boot/dts/kmeter1.dts
index 983aee185793..9fa33d9ba966 100644
--- a/arch/powerpc/boot/dts/kmeter1.dts
+++ b/arch/powerpc/boot/dts/kmeter1.dts
@@ -434,27 +434,27 @@
 				compatible = "fsl,ucc-mdio";
 
 				/* Piggy2 (UCC4, MDIO 0x00, RMII) */
-				phy_piggy2: ethernet-phy@00 {
+				phy_piggy2: ethernet-phy@0 {
 					reg = <0x0>;
 				};
 
 				/* Eth-1 (UCC5, MDIO 0x08, RMII) */
-				phy_eth1: ethernet-phy@08 {
+				phy_eth1: ethernet-phy@8 {
 					reg = <0x08>;
 				};
 
 				/* Eth-2 (UCC6, MDIO 0x09, RMII) */
-				phy_eth2: ethernet-phy@09 {
+				phy_eth2: ethernet-phy@9 {
 					reg = <0x09>;
 				};
 
 				/* Eth-3 (UCC7, MDIO 0x0a, RMII) */
-				phy_eth3: ethernet-phy@0a {
+				phy_eth3: ethernet-phy@a {
 					reg = <0x0a>;
 				};
 
 				/* Eth-4 (UCC8, MDIO 0x0b, RMII) */
-				phy_eth4: ethernet-phy@0b {
+				phy_eth4: ethernet-phy@b {
 					reg = <0x0b>;
 				};
 
diff --git a/arch/powerpc/boot/dts/makalu.dts b/arch/powerpc/boot/dts/makalu.dts
index 63d48b632c84..bf8fe1629392 100644
--- a/arch/powerpc/boot/dts/makalu.dts
+++ b/arch/powerpc/boot/dts/makalu.dts
@@ -268,7 +268,7 @@
 			};
 		};
 
-		PCIE0: pciex@0a0000000 {
+		PCIE0: pciex@a0000000 {
 			device_type = "pci";
 			#interrupt-cells = <1>;
 			#size-cells = <2>;
@@ -309,7 +309,7 @@
 				0x0 0x0 0x0 0x4 &UIC2 0x3 0x4 /* swizzled int D */>;
 		};
 
-		PCIE1: pciex@0c0000000 {
+		PCIE1: pciex@c0000000 {
 			device_type = "pci";
 			#interrupt-cells = <1>;
 			#size-cells = <2>;
diff --git a/arch/powerpc/boot/dts/mpc832x_mds.dts b/arch/powerpc/boot/dts/mpc832x_mds.dts
index 0793cdf0d46e..49c7d657118a 100644
--- a/arch/powerpc/boot/dts/mpc832x_mds.dts
+++ b/arch/powerpc/boot/dts/mpc832x_mds.dts
@@ -186,7 +186,7 @@
 			device_type = "par_io";
 			num-ports = <7>;
 
-			pio3: ucc_pin@03 {
+			pio3: ucc_pin@3 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					3  4  3  0  2  0  /* MDIO */
@@ -208,7 +208,7 @@
 					1 12  1  0  1  0 	/* TX_EN */
 					1 13  2  0  1  0>;	/* CRS */
 			};
-			pio4: ucc_pin@04 {
+			pio4: ucc_pin@4 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					3 31  2  0  1  0 	/* RX_CLK (CLK7) */
@@ -228,7 +228,7 @@
 					1 30  1  0  1  0 	/* TX_EN */
 					1 31  2  0  1  0>;	/* CRS */
 			};
-			pio5: ucc_pin@05 {
+			pio5: ucc_pin@5 {
 				pio-map = <
 				/*
 				 *    		      open       has
@@ -352,12 +352,12 @@
 			reg = <0x2320 0x18>;
 			compatible = "fsl,ucc-mdio";
 
-			phy3: ethernet-phy@03 {
+			phy3: ethernet-phy@3 {
 				interrupt-parent = <&ipic>;
 				interrupts = <17 0x8>;
 				reg = <0x3>;
 			};
-			phy4: ethernet-phy@04 {
+			phy4: ethernet-phy@4 {
 				interrupt-parent = <&ipic>;
 				interrupts = <18 0x8>;
 				reg = <0x4>;
diff --git a/arch/powerpc/boot/dts/mpc832x_rdb.dts b/arch/powerpc/boot/dts/mpc832x_rdb.dts
index 91df1eb16667..647cae14c16d 100644
--- a/arch/powerpc/boot/dts/mpc832x_rdb.dts
+++ b/arch/powerpc/boot/dts/mpc832x_rdb.dts
@@ -175,7 +175,7 @@
 				gpio-controller;
 			};
 
-			ucc2pio:ucc_pin@02 {
+			ucc2pio:ucc_pin@2 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					3  4  3  0  2  0 	/* MDIO */
@@ -197,7 +197,7 @@
 					0 30  1  0  1  0 	/* TX_EN */
 					0 31  2  0  1  0>;      /* CRS */
 			};
-			ucc3pio:ucc_pin@03 {
+			ucc3pio:ucc_pin@3 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0 13  2  0  1  0 	/* RX_CLK (CLK9) */
@@ -310,12 +310,12 @@
 			reg = <0x3120 0x18>;
 			compatible = "fsl,ucc-mdio";
 
-			phy00:ethernet-phy@00 {
+			phy00:ethernet-phy@0 {
 				interrupt-parent = <&ipic>;
 				interrupts = <0>;
 				reg = <0x0>;
 			};
-			phy04:ethernet-phy@04 {
+			phy04:ethernet-phy@4 {
 				interrupt-parent = <&ipic>;
 				interrupts = <0>;
 				reg = <0x4>;
diff --git a/arch/powerpc/boot/dts/mpc836x_mds.dts b/arch/powerpc/boot/dts/mpc836x_mds.dts
index ecb6ccd3a6aa..539fd9f72eda 100644
--- a/arch/powerpc/boot/dts/mpc836x_mds.dts
+++ b/arch/powerpc/boot/dts/mpc836x_mds.dts
@@ -228,7 +228,7 @@
 				gpio-controller;
 			};
 
-			pio1: ucc_pin@01 {
+			pio1: ucc_pin@1 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0  3  1  0  1  0 	/* TxD0 */
@@ -255,7 +255,7 @@
 					2  9  1  0  3  0 	/* GTX_CLK - CLK10 */
 					2  8  2  0  1  0>;	/* GTX125 - CLK9 */
 			};
-			pio2: ucc_pin@02 {
+			pio2: ucc_pin@2 {
 				pio-map = <
 			/* port  pin  dir  open_drain  assignment  has_irq */
 					0  17 1  0  1  0   /* TxD0 */
@@ -393,12 +393,12 @@
 			reg = <0x2120 0x18>;
 			compatible = "fsl,ucc-mdio";
 
-			phy0: ethernet-phy@00 {
+			phy0: ethernet-phy@0 {
 				interrupt-parent = <&ipic>;
 				interrupts = <17 0x8>;
 				reg = <0x0>;
 			};
-			phy1: ethernet-phy@01 {
+			phy1: ethernet-phy@1 {
 				interrupt-parent = <&ipic>;
 				interrupts = <18 0x8>;
 				reg = <0x1>;
diff --git a/arch/powerpc/boot/dts/sbc8548-altflash.dts b/arch/powerpc/boot/dts/sbc8548-altflash.dts
index 0b38a0defd2c..8967a56adad4 100644
--- a/arch/powerpc/boot/dts/sbc8548-altflash.dts
+++ b/arch/powerpc/boot/dts/sbc8548-altflash.dts
@@ -40,12 +40,12 @@
 			compatible = "intel,JS28F128", "cfi-flash";
 			bank-width = <4>;
 			device-width = <1>;
-			partition@0x0 {
+			partition@0 {
 				label = "space";
 				/* FC000000 -> FFEFFFFF */
 				reg = <0x00000000 0x03f00000>;
 			};
-			partition@0x03f00000 {
+			partition@3f00000 {
 				label = "bootloader";
 				/* FFF00000 -> FFFFFFFF */
 				reg = <0x03f00000 0x00100000>;
@@ -95,12 +95,12 @@
 			reg = <0x6 0x0 0x800000>;
 			bank-width = <1>;
 			device-width = <1>;
-			partition@0x0 {
+			partition@0 {
 				label = "space";
 				/* EF800000 -> EFF9FFFF */
 				reg = <0x00000000 0x007a0000>;
 			};
-			partition@0x7a0000 {
+			partition@7a0000 {
 				label = "bootloader";
 				/* EFFA0000 -> EFFFFFFF */
 				reg = <0x007a0000 0x00060000>;
diff --git a/arch/powerpc/boot/dts/sbc8548.dts b/arch/powerpc/boot/dts/sbc8548.dts
index 1df2a0955668..9bdb828a504e 100644
--- a/arch/powerpc/boot/dts/sbc8548.dts
+++ b/arch/powerpc/boot/dts/sbc8548.dts
@@ -38,12 +38,12 @@
 			reg = <0x0 0x0 0x800000>;
 			bank-width = <1>;
 			device-width = <1>;
-			partition@0x0 {
+			partition@0 {
 				label = "space";
 				/* FF800000 -> FFF9FFFF */
 				reg = <0x00000000 0x007a0000>;
 			};
-			partition@0x7a0000 {
+			partition@7a0000 {
 				label = "bootloader";
 				/* FFFA0000 -> FFFFFFFF */
 				reg = <0x007a0000 0x00060000>;
@@ -92,12 +92,12 @@
 			compatible = "intel,JS28F128", "cfi-flash";
 			bank-width = <4>;
 			device-width = <1>;
-			partition@0x0 {
+			partition@0 {
 				label = "space";
 				/* EC000000 -> EFEFFFFF */
 				reg = <0x00000000 0x03f00000>;
 			};
-			partition@0x03f00000 {
+			partition@3f00000 {
 				label = "bootloader";
 				/* EFF00000 -> EFFFFFFF */
 				reg = <0x03f00000 0x00100000>;
diff --git a/arch/powerpc/boot/dts/wii.dts b/arch/powerpc/boot/dts/wii.dts
index 77528c9a8dbd..17a5babb098d 100644
--- a/arch/powerpc/boot/dts/wii.dts
+++ b/arch/powerpc/boot/dts/wii.dts
@@ -65,14 +65,14 @@
 			  0x0d800000 0x0d800000 0x00800000>;
 		interrupt-parent = <&PIC0>;
 
-		video@0c002000 {
+		video@c002000 {
 			compatible = "nintendo,hollywood-vi",
 					"nintendo,flipper-vi";
 			reg = <0x0c002000 0x100>;
 			interrupts = <8>;
 		};
 
-		processor-interface@0c003000 {
+		processor-interface@c003000 {
 			compatible = "nintendo,hollywood-pi",
 					"nintendo,flipper-pi";
 			reg = <0x0c003000 0x100>;
@@ -84,7 +84,7 @@
 			};
 		};
 
-		dsp@0c005000 {
+		dsp@c005000 {
 			#address-cells = <1>;
 			#size-cells = <1>;
 			compatible = "nintendo,hollywood-dsp",
@@ -93,14 +93,14 @@
 			interrupts = <6>;
 		};
 
-		gamepad-controller@0d006400 {
+		gamepad-controller@d006400 {
 			compatible = "nintendo,hollywood-si",
 					"nintendo,flipper-si";
 			reg = <0x0d006400 0x100>;
 			interrupts = <3>;
 		};
 
-		audio@0c006c00 {
+		audio@c006c00 {
 			compatible = "nintendo,hollywood-ai",
 					"nintendo,flipper-ai";
 			reg = <0x0d006c00 0x20>;
@@ -108,7 +108,7 @@
 		};
 
 		/* External Interface bus */
-		exi@0d006800 {
+		exi@d006800 {
 			compatible = "nintendo,hollywood-exi",
 					"nintendo,flipper-exi";
 			reg = <0x0d006800 0x40>;
@@ -116,7 +116,7 @@
 			interrupts = <4>;
 		};
 
-		usb@0d040000 {
+		usb@d040000 {
 			compatible = "nintendo,hollywood-usb-ehci",
 					"usb-ehci";
 			reg = <0x0d040000 0x100>;
@@ -124,7 +124,7 @@
 			interrupt-parent = <&PIC1>;
 		};
 
-		usb@0d050000 {
+		usb@d050000 {
 			compatible = "nintendo,hollywood-usb-ohci",
 					"usb-ohci";
 			reg = <0x0d050000 0x100>;
@@ -132,7 +132,7 @@
 			interrupt-parent = <&PIC1>;
 		};
 
-		usb@0d060000 {
+		usb@d060000 {
 			compatible = "nintendo,hollywood-usb-ohci",
 					"usb-ohci";
 			reg = <0x0d060000 0x100>;
@@ -140,7 +140,7 @@
 			interrupt-parent = <&PIC1>;
 		};
 
-		sd@0d070000 {
+		sd@d070000 {
 			compatible = "nintendo,hollywood-sdhci",
 					"sdhci";
 			reg = <0x0d070000 0x200>;
@@ -148,7 +148,7 @@
 			interrupt-parent = <&PIC1>;
 		};
 
-		sdio@0d080000 {
+		sdio@d080000 {
 			compatible = "nintendo,hollywood-sdhci",
 					"sdhci";
 			reg = <0x0d080000 0x200>;
@@ -156,14 +156,14 @@
 			interrupt-parent = <&PIC1>;
 		};
 
-		ipc@0d000000 {
+		ipc@d000000 {
 			compatible = "nintendo,hollywood-ipc";
 			reg = <0x0d000000 0x10>;
 			interrupts = <30>;
 			interrupt-parent = <&PIC1>;
 		};
 
-		PIC1: pic1@0d800030 {
+		PIC1: pic1@d800030 {
 			#interrupt-cells = <1>;
 			compatible = "nintendo,hollywood-pic";
 			reg = <0x0d800030 0x10>;
@@ -171,7 +171,7 @@
 			interrupts = <14>;
 		};
 
-		GPIO: gpio@0d8000c0 {
+		GPIO: gpio@d8000c0 {
 			#gpio-cells = <2>;
 			compatible = "nintendo,hollywood-gpio";
 			reg = <0x0d8000c0 0x40>;
@@ -203,12 +203,12 @@
 			*/
 		};
 
-		control@0d800100 {
+		control@d800100 {
 			compatible = "nintendo,hollywood-control";
 			reg = <0x0d800100 0x300>;
 		};
 
-		disk@0d806000 {
+		disk@d806000 {
 			compatible = "nintendo,hollywood-di";
 			reg = <0x0d806000 0x40>;
 			interrupts = <2>;
diff --git a/arch/powerpc/boot/serial.c b/arch/powerpc/boot/serial.c
index 7b5c02b1afd0..88955095ec07 100644
--- a/arch/powerpc/boot/serial.c
+++ b/arch/powerpc/boot/serial.c
@@ -124,20 +124,26 @@ int serial_console_init(void)
 	else if (dt_is_compatible(devp, "marvell,mv64360-mpsc"))
 		rc = mpsc_console_init(devp, &serial_cd);
 #endif
+#ifdef CONFIG_CPM
 	else if (dt_is_compatible(devp, "fsl,cpm1-scc-uart") ||
 	         dt_is_compatible(devp, "fsl,cpm1-smc-uart") ||
 	         dt_is_compatible(devp, "fsl,cpm2-scc-uart") ||
 	         dt_is_compatible(devp, "fsl,cpm2-smc-uart"))
 		rc = cpm_console_init(devp, &serial_cd);
+#endif
 #ifdef CONFIG_PPC_MPC52XX
 	else if (dt_is_compatible(devp, "fsl,mpc5200-psc-uart"))
 		rc = mpc5200_psc_console_init(devp, &serial_cd);
 #endif
+#ifdef CONFIG_XILINX_VIRTEX
 	else if (dt_is_compatible(devp, "xlnx,opb-uartlite-1.00.b") ||
 		 dt_is_compatible(devp, "xlnx,xps-uartlite-1.00.a"))
 		rc = uartlite_console_init(devp, &serial_cd);
+#endif
+#ifdef CONFIG_PPC64_BOOT_WRAPPER
 	else if (dt_is_compatible(devp, "ibm,opal-console-raw"))
 		rc = opal_console_init(devp, &serial_cd);
+#endif
 
 	/* Add other serial console driver calls here */
 
diff --git a/arch/powerpc/configs/44x/warp_defconfig b/arch/powerpc/configs/44x/warp_defconfig
index b5c866073efd..6c02f53271cd 100644
--- a/arch/powerpc/configs/44x/warp_defconfig
+++ b/arch/powerpc/configs/44x/warp_defconfig
@@ -28,7 +28,6 @@ CONFIG_NETFILTER=y
 CONFIG_VLAN_8021Q=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 # CONFIG_STANDALONE is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/powerpc/configs/c2k_defconfig b/arch/powerpc/configs/c2k_defconfig
index f1552af9eecc..4bb832a41d55 100644
--- a/arch/powerpc/configs/c2k_defconfig
+++ b/arch/powerpc/configs/c2k_defconfig
@@ -272,18 +272,6 @@ CONFIG_USB_SERIAL_EDGEPORT=m
 CONFIG_USB_SERIAL_EDGEPORT_TI=m
 CONFIG_USB_SERIAL_KEYSPAN_PDA=m
 CONFIG_USB_SERIAL_KEYSPAN=m
-CONFIG_USB_SERIAL_KEYSPAN_MPR=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XA=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XB=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19=y
-CONFIG_USB_SERIAL_KEYSPAN_USA18X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QW=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QI=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49WLC=y
 CONFIG_USB_SERIAL_KLSI=m
 CONFIG_USB_SERIAL_KOBIL_SCT=m
 CONFIG_USB_SERIAL_MCT_U232=m
diff --git a/arch/powerpc/configs/fsl-emb-nonhw.config b/arch/powerpc/configs/fsl-emb-nonhw.config
index cc49c95494da..e0567dc41968 100644
--- a/arch/powerpc/configs/fsl-emb-nonhw.config
+++ b/arch/powerpc/configs/fsl-emb-nonhw.config
@@ -71,7 +71,6 @@ CONFIG_IP_ROUTE_MULTIPATH=y
 CONFIG_IP_ROUTE_VERBOSE=y
 CONFIG_IP_SCTP=m
 CONFIG_IPV6=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_ISO9660_FS=m
 CONFIG_JFFS2_FS_DEBUG=1
 CONFIG_JFFS2_FS=y
diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig
index 063817fee61c..67c39f4acede 100644
--- a/arch/powerpc/configs/g5_defconfig
+++ b/arch/powerpc/configs/g5_defconfig
@@ -189,18 +189,6 @@ CONFIG_USB_SERIAL_GARMIN=m
 CONFIG_USB_SERIAL_IPW=m
 CONFIG_USB_SERIAL_KEYSPAN_PDA=m
 CONFIG_USB_SERIAL_KEYSPAN=m
-CONFIG_USB_SERIAL_KEYSPAN_MPR=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XA=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XB=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19=y
-CONFIG_USB_SERIAL_KEYSPAN_USA18X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QW=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QI=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49WLC=y
 CONFIG_USB_SERIAL_KLSI=m
 CONFIG_USB_SERIAL_KOBIL_SCT=m
 CONFIG_USB_SERIAL_MCT_U232=m
diff --git a/arch/powerpc/configs/maple_defconfig b/arch/powerpc/configs/maple_defconfig
index 078cdb427fc9..59e47ec85336 100644
--- a/arch/powerpc/configs/maple_defconfig
+++ b/arch/powerpc/configs/maple_defconfig
@@ -82,18 +82,6 @@ CONFIG_USB_SERIAL_CYPRESS_M8=m
 CONFIG_USB_SERIAL_GARMIN=m
 CONFIG_USB_SERIAL_IPW=m
 CONFIG_USB_SERIAL_KEYSPAN=y
-CONFIG_USB_SERIAL_KEYSPAN_MPR=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XA=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XB=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19=y
-CONFIG_USB_SERIAL_KEYSPAN_USA18X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QW=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QI=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49WLC=y
 CONFIG_USB_SERIAL_TI=m
 CONFIG_EXT2_FS=y
 CONFIG_EXT4_FS=y
diff --git a/arch/powerpc/configs/mpc512x_defconfig b/arch/powerpc/configs/mpc512x_defconfig
index 10be5773ad5d..c2b1c4404683 100644
--- a/arch/powerpc/configs/mpc512x_defconfig
+++ b/arch/powerpc/configs/mpc512x_defconfig
@@ -39,7 +39,6 @@ CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_PREVENT_FIRMWARE_BUILD is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/powerpc/configs/mpc866_ads_defconfig b/arch/powerpc/configs/mpc866_ads_defconfig
index f1f176c29fa3..5320735395e7 100644
--- a/arch/powerpc/configs/mpc866_ads_defconfig
+++ b/arch/powerpc/configs/mpc866_ads_defconfig
@@ -13,7 +13,6 @@ CONFIG_EXPERT=y
 CONFIG_PARTITION_ADVANCED=y
 CONFIG_MPC86XADS=y
 CONFIG_8xx_COPYBACK=y
-CONFIG_8xx_CPU6=y
 CONFIG_GEN_RTC=y
 CONFIG_HZ_1000=y
 CONFIG_MATH_EMULATION=y
diff --git a/arch/powerpc/configs/pmac32_defconfig b/arch/powerpc/configs/pmac32_defconfig
index 1aab9a62a681..62948d198d7f 100644
--- a/arch/powerpc/configs/pmac32_defconfig
+++ b/arch/powerpc/configs/pmac32_defconfig
@@ -264,18 +264,6 @@ CONFIG_USB_SERIAL_VISOR=m
 CONFIG_USB_SERIAL_IPAQ=m
 CONFIG_USB_SERIAL_KEYSPAN_PDA=m
 CONFIG_USB_SERIAL_KEYSPAN=m
-CONFIG_USB_SERIAL_KEYSPAN_MPR=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XA=y
-CONFIG_USB_SERIAL_KEYSPAN_USA28XB=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19=y
-CONFIG_USB_SERIAL_KEYSPAN_USA18X=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QW=y
-CONFIG_USB_SERIAL_KEYSPAN_USA19QI=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49W=y
-CONFIG_USB_SERIAL_KEYSPAN_USA49WLC=y
 CONFIG_USB_APPLEDISPLAY=m
 CONFIG_LEDS_TRIGGER_DEFAULT_ON=y
 CONFIG_EXT2_FS=y
diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig
index 4891bbed6258..9e92aa6a52ba 100644
--- a/arch/powerpc/configs/powernv_defconfig
+++ b/arch/powerpc/configs/powernv_defconfig
@@ -4,7 +4,6 @@ CONFIG_CPU_LITTLE_ENDIAN=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_AUDIT=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_TASKSTATS=y
@@ -97,6 +96,7 @@ CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=65536
 CONFIG_VIRTIO_BLK=m
+CONFIG_BLK_DEV_NVME=y
 CONFIG_IDE=y
 CONFIG_BLK_DEV_IDECD=y
 CONFIG_BLK_DEV_GENERIC=y
@@ -113,6 +113,7 @@ CONFIG_SCSI_CXGB3_ISCSI=m
 CONFIG_SCSI_CXGB4_ISCSI=m
 CONFIG_SCSI_BNX2_ISCSI=m
 CONFIG_BE2ISCSI=m
+CONFIG_SCSI_AACRAID=y
 CONFIG_SCSI_MPT2SAS=m
 CONFIG_SCSI_SYM53C8XX_2=m
 CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=0
diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig
index 6ddca80c52c3..5033e630afea 100644
--- a/arch/powerpc/configs/ppc64_defconfig
+++ b/arch/powerpc/configs/ppc64_defconfig
@@ -1,7 +1,6 @@
 CONFIG_PPC64=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_TASKSTATS=y
diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
index da0e8d535eb8..7ee736f20774 100644
--- a/arch/powerpc/configs/ppc6xx_defconfig
+++ b/arch/powerpc/configs/ppc6xx_defconfig
@@ -347,7 +347,6 @@ CONFIG_MAC80211_DEBUGFS=y
 CONFIG_NET_9P=m
 CONFIG_NET_9P_VIRTIO=m
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_DEBUG_DEVRES=y
 CONFIG_CONNECTOR=y
 CONFIG_PARPORT=m
diff --git a/arch/powerpc/configs/ps3_defconfig b/arch/powerpc/configs/ps3_defconfig
index 2efa025bf483..187e2f7c12c8 100644
--- a/arch/powerpc/configs/ps3_defconfig
+++ b/arch/powerpc/configs/ps3_defconfig
@@ -64,7 +64,6 @@ CONFIG_CFG80211_WEXT=y
 CONFIG_MAC80211=m
 # CONFIG_MAC80211_RC_MINSTREL is not set
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=65535
diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig
index bde2cd1005a2..0dd5cf7b566d 100644
--- a/arch/powerpc/configs/pseries_defconfig
+++ b/arch/powerpc/configs/pseries_defconfig
@@ -3,7 +3,6 @@ CONFIG_NR_CPUS=2048
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_AUDIT=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_TASKSTATS=y
diff --git a/arch/powerpc/configs/wii_defconfig b/arch/powerpc/configs/wii_defconfig
index 9c7400a19e9d..0b0f78823a1b 100644
--- a/arch/powerpc/configs/wii_defconfig
+++ b/arch/powerpc/configs/wii_defconfig
@@ -43,7 +43,6 @@ CONFIG_CFG80211=y
 CONFIG_MAC80211=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 # CONFIG_STANDALONE is not set
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_COUNT=2
diff --git a/arch/powerpc/crypto/crc32c-vpmsum_glue.c b/arch/powerpc/crypto/crc32c-vpmsum_glue.c
index f058e0c3e4d4..fd1d6c83f0c0 100644
--- a/arch/powerpc/crypto/crc32c-vpmsum_glue.c
+++ b/arch/powerpc/crypto/crc32c-vpmsum_glue.c
@@ -141,6 +141,7 @@ static struct shash_alg alg = {
 		.cra_name		= "crc32c",
 		.cra_driver_name	= "crc32c-vpmsum",
 		.cra_priority		= 200,
+		.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 		.cra_blocksize		= CHKSUM_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(u32),
 		.cra_module		= THIS_MODULE,
diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild
index 2542ea15d338..3196d227e351 100644
--- a/arch/powerpc/include/asm/Kbuild
+++ b/arch/powerpc/include/asm/Kbuild
@@ -1,4 +1,3 @@
-generic-y += clkdev.h
 generic-y += div64.h
 generic-y += export.h
 generic-y += irq_regs.h
diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h
index 016579ef16d3..c615abdce119 100644
--- a/arch/powerpc/include/asm/book3s/32/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/32/pgtable.h
@@ -16,6 +16,7 @@
 #define PGD_INDEX_SIZE	(32 - PGDIR_SHIFT)
 
 #define PMD_CACHE_INDEX	PMD_INDEX_SIZE
+#define PUD_CACHE_INDEX	PUD_INDEX_SIZE
 
 #ifndef __ASSEMBLY__
 #define PTE_TABLE_SIZE	(sizeof(pte_t) << PTE_INDEX_SIZE)
@@ -311,6 +312,29 @@ static inline int pte_present(pte_t pte)
 	return pte_val(pte) & _PAGE_PRESENT;
 }
 
+/*
+ * We only find page table entry in the last level
+ * Hence no need for other accessors
+ */
+#define pte_access_permitted pte_access_permitted
+static inline bool pte_access_permitted(pte_t pte, bool write)
+{
+	unsigned long pteval = pte_val(pte);
+	/*
+	 * A read-only access is controlled by _PAGE_USER bit.
+	 * We have _PAGE_READ set for WRITE and EXECUTE
+	 */
+	unsigned long need_pte_bits = _PAGE_PRESENT | _PAGE_USER;
+
+	if (write)
+		need_pte_bits |= _PAGE_WRITE;
+
+	if ((pteval & need_pte_bits) != need_pte_bits)
+		return false;
+
+	return true;
+}
+
 /* Conversion functions: convert a page and protection to a page entry,
  * and a page entry and page directory to the page they refer to.
  *
diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h
index 197ced1eaaa0..67c5475311ee 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-4k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h
@@ -17,6 +17,12 @@
 #define H_PUD_TABLE_SIZE	(sizeof(pud_t) << H_PUD_INDEX_SIZE)
 #define H_PGD_TABLE_SIZE	(sizeof(pgd_t) << H_PGD_INDEX_SIZE)
 
+#define H_PAGE_F_GIX_SHIFT	53
+#define H_PAGE_F_SECOND	_RPAGE_RPN44	/* HPTE is in 2ndary HPTEG */
+#define H_PAGE_F_GIX	(_RPAGE_RPN43 | _RPAGE_RPN42 | _RPAGE_RPN41)
+#define H_PAGE_BUSY	_RPAGE_RSV1     /* software: PTE & hash are busy */
+#define H_PAGE_HASHPTE	_RPAGE_RSV2     /* software: PTE & hash are busy */
+
 /* PTE flags to conserve for HPTE identification */
 #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \
 			 H_PAGE_F_SECOND | H_PAGE_F_GIX)
@@ -49,6 +55,21 @@ static inline int hash__hugepd_ok(hugepd_t hpd)
 }
 #endif
 
+/*
+ * 4K PTE format is different from 64K PTE format. Saving the hash_slot is just
+ * a matter of returning the PTE bits that need to be modified. On 64K PTE,
+ * things are a little more involved and hence needs many more parameters to
+ * accomplish the same. However we want to abstract this out from the caller by
+ * keeping the prototype consistent across the two formats.
+ */
+static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
+					 unsigned int subpg_index, unsigned long hidx,
+					 int offset)
+{
+	return (hidx << H_PAGE_F_GIX_SHIFT) &
+		(H_PAGE_F_SECOND | H_PAGE_F_GIX);
+}
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 
 static inline char *get_hpte_slot_array(pmd_t *pmdp)
@@ -101,8 +122,6 @@ extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 					 pgtable_t pgtable);
 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
-extern void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
-				      unsigned long address, pmd_t *pmdp);
 extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
 				       unsigned long addr, pmd_t *pmdp);
 extern int hash__has_transparent_hugepage(void);
diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h
index 8d40cf03cb67..3bcf269f8f55 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h
@@ -13,21 +13,17 @@
  */
 #define H_PAGE_COMBO	_RPAGE_RPN0 /* this is a combo 4k page */
 #define H_PAGE_4K_PFN	_RPAGE_RPN1 /* PFN is for a single 4k page */
+#define H_PAGE_BUSY	_RPAGE_RPN44     /* software: PTE & hash are busy */
+#define H_PAGE_HASHPTE	_RPAGE_RPN43	/* PTE has associated HPTE */
+
 /*
  * We need to differentiate between explicit huge page and THP huge
  * page, since THP huge page also need to track real subpage details
  */
 #define H_PAGE_THP_HUGE  H_PAGE_4K_PFN
 
-/*
- * Used to track subpage group valid if H_PAGE_COMBO is set
- * This overloads H_PAGE_F_GIX and H_PAGE_F_SECOND
- */
-#define H_PAGE_COMBO_VALID	(H_PAGE_F_GIX | H_PAGE_F_SECOND)
-
 /* PTE flags to conserve for HPTE identification */
-#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \
-			 H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO)
+#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | H_PAGE_COMBO)
 /*
  * we support 16 fragments per PTE page of 64K size.
  */
@@ -49,30 +45,64 @@
  * generic accessors and iterators here
  */
 #define __real_pte __real_pte
-static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
+static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset)
 {
 	real_pte_t rpte;
 	unsigned long *hidxp;
 
 	rpte.pte = pte;
-	rpte.hidx = 0;
-	if (pte_val(pte) & H_PAGE_COMBO) {
-		/*
-		 * Make sure we order the hidx load against the H_PAGE_COMBO
-		 * check. The store side ordering is done in __hash_page_4K
-		 */
-		smp_rmb();
-		hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
-		rpte.hidx = *hidxp;
-	}
+
+	/*
+	 * Ensure that we do not read the hidx before we read the PTE. Because
+	 * the writer side is expected to finish writing the hidx first followed
+	 * by the PTE, by using smp_wmb(). pte_set_hash_slot() ensures that.
+	 */
+	smp_rmb();
+
+	hidxp = (unsigned long *)(ptep + offset);
+	rpte.hidx = *hidxp;
 	return rpte;
 }
 
+/*
+ * shift the hidx representation by one-modulo-0xf; i.e hidx 0 is respresented
+ * as 1, 1 as 2,... , and 0xf as 0.  This convention lets us represent a
+ * invalid hidx 0xf with a 0x0 bit value. PTEs are anyway zero'd when
+ * allocated. We dont have to zero them gain; thus save on the initialization.
+ */
+#define HIDX_UNSHIFT_BY_ONE(x) ((x + 0xfUL) & 0xfUL) /* shift backward by one */
+#define HIDX_SHIFT_BY_ONE(x) ((x + 0x1UL) & 0xfUL)   /* shift forward by one */
+#define HIDX_BITS(x, index)  (x << (index << 2))
+#define BITS_TO_HIDX(x, index)  ((x >> (index << 2)) & 0xfUL)
+#define INVALID_RPTE_HIDX  0x0UL
+
 static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
 {
-	if ((pte_val(rpte.pte) & H_PAGE_COMBO))
-		return (rpte.hidx >> (index<<2)) & 0xf;
-	return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf;
+	return HIDX_UNSHIFT_BY_ONE(BITS_TO_HIDX(rpte.hidx, index));
+}
+
+/*
+ * Commit the hidx and return PTE bits that needs to be modified. The caller is
+ * expected to modify the PTE bits accordingly and commit the PTE to memory.
+ */
+static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
+					 unsigned int subpg_index,
+					 unsigned long hidx, int offset)
+{
+	unsigned long *hidxp = (unsigned long *)(ptep + offset);
+
+	rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index);
+	*hidxp = rpte.hidx  | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index);
+
+	/*
+	 * Anyone reading PTE must ensure hidx bits are read after reading the
+	 * PTE by using the read-side barrier smp_rmb(). __real_pte() can be
+	 * used for that.
+	 */
+	smp_wmb();
+
+	/* No PTE bits to be modified, return 0x0UL */
+	return 0x0UL;
 }
 
 #define __rpte_to_pte(r)	((r).pte)
@@ -111,13 +141,18 @@ static inline int hash__remap_4k_pfn(struct vm_area_struct *vma, unsigned long a
 }
 
 #define H_PTE_TABLE_SIZE	PTE_FRAG_SIZE
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined (CONFIG_HUGETLB_PAGE)
 #define H_PMD_TABLE_SIZE	((sizeof(pmd_t) << PMD_INDEX_SIZE) + \
 				 (sizeof(unsigned long) << PMD_INDEX_SIZE))
 #else
 #define H_PMD_TABLE_SIZE	(sizeof(pmd_t) << PMD_INDEX_SIZE)
 #endif
+#ifdef CONFIG_HUGETLB_PAGE
+#define H_PUD_TABLE_SIZE	((sizeof(pud_t) << PUD_INDEX_SIZE) +	\
+				 (sizeof(unsigned long) << PUD_INDEX_SIZE))
+#else
 #define H_PUD_TABLE_SIZE	(sizeof(pud_t) << PUD_INDEX_SIZE)
+#endif
 #define H_PGD_TABLE_SIZE	(sizeof(pgd_t) << PGD_INDEX_SIZE)
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -203,8 +238,6 @@ extern pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma,
 extern void hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 					 pgtable_t pgtable);
 extern pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
-extern void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
-				      unsigned long address, pmd_t *pmdp);
 extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm,
 				       unsigned long addr, pmd_t *pmdp);
 extern int hash__has_transparent_hugepage(void);
diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h
index ecb1239d74f4..935adcd92a81 100644
--- a/arch/powerpc/include/asm/book3s/64/hash.h
+++ b/arch/powerpc/include/asm/book3s/64/hash.h
@@ -9,11 +9,6 @@
  *
  */
 #define H_PTE_NONE_MASK		_PAGE_HPTEFLAGS
-#define H_PAGE_F_GIX_SHIFT	56
-#define H_PAGE_BUSY		_RPAGE_RSV1 /* software: PTE & hash are busy */
-#define H_PAGE_F_SECOND		_RPAGE_RSV2	/* HPTE is in 2ndary HPTEG */
-#define H_PAGE_F_GIX		(_RPAGE_RSV3 | _RPAGE_RSV4 | _RPAGE_RPN44)
-#define H_PAGE_HASHPTE		_RPAGE_RPN43	/* PTE has associated HPTE */
 
 #ifdef CONFIG_PPC_64K_PAGES
 #include <asm/book3s/64/hash-64k.h>
@@ -28,7 +23,8 @@
 				 H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT)
 #define H_PGTABLE_RANGE		(ASM_CONST(1) << H_PGTABLE_EADDR_SIZE)
 
-#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&  defined(CONFIG_PPC_64K_PAGES)
+#if (defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE)) && \
+	defined(CONFIG_PPC_64K_PAGES)
 /*
  * only with hash 64k we need to use the second half of pmd page table
  * to store pointer to deposited pgtable_t
@@ -37,6 +33,16 @@
 #else
 #define H_PMD_CACHE_INDEX	H_PMD_INDEX_SIZE
 #endif
+/*
+ * We store the slot details in the second half of page table.
+ * Increase the pud level table so that hugetlb ptes can be stored
+ * at pud level.
+ */
+#if defined(CONFIG_HUGETLB_PAGE) &&  defined(CONFIG_PPC_64K_PAGES)
+#define H_PUD_CACHE_INDEX	(H_PUD_INDEX_SIZE + 1)
+#else
+#define H_PUD_CACHE_INDEX	(H_PUD_INDEX_SIZE)
+#endif
 /*
  * Define the address range of the kernel non-linear virtual area
  */
@@ -167,6 +173,9 @@ static inline int hash__pte_none(pte_t pte)
 	return (pte_val(pte) & ~H_PTE_NONE_MASK) == 0;
 }
 
+unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift,
+		int ssize, real_pte_t rpte, unsigned int subpg_index);
+
 /* This low level function performs the actual PTE insertion
  * Setting the PTE depends on the MMU type and other factors. It's
  * an horrible mess that I'm not going to try to clean up now but
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index e91e115a816f..50ed64fba4ae 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -90,6 +90,8 @@
 #define HPTE_R_PP0		ASM_CONST(0x8000000000000000)
 #define HPTE_R_TS		ASM_CONST(0x4000000000000000)
 #define HPTE_R_KEY_HI		ASM_CONST(0x3000000000000000)
+#define HPTE_R_KEY_BIT0		ASM_CONST(0x2000000000000000)
+#define HPTE_R_KEY_BIT1		ASM_CONST(0x1000000000000000)
 #define HPTE_R_RPN_SHIFT	12
 #define HPTE_R_RPN		ASM_CONST(0x0ffffffffffff000)
 #define HPTE_R_RPN_3_0		ASM_CONST(0x01fffffffffff000)
@@ -104,6 +106,9 @@
 #define HPTE_R_C		ASM_CONST(0x0000000000000080)
 #define HPTE_R_R		ASM_CONST(0x0000000000000100)
 #define HPTE_R_KEY_LO		ASM_CONST(0x0000000000000e00)
+#define HPTE_R_KEY_BIT2		ASM_CONST(0x0000000000000800)
+#define HPTE_R_KEY_BIT3		ASM_CONST(0x0000000000000400)
+#define HPTE_R_KEY_BIT4		ASM_CONST(0x0000000000000200)
 #define HPTE_R_KEY		(HPTE_R_KEY_LO | HPTE_R_KEY_HI)
 
 #define HPTE_V_1TB_SEG		ASM_CONST(0x4000000000000000)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
index c9448e19847a..37671feb2bf6 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu.h
@@ -87,6 +87,9 @@ typedef struct {
 	/* Number of bits in the mm_cpumask */
 	atomic_t active_cpus;
 
+	/* Number of users of the external (Nest) MMU */
+	atomic_t copros;
+
 	/* NPU NMMU context */
 	struct npu_context *npu_context;
 
@@ -108,6 +111,16 @@ typedef struct {
 #ifdef CONFIG_SPAPR_TCE_IOMMU
 	struct list_head iommu_group_mem_list;
 #endif
+
+#ifdef CONFIG_PPC_MEM_KEYS
+	/*
+	 * Each bit represents one protection key.
+	 * bit set   -> key allocated
+	 * bit unset -> key available for allocation
+	 */
+	u32 pkey_allocation_map;
+	s16 execute_only_pkey; /* key holding execute-only protection */
+#endif
 } mm_context_t;
 
 /*
diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h
index 1fcfa425cefa..4746bc68d446 100644
--- a/arch/powerpc/include/asm/book3s/64/pgalloc.h
+++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h
@@ -73,10 +73,16 @@ static inline void radix__pgd_free(struct mm_struct *mm, pgd_t *pgd)
 
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
+	pgd_t *pgd;
+
 	if (radix_enabled())
 		return radix__pgd_alloc(mm);
-	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
-		pgtable_gfp_flags(mm, GFP_KERNEL));
+
+	pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE),
+			       pgtable_gfp_flags(mm, GFP_KERNEL));
+	memset(pgd, 0, PGD_TABLE_SIZE);
+
+	return pgd;
 }
 
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
@@ -93,13 +99,13 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
 
 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
-	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
+	return kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX),
 		pgtable_gfp_flags(mm, GFP_KERNEL));
 }
 
 static inline void pud_free(struct mm_struct *mm, pud_t *pud)
 {
-	kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud);
+	kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud);
 }
 
 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
@@ -115,7 +121,7 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
 	 * ahead and flush the page walk cache
 	 */
 	flush_tlb_pgtable(tlb, address);
-        pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE);
+	pgtable_free_tlb(tlb, pud, PUD_CACHE_INDEX);
 }
 
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 44697817ccc6..a6b9f1d74600 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -14,8 +14,9 @@
  */
 #define _PAGE_BIT_SWAP_TYPE	0
 
+#define _PAGE_NA		0
 #define _PAGE_RO		0
-#define _PAGE_SHARED		0
+#define _PAGE_USER		0
 
 #define _PAGE_EXEC		0x00001 /* execute permission */
 #define _PAGE_WRITE		0x00002 /* write access allowed */
@@ -39,6 +40,7 @@
 #define _RPAGE_RSV2		0x0800000000000000UL
 #define _RPAGE_RSV3		0x0400000000000000UL
 #define _RPAGE_RSV4		0x0200000000000000UL
+#define _RPAGE_RSV5		0x00040UL
 
 #define _PAGE_PTE		0x4000000000000000UL	/* distinguishes PTEs from pointers */
 #define _PAGE_PRESENT		0x8000000000000000UL	/* pte contains a translation */
@@ -58,6 +60,25 @@
 /* Max physical address bit as per radix table */
 #define _RPAGE_PA_MAX		57
 
+#ifdef CONFIG_PPC_MEM_KEYS
+#ifdef CONFIG_PPC_64K_PAGES
+#define H_PTE_PKEY_BIT0	_RPAGE_RSV1
+#define H_PTE_PKEY_BIT1	_RPAGE_RSV2
+#else /* CONFIG_PPC_64K_PAGES */
+#define H_PTE_PKEY_BIT0	0 /* _RPAGE_RSV1 is not available */
+#define H_PTE_PKEY_BIT1	0 /* _RPAGE_RSV2 is not available */
+#endif /* CONFIG_PPC_64K_PAGES */
+#define H_PTE_PKEY_BIT2	_RPAGE_RSV3
+#define H_PTE_PKEY_BIT3	_RPAGE_RSV4
+#define H_PTE_PKEY_BIT4	_RPAGE_RSV5
+#else /*  CONFIG_PPC_MEM_KEYS */
+#define H_PTE_PKEY_BIT0	0
+#define H_PTE_PKEY_BIT1	0
+#define H_PTE_PKEY_BIT2	0
+#define H_PTE_PKEY_BIT3	0
+#define H_PTE_PKEY_BIT4	0
+#endif /*  CONFIG_PPC_MEM_KEYS */
+
 /*
  * Max physical address bit we will use for now.
  *
@@ -121,13 +142,16 @@
 #define _PAGE_CHG_MASK	(PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \
 			 _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE |	\
 			 _PAGE_SOFT_DIRTY)
+
+#define H_PTE_PKEY  (H_PTE_PKEY_BIT0 | H_PTE_PKEY_BIT1 | H_PTE_PKEY_BIT2 | \
+		     H_PTE_PKEY_BIT3 | H_PTE_PKEY_BIT4)
 /*
  * Mask of bits returned by pte_pgprot()
  */
 #define PAGE_PROT_BITS  (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT | \
 			 H_PAGE_4K_PFN | _PAGE_PRIVILEGED | _PAGE_ACCESSED | \
 			 _PAGE_READ | _PAGE_WRITE |  _PAGE_DIRTY | _PAGE_EXEC | \
-			 _PAGE_SOFT_DIRTY)
+			 _PAGE_SOFT_DIRTY | H_PTE_PKEY)
 /*
  * We define 2 sets of base prot bits, one for basic pages (ie,
  * cacheable kernel and user pages) and one for non cacheable
@@ -208,11 +232,13 @@ extern unsigned long __pmd_index_size;
 extern unsigned long __pud_index_size;
 extern unsigned long __pgd_index_size;
 extern unsigned long __pmd_cache_index;
+extern unsigned long __pud_cache_index;
 #define PTE_INDEX_SIZE  __pte_index_size
 #define PMD_INDEX_SIZE  __pmd_index_size
 #define PUD_INDEX_SIZE  __pud_index_size
 #define PGD_INDEX_SIZE  __pgd_index_size
 #define PMD_CACHE_INDEX __pmd_cache_index
+#define PUD_CACHE_INDEX __pud_cache_index
 /*
  * Because of use of pte fragments and THP, size of page table
  * are not always derived out of index size above.
@@ -324,7 +350,7 @@ extern unsigned long pci_io_base;
  */
 #ifndef __real_pte
 
-#define __real_pte(e,p)		((real_pte_t){(e)})
+#define __real_pte(e, p, o)		((real_pte_t){(e)})
 #define __rpte_to_pte(r)	((r).pte)
 #define __rpte_to_hidx(r,index)	(pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT)
 
@@ -546,6 +572,40 @@ static inline int pte_present(pte_t pte)
 {
 	return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT));
 }
+
+#ifdef CONFIG_PPC_MEM_KEYS
+extern bool arch_pte_access_permitted(u64 pte, bool write, bool execute);
+#else
+static inline bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
+{
+	return true;
+}
+#endif /* CONFIG_PPC_MEM_KEYS */
+
+#define pte_access_permitted pte_access_permitted
+static inline bool pte_access_permitted(pte_t pte, bool write)
+{
+	unsigned long pteval = pte_val(pte);
+	/* Also check for pte_user */
+	unsigned long clear_pte_bits = _PAGE_PRIVILEGED;
+	/*
+	 * _PAGE_READ is needed for any access and will be
+	 * cleared for PROT_NONE
+	 */
+	unsigned long need_pte_bits = _PAGE_PRESENT | _PAGE_READ;
+
+	if (write)
+		need_pte_bits |= _PAGE_WRITE;
+
+	if ((pteval & need_pte_bits) != need_pte_bits)
+		return false;
+
+	if ((pteval & clear_pte_bits) == clear_pte_bits)
+		return false;
+
+	return arch_pte_access_permitted(pte_val(pte), write, 0);
+}
+
 /*
  * Conversion functions: convert a page and protection to a page entry,
  * and a page entry and page directory to the page they refer to.
@@ -850,6 +910,11 @@ static inline int pud_bad(pud_t pud)
 	return hash__pud_bad(pud);
 }
 
+#define pud_access_permitted pud_access_permitted
+static inline bool pud_access_permitted(pud_t pud, bool write)
+{
+	return pte_access_permitted(pud_pte(pud), write);
+}
 
 #define pgd_write(pgd)		pte_write(pgd_pte(pgd))
 static inline void pgd_set(pgd_t *pgdp, unsigned long val)
@@ -889,6 +954,12 @@ static inline int pgd_bad(pgd_t pgd)
 	return hash__pgd_bad(pgd);
 }
 
+#define pgd_access_permitted pgd_access_permitted
+static inline bool pgd_access_permitted(pgd_t pgd, bool write)
+{
+	return pte_access_permitted(pgd_pte(pgd), write);
+}
+
 extern struct page *pgd_page(pgd_t pgd);
 
 /* Pointers in the page table tree are physical addresses */
@@ -1009,6 +1080,12 @@ static inline int pmd_protnone(pmd_t pmd)
 #define __pmd_write(pmd)	__pte_write(pmd_pte(pmd))
 #define pmd_savedwrite(pmd)	pte_savedwrite(pmd_pte(pmd))
 
+#define pmd_access_permitted pmd_access_permitted
+static inline bool pmd_access_permitted(pmd_t pmd, bool write)
+{
+	return pte_access_permitted(pmd_pte(pmd), write);
+}
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot);
 extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot);
@@ -1137,17 +1214,8 @@ static inline pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm,
 }
 
 #define __HAVE_ARCH_PMDP_INVALIDATE
-extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
-			    pmd_t *pmdp);
-
-#define __HAVE_ARCH_PMDP_HUGE_SPLIT_PREPARE
-static inline void pmdp_huge_split_prepare(struct vm_area_struct *vma,
-					   unsigned long address, pmd_t *pmdp)
-{
-	if (radix_enabled())
-		return radix__pmdp_huge_split_prepare(vma, address, pmdp);
-	return hash__pmdp_huge_split_prepare(vma, address, pmdp);
-}
+extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
+			     pmd_t *pmdp);
 
 #define pmd_move_must_withdraw pmd_move_must_withdraw
 struct spinlock;
diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h
index 19c44e1495ae..365010f66570 100644
--- a/arch/powerpc/include/asm/book3s/64/radix.h
+++ b/arch/powerpc/include/asm/book3s/64/radix.h
@@ -269,12 +269,6 @@ static inline pmd_t radix__pmd_mkhuge(pmd_t pmd)
 		return __pmd(pmd_val(pmd) | _PAGE_PTE | R_PAGE_LARGE);
 	return __pmd(pmd_val(pmd) | _PAGE_PTE);
 }
-static inline void radix__pmdp_huge_split_prepare(struct vm_area_struct *vma,
-					    unsigned long address, pmd_t *pmdp)
-{
-	/* Nothing to do for radix. */
-	return;
-}
 
 extern unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
 					  pmd_t *pmdp, unsigned long clr,
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
index 849ecaae9e79..64d02a704bcb 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
@@ -51,6 +51,7 @@ static inline void arch_leave_lazy_mmu_mode(void)
 
 #define arch_flush_lazy_mmu_mode()      do {} while (0)
 
+extern void hash__tlbiel_all(unsigned int action);
 
 extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,
 			    int ssize, unsigned long flags);
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 6a9e68003387..19b45ba6caf9 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -11,6 +11,12 @@ static inline int mmu_get_ap(int psize)
 	return mmu_psize_defs[psize].ap;
 }
 
+#ifdef CONFIG_PPC_RADIX_MMU
+extern void radix__tlbiel_all(unsigned int action);
+#else
+static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); };
+#endif
+
 extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
 					   unsigned long start, unsigned long end);
 extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
@@ -41,10 +47,8 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad
 #endif
 extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
 extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
-extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
-				     unsigned long page_size);
-extern void radix__flush_tlb_lpid(unsigned long lpid);
 extern void radix__flush_tlb_all(void);
 extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
 					unsigned long address);
+
 #endif
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index 58b576f654b3..0cac17253513 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -8,6 +8,44 @@
 #include <asm/book3s/64/tlbflush-hash.h>
 #include <asm/book3s/64/tlbflush-radix.h>
 
+/* TLB flush actions. Used as argument to tlbiel_all() */
+enum {
+	TLB_INVAL_SCOPE_GLOBAL = 0,	/* invalidate all TLBs */
+	TLB_INVAL_SCOPE_LPID = 1,	/* invalidate TLBs for current LPID */
+};
+
+#ifdef CONFIG_PPC_NATIVE
+static inline void tlbiel_all(void)
+{
+	/*
+	 * This is used for host machine check and bootup.
+	 *
+	 * This uses early_radix_enabled and implementations use
+	 * early_cpu_has_feature etc because that works early in boot
+	 * and this is the machine check path which is not performance
+	 * critical.
+	 */
+	if (early_radix_enabled())
+		radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
+	else
+		hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
+}
+#else
+static inline void tlbiel_all(void) { BUG(); };
+#endif
+
+static inline void tlbiel_all_lpid(bool radix)
+{
+	/*
+	 * This is used for guest machine check.
+	 */
+	if (radix)
+		radix__tlbiel_all(TLB_INVAL_SCOPE_LPID);
+	else
+		hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
+}
+
+
 #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
 static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
 				       unsigned long start, unsigned long end)
diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h
index 3c04249bcf39..fd06dbe7d7d3 100644
--- a/arch/powerpc/include/asm/bug.h
+++ b/arch/powerpc/include/asm/bug.h
@@ -133,9 +133,11 @@ struct pt_regs;
 extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long);
 extern void bad_page_fault(struct pt_regs *, unsigned long, int);
 extern void _exception(int, struct pt_regs *, int, unsigned long);
+extern void _exception_pkey(int, struct pt_regs *, int, unsigned long, int);
 extern void die(const char *, struct pt_regs *, long);
 extern bool die_will_crash(void);
-
+extern void panic_flush_kmsg_start(void);
+extern void panic_flush_kmsg_end(void);
 #endif /* !__ASSEMBLY__ */
 
 #endif /* __KERNEL__ */
diff --git a/arch/powerpc/include/asm/code-patching.h b/arch/powerpc/include/asm/code-patching.h
index abef812de7f8..812535f40124 100644
--- a/arch/powerpc/include/asm/code-patching.h
+++ b/arch/powerpc/include/asm/code-patching.h
@@ -31,8 +31,10 @@ unsigned int create_cond_branch(const unsigned int *addr,
 				unsigned long target, int flags);
 int patch_branch(unsigned int *addr, unsigned long target, int flags);
 int patch_instruction(unsigned int *addr, unsigned int instr);
+int raw_patch_instruction(unsigned int *addr, unsigned int instr);
 
 int instr_is_relative_branch(unsigned int instr);
+int instr_is_relative_link_branch(unsigned int instr);
 int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr);
 unsigned long branch_target(const unsigned int *instr);
 unsigned int translate_branch(const unsigned int *dest,
diff --git a/arch/powerpc/include/asm/compat.h b/arch/powerpc/include/asm/compat.h
index 8a2aecfe9b02..62168e1158f1 100644
--- a/arch/powerpc/include/asm/compat.h
+++ b/arch/powerpc/include/asm/compat.h
@@ -119,71 +119,6 @@ typedef u32		compat_old_sigset_t;
 
 typedef u32		compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-#define SI_PAD_SIZE32	(128/sizeof(int) - 3)
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[SI_PAD_SIZE32];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			__compat_uid_t _uid;		/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;		/* timer id */
-			int _overrun;			/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			__compat_uid_t _uid;		/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;		/* which child */
-			__compat_uid_t _uid;		/* sender's uid */
-			int _status;			/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */
-		struct {
-			unsigned int _addr; /* faulting insn/memory ref. */
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		/* SIGSYS */
-		struct {
-			unsigned int _call_addr; /* calling insn */
-			int _syscall;		 /* triggering system call number */
-			unsigned int _arch;	 /* AUDIT_ARCH_* of syscall */
-		} _sigsys;
-	} _sifields;
-} compat_siginfo_t;
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 /*
diff --git a/arch/powerpc/include/asm/cpm.h b/arch/powerpc/include/asm/cpm.h
index b925df1b87d0..4c24ea8209bb 100644
--- a/arch/powerpc/include/asm/cpm.h
+++ b/arch/powerpc/include/asm/cpm.h
@@ -166,6 +166,6 @@ static inline int cpm_command(u32 command, u8 opcode)
 }
 #endif /* CONFIG_CPM */
 
-int cpm2_gpiochip_add32(struct device_node *np);
+int cpm2_gpiochip_add32(struct device *dev);
 
 #endif
diff --git a/arch/powerpc/include/asm/cpm1.h b/arch/powerpc/include/asm/cpm1.h
index 3db821876d48..a116fe931789 100644
--- a/arch/powerpc/include/asm/cpm1.h
+++ b/arch/powerpc/include/asm/cpm1.h
@@ -605,5 +605,7 @@ enum cpm_clk {
 };
 
 int cpm1_clk_setup(enum cpm_clk_target target, int clock, int mode);
+int cpm1_gpiochip_add16(struct device *dev);
+int cpm1_gpiochip_add32(struct device *dev);
 
 #endif /* __CPM1__ */
diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index 0546663a98db..2e2bacbdf6ed 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -107,12 +107,6 @@ struct cpu_spec {
 	 * called in real mode to handle SLB and TLB errors.
 	 */
 	long		(*machine_check_early)(struct pt_regs *regs);
-
-	/*
-	 * Processor specific routine to flush tlbs.
-	 */
-	void		(*flush_tlb)(unsigned int action);
-
 };
 
 extern struct cpu_spec		*cur_cpu_spec;
@@ -133,12 +127,6 @@ extern void cpu_feature_keys_init(void);
 static inline void cpu_feature_keys_init(void) { }
 #endif
 
-/* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */
-enum {
-	TLB_INVAL_SCOPE_GLOBAL = 0,	/* invalidate all TLBs */
-	TLB_INVAL_SCOPE_LPID = 1,	/* invalidate TLBs for current LPID */
-};
-
 #endif /* __ASSEMBLY__ */
 
 /* CPU kernel features */
@@ -207,7 +195,7 @@ enum {
 #define CPU_FTR_STCX_CHECKS_ADDRESS	LONG_ASM_CONST(0x0004000000000000)
 #define CPU_FTR_POPCNTB			LONG_ASM_CONST(0x0008000000000000)
 #define CPU_FTR_POPCNTD			LONG_ASM_CONST(0x0010000000000000)
-/* Free					LONG_ASM_CONST(0x0020000000000000) */
+#define CPU_FTR_PKEY			LONG_ASM_CONST(0x0020000000000000)
 #define CPU_FTR_VMX_COPY		LONG_ASM_CONST(0x0040000000000000)
 #define CPU_FTR_TM			LONG_ASM_CONST(0x0080000000000000)
 #define CPU_FTR_CFAR			LONG_ASM_CONST(0x0100000000000000)
@@ -215,6 +203,7 @@ enum {
 #define CPU_FTR_DAWR			LONG_ASM_CONST(0x0400000000000000)
 #define CPU_FTR_DABRX			LONG_ASM_CONST(0x0800000000000000)
 #define CPU_FTR_PMAO_BUG		LONG_ASM_CONST(0x1000000000000000)
+#define CPU_FTR_P9_TLBIE_BUG		LONG_ASM_CONST(0x2000000000000000)
 #define CPU_FTR_POWER9_DD1		LONG_ASM_CONST(0x4000000000000000)
 #define CPU_FTR_POWER9_DD2_1		LONG_ASM_CONST(0x8000000000000000)
 
@@ -454,7 +443,7 @@ enum {
 	    CPU_FTR_DSCR | CPU_FTR_SAO  | CPU_FTR_ASYM_SMT | \
 	    CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
 	    CPU_FTR_CFAR | CPU_FTR_HVMODE | \
-	    CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX)
+	    CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX | CPU_FTR_PKEY)
 #define CPU_FTRS_POWER8 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
 	    CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\
 	    CPU_FTR_MMCRA | CPU_FTR_SMT | \
@@ -464,7 +453,7 @@ enum {
 	    CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
 	    CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
 	    CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
-	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP)
+	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_PKEY)
 #define CPU_FTRS_POWER8E (CPU_FTRS_POWER8 | CPU_FTR_PMAO_BUG)
 #define CPU_FTRS_POWER8_DD1 (CPU_FTRS_POWER8 & ~CPU_FTR_DBELL)
 #define CPU_FTRS_POWER9 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
@@ -476,7 +465,8 @@ enum {
 	    CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
 	    CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
 	    CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
-	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300)
+	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \
+	    CPU_FTR_PKEY | CPU_FTR_P9_TLBIE_BUG)
 #define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
 			     (~CPU_FTR_SAO))
 #define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9
diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h
index 14e71ff6579e..fc97404de0a3 100644
--- a/arch/powerpc/include/asm/debug.h
+++ b/arch/powerpc/include/asm/debug.h
@@ -49,7 +49,7 @@ void set_breakpoint(struct arch_hw_breakpoint *brk);
 void __set_breakpoint(struct arch_hw_breakpoint *brk);
 #ifdef CONFIG_PPC_ADV_DEBUG_REGS
 extern void do_send_trap(struct pt_regs *regs, unsigned long address,
-			 unsigned long error_code, int signal_code, int brkpt);
+			 unsigned long error_code, int brkpt);
 #else
 
 extern void do_break(struct pt_regs *regs, unsigned long address,
diff --git a/arch/powerpc/include/asm/dma-direct.h b/arch/powerpc/include/asm/dma-direct.h
new file mode 100644
index 000000000000..a5b59c765426
--- /dev/null
+++ b/arch/powerpc/include/asm/dma-direct.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef ASM_POWERPC_DMA_DIRECT_H
+#define ASM_POWERPC_DMA_DIRECT_H 1
+
+static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
+{
+#ifdef CONFIG_SWIOTLB
+	struct dev_archdata *sd = &dev->archdata;
+
+	if (sd->max_direct_dma_addr && addr + size > sd->max_direct_dma_addr)
+		return false;
+#endif
+
+	if (!dev->dma_mask)
+		return false;
+
+	return addr + size - 1 <= *dev->dma_mask;
+}
+
+static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
+{
+	return paddr + get_dma_offset(dev);
+}
+
+static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
+{
+	return daddr - get_dma_offset(dev);
+}
+#endif /* ASM_POWERPC_DMA_DIRECT_H */
diff --git a/arch/powerpc/include/asm/dma-mapping.h b/arch/powerpc/include/asm/dma-mapping.h
index 5a6cbe11db6f..8fa394520af6 100644
--- a/arch/powerpc/include/asm/dma-mapping.h
+++ b/arch/powerpc/include/asm/dma-mapping.h
@@ -19,13 +19,13 @@
 #include <asm/swiotlb.h>
 
 /* Some dma direct funcs must be visible for use in other dma_ops */
-extern void *__dma_direct_alloc_coherent(struct device *dev, size_t size,
+extern void *__dma_nommu_alloc_coherent(struct device *dev, size_t size,
 					 dma_addr_t *dma_handle, gfp_t flag,
 					 unsigned long attrs);
-extern void __dma_direct_free_coherent(struct device *dev, size_t size,
+extern void __dma_nommu_free_coherent(struct device *dev, size_t size,
 				       void *vaddr, dma_addr_t dma_handle,
 				       unsigned long attrs);
-extern int dma_direct_mmap_coherent(struct device *dev,
+extern int dma_nommu_mmap_coherent(struct device *dev,
 				    struct vm_area_struct *vma,
 				    void *cpu_addr, dma_addr_t handle,
 				    size_t size, unsigned long attrs);
@@ -73,7 +73,7 @@ static inline unsigned long device_to_mask(struct device *dev)
 #ifdef CONFIG_PPC64
 extern struct dma_map_ops dma_iommu_ops;
 #endif
-extern const struct dma_map_ops dma_direct_ops;
+extern const struct dma_map_ops dma_nommu_ops;
 
 static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 {
@@ -107,39 +107,11 @@ static inline void set_dma_offset(struct device *dev, dma_addr_t off)
 		dev->archdata.dma_offset = off;
 }
 
-/* this will be removed soon */
-#define flush_write_buffers()
-
 #define HAVE_ARCH_DMA_SET_MASK 1
 extern int dma_set_mask(struct device *dev, u64 dma_mask);
 
 extern u64 __dma_get_required_mask(struct device *dev);
 
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-#ifdef CONFIG_SWIOTLB
-	struct dev_archdata *sd = &dev->archdata;
-
-	if (sd->max_direct_dma_addr && addr + size > sd->max_direct_dma_addr)
-		return false;
-#endif
-
-	if (!dev->dma_mask)
-		return false;
-
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return paddr + get_dma_offset(dev);
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
-{
-	return daddr - get_dma_offset(dev);
-}
-
 #define ARCH_HAS_DMA_MMAP_COHERENT
 
 #endif /* __KERNEL__ */
diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h
new file mode 100644
index 000000000000..ce242b9ea8c6
--- /dev/null
+++ b/arch/powerpc/include/asm/drmem.h
@@ -0,0 +1,102 @@
+/*
+ * drmem.h: Power specific logical memory block representation
+ *
+ * Copyright 2017 IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef _ASM_POWERPC_LMB_H
+#define _ASM_POWERPC_LMB_H
+
+struct drmem_lmb {
+	u64     base_addr;
+	u32     drc_index;
+	u32     aa_index;
+	u32     flags;
+};
+
+struct drmem_lmb_info {
+	struct drmem_lmb        *lmbs;
+	int                     n_lmbs;
+	u32                     lmb_size;
+};
+
+extern struct drmem_lmb_info *drmem_info;
+
+#define for_each_drmem_lmb_in_range(lmb, start, end)		\
+	for ((lmb) = (start); (lmb) <= (end); (lmb)++)
+
+#define for_each_drmem_lmb(lmb)					\
+	for_each_drmem_lmb_in_range((lmb),			\
+		&drmem_info->lmbs[0],				\
+		&drmem_info->lmbs[drmem_info->n_lmbs - 1])
+
+/*
+ * The of_drconf_cell_v1 struct defines the layout of the LMB data
+ * specified in the ibm,dynamic-memory device tree property.
+ * The property itself is a 32-bit value specifying the number of
+ * LMBs followed by an array of of_drconf_cell_v1 entries, one
+ * per LMB.
+ */
+struct of_drconf_cell_v1 {
+	__be64	base_addr;
+	__be32	drc_index;
+	__be32	reserved;
+	__be32	aa_index;
+	__be32	flags;
+};
+
+/*
+ * Version 2 of the ibm,dynamic-memory property is defined as a
+ * 32-bit value specifying the number of LMB sets followed by an
+ * array of of_drconf_cell_v2 entries, one per LMB set.
+ */
+struct of_drconf_cell_v2 {
+	u32	seq_lmbs;
+	u64	base_addr;
+	u32	drc_index;
+	u32	aa_index;
+	u32	flags;
+} __packed;
+
+#define DRCONF_MEM_ASSIGNED	0x00000008
+#define DRCONF_MEM_AI_INVALID	0x00000040
+#define DRCONF_MEM_RESERVED	0x00000080
+
+static inline u32 drmem_lmb_size(void)
+{
+	return drmem_info->lmb_size;
+}
+
+#define DRMEM_LMB_RESERVED	0x80000000
+
+static inline void drmem_mark_lmb_reserved(struct drmem_lmb *lmb)
+{
+	lmb->flags |= DRMEM_LMB_RESERVED;
+}
+
+static inline void drmem_remove_lmb_reservation(struct drmem_lmb *lmb)
+{
+	lmb->flags &= ~DRMEM_LMB_RESERVED;
+}
+
+static inline bool drmem_lmb_reserved(struct drmem_lmb *lmb)
+{
+	return lmb->flags & DRMEM_LMB_RESERVED;
+}
+
+u64 drmem_lmb_memory_max(void);
+void __init walk_drmem_lmbs(struct device_node *dn,
+			void (*func)(struct drmem_lmb *, const __be32 **));
+int drmem_update_dt(void);
+
+#ifdef CONFIG_PPC_PSERIES
+void __init walk_drmem_lmbs_early(unsigned long node,
+			void (*func)(struct drmem_lmb *, const __be32 **));
+#endif
+
+#endif /* _ASM_POWERPC_LMB_H */
diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
index 5161c37dd039..fd37cc101f4f 100644
--- a/arch/powerpc/include/asm/eeh.h
+++ b/arch/powerpc/include/asm/eeh.h
@@ -214,6 +214,7 @@ struct eeh_ops {
 	int (*write_config)(struct pci_dn *pdn, int where, int size, u32 val);
 	int (*next_error)(struct eeh_pe **pe);
 	int (*restore_config)(struct pci_dn *pdn);
+	int (*notify_resume)(struct pci_dn *pdn);
 };
 
 extern int eeh_subsystem_flags;
@@ -297,6 +298,7 @@ int eeh_pe_reset(struct eeh_pe *pe, int option);
 int eeh_pe_configure(struct eeh_pe *pe);
 int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func,
 		      unsigned long addr, unsigned long mask);
+int eeh_restore_vf_config(struct pci_dn *pdn);
 
 /**
  * EEH_POSSIBLE_ERROR() -- test for possible MMIO failure.
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 7197b179c1b1..471b2274fbeb 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -251,18 +251,40 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	std	r10,area+EX_R10(r13);	/* save r10 - r12 */		\
 	OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)
 
-#define __EXCEPTION_PROLOG_1(area, extra, vec)				\
+#define __EXCEPTION_PROLOG_1_PRE(area)					\
 	OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR);		\
 	OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR);		\
 	SAVE_CTR(r10, area);						\
-	mfcr	r9;							\
-	extra(vec);							\
+	mfcr	r9;
+
+#define __EXCEPTION_PROLOG_1_POST(area)					\
 	std	r11,area+EX_R11(r13);					\
 	std	r12,area+EX_R12(r13);					\
 	GET_SCRATCH0(r10);						\
 	std	r10,area+EX_R13(r13)
+
+/*
+ * This version of the EXCEPTION_PROLOG_1 will carry
+ * addition parameter called "bitmask" to support
+ * checking of the interrupt maskable level in the SOFTEN_TEST.
+ * Intended to be used in MASKABLE_EXCPETION_* macros.
+ */
+#define MASKABLE_EXCEPTION_PROLOG_1(area, extra, vec, bitmask)			\
+	__EXCEPTION_PROLOG_1_PRE(area);					\
+	extra(vec, bitmask);						\
+	__EXCEPTION_PROLOG_1_POST(area);
+
+/*
+ * This version of the EXCEPTION_PROLOG_1 is intended
+ * to be used in STD_EXCEPTION* macros
+ */
+#define _EXCEPTION_PROLOG_1(area, extra, vec)				\
+	__EXCEPTION_PROLOG_1_PRE(area);					\
+	extra(vec);							\
+	__EXCEPTION_PROLOG_1_POST(area);
+
 #define EXCEPTION_PROLOG_1(area, extra, vec)				\
-	__EXCEPTION_PROLOG_1(area, extra, vec)
+	_EXCEPTION_PROLOG_1(area, extra, vec)
 
 #define __EXCEPTION_PROLOG_PSERIES_1(label, h)				\
 	ld	r10,PACAKMSR(r13);	/* get MSR value for kernel */	\
@@ -485,7 +507,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	mflr	r9;			/* Get LR, later save to stack	*/ \
 	ld	r2,PACATOC(r13);	/* get kernel TOC into r2	*/ \
 	std	r9,_LINK(r1);						   \
-	lbz	r10,PACASOFTIRQEN(r13);				   \
+	lbz	r10,PACAIRQSOFTMASK(r13);				   \
 	mfspr	r11,SPRN_XER;		/* save XER in stackframe	*/ \
 	std	r10,SOFTE(r1);						   \
 	std	r11,_XER(r1);						   \
@@ -549,22 +571,23 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 #define SOFTEN_VALUE_0xe80	PACA_IRQ_DBELL
 #define SOFTEN_VALUE_0xe60	PACA_IRQ_HMI
 #define SOFTEN_VALUE_0xea0	PACA_IRQ_EE
+#define SOFTEN_VALUE_0xf00	PACA_IRQ_PMI
 
-#define __SOFTEN_TEST(h, vec)						\
-	lbz	r10,PACASOFTIRQEN(r13);					\
-	cmpwi	r10,0;							\
+#define __SOFTEN_TEST(h, vec, bitmask)					\
+	lbz	r10,PACAIRQSOFTMASK(r13);				\
+	andi.	r10,r10,bitmask;					\
 	li	r10,SOFTEN_VALUE_##vec;					\
-	beq	masked_##h##interrupt
+	bne	masked_##h##interrupt
 
-#define _SOFTEN_TEST(h, vec)	__SOFTEN_TEST(h, vec)
+#define _SOFTEN_TEST(h, vec, bitmask)	__SOFTEN_TEST(h, vec, bitmask)
 
-#define SOFTEN_TEST_PR(vec)						\
+#define SOFTEN_TEST_PR(vec, bitmask)					\
 	KVMTEST(EXC_STD, vec);						\
-	_SOFTEN_TEST(EXC_STD, vec)
+	_SOFTEN_TEST(EXC_STD, vec, bitmask)
 
-#define SOFTEN_TEST_HV(vec)						\
+#define SOFTEN_TEST_HV(vec, bitmask)					\
 	KVMTEST(EXC_HV, vec);						\
-	_SOFTEN_TEST(EXC_HV, vec)
+	_SOFTEN_TEST(EXC_HV, vec, bitmask)
 
 #define KVMTEST_PR(vec)							\
 	KVMTEST(EXC_STD, vec)
@@ -572,53 +595,57 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 #define KVMTEST_HV(vec)							\
 	KVMTEST(EXC_HV, vec)
 
-#define SOFTEN_NOTEST_PR(vec)		_SOFTEN_TEST(EXC_STD, vec)
-#define SOFTEN_NOTEST_HV(vec)		_SOFTEN_TEST(EXC_HV, vec)
+#define SOFTEN_NOTEST_PR(vec, bitmask)	_SOFTEN_TEST(EXC_STD, vec, bitmask)
+#define SOFTEN_NOTEST_HV(vec, bitmask)	_SOFTEN_TEST(EXC_HV, vec, bitmask)
 
-#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)		\
+#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)	\
 	SET_SCRATCH0(r13);    /* save r13 */				\
 	EXCEPTION_PROLOG_0(PACA_EXGEN);					\
-	__EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec);			\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask);	\
 	EXCEPTION_PROLOG_PSERIES_1(label, h);
 
-#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)		\
-	__MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)
+#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)	\
+	__MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)
 
-#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label)			\
+#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label, bitmask)		\
 	_MASKABLE_EXCEPTION_PSERIES(vec, label,				\
-				    EXC_STD, SOFTEN_TEST_PR)
+				    EXC_STD, SOFTEN_TEST_PR, bitmask)
 
-#define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label)			\
-	EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec);		\
+#define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label, bitmask)		\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\
 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD)
 
-#define MASKABLE_EXCEPTION_HV(loc, vec, label)				\
+#define MASKABLE_EXCEPTION_HV(loc, vec, label, bitmask)			\
 	_MASKABLE_EXCEPTION_PSERIES(vec, label,				\
-				    EXC_HV, SOFTEN_TEST_HV)
+				    EXC_HV, SOFTEN_TEST_HV, bitmask)
 
-#define MASKABLE_EXCEPTION_HV_OOL(vec, label)				\
-	EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec);		\
+#define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask)			\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\
 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
 
-#define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra)	\
+#define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \
 	SET_SCRATCH0(r13);    /* save r13 */				\
 	EXCEPTION_PROLOG_0(PACA_EXGEN);					\
-	__EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec);			\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask);	\
 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)
 
-#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra)		\
-	__MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra)
+#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)\
+	__MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)
 
-#define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label)		\
+#define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label, bitmask)	\
 	_MASKABLE_RELON_EXCEPTION_PSERIES(vec, label,			\
-					  EXC_STD, SOFTEN_NOTEST_PR)
+					  EXC_STD, SOFTEN_NOTEST_PR, bitmask)
+
+#define MASKABLE_RELON_EXCEPTION_PSERIES_OOL(vec, label, bitmask)	\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\
+	EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD);
 
-#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label)			\
+#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label, bitmask)		\
 	_MASKABLE_RELON_EXCEPTION_PSERIES(vec, label,			\
-					  EXC_HV, SOFTEN_TEST_HV)
+					  EXC_HV, SOFTEN_TEST_HV, bitmask)
 
-#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label)			\
-	EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec);		\
+#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask)		\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\
 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
 
 /*
diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h
index 8645897472b1..535add3f7791 100644
--- a/arch/powerpc/include/asm/firmware.h
+++ b/arch/powerpc/include/asm/firmware.h
@@ -51,6 +51,8 @@
 #define FW_FEATURE_BEST_ENERGY	ASM_CONST(0x0000000080000000)
 #define FW_FEATURE_TYPE1_AFFINITY ASM_CONST(0x0000000100000000)
 #define FW_FEATURE_PRRN		ASM_CONST(0x0000000200000000)
+#define FW_FEATURE_DRMEM_V2	ASM_CONST(0x0000000400000000)
+#define FW_FEATURE_DRC_INFO	ASM_CONST(0x0000000800000000)
 
 #ifndef __ASSEMBLY__
 
@@ -67,7 +69,8 @@ enum {
 		FW_FEATURE_CMO | FW_FEATURE_VPHN | FW_FEATURE_XCMO |
 		FW_FEATURE_SET_MODE | FW_FEATURE_BEST_ENERGY |
 		FW_FEATURE_TYPE1_AFFINITY | FW_FEATURE_PRRN |
-		FW_FEATURE_HPT_RESIZE,
+		FW_FEATURE_HPT_RESIZE | FW_FEATURE_DRMEM_V2 |
+		FW_FEATURE_DRC_INFO,
 	FW_FEATURE_PSERIES_ALWAYS = 0,
 	FW_FEATURE_POWERNV_POSSIBLE = FW_FEATURE_OPAL,
 	FW_FEATURE_POWERNV_ALWAYS = 0,
diff --git a/arch/powerpc/include/asm/hardirq.h b/arch/powerpc/include/asm/hardirq.h
index 456f9e7b8d83..5986d473722b 100644
--- a/arch/powerpc/include/asm/hardirq.h
+++ b/arch/powerpc/include/asm/hardirq.h
@@ -29,6 +29,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 #define local_softirq_pending()	__this_cpu_read(irq_stat.__softirq_pending)
 
 #define __ARCH_SET_SOFTIRQ_PENDING
+#define __ARCH_IRQ_EXIT_IRQS_DISABLED
 
 #define set_softirq_pending(x) __this_cpu_write(irq_stat.__softirq_pending, (x))
 #define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h
index fdcff76e9a25..7e0e93f24cb7 100644
--- a/arch/powerpc/include/asm/head-64.h
+++ b/arch/powerpc/include/asm/head-64.h
@@ -178,7 +178,7 @@ end_##sname:
  * TRAMP_REAL_*   - real, unrelocated helpers (virt can call these)
  * TRAMP_VIRT_*   - virt, unreloc helpers (in practice, real can use)
  * TRAMP_KVM      - KVM handlers that get put into real, unrelocated
- * EXC_COMMON_*   - virt, relocated common handlers
+ * EXC_COMMON     - virt, relocated common handlers
  *
  * The EXC handlers are given a name, and branch to name_common, or the
  * appropriate KVM or masking function. Vector handler verieties are as
@@ -211,7 +211,6 @@ end_##sname:
  * EXC_COMMON_BEGIN/END - used to open-code the handler
  * EXC_COMMON
  * EXC_COMMON_ASYNC
- * EXC_COMMON_HV
  *
  * TRAMP_REAL and TRAMP_VIRT can be used with BEGIN/END. KVM
  * and OOL handlers are implemented as types of TRAMP and TRAMP_VIRT handlers.
@@ -269,14 +268,14 @@ end_##sname:
 	STD_RELON_EXCEPTION_PSERIES(start, realvec, name##_common);	\
 	EXC_VIRT_END(name, start, size);
 
-#define EXC_REAL_MASKABLE(name, start, size)				\
+#define EXC_REAL_MASKABLE(name, start, size, bitmask)			\
 	EXC_REAL_BEGIN(name, start, size);				\
-	MASKABLE_EXCEPTION_PSERIES(start, start, name##_common);	\
+	MASKABLE_EXCEPTION_PSERIES(start, start, name##_common, bitmask);\
 	EXC_REAL_END(name, start, size);
 
-#define EXC_VIRT_MASKABLE(name, start, size, realvec)			\
+#define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask)		\
 	EXC_VIRT_BEGIN(name, start, size);				\
-	MASKABLE_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \
+	MASKABLE_RELON_EXCEPTION_PSERIES(start, realvec, name##_common, bitmask);\
 	EXC_VIRT_END(name, start, size);
 
 #define EXC_REAL_HV(name, start, size)					\
@@ -305,13 +304,13 @@ end_##sname:
 #define __EXC_REAL_OOL_MASKABLE(name, start, size)			\
 	__EXC_REAL_OOL(name, start, size);
 
-#define __TRAMP_REAL_OOL_MASKABLE(name, vec)				\
+#define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask)			\
 	TRAMP_REAL_BEGIN(tramp_real_##name);				\
-	MASKABLE_EXCEPTION_PSERIES_OOL(vec, name##_common);		\
+	MASKABLE_EXCEPTION_PSERIES_OOL(vec, name##_common, bitmask);	\
 
-#define EXC_REAL_OOL_MASKABLE(name, start, size)			\
+#define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask)		\
 	__EXC_REAL_OOL_MASKABLE(name, start, size);			\
-	__TRAMP_REAL_OOL_MASKABLE(name, start);
+	__TRAMP_REAL_OOL_MASKABLE(name, start, bitmask);
 
 #define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler)		\
 	EXC_REAL_BEGIN(name, start, size);				\
@@ -332,13 +331,13 @@ end_##sname:
 #define __EXC_REAL_OOL_MASKABLE_HV(name, start, size)			\
 	__EXC_REAL_OOL(name, start, size);
 
-#define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec)				\
+#define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask)		\
 	TRAMP_REAL_BEGIN(tramp_real_##name);				\
-	MASKABLE_EXCEPTION_HV_OOL(vec, name##_common);			\
+	MASKABLE_EXCEPTION_HV_OOL(vec, name##_common, bitmask);		\
 
-#define EXC_REAL_OOL_MASKABLE_HV(name, start, size)			\
+#define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask)		\
 	__EXC_REAL_OOL_MASKABLE_HV(name, start, size);			\
-	__TRAMP_REAL_OOL_MASKABLE_HV(name, start);
+	__TRAMP_REAL_OOL_MASKABLE_HV(name, start, bitmask);
 
 #define __EXC_VIRT_OOL(name, start, size)				\
 	EXC_VIRT_BEGIN(name, start, size);				\
@@ -356,13 +355,13 @@ end_##sname:
 #define __EXC_VIRT_OOL_MASKABLE(name, start, size)			\
 	__EXC_VIRT_OOL(name, start, size);
 
-#define __TRAMP_VIRT_OOL_MASKABLE(name, realvec)			\
+#define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask)		\
 	TRAMP_VIRT_BEGIN(tramp_virt_##name);				\
-	MASKABLE_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common);	\
+	MASKABLE_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common, bitmask);\
 
-#define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec)		\
+#define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask)	\
 	__EXC_VIRT_OOL_MASKABLE(name, start, size);			\
-	__TRAMP_VIRT_OOL_MASKABLE(name, realvec);
+	__TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask);
 
 #define __EXC_VIRT_OOL_HV(name, start, size)				\
 	__EXC_VIRT_OOL(name, start, size);
@@ -378,13 +377,13 @@ end_##sname:
 #define __EXC_VIRT_OOL_MASKABLE_HV(name, start, size)			\
 	__EXC_VIRT_OOL(name, start, size);
 
-#define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec)			\
+#define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask)		\
 	TRAMP_VIRT_BEGIN(tramp_virt_##name);				\
-	MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common);	\
+	MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common, bitmask);\
 
-#define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec)		\
+#define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask)	\
 	__EXC_VIRT_OOL_MASKABLE_HV(name, start, size);			\
-	__TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec);
+	__TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask);
 
 #define TRAMP_KVM(area, n)						\
 	TRAMP_KVM_BEGIN(do_kvm_##n);					\
@@ -413,10 +412,6 @@ end_##sname:
 	EXC_COMMON_BEGIN(name);						\
 	STD_EXCEPTION_COMMON_ASYNC(realvec, name, hdlr);		\
 
-#define EXC_COMMON_HV(name, realvec, hdlr)				\
-	EXC_COMMON_BEGIN(name);						\
-	STD_EXCEPTION_COMMON(realvec + 0x2, name, hdlr);		\
-
 #endif /* __ASSEMBLY__ */
 
 #endif	/* _ASM_POWERPC_HEAD_64_H */
diff --git a/arch/powerpc/include/asm/hmi.h b/arch/powerpc/include/asm/hmi.h
index 85b7a1a21e22..9c14f7b5c46c 100644
--- a/arch/powerpc/include/asm/hmi.h
+++ b/arch/powerpc/include/asm/hmi.h
@@ -42,4 +42,8 @@ extern void wait_for_tb_resync(void);
 static inline void wait_for_subcore_guest_exit(void) { }
 static inline void wait_for_tb_resync(void) { }
 #endif
+
+struct pt_regs;
+extern long hmi_handle_debugtrig(struct pt_regs *regs);
+
 #endif /* __ASM_PPC64_HMI_H__ */
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index 14c9d44f355b..1a4847f67ea8 100644
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -47,8 +47,7 @@ static inline pte_t *hugepd_page(hugepd_t hpd)
 {
 	BUG_ON(!hugepd_ok(hpd));
 #ifdef CONFIG_PPC_8xx
-	return (pte_t *)__va(hpd_val(hpd) &
-			     ~(_PMD_PAGE_MASK | _PMD_PRESENT_MASK));
+	return (pte_t *)__va(hpd_val(hpd) & ~HUGEPD_SHIFT_MASK);
 #else
 	return (pte_t *)((hpd_val(hpd) &
 			  ~HUGEPD_SHIFT_MASK) | PD_HUGE);
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 3818fa0164f0..855e17d158b1 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -27,6 +27,25 @@
 #define PACA_IRQ_DEC		0x08 /* Or FIT */
 #define PACA_IRQ_EE_EDGE	0x10 /* BookE only */
 #define PACA_IRQ_HMI		0x20
+#define PACA_IRQ_PMI		0x40
+
+/*
+ * Some soft-masked interrupts must be hard masked until they are replayed
+ * (e.g., because the soft-masked handler does not clear the exception).
+ */
+#ifdef CONFIG_PPC_BOOK3S
+#define PACA_IRQ_MUST_HARD_MASK	(PACA_IRQ_EE|PACA_IRQ_PMI)
+#else
+#define PACA_IRQ_MUST_HARD_MASK	(PACA_IRQ_EE)
+#endif
+
+/*
+ * flags for paca->irq_soft_mask
+ */
+#define IRQS_ENABLED		0
+#define IRQS_DISABLED		1 /* local_irq_disable() interrupts */
+#define IRQS_PMI_DISABLED	2
+#define IRQS_ALL_DISABLED	(IRQS_DISABLED | IRQS_PMI_DISABLED)
 
 #endif /* CONFIG_PPC64 */
 
@@ -43,46 +62,112 @@ extern void unknown_exception(struct pt_regs *regs);
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
 
-static inline unsigned long arch_local_save_flags(void)
+static inline notrace unsigned long irq_soft_mask_return(void)
 {
 	unsigned long flags;
 
 	asm volatile(
 		"lbz %0,%1(13)"
 		: "=r" (flags)
-		: "i" (offsetof(struct paca_struct, soft_enabled)));
+		: "i" (offsetof(struct paca_struct, irq_soft_mask)));
 
 	return flags;
 }
 
-static inline unsigned long arch_local_irq_disable(void)
+/*
+ * The "memory" clobber acts as both a compiler barrier
+ * for the critical section and as a clobber because
+ * we changed paca->irq_soft_mask
+ */
+static inline notrace void irq_soft_mask_set(unsigned long mask)
+{
+#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
+	/*
+	 * The irq mask must always include the STD bit if any are set.
+	 *
+	 * and interrupts don't get replayed until the standard
+	 * interrupt (local_irq_disable()) is unmasked.
+	 *
+	 * Other masks must only provide additional masking beyond
+	 * the standard, and they are also not replayed until the
+	 * standard interrupt becomes unmasked.
+	 *
+	 * This could be changed, but it will require partial
+	 * unmasks to be replayed, among other things. For now, take
+	 * the simple approach.
+	 */
+	WARN_ON(mask && !(mask & IRQS_DISABLED));
+#endif
+
+	asm volatile(
+		"stb %0,%1(13)"
+		:
+		: "r" (mask),
+		  "i" (offsetof(struct paca_struct, irq_soft_mask))
+		: "memory");
+}
+
+static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask)
 {
-	unsigned long flags, zero;
+	unsigned long flags;
+
+#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
+	WARN_ON(mask && !(mask & IRQS_DISABLED));
+#endif
 
 	asm volatile(
-		"li %1,0; lbz %0,%2(13); stb %1,%2(13)"
-		: "=r" (flags), "=&r" (zero)
-		: "i" (offsetof(struct paca_struct, soft_enabled))
+		"lbz %0,%1(13); stb %2,%1(13)"
+		: "=&r" (flags)
+		: "i" (offsetof(struct paca_struct, irq_soft_mask)),
+		  "r" (mask)
 		: "memory");
 
 	return flags;
 }
 
+static inline notrace unsigned long irq_soft_mask_or_return(unsigned long mask)
+{
+	unsigned long flags, tmp;
+
+	asm volatile(
+		"lbz %0,%2(13); or %1,%0,%3; stb %1,%2(13)"
+		: "=&r" (flags), "=r" (tmp)
+		: "i" (offsetof(struct paca_struct, irq_soft_mask)),
+		  "r" (mask)
+		: "memory");
+
+#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
+	WARN_ON((mask | flags) && !((mask | flags) & IRQS_DISABLED));
+#endif
+
+	return flags;
+}
+
+static inline unsigned long arch_local_save_flags(void)
+{
+	return irq_soft_mask_return();
+}
+
+static inline void arch_local_irq_disable(void)
+{
+	irq_soft_mask_set(IRQS_DISABLED);
+}
+
 extern void arch_local_irq_restore(unsigned long);
 
 static inline void arch_local_irq_enable(void)
 {
-	arch_local_irq_restore(1);
+	arch_local_irq_restore(IRQS_ENABLED);
 }
 
 static inline unsigned long arch_local_irq_save(void)
 {
-	return arch_local_irq_disable();
+	return irq_soft_mask_set_return(IRQS_DISABLED);
 }
 
 static inline bool arch_irqs_disabled_flags(unsigned long flags)
 {
-	return flags == 0;
+	return flags & IRQS_DISABLED;
 }
 
 static inline bool arch_irqs_disabled(void)
@@ -90,6 +175,55 @@ static inline bool arch_irqs_disabled(void)
 	return arch_irqs_disabled_flags(arch_local_save_flags());
 }
 
+#ifdef CONFIG_PPC_BOOK3S
+/*
+ * To support disabling and enabling of irq with PMI, set of
+ * new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore()
+ * functions are added. These macros are implemented using generic
+ * linux local_irq_* code from include/linux/irqflags.h.
+ */
+#define raw_local_irq_pmu_save(flags)					\
+	do {								\
+		typecheck(unsigned long, flags);			\
+		flags = irq_soft_mask_or_return(IRQS_DISABLED |	\
+				IRQS_PMI_DISABLED);			\
+	} while(0)
+
+#define raw_local_irq_pmu_restore(flags)				\
+	do {								\
+		typecheck(unsigned long, flags);			\
+		arch_local_irq_restore(flags);				\
+	} while(0)
+
+#ifdef CONFIG_TRACE_IRQFLAGS
+#define powerpc_local_irq_pmu_save(flags)			\
+	 do {							\
+		raw_local_irq_pmu_save(flags);			\
+		trace_hardirqs_off();				\
+	} while(0)
+#define powerpc_local_irq_pmu_restore(flags)			\
+	do {							\
+		if (raw_irqs_disabled_flags(flags)) {		\
+			raw_local_irq_pmu_restore(flags);	\
+			trace_hardirqs_off();			\
+		} else {					\
+			trace_hardirqs_on();			\
+			raw_local_irq_pmu_restore(flags);	\
+		}						\
+	} while(0)
+#else
+#define powerpc_local_irq_pmu_save(flags)			\
+	do {							\
+		raw_local_irq_pmu_save(flags);			\
+	} while(0)
+#define powerpc_local_irq_pmu_restore(flags)			\
+	do {							\
+		raw_local_irq_pmu_restore(flags);		\
+	} while (0)
+#endif  /* CONFIG_TRACE_IRQFLAGS */
+
+#endif /* CONFIG_PPC_BOOK3S */
+
 #ifdef CONFIG_PPC_BOOK3E
 #define __hard_irq_enable()	asm volatile("wrteei 1" : : : "memory")
 #define __hard_irq_disable()	asm volatile("wrteei 0" : : : "memory")
@@ -98,14 +232,13 @@ static inline bool arch_irqs_disabled(void)
 #define __hard_irq_disable()	__mtmsrd(local_paca->kernel_msr, 1)
 #endif
 
-#define hard_irq_disable()	do {			\
-	u8 _was_enabled;				\
-	__hard_irq_disable();				\
-	_was_enabled = local_paca->soft_enabled;	\
-	local_paca->soft_enabled = 0;			\
-	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;	\
-	if (_was_enabled)				\
-		trace_hardirqs_off();			\
+#define hard_irq_disable()	do {					\
+	unsigned long flags;						\
+	__hard_irq_disable();						\
+	flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED);		\
+	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;			\
+	if (!arch_irqs_disabled_flags(flags))				\
+		trace_hardirqs_off();					\
 } while(0)
 
 static inline bool lazy_irq_pending(void)
@@ -121,13 +254,13 @@ static inline bool lazy_irq_pending(void)
 static inline void may_hard_irq_enable(void)
 {
 	get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
-	if (!(get_paca()->irq_happened & PACA_IRQ_EE))
+	if (!(get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK))
 		__hard_irq_enable();
 }
 
 static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
 {
-	return !regs->softe;
+	return (regs->softe & IRQS_DISABLED);
 }
 
 extern bool prep_irq_for_idle(void);
diff --git a/arch/powerpc/include/asm/imc-pmu.h b/arch/powerpc/include/asm/imc-pmu.h
index fad0e6ff460f..d76cb11be3e3 100644
--- a/arch/powerpc/include/asm/imc-pmu.h
+++ b/arch/powerpc/include/asm/imc-pmu.h
@@ -34,6 +34,13 @@
 #define THREAD_IMC_LDBAR_MASK           0x0003ffffffffe000ULL
 #define THREAD_IMC_ENABLE               0x8000000000000000ULL
 
+/*
+ * For debugfs interface for imc-mode and imc-command
+ */
+#define IMC_CNTL_BLK_OFFSET		0x3FC00
+#define IMC_CNTL_BLK_CMD_OFFSET		8
+#define IMC_CNTL_BLK_MODE_OFFSET	32
+
 /*
  * Structure to hold memory address information for imc units.
  */
@@ -71,7 +78,7 @@ struct imc_events {
 struct imc_pmu {
 	struct pmu pmu;
 	struct imc_mem_info *mem_info;
-	struct imc_events **events;
+	struct imc_events *events;
 	/*
 	 * Attribute groups for the PMU. Slot 0 used for
 	 * format attribute, slot 1 used for cpusmask attribute,
diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h
index 1aeb5f13b8c4..1a6c1ce17735 100644
--- a/arch/powerpc/include/asm/irqflags.h
+++ b/arch/powerpc/include/asm/irqflags.h
@@ -47,14 +47,14 @@
  * be clobbered.
  */
 #define RECONCILE_IRQ_STATE(__rA, __rB)		\
-	lbz	__rA,PACASOFTIRQEN(r13);	\
+	lbz	__rA,PACAIRQSOFTMASK(r13);	\
 	lbz	__rB,PACAIRQHAPPENED(r13);	\
-	cmpwi	cr0,__rA,0;			\
-	li	__rA,0;				\
+	andi.	__rA,__rA,IRQS_DISABLED;	\
+	li	__rA,IRQS_DISABLED;		\
 	ori	__rB,__rB,PACA_IRQ_HARD_DIS;	\
 	stb	__rB,PACAIRQHAPPENED(r13);	\
-	beq	44f;				\
-	stb	__rA,PACASOFTIRQEN(r13);	\
+	bne	44f;				\
+	stb	__rA,PACAIRQSOFTMASK(r13);	\
 	TRACE_DISABLE_INTS;			\
 44:
 
@@ -64,9 +64,9 @@
 
 #define RECONCILE_IRQ_STATE(__rA, __rB)		\
 	lbz	__rA,PACAIRQHAPPENED(r13);	\
-	li	__rB,0;				\
+	li	__rB,IRQS_DISABLED;		\
 	ori	__rA,__rA,PACA_IRQ_HARD_DIS;	\
-	stb	__rB,PACASOFTIRQEN(r13);	\
+	stb	__rB,PACAIRQSOFTMASK(r13);	\
 	stb	__rA,PACAIRQHAPPENED(r13)
 #endif
 #endif
diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h
index 4419d435639a..d8b1e8e7e035 100644
--- a/arch/powerpc/include/asm/kexec.h
+++ b/arch/powerpc/include/asm/kexec.h
@@ -73,6 +73,8 @@ extern void kexec_smp_wait(void);	/* get and clear naca physid, wait for
 					  master to copy new code to 0 */
 extern int crashing_cpu;
 extern void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *));
+extern void crash_ipi_callback(struct pt_regs *);
+extern int crash_wake_offline;
 
 struct kimage;
 struct pt_regs;
@@ -138,6 +140,12 @@ static inline bool kdump_in_progress(void)
 	return false;
 }
 
+static inline void crash_ipi_callback(struct pt_regs *regs) { }
+
+static inline void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
+{
+}
+
 #endif /* CONFIG_KEXEC_CORE */
 #endif /* ! __ASSEMBLY__ */
 #endif /* __KERNEL__ */
diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
index 9a667007bff8..376ae803b69c 100644
--- a/arch/powerpc/include/asm/kvm_book3s.h
+++ b/arch/powerpc/include/asm/kvm_book3s.h
@@ -249,10 +249,8 @@ extern int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd);
 extern void kvmppc_pr_init_default_hcalls(struct kvm *kvm);
 extern int kvmppc_hcall_impl_pr(unsigned long cmd);
 extern int kvmppc_hcall_impl_hv_realmode(unsigned long cmd);
-extern void kvmppc_copy_to_svcpu(struct kvmppc_book3s_shadow_vcpu *svcpu,
-				 struct kvm_vcpu *vcpu);
-extern void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu,
-				   struct kvmppc_book3s_shadow_vcpu *svcpu);
+extern void kvmppc_copy_to_svcpu(struct kvm_vcpu *vcpu);
+extern void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu);
 extern int kvm_irq_bypass;
 
 static inline struct kvmppc_vcpu_book3s *to_book3s(struct kvm_vcpu *vcpu)
diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h
index 735cfa35298a..998f7b7aaa9e 100644
--- a/arch/powerpc/include/asm/kvm_book3s_64.h
+++ b/arch/powerpc/include/asm/kvm_book3s_64.h
@@ -122,13 +122,13 @@ static inline int kvmppc_hpte_page_shifts(unsigned long h, unsigned long l)
 	lphi = (l >> 16) & 0xf;
 	switch ((l >> 12) & 0xf) {
 	case 0:
-		return !lphi ? 24 : -1;		/* 16MB */
+		return !lphi ? 24 : 0;		/* 16MB */
 		break;
 	case 1:
 		return 16;			/* 64kB */
 		break;
 	case 3:
-		return !lphi ? 34 : -1;		/* 16GB */
+		return !lphi ? 34 : 0;		/* 16GB */
 		break;
 	case 7:
 		return (16 << 8) + 12;		/* 64kB in 4kB */
@@ -140,7 +140,7 @@ static inline int kvmppc_hpte_page_shifts(unsigned long h, unsigned long l)
 			return (24 << 8) + 12;	/* 16MB in 4kB */
 		break;
 	}
-	return -1;
+	return 0;
 }
 
 static inline int kvmppc_hpte_base_page_shift(unsigned long h, unsigned long l)
@@ -159,7 +159,11 @@ static inline int kvmppc_hpte_actual_page_shift(unsigned long h, unsigned long l
 
 static inline unsigned long kvmppc_actual_pgsz(unsigned long v, unsigned long r)
 {
-	return 1ul << kvmppc_hpte_actual_page_shift(v, r);
+	int shift = kvmppc_hpte_actual_page_shift(v, r);
+
+	if (shift)
+		return 1ul << shift;
+	return 0;
 }
 
 static inline int kvmppc_pgsize_lp_encoding(int base_shift, int actual_shift)
@@ -232,7 +236,7 @@ static inline unsigned long compute_tlbie_rb(unsigned long v, unsigned long r,
 		va_low ^= v >> (SID_SHIFT_1T - 16);
 	va_low &= 0x7ff;
 
-	if (b_pgshift == 12) {
+	if (b_pgshift <= 12) {
 		if (a_pgshift > 12) {
 			sllp = (a_pgshift == 16) ? 5 : 4;
 			rb |= sllp << 5;	/*  AP field */
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 3aa5b577cd60..1f53b562726f 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -690,6 +690,7 @@ struct kvm_vcpu_arch {
 	u8 mmio_vsx_offset;
 	u8 mmio_vsx_copy_type;
 	u8 mmio_vsx_tx_sx_enabled;
+	u8 mmio_vmx_copy_nums;
 	u8 osi_needed;
 	u8 osi_enabled;
 	u8 papr_enabled;
@@ -709,6 +710,7 @@ struct kvm_vcpu_arch {
 	u8 ceded;
 	u8 prodded;
 	u8 doorbell_request;
+	u8 irq_pending; /* Used by XIVE to signal pending guest irqs */
 	u32 last_inst;
 
 	struct swait_queue_head *wqp;
@@ -738,8 +740,11 @@ struct kvm_vcpu_arch {
 	struct kvmppc_icp *icp; /* XICS presentation controller */
 	struct kvmppc_xive_vcpu *xive_vcpu; /* XIVE virtual CPU data */
 	__be32 xive_cam_word;    /* Cooked W2 in proper endian with valid bit */
-	u32 xive_pushed;	 /* Is the VP pushed on the physical CPU ? */
+	u8 xive_pushed;		 /* Is the VP pushed on the physical CPU ? */
+	u8 xive_esc_on;		 /* Is the escalation irq enabled ? */
 	union xive_tma_w01 xive_saved_state; /* W0..1 of XIVE thread state */
+	u64 xive_esc_raddr;	 /* Escalation interrupt ESB real addr */
+	u64 xive_esc_vaddr;	 /* Escalation interrupt ESB virt addr */
 #endif
 
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
@@ -800,6 +805,7 @@ struct kvm_vcpu_arch {
 #define KVM_MMIO_REG_QPR	0x0040
 #define KVM_MMIO_REG_FQPR	0x0060
 #define KVM_MMIO_REG_VSX	0x0080
+#define KVM_MMIO_REG_VMX	0x00c0
 
 #define __KVM_HAVE_ARCH_WQP
 #define __KVM_HAVE_CREATE_DEVICE
diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h
index 941c2a3f231b..7765a800ddae 100644
--- a/arch/powerpc/include/asm/kvm_ppc.h
+++ b/arch/powerpc/include/asm/kvm_ppc.h
@@ -81,6 +81,10 @@ extern int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu,
 extern int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
 				unsigned int rt, unsigned int bytes,
 			int is_default_endian, int mmio_sign_extend);
+extern int kvmppc_handle_load128_by2x64(struct kvm_run *run,
+		struct kvm_vcpu *vcpu, unsigned int rt, int is_default_endian);
+extern int kvmppc_handle_store128_by2x64(struct kvm_run *run,
+		struct kvm_vcpu *vcpu, unsigned int rs, int is_default_endian);
 extern int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
 			       u64 val, unsigned int bytes,
 			       int is_default_endian);
@@ -873,7 +877,7 @@ static inline void kvmppc_fix_ee_before_entry(void)
 
 	/* Only need to enable IRQs by hard enabling them after this */
 	local_paca->irq_happened = 0;
-	local_paca->soft_enabled = 1;
+	irq_soft_mask_set(IRQS_ENABLED);
 #endif
 }
 
diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h
index 600a68bd77f5..fdd00939270b 100644
--- a/arch/powerpc/include/asm/local.h
+++ b/arch/powerpc/include/asm/local.h
@@ -2,76 +2,64 @@
 #ifndef _ARCH_POWERPC_LOCAL_H
 #define _ARCH_POWERPC_LOCAL_H
 
+#ifdef CONFIG_PPC_BOOK3S_64
+
 #include <linux/percpu.h>
 #include <linux/atomic.h>
+#include <linux/irqflags.h>
+
+#include <asm/hw_irq.h>
 
 typedef struct
 {
-	atomic_long_t a;
+	long v;
 } local_t;
 
-#define LOCAL_INIT(i)	{ ATOMIC_LONG_INIT(i) }
-
-#define local_read(l)	atomic_long_read(&(l)->a)
-#define local_set(l,i)	atomic_long_set(&(l)->a, (i))
+#define LOCAL_INIT(i)	{ (i) }
 
-#define local_add(i,l)	atomic_long_add((i),(&(l)->a))
-#define local_sub(i,l)	atomic_long_sub((i),(&(l)->a))
-#define local_inc(l)	atomic_long_inc(&(l)->a)
-#define local_dec(l)	atomic_long_dec(&(l)->a)
-
-static __inline__ long local_add_return(long a, local_t *l)
+static __inline__ long local_read(local_t *l)
 {
-	long t;
-
-	__asm__ __volatile__(
-"1:"	PPC_LLARX(%0,0,%2,0) "			# local_add_return\n\
-	add	%0,%1,%0\n"
-	PPC405_ERR77(0,%2)
-	PPC_STLCX	"%0,0,%2 \n\
-	bne-	1b"
-	: "=&r" (t)
-	: "r" (a), "r" (&(l->a.counter))
-	: "cc", "memory");
-
-	return t;
+	return READ_ONCE(l->v);
 }
 
-#define local_add_negative(a, l)	(local_add_return((a), (l)) < 0)
-
-static __inline__ long local_sub_return(long a, local_t *l)
+static __inline__ void local_set(local_t *l, long i)
 {
-	long t;
+	WRITE_ONCE(l->v, i);
+}
 
-	__asm__ __volatile__(
-"1:"	PPC_LLARX(%0,0,%2,0) "			# local_sub_return\n\
-	subf	%0,%1,%0\n"
-	PPC405_ERR77(0,%2)
-	PPC_STLCX	"%0,0,%2 \n\
-	bne-	1b"
-	: "=&r" (t)
-	: "r" (a), "r" (&(l->a.counter))
-	: "cc", "memory");
+#define LOCAL_OP(op, c_op)						\
+static __inline__ void local_##op(long i, local_t *l)			\
+{									\
+	unsigned long flags;						\
+									\
+	powerpc_local_irq_pmu_save(flags);				\
+	l->v c_op i;						\
+	powerpc_local_irq_pmu_restore(flags);				\
+}
 
-	return t;
+#define LOCAL_OP_RETURN(op, c_op)					\
+static __inline__ long local_##op##_return(long a, local_t *l)		\
+{									\
+	long t;								\
+	unsigned long flags;						\
+									\
+	powerpc_local_irq_pmu_save(flags);				\
+	t = (l->v c_op a);						\
+	powerpc_local_irq_pmu_restore(flags);				\
+									\
+	return t;							\
 }
 
-static __inline__ long local_inc_return(local_t *l)
-{
-	long t;
+#define LOCAL_OPS(op, c_op)		\
+	LOCAL_OP(op, c_op)		\
+	LOCAL_OP_RETURN(op, c_op)
 
-	__asm__ __volatile__(
-"1:"	PPC_LLARX(%0,0,%1,0) "			# local_inc_return\n\
-	addic	%0,%0,1\n"
-	PPC405_ERR77(0,%1)
-	PPC_STLCX	"%0,0,%1 \n\
-	bne-	1b"
-	: "=&r" (t)
-	: "r" (&(l->a.counter))
-	: "cc", "xer", "memory");
+LOCAL_OPS(add, +=)
+LOCAL_OPS(sub, -=)
 
-	return t;
-}
+#define local_add_negative(a, l)	(local_add_return((a), (l)) < 0)
+#define local_inc_return(l)		local_add_return(1LL, l)
+#define local_inc(l)			local_inc_return(l)
 
 /*
  * local_inc_and_test - increment and test
@@ -81,28 +69,39 @@ static __inline__ long local_inc_return(local_t *l)
  * and returns true if the result is zero, or false for all
  * other cases.
  */
-#define local_inc_and_test(l) (local_inc_return(l) == 0)
+#define local_inc_and_test(l)		(local_inc_return(l) == 0)
 
-static __inline__ long local_dec_return(local_t *l)
+#define local_dec_return(l)		local_sub_return(1LL, l)
+#define local_dec(l)			local_dec_return(l)
+#define local_sub_and_test(a, l)	(local_sub_return((a), (l)) == 0)
+#define local_dec_and_test(l)		(local_dec_return((l)) == 0)
+
+static __inline__ long local_cmpxchg(local_t *l, long o, long n)
 {
 	long t;
+	unsigned long flags;
 
-	__asm__ __volatile__(
-"1:"	PPC_LLARX(%0,0,%1,0) "			# local_dec_return\n\
-	addic	%0,%0,-1\n"
-	PPC405_ERR77(0,%1)
-	PPC_STLCX	"%0,0,%1\n\
-	bne-	1b"
-	: "=&r" (t)
-	: "r" (&(l->a.counter))
-	: "cc", "xer", "memory");
+	powerpc_local_irq_pmu_save(flags);
+	t = l->v;
+	if (t == o)
+		l->v = n;
+	powerpc_local_irq_pmu_restore(flags);
 
 	return t;
 }
 
-#define local_cmpxchg(l, o, n) \
-	(cmpxchg_local(&((l)->a.counter), (o), (n)))
-#define local_xchg(l, n) (xchg_local(&((l)->a.counter), (n)))
+static __inline__ long local_xchg(local_t *l, long n)
+{
+	long t;
+	unsigned long flags;
+
+	powerpc_local_irq_pmu_save(flags);
+	t = l->v;
+	l->v = n;
+	powerpc_local_irq_pmu_restore(flags);
+
+	return t;
+}
 
 /**
  * local_add_unless - add unless the number is a given value
@@ -115,62 +114,35 @@ static __inline__ long local_dec_return(local_t *l)
  */
 static __inline__ int local_add_unless(local_t *l, long a, long u)
 {
-	long t;
-
-	__asm__ __volatile__ (
-"1:"	PPC_LLARX(%0,0,%1,0) "			# local_add_unless\n\
-	cmpw	0,%0,%3 \n\
-	beq-	2f \n\
-	add	%0,%2,%0 \n"
-	PPC405_ERR77(0,%2)
-	PPC_STLCX	"%0,0,%1 \n\
-	bne-	1b \n"
-"	subf	%0,%2,%0 \n\
-2:"
-	: "=&r" (t)
-	: "r" (&(l->a.counter)), "r" (a), "r" (u)
-	: "cc", "memory");
-
-	return t != u;
-}
-
-#define local_inc_not_zero(l) local_add_unless((l), 1, 0)
-
-#define local_sub_and_test(a, l)	(local_sub_return((a), (l)) == 0)
-#define local_dec_and_test(l)		(local_dec_return((l)) == 0)
-
-/*
- * Atomically test *l and decrement if it is greater than 0.
- * The function returns the old value of *l minus 1.
- */
-static __inline__ long local_dec_if_positive(local_t *l)
-{
-	long t;
+	unsigned long flags;
+	int ret = 0;
 
-	__asm__ __volatile__(
-"1:"	PPC_LLARX(%0,0,%1,0) "			# local_dec_if_positive\n\
-	cmpwi	%0,1\n\
-	addi	%0,%0,-1\n\
-	blt-	2f\n"
-	PPC405_ERR77(0,%1)
-	PPC_STLCX	"%0,0,%1\n\
-	bne-	1b"
-	"\n\
-2:"	: "=&b" (t)
-	: "r" (&(l->a.counter))
-	: "cc", "memory");
+	powerpc_local_irq_pmu_save(flags);
+	if (l->v != u) {
+		l->v += a;
+		ret = 1;
+	}
+	powerpc_local_irq_pmu_restore(flags);
 
-	return t;
+	return ret;
 }
 
+#define local_inc_not_zero(l)		local_add_unless((l), 1, 0)
+
 /* Use these for per-cpu local_t variables: on some archs they are
  * much more efficient than these naive implementations.  Note they take
  * a variable, not an address.
  */
 
-#define __local_inc(l)		((l)->a.counter++)
-#define __local_dec(l)		((l)->a.counter++)
-#define __local_add(i,l)	((l)->a.counter+=(i))
-#define __local_sub(i,l)	((l)->a.counter-=(i))
+#define __local_inc(l)		((l)->v++)
+#define __local_dec(l)		((l)->v++)
+#define __local_add(i,l)	((l)->v+=(i))
+#define __local_sub(i,l)	((l)->v-=(i))
+
+#else /* CONFIG_PPC64 */
+
+#include <asm-generic/local.h>
+
+#endif /* CONFIG_PPC64 */
 
 #endif /* _ARCH_POWERPC_LOCAL_H */
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index cd2fc1cc1cc7..ffe7c71e1132 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -173,11 +173,19 @@ struct machdep_calls {
 	/* Called after scan and before resource survey */
 	void (*pcibios_fixup_phb)(struct pci_controller *hose);
 
+	/*
+	 * Called after device has been added to bus and
+	 * before sysfs has been created.
+	 */
+	void (*pcibios_bus_add_device)(struct pci_dev *pdev);
+
 	resource_size_t (*pcibios_default_alignment)(void);
 
 #ifdef CONFIG_PCI_IOV
 	void (*pcibios_fixup_sriov)(struct pci_dev *pdev);
 	resource_size_t (*pcibios_iov_resource_alignment)(struct pci_dev *, int resno);
+	int (*pcibios_sriov_enable)(struct pci_dev *pdev, u16 num_vfs);
+	int (*pcibios_sriov_disable)(struct pci_dev *pdev);
 #endif /* CONFIG_PCI_IOV */
 
 	/* Called to shutdown machine specific hardware not already controlled
diff --git a/arch/powerpc/include/asm/membarrier.h b/arch/powerpc/include/asm/membarrier.h
new file mode 100644
index 000000000000..6e20bb5c74ea
--- /dev/null
+++ b/arch/powerpc/include/asm/membarrier.h
@@ -0,0 +1,27 @@
+#ifndef _ASM_POWERPC_MEMBARRIER_H
+#define _ASM_POWERPC_MEMBARRIER_H
+
+static inline void membarrier_arch_switch_mm(struct mm_struct *prev,
+					     struct mm_struct *next,
+					     struct task_struct *tsk)
+{
+	/*
+	 * Only need the full barrier when switching between processes.
+	 * Barrier when switching from kernel to userspace is not
+	 * required here, given that it is implied by mmdrop(). Barrier
+	 * when switching from userspace to kernel is not needed after
+	 * store to rq->curr.
+	 */
+	if (likely(!(atomic_read(&next->membarrier_state) &
+		     (MEMBARRIER_STATE_PRIVATE_EXPEDITED |
+		      MEMBARRIER_STATE_GLOBAL_EXPEDITED)) || !prev))
+		return;
+
+	/*
+	 * The membarrier system call requires a full memory barrier
+	 * after storing to rq->curr, before going back to user-space.
+	 */
+	smp_mb();
+}
+
+#endif /* _ASM_POWERPC_MEMBARRIER_H */
diff --git a/arch/powerpc/include/asm/mman.h b/arch/powerpc/include/asm/mman.h
index 30922f699341..07e3f54de9e3 100644
--- a/arch/powerpc/include/asm/mman.h
+++ b/arch/powerpc/include/asm/mman.h
@@ -13,6 +13,7 @@
 
 #include <asm/cputable.h>
 #include <linux/mm.h>
+#include <linux/pkeys.h>
 #include <asm/cpu_has_feature.h>
 
 /*
@@ -22,13 +23,23 @@
 static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
 		unsigned long pkey)
 {
-	return (prot & PROT_SAO) ? VM_SAO : 0;
+#ifdef CONFIG_PPC_MEM_KEYS
+	return (((prot & PROT_SAO) ? VM_SAO : 0) | pkey_to_vmflag_bits(pkey));
+#else
+	return ((prot & PROT_SAO) ? VM_SAO : 0);
+#endif
 }
 #define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey)
 
 static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags)
 {
+#ifdef CONFIG_PPC_MEM_KEYS
+	return (vm_flags & VM_SAO) ?
+		__pgprot(_PAGE_SAO | vmflag_to_pte_pkey_bits(vm_flags)) :
+		__pgprot(0 | vmflag_to_pte_pkey_bits(vm_flags));
+#else
 	return (vm_flags & VM_SAO) ? __pgprot(_PAGE_SAO) : __pgprot(0);
+#endif
 }
 #define arch_vm_get_page_prot(vm_flags) arch_vm_get_page_prot(vm_flags)
 
diff --git a/arch/powerpc/include/asm/mmu-8xx.h b/arch/powerpc/include/asm/mmu-8xx.h
index 5bb3dbede41a..2f806e329648 100644
--- a/arch/powerpc/include/asm/mmu-8xx.h
+++ b/arch/powerpc/include/asm/mmu-8xx.h
@@ -29,17 +29,25 @@
 #define MI_Kp		0x40000000	/* Should always be set */
 
 /*
- * All pages' PP exec bits are set to 000, which means Execute for Supervisor
- * and no Execute for User.
- * Then we use the APG to say whether accesses are according to Page rules,
- * "all Supervisor" rules (Exec for all) and "all User" rules (Exec for noone)
- * Therefore, we define 4 APG groups. msb is _PAGE_EXEC, lsb is _PAGE_USER
- * 0 (00) => Not User, no exec => 11 (all accesses performed as user)
- * 1 (01) => User but no exec => 11 (all accesses performed as user)
- * 2 (10) => Not User, exec => 01 (rights according to page definition)
- * 3 (11) => User, exec => 00 (all accesses performed as supervisor)
- */
-#define MI_APG_INIT	0xf4ffffff
+ * All pages' PP data bits are set to either 001 or 011 by copying _PAGE_EXEC
+ * into bit 21 in the ITLBmiss handler (bit 21 is the middle bit), which means
+ * respectively NA for All or X for Supervisor and no access for User.
+ * Then we use the APG to say whether accesses are according to Page rules or
+ * "all Supervisor" rules (Access to all)
+ * We also use the 2nd APG bit for _PAGE_ACCESSED when having SWAP:
+ * When that bit is not set access is done iaw "all user"
+ * which means no access iaw page rules.
+ * Therefore, we define 4 APG groups. lsb is _PMD_USER, 2nd is _PAGE_ACCESSED
+ * 0x => No access => 11 (all accesses performed as user iaw page definition)
+ * 10 => No user => 01 (all accesses performed according to page definition)
+ * 11 => User => 00 (all accesses performed as supervisor iaw page definition)
+ * We define all 16 groups so that all other bits of APG can take any value
+ */
+#ifdef CONFIG_SWAP
+#define MI_APG_INIT	0xf4f4f4f4
+#else
+#define MI_APG_INIT	0x44444444
+#endif
 
 /* The effective page number register.  When read, contains the information
  * about the last instruction TLB miss.  When MI_RPN is written, bits in
@@ -102,17 +110,25 @@
 #define MD_Kp		0x40000000	/* Should always be set */
 
 /*
- * All pages' PP data bits are set to either 000 or 011, which means
+ * All pages' PP data bits are set to either 000 or 011 or 001, which means
  * respectively RW for Supervisor and no access for User, or RO for
- * Supervisor and no access for user.
+ * Supervisor and no access for user and NA for ALL.
  * Then we use the APG to say whether accesses are according to Page rules or
  * "all Supervisor" rules (Access to all)
- * Therefore, we define 2 APG groups. lsb is _PAGE_USER
- * 0 => No user => 01 (all accesses performed according to page definition)
- * 1 => User => 00 (all accesses performed as supervisor
- *                                 according to page definition)
- */
-#define MD_APG_INIT	0x4fffffff
+ * We also use the 2nd APG bit for _PAGE_ACCESSED when having SWAP:
+ * When that bit is not set access is done iaw "all user"
+ * which means no access iaw page rules.
+ * Therefore, we define 4 APG groups. lsb is _PMD_USER, 2nd is _PAGE_ACCESSED
+ * 0x => No access => 11 (all accesses performed as user iaw page definition)
+ * 10 => No user => 01 (all accesses performed according to page definition)
+ * 11 => User => 00 (all accesses performed as supervisor iaw page definition)
+ * We define all 16 groups so that all other bits of APG can take any value
+ */
+#ifdef CONFIG_SWAP
+#define MD_APG_INIT	0xf4f4f4f4
+#else
+#define MD_APG_INIT	0x44444444
+#endif
 
 /* The effective page number register.  When read, contains the information
  * about the last instruction TLB miss.  When MD_RPN is written, bits in
@@ -164,6 +180,12 @@
  */
 #define SPRN_M_TW	799
 
+/* APGs */
+#define M_APG0		0x00000000
+#define M_APG1		0x00000020
+#define M_APG2		0x00000040
+#define M_APG3		0x00000060
+
 #ifndef __ASSEMBLY__
 typedef struct {
 	unsigned int id;
diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h
index 6364f5c2cc3e..bb38312cff28 100644
--- a/arch/powerpc/include/asm/mmu.h
+++ b/arch/powerpc/include/asm/mmu.h
@@ -260,6 +260,15 @@ static inline bool early_radix_enabled(void)
 }
 #endif
 
+#ifdef CONFIG_PPC_MEM_KEYS
+extern u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address);
+#else
+static inline u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address)
+{
+	return 0;
+}
+#endif /* CONFIG_PPC_MEM_KEYS */
+
 #endif /* !__ASSEMBLY__ */
 
 /* The kernel use the constants below to index in the page sizes array.
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index e2a2b8400490..3a15b6db9501 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -92,15 +92,23 @@ static inline void dec_mm_active_cpus(struct mm_struct *mm)
 static inline void mm_context_add_copro(struct mm_struct *mm)
 {
 	/*
-	 * On hash, should only be called once over the lifetime of
-	 * the context, as we can't decrement the active cpus count
-	 * and flush properly for the time being.
+	 * If any copro is in use, increment the active CPU count
+	 * in order to force TLB invalidations to be global as to
+	 * propagate to the Nest MMU.
 	 */
-	inc_mm_active_cpus(mm);
+	if (atomic_inc_return(&mm->context.copros) == 1)
+		inc_mm_active_cpus(mm);
 }
 
 static inline void mm_context_remove_copro(struct mm_struct *mm)
 {
+	int c;
+
+	c = atomic_dec_if_positive(&mm->context.copros);
+
+	/* Detect imbalance between add and remove */
+	WARN_ON(c < 0);
+
 	/*
 	 * Need to broadcast a global flush of the full mm before
 	 * decrementing active_cpus count, as the next TLBI may be
@@ -111,7 +119,7 @@ static inline void mm_context_remove_copro(struct mm_struct *mm)
 	 * for the time being. Invalidations will remain global if
 	 * used on hash.
 	 */
-	if (radix_enabled()) {
+	if (c == 0 && radix_enabled()) {
 		flush_all_mm(mm);
 		dec_mm_active_cpus(mm);
 	}
@@ -187,11 +195,33 @@ static inline void arch_bprm_mm_init(struct mm_struct *mm,
 {
 }
 
+#ifdef CONFIG_PPC_MEM_KEYS
+bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
+			       bool execute, bool foreign);
+#else /* CONFIG_PPC_MEM_KEYS */
 static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
 		bool write, bool execute, bool foreign)
 {
 	/* by default, allow everything */
 	return true;
 }
+
+#define pkey_mm_init(mm)
+#define thread_pkey_regs_save(thread)
+#define thread_pkey_regs_restore(new_thread, old_thread)
+#define thread_pkey_regs_init(thread)
+
+static inline int vma_pkey(struct vm_area_struct *vma)
+{
+	return 0;
+}
+
+static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
+{
+	return 0x0UL;
+}
+
+#endif /* CONFIG_PPC_MEM_KEYS */
+
 #endif /* __KERNEL__ */
 #endif /* __ASM_POWERPC_MMU_CONTEXT_H */
diff --git a/arch/powerpc/include/asm/module.h b/arch/powerpc/include/asm/module.h
index 6c0132c7212f..7e28442827f1 100644
--- a/arch/powerpc/include/asm/module.h
+++ b/arch/powerpc/include/asm/module.h
@@ -45,6 +45,9 @@ struct mod_arch_specific {
 	unsigned long tramp;
 #endif
 
+	/* For module function descriptor dereference */
+	unsigned long start_opd;
+	unsigned long end_opd;
 #else /* powerpc64 */
 	/* Indices of PLT sections within module. */
 	unsigned int core_plt_section;
diff --git a/arch/powerpc/include/asm/mpic_timer.h b/arch/powerpc/include/asm/mpic_timer.h
index 0e23cd4ac8aa..13e6702ec458 100644
--- a/arch/powerpc/include/asm/mpic_timer.h
+++ b/arch/powerpc/include/asm/mpic_timer.h
@@ -29,17 +29,17 @@ struct mpic_timer {
 
 #ifdef CONFIG_MPIC_TIMER
 struct mpic_timer *mpic_request_timer(irq_handler_t fn,  void *dev,
-		const struct timeval *time);
+		time64_t time);
 void mpic_start_timer(struct mpic_timer *handle);
 void mpic_stop_timer(struct mpic_timer *handle);
-void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time);
+void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time);
 void mpic_free_timer(struct mpic_timer *handle);
 #else
 struct mpic_timer *mpic_request_timer(irq_handler_t fn,  void *dev,
-		const struct timeval *time) { return NULL; }
+		time64_t time) { return NULL; }
 void mpic_start_timer(struct mpic_timer *handle) { }
 void mpic_stop_timer(struct mpic_timer *handle) { }
-void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time) { }
+void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time) { }
 void mpic_free_timer(struct mpic_timer *handle) { }
 #endif
 
diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h
index e97f58689ca7..9c80939b4d14 100644
--- a/arch/powerpc/include/asm/nmi.h
+++ b/arch/powerpc/include/asm/nmi.h
@@ -4,10 +4,6 @@
 
 #ifdef CONFIG_PPC_WATCHDOG
 extern void arch_touch_nmi_watchdog(void);
-extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask,
-					   bool exclude_self);
-#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace
-
 #else
 static inline void arch_touch_nmi_watchdog(void) {}
 #endif
diff --git a/arch/powerpc/include/asm/nohash/32/pgalloc.h b/arch/powerpc/include/asm/nohash/32/pgalloc.h
index d072139ff2e5..29d37bd1f3b3 100644
--- a/arch/powerpc/include/asm/nohash/32/pgalloc.h
+++ b/arch/powerpc/include/asm/nohash/32/pgalloc.h
@@ -61,7 +61,8 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp,
 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
 				pgtable_t pte_page)
 {
-	*pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_PRESENT);
+	*pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_USER |
+		      _PMD_PRESENT);
 }
 
 #define pmd_pgtable(pmd) pmd_page(pmd)
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index cc2bfec3aa3b..03bbd1149530 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -24,6 +24,7 @@ extern int icache_44x_need_flush;
 #define PGD_INDEX_SIZE	(32 - PGDIR_SHIFT)
 
 #define PMD_CACHE_INDEX	PMD_INDEX_SIZE
+#define PUD_CACHE_INDEX	PUD_INDEX_SIZE
 
 #ifndef __ASSEMBLY__
 #define PTE_TABLE_SIZE	(sizeof(pte_t) << PTE_INDEX_SIZE)
@@ -282,7 +283,7 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
 {
 	unsigned long set = pte_val(entry) &
 		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
-	unsigned long clr = ~pte_val(entry) & _PAGE_RO;
+	unsigned long clr = ~pte_val(entry) & (_PAGE_RO | _PAGE_NA);
 
 	pte_update(ptep, clr, set);
 }
diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
index 6dc0180fd5c7..f04cb46ae8a1 100644
--- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
@@ -31,37 +31,34 @@
 /* Definitions for 8xx embedded chips. */
 #define _PAGE_PRESENT	0x0001	/* Page is valid */
 #define _PAGE_NO_CACHE	0x0002	/* I: cache inhibit */
-#define _PAGE_SHARED	0x0004	/* No ASID (context) compare */
-#define _PAGE_SPECIAL	0x0008	/* SW entry, forced to 0 by the TLB miss */
+#define _PAGE_PRIVILEGED	0x0004	/* No ASID (context) compare */
+#define _PAGE_HUGE	0x0008	/* SPS: Small Page Size (1 if 16k, 512k or 8M)*/
 #define _PAGE_DIRTY	0x0100	/* C: page changed */
 
 /* These 4 software bits must be masked out when the L2 entry is loaded
  * into the TLB.
  */
 #define _PAGE_GUARDED	0x0010	/* Copied to L1 G entry in DTLB */
-#define _PAGE_USER	0x0020	/* Copied to L1 APG lsb */
-#define _PAGE_EXEC	0x0040	/* Copied to L1 APG */
-#define _PAGE_WRITETHRU	0x0080	/* software: caching is write through */
-#define _PAGE_ACCESSED	0x0800	/* software: page referenced */
+#define _PAGE_SPECIAL	0x0020	/* SW entry */
+#define _PAGE_EXEC	0x0040	/* Copied to PP (bit 21) in ITLB */
+#define _PAGE_ACCESSED	0x0080	/* software: page referenced */
 
+#define _PAGE_NA	0x0200	/* Supervisor NA, User no access */
 #define _PAGE_RO	0x0600	/* Supervisor RO, User no access */
 
 #define _PMD_PRESENT	0x0001
-#define _PMD_BAD	0x0ff0
+#define _PMD_BAD	0x0fd0
 #define _PMD_PAGE_MASK	0x000c
 #define _PMD_PAGE_8M	0x000c
 #define _PMD_PAGE_512K	0x0004
+#define _PMD_USER	0x0020	/* APG 1 */
 
 /* Until my rework is finished, 8xx still needs atomic PTE updates */
 #define PTE_ATOMIC_UPDATES	1
 
-/* We need to add _PAGE_SHARED to kernel pages */
-#define _PAGE_KERNEL_RO		(_PAGE_SHARED | _PAGE_RO)
-#define _PAGE_KERNEL_ROX	(_PAGE_SHARED | _PAGE_RO | _PAGE_EXEC)
-#define _PAGE_KERNEL_RW		(_PAGE_SHARED | _PAGE_DIRTY | _PAGE_RW | \
-				 _PAGE_HWWRITE)
-#define _PAGE_KERNEL_RWX	(_PAGE_SHARED | _PAGE_DIRTY | _PAGE_RW | \
-				 _PAGE_HWWRITE | _PAGE_EXEC)
+#ifdef CONFIG_PPC_16K_PAGES
+#define _PAGE_PSIZE	_PAGE_HUGE
+#endif
 
 #endif /* __KERNEL__ */
 #endif /*  _ASM_POWERPC_NOHASH_32_PTE_8xx_H */
diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h
index abddf5830ad5..5c5f75d005ad 100644
--- a/arch/powerpc/include/asm/nohash/64/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
@@ -27,6 +27,7 @@
 #else
 #define PMD_CACHE_INDEX	PMD_INDEX_SIZE
 #endif
+#define PUD_CACHE_INDEX PUD_INDEX_SIZE
 
 /*
  * Define the address range of the kernel non-linear virtual area
diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h
index 5c68f4a59f75..c56de1e8026f 100644
--- a/arch/powerpc/include/asm/nohash/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/pgtable.h
@@ -45,6 +45,29 @@ static inline int pte_present(pte_t pte)
 	return pte_val(pte) & _PAGE_PRESENT;
 }
 
+/*
+ * We only find page table entry in the last level
+ * Hence no need for other accessors
+ */
+#define pte_access_permitted pte_access_permitted
+static inline bool pte_access_permitted(pte_t pte, bool write)
+{
+	unsigned long pteval = pte_val(pte);
+	/*
+	 * A read-only access is controlled by _PAGE_USER bit.
+	 * We have _PAGE_READ set for WRITE and EXECUTE
+	 */
+	unsigned long need_pte_bits = _PAGE_PRESENT | _PAGE_USER;
+
+	if (write)
+		need_pte_bits |= _PAGE_WRITE;
+
+	if ((pteval & need_pte_bits) != need_pte_bits)
+		return false;
+
+	return true;
+}
+
 /* Conversion functions: convert a page and protection to a page entry,
  * and a page entry and page directory to the page they refer to.
  *
@@ -103,7 +126,7 @@ static inline pte_t pte_mkspecial(pte_t pte)
 
 static inline pte_t pte_mkhuge(pte_t pte)
 {
-	return pte;
+	return __pte(pte_val(pte) | _PAGE_HUGE);
 }
 
 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
@@ -212,8 +235,10 @@ extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addre
 #define pgprot_cached(prot)       (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \
 				            _PAGE_COHERENT))
 
+#if _PAGE_WRITETHRU != 0
 #define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \
 				            _PAGE_COHERENT | _PAGE_WRITETHRU))
+#endif
 
 #define pgprot_cached_noncoherent(prot) \
 		(__pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL))
diff --git a/arch/powerpc/include/asm/nohash/pte-book3e.h b/arch/powerpc/include/asm/nohash/pte-book3e.h
index 2da4532ca377..ccee8eb509bb 100644
--- a/arch/powerpc/include/asm/nohash/pte-book3e.h
+++ b/arch/powerpc/include/asm/nohash/pte-book3e.h
@@ -55,6 +55,7 @@
 #define _PAGE_KERNEL_RWX	(_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY | _PAGE_BAP_SX)
 #define _PAGE_KERNEL_ROX	(_PAGE_BAP_SR | _PAGE_BAP_SX)
 #define _PAGE_USER		(_PAGE_BAP_UR | _PAGE_BAP_SR) /* Can be read */
+#define _PAGE_PRIVILEGED	(_PAGE_BAP_SR)
 
 #define _PAGE_HASHPTE	0
 #define _PAGE_BUSY	0
diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h
index 233c7504b1f2..94bd1bf2c873 100644
--- a/arch/powerpc/include/asm/opal-api.h
+++ b/arch/powerpc/include/asm/opal-api.h
@@ -201,7 +201,10 @@
 #define OPAL_SET_POWER_SHIFT_RATIO		155
 #define OPAL_SENSOR_GROUP_CLEAR			156
 #define OPAL_PCI_SET_P2P			157
-#define OPAL_LAST				157
+#define OPAL_NPU_SPA_SETUP			159
+#define OPAL_NPU_SPA_CLEAR_CACHE		160
+#define OPAL_NPU_TL_SET				161
+#define OPAL_LAST				161
 
 /* Device tree flags */
 
@@ -1073,6 +1076,7 @@ enum {
 /* Flags for OPAL_XIVE_GET/SET_VP_INFO */
 enum {
 	OPAL_XIVE_VP_ENABLED		= 0x00000001,
+	OPAL_XIVE_VP_SINGLE_ESCALATION	= 0x00000002,
 };
 
 /* "Any chip" replacement for chip ID for allocation functions */
diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h
index 0c545f7fc77b..12e70fb58700 100644
--- a/arch/powerpc/include/asm/opal.h
+++ b/arch/powerpc/include/asm/opal.h
@@ -34,6 +34,12 @@ int64_t opal_npu_init_context(uint64_t phb_id, int pasid, uint64_t msr,
 			uint64_t bdf);
 int64_t opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid,
 			uint64_t lpcr);
+int64_t opal_npu_spa_setup(uint64_t phb_id, uint32_t bdfn,
+			uint64_t addr, uint64_t PE_mask);
+int64_t opal_npu_spa_clear_cache(uint64_t phb_id, uint32_t bdfn,
+				uint64_t PE_handle);
+int64_t opal_npu_tl_set(uint64_t phb_id, uint32_t bdfn, long cap,
+			uint64_t rate_phys, uint32_t size);
 int64_t opal_console_write(int64_t term_number, __be64 *length,
 			   const uint8_t *buffer);
 int64_t opal_console_read(int64_t term_number, __be64 *length,
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 23ac7fc0af23..b62c31037cad 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -159,7 +159,7 @@ struct paca_struct {
 	u64 saved_r1;			/* r1 save for RTAS calls or PM */
 	u64 saved_msr;			/* MSR saved here by enter_rtas */
 	u16 trap_save;			/* Used when bad stack is encountered */
-	u8 soft_enabled;		/* irq soft-enable flag */
+	u8 irq_soft_mask;		/* mask for irq soft masking */
 	u8 irq_happened;		/* irq happened while soft-disabled */
 	u8 io_sync;			/* writel() needs spin_unlock sync */
 	u8 irq_work_pending;		/* IRQ_WORK interrupt while soft-disable */
@@ -239,8 +239,7 @@ struct paca_struct {
 	 */
 	u64 exrfi[EX_SIZE] __aligned(0x80);
 	void *rfi_flush_fallback_area;
-	u64 l1d_flush_congruence;
-	u64 l1d_flush_sets;
+	u64 l1d_flush_size;
 #endif
 };
 
diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h
index 62ed83db04ae..94d449031b18 100644
--- a/arch/powerpc/include/asm/pci-bridge.h
+++ b/arch/powerpc/include/asm/pci-bridge.h
@@ -197,25 +197,22 @@ struct pci_dn {
 	struct	iommu_table_group *table_group;	/* for phb's or bridges */
 
 	int	pci_ext_config_space;	/* for pci devices */
-
-	struct	pci_dev *pcidev;	/* back-pointer to the pci device */
 #ifdef CONFIG_EEH
 	struct eeh_dev *edev;		/* eeh device */
 #endif
 #define IODA_INVALID_PE		0xFFFFFFFF
-#ifdef CONFIG_PPC_POWERNV
 	unsigned int pe_number;
-	int     vf_index;		/* VF index in the PF */
 #ifdef CONFIG_PCI_IOV
+	int     vf_index;		/* VF index in the PF */
 	u16     vfs_expanded;		/* number of VFs IOV BAR expanded */
 	u16     num_vfs;		/* number of VFs enabled*/
 	unsigned int *pe_num_map;	/* PE# for the first VF PE or array */
 	bool    m64_single_mode;	/* Use M64 BAR in Single Mode */
 #define IODA_INVALID_M64        (-1)
-	int     (*m64_map)[PCI_SRIOV_NUM_BARS];
+	int     (*m64_map)[PCI_SRIOV_NUM_BARS];	/* Only used on powernv */
+	int     last_allow_rc;			/* Only used on pseries */
 #endif /* CONFIG_PCI_IOV */
 	int	mps;			/* Maximum Payload Size */
-#endif
 	struct list_head child_list;
 	struct list_head list;
 	struct resource holes[PCI_SRIOV_NUM_BARS];
diff --git a/arch/powerpc/include/asm/pci.h b/arch/powerpc/include/asm/pci.h
index 8dc32eacc97c..d82802ff5088 100644
--- a/arch/powerpc/include/asm/pci.h
+++ b/arch/powerpc/include/asm/pci.h
@@ -121,6 +121,8 @@ extern int remove_phb_dynamic(struct pci_controller *phb);
 extern struct pci_dev *of_create_pci_dev(struct device_node *node,
 					struct pci_bus *bus, int devfn);
 
+extern unsigned int pci_parse_of_flags(u32 addr0, int bridge);
+
 extern void of_scan_pci_bridge(struct pci_dev *dev);
 
 extern void of_scan_bus(struct device_node *node, struct pci_bus *bus);
diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h
new file mode 100644
index 000000000000..0409c80c32c0
--- /dev/null
+++ b/arch/powerpc/include/asm/pkeys.h
@@ -0,0 +1,218 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * PowerPC Memory Protection Keys management
+ *
+ * Copyright 2017, Ram Pai, IBM Corporation.
+ */
+
+#ifndef _ASM_POWERPC_KEYS_H
+#define _ASM_POWERPC_KEYS_H
+
+#include <linux/jump_label.h>
+#include <asm/firmware.h>
+
+DECLARE_STATIC_KEY_TRUE(pkey_disabled);
+extern int pkeys_total; /* total pkeys as per device tree */
+extern u32 initial_allocation_mask; /* bits set for reserved keys */
+
+/*
+ * Define these here temporarily so we're not dependent on patching linux/mm.h.
+ * Once it's updated we can drop these.
+ */
+#ifndef VM_PKEY_BIT0
+# define VM_PKEY_SHIFT	VM_HIGH_ARCH_BIT_0
+# define VM_PKEY_BIT0	VM_HIGH_ARCH_0
+# define VM_PKEY_BIT1	VM_HIGH_ARCH_1
+# define VM_PKEY_BIT2	VM_HIGH_ARCH_2
+# define VM_PKEY_BIT3	VM_HIGH_ARCH_3
+# define VM_PKEY_BIT4	VM_HIGH_ARCH_4
+#endif
+
+#define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | \
+			    VM_PKEY_BIT3 | VM_PKEY_BIT4)
+
+/* Override any generic PKEY permission defines */
+#define PKEY_DISABLE_EXECUTE   0x4
+#define PKEY_ACCESS_MASK       (PKEY_DISABLE_ACCESS | \
+				PKEY_DISABLE_WRITE  | \
+				PKEY_DISABLE_EXECUTE)
+
+static inline u64 pkey_to_vmflag_bits(u16 pkey)
+{
+	return (((u64)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS);
+}
+
+static inline u64 vmflag_to_pte_pkey_bits(u64 vm_flags)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return 0x0UL;
+
+	return (((vm_flags & VM_PKEY_BIT0) ? H_PTE_PKEY_BIT4 : 0x0UL) |
+		((vm_flags & VM_PKEY_BIT1) ? H_PTE_PKEY_BIT3 : 0x0UL) |
+		((vm_flags & VM_PKEY_BIT2) ? H_PTE_PKEY_BIT2 : 0x0UL) |
+		((vm_flags & VM_PKEY_BIT3) ? H_PTE_PKEY_BIT1 : 0x0UL) |
+		((vm_flags & VM_PKEY_BIT4) ? H_PTE_PKEY_BIT0 : 0x0UL));
+}
+
+static inline int vma_pkey(struct vm_area_struct *vma)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return 0;
+	return (vma->vm_flags & ARCH_VM_PKEY_FLAGS) >> VM_PKEY_SHIFT;
+}
+
+#define arch_max_pkey() pkeys_total
+
+static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
+{
+	return (((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT2) ? HPTE_R_KEY_BIT2 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT3) ? HPTE_R_KEY_BIT3 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL));
+}
+
+static inline u16 pte_to_pkey_bits(u64 pteflags)
+{
+	return (((pteflags & H_PTE_PKEY_BIT0) ? 0x10 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT1) ? 0x8 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT2) ? 0x4 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT3) ? 0x2 : 0x0UL) |
+		((pteflags & H_PTE_PKEY_BIT4) ? 0x1 : 0x0UL));
+}
+
+#define pkey_alloc_mask(pkey) (0x1 << pkey)
+
+#define mm_pkey_allocation_map(mm) (mm->context.pkey_allocation_map)
+
+#define __mm_pkey_allocated(mm, pkey) {	\
+	mm_pkey_allocation_map(mm) |= pkey_alloc_mask(pkey); \
+}
+
+#define __mm_pkey_free(mm, pkey) {	\
+	mm_pkey_allocation_map(mm) &= ~pkey_alloc_mask(pkey);	\
+}
+
+#define __mm_pkey_is_allocated(mm, pkey)	\
+	(mm_pkey_allocation_map(mm) & pkey_alloc_mask(pkey))
+
+#define __mm_pkey_is_reserved(pkey) (initial_allocation_mask & \
+				       pkey_alloc_mask(pkey))
+
+static inline bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey)
+{
+	/* A reserved key is never considered as 'explicitly allocated' */
+	return ((pkey < arch_max_pkey()) &&
+		!__mm_pkey_is_reserved(pkey) &&
+		__mm_pkey_is_allocated(mm, pkey));
+}
+
+extern void __arch_activate_pkey(int pkey);
+extern void __arch_deactivate_pkey(int pkey);
+/*
+ * Returns a positive, 5-bit key on success, or -1 on failure.
+ * Relies on the mmap_sem to protect against concurrency in mm_pkey_alloc() and
+ * mm_pkey_free().
+ */
+static inline int mm_pkey_alloc(struct mm_struct *mm)
+{
+	/*
+	 * Note: this is the one and only place we make sure that the pkey is
+	 * valid as far as the hardware is concerned. The rest of the kernel
+	 * trusts that only good, valid pkeys come out of here.
+	 */
+	u32 all_pkeys_mask = (u32)(~(0x0));
+	int ret;
+
+	if (static_branch_likely(&pkey_disabled))
+		return -1;
+
+	/*
+	 * Are we out of pkeys? We must handle this specially because ffz()
+	 * behavior is undefined if there are no zeros.
+	 */
+	if (mm_pkey_allocation_map(mm) == all_pkeys_mask)
+		return -1;
+
+	ret = ffz((u32)mm_pkey_allocation_map(mm));
+	__mm_pkey_allocated(mm, ret);
+
+	/*
+	 * Enable the key in the hardware
+	 */
+	if (ret > 0)
+		__arch_activate_pkey(ret);
+	return ret;
+}
+
+static inline int mm_pkey_free(struct mm_struct *mm, int pkey)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return -1;
+
+	if (!mm_pkey_is_allocated(mm, pkey))
+		return -EINVAL;
+
+	/*
+	 * Disable the key in the hardware
+	 */
+	__arch_deactivate_pkey(pkey);
+	__mm_pkey_free(mm, pkey);
+
+	return 0;
+}
+
+/*
+ * Try to dedicate one of the protection keys to be used as an
+ * execute-only protection key.
+ */
+extern int __execute_only_pkey(struct mm_struct *mm);
+static inline int execute_only_pkey(struct mm_struct *mm)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return -1;
+
+	return __execute_only_pkey(mm);
+}
+
+extern int __arch_override_mprotect_pkey(struct vm_area_struct *vma,
+					 int prot, int pkey);
+static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma,
+					      int prot, int pkey)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return 0;
+
+	/*
+	 * Is this an mprotect_pkey() call? If so, never override the value that
+	 * came from the user.
+	 */
+	if (pkey != -1)
+		return pkey;
+
+	return __arch_override_mprotect_pkey(vma, prot, pkey);
+}
+
+extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
+				       unsigned long init_val);
+static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
+					    unsigned long init_val)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return -EINVAL;
+	return __arch_set_user_pkey_access(tsk, pkey, init_val);
+}
+
+static inline bool arch_pkeys_enabled(void)
+{
+	return !static_branch_likely(&pkey_disabled);
+}
+
+extern void pkey_mm_init(struct mm_struct *mm);
+extern bool arch_supports_pkeys(int cap);
+extern unsigned int arch_usable_pkeys(void);
+extern void thread_pkey_regs_save(struct thread_struct *thread);
+extern void thread_pkey_regs_restore(struct thread_struct *new_thread,
+				     struct thread_struct *old_thread);
+extern void thread_pkey_regs_init(struct thread_struct *thread);
+#endif /*_ASM_POWERPC_KEYS_H */
diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h
new file mode 100644
index 000000000000..f6945d3bc971
--- /dev/null
+++ b/arch/powerpc/include/asm/pnv-ocxl.h
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#ifndef _ASM_PNV_OCXL_H
+#define _ASM_PNV_OCXL_H
+
+#include <linux/pci.h>
+
+#define PNV_OCXL_TL_MAX_TEMPLATE        63
+#define PNV_OCXL_TL_BITS_PER_RATE       4
+#define PNV_OCXL_TL_RATE_BUF_SIZE       ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8)
+
+extern int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled,
+			u16 *supported);
+extern int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count);
+
+extern int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap,
+			char *rate_buf, int rate_buf_size);
+extern int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap,
+			uint64_t rate_buf_phys, int rate_buf_size);
+
+extern int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq);
+extern void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar,
+				void __iomem *tfc, void __iomem *pe_handle);
+extern int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr,
+				void __iomem **dar, void __iomem **tfc,
+				void __iomem **pe_handle);
+
+extern int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask,
+			void **platform_data);
+extern void pnv_ocxl_spa_release(void *platform_data);
+extern int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle);
+
+extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr);
+extern void pnv_ocxl_free_xive_irq(u32 irq);
+
+#endif /* _ASM_PNV_OCXL_H */
diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index ce0930d68857..f1083bcf449c 100644
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h
@@ -156,6 +156,12 @@
 #define OP_31_XOP_LFDX          599
 #define OP_31_XOP_LFDUX		631
 
+/* VMX Vector Load Instructions */
+#define OP_31_XOP_LVX           103
+
+/* VMX Vector Store Instructions */
+#define OP_31_XOP_STVX          231
+
 #define OP_LWZ  32
 #define OP_STFS 52
 #define OP_STFSU 53
@@ -236,6 +242,7 @@
 #define PPC_INST_RFCI			0x4c000066
 #define PPC_INST_RFDI			0x4c00004e
 #define PPC_INST_RFMCI			0x4c00004c
+#define PPC_INST_MFSPR			0x7c0002a6
 #define PPC_INST_MFSPR_DSCR		0x7c1102a6
 #define PPC_INST_MFSPR_DSCR_MASK	0xfc1ffffe
 #define PPC_INST_MTSPR_DSCR		0x7c1103a6
@@ -383,6 +390,7 @@
 #define __PPC_ME64(s)	__PPC_MB64(s)
 #define __PPC_BI(s)	(((s) & 0x1f) << 16)
 #define __PPC_CT(t)	(((t) & 0x0f) << 21)
+#define __PPC_SPR(r)	((((r) & 0x1f) << 16) | ((((r) >> 5) & 0x1f) << 11))
 
 /*
  * Only use the larx hint bit on 64bit CPUs. e500v1/v2 based CPUs will treat a
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index bdab3b74eb98..01299cdc9806 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -309,6 +309,11 @@ struct thread_struct {
 	struct thread_vr_state ckvr_state; /* Checkpointed VR state */
 	unsigned long	ckvrsave; /* Checkpointed VRSAVE */
 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
+#ifdef CONFIG_PPC_MEM_KEYS
+	unsigned long	amr;
+	unsigned long	iamr;
+	unsigned long	uamor;
+#endif
 #ifdef CONFIG_KVM_BOOK3S_32_HANDLER
 	void*		kvm_shadow_vcpu; /* KVM internal data */
 #endif /* CONFIG_KVM_BOOK3S_32_HANDLER */
diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h
index 825bd5998701..b04c5ce8191b 100644
--- a/arch/powerpc/include/asm/prom.h
+++ b/arch/powerpc/include/asm/prom.h
@@ -80,21 +80,20 @@ extern void of_instantiate_rtc(void);
 
 extern int of_get_ibm_chip_id(struct device_node *np);
 
-/* The of_drconf_cell struct defines the layout of the LMB array
- * specified in the device tree property
- * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory
- */
-struct of_drconf_cell {
-	u64	base_addr;
-	u32	drc_index;
-	u32	reserved;
-	u32	aa_index;
-	u32	flags;
+struct of_drc_info {
+	char *drc_type;
+	char *drc_name_prefix;
+	u32 drc_index_start;
+	u32 drc_name_suffix_start;
+	u32 num_sequential_elems;
+	u32 sequential_inc;
+	u32 drc_power_domain;
+	u32 last_drc_index;
 };
 
-#define DRCONF_MEM_ASSIGNED	0x00000008
-#define DRCONF_MEM_AI_INVALID	0x00000040
-#define DRCONF_MEM_RESERVED	0x00000080
+extern int of_read_drc_info_cell(struct property **prop,
+			const __be32 **curval, struct of_drc_info *data);
+
 
 /*
  * There are two methods for telling firmware what our capabilities are.
@@ -159,6 +158,7 @@ struct of_drconf_cell {
 #define OV5_PFO_HW_842		0x1140	/* PFO Compression Accelerator */
 #define OV5_PFO_HW_ENCR		0x1120	/* PFO Encryption Accelerator */
 #define OV5_SUB_PROCESSORS	0x1501	/* 1,2,or 4 Sub-Processors supported */
+#define OV5_DRMEM_V2		0x1680	/* ibm,dynamic-reconfiguration-v2 */
 #define OV5_XIVE_SUPPORT	0x17C0	/* XIVE Exploitation Support Mask */
 #define OV5_XIVE_LEGACY		0x1700	/* XIVE legacy mode Only */
 #define OV5_XIVE_EXPLOIT	0x1740	/* XIVE exploitation mode Only */
@@ -175,6 +175,7 @@ struct of_drconf_cell {
 #define OV5_HASH_GTSE		0x1940	/* Guest Translation Shoot Down Avail */
 /* Radix Table Extensions */
 #define OV5_RADIX_GTSE		0x1A40	/* Guest Translation Shoot Down Avail */
+#define OV5_DRC_INFO		0x1640	/* Redef Prop Structures: drc-info   */
 
 /* Option Vector 6: IBM PAPR hints */
 #define OV6_LINUX		0x02	/* Linux is our OS */
diff --git a/arch/powerpc/include/asm/pte-common.h b/arch/powerpc/include/asm/pte-common.h
index ce142ef99ba7..c4a72c7a8c83 100644
--- a/arch/powerpc/include/asm/pte-common.h
+++ b/arch/powerpc/include/asm/pte-common.h
@@ -8,9 +8,6 @@
 #ifndef _PAGE_HASHPTE
 #define _PAGE_HASHPTE	0
 #endif
-#ifndef _PAGE_SHARED
-#define _PAGE_SHARED	0
-#endif
 #ifndef _PAGE_HWWRITE
 #define _PAGE_HWWRITE	0
 #endif
@@ -45,6 +42,20 @@
 #ifndef _PAGE_PTE
 #define _PAGE_PTE 0
 #endif
+/* At least one of _PAGE_PRIVILEGED or _PAGE_USER must be defined */
+#ifndef _PAGE_PRIVILEGED
+#define _PAGE_PRIVILEGED 0
+#else
+#ifndef _PAGE_USER
+#define _PAGE_USER 0
+#endif
+#endif
+#ifndef _PAGE_NA
+#define _PAGE_NA 0
+#endif
+#ifndef _PAGE_HUGE
+#define _PAGE_HUGE 0
+#endif
 
 #ifndef _PMD_PRESENT_MASK
 #define _PMD_PRESENT_MASK	_PMD_PRESENT
@@ -53,17 +64,22 @@
 #define _PMD_SIZE	0
 #define PMD_PAGE_SIZE(pmd)	bad_call_to_PMD_PAGE_SIZE()
 #endif
+#ifndef _PMD_USER
+#define _PMD_USER	0
+#endif
 #ifndef _PAGE_KERNEL_RO
-#define _PAGE_KERNEL_RO		(_PAGE_RO)
+#define _PAGE_KERNEL_RO		(_PAGE_PRIVILEGED | _PAGE_RO)
 #endif
 #ifndef _PAGE_KERNEL_ROX
-#define _PAGE_KERNEL_ROX	(_PAGE_EXEC | _PAGE_RO)
+#define _PAGE_KERNEL_ROX	(_PAGE_PRIVILEGED | _PAGE_RO | _PAGE_EXEC)
 #endif
 #ifndef _PAGE_KERNEL_RW
-#define _PAGE_KERNEL_RW		(_PAGE_DIRTY | _PAGE_RW | _PAGE_HWWRITE)
+#define _PAGE_KERNEL_RW		(_PAGE_PRIVILEGED | _PAGE_DIRTY | _PAGE_RW | \
+				 _PAGE_HWWRITE)
 #endif
 #ifndef _PAGE_KERNEL_RWX
-#define _PAGE_KERNEL_RWX	(_PAGE_DIRTY | _PAGE_RW | _PAGE_HWWRITE | _PAGE_EXEC)
+#define _PAGE_KERNEL_RWX	(_PAGE_PRIVILEGED | _PAGE_DIRTY | _PAGE_RW | \
+				 _PAGE_HWWRITE | _PAGE_EXEC)
 #endif
 #ifndef _PAGE_HPTEFLAGS
 #define _PAGE_HPTEFLAGS _PAGE_HASHPTE
@@ -85,7 +101,7 @@ extern unsigned long bad_call_to_PMD_PAGE_SIZE(void);
  */
 static inline bool pte_user(pte_t pte)
 {
-	return (pte_val(pte) & _PAGE_USER) == _PAGE_USER;
+	return (pte_val(pte) & (_PAGE_USER | _PAGE_PRIVILEGED)) == _PAGE_USER;
 }
 #endif /* __ASSEMBLY__ */
 
@@ -115,7 +131,8 @@ static inline bool pte_user(pte_t pte)
 /* Mask of bits returned by pte_pgprot() */
 #define PAGE_PROT_BITS	(_PAGE_GUARDED | _PAGE_COHERENT | _PAGE_NO_CACHE | \
 			 _PAGE_WRITETHRU | _PAGE_ENDIAN | _PAGE_4K_PFN | \
-			 _PAGE_USER | _PAGE_ACCESSED | _PAGE_RO | \
+			 _PAGE_USER | _PAGE_ACCESSED | _PAGE_RO | _PAGE_NA | \
+			 _PAGE_PRIVILEGED | \
 			 _PAGE_RW | _PAGE_HWWRITE | _PAGE_DIRTY | _PAGE_EXEC)
 
 /*
@@ -142,7 +159,7 @@ static inline bool pte_user(pte_t pte)
  *
  * Note due to the way vm flags are laid out, the bits are XWR
  */
-#define PAGE_NONE	__pgprot(_PAGE_BASE)
+#define PAGE_NONE	__pgprot(_PAGE_BASE | _PAGE_NA)
 #define PAGE_SHARED	__pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW)
 #define PAGE_SHARED_X	__pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | \
 				 _PAGE_EXEC)
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index b779f3ccd412..e6c7eadf6bce 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -312,7 +312,6 @@
 				 DSISR_BAD_EXT_CTRL)
 #define	  DSISR_BAD_FAULT_64S	(DSISR_BAD_FAULT_32S	| \
 				 DSISR_ATTR_CONFLICT	| \
-				 DSISR_KEYFAULT		| \
 				 DSISR_UNSUPP_MMU	| \
 				 DSISR_PRTABLE_FAULT	| \
 				 DSISR_ICSWX_NO_CT	| \
@@ -432,8 +431,9 @@
 #define SPRN_LPID	0x13F	/* Logical Partition Identifier */
 #endif
 #define   LPID_RSVD	0x3ff		/* Reserved LPID for partn switching */
-#define	SPRN_HMER	0x150	/* Hardware m? error recovery */
-#define	SPRN_HMEER	0x151	/* Hardware m? enable error recovery */
+#define	SPRN_HMER	0x150	/* Hypervisor maintenance exception reg */
+#define   HMER_DEBUG_TRIG	(1ul << (63 - 17)) /* Debug trigger */
+#define	SPRN_HMEER	0x151	/* Hyp maintenance exception enable reg */
 #define SPRN_PCR	0x152	/* Processor compatibility register */
 #define   PCR_VEC_DIS	(1ul << (63-0))	/* Vec. disable (bit NA since POWER8) */
 #define   PCR_VSX_DIS	(1ul << (63-1))	/* VSX disable (bit NA since POWER8) */
diff --git a/arch/powerpc/include/asm/reg_8xx.h b/arch/powerpc/include/asm/reg_8xx.h
index 53a7e2955d3e..7192eece6c3e 100644
--- a/arch/powerpc/include/asm/reg_8xx.h
+++ b/arch/powerpc/include/asm/reg_8xx.h
@@ -66,86 +66,4 @@
 #define DC_DFWT		0x40000000	/* Data cache is forced write through */
 #define DC_LES		0x20000000	/* Caches are little endian mode */
 
-#ifdef CONFIG_8xx_CPU6
-#define do_mtspr_cpu6(rn, rn_addr, v)	\
-	do {								\
-		int _reg_cpu6 = rn_addr, _tmp_cpu6;		\
-		asm volatile("stw %0, %1;"				\
-			     "lwz %0, %1;"				\
-			     "mtspr " __stringify(rn) ",%2" :		\
-			     : "r" (_reg_cpu6), "m"(_tmp_cpu6),		\
-			       "r" ((unsigned long)(v))			\
-			     : "memory");				\
-	} while (0)
-
-#define do_mtspr(rn, v)	asm volatile("mtspr " __stringify(rn) ",%0" :	\
-				     : "r" ((unsigned long)(v))		\
-				     : "memory")
-#define mtspr(rn, v) \
-	do {								\
-		if (rn == SPRN_IMMR)					\
-			do_mtspr_cpu6(rn, 0x3d30, v);			\
-		else if (rn == SPRN_IC_CST)				\
-			do_mtspr_cpu6(rn, 0x2110, v);			\
-		else if (rn == SPRN_IC_ADR)				\
-			do_mtspr_cpu6(rn, 0x2310, v);			\
-		else if (rn == SPRN_IC_DAT)				\
-			do_mtspr_cpu6(rn, 0x2510, v);			\
-		else if (rn == SPRN_DC_CST)				\
-			do_mtspr_cpu6(rn, 0x3110, v);			\
-		else if (rn == SPRN_DC_ADR)				\
-			do_mtspr_cpu6(rn, 0x3310, v);			\
-		else if (rn == SPRN_DC_DAT)				\
-			do_mtspr_cpu6(rn, 0x3510, v);			\
-		else if (rn == SPRN_MI_CTR)				\
-			do_mtspr_cpu6(rn, 0x2180, v);			\
-		else if (rn == SPRN_MI_AP)				\
-			do_mtspr_cpu6(rn, 0x2580, v);			\
-		else if (rn == SPRN_MI_EPN)				\
-			do_mtspr_cpu6(rn, 0x2780, v);			\
-		else if (rn == SPRN_MI_TWC)				\
-			do_mtspr_cpu6(rn, 0x2b80, v);			\
-		else if (rn == SPRN_MI_RPN)				\
-			do_mtspr_cpu6(rn, 0x2d80, v);			\
-		else if (rn == SPRN_MI_CAM)				\
-			do_mtspr_cpu6(rn, 0x2190, v);			\
-		else if (rn == SPRN_MI_RAM0)				\
-			do_mtspr_cpu6(rn, 0x2390, v);			\
-		else if (rn == SPRN_MI_RAM1)				\
-			do_mtspr_cpu6(rn, 0x2590, v);			\
-		else if (rn == SPRN_MD_CTR)				\
-			do_mtspr_cpu6(rn, 0x3180, v);			\
-		else if (rn == SPRN_M_CASID)				\
-			do_mtspr_cpu6(rn, 0x3380, v);			\
-		else if (rn == SPRN_MD_AP)				\
-			do_mtspr_cpu6(rn, 0x3580, v);			\
-		else if (rn == SPRN_MD_EPN)				\
-			do_mtspr_cpu6(rn, 0x3780, v);			\
-		else if (rn == SPRN_M_TWB)				\
-			do_mtspr_cpu6(rn, 0x3980, v);			\
-		else if (rn == SPRN_MD_TWC)				\
-			do_mtspr_cpu6(rn, 0x3b80, v);			\
-		else if (rn == SPRN_MD_RPN)				\
-			do_mtspr_cpu6(rn, 0x3d80, v);			\
-		else if (rn == SPRN_M_TW)				\
-			do_mtspr_cpu6(rn, 0x3f80, v);			\
-		else if (rn == SPRN_MD_CAM)				\
-			do_mtspr_cpu6(rn, 0x3190, v);			\
-		else if (rn == SPRN_MD_RAM0)				\
-			do_mtspr_cpu6(rn, 0x3390, v);			\
-		else if (rn == SPRN_MD_RAM1)				\
-			do_mtspr_cpu6(rn, 0x3590, v);			\
-		else if (rn == SPRN_DEC)				\
-			do_mtspr_cpu6(rn, 0x2c00, v);			\
-		else if (rn == SPRN_TBWL)				\
-			do_mtspr_cpu6(rn, 0x3880, v);			\
-		else if (rn == SPRN_TBWU)				\
-			do_mtspr_cpu6(rn, 0x3a80, v);			\
-		else if (rn == SPRN_DPDR)				\
-			do_mtspr_cpu6(rn, 0x2d30, v);			\
-		else							\
-			do_mtspr(rn, v);				\
-	} while (0)
-#endif
-
 #endif /* _ASM_POWERPC_REG_8xx_H */
diff --git a/arch/powerpc/include/asm/sections.h b/arch/powerpc/include/asm/sections.h
index 82bec63bbd4f..e335a8f846af 100644
--- a/arch/powerpc/include/asm/sections.h
+++ b/arch/powerpc/include/asm/sections.h
@@ -66,6 +66,9 @@ static inline int overlaps_kvm_tmp(unsigned long start, unsigned long end)
 }
 
 #ifdef PPC64_ELF_ABI_v1
+
+#define HAVE_DEREFERENCE_FUNCTION_DESCRIPTOR 1
+
 #undef dereference_function_descriptor
 static inline void *dereference_function_descriptor(void *ptr)
 {
@@ -76,6 +79,15 @@ static inline void *dereference_function_descriptor(void *ptr)
 		ptr = p;
 	return ptr;
 }
+
+#undef dereference_kernel_function_descriptor
+static inline void *dereference_kernel_function_descriptor(void *ptr)
+{
+	if (ptr < (void *)__start_opd || ptr >= (void *)__end_opd)
+		return ptr;
+
+	return dereference_function_descriptor(ptr);
+}
 #endif /* PPC64_ELF_ABI_v1 */
 
 #endif
diff --git a/arch/powerpc/include/asm/swiotlb.h b/arch/powerpc/include/asm/swiotlb.h
index 01d45a5fd00b..f65ecf57b66c 100644
--- a/arch/powerpc/include/asm/swiotlb.h
+++ b/arch/powerpc/include/asm/swiotlb.h
@@ -13,9 +13,7 @@
 
 #include <linux/swiotlb.h>
 
-extern const struct dma_map_ops swiotlb_dma_ops;
-
-static inline void dma_mark_clean(void *addr, size_t size) {}
+extern const struct dma_map_ops powerpc_swiotlb_dma_ops;
 
 extern unsigned int ppc_swiotlb_enable;
 int __init swiotlb_setup_bus_notifier(void);
diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h
index 449912f057f6..d61f9c96d916 100644
--- a/arch/powerpc/include/asm/systbl.h
+++ b/arch/powerpc/include/asm/systbl.h
@@ -389,3 +389,6 @@ COMPAT_SYS_SPU(preadv2)
 COMPAT_SYS_SPU(pwritev2)
 SYSCALL(kexec_file_load)
 SYSCALL(statx)
+SYSCALL(pkey_alloc)
+SYSCALL(pkey_free)
+SYSCALL(pkey_mprotect)
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index a264c3ad366b..4a12c00f8de3 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -58,9 +58,6 @@ struct thread_info {
 	.flags =	0,			\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 #define THREAD_SIZE_ORDER	(THREAD_SHIFT - PAGE_SHIFT)
 
 /* how to get the thread information struct from C */
diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index 88187c285c70..9f421641a35c 100644
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -44,6 +44,11 @@ extern int sysfs_add_device_to_node(struct device *dev, int nid);
 extern void sysfs_remove_device_from_node(struct device *dev, int nid);
 extern int numa_update_cpu_topology(bool cpus_locked);
 
+static inline void update_numa_cpu_lookup_table(unsigned int cpu, int node)
+{
+	numa_cpu_lookup_table[cpu] = node;
+}
+
 static inline int early_cpu_to_node(int cpu)
 {
 	int nid;
@@ -76,12 +81,16 @@ static inline int numa_update_cpu_topology(bool cpus_locked)
 {
 	return 0;
 }
+
+static inline void update_numa_cpu_lookup_table(unsigned int cpu, int node) {}
+
 #endif /* CONFIG_NUMA */
 
 #if defined(CONFIG_NUMA) && defined(CONFIG_PPC_SPLPAR)
 extern int start_topology_update(void);
 extern int stop_topology_update(void);
 extern int prrn_is_enabled(void);
+extern int find_and_online_cpu_nid(int cpu);
 #else
 static inline int start_topology_update(void)
 {
@@ -95,6 +104,10 @@ static inline int prrn_is_enabled(void)
 {
 	return 0;
 }
+static inline int find_and_online_cpu_nid(int cpu)
+{
+	return 0;
+}
 #endif /* CONFIG_NUMA && CONFIG_PPC_SPLPAR */
 
 #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_NEED_MULTIPLE_NODES)
diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h
index 9ba11dbcaca9..daf1ba97a00c 100644
--- a/arch/powerpc/include/asm/unistd.h
+++ b/arch/powerpc/include/asm/unistd.h
@@ -12,14 +12,10 @@
 #include <uapi/asm/unistd.h>
 
 
-#define NR_syscalls		384
+#define NR_syscalls		387
 
 #define __NR__exit __NR_exit
 
-#define __IGNORE_pkey_mprotect
-#define __IGNORE_pkey_alloc
-#define __IGNORE_pkey_free
-
 #ifndef __ASSEMBLY__
 
 #include <linux/types.h>
diff --git a/arch/powerpc/include/asm/xive-regs.h b/arch/powerpc/include/asm/xive-regs.h
index 1d3f2be5ae39..fa4288822b68 100644
--- a/arch/powerpc/include/asm/xive-regs.h
+++ b/arch/powerpc/include/asm/xive-regs.h
@@ -9,6 +9,41 @@
 #ifndef _ASM_POWERPC_XIVE_REGS_H
 #define _ASM_POWERPC_XIVE_REGS_H
 
+/*
+ * "magic" Event State Buffer (ESB) MMIO offsets.
+ *
+ * Each interrupt source has a 2-bit state machine called ESB
+ * which can be controlled by MMIO. It's made of 2 bits, P and
+ * Q. P indicates that an interrupt is pending (has been sent
+ * to a queue and is waiting for an EOI). Q indicates that the
+ * interrupt has been triggered while pending.
+ *
+ * This acts as a coalescing mechanism in order to guarantee
+ * that a given interrupt only occurs at most once in a queue.
+ *
+ * When doing an EOI, the Q bit will indicate if the interrupt
+ * needs to be re-triggered.
+ *
+ * The following offsets into the ESB MMIO allow to read or
+ * manipulate the PQ bits. They must be used with an 8-bytes
+ * load instruction. They all return the previous state of the
+ * interrupt (atomically).
+ *
+ * Additionally, some ESB pages support doing an EOI via a
+ * store at 0 and some ESBs support doing a trigger via a
+ * separate trigger page.
+ */
+#define XIVE_ESB_STORE_EOI	0x400 /* Store */
+#define XIVE_ESB_LOAD_EOI	0x000 /* Load */
+#define XIVE_ESB_GET		0x800 /* Load */
+#define XIVE_ESB_SET_PQ_00	0xc00 /* Load */
+#define XIVE_ESB_SET_PQ_01	0xd00 /* Load */
+#define XIVE_ESB_SET_PQ_10	0xe00 /* Load */
+#define XIVE_ESB_SET_PQ_11	0xf00 /* Load */
+
+#define XIVE_ESB_VAL_P		0x2
+#define XIVE_ESB_VAL_Q		0x1
+
 /*
  * Thread Management (aka "TM") registers
  */
diff --git a/arch/powerpc/include/asm/xive.h b/arch/powerpc/include/asm/xive.h
index 371fbebf1ec9..8d1a2792484f 100644
--- a/arch/powerpc/include/asm/xive.h
+++ b/arch/powerpc/include/asm/xive.h
@@ -58,6 +58,9 @@ struct xive_irq_data {
 #define XIVE_IRQ_FLAG_EOI_FW	0x10
 #define XIVE_IRQ_FLAG_H_INT_ESB	0x20
 
+/* Special flag set by KVM for excalation interrupts */
+#define XIVE_IRQ_NO_EOI		0x80
+
 #define XIVE_INVALID_CHIP_ID	-1
 
 /* A queue tracking structure in a CPU */
@@ -72,41 +75,6 @@ struct xive_q {
 	atomic_t		pending_count;
 };
 
-/*
- * "magic" Event State Buffer (ESB) MMIO offsets.
- *
- * Each interrupt source has a 2-bit state machine called ESB
- * which can be controlled by MMIO. It's made of 2 bits, P and
- * Q. P indicates that an interrupt is pending (has been sent
- * to a queue and is waiting for an EOI). Q indicates that the
- * interrupt has been triggered while pending.
- *
- * This acts as a coalescing mechanism in order to guarantee
- * that a given interrupt only occurs at most once in a queue.
- *
- * When doing an EOI, the Q bit will indicate if the interrupt
- * needs to be re-triggered.
- *
- * The following offsets into the ESB MMIO allow to read or
- * manipulate the PQ bits. They must be used with an 8-bytes
- * load instruction. They all return the previous state of the
- * interrupt (atomically).
- *
- * Additionally, some ESB pages support doing an EOI via a
- * store at 0 and some ESBs support doing a trigger via a
- * separate trigger page.
- */
-#define XIVE_ESB_STORE_EOI	0x400 /* Store */
-#define XIVE_ESB_LOAD_EOI	0x000 /* Load */
-#define XIVE_ESB_GET		0x800 /* Load */
-#define XIVE_ESB_SET_PQ_00	0xc00 /* Load */
-#define XIVE_ESB_SET_PQ_01	0xd00 /* Load */
-#define XIVE_ESB_SET_PQ_10	0xe00 /* Load */
-#define XIVE_ESB_SET_PQ_11	0xf00 /* Load */
-
-#define XIVE_ESB_VAL_P		0x2
-#define XIVE_ESB_VAL_Q		0x1
-
 /* Global enable flags for the XIVE support */
 extern bool __xive_enabled;
 
@@ -143,9 +111,10 @@ extern void xive_native_disable_queue(u32 vp_id, struct xive_q *q, u8 prio);
 
 extern void xive_native_sync_source(u32 hw_irq);
 extern bool is_xive_irq(struct irq_chip *chip);
-extern int xive_native_enable_vp(u32 vp_id);
+extern int xive_native_enable_vp(u32 vp_id, bool single_escalation);
 extern int xive_native_disable_vp(u32 vp_id);
 extern int xive_native_get_vp_info(u32 vp_id, u32 *out_cam_id, u32 *out_chip_id);
+extern bool xive_native_has_single_escalation(void);
 
 #else
 
@@ -154,7 +123,7 @@ static inline bool xive_enabled(void) { return false; }
 static inline bool xive_spapr_init(void) { return false; }
 static inline bool xive_native_init(void) { return false; }
 static inline void xive_smp_probe(void) { }
-extern inline int  xive_smp_prepare_cpu(unsigned int cpu) { return -EINVAL; }
+static inline int  xive_smp_prepare_cpu(unsigned int cpu) { return -EINVAL; }
 static inline void xive_smp_setup_cpu(void) { }
 static inline void xive_smp_disable_cpu(void) { }
 static inline void xive_kexec_teardown_cpu(int secondary) { }
diff --git a/arch/powerpc/include/uapi/asm/elf.h b/arch/powerpc/include/uapi/asm/elf.h
index 5f201d40bcca..860c59291bfc 100644
--- a/arch/powerpc/include/uapi/asm/elf.h
+++ b/arch/powerpc/include/uapi/asm/elf.h
@@ -97,6 +97,7 @@
 #define ELF_NTMSPRREG	3	/* include tfhar, tfiar, texasr */
 #define ELF_NEBB	3	/* includes ebbrr, ebbhr, bescr */
 #define ELF_NPMU	5	/* includes siar, sdar, sier, mmcr2, mmcr0 */
+#define ELF_NPKEY	3	/* includes amr, iamr, uamor */
 
 typedef unsigned long elf_greg_t64;
 typedef elf_greg_t64 elf_gregset_t64[ELF_NGREG];
diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
index 637b7263cb86..833ed9a16adf 100644
--- a/arch/powerpc/include/uapi/asm/kvm.h
+++ b/arch/powerpc/include/uapi/asm/kvm.h
@@ -632,6 +632,8 @@ struct kvm_ppc_cpu_char {
 #define KVM_REG_PPC_TIDR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbc)
 #define KVM_REG_PPC_PSSCR	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbd)
 
+#define KVM_REG_PPC_DEC_EXPIRY	(KVM_REG_PPC | KVM_REG_SIZE_U64 | 0xbe)
+
 /* Transactional Memory checkpointed state:
  * This is all GPRs, all VSX regs and a subset of SPRs
  */
diff --git a/arch/powerpc/include/uapi/asm/mman.h b/arch/powerpc/include/uapi/asm/mman.h
index e63bc37e33af..65065ce32814 100644
--- a/arch/powerpc/include/uapi/asm/mman.h
+++ b/arch/powerpc/include/uapi/asm/mman.h
@@ -30,4 +30,10 @@
 #define MAP_STACK	0x20000		/* give out an address that is best suited for process/thread stacks */
 #define MAP_HUGETLB	0x40000		/* create a huge page mapping */
 
+/* Override any generic PKEY permission defines */
+#define PKEY_DISABLE_EXECUTE   0x4
+#undef PKEY_ACCESS_MASK
+#define PKEY_ACCESS_MASK       (PKEY_DISABLE_ACCESS |\
+				PKEY_DISABLE_WRITE  |\
+				PKEY_DISABLE_EXECUTE)
 #endif /* _UAPI_ASM_POWERPC_MMAN_H */
diff --git a/arch/powerpc/include/uapi/asm/siginfo.h b/arch/powerpc/include/uapi/asm/siginfo.h
index 1a691141e49f..9f142451a01f 100644
--- a/arch/powerpc/include/uapi/asm/siginfo.h
+++ b/arch/powerpc/include/uapi/asm/siginfo.h
@@ -15,7 +15,19 @@
 
 #include <asm-generic/siginfo.h>
 
-#undef NSIGTRAP
-#define NSIGTRAP	4
+/*
+ * SIGFPE si_codes
+ */
+#ifdef __KERNEL__
+#define FPE_FIXME	0	/* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
+/*
+ * SIGTRAP si_codes
+ */
+#ifdef __KERNEL__
+#define TRAP_FIXME	0	/* Broken dup of SI_USER */
+#endif /* __KERNEL__ */
+
 
 #endif	/* _ASM_POWERPC_SIGINFO_H */
diff --git a/arch/powerpc/include/uapi/asm/unistd.h b/arch/powerpc/include/uapi/asm/unistd.h
index df8684f31919..389c36fd8299 100644
--- a/arch/powerpc/include/uapi/asm/unistd.h
+++ b/arch/powerpc/include/uapi/asm/unistd.h
@@ -395,5 +395,8 @@
 #define __NR_pwritev2		381
 #define __NR_kexec_file_load	382
 #define __NR_statx		383
+#define __NR_pkey_alloc		384
+#define __NR_pkey_free		385
+#define __NR_pkey_mprotect	386
 
 #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index f390d57cf2e1..ea5eb91b836e 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -178,7 +178,7 @@ int main(void)
 	OFFSET(PACATOC, paca_struct, kernel_toc);
 	OFFSET(PACAKBASE, paca_struct, kernelbase);
 	OFFSET(PACAKMSR, paca_struct, kernel_msr);
-	OFFSET(PACASOFTIRQEN, paca_struct, soft_enabled);
+	OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask);
 	OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
 #ifdef CONFIG_PPC_BOOK3S
 	OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id);
@@ -239,8 +239,7 @@ int main(void)
 	OFFSET(PACA_IN_NMI, paca_struct, in_nmi);
 	OFFSET(PACA_RFI_FLUSH_FALLBACK_AREA, paca_struct, rfi_flush_fallback_area);
 	OFFSET(PACA_EXRFI, paca_struct, exrfi);
-	OFFSET(PACA_L1D_FLUSH_CONGRUENCE, paca_struct, l1d_flush_congruence);
-	OFFSET(PACA_L1D_FLUSH_SETS, paca_struct, l1d_flush_sets);
+	OFFSET(PACA_L1D_FLUSH_SIZE, paca_struct, l1d_flush_size);
 
 #endif
 	OFFSET(PACAHWCPUID, paca_struct, hw_cpu_id);
@@ -401,6 +400,8 @@ int main(void)
 	/* Other bits used by the vdso */
 	DEFINE(CLOCK_REALTIME, CLOCK_REALTIME);
 	DEFINE(CLOCK_MONOTONIC, CLOCK_MONOTONIC);
+	DEFINE(CLOCK_REALTIME_COARSE, CLOCK_REALTIME_COARSE);
+	DEFINE(CLOCK_MONOTONIC_COARSE, CLOCK_MONOTONIC_COARSE);
 	DEFINE(NSEC_PER_SEC, NSEC_PER_SEC);
 	DEFINE(CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC);
 
@@ -519,6 +520,7 @@ int main(void)
 	OFFSET(VCPU_PENDING_EXC, kvm_vcpu, arch.pending_exceptions);
 	OFFSET(VCPU_CEDED, kvm_vcpu, arch.ceded);
 	OFFSET(VCPU_PRODDED, kvm_vcpu, arch.prodded);
+	OFFSET(VCPU_IRQ_PENDING, kvm_vcpu, arch.irq_pending);
 	OFFSET(VCPU_DBELL_REQ, kvm_vcpu, arch.doorbell_request);
 	OFFSET(VCPU_MMCR, kvm_vcpu, arch.mmcr);
 	OFFSET(VCPU_PMC, kvm_vcpu, arch.pmc);
@@ -738,6 +740,9 @@ int main(void)
 	DEFINE(VCPU_XIVE_CAM_WORD, offsetof(struct kvm_vcpu,
 					    arch.xive_cam_word));
 	DEFINE(VCPU_XIVE_PUSHED, offsetof(struct kvm_vcpu, arch.xive_pushed));
+	DEFINE(VCPU_XIVE_ESC_ON, offsetof(struct kvm_vcpu, arch.xive_esc_on));
+	DEFINE(VCPU_XIVE_ESC_RADDR, offsetof(struct kvm_vcpu, arch.xive_esc_raddr));
+	DEFINE(VCPU_XIVE_ESC_VADDR, offsetof(struct kvm_vcpu, arch.xive_esc_vaddr));
 #endif
 
 #ifdef CONFIG_KVM_EXIT_TIMING
diff --git a/arch/powerpc/kernel/cpu_setup_power.S b/arch/powerpc/kernel/cpu_setup_power.S
index 679bbe714e85..3f30c994e931 100644
--- a/arch/powerpc/kernel/cpu_setup_power.S
+++ b/arch/powerpc/kernel/cpu_setup_power.S
@@ -31,7 +31,6 @@ _GLOBAL(__setup_cpu_power7)
 	mfspr	r3,SPRN_LPCR
 	li	r4,(LPCR_LPES1 >> LPCR_LPES_SH)
 	bl	__init_LPCR_ISA206
-	bl	__init_tlb_power7
 	mtlr	r11
 	blr
 
@@ -45,7 +44,6 @@ _GLOBAL(__restore_cpu_power7)
 	mfspr	r3,SPRN_LPCR
 	li	r4,(LPCR_LPES1 >> LPCR_LPES_SH)
 	bl	__init_LPCR_ISA206
-	bl	__init_tlb_power7
 	mtlr	r11
 	blr
 
@@ -64,7 +62,6 @@ _GLOBAL(__setup_cpu_power8)
 	li	r4,0 /* LPES = 0 */
 	bl	__init_LPCR_ISA206
 	bl	__init_HFSCR
-	bl	__init_tlb_power8
 	bl	__init_PMU_HV
 	bl	__init_PMU_HV_ISA207
 	mtlr	r11
@@ -86,7 +83,6 @@ _GLOBAL(__restore_cpu_power8)
 	li	r4,0 /* LPES = 0 */
 	bl	__init_LPCR_ISA206
 	bl	__init_HFSCR
-	bl	__init_tlb_power8
 	bl	__init_PMU_HV
 	bl	__init_PMU_HV_ISA207
 	mtlr	r11
@@ -111,7 +107,6 @@ _GLOBAL(__setup_cpu_power9)
 	li	r4,0 /* LPES = 0 */
 	bl	__init_LPCR_ISA300
 	bl	__init_HFSCR
-	bl	__init_tlb_power9
 	bl	__init_PMU_HV
 	mtlr	r11
 	blr
@@ -136,7 +131,6 @@ _GLOBAL(__restore_cpu_power9)
 	li	r4,0 /* LPES = 0 */
 	bl	__init_LPCR_ISA300
 	bl	__init_HFSCR
-	bl	__init_tlb_power9
 	bl	__init_PMU_HV
 	mtlr	r11
 	blr
@@ -194,50 +188,6 @@ __init_HFSCR:
 	mtspr	SPRN_HFSCR,r3
 	blr
 
-/*
- * Clear the TLB using the specified IS form of tlbiel instruction
- * (invalidate by congruence class). P7 has 128 CCs., P8 has 512.
- */
-__init_tlb_power7:
-	li	r6,POWER7_TLB_SETS
-	mtctr	r6
-	li	r7,0xc00	/* IS field = 0b11 */
-	ptesync
-2:	tlbiel	r7
-	addi	r7,r7,0x1000
-	bdnz	2b
-	ptesync
-1:	blr
-
-__init_tlb_power8:
-	li	r6,POWER8_TLB_SETS
-	mtctr	r6
-	li	r7,0xc00	/* IS field = 0b11 */
-	ptesync
-2:	tlbiel	r7
-	addi	r7,r7,0x1000
-	bdnz	2b
-	ptesync
-1:	blr
-
-/*
- * Flush the TLB in hash mode. Hash must flush with RIC=2 once for process
- * and one for partition scope to clear process and partition table entries.
- */
-__init_tlb_power9:
-	li	r6,POWER9_TLB_SETS_HASH - 1
-	mtctr	r6
-	li	r7,0xc00	/* IS field = 0b11 */
-	li	r8,0
-	ptesync
-	PPC_TLBIEL(7, 8, 2, 1, 0)
-	PPC_TLBIEL(7, 8, 2, 0, 0)
-2:	addi	r7,r7,0x1000
-	PPC_TLBIEL(7, 8, 0, 0, 0)
-	bdnz	2b
-	ptesync
-1:	blr
-
 __init_PMU_HV:
 	li	r5,0
 	mtspr	SPRN_MMCRC,r5
diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
index 1350f49d81a8..c40a9fc1e5d1 100644
--- a/arch/powerpc/kernel/cputable.c
+++ b/arch/powerpc/kernel/cputable.c
@@ -74,9 +74,6 @@ extern void __setup_cpu_power8(unsigned long offset, struct cpu_spec* spec);
 extern void __restore_cpu_power8(void);
 extern void __setup_cpu_power9(unsigned long offset, struct cpu_spec* spec);
 extern void __restore_cpu_power9(void);
-extern void __flush_tlb_power7(unsigned int action);
-extern void __flush_tlb_power8(unsigned int action);
-extern void __flush_tlb_power9(unsigned int action);
 extern long __machine_check_early_realmode_p7(struct pt_regs *regs);
 extern long __machine_check_early_realmode_p8(struct pt_regs *regs);
 extern long __machine_check_early_realmode_p9(struct pt_regs *regs);
@@ -368,7 +365,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_cpu_type	= "ppc64/ibm-compat-v1",
 		.cpu_setup		= __setup_cpu_power7,
 		.cpu_restore		= __restore_cpu_power7,
-		.flush_tlb		= __flush_tlb_power7,
 		.machine_check_early	= __machine_check_early_realmode_p7,
 		.platform		= "power7",
 	},
@@ -386,7 +382,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_cpu_type	= "ppc64/ibm-compat-v1",
 		.cpu_setup		= __setup_cpu_power8,
 		.cpu_restore		= __restore_cpu_power8,
-		.flush_tlb		= __flush_tlb_power8,
 		.machine_check_early	= __machine_check_early_realmode_p8,
 		.platform		= "power8",
 	},
@@ -404,7 +399,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_cpu_type	= "ppc64/ibm-compat-v1",
 		.cpu_setup		= __setup_cpu_power9,
 		.cpu_restore		= __restore_cpu_power9,
-		.flush_tlb		= __flush_tlb_power9,
 		.platform		= "power9",
 	},
 	{	/* Power7 */
@@ -423,7 +417,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_POWER4,
 		.cpu_setup		= __setup_cpu_power7,
 		.cpu_restore		= __restore_cpu_power7,
-		.flush_tlb		= __flush_tlb_power7,
 		.machine_check_early	= __machine_check_early_realmode_p7,
 		.platform		= "power7",
 	},
@@ -443,7 +436,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_POWER4,
 		.cpu_setup		= __setup_cpu_power7,
 		.cpu_restore		= __restore_cpu_power7,
-		.flush_tlb		= __flush_tlb_power7,
 		.machine_check_early	= __machine_check_early_realmode_p7,
 		.platform		= "power7+",
 	},
@@ -463,7 +455,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_INVALID,
 		.cpu_setup		= __setup_cpu_power8,
 		.cpu_restore		= __restore_cpu_power8,
-		.flush_tlb		= __flush_tlb_power8,
 		.machine_check_early	= __machine_check_early_realmode_p8,
 		.platform		= "power8",
 	},
@@ -483,7 +474,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_INVALID,
 		.cpu_setup		= __setup_cpu_power8,
 		.cpu_restore		= __restore_cpu_power8,
-		.flush_tlb		= __flush_tlb_power8,
 		.machine_check_early	= __machine_check_early_realmode_p8,
 		.platform		= "power8",
 	},
@@ -503,7 +493,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_INVALID,
 		.cpu_setup		= __setup_cpu_power8,
 		.cpu_restore		= __restore_cpu_power8,
-		.flush_tlb		= __flush_tlb_power8,
 		.machine_check_early	= __machine_check_early_realmode_p8,
 		.platform		= "power8",
 	},
@@ -523,7 +512,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_INVALID,
 		.cpu_setup		= __setup_cpu_power8,
 		.cpu_restore		= __restore_cpu_power8,
-		.flush_tlb		= __flush_tlb_power8,
 		.machine_check_early	= __machine_check_early_realmode_p8,
 		.platform		= "power8",
 	},
@@ -543,7 +531,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_INVALID,
 		.cpu_setup		= __setup_cpu_power9,
 		.cpu_restore		= __restore_cpu_power9,
-		.flush_tlb		= __flush_tlb_power9,
 		.machine_check_early	= __machine_check_early_realmode_p9,
 		.platform		= "power9",
 	},
@@ -563,7 +550,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_INVALID,
 		.cpu_setup		= __setup_cpu_power9,
 		.cpu_restore		= __restore_cpu_power9,
-		.flush_tlb		= __flush_tlb_power9,
 		.machine_check_early	= __machine_check_early_realmode_p9,
 		.platform		= "power9",
 	},
@@ -583,7 +569,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
 		.oprofile_type		= PPC_OPROFILE_INVALID,
 		.cpu_setup		= __setup_cpu_power9,
 		.cpu_restore		= __restore_cpu_power9,
-		.flush_tlb		= __flush_tlb_power9,
 		.machine_check_early	= __machine_check_early_realmode_p9,
 		.platform		= "power9",
 	},
diff --git a/arch/powerpc/kernel/crash.c b/arch/powerpc/kernel/crash.c
index cbabb5adccd9..00b215125d3e 100644
--- a/arch/powerpc/kernel/crash.c
+++ b/arch/powerpc/kernel/crash.c
@@ -44,6 +44,14 @@
 #define REAL_MODE_TIMEOUT	10000
 
 static int time_to_dump;
+/*
+ * crash_wake_offline should be set to 1 by platforms that intend to wake
+ * up offline cpus prior to jumping to a kdump kernel. Currently powernv
+ * sets it to 1, since we want to avoid things from happening when an
+ * offline CPU wakes up due to something like an HMI (malfunction error),
+ * which propagates to all threads.
+ */
+int crash_wake_offline;
 
 #define CRASH_HANDLER_MAX 3
 /* List of shutdown handles */
@@ -63,15 +71,12 @@ static int handle_fault(struct pt_regs *regs)
 #ifdef CONFIG_SMP
 
 static atomic_t cpus_in_crash;
-static void crash_ipi_callback(struct pt_regs *regs)
+void crash_ipi_callback(struct pt_regs *regs)
 {
 	static cpumask_t cpus_state_saved = CPU_MASK_NONE;
 
 	int cpu = smp_processor_id();
 
-	if (!cpu_online(cpu))
-		return;
-
 	hard_irq_disable();
 	if (!cpumask_test_cpu(cpu, &cpus_state_saved)) {
 		crash_save_cpu(regs, cpu);
@@ -109,6 +114,9 @@ static void crash_kexec_prepare_cpus(int cpu)
 
 	printk(KERN_EMERG "Sending IPI to other CPUs\n");
 
+	if (crash_wake_offline)
+		ncpus = num_present_cpus() - 1;
+
 	crash_send_ipi(crash_ipi_callback);
 	smp_wmb();
 
diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c
index 66f33e7f8d40..f9fe2080ceb9 100644
--- a/arch/powerpc/kernel/dma-iommu.c
+++ b/arch/powerpc/kernel/dma-iommu.c
@@ -114,7 +114,7 @@ int dma_iommu_mapping_error(struct device *dev, dma_addr_t dma_addr)
 struct dma_map_ops dma_iommu_ops = {
 	.alloc			= dma_iommu_alloc_coherent,
 	.free			= dma_iommu_free_coherent,
-	.mmap			= dma_direct_mmap_coherent,
+	.mmap			= dma_nommu_mmap_coherent,
 	.map_sg			= dma_iommu_map_sg,
 	.unmap_sg		= dma_iommu_unmap_sg,
 	.dma_supported		= dma_iommu_dma_supported,
diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c
index d0ea7860e02b..88f3963ca30f 100644
--- a/arch/powerpc/kernel/dma-swiotlb.c
+++ b/arch/powerpc/kernel/dma-swiotlb.c
@@ -46,10 +46,10 @@ static u64 swiotlb_powerpc_get_required(struct device *dev)
  * map_page, and unmap_page on highmem, use normal dma_ops
  * for everything else.
  */
-const struct dma_map_ops swiotlb_dma_ops = {
-	.alloc = __dma_direct_alloc_coherent,
-	.free = __dma_direct_free_coherent,
-	.mmap = dma_direct_mmap_coherent,
+const struct dma_map_ops powerpc_swiotlb_dma_ops = {
+	.alloc = __dma_nommu_alloc_coherent,
+	.free = __dma_nommu_free_coherent,
+	.mmap = dma_nommu_mmap_coherent,
 	.map_sg = swiotlb_map_sg_attrs,
 	.unmap_sg = swiotlb_unmap_sg_attrs,
 	.dma_supported = swiotlb_dma_supported,
@@ -89,7 +89,7 @@ static int ppc_swiotlb_bus_notify(struct notifier_block *nb,
 
 	/* May need to bounce if the device can't address all of DRAM */
 	if ((dma_get_mask(dev) + 1) < memblock_end_of_DRAM())
-		set_dma_ops(dev, &swiotlb_dma_ops);
+		set_dma_ops(dev, &powerpc_swiotlb_dma_ops);
 
 	return NOTIFY_DONE;
 }
@@ -121,7 +121,7 @@ static int __init check_swiotlb_enabled(void)
 	if (ppc_swiotlb_enable)
 		swiotlb_print_info();
 	else
-		swiotlb_free();
+		swiotlb_exit();
 
 	return 0;
 }
diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c
index 4194bbbbdb10..da20569de9d4 100644
--- a/arch/powerpc/kernel/dma.c
+++ b/arch/powerpc/kernel/dma.c
@@ -33,14 +33,14 @@ static u64 __maybe_unused get_pfn_limit(struct device *dev)
 	struct dev_archdata __maybe_unused *sd = &dev->archdata;
 
 #ifdef CONFIG_SWIOTLB
-	if (sd->max_direct_dma_addr && dev->dma_ops == &swiotlb_dma_ops)
+	if (sd->max_direct_dma_addr && dev->dma_ops == &powerpc_swiotlb_dma_ops)
 		pfn = min_t(u64, pfn, sd->max_direct_dma_addr >> PAGE_SHIFT);
 #endif
 
 	return pfn;
 }
 
-static int dma_direct_dma_supported(struct device *dev, u64 mask)
+static int dma_nommu_dma_supported(struct device *dev, u64 mask)
 {
 #ifdef CONFIG_PPC64
 	u64 limit = get_dma_offset(dev) + (memblock_end_of_DRAM() - 1);
@@ -62,7 +62,7 @@ static int dma_direct_dma_supported(struct device *dev, u64 mask)
 #endif
 }
 
-void *__dma_direct_alloc_coherent(struct device *dev, size_t size,
+void *__dma_nommu_alloc_coherent(struct device *dev, size_t size,
 				  dma_addr_t *dma_handle, gfp_t flag,
 				  unsigned long attrs)
 {
@@ -105,9 +105,6 @@ void *__dma_direct_alloc_coherent(struct device *dev, size_t size,
 	};
 #endif /* CONFIG_FSL_SOC */
 
-	/* ignore region specifiers */
-	flag  &= ~(__GFP_HIGHMEM);
-
 	page = alloc_pages_node(node, flag, get_order(size));
 	if (page == NULL)
 		return NULL;
@@ -119,7 +116,7 @@ void *__dma_direct_alloc_coherent(struct device *dev, size_t size,
 #endif
 }
 
-void __dma_direct_free_coherent(struct device *dev, size_t size,
+void __dma_nommu_free_coherent(struct device *dev, size_t size,
 				void *vaddr, dma_addr_t dma_handle,
 				unsigned long attrs)
 {
@@ -130,7 +127,7 @@ void __dma_direct_free_coherent(struct device *dev, size_t size,
 #endif
 }
 
-static void *dma_direct_alloc_coherent(struct device *dev, size_t size,
+static void *dma_nommu_alloc_coherent(struct device *dev, size_t size,
 				       dma_addr_t *dma_handle, gfp_t flag,
 				       unsigned long attrs)
 {
@@ -139,8 +136,8 @@ static void *dma_direct_alloc_coherent(struct device *dev, size_t size,
 	/* The coherent mask may be smaller than the real mask, check if
 	 * we can really use the direct ops
 	 */
-	if (dma_direct_dma_supported(dev, dev->coherent_dma_mask))
-		return __dma_direct_alloc_coherent(dev, size, dma_handle,
+	if (dma_nommu_dma_supported(dev, dev->coherent_dma_mask))
+		return __dma_nommu_alloc_coherent(dev, size, dma_handle,
 						   flag, attrs);
 
 	/* Ok we can't ... do we have an iommu ? If not, fail */
@@ -154,15 +151,15 @@ static void *dma_direct_alloc_coherent(struct device *dev, size_t size,
 				    dev_to_node(dev));
 }
 
-static void dma_direct_free_coherent(struct device *dev, size_t size,
+static void dma_nommu_free_coherent(struct device *dev, size_t size,
 				     void *vaddr, dma_addr_t dma_handle,
 				     unsigned long attrs)
 {
 	struct iommu_table *iommu;
 
-	/* See comments in dma_direct_alloc_coherent() */
-	if (dma_direct_dma_supported(dev, dev->coherent_dma_mask))
-		return __dma_direct_free_coherent(dev, size, vaddr, dma_handle,
+	/* See comments in dma_nommu_alloc_coherent() */
+	if (dma_nommu_dma_supported(dev, dev->coherent_dma_mask))
+		return __dma_nommu_free_coherent(dev, size, vaddr, dma_handle,
 						  attrs);
 	/* Maybe we used an iommu ... */
 	iommu = get_iommu_table_base(dev);
@@ -175,7 +172,7 @@ static void dma_direct_free_coherent(struct device *dev, size_t size,
 	iommu_free_coherent(iommu, size, vaddr, dma_handle);
 }
 
-int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
+int dma_nommu_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
 			     void *cpu_addr, dma_addr_t handle, size_t size,
 			     unsigned long attrs)
 {
@@ -193,7 +190,7 @@ int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
 			       vma->vm_page_prot);
 }
 
-static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
+static int dma_nommu_map_sg(struct device *dev, struct scatterlist *sgl,
 			     int nents, enum dma_data_direction direction,
 			     unsigned long attrs)
 {
@@ -213,13 +210,13 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl,
 	return nents;
 }
 
-static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sg,
+static void dma_nommu_unmap_sg(struct device *dev, struct scatterlist *sg,
 				int nents, enum dma_data_direction direction,
 				unsigned long attrs)
 {
 }
 
-static u64 dma_direct_get_required_mask(struct device *dev)
+static u64 dma_nommu_get_required_mask(struct device *dev)
 {
 	u64 end, mask;
 
@@ -231,7 +228,7 @@ static u64 dma_direct_get_required_mask(struct device *dev)
 	return mask;
 }
 
-static inline dma_addr_t dma_direct_map_page(struct device *dev,
+static inline dma_addr_t dma_nommu_map_page(struct device *dev,
 					     struct page *page,
 					     unsigned long offset,
 					     size_t size,
@@ -246,7 +243,7 @@ static inline dma_addr_t dma_direct_map_page(struct device *dev,
 	return page_to_phys(page) + offset + get_dma_offset(dev);
 }
 
-static inline void dma_direct_unmap_page(struct device *dev,
+static inline void dma_nommu_unmap_page(struct device *dev,
 					 dma_addr_t dma_address,
 					 size_t size,
 					 enum dma_data_direction direction,
@@ -255,7 +252,7 @@ static inline void dma_direct_unmap_page(struct device *dev,
 }
 
 #ifdef CONFIG_NOT_COHERENT_CACHE
-static inline void dma_direct_sync_sg(struct device *dev,
+static inline void dma_nommu_sync_sg(struct device *dev,
 		struct scatterlist *sgl, int nents,
 		enum dma_data_direction direction)
 {
@@ -266,7 +263,7 @@ static inline void dma_direct_sync_sg(struct device *dev,
 		__dma_sync_page(sg_page(sg), sg->offset, sg->length, direction);
 }
 
-static inline void dma_direct_sync_single(struct device *dev,
+static inline void dma_nommu_sync_single(struct device *dev,
 					  dma_addr_t dma_handle, size_t size,
 					  enum dma_data_direction direction)
 {
@@ -274,24 +271,24 @@ static inline void dma_direct_sync_single(struct device *dev,
 }
 #endif
 
-const struct dma_map_ops dma_direct_ops = {
-	.alloc				= dma_direct_alloc_coherent,
-	.free				= dma_direct_free_coherent,
-	.mmap				= dma_direct_mmap_coherent,
-	.map_sg				= dma_direct_map_sg,
-	.unmap_sg			= dma_direct_unmap_sg,
-	.dma_supported			= dma_direct_dma_supported,
-	.map_page			= dma_direct_map_page,
-	.unmap_page			= dma_direct_unmap_page,
-	.get_required_mask		= dma_direct_get_required_mask,
+const struct dma_map_ops dma_nommu_ops = {
+	.alloc				= dma_nommu_alloc_coherent,
+	.free				= dma_nommu_free_coherent,
+	.mmap				= dma_nommu_mmap_coherent,
+	.map_sg				= dma_nommu_map_sg,
+	.unmap_sg			= dma_nommu_unmap_sg,
+	.dma_supported			= dma_nommu_dma_supported,
+	.map_page			= dma_nommu_map_page,
+	.unmap_page			= dma_nommu_unmap_page,
+	.get_required_mask		= dma_nommu_get_required_mask,
 #ifdef CONFIG_NOT_COHERENT_CACHE
-	.sync_single_for_cpu 		= dma_direct_sync_single,
-	.sync_single_for_device 	= dma_direct_sync_single,
-	.sync_sg_for_cpu 		= dma_direct_sync_sg,
-	.sync_sg_for_device 		= dma_direct_sync_sg,
+	.sync_single_for_cpu 		= dma_nommu_sync_single,
+	.sync_single_for_device 	= dma_nommu_sync_single,
+	.sync_sg_for_cpu 		= dma_nommu_sync_sg,
+	.sync_sg_for_device 		= dma_nommu_sync_sg,
 #endif
 };
-EXPORT_SYMBOL(dma_direct_ops);
+EXPORT_SYMBOL(dma_nommu_ops);
 
 int dma_set_coherent_mask(struct device *dev, u64 mask)
 {
@@ -302,7 +299,7 @@ int dma_set_coherent_mask(struct device *dev, u64 mask)
 		 * is no dma_op->set_coherent_mask() so we have to do
 		 * things the hard way:
 		 */
-		if (get_dma_ops(dev) != &dma_direct_ops ||
+		if (get_dma_ops(dev) != &dma_nommu_ops ||
 		    get_iommu_table_base(dev) == NULL ||
 		    !dma_iommu_dma_supported(dev, mask))
 			return -EIO;
diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c
index 8bdc2f96c5d6..8ca5d5b74618 100644
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -77,8 +77,6 @@ struct dt_cpu_feature {
  * Set up the base CPU
  */
 
-extern void __flush_tlb_power8(unsigned int action);
-extern void __flush_tlb_power9(unsigned int action);
 extern long __machine_check_early_realmode_p8(struct pt_regs *regs);
 extern long __machine_check_early_realmode_p9(struct pt_regs *regs);
 
@@ -92,27 +90,6 @@ static struct {
 
 static void (*init_pmu_registers)(void);
 
-static void cpufeatures_flush_tlb(void)
-{
-	/*
-	 * This is a temporary measure to keep equivalent TLB flush as the
-	 * cputable based setup code.
-	 */
-	switch (PVR_VER(mfspr(SPRN_PVR))) {
-	case PVR_POWER8:
-	case PVR_POWER8E:
-	case PVR_POWER8NVL:
-		__flush_tlb_power8(TLB_INVAL_SCOPE_GLOBAL);
-		break;
-	case PVR_POWER9:
-		__flush_tlb_power9(TLB_INVAL_SCOPE_GLOBAL);
-		break;
-	default:
-		pr_err("unknown CPU version for boot TLB flush\n");
-		break;
-	}
-}
-
 static void __restore_cpu_cpufeatures(void)
 {
 	/*
@@ -137,8 +114,6 @@ static void __restore_cpu_cpufeatures(void)
 
 	if (init_pmu_registers)
 		init_pmu_registers();
-
-	cpufeatures_flush_tlb();
 }
 
 static char dt_cpu_name[64];
@@ -157,7 +132,6 @@ static struct cpu_spec __initdata base_cpu_spec = {
 	.oprofile_type		= PPC_OPROFILE_INVALID,
 	.cpu_setup		= NULL,
 	.cpu_restore		= __restore_cpu_cpufeatures,
-	.flush_tlb		= NULL,
 	.machine_check_early	= NULL,
 	.platform		= NULL,
 };
@@ -412,7 +386,6 @@ static void init_pmu_power8(void)
 static int __init feat_enable_mce_power8(struct dt_cpu_feature *f)
 {
 	cur_cpu_spec->platform = "power8";
-	cur_cpu_spec->flush_tlb = __flush_tlb_power8;
 	cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8;
 
 	return 1;
@@ -451,7 +424,6 @@ static void init_pmu_power9(void)
 static int __init feat_enable_mce_power9(struct dt_cpu_feature *f)
 {
 	cur_cpu_spec->platform = "power9";
-	cur_cpu_spec->flush_tlb = __flush_tlb_power9;
 	cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9;
 
 	return 1;
@@ -737,6 +709,9 @@ static __init void cpufeatures_cpu_quirks(void)
 		cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1;
 	else if ((version & 0xffffefff) == 0x004e0201)
 		cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1;
+
+	if ((version & 0xffff0000) == 0x004e0000)
+		cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG;
 }
 
 static void __init cpufeatures_setup_finished(void)
@@ -748,12 +723,13 @@ static void __init cpufeatures_setup_finished(void)
 		cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE;
 	}
 
+	/* Make sure powerpc_base_platform is non-NULL */
+	powerpc_base_platform = cur_cpu_spec->platform;
+
 	system_registers.lpcr = mfspr(SPRN_LPCR);
 	system_registers.hfscr = mfspr(SPRN_HFSCR);
 	system_registers.fscr = mfspr(SPRN_FSCR);
 
-	cpufeatures_flush_tlb();
-
 	pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n",
 		cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features);
 }
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index cbca0a667682..2b9df0040d6b 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -740,6 +740,65 @@ static void *eeh_restore_dev_state(void *data, void *userdata)
 	return NULL;
 }
 
+int eeh_restore_vf_config(struct pci_dn *pdn)
+{
+	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
+	u32 devctl, cmd, cap2, aer_capctl;
+	int old_mps;
+
+	if (edev->pcie_cap) {
+		/* Restore MPS */
+		old_mps = (ffs(pdn->mps) - 8) << 5;
+		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+				     2, &devctl);
+		devctl &= ~PCI_EXP_DEVCTL_PAYLOAD;
+		devctl |= old_mps;
+		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+				      2, devctl);
+
+		/* Disable Completion Timeout if possible */
+		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP2,
+				     4, &cap2);
+		if (cap2 & PCI_EXP_DEVCAP2_COMP_TMOUT_DIS) {
+			eeh_ops->read_config(pdn,
+					     edev->pcie_cap + PCI_EXP_DEVCTL2,
+					     4, &cap2);
+			cap2 |= PCI_EXP_DEVCTL2_COMP_TMOUT_DIS;
+			eeh_ops->write_config(pdn,
+					      edev->pcie_cap + PCI_EXP_DEVCTL2,
+					      4, cap2);
+		}
+	}
+
+	/* Enable SERR and parity checking */
+	eeh_ops->read_config(pdn, PCI_COMMAND, 2, &cmd);
+	cmd |= (PCI_COMMAND_PARITY | PCI_COMMAND_SERR);
+	eeh_ops->write_config(pdn, PCI_COMMAND, 2, cmd);
+
+	/* Enable report various errors */
+	if (edev->pcie_cap) {
+		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+				     2, &devctl);
+		devctl &= ~PCI_EXP_DEVCTL_CERE;
+		devctl |= (PCI_EXP_DEVCTL_NFERE |
+			   PCI_EXP_DEVCTL_FERE |
+			   PCI_EXP_DEVCTL_URRE);
+		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
+				      2, devctl);
+	}
+
+	/* Enable ECRC generation and check */
+	if (edev->pcie_cap && edev->aer_cap) {
+		eeh_ops->read_config(pdn, edev->aer_cap + PCI_ERR_CAP,
+				     4, &aer_capctl);
+		aer_capctl |= (PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
+		eeh_ops->write_config(pdn, edev->aer_cap + PCI_ERR_CAP,
+				      4, aer_capctl);
+	}
+
+	return 0;
+}
+
 /**
  * pcibios_set_pcie_reset_state - Set PCI-E reset state
  * @dev: pci device struct
diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c
index 4f71e4c9beb7..0c0b66fc5bfb 100644
--- a/arch/powerpc/kernel/eeh_driver.c
+++ b/arch/powerpc/kernel/eeh_driver.c
@@ -228,6 +228,7 @@ static void *eeh_report_error(void *data, void *userdata)
 
 	edev->in_error = true;
 	eeh_pcid_put(dev);
+	pci_uevent_ers(dev, PCI_ERS_RESULT_NONE);
 	return NULL;
 }
 
@@ -381,6 +382,11 @@ static void *eeh_report_resume(void *data, void *userdata)
 	driver->err_handler->resume(dev);
 
 	eeh_pcid_put(dev);
+	pci_uevent_ers(dev, PCI_ERS_RESULT_RECOVERED);
+#ifdef CONFIG_PCI_IOV
+	if (eeh_ops->notify_resume && eeh_dev_to_pdn(edev))
+		eeh_ops->notify_resume(eeh_dev_to_pdn(edev));
+#endif
 	return NULL;
 }
 
@@ -416,6 +422,7 @@ static void *eeh_report_failure(void *data, void *userdata)
 	driver->err_handler->error_detected(dev, pci_channel_io_perm_failure);
 
 	eeh_pcid_put(dev);
+	pci_uevent_ers(dev, PCI_ERS_RESULT_DISCONNECT);
 	return NULL;
 }
 
@@ -440,7 +447,7 @@ static void *eeh_add_virt_device(void *data, void *userdata)
 			return NULL;
 	}
 
-#ifdef CONFIG_PPC_POWERNV
+#ifdef CONFIG_PCI_IOV
 	pci_iov_add_virtfn(edev->physfn, pdn->vf_index);
 #endif
 	return NULL;
@@ -496,7 +503,7 @@ static void *eeh_rmv_device(void *data, void *userdata)
 		(*removed)++;
 
 	if (edev->physfn) {
-#ifdef CONFIG_PPC_POWERNV
+#ifdef CONFIG_PCI_IOV
 		struct pci_dn *pdn = eeh_dev_to_pdn(edev);
 
 		pci_iov_remove_virtfn(edev->physfn, pdn->vf_index);
diff --git a/arch/powerpc/kernel/eeh_sysfs.c b/arch/powerpc/kernel/eeh_sysfs.c
index 797549289798..deed906dd8f1 100644
--- a/arch/powerpc/kernel/eeh_sysfs.c
+++ b/arch/powerpc/kernel/eeh_sysfs.c
@@ -48,7 +48,7 @@ static ssize_t eeh_show_##_name(struct device *dev,      \
 	                                                      \
 	return sprintf(buf, _format "\n", edev->_memb);       \
 }                                                        \
-static DEVICE_ATTR(_name, S_IRUGO, eeh_show_##_name, NULL);
+static DEVICE_ATTR(_name, 0444, eeh_show_##_name, NULL);
 
 EEH_SHOW_ATTR(eeh_mode,            mode,            "0x%x");
 EEH_SHOW_ATTR(eeh_pe_config_addr,  pe_config_addr,  "0x%x");
@@ -90,6 +90,65 @@ static ssize_t eeh_pe_state_store(struct device *dev,
 
 static DEVICE_ATTR_RW(eeh_pe_state);
 
+#ifdef CONFIG_PCI_IOV
+static ssize_t eeh_notify_resume_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct eeh_dev *edev = pci_dev_to_eeh_dev(pdev);
+	struct pci_dn *pdn = pci_get_pdn(pdev);
+
+	if (!edev || !edev->pe)
+		return -ENODEV;
+
+	pdn = pci_get_pdn(pdev);
+	return sprintf(buf, "%d\n", pdn->last_allow_rc);
+}
+
+static ssize_t eeh_notify_resume_store(struct device *dev,
+				       struct device_attribute *attr,
+				       const char *buf, size_t count)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct eeh_dev *edev = pci_dev_to_eeh_dev(pdev);
+
+	if (!edev || !edev->pe || !eeh_ops->notify_resume)
+		return -ENODEV;
+
+	if (eeh_ops->notify_resume(pci_get_pdn(pdev)))
+		return -EIO;
+
+	return count;
+}
+static DEVICE_ATTR_RW(eeh_notify_resume);
+
+static int eeh_notify_resume_add(struct pci_dev *pdev)
+{
+	struct device_node *np;
+	int rc = 0;
+
+	np = pci_device_to_OF_node(pdev->is_physfn ? pdev : pdev->physfn);
+
+	if (of_property_read_bool(np, "ibm,is-open-sriov-pf"))
+		rc = device_create_file(&pdev->dev, &dev_attr_eeh_notify_resume);
+
+	return rc;
+}
+
+static void eeh_notify_resume_remove(struct pci_dev *pdev)
+{
+	struct device_node *np;
+
+	np = pci_device_to_OF_node(pdev->is_physfn ? pdev : pdev->physfn);
+
+	if (of_property_read_bool(np, "ibm,is-open-sriov-pf"))
+		device_remove_file(&pdev->dev, &dev_attr_eeh_notify_resume);
+}
+#else
+static inline int eeh_notify_resume_add(struct pci_dev *pdev) { return 0; }
+static inline void eeh_notify_resume_remove(struct pci_dev *pdev) { }
+#endif /* CONFIG_PCI_IOV */
+
 void eeh_sysfs_add_device(struct pci_dev *pdev)
 {
 	struct eeh_dev *edev = pci_dev_to_eeh_dev(pdev);
@@ -104,6 +163,7 @@ void eeh_sysfs_add_device(struct pci_dev *pdev)
 	rc += device_create_file(&pdev->dev, &dev_attr_eeh_mode);
 	rc += device_create_file(&pdev->dev, &dev_attr_eeh_pe_config_addr);
 	rc += device_create_file(&pdev->dev, &dev_attr_eeh_pe_state);
+	rc += eeh_notify_resume_add(pdev);
 
 	if (rc)
 		pr_warn("EEH: Unable to create sysfs entries\n");
@@ -129,6 +189,8 @@ void eeh_sysfs_remove_device(struct pci_dev *pdev)
 	device_remove_file(&pdev->dev, &dev_attr_eeh_pe_config_addr);
 	device_remove_file(&pdev->dev, &dev_attr_eeh_pe_state);
 
+	eeh_notify_resume_remove(pdev);
+
 	if (edev)
 		edev->mode &= ~EEH_DEV_SYSFS;
 }
diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S
index e780e1fbf6c2..eb8d01bae8c6 100644
--- a/arch/powerpc/kernel/entry_32.S
+++ b/arch/powerpc/kernel/entry_32.S
@@ -211,7 +211,7 @@ transfer_to_handler_cont:
 	mflr	r9
 	lwz	r11,0(r9)		/* virtual address of handler */
 	lwz	r9,4(r9)		/* where to go when done */
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 	mtspr	SPRN_NRI, r0
 #endif
 #ifdef CONFIG_TRACE_IRQFLAGS
@@ -301,7 +301,7 @@ stack_ovf:
 	lis	r9,StackOverflow@ha
 	addi	r9,r9,StackOverflow@l
 	LOAD_MSR_KERNEL(r10,MSR_KERNEL)
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 	mtspr	SPRN_NRI, r0
 #endif
 	mtspr	SPRN_SRR0,r9
@@ -430,7 +430,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
 	lwz	r7,_NIP(r1)
 	lwz	r2,GPR2(r1)
 	lwz	r1,GPR1(r1)
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 	mtspr	SPRN_NRI, r0
 #endif
 	mtspr	SPRN_SRR0,r7
@@ -727,7 +727,7 @@ fast_exception_return:
 	lwz	r10,_LINK(r11)
 	mtlr	r10
 	REST_GPR(10, r11)
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 	mtspr	SPRN_NRI, r0
 #endif
 	mtspr	SPRN_SRR1,r9
@@ -978,7 +978,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
 	.globl exc_exit_restart
 exc_exit_restart:
 	lwz	r12,_NIP(r1)
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS)
 	mtspr	SPRN_NRI, r0
 #endif
 	mtspr	SPRN_SRR0,r12
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 2748584b767d..2cb5109a7ea3 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -133,10 +133,9 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
 	 * of irq tracing is used, we additionally check that condition
 	 * is correct
 	 */
-#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
-	lbz	r10,PACASOFTIRQEN(r13)
-	xori	r10,r10,1
-1:	tdnei	r10,0
+#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
+	lbz	r10,PACAIRQSOFTMASK(r13)
+1:	tdnei	r10,IRQS_ENABLED
 	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
 #endif
 
@@ -152,7 +151,7 @@ system_call:			/* label this so stack traces look sane */
 	/* We do need to set SOFTE in the stack frame or the return
 	 * from interrupt will be painful
 	 */
-	li	r10,1
+	li	r10,IRQS_ENABLED
 	std	r10,SOFTE(r1)
 
 	CURRENT_THREAD_INFO(r11, r1)
@@ -755,10 +754,10 @@ resume_kernel:
 	beq+	restore
 	/* Check that preempt_count() == 0 and interrupts are enabled */
 	lwz	r8,TI_PREEMPT(r9)
-	cmpwi	cr1,r8,0
+	cmpwi	cr0,r8,0
+	bne	restore
 	ld	r0,SOFTE(r1)
-	cmpdi	r0,0
-	crandc	eq,cr1*4+eq,eq
+	andi.	r0,r0,IRQS_DISABLED
 	bne	restore
 
 	/*
@@ -796,12 +795,12 @@ restore:
 	 * are about to re-enable interrupts
 	 */
 	ld	r5,SOFTE(r1)
-	lbz	r6,PACASOFTIRQEN(r13)
-	cmpwi	cr0,r5,0
-	beq	.Lrestore_irq_off
+	lbz	r6,PACAIRQSOFTMASK(r13)
+	andi.	r5,r5,IRQS_DISABLED
+	bne	.Lrestore_irq_off
 
 	/* We are enabling, were we already enabled ? Yes, just return */
-	cmpwi	cr0,r6,1
+	andi.	r6,r6,IRQS_DISABLED
 	beq	cr0,.Ldo_restore
 
 	/*
@@ -820,8 +819,8 @@ restore:
 	 */
 .Lrestore_no_replay:
 	TRACE_ENABLE_INTS
-	li	r0,1
-	stb	r0,PACASOFTIRQEN(r13);
+	li	r0,IRQS_ENABLED
+	stb	r0,PACAIRQSOFTMASK(r13);
 
 	/*
 	 * Final return path. BookE is handled in a different file
@@ -939,9 +938,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 	beq	1f
 	rlwinm	r7,r7,0,~PACA_IRQ_HARD_DIS
 	stb	r7,PACAIRQHAPPENED(r13)
-1:	li	r0,0
-	stb	r0,PACASOFTIRQEN(r13);
-	TRACE_DISABLE_INTS
+1:
+#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG)
+	/* The interrupt should not have soft enabled. */
+	lbz	r7,PACAIRQSOFTMASK(r13)
+1:	tdeqi	r7,IRQS_ENABLED
+	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
+#endif
 	b	.Ldo_restore
 
 	/*
@@ -979,6 +982,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 	addi	r3,r1,STACK_FRAME_OVERHEAD;
  	bl	do_IRQ
 	b	ret_from_except
+1:	cmpwi	cr0,r3,0xf00
+	bne	1f
+	addi	r3,r1,STACK_FRAME_OVERHEAD;
+	bl	performance_monitor_exception
+	b	ret_from_except
 1:	cmpwi	cr0,r3,0xe60
 	bne	1f
 	addi	r3,r1,STACK_FRAME_OVERHEAD;
@@ -1055,15 +1063,15 @@ _GLOBAL(enter_rtas)
 	li	r0,0
 	mtcr	r0
 
-#ifdef CONFIG_BUG	
+#ifdef CONFIG_BUG
 	/* There is no way it is acceptable to get here with interrupts enabled,
 	 * check it with the asm equivalent of WARN_ON
 	 */
-	lbz	r0,PACASOFTIRQEN(r13)
-1:	tdnei	r0,0
+	lbz	r0,PACAIRQSOFTMASK(r13)
+1:	tdeqi	r0,IRQS_ENABLED
 	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
 #endif
-	
+
 	/* Hard-disable interrupts */
 	mfmsr	r6
 	rldicl	r7,r6,48,1
@@ -1107,6 +1115,17 @@ __enter_rtas:
 rtas_return_loc:
 	FIXUP_ENDIAN
 
+	/*
+	 * Clear RI and set SF before anything.
+	 */
+	mfmsr   r6
+	li	r0,MSR_RI
+	andc	r6,r6,r0
+	sldi	r0,r0,(MSR_SF_LG - MSR_RI_LG)
+	or	r6,r6,r0
+	sync
+	mtmsrd  r6
+
 	/* relocation is off at this point */
 	GET_PACA(r4)
 	clrldi	r4,r4,2			/* convert to realmode address */
@@ -1115,12 +1134,6 @@ rtas_return_loc:
 0:	mflr	r3
 	ld	r3,(1f-0b)(r3)		/* get &rtas_restore_regs */
 
-	mfmsr   r6
-	li	r0,MSR_RI
-	andc	r6,r6,r0
-	sync	
-	mtmsrd  r6
-        
         ld	r1,PACAR1(r4)           /* Restore our SP */
         ld	r4,PACASAVEDMSR(r4)     /* Restore our MSR */
 
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index acd8ca76233e..9b6e653e501a 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -139,7 +139,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
 	mfspr	r10,SPRN_ESR
 	SPECIAL_EXC_STORE(r10,ESR)
 
-	lbz	r10,PACASOFTIRQEN(r13)
+	lbz	r10,PACAIRQSOFTMASK(r13)
 	SPECIAL_EXC_STORE(r10,SOFTE)
 	ld	r10,_NIP(r1)
 	SPECIAL_EXC_STORE(r10,CSRR0)
@@ -206,17 +206,17 @@ BEGIN_FTR_SECTION
 	mtspr	SPRN_MAS8,r10
 END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
 
-	lbz	r6,PACASOFTIRQEN(r13)
+	lbz	r6,PACAIRQSOFTMASK(r13)
 	ld	r5,SOFTE(r1)
 
 	/* Interrupts had better not already be enabled... */
-	twnei	r6,0
+	tweqi	r6,IRQS_ENABLED
 
-	cmpwi	cr0,r5,0
-	beq	1f
+	andi.	r6,r5,IRQS_DISABLED
+	bne	1f
 
 	TRACE_ENABLE_INTS
-	stb	r5,PACASOFTIRQEN(r13)
+	stb	r5,PACAIRQSOFTMASK(r13)
 1:
 	/*
 	 * Restore PACAIRQHAPPENED rather than setting it based on
@@ -351,9 +351,9 @@ ret_from_mc_except:
 #define PROLOG_ADDITION_NONE_MC(n)
 
 #define PROLOG_ADDITION_MASKABLE_GEN(n)					    \
-	lbz	r10,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */	    \
-	cmpwi	cr0,r10,0;		/* yes -> go out of line */	    \
-	beq	masked_interrupt_book3e_##n
+	lbz	r10,PACAIRQSOFTMASK(r13);	/* are irqs soft-masked? */ \
+	andi.	r10,r10,IRQS_DISABLED;	/* yes -> go out of line */ \
+	bne	masked_interrupt_book3e_##n
 
 #define PROLOG_ADDITION_2REGS_GEN(n)					    \
 	std	r14,PACA_EXGEN+EX_R14(r13);				    \
@@ -397,7 +397,7 @@ exc_##n##_common:							    \
 	mfspr	r8,SPRN_XER;		/* save XER in stackframe */	    \
 	ld	r9,excf+EX_R1(r13);	/* load orig r1 back from PACA */   \
 	lwz	r10,excf+EX_CR(r13);	/* load orig CR back from PACA	*/  \
-	lbz	r11,PACASOFTIRQEN(r13);	/* get current IRQ softe */	    \
+	lbz	r11,PACAIRQSOFTMASK(r13); /* get current IRQ softe */	    \
 	ld	r12,exception_marker@toc(r2);				    \
 	li	r0,0;							    \
 	std	r3,GPR10(r1);		/* save r10 to stackframe */	    \
@@ -943,6 +943,8 @@ kernel_dbg_exc:
 /*
  * An interrupt came in while soft-disabled; We mark paca->irq_happened
  * accordingly and if the interrupt is level sensitive, we hard disable
+ * hard disable (full_mask) corresponds to PACA_IRQ_MUST_HARD_MASK, so
+ * keep these in synch.
  */
 
 .macro masked_interrupt_book3e paca_irq full_mask
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 2dc10bf646b8..1ecfd8ffb098 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -706,7 +706,7 @@ EXC_COMMON_BEGIN(bad_addr_slb)
 	ld	r3, PACA_EXSLB+EX_DAR(r13)
 	std	r3, _DAR(r1)
 	beq	cr6, 2f
-	li	r10, 0x480		/* fix trap number for I-SLB miss */
+	li	r10, 0x481		/* fix trap number for I-SLB miss */
 	std	r10, _TRAP(r1)
 2:	bl	save_nvgprs
 	addi	r3, r1, STACK_FRAME_OVERHEAD
@@ -718,10 +718,12 @@ EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100)
 hardware_interrupt_hv:
 	BEGIN_FTR_SECTION
 		_MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
-					    EXC_HV, SOFTEN_TEST_HV)
+					    EXC_HV, SOFTEN_TEST_HV,
+					    IRQS_DISABLED)
 	FTR_SECTION_ELSE
 		_MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
-					    EXC_STD, SOFTEN_TEST_PR)
+					    EXC_STD, SOFTEN_TEST_PR,
+					    IRQS_DISABLED)
 	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
 EXC_REAL_END(hardware_interrupt, 0x500, 0x100)
 
@@ -729,9 +731,13 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
 	.globl hardware_interrupt_relon_hv;
 hardware_interrupt_relon_hv:
 	BEGIN_FTR_SECTION
-		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_HV, SOFTEN_TEST_HV)
+		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
+						  EXC_HV, SOFTEN_TEST_HV,
+						  IRQS_DISABLED)
 	FTR_SECTION_ELSE
-		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_STD, SOFTEN_TEST_PR)
+		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
+						  EXC_STD, SOFTEN_TEST_PR,
+						  IRQS_DISABLED)
 	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
 EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
 
@@ -827,8 +833,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM)
 #endif
 
 
-EXC_REAL_MASKABLE(decrementer, 0x900, 0x80)
-EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900)
+EXC_REAL_MASKABLE(decrementer, 0x900, 0x80, IRQS_DISABLED)
+EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900, IRQS_DISABLED)
 TRAMP_KVM(PACA_EXGEN, 0x900)
 EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
 
@@ -839,8 +845,8 @@ TRAMP_KVM_HV(PACA_EXGEN, 0x980)
 EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt)
 
 
-EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0x100)
-EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x100, 0xa00)
+EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0x100, IRQS_DISABLED)
+EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x100, 0xa00, IRQS_DISABLED)
 TRAMP_KVM(PACA_EXGEN, 0xa00)
 #ifdef CONFIG_PPC_DOORBELL
 EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, doorbell_exception)
@@ -1052,7 +1058,7 @@ EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt)
  * mode.
  */
 __EXC_REAL_OOL_HV_DIRECT(hmi_exception, 0xe60, 0x20, hmi_exception_early)
-__TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60)
+__TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED)
 EXC_VIRT_NONE(0x4e60, 0x20)
 TRAMP_KVM_HV(PACA_EXGEN, 0xe60)
 TRAMP_REAL_BEGIN(hmi_exception_early)
@@ -1110,8 +1116,8 @@ EXC_COMMON_BEGIN(hmi_exception_common)
 EXCEPTION_COMMON(PACA_EXGEN, 0xe60, hmi_exception_common, handle_hmi_exception,
         ret_from_except, FINISH_NAP;ADD_NVGPRS;ADD_RECONCILE;RUNLATCH_ON)
 
-EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20)
-EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80)
+EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20, IRQS_DISABLED)
+EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80, IRQS_DISABLED)
 TRAMP_KVM_HV(PACA_EXGEN, 0xe80)
 #ifdef CONFIG_PPC_DOORBELL
 EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception)
@@ -1120,8 +1126,8 @@ EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception)
 #endif
 
 
-EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20)
-EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0)
+EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20, IRQS_DISABLED)
+EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0, IRQS_DISABLED)
 TRAMP_KVM_HV(PACA_EXGEN, 0xea0)
 EXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ)
 
@@ -1132,8 +1138,8 @@ EXC_REAL_NONE(0xee0, 0x20)
 EXC_VIRT_NONE(0x4ee0, 0x20)
 
 
-EXC_REAL_OOL(performance_monitor, 0xf00, 0x20)
-EXC_VIRT_OOL(performance_monitor, 0x4f00, 0x20, 0xf00)
+EXC_REAL_OOL_MASKABLE(performance_monitor, 0xf00, 0x20, IRQS_PMI_DISABLED)
+EXC_VIRT_OOL_MASKABLE(performance_monitor, 0x4f00, 0x20, 0xf00, IRQS_PMI_DISABLED)
 TRAMP_KVM(PACA_EXGEN, 0xf00)
 EXC_COMMON_ASYNC(performance_monitor_common, 0xf00, performance_monitor_exception)
 
@@ -1345,7 +1351,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
 	b	.
 #endif
 
-EXC_COMMON_HV(denorm_common, 0x1500, unknown_exception)
+EXC_COMMON(denorm_common, 0x1500, unknown_exception)
 
 
 #ifdef CONFIG_CBE_RAS
@@ -1420,7 +1426,7 @@ EXC_COMMON_BEGIN(soft_nmi_common)
  *   triggered and won't automatically refire.
  * - If it was a HMI we return immediately since we handled it in realmode
  *   and it won't refire.
- * - else we hard disable and return.
+ * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return.
  * This is called with r10 containing the value to OR to the paca field.
  */
 #define MASKED_INTERRUPT(_H)				\
@@ -1435,8 +1441,8 @@ masked_##_H##interrupt:					\
 	ori	r10,r10,0xffff;				\
 	mtspr	SPRN_DEC,r10;				\
 	b	MASKED_DEC_HANDLER_LABEL;		\
-1:	andi.	r10,r10,(PACA_IRQ_DBELL|PACA_IRQ_HMI);	\
-	bne	2f;					\
+1:	andi.	r10,r10,PACA_IRQ_MUST_HARD_MASK;	\
+	beq	2f;					\
 	mfspr	r10,SPRN_##_H##SRR1;			\
 	xori	r10,r10,MSR_EE; /* clear MSR_EE */	\
 	mtspr	SPRN_##_H##SRR1,r10;			\
@@ -1455,39 +1461,37 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback)
 	std	r9,PACA_EXRFI+EX_R9(r13)
 	std	r10,PACA_EXRFI+EX_R10(r13)
 	std	r11,PACA_EXRFI+EX_R11(r13)
-	std	r12,PACA_EXRFI+EX_R12(r13)
-	std	r8,PACA_EXRFI+EX_R13(r13)
 	mfctr	r9
 	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
-	ld	r11,PACA_L1D_FLUSH_SETS(r13)
-	ld	r12,PACA_L1D_FLUSH_CONGRUENCE(r13)
-	/*
-	 * The load adresses are at staggered offsets within cachelines,
-	 * which suits some pipelines better (on others it should not
-	 * hurt).
-	 */
-	addi	r12,r12,8
+	ld	r11,PACA_L1D_FLUSH_SIZE(r13)
+	srdi	r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
 	mtctr	r11
 	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
 
 	/* order ld/st prior to dcbt stop all streams with flushing */
 	sync
-1:	li	r8,0
-	.rept	8 /* 8-way set associative */
-	ldx	r11,r10,r8
-	add	r8,r8,r12
-	xor	r11,r11,r11	// Ensure r11 is 0 even if fallback area is not
-	add	r8,r8,r11	// Add 0, this creates a dependency on the ldx
-	.endr
-	addi	r10,r10,128 /* 128 byte cache line */
+
+	/*
+	 * The load adresses are at staggered offsets within cachelines,
+	 * which suits some pipelines better (on others it should not
+	 * hurt).
+	 */
+1:
+	ld	r11,(0x80 + 8)*0(r10)
+	ld	r11,(0x80 + 8)*1(r10)
+	ld	r11,(0x80 + 8)*2(r10)
+	ld	r11,(0x80 + 8)*3(r10)
+	ld	r11,(0x80 + 8)*4(r10)
+	ld	r11,(0x80 + 8)*5(r10)
+	ld	r11,(0x80 + 8)*6(r10)
+	ld	r11,(0x80 + 8)*7(r10)
+	addi	r10,r10,0x80*8
 	bdnz	1b
 
 	mtctr	r9
 	ld	r9,PACA_EXRFI+EX_R9(r13)
 	ld	r10,PACA_EXRFI+EX_R10(r13)
 	ld	r11,PACA_EXRFI+EX_R11(r13)
-	ld	r12,PACA_EXRFI+EX_R12(r13)
-	ld	r8,PACA_EXRFI+EX_R13(r13)
 	GET_SCRATCH0(r13);
 	rfid
 
@@ -1497,39 +1501,37 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback)
 	std	r9,PACA_EXRFI+EX_R9(r13)
 	std	r10,PACA_EXRFI+EX_R10(r13)
 	std	r11,PACA_EXRFI+EX_R11(r13)
-	std	r12,PACA_EXRFI+EX_R12(r13)
-	std	r8,PACA_EXRFI+EX_R13(r13)
 	mfctr	r9
 	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
-	ld	r11,PACA_L1D_FLUSH_SETS(r13)
-	ld	r12,PACA_L1D_FLUSH_CONGRUENCE(r13)
-	/*
-	 * The load adresses are at staggered offsets within cachelines,
-	 * which suits some pipelines better (on others it should not
-	 * hurt).
-	 */
-	addi	r12,r12,8
+	ld	r11,PACA_L1D_FLUSH_SIZE(r13)
+	srdi	r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
 	mtctr	r11
 	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
 
 	/* order ld/st prior to dcbt stop all streams with flushing */
 	sync
-1:	li	r8,0
-	.rept	8 /* 8-way set associative */
-	ldx	r11,r10,r8
-	add	r8,r8,r12
-	xor	r11,r11,r11	// Ensure r11 is 0 even if fallback area is not
-	add	r8,r8,r11	// Add 0, this creates a dependency on the ldx
-	.endr
-	addi	r10,r10,128 /* 128 byte cache line */
+
+	/*
+	 * The load adresses are at staggered offsets within cachelines,
+	 * which suits some pipelines better (on others it should not
+	 * hurt).
+	 */
+1:
+	ld	r11,(0x80 + 8)*0(r10)
+	ld	r11,(0x80 + 8)*1(r10)
+	ld	r11,(0x80 + 8)*2(r10)
+	ld	r11,(0x80 + 8)*3(r10)
+	ld	r11,(0x80 + 8)*4(r10)
+	ld	r11,(0x80 + 8)*5(r10)
+	ld	r11,(0x80 + 8)*6(r10)
+	ld	r11,(0x80 + 8)*7(r10)
+	addi	r10,r10,0x80*8
 	bdnz	1b
 
 	mtctr	r9
 	ld	r9,PACA_EXRFI+EX_R9(r13)
 	ld	r10,PACA_EXRFI+EX_R10(r13)
 	ld	r11,PACA_EXRFI+EX_R11(r13)
-	ld	r12,PACA_EXRFI+EX_R12(r13)
-	ld	r8,PACA_EXRFI+EX_R13(r13)
 	GET_SCRATCH0(r13);
 	hrfid
 
@@ -1632,7 +1634,7 @@ USE_TEXT_SECTION()
 	.balign	IFETCH_ALIGN_BYTES
 do_hash_page:
 #ifdef CONFIG_PPC_BOOK3S_64
-	lis	r0,(DSISR_BAD_FAULT_64S|DSISR_DABRMATCH)@h
+	lis	r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h
 	ori	r0,r0,DSISR_BAD_FAULT_64S@l
 	and.	r0,r4,r0		/* weird error? */
 	bne-	handle_page_fault	/* if not, try to insert a HPTE */
@@ -1828,6 +1830,8 @@ BEGIN_FTR_SECTION
 FTR_SECTION_ELSE
 	beq	hardware_interrupt_common
 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300)
+	cmpwi	r3,0xf00
+	beq	performance_monitor_common
 BEGIN_FTR_SECTION
 	cmpwi	r3,0xa00
 	beq	h_doorbell_common_msgclr
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index aa71a90f5222..a61151a6ea5e 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -765,8 +765,8 @@ _GLOBAL(pmac_secondary_start)
 	/* Mark interrupts soft and hard disabled (they might be enabled
 	 * in the PACA when doing hotplug)
 	 */
-	li	r0,0
-	stb	r0,PACASOFTIRQEN(r13)
+	li	r0,IRQS_DISABLED
+	stb	r0,PACAIRQSOFTMASK(r13)
 	li	r0,PACA_IRQ_HARD_DIS
 	stb	r0,PACAIRQHAPPENED(r13)
 
@@ -822,7 +822,8 @@ __secondary_start:
 	/* Mark interrupts soft and hard disabled (they might be enabled
 	 * in the PACA when doing hotplug)
 	 */
-	stb	r7,PACASOFTIRQEN(r13)
+	li	r7,IRQS_DISABLED
+	stb	r7,PACAIRQSOFTMASK(r13)
 	li	r0,PACA_IRQ_HARD_DIS
 	stb	r0,PACAIRQHAPPENED(r13)
 
@@ -988,8 +989,8 @@ start_here_common:
 	/* Mark interrupts soft and hard disabled (they might be enabled
 	 * in the PACA when doing hotplug)
 	 */
-	li	r0,0
-	stb	r0,PACASOFTIRQEN(r13)
+	li	r0,IRQS_DISABLED
+	stb	r0,PACAIRQSOFTMASK(r13)
 	li	r0,PACA_IRQ_HARD_DIS
 	stb	r0,PACAIRQHAPPENED(r13)
 
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 4fee00d414e8..d8670a37d70c 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -33,23 +33,6 @@
 #include <asm/fixmap.h>
 #include <asm/export.h>
 
-/* Macro to make the code more readable. */
-#ifdef CONFIG_8xx_CPU6
-#define SPRN_MI_TWC_ADDR	0x2b80
-#define SPRN_MI_RPN_ADDR	0x2d80
-#define SPRN_MD_TWC_ADDR	0x3b80
-#define SPRN_MD_RPN_ADDR	0x3d80
-
-#define MTSPR_CPU6(spr, reg, treg)	\
-	li	treg, spr##_ADDR;	\
-	stw	treg, 12(r0);		\
-	lwz	treg, 12(r0);		\
-	mtspr	spr, reg
-#else
-#define MTSPR_CPU6(spr, reg, treg)	\
-	mtspr	spr, reg
-#endif
-
 #if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000
 /* By simply checking Address >= 0x80000000, we know if its a kernel address */
 #define SIMPLE_KERNEL_ADDRESS		1
@@ -69,11 +52,7 @@
  * Value for the bits that have fixed value in RPN entries.
  * Also used for tagging DAR for DTLBerror.
  */
-#ifdef CONFIG_PPC_16K_PAGES
-#define RPN_PATTERN	(0x00f0 | MD_SPS16K)
-#else
 #define RPN_PATTERN	0x00f0
-#endif
 
 #define PAGE_SHIFT_512K		19
 #define PAGE_SHIFT_8M		23
@@ -134,15 +113,12 @@ turn_on_mmu:
  * task's thread_struct.
  */
 #define EXCEPTION_PROLOG	\
-	EXCEPTION_PROLOG_0;	\
+	mtspr	SPRN_SPRG_SCRATCH0, r10;	\
+	mtspr	SPRN_SPRG_SCRATCH1, r11;	\
 	mfcr	r10;		\
 	EXCEPTION_PROLOG_1;	\
 	EXCEPTION_PROLOG_2
 
-#define EXCEPTION_PROLOG_0	\
-	mtspr	SPRN_SPRG_SCRATCH0,r10;	\
-	mtspr	SPRN_SPRG_SCRATCH1,r11
-
 #define EXCEPTION_PROLOG_1	\
 	mfspr	r11,SPRN_SRR1;		/* check whether user or kernel */ \
 	andi.	r11,r11,MSR_PR;	\
@@ -176,13 +152,6 @@ turn_on_mmu:
 	SAVE_4GPRS(3, r11);	\
 	SAVE_2GPRS(7, r11)
 
-/*
- * Exception exit code.
- */
-#define EXCEPTION_EPILOG_0	\
-	mfspr	r10,SPRN_SPRG_SCRATCH0;	\
-	mfspr	r11,SPRN_SPRG_SCRATCH1
-
 /*
  * Note: code which follows this uses cr0.eq (set if from kernel),
  * r11, r12 (SRR0), and r9 (SRR1).
@@ -326,15 +295,10 @@ SystemCall:
 #endif
 
 InstructionTLBMiss:
-#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
-	mtspr	SPRN_SPRG_SCRATCH2, r3
-#endif
-	EXCEPTION_PROLOG_0
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
-	lis	r10, (itlb_miss_counter - PAGE_OFFSET)@ha
-	lwz	r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
-	addi	r11, r11, 1
-	stw	r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
+	mtspr	SPRN_SPRG_SCRATCH0, r10
+	mtspr	SPRN_SPRG_SCRATCH1, r11
+#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
+	mtspr	SPRN_SPRG_SCRATCH2, r12
 #endif
 
 	/* If we are faulting a kernel address, we have to use the
@@ -345,7 +309,7 @@ InstructionTLBMiss:
 	/* Only modules will cause ITLB Misses as we always
 	 * pin the first 8MB of kernel memory */
 #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
-	mfcr	r3
+	mfcr	r12
 #endif
 #ifdef ITLB_MISS_KERNEL
 #if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
@@ -388,40 +352,46 @@ _ENTRY(ITLBMiss_cmp)
 	lwz	r10, 0(r10)	/* Get the pte */
 4:
 #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
-	mtcr	r3
+	mtcr	r12
 #endif
-	/* Insert the APG into the TWC from the Linux PTE. */
-	rlwimi	r11, r10, 0, 25, 26
-	/* Load the MI_TWC with the attributes for this "segment." */
-	MTSPR_CPU6(SPRN_MI_TWC, r11, r3)	/* Set segment attributes */
 
-#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
-	rlwimi	r10, r11, 1, MI_SPS16K
-#endif
 #ifdef CONFIG_SWAP
-	rlwinm	r11, r10, 32-5, _PAGE_PRESENT
-	and	r11, r11, r10
-	rlwimi	r10, r11, 0, _PAGE_PRESENT
+	rlwinm	r11, r10, 31, _PAGE_ACCESSED >> 1
 #endif
-	li	r11, RPN_PATTERN
+	/* Load the MI_TWC with the attributes for this "segment." */
+	mtspr	SPRN_MI_TWC, r11	/* Set segment attributes */
+
+	li	r11, RPN_PATTERN | 0x200
 	/* The Linux PTE won't go exactly into the MMU TLB.
-	 * Software indicator bits 20-23 and 28 must be clear.
-	 * Software indicator bits 24, 25, 26, and 27 must be
+	 * Software indicator bits 20 and 23 must be clear.
+	 * Software indicator bits 22, 24, 25, 26, and 27 must be
 	 * set.  All other Linux PTE bits control the behavior
 	 * of the MMU.
 	 */
-#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
-	rlwimi	r10, r11, 0, 0x0ff0	/* Set 24-27, clear 20-23 */
-#else
-	rlwimi	r10, r11, 0, 0x0ff8	/* Set 24-27, clear 20-23,28 */
-#endif
-	MTSPR_CPU6(SPRN_MI_RPN, r10, r3)	/* Update TLB entry */
+	rlwimi	r11, r10, 4, 0x0400	/* Copy _PAGE_EXEC into bit 21 */
+	rlwimi	r10, r11, 0, 0x0ff0	/* Set 22, 24-27, clear 20,23 */
+	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
 
 	/* Restore registers */
-#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
-	mfspr	r3, SPRN_SPRG_SCRATCH2
+_ENTRY(itlb_miss_exit_1)
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
+#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
+	mfspr	r12, SPRN_SPRG_SCRATCH2
+#endif
+	rfi
+#ifdef CONFIG_PERF_EVENTS
+_ENTRY(itlb_miss_perf)
+	lis	r10, (itlb_miss_counter - PAGE_OFFSET)@ha
+	lwz	r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
+	addi	r11, r11, 1
+	stw	r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
+#endif
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
+#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
+	mfspr	r12, SPRN_SPRG_SCRATCH2
 #endif
-	EXCEPTION_EPILOG_0
 	rfi
 
 #ifdef CONFIG_HUGETLB_PAGE
@@ -436,7 +406,6 @@ _ENTRY(ITLBMiss_cmp)
 	rlwinm	r10, r11, 0, ~HUGEPD_SHIFT_MASK
 #endif
 	lwz	r10, 0(r10)	/* Get the pte */
-	rlwinm	r11, r11, 0, 0xf
 	b	4b
 
 20:	/* 512k pages */
@@ -445,21 +414,15 @@ _ENTRY(ITLBMiss_cmp)
 	/* Add level 2 base */
 	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
 	lwz	r10, 0(r10)	/* Get the pte */
-	rlwinm	r11, r11, 0, 0xf
 	b	4b
 #endif
 
 	. = 0x1200
 DataStoreTLBMiss:
-	mtspr	SPRN_SPRG_SCRATCH2, r3
-	EXCEPTION_PROLOG_0
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
-	lis	r10, (dtlb_miss_counter - PAGE_OFFSET)@ha
-	lwz	r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
-	addi	r11, r11, 1
-	stw	r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
-#endif
-	mfcr	r3
+	mtspr	SPRN_SPRG_SCRATCH0, r10
+	mtspr	SPRN_SPRG_SCRATCH1, r11
+	mtspr	SPRN_SPRG_SCRATCH2, r12
+	mfcr	r12
 
 	/* If we are faulting a kernel address, we have to use the
 	 * kernel page tables.
@@ -499,59 +462,49 @@ _ENTRY(DTLBMiss_jmp)
 	rlwimi	r10, r11, 0, 0, 32 - PAGE_SHIFT - 1	/* Add level 2 base */
 	lwz	r10, 0(r10)	/* Get the pte */
 4:
-	mtcr	r3
+	mtcr	r12
 
-	/* Insert the Guarded flag and APG into the TWC from the Linux PTE.
-	 * It is bit 26-27 of both the Linux PTE and the TWC (at least
+	/* Insert the Guarded flag into the TWC from the Linux PTE.
+	 * It is bit 27 of both the Linux PTE and the TWC (at least
 	 * I got that right :-).  It will be better when we can put
 	 * this into the Linux pgd/pmd and load it in the operation
 	 * above.
 	 */
-	rlwimi	r11, r10, 0, 26, 27
-	/* Insert the WriteThru flag into the TWC from the Linux PTE.
-	 * It is bit 25 in the Linux PTE and bit 30 in the TWC
-	 */
-	rlwimi	r11, r10, 32-5, 30, 30
-	MTSPR_CPU6(SPRN_MD_TWC, r11, r3)
-
-	/* In 4k pages mode, SPS (bit 28) in RPN must match PS[1] (bit 29)
-	 * In 16k pages mode, SPS is always 1 */
-#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
-	rlwimi	r10, r11, 1, MD_SPS16K
-#endif
-	/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
-	 * We also need to know if the insn is a load/store, so:
-	 * Clear _PAGE_PRESENT and load that which will
-	 * trap into DTLB Error with store bit set accordinly.
-	 */
-	/* PRESENT=0x1, ACCESSED=0x20
-	 * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5));
-	 * r10 = (r10 & ~PRESENT) | r11;
-	 */
+	rlwimi	r11, r10, 0, _PAGE_GUARDED
 #ifdef CONFIG_SWAP
-	rlwinm	r11, r10, 32-5, _PAGE_PRESENT
-	and	r11, r11, r10
-	rlwimi	r10, r11, 0, _PAGE_PRESENT
+	/* _PAGE_ACCESSED has to be set. We use second APG bit for that, 0
+	 * on that bit will represent a Non Access group
+	 */
+	rlwinm	r11, r10, 31, _PAGE_ACCESSED >> 1
 #endif
+	mtspr	SPRN_MD_TWC, r11
+
 	/* The Linux PTE won't go exactly into the MMU TLB.
-	 * Software indicator bits 22 and 28 must be clear.
 	 * Software indicator bits 24, 25, 26, and 27 must be
 	 * set.  All other Linux PTE bits control the behavior
 	 * of the MMU.
 	 */
 	li	r11, RPN_PATTERN
-#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES)
 	rlwimi	r10, r11, 0, 24, 27	/* Set 24-27 */
-#else
-	rlwimi	r10, r11, 0, 24, 28	/* Set 24-27, clear 28 */
-#endif
-	rlwimi	r10, r11, 0, 20, 20	/* clear 20 */
-	MTSPR_CPU6(SPRN_MD_RPN, r10, r3)	/* Update TLB entry */
+	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
 
 	/* Restore registers */
-	mfspr	r3, SPRN_SPRG_SCRATCH2
 	mtspr	SPRN_DAR, r11	/* Tag DAR */
-	EXCEPTION_EPILOG_0
+_ENTRY(dtlb_miss_exit_1)
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
+	mfspr	r12, SPRN_SPRG_SCRATCH2
+	rfi
+#ifdef CONFIG_PERF_EVENTS
+_ENTRY(dtlb_miss_perf)
+	lis	r10, (dtlb_miss_counter - PAGE_OFFSET)@ha
+	lwz	r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
+	addi	r11, r11, 1
+	stw	r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10)
+#endif
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
+	mfspr	r12, SPRN_SPRG_SCRATCH2
 	rfi
 
 #ifdef CONFIG_HUGETLB_PAGE
@@ -566,7 +519,6 @@ _ENTRY(DTLBMiss_jmp)
 	rlwinm	r10, r11, 0, ~HUGEPD_SHIFT_MASK
 #endif
 	lwz	r10, 0(r10)	/* Get the pte */
-	rlwinm	r11, r11, 0, 0xf
 	b	4b
 
 20:	/* 512k pages */
@@ -575,7 +527,6 @@ _ENTRY(DTLBMiss_jmp)
 	/* Add level 2 base */
 	rlwimi	r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
 	lwz	r10, 0(r10)	/* Get the pte */
-	rlwinm	r11, r11, 0, 0xf
 	b	4b
 #endif
 
@@ -601,7 +552,8 @@ itlbie:
  */
 	. = 0x1400
 DataTLBError:
-	EXCEPTION_PROLOG_0
+	mtspr	SPRN_SPRG_SCRATCH0, r10
+	mtspr	SPRN_SPRG_SCRATCH1, r11
 	mfcr	r10
 
 	mfspr	r11, SPRN_DAR
@@ -636,7 +588,8 @@ dtlbie:
  */
 	. = 0x1c00
 DataBreakpoint:
-	EXCEPTION_PROLOG_0
+	mtspr	SPRN_SPRG_SCRATCH0, r10
+	mtspr	SPRN_SPRG_SCRATCH1, r11
 	mfcr	r10
 	mfspr	r11, SPRN_SRR0
 	cmplwi	cr0, r11, (dtlbie - PAGE_OFFSET)@l
@@ -652,13 +605,15 @@ DataBreakpoint:
 	EXC_XFER_EE(0x1c00, do_break)
 11:
 	mtcr	r10
-	EXCEPTION_EPILOG_0
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
 	rfi
 
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#ifdef CONFIG_PERF_EVENTS
 	. = 0x1d00
 InstructionBreakpoint:
-	EXCEPTION_PROLOG_0
+	mtspr	SPRN_SPRG_SCRATCH0, r10
+	mtspr	SPRN_SPRG_SCRATCH1, r11
 	lis	r10, (instruction_counter - PAGE_OFFSET)@ha
 	lwz	r11, (instruction_counter - PAGE_OFFSET)@l(r10)
 	addi	r11, r11, -1
@@ -666,7 +621,8 @@ InstructionBreakpoint:
 	lis	r10, 0xffff
 	ori	r10, r10, 0x01
 	mtspr	SPRN_COUNTA, r10
-	EXCEPTION_EPILOG_0
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
 	rfi
 #else
 	EXCEPTION(0x1d00, Trap_1d, unknown_exception, EXC_XFER_EE)
@@ -681,51 +637,57 @@ InstructionBreakpoint:
  * not enough space in the DataStoreTLBMiss area.
  */
 DTLBMissIMMR:
-	mtcr	r3
-	/* Set 512k byte guarded page and mark it valid */
-	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID
-	MTSPR_CPU6(SPRN_MD_TWC, r10, r11)
+	mtcr	r12
+	/* Set 512k byte guarded page and mark it valid and accessed */
+	li	r10, MD_PS512K | MD_GUARDED | MD_SVALID | M_APG2
+	mtspr	SPRN_MD_TWC, r10
 	mfspr	r10, SPRN_IMMR			/* Get current IMMR */
 	rlwinm	r10, r10, 0, 0xfff80000		/* Get 512 kbytes boundary */
-	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \
+	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
 			  _PAGE_PRESENT | _PAGE_NO_CACHE
-	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */
+	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
 
 	li	r11, RPN_PATTERN
 	mtspr	SPRN_DAR, r11	/* Tag DAR */
-	mfspr	r3, SPRN_SPRG_SCRATCH2
-	EXCEPTION_EPILOG_0
+_ENTRY(dtlb_miss_exit_2)
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
+	mfspr	r12, SPRN_SPRG_SCRATCH2
 	rfi
 
 DTLBMissLinear:
-	mtcr	r3
-	/* Set 8M byte page and mark it valid */
-	li	r11, MD_PS8MEG | MD_SVALID
-	MTSPR_CPU6(SPRN_MD_TWC, r11, r3)
+	mtcr	r12
+	/* Set 8M byte page and mark it valid and accessed */
+	li	r11, MD_PS8MEG | MD_SVALID | M_APG2
+	mtspr	SPRN_MD_TWC, r11
 	rlwinm	r10, r10, 0, 0x0f800000	/* 8xx supports max 256Mb RAM */
-	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \
+	ori	r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
 			  _PAGE_PRESENT
-	MTSPR_CPU6(SPRN_MD_RPN, r10, r11)	/* Update TLB entry */
+	mtspr	SPRN_MD_RPN, r10	/* Update TLB entry */
 
 	li	r11, RPN_PATTERN
 	mtspr	SPRN_DAR, r11	/* Tag DAR */
-	mfspr	r3, SPRN_SPRG_SCRATCH2
-	EXCEPTION_EPILOG_0
+_ENTRY(dtlb_miss_exit_3)
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
+	mfspr	r12, SPRN_SPRG_SCRATCH2
 	rfi
 
 #ifndef CONFIG_PIN_TLB_TEXT
 ITLBMissLinear:
-	mtcr	r3
-	/* Set 8M byte page and mark it valid */
-	li	r11, MI_PS8MEG | MI_SVALID | _PAGE_EXEC
-	MTSPR_CPU6(SPRN_MI_TWC, r11, r3)
+	mtcr	r12
+	/* Set 8M byte page and mark it valid,accessed */
+	li	r11, MI_PS8MEG | MI_SVALID | M_APG2
+	mtspr	SPRN_MI_TWC, r11
 	rlwinm	r10, r10, 0, 0x0f800000	/* 8xx supports max 256Mb RAM */
-	ori	r10, r10, 0xf0 | MI_SPS16K | _PAGE_SHARED | _PAGE_DIRTY	| \
+	ori	r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
 			  _PAGE_PRESENT
-	MTSPR_CPU6(SPRN_MI_RPN, r10, r11)	/* Update TLB entry */
+	mtspr	SPRN_MI_RPN, r10	/* Update TLB entry */
 
-	mfspr	r3, SPRN_SPRG_SCRATCH2
-	EXCEPTION_EPILOG_0
+_ENTRY(itlb_miss_exit_2)
+	mfspr	r10, SPRN_SPRG_SCRATCH0
+	mfspr	r11, SPRN_SPRG_SCRATCH1
+	mfspr	r12, SPRN_SPRG_SCRATCH2
 	rfi
 #endif
 
@@ -933,13 +895,6 @@ start_here:
 	 */
 	lis	r6, swapper_pg_dir@ha
 	tophys(r6,r6)
-#ifdef CONFIG_8xx_CPU6
-	lis	r4, cpu6_errata_word@h
-	ori	r4, r4, cpu6_errata_word@l
-	li	r3, 0x3f80
-	stw	r3, 12(r4)
-	lwz	r3, 12(r4)
-#endif
 	mtspr	SPRN_M_TW, r6
 	lis	r4,2f@h
 	ori	r4,r4,2f@l
@@ -1004,8 +959,8 @@ initial_mmu:
 	lis	r8, KERNELBASE@h	/* Create vaddr for TLB */
 	ori	r8, r8, MI_EVALID	/* Mark it valid */
 	mtspr	SPRN_MI_EPN, r8
-	li	r8, MI_PS8MEG | (2 << 5)	/* Set 8M byte page, APG 2 */
-	ori	r8, r8, MI_SVALID	/* Make it valid */
+	li	r8, MI_PS8MEG /* Set 8M byte page */
+	ori	r8, r8, MI_SVALID | M_APG2	/* Make it valid, APG 2 */
 	mtspr	SPRN_MI_TWC, r8
 	li	r8, MI_BOOTINIT		/* Create RPN for address 0 */
 	mtspr	SPRN_MI_RPN, r8		/* Store TLB entry */
@@ -1032,7 +987,7 @@ initial_mmu:
 	ori	r8, r8, MD_EVALID	/* Mark it valid */
 	mtspr	SPRN_MD_EPN, r8
 	li	r8, MD_PS512K | MD_GUARDED	/* Set 512k byte page */
-	ori	r8, r8, MD_SVALID	/* Make it valid */
+	ori	r8, r8, MD_SVALID | M_APG2	/* Make it valid and accessed */
 	mtspr	SPRN_MD_TWC, r8
 	mr	r8, r9			/* Create paddr for TLB */
 	ori	r8, r8, MI_BOOTINIT|0x2 /* Inhibit cache -- Cort */
@@ -1061,7 +1016,7 @@ initial_mmu:
 #endif
 	/* Disable debug mode entry on breakpoints */
 	mfspr	r8, SPRN_DER
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#ifdef CONFIG_PERF_EVENTS
 	rlwinm	r8, r8, 0, ~0xc
 #else
 	rlwinm	r8, r8, 0, ~0x8
@@ -1094,13 +1049,7 @@ swapper_pg_dir:
 abatron_pteptrs:
 	.space	8
 
-#ifdef CONFIG_8xx_CPU6
-	.globl	cpu6_errata_word
-cpu6_errata_word:
-	.space	16
-#endif
-
-#ifdef CONFIG_PPC_8xx_PERF_EVENT
+#ifdef CONFIG_PERF_EVENTS
 	.globl	itlb_miss_counter
 itlb_miss_counter:
 	.space	4
diff --git a/arch/powerpc/kernel/idle_book3e.S b/arch/powerpc/kernel/idle_book3e.S
index 48c21acef915..2b269315d377 100644
--- a/arch/powerpc/kernel/idle_book3e.S
+++ b/arch/powerpc/kernel/idle_book3e.S
@@ -17,6 +17,7 @@
 #include <asm/processor.h>
 #include <asm/thread_info.h>
 #include <asm/epapr_hcalls.h>
+#include <asm/hw_irq.h>
 
 /* 64-bit version only for now */
 #ifdef CONFIG_PPC64
@@ -46,8 +47,8 @@ _GLOBAL(\name)
 	bl	trace_hardirqs_on
 	addi    r1,r1,128
 #endif
-	li	r0,1
-	stb	r0,PACASOFTIRQEN(r13)
+	li	r0,IRQS_ENABLED
+	stb	r0,PACAIRQSOFTMASK(r13)
 	
 	/* Interrupts will make use return to LR, so get something we want
 	 * in there
diff --git a/arch/powerpc/kernel/idle_power4.S b/arch/powerpc/kernel/idle_power4.S
index f57a19348bdd..08faa93755f9 100644
--- a/arch/powerpc/kernel/idle_power4.S
+++ b/arch/powerpc/kernel/idle_power4.S
@@ -15,6 +15,7 @@
 #include <asm/ppc_asm.h>
 #include <asm/asm-offsets.h>
 #include <asm/irqflags.h>
+#include <asm/hw_irq.h>
 
 #undef DEBUG
 
@@ -53,8 +54,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP)
 	mfmsr	r7
 #endif /* CONFIG_TRACE_IRQFLAGS */
 
-	li	r0,1
-	stb	r0,PACASOFTIRQEN(r13)	/* we'll hard-enable shortly */
+	li	r0,IRQS_ENABLED
+	stb	r0,PACAIRQSOFTMASK(r13)	/* we'll hard-enable shortly */
 BEGIN_FTR_SECTION
 	DSSALL
 	sync
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index b7a84522e652..061aa0f47bb1 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -67,6 +67,7 @@
 #include <asm/smp.h>
 #include <asm/livepatch.h>
 #include <asm/asm-prototypes.h>
+#include <asm/hw_irq.h>
 
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
@@ -106,12 +107,6 @@ static inline notrace unsigned long get_irq_happened(void)
 	return happened;
 }
 
-static inline notrace void set_soft_enabled(unsigned long enable)
-{
-	__asm__ __volatile__("stb %0,%1(13)"
-	: : "r" (enable), "i" (offsetof(struct paca_struct, soft_enabled)));
-}
-
 static inline notrace int decrementer_check_overflow(void)
 {
  	u64 now = get_tb_or_rtc();
@@ -191,6 +186,11 @@ notrace unsigned int __check_irq_replay(void)
 		return 0x900;
 	}
 
+	if (happened & PACA_IRQ_PMI) {
+		local_paca->irq_happened &= ~PACA_IRQ_PMI;
+		return 0xf00;
+	}
+
 	if (happened & PACA_IRQ_EE) {
 		local_paca->irq_happened &= ~PACA_IRQ_EE;
 		return 0x500;
@@ -224,15 +224,16 @@ notrace unsigned int __check_irq_replay(void)
 	return 0;
 }
 
-notrace void arch_local_irq_restore(unsigned long en)
+notrace void arch_local_irq_restore(unsigned long mask)
 {
 	unsigned char irq_happened;
 	unsigned int replay;
 
 	/* Write the new soft-enabled value */
-	set_soft_enabled(en);
-	if (!en)
+	irq_soft_mask_set(mask);
+	if (mask)
 		return;
+
 	/*
 	 * From this point onward, we can take interrupts, preempt,
 	 * etc... unless we got hard-disabled. We check if an event
@@ -263,7 +264,7 @@ notrace void arch_local_irq_restore(unsigned long en)
 	 */
 	if (unlikely(irq_happened != PACA_IRQ_HARD_DIS))
 		__hard_irq_disable();
-#ifdef CONFIG_TRACE_IRQFLAGS
+#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
 	else {
 		/*
 		 * We should already be hard disabled here. We had bugs
@@ -274,9 +275,9 @@ notrace void arch_local_irq_restore(unsigned long en)
 		if (WARN_ON(mfmsr() & MSR_EE))
 			__hard_irq_disable();
 	}
-#endif /* CONFIG_TRACE_IRQFLAGS */
+#endif
 
-	set_soft_enabled(0);
+	irq_soft_mask_set(IRQS_ALL_DISABLED);
 	trace_hardirqs_off();
 
 	/*
@@ -288,7 +289,7 @@ notrace void arch_local_irq_restore(unsigned long en)
 
 	/* We can soft-enable now */
 	trace_hardirqs_on();
-	set_soft_enabled(1);
+	irq_soft_mask_set(IRQS_ENABLED);
 
 	/*
 	 * And replay if we have to. This will return with interrupts
@@ -363,7 +364,7 @@ bool prep_irq_for_idle(void)
 	 * of entering the low power state.
 	 */
 	local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
-	local_paca->soft_enabled = 1;
+	irq_soft_mask_set(IRQS_ENABLED);
 
 	/* Tell the caller to enter the low power state */
 	return true;
@@ -475,6 +476,14 @@ void force_external_irq_replay(void)
 	 */
 	WARN_ON(!arch_irqs_disabled());
 
+	/*
+	 * Interrupts must always be hard disabled before irq_happened is
+	 * modified (to prevent lost update in case of interrupt between
+	 * load and store).
+	 */
+	__hard_irq_disable();
+	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
+
 	/* Indicate in the PACA that we have an interrupt to replay */
 	local_paca->irq_happened |= PACA_IRQ_EE;
 }
diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c
index 742e4658c5dc..efdd16a79075 100644
--- a/arch/powerpc/kernel/mce.c
+++ b/arch/powerpc/kernel/mce.c
@@ -273,7 +273,7 @@ static void machine_process_ue_event(struct work_struct *work)
 
 				pfn = evt->u.ue_error.physical_address >>
 					PAGE_SHIFT;
-				memory_failure(pfn, SIGBUS, 0);
+				memory_failure(pfn, 0);
 			} else
 				pr_warn("Failed to identify bad address from "
 					"where the uncorrectable error (UE) "
@@ -495,37 +495,123 @@ long machine_check_early(struct pt_regs *regs)
 	return handled;
 }
 
-long hmi_exception_realmode(struct pt_regs *regs)
-{
-	__this_cpu_inc(irq_stat.hmi_exceptions);
+/* Possible meanings for HMER_DEBUG_TRIG bit being set on POWER9 */
+static enum {
+	DTRIG_UNKNOWN,
+	DTRIG_VECTOR_CI,	/* need to emulate vector CI load instr */
+	DTRIG_SUSPEND_ESCAPE,	/* need to escape from TM suspend mode */
+} hmer_debug_trig_function;
 
-#ifdef CONFIG_PPC_BOOK3S_64
-	/* Workaround for P9 vector CI loads (see p9_hmi_special_emu) */
-	if (pvr_version_is(PVR_POWER9)) {
-		unsigned long hmer = mfspr(SPRN_HMER);
-
-		/* Do we have the debug bit set */
-		if (hmer & PPC_BIT(17)) {
-			hmer &= ~PPC_BIT(17);
-			mtspr(SPRN_HMER, hmer);
-
-			/*
-			 * Now to avoid problems with soft-disable we
-			 * only do the emulation if we are coming from
-			 * user space
-			 */
-			if (user_mode(regs))
-				local_paca->hmi_p9_special_emu = 1;
-
-			/*
-			 * Don't bother going to OPAL if that's the
-			 * only relevant bit.
-			 */
-			if (!(hmer & mfspr(SPRN_HMEER)))
-				return local_paca->hmi_p9_special_emu;
+static int init_debug_trig_function(void)
+{
+	int pvr;
+	struct device_node *cpun;
+	struct property *prop = NULL;
+	const char *str;
+
+	/* First look in the device tree */
+	preempt_disable();
+	cpun = of_get_cpu_node(smp_processor_id(), NULL);
+	if (cpun) {
+		of_property_for_each_string(cpun, "ibm,hmi-special-triggers",
+					    prop, str) {
+			if (strcmp(str, "bit17-vector-ci-load") == 0)
+				hmer_debug_trig_function = DTRIG_VECTOR_CI;
+			else if (strcmp(str, "bit17-tm-suspend-escape") == 0)
+				hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE;
 		}
+		of_node_put(cpun);
+	}
+	preempt_enable();
+
+	/* If we found the property, don't look at PVR */
+	if (prop)
+		goto out;
+
+	pvr = mfspr(SPRN_PVR);
+	/* Check for POWER9 Nimbus (scale-out) */
+	if ((PVR_VER(pvr) == PVR_POWER9) && (pvr & 0xe000) == 0) {
+		/* DD2.2 and later */
+		if ((pvr & 0xfff) >= 0x202)
+			hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE;
+		/* DD2.0 and DD2.1 - used for vector CI load emulation */
+		else if ((pvr & 0xfff) >= 0x200)
+			hmer_debug_trig_function = DTRIG_VECTOR_CI;
+	}
+
+ out:
+	switch (hmer_debug_trig_function) {
+	case DTRIG_VECTOR_CI:
+		pr_debug("HMI debug trigger used for vector CI load\n");
+		break;
+	case DTRIG_SUSPEND_ESCAPE:
+		pr_debug("HMI debug trigger used for TM suspend escape\n");
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+__initcall(init_debug_trig_function);
+
+/*
+ * Handle HMIs that occur as a result of a debug trigger.
+ * Return values:
+ * -1 means this is not a HMI cause that we know about
+ *  0 means no further handling is required
+ *  1 means further handling is required
+ */
+long hmi_handle_debugtrig(struct pt_regs *regs)
+{
+	unsigned long hmer = mfspr(SPRN_HMER);
+	long ret = 0;
+
+	/* HMER_DEBUG_TRIG bit is used for various workarounds on P9 */
+	if (!((hmer & HMER_DEBUG_TRIG)
+	      && hmer_debug_trig_function != DTRIG_UNKNOWN))
+		return -1;
+		
+	hmer &= ~HMER_DEBUG_TRIG;
+	/* HMER is a write-AND register */
+	mtspr(SPRN_HMER, ~HMER_DEBUG_TRIG);
+
+	switch (hmer_debug_trig_function) {
+	case DTRIG_VECTOR_CI:
+		/*
+		 * Now to avoid problems with soft-disable we
+		 * only do the emulation if we are coming from
+		 * host user space
+		 */
+		if (regs && user_mode(regs))
+			ret = local_paca->hmi_p9_special_emu = 1;
+
+		break;
+
+	default:
+		break;
 	}
-#endif /* CONFIG_PPC_BOOK3S_64 */
+
+	/*
+	 * See if any other HMI causes remain to be handled
+	 */
+	if (hmer & mfspr(SPRN_HMEER))
+		return -1;
+
+	return ret;
+}
+
+/*
+ * Return values:
+ */
+long hmi_exception_realmode(struct pt_regs *regs)
+{	
+	int ret;
+
+	__this_cpu_inc(irq_stat.hmi_exceptions);
+
+	ret = hmi_handle_debugtrig(regs);
+	if (ret >= 0)
+		return ret;
 
 	wait_for_subcore_guest_exit();
 
diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c
index 644f7040b91c..fe6fc63251fe 100644
--- a/arch/powerpc/kernel/mce_power.c
+++ b/arch/powerpc/kernel/mce_power.c
@@ -58,115 +58,6 @@ static unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr)
 	return pte_pfn(*ptep);
 }
 
-static void flush_tlb_206(unsigned int num_sets, unsigned int action)
-{
-	unsigned long rb;
-	unsigned int i;
-
-	switch (action) {
-	case TLB_INVAL_SCOPE_GLOBAL:
-		rb = TLBIEL_INVAL_SET;
-		break;
-	case TLB_INVAL_SCOPE_LPID:
-		rb = TLBIEL_INVAL_SET_LPID;
-		break;
-	default:
-		BUG();
-		break;
-	}
-
-	asm volatile("ptesync" : : : "memory");
-	for (i = 0; i < num_sets; i++) {
-		asm volatile("tlbiel %0" : : "r" (rb));
-		rb += 1 << TLBIEL_INVAL_SET_SHIFT;
-	}
-	asm volatile("ptesync" : : : "memory");
-}
-
-static void flush_tlb_300(unsigned int num_sets, unsigned int action)
-{
-	unsigned long rb;
-	unsigned int i;
-	unsigned int r;
-
-	switch (action) {
-	case TLB_INVAL_SCOPE_GLOBAL:
-		rb = TLBIEL_INVAL_SET;
-		break;
-	case TLB_INVAL_SCOPE_LPID:
-		rb = TLBIEL_INVAL_SET_LPID;
-		break;
-	default:
-		BUG();
-		break;
-	}
-
-	asm volatile("ptesync" : : : "memory");
-
-	if (early_radix_enabled())
-		r = 1;
-	else
-		r = 0;
-
-	/*
-	 * First flush table/PWC caches with set 0, then flush the
-	 * rest of the sets, partition scope. Radix must then do it
-	 * all again with process scope. Hash just has to flush
-	 * process table.
-	 */
-	asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) : :
-			"r"(rb), "r"(0), "i"(2), "i"(0), "r"(r));
-	for (i = 1; i < num_sets; i++) {
-		unsigned long set = i * (1<<TLBIEL_INVAL_SET_SHIFT);
-
-		asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) : :
-				"r"(rb+set), "r"(0), "i"(2), "i"(0), "r"(r));
-	}
-
-	asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) : :
-			"r"(rb), "r"(0), "i"(2), "i"(1), "r"(r));
-	if (early_radix_enabled()) {
-		for (i = 1; i < num_sets; i++) {
-			unsigned long set = i * (1<<TLBIEL_INVAL_SET_SHIFT);
-
-			asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) : :
-				"r"(rb+set), "r"(0), "i"(2), "i"(1), "r"(r));
-		}
-	}
-
-	asm volatile("ptesync" : : : "memory");
-}
-
-/*
- * Generic routines to flush TLB on POWER processors. These routines
- * are used as flush_tlb hook in the cpu_spec.
- *
- * action => TLB_INVAL_SCOPE_GLOBAL:  Invalidate all TLBs.
- *	     TLB_INVAL_SCOPE_LPID: Invalidate TLB for current LPID.
- */
-void __flush_tlb_power7(unsigned int action)
-{
-	flush_tlb_206(POWER7_TLB_SETS, action);
-}
-
-void __flush_tlb_power8(unsigned int action)
-{
-	flush_tlb_206(POWER8_TLB_SETS, action);
-}
-
-void __flush_tlb_power9(unsigned int action)
-{
-	unsigned int num_sets;
-
-	if (early_radix_enabled())
-		num_sets = POWER9_TLB_SETS_RADIX;
-	else
-		num_sets = POWER9_TLB_SETS_HASH;
-
-	flush_tlb_300(num_sets, action);
-}
-
-
 /* flush SLBs and reload */
 #ifdef CONFIG_PPC_BOOK3S_64
 static void flush_and_reload_slb(void)
@@ -226,10 +117,8 @@ static int mce_flush(int what)
 		return 1;
 	}
 	if (what == MCE_FLUSH_TLB) {
-		if (cur_cpu_spec && cur_cpu_spec->flush_tlb) {
-			cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_GLOBAL);
-			return 1;
-		}
+		tlbiel_all();
+		return 1;
 	}
 
 	return 0;
diff --git a/arch/powerpc/kernel/module.lds b/arch/powerpc/kernel/module.lds
new file mode 100644
index 000000000000..cea5dc124be4
--- /dev/null
+++ b/arch/powerpc/kernel/module.lds
@@ -0,0 +1,8 @@
+/* Force alignment of .toc section.  */
+SECTIONS
+{
+	.toc 0 : ALIGN(256)
+	{
+		*(.got .toc)
+	}
+}
diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c
index 759104b99f9f..a2636c250b7b 100644
--- a/arch/powerpc/kernel/module_64.c
+++ b/arch/powerpc/kernel/module_64.c
@@ -93,6 +93,15 @@ static unsigned int local_entry_offset(const Elf64_Sym *sym)
 {
 	return 0;
 }
+
+void *dereference_module_function_descriptor(struct module *mod, void *ptr)
+{
+	if (ptr < (void *)mod->arch.start_opd ||
+			ptr >= (void *)mod->arch.end_opd)
+		return ptr;
+
+	return dereference_function_descriptor(ptr);
+}
 #endif
 
 #define STUB_MAGIC 0x73747562 /* stub */
@@ -339,11 +348,19 @@ int module_frob_arch_sections(Elf64_Ehdr *hdr,
 		char *p;
 		if (strcmp(secstrings + sechdrs[i].sh_name, ".stubs") == 0)
 			me->arch.stubs_section = i;
-		else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0)
+		else if (strcmp(secstrings + sechdrs[i].sh_name, ".toc") == 0) {
 			me->arch.toc_section = i;
+			if (sechdrs[i].sh_addralign < 8)
+				sechdrs[i].sh_addralign = 8;
+		}
 		else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0)
 			dedotify_versions((void *)hdr + sechdrs[i].sh_offset,
 					  sechdrs[i].sh_size);
+		else if (!strcmp(secstrings + sechdrs[i].sh_name, ".opd")) {
+			me->arch.start_opd = sechdrs[i].sh_addr;
+			me->arch.end_opd = sechdrs[i].sh_addr +
+					   sechdrs[i].sh_size;
+		}
 
 		/* We don't handle .init for the moment: rename to _init */
 		while ((p = strstr(secstrings + sechdrs[i].sh_name, ".init")))
@@ -373,12 +390,15 @@ int module_frob_arch_sections(Elf64_Ehdr *hdr,
 	return 0;
 }
 
-/* r2 is the TOC pointer: it actually points 0x8000 into the TOC (this
-   gives the value maximum span in an instruction which uses a signed
-   offset) */
+/*
+ * r2 is the TOC pointer: it actually points 0x8000 into the TOC (this gives the
+ * value maximum span in an instruction which uses a signed offset). Round down
+ * to a 256 byte boundary for the odd case where we are setting up r2 without a
+ * .toc section.
+ */
 static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me)
 {
-	return sechdrs[me->arch.toc_section].sh_addr + 0x8000;
+	return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000;
 }
 
 /* Both low and high 16 bits are added as SIGNED additions, so if low
@@ -487,12 +507,22 @@ static bool is_early_mcount_callsite(u32 *instruction)
    restore r2. */
 static int restore_r2(u32 *instruction, struct module *me)
 {
-	if (is_early_mcount_callsite(instruction - 1))
+	u32 *prev_insn = instruction - 1;
+
+	if (is_early_mcount_callsite(prev_insn))
+		return 1;
+
+	/*
+	 * Make sure the branch isn't a sibling call.  Sibling calls aren't
+	 * "link" branches and they don't return, so they don't need the r2
+	 * restore afterwards.
+	 */
+	if (!instr_is_relative_link_branch(*prev_insn))
 		return 1;
 
 	if (*instruction != PPC_INST_NOP) {
-		pr_err("%s: Expect noop after relocate, got %08x\n",
-		       me->name, *instruction);
+		pr_err("%s: Expected nop after call, got %08x at %pS\n",
+			me->name, *instruction, instruction);
 		return 0;
 	}
 	/* ld r2,R2_STACK_OFFSET(r1) */
@@ -614,7 +644,8 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 
 		case R_PPC_REL24:
 			/* FIXME: Handle weak symbols here --RR */
-			if (sym->st_shndx == SHN_UNDEF) {
+			if (sym->st_shndx == SHN_UNDEF ||
+			    sym->st_shndx == SHN_LIVEPATCH) {
 				/* External: go via stub */
 				value = stub_for_addr(sechdrs, value, me);
 				if (!value)
diff --git a/arch/powerpc/kernel/optprobes_head.S b/arch/powerpc/kernel/optprobes_head.S
index 52fc864cdec4..98a3aeeb3c8c 100644
--- a/arch/powerpc/kernel/optprobes_head.S
+++ b/arch/powerpc/kernel/optprobes_head.S
@@ -58,7 +58,7 @@ optprobe_template_entry:
 	std	r5,_XER(r1)
 	mfcr	r5
 	std	r5,_CCR(r1)
-	lbz     r5,PACASOFTIRQEN(r13)
+	lbz     r5,PACAIRQSOFTMASK(r13)
 	std     r5,SOFTE(r1)
 
 	/*
diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c
index d6597038931d..95ffedf14885 100644
--- a/arch/powerpc/kernel/paca.c
+++ b/arch/powerpc/kernel/paca.c
@@ -18,6 +18,8 @@
 #include <asm/pgtable.h>
 #include <asm/kexec.h>
 
+#include "setup.h"
+
 #ifdef CONFIG_PPC_BOOK3S
 
 /*
@@ -208,15 +210,14 @@ void __init allocate_pacas(void)
 	u64 limit;
 	int cpu;
 
-	limit = ppc64_rma_size;
-
 #ifdef CONFIG_PPC_BOOK3S_64
 	/*
-	 * We can't take SLB misses on the paca, and we want to access them
-	 * in real mode, so allocate them within the RMA and also within
-	 * the first segment.
+	 * We access pacas in real mode, and cannot take SLB faults
+	 * on them when in virtual mode, so allocate them accordingly.
 	 */
-	limit = min(0x10000000ULL, limit);
+	limit = min(ppc64_bolted_size(), ppc64_rma_size);
+#else
+	limit = ppc64_rma_size;
 #endif
 
 	paca_size = PAGE_ALIGN(sizeof(struct paca_struct) * nr_cpu_ids);
diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
index 0ac7aa346c69..446c79611d56 100644
--- a/arch/powerpc/kernel/pci-common.c
+++ b/arch/powerpc/kernel/pci-common.c
@@ -60,7 +60,7 @@ resource_size_t isa_mem_base;
 EXPORT_SYMBOL(isa_mem_base);
 
 
-static const struct dma_map_ops *pci_dma_ops = &dma_direct_ops;
+static const struct dma_map_ops *pci_dma_ops = &dma_nommu_ops;
 
 void set_pci_dma_ops(const struct dma_map_ops *dma_ops)
 {
@@ -249,8 +249,31 @@ resource_size_t pcibios_iov_resource_alignment(struct pci_dev *pdev, int resno)
 
 	return pci_iov_resource_size(pdev, resno);
 }
+
+int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
+{
+	if (ppc_md.pcibios_sriov_enable)
+		return ppc_md.pcibios_sriov_enable(pdev, num_vfs);
+
+	return 0;
+}
+
+int pcibios_sriov_disable(struct pci_dev *pdev)
+{
+	if (ppc_md.pcibios_sriov_disable)
+		return ppc_md.pcibios_sriov_disable(pdev);
+
+	return 0;
+}
+
 #endif /* CONFIG_PCI_IOV */
 
+void pcibios_bus_add_device(struct pci_dev *pdev)
+{
+	if (ppc_md.pcibios_bus_add_device)
+		ppc_md.pcibios_bus_add_device(pdev);
+}
+
 static resource_size_t pcibios_io_size(const struct pci_controller *hose)
 {
 #ifdef CONFIG_PPC64
@@ -339,8 +362,7 @@ struct pci_controller* pci_find_hose_for_OF_device(struct device_node* node)
  */
 static int pci_read_irq_line(struct pci_dev *pci_dev)
 {
-	struct of_phandle_args oirq;
-	unsigned int virq;
+	int virq;
 
 	pr_debug("PCI: Try to map irq for %s...\n", pci_name(pci_dev));
 
@@ -348,7 +370,8 @@ static int pci_read_irq_line(struct pci_dev *pci_dev)
 	memset(&oirq, 0xff, sizeof(oirq));
 #endif
 	/* Try to get a mapping from the device-tree */
-	if (of_irq_parse_pci(pci_dev, &oirq)) {
+	virq = of_irq_parse_and_map_pci(pci_dev, 0, 0);
+	if (virq <= 0) {
 		u8 line, pin;
 
 		/* If that fails, lets fallback to what is in the config
@@ -372,11 +395,6 @@ static int pci_read_irq_line(struct pci_dev *pci_dev)
 		virq = irq_create_mapping(NULL, line);
 		if (virq)
 			irq_set_irq_type(virq, IRQ_TYPE_LEVEL_LOW);
-	} else {
-		pr_debug(" Got one, spec %d cells (0x%08x 0x%08x...) on %pOF\n",
-			 oirq.args_count, oirq.args[0], oirq.args[1], oirq.np);
-
-		virq = irq_create_of_mapping(&oirq);
 	}
 
 	if (!virq) {
@@ -1276,8 +1294,8 @@ static void pcibios_allocate_bus_resources(struct pci_bus *bus)
 						i + PCI_BRIDGE_RESOURCES) == 0)
 				continue;
 		}
-		pr_warning("PCI: Cannot allocate resource region "
-			   "%d of PCI bridge %d, will remap\n", i, bus->number);
+		pr_warn("PCI: Cannot allocate resource region %d of PCI bridge %d, will remap\n",
+			i, bus->number);
 	clear_resource:
 		/* The resource might be figured out when doing
 		 * reassignment based on the resources required
diff --git a/arch/powerpc/kernel/pci-hotplug.c b/arch/powerpc/kernel/pci-hotplug.c
index 2d71269e7dc1..cf47b1aec4c2 100644
--- a/arch/powerpc/kernel/pci-hotplug.c
+++ b/arch/powerpc/kernel/pci-hotplug.c
@@ -104,7 +104,7 @@ EXPORT_SYMBOL_GPL(pci_hp_remove_devices);
  */
 void pci_hp_add_devices(struct pci_bus *bus)
 {
-	int slotno, mode, pass, max;
+	int slotno, mode, max;
 	struct pci_dev *dev;
 	struct pci_controller *phb;
 	struct device_node *dn = pci_bus_to_OF_node(bus);
@@ -133,13 +133,17 @@ void pci_hp_add_devices(struct pci_bus *bus)
 		pci_scan_slot(bus, PCI_DEVFN(slotno, 0));
 		pcibios_setup_bus_devices(bus);
 		max = bus->busn_res.start;
-		for (pass = 0; pass < 2; pass++) {
-			list_for_each_entry(dev, &bus->devices, bus_list) {
-				if (pci_is_bridge(dev))
-					max = pci_scan_bridge(bus, dev,
-							      max, pass);
-			}
-		}
+		/*
+		 * Scan bridges that are already configured. We don't touch
+		 * them unless they are misconfigured (which will be done in
+		 * the second scan below).
+		 */
+		for_each_pci_bridge(dev, bus)
+			max = pci_scan_bridge(bus, dev, max, 0);
+
+		/* Scan bridges that need to be reconfigured */
+		for_each_pci_bridge(dev, bus)
+			max = pci_scan_bridge(bus, dev, max, 1);
 	}
 	pcibios_finish_adding_to_bus(bus);
 }
diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c
index 1d817f4d97d9..85ad2f78b889 100644
--- a/arch/powerpc/kernel/pci_32.c
+++ b/arch/powerpc/kernel/pci_32.c
@@ -96,7 +96,8 @@ make_one_node_map(struct device_node* node, u8 pci_bus)
 		reg = of_get_property(node, "reg", NULL);
 		if (!reg)
 			continue;
-		dev = pci_get_bus_and_slot(pci_bus, ((reg[0] >> 8) & 0xff));
+		dev = pci_get_domain_bus_and_slot(0, pci_bus,
+						  ((reg[0] >> 8) & 0xff));
 		if (!dev || !dev->subordinate) {
 			pci_dev_put(dev);
 			continue;
diff --git a/arch/powerpc/kernel/pci_dn.c b/arch/powerpc/kernel/pci_dn.c
index 0e395afbf0f4..ab147a1909c8 100644
--- a/arch/powerpc/kernel/pci_dn.c
+++ b/arch/powerpc/kernel/pci_dn.c
@@ -156,10 +156,8 @@ static struct pci_dn *add_one_dev_pci_data(struct pci_dn *parent,
 	pdn->parent = parent;
 	pdn->busno = busno;
 	pdn->devfn = devfn;
-#ifdef CONFIG_PPC_POWERNV
 	pdn->vf_index = vf_index;
 	pdn->pe_number = IODA_INVALID_PE;
-#endif
 	INIT_LIST_HEAD(&pdn->child_list);
 	INIT_LIST_HEAD(&pdn->list);
 	list_add_tail(&pdn->list, &parent->child_list);
@@ -226,9 +224,7 @@ void remove_dev_pci_data(struct pci_dev *pdev)
 	 */
 	if (pdev->is_virtfn) {
 		pdn = pci_get_pdn(pdev);
-#ifdef CONFIG_PPC_POWERNV
 		pdn->pe_number = IODA_INVALID_PE;
-#endif
 		return;
 	}
 
@@ -294,9 +290,7 @@ struct pci_dn *pci_add_device_node_info(struct pci_controller *hose,
 		return NULL;
 	dn->data = pdn;
 	pdn->phb = hose;
-#ifdef CONFIG_PPC_POWERNV
 	pdn->pe_number = IODA_INVALID_PE;
-#endif
 	regs = of_get_property(dn, "reg", NULL);
 	if (regs) {
 		u32 addr = of_read_number(regs, 1);
diff --git a/arch/powerpc/kernel/pci_of_scan.c b/arch/powerpc/kernel/pci_of_scan.c
index 0d790f8432d2..98f04725def7 100644
--- a/arch/powerpc/kernel/pci_of_scan.c
+++ b/arch/powerpc/kernel/pci_of_scan.c
@@ -38,7 +38,7 @@ static u32 get_int_prop(struct device_node *np, const char *name, u32 def)
  * @addr0: value of 1st cell of a device tree PCI address.
  * @bridge: Set this flag if the address is from a bridge 'ranges' property
  */
-static unsigned int pci_parse_of_flags(u32 addr0, int bridge)
+unsigned int pci_parse_of_flags(u32 addr0, int bridge)
 {
 	unsigned int flags = 0;
 
@@ -369,11 +369,8 @@ static void __of_scan_bus(struct device_node *node, struct pci_bus *bus,
 	pcibios_setup_bus_devices(bus);
 
 	/* Now scan child busses */
-	list_for_each_entry(dev, &bus->devices, bus_list) {
-		if (pci_is_bridge(dev)) {
-			of_scan_pci_bridge(dev);
-		}
-	}
+	for_each_pci_bridge(dev, bus)
+		of_scan_pci_bridge(dev);
 }
 
 /**
diff --git a/arch/powerpc/kernel/proc_powerpc.c b/arch/powerpc/kernel/proc_powerpc.c
index 56548bf6231f..9bfbd800d32f 100644
--- a/arch/powerpc/kernel/proc_powerpc.c
+++ b/arch/powerpc/kernel/proc_powerpc.c
@@ -63,7 +63,7 @@ static int __init proc_ppc64_init(void)
 {
 	struct proc_dir_entry *pde;
 
-	pde = proc_create_data("powerpc/systemcfg", S_IFREG|S_IRUGO, NULL,
+	pde = proc_create_data("powerpc/systemcfg", S_IFREG | 0444, NULL,
 			       &page_map_fops, vdso_data);
 	if (!pde)
 		return 1;
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 72be0c32e902..1738c4127b32 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -42,6 +42,7 @@
 #include <linux/hw_breakpoint.h>
 #include <linux/uaccess.h>
 #include <linux/elf-randomize.h>
+#include <linux/pkeys.h>
 
 #include <asm/pgtable.h>
 #include <asm/io.h>
@@ -57,6 +58,7 @@
 #include <asm/debug.h>
 #ifdef CONFIG_PPC64
 #include <asm/firmware.h>
+#include <asm/hw_irq.h>
 #endif
 #include <asm/code-patching.h>
 #include <asm/exec.h>
@@ -601,21 +603,16 @@ EXPORT_SYMBOL(flush_all_to_thread);
 
 #ifdef CONFIG_PPC_ADV_DEBUG_REGS
 void do_send_trap(struct pt_regs *regs, unsigned long address,
-		  unsigned long error_code, int signal_code, int breakpt)
+		  unsigned long error_code, int breakpt)
 {
-	siginfo_t info;
-
-	current->thread.trap_nr = signal_code;
+	current->thread.trap_nr = TRAP_HWBKPT;
 	if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code,
 			11, SIGSEGV) == NOTIFY_STOP)
 		return;
 
 	/* Deliver the signal to userspace */
-	info.si_signo = SIGTRAP;
-	info.si_errno = breakpt;	/* breakpoint or watchpoint id */
-	info.si_code = signal_code;
-	info.si_addr = (void __user *)address;
-	force_sig_info(SIGTRAP, &info, current);
+	force_sig_ptrace_errno_trap(breakpt, /* breakpoint or watchpoint id */
+				    (void __user *)address);
 }
 #else	/* !CONFIG_PPC_ADV_DEBUG_REGS */
 void do_break (struct pt_regs *regs, unsigned long address,
@@ -1102,6 +1099,8 @@ static inline void save_sprs(struct thread_struct *t)
 		t->tar = mfspr(SPRN_TAR);
 	}
 #endif
+
+	thread_pkey_regs_save(t);
 }
 
 static inline void restore_sprs(struct thread_struct *old_thread,
@@ -1141,6 +1140,8 @@ static inline void restore_sprs(struct thread_struct *old_thread,
 	    old_thread->tidr != new_thread->tidr)
 		mtspr(SPRN_TIDR, new_thread->tidr);
 #endif
+
+	thread_pkey_regs_restore(new_thread, old_thread);
 }
 
 #ifdef CONFIG_PPC_BOOK3S_64
@@ -1409,7 +1410,7 @@ void show_regs(struct pt_regs * regs)
 	print_msr_bits(regs->msr);
 	pr_cont("  CR: %08lx  XER: %08lx\n", regs->ccr, regs->xer);
 	trap = TRAP(regs);
-	if ((regs->trap != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
+	if ((TRAP(regs) != 0xc00) && cpu_has_feature(CPU_FTR_CFAR))
 		pr_cont("CFAR: "REG" ", regs->orig_gpr3);
 	if (trap == 0x200 || trap == 0x300 || trap == 0x600)
 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
@@ -1509,14 +1510,15 @@ static int assign_thread_tidr(void)
 {
 	int index;
 	int err;
+	unsigned long flags;
 
 again:
 	if (!ida_pre_get(&vas_thread_ida, GFP_KERNEL))
 		return -ENOMEM;
 
-	spin_lock(&vas_thread_id_lock);
+	spin_lock_irqsave(&vas_thread_id_lock, flags);
 	err = ida_get_new_above(&vas_thread_ida, 1, &index);
-	spin_unlock(&vas_thread_id_lock);
+	spin_unlock_irqrestore(&vas_thread_id_lock, flags);
 
 	if (err == -EAGAIN)
 		goto again;
@@ -1524,9 +1526,9 @@ static int assign_thread_tidr(void)
 		return err;
 
 	if (index > MAX_THREAD_CONTEXT) {
-		spin_lock(&vas_thread_id_lock);
+		spin_lock_irqsave(&vas_thread_id_lock, flags);
 		ida_remove(&vas_thread_ida, index);
-		spin_unlock(&vas_thread_id_lock);
+		spin_unlock_irqrestore(&vas_thread_id_lock, flags);
 		return -ENOMEM;
 	}
 
@@ -1535,9 +1537,11 @@ static int assign_thread_tidr(void)
 
 static void free_thread_tidr(int id)
 {
-	spin_lock(&vas_thread_id_lock);
+	unsigned long flags;
+
+	spin_lock_irqsave(&vas_thread_id_lock, flags);
 	ida_remove(&vas_thread_ida, id);
-	spin_unlock(&vas_thread_id_lock);
+	spin_unlock_irqrestore(&vas_thread_id_lock, flags);
 }
 
 /*
@@ -1589,6 +1593,7 @@ int set_thread_tidr(struct task_struct *t)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(set_thread_tidr);
 
 #endif /* CONFIG_PPC64 */
 
@@ -1674,7 +1679,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
 			childregs->gpr[14] = ppc_function_entry((void *)usp);
 #ifdef CONFIG_PPC64
 		clear_tsk_thread_flag(p, TIF_32BIT);
-		childregs->softe = 1;
+		childregs->softe = IRQS_ENABLED;
 #endif
 		childregs->gpr[15] = kthread_arg;
 		p->thread.regs = NULL;	/* no user register state */
@@ -1865,6 +1870,8 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
 	current->thread.tm_tfiar = 0;
 	current->thread.load_tm = 0;
 #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
+
+	thread_pkey_regs_init(&current->thread);
 }
 EXPORT_SYMBOL(start_thread);
 
diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c
index b15bae265c90..4dffef947b8a 100644
--- a/arch/powerpc/kernel/prom.c
+++ b/arch/powerpc/kernel/prom.c
@@ -59,6 +59,7 @@
 #include <asm/epapr_hcalls.h>
 #include <asm/firmware.h>
 #include <asm/dt_cpu_ftrs.h>
+#include <asm/drmem.h>
 
 #include <mm/mmu_decl.h>
 
@@ -455,92 +456,74 @@ static int __init early_init_dt_scan_chosen_ppc(unsigned long node,
 
 #ifdef CONFIG_PPC_PSERIES
 /*
- * Interpret the ibm,dynamic-memory property in the
- * /ibm,dynamic-reconfiguration-memory node.
+ * Interpret the ibm dynamic reconfiguration memory LMBs.
  * This contains a list of memory blocks along with NUMA affinity
  * information.
  */
-static int __init early_init_dt_scan_drconf_memory(unsigned long node)
+static void __init early_init_drmem_lmb(struct drmem_lmb *lmb,
+					const __be32 **usm)
 {
-	const __be32 *dm, *ls, *usm;
-	int l;
-	unsigned long n, flags;
-	u64 base, size, memblock_size;
-	unsigned int is_kexec_kdump = 0, rngs;
-
-	ls = of_get_flat_dt_prop(node, "ibm,lmb-size", &l);
-	if (ls == NULL || l < dt_root_size_cells * sizeof(__be32))
-		return 0;
-	memblock_size = dt_mem_next_cell(dt_root_size_cells, &ls);
+	u64 base, size;
+	int is_kexec_kdump = 0, rngs;
 
-	dm = of_get_flat_dt_prop(node, "ibm,dynamic-memory", &l);
-	if (dm == NULL || l < sizeof(__be32))
-		return 0;
+	base = lmb->base_addr;
+	size = drmem_lmb_size();
+	rngs = 1;
 
-	n = of_read_number(dm++, 1);	/* number of entries */
-	if (l < (n * (dt_root_addr_cells + 4) + 1) * sizeof(__be32))
-		return 0;
+	/*
+	 * Skip this block if the reserved bit is set in flags
+	 * or if the block is not assigned to this partition.
+	 */
+	if ((lmb->flags & DRCONF_MEM_RESERVED) ||
+	    !(lmb->flags & DRCONF_MEM_ASSIGNED))
+		return;
 
-	/* check if this is a kexec/kdump kernel. */
-	usm = of_get_flat_dt_prop(node, "linux,drconf-usable-memory",
-						 &l);
-	if (usm != NULL)
+	if (*usm)
 		is_kexec_kdump = 1;
 
-	for (; n != 0; --n) {
-		base = dt_mem_next_cell(dt_root_addr_cells, &dm);
-		flags = of_read_number(&dm[3], 1);
-		/* skip DRC index, pad, assoc. list index, flags */
-		dm += 4;
-		/* skip this block if the reserved bit is set in flags
-		   or if the block is not assigned to this partition */
-		if ((flags & DRCONF_MEM_RESERVED) ||
-				!(flags & DRCONF_MEM_ASSIGNED))
-			continue;
-		size = memblock_size;
-		rngs = 1;
+	if (is_kexec_kdump) {
+		/*
+		 * For each memblock in ibm,dynamic-memory, a
+		 * corresponding entry in linux,drconf-usable-memory
+		 * property contains a counter 'p' followed by 'p'
+		 * (base, size) duple. Now read the counter from
+		 * linux,drconf-usable-memory property
+		 */
+		rngs = dt_mem_next_cell(dt_root_size_cells, usm);
+		if (!rngs) /* there are no (base, size) duple */
+			return;
+	}
+
+	do {
 		if (is_kexec_kdump) {
-			/*
-			 * For each memblock in ibm,dynamic-memory, a corresponding
-			 * entry in linux,drconf-usable-memory property contains
-			 * a counter 'p' followed by 'p' (base, size) duple.
-			 * Now read the counter from
-			 * linux,drconf-usable-memory property
-			 */
-			rngs = dt_mem_next_cell(dt_root_size_cells, &usm);
-			if (!rngs) /* there are no (base, size) duple */
+			base = dt_mem_next_cell(dt_root_addr_cells, usm);
+			size = dt_mem_next_cell(dt_root_size_cells, usm);
+		}
+
+		if (iommu_is_off) {
+			if (base >= 0x80000000ul)
 				continue;
+			if ((base + size) > 0x80000000ul)
+				size = 0x80000000ul - base;
 		}
-		do {
-			if (is_kexec_kdump) {
-				base = dt_mem_next_cell(dt_root_addr_cells,
-							 &usm);
-				size = dt_mem_next_cell(dt_root_size_cells,
-							 &usm);
-			}
-			if (iommu_is_off) {
-				if (base >= 0x80000000ul)
-					continue;
-				if ((base + size) > 0x80000000ul)
-					size = 0x80000000ul - base;
-			}
-			memblock_add(base, size);
-		} while (--rngs);
-	}
-	memblock_dump_all();
-	return 0;
+
+		DBG("Adding: %llx -> %llx\n", base, size);
+		memblock_add(base, size);
+	} while (--rngs);
 }
-#else
-#define early_init_dt_scan_drconf_memory(node)	0
 #endif /* CONFIG_PPC_PSERIES */
 
 static int __init early_init_dt_scan_memory_ppc(unsigned long node,
 						const char *uname,
 						int depth, void *data)
 {
+#ifdef CONFIG_PPC_PSERIES
 	if (depth == 1 &&
-	    strcmp(uname, "ibm,dynamic-reconfiguration-memory") == 0)
-		return early_init_dt_scan_drconf_memory(node);
+	    strcmp(uname, "ibm,dynamic-reconfiguration-memory") == 0) {
+		walk_drmem_lmbs_early(node, early_init_drmem_lmb);
+		return 0;
+	}
+#endif
 	
 	return early_init_dt_scan_memory(node, uname, depth, data);
 }
diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c
index 02190e90c7ae..acf4b2e0530c 100644
--- a/arch/powerpc/kernel/prom_init.c
+++ b/arch/powerpc/kernel/prom_init.c
@@ -869,6 +869,7 @@ struct ibm_arch_vec __cacheline_aligned ibm_architecture_vec = {
 		.reserved2 = 0,
 		.reserved3 = 0,
 		.subprocessors = 1,
+		.byte22 = OV5_FEAT(OV5_DRMEM_V2),
 		.intarch = 0,
 		.mmu = 0,
 		.hash_ext = 0,
diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c
index f52ad5bb7109..ca72d7391d40 100644
--- a/arch/powerpc/kernel/ptrace.c
+++ b/arch/powerpc/kernel/ptrace.c
@@ -35,6 +35,7 @@
 #include <linux/context_tracking.h>
 
 #include <linux/uaccess.h>
+#include <linux/pkeys.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/switch_to.h>
@@ -283,6 +284,18 @@ int ptrace_get_reg(struct task_struct *task, int regno, unsigned long *data)
 	if (regno == PT_DSCR)
 		return get_user_dscr(task, data);
 
+#ifdef CONFIG_PPC64
+	/*
+	 * softe copies paca->irq_soft_mask variable state. Since irq_soft_mask is
+	 * no more used as a flag, lets force usr to alway see the softe value as 1
+	 * which means interrupts are not soft disabled.
+	 */
+	if (regno == PT_SOFTE) {
+		*data = 1;
+		return  0;
+	}
+#endif
+
 	if (regno < (sizeof(struct pt_regs) / sizeof(unsigned long))) {
 		*data = ((unsigned long *)task->thread.regs)[regno];
 		return 0;
@@ -1775,6 +1788,61 @@ static int pmu_set(struct task_struct *target,
 	return ret;
 }
 #endif
+
+#ifdef CONFIG_PPC_MEM_KEYS
+static int pkey_active(struct task_struct *target,
+		       const struct user_regset *regset)
+{
+	if (!arch_pkeys_enabled())
+		return -ENODEV;
+
+	return regset->n;
+}
+
+static int pkey_get(struct task_struct *target,
+		    const struct user_regset *regset,
+		    unsigned int pos, unsigned int count,
+		    void *kbuf, void __user *ubuf)
+{
+	BUILD_BUG_ON(TSO(amr) + sizeof(unsigned long) != TSO(iamr));
+	BUILD_BUG_ON(TSO(iamr) + sizeof(unsigned long) != TSO(uamor));
+
+	if (!arch_pkeys_enabled())
+		return -ENODEV;
+
+	return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
+				   &target->thread.amr, 0,
+				   ELF_NPKEY * sizeof(unsigned long));
+}
+
+static int pkey_set(struct task_struct *target,
+		      const struct user_regset *regset,
+		      unsigned int pos, unsigned int count,
+		      const void *kbuf, const void __user *ubuf)
+{
+	u64 new_amr;
+	int ret;
+
+	if (!arch_pkeys_enabled())
+		return -ENODEV;
+
+	/* Only the AMR can be set from userspace */
+	if (pos != 0 || count != sizeof(new_amr))
+		return -EINVAL;
+
+	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+				 &new_amr, 0, sizeof(new_amr));
+	if (ret)
+		return ret;
+
+	/* UAMOR determines which bits of the AMR can be set from userspace. */
+	target->thread.amr = (new_amr & target->thread.uamor) |
+		(target->thread.amr & ~target->thread.uamor);
+
+	return 0;
+}
+#endif /* CONFIG_PPC_MEM_KEYS */
+
 /*
  * These are our native regset flavors.
  */
@@ -1809,6 +1877,9 @@ enum powerpc_regset {
 	REGSET_EBB,		/* EBB registers */
 	REGSET_PMR,		/* Performance Monitor Registers */
 #endif
+#ifdef CONFIG_PPC_MEM_KEYS
+	REGSET_PKEY,		/* AMR register */
+#endif
 };
 
 static const struct user_regset native_regsets[] = {
@@ -1914,6 +1985,13 @@ static const struct user_regset native_regsets[] = {
 		.active = pmu_active, .get = pmu_get, .set = pmu_set
 	},
 #endif
+#ifdef CONFIG_PPC_MEM_KEYS
+	[REGSET_PKEY] = {
+		.core_note_type = NT_PPC_PKEY, .n = ELF_NPKEY,
+		.size = sizeof(u64), .align = sizeof(u64),
+		.active = pkey_active, .get = pkey_get, .set = pkey_set
+	},
+#endif
 };
 
 static const struct user_regset_view user_ppc_native_view = {
diff --git a/arch/powerpc/kernel/rtas-proc.c b/arch/powerpc/kernel/rtas-proc.c
index c8c5f3a550c2..fb070d8cad07 100644
--- a/arch/powerpc/kernel/rtas-proc.c
+++ b/arch/powerpc/kernel/rtas-proc.c
@@ -261,19 +261,19 @@ static int __init proc_rtas_init(void)
 	if (rtas_node == NULL)
 		return -ENODEV;
 
-	proc_create("powerpc/rtas/progress", S_IRUGO|S_IWUSR, NULL,
+	proc_create("powerpc/rtas/progress", 0644, NULL,
 		    &ppc_rtas_progress_operations);
-	proc_create("powerpc/rtas/clock", S_IRUGO|S_IWUSR, NULL,
+	proc_create("powerpc/rtas/clock", 0644, NULL,
 		    &ppc_rtas_clock_operations);
-	proc_create("powerpc/rtas/poweron", S_IWUSR|S_IRUGO, NULL,
+	proc_create("powerpc/rtas/poweron", 0644, NULL,
 		    &ppc_rtas_poweron_operations);
-	proc_create("powerpc/rtas/sensors", S_IRUGO, NULL,
+	proc_create("powerpc/rtas/sensors", 0444, NULL,
 		    &ppc_rtas_sensors_operations);
-	proc_create("powerpc/rtas/frequency", S_IWUSR|S_IRUGO, NULL,
+	proc_create("powerpc/rtas/frequency", 0644, NULL,
 		    &ppc_rtas_tone_freq_operations);
-	proc_create("powerpc/rtas/volume", S_IWUSR|S_IRUGO, NULL,
+	proc_create("powerpc/rtas/volume", 0644, NULL,
 		    &ppc_rtas_tone_volume_operations);
-	proc_create("powerpc/rtas/rmo_buffer", S_IRUSR, NULL,
+	proc_create("powerpc/rtas/rmo_buffer", 0400, NULL,
 		    &ppc_rtas_rmo_buf_ops);
 	return 0;
 }
diff --git a/arch/powerpc/kernel/rtas_flash.c b/arch/powerpc/kernel/rtas_flash.c
index f6f6a8a5103a..10fabae2574d 100644
--- a/arch/powerpc/kernel/rtas_flash.c
+++ b/arch/powerpc/kernel/rtas_flash.c
@@ -727,7 +727,7 @@ static int __init rtas_flash_init(void)
 		const struct rtas_flash_file *f = &rtas_flash_files[i];
 		int token;
 
-		if (!proc_create(f->filename, S_IRUSR | S_IWUSR, NULL, &f->fops))
+		if (!proc_create(f->filename, 0600, NULL, &f->fops))
 			goto enomem;
 
 		/*
diff --git a/arch/powerpc/kernel/rtasd.c b/arch/powerpc/kernel/rtasd.c
index 0f0b1b2f3b60..f915db93cd42 100644
--- a/arch/powerpc/kernel/rtasd.c
+++ b/arch/powerpc/kernel/rtasd.c
@@ -388,11 +388,11 @@ static ssize_t rtas_log_read(struct file * file, char __user * buf,
 	return error;
 }
 
-static unsigned int rtas_log_poll(struct file *file, poll_table * wait)
+static __poll_t rtas_log_poll(struct file *file, poll_table * wait)
 {
 	poll_wait(file, &rtas_log_wait, wait);
 	if (rtas_log_size)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 
@@ -581,7 +581,7 @@ static int __init rtas_init(void)
 	if (!rtas_log_buf)
 		return -ENODEV;
 
-	entry = proc_create("powerpc/rtas/error_log", S_IRUSR, NULL,
+	entry = proc_create("powerpc/rtas/error_log", 0400, NULL,
 			    &proc_rtas_log_operations);
 	if (!entry)
 		printk(KERN_ERR "Failed to create error_log proc entry\n");
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 8fd3a70047f1..d73ec518ef80 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -346,10 +346,8 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 		   loops_per_jiffy / (500000/HZ),
 		   (loops_per_jiffy / (5000/HZ)) % 100);
 #endif
-
-#ifdef CONFIG_SMP
 	seq_printf(m, "\n");
-#endif
+
 	/* If this is the last cpu, print the summary */
 	if (cpumask_next(cpu_id, cpu_online_mask) >= nr_cpu_ids)
 		show_cpuinfo_summary(m);
@@ -379,10 +377,10 @@ static void c_stop(struct seq_file *m, void *v)
 }
 
 const struct seq_operations cpuinfo_op = {
-	.start =c_start,
-	.next =	c_next,
-	.stop =	c_stop,
-	.show =	show_cpuinfo,
+	.start	= c_start,
+	.next	= c_next,
+	.stop	= c_stop,
+	.show	= show_cpuinfo,
 };
 
 void __init check_for_initrd(void)
@@ -459,13 +457,13 @@ static void __init cpu_init_thread_core_maps(int tpc)
  */
 void __init smp_setup_cpu_maps(void)
 {
-	struct device_node *dn = NULL;
+	struct device_node *dn;
 	int cpu = 0;
 	int nthreads = 1;
 
 	DBG("smp_setup_cpu_maps()\n");
 
-	while ((dn = of_find_node_by_type(dn, "cpu")) && cpu < nr_cpu_ids) {
+	for_each_node_by_type(dn, "cpu") {
 		const __be32 *intserv;
 		__be32 cpu_be;
 		int j, len;
@@ -505,6 +503,11 @@ void __init smp_setup_cpu_maps(void)
 			set_cpu_possible(cpu, true);
 			cpu++;
 		}
+
+		if (cpu >= nr_cpu_ids) {
+			of_node_put(dn);
+			break;
+		}
 	}
 
 	/* If no SMT supported, nthreads is forced to 1 */
@@ -780,7 +783,7 @@ void arch_setup_pdev_archdata(struct platform_device *pdev)
 {
 	pdev->archdata.dma_mask = DMA_BIT_MASK(32);
 	pdev->dev.dma_mask = &pdev->archdata.dma_mask;
- 	set_dma_ops(&pdev->dev, &dma_direct_ops);
+ 	set_dma_ops(&pdev->dev, &dma_nommu_ops);
 }
 
 static __init void print_system_info(void)
diff --git a/arch/powerpc/kernel/setup.h b/arch/powerpc/kernel/setup.h
index 21c18071d9d5..3fc11e30308f 100644
--- a/arch/powerpc/kernel/setup.h
+++ b/arch/powerpc/kernel/setup.h
@@ -51,6 +51,10 @@ void record_spr_defaults(void);
 static inline void record_spr_defaults(void) { };
 #endif
 
+#ifdef CONFIG_PPC64
+u64 ppc64_bolted_size(void);
+#endif
+
 /*
  * Having this in kvm_ppc.h makes include dependencies too
  * tricky to solve for setup-common.c so have it here.
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index e67413f4a8f0..c388cc3357fa 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -10,8 +10,6 @@
  *      2 of the License, or (at your option) any later version.
  */
 
-#define DEBUG
-
 #include <linux/export.h>
 #include <linux/string.h>
 #include <linux/sched.h>
@@ -69,6 +67,7 @@
 #include <asm/livepatch.h>
 #include <asm/opal.h>
 #include <asm/cputhreads.h>
+#include <asm/hw_irq.h>
 
 #include "setup.h"
 
@@ -190,6 +189,8 @@ static void __init fixup_boot_paca(void)
 	get_paca()->cpu_start = 1;
 	/* Allow percpu accesses to work until we setup percpu data */
 	get_paca()->data_offset = 0;
+	/* Mark interrupts disabled in PACA */
+	irq_soft_mask_set(IRQS_DISABLED);
 }
 
 static void __init configure_exceptions(void)
@@ -352,7 +353,7 @@ void __init early_setup(unsigned long dt_ptr)
 void early_setup_secondary(void)
 {
 	/* Mark interrupts disabled in PACA */
-	get_paca()->soft_enabled = 0;
+	irq_soft_mask_set(IRQS_DISABLED);
 
 	/* Initialize the hash table or TLB handling */
 	early_init_mmu_secondary();
@@ -568,25 +569,31 @@ void __init initialize_cache_info(void)
 	DBG(" <- initialize_cache_info()\n");
 }
 
-/* This returns the limit below which memory accesses to the linear
- * mapping are guarnateed not to cause a TLB or SLB miss. This is
- * used to allocate interrupt or emergency stacks for which our
- * exception entry path doesn't deal with being interrupted.
+/*
+ * This returns the limit below which memory accesses to the linear
+ * mapping are guarnateed not to cause an architectural exception (e.g.,
+ * TLB or SLB miss fault).
+ *
+ * This is used to allocate PACAs and various interrupt stacks that
+ * that are accessed early in interrupt handlers that must not cause
+ * re-entrant interrupts.
  */
-static __init u64 safe_stack_limit(void)
+__init u64 ppc64_bolted_size(void)
 {
 #ifdef CONFIG_PPC_BOOK3E
 	/* Freescale BookE bolts the entire linear mapping */
-	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E))
+	/* XXX: BookE ppc64_rma_limit setup seems to disagree? */
+	if (early_mmu_has_feature(MMU_FTR_TYPE_FSL_E))
 		return linear_map_top;
 	/* Other BookE, we assume the first GB is bolted */
 	return 1ul << 30;
 #else
+	/* BookS radix, does not take faults on linear mapping */
 	if (early_radix_enabled())
 		return ULONG_MAX;
 
-	/* BookS, the first segment is bolted */
-	if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
+	/* BookS hash, the first segment is bolted */
+	if (early_mmu_has_feature(MMU_FTR_1T_SEGMENT))
 		return 1UL << SID_SHIFT_1T;
 	return 1UL << SID_SHIFT;
 #endif
@@ -594,7 +601,7 @@ static __init u64 safe_stack_limit(void)
 
 void __init irqstack_early_init(void)
 {
-	u64 limit = safe_stack_limit();
+	u64 limit = ppc64_bolted_size();
 	unsigned int i;
 
 	/*
@@ -679,7 +686,7 @@ void __init emergency_stack_init(void)
 	 * initialized in kernel/irq.c. These are initialized here in order
 	 * to have emergency stacks available as early as possible.
 	 */
-	limit = min(safe_stack_limit(), ppc64_rma_size);
+	limit = min(ppc64_bolted_size(), ppc64_rma_size);
 
 	for_each_possible_cpu(i) {
 		struct thread_info *ti;
@@ -857,7 +864,7 @@ static void init_fallback_flush(void)
 	int cpu;
 
 	l1d_size = ppc64_caches.l1d.size;
-	limit = min(safe_stack_limit(), ppc64_rma_size);
+	limit = min(ppc64_bolted_size(), ppc64_rma_size);
 
 	/*
 	 * Align to L1d size, and size it at 2x L1d size, to catch possible
@@ -868,19 +875,8 @@ static void init_fallback_flush(void)
 	memset(l1d_flush_fallback_area, 0, l1d_size * 2);
 
 	for_each_possible_cpu(cpu) {
-		/*
-		 * The fallback flush is currently coded for 8-way
-		 * associativity. Different associativity is possible, but it
-		 * will be treated as 8-way and may not evict the lines as
-		 * effectively.
-		 *
-		 * 128 byte lines are mandatory.
-		 */
-		u64 c = l1d_size / 8;
-
 		paca[cpu].rfi_flush_fallback_area = l1d_flush_fallback_area;
-		paca[cpu].l1d_flush_congruence = c;
-		paca[cpu].l1d_flush_sets = c / 128;
+		paca[cpu].l1d_flush_size = l1d_size;
 	}
 }
 
diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c
index 3d7539b90010..61db86ecd318 100644
--- a/arch/powerpc/kernel/signal.c
+++ b/arch/powerpc/kernel/signal.c
@@ -153,6 +153,9 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)
 	if (thread_info_flags & _TIF_UPROBE)
 		uprobe_notify_resume(regs);
 
+	if (thread_info_flags & _TIF_PATCH_PENDING)
+		klp_update_patch_state(current);
+
 	if (thread_info_flags & _TIF_SIGPENDING) {
 		BUG_ON(regs != current->thread.regs);
 		do_signal(current);
@@ -163,9 +166,6 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags)
 		tracehook_notify_resume(regs);
 	}
 
-	if (thread_info_flags & _TIF_PATCH_PENDING)
-		klp_update_patch_state(current);
-
 	user_enter();
 }
 
diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c
index 9ffd73296f64..a46de0035214 100644
--- a/arch/powerpc/kernel/signal_32.c
+++ b/arch/powerpc/kernel/signal_32.c
@@ -111,12 +111,20 @@ static inline int save_general_regs(struct pt_regs *regs,
 {
 	elf_greg_t64 *gregs = (elf_greg_t64 *)regs;
 	int i;
+	/* Force usr to alway see softe as 1 (interrupts enabled) */
+	elf_greg_t64 softe = 0x1;
 
 	WARN_ON(!FULL_REGS(regs));
 
 	for (i = 0; i <= PT_RESULT; i ++) {
 		if (i == 14 && !FULL_REGS(regs))
 			i = 32;
+		if ( i == PT_SOFTE) {
+			if(__put_user((unsigned int)softe, &frame->mc_gregs[i]))
+				return -EFAULT;
+			else
+				continue;
+		}
 		if (__put_user((unsigned int)gregs[i], &frame->mc_gregs[i]))
 			return -EFAULT;
 	}
@@ -873,75 +881,9 @@ static long restore_tm_user_regs(struct pt_regs *regs,
 #endif
 
 #ifdef CONFIG_PPC64
-int copy_siginfo_to_user32(struct compat_siginfo __user *d, const siginfo_t *s)
-{
-	int err;
-
-	if (!access_ok (VERIFY_WRITE, d, sizeof(*d)))
-		return -EFAULT;
-
-	/* If you change siginfo_t structure, please be sure
-	 * this code is fixed accordingly.
-	 * It should never copy any pad contained in the structure
-	 * to avoid security leaks, but must copy the generic
-	 * 3 ints plus the relevant union member.
-	 * This routine must convert siginfo from 64bit to 32bit as well
-	 * at the same time.
-	 */
-	err = __put_user(s->si_signo, &d->si_signo);
-	err |= __put_user(s->si_errno, &d->si_errno);
-	err |= __put_user(s->si_code, &d->si_code);
-	if (s->si_code < 0)
-		err |= __copy_to_user(&d->_sifields._pad, &s->_sifields._pad,
-				      SI_PAD_SIZE32);
-	else switch(siginfo_layout(s->si_signo, s->si_code)) {
-	case SIL_CHLD:
-		err |= __put_user(s->si_pid, &d->si_pid);
-		err |= __put_user(s->si_uid, &d->si_uid);
-		err |= __put_user(s->si_utime, &d->si_utime);
-		err |= __put_user(s->si_stime, &d->si_stime);
-		err |= __put_user(s->si_status, &d->si_status);
-		break;
-	case SIL_FAULT:
-		err |= __put_user((unsigned int)(unsigned long)s->si_addr,
-				  &d->si_addr);
-		break;
-	case SIL_POLL:
-		err |= __put_user(s->si_band, &d->si_band);
-		err |= __put_user(s->si_fd, &d->si_fd);
-		break;
-	case SIL_TIMER:
-		err |= __put_user(s->si_tid, &d->si_tid);
-		err |= __put_user(s->si_overrun, &d->si_overrun);
-		err |= __put_user(s->si_int, &d->si_int);
-		break;
-	case SIL_SYS:
-		err |= __put_user(ptr_to_compat(s->si_call_addr), &d->si_call_addr);
-		err |= __put_user(s->si_syscall, &d->si_syscall);
-		err |= __put_user(s->si_arch, &d->si_arch);
-		break;
-	case SIL_RT:
-		err |= __put_user(s->si_int, &d->si_int);
-		/* fallthrough */
-	case SIL_KILL:
-		err |= __put_user(s->si_pid, &d->si_pid);
-		err |= __put_user(s->si_uid, &d->si_uid);
-		break;
-	}
-	return err;
-}
 
 #define copy_siginfo_to_user	copy_siginfo_to_user32
 
-int copy_siginfo_from_user32(siginfo_t *to, struct compat_siginfo __user *from)
-{
-	if (copy_from_user(to, from, 3*sizeof(int)) ||
-	    copy_from_user(to->_sifields._pad,
-			   from->_sifields._pad, SI_PAD_SIZE32))
-		return -EFAULT;
-
-	return 0;
-}
 #endif /* CONFIG_PPC64 */
 
 /*
diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c
index 4b9ca3570344..720117690822 100644
--- a/arch/powerpc/kernel/signal_64.c
+++ b/arch/powerpc/kernel/signal_64.c
@@ -110,6 +110,8 @@ static long setup_sigcontext(struct sigcontext __user *sc,
 	struct pt_regs *regs = tsk->thread.regs;
 	unsigned long msr = regs->msr;
 	long err = 0;
+	/* Force usr to alway see softe as 1 (interrupts enabled) */
+	unsigned long softe = 0x1;
 
 	BUG_ON(tsk != current);
 
@@ -169,6 +171,7 @@ static long setup_sigcontext(struct sigcontext __user *sc,
 	WARN_ON(!FULL_REGS(regs));
 	err |= __copy_to_user(&sc->gp_regs, regs, GP_REGS_SIZE);
 	err |= __put_user(msr, &sc->gp_regs[PT_MSR]);
+	err |= __put_user(softe, &sc->gp_regs[PT_SOFTE]);
 	err |= __put_user(signr, &sc->signal);
 	err |= __put_user(handler, &sc->handler);
 	if (set != NULL)
@@ -207,7 +210,7 @@ static long setup_tm_sigcontexts(struct sigcontext __user *sc,
 	elf_vrreg_t __user *tm_v_regs = sigcontext_vmx_regs(tm_sc);
 #endif
 	struct pt_regs *regs = tsk->thread.regs;
-	unsigned long msr = tsk->thread.ckpt_regs.msr;
+	unsigned long msr = tsk->thread.regs->msr;
 	long err = 0;
 
 	BUG_ON(tsk != current);
@@ -216,6 +219,12 @@ static long setup_tm_sigcontexts(struct sigcontext __user *sc,
 
 	WARN_ON(tm_suspend_disabled);
 
+	/* Restore checkpointed FP, VEC, and VSX bits from ckpt_regs as
+	 * it contains the correct FP, VEC, VSX state after we treclaimed
+	 * the transaction and giveup_all() was called on reclaiming.
+	 */
+	msr |= tsk->thread.ckpt_regs.msr & (MSR_FP | MSR_VEC | MSR_VSX);
+
 	/* Remove TM bits from thread's MSR.  The MSR in the sigcontext
 	 * just indicates to userland that we were doing a transaction, but we
 	 * don't want to return in transactional state.  This also ensures
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index e0a4c1f82e25..bbe7634b3a43 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -543,7 +543,25 @@ void smp_send_debugger_break(void)
 #ifdef CONFIG_KEXEC_CORE
 void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
 {
+	int cpu;
+
 	smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000);
+	if (kdump_in_progress() && crash_wake_offline) {
+		for_each_present_cpu(cpu) {
+			if (cpu_online(cpu))
+				continue;
+			/*
+			 * crash_ipi_callback will wait for
+			 * all cpus, including offline CPUs.
+			 * We don't care about nmi_ipi_function.
+			 * Offline cpus will jump straight into
+			 * crash_ipi_callback, we can skip the
+			 * entire NMI dance and waiting for
+			 * cpus to clear pending mask, etc.
+			 */
+			do_smp_send_nmi_ipi(cpu);
+		}
+	}
 }
 #endif
 
diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c
index b8d4a1dac39f..04d0bbd7a1dd 100644
--- a/arch/powerpc/kernel/sysfs.c
+++ b/arch/powerpc/kernel/sysfs.c
@@ -485,6 +485,7 @@ SYSFS_PMCSETUP(mmcra, SPRN_MMCRA);
 SYSFS_SPRSETUP(purr, SPRN_PURR);
 SYSFS_SPRSETUP(spurr, SPRN_SPURR);
 SYSFS_SPRSETUP(pir, SPRN_PIR);
+SYSFS_SPRSETUP(tscr, SPRN_TSCR);
 
 /*
   Lets only enable read for phyp resources and
@@ -495,6 +496,7 @@ static DEVICE_ATTR(mmcra, 0600, show_mmcra, store_mmcra);
 static DEVICE_ATTR(spurr, 0400, show_spurr, NULL);
 static DEVICE_ATTR(purr, 0400, show_purr, store_purr);
 static DEVICE_ATTR(pir, 0400, show_pir, NULL);
+static DEVICE_ATTR(tscr, 0600, show_tscr, store_tscr);
 
 /*
  * This is the system wide DSCR register default value. Any
@@ -785,6 +787,10 @@ static int register_cpu_online(unsigned int cpu)
 
 	if (cpu_has_feature(CPU_FTR_PPCAS_ARCH_V2))
 		device_create_file(s, &dev_attr_pir);
+
+	if (cpu_has_feature(CPU_FTR_ARCH_206) &&
+		!firmware_has_feature(FW_FEATURE_LPAR))
+		device_create_file(s, &dev_attr_tscr);
 #endif /* CONFIG_PPC64 */
 
 #ifdef CONFIG_PPC_FSL_BOOK3E
@@ -867,6 +873,10 @@ static int unregister_cpu_online(unsigned int cpu)
 
 	if (cpu_has_feature(CPU_FTR_PPCAS_ARCH_V2))
 		device_remove_file(s, &dev_attr_pir);
+
+	if (cpu_has_feature(CPU_FTR_ARCH_206) &&
+		!firmware_has_feature(FW_FEATURE_LPAR))
+		device_remove_file(s, &dev_attr_tscr);
 #endif /* CONFIG_PPC64 */
 
 #ifdef CONFIG_PPC_FSL_BOOK3E
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index fe6f3a285455..a32823dcd9a4 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -244,7 +244,7 @@ static u64 scan_dispatch_log(u64 stop_tb)
 void accumulate_stolen_time(void)
 {
 	u64 sst, ust;
-	u8 save_soft_enabled = local_paca->soft_enabled;
+	unsigned long save_irq_soft_mask = irq_soft_mask_return();
 	struct cpu_accounting_data *acct = &local_paca->accounting;
 
 	/* We are called early in the exception entry, before
@@ -253,7 +253,7 @@ void accumulate_stolen_time(void)
 	 * needs to reflect that so various debug stuff doesn't
 	 * complain
 	 */
-	local_paca->soft_enabled = 0;
+	irq_soft_mask_set(IRQS_DISABLED);
 
 	sst = scan_dispatch_log(acct->starttime_user);
 	ust = scan_dispatch_log(acct->starttime);
@@ -261,7 +261,7 @@ void accumulate_stolen_time(void)
 	acct->utime -= ust;
 	acct->steal_time += ust + sst;
 
-	local_paca->soft_enabled = save_soft_enabled;
+	irq_soft_mask_set(save_irq_soft_mask);
 }
 
 static inline u64 calculate_stolen_time(u64 stop_tb)
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index f3eb61be0d30..1e48d157196a 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -20,6 +20,7 @@
 #include <linux/sched/debug.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>
+#include <linux/pkeys.h>
 #include <linux/stddef.h>
 #include <linux/unistd.h>
 #include <linux/ptrace.h>
@@ -38,6 +39,8 @@
 #include <linux/ratelimit.h>
 #include <linux/context_tracking.h>
 #include <linux/smp.h>
+#include <linux/console.h>
+#include <linux/kmsg_dump.h>
 
 #include <asm/emulated_ops.h>
 #include <asm/pgtable.h>
@@ -142,6 +145,28 @@ static int die_owner = -1;
 static unsigned int die_nest_count;
 static int die_counter;
 
+extern void panic_flush_kmsg_start(void)
+{
+	/*
+	 * These are mostly taken from kernel/panic.c, but tries to do
+	 * relatively minimal work. Don't use delay functions (TB may
+	 * be broken), don't crash dump (need to set a firmware log),
+	 * don't run notifiers. We do want to get some information to
+	 * Linux console.
+	 */
+	console_verbose();
+	bust_spinlocks(1);
+}
+
+extern void panic_flush_kmsg_end(void)
+{
+	printk_safe_flush_on_panic();
+	kmsg_dump(KMSG_DUMP_PANIC);
+	bust_spinlocks(0);
+	debug_locks_off();
+	console_flush_on_panic();
+}
+
 static unsigned long oops_begin(struct pt_regs *regs)
 {
 	int cpu;
@@ -266,7 +291,9 @@ void user_single_step_siginfo(struct task_struct *tsk,
 	info->si_addr = (void __user *)regs->nip;
 }
 
-void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr)
+
+void _exception_pkey(int signr, struct pt_regs *regs, int code,
+		unsigned long addr, int key)
 {
 	siginfo_t info;
 	const char fmt32[] = KERN_INFO "%s[%d]: unhandled signal %d " \
@@ -289,13 +316,27 @@ void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr)
 		local_irq_enable();
 
 	current->thread.trap_nr = code;
+
+	/*
+	 * Save all the pkey registers AMR/IAMR/UAMOR. Eg: Core dumps need
+	 * to capture the content, if the task gets killed.
+	 */
+	thread_pkey_regs_save(&current->thread);
+
 	memset(&info, 0, sizeof(info));
 	info.si_signo = signr;
 	info.si_code = code;
 	info.si_addr = (void __user *) addr;
+	info.si_pkey = key;
+
 	force_sig_info(signr, &info, current);
 }
 
+void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr)
+{
+	_exception_pkey(signr, regs, code, addr, 0);
+}
+
 void system_reset_exception(struct pt_regs *regs)
 {
 	/*
@@ -337,7 +378,7 @@ void system_reset_exception(struct pt_regs *regs)
 	 * No debugger or crash dump registered, print logs then
 	 * panic.
 	 */
-	__die("System Reset", regs, SIGABRT);
+	die("System Reset", regs, SIGABRT);
 
 	mdelay(2*MSEC_PER_SEC); /* Wait a little while for others to print */
 	add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
@@ -917,7 +958,7 @@ void unknown_exception(struct pt_regs *regs)
 	printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
 	       regs->nip, regs->msr, regs->trap);
 
-	_exception(SIGTRAP, regs, 0, 0);
+	_exception(SIGTRAP, regs, TRAP_FIXME, 0);
 
 	exception_exit(prev_state);
 }
@@ -939,7 +980,7 @@ void instruction_breakpoint_exception(struct pt_regs *regs)
 
 void RunModeException(struct pt_regs *regs)
 {
-	_exception(SIGTRAP, regs, 0, 0);
+	_exception(SIGTRAP, regs, TRAP_FIXME, 0);
 }
 
 void single_step_exception(struct pt_regs *regs)
@@ -978,7 +1019,7 @@ static void emulate_single_step(struct pt_regs *regs)
 
 static inline int __parse_fpscr(unsigned long fpscr)
 {
-	int ret = 0;
+	int ret = FPE_FIXME;
 
 	/* Invalid operation */
 	if ((fpscr & FPSCR_VE) && (fpscr & FPSCR_VX))
@@ -1564,7 +1605,7 @@ void facility_unavailable_exception(struct pt_regs *regs)
 	u8 status;
 	bool hv;
 
-	hv = (regs->trap == 0xf80);
+	hv = (TRAP(regs) == 0xf80);
 	if (hv)
 		value = mfspr(SPRN_HFSCR);
 	else
@@ -1750,34 +1791,34 @@ static void handle_debug(struct pt_regs *regs, unsigned long debug_status)
 #ifdef CONFIG_PPC_ADV_DEBUG_DAC_RANGE
 		current->thread.debug.dbcr2 &= ~DBCR2_DAC12MODE;
 #endif
-		do_send_trap(regs, mfspr(SPRN_DAC1), debug_status, TRAP_HWBKPT,
+		do_send_trap(regs, mfspr(SPRN_DAC1), debug_status,
 			     5);
 		changed |= 0x01;
 	}  else if (debug_status & (DBSR_DAC2R | DBSR_DAC2W)) {
 		dbcr_dac(current) &= ~(DBCR_DAC2R | DBCR_DAC2W);
-		do_send_trap(regs, mfspr(SPRN_DAC2), debug_status, TRAP_HWBKPT,
+		do_send_trap(regs, mfspr(SPRN_DAC2), debug_status,
 			     6);
 		changed |= 0x01;
 	}  else if (debug_status & DBSR_IAC1) {
 		current->thread.debug.dbcr0 &= ~DBCR0_IAC1;
 		dbcr_iac_range(current) &= ~DBCR_IAC12MODE;
-		do_send_trap(regs, mfspr(SPRN_IAC1), debug_status, TRAP_HWBKPT,
+		do_send_trap(regs, mfspr(SPRN_IAC1), debug_status,
 			     1);
 		changed |= 0x01;
 	}  else if (debug_status & DBSR_IAC2) {
 		current->thread.debug.dbcr0 &= ~DBCR0_IAC2;
-		do_send_trap(regs, mfspr(SPRN_IAC2), debug_status, TRAP_HWBKPT,
+		do_send_trap(regs, mfspr(SPRN_IAC2), debug_status,
 			     2);
 		changed |= 0x01;
 	}  else if (debug_status & DBSR_IAC3) {
 		current->thread.debug.dbcr0 &= ~DBCR0_IAC3;
 		dbcr_iac_range(current) &= ~DBCR_IAC34MODE;
-		do_send_trap(regs, mfspr(SPRN_IAC3), debug_status, TRAP_HWBKPT,
+		do_send_trap(regs, mfspr(SPRN_IAC3), debug_status,
 			     3);
 		changed |= 0x01;
 	}  else if (debug_status & DBSR_IAC4) {
 		current->thread.debug.dbcr0 &= ~DBCR0_IAC4;
-		do_send_trap(regs, mfspr(SPRN_IAC4), debug_status, TRAP_HWBKPT,
+		do_send_trap(regs, mfspr(SPRN_IAC4), debug_status,
 			     4);
 		changed |= 0x01;
 	}
@@ -1929,7 +1970,7 @@ void SPEFloatingPointException(struct pt_regs *regs)
 	extern int do_spe_mathemu(struct pt_regs *regs);
 	unsigned long spefscr;
 	int fpexc_mode;
-	int code = 0;
+	int code = FPE_FIXME;
 	int err;
 
 	flush_spe_to_thread(current);
@@ -1998,7 +2039,7 @@ void SPEFloatingPointRoundException(struct pt_regs *regs)
 		printk(KERN_ERR "unrecognized spe instruction "
 		       "in %s at %lx\n", current->comm, regs->nip);
 	} else {
-		_exception(SIGFPE, regs, 0, regs->nip);
+		_exception(SIGFPE, regs, FPE_FIXME, regs->nip);
 		return;
 	}
 }
@@ -2113,13 +2154,13 @@ static int __init ppc_warn_emulated_init(void)
 	if (!dir)
 		return -ENOMEM;
 
-	d = debugfs_create_u32("do_warn", S_IRUGO | S_IWUSR, dir,
+	d = debugfs_create_u32("do_warn", 0644, dir,
 			       &ppc_warn_emulated);
 	if (!d)
 		goto fail;
 
 	for (i = 0; i < sizeof(ppc_emulated)/sizeof(*entries); i++) {
-		d = debugfs_create_u32(entries[i].name, S_IRUGO | S_IWUSR, dir,
+		d = debugfs_create_u32(entries[i].name, 0644, dir,
 				       (u32 *)&entries[i].val.counter);
 		if (!d)
 			goto fail;
diff --git a/arch/powerpc/kernel/vdso64/gettimeofday.S b/arch/powerpc/kernel/vdso64/gettimeofday.S
index 382021324883..c002adcc694c 100644
--- a/arch/powerpc/kernel/vdso64/gettimeofday.S
+++ b/arch/powerpc/kernel/vdso64/gettimeofday.S
@@ -64,6 +64,12 @@ V_FUNCTION_BEGIN(__kernel_clock_gettime)
 	cmpwi	cr0,r3,CLOCK_REALTIME
 	cmpwi	cr1,r3,CLOCK_MONOTONIC
 	cror	cr0*4+eq,cr0*4+eq,cr1*4+eq
+
+	cmpwi	cr5,r3,CLOCK_REALTIME_COARSE
+	cmpwi	cr6,r3,CLOCK_MONOTONIC_COARSE
+	cror	cr5*4+eq,cr5*4+eq,cr6*4+eq
+
+	cror	cr0*4+eq,cr0*4+eq,cr5*4+eq
 	bne	cr0,99f
 
 	mflr	r12			/* r12 saves lr */
@@ -72,6 +78,7 @@ V_FUNCTION_BEGIN(__kernel_clock_gettime)
 	bl	V_LOCAL_FUNC(__get_datapage)	/* get data page */
 	lis	r7,NSEC_PER_SEC@h	/* want nanoseconds */
 	ori	r7,r7,NSEC_PER_SEC@l
+	beq	cr5,70f
 50:	bl	V_LOCAL_FUNC(__do_get_tspec)	/* get time from tb & kernel */
 	bne	cr1,80f			/* if not monotonic, all done */
 
@@ -97,19 +104,57 @@ V_FUNCTION_BEGIN(__kernel_clock_gettime)
 	ld	r0,CFG_TB_UPDATE_COUNT(r3)
         cmpld   cr0,r0,r8		/* check if updated */
 	bne-	50b
+	b	78f
 
-	/* Add wall->monotonic offset and check for overflow or underflow.
+	/*
+	 * For coarse clocks we get data directly from the vdso data page, so
+	 * we don't need to call __do_get_tspec, but we still need to do the
+	 * counter trick.
 	 */
-	add	r4,r4,r6
-	add	r5,r5,r9
-	cmpd	cr0,r5,r7
-	cmpdi	cr1,r5,0
-	blt	1f
-	subf	r5,r7,r5
-	addi	r4,r4,1
-1:	bge	cr1,80f
-	addi	r4,r4,-1
-	add	r5,r5,r7
+70:	ld      r8,CFG_TB_UPDATE_COUNT(r3)
+	andi.   r0,r8,1                 /* pending update ? loop */
+	bne-    70b
+	add     r3,r3,r0		/* r0 is already 0 */
+
+	/*
+	 * CLOCK_REALTIME_COARSE, below values are needed for MONOTONIC_COARSE
+	 * too
+	 */
+	ld      r4,STAMP_XTIME+TSPC64_TV_SEC(r3)
+	ld      r5,STAMP_XTIME+TSPC64_TV_NSEC(r3)
+	bne     cr6,75f
+
+	/* CLOCK_MONOTONIC_COARSE */
+	lwa     r6,WTOM_CLOCK_SEC(r3)
+	lwa     r9,WTOM_CLOCK_NSEC(r3)
+
+	/* check if counter has updated */
+	or      r0,r6,r9
+75:	or	r0,r0,r4
+	or	r0,r0,r5
+	xor     r0,r0,r0
+	add     r3,r3,r0
+	ld      r0,CFG_TB_UPDATE_COUNT(r3)
+	cmpld   cr0,r0,r8               /* check if updated */
+	bne-    70b
+
+	/* Counter has not updated, so continue calculating proper values for
+	 * sec and nsec if monotonic coarse, or just return with the proper
+	 * values for realtime.
+	 */
+	bne     cr6,80f
+
+	/* Add wall->monotonic offset and check for overflow or underflow */
+78:	add     r4,r4,r6
+	add     r5,r5,r9
+	cmpd    cr0,r5,r7
+	cmpdi   cr1,r5,0
+	blt     79f
+	subf    r5,r7,r5
+	addi    r4,r4,1
+79:	bge     cr1,80f
+	addi    r4,r4,-1
+	add     r5,r5,r7
 
 80:	std	r4,TSPC64_TV_SEC(r11)
 	std	r5,TSPC64_TV_NSEC(r11)
diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index 307843d23682..c8af90ff49f0 100644
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -273,6 +273,7 @@ SECTIONS
 #ifdef CONFIG_PPC32
 	.data : AT(ADDR(.data) - LOAD_OFFSET) {
 		DATA_DATA
+		*(.data.rel*)
 		*(.sdata)
 		*(.sdata2)
 		*(.got.plt) *(.got)
@@ -287,7 +288,9 @@ SECTIONS
 	}
 
 	.opd : AT(ADDR(.opd) - LOAD_OFFSET) {
+		__start_opd = .;
 		*(.opd)
+		__end_opd = .;
 	}
 
 	. = ALIGN(256);
diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c
index 87da80ccced1..6256dc3b0087 100644
--- a/arch/powerpc/kernel/watchdog.c
+++ b/arch/powerpc/kernel/watchdog.c
@@ -6,6 +6,9 @@
  *
  * This uses code from arch/sparc/kernel/nmi.c and kernel/watchdog.c
  */
+
+#define pr_fmt(fmt) "watchdog: " fmt
+
 #include <linux/kernel.h>
 #include <linux/param.h>
 #include <linux/init.h>
@@ -26,15 +29,45 @@
 #include <asm/paca.h>
 
 /*
- * The watchdog has a simple timer that runs on each CPU, once per timer
- * period. This is the heartbeat.
+ * The powerpc watchdog ensures that each CPU is able to service timers.
+ * The watchdog sets up a simple timer on each CPU to run once per timer
+ * period, and updates a per-cpu timestamp and a "pending" cpumask. This is
+ * the heartbeat.
+ *
+ * Then there are two systems to check that the heartbeat is still running.
+ * The local soft-NMI, and the SMP checker.
+ *
+ * The soft-NMI checker can detect lockups on the local CPU. When interrupts
+ * are disabled with local_irq_disable(), platforms that use soft-masking
+ * can leave hardware interrupts enabled and handle them with a masked
+ * interrupt handler. The masked handler can send the timer interrupt to the
+ * watchdog's soft_nmi_interrupt(), which appears to Linux as an NMI
+ * interrupt, and can be used to detect CPUs stuck with IRQs disabled.
+ *
+ * The soft-NMI checker will compare the heartbeat timestamp for this CPU
+ * with the current time, and take action if the difference exceeds the
+ * watchdog threshold.
+ *
+ * The limitation of the soft-NMI watchdog is that it does not work when
+ * interrupts are hard disabled or otherwise not being serviced. This is
+ * solved by also having a SMP watchdog where all CPUs check all other
+ * CPUs heartbeat.
  *
- * Then there are checks to see if the heartbeat has not triggered on a CPU
- * for the panic timeout period. Currently the watchdog only supports an
- * SMP check, so the heartbeat only turns on when we have 2 or more CPUs.
+ * The SMP checker can detect lockups on other CPUs. A gobal "pending"
+ * cpumask is kept, containing all CPUs which enable the watchdog. Each
+ * CPU clears their pending bit in their heartbeat timer. When the bitmask
+ * becomes empty, the last CPU to clear its pending bit updates a global
+ * timestamp and refills the pending bitmask.
  *
- * This is not an NMI watchdog, but Linux uses that name for a generic
- * watchdog in some cases, so NMI gets used in some places.
+ * In the heartbeat timer, if any CPU notices that the global timestamp has
+ * not been updated for a period exceeding the watchdog threshold, then it
+ * means the CPU(s) with their bit still set in the pending mask have had
+ * their heartbeat stop, and action is taken.
+ *
+ * Some platforms implement true NMI IPIs, which can by used by the SMP
+ * watchdog to detect an unresponsive CPU and pull it out of its stuck
+ * state with the NMI IPI, to get crash/debug data from it. This way the
+ * SMP watchdog can detect hardware interrupts off lockups.
  */
 
 static cpumask_t wd_cpus_enabled __read_mostly;
@@ -47,19 +80,7 @@ static u64 wd_timer_period_ms __read_mostly;  /* interval between heartbeat */
 static DEFINE_PER_CPU(struct timer_list, wd_timer);
 static DEFINE_PER_CPU(u64, wd_timer_tb);
 
-/*
- * These are for the SMP checker. CPUs clear their pending bit in their
- * heartbeat. If the bitmask becomes empty, the time is noted and the
- * bitmask is refilled.
- *
- * All CPUs clear their bit in the pending mask every timer period.
- * Once all have cleared, the time is noted and the bits are reset.
- * If the time since all clear was greater than the panic timeout,
- * we can panic with the list of stuck CPUs.
- *
- * This will work best with NMI IPIs for crash code so the stuck CPUs
- * can be pulled out to get their backtraces.
- */
+/* SMP checker bits */
 static unsigned long __wd_smp_lock;
 static cpumask_t wd_smp_cpus_pending;
 static cpumask_t wd_smp_cpus_stuck;
@@ -90,7 +111,7 @@ static inline void wd_smp_unlock(unsigned long *flags)
 
 static void wd_lockup_ipi(struct pt_regs *regs)
 {
-	pr_emerg("Watchdog CPU:%d Hard LOCKUP\n", raw_smp_processor_id());
+	pr_emerg("CPU %d Hard LOCKUP\n", raw_smp_processor_id());
 	print_modules();
 	print_irqtrace_events(current);
 	if (regs)
@@ -131,8 +152,8 @@ static void watchdog_smp_panic(int cpu, u64 tb)
 	if (cpumask_weight(&wd_smp_cpus_pending) == 0)
 		goto out;
 
-	pr_emerg("Watchdog CPU:%d detected Hard LOCKUP other CPUS:%*pbl\n",
-			cpu, cpumask_pr_args(&wd_smp_cpus_pending));
+	pr_emerg("CPU %d detected hard LOCKUP on other CPUs %*pbl\n",
+		 cpu, cpumask_pr_args(&wd_smp_cpus_pending));
 
 	if (!sysctl_hardlockup_all_cpu_backtrace) {
 		/*
@@ -175,7 +196,7 @@ static void wd_smp_clear_cpu_pending(int cpu, u64 tb)
 		if (unlikely(cpumask_test_cpu(cpu, &wd_smp_cpus_stuck))) {
 			unsigned long flags;
 
-			pr_emerg("Watchdog CPU:%d became unstuck\n", cpu);
+			pr_emerg("CPU %d became unstuck\n", cpu);
 			wd_smp_lock(&flags);
 			cpumask_clear_cpu(cpu, &wd_smp_cpus_stuck);
 			wd_smp_unlock(&flags);
@@ -233,13 +254,10 @@ void soft_nmi_interrupt(struct pt_regs *regs)
 		}
 		set_cpu_stuck(cpu, tb);
 
-		pr_emerg("Watchdog CPU:%d Hard LOCKUP\n", cpu);
+		pr_emerg("CPU %d self-detected hard LOCKUP @ %pS\n", cpu, (void *)regs->nip);
 		print_modules();
 		print_irqtrace_events(current);
-		if (regs)
-			show_regs(regs);
-		else
-			dump_stack();
+		show_regs(regs);
 
 		wd_smp_unlock(&flags);
 
@@ -388,30 +406,8 @@ int __init watchdog_nmi_probe(void)
 					"powerpc/watchdog:online",
 					start_wd_on_cpu, stop_wd_on_cpu);
 	if (err < 0) {
-		pr_warn("Watchdog could not be initialized");
+		pr_warn("could not be initialized");
 		return err;
 	}
 	return 0;
 }
-
-static void handle_backtrace_ipi(struct pt_regs *regs)
-{
-	nmi_cpu_backtrace(regs);
-}
-
-static void raise_backtrace_ipi(cpumask_t *mask)
-{
-	unsigned int cpu;
-
-	for_each_cpu(cpu, mask) {
-		if (cpu == smp_processor_id())
-			handle_backtrace_ipi(NULL);
-		else
-			smp_send_nmi_ipi(cpu, handle_backtrace_ipi, 1000000);
-	}
-}
-
-void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self)
-{
-	nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace_ipi);
-}
diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig
index b12b8eb39c29..68a0e9d5b440 100644
--- a/arch/powerpc/kvm/Kconfig
+++ b/arch/powerpc/kvm/Kconfig
@@ -22,6 +22,7 @@ config KVM
 	select PREEMPT_NOTIFIERS
 	select ANON_INODES
 	select HAVE_KVM_EVENTFD
+	select HAVE_KVM_VCPU_ASYNC_IOCTL
 	select SRCU
 	select KVM_VFIO
 	select IRQ_BYPASS_MANAGER
@@ -68,7 +69,7 @@ config KVM_BOOK3S_64
 	select KVM_BOOK3S_64_HANDLER
 	select KVM
 	select KVM_BOOK3S_PR_POSSIBLE if !KVM_BOOK3S_HV_POSSIBLE
-	select SPAPR_TCE_IOMMU if IOMMU_SUPPORT && (PPC_SERIES || PPC_POWERNV)
+	select SPAPR_TCE_IOMMU if IOMMU_SUPPORT && (PPC_PSERIES || PPC_POWERNV)
 	---help---
 	  Support running unmodified book3s_64 and book3s_32 guest kernels
 	  in virtual machines on book3s_64 host processors.
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index 72d977e30952..234531d1bee1 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -484,19 +484,33 @@ void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu)
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
-	return vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
+	int ret;
+
+	vcpu_load(vcpu);
+	ret = vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
+	vcpu_put(vcpu);
+
+	return ret;
 }
 
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
-	return vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
+	int ret;
+
+	vcpu_load(vcpu);
+	ret = vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
+	vcpu_put(vcpu);
+
+	return ret;
 }
 
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
 
+	vcpu_load(vcpu);
+
 	regs->pc = kvmppc_get_pc(vcpu);
 	regs->cr = kvmppc_get_cr(vcpu);
 	regs->ctr = kvmppc_get_ctr(vcpu);
@@ -518,6 +532,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -525,6 +540,8 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
 
+	vcpu_load(vcpu);
+
 	kvmppc_set_pc(vcpu, regs->pc);
 	kvmppc_set_cr(vcpu, regs->cr);
 	kvmppc_set_ctr(vcpu, regs->ctr);
@@ -545,6 +562,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -737,7 +755,9 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
+	vcpu_load(vcpu);
 	vcpu->guest_debug = dbg->control;
+	vcpu_put(vcpu);
 	return 0;
 }
 
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
index b73dbc9e797d..ef243fed2f2b 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -1269,6 +1269,11 @@ static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
 		/* Nothing to do */
 		goto out;
 
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		rpte = be64_to_cpu(hptep[1]);
+		vpte = hpte_new_to_old_v(vpte, rpte);
+	}
+
 	/* Unmap */
 	rev = &old->rev[idx];
 	guest_rpte = rev->guest_rpte;
@@ -1298,7 +1303,6 @@ static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
 
 	/* Reload PTE after unmap */
 	vpte = be64_to_cpu(hptep[0]);
-
 	BUG_ON(vpte & HPTE_V_VALID);
 	BUG_ON(!(vpte & HPTE_V_ABSENT));
 
@@ -1307,6 +1311,12 @@ static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
 		goto out;
 
 	rpte = be64_to_cpu(hptep[1]);
+
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		vpte = hpte_new_to_old_v(vpte, rpte);
+		rpte = hpte_new_to_old_r(rpte);
+	}
+
 	pshift = kvmppc_hpte_base_page_shift(vpte, rpte);
 	avpn = HPTE_V_AVPN_VAL(vpte) & ~(((1ul << pshift) - 1) >> 23);
 	pteg = idx / HPTES_PER_GROUP;
@@ -1337,17 +1347,17 @@ static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
 	}
 
 	new_pteg = hash & new_hash_mask;
-	if (vpte & HPTE_V_SECONDARY) {
-		BUG_ON(~pteg != (hash & old_hash_mask));
-		new_pteg = ~new_pteg;
-	} else {
-		BUG_ON(pteg != (hash & old_hash_mask));
-	}
+	if (vpte & HPTE_V_SECONDARY)
+		new_pteg = ~hash & new_hash_mask;
 
 	new_idx = new_pteg * HPTES_PER_GROUP + (idx % HPTES_PER_GROUP);
 	new_hptep = (__be64 *)(new->virt + (new_idx << 4));
 
 	replace_vpte = be64_to_cpu(new_hptep[0]);
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		unsigned long replace_rpte = be64_to_cpu(new_hptep[1]);
+		replace_vpte = hpte_new_to_old_v(replace_vpte, replace_rpte);
+	}
 
 	if (replace_vpte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
 		BUG_ON(new->order >= old->order);
@@ -1363,6 +1373,11 @@ static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
 		/* Discard the previous HPTE */
 	}
 
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		rpte = hpte_old_to_new_r(vpte, rpte);
+		vpte = hpte_old_to_new_v(vpte);
+	}
+
 	new_hptep[1] = cpu_to_be64(rpte);
 	new->rev[new_idx].guest_rpte = guest_rpte;
 	/* No need for a barrier, since new HPT isn't active */
@@ -1380,12 +1395,6 @@ static int resize_hpt_rehash(struct kvm_resize_hpt *resize)
 	unsigned  long i;
 	int rc;
 
-	/*
-	 * resize_hpt_rehash_hpte() doesn't handle the new-format HPTEs
-	 * that POWER9 uses, and could well hit a BUG_ON on POWER9.
-	 */
-	if (cpu_has_feature(CPU_FTR_ARCH_300))
-		return -EIO;
 	for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) {
 		rc = resize_hpt_rehash_hpte(resize, i);
 		if (rc != 0)
@@ -1416,6 +1425,9 @@ static void resize_hpt_pivot(struct kvm_resize_hpt *resize)
 
 	synchronize_srcu_expedited(&kvm->srcu);
 
+	if (cpu_has_feature(CPU_FTR_ARCH_300))
+		kvmppc_setup_partition_table(kvm);
+
 	resize_hpt_debug(resize, "resize_hpt_pivot() done\n");
 }
 
diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index 58618f644c56..5d9bafe9a371 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -157,6 +157,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
 	asm volatile("ptesync": : :"memory");
 	asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
 		     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
+	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
+		asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
+			     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
 	asm volatile("ptesync": : :"memory");
 }
 
@@ -195,6 +198,12 @@ static void kvmppc_pte_free(pte_t *ptep)
 	kmem_cache_free(kvm_pte_cache, ptep);
 }
 
+/* Like pmd_huge() and pmd_large(), but works regardless of config options */
+static inline int pmd_is_leaf(pmd_t pmd)
+{
+	return !!(pmd_val(pmd) & _PAGE_PTE);
+}
+
 static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 			     unsigned int level, unsigned long mmu_seq)
 {
@@ -219,7 +228,7 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	else
 		new_pmd = pmd_alloc_one(kvm->mm, gpa);
 
-	if (level == 0 && !(pmd && pmd_present(*pmd)))
+	if (level == 0 && !(pmd && pmd_present(*pmd) && !pmd_is_leaf(*pmd)))
 		new_ptep = kvmppc_pte_alloc();
 
 	/* Check if we might have been invalidated; let the guest retry if so */
@@ -244,12 +253,30 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 		new_pmd = NULL;
 	}
 	pmd = pmd_offset(pud, gpa);
-	if (pmd_large(*pmd)) {
-		/* Someone else has instantiated a large page here; retry */
-		ret = -EAGAIN;
-		goto out_unlock;
-	}
-	if (level == 1 && !pmd_none(*pmd)) {
+	if (pmd_is_leaf(*pmd)) {
+		unsigned long lgpa = gpa & PMD_MASK;
+
+		/*
+		 * If we raced with another CPU which has just put
+		 * a 2MB pte in after we saw a pte page, try again.
+		 */
+		if (level == 0 && !new_ptep) {
+			ret = -EAGAIN;
+			goto out_unlock;
+		}
+		/* Valid 2MB page here already, remove it */
+		old = kvmppc_radix_update_pte(kvm, pmdp_ptep(pmd),
+					      ~0UL, 0, lgpa, PMD_SHIFT);
+		kvmppc_radix_tlbie_page(kvm, lgpa, PMD_SHIFT);
+		if (old & _PAGE_DIRTY) {
+			unsigned long gfn = lgpa >> PAGE_SHIFT;
+			struct kvm_memory_slot *memslot;
+			memslot = gfn_to_memslot(kvm, gfn);
+			if (memslot && memslot->dirty_bitmap)
+				kvmppc_update_dirty_map(memslot,
+							gfn, PMD_SIZE);
+		}
+	} else if (level == 1 && !pmd_none(*pmd)) {
 		/*
 		 * There's a page table page here, but we wanted
 		 * to install a large page.  Tell the caller and let
@@ -412,28 +439,24 @@ int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	} else {
 		page = pages[0];
 		pfn = page_to_pfn(page);
-		if (PageHuge(page)) {
-			page = compound_head(page);
-			pte_size <<= compound_order(page);
+		if (PageCompound(page)) {
+			pte_size <<= compound_order(compound_head(page));
 			/* See if we can insert a 2MB large-page PTE here */
 			if (pte_size >= PMD_SIZE &&
-			    (gpa & PMD_MASK & PAGE_MASK) ==
-			    (hva & PMD_MASK & PAGE_MASK)) {
+			    (gpa & (PMD_SIZE - PAGE_SIZE)) ==
+			    (hva & (PMD_SIZE - PAGE_SIZE))) {
 				level = 1;
 				pfn &= ~((PMD_SIZE >> PAGE_SHIFT) - 1);
 			}
 		}
 		/* See if we can provide write access */
 		if (writing) {
-			/*
-			 * We assume gup_fast has set dirty on the host PTE.
-			 */
 			pgflags |= _PAGE_WRITE;
 		} else {
 			local_irq_save(flags);
 			ptep = find_current_mm_pte(current->mm->pgd,
 						   hva, NULL, NULL);
-			if (ptep && pte_write(*ptep) && pte_dirty(*ptep))
+			if (ptep && pte_write(*ptep))
 				pgflags |= _PAGE_WRITE;
 			local_irq_restore(flags);
 		}
@@ -459,18 +482,15 @@ int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 		pte = pfn_pte(pfn, __pgprot(pgflags));
 		ret = kvmppc_create_pte(kvm, pte, gpa, level, mmu_seq);
 	}
-	if (ret == 0 || ret == -EAGAIN)
-		ret = RESUME_GUEST;
 
 	if (page) {
-		/*
-		 * We drop pages[0] here, not page because page might
-		 * have been set to the head page of a compound, but
-		 * we have to drop the reference on the correct tail
-		 * page to match the get inside gup()
-		 */
-		put_page(pages[0]);
+		if (!ret && (pgflags & _PAGE_WRITE))
+			set_page_dirty_lock(page);
+		put_page(page);
 	}
+
+	if (ret == 0 || ret == -EAGAIN)
+		ret = RESUME_GUEST;
 	return ret;
 }
 
@@ -573,7 +593,7 @@ long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
 		j = i + 1;
 		if (npages) {
 			set_dirty_bits(map, i, npages);
-			i = j + npages;
+			j = i + npages;
 		}
 	}
 	return 0;
@@ -644,7 +664,7 @@ void kvmppc_free_radix(struct kvm *kvm)
 				continue;
 			pmd = pmd_offset(pud, 0);
 			for (im = 0; im < PTRS_PER_PMD; ++im, ++pmd) {
-				if (pmd_huge(*pmd)) {
+				if (pmd_is_leaf(*pmd)) {
 					pmd_clear(pmd);
 					continue;
 				}
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 2d46037ce936..9cb9448163c4 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -93,10 +93,10 @@
 static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1);
 
 static int dynamic_mt_modes = 6;
-module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR);
+module_param(dynamic_mt_modes, int, 0644);
 MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)");
 static int target_smt_mode;
-module_param(target_smt_mode, int, S_IRUGO | S_IWUSR);
+module_param(target_smt_mode, int, 0644);
 MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)");
 
 static bool indep_threads_mode = true;
@@ -109,15 +109,16 @@ static struct kernel_param_ops module_param_ops = {
 	.get = param_get_int,
 };
 
-module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass,
-							S_IRUGO | S_IWUSR);
+module_param_cb(kvm_irq_bypass, &module_param_ops, &kvm_irq_bypass, 0644);
 MODULE_PARM_DESC(kvm_irq_bypass, "Bypass passthrough interrupt optimization");
 
-module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect,
-							S_IRUGO | S_IWUSR);
+module_param_cb(h_ipi_redirect, &module_param_ops, &h_ipi_redirect, 0644);
 MODULE_PARM_DESC(h_ipi_redirect, "Redirect H_IPI wakeup to a free host core");
 #endif
 
+/* If set, the threads on each CPU core have to be in the same MMU mode */
+static bool no_mixing_hpt_and_radix;
+
 static void kvmppc_end_cede(struct kvm_vcpu *vcpu);
 static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu);
 
@@ -1005,8 +1006,6 @@ static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
 	struct kvm *kvm = vcpu->kvm;
 	struct kvm_vcpu *tvcpu;
 
-	if (!cpu_has_feature(CPU_FTR_ARCH_300))
-		return EMULATE_FAIL;
 	if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &inst) != EMULATE_DONE)
 		return RESUME_GUEST;
 	if (get_op(inst) != 31)
@@ -1056,6 +1055,7 @@ static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu)
 	return RESUME_GUEST;
 }
 
+/* Called with vcpu->arch.vcore->lock held */
 static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
 				 struct task_struct *tsk)
 {
@@ -1176,7 +1176,10 @@ static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
 				swab32(vcpu->arch.emul_inst) :
 				vcpu->arch.emul_inst;
 		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) {
+			/* Need vcore unlocked to call kvmppc_get_last_inst */
+			spin_unlock(&vcpu->arch.vcore->lock);
 			r = kvmppc_emulate_debug_inst(run, vcpu);
+			spin_lock(&vcpu->arch.vcore->lock);
 		} else {
 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
 			r = RESUME_GUEST;
@@ -1191,8 +1194,13 @@ static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu,
 	 */
 	case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
 		r = EMULATE_FAIL;
-		if ((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG)
+		if (((vcpu->arch.hfscr >> 56) == FSCR_MSGP_LG) &&
+		    cpu_has_feature(CPU_FTR_ARCH_300)) {
+			/* Need vcore unlocked to call kvmppc_get_last_inst */
+			spin_unlock(&vcpu->arch.vcore->lock);
 			r = kvmppc_emulate_doorbell_instr(vcpu);
+			spin_lock(&vcpu->arch.vcore->lock);
+		}
 		if (r == EMULATE_FAIL) {
 			kvmppc_core_queue_program(vcpu, SRR1_PROGILL);
 			r = RESUME_GUEST;
@@ -1497,6 +1505,10 @@ static int kvmppc_get_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 	case KVM_REG_PPC_ARCH_COMPAT:
 		*val = get_reg_val(id, vcpu->arch.vcore->arch_compat);
 		break;
+	case KVM_REG_PPC_DEC_EXPIRY:
+		*val = get_reg_val(id, vcpu->arch.dec_expires +
+				   vcpu->arch.vcore->tb_offset);
+		break;
 	default:
 		r = -EINVAL;
 		break;
@@ -1724,6 +1736,10 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id,
 	case KVM_REG_PPC_ARCH_COMPAT:
 		r = kvmppc_set_arch_compat(vcpu, set_reg_val(id, *val));
 		break;
+	case KVM_REG_PPC_DEC_EXPIRY:
+		vcpu->arch.dec_expires = set_reg_val(id, *val) -
+			vcpu->arch.vcore->tb_offset;
+		break;
 	default:
 		r = -EINVAL;
 		break;
@@ -2378,8 +2394,8 @@ static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc)
 static bool subcore_config_ok(int n_subcores, int n_threads)
 {
 	/*
-	 * POWER9 "SMT4" cores are permanently in what is effectively a 4-way split-core
-	 * mode, with one thread per subcore.
+	 * POWER9 "SMT4" cores are permanently in what is effectively a 4-way
+	 * split-core mode, with one thread per subcore.
 	 */
 	if (cpu_has_feature(CPU_FTR_ARCH_300))
 		return n_subcores <= 4 && n_threads == 1;
@@ -2415,8 +2431,8 @@ static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip)
 	if (!cpu_has_feature(CPU_FTR_ARCH_207S))
 		return false;
 
-	/* POWER9 currently requires all threads to be in the same MMU mode */
-	if (cpu_has_feature(CPU_FTR_ARCH_300) &&
+	/* Some POWER9 chips require all threads to be in the same MMU mode */
+	if (no_mixing_hpt_and_radix &&
 	    kvm_is_radix(vc->kvm) != kvm_is_radix(cip->vc[0]->kvm))
 		return false;
 
@@ -2679,9 +2695,11 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	 * threads are offline.  Also check if the number of threads in this
 	 * guest are greater than the current system threads per guest.
 	 * On POWER9, we need to be not in independent-threads mode if
-	 * this is a HPT guest on a radix host.
+	 * this is a HPT guest on a radix host machine where the
+	 * CPU threads may not be in different MMU modes.
 	 */
-	hpt_on_radix = radix_enabled() && !kvm_is_radix(vc->kvm);
+	hpt_on_radix = no_mixing_hpt_and_radix && radix_enabled() &&
+		!kvm_is_radix(vc->kvm);
 	if (((controlled_threads > 1) &&
 	     ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) ||
 	    (hpt_on_radix && vc->kvm->arch.threads_indep)) {
@@ -2831,7 +2849,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 		 */
 		if (!thr0_done)
 			kvmppc_start_thread(NULL, pvc);
-		thr += pvc->num_threads;
 	}
 
 	/*
@@ -2868,7 +2885,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	 */
 	trace_hardirqs_on();
 
-	guest_enter();
+	guest_enter_irqoff();
 
 	srcu_idx = srcu_read_lock(&vc->kvm->srcu);
 
@@ -2876,8 +2893,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 
 	srcu_read_unlock(&vc->kvm->srcu, srcu_idx);
 
-	guest_exit();
-
 	trace_hardirqs_off();
 	set_irq_happened(trap);
 
@@ -2920,6 +2935,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	kvmppc_set_host_core(pcpu);
 
 	local_irq_enable();
+	guest_exit();
 
 	/* Let secondaries go back to the offline loop */
 	for (i = 0; i < controlled_threads; ++i) {
@@ -2934,13 +2950,14 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
 	/* make sure updates to secondary vcpu structs are visible now */
 	smp_mb();
 
+	preempt_enable();
+
 	for (sub = 0; sub < core_info.n_subcores; ++sub) {
 		pvc = core_info.vc[sub];
 		post_guest_process(pvc, pvc == vc);
 	}
 
 	spin_lock(&vc->lock);
-	preempt_enable();
 
  out:
 	vc->vcore_state = VCORE_INACTIVE;
@@ -2987,7 +3004,7 @@ static inline bool xive_interrupt_pending(struct kvm_vcpu *vcpu)
 {
 	if (!xive_enabled())
 		return false;
-	return vcpu->arch.xive_saved_state.pipr <
+	return vcpu->arch.irq_pending || vcpu->arch.xive_saved_state.pipr <
 		vcpu->arch.xive_saved_state.cppr;
 }
 #else
@@ -3176,17 +3193,8 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
 	 * this thread straight away and have it join in.
 	 */
 	if (!signal_pending(current)) {
-		if (vc->vcore_state == VCORE_PIGGYBACK) {
-			if (spin_trylock(&vc->lock)) {
-				if (vc->vcore_state == VCORE_RUNNING &&
-				    !VCORE_IS_EXITING(vc)) {
-					kvmppc_create_dtl_entry(vcpu, vc);
-					kvmppc_start_thread(vcpu, vc);
-					trace_kvm_guest_enter(vcpu);
-				}
-				spin_unlock(&vc->lock);
-			}
-		} else if (vc->vcore_state == VCORE_RUNNING &&
+		if ((vc->vcore_state == VCORE_PIGGYBACK ||
+		     vc->vcore_state == VCORE_RUNNING) &&
 			   !VCORE_IS_EXITING(vc)) {
 			kvmppc_create_dtl_entry(vcpu, vc);
 			kvmppc_start_thread(vcpu, vc);
@@ -3647,15 +3655,17 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu)
 		goto up_out;
 
 	psize = vma_kernel_pagesize(vma);
-	porder = __ilog2(psize);
 
 	up_read(&current->mm->mmap_sem);
 
 	/* We can handle 4k, 64k or 16M pages in the VRMA */
-	err = -EINVAL;
-	if (!(psize == 0x1000 || psize == 0x10000 ||
-	      psize == 0x1000000))
-		goto out_srcu;
+	if (psize >= 0x1000000)
+		psize = 0x1000000;
+	else if (psize >= 0x10000)
+		psize = 0x10000;
+	else
+		psize = 0x1000;
+	porder = __ilog2(psize);
 
 	senc = slb_pgsize_encoding(psize);
 	kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
@@ -4448,6 +4458,19 @@ static int kvmppc_book3s_init_hv(void)
 
 	if (kvmppc_radix_possible())
 		r = kvmppc_radix_init();
+
+	/*
+	 * POWER9 chips before version 2.02 can't have some threads in
+	 * HPT mode and some in radix mode on the same core.
+	 */
+	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
+		unsigned int pvr = mfspr(SPRN_PVR);
+		if ((pvr >> 16) == PVR_POWER9 &&
+		    (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) ||
+		     ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101)))
+			no_mixing_hpt_and_radix = true;
+	}
+
 	return r;
 }
 
diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c
index c356f9a40b24..b11043b23c18 100644
--- a/arch/powerpc/kvm/book3s_hv_ras.c
+++ b/arch/powerpc/kvm/book3s_hv_ras.c
@@ -87,8 +87,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu)
 				   DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI);
 		}
 		if (dsisr & DSISR_MC_TLB_MULTI) {
-			if (cur_cpu_spec && cur_cpu_spec->flush_tlb)
-				cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID);
+			tlbiel_all_lpid(vcpu->kvm->arch.radix);
 			dsisr &= ~DSISR_MC_TLB_MULTI;
 		}
 		/* Any other errors we don't understand? */
@@ -105,8 +104,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu)
 		reload_slb(vcpu);
 		break;
 	case SRR1_MC_IFETCH_TLBMULTI:
-		if (cur_cpu_spec && cur_cpu_spec->flush_tlb)
-			cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID);
+		tlbiel_all_lpid(vcpu->kvm->arch.radix);
 		break;
 	default:
 		handled = 0;
@@ -268,17 +266,19 @@ static void kvmppc_tb_resync_done(void)
  *   secondary threads to proceed.
  * - All secondary threads will eventually call opal hmi handler on
  *   their exit path.
+ *
+ * Returns 1 if the timebase offset should be applied, 0 if not.
  */
 
 long kvmppc_realmode_hmi_handler(void)
 {
-	int ptid = local_paca->kvm_hstate.ptid;
 	bool resync_req;
 
-	/* This is only called on primary thread. */
-	BUG_ON(ptid != 0);
 	__this_cpu_inc(irq_stat.hmi_exceptions);
 
+	if (hmi_handle_debugtrig(NULL) >= 0)
+		return 1;
+
 	/*
 	 * By now primary thread has already completed guest->host
 	 * partition switch but haven't signaled secondaries yet.
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 26c11f678fbf..e1c083fbe434 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -42,7 +42,7 @@ static void *real_vmalloc_addr(void *x)
 }
 
 /* Return 1 if we need to do a global tlbie, 0 if we can use tlbiel */
-static int global_invalidates(struct kvm *kvm, unsigned long flags)
+static int global_invalidates(struct kvm *kvm)
 {
 	int global;
 	int cpu;
@@ -473,6 +473,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
 			trace_tlbie(kvm->arch.lpid, 0, rbvalues[i],
 				kvm->arch.lpid, 0, 0, 0);
 		}
+
+		if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+			/*
+			 * Need the extra ptesync to make sure we don't
+			 * re-order the tlbie
+			 */
+			asm volatile("ptesync": : :"memory");
+			asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
+				     "r" (rbvalues[0]), "r" (kvm->arch.lpid));
+		}
+
 		asm volatile("eieio; tlbsync; ptesync" : : : "memory");
 		kvm->arch.tlbie_lock = 0;
 	} else {
@@ -522,7 +533,7 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
 	if (v & HPTE_V_VALID) {
 		hpte[0] &= ~cpu_to_be64(HPTE_V_VALID);
 		rb = compute_tlbie_rb(v, pte_r, pte_index);
-		do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true);
+		do_tlbies(kvm, &rb, 1, global_invalidates(kvm), true);
 		/*
 		 * The reference (R) and change (C) bits in a HPT
 		 * entry can be set by hardware at any time up until
@@ -572,7 +583,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu)
 
 	if (kvm_is_radix(kvm))
 		return H_FUNCTION;
-	global = global_invalidates(kvm, 0);
+	global = global_invalidates(kvm);
 	for (i = 0; i < 4 && ret == H_SUCCESS; ) {
 		n = 0;
 		for (; i < 4; ++i) {
@@ -732,8 +743,7 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags,
 			rb = compute_tlbie_rb(v, r, pte_index);
 			hpte[0] = cpu_to_be64((pte_v & ~HPTE_V_VALID) |
 					      HPTE_V_ABSENT);
-			do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags),
-				  true);
+			do_tlbies(kvm, &rb, 1, global_invalidates(kvm), true);
 			/* Don't lose R/C bit updates done by hardware */
 			r |= be64_to_cpu(hpte[1]) & (HPTE_R_R | HPTE_R_C);
 			hpte[1] = cpu_to_be64(r);
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 9c61f736c75b..f86a20270e50 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -320,7 +320,6 @@ kvm_novcpu_exit:
 	stw	r12, STACK_SLOT_TRAP(r1)
 	bl	kvmhv_commence_exit
 	nop
-	lwz	r12, STACK_SLOT_TRAP(r1)
 	b	kvmhv_switch_to_host
 
 /*
@@ -413,10 +412,11 @@ FTR_SECTION_ELSE
 	/* On P9 we use the split_info for coordinating LPCR changes */
 	lwz	r4, KVM_SPLIT_DO_SET(r6)
 	cmpwi	r4, 0
-	beq	63f
+	beq	1f
 	mr	r3, r6
 	bl	kvmhv_p9_set_lpcr
 	nop
+1:
 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
 63:
 	/* Order load of vcpu after load of vcore */
@@ -617,13 +617,6 @@ kvmppc_hv_entry:
 	lbz	r0, KVM_RADIX(r9)
 	cmpwi	cr7, r0, 0
 
-	/* Clear out SLB if hash */
-	bne	cr7, 2f
-	li	r6,0
-	slbmte	r6,r6
-	slbia
-	ptesync
-2:
 	/*
 	 * POWER7/POWER8 host -> guest partition switch code.
 	 * We don't have to lock against concurrent tlbies,
@@ -738,19 +731,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 10:	cmpdi	r4, 0
 	beq	kvmppc_primary_no_guest
 kvmppc_got_guest:
-
-	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
-	lwz	r5,VCPU_SLB_MAX(r4)
-	cmpwi	r5,0
-	beq	9f
-	mtctr	r5
-	addi	r6,r4,VCPU_SLB
-1:	ld	r8,VCPU_SLB_E(r6)
-	ld	r9,VCPU_SLB_V(r6)
-	slbmte	r9,r8
-	addi	r6,r6,VCPU_SLB_SIZE
-	bdnz	1b
-9:
 	/* Increment yield count if they have a VPA */
 	ld	r3, VCPU_VPA(r4)
 	cmpdi	r3, 0
@@ -957,7 +937,6 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
 	mftb	r7
 	subf	r3,r7,r8
 	mtspr	SPRN_DEC,r3
-	std	r3,VCPU_DEC(r4)
 
 	ld	r5, VCPU_SPRG0(r4)
 	ld	r6, VCPU_SPRG1(r4)
@@ -1018,6 +997,29 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
 	cmpdi	r3, 512		/* 1 microsecond */
 	blt	hdec_soon
 
+	/* For hash guest, clear out and reload the SLB */
+	ld	r6, VCPU_KVM(r4)
+	lbz	r0, KVM_RADIX(r6)
+	cmpwi	r0, 0
+	bne	9f
+	li	r6, 0
+	slbmte	r6, r6
+	slbia
+	ptesync
+
+	/* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */
+	lwz	r5,VCPU_SLB_MAX(r4)
+	cmpwi	r5,0
+	beq	9f
+	mtctr	r5
+	addi	r6,r4,VCPU_SLB
+1:	ld	r8,VCPU_SLB_E(r6)
+	ld	r9,VCPU_SLB_V(r6)
+	slbmte	r9,r8
+	addi	r6,r6,VCPU_SLB_SIZE
+	bdnz	1b
+9:
+
 #ifdef CONFIG_KVM_XICS
 	/* We are entering the guest on that thread, push VCPU to XIVE */
 	ld	r10, HSTATE_XIVE_TIMA_PHYS(r13)
@@ -1031,8 +1033,53 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
 	li	r9, TM_QW1_OS + TM_WORD2
 	stwcix	r11,r9,r10
 	li	r9, 1
-	stw	r9, VCPU_XIVE_PUSHED(r4)
+	stb	r9, VCPU_XIVE_PUSHED(r4)
 	eieio
+
+	/*
+	 * We clear the irq_pending flag. There is a small chance of a
+	 * race vs. the escalation interrupt happening on another
+	 * processor setting it again, but the only consequence is to
+	 * cause a spurrious wakeup on the next H_CEDE which is not an
+	 * issue.
+	 */
+	li	r0,0
+	stb	r0, VCPU_IRQ_PENDING(r4)
+
+	/*
+	 * In single escalation mode, if the escalation interrupt is
+	 * on, we mask it.
+	 */
+	lbz	r0, VCPU_XIVE_ESC_ON(r4)
+	cmpwi	r0,0
+	beq	1f
+	ld	r10, VCPU_XIVE_ESC_RADDR(r4)
+	li	r9, XIVE_ESB_SET_PQ_01
+	ldcix	r0, r10, r9
+	sync
+
+	/* We have a possible subtle race here: The escalation interrupt might
+	 * have fired and be on its way to the host queue while we mask it,
+	 * and if we unmask it early enough (re-cede right away), there is
+	 * a theorical possibility that it fires again, thus landing in the
+	 * target queue more than once which is a big no-no.
+	 *
+	 * Fortunately, solving this is rather easy. If the above load setting
+	 * PQ to 01 returns a previous value where P is set, then we know the
+	 * escalation interrupt is somewhere on its way to the host. In that
+	 * case we simply don't clear the xive_esc_on flag below. It will be
+	 * eventually cleared by the handler for the escalation interrupt.
+	 *
+	 * Then, when doing a cede, we check that flag again before re-enabling
+	 * the escalation interrupt, and if set, we abort the cede.
+	 */
+	andi.	r0, r0, XIVE_ESB_VAL_P
+	bne-	1f
+
+	/* Now P is 0, we can clear the flag */
+	li	r0, 0
+	stb	r0, VCPU_XIVE_ESC_ON(r4)
+1:
 no_xive:
 #endif /* CONFIG_KVM_XICS */
 
@@ -1172,6 +1219,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 
 secondary_too_late:
 	li	r12, 0
+	stw	r12, STACK_SLOT_TRAP(r1)
 	cmpdi	r4, 0
 	beq	11f
 	stw	r12, VCPU_TRAP(r4)
@@ -1193,7 +1241,7 @@ hdec_soon:
 	addi	r3, r4, VCPU_TB_RMEXIT
 	bl	kvmhv_accumulate_time
 #endif
-	b	guest_exit_cont
+	b	guest_bypass
 
 /******************************************************************************
  *                                                                            *
@@ -1423,15 +1471,35 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 	blt	deliver_guest_interrupt
 
 guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
+	/* Save more register state  */
+	mfdar	r6
+	mfdsisr	r7
+	std	r6, VCPU_DAR(r9)
+	stw	r7, VCPU_DSISR(r9)
+	/* don't overwrite fault_dar/fault_dsisr if HDSI */
+	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
+	beq	mc_cont
+	std	r6, VCPU_FAULT_DAR(r9)
+	stw	r7, VCPU_FAULT_DSISR(r9)
+
+	/* See if it is a machine check */
+	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
+	beq	machine_check_realmode
+mc_cont:
+#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
+	addi	r3, r9, VCPU_TB_RMEXIT
+	mr	r4, r9
+	bl	kvmhv_accumulate_time
+#endif
 #ifdef CONFIG_KVM_XICS
 	/* We are exiting, pull the VP from the XIVE */
-	lwz	r0, VCPU_XIVE_PUSHED(r9)
+	lbz	r0, VCPU_XIVE_PUSHED(r9)
 	cmpwi	cr0, r0, 0
 	beq	1f
 	li	r7, TM_SPC_PULL_OS_CTX
 	li	r6, TM_QW1_OS
 	mfmsr	r0
-	andi.	r0, r0, MSR_IR		/* in real mode? */
+	andi.	r0, r0, MSR_DR		/* in real mode? */
 	beq	2f
 	ld	r10, HSTATE_XIVE_TIMA_VIRT(r13)
 	cmpldi	cr0, r10, 0
@@ -1454,61 +1522,19 @@ guest_exit_cont:		/* r9 = vcpu, r12 = trap, r13 = paca */
 	/* Fixup some of the state for the next load */
 	li	r10, 0
 	li	r0, 0xff
-	stw	r10, VCPU_XIVE_PUSHED(r9)
+	stb	r10, VCPU_XIVE_PUSHED(r9)
 	stb	r10, (VCPU_XIVE_SAVED_STATE+3)(r9)
 	stb	r0, (VCPU_XIVE_SAVED_STATE+4)(r9)
 	eieio
 1:
 #endif /* CONFIG_KVM_XICS */
-	/* Save more register state  */
-	mfdar	r6
-	mfdsisr	r7
-	std	r6, VCPU_DAR(r9)
-	stw	r7, VCPU_DSISR(r9)
-	/* don't overwrite fault_dar/fault_dsisr if HDSI */
-	cmpwi	r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
-	beq	mc_cont
-	std	r6, VCPU_FAULT_DAR(r9)
-	stw	r7, VCPU_FAULT_DSISR(r9)
-
-	/* See if it is a machine check */
-	cmpwi	r12, BOOK3S_INTERRUPT_MACHINE_CHECK
-	beq	machine_check_realmode
-mc_cont:
-#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
-	addi	r3, r9, VCPU_TB_RMEXIT
-	mr	r4, r9
-	bl	kvmhv_accumulate_time
-#endif
-
-	mr 	r3, r12
-	/* Increment exit count, poke other threads to exit */
-	bl	kvmhv_commence_exit
-	nop
-	ld	r9, HSTATE_KVM_VCPU(r13)
-	lwz	r12, VCPU_TRAP(r9)
 
-	/* Stop others sending VCPU interrupts to this physical CPU */
-	li	r0, -1
-	stw	r0, VCPU_CPU(r9)
-	stw	r0, VCPU_THREAD_CPU(r9)
-
-	/* Save guest CTRL register, set runlatch to 1 */
-	mfspr	r6,SPRN_CTRLF
-	stw	r6,VCPU_CTRL(r9)
-	andi.	r0,r6,1
-	bne	4f
-	ori	r6,r6,1
-	mtspr	SPRN_CTRLT,r6
-4:
-	/* Check if we are running hash or radix and store it in cr2 */
+	/* For hash guest, read the guest SLB and save it away */
 	ld	r5, VCPU_KVM(r9)
 	lbz	r0, KVM_RADIX(r5)
-	cmpwi	cr2,r0,0
-
-	/* Read the guest SLB and save it away */
 	li	r5, 0
-	bne	cr2, 3f			/* for radix, save 0 entries */
+	cmpwi	r0, 0
+	bne	3f			/* for radix, save 0 entries */
 	lwz	r0,VCPU_SLB_NR(r9)	/* number of entries in SLB */
 	mtctr	r0
 	li	r6,0
@@ -1524,8 +1550,52 @@ mc_cont:
 	addi	r5,r5,1
 2:	addi	r6,r6,1
 	bdnz	1b
+	/* Finally clear out the SLB */
+	li	r0,0
+	slbmte	r0,r0
+	slbia
+	ptesync
 3:	stw	r5,VCPU_SLB_MAX(r9)
 
+	/* load host SLB entries */
+BEGIN_MMU_FTR_SECTION
+	b	0f
+END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
+	ld	r8,PACA_SLBSHADOWPTR(r13)
+
+	.rept	SLB_NUM_BOLTED
+	li	r3, SLBSHADOW_SAVEAREA
+	LDX_BE	r5, r8, r3
+	addi	r3, r3, 8
+	LDX_BE	r6, r8, r3
+	andis.	r7,r5,SLB_ESID_V@h
+	beq	1f
+	slbmte	r6,r5
+1:	addi	r8,r8,16
+	.endr
+0:
+
+guest_bypass:
+	stw	r12, STACK_SLOT_TRAP(r1)
+	mr 	r3, r12
+	/* Increment exit count, poke other threads to exit */
+	bl	kvmhv_commence_exit
+	nop
+	ld	r9, HSTATE_KVM_VCPU(r13)
+
+	/* Stop others sending VCPU interrupts to this physical CPU */
+	li	r0, -1
+	stw	r0, VCPU_CPU(r9)
+	stw	r0, VCPU_THREAD_CPU(r9)
+
+	/* Save guest CTRL register, set runlatch to 1 */
+	mfspr	r6,SPRN_CTRLF
+	stw	r6,VCPU_CTRL(r9)
+	andi.	r0,r6,1
+	bne	4f
+	ori	r6,r6,1
+	mtspr	SPRN_CTRLT,r6
+4:
 	/*
 	 * Save the guest PURR/SPURR
 	 */
@@ -1803,7 +1873,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 	ld	r5, VCPU_KVM(r9)
 	lbz	r0, KVM_RADIX(r5)
 	cmpwi	cr2, r0, 0
-	beq	cr2, 3f
+	beq	cr2, 4f
 
 	/* Radix: Handle the case where the guest used an illegal PID */
 	LOAD_REG_ADDR(r4, mmu_base_pid)
@@ -1839,19 +1909,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 BEGIN_FTR_SECTION
 	PPC_INVALIDATE_ERAT
 END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1)
-	b	4f
+4:
 #endif /* CONFIG_PPC_RADIX_MMU */
 
-	/* Hash: clear out SLB */
-3:	li	r5,0
-	slbmte	r5,r5
-	slbia
-	ptesync
-4:
 	/*
 	 * POWER7/POWER8 guest -> host partition switch code.
 	 * We don't have to lock against tlbies but we do
 	 * have to coordinate the hardware threads.
+	 * Here STACK_SLOT_TRAP(r1) contains the trap number.
 	 */
 kvmhv_switch_to_host:
 	/* Secondary threads wait for primary to do partition switch */
@@ -1904,20 +1969,21 @@ BEGIN_FTR_SECTION
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
 
 	/* If HMI, call kvmppc_realmode_hmi_handler() */
+	lwz	r12, STACK_SLOT_TRAP(r1)
 	cmpwi	r12, BOOK3S_INTERRUPT_HMI
 	bne	27f
 	bl	kvmppc_realmode_hmi_handler
 	nop
-	li	r12, BOOK3S_INTERRUPT_HMI
+	cmpdi	r3, 0
 	/*
-	 * At this point kvmppc_realmode_hmi_handler would have resync-ed
-	 * the TB. Hence it is not required to subtract guest timebase
-	 * offset from timebase. So, skip it.
+	 * At this point kvmppc_realmode_hmi_handler may have resync-ed
+	 * the TB, and if it has, we must not subtract the guest timebase
+	 * offset from the timebase. So, skip it.
 	 *
 	 * Also, do not call kvmppc_subcore_exit_guest() because it has
 	 * been invoked as part of kvmppc_realmode_hmi_handler().
 	 */
-	b	30f
+	beq	30f
 
 27:
 	/* Subtract timebase offset from timebase */
@@ -1961,10 +2027,8 @@ BEGIN_FTR_SECTION
 	lwz	r8, KVM_SPLIT_DO_RESTORE(r3)
 	cmpwi	r8, 0
 	beq	47f
-	stw	r12, STACK_SLOT_TRAP(r1)
 	bl	kvmhv_p9_restore_lpcr
 	nop
-	lwz	r12, STACK_SLOT_TRAP(r1)
 	b	48f
 47:
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
@@ -1972,23 +2036,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 	mtspr	SPRN_LPCR,r8
 	isync
 48:
-	/* load host SLB entries */
-BEGIN_MMU_FTR_SECTION
-	b	0f
-END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
-	ld	r8,PACA_SLBSHADOWPTR(r13)
-
-	.rept	SLB_NUM_BOLTED
-	li	r3, SLBSHADOW_SAVEAREA
-	LDX_BE	r5, r8, r3
-	addi	r3, r3, 8
-	LDX_BE	r6, r8, r3
-	andis.	r7,r5,SLB_ESID_V@h
-	beq	1f
-	slbmte	r6,r5
-1:	addi	r8,r8,16
-	.endr
-0:
 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
 	/* Finish timing, if we have a vcpu */
 	ld	r4, HSTATE_KVM_VCPU(r13)
@@ -2002,6 +2049,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
 	li	r0, KVM_GUEST_MODE_NONE
 	stb	r0, HSTATE_IN_GUEST(r13)
 
+	lwz	r12, STACK_SLOT_TRAP(r1)	/* return trap # in r12 */
 	ld	r0, SFS+PPC_LR_STKOFF(r1)
 	addi	r1, r1, SFS
 	mtlr	r0
@@ -2744,7 +2792,32 @@ kvm_cede_prodded:
 	/* we've ceded but we want to give control to the host */
 kvm_cede_exit:
 	ld	r9, HSTATE_KVM_VCPU(r13)
-	b	guest_exit_cont
+#ifdef CONFIG_KVM_XICS
+	/* Abort if we still have a pending escalation */
+	lbz	r5, VCPU_XIVE_ESC_ON(r9)
+	cmpwi	r5, 0
+	beq	1f
+	li	r0, 0
+	stb	r0, VCPU_CEDED(r9)
+1:	/* Enable XIVE escalation */
+	li	r5, XIVE_ESB_SET_PQ_00
+	mfmsr	r0
+	andi.	r0, r0, MSR_DR		/* in real mode? */
+	beq	1f
+	ld	r10, VCPU_XIVE_ESC_VADDR(r9)
+	cmpdi	r10, 0
+	beq	3f
+	ldx	r0, r10, r5
+	b	2f
+1:	ld	r10, VCPU_XIVE_ESC_RADDR(r9)
+	cmpdi	r10, 0
+	beq	3f
+	ldcix	r0, r10, r5
+2:	sync
+	li	r0, 1
+	stb	r0, VCPU_XIVE_ESC_ON(r9)
+#endif /* CONFIG_KVM_XICS */
+3:	b	guest_exit_cont
 
 	/* Try to handle a machine check in real mode */
 machine_check_realmode:
@@ -3248,7 +3321,7 @@ kvmppc_bad_host_intr:
 	mfctr	r4
 #endif
 	mfxer	r5
-	lbz	r6, PACASOFTIRQEN(r13)
+	lbz	r6, PACAIRQSOFTMASK(r13)
 	std	r3, _LINK(r1)
 	std	r4, _CTR(r1)
 	std	r5, _XER(r1)
diff --git a/arch/powerpc/kvm/book3s_interrupts.S b/arch/powerpc/kvm/book3s_interrupts.S
index 901e6fe00c39..c18e845019ec 100644
--- a/arch/powerpc/kvm/book3s_interrupts.S
+++ b/arch/powerpc/kvm/book3s_interrupts.S
@@ -96,7 +96,7 @@ kvm_start_entry:
 
 kvm_start_lightweight:
 	/* Copy registers into shadow vcpu so we can access them in real mode */
-	GET_SHADOW_VCPU(r3)
+	mr	r3, r4
 	bl	FUNC(kvmppc_copy_to_svcpu)
 	nop
 	REST_GPR(4, r1)
@@ -165,9 +165,7 @@ after_sprg3_load:
 	stw	r12, VCPU_TRAP(r3)
 
 	/* Transfer reg values from shadow vcpu back to vcpu struct */
-	/* On 64-bit, interrupts are still off at this point */
 
-	GET_SHADOW_VCPU(r4)
 	bl	FUNC(kvmppc_copy_from_svcpu)
 	nop
 
diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c
index 7deaeeb14b93..3ae752314b34 100644
--- a/arch/powerpc/kvm/book3s_pr.c
+++ b/arch/powerpc/kvm/book3s_pr.c
@@ -121,7 +121,7 @@ static void kvmppc_core_vcpu_put_pr(struct kvm_vcpu *vcpu)
 #ifdef CONFIG_PPC_BOOK3S_64
 	struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu);
 	if (svcpu->in_use) {
-		kvmppc_copy_from_svcpu(vcpu, svcpu);
+		kvmppc_copy_from_svcpu(vcpu);
 	}
 	memcpy(to_book3s(vcpu)->slb_shadow, svcpu->slb, sizeof(svcpu->slb));
 	to_book3s(vcpu)->slb_shadow_max = svcpu->slb_max;
@@ -143,9 +143,10 @@ static void kvmppc_core_vcpu_put_pr(struct kvm_vcpu *vcpu)
 }
 
 /* Copy data needed by real-mode code from vcpu to shadow vcpu */
-void kvmppc_copy_to_svcpu(struct kvmppc_book3s_shadow_vcpu *svcpu,
-			  struct kvm_vcpu *vcpu)
+void kvmppc_copy_to_svcpu(struct kvm_vcpu *vcpu)
 {
+	struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu);
+
 	svcpu->gpr[0] = vcpu->arch.gpr[0];
 	svcpu->gpr[1] = vcpu->arch.gpr[1];
 	svcpu->gpr[2] = vcpu->arch.gpr[2];
@@ -177,17 +178,14 @@ void kvmppc_copy_to_svcpu(struct kvmppc_book3s_shadow_vcpu *svcpu,
 	if (cpu_has_feature(CPU_FTR_ARCH_207S))
 		vcpu->arch.entry_ic = mfspr(SPRN_IC);
 	svcpu->in_use = true;
+
+	svcpu_put(svcpu);
 }
 
 /* Copy data touched by real-mode code from shadow vcpu back to vcpu */
-void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu,
-			    struct kvmppc_book3s_shadow_vcpu *svcpu)
+void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu)
 {
-	/*
-	 * vcpu_put would just call us again because in_use hasn't
-	 * been updated yet.
-	 */
-	preempt_disable();
+	struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu);
 
 	/*
 	 * Maybe we were already preempted and synced the svcpu from
@@ -233,7 +231,7 @@ void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu,
 	svcpu->in_use = false;
 
 out:
-	preempt_enable();
+	svcpu_put(svcpu);
 }
 
 static int kvmppc_core_check_requests_pr(struct kvm_vcpu *vcpu)
diff --git a/arch/powerpc/kvm/book3s_xics.c b/arch/powerpc/kvm/book3s_xics.c
index d329b2add7e2..b8356cdc0c04 100644
--- a/arch/powerpc/kvm/book3s_xics.c
+++ b/arch/powerpc/kvm/book3s_xics.c
@@ -1039,7 +1039,7 @@ static void xics_debugfs_init(struct kvmppc_xics *xics)
 		return;
 	}
 
-	xics->dentry = debugfs_create_file(name, S_IRUGO, powerpc_debugfs_root,
+	xics->dentry = debugfs_create_file(name, 0444, powerpc_debugfs_root,
 					   xics, &xics_debug_fops);
 
 	pr_debug("%s: created %s\n", __func__, name);
diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c
index 0d750d274c4e..f9818d7d3381 100644
--- a/arch/powerpc/kvm/book3s_xive.c
+++ b/arch/powerpc/kvm/book3s_xive.c
@@ -16,7 +16,7 @@
 #include <linux/delay.h>
 #include <linux/percpu.h>
 #include <linux/cpumask.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <asm/kvm_book3s.h>
 #include <asm/kvm_ppc.h>
 #include <asm/hvcall.h>
@@ -84,12 +84,22 @@ static irqreturn_t xive_esc_irq(int irq, void *data)
 {
 	struct kvm_vcpu *vcpu = data;
 
-	/* We use the existing H_PROD mechanism to wake up the target */
-	vcpu->arch.prodded = 1;
+	vcpu->arch.irq_pending = 1;
 	smp_mb();
 	if (vcpu->arch.ceded)
 		kvmppc_fast_vcpu_kick(vcpu);
 
+	/* Since we have the no-EOI flag, the interrupt is effectively
+	 * disabled now. Clearing xive_esc_on means we won't bother
+	 * doing so on the next entry.
+	 *
+	 * This also allows the entry code to know that if a PQ combination
+	 * of 10 is observed while xive_esc_on is true, it means the queue
+	 * contains an unprocessed escalation interrupt. We don't make use of
+	 * that knowledge today but might (see comment in book3s_hv_rmhandler.S)
+	 */
+	vcpu->arch.xive_esc_on = false;
+
 	return IRQ_HANDLED;
 }
 
@@ -112,19 +122,21 @@ static int xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio)
 		return -EIO;
 	}
 
-	/*
-	 * Future improvement: start with them disabled
-	 * and handle DD2 and later scheme of merged escalation
-	 * interrupts
-	 */
-	name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
-			 vcpu->kvm->arch.lpid, xc->server_num, prio);
+	if (xc->xive->single_escalation)
+		name = kasprintf(GFP_KERNEL, "kvm-%d-%d",
+				 vcpu->kvm->arch.lpid, xc->server_num);
+	else
+		name = kasprintf(GFP_KERNEL, "kvm-%d-%d-%d",
+				 vcpu->kvm->arch.lpid, xc->server_num, prio);
 	if (!name) {
 		pr_err("Failed to allocate escalation irq name for queue %d of VCPU %d\n",
 		       prio, xc->server_num);
 		rc = -ENOMEM;
 		goto error;
 	}
+
+	pr_devel("Escalation %s irq %d (prio %d)\n", name, xc->esc_virq[prio], prio);
+
 	rc = request_irq(xc->esc_virq[prio], xive_esc_irq,
 			 IRQF_NO_THREAD, name, vcpu);
 	if (rc) {
@@ -133,6 +145,25 @@ static int xive_attach_escalation(struct kvm_vcpu *vcpu, u8 prio)
 		goto error;
 	}
 	xc->esc_virq_names[prio] = name;
+
+	/* In single escalation mode, we grab the ESB MMIO of the
+	 * interrupt and mask it. Also populate the VCPU v/raddr
+	 * of the ESB page for use by asm entry/exit code. Finally
+	 * set the XIVE_IRQ_NO_EOI flag which will prevent the
+	 * core code from performing an EOI on the escalation
+	 * interrupt, thus leaving it effectively masked after
+	 * it fires once.
+	 */
+	if (xc->xive->single_escalation) {
+		struct irq_data *d = irq_get_irq_data(xc->esc_virq[prio]);
+		struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
+
+		xive_vm_esb_load(xd, XIVE_ESB_SET_PQ_01);
+		vcpu->arch.xive_esc_raddr = xd->eoi_page;
+		vcpu->arch.xive_esc_vaddr = (__force u64)xd->eoi_mmio;
+		xd->flags |= XIVE_IRQ_NO_EOI;
+	}
+
 	return 0;
 error:
 	irq_dispose_mapping(xc->esc_virq[prio]);
@@ -157,7 +188,7 @@ static int xive_provision_queue(struct kvm_vcpu *vcpu, u8 prio)
 	if (!qpage) {
 		pr_err("Failed to allocate queue %d for VCPU %d\n",
 		       prio, xc->server_num);
-		return -ENOMEM;;
+		return -ENOMEM;
 	}
 	memset(qpage, 0, 1 << xive->q_order);
 
@@ -191,12 +222,12 @@ static int xive_check_provisioning(struct kvm *kvm, u8 prio)
 
 	pr_devel("Provisioning prio... %d\n", prio);
 
-	/* Provision each VCPU and enable escalations */
+	/* Provision each VCPU and enable escalations if needed */
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		if (!vcpu->arch.xive_vcpu)
 			continue;
 		rc = xive_provision_queue(vcpu, prio);
-		if (rc == 0)
+		if (rc == 0 && !xive->single_escalation)
 			xive_attach_escalation(vcpu, prio);
 		if (rc)
 			return rc;
@@ -1082,6 +1113,7 @@ int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
 	/* Allocate IPI */
 	xc->vp_ipi = xive_native_alloc_irq();
 	if (!xc->vp_ipi) {
+		pr_err("Failed to allocate xive irq for VCPU IPI\n");
 		r = -EIO;
 		goto bail;
 	}
@@ -1091,19 +1123,34 @@ int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
 	if (r)
 		goto bail;
 
+	/*
+	 * Enable the VP first as the single escalation mode will
+	 * affect escalation interrupts numbering
+	 */
+	r = xive_native_enable_vp(xc->vp_id, xive->single_escalation);
+	if (r) {
+		pr_err("Failed to enable VP in OPAL, err %d\n", r);
+		goto bail;
+	}
+
 	/*
 	 * Initialize queues. Initially we set them all for no queueing
 	 * and we enable escalation for queue 0 only which we'll use for
 	 * our mfrr change notifications. If the VCPU is hot-plugged, we
-	 * do handle provisioning however.
+	 * do handle provisioning however based on the existing "map"
+	 * of enabled queues.
 	 */
 	for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
 		struct xive_q *q = &xc->queues[i];
 
+		/* Single escalation, no queue 7 */
+		if (i == 7 && xive->single_escalation)
+			break;
+
 		/* Is queue already enabled ? Provision it */
 		if (xive->qmap & (1 << i)) {
 			r = xive_provision_queue(vcpu, i);
-			if (r == 0)
+			if (r == 0 && !xive->single_escalation)
 				xive_attach_escalation(vcpu, i);
 			if (r)
 				goto bail;
@@ -1123,11 +1170,6 @@ int kvmppc_xive_connect_vcpu(struct kvm_device *dev,
 	if (r)
 		goto bail;
 
-	/* Enable the VP */
-	r = xive_native_enable_vp(xc->vp_id);
-	if (r)
-		goto bail;
-
 	/* Route the IPI */
 	r = xive_native_configure_irq(xc->vp_ipi, xc->vp_id, 0, XICS_IPI);
 	if (!r)
@@ -1474,6 +1516,7 @@ static int xive_set_source(struct kvmppc_xive *xive, long irq, u64 addr)
 
 	pr_devel("  val=0x016%llx (server=0x%x, guest_prio=%d)\n",
 		 val, server, guest_prio);
+
 	/*
 	 * If the source doesn't already have an IPI, allocate
 	 * one and get the corresponding data
@@ -1762,6 +1805,8 @@ static int kvmppc_xive_create(struct kvm_device *dev, u32 type)
 	if (xive->vp_base == XIVE_INVALID_VP)
 		ret = -ENOMEM;
 
+	xive->single_escalation = xive_native_has_single_escalation();
+
 	if (ret) {
 		kfree(xive);
 		return ret;
@@ -1795,6 +1840,7 @@ static int xive_debug_show(struct seq_file *m, void *private)
 
 	kvm_for_each_vcpu(i, vcpu, kvm) {
 		struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu;
+		unsigned int i;
 
 		if (!xc)
 			continue;
@@ -1804,6 +1850,33 @@ static int xive_debug_show(struct seq_file *m, void *private)
 			   xc->server_num, xc->cppr, xc->hw_cppr,
 			   xc->mfrr, xc->pending,
 			   xc->stat_rm_h_xirr, xc->stat_vm_h_xirr);
+		for (i = 0; i < KVMPPC_XIVE_Q_COUNT; i++) {
+			struct xive_q *q = &xc->queues[i];
+			u32 i0, i1, idx;
+
+			if (!q->qpage && !xc->esc_virq[i])
+				continue;
+
+			seq_printf(m, " [q%d]: ", i);
+
+			if (q->qpage) {
+				idx = q->idx;
+				i0 = be32_to_cpup(q->qpage + idx);
+				idx = (idx + 1) & q->msk;
+				i1 = be32_to_cpup(q->qpage + idx);
+				seq_printf(m, "T=%d %08x %08x... \n", q->toggle, i0, i1);
+			}
+			if (xc->esc_virq[i]) {
+				struct irq_data *d = irq_get_irq_data(xc->esc_virq[i]);
+				struct xive_irq_data *xd = irq_data_get_irq_handler_data(d);
+				u64 pq = xive_vm_esb_load(xd, XIVE_ESB_GET);
+				seq_printf(m, "E:%c%c I(%d:%llx:%llx)",
+					   (pq & XIVE_ESB_VAL_P) ? 'P' : 'p',
+					   (pq & XIVE_ESB_VAL_Q) ? 'Q' : 'q',
+					   xc->esc_virq[i], pq, xd->eoi_page);
+				seq_printf(m, "\n");
+			}
+		}
 
 		t_rm_h_xirr += xc->stat_rm_h_xirr;
 		t_rm_h_ipoll += xc->stat_rm_h_ipoll;
diff --git a/arch/powerpc/kvm/book3s_xive.h b/arch/powerpc/kvm/book3s_xive.h
index 6ba63f8e8a61..a08ae6fd4c51 100644
--- a/arch/powerpc/kvm/book3s_xive.h
+++ b/arch/powerpc/kvm/book3s_xive.h
@@ -120,6 +120,8 @@ struct kvmppc_xive {
 	u32	q_order;
 	u32	q_page_order;
 
+	/* Flags */
+	u8	single_escalation;
 };
 
 #define KVMPPC_XIVE_Q_COUNT	8
@@ -201,25 +203,20 @@ static inline struct kvmppc_xive_src_block *kvmppc_xive_find_source(struct kvmpp
  * is as follow.
  *
  * Guest request for 0...6 are honored. Guest request for anything
- * higher results in a priority of 7 being applied.
- *
- * However, when XIRR is returned via H_XIRR, 7 is translated to 0xb
- * in order to match AIX expectations
+ * higher results in a priority of 6 being applied.
  *
  * Similar mapping is done for CPPR values
  */
 static inline u8 xive_prio_from_guest(u8 prio)
 {
-	if (prio == 0xff || prio < 8)
+	if (prio == 0xff || prio < 6)
 		return prio;
-	return 7;
+	return 6;
 }
 
 static inline u8 xive_prio_to_guest(u8 prio)
 {
-	if (prio == 0xff || prio < 7)
-		return prio;
-	return 0xb;
+	return prio;
 }
 
 static inline u32 __xive_read_eq(__be32 *qpage, u32 msk, u32 *idx, u32 *toggle)
diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c
index 83b485810aea..6038e2e7aee0 100644
--- a/arch/powerpc/kvm/booke.c
+++ b/arch/powerpc/kvm/booke.c
@@ -1431,6 +1431,8 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
 
+	vcpu_load(vcpu);
+
 	regs->pc = vcpu->arch.pc;
 	regs->cr = kvmppc_get_cr(vcpu);
 	regs->ctr = vcpu->arch.ctr;
@@ -1452,6 +1454,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -1459,6 +1462,8 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
 	int i;
 
+	vcpu_load(vcpu);
+
 	vcpu->arch.pc = regs->pc;
 	kvmppc_set_cr(vcpu, regs->cr);
 	vcpu->arch.ctr = regs->ctr;
@@ -1480,6 +1485,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
 		kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -1607,30 +1613,42 @@ int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
+	int ret;
+
+	vcpu_load(vcpu);
+
 	sregs->pvr = vcpu->arch.pvr;
 
 	get_sregs_base(vcpu, sregs);
 	get_sregs_arch206(vcpu, sregs);
-	return vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
+	ret = vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
+
+	vcpu_put(vcpu);
+	return ret;
 }
 
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
                                   struct kvm_sregs *sregs)
 {
-	int ret;
+	int ret = -EINVAL;
 
+	vcpu_load(vcpu);
 	if (vcpu->arch.pvr != sregs->pvr)
-		return -EINVAL;
+		goto out;
 
 	ret = set_sregs_base(vcpu, sregs);
 	if (ret < 0)
-		return ret;
+		goto out;
 
 	ret = set_sregs_arch206(vcpu, sregs);
 	if (ret < 0)
-		return ret;
+		goto out;
 
-	return vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
+	ret = vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
+
+out:
+	vcpu_put(vcpu);
+	return ret;
 }
 
 int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id,
@@ -1773,7 +1791,9 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 {
 	int r;
 
+	vcpu_load(vcpu);
 	r = kvmppc_core_vcpu_translate(vcpu, tr);
+	vcpu_put(vcpu);
 	return r;
 }
 
@@ -1996,12 +2016,15 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 {
 	struct debug_reg *dbg_reg;
 	int n, b = 0, w = 0;
+	int ret = 0;
+
+	vcpu_load(vcpu);
 
 	if (!(dbg->control & KVM_GUESTDBG_ENABLE)) {
 		vcpu->arch.dbg_reg.dbcr0 = 0;
 		vcpu->guest_debug = 0;
 		kvm_guest_protect_msr(vcpu, MSR_DE, false);
-		return 0;
+		goto out;
 	}
 
 	kvm_guest_protect_msr(vcpu, MSR_DE, true);
@@ -2033,8 +2056,9 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 #endif
 
 	if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
-		return 0;
+		goto out;
 
+	ret = -EINVAL;
 	for (n = 0; n < (KVMPPC_BOOKE_IAC_NUM + KVMPPC_BOOKE_DAC_NUM); n++) {
 		uint64_t addr = dbg->arch.bp[n].addr;
 		uint32_t type = dbg->arch.bp[n].type;
@@ -2045,21 +2069,24 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 		if (type & ~(KVMPPC_DEBUG_WATCH_READ |
 			     KVMPPC_DEBUG_WATCH_WRITE |
 			     KVMPPC_DEBUG_BREAKPOINT))
-			return -EINVAL;
+			goto out;
 
 		if (type & KVMPPC_DEBUG_BREAKPOINT) {
 			/* Setting H/W breakpoint */
 			if (kvmppc_booke_add_breakpoint(dbg_reg, addr, b++))
-				return -EINVAL;
+				goto out;
 		} else {
 			/* Setting H/W watchpoint */
 			if (kvmppc_booke_add_watchpoint(dbg_reg, addr,
 							type, w++))
-				return -EINVAL;
+				goto out;
 		}
 	}
 
-	return 0;
+	ret = 0;
+out:
+	vcpu_put(vcpu);
+	return ret;
 }
 
 void kvmppc_booke_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
diff --git a/arch/powerpc/kvm/emulate_loadstore.c b/arch/powerpc/kvm/emulate_loadstore.c
index af833531af31..a382e15135e6 100644
--- a/arch/powerpc/kvm/emulate_loadstore.c
+++ b/arch/powerpc/kvm/emulate_loadstore.c
@@ -58,6 +58,18 @@ static bool kvmppc_check_vsx_disabled(struct kvm_vcpu *vcpu)
 }
 #endif /* CONFIG_VSX */
 
+#ifdef CONFIG_ALTIVEC
+static bool kvmppc_check_altivec_disabled(struct kvm_vcpu *vcpu)
+{
+	if (!(kvmppc_get_msr(vcpu) & MSR_VEC)) {
+		kvmppc_core_queue_vec_unavail(vcpu);
+		return true;
+	}
+
+	return false;
+}
+#endif /* CONFIG_ALTIVEC */
+
 /*
  * XXX to do:
  * lfiwax, lfiwzx
@@ -98,6 +110,7 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu)
 	vcpu->arch.mmio_vsx_copy_type = KVMPPC_VSX_COPY_NONE;
 	vcpu->arch.mmio_sp64_extend = 0;
 	vcpu->arch.mmio_sign_extend = 0;
+	vcpu->arch.mmio_vmx_copy_nums = 0;
 
 	switch (get_op(inst)) {
 	case 31:
@@ -459,6 +472,29 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu)
 							 rs, 4, 1);
 			break;
 #endif /* CONFIG_VSX */
+
+#ifdef CONFIG_ALTIVEC
+		case OP_31_XOP_LVX:
+			if (kvmppc_check_altivec_disabled(vcpu))
+				return EMULATE_DONE;
+			vcpu->arch.vaddr_accessed &= ~0xFULL;
+			vcpu->arch.paddr_accessed &= ~0xFULL;
+			vcpu->arch.mmio_vmx_copy_nums = 2;
+			emulated = kvmppc_handle_load128_by2x64(run, vcpu,
+					KVM_MMIO_REG_VMX|rt, 1);
+			break;
+
+		case OP_31_XOP_STVX:
+			if (kvmppc_check_altivec_disabled(vcpu))
+				return EMULATE_DONE;
+			vcpu->arch.vaddr_accessed &= ~0xFULL;
+			vcpu->arch.paddr_accessed &= ~0xFULL;
+			vcpu->arch.mmio_vmx_copy_nums = 2;
+			emulated = kvmppc_handle_store128_by2x64(run, vcpu,
+					rs, 1);
+			break;
+#endif /* CONFIG_ALTIVEC */
+
 		default:
 			emulated = EMULATE_FAIL;
 			break;
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 0a7c88786ec0..52c205373986 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -638,8 +638,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 		r = 1;
 		break;
 	case KVM_CAP_SPAPR_RESIZE_HPT:
-		/* Disable this on POWER9 until code handles new HPTE format */
-		r = !!hv_enabled && !cpu_has_feature(CPU_FTR_ARCH_300);
+		r = !!hv_enabled;
 		break;
 #endif
 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
@@ -763,7 +762,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 
 	hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 	vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup;
-	vcpu->arch.dec_expires = ~(u64)0;
+	vcpu->arch.dec_expires = get_tb();
 
 #ifdef CONFIG_KVM_EXIT_TIMING
 	mutex_init(&vcpu->arch.exit_timing_lock);
@@ -930,6 +929,34 @@ static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu,
 }
 #endif /* CONFIG_VSX */
 
+#ifdef CONFIG_ALTIVEC
+static inline void kvmppc_set_vmx_dword(struct kvm_vcpu *vcpu,
+		u64 gpr)
+{
+	int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
+	u32 hi, lo;
+	u32 di;
+
+#ifdef __BIG_ENDIAN
+	hi = gpr >> 32;
+	lo = gpr & 0xffffffff;
+#else
+	lo = gpr >> 32;
+	hi = gpr & 0xffffffff;
+#endif
+
+	di = 2 - vcpu->arch.mmio_vmx_copy_nums;		/* doubleword index */
+	if (di > 1)
+		return;
+
+	if (vcpu->arch.mmio_host_swabbed)
+		di = 1 - di;
+
+	VCPU_VSX_VR(vcpu, index).u[di * 2] = hi;
+	VCPU_VSX_VR(vcpu, index).u[di * 2 + 1] = lo;
+}
+#endif /* CONFIG_ALTIVEC */
+
 #ifdef CONFIG_PPC_FPU
 static inline u64 sp_to_dp(u32 fprs)
 {
@@ -1032,6 +1059,11 @@ static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu,
 				KVMPPC_VSX_COPY_DWORD_LOAD_DUMP)
 			kvmppc_set_vsr_dword_dump(vcpu, gpr);
 		break;
+#endif
+#ifdef CONFIG_ALTIVEC
+	case KVM_MMIO_REG_VMX:
+		kvmppc_set_vmx_dword(vcpu, gpr);
+		break;
 #endif
 	default:
 		BUG();
@@ -1106,11 +1138,9 @@ int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
 {
 	enum emulation_result emulated = EMULATE_DONE;
 
-	/* Currently, mmio_vsx_copy_nums only allowed to be less than 4 */
-	if ( (vcpu->arch.mmio_vsx_copy_nums > 4) ||
-		(vcpu->arch.mmio_vsx_copy_nums < 0) ) {
+	/* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
+	if (vcpu->arch.mmio_vsx_copy_nums > 4)
 		return EMULATE_FAIL;
-	}
 
 	while (vcpu->arch.mmio_vsx_copy_nums) {
 		emulated = __kvmppc_handle_load(run, vcpu, rt, bytes,
@@ -1252,11 +1282,9 @@ int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
 
 	vcpu->arch.io_gpr = rs;
 
-	/* Currently, mmio_vsx_copy_nums only allowed to be less than 4 */
-	if ( (vcpu->arch.mmio_vsx_copy_nums > 4) ||
-		(vcpu->arch.mmio_vsx_copy_nums < 0) ) {
+	/* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
+	if (vcpu->arch.mmio_vsx_copy_nums > 4)
 		return EMULATE_FAIL;
-	}
 
 	while (vcpu->arch.mmio_vsx_copy_nums) {
 		if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1)
@@ -1312,6 +1340,111 @@ static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu,
 }
 #endif /* CONFIG_VSX */
 
+#ifdef CONFIG_ALTIVEC
+/* handle quadword load access in two halves */
+int kvmppc_handle_load128_by2x64(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		unsigned int rt, int is_default_endian)
+{
+	enum emulation_result emulated = EMULATE_DONE;
+
+	while (vcpu->arch.mmio_vmx_copy_nums) {
+		emulated = __kvmppc_handle_load(run, vcpu, rt, 8,
+				is_default_endian, 0);
+
+		if (emulated != EMULATE_DONE)
+			break;
+
+		vcpu->arch.paddr_accessed += run->mmio.len;
+		vcpu->arch.mmio_vmx_copy_nums--;
+	}
+
+	return emulated;
+}
+
+static inline int kvmppc_get_vmx_data(struct kvm_vcpu *vcpu, int rs, u64 *val)
+{
+	vector128 vrs = VCPU_VSX_VR(vcpu, rs);
+	u32 di;
+	u64 w0, w1;
+
+	di = 2 - vcpu->arch.mmio_vmx_copy_nums;		/* doubleword index */
+	if (di > 1)
+		return -1;
+
+	if (vcpu->arch.mmio_host_swabbed)
+		di = 1 - di;
+
+	w0 = vrs.u[di * 2];
+	w1 = vrs.u[di * 2 + 1];
+
+#ifdef __BIG_ENDIAN
+	*val = (w0 << 32) | w1;
+#else
+	*val = (w1 << 32) | w0;
+#endif
+	return 0;
+}
+
+/* handle quadword store in two halves */
+int kvmppc_handle_store128_by2x64(struct kvm_run *run, struct kvm_vcpu *vcpu,
+		unsigned int rs, int is_default_endian)
+{
+	u64 val = 0;
+	enum emulation_result emulated = EMULATE_DONE;
+
+	vcpu->arch.io_gpr = rs;
+
+	while (vcpu->arch.mmio_vmx_copy_nums) {
+		if (kvmppc_get_vmx_data(vcpu, rs, &val) == -1)
+			return EMULATE_FAIL;
+
+		emulated = kvmppc_handle_store(run, vcpu, val, 8,
+				is_default_endian);
+		if (emulated != EMULATE_DONE)
+			break;
+
+		vcpu->arch.paddr_accessed += run->mmio.len;
+		vcpu->arch.mmio_vmx_copy_nums--;
+	}
+
+	return emulated;
+}
+
+static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu,
+		struct kvm_run *run)
+{
+	enum emulation_result emulated = EMULATE_FAIL;
+	int r;
+
+	vcpu->arch.paddr_accessed += run->mmio.len;
+
+	if (!vcpu->mmio_is_write) {
+		emulated = kvmppc_handle_load128_by2x64(run, vcpu,
+				vcpu->arch.io_gpr, 1);
+	} else {
+		emulated = kvmppc_handle_store128_by2x64(run, vcpu,
+				vcpu->arch.io_gpr, 1);
+	}
+
+	switch (emulated) {
+	case EMULATE_DO_MMIO:
+		run->exit_reason = KVM_EXIT_MMIO;
+		r = RESUME_HOST;
+		break;
+	case EMULATE_FAIL:
+		pr_info("KVM: MMIO emulation failed (VMX repeat)\n");
+		run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+		run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
+		r = RESUME_HOST;
+		break;
+	default:
+		r = RESUME_GUEST;
+		break;
+	}
+	return r;
+}
+#endif /* CONFIG_ALTIVEC */
+
 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
 {
 	int r = 0;
@@ -1413,6 +1546,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	int r;
 
+	vcpu_load(vcpu);
+
 	if (vcpu->mmio_needed) {
 		vcpu->mmio_needed = 0;
 		if (!vcpu->mmio_is_write)
@@ -1427,7 +1562,19 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 			r = kvmppc_emulate_mmio_vsx_loadstore(vcpu, run);
 			if (r == RESUME_HOST) {
 				vcpu->mmio_needed = 1;
-				return r;
+				goto out;
+			}
+		}
+#endif
+#ifdef CONFIG_ALTIVEC
+		if (vcpu->arch.mmio_vmx_copy_nums > 0)
+			vcpu->arch.mmio_vmx_copy_nums--;
+
+		if (vcpu->arch.mmio_vmx_copy_nums > 0) {
+			r = kvmppc_emulate_mmio_vmx_loadstore(vcpu, run);
+			if (r == RESUME_HOST) {
+				vcpu->mmio_needed = 1;
+				goto out;
 			}
 		}
 #endif
@@ -1461,6 +1608,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 	kvm_sigset_deactivate(vcpu);
 
+#ifdef CONFIG_ALTIVEC
+out:
+#endif
+	vcpu_put(vcpu);
 	return r;
 }
 
@@ -1608,23 +1759,31 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 	return -EINVAL;
 }
 
-long kvm_arch_vcpu_ioctl(struct file *filp,
-                         unsigned int ioctl, unsigned long arg)
+long kvm_arch_vcpu_async_ioctl(struct file *filp,
+			       unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
-	long r;
 
-	switch (ioctl) {
-	case KVM_INTERRUPT: {
+	if (ioctl == KVM_INTERRUPT) {
 		struct kvm_interrupt irq;
-		r = -EFAULT;
 		if (copy_from_user(&irq, argp, sizeof(irq)))
-			goto out;
-		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
-		goto out;
+			return -EFAULT;
+		return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
 	}
+	return -ENOIOCTLCMD;
+}
+
+long kvm_arch_vcpu_ioctl(struct file *filp,
+                         unsigned int ioctl, unsigned long arg)
+{
+	struct kvm_vcpu *vcpu = filp->private_data;
+	void __user *argp = (void __user *)arg;
+	long r;
+
+	vcpu_load(vcpu);
 
+	switch (ioctl) {
 	case KVM_ENABLE_CAP:
 	{
 		struct kvm_enable_cap cap;
@@ -1664,6 +1823,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 	}
 
 out:
+	vcpu_put(vcpu);
 	return r;
 }
 
diff --git a/arch/powerpc/kvm/timing.c b/arch/powerpc/kvm/timing.c
index e44d2b2ea97e..1c03c978eb18 100644
--- a/arch/powerpc/kvm/timing.c
+++ b/arch/powerpc/kvm/timing.c
@@ -143,8 +143,7 @@ static int kvmppc_exit_timing_show(struct seq_file *m, void *private)
 	int i;
 	u64 min, max, sum, sum_quad;
 
-	seq_printf(m, "%s", "type	count	min	max	sum	sum_squared\n");
-
+	seq_puts(m, "type	count	min	max	sum	sum_squared\n");
 
 	for (i = 0; i < __NUMBER_OF_KVM_EXIT_TYPES; i++) {
 
diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c
index d469224c4ada..e0d881ab304e 100644
--- a/arch/powerpc/lib/code-patching.c
+++ b/arch/powerpc/lib/code-patching.c
@@ -23,19 +23,26 @@
 #include <asm/code-patching.h>
 #include <asm/setup.h>
 
-static int __patch_instruction(unsigned int *addr, unsigned int instr)
+static int __patch_instruction(unsigned int *exec_addr, unsigned int instr,
+			       unsigned int *patch_addr)
 {
 	int err;
 
-	__put_user_size(instr, addr, 4, err);
+	__put_user_size(instr, patch_addr, 4, err);
 	if (err)
 		return err;
 
-	asm ("dcbst 0, %0; sync; icbi 0,%0; sync; isync" :: "r" (addr));
+	asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
+							    "r" (exec_addr));
 
 	return 0;
 }
 
+int raw_patch_instruction(unsigned int *addr, unsigned int instr)
+{
+	return __patch_instruction(addr, instr, addr);
+}
+
 #ifdef CONFIG_STRICT_KERNEL_RWX
 static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
 
@@ -138,7 +145,7 @@ static inline int unmap_patch_area(unsigned long addr)
 int patch_instruction(unsigned int *addr, unsigned int instr)
 {
 	int err;
-	unsigned int *dest = NULL;
+	unsigned int *patch_addr = NULL;
 	unsigned long flags;
 	unsigned long text_poke_addr;
 	unsigned long kaddr = (unsigned long)addr;
@@ -148,8 +155,8 @@ int patch_instruction(unsigned int *addr, unsigned int instr)
 	 * when text_poke_area is not ready, but we still need
 	 * to allow patching. We just do the plain old patching
 	 */
-	if (!this_cpu_read(*PTRRELOC(&text_poke_area)))
-		return __patch_instruction(addr, instr);
+	if (!this_cpu_read(text_poke_area))
+		return raw_patch_instruction(addr, instr);
 
 	local_irq_save(flags);
 
@@ -159,17 +166,10 @@ int patch_instruction(unsigned int *addr, unsigned int instr)
 		goto out;
 	}
 
-	dest = (unsigned int *)(text_poke_addr) +
+	patch_addr = (unsigned int *)(text_poke_addr) +
 			((kaddr & ~PAGE_MASK) / sizeof(unsigned int));
 
-	/*
-	 * We use __put_user_size so that we can handle faults while
-	 * writing to dest and return err to handle faults gracefully
-	 */
-	__put_user_size(instr, dest, 4, err);
-	if (!err)
-		asm ("dcbst 0, %0; sync; icbi 0,%0; icbi 0,%1; sync; isync"
-			::"r" (dest), "r"(addr));
+	__patch_instruction(addr, instr, patch_addr);
 
 	err = unmap_patch_area(text_poke_addr);
 	if (err)
@@ -184,7 +184,7 @@ int patch_instruction(unsigned int *addr, unsigned int instr)
 
 int patch_instruction(unsigned int *addr, unsigned int instr)
 {
-	return __patch_instruction(addr, instr);
+	return raw_patch_instruction(addr, instr);
 }
 
 #endif /* CONFIG_STRICT_KERNEL_RWX */
@@ -302,6 +302,11 @@ int instr_is_relative_branch(unsigned int instr)
 	return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
 }
 
+int instr_is_relative_link_branch(unsigned int instr)
+{
+	return instr_is_relative_branch(instr) && (instr & BRANCH_SET_LINK);
+}
+
 static unsigned long branch_iform_target(const unsigned int *instr)
 {
 	signed long imm;
diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c
index a95ea007d654..73697c4e3468 100644
--- a/arch/powerpc/lib/feature-fixups.c
+++ b/arch/powerpc/lib/feature-fixups.c
@@ -62,7 +62,7 @@ static int patch_alt_instruction(unsigned int *src, unsigned int *dest,
 		}
 	}
 
-	patch_instruction(dest, instr);
+	raw_patch_instruction(dest, instr);
 
 	return 0;
 }
@@ -91,7 +91,7 @@ static int patch_feature_section(unsigned long value, struct fixup_entry *fcur)
 	}
 
 	for (; dest < end; dest++)
-		patch_instruction(dest, PPC_INST_NOP);
+		raw_patch_instruction(dest, PPC_INST_NOP);
 
 	return 0;
 }
@@ -170,7 +170,7 @@ void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end)
 
 	for (; start < end; start++) {
 		dest = (void *)start + *start;
-		patch_instruction(dest, PPC_INST_LWSYNC);
+		raw_patch_instruction(dest, PPC_INST_LWSYNC);
 	}
 }
 
@@ -188,7 +188,7 @@ static void do_final_fixups(void)
 	length = (__end_interrupts - _stext) / sizeof(int);
 
 	while (length--) {
-		patch_instruction(dest, *src);
+		raw_patch_instruction(dest, *src);
 		src++;
 		dest++;
 	}
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index f29212e40f40..849f50cd62f2 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -67,7 +67,7 @@ void __init MMU_init_hw(void)
 	/* PIN up to the 3 first 8Mb after IMMR in DTLB table */
 #ifdef CONFIG_PIN_TLB_DATA
 	unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000;
-	unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY;
+	unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY;
 #ifdef CONFIG_PIN_TLB_IMMR
 	int i = 29;
 #else
@@ -79,7 +79,7 @@ void __init MMU_init_hw(void)
 	for (; i < 32 && mem >= LARGE_PAGE_SIZE_8M; i++) {
 		mtspr(SPRN_MD_CTR, ctr | (i << 8));
 		mtspr(SPRN_MD_EPN, (unsigned long)__va(addr) | MD_EVALID);
-		mtspr(SPRN_MD_TWC, MD_PS8MEG | MD_SVALID);
+		mtspr(SPRN_MD_TWC, MD_PS8MEG | MD_SVALID | M_APG2);
 		mtspr(SPRN_MD_RPN, addr | flags | _PAGE_PRESENT);
 		addr += LARGE_PAGE_SIZE_8M;
 		mem -= LARGE_PAGE_SIZE_8M;
diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile
index 76a6b057d454..f06f3577d8d1 100644
--- a/arch/powerpc/mm/Makefile
+++ b/arch/powerpc/mm/Makefile
@@ -9,7 +9,7 @@ ccflags-$(CONFIG_PPC64)	:= $(NO_MINIMAL_TOC)
 
 obj-y				:= fault.o mem.o pgtable.o mmap.o \
 				   init_$(BITS).o pgtable_$(BITS).o \
-				   init-common.o mmu_context.o
+				   init-common.o mmu_context.o drmem.o
 obj-$(CONFIG_PPC_MMU_NOHASH)	+= mmu_context_nohash.o tlb_nohash.o \
 				   tlb_nohash_low.o
 obj-$(CONFIG_PPC_BOOK3E)	+= tlb_low_$(BITS)e.o
@@ -44,3 +44,4 @@ obj-$(CONFIG_PPC_COPRO_BASE)	+= copro_fault.o
 obj-$(CONFIG_SPAPR_TCE_IOMMU)	+= mmu_context_iommu.o
 obj-$(CONFIG_PPC_PTDUMP)	+= dump_linuxpagetables.o
 obj-$(CONFIG_PPC_HTDUMP)	+= dump_hashpagetable.o
+obj-$(CONFIG_PPC_MEM_KEYS)	+= pkeys.o
diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c
new file mode 100644
index 000000000000..3f1803672c9b
--- /dev/null
+++ b/arch/powerpc/mm/drmem.c
@@ -0,0 +1,447 @@
+/*
+ * Dynamic reconfiguration memory support
+ *
+ * Copyright 2017 IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#define pr_fmt(fmt) "drmem: " fmt
+
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/memblock.h>
+#include <asm/prom.h>
+#include <asm/drmem.h>
+
+static struct drmem_lmb_info __drmem_info;
+struct drmem_lmb_info *drmem_info = &__drmem_info;
+
+u64 drmem_lmb_memory_max(void)
+{
+	struct drmem_lmb *last_lmb;
+
+	last_lmb = &drmem_info->lmbs[drmem_info->n_lmbs - 1];
+	return last_lmb->base_addr + drmem_lmb_size();
+}
+
+static u32 drmem_lmb_flags(struct drmem_lmb *lmb)
+{
+	/*
+	 * Return the value of the lmb flags field minus the reserved
+	 * bit used internally for hotplug processing.
+	 */
+	return lmb->flags & ~DRMEM_LMB_RESERVED;
+}
+
+static struct property *clone_property(struct property *prop, u32 prop_sz)
+{
+	struct property *new_prop;
+
+	new_prop = kzalloc(sizeof(*new_prop), GFP_KERNEL);
+	if (!new_prop)
+		return NULL;
+
+	new_prop->name = kstrdup(prop->name, GFP_KERNEL);
+	new_prop->value = kzalloc(prop_sz, GFP_KERNEL);
+	if (!new_prop->name || !new_prop->value) {
+		kfree(new_prop->name);
+		kfree(new_prop->value);
+		kfree(new_prop);
+		return NULL;
+	}
+
+	new_prop->length = prop_sz;
+#if defined(CONFIG_OF_DYNAMIC)
+	of_property_set_flag(new_prop, OF_DYNAMIC);
+#endif
+	return new_prop;
+}
+
+static int drmem_update_dt_v1(struct device_node *memory,
+			      struct property *prop)
+{
+	struct property *new_prop;
+	struct of_drconf_cell_v1 *dr_cell;
+	struct drmem_lmb *lmb;
+	u32 *p;
+
+	new_prop = clone_property(prop, prop->length);
+	if (!new_prop)
+		return -1;
+
+	p = new_prop->value;
+	*p++ = cpu_to_be32(drmem_info->n_lmbs);
+
+	dr_cell = (struct of_drconf_cell_v1 *)p;
+
+	for_each_drmem_lmb(lmb) {
+		dr_cell->base_addr = cpu_to_be64(lmb->base_addr);
+		dr_cell->drc_index = cpu_to_be32(lmb->drc_index);
+		dr_cell->aa_index = cpu_to_be32(lmb->aa_index);
+		dr_cell->flags = cpu_to_be32(drmem_lmb_flags(lmb));
+
+		dr_cell++;
+	}
+
+	of_update_property(memory, new_prop);
+	return 0;
+}
+
+static void init_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,
+				struct drmem_lmb *lmb)
+{
+	dr_cell->base_addr = cpu_to_be64(lmb->base_addr);
+	dr_cell->drc_index = cpu_to_be32(lmb->drc_index);
+	dr_cell->aa_index = cpu_to_be32(lmb->aa_index);
+	dr_cell->flags = cpu_to_be32(drmem_lmb_flags(lmb));
+}
+
+static int drmem_update_dt_v2(struct device_node *memory,
+			      struct property *prop)
+{
+	struct property *new_prop;
+	struct of_drconf_cell_v2 *dr_cell;
+	struct drmem_lmb *lmb, *prev_lmb;
+	u32 lmb_sets, prop_sz, seq_lmbs;
+	u32 *p;
+
+	/* First pass, determine how many LMB sets are needed. */
+	lmb_sets = 0;
+	prev_lmb = NULL;
+	for_each_drmem_lmb(lmb) {
+		if (!prev_lmb) {
+			prev_lmb = lmb;
+			lmb_sets++;
+			continue;
+		}
+
+		if (prev_lmb->aa_index != lmb->aa_index ||
+		    drmem_lmb_flags(prev_lmb) != drmem_lmb_flags(lmb))
+			lmb_sets++;
+
+		prev_lmb = lmb;
+	}
+
+	prop_sz = lmb_sets * sizeof(*dr_cell) + sizeof(__be32);
+	new_prop = clone_property(prop, prop_sz);
+	if (!new_prop)
+		return -1;
+
+	p = new_prop->value;
+	*p++ = cpu_to_be32(lmb_sets);
+
+	dr_cell = (struct of_drconf_cell_v2 *)p;
+
+	/* Second pass, populate the LMB set data */
+	prev_lmb = NULL;
+	seq_lmbs = 0;
+	for_each_drmem_lmb(lmb) {
+		if (prev_lmb == NULL) {
+			/* Start of first LMB set */
+			prev_lmb = lmb;
+			init_drconf_v2_cell(dr_cell, lmb);
+			seq_lmbs++;
+			continue;
+		}
+
+		if (prev_lmb->aa_index != lmb->aa_index ||
+		    drmem_lmb_flags(prev_lmb) != drmem_lmb_flags(lmb)) {
+			/* end of one set, start of another */
+			dr_cell->seq_lmbs = cpu_to_be32(seq_lmbs);
+			dr_cell++;
+
+			init_drconf_v2_cell(dr_cell, lmb);
+			seq_lmbs = 1;
+		} else {
+			seq_lmbs++;
+		}
+
+		prev_lmb = lmb;
+	}
+
+	/* close out last LMB set */
+	dr_cell->seq_lmbs = cpu_to_be32(seq_lmbs);
+	of_update_property(memory, new_prop);
+	return 0;
+}
+
+int drmem_update_dt(void)
+{
+	struct device_node *memory;
+	struct property *prop;
+	int rc = -1;
+
+	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
+	if (!memory)
+		return -1;
+
+	prop = of_find_property(memory, "ibm,dynamic-memory", NULL);
+	if (prop) {
+		rc = drmem_update_dt_v1(memory, prop);
+	} else {
+		prop = of_find_property(memory, "ibm,dynamic-memory-v2", NULL);
+		if (prop)
+			rc = drmem_update_dt_v2(memory, prop);
+	}
+
+	of_node_put(memory);
+	return rc;
+}
+
+static void __init read_drconf_v1_cell(struct drmem_lmb *lmb,
+				       const __be32 **prop)
+{
+	const __be32 *p = *prop;
+
+	lmb->base_addr = dt_mem_next_cell(dt_root_addr_cells, &p);
+	lmb->drc_index = of_read_number(p++, 1);
+
+	p++; /* skip reserved field */
+
+	lmb->aa_index = of_read_number(p++, 1);
+	lmb->flags = of_read_number(p++, 1);
+
+	*prop = p;
+}
+
+static void __init __walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm,
+			void (*func)(struct drmem_lmb *, const __be32 **))
+{
+	struct drmem_lmb lmb;
+	u32 i, n_lmbs;
+
+	n_lmbs = of_read_number(prop++, 1);
+	if (n_lmbs == 0)
+		return;
+
+	for (i = 0; i < n_lmbs; i++) {
+		read_drconf_v1_cell(&lmb, &prop);
+		func(&lmb, &usm);
+	}
+}
+
+static void __init read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell,
+				       const __be32 **prop)
+{
+	const __be32 *p = *prop;
+
+	dr_cell->seq_lmbs = of_read_number(p++, 1);
+	dr_cell->base_addr = dt_mem_next_cell(dt_root_addr_cells, &p);
+	dr_cell->drc_index = of_read_number(p++, 1);
+	dr_cell->aa_index = of_read_number(p++, 1);
+	dr_cell->flags = of_read_number(p++, 1);
+
+	*prop = p;
+}
+
+static void __init __walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *usm,
+			void (*func)(struct drmem_lmb *, const __be32 **))
+{
+	struct of_drconf_cell_v2 dr_cell;
+	struct drmem_lmb lmb;
+	u32 i, j, lmb_sets;
+
+	lmb_sets = of_read_number(prop++, 1);
+	if (lmb_sets == 0)
+		return;
+
+	for (i = 0; i < lmb_sets; i++) {
+		read_drconf_v2_cell(&dr_cell, &prop);
+
+		for (j = 0; j < dr_cell.seq_lmbs; j++) {
+			lmb.base_addr = dr_cell.base_addr;
+			dr_cell.base_addr += drmem_lmb_size();
+
+			lmb.drc_index = dr_cell.drc_index;
+			dr_cell.drc_index++;
+
+			lmb.aa_index = dr_cell.aa_index;
+			lmb.flags = dr_cell.flags;
+
+			func(&lmb, &usm);
+		}
+	}
+}
+
+#ifdef CONFIG_PPC_PSERIES
+void __init walk_drmem_lmbs_early(unsigned long node,
+			void (*func)(struct drmem_lmb *, const __be32 **))
+{
+	const __be32 *prop, *usm;
+	int len;
+
+	prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
+	if (!prop || len < dt_root_size_cells * sizeof(__be32))
+		return;
+
+	drmem_info->lmb_size = dt_mem_next_cell(dt_root_size_cells, &prop);
+
+	usm = of_get_flat_dt_prop(node, "linux,drconf-usable-memory", &len);
+
+	prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory", &len);
+	if (prop) {
+		__walk_drmem_v1_lmbs(prop, usm, func);
+	} else {
+		prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory-v2",
+					   &len);
+		if (prop)
+			__walk_drmem_v2_lmbs(prop, usm, func);
+	}
+
+	memblock_dump_all();
+}
+
+#endif
+
+static int __init init_drmem_lmb_size(struct device_node *dn)
+{
+	const __be32 *prop;
+	int len;
+
+	if (drmem_info->lmb_size)
+		return 0;
+
+	prop = of_get_property(dn, "ibm,lmb-size", &len);
+	if (!prop || len < dt_root_size_cells * sizeof(__be32)) {
+		pr_info("Could not determine LMB size\n");
+		return -1;
+	}
+
+	drmem_info->lmb_size = dt_mem_next_cell(dt_root_size_cells, &prop);
+	return 0;
+}
+
+/*
+ * Returns the property linux,drconf-usable-memory if
+ * it exists (the property exists only in kexec/kdump kernels,
+ * added by kexec-tools)
+ */
+static const __be32 *of_get_usable_memory(struct device_node *dn)
+{
+	const __be32 *prop;
+	u32 len;
+
+	prop = of_get_property(dn, "linux,drconf-usable-memory", &len);
+	if (!prop || len < sizeof(unsigned int))
+		return NULL;
+
+	return prop;
+}
+
+void __init walk_drmem_lmbs(struct device_node *dn,
+			    void (*func)(struct drmem_lmb *, const __be32 **))
+{
+	const __be32 *prop, *usm;
+
+	if (init_drmem_lmb_size(dn))
+		return;
+
+	usm = of_get_usable_memory(dn);
+
+	prop = of_get_property(dn, "ibm,dynamic-memory", NULL);
+	if (prop) {
+		__walk_drmem_v1_lmbs(prop, usm, func);
+	} else {
+		prop = of_get_property(dn, "ibm,dynamic-memory-v2", NULL);
+		if (prop)
+			__walk_drmem_v2_lmbs(prop, usm, func);
+	}
+}
+
+static void __init init_drmem_v1_lmbs(const __be32 *prop)
+{
+	struct drmem_lmb *lmb;
+
+	drmem_info->n_lmbs = of_read_number(prop++, 1);
+	if (drmem_info->n_lmbs == 0)
+		return;
+
+	drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
+				   GFP_KERNEL);
+	if (!drmem_info->lmbs)
+		return;
+
+	for_each_drmem_lmb(lmb)
+		read_drconf_v1_cell(lmb, &prop);
+}
+
+static void __init init_drmem_v2_lmbs(const __be32 *prop)
+{
+	struct drmem_lmb *lmb;
+	struct of_drconf_cell_v2 dr_cell;
+	const __be32 *p;
+	u32 i, j, lmb_sets;
+	int lmb_index;
+
+	lmb_sets = of_read_number(prop++, 1);
+	if (lmb_sets == 0)
+		return;
+
+	/* first pass, calculate the number of LMBs */
+	p = prop;
+	for (i = 0; i < lmb_sets; i++) {
+		read_drconf_v2_cell(&dr_cell, &p);
+		drmem_info->n_lmbs += dr_cell.seq_lmbs;
+	}
+
+	drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb),
+				   GFP_KERNEL);
+	if (!drmem_info->lmbs)
+		return;
+
+	/* second pass, read in the LMB information */
+	lmb_index = 0;
+	p = prop;
+
+	for (i = 0; i < lmb_sets; i++) {
+		read_drconf_v2_cell(&dr_cell, &p);
+
+		for (j = 0; j < dr_cell.seq_lmbs; j++) {
+			lmb = &drmem_info->lmbs[lmb_index++];
+
+			lmb->base_addr = dr_cell.base_addr;
+			dr_cell.base_addr += drmem_info->lmb_size;
+
+			lmb->drc_index = dr_cell.drc_index;
+			dr_cell.drc_index++;
+
+			lmb->aa_index = dr_cell.aa_index;
+			lmb->flags = dr_cell.flags;
+		}
+	}
+}
+
+static int __init drmem_init(void)
+{
+	struct device_node *dn;
+	const __be32 *prop;
+
+	dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
+	if (!dn) {
+		pr_info("No dynamic reconfiguration memory found\n");
+		return 0;
+	}
+
+	if (init_drmem_lmb_size(dn)) {
+		of_node_put(dn);
+		return 0;
+	}
+
+	prop = of_get_property(dn, "ibm,dynamic-memory", NULL);
+	if (prop) {
+		init_drmem_v1_lmbs(prop);
+	} else {
+		prop = of_get_property(dn, "ibm,dynamic-memory-v2", NULL);
+		if (prop)
+			init_drmem_v2_lmbs(prop);
+	}
+
+	of_node_put(dn);
+	return 0;
+}
+late_initcall(drmem_init);
diff --git a/arch/powerpc/mm/dump_linuxpagetables.c b/arch/powerpc/mm/dump_linuxpagetables.c
index c2e7dea59490..876e2a3c79f2 100644
--- a/arch/powerpc/mm/dump_linuxpagetables.c
+++ b/arch/powerpc/mm/dump_linuxpagetables.c
@@ -112,26 +112,25 @@ struct flag_info {
 
 static const struct flag_info flag_array[] = {
 	{
-#ifdef CONFIG_PPC_BOOK3S_64
-		.mask	= _PAGE_PRIVILEGED,
-		.val	= 0,
-#else
-		.mask	= _PAGE_USER,
+		.mask	= _PAGE_USER | _PAGE_PRIVILEGED,
 		.val	= _PAGE_USER,
-#endif
 		.set	= "user",
 		.clear	= "    ",
 	}, {
-#if _PAGE_RO == 0
-		.mask	= _PAGE_RW,
+		.mask	= _PAGE_RW | _PAGE_RO | _PAGE_NA,
 		.val	= _PAGE_RW,
-#else
-		.mask	= _PAGE_RO,
-		.val	= 0,
-#endif
 		.set	= "rw",
-		.clear	= "ro",
 	}, {
+		.mask	= _PAGE_RW | _PAGE_RO | _PAGE_NA,
+		.val	= _PAGE_RO,
+		.set	= "ro",
+	}, {
+#if _PAGE_NA != 0
+		.mask	= _PAGE_RW | _PAGE_RO | _PAGE_NA,
+		.val	= _PAGE_RO,
+		.set	= "na",
+	}, {
+#endif
 		.mask	= _PAGE_EXEC,
 		.val	= _PAGE_EXEC,
 		.set	= " X ",
@@ -213,7 +212,7 @@ static const struct flag_info flag_array[] = {
 		.val	= H_PAGE_4K_PFN,
 		.set	= "4K_pfn",
 	}, {
-#endif
+#else /* CONFIG_PPC_64K_PAGES */
 		.mask	= H_PAGE_F_GIX,
 		.val	= H_PAGE_F_GIX,
 		.set	= "f_gix",
@@ -224,14 +223,11 @@ static const struct flag_info flag_array[] = {
 		.val	= H_PAGE_F_SECOND,
 		.set	= "f_second",
 	}, {
+#endif /* CONFIG_PPC_64K_PAGES */
 #endif
 		.mask	= _PAGE_SPECIAL,
 		.val	= _PAGE_SPECIAL,
 		.set	= "special",
-	}, {
-		.mask	= _PAGE_SHARED,
-		.val	= _PAGE_SHARED,
-		.set	= "shared",
 	}
 };
 
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 6e1e39035380..866446cf2d9a 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -107,7 +107,8 @@ static bool store_updates_sp(struct pt_regs *regs)
  */
 
 static int
-__bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code)
+__bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code,
+		int pkey)
 {
 	/*
 	 * If we are in kernel mode, bail out with a SEGV, this will
@@ -117,17 +118,18 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code)
 	if (!user_mode(regs))
 		return SIGSEGV;
 
-	_exception(SIGSEGV, regs, si_code, address);
+	_exception_pkey(SIGSEGV, regs, si_code, address, pkey);
 
 	return 0;
 }
 
 static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long address)
 {
-	return __bad_area_nosemaphore(regs, address, SEGV_MAPERR);
+	return __bad_area_nosemaphore(regs, address, SEGV_MAPERR, 0);
 }
 
-static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code)
+static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code,
+			int pkey)
 {
 	struct mm_struct *mm = current->mm;
 
@@ -137,17 +139,23 @@ static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code)
 	 */
 	up_read(&mm->mmap_sem);
 
-	return __bad_area_nosemaphore(regs, address, si_code);
+	return __bad_area_nosemaphore(regs, address, si_code, pkey);
 }
 
 static noinline int bad_area(struct pt_regs *regs, unsigned long address)
 {
-	return __bad_area(regs, address, SEGV_MAPERR);
+	return __bad_area(regs, address, SEGV_MAPERR, 0);
+}
+
+static int bad_key_fault_exception(struct pt_regs *regs, unsigned long address,
+				    int pkey)
+{
+	return __bad_area_nosemaphore(regs, address, SEGV_PKUERR, pkey);
 }
 
 static noinline int bad_access(struct pt_regs *regs, unsigned long address)
 {
-	return __bad_area(regs, address, SEGV_ACCERR);
+	return __bad_area(regs, address, SEGV_ACCERR, 0);
 }
 
 static int do_sigbus(struct pt_regs *regs, unsigned long address,
@@ -432,6 +440,10 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
 
 	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 
+	if (error_code & DSISR_KEYFAULT)
+		return bad_key_fault_exception(regs, address,
+					       get_mm_addr_key(mm, address));
+
 	/*
 	 * We want to do this outside mmap_sem, because reading code around nip
 	 * can result in fault, which will cause a deadlock when called with
@@ -503,6 +515,31 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
 	 * the fault.
 	 */
 	fault = handle_mm_fault(vma, address, flags);
+
+#ifdef CONFIG_PPC_MEM_KEYS
+	/*
+	 * if the HPTE is not hashed, hardware will not detect
+	 * a key fault. Lets check if we failed because of a
+	 * software detected key fault.
+	 */
+	if (unlikely(fault & VM_FAULT_SIGSEGV) &&
+		!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
+			is_exec, 0)) {
+		/*
+		 * The PGD-PDT...PMD-PTE tree may not have been fully setup.
+		 * Hence we cannot walk the tree to locate the PTE, to locate
+		 * the key. Hence let's use vma_pkey() to get the key; instead
+		 * of get_mm_addr_key().
+		 */
+		int pkey = vma_pkey(vma);
+
+		if (likely(pkey)) {
+			up_read(&mm->mmap_sem);
+			return bad_key_fault_exception(regs, address, pkey);
+		}
+	}
+#endif /* CONFIG_PPC_MEM_KEYS */
+
 	major |= fault & VM_FAULT_MAJOR;
 
 	/*
@@ -576,7 +613,7 @@ void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
 
 	/* kernel has accessed a bad area */
 
-	switch (regs->trap) {
+	switch (TRAP(regs)) {
 	case 0x300:
 	case 0x380:
 		printk(KERN_ALERT "Unable to handle kernel paging request for "
diff --git a/arch/powerpc/mm/hash64_4k.c b/arch/powerpc/mm/hash64_4k.c
index 6fa450c12d6d..d573d7d07f25 100644
--- a/arch/powerpc/mm/hash64_4k.c
+++ b/arch/powerpc/mm/hash64_4k.c
@@ -20,6 +20,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 		   pte_t *ptep, unsigned long trap, unsigned long flags,
 		   int ssize, int subpg_prot)
 {
+	real_pte_t rpte;
 	unsigned long hpte_group;
 	unsigned long rflags, pa;
 	unsigned long old_pte, new_pte;
@@ -54,6 +55,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 	 * need to add in 0x1 if it's a read-only user page
 	 */
 	rflags = htab_convert_pte_flags(new_pte);
+	rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
 
 	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
 	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
@@ -64,13 +66,10 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 		/*
 		 * There MIGHT be an HPTE for this pte
 		 */
-		hash = hpt_hash(vpn, shift, ssize);
-		if (old_pte & H_PAGE_F_SECOND)
-			hash = ~hash;
-		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
-		slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
+		unsigned long gslot = pte_get_hash_gslot(vpn, shift, ssize,
+							 rpte, 0);
 
-		if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_4K,
+		if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, MMU_PAGE_4K,
 					       MMU_PAGE_4K, ssize, flags) == -1)
 			old_pte &= ~_PAGE_HPTEFLAGS;
 	}
@@ -118,8 +117,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 			return -1;
 		}
 		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
-		new_pte |= (slot << H_PAGE_F_GIX_SHIFT) &
-			(H_PAGE_F_SECOND | H_PAGE_F_GIX);
+		new_pte |= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
 	}
 	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
 	return 0;
diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c
index 1a68cb19b0e3..e601d95c3b20 100644
--- a/arch/powerpc/mm/hash64_64k.c
+++ b/arch/powerpc/mm/hash64_64k.c
@@ -15,34 +15,22 @@
 #include <linux/mm.h>
 #include <asm/machdep.h>
 #include <asm/mmu.h>
+
 /*
- * index from 0 - 15
+ * Return true, if the entry has a slot value which
+ * the software considers as invalid.
  */
-bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
+static inline bool hpte_soft_invalid(unsigned long hidx)
 {
-	unsigned long g_idx;
-	unsigned long ptev = pte_val(rpte.pte);
-
-	g_idx = (ptev & H_PAGE_COMBO_VALID) >> H_PAGE_F_GIX_SHIFT;
-	index = index >> 2;
-	if (g_idx & (0x1 << index))
-		return true;
-	else
-		return false;
+	return ((hidx & 0xfUL) == 0xfUL);
 }
+
 /*
  * index from 0 - 15
  */
-static unsigned long mark_subptegroup_valid(unsigned long ptev, unsigned long index)
+bool __rpte_sub_valid(real_pte_t rpte, unsigned long index)
 {
-	unsigned long g_idx;
-
-	if (!(ptev & H_PAGE_COMBO))
-		return ptev;
-	index = index >> 2;
-	g_idx = 0x1 << index;
-
-	return ptev | (g_idx << H_PAGE_F_GIX_SHIFT);
+	return !(hpte_soft_invalid(__rpte_to_hidx(rpte, index)));
 }
 
 int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
@@ -50,12 +38,11 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 		   int ssize, int subpg_prot)
 {
 	real_pte_t rpte;
-	unsigned long *hidxp;
 	unsigned long hpte_group;
 	unsigned int subpg_index;
-	unsigned long rflags, pa, hidx;
+	unsigned long rflags, pa;
 	unsigned long old_pte, new_pte, subpg_pte;
-	unsigned long vpn, hash, slot;
+	unsigned long vpn, hash, slot, gslot;
 	unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift;
 
 	/*
@@ -99,7 +86,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 
 	subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
 	vpn  = hpt_vpn(ea, vsid, ssize);
-	rpte = __real_pte(__pte(old_pte), ptep);
+	rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
 	/*
 	 *None of the sub 4k page is hashed
 	 */
@@ -116,8 +103,8 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 		 * On hash insert failure we use old pte value and we don't
 		 * want slot information there if we have a insert failure.
 		 */
-		old_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND);
-		new_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND);
+		old_pte &= ~H_PAGE_HASHPTE;
+		new_pte &= ~H_PAGE_HASHPTE;
 		goto htab_insert_hpte;
 	}
 	/*
@@ -126,18 +113,14 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 	if (__rpte_sub_valid(rpte, subpg_index)) {
 		int ret;
 
-		hash = hpt_hash(vpn, shift, ssize);
-		hidx = __rpte_to_hidx(rpte, subpg_index);
-		if (hidx & _PTEIDX_SECONDARY)
-			hash = ~hash;
-		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
-		slot += hidx & _PTEIDX_GROUP_IX;
-
-		ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn,
+		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte,
+					   subpg_index);
+		ret = mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn,
 						 MMU_PAGE_4K, MMU_PAGE_4K,
 						 ssize, flags);
+
 		/*
-		 *if we failed because typically the HPTE wasn't really here
+		 * If we failed because typically the HPTE wasn't really here
 		 * we try an insertion.
 		 */
 		if (ret == -1)
@@ -148,6 +131,14 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 	}
 
 htab_insert_hpte:
+
+	/*
+	 * Initialize all hidx entries to invalid value, the first time
+	 * the PTE is about to allocate a 4K HPTE.
+	 */
+	if (!(old_pte & H_PAGE_COMBO))
+		rpte.hidx = INVALID_RPTE_HIDX;
+
 	/*
 	 * handle H_PAGE_4K_PFN case
 	 */
@@ -172,15 +163,39 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 	 * Primary is full, try the secondary
 	 */
 	if (unlikely(slot == -1)) {
+		bool soft_invalid;
+
 		hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL;
 		slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa,
 						rflags, HPTE_V_SECONDARY,
 						MMU_PAGE_4K, MMU_PAGE_4K,
 						ssize);
-		if (slot == -1) {
-			if (mftb() & 0x1)
+
+		soft_invalid = hpte_soft_invalid(slot);
+		if (unlikely(soft_invalid)) {
+			/*
+			 * We got a valid slot from a hardware point of view.
+			 * but we cannot use it, because we use this special
+			 * value; as defined by hpte_soft_invalid(), to track
+			 * invalid slots. We cannot use it. So invalidate it.
+			 */
+			gslot = slot & _PTEIDX_GROUP_IX;
+			mmu_hash_ops.hpte_invalidate(hpte_group + gslot, vpn,
+						     MMU_PAGE_4K, MMU_PAGE_4K,
+						     ssize, 0);
+		}
+
+		if (unlikely(slot == -1 || soft_invalid)) {
+			/*
+			 * For soft invalid slot, let's ensure that we release a
+			 * slot from the primary, with the hope that we will
+			 * acquire that slot next time we try. This will ensure
+			 * that we do not get the same soft-invalid slot.
+			 */
+			if (soft_invalid || (mftb() & 0x1))
 				hpte_group = ((hash & htab_hash_mask) *
 					      HPTES_PER_GROUP) & ~0x7UL;
+
 			mmu_hash_ops.hpte_remove(hpte_group);
 			/*
 			 * FIXME!! Should be try the group from which we removed ?
@@ -198,21 +213,10 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 				   MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
 		return -1;
 	}
-	/*
-	 * Insert slot number & secondary bit in PTE second half,
-	 * clear H_PAGE_BUSY and set appropriate HPTE slot bit
-	 * Since we have H_PAGE_BUSY set on ptep, we can be sure
-	 * nobody is undating hidx.
-	 */
-	hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
-	rpte.hidx &= ~(0xfUL << (subpg_index << 2));
-	*hidxp = rpte.hidx  | (slot << (subpg_index << 2));
-	new_pte = mark_subptegroup_valid(new_pte, subpg_index);
-	new_pte |=  H_PAGE_HASHPTE;
-	/*
-	 * check __real_pte for details on matching smp_rmb()
-	 */
-	smp_wmb();
+
+	new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot, PTRS_PER_PTE);
+	new_pte |= H_PAGE_HASHPTE;
+
 	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
 	return 0;
 }
@@ -221,6 +225,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
 		    unsigned long vsid, pte_t *ptep, unsigned long trap,
 		    unsigned long flags, int ssize)
 {
+	real_pte_t rpte;
 	unsigned long hpte_group;
 	unsigned long rflags, pa;
 	unsigned long old_pte, new_pte;
@@ -257,6 +262,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
 	} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
 
 	rflags = htab_convert_pte_flags(new_pte);
+	rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
 
 	if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
 	    !cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
@@ -264,16 +270,13 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
 
 	vpn  = hpt_vpn(ea, vsid, ssize);
 	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
+		unsigned long gslot;
+
 		/*
 		 * There MIGHT be an HPTE for this pte
 		 */
-		hash = hpt_hash(vpn, shift, ssize);
-		if (old_pte & H_PAGE_F_SECOND)
-			hash = ~hash;
-		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
-		slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
-
-		if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K,
+		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
+		if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, MMU_PAGE_64K,
 					       MMU_PAGE_64K, ssize,
 					       flags) == -1)
 			old_pte &= ~_PAGE_HPTEFLAGS;
@@ -322,9 +325,9 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
 					   MMU_PAGE_64K, MMU_PAGE_64K, old_pte);
 			return -1;
 		}
+
 		new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
-		new_pte |= (slot << H_PAGE_F_GIX_SHIFT) &
-			(H_PAGE_F_SECOND | H_PAGE_F_GIX);
+		new_pte |= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
 	}
 	*ptep = __pte(new_pte & ~H_PAGE_BUSY);
 	return 0;
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 640cf566e986..656933c85925 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -47,6 +47,103 @@
 
 DEFINE_RAW_SPINLOCK(native_tlbie_lock);
 
+static inline void tlbiel_hash_set_isa206(unsigned int set, unsigned int is)
+{
+	unsigned long rb;
+
+	rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
+
+	asm volatile("tlbiel %0" : : "r" (rb));
+}
+
+/*
+ * tlbiel instruction for hash, set invalidation
+ * i.e., r=1 and is=01 or is=10 or is=11
+ */
+static inline void tlbiel_hash_set_isa300(unsigned int set, unsigned int is,
+					unsigned int pid,
+					unsigned int ric, unsigned int prs)
+{
+	unsigned long rb;
+	unsigned long rs;
+	unsigned int r = 0; /* hash format */
+
+	rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
+	rs = ((unsigned long)pid << PPC_BITLSHIFT(31));
+
+	asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4)
+		     : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r)
+		     : "memory");
+}
+
+
+static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is)
+{
+	unsigned int set;
+
+	asm volatile("ptesync": : :"memory");
+
+	for (set = 0; set < num_sets; set++)
+		tlbiel_hash_set_isa206(set, is);
+
+	asm volatile("ptesync": : :"memory");
+}
+
+static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)
+{
+	unsigned int set;
+
+	asm volatile("ptesync": : :"memory");
+
+	/*
+	 * Flush the first set of the TLB, and any caching of partition table
+	 * entries. Then flush the remaining sets of the TLB. Hash mode uses
+	 * partition scoped TLB translations.
+	 */
+	tlbiel_hash_set_isa300(0, is, 0, 2, 0);
+	for (set = 1; set < num_sets; set++)
+		tlbiel_hash_set_isa300(set, is, 0, 0, 0);
+
+	/*
+	 * Now invalidate the process table cache.
+	 *
+	 * From ISA v3.0B p. 1078:
+	 *     The following forms are invalid.
+	 *      * PRS=1, R=0, and RIC!=2 (The only process-scoped
+	 *        HPT caching is of the Process Table.)
+	 */
+	tlbiel_hash_set_isa300(0, is, 0, 2, 1);
+
+	asm volatile("ptesync": : :"memory");
+}
+
+void hash__tlbiel_all(unsigned int action)
+{
+	unsigned int is;
+
+	switch (action) {
+	case TLB_INVAL_SCOPE_GLOBAL:
+		is = 3;
+		break;
+	case TLB_INVAL_SCOPE_LPID:
+		is = 2;
+		break;
+	default:
+		BUG();
+	}
+
+	if (early_cpu_has_feature(CPU_FTR_ARCH_300))
+		tlbiel_all_isa300(POWER9_TLB_SETS_HASH, is);
+	else if (early_cpu_has_feature(CPU_FTR_ARCH_207S))
+		tlbiel_all_isa206(POWER8_TLB_SETS, is);
+	else if (early_cpu_has_feature(CPU_FTR_ARCH_206))
+		tlbiel_all_isa206(POWER7_TLB_SETS, is);
+	else
+		WARN(1, "%s called on pre-POWER7 CPU\n", __func__);
+
+	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
+}
+
 static inline unsigned long  ___tlbie(unsigned long vpn, int psize,
 						int apsize, int ssize)
 {
@@ -104,6 +201,15 @@ static inline unsigned long  ___tlbie(unsigned long vpn, int psize,
 	return va;
 }
 
+static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)
+{
+	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+		/* Need the extra ptesync to ensure we don't reorder tlbie*/
+		asm volatile("ptesync": : :"memory");
+		___tlbie(vpn, psize, apsize, ssize);
+	}
+}
+
 static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
 {
 	unsigned long rb;
@@ -181,6 +287,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize,
 		asm volatile("ptesync": : :"memory");
 	} else {
 		__tlbie(vpn, psize, apsize, ssize);
+		fixup_tlbie(vpn, psize, apsize, ssize);
 		asm volatile("eieio; tlbsync; ptesync": : :"memory");
 	}
 	if (lock_tlbie && !use_local)
@@ -674,7 +781,7 @@ static void native_hpte_clear(void)
  */
 static void native_flush_hash_range(unsigned long number, int local)
 {
-	unsigned long vpn;
+	unsigned long vpn = 0;
 	unsigned long hash, index, hidx, shift, slot;
 	struct hash_pte *hptep;
 	unsigned long hpte_v;
@@ -746,6 +853,10 @@ static void native_flush_hash_range(unsigned long number, int local)
 				__tlbie(vpn, psize, psize, ssize);
 			} pte_iterate_hashed_end();
 		}
+		/*
+		 * Just do one more with the last used values.
+		 */
+		fixup_tlbie(vpn, psize, psize, ssize);
 		asm volatile("eieio; tlbsync; ptesync":::"memory");
 
 		if (lock_tlbie)
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 655a5a9a183d..cf290d415dcd 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -36,6 +36,7 @@
 #include <linux/memblock.h>
 #include <linux/context_tracking.h>
 #include <linux/libfdt.h>
+#include <linux/pkeys.h>
 
 #include <asm/debugfs.h>
 #include <asm/processor.h>
@@ -232,6 +233,7 @@ unsigned long htab_convert_pte_flags(unsigned long pteflags)
 		 */
 		rflags |= HPTE_R_M;
 
+	rflags |= pte_to_hpte_pkey_bits(pteflags);
 	return rflags;
 }
 
@@ -606,7 +608,7 @@ static void init_hpte_page_sizes(void)
 			continue;	/* not a supported page size */
 		for (ap = bp; ap < MMU_PAGE_COUNT; ++ap) {
 			penc = mmu_psize_defs[bp].penc[ap];
-			if (penc == -1)
+			if (penc == -1 || !mmu_psize_defs[ap].shift)
 				continue;
 			shift = mmu_psize_defs[ap].shift - LP_SHIFT;
 			if (shift <= 0)
@@ -772,7 +774,7 @@ void resize_hpt_for_hotplug(unsigned long new_mem_size)
 		int rc;
 
 		rc = mmu_hash_ops.resize_hpt(target_hpt_shift);
-		if (rc)
+		if (rc && (rc != -ENODEV))
 			printk(KERN_WARNING
 			       "Unable to resize hash page table to target order %d: %d\n",
 			       target_hpt_shift, rc);
@@ -979,8 +981,9 @@ void __init hash__early_init_devtree(void)
 
 void __init hash__early_init_mmu(void)
 {
+#ifndef CONFIG_PPC_64K_PAGES
 	/*
-	 * We have code in __hash_page_64K() and elsewhere, which assumes it can
+	 * We have code in __hash_page_4K() and elsewhere, which assumes it can
 	 * do the following:
 	 *   new_pte |= (slot << H_PAGE_F_GIX_SHIFT) & (H_PAGE_F_SECOND | H_PAGE_F_GIX);
 	 *
@@ -991,6 +994,7 @@ void __init hash__early_init_mmu(void)
 	 * with a BUILD_BUG_ON().
 	 */
 	BUILD_BUG_ON(H_PAGE_F_SECOND != (1ul  << (H_PAGE_F_GIX_SHIFT + 3)));
+#endif /* CONFIG_PPC_64K_PAGES */
 
 	htab_init_page_sizes();
 
@@ -1004,6 +1008,7 @@ void __init hash__early_init_mmu(void)
 	__pmd_index_size = H_PMD_INDEX_SIZE;
 	__pud_index_size = H_PUD_INDEX_SIZE;
 	__pgd_index_size = H_PGD_INDEX_SIZE;
+	__pud_cache_index = H_PUD_CACHE_INDEX;
 	__pmd_cache_index = H_PMD_CACHE_INDEX;
 	__pte_table_size = H_PTE_TABLE_SIZE;
 	__pmd_table_size = H_PMD_TABLE_SIZE;
@@ -1049,6 +1054,10 @@ void __init hash__early_init_mmu(void)
 	pr_info("Initializing hash mmu with SLB\n");
 	/* Initialize SLB management */
 	slb_initialize();
+
+	if (cpu_has_feature(CPU_FTR_ARCH_206)
+			&& cpu_has_feature(CPU_FTR_HVMODE))
+		tlbiel_all();
 }
 
 #ifdef CONFIG_SMP
@@ -1068,6 +1077,10 @@ void hash__early_init_mmu_secondary(void)
 	}
 	/* Initialize SLB */
 	slb_initialize();
+
+	if (cpu_has_feature(CPU_FTR_ARCH_206)
+			&& cpu_has_feature(CPU_FTR_HVMODE))
+		tlbiel_all();
 }
 #endif /* CONFIG_SMP */
 
@@ -1569,6 +1582,30 @@ void hash_preload(struct mm_struct *mm, unsigned long ea,
 	local_irq_restore(flags);
 }
 
+#ifdef CONFIG_PPC_MEM_KEYS
+/*
+ * Return the protection key associated with the given address and the
+ * mm_struct.
+ */
+u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address)
+{
+	pte_t *ptep;
+	u16 pkey = 0;
+	unsigned long flags;
+
+	if (!mm || !mm->pgd)
+		return 0;
+
+	local_irq_save(flags);
+	ptep = find_linux_pte(mm->pgd, address, NULL, NULL);
+	if (ptep)
+		pkey = pte_to_pkey_bits(pte_val(READ_ONCE(*ptep)));
+	local_irq_restore(flags);
+
+	return pkey;
+}
+#endif /* CONFIG_PPC_MEM_KEYS */
+
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 static inline void tm_flush_hash_page(int local)
 {
@@ -1592,29 +1629,42 @@ static inline void tm_flush_hash_page(int local)
 }
 #endif
 
+/*
+ * Return the global hash slot, corresponding to the given PTE, which contains
+ * the HPTE.
+ */
+unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift,
+		int ssize, real_pte_t rpte, unsigned int subpg_index)
+{
+	unsigned long hash, gslot, hidx;
+
+	hash = hpt_hash(vpn, shift, ssize);
+	hidx = __rpte_to_hidx(rpte, subpg_index);
+	if (hidx & _PTEIDX_SECONDARY)
+		hash = ~hash;
+	gslot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
+	gslot += hidx & _PTEIDX_GROUP_IX;
+	return gslot;
+}
+
 /* WARNING: This is called from hash_low_64.S, if you change this prototype,
  *          do not forget to update the assembly call site !
  */
 void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
 		     unsigned long flags)
 {
-	unsigned long hash, index, shift, hidx, slot;
+	unsigned long index, shift, gslot;
 	int local = flags & HPTE_LOCAL_UPDATE;
 
 	DBG_LOW("flush_hash_page(vpn=%016lx)\n", vpn);
 	pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
-		hash = hpt_hash(vpn, shift, ssize);
-		hidx = __rpte_to_hidx(pte, index);
-		if (hidx & _PTEIDX_SECONDARY)
-			hash = ~hash;
-		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
-		slot += hidx & _PTEIDX_GROUP_IX;
-		DBG_LOW(" sub %ld: hash=%lx, hidx=%lx\n", index, slot, hidx);
+		gslot = pte_get_hash_gslot(vpn, shift, ssize, pte, index);
+		DBG_LOW(" sub %ld: gslot=%lx\n", index, gslot);
 		/*
 		 * We use same base page size and actual psize, because we don't
 		 * use these functions for hugepage
 		 */
-		mmu_hash_ops.hpte_invalidate(slot, vpn, psize, psize,
+		mmu_hash_ops.hpte_invalidate(gslot, vpn, psize, psize,
 					     ssize, local);
 	} pte_iterate_hashed_end();
 
@@ -1825,16 +1875,24 @@ void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base,
 	 */
 	BUG_ON(first_memblock_base != 0);
 
-	/* On LPAR systems, the first entry is our RMA region,
-	 * non-LPAR 64-bit hash MMU systems don't have a limitation
-	 * on real mode access, but using the first entry works well
-	 * enough. We also clamp it to 1G to avoid some funky things
-	 * such as RTAS bugs etc...
+	/*
+	 * On virtualized systems the first entry is our RMA region aka VRMA,
+	 * non-virtualized 64-bit hash MMU systems don't have a limitation
+	 * on real mode access.
+	 *
+	 * For guests on platforms before POWER9, we clamp the it limit to 1G
+	 * to avoid some funky things such as RTAS bugs etc...
 	 */
-	ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
+	if (!early_cpu_has_feature(CPU_FTR_HVMODE)) {
+		ppc64_rma_size = first_memblock_size;
+		if (!early_cpu_has_feature(CPU_FTR_ARCH_300))
+			ppc64_rma_size = min_t(u64, ppc64_rma_size, 0x40000000);
 
-	/* Finally limit subsequent allocations */
-	memblock_set_current_limit(ppc64_rma_size);
+		/* Finally limit subsequent allocations */
+		memblock_set_current_limit(ppc64_rma_size);
+	} else {
+		ppc64_rma_size = ULONG_MAX;
+	}
 }
 
 #ifdef CONFIG_DEBUG_FS
diff --git a/arch/powerpc/mm/hugetlbpage-hash64.c b/arch/powerpc/mm/hugetlbpage-hash64.c
index 0c2a91df3210..b320f5097a06 100644
--- a/arch/powerpc/mm/hugetlbpage-hash64.c
+++ b/arch/powerpc/mm/hugetlbpage-hash64.c
@@ -23,10 +23,11 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
 		     pte_t *ptep, unsigned long trap, unsigned long flags,
 		     int ssize, unsigned int shift, unsigned int mmu_psize)
 {
+	real_pte_t rpte;
 	unsigned long vpn;
 	unsigned long old_pte, new_pte;
 	unsigned long rflags, pa, sz;
-	long slot;
+	long slot, offset;
 
 	BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
 
@@ -62,6 +63,11 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
 	} while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
 
 	rflags = htab_convert_pte_flags(new_pte);
+	if (unlikely(mmu_psize == MMU_PAGE_16G))
+		offset = PTRS_PER_PUD;
+	else
+		offset = PTRS_PER_PMD;
+	rpte = __real_pte(__pte(old_pte), ptep, offset);
 
 	sz = ((1UL) << shift);
 	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
@@ -72,15 +78,10 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
 	/* Check if pte already has an hpte (case 2) */
 	if (unlikely(old_pte & H_PAGE_HASHPTE)) {
 		/* There MIGHT be an HPTE for this pte */
-		unsigned long hash, slot;
+		unsigned long gslot;
 
-		hash = hpt_hash(vpn, shift, ssize);
-		if (old_pte & H_PAGE_F_SECOND)
-			hash = ~hash;
-		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
-		slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT;
-
-		if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, mmu_psize,
+		gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0);
+		if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, mmu_psize,
 					       mmu_psize, ssize, flags) == -1)
 			old_pte &= ~_PAGE_HPTEFLAGS;
 	}
@@ -107,8 +108,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
 			return -1;
 		}
 
-		new_pte |= (slot << H_PAGE_F_GIX_SHIFT) &
-			(H_PAGE_F_SECOND | H_PAGE_F_GIX);
+		new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
 	}
 
 	/*
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index a9b9083c5e49..876da2bc1796 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -96,7 +96,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
 			*hpdp = __hugepd(__pa(new) |
 					 (shift_to_mmu_psize(pshift) << 2));
 #elif defined(CONFIG_PPC_8xx)
-			*hpdp = __hugepd(__pa(new) |
+			*hpdp = __hugepd(__pa(new) | _PMD_USER |
 					 (pshift == PAGE_SHIFT_8M ? _PMD_PAGE_8M :
 					  _PMD_PAGE_512K) | _PMD_PRESENT);
 #else
@@ -752,7 +752,7 @@ void flush_dcache_icache_hugepage(struct page *page)
  * So long as we atomically load page table pointers we are safe against teardown,
  * we can follow the address down to the the page and take a ref on it.
  * This function need to be called with interrupts disabled. We use this variant
- * when we have MSR[EE] = 0 but the paca->soft_enabled = 1
+ * when we have MSR[EE] = 0 but the paca->irq_soft_mask = IRQS_ENABLED
  */
 pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea,
 			bool *is_thp, unsigned *hpage_shift)
@@ -855,9 +855,7 @@ int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
 
 	pte = READ_ONCE(*ptep);
 
-	if (!pte_present(pte) || !pte_read(pte))
-		return 0;
-	if (write && !pte_write(pte))
+	if (!pte_access_permitted(pte, write))
 		return 0;
 
 	/* hugepages are never "special" */
diff --git a/arch/powerpc/mm/init-common.c b/arch/powerpc/mm/init-common.c
index eb8c6c8c4851..2b656e67f2ea 100644
--- a/arch/powerpc/mm/init-common.c
+++ b/arch/powerpc/mm/init-common.c
@@ -100,6 +100,6 @@ void pgtable_cache_init(void)
 	 * same size as either the pgd or pmd index except with THP enabled
 	 * on book3s 64
 	 */
-	if (PUD_INDEX_SIZE && !PGT_CACHE(PUD_INDEX_SIZE))
-		pgtable_cache_add(PUD_INDEX_SIZE, pud_ctor);
+	if (PUD_CACHE_INDEX && !PGT_CACHE(PUD_CACHE_INDEX))
+		pgtable_cache_add(PUD_CACHE_INDEX, pud_ctor);
 }
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index a07722531b32..fdb424a29f03 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -183,7 +183,8 @@ static __meminit void vmemmap_list_populate(unsigned long phys,
 	vmemmap_list = vmem_back;
 }
 
-int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
+int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
+		struct vmem_altmap *altmap)
 {
 	unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
 
@@ -193,17 +194,16 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 	pr_debug("vmemmap_populate %lx..%lx, node %d\n", start, end, node);
 
 	for (; start < end; start += page_size) {
-		struct vmem_altmap *altmap;
 		void *p;
 		int rc;
 
 		if (vmemmap_populated(start, page_size))
 			continue;
 
-		/* altmap lookups only work at section boundaries */
-		altmap = to_vmem_altmap(SECTION_ALIGN_DOWN(start));
-
-		p =  __vmemmap_alloc_block_buf(page_size, node, altmap);
+		if (altmap)
+			p = altmap_alloc_block_buf(page_size, altmap);
+		else
+			p = vmemmap_alloc_block_buf(page_size, node);
 		if (!p)
 			return -ENOMEM;
 
@@ -214,9 +214,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 
 		rc = vmemmap_create_mapping(start, page_size, __pa(p));
 		if (rc < 0) {
-			pr_warning(
-				"vmemmap_populate: Unable to create vmemmap mapping: %d\n",
-				rc);
+			pr_warn("%s: Unable to create vmemmap mapping: %d\n",
+				__func__, rc);
 			return -EFAULT;
 		}
 	}
@@ -257,7 +256,8 @@ static unsigned long vmemmap_list_free(unsigned long start)
 	return vmem_back->phys;
 }
 
-void __ref vmemmap_free(unsigned long start, unsigned long end)
+void __ref vmemmap_free(unsigned long start, unsigned long end,
+		struct vmem_altmap *altmap)
 {
 	unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
 	unsigned long page_order = get_order(page_size);
@@ -268,7 +268,6 @@ void __ref vmemmap_free(unsigned long start, unsigned long end)
 
 	for (; start < end; start += page_size) {
 		unsigned long nr_pages, addr;
-		struct vmem_altmap *altmap;
 		struct page *section_base;
 		struct page *page;
 
@@ -288,7 +287,6 @@ void __ref vmemmap_free(unsigned long start, unsigned long end)
 		section_base = pfn_to_page(vmemmap_section_start(start));
 		nr_pages = 1 << page_order;
 
-		altmap = to_vmem_altmap((unsigned long) section_base);
 		if (altmap) {
 			vmem_altmap_free(altmap, nr_pages);
 		} else if (PageReserved(page)) {
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 4362b86ef84c..fe8c61149fb8 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -127,7 +127,8 @@ int __weak remove_section_mapping(unsigned long start, unsigned long end)
 	return -ENODEV;
 }
 
-int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
+int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
+		bool want_memblock)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
@@ -138,21 +139,19 @@ int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
 	start = (unsigned long)__va(start);
 	rc = create_section_mapping(start, start + size);
 	if (rc) {
-		pr_warning(
-			"Unable to create mapping for hot added memory 0x%llx..0x%llx: %d\n",
+		pr_warn("Unable to create mapping for hot added memory 0x%llx..0x%llx: %d\n",
 			start, start + size, rc);
 		return -EFAULT;
 	}
 
-	return __add_pages(nid, start_pfn, nr_pages, want_memblock);
+	return __add_pages(nid, start_pfn, nr_pages, altmap, want_memblock);
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
-int arch_remove_memory(u64 start, u64 size)
+int arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
-	struct vmem_altmap *altmap;
 	struct page *page;
 	int ret;
 
@@ -161,11 +160,10 @@ int arch_remove_memory(u64 start, u64 size)
 	 * when querying the zone.
 	 */
 	page = pfn_to_page(start_pfn);
-	altmap = to_vmem_altmap((unsigned long) page);
 	if (altmap)
 		page += vmem_altmap_offset(altmap);
 
-	ret = __remove_pages(page_zone(page), start_pfn, nr_pages);
+	ret = __remove_pages(page_zone(page), start_pfn, nr_pages, altmap);
 	if (ret)
 		return ret;
 
diff --git a/arch/powerpc/mm/mmu_context.c b/arch/powerpc/mm/mmu_context.c
index d60a62bf4fc7..0ab297c4cfad 100644
--- a/arch/powerpc/mm/mmu_context.c
+++ b/arch/powerpc/mm/mmu_context.c
@@ -12,6 +12,7 @@
 
 #include <linux/mm.h>
 #include <linux/cpu.h>
+#include <linux/sched/mm.h>
 
 #include <asm/mmu_context.h>
 
@@ -58,6 +59,10 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 		 *
 		 * On the read side the barrier is in pte_xchg(), which orders
 		 * the store to the PTE vs the load of mm_cpumask.
+		 *
+		 * This full barrier is needed by membarrier when switching
+		 * between processes after store to rq->curr, before user-space
+		 * memory accesses.
 		 */
 		smp_mb();
 
@@ -80,6 +85,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 
 	if (new_on_cpu)
 		radix_kvm_prefetch_workaround(next);
+	else
+		membarrier_arch_switch_mm(prev, next, tsk);
 
 	/*
 	 * The actual HW switching method differs between the various
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index 59c0766ae4e0..3f980baade4c 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -16,6 +16,7 @@
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/mm.h>
+#include <linux/pkeys.h>
 #include <linux/spinlock.h>
 #include <linux/idr.h>
 #include <linux/export.h>
@@ -118,6 +119,7 @@ static int hash__init_new_context(struct mm_struct *mm)
 
 	subpage_prot_init_new_context(mm);
 
+	pkey_mm_init(mm);
 	return index;
 }
 
@@ -171,6 +173,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 	mm_iommu_init(mm);
 #endif
 	atomic_set(&mm->context.active_cpus, 0);
+	atomic_set(&mm->context.copros, 0);
 
 	return 0;
 }
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index adb6364f4091..edd8d0bc9364 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -40,6 +40,7 @@
 #include <asm/hvcall.h>
 #include <asm/setup.h>
 #include <asm/vdso.h>
+#include <asm/drmem.h>
 
 static int numa_enabled = 1;
 
@@ -142,11 +143,6 @@ static void reset_numa_cpu_lookup_table(void)
 		numa_cpu_lookup_table[cpu] = -1;
 }
 
-static void update_numa_cpu_lookup_table(unsigned int cpu, int node)
-{
-	numa_cpu_lookup_table[cpu] = node;
-}
-
 static void map_cpu_to_node(int cpu, int node)
 {
 	update_numa_cpu_lookup_table(cpu, node);
@@ -179,21 +175,6 @@ static const __be32 *of_get_associativity(struct device_node *dev)
 	return of_get_property(dev, "ibm,associativity", NULL);
 }
 
-/*
- * Returns the property linux,drconf-usable-memory if
- * it exists (the property exists only in kexec/kdump kernels,
- * added by kexec-tools)
- */
-static const __be32 *of_get_usable_memory(struct device_node *memory)
-{
-	const __be32 *prop;
-	u32 len;
-	prop = of_get_property(memory, "linux,drconf-usable-memory", &len);
-	if (!prop || len < sizeof(unsigned int))
-		return NULL;
-	return prop;
-}
-
 int __node_distance(int a, int b)
 {
 	int i;
@@ -387,69 +368,6 @@ static unsigned long read_n_cells(int n, const __be32 **buf)
 	return result;
 }
 
-/*
- * Read the next memblock list entry from the ibm,dynamic-memory property
- * and return the information in the provided of_drconf_cell structure.
- */
-static void read_drconf_cell(struct of_drconf_cell *drmem, const __be32 **cellp)
-{
-	const __be32 *cp;
-
-	drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp);
-
-	cp = *cellp;
-	drmem->drc_index = of_read_number(cp, 1);
-	drmem->reserved = of_read_number(&cp[1], 1);
-	drmem->aa_index = of_read_number(&cp[2], 1);
-	drmem->flags = of_read_number(&cp[3], 1);
-
-	*cellp = cp + 4;
-}
-
-/*
- * Retrieve and validate the ibm,dynamic-memory property of the device tree.
- *
- * The layout of the ibm,dynamic-memory property is a number N of memblock
- * list entries followed by N memblock list entries.  Each memblock list entry
- * contains information as laid out in the of_drconf_cell struct above.
- */
-static int of_get_drconf_memory(struct device_node *memory, const __be32 **dm)
-{
-	const __be32 *prop;
-	u32 len, entries;
-
-	prop = of_get_property(memory, "ibm,dynamic-memory", &len);
-	if (!prop || len < sizeof(unsigned int))
-		return 0;
-
-	entries = of_read_number(prop++, 1);
-
-	/* Now that we know the number of entries, revalidate the size
-	 * of the property read in to ensure we have everything
-	 */
-	if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int))
-		return 0;
-
-	*dm = prop;
-	return entries;
-}
-
-/*
- * Retrieve and validate the ibm,lmb-size property for drconf memory
- * from the device tree.
- */
-static u64 of_get_lmb_size(struct device_node *memory)
-{
-	const __be32 *prop;
-	u32 len;
-
-	prop = of_get_property(memory, "ibm,lmb-size", &len);
-	if (!prop || len < sizeof(unsigned int))
-		return 0;
-
-	return read_n_cells(n_mem_size_cells, &prop);
-}
-
 struct assoc_arrays {
 	u32	n_arrays;
 	u32	array_sz;
@@ -466,19 +384,27 @@ struct assoc_arrays {
  * indicating the size of each associativity array, followed by a list
  * of N associativity arrays.
  */
-static int of_get_assoc_arrays(struct device_node *memory,
-			       struct assoc_arrays *aa)
+static int of_get_assoc_arrays(struct assoc_arrays *aa)
 {
+	struct device_node *memory;
 	const __be32 *prop;
 	u32 len;
 
+	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
+	if (!memory)
+		return -1;
+
 	prop = of_get_property(memory, "ibm,associativity-lookup-arrays", &len);
-	if (!prop || len < 2 * sizeof(unsigned int))
+	if (!prop || len < 2 * sizeof(unsigned int)) {
+		of_node_put(memory);
 		return -1;
+	}
 
 	aa->n_arrays = of_read_number(prop++, 1);
 	aa->array_sz = of_read_number(prop++, 1);
 
+	of_node_put(memory);
+
 	/* Now that we know the number of arrays and size of each array,
 	 * revalidate the size of the property read in.
 	 */
@@ -493,26 +419,30 @@ static int of_get_assoc_arrays(struct device_node *memory,
  * This is like of_node_to_nid_single() for memory represented in the
  * ibm,dynamic-reconfiguration-memory node.
  */
-static int of_drconf_to_nid_single(struct of_drconf_cell *drmem,
-				   struct assoc_arrays *aa)
+static int of_drconf_to_nid_single(struct drmem_lmb *lmb)
 {
+	struct assoc_arrays aa = { .arrays = NULL };
 	int default_nid = 0;
 	int nid = default_nid;
-	int index;
+	int rc, index;
+
+	rc = of_get_assoc_arrays(&aa);
+	if (rc)
+		return default_nid;
 
-	if (min_common_depth > 0 && min_common_depth <= aa->array_sz &&
-	    !(drmem->flags & DRCONF_MEM_AI_INVALID) &&
-	    drmem->aa_index < aa->n_arrays) {
-		index = drmem->aa_index * aa->array_sz + min_common_depth - 1;
-		nid = of_read_number(&aa->arrays[index], 1);
+	if (min_common_depth > 0 && min_common_depth <= aa.array_sz &&
+	    !(lmb->flags & DRCONF_MEM_AI_INVALID) &&
+	    lmb->aa_index < aa.n_arrays) {
+		index = lmb->aa_index * aa.array_sz + min_common_depth - 1;
+		nid = of_read_number(&aa.arrays[index], 1);
 
 		if (nid == 0xffff || nid >= MAX_NUMNODES)
 			nid = default_nid;
 
 		if (nid > 0) {
-			index = drmem->aa_index * aa->array_sz;
+			index = lmb->aa_index * aa.array_sz;
 			initialize_distance_lookup_table(nid,
-							&aa->arrays[index]);
+							&aa.arrays[index]);
 		}
 	}
 
@@ -551,7 +481,7 @@ static int numa_setup_cpu(unsigned long lcpu)
 	nid = of_node_to_nid_single(cpu);
 
 out_present:
-	if (nid < 0 || !node_online(nid))
+	if (nid < 0 || !node_possible(nid))
 		nid = first_online_node;
 
 	map_cpu_to_node(lcpu, nid);
@@ -645,67 +575,48 @@ static inline int __init read_usm_ranges(const __be32 **usm)
  * Extract NUMA information from the ibm,dynamic-reconfiguration-memory
  * node.  This assumes n_mem_{addr,size}_cells have been set.
  */
-static void __init parse_drconf_memory(struct device_node *memory)
+static void __init numa_setup_drmem_lmb(struct drmem_lmb *lmb,
+					const __be32 **usm)
 {
-	const __be32 *uninitialized_var(dm), *usm;
-	unsigned int n, rc, ranges, is_kexec_kdump = 0;
-	unsigned long lmb_size, base, size, sz;
+	unsigned int ranges, is_kexec_kdump = 0;
+	unsigned long base, size, sz;
 	int nid;
-	struct assoc_arrays aa = { .arrays = NULL };
-
-	n = of_get_drconf_memory(memory, &dm);
-	if (!n)
-		return;
 
-	lmb_size = of_get_lmb_size(memory);
-	if (!lmb_size)
-		return;
-
-	rc = of_get_assoc_arrays(memory, &aa);
-	if (rc)
+	/*
+	 * Skip this block if the reserved bit is set in flags (0x80)
+	 * or if the block is not assigned to this partition (0x8)
+	 */
+	if ((lmb->flags & DRCONF_MEM_RESERVED)
+	    || !(lmb->flags & DRCONF_MEM_ASSIGNED))
 		return;
 
-	/* check if this is a kexec/kdump kernel */
-	usm = of_get_usable_memory(memory);
-	if (usm != NULL)
+	if (*usm)
 		is_kexec_kdump = 1;
 
-	for (; n != 0; --n) {
-		struct of_drconf_cell drmem;
-
-		read_drconf_cell(&drmem, &dm);
-
-		/* skip this block if the reserved bit is set in flags (0x80)
-		   or if the block is not assigned to this partition (0x8) */
-		if ((drmem.flags & DRCONF_MEM_RESERVED)
-		    || !(drmem.flags & DRCONF_MEM_ASSIGNED))
-			continue;
+	base = lmb->base_addr;
+	size = drmem_lmb_size();
+	ranges = 1;
 
-		base = drmem.base_addr;
-		size = lmb_size;
-		ranges = 1;
+	if (is_kexec_kdump) {
+		ranges = read_usm_ranges(usm);
+		if (!ranges) /* there are no (base, size) duple */
+			return;
+	}
 
+	do {
 		if (is_kexec_kdump) {
-			ranges = read_usm_ranges(&usm);
-			if (!ranges) /* there are no (base, size) duple */
-				continue;
+			base = read_n_cells(n_mem_addr_cells, usm);
+			size = read_n_cells(n_mem_size_cells, usm);
 		}
-		do {
-			if (is_kexec_kdump) {
-				base = read_n_cells(n_mem_addr_cells, &usm);
-				size = read_n_cells(n_mem_size_cells, &usm);
-			}
-			nid = of_drconf_to_nid_single(&drmem, &aa);
-			fake_numa_create_new_node(
-				((base + size) >> PAGE_SHIFT),
-					   &nid);
-			node_set_online(nid);
-			sz = numa_enforce_memory_limit(base, size);
-			if (sz)
-				memblock_set_node(base, sz,
-						  &memblock.memory, nid);
-		} while (--ranges);
-	}
+
+		nid = of_drconf_to_nid_single(lmb);
+		fake_numa_create_new_node(((base + size) >> PAGE_SHIFT),
+					  &nid);
+		node_set_online(nid);
+		sz = numa_enforce_memory_limit(base, size);
+		if (sz)
+			memblock_set_node(base, sz, &memblock.memory, nid);
+	} while (--ranges);
 }
 
 static int __init parse_numa_properties(void)
@@ -800,8 +711,10 @@ static int __init parse_numa_properties(void)
 	 * ibm,dynamic-reconfiguration-memory node.
 	 */
 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
-	if (memory)
-		parse_drconf_memory(memory);
+	if (memory) {
+		walk_drmem_lmbs(memory, numa_setup_drmem_lmb);
+		of_node_put(memory);
+	}
 
 	return 0;
 }
@@ -892,6 +805,32 @@ static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
 	NODE_DATA(nid)->node_spanned_pages = spanned_pages;
 }
 
+static void __init find_possible_nodes(void)
+{
+	struct device_node *rtas;
+	u32 numnodes, i;
+
+	if (min_common_depth <= 0)
+		return;
+
+	rtas = of_find_node_by_path("/rtas");
+	if (!rtas)
+		return;
+
+	if (of_property_read_u32_index(rtas,
+				"ibm,max-associativity-domains",
+				min_common_depth, &numnodes))
+		goto out;
+
+	for (i = 0; i < numnodes; i++) {
+		if (!node_possible(i))
+			node_set(i, node_possible_map);
+	}
+
+out:
+	of_node_put(rtas);
+}
+
 void __init initmem_init(void)
 {
 	int nid, cpu;
@@ -905,12 +844,15 @@ void __init initmem_init(void)
 	memblock_dump_all();
 
 	/*
-	 * Reduce the possible NUMA nodes to the online NUMA nodes,
-	 * since we do not support node hotplug. This ensures that  we
-	 * lower the maximum NUMA node ID to what is actually present.
+	 * Modify the set of possible NUMA nodes to reflect information
+	 * available about the set of online nodes, and the set of nodes
+	 * that we expect to make use of for this platform's affinity
+	 * calculations.
 	 */
 	nodes_and(node_possible_map, node_possible_map, node_online_map);
 
+	find_possible_nodes();
+
 	for_each_online_node(nid) {
 		unsigned long start_pfn, end_pfn;
 
@@ -979,43 +921,26 @@ early_param("topology_updates", early_topology_updates);
  * memory represented in the device tree by the property
  * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory.
  */
-static int hot_add_drconf_scn_to_nid(struct device_node *memory,
-				     unsigned long scn_addr)
+static int hot_add_drconf_scn_to_nid(unsigned long scn_addr)
 {
-	const __be32 *dm;
-	unsigned int drconf_cell_cnt, rc;
+	struct drmem_lmb *lmb;
 	unsigned long lmb_size;
-	struct assoc_arrays aa;
 	int nid = -1;
 
-	drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
-	if (!drconf_cell_cnt)
-		return -1;
-
-	lmb_size = of_get_lmb_size(memory);
-	if (!lmb_size)
-		return -1;
-
-	rc = of_get_assoc_arrays(memory, &aa);
-	if (rc)
-		return -1;
-
-	for (; drconf_cell_cnt != 0; --drconf_cell_cnt) {
-		struct of_drconf_cell drmem;
-
-		read_drconf_cell(&drmem, &dm);
+	lmb_size = drmem_lmb_size();
 
+	for_each_drmem_lmb(lmb) {
 		/* skip this block if it is reserved or not assigned to
 		 * this partition */
-		if ((drmem.flags & DRCONF_MEM_RESERVED)
-		    || !(drmem.flags & DRCONF_MEM_ASSIGNED))
+		if ((lmb->flags & DRCONF_MEM_RESERVED)
+		    || !(lmb->flags & DRCONF_MEM_ASSIGNED))
 			continue;
 
-		if ((scn_addr < drmem.base_addr)
-		    || (scn_addr >= (drmem.base_addr + lmb_size)))
+		if ((scn_addr < lmb->base_addr)
+		    || (scn_addr >= (lmb->base_addr + lmb_size)))
 			continue;
 
-		nid = of_drconf_to_nid_single(&drmem, &aa);
+		nid = of_drconf_to_nid_single(lmb);
 		break;
 	}
 
@@ -1080,7 +1005,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr)
 
 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
 	if (memory) {
-		nid = hot_add_drconf_scn_to_nid(memory, scn_addr);
+		nid = hot_add_drconf_scn_to_nid(scn_addr);
 		of_node_put(memory);
 	} else {
 		nid = hot_add_node_scn_to_nid(scn_addr);
@@ -1096,11 +1021,7 @@ static u64 hot_add_drconf_memory_max(void)
 {
 	struct device_node *memory = NULL;
 	struct device_node *dn = NULL;
-	unsigned int drconf_cell_cnt = 0;
-	u64 lmb_size = 0;
-	const __be32 *dm = NULL;
 	const __be64 *lrdr = NULL;
-	struct of_drconf_cell drmem;
 
 	dn = of_find_node_by_path("/rtas");
 	if (dn) {
@@ -1112,14 +1033,8 @@ static u64 hot_add_drconf_memory_max(void)
 
 	memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
 	if (memory) {
-		drconf_cell_cnt = of_get_drconf_memory(memory, &dm);
-		lmb_size = of_get_lmb_size(memory);
-
-		/* Advance to the last cell, each cell has 6 32 bit integers */
-		dm += (drconf_cell_cnt - 1) * 6;
-		read_drconf_cell(&drmem, &dm);
 		of_node_put(memory);
-		return drmem.base_addr + lmb_size;
+		return drmem_lmb_memory_max();
 	}
 	return 0;
 }
@@ -1278,6 +1193,42 @@ static long vphn_get_associativity(unsigned long cpu,
 	return rc;
 }
 
+int find_and_online_cpu_nid(int cpu)
+{
+	__be32 associativity[VPHN_ASSOC_BUFSIZE] = {0};
+	int new_nid;
+
+	/* Use associativity from first thread for all siblings */
+	vphn_get_associativity(cpu, associativity);
+	new_nid = associativity_to_nid(associativity);
+	if (new_nid < 0 || !node_possible(new_nid))
+		new_nid = first_online_node;
+
+	if (NODE_DATA(new_nid) == NULL) {
+#ifdef CONFIG_MEMORY_HOTPLUG
+		/*
+		 * Need to ensure that NODE_DATA is initialized for a node from
+		 * available memory (see memblock_alloc_try_nid). If unable to
+		 * init the node, then default to nearest node that has memory
+		 * installed.
+		 */
+		if (try_online_node(new_nid))
+			new_nid = first_online_node;
+#else
+		/*
+		 * Default to using the nearest node that has memory installed.
+		 * Otherwise, it would be necessary to patch the kernel MM code
+		 * to deal with more memoryless-node error conditions.
+		 */
+		new_nid = first_online_node;
+#endif
+	}
+
+	pr_debug("%s:%d cpu %d nid %d\n", __FUNCTION__, __LINE__,
+		cpu, new_nid);
+	return new_nid;
+}
+
 /*
  * Update the CPU maps and sysfs entries for a single CPU when its NUMA
  * characteristics change. This function doesn't perform any locking and is
@@ -1345,7 +1296,6 @@ int numa_update_cpu_topology(bool cpus_locked)
 {
 	unsigned int cpu, sibling, changed = 0;
 	struct topology_update_data *updates, *ud;
-	__be32 associativity[VPHN_ASSOC_BUFSIZE] = {0};
 	cpumask_t updated_cpus;
 	struct device *dev;
 	int weight, new_nid, i = 0;
@@ -1383,11 +1333,7 @@ int numa_update_cpu_topology(bool cpus_locked)
 			continue;
 		}
 
-		/* Use associativity from first thread for all siblings */
-		vphn_get_associativity(cpu, associativity);
-		new_nid = associativity_to_nid(associativity);
-		if (new_nid < 0 || !node_online(new_nid))
-			new_nid = first_online_node;
+		new_nid = find_and_online_cpu_nid(cpu);
 
 		if (new_nid == numa_cpu_lookup_table[cpu]) {
 			cpumask_andnot(&cpu_associativity_changes_mask,
diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
index 3b65917785a5..422e80253a33 100644
--- a/arch/powerpc/mm/pgtable-book3s64.c
+++ b/arch/powerpc/mm/pgtable-book3s64.c
@@ -90,16 +90,19 @@ void serialize_against_pte_lookup(struct mm_struct *mm)
  * We use this to invalidate a pmdp entry before switching from a
  * hugepte to regular pmd entry.
  */
-void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
+pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 		     pmd_t *pmdp)
 {
-	pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
+	unsigned long old_pmd;
+
+	old_pmd = pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
 	flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 	/*
 	 * This ensures that generic code that rely on IRQ disabling
 	 * to prevent a parallel THP split work as expected.
 	 */
 	serialize_against_pte_lookup(vma->vm_mm);
+	return __pmd(old_pmd);
 }
 
 static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot)
diff --git a/arch/powerpc/mm/pgtable-hash64.c b/arch/powerpc/mm/pgtable-hash64.c
index ec277913e01b..469808e77e58 100644
--- a/arch/powerpc/mm/pgtable-hash64.c
+++ b/arch/powerpc/mm/pgtable-hash64.c
@@ -296,28 +296,6 @@ pgtable_t hash__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
 	return pgtable;
 }
 
-void hash__pmdp_huge_split_prepare(struct vm_area_struct *vma,
-			       unsigned long address, pmd_t *pmdp)
-{
-	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
-	VM_BUG_ON(REGION_ID(address) != USER_REGION_ID);
-	VM_BUG_ON(pmd_devmap(*pmdp));
-
-	/*
-	 * We can't mark the pmd none here, because that will cause a race
-	 * against exit_mmap. We need to continue mark pmd TRANS HUGE, while
-	 * we spilt, but at the same time we wan't rest of the ppc64 code
-	 * not to insert hash pte on this, because we will be modifying
-	 * the deposited pgtable in the caller of this function. Hence
-	 * clear the _PAGE_USER so that we move the fault handling to
-	 * higher level function and that will serialize against ptl.
-	 * We need to flush existing hash pte entries here even though,
-	 * the translation is still valid, because we will withdraw
-	 * pgtable_t after this.
-	 */
-	pmd_hugepage_update(vma->vm_mm, address, pmdp, 0, _PAGE_PRIVILEGED);
-}
-
 /*
  * A linux hugepage PMD was changed and the corresponding hash table entries
  * neesd to be flushed.
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index cfbbee941a76..2e10a964e290 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -17,9 +17,11 @@
 #include <linux/of_fdt.h>
 #include <linux/mm.h>
 #include <linux/string_helpers.h>
+#include <linux/stop_machine.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
+#include <asm/mmu_context.h>
 #include <asm/dma.h>
 #include <asm/machdep.h>
 #include <asm/mmu.h>
@@ -333,6 +335,22 @@ static void __init radix_init_pgtable(void)
 		     "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
 	asm volatile("eieio; tlbsync; ptesync" : : : "memory");
 	trace_tlbie(0, 0, TLBIEL_INVAL_SET_LPID, 0, 2, 1, 1);
+
+	/*
+	 * The init_mm context is given the first available (non-zero) PID,
+	 * which is the "guard PID" and contains no page table. PIDR should
+	 * never be set to zero because that duplicates the kernel address
+	 * space at the 0x0... offset (quadrant 0)!
+	 *
+	 * An arbitrary PID that may later be allocated by the PID allocator
+	 * for userspace processes must not be used either, because that
+	 * would cause stale user mappings for that PID on CPUs outside of
+	 * the TLB invalidation scheme (because it won't be in mm_cpumask).
+	 *
+	 * So permanently carve out one PID for the purpose of a guard PID.
+	 */
+	init_mm.context.id = mmu_base_pid;
+	mmu_base_pid++;
 }
 
 static void __init radix_init_partition_table(void)
@@ -535,6 +553,7 @@ void __init radix__early_init_mmu(void)
 	__pmd_index_size = RADIX_PMD_INDEX_SIZE;
 	__pud_index_size = RADIX_PUD_INDEX_SIZE;
 	__pgd_index_size = RADIX_PGD_INDEX_SIZE;
+	__pud_cache_index = RADIX_PUD_INDEX_SIZE;
 	__pmd_cache_index = RADIX_PMD_INDEX_SIZE;
 	__pte_table_size = RADIX_PTE_TABLE_SIZE;
 	__pmd_table_size = RADIX_PMD_TABLE_SIZE;
@@ -579,6 +598,10 @@ void __init radix__early_init_mmu(void)
 
 	radix_init_iamr();
 	radix_init_pgtable();
+	/* Switch to the guard PID before turning on MMU */
+	radix__switch_mmu_context(NULL, &init_mm);
+	if (cpu_has_feature(CPU_FTR_HVMODE))
+		tlbiel_all();
 }
 
 void radix__early_init_mmu_secondary(void)
@@ -600,6 +623,10 @@ void radix__early_init_mmu_secondary(void)
 		radix_init_amor();
 	}
 	radix_init_iamr();
+
+	radix__switch_mmu_context(NULL, &init_mm);
+	if (cpu_has_feature(CPU_FTR_HVMODE))
+		tlbiel_all();
 }
 
 void radix__mmu_cleanup_all(void)
@@ -622,22 +649,11 @@ void radix__setup_initial_memory_limit(phys_addr_t first_memblock_base,
 	 * physical on those processors
 	 */
 	BUG_ON(first_memblock_base != 0);
+
 	/*
-	 * We limit the allocation that depend on ppc64_rma_size
-	 * to first_memblock_size. We also clamp it to 1GB to
-	 * avoid some funky things such as RTAS bugs.
-	 *
-	 * On radix config we really don't have a limitation
-	 * on real mode access. But keeping it as above works
-	 * well enough.
-	 */
-	ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
-	/*
-	 * Finally limit subsequent allocations. We really don't want
-	 * to limit the memblock allocations to rma_size. FIXME!! should
-	 * we even limit at all ?
+	 * Radix mode is not limited by RMA / VRMA addressing.
 	 */
-	memblock_set_current_limit(first_memblock_base + first_memblock_size);
+	ppc64_rma_size = ULONG_MAX;
 }
 
 #ifdef CONFIG_MEMORY_HOTPLUG
@@ -671,6 +687,30 @@ static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
 	pud_clear(pud);
 }
 
+struct change_mapping_params {
+	pte_t *pte;
+	unsigned long start;
+	unsigned long end;
+	unsigned long aligned_start;
+	unsigned long aligned_end;
+};
+
+static int stop_machine_change_mapping(void *data)
+{
+	struct change_mapping_params *params =
+			(struct change_mapping_params *)data;
+
+	if (!data)
+		return -1;
+
+	spin_unlock(&init_mm.page_table_lock);
+	pte_clear(&init_mm, params->aligned_start, params->pte);
+	create_physical_mapping(params->aligned_start, params->start);
+	create_physical_mapping(params->end, params->aligned_end);
+	spin_lock(&init_mm.page_table_lock);
+	return 0;
+}
+
 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
 			     unsigned long end)
 {
@@ -699,6 +739,52 @@ static void remove_pte_table(pte_t *pte_start, unsigned long addr,
 	}
 }
 
+/*
+ * clear the pte and potentially split the mapping helper
+ */
+static void split_kernel_mapping(unsigned long addr, unsigned long end,
+				unsigned long size, pte_t *pte)
+{
+	unsigned long mask = ~(size - 1);
+	unsigned long aligned_start = addr & mask;
+	unsigned long aligned_end = addr + size;
+	struct change_mapping_params params;
+	bool split_region = false;
+
+	if ((end - addr) < size) {
+		/*
+		 * We're going to clear the PTE, but not flushed
+		 * the mapping, time to remap and flush. The
+		 * effects if visible outside the processor or
+		 * if we are running in code close to the
+		 * mapping we cleared, we are in trouble.
+		 */
+		if (overlaps_kernel_text(aligned_start, addr) ||
+			overlaps_kernel_text(end, aligned_end)) {
+			/*
+			 * Hack, just return, don't pte_clear
+			 */
+			WARN_ONCE(1, "Linear mapping %lx->%lx overlaps kernel "
+				  "text, not splitting\n", addr, end);
+			return;
+		}
+		split_region = true;
+	}
+
+	if (split_region) {
+		params.pte = pte;
+		params.start = addr;
+		params.end = end;
+		params.aligned_start = addr & ~(size - 1);
+		params.aligned_end = min_t(unsigned long, aligned_end,
+				(unsigned long)__va(memblock_end_of_DRAM()));
+		stop_machine(stop_machine_change_mapping, &params, NULL);
+		return;
+	}
+
+	pte_clear(&init_mm, addr, pte);
+}
+
 static void remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
 			     unsigned long end)
 {
@@ -714,13 +800,7 @@ static void remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
 			continue;
 
 		if (pmd_huge(*pmd)) {
-			if (!IS_ALIGNED(addr, PMD_SIZE) ||
-			    !IS_ALIGNED(next, PMD_SIZE)) {
-				WARN_ONCE(1, "%s: unaligned range\n", __func__);
-				continue;
-			}
-
-			pte_clear(&init_mm, addr, (pte_t *)pmd);
+			split_kernel_mapping(addr, end, PMD_SIZE, (pte_t *)pmd);
 			continue;
 		}
 
@@ -745,13 +825,7 @@ static void remove_pud_table(pud_t *pud_start, unsigned long addr,
 			continue;
 
 		if (pud_huge(*pud)) {
-			if (!IS_ALIGNED(addr, PUD_SIZE) ||
-			    !IS_ALIGNED(next, PUD_SIZE)) {
-				WARN_ONCE(1, "%s: unaligned range\n", __func__);
-				continue;
-			}
-
-			pte_clear(&init_mm, addr, (pte_t *)pud);
+			split_kernel_mapping(addr, end, PUD_SIZE, (pte_t *)pud);
 			continue;
 		}
 
@@ -777,13 +851,7 @@ static void remove_pagetable(unsigned long start, unsigned long end)
 			continue;
 
 		if (pgd_huge(*pgd)) {
-			if (!IS_ALIGNED(addr, PGDIR_SIZE) ||
-			    !IS_ALIGNED(next, PGDIR_SIZE)) {
-				WARN_ONCE(1, "%s: unaligned range\n", __func__);
-				continue;
-			}
-
-			pte_clear(&init_mm, addr, (pte_t *)pgd);
+			split_kernel_mapping(addr, end, PGDIR_SIZE, (pte_t *)pgd);
 			continue;
 		}
 
diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index a03ff3d99e0c..9f361ae571e9 100644
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -54,7 +54,8 @@ static inline int pte_looks_normal(pte_t pte)
 	return 0;
 #else
 	return (pte_val(pte) &
-		(_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE | _PAGE_USER)) ==
+		(_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE | _PAGE_USER |
+		 _PAGE_PRIVILEGED)) ==
 		(_PAGE_PRESENT | _PAGE_USER);
 #endif
 }
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index f6c7f54c0515..d35d9ad3c1cd 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -98,14 +98,7 @@ ioremap_prot(phys_addr_t addr, unsigned long size, unsigned long flags)
 
 	/* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */
 	flags &= ~(_PAGE_USER | _PAGE_EXEC);
-
-#ifdef _PAGE_BAP_SR
-	/* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format
-	 * which means that we just cleared supervisor access... oops ;-) This
-	 * restores it
-	 */
-	flags |= _PAGE_BAP_SR;
-#endif
+	flags |= _PAGE_PRIVILEGED;
 
 	return __ioremap_caller(addr, size, flags, __builtin_return_address(0));
 }
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 813ea22c3e00..adf469f312f2 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -82,6 +82,8 @@ unsigned long __pgd_index_size;
 EXPORT_SYMBOL(__pgd_index_size);
 unsigned long __pmd_cache_index;
 EXPORT_SYMBOL(__pmd_cache_index);
+unsigned long __pud_cache_index;
+EXPORT_SYMBOL(__pud_cache_index);
 unsigned long __pte_table_size;
 EXPORT_SYMBOL(__pte_table_size);
 unsigned long __pmd_table_size;
@@ -244,20 +246,8 @@ void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size,
 	/*
 	 * Force kernel mapping.
 	 */
-#if defined(CONFIG_PPC_BOOK3S_64)
-	flags |= _PAGE_PRIVILEGED;
-#else
 	flags &= ~_PAGE_USER;
-#endif
-
-
-#ifdef _PAGE_BAP_SR
-	/* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format
-	 * which means that we just cleared supervisor access... oops ;-) This
-	 * restores it
-	 */
-	flags |= _PAGE_BAP_SR;
-#endif
+	flags |= _PAGE_PRIVILEGED;
 
 	if (ppc_md.ioremap)
 		return ppc_md.ioremap(addr, size, flags, caller);
@@ -483,12 +473,15 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
 	if (old & PATB_HR) {
 		asm volatile(PPC_TLBIE_5(%0,%1,2,0,1) : :
 			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
+		asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
+			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
 		trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 1);
 	} else {
 		asm volatile(PPC_TLBIE_5(%0,%1,2,0,0) : :
 			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
 		trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
 	}
+	/* do we need fixup here ?*/
 	asm volatile("eieio; tlbsync; ptesync" : : : "memory");
 }
 EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c
new file mode 100644
index 000000000000..ba71c5481f42
--- /dev/null
+++ b/arch/powerpc/mm/pkeys.c
@@ -0,0 +1,468 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * PowerPC Memory Protection Keys management
+ *
+ * Copyright 2017, Ram Pai, IBM Corporation.
+ */
+
+#include <asm/mman.h>
+#include <asm/setup.h>
+#include <linux/pkeys.h>
+#include <linux/of_device.h>
+
+DEFINE_STATIC_KEY_TRUE(pkey_disabled);
+bool pkey_execute_disable_supported;
+int  pkeys_total;		/* Total pkeys as per device tree */
+bool pkeys_devtree_defined;	/* pkey property exported by device tree */
+u32  initial_allocation_mask;	/* Bits set for reserved keys */
+u64  pkey_amr_uamor_mask;	/* Bits in AMR/UMOR not to be touched */
+u64  pkey_iamr_mask;		/* Bits in AMR not to be touched */
+
+#define AMR_BITS_PER_PKEY 2
+#define AMR_RD_BIT 0x1UL
+#define AMR_WR_BIT 0x2UL
+#define IAMR_EX_BIT 0x1UL
+#define PKEY_REG_BITS (sizeof(u64)*8)
+#define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY))
+
+static void scan_pkey_feature(void)
+{
+	u32 vals[2];
+	struct device_node *cpu;
+
+	cpu = of_find_node_by_type(NULL, "cpu");
+	if (!cpu)
+		return;
+
+	if (of_property_read_u32_array(cpu,
+			"ibm,processor-storage-keys", vals, 2))
+		return;
+
+	/*
+	 * Since any pkey can be used for data or execute, we will just treat
+	 * all keys as equal and track them as one entity.
+	 */
+	pkeys_total = be32_to_cpu(vals[0]);
+	pkeys_devtree_defined = true;
+}
+
+static inline bool pkey_mmu_enabled(void)
+{
+	if (firmware_has_feature(FW_FEATURE_LPAR))
+		return pkeys_total;
+	else
+		return cpu_has_feature(CPU_FTR_PKEY);
+}
+
+int pkey_initialize(void)
+{
+	int os_reserved, i;
+
+	/*
+	 * We define PKEY_DISABLE_EXECUTE in addition to the arch-neutral
+	 * generic defines for PKEY_DISABLE_ACCESS and PKEY_DISABLE_WRITE.
+	 * Ensure that the bits a distinct.
+	 */
+	BUILD_BUG_ON(PKEY_DISABLE_EXECUTE &
+		     (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
+
+	/*
+	 * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous
+	 * in the vmaflag. Make sure that is really the case.
+	 */
+	BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) +
+		     __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)
+				!= (sizeof(u64) * BITS_PER_BYTE));
+
+	/* scan the device tree for pkey feature */
+	scan_pkey_feature();
+
+	/*
+	 * Let's assume 32 pkeys on P8 bare metal, if its not defined by device
+	 * tree. We make this exception since skiboot forgot to expose this
+	 * property on power8.
+	 */
+	if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) &&
+			cpu_has_feature(CPU_FTRS_POWER8))
+		pkeys_total = 32;
+
+	/*
+	 * Adjust the upper limit, based on the number of bits supported by
+	 * arch-neutral code.
+	 */
+	pkeys_total = min_t(int, pkeys_total,
+			(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT));
+
+	if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total)
+		static_branch_enable(&pkey_disabled);
+	else
+		static_branch_disable(&pkey_disabled);
+
+	if (static_branch_likely(&pkey_disabled))
+		return 0;
+
+	/*
+	 * The device tree cannot be relied to indicate support for
+	 * execute_disable support. Instead we use a PVR check.
+	 */
+	if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p))
+		pkey_execute_disable_supported = false;
+	else
+		pkey_execute_disable_supported = true;
+
+#ifdef CONFIG_PPC_4K_PAGES
+	/*
+	 * The OS can manage only 8 pkeys due to its inability to represent them
+	 * in the Linux 4K PTE.
+	 */
+	os_reserved = pkeys_total - 8;
+#else
+	os_reserved = 0;
+#endif
+	/*
+	 * Bits are in LE format. NOTE: 1, 0 are reserved.
+	 * key 0 is the default key, which allows read/write/execute.
+	 * key 1 is recommended not to be used. PowerISA(3.0) page 1015,
+	 * programming note.
+	 */
+	initial_allocation_mask = ~0x0;
+
+	/* register mask is in BE format */
+	pkey_amr_uamor_mask = ~0x0ul;
+	pkey_iamr_mask = ~0x0ul;
+
+	for (i = 2; i < (pkeys_total - os_reserved); i++) {
+		initial_allocation_mask &= ~(0x1 << i);
+		pkey_amr_uamor_mask &= ~(0x3ul << pkeyshift(i));
+		pkey_iamr_mask &= ~(0x1ul << pkeyshift(i));
+	}
+	return 0;
+}
+
+arch_initcall(pkey_initialize);
+
+void pkey_mm_init(struct mm_struct *mm)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return;
+	mm_pkey_allocation_map(mm) = initial_allocation_mask;
+	/* -1 means unallocated or invalid */
+	mm->context.execute_only_pkey = -1;
+}
+
+static inline u64 read_amr(void)
+{
+	return mfspr(SPRN_AMR);
+}
+
+static inline void write_amr(u64 value)
+{
+	mtspr(SPRN_AMR, value);
+}
+
+static inline u64 read_iamr(void)
+{
+	if (!likely(pkey_execute_disable_supported))
+		return 0x0UL;
+
+	return mfspr(SPRN_IAMR);
+}
+
+static inline void write_iamr(u64 value)
+{
+	if (!likely(pkey_execute_disable_supported))
+		return;
+
+	mtspr(SPRN_IAMR, value);
+}
+
+static inline u64 read_uamor(void)
+{
+	return mfspr(SPRN_UAMOR);
+}
+
+static inline void write_uamor(u64 value)
+{
+	mtspr(SPRN_UAMOR, value);
+}
+
+static bool is_pkey_enabled(int pkey)
+{
+	u64 uamor = read_uamor();
+	u64 pkey_bits = 0x3ul << pkeyshift(pkey);
+	u64 uamor_pkey_bits = (uamor & pkey_bits);
+
+	/*
+	 * Both the bits in UAMOR corresponding to the key should be set or
+	 * reset.
+	 */
+	WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits));
+	return !!(uamor_pkey_bits);
+}
+
+static inline void init_amr(int pkey, u8 init_bits)
+{
+	u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey));
+	u64 old_amr = read_amr() & ~((u64)(0x3ul) << pkeyshift(pkey));
+
+	write_amr(old_amr | new_amr_bits);
+}
+
+static inline void init_iamr(int pkey, u8 init_bits)
+{
+	u64 new_iamr_bits = (((u64)init_bits & 0x1UL) << pkeyshift(pkey));
+	u64 old_iamr = read_iamr() & ~((u64)(0x1ul) << pkeyshift(pkey));
+
+	write_iamr(old_iamr | new_iamr_bits);
+}
+
+static void pkey_status_change(int pkey, bool enable)
+{
+	u64 old_uamor;
+
+	/* Reset the AMR and IAMR bits for this key */
+	init_amr(pkey, 0x0);
+	init_iamr(pkey, 0x0);
+
+	/* Enable/disable key */
+	old_uamor = read_uamor();
+	if (enable)
+		old_uamor |= (0x3ul << pkeyshift(pkey));
+	else
+		old_uamor &= ~(0x3ul << pkeyshift(pkey));
+	write_uamor(old_uamor);
+}
+
+void __arch_activate_pkey(int pkey)
+{
+	pkey_status_change(pkey, true);
+}
+
+void __arch_deactivate_pkey(int pkey)
+{
+	pkey_status_change(pkey, false);
+}
+
+/*
+ * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that
+ * specified in @init_val.
+ */
+int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
+				unsigned long init_val)
+{
+	u64 new_amr_bits = 0x0ul;
+	u64 new_iamr_bits = 0x0ul;
+
+	if (!is_pkey_enabled(pkey))
+		return -EINVAL;
+
+	if (init_val & PKEY_DISABLE_EXECUTE) {
+		if (!pkey_execute_disable_supported)
+			return -EINVAL;
+		new_iamr_bits |= IAMR_EX_BIT;
+	}
+	init_iamr(pkey, new_iamr_bits);
+
+	/* Set the bits we need in AMR: */
+	if (init_val & PKEY_DISABLE_ACCESS)
+		new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT;
+	else if (init_val & PKEY_DISABLE_WRITE)
+		new_amr_bits |= AMR_WR_BIT;
+
+	init_amr(pkey, new_amr_bits);
+	return 0;
+}
+
+void thread_pkey_regs_save(struct thread_struct *thread)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return;
+
+	/*
+	 * TODO: Skip saving registers if @thread hasn't used any keys yet.
+	 */
+	thread->amr = read_amr();
+	thread->iamr = read_iamr();
+	thread->uamor = read_uamor();
+}
+
+void thread_pkey_regs_restore(struct thread_struct *new_thread,
+			      struct thread_struct *old_thread)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return;
+
+	/*
+	 * TODO: Just set UAMOR to zero if @new_thread hasn't used any keys yet.
+	 */
+	if (old_thread->amr != new_thread->amr)
+		write_amr(new_thread->amr);
+	if (old_thread->iamr != new_thread->iamr)
+		write_iamr(new_thread->iamr);
+	if (old_thread->uamor != new_thread->uamor)
+		write_uamor(new_thread->uamor);
+}
+
+void thread_pkey_regs_init(struct thread_struct *thread)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return;
+
+	write_amr(read_amr() & pkey_amr_uamor_mask);
+	write_iamr(read_iamr() & pkey_iamr_mask);
+	write_uamor(read_uamor() & pkey_amr_uamor_mask);
+}
+
+static inline bool pkey_allows_readwrite(int pkey)
+{
+	int pkey_shift = pkeyshift(pkey);
+
+	if (!is_pkey_enabled(pkey))
+		return true;
+
+	return !(read_amr() & ((AMR_RD_BIT|AMR_WR_BIT) << pkey_shift));
+}
+
+int __execute_only_pkey(struct mm_struct *mm)
+{
+	bool need_to_set_mm_pkey = false;
+	int execute_only_pkey = mm->context.execute_only_pkey;
+	int ret;
+
+	/* Do we need to assign a pkey for mm's execute-only maps? */
+	if (execute_only_pkey == -1) {
+		/* Go allocate one to use, which might fail */
+		execute_only_pkey = mm_pkey_alloc(mm);
+		if (execute_only_pkey < 0)
+			return -1;
+		need_to_set_mm_pkey = true;
+	}
+
+	/*
+	 * We do not want to go through the relatively costly dance to set AMR
+	 * if we do not need to. Check it first and assume that if the
+	 * execute-only pkey is readwrite-disabled than we do not have to set it
+	 * ourselves.
+	 */
+	if (!need_to_set_mm_pkey && !pkey_allows_readwrite(execute_only_pkey))
+		return execute_only_pkey;
+
+	/*
+	 * Set up AMR so that it denies access for everything other than
+	 * execution.
+	 */
+	ret = __arch_set_user_pkey_access(current, execute_only_pkey,
+					  PKEY_DISABLE_ACCESS |
+					  PKEY_DISABLE_WRITE);
+	/*
+	 * If the AMR-set operation failed somehow, just return 0 and
+	 * effectively disable execute-only support.
+	 */
+	if (ret) {
+		mm_pkey_free(mm, execute_only_pkey);
+		return -1;
+	}
+
+	/* We got one, store it and use it from here on out */
+	if (need_to_set_mm_pkey)
+		mm->context.execute_only_pkey = execute_only_pkey;
+	return execute_only_pkey;
+}
+
+static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma)
+{
+	/* Do this check first since the vm_flags should be hot */
+	if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC)
+		return false;
+
+	return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey);
+}
+
+/*
+ * This should only be called for *plain* mprotect calls.
+ */
+int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot,
+				  int pkey)
+{
+	/*
+	 * If the currently associated pkey is execute-only, but the requested
+	 * protection requires read or write, move it back to the default pkey.
+	 */
+	if (vma_is_pkey_exec_only(vma) && (prot & (PROT_READ | PROT_WRITE)))
+		return 0;
+
+	/*
+	 * The requested protection is execute-only. Hence let's use an
+	 * execute-only pkey.
+	 */
+	if (prot == PROT_EXEC) {
+		pkey = execute_only_pkey(vma->vm_mm);
+		if (pkey > 0)
+			return pkey;
+	}
+
+	/* Nothing to override. */
+	return vma_pkey(vma);
+}
+
+static bool pkey_access_permitted(int pkey, bool write, bool execute)
+{
+	int pkey_shift;
+	u64 amr;
+
+	if (!pkey)
+		return true;
+
+	if (!is_pkey_enabled(pkey))
+		return true;
+
+	pkey_shift = pkeyshift(pkey);
+	if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift)))
+		return true;
+
+	amr = read_amr(); /* Delay reading amr until absolutely needed */
+	return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) ||
+		(write &&  !(amr & (AMR_WR_BIT << pkey_shift))));
+}
+
+bool arch_pte_access_permitted(u64 pte, bool write, bool execute)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return true;
+
+	return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute);
+}
+
+/*
+ * We only want to enforce protection keys on the current thread because we
+ * effectively have no access to AMR/IAMR for other threads or any way to tell
+ * which AMR/IAMR in a threaded process we could use.
+ *
+ * So do not enforce things if the VMA is not from the current mm, or if we are
+ * in a kernel thread.
+ */
+static inline bool vma_is_foreign(struct vm_area_struct *vma)
+{
+	if (!current->mm)
+		return true;
+
+	/* if it is not our ->mm, it has to be foreign */
+	if (current->mm != vma->vm_mm)
+		return true;
+
+	return false;
+}
+
+bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write,
+			       bool execute, bool foreign)
+{
+	if (static_branch_likely(&pkey_disabled))
+		return true;
+	/*
+	 * Do not enforce our key-permissions on a foreign vma.
+	 */
+	if (foreign || vma_is_foreign(vma))
+		return true;
+
+	return pkey_access_permitted(vma_pkey(vma), write, execute);
+}
diff --git a/arch/powerpc/mm/subpage-prot.c b/arch/powerpc/mm/subpage-prot.c
index 781532d7bc4d..f14a07c2fb90 100644
--- a/arch/powerpc/mm/subpage-prot.c
+++ b/arch/powerpc/mm/subpage-prot.c
@@ -195,6 +195,9 @@ long sys_subpage_prot(unsigned long addr, unsigned long len, u32 __user *map)
 	unsigned long next, limit;
 	int err;
 
+	if (radix_enabled())
+		return -ENOENT;
+
 	/* Check parameters */
 	if ((addr & ~PAGE_MASK) || (len & ~PAGE_MASK) ||
 	    addr >= mm->task_size || len >= mm->task_size ||
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 884f4b705b57..a07f5372a4bf 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -23,6 +23,72 @@
 #define RIC_FLUSH_PWC 1
 #define RIC_FLUSH_ALL 2
 
+/*
+ * tlbiel instruction for radix, set invalidation
+ * i.e., r=1 and is=01 or is=10 or is=11
+ */
+static inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is,
+					unsigned int pid,
+					unsigned int ric, unsigned int prs)
+{
+	unsigned long rb;
+	unsigned long rs;
+	unsigned int r = 1; /* radix format */
+
+	rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
+	rs = ((unsigned long)pid << PPC_BITLSHIFT(31));
+
+	asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4)
+		     : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r)
+		     : "memory");
+}
+
+static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)
+{
+	unsigned int set;
+
+	asm volatile("ptesync": : :"memory");
+
+	/*
+	 * Flush the first set of the TLB, and the entire Page Walk Cache
+	 * and partition table entries. Then flush the remaining sets of the
+	 * TLB.
+	 */
+	tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0);
+	for (set = 1; set < num_sets; set++)
+		tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0);
+
+	/* Do the same for process scoped entries. */
+	tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1);
+	for (set = 1; set < num_sets; set++)
+		tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1);
+
+	asm volatile("ptesync": : :"memory");
+}
+
+void radix__tlbiel_all(unsigned int action)
+{
+	unsigned int is;
+
+	switch (action) {
+	case TLB_INVAL_SCOPE_GLOBAL:
+		is = 3;
+		break;
+	case TLB_INVAL_SCOPE_LPID:
+		is = 2;
+		break;
+	default:
+		BUG();
+	}
+
+	if (early_cpu_has_feature(CPU_FTR_ARCH_300))
+		tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is);
+	else
+		WARN(1, "%s called on pre-POWER9 CPU\n", __func__);
+
+	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
+}
+
 static inline void __tlbiel_pid(unsigned long pid, int set,
 				unsigned long ric)
 {
@@ -53,6 +119,49 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
 	trace_tlbie(0, 0, rb, rs, ric, prs, r);
 }
 
+static inline void __tlbiel_va(unsigned long va, unsigned long pid,
+			       unsigned long ap, unsigned long ric)
+{
+	unsigned long rb,rs,prs,r;
+
+	rb = va & ~(PPC_BITMASK(52, 63));
+	rb |= ap << PPC_BITLSHIFT(58);
+	rs = pid << PPC_BITLSHIFT(31);
+	prs = 1; /* process scoped */
+	r = 1;   /* raidx format */
+
+	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
+		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	trace_tlbie(0, 1, rb, rs, ric, prs, r);
+}
+
+static inline void __tlbie_va(unsigned long va, unsigned long pid,
+			      unsigned long ap, unsigned long ric)
+{
+	unsigned long rb,rs,prs,r;
+
+	rb = va & ~(PPC_BITMASK(52, 63));
+	rb |= ap << PPC_BITLSHIFT(58);
+	rs = pid << PPC_BITLSHIFT(31);
+	prs = 1; /* process scoped */
+	r = 1;   /* raidx format */
+
+	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	trace_tlbie(0, 0, rb, rs, ric, prs, r);
+}
+
+static inline void fixup_tlbie(void)
+{
+	unsigned long pid = 0;
+	unsigned long va = ((1UL << 52) - 1);
+
+	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+		asm volatile("ptesync": : :"memory");
+		__tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
+	}
+}
+
 /*
  * We use 128 set in radix mode and 256 set in hpt mode.
  */
@@ -85,24 +194,25 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
 {
 	asm volatile("ptesync": : :"memory");
-	__tlbie_pid(pid, ric);
-	asm volatile("eieio; tlbsync; ptesync": : :"memory");
-}
-
-static inline void __tlbiel_va(unsigned long va, unsigned long pid,
-			       unsigned long ap, unsigned long ric)
-{
-	unsigned long rb,rs,prs,r;
 
-	rb = va & ~(PPC_BITMASK(52, 63));
-	rb |= ap << PPC_BITLSHIFT(58);
-	rs = pid << PPC_BITLSHIFT(31);
-	prs = 1; /* process scoped */
-	r = 1;   /* raidx format */
-
-	asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
-		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-	trace_tlbie(0, 1, rb, rs, ric, prs, r);
+	/*
+	 * Workaround the fact that the "ric" argument to __tlbie_pid
+	 * must be a compile-time contraint to match the "i" constraint
+	 * in the asm statement.
+	 */
+	switch (ric) {
+	case RIC_FLUSH_TLB:
+		__tlbie_pid(pid, RIC_FLUSH_TLB);
+		break;
+	case RIC_FLUSH_PWC:
+		__tlbie_pid(pid, RIC_FLUSH_PWC);
+		break;
+	case RIC_FLUSH_ALL:
+	default:
+		__tlbie_pid(pid, RIC_FLUSH_ALL);
+	}
+	fixup_tlbie();
+	asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
 static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
@@ -137,22 +247,6 @@ static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
 	asm volatile("ptesync": : :"memory");
 }
 
-static inline void __tlbie_va(unsigned long va, unsigned long pid,
-			     unsigned long ap, unsigned long ric)
-{
-	unsigned long rb,rs,prs,r;
-
-	rb = va & ~(PPC_BITMASK(52, 63));
-	rb |= ap << PPC_BITLSHIFT(58);
-	rs = pid << PPC_BITLSHIFT(31);
-	prs = 1; /* process scoped */
-	r = 1;   /* raidx format */
-
-	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
-		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-	trace_tlbie(0, 0, rb, rs, ric, prs, r);
-}
-
 static inline void __tlbie_va_range(unsigned long start, unsigned long end,
 				    unsigned long pid, unsigned long page_size,
 				    unsigned long psize)
@@ -171,6 +265,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
 
 	asm volatile("ptesync": : :"memory");
 	__tlbie_va(va, pid, ap, ric);
+	fixup_tlbie();
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
@@ -182,6 +277,7 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end,
 	if (also_pwc)
 		__tlbie_pid(pid, RIC_FLUSH_PWC);
 	__tlbie_va_range(start, end, pid, page_size, psize);
+	fixup_tlbie();
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
@@ -245,6 +341,16 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
 }
 EXPORT_SYMBOL(radix__local_flush_tlb_page);
 
+static bool mm_needs_flush_escalation(struct mm_struct *mm)
+{
+	/*
+	 * P9 nest MMU has issues with the page walk cache
+	 * caching PTEs and not flushing them properly when
+	 * RIC = 0 for a PID/LPID invalidate
+	 */
+	return atomic_read(&mm->context.copros) != 0;
+}
+
 #ifdef CONFIG_SMP
 void radix__flush_tlb_mm(struct mm_struct *mm)
 {
@@ -255,9 +361,12 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
 		return;
 
 	preempt_disable();
-	if (!mm_is_thread_local(mm))
-		_tlbie_pid(pid, RIC_FLUSH_TLB);
-	else
+	if (!mm_is_thread_local(mm)) {
+		if (mm_needs_flush_escalation(mm))
+			_tlbie_pid(pid, RIC_FLUSH_ALL);
+		else
+			_tlbie_pid(pid, RIC_FLUSH_TLB);
+	} else
 		_tlbiel_pid(pid, RIC_FLUSH_TLB);
 	preempt_enable();
 }
@@ -369,10 +478,14 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 	}
 
 	if (full) {
-		if (local)
+		if (local) {
 			_tlbiel_pid(pid, RIC_FLUSH_TLB);
-		else
-			_tlbie_pid(pid, RIC_FLUSH_TLB);
+		} else {
+			if (mm_needs_flush_escalation(mm))
+				_tlbie_pid(pid, RIC_FLUSH_ALL);
+			else
+				_tlbie_pid(pid, RIC_FLUSH_TLB);
+		}
 	} else {
 		bool hflush = false;
 		unsigned long hstart, hend;
@@ -399,6 +512,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 			if (hflush)
 				__tlbie_va_range(hstart, hend, pid,
 						HPAGE_PMD_SIZE, MMU_PAGE_2M);
+			fixup_tlbie();
 			asm volatile("eieio; tlbsync; ptesync": : :"memory");
 		}
 	}
@@ -482,6 +596,9 @@ static inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
 	}
 
 	if (full) {
+		if (!local && mm_needs_flush_escalation(mm))
+			also_pwc = true;
+
 		if (local)
 			_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
 		else
@@ -537,46 +654,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
-void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
-			      unsigned long page_size)
-{
-	unsigned long rb,rs,prs,r;
-	unsigned long ap;
-	unsigned long ric = RIC_FLUSH_TLB;
-
-	ap = mmu_get_ap(radix_get_mmu_psize(page_size));
-	rb = gpa & ~(PPC_BITMASK(52, 63));
-	rb |= ap << PPC_BITLSHIFT(58);
-	rs = lpid & ((1UL << 32) - 1);
-	prs = 0; /* process scoped */
-	r = 1;   /* raidx format */
-
-	asm volatile("ptesync": : :"memory");
-	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
-		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-	asm volatile("eieio; tlbsync; ptesync": : :"memory");
-	trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
-}
-EXPORT_SYMBOL(radix__flush_tlb_lpid_va);
-
-void radix__flush_tlb_lpid(unsigned long lpid)
-{
-	unsigned long rb,rs,prs,r;
-	unsigned long ric = RIC_FLUSH_ALL;
-
-	rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
-	rs = lpid & ((1UL << 32) - 1);
-	prs = 0; /* partition scoped */
-	r = 1;   /* raidx format */
-
-	asm volatile("ptesync": : :"memory");
-	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
-		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-	asm volatile("eieio; tlbsync; ptesync": : :"memory");
-	trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
-}
-EXPORT_SYMBOL(radix__flush_tlb_lpid);
-
 void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
 				unsigned long start, unsigned long end)
 {
@@ -600,14 +677,12 @@ void radix__flush_tlb_all(void)
 	 */
 	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
 		     : : "r"(rb), "i"(r), "i"(1), "i"(ric), "r"(rs) : "memory");
-	trace_tlbie(0, 0, rb, rs, ric, prs, r);
 	/*
 	 * now flush host entires by passing PRS = 0 and LPID == 0
 	 */
 	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(0) : "memory");
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
-	trace_tlbie(0, 0, rb, 0, ric, prs, r);
 }
 
 void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
diff --git a/arch/powerpc/mm/tlb_hash64.c b/arch/powerpc/mm/tlb_hash64.c
index 881ebd53ffc2..9b23f12e863c 100644
--- a/arch/powerpc/mm/tlb_hash64.c
+++ b/arch/powerpc/mm/tlb_hash64.c
@@ -51,7 +51,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
 	unsigned int psize;
 	int ssize;
 	real_pte_t rpte;
-	int i;
+	int i, offset;
 
 	i = batch->index;
 
@@ -67,6 +67,10 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
 		psize = get_slice_psize(mm, addr);
 		/* Mask the address for the correct page size */
 		addr &= ~((1UL << mmu_psize_defs[psize].shift) - 1);
+		if (unlikely(psize == MMU_PAGE_16G))
+			offset = PTRS_PER_PUD;
+		else
+			offset = PTRS_PER_PMD;
 #else
 		BUG();
 		psize = pte_pagesize_index(mm, addr, pte); /* shutup gcc */
@@ -78,6 +82,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
 		 * support 64k pages, this might be different from the
 		 * hardware page size encoded in the slice table. */
 		addr &= PAGE_MASK;
+		offset = PTRS_PER_PTE;
 	}
 
 
@@ -91,7 +96,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
 	}
 	WARN_ON(vsid == 0);
 	vpn = hpt_vpn(addr, vsid, ssize);
-	rpte = __real_pte(__pte(pte), ptep);
+	rpte = __real_pte(__pte(pte), ptep, offset);
 
 	/*
 	 * Check if we have an active batch on this CPU. If not, just
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index bfc4a0869609..15fe5f0c8665 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -388,7 +388,10 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 		     unsigned long end)
 
 {
-	flush_tlb_mm(vma->vm_mm);
+	if (end - start == PAGE_SIZE && !(start & ~PAGE_MASK))
+		flush_tlb_page(vma, start);
+	else
+		flush_tlb_mm(vma->vm_mm);
 }
 EXPORT_SYMBOL(flush_tlb_range);
 
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index f9941b3b5770..a9636d8cba15 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -18,8 +18,6 @@
 
 #include "bpf_jit32.h"
 
-int bpf_jit_enable __read_mostly;
-
 static inline void bpf_flush_icache(void *start, void *end)
 {
 	smp_wmb();
@@ -329,6 +327,9 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 			BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
 			PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, len));
 			break;
+		case BPF_LDX | BPF_W | BPF_ABS: /* A = *((u32 *)(seccomp_data + K)); */
+			PPC_LWZ_OFFS(r_A, r_skb, K);
+			break;
 		case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */
 			PPC_LWZ_OFFS(r_X, r_skb, offsetof(struct sk_buff, len));
 			break;
diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index d183b4801bdb..0ef3d9580e98 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -21,8 +21,6 @@
 
 #include "bpf_jit64.h"
 
-int bpf_jit_enable __read_mostly;
-
 static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
 {
 	memset32(area, BREAKPOINT_INSTRUCTION, size/4);
@@ -242,6 +240,7 @@ static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32
 	 *   goto out;
 	 */
 	PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries));
+	PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31);
 	PPC_CMPLW(b2p_index, b2p[TMP_REG_1]);
 	PPC_BCC(COND_GE, out);
 
@@ -383,10 +382,6 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 			goto bpf_alu32_trunc;
 		case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
 		case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
-			PPC_CMPWI(src_reg, 0);
-			PPC_BCC_SHORT(COND_NE, (ctx->idx * 4) + 12);
-			PPC_LI(b2p[BPF_REG_0], 0);
-			PPC_JMP(exit_addr);
 			if (BPF_OP(code) == BPF_MOD) {
 				PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg);
 				PPC_MULW(b2p[TMP_REG_1], src_reg,
@@ -397,10 +392,6 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
 			goto bpf_alu32_trunc;
 		case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
 		case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
-			PPC_CMPDI(src_reg, 0);
-			PPC_BCC_SHORT(COND_NE, (ctx->idx * 4) + 12);
-			PPC_LI(b2p[BPF_REG_0], 0);
-			PPC_JMP(exit_addr);
 			if (BPF_OP(code) == BPF_MOD) {
 				PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg);
 				PPC_MULD(b2p[TMP_REG_1], src_reg,
@@ -995,7 +986,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	struct bpf_prog *tmp_fp;
 	bool bpf_blinded = false;
 
-	if (!bpf_jit_enable)
+	if (!fp->jit_requested)
 		return org_fp;
 
 	tmp_fp = bpf_jit_blind_constants(org_fp);
diff --git a/arch/powerpc/perf/8xx-pmu.c b/arch/powerpc/perf/8xx-pmu.c
index 3c39f05f0af3..6c0020d1c561 100644
--- a/arch/powerpc/perf/8xx-pmu.c
+++ b/arch/powerpc/perf/8xx-pmu.c
@@ -18,6 +18,7 @@
 #include <asm/machdep.h>
 #include <asm/firmware.h>
 #include <asm/ptrace.h>
+#include <asm/code-patching.h>
 
 #define PERF_8xx_ID_CPU_CYCLES		1
 #define PERF_8xx_ID_HW_INSTRUCTIONS	2
@@ -30,8 +31,13 @@
 
 extern unsigned long itlb_miss_counter, dtlb_miss_counter;
 extern atomic_t instruction_counter;
+extern unsigned int itlb_miss_perf, dtlb_miss_perf;
+extern unsigned int itlb_miss_exit_1, itlb_miss_exit_2;
+extern unsigned int dtlb_miss_exit_1, dtlb_miss_exit_2, dtlb_miss_exit_3;
 
 static atomic_t insn_ctr_ref;
+static atomic_t itlb_miss_ref;
+static atomic_t dtlb_miss_ref;
 
 static s64 get_insn_ctr(void)
 {
@@ -96,9 +102,24 @@ static int mpc8xx_pmu_add(struct perf_event *event, int flags)
 		val = get_insn_ctr();
 		break;
 	case PERF_8xx_ID_ITLB_LOAD_MISS:
+		if (atomic_inc_return(&itlb_miss_ref) == 1) {
+			unsigned long target = (unsigned long)&itlb_miss_perf;
+
+			patch_branch(&itlb_miss_exit_1, target, 0);
+#ifndef CONFIG_PIN_TLB_TEXT
+			patch_branch(&itlb_miss_exit_2, target, 0);
+#endif
+		}
 		val = itlb_miss_counter;
 		break;
 	case PERF_8xx_ID_DTLB_LOAD_MISS:
+		if (atomic_inc_return(&dtlb_miss_ref) == 1) {
+			unsigned long target = (unsigned long)&dtlb_miss_perf;
+
+			patch_branch(&dtlb_miss_exit_1, target, 0);
+			patch_branch(&dtlb_miss_exit_2, target, 0);
+			patch_branch(&dtlb_miss_exit_3, target, 0);
+		}
 		val = dtlb_miss_counter;
 		break;
 	}
@@ -143,13 +164,36 @@ static void mpc8xx_pmu_read(struct perf_event *event)
 
 static void mpc8xx_pmu_del(struct perf_event *event, int flags)
 {
+	/* mfspr r10, SPRN_SPRG_SCRATCH0 */
+	unsigned int insn = PPC_INST_MFSPR | __PPC_RS(R10) |
+			    __PPC_SPR(SPRN_SPRG_SCRATCH0);
+
 	mpc8xx_pmu_read(event);
-	if (event_type(event) != PERF_8xx_ID_HW_INSTRUCTIONS)
-		return;
 
 	/* If it was the last user, stop counting to avoid useles overhead */
-	if (atomic_dec_return(&insn_ctr_ref) == 0)
-		mtspr(SPRN_ICTRL, 7);
+	switch (event_type(event)) {
+	case PERF_8xx_ID_CPU_CYCLES:
+		break;
+	case PERF_8xx_ID_HW_INSTRUCTIONS:
+		if (atomic_dec_return(&insn_ctr_ref) == 0)
+			mtspr(SPRN_ICTRL, 7);
+		break;
+	case PERF_8xx_ID_ITLB_LOAD_MISS:
+		if (atomic_dec_return(&itlb_miss_ref) == 0) {
+			patch_instruction(&itlb_miss_exit_1, insn);
+#ifndef CONFIG_PIN_TLB_TEXT
+			patch_instruction(&itlb_miss_exit_2, insn);
+#endif
+		}
+		break;
+	case PERF_8xx_ID_DTLB_LOAD_MISS:
+		if (atomic_dec_return(&dtlb_miss_ref) == 0) {
+			patch_instruction(&dtlb_miss_exit_1, insn);
+			patch_instruction(&dtlb_miss_exit_2, insn);
+			patch_instruction(&dtlb_miss_exit_3, insn);
+		}
+		break;
+	}
 }
 
 static struct pmu mpc8xx_pmu = {
diff --git a/arch/powerpc/perf/Makefile b/arch/powerpc/perf/Makefile
index 225c9c86d7c0..57ebc655d2ac 100644
--- a/arch/powerpc/perf/Makefile
+++ b/arch/powerpc/perf/Makefile
@@ -15,7 +15,7 @@ obj-$(CONFIG_FSL_EMB_PERF_EVENT_E500) += e500-pmu.o e6500-pmu.o
 
 obj-$(CONFIG_HV_PERF_CTRS) += hv-24x7.o hv-gpci.o hv-common.o
 
-obj-$(CONFIG_PPC_8xx_PERF_EVENT) += 8xx-pmu.o
+obj-$(CONFIG_PPC_8xx) += 8xx-pmu.o
 
 obj-$(CONFIG_PPC64)		+= $(obj64-y)
 obj-$(CONFIG_PPC32)		+= $(obj32-y)
diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index fce545774d50..f89bbd54ecec 100644
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -322,7 +322,7 @@ static inline void perf_read_regs(struct pt_regs *regs)
  */
 static inline int perf_intr_is_nmi(struct pt_regs *regs)
 {
-	return !regs->softe;
+	return (regs->softe & IRQS_DISABLED);
 }
 
 /*
diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c
index be4e7f84f70a..d7532e7b9ab5 100644
--- a/arch/powerpc/perf/imc-pmu.c
+++ b/arch/powerpc/perf/imc-pmu.c
@@ -40,7 +40,6 @@ static struct imc_pmu *core_imc_pmu;
 /* Thread IMC data structures and variables */
 
 static DEFINE_PER_CPU(u64 *, thread_imc_mem);
-static struct imc_pmu *thread_imc_pmu;
 static int thread_imc_mem_size;
 
 struct imc_pmu *imc_event_to_pmu(struct perf_event *event)
@@ -117,17 +116,13 @@ static struct attribute *device_str_attr_create(const char *name, const char *st
 	return &attr->attr.attr;
 }
 
-struct imc_events *imc_parse_event(struct device_node *np, const char *scale,
-				  const char *unit, const char *prefix, u32 base)
+static int imc_parse_event(struct device_node *np, const char *scale,
+				  const char *unit, const char *prefix,
+				  u32 base, struct imc_events *event)
 {
-	struct imc_events *event;
 	const char *s;
 	u32 reg;
 
-	event = kzalloc(sizeof(struct imc_events), GFP_KERNEL);
-	if (!event)
-		return NULL;
-
 	if (of_property_read_u32(np, "reg", &reg))
 		goto error;
 	/* Add the base_reg value to the "reg" */
@@ -158,14 +153,32 @@ struct imc_events *imc_parse_event(struct device_node *np, const char *scale,
 			goto error;
 	}
 
-	return event;
+	return 0;
 error:
 	kfree(event->unit);
 	kfree(event->scale);
 	kfree(event->name);
-	kfree(event);
+	return -EINVAL;
+}
+
+/*
+ * imc_free_events: Function to cleanup the events list, having
+ * 		    "nr_entries".
+ */
+static void imc_free_events(struct imc_events *events, int nr_entries)
+{
+	int i;
 
-	return NULL;
+	/* Nothing to clean, return */
+	if (!events)
+		return;
+	for (i = 0; i < nr_entries; i++) {
+		kfree(events[i].unit);
+		kfree(events[i].scale);
+		kfree(events[i].name);
+	}
+
+	kfree(events);
 }
 
 /*
@@ -177,9 +190,8 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu)
 	struct attribute_group *attr_group;
 	struct attribute **attrs, *dev_str;
 	struct device_node *np, *pmu_events;
-	struct imc_events *ev;
 	u32 handle, base_reg;
-	int i=0, j=0, ct;
+	int i = 0, j = 0, ct, ret;
 	const char *prefix, *g_scale, *g_unit;
 	const char *ev_val_str, *ev_scale_str, *ev_unit_str;
 
@@ -217,15 +229,17 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu)
 	ct = 0;
 	/* Parse the events and update the struct */
 	for_each_child_of_node(pmu_events, np) {
-		ev = imc_parse_event(np, g_scale, g_unit, prefix, base_reg);
-		if (ev)
-			pmu->events[ct++] = ev;
+		ret = imc_parse_event(np, g_scale, g_unit, prefix, base_reg, &pmu->events[ct]);
+		if (!ret)
+			ct++;
 	}
 
 	/* Allocate memory for attribute group */
 	attr_group = kzalloc(sizeof(*attr_group), GFP_KERNEL);
-	if (!attr_group)
+	if (!attr_group) {
+		imc_free_events(pmu->events, ct);
 		return -ENOMEM;
+	}
 
 	/*
 	 * Allocate memory for attributes.
@@ -238,31 +252,31 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu)
 	attrs = kcalloc(((ct * 3) + 1), sizeof(struct attribute *), GFP_KERNEL);
 	if (!attrs) {
 		kfree(attr_group);
-		kfree(pmu->events);
+		imc_free_events(pmu->events, ct);
 		return -ENOMEM;
 	}
 
 	attr_group->name = "events";
 	attr_group->attrs = attrs;
 	do {
-		ev_val_str = kasprintf(GFP_KERNEL, "event=0x%x", pmu->events[i]->value);
-		dev_str = device_str_attr_create(pmu->events[i]->name, ev_val_str);
+		ev_val_str = kasprintf(GFP_KERNEL, "event=0x%x", pmu->events[i].value);
+		dev_str = device_str_attr_create(pmu->events[i].name, ev_val_str);
 		if (!dev_str)
 			continue;
 
 		attrs[j++] = dev_str;
-		if (pmu->events[i]->scale) {
-			ev_scale_str = kasprintf(GFP_KERNEL, "%s.scale",pmu->events[i]->name);
-			dev_str = device_str_attr_create(ev_scale_str, pmu->events[i]->scale);
+		if (pmu->events[i].scale) {
+			ev_scale_str = kasprintf(GFP_KERNEL, "%s.scale", pmu->events[i].name);
+			dev_str = device_str_attr_create(ev_scale_str, pmu->events[i].scale);
 			if (!dev_str)
 				continue;
 
 			attrs[j++] = dev_str;
 		}
 
-		if (pmu->events[i]->unit) {
-			ev_unit_str = kasprintf(GFP_KERNEL, "%s.unit",pmu->events[i]->name);
-			dev_str = device_str_attr_create(ev_unit_str, pmu->events[i]->unit);
+		if (pmu->events[i].unit) {
+			ev_unit_str = kasprintf(GFP_KERNEL, "%s.unit", pmu->events[i].name);
+			dev_str = device_str_attr_create(ev_unit_str, pmu->events[i].unit);
 			if (!dev_str)
 				continue;
 
@@ -273,7 +287,6 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu)
 	/* Save the event attribute */
 	pmu->attr_groups[IMC_EVENT_ATTR] = attr_group;
 
-	kfree(pmu->events);
 	return 0;
 }
 
@@ -611,7 +624,8 @@ static int ppc_core_imc_cpu_online(unsigned int cpu)
 
 static int ppc_core_imc_cpu_offline(unsigned int cpu)
 {
-	unsigned int ncpu, core_id;
+	unsigned int core_id;
+	int ncpu;
 	struct imc_pmu_ref *ref;
 
 	/*
@@ -1171,6 +1185,15 @@ static void cleanup_all_thread_imc_memory(void)
 	}
 }
 
+/* Function to free the attr_groups which are dynamically allocated */
+static void imc_common_mem_free(struct imc_pmu *pmu_ptr)
+{
+	if (pmu_ptr->attr_groups[IMC_EVENT_ATTR])
+		kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]->attrs);
+	kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]);
+	kfree(pmu_ptr);
+}
+
 /*
  * Common function to unregister cpu hotplug callback and
  * free the memory.
@@ -1203,13 +1226,6 @@ static void imc_common_cpuhp_mem_free(struct imc_pmu *pmu_ptr)
 		cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE);
 		cleanup_all_thread_imc_memory();
 	}
-
-	/* Only free the attr_groups which are dynamically allocated  */
-	if (pmu_ptr->attr_groups[IMC_EVENT_ATTR])
-		kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]->attrs);
-	kfree(pmu_ptr->attr_groups[IMC_EVENT_ATTR]);
-	kfree(pmu_ptr);
-	return;
 }
 
 
@@ -1258,8 +1274,10 @@ static int imc_mem_init(struct imc_pmu *pmu_ptr, struct device_node *parent,
 		core_imc_refc = kcalloc(nr_cores, sizeof(struct imc_pmu_ref),
 								GFP_KERNEL);
 
-		if (!core_imc_refc)
+		if (!core_imc_refc) {
+			kfree(pmu_ptr->mem_info);
 			return -ENOMEM;
+		}
 
 		core_imc_pmu = pmu_ptr;
 		break;
@@ -1272,11 +1290,12 @@ static int imc_mem_init(struct imc_pmu *pmu_ptr, struct device_node *parent,
 		thread_imc_mem_size = pmu_ptr->counter_mem_size;
 		for_each_online_cpu(cpu) {
 			res = thread_imc_mem_alloc(cpu, pmu_ptr->counter_mem_size);
-			if (res)
+			if (res) {
+				cleanup_all_thread_imc_memory();
 				return res;
+			}
 		}
 
-		thread_imc_pmu = pmu_ptr;
 		break;
 	default:
 		return -EINVAL;
@@ -1300,8 +1319,10 @@ int init_imc_pmu(struct device_node *parent, struct imc_pmu *pmu_ptr, int pmu_id
 	int ret;
 
 	ret = imc_mem_init(pmu_ptr, parent, pmu_idx);
-	if (ret)
-		goto err_free;
+	if (ret) {
+		imc_common_mem_free(pmu_ptr);
+		return ret;
+	}
 
 	switch (pmu_ptr->domain) {
 	case IMC_DOMAIN_NEST:
@@ -1368,6 +1389,7 @@ int init_imc_pmu(struct device_node *parent, struct imc_pmu *pmu_ptr, int pmu_id
 	return 0;
 
 err_free:
+	imc_common_mem_free(pmu_ptr);
 	imc_common_cpuhp_mem_free(pmu_ptr);
 	return ret;
 }
diff --git a/arch/powerpc/platforms/44x/fsp2.c b/arch/powerpc/platforms/44x/fsp2.c
index 92e98048404f..04f0c73a9b4f 100644
--- a/arch/powerpc/platforms/44x/fsp2.c
+++ b/arch/powerpc/platforms/44x/fsp2.c
@@ -27,6 +27,17 @@
 #include <asm/time.h>
 #include <asm/uic.h>
 #include <asm/ppc4xx.h>
+#include <asm/dcr.h>
+#include <linux/interrupt.h>
+#include <linux/of_irq.h>
+#include "fsp2.h"
+
+#define FSP2_BUS_ERR	"ibm,bus-error-irq"
+#define FSP2_CMU_ERR	"ibm,cmu-error-irq"
+#define FSP2_CONF_ERR	"ibm,conf-error-irq"
+#define FSP2_OPBD_ERR	"ibm,opbd-error-irq"
+#define FSP2_MCUE	"ibm,mc-ue-irq"
+#define FSP2_RST_WRN	"ibm,reset-warning-irq"
 
 static __initdata struct of_device_id fsp2_of_bus[] = {
 	{ .compatible = "ibm,plb4", },
@@ -35,6 +46,194 @@ static __initdata struct of_device_id fsp2_of_bus[] = {
 	{},
 };
 
+static void l2regs(void)
+{
+	pr_err("L2 Controller:\n");
+	pr_err("MCK:      0x%08x\n", mfl2(L2MCK));
+	pr_err("INT:      0x%08x\n", mfl2(L2INT));
+	pr_err("PLBSTAT0: 0x%08x\n", mfl2(L2PLBSTAT0));
+	pr_err("PLBSTAT1: 0x%08x\n", mfl2(L2PLBSTAT1));
+	pr_err("ARRSTAT0: 0x%08x\n", mfl2(L2ARRSTAT0));
+	pr_err("ARRSTAT1: 0x%08x\n", mfl2(L2ARRSTAT1));
+	pr_err("ARRSTAT2: 0x%08x\n", mfl2(L2ARRSTAT2));
+	pr_err("CPUSTAT:  0x%08x\n", mfl2(L2CPUSTAT));
+	pr_err("RACSTAT0: 0x%08x\n", mfl2(L2RACSTAT0));
+	pr_err("WACSTAT0: 0x%08x\n", mfl2(L2WACSTAT0));
+	pr_err("WACSTAT1: 0x%08x\n", mfl2(L2WACSTAT1));
+	pr_err("WACSTAT2: 0x%08x\n", mfl2(L2WACSTAT2));
+	pr_err("WDFSTAT:  0x%08x\n", mfl2(L2WDFSTAT));
+	pr_err("LOG0:     0x%08x\n", mfl2(L2LOG0));
+	pr_err("LOG1:     0x%08x\n", mfl2(L2LOG1));
+	pr_err("LOG2:     0x%08x\n", mfl2(L2LOG2));
+	pr_err("LOG3:     0x%08x\n", mfl2(L2LOG3));
+	pr_err("LOG4:     0x%08x\n", mfl2(L2LOG4));
+	pr_err("LOG5:     0x%08x\n", mfl2(L2LOG5));
+}
+
+static void show_plbopb_regs(u32 base, int num)
+{
+	pr_err("\nPLBOPB Bridge %d:\n", num);
+	pr_err("GESR0: 0x%08x\n", mfdcr(base + PLB4OPB_GESR0));
+	pr_err("GESR1: 0x%08x\n", mfdcr(base + PLB4OPB_GESR1));
+	pr_err("GESR2: 0x%08x\n", mfdcr(base + PLB4OPB_GESR2));
+	pr_err("GEARU: 0x%08x\n", mfdcr(base + PLB4OPB_GEARU));
+	pr_err("GEAR:  0x%08x\n", mfdcr(base + PLB4OPB_GEAR));
+}
+
+static irqreturn_t bus_err_handler(int irq, void *data)
+{
+	pr_err("Bus Error\n");
+
+	l2regs();
+
+	pr_err("\nPLB6 Controller:\n");
+	pr_err("BC_SHD: 0x%08x\n", mfdcr(DCRN_PLB6_SHD));
+	pr_err("BC_ERR: 0x%08x\n", mfdcr(DCRN_PLB6_ERR));
+
+	pr_err("\nPLB6-to-PLB4 Bridge:\n");
+	pr_err("ESR:  0x%08x\n", mfdcr(DCRN_PLB6PLB4_ESR));
+	pr_err("EARH: 0x%08x\n", mfdcr(DCRN_PLB6PLB4_EARH));
+	pr_err("EARL: 0x%08x\n", mfdcr(DCRN_PLB6PLB4_EARL));
+
+	pr_err("\nPLB4-to-PLB6 Bridge:\n");
+	pr_err("ESR:  0x%08x\n", mfdcr(DCRN_PLB4PLB6_ESR));
+	pr_err("EARH: 0x%08x\n", mfdcr(DCRN_PLB4PLB6_EARH));
+	pr_err("EARL: 0x%08x\n", mfdcr(DCRN_PLB4PLB6_EARL));
+
+	pr_err("\nPLB6-to-MCIF Bridge:\n");
+	pr_err("BESR0: 0x%08x\n", mfdcr(DCRN_PLB6MCIF_BESR0));
+	pr_err("BESR1: 0x%08x\n", mfdcr(DCRN_PLB6MCIF_BESR1));
+	pr_err("BEARH: 0x%08x\n", mfdcr(DCRN_PLB6MCIF_BEARH));
+	pr_err("BEARL: 0x%08x\n", mfdcr(DCRN_PLB6MCIF_BEARL));
+
+	pr_err("\nPLB4 Arbiter:\n");
+	pr_err("P0ESRH 0x%08x\n", mfdcr(DCRN_PLB4_P0ESRH));
+	pr_err("P0ESRL 0x%08x\n", mfdcr(DCRN_PLB4_P0ESRL));
+	pr_err("P0EARH 0x%08x\n", mfdcr(DCRN_PLB4_P0EARH));
+	pr_err("P0EARH 0x%08x\n", mfdcr(DCRN_PLB4_P0EARH));
+	pr_err("P1ESRH 0x%08x\n", mfdcr(DCRN_PLB4_P1ESRH));
+	pr_err("P1ESRL 0x%08x\n", mfdcr(DCRN_PLB4_P1ESRL));
+	pr_err("P1EARH 0x%08x\n", mfdcr(DCRN_PLB4_P1EARH));
+	pr_err("P1EARH 0x%08x\n", mfdcr(DCRN_PLB4_P1EARH));
+
+	show_plbopb_regs(DCRN_PLB4OPB0_BASE, 0);
+	show_plbopb_regs(DCRN_PLB4OPB1_BASE, 1);
+	show_plbopb_regs(DCRN_PLB4OPB2_BASE, 2);
+	show_plbopb_regs(DCRN_PLB4OPB3_BASE, 3);
+
+	pr_err("\nPLB4-to-AHB Bridge:\n");
+	pr_err("ESR:   0x%08x\n", mfdcr(DCRN_PLB4AHB_ESR));
+	pr_err("SEUAR: 0x%08x\n", mfdcr(DCRN_PLB4AHB_SEUAR));
+	pr_err("SELAR: 0x%08x\n", mfdcr(DCRN_PLB4AHB_SELAR));
+
+	pr_err("\nAHB-to-PLB4 Bridge:\n");
+	pr_err("\nESR: 0x%08x\n", mfdcr(DCRN_AHBPLB4_ESR));
+	pr_err("\nEAR: 0x%08x\n", mfdcr(DCRN_AHBPLB4_EAR));
+	panic("Bus Error\n");
+}
+
+static irqreturn_t cmu_err_handler(int irq, void *data) {
+	pr_err("CMU Error\n");
+	pr_err("FIR0: 0x%08x\n", mfcmu(CMUN_FIR0));
+	panic("CMU Error\n");
+}
+
+static irqreturn_t conf_err_handler(int irq, void *data) {
+	pr_err("Configuration Logic Error\n");
+	pr_err("CONF_FIR: 0x%08x\n", mfdcr(DCRN_CONF_FIR_RWC));
+	pr_err("RPERR0:   0x%08x\n", mfdcr(DCRN_CONF_RPERR0));
+	pr_err("RPERR1:   0x%08x\n", mfdcr(DCRN_CONF_RPERR1));
+	panic("Configuration Logic Error\n");
+}
+
+static irqreturn_t opbd_err_handler(int irq, void *data) {
+	panic("OPBD Error\n");
+}
+
+static irqreturn_t mcue_handler(int irq, void *data) {
+	pr_err("DDR: Uncorrectable Error\n");
+	pr_err("MCSTAT:            0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_MCSTAT));
+	pr_err("MCOPT1:            0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_MCOPT1));
+	pr_err("MCOPT2:            0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_MCOPT2));
+	pr_err("PHYSTAT:           0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_PHYSTAT));
+	pr_err("CFGR0:             0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_CFGR0));
+	pr_err("CFGR1:             0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_CFGR1));
+	pr_err("CFGR2:             0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_CFGR2));
+	pr_err("CFGR3:             0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_CFGR3));
+	pr_err("SCRUB_CNTL:        0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_SCRUB_CNTL));
+	pr_err("ECCERR_PORT0:      0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_ECCERR_PORT0));
+	pr_err("ECCERR_ADDR_PORT0: 0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_ECCERR_ADDR_PORT0));
+	pr_err("ECCERR_CNT_PORT0:  0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_ECCERR_COUNT_PORT0));
+	pr_err("ECC_CHECK_PORT0:   0x%08x\n",
+		mfdcr(DCRN_DDR34_BASE + DCRN_DDR34_ECC_CHECK_PORT0));
+	pr_err("MCER0:            0x%08x\n",
+		mfdcr(DCRN_CW_BASE + DCRN_CW_MCER0));
+	pr_err("MCER1:            0x%08x\n",
+		mfdcr(DCRN_CW_BASE + DCRN_CW_MCER1));
+	pr_err("BESR:             0x%08x\n",
+		mfdcr(DCRN_PLB6MCIF_BESR0));
+	pr_err("BEARL:            0x%08x\n",
+		mfdcr(DCRN_PLB6MCIF_BEARL));
+	pr_err("BEARH:            0x%08x\n",
+		mfdcr(DCRN_PLB6MCIF_BEARH));
+	panic("DDR: Uncorrectable Error\n");
+}
+
+static irqreturn_t rst_wrn_handler(int irq, void *data) {
+	u32 crcs = mfcmu(CMUN_CRCS);
+	switch (crcs & CRCS_STAT_MASK) {
+	case CRCS_STAT_CHIP_RST_B:
+		panic("Received chassis-initiated reset request");
+	default:
+		panic("Unknown external reset: CRCS=0x%x", crcs);
+	}
+}
+
+static void node_irq_request(const char *compat, irq_handler_t errirq_handler)
+{
+	struct device_node *np;
+	unsigned int irq;
+	int32_t rc;
+
+	for_each_compatible_node(np, NULL, compat) {
+		irq = irq_of_parse_and_map(np, 0);
+		if (irq == NO_IRQ) {
+			pr_err("device tree node %s is missing a interrupt",
+			      np->name);
+			return;
+		}
+
+		rc = request_irq(irq, errirq_handler, 0, np->name, np);
+		if (rc) {
+			pr_err("fsp_of_probe: request_irq failed: np=%s rc=%d",
+			      np->full_name, rc);
+			return;
+		}
+	}
+}
+
+static void critical_irq_setup(void)
+{
+	node_irq_request(FSP2_CMU_ERR, cmu_err_handler);
+	node_irq_request(FSP2_BUS_ERR, bus_err_handler);
+	node_irq_request(FSP2_CONF_ERR, conf_err_handler);
+	node_irq_request(FSP2_OPBD_ERR, opbd_err_handler);
+	node_irq_request(FSP2_MCUE, mcue_handler);
+	node_irq_request(FSP2_RST_WRN, rst_wrn_handler);
+}
+
 static int __init fsp2_device_probe(void)
 {
 	of_platform_bus_probe(NULL, fsp2_of_bus, NULL);
@@ -44,18 +243,76 @@ machine_device_initcall(fsp2, fsp2_device_probe);
 
 static int __init fsp2_probe(void)
 {
+	u32 val;
 	unsigned long root = of_get_flat_dt_root();
 
 	if (!of_flat_dt_is_compatible(root, "ibm,fsp2"))
 		return 0;
+
+	/* Clear BC_ERR and mask snoopable request plb errors. */
+	val = mfdcr(DCRN_PLB6_CR0);
+	val |= 0x20000000;
+	mtdcr(DCRN_PLB6_BASE, val);
+	mtdcr(DCRN_PLB6_HD, 0xffff0000);
+	mtdcr(DCRN_PLB6_SHD, 0xffff0000);
+
+	/* TVSENSE reset is blocked (clock gated) by the POR default of the TVS
+	 * sleep config bit. As a consequence, TVSENSE will provide erratic
+	 * sensor values, which may result in spurious (parity) errors
+	 * recorded in the CMU FIR and leading to erroneous interrupt requests
+	 * once the CMU interrupt is unmasked.
+	 */
+
+	/* 1. set TVS1[UNDOZE] */
+	val = mfcmu(CMUN_TVS1);
+	val |= 0x4;
+	mtcmu(CMUN_TVS1, val);
+
+	/* 2. clear FIR[TVS] and FIR[TVSPAR] */
+	val = mfcmu(CMUN_FIR0);
+	val |= 0x30000000;
+	mtcmu(CMUN_FIR0, val);
+
+	/* L2 machine checks */
+	mtl2(L2PLBMCKEN0, 0xffffffff);
+	mtl2(L2PLBMCKEN1, 0x0000ffff);
+	mtl2(L2ARRMCKEN0, 0xffffffff);
+	mtl2(L2ARRMCKEN1, 0xffffffff);
+	mtl2(L2ARRMCKEN2, 0xfffff000);
+	mtl2(L2CPUMCKEN,  0xffffffff);
+	mtl2(L2RACMCKEN0, 0xffffffff);
+	mtl2(L2WACMCKEN0, 0xffffffff);
+	mtl2(L2WACMCKEN1, 0xffffffff);
+	mtl2(L2WACMCKEN2, 0xffffffff);
+	mtl2(L2WDFMCKEN,  0xffffffff);
+
+	/* L2 interrupts */
+	mtl2(L2PLBINTEN1, 0xffff0000);
+
+	/*
+	 * At a global level, enable all L2 machine checks and interrupts
+	 * reported by the L2 subsystems, except for the external machine check
+	 * input (UIC0.1).
+	 */
+	mtl2(L2MCKEN, 0x000007ff);
+	mtl2(L2INTEN, 0x000004ff);
+
+	/* Enable FSP-2 configuration logic parity errors */
+	mtdcr(DCRN_CONF_EIR_RS, 0x80000000);
 	return 1;
 }
 
+static void __init fsp2_irq_init(void)
+{
+	uic_init_tree();
+	critical_irq_setup();
+}
+
 define_machine(fsp2) {
 	.name			= "FSP-2",
 	.probe			= fsp2_probe,
 	.progress		= udbg_progress,
-	.init_IRQ		= uic_init_tree,
+	.init_IRQ		= fsp2_irq_init,
 	.get_irq		= uic_get_irq,
 	.restart		= ppc4xx_reset_system,
 	.calibrate_decr		= generic_calibrate_decr,
diff --git a/arch/powerpc/platforms/44x/fsp2.h b/arch/powerpc/platforms/44x/fsp2.h
new file mode 100644
index 000000000000..9e1d52754c8b
--- /dev/null
+++ b/arch/powerpc/platforms/44x/fsp2.h
@@ -0,0 +1,272 @@
+#ifndef _ASM_POWERPC_FSP_DCR_H_
+#define _ASM_POWERPC_FSP_DCR_H_
+#ifdef __KERNEL__
+#include <asm/dcr.h>
+
+#define DCRN_CMU_ADDR		0x00C	/* Chip management unic addr */
+#define DCRN_CMU_DATA		0x00D	/* Chip management unic data */
+
+/* PLB4 Arbiter */
+#define DCRN_PLB4_PCBI		0x010	/* PLB Crossbar ID/Rev Register */
+#define DCRN_PLB4_P0ACR		0x011	/* PLB0 Arbiter Control Register */
+#define DCRN_PLB4_P0ESRL	0x012	/* PLB0 Error Status Register Low */
+#define DCRN_PLB4_P0ESRH	0x013	/* PLB0 Error Status Register High */
+#define DCRN_PLB4_P0EARL	0x014	/* PLB0 Error Address Register Low */
+#define DCRN_PLB4_P0EARH	0x015	/* PLB0 Error Address Register High */
+#define DCRN_PLB4_P0ESRLS	0x016	/* PLB0 Error Status Register Low Set*/
+#define DCRN_PLB4_P0ESRHS	0x017	/* PLB0 Error Status Register High */
+#define DCRN_PLB4_PCBC		0x018	/* PLB Crossbar Control Register */
+#define DCRN_PLB4_P1ACR		0x019	/* PLB1 Arbiter Control Register */
+#define DCRN_PLB4_P1ESRL	0x01A	/* PLB1 Error Status Register Low */
+#define DCRN_PLB4_P1ESRH	0x01B	/* PLB1 Error Status Register High */
+#define DCRN_PLB4_P1EARL	0x01C	/* PLB1 Error Address Register Low */
+#define DCRN_PLB4_P1EARH	0x01D	/* PLB1 Error Address Register High */
+#define DCRN_PLB4_P1ESRLS	0x01E	/* PLB1 Error Status Register Low Set*/
+#define DCRN_PLB4_P1ESRHS	0x01F	/*PLB1 Error Status Register High Set*/
+
+/* PLB4/OPB bridge 0, 1, 2, 3 */
+#define DCRN_PLB4OPB0_BASE	0x020
+#define DCRN_PLB4OPB1_BASE	0x030
+#define DCRN_PLB4OPB2_BASE	0x040
+#define DCRN_PLB4OPB3_BASE	0x050
+
+#define PLB4OPB_GESR0		0x0	/* Error status 0: Master Dev 0-3 */
+#define PLB4OPB_GEAR		0x2	/* Error Address Register */
+#define PLB4OPB_GEARU		0x3	/* Error Upper Address Register */
+#define PLB4OPB_GESR1		0x4	/* Error Status 1: Master Dev 4-7 */
+#define PLB4OPB_GESR2		0xC	/* Error Status 2: Master Dev 8-11 */
+
+/* PLB4-to-AHB Bridge */
+#define DCRN_PLB4AHB_BASE	0x400
+#define DCRN_PLB4AHB_SEUAR	(DCRN_PLB4AHB_BASE + 1)
+#define DCRN_PLB4AHB_SELAR	(DCRN_PLB4AHB_BASE + 2)
+#define DCRN_PLB4AHB_ESR	(DCRN_PLB4AHB_BASE + 3)
+#define DCRN_AHBPLB4_ESR	(DCRN_PLB4AHB_BASE + 8)
+#define DCRN_AHBPLB4_EAR	(DCRN_PLB4AHB_BASE + 9)
+
+/* PLB6 Controller */
+#define DCRN_PLB6_BASE		0x11111300
+#define DCRN_PLB6_CR0		(DCRN_PLB6_BASE)
+#define DCRN_PLB6_ERR		(DCRN_PLB6_BASE + 0x0B)
+#define DCRN_PLB6_HD		(DCRN_PLB6_BASE + 0x0E)
+#define DCRN_PLB6_SHD		(DCRN_PLB6_BASE + 0x10)
+
+/* PLB4-to-PLB6 Bridge */
+#define DCRN_PLB4PLB6_BASE	0x11111320
+#define DCRN_PLB4PLB6_ESR	(DCRN_PLB4PLB6_BASE + 1)
+#define DCRN_PLB4PLB6_EARH	(DCRN_PLB4PLB6_BASE + 3)
+#define DCRN_PLB4PLB6_EARL	(DCRN_PLB4PLB6_BASE + 4)
+
+/* PLB6-to-PLB4 Bridge */
+#define DCRN_PLB6PLB4_BASE	0x11111350
+#define DCRN_PLB6PLB4_ESR	(DCRN_PLB6PLB4_BASE + 1)
+#define DCRN_PLB6PLB4_EARH	(DCRN_PLB6PLB4_BASE + 3)
+#define DCRN_PLB6PLB4_EARL	(DCRN_PLB6PLB4_BASE + 4)
+
+/* PLB6-to-MCIF Bridge */
+#define DCRN_PLB6MCIF_BASE	0x11111380
+#define DCRN_PLB6MCIF_BESR0	(DCRN_PLB6MCIF_BASE + 0)
+#define DCRN_PLB6MCIF_BESR1	(DCRN_PLB6MCIF_BASE + 1)
+#define DCRN_PLB6MCIF_BEARL	(DCRN_PLB6MCIF_BASE + 2)
+#define DCRN_PLB6MCIF_BEARH	(DCRN_PLB6MCIF_BASE + 3)
+
+/* Configuration Logic Registers */
+#define DCRN_CONF_BASE		0x11111400
+#define DCRN_CONF_FIR_RWC	(DCRN_CONF_BASE + 0x3A)
+#define DCRN_CONF_EIR_RS	(DCRN_CONF_BASE + 0x3E)
+#define DCRN_CONF_RPERR0	(DCRN_CONF_BASE + 0x4D)
+#define DCRN_CONF_RPERR1	(DCRN_CONF_BASE + 0x4E)
+
+#define DCRN_L2CDCRAI		0x11111100
+#define DCRN_L2CDCRDI		0x11111104
+/* L2 indirect addresses */
+#define L2MCK		0x120
+#define L2MCKEN		0x130
+#define L2INT		0x150
+#define L2INTEN		0x160
+#define L2LOG0		0x180
+#define L2LOG1		0x184
+#define L2LOG2		0x188
+#define L2LOG3		0x18C
+#define L2LOG4		0x190
+#define L2LOG5		0x194
+#define L2PLBSTAT0	0x300
+#define L2PLBSTAT1	0x304
+#define L2PLBMCKEN0	0x330
+#define L2PLBMCKEN1	0x334
+#define L2PLBINTEN0	0x360
+#define L2PLBINTEN1	0x364
+#define L2ARRSTAT0	0x500
+#define L2ARRSTAT1	0x504
+#define L2ARRSTAT2	0x508
+#define L2ARRMCKEN0	0x530
+#define L2ARRMCKEN1	0x534
+#define L2ARRMCKEN2	0x538
+#define L2ARRINTEN0	0x560
+#define L2ARRINTEN1	0x564
+#define L2ARRINTEN2	0x568
+#define L2CPUSTAT	0x700
+#define L2CPUMCKEN	0x730
+#define L2CPUINTEN	0x760
+#define L2RACSTAT0	0x900
+#define L2RACMCKEN0	0x930
+#define L2RACINTEN0	0x960
+#define L2WACSTAT0	0xD00
+#define L2WACSTAT1	0xD04
+#define L2WACSTAT2	0xD08
+#define L2WACMCKEN0	0xD30
+#define L2WACMCKEN1	0xD34
+#define L2WACMCKEN2	0xD38
+#define L2WACINTEN0	0xD60
+#define L2WACINTEN1	0xD64
+#define L2WACINTEN2	0xD68
+#define L2WDFSTAT	0xF00
+#define L2WDFMCKEN	0xF30
+#define L2WDFINTEN	0xF60
+
+/* DDR3/4 Memory Controller */
+#define DCRN_DDR34_BASE			0x11120000
+#define DCRN_DDR34_MCSTAT		0x10
+#define DCRN_DDR34_MCOPT1		0x20
+#define DCRN_DDR34_MCOPT2		0x21
+#define DCRN_DDR34_PHYSTAT		0x32
+#define DCRN_DDR34_CFGR0		0x40
+#define DCRN_DDR34_CFGR1		0x41
+#define DCRN_DDR34_CFGR2		0x42
+#define DCRN_DDR34_CFGR3		0x43
+#define DCRN_DDR34_SCRUB_CNTL		0xAA
+#define DCRN_DDR34_SCRUB_INT		0xAB
+#define DCRN_DDR34_SCRUB_START_ADDR	0xB0
+#define DCRN_DDR34_SCRUB_END_ADDR	0xD0
+#define DCRN_DDR34_ECCERR_ADDR_PORT0	0xE0
+#define DCRN_DDR34_ECCERR_ADDR_PORT1	0xE1
+#define DCRN_DDR34_ECCERR_ADDR_PORT2	0xE2
+#define DCRN_DDR34_ECCERR_ADDR_PORT3	0xE3
+#define DCRN_DDR34_ECCERR_COUNT_PORT0	0xE4
+#define DCRN_DDR34_ECCERR_COUNT_PORT1	0xE5
+#define DCRN_DDR34_ECCERR_COUNT_PORT2	0xE6
+#define DCRN_DDR34_ECCERR_COUNT_PORT3	0xE7
+#define DCRN_DDR34_ECCERR_PORT0		0xF0
+#define DCRN_DDR34_ECCERR_PORT1		0xF2
+#define DCRN_DDR34_ECCERR_PORT2		0xF4
+#define DCRN_DDR34_ECCERR_PORT3		0xF6
+#define DCRN_DDR34_ECC_CHECK_PORT0	0xF8
+#define DCRN_DDR34_ECC_CHECK_PORT1	0xF9
+#define DCRN_DDR34_ECC_CHECK_PORT2	0xF9
+#define DCRN_DDR34_ECC_CHECK_PORT3	0xFB
+
+#define DDR34_SCRUB_CNTL_STOP		0x00000000
+#define DDR34_SCRUB_CNTL_SCRUB		0x80000000
+#define DDR34_SCRUB_CNTL_UE_STOP	0x20000000
+#define DDR34_SCRUB_CNTL_CE_STOP	0x10000000
+#define DDR34_SCRUB_CNTL_RANK_EN	0x00008000
+
+/* PLB-Attached DDR3/4 Core Wrapper */
+#define DCRN_CW_BASE			0x11111800
+#define DCRN_CW_MCER0			0x00
+#define DCRN_CW_MCER1			0x01
+#define DCRN_CW_MCER_AND0		0x02
+#define DCRN_CW_MCER_AND1		0x03
+#define DCRN_CW_MCER_OR0		0x04
+#define DCRN_CW_MCER_OR1		0x05
+#define DCRN_CW_MCER_MASK0		0x06
+#define DCRN_CW_MCER_MASK1		0x07
+#define DCRN_CW_MCER_MASK_AND0		0x08
+#define DCRN_CW_MCER_MASK_AND1		0x09
+#define DCRN_CW_MCER_MASK_OR0		0x0A
+#define DCRN_CW_MCER_MASK_OR1		0x0B
+#define DCRN_CW_MCER_ACTION0		0x0C
+#define DCRN_CW_MCER_ACTION1		0x0D
+#define DCRN_CW_MCER_WOF0		0x0E
+#define DCRN_CW_MCER_WOF1		0x0F
+#define DCRN_CW_LFIR			0x10
+#define DCRN_CW_LFIR_AND		0x11
+#define DCRN_CW_LFIR_OR			0x12
+#define DCRN_CW_LFIR_MASK		0x13
+#define DCRN_CW_LFIR_MASK_AND		0x14
+#define DCRN_CW_LFIR_MASK_OR		0x15
+
+#define CW_MCER0_MEM_CE			0x00020000
+/* CMU addresses */
+#define CMUN_CRCS		0x00 /* Chip Reset Control/Status */
+#define CMUN_CONFFIR0		0x20 /* Config Reg Parity FIR 0 */
+#define CMUN_CONFFIR1		0x21 /* Config Reg Parity FIR 1 */
+#define CMUN_CONFFIR2		0x22 /* Config Reg Parity FIR 2 */
+#define CMUN_CONFFIR3		0x23 /* Config Reg Parity FIR 3 */
+#define CMUN_URCR3_RS		0x24 /* Unit Reset Control Reg 3 Set */
+#define CMUN_URCR3_C		0x25 /* Unit Reset Control Reg 3 Clear */
+#define CMUN_URCR3_P		0x26 /* Unit Reset Control Reg 3 Pulse */
+#define CMUN_PW0		0x2C /* Pulse Width Register */
+#define CMUN_URCR0_P		0x2D /* Unit Reset Control Reg 0 Pulse */
+#define CMUN_URCR1_P		0x2E /* Unit Reset Control Reg 1 Pulse */
+#define CMUN_URCR2_P		0x2F /* Unit Reset Control Reg 2 Pulse */
+#define CMUN_CLS_RW		0x30 /* Code Load Status (Read/Write) */
+#define CMUN_CLS_S		0x31 /* Code Load Status (Set) */
+#define CMUN_CLS_C		0x32 /* Code Load Status (Clear */
+#define CMUN_URCR2_RS		0x33 /* Unit Reset Control Reg 2 Set */
+#define CMUN_URCR2_C		0x34 /* Unit Reset Control Reg 2 Clear */
+#define CMUN_CLKEN0		0x35 /* Clock Enable 0 */
+#define CMUN_CLKEN1		0x36 /* Clock Enable 1 */
+#define CMUN_PCD0		0x37 /* PSI clock divider 0 */
+#define CMUN_PCD1		0x38 /* PSI clock divider 1 */
+#define CMUN_TMR0		0x39 /* Reset Timer */
+#define CMUN_TVS0		0x3A /* TV Sense Reg 0 */
+#define CMUN_TVS1		0x3B /* TV Sense Reg 1 */
+#define CMUN_MCCR		0x3C /* DRAM Configuration Reg */
+#define CMUN_FIR0		0x3D /* Fault Isolation Reg 0 */
+#define CMUN_FMR0		0x3E /* FIR Mask Reg 0 */
+#define CMUN_ETDRB		0x3F /* ETDR Backdoor */
+
+/* CRCS bit fields */
+#define CRCS_STAT_MASK		0xF0000000
+#define CRCS_STAT_POR		0x10000000
+#define CRCS_STAT_PHR		0x20000000
+#define CRCS_STAT_PCIE		0x30000000
+#define CRCS_STAT_CRCS_SYS	0x40000000
+#define CRCS_STAT_DBCR_SYS	0x50000000
+#define CRCS_STAT_HOST_SYS	0x60000000
+#define CRCS_STAT_CHIP_RST_B	0x70000000
+#define CRCS_STAT_CRCS_CHIP	0x80000000
+#define CRCS_STAT_DBCR_CHIP	0x90000000
+#define CRCS_STAT_HOST_CHIP	0xA0000000
+#define CRCS_STAT_PSI_CHIP	0xB0000000
+#define CRCS_STAT_CRCS_CORE	0xC0000000
+#define CRCS_STAT_DBCR_CORE	0xD0000000
+#define CRCS_STAT_HOST_CORE	0xE0000000
+#define CRCS_STAT_PCIE_HOT	0xF0000000
+#define CRCS_STAT_SELF_CORE	0x40000000
+#define CRCS_STAT_SELF_CHIP	0x50000000
+#define CRCS_WATCHE		0x08000000
+#define CRCS_CORE		0x04000000 /* Reset PPC440 core */
+#define CRCS_CHIP		0x02000000 /* Chip Reset */
+#define CRCS_SYS		0x01000000 /* System Reset */
+#define CRCS_WRCR		0x00800000 /* Watchdog reset on core reset */
+#define CRCS_EXTCR		0x00080000 /* CHIP_RST_B triggers chip reset */
+#define CRCS_PLOCK		0x00000002 /* PLL Locked */
+
+#define mtcmu(reg, data)		\
+do {					\
+	mtdcr(DCRN_CMU_ADDR, reg);	\
+	mtdcr(DCRN_CMU_DATA, data);	\
+} while (0)
+
+#define mfcmu(reg)\
+	({u32 data;			\
+	mtdcr(DCRN_CMU_ADDR, reg);	\
+	data = mfdcr(DCRN_CMU_DATA);	\
+	data; })
+
+#define mtl2(reg, data)			\
+do {					\
+	mtdcr(DCRN_L2CDCRAI, reg);	\
+	mtdcr(DCRN_L2CDCRDI, data);	\
+} while (0)
+
+#define mfl2(reg)			\
+	({u32 data;			\
+	mtdcr(DCRN_L2CDCRAI, reg);	\
+	data = mfdcr(DCRN_L2CDCRDI);	\
+	data; })
+
+#endif /* __KERNEL__ */
+#endif /* _ASM_POWERPC_FSP2_DCR_H_ */
diff --git a/arch/powerpc/platforms/512x/mpc512x_shared.c b/arch/powerpc/platforms/512x/mpc512x_shared.c
index f99e79ee060e..48abb4cb304c 100644
--- a/arch/powerpc/platforms/512x/mpc512x_shared.c
+++ b/arch/powerpc/platforms/512x/mpc512x_shared.c
@@ -387,8 +387,8 @@ static unsigned int __init get_fifo_size(struct device_node *np,
 	if (fp)
 		return *fp;
 
-	pr_warning("no %s property in %pOF node, defaulting to %d\n",
-		   prop_name, np, DEFAULT_FIFO_SIZE);
+	pr_warn("no %s property in %pOF node, defaulting to %d\n",
+		prop_name, np, DEFAULT_FIFO_SIZE);
 
 	return DEFAULT_FIFO_SIZE;
 }
diff --git a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c
index 9e974b1e1697..17cf249b18ee 100644
--- a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c
+++ b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c
@@ -90,7 +90,7 @@ struct mpc52xx_gpt_priv {
 	struct list_head list;		/* List of all GPT devices */
 	struct device *dev;
 	struct mpc52xx_gpt __iomem *regs;
-	spinlock_t lock;
+	raw_spinlock_t lock;
 	struct irq_domain *irqhost;
 	u32 ipb_freq;
 	u8 wdt_mode;
@@ -141,9 +141,9 @@ static void mpc52xx_gpt_irq_unmask(struct irq_data *d)
 	struct mpc52xx_gpt_priv *gpt = irq_data_get_irq_chip_data(d);
 	unsigned long flags;
 
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	setbits32(&gpt->regs->mode, MPC52xx_GPT_MODE_IRQ_EN);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 }
 
 static void mpc52xx_gpt_irq_mask(struct irq_data *d)
@@ -151,9 +151,9 @@ static void mpc52xx_gpt_irq_mask(struct irq_data *d)
 	struct mpc52xx_gpt_priv *gpt = irq_data_get_irq_chip_data(d);
 	unsigned long flags;
 
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	clrbits32(&gpt->regs->mode, MPC52xx_GPT_MODE_IRQ_EN);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 }
 
 static void mpc52xx_gpt_irq_ack(struct irq_data *d)
@@ -171,14 +171,14 @@ static int mpc52xx_gpt_irq_set_type(struct irq_data *d, unsigned int flow_type)
 
 	dev_dbg(gpt->dev, "%s: virq=%i type=%x\n", __func__, d->irq, flow_type);
 
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	reg = in_be32(&gpt->regs->mode) & ~MPC52xx_GPT_MODE_ICT_MASK;
 	if (flow_type & IRQF_TRIGGER_RISING)
 		reg |= MPC52xx_GPT_MODE_ICT_RISING;
 	if (flow_type & IRQF_TRIGGER_FALLING)
 		reg |= MPC52xx_GPT_MODE_ICT_FALLING;
 	out_be32(&gpt->regs->mode, reg);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 
 	return 0;
 }
@@ -264,11 +264,11 @@ mpc52xx_gpt_irq_setup(struct mpc52xx_gpt_priv *gpt, struct device_node *node)
 	/* If the GPT is currently disabled, then change it to be in Input
 	 * Capture mode.  If the mode is non-zero, then the pin could be
 	 * already in use for something. */
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	mode = in_be32(&gpt->regs->mode);
 	if ((mode & MPC52xx_GPT_MODE_MS_MASK) == 0)
 		out_be32(&gpt->regs->mode, mode | MPC52xx_GPT_MODE_MS_IC);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 
 	dev_dbg(gpt->dev, "%s() complete. virq=%i\n", __func__, cascade_virq);
 }
@@ -295,9 +295,9 @@ mpc52xx_gpt_gpio_set(struct gpio_chip *gc, unsigned int gpio, int v)
 	dev_dbg(gpt->dev, "%s: gpio:%d v:%d\n", __func__, gpio, v);
 	r = v ? MPC52xx_GPT_MODE_GPIO_OUT_HIGH : MPC52xx_GPT_MODE_GPIO_OUT_LOW;
 
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	clrsetbits_be32(&gpt->regs->mode, MPC52xx_GPT_MODE_GPIO_MASK, r);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 }
 
 static int mpc52xx_gpt_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
@@ -307,9 +307,9 @@ static int mpc52xx_gpt_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio)
 
 	dev_dbg(gpt->dev, "%s: gpio:%d\n", __func__, gpio);
 
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	clrbits32(&gpt->regs->mode, MPC52xx_GPT_MODE_GPIO_MASK);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 
 	return 0;
 }
@@ -436,16 +436,16 @@ static int mpc52xx_gpt_do_start(struct mpc52xx_gpt_priv *gpt, u64 period,
 	}
 
 	/* Set and enable the timer, reject an attempt to use a wdt as gpt */
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	if (as_wdt)
 		gpt->wdt_mode |= MPC52xx_GPT_IS_WDT;
 	else if ((gpt->wdt_mode & MPC52xx_GPT_IS_WDT) != 0) {
-		spin_unlock_irqrestore(&gpt->lock, flags);
+		raw_spin_unlock_irqrestore(&gpt->lock, flags);
 		return -EBUSY;
 	}
 	out_be32(&gpt->regs->count, prescale << 16 | clocks);
 	clrsetbits_be32(&gpt->regs->mode, clear, set);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 
 	return 0;
 }
@@ -476,14 +476,14 @@ int mpc52xx_gpt_stop_timer(struct mpc52xx_gpt_priv *gpt)
 	unsigned long flags;
 
 	/* reject the operation if the timer is used as watchdog (gpt 0 only) */
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	if ((gpt->wdt_mode & MPC52xx_GPT_IS_WDT) != 0) {
-		spin_unlock_irqrestore(&gpt->lock, flags);
+		raw_spin_unlock_irqrestore(&gpt->lock, flags);
 		return -EBUSY;
 	}
 
 	clrbits32(&gpt->regs->mode, MPC52xx_GPT_MODE_COUNTER_ENABLE);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 	return 0;
 }
 EXPORT_SYMBOL(mpc52xx_gpt_stop_timer);
@@ -500,9 +500,9 @@ u64 mpc52xx_gpt_timer_period(struct mpc52xx_gpt_priv *gpt)
 	u64 prescale;
 	unsigned long flags;
 
-	spin_lock_irqsave(&gpt->lock, flags);
+	raw_spin_lock_irqsave(&gpt->lock, flags);
 	period = in_be32(&gpt->regs->count);
-	spin_unlock_irqrestore(&gpt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt->lock, flags);
 
 	prescale = period >> 16;
 	period &= 0xffff;
@@ -532,9 +532,9 @@ static inline void mpc52xx_gpt_wdt_ping(struct mpc52xx_gpt_priv *gpt_wdt)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&gpt_wdt->lock, flags);
+	raw_spin_lock_irqsave(&gpt_wdt->lock, flags);
 	out_8((u8 *) &gpt_wdt->regs->mode, MPC52xx_GPT_MODE_WDT_PING);
-	spin_unlock_irqrestore(&gpt_wdt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt_wdt->lock, flags);
 }
 
 /* wdt misc device api */
@@ -638,11 +638,11 @@ static int mpc52xx_wdt_release(struct inode *inode, struct file *file)
 	struct mpc52xx_gpt_priv *gpt_wdt = file->private_data;
 	unsigned long flags;
 
-	spin_lock_irqsave(&gpt_wdt->lock, flags);
+	raw_spin_lock_irqsave(&gpt_wdt->lock, flags);
 	clrbits32(&gpt_wdt->regs->mode,
 		  MPC52xx_GPT_MODE_COUNTER_ENABLE | MPC52xx_GPT_MODE_WDT_EN);
 	gpt_wdt->wdt_mode &= ~MPC52xx_GPT_IS_WDT;
-	spin_unlock_irqrestore(&gpt_wdt->lock, flags);
+	raw_spin_unlock_irqrestore(&gpt_wdt->lock, flags);
 #endif
 	clear_bit(0, &wdt_is_active);
 	return 0;
@@ -723,7 +723,7 @@ static int mpc52xx_gpt_probe(struct platform_device *ofdev)
 	if (!gpt)
 		return -ENOMEM;
 
-	spin_lock_init(&gpt->lock);
+	raw_spin_lock_init(&gpt->lock);
 	gpt->dev = &ofdev->dev;
 	gpt->ipb_freq = mpc5xxx_get_bus_frequency(ofdev->dev.of_node);
 	gpt->regs = of_iomap(ofdev->dev.of_node, 0);
diff --git a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c
index 96bb55ca61d3..d2ef39f0edc8 100644
--- a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c
+++ b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c
@@ -84,7 +84,7 @@ static ssize_t show_status(struct device *d,
 
 	return sprintf(buf, "%02x\n", ret);
 }
-static DEVICE_ATTR(status, S_IRUGO, show_status, NULL);
+static DEVICE_ATTR(status, 0444, show_status, NULL);
 
 static void mcu_power_off(void)
 {
diff --git a/arch/powerpc/platforms/83xx/mpc832x_mds.c b/arch/powerpc/platforms/83xx/mpc832x_mds.c
index bb7b25acf26f..74c154e67c8b 100644
--- a/arch/powerpc/platforms/83xx/mpc832x_mds.c
+++ b/arch/powerpc/platforms/83xx/mpc832x_mds.c
@@ -75,7 +75,7 @@ static void __init mpc832x_sys_setup_arch(void)
 		par_io_init(np);
 		of_node_put(np);
 
-		for (np = NULL; (np = of_find_node_by_name(np, "ucc")) != NULL;)
+		for_each_node_by_name(np, "ucc")
 			par_io_of_config(np);
 	}
 
diff --git a/arch/powerpc/platforms/83xx/mpc832x_rdb.c b/arch/powerpc/platforms/83xx/mpc832x_rdb.c
index a4539c5accb0..438986593873 100644
--- a/arch/powerpc/platforms/83xx/mpc832x_rdb.c
+++ b/arch/powerpc/platforms/83xx/mpc832x_rdb.c
@@ -204,7 +204,7 @@ static void __init mpc832x_rdb_setup_arch(void)
 		par_io_init(np);
 		of_node_put(np);
 
-		for (np = NULL; (np = of_find_node_by_name(np, "ucc")) != NULL;)
+		for_each_node_by_name(np, "ucc")
 			par_io_of_config(np);
 	}
 #endif				/* CONFIG_QUICC_ENGINE */
diff --git a/arch/powerpc/platforms/83xx/mpc836x_mds.c b/arch/powerpc/platforms/83xx/mpc836x_mds.c
index 4fc3051c2b2e..fd44dd03e1f3 100644
--- a/arch/powerpc/platforms/83xx/mpc836x_mds.c
+++ b/arch/powerpc/platforms/83xx/mpc836x_mds.c
@@ -83,7 +83,7 @@ static void __init mpc836x_mds_setup_arch(void)
 		par_io_init(np);
 		of_node_put(np);
 
-		for (np = NULL; (np = of_find_node_by_name(np, "ucc")) != NULL;)
+		for_each_node_by_name(np, "ucc")
 			par_io_of_config(np);
 #ifdef CONFIG_QE_USB
 		/* Must fixup Par IO before QE GPIO chips are registered. */
diff --git a/arch/powerpc/platforms/85xx/socrates_fpga_pic.c b/arch/powerpc/platforms/85xx/socrates_fpga_pic.c
index 82f8490b5aa7..38d4ba9f37b5 100644
--- a/arch/powerpc/platforms/85xx/socrates_fpga_pic.c
+++ b/arch/powerpc/platforms/85xx/socrates_fpga_pic.c
@@ -252,8 +252,7 @@ static int socrates_fpga_pic_host_xlate(struct irq_domain *h,
 		/* type is configurable */
 		if (intspec[1] != IRQ_TYPE_LEVEL_LOW &&
 		    intspec[1] != IRQ_TYPE_LEVEL_HIGH) {
-			pr_warning("FPGA PIC: invalid irq type, "
-				   "setting default active low\n");
+			pr_warn("FPGA PIC: invalid irq type, setting default active low\n");
 			*out_flags = IRQ_TYPE_LEVEL_LOW;
 		} else {
 			*out_flags = intspec[1];
@@ -267,7 +266,7 @@ static int socrates_fpga_pic_host_xlate(struct irq_domain *h,
 	if (intspec[2] <= 2)
 		fpga_irq->irq_line = intspec[2];
 	else
-		pr_warning("FPGA PIC: invalid irq routing\n");
+		pr_warn("FPGA PIC: invalid irq routing\n");
 
 	return 0;
 }
@@ -293,7 +292,7 @@ void socrates_fpga_pic_init(struct device_node *pic)
 	for (i = 0; i < 3; i++) {
 		socrates_fpga_irqs[i] = irq_of_parse_and_map(pic, i);
 		if (!socrates_fpga_irqs[i]) {
-			pr_warning("FPGA PIC: can't get irq%d.\n", i);
+			pr_warn("FPGA PIC: can't get irq%d\n", i);
 			continue;
 		}
 		irq_set_chained_handler(socrates_fpga_irqs[i],
diff --git a/arch/powerpc/platforms/86xx/mpc86xx_hpcn.c b/arch/powerpc/platforms/86xx/mpc86xx_hpcn.c
index a0e989ed4b6f..17c6cd3d02e6 100644
--- a/arch/powerpc/platforms/86xx/mpc86xx_hpcn.c
+++ b/arch/powerpc/platforms/86xx/mpc86xx_hpcn.c
@@ -101,7 +101,7 @@ static int __init mpc86xx_hpcn_probe(void)
 
 	/* Be nice and don't give silent boot death.  Delete this in 2.6.27 */
 	if (of_machine_is_compatible("mpc86xx")) {
-		pr_warning("WARNING: your dts/dtb is old. You must update before the next kernel release\n");
+		pr_warn("WARNING: your dts/dtb is old. You must update before the next kernel release.\n");
 		return 1;
 	}
 
diff --git a/arch/powerpc/platforms/8xx/Kconfig b/arch/powerpc/platforms/8xx/Kconfig
index e2089d3de00c..d408162d5af4 100644
--- a/arch/powerpc/platforms/8xx/Kconfig
+++ b/arch/powerpc/platforms/8xx/Kconfig
@@ -116,18 +116,6 @@ config 8xx_GPIO
 
 	  If in doubt, say Y here.
 
-config 8xx_CPU6
-	bool "CPU6 Silicon Errata (860 Pre Rev. C)"
-	help
-	  MPC860 CPUs, prior to Rev C have some bugs in the silicon, which
-	  require workarounds for Linux (and most other OSes to work).  If you
-	  get a BUG() very early in boot, this might fix the problem.  For
-	  more details read the document entitled "MPC860 Family Device Errata
-	  Reference" on Freescale's website.  This option also incurs a
-	  performance hit.
-
-	  If in doubt, say N here.
-
 config 8xx_CPU15
 	bool "CPU15 Silicon Errata"
 	depends on !HUGETLB_PAGE
diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig
index 5a96a2763e4a..14ef17e10ec9 100644
--- a/arch/powerpc/platforms/Kconfig
+++ b/arch/powerpc/platforms/Kconfig
@@ -293,17 +293,6 @@ config CPM2
 	  you wish to build a kernel for a machine with a CPM2 coprocessor
 	  on it (826x, 827x, 8560).
 
-config AXON_RAM
-	tristate "Axon DDR2 memory device driver"
-	depends on PPC_IBM_CELL_BLADE && BLOCK
-	select DAX
-	default m
-	help
-	  It registers one block device per Axon's DDR2 memory bank found
-	  on a system. Block devices are called axonram?, their major and
-	  minor numbers are available in /proc/devices, /proc/partitions or
-	  in /sys/block/axonram?/dev.
-
 config FSL_ULI1575
 	bool
 	default n
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index ae07470fde3c..a429d859f15d 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -33,7 +33,6 @@ config PPC_85xx
 config PPC_8xx
 	bool "Freescale 8xx"
 	select FSL_SOC
-	select PPC_LIB_RHEAP
 	select SYS_SUPPORTS_HUGETLBFS
 
 config 40x
@@ -168,13 +167,6 @@ config PPC_FPU
 	bool
 	default y if PPC64
 
-config PPC_8xx_PERF_EVENT
-	bool "PPC 8xx perf events"
-	depends on PPC_8xx && PERF_EVENTS
-	help
-	  This is Performance Events support for PPC 8xx. The 8xx doesn't
-	  have a PMU but some events are emulated using 8xx features.
-
 config FSL_EMB_PERFMON
 	bool "Freescale Embedded Perfmon"
 	depends on E500 || PPC_83xx
diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c
index 6fc85e29dc08..5d4bf9aed51a 100644
--- a/arch/powerpc/platforms/cell/interrupt.c
+++ b/arch/powerpc/platforms/cell/interrupt.c
@@ -315,8 +315,7 @@ static int __init setup_iic(void)
 	struct cbe_iic_regs __iomem *node_iic;
 	const u32 *np;
 
-	for (dn = NULL;
-	     (dn = of_find_node_by_name(dn,"interrupt-controller")) != NULL;) {
+	for_each_node_by_name(dn, "interrupt-controller") {
 		if (!of_device_is_compatible(dn,
 				     "IBM,CBEA-Internal-Interrupt-Controller"))
 			continue;
diff --git a/arch/powerpc/platforms/cell/iommu.c b/arch/powerpc/platforms/cell/iommu.c
index 4b91ad08eefd..12352a58072a 100644
--- a/arch/powerpc/platforms/cell/iommu.c
+++ b/arch/powerpc/platforms/cell/iommu.c
@@ -541,7 +541,7 @@ static struct cbe_iommu *cell_iommu_for_node(int nid)
 	return NULL;
 }
 
-static unsigned long cell_dma_direct_offset;
+static unsigned long cell_dma_nommu_offset;
 
 static unsigned long dma_iommu_fixed_base;
 
@@ -580,7 +580,7 @@ static void *dma_fixed_alloc_coherent(struct device *dev, size_t size,
 					    device_to_mask(dev), flag,
 					    dev_to_node(dev));
 	else
-		return dma_direct_ops.alloc(dev, size, dma_handle, flag,
+		return dma_nommu_ops.alloc(dev, size, dma_handle, flag,
 					    attrs);
 }
 
@@ -592,7 +592,7 @@ static void dma_fixed_free_coherent(struct device *dev, size_t size,
 		iommu_free_coherent(cell_get_iommu_table(dev), size, vaddr,
 				    dma_handle);
 	else
-		dma_direct_ops.free(dev, size, vaddr, dma_handle, attrs);
+		dma_nommu_ops.free(dev, size, vaddr, dma_handle, attrs);
 }
 
 static dma_addr_t dma_fixed_map_page(struct device *dev, struct page *page,
@@ -601,7 +601,7 @@ static dma_addr_t dma_fixed_map_page(struct device *dev, struct page *page,
 				     unsigned long attrs)
 {
 	if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING))
-		return dma_direct_ops.map_page(dev, page, offset, size,
+		return dma_nommu_ops.map_page(dev, page, offset, size,
 					       direction, attrs);
 	else
 		return iommu_map_page(dev, cell_get_iommu_table(dev), page,
@@ -614,7 +614,7 @@ static void dma_fixed_unmap_page(struct device *dev, dma_addr_t dma_addr,
 				 unsigned long attrs)
 {
 	if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING))
-		dma_direct_ops.unmap_page(dev, dma_addr, size, direction,
+		dma_nommu_ops.unmap_page(dev, dma_addr, size, direction,
 					  attrs);
 	else
 		iommu_unmap_page(cell_get_iommu_table(dev), dma_addr, size,
@@ -626,7 +626,7 @@ static int dma_fixed_map_sg(struct device *dev, struct scatterlist *sg,
 			   unsigned long attrs)
 {
 	if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING))
-		return dma_direct_ops.map_sg(dev, sg, nents, direction, attrs);
+		return dma_nommu_ops.map_sg(dev, sg, nents, direction, attrs);
 	else
 		return ppc_iommu_map_sg(dev, cell_get_iommu_table(dev), sg,
 					nents, device_to_mask(dev),
@@ -638,7 +638,7 @@ static void dma_fixed_unmap_sg(struct device *dev, struct scatterlist *sg,
 			       unsigned long attrs)
 {
 	if (iommu_fixed_is_weak == (attrs & DMA_ATTR_WEAK_ORDERING))
-		dma_direct_ops.unmap_sg(dev, sg, nents, direction, attrs);
+		dma_nommu_ops.unmap_sg(dev, sg, nents, direction, attrs);
 	else
 		ppc_iommu_unmap_sg(cell_get_iommu_table(dev), sg, nents,
 				   direction, attrs);
@@ -661,8 +661,8 @@ static void cell_dma_dev_setup(struct device *dev)
 {
 	if (get_pci_dma_ops() == &dma_iommu_ops)
 		set_iommu_table_base(dev, cell_get_iommu_table(dev));
-	else if (get_pci_dma_ops() == &dma_direct_ops)
-		set_dma_offset(dev, cell_dma_direct_offset);
+	else if (get_pci_dma_ops() == &dma_nommu_ops)
+		set_dma_offset(dev, cell_dma_nommu_offset);
 	else
 		BUG();
 }
@@ -810,14 +810,14 @@ static int __init cell_iommu_init_disabled(void)
 	unsigned long base = 0, size;
 
 	/* When no iommu is present, we use direct DMA ops */
-	set_pci_dma_ops(&dma_direct_ops);
+	set_pci_dma_ops(&dma_nommu_ops);
 
 	/* First make sure all IOC translation is turned off */
 	cell_disable_iommus();
 
 	/* If we have no Axon, we set up the spider DMA magic offset */
 	if (of_find_node_by_name(NULL, "axon") == NULL)
-		cell_dma_direct_offset = SPIDER_DMA_OFFSET;
+		cell_dma_nommu_offset = SPIDER_DMA_OFFSET;
 
 	/* Now we need to check to see where the memory is mapped
 	 * in PCI space. We assume that all busses use the same dma
@@ -851,13 +851,13 @@ static int __init cell_iommu_init_disabled(void)
 		return -ENODEV;
 	}
 
-	cell_dma_direct_offset += base;
+	cell_dma_nommu_offset += base;
 
-	if (cell_dma_direct_offset != 0)
+	if (cell_dma_nommu_offset != 0)
 		cell_pci_controller_ops.dma_dev_setup = cell_pci_dma_dev_setup;
 
 	printk("iommu: disabled, direct DMA offset is 0x%lx\n",
-	       cell_dma_direct_offset);
+	       cell_dma_nommu_offset);
 
 	return 0;
 }
diff --git a/arch/powerpc/platforms/cell/setup.c b/arch/powerpc/platforms/cell/setup.c
index d3543e68efe8..7d31b8d14661 100644
--- a/arch/powerpc/platforms/cell/setup.c
+++ b/arch/powerpc/platforms/cell/setup.c
@@ -192,8 +192,7 @@ static void __init mpic_init_IRQ(void)
 	struct device_node *dn;
 	struct mpic *mpic;
 
-	for (dn = NULL;
-	     (dn = of_find_node_by_name(dn, "interrupt-controller"));) {
+	for_each_node_by_name(dn, "interrupt-controller") {
 		if (!of_device_is_compatible(dn, "CBEA,platform-open-pic"))
 			continue;
 
diff --git a/arch/powerpc/platforms/cell/spider-pic.c b/arch/powerpc/platforms/cell/spider-pic.c
index aa44bfc46467..c137f0cb4151 100644
--- a/arch/powerpc/platforms/cell/spider-pic.c
+++ b/arch/powerpc/platforms/cell/spider-pic.c
@@ -343,8 +343,7 @@ void __init spider_init_IRQ(void)
 	 * device-tree is bogus anyway) so all we can do is pray or maybe test
 	 * the address and deduce the node-id
 	 */
-	for (dn = NULL;
-	     (dn = of_find_node_by_name(dn, "interrupt-controller"));) {
+	for_each_node_by_name(dn, "interrupt-controller") {
 		if (of_device_is_compatible(dn, "CBEA,platform-spider-pic")) {
 			if (of_address_to_resource(dn, 0, &r)) {
 				printk(KERN_WARNING "spider-pic: Failed\n");
diff --git a/arch/powerpc/platforms/cell/spu_manage.c b/arch/powerpc/platforms/cell/spu_manage.c
index f636ee22b203..5c409c98cca8 100644
--- a/arch/powerpc/platforms/cell/spu_manage.c
+++ b/arch/powerpc/platforms/cell/spu_manage.c
@@ -292,12 +292,12 @@ static int __init of_enumerate_spus(int (*fn)(void *data))
 	unsigned int n = 0;
 
 	ret = -ENODEV;
-	for (node = of_find_node_by_type(NULL, "spe");
-			node; node = of_find_node_by_type(node, "spe")) {
+	for_each_node_by_type(node, "spe") {
 		ret = fn(node);
 		if (ret) {
 			printk(KERN_WARNING "%s: Error initializing %s\n",
 				__func__, node->name);
+			of_node_put(node);
 			break;
 		}
 		n++;
diff --git a/arch/powerpc/platforms/cell/spufs/backing_ops.c b/arch/powerpc/platforms/cell/spufs/backing_ops.c
index 6e8a9ef8590e..857580a78bbd 100644
--- a/arch/powerpc/platforms/cell/spufs/backing_ops.c
+++ b/arch/powerpc/platforms/cell/spufs/backing_ops.c
@@ -86,10 +86,10 @@ static u32 spu_backing_mbox_stat_read(struct spu_context *ctx)
 	return ctx->csa.prob.mb_stat_R;
 }
 
-static unsigned int spu_backing_mbox_stat_poll(struct spu_context *ctx,
-					  unsigned int events)
+static __poll_t spu_backing_mbox_stat_poll(struct spu_context *ctx,
+					  __poll_t events)
 {
-	int ret;
+	__poll_t ret;
 	u32 stat;
 
 	ret = 0;
@@ -101,9 +101,9 @@ static unsigned int spu_backing_mbox_stat_poll(struct spu_context *ctx,
 	   but first mark any pending interrupts as done so
 	   we don't get woken up unnecessarily */
 
-	if (events & (POLLIN | POLLRDNORM)) {
+	if (events & (EPOLLIN | EPOLLRDNORM)) {
 		if (stat & 0xff0000)
-			ret |= POLLIN | POLLRDNORM;
+			ret |= EPOLLIN | EPOLLRDNORM;
 		else {
 			ctx->csa.priv1.int_stat_class2_RW &=
 				~CLASS2_MAILBOX_INTR;
@@ -111,9 +111,9 @@ static unsigned int spu_backing_mbox_stat_poll(struct spu_context *ctx,
 				CLASS2_ENABLE_MAILBOX_INTR;
 		}
 	}
-	if (events & (POLLOUT | POLLWRNORM)) {
+	if (events & (EPOLLOUT | EPOLLWRNORM)) {
 		if (stat & 0x00ff00)
-			ret = POLLOUT | POLLWRNORM;
+			ret = EPOLLOUT | EPOLLWRNORM;
 		else {
 			ctx->csa.priv1.int_stat_class2_RW &=
 				~CLASS2_MAILBOX_THRESHOLD_INTR;
diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c
index 5ffcdeb1eb17..469bdd0b748f 100644
--- a/arch/powerpc/platforms/cell/spufs/file.c
+++ b/arch/powerpc/platforms/cell/spufs/file.c
@@ -762,10 +762,10 @@ static ssize_t spufs_ibox_read(struct file *file, char __user *buf,
 	return count;
 }
 
-static unsigned int spufs_ibox_poll(struct file *file, poll_table *wait)
+static __poll_t spufs_ibox_poll(struct file *file, poll_table *wait)
 {
 	struct spu_context *ctx = file->private_data;
-	unsigned int mask;
+	__poll_t mask;
 
 	poll_wait(file, &ctx->ibox_wq, wait);
 
@@ -774,7 +774,7 @@ static unsigned int spufs_ibox_poll(struct file *file, poll_table *wait)
 	 * that poll should not sleep.  Will be fixed later.
 	 */
 	mutex_lock(&ctx->state_mutex);
-	mask = ctx->ops->mbox_stat_poll(ctx, POLLIN | POLLRDNORM);
+	mask = ctx->ops->mbox_stat_poll(ctx, EPOLLIN | EPOLLRDNORM);
 	spu_release(ctx);
 
 	return mask;
@@ -898,10 +898,10 @@ static ssize_t spufs_wbox_write(struct file *file, const char __user *buf,
 	return count;
 }
 
-static unsigned int spufs_wbox_poll(struct file *file, poll_table *wait)
+static __poll_t spufs_wbox_poll(struct file *file, poll_table *wait)
 {
 	struct spu_context *ctx = file->private_data;
-	unsigned int mask;
+	__poll_t mask;
 
 	poll_wait(file, &ctx->wbox_wq, wait);
 
@@ -910,7 +910,7 @@ static unsigned int spufs_wbox_poll(struct file *file, poll_table *wait)
 	 * that poll should not sleep.  Will be fixed later.
 	 */
 	mutex_lock(&ctx->state_mutex);
-	mask = ctx->ops->mbox_stat_poll(ctx, POLLOUT | POLLWRNORM);
+	mask = ctx->ops->mbox_stat_poll(ctx, EPOLLOUT | EPOLLWRNORM);
 	spu_release(ctx);
 
 	return mask;
@@ -1690,11 +1690,11 @@ static ssize_t spufs_mfc_write(struct file *file, const char __user *buffer,
 	return ret;
 }
 
-static unsigned int spufs_mfc_poll(struct file *file,poll_table *wait)
+static __poll_t spufs_mfc_poll(struct file *file,poll_table *wait)
 {
 	struct spu_context *ctx = file->private_data;
 	u32 free_elements, tagstatus;
-	unsigned int mask;
+	__poll_t mask;
 
 	poll_wait(file, &ctx->mfc_wq, wait);
 
@@ -1710,9 +1710,9 @@ static unsigned int spufs_mfc_poll(struct file *file,poll_table *wait)
 
 	mask = 0;
 	if (free_elements & 0xffff)
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 	if (tagstatus & ctx->tagwait)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	pr_debug("%s: free %d tagstatus %d tagwait %d\n", __func__,
 		free_elements, tagstatus, ctx->tagwait);
@@ -2375,8 +2375,8 @@ static int switch_log_sprint(struct spu_context *ctx, char *tbuf, int n)
 
 	p = ctx->switch_log->log + ctx->switch_log->tail % SWITCH_LOG_BUFSIZE;
 
-	return snprintf(tbuf, n, "%u.%09u %d %u %u %llu\n",
-			(unsigned int) p->tstamp.tv_sec,
+	return snprintf(tbuf, n, "%llu.%09u %d %u %u %llu\n",
+			(unsigned long long) p->tstamp.tv_sec,
 			(unsigned int) p->tstamp.tv_nsec,
 			p->spu_id,
 			(unsigned int) p->type,
@@ -2455,11 +2455,11 @@ static ssize_t spufs_switch_log_read(struct file *file, char __user *buf,
 	return cnt == 0 ? error : cnt;
 }
 
-static unsigned int spufs_switch_log_poll(struct file *file, poll_table *wait)
+static __poll_t spufs_switch_log_poll(struct file *file, poll_table *wait)
 {
 	struct inode *inode = file_inode(file);
 	struct spu_context *ctx = SPUFS_I(inode)->i_ctx;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	int rc;
 
 	poll_wait(file, &ctx->switch_log->wait, wait);
@@ -2469,7 +2469,7 @@ static unsigned int spufs_switch_log_poll(struct file *file, poll_table *wait)
 		return rc;
 
 	if (spufs_switch_log_used(ctx) > 0)
-		mask |= POLLIN;
+		mask |= EPOLLIN;
 
 	spu_release(ctx);
 
@@ -2499,7 +2499,7 @@ void spu_switch_log_notify(struct spu *spu, struct spu_context *ctx,
 		struct switch_log_entry *p;
 
 		p = ctx->switch_log->log + ctx->switch_log->head;
-		ktime_get_ts(&p->tstamp);
+		ktime_get_ts64(&p->tstamp);
 		p->timebase = get_tb();
 		p->spu_id = spu ? spu->number : -1;
 		p->type = type;
diff --git a/arch/powerpc/platforms/cell/spufs/hw_ops.c b/arch/powerpc/platforms/cell/spufs/hw_ops.c
index 8655c4cbefc2..ae9d24d31eed 100644
--- a/arch/powerpc/platforms/cell/spufs/hw_ops.c
+++ b/arch/powerpc/platforms/cell/spufs/hw_ops.c
@@ -56,11 +56,10 @@ static u32 spu_hw_mbox_stat_read(struct spu_context *ctx)
 	return in_be32(&ctx->spu->problem->mb_stat_R);
 }
 
-static unsigned int spu_hw_mbox_stat_poll(struct spu_context *ctx,
-					  unsigned int events)
+static __poll_t spu_hw_mbox_stat_poll(struct spu_context *ctx, __poll_t events)
 {
 	struct spu *spu = ctx->spu;
-	int ret = 0;
+	__poll_t ret = 0;
 	u32 stat;
 
 	spin_lock_irq(&spu->register_lock);
@@ -71,17 +70,17 @@ static unsigned int spu_hw_mbox_stat_poll(struct spu_context *ctx,
 	   but first mark any pending interrupts as done so
 	   we don't get woken up unnecessarily */
 
-	if (events & (POLLIN | POLLRDNORM)) {
+	if (events & (EPOLLIN | EPOLLRDNORM)) {
 		if (stat & 0xff0000)
-			ret |= POLLIN | POLLRDNORM;
+			ret |= EPOLLIN | EPOLLRDNORM;
 		else {
 			spu_int_stat_clear(spu, 2, CLASS2_MAILBOX_INTR);
 			spu_int_mask_or(spu, 2, CLASS2_ENABLE_MAILBOX_INTR);
 		}
 	}
-	if (events & (POLLOUT | POLLWRNORM)) {
+	if (events & (EPOLLOUT | EPOLLWRNORM)) {
 		if (stat & 0x00ff00)
-			ret = POLLOUT | POLLWRNORM;
+			ret = EPOLLOUT | EPOLLWRNORM;
 		else {
 			spu_int_stat_clear(spu, 2,
 					CLASS2_MAILBOX_THRESHOLD_INTR);
diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c
index 9558d725a99b..db329d4bf1c3 100644
--- a/arch/powerpc/platforms/cell/spufs/inode.c
+++ b/arch/powerpc/platforms/cell/spufs/inode.c
@@ -455,7 +455,7 @@ spufs_create_context(struct inode *inode, struct dentry *dentry,
 		}
 	}
 
-	ret = spufs_mkdir(inode, dentry, flags, mode & S_IRWXUGO);
+	ret = spufs_mkdir(inode, dentry, flags, mode & 0777);
 	if (ret)
 		goto out_aff_unlock;
 
@@ -546,7 +546,7 @@ static int spufs_create_gang(struct inode *inode,
 	struct path path = {.mnt = mnt, .dentry = dentry};
 	int ret;
 
-	ret = spufs_mkgang(inode, dentry, mode & S_IRWXUGO);
+	ret = spufs_mkgang(inode, dentry, mode & 0777);
 	if (!ret) {
 		ret = spufs_gang_open(&path);
 		if (ret < 0) {
diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h
index 5e59f80e95db..b5fc1b3fe538 100644
--- a/arch/powerpc/platforms/cell/spufs/spufs.h
+++ b/arch/powerpc/platforms/cell/spufs/spufs.h
@@ -69,7 +69,7 @@ struct switch_log {
 	unsigned long		head;
 	unsigned long		tail;
 	struct switch_log_entry {
-		struct timespec	tstamp;
+		struct timespec64 tstamp;
 		s32		spu_id;
 		u32		type;
 		u32		val;
@@ -185,8 +185,7 @@ struct mfc_dma_command {
 struct spu_context_ops {
 	int (*mbox_read) (struct spu_context * ctx, u32 * data);
 	 u32(*mbox_stat_read) (struct spu_context * ctx);
-	unsigned int (*mbox_stat_poll)(struct spu_context *ctx,
-					unsigned int events);
+	__poll_t (*mbox_stat_poll)(struct spu_context *ctx, __poll_t events);
 	int (*ibox_read) (struct spu_context * ctx, u32 * data);
 	int (*wbox_write) (struct spu_context * ctx, u32 data);
 	 u32(*signal1_read) (struct spu_context * ctx);
diff --git a/arch/powerpc/platforms/maple/time.c b/arch/powerpc/platforms/maple/time.c
index 81799d70a1ee..cfddc87f81bf 100644
--- a/arch/powerpc/platforms/maple/time.c
+++ b/arch/powerpc/platforms/maple/time.c
@@ -134,7 +134,7 @@ int maple_set_rtc_time(struct rtc_time *tm)
 
 static struct resource rtc_iores = {
 	.name = "rtc",
-	.flags = IORESOURCE_BUSY,
+	.flags = IORESOURCE_IO | IORESOURCE_BUSY,
 };
 
 unsigned long __init maple_get_boot_time(void)
diff --git a/arch/powerpc/platforms/pasemi/dma_lib.c b/arch/powerpc/platforms/pasemi/dma_lib.c
index aafa01ba062f..2c72263ad6ab 100644
--- a/arch/powerpc/platforms/pasemi/dma_lib.c
+++ b/arch/powerpc/platforms/pasemi/dma_lib.c
@@ -589,7 +589,7 @@ int pasemi_dma_init(void)
 	pasemi_write_dma_reg(PAS_DMA_COM_RXCMD, 0);
 	while (pasemi_read_dma_reg(PAS_DMA_COM_RXSTA) & 1) {
 		if (time_after(jiffies, timeout)) {
-			pr_warning("Warning: Could not disable RX section\n");
+			pr_warn("Warning: Could not disable RX section\n");
 			break;
 		}
 	}
@@ -598,7 +598,7 @@ int pasemi_dma_init(void)
 	pasemi_write_dma_reg(PAS_DMA_COM_TXCMD, 0);
 	while (pasemi_read_dma_reg(PAS_DMA_COM_TXSTA) & 1) {
 		if (time_after(jiffies, timeout)) {
-			pr_warning("Warning: Could not disable TX section\n");
+			pr_warn("Warning: Could not disable TX section\n");
 			break;
 		}
 	}
diff --git a/arch/powerpc/platforms/pasemi/iommu.c b/arch/powerpc/platforms/pasemi/iommu.c
index 7fec04de27fc..78b80cbd9768 100644
--- a/arch/powerpc/platforms/pasemi/iommu.c
+++ b/arch/powerpc/platforms/pasemi/iommu.c
@@ -186,7 +186,7 @@ static void pci_dma_dev_setup_pasemi(struct pci_dev *dev)
 	 */
 	if (dev->vendor == 0x1959 && dev->device == 0xa007 &&
 	    !firmware_has_feature(FW_FEATURE_LPAR)) {
-		dev->dev.dma_ops = &dma_direct_ops;
+		dev->dev.dma_ops = &dma_nommu_ops;
 		/*
 		 * Set the coherent DMA mask to prevent the iommu
 		 * being used unnecessarily
diff --git a/arch/powerpc/platforms/pasemi/setup.c b/arch/powerpc/platforms/pasemi/setup.c
index c4a3e93dc324..d0b8ae53660d 100644
--- a/arch/powerpc/platforms/pasemi/setup.c
+++ b/arch/powerpc/platforms/pasemi/setup.c
@@ -363,7 +363,7 @@ static int pcmcia_notify(struct notifier_block *nb, unsigned long action,
 		return 0;
 
 	/* We use the direct ops for localbus */
-	dev->dma_ops = &dma_direct_ops;
+	dev->dma_ops = &dma_nommu_ops;
 
 	return 0;
 }
diff --git a/arch/powerpc/platforms/powermac/backlight.c b/arch/powerpc/platforms/powermac/backlight.c
index a00096b1c713..6b5dcccae1d3 100644
--- a/arch/powerpc/platforms/powermac/backlight.c
+++ b/arch/powerpc/platforms/powermac/backlight.c
@@ -186,7 +186,7 @@ int pmac_backlight_set_legacy_brightness(int brightness)
 	return __pmac_backlight_set_legacy_brightness(brightness);
 }
 
-int pmac_backlight_get_legacy_brightness()
+int pmac_backlight_get_legacy_brightness(void)
 {
 	int result = -ENXIO;
 
@@ -205,12 +205,12 @@ int pmac_backlight_get_legacy_brightness()
 	return result;
 }
 
-void pmac_backlight_disable()
+void pmac_backlight_disable(void)
 {
 	atomic_inc(&kernel_backlight_disabled);
 }
 
-void pmac_backlight_enable()
+void pmac_backlight_enable(void)
 {
 	atomic_dec(&kernel_backlight_disabled);
 }
diff --git a/arch/powerpc/platforms/powermac/feature.c b/arch/powerpc/platforms/powermac/feature.c
index 9e3f39d36e88..3f82cb24eb2b 100644
--- a/arch/powerpc/platforms/powermac/feature.c
+++ b/arch/powerpc/platforms/powermac/feature.c
@@ -829,7 +829,7 @@ core99_ata100_enable(struct device_node *node, long value)
 
 	if (value) {
 		if (pci_device_from_OF_node(node, &pbus, &pid) == 0)
-			pdev = pci_get_bus_and_slot(pbus, pid);
+			pdev = pci_get_domain_bus_and_slot(0, pbus, pid);
 		if (pdev == NULL)
 			return 0;
 		rc = pci_enable_device(pdev);
@@ -2641,7 +2641,7 @@ static void __init probe_one_macio(const char *name, const char *compat, int typ
 	phys_addr_t		addr;
 	u64			size;
 
-	for (node = NULL; (node = of_find_node_by_name(node, name)) != NULL;) {
+	for_each_node_by_name(node, name) {
 		if (!compat)
 			break;
 		if (of_device_is_compatible(node, compat))
@@ -2853,7 +2853,6 @@ set_initial_features(void)
 		}
 
 		/* Enable ATA-100 before PCI probe. */
-		np = of_find_node_by_name(NULL, "ata-6");
 		for_each_node_by_name(np, "ata-6") {
 			if (np->parent
 			    && of_device_is_compatible(np->parent, "uni-north")
diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c
index 5e0719b27294..57bbff465964 100644
--- a/arch/powerpc/platforms/powermac/pic.c
+++ b/arch/powerpc/platforms/powermac/pic.c
@@ -486,15 +486,16 @@ static int __init pmac_pic_probe_mpic(void)
 	struct device_node *np, *master = NULL, *slave = NULL;
 
 	/* We can have up to 2 MPICs cascaded */
-	for (np = NULL; (np = of_find_node_by_type(np, "open-pic"))
-		     != NULL;) {
+	for_each_node_by_type(np, "open-pic") {
 		if (master == NULL &&
 		    of_get_property(np, "interrupts", NULL) == NULL)
 			master = of_node_get(np);
 		else if (slave == NULL)
 			slave = of_node_get(np);
-		if (master && slave)
+		if (master && slave) {
+			of_node_put(np);
 			break;
+		}
 	}
 
 	/* Check for bogus setups */
@@ -604,6 +605,7 @@ static int pmacpic_find_viaint(void)
 	if (np == NULL)
 		goto not_found;
 	viaint = irq_of_parse_and_map(np, 0);
+	of_node_put(np);
 
 not_found:
 #endif /* CONFIG_ADB_PMU */
diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c
index 2cd99eb30762..95275e0e2efa 100644
--- a/arch/powerpc/platforms/powermac/smp.c
+++ b/arch/powerpc/platforms/powermac/smp.c
@@ -774,8 +774,8 @@ static void __init smp_core99_probe(void)
 	if (ppc_md.progress) ppc_md.progress("smp_core99_probe", 0x345);
 
 	/* Count CPUs in the device-tree */
-       	for (cpus = NULL; (cpus = of_find_node_by_type(cpus, "cpu")) != NULL;)
-	       	++ncpus;
+	for_each_node_by_type(cpus, "cpu")
+		++ncpus;
 
 	printk(KERN_INFO "PowerMac SMP probe found %d cpus\n", ncpus);
 
diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile
index 3732118a0482..6c9d5199a7e2 100644
--- a/arch/powerpc/platforms/powernv/Makefile
+++ b/arch/powerpc/platforms/powernv/Makefile
@@ -17,3 +17,4 @@ obj-$(CONFIG_PERF_EVENTS) += opal-imc.o
 obj-$(CONFIG_PPC_MEMTRACE)	+= memtrace.o
 obj-$(CONFIG_PPC_VAS)	+= vas.o vas-window.o vas-debug.o
 obj-$(CONFIG_PPC_FTW)	+= nx-ftw.o
+obj-$(CONFIG_OCXL_BASE)	+= ocxl.o
diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c
index 4650fb294e7a..33c86c1a1720 100644
--- a/arch/powerpc/platforms/powernv/eeh-powernv.c
+++ b/arch/powerpc/platforms/powernv/eeh-powernv.c
@@ -43,6 +43,22 @@
 
 static int eeh_event_irq = -EINVAL;
 
+void pnv_pcibios_bus_add_device(struct pci_dev *pdev)
+{
+	struct pci_dn *pdn = pci_get_pdn(pdev);
+
+	if (!pdev->is_virtfn)
+		return;
+
+	/*
+	 * The following operations will fail if VF's sysfs files
+	 * aren't created or its resources aren't finalized.
+	 */
+	eeh_add_device_early(pdn);
+	eeh_add_device_late(pdev);
+	eeh_sysfs_add_device(pdev);
+}
+
 static int pnv_eeh_init(void)
 {
 	struct pci_controller *hose;
@@ -86,6 +102,7 @@ static int pnv_eeh_init(void)
 	}
 
 	eeh_set_pe_aux_size(max_diag_size);
+	ppc_md.pcibios_bus_add_device = pnv_pcibios_bus_add_device;
 
 	return 0;
 }
@@ -1638,70 +1655,11 @@ static int pnv_eeh_next_error(struct eeh_pe **pe)
 	return ret;
 }
 
-static int pnv_eeh_restore_vf_config(struct pci_dn *pdn)
-{
-	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
-	u32 devctl, cmd, cap2, aer_capctl;
-	int old_mps;
-
-	if (edev->pcie_cap) {
-		/* Restore MPS */
-		old_mps = (ffs(pdn->mps) - 8) << 5;
-		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
-				     2, &devctl);
-		devctl &= ~PCI_EXP_DEVCTL_PAYLOAD;
-		devctl |= old_mps;
-		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
-				      2, devctl);
-
-		/* Disable Completion Timeout */
-		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP2,
-				     4, &cap2);
-		if (cap2 & 0x10) {
-			eeh_ops->read_config(pdn,
-					     edev->pcie_cap + PCI_EXP_DEVCTL2,
-					     4, &cap2);
-			cap2 |= 0x10;
-			eeh_ops->write_config(pdn,
-					      edev->pcie_cap + PCI_EXP_DEVCTL2,
-					      4, cap2);
-		}
-	}
-
-	/* Enable SERR and parity checking */
-	eeh_ops->read_config(pdn, PCI_COMMAND, 2, &cmd);
-	cmd |= (PCI_COMMAND_PARITY | PCI_COMMAND_SERR);
-	eeh_ops->write_config(pdn, PCI_COMMAND, 2, cmd);
-
-	/* Enable report various errors */
-	if (edev->pcie_cap) {
-		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
-				     2, &devctl);
-		devctl &= ~PCI_EXP_DEVCTL_CERE;
-		devctl |= (PCI_EXP_DEVCTL_NFERE |
-			   PCI_EXP_DEVCTL_FERE |
-			   PCI_EXP_DEVCTL_URRE);
-		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
-				      2, devctl);
-	}
-
-	/* Enable ECRC generation and check */
-	if (edev->pcie_cap && edev->aer_cap) {
-		eeh_ops->read_config(pdn, edev->aer_cap + PCI_ERR_CAP,
-				     4, &aer_capctl);
-		aer_capctl |= (PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
-		eeh_ops->write_config(pdn, edev->aer_cap + PCI_ERR_CAP,
-				      4, aer_capctl);
-	}
-
-	return 0;
-}
-
 static int pnv_eeh_restore_config(struct pci_dn *pdn)
 {
 	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
 	struct pnv_phb *phb;
-	s64 ret;
+	s64 ret = 0;
 	int config_addr = (pdn->busno << 8) | (pdn->devfn);
 
 	if (!edev)
@@ -1715,7 +1673,7 @@ static int pnv_eeh_restore_config(struct pci_dn *pdn)
 	 * to be exported by firmware in extendible way.
 	 */
 	if (edev->physfn) {
-		ret = pnv_eeh_restore_vf_config(pdn);
+		ret = eeh_restore_vf_config(pdn);
 	} else {
 		phb = pdn->phb->private_data;
 		ret = opal_pci_reinit(phb->opal_id,
@@ -1728,7 +1686,7 @@ static int pnv_eeh_restore_config(struct pci_dn *pdn)
 		return -EIO;
 	}
 
-	return 0;
+	return ret;
 }
 
 static struct eeh_ops pnv_eeh_ops = {
@@ -1746,25 +1704,10 @@ static struct eeh_ops pnv_eeh_ops = {
 	.read_config            = pnv_eeh_read_config,
 	.write_config           = pnv_eeh_write_config,
 	.next_error		= pnv_eeh_next_error,
-	.restore_config		= pnv_eeh_restore_config
+	.restore_config		= pnv_eeh_restore_config,
+	.notify_resume		= NULL
 };
 
-void pcibios_bus_add_device(struct pci_dev *pdev)
-{
-	struct pci_dn *pdn = pci_get_pdn(pdev);
-
-	if (!pdev->is_virtfn)
-		return;
-
-	/*
-	 * The following operations will fail if VF's sysfs files
-	 * aren't created or its resources aren't finalized.
-	 */
-	eeh_add_device_early(pdn);
-	eeh_add_device_late(pdev);
-	eeh_sysfs_add_device(pdev);
-}
-
 #ifdef CONFIG_PCI_IOV
 static void pnv_pci_fixup_vf_mps(struct pci_dev *pdev)
 {
diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c
index f6cbc1a71472..0a253b64ac5f 100644
--- a/arch/powerpc/platforms/powernv/npu-dma.c
+++ b/arch/powerpc/platforms/powernv/npu-dma.c
@@ -39,7 +39,10 @@
  */
 static struct pci_dev *get_pci_dev(struct device_node *dn)
 {
-	return PCI_DN(dn)->pcidev;
+	struct pci_dn *pdn = PCI_DN(dn);
+
+	return pci_get_domain_bus_and_slot(pci_domain_nr(pdn->phb->bus),
+					   pdn->busno, pdn->devfn);
 }
 
 /* Given a NPU device get the associated PCI device. */
@@ -277,7 +280,7 @@ static int pnv_npu_dma_set_bypass(struct pnv_ioda_pe *npe)
 	int64_t rc = 0;
 	phys_addr_t top = memblock_end_of_DRAM();
 
-	if (phb->type != PNV_PHB_NPU || !npe->pdev)
+	if (phb->type != PNV_PHB_NPU_NVLINK || !npe->pdev)
 		return -EINVAL;
 
 	rc = pnv_npu_unset_window(npe, 0);
diff --git a/arch/powerpc/platforms/powernv/ocxl.c b/arch/powerpc/platforms/powernv/ocxl.c
new file mode 100644
index 000000000000..fa9b53af3c7b
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/ocxl.c
@@ -0,0 +1,515 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <asm/pnv-ocxl.h>
+#include <asm/opal.h>
+#include <asm/xive.h>
+#include <misc/ocxl-config.h>
+#include "pci.h"
+
+#define PNV_OCXL_TL_P9_RECV_CAP		0x000000000000000Full
+#define PNV_OCXL_ACTAG_MAX		64
+/* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */
+#define PNV_OCXL_PASID_BITS		15
+#define PNV_OCXL_PASID_MAX		((1 << PNV_OCXL_PASID_BITS) - 1)
+
+#define AFU_PRESENT (1 << 31)
+#define AFU_INDEX_MASK 0x3F000000
+#define AFU_INDEX_SHIFT 24
+#define ACTAG_MASK 0xFFF
+
+
+struct actag_range {
+	u16 start;
+	u16 count;
+};
+
+struct npu_link {
+	struct list_head list;
+	int domain;
+	int bus;
+	int dev;
+	u16 fn_desired_actags[8];
+	struct actag_range fn_actags[8];
+	bool assignment_done;
+};
+static struct list_head links_list = LIST_HEAD_INIT(links_list);
+static DEFINE_MUTEX(links_list_lock);
+
+
+/*
+ * opencapi actags handling:
+ *
+ * When sending commands, the opencapi device references the memory
+ * context it's targeting with an 'actag', which is really an alias
+ * for a (BDF, pasid) combination. When it receives a command, the NPU
+ * must do a lookup of the actag to identify the memory context. The
+ * hardware supports a finite number of actags per link (64 for
+ * POWER9).
+ *
+ * The device can carry multiple functions, and each function can have
+ * multiple AFUs. Each AFU advertises in its config space the number
+ * of desired actags. The host must configure in the config space of
+ * the AFU how many actags the AFU is really allowed to use (which can
+ * be less than what the AFU desires).
+ *
+ * When a PCI function is probed by the driver, it has no visibility
+ * about the other PCI functions and how many actags they'd like,
+ * which makes it impossible to distribute actags fairly among AFUs.
+ *
+ * Unfortunately, the only way to know how many actags a function
+ * desires is by looking at the data for each AFU in the config space
+ * and add them up. Similarly, the only way to know how many actags
+ * all the functions of the physical device desire is by adding the
+ * previously computed function counts. Then we can match that against
+ * what the hardware supports.
+ *
+ * To get a comprehensive view, we use a 'pci fixup': at the end of
+ * PCI enumeration, each function counts how many actags its AFUs
+ * desire and we save it in a 'npu_link' structure, shared between all
+ * the PCI functions of a same device. Therefore, when the first
+ * function is probed by the driver, we can get an idea of the total
+ * count of desired actags for the device, and assign the actags to
+ * the AFUs, by pro-rating if needed.
+ */
+
+static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos)
+{
+	int vsec = pos;
+	u16 vendor, id;
+
+	while ((vsec = pci_find_next_ext_capability(dev, vsec,
+						    OCXL_EXT_CAP_ID_DVSEC))) {
+		pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
+				&vendor);
+		pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
+		if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
+			return vsec;
+	}
+	return 0;
+}
+
+static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
+{
+	int vsec = 0;
+	u8 idx;
+
+	while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID,
+					   vsec))) {
+		pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
+				&idx);
+		if (idx == afu_idx)
+			return vsec;
+	}
+	return 0;
+}
+
+static int get_max_afu_index(struct pci_dev *dev, int *afu_idx)
+{
+	int pos;
+	u32 val;
+
+	pos = find_dvsec_from_pos(dev, OCXL_DVSEC_FUNC_ID, 0);
+	if (!pos)
+		return -ESRCH;
+
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
+	if (val & AFU_PRESENT)
+		*afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT;
+	else
+		*afu_idx = -1;
+	return 0;
+}
+
+static int get_actag_count(struct pci_dev *dev, int afu_idx, int *actag)
+{
+	int pos;
+	u16 actag_sup;
+
+	pos = find_dvsec_afu_ctrl(dev, afu_idx);
+	if (!pos)
+		return -ESRCH;
+
+	pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP,
+			&actag_sup);
+	*actag = actag_sup & ACTAG_MASK;
+	return 0;
+}
+
+static struct npu_link *find_link(struct pci_dev *dev)
+{
+	struct npu_link *link;
+
+	list_for_each_entry(link, &links_list, list) {
+		/* The functions of a device all share the same link */
+		if (link->domain == pci_domain_nr(dev->bus) &&
+			link->bus == dev->bus->number &&
+			link->dev == PCI_SLOT(dev->devfn)) {
+			return link;
+		}
+	}
+
+	/* link doesn't exist yet. Allocate one */
+	link = kzalloc(sizeof(struct npu_link), GFP_KERNEL);
+	if (!link)
+		return NULL;
+	link->domain = pci_domain_nr(dev->bus);
+	link->bus = dev->bus->number;
+	link->dev = PCI_SLOT(dev->devfn);
+	list_add(&link->list, &links_list);
+	return link;
+}
+
+static void pnv_ocxl_fixup_actag(struct pci_dev *dev)
+{
+	struct pci_controller *hose = pci_bus_to_host(dev->bus);
+	struct pnv_phb *phb = hose->private_data;
+	struct npu_link *link;
+	int rc, afu_idx = -1, i, actag;
+
+	if (!machine_is(powernv))
+		return;
+
+	if (phb->type != PNV_PHB_NPU_OCAPI)
+		return;
+
+	mutex_lock(&links_list_lock);
+
+	link = find_link(dev);
+	if (!link) {
+		dev_warn(&dev->dev, "couldn't update actag information\n");
+		mutex_unlock(&links_list_lock);
+		return;
+	}
+
+	/*
+	 * Check how many actags are desired for the AFUs under that
+	 * function and add it to the count for the link
+	 */
+	rc = get_max_afu_index(dev, &afu_idx);
+	if (rc) {
+		/* Most likely an invalid config space */
+		dev_dbg(&dev->dev, "couldn't find AFU information\n");
+		afu_idx = -1;
+	}
+
+	link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0;
+	for (i = 0; i <= afu_idx; i++) {
+		/*
+		 * AFU index 'holes' are allowed. So don't fail if we
+		 * can't read the actag info for an index
+		 */
+		rc = get_actag_count(dev, i, &actag);
+		if (rc)
+			continue;
+		link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag;
+	}
+	dev_dbg(&dev->dev, "total actags for function: %d\n",
+		link->fn_desired_actags[PCI_FUNC(dev->devfn)]);
+
+	mutex_unlock(&links_list_lock);
+}
+DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag);
+
+static u16 assign_fn_actags(u16 desired, u16 total)
+{
+	u16 count;
+
+	if (total <= PNV_OCXL_ACTAG_MAX)
+		count = desired;
+	else
+		count = PNV_OCXL_ACTAG_MAX * desired / total;
+
+	return count;
+}
+
+static void assign_actags(struct npu_link *link)
+{
+	u16 actag_count, range_start = 0, total_desired = 0;
+	int i;
+
+	for (i = 0; i < 8; i++)
+		total_desired += link->fn_desired_actags[i];
+
+	for (i = 0; i < 8; i++) {
+		if (link->fn_desired_actags[i]) {
+			actag_count = assign_fn_actags(
+				link->fn_desired_actags[i],
+				total_desired);
+			link->fn_actags[i].start = range_start;
+			link->fn_actags[i].count = actag_count;
+			range_start += actag_count;
+			WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX);
+		}
+		pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n",
+			link->domain, link->bus, link->dev, i,
+			link->fn_actags[i].start, link->fn_actags[i].count,
+			link->fn_desired_actags[i]);
+	}
+	link->assignment_done = true;
+}
+
+int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled,
+		u16 *supported)
+{
+	struct npu_link *link;
+
+	mutex_lock(&links_list_lock);
+
+	link = find_link(dev);
+	if (!link) {
+		dev_err(&dev->dev, "actag information not found\n");
+		mutex_unlock(&links_list_lock);
+		return -ENODEV;
+	}
+	/*
+	 * On p9, we only have 64 actags per link, so they must be
+	 * shared by all the functions of the same adapter. We counted
+	 * the desired actag counts during PCI enumeration, so that we
+	 * can allocate a pro-rated number of actags to each function.
+	 */
+	if (!link->assignment_done)
+		assign_actags(link);
+
+	*base      = link->fn_actags[PCI_FUNC(dev->devfn)].start;
+	*enabled   = link->fn_actags[PCI_FUNC(dev->devfn)].count;
+	*supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)];
+
+	mutex_unlock(&links_list_lock);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag);
+
+int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count)
+{
+	struct npu_link *link;
+	int i, rc = -EINVAL;
+
+	/*
+	 * The number of PASIDs (process address space ID) which can
+	 * be used by a function depends on how many functions exist
+	 * on the device. The NPU needs to be configured to know how
+	 * many bits are available to PASIDs and how many are to be
+	 * used by the function BDF indentifier.
+	 *
+	 * We only support one AFU-carrying function for now.
+	 */
+	mutex_lock(&links_list_lock);
+
+	link = find_link(dev);
+	if (!link) {
+		dev_err(&dev->dev, "actag information not found\n");
+		mutex_unlock(&links_list_lock);
+		return -ENODEV;
+	}
+
+	for (i = 0; i < 8; i++)
+		if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) {
+			*count = PNV_OCXL_PASID_MAX;
+			rc = 0;
+			break;
+		}
+
+	mutex_unlock(&links_list_lock);
+	dev_dbg(&dev->dev, "%d PASIDs available for function\n",
+		rc ? 0 : *count);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count);
+
+static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf)
+{
+	int shift, idx;
+
+	WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE);
+	idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2;
+	shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2));
+	buf[idx] |= rate << shift;
+}
+
+int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap,
+			char *rate_buf, int rate_buf_size)
+{
+	if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
+		return -EINVAL;
+	/*
+	 * The TL capabilities are a characteristic of the NPU, so
+	 * we go with hard-coded values.
+	 *
+	 * The receiving rate of each template is encoded on 4 bits.
+	 *
+	 * On P9:
+	 * - templates 0 -> 3 are supported
+	 * - templates 0, 1 and 3 have a 0 receiving rate
+	 * - template 2 has receiving rate of 1 (extra cycle)
+	 */
+	memset(rate_buf, 0, rate_buf_size);
+	set_templ_rate(2, 1, rate_buf);
+	*cap = PNV_OCXL_TL_P9_RECV_CAP;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap);
+
+int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap,
+			uint64_t rate_buf_phys, int rate_buf_size)
+{
+	struct pci_controller *hose = pci_bus_to_host(dev->bus);
+	struct pnv_phb *phb = hose->private_data;
+	int rc;
+
+	if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE)
+		return -EINVAL;
+
+	rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap,
+			rate_buf_phys, rate_buf_size);
+	if (rc) {
+		dev_err(&dev->dev, "Can't configure host TL: %d\n", rc);
+		return -EINVAL;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf);
+
+int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq)
+{
+	int rc;
+
+	rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq);
+	if (rc) {
+		dev_err(&dev->dev,
+			"Can't get translation interrupt for device\n");
+		return rc;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq);
+
+void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar,
+			void __iomem *tfc, void __iomem *pe_handle)
+{
+	iounmap(dsisr);
+	iounmap(dar);
+	iounmap(tfc);
+	iounmap(pe_handle);
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs);
+
+int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr,
+			void __iomem **dar, void __iomem **tfc,
+			void __iomem **pe_handle)
+{
+	u64 reg;
+	int i, j, rc = 0;
+	void __iomem *regs[4];
+
+	/*
+	 * opal stores the mmio addresses of the DSISR, DAR, TFC and
+	 * PE_HANDLE registers in a device tree property, in that
+	 * order
+	 */
+	for (i = 0; i < 4; i++) {
+		rc = of_property_read_u64_index(dev->dev.of_node,
+						"ibm,opal-xsl-mmio", i, &reg);
+		if (rc)
+			break;
+		regs[i] = ioremap(reg, 8);
+		if (!regs[i]) {
+			rc = -EINVAL;
+			break;
+		}
+	}
+	if (rc) {
+		dev_err(&dev->dev, "Can't map translation mmio registers\n");
+		for (j = i - 1; j >= 0; j--)
+			iounmap(regs[j]);
+	} else {
+		*dsisr = regs[0];
+		*dar = regs[1];
+		*tfc = regs[2];
+		*pe_handle = regs[3];
+	}
+	return rc;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs);
+
+struct spa_data {
+	u64 phb_opal_id;
+	u32 bdfn;
+};
+
+int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask,
+		void **platform_data)
+{
+	struct pci_controller *hose = pci_bus_to_host(dev->bus);
+	struct pnv_phb *phb = hose->private_data;
+	struct spa_data *data;
+	u32 bdfn;
+	int rc;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	bdfn = (dev->bus->number << 8) | dev->devfn;
+	rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem),
+				PE_mask);
+	if (rc) {
+		dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc);
+		kfree(data);
+		return rc;
+	}
+	data->phb_opal_id = phb->opal_id;
+	data->bdfn = bdfn;
+	*platform_data = (void *) data;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup);
+
+void pnv_ocxl_spa_release(void *platform_data)
+{
+	struct spa_data *data = (struct spa_data *) platform_data;
+	int rc;
+
+	rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0);
+	WARN_ON(rc);
+	kfree(data);
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release);
+
+int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle)
+{
+	struct spa_data *data = (struct spa_data *) platform_data;
+	int rc;
+
+	rc = opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe);
+
+int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr)
+{
+	__be64 flags, trigger_page;
+	s64 rc;
+	u32 hwirq;
+
+	hwirq = xive_native_alloc_irq();
+	if (!hwirq)
+		return -ENOENT;
+
+	rc = opal_xive_get_irq_info(hwirq, &flags, NULL, &trigger_page, NULL,
+				NULL);
+	if (rc || !trigger_page) {
+		xive_native_free_irq(hwirq);
+		return -ENOENT;
+	}
+	*irq = hwirq;
+	*trigger_addr = be64_to_cpu(trigger_page);
+	return 0;
+
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_alloc_xive_irq);
+
+void pnv_ocxl_free_xive_irq(u32 irq)
+{
+	xive_native_free_irq(irq);
+}
+EXPORT_SYMBOL_GPL(pnv_ocxl_free_xive_irq);
diff --git a/arch/powerpc/platforms/powernv/opal-dump.c b/arch/powerpc/platforms/powernv/opal-dump.c
index 4c827826c05e..0dc8fa4e0af2 100644
--- a/arch/powerpc/platforms/powernv/opal-dump.c
+++ b/arch/powerpc/platforms/powernv/opal-dump.c
@@ -103,9 +103,9 @@ static ssize_t dump_ack_store(struct dump_obj *dump_obj,
  * due to the dynamic size of the dump
  */
 static struct dump_attribute id_attribute =
-	__ATTR(id, S_IRUGO, dump_id_show, NULL);
+	__ATTR(id, 0444, dump_id_show, NULL);
 static struct dump_attribute type_attribute =
-	__ATTR(type, S_IRUGO, dump_type_show, NULL);
+	__ATTR(type, 0444, dump_type_show, NULL);
 static struct dump_attribute ack_attribute =
 	__ATTR(acknowledge, 0660, dump_ack_show, dump_ack_store);
 
diff --git a/arch/powerpc/platforms/powernv/opal-elog.c b/arch/powerpc/platforms/powernv/opal-elog.c
index ecd6d9177d13..ba6e437abb4b 100644
--- a/arch/powerpc/platforms/powernv/opal-elog.c
+++ b/arch/powerpc/platforms/powernv/opal-elog.c
@@ -83,9 +83,9 @@ static ssize_t elog_ack_store(struct elog_obj *elog_obj,
 }
 
 static struct elog_attribute id_attribute =
-	__ATTR(id, S_IRUGO, elog_id_show, NULL);
+	__ATTR(id, 0444, elog_id_show, NULL);
 static struct elog_attribute type_attribute =
-	__ATTR(type, S_IRUGO, elog_type_show, NULL);
+	__ATTR(type, 0444, elog_type_show, NULL);
 static struct elog_attribute ack_attribute =
 	__ATTR(acknowledge, 0660, elog_ack_show, elog_ack_store);
 
diff --git a/arch/powerpc/platforms/powernv/opal-imc.c b/arch/powerpc/platforms/powernv/opal-imc.c
index 465ea105b771..f6f55ab4980e 100644
--- a/arch/powerpc/platforms/powernv/opal-imc.c
+++ b/arch/powerpc/platforms/powernv/opal-imc.c
@@ -21,6 +21,78 @@
 #include <asm/io.h>
 #include <asm/imc-pmu.h>
 #include <asm/cputhreads.h>
+#include <asm/debugfs.h>
+
+static struct dentry *imc_debugfs_parent;
+
+/* Helpers to export imc command and mode via debugfs */
+static int imc_mem_get(void *data, u64 *val)
+{
+	*val = cpu_to_be64(*(u64 *)data);
+	return 0;
+}
+
+static int imc_mem_set(void *data, u64 val)
+{
+	*(u64 *)data = cpu_to_be64(val);
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(fops_imc_x64, imc_mem_get, imc_mem_set, "0x%016llx\n");
+
+static struct dentry *imc_debugfs_create_x64(const char *name, umode_t mode,
+					     struct dentry *parent, u64  *value)
+{
+	return debugfs_create_file_unsafe(name, mode, parent,
+					  value, &fops_imc_x64);
+}
+
+/*
+ * export_imc_mode_and_cmd: Create a debugfs interface
+ *                     for imc_cmd and imc_mode
+ *                     for each node in the system.
+ *  imc_mode and imc_cmd can be changed by echo into
+ *  this interface.
+ */
+static void export_imc_mode_and_cmd(struct device_node *node,
+				    struct imc_pmu *pmu_ptr)
+{
+	static u64 loc, *imc_mode_addr, *imc_cmd_addr;
+	int chip = 0, nid;
+	char mode[16], cmd[16];
+	u32 cb_offset;
+
+	imc_debugfs_parent = debugfs_create_dir("imc", powerpc_debugfs_root);
+
+	/*
+	 * Return here, either because 'imc' directory already exists,
+	 * Or failed to create a new one.
+	 */
+	if (!imc_debugfs_parent)
+		return;
+
+	if (of_property_read_u32(node, "cb_offset", &cb_offset))
+		cb_offset = IMC_CNTL_BLK_OFFSET;
+
+	for_each_node(nid) {
+		loc = (u64)(pmu_ptr->mem_info[chip].vbase) + cb_offset;
+		imc_mode_addr = (u64 *)(loc + IMC_CNTL_BLK_MODE_OFFSET);
+		sprintf(mode, "imc_mode_%d", nid);
+		if (!imc_debugfs_create_x64(mode, 0600, imc_debugfs_parent,
+					    imc_mode_addr))
+			goto err;
+
+		imc_cmd_addr = (u64 *)(loc + IMC_CNTL_BLK_CMD_OFFSET);
+		sprintf(cmd, "imc_cmd_%d", nid);
+		if (!imc_debugfs_create_x64(cmd, 0600, imc_debugfs_parent,
+					    imc_cmd_addr))
+			goto err;
+		chip++;
+	}
+	return;
+
+err:
+	debugfs_remove_recursive(imc_debugfs_parent);
+}
 
 /*
  * imc_get_mem_addr_nest: Function to get nest counter memory region
@@ -65,6 +137,7 @@ static int imc_get_mem_addr_nest(struct device_node *node,
 	}
 
 	pmu_ptr->imc_counter_mmaped = true;
+	export_imc_mode_and_cmd(node, pmu_ptr);
 	kfree(base_addr_arr);
 	kfree(chipid_arr);
 	return 0;
@@ -126,9 +199,11 @@ static void disable_nest_pmu_counters(void)
 	const struct cpumask *l_cpumask;
 
 	get_online_cpus();
-	for_each_online_node(nid) {
+	for_each_node_with_cpus(nid) {
 		l_cpumask = cpumask_of_node(nid);
-		cpu = cpumask_first(l_cpumask);
+		cpu = cpumask_first_and(l_cpumask, cpu_online_mask);
+		if (cpu >= nr_cpu_ids)
+			continue;
 		opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
 				       get_hard_smp_processor_id(cpu));
 	}
@@ -213,6 +288,10 @@ static int opal_imc_counters_probe(struct platform_device *pdev)
 		}
 	}
 
+	/* If none of the nest units are registered, remove debugfs interface */
+	if (pmu_count == 0)
+		debugfs_remove_recursive(imc_debugfs_parent);
+
 	return 0;
 }
 
diff --git a/arch/powerpc/platforms/powernv/opal-memory-errors.c b/arch/powerpc/platforms/powernv/opal-memory-errors.c
index d9916ea62305..8ddc1accf199 100644
--- a/arch/powerpc/platforms/powernv/opal-memory-errors.c
+++ b/arch/powerpc/platforms/powernv/opal-memory-errors.c
@@ -60,7 +60,7 @@ static void handle_memory_error_event(struct OpalMemoryErrorData *merr_evt)
 	}
 
 	for (; paddr_start < paddr_end; paddr_start += PAGE_SIZE) {
-		memory_failure(paddr_start >> PAGE_SHIFT, 0, 0);
+		memory_failure(paddr_start >> PAGE_SHIFT, 0);
 	}
 }
 
diff --git a/arch/powerpc/platforms/powernv/opal-prd.c b/arch/powerpc/platforms/powernv/opal-prd.c
index de4dd09f4a15..4070bb4e9da4 100644
--- a/arch/powerpc/platforms/powernv/opal-prd.c
+++ b/arch/powerpc/platforms/powernv/opal-prd.c
@@ -147,13 +147,13 @@ static bool opal_msg_queue_empty(void)
 	return ret;
 }
 
-static unsigned int opal_prd_poll(struct file *file,
+static __poll_t opal_prd_poll(struct file *file,
 		struct poll_table_struct *wait)
 {
 	poll_wait(file, &opal_prd_msg_wait, wait);
 
 	if (!opal_msg_queue_empty())
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/arch/powerpc/platforms/powernv/opal-sysparam.c b/arch/powerpc/platforms/powernv/opal-sysparam.c
index 23fb6647dced..6fd4092798d5 100644
--- a/arch/powerpc/platforms/powernv/opal-sysparam.c
+++ b/arch/powerpc/platforms/powernv/opal-sysparam.c
@@ -260,13 +260,13 @@ void __init opal_sys_param_init(void)
 		/* If the parameter is read-only or read-write */
 		switch (perm[i] & 3) {
 		case OPAL_SYSPARAM_READ:
-			attr[i].kobj_attr.attr.mode = S_IRUGO;
+			attr[i].kobj_attr.attr.mode = 0444;
 			break;
 		case OPAL_SYSPARAM_WRITE:
-			attr[i].kobj_attr.attr.mode = S_IWUSR;
+			attr[i].kobj_attr.attr.mode = 0200;
 			break;
 		case OPAL_SYSPARAM_RW:
-			attr[i].kobj_attr.attr.mode = S_IRUGO | S_IWUSR;
+			attr[i].kobj_attr.attr.mode = 0644;
 			break;
 		default:
 			break;
diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S
index 6f4b00a2ac46..1b2936ba6040 100644
--- a/arch/powerpc/platforms/powernv/opal-wrappers.S
+++ b/arch/powerpc/platforms/powernv/opal-wrappers.S
@@ -320,3 +320,6 @@ OPAL_CALL(opal_set_powercap,			OPAL_SET_POWERCAP);
 OPAL_CALL(opal_get_power_shift_ratio,		OPAL_GET_POWER_SHIFT_RATIO);
 OPAL_CALL(opal_set_power_shift_ratio,		OPAL_SET_POWER_SHIFT_RATIO);
 OPAL_CALL(opal_sensor_group_clear,		OPAL_SENSOR_GROUP_CLEAR);
+OPAL_CALL(opal_npu_spa_setup,			OPAL_NPU_SPA_SETUP);
+OPAL_CALL(opal_npu_spa_clear_cache,		OPAL_NPU_SPA_CLEAR_CACHE);
+OPAL_CALL(opal_npu_tl_set,			OPAL_NPU_TL_SET);
diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c
index 041ddbd1fc57..c15182765ff5 100644
--- a/arch/powerpc/platforms/powernv/opal.c
+++ b/arch/powerpc/platforms/powernv/opal.c
@@ -127,7 +127,7 @@ int __init early_init_dt_scan_opal(unsigned long node,
 
 	if (of_flat_dt_is_compatible(node, "ibm,opal-v3")) {
 		powerpc_firmware_features |= FW_FEATURE_OPAL;
-		pr_info("OPAL detected !\n");
+		pr_debug("OPAL detected !\n");
 	} else {
 		panic("OPAL != V3 detected, no longer supported.\n");
 	}
@@ -239,8 +239,8 @@ int opal_message_notifier_register(enum opal_msg_type msg_type,
 					struct notifier_block *nb)
 {
 	if (!nb || msg_type >= OPAL_MSG_TYPE_MAX) {
-		pr_warning("%s: Invalid arguments, msg_type:%d\n",
-			   __func__, msg_type);
+		pr_warn("%s: Invalid arguments, msg_type:%d\n",
+			__func__, msg_type);
 		return -EINVAL;
 	}
 
@@ -281,8 +281,8 @@ static void opal_handle_message(void)
 
 	/* check for errors. */
 	if (ret) {
-		pr_warning("%s: Failed to retrieve opal message, err=%lld\n",
-				__func__, ret);
+		pr_warn("%s: Failed to retrieve opal message, err=%lld\n",
+			__func__, ret);
 		return;
 	}
 
@@ -461,24 +461,14 @@ static int opal_recover_mce(struct pt_regs *regs,
 
 void pnv_platform_error_reboot(struct pt_regs *regs, const char *msg)
 {
-	/*
-	 * This is mostly taken from kernel/panic.c, but tries to do
-	 * relatively minimal work. Don't use delay functions (TB may
-	 * be broken), don't crash dump (need to set a firmware log),
-	 * don't run notifiers. We do want to get some information to
-	 * Linux console.
-	 */
-	console_verbose();
-	bust_spinlocks(1);
+	panic_flush_kmsg_start();
+
 	pr_emerg("Hardware platform error: %s\n", msg);
 	if (regs)
 		show_regs(regs);
 	smp_send_stop();
-	printk_safe_flush_on_panic();
-	kmsg_dump(KMSG_DUMP_PANIC);
-	bust_spinlocks(0);
-	debug_locks_off();
-	console_flush_on_panic();
+
+	panic_flush_kmsg_end();
 
 	/*
 	 * Don't bother to shut things down because this will
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 749055553064..a6c92c78c9b2 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -54,7 +54,8 @@
 #define POWERNV_IOMMU_DEFAULT_LEVELS	1
 #define POWERNV_IOMMU_MAX_LEVELS	5
 
-static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU" };
+static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_NVLINK",
+					      "NPU_OCAPI" };
 static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
 
 void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
@@ -89,6 +90,7 @@ void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level,
 }
 
 static bool pnv_iommu_bypass_disabled __read_mostly;
+static bool pci_reset_phbs __read_mostly;
 
 static int __init iommu_setup(char *str)
 {
@@ -110,6 +112,14 @@ static int __init iommu_setup(char *str)
 }
 early_param("iommu", iommu_setup);
 
+static int __init pci_reset_phbs_setup(char *str)
+{
+	pci_reset_phbs = true;
+	return 0;
+}
+
+early_param("ppc_pci_reset_phbs", pci_reset_phbs_setup);
+
 static inline bool pnv_pci_is_m64(struct pnv_phb *phb, struct resource *r)
 {
 	/*
@@ -924,7 +934,7 @@ static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
 	 * Configure PELTV. NPUs don't have a PELTV table so skip
 	 * configuration on them.
 	 */
-	if (phb->type != PNV_PHB_NPU)
+	if (phb->type != PNV_PHB_NPU_NVLINK && phb->type != PNV_PHB_NPU_OCAPI)
 		pnv_ioda_set_peltv(phb, pe, true);
 
 	/* Setup reverse map */
@@ -1059,8 +1069,8 @@ static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 
 	pe = pnv_ioda_alloc_pe(phb);
 	if (!pe) {
-		pr_warning("%s: Not enough PE# available, disabling device\n",
-			   pci_name(dev));
+		pr_warn("%s: Not enough PE# available, disabling device\n",
+			pci_name(dev));
 		return NULL;
 	}
 
@@ -1072,7 +1082,6 @@ static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
 	 * At some point we want to remove the PDN completely anyways
 	 */
 	pci_dev_get(dev);
-	pdn->pcidev = dev;
 	pdn->pe_number = pe->pe_number;
 	pe->flags = PNV_IODA_PE_DEV;
 	pe->pdev = dev;
@@ -1119,7 +1128,6 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe)
 			continue;
 
 		pe->device_count++;
-		pdn->pcidev = dev;
 		pdn->pe_number = pe->pe_number;
 		if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate)
 			pnv_ioda_setup_same_PE(dev->subordinate, pe);
@@ -1164,7 +1172,7 @@ static struct pnv_ioda_pe *pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all)
 		pe = pnv_ioda_alloc_pe(phb);
 
 	if (!pe) {
-		pr_warning("%s: Not enough PE# available for PCI bus %04x:%02x\n",
+		pr_warn("%s: Not enough PE# available for PCI bus %04x:%02x\n",
 			__func__, pci_domain_nr(bus), bus->number);
 		return NULL;
 	}
@@ -1234,7 +1242,6 @@ static struct pnv_ioda_pe *pnv_ioda_setup_npu_PE(struct pci_dev *npu_pdev)
 			pci_dev_get(npu_pdev);
 			npu_pdn = pci_get_pdn(npu_pdev);
 			rid = npu_pdev->bus->number << 8 | npu_pdn->devfn;
-			npu_pdn->pcidev = npu_pdev;
 			npu_pdn->pe_number = pe_num;
 			phb->ioda.pe_rmap[rid] = pe->pe_number;
 
@@ -1272,16 +1279,23 @@ static void pnv_pci_ioda_setup_PEs(void)
 {
 	struct pci_controller *hose, *tmp;
 	struct pnv_phb *phb;
+	struct pci_bus *bus;
+	struct pci_dev *pdev;
 
 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
 		phb = hose->private_data;
-		if (phb->type == PNV_PHB_NPU) {
+		if (phb->type == PNV_PHB_NPU_NVLINK) {
 			/* PE#0 is needed for error reporting */
 			pnv_ioda_reserve_pe(phb, 0);
 			pnv_ioda_setup_npu_PEs(hose->bus);
 			if (phb->model == PNV_PHB_MODEL_NPU2)
 				pnv_npu2_init(phb);
 		}
+		if (phb->type == PNV_PHB_NPU_OCAPI) {
+			bus = hose->bus;
+			list_for_each_entry(pdev, &bus->devices, bus_list)
+				pnv_ioda_setup_dev_PE(pdev);
+		}
 	}
 }
 
@@ -1692,7 +1706,7 @@ int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
 	return ret;
 }
 
-int pcibios_sriov_disable(struct pci_dev *pdev)
+int pnv_pcibios_sriov_disable(struct pci_dev *pdev)
 {
 	pnv_pci_sriov_disable(pdev);
 
@@ -1701,7 +1715,7 @@ int pcibios_sriov_disable(struct pci_dev *pdev)
 	return 0;
 }
 
-int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
+int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
 {
 	/* Allocate PCI data */
 	add_dev_pci_data(pdev);
@@ -1840,7 +1854,7 @@ static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
 	s64 rc;
 
 	if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
-		return -ENODEV;;
+		return -ENODEV;
 
 	pe = &phb->ioda.pe_array[pdn->pe_number];
 	if (pe->tce_bypass_enabled) {
@@ -1850,7 +1864,7 @@ static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
 
 	if (bypass) {
 		dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n");
-		set_dma_ops(&pdev->dev, &dma_direct_ops);
+		set_dma_ops(&pdev->dev, &dma_nommu_ops);
 	} else {
 		/*
 		 * If the device can't set the TCE bypass bit but still wants
@@ -1868,7 +1882,7 @@ static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
 				return rc;
 			/* 4GB offset bypasses 32-bit space */
 			set_dma_offset(&pdev->dev, (1ULL << 32));
-			set_dma_ops(&pdev->dev, &dma_direct_ops);
+			set_dma_ops(&pdev->dev, &dma_nommu_ops);
 		} else if (dma_mask >> 32 && dma_mask != DMA_BIT_MASK(64)) {
 			/*
 			 * Fail the request if a DMA mask between 32 and 64 bits
@@ -2572,7 +2586,6 @@ static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift,
 	unsigned long direct_table_size;
 
 	if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS) ||
-			(window_size > memory_hotplug_max()) ||
 			!is_power_of_2(window_size))
 		return 0;
 
@@ -2640,7 +2653,7 @@ static int gpe_table_group_to_npe_cb(struct device *dev, void *opaque)
 
 	hose = pci_bus_to_host(pdev->bus);
 	phb = hose->private_data;
-	if (phb->type != PNV_PHB_NPU)
+	if (phb->type != PNV_PHB_NPU_NVLINK)
 		return 0;
 
 	*ptmppe = &phb->ioda.pe_array[pdn->pe_number];
@@ -2724,7 +2737,7 @@ static void pnv_pci_ioda_setup_iommu_api(void)
 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
 		phb = hose->private_data;
 
-		if (phb->type != PNV_PHB_NPU)
+		if (phb->type != PNV_PHB_NPU_NVLINK)
 			continue;
 
 		list_for_each_entry(pe, &phb->ioda.pe_list, list) {
@@ -3293,7 +3306,7 @@ static void pnv_pci_ioda_create_dbgfs(void)
 		sprintf(name, "PCI%04x", hose->global_number);
 		phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root);
 		if (!phb->dbgfs) {
-			pr_warning("%s: Error on creating debugfs on PHB#%x\n",
+			pr_warn("%s: Error on creating debugfs on PHB#%x\n",
 				__func__, hose->global_number);
 			continue;
 		}
@@ -3774,6 +3787,13 @@ static const struct pci_controller_ops pnv_npu_ioda_controller_ops = {
 	.shutdown		= pnv_pci_ioda_shutdown,
 };
 
+static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = {
+	.enable_device_hook	= pnv_pci_enable_device_hook,
+	.window_alignment	= pnv_pci_window_alignment,
+	.reset_secondary_bus	= pnv_pci_reset_secondary_bus,
+	.shutdown		= pnv_pci_ioda_shutdown,
+};
+
 #ifdef CONFIG_CXL_BASE
 const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = {
 	.dma_dev_setup		= pnv_pci_dma_dev_setup,
@@ -4007,9 +4027,14 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np,
 	 */
 	ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
 
-	if (phb->type == PNV_PHB_NPU) {
+	switch (phb->type) {
+	case PNV_PHB_NPU_NVLINK:
 		hose->controller_ops = pnv_npu_ioda_controller_ops;
-	} else {
+		break;
+	case PNV_PHB_NPU_OCAPI:
+		hose->controller_ops = pnv_npu_ocapi_ioda_controller_ops;
+		break;
+	default:
 		phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup;
 		hose->controller_ops = pnv_pci_ioda_controller_ops;
 	}
@@ -4019,6 +4044,8 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np,
 #ifdef CONFIG_PCI_IOV
 	ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources;
 	ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment;
+	ppc_md.pcibios_sriov_enable = pnv_pcibios_sriov_enable;
+	ppc_md.pcibios_sriov_disable = pnv_pcibios_sriov_disable;
 #endif
 
 	pci_add_flags(PCI_REASSIGN_ALL_RSRC);
@@ -4026,15 +4053,16 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np,
 	/* Reset IODA tables to a clean state */
 	rc = opal_pci_reset(phb_id, OPAL_RESET_PCI_IODA_TABLE, OPAL_ASSERT_RESET);
 	if (rc)
-		pr_warning("  OPAL Error %ld performing IODA table reset !\n", rc);
+		pr_warn("  OPAL Error %ld performing IODA table reset !\n", rc);
 
 	/*
 	 * If we're running in kdump kernel, the previous kernel never
 	 * shutdown PCI devices correctly. We already got IODA table
 	 * cleaned out. So we have to issue PHB reset to stop all PCI
-	 * transactions from previous kernel.
+	 * transactions from previous kernel. The ppc_pci_reset_phbs
+	 * kernel parameter will force this reset too.
 	 */
-	if (is_kdump_kernel()) {
+	if (is_kdump_kernel() || pci_reset_phbs) {
 		pr_info("  Issue PHB reset ...\n");
 		pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL);
 		pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE);
@@ -4052,8 +4080,26 @@ void __init pnv_pci_init_ioda2_phb(struct device_node *np)
 
 void __init pnv_pci_init_npu_phb(struct device_node *np)
 {
-	pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU);
+	pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_NVLINK);
+}
+
+void __init pnv_pci_init_npu2_opencapi_phb(struct device_node *np)
+{
+	pnv_pci_init_ioda_phb(np, 0, PNV_PHB_NPU_OCAPI);
+}
+
+static void pnv_npu2_opencapi_cfg_size_fixup(struct pci_dev *dev)
+{
+	struct pci_controller *hose = pci_bus_to_host(dev->bus);
+	struct pnv_phb *phb = hose->private_data;
+
+	if (!machine_is(powernv))
+		return;
+
+	if (phb->type == PNV_PHB_NPU_OCAPI)
+		dev->cfg_size = PCI_CFG_SPACE_EXP_SIZE;
 }
+DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, pnv_npu2_opencapi_cfg_size_fixup);
 
 void __init pnv_pci_init_ioda_hub(struct device_node *np)
 {
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
index 5422f4a6317c..69d102cbf48f 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
@@ -1142,6 +1142,10 @@ void __init pnv_pci_init(void)
 	for_each_compatible_node(np, NULL, "ibm,ioda2-npu2-phb")
 		pnv_pci_init_npu_phb(np);
 
+	/* Look for NPU2 OpenCAPI PHBs */
+	for_each_compatible_node(np, NULL, "ibm,ioda2-npu2-opencapi-phb")
+		pnv_pci_init_npu2_opencapi_phb(np);
+
 	/* Configure IOMMU DMA hooks */
 	set_pci_dma_ops(&dma_iommu_ops);
 }
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index b772d7473896..eada4b6068cb 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -12,9 +12,10 @@ struct pci_dn;
 #define NV_NMMU_ATSD_REGS 8
 
 enum pnv_phb_type {
-	PNV_PHB_IODA1	= 0,
-	PNV_PHB_IODA2	= 1,
-	PNV_PHB_NPU	= 2,
+	PNV_PHB_IODA1		= 0,
+	PNV_PHB_IODA2		= 1,
+	PNV_PHB_NPU_NVLINK	= 2,
+	PNV_PHB_NPU_OCAPI	= 3,
 };
 
 /* Precise PHB model for error management */
@@ -227,6 +228,7 @@ extern void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
 extern void pnv_pci_init_ioda_hub(struct device_node *np);
 extern void pnv_pci_init_ioda2_phb(struct device_node *np);
 extern void pnv_pci_init_npu_phb(struct device_node *np);
+extern void pnv_pci_init_npu2_opencapi_phb(struct device_node *np);
 extern void pnv_pci_reset_secondary_bus(struct pci_dev *dev);
 extern int pnv_eeh_phb_reset(struct pci_controller *hose, int option);
 
diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c
index 4fb21e17504a..092715b9674b 100644
--- a/arch/powerpc/platforms/powernv/setup.c
+++ b/arch/powerpc/platforms/powernv/setup.c
@@ -80,6 +80,10 @@ static void pnv_setup_rfi_flush(void)
 		if (np && of_property_read_bool(np, "disabled"))
 			enable--;
 
+		np = of_get_child_by_name(fw_features, "speculation-policy-favor-security");
+		if (np && of_property_read_bool(np, "disabled"))
+			enable = 0;
+
 		of_node_put(np);
 		of_node_put(fw_features);
 	}
diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c
index ba030669eca1..9664c8461f03 100644
--- a/arch/powerpc/platforms/powernv/smp.c
+++ b/arch/powerpc/platforms/powernv/smp.c
@@ -37,6 +37,8 @@
 #include <asm/kvm_ppc.h>
 #include <asm/ppc-opcode.h>
 #include <asm/cpuidle.h>
+#include <asm/kexec.h>
+#include <asm/reg.h>
 
 #include "powernv.h"
 
@@ -209,9 +211,32 @@ static void pnv_smp_cpu_kill_self(void)
 		} else if ((srr1 & wmask) == SRR1_WAKEHDBELL) {
 			unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
 			asm volatile(PPC_MSGCLR(%0) : : "r" (msg));
+		} else if ((srr1 & wmask) == SRR1_WAKERESET) {
+			irq_set_pending_from_srr1(srr1);
+			/* Does not return */
 		}
+
 		smp_mb();
 
+		/*
+		 * For kdump kernels, we process the ipi and jump to
+		 * crash_ipi_callback
+		 */
+		if (kdump_in_progress()) {
+			/*
+			 * If we got to this point, we've not used
+			 * NMI's, otherwise we would have gone
+			 * via the SRR1_WAKERESET path. We are
+			 * using regular IPI's for waking up offline
+			 * threads.
+			 */
+			struct pt_regs regs;
+
+			ppc_save_regs(&regs);
+			crash_ipi_callback(&regs);
+			/* Does not return */
+		}
+
 		if (cpu_core_split_required())
 			continue;
 
@@ -371,5 +396,8 @@ void __init pnv_smp_init(void)
 
 #ifdef CONFIG_HOTPLUG_CPU
 	ppc_md.cpu_die	= pnv_smp_cpu_kill_self;
+#ifdef CONFIG_KEXEC_CORE
+	crash_wake_offline = 1;
+#endif
 #endif
 }
diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
index 2b3eb01ab110..b7c53a51c31b 100644
--- a/arch/powerpc/platforms/powernv/vas-window.c
+++ b/arch/powerpc/platforms/powernv/vas-window.c
@@ -1063,16 +1063,16 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
 			rc = PTR_ERR(txwin->paste_kaddr);
 			goto free_window;
 		}
+	} else {
+		/*
+		 * A user mapping must ensure that context switch issues
+		 * CP_ABORT for this thread.
+		 */
+		rc = set_thread_uses_vas();
+		if (rc)
+			goto free_window;
 	}
 
-	/*
-	 * Now that we have a send window, ensure context switch issues
-	 * CP_ABORT for this thread.
-	 */
-	rc = -EINVAL;
-	if (set_thread_uses_vas() < 0)
-		goto free_window;
-
 	set_vinst_win(vinst, txwin);
 
 	return txwin;
diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c
index e48462447ff0..e7075aaff1bb 100644
--- a/arch/powerpc/platforms/ps3/device-init.c
+++ b/arch/powerpc/platforms/ps3/device-init.c
@@ -663,8 +663,8 @@ static void ps3_find_and_add_device(u64 bus_id, u64 dev_id)
 		if (rem)
 			break;
 	}
-	pr_warning("%s:%u: device %llu:%llu not found\n", __func__, __LINE__,
-		   bus_id, dev_id);
+	pr_warn("%s:%u: device %llu:%llu not found\n",
+		__func__, __LINE__, bus_id, dev_id);
 	return;
 
 found:
@@ -859,11 +859,9 @@ static int ps3_probe_thread(void *data)
 
 		if (notify_event->event_type != notify_region_probe ||
 		    notify_event->bus_id != dev.sbd.bus_id) {
-			pr_warning("%s:%u: bad notify_event: event %llu, "
-				   "dev_id %llu, dev_type %llu\n",
-				   __func__, __LINE__, notify_event->event_type,
-				   notify_event->dev_id,
-				   notify_event->dev_type);
+			pr_warn("%s:%u: bad notify_event: event %llu, dev_id %llu, dev_type %llu\n",
+				__func__, __LINE__, notify_event->event_type,
+				notify_event->dev_id, notify_event->dev_type);
 			continue;
 		}
 
diff --git a/arch/powerpc/platforms/ps3/mm.c b/arch/powerpc/platforms/ps3/mm.c
index b0f34663b1ae..7f870ec29daf 100644
--- a/arch/powerpc/platforms/ps3/mm.c
+++ b/arch/powerpc/platforms/ps3/mm.c
@@ -607,8 +607,8 @@ static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr,
 				       r->ioid,
 				       iopte_flag);
 		if (result) {
-			pr_warning("%s:%d: lv1_put_iopte failed: %s\n",
-				   __func__, __LINE__, ps3_result(result));
+			pr_warn("%s:%d: lv1_put_iopte failed: %s\n",
+				__func__, __LINE__, ps3_result(result));
 			goto fail_map;
 		}
 		DBG("%s: pg=%d bus=%#lx, lpar=%#lx, ioid=%#x\n", __func__,
diff --git a/arch/powerpc/platforms/ps3/os-area.c b/arch/powerpc/platforms/ps3/os-area.c
index 3db53e8aff92..cdbfc5cfd6f3 100644
--- a/arch/powerpc/platforms/ps3/os-area.c
+++ b/arch/powerpc/platforms/ps3/os-area.c
@@ -699,7 +699,7 @@ static void os_area_queue_work_handler(struct work_struct *work)
 
 	error = update_flash_db();
 	if (error)
-		pr_warning("%s: Could not update FLASH ROM\n", __func__);
+		pr_warn("%s: Could not update FLASH ROM\n", __func__);
 
 	pr_debug(" <- %s:%d\n", __func__, __LINE__);
 }
diff --git a/arch/powerpc/platforms/ps3/setup.c b/arch/powerpc/platforms/ps3/setup.c
index 6244bc849469..77a37520068d 100644
--- a/arch/powerpc/platforms/ps3/setup.c
+++ b/arch/powerpc/platforms/ps3/setup.c
@@ -113,6 +113,7 @@ static void ps3_panic(char *str)
 	printk("   System does not reboot automatically.\n");
 	printk("   Please press POWER button.\n");
 	printk("\n");
+	panic_flush_kmsg_end();
 
 	while(1)
 		lv1_pause(1);
diff --git a/arch/powerpc/platforms/pseries/cmm.c b/arch/powerpc/platforms/pseries/cmm.c
index 560aefde06c0..25427a48feae 100644
--- a/arch/powerpc/platforms/pseries/cmm.c
+++ b/arch/powerpc/platforms/pseries/cmm.c
@@ -72,20 +72,20 @@ MODULE_DESCRIPTION("IBM System p Collaborative Memory Manager");
 MODULE_LICENSE("GPL");
 MODULE_VERSION(CMM_DRIVER_VERSION);
 
-module_param_named(delay, delay, uint, S_IRUGO | S_IWUSR);
+module_param_named(delay, delay, uint, 0644);
 MODULE_PARM_DESC(delay, "Delay (in seconds) between polls to query hypervisor paging requests. "
 		 "[Default=" __stringify(CMM_DEFAULT_DELAY) "]");
-module_param_named(hotplug_delay, hotplug_delay, uint, S_IRUGO | S_IWUSR);
+module_param_named(hotplug_delay, hotplug_delay, uint, 0644);
 MODULE_PARM_DESC(hotplug_delay, "Delay (in seconds) after memory hotplug remove "
 		 "before loaning resumes. "
 		 "[Default=" __stringify(CMM_HOTPLUG_DELAY) "]");
-module_param_named(oom_kb, oom_kb, uint, S_IRUGO | S_IWUSR);
+module_param_named(oom_kb, oom_kb, uint, 0644);
 MODULE_PARM_DESC(oom_kb, "Amount of memory in kb to free on OOM. "
 		 "[Default=" __stringify(CMM_OOM_KB) "]");
-module_param_named(min_mem_mb, min_mem_mb, ulong, S_IRUGO | S_IWUSR);
+module_param_named(min_mem_mb, min_mem_mb, ulong, 0644);
 MODULE_PARM_DESC(min_mem_mb, "Minimum amount of memory (in MB) to not balloon. "
 		 "[Default=" __stringify(CMM_MIN_MEM_MB) "]");
-module_param_named(debug, cmm_debug, uint, S_IRUGO | S_IWUSR);
+module_param_named(debug, cmm_debug, uint, 0644);
 MODULE_PARM_DESC(debug, "Enable module debugging logging. Set to 1 to enable. "
 		 "[Default=" __stringify(CMM_DEBUG) "]");
 
@@ -385,7 +385,7 @@ static int cmm_thread(void *dummy)
 	{							\
 		return sprintf(buf, format, ##args);		\
 	}							\
-	static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
+	static DEVICE_ATTR(name, 0444, show_##name, NULL)
 
 CMM_SHOW(loaned_kb, "%lu\n", PAGES2KB(loaned_pages));
 CMM_SHOW(loaned_target_kb, "%lu\n", PAGES2KB(loaned_pages_target));
@@ -411,7 +411,7 @@ static ssize_t store_oom_pages(struct device *dev,
 	return count;
 }
 
-static DEVICE_ATTR(oom_freed_kb, S_IWUSR | S_IRUGO,
+static DEVICE_ATTR(oom_freed_kb, 0644,
 		   show_oom_pages, store_oom_pages);
 
 static struct device_attribute *cmm_attrs[] = {
@@ -765,7 +765,7 @@ static int cmm_set_disable(const char *val, const struct kernel_param *kp)
 }
 
 module_param_call(disable, cmm_set_disable, param_get_uint,
-		  &cmm_disabled, S_IRUGO | S_IWUSR);
+		  &cmm_disabled, 0644);
 MODULE_PARM_DESC(disable, "Disable CMM. Set to 1 to disable. "
 		 "[Default=" __stringify(CMM_DISABLE) "]");
 
diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c
index 6b812ad990e4..823cb27efa8b 100644
--- a/arch/powerpc/platforms/pseries/eeh_pseries.c
+++ b/arch/powerpc/platforms/pseries/eeh_pseries.c
@@ -55,6 +55,43 @@ static int ibm_get_config_addr_info;
 static int ibm_get_config_addr_info2;
 static int ibm_configure_pe;
 
+#ifdef CONFIG_PCI_IOV
+void pseries_pcibios_bus_add_device(struct pci_dev *pdev)
+{
+	struct pci_dn *pdn = pci_get_pdn(pdev);
+	struct pci_dn *physfn_pdn;
+	struct eeh_dev *edev;
+
+	if (!pdev->is_virtfn)
+		return;
+
+	pdn->device_id  =  pdev->device;
+	pdn->vendor_id  =  pdev->vendor;
+	pdn->class_code =  pdev->class;
+	/*
+	 * Last allow unfreeze return code used for retrieval
+	 * by user space in eeh-sysfs to show the last command
+	 * completion from platform.
+	 */
+	pdn->last_allow_rc =  0;
+	physfn_pdn      =  pci_get_pdn(pdev->physfn);
+	pdn->pe_number  =  physfn_pdn->pe_num_map[pdn->vf_index];
+	edev = pdn_to_eeh_dev(pdn);
+
+	/*
+	 * The following operations will fail if VF's sysfs files
+	 * aren't created or its resources aren't finalized.
+	 */
+	eeh_add_device_early(pdn);
+	eeh_add_device_late(pdev);
+	edev->pe_config_addr =  (pdn->busno << 16) | (pdn->devfn << 8);
+	eeh_rmv_from_parent_pe(edev); /* Remove as it is adding to bus pe */
+	eeh_add_to_parent_pe(edev);   /* Add as VF PE type */
+	eeh_sysfs_add_device(pdev);
+
+}
+#endif
+
 /*
  * Buffer for reporting slot-error-detail rtas calls. Its here
  * in BSS, and not dynamically alloced, so that it ends up in
@@ -120,6 +157,11 @@ static int pseries_eeh_init(void)
 	/* Set EEH probe mode */
 	eeh_add_flag(EEH_PROBE_MODE_DEVTREE | EEH_ENABLE_IO_FOR_LOG);
 
+#ifdef CONFIG_PCI_IOV
+	/* Set EEH machine dependent code */
+	ppc_md.pcibios_bus_add_device = pseries_pcibios_bus_add_device;
+#endif
+
 	return 0;
 }
 
@@ -684,6 +726,121 @@ static int pseries_eeh_write_config(struct pci_dn *pdn, int where, int size, u32
 	return rtas_write_config(pdn, where, size, val);
 }
 
+static int pseries_eeh_restore_config(struct pci_dn *pdn)
+{
+	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
+	s64 ret = 0;
+
+	if (!edev)
+		return -EEXIST;
+
+	/*
+	 * FIXME: The MPS, error routing rules, timeout setting are worthy
+	 * to be exported by firmware in extendible way.
+	 */
+	if (edev->physfn)
+		ret = eeh_restore_vf_config(pdn);
+
+	if (ret) {
+		pr_warn("%s: Can't reinit PCI dev 0x%x (%lld)\n",
+			__func__, edev->pe_config_addr, ret);
+		return -EIO;
+	}
+
+	return ret;
+}
+
+#ifdef CONFIG_PCI_IOV
+int pseries_send_allow_unfreeze(struct pci_dn *pdn,
+				u16 *vf_pe_array, int cur_vfs)
+{
+	int rc;
+	int ibm_allow_unfreeze = rtas_token("ibm,open-sriov-allow-unfreeze");
+	unsigned long buid, addr;
+
+	addr = rtas_config_addr(pdn->busno, pdn->devfn, 0);
+	buid = pdn->phb->buid;
+	spin_lock(&rtas_data_buf_lock);
+	memcpy(rtas_data_buf, vf_pe_array, RTAS_DATA_BUF_SIZE);
+	rc = rtas_call(ibm_allow_unfreeze, 5, 1, NULL,
+		       addr,
+		       BUID_HI(buid),
+		       BUID_LO(buid),
+		       rtas_data_buf, cur_vfs * sizeof(u16));
+	spin_unlock(&rtas_data_buf_lock);
+	if (rc)
+		pr_warn("%s: Failed to allow unfreeze for PHB#%x-PE#%lx, rc=%x\n",
+			__func__,
+			pdn->phb->global_number, addr, rc);
+	return rc;
+}
+
+static int pseries_call_allow_unfreeze(struct eeh_dev *edev)
+{
+	struct pci_dn *pdn, *tmp, *parent, *physfn_pdn;
+	int cur_vfs = 0, rc = 0, vf_index, bus, devfn;
+	u16 *vf_pe_array;
+
+	vf_pe_array = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL);
+	if (!vf_pe_array)
+		return -ENOMEM;
+	if (pci_num_vf(edev->physfn ? edev->physfn : edev->pdev)) {
+		if (edev->pdev->is_physfn) {
+			cur_vfs = pci_num_vf(edev->pdev);
+			pdn = eeh_dev_to_pdn(edev);
+			parent = pdn->parent;
+			for (vf_index = 0; vf_index < cur_vfs; vf_index++)
+				vf_pe_array[vf_index] =
+					cpu_to_be16(pdn->pe_num_map[vf_index]);
+			rc = pseries_send_allow_unfreeze(pdn, vf_pe_array,
+							 cur_vfs);
+			pdn->last_allow_rc = rc;
+			for (vf_index = 0; vf_index < cur_vfs; vf_index++) {
+				list_for_each_entry_safe(pdn, tmp,
+							 &parent->child_list,
+							 list) {
+					bus = pci_iov_virtfn_bus(edev->pdev,
+								 vf_index);
+					devfn = pci_iov_virtfn_devfn(edev->pdev,
+								     vf_index);
+					if (pdn->busno != bus ||
+					    pdn->devfn != devfn)
+						continue;
+					pdn->last_allow_rc = rc;
+				}
+			}
+		} else {
+			pdn = pci_get_pdn(edev->pdev);
+			vf_pe_array[0] = cpu_to_be16(pdn->pe_number);
+			physfn_pdn = pci_get_pdn(edev->physfn);
+			rc = pseries_send_allow_unfreeze(physfn_pdn,
+							 vf_pe_array, 1);
+			pdn->last_allow_rc = rc;
+		}
+	}
+
+	kfree(vf_pe_array);
+	return rc;
+}
+
+static int pseries_notify_resume(struct pci_dn *pdn)
+{
+	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
+
+	if (!edev)
+		return -EEXIST;
+
+	if (rtas_token("ibm,open-sriov-allow-unfreeze")
+	    == RTAS_UNKNOWN_SERVICE)
+		return -EINVAL;
+
+	if (edev->pdev->is_physfn || edev->pdev->is_virtfn)
+		return pseries_call_allow_unfreeze(edev);
+
+	return 0;
+}
+#endif
+
 static struct eeh_ops pseries_eeh_ops = {
 	.name			= "pseries",
 	.init			= pseries_eeh_init,
@@ -699,7 +856,10 @@ static struct eeh_ops pseries_eeh_ops = {
 	.read_config		= pseries_eeh_read_config,
 	.write_config		= pseries_eeh_write_config,
 	.next_error		= NULL,
-	.restore_config		= NULL
+	.restore_config		= pseries_eeh_restore_config,
+#ifdef CONFIG_PCI_IOV
+	.notify_resume		= pseries_notify_resume
+#endif
 };
 
 /**
diff --git a/arch/powerpc/platforms/pseries/firmware.c b/arch/powerpc/platforms/pseries/firmware.c
index 63cc82ad58ac..a3bbeb43689e 100644
--- a/arch/powerpc/platforms/pseries/firmware.c
+++ b/arch/powerpc/platforms/pseries/firmware.c
@@ -114,6 +114,8 @@ static __initdata struct vec5_fw_feature
 vec5_fw_features_table[] = {
 	{FW_FEATURE_TYPE1_AFFINITY,	OV5_TYPE1_AFFINITY},
 	{FW_FEATURE_PRRN,		OV5_PRRN},
+	{FW_FEATURE_DRMEM_V2,		OV5_DRMEM_V2},
+	{FW_FEATURE_DRC_INFO,		OV5_DRC_INFO},
 };
 
 static void __init fw_vec5_feature_init(const char *vec5, unsigned long len)
diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c
index a7d14aa7bb7c..652d3e96b812 100644
--- a/arch/powerpc/platforms/pseries/hotplug-cpu.c
+++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c
@@ -36,6 +36,7 @@
 #include <asm/xics.h>
 #include <asm/xive.h>
 #include <asm/plpar_wrappers.h>
+#include <asm/topology.h>
 
 #include "pseries.h"
 #include "offline_states.h"
@@ -331,6 +332,7 @@ static void pseries_remove_processor(struct device_node *np)
 			BUG_ON(cpu_online(cpu));
 			set_cpu_present(cpu, false);
 			set_hard_smp_processor_id(cpu, -1);
+			update_numa_cpu_lookup_table(cpu, -1);
 			break;
 		}
 		if (cpu >= nr_cpu_ids)
@@ -364,6 +366,7 @@ static int dlpar_online_cpu(struct device_node *dn)
 					!= CPU_STATE_OFFLINE);
 			cpu_maps_update_done();
 			timed_topology_update(1);
+			find_and_online_cpu_nid(cpu);
 			rc = device_online(get_cpu_device(cpu));
 			if (rc)
 				goto out;
diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index 1d48ab424bd9..c1578f54c626 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c
@@ -23,6 +23,7 @@
 #include <asm/prom.h>
 #include <asm/sparsemem.h>
 #include <asm/fadump.h>
+#include <asm/drmem.h>
 #include "pseries.h"
 
 static bool rtas_hp_event;
@@ -100,100 +101,6 @@ static struct property *dlpar_clone_property(struct property *prop,
 	return new_prop;
 }
 
-static struct property *dlpar_clone_drconf_property(struct device_node *dn)
-{
-	struct property *prop, *new_prop;
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
-	int i;
-
-	prop = of_find_property(dn, "ibm,dynamic-memory", NULL);
-	if (!prop)
-		return NULL;
-
-	new_prop = dlpar_clone_property(prop, prop->length);
-	if (!new_prop)
-		return NULL;
-
-	/* Convert the property to cpu endian-ness */
-	p = new_prop->value;
-	*p = be32_to_cpu(*p);
-
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-
-	for (i = 0; i < num_lmbs; i++) {
-		lmbs[i].base_addr = be64_to_cpu(lmbs[i].base_addr);
-		lmbs[i].drc_index = be32_to_cpu(lmbs[i].drc_index);
-		lmbs[i].aa_index = be32_to_cpu(lmbs[i].aa_index);
-		lmbs[i].flags = be32_to_cpu(lmbs[i].flags);
-	}
-
-	return new_prop;
-}
-
-static void dlpar_update_drconf_property(struct device_node *dn,
-					 struct property *prop)
-{
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
-	int i;
-
-	/* Convert the property back to BE */
-	p = prop->value;
-	num_lmbs = *p;
-	*p = cpu_to_be32(*p);
-	p++;
-
-	lmbs = (struct of_drconf_cell *)p;
-	for (i = 0; i < num_lmbs; i++) {
-		lmbs[i].base_addr = cpu_to_be64(lmbs[i].base_addr);
-		lmbs[i].drc_index = cpu_to_be32(lmbs[i].drc_index);
-		lmbs[i].aa_index = cpu_to_be32(lmbs[i].aa_index);
-		lmbs[i].flags = cpu_to_be32(lmbs[i].flags);
-	}
-
-	rtas_hp_event = true;
-	of_update_property(dn, prop);
-	rtas_hp_event = false;
-}
-
-static int dlpar_update_device_tree_lmb(struct of_drconf_cell *lmb)
-{
-	struct device_node *dn;
-	struct property *prop;
-	struct of_drconf_cell *lmbs;
-	u32 *p, num_lmbs;
-	int i;
-
-	dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
-	if (!dn)
-		return -ENODEV;
-
-	prop = dlpar_clone_drconf_property(dn);
-	if (!prop) {
-		of_node_put(dn);
-		return -ENODEV;
-	}
-
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-
-	for (i = 0; i < num_lmbs; i++) {
-		if (lmbs[i].drc_index == lmb->drc_index) {
-			lmbs[i].flags = lmb->flags;
-			lmbs[i].aa_index = lmb->aa_index;
-
-			dlpar_update_drconf_property(dn, prop);
-			break;
-		}
-	}
-
-	of_node_put(dn);
-	return 0;
-}
-
 static u32 find_aa_index(struct device_node *dr_node,
 			 struct property *ala_prop, const u32 *lmb_assoc)
 {
@@ -256,7 +163,7 @@ static u32 find_aa_index(struct device_node *dr_node,
 	return aa_index;
 }
 
-static u32 lookup_lmb_associativity_index(struct of_drconf_cell *lmb)
+static u32 lookup_lmb_associativity_index(struct drmem_lmb *lmb)
 {
 	struct device_node *parent, *lmb_node, *dr_node;
 	struct property *ala_prop;
@@ -299,9 +206,9 @@ static u32 lookup_lmb_associativity_index(struct of_drconf_cell *lmb)
 	return aa_index;
 }
 
-static int dlpar_add_device_tree_lmb(struct of_drconf_cell *lmb)
+static int dlpar_add_device_tree_lmb(struct drmem_lmb *lmb)
 {
-	int aa_index;
+	int rc, aa_index;
 
 	lmb->flags |= DRCONF_MEM_ASSIGNED;
 
@@ -313,17 +220,29 @@ static int dlpar_add_device_tree_lmb(struct of_drconf_cell *lmb)
 	}
 
 	lmb->aa_index = aa_index;
-	return dlpar_update_device_tree_lmb(lmb);
+
+	rtas_hp_event = true;
+	rc = drmem_update_dt();
+	rtas_hp_event = false;
+
+	return rc;
 }
 
-static int dlpar_remove_device_tree_lmb(struct of_drconf_cell *lmb)
+static int dlpar_remove_device_tree_lmb(struct drmem_lmb *lmb)
 {
+	int rc;
+
 	lmb->flags &= ~DRCONF_MEM_ASSIGNED;
 	lmb->aa_index = 0xffffffff;
-	return dlpar_update_device_tree_lmb(lmb);
+
+	rtas_hp_event = true;
+	rc = drmem_update_dt();
+	rtas_hp_event = false;
+
+	return rc;
 }
 
-static struct memory_block *lmb_to_memblock(struct of_drconf_cell *lmb)
+static struct memory_block *lmb_to_memblock(struct drmem_lmb *lmb)
 {
 	unsigned long section_nr;
 	struct mem_section *mem_sect;
@@ -336,7 +255,36 @@ static struct memory_block *lmb_to_memblock(struct of_drconf_cell *lmb)
 	return mem_block;
 }
 
-static int dlpar_change_lmb_state(struct of_drconf_cell *lmb, bool online)
+static int get_lmb_range(u32 drc_index, int n_lmbs,
+			 struct drmem_lmb **start_lmb,
+			 struct drmem_lmb **end_lmb)
+{
+	struct drmem_lmb *lmb, *start, *end;
+	struct drmem_lmb *last_lmb;
+
+	start = NULL;
+	for_each_drmem_lmb(lmb) {
+		if (lmb->drc_index == drc_index) {
+			start = lmb;
+			break;
+		}
+	}
+
+	if (!start)
+		return -EINVAL;
+
+	end = &start[n_lmbs - 1];
+
+	last_lmb = &drmem_info->lmbs[drmem_info->n_lmbs - 1];
+	if (end > last_lmb)
+		return -EINVAL;
+
+	*start_lmb = start;
+	*end_lmb = end;
+	return 0;
+}
+
+static int dlpar_change_lmb_state(struct drmem_lmb *lmb, bool online)
 {
 	struct memory_block *mem_block;
 	int rc;
@@ -357,13 +305,13 @@ static int dlpar_change_lmb_state(struct of_drconf_cell *lmb, bool online)
 	return rc;
 }
 
-static int dlpar_online_lmb(struct of_drconf_cell *lmb)
+static int dlpar_online_lmb(struct drmem_lmb *lmb)
 {
 	return dlpar_change_lmb_state(lmb, true);
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
-static int dlpar_offline_lmb(struct of_drconf_cell *lmb)
+static int dlpar_offline_lmb(struct drmem_lmb *lmb)
 {
 	return dlpar_change_lmb_state(lmb, false);
 }
@@ -426,7 +374,7 @@ static int pseries_remove_mem_node(struct device_node *np)
 	return 0;
 }
 
-static bool lmb_is_removable(struct of_drconf_cell *lmb)
+static bool lmb_is_removable(struct drmem_lmb *lmb)
 {
 	int i, scns_per_block;
 	int rc = 1;
@@ -458,9 +406,9 @@ static bool lmb_is_removable(struct of_drconf_cell *lmb)
 	return rc ? true : false;
 }
 
-static int dlpar_add_lmb(struct of_drconf_cell *);
+static int dlpar_add_lmb(struct drmem_lmb *);
 
-static int dlpar_remove_lmb(struct of_drconf_cell *lmb)
+static int dlpar_remove_lmb(struct drmem_lmb *lmb)
 {
 	unsigned long block_sz;
 	int nid, rc;
@@ -484,28 +432,25 @@ static int dlpar_remove_lmb(struct of_drconf_cell *lmb)
 	return 0;
 }
 
-static int dlpar_memory_remove_by_count(u32 lmbs_to_remove,
-					struct property *prop)
+static int dlpar_memory_remove_by_count(u32 lmbs_to_remove)
 {
-	struct of_drconf_cell *lmbs;
+	struct drmem_lmb *lmb;
 	int lmbs_removed = 0;
 	int lmbs_available = 0;
-	u32 num_lmbs, *p;
-	int i, rc;
+	int rc;
 
 	pr_info("Attempting to hot-remove %d LMB(s)\n", lmbs_to_remove);
 
 	if (lmbs_to_remove == 0)
 		return -EINVAL;
 
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-
 	/* Validate that there are enough LMBs to satisfy the request */
-	for (i = 0; i < num_lmbs; i++) {
-		if (lmb_is_removable(&lmbs[i]))
+	for_each_drmem_lmb(lmb) {
+		if (lmb_is_removable(lmb))
 			lmbs_available++;
+
+		if (lmbs_available == lmbs_to_remove)
+			break;
 	}
 
 	if (lmbs_available < lmbs_to_remove) {
@@ -514,45 +459,47 @@ static int dlpar_memory_remove_by_count(u32 lmbs_to_remove,
 		return -EINVAL;
 	}
 
-	for (i = 0; i < num_lmbs && lmbs_removed < lmbs_to_remove; i++) {
-		rc = dlpar_remove_lmb(&lmbs[i]);
+	for_each_drmem_lmb(lmb) {
+		rc = dlpar_remove_lmb(lmb);
 		if (rc)
 			continue;
 
-		lmbs_removed++;
-
 		/* Mark this lmb so we can add it later if all of the
 		 * requested LMBs cannot be removed.
 		 */
-		lmbs[i].reserved = 1;
+		drmem_mark_lmb_reserved(lmb);
+
+		lmbs_removed++;
+		if (lmbs_removed == lmbs_to_remove)
+			break;
 	}
 
 	if (lmbs_removed != lmbs_to_remove) {
 		pr_err("Memory hot-remove failed, adding LMB's back\n");
 
-		for (i = 0; i < num_lmbs; i++) {
-			if (!lmbs[i].reserved)
+		for_each_drmem_lmb(lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
-			rc = dlpar_add_lmb(&lmbs[i]);
+			rc = dlpar_add_lmb(lmb);
 			if (rc)
 				pr_err("Failed to add LMB back, drc index %x\n",
-				       lmbs[i].drc_index);
+				       lmb->drc_index);
 
-			lmbs[i].reserved = 0;
+			drmem_remove_lmb_reservation(lmb);
 		}
 
 		rc = -EINVAL;
 	} else {
-		for (i = 0; i < num_lmbs; i++) {
-			if (!lmbs[i].reserved)
+		for_each_drmem_lmb(lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
-			dlpar_release_drc(lmbs[i].drc_index);
+			dlpar_release_drc(lmb->drc_index);
 			pr_info("Memory at %llx was hot-removed\n",
-				lmbs[i].base_addr);
+				lmb->base_addr);
 
-			lmbs[i].reserved = 0;
+			drmem_remove_lmb_reservation(lmb);
 		}
 		rc = 0;
 	}
@@ -560,26 +507,21 @@ static int dlpar_memory_remove_by_count(u32 lmbs_to_remove,
 	return rc;
 }
 
-static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop)
+static int dlpar_memory_remove_by_index(u32 drc_index)
 {
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
+	struct drmem_lmb *lmb;
 	int lmb_found;
-	int i, rc;
+	int rc;
 
 	pr_info("Attempting to hot-remove LMB, drc index %x\n", drc_index);
 
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-
 	lmb_found = 0;
-	for (i = 0; i < num_lmbs; i++) {
-		if (lmbs[i].drc_index == drc_index) {
+	for_each_drmem_lmb(lmb) {
+		if (lmb->drc_index == drc_index) {
 			lmb_found = 1;
-			rc = dlpar_remove_lmb(&lmbs[i]);
+			rc = dlpar_remove_lmb(lmb);
 			if (!rc)
-				dlpar_release_drc(lmbs[i].drc_index);
+				dlpar_release_drc(lmb->drc_index);
 
 			break;
 		}
@@ -590,35 +532,30 @@ static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop)
 
 	if (rc)
 		pr_info("Failed to hot-remove memory at %llx\n",
-			lmbs[i].base_addr);
+			lmb->base_addr);
 	else
-		pr_info("Memory at %llx was hot-removed\n", lmbs[i].base_addr);
+		pr_info("Memory at %llx was hot-removed\n", lmb->base_addr);
 
 	return rc;
 }
 
-static int dlpar_memory_readd_by_index(u32 drc_index, struct property *prop)
+static int dlpar_memory_readd_by_index(u32 drc_index)
 {
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
+	struct drmem_lmb *lmb;
 	int lmb_found;
-	int i, rc;
+	int rc;
 
 	pr_info("Attempting to update LMB, drc index %x\n", drc_index);
 
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-
 	lmb_found = 0;
-	for (i = 0; i < num_lmbs; i++) {
-		if (lmbs[i].drc_index == drc_index) {
+	for_each_drmem_lmb(lmb) {
+		if (lmb->drc_index == drc_index) {
 			lmb_found = 1;
-			rc = dlpar_remove_lmb(&lmbs[i]);
+			rc = dlpar_remove_lmb(lmb);
 			if (!rc) {
-				rc = dlpar_add_lmb(&lmbs[i]);
+				rc = dlpar_add_lmb(lmb);
 				if (rc)
-					dlpar_release_drc(lmbs[i].drc_index);
+					dlpar_release_drc(lmb->drc_index);
 			}
 			break;
 		}
@@ -629,20 +566,18 @@ static int dlpar_memory_readd_by_index(u32 drc_index, struct property *prop)
 
 	if (rc)
 		pr_info("Failed to update memory at %llx\n",
-			lmbs[i].base_addr);
+			lmb->base_addr);
 	else
-		pr_info("Memory at %llx was updated\n", lmbs[i].base_addr);
+		pr_info("Memory at %llx was updated\n", lmb->base_addr);
 
 	return rc;
 }
 
-static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index,
-				     struct property *prop)
+static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index)
 {
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
-	int i, rc, start_lmb_found;
-	int lmbs_available = 0, start_index = 0, end_index;
+	struct drmem_lmb *lmb, *start_lmb, *end_lmb;
+	int lmbs_available = 0;
+	int rc;
 
 	pr_info("Attempting to hot-remove %u LMB(s) at %x\n",
 		lmbs_to_remove, drc_index);
@@ -650,29 +585,13 @@ static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index,
 	if (lmbs_to_remove == 0)
 		return -EINVAL;
 
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-	start_lmb_found = 0;
-
-	/* Navigate to drc_index */
-	while (start_index < num_lmbs) {
-		if (lmbs[start_index].drc_index == drc_index) {
-			start_lmb_found = 1;
-			break;
-		}
-
-		start_index++;
-	}
-
-	if (!start_lmb_found)
+	rc = get_lmb_range(drc_index, lmbs_to_remove, &start_lmb, &end_lmb);
+	if (rc)
 		return -EINVAL;
 
-	end_index = start_index + lmbs_to_remove;
-
 	/* Validate that there are enough LMBs to satisfy the request */
-	for (i = start_index; i < end_index; i++) {
-		if (lmbs[i].flags & DRCONF_MEM_RESERVED)
+	for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+		if (lmb->flags & DRCONF_MEM_RESERVED)
 			break;
 
 		lmbs_available++;
@@ -681,42 +600,43 @@ static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index,
 	if (lmbs_available < lmbs_to_remove)
 		return -EINVAL;
 
-	for (i = start_index; i < end_index; i++) {
-		if (!(lmbs[i].flags & DRCONF_MEM_ASSIGNED))
+	for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+		if (!(lmb->flags & DRCONF_MEM_ASSIGNED))
 			continue;
 
-		rc = dlpar_remove_lmb(&lmbs[i]);
+		rc = dlpar_remove_lmb(lmb);
 		if (rc)
 			break;
 
-		lmbs[i].reserved = 1;
+		drmem_mark_lmb_reserved(lmb);
 	}
 
 	if (rc) {
 		pr_err("Memory indexed-count-remove failed, adding any removed LMBs\n");
 
-		for (i = start_index; i < end_index; i++) {
-			if (!lmbs[i].reserved)
+
+		for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
-			rc = dlpar_add_lmb(&lmbs[i]);
+			rc = dlpar_add_lmb(lmb);
 			if (rc)
 				pr_err("Failed to add LMB, drc index %x\n",
-				       be32_to_cpu(lmbs[i].drc_index));
+				       lmb->drc_index);
 
-			lmbs[i].reserved = 0;
+			drmem_remove_lmb_reservation(lmb);
 		}
 		rc = -EINVAL;
 	} else {
-		for (i = start_index; i < end_index; i++) {
-			if (!lmbs[i].reserved)
+		for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
-			dlpar_release_drc(lmbs[i].drc_index);
+			dlpar_release_drc(lmb->drc_index);
 			pr_info("Memory at %llx (drc index %x) was hot-removed\n",
-				lmbs[i].base_addr, lmbs[i].drc_index);
+				lmb->base_addr, lmb->drc_index);
 
-			lmbs[i].reserved = 0;
+			drmem_remove_lmb_reservation(lmb);
 		}
 	}
 
@@ -737,32 +657,30 @@ static inline int dlpar_memory_remove(struct pseries_hp_errorlog *hp_elog)
 {
 	return -EOPNOTSUPP;
 }
-static int dlpar_remove_lmb(struct of_drconf_cell *lmb)
+static int dlpar_remove_lmb(struct drmem_lmb *lmb)
 {
 	return -EOPNOTSUPP;
 }
-static int dlpar_memory_remove_by_count(u32 lmbs_to_remove,
-					struct property *prop)
+static int dlpar_memory_remove_by_count(u32 lmbs_to_remove)
 {
 	return -EOPNOTSUPP;
 }
-static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop)
+static int dlpar_memory_remove_by_index(u32 drc_index)
 {
 	return -EOPNOTSUPP;
 }
-static int dlpar_memory_readd_by_index(u32 drc_index, struct property *prop)
+static int dlpar_memory_readd_by_index(u32 drc_index)
 {
 	return -EOPNOTSUPP;
 }
 
-static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index,
-				     struct property *prop)
+static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index)
 {
 	return -EOPNOTSUPP;
 }
 #endif /* CONFIG_MEMORY_HOTREMOVE */
 
-static int dlpar_add_lmb(struct of_drconf_cell *lmb)
+static int dlpar_add_lmb(struct drmem_lmb *lmb)
 {
 	unsigned long block_sz;
 	int nid, rc;
@@ -801,77 +719,79 @@ static int dlpar_add_lmb(struct of_drconf_cell *lmb)
 	return rc;
 }
 
-static int dlpar_memory_add_by_count(u32 lmbs_to_add, struct property *prop)
+static int dlpar_memory_add_by_count(u32 lmbs_to_add)
 {
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
+	struct drmem_lmb *lmb;
 	int lmbs_available = 0;
 	int lmbs_added = 0;
-	int i, rc;
+	int rc;
 
 	pr_info("Attempting to hot-add %d LMB(s)\n", lmbs_to_add);
 
 	if (lmbs_to_add == 0)
 		return -EINVAL;
 
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-
 	/* Validate that there are enough LMBs to satisfy the request */
-	for (i = 0; i < num_lmbs; i++) {
-		if (!(lmbs[i].flags & DRCONF_MEM_ASSIGNED))
+	for_each_drmem_lmb(lmb) {
+		if (!(lmb->flags & DRCONF_MEM_ASSIGNED))
 			lmbs_available++;
+
+		if (lmbs_available == lmbs_to_add)
+			break;
 	}
 
 	if (lmbs_available < lmbs_to_add)
 		return -EINVAL;
 
-	for (i = 0; i < num_lmbs && lmbs_to_add != lmbs_added; i++) {
-		if (lmbs[i].flags & DRCONF_MEM_ASSIGNED)
+	for_each_drmem_lmb(lmb) {
+		if (lmb->flags & DRCONF_MEM_ASSIGNED)
 			continue;
 
-		rc = dlpar_acquire_drc(lmbs[i].drc_index);
+		rc = dlpar_acquire_drc(lmb->drc_index);
 		if (rc)
 			continue;
 
-		rc = dlpar_add_lmb(&lmbs[i]);
+		rc = dlpar_add_lmb(lmb);
 		if (rc) {
-			dlpar_release_drc(lmbs[i].drc_index);
+			dlpar_release_drc(lmb->drc_index);
 			continue;
 		}
 
-		lmbs_added++;
-
 		/* Mark this lmb so we can remove it later if all of the
 		 * requested LMBs cannot be added.
 		 */
-		lmbs[i].reserved = 1;
+		drmem_mark_lmb_reserved(lmb);
+
+		lmbs_added++;
+		if (lmbs_added == lmbs_to_add)
+			break;
 	}
 
 	if (lmbs_added != lmbs_to_add) {
 		pr_err("Memory hot-add failed, removing any added LMBs\n");
 
-		for (i = 0; i < num_lmbs; i++) {
-			if (!lmbs[i].reserved)
+		for_each_drmem_lmb(lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
-			rc = dlpar_remove_lmb(&lmbs[i]);
+			rc = dlpar_remove_lmb(lmb);
 			if (rc)
 				pr_err("Failed to remove LMB, drc index %x\n",
-				       be32_to_cpu(lmbs[i].drc_index));
+				       lmb->drc_index);
 			else
-				dlpar_release_drc(lmbs[i].drc_index);
+				dlpar_release_drc(lmb->drc_index);
+
+			drmem_remove_lmb_reservation(lmb);
 		}
 		rc = -EINVAL;
 	} else {
-		for (i = 0; i < num_lmbs; i++) {
-			if (!lmbs[i].reserved)
+		for_each_drmem_lmb(lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
 			pr_info("Memory at %llx (drc index %x) was hot-added\n",
-				lmbs[i].base_addr, lmbs[i].drc_index);
-			lmbs[i].reserved = 0;
+				lmb->base_addr, lmb->drc_index);
+			drmem_remove_lmb_reservation(lmb);
 		}
 		rc = 0;
 	}
@@ -879,28 +799,22 @@ static int dlpar_memory_add_by_count(u32 lmbs_to_add, struct property *prop)
 	return rc;
 }
 
-static int dlpar_memory_add_by_index(u32 drc_index, struct property *prop)
+static int dlpar_memory_add_by_index(u32 drc_index)
 {
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
-	int i, lmb_found;
-	int rc;
+	struct drmem_lmb *lmb;
+	int rc, lmb_found;
 
 	pr_info("Attempting to hot-add LMB, drc index %x\n", drc_index);
 
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-
 	lmb_found = 0;
-	for (i = 0; i < num_lmbs; i++) {
-		if (lmbs[i].drc_index == drc_index) {
+	for_each_drmem_lmb(lmb) {
+		if (lmb->drc_index == drc_index) {
 			lmb_found = 1;
-			rc = dlpar_acquire_drc(lmbs[i].drc_index);
+			rc = dlpar_acquire_drc(lmb->drc_index);
 			if (!rc) {
-				rc = dlpar_add_lmb(&lmbs[i]);
+				rc = dlpar_add_lmb(lmb);
 				if (rc)
-					dlpar_release_drc(lmbs[i].drc_index);
+					dlpar_release_drc(lmb->drc_index);
 			}
 
 			break;
@@ -914,18 +828,16 @@ static int dlpar_memory_add_by_index(u32 drc_index, struct property *prop)
 		pr_info("Failed to hot-add memory, drc index %x\n", drc_index);
 	else
 		pr_info("Memory at %llx (drc index %x) was hot-added\n",
-			lmbs[i].base_addr, drc_index);
+			lmb->base_addr, drc_index);
 
 	return rc;
 }
 
-static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index,
-				  struct property *prop)
+static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index)
 {
-	struct of_drconf_cell *lmbs;
-	u32 num_lmbs, *p;
-	int i, rc, start_lmb_found;
-	int lmbs_available = 0, start_index = 0, end_index;
+	struct drmem_lmb *lmb, *start_lmb, *end_lmb;
+	int lmbs_available = 0;
+	int rc;
 
 	pr_info("Attempting to hot-add %u LMB(s) at index %x\n",
 		lmbs_to_add, drc_index);
@@ -933,29 +845,13 @@ static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index,
 	if (lmbs_to_add == 0)
 		return -EINVAL;
 
-	p = prop->value;
-	num_lmbs = *p++;
-	lmbs = (struct of_drconf_cell *)p;
-	start_lmb_found = 0;
-
-	/* Navigate to drc_index */
-	while (start_index < num_lmbs) {
-		if (lmbs[start_index].drc_index == drc_index) {
-			start_lmb_found = 1;
-			break;
-		}
-
-		start_index++;
-	}
-
-	if (!start_lmb_found)
+	rc = get_lmb_range(drc_index, lmbs_to_add, &start_lmb, &end_lmb);
+	if (rc)
 		return -EINVAL;
 
-	end_index = start_index + lmbs_to_add;
-
 	/* Validate that the LMBs in this range are not reserved */
-	for (i = start_index; i < end_index; i++) {
-		if (lmbs[i].flags & DRCONF_MEM_RESERVED)
+	for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+		if (lmb->flags & DRCONF_MEM_RESERVED)
 			break;
 
 		lmbs_available++;
@@ -964,46 +860,48 @@ static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index,
 	if (lmbs_available < lmbs_to_add)
 		return -EINVAL;
 
-	for (i = start_index; i < end_index; i++) {
-		if (lmbs[i].flags & DRCONF_MEM_ASSIGNED)
+	for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+		if (lmb->flags & DRCONF_MEM_ASSIGNED)
 			continue;
 
-		rc = dlpar_acquire_drc(lmbs[i].drc_index);
+		rc = dlpar_acquire_drc(lmb->drc_index);
 		if (rc)
 			break;
 
-		rc = dlpar_add_lmb(&lmbs[i]);
+		rc = dlpar_add_lmb(lmb);
 		if (rc) {
-			dlpar_release_drc(lmbs[i].drc_index);
+			dlpar_release_drc(lmb->drc_index);
 			break;
 		}
 
-		lmbs[i].reserved = 1;
+		drmem_mark_lmb_reserved(lmb);
 	}
 
 	if (rc) {
 		pr_err("Memory indexed-count-add failed, removing any added LMBs\n");
 
-		for (i = start_index; i < end_index; i++) {
-			if (!lmbs[i].reserved)
+		for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
-			rc = dlpar_remove_lmb(&lmbs[i]);
+			rc = dlpar_remove_lmb(lmb);
 			if (rc)
 				pr_err("Failed to remove LMB, drc index %x\n",
-				       be32_to_cpu(lmbs[i].drc_index));
+				       lmb->drc_index);
 			else
-				dlpar_release_drc(lmbs[i].drc_index);
+				dlpar_release_drc(lmb->drc_index);
+
+			drmem_remove_lmb_reservation(lmb);
 		}
 		rc = -EINVAL;
 	} else {
-		for (i = start_index; i < end_index; i++) {
-			if (!lmbs[i].reserved)
+		for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) {
+			if (!drmem_lmb_reserved(lmb))
 				continue;
 
 			pr_info("Memory at %llx (drc index %x) was hot-added\n",
-				lmbs[i].base_addr, lmbs[i].drc_index);
-			lmbs[i].reserved = 0;
+				lmb->base_addr, lmb->drc_index);
+			drmem_remove_lmb_reservation(lmb);
 		}
 	}
 
@@ -1012,37 +910,23 @@ static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index,
 
 int dlpar_memory(struct pseries_hp_errorlog *hp_elog)
 {
-	struct device_node *dn;
-	struct property *prop;
 	u32 count, drc_index;
 	int rc;
 
 	lock_device_hotplug();
 
-	dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory");
-	if (!dn) {
-		rc = -EINVAL;
-		goto dlpar_memory_out;
-	}
-
-	prop = dlpar_clone_drconf_property(dn);
-	if (!prop) {
-		rc = -EINVAL;
-		goto dlpar_memory_out;
-	}
-
 	switch (hp_elog->action) {
 	case PSERIES_HP_ELOG_ACTION_ADD:
 		if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT) {
 			count = hp_elog->_drc_u.drc_count;
-			rc = dlpar_memory_add_by_count(count, prop);
+			rc = dlpar_memory_add_by_count(count);
 		} else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) {
 			drc_index = hp_elog->_drc_u.drc_index;
-			rc = dlpar_memory_add_by_index(drc_index, prop);
+			rc = dlpar_memory_add_by_index(drc_index);
 		} else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_IC) {
 			count = hp_elog->_drc_u.ic.count;
 			drc_index = hp_elog->_drc_u.ic.index;
-			rc = dlpar_memory_add_by_ic(count, drc_index, prop);
+			rc = dlpar_memory_add_by_ic(count, drc_index);
 		} else {
 			rc = -EINVAL;
 		}
@@ -1051,14 +935,14 @@ int dlpar_memory(struct pseries_hp_errorlog *hp_elog)
 	case PSERIES_HP_ELOG_ACTION_REMOVE:
 		if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT) {
 			count = hp_elog->_drc_u.drc_count;
-			rc = dlpar_memory_remove_by_count(count, prop);
+			rc = dlpar_memory_remove_by_count(count);
 		} else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) {
 			drc_index = hp_elog->_drc_u.drc_index;
-			rc = dlpar_memory_remove_by_index(drc_index, prop);
+			rc = dlpar_memory_remove_by_index(drc_index);
 		} else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_IC) {
 			count = hp_elog->_drc_u.ic.count;
 			drc_index = hp_elog->_drc_u.ic.index;
-			rc = dlpar_memory_remove_by_ic(count, drc_index, prop);
+			rc = dlpar_memory_remove_by_ic(count, drc_index);
 		} else {
 			rc = -EINVAL;
 		}
@@ -1066,7 +950,7 @@ int dlpar_memory(struct pseries_hp_errorlog *hp_elog)
 		break;
 	case PSERIES_HP_ELOG_ACTION_READD:
 		drc_index = hp_elog->_drc_u.drc_index;
-		rc = dlpar_memory_readd_by_index(drc_index, prop);
+		rc = dlpar_memory_readd_by_index(drc_index);
 		break;
 	default:
 		pr_err("Invalid action (%d) specified\n", hp_elog->action);
@@ -1074,10 +958,6 @@ int dlpar_memory(struct pseries_hp_errorlog *hp_elog)
 		break;
 	}
 
-	dlpar_free_property(prop);
-
-dlpar_memory_out:
-	of_node_put(dn);
 	unlock_device_hotplug();
 	return rc;
 }
@@ -1116,7 +996,7 @@ static int pseries_add_mem_node(struct device_node *np)
 
 static int pseries_update_drconf_memory(struct of_reconfig_data *pr)
 {
-	struct of_drconf_cell *new_drmem, *old_drmem;
+	struct of_drconf_cell_v1 *new_drmem, *old_drmem;
 	unsigned long memblock_size;
 	u32 entries;
 	__be32 *p;
@@ -1139,11 +1019,11 @@ static int pseries_update_drconf_memory(struct of_reconfig_data *pr)
 	 * of_drconf_cell's.
 	 */
 	entries = be32_to_cpu(*p++);
-	old_drmem = (struct of_drconf_cell *)p;
+	old_drmem = (struct of_drconf_cell_v1 *)p;
 
 	p = (__be32 *)pr->prop->value;
 	p++;
-	new_drmem = (struct of_drconf_cell *)p;
+	new_drmem = (struct of_drconf_cell_v1 *)p;
 
 	for (i = 0; i < entries; i++) {
 		if ((be32_to_cpu(old_drmem[i].flags) & DRCONF_MEM_ASSIGNED) &&
diff --git a/arch/powerpc/platforms/pseries/hvCall_inst.c b/arch/powerpc/platforms/pseries/hvCall_inst.c
index 957ae347b0b3..89b7ce807e70 100644
--- a/arch/powerpc/platforms/pseries/hvCall_inst.c
+++ b/arch/powerpc/platforms/pseries/hvCall_inst.c
@@ -163,7 +163,7 @@ static int __init hcall_inst_init(void)
 
 	for_each_possible_cpu(cpu) {
 		snprintf(cpu_name_buf, CPU_NAME_BUF_SIZE, "cpu%d", cpu);
-		hcall_file = debugfs_create_file(cpu_name_buf, S_IRUGO,
+		hcall_file = debugfs_create_file(cpu_name_buf, 0444,
 						 hcall_root,
 						 per_cpu(hcall_stats, cpu),
 						 &hcall_inst_seq_fops);
diff --git a/arch/powerpc/platforms/pseries/ibmebus.c b/arch/powerpc/platforms/pseries/ibmebus.c
index 408a86044133..c7c1140c13b6 100644
--- a/arch/powerpc/platforms/pseries/ibmebus.c
+++ b/arch/powerpc/platforms/pseries/ibmebus.c
@@ -298,7 +298,7 @@ static ssize_t ibmebus_store_probe(struct bus_type *bus,
 		return rc;
 	return count;
 }
-static BUS_ATTR(probe, S_IWUSR, NULL, ibmebus_store_probe);
+static BUS_ATTR(probe, 0200, NULL, ibmebus_store_probe);
 
 static ssize_t ibmebus_store_remove(struct bus_type *bus,
 				    const char *buf, size_t count)
@@ -325,7 +325,7 @@ static ssize_t ibmebus_store_remove(struct bus_type *bus,
 		return -ENODEV;
 	}
 }
-static BUS_ATTR(remove, S_IWUSR, NULL, ibmebus_store_remove);
+static BUS_ATTR(remove, 0200, NULL, ibmebus_store_remove);
 
 static struct attribute *ibmbus_bus_attrs[] = {
 	&bus_attr_probe.attr,
diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c
index 69921f72e2da..06f02960b439 100644
--- a/arch/powerpc/platforms/pseries/iommu.c
+++ b/arch/powerpc/platforms/pseries/iommu.c
@@ -816,15 +816,15 @@ static void remove_ddw(struct device_node *np, bool remove_prop)
 	ret = tce_clearrange_multi_pSeriesLP(0,
 		1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
 	if (ret)
-		pr_warning("%pOF failed to clear tces in window.\n",
-			 np);
+		pr_warn("%pOF failed to clear tces in window.\n",
+			np);
 	else
 		pr_debug("%pOF successfully cleared tces in window.\n",
 			 np);
 
 	ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn);
 	if (ret)
-		pr_warning("%pOF: failed to remove direct window: rtas returned "
+		pr_warn("%pOF: failed to remove direct window: rtas returned "
 			"%d to ibm,remove-pe-dma-window(%x) %llx\n",
 			np, ret, ddw_avail[2], liobn);
 	else
@@ -836,7 +836,7 @@ static void remove_ddw(struct device_node *np, bool remove_prop)
 	if (remove_prop)
 		ret = of_remove_property(np, win64);
 	if (ret)
-		pr_warning("%pOF: failed to remove direct window property: %d\n",
+		pr_warn("%pOF: failed to remove direct window property: %d\n",
 			np, ret);
 }
 
@@ -1231,7 +1231,7 @@ static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask)
 			if (dma_offset != 0) {
 				dev_info(dev, "Using 64-bit direct DMA at offset %llx\n", dma_offset);
 				set_dma_offset(dev, dma_offset);
-				set_dma_ops(dev, &dma_direct_ops);
+				set_dma_ops(dev, &dma_nommu_ops);
 				ddw_enabled = true;
 			}
 		}
diff --git a/arch/powerpc/platforms/pseries/lparcfg.c b/arch/powerpc/platforms/pseries/lparcfg.c
index b2706c483067..c508c938dc71 100644
--- a/arch/powerpc/platforms/pseries/lparcfg.c
+++ b/arch/powerpc/platforms/pseries/lparcfg.c
@@ -370,10 +370,10 @@ static void parse_system_parameter_string(struct seq_file *m)
  */
 static int lparcfg_count_active_processors(void)
 {
-	struct device_node *cpus_dn = NULL;
+	struct device_node *cpus_dn;
 	int count = 0;
 
-	while ((cpus_dn = of_find_node_by_type(cpus_dn, "cpu"))) {
+	for_each_node_by_type(cpus_dn, "cpu") {
 #ifdef LPARCFG_DEBUG
 		printk(KERN_ERR "cpus_dn %p\n", cpus_dn);
 #endif
@@ -697,11 +697,11 @@ static const struct file_operations lparcfg_fops = {
 
 static int __init lparcfg_init(void)
 {
-	umode_t mode = S_IRUSR | S_IRGRP | S_IROTH;
+	umode_t mode = 0444;
 
 	/* Allow writing if we have FW_FEATURE_SPLPAR */
 	if (firmware_has_feature(FW_FEATURE_SPLPAR))
-		mode |= S_IWUSR;
+		mode |= 0200;
 
 	if (!proc_create("powerpc/lparcfg", mode, NULL, &lparcfg_fops)) {
 		printk(KERN_ERR "Failed to create powerpc/lparcfg\n");
diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c
index f7042ad492ba..0f7fb7170b03 100644
--- a/arch/powerpc/platforms/pseries/mobility.c
+++ b/arch/powerpc/platforms/pseries/mobility.c
@@ -384,7 +384,7 @@ static ssize_t migration_store(struct class *class,
 #define MIGRATION_API_VERSION	1
 
 static CLASS_ATTR_WO(migration);
-static CLASS_ATTR_STRING(api_version, S_IRUGO, __stringify(MIGRATION_API_VERSION));
+static CLASS_ATTR_STRING(api_version, 0444, __stringify(MIGRATION_API_VERSION));
 
 static int __init mobility_sysfs_init(void)
 {
diff --git a/arch/powerpc/platforms/pseries/of_helpers.c b/arch/powerpc/platforms/pseries/of_helpers.c
index 7e75101fa522..6df192f38f80 100644
--- a/arch/powerpc/platforms/pseries/of_helpers.c
+++ b/arch/powerpc/platforms/pseries/of_helpers.c
@@ -3,6 +3,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <asm/prom.h>
 
 #include "of_helpers.h"
 
@@ -37,3 +38,62 @@ struct device_node *pseries_of_derive_parent(const char *path)
 		kfree(parent_path);
 	return parent ? parent : ERR_PTR(-EINVAL);
 }
+
+
+/* Helper Routines to convert between drc_index to cpu numbers */
+
+int of_read_drc_info_cell(struct property **prop, const __be32 **curval,
+			struct of_drc_info *data)
+{
+	const char *p;
+	const __be32 *p2;
+
+	if (!data)
+		return -EINVAL;
+
+	/* Get drc-type:encode-string */
+	p = data->drc_type = (char*) (*curval);
+	p = of_prop_next_string(*prop, p);
+	if (!p)
+		return -EINVAL;
+
+	/* Get drc-name-prefix:encode-string */
+	data->drc_name_prefix = (char *)p;
+	p = of_prop_next_string(*prop, p);
+	if (!p)
+		return -EINVAL;
+
+	/* Get drc-index-start:encode-int */
+	p2 = (const __be32 *)p;
+	p2 = of_prop_next_u32(*prop, p2, &data->drc_index_start);
+	if (!p2)
+		return -EINVAL;
+
+	/* Get drc-name-suffix-start:encode-int */
+	p2 = of_prop_next_u32(*prop, p2, &data->drc_name_suffix_start);
+	if (!p2)
+		return -EINVAL;
+
+	/* Get number-sequential-elements:encode-int */
+	p2 = of_prop_next_u32(*prop, p2, &data->num_sequential_elems);
+	if (!p2)
+		return -EINVAL;
+
+	/* Get sequential-increment:encode-int */
+	p2 = of_prop_next_u32(*prop, p2, &data->sequential_inc);
+	if (!p2)
+		return -EINVAL;
+
+	/* Get drc-power-domain:encode-int */
+	p2 = of_prop_next_u32(*prop, p2, &data->drc_power_domain);
+	if (!p2)
+		return -EINVAL;
+
+	/* Should now know end of current entry */
+	(*curval) = (void *)p2;
+	data->last_drc_index = data->drc_index_start +
+		((data->num_sequential_elems - 1) * data->sequential_inc);
+
+	return 0;
+}
+EXPORT_SYMBOL(of_read_drc_info_cell);
diff --git a/arch/powerpc/platforms/pseries/pci.c b/arch/powerpc/platforms/pseries/pci.c
index 09eba5a9929a..eab96637d6cf 100644
--- a/arch/powerpc/platforms/pseries/pci.c
+++ b/arch/powerpc/platforms/pseries/pci.c
@@ -3,17 +3,17 @@
  * Copyright (C) 2003 Anton Blanchard <anton@au.ibm.com>, IBM
  *
  * pSeries specific routines for PCI.
- * 
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
- *    
+ *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- * 
+ *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
@@ -54,10 +54,174 @@ void pcibios_name_device(struct pci_dev *dev)
 			}
 		}
 	}
-}   
+}
 DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pcibios_name_device);
 #endif
 
+#ifdef CONFIG_PCI_IOV
+#define MAX_VFS_FOR_MAP_PE 256
+struct pe_map_bar_entry {
+	__be64     bar;       /* Input:  Virtual Function BAR */
+	__be16     rid;       /* Input:  Virtual Function Router ID */
+	__be16     pe_num;    /* Output: Virtual Function PE Number */
+	__be32     reserved;  /* Reserved Space */
+};
+
+int pseries_send_map_pe(struct pci_dev *pdev,
+			u16 num_vfs,
+			struct pe_map_bar_entry *vf_pe_array)
+{
+	struct pci_dn *pdn;
+	int rc;
+	unsigned long buid, addr;
+	int ibm_map_pes = rtas_token("ibm,open-sriov-map-pe-number");
+
+	if (ibm_map_pes == RTAS_UNKNOWN_SERVICE)
+		return -EINVAL;
+
+	pdn = pci_get_pdn(pdev);
+	addr = rtas_config_addr(pdn->busno, pdn->devfn, 0);
+	buid = pdn->phb->buid;
+	spin_lock(&rtas_data_buf_lock);
+	memcpy(rtas_data_buf, vf_pe_array,
+	       RTAS_DATA_BUF_SIZE);
+	rc = rtas_call(ibm_map_pes, 5, 1, NULL, addr,
+		       BUID_HI(buid), BUID_LO(buid),
+		       rtas_data_buf,
+		       num_vfs * sizeof(struct pe_map_bar_entry));
+	memcpy(vf_pe_array, rtas_data_buf, RTAS_DATA_BUF_SIZE);
+	spin_unlock(&rtas_data_buf_lock);
+
+	if (rc)
+		dev_err(&pdev->dev,
+			"%s: Failed to associate pes PE#%lx, rc=%x\n",
+			__func__,  addr, rc);
+
+	return rc;
+}
+
+void pseries_set_pe_num(struct pci_dev *pdev, u16 vf_index, __be16 pe_num)
+{
+	struct pci_dn *pdn;
+
+	pdn = pci_get_pdn(pdev);
+	pdn->pe_num_map[vf_index] = be16_to_cpu(pe_num);
+	dev_dbg(&pdev->dev, "VF %04x:%02x:%02x.%x associated with PE#%x\n",
+		pci_domain_nr(pdev->bus),
+		pdev->bus->number,
+		PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
+		PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)),
+		pdn->pe_num_map[vf_index]);
+}
+
+int pseries_associate_pes(struct pci_dev *pdev, u16 num_vfs)
+{
+	struct pci_dn *pdn;
+	int i, rc, vf_index;
+	struct pe_map_bar_entry *vf_pe_array;
+	struct resource *res;
+	u64 size;
+
+	vf_pe_array = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL);
+	if (!vf_pe_array)
+		return -ENOMEM;
+
+	pdn = pci_get_pdn(pdev);
+	/* create firmware structure to associate pes */
+	for (vf_index = 0; vf_index < num_vfs; vf_index++) {
+		pdn->pe_num_map[vf_index] = IODA_INVALID_PE;
+		for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+			res = &pdev->resource[i + PCI_IOV_RESOURCES];
+			if (!res->parent)
+				continue;
+			size = pcibios_iov_resource_alignment(pdev, i +
+					PCI_IOV_RESOURCES);
+			vf_pe_array[vf_index].bar =
+				cpu_to_be64(res->start + size * vf_index);
+			vf_pe_array[vf_index].rid =
+				cpu_to_be16((pci_iov_virtfn_bus(pdev, vf_index)
+					    << 8) | pci_iov_virtfn_devfn(pdev,
+					    vf_index));
+			vf_pe_array[vf_index].pe_num =
+				cpu_to_be16(IODA_INVALID_PE);
+		}
+	}
+
+	rc = pseries_send_map_pe(pdev, num_vfs, vf_pe_array);
+	/* Only zero is success */
+	if (!rc)
+		for (vf_index = 0; vf_index < num_vfs; vf_index++)
+			pseries_set_pe_num(pdev, vf_index,
+					   vf_pe_array[vf_index].pe_num);
+
+	kfree(vf_pe_array);
+	return rc;
+}
+
+int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
+{
+	struct pci_dn         *pdn;
+	int                    rc;
+	const int *max_vfs;
+	int max_config_vfs;
+	struct device_node *dn = pci_device_to_OF_node(pdev);
+
+	max_vfs = of_get_property(dn, "ibm,number-of-configurable-vfs", NULL);
+
+	if (!max_vfs)
+		return -EINVAL;
+
+	/* First integer stores max config */
+	max_config_vfs = of_read_number(&max_vfs[0], 1);
+	if (max_config_vfs < num_vfs && num_vfs > MAX_VFS_FOR_MAP_PE) {
+		dev_err(&pdev->dev,
+			"Num VFs %x > %x Configurable VFs\n",
+			num_vfs, (num_vfs > MAX_VFS_FOR_MAP_PE) ?
+			MAX_VFS_FOR_MAP_PE : max_config_vfs);
+		return -EINVAL;
+	}
+
+	pdn = pci_get_pdn(pdev);
+	pdn->pe_num_map = kmalloc_array(num_vfs,
+					sizeof(*pdn->pe_num_map),
+					GFP_KERNEL);
+	if (!pdn->pe_num_map)
+		return -ENOMEM;
+
+	rc = pseries_associate_pes(pdev, num_vfs);
+
+	/* Anything other than zero is failure */
+	if (rc) {
+		dev_err(&pdev->dev, "Failure to enable sriov: %x\n", rc);
+		kfree(pdn->pe_num_map);
+	} else {
+		pci_vf_drivers_autoprobe(pdev, false);
+	}
+
+	return rc;
+}
+
+int pseries_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
+{
+	/* Allocate PCI data */
+	add_dev_pci_data(pdev);
+	return pseries_pci_sriov_enable(pdev, num_vfs);
+}
+
+int pseries_pcibios_sriov_disable(struct pci_dev *pdev)
+{
+	struct pci_dn         *pdn;
+
+	pdn = pci_get_pdn(pdev);
+	/* Releasing pe_num_map */
+	kfree(pdn->pe_num_map);
+	/* Release PCI data */
+	remove_dev_pci_data(pdev);
+	pci_vf_drivers_autoprobe(pdev, true);
+	return 0;
+}
+#endif
+
 static void __init pSeries_request_regions(void)
 {
 	if (!isa_io_base)
@@ -76,6 +240,11 @@ void __init pSeries_final_fixup(void)
 	pSeries_request_regions();
 
 	eeh_addr_cache_build();
+
+#ifdef CONFIG_PCI_IOV
+	ppc_md.pcibios_sriov_enable = pseries_pcibios_sriov_enable;
+	ppc_md.pcibios_sriov_disable = pseries_pcibios_sriov_disable;
+#endif
 }
 
 /*
diff --git a/arch/powerpc/platforms/pseries/pseries_energy.c b/arch/powerpc/platforms/pseries/pseries_energy.c
index 35c891aabef0..6ed22127391b 100644
--- a/arch/powerpc/platforms/pseries/pseries_energy.c
+++ b/arch/powerpc/platforms/pseries/pseries_energy.c
@@ -22,6 +22,7 @@
 #include <asm/page.h>
 #include <asm/hvcall.h>
 #include <asm/firmware.h>
+#include <asm/prom.h>
 
 
 #define MODULE_VERS "1.0"
@@ -38,26 +39,58 @@ static int sysfs_entries;
 static u32 cpu_to_drc_index(int cpu)
 {
 	struct device_node *dn = NULL;
-	const int *indexes;
-	int i;
+	int thread_index;
 	int rc = 1;
 	u32 ret = 0;
 
 	dn = of_find_node_by_path("/cpus");
 	if (dn == NULL)
 		goto err;
-	indexes = of_get_property(dn, "ibm,drc-indexes", NULL);
-	if (indexes == NULL)
-		goto err_of_node_put;
+
 	/* Convert logical cpu number to core number */
-	i = cpu_core_index_of_thread(cpu);
-	/*
-	 * The first element indexes[0] is the number of drc_indexes
-	 * returned in the list.  Hence i+1 will get the drc_index
-	 * corresponding to core number i.
-	 */
-	WARN_ON(i > indexes[0]);
-	ret = indexes[i + 1];
+	thread_index = cpu_core_index_of_thread(cpu);
+
+	if (firmware_has_feature(FW_FEATURE_DRC_INFO)) {
+		struct property *info = NULL;
+		struct of_drc_info drc;
+		int j;
+		u32 num_set_entries;
+		const __be32 *value;
+
+		info = of_find_property(dn, "ibm,drc-info", NULL);
+		if (info == NULL)
+			goto err_of_node_put;
+
+		value = of_prop_next_u32(info, NULL, &num_set_entries);
+		if (!value)
+			goto err_of_node_put;
+
+		for (j = 0; j < num_set_entries; j++) {
+
+			of_read_drc_info_cell(&info, &value, &drc);
+			if (strncmp(drc.drc_type, "CPU", 3))
+				goto err;
+
+			if (thread_index < drc.last_drc_index)
+				break;
+		}
+
+		ret = drc.drc_index_start + (thread_index * drc.sequential_inc);
+	} else {
+		const __be32 *indexes;
+
+		indexes = of_get_property(dn, "ibm,drc-indexes", NULL);
+		if (indexes == NULL)
+			goto err_of_node_put;
+
+		/*
+		 * The first element indexes[0] is the number of drc_indexes
+		 * returned in the list.  Hence thread_index+1 will get the
+		 * drc_index corresponding to core number thread_index.
+		 */
+		ret = indexes[thread_index + 1];
+	}
+
 	rc = 0;
 
 err_of_node_put:
@@ -72,34 +105,71 @@ static int drc_index_to_cpu(u32 drc_index)
 {
 	struct device_node *dn = NULL;
 	const int *indexes;
-	int i, cpu = 0;
+	int thread_index = 0, cpu = 0;
 	int rc = 1;
 
 	dn = of_find_node_by_path("/cpus");
 	if (dn == NULL)
 		goto err;
-	indexes = of_get_property(dn, "ibm,drc-indexes", NULL);
-	if (indexes == NULL)
-		goto err_of_node_put;
-	/*
-	 * First element in the array is the number of drc_indexes
-	 * returned.  Search through the list to find the matching
-	 * drc_index and get the core number
-	 */
-	for (i = 0; i < indexes[0]; i++) {
-		if (indexes[i + 1] == drc_index)
+
+	if (firmware_has_feature(FW_FEATURE_DRC_INFO)) {
+		struct property *info = NULL;
+		struct of_drc_info drc;
+		int j;
+		u32 num_set_entries;
+		const __be32 *value;
+
+		info = of_find_property(dn, "ibm,drc-info", NULL);
+		if (info == NULL)
+			goto err_of_node_put;
+
+		value = of_prop_next_u32(info, NULL, &num_set_entries);
+		if (!value)
+			goto err_of_node_put;
+
+		for (j = 0; j < num_set_entries; j++) {
+
+			of_read_drc_info_cell(&info, &value, &drc);
+			if (strncmp(drc.drc_type, "CPU", 3))
+				goto err;
+
+			if (drc_index > drc.last_drc_index) {
+				cpu += drc.num_sequential_elems;
+				continue;
+			}
+			cpu += ((drc_index - drc.drc_index_start) /
+				drc.sequential_inc);
+
+			thread_index = cpu_first_thread_of_core(cpu);
+			rc = 0;
 			break;
+		}
+	} else {
+		unsigned long int i;
+
+		indexes = of_get_property(dn, "ibm,drc-indexes", NULL);
+		if (indexes == NULL)
+			goto err_of_node_put;
+		/*
+		 * First element in the array is the number of drc_indexes
+		 * returned.  Search through the list to find the matching
+		 * drc_index and get the core number
+		 */
+		for (i = 0; i < indexes[0]; i++) {
+			if (indexes[i + 1] == drc_index)
+				break;
+		}
+		/* Convert core number to logical cpu number */
+		thread_index = cpu_first_thread_of_core(i);
+		rc = 0;
 	}
-	/* Convert core number to logical cpu number */
-	cpu = cpu_first_thread_of_core(i);
-	rc = 0;
 
 err_of_node_put:
 	of_node_put(dn);
 err:
 	if (rc)
 		printk(KERN_WARNING "drc_index_to_cpu(%d) failed", drc_index);
-	return cpu;
+	return thread_index;
 }
 
 /*
diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c
index 81d8614e7379..5e1ef9150182 100644
--- a/arch/powerpc/platforms/pseries/ras.c
+++ b/arch/powerpc/platforms/pseries/ras.c
@@ -48,6 +48,28 @@ static irqreturn_t ras_epow_interrupt(int irq, void *dev_id);
 static irqreturn_t ras_error_interrupt(int irq, void *dev_id);
 
 
+/*
+ * Enable the hotplug interrupt late because processing them may touch other
+ * devices or systems (e.g. hugepages) that have not been initialized at the
+ * subsys stage.
+ */
+int __init init_ras_hotplug_IRQ(void)
+{
+	struct device_node *np;
+
+	/* Hotplug Events */
+	np = of_find_node_by_path("/event-sources/hot-plug-events");
+	if (np != NULL) {
+		if (dlpar_workqueue_init() == 0)
+			request_event_sources_irqs(np, ras_hotplug_interrupt,
+						   "RAS_HOTPLUG");
+		of_node_put(np);
+	}
+
+	return 0;
+}
+machine_late_initcall(pseries, init_ras_hotplug_IRQ);
+
 /*
  * Initialize handlers for the set of interrupts caused by hardware errors
  * and power system events.
@@ -66,15 +88,6 @@ static int __init init_ras_IRQ(void)
 		of_node_put(np);
 	}
 
-	/* Hotplug Events */
-	np = of_find_node_by_path("/event-sources/hot-plug-events");
-	if (np != NULL) {
-		if (dlpar_workqueue_init() == 0)
-			request_event_sources_irqs(np, ras_hotplug_interrupt,
-					   "RAS_HOTPLUG");
-		of_node_put(np);
-	}
-
 	/* EPOW Events */
 	np = of_find_node_by_path("/event-sources/epow-events");
 	if (np != NULL) {
diff --git a/arch/powerpc/platforms/pseries/reconfig.c b/arch/powerpc/platforms/pseries/reconfig.c
index f24d8159c9e1..0e0208117e77 100644
--- a/arch/powerpc/platforms/pseries/reconfig.c
+++ b/arch/powerpc/platforms/pseries/reconfig.c
@@ -405,7 +405,7 @@ static int proc_ppc64_create_ofdt(void)
 {
 	struct proc_dir_entry *ent;
 
-	ent = proc_create("powerpc/ofdt", S_IWUSR, NULL, &ofdt_fops);
+	ent = proc_create("powerpc/ofdt", 0200, NULL, &ofdt_fops);
 	if (ent)
 		proc_set_size(ent, 0);
 
diff --git a/arch/powerpc/platforms/pseries/scanlog.c b/arch/powerpc/platforms/pseries/scanlog.c
index c47585a78b69..054ce7a16fc3 100644
--- a/arch/powerpc/platforms/pseries/scanlog.c
+++ b/arch/powerpc/platforms/pseries/scanlog.c
@@ -179,7 +179,7 @@ static int __init scanlog_init(void)
 	if (!scanlog_buffer)
 		goto err;
 
-	ent = proc_create("powerpc/rtas/scan-log-dump", S_IRUSR, NULL,
+	ent = proc_create("powerpc/rtas/scan-log-dump", 0400, NULL,
 			  &scanlog_fops);
 	if (!ent)
 		goto err;
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index ae4f596273b5..1a527625acf7 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -371,8 +371,8 @@ void pseries_disable_reloc_on_exc(void)
 		mdelay(get_longbusy_msecs(rc));
 	}
 	if (rc != H_SUCCESS)
-		pr_warning("Warning: Failed to disable relocation on "
-			   "exceptions: %ld\n", rc);
+		pr_warn("Warning: Failed to disable relocation on exceptions: %ld\n",
+			rc);
 }
 EXPORT_SYMBOL(pseries_disable_reloc_on_exc);
 
@@ -482,7 +482,8 @@ static void pseries_setup_rfi_flush(void)
 		if (types == L1D_FLUSH_NONE)
 			types = L1D_FLUSH_FALLBACK;
 
-		if (!(result.behaviour & H_CPU_BEHAV_L1D_FLUSH_PR))
+		if ((!(result.behaviour & H_CPU_BEHAV_L1D_FLUSH_PR)) ||
+		    (!(result.behaviour & H_CPU_BEHAV_FAVOUR_SECURITY)))
 			enable = false;
 	} else {
 		/* Default to fallback if case hcall is not available */
@@ -492,6 +493,162 @@ static void pseries_setup_rfi_flush(void)
 	setup_rfi_flush(types, enable);
 }
 
+#ifdef CONFIG_PCI_IOV
+enum rtas_iov_fw_value_map {
+	NUM_RES_PROPERTY  = 0, /* Number of Resources */
+	LOW_INT           = 1, /* Lowest 32 bits of Address */
+	START_OF_ENTRIES  = 2, /* Always start of entry */
+	APERTURE_PROPERTY = 2, /* Start of entry+ to  Aperture Size */
+	WDW_SIZE_PROPERTY = 4, /* Start of entry+ to Window Size */
+	NEXT_ENTRY        = 7  /* Go to next entry on array */
+};
+
+enum get_iov_fw_value_index {
+	BAR_ADDRS     = 1,    /*  Get Bar Address */
+	APERTURE_SIZE = 2,    /*  Get Aperture Size */
+	WDW_SIZE      = 3     /*  Get Window Size */
+};
+
+resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno,
+					 enum get_iov_fw_value_index value)
+{
+	const int *indexes;
+	struct device_node *dn = pci_device_to_OF_node(dev);
+	int i, num_res, ret = 0;
+
+	indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL);
+	if (!indexes)
+		return  0;
+
+	/*
+	 * First element in the array is the number of Bars
+	 * returned.  Search through the list to find the matching
+	 * bar
+	 */
+	num_res = of_read_number(&indexes[NUM_RES_PROPERTY], 1);
+	if (resno >= num_res)
+		return 0; /* or an errror */
+
+	i = START_OF_ENTRIES + NEXT_ENTRY * resno;
+	switch (value) {
+	case BAR_ADDRS:
+		ret = of_read_number(&indexes[i], 2);
+		break;
+	case APERTURE_SIZE:
+		ret = of_read_number(&indexes[i + APERTURE_PROPERTY], 2);
+		break;
+	case WDW_SIZE:
+		ret = of_read_number(&indexes[i + WDW_SIZE_PROPERTY], 2);
+		break;
+	}
+
+	return ret;
+}
+
+void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes)
+{
+	struct resource *res;
+	resource_size_t base, size;
+	int i, r, num_res;
+
+	num_res = of_read_number(&indexes[NUM_RES_PROPERTY], 1);
+	num_res = min_t(int, num_res, PCI_SRIOV_NUM_BARS);
+	for (i = START_OF_ENTRIES, r = 0; r < num_res && r < PCI_SRIOV_NUM_BARS;
+	     i += NEXT_ENTRY, r++) {
+		res = &dev->resource[r + PCI_IOV_RESOURCES];
+		base = of_read_number(&indexes[i], 2);
+		size = of_read_number(&indexes[i + APERTURE_PROPERTY], 2);
+		res->flags = pci_parse_of_flags(of_read_number
+						(&indexes[i + LOW_INT], 1), 0);
+		res->flags |= (IORESOURCE_MEM_64 | IORESOURCE_PCI_FIXED);
+		res->name = pci_name(dev);
+		res->start = base;
+		res->end = base + size - 1;
+	}
+}
+
+void of_pci_parse_iov_addrs(struct pci_dev *dev, const int *indexes)
+{
+	struct resource *res, *root, *conflict;
+	resource_size_t base, size;
+	int i, r, num_res;
+
+	/*
+	 * First element in the array is the number of Bars
+	 * returned.  Search through the list to find the matching
+	 * bars assign them from firmware into resources structure.
+	 */
+	num_res = of_read_number(&indexes[NUM_RES_PROPERTY], 1);
+	for (i = START_OF_ENTRIES, r = 0; r < num_res && r < PCI_SRIOV_NUM_BARS;
+	     i += NEXT_ENTRY, r++) {
+		res = &dev->resource[r + PCI_IOV_RESOURCES];
+		base = of_read_number(&indexes[i], 2);
+		size = of_read_number(&indexes[i + WDW_SIZE_PROPERTY], 2);
+		res->name = pci_name(dev);
+		res->start = base;
+		res->end = base + size - 1;
+		root = &iomem_resource;
+		dev_dbg(&dev->dev,
+			"pSeries IOV BAR %d: trying firmware assignment %pR\n",
+			 r + PCI_IOV_RESOURCES, res);
+		conflict = request_resource_conflict(root, res);
+		if (conflict) {
+			dev_info(&dev->dev,
+				 "BAR %d: %pR conflicts with %s %pR\n",
+				 r + PCI_IOV_RESOURCES, res,
+				 conflict->name, conflict);
+			res->flags |= IORESOURCE_UNSET;
+		}
+	}
+}
+
+static void pseries_pci_fixup_resources(struct pci_dev *pdev)
+{
+	const int *indexes;
+	struct device_node *dn = pci_device_to_OF_node(pdev);
+
+	/*Firmware must support open sriov otherwise dont configure*/
+	indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL);
+	if (!indexes)
+		return;
+	/* Assign the addresses from device tree*/
+	of_pci_set_vf_bar_size(pdev, indexes);
+}
+
+static void pseries_pci_fixup_iov_resources(struct pci_dev *pdev)
+{
+	const int *indexes;
+	struct device_node *dn = pci_device_to_OF_node(pdev);
+
+	if (!pdev->is_physfn || pdev->is_added)
+		return;
+	/*Firmware must support open sriov otherwise dont configure*/
+	indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL);
+	if (!indexes)
+		return;
+	/* Assign the addresses from device tree*/
+	of_pci_parse_iov_addrs(pdev, indexes);
+}
+
+static resource_size_t pseries_pci_iov_resource_alignment(struct pci_dev *pdev,
+							  int resno)
+{
+	const __be32 *reg;
+	struct device_node *dn = pci_device_to_OF_node(pdev);
+
+	/*Firmware must support open sriov otherwise report regular alignment*/
+	reg = of_get_property(dn, "ibm,is-open-sriov-pf", NULL);
+	if (!reg)
+		return pci_iov_resource_size(pdev, resno);
+
+	if (!pdev->is_physfn)
+		return 0;
+	return pseries_get_iov_fw_value(pdev,
+					resno - PCI_IOV_RESOURCES,
+					APERTURE_SIZE);
+}
+#endif
+
 static void __init pSeries_setup_arch(void)
 {
 	set_arch_panic_timeout(10, ARCH_PANIC_TIMEOUT);
@@ -525,6 +682,14 @@ static void __init pSeries_setup_arch(void)
 		vpa_init(boot_cpuid);
 		ppc_md.power_save = pseries_lpar_idle;
 		ppc_md.enable_pmcs = pseries_lpar_enable_pmcs;
+#ifdef CONFIG_PCI_IOV
+		ppc_md.pcibios_fixup_resources =
+			pseries_pci_fixup_resources;
+		ppc_md.pcibios_fixup_sriov =
+			pseries_pci_fixup_iov_resources;
+		ppc_md.pcibios_iov_resource_alignment =
+			pseries_pci_iov_resource_alignment;
+#endif
 	} else {
 		/* No special idle routine */
 		ppc_md.enable_pmcs = power4_enable_pmcs;
@@ -533,6 +698,12 @@ static void __init pSeries_setup_arch(void)
 	ppc_md.pcibios_root_bridge_prepare = pseries_root_bridge_prepare;
 }
 
+static void pseries_panic(char *str)
+{
+	panic_flush_kmsg_end();
+	rtas_os_term(str);
+}
+
 static int __init pSeries_init_panel(void)
 {
 	/* Manually leave the kernel version on the panel. */
@@ -761,7 +932,7 @@ define_machine(pseries) {
 	.pcibios_fixup		= pSeries_final_fixup,
 	.restart		= rtas_restart,
 	.halt			= rtas_halt,
-	.panic			= rtas_os_term,
+	.panic			= pseries_panic,
 	.get_boot_time		= rtas_get_boot_time,
 	.get_rtc_time		= rtas_get_rtc_time,
 	.set_rtc_time		= rtas_set_rtc_time,
diff --git a/arch/powerpc/platforms/pseries/suspend.c b/arch/powerpc/platforms/pseries/suspend.c
index 89726f07d249..52a021e1f86b 100644
--- a/arch/powerpc/platforms/pseries/suspend.c
+++ b/arch/powerpc/platforms/pseries/suspend.c
@@ -214,8 +214,7 @@ static ssize_t show_hibernate(struct device *dev,
 	return sprintf(buf, "%d\n", KERN_DT_UPDATE);
 }
 
-static DEVICE_ATTR(hibernate, S_IWUSR | S_IRUGO,
-		   show_hibernate, store_hibernate);
+static DEVICE_ATTR(hibernate, 0644, show_hibernate, store_hibernate);
 
 static struct bus_type suspend_subsys = {
 	.name = "power",
diff --git a/arch/powerpc/platforms/pseries/vio.c b/arch/powerpc/platforms/pseries/vio.c
index d86938260a86..49e04ec19238 100644
--- a/arch/powerpc/platforms/pseries/vio.c
+++ b/arch/powerpc/platforms/pseries/vio.c
@@ -618,7 +618,7 @@ static u64 vio_dma_get_required_mask(struct device *dev)
 static const struct dma_map_ops vio_dma_mapping_ops = {
 	.alloc             = vio_dma_iommu_alloc_coherent,
 	.free              = vio_dma_iommu_free_coherent,
-	.mmap		   = dma_direct_mmap_coherent,
+	.mmap		   = dma_nommu_mmap_coherent,
 	.map_sg            = vio_dma_iommu_map_sg,
 	.unmap_sg          = vio_dma_iommu_unmap_sg,
 	.map_page          = vio_dma_iommu_map_page,
diff --git a/arch/powerpc/sysdev/Makefile b/arch/powerpc/sysdev/Makefile
index 0baba21404dc..9861407d644a 100644
--- a/arch/powerpc/sysdev/Makefile
+++ b/arch/powerpc/sysdev/Makefile
@@ -32,7 +32,6 @@ mv64x60-$(CONFIG_PCI)		+= mv64x60_pci.o
 obj-$(CONFIG_MV64X60)		+= $(mv64x60-y) mv64x60_pic.o mv64x60_dev.o \
 				   mv64x60_udbg.o
 obj-$(CONFIG_RTC_DRV_CMOS)	+= rtc_cmos_setup.o
-obj-$(CONFIG_AXON_RAM)		+= axonram.o
 
 obj-$(CONFIG_PPC_INDIRECT_PCI)	+= indirect_pci.o
 obj-$(CONFIG_PPC_I8259)		+= i8259.o
@@ -43,7 +42,8 @@ obj-$(CONFIG_OF_RTC)		+= of_rtc.o
 
 obj-$(CONFIG_CPM)		+= cpm_common.o
 obj-$(CONFIG_CPM1)		+= cpm1.o
-obj-$(CONFIG_CPM2)		+= cpm2.o cpm2_pic.o
+obj-$(CONFIG_CPM2)		+= cpm2.o cpm2_pic.o cpm_gpio.o
+obj-$(CONFIG_8xx_GPIO)		+= cpm_gpio.o
 obj-$(CONFIG_QUICC_ENGINE)	+= cpm_common.o
 obj-$(CONFIG_PPC_DCR)		+= dcr.o
 obj-$(CONFIG_UCODE_PATCH)	+= micropatch.o
diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c
deleted file mode 100644
index 1b307c80b401..000000000000
--- a/arch/powerpc/sysdev/axonram.c
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * (C) Copyright IBM Deutschland Entwicklung GmbH 2006
- *
- * Author: Maxim Shchetynin <maxim@de.ibm.com>
- *
- * Axon DDR2 device driver.
- * It registers one block device per Axon's DDR2 memory bank found on a system.
- * Block devices are called axonram?, their major and minor numbers are
- * available in /proc/devices, /proc/partitions or in /sys/block/axonram?/dev.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#include <linux/bio.h>
-#include <linux/blkdev.h>
-#include <linux/dax.h>
-#include <linux/device.h>
-#include <linux/errno.h>
-#include <linux/fs.h>
-#include <linux/genhd.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/ioport.h>
-#include <linux/irq.h>
-#include <linux/irqreturn.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/mod_devicetable.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include <linux/types.h>
-#include <linux/of_device.h>
-#include <linux/of_platform.h>
-#include <linux/pfn_t.h>
-#include <linux/uio.h>
-
-#include <asm/page.h>
-#include <asm/prom.h>
-
-#define AXON_RAM_MODULE_NAME		"axonram"
-#define AXON_RAM_DEVICE_NAME		"axonram"
-#define AXON_RAM_MINORS_PER_DISK	16
-#define AXON_RAM_BLOCK_SHIFT		PAGE_SHIFT
-#define AXON_RAM_BLOCK_SIZE		1 << AXON_RAM_BLOCK_SHIFT
-#define AXON_RAM_SECTOR_SHIFT		9
-#define AXON_RAM_SECTOR_SIZE		1 << AXON_RAM_SECTOR_SHIFT
-#define AXON_RAM_IRQ_FLAGS		IRQF_SHARED | IRQF_TRIGGER_RISING
-
-static int azfs_major, azfs_minor;
-
-struct axon_ram_bank {
-	struct platform_device	*device;
-	struct gendisk		*disk;
-	struct dax_device	*dax_dev;
-	unsigned int		irq_id;
-	unsigned long		ph_addr;
-	unsigned long		io_addr;
-	unsigned long		size;
-	unsigned long		ecc_counter;
-};
-
-static ssize_t
-axon_ram_sysfs_ecc(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct platform_device *device = to_platform_device(dev);
-	struct axon_ram_bank *bank = device->dev.platform_data;
-
-	BUG_ON(!bank);
-
-	return sprintf(buf, "%ld\n", bank->ecc_counter);
-}
-
-static DEVICE_ATTR(ecc, S_IRUGO, axon_ram_sysfs_ecc, NULL);
-
-/**
- * axon_ram_irq_handler - interrupt handler for Axon RAM ECC
- * @irq: interrupt ID
- * @dev: pointer to of_device
- */
-static irqreturn_t
-axon_ram_irq_handler(int irq, void *dev)
-{
-	struct platform_device *device = dev;
-	struct axon_ram_bank *bank = device->dev.platform_data;
-
-	BUG_ON(!bank);
-
-	dev_err(&device->dev, "Correctable memory error occurred\n");
-	bank->ecc_counter++;
-	return IRQ_HANDLED;
-}
-
-/**
- * axon_ram_make_request - make_request() method for block device
- * @queue, @bio: see blk_queue_make_request()
- */
-static blk_qc_t
-axon_ram_make_request(struct request_queue *queue, struct bio *bio)
-{
-	struct axon_ram_bank *bank = bio->bi_disk->private_data;
-	unsigned long phys_mem, phys_end;
-	void *user_mem;
-	struct bio_vec vec;
-	unsigned int transfered;
-	struct bvec_iter iter;
-
-	phys_mem = bank->io_addr + (bio->bi_iter.bi_sector <<
-				    AXON_RAM_SECTOR_SHIFT);
-	phys_end = bank->io_addr + bank->size;
-	transfered = 0;
-	bio_for_each_segment(vec, bio, iter) {
-		if (unlikely(phys_mem + vec.bv_len > phys_end)) {
-			bio_io_error(bio);
-			return BLK_QC_T_NONE;
-		}
-
-		user_mem = page_address(vec.bv_page) + vec.bv_offset;
-		if (bio_data_dir(bio) == READ)
-			memcpy(user_mem, (void *) phys_mem, vec.bv_len);
-		else
-			memcpy((void *) phys_mem, user_mem, vec.bv_len);
-
-		phys_mem += vec.bv_len;
-		transfered += vec.bv_len;
-	}
-	bio_endio(bio);
-	return BLK_QC_T_NONE;
-}
-
-static const struct block_device_operations axon_ram_devops = {
-	.owner		= THIS_MODULE,
-};
-
-static long
-__axon_ram_direct_access(struct axon_ram_bank *bank, pgoff_t pgoff, long nr_pages,
-		       void **kaddr, pfn_t *pfn)
-{
-	resource_size_t offset = pgoff * PAGE_SIZE;
-
-	*kaddr = (void *) bank->io_addr + offset;
-	*pfn = phys_to_pfn_t(bank->ph_addr + offset, PFN_DEV);
-	return (bank->size - offset) / PAGE_SIZE;
-}
-
-static long
-axon_ram_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
-		       void **kaddr, pfn_t *pfn)
-{
-	struct axon_ram_bank *bank = dax_get_private(dax_dev);
-
-	return __axon_ram_direct_access(bank, pgoff, nr_pages, kaddr, pfn);
-}
-
-static size_t axon_ram_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
-		void *addr, size_t bytes, struct iov_iter *i)
-{
-	return copy_from_iter(addr, bytes, i);
-}
-
-static const struct dax_operations axon_ram_dax_ops = {
-	.direct_access = axon_ram_dax_direct_access,
-	.copy_from_iter = axon_ram_copy_from_iter,
-};
-
-/**
- * axon_ram_probe - probe() method for platform driver
- * @device: see platform_driver method
- */
-static int axon_ram_probe(struct platform_device *device)
-{
-	static int axon_ram_bank_id = -1;
-	struct axon_ram_bank *bank;
-	struct resource resource;
-	int rc;
-
-	axon_ram_bank_id++;
-
-	dev_info(&device->dev, "Found memory controller on %pOF\n",
-			device->dev.of_node);
-
-	bank = kzalloc(sizeof(*bank), GFP_KERNEL);
-	if (!bank)
-		return -ENOMEM;
-
-	device->dev.platform_data = bank;
-
-	bank->device = device;
-
-	if (of_address_to_resource(device->dev.of_node, 0, &resource) != 0) {
-		dev_err(&device->dev, "Cannot access device tree\n");
-		rc = -EFAULT;
-		goto failed;
-	}
-
-	bank->size = resource_size(&resource);
-
-	if (bank->size == 0) {
-		dev_err(&device->dev, "No DDR2 memory found for %s%d\n",
-				AXON_RAM_DEVICE_NAME, axon_ram_bank_id);
-		rc = -ENODEV;
-		goto failed;
-	}
-
-	dev_info(&device->dev, "Register DDR2 memory device %s%d with %luMB\n",
-			AXON_RAM_DEVICE_NAME, axon_ram_bank_id, bank->size >> 20);
-
-	bank->ph_addr = resource.start;
-	bank->io_addr = (unsigned long) ioremap_prot(
-			bank->ph_addr, bank->size, _PAGE_NO_CACHE);
-	if (bank->io_addr == 0) {
-		dev_err(&device->dev, "ioremap() failed\n");
-		rc = -EFAULT;
-		goto failed;
-	}
-
-	bank->disk = alloc_disk(AXON_RAM_MINORS_PER_DISK);
-	if (bank->disk == NULL) {
-		dev_err(&device->dev, "Cannot register disk\n");
-		rc = -EFAULT;
-		goto failed;
-	}
-
-
-	bank->disk->major = azfs_major;
-	bank->disk->first_minor = azfs_minor;
-	bank->disk->fops = &axon_ram_devops;
-	bank->disk->private_data = bank;
-
-	sprintf(bank->disk->disk_name, "%s%d",
-			AXON_RAM_DEVICE_NAME, axon_ram_bank_id);
-
-	bank->dax_dev = alloc_dax(bank, bank->disk->disk_name,
-			&axon_ram_dax_ops);
-	if (!bank->dax_dev) {
-		rc = -ENOMEM;
-		goto failed;
-	}
-
-	bank->disk->queue = blk_alloc_queue(GFP_KERNEL);
-	if (bank->disk->queue == NULL) {
-		dev_err(&device->dev, "Cannot register disk queue\n");
-		rc = -EFAULT;
-		goto failed;
-	}
-
-	set_capacity(bank->disk, bank->size >> AXON_RAM_SECTOR_SHIFT);
-	blk_queue_make_request(bank->disk->queue, axon_ram_make_request);
-	blk_queue_logical_block_size(bank->disk->queue, AXON_RAM_SECTOR_SIZE);
-	device_add_disk(&device->dev, bank->disk);
-
-	bank->irq_id = irq_of_parse_and_map(device->dev.of_node, 0);
-	if (!bank->irq_id) {
-		dev_err(&device->dev, "Cannot access ECC interrupt ID\n");
-		rc = -EFAULT;
-		goto failed;
-	}
-
-	rc = request_irq(bank->irq_id, axon_ram_irq_handler,
-			AXON_RAM_IRQ_FLAGS, bank->disk->disk_name, device);
-	if (rc != 0) {
-		dev_err(&device->dev, "Cannot register ECC interrupt handler\n");
-		bank->irq_id = 0;
-		rc = -EFAULT;
-		goto failed;
-	}
-
-	rc = device_create_file(&device->dev, &dev_attr_ecc);
-	if (rc != 0) {
-		dev_err(&device->dev, "Cannot create sysfs file\n");
-		rc = -EFAULT;
-		goto failed;
-	}
-
-	azfs_minor += bank->disk->minors;
-
-	return 0;
-
-failed:
-	if (bank->irq_id)
-		free_irq(bank->irq_id, device);
-	if (bank->disk != NULL) {
-		if (bank->disk->major > 0)
-			unregister_blkdev(bank->disk->major,
-					bank->disk->disk_name);
-		if (bank->disk->flags & GENHD_FL_UP)
-			del_gendisk(bank->disk);
-		put_disk(bank->disk);
-	}
-	kill_dax(bank->dax_dev);
-	put_dax(bank->dax_dev);
-	device->dev.platform_data = NULL;
-	if (bank->io_addr != 0)
-		iounmap((void __iomem *) bank->io_addr);
-	kfree(bank);
-	return rc;
-}
-
-/**
- * axon_ram_remove - remove() method for platform driver
- * @device: see of_platform_driver method
- */
-static int
-axon_ram_remove(struct platform_device *device)
-{
-	struct axon_ram_bank *bank = device->dev.platform_data;
-
-	BUG_ON(!bank || !bank->disk);
-
-	device_remove_file(&device->dev, &dev_attr_ecc);
-	free_irq(bank->irq_id, device);
-	kill_dax(bank->dax_dev);
-	put_dax(bank->dax_dev);
-	del_gendisk(bank->disk);
-	put_disk(bank->disk);
-	iounmap((void __iomem *) bank->io_addr);
-	kfree(bank);
-
-	return 0;
-}
-
-static const struct of_device_id axon_ram_device_id[] = {
-	{
-		.type	= "dma-memory"
-	},
-	{}
-};
-MODULE_DEVICE_TABLE(of, axon_ram_device_id);
-
-static struct platform_driver axon_ram_driver = {
-	.probe		= axon_ram_probe,
-	.remove		= axon_ram_remove,
-	.driver = {
-		.name = AXON_RAM_MODULE_NAME,
-		.of_match_table = axon_ram_device_id,
-	},
-};
-
-/**
- * axon_ram_init
- */
-static int __init
-axon_ram_init(void)
-{
-	azfs_major = register_blkdev(azfs_major, AXON_RAM_DEVICE_NAME);
-	if (azfs_major < 0) {
-		printk(KERN_ERR "%s cannot become block device major number\n",
-				AXON_RAM_MODULE_NAME);
-		return -EFAULT;
-	}
-	azfs_minor = 0;
-
-	return platform_driver_register(&axon_ram_driver);
-}
-
-/**
- * axon_ram_exit
- */
-static void __exit
-axon_ram_exit(void)
-{
-	platform_driver_unregister(&axon_ram_driver);
-	unregister_blkdev(azfs_major, AXON_RAM_DEVICE_NAME);
-}
-
-module_init(axon_ram_init);
-module_exit(axon_ram_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Maxim Shchetynin <maxim@de.ibm.com>");
-MODULE_DESCRIPTION("Axon DDR2 RAM device driver for IBM Cell BE");
diff --git a/arch/powerpc/sysdev/cpm1.c b/arch/powerpc/sysdev/cpm1.c
index c6f154b602fb..5240d3a74a10 100644
--- a/arch/powerpc/sysdev/cpm1.c
+++ b/arch/powerpc/sysdev/cpm1.c
@@ -629,8 +629,9 @@ static int cpm1_gpio16_dir_in(struct gpio_chip *gc, unsigned int gpio)
 	return 0;
 }
 
-int cpm1_gpiochip_add16(struct device_node *np)
+int cpm1_gpiochip_add16(struct device *dev)
 {
+	struct device_node *np = dev->of_node;
 	struct cpm1_gpio16_chip *cpm1_gc;
 	struct of_mm_gpio_chip *mm_gc;
 	struct gpio_chip *gc;
@@ -660,6 +661,8 @@ int cpm1_gpiochip_add16(struct device_node *np)
 	gc->get = cpm1_gpio16_get;
 	gc->set = cpm1_gpio16_set;
 	gc->to_irq = cpm1_gpio16_to_irq;
+	gc->parent = dev;
+	gc->owner = THIS_MODULE;
 
 	return of_mm_gpiochip_add_data(np, mm_gc, cpm1_gc);
 }
@@ -755,8 +758,9 @@ static int cpm1_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio)
 	return 0;
 }
 
-int cpm1_gpiochip_add32(struct device_node *np)
+int cpm1_gpiochip_add32(struct device *dev)
 {
+	struct device_node *np = dev->of_node;
 	struct cpm1_gpio32_chip *cpm1_gc;
 	struct of_mm_gpio_chip *mm_gc;
 	struct gpio_chip *gc;
@@ -776,31 +780,10 @@ int cpm1_gpiochip_add32(struct device_node *np)
 	gc->direction_output = cpm1_gpio32_dir_out;
 	gc->get = cpm1_gpio32_get;
 	gc->set = cpm1_gpio32_set;
+	gc->parent = dev;
+	gc->owner = THIS_MODULE;
 
 	return of_mm_gpiochip_add_data(np, mm_gc, cpm1_gc);
 }
 
-static int cpm_init_par_io(void)
-{
-	struct device_node *np;
-
-	for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-a")
-		cpm1_gpiochip_add16(np);
-
-	for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-b")
-		cpm1_gpiochip_add32(np);
-
-	for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-c")
-		cpm1_gpiochip_add16(np);
-
-	for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-d")
-		cpm1_gpiochip_add16(np);
-
-	/* Port E uses CPM2 layout */
-	for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-e")
-		cpm2_gpiochip_add32(np);
-	return 0;
-}
-arch_initcall(cpm_init_par_io);
-
 #endif /* CONFIG_8xx_GPIO */
diff --git a/arch/powerpc/sysdev/cpm2.c b/arch/powerpc/sysdev/cpm2.c
index f78ff841652c..07718b9a2c99 100644
--- a/arch/powerpc/sysdev/cpm2.c
+++ b/arch/powerpc/sysdev/cpm2.c
@@ -354,14 +354,3 @@ void cpm2_set_pin(int port, int pin, int flags)
 	else
 		clrbits32(&iop[port].odr, pin);
 }
-
-static int cpm_init_par_io(void)
-{
-	struct device_node *np;
-
-	for_each_compatible_node(np, NULL, "fsl,cpm2-pario-bank")
-		cpm2_gpiochip_add32(np);
-	return 0;
-}
-arch_initcall(cpm_init_par_io);
-
diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c
index 51bf749a4f3a..b74508175b67 100644
--- a/arch/powerpc/sysdev/cpm_common.c
+++ b/arch/powerpc/sysdev/cpm_common.c
@@ -190,8 +190,9 @@ static int cpm2_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio)
 	return 0;
 }
 
-int cpm2_gpiochip_add32(struct device_node *np)
+int cpm2_gpiochip_add32(struct device *dev)
 {
+	struct device_node *np = dev->of_node;
 	struct cpm2_gpio32_chip *cpm2_gc;
 	struct of_mm_gpio_chip *mm_gc;
 	struct gpio_chip *gc;
@@ -211,6 +212,8 @@ int cpm2_gpiochip_add32(struct device_node *np)
 	gc->direction_output = cpm2_gpio32_dir_out;
 	gc->get = cpm2_gpio32_get;
 	gc->set = cpm2_gpio32_set;
+	gc->parent = dev;
+	gc->owner = THIS_MODULE;
 
 	return of_mm_gpiochip_add_data(np, mm_gc, cpm2_gc);
 }
diff --git a/arch/powerpc/sysdev/cpm_gpio.c b/arch/powerpc/sysdev/cpm_gpio.c
new file mode 100644
index 000000000000..0badc90be666
--- /dev/null
+++ b/arch/powerpc/sysdev/cpm_gpio.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Common CPM GPIO wrapper for the CPM GPIO ports
+ *
+ * Author: Christophe Leroy <christophe.leroy@c-s.fr>
+ *
+ * Copyright 2017 CS Systemes d'Information.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/of_device.h>
+
+#include <asm/cpm.h>
+#ifdef CONFIG_8xx_GPIO
+#include <asm/cpm1.h>
+#endif
+
+static int cpm_gpio_probe(struct platform_device *ofdev)
+{
+	struct device *dev = &ofdev->dev;
+	int (*gp_add)(struct device *dev) = of_device_get_match_data(dev);
+
+	if (!gp_add)
+		return -ENODEV;
+
+	return gp_add(dev);
+}
+
+static const struct of_device_id cpm_gpio_match[] = {
+#ifdef CONFIG_8xx_GPIO
+	{
+		.compatible = "fsl,cpm1-pario-bank-a",
+		.data = cpm1_gpiochip_add16,
+	},
+	{
+		.compatible = "fsl,cpm1-pario-bank-b",
+		.data = cpm1_gpiochip_add32,
+	},
+	{
+		.compatible = "fsl,cpm1-pario-bank-c",
+		.data = cpm1_gpiochip_add16,
+	},
+	{
+		.compatible = "fsl,cpm1-pario-bank-d",
+		.data = cpm1_gpiochip_add16,
+	},
+	/* Port E uses CPM2 layout */
+	{
+		.compatible = "fsl,cpm1-pario-bank-e",
+		.data = cpm2_gpiochip_add32,
+	},
+#endif
+	{
+		.compatible = "fsl,cpm2-pario-bank",
+		.data = cpm2_gpiochip_add32,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, cpm_gpio_match);
+
+static struct platform_driver cpm_gpio_driver = {
+	.probe		= cpm_gpio_probe,
+	.driver		= {
+		.name	= "cpm-gpio",
+		.owner	= THIS_MODULE,
+		.of_match_table	= cpm_gpio_match,
+	},
+};
+
+static int __init cpm_gpio_init(void)
+{
+	return platform_driver_register(&cpm_gpio_driver);
+}
+arch_initcall(cpm_gpio_init);
+
+MODULE_AUTHOR("Christophe Leroy <christophe.leroy@c-s.fr>");
+MODULE_DESCRIPTION("Driver for CPM GPIO");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:cpm-gpio");
diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c
index 3573d54b2770..a6198d4f0f03 100644
--- a/arch/powerpc/sysdev/dart_iommu.c
+++ b/arch/powerpc/sysdev/dart_iommu.c
@@ -402,7 +402,7 @@ static int dart_dma_set_mask(struct device *dev, u64 dma_mask)
 	 */
 	if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) {
 		dev_info(dev, "Using 64-bit DMA iommu bypass\n");
-		set_dma_ops(dev, &dma_direct_ops);
+		set_dma_ops(dev, &dma_nommu_ops);
 	} else {
 		dev_info(dev, "Using 32-bit DMA via iommu\n");
 		set_dma_ops(dev, &dma_iommu_ops);
@@ -446,7 +446,7 @@ void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops)
 	controller_ops->dma_bus_setup = NULL;
 
 	/* Setup pci_dma ops */
-	set_pci_dma_ops(&dma_direct_ops);
+	set_pci_dma_ops(&dma_nommu_ops);
 }
 
 #ifdef CONFIG_PM
diff --git a/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c b/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c
index 1707bf04dec6..94278e8af192 100644
--- a/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c
+++ b/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c
@@ -56,17 +56,16 @@ static ssize_t fsl_timer_wakeup_show(struct device *dev,
 				struct device_attribute *attr,
 				char *buf)
 {
-	struct timeval interval;
-	int val = 0;
+	time64_t interval = 0;
 
 	mutex_lock(&sysfs_lock);
 	if (fsl_wakeup->timer) {
 		mpic_get_remain_time(fsl_wakeup->timer, &interval);
-		val = interval.tv_sec + 1;
+		interval++;
 	}
 	mutex_unlock(&sysfs_lock);
 
-	return sprintf(buf, "%d\n", val);
+	return sprintf(buf, "%lld\n", interval);
 }
 
 static ssize_t fsl_timer_wakeup_store(struct device *dev,
@@ -74,11 +73,10 @@ static ssize_t fsl_timer_wakeup_store(struct device *dev,
 				const char *buf,
 				size_t count)
 {
-	struct timeval interval;
+	time64_t interval;
 	int ret;
 
-	interval.tv_usec = 0;
-	if (kstrtol(buf, 0, &interval.tv_sec))
+	if (kstrtoll(buf, 0, &interval))
 		return -EINVAL;
 
 	mutex_lock(&sysfs_lock);
@@ -89,13 +87,13 @@ static ssize_t fsl_timer_wakeup_store(struct device *dev,
 		fsl_wakeup->timer = NULL;
 	}
 
-	if (!interval.tv_sec) {
+	if (!interval) {
 		mutex_unlock(&sysfs_lock);
 		return count;
 	}
 
 	fsl_wakeup->timer = mpic_request_timer(fsl_mpic_timer_irq,
-						fsl_wakeup, &interval);
+						fsl_wakeup, interval);
 	if (!fsl_wakeup->timer) {
 		mutex_unlock(&sysfs_lock);
 		return -EINVAL;
diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c
index 22d98057f773..918be816b097 100644
--- a/arch/powerpc/sysdev/fsl_pci.c
+++ b/arch/powerpc/sysdev/fsl_pci.c
@@ -118,7 +118,7 @@ static void setup_swiotlb_ops(struct pci_controller *hose)
 {
 	if (ppc_swiotlb_enable) {
 		hose->controller_ops.dma_dev_setup = pci_dma_dev_setup_swiotlb;
-		set_pci_dma_ops(&swiotlb_dma_ops);
+		set_pci_dma_ops(&powerpc_swiotlb_dma_ops);
 	}
 }
 #else
@@ -135,7 +135,7 @@ static int fsl_pci_dma_set_mask(struct device *dev, u64 dma_mask)
 	 * mapping that allows addressing any RAM address from across PCI.
 	 */
 	if (dev_is_pci(dev) && dma_mask >= pci64_dma_offset * 2 - 1) {
-		set_dma_ops(dev, &dma_direct_ops);
+		set_dma_ops(dev, &dma_nommu_ops);
 		set_dma_offset(dev, pci64_dma_offset);
 	}
 
@@ -448,7 +448,7 @@ static void setup_pci_atmu(struct pci_controller *hose)
 #endif
 		/* adjusting outbound windows could reclaim space in mem map */
 		if (paddr_hi < 0xffffffffull)
-			pr_warning("%pOF: WARNING: Outbound window cfg leaves "
+			pr_warn("%pOF: WARNING: Outbound window cfg leaves "
 				"gaps in memory map. Adjusting the memory map "
 				"could reduce unnecessary bounce buffering.\n",
 				hose->dn);
@@ -531,7 +531,7 @@ int fsl_add_bridge(struct platform_device *pdev, int is_primary)
 	dev = pdev->dev.of_node;
 
 	if (!of_device_is_available(dev)) {
-		pr_warning("%pOF: disabled\n", dev);
+		pr_warn("%pOF: disabled\n", dev);
 		return -ENODEV;
 	}
 
@@ -808,8 +808,8 @@ int __init mpc83xx_add_bridge(struct device_node *dev)
 	is_mpc83xx_pci = 1;
 
 	if (!of_device_is_available(dev)) {
-		pr_warning("%pOF: disabled by the firmware.\n",
-			   dev);
+		pr_warn("%pOF: disabled by the firmware.\n",
+			dev);
 		return -ENODEV;
 	}
 	pr_debug("Adding PCI host bridge %pOF\n", dev);
@@ -1070,7 +1070,7 @@ int fsl_pci_mcheck_exception(struct pt_regs *regs)
 	if (is_in_pci_mem_space(addr)) {
 		if (user_mode(regs)) {
 			pagefault_disable();
-			ret = get_user(regs->nip, &inst);
+			ret = get_user(inst, (__u32 __user *)regs->nip);
 			pagefault_enable();
 		} else {
 			ret = probe_kernel_address((void *)regs->nip, inst);
@@ -1304,10 +1304,8 @@ static int add_err_dev(struct platform_device *pdev)
 						   pdev->resource,
 						   pdev->num_resources,
 						   &pd, sizeof(pd));
-	if (IS_ERR(errdev))
-		return PTR_ERR(errdev);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(errdev);
 }
 
 static int fsl_pci_probe(struct platform_device *pdev)
diff --git a/arch/powerpc/sysdev/i8259.c b/arch/powerpc/sysdev/i8259.c
index bafb014e1a7e..cb9a8b71fd0f 100644
--- a/arch/powerpc/sysdev/i8259.c
+++ b/arch/powerpc/sysdev/i8259.c
@@ -145,21 +145,21 @@ static struct resource pic1_iores = {
 	.name = "8259 (master)",
 	.start = 0x20,
 	.end = 0x21,
-	.flags = IORESOURCE_BUSY,
+	.flags = IORESOURCE_IO | IORESOURCE_BUSY,
 };
 
 static struct resource pic2_iores = {
 	.name = "8259 (slave)",
 	.start = 0xa0,
 	.end = 0xa1,
-	.flags = IORESOURCE_BUSY,
+	.flags = IORESOURCE_IO | IORESOURCE_BUSY,
 };
 
 static struct resource pic_edgectrl_iores = {
 	.name = "8259 edge control",
 	.start = 0x4d0,
 	.end = 0x4d1,
-	.flags = IORESOURCE_BUSY,
+	.flags = IORESOURCE_IO | IORESOURCE_BUSY,
 };
 
 static int i8259_host_match(struct irq_domain *h, struct device_node *node,
diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c
index ead3e2549ebf..73067805300a 100644
--- a/arch/powerpc/sysdev/mpic.c
+++ b/arch/powerpc/sysdev/mpic.c
@@ -1008,9 +1008,8 @@ static int mpic_host_map(struct irq_domain *h, unsigned int virq,
 	if (hw == mpic->spurious_vec)
 		return -EINVAL;
 	if (mpic->protected && test_bit(hw, mpic->protected)) {
-		pr_warning("mpic: Mapping of source 0x%x failed, "
-			   "source protected by firmware !\n",\
-			   (unsigned int)hw);
+		pr_warn("mpic: Mapping of source 0x%x failed, source protected by firmware !\n",
+			(unsigned int)hw);
 		return -EPERM;
 	}
 
@@ -1040,9 +1039,8 @@ static int mpic_host_map(struct irq_domain *h, unsigned int virq,
 		return 0;
 
 	if (hw >= mpic->num_sources) {
-		pr_warning("mpic: Mapping of source 0x%x failed, "
-			   "source out of range !\n",\
-			   (unsigned int)hw);
+		pr_warn("mpic: Mapping of source 0x%x failed, source out of range !\n",
+			(unsigned int)hw);
 		return -EINVAL;
 	}
 
diff --git a/arch/powerpc/sysdev/mpic_timer.c b/arch/powerpc/sysdev/mpic_timer.c
index a418579591be..87e7c42777a8 100644
--- a/arch/powerpc/sysdev/mpic_timer.c
+++ b/arch/powerpc/sysdev/mpic_timer.c
@@ -47,9 +47,6 @@
 #define MAX_TICKS_CASCADE		(~0U)
 #define TIMER_OFFSET(num)		(1 << (TIMERS_PER_GROUP - 1 - num))
 
-/* tv_usec should be less than ONE_SECOND, otherwise use tv_sec */
-#define ONE_SECOND			1000000
-
 struct timer_regs {
 	u32	gtccr;
 	u32	res0[3];
@@ -90,51 +87,23 @@ static struct cascade_priv cascade_timer[] = {
 static LIST_HEAD(timer_group_list);
 
 static void convert_ticks_to_time(struct timer_group_priv *priv,
-		const u64 ticks, struct timeval *time)
+		const u64 ticks, time64_t *time)
 {
-	u64 tmp_sec;
-
-	time->tv_sec = (__kernel_time_t)div_u64(ticks, priv->timerfreq);
-	tmp_sec = (u64)time->tv_sec * (u64)priv->timerfreq;
-
-	time->tv_usec = 0;
-
-	if (tmp_sec <= ticks)
-		time->tv_usec = (__kernel_suseconds_t)
-			div_u64((ticks - tmp_sec) * 1000000, priv->timerfreq);
-
-	return;
+	*time = (u64)div_u64(ticks, priv->timerfreq);
 }
 
 /* the time set by the user is converted to "ticks" */
 static int convert_time_to_ticks(struct timer_group_priv *priv,
-		const struct timeval *time, u64 *ticks)
+		time64_t time, u64 *ticks)
 {
 	u64 max_value;		/* prevent u64 overflow */
-	u64 tmp = 0;
-
-	u64 tmp_sec;
-	u64 tmp_ms;
-	u64 tmp_us;
 
 	max_value = div_u64(ULLONG_MAX, priv->timerfreq);
 
-	if (time->tv_sec > max_value ||
-			(time->tv_sec == max_value && time->tv_usec > 0))
+	if (time > max_value)
 		return -EINVAL;
 
-	tmp_sec = (u64)time->tv_sec * (u64)priv->timerfreq;
-	tmp += tmp_sec;
-
-	tmp_ms = time->tv_usec / 1000;
-	tmp_ms = div_u64((u64)tmp_ms * (u64)priv->timerfreq, 1000);
-	tmp += tmp_ms;
-
-	tmp_us = time->tv_usec % 1000;
-	tmp_us = div_u64((u64)tmp_us * (u64)priv->timerfreq, 1000000);
-	tmp += tmp_us;
-
-	*ticks = tmp;
+	*ticks = (u64)time * (u64)priv->timerfreq;
 
 	return 0;
 }
@@ -223,7 +192,7 @@ static struct mpic_timer *get_cascade_timer(struct timer_group_priv *priv,
 	return allocated_timer;
 }
 
-static struct mpic_timer *get_timer(const struct timeval *time)
+static struct mpic_timer *get_timer(time64_t time)
 {
 	struct timer_group_priv *priv;
 	struct mpic_timer *timer;
@@ -277,7 +246,7 @@ static struct mpic_timer *get_timer(const struct timeval *time)
  * @handle: the timer to be started.
  *
  * It will do ->fn(->dev) callback from the hardware interrupt at
- * the ->timeval point in the future.
+ * the 'time64_t' point in the future.
  */
 void mpic_start_timer(struct mpic_timer *handle)
 {
@@ -319,7 +288,7 @@ EXPORT_SYMBOL(mpic_stop_timer);
  *
  * Query timer remaining time.
  */
-void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time)
+void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time)
 {
 	struct timer_group_priv *priv = container_of(handle,
 			struct timer_group_priv, timer[handle->num]);
@@ -391,7 +360,7 @@ EXPORT_SYMBOL(mpic_free_timer);
  * else "handle" on success.
  */
 struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev,
-					const struct timeval *time)
+				      time64_t time)
 {
 	struct mpic_timer *allocated_timer;
 	int ret;
@@ -399,11 +368,7 @@ struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev,
 	if (list_empty(&timer_group_list))
 		return NULL;
 
-	if (!(time->tv_sec + time->tv_usec) ||
-			time->tv_sec < 0 || time->tv_usec < 0)
-		return NULL;
-
-	if (time->tv_usec > ONE_SECOND)
+	if (time < 0)
 		return NULL;
 
 	allocated_timer = get_timer(time);
diff --git a/arch/powerpc/sysdev/mv64x60_pci.c b/arch/powerpc/sysdev/mv64x60_pci.c
index d52b3b81e05f..1afcdb428e51 100644
--- a/arch/powerpc/sysdev/mv64x60_pci.c
+++ b/arch/powerpc/sysdev/mv64x60_pci.c
@@ -37,7 +37,7 @@ static ssize_t mv64x60_hs_reg_read(struct file *filp, struct kobject *kobj,
 	if (count < MV64X60_VAL_LEN_MAX)
 		return -EINVAL;
 
-	phb = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
+	phb = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0));
 	if (!phb)
 		return -ENODEV;
 	pci_read_config_dword(phb, MV64X60_PCICFG_CPCI_HOTSWAP, &v);
@@ -61,7 +61,7 @@ static ssize_t mv64x60_hs_reg_write(struct file *filp, struct kobject *kobj,
 	if (sscanf(buf, "%i", &v) != 1)
 		return -EINVAL;
 
-	phb = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
+	phb = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0));
 	if (!phb)
 		return -ENODEV;
 	pci_write_config_dword(phb, MV64X60_PCICFG_CPCI_HOTSWAP, v);
@@ -73,7 +73,7 @@ static ssize_t mv64x60_hs_reg_write(struct file *filp, struct kobject *kobj,
 static const struct bin_attribute mv64x60_hs_reg_attr = { /* Hotswap register */
 	.attr = {
 		.name = "hs_reg",
-		.mode = S_IRUGO | S_IWUSR,
+		.mode = 0644,
 	},
 	.size  = MV64X60_VAL_LEN_MAX,
 	.read  = mv64x60_hs_reg_read,
diff --git a/arch/powerpc/sysdev/xics/icp-native.c b/arch/powerpc/sysdev/xics/icp-native.c
index 2bfb9968d562..1459f4e8b698 100644
--- a/arch/powerpc/sysdev/xics/icp-native.c
+++ b/arch/powerpc/sysdev/xics/icp-native.c
@@ -241,18 +241,16 @@ static int __init icp_native_map_one_cpu(int hw_id, unsigned long addr,
 			  cpu, hw_id);
 
 	if (!request_mem_region(addr, size, rname)) {
-		pr_warning("icp_native: Could not reserve ICP MMIO"
-			   " for CPU %d, interrupt server #0x%x\n",
-			   cpu, hw_id);
+		pr_warn("icp_native: Could not reserve ICP MMIO for CPU %d, interrupt server #0x%x\n",
+			cpu, hw_id);
 		return -EBUSY;
 	}
 
 	icp_native_regs[cpu] = ioremap(addr, size);
 	kvmppc_set_xics_phys(cpu, addr);
 	if (!icp_native_regs[cpu]) {
-		pr_warning("icp_native: Failed ioremap for CPU %d, "
-			   "interrupt server #0x%x, addr %#lx\n",
-			   cpu, hw_id, addr);
+		pr_warn("icp_native: Failed ioremap for CPU %d, interrupt server #0x%x, addr %#lx\n",
+			cpu, hw_id, addr);
 		release_mem_region(addr, size);
 		return -ENOMEM;
 	}
diff --git a/arch/powerpc/sysdev/xics/ics-opal.c b/arch/powerpc/sysdev/xics/ics-opal.c
index 1c6bf4b66f56..f85f916ba432 100644
--- a/arch/powerpc/sysdev/xics/ics-opal.c
+++ b/arch/powerpc/sysdev/xics/ics-opal.c
@@ -131,8 +131,8 @@ static int ics_opal_set_affinity(struct irq_data *d,
 
 	wanted_server = xics_get_irq_server(d->irq, cpumask, 1);
 	if (wanted_server < 0) {
-		pr_warning("%s: No online cpus in the mask %*pb for irq %d\n",
-			   __func__, cpumask_pr_args(cpumask), d->irq);
+		pr_warn("%s: No online cpus in the mask %*pb for irq %d\n",
+			__func__, cpumask_pr_args(cpumask), d->irq);
 		return -1;
 	}
 	server = ics_opal_mangle_server(wanted_server);
diff --git a/arch/powerpc/sysdev/xics/ics-rtas.c b/arch/powerpc/sysdev/xics/ics-rtas.c
index 42e0c56ff81c..6aabc74688a6 100644
--- a/arch/powerpc/sysdev/xics/ics-rtas.c
+++ b/arch/powerpc/sysdev/xics/ics-rtas.c
@@ -141,8 +141,8 @@ static int ics_rtas_set_affinity(struct irq_data *d,
 
 	irq_server = xics_get_irq_server(d->irq, cpumask, 1);
 	if (irq_server == -1) {
-		pr_warning("%s: No online cpus in the mask %*pb for irq %d\n",
-			   __func__, cpumask_pr_args(cpumask), d->irq);
+		pr_warn("%s: No online cpus in the mask %*pb for irq %d\n",
+			__func__, cpumask_pr_args(cpumask), d->irq);
 		return -1;
 	}
 
diff --git a/arch/powerpc/sysdev/xics/xics-common.c b/arch/powerpc/sysdev/xics/xics-common.c
index ffe138b8b9dc..77e864d5506d 100644
--- a/arch/powerpc/sysdev/xics/xics-common.c
+++ b/arch/powerpc/sysdev/xics/xics-common.c
@@ -243,8 +243,8 @@ void xics_migrate_irqs_away(void)
 
 		/* This is expected during cpu offline. */
 		if (cpu_online(cpu))
-			pr_warning("IRQ %u affinity broken off cpu %u\n",
-			       virq, cpu);
+			pr_warn("IRQ %u affinity broken off cpu %u\n",
+				virq, cpu);
 
 		/* Reset affinity to all cpus */
 		raw_spin_unlock_irqrestore(&desc->lock, flags);
@@ -466,7 +466,7 @@ void __init xics_init(void)
 		    rc = icp_opal_init();
 	}
 	if (rc < 0) {
-		pr_warning("XICS: Cannot find a Presentation Controller !\n");
+		pr_warn("XICS: Cannot find a Presentation Controller !\n");
 		return;
 	}
 
@@ -481,7 +481,7 @@ void __init xics_init(void)
 	if (rc < 0)
 		rc = ics_opal_init();
 	if (rc < 0)
-		pr_warning("XICS: Cannot find a Source Controller !\n");
+		pr_warn("XICS: Cannot find a Source Controller !\n");
 
 	/* Initialize common bits */
 	xics_get_server_size();
diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c
index a3b8d7d1316e..40c06110821c 100644
--- a/arch/powerpc/sysdev/xive/common.c
+++ b/arch/powerpc/sysdev/xive/common.c
@@ -367,7 +367,8 @@ static void xive_irq_eoi(struct irq_data *d)
 	 * EOI the source if it hasn't been disabled and hasn't
 	 * been passed-through to a KVM guest
 	 */
-	if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d))
+	if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d) &&
+	    !(xd->flags & XIVE_IRQ_NO_EOI))
 		xive_do_source_eoi(irqd_to_hwirq(d), xd);
 
 	/*
@@ -1269,11 +1270,6 @@ static void xive_setup_cpu(void)
 {
 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);
 
-	/* Debug: Dump the TM state */
-	pr_devel("CPU %d [HW 0x%02x] VT=%02x\n",
-	    smp_processor_id(), hard_smp_processor_id(),
-	    in_8(xive_tima + xive_tima_offset + TM_WORD2));
-
 	/* The backend might have additional things to do */
 	if (xive_ops->setup_cpu)
 		xive_ops->setup_cpu(smp_processor_id(), xc);
diff --git a/arch/powerpc/sysdev/xive/native.c b/arch/powerpc/sysdev/xive/native.c
index ebc244b08d67..d22aeb0b69e1 100644
--- a/arch/powerpc/sysdev/xive/native.c
+++ b/arch/powerpc/sysdev/xive/native.c
@@ -42,6 +42,7 @@ static u32 xive_provision_chip_count;
 static u32 xive_queue_shift;
 static u32 xive_pool_vps = XIVE_INVALID_VP;
 static struct kmem_cache *xive_provision_cache;
+static bool xive_has_single_esc;
 
 int xive_native_populate_irq_data(u32 hw_irq, struct xive_irq_data *data)
 {
@@ -571,6 +572,10 @@ bool __init xive_native_init(void)
 			break;
 	}
 
+	/* Do we support single escalation */
+	if (of_get_property(np, "single-escalation-support", NULL) != NULL)
+		xive_has_single_esc = true;
+
 	/* Configure Thread Management areas for KVM */
 	for_each_possible_cpu(cpu)
 		kvmppc_set_xive_tima(cpu, r.start, tima);
@@ -667,12 +672,15 @@ void xive_native_free_vp_block(u32 vp_base)
 }
 EXPORT_SYMBOL_GPL(xive_native_free_vp_block);
 
-int xive_native_enable_vp(u32 vp_id)
+int xive_native_enable_vp(u32 vp_id, bool single_escalation)
 {
 	s64 rc;
+	u64 flags = OPAL_XIVE_VP_ENABLED;
 
+	if (single_escalation)
+		flags |= OPAL_XIVE_VP_SINGLE_ESCALATION;
 	for (;;) {
-		rc = opal_xive_set_vp_info(vp_id, OPAL_XIVE_VP_ENABLED, 0);
+		rc = opal_xive_set_vp_info(vp_id, flags, 0);
 		if (rc != OPAL_BUSY)
 			break;
 		msleep(1);
@@ -710,3 +718,9 @@ int xive_native_get_vp_info(u32 vp_id, u32 *out_cam_id, u32 *out_chip_id)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(xive_native_get_vp_info);
+
+bool xive_native_has_single_escalation(void)
+{
+	return xive_has_single_esc;
+}
+EXPORT_SYMBOL_GPL(xive_native_has_single_escalation);
diff --git a/arch/powerpc/sysdev/xive/spapr.c b/arch/powerpc/sysdev/xive/spapr.c
index d9c4c9366049..091f1d0d0af1 100644
--- a/arch/powerpc/sysdev/xive/spapr.c
+++ b/arch/powerpc/sysdev/xive/spapr.c
@@ -356,7 +356,8 @@ static int xive_spapr_configure_queue(u32 target, struct xive_q *q, u8 prio,
 
 	rc = plpar_int_get_queue_info(0, target, prio, &esn_page, &esn_size);
 	if (rc) {
-		pr_err("Error %lld getting queue info prio %d\n", rc, prio);
+		pr_err("Error %lld getting queue info CPU %d prio %d\n", rc,
+		       target, prio);
 		rc = -EIO;
 		goto fail;
 	}
@@ -370,7 +371,8 @@ static int xive_spapr_configure_queue(u32 target, struct xive_q *q, u8 prio,
 	/* Configure and enable the queue in HW */
 	rc = plpar_int_set_queue_config(flags, target, prio, qpage_phys, order);
 	if (rc) {
-		pr_err("Error %lld setting queue for prio %d\n", rc, prio);
+		pr_err("Error %lld setting queue for CPU %d prio %d\n", rc,
+		       target, prio);
 		rc = -EIO;
 	} else {
 		q->qpage = qpage;
@@ -389,8 +391,8 @@ static int xive_spapr_setup_queue(unsigned int cpu, struct xive_cpu *xc,
 	if (IS_ERR(qpage))
 		return PTR_ERR(qpage);
 
-	return xive_spapr_configure_queue(cpu, q, prio, qpage,
-					  xive_queue_shift);
+	return xive_spapr_configure_queue(get_hard_smp_processor_id(cpu),
+					  q, prio, qpage, xive_queue_shift);
 }
 
 static void xive_spapr_cleanup_queue(unsigned int cpu, struct xive_cpu *xc,
@@ -399,10 +401,12 @@ static void xive_spapr_cleanup_queue(unsigned int cpu, struct xive_cpu *xc,
 	struct xive_q *q = &xc->queue[prio];
 	unsigned int alloc_order;
 	long rc;
+	int hw_cpu = get_hard_smp_processor_id(cpu);
 
-	rc = plpar_int_set_queue_config(0, cpu, prio, 0, 0);
+	rc = plpar_int_set_queue_config(0, hw_cpu, prio, 0, 0);
 	if (rc)
-		pr_err("Error %ld setting queue for prio %d\n", rc, prio);
+		pr_err("Error %ld setting queue for CPU %d prio %d\n", rc,
+		       hw_cpu, prio);
 
 	alloc_order = xive_alloc_order(xive_queue_shift);
 	free_pages((unsigned long)q->qpage, alloc_order);
diff --git a/arch/powerpc/xmon/ppc-dis.c b/arch/powerpc/xmon/ppc-dis.c
index 31db8c072acd..9deea5ee13f6 100644
--- a/arch/powerpc/xmon/ppc-dis.c
+++ b/arch/powerpc/xmon/ppc-dis.c
@@ -93,10 +93,6 @@ lookup_powerpc (unsigned long insn, ppc_cpu_t dialect)
 {
   const struct powerpc_opcode *opcode;
   const struct powerpc_opcode *opcode_end;
-  unsigned long op;
-
-  /* Get the major opcode of the instruction.  */
-  op = PPC_OP (insn);
 
   opcode_end = powerpc_opcodes + powerpc_num_opcodes;
   /* Find the first match in the opcode table for this major opcode.  */
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 0ddc7ac6c5f1..82e1a3ee6e0f 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -1623,7 +1623,7 @@ static void excprint(struct pt_regs *fp)
 	printf("  current = 0x%lx\n", current);
 #ifdef CONFIG_PPC64
 	printf("  paca    = 0x%lx\t softe: %d\t irq_happened: 0x%02x\n",
-	       local_paca, local_paca->soft_enabled, local_paca->irq_happened);
+	       local_paca, local_paca->irq_soft_mask, local_paca->irq_happened);
 #endif
 	if (current) {
 		printf("    pid   = %ld, comm = %s\n",
@@ -2377,8 +2377,6 @@ static void dump_one_paca(int cpu)
 		printf(" slb_cache[%d]:        = 0x%016lx\n", i, p->slb_cache[i]);
 
 	DUMP(p, rfi_flush_fallback_area, "px");
-	DUMP(p, l1d_flush_congruence, "llx");
-	DUMP(p, l1d_flush_sets, "llx");
 #endif
 	DUMP(p, dscr_default, "llx");
 #ifdef CONFIG_PPC_BOOK3E
@@ -2395,7 +2393,7 @@ static void dump_one_paca(int cpu)
 	DUMP(p, stab_rr, "lx");
 	DUMP(p, saved_r1, "lx");
 	DUMP(p, trap_save, "x");
-	DUMP(p, soft_enabled, "x");
+	DUMP(p, irq_soft_mask, "x");
 	DUMP(p, irq_happened, "x");
 	DUMP(p, io_sync, "x");
 	DUMP(p, irq_work_pending, "x");
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 2c6adf12713a..04807c7f64cc 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -8,7 +8,6 @@ config RISCV
 	select OF
 	select OF_EARLY_FLATTREE
 	select OF_IRQ
-	select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
 	select ARCH_WANT_FRAME_POINTERS
 	select CLONE_BACKWARDS
 	select COMMON_CLK
@@ -20,8 +19,8 @@ config RISCV
 	select GENERIC_STRNLEN_USER
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_ATOMIC64 if !64BIT || !RISCV_ISA_A
-	select ARCH_WANT_OPTIONAL_GPIOLIB
 	select HAVE_MEMBLOCK
+	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
 	select HAVE_GENERIC_DMA_COHERENT
@@ -33,7 +32,6 @@ config RISCV
 	select HAVE_ARCH_TRACEHOOK
 	select MODULES_USE_ELF_RELA if MODULES
 	select THREAD_INFO_IN_TASK
-	select RISCV_IRQ_INTC
 	select RISCV_TIMER
 
 config MMU
@@ -43,6 +41,10 @@ config MMU
 config ARCH_PHYS_ADDR_T_64BIT
 	def_bool y
 
+config ZONE_DMA32
+	bool
+	default y
+
 config ARCH_DMA_ADDR_T_64BIT
 	def_bool y
 
@@ -55,6 +57,9 @@ config PAGE_OFFSET
 config STACKTRACE_SUPPORT
 	def_bool y
 
+config TRACE_IRQFLAGS_SUPPORT
+	def_bool y
+
 config RWSEM_GENERIC_SPINLOCK
 	def_bool y
 
@@ -83,7 +88,7 @@ config PGTABLE_LEVELS
 config HAVE_KPROBES
 	def_bool n
 
-config DMA_NOOP_OPS
+config DMA_DIRECT_OPS
 	def_bool y
 
 menu "Platform type"
@@ -107,6 +112,8 @@ config ARCH_RV64I
 	bool "RV64I"
 	select CPU_SUPPORTS_64BIT_KERNEL
 	select 64BIT
+	select HAVE_FUNCTION_TRACER
+	select HAVE_FUNCTION_GRAPH_TRACER
 
 endchoice
 
diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild
index 970460a0b492..4286a5f83876 100644
--- a/arch/riscv/include/asm/Kbuild
+++ b/arch/riscv/include/asm/Kbuild
@@ -1,18 +1,17 @@
 generic-y += bugs.h
 generic-y += cacheflush.h
 generic-y += checksum.h
-generic-y += clkdev.h
 generic-y += cputime.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += dma.h
 generic-y += dma-contiguous.h
+generic-y += dma-mapping.h
 generic-y += emergency-restart.h
 generic-y += errno.h
 generic-y += exec.h
 generic-y += fb.h
 generic-y += fcntl.h
-generic-y += ftrace.h
 generic-y += futex.h
 generic-y += hardirq.h
 generic-y += hash.h
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index c0319cbf1eec..5510366d169a 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -34,9 +34,9 @@
 #define wmb()		RISCV_FENCE(ow,ow)
 
 /* These barriers do not need to enforce ordering on devices, just memory. */
-#define smp_mb()	RISCV_FENCE(rw,rw)
-#define smp_rmb()	RISCV_FENCE(r,r)
-#define smp_wmb()	RISCV_FENCE(w,w)
+#define __smp_mb()	RISCV_FENCE(rw,rw)
+#define __smp_rmb()	RISCV_FENCE(r,r)
+#define __smp_wmb()	RISCV_FENCE(w,w)
 
 /*
  * This is a very specific barrier: it's currently only used in two places in
diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h
index 3c7a2c97e377..421fa3585798 100644
--- a/arch/riscv/include/asm/csr.h
+++ b/arch/riscv/include/asm/csr.h
@@ -40,15 +40,15 @@
 #define SR_SD   _AC(0x8000000000000000, UL) /* FS/XS dirty */
 #endif
 
-/* SPTBR flags */
+/* SATP flags */
 #if __riscv_xlen == 32
-#define SPTBR_PPN     _AC(0x003FFFFF, UL)
-#define SPTBR_MODE_32 _AC(0x80000000, UL)
-#define SPTBR_MODE    SPTBR_MODE_32
+#define SATP_PPN     _AC(0x003FFFFF, UL)
+#define SATP_MODE_32 _AC(0x80000000, UL)
+#define SATP_MODE    SATP_MODE_32
 #else
-#define SPTBR_PPN     _AC(0x00000FFFFFFFFFFF, UL)
-#define SPTBR_MODE_39 _AC(0x8000000000000000, UL)
-#define SPTBR_MODE    SPTBR_MODE_39
+#define SATP_PPN     _AC(0x00000FFFFFFFFFFF, UL)
+#define SATP_MODE_39 _AC(0x8000000000000000, UL)
+#define SATP_MODE    SATP_MODE_39
 #endif
 
 /* Interrupt Enable and Interrupt Pending flags */
diff --git a/arch/riscv/include/asm/dma-mapping.h b/arch/riscv/include/asm/dma-mapping.h
deleted file mode 100644
index 3eec1000196d..000000000000
--- a/arch/riscv/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (C) 2003-2004 Hewlett-Packard Co
- *	David Mosberger-Tang <davidm@hpl.hp.com>
- * Copyright (C) 2012 ARM Ltd.
- * Copyright (C) 2016 SiFive, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef __ASM_RISCV_DMA_MAPPING_H
-#define __ASM_RISCV_DMA_MAPPING_H
-
-/* Use ops->dma_mapping_error (if it exists) or assume success */
-// #undef DMA_ERROR_CODE
-
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &dma_noop_ops;
-}
-
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return false;
-
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
-#endif	/* __ASM_RISCV_DMA_MAPPING_H */
diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h
new file mode 100644
index 000000000000..66d4175eb13e
--- /dev/null
+++ b/arch/riscv/include/asm/ftrace.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2017 Andes Technology Corporation */
+
+/*
+ * The graph frame test is not possible if CONFIG_FRAME_POINTER is not enabled.
+ * Check arch/riscv/kernel/mcount.S for detail.
+ */
+#if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_FRAME_POINTER)
+#define HAVE_FUNCTION_GRAPH_FP_TEST
+#endif
diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h
index 97424834dce2..336d60ec5698 100644
--- a/arch/riscv/include/asm/mmu_context.h
+++ b/arch/riscv/include/asm/mmu_context.h
@@ -39,16 +39,6 @@ static inline void destroy_context(struct mm_struct *mm)
 {
 }
 
-static inline pgd_t *current_pgdir(void)
-{
-	return pfn_to_virt(csr_read(sptbr) & SPTBR_PPN);
-}
-
-static inline void set_pgdir(pgd_t *pgd)
-{
-	csr_write(sptbr, virt_to_pfn(pgd) | SPTBR_MODE);
-}
-
 /*
  * When necessary, performs a deferred icache flush for the given MM context,
  * on the local CPU.  RISC-V has no direct mechanism for instruction cache
@@ -93,7 +83,12 @@ static inline void switch_mm(struct mm_struct *prev,
 		cpumask_clear_cpu(cpu, mm_cpumask(prev));
 		cpumask_set_cpu(cpu, mm_cpumask(next));
 
-		set_pgdir(next->pgd);
+		/*
+		 * Use the old spbtr name instead of using the current satp
+		 * name to support binutils 2.29 which doesn't know about the
+		 * privileged ISA 1.10 yet.
+		 */
+		csr_write(sptbr, virt_to_pfn(next->pgd) | SATP_MODE);
 		local_flush_tlb_all();
 
 		flush_icache_deferred(next);
diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index 22c3536ed281..f8fa1cd2dad9 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -64,8 +64,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_stack		(init_thread_union.stack)
-
 #endif /* !__ASSEMBLY__ */
 
 /*
diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h
index 7b9c24ebdf52..7b209aec355d 100644
--- a/arch/riscv/include/asm/tlbflush.h
+++ b/arch/riscv/include/asm/tlbflush.h
@@ -36,7 +36,14 @@ static inline void local_flush_tlb_page(unsigned long addr)
 
 #define flush_tlb_all() local_flush_tlb_all()
 #define flush_tlb_page(vma, addr) local_flush_tlb_page(addr)
-#define flush_tlb_range(vma, start, end) local_flush_tlb_all()
+
+static inline void flush_tlb_range(struct vm_area_struct *vma,
+		unsigned long start, unsigned long end)
+{
+	local_flush_tlb_all();
+}
+
+#define flush_tlb_mm(mm) flush_tlb_all()
 
 #else /* CONFIG_SMP */
 
@@ -45,16 +52,13 @@ static inline void local_flush_tlb_page(unsigned long addr)
 #define flush_tlb_all() sbi_remote_sfence_vma(0, 0, -1)
 #define flush_tlb_page(vma, addr) flush_tlb_range(vma, addr, 0)
 #define flush_tlb_range(vma, start, end) \
-	sbi_remote_sfence_vma(0, start, (end) - (start))
+	sbi_remote_sfence_vma(mm_cpumask((vma)->vm_mm)->bits, \
+			      start, (end) - (start))
+#define flush_tlb_mm(mm) \
+	sbi_remote_sfence_vma(mm_cpumask(mm)->bits, 0, -1)
 
 #endif /* CONFIG_SMP */
 
-/* Flush the TLB entries of the specified mm context */
-static inline void flush_tlb_mm(struct mm_struct *mm)
-{
-	flush_tlb_all();
-}
-
 /* Flush a range of kernel pages */
 static inline void flush_tlb_kernel_range(unsigned long start,
 	unsigned long end)
diff --git a/arch/riscv/include/asm/unistd.h b/arch/riscv/include/asm/unistd.h
index 2f704a5c4196..080fb28061de 100644
--- a/arch/riscv/include/asm/unistd.h
+++ b/arch/riscv/include/asm/unistd.h
@@ -11,7 +11,6 @@
  *   GNU General Public License for more details.
  */
 
-#define __ARCH_HAVE_MMU
 #define __ARCH_WANT_SYS_CLONE
 #include <uapi/asm/unistd.h>
 #include <uapi/asm/syscalls.h>
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index ab8baf7bd142..196f62ffc428 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -2,6 +2,11 @@
 # Makefile for the RISC-V Linux kernel
 #
 
+ifdef CONFIG_FTRACE
+CFLAGS_REMOVE_ftrace.o = -pg
+CFLAGS_REMOVE_setup.o = -pg
+endif
+
 extra-y += head.o
 extra-y += vmlinux.lds
 
@@ -29,5 +34,7 @@ CFLAGS_setup.o := -mcmodel=medany
 obj-$(CONFIG_SMP)		+= smpboot.o
 obj-$(CONFIG_SMP)		+= smp.o
 obj-$(CONFIG_MODULES)		+= module.o
+obj-$(CONFIG_FUNCTION_TRACER)	+= mcount.o
+obj-$(CONFIG_FUNCTION_GRAPH_TRACER)	+= ftrace.o
 
 clean:
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 7404ec222406..56fa592cfa34 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -78,10 +78,13 @@ _save_context:
 	REG_S x31, PT_T6(sp)
 
 	/*
-	 * Disable FPU to detect illegal usage of
-	 * floating point in kernel space
+	 * Disable user-mode memory access as it should only be set in the
+	 * actual user copy routines.
+	 *
+	 * Disable the FPU to detect illegal usage of floating point in kernel
+	 * space.
 	 */
-	li t0, SR_FS
+	li t0, SR_SUM | SR_FS
 
 	REG_L s0, TASK_TI_USER_SP(tp)
 	csrrc s1, sstatus, t0
@@ -169,6 +172,9 @@ ENTRY(handle_exception)
 	move a1, sp /* pt_regs */
 	tail do_IRQ
 1:
+	/* Exceptions run with interrupts enabled */
+	csrs sstatus, SR_SIE
+
 	/* Handle syscalls */
 	li t0, EXC_SYSCALL
 	beq s4, t0, handle_syscall
@@ -195,8 +201,6 @@ handle_syscall:
 	 */
 	addi s2, s2, 0x4
 	REG_S s2, PT_SEPC(sp)
-	/* System calls run with interrupts enabled */
-	csrs sstatus, SR_SIE
 	/* Trace syscalls, but only if requested by the user. */
 	REG_L t0, TASK_TI_FLAGS(tp)
 	andi t0, t0, _TIF_SYSCALL_TRACE
diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c
new file mode 100644
index 000000000000..d0de68d144cb
--- /dev/null
+++ b/arch/riscv/kernel/ftrace.c
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2013 Linaro Limited
+ * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
+ * Copyright (C) 2017 Andes Technology Corporation
+ */
+
+#include <linux/ftrace.h>
+
+/*
+ * Most of this file is copied from arm64.
+ */
+void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
+			   unsigned long frame_pointer)
+{
+	unsigned long return_hooker = (unsigned long)&return_to_handler;
+	unsigned long old;
+	struct ftrace_graph_ent trace;
+	int err;
+
+	if (unlikely(atomic_read(&current->tracing_graph_pause)))
+		return;
+
+	/*
+	 * We don't suffer access faults, so no extra fault-recovery assembly
+	 * is needed here.
+	 */
+	old = *parent;
+
+	trace.func = self_addr;
+	trace.depth = current->curr_ret_stack + 1;
+
+	if (!ftrace_graph_entry(&trace))
+		return;
+
+	err = ftrace_push_return_trace(old, self_addr, &trace.depth,
+				       frame_pointer, NULL);
+	if (err == -EBUSY)
+		return;
+	*parent = return_hooker;
+}
diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index 78f670d70133..6e07ed37bbff 100644
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -64,7 +64,7 @@ ENTRY(_start)
 	/* Start the kernel */
 	mv a0, s0
 	mv a1, s1
-	call sbi_save
+	call parse_dtb
 	tail start_kernel
 
 relocate:
@@ -74,15 +74,15 @@ relocate:
 	sub a1, a1, a0
 	add ra, ra, a1
 
-	/* Point stvec to virtual address of intruction after sptbr write */
+	/* Point stvec to virtual address of intruction after satp write */
 	la a0, 1f
 	add a0, a0, a1
 	csrw stvec, a0
 
-	/* Compute sptbr for kernel page tables, but don't load it yet */
+	/* Compute satp for kernel page tables, but don't load it yet */
 	la a2, swapper_pg_dir
 	srl a2, a2, PAGE_SHIFT
-	li a1, SPTBR_MODE
+	li a1, SATP_MODE
 	or a2, a2, a1
 
 	/*
diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S
new file mode 100644
index 000000000000..c46a778627be
--- /dev/null
+++ b/arch/riscv/kernel/mcount.S
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2017 Andes Technology Corporation */
+
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <asm/asm.h>
+#include <asm/csr.h>
+#include <asm/unistd.h>
+#include <asm/thread_info.h>
+#include <asm/asm-offsets.h>
+#include <asm-generic/export.h>
+#include <asm/ftrace.h>
+
+	.text
+
+	.macro SAVE_ABI_STATE
+	addi	sp, sp, -16
+	sd	s0, 0(sp)
+	sd	ra, 8(sp)
+	addi	s0, sp, 16
+	.endm
+
+	/*
+	 * The call to ftrace_return_to_handler would overwrite the return
+	 * register if a0 was not saved.
+	 */
+	.macro SAVE_RET_ABI_STATE
+	addi	sp, sp, -32
+	sd	s0, 16(sp)
+	sd	ra, 24(sp)
+	sd	a0, 8(sp)
+	addi	s0, sp, 32
+	.endm
+
+	.macro STORE_ABI_STATE
+	ld	ra, 8(sp)
+	ld	s0, 0(sp)
+	addi	sp, sp, 16
+	.endm
+
+	.macro STORE_RET_ABI_STATE
+	ld	ra, 24(sp)
+	ld	s0, 16(sp)
+	ld	a0, 8(sp)
+	addi	sp, sp, 32
+	.endm
+
+ENTRY(ftrace_stub)
+	ret
+ENDPROC(ftrace_stub)
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ENTRY(return_to_handler)
+/*
+ * On implementing the frame point test, the ideal way is to compare the
+ * s0 (frame pointer, if enabled) on entry and the sp (stack pointer) on return.
+ * However, the psABI of variable-length-argument functions does not allow this.
+ *
+ * So alternatively we check the *old* frame pointer position, that is, the
+ * value stored in -16(s0) on entry, and the s0 on return.
+ */
+#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
+	mv	t6, s0
+#endif
+	SAVE_RET_ABI_STATE
+#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
+	mv	a0, t6
+#endif
+	la	t0, ftrace_return_to_handler
+	jalr	t0
+	mv	a1, a0
+	STORE_RET_ABI_STATE
+	jalr	a1
+ENDPROC(return_to_handler)
+EXPORT_SYMBOL(return_to_handler)
+#endif
+
+ENTRY(_mcount)
+	la	t4, ftrace_stub
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+	la	t0, ftrace_graph_return
+	ld	t1, 0(t0)
+	bne	t1, t4, do_ftrace_graph_caller
+
+	la	t3, ftrace_graph_entry
+	ld	t2, 0(t3)
+	la	t6, ftrace_graph_entry_stub
+	bne	t2, t6, do_ftrace_graph_caller
+#endif
+	la	t3, ftrace_trace_function
+	ld	t5, 0(t3)
+	bne	t5, t4, do_trace
+	ret
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+/*
+ * A pseudo representation for the function graph tracer:
+ * prepare_to_return(&ra_to_caller_of_caller, ra_to_caller)
+ */
+do_ftrace_graph_caller:
+	addi	a0, s0, -8
+	mv	a1, ra
+#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
+	ld	a2, -16(s0)
+#endif
+	SAVE_ABI_STATE
+	la	t0, prepare_ftrace_return
+	jalr	t0
+	STORE_ABI_STATE
+	ret
+#endif
+
+/*
+ * A pseudo representation for the function tracer:
+ * (*ftrace_trace_function)(ra_to_caller, ra_to_caller_of_caller)
+ */
+do_trace:
+	ld	a1, -8(s0)
+	mv	a0, ra
+
+	SAVE_ABI_STATE
+	jalr	t5
+	STORE_ABI_STATE
+	ret
+ENDPROC(_mcount)
+EXPORT_SYMBOL(_mcount)
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index d74d4adf2d54..d7c6ca7c95ae 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -25,9 +25,9 @@
 #include <linux/sched/task_stack.h>
 #include <linux/tick.h>
 #include <linux/ptrace.h>
+#include <linux/uaccess.h>
 
 #include <asm/unistd.h>
-#include <asm/uaccess.h>
 #include <asm/processor.h>
 #include <asm/csr.h>
 #include <asm/string.h>
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index cb7b0c63014e..c11f40c1b2a8 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -49,10 +49,6 @@ struct screen_info screen_info = {
 };
 #endif
 
-#ifdef CONFIG_CMDLINE_BOOL
-static char __initdata builtin_cmdline[COMMAND_LINE_SIZE] = CONFIG_CMDLINE;
-#endif /* CONFIG_CMDLINE_BOOL */
-
 unsigned long va_pa_offset;
 EXPORT_SYMBOL(va_pa_offset);
 unsigned long pfn_base;
@@ -148,30 +144,11 @@ asmlinkage void __init setup_vm(void)
 #endif
 }
 
-void __init sbi_save(unsigned int hartid, void *dtb)
+void __init parse_dtb(unsigned int hartid, void *dtb)
 {
 	early_init_dt_scan(__va(dtb));
 }
 
-/*
- * Allow the user to manually add a memory region (in case DTS is broken);
- * "mem_end=nn[KkMmGg]"
- */
-static int __init mem_end_override(char *p)
-{
-	resource_size_t base, end;
-
-	if (!p)
-		return -EINVAL;
-	base = (uintptr_t) __pa(PAGE_OFFSET);
-	end = memparse(p, &p) & PMD_MASK;
-	if (end == 0)
-		return -EINVAL;
-	memblock_add(base, end - base);
-	return 0;
-}
-early_param("mem_end", mem_end_override);
-
 static void __init setup_bootmem(void)
 {
 	struct memblock_region *reg;
@@ -204,22 +181,19 @@ static void __init setup_bootmem(void)
 	early_init_fdt_scan_reserved_mem();
 	memblock_allow_resize();
 	memblock_dump_all();
+
+	for_each_memblock(memory, reg) {
+		unsigned long start_pfn = memblock_region_memory_base_pfn(reg);
+		unsigned long end_pfn = memblock_region_memory_end_pfn(reg);
+
+		memblock_set_node(PFN_PHYS(start_pfn),
+		                  PFN_PHYS(end_pfn - start_pfn),
+		                  &memblock.memory, 0);
+	}
 }
 
 void __init setup_arch(char **cmdline_p)
 {
-#ifdef CONFIG_CMDLINE_BOOL
-#ifdef CONFIG_CMDLINE_OVERRIDE
-	strlcpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
-#else
-	if (builtin_cmdline[0] != '\0') {
-		/* Append bootloader command line to built-in */
-		strlcat(builtin_cmdline, " ", COMMAND_LINE_SIZE);
-		strlcat(builtin_cmdline, boot_command_line, COMMAND_LINE_SIZE);
-		strlcpy(boot_command_line, builtin_cmdline, COMMAND_LINE_SIZE);
-	}
-#endif /* CONFIG_CMDLINE_OVERRIDE */
-#endif /* CONFIG_CMDLINE_BOOL */
 	*cmdline_p = boot_command_line;
 
 	parse_early_param();
diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c
index e8a178df8144..582cb153eb24 100644
--- a/arch/riscv/kernel/vdso.c
+++ b/arch/riscv/kernel/vdso.c
@@ -74,7 +74,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm,
 
 	down_write(&mm->mmap_sem);
 	vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0);
-	if (unlikely(IS_ERR_VALUE(vdso_base))) {
+	if (IS_ERR_VALUE(vdso_base)) {
 		ret = vdso_base;
 		goto end;
 	}
diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c
index 0713f3c67ab4..148c98ca9b45 100644
--- a/arch/riscv/mm/fault.c
+++ b/arch/riscv/mm/fault.c
@@ -29,7 +29,6 @@
 
 #include <asm/pgalloc.h>
 #include <asm/ptrace.h>
-#include <asm/uaccess.h>
 
 /*
  * This routine handles page faults.  It determines the address and the
@@ -239,6 +238,10 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
 		 * Do _not_ use "tsk->active_mm->pgd" here.
 		 * We might be inside an interrupt in the middle
 		 * of a task switch.
+		 *
+		 * Note: Use the old spbtr name instead of using the current
+		 * satp name to support binutils 2.29 which doesn't know about
+		 * the privileged ISA 1.10 yet.
 		 */
 		index = pgd_index(addr);
 		pgd = (pgd_t *)pfn_to_virt(csr_read(sptbr)) + index;
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index 9f4bee5e51fd..c77df8142be2 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -17,6 +17,7 @@
 #include <linux/initrd.h>
 #include <linux/memblock.h>
 #include <linux/swap.h>
+#include <linux/sizes.h>
 
 #include <asm/tlbflush.h>
 #include <asm/sections.h>
@@ -25,11 +26,12 @@
 
 static void __init zone_sizes_init(void)
 {
-	unsigned long zones_size[MAX_NR_ZONES];
+	unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, };
 
-	memset(zones_size, 0, sizeof(zones_size));
-	zones_size[ZONE_NORMAL] = max_mapnr;
-	free_area_init_node(0, zones_size, pfn_base, NULL);
+	max_zone_pfns[ZONE_DMA32] = PFN_DOWN(min(4UL * SZ_1G, max_low_pfn));
+	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
+
+	free_area_init_nodes(max_zone_pfns);
 }
 
 void setup_zero_page(void)
@@ -39,8 +41,6 @@ void setup_zero_page(void)
 
 void __init paging_init(void)
 {
-	init_mm.pgd = (pgd_t *)pfn_to_virt(csr_read(sptbr));
-
 	setup_zero_page();
 	local_flush_tlb_all();
 	zone_sizes_init();
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 829c67986db7..eaee7087886f 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -108,12 +108,10 @@ config S390
 	select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
 	select ARCH_SAVE_PAGE_KEYS if HIBERNATION
 	select ARCH_SUPPORTS_ATOMIC_RMW
-	select ARCH_SUPPORTS_DEFERRED_STRUCT_PAGE_INIT
 	select ARCH_SUPPORTS_NUMA_BALANCING
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_CMPXCHG_LOCKREF
 	select ARCH_WANTS_DYNAMIC_TASK_STRUCT
-	select ARCH_WANTS_PROT_NUMA_PROT_NONE
 	select ARCH_WANTS_UBSAN_NO_NULL
 	select ARCH_WANT_IPC_PARSE_VERSION
 	select BUILDTIME_EXTABLE_SORT
@@ -140,7 +138,7 @@ config S390
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
-	select DMA_NOOP_OPS
+	select DMA_DIRECT_OPS
 	select HAVE_DYNAMIC_FTRACE
 	select HAVE_DYNAMIC_FTRACE_WITH_REGS
 	select HAVE_EFFICIENT_UNALIGNED_ACCESS
@@ -541,6 +539,51 @@ config ARCH_RANDOM
 
 	  If unsure, say Y.
 
+config KERNEL_NOBP
+	def_bool n
+	prompt "Enable modified branch prediction for the kernel by default"
+	help
+	  If this option is selected the kernel will switch to a modified
+	  branch prediction mode if the firmware interface is available.
+	  The modified branch prediction mode improves the behaviour in
+	  regard to speculative execution.
+
+	  With the option enabled the kernel parameter "nobp=0" or "nospec"
+	  can be used to run the kernel in the normal branch prediction mode.
+
+	  With the option disabled the modified branch prediction mode is
+	  enabled with the "nobp=1" kernel parameter.
+
+	  If unsure, say N.
+
+config EXPOLINE
+	def_bool n
+	prompt "Avoid speculative indirect branches in the kernel"
+	help
+	  Compile the kernel with the expoline compiler options to guard
+	  against kernel-to-user data leaks by avoiding speculative indirect
+	  branches.
+	  Requires a compiler with -mindirect-branch=thunk support for full
+	  protection. The kernel may run slower.
+
+	  If unsure, say N.
+
+choice
+	prompt "Expoline default"
+	depends on EXPOLINE
+	default EXPOLINE_FULL
+
+config EXPOLINE_OFF
+	bool "spectre_v2=off"
+
+config EXPOLINE_MEDIUM
+	bool "spectre_v2=auto"
+
+config EXPOLINE_FULL
+	bool "spectre_v2=on"
+
+endchoice
+
 endmenu
 
 menu "Memory setup"
diff --git a/arch/s390/Makefile b/arch/s390/Makefile
index de54cfc6109d..2ced3239cb84 100644
--- a/arch/s390/Makefile
+++ b/arch/s390/Makefile
@@ -78,6 +78,16 @@ ifeq ($(call cc-option-yn,-mwarn-dynamicstack),y)
 cflags-$(CONFIG_WARN_DYNAMIC_STACK) += -mwarn-dynamicstack
 endif
 
+ifdef CONFIG_EXPOLINE
+  ifeq ($(call cc-option-yn,$(CC_FLAGS_MARCH) -mindirect-branch=thunk),y)
+    CC_FLAGS_EXPOLINE := -mindirect-branch=thunk
+    CC_FLAGS_EXPOLINE += -mfunction-return=thunk
+    CC_FLAGS_EXPOLINE += -mindirect-branch-table
+    export CC_FLAGS_EXPOLINE
+    cflags-y += $(CC_FLAGS_EXPOLINE)
+  endif
+endif
+
 ifdef CONFIG_FUNCTION_TRACER
 # make use of hotpatch feature if the compiler supports it
 cc_hotpatch	:= -mhotpatch=0,3
@@ -88,9 +98,13 @@ KBUILD_CFLAGS	+= -DCC_USING_HOTPATCH
 endif
 endif
 
+# Test CFI features of binutils
+cfi := $(call as-instr,.cfi_startproc\n.cfi_val_offset 15$(comma)-160\n.cfi_endproc,-DCONFIG_AS_CFI_VAL_OFFSET=1)
+
 KBUILD_CFLAGS	+= -mbackchain -msoft-float $(cflags-y)
 KBUILD_CFLAGS	+= -pipe -fno-strength-reduce -Wno-sign-compare
-KBUILD_AFLAGS	+= $(aflags-y)
+KBUILD_CFLAGS	+= -fno-asynchronous-unwind-tables $(cfi)
+KBUILD_AFLAGS	+= $(aflags-y) $(cfi)
 
 OBJCOPYFLAGS	:= -O binary
 
@@ -107,6 +121,7 @@ drivers-y	+= drivers/s390/
 drivers-$(CONFIG_OPROFILE)	+= arch/s390/oprofile/
 
 boot		:= arch/s390/boot
+syscalls	:= arch/s390/kernel/syscalls
 tools		:= arch/s390/tools
 
 all: image bzImage
@@ -128,9 +143,12 @@ archclean:
 	$(Q)$(MAKE) $(clean)=$(boot)
 	$(Q)$(MAKE) $(clean)=$(tools)
 
+archheaders:
+	$(Q)$(MAKE) $(build)=$(syscalls) uapi
+
 archprepare:
-	$(Q)$(MAKE) $(build)=$(tools) include/generated/facilities.h
-	$(Q)$(MAKE) $(build)=$(tools) include/generated/dis.h
+	$(Q)$(MAKE) $(build)=$(syscalls) kapi
+	$(Q)$(MAKE) $(build)=$(tools) kapi
 
 # Don't use tabs in echo arguments
 define archhelp
diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile
index 29e3dc99b916..26d6a94f40f6 100644
--- a/arch/s390/boot/compressed/Makefile
+++ b/arch/s390/boot/compressed/Makefile
@@ -14,6 +14,7 @@ targets += misc.o piggy.o sizes.h head.o
 KBUILD_CFLAGS := -m64 -D__KERNEL__ -O2
 KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY
 KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks -msoft-float
+KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 KBUILD_CFLAGS += $(call cc-option,-mpacked-stack)
 KBUILD_CFLAGS += $(call cc-option,-ffreestanding)
 
diff --git a/arch/s390/boot/compressed/vmlinux.lds.S b/arch/s390/boot/compressed/vmlinux.lds.S
index 88e46d4a7784..8150132b144f 100644
--- a/arch/s390/boot/compressed/vmlinux.lds.S
+++ b/arch/s390/boot/compressed/vmlinux.lds.S
@@ -17,15 +17,15 @@ SECTIONS
 		HEAD_TEXT
 		_ehead = . ;
 	}
-	.rodata.compressed : {
-		*(.rodata.compressed)
-	}
 	.text :	{
 		_text = .;	/* Text */
 		*(.text)
 		*(.text.*)
 		_etext = . ;
 	}
+	.rodata.compressed : {
+		*(.rodata.compressed)
+	}
 	.rodata : {
 		_rodata = . ;
 		*(.rodata)	 /* read-only data */
@@ -48,4 +48,10 @@ SECTIONS
 		_ebss = .;
 	}
 	_end = .;
+
+	/* Sections to be discarded */
+	/DISCARD/ : {
+		*(.eh_frame)
+		*(*__ksymtab*)
+	}
 }
diff --git a/arch/s390/configs/zfcpdump_defconfig b/arch/s390/configs/zfcpdump_defconfig
index 04e042edbab7..7dc7f58c4287 100644
--- a/arch/s390/configs/zfcpdump_defconfig
+++ b/arch/s390/configs/zfcpdump_defconfig
@@ -26,7 +26,6 @@ CONFIG_NET=y
 # CONFIG_IUCV is not set
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_BLK_DEV_RAM=y
 # CONFIG_BLK_DEV_XPRAM is not set
 # CONFIG_DCSSBLK is not set
diff --git a/arch/s390/crypto/crc32-vx.c b/arch/s390/crypto/crc32-vx.c
index 436865926c26..423ee05887e6 100644
--- a/arch/s390/crypto/crc32-vx.c
+++ b/arch/s390/crypto/crc32-vx.c
@@ -239,6 +239,7 @@ static struct shash_alg crc32_vx_algs[] = {
 			.cra_name	 = "crc32",
 			.cra_driver_name = "crc32-vx",
 			.cra_priority	 = 200,
+			.cra_flags	 = CRYPTO_ALG_OPTIONAL_KEY,
 			.cra_blocksize	 = CRC32_BLOCK_SIZE,
 			.cra_ctxsize	 = sizeof(struct crc_ctx),
 			.cra_module	 = THIS_MODULE,
@@ -259,6 +260,7 @@ static struct shash_alg crc32_vx_algs[] = {
 			.cra_name	 = "crc32be",
 			.cra_driver_name = "crc32be-vx",
 			.cra_priority	 = 200,
+			.cra_flags	 = CRYPTO_ALG_OPTIONAL_KEY,
 			.cra_blocksize	 = CRC32_BLOCK_SIZE,
 			.cra_ctxsize	 = sizeof(struct crc_ctx),
 			.cra_module	 = THIS_MODULE,
@@ -279,6 +281,7 @@ static struct shash_alg crc32_vx_algs[] = {
 			.cra_name	 = "crc32c",
 			.cra_driver_name = "crc32c-vx",
 			.cra_priority	 = 200,
+			.cra_flags	 = CRYPTO_ALG_OPTIONAL_KEY,
 			.cra_blocksize	 = CRC32_BLOCK_SIZE,
 			.cra_ctxsize	 = sizeof(struct crc_ctx),
 			.cra_module	 = THIS_MODULE,
diff --git a/arch/s390/include/asm/Kbuild b/arch/s390/include/asm/Kbuild
index 048450869328..e3239772887a 100644
--- a/arch/s390/include/asm/Kbuild
+++ b/arch/s390/include/asm/Kbuild
@@ -1,9 +1,14 @@
 # SPDX-License-Identifier: GPL-2.0
+generated-y += dis-defs.h
+generated-y += facility-defs.h
+generated-y += syscall_table.h
+generated-y += unistd_nr.h
+
 generic-y += asm-offsets.h
 generic-y += cacheflush.h
-generic-y += clkdev.h
 generic-y += device.h
 generic-y += dma-contiguous.h
+generic-y += dma-mapping.h
 generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += export.h
diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h
index 10432607a573..f9eddbca79d2 100644
--- a/arch/s390/include/asm/barrier.h
+++ b/arch/s390/include/asm/barrier.h
@@ -49,6 +49,30 @@ do {									\
 #define __smp_mb__before_atomic()	barrier()
 #define __smp_mb__after_atomic()	barrier()
 
+/**
+ * array_index_mask_nospec - generate a mask for array_idx() that is
+ * ~0UL when the bounds check succeeds and 0 otherwise
+ * @index: array element index
+ * @size: number of elements in array
+ */
+#define array_index_mask_nospec array_index_mask_nospec
+static inline unsigned long array_index_mask_nospec(unsigned long index,
+						    unsigned long size)
+{
+	unsigned long mask;
+
+	if (__builtin_constant_p(size) && size > 0) {
+		asm("	clgr	%2,%1\n"
+		    "	slbgr	%0,%0\n"
+		    :"=d" (mask) : "d" (size-1), "d" (index) :"cc");
+		return mask;
+	}
+	asm("	clgr	%1,%2\n"
+	    "	slbgr	%0,%0\n"
+	    :"=d" (mask) : "d" (size), "d" (index) :"cc");
+	return ~mask;
+}
+
 #include <asm-generic/barrier.h>
 
 #endif /* __ASM_BARRIER_H */
diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h
index 31e400c1a1f3..86e5b2fdee3c 100644
--- a/arch/s390/include/asm/bitops.h
+++ b/arch/s390/include/asm/bitops.h
@@ -261,6 +261,11 @@ static inline void clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 	return clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
 }
 
+static inline int test_and_clear_bit_inv(unsigned long nr, volatile unsigned long *ptr)
+{
+	return test_and_clear_bit(nr ^ (BITS_PER_LONG - 1), ptr);
+}
+
 static inline void __set_bit_inv(unsigned long nr, volatile unsigned long *ptr)
 {
 	return __set_bit(nr ^ (BITS_PER_LONG - 1), ptr);
diff --git a/arch/s390/include/asm/compat.h b/arch/s390/include/asm/compat.h
index 5e6a63641a5f..9830fb6b076e 100644
--- a/arch/s390/include/asm/compat.h
+++ b/arch/s390/include/asm/compat.h
@@ -189,79 +189,6 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32		compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-typedef struct compat_siginfo {
-	int	si_signo;
-	int	si_errno;
-	int	si_code;
-
-	union {
-		int _pad[128/sizeof(int) - 3];
-
-		/* kill() */
-		struct {
-			pid_t	_pid;	/* sender's pid */
-			uid_t	_uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;		/* timer id */
-			int _overrun;			/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			pid_t			_pid;	/* sender's pid */
-			uid_t			_uid;	/* sender's uid */
-			compat_sigval_t		_sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			pid_t			_pid;	/* which child */
-			uid_t			_uid;	/* sender's uid */
-			int			_status;/* exit code */
-			compat_clock_t		_utime;
-			compat_clock_t		_stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			__u32	_addr;	/* faulting insn/memory ref. - pointer */
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int	_band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int	_fd;
-		} _sigpoll;
-	} _sifields;
-} compat_siginfo_t;
-
-/*
- * How these fields are to be accessed.
- */
-#define si_pid		_sifields._kill._pid
-#define si_uid		_sifields._kill._uid
-#define si_status	_sifields._sigchld._status
-#define si_utime	_sifields._sigchld._utime
-#define si_stime	_sifields._sigchld._stime
-#define si_value	_sifields._rt._sigval
-#define si_int		_sifields._rt._sigval.sival_int
-#define si_ptr		_sifields._rt._sigval.sival_ptr
-#define si_addr		_sifields._sigfault._addr
-#define si_band		_sifields._sigpoll._band
-#define si_fd		_sifields._sigpoll._fd
-#define si_tid		_sifields._timer._tid
-#define si_overrun	_sifields._timer._overrun
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 /*
diff --git a/arch/s390/include/asm/css_chars.h b/arch/s390/include/asm/css_chars.h
index a478eb61aaf7..fb56fa3283a2 100644
--- a/arch/s390/include/asm/css_chars.h
+++ b/arch/s390/include/asm/css_chars.h
@@ -20,7 +20,9 @@ struct css_general_char {
 	u32 aif_tdd : 1; /* bit 56 */
 	u32 : 1;
 	u32 qebsm : 1;	 /* bit 58 */
-	u32 : 8;
+	u32 : 2;
+	u32 aiv : 1;     /* bit 61 */
+	u32 : 5;
 	u32 aif_osa : 1; /* bit 67 */
 	u32 : 12;
 	u32 eadm_rf : 1; /* bit 80 */
diff --git a/arch/s390/include/asm/diag.h b/arch/s390/include/asm/diag.h
index 6db78567294c..cdbaad50c7c7 100644
--- a/arch/s390/include/asm/diag.h
+++ b/arch/s390/include/asm/diag.h
@@ -229,13 +229,55 @@ struct diag204_x_phys_block {
 } __packed;
 
 enum diag26c_sc {
+	DIAG26C_PORT_VNIC    = 0x00000024,
 	DIAG26C_MAC_SERVICES = 0x00000030
 };
 
 enum diag26c_version {
-	DIAG26C_VERSION2 = 0x00000002	/* z/VM 5.4.0 */
+	DIAG26C_VERSION2	 = 0x00000002,	/* z/VM 5.4.0 */
+	DIAG26C_VERSION6_VM65918 = 0x00020006	/* z/VM 6.4.0 + VM65918 */
 };
 
+#define DIAG26C_VNIC_INFO	0x0002
+struct diag26c_vnic_req {
+	u32	resp_buf_len;
+	u32	resp_version;
+	u16	req_format;
+	u16	vlan_id;
+	u64	sys_name;
+	u8	res[2];
+	u16	devno;
+} __packed __aligned(8);
+
+#define VNIC_INFO_PROT_L3	1
+#define VNIC_INFO_PROT_L2	2
+/* Note: this is the bare minimum, use it for uninitialized VNICs only. */
+struct diag26c_vnic_resp {
+	u32	version;
+	u32	entry_cnt;
+	/* VNIC info: */
+	u32	next_entry;
+	u64	owner;
+	u16	devno;
+	u8	status;
+	u8	type;
+	u64	lan_owner;
+	u64	lan_name;
+	u64	port_name;
+	u8	port_type;
+	u8	ext_status:6;
+	u8	protocol:2;
+	u16	base_devno;
+	u32	port_num;
+	u32	ifindex;
+	u32	maxinfo;
+	u32	dev_count;
+	/* 3x device info: */
+	u8	dev_info1[28];
+	u8	dev_info2[28];
+	u8	dev_info3[28];
+} __packed __aligned(8);
+
 #define DIAG26C_GET_MAC	0x0000
 struct diag26c_mac_req {
 	u32	resp_buf_len;
diff --git a/arch/s390/include/asm/dis.h b/arch/s390/include/asm/dis.h
index b0480c60a8e1..c18ed6091914 100644
--- a/arch/s390/include/asm/dis.h
+++ b/arch/s390/include/asm/dis.h
@@ -9,7 +9,7 @@
 #ifndef __ASM_S390_DIS_H__
 #define __ASM_S390_DIS_H__
 
-#include <generated/dis.h>
+#include <asm/dis-defs.h>
 
 static inline int insn_length(unsigned char code)
 {
diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h
deleted file mode 100644
index eaf490f9c5bc..000000000000
--- a/arch/s390/include/asm/dma-mapping.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_S390_DMA_MAPPING_H
-#define _ASM_S390_DMA_MAPPING_H
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/mm.h>
-#include <linux/scatterlist.h>
-#include <linux/dma-debug.h>
-#include <linux/io.h>
-
-extern const struct dma_map_ops s390_pci_dma_ops;
-
-static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
-{
-	return &dma_noop_ops;
-}
-
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return false;
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
-#endif /* _ASM_S390_DMA_MAPPING_H */
diff --git a/arch/s390/include/asm/dwarf.h b/arch/s390/include/asm/dwarf.h
new file mode 100644
index 000000000000..4f21ae561e4d
--- /dev/null
+++ b/arch/s390/include/asm/dwarf.h
@@ -0,0 +1,37 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_S390_DWARF_H
+#define _ASM_S390_DWARF_H
+
+#ifdef __ASSEMBLY__
+
+#define CFI_STARTPROC		.cfi_startproc
+#define CFI_ENDPROC		.cfi_endproc
+#define CFI_DEF_CFA_OFFSET	.cfi_def_cfa_offset
+#define CFI_ADJUST_CFA_OFFSET	.cfi_adjust_cfa_offset
+#define CFI_RESTORE		.cfi_restore
+
+#ifdef CONFIG_AS_CFI_VAL_OFFSET
+#define CFI_VAL_OFFSET		.cfi_val_offset
+#else
+#define CFI_VAL_OFFSET		#
+#endif
+
+#ifndef BUILD_VDSO
+	/*
+	 * Emit CFI data in .debug_frame sections and not in .eh_frame
+	 * sections.  The .eh_frame CFI is used for runtime unwind
+	 * information that is not being used.  Hence, vmlinux.lds.S
+	 * can discard the .eh_frame sections.
+	 */
+	.cfi_sections .debug_frame
+#else
+	/*
+	 * For vDSO, emit CFI data in both, .eh_frame and .debug_frame
+	 * sections.
+	 */
+	.cfi_sections .eh_frame, .debug_frame
+#endif
+
+#endif	/* __ASSEMBLY__ */
+
+#endif	/* _ASM_S390_DWARF_H */
diff --git a/arch/s390/include/asm/eadm.h b/arch/s390/include/asm/eadm.h
index eb5323161f11..bb63b2afdf6f 100644
--- a/arch/s390/include/asm/eadm.h
+++ b/arch/s390/include/asm/eadm.h
@@ -4,7 +4,7 @@
 
 #include <linux/types.h>
 #include <linux/device.h>
-#include <linux/blkdev.h>
+#include <linux/blk_types.h>
 
 struct arqb {
 	u64 data;
diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h
index f040644575b7..99c8ce30b3cd 100644
--- a/arch/s390/include/asm/facility.h
+++ b/arch/s390/include/asm/facility.h
@@ -8,13 +8,31 @@
 #ifndef __ASM_FACILITY_H
 #define __ASM_FACILITY_H
 
-#include <generated/facilities.h>
+#include <asm/facility-defs.h>
 #include <linux/string.h>
 #include <linux/preempt.h>
 #include <asm/lowcore.h>
 
 #define MAX_FACILITY_BIT (sizeof(((struct lowcore *)0)->stfle_fac_list) * 8)
 
+static inline void __set_facility(unsigned long nr, void *facilities)
+{
+	unsigned char *ptr = (unsigned char *) facilities;
+
+	if (nr >= MAX_FACILITY_BIT)
+		return;
+	ptr[nr >> 3] |= 0x80 >> (nr & 7);
+}
+
+static inline void __clear_facility(unsigned long nr, void *facilities)
+{
+	unsigned char *ptr = (unsigned char *) facilities;
+
+	if (nr >= MAX_FACILITY_BIT)
+		return;
+	ptr[nr >> 3] &= ~(0x80 >> (nr & 7));
+}
+
 static inline int __test_facility(unsigned long nr, void *facilities)
 {
 	unsigned char *ptr;
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index c1b0a9ac1dc8..afb0f08b8021 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -2,7 +2,7 @@
 /*
  * definition for kernel virtual machines on s390
  *
- * Copyright IBM Corp. 2008, 2009
+ * Copyright IBM Corp. 2008, 2018
  *
  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  */
@@ -183,6 +183,7 @@ struct kvm_s390_sie_block {
 #define ECA_IB		0x40000000
 #define ECA_SIGPI	0x10000000
 #define ECA_MVPGI	0x01000000
+#define ECA_AIV		0x00200000
 #define ECA_VX		0x00020000
 #define ECA_PROTEXCI	0x00002000
 #define ECA_SII		0x00000001
@@ -228,7 +229,9 @@ struct kvm_s390_sie_block {
 	__u8    epdx;			/* 0x0069 */
 	__u8    reserved6a[2];		/* 0x006a */
 	__u32	todpr;			/* 0x006c */
-	__u8	reserved70[16];		/* 0x0070 */
+#define GISA_FORMAT1 0x00000001
+	__u32	gd;			/* 0x0070 */
+	__u8	reserved74[12];		/* 0x0074 */
 	__u64	mso;			/* 0x0080 */
 	__u64	msl;			/* 0x0088 */
 	psw_t	gpsw;			/* 0x0090 */
@@ -317,18 +320,30 @@ struct kvm_vcpu_stat {
 	u64 deliver_program_int;
 	u64 deliver_io_int;
 	u64 exit_wait_state;
+	u64 instruction_epsw;
+	u64 instruction_gs;
+	u64 instruction_io_other;
+	u64 instruction_lpsw;
+	u64 instruction_lpswe;
 	u64 instruction_pfmf;
+	u64 instruction_ptff;
+	u64 instruction_sck;
+	u64 instruction_sckpf;
 	u64 instruction_stidp;
 	u64 instruction_spx;
 	u64 instruction_stpx;
 	u64 instruction_stap;
-	u64 instruction_storage_key;
+	u64 instruction_iske;
+	u64 instruction_ri;
+	u64 instruction_rrbe;
+	u64 instruction_sske;
 	u64 instruction_ipte_interlock;
-	u64 instruction_stsch;
-	u64 instruction_chsc;
 	u64 instruction_stsi;
 	u64 instruction_stfl;
+	u64 instruction_tb;
+	u64 instruction_tpi;
 	u64 instruction_tprot;
+	u64 instruction_tsch;
 	u64 instruction_sie;
 	u64 instruction_essa;
 	u64 instruction_sthyi;
@@ -354,6 +369,7 @@ struct kvm_vcpu_stat {
 	u64 diagnose_258;
 	u64 diagnose_308;
 	u64 diagnose_500;
+	u64 diagnose_other;
 };
 
 #define PGM_OPERATION			0x01
@@ -410,35 +426,35 @@ struct kvm_vcpu_stat {
 #define PGM_PER				0x80
 #define PGM_CRYPTO_OPERATION		0x119
 
-/* irq types in order of priority */
+/* irq types in ascend order of priorities */
 enum irq_types {
-	IRQ_PEND_MCHK_EX = 0,
-	IRQ_PEND_SVC,
-	IRQ_PEND_PROG,
-	IRQ_PEND_MCHK_REP,
-	IRQ_PEND_EXT_IRQ_KEY,
-	IRQ_PEND_EXT_MALFUNC,
-	IRQ_PEND_EXT_EMERGENCY,
-	IRQ_PEND_EXT_EXTERNAL,
-	IRQ_PEND_EXT_CLOCK_COMP,
-	IRQ_PEND_EXT_CPU_TIMER,
-	IRQ_PEND_EXT_TIMING,
-	IRQ_PEND_EXT_SERVICE,
-	IRQ_PEND_EXT_HOST,
-	IRQ_PEND_PFAULT_INIT,
-	IRQ_PEND_PFAULT_DONE,
-	IRQ_PEND_VIRTIO,
-	IRQ_PEND_IO_ISC_0,
-	IRQ_PEND_IO_ISC_1,
-	IRQ_PEND_IO_ISC_2,
-	IRQ_PEND_IO_ISC_3,
-	IRQ_PEND_IO_ISC_4,
-	IRQ_PEND_IO_ISC_5,
-	IRQ_PEND_IO_ISC_6,
-	IRQ_PEND_IO_ISC_7,
-	IRQ_PEND_SIGP_STOP,
+	IRQ_PEND_SET_PREFIX = 0,
 	IRQ_PEND_RESTART,
-	IRQ_PEND_SET_PREFIX,
+	IRQ_PEND_SIGP_STOP,
+	IRQ_PEND_IO_ISC_7,
+	IRQ_PEND_IO_ISC_6,
+	IRQ_PEND_IO_ISC_5,
+	IRQ_PEND_IO_ISC_4,
+	IRQ_PEND_IO_ISC_3,
+	IRQ_PEND_IO_ISC_2,
+	IRQ_PEND_IO_ISC_1,
+	IRQ_PEND_IO_ISC_0,
+	IRQ_PEND_VIRTIO,
+	IRQ_PEND_PFAULT_DONE,
+	IRQ_PEND_PFAULT_INIT,
+	IRQ_PEND_EXT_HOST,
+	IRQ_PEND_EXT_SERVICE,
+	IRQ_PEND_EXT_TIMING,
+	IRQ_PEND_EXT_CPU_TIMER,
+	IRQ_PEND_EXT_CLOCK_COMP,
+	IRQ_PEND_EXT_EXTERNAL,
+	IRQ_PEND_EXT_EMERGENCY,
+	IRQ_PEND_EXT_MALFUNC,
+	IRQ_PEND_EXT_IRQ_KEY,
+	IRQ_PEND_MCHK_REP,
+	IRQ_PEND_PROG,
+	IRQ_PEND_SVC,
+	IRQ_PEND_MCHK_EX,
 	IRQ_PEND_COUNT
 };
 
@@ -516,9 +532,6 @@ struct kvm_s390_irq_payload {
 
 struct kvm_s390_local_interrupt {
 	spinlock_t lock;
-	struct kvm_s390_float_interrupt *float_int;
-	struct swait_queue_head *wq;
-	atomic_t *cpuflags;
 	DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS);
 	struct kvm_s390_irq_payload irq;
 	unsigned long pending_irqs;
@@ -707,14 +720,50 @@ struct kvm_s390_crypto_cb {
 	struct kvm_s390_apcb1 apcb1;		/* 0x0080 */
 };
 
+struct kvm_s390_gisa {
+	union {
+		struct { /* common to all formats */
+			u32 next_alert;
+			u8  ipm;
+			u8  reserved01[2];
+			u8  iam;
+		};
+		struct { /* format 0 */
+			u32 next_alert;
+			u8  ipm;
+			u8  reserved01;
+			u8  : 6;
+			u8  g : 1;
+			u8  c : 1;
+			u8  iam;
+			u8  reserved02[4];
+			u32 airq_count;
+		} g0;
+		struct { /* format 1 */
+			u32 next_alert;
+			u8  ipm;
+			u8  simm;
+			u8  nimm;
+			u8  iam;
+			u8  aism[8];
+			u8  : 6;
+			u8  g : 1;
+			u8  c : 1;
+			u8  reserved03[11];
+			u32 airq_count;
+		} g1;
+	};
+};
+
 /*
- * sie_page2 has to be allocated as DMA because fac_list and crycb need
- * 31bit addresses in the sie control block.
+ * sie_page2 has to be allocated as DMA because fac_list, crycb and
+ * gisa need 31bit addresses in the sie control block.
  */
 struct sie_page2 {
 	__u64 fac_list[S390_ARCH_FAC_LIST_SIZE_U64];	/* 0x0000 */
 	struct kvm_s390_crypto_cb crycb;		/* 0x0800 */
-	u8 reserved900[0x1000 - 0x900];			/* 0x0900 */
+	struct kvm_s390_gisa gisa;			/* 0x0900 */
+	u8 reserved920[0x1000 - 0x920];			/* 0x0920 */
 };
 
 struct kvm_s390_vsie {
@@ -761,6 +810,7 @@ struct kvm_arch{
 	struct kvm_s390_migration_state *migration_state;
 	/* subset of available cpu features enabled by user space */
 	DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
+	struct kvm_s390_gisa *gisa;
 };
 
 #define KVM_HVA_ERR_BAD		(-1UL)
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index ec6592e8ba36..5bc888841eaf 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -136,7 +136,11 @@ struct lowcore {
 	__u64	vdso_per_cpu_data;		/* 0x03b8 */
 	__u64	machine_flags;			/* 0x03c0 */
 	__u64	gmap;				/* 0x03c8 */
-	__u8	pad_0x03d0[0x0e00-0x03d0];	/* 0x03d0 */
+	__u8	pad_0x03d0[0x0400-0x03d0];	/* 0x03d0 */
+
+	/* br %r1 trampoline */
+	__u16	br_r1_trampoline;		/* 0x0400 */
+	__u8	pad_0x0402[0x0e00-0x0402];	/* 0x0402 */
 
 	/*
 	 * 0xe00 contains the address of the IPL Parameter Information
@@ -151,7 +155,8 @@ struct lowcore {
 	__u8	pad_0x0e20[0x0f00-0x0e20];	/* 0x0e20 */
 
 	/* Extended facility list */
-	__u64	stfle_fac_list[32];		/* 0x0f00 */
+	__u64	stfle_fac_list[16];		/* 0x0f00 */
+	__u64	alt_stfle_fac_list[16];		/* 0x0f80 */
 	__u8	pad_0x1000[0x11b0-0x1000];	/* 0x1000 */
 
 	/* Pointer to the machine check extended save area */
diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h
index 65154eaa3714..6c8ce15cde7b 100644
--- a/arch/s390/include/asm/mmu_context.h
+++ b/arch/s390/include/asm/mmu_context.h
@@ -63,6 +63,7 @@ static inline int init_new_context(struct task_struct *tsk,
 				   _ASCE_USER_BITS | _ASCE_TYPE_SEGMENT;
 		/* pgd_alloc() did not account this pmd */
 		mm_inc_nr_pmds(mm);
+		mm_inc_nr_puds(mm);
 	}
 	crst_table_init((unsigned long *) mm->pgd, pgd_entry_type(mm));
 	return 0;
diff --git a/arch/s390/include/asm/nospec-branch.h b/arch/s390/include/asm/nospec-branch.h
new file mode 100644
index 000000000000..7df48e5cf36f
--- /dev/null
+++ b/arch/s390/include/asm/nospec-branch.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_S390_EXPOLINE_H
+#define _ASM_S390_EXPOLINE_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+extern int nospec_call_disable;
+extern int nospec_return_disable;
+
+void nospec_init_branches(void);
+void nospec_call_revert(s32 *start, s32 *end);
+void nospec_return_revert(s32 *start, s32 *end);
+
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_S390_EXPOLINE_H */
diff --git a/arch/s390/include/asm/pci_dma.h b/arch/s390/include/asm/pci_dma.h
index e8d9161fa17a..419fac7a62c0 100644
--- a/arch/s390/include/asm/pci_dma.h
+++ b/arch/s390/include/asm/pci_dma.h
@@ -201,4 +201,7 @@ void dma_cleanup_tables(unsigned long *);
 unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr);
 void dma_update_cpu_trans(unsigned long *entry, void *page_addr, int flags);
 
+extern const struct dma_map_ops s390_pci_dma_ops;
+
+
 #endif
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 0a6b0286c32e..2d24d33bf188 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -1505,12 +1505,12 @@ static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
 }
 
 #define __HAVE_ARCH_PMDP_INVALIDATE
-static inline void pmdp_invalidate(struct vm_area_struct *vma,
+static inline pmd_t pmdp_invalidate(struct vm_area_struct *vma,
 				   unsigned long addr, pmd_t *pmdp)
 {
 	pmd_t pmd = __pmd(pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID);
 
-	pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
+	return pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd);
 }
 
 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index bfbfad482289..7f2953c15c37 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -91,6 +91,7 @@ void cpu_detect_mhz_feature(void);
 extern const struct seq_operations cpuinfo_op;
 extern int sysctl_ieee_emulation_warnings;
 extern void execve_tail(void);
+extern void __bpon(void);
 
 /*
  * User space process size: 2GB for 31 bit, 4TB or 8PT for 64 bit.
@@ -377,6 +378,9 @@ extern void memcpy_absolute(void *, void *, size_t);
 	memcpy_absolute(&(dest), &__tmp, sizeof(__tmp));	\
 } while (0)
 
+extern int s390_isolate_bp(void);
+extern int s390_isolate_bp_guest(void);
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* __ASM_S390_PROCESSOR_H */
diff --git a/arch/s390/include/asm/runtime_instr.h b/arch/s390/include/asm/runtime_instr.h
index 6b1540337ed6..0e1605538cd4 100644
--- a/arch/s390/include/asm/runtime_instr.h
+++ b/arch/s390/include/asm/runtime_instr.h
@@ -2,75 +2,10 @@
 #ifndef _RUNTIME_INSTR_H
 #define _RUNTIME_INSTR_H
 
-#define S390_RUNTIME_INSTR_START	0x1
-#define S390_RUNTIME_INSTR_STOP		0x2
-
-struct runtime_instr_cb {
-	__u64 rca;
-	__u64 roa;
-	__u64 rla;
-
-	__u32 v			: 1;
-	__u32 s			: 1;
-	__u32 k			: 1;
-	__u32 h			: 1;
-	__u32 a			: 1;
-	__u32 reserved1		: 3;
-	__u32 ps		: 1;
-	__u32 qs		: 1;
-	__u32 pc		: 1;
-	__u32 qc		: 1;
-	__u32 reserved2		: 1;
-	__u32 g			: 1;
-	__u32 u			: 1;
-	__u32 l			: 1;
-	__u32 key		: 4;
-	__u32 reserved3		: 8;
-	__u32 t			: 1;
-	__u32 rgs		: 3;
-
-	__u32 m			: 4;
-	__u32 n			: 1;
-	__u32 mae		: 1;
-	__u32 reserved4		: 2;
-	__u32 c			: 1;
-	__u32 r			: 1;
-	__u32 b			: 1;
-	__u32 j			: 1;
-	__u32 e			: 1;
-	__u32 x			: 1;
-	__u32 reserved5		: 2;
-	__u32 bpxn		: 1;
-	__u32 bpxt		: 1;
-	__u32 bpti		: 1;
-	__u32 bpni		: 1;
-	__u32 reserved6		: 2;
-
-	__u32 d			: 1;
-	__u32 f			: 1;
-	__u32 ic		: 4;
-	__u32 dc		: 4;
-
-	__u64 reserved7;
-	__u64 sf;
-	__u64 rsic;
-	__u64 reserved8;
-} __packed __aligned(8);
+#include <uapi/asm/runtime_instr.h>
 
 extern struct runtime_instr_cb runtime_instr_empty_cb;
 
-static inline void load_runtime_instr_cb(struct runtime_instr_cb *cb)
-{
-	asm volatile(".insn	rsy,0xeb0000000060,0,0,%0"	/* LRIC */
-		: : "Q" (*cb));
-}
-
-static inline void store_runtime_instr_cb(struct runtime_instr_cb *cb)
-{
-	asm volatile(".insn	rsy,0xeb0000000061,0,0,%0"	/* STRIC */
-		: "=Q" (*cb) : : "cc");
-}
-
 static inline void save_ri_cb(struct runtime_instr_cb *cb_prev)
 {
 	if (cb_prev)
diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h
index d3c1a8a2e3ad..3cae9168f63c 100644
--- a/arch/s390/include/asm/sclp.h
+++ b/arch/s390/include/asm/sclp.h
@@ -77,6 +77,7 @@ struct sclp_info {
 	unsigned char has_ibs : 1;
 	unsigned char has_skey : 1;
 	unsigned char has_kss : 1;
+	unsigned char has_gisaf : 1;
 	unsigned int ibc;
 	unsigned int mtid;
 	unsigned int mtid_cp;
diff --git a/arch/s390/include/asm/sysinfo.h b/arch/s390/include/asm/sysinfo.h
index 25057c118d56..fe7b3f8f0791 100644
--- a/arch/s390/include/asm/sysinfo.h
+++ b/arch/s390/include/asm/sysinfo.h
@@ -21,7 +21,8 @@ struct sysinfo_1_1_1 {
 	unsigned char :8;
 	unsigned char ccr;
 	unsigned char cai;
-	char reserved_0[28];
+	char reserved_0[20];
+	unsigned long lic;
 	char manufacturer[16];
 	char type[4];
 	char reserved_1[12];
diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h
index 0880a37b6d3b..83ba57533ce6 100644
--- a/arch/s390/include/asm/thread_info.h
+++ b/arch/s390/include/asm/thread_info.h
@@ -42,8 +42,6 @@ struct thread_info {
 	.flags		= 0,			\
 }
 
-#define init_stack		(init_thread_union.stack)
-
 void arch_release_task_struct(struct task_struct *tsk);
 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 
@@ -60,6 +58,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 #define TIF_GUARDED_STORAGE	4	/* load guarded storage control block */
 #define TIF_PATCH_PENDING	5	/* pending live patching update */
 #define TIF_PGSTE		6	/* New mm's will use 4K page tables */
+#define TIF_ISOLATE_BP		8	/* Run process with isolated BP */
+#define TIF_ISOLATE_BP_GUEST	9	/* Run KVM guests with isolated BP */
 
 #define TIF_31BIT		16	/* 32bit process */
 #define TIF_MEMDIE		17	/* is terminating due to OOM killer */
@@ -80,6 +80,8 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 #define _TIF_UPROBE		_BITUL(TIF_UPROBE)
 #define _TIF_GUARDED_STORAGE	_BITUL(TIF_GUARDED_STORAGE)
 #define _TIF_PATCH_PENDING	_BITUL(TIF_PATCH_PENDING)
+#define _TIF_ISOLATE_BP		_BITUL(TIF_ISOLATE_BP)
+#define _TIF_ISOLATE_BP_GUEST	_BITUL(TIF_ISOLATE_BP_GUEST)
 
 #define _TIF_31BIT		_BITUL(TIF_31BIT)
 #define _TIF_SINGLE_STEP	_BITUL(TIF_SINGLE_STEP)
diff --git a/arch/s390/include/asm/unistd.h b/arch/s390/include/asm/unistd.h
index 7807093b73be..fd79c0d35dc4 100644
--- a/arch/s390/include/asm/unistd.h
+++ b/arch/s390/include/asm/unistd.h
@@ -8,6 +8,7 @@
 #define _ASM_S390_UNISTD_H_
 
 #include <uapi/asm/unistd.h>
+#include <asm/unistd_nr.h>
 
 #define __IGNORE_time
 #define __IGNORE_pkey_mprotect
diff --git a/arch/s390/include/uapi/asm/Kbuild b/arch/s390/include/uapi/asm/Kbuild
index 92b7c9b3e641..faef3f7e8353 100644
--- a/arch/s390/include/uapi/asm/Kbuild
+++ b/arch/s390/include/uapi/asm/Kbuild
@@ -2,6 +2,9 @@
 # UAPI Header export list
 include include/uapi/asm-generic/Kbuild.asm
 
+generated-y += unistd_32.h
+generated-y += unistd_64.h
+
 generic-y += errno.h
 generic-y += fcntl.h
 generic-y += ioctl.h
diff --git a/arch/s390/include/uapi/asm/runtime_instr.h b/arch/s390/include/uapi/asm/runtime_instr.h
new file mode 100644
index 000000000000..45c9ec984e6b
--- /dev/null
+++ b/arch/s390/include/uapi/asm/runtime_instr.h
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _S390_UAPI_RUNTIME_INSTR_H
+#define _S390_UAPI_RUNTIME_INSTR_H
+
+#include <linux/types.h>
+
+#define S390_RUNTIME_INSTR_START	0x1
+#define S390_RUNTIME_INSTR_STOP		0x2
+
+struct runtime_instr_cb {
+	__u64 rca;
+	__u64 roa;
+	__u64 rla;
+
+	__u32 v			: 1;
+	__u32 s			: 1;
+	__u32 k			: 1;
+	__u32 h			: 1;
+	__u32 a			: 1;
+	__u32 reserved1		: 3;
+	__u32 ps		: 1;
+	__u32 qs		: 1;
+	__u32 pc		: 1;
+	__u32 qc		: 1;
+	__u32 reserved2		: 1;
+	__u32 g			: 1;
+	__u32 u			: 1;
+	__u32 l			: 1;
+	__u32 key		: 4;
+	__u32 reserved3		: 8;
+	__u32 t			: 1;
+	__u32 rgs		: 3;
+
+	__u32 m			: 4;
+	__u32 n			: 1;
+	__u32 mae		: 1;
+	__u32 reserved4		: 2;
+	__u32 c			: 1;
+	__u32 r			: 1;
+	__u32 b			: 1;
+	__u32 j			: 1;
+	__u32 e			: 1;
+	__u32 x			: 1;
+	__u32 reserved5		: 2;
+	__u32 bpxn		: 1;
+	__u32 bpxt		: 1;
+	__u32 bpti		: 1;
+	__u32 bpni		: 1;
+	__u32 reserved6		: 2;
+
+	__u32 d			: 1;
+	__u32 f			: 1;
+	__u32 ic		: 4;
+	__u32 dc		: 4;
+
+	__u64 reserved7;
+	__u64 sf;
+	__u64 rsic;
+	__u64 reserved8;
+} __packed __aligned(8);
+
+static inline void load_runtime_instr_cb(struct runtime_instr_cb *cb)
+{
+	asm volatile(".insn	rsy,0xeb0000000060,0,0,%0"	/* LRIC */
+		: : "Q" (*cb));
+}
+
+static inline void store_runtime_instr_cb(struct runtime_instr_cb *cb)
+{
+	asm volatile(".insn	rsy,0xeb0000000061,0,0,%0"	/* STRIC */
+		: "=Q" (*cb) : : "cc");
+}
+
+#endif /* _S390_UAPI_RUNTIME_INSTR_H */
diff --git a/arch/s390/include/uapi/asm/unistd.h b/arch/s390/include/uapi/asm/unistd.h
index 725120939051..01b5fe8b9db6 100644
--- a/arch/s390/include/uapi/asm/unistd.h
+++ b/arch/s390/include/uapi/asm/unistd.h
@@ -8,405 +8,10 @@
 #ifndef _UAPI_ASM_S390_UNISTD_H_
 #define _UAPI_ASM_S390_UNISTD_H_
 
-/*
- * This file contains the system call numbers.
- */
-
-#define __NR_exit                 1
-#define __NR_fork                 2
-#define __NR_read                 3
-#define __NR_write                4
-#define __NR_open                 5
-#define __NR_close                6
-#define __NR_restart_syscall	  7
-#define __NR_creat                8
-#define __NR_link                 9
-#define __NR_unlink              10
-#define __NR_execve              11
-#define __NR_chdir               12
-#define __NR_mknod               14
-#define __NR_chmod               15
-#define __NR_lseek               19
-#define __NR_getpid              20
-#define __NR_mount               21
-#define __NR_umount              22
-#define __NR_ptrace              26
-#define __NR_alarm               27
-#define __NR_pause               29
-#define __NR_utime               30
-#define __NR_access              33
-#define __NR_nice                34
-#define __NR_sync                36
-#define __NR_kill                37
-#define __NR_rename              38
-#define __NR_mkdir               39
-#define __NR_rmdir               40
-#define __NR_dup                 41
-#define __NR_pipe                42
-#define __NR_times               43
-#define __NR_brk                 45
-#define __NR_signal              48
-#define __NR_acct                51
-#define __NR_umount2             52
-#define __NR_ioctl               54
-#define __NR_fcntl               55
-#define __NR_setpgid             57
-#define __NR_umask               60
-#define __NR_chroot              61
-#define __NR_ustat               62
-#define __NR_dup2                63
-#define __NR_getppid             64
-#define __NR_getpgrp             65
-#define __NR_setsid              66
-#define __NR_sigaction           67
-#define __NR_sigsuspend          72
-#define __NR_sigpending          73
-#define __NR_sethostname         74
-#define __NR_setrlimit           75
-#define __NR_getrusage           77
-#define __NR_gettimeofday        78
-#define __NR_settimeofday        79
-#define __NR_symlink             83
-#define __NR_readlink            85
-#define __NR_uselib              86
-#define __NR_swapon              87
-#define __NR_reboot              88
-#define __NR_readdir             89
-#define __NR_mmap                90
-#define __NR_munmap              91
-#define __NR_truncate            92
-#define __NR_ftruncate           93
-#define __NR_fchmod              94
-#define __NR_getpriority         96
-#define __NR_setpriority         97
-#define __NR_statfs              99
-#define __NR_fstatfs            100
-#define __NR_socketcall         102
-#define __NR_syslog             103
-#define __NR_setitimer          104
-#define __NR_getitimer          105
-#define __NR_stat               106
-#define __NR_lstat              107
-#define __NR_fstat              108
-#define __NR_lookup_dcookie     110
-#define __NR_vhangup            111
-#define __NR_idle               112
-#define __NR_wait4              114
-#define __NR_swapoff            115
-#define __NR_sysinfo            116
-#define __NR_ipc                117
-#define __NR_fsync              118
-#define __NR_sigreturn          119
-#define __NR_clone              120
-#define __NR_setdomainname      121
-#define __NR_uname              122
-#define __NR_adjtimex           124
-#define __NR_mprotect           125
-#define __NR_sigprocmask        126
-#define __NR_create_module      127
-#define __NR_init_module        128
-#define __NR_delete_module      129
-#define __NR_get_kernel_syms    130
-#define __NR_quotactl           131
-#define __NR_getpgid            132
-#define __NR_fchdir             133
-#define __NR_bdflush            134
-#define __NR_sysfs              135
-#define __NR_personality        136
-#define __NR_afs_syscall        137 /* Syscall for Andrew File System */
-#define __NR_getdents           141
-#define __NR_flock              143
-#define __NR_msync              144
-#define __NR_readv              145
-#define __NR_writev             146
-#define __NR_getsid             147
-#define __NR_fdatasync          148
-#define __NR__sysctl            149
-#define __NR_mlock              150
-#define __NR_munlock            151
-#define __NR_mlockall           152
-#define __NR_munlockall         153
-#define __NR_sched_setparam             154
-#define __NR_sched_getparam             155
-#define __NR_sched_setscheduler         156
-#define __NR_sched_getscheduler         157
-#define __NR_sched_yield                158
-#define __NR_sched_get_priority_max     159
-#define __NR_sched_get_priority_min     160
-#define __NR_sched_rr_get_interval      161
-#define __NR_nanosleep          162
-#define __NR_mremap             163
-#define __NR_query_module       167
-#define __NR_poll               168
-#define __NR_nfsservctl         169
-#define __NR_prctl              172
-#define __NR_rt_sigreturn       173
-#define __NR_rt_sigaction       174
-#define __NR_rt_sigprocmask     175
-#define __NR_rt_sigpending      176
-#define __NR_rt_sigtimedwait    177
-#define __NR_rt_sigqueueinfo    178
-#define __NR_rt_sigsuspend      179
-#define __NR_pread64            180
-#define __NR_pwrite64           181
-#define __NR_getcwd             183
-#define __NR_capget             184
-#define __NR_capset             185
-#define __NR_sigaltstack        186
-#define __NR_sendfile           187
-#define __NR_getpmsg		188
-#define __NR_putpmsg		189
-#define __NR_vfork		190
-#define __NR_pivot_root         217
-#define __NR_mincore            218
-#define __NR_madvise            219
-#define __NR_getdents64		220
-#define __NR_readahead		222
-#define __NR_setxattr		224
-#define __NR_lsetxattr		225
-#define __NR_fsetxattr		226
-#define __NR_getxattr		227
-#define __NR_lgetxattr		228
-#define __NR_fgetxattr		229
-#define __NR_listxattr		230
-#define __NR_llistxattr		231
-#define __NR_flistxattr		232
-#define __NR_removexattr	233
-#define __NR_lremovexattr	234
-#define __NR_fremovexattr	235
-#define __NR_gettid		236
-#define __NR_tkill		237
-#define __NR_futex		238
-#define __NR_sched_setaffinity	239
-#define __NR_sched_getaffinity	240
-#define __NR_tgkill		241
-/* Number 242 is reserved for tux */
-#define __NR_io_setup		243
-#define __NR_io_destroy		244
-#define __NR_io_getevents	245
-#define __NR_io_submit		246
-#define __NR_io_cancel		247
-#define __NR_exit_group		248
-#define __NR_epoll_create	249
-#define __NR_epoll_ctl		250
-#define __NR_epoll_wait		251
-#define __NR_set_tid_address	252
-#define __NR_fadvise64		253
-#define __NR_timer_create	254
-#define __NR_timer_settime	255
-#define __NR_timer_gettime	256
-#define __NR_timer_getoverrun	257
-#define __NR_timer_delete	258
-#define __NR_clock_settime	259
-#define __NR_clock_gettime	260
-#define __NR_clock_getres	261
-#define __NR_clock_nanosleep	262
-/* Number 263 is reserved for vserver */
-#define __NR_statfs64		265
-#define __NR_fstatfs64		266
-#define __NR_remap_file_pages	267
-#define __NR_mbind		268
-#define __NR_get_mempolicy	269
-#define __NR_set_mempolicy	270
-#define __NR_mq_open		271
-#define __NR_mq_unlink		272
-#define __NR_mq_timedsend	273
-#define __NR_mq_timedreceive	274
-#define __NR_mq_notify		275
-#define __NR_mq_getsetattr	276
-#define __NR_kexec_load		277
-#define __NR_add_key		278
-#define __NR_request_key	279
-#define __NR_keyctl		280
-#define __NR_waitid		281
-#define __NR_ioprio_set		282
-#define __NR_ioprio_get		283
-#define __NR_inotify_init	284
-#define __NR_inotify_add_watch	285
-#define __NR_inotify_rm_watch	286
-#define __NR_migrate_pages	287
-#define __NR_openat		288
-#define __NR_mkdirat		289
-#define __NR_mknodat		290
-#define __NR_fchownat		291
-#define __NR_futimesat		292
-#define __NR_unlinkat		294
-#define __NR_renameat		295
-#define __NR_linkat		296
-#define __NR_symlinkat		297
-#define __NR_readlinkat		298
-#define __NR_fchmodat		299
-#define __NR_faccessat		300
-#define __NR_pselect6		301
-#define __NR_ppoll		302
-#define __NR_unshare		303
-#define __NR_set_robust_list	304
-#define __NR_get_robust_list	305
-#define __NR_splice		306
-#define __NR_sync_file_range	307
-#define __NR_tee		308
-#define __NR_vmsplice		309
-#define __NR_move_pages		310
-#define __NR_getcpu		311
-#define __NR_epoll_pwait	312
-#define __NR_utimes		313
-#define __NR_fallocate		314
-#define __NR_utimensat		315
-#define __NR_signalfd		316
-#define __NR_timerfd		317
-#define __NR_eventfd		318
-#define __NR_timerfd_create	319
-#define __NR_timerfd_settime	320
-#define __NR_timerfd_gettime	321
-#define __NR_signalfd4		322
-#define __NR_eventfd2		323
-#define __NR_inotify_init1	324
-#define __NR_pipe2		325
-#define __NR_dup3		326
-#define __NR_epoll_create1	327
-#define	__NR_preadv		328
-#define	__NR_pwritev		329
-#define __NR_rt_tgsigqueueinfo	330
-#define __NR_perf_event_open	331
-#define __NR_fanotify_init	332
-#define __NR_fanotify_mark	333
-#define __NR_prlimit64		334
-#define __NR_name_to_handle_at	335
-#define __NR_open_by_handle_at	336
-#define __NR_clock_adjtime	337
-#define __NR_syncfs		338
-#define __NR_setns		339
-#define __NR_process_vm_readv	340
-#define __NR_process_vm_writev	341
-#define __NR_s390_runtime_instr 342
-#define __NR_kcmp		343
-#define __NR_finit_module	344
-#define __NR_sched_setattr	345
-#define __NR_sched_getattr	346
-#define __NR_renameat2		347
-#define __NR_seccomp		348
-#define __NR_getrandom		349
-#define __NR_memfd_create	350
-#define __NR_bpf		351
-#define __NR_s390_pci_mmio_write	352
-#define __NR_s390_pci_mmio_read		353
-#define __NR_execveat		354
-#define __NR_userfaultfd	355
-#define __NR_membarrier		356
-#define __NR_recvmmsg		357
-#define __NR_sendmmsg		358
-#define __NR_socket		359
-#define __NR_socketpair		360
-#define __NR_bind		361
-#define __NR_connect		362
-#define __NR_listen		363
-#define __NR_accept4		364
-#define __NR_getsockopt		365
-#define __NR_setsockopt		366
-#define __NR_getsockname	367
-#define __NR_getpeername	368
-#define __NR_sendto		369
-#define __NR_sendmsg		370
-#define __NR_recvfrom		371
-#define __NR_recvmsg		372
-#define __NR_shutdown		373
-#define __NR_mlock2		374
-#define __NR_copy_file_range	375
-#define __NR_preadv2		376
-#define __NR_pwritev2		377
-#define __NR_s390_guarded_storage	378
-#define __NR_statx		379
-#define __NR_s390_sthyi		380
-#define NR_syscalls 381
-
-/* 
- * There are some system calls that are not present on 64 bit, some
- * have a different name although they do the same (e.g. __NR_chown32
- * is __NR_chown on 64 bit).
- */
-#ifndef __s390x__
-
-#define __NR_time		 13
-#define __NR_lchown		 16
-#define __NR_setuid		 23
-#define __NR_getuid		 24
-#define __NR_stime		 25
-#define __NR_setgid		 46
-#define __NR_getgid		 47
-#define __NR_geteuid		 49
-#define __NR_getegid		 50
-#define __NR_setreuid		 70
-#define __NR_setregid		 71
-#define __NR_getrlimit		 76
-#define __NR_getgroups		 80
-#define __NR_setgroups		 81
-#define __NR_fchown		 95
-#define __NR_ioperm		101
-#define __NR_setfsuid		138
-#define __NR_setfsgid		139
-#define __NR__llseek		140
-#define __NR__newselect 	142
-#define __NR_setresuid		164
-#define __NR_getresuid		165
-#define __NR_setresgid		170
-#define __NR_getresgid		171
-#define __NR_chown		182
-#define __NR_ugetrlimit		191	/* SuS compliant getrlimit */
-#define __NR_mmap2		192
-#define __NR_truncate64		193
-#define __NR_ftruncate64	194
-#define __NR_stat64		195
-#define __NR_lstat64		196
-#define __NR_fstat64		197
-#define __NR_lchown32		198
-#define __NR_getuid32		199
-#define __NR_getgid32		200
-#define __NR_geteuid32		201
-#define __NR_getegid32		202
-#define __NR_setreuid32		203
-#define __NR_setregid32		204
-#define __NR_getgroups32	205
-#define __NR_setgroups32	206
-#define __NR_fchown32		207
-#define __NR_setresuid32	208
-#define __NR_getresuid32	209
-#define __NR_setresgid32	210
-#define __NR_getresgid32	211
-#define __NR_chown32		212
-#define __NR_setuid32		213
-#define __NR_setgid32		214
-#define __NR_setfsuid32		215
-#define __NR_setfsgid32		216
-#define __NR_fcntl64		221
-#define __NR_sendfile64		223
-#define __NR_fadvise64_64	264
-#define __NR_fstatat64		293
-
+#ifdef __s390x__
+#include <asm/unistd_64.h>
 #else
-
-#define __NR_select		142
-#define __NR_getrlimit		191	/* SuS compliant getrlimit */
-#define __NR_lchown  		198
-#define __NR_getuid  		199
-#define __NR_getgid  		200
-#define __NR_geteuid  		201
-#define __NR_getegid  		202
-#define __NR_setreuid  		203
-#define __NR_setregid  		204
-#define __NR_getgroups  	205
-#define __NR_setgroups  	206
-#define __NR_fchown  		207
-#define __NR_setresuid  	208
-#define __NR_getresuid  	209
-#define __NR_setresgid  	210
-#define __NR_getresgid  	211
-#define __NR_chown  		212
-#define __NR_setuid  		213
-#define __NR_setgid  		214
-#define __NR_setfsuid  		215
-#define __NR_setfsgid  		216
-#define __NR_newfstatat		293
-
+#include <asm/unistd_32.h>
 #endif
 
 #endif /* _UAPI_ASM_S390_UNISTD_H_ */
diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile
index 909bce65cb2b..7f27e3da9709 100644
--- a/arch/s390/kernel/Makefile
+++ b/arch/s390/kernel/Makefile
@@ -29,6 +29,7 @@ UBSAN_SANITIZE_early.o	:= n
 #
 ifneq ($(CC_FLAGS_MARCH),-march=z900)
 CFLAGS_REMOVE_als.o	+= $(CC_FLAGS_MARCH)
+CFLAGS_REMOVE_als.o	+= $(CC_FLAGS_EXPOLINE)
 CFLAGS_als.o		+= -march=z900
 AFLAGS_REMOVE_head.o	+= $(CC_FLAGS_MARCH)
 AFLAGS_head.o		+= -march=z900
@@ -63,6 +64,9 @@ obj-y	+= entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o
 
 extra-y				+= head.o head64.o vmlinux.lds
 
+obj-$(CONFIG_EXPOLINE)		+= nospec-branch.o
+CFLAGS_REMOVE_expoline.o	+= $(CC_FLAGS_EXPOLINE)
+
 obj-$(CONFIG_MODULES)		+= module.o
 obj-$(CONFIG_SMP)		+= smp.o
 obj-$(CONFIG_SCHED_TOPOLOGY)	+= topology.o
diff --git a/arch/s390/kernel/alternative.c b/arch/s390/kernel/alternative.c
index 574e77622c04..22476135f738 100644
--- a/arch/s390/kernel/alternative.c
+++ b/arch/s390/kernel/alternative.c
@@ -15,6 +15,29 @@ static int __init disable_alternative_instructions(char *str)
 
 early_param("noaltinstr", disable_alternative_instructions);
 
+static int __init nobp_setup_early(char *str)
+{
+	bool enabled;
+	int rc;
+
+	rc = kstrtobool(str, &enabled);
+	if (rc)
+		return rc;
+	if (enabled && test_facility(82))
+		__set_facility(82, S390_lowcore.alt_stfle_fac_list);
+	else
+		__clear_facility(82, S390_lowcore.alt_stfle_fac_list);
+	return 0;
+}
+early_param("nobp", nobp_setup_early);
+
+static int __init nospec_setup_early(char *str)
+{
+	__clear_facility(82, S390_lowcore.alt_stfle_fac_list);
+	return 0;
+}
+early_param("nospec", nospec_setup_early);
+
 struct brcl_insn {
 	u16 opc;
 	s32 disp;
@@ -75,7 +98,8 @@ static void __init_or_module __apply_alternatives(struct alt_instr *start,
 		instr = (u8 *)&a->instr_offset + a->instr_offset;
 		replacement = (u8 *)&a->repl_offset + a->repl_offset;
 
-		if (!test_facility(a->facility))
+		if (!__test_facility(a->facility,
+				     S390_lowcore.alt_stfle_fac_list))
 			continue;
 
 		if (unlikely(a->instrlen % 2 || a->replacementlen % 2)) {
diff --git a/arch/s390/kernel/compat_linux.c b/arch/s390/kernel/compat_linux.c
index 59eea9c65d3e..79b7a3438d54 100644
--- a/arch/s390/kernel/compat_linux.c
+++ b/arch/s390/kernel/compat_linux.c
@@ -110,7 +110,7 @@ COMPAT_SYSCALL_DEFINE2(s390_setregid16, u16, rgid, u16, egid)
 
 COMPAT_SYSCALL_DEFINE1(s390_setgid16, u16, gid)
 {
-	return sys_setgid((gid_t)gid);
+	return sys_setgid(low2highgid(gid));
 }
 
 COMPAT_SYSCALL_DEFINE2(s390_setreuid16, u16, ruid, u16, euid)
@@ -120,7 +120,7 @@ COMPAT_SYSCALL_DEFINE2(s390_setreuid16, u16, ruid, u16, euid)
 
 COMPAT_SYSCALL_DEFINE1(s390_setuid16, u16, uid)
 {
-	return sys_setuid((uid_t)uid);
+	return sys_setuid(low2highuid(uid));
 }
 
 COMPAT_SYSCALL_DEFINE3(s390_setresuid16, u16, ruid, u16, euid, u16, suid)
@@ -173,12 +173,12 @@ COMPAT_SYSCALL_DEFINE3(s390_getresgid16, u16 __user *, rgidp,
 
 COMPAT_SYSCALL_DEFINE1(s390_setfsuid16, u16, uid)
 {
-	return sys_setfsuid((uid_t)uid);
+	return sys_setfsuid(low2highuid(uid));
 }
 
 COMPAT_SYSCALL_DEFINE1(s390_setfsgid16, u16, gid)
 {
-	return sys_setfsgid((gid_t)gid);
+	return sys_setfsgid(low2highgid(gid));
 }
 
 static int groups16_to_user(u16 __user *grouplist, struct group_info *group_info)
diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c
index ef246940b44c..18c1eeb847b2 100644
--- a/arch/s390/kernel/compat_signal.c
+++ b/arch/s390/kernel/compat_signal.c
@@ -50,106 +50,6 @@ typedef struct
 	struct ucontext32 uc;
 } rt_sigframe32;
 
-int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from)
-{
-	int err;
-
-	/* If you change siginfo_t structure, please be sure
-	   this code is fixed accordingly.
-	   It should never copy any pad contained in the structure
-	   to avoid security leaks, but must copy the generic
-	   3 ints plus the relevant union member.  
-	   This routine must convert siginfo from 64bit to 32bit as well
-	   at the same time.  */
-	err = __put_user(from->si_signo, &to->si_signo);
-	err |= __put_user(from->si_errno, &to->si_errno);
-	err |= __put_user(from->si_code, &to->si_code);
-	if (from->si_code < 0)
-		err |= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE);
-	else {
-		switch (siginfo_layout(from->si_signo, from->si_code)) {
-		case SIL_RT:
-			err |= __put_user(from->si_int, &to->si_int);
-			/* fallthrough */
-		case SIL_KILL:
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_uid, &to->si_uid);
-			break;
-		case SIL_CHLD:
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_uid, &to->si_uid);
-			err |= __put_user(from->si_utime, &to->si_utime);
-			err |= __put_user(from->si_stime, &to->si_stime);
-			err |= __put_user(from->si_status, &to->si_status);
-			break;
-		case SIL_FAULT:
-			err |= __put_user((unsigned long) from->si_addr,
-					  &to->si_addr);
-			break;
-		case SIL_POLL:
-			err |= __put_user(from->si_band, &to->si_band);
-			err |= __put_user(from->si_fd, &to->si_fd);
-			break;
-		case SIL_TIMER:
-			err |= __put_user(from->si_tid, &to->si_tid);
-			err |= __put_user(from->si_overrun, &to->si_overrun);
-			err |= __put_user(from->si_int, &to->si_int);
-			break;
-		default:
-			break;
-		}
-	}
-	return err ? -EFAULT : 0;
-}
-
-int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from)
-{
-	int err;
-	u32 tmp;
-
-	err = __get_user(to->si_signo, &from->si_signo);
-	err |= __get_user(to->si_errno, &from->si_errno);
-	err |= __get_user(to->si_code, &from->si_code);
-
-	if (to->si_code < 0)
-		err |= __copy_from_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE);
-	else {
-		switch (siginfo_layout(to->si_signo, to->si_code)) {
-		case SIL_RT:
-			err |= __get_user(to->si_int, &from->si_int);
-			/* fallthrough */
-		case SIL_KILL:
-			err |= __get_user(to->si_pid, &from->si_pid);
-			err |= __get_user(to->si_uid, &from->si_uid);
-			break;
-		case SIL_CHLD:
-			err |= __get_user(to->si_pid, &from->si_pid);
-			err |= __get_user(to->si_uid, &from->si_uid);
-			err |= __get_user(to->si_utime, &from->si_utime);
-			err |= __get_user(to->si_stime, &from->si_stime);
-			err |= __get_user(to->si_status, &from->si_status);
-			break;
-		case SIL_FAULT:
-			err |= __get_user(tmp, &from->si_addr);
-			to->si_addr = (void __force __user *)
-				(u64) (tmp & PSW32_ADDR_INSN);
-			break;
-		case SIL_POLL:
-			err |= __get_user(to->si_band, &from->si_band);
-			err |= __get_user(to->si_fd, &from->si_fd);
-			break;
-		case SIL_TIMER:
-			err |= __get_user(to->si_tid, &from->si_tid);
-			err |= __get_user(to->si_overrun, &from->si_overrun);
-			err |= __get_user(to->si_int, &from->si_int);
-			break;
-		default:
-			break;
-		}
-	}
-	return err ? -EFAULT : 0;
-}
-
 /* Store registers needed to create the signal frame */
 static void store_sigregs(void)
 {
diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index 497a92047591..ac707a9f729e 100644
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -193,6 +193,11 @@ static noinline __init void setup_facility_list(void)
 {
 	stfle(S390_lowcore.stfle_fac_list,
 	      ARRAY_SIZE(S390_lowcore.stfle_fac_list));
+	memcpy(S390_lowcore.alt_stfle_fac_list,
+	       S390_lowcore.stfle_fac_list,
+	       sizeof(S390_lowcore.alt_stfle_fac_list));
+	if (!IS_ENABLED(CONFIG_KERNEL_NOBP))
+		__clear_facility(82, S390_lowcore.alt_stfle_fac_list);
 }
 
 static __init void detect_diag9c(void)
diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S
index 9e5f6cd8e4c2..a5621ea6d123 100644
--- a/arch/s390/kernel/entry.S
+++ b/arch/s390/kernel/entry.S
@@ -14,6 +14,7 @@
 #include <asm/processor.h>
 #include <asm/cache.h>
 #include <asm/ctl_reg.h>
+#include <asm/dwarf.h>
 #include <asm/errno.h>
 #include <asm/ptrace.h>
 #include <asm/thread_info.h>
@@ -107,6 +108,7 @@ _PIF_WORK	= (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
 	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
 	j	3f
 1:	UPDATE_VTIME %r14,%r15,\timer
+	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 2:	lg	%r15,__LC_ASYNC_STACK	# load async stack
 3:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
 	.endm
@@ -159,6 +161,130 @@ _PIF_WORK	= (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
 		tm	off+\addr, \mask
 	.endm
 
+	.macro BPOFF
+	.pushsection .altinstr_replacement, "ax"
+660:	.long	0xb2e8c000
+	.popsection
+661:	.long	0x47000000
+	.pushsection .altinstructions, "a"
+	.long 661b - .
+	.long 660b - .
+	.word 82
+	.byte 4
+	.byte 4
+	.popsection
+	.endm
+
+	.macro BPON
+	.pushsection .altinstr_replacement, "ax"
+662:	.long	0xb2e8d000
+	.popsection
+663:	.long	0x47000000
+	.pushsection .altinstructions, "a"
+	.long 663b - .
+	.long 662b - .
+	.word 82
+	.byte 4
+	.byte 4
+	.popsection
+	.endm
+
+	.macro BPENTER tif_ptr,tif_mask
+	.pushsection .altinstr_replacement, "ax"
+662:	.word	0xc004, 0x0000, 0x0000	# 6 byte nop
+	.word	0xc004, 0x0000, 0x0000	# 6 byte nop
+	.popsection
+664:	TSTMSK	\tif_ptr,\tif_mask
+	jz	. + 8
+	.long	0xb2e8d000
+	.pushsection .altinstructions, "a"
+	.long 664b - .
+	.long 662b - .
+	.word 82
+	.byte 12
+	.byte 12
+	.popsection
+	.endm
+
+	.macro BPEXIT tif_ptr,tif_mask
+	TSTMSK	\tif_ptr,\tif_mask
+	.pushsection .altinstr_replacement, "ax"
+662:	jnz	. + 8
+	.long	0xb2e8d000
+	.popsection
+664:	jz	. + 8
+	.long	0xb2e8c000
+	.pushsection .altinstructions, "a"
+	.long 664b - .
+	.long 662b - .
+	.word 82
+	.byte 8
+	.byte 8
+	.popsection
+	.endm
+
+#ifdef CONFIG_EXPOLINE
+
+	.macro GEN_BR_THUNK name,reg,tmp
+	.section .text.\name,"axG",@progbits,\name,comdat
+	.globl \name
+	.hidden \name
+	.type \name,@function
+\name:
+	CFI_STARTPROC
+#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES
+	exrl	0,0f
+#else
+	larl	\tmp,0f
+	ex	0,0(\tmp)
+#endif
+	j	.
+0:	br	\reg
+	CFI_ENDPROC
+	.endm
+
+	GEN_BR_THUNK __s390x_indirect_jump_r1use_r9,%r9,%r1
+	GEN_BR_THUNK __s390x_indirect_jump_r1use_r14,%r14,%r1
+	GEN_BR_THUNK __s390x_indirect_jump_r11use_r14,%r14,%r11
+
+	.macro BASR_R14_R9
+0:	brasl	%r14,__s390x_indirect_jump_r1use_r9
+	.pushsection .s390_indirect_branches,"a",@progbits
+	.long	0b-.
+	.popsection
+	.endm
+
+	.macro BR_R1USE_R14
+0:	jg	__s390x_indirect_jump_r1use_r14
+	.pushsection .s390_indirect_branches,"a",@progbits
+	.long	0b-.
+	.popsection
+	.endm
+
+	.macro BR_R11USE_R14
+0:	jg	__s390x_indirect_jump_r11use_r14
+	.pushsection .s390_indirect_branches,"a",@progbits
+	.long	0b-.
+	.popsection
+	.endm
+
+#else	/* CONFIG_EXPOLINE */
+
+	.macro BASR_R14_R9
+	basr	%r14,%r9
+	.endm
+
+	.macro BR_R1USE_R14
+	br	%r14
+	.endm
+
+	.macro BR_R11USE_R14
+	br	%r14
+	.endm
+
+#endif /* CONFIG_EXPOLINE */
+
+
 	.section .kprobes.text, "ax"
 .Ldummy:
 	/*
@@ -171,6 +297,11 @@ _PIF_WORK	= (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
 	 */
 	nop	0
 
+ENTRY(__bpon)
+	.globl __bpon
+	BPON
+	BR_R1USE_R14
+
 /*
  * Scheduler resume function, called by switch_to
  *  gpr2 = (task_struct *) prev
@@ -193,9 +324,9 @@ ENTRY(__switch_to)
 	mvc	__LC_CURRENT_PID(4,%r0),0(%r3)	# store pid of next
 	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
 	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_LPP
-	bzr	%r14
+	jz	0f
 	.insn	s,0xb2800000,__LC_LPP		# set program parameter
-	br	%r14
+0:	BR_R1USE_R14
 
 .L__critical_start:
 
@@ -207,9 +338,11 @@ ENTRY(__switch_to)
  */
 ENTRY(sie64a)
 	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
+	lg	%r12,__LC_CURRENT
 	stg	%r2,__SF_EMPTY(%r15)		# save control block pointer
 	stg	%r3,__SF_EMPTY+8(%r15)		# save guest register save area
 	xc	__SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0
+	mvc	__SF_EMPTY+24(8,%r15),__TI_flags(%r12) # copy thread flags
 	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU		# load guest fp/vx registers ?
 	jno	.Lsie_load_guest_gprs
 	brasl	%r14,load_fpu_regs		# load guest fp/vx regs
@@ -226,8 +359,12 @@ ENTRY(sie64a)
 	jnz	.Lsie_skip
 	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
 	jo	.Lsie_skip			# exit if fp/vx regs changed
+	BPEXIT	__SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
 .Lsie_entry:
 	sie	0(%r14)
+.Lsie_exit:
+	BPOFF
+	BPENTER	__SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
 .Lsie_skip:
 	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
 	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
@@ -248,9 +385,15 @@ ENTRY(sie64a)
 sie_exit:
 	lg	%r14,__SF_EMPTY+8(%r15)		# load guest register save area
 	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
+	xgr	%r0,%r0				# clear guest registers to
+	xgr	%r1,%r1				# prevent speculative use
+	xgr	%r2,%r2
+	xgr	%r3,%r3
+	xgr	%r4,%r4
+	xgr	%r5,%r5
 	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
 	lg	%r2,__SF_EMPTY+16(%r15)		# return exit reason code
-	br	%r14
+	BR_R1USE_R14
 .Lsie_fault:
 	lghi	%r14,-EFAULT
 	stg	%r14,__SF_EMPTY+16(%r15)	# set exit reason code
@@ -273,6 +416,7 @@ ENTRY(system_call)
 	stpt	__LC_SYNC_ENTER_TIMER
 .Lsysc_stmg:
 	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
+	BPOFF
 	lg	%r12,__LC_CURRENT
 	lghi	%r13,__TASK_thread
 	lghi	%r14,_PIF_SYSCALL
@@ -281,12 +425,15 @@ ENTRY(system_call)
 	la	%r11,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
 .Lsysc_vtime:
 	UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
+	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 	stmg	%r0,%r7,__PT_R0(%r11)
 	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
 	mvc	__PT_PSW(16,%r11),__LC_SVC_OLD_PSW
 	mvc	__PT_INT_CODE(4,%r11),__LC_SVC_ILC
 	stg	%r14,__PT_FLAGS(%r11)
 .Lsysc_do_svc:
+	# clear user controlled register to prevent speculative use
+	xgr	%r0,%r0
 	# load address of system call table
 	lg	%r10,__THREAD_sysc_table(%r13,%r12)
 	llgh	%r8,__PT_INT_CODE+2(%r11)
@@ -305,7 +452,7 @@ ENTRY(system_call)
 	lgf	%r9,0(%r8,%r10)			# get system call add.
 	TSTMSK	__TI_flags(%r12),_TIF_TRACE
 	jnz	.Lsysc_tracesys
-	basr	%r14,%r9			# call sys_xxxx
+	BASR_R14_R9				# call sys_xxxx
 	stg	%r2,__PT_R2(%r11)		# store return value
 
 .Lsysc_return:
@@ -317,6 +464,7 @@ ENTRY(system_call)
 	jnz	.Lsysc_work			# check for work
 	TSTMSK	__LC_CPU_FLAGS,_CIF_WORK
 	jnz	.Lsysc_work
+	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
 .Lsysc_restore:
 	lg	%r14,__LC_VDSO_PER_CPU
 	lmg	%r0,%r10,__PT_R0(%r11)
@@ -489,7 +637,7 @@ ENTRY(system_call)
 	lmg	%r3,%r7,__PT_R3(%r11)
 	stg	%r7,STACK_FRAME_OVERHEAD(%r15)
 	lg	%r2,__PT_ORIG_GPR2(%r11)
-	basr	%r14,%r9		# call sys_xxx
+	BASR_R14_R9			# call sys_xxx
 	stg	%r2,__PT_R2(%r11)	# store return value
 .Lsysc_tracenogo:
 	TSTMSK	__TI_flags(%r12),_TIF_TRACE
@@ -513,7 +661,7 @@ ENTRY(ret_from_fork)
 	lmg	%r9,%r10,__PT_R9(%r11)	# load gprs
 ENTRY(kernel_thread_starter)
 	la	%r2,0(%r10)
-	basr	%r14,%r9
+	BASR_R14_R9
 	j	.Lsysc_tracenogo
 
 /*
@@ -522,6 +670,7 @@ ENTRY(kernel_thread_starter)
 
 ENTRY(pgm_check_handler)
 	stpt	__LC_SYNC_ENTER_TIMER
+	BPOFF
 	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
 	lg	%r10,__LC_LAST_BREAK
 	lg	%r12,__LC_CURRENT
@@ -550,6 +699,7 @@ ENTRY(pgm_check_handler)
 	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
 	j	4f
 2:	UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
+	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 	lg	%r15,__LC_KERNEL_STACK
 	lgr	%r14,%r12
 	aghi	%r14,__TASK_thread	# pointer to thread_struct
@@ -561,6 +711,15 @@ ENTRY(pgm_check_handler)
 4:	lgr	%r13,%r11
 	la	%r11,STACK_FRAME_OVERHEAD(%r15)
 	stmg	%r0,%r7,__PT_R0(%r11)
+	# clear user controlled registers to prevent speculative use
+	xgr	%r0,%r0
+	xgr	%r1,%r1
+	xgr	%r2,%r2
+	xgr	%r3,%r3
+	xgr	%r4,%r4
+	xgr	%r5,%r5
+	xgr	%r6,%r6
+	xgr	%r7,%r7
 	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
 	stmg	%r8,%r9,__PT_PSW(%r11)
 	mvc	__PT_INT_CODE(4,%r11),__LC_PGM_ILC
@@ -582,9 +741,9 @@ ENTRY(pgm_check_handler)
 	nill	%r10,0x007f
 	sll	%r10,2
 	je	.Lpgm_return
-	lgf	%r1,0(%r10,%r1)		# load address of handler routine
+	lgf	%r9,0(%r10,%r1)		# load address of handler routine
 	lgr	%r2,%r11		# pass pointer to pt_regs
-	basr	%r14,%r1		# branch to interrupt-handler
+	BASR_R14_R9			# branch to interrupt-handler
 .Lpgm_return:
 	LOCKDEP_SYS_EXIT
 	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
@@ -620,12 +779,23 @@ ENTRY(pgm_check_handler)
 ENTRY(io_int_handler)
 	STCK	__LC_INT_CLOCK
 	stpt	__LC_ASYNC_ENTER_TIMER
+	BPOFF
 	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
 	lg	%r12,__LC_CURRENT
 	larl	%r13,cleanup_critical
 	lmg	%r8,%r9,__LC_IO_OLD_PSW
 	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
 	stmg	%r0,%r7,__PT_R0(%r11)
+	# clear user controlled registers to prevent speculative use
+	xgr	%r0,%r0
+	xgr	%r1,%r1
+	xgr	%r2,%r2
+	xgr	%r3,%r3
+	xgr	%r4,%r4
+	xgr	%r5,%r5
+	xgr	%r6,%r6
+	xgr	%r7,%r7
+	xgr	%r10,%r10
 	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
 	stmg	%r8,%r9,__PT_PSW(%r11)
 	mvc	__PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
@@ -660,9 +830,13 @@ ENTRY(io_int_handler)
 	lg	%r14,__LC_VDSO_PER_CPU
 	lmg	%r0,%r10,__PT_R0(%r11)
 	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
+	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
+	jno	.Lio_exit_kernel
+	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
 .Lio_exit_timer:
 	stpt	__LC_EXIT_TIMER
 	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
+.Lio_exit_kernel:
 	lmg	%r11,%r15,__PT_R11(%r11)
 	lpswe	__LC_RETURN_PSW
 .Lio_done:
@@ -833,12 +1007,23 @@ ENTRY(io_int_handler)
 ENTRY(ext_int_handler)
 	STCK	__LC_INT_CLOCK
 	stpt	__LC_ASYNC_ENTER_TIMER
+	BPOFF
 	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
 	lg	%r12,__LC_CURRENT
 	larl	%r13,cleanup_critical
 	lmg	%r8,%r9,__LC_EXT_OLD_PSW
 	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
 	stmg	%r0,%r7,__PT_R0(%r11)
+	# clear user controlled registers to prevent speculative use
+	xgr	%r0,%r0
+	xgr	%r1,%r1
+	xgr	%r2,%r2
+	xgr	%r3,%r3
+	xgr	%r4,%r4
+	xgr	%r5,%r5
+	xgr	%r6,%r6
+	xgr	%r7,%r7
+	xgr	%r10,%r10
 	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
 	stmg	%r8,%r9,__PT_PSW(%r11)
 	lghi	%r1,__LC_EXT_PARAMS2
@@ -871,11 +1056,12 @@ ENTRY(psw_idle)
 .Lpsw_idle_stcctm:
 #endif
 	oi	__LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
+	BPON
 	STCK	__CLOCK_IDLE_ENTER(%r2)
 	stpt	__TIMER_IDLE_ENTER(%r2)
 .Lpsw_idle_lpsw:
 	lpswe	__SF_EMPTY(%r15)
-	br	%r14
+	BR_R1USE_R14
 .Lpsw_idle_end:
 
 /*
@@ -889,7 +1075,7 @@ ENTRY(save_fpu_regs)
 	lg	%r2,__LC_CURRENT
 	aghi	%r2,__TASK_thread
 	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
-	bor	%r14
+	jo	.Lsave_fpu_regs_exit
 	stfpc	__THREAD_FPU_fpc(%r2)
 	lg	%r3,__THREAD_FPU_regs(%r2)
 	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
@@ -916,7 +1102,8 @@ ENTRY(save_fpu_regs)
 	std	15,120(%r3)
 .Lsave_fpu_regs_done:
 	oi	__LC_CPU_FLAGS+7,_CIF_FPU
-	br	%r14
+.Lsave_fpu_regs_exit:
+	BR_R1USE_R14
 .Lsave_fpu_regs_end:
 EXPORT_SYMBOL(save_fpu_regs)
 
@@ -934,7 +1121,7 @@ load_fpu_regs:
 	lg	%r4,__LC_CURRENT
 	aghi	%r4,__TASK_thread
 	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
-	bnor	%r14
+	jno	.Lload_fpu_regs_exit
 	lfpc	__THREAD_FPU_fpc(%r4)
 	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
 	lg	%r4,__THREAD_FPU_regs(%r4)	# %r4 <- reg save area
@@ -961,7 +1148,8 @@ load_fpu_regs:
 	ld	15,120(%r4)
 .Lload_fpu_regs_done:
 	ni	__LC_CPU_FLAGS+7,255-_CIF_FPU
-	br	%r14
+.Lload_fpu_regs_exit:
+	BR_R1USE_R14
 .Lload_fpu_regs_end:
 
 .L__critical_end:
@@ -971,6 +1159,7 @@ load_fpu_regs:
  */
 ENTRY(mcck_int_handler)
 	STCK	__LC_MCCK_CLOCK
+	BPOFF
 	la	%r1,4095		# validate r1
 	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# validate cpu timer
 	sckc	__LC_CLOCK_COMPARATOR			# validate comparator
@@ -1046,6 +1235,16 @@ ENTRY(mcck_int_handler)
 .Lmcck_skip:
 	lghi	%r14,__LC_GPREGS_SAVE_AREA+64
 	stmg	%r0,%r7,__PT_R0(%r11)
+	# clear user controlled registers to prevent speculative use
+	xgr	%r0,%r0
+	xgr	%r1,%r1
+	xgr	%r2,%r2
+	xgr	%r3,%r3
+	xgr	%r4,%r4
+	xgr	%r5,%r5
+	xgr	%r6,%r6
+	xgr	%r7,%r7
+	xgr	%r10,%r10
 	mvc	__PT_R8(64,%r11),0(%r14)
 	stmg	%r8,%r9,__PT_PSW(%r11)
 	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
@@ -1071,6 +1270,7 @@ ENTRY(mcck_int_handler)
 	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
 	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
 	jno	0f
+	BPEXIT	__TI_flags(%r12),_TIF_ISOLATE_BP
 	stpt	__LC_EXIT_TIMER
 	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
 0:	lmg	%r11,%r15,__PT_R11(%r11)
@@ -1166,7 +1366,7 @@ cleanup_critical:
 	jl	0f
 	clg	%r9,BASED(.Lcleanup_table+104)	# .Lload_fpu_regs_end
 	jl	.Lcleanup_load_fpu_regs
-0:	br	%r14
+0:	BR_R11USE_R14
 
 	.align	8
 .Lcleanup_table:
@@ -1197,11 +1397,12 @@ cleanup_critical:
 	clg     %r9,BASED(.Lsie_crit_mcck_length)
 	jh      1f
 	oi      __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
-1:	lg	%r9,__SF_EMPTY(%r15)		# get control block pointer
+1:	BPENTER __SF_EMPTY+24(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
+	lg	%r9,__SF_EMPTY(%r15)		# get control block pointer
 	ni	__SIE_PROG0C+3(%r9),0xfe	# no longer in SIE
 	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
 	larl	%r9,sie_exit			# skip forward to sie_exit
-	br	%r14
+	BR_R11USE_R14
 #endif
 
 .Lcleanup_system_call:
@@ -1239,6 +1440,7 @@ cleanup_critical:
 	stg	%r15,__LC_SYSTEM_TIMER
 0:	# update accounting time stamp
 	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
+	BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
 	# set up saved register r11
 	lg	%r15,__LC_KERNEL_STACK
 	la	%r9,STACK_FRAME_OVERHEAD(%r15)
@@ -1254,7 +1456,7 @@ cleanup_critical:
 	stg	%r15,56(%r11)		# r15 stack pointer
 	# set new psw address and exit
 	larl	%r9,.Lsysc_do_svc
-	br	%r14
+	BR_R11USE_R14
 .Lcleanup_system_call_insn:
 	.quad	system_call
 	.quad	.Lsysc_stmg
@@ -1266,7 +1468,7 @@ cleanup_critical:
 
 .Lcleanup_sysc_tif:
 	larl	%r9,.Lsysc_tif
-	br	%r14
+	BR_R11USE_R14
 
 .Lcleanup_sysc_restore:
 	# check if stpt has been executed
@@ -1283,14 +1485,14 @@ cleanup_critical:
 	mvc	0(64,%r11),__PT_R8(%r9)
 	lmg	%r0,%r7,__PT_R0(%r9)
 1:	lmg	%r8,%r9,__LC_RETURN_PSW
-	br	%r14
+	BR_R11USE_R14
 .Lcleanup_sysc_restore_insn:
 	.quad	.Lsysc_exit_timer
 	.quad	.Lsysc_done - 4
 
 .Lcleanup_io_tif:
 	larl	%r9,.Lio_tif
-	br	%r14
+	BR_R11USE_R14
 
 .Lcleanup_io_restore:
 	# check if stpt has been executed
@@ -1304,7 +1506,7 @@ cleanup_critical:
 	mvc	0(64,%r11),__PT_R8(%r9)
 	lmg	%r0,%r7,__PT_R0(%r9)
 1:	lmg	%r8,%r9,__LC_RETURN_PSW
-	br	%r14
+	BR_R11USE_R14
 .Lcleanup_io_restore_insn:
 	.quad	.Lio_exit_timer
 	.quad	.Lio_done - 4
@@ -1357,17 +1559,17 @@ cleanup_critical:
 	# prepare return psw
 	nihh	%r8,0xfcfd		# clear irq & wait state bits
 	lg	%r9,48(%r11)		# return from psw_idle
-	br	%r14
+	BR_R11USE_R14
 .Lcleanup_idle_insn:
 	.quad	.Lpsw_idle_lpsw
 
 .Lcleanup_save_fpu_regs:
 	larl	%r9,save_fpu_regs
-	br	%r14
+	BR_R11USE_R14
 
 .Lcleanup_load_fpu_regs:
 	larl	%r9,load_fpu_regs
-	br	%r14
+	BR_R11USE_R14
 
 /*
  * Integer constants
@@ -1387,12 +1589,11 @@ cleanup_critical:
 .Lsie_crit_mcck_length:
 	.quad   .Lsie_skip - .Lsie_entry
 #endif
-
 	.section .rodata, "a"
 #define SYSCALL(esame,emu)	.long esame
 	.globl	sys_call_table
 sys_call_table:
-#include "syscalls.S"
+#include "asm/syscall_table.h"
 #undef SYSCALL
 
 #ifdef CONFIG_COMPAT
@@ -1400,6 +1601,6 @@ sys_call_table:
 #define SYSCALL(esame,emu)	.long emu
 	.globl	sys_call_table_emu
 sys_call_table_emu:
-#include "syscalls.S"
+#include "asm/syscall_table.h"
 #undef SYSCALL
 #endif
diff --git a/arch/s390/kernel/head.S b/arch/s390/kernel/head.S
index b86fa3ee4927..5c42f16a54c4 100644
--- a/arch/s390/kernel/head.S
+++ b/arch/s390/kernel/head.S
@@ -66,7 +66,7 @@ __HEAD
 # subroutine to wait for end I/O
 #
 .Lirqwait:
-	mvc	0x1f0(16),.Lnewpsw	# set up IO interrupt psw
+	mvc	__LC_IO_NEW_PSW(16),.Lnewpsw	# set up IO interrupt psw
 	lpsw	.Lwaitpsw
 .Lioint:
 	br	%r14
@@ -98,7 +98,7 @@ __HEAD
 	bnz	.Llderr
 .Lwait4irq:
 	bas	%r14,.Lirqwait
-	c	%r1,0xb8		# compare subchannel number
+	c	%r1,__LC_SUBCHANNEL_ID	# compare subchannel number
 	bne	.Lwait4irq
 	tsch	0(%r5)
 
@@ -156,9 +156,9 @@ iplstart:
 	.fill	16,4,0x0
 0:	lmh	%r0,%r15,0(%r13)	# clear high-order half of gprs
 	sam31				# switch to 31 bit addressing mode
-	lh	%r1,0xb8		# test if subchannel number
+	lh	%r1,__LC_SUBCHANNEL_ID	# test if subchannel number
 	bct	%r1,.Lnoload		#  is valid
-	l	%r1,0xb8		# load ipl subchannel number
+	l	%r1,__LC_SUBCHANNEL_ID	# load ipl subchannel number
 	la	%r2,IPL_BS		# load start address
 	bas	%r14,.Lloader		# load rest of ipl image
 	l	%r12,.Lparm		# pointer to parameter area
@@ -239,7 +239,7 @@ iplstart:
 	bz	.Lnoreset
 .Lwaitforirq:
 	bas	%r14,.Lirqwait		# wait for IO interrupt
-	c	%r1,0xb8		# compare subchannel number
+	c	%r1,__LC_SUBCHANNEL_ID	# compare subchannel number
 	bne	.Lwaitforirq
 	la	%r5,.Lirb
 	tsch	0(%r5)
diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c
index 8ecb8726ac47..34477c1aee6d 100644
--- a/arch/s390/kernel/ipl.c
+++ b/arch/s390/kernel/ipl.c
@@ -119,13 +119,9 @@ static char *dump_type_str(enum dump_type type)
 	}
 }
 
-/*
- * Must be in data section since the bss section
- * is not cleared when these are accessed.
- */
-static u8 ipl_ssid __section(.data) = 0;
-static u16 ipl_devno __section(.data) = 0;
-u32 ipl_flags __section(.data) = 0;
+static u8 ipl_ssid;
+static u16 ipl_devno;
+u32 ipl_flags;
 
 enum ipl_method {
 	REIPL_METHOD_CCW_CIO,
@@ -148,7 +144,7 @@ enum dump_method {
 	DUMP_METHOD_FCP_DIAG,
 };
 
-static int diag308_set_works = 0;
+static int diag308_set_works;
 
 static struct ipl_parameter_block ipl_block;
 
@@ -547,6 +543,7 @@ static struct kset *ipl_kset;
 
 static void __ipl_run(void *unused)
 {
+	__bpon();
 	diag308(DIAG308_LOAD_CLEAR, NULL);
 	if (MACHINE_IS_VM)
 		__cpcmd("IPL", NULL, 0, NULL);
diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c
index af3722c28fd9..60f60afa645c 100644
--- a/arch/s390/kernel/kprobes.c
+++ b/arch/s390/kernel/kprobes.c
@@ -20,7 +20,6 @@
 #include <linux/ftrace.h>
 #include <asm/set_memory.h>
 #include <asm/sections.h>
-#include <linux/uaccess.h>
 #include <asm/dis.h>
 
 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
@@ -282,7 +281,7 @@ static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 		 * is a BUG. The code path resides in the .kprobes.text
 		 * section and is executed with interrupts disabled.
 		 */
-		printk(KERN_EMERG "Invalid kprobe detected at %p.\n", p->addr);
+		pr_err("Invalid kprobe detected.\n");
 		dump_kprobe(p);
 		BUG();
 	}
diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c
index b7abfad4fd7d..1fc6d1ff92d3 100644
--- a/arch/s390/kernel/module.c
+++ b/arch/s390/kernel/module.c
@@ -19,6 +19,8 @@
 #include <linux/moduleloader.h>
 #include <linux/bug.h>
 #include <asm/alternative.h>
+#include <asm/nospec-branch.h>
+#include <asm/facility.h>
 
 #if 0
 #define DEBUGP printk
@@ -156,7 +158,11 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
 	me->arch.got_offset = me->core_layout.size;
 	me->core_layout.size += me->arch.got_size;
 	me->arch.plt_offset = me->core_layout.size;
-	me->core_layout.size += me->arch.plt_size;
+	if (me->arch.plt_size) {
+		if (IS_ENABLED(CONFIG_EXPOLINE) && !nospec_call_disable)
+			me->arch.plt_size += PLT_ENTRY_SIZE;
+		me->core_layout.size += me->arch.plt_size;
+	}
 	return 0;
 }
 
@@ -310,9 +316,21 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab,
 			unsigned int *ip;
 			ip = me->core_layout.base + me->arch.plt_offset +
 				info->plt_offset;
-			ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */
-			ip[1] = 0x100a0004;
-			ip[2] = 0x07f10000;
+			ip[0] = 0x0d10e310;	/* basr 1,0  */
+			ip[1] = 0x100a0004;	/* lg	1,10(1) */
+			if (IS_ENABLED(CONFIG_EXPOLINE) &&
+			    !nospec_call_disable) {
+				unsigned int *ij;
+				ij = me->core_layout.base +
+					me->arch.plt_offset +
+					me->arch.plt_size - PLT_ENTRY_SIZE;
+				ip[2] = 0xa7f40000 +	/* j __jump_r1 */
+					(unsigned int)(u16)
+					(((unsigned long) ij - 8 -
+					  (unsigned long) ip) / 2);
+			} else {
+				ip[2] = 0x07f10000;	/* br %r1 */
+			}
 			ip[3] = (unsigned int) (val >> 32);
 			ip[4] = (unsigned int) val;
 			info->plt_initialized = 1;
@@ -418,16 +436,42 @@ int module_finalize(const Elf_Ehdr *hdr,
 		    struct module *me)
 {
 	const Elf_Shdr *s;
-	char *secstrings;
+	char *secstrings, *secname;
+	void *aseg;
+
+	if (IS_ENABLED(CONFIG_EXPOLINE) &&
+	    !nospec_call_disable && me->arch.plt_size) {
+		unsigned int *ij;
+
+		ij = me->core_layout.base + me->arch.plt_offset +
+			me->arch.plt_size - PLT_ENTRY_SIZE;
+		if (test_facility(35)) {
+			ij[0] = 0xc6000000;	/* exrl	%r0,.+10	*/
+			ij[1] = 0x0005a7f4;	/* j	.		*/
+			ij[2] = 0x000007f1;	/* br	%r1		*/
+		} else {
+			ij[0] = 0x44000000 | (unsigned int)
+				offsetof(struct lowcore, br_r1_trampoline);
+			ij[1] = 0xa7f40000;	/* j	.		*/
+		}
+	}
 
 	secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 	for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) {
-		if (!strcmp(".altinstructions", secstrings + s->sh_name)) {
-			/* patch .altinstructions */
-			void *aseg = (void *)s->sh_addr;
+		aseg = (void *) s->sh_addr;
+		secname = secstrings + s->sh_name;
 
+		if (!strcmp(".altinstructions", secname))
+			/* patch .altinstructions */
 			apply_alternatives(aseg, aseg + s->sh_size);
-		}
+
+		if (IS_ENABLED(CONFIG_EXPOLINE) &&
+		    (!strcmp(".nospec_call_table", secname)))
+			nospec_call_revert(aseg, aseg + s->sh_size);
+
+		if (IS_ENABLED(CONFIG_EXPOLINE) &&
+		    (!strcmp(".nospec_return_table", secname)))
+			nospec_return_revert(aseg, aseg + s->sh_size);
 	}
 
 	jump_label_apply_nops(me);
diff --git a/arch/s390/kernel/nospec-branch.c b/arch/s390/kernel/nospec-branch.c
new file mode 100644
index 000000000000..9aff72d3abda
--- /dev/null
+++ b/arch/s390/kernel/nospec-branch.c
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/module.h>
+#include <asm/nospec-branch.h>
+
+int nospec_call_disable = IS_ENABLED(CONFIG_EXPOLINE_OFF);
+int nospec_return_disable = !IS_ENABLED(CONFIG_EXPOLINE_FULL);
+
+static int __init nospectre_v2_setup_early(char *str)
+{
+	nospec_call_disable = 1;
+	nospec_return_disable = 1;
+	return 0;
+}
+early_param("nospectre_v2", nospectre_v2_setup_early);
+
+static int __init spectre_v2_setup_early(char *str)
+{
+	if (str && !strncmp(str, "on", 2)) {
+		nospec_call_disable = 0;
+		nospec_return_disable = 0;
+	}
+	if (str && !strncmp(str, "off", 3)) {
+		nospec_call_disable = 1;
+		nospec_return_disable = 1;
+	}
+	if (str && !strncmp(str, "auto", 4)) {
+		nospec_call_disable = 0;
+		nospec_return_disable = 1;
+	}
+	return 0;
+}
+early_param("spectre_v2", spectre_v2_setup_early);
+
+static void __init_or_module __nospec_revert(s32 *start, s32 *end)
+{
+	enum { BRCL_EXPOLINE, BRASL_EXPOLINE } type;
+	u8 *instr, *thunk, *br;
+	u8 insnbuf[6];
+	s32 *epo;
+
+	/* Second part of the instruction replace is always a nop */
+	memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x00, 0x00 }, 4);
+	for (epo = start; epo < end; epo++) {
+		instr = (u8 *) epo + *epo;
+		if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04)
+			type = BRCL_EXPOLINE;	/* brcl instruction */
+		else if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x05)
+			type = BRASL_EXPOLINE;	/* brasl instruction */
+		else
+			continue;
+		thunk = instr + (*(int *)(instr + 2)) * 2;
+		if (thunk[0] == 0xc6 && thunk[1] == 0x00)
+			/* exrl %r0,<target-br> */
+			br = thunk + (*(int *)(thunk + 2)) * 2;
+		else if (thunk[0] == 0xc0 && (thunk[1] & 0x0f) == 0x00 &&
+			 thunk[6] == 0x44 && thunk[7] == 0x00 &&
+			 (thunk[8] & 0x0f) == 0x00 && thunk[9] == 0x00 &&
+			 (thunk[1] & 0xf0) == (thunk[8] & 0xf0))
+			/* larl %rx,<target br> + ex %r0,0(%rx) */
+			br = thunk + (*(int *)(thunk + 2)) * 2;
+		else
+			continue;
+		if (br[0] != 0x07 || (br[1] & 0xf0) != 0xf0)
+			continue;
+		switch (type) {
+		case BRCL_EXPOLINE:
+			/* brcl to thunk, replace with br + nop */
+			insnbuf[0] = br[0];
+			insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
+			break;
+		case BRASL_EXPOLINE:
+			/* brasl to thunk, replace with basr + nop */
+			insnbuf[0] = 0x0d;
+			insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
+			break;
+		}
+
+		s390_kernel_write(instr, insnbuf, 6);
+	}
+}
+
+void __init_or_module nospec_call_revert(s32 *start, s32 *end)
+{
+	if (nospec_call_disable)
+		__nospec_revert(start, end);
+}
+
+void __init_or_module nospec_return_revert(s32 *start, s32 *end)
+{
+	if (nospec_return_disable)
+		__nospec_revert(start, end);
+}
+
+extern s32 __nospec_call_start[], __nospec_call_end[];
+extern s32 __nospec_return_start[], __nospec_return_end[];
+void __init nospec_init_branches(void)
+{
+	nospec_call_revert(__nospec_call_start, __nospec_call_end);
+	nospec_return_revert(__nospec_return_start, __nospec_return_end);
+}
diff --git a/arch/s390/kernel/perf_cpum_cf_events.c b/arch/s390/kernel/perf_cpum_cf_events.c
index 94f90cefbffc..c5bc3f209652 100644
--- a/arch/s390/kernel/perf_cpum_cf_events.c
+++ b/arch/s390/kernel/perf_cpum_cf_events.c
@@ -226,7 +226,7 @@ CPUMF_EVENT_ATTR(cf_z14, L1I_OFFDRAWER_L4_SOURCED_WRITES, 0x00af);
 CPUMF_EVENT_ATTR(cf_z14, BCD_DFP_EXECUTION_SLOTS, 0x00e0);
 CPUMF_EVENT_ATTR(cf_z14, VX_BCD_EXECUTION_SLOTS, 0x00e1);
 CPUMF_EVENT_ATTR(cf_z14, DECIMAL_INSTRUCTIONS, 0x00e2);
-CPUMF_EVENT_ATTR(cf_z14, LAST_HOST_TRANSLATIONS, 0x00e9);
+CPUMF_EVENT_ATTR(cf_z14, LAST_HOST_TRANSLATIONS, 0x00e8);
 CPUMF_EVENT_ATTR(cf_z14, TX_NC_TABORT, 0x00f3);
 CPUMF_EVENT_ATTR(cf_z14, TX_C_TABORT_NO_SPECIAL, 0x00f4);
 CPUMF_EVENT_ATTR(cf_z14, TX_C_TABORT_SPECIAL, 0x00f5);
diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c
index 5362fd868d0d..6fe2e1875058 100644
--- a/arch/s390/kernel/processor.c
+++ b/arch/s390/kernel/processor.c
@@ -197,3 +197,21 @@ const struct seq_operations cpuinfo_op = {
 	.stop	= c_stop,
 	.show	= show_cpuinfo,
 };
+
+int s390_isolate_bp(void)
+{
+	if (!test_facility(82))
+		return -EOPNOTSUPP;
+	set_thread_flag(TIF_ISOLATE_BP);
+	return 0;
+}
+EXPORT_SYMBOL(s390_isolate_bp);
+
+int s390_isolate_bp_guest(void)
+{
+	if (!test_facility(82))
+		return -EOPNOTSUPP;
+	set_thread_flag(TIF_ISOLATE_BP_GUEST);
+	return 0;
+}
+EXPORT_SYMBOL(s390_isolate_bp_guest);
diff --git a/arch/s390/kernel/runtime_instr.c b/arch/s390/kernel/runtime_instr.c
index 09f5bf0d5c0c..125c7f6e8715 100644
--- a/arch/s390/kernel/runtime_instr.c
+++ b/arch/s390/kernel/runtime_instr.c
@@ -18,6 +18,8 @@
 #include <asm/cpu_mf.h>
 #include <asm/irq.h>
 
+#include "entry.h"
+
 /* empty control block to disable RI by loading it */
 struct runtime_instr_cb runtime_instr_empty_cb;
 
@@ -59,7 +61,13 @@ static void init_runtime_instr_cb(struct runtime_instr_cb *cb)
 	cb->v = 1;
 }
 
-SYSCALL_DEFINE1(s390_runtime_instr, int, command)
+/*
+ * The signum argument is unused. In older kernels it was used to
+ * specify a real-time signal. For backwards compatibility user space
+ * should pass a valid real-time signal number (the signum argument
+ * was checked in older kernels).
+ */
+SYSCALL_DEFINE2(s390_runtime_instr, int, command, int, signum)
 {
 	struct runtime_instr_cb *cb;
 
diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c
index 793da97f9a6e..a6a91f01a17a 100644
--- a/arch/s390/kernel/setup.c
+++ b/arch/s390/kernel/setup.c
@@ -68,6 +68,7 @@
 #include <asm/sysinfo.h>
 #include <asm/numa.h>
 #include <asm/alternative.h>
+#include <asm/nospec-branch.h>
 #include "entry.h"
 
 /*
@@ -340,7 +341,9 @@ static void __init setup_lowcore(void)
 	lc->preempt_count = S390_lowcore.preempt_count;
 	lc->stfl_fac_list = S390_lowcore.stfl_fac_list;
 	memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
-	       MAX_FACILITY_BIT/8);
+	       sizeof(lc->stfle_fac_list));
+	memcpy(lc->alt_stfle_fac_list, S390_lowcore.alt_stfle_fac_list,
+	       sizeof(lc->alt_stfle_fac_list));
 	nmi_alloc_boot_cpu(lc);
 	vdso_alloc_boot_cpu(lc);
 	lc->sync_enter_timer = S390_lowcore.sync_enter_timer;
@@ -377,6 +380,7 @@ static void __init setup_lowcore(void)
 	lc->spinlock_index = 0;
 	arch_spin_lock_setup(0);
 #endif
+	lc->br_r1_trampoline = 0x07f1;	/* br %r1 */
 
 	set_prefix((u32)(unsigned long) lc);
 	lowcore_ptr[0] = lc;
@@ -952,6 +956,8 @@ void __init setup_arch(char **cmdline_p)
 	set_preferred_console();
 
 	apply_alternative_instructions();
+	if (IS_ENABLED(CONFIG_EXPOLINE))
+		nospec_init_branches();
 
 	/* Setup zfcpdump support */
 	setup_zfcpdump();
diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index b8c1a85bcf2d..a4a9fe1934e9 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -214,6 +214,7 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
 	lc->cpu_nr = cpu;
 	lc->spinlock_lockval = arch_spin_lockval(cpu);
 	lc->spinlock_index = 0;
+	lc->br_r1_trampoline = 0x07f1;	/* br %r1 */
 	if (nmi_alloc_per_cpu(lc))
 		goto out;
 	if (vdso_alloc_per_cpu(lc))
@@ -266,7 +267,9 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
 	__ctl_store(lc->cregs_save_area, 0, 15);
 	save_access_regs((unsigned int *) lc->access_regs_save_area);
 	memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
-	       MAX_FACILITY_BIT/8);
+	       sizeof(lc->stfle_fac_list));
+	memcpy(lc->alt_stfle_fac_list, S390_lowcore.alt_stfle_fac_list,
+	       sizeof(lc->alt_stfle_fac_list));
 	arch_spin_lock_setup(cpu);
 }
 
@@ -317,6 +320,7 @@ static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
 	mem_assign_absolute(lc->restart_fn, (unsigned long) func);
 	mem_assign_absolute(lc->restart_data, (unsigned long) data);
 	mem_assign_absolute(lc->restart_source, source_cpu);
+	__bpon();
 	asm volatile(
 		"0:	sigp	0,%0,%2	# sigp restart to target cpu\n"
 		"	brc	2,0b	# busy, try again\n"
@@ -901,6 +905,7 @@ void __cpu_die(unsigned int cpu)
 void __noreturn cpu_die(void)
 {
 	idle_task_exit();
+	__bpon();
 	pcpu_sigp_retry(pcpu_devices + smp_processor_id(), SIGP_STOP, 0);
 	for (;;) ;
 }
@@ -1151,7 +1156,7 @@ static ssize_t __ref rescan_store(struct device *dev,
 	rc = smp_rescan_cpus();
 	return rc ? rc : count;
 }
-static DEVICE_ATTR(rescan, 0200, NULL, rescan_store);
+static DEVICE_ATTR_WO(rescan);
 #endif /* CONFIG_HOTPLUG_CPU */
 
 static int __init s390_smp_init(void)
diff --git a/arch/s390/kernel/syscalls.S b/arch/s390/kernel/syscalls.S
deleted file mode 100644
index f7fc63385553..000000000000
--- a/arch/s390/kernel/syscalls.S
+++ /dev/null
@@ -1,392 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * definitions for sys_call_table, each line represents an
- * entry in the table in the form
- * SYSCALL(64 bit syscall, 31 bit emulated syscall)
- *
- * this file is meant to be included from entry.S
- */
-
-#define NI_SYSCALL SYSCALL(sys_ni_syscall,sys_ni_syscall)
-
-NI_SYSCALL						/* 0 */
-SYSCALL(sys_exit,sys_exit)
-SYSCALL(sys_fork,sys_fork)
-SYSCALL(sys_read,compat_sys_s390_read)
-SYSCALL(sys_write,compat_sys_s390_write)
-SYSCALL(sys_open,compat_sys_open)			/* 5 */
-SYSCALL(sys_close,sys_close)
-SYSCALL(sys_restart_syscall,sys_restart_syscall)
-SYSCALL(sys_creat,compat_sys_creat)
-SYSCALL(sys_link,compat_sys_link)
-SYSCALL(sys_unlink,compat_sys_unlink)			/* 10 */
-SYSCALL(sys_execve,compat_sys_execve)
-SYSCALL(sys_chdir,compat_sys_chdir)
-SYSCALL(sys_ni_syscall,compat_sys_time)			/* old time syscall */
-SYSCALL(sys_mknod,compat_sys_mknod)
-SYSCALL(sys_chmod,compat_sys_chmod)			/* 15 */
-SYSCALL(sys_ni_syscall,compat_sys_s390_lchown16)	/* old lchown16 syscall*/
-NI_SYSCALL						/* old break syscall holder */
-NI_SYSCALL						/* old stat syscall holder */
-SYSCALL(sys_lseek,compat_sys_lseek)
-SYSCALL(sys_getpid,sys_getpid)				/* 20 */
-SYSCALL(sys_mount,compat_sys_mount)
-SYSCALL(sys_oldumount,compat_sys_oldumount)
-SYSCALL(sys_ni_syscall,compat_sys_s390_setuid16)	/* old setuid16 syscall*/
-SYSCALL(sys_ni_syscall,compat_sys_s390_getuid16)	/* old getuid16 syscall*/
-SYSCALL(sys_ni_syscall,compat_sys_stime)		/* 25 old stime syscall */
-SYSCALL(sys_ptrace,compat_sys_ptrace)
-SYSCALL(sys_alarm,sys_alarm)
-NI_SYSCALL						/* old fstat syscall */
-SYSCALL(sys_pause,sys_pause)
-SYSCALL(sys_utime,compat_sys_utime)			/* 30 */
-NI_SYSCALL						/* old stty syscall */
-NI_SYSCALL						/* old gtty syscall */
-SYSCALL(sys_access,compat_sys_access)
-SYSCALL(sys_nice,sys_nice)
-NI_SYSCALL						/* 35 old ftime syscall */
-SYSCALL(sys_sync,sys_sync)
-SYSCALL(sys_kill,sys_kill)
-SYSCALL(sys_rename,compat_sys_rename)
-SYSCALL(sys_mkdir,compat_sys_mkdir)
-SYSCALL(sys_rmdir,compat_sys_rmdir)			/* 40 */
-SYSCALL(sys_dup,sys_dup)
-SYSCALL(sys_pipe,compat_sys_pipe)
-SYSCALL(sys_times,compat_sys_times)
-NI_SYSCALL						/* old prof syscall */
-SYSCALL(sys_brk,compat_sys_brk)				/* 45 */
-SYSCALL(sys_ni_syscall,compat_sys_s390_setgid16)	/* old setgid16 syscall*/
-SYSCALL(sys_ni_syscall,compat_sys_s390_getgid16)	/* old getgid16 syscall*/
-SYSCALL(sys_signal,compat_sys_signal)
-SYSCALL(sys_ni_syscall,compat_sys_s390_geteuid16)	/* old geteuid16 syscall */
-SYSCALL(sys_ni_syscall,compat_sys_s390_getegid16)	/* 50 old getegid16 syscall */
-SYSCALL(sys_acct,compat_sys_acct)
-SYSCALL(sys_umount,compat_sys_umount)
-NI_SYSCALL						/* old lock syscall */
-SYSCALL(sys_ioctl,compat_sys_ioctl)
-SYSCALL(sys_fcntl,compat_sys_fcntl)			/* 55 */
-NI_SYSCALL						/* intel mpx syscall */
-SYSCALL(sys_setpgid,sys_setpgid)
-NI_SYSCALL						/* old ulimit syscall */
-NI_SYSCALL						/* old uname syscall */
-SYSCALL(sys_umask,sys_umask)				/* 60 */
-SYSCALL(sys_chroot,compat_sys_chroot)
-SYSCALL(sys_ustat,compat_sys_ustat)
-SYSCALL(sys_dup2,sys_dup2)
-SYSCALL(sys_getppid,sys_getppid)
-SYSCALL(sys_getpgrp,sys_getpgrp)			/* 65 */
-SYSCALL(sys_setsid,sys_setsid)
-SYSCALL(sys_sigaction,compat_sys_sigaction)
-NI_SYSCALL						/* old sgetmask syscall*/
-NI_SYSCALL						/* old ssetmask syscall*/
-SYSCALL(sys_ni_syscall,compat_sys_s390_setreuid16)	/* old setreuid16 syscall */
-SYSCALL(sys_ni_syscall,compat_sys_s390_setregid16)	/* old setregid16 syscall */
-SYSCALL(sys_sigsuspend,compat_sys_sigsuspend)
-SYSCALL(sys_sigpending,compat_sys_sigpending)
-SYSCALL(sys_sethostname,compat_sys_sethostname)
-SYSCALL(sys_setrlimit,compat_sys_setrlimit)		/* 75 */
-SYSCALL(sys_getrlimit,compat_sys_old_getrlimit)
-SYSCALL(sys_getrusage,compat_sys_getrusage)
-SYSCALL(sys_gettimeofday,compat_sys_gettimeofday)
-SYSCALL(sys_settimeofday,compat_sys_settimeofday)
-SYSCALL(sys_ni_syscall,compat_sys_s390_getgroups16)	/* 80 old getgroups16 syscall */
-SYSCALL(sys_ni_syscall,compat_sys_s390_setgroups16)	/* old setgroups16 syscall */
-NI_SYSCALL						/* old select syscall */
-SYSCALL(sys_symlink,compat_sys_symlink)
-NI_SYSCALL						/* old lstat syscall */
-SYSCALL(sys_readlink,compat_sys_readlink)		/* 85 */
-SYSCALL(sys_uselib,compat_sys_uselib)
-SYSCALL(sys_swapon,compat_sys_swapon)
-SYSCALL(sys_reboot,compat_sys_reboot)
-SYSCALL(sys_ni_syscall,compat_sys_old_readdir)		/* old readdir syscall */
-SYSCALL(sys_old_mmap,compat_sys_s390_old_mmap)		/* 90 */
-SYSCALL(sys_munmap,compat_sys_munmap)
-SYSCALL(sys_truncate,compat_sys_truncate)
-SYSCALL(sys_ftruncate,compat_sys_ftruncate)
-SYSCALL(sys_fchmod,sys_fchmod)
-SYSCALL(sys_ni_syscall,compat_sys_s390_fchown16)	/* 95 old fchown16 syscall*/
-SYSCALL(sys_getpriority,sys_getpriority)
-SYSCALL(sys_setpriority,sys_setpriority)
-NI_SYSCALL						/* old profil syscall */
-SYSCALL(sys_statfs,compat_sys_statfs)
-SYSCALL(sys_fstatfs,compat_sys_fstatfs)			/* 100 */
-NI_SYSCALL						/* ioperm for i386 */
-SYSCALL(sys_socketcall,compat_sys_socketcall)
-SYSCALL(sys_syslog,compat_sys_syslog)
-SYSCALL(sys_setitimer,compat_sys_setitimer)
-SYSCALL(sys_getitimer,compat_sys_getitimer)		/* 105 */
-SYSCALL(sys_newstat,compat_sys_newstat)
-SYSCALL(sys_newlstat,compat_sys_newlstat)
-SYSCALL(sys_newfstat,compat_sys_newfstat)
-NI_SYSCALL						/* old uname syscall */
-SYSCALL(sys_lookup_dcookie,compat_sys_lookup_dcookie)	/* 110 */
-SYSCALL(sys_vhangup,sys_vhangup)
-NI_SYSCALL						/* old "idle" system call */
-NI_SYSCALL						/* vm86old for i386 */
-SYSCALL(sys_wait4,compat_sys_wait4)
-SYSCALL(sys_swapoff,compat_sys_swapoff)			/* 115 */
-SYSCALL(sys_sysinfo,compat_sys_sysinfo)
-SYSCALL(sys_s390_ipc,compat_sys_s390_ipc)
-SYSCALL(sys_fsync,sys_fsync)
-SYSCALL(sys_sigreturn,compat_sys_sigreturn)
-SYSCALL(sys_clone,compat_sys_clone)			/* 120 */
-SYSCALL(sys_setdomainname,compat_sys_setdomainname)
-SYSCALL(sys_newuname,compat_sys_newuname)
-NI_SYSCALL						/* modify_ldt for i386 */
-SYSCALL(sys_adjtimex,compat_sys_adjtimex)
-SYSCALL(sys_mprotect,compat_sys_mprotect)		/* 125 */
-SYSCALL(sys_sigprocmask,compat_sys_sigprocmask)
-NI_SYSCALL						/* old "create module" */
-SYSCALL(sys_init_module,compat_sys_init_module)
-SYSCALL(sys_delete_module,compat_sys_delete_module)
-NI_SYSCALL						/* 130: old get_kernel_syms */
-SYSCALL(sys_quotactl,compat_sys_quotactl)
-SYSCALL(sys_getpgid,sys_getpgid)
-SYSCALL(sys_fchdir,sys_fchdir)
-SYSCALL(sys_bdflush,compat_sys_bdflush)
-SYSCALL(sys_sysfs,compat_sys_sysfs)			/* 135 */
-SYSCALL(sys_s390_personality,sys_s390_personality)
-NI_SYSCALL						/* for afs_syscall */
-SYSCALL(sys_ni_syscall,compat_sys_s390_setfsuid16)	/* old setfsuid16 syscall */
-SYSCALL(sys_ni_syscall,compat_sys_s390_setfsgid16)	/* old setfsgid16 syscall */
-SYSCALL(sys_llseek,compat_sys_llseek)			/* 140 */
-SYSCALL(sys_getdents,compat_sys_getdents)
-SYSCALL(sys_select,compat_sys_select)
-SYSCALL(sys_flock,sys_flock)
-SYSCALL(sys_msync,compat_sys_msync)
-SYSCALL(sys_readv,compat_sys_readv)			/* 145 */
-SYSCALL(sys_writev,compat_sys_writev)
-SYSCALL(sys_getsid,sys_getsid)
-SYSCALL(sys_fdatasync,sys_fdatasync)
-SYSCALL(sys_sysctl,compat_sys_sysctl)
-SYSCALL(sys_mlock,compat_sys_mlock)			/* 150 */
-SYSCALL(sys_munlock,compat_sys_munlock)
-SYSCALL(sys_mlockall,sys_mlockall)
-SYSCALL(sys_munlockall,sys_munlockall)
-SYSCALL(sys_sched_setparam,compat_sys_sched_setparam)
-SYSCALL(sys_sched_getparam,compat_sys_sched_getparam)	/* 155 */
-SYSCALL(sys_sched_setscheduler,compat_sys_sched_setscheduler)
-SYSCALL(sys_sched_getscheduler,sys_sched_getscheduler)
-SYSCALL(sys_sched_yield,sys_sched_yield)
-SYSCALL(sys_sched_get_priority_max,sys_sched_get_priority_max)
-SYSCALL(sys_sched_get_priority_min,sys_sched_get_priority_min)	/* 160 */
-SYSCALL(sys_sched_rr_get_interval,compat_sys_sched_rr_get_interval)
-SYSCALL(sys_nanosleep,compat_sys_nanosleep)
-SYSCALL(sys_mremap,compat_sys_mremap)
-SYSCALL(sys_ni_syscall,compat_sys_s390_setresuid16)	/* old setresuid16 syscall */
-SYSCALL(sys_ni_syscall,compat_sys_s390_getresuid16)	/* 165 old getresuid16 syscall */
-NI_SYSCALL						/* for vm86 */
-NI_SYSCALL						/* old sys_query_module */
-SYSCALL(sys_poll,compat_sys_poll)
-NI_SYSCALL						/* old nfsservctl */
-SYSCALL(sys_ni_syscall,compat_sys_s390_setresgid16)	/* 170 old setresgid16 syscall */
-SYSCALL(sys_ni_syscall,compat_sys_s390_getresgid16)	/* old getresgid16 syscall */
-SYSCALL(sys_prctl,compat_sys_prctl)
-SYSCALL(sys_rt_sigreturn,compat_sys_rt_sigreturn)
-SYSCALL(sys_rt_sigaction,compat_sys_rt_sigaction)
-SYSCALL(sys_rt_sigprocmask,compat_sys_rt_sigprocmask)	/* 175 */
-SYSCALL(sys_rt_sigpending,compat_sys_rt_sigpending)
-SYSCALL(sys_rt_sigtimedwait,compat_sys_rt_sigtimedwait)
-SYSCALL(sys_rt_sigqueueinfo,compat_sys_rt_sigqueueinfo)
-SYSCALL(sys_rt_sigsuspend,compat_sys_rt_sigsuspend)
-SYSCALL(sys_pread64,compat_sys_s390_pread64)		/* 180 */
-SYSCALL(sys_pwrite64,compat_sys_s390_pwrite64)
-SYSCALL(sys_ni_syscall,compat_sys_s390_chown16)		/* old chown16 syscall */
-SYSCALL(sys_getcwd,compat_sys_getcwd)
-SYSCALL(sys_capget,compat_sys_capget)
-SYSCALL(sys_capset,compat_sys_capset)			/* 185 */
-SYSCALL(sys_sigaltstack,compat_sys_sigaltstack)
-SYSCALL(sys_sendfile64,compat_sys_sendfile)
-NI_SYSCALL						/* streams1 */
-NI_SYSCALL						/* streams2 */
-SYSCALL(sys_vfork,sys_vfork)				/* 190 */
-SYSCALL(sys_getrlimit,compat_sys_getrlimit)
-SYSCALL(sys_mmap2,compat_sys_s390_mmap2)
-SYSCALL(sys_ni_syscall,compat_sys_s390_truncate64)
-SYSCALL(sys_ni_syscall,compat_sys_s390_ftruncate64)
-SYSCALL(sys_ni_syscall,compat_sys_s390_stat64)		/* 195 */
-SYSCALL(sys_ni_syscall,compat_sys_s390_lstat64)
-SYSCALL(sys_ni_syscall,compat_sys_s390_fstat64)
-SYSCALL(sys_lchown,compat_sys_lchown)
-SYSCALL(sys_getuid,sys_getuid)
-SYSCALL(sys_getgid,sys_getgid)				/* 200 */
-SYSCALL(sys_geteuid,sys_geteuid)
-SYSCALL(sys_getegid,sys_getegid)
-SYSCALL(sys_setreuid,sys_setreuid)
-SYSCALL(sys_setregid,sys_setregid)
-SYSCALL(sys_getgroups,compat_sys_getgroups)		/* 205 */
-SYSCALL(sys_setgroups,compat_sys_setgroups)
-SYSCALL(sys_fchown,sys_fchown)
-SYSCALL(sys_setresuid,sys_setresuid)
-SYSCALL(sys_getresuid,compat_sys_getresuid)
-SYSCALL(sys_setresgid,sys_setresgid)			/* 210 */
-SYSCALL(sys_getresgid,compat_sys_getresgid)
-SYSCALL(sys_chown,compat_sys_chown)
-SYSCALL(sys_setuid,sys_setuid)
-SYSCALL(sys_setgid,sys_setgid)
-SYSCALL(sys_setfsuid,sys_setfsuid)			/* 215 */
-SYSCALL(sys_setfsgid,sys_setfsgid)
-SYSCALL(sys_pivot_root,compat_sys_pivot_root)
-SYSCALL(sys_mincore,compat_sys_mincore)
-SYSCALL(sys_madvise,compat_sys_madvise)
-SYSCALL(sys_getdents64,compat_sys_getdents64)		/* 220 */
-SYSCALL(sys_ni_syscall,compat_sys_fcntl64)
-SYSCALL(sys_readahead,compat_sys_s390_readahead)
-SYSCALL(sys_ni_syscall,compat_sys_sendfile64)
-SYSCALL(sys_setxattr,compat_sys_setxattr)
-SYSCALL(sys_lsetxattr,compat_sys_lsetxattr)		/* 225 */
-SYSCALL(sys_fsetxattr,compat_sys_fsetxattr)
-SYSCALL(sys_getxattr,compat_sys_getxattr)
-SYSCALL(sys_lgetxattr,compat_sys_lgetxattr)
-SYSCALL(sys_fgetxattr,compat_sys_fgetxattr)
-SYSCALL(sys_listxattr,compat_sys_listxattr)		/* 230 */
-SYSCALL(sys_llistxattr,compat_sys_llistxattr)
-SYSCALL(sys_flistxattr,compat_sys_flistxattr)
-SYSCALL(sys_removexattr,compat_sys_removexattr)
-SYSCALL(sys_lremovexattr,compat_sys_lremovexattr)
-SYSCALL(sys_fremovexattr,compat_sys_fremovexattr)	/* 235 */
-SYSCALL(sys_gettid,sys_gettid)
-SYSCALL(sys_tkill,sys_tkill)
-SYSCALL(sys_futex,compat_sys_futex)
-SYSCALL(sys_sched_setaffinity,compat_sys_sched_setaffinity)
-SYSCALL(sys_sched_getaffinity,compat_sys_sched_getaffinity)	/* 240 */
-SYSCALL(sys_tgkill,sys_tgkill)
-NI_SYSCALL						/* reserved for TUX */
-SYSCALL(sys_io_setup,compat_sys_io_setup)
-SYSCALL(sys_io_destroy,compat_sys_io_destroy)
-SYSCALL(sys_io_getevents,compat_sys_io_getevents)	/* 245 */
-SYSCALL(sys_io_submit,compat_sys_io_submit)
-SYSCALL(sys_io_cancel,compat_sys_io_cancel)
-SYSCALL(sys_exit_group,sys_exit_group)
-SYSCALL(sys_epoll_create,sys_epoll_create)
-SYSCALL(sys_epoll_ctl,compat_sys_epoll_ctl)		/* 250 */
-SYSCALL(sys_epoll_wait,compat_sys_epoll_wait)
-SYSCALL(sys_set_tid_address,compat_sys_set_tid_address)
-SYSCALL(sys_fadvise64_64,compat_sys_s390_fadvise64)
-SYSCALL(sys_timer_create,compat_sys_timer_create)
-SYSCALL(sys_timer_settime,compat_sys_timer_settime)	/* 255 */
-SYSCALL(sys_timer_gettime,compat_sys_timer_gettime)
-SYSCALL(sys_timer_getoverrun,sys_timer_getoverrun)
-SYSCALL(sys_timer_delete,sys_timer_delete)
-SYSCALL(sys_clock_settime,compat_sys_clock_settime)
-SYSCALL(sys_clock_gettime,compat_sys_clock_gettime)	/* 260 */
-SYSCALL(sys_clock_getres,compat_sys_clock_getres)
-SYSCALL(sys_clock_nanosleep,compat_sys_clock_nanosleep)
-NI_SYSCALL						/* reserved for vserver */
-SYSCALL(sys_ni_syscall,compat_sys_s390_fadvise64_64)
-SYSCALL(sys_statfs64,compat_sys_statfs64)
-SYSCALL(sys_fstatfs64,compat_sys_fstatfs64)
-SYSCALL(sys_remap_file_pages,compat_sys_remap_file_pages)
-SYSCALL(sys_mbind,compat_sys_mbind)
-SYSCALL(sys_get_mempolicy,compat_sys_get_mempolicy)
-SYSCALL(sys_set_mempolicy,compat_sys_set_mempolicy)
-SYSCALL(sys_mq_open,compat_sys_mq_open)
-SYSCALL(sys_mq_unlink,compat_sys_mq_unlink)
-SYSCALL(sys_mq_timedsend,compat_sys_mq_timedsend)
-SYSCALL(sys_mq_timedreceive,compat_sys_mq_timedreceive)
-SYSCALL(sys_mq_notify,compat_sys_mq_notify)		/* 275 */
-SYSCALL(sys_mq_getsetattr,compat_sys_mq_getsetattr)
-SYSCALL(sys_kexec_load,compat_sys_kexec_load)
-SYSCALL(sys_add_key,compat_sys_add_key)
-SYSCALL(sys_request_key,compat_sys_request_key)
-SYSCALL(sys_keyctl,compat_sys_keyctl)			/* 280 */
-SYSCALL(sys_waitid,compat_sys_waitid)
-SYSCALL(sys_ioprio_set,sys_ioprio_set)
-SYSCALL(sys_ioprio_get,sys_ioprio_get)
-SYSCALL(sys_inotify_init,sys_inotify_init)
-SYSCALL(sys_inotify_add_watch,compat_sys_inotify_add_watch)	/* 285 */
-SYSCALL(sys_inotify_rm_watch,sys_inotify_rm_watch)
-SYSCALL(sys_migrate_pages,compat_sys_migrate_pages)
-SYSCALL(sys_openat,compat_sys_openat)
-SYSCALL(sys_mkdirat,compat_sys_mkdirat)
-SYSCALL(sys_mknodat,compat_sys_mknodat)			/* 290 */
-SYSCALL(sys_fchownat,compat_sys_fchownat)
-SYSCALL(sys_futimesat,compat_sys_futimesat)
-SYSCALL(sys_newfstatat,compat_sys_s390_fstatat64)
-SYSCALL(sys_unlinkat,compat_sys_unlinkat)
-SYSCALL(sys_renameat,compat_sys_renameat)		/* 295 */
-SYSCALL(sys_linkat,compat_sys_linkat)
-SYSCALL(sys_symlinkat,compat_sys_symlinkat)
-SYSCALL(sys_readlinkat,compat_sys_readlinkat)
-SYSCALL(sys_fchmodat,compat_sys_fchmodat)
-SYSCALL(sys_faccessat,compat_sys_faccessat)		/* 300 */
-SYSCALL(sys_pselect6,compat_sys_pselect6)
-SYSCALL(sys_ppoll,compat_sys_ppoll)
-SYSCALL(sys_unshare,compat_sys_unshare)
-SYSCALL(sys_set_robust_list,compat_sys_set_robust_list)
-SYSCALL(sys_get_robust_list,compat_sys_get_robust_list)
-SYSCALL(sys_splice,compat_sys_splice)
-SYSCALL(sys_sync_file_range,compat_sys_s390_sync_file_range)
-SYSCALL(sys_tee,compat_sys_tee)
-SYSCALL(sys_vmsplice,compat_sys_vmsplice)
-SYSCALL(sys_move_pages,compat_sys_move_pages)
-SYSCALL(sys_getcpu,compat_sys_getcpu)
-SYSCALL(sys_epoll_pwait,compat_sys_epoll_pwait)
-SYSCALL(sys_utimes,compat_sys_utimes)
-SYSCALL(sys_fallocate,compat_sys_s390_fallocate)
-SYSCALL(sys_utimensat,compat_sys_utimensat)		/* 315 */
-SYSCALL(sys_signalfd,compat_sys_signalfd)
-NI_SYSCALL						/* 317 old sys_timer_fd */
-SYSCALL(sys_eventfd,sys_eventfd)
-SYSCALL(sys_timerfd_create,sys_timerfd_create)
-SYSCALL(sys_timerfd_settime,compat_sys_timerfd_settime) /* 320 */
-SYSCALL(sys_timerfd_gettime,compat_sys_timerfd_gettime)
-SYSCALL(sys_signalfd4,compat_sys_signalfd4)
-SYSCALL(sys_eventfd2,sys_eventfd2)
-SYSCALL(sys_inotify_init1,sys_inotify_init1)
-SYSCALL(sys_pipe2,compat_sys_pipe2)			/* 325 */
-SYSCALL(sys_dup3,sys_dup3)
-SYSCALL(sys_epoll_create1,sys_epoll_create1)
-SYSCALL(sys_preadv,compat_sys_preadv)
-SYSCALL(sys_pwritev,compat_sys_pwritev)
-SYSCALL(sys_rt_tgsigqueueinfo,compat_sys_rt_tgsigqueueinfo) /* 330 */
-SYSCALL(sys_perf_event_open,compat_sys_perf_event_open)
-SYSCALL(sys_fanotify_init,sys_fanotify_init)
-SYSCALL(sys_fanotify_mark,compat_sys_fanotify_mark)
-SYSCALL(sys_prlimit64,compat_sys_prlimit64)
-SYSCALL(sys_name_to_handle_at,compat_sys_name_to_handle_at) /* 335 */
-SYSCALL(sys_open_by_handle_at,compat_sys_open_by_handle_at)
-SYSCALL(sys_clock_adjtime,compat_sys_clock_adjtime)
-SYSCALL(sys_syncfs,sys_syncfs)
-SYSCALL(sys_setns,sys_setns)
-SYSCALL(sys_process_vm_readv,compat_sys_process_vm_readv) /* 340 */
-SYSCALL(sys_process_vm_writev,compat_sys_process_vm_writev)
-SYSCALL(sys_s390_runtime_instr,sys_s390_runtime_instr)
-SYSCALL(sys_kcmp,compat_sys_kcmp)
-SYSCALL(sys_finit_module,compat_sys_finit_module)
-SYSCALL(sys_sched_setattr,compat_sys_sched_setattr)	/* 345 */
-SYSCALL(sys_sched_getattr,compat_sys_sched_getattr)
-SYSCALL(sys_renameat2,compat_sys_renameat2)
-SYSCALL(sys_seccomp,compat_sys_seccomp)
-SYSCALL(sys_getrandom,compat_sys_getrandom)
-SYSCALL(sys_memfd_create,compat_sys_memfd_create)	/* 350 */
-SYSCALL(sys_bpf,compat_sys_bpf)
-SYSCALL(sys_s390_pci_mmio_write,compat_sys_s390_pci_mmio_write)
-SYSCALL(sys_s390_pci_mmio_read,compat_sys_s390_pci_mmio_read)
-SYSCALL(sys_execveat,compat_sys_execveat)
-SYSCALL(sys_userfaultfd,sys_userfaultfd)		/* 355 */
-SYSCALL(sys_membarrier,sys_membarrier)
-SYSCALL(sys_recvmmsg,compat_sys_recvmmsg)
-SYSCALL(sys_sendmmsg,compat_sys_sendmmsg)
-SYSCALL(sys_socket,sys_socket)
-SYSCALL(sys_socketpair,compat_sys_socketpair)		/* 360 */
-SYSCALL(sys_bind,compat_sys_bind)
-SYSCALL(sys_connect,compat_sys_connect)
-SYSCALL(sys_listen,sys_listen)
-SYSCALL(sys_accept4,compat_sys_accept4)
-SYSCALL(sys_getsockopt,compat_sys_getsockopt)		/* 365 */
-SYSCALL(sys_setsockopt,compat_sys_setsockopt)
-SYSCALL(sys_getsockname,compat_sys_getsockname)
-SYSCALL(sys_getpeername,compat_sys_getpeername)
-SYSCALL(sys_sendto,compat_sys_sendto)
-SYSCALL(sys_sendmsg,compat_sys_sendmsg)			/* 370 */
-SYSCALL(sys_recvfrom,compat_sys_recvfrom)
-SYSCALL(sys_recvmsg,compat_sys_recvmsg)
-SYSCALL(sys_shutdown,sys_shutdown)
-SYSCALL(sys_mlock2,compat_sys_mlock2)
-SYSCALL(sys_copy_file_range,compat_sys_copy_file_range) /* 375 */
-SYSCALL(sys_preadv2,compat_sys_preadv2)
-SYSCALL(sys_pwritev2,compat_sys_pwritev2)
-SYSCALL(sys_s390_guarded_storage,compat_sys_s390_guarded_storage) /* 378 */
-SYSCALL(sys_statx,compat_sys_statx)
-SYSCALL(sys_s390_sthyi,compat_sys_s390_sthyi)
diff --git a/arch/s390/kernel/syscalls/Makefile b/arch/s390/kernel/syscalls/Makefile
new file mode 100644
index 000000000000..8ff96c08955f
--- /dev/null
+++ b/arch/s390/kernel/syscalls/Makefile
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-2.0
+
+gen	:= arch/$(ARCH)/include/generated
+kapi	:= $(gen)/asm
+uapi	:= $(gen)/uapi/asm
+
+syscall	:= $(srctree)/$(src)/syscall.tbl
+systbl	:= $(srctree)/$(src)/syscalltbl
+
+gen-y := $(kapi)/syscall_table.h
+kapi-hdrs-y := $(kapi)/unistd_nr.h
+uapi-hdrs-y := $(uapi)/unistd_32.h
+uapi-hdrs-y += $(uapi)/unistd_64.h
+
+targets += $(addprefix ../../../,$(gen-y) $(kapi-hdrs-y) $(uapi-hdrs-y))
+
+PHONY += kapi uapi
+
+kapi:	$(gen-y) $(kapi-hdrs-y)
+uapi:	$(uapi-hdrs-y)
+
+
+# Create output directory if not already present
+_dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \
+	  $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)')
+
+define filechk_syshdr
+	$(CONFIG_SHELL) '$(systbl)' -H -a $(syshdr_abi_$(basetarget)) -f "$2"
+endef
+
+define filechk_sysnr
+	$(CONFIG_SHELL) '$(systbl)' -N -a $(sysnr_abi_$(basetarget))
+endef
+
+define filechk_syscalls
+	$(CONFIG_SHELL) '$(systbl)' -S
+endef
+
+syshdr_abi_unistd_32 := common,32
+$(uapi)/unistd_32.h: $(syscall) FORCE
+	$(call filechk,syshdr,$@)
+
+syshdr_abi_unistd_64 := common,64
+$(uapi)/unistd_64.h: $(syscall) FORCE
+	$(call filechk,syshdr,$@)
+
+$(kapi)/syscall_table.h: $(syscall) FORCE
+	$(call filechk,syscalls)
+
+sysnr_abi_unistd_nr := common,32,64
+$(kapi)/unistd_nr.h: $(syscall) FORCE
+	$(call filechk,sysnr)
diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl
new file mode 100644
index 000000000000..b38d48464368
--- /dev/null
+++ b/arch/s390/kernel/syscalls/syscall.tbl
@@ -0,0 +1,390 @@
+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+#
+# System call table for s390
+#
+# Format:
+#
+# <nr> <abi> <syscall> <entry-64bit> <compat-entry>
+#
+# where <abi> can be common, 64, or 32
+
+1    common	exit			sys_exit			sys_exit
+2    common	fork			sys_fork			sys_fork
+3    common	read			sys_read			compat_sys_s390_read
+4    common	write			sys_write			compat_sys_s390_write
+5    common	open			sys_open			compat_sys_open
+6    common	close			sys_close			sys_close
+7    common	restart_syscall		sys_restart_syscall		sys_restart_syscall
+8    common	creat			sys_creat			compat_sys_creat
+9    common	link			sys_link			compat_sys_link
+10   common	unlink			sys_unlink			compat_sys_unlink
+11   common	execve			sys_execve			compat_sys_execve
+12   common	chdir			sys_chdir			compat_sys_chdir
+13   32		time			-				compat_sys_time
+14   common	mknod			sys_mknod			compat_sys_mknod
+15   common	chmod			sys_chmod			compat_sys_chmod
+16   32		lchown			-				compat_sys_s390_lchown16
+19   common	lseek			sys_lseek			compat_sys_lseek
+20   common	getpid			sys_getpid			sys_getpid
+21   common	mount			sys_mount			compat_sys_mount
+22   common	umount			sys_oldumount			compat_sys_oldumount
+23   32		setuid			-				compat_sys_s390_setuid16
+24   32		getuid			-				compat_sys_s390_getuid16
+25   32		stime			-				compat_sys_stime
+26   common	ptrace			sys_ptrace			compat_sys_ptrace
+27   common	alarm			sys_alarm			sys_alarm
+29   common	pause			sys_pause			sys_pause
+30   common	utime			sys_utime			compat_sys_utime
+33   common	access			sys_access			compat_sys_access
+34   common	nice			sys_nice			sys_nice
+36   common	sync			sys_sync			sys_sync
+37   common	kill			sys_kill			sys_kill
+38   common	rename			sys_rename			compat_sys_rename
+39   common	mkdir			sys_mkdir			compat_sys_mkdir
+40   common	rmdir			sys_rmdir			compat_sys_rmdir
+41   common	dup			sys_dup				sys_dup
+42   common	pipe			sys_pipe			compat_sys_pipe
+43   common	times			sys_times			compat_sys_times
+45   common	brk			sys_brk				compat_sys_brk
+46   32		setgid			-				compat_sys_s390_setgid16
+47   32		getgid			-				compat_sys_s390_getgid16
+48   common	signal			sys_signal			compat_sys_signal
+49   32		geteuid			-				compat_sys_s390_geteuid16
+50   32		getegid			-				compat_sys_s390_getegid16
+51   common	acct			sys_acct			compat_sys_acct
+52   common	umount2			sys_umount			compat_sys_umount
+54   common	ioctl			sys_ioctl			compat_sys_ioctl
+55   common	fcntl			sys_fcntl			compat_sys_fcntl
+57   common	setpgid			sys_setpgid			sys_setpgid
+60   common	umask			sys_umask			sys_umask
+61   common	chroot			sys_chroot			compat_sys_chroot
+62   common	ustat			sys_ustat			compat_sys_ustat
+63   common	dup2			sys_dup2			sys_dup2
+64   common	getppid			sys_getppid			sys_getppid
+65   common	getpgrp			sys_getpgrp			sys_getpgrp
+66   common	setsid			sys_setsid			sys_setsid
+67   common	sigaction		sys_sigaction			compat_sys_sigaction
+70   32		setreuid		-				compat_sys_s390_setreuid16
+71   32		setregid		-				compat_sys_s390_setregid16
+72   common	sigsuspend		sys_sigsuspend			compat_sys_sigsuspend
+73   common	sigpending		sys_sigpending			compat_sys_sigpending
+74   common	sethostname		sys_sethostname			compat_sys_sethostname
+75   common	setrlimit		sys_setrlimit			compat_sys_setrlimit
+76   32		getrlimit		-				compat_sys_old_getrlimit
+77   common	getrusage		sys_getrusage			compat_sys_getrusage
+78   common	gettimeofday		sys_gettimeofday		compat_sys_gettimeofday
+79   common	settimeofday		sys_settimeofday		compat_sys_settimeofday
+80   32		getgroups		-				compat_sys_s390_getgroups16
+81   32		setgroups		-				compat_sys_s390_setgroups16
+83   common	symlink			sys_symlink			compat_sys_symlink
+85   common	readlink		sys_readlink			compat_sys_readlink
+86   common	uselib			sys_uselib			compat_sys_uselib
+87   common	swapon			sys_swapon			compat_sys_swapon
+88   common	reboot			sys_reboot			compat_sys_reboot
+89   common	readdir			-				compat_sys_old_readdir
+90   common	mmap			sys_old_mmap			compat_sys_s390_old_mmap
+91   common	munmap			sys_munmap			compat_sys_munmap
+92   common	truncate		sys_truncate			compat_sys_truncate
+93   common	ftruncate		sys_ftruncate			compat_sys_ftruncate
+94   common	fchmod			sys_fchmod			sys_fchmod
+95   32		fchown			-				compat_sys_s390_fchown16
+96   common	getpriority		sys_getpriority			sys_getpriority
+97   common	setpriority		sys_setpriority			sys_setpriority
+99   common	statfs			sys_statfs			compat_sys_statfs
+100  common	fstatfs			sys_fstatfs			compat_sys_fstatfs
+101  32		ioperm			-				-
+102  common	socketcall		sys_socketcall			compat_sys_socketcall
+103  common	syslog			sys_syslog			compat_sys_syslog
+104  common	setitimer		sys_setitimer			compat_sys_setitimer
+105  common	getitimer		sys_getitimer			compat_sys_getitimer
+106  common	stat			sys_newstat			compat_sys_newstat
+107  common	lstat			sys_newlstat			compat_sys_newlstat
+108  common	fstat			sys_newfstat			compat_sys_newfstat
+110  common	lookup_dcookie		sys_lookup_dcookie		compat_sys_lookup_dcookie
+111  common	vhangup			sys_vhangup			sys_vhangup
+112  common	idle			-				-
+114  common	wait4			sys_wait4			compat_sys_wait4
+115  common	swapoff			sys_swapoff			compat_sys_swapoff
+116  common	sysinfo			sys_sysinfo			compat_sys_sysinfo
+117  common	ipc			sys_s390_ipc			compat_sys_s390_ipc
+118  common	fsync			sys_fsync			sys_fsync
+119  common	sigreturn		sys_sigreturn			compat_sys_sigreturn
+120  common	clone			sys_clone			compat_sys_clone
+121  common	setdomainname		sys_setdomainname		compat_sys_setdomainname
+122  common	uname			sys_newuname			compat_sys_newuname
+124  common	adjtimex		sys_adjtimex			compat_sys_adjtimex
+125  common	mprotect		sys_mprotect			compat_sys_mprotect
+126  common	sigprocmask		sys_sigprocmask			compat_sys_sigprocmask
+127  common	create_module		-				-
+128  common	init_module		sys_init_module			compat_sys_init_module
+129  common	delete_module		sys_delete_module		compat_sys_delete_module
+130  common	get_kernel_syms		-				-
+131  common	quotactl		sys_quotactl			compat_sys_quotactl
+132  common	getpgid			sys_getpgid			sys_getpgid
+133  common	fchdir			sys_fchdir			sys_fchdir
+134  common	bdflush			sys_bdflush			compat_sys_bdflush
+135  common	sysfs			sys_sysfs			compat_sys_sysfs
+136  common	personality		sys_s390_personality		sys_s390_personality
+137  common	afs_syscall		-				-
+138  32		setfsuid		-				compat_sys_s390_setfsuid16
+139  32		setfsgid		-				compat_sys_s390_setfsgid16
+140  32		_llseek			-				compat_sys_llseek
+141  common	getdents		sys_getdents			compat_sys_getdents
+142  32		_newselect		-				compat_sys_select
+142  64		select			sys_select			-
+143  common	flock			sys_flock			sys_flock
+144  common	msync			sys_msync			compat_sys_msync
+145  common	readv			sys_readv			compat_sys_readv
+146  common	writev			sys_writev			compat_sys_writev
+147  common	getsid			sys_getsid			sys_getsid
+148  common	fdatasync		sys_fdatasync			sys_fdatasync
+149  common	_sysctl			sys_sysctl			compat_sys_sysctl
+150  common	mlock			sys_mlock			compat_sys_mlock
+151  common	munlock			sys_munlock			compat_sys_munlock
+152  common	mlockall		sys_mlockall			sys_mlockall
+153  common	munlockall		sys_munlockall			sys_munlockall
+154  common	sched_setparam		sys_sched_setparam		compat_sys_sched_setparam
+155  common	sched_getparam		sys_sched_getparam		compat_sys_sched_getparam
+156  common	sched_setscheduler	sys_sched_setscheduler		compat_sys_sched_setscheduler
+157  common	sched_getscheduler	sys_sched_getscheduler		sys_sched_getscheduler
+158  common	sched_yield		sys_sched_yield			sys_sched_yield
+159  common	sched_get_priority_max	sys_sched_get_priority_max	sys_sched_get_priority_max
+160  common	sched_get_priority_min	sys_sched_get_priority_min	sys_sched_get_priority_min
+161  common	sched_rr_get_interval	sys_sched_rr_get_interval	compat_sys_sched_rr_get_interval
+162  common	nanosleep		sys_nanosleep			compat_sys_nanosleep
+163  common	mremap			sys_mremap			compat_sys_mremap
+164  32		setresuid		-				compat_sys_s390_setresuid16
+165  32		getresuid		-				compat_sys_s390_getresuid16
+167  common	query_module		-				-
+168  common	poll			sys_poll			compat_sys_poll
+169  common	nfsservctl		-				-
+170  32		setresgid		-				compat_sys_s390_setresgid16
+171  32		getresgid		-				compat_sys_s390_getresgid16
+172  common	prctl			sys_prctl			compat_sys_prctl
+173  common	rt_sigreturn		sys_rt_sigreturn		compat_sys_rt_sigreturn
+174  common	rt_sigaction		sys_rt_sigaction		compat_sys_rt_sigaction
+175  common	rt_sigprocmask		sys_rt_sigprocmask		compat_sys_rt_sigprocmask
+176  common	rt_sigpending		sys_rt_sigpending		compat_sys_rt_sigpending
+177  common	rt_sigtimedwait		sys_rt_sigtimedwait		compat_sys_rt_sigtimedwait
+178  common	rt_sigqueueinfo		sys_rt_sigqueueinfo		compat_sys_rt_sigqueueinfo
+179  common	rt_sigsuspend		sys_rt_sigsuspend		compat_sys_rt_sigsuspend
+180  common	pread64			sys_pread64			compat_sys_s390_pread64
+181  common	pwrite64		sys_pwrite64			compat_sys_s390_pwrite64
+182  32		chown			-				compat_sys_s390_chown16
+183  common	getcwd			sys_getcwd			compat_sys_getcwd
+184  common	capget			sys_capget			compat_sys_capget
+185  common	capset			sys_capset			compat_sys_capset
+186  common	sigaltstack		sys_sigaltstack			compat_sys_sigaltstack
+187  common	sendfile		sys_sendfile64			compat_sys_sendfile
+188  common	getpmsg			-				-
+189  common	putpmsg			-				-
+190  common	vfork			sys_vfork			sys_vfork
+191  32		ugetrlimit		-				compat_sys_getrlimit
+191  64		getrlimit		sys_getrlimit			-
+192  32		mmap2			-				compat_sys_s390_mmap2
+193  32		truncate64		-				compat_sys_s390_truncate64
+194  32		ftruncate64		-				compat_sys_s390_ftruncate64
+195  32		stat64			-				compat_sys_s390_stat64
+196  32		lstat64			-				compat_sys_s390_lstat64
+197  32		fstat64			-				compat_sys_s390_fstat64
+198  32		lchown32		-				compat_sys_lchown
+198  64		lchown			sys_lchown			-
+199  32		getuid32		-				sys_getuid
+199  64		getuid			sys_getuid			-
+200  32		getgid32		-				sys_getgid
+200  64		getgid			sys_getgid			-
+201  32		geteuid32		-				sys_geteuid
+201  64		geteuid			sys_geteuid			-
+202  32		getegid32		-				sys_getegid
+202  64		getegid			sys_getegid			-
+203  32		setreuid32		-				sys_setreuid
+203  64		setreuid		sys_setreuid			-
+204  32		setregid32		-				sys_setregid
+204  64		setregid		sys_setregid			-
+205  32		getgroups32		-				compat_sys_getgroups
+205  64		getgroups		sys_getgroups			-
+206  32		setgroups32		-				compat_sys_setgroups
+206  64		setgroups		sys_setgroups			-
+207  32		fchown32		-				sys_fchown
+207  64		fchown			sys_fchown			-
+208  32		setresuid32		-				sys_setresuid
+208  64		setresuid		sys_setresuid			-
+209  32		getresuid32		-				compat_sys_getresuid
+209  64		getresuid		sys_getresuid			-
+210  32		setresgid32		-				sys_setresgid
+210  64		setresgid		sys_setresgid			-
+211  32		getresgid32		-				compat_sys_getresgid
+211  64		getresgid		sys_getresgid			-
+212  32		chown32			-				compat_sys_chown
+212  64		chown			sys_chown			-
+213  32		setuid32		-				sys_setuid
+213  64		setuid			sys_setuid			-
+214  32		setgid32		-				sys_setgid
+214  64		setgid			sys_setgid			-
+215  32		setfsuid32		-				sys_setfsuid
+215  64		setfsuid		sys_setfsuid			-
+216  32		setfsgid32		-				sys_setfsgid
+216  64		setfsgid		sys_setfsgid			-
+217  common	pivot_root		sys_pivot_root			compat_sys_pivot_root
+218  common	mincore			sys_mincore			compat_sys_mincore
+219  common	madvise			sys_madvise			compat_sys_madvise
+220  common	getdents64		sys_getdents64			compat_sys_getdents64
+221  32		fcntl64			-				compat_sys_fcntl64
+222  common	readahead		sys_readahead			compat_sys_s390_readahead
+223  32		sendfile64		-				compat_sys_sendfile64
+224  common	setxattr		sys_setxattr			compat_sys_setxattr
+225  common	lsetxattr		sys_lsetxattr			compat_sys_lsetxattr
+226  common	fsetxattr		sys_fsetxattr			compat_sys_fsetxattr
+227  common	getxattr		sys_getxattr			compat_sys_getxattr
+228  common	lgetxattr		sys_lgetxattr			compat_sys_lgetxattr
+229  common	fgetxattr		sys_fgetxattr			compat_sys_fgetxattr
+230  common	listxattr		sys_listxattr			compat_sys_listxattr
+231  common	llistxattr		sys_llistxattr			compat_sys_llistxattr
+232  common	flistxattr		sys_flistxattr			compat_sys_flistxattr
+233  common	removexattr		sys_removexattr			compat_sys_removexattr
+234  common	lremovexattr		sys_lremovexattr		compat_sys_lremovexattr
+235  common	fremovexattr		sys_fremovexattr		compat_sys_fremovexattr
+236  common	gettid			sys_gettid			sys_gettid
+237  common	tkill			sys_tkill			sys_tkill
+238  common	futex			sys_futex			compat_sys_futex
+239  common	sched_setaffinity	sys_sched_setaffinity		compat_sys_sched_setaffinity
+240  common	sched_getaffinity	sys_sched_getaffinity		compat_sys_sched_getaffinity
+241  common	tgkill			sys_tgkill			sys_tgkill
+243  common	io_setup		sys_io_setup			compat_sys_io_setup
+244  common	io_destroy		sys_io_destroy			compat_sys_io_destroy
+245  common	io_getevents		sys_io_getevents		compat_sys_io_getevents
+246  common	io_submit		sys_io_submit			compat_sys_io_submit
+247  common	io_cancel		sys_io_cancel			compat_sys_io_cancel
+248  common	exit_group		sys_exit_group			sys_exit_group
+249  common	epoll_create		sys_epoll_create		sys_epoll_create
+250  common	epoll_ctl		sys_epoll_ctl			compat_sys_epoll_ctl
+251  common	epoll_wait		sys_epoll_wait			compat_sys_epoll_wait
+252  common	set_tid_address		sys_set_tid_address		compat_sys_set_tid_address
+253  common	fadvise64		sys_fadvise64_64		compat_sys_s390_fadvise64
+254  common	timer_create		sys_timer_create		compat_sys_timer_create
+255  common	timer_settime		sys_timer_settime		compat_sys_timer_settime
+256  common	timer_gettime		sys_timer_gettime		compat_sys_timer_gettime
+257  common	timer_getoverrun	sys_timer_getoverrun		sys_timer_getoverrun
+258  common	timer_delete		sys_timer_delete		sys_timer_delete
+259  common	clock_settime		sys_clock_settime		compat_sys_clock_settime
+260  common	clock_gettime		sys_clock_gettime		compat_sys_clock_gettime
+261  common	clock_getres		sys_clock_getres		compat_sys_clock_getres
+262  common	clock_nanosleep		sys_clock_nanosleep		compat_sys_clock_nanosleep
+264  32		fadvise64_64		-				compat_sys_s390_fadvise64_64
+265  common	statfs64		sys_statfs64			compat_sys_statfs64
+266  common	fstatfs64		sys_fstatfs64			compat_sys_fstatfs64
+267  common	remap_file_pages	sys_remap_file_pages		compat_sys_remap_file_pages
+268  common	mbind			sys_mbind			compat_sys_mbind
+269  common	get_mempolicy		sys_get_mempolicy		compat_sys_get_mempolicy
+270  common	set_mempolicy		sys_set_mempolicy		compat_sys_set_mempolicy
+271  common	mq_open			sys_mq_open			compat_sys_mq_open
+272  common	mq_unlink		sys_mq_unlink			compat_sys_mq_unlink
+273  common	mq_timedsend		sys_mq_timedsend		compat_sys_mq_timedsend
+274  common	mq_timedreceive		sys_mq_timedreceive		compat_sys_mq_timedreceive
+275  common	mq_notify		sys_mq_notify			compat_sys_mq_notify
+276  common	mq_getsetattr		sys_mq_getsetattr		compat_sys_mq_getsetattr
+277  common	kexec_load		sys_kexec_load			compat_sys_kexec_load
+278  common	add_key			sys_add_key			compat_sys_add_key
+279  common	request_key		sys_request_key			compat_sys_request_key
+280  common	keyctl			sys_keyctl			compat_sys_keyctl
+281  common	waitid			sys_waitid			compat_sys_waitid
+282  common	ioprio_set		sys_ioprio_set			sys_ioprio_set
+283  common	ioprio_get		sys_ioprio_get			sys_ioprio_get
+284  common	inotify_init		sys_inotify_init		sys_inotify_init
+285  common	inotify_add_watch	sys_inotify_add_watch		compat_sys_inotify_add_watch
+286  common	inotify_rm_watch	sys_inotify_rm_watch		sys_inotify_rm_watch
+287  common	migrate_pages		sys_migrate_pages		compat_sys_migrate_pages
+288  common	openat			sys_openat			compat_sys_openat
+289  common	mkdirat			sys_mkdirat			compat_sys_mkdirat
+290  common	mknodat			sys_mknodat			compat_sys_mknodat
+291  common	fchownat		sys_fchownat			compat_sys_fchownat
+292  common	futimesat		sys_futimesat			compat_sys_futimesat
+293  32		fstatat64		-				compat_sys_s390_fstatat64
+293  64		newfstatat		sys_newfstatat			-
+294  common	unlinkat		sys_unlinkat			compat_sys_unlinkat
+295  common	renameat		sys_renameat			compat_sys_renameat
+296  common	linkat			sys_linkat			compat_sys_linkat
+297  common	symlinkat		sys_symlinkat			compat_sys_symlinkat
+298  common	readlinkat		sys_readlinkat			compat_sys_readlinkat
+299  common	fchmodat		sys_fchmodat			compat_sys_fchmodat
+300  common	faccessat		sys_faccessat			compat_sys_faccessat
+301  common	pselect6		sys_pselect6			compat_sys_pselect6
+302  common	ppoll			sys_ppoll			compat_sys_ppoll
+303  common	unshare			sys_unshare			compat_sys_unshare
+304  common	set_robust_list		sys_set_robust_list		compat_sys_set_robust_list
+305  common	get_robust_list		sys_get_robust_list		compat_sys_get_robust_list
+306  common	splice			sys_splice			compat_sys_splice
+307  common	sync_file_range		sys_sync_file_range		compat_sys_s390_sync_file_range
+308  common	tee			sys_tee				compat_sys_tee
+309  common	vmsplice		sys_vmsplice			compat_sys_vmsplice
+310  common	move_pages		sys_move_pages			compat_sys_move_pages
+311  common	getcpu			sys_getcpu			compat_sys_getcpu
+312  common	epoll_pwait		sys_epoll_pwait			compat_sys_epoll_pwait
+313  common	utimes			sys_utimes			compat_sys_utimes
+314  common	fallocate		sys_fallocate			compat_sys_s390_fallocate
+315  common	utimensat		sys_utimensat			compat_sys_utimensat
+316  common	signalfd		sys_signalfd			compat_sys_signalfd
+317  common	timerfd			-				-
+318  common	eventfd			sys_eventfd			sys_eventfd
+319  common	timerfd_create		sys_timerfd_create		sys_timerfd_create
+320  common	timerfd_settime		sys_timerfd_settime		compat_sys_timerfd_settime
+321  common	timerfd_gettime		sys_timerfd_gettime		compat_sys_timerfd_gettime
+322  common	signalfd4		sys_signalfd4			compat_sys_signalfd4
+323  common	eventfd2		sys_eventfd2			sys_eventfd2
+324  common	inotify_init1		sys_inotify_init1		sys_inotify_init1
+325  common	pipe2			sys_pipe2			compat_sys_pipe2
+326  common	dup3			sys_dup3			sys_dup3
+327  common	epoll_create1		sys_epoll_create1		sys_epoll_create1
+328  common	preadv			sys_preadv			compat_sys_preadv
+329  common	pwritev			sys_pwritev			compat_sys_pwritev
+330  common	rt_tgsigqueueinfo	sys_rt_tgsigqueueinfo		compat_sys_rt_tgsigqueueinfo
+331  common	perf_event_open		sys_perf_event_open		compat_sys_perf_event_open
+332  common	fanotify_init		sys_fanotify_init		sys_fanotify_init
+333  common	fanotify_mark		sys_fanotify_mark		compat_sys_fanotify_mark
+334  common	prlimit64		sys_prlimit64			compat_sys_prlimit64
+335  common	name_to_handle_at	sys_name_to_handle_at		compat_sys_name_to_handle_at
+336  common	open_by_handle_at	sys_open_by_handle_at		compat_sys_open_by_handle_at
+337  common	clock_adjtime		sys_clock_adjtime		compat_sys_clock_adjtime
+338  common	syncfs			sys_syncfs			sys_syncfs
+339  common	setns			sys_setns			sys_setns
+340  common	process_vm_readv	sys_process_vm_readv		compat_sys_process_vm_readv
+341  common	process_vm_writev	sys_process_vm_writev		compat_sys_process_vm_writev
+342  common	s390_runtime_instr	sys_s390_runtime_instr		sys_s390_runtime_instr
+343  common	kcmp			sys_kcmp			compat_sys_kcmp
+344  common	finit_module		sys_finit_module		compat_sys_finit_module
+345  common	sched_setattr		sys_sched_setattr		compat_sys_sched_setattr
+346  common	sched_getattr		sys_sched_getattr		compat_sys_sched_getattr
+347  common	renameat2		sys_renameat2			compat_sys_renameat2
+348  common	seccomp			sys_seccomp			compat_sys_seccomp
+349  common	getrandom		sys_getrandom			compat_sys_getrandom
+350  common	memfd_create		sys_memfd_create		compat_sys_memfd_create
+351  common	bpf			sys_bpf				compat_sys_bpf
+352  common	s390_pci_mmio_write	sys_s390_pci_mmio_write		compat_sys_s390_pci_mmio_write
+353  common	s390_pci_mmio_read	sys_s390_pci_mmio_read		compat_sys_s390_pci_mmio_read
+354  common	execveat		sys_execveat			compat_sys_execveat
+355  common	userfaultfd		sys_userfaultfd			sys_userfaultfd
+356  common	membarrier		sys_membarrier			sys_membarrier
+357  common	recvmmsg		sys_recvmmsg			compat_sys_recvmmsg
+358  common	sendmmsg		sys_sendmmsg			compat_sys_sendmmsg
+359  common	socket			sys_socket			sys_socket
+360  common	socketpair		sys_socketpair			compat_sys_socketpair
+361  common	bind			sys_bind			compat_sys_bind
+362  common	connect			sys_connect			compat_sys_connect
+363  common	listen			sys_listen			sys_listen
+364  common	accept4			sys_accept4			compat_sys_accept4
+365  common	getsockopt		sys_getsockopt			compat_sys_getsockopt
+366  common	setsockopt		sys_setsockopt			compat_sys_setsockopt
+367  common	getsockname		sys_getsockname			compat_sys_getsockname
+368  common	getpeername		sys_getpeername			compat_sys_getpeername
+369  common	sendto			sys_sendto			compat_sys_sendto
+370  common	sendmsg			sys_sendmsg			compat_sys_sendmsg
+371  common	recvfrom		sys_recvfrom			compat_sys_recvfrom
+372  common	recvmsg			sys_recvmsg			compat_sys_recvmsg
+373  common	shutdown		sys_shutdown			sys_shutdown
+374  common	mlock2			sys_mlock2			compat_sys_mlock2
+375  common	copy_file_range		sys_copy_file_range		compat_sys_copy_file_range
+376  common	preadv2			sys_preadv2			compat_sys_preadv2
+377  common	pwritev2		sys_pwritev2			compat_sys_pwritev2
+378  common	s390_guarded_storage	sys_s390_guarded_storage	compat_sys_s390_guarded_storage
+379  common	statx			sys_statx			compat_sys_statx
+380  common	s390_sthyi		sys_s390_sthyi			compat_sys_s390_sthyi
diff --git a/arch/s390/kernel/syscalls/syscalltbl b/arch/s390/kernel/syscalls/syscalltbl
new file mode 100755
index 000000000000..fbac1732f874
--- /dev/null
+++ b/arch/s390/kernel/syscalls/syscalltbl
@@ -0,0 +1,232 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+#
+# Generate system call table and header files
+#
+# Copyright IBM Corp. 2018
+# Author(s):  Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
+
+#
+# File path to the system call table definition.
+# You can set the path with the -i option.  If omitted,
+# system call table definitions are read from standard input.
+#
+SYSCALL_TBL=""
+
+
+create_syscall_table_entries()
+{
+	local nr abi name entry64 entry32 _ignore
+	local temp=$(mktemp ${TMPDIR:-/tmp}/syscalltbl-common.XXXXXXXXX)
+
+	(
+	#
+	# Initialize with 0 to create an NI_SYSCALL for 0
+	#
+	local prev_nr=0 prev_32=sys_ni_syscall prev_64=sys_ni_syscall
+	while read nr abi name entry64 entry32 _ignore; do
+		test x$entry32 = x- && entry32=sys_ni_syscall
+		test x$entry64 = x- && entry64=sys_ni_syscall
+
+		if test $prev_nr -eq $nr; then
+			#
+			# Same syscall but different ABI, just update
+			# the respective entry point
+			#
+			case $abi in
+			32)
+				prev_32=$entry32
+			;;
+			64)
+				prev_64=$entry64
+			;;
+			esac
+			continue;
+		else
+			printf "%d\t%s\t%s\n" $prev_nr $prev_64 $prev_32
+		fi
+
+		prev_nr=$nr
+		prev_64=$entry64
+		prev_32=$entry32
+	done
+	printf "%d\t%s\t%s\n" $prev_nr $prev_64 $prev_32
+	) >> $temp
+
+	#
+	# Check for duplicate syscall numbers
+	#
+	if ! cat $temp |cut -f1 |uniq -d 2>&1; then
+		echo "Error: generated system call table contains duplicate entries: $temp" >&2
+		exit 1
+	fi
+
+	#
+	# Generate syscall table
+	#
+	prev_nr=0
+	while read nr entry64 entry32; do
+		while test $prev_nr -lt $((nr - 1)); do
+			printf "NI_SYSCALL\n"
+			prev_nr=$((prev_nr + 1))
+		done
+		if test x$entry64 = xsys_ni_syscall &&
+		   test x$entry32 = xsys_ni_syscall; then
+			printf "NI_SYSCALL\n"
+		else
+			printf "SYSCALL(%s,%s)\n" $entry64 $entry32
+		fi
+		prev_nr=$nr
+	done < $temp
+	rm $temp
+}
+
+generate_syscall_table()
+{
+	cat <<-EoHEADER
+	/* SPDX-License-Identifier: GPL-2.0 */
+	/*
+	 * Definitions for sys_call_table, each line represents an
+	 * entry in the table in the form
+	 * SYSCALL(64 bit syscall, 31 bit emulated syscall)
+	 *
+	 * This file is meant to be included from entry.S.
+	 */
+
+	#define NI_SYSCALL SYSCALL(sys_ni_syscall,sys_ni_syscall)
+
+EoHEADER
+	grep -Ev '^(#|[[:blank:]]*$)' $SYSCALL_TBL	\
+		|sort -k1 -n				\
+		|create_syscall_table_entries
+}
+
+create_header_defines()
+{
+	local nr abi name _ignore
+
+	while read nr abi name _ignore; do
+		printf "#define __NR_%s %d\n" $name $nr
+	done
+}
+
+normalize_fileguard()
+{
+	local fileguard="$1"
+
+	echo "$1" |tr '[[:lower:]]' '[[:upper:]]' \
+		  |sed -e 's/[^A-Z0-9_]/_/g' -e 's/__/_/g'
+}
+
+generate_syscall_header()
+{
+	local abis=$(echo "($1)" | tr ',' '|')
+	local filename="$2"
+	local fileguard suffix
+
+	if test "$filename"; then
+		fileguard=$(normalize_fileguard "__UAPI_ASM_S390_$2")
+	else
+		case "$abis" in
+		*64*) suffix=64 ;;
+		*32*) suffix=32 ;;
+		esac
+		fileguard=$(normalize_fileguard "__UAPI_ASM_S390_SYSCALLS_$suffix")
+	fi
+
+	cat <<-EoHEADER
+	/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+	#ifndef ${fileguard}
+	#define ${fileguard}
+
+EoHEADER
+
+	grep -E "^[[:digit:]]+[[:space:]]+${abis}" $SYSCALL_TBL	\
+		|sort -k1 -n					\
+		|create_header_defines
+
+	cat <<-EoFOOTER
+
+	#endif /* ${fileguard} */
+EoFOOTER
+}
+
+__max_syscall_nr()
+{
+	local abis=$(echo "($1)" | tr ',' '|')
+
+	grep -E "^[[:digit:]]+[[:space:]]+${abis}" $SYSCALL_TBL	 \
+		|sed -ne 's/^\([[:digit:]]*\)[[:space:]].*/\1/p' \
+		|sort -n					 \
+		|tail -1
+}
+
+
+generate_syscall_nr()
+{
+	local abis="$1"
+	local max_syscall_nr num_syscalls
+
+	max_syscall_nr=$(__max_syscall_nr "$abis")
+	num_syscalls=$((max_syscall_nr + 1))
+
+	cat <<-EoHEADER
+	/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+	#ifndef __ASM_S390_SYSCALLS_NR
+	#define __ASM_S390_SYSCALLS_NR
+
+	#define NR_syscalls ${num_syscalls}
+
+	#endif /* __ASM_S390_SYSCALLS_NR */
+EoHEADER
+}
+
+
+#
+# Parse command line arguments
+#
+do_syscall_header=""
+do_syscall_table=""
+do_syscall_nr=""
+output_file=""
+abi_list="common,64"
+filename=""
+while getopts ":HNSXi:a:f:" arg; do
+	case $arg in
+	a)
+		abi_list="$OPTARG"
+		;;
+	i)
+		SYSCALL_TBL="$OPTARG"
+		;;
+	f)
+		filename=${OPTARG##*/}
+		;;
+	H)
+		do_syscall_header=1
+		;;
+	N)
+		do_syscall_nr=1
+		;;
+	S)
+		do_syscall_table=1
+		;;
+	X)
+		set -x
+		;;
+	:)
+		echo "Missing argument for -$OPTARG" >&2
+		exit 1
+	;;
+	\?)
+		echo "Invalid option specified" >&2
+		exit 1
+	;;
+	esac
+done
+
+test "$do_syscall_header" && generate_syscall_header "$abi_list" "$filename"
+test "$do_syscall_table" && generate_syscall_table
+test "$do_syscall_nr" && generate_syscall_nr "$abi_list"
+
+exit 0
diff --git a/arch/s390/kernel/sysinfo.c b/arch/s390/kernel/sysinfo.c
index a441cba8d165..fc7e04c2195b 100644
--- a/arch/s390/kernel/sysinfo.c
+++ b/arch/s390/kernel/sysinfo.c
@@ -89,6 +89,8 @@ static void stsi_1_1_1(struct seq_file *m, struct sysinfo_1_1_1 *info)
 	EBCASC(info->model_temp_cap, sizeof(info->model_temp_cap));
 	seq_printf(m, "Manufacturer:         %-16.16s\n", info->manufacturer);
 	seq_printf(m, "Type:                 %-4.4s\n", info->type);
+	if (info->lic)
+		seq_printf(m, "LIC Identifier:       %016lx\n", info->lic);
 	/*
 	 * Sigh: the model field has been renamed with System z9
 	 * to model_capacity and a new model field has been added
diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c
index 4d5b65e527b5..4b6e0397f66d 100644
--- a/arch/s390/kernel/topology.c
+++ b/arch/s390/kernel/topology.c
@@ -404,8 +404,7 @@ static ssize_t dispatching_store(struct device *dev,
 	put_online_cpus();
 	return rc ? rc : count;
 }
-static DEVICE_ATTR(dispatching, 0644, dispatching_show,
-			 dispatching_store);
+static DEVICE_ATTR_RW(dispatching);
 
 static ssize_t cpu_polarization_show(struct device *dev,
 				     struct device_attribute *attr, char *buf)
diff --git a/arch/s390/kernel/vdso32/Makefile b/arch/s390/kernel/vdso32/Makefile
index 308564b9bf68..c5c856f320bc 100644
--- a/arch/s390/kernel/vdso32/Makefile
+++ b/arch/s390/kernel/vdso32/Makefile
@@ -10,6 +10,9 @@ obj-vdso32 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o
 targets := $(obj-vdso32) vdso32.so vdso32.so.dbg
 obj-vdso32 := $(addprefix $(obj)/, $(obj-vdso32))
 
+KBUILD_AFLAGS += -DBUILD_VDSO
+KBUILD_CFLAGS += -DBUILD_VDSO
+
 KBUILD_AFLAGS_31 := $(filter-out -m64,$(KBUILD_AFLAGS))
 KBUILD_AFLAGS_31 += -m31 -s
 
diff --git a/arch/s390/kernel/vdso32/clock_getres.S b/arch/s390/kernel/vdso32/clock_getres.S
index f61df5253c23..eaf9cf1417f6 100644
--- a/arch/s390/kernel/vdso32/clock_getres.S
+++ b/arch/s390/kernel/vdso32/clock_getres.S
@@ -9,13 +9,14 @@
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
 #include <asm/unistd.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_clock_getres
 	.type  __kernel_clock_getres,@function
 __kernel_clock_getres:
-	.cfi_startproc
+	CFI_STARTPROC
 	basr	%r1,0
 	la	%r1,4f-.(%r1)
 	chi	%r2,__CLOCK_REALTIME
@@ -37,7 +38,7 @@ __kernel_clock_getres:
 3:	lhi	%r1,__NR_clock_getres		/* fallback to svc */
 	svc	0
 	br	%r14
+	CFI_ENDPROC
 4:	.long	__CLOCK_REALTIME_RES
 5:	.long	__CLOCK_COARSE_RES
-	.cfi_endproc
 	.size	__kernel_clock_getres,.-__kernel_clock_getres
diff --git a/arch/s390/kernel/vdso32/clock_gettime.S b/arch/s390/kernel/vdso32/clock_gettime.S
index 2d6ec3abe095..a9418bf975db 100644
--- a/arch/s390/kernel/vdso32/clock_gettime.S
+++ b/arch/s390/kernel/vdso32/clock_gettime.S
@@ -9,14 +9,17 @@
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
 #include <asm/unistd.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_clock_gettime
 	.type  __kernel_clock_gettime,@function
 __kernel_clock_gettime:
-	.cfi_startproc
+	CFI_STARTPROC
 	ahi	%r15,-16
+	CFI_DEF_CFA_OFFSET 176
+	CFI_VAL_OFFSET 15, -160
 	basr	%r5,0
 0:	al	%r5,21f-0b(%r5)			/* get &_vdso_data */
 	chi	%r2,__CLOCK_REALTIME_COARSE
@@ -69,9 +72,13 @@ __kernel_clock_gettime:
 	st	%r1,4(%r3)			/* store tp->tv_nsec */
 	lhi	%r2,0
 	ahi	%r15,16
+	CFI_DEF_CFA_OFFSET 160
+	CFI_RESTORE 15
 	br	%r14
 
 	/* CLOCK_MONOTONIC_COARSE */
+	CFI_DEF_CFA_OFFSET 176
+	CFI_VAL_OFFSET 15, -160
 9:	l	%r4,__VDSO_UPD_COUNT+4(%r5)	/* load update counter */
 	tml	%r4,0x0001			/* pending update ? loop */
 	jnz	9b
@@ -151,15 +158,21 @@ __kernel_clock_gettime:
 	st	%r1,4(%r3)			/* store tp->tv_nsec */
 	lhi	%r2,0
 	ahi	%r15,16
+	CFI_DEF_CFA_OFFSET 160
+	CFI_RESTORE 15
 	br	%r14
 
 	/* Fallback to system call */
+	CFI_DEF_CFA_OFFSET 176
+	CFI_VAL_OFFSET 15, -160
 19:	lhi	%r1,__NR_clock_gettime
 	svc	0
 	ahi	%r15,16
+	CFI_DEF_CFA_OFFSET 160
+	CFI_RESTORE 15
 	br	%r14
+	CFI_ENDPROC
 
 20:	.long	1000000000
 21:	.long	_vdso_data - 0b
-	.cfi_endproc
 	.size	__kernel_clock_gettime,.-__kernel_clock_gettime
diff --git a/arch/s390/kernel/vdso32/getcpu.S b/arch/s390/kernel/vdso32/getcpu.S
index 5477a2c112fb..25515f3fbcea 100644
--- a/arch/s390/kernel/vdso32/getcpu.S
+++ b/arch/s390/kernel/vdso32/getcpu.S
@@ -8,13 +8,14 @@
  */
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_getcpu
 	.type  __kernel_getcpu,@function
 __kernel_getcpu:
-	.cfi_startproc
+	CFI_STARTPROC
 	la	%r4,0
 	sacf	256
 	l	%r5,__VDSO_CPU_NR(%r4)
@@ -28,5 +29,5 @@ __kernel_getcpu:
 	st	%r4,0(%r3)
 3:	lhi	%r2,0
 	br	%r14
-	.cfi_endproc
+	CFI_ENDPROC
 	.size	__kernel_getcpu,.-__kernel_getcpu
diff --git a/arch/s390/kernel/vdso32/gettimeofday.S b/arch/s390/kernel/vdso32/gettimeofday.S
index aa8bf13a2edb..3c0db0fa6ad9 100644
--- a/arch/s390/kernel/vdso32/gettimeofday.S
+++ b/arch/s390/kernel/vdso32/gettimeofday.S
@@ -9,14 +9,17 @@
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
 #include <asm/unistd.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_gettimeofday
 	.type  __kernel_gettimeofday,@function
 __kernel_gettimeofday:
-	.cfi_startproc
+	CFI_STARTPROC
 	ahi	%r15,-16
+	CFI_ADJUST_CFA_OFFSET 16
+	CFI_VAL_OFFSET 15, -160
 	basr	%r5,0
 0:	al	%r5,13f-0b(%r5)			/* get &_vdso_data */
 1:	ltr	%r3,%r3				/* check if tz is NULL */
@@ -89,9 +92,11 @@ __kernel_gettimeofday:
 	st	%r0,4(%r2)			/* store tv->tv_usec */
 10:	slr	%r2,%r2
 	ahi	%r15,16
+	CFI_ADJUST_CFA_OFFSET -16
+	CFI_RESTORE 15
 	br	%r14
+	CFI_ENDPROC
 11:	.long	1000000000
 12:	.long	274877907
 13:	.long	_vdso_data - 0b
-	.cfi_endproc
 	.size	__kernel_gettimeofday,.-__kernel_gettimeofday
diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile
index f81ae7998883..15b1ceafc4c1 100644
--- a/arch/s390/kernel/vdso64/Makefile
+++ b/arch/s390/kernel/vdso64/Makefile
@@ -10,6 +10,9 @@ obj-vdso64 = gettimeofday.o clock_getres.o clock_gettime.o note.o getcpu.o
 targets := $(obj-vdso64) vdso64.so vdso64.so.dbg
 obj-vdso64 := $(addprefix $(obj)/, $(obj-vdso64))
 
+KBUILD_AFLAGS += -DBUILD_VDSO
+KBUILD_CFLAGS += -DBUILD_VDSO
+
 KBUILD_AFLAGS_64 := $(filter-out -m64,$(KBUILD_AFLAGS))
 KBUILD_AFLAGS_64 += -m64 -s
 
diff --git a/arch/s390/kernel/vdso64/clock_getres.S b/arch/s390/kernel/vdso64/clock_getres.S
index faf5213b15df..081435398e0a 100644
--- a/arch/s390/kernel/vdso64/clock_getres.S
+++ b/arch/s390/kernel/vdso64/clock_getres.S
@@ -9,13 +9,14 @@
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
 #include <asm/unistd.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_clock_getres
 	.type  __kernel_clock_getres,@function
 __kernel_clock_getres:
-	.cfi_startproc
+	CFI_STARTPROC
 	larl	%r1,4f
 	cghi	%r2,__CLOCK_REALTIME_COARSE
 	je	0f
@@ -43,7 +44,7 @@ __kernel_clock_getres:
 2:	lghi	%r1,__NR_clock_getres		/* fallback to svc */
 	svc	0
 	br	%r14
+	CFI_ENDPROC
 3:	.quad	__CLOCK_REALTIME_RES
 4:	.quad	__CLOCK_COARSE_RES
-	.cfi_endproc
 	.size	__kernel_clock_getres,.-__kernel_clock_getres
diff --git a/arch/s390/kernel/vdso64/clock_gettime.S b/arch/s390/kernel/vdso64/clock_gettime.S
index 6046b3bfca46..fac3ab5ec83a 100644
--- a/arch/s390/kernel/vdso64/clock_gettime.S
+++ b/arch/s390/kernel/vdso64/clock_gettime.S
@@ -9,14 +9,17 @@
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
 #include <asm/unistd.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_clock_gettime
 	.type  __kernel_clock_gettime,@function
 __kernel_clock_gettime:
-	.cfi_startproc
+	CFI_STARTPROC
 	aghi	%r15,-16
+	CFI_DEF_CFA_OFFSET 176
+	CFI_VAL_OFFSET 15, -160
 	larl	%r5,_vdso_data
 	cghi	%r2,__CLOCK_REALTIME_COARSE
 	je	4f
@@ -53,9 +56,13 @@ __kernel_clock_gettime:
 	stg	%r1,8(%r3)			/* store tp->tv_nsec */
 	lghi	%r2,0
 	aghi	%r15,16
+	CFI_DEF_CFA_OFFSET 160
+	CFI_RESTORE 15
 	br	%r14
 
 	/* CLOCK_MONOTONIC_COARSE */
+	CFI_DEF_CFA_OFFSET 176
+	CFI_VAL_OFFSET 15, -160
 3:	lg	%r4,__VDSO_UPD_COUNT(%r5)	/* load update counter */
 	tmll	%r4,0x0001			/* pending update ? loop */
 	jnz	3b
@@ -108,9 +115,13 @@ __kernel_clock_gettime:
 	stg	%r1,8(%r3)			/* store tp->tv_nsec */
 	lghi	%r2,0
 	aghi	%r15,16
+	CFI_DEF_CFA_OFFSET 160
+	CFI_RESTORE 15
 	br	%r14
 
 	/* CPUCLOCK_VIRT for this thread */
+	CFI_DEF_CFA_OFFSET 176
+	CFI_VAL_OFFSET 15, -160
 9:	lghi	%r4,0
 	icm	%r0,15,__VDSO_ECTG_OK(%r5)
 	jz	12f
@@ -131,15 +142,21 @@ __kernel_clock_gettime:
 	stg	%r4,8(%r3)
 	lghi	%r2,0
 	aghi	%r15,16
+	CFI_DEF_CFA_OFFSET 160
+	CFI_RESTORE 15
 	br	%r14
 
 	/* Fallback to system call */
+	CFI_DEF_CFA_OFFSET 176
+	CFI_VAL_OFFSET 15, -160
 12:	lghi	%r1,__NR_clock_gettime
 	svc	0
 	aghi	%r15,16
+	CFI_DEF_CFA_OFFSET 160
+	CFI_RESTORE 15
 	br	%r14
+	CFI_ENDPROC
 
 13:	.quad	1000000000
 14:	.quad	19342813113834067
-	.cfi_endproc
 	.size	__kernel_clock_gettime,.-__kernel_clock_gettime
diff --git a/arch/s390/kernel/vdso64/getcpu.S b/arch/s390/kernel/vdso64/getcpu.S
index e9c34364d97b..2446e9dac8ab 100644
--- a/arch/s390/kernel/vdso64/getcpu.S
+++ b/arch/s390/kernel/vdso64/getcpu.S
@@ -8,13 +8,14 @@
  */
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_getcpu
 	.type  __kernel_getcpu,@function
 __kernel_getcpu:
-	.cfi_startproc
+	CFI_STARTPROC
 	la	%r4,0
 	sacf	256
 	l	%r5,__VDSO_CPU_NR(%r4)
@@ -28,5 +29,5 @@ __kernel_getcpu:
 	st	%r4,0(%r3)
 3:	lghi	%r2,0
 	br	%r14
-	.cfi_endproc
+	CFI_ENDPROC
 	.size	__kernel_getcpu,.-__kernel_getcpu
diff --git a/arch/s390/kernel/vdso64/gettimeofday.S b/arch/s390/kernel/vdso64/gettimeofday.S
index cc9dbc27da6f..6e1f0b421695 100644
--- a/arch/s390/kernel/vdso64/gettimeofday.S
+++ b/arch/s390/kernel/vdso64/gettimeofday.S
@@ -9,14 +9,17 @@
 #include <asm/vdso.h>
 #include <asm/asm-offsets.h>
 #include <asm/unistd.h>
+#include <asm/dwarf.h>
 
 	.text
 	.align 4
 	.globl __kernel_gettimeofday
 	.type  __kernel_gettimeofday,@function
 __kernel_gettimeofday:
-	.cfi_startproc
+	CFI_STARTPROC
 	aghi	%r15,-16
+	CFI_ADJUST_CFA_OFFSET 16
+	CFI_VAL_OFFSET 15, -160
 	larl	%r5,_vdso_data
 0:	ltgr	%r3,%r3				/* check if tz is NULL */
 	je	1f
@@ -58,8 +61,10 @@ __kernel_gettimeofday:
 	stg	%r0,8(%r2)			/* store tv->tv_usec */
 4:	lghi	%r2,0
 	aghi	%r15,16
+	CFI_ADJUST_CFA_OFFSET -16
+	CFI_RESTORE 15
 	br	%r14
+	CFI_ENDPROC
 5:	.quad	1000000000
 	.long	274877907
-	.cfi_endproc
 	.size	__kernel_gettimeofday,.-__kernel_gettimeofday
diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index a049ff005f03..08d12cfaf091 100644
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -123,6 +123,20 @@ SECTIONS
 		*(.altinstr_replacement)
 	}
 
+	/*
+	 * Table with the patch locations to undo expolines
+	*/
+	.nospec_call_table : {
+		__nospec_call_start = . ;
+		*(.s390_indirect*)
+		__nospec_call_end = . ;
+	}
+	.nospec_return_table : {
+		__nospec_return_start = . ;
+		*(.s390_return*)
+		__nospec_return_end = . ;
+	}
+
 	/* early.c uses stsi, which requires page aligned data. */
 	. = ALIGN(PAGE_SIZE);
 	INIT_DATA_SECTION(0x100)
@@ -141,4 +155,7 @@ SECTIONS
 
 	/* Sections to be discarded */
 	DISCARDS
+	/DISCARD/ : {
+		*(.eh_frame)
+	}
 }
diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
index 9a4594e0a1ff..a3dbd459cce9 100644
--- a/arch/s390/kvm/Kconfig
+++ b/arch/s390/kvm/Kconfig
@@ -23,6 +23,7 @@ config KVM
 	select PREEMPT_NOTIFIERS
 	select ANON_INODES
 	select HAVE_KVM_CPU_RELAX_INTERCEPT
+	select HAVE_KVM_VCPU_ASYNC_IOCTL
 	select HAVE_KVM_EVENTFD
 	select KVM_ASYNC_PF
 	select KVM_ASYNC_PF_SYNC
diff --git a/arch/s390/kvm/diag.c b/arch/s390/kvm/diag.c
index 89aa114a2cba..45634b3d2e0a 100644
--- a/arch/s390/kvm/diag.c
+++ b/arch/s390/kvm/diag.c
@@ -257,6 +257,7 @@ int kvm_s390_handle_diag(struct kvm_vcpu *vcpu)
 	case 0x500:
 		return __diag_virtio_hypercall(vcpu);
 	default:
+		vcpu->stat.diagnose_other++;
 		return -EOPNOTSUPP;
 	}
 }
diff --git a/arch/s390/kvm/intercept.c b/arch/s390/kvm/intercept.c
index 9c7d70715862..07c6e81163bf 100644
--- a/arch/s390/kvm/intercept.c
+++ b/arch/s390/kvm/intercept.c
@@ -22,22 +22,6 @@
 #include "trace.h"
 #include "trace-s390.h"
 
-
-static const intercept_handler_t instruction_handlers[256] = {
-	[0x01] = kvm_s390_handle_01,
-	[0x82] = kvm_s390_handle_lpsw,
-	[0x83] = kvm_s390_handle_diag,
-	[0xaa] = kvm_s390_handle_aa,
-	[0xae] = kvm_s390_handle_sigp,
-	[0xb2] = kvm_s390_handle_b2,
-	[0xb6] = kvm_s390_handle_stctl,
-	[0xb7] = kvm_s390_handle_lctl,
-	[0xb9] = kvm_s390_handle_b9,
-	[0xe3] = kvm_s390_handle_e3,
-	[0xe5] = kvm_s390_handle_e5,
-	[0xeb] = kvm_s390_handle_eb,
-};
-
 u8 kvm_s390_get_ilen(struct kvm_vcpu *vcpu)
 {
 	struct kvm_s390_sie_block *sie_block = vcpu->arch.sie_block;
@@ -129,16 +113,39 @@ static int handle_validity(struct kvm_vcpu *vcpu)
 
 static int handle_instruction(struct kvm_vcpu *vcpu)
 {
-	intercept_handler_t handler;
-
 	vcpu->stat.exit_instruction++;
 	trace_kvm_s390_intercept_instruction(vcpu,
 					     vcpu->arch.sie_block->ipa,
 					     vcpu->arch.sie_block->ipb);
-	handler = instruction_handlers[vcpu->arch.sie_block->ipa >> 8];
-	if (handler)
-		return handler(vcpu);
-	return -EOPNOTSUPP;
+
+	switch (vcpu->arch.sie_block->ipa >> 8) {
+	case 0x01:
+		return kvm_s390_handle_01(vcpu);
+	case 0x82:
+		return kvm_s390_handle_lpsw(vcpu);
+	case 0x83:
+		return kvm_s390_handle_diag(vcpu);
+	case 0xaa:
+		return kvm_s390_handle_aa(vcpu);
+	case 0xae:
+		return kvm_s390_handle_sigp(vcpu);
+	case 0xb2:
+		return kvm_s390_handle_b2(vcpu);
+	case 0xb6:
+		return kvm_s390_handle_stctl(vcpu);
+	case 0xb7:
+		return kvm_s390_handle_lctl(vcpu);
+	case 0xb9:
+		return kvm_s390_handle_b9(vcpu);
+	case 0xe3:
+		return kvm_s390_handle_e3(vcpu);
+	case 0xe5:
+		return kvm_s390_handle_e5(vcpu);
+	case 0xeb:
+		return kvm_s390_handle_eb(vcpu);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 static int inject_prog_on_prog_intercept(struct kvm_vcpu *vcpu)
diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c
index 024ad8bcc516..b04616b57a94 100644
--- a/arch/s390/kvm/interrupt.c
+++ b/arch/s390/kvm/interrupt.c
@@ -36,7 +36,7 @@ static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id)
 {
 	int c, scn;
 
-	if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND))
+	if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND))
 		return 0;
 
 	BUG_ON(!kvm_s390_use_sca_entries());
@@ -101,18 +101,17 @@ static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id)
 		/* another external call is pending */
 		return -EBUSY;
 	}
-	atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 	return 0;
 }
 
 static void sca_clear_ext_call(struct kvm_vcpu *vcpu)
 {
-	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
 	int rc, expect;
 
 	if (!kvm_s390_use_sca_entries())
 		return;
-	atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags);
+	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND);
 	read_lock(&vcpu->kvm->arch.sca_lock);
 	if (vcpu->kvm->arch.use_esca) {
 		struct esca_block *sca = vcpu->kvm->arch.sca;
@@ -170,8 +169,15 @@ static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
 
 static int ckc_irq_pending(struct kvm_vcpu *vcpu)
 {
-	if (vcpu->arch.sie_block->ckc >= kvm_s390_get_tod_clock_fast(vcpu->kvm))
+	const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
+	const u64 ckc = vcpu->arch.sie_block->ckc;
+
+	if (vcpu->arch.sie_block->gcr[0] & 0x0020000000000000ul) {
+		if ((s64)ckc >= (s64)now)
+			return 0;
+	} else if (ckc >= now) {
 		return 0;
+	}
 	return ckc_interrupts_enabled(vcpu);
 }
 
@@ -188,15 +194,14 @@ static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
 	return kvm_s390_get_cpu_timer(vcpu) >> 63;
 }
 
-static inline int is_ioirq(unsigned long irq_type)
+static uint64_t isc_to_isc_bits(int isc)
 {
-	return ((irq_type >= IRQ_PEND_IO_ISC_0) &&
-		(irq_type <= IRQ_PEND_IO_ISC_7));
+	return (0x80 >> isc) << 24;
 }
 
-static uint64_t isc_to_isc_bits(int isc)
+static inline u32 isc_to_int_word(u8 isc)
 {
-	return (0x80 >> isc) << 24;
+	return ((u32)isc << 27) | 0x80000000;
 }
 
 static inline u8 int_word_to_isc(u32 int_word)
@@ -204,20 +209,54 @@ static inline u8 int_word_to_isc(u32 int_word)
 	return (int_word & 0x38000000) >> 27;
 }
 
-static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
+/*
+ * To use atomic bitmap functions, we have to provide a bitmap address
+ * that is u64 aligned. However, the ipm might be u32 aligned.
+ * Therefore, we logically start the bitmap at the very beginning of the
+ * struct and fixup the bit number.
+ */
+#define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE)
+
+static inline void kvm_s390_gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
+{
+	set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
+}
+
+static inline u8 kvm_s390_gisa_get_ipm(struct kvm_s390_gisa *gisa)
+{
+	return READ_ONCE(gisa->ipm);
+}
+
+static inline void kvm_s390_gisa_clear_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
+{
+	clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
+}
+
+static inline int kvm_s390_gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc)
+{
+	return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa);
+}
+
+static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu)
 {
 	return vcpu->kvm->arch.float_int.pending_irqs |
-	       vcpu->arch.local_int.pending_irqs;
+		vcpu->arch.local_int.pending_irqs;
+}
+
+static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu)
+{
+	return pending_irqs_no_gisa(vcpu) |
+		kvm_s390_gisa_get_ipm(vcpu->kvm->arch.gisa) << IRQ_PEND_IO_ISC_7;
 }
 
 static inline int isc_to_irq_type(unsigned long isc)
 {
-	return IRQ_PEND_IO_ISC_0 + isc;
+	return IRQ_PEND_IO_ISC_0 - isc;
 }
 
 static inline int irq_type_to_isc(unsigned long irq_type)
 {
-	return irq_type - IRQ_PEND_IO_ISC_0;
+	return IRQ_PEND_IO_ISC_0 - irq_type;
 }
 
 static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
@@ -278,20 +317,20 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 
 static void __set_cpu_idle(struct kvm_vcpu *vcpu)
 {
-	atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
-	set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
+	set_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask);
 }
 
 static void __unset_cpu_idle(struct kvm_vcpu *vcpu)
 {
-	atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
-	clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
+	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
+	clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask);
 }
 
 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 {
-	atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT,
-		    &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT |
+				      CPUSTAT_STOP_INT);
 	vcpu->arch.sie_block->lctl = 0x0000;
 	vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT);
 
@@ -302,17 +341,12 @@ static void __reset_intercept_indicators(struct kvm_vcpu *vcpu)
 	}
 }
 
-static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
-{
-	atomic_or(flag, &vcpu->arch.sie_block->cpuflags);
-}
-
 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
 {
-	if (!(pending_irqs(vcpu) & IRQ_PEND_IO_MASK))
+	if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK))
 		return;
 	else if (psw_ioint_disabled(vcpu))
-		__set_cpuflag(vcpu, CPUSTAT_IO_INT);
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT);
 	else
 		vcpu->arch.sie_block->lctl |= LCTL_CR6;
 }
@@ -322,7 +356,7 @@ static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 	if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK))
 		return;
 	if (psw_extint_disabled(vcpu))
-		__set_cpuflag(vcpu, CPUSTAT_EXT_INT);
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 	else
 		vcpu->arch.sie_block->lctl |= LCTL_CR0;
 }
@@ -340,7 +374,7 @@ static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu)
 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
 {
 	if (kvm_s390_is_stop_irq_pending(vcpu))
-		__set_cpuflag(vcpu, CPUSTAT_STOP_INT);
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 }
 
 /* Set interception request for non-deliverable interrupts */
@@ -897,18 +931,38 @@ static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
 	return rc ? -EFAULT : 0;
 }
 
+static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io)
+{
+	int rc;
+
+	rc  = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID);
+	rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR);
+	rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM);
+	rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD);
+	rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
+			     &vcpu->arch.sie_block->gpsw,
+			     sizeof(psw_t));
+	rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
+			    &vcpu->arch.sie_block->gpsw,
+			    sizeof(psw_t));
+	return rc ? -EFAULT : 0;
+}
+
 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
 				     unsigned long irq_type)
 {
 	struct list_head *isc_list;
 	struct kvm_s390_float_interrupt *fi;
 	struct kvm_s390_interrupt_info *inti = NULL;
+	struct kvm_s390_io_info io;
+	u32 isc;
 	int rc = 0;
 
 	fi = &vcpu->kvm->arch.float_int;
 
 	spin_lock(&fi->lock);
-	isc_list = &fi->lists[irq_type_to_isc(irq_type)];
+	isc = irq_type_to_isc(irq_type);
+	isc_list = &fi->lists[isc];
 	inti = list_first_entry_or_null(isc_list,
 					struct kvm_s390_interrupt_info,
 					list);
@@ -936,44 +990,33 @@ static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
 	spin_unlock(&fi->lock);
 
 	if (inti) {
-		rc  = put_guest_lc(vcpu, inti->io.subchannel_id,
-				(u16 *)__LC_SUBCHANNEL_ID);
-		rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
-				(u16 *)__LC_SUBCHANNEL_NR);
-		rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
-				(u32 *)__LC_IO_INT_PARM);
-		rc |= put_guest_lc(vcpu, inti->io.io_int_word,
-				(u32 *)__LC_IO_INT_WORD);
-		rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
-				&vcpu->arch.sie_block->gpsw,
-				sizeof(psw_t));
-		rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
-				&vcpu->arch.sie_block->gpsw,
-				sizeof(psw_t));
+		rc = __do_deliver_io(vcpu, &(inti->io));
 		kfree(inti);
+		goto out;
 	}
 
-	return rc ? -EFAULT : 0;
+	if (vcpu->kvm->arch.gisa &&
+	    kvm_s390_gisa_tac_ipm_gisc(vcpu->kvm->arch.gisa, isc)) {
+		/*
+		 * in case an adapter interrupt was not delivered
+		 * in SIE context KVM will handle the delivery
+		 */
+		VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc);
+		memset(&io, 0, sizeof(io));
+		io.io_int_word = isc_to_int_word(isc);
+		vcpu->stat.deliver_io_int++;
+		trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
+			KVM_S390_INT_IO(1, 0, 0, 0),
+			((__u32)io.subchannel_id << 16) |
+			io.subchannel_nr,
+			((__u64)io.io_int_parm << 32) |
+			io.io_int_word);
+		rc = __do_deliver_io(vcpu, &io);
+	}
+out:
+	return rc;
 }
 
-typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu);
-
-static const deliver_irq_t deliver_irq_funcs[] = {
-	[IRQ_PEND_MCHK_EX]        = __deliver_machine_check,
-	[IRQ_PEND_MCHK_REP]       = __deliver_machine_check,
-	[IRQ_PEND_PROG]           = __deliver_prog,
-	[IRQ_PEND_EXT_EMERGENCY]  = __deliver_emergency_signal,
-	[IRQ_PEND_EXT_EXTERNAL]   = __deliver_external_call,
-	[IRQ_PEND_EXT_CLOCK_COMP] = __deliver_ckc,
-	[IRQ_PEND_EXT_CPU_TIMER]  = __deliver_cpu_timer,
-	[IRQ_PEND_RESTART]        = __deliver_restart,
-	[IRQ_PEND_SET_PREFIX]     = __deliver_set_prefix,
-	[IRQ_PEND_PFAULT_INIT]    = __deliver_pfault_init,
-	[IRQ_PEND_EXT_SERVICE]    = __deliver_service,
-	[IRQ_PEND_PFAULT_DONE]    = __deliver_pfault_done,
-	[IRQ_PEND_VIRTIO]         = __deliver_virtio,
-};
-
 /* Check whether an external call is pending (deliverable or not) */
 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
 {
@@ -1011,13 +1054,19 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 
 static u64 __calculate_sltime(struct kvm_vcpu *vcpu)
 {
-	u64 now, cputm, sltime = 0;
+	const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
+	const u64 ckc = vcpu->arch.sie_block->ckc;
+	u64 cputm, sltime = 0;
 
 	if (ckc_interrupts_enabled(vcpu)) {
-		now = kvm_s390_get_tod_clock_fast(vcpu->kvm);
-		sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now);
-		/* already expired or overflow? */
-		if (!sltime || vcpu->arch.sie_block->ckc <= now)
+		if (vcpu->arch.sie_block->gcr[0] & 0x0020000000000000ul) {
+			if ((s64)now < (s64)ckc)
+				sltime = tod_to_ns((s64)ckc - (s64)now);
+		} else if (now < ckc) {
+			sltime = tod_to_ns(ckc - now);
+		}
+		/* already expired */
+		if (!sltime)
 			return 0;
 		if (cpu_timer_interrupts_enabled(vcpu)) {
 			cputm = kvm_s390_get_cpu_timer(vcpu);
@@ -1137,7 +1186,6 @@ void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
 {
 	struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
-	deliver_irq_t func;
 	int rc = 0;
 	unsigned long irq_type;
 	unsigned long irqs;
@@ -1155,18 +1203,59 @@ int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
 		set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
 
 	while ((irqs = deliverable_irqs(vcpu)) && !rc) {
-		/* bits are in the order of interrupt priority */
-		irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT);
-		if (is_ioirq(irq_type)) {
+		/* bits are in the reverse order of interrupt priority */
+		irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT);
+		switch (irq_type) {
+		case IRQ_PEND_IO_ISC_0:
+		case IRQ_PEND_IO_ISC_1:
+		case IRQ_PEND_IO_ISC_2:
+		case IRQ_PEND_IO_ISC_3:
+		case IRQ_PEND_IO_ISC_4:
+		case IRQ_PEND_IO_ISC_5:
+		case IRQ_PEND_IO_ISC_6:
+		case IRQ_PEND_IO_ISC_7:
 			rc = __deliver_io(vcpu, irq_type);
-		} else {
-			func = deliver_irq_funcs[irq_type];
-			if (!func) {
-				WARN_ON_ONCE(func == NULL);
-				clear_bit(irq_type, &li->pending_irqs);
-				continue;
-			}
-			rc = func(vcpu);
+			break;
+		case IRQ_PEND_MCHK_EX:
+		case IRQ_PEND_MCHK_REP:
+			rc = __deliver_machine_check(vcpu);
+			break;
+		case IRQ_PEND_PROG:
+			rc = __deliver_prog(vcpu);
+			break;
+		case IRQ_PEND_EXT_EMERGENCY:
+			rc = __deliver_emergency_signal(vcpu);
+			break;
+		case IRQ_PEND_EXT_EXTERNAL:
+			rc = __deliver_external_call(vcpu);
+			break;
+		case IRQ_PEND_EXT_CLOCK_COMP:
+			rc = __deliver_ckc(vcpu);
+			break;
+		case IRQ_PEND_EXT_CPU_TIMER:
+			rc = __deliver_cpu_timer(vcpu);
+			break;
+		case IRQ_PEND_RESTART:
+			rc = __deliver_restart(vcpu);
+			break;
+		case IRQ_PEND_SET_PREFIX:
+			rc = __deliver_set_prefix(vcpu);
+			break;
+		case IRQ_PEND_PFAULT_INIT:
+			rc = __deliver_pfault_init(vcpu);
+			break;
+		case IRQ_PEND_EXT_SERVICE:
+			rc = __deliver_service(vcpu);
+			break;
+		case IRQ_PEND_PFAULT_DONE:
+			rc = __deliver_pfault_done(vcpu);
+			break;
+		case IRQ_PEND_VIRTIO:
+			rc = __deliver_virtio(vcpu);
+			break;
+		default:
+			WARN_ONCE(1, "Unknown pending irq type %ld", irq_type);
+			clear_bit(irq_type, &li->pending_irqs);
 		}
 	}
 
@@ -1228,7 +1317,7 @@ static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
 
 	li->irq.ext = irq->u.ext;
 	set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs);
-	atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 	return 0;
 }
 
@@ -1253,7 +1342,7 @@ static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
 	if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs))
 		return -EBUSY;
 	*extcall = irq->u.extcall;
-	atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 	return 0;
 }
 
@@ -1297,7 +1386,7 @@ static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
 	if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs))
 		return -EBUSY;
 	stop->flags = irq->u.stop.flags;
-	__set_cpuflag(vcpu, CPUSTAT_STOP_INT);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 	return 0;
 }
 
@@ -1329,7 +1418,7 @@ static int __inject_sigp_emergency(struct kvm_vcpu *vcpu,
 
 	set_bit(irq->u.emerg.code, li->sigp_emerg_pending);
 	set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs);
-	atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 	return 0;
 }
 
@@ -1373,7 +1462,7 @@ static int __inject_ckc(struct kvm_vcpu *vcpu)
 				   0, 0);
 
 	set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
-	atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 	return 0;
 }
 
@@ -1386,7 +1475,7 @@ static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
 				   0, 0);
 
 	set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
-	atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT);
 	return 0;
 }
 
@@ -1416,20 +1505,86 @@ static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
 	return NULL;
 }
 
+static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm,
+						      u64 isc_mask, u32 schid)
+{
+	struct kvm_s390_interrupt_info *inti = NULL;
+	int isc;
+
+	for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
+		if (isc_mask & isc_to_isc_bits(isc))
+			inti = get_io_int(kvm, isc, schid);
+	}
+	return inti;
+}
+
+static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid)
+{
+	unsigned long active_mask;
+	int isc;
+
+	if (schid)
+		goto out;
+	if (!kvm->arch.gisa)
+		goto out;
+
+	active_mask = (isc_mask & kvm_s390_gisa_get_ipm(kvm->arch.gisa) << 24) << 32;
+	while (active_mask) {
+		isc = __fls(active_mask) ^ (BITS_PER_LONG - 1);
+		if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, isc))
+			return isc;
+		clear_bit_inv(isc, &active_mask);
+	}
+out:
+	return -EINVAL;
+}
+
 /*
  * Dequeue and return an I/O interrupt matching any of the interruption
  * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
+ * Take into account the interrupts pending in the interrupt list and in GISA.
+ *
+ * Note that for a guest that does not enable I/O interrupts
+ * but relies on TPI, a flood of classic interrupts may starve
+ * out adapter interrupts on the same isc. Linux does not do
+ * that, and it is possible to work around the issue by configuring
+ * different iscs for classic and adapter interrupts in the guest,
+ * but we may want to revisit this in the future.
  */
 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
 						    u64 isc_mask, u32 schid)
 {
-	struct kvm_s390_interrupt_info *inti = NULL;
+	struct kvm_s390_interrupt_info *inti, *tmp_inti;
 	int isc;
 
-	for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
-		if (isc_mask & isc_to_isc_bits(isc))
-			inti = get_io_int(kvm, isc, schid);
+	inti = get_top_io_int(kvm, isc_mask, schid);
+
+	isc = get_top_gisa_isc(kvm, isc_mask, schid);
+	if (isc < 0)
+		/* no AI in GISA */
+		goto out;
+
+	if (!inti)
+		/* AI in GISA but no classical IO int */
+		goto gisa_out;
+
+	/* both types of interrupts present */
+	if (int_word_to_isc(inti->io.io_int_word) <= isc) {
+		/* classical IO int with higher priority */
+		kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
+		goto out;
 	}
+gisa_out:
+	tmp_inti = kzalloc(sizeof(*inti), GFP_KERNEL);
+	if (tmp_inti) {
+		tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0);
+		tmp_inti->io.io_int_word = isc_to_int_word(isc);
+		if (inti)
+			kvm_s390_reinject_io_int(kvm, inti);
+		inti = tmp_inti;
+	} else
+		kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
+out:
 	return inti;
 }
 
@@ -1517,6 +1672,15 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 	struct list_head *list;
 	int isc;
 
+	isc = int_word_to_isc(inti->io.io_int_word);
+
+	if (kvm->arch.gisa && inti->type & KVM_S390_INT_IO_AI_MASK) {
+		VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc);
+		kvm_s390_gisa_set_ipm_gisc(kvm->arch.gisa, isc);
+		kfree(inti);
+		return 0;
+	}
+
 	fi = &kvm->arch.float_int;
 	spin_lock(&fi->lock);
 	if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
@@ -1532,7 +1696,6 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 			inti->io.subchannel_id >> 8,
 			inti->io.subchannel_id >> 1 & 0x3,
 			inti->io.subchannel_nr);
-	isc = int_word_to_isc(inti->io.io_int_word);
 	list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
 	list_add_tail(&inti->list, list);
 	set_bit(isc_to_irq_type(isc), &fi->pending_irqs);
@@ -1546,7 +1709,6 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 static void __floating_irq_kick(struct kvm *kvm, u64 type)
 {
 	struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
-	struct kvm_s390_local_interrupt *li;
 	struct kvm_vcpu *dst_vcpu;
 	int sigcpu, online_vcpus, nr_tries = 0;
 
@@ -1568,20 +1730,18 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type)
 	dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
 
 	/* make the VCPU drop out of the SIE, or wake it up if sleeping */
-	li = &dst_vcpu->arch.local_int;
-	spin_lock(&li->lock);
 	switch (type) {
 	case KVM_S390_MCHK:
-		atomic_or(CPUSTAT_STOP_INT, li->cpuflags);
+		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT);
 		break;
 	case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
-		atomic_or(CPUSTAT_IO_INT, li->cpuflags);
+		if (!(type & KVM_S390_INT_IO_AI_MASK && kvm->arch.gisa))
+			kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT);
 		break;
 	default:
-		atomic_or(CPUSTAT_EXT_INT, li->cpuflags);
+		kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT);
 		break;
 	}
-	spin_unlock(&li->lock);
 	kvm_s390_vcpu_wakeup(dst_vcpu);
 }
 
@@ -1820,6 +1980,7 @@ void kvm_s390_clear_float_irqs(struct kvm *kvm)
 	for (i = 0; i < FIRQ_MAX_COUNT; i++)
 		fi->counters[i] = 0;
 	spin_unlock(&fi->lock);
+	kvm_s390_gisa_clear(kvm);
 };
 
 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
@@ -1847,6 +2008,22 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
 
 	max_irqs = len / sizeof(struct kvm_s390_irq);
 
+	if (kvm->arch.gisa &&
+	    kvm_s390_gisa_get_ipm(kvm->arch.gisa)) {
+		for (i = 0; i <= MAX_ISC; i++) {
+			if (n == max_irqs) {
+				/* signal userspace to try again */
+				ret = -ENOMEM;
+				goto out_nolock;
+			}
+			if (kvm_s390_gisa_tac_ipm_gisc(kvm->arch.gisa, i)) {
+				irq = (struct kvm_s390_irq *) &buf[n];
+				irq->type = KVM_S390_INT_IO(1, 0, 0, 0);
+				irq->u.io.io_int_word = isc_to_int_word(i);
+				n++;
+			}
+		}
+	}
 	fi = &kvm->arch.float_int;
 	spin_lock(&fi->lock);
 	for (i = 0; i < FIRQ_LIST_COUNT; i++) {
@@ -1885,6 +2062,7 @@ static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len)
 
 out:
 	spin_unlock(&fi->lock);
+out_nolock:
 	if (!ret && n > 0) {
 		if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n))
 			ret = -EFAULT;
@@ -2245,7 +2423,7 @@ static int kvm_s390_inject_airq(struct kvm *kvm,
 	struct kvm_s390_interrupt s390int = {
 		.type = KVM_S390_INT_IO(1, 0, 0, 0),
 		.parm = 0,
-		.parm64 = (adapter->isc << 27) | 0x80000000,
+		.parm64 = isc_to_int_word(adapter->isc),
 	};
 	int ret = 0;
 
@@ -2687,3 +2865,28 @@ int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len)
 
 	return n;
 }
+
+void kvm_s390_gisa_clear(struct kvm *kvm)
+{
+	if (kvm->arch.gisa) {
+		memset(kvm->arch.gisa, 0, sizeof(struct kvm_s390_gisa));
+		kvm->arch.gisa->next_alert = (u32)(u64)kvm->arch.gisa;
+		VM_EVENT(kvm, 3, "gisa 0x%pK cleared", kvm->arch.gisa);
+	}
+}
+
+void kvm_s390_gisa_init(struct kvm *kvm)
+{
+	if (css_general_characteristics.aiv) {
+		kvm->arch.gisa = &kvm->arch.sie_page2->gisa;
+		VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa);
+		kvm_s390_gisa_clear(kvm);
+	}
+}
+
+void kvm_s390_gisa_destroy(struct kvm *kvm)
+{
+	if (!kvm->arch.gisa)
+		return;
+	kvm->arch.gisa = NULL;
+}
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 1371dff2b90d..339ac0964590 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -2,7 +2,7 @@
 /*
  * hosting IBM Z kernel virtual machines (s390x)
  *
- * Copyright IBM Corp. 2008, 2017
+ * Copyright IBM Corp. 2008, 2018
  *
  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  *               Christian Borntraeger <borntraeger@de.ibm.com>
@@ -86,20 +86,33 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
 	{ "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
 	{ "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
+	{ "deliver_io_interrupt", VCPU_STAT(deliver_io_int) },
 	{ "exit_wait_state", VCPU_STAT(exit_wait_state) },
+	{ "instruction_epsw", VCPU_STAT(instruction_epsw) },
+	{ "instruction_gs", VCPU_STAT(instruction_gs) },
+	{ "instruction_io_other", VCPU_STAT(instruction_io_other) },
+	{ "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
+	{ "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
 	{ "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
+	{ "instruction_ptff", VCPU_STAT(instruction_ptff) },
 	{ "instruction_stidp", VCPU_STAT(instruction_stidp) },
+	{ "instruction_sck", VCPU_STAT(instruction_sck) },
+	{ "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
 	{ "instruction_spx", VCPU_STAT(instruction_spx) },
 	{ "instruction_stpx", VCPU_STAT(instruction_stpx) },
 	{ "instruction_stap", VCPU_STAT(instruction_stap) },
-	{ "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
+	{ "instruction_iske", VCPU_STAT(instruction_iske) },
+	{ "instruction_ri", VCPU_STAT(instruction_ri) },
+	{ "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
+	{ "instruction_sske", VCPU_STAT(instruction_sske) },
 	{ "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
-	{ "instruction_stsch", VCPU_STAT(instruction_stsch) },
-	{ "instruction_chsc", VCPU_STAT(instruction_chsc) },
 	{ "instruction_essa", VCPU_STAT(instruction_essa) },
 	{ "instruction_stsi", VCPU_STAT(instruction_stsi) },
 	{ "instruction_stfl", VCPU_STAT(instruction_stfl) },
+	{ "instruction_tb", VCPU_STAT(instruction_tb) },
+	{ "instruction_tpi", VCPU_STAT(instruction_tpi) },
 	{ "instruction_tprot", VCPU_STAT(instruction_tprot) },
+	{ "instruction_tsch", VCPU_STAT(instruction_tsch) },
 	{ "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 	{ "instruction_sie", VCPU_STAT(instruction_sie) },
 	{ "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
@@ -118,12 +131,13 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 	{ "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 	{ "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
-	{ "diagnose_10", VCPU_STAT(diagnose_10) },
-	{ "diagnose_44", VCPU_STAT(diagnose_44) },
-	{ "diagnose_9c", VCPU_STAT(diagnose_9c) },
-	{ "diagnose_258", VCPU_STAT(diagnose_258) },
-	{ "diagnose_308", VCPU_STAT(diagnose_308) },
-	{ "diagnose_500", VCPU_STAT(diagnose_500) },
+	{ "instruction_diag_10", VCPU_STAT(diagnose_10) },
+	{ "instruction_diag_44", VCPU_STAT(diagnose_44) },
+	{ "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
+	{ "instruction_diag_258", VCPU_STAT(diagnose_258) },
+	{ "instruction_diag_308", VCPU_STAT(diagnose_308) },
+	{ "instruction_diag_500", VCPU_STAT(diagnose_500) },
+	{ "instruction_diag_other", VCPU_STAT(diagnose_other) },
 	{ NULL }
 };
 
@@ -166,6 +180,28 @@ int kvm_arch_hardware_enable(void)
 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 			      unsigned long end);
 
+static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
+{
+	u8 delta_idx = 0;
+
+	/*
+	 * The TOD jumps by delta, we have to compensate this by adding
+	 * -delta to the epoch.
+	 */
+	delta = -delta;
+
+	/* sign-extension - we're adding to signed values below */
+	if ((s64)delta < 0)
+		delta_idx = -1;
+
+	scb->epoch += delta;
+	if (scb->ecd & ECD_MEF) {
+		scb->epdx += delta_idx;
+		if (scb->epoch < delta)
+			scb->epdx += 1;
+	}
+}
+
 /*
  * This callback is executed during stop_machine(). All CPUs are therefore
  * temporarily stopped. In order not to change guest behavior, we have to
@@ -181,13 +217,17 @@ static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 	unsigned long long *delta = v;
 
 	list_for_each_entry(kvm, &vm_list, vm_list) {
-		kvm->arch.epoch -= *delta;
 		kvm_for_each_vcpu(i, vcpu, kvm) {
-			vcpu->arch.sie_block->epoch -= *delta;
+			kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
+			if (i == 0) {
+				kvm->arch.epoch = vcpu->arch.sie_block->epoch;
+				kvm->arch.epdx = vcpu->arch.sie_block->epdx;
+			}
 			if (vcpu->arch.cputm_enabled)
 				vcpu->arch.cputm_start += *delta;
 			if (vcpu->arch.vsie_block)
-				vcpu->arch.vsie_block->epoch -= *delta;
+				kvm_clock_sync_scb(vcpu->arch.vsie_block,
+						   *delta);
 		}
 	}
 	return NOTIFY_OK;
@@ -576,7 +616,7 @@ static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 	case KVM_CAP_S390_GS:
 		r = -EINVAL;
 		mutex_lock(&kvm->lock);
-		if (atomic_read(&kvm->online_vcpus)) {
+		if (kvm->created_vcpus) {
 			r = -EBUSY;
 		} else if (test_facility(133)) {
 			set_kvm_facility(kvm->arch.model.fac_mask, 133);
@@ -889,12 +929,9 @@ static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
 	if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 		return -EFAULT;
 
-	if (test_kvm_facility(kvm, 139))
-		kvm_s390_set_tod_clock_ext(kvm, &gtod);
-	else if (gtod.epoch_idx == 0)
-		kvm_s390_set_tod_clock(kvm, gtod.tod);
-	else
+	if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
 		return -EINVAL;
+	kvm_s390_set_tod_clock(kvm, &gtod);
 
 	VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
 		gtod.epoch_idx, gtod.tod);
@@ -919,13 +956,14 @@ static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 
 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 {
-	u64 gtod;
+	struct kvm_s390_vm_tod_clock gtod = { 0 };
 
-	if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
+	if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
+			   sizeof(gtod.tod)))
 		return -EFAULT;
 
-	kvm_s390_set_tod_clock(kvm, gtod);
-	VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
+	kvm_s390_set_tod_clock(kvm, &gtod);
+	VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
 	return 0;
 }
 
@@ -1088,7 +1126,6 @@ static int kvm_s390_set_processor_feat(struct kvm *kvm,
 				       struct kvm_device_attr *attr)
 {
 	struct kvm_s390_vm_cpu_feat data;
-	int ret = -EBUSY;
 
 	if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
 		return -EFAULT;
@@ -1098,13 +1135,18 @@ static int kvm_s390_set_processor_feat(struct kvm *kvm,
 		return -EINVAL;
 
 	mutex_lock(&kvm->lock);
-	if (!atomic_read(&kvm->online_vcpus)) {
-		bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
-			    KVM_S390_VM_CPU_FEAT_NR_BITS);
-		ret = 0;
+	if (kvm->created_vcpus) {
+		mutex_unlock(&kvm->lock);
+		return -EBUSY;
 	}
+	bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
+		    KVM_S390_VM_CPU_FEAT_NR_BITS);
 	mutex_unlock(&kvm->lock);
-	return ret;
+	VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
+			 data.feat[0],
+			 data.feat[1],
+			 data.feat[2]);
+	return 0;
 }
 
 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
@@ -1206,6 +1248,10 @@ static int kvm_s390_get_processor_feat(struct kvm *kvm,
 		    KVM_S390_VM_CPU_FEAT_NR_BITS);
 	if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 		return -EFAULT;
+	VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
+			 data.feat[0],
+			 data.feat[1],
+			 data.feat[2]);
 	return 0;
 }
 
@@ -1219,6 +1265,10 @@ static int kvm_s390_get_machine_feat(struct kvm *kvm,
 		    KVM_S390_VM_CPU_FEAT_NR_BITS);
 	if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 		return -EFAULT;
+	VM_EVENT(kvm, 3, "GET: host feat:  0x%16.16llx.0x%16.16llx.0x%16.16llx",
+			 data.feat[0],
+			 data.feat[1],
+			 data.feat[2]);
 	return 0;
 }
 
@@ -1911,6 +1961,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	if (!kvm->arch.dbf)
 		goto out_err;
 
+	BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
 	kvm->arch.sie_page2 =
 	     (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!kvm->arch.sie_page2)
@@ -1981,6 +2032,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
 	spin_lock_init(&kvm->arch.start_stop_lock);
 	kvm_s390_vsie_init(kvm);
+	kvm_s390_gisa_init(kvm);
 	KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
 
 	return 0;
@@ -2043,6 +2095,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
 	kvm_free_vcpus(kvm);
 	sca_dispose(kvm);
 	debug_unregister(kvm->arch.dbf);
+	kvm_s390_gisa_destroy(kvm);
 	free_page((unsigned long)kvm->arch.sie_page2);
 	if (!kvm_is_ucontrol(kvm))
 		gmap_remove(kvm->arch.gmap);
@@ -2094,6 +2147,7 @@ static void sca_add_vcpu(struct kvm_vcpu *vcpu)
 		/* we still need the basic sca for the ipte control */
 		vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
 		vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
+		return;
 	}
 	read_lock(&vcpu->kvm->arch.sca_lock);
 	if (vcpu->kvm->arch.use_esca) {
@@ -2314,7 +2368,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 
 	gmap_enable(vcpu->arch.enabled_gmap);
-	atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
 	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
 		__start_cpu_timer_accounting(vcpu);
 	vcpu->cpu = cpu;
@@ -2325,7 +2379,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	vcpu->cpu = -1;
 	if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
 		__stop_cpu_timer_accounting(vcpu);
-	atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
 	vcpu->arch.enabled_gmap = gmap_get_enabled();
 	gmap_disable(vcpu->arch.enabled_gmap);
 
@@ -2361,6 +2415,7 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 	mutex_lock(&vcpu->kvm->lock);
 	preempt_disable();
 	vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
+	vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
 	preempt_enable();
 	mutex_unlock(&vcpu->kvm->lock);
 	if (!kvm_is_ucontrol(vcpu->kvm)) {
@@ -2422,9 +2477,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 						    CPUSTAT_STOPPED);
 
 	if (test_kvm_facility(vcpu->kvm, 78))
-		atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
 	else if (test_kvm_facility(vcpu->kvm, 8))
-		atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
 
 	kvm_s390_vcpu_setup_model(vcpu);
 
@@ -2456,12 +2511,17 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 	if (test_kvm_facility(vcpu->kvm, 139))
 		vcpu->arch.sie_block->ecd |= ECD_MEF;
 
+	if (vcpu->arch.sie_block->gd) {
+		vcpu->arch.sie_block->eca |= ECA_AIV;
+		VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
+			   vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
+	}
 	vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
 					| SDNXC;
 	vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
 
 	if (sclp.has_kss)
-		atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags);
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
 	else
 		vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
 
@@ -2508,9 +2568,9 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 
 	vcpu->arch.sie_block->icpua = id;
 	spin_lock_init(&vcpu->arch.local_int.lock);
-	vcpu->arch.local_int.float_int = &kvm->arch.float_int;
-	vcpu->arch.local_int.wq = &vcpu->wq;
-	vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
+	vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa;
+	if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
+		vcpu->arch.sie_block->gd |= GISA_FORMAT1;
 	seqcount_init(&vcpu->arch.cputm_seqcount);
 
 	rc = kvm_vcpu_init(vcpu, kvm, id);
@@ -2567,7 +2627,7 @@ static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
  * return immediately. */
 void exit_sie(struct kvm_vcpu *vcpu)
 {
-	atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
 	while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
 		cpu_relax();
 }
@@ -2720,47 +2780,70 @@ static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
 
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
+	vcpu_load(vcpu);
 	memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
+	vcpu_load(vcpu);
 	memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
+	vcpu_load(vcpu);
+
 	memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
 	memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
+
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 				  struct kvm_sregs *sregs)
 {
+	vcpu_load(vcpu);
+
 	memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
 	memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
+
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
-	if (test_fp_ctl(fpu->fpc))
-		return -EINVAL;
+	int ret = 0;
+
+	vcpu_load(vcpu);
+
+	if (test_fp_ctl(fpu->fpc)) {
+		ret = -EINVAL;
+		goto out;
+	}
 	vcpu->run->s.regs.fpc = fpu->fpc;
 	if (MACHINE_HAS_VX)
 		convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
 				 (freg_t *) fpu->fprs);
 	else
 		memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
-	return 0;
+
+out:
+	vcpu_put(vcpu);
+	return ret;
 }
 
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
+	vcpu_load(vcpu);
+
 	/* make sure we have the latest values */
 	save_fpu_regs();
 	if (MACHINE_HAS_VX)
@@ -2769,6 +2852,8 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 	else
 		memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
 	fpu->fpc = vcpu->run->s.regs.fpc;
+
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -2800,41 +2885,56 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 {
 	int rc = 0;
 
+	vcpu_load(vcpu);
+
 	vcpu->guest_debug = 0;
 	kvm_s390_clear_bp_data(vcpu);
 
-	if (dbg->control & ~VALID_GUESTDBG_FLAGS)
-		return -EINVAL;
-	if (!sclp.has_gpere)
-		return -EINVAL;
+	if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
+		rc = -EINVAL;
+		goto out;
+	}
+	if (!sclp.has_gpere) {
+		rc = -EINVAL;
+		goto out;
+	}
 
 	if (dbg->control & KVM_GUESTDBG_ENABLE) {
 		vcpu->guest_debug = dbg->control;
 		/* enforce guest PER */
-		atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
+		kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
 
 		if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
 			rc = kvm_s390_import_bp_data(vcpu, dbg);
 	} else {
-		atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
+		kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
 		vcpu->arch.guestdbg.last_bp = 0;
 	}
 
 	if (rc) {
 		vcpu->guest_debug = 0;
 		kvm_s390_clear_bp_data(vcpu);
-		atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
+		kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
 	}
 
+out:
+	vcpu_put(vcpu);
 	return rc;
 }
 
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
+	int ret;
+
+	vcpu_load(vcpu);
+
 	/* CHECK_STOP and LOAD are not supported yet */
-	return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
-				       KVM_MP_STATE_OPERATING;
+	ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
+				      KVM_MP_STATE_OPERATING;
+
+	vcpu_put(vcpu);
+	return ret;
 }
 
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
@@ -2842,6 +2942,8 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 {
 	int rc = 0;
 
+	vcpu_load(vcpu);
+
 	/* user space knows about this interface - let it control the state */
 	vcpu->kvm->arch.user_cpu_state_ctrl = 1;
 
@@ -2859,12 +2961,13 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 		rc = -ENXIO;
 	}
 
+	vcpu_put(vcpu);
 	return rc;
 }
 
 static bool ibs_enabled(struct kvm_vcpu *vcpu)
 {
-	return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
+	return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
 }
 
 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
@@ -2900,8 +3003,7 @@ static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
 	if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
 		if (!ibs_enabled(vcpu)) {
 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
-			atomic_or(CPUSTAT_IBS,
-					&vcpu->arch.sie_block->cpuflags);
+			kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
 		}
 		goto retry;
 	}
@@ -2909,8 +3011,7 @@ static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
 	if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
 		if (ibs_enabled(vcpu)) {
 			trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
-			atomic_andnot(CPUSTAT_IBS,
-					  &vcpu->arch.sie_block->cpuflags);
+			kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
 		}
 		goto retry;
 	}
@@ -2947,8 +3048,8 @@ static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
 	return 0;
 }
 
-void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
-				 const struct kvm_s390_vm_tod_clock *gtod)
+void kvm_s390_set_tod_clock(struct kvm *kvm,
+			    const struct kvm_s390_vm_tod_clock *gtod)
 {
 	struct kvm_vcpu *vcpu;
 	struct kvm_s390_tod_clock_ext htod;
@@ -2960,10 +3061,12 @@ void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
 	get_tod_clock_ext((char *)&htod);
 
 	kvm->arch.epoch = gtod->tod - htod.tod;
-	kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
-
-	if (kvm->arch.epoch > gtod->tod)
-		kvm->arch.epdx -= 1;
+	kvm->arch.epdx = 0;
+	if (test_kvm_facility(kvm, 139)) {
+		kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
+		if (kvm->arch.epoch > gtod->tod)
+			kvm->arch.epdx -= 1;
+	}
 
 	kvm_s390_vcpu_block_all(kvm);
 	kvm_for_each_vcpu(i, vcpu, kvm) {
@@ -2976,22 +3079,6 @@ void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
 	mutex_unlock(&kvm->lock);
 }
 
-void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
-{
-	struct kvm_vcpu *vcpu;
-	int i;
-
-	mutex_lock(&kvm->lock);
-	preempt_disable();
-	kvm->arch.epoch = tod - get_tod_clock();
-	kvm_s390_vcpu_block_all(kvm);
-	kvm_for_each_vcpu(i, vcpu, kvm)
-		vcpu->arch.sie_block->epoch = kvm->arch.epoch;
-	kvm_s390_vcpu_unblock_all(kvm);
-	preempt_enable();
-	mutex_unlock(&kvm->lock);
-}
-
 /**
  * kvm_arch_fault_in_page - fault-in guest page if necessary
  * @vcpu: The corresponding virtual cpu
@@ -3390,9 +3477,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 	if (kvm_run->immediate_exit)
 		return -EINTR;
 
+	vcpu_load(vcpu);
+
 	if (guestdbg_exit_pending(vcpu)) {
 		kvm_s390_prepare_debug_exit(vcpu);
-		return 0;
+		rc = 0;
+		goto out;
 	}
 
 	kvm_sigset_activate(vcpu);
@@ -3402,7 +3492,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 	} else if (is_vcpu_stopped(vcpu)) {
 		pr_err_ratelimited("can't run stopped vcpu %d\n",
 				   vcpu->vcpu_id);
-		return -EINVAL;
+		rc = -EINVAL;
+		goto out;
 	}
 
 	sync_regs(vcpu, kvm_run);
@@ -3432,6 +3523,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 	kvm_sigset_deactivate(vcpu);
 
 	vcpu->stat.exit_userspace++;
+out:
+	vcpu_put(vcpu);
 	return rc;
 }
 
@@ -3560,7 +3653,7 @@ void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
 		__disable_ibs_on_all_vcpus(vcpu->kvm);
 	}
 
-	atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
 	/*
 	 * Another VCPU might have used IBS while we were offline.
 	 * Let's play safe and flush the VCPU at startup.
@@ -3586,7 +3679,7 @@ void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
 	/* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
 	kvm_s390_clear_stop_irq(vcpu);
 
-	atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
 	__disable_ibs_on_vcpu(vcpu);
 
 	for (i = 0; i < online_vcpus; i++) {
@@ -3693,36 +3786,45 @@ static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
 	return r;
 }
 
-long kvm_arch_vcpu_ioctl(struct file *filp,
-			 unsigned int ioctl, unsigned long arg)
+long kvm_arch_vcpu_async_ioctl(struct file *filp,
+			       unsigned int ioctl, unsigned long arg)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 	void __user *argp = (void __user *)arg;
-	int idx;
-	long r;
 
 	switch (ioctl) {
 	case KVM_S390_IRQ: {
 		struct kvm_s390_irq s390irq;
 
-		r = -EFAULT;
 		if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
-			break;
-		r = kvm_s390_inject_vcpu(vcpu, &s390irq);
-		break;
+			return -EFAULT;
+		return kvm_s390_inject_vcpu(vcpu, &s390irq);
 	}
 	case KVM_S390_INTERRUPT: {
 		struct kvm_s390_interrupt s390int;
 		struct kvm_s390_irq s390irq;
 
-		r = -EFAULT;
 		if (copy_from_user(&s390int, argp, sizeof(s390int)))
-			break;
+			return -EFAULT;
 		if (s390int_to_s390irq(&s390int, &s390irq))
 			return -EINVAL;
-		r = kvm_s390_inject_vcpu(vcpu, &s390irq);
-		break;
+		return kvm_s390_inject_vcpu(vcpu, &s390irq);
+	}
 	}
+	return -ENOIOCTLCMD;
+}
+
+long kvm_arch_vcpu_ioctl(struct file *filp,
+			 unsigned int ioctl, unsigned long arg)
+{
+	struct kvm_vcpu *vcpu = filp->private_data;
+	void __user *argp = (void __user *)arg;
+	int idx;
+	long r;
+
+	vcpu_load(vcpu);
+
+	switch (ioctl) {
 	case KVM_S390_STORE_STATUS:
 		idx = srcu_read_lock(&vcpu->kvm->srcu);
 		r = kvm_s390_vcpu_store_status(vcpu, arg);
@@ -3847,6 +3949,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 	default:
 		r = -ENOTTY;
 	}
+
+	vcpu_put(vcpu);
 	return r;
 }
 
diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h
index 5e46ba429bcb..f55ac0ef99ea 100644
--- a/arch/s390/kvm/kvm-s390.h
+++ b/arch/s390/kvm/kvm-s390.h
@@ -19,8 +19,6 @@
 #include <asm/processor.h>
 #include <asm/sclp.h>
 
-typedef int (*intercept_handler_t)(struct kvm_vcpu *vcpu);
-
 /* Transactional Memory Execution related macros */
 #define IS_TE_ENABLED(vcpu)	((vcpu->arch.sie_block->ecb & ECB_TE))
 #define TDB_FORMAT1		1
@@ -47,14 +45,29 @@ do { \
 	  d_args); \
 } while (0)
 
+static inline void kvm_s390_set_cpuflags(struct kvm_vcpu *vcpu, u32 flags)
+{
+	atomic_or(flags, &vcpu->arch.sie_block->cpuflags);
+}
+
+static inline void kvm_s390_clear_cpuflags(struct kvm_vcpu *vcpu, u32 flags)
+{
+	atomic_andnot(flags, &vcpu->arch.sie_block->cpuflags);
+}
+
+static inline bool kvm_s390_test_cpuflags(struct kvm_vcpu *vcpu, u32 flags)
+{
+	return (atomic_read(&vcpu->arch.sie_block->cpuflags) & flags) == flags;
+}
+
 static inline int is_vcpu_stopped(struct kvm_vcpu *vcpu)
 {
-	return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_STOPPED;
+	return kvm_s390_test_cpuflags(vcpu, CPUSTAT_STOPPED);
 }
 
 static inline int is_vcpu_idle(struct kvm_vcpu *vcpu)
 {
-	return test_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask);
+	return test_bit(vcpu->vcpu_id, vcpu->kvm->arch.float_int.idle_mask);
 }
 
 static inline int kvm_is_ucontrol(struct kvm *kvm)
@@ -268,9 +281,8 @@ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu);
 int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu);
 
 /* implemented in kvm-s390.c */
-void kvm_s390_set_tod_clock_ext(struct kvm *kvm,
-				 const struct kvm_s390_vm_tod_clock *gtod);
-void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod);
+void kvm_s390_set_tod_clock(struct kvm *kvm,
+			    const struct kvm_s390_vm_tod_clock *gtod);
 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable);
 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr);
 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr);
@@ -367,6 +379,9 @@ int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu,
 			   void __user *buf, int len);
 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu,
 			   __u8 __user *buf, int len);
+void kvm_s390_gisa_init(struct kvm *kvm);
+void kvm_s390_gisa_clear(struct kvm *kvm);
+void kvm_s390_gisa_destroy(struct kvm *kvm);
 
 /* implemented in guestdbg.c */
 void kvm_s390_backup_guest_per_regs(struct kvm_vcpu *vcpu);
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 0714bfa56da0..f0b4185158af 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -2,7 +2,7 @@
 /*
  * handling privileged instructions
  *
- * Copyright IBM Corp. 2008, 2013
+ * Copyright IBM Corp. 2008, 2018
  *
  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  *               Christian Borntraeger <borntraeger@de.ibm.com>
@@ -34,6 +34,8 @@
 
 static int handle_ri(struct kvm_vcpu *vcpu)
 {
+	vcpu->stat.instruction_ri++;
+
 	if (test_kvm_facility(vcpu->kvm, 64)) {
 		VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)");
 		vcpu->arch.sie_block->ecb3 |= ECB3_RI;
@@ -53,6 +55,8 @@ int kvm_s390_handle_aa(struct kvm_vcpu *vcpu)
 
 static int handle_gs(struct kvm_vcpu *vcpu)
 {
+	vcpu->stat.instruction_gs++;
+
 	if (test_kvm_facility(vcpu->kvm, 133)) {
 		VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)");
 		preempt_disable();
@@ -81,9 +85,12 @@ int kvm_s390_handle_e3(struct kvm_vcpu *vcpu)
 /* Handle SCK (SET CLOCK) interception */
 static int handle_set_clock(struct kvm_vcpu *vcpu)
 {
+	struct kvm_s390_vm_tod_clock gtod = { 0 };
 	int rc;
 	u8 ar;
-	u64 op2, val;
+	u64 op2;
+
+	vcpu->stat.instruction_sck++;
 
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
@@ -91,12 +98,12 @@ static int handle_set_clock(struct kvm_vcpu *vcpu)
 	op2 = kvm_s390_get_base_disp_s(vcpu, &ar);
 	if (op2 & 7)	/* Operand must be on a doubleword boundary */
 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
-	rc = read_guest(vcpu, op2, ar, &val, sizeof(val));
+	rc = read_guest(vcpu, op2, ar, &gtod.tod, sizeof(gtod.tod));
 	if (rc)
 		return kvm_s390_inject_prog_cond(vcpu, rc);
 
-	VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", val);
-	kvm_s390_set_tod_clock(vcpu->kvm, val);
+	VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", gtod.tod);
+	kvm_s390_set_tod_clock(vcpu->kvm, &gtod);
 
 	kvm_s390_set_psw_cc(vcpu, 0);
 	return 0;
@@ -203,14 +210,14 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
 
 	trace_kvm_s390_skey_related_inst(vcpu);
 	if (!(sie_block->ictl & (ICTL_ISKE | ICTL_SSKE | ICTL_RRBE)) &&
-	    !(atomic_read(&sie_block->cpuflags) & CPUSTAT_KSS))
+	    !kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
 		return rc;
 
 	rc = s390_enable_skey();
 	VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc);
 	if (!rc) {
-		if (atomic_read(&sie_block->cpuflags) & CPUSTAT_KSS)
-			atomic_andnot(CPUSTAT_KSS, &sie_block->cpuflags);
+		if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
+			kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
 		else
 			sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE |
 					     ICTL_RRBE);
@@ -222,7 +229,6 @@ static int try_handle_skey(struct kvm_vcpu *vcpu)
 {
 	int rc;
 
-	vcpu->stat.instruction_storage_key++;
 	rc = kvm_s390_skey_check_enable(vcpu);
 	if (rc)
 		return rc;
@@ -242,6 +248,8 @@ static int handle_iske(struct kvm_vcpu *vcpu)
 	int reg1, reg2;
 	int rc;
 
+	vcpu->stat.instruction_iske++;
+
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
@@ -274,6 +282,8 @@ static int handle_rrbe(struct kvm_vcpu *vcpu)
 	int reg1, reg2;
 	int rc;
 
+	vcpu->stat.instruction_rrbe++;
+
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
@@ -312,6 +322,8 @@ static int handle_sske(struct kvm_vcpu *vcpu)
 	int reg1, reg2;
 	int rc;
 
+	vcpu->stat.instruction_sske++;
+
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
@@ -392,6 +404,8 @@ static int handle_test_block(struct kvm_vcpu *vcpu)
 	gpa_t addr;
 	int reg2;
 
+	vcpu->stat.instruction_tb++;
+
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
@@ -424,6 +438,8 @@ static int handle_tpi(struct kvm_vcpu *vcpu)
 	u64 addr;
 	u8 ar;
 
+	vcpu->stat.instruction_tpi++;
+
 	addr = kvm_s390_get_base_disp_s(vcpu, &ar);
 	if (addr & 3)
 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
@@ -484,6 +500,8 @@ static int handle_tsch(struct kvm_vcpu *vcpu)
 	struct kvm_s390_interrupt_info *inti = NULL;
 	const u64 isc_mask = 0xffUL << 24; /* all iscs set */
 
+	vcpu->stat.instruction_tsch++;
+
 	/* a valid schid has at least one bit set */
 	if (vcpu->run->s.regs.gprs[1])
 		inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask,
@@ -527,6 +545,7 @@ static int handle_io_inst(struct kvm_vcpu *vcpu)
 		if (vcpu->arch.sie_block->ipa == 0xb235)
 			return handle_tsch(vcpu);
 		/* Handle in userspace. */
+		vcpu->stat.instruction_io_other++;
 		return -EOPNOTSUPP;
 	} else {
 		/*
@@ -592,6 +611,8 @@ int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu)
 	int rc;
 	u8 ar;
 
+	vcpu->stat.instruction_lpsw++;
+
 	if (gpsw->mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
@@ -619,6 +640,8 @@ static int handle_lpswe(struct kvm_vcpu *vcpu)
 	int rc;
 	u8 ar;
 
+	vcpu->stat.instruction_lpswe++;
+
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
@@ -773,61 +796,68 @@ static int handle_stsi(struct kvm_vcpu *vcpu)
 	return rc;
 }
 
-static const intercept_handler_t b2_handlers[256] = {
-	[0x02] = handle_stidp,
-	[0x04] = handle_set_clock,
-	[0x10] = handle_set_prefix,
-	[0x11] = handle_store_prefix,
-	[0x12] = handle_store_cpu_address,
-	[0x14] = kvm_s390_handle_vsie,
-	[0x21] = handle_ipte_interlock,
-	[0x29] = handle_iske,
-	[0x2a] = handle_rrbe,
-	[0x2b] = handle_sske,
-	[0x2c] = handle_test_block,
-	[0x30] = handle_io_inst,
-	[0x31] = handle_io_inst,
-	[0x32] = handle_io_inst,
-	[0x33] = handle_io_inst,
-	[0x34] = handle_io_inst,
-	[0x35] = handle_io_inst,
-	[0x36] = handle_io_inst,
-	[0x37] = handle_io_inst,
-	[0x38] = handle_io_inst,
-	[0x39] = handle_io_inst,
-	[0x3a] = handle_io_inst,
-	[0x3b] = handle_io_inst,
-	[0x3c] = handle_io_inst,
-	[0x50] = handle_ipte_interlock,
-	[0x56] = handle_sthyi,
-	[0x5f] = handle_io_inst,
-	[0x74] = handle_io_inst,
-	[0x76] = handle_io_inst,
-	[0x7d] = handle_stsi,
-	[0xb1] = handle_stfl,
-	[0xb2] = handle_lpswe,
-};
-
 int kvm_s390_handle_b2(struct kvm_vcpu *vcpu)
 {
-	intercept_handler_t handler;
-
-	/*
-	 * A lot of B2 instructions are priviledged. Here we check for
-	 * the privileged ones, that we can handle in the kernel.
-	 * Anything else goes to userspace.
-	 */
-	handler = b2_handlers[vcpu->arch.sie_block->ipa & 0x00ff];
-	if (handler)
-		return handler(vcpu);
-
-	return -EOPNOTSUPP;
+	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
+	case 0x02:
+		return handle_stidp(vcpu);
+	case 0x04:
+		return handle_set_clock(vcpu);
+	case 0x10:
+		return handle_set_prefix(vcpu);
+	case 0x11:
+		return handle_store_prefix(vcpu);
+	case 0x12:
+		return handle_store_cpu_address(vcpu);
+	case 0x14:
+		return kvm_s390_handle_vsie(vcpu);
+	case 0x21:
+	case 0x50:
+		return handle_ipte_interlock(vcpu);
+	case 0x29:
+		return handle_iske(vcpu);
+	case 0x2a:
+		return handle_rrbe(vcpu);
+	case 0x2b:
+		return handle_sske(vcpu);
+	case 0x2c:
+		return handle_test_block(vcpu);
+	case 0x30:
+	case 0x31:
+	case 0x32:
+	case 0x33:
+	case 0x34:
+	case 0x35:
+	case 0x36:
+	case 0x37:
+	case 0x38:
+	case 0x39:
+	case 0x3a:
+	case 0x3b:
+	case 0x3c:
+	case 0x5f:
+	case 0x74:
+	case 0x76:
+		return handle_io_inst(vcpu);
+	case 0x56:
+		return handle_sthyi(vcpu);
+	case 0x7d:
+		return handle_stsi(vcpu);
+	case 0xb1:
+		return handle_stfl(vcpu);
+	case 0xb2:
+		return handle_lpswe(vcpu);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 static int handle_epsw(struct kvm_vcpu *vcpu)
 {
 	int reg1, reg2;
 
+	vcpu->stat.instruction_epsw++;
+
 	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
 
 	/* This basically extracts the mask half of the psw. */
@@ -1081,25 +1111,22 @@ static int handle_essa(struct kvm_vcpu *vcpu)
 	return 0;
 }
 
-static const intercept_handler_t b9_handlers[256] = {
-	[0x8a] = handle_ipte_interlock,
-	[0x8d] = handle_epsw,
-	[0x8e] = handle_ipte_interlock,
-	[0x8f] = handle_ipte_interlock,
-	[0xab] = handle_essa,
-	[0xaf] = handle_pfmf,
-};
-
 int kvm_s390_handle_b9(struct kvm_vcpu *vcpu)
 {
-	intercept_handler_t handler;
-
-	/* This is handled just as for the B2 instructions. */
-	handler = b9_handlers[vcpu->arch.sie_block->ipa & 0x00ff];
-	if (handler)
-		return handler(vcpu);
-
-	return -EOPNOTSUPP;
+	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
+	case 0x8a:
+	case 0x8e:
+	case 0x8f:
+		return handle_ipte_interlock(vcpu);
+	case 0x8d:
+		return handle_epsw(vcpu);
+	case 0xab:
+		return handle_essa(vcpu);
+	case 0xaf:
+		return handle_pfmf(vcpu);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu)
@@ -1247,22 +1274,20 @@ static int handle_stctg(struct kvm_vcpu *vcpu)
 	return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0;
 }
 
-static const intercept_handler_t eb_handlers[256] = {
-	[0x2f] = handle_lctlg,
-	[0x25] = handle_stctg,
-	[0x60] = handle_ri,
-	[0x61] = handle_ri,
-	[0x62] = handle_ri,
-};
-
 int kvm_s390_handle_eb(struct kvm_vcpu *vcpu)
 {
-	intercept_handler_t handler;
-
-	handler = eb_handlers[vcpu->arch.sie_block->ipb & 0xff];
-	if (handler)
-		return handler(vcpu);
-	return -EOPNOTSUPP;
+	switch (vcpu->arch.sie_block->ipb & 0x000000ff) {
+	case 0x25:
+		return handle_stctg(vcpu);
+	case 0x2f:
+		return handle_lctlg(vcpu);
+	case 0x60:
+	case 0x61:
+	case 0x62:
+		return handle_ri(vcpu);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 static int handle_tprot(struct kvm_vcpu *vcpu)
@@ -1322,16 +1347,20 @@ static int handle_tprot(struct kvm_vcpu *vcpu)
 
 int kvm_s390_handle_e5(struct kvm_vcpu *vcpu)
 {
-	/* For e5xx... instructions we only handle TPROT */
-	if ((vcpu->arch.sie_block->ipa & 0x00ff) == 0x01)
+	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
+	case 0x01:
 		return handle_tprot(vcpu);
-	return -EOPNOTSUPP;
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 static int handle_sckpf(struct kvm_vcpu *vcpu)
 {
 	u32 value;
 
+	vcpu->stat.instruction_sckpf++;
+
 	if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
 		return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
 
@@ -1347,22 +1376,21 @@ static int handle_sckpf(struct kvm_vcpu *vcpu)
 
 static int handle_ptff(struct kvm_vcpu *vcpu)
 {
+	vcpu->stat.instruction_ptff++;
+
 	/* we don't emulate any control instructions yet */
 	kvm_s390_set_psw_cc(vcpu, 3);
 	return 0;
 }
 
-static const intercept_handler_t x01_handlers[256] = {
-	[0x04] = handle_ptff,
-	[0x07] = handle_sckpf,
-};
-
 int kvm_s390_handle_01(struct kvm_vcpu *vcpu)
 {
-	intercept_handler_t handler;
-
-	handler = x01_handlers[vcpu->arch.sie_block->ipa & 0x00ff];
-	if (handler)
-		return handler(vcpu);
-	return -EOPNOTSUPP;
+	switch (vcpu->arch.sie_block->ipa & 0x00ff) {
+	case 0x04:
+		return handle_ptff(vcpu);
+	case 0x07:
+		return handle_sckpf(vcpu);
+	default:
+		return -EOPNOTSUPP;
+	}
 }
diff --git a/arch/s390/kvm/sigp.c b/arch/s390/kvm/sigp.c
index c1f5cde2c878..683036c1c92a 100644
--- a/arch/s390/kvm/sigp.c
+++ b/arch/s390/kvm/sigp.c
@@ -20,22 +20,18 @@
 static int __sigp_sense(struct kvm_vcpu *vcpu, struct kvm_vcpu *dst_vcpu,
 			u64 *reg)
 {
-	struct kvm_s390_local_interrupt *li;
-	int cpuflags;
+	const bool stopped = kvm_s390_test_cpuflags(dst_vcpu, CPUSTAT_STOPPED);
 	int rc;
 	int ext_call_pending;
 
-	li = &dst_vcpu->arch.local_int;
-
-	cpuflags = atomic_read(li->cpuflags);
 	ext_call_pending = kvm_s390_ext_call_pending(dst_vcpu);
-	if (!(cpuflags & CPUSTAT_STOPPED) && !ext_call_pending)
+	if (!stopped && !ext_call_pending)
 		rc = SIGP_CC_ORDER_CODE_ACCEPTED;
 	else {
 		*reg &= 0xffffffff00000000UL;
 		if (ext_call_pending)
 			*reg |= SIGP_STATUS_EXT_CALL_PENDING;
-		if (cpuflags & CPUSTAT_STOPPED)
+		if (stopped)
 			*reg |= SIGP_STATUS_STOPPED;
 		rc = SIGP_CC_STATUS_STORED;
 	}
@@ -208,11 +204,9 @@ static int __sigp_store_status_at_addr(struct kvm_vcpu *vcpu,
 				       struct kvm_vcpu *dst_vcpu,
 				       u32 addr, u64 *reg)
 {
-	int flags;
 	int rc;
 
-	flags = atomic_read(dst_vcpu->arch.local_int.cpuflags);
-	if (!(flags & CPUSTAT_STOPPED)) {
+	if (!kvm_s390_test_cpuflags(dst_vcpu, CPUSTAT_STOPPED)) {
 		*reg &= 0xffffffff00000000UL;
 		*reg |= SIGP_STATUS_INCORRECT_STATE;
 		return SIGP_CC_STATUS_STORED;
@@ -231,7 +225,6 @@ static int __sigp_store_status_at_addr(struct kvm_vcpu *vcpu,
 static int __sigp_sense_running(struct kvm_vcpu *vcpu,
 				struct kvm_vcpu *dst_vcpu, u64 *reg)
 {
-	struct kvm_s390_local_interrupt *li;
 	int rc;
 
 	if (!test_kvm_facility(vcpu->kvm, 9)) {
@@ -240,8 +233,7 @@ static int __sigp_sense_running(struct kvm_vcpu *vcpu,
 		return SIGP_CC_STATUS_STORED;
 	}
 
-	li = &dst_vcpu->arch.local_int;
-	if (atomic_read(li->cpuflags) & CPUSTAT_RUNNING) {
+	if (kvm_s390_test_cpuflags(dst_vcpu, CPUSTAT_RUNNING)) {
 		/* running */
 		rc = SIGP_CC_ORDER_CODE_ACCEPTED;
 	} else {
diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index 751348348477..8961e3970901 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -28,13 +28,23 @@ struct vsie_page {
 	 * the same offset as that in struct sie_page!
 	 */
 	struct mcck_volatile_info mcck_info;    /* 0x0200 */
-	/* the pinned originial scb */
+	/*
+	 * The pinned original scb. Be aware that other VCPUs can modify
+	 * it while we read from it. Values that are used for conditions or
+	 * are reused conditionally, should be accessed via READ_ONCE.
+	 */
 	struct kvm_s390_sie_block *scb_o;	/* 0x0218 */
 	/* the shadow gmap in use by the vsie_page */
 	struct gmap *gmap;			/* 0x0220 */
 	/* address of the last reported fault to guest2 */
 	unsigned long fault_addr;		/* 0x0228 */
-	__u8 reserved[0x0700 - 0x0230];		/* 0x0230 */
+	/* calculated guest addresses of satellite control blocks */
+	gpa_t sca_gpa;				/* 0x0230 */
+	gpa_t itdba_gpa;			/* 0x0238 */
+	gpa_t gvrd_gpa;				/* 0x0240 */
+	gpa_t riccbd_gpa;			/* 0x0248 */
+	gpa_t sdnx_gpa;				/* 0x0250 */
+	__u8 reserved[0x0700 - 0x0258];		/* 0x0258 */
 	struct kvm_s390_crypto_cb crycb;	/* 0x0700 */
 	__u8 fac[S390_ARCH_FAC_LIST_SIZE_BYTE];	/* 0x0800 */
 };
@@ -140,12 +150,13 @@ static int shadow_crycb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 {
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
-	u32 crycb_addr = scb_o->crycbd & 0x7ffffff8U;
+	const uint32_t crycbd_o = READ_ONCE(scb_o->crycbd);
+	const u32 crycb_addr = crycbd_o & 0x7ffffff8U;
 	unsigned long *b1, *b2;
 	u8 ecb3_flags;
 
 	scb_s->crycbd = 0;
-	if (!(scb_o->crycbd & vcpu->arch.sie_block->crycbd & CRYCB_FORMAT1))
+	if (!(crycbd_o & vcpu->arch.sie_block->crycbd & CRYCB_FORMAT1))
 		return 0;
 	/* format-1 is supported with message-security-assist extension 3 */
 	if (!test_kvm_facility(vcpu->kvm, 76))
@@ -183,12 +194,15 @@ static void prepare_ibc(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 {
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
+	/* READ_ONCE does not work on bitfields - use a temporary variable */
+	const uint32_t __new_ibc = scb_o->ibc;
+	const uint32_t new_ibc = READ_ONCE(__new_ibc) & 0x0fffU;
 	__u64 min_ibc = (sclp.ibc >> 16) & 0x0fffU;
 
 	scb_s->ibc = 0;
 	/* ibc installed in g2 and requested for g3 */
-	if (vcpu->kvm->arch.model.ibc && (scb_o->ibc & 0x0fffU)) {
-		scb_s->ibc = scb_o->ibc & 0x0fffU;
+	if (vcpu->kvm->arch.model.ibc && new_ibc) {
+		scb_s->ibc = new_ibc;
 		/* takte care of the minimum ibc level of the machine */
 		if (scb_s->ibc < min_ibc)
 			scb_s->ibc = min_ibc;
@@ -259,6 +273,10 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 {
 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
+	/* READ_ONCE does not work on bitfields - use a temporary variable */
+	const uint32_t __new_prefix = scb_o->prefix;
+	const uint32_t new_prefix = READ_ONCE(__new_prefix);
+	const bool wants_tx = READ_ONCE(scb_o->ecb) & ECB_TE;
 	bool had_tx = scb_s->ecb & ECB_TE;
 	unsigned long new_mso = 0;
 	int rc;
@@ -306,14 +324,14 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 	scb_s->icpua = scb_o->icpua;
 
 	if (!(atomic_read(&scb_s->cpuflags) & CPUSTAT_SM))
-		new_mso = scb_o->mso & 0xfffffffffff00000UL;
+		new_mso = READ_ONCE(scb_o->mso) & 0xfffffffffff00000UL;
 	/* if the hva of the prefix changes, we have to remap the prefix */
-	if (scb_s->mso != new_mso || scb_s->prefix != scb_o->prefix)
+	if (scb_s->mso != new_mso || scb_s->prefix != new_prefix)
 		prefix_unmapped(vsie_page);
 	 /* SIE will do mso/msl validity and exception checks for us */
 	scb_s->msl = scb_o->msl & 0xfffffffffff00000UL;
 	scb_s->mso = new_mso;
-	scb_s->prefix = scb_o->prefix;
+	scb_s->prefix = new_prefix;
 
 	/* We have to definetly flush the tlb if this scb never ran */
 	if (scb_s->ihcpu != 0xffffU)
@@ -325,11 +343,11 @@ static int shadow_scb(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 	if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_ESOP))
 		scb_s->ecb |= scb_o->ecb & ECB_HOSTPROTINT;
 	/* transactional execution */
-	if (test_kvm_facility(vcpu->kvm, 73)) {
+	if (test_kvm_facility(vcpu->kvm, 73) && wants_tx) {
 		/* remap the prefix is tx is toggled on */
-		if ((scb_o->ecb & ECB_TE) && !had_tx)
+		if (!had_tx)
 			prefix_unmapped(vsie_page);
-		scb_s->ecb |= scb_o->ecb & ECB_TE;
+		scb_s->ecb |= ECB_TE;
 	}
 	/* branch prediction */
 	if (test_kvm_facility(vcpu->kvm, 82))
@@ -473,46 +491,42 @@ static void unpin_guest_page(struct kvm *kvm, gpa_t gpa, hpa_t hpa)
 /* unpin all blocks previously pinned by pin_blocks(), marking them dirty */
 static void unpin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 {
-	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
 	hpa_t hpa;
-	gpa_t gpa;
 
 	hpa = (u64) scb_s->scaoh << 32 | scb_s->scaol;
 	if (hpa) {
-		gpa = scb_o->scaol & ~0xfUL;
-		if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO))
-			gpa |= (u64) scb_o->scaoh << 32;
-		unpin_guest_page(vcpu->kvm, gpa, hpa);
+		unpin_guest_page(vcpu->kvm, vsie_page->sca_gpa, hpa);
+		vsie_page->sca_gpa = 0;
 		scb_s->scaol = 0;
 		scb_s->scaoh = 0;
 	}
 
 	hpa = scb_s->itdba;
 	if (hpa) {
-		gpa = scb_o->itdba & ~0xffUL;
-		unpin_guest_page(vcpu->kvm, gpa, hpa);
+		unpin_guest_page(vcpu->kvm, vsie_page->itdba_gpa, hpa);
+		vsie_page->itdba_gpa = 0;
 		scb_s->itdba = 0;
 	}
 
 	hpa = scb_s->gvrd;
 	if (hpa) {
-		gpa = scb_o->gvrd & ~0x1ffUL;
-		unpin_guest_page(vcpu->kvm, gpa, hpa);
+		unpin_guest_page(vcpu->kvm, vsie_page->gvrd_gpa, hpa);
+		vsie_page->gvrd_gpa = 0;
 		scb_s->gvrd = 0;
 	}
 
 	hpa = scb_s->riccbd;
 	if (hpa) {
-		gpa = scb_o->riccbd & ~0x3fUL;
-		unpin_guest_page(vcpu->kvm, gpa, hpa);
+		unpin_guest_page(vcpu->kvm, vsie_page->riccbd_gpa, hpa);
+		vsie_page->riccbd_gpa = 0;
 		scb_s->riccbd = 0;
 	}
 
 	hpa = scb_s->sdnxo;
 	if (hpa) {
-		gpa = scb_o->sdnxo;
-		unpin_guest_page(vcpu->kvm, gpa, hpa);
+		unpin_guest_page(vcpu->kvm, vsie_page->sdnx_gpa, hpa);
+		vsie_page->sdnx_gpa = 0;
 		scb_s->sdnxo = 0;
 	}
 }
@@ -539,9 +553,9 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 	gpa_t gpa;
 	int rc = 0;
 
-	gpa = scb_o->scaol & ~0xfUL;
+	gpa = READ_ONCE(scb_o->scaol) & ~0xfUL;
 	if (test_kvm_cpu_feat(vcpu->kvm, KVM_S390_VM_CPU_FEAT_64BSCAO))
-		gpa |= (u64) scb_o->scaoh << 32;
+		gpa |= (u64) READ_ONCE(scb_o->scaoh) << 32;
 	if (gpa) {
 		if (!(gpa & ~0x1fffUL))
 			rc = set_validity_icpt(scb_s, 0x0038U);
@@ -557,11 +571,12 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 		}
 		if (rc)
 			goto unpin;
+		vsie_page->sca_gpa = gpa;
 		scb_s->scaoh = (u32)((u64)hpa >> 32);
 		scb_s->scaol = (u32)(u64)hpa;
 	}
 
-	gpa = scb_o->itdba & ~0xffUL;
+	gpa = READ_ONCE(scb_o->itdba) & ~0xffUL;
 	if (gpa && (scb_s->ecb & ECB_TE)) {
 		if (!(gpa & ~0x1fffU)) {
 			rc = set_validity_icpt(scb_s, 0x0080U);
@@ -573,10 +588,11 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 			rc = set_validity_icpt(scb_s, 0x0080U);
 			goto unpin;
 		}
+		vsie_page->itdba_gpa = gpa;
 		scb_s->itdba = hpa;
 	}
 
-	gpa = scb_o->gvrd & ~0x1ffUL;
+	gpa = READ_ONCE(scb_o->gvrd) & ~0x1ffUL;
 	if (gpa && (scb_s->eca & ECA_VX) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
 		if (!(gpa & ~0x1fffUL)) {
 			rc = set_validity_icpt(scb_s, 0x1310U);
@@ -591,10 +607,11 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 			rc = set_validity_icpt(scb_s, 0x1310U);
 			goto unpin;
 		}
+		vsie_page->gvrd_gpa = gpa;
 		scb_s->gvrd = hpa;
 	}
 
-	gpa = scb_o->riccbd & ~0x3fUL;
+	gpa = READ_ONCE(scb_o->riccbd) & ~0x3fUL;
 	if (gpa && (scb_s->ecb3 & ECB3_RI)) {
 		if (!(gpa & ~0x1fffUL)) {
 			rc = set_validity_icpt(scb_s, 0x0043U);
@@ -607,13 +624,14 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 			goto unpin;
 		}
 		/* Validity 0x0044 will be checked by SIE */
+		vsie_page->riccbd_gpa = gpa;
 		scb_s->riccbd = hpa;
 	}
 	if ((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
 		unsigned long sdnxc;
 
-		gpa = scb_o->sdnxo & ~0xfUL;
-		sdnxc = scb_o->sdnxo & 0xfUL;
+		gpa = READ_ONCE(scb_o->sdnxo) & ~0xfUL;
+		sdnxc = READ_ONCE(scb_o->sdnxo) & 0xfUL;
 		if (!gpa || !(gpa & ~0x1fffUL)) {
 			rc = set_validity_icpt(scb_s, 0x10b0U);
 			goto unpin;
@@ -634,6 +652,7 @@ static int pin_blocks(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 			rc = set_validity_icpt(scb_s, 0x10b0U);
 			goto unpin;
 		}
+		vsie_page->sdnx_gpa = gpa;
 		scb_s->sdnxo = hpa | sdnxc;
 	}
 	return 0;
@@ -778,7 +797,7 @@ static void retry_vsie_icpt(struct vsie_page *vsie_page)
 static int handle_stfle(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 {
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
-	__u32 fac = vsie_page->scb_o->fac & 0x7ffffff8U;
+	__u32 fac = READ_ONCE(vsie_page->scb_o->fac) & 0x7ffffff8U;
 
 	if (fac && test_kvm_facility(vcpu->kvm, 7)) {
 		retry_vsie_icpt(vsie_page);
@@ -802,6 +821,7 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 {
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;
+	int guest_bp_isolation;
 	int rc;
 
 	handle_last_fault(vcpu, vsie_page);
@@ -812,6 +832,20 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 		s390_handle_mcck();
 
 	srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
+
+	/* save current guest state of bp isolation override */
+	guest_bp_isolation = test_thread_flag(TIF_ISOLATE_BP_GUEST);
+
+	/*
+	 * The guest is running with BPBC, so we have to force it on for our
+	 * nested guest. This is done by enabling BPBC globally, so the BPBC
+	 * control in the SCB (which the nested guest can modify) is simply
+	 * ignored.
+	 */
+	if (test_kvm_facility(vcpu->kvm, 82) &&
+	    vcpu->arch.sie_block->fpf & FPF_BPBC)
+		set_thread_flag(TIF_ISOLATE_BP_GUEST);
+
 	local_irq_disable();
 	guest_enter_irqoff();
 	local_irq_enable();
@@ -821,6 +855,11 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
 	local_irq_disable();
 	guest_exit_irqoff();
 	local_irq_enable();
+
+	/* restore guest state for bp isolation override */
+	if (!guest_bp_isolation)
+		clear_thread_flag(TIF_ISOLATE_BP_GUEST);
+
 	vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 
 	if (rc == -EINTR) {
@@ -904,7 +943,7 @@ static void register_shadow_scb(struct kvm_vcpu *vcpu,
 	 * External calls have to lead to a kick of the vcpu and
 	 * therefore the vsie -> Simulate Wait state.
 	 */
-	atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT);
 	/*
 	 * We have to adjust the g3 epoch by the g2 epoch. The epoch will
 	 * automatically be adjusted on tod clock changes via kvm_sync_clock.
@@ -926,7 +965,7 @@ static void register_shadow_scb(struct kvm_vcpu *vcpu,
  */
 static void unregister_shadow_scb(struct kvm_vcpu *vcpu)
 {
-	atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags);
+	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT);
 	WRITE_ONCE(vcpu->arch.vsie_block, NULL);
 }
 
diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
index 05d459b638f5..2c55a2b9d6c6 100644
--- a/arch/s390/mm/gmap.c
+++ b/arch/s390/mm/gmap.c
@@ -815,27 +815,17 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap,
  * @ptl: pointer to the spinlock pointer
  *
  * Returns a pointer to the locked pte for a guest address, or NULL
- *
- * Note: Can also be called for shadow gmaps.
  */
 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr,
 			       spinlock_t **ptl)
 {
 	unsigned long *table;
 
-	if (gmap_is_shadow(gmap))
-		spin_lock(&gmap->guest_table_lock);
+	BUG_ON(gmap_is_shadow(gmap));
 	/* Walk the gmap page table, lock and get pte pointer */
 	table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */
-	if (!table || *table & _SEGMENT_ENTRY_INVALID) {
-		if (gmap_is_shadow(gmap))
-			spin_unlock(&gmap->guest_table_lock);
+	if (!table || *table & _SEGMENT_ENTRY_INVALID)
 		return NULL;
-	}
-	if (gmap_is_shadow(gmap)) {
-		*ptl = &gmap->guest_table_lock;
-		return pte_offset_map((pmd_t *) table, gaddr);
-	}
 	return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl);
 }
 
@@ -889,8 +879,6 @@ static void gmap_pte_op_end(spinlock_t *ptl)
  * -EFAULT if gaddr is invalid (or mapping for shadows is missing).
  *
  * Called with sg->mm->mmap_sem in read.
- *
- * Note: Can also be called for shadow gmaps.
  */
 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
 			      unsigned long len, int prot, unsigned long bits)
@@ -900,6 +888,7 @@ static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
 	pte_t *ptep;
 	int rc;
 
+	BUG_ON(gmap_is_shadow(gmap));
 	while (len) {
 		rc = -EAGAIN;
 		ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);
@@ -960,7 +949,8 @@ EXPORT_SYMBOL_GPL(gmap_mprotect_notify);
  * @val: pointer to the unsigned long value to return
  *
  * Returns 0 if the value was read, -ENOMEM if out of memory and -EFAULT
- * if reading using the virtual address failed.
+ * if reading using the virtual address failed. -EINVAL if called on a gmap
+ * shadow.
  *
  * Called with gmap->mm->mmap_sem in read.
  */
@@ -971,6 +961,9 @@ int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)
 	pte_t *ptep, pte;
 	int rc;
 
+	if (gmap_is_shadow(gmap))
+		return -EINVAL;
+
 	while (1) {
 		rc = -EAGAIN;
 		ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);
@@ -1028,18 +1021,17 @@ static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,
 }
 
 /**
- * gmap_protect_rmap - modify access rights to memory and create an rmap
+ * gmap_protect_rmap - restrict access rights to memory (RO) and create an rmap
  * @sg: pointer to the shadow guest address space structure
  * @raddr: rmap address in the shadow gmap
  * @paddr: address in the parent guest address space
  * @len: length of the memory area to protect
- * @prot: indicates access rights: none, read-only or read-write
  *
  * Returns 0 if successfully protected and the rmap was created, -ENOMEM
  * if out of memory and -EFAULT if paddr is invalid.
  */
 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
-			     unsigned long paddr, unsigned long len, int prot)
+			     unsigned long paddr, unsigned long len)
 {
 	struct gmap *parent;
 	struct gmap_rmap *rmap;
@@ -1067,7 +1059,7 @@ static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
 		ptep = gmap_pte_op_walk(parent, paddr, &ptl);
 		if (ptep) {
 			spin_lock(&sg->guest_table_lock);
-			rc = ptep_force_prot(parent->mm, paddr, ptep, prot,
+			rc = ptep_force_prot(parent->mm, paddr, ptep, PROT_READ,
 					     PGSTE_VSIE_BIT);
 			if (!rc)
 				gmap_insert_rmap(sg, vmaddr, rmap);
@@ -1077,7 +1069,7 @@ static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
 		radix_tree_preload_end();
 		if (rc) {
 			kfree(rmap);
-			rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot);
+			rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ);
 			if (rc)
 				return rc;
 			continue;
@@ -1616,7 +1608,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
 	origin = r2t & _REGION_ENTRY_ORIGIN;
 	offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
 	len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
-	rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ);
+	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
 	spin_lock(&sg->guest_table_lock);
 	if (!rc) {
 		table = gmap_table_walk(sg, saddr, 4);
@@ -1699,7 +1691,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
 	origin = r3t & _REGION_ENTRY_ORIGIN;
 	offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
 	len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
-	rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ);
+	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
 	spin_lock(&sg->guest_table_lock);
 	if (!rc) {
 		table = gmap_table_walk(sg, saddr, 3);
@@ -1783,7 +1775,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
 	origin = sgt & _REGION_ENTRY_ORIGIN;
 	offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
 	len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
-	rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ);
+	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
 	spin_lock(&sg->guest_table_lock);
 	if (!rc) {
 		table = gmap_table_walk(sg, saddr, 2);
@@ -1902,7 +1894,7 @@ int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
 	/* Make pgt read-only in parent gmap page table (not the pgste) */
 	raddr = (saddr & _SEGMENT_MASK) | _SHADOW_RMAP_SEGMENT;
 	origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK;
-	rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE, PROT_READ);
+	rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);
 	spin_lock(&sg->guest_table_lock);
 	if (!rc) {
 		table = gmap_table_walk(sg, saddr, 1);
@@ -2005,7 +1997,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_page);
  * Called with sg->parent->shadow_lock.
  */
 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
-			       unsigned long gaddr, pte_t *pte)
+			       unsigned long gaddr)
 {
 	struct gmap_rmap *rmap, *rnext, *head;
 	unsigned long start, end, bits, raddr;
@@ -2090,7 +2082,7 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,
 			spin_lock(&gmap->shadow_lock);
 			list_for_each_entry_safe(sg, next,
 						 &gmap->children, list)
-				gmap_shadow_notify(sg, vmaddr, gaddr, pte);
+				gmap_shadow_notify(sg, vmaddr, gaddr);
 			spin_unlock(&gmap->shadow_lock);
 		}
 		if (bits & PGSTE_IN_BIT)
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 671535e64aba..3fa3e5323612 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -222,7 +222,8 @@ device_initcall(s390_cma_mem_init);
 
 #endif /* CONFIG_CMA */
 
-int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
+int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
+		bool want_memblock)
 {
 	unsigned long start_pfn = PFN_DOWN(start);
 	unsigned long size_pages = PFN_DOWN(size);
@@ -232,14 +233,14 @@ int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
 	if (rc)
 		return rc;
 
-	rc = __add_pages(nid, start_pfn, size_pages, want_memblock);
+	rc = __add_pages(nid, start_pfn, size_pages, altmap, want_memblock);
 	if (rc)
 		vmem_remove_mapping(start, size);
 	return rc;
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
-int arch_remove_memory(u64 start, u64 size)
+int arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 {
 	/*
 	 * There is no hardware or firmware interface which could trigger a
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index 3316d463fc29..db55561c5981 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -211,7 +211,8 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
 /*
  * Add a backed mem_map array to the virtual mem_map array.
  */
-int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
+int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
+		struct vmem_altmap *altmap)
 {
 	unsigned long pgt_prot, sgt_prot;
 	unsigned long address = start;
@@ -296,7 +297,8 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 	return ret;
 }
 
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+		struct vmem_altmap *altmap)
 {
 }
 
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 9557d8b516df..78a19c93b380 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -28,8 +28,6 @@
 #include <asm/set_memory.h>
 #include "bpf_jit.h"
 
-int bpf_jit_enable __read_mostly;
-
 struct bpf_jit {
 	u32 seen;		/* Flags to remember seen eBPF instructions */
 	u32 seen_reg[16];	/* Array to remember which registers are used */
@@ -612,11 +610,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
 	{
 		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
 
-		jit->seen |= SEEN_RET0;
-		/* ltr %src,%src (if src == 0 goto fail) */
-		EMIT2(0x1200, src_reg, src_reg);
-		/* jz <ret0> */
-		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
 		/* lhi %w0,0 */
 		EMIT4_IMM(0xa7080000, REG_W0, 0);
 		/* lr %w1,%dst */
@@ -632,11 +625,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
 	{
 		int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
 
-		jit->seen |= SEEN_RET0;
-		/* ltgr %src,%src (if src == 0 goto fail) */
-		EMIT4(0xb9020000, src_reg, src_reg);
-		/* jz <ret0> */
-		EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
 		/* lghi %w0,0 */
 		EMIT4_IMM(0xa7090000, REG_W0, 0);
 		/* lgr %w1,%dst */
@@ -1299,7 +1287,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 	struct bpf_jit jit;
 	int pass;
 
-	if (!bpf_jit_enable)
+	if (!fp->jit_requested)
 		return orig_fp;
 
 	tmp = bpf_jit_blind_constants(fp);
diff --git a/arch/s390/tools/.gitignore b/arch/s390/tools/.gitignore
index 72a4b2cf1365..71bd6f8eebaf 100644
--- a/arch/s390/tools/.gitignore
+++ b/arch/s390/tools/.gitignore
@@ -1 +1,2 @@
 gen_facilities
+gen_opcode_table
diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile
index 2e70e25de07a..48cdac1143a9 100644
--- a/arch/s390/tools/Makefile
+++ b/arch/s390/tools/Makefile
@@ -3,22 +3,33 @@
 # Makefile for s390 specific build tools
 #
 
+kapi := arch/$(ARCH)/include/generated/asm
+kapi-hdrs-y := $(kapi)/facility-defs.h $(kapi)/dis-defs.h
+
+targets += $(addprefix ../../../,$(kapi-hdrs-y))
+PHONY += kapi
+
+kapi:	$(kapi-hdrs-y)
+
 hostprogs-y		    += gen_facilities
 hostprogs-y		    += gen_opcode_table
 
 HOSTCFLAGS_gen_facilities.o += -Wall $(LINUXINCLUDE)
 HOSTCFLAGS_gen_opcode_table.o += -Wall $(LINUXINCLUDE)
 
-define filechk_facilities.h
+# Ensure output directory exists
+_dummy := $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)')
+
+define filechk_facility-defs.h
 	$(obj)/gen_facilities
 endef
 
-define filechk_dis.h
+define filechk_dis-defs.h
 	( $(obj)/gen_opcode_table < $(srctree)/arch/$(ARCH)/tools/opcodes.txt )
 endef
 
-include/generated/facilities.h: $(obj)/gen_facilities FORCE
-	$(call filechk,facilities.h)
+$(kapi)/facility-defs.h: $(obj)/gen_facilities FORCE
+	$(call filechk,facility-defs.h)
 
-include/generated/dis.h: $(obj)/gen_opcode_table FORCE
-	$(call filechk,dis.h)
+$(kapi)/dis-defs.h: $(obj)/gen_opcode_table FORCE
+	$(call filechk,dis-defs.h)
diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c
index 0373801d9860..424a1ba4f874 100644
--- a/arch/s390/tools/gen_facilities.c
+++ b/arch/s390/tools/gen_facilities.c
@@ -128,8 +128,8 @@ static void print_facility_lists(void)
 
 int main(int argc, char **argv)
 {
-	printf("#ifndef __ASM_S390_FACILITIES__\n");
-	printf("#define __ASM_S390_FACILITIES__\n");
+	printf("#ifndef __ASM_S390_FACILITY_DEFS__\n");
+	printf("#define __ASM_S390_FACILITY_DEFS__\n");
 	printf("/*\n");
 	printf(" * DO NOT MODIFY.\n");
 	printf(" *\n");
diff --git a/arch/s390/tools/gen_opcode_table.c b/arch/s390/tools/gen_opcode_table.c
index 357d42681cef..259aa0680d1a 100644
--- a/arch/s390/tools/gen_opcode_table.c
+++ b/arch/s390/tools/gen_opcode_table.c
@@ -321,8 +321,8 @@ int main(int argc, char **argv)
 	struct gen_opcode *desc = &_desc;
 
 	read_instructions(desc);
-	printf("#ifndef __S390_GENERATED_DIS_H__\n");
-	printf("#define __S390_GENERATED_DIS_H__\n");
+	printf("#ifndef __S390_GENERATED_DIS_DEFS_H__\n");
+	printf("#define __S390_GENERATED_DIS_DEFS_H__\n");
 	printf("/*\n");
 	printf(" * DO NOT MODIFY.\n");
 	printf(" *\n");
diff --git a/arch/score/include/asm/Kbuild b/arch/score/include/asm/Kbuild
index 54b3b2039af1..1a0ee72f6a7c 100644
--- a/arch/score/include/asm/Kbuild
+++ b/arch/score/include/asm/Kbuild
@@ -1,5 +1,4 @@
 generic-y += barrier.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += extable.h
 generic-y += irq_work.h
diff --git a/arch/score/include/asm/thread_info.h b/arch/score/include/asm/thread_info.h
index ad51b56e51bd..bc4c7c90550f 100644
--- a/arch/score/include/asm/thread_info.h
+++ b/arch/score/include/asm/thread_info.h
@@ -58,9 +58,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* How to get the thread information struct from C. */
 register struct thread_info *__current_thread_info __asm__("r28");
 #define current_thread_info()	__current_thread_info
diff --git a/arch/score/include/uapi/asm/Kbuild b/arch/score/include/uapi/asm/Kbuild
index 81271d3af47c..b04fd1632051 100644
--- a/arch/score/include/uapi/asm/Kbuild
+++ b/arch/score/include/uapi/asm/Kbuild
@@ -2,4 +2,5 @@
 include include/uapi/asm-generic/Kbuild.asm
 
 generic-y	+= bpf_perf_event.h
+generic-y	+= poll.h
 generic-y	+= siginfo.h
diff --git a/arch/score/include/uapi/asm/poll.h b/arch/score/include/uapi/asm/poll.h
deleted file mode 100644
index c636b85843cd..000000000000
--- a/arch/score/include/uapi/asm/poll.h
+++ /dev/null
@@ -1,7 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _ASM_SCORE_POLL_H
-#define _ASM_SCORE_POLL_H
-
-#include <asm-generic/poll.h>
-
-#endif /* _ASM_SCORE_POLL_H */
diff --git a/arch/score/kernel/setup.c b/arch/score/kernel/setup.c
index f3a0649ab521..627416bbd0b1 100644
--- a/arch/score/kernel/setup.c
+++ b/arch/score/kernel/setup.c
@@ -124,9 +124,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 {
 	unsigned long n = (unsigned long) v - 1;
 
-	seq_printf(m, "processor\t\t: %ld\n", n);
-	seq_printf(m, "\n");
-
+	seq_printf(m, "processor\t\t: %ld\n\n", n);
 	return 0;
 }
 
diff --git a/arch/sh/boards/board-espt.c b/arch/sh/boards/board-espt.c
index 7291e2f11a47..4d6be53058d6 100644
--- a/arch/sh/boards/board-espt.c
+++ b/arch/sh/boards/board-espt.c
@@ -79,7 +79,6 @@ static struct resource sh_eth_resources[] = {
 
 static struct sh_eth_plat_data sh7763_eth_pdata = {
 	.phy = 0,
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.phy_interface = PHY_INTERFACE_MODE_MII,
 };
 
diff --git a/arch/sh/boards/board-sh7757lcr.c b/arch/sh/boards/board-sh7757lcr.c
index 0104c8199c48..1bde08dc067d 100644
--- a/arch/sh/boards/board-sh7757lcr.c
+++ b/arch/sh/boards/board-sh7757lcr.c
@@ -76,7 +76,6 @@ static struct resource sh_eth0_resources[] = {
 
 static struct sh_eth_plat_data sh7757_eth0_pdata = {
 	.phy = 1,
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.set_mdio_gate = sh7757_eth_set_mdio_gate,
 };
 
@@ -104,7 +103,6 @@ static struct resource sh_eth1_resources[] = {
 
 static struct sh_eth_plat_data sh7757_eth1_pdata = {
 	.phy = 1,
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.set_mdio_gate = sh7757_eth_set_mdio_gate,
 };
 
@@ -148,7 +146,6 @@ static struct resource sh_eth_giga0_resources[] = {
 
 static struct sh_eth_plat_data sh7757_eth_giga0_pdata = {
 	.phy = 18,
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.set_mdio_gate = sh7757_eth_giga_set_mdio_gate,
 	.phy_interface = PHY_INTERFACE_MODE_RGMII_ID,
 };
@@ -182,7 +179,6 @@ static struct resource sh_eth_giga1_resources[] = {
 
 static struct sh_eth_plat_data sh7757_eth_giga1_pdata = {
 	.phy = 19,
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.set_mdio_gate = sh7757_eth_giga_set_mdio_gate,
 	.phy_interface = PHY_INTERFACE_MODE_RGMII_ID,
 };
diff --git a/arch/sh/boards/mach-ecovec24/setup.c b/arch/sh/boards/mach-ecovec24/setup.c
index 1faf6cb93dcb..6f929abe0b50 100644
--- a/arch/sh/boards/mach-ecovec24/setup.c
+++ b/arch/sh/boards/mach-ecovec24/setup.c
@@ -159,7 +159,6 @@ static struct resource sh_eth_resources[] = {
 
 static struct sh_eth_plat_data sh_eth_plat = {
 	.phy = 0x1f, /* SMSC LAN8700 */
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.phy_interface = PHY_INTERFACE_MODE_MII,
 	.ether_link_active_low = 1
 };
diff --git a/arch/sh/boards/mach-se/7724/setup.c b/arch/sh/boards/mach-se/7724/setup.c
index f1fecd395679..255952555656 100644
--- a/arch/sh/boards/mach-se/7724/setup.c
+++ b/arch/sh/boards/mach-se/7724/setup.c
@@ -374,7 +374,6 @@ static struct resource sh_eth_resources[] = {
 
 static struct sh_eth_plat_data sh_eth_plat = {
 	.phy = 0x1f, /* SMSC LAN8187 */
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.phy_interface = PHY_INTERFACE_MODE_MII,
 };
 
diff --git a/arch/sh/boards/mach-sh7763rdp/setup.c b/arch/sh/boards/mach-sh7763rdp/setup.c
index 2c8fb04685d4..6e62686b81b1 100644
--- a/arch/sh/boards/mach-sh7763rdp/setup.c
+++ b/arch/sh/boards/mach-sh7763rdp/setup.c
@@ -87,7 +87,6 @@ static struct resource sh_eth_resources[] = {
 
 static struct sh_eth_plat_data sh7763_eth_pdata = {
 	.phy = 1,
-	.edmac_endian = EDMAC_LITTLE_ENDIAN,
 	.phy_interface = PHY_INTERFACE_MODE_MII,
 };
 
diff --git a/arch/sh/boot/dts/Makefile b/arch/sh/boot/dts/Makefile
index 715def00a436..01d0f7fb14cc 100644
--- a/arch/sh/boot/dts/Makefile
+++ b/arch/sh/boot/dts/Makefile
@@ -1 +1,3 @@
-obj-$(CONFIG_USE_BUILTIN_DTB) += $(patsubst "%",%,$(CONFIG_BUILTIN_DTB_SOURCE)).dtb.o
+ifneq ($(CONFIG_BUILTIN_DTB_SOURCE),"")
+obj-y += $(patsubst "%",%,$(CONFIG_BUILTIN_DTB_SOURCE)).dtb.o
+endif
diff --git a/arch/sh/configs/polaris_defconfig b/arch/sh/configs/polaris_defconfig
index 0a432b5f50e7..87641b7d6c4e 100644
--- a/arch/sh/configs/polaris_defconfig
+++ b/arch/sh/configs/polaris_defconfig
@@ -38,7 +38,6 @@ CONFIG_IP_MULTICAST=y
 # CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_IPV6 is not set
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/sh/drivers/push-switch.c b/arch/sh/drivers/push-switch.c
index a17181160233..762bc5619910 100644
--- a/arch/sh/drivers/push-switch.c
+++ b/arch/sh/drivers/push-switch.c
@@ -24,7 +24,7 @@ static ssize_t switch_show(struct device *dev,
 	struct push_switch_platform_info *psw_info = dev->platform_data;
 	return sprintf(buf, "%s\n", psw_info->name);
 }
-static DEVICE_ATTR(switch, S_IRUGO, switch_show, NULL);
+static DEVICE_ATTR_RO(switch);
 
 static void switch_timer(struct timer_list *t)
 {
diff --git a/arch/sh/include/asm/Kbuild b/arch/sh/include/asm/Kbuild
index 1a6f9c39feef..1efcce74997b 100644
--- a/arch/sh/include/asm/Kbuild
+++ b/arch/sh/include/asm/Kbuild
@@ -1,4 +1,3 @@
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += delay.h
 generic-y += div64.h
diff --git a/arch/sh/include/asm/thread_info.h b/arch/sh/include/asm/thread_info.h
index becb798f1b04..cf5c792bf70b 100644
--- a/arch/sh/include/asm/thread_info.h
+++ b/arch/sh/include/asm/thread_info.h
@@ -63,9 +63,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the current stack pointer from C */
 register unsigned long current_stack_pointer asm("r15") __used;
 
diff --git a/arch/sh/kernel/cpu/sh2/setup-sh7619.c b/arch/sh/kernel/cpu/sh2/setup-sh7619.c
index 95796ad00fbe..d08db08dec38 100644
--- a/arch/sh/kernel/cpu/sh2/setup-sh7619.c
+++ b/arch/sh/kernel/cpu/sh2/setup-sh7619.c
@@ -122,7 +122,6 @@ static struct platform_device scif2_device = {
 
 static struct sh_eth_plat_data eth_platform_data = {
 	.phy		= 1,
-	.edmac_endian	= EDMAC_LITTLE_ENDIAN,
 	.phy_interface	= PHY_INTERFACE_MODE_MII,
 };
 
diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c
index 2c7bdf8cb934..93522069cb15 100644
--- a/arch/sh/kernel/process_32.c
+++ b/arch/sh/kernel/process_32.c
@@ -20,7 +20,6 @@
 #include <linux/sched/task_stack.h>
 #include <linux/slab.h>
 #include <linux/elfcore.h>
-#include <linux/kallsyms.h>
 #include <linux/fs.h>
 #include <linux/ftrace.h>
 #include <linux/hw_breakpoint.h>
@@ -37,8 +36,8 @@ void show_regs(struct pt_regs * regs)
 	printk("\n");
 	show_regs_print_info(KERN_DEFAULT);
 
-	print_symbol("PC is at %s\n", instruction_pointer(regs));
-	print_symbol("PR is at %s\n", regs->pr);
+	printk("PC is at %pS\n", (void *)instruction_pointer(regs));
+	printk("PR is at %pS\n", (void *)regs->pr);
 
 	printk("PC  : %08lx SP  : %08lx SR  : %08lx ",
 	       regs->pc, regs->regs[15], regs->sr);
diff --git a/arch/sh/kernel/traps_32.c b/arch/sh/kernel/traps_32.c
index 57cff00cad17..b3770bb26211 100644
--- a/arch/sh/kernel/traps_32.c
+++ b/arch/sh/kernel/traps_32.c
@@ -609,7 +609,8 @@ asmlinkage void do_divide_error(unsigned long r4)
 		break;
 	}
 
-	force_sig_info(SIGFPE, &info, current);
+	info.si_signo = SIGFPE;
+	force_sig_info(info.si_signo, &info, current);
 }
 #endif
 
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index bf726af5f1a5..ce0bbaa7e404 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -485,20 +485,20 @@ void free_initrd_mem(unsigned long start, unsigned long end)
 #endif
 
 #ifdef CONFIG_MEMORY_HOTPLUG
-int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
+int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
+		bool want_memblock)
 {
 	unsigned long start_pfn = PFN_DOWN(start);
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 	int ret;
 
 	/* We only have ZONE_NORMAL, so this is easy.. */
-	ret = __add_pages(nid, start_pfn, nr_pages, want_memblock);
+	ret = __add_pages(nid, start_pfn, nr_pages, altmap, want_memblock);
 	if (unlikely(ret))
 		printk("%s: Failed, __add_pages() == %d\n", __func__, ret);
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(arch_add_memory);
 
 #ifdef CONFIG_NUMA
 int memory_add_physaddr_to_nid(u64 addr)
@@ -510,7 +510,7 @@ EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
 #endif
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
-int arch_remove_memory(u64 start, u64 size)
+int arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 {
 	unsigned long start_pfn = PFN_DOWN(start);
 	unsigned long nr_pages = size >> PAGE_SHIFT;
@@ -518,7 +518,7 @@ int arch_remove_memory(u64 start, u64 size)
 	int ret;
 
 	zone = page_zone(pfn_to_page(start_pfn));
-	ret = __remove_pages(zone, start_pfn, nr_pages);
+	ret = __remove_pages(zone, start_pfn, nr_pages, altmap);
 	if (unlikely(ret))
 		pr_warn("%s: Failed, __remove_pages() == %d\n", __func__,
 			ret);
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 6bf594ace663..8767e45f1b2b 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -430,6 +430,8 @@ config SPARC_LEON
 	depends on SPARC32
 	select USB_EHCI_BIG_ENDIAN_MMIO
 	select USB_EHCI_BIG_ENDIAN_DESC
+	select USB_UHCI_BIG_ENDIAN_MMIO
+	select USB_UHCI_BIG_ENDIAN_DESC
 	---help---
 	  If you say Y here if you are running on a SPARC-LEON processor.
 	  The LEON processor is a synthesizable VHDL model of the
diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c
index d1064e46efe8..8aa664638c3c 100644
--- a/arch/sparc/crypto/crc32c_glue.c
+++ b/arch/sparc/crypto/crc32c_glue.c
@@ -133,6 +133,7 @@ static struct shash_alg alg = {
 		.cra_name		=	"crc32c",
 		.cra_driver_name	=	"crc32c-sparc64",
 		.cra_priority		=	SPARC_CR_OPCODE_PRIORITY,
+		.cra_flags		=	CRYPTO_ALG_OPTIONAL_KEY,
 		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
 		.cra_ctxsize		=	sizeof(u32),
 		.cra_alignmask		=	7,
diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild
index 80ddc01f57ac..ac67828da201 100644
--- a/arch/sparc/include/asm/Kbuild
+++ b/arch/sparc/include/asm/Kbuild
@@ -1,7 +1,6 @@
 # User exported sparc header files
 
 
-generic-y += clkdev.h
 generic-y += div64.h
 generic-y += emergency-restart.h
 generic-y += exec.h
diff --git a/arch/sparc/include/asm/asm-prototypes.h b/arch/sparc/include/asm/asm-prototypes.h
index 96e0972e8dbf..4987c735ff56 100644
--- a/arch/sparc/include/asm/asm-prototypes.h
+++ b/arch/sparc/include/asm/asm-prototypes.h
@@ -6,7 +6,7 @@
 #include <asm/xor.h>
 #include <asm/checksum.h>
 #include <asm/trap_block.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <asm/atomic.h>
 #include <asm/ftrace.h>
 #include <asm/cacheflush.h>
diff --git a/arch/sparc/include/asm/bug.h b/arch/sparc/include/asm/bug.h
index 6f17528356b2..ea53e418f6c0 100644
--- a/arch/sparc/include/asm/bug.h
+++ b/arch/sparc/include/asm/bug.h
@@ -9,10 +9,14 @@
 void do_BUG(const char *file, int line);
 #define BUG() do {					\
 	do_BUG(__FILE__, __LINE__);			\
+	barrier_before_unreachable();			\
 	__builtin_trap();				\
 } while (0)
 #else
-#define BUG()		__builtin_trap()
+#define BUG() do {					\
+	barrier_before_unreachable();			\
+	__builtin_trap();				\
+} while (0)
 #endif
 
 #define HAVE_ARCH_BUG
diff --git a/arch/sparc/include/asm/compat.h b/arch/sparc/include/asm/compat.h
index fa38c78de0f0..615283e16f22 100644
--- a/arch/sparc/include/asm/compat.h
+++ b/arch/sparc/include/asm/compat.h
@@ -149,65 +149,6 @@ typedef u32		compat_old_sigset_t;
 
 typedef u32		compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-#define SI_PAD_SIZE32	(128/sizeof(int) - 3)
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[SI_PAD_SIZE32];
-
-		/* kill() */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			unsigned int _uid;		/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;		/* timer id */
-			int _overrun;			/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			compat_pid_t _pid;		/* sender's pid */
-			unsigned int _uid;		/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			compat_pid_t _pid;		/* which child */
-			unsigned int _uid;		/* sender's uid */
-			int _status;			/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */
-		struct {
-			u32 _addr; /* faulting insn/memory ref. */
-			int _trapno;
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-	} _sifields;
-} compat_siginfo_t;
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 /*
diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h
index 89a0c57aed59..ab9c6b027b75 100644
--- a/arch/sparc/include/asm/hypervisor.h
+++ b/arch/sparc/include/asm/hypervisor.h
@@ -76,6 +76,10 @@
 #define HV_ETOOMANY			15 /* Too many items specified     */
 #define HV_ECHANNEL			16 /* Invalid LDC channel          */
 #define HV_EBUSY			17 /* Resource busy                */
+#define HV_EUNAVAILABLE			23 /* Resource or operation not
+					    * currently available, but may
+					    * become available in the future
+					    */
 
 /* mach_exit()
  * TRAP:	HV_FAST_TRAP
@@ -941,6 +945,139 @@ unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr,
  */
 #define HV_FAST_MEM_SYNC		0x32
 
+/* Coprocessor services
+ *
+ * M7 and later processors provide an on-chip coprocessor which
+ * accelerates database operations, and is known internally as
+ * DAX.
+ */
+
+/* ccb_submit()
+ * TRAP:	HV_FAST_TRAP
+ * FUNCTION:	HV_CCB_SUBMIT
+ * ARG0:	address of CCB array
+ * ARG1:	size (in bytes) of CCB array being submitted
+ * ARG2:	flags
+ * ARG3:	reserved
+ * RET0:	status (success or error code)
+ * RET1:	size (in bytes) of CCB array that was accepted (might be less
+ *		than arg1)
+ * RET2:	status data
+ *		if status == ENOMAP or ENOACCESS, identifies the VA in question
+ *		if status == EUNAVAILBLE, unavailable code
+ * RET3:	reserved
+ *
+ * ERRORS:	EOK		successful submission (check size)
+ *		EWOULDBLOCK	could not finish submissions, try again
+ *		EBADALIGN	array not 64B aligned or size not 64B multiple
+ *		ENORADDR	invalid RA for array or in CCB
+ *		ENOMAP		could not translate address (see status data)
+ *		EINVAL		invalid ccb or arguments
+ *		ETOOMANY	too many ccbs with all-or-nothing flag
+ *		ENOACCESS	guest has no access to submit ccbs or address
+ *				in CCB does not have correct permissions (check
+ *				status data)
+ *		EUNAVAILABLE	ccb operation could not be performed at this
+ *				time (check status data)
+ *				Status data codes:
+ *					0 - exact CCB could not be executed
+ *					1 - CCB opcode cannot be executed
+ *					2 - CCB version cannot be executed
+ *					3 - vcpu cannot execute CCBs
+ *					4 - no CCBs can be executed
+ */
+
+#define HV_CCB_SUBMIT               0x34
+#ifndef __ASSEMBLY__
+unsigned long sun4v_ccb_submit(unsigned long ccb_buf,
+			       unsigned long len,
+			       unsigned long flags,
+			       unsigned long reserved,
+			       void *submitted_len,
+			       void *status_data);
+#endif
+
+/* flags (ARG2) */
+#define HV_CCB_QUERY_CMD		BIT(1)
+#define HV_CCB_ARG0_TYPE_REAL		0UL
+#define HV_CCB_ARG0_TYPE_PRIMARY	BIT(4)
+#define HV_CCB_ARG0_TYPE_SECONDARY	BIT(5)
+#define HV_CCB_ARG0_TYPE_NUCLEUS	GENMASK(5, 4)
+#define HV_CCB_ARG0_PRIVILEGED		BIT(6)
+#define HV_CCB_ALL_OR_NOTHING		BIT(7)
+#define HV_CCB_QUEUE_INFO		BIT(8)
+#define HV_CCB_VA_REJECT		0UL
+#define HV_CCB_VA_SECONDARY		BIT(13)
+#define HV_CCB_VA_NUCLEUS		GENMASK(13, 12)
+#define HV_CCB_VA_PRIVILEGED		BIT(14)
+#define HV_CCB_VA_READ_ADI_DISABLE	BIT(15)	/* DAX2 only */
+
+/* ccb_info()
+ * TRAP:	HV_FAST_TRAP
+ * FUNCTION:	HV_CCB_INFO
+ * ARG0:	real address of CCB completion area
+ * RET0:	status (success or error code)
+ * RET1:	info array
+ *			- RET1[0]: CCB state
+ *			- RET1[1]: dax unit
+ *			- RET1[2]: queue number
+ *			- RET1[3]: queue position
+ *
+ * ERRORS:	EOK		operation successful
+ *		EBADALIGN	address not 64B aligned
+ *		ENORADDR	RA in address not valid
+ *		EINVAL		CA not valid
+ *		EWOULDBLOCK	info not available for this CCB currently, try
+ *				again
+ *		ENOACCESS	guest cannot use dax
+ */
+
+#define HV_CCB_INFO                 0x35
+#ifndef __ASSEMBLY__
+unsigned long sun4v_ccb_info(unsigned long ca,
+			     void *info_arr);
+#endif
+
+/* info array byte offsets (RET1) */
+#define CCB_INFO_OFFSET_CCB_STATE	0
+#define CCB_INFO_OFFSET_DAX_UNIT	2
+#define CCB_INFO_OFFSET_QUEUE_NUM	4
+#define CCB_INFO_OFFSET_QUEUE_POS	6
+
+/* CCB state (RET1[0]) */
+#define HV_CCB_STATE_COMPLETED      0
+#define HV_CCB_STATE_ENQUEUED       1
+#define HV_CCB_STATE_INPROGRESS     2
+#define HV_CCB_STATE_NOTFOUND       3
+
+/* ccb_kill()
+ * TRAP:	HV_FAST_TRAP
+ * FUNCTION:	HV_CCB_KILL
+ * ARG0:	real address of CCB completion area
+ * RET0:	status (success or error code)
+ * RET1:	CCB kill status
+ *
+ * ERRORS:	EOK		operation successful
+ *		EBADALIGN	address not 64B aligned
+ *		ENORADDR	RA in address not valid
+ *		EINVAL		CA not valid
+ *		EWOULDBLOCK	kill not available for this CCB currently, try
+ *				again
+ *		ENOACCESS	guest cannot use dax
+ */
+
+#define HV_CCB_KILL                 0x36
+#ifndef __ASSEMBLY__
+unsigned long sun4v_ccb_kill(unsigned long ca,
+			     void *kill_status);
+#endif
+
+/* CCB kill status (RET1) */
+#define HV_CCB_KILL_COMPLETED       0
+#define HV_CCB_KILL_DEQUEUED        1
+#define HV_CCB_KILL_KILLED          2
+#define HV_CCB_KILL_NOTFOUND        3
+
 /* Time of day services.
  *
  * The hypervisor maintains the time of day on a per-domain basis.
@@ -3355,6 +3492,7 @@ unsigned long sun4v_m7_set_perfreg(unsigned long reg_num,
 #define HV_GRP_SDIO_ERR			0x0109
 #define HV_GRP_REBOOT_DATA		0x0110
 #define HV_GRP_ATU			0x0111
+#define HV_GRP_DAX			0x0113
 #define HV_GRP_M7_PERF			0x0114
 #define HV_GRP_NIAG_PERF		0x0200
 #define HV_GRP_FIRE_PERF		0x0201
diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index 9937c5ff94a9..339920fdf9ed 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -1010,7 +1010,7 @@ void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
 			  pmd_t *pmd);
 
 #define __HAVE_ARCH_PMDP_INVALIDATE
-extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
+extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 			    pmd_t *pmdp);
 
 #define __HAVE_ARCH_PGTABLE_DEPOSIT
diff --git a/arch/sparc/include/asm/thread_info_32.h b/arch/sparc/include/asm/thread_info_32.h
index febaaeb1a0fe..548b366165dd 100644
--- a/arch/sparc/include/asm/thread_info_32.h
+++ b/arch/sparc/include/asm/thread_info_32.h
@@ -63,9 +63,6 @@ struct thread_info {
 	.preempt_count	=	INIT_PREEMPT_COUNT,	\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 register struct thread_info *current_thread_info_reg asm("g6");
 #define current_thread_info()   (current_thread_info_reg)
diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h
index caf915321ba9..f7e7b0baec9f 100644
--- a/arch/sparc/include/asm/thread_info_64.h
+++ b/arch/sparc/include/asm/thread_info_64.h
@@ -120,9 +120,6 @@ struct thread_info {
 	.preempt_count	=	INIT_PREEMPT_COUNT,	\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 register struct thread_info *current_thread_info_reg asm("g6");
 #define current_thread_info()	(current_thread_info_reg)
diff --git a/arch/sparc/include/uapi/asm/oradax.h b/arch/sparc/include/uapi/asm/oradax.h
new file mode 100644
index 000000000000..722951908b0a
--- /dev/null
+++ b/arch/sparc/include/uapi/asm/oradax.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Oracle DAX driver API definitions
+ */
+
+#ifndef _ORADAX_H
+#define	_ORADAX_H
+
+#include <linux/types.h>
+
+#define	CCB_KILL 0
+#define	CCB_INFO 1
+#define	CCB_DEQUEUE 2
+
+struct dax_command {
+	__u16 command;		/* CCB_KILL/INFO/DEQUEUE */
+	__u16 ca_offset;	/* offset into mmapped completion area */
+};
+
+struct ccb_kill_result {
+	__u16 action;		/* action taken to kill ccb */
+};
+
+struct ccb_info_result {
+	__u16 state;		/* state of enqueued ccb */
+	__u16 inst_num;		/* dax instance number of enqueued ccb */
+	__u16 q_num;		/* queue number of enqueued ccb */
+	__u16 q_pos;		/* ccb position in queue */
+};
+
+struct ccb_exec_result {
+	__u64	status_data;	/* additional status data (e.g. bad VA) */
+	__u32	status;		/* one of DAX_SUBMIT_* */
+};
+
+union ccb_result {
+	struct ccb_exec_result exec;
+	struct ccb_info_result info;
+	struct ccb_kill_result kill;
+};
+
+#define	DAX_MMAP_LEN		(16 * 1024)
+#define	DAX_MAX_CCBS		15
+#define	DAX_CCB_BUF_MAXLEN	(DAX_MAX_CCBS * 64)
+#define	DAX_NAME		"oradax"
+
+/* CCB_EXEC status */
+#define	DAX_SUBMIT_OK			0
+#define	DAX_SUBMIT_ERR_RETRY		1
+#define	DAX_SUBMIT_ERR_WOULDBLOCK	2
+#define	DAX_SUBMIT_ERR_BUSY		3
+#define	DAX_SUBMIT_ERR_THR_INIT		4
+#define	DAX_SUBMIT_ERR_ARG_INVAL	5
+#define	DAX_SUBMIT_ERR_CCB_INVAL	6
+#define	DAX_SUBMIT_ERR_NO_CA_AVAIL	7
+#define	DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS	8
+#define	DAX_SUBMIT_ERR_NOMAP		9
+#define	DAX_SUBMIT_ERR_NOACCESS		10
+#define	DAX_SUBMIT_ERR_TOOMANY		11
+#define	DAX_SUBMIT_ERR_UNAVAIL		12
+#define	DAX_SUBMIT_ERR_INTERNAL		13
+
+/* CCB_INFO states - must match HV_CCB_STATE_* definitions */
+#define	DAX_CCB_COMPLETED	0
+#define	DAX_CCB_ENQUEUED	1
+#define	DAX_CCB_INPROGRESS	2
+#define	DAX_CCB_NOTFOUND	3
+
+/* CCB_KILL actions - must match HV_CCB_KILL_* definitions */
+#define	DAX_KILL_COMPLETED	0
+#define	DAX_KILL_DEQUEUED	1
+#define	DAX_KILL_KILLED		2
+#define	DAX_KILL_NOTFOUND	3
+
+#endif /* _ORADAX_H */
diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c
index a2bc1b2955b4..717ec7ef07f9 100644
--- a/arch/sparc/kernel/hvapi.c
+++ b/arch/sparc/kernel/hvapi.c
@@ -41,6 +41,7 @@ static struct api_info api_table[] = {
 	{ .group = HV_GRP_SDIO_ERR,				},
 	{ .group = HV_GRP_REBOOT_DATA,				},
 	{ .group = HV_GRP_ATU,		.flags = FLAG_PRE_API	},
+	{ .group = HV_GRP_DAX,					},
 	{ .group = HV_GRP_NIAG_PERF,	.flags = FLAG_PRE_API	},
 	{ .group = HV_GRP_FIRE_PERF,				},
 	{ .group = HV_GRP_N2_CPU,				},
diff --git a/arch/sparc/kernel/hvcalls.S b/arch/sparc/kernel/hvcalls.S
index bdfd3d8c6707..2f865a464576 100644
--- a/arch/sparc/kernel/hvcalls.S
+++ b/arch/sparc/kernel/hvcalls.S
@@ -871,3 +871,60 @@ ENTRY(sun4v_m7_set_perfreg)
 	retl
 	nop
 ENDPROC(sun4v_m7_set_perfreg)
+
+	/* %o0: address of CCB array
+	 * %o1: size (in bytes) of CCB array
+	 * %o2: flags
+	 * %o3: reserved
+	 *
+	 * returns:
+	 * %o0: status
+	 * %o1: size (in bytes) of the CCB array that was accepted
+	 * %o2: status data
+	 * %o3: reserved
+	 */
+ENTRY(sun4v_ccb_submit)
+	mov	%o5, %g1
+	mov	HV_CCB_SUBMIT, %o5
+	ta	HV_FAST_TRAP
+	stx	%o1, [%o4]
+	retl
+	 stx	%o2, [%g1]
+ENDPROC(sun4v_ccb_submit)
+EXPORT_SYMBOL(sun4v_ccb_submit)
+
+	/* %o0: completion area ra for the ccb to get info
+	 *
+	 * returns:
+	 * %o0: status
+	 * %o1: CCB state
+	 * %o2: position
+	 * %o3: dax unit
+	 * %o4: queue
+	 */
+ENTRY(sun4v_ccb_info)
+	mov	%o1, %g1
+	mov	HV_CCB_INFO, %o5
+	ta	HV_FAST_TRAP
+	sth	%o1, [%g1 + CCB_INFO_OFFSET_CCB_STATE]
+	sth	%o2, [%g1 + CCB_INFO_OFFSET_QUEUE_POS]
+	sth	%o3, [%g1 + CCB_INFO_OFFSET_DAX_UNIT]
+	retl
+	 sth	%o4, [%g1 + CCB_INFO_OFFSET_QUEUE_NUM]
+ENDPROC(sun4v_ccb_info)
+EXPORT_SYMBOL(sun4v_ccb_info)
+
+	/* %o0: completion area ra for the ccb to kill
+	 *
+	 * returns:
+	 * %o0: status
+	 * %o1: result of the kill
+	 */
+ENTRY(sun4v_ccb_kill)
+	mov	%o1, %g1
+	mov	HV_CCB_KILL, %o5
+	ta	HV_FAST_TRAP
+	retl
+	 sth	%o1, [%g1]
+ENDPROC(sun4v_ccb_kill)
+EXPORT_SYMBOL(sun4v_ccb_kill)
diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c
index 54a6159b9cd8..44d379db3f64 100644
--- a/arch/sparc/kernel/signal32.c
+++ b/arch/sparc/kernel/signal32.c
@@ -70,75 +70,6 @@ struct rt_signal_frame32 {
 	/* __siginfo_rwin_t * */u32 rwin_save;
 } __attribute__((aligned(8)));
 
-int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from)
-{
-	int err;
-
-	if (!access_ok(VERIFY_WRITE, to, sizeof(compat_siginfo_t)))
-		return -EFAULT;
-
-	/* If you change siginfo_t structure, please be sure
-	   this code is fixed accordingly.
-	   It should never copy any pad contained in the structure
-	   to avoid security leaks, but must copy the generic
-	   3 ints plus the relevant union member.
-	   This routine must convert siginfo from 64bit to 32bit as well
-	   at the same time.  */
-	err = __put_user(from->si_signo, &to->si_signo);
-	err |= __put_user(from->si_errno, &to->si_errno);
-	err |= __put_user(from->si_code, &to->si_code);
-	if (from->si_code < 0)
-		err |= __copy_to_user(&to->_sifields._pad, &from->_sifields._pad, SI_PAD_SIZE);
-	else {
-		switch (siginfo_layout(from->si_signo, from->si_code)) {
-		case SIL_TIMER:
-			err |= __put_user(from->si_tid, &to->si_tid);
-			err |= __put_user(from->si_overrun, &to->si_overrun);
-			err |= __put_user(from->si_int, &to->si_int);
-			break;
-		case SIL_CHLD:
-			err |= __put_user(from->si_utime, &to->si_utime);
-			err |= __put_user(from->si_stime, &to->si_stime);
-			err |= __put_user(from->si_status, &to->si_status);
-		default:
-		case SIL_KILL:
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_uid, &to->si_uid);
-			break;
-		case SIL_FAULT:
-			err |= __put_user(from->si_trapno, &to->si_trapno);
-			err |= __put_user((unsigned long)from->si_addr, &to->si_addr);
-			break;
-		case SIL_POLL:
-			err |= __put_user(from->si_band, &to->si_band);
-			err |= __put_user(from->si_fd, &to->si_fd);
-			break;
-		case SIL_RT:
-			err |= __put_user(from->si_pid, &to->si_pid);
-			err |= __put_user(from->si_uid, &to->si_uid);
-			err |= __put_user(from->si_int, &to->si_int);
-			break;
-		}
-	}
-	return err;
-}
-
-/* CAUTION: This is just a very minimalist implementation for the
- *          sake of compat_sys_rt_sigqueueinfo()
- */
-int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from)
-{
-	if (!access_ok(VERIFY_WRITE, from, sizeof(compat_siginfo_t)))
-		return -EFAULT;
-
-	if (copy_from_user(to, from, 3*sizeof(int)) ||
-	    copy_from_user(to->_sifields._pad, from->_sifields._pad,
-			   SI_PAD_SIZE))
-		return -EFAULT;
-
-	return 0;
-}
-
 /* Checks if the fp is valid.  We always build signal frames which are
  * 16-byte aligned, therefore we can always enforce that the restore
  * frame has that property as well.
diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c
index 20426a1c28f2..48366e5eb5b2 100644
--- a/arch/sparc/kernel/signal_64.c
+++ b/arch/sparc/kernel/signal_64.c
@@ -9,9 +9,6 @@
  *  Copyright (C) 1997,1998 Jakub Jelinek   (jj@sunsite.mff.cuni.cz)
  */
 
-#ifdef CONFIG_COMPAT
-#include <linux/compat.h>	/* for compat_old_sigset_t */
-#endif
 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/signal.h>
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 55ba62957e64..995f9490334d 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2628,7 +2628,7 @@ EXPORT_SYMBOL(_PAGE_CACHE);
 
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
-			       int node)
+			       int node, struct vmem_altmap *altmap)
 {
 	unsigned long pte_base;
 
@@ -2671,7 +2671,8 @@ int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
 	return 0;
 }
 
-void vmemmap_free(unsigned long start, unsigned long end)
+void vmemmap_free(unsigned long start, unsigned long end,
+		struct vmem_altmap *altmap)
 {
 }
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 4ae86bc0d35c..b5cfab711651 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -163,13 +163,10 @@ static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
 	pte_unmap(pte);
 }
 
-void set_pmd_at(struct mm_struct *mm, unsigned long addr,
-		pmd_t *pmdp, pmd_t pmd)
-{
-	pmd_t orig = *pmdp;
-
-	*pmdp = pmd;
 
+static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr,
+			   pmd_t orig, pmd_t pmd)
+{
 	if (mm == &init_mm)
 		return;
 
@@ -219,17 +216,38 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 	}
 }
 
+void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+		pmd_t *pmdp, pmd_t pmd)
+{
+	pmd_t orig = *pmdp;
+
+	*pmdp = pmd;
+	__set_pmd_acct(mm, addr, orig, pmd);
+}
+
+static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
+		unsigned long address, pmd_t *pmdp, pmd_t pmd)
+{
+	pmd_t old;
+
+	do {
+		old = *pmdp;
+	} while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
+	__set_pmd_acct(vma->vm_mm, address, old, pmd);
+
+	return old;
+}
+
 /*
  * This routine is only called when splitting a THP
  */
-void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
+pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 		     pmd_t *pmdp)
 {
-	pmd_t entry = *pmdp;
+	pmd_t old, entry;
 
-	pmd_val(entry) &= ~_PAGE_VALID;
-
-	set_pmd_at(vma->vm_mm, address, pmdp, entry);
+	entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID);
+	old = pmdp_establish(vma, address, pmdp, entry);
 	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 
 	/*
@@ -240,6 +258,8 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 	if ((pmd_val(entry) & _PAGE_PMD_HUGE) &&
 	    !is_huge_zero_page(pmd_page(entry)))
 		(vma->vm_mm)->context.thp_pte_count--;
+
+	return old;
 }
 
 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
diff --git a/arch/sparc/net/bpf_jit_comp_32.c b/arch/sparc/net/bpf_jit_comp_32.c
index 09e318eb34ee..3bd8ca95e521 100644
--- a/arch/sparc/net/bpf_jit_comp_32.c
+++ b/arch/sparc/net/bpf_jit_comp_32.c
@@ -11,8 +11,6 @@
 
 #include "bpf_jit_32.h"
 
-int bpf_jit_enable __read_mostly;
-
 static inline bool is_simm13(unsigned int value)
 {
 	return value + 0x1000 < 0x2000;
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index ff5f9cb3039a..48a25869349b 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -12,8 +12,6 @@
 
 #include "bpf_jit_64.h"
 
-int bpf_jit_enable __read_mostly;
-
 static inline bool is_simm13(unsigned int value)
 {
 	return value + 0x1000 < 0x2000;
@@ -969,31 +967,17 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 		emit_alu(MULX, src, dst, ctx);
 		break;
 	case BPF_ALU | BPF_DIV | BPF_X:
-		emit_cmp(src, G0, ctx);
-		emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
-		emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
-
 		emit_write_y(G0, ctx);
 		emit_alu(DIV, src, dst, ctx);
 		break;
-
 	case BPF_ALU64 | BPF_DIV | BPF_X:
-		emit_cmp(src, G0, ctx);
-		emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
-		emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
-
 		emit_alu(UDIVX, src, dst, ctx);
 		break;
-
 	case BPF_ALU | BPF_MOD | BPF_X: {
 		const u8 tmp = bpf2sparc[TMP_REG_1];
 
 		ctx->tmp_1_used = true;
 
-		emit_cmp(src, G0, ctx);
-		emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
-		emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
-
 		emit_write_y(G0, ctx);
 		emit_alu3(DIV, dst, src, tmp, ctx);
 		emit_alu3(MULX, tmp, src, tmp, ctx);
@@ -1005,10 +989,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx)
 
 		ctx->tmp_1_used = true;
 
-		emit_cmp(src, G0, ctx);
-		emit_branch(BE|ANNUL, ctx->idx, ctx->epilogue_offset, ctx);
-		emit_loadimm(0, bpf2sparc[BPF_REG_0], ctx);
-
 		emit_alu3(UDIVX, dst, src, tmp, ctx);
 		emit_alu3(MULX, tmp, src, tmp, ctx);
 		emit_alu3(SUB, dst, tmp, dst, ctx);
@@ -1509,17 +1489,25 @@ static void jit_fill_hole(void *area, unsigned int size)
 		*ptr++ = 0x91d02005; /* ta 5 */
 }
 
+struct sparc64_jit_data {
+	struct bpf_binary_header *header;
+	u8 *image;
+	struct jit_ctx ctx;
+};
+
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 {
 	struct bpf_prog *tmp, *orig_prog = prog;
+	struct sparc64_jit_data *jit_data;
 	struct bpf_binary_header *header;
 	bool tmp_blinded = false;
+	bool extra_pass = false;
 	struct jit_ctx ctx;
 	u32 image_size;
 	u8 *image_ptr;
 	int pass;
 
-	if (!bpf_jit_enable)
+	if (!prog->jit_requested)
 		return orig_prog;
 
 	tmp = bpf_jit_blind_constants(prog);
@@ -1533,13 +1521,31 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		prog = tmp;
 	}
 
+	jit_data = prog->aux->jit_data;
+	if (!jit_data) {
+		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+		if (!jit_data) {
+			prog = orig_prog;
+			goto out;
+		}
+		prog->aux->jit_data = jit_data;
+	}
+	if (jit_data->ctx.offset) {
+		ctx = jit_data->ctx;
+		image_ptr = jit_data->image;
+		header = jit_data->header;
+		extra_pass = true;
+		image_size = sizeof(u32) * ctx.idx;
+		goto skip_init_ctx;
+	}
+
 	memset(&ctx, 0, sizeof(ctx));
 	ctx.prog = prog;
 
 	ctx.offset = kcalloc(prog->len, sizeof(unsigned int), GFP_KERNEL);
 	if (ctx.offset == NULL) {
 		prog = orig_prog;
-		goto out;
+		goto out_off;
 	}
 
 	/* Fake pass to detect features used, and get an accurate assessment
@@ -1562,7 +1568,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	}
 
 	ctx.image = (u32 *)image_ptr;
-
+skip_init_ctx:
 	for (pass = 1; pass < 3; pass++) {
 		ctx.idx = 0;
 
@@ -1593,14 +1599,24 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 
 	bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
 
-	bpf_jit_binary_lock_ro(header);
+	if (!prog->is_func || extra_pass) {
+		bpf_jit_binary_lock_ro(header);
+	} else {
+		jit_data->ctx = ctx;
+		jit_data->image = image_ptr;
+		jit_data->header = header;
+	}
 
 	prog->bpf_func = (void *)ctx.image;
 	prog->jited = 1;
 	prog->jited_len = image_size;
 
+	if (!prog->is_func || extra_pass) {
 out_off:
-	kfree(ctx.offset);
+		kfree(ctx.offset);
+		kfree(jit_data);
+		prog->aux->jit_data = NULL;
+	}
 out:
 	if (tmp_blinded)
 		bpf_jit_prog_release_other(prog, prog == orig_prog ?
diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c
index 0a6f50098e23..f51595f861b8 100644
--- a/arch/sparc/vdso/vma.c
+++ b/arch/sparc/vdso/vma.c
@@ -251,7 +251,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
 	else
 		return map_vdso(&vdso_image_32_builtin, &vdso_mapping32);
 #else
-		return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
+	return map_vdso(&vdso_image_64_builtin, &vdso_mapping64);
 #endif
 
 }
diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig
index 02f269cfa538..ef9d403cbbe4 100644
--- a/arch/tile/Kconfig
+++ b/arch/tile/Kconfig
@@ -249,7 +249,7 @@ config HIGHMEM
 
 	  If unsure, say "true".
 
-config ZONE_DMA
+config ZONE_DMA32
 	def_bool y
 
 config IOMMU_HELPER
@@ -261,6 +261,7 @@ config NEED_SG_DMA_LENGTH
 config SWIOTLB
 	bool
 	default TILEGX
+	select DMA_DIRECT_OPS
 	select IOMMU_HELPER
 	select NEED_SG_DMA_LENGTH
 	select ARCH_HAS_DMA_SET_COHERENT_MASK
diff --git a/arch/tile/configs/tilegx_defconfig b/arch/tile/configs/tilegx_defconfig
index 9f94435cc44f..357a4c271ad4 100644
--- a/arch/tile/configs/tilegx_defconfig
+++ b/arch/tile/configs/tilegx_defconfig
@@ -159,7 +159,6 @@ CONFIG_DNS_RESOLVER=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_CONNECTOR=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_CRYPTOLOOP=m
diff --git a/arch/tile/configs/tilepro_defconfig b/arch/tile/configs/tilepro_defconfig
index 1c5bd4f8ffca..da2858755fa1 100644
--- a/arch/tile/configs/tilepro_defconfig
+++ b/arch/tile/configs/tilepro_defconfig
@@ -289,7 +289,6 @@ CONFIG_DNS_RESOLVER=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
-# CONFIG_FIRMWARE_IN_KERNEL is not set
 CONFIG_CONNECTOR=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_CRYPTOLOOP=m
diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild
index d28d2b8932c7..414dfc3a1808 100644
--- a/arch/tile/include/asm/Kbuild
+++ b/arch/tile/include/asm/Kbuild
@@ -1,6 +1,5 @@
 generic-y += bug.h
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += extable.h
diff --git a/arch/tile/include/asm/compat.h b/arch/tile/include/asm/compat.h
index 62a7b83025dd..769ff6ac0bf5 100644
--- a/arch/tile/include/asm/compat.h
+++ b/arch/tile/include/asm/compat.h
@@ -110,68 +110,6 @@ struct compat_flock64 {
 
 typedef u32               compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-#define COMPAT_SI_PAD_SIZE	(128/sizeof(int) - 3)
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[COMPAT_SI_PAD_SIZE];
-
-		/* kill() */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-			int _overrun_incr;	/* amount to add to overrun */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			unsigned int _pid;	/* which child */
-			unsigned int _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			unsigned int _addr;	/* faulting insn/memory ref. */
-#ifdef __ARCH_SI_TRAPNO
-			int _trapno;	/* TRAP # which caused the signal */
-#endif
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-	} _sifields;
-} compat_siginfo_t;
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 struct compat_ipc64_perm {
diff --git a/arch/tile/include/asm/dma-mapping.h b/arch/tile/include/asm/dma-mapping.h
index 97ad62878290..d25fce101fc0 100644
--- a/arch/tile/include/asm/dma-mapping.h
+++ b/arch/tile/include/asm/dma-mapping.h
@@ -44,26 +44,6 @@ static inline void set_dma_offset(struct device *dev, dma_addr_t off)
 	dev->archdata.dma_offset = off;
 }
 
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return paddr;
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
-{
-	return daddr;
-}
-
-static inline void dma_mark_clean(void *addr, size_t size) {}
-
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return 0;
-
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
 #define HAVE_ARCH_DMA_SET_MASK 1
 int dma_set_mask(struct device *dev, u64 mask);
 
diff --git a/arch/tile/include/asm/thread_info.h b/arch/tile/include/asm/thread_info.h
index b7659b8f1117..2adcacd85749 100644
--- a/arch/tile/include/asm/thread_info.h
+++ b/arch/tile/include/asm/thread_info.h
@@ -59,9 +59,6 @@ struct thread_info {
 	.align_ctl	= 0,			\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 #endif /* !__ASSEMBLY__ */
 
 #if PAGE_SIZE < 8192
diff --git a/arch/tile/include/uapi/asm/siginfo.h b/arch/tile/include/uapi/asm/siginfo.h
index f234d24fff55..a812fcbf4267 100644
--- a/arch/tile/include/uapi/asm/siginfo.h
+++ b/arch/tile/include/uapi/asm/siginfo.h
@@ -24,12 +24,4 @@
 
 #include <asm-generic/siginfo.h>
 
-/*
- * Additional Tile-specific SIGILL si_codes
- */
-#define ILL_DBLFLT	9	/* double fault */
-#define ILL_HARDWALL	10	/* user networks hardwall violation */
-#undef NSIGILL
-#define NSIGILL		10
-
 #endif /* _ASM_TILE_SIGINFO_H */
diff --git a/arch/tile/kernel/compat_signal.c b/arch/tile/kernel/compat_signal.c
index 971d87a1d8cf..a703bd0e0488 100644
--- a/arch/tile/kernel/compat_signal.c
+++ b/arch/tile/kernel/compat_signal.c
@@ -50,79 +50,6 @@ struct compat_rt_sigframe {
 	struct compat_ucontext uc;
 };
 
-int copy_siginfo_to_user32(struct compat_siginfo __user *to, const siginfo_t *from)
-{
-	int err;
-
-	if (!access_ok(VERIFY_WRITE, to, sizeof(struct compat_siginfo)))
-		return -EFAULT;
-
-	/* If you change siginfo_t structure, please make sure that
-	   this code is fixed accordingly.
-	   It should never copy any pad contained in the structure
-	   to avoid security leaks, but must copy the generic
-	   3 ints plus the relevant union member.  */
-	err = __put_user(from->si_signo, &to->si_signo);
-	err |= __put_user(from->si_errno, &to->si_errno);
-	err |= __put_user(from->si_code, &to->si_code);
-
-	if (from->si_code < 0) {
-		err |= __put_user(from->si_pid, &to->si_pid);
-		err |= __put_user(from->si_uid, &to->si_uid);
-		err |= __put_user(from->si_int, &to->si_int);
-	} else {
-		/*
-		 * First 32bits of unions are always present:
-		 * si_pid === si_band === si_tid === si_addr(LS half)
-		 */
-		err |= __put_user(from->_sifields._pad[0],
-				  &to->_sifields._pad[0]);
-		switch (siginfo_layout(from->si_signo, from->si_code)) {
-		case SIL_FAULT:
-			break;
-		case SIL_CHLD:
-			err |= __put_user(from->si_utime, &to->si_utime);
-			err |= __put_user(from->si_stime, &to->si_stime);
-			err |= __put_user(from->si_status, &to->si_status);
-			/* FALL THROUGH */
-		default:
-		case SIL_KILL:
-			err |= __put_user(from->si_uid, &to->si_uid);
-			break;
-		case SIL_POLL:
-			err |= __put_user(from->si_fd, &to->si_fd);
-			break;
-		case SIL_TIMER:
-			err |= __put_user(from->si_overrun, &to->si_overrun);
-			err |= __put_user(from->si_int, &to->si_int);
-			break;
-		case SIL_RT:
-			err |= __put_user(from->si_uid, &to->si_uid);
-			err |= __put_user(from->si_int, &to->si_int);
-			break;
-		}
-	}
-	return err;
-}
-
-int copy_siginfo_from_user32(siginfo_t *to, struct compat_siginfo __user *from)
-{
-	int err;
-
-	if (!access_ok(VERIFY_READ, from, sizeof(struct compat_siginfo)))
-		return -EFAULT;
-
-	err = __get_user(to->si_signo, &from->si_signo);
-	err |= __get_user(to->si_errno, &from->si_errno);
-	err |= __get_user(to->si_code, &from->si_code);
-
-	err |= __get_user(to->si_pid, &from->si_pid);
-	err |= __get_user(to->si_uid, &from->si_uid);
-	err |= __get_user(to->si_int, &from->si_int);
-
-	return err;
-}
-
 /* The assembly shim for this function arranges to ignore the return value. */
 long compat_sys_rt_sigreturn(void)
 {
diff --git a/arch/tile/kernel/pci-dma.c b/arch/tile/kernel/pci-dma.c
index f2abedc8a080..6a1efe5543fa 100644
--- a/arch/tile/kernel/pci-dma.c
+++ b/arch/tile/kernel/pci-dma.c
@@ -54,7 +54,7 @@ static void *tile_dma_alloc_coherent(struct device *dev, size_t size,
 	 * which case we will return NULL.  But such devices are uncommon.
 	 */
 	if (dma_mask <= DMA_BIT_MASK(32)) {
-		gfp |= GFP_DMA;
+		gfp |= GFP_DMA32;
 		node = 0;
 	}
 
@@ -509,39 +509,9 @@ EXPORT_SYMBOL(gx_pci_dma_map_ops);
 /* PCI DMA mapping functions for legacy PCI devices */
 
 #ifdef CONFIG_SWIOTLB
-static void *tile_swiotlb_alloc_coherent(struct device *dev, size_t size,
-					 dma_addr_t *dma_handle, gfp_t gfp,
-					 unsigned long attrs)
-{
-	gfp |= GFP_DMA;
-	return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
-}
-
-static void tile_swiotlb_free_coherent(struct device *dev, size_t size,
-				       void *vaddr, dma_addr_t dma_addr,
-				       unsigned long attrs)
-{
-	swiotlb_free_coherent(dev, size, vaddr, dma_addr);
-}
-
-static const struct dma_map_ops pci_swiotlb_dma_ops = {
-	.alloc = tile_swiotlb_alloc_coherent,
-	.free = tile_swiotlb_free_coherent,
-	.map_page = swiotlb_map_page,
-	.unmap_page = swiotlb_unmap_page,
-	.map_sg = swiotlb_map_sg_attrs,
-	.unmap_sg = swiotlb_unmap_sg_attrs,
-	.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
-	.sync_single_for_device = swiotlb_sync_single_for_device,
-	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
-	.sync_sg_for_device = swiotlb_sync_sg_for_device,
-	.dma_supported = swiotlb_dma_supported,
-	.mapping_error = swiotlb_dma_mapping_error,
-};
-
 static const struct dma_map_ops pci_hybrid_dma_ops = {
-	.alloc = tile_swiotlb_alloc_coherent,
-	.free = tile_swiotlb_free_coherent,
+	.alloc = swiotlb_alloc,
+	.free = swiotlb_free,
 	.map_page = tile_pci_dma_map_page,
 	.unmap_page = tile_pci_dma_unmap_page,
 	.map_sg = tile_pci_dma_map_sg,
@@ -552,7 +522,7 @@ static const struct dma_map_ops pci_hybrid_dma_ops = {
 	.sync_sg_for_device = tile_pci_dma_sync_sg_for_device,
 };
 
-const struct dma_map_ops *gx_legacy_pci_dma_map_ops = &pci_swiotlb_dma_ops;
+const struct dma_map_ops *gx_legacy_pci_dma_map_ops = &swiotlb_dma_ops;
 const struct dma_map_ops *gx_hybrid_pci_dma_map_ops = &pci_hybrid_dma_ops;
 #else
 const struct dma_map_ops *gx_legacy_pci_dma_map_ops;
diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c
index ad83c1e66dbd..eb4e198f6f93 100644
--- a/arch/tile/kernel/setup.c
+++ b/arch/tile/kernel/setup.c
@@ -814,11 +814,11 @@ static void __init zone_sizes_init(void)
 #endif
 
 		if (start < dma_end) {
-			zones_size[ZONE_DMA] = min(zones_size[ZONE_NORMAL],
+			zones_size[ZONE_DMA32] = min(zones_size[ZONE_NORMAL],
 						   dma_end - start);
-			zones_size[ZONE_NORMAL] -= zones_size[ZONE_DMA];
+			zones_size[ZONE_NORMAL] -= zones_size[ZONE_DMA32];
 		} else {
-			zones_size[ZONE_DMA] = 0;
+			zones_size[ZONE_DMA32] = 0;
 		}
 
 		/* Take zone metadata from controller 0 if we're isolnode. */
@@ -830,7 +830,7 @@ static void __init zone_sizes_init(void)
 		       PFN_UP(node_percpu[i]));
 
 		/* Track the type of memory on each node */
-		if (zones_size[ZONE_NORMAL] || zones_size[ZONE_DMA])
+		if (zones_size[ZONE_NORMAL] || zones_size[ZONE_DMA32])
 			node_set_state(i, N_NORMAL_MEMORY);
 #ifdef CONFIG_HIGHMEM
 		if (end != start)
diff --git a/arch/tile/kernel/single_step.c b/arch/tile/kernel/single_step.c
index de3eae813e52..479d8033a801 100644
--- a/arch/tile/kernel/single_step.c
+++ b/arch/tile/kernel/single_step.c
@@ -163,11 +163,13 @@ static tilepro_bundle_bits rewrite_load_store_unaligned(
 	 * actual bad address in an SPR, which it doesn't.
 	 */
 	if (align_ctl == 0) {
-		siginfo_t info = {
-			.si_signo = SIGBUS,
-			.si_code = BUS_ADRALN,
-			.si_addr = addr
-		};
+		siginfo_t info;
+
+		clear_siginfo(&info);
+		info.si_signo = SIGBUS;
+		info.si_code = BUS_ADRALN;
+		info.si_addr = addr;
+
 		trace_unhandled_signal("unaligned trap", regs,
 				       (unsigned long)addr, SIGBUS);
 		force_sig_info(info.si_signo, &info, current);
@@ -210,11 +212,13 @@ static tilepro_bundle_bits rewrite_load_store_unaligned(
 	}
 
 	if (err) {
-		siginfo_t info = {
-			.si_signo = SIGBUS,
-			.si_code = BUS_ADRALN,
-			.si_addr = addr
-		};
+		siginfo_t info;
+
+		clear_siginfo(&info);
+		info.si_signo = SIGBUS;
+		info.si_code = BUS_ADRALN;
+		info.si_addr = addr;
+
 		trace_unhandled_signal("bad address for unaligned fixup", regs,
 				       (unsigned long)addr, SIGBUS);
 		force_sig_info(info.si_signo, &info, current);
diff --git a/arch/tile/kernel/sysfs.c b/arch/tile/kernel/sysfs.c
index 825867c53853..b09456a3d77a 100644
--- a/arch/tile/kernel/sysfs.c
+++ b/arch/tile/kernel/sysfs.c
@@ -38,7 +38,7 @@ static ssize_t chip_width_show(struct device *dev,
 {
 	return sprintf(page, "%u\n", smp_width);
 }
-static DEVICE_ATTR(chip_width, 0444, chip_width_show, NULL);
+static DEVICE_ATTR_RO(chip_width);
 
 static ssize_t chip_height_show(struct device *dev,
 				struct device_attribute *attr,
@@ -46,7 +46,7 @@ static ssize_t chip_height_show(struct device *dev,
 {
 	return sprintf(page, "%u\n", smp_height);
 }
-static DEVICE_ATTR(chip_height, 0444, chip_height_show, NULL);
+static DEVICE_ATTR_RO(chip_height);
 
 static ssize_t chip_serial_show(struct device *dev,
 				struct device_attribute *attr,
@@ -54,7 +54,7 @@ static ssize_t chip_serial_show(struct device *dev,
 {
 	return get_hv_confstr(page, HV_CONFSTR_CHIP_SERIAL_NUM);
 }
-static DEVICE_ATTR(chip_serial, 0444, chip_serial_show, NULL);
+static DEVICE_ATTR_RO(chip_serial);
 
 static ssize_t chip_revision_show(struct device *dev,
 				  struct device_attribute *attr,
@@ -62,7 +62,7 @@ static ssize_t chip_revision_show(struct device *dev,
 {
 	return get_hv_confstr(page, HV_CONFSTR_CHIP_REV);
 }
-static DEVICE_ATTR(chip_revision, 0444, chip_revision_show, NULL);
+static DEVICE_ATTR_RO(chip_revision);
 
 
 static ssize_t type_show(struct device *dev,
@@ -71,7 +71,7 @@ static ssize_t type_show(struct device *dev,
 {
 	return sprintf(page, "tilera\n");
 }
-static DEVICE_ATTR(type, 0444, type_show, NULL);
+static DEVICE_ATTR_RO(type);
 
 #define HV_CONF_ATTR(name, conf)					\
 	static ssize_t name ## _show(struct device *dev,		\
@@ -184,7 +184,7 @@ static ssize_t hv_stats_store(struct device *dev,
 	return n < 0 ? n : count;
 }
 
-static DEVICE_ATTR(hv_stats, 0644, hv_stats_show, hv_stats_store);
+static DEVICE_ATTR_RW(hv_stats);
 
 static int hv_stats_device_add(struct device *dev, struct subsys_interface *sif)
 {
diff --git a/arch/tile/kernel/traps.c b/arch/tile/kernel/traps.c
index 9b08c6055f15..83a7186198d7 100644
--- a/arch/tile/kernel/traps.c
+++ b/arch/tile/kernel/traps.c
@@ -256,12 +256,14 @@ static int do_bpt(struct pt_regs *regs)
 void __kprobes do_trap(struct pt_regs *regs, int fault_num,
 		       unsigned long reason)
 {
-	siginfo_t info = { 0 };
+	siginfo_t info;
 	int signo, code;
 	unsigned long address = 0;
 	tile_bundle_bits instr;
 	int is_kernel = !user_mode(regs);
 
+	clear_siginfo(&info);
+
 	/* Handle breakpoints, etc. */
 	if (is_kernel && fault_num == INT_ILL && do_bpt(regs))
 		return;
diff --git a/arch/tile/kernel/unaligned.c b/arch/tile/kernel/unaligned.c
index 8149c38f67b6..77a0b6b6a2a1 100644
--- a/arch/tile/kernel/unaligned.c
+++ b/arch/tile/kernel/unaligned.c
@@ -980,11 +980,13 @@ void jit_bundle_gen(struct pt_regs *regs, tilegx_bundle_bits bundle,
 	}
 
 	if ((align_ctl == 0) || unexpected) {
-		siginfo_t info = {
-			.si_signo = SIGBUS,
-			.si_code = BUS_ADRALN,
-			.si_addr = (unsigned char __user *)0
-		};
+		siginfo_t info;
+
+		clear_siginfo(&info);
+		info.si_signo = SIGBUS;
+		info.si_code = BUS_ADRALN;
+		info.si_addr = (unsigned char __user *)0;
+
 		if (unaligned_printk)
 			pr_info("Unalign bundle: unexp @%llx, %llx\n",
 				(unsigned long long)regs->pc,
@@ -1396,11 +1398,12 @@ void jit_bundle_gen(struct pt_regs *regs, tilegx_bundle_bits bundle,
 				      &frag, sizeof(frag));
 		if (status) {
 			/* Fail to copy JIT into user land. send SIGSEGV. */
-			siginfo_t info = {
-				.si_signo = SIGSEGV,
-				.si_code = SEGV_MAPERR,
-				.si_addr = (void __user *)&jit_code_area[idx]
-			};
+			siginfo_t info;
+
+			clear_siginfo(&info);
+			info.si_signo = SIGSEGV;
+			info.si_code = SEGV_MAPERR;
+			info.si_addr = (void __user *)&jit_code_area[idx];
 
 			pr_warn("Unalign fixup: pid=%d %s jit_code_area=%llx\n",
 				current->pid, current->comm,
@@ -1511,11 +1514,12 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
 	 * If so, we will trigger SIGBUS.
 	 */
 	if ((regs->sp & 0x7) || (regs->ex1) || (align_ctl < 0)) {
-		siginfo_t info = {
-			.si_signo = SIGBUS,
-			.si_code = BUS_ADRALN,
-			.si_addr = (unsigned char __user *)0
-		};
+		siginfo_t info;
+
+		clear_siginfo(&info);
+		info.si_signo = SIGBUS;
+		info.si_code = BUS_ADRALN;
+		info.si_addr = (unsigned char __user *)0;
 
 		if (unaligned_printk)
 			pr_info("Unalign fixup: %d %llx @%llx\n",
@@ -1535,11 +1539,13 @@ void do_unaligned(struct pt_regs *regs, int vecnum)
 	pc = (tilegx_bundle_bits __user *)(regs->pc);
 	if (get_user(bundle, pc) != 0) {
 		/* Probably never be here since pc is valid user address.*/
-		siginfo_t info = {
-			.si_signo = SIGSEGV,
-			.si_code = SEGV_MAPERR,
-			.si_addr = (void __user *)pc
-		};
+		siginfo_t info;
+
+		clear_siginfo(&info);
+		info.si_signo = SIGSEGV;
+		info.si_code = SEGV_MAPERR;
+		info.si_addr = (void __user *)pc;
+
 		pr_err("Couldn't read instruction at %p trying to step\n", pc);
 		trace_unhandled_signal("segfault in unalign fixup", regs,
 				       (unsigned long)info.si_addr, SIGSEGV);
diff --git a/arch/um/Makefile b/arch/um/Makefile
index b76fcce397a1..e54dda8a0363 100644
--- a/arch/um/Makefile
+++ b/arch/um/Makefile
@@ -116,8 +116,15 @@ endef
 KBUILD_KCONFIG := $(HOST_DIR)/um/Kconfig
 
 archheaders:
+	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.asm-generic \
+	            kbuild-file=$(HOST_DIR)/include/asm/Kbuild \
+		    obj=$(HOST_DIR)/include/generated/asm
+	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.asm-generic \
+	            kbuild-file=$(HOST_DIR)/include/uapi/asm/Kbuild \
+		    obj=$(HOST_DIR)/include/generated/uapi/asm
 	$(Q)$(MAKE) KBUILD_SRC= ARCH=$(HEADER_ARCH) archheaders
 
+
 archprepare: include/generated/user_constants.h
 
 LINK-$(CONFIG_LD_SCRIPT_STATIC) += -static
diff --git a/arch/um/drivers/hostaudio_kern.c b/arch/um/drivers/hostaudio_kern.c
index 12bdb5996bf5..7f9dbdbc4eb7 100644
--- a/arch/um/drivers/hostaudio_kern.c
+++ b/arch/um/drivers/hostaudio_kern.c
@@ -119,10 +119,10 @@ static ssize_t hostaudio_write(struct file *file, const char __user *buffer,
 	return err;
 }
 
-static unsigned int hostaudio_poll(struct file *file,
-				   struct poll_table_struct *wait)
+static __poll_t hostaudio_poll(struct file *file,
+				struct poll_table_struct *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 #ifdef DEBUG
 	printk(KERN_DEBUG "hostaudio: poll called (unimplemented)\n");
diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c
index c4d162a94be9..d5f9a2d1da1b 100644
--- a/arch/um/drivers/mconsole_kern.c
+++ b/arch/um/drivers/mconsole_kern.c
@@ -130,6 +130,7 @@ void mconsole_proc(struct mc_request *req)
 	struct file *file;
 	int first_chunk = 1;
 	char *ptr = req->request.data;
+	loff_t pos = 0;
 
 	ptr += strlen("proc");
 	ptr = skip_spaces(ptr);
@@ -148,7 +149,7 @@ void mconsole_proc(struct mc_request *req)
 	}
 
 	do {
-		len = kernel_read(file, buf, PAGE_SIZE - 1, &file->f_pos);
+		len = kernel_read(file, buf, PAGE_SIZE - 1, &pos);
 		if (len < 0) {
 			mconsole_reply(req, "Read of file failed", 1, 0);
 			goto out_free;
diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild
index 73c57f614c9e..bb5a196c3061 100644
--- a/arch/um/include/asm/Kbuild
+++ b/arch/um/include/asm/Kbuild
@@ -1,7 +1,6 @@
 generic-y += barrier.h
 generic-y += bpf_perf_event.h
 generic-y += bug.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += delay.h
 generic-y += device.h
diff --git a/arch/um/include/asm/processor-generic.h b/arch/um/include/asm/processor-generic.h
index 86942a492454..b58b746d3f2c 100644
--- a/arch/um/include/asm/processor-generic.h
+++ b/arch/um/include/asm/processor-generic.h
@@ -58,7 +58,10 @@ static inline void release_thread(struct task_struct *task)
 {
 }
 
-#define init_stack	(init_thread_union.stack)
+static inline void mm_copy_segments(struct mm_struct *from_mm,
+				    struct mm_struct *new_mm)
+{
+}
 
 /*
  * User space process size: 3GB (default).
diff --git a/arch/um/include/asm/thread_info.h b/arch/um/include/asm/thread_info.h
index 9300f7630d2a..4eecd960ee8c 100644
--- a/arch/um/include/asm/thread_info.h
+++ b/arch/um/include/asm/thread_info.h
@@ -6,6 +6,9 @@
 #ifndef __UM_THREAD_INFO_H
 #define __UM_THREAD_INFO_H
 
+#define THREAD_SIZE_ORDER CONFIG_KERNEL_STACK_ORDER
+#define THREAD_SIZE ((1 << CONFIG_KERNEL_STACK_ORDER) * PAGE_SIZE)
+
 #ifndef __ASSEMBLY__
 
 #include <asm/types.h>
@@ -37,10 +40,6 @@ struct thread_info {
 	.real_thread = NULL,			\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
-#define THREAD_SIZE ((1 << CONFIG_KERNEL_STACK_ORDER) * PAGE_SIZE)
 /* how to get the thread information struct from C */
 static inline struct thread_info *current_thread_info(void)
 {
@@ -53,8 +52,6 @@ static inline struct thread_info *current_thread_info(void)
 	return ti;
 }
 
-#define THREAD_SIZE_ORDER CONFIG_KERNEL_STACK_ORDER
-
 #endif
 
 #define TIF_SYSCALL_TRACE	0	/* syscall trace active */
diff --git a/arch/um/include/asm/vmlinux.lds.h b/arch/um/include/asm/vmlinux.lds.h
new file mode 100644
index 000000000000..149494ae78ea
--- /dev/null
+++ b/arch/um/include/asm/vmlinux.lds.h
@@ -0,0 +1,2 @@
+#include <asm/thread_info.h>
+#include <asm-generic/vmlinux.lds.h>
diff --git a/arch/um/kernel/dyn.lds.S b/arch/um/kernel/dyn.lds.S
index d417e3899700..5568cf882371 100644
--- a/arch/um/kernel/dyn.lds.S
+++ b/arch/um/kernel/dyn.lds.S
@@ -1,5 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#include <asm-generic/vmlinux.lds.h>
+#include <asm/vmlinux.lds.h>
 #include <asm/page.h>
 
 OUTPUT_FORMAT(ELF_FORMAT)
diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
index 428644175956..b2b02df9896e 100644
--- a/arch/um/kernel/trap.c
+++ b/arch/um/kernel/trap.c
@@ -306,7 +306,7 @@ void relay_signal(int sig, struct siginfo *si, struct uml_pt_regs *regs)
 
 	arch_examine_signal(sig, regs);
 
-	memset(&clean_si, 0, sizeof(clean_si));
+	clear_siginfo(&clean_si);
 	clean_si.si_signo = si->si_signo;
 	clean_si.si_errno = si->si_errno;
 	clean_si.si_code = si->si_code;
diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c
index f433690b9b37..a818ccef30ca 100644
--- a/arch/um/kernel/um_arch.c
+++ b/arch/um/kernel/um_arch.c
@@ -54,7 +54,7 @@ struct cpuinfo_um boot_cpu_data = {
 
 union thread_union cpu0_irqstack
 	__attribute__((__section__(".data..init_irqstack"))) =
-		{ INIT_THREAD_INFO(init_task) };
+		{ .thread_info = INIT_THREAD_INFO(init_task) };
 
 /* Changed in setup_arch, which is called in early boot */
 static char host_info[(__NEW_UTS_LEN + 1) * 5];
diff --git a/arch/um/kernel/uml.lds.S b/arch/um/kernel/uml.lds.S
index 3d6ed6ba5b78..36b07ec09742 100644
--- a/arch/um/kernel/uml.lds.S
+++ b/arch/um/kernel/uml.lds.S
@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-#include <asm-generic/vmlinux.lds.h>
+#include <asm/vmlinux.lds.h>
 #include <asm/page.h>
 
 OUTPUT_FORMAT(ELF_FORMAT)
diff --git a/arch/unicore32/include/asm/Kbuild b/arch/unicore32/include/asm/Kbuild
index fda7e2153086..6f70c76c81fc 100644
--- a/arch/unicore32/include/asm/Kbuild
+++ b/arch/unicore32/include/asm/Kbuild
@@ -1,6 +1,5 @@
 generic-y += atomic.h
 generic-y += bugs.h
-generic-y += clkdev.h
 generic-y += current.h
 generic-y += device.h
 generic-y += div64.h
diff --git a/arch/unicore32/include/asm/bitops.h b/arch/unicore32/include/asm/bitops.h
index 401f597bc38c..c0cbdbe17168 100644
--- a/arch/unicore32/include/asm/bitops.h
+++ b/arch/unicore32/include/asm/bitops.h
@@ -44,4 +44,6 @@ static inline int fls(int x)
 #define find_first_bit		find_first_bit
 #define find_first_zero_bit	find_first_zero_bit
 
+#include <asm-generic/bitops/find.h>
+
 #endif /* __UNICORE_BITOPS_H__ */
diff --git a/arch/unicore32/include/asm/dma-mapping.h b/arch/unicore32/include/asm/dma-mapping.h
index ac608c2f6af6..790bc2ef4af2 100644
--- a/arch/unicore32/include/asm/dma-mapping.h
+++ b/arch/unicore32/include/asm/dma-mapping.h
@@ -12,38 +12,11 @@
 #ifndef __UNICORE_DMA_MAPPING_H__
 #define __UNICORE_DMA_MAPPING_H__
 
-#ifdef __KERNEL__
-
-#include <linux/mm_types.h>
-#include <linux/scatterlist.h>
 #include <linux/swiotlb.h>
 
-extern const struct dma_map_ops swiotlb_dma_map_ops;
-
 static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 {
-	return &swiotlb_dma_map_ops;
+	return &swiotlb_dma_ops;
 }
 
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (dev && dev->dma_mask)
-		return addr + size - 1 <= *dev->dma_mask;
-
-	return 1;
-}
-
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return paddr;
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
-{
-	return daddr;
-}
-
-static inline void dma_mark_clean(void *addr, size_t size) {}
-
-#endif /* __KERNEL__ */
 #endif
diff --git a/arch/unicore32/include/asm/thread_info.h b/arch/unicore32/include/asm/thread_info.h
index e79ad6d5b5b2..5fb728f3b49a 100644
--- a/arch/unicore32/include/asm/thread_info.h
+++ b/arch/unicore32/include/asm/thread_info.h
@@ -87,9 +87,6 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,					\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /*
  * how to get the thread information struct from C
  */
diff --git a/arch/unicore32/kernel/process.c b/arch/unicore32/kernel/process.c
index ddaf78ae6854..2bc10b8e9cf4 100644
--- a/arch/unicore32/kernel/process.c
+++ b/arch/unicore32/kernel/process.c
@@ -23,7 +23,6 @@
 #include <linux/delay.h>
 #include <linux/reboot.h>
 #include <linux/interrupt.h>
-#include <linux/kallsyms.h>
 #include <linux/init.h>
 #include <linux/cpu.h>
 #include <linux/elfcore.h>
@@ -139,8 +138,8 @@ void __show_regs(struct pt_regs *regs)
 	char buf[64];
 
 	show_regs_print_info(KERN_DEFAULT);
-	print_symbol("PC is at %s\n", instruction_pointer(regs));
-	print_symbol("LR is at %s\n", regs->UCreg_lr);
+	printk("PC is at %pS\n", (void *)instruction_pointer(regs));
+	printk("LR is at %pS\n", (void *)regs->UCreg_lr);
 	printk(KERN_DEFAULT "pc : [<%08lx>]    lr : [<%08lx>]    psr: %08lx\n"
 	       "sp : %08lx  ip : %08lx  fp : %08lx\n",
 		regs->UCreg_pc, regs->UCreg_lr, regs->UCreg_asr,
diff --git a/arch/unicore32/mm/Kconfig b/arch/unicore32/mm/Kconfig
index c256460cd363..e9154a59d561 100644
--- a/arch/unicore32/mm/Kconfig
+++ b/arch/unicore32/mm/Kconfig
@@ -42,6 +42,7 @@ config CPU_TLB_SINGLE_ENTRY_DISABLE
 
 config SWIOTLB
 	def_bool y
+	select DMA_DIRECT_OPS
 
 config IOMMU_HELPER
 	def_bool SWIOTLB
diff --git a/arch/unicore32/mm/Makefile b/arch/unicore32/mm/Makefile
index 681c0ef5ec9e..8106260583ab 100644
--- a/arch/unicore32/mm/Makefile
+++ b/arch/unicore32/mm/Makefile
@@ -6,8 +6,6 @@
 obj-y				:= extable.o fault.o init.o pgd.o mmu.o
 obj-y				+= flush.o ioremap.o
 
-obj-$(CONFIG_SWIOTLB)		+= dma-swiotlb.o
-
 obj-$(CONFIG_MODULES)		+= proc-syms.o
 
 obj-$(CONFIG_ALIGNMENT_TRAP)	+= alignment.o
diff --git a/arch/unicore32/mm/dma-swiotlb.c b/arch/unicore32/mm/dma-swiotlb.c
deleted file mode 100644
index 525413d6690e..000000000000
--- a/arch/unicore32/mm/dma-swiotlb.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Contains routines needed to support swiotlb for UniCore32.
- *
- * Copyright (C) 2010 Guan Xuetao
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-#include <linux/pci.h>
-#include <linux/cache.h>
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/swiotlb.h>
-#include <linux/bootmem.h>
-
-#include <asm/dma.h>
-
-static void *unicore_swiotlb_alloc_coherent(struct device *dev, size_t size,
-					    dma_addr_t *dma_handle, gfp_t flags,
-					    unsigned long attrs)
-{
-	return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
-}
-
-static void unicore_swiotlb_free_coherent(struct device *dev, size_t size,
-					  void *vaddr, dma_addr_t dma_addr,
-					  unsigned long attrs)
-{
-	swiotlb_free_coherent(dev, size, vaddr, dma_addr);
-}
-
-const struct dma_map_ops swiotlb_dma_map_ops = {
-	.alloc = unicore_swiotlb_alloc_coherent,
-	.free = unicore_swiotlb_free_coherent,
-	.map_sg = swiotlb_map_sg_attrs,
-	.unmap_sg = swiotlb_unmap_sg_attrs,
-	.dma_supported = swiotlb_dma_supported,
-	.map_page = swiotlb_map_page,
-	.unmap_page = swiotlb_unmap_page,
-	.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
-	.sync_single_for_device = swiotlb_sync_single_for_device,
-	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
-	.sync_sg_for_device = swiotlb_sync_sg_for_device,
-	.mapping_error = swiotlb_dma_mapping_error,
-};
-EXPORT_SYMBOL(swiotlb_dma_map_ops);
diff --git a/arch/x86/.gitignore b/arch/x86/.gitignore
index aff152c87cf4..5a82bac5e0bc 100644
--- a/arch/x86/.gitignore
+++ b/arch/x86/.gitignore
@@ -1,6 +1,7 @@
 boot/compressed/vmlinux
 tools/test_get_len
 tools/insn_sanity
+tools/insn_decoder_test
 purgatory/kexec-purgatory.c
 purgatory/purgatory.ro
 
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 20da391b5f32..0fa71a78ec99 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -54,6 +54,8 @@ config X86
 	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_HAS_KCOV			if X86_64
+	select ARCH_HAS_PHYS_TO_DMA
+	select ARCH_HAS_MEMBARRIER_SYNC_CORE
 	select ARCH_HAS_PMEM_API		if X86_64
 	select ARCH_HAS_REFCOUNT
 	select ARCH_HAS_UACCESS_FLUSHCACHE	if X86_64
@@ -61,6 +63,7 @@ config X86
 	select ARCH_HAS_SG_CHAIN
 	select ARCH_HAS_STRICT_KERNEL_RWX
 	select ARCH_HAS_STRICT_MODULE_RWX
+	select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE
 	select ARCH_HAS_UBSAN_SANITIZE_ALL
 	select ARCH_HAS_ZONE_DEVICE		if X86_64
 	select ARCH_HAVE_NMI_SAFE_CMPXCHG
@@ -68,7 +71,6 @@ config X86
 	select ARCH_MIGHT_HAVE_PC_PARPORT
 	select ARCH_MIGHT_HAVE_PC_SERIO
 	select ARCH_SUPPORTS_ATOMIC_RMW
-	select ARCH_SUPPORTS_DEFERRED_STRUCT_PAGE_INIT
 	select ARCH_SUPPORTS_NUMA_BALANCING	if X86_64
 	select ARCH_USE_BUILTIN_BSWAP
 	select ARCH_USE_QUEUED_RWLOCKS
@@ -116,6 +118,7 @@ config X86
 	select HAVE_ARCH_MMAP_RND_COMPAT_BITS	if MMU && COMPAT
 	select HAVE_ARCH_COMPAT_MMAP_BASES	if MMU && COMPAT
 	select HAVE_ARCH_SECCOMP_FILTER
+	select HAVE_ARCH_THREAD_STRUCT_WHITELIST
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD if X86_64
@@ -154,6 +157,7 @@ config X86
 	select HAVE_KERNEL_XZ
 	select HAVE_KPROBES
 	select HAVE_KPROBES_ON_FTRACE
+	select HAVE_FUNCTION_ERROR_INJECTION
 	select HAVE_KRETPROBES
 	select HAVE_KVM
 	select HAVE_LIVEPATCH			if X86_64
@@ -320,7 +324,7 @@ config X86_64_SMP
 
 config X86_32_LAZY_GS
 	def_bool y
-	depends on X86_32 && !CC_STACKPROTECTOR
+	depends on X86_32 && CC_STACKPROTECTOR_NONE
 
 config ARCH_SUPPORTS_UPROBES
 	def_bool y
@@ -419,12 +423,6 @@ config X86_MPPARSE
 	  For old smp systems that do not have proper acpi support. Newer systems
 	  (esp with 64bit cpus) with acpi support, MADT and DSDT will override it
 
-config X86_BIGSMP
-	bool "Support for big SMP systems with more than 8 CPUs"
-	depends on X86_32 && SMP
-	---help---
-	  This option is needed for the systems that have more than 8 CPUs
-
 config GOLDFISH
        def_bool y
        depends on X86_GOLDFISH
@@ -432,6 +430,7 @@ config GOLDFISH
 config RETPOLINE
 	bool "Avoid speculative indirect branches in kernel"
 	default y
+	select STACK_VALIDATION if HAVE_STACK_VALIDATION
 	help
 	  Compile kernel with the retpoline compiler options to guard against
 	  kernel-to-user data leaks by avoiding speculative indirect
@@ -456,6 +455,12 @@ config INTEL_RDT
 	  Say N if unsure.
 
 if X86_32
+config X86_BIGSMP
+	bool "Support for big SMP systems with more than 8 CPUs"
+	depends on SMP
+	---help---
+	  This option is needed for the systems that have more than 8 CPUs
+
 config X86_EXTENDED_PLATFORM
 	bool "Support for extended (non-PC) x86 platforms"
 	default y
@@ -810,6 +815,15 @@ config PARAVIRT_TIME_ACCOUNTING
 config PARAVIRT_CLOCK
 	bool
 
+config JAILHOUSE_GUEST
+	bool "Jailhouse non-root cell support"
+	depends on X86_64 && PCI
+	select X86_PM_TIMER
+	---help---
+	  This option allows to run Linux as guest in a Jailhouse non-root
+	  cell. You can leave this option disabled if you only want to start
+	  Jailhouse and run Linux afterwards in the root cell.
+
 endif #HYPERVISOR_GUEST
 
 config NO_BOOTMEM
@@ -936,25 +950,66 @@ config MAXSMP
 	  Enable maximum number of CPUS and NUMA Nodes for this architecture.
 	  If unsure, say N.
 
+#
+# The maximum number of CPUs supported:
+#
+# The main config value is NR_CPUS, which defaults to NR_CPUS_DEFAULT,
+# and which can be configured interactively in the
+# [NR_CPUS_RANGE_BEGIN ... NR_CPUS_RANGE_END] range.
+#
+# The ranges are different on 32-bit and 64-bit kernels, depending on
+# hardware capabilities and scalability features of the kernel.
+#
+# ( If MAXSMP is enabled we just use the highest possible value and disable
+#   interactive configuration. )
+#
+
+config NR_CPUS_RANGE_BEGIN
+	int
+	default NR_CPUS_RANGE_END if MAXSMP
+	default    1 if !SMP
+	default    2
+
+config NR_CPUS_RANGE_END
+	int
+	depends on X86_32
+	default   64 if  SMP &&  X86_BIGSMP
+	default    8 if  SMP && !X86_BIGSMP
+	default    1 if !SMP
+
+config NR_CPUS_RANGE_END
+	int
+	depends on X86_64
+	default 8192 if  SMP && ( MAXSMP ||  CPUMASK_OFFSTACK)
+	default  512 if  SMP && (!MAXSMP && !CPUMASK_OFFSTACK)
+	default    1 if !SMP
+
+config NR_CPUS_DEFAULT
+	int
+	depends on X86_32
+	default   32 if  X86_BIGSMP
+	default    8 if  SMP
+	default    1 if !SMP
+
+config NR_CPUS_DEFAULT
+	int
+	depends on X86_64
+	default 8192 if  MAXSMP
+	default   64 if  SMP
+	default    1 if !SMP
+
 config NR_CPUS
 	int "Maximum number of CPUs" if SMP && !MAXSMP
-	range 2 8 if SMP && X86_32 && !X86_BIGSMP
-	range 2 64 if SMP && X86_32 && X86_BIGSMP
-	range 2 512 if SMP && !MAXSMP && !CPUMASK_OFFSTACK && X86_64
-	range 2 8192 if SMP && !MAXSMP && CPUMASK_OFFSTACK && X86_64
-	default "1" if !SMP
-	default "8192" if MAXSMP
-	default "32" if SMP && X86_BIGSMP
-	default "8" if SMP && X86_32
-	default "64" if SMP
+	range NR_CPUS_RANGE_BEGIN NR_CPUS_RANGE_END
+	default NR_CPUS_DEFAULT
 	---help---
 	  This allows you to specify the maximum number of CPUs which this
 	  kernel will support.  If CPUMASK_OFFSTACK is enabled, the maximum
 	  supported value is 8192, otherwise the maximum value is 512.  The
 	  minimum value which makes sense is 2.
 
-	  This is purely to save memory - each supported CPU adds
-	  approximately eight kilobytes to the kernel image.
+	  This is purely to save memory: each supported CPU adds about 8KB
+	  to the kernel image.
 
 config SCHED_SMT
 	bool "SMT (Hyperthreading) scheduler support"
@@ -1255,9 +1310,9 @@ config MICROCODE
 	  CONFIG_BLK_DEV_INITRD in order for the loader to be able to scan the
 	  initrd for microcode blobs.
 
-	  In addition, you can build-in the microcode into the kernel. For that you
-	  need to enable FIRMWARE_IN_KERNEL and add the vendor-supplied microcode
-	  to the CONFIG_EXTRA_FIRMWARE config option.
+	  In addition, you can build the microcode into the kernel. For that you
+	  need to add the vendor-supplied microcode to the CONFIG_EXTRA_FIRMWARE
+	  config option.
 
 config MICROCODE_INTEL
 	bool "Intel microcode loading support"
@@ -1350,7 +1405,7 @@ config HIGHMEM4G
 
 config HIGHMEM64G
 	bool "64GB"
-	depends on !M486
+	depends on !M486 && !M586 && !M586TSC && !M586MMX && !MGEODE_LX && !MGEODEGX1 && !MCYRIXIII && !MELAN && !MWINCHIPC6 && !WINCHIP3D && !MK6
 	select X86_PAE
 	---help---
 	  Select this if you have a 32-bit processor and more than 4
@@ -2252,7 +2307,7 @@ choice
 	  it can be used to assist security vulnerability exploitation.
 
 	  This setting can be changed at boot time via the kernel command
-	  line parameter vsyscall=[native|emulate|none].
+	  line parameter vsyscall=[emulate|none].
 
 	  On a system with recent enough glibc (2.14 or newer) and no
 	  static binaries, you can say None without a performance penalty
@@ -2260,15 +2315,6 @@ choice
 
 	  If unsure, select "Emulate".
 
-	config LEGACY_VSYSCALL_NATIVE
-		bool "Native"
-		help
-		  Actual executable code is located in the fixed vsyscall
-		  address mapping, implementing time() efficiently. Since
-		  this makes the mapping executable, it can be used during
-		  security vulnerability exploitation (traditionally as
-		  ROP gadgets). This configuration is not recommended.
-
 	config LEGACY_VSYSCALL_EMULATE
 		bool "Emulate"
 		help
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index 65a9a4716e34..638411f22267 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -315,19 +315,6 @@ config X86_L1_CACHE_SHIFT
 	default "4" if MELAN || M486 || MGEODEGX1
 	default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
 
-config X86_PPRO_FENCE
-	bool "PentiumPro memory ordering errata workaround"
-	depends on M686 || M586MMX || M586TSC || M586 || M486 || MGEODEGX1
-	---help---
-	  Old PentiumPro multiprocessor systems had errata that could cause
-	  memory operations to violate the x86 ordering standard in rare cases.
-	  Enabling this option will attempt to work around some (but not all)
-	  occurrences of this problem, at the cost of much heavier spinlock and
-	  memory barrier operations.
-
-	  If unsure, say n here. Even distro kernels should think twice before
-	  enabling this: there are few systems, and an unlikely bug.
-
 config X86_F00F_BUG
 	def_bool y
 	depends on M586MMX || M586TSC || M586 || M486
@@ -374,7 +361,7 @@ config X86_TSC
 
 config X86_CMPXCHG64
 	def_bool y
-	depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MATOM
+	depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8
 
 # this should be set for all -march=.. options where the compiler
 # generates cmov.
@@ -385,7 +372,7 @@ config X86_CMOV
 config X86_MINIMUM_CPU_FAMILY
 	int
 	default "64" if X86_64
-	default "6" if X86_32 && X86_P6_NOP
+	default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8)
 	default "5" if X86_32 && X86_CMPXCHG64
 	default "4"
 
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index 672441c008c7..192e4d2f9efc 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -169,14 +169,6 @@ config IOMMU_DEBUG
 	  options. See Documentation/x86/x86_64/boot-options.txt for more
 	  details.
 
-config IOMMU_STRESS
-	bool "Enable IOMMU stress-test mode"
-	---help---
-	  This option disables various optimizations in IOMMU related
-	  code to do real stress testing of the IOMMU code. This option
-	  will cause a performance drop and should only be enabled for
-	  testing.
-
 config IOMMU_LEAK
 	bool "IOMMU leak tracing"
 	depends on IOMMU_DEBUG && DMA_API_DEBUG
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index fad55160dcb9..1c4d012550ec 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -223,6 +223,15 @@ KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr)
 
 LDFLAGS := -m elf_$(UTS_MACHINE)
 
+#
+# The 64-bit kernel must be aligned to 2MB.  Pass -z max-page-size=0x200000 to
+# the linker to force 2MB page size regardless of the default page size used
+# by the linker.
+#
+ifdef CONFIG_X86_64
+LDFLAGS += $(call ld-option, -z max-page-size=0x200000)
+endif
+
 # Speed up the build
 KBUILD_CFLAGS += -pipe
 # Workaround for a gcc prelease that unfortunately was shipped in a suse release
@@ -232,10 +241,9 @@ KBUILD_CFLAGS += -fno-asynchronous-unwind-tables
 
 # Avoid indirect branches in kernel to deal with Spectre
 ifdef CONFIG_RETPOLINE
-    RETPOLINE_CFLAGS += $(call cc-option,-mindirect-branch=thunk-extern -mindirect-branch-register)
-    ifneq ($(RETPOLINE_CFLAGS),)
-        KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
-    endif
+ifneq ($(RETPOLINE_CFLAGS),)
+  KBUILD_CFLAGS += $(RETPOLINE_CFLAGS) -DRETPOLINE
+endif
 endif
 
 archscripts: scripts_basic
diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c
index e56dbc67e837..886a9115af62 100644
--- a/arch/x86/boot/compressed/eboot.c
+++ b/arch/x86/boot/compressed/eboot.c
@@ -439,7 +439,7 @@ setup_uga32(void **uga_handle, unsigned long size, u32 *width, u32 *height)
 	struct efi_uga_draw_protocol *uga = NULL, *first_uga;
 	efi_guid_t uga_proto = EFI_UGA_PROTOCOL_GUID;
 	unsigned long nr_ugas;
-	u32 *handles = (u32 *)uga_handle;;
+	u32 *handles = (u32 *)uga_handle;
 	efi_status_t status = EFI_INVALID_PARAMETER;
 	int i;
 
@@ -484,7 +484,7 @@ setup_uga64(void **uga_handle, unsigned long size, u32 *width, u32 *height)
 	struct efi_uga_draw_protocol *uga = NULL, *first_uga;
 	efi_guid_t uga_proto = EFI_UGA_PROTOCOL_GUID;
 	unsigned long nr_ugas;
-	u64 *handles = (u64 *)uga_handle;;
+	u64 *handles = (u64 *)uga_handle;
 	efi_status_t status = EFI_INVALID_PARAMETER;
 	int i;
 
@@ -999,6 +999,7 @@ struct boot_params *efi_main(struct efi_config *c,
 
 	/* Ask the firmware to clear memory on unclean shutdown */
 	efi_enable_reset_attack_mitigation(sys_table);
+	efi_retrieve_tpm2_eventlog(sys_table);
 
 	setup_graphics(boot_params);
 
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 98761a1576ce..252fee320816 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -309,6 +309,10 @@ static void parse_elf(void *output)
 
 		switch (phdr->p_type) {
 		case PT_LOAD:
+#ifdef CONFIG_X86_64
+			if ((phdr->p_align % 0x200000) != 0)
+				error("Alignment of LOAD segment isn't multiple of 2MB");
+#endif
 #ifdef CONFIG_RELOCATABLE
 			dest = output;
 			dest += (phdr->p_paddr - LOAD_PHYSICAL_ADDR);
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index 3d09e3aca18d..12e8484a8ee7 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -90,30 +90,6 @@ SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
 ALL_F:      .octa 0xffffffffffffffffffffffffffffffff
             .octa 0x00000000000000000000000000000000
 
-.section .rodata
-.align 16
-.type aad_shift_arr, @object
-.size aad_shift_arr, 272
-aad_shift_arr:
-        .octa     0xffffffffffffffffffffffffffffffff
-        .octa     0xffffffffffffffffffffffffffffff0C
-        .octa     0xffffffffffffffffffffffffffff0D0C
-        .octa     0xffffffffffffffffffffffffff0E0D0C
-        .octa     0xffffffffffffffffffffffff0F0E0D0C
-        .octa     0xffffffffffffffffffffff0C0B0A0908
-        .octa     0xffffffffffffffffffff0D0C0B0A0908
-        .octa     0xffffffffffffffffff0E0D0C0B0A0908
-        .octa     0xffffffffffffffff0F0E0D0C0B0A0908
-        .octa     0xffffffffffffff0C0B0A090807060504
-        .octa     0xffffffffffff0D0C0B0A090807060504
-        .octa     0xffffffffff0E0D0C0B0A090807060504
-        .octa     0xffffffff0F0E0D0C0B0A090807060504
-        .octa     0xffffff0C0B0A09080706050403020100
-        .octa     0xffff0D0C0B0A09080706050403020100
-        .octa     0xff0E0D0C0B0A09080706050403020100
-        .octa     0x0F0E0D0C0B0A09080706050403020100
-
-
 .text
 
 
@@ -257,6 +233,37 @@ aad_shift_arr:
 	pxor      \TMP1, \GH            # result is in TMP1
 .endm
 
+# Reads DLEN bytes starting at DPTR and stores in XMMDst
+# where 0 < DLEN < 16
+# Clobbers %rax, DLEN and XMM1
+.macro READ_PARTIAL_BLOCK DPTR DLEN XMM1 XMMDst
+        cmp $8, \DLEN
+        jl _read_lt8_\@
+        mov (\DPTR), %rax
+        MOVQ_R64_XMM %rax, \XMMDst
+        sub $8, \DLEN
+        jz _done_read_partial_block_\@
+	xor %eax, %eax
+_read_next_byte_\@:
+        shl $8, %rax
+        mov 7(\DPTR, \DLEN, 1), %al
+        dec \DLEN
+        jnz _read_next_byte_\@
+        MOVQ_R64_XMM %rax, \XMM1
+	pslldq $8, \XMM1
+        por \XMM1, \XMMDst
+	jmp _done_read_partial_block_\@
+_read_lt8_\@:
+	xor %eax, %eax
+_read_next_byte_lt8_\@:
+        shl $8, %rax
+        mov -1(\DPTR, \DLEN, 1), %al
+        dec \DLEN
+        jnz _read_next_byte_lt8_\@
+        MOVQ_R64_XMM %rax, \XMMDst
+_done_read_partial_block_\@:
+.endm
+
 /*
 * if a = number of total plaintext bytes
 * b = floor(a/16)
@@ -273,62 +280,30 @@ aad_shift_arr:
 XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
         MOVADQ     SHUF_MASK(%rip), %xmm14
 	mov	   arg7, %r10           # %r10 = AAD
-	mov	   arg8, %r12           # %r12 = aadLen
-	mov	   %r12, %r11
+	mov	   arg8, %r11           # %r11 = aadLen
 	pxor	   %xmm\i, %xmm\i
 	pxor       \XMM2, \XMM2
 
 	cmp	   $16, %r11
-	jl	   _get_AAD_rest8\num_initial_blocks\operation
+	jl	   _get_AAD_rest\num_initial_blocks\operation
 _get_AAD_blocks\num_initial_blocks\operation:
 	movdqu	   (%r10), %xmm\i
 	PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data
 	pxor	   %xmm\i, \XMM2
 	GHASH_MUL  \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 	add	   $16, %r10
-	sub	   $16, %r12
 	sub	   $16, %r11
 	cmp	   $16, %r11
 	jge	   _get_AAD_blocks\num_initial_blocks\operation
 
 	movdqu	   \XMM2, %xmm\i
+
+	/* read the last <16B of AAD */
+_get_AAD_rest\num_initial_blocks\operation:
 	cmp	   $0, %r11
 	je	   _get_AAD_done\num_initial_blocks\operation
 
-	pxor	   %xmm\i,%xmm\i
-
-	/* read the last <16B of AAD. since we have at least 4B of
-	data right after the AAD (the ICV, and maybe some CT), we can
-	read 4B/8B blocks safely, and then get rid of the extra stuff */
-_get_AAD_rest8\num_initial_blocks\operation:
-	cmp	   $4, %r11
-	jle	   _get_AAD_rest4\num_initial_blocks\operation
-	movq	   (%r10), \TMP1
-	add	   $8, %r10
-	sub	   $8, %r11
-	pslldq	   $8, \TMP1
-	psrldq	   $8, %xmm\i
-	pxor	   \TMP1, %xmm\i
-	jmp	   _get_AAD_rest8\num_initial_blocks\operation
-_get_AAD_rest4\num_initial_blocks\operation:
-	cmp	   $0, %r11
-	jle	   _get_AAD_rest0\num_initial_blocks\operation
-	mov	   (%r10), %eax
-	movq	   %rax, \TMP1
-	add	   $4, %r10
-	sub	   $4, %r10
-	pslldq	   $12, \TMP1
-	psrldq	   $4, %xmm\i
-	pxor	   \TMP1, %xmm\i
-_get_AAD_rest0\num_initial_blocks\operation:
-	/* finalize: shift out the extra bytes we read, and align
-	left. since pslldq can only shift by an immediate, we use
-	vpshufb and an array of shuffle masks */
-	movq	   %r12, %r11
-	salq	   $4, %r11
-	movdqu	   aad_shift_arr(%r11), \TMP1
-	PSHUFB_XMM \TMP1, %xmm\i
-_get_AAD_rest_final\num_initial_blocks\operation:
+	READ_PARTIAL_BLOCK %r10, %r11, \TMP1, %xmm\i
 	PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
 	pxor	   \XMM2, %xmm\i
 	GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
@@ -532,62 +507,30 @@ _initial_blocks_done\num_initial_blocks\operation:
 XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
         MOVADQ     SHUF_MASK(%rip), %xmm14
 	mov	   arg7, %r10           # %r10 = AAD
-	mov	   arg8, %r12           # %r12 = aadLen
-	mov	   %r12, %r11
+	mov	   arg8, %r11           # %r11 = aadLen
 	pxor	   %xmm\i, %xmm\i
 	pxor	   \XMM2, \XMM2
 
 	cmp	   $16, %r11
-	jl	   _get_AAD_rest8\num_initial_blocks\operation
+	jl	   _get_AAD_rest\num_initial_blocks\operation
 _get_AAD_blocks\num_initial_blocks\operation:
 	movdqu	   (%r10), %xmm\i
 	PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
 	pxor	   %xmm\i, \XMM2
 	GHASH_MUL  \XMM2, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 	add	   $16, %r10
-	sub	   $16, %r12
 	sub	   $16, %r11
 	cmp	   $16, %r11
 	jge	   _get_AAD_blocks\num_initial_blocks\operation
 
 	movdqu	   \XMM2, %xmm\i
+
+	/* read the last <16B of AAD */
+_get_AAD_rest\num_initial_blocks\operation:
 	cmp	   $0, %r11
 	je	   _get_AAD_done\num_initial_blocks\operation
 
-	pxor	   %xmm\i,%xmm\i
-
-	/* read the last <16B of AAD. since we have at least 4B of
-	data right after the AAD (the ICV, and maybe some PT), we can
-	read 4B/8B blocks safely, and then get rid of the extra stuff */
-_get_AAD_rest8\num_initial_blocks\operation:
-	cmp	   $4, %r11
-	jle	   _get_AAD_rest4\num_initial_blocks\operation
-	movq	   (%r10), \TMP1
-	add	   $8, %r10
-	sub	   $8, %r11
-	pslldq	   $8, \TMP1
-	psrldq	   $8, %xmm\i
-	pxor	   \TMP1, %xmm\i
-	jmp	   _get_AAD_rest8\num_initial_blocks\operation
-_get_AAD_rest4\num_initial_blocks\operation:
-	cmp	   $0, %r11
-	jle	   _get_AAD_rest0\num_initial_blocks\operation
-	mov	   (%r10), %eax
-	movq	   %rax, \TMP1
-	add	   $4, %r10
-	sub	   $4, %r10
-	pslldq	   $12, \TMP1
-	psrldq	   $4, %xmm\i
-	pxor	   \TMP1, %xmm\i
-_get_AAD_rest0\num_initial_blocks\operation:
-	/* finalize: shift out the extra bytes we read, and align
-	left. since pslldq can only shift by an immediate, we use
-	vpshufb and an array of shuffle masks */
-	movq	   %r12, %r11
-	salq	   $4, %r11
-	movdqu	   aad_shift_arr(%r11), \TMP1
-	PSHUFB_XMM \TMP1, %xmm\i
-_get_AAD_rest_final\num_initial_blocks\operation:
+	READ_PARTIAL_BLOCK %r10, %r11, \TMP1, %xmm\i
 	PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
 	pxor	   \XMM2, %xmm\i
 	GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
@@ -1386,14 +1329,6 @@ _esb_loop_\@:
 *
 *                        AAD Format with 64-bit Extended Sequence Number
 *
-* aadLen:
-*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
-*       The code supports 16 too but for other sizes, the code will fail.
-*
-* TLen:
-*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
-*       For other sizes, the code will fail.
-*
 * poly = x^128 + x^127 + x^126 + x^121 + 1
 *
 *****************************************************************************/
@@ -1487,19 +1422,16 @@ _zero_cipher_left_decrypt:
 	PSHUFB_XMM %xmm10, %xmm0
 
 	ENCRYPT_SINGLE_BLOCK  %xmm0, %xmm1    # E(K, Yn)
-	sub $16, %r11
-	add %r13, %r11
-	movdqu (%arg3,%r11,1), %xmm1   # receive the last <16 byte block
-	lea SHIFT_MASK+16(%rip), %r12
-	sub %r13, %r12
-# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes
-# (%r13 is the number of bytes in plaintext mod 16)
-	movdqu (%r12), %xmm2           # get the appropriate shuffle mask
-	PSHUFB_XMM %xmm2, %xmm1            # right shift 16-%r13 butes
 
+	lea (%arg3,%r11,1), %r10
+	mov %r13, %r12
+	READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1
+
+	lea ALL_F+16(%rip), %r12
+	sub %r13, %r12
 	movdqa  %xmm1, %xmm2
 	pxor %xmm1, %xmm0            # Ciphertext XOR E(K, Yn)
-	movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
+	movdqu (%r12), %xmm1
 	# get the appropriate mask to mask out top 16-%r13 bytes of %xmm0
 	pand %xmm1, %xmm0            # mask out top 16-%r13 bytes of %xmm0
 	pand    %xmm1, %xmm2
@@ -1508,9 +1440,6 @@ _zero_cipher_left_decrypt:
 
 	pxor %xmm2, %xmm8
 	GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
-	          # GHASH computation for the last <16 byte block
-	sub %r13, %r11
-	add $16, %r11
 
         # output %r13 bytes
 	MOVQ_R64_XMM	%xmm0, %rax
@@ -1664,14 +1593,6 @@ ENDPROC(aesni_gcm_dec)
 *
 *                         AAD Format with 64-bit Extended Sequence Number
 *
-* aadLen:
-*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
-*       The code supports 16 too but for other sizes, the code will fail.
-*
-* TLen:
-*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
-*       For other sizes, the code will fail.
-*
 * poly = x^128 + x^127 + x^126 + x^121 + 1
 ***************************************************************************/
 ENTRY(aesni_gcm_enc)
@@ -1764,19 +1685,16 @@ _zero_cipher_left_encrypt:
         movdqa SHUF_MASK(%rip), %xmm10
 	PSHUFB_XMM %xmm10, %xmm0
 
-
 	ENCRYPT_SINGLE_BLOCK	%xmm0, %xmm1        # Encrypt(K, Yn)
-	sub $16, %r11
-	add %r13, %r11
-	movdqu (%arg3,%r11,1), %xmm1     # receive the last <16 byte blocks
-	lea SHIFT_MASK+16(%rip), %r12
+
+	lea (%arg3,%r11,1), %r10
+	mov %r13, %r12
+	READ_PARTIAL_BLOCK %r10 %r12 %xmm2 %xmm1
+
+	lea ALL_F+16(%rip), %r12
 	sub %r13, %r12
-	# adjust the shuffle mask pointer to be able to shift 16-r13 bytes
-	# (%r13 is the number of bytes in plaintext mod 16)
-	movdqu	(%r12), %xmm2           # get the appropriate shuffle mask
-	PSHUFB_XMM	%xmm2, %xmm1            # shift right 16-r13 byte
 	pxor	%xmm1, %xmm0            # Plaintext XOR Encrypt(K, Yn)
-	movdqu	ALL_F-SHIFT_MASK(%r12), %xmm1
+	movdqu	(%r12), %xmm1
 	# get the appropriate mask to mask out top 16-r13 bytes of xmm0
 	pand	%xmm1, %xmm0            # mask out top 16-r13 bytes of xmm0
         movdqa SHUF_MASK(%rip), %xmm10
@@ -1785,9 +1703,6 @@ _zero_cipher_left_encrypt:
 	pxor	%xmm0, %xmm8
 	GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
 	# GHASH computation for the last <16 byte block
-	sub	%r13, %r11
-	add	$16, %r11
-
 	movdqa SHUF_MASK(%rip), %xmm10
 	PSHUFB_XMM %xmm10, %xmm0
 
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 3bf3dcf29825..34cf1c1f8c98 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -690,8 +690,8 @@ static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
 	       rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 }
 
-static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
-			   unsigned int key_len)
+static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
+				  unsigned int key_len)
 {
 	struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 	struct cryptd_aead *cryptd_tfm = *ctx;
@@ -716,8 +716,8 @@ static int common_rfc4106_set_authsize(struct crypto_aead *aead,
 
 /* This is the Integrity Check Value (aka the authentication tag length and can
  * be 8, 12 or 16 bytes long. */
-static int rfc4106_set_authsize(struct crypto_aead *parent,
-				unsigned int authsize)
+static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
+				       unsigned int authsize)
 {
 	struct cryptd_aead **ctx = crypto_aead_ctx(parent);
 	struct cryptd_aead *cryptd_tfm = *ctx;
@@ -824,7 +824,7 @@ static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
 	if (sg_is_last(req->src) &&
 	    (!PageHighMem(sg_page(req->src)) ||
 	    req->src->offset + req->src->length <= PAGE_SIZE) &&
-	    sg_is_last(req->dst) &&
+	    sg_is_last(req->dst) && req->dst->length &&
 	    (!PageHighMem(sg_page(req->dst)) ||
 	    req->dst->offset + req->dst->length <= PAGE_SIZE)) {
 		one_entry_in_sg = 1;
@@ -929,7 +929,7 @@ static int helper_rfc4106_decrypt(struct aead_request *req)
 			      aes_ctx);
 }
 
-static int rfc4106_encrypt(struct aead_request *req)
+static int gcmaes_wrapper_encrypt(struct aead_request *req)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
@@ -945,7 +945,7 @@ static int rfc4106_encrypt(struct aead_request *req)
 	return crypto_aead_encrypt(req);
 }
 
-static int rfc4106_decrypt(struct aead_request *req)
+static int gcmaes_wrapper_decrypt(struct aead_request *req)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 	struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
@@ -1117,7 +1117,7 @@ static int generic_gcmaes_decrypt(struct aead_request *req)
 {
 	__be32 counter = cpu_to_be32(1);
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-	struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
+	struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
 	void *aes_ctx = &(ctx->aes_key_expanded);
 	u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
 
@@ -1128,6 +1128,30 @@ static int generic_gcmaes_decrypt(struct aead_request *req)
 			      aes_ctx);
 }
 
+static int generic_gcmaes_init(struct crypto_aead *aead)
+{
+	struct cryptd_aead *cryptd_tfm;
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni",
+				       CRYPTO_ALG_INTERNAL,
+				       CRYPTO_ALG_INTERNAL);
+	if (IS_ERR(cryptd_tfm))
+		return PTR_ERR(cryptd_tfm);
+
+	*ctx = cryptd_tfm;
+	crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
+
+	return 0;
+}
+
+static void generic_gcmaes_exit(struct crypto_aead *aead)
+{
+	struct cryptd_aead **ctx = crypto_aead_ctx(aead);
+
+	cryptd_free_aead(*ctx);
+}
+
 static struct aead_alg aesni_aead_algs[] = { {
 	.setkey			= common_rfc4106_set_key,
 	.setauthsize		= common_rfc4106_set_authsize,
@@ -1147,10 +1171,10 @@ static struct aead_alg aesni_aead_algs[] = { {
 }, {
 	.init			= rfc4106_init,
 	.exit			= rfc4106_exit,
-	.setkey			= rfc4106_set_key,
-	.setauthsize		= rfc4106_set_authsize,
-	.encrypt		= rfc4106_encrypt,
-	.decrypt		= rfc4106_decrypt,
+	.setkey			= gcmaes_wrapper_set_key,
+	.setauthsize		= gcmaes_wrapper_set_authsize,
+	.encrypt		= gcmaes_wrapper_encrypt,
+	.decrypt		= gcmaes_wrapper_decrypt,
 	.ivsize			= GCM_RFC4106_IV_SIZE,
 	.maxauthsize		= 16,
 	.base = {
@@ -1169,14 +1193,32 @@ static struct aead_alg aesni_aead_algs[] = { {
 	.decrypt		= generic_gcmaes_decrypt,
 	.ivsize			= GCM_AES_IV_SIZE,
 	.maxauthsize		= 16,
+	.base = {
+		.cra_name		= "__generic-gcm-aes-aesni",
+		.cra_driver_name	= "__driver-generic-gcm-aes-aesni",
+		.cra_priority		= 0,
+		.cra_flags		= CRYPTO_ALG_INTERNAL,
+		.cra_blocksize		= 1,
+		.cra_ctxsize		= sizeof(struct generic_gcmaes_ctx),
+		.cra_alignmask		= AESNI_ALIGN - 1,
+		.cra_module		= THIS_MODULE,
+	},
+}, {
+	.init			= generic_gcmaes_init,
+	.exit			= generic_gcmaes_exit,
+	.setkey			= gcmaes_wrapper_set_key,
+	.setauthsize		= gcmaes_wrapper_set_authsize,
+	.encrypt		= gcmaes_wrapper_encrypt,
+	.decrypt		= gcmaes_wrapper_decrypt,
+	.ivsize			= GCM_AES_IV_SIZE,
+	.maxauthsize		= 16,
 	.base = {
 		.cra_name		= "gcm(aes)",
 		.cra_driver_name	= "generic-gcm-aesni",
 		.cra_priority		= 400,
 		.cra_flags		= CRYPTO_ALG_ASYNC,
 		.cra_blocksize		= 1,
-		.cra_ctxsize		= sizeof(struct generic_gcmaes_ctx),
-		.cra_alignmask		= AESNI_ALIGN - 1,
+		.cra_ctxsize		= sizeof(struct cryptd_aead *),
 		.cra_module		= THIS_MODULE,
 	},
 } };
diff --git a/arch/x86/crypto/chacha20_glue.c b/arch/x86/crypto/chacha20_glue.c
index 1e6af1b35f7b..dce7c5d39c2f 100644
--- a/arch/x86/crypto/chacha20_glue.c
+++ b/arch/x86/crypto/chacha20_glue.c
@@ -107,7 +107,6 @@ static struct skcipher_alg alg = {
 	.base.cra_priority	= 300,
 	.base.cra_blocksize	= 1,
 	.base.cra_ctxsize	= sizeof(struct chacha20_ctx),
-	.base.cra_alignmask	= sizeof(u32) - 1,
 	.base.cra_module	= THIS_MODULE,
 
 	.min_keysize		= CHACHA20_KEY_SIZE,
diff --git a/arch/x86/crypto/crc32-pclmul_glue.c b/arch/x86/crypto/crc32-pclmul_glue.c
index 27226df3f7d8..c8d9cdacbf10 100644
--- a/arch/x86/crypto/crc32-pclmul_glue.c
+++ b/arch/x86/crypto/crc32-pclmul_glue.c
@@ -162,6 +162,7 @@ static struct shash_alg alg = {
 			.cra_name		= "crc32",
 			.cra_driver_name	= "crc32-pclmul",
 			.cra_priority		= 200,
+			.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 			.cra_blocksize		= CHKSUM_BLOCK_SIZE,
 			.cra_ctxsize		= sizeof(u32),
 			.cra_module		= THIS_MODULE,
diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c
index c194d5717ae5..5773e1161072 100644
--- a/arch/x86/crypto/crc32c-intel_glue.c
+++ b/arch/x86/crypto/crc32c-intel_glue.c
@@ -226,6 +226,7 @@ static struct shash_alg alg = {
 		.cra_name		=	"crc32c",
 		.cra_driver_name	=	"crc32c-intel",
 		.cra_priority		=	200,
+		.cra_flags		=	CRYPTO_ALG_OPTIONAL_KEY,
 		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
 		.cra_ctxsize		=	sizeof(u32),
 		.cra_module		=	THIS_MODULE,
diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c
index e32142bc071d..790377797544 100644
--- a/arch/x86/crypto/poly1305_glue.c
+++ b/arch/x86/crypto/poly1305_glue.c
@@ -164,14 +164,12 @@ static struct shash_alg alg = {
 	.init		= poly1305_simd_init,
 	.update		= poly1305_simd_update,
 	.final		= crypto_poly1305_final,
-	.setkey		= crypto_poly1305_setkey,
 	.descsize	= sizeof(struct poly1305_simd_desc_ctx),
 	.base		= {
 		.cra_name		= "poly1305",
 		.cra_driver_name	= "poly1305-simd",
 		.cra_priority		= 300,
 		.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
-		.cra_alignmask		= sizeof(u32) - 1,
 		.cra_blocksize		= POLY1305_BLOCK_SIZE,
 		.cra_module		= THIS_MODULE,
 	},
diff --git a/arch/x86/crypto/salsa20-i586-asm_32.S b/arch/x86/crypto/salsa20-i586-asm_32.S
index 329452b8f794..6014b7b9e52a 100644
--- a/arch/x86/crypto/salsa20-i586-asm_32.S
+++ b/arch/x86/crypto/salsa20-i586-asm_32.S
@@ -1,6 +1,7 @@
-# salsa20_pm.s version 20051229
-# D. J. Bernstein
-# Public domain.
+# Derived from:
+#	salsa20_pm.s version 20051229
+#	D. J. Bernstein
+#	Public domain.
 
 #include <linux/linkage.h>
 
@@ -935,180 +936,3 @@ ENTRY(salsa20_encrypt_bytes)
 	# goto bytesatleast1
 	jmp	._bytesatleast1
 ENDPROC(salsa20_encrypt_bytes)
-
-# enter salsa20_keysetup
-ENTRY(salsa20_keysetup)
-	mov	%esp,%eax
-	and	$31,%eax
-	add	$256,%eax
-	sub	%eax,%esp
-	#   eax_stack = eax
-	movl	%eax,64(%esp)
-	#   ebx_stack = ebx
-	movl	%ebx,68(%esp)
-	#   esi_stack = esi
-	movl	%esi,72(%esp)
-	#   edi_stack = edi
-	movl	%edi,76(%esp)
-	#   ebp_stack = ebp
-	movl	%ebp,80(%esp)
-	#   k = arg2
-	movl	8(%esp,%eax),%ecx
-	#   kbits = arg3
-	movl	12(%esp,%eax),%edx
-	#   x = arg1
-	movl	4(%esp,%eax),%eax
-	#   in1 = *(uint32 *) (k + 0)
-	movl	0(%ecx),%ebx
-	#   in2 = *(uint32 *) (k + 4)
-	movl	4(%ecx),%esi
-	#   in3 = *(uint32 *) (k + 8)
-	movl	8(%ecx),%edi
-	#   in4 = *(uint32 *) (k + 12)
-	movl	12(%ecx),%ebp
-	#   *(uint32 *) (x + 4) = in1
-	movl	%ebx,4(%eax)
-	#   *(uint32 *) (x + 8) = in2
-	movl	%esi,8(%eax)
-	#   *(uint32 *) (x + 12) = in3
-	movl	%edi,12(%eax)
-	#   *(uint32 *) (x + 16) = in4
-	movl	%ebp,16(%eax)
-	#   kbits - 256
-	cmp	$256,%edx
-	#   goto kbits128 if unsigned<
-	jb	._kbits128
-._kbits256:
-	#     in11 = *(uint32 *) (k + 16)
-	movl	16(%ecx),%edx
-	#     in12 = *(uint32 *) (k + 20)
-	movl	20(%ecx),%ebx
-	#     in13 = *(uint32 *) (k + 24)
-	movl	24(%ecx),%esi
-	#     in14 = *(uint32 *) (k + 28)
-	movl	28(%ecx),%ecx
-	#     *(uint32 *) (x + 44) = in11
-	movl	%edx,44(%eax)
-	#     *(uint32 *) (x + 48) = in12
-	movl	%ebx,48(%eax)
-	#     *(uint32 *) (x + 52) = in13
-	movl	%esi,52(%eax)
-	#     *(uint32 *) (x + 56) = in14
-	movl	%ecx,56(%eax)
-	#     in0 = 1634760805
-	mov	$1634760805,%ecx
-	#     in5 = 857760878
-	mov	$857760878,%edx
-	#     in10 = 2036477234
-	mov	$2036477234,%ebx
-	#     in15 = 1797285236
-	mov	$1797285236,%esi
-	#     *(uint32 *) (x + 0) = in0
-	movl	%ecx,0(%eax)
-	#     *(uint32 *) (x + 20) = in5
-	movl	%edx,20(%eax)
-	#     *(uint32 *) (x + 40) = in10
-	movl	%ebx,40(%eax)
-	#     *(uint32 *) (x + 60) = in15
-	movl	%esi,60(%eax)
-	#   goto keysetupdone
-	jmp	._keysetupdone
-._kbits128:
-	#     in11 = *(uint32 *) (k + 0)
-	movl	0(%ecx),%edx
-	#     in12 = *(uint32 *) (k + 4)
-	movl	4(%ecx),%ebx
-	#     in13 = *(uint32 *) (k + 8)
-	movl	8(%ecx),%esi
-	#     in14 = *(uint32 *) (k + 12)
-	movl	12(%ecx),%ecx
-	#     *(uint32 *) (x + 44) = in11
-	movl	%edx,44(%eax)
-	#     *(uint32 *) (x + 48) = in12
-	movl	%ebx,48(%eax)
-	#     *(uint32 *) (x + 52) = in13
-	movl	%esi,52(%eax)
-	#     *(uint32 *) (x + 56) = in14
-	movl	%ecx,56(%eax)
-	#     in0 = 1634760805
-	mov	$1634760805,%ecx
-	#     in5 = 824206446
-	mov	$824206446,%edx
-	#     in10 = 2036477238
-	mov	$2036477238,%ebx
-	#     in15 = 1797285236
-	mov	$1797285236,%esi
-	#     *(uint32 *) (x + 0) = in0
-	movl	%ecx,0(%eax)
-	#     *(uint32 *) (x + 20) = in5
-	movl	%edx,20(%eax)
-	#     *(uint32 *) (x + 40) = in10
-	movl	%ebx,40(%eax)
-	#     *(uint32 *) (x + 60) = in15
-	movl	%esi,60(%eax)
-._keysetupdone:
-	#   eax = eax_stack
-	movl	64(%esp),%eax
-	#   ebx = ebx_stack
-	movl	68(%esp),%ebx
-	#   esi = esi_stack
-	movl	72(%esp),%esi
-	#   edi = edi_stack
-	movl	76(%esp),%edi
-	#   ebp = ebp_stack
-	movl	80(%esp),%ebp
-	# leave
-	add	%eax,%esp
-	ret
-ENDPROC(salsa20_keysetup)
-
-# enter salsa20_ivsetup
-ENTRY(salsa20_ivsetup)
-	mov	%esp,%eax
-	and	$31,%eax
-	add	$256,%eax
-	sub	%eax,%esp
-	#   eax_stack = eax
-	movl	%eax,64(%esp)
-	#   ebx_stack = ebx
-	movl	%ebx,68(%esp)
-	#   esi_stack = esi
-	movl	%esi,72(%esp)
-	#   edi_stack = edi
-	movl	%edi,76(%esp)
-	#   ebp_stack = ebp
-	movl	%ebp,80(%esp)
-	#   iv = arg2
-	movl	8(%esp,%eax),%ecx
-	#   x = arg1
-	movl	4(%esp,%eax),%eax
-	#   in6 = *(uint32 *) (iv + 0)
-	movl	0(%ecx),%edx
-	#   in7 = *(uint32 *) (iv + 4)
-	movl	4(%ecx),%ecx
-	#   in8 = 0
-	mov	$0,%ebx
-	#   in9 = 0
-	mov	$0,%esi
-	#   *(uint32 *) (x + 24) = in6
-	movl	%edx,24(%eax)
-	#   *(uint32 *) (x + 28) = in7
-	movl	%ecx,28(%eax)
-	#   *(uint32 *) (x + 32) = in8
-	movl	%ebx,32(%eax)
-	#   *(uint32 *) (x + 36) = in9
-	movl	%esi,36(%eax)
-	#   eax = eax_stack
-	movl	64(%esp),%eax
-	#   ebx = ebx_stack
-	movl	68(%esp),%ebx
-	#   esi = esi_stack
-	movl	72(%esp),%esi
-	#   edi = edi_stack
-	movl	76(%esp),%edi
-	#   ebp = ebp_stack
-	movl	80(%esp),%ebp
-	# leave
-	add	%eax,%esp
-	ret
-ENDPROC(salsa20_ivsetup)
diff --git a/arch/x86/crypto/salsa20-x86_64-asm_64.S b/arch/x86/crypto/salsa20-x86_64-asm_64.S
index 10db30d58006..03a4918f41ee 100644
--- a/arch/x86/crypto/salsa20-x86_64-asm_64.S
+++ b/arch/x86/crypto/salsa20-x86_64-asm_64.S
@@ -803,117 +803,3 @@ ENTRY(salsa20_encrypt_bytes)
 	# goto bytesatleast1
 	jmp	._bytesatleast1
 ENDPROC(salsa20_encrypt_bytes)
-
-# enter salsa20_keysetup
-ENTRY(salsa20_keysetup)
-	mov	%rsp,%r11
-	and	$31,%r11
-	add	$256,%r11
-	sub	%r11,%rsp
-	#   k = arg2
-	mov	%rsi,%rsi
-	#   kbits = arg3
-	mov	%rdx,%rdx
-	#   x = arg1
-	mov	%rdi,%rdi
-	#   in0 = *(uint64 *) (k + 0)
-	movq	0(%rsi),%r8
-	#   in2 = *(uint64 *) (k + 8)
-	movq	8(%rsi),%r9
-	#   *(uint64 *) (x + 4) = in0
-	movq	%r8,4(%rdi)
-	#   *(uint64 *) (x + 12) = in2
-	movq	%r9,12(%rdi)
-	#                    unsigned<? kbits - 256
-	cmp	$256,%rdx
-	# comment:fp stack unchanged by jump
-	#   goto kbits128 if unsigned<
-	jb	._kbits128
-#   kbits256:
-._kbits256:
-	#     in10 = *(uint64 *) (k + 16)
-	movq	16(%rsi),%rdx
-	#     in12 = *(uint64 *) (k + 24)
-	movq	24(%rsi),%rsi
-	#     *(uint64 *) (x + 44) = in10
-	movq	%rdx,44(%rdi)
-	#     *(uint64 *) (x + 52) = in12
-	movq	%rsi,52(%rdi)
-	#     in0 = 1634760805
-	mov	$1634760805,%rsi
-	#     in4 = 857760878
-	mov	$857760878,%rdx
-	#     in10 = 2036477234
-	mov	$2036477234,%rcx
-	#     in14 = 1797285236
-	mov	$1797285236,%r8
-	#     *(uint32 *) (x + 0) = in0
-	movl	%esi,0(%rdi)
-	#     *(uint32 *) (x + 20) = in4
-	movl	%edx,20(%rdi)
-	#     *(uint32 *) (x + 40) = in10
-	movl	%ecx,40(%rdi)
-	#     *(uint32 *) (x + 60) = in14
-	movl	%r8d,60(%rdi)
-	# comment:fp stack unchanged by jump
-	#   goto keysetupdone
-	jmp	._keysetupdone
-#   kbits128:
-._kbits128:
-	#     in10 = *(uint64 *) (k + 0)
-	movq	0(%rsi),%rdx
-	#     in12 = *(uint64 *) (k + 8)
-	movq	8(%rsi),%rsi
-	#     *(uint64 *) (x + 44) = in10
-	movq	%rdx,44(%rdi)
-	#     *(uint64 *) (x + 52) = in12
-	movq	%rsi,52(%rdi)
-	#     in0 = 1634760805
-	mov	$1634760805,%rsi
-	#     in4 = 824206446
-	mov	$824206446,%rdx
-	#     in10 = 2036477238
-	mov	$2036477238,%rcx
-	#     in14 = 1797285236
-	mov	$1797285236,%r8
-	#     *(uint32 *) (x + 0) = in0
-	movl	%esi,0(%rdi)
-	#     *(uint32 *) (x + 20) = in4
-	movl	%edx,20(%rdi)
-	#     *(uint32 *) (x + 40) = in10
-	movl	%ecx,40(%rdi)
-	#     *(uint32 *) (x + 60) = in14
-	movl	%r8d,60(%rdi)
-#   keysetupdone:
-._keysetupdone:
-	# leave
-	add	%r11,%rsp
-	mov	%rdi,%rax
-	mov	%rsi,%rdx
-	ret
-ENDPROC(salsa20_keysetup)
-
-# enter salsa20_ivsetup
-ENTRY(salsa20_ivsetup)
-	mov	%rsp,%r11
-	and	$31,%r11
-	add	$256,%r11
-	sub	%r11,%rsp
-	#   iv = arg2
-	mov	%rsi,%rsi
-	#   x = arg1
-	mov	%rdi,%rdi
-	#   in6 = *(uint64 *) (iv + 0)
-	movq	0(%rsi),%rsi
-	#   in8 = 0
-	mov	$0,%r8
-	#   *(uint64 *) (x + 24) = in6
-	movq	%rsi,24(%rdi)
-	#   *(uint64 *) (x + 32) = in8
-	movq	%r8,32(%rdi)
-	# leave
-	add	%r11,%rsp
-	mov	%rdi,%rax
-	mov	%rsi,%rdx
-	ret
-ENDPROC(salsa20_ivsetup)
diff --git a/arch/x86/crypto/salsa20_glue.c b/arch/x86/crypto/salsa20_glue.c
index cb91a64a99e7..b07d7d959806 100644
--- a/arch/x86/crypto/salsa20_glue.c
+++ b/arch/x86/crypto/salsa20_glue.c
@@ -11,6 +11,9 @@
  * - x86-64 version, renamed as salsa20-x86_64-asm_64.S
  *   available from <http://cr.yp.to/snuffle/salsa20/amd64-3/salsa20.s>
  *
+ * Also modified to set up the initial state using the generic C code rather
+ * than in assembly.
+ *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
@@ -18,93 +21,65 @@
  *
  */
 
-#include <crypto/algapi.h>
+#include <asm/unaligned.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/salsa20.h>
 #include <linux/module.h>
-#include <linux/crypto.h>
-
-#define SALSA20_IV_SIZE        8U
-#define SALSA20_MIN_KEY_SIZE  16U
-#define SALSA20_MAX_KEY_SIZE  32U
-
-struct salsa20_ctx
-{
-	u32 input[16];
-};
 
-asmlinkage void salsa20_keysetup(struct salsa20_ctx *ctx, const u8 *k,
-				 u32 keysize, u32 ivsize);
-asmlinkage void salsa20_ivsetup(struct salsa20_ctx *ctx, const u8 *iv);
-asmlinkage void salsa20_encrypt_bytes(struct salsa20_ctx *ctx,
-				      const u8 *src, u8 *dst, u32 bytes);
+asmlinkage void salsa20_encrypt_bytes(u32 state[16], const u8 *src, u8 *dst,
+				      u32 bytes);
 
-static int setkey(struct crypto_tfm *tfm, const u8 *key,
-		  unsigned int keysize)
+static int salsa20_asm_crypt(struct skcipher_request *req)
 {
-	struct salsa20_ctx *ctx = crypto_tfm_ctx(tfm);
-	salsa20_keysetup(ctx, key, keysize*8, SALSA20_IV_SIZE*8);
-	return 0;
-}
-
-static int encrypt(struct blkcipher_desc *desc,
-		   struct scatterlist *dst, struct scatterlist *src,
-		   unsigned int nbytes)
-{
-	struct blkcipher_walk walk;
-	struct crypto_blkcipher *tfm = desc->tfm;
-	struct salsa20_ctx *ctx = crypto_blkcipher_ctx(tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	const struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
+	u32 state[16];
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, 64);
+	err = skcipher_walk_virt(&walk, req, true);
 
-	salsa20_ivsetup(ctx, walk.iv);
+	crypto_salsa20_init(state, ctx, walk.iv);
 
-	while (walk.nbytes >= 64) {
-		salsa20_encrypt_bytes(ctx, walk.src.virt.addr,
-				      walk.dst.virt.addr,
-				      walk.nbytes - (walk.nbytes % 64));
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % 64);
-	}
+	while (walk.nbytes > 0) {
+		unsigned int nbytes = walk.nbytes;
 
-	if (walk.nbytes) {
-		salsa20_encrypt_bytes(ctx, walk.src.virt.addr,
-				      walk.dst.virt.addr, walk.nbytes);
-		err = blkcipher_walk_done(desc, &walk, 0);
+		if (nbytes < walk.total)
+			nbytes = round_down(nbytes, walk.stride);
+
+		salsa20_encrypt_bytes(state, walk.src.virt.addr,
+				      walk.dst.virt.addr, nbytes);
+		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 	}
 
 	return err;
 }
 
-static struct crypto_alg alg = {
-	.cra_name           =   "salsa20",
-	.cra_driver_name    =   "salsa20-asm",
-	.cra_priority       =   200,
-	.cra_flags          =   CRYPTO_ALG_TYPE_BLKCIPHER,
-	.cra_type           =   &crypto_blkcipher_type,
-	.cra_blocksize      =   1,
-	.cra_ctxsize        =   sizeof(struct salsa20_ctx),
-	.cra_alignmask      =	3,
-	.cra_module         =   THIS_MODULE,
-	.cra_u              =   {
-		.blkcipher = {
-			.setkey         =   setkey,
-			.encrypt        =   encrypt,
-			.decrypt        =   encrypt,
-			.min_keysize    =   SALSA20_MIN_KEY_SIZE,
-			.max_keysize    =   SALSA20_MAX_KEY_SIZE,
-			.ivsize         =   SALSA20_IV_SIZE,
-		}
-	}
+static struct skcipher_alg alg = {
+	.base.cra_name		= "salsa20",
+	.base.cra_driver_name	= "salsa20-asm",
+	.base.cra_priority	= 200,
+	.base.cra_blocksize	= 1,
+	.base.cra_ctxsize	= sizeof(struct salsa20_ctx),
+	.base.cra_module	= THIS_MODULE,
+
+	.min_keysize		= SALSA20_MIN_KEY_SIZE,
+	.max_keysize		= SALSA20_MAX_KEY_SIZE,
+	.ivsize			= SALSA20_IV_SIZE,
+	.chunksize		= SALSA20_BLOCK_SIZE,
+	.setkey			= crypto_salsa20_setkey,
+	.encrypt		= salsa20_asm_crypt,
+	.decrypt		= salsa20_asm_crypt,
 };
 
 static int __init init(void)
 {
-	return crypto_register_alg(&alg);
+	return crypto_register_skcipher(&alg);
 }
 
 static void __exit fini(void)
 {
-	crypto_unregister_alg(&alg);
+	crypto_unregister_skcipher(&alg);
 }
 
 module_init(init);
diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c
index 36870b26067a..d08805032f01 100644
--- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c
+++ b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c
@@ -57,10 +57,12 @@ void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state)
 {
 	unsigned int j;
 
-	state->lens[0] = 0;
-	state->lens[1] = 1;
-	state->lens[2] = 2;
-	state->lens[3] = 3;
+	/* initially all lanes are unused */
+	state->lens[0] = 0xFFFFFFFF00000000;
+	state->lens[1] = 0xFFFFFFFF00000001;
+	state->lens[2] = 0xFFFFFFFF00000002;
+	state->lens[3] = 0xFFFFFFFF00000003;
+
 	state->unused_lanes = 0xFF03020100;
 	for (j = 0; j < 4; j++)
 		state->ldata[j].job_in_lane = NULL;
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
index 1c3b7ceb36d2..e7273a606a07 100644
--- a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
+++ b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
@@ -55,29 +55,31 @@
 #define RAB1bl %bl
 #define RAB2bl %cl
 
+#define CD0 0x0(%rsp)
+#define CD1 0x8(%rsp)
+#define CD2 0x10(%rsp)
+
+# used only before/after all rounds
 #define RCD0 %r8
 #define RCD1 %r9
 #define RCD2 %r10
 
-#define RCD0d %r8d
-#define RCD1d %r9d
-#define RCD2d %r10d
-
-#define RX0 %rbp
-#define RX1 %r11
-#define RX2 %r12
+# used only during rounds
+#define RX0 %r8
+#define RX1 %r9
+#define RX2 %r10
 
-#define RX0d %ebp
-#define RX1d %r11d
-#define RX2d %r12d
+#define RX0d %r8d
+#define RX1d %r9d
+#define RX2d %r10d
 
-#define RY0 %r13
-#define RY1 %r14
-#define RY2 %r15
+#define RY0 %r11
+#define RY1 %r12
+#define RY2 %r13
 
-#define RY0d %r13d
-#define RY1d %r14d
-#define RY2d %r15d
+#define RY0d %r11d
+#define RY1d %r12d
+#define RY2d %r13d
 
 #define RT0 %rdx
 #define RT1 %rsi
@@ -85,6 +87,8 @@
 #define RT0d %edx
 #define RT1d %esi
 
+#define RT1bl %sil
+
 #define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \
 	movzbl ab ## bl,		tmp2 ## d; \
 	movzbl ab ## bh,		tmp1 ## d; \
@@ -92,6 +96,11 @@
 	op1##l T0(CTX, tmp2, 4),	dst ## d; \
 	op2##l T1(CTX, tmp1, 4),	dst ## d;
 
+#define swap_ab_with_cd(ab, cd, tmp)	\
+	movq cd, tmp;			\
+	movq ab, cd;			\
+	movq tmp, ab;
+
 /*
  * Combined G1 & G2 function. Reordered with help of rotates to have moves
  * at begining.
@@ -110,15 +119,15 @@
 	/* G1,2 && G2,2 */ \
 	do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 0, x ## 0); \
 	do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 0, y ## 0); \
-	xchgq cd ## 0, ab ## 0; \
+	swap_ab_with_cd(ab ## 0, cd ## 0, RT0); \
 	\
 	do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 1, x ## 1); \
 	do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 1, y ## 1); \
-	xchgq cd ## 1, ab ## 1; \
+	swap_ab_with_cd(ab ## 1, cd ## 1, RT0); \
 	\
 	do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 2, x ## 2); \
 	do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 2, y ## 2); \
-	xchgq cd ## 2, ab ## 2;
+	swap_ab_with_cd(ab ## 2, cd ## 2, RT0);
 
 #define enc_round_end(ab, x, y, n) \
 	addl y ## d,			x ## d; \
@@ -168,6 +177,16 @@
 	decrypt_round3(ba, dc, (n*2)+1); \
 	decrypt_round3(ba, dc, (n*2));
 
+#define push_cd()	\
+	pushq RCD2;	\
+	pushq RCD1;	\
+	pushq RCD0;
+
+#define pop_cd()	\
+	popq RCD0;	\
+	popq RCD1;	\
+	popq RCD2;
+
 #define inpack3(in, n, xy, m) \
 	movq 4*(n)(in),			xy ## 0; \
 	xorq w+4*m(CTX),		xy ## 0; \
@@ -223,11 +242,8 @@ ENTRY(__twofish_enc_blk_3way)
 	 *	%rdx: src, RIO
 	 *	%rcx: bool, if true: xor output
 	 */
-	pushq %r15;
-	pushq %r14;
 	pushq %r13;
 	pushq %r12;
-	pushq %rbp;
 	pushq %rbx;
 
 	pushq %rcx; /* bool xor */
@@ -235,40 +251,36 @@ ENTRY(__twofish_enc_blk_3way)
 
 	inpack_enc3();
 
-	encrypt_cycle3(RAB, RCD, 0);
-	encrypt_cycle3(RAB, RCD, 1);
-	encrypt_cycle3(RAB, RCD, 2);
-	encrypt_cycle3(RAB, RCD, 3);
-	encrypt_cycle3(RAB, RCD, 4);
-	encrypt_cycle3(RAB, RCD, 5);
-	encrypt_cycle3(RAB, RCD, 6);
-	encrypt_cycle3(RAB, RCD, 7);
+	push_cd();
+	encrypt_cycle3(RAB, CD, 0);
+	encrypt_cycle3(RAB, CD, 1);
+	encrypt_cycle3(RAB, CD, 2);
+	encrypt_cycle3(RAB, CD, 3);
+	encrypt_cycle3(RAB, CD, 4);
+	encrypt_cycle3(RAB, CD, 5);
+	encrypt_cycle3(RAB, CD, 6);
+	encrypt_cycle3(RAB, CD, 7);
+	pop_cd();
 
 	popq RIO; /* dst */
-	popq %rbp; /* bool xor */
+	popq RT1; /* bool xor */
 
-	testb %bpl, %bpl;
+	testb RT1bl, RT1bl;
 	jnz .L__enc_xor3;
 
 	outunpack_enc3(mov);
 
 	popq %rbx;
-	popq %rbp;
 	popq %r12;
 	popq %r13;
-	popq %r14;
-	popq %r15;
 	ret;
 
 .L__enc_xor3:
 	outunpack_enc3(xor);
 
 	popq %rbx;
-	popq %rbp;
 	popq %r12;
 	popq %r13;
-	popq %r14;
-	popq %r15;
 	ret;
 ENDPROC(__twofish_enc_blk_3way)
 
@@ -278,35 +290,31 @@ ENTRY(twofish_dec_blk_3way)
 	 *	%rsi: dst
 	 *	%rdx: src, RIO
 	 */
-	pushq %r15;
-	pushq %r14;
 	pushq %r13;
 	pushq %r12;
-	pushq %rbp;
 	pushq %rbx;
 
 	pushq %rsi; /* dst */
 
 	inpack_dec3();
 
-	decrypt_cycle3(RAB, RCD, 7);
-	decrypt_cycle3(RAB, RCD, 6);
-	decrypt_cycle3(RAB, RCD, 5);
-	decrypt_cycle3(RAB, RCD, 4);
-	decrypt_cycle3(RAB, RCD, 3);
-	decrypt_cycle3(RAB, RCD, 2);
-	decrypt_cycle3(RAB, RCD, 1);
-	decrypt_cycle3(RAB, RCD, 0);
+	push_cd();
+	decrypt_cycle3(RAB, CD, 7);
+	decrypt_cycle3(RAB, CD, 6);
+	decrypt_cycle3(RAB, CD, 5);
+	decrypt_cycle3(RAB, CD, 4);
+	decrypt_cycle3(RAB, CD, 3);
+	decrypt_cycle3(RAB, CD, 2);
+	decrypt_cycle3(RAB, CD, 1);
+	decrypt_cycle3(RAB, CD, 0);
+	pop_cd();
 
 	popq RIO; /* dst */
 
 	outunpack_dec3();
 
 	popq %rbx;
-	popq %rbp;
 	popq %r12;
 	popq %r13;
-	popq %r14;
-	popq %r15;
 	ret;
 ENDPROC(twofish_dec_blk_3way)
diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h
index 3f48f695d5e6..be63330c5511 100644
--- a/arch/x86/entry/calling.h
+++ b/arch/x86/entry/calling.h
@@ -97,80 +97,78 @@ For 32-bit we have the following conventions - kernel is built with
 
 #define SIZEOF_PTREGS	21*8
 
-	.macro ALLOC_PT_GPREGS_ON_STACK
-	addq	$-(15*8), %rsp
-	.endm
-
-	.macro SAVE_C_REGS_HELPER offset=0 rax=1 rcx=1 r8910=1 r11=1
-	.if \r11
-	movq %r11, 6*8+\offset(%rsp)
-	.endif
-	.if \r8910
-	movq %r10, 7*8+\offset(%rsp)
-	movq %r9,  8*8+\offset(%rsp)
-	movq %r8,  9*8+\offset(%rsp)
-	.endif
-	.if \rax
-	movq %rax, 10*8+\offset(%rsp)
+.macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax save_ret=0
+	/*
+	 * Push registers and sanitize registers of values that a
+	 * speculation attack might otherwise want to exploit. The
+	 * lower registers are likely clobbered well before they
+	 * could be put to use in a speculative execution gadget.
+	 * Interleave XOR with PUSH for better uop scheduling:
+	 */
+	.if \save_ret
+	pushq	%rsi		/* pt_regs->si */
+	movq	8(%rsp), %rsi	/* temporarily store the return address in %rsi */
+	movq	%rdi, 8(%rsp)	/* pt_regs->di (overwriting original return address) */
+	.else
+	pushq   %rdi		/* pt_regs->di */
+	pushq   %rsi		/* pt_regs->si */
 	.endif
-	.if \rcx
-	movq %rcx, 11*8+\offset(%rsp)
+	pushq	\rdx		/* pt_regs->dx */
+	pushq   %rcx		/* pt_regs->cx */
+	pushq   \rax		/* pt_regs->ax */
+	pushq   %r8		/* pt_regs->r8 */
+	xorl	%r8d, %r8d	/* nospec   r8 */
+	pushq   %r9		/* pt_regs->r9 */
+	xorl	%r9d, %r9d	/* nospec   r9 */
+	pushq   %r10		/* pt_regs->r10 */
+	xorl	%r10d, %r10d	/* nospec   r10 */
+	pushq   %r11		/* pt_regs->r11 */
+	xorl	%r11d, %r11d	/* nospec   r11*/
+	pushq	%rbx		/* pt_regs->rbx */
+	xorl    %ebx, %ebx	/* nospec   rbx*/
+	pushq	%rbp		/* pt_regs->rbp */
+	xorl    %ebp, %ebp	/* nospec   rbp*/
+	pushq	%r12		/* pt_regs->r12 */
+	xorl	%r12d, %r12d	/* nospec   r12*/
+	pushq	%r13		/* pt_regs->r13 */
+	xorl	%r13d, %r13d	/* nospec   r13*/
+	pushq	%r14		/* pt_regs->r14 */
+	xorl	%r14d, %r14d	/* nospec   r14*/
+	pushq	%r15		/* pt_regs->r15 */
+	xorl	%r15d, %r15d	/* nospec   r15*/
+	UNWIND_HINT_REGS
+	.if \save_ret
+	pushq	%rsi		/* return address on top of stack */
 	.endif
-	movq %rdx, 12*8+\offset(%rsp)
-	movq %rsi, 13*8+\offset(%rsp)
-	movq %rdi, 14*8+\offset(%rsp)
-	UNWIND_HINT_REGS offset=\offset extra=0
-	.endm
-	.macro SAVE_C_REGS offset=0
-	SAVE_C_REGS_HELPER \offset, 1, 1, 1, 1
-	.endm
-	.macro SAVE_C_REGS_EXCEPT_RAX_RCX offset=0
-	SAVE_C_REGS_HELPER \offset, 0, 0, 1, 1
-	.endm
-	.macro SAVE_C_REGS_EXCEPT_R891011
-	SAVE_C_REGS_HELPER 0, 1, 1, 0, 0
-	.endm
-	.macro SAVE_C_REGS_EXCEPT_RCX_R891011
-	SAVE_C_REGS_HELPER 0, 1, 0, 0, 0
-	.endm
-	.macro SAVE_C_REGS_EXCEPT_RAX_RCX_R11
-	SAVE_C_REGS_HELPER 0, 0, 0, 1, 0
-	.endm
-
-	.macro SAVE_EXTRA_REGS offset=0
-	movq %r15, 0*8+\offset(%rsp)
-	movq %r14, 1*8+\offset(%rsp)
-	movq %r13, 2*8+\offset(%rsp)
-	movq %r12, 3*8+\offset(%rsp)
-	movq %rbp, 4*8+\offset(%rsp)
-	movq %rbx, 5*8+\offset(%rsp)
-	UNWIND_HINT_REGS offset=\offset
-	.endm
-
-	.macro POP_EXTRA_REGS
+.endm
+
+.macro POP_REGS pop_rdi=1 skip_r11rcx=0
 	popq %r15
 	popq %r14
 	popq %r13
 	popq %r12
 	popq %rbp
 	popq %rbx
-	.endm
-
-	.macro POP_C_REGS
+	.if \skip_r11rcx
+	popq %rsi
+	.else
 	popq %r11
+	.endif
 	popq %r10
 	popq %r9
 	popq %r8
 	popq %rax
+	.if \skip_r11rcx
+	popq %rsi
+	.else
 	popq %rcx
+	.endif
 	popq %rdx
 	popq %rsi
+	.if \pop_rdi
 	popq %rdi
-	.endm
-
-	.macro icebp
-	.byte 0xf1
-	.endm
+	.endif
+.endm
 
 /*
  * This is a sneaky trick to help the unwinder find pt_regs on the stack.  The
@@ -178,17 +176,12 @@ For 32-bit we have the following conventions - kernel is built with
  * is just setting the LSB, which makes it an invalid stack address and is also
  * a signal to the unwinder that it's a pt_regs pointer in disguise.
  *
- * NOTE: This macro must be used *after* SAVE_EXTRA_REGS because it corrupts
+ * NOTE: This macro must be used *after* PUSH_AND_CLEAR_REGS because it corrupts
  * the original rbp.
  */
 .macro ENCODE_FRAME_POINTER ptregs_offset=0
 #ifdef CONFIG_FRAME_POINTER
-	.if \ptregs_offset
-		leaq \ptregs_offset(%rsp), %rbp
-	.else
-		mov %rsp, %rbp
-	.endif
-	orq	$0x1, %rbp
+	leaq 1+\ptregs_offset(%rsp), %rbp
 #endif
 .endm
 
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index d7d3cc24baf4..74f6eee15179 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -21,6 +21,7 @@
 #include <linux/export.h>
 #include <linux/context_tracking.h>
 #include <linux/user-return-notifier.h>
+#include <linux/nospec.h>
 #include <linux/uprobes.h>
 #include <linux/livepatch.h>
 #include <linux/syscalls.h>
@@ -153,6 +154,9 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
 		if (cached_flags & _TIF_UPROBE)
 			uprobe_notify_resume(regs);
 
+		if (cached_flags & _TIF_PATCH_PENDING)
+			klp_update_patch_state(current);
+
 		/* deal with pending signal delivery */
 		if (cached_flags & _TIF_SIGPENDING)
 			do_signal(regs);
@@ -165,9 +169,6 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
 		if (cached_flags & _TIF_USER_RETURN_NOTIFY)
 			fire_user_return_notifiers();
 
-		if (cached_flags & _TIF_PATCH_PENDING)
-			klp_update_patch_state(current);
-
 		/* Disable IRQs and retry */
 		local_irq_disable();
 
@@ -206,7 +207,7 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
 	 * special case only applies after poking regs and before the
 	 * very next return to user mode.
 	 */
-	current->thread.status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
+	ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED);
 #endif
 
 	user_enter_irqoff();
@@ -282,7 +283,8 @@ __visible void do_syscall_64(struct pt_regs *regs)
 	 * regs->orig_ax, which changes the behavior of some syscalls.
 	 */
 	if (likely((nr & __SYSCALL_MASK) < NR_syscalls)) {
-		regs->ax = sys_call_table[nr & __SYSCALL_MASK](
+		nr = array_index_nospec(nr & __SYSCALL_MASK, NR_syscalls);
+		regs->ax = sys_call_table[nr](
 			regs->di, regs->si, regs->dx,
 			regs->r10, regs->r8, regs->r9);
 	}
@@ -304,7 +306,7 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
 	unsigned int nr = (unsigned int)regs->orig_ax;
 
 #ifdef CONFIG_IA32_EMULATION
-	current->thread.status |= TS_COMPAT;
+	ti->status |= TS_COMPAT;
 #endif
 
 	if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
@@ -318,6 +320,7 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
 	}
 
 	if (likely(nr < IA32_NR_syscalls)) {
+		nr = array_index_nospec(nr, IA32_NR_syscalls);
 		/*
 		 * It's possible that a 32-bit syscall implementation
 		 * takes a 64-bit parameter but nonetheless assumes that
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 60c4c342316c..6ad064c8cf35 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -565,6 +565,11 @@ restore_all:
 .Lrestore_nocheck:
 	RESTORE_REGS 4				# skip orig_eax/error_code
 .Lirq_return:
+	/*
+	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
+	 * when returning from IPI handler and when returning from
+	 * scheduler to user-space.
+	 */
 	INTERRUPT_RETURN
 
 .section .fixup, "ax"
@@ -894,6 +899,9 @@ BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
 BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
 		 hyperv_vector_handler)
 
+BUILD_INTERRUPT3(hyperv_reenlightenment_vector, HYPERV_REENLIGHTENMENT_VECTOR,
+		 hyperv_reenlightenment_intr)
+
 #endif /* CONFIG_HYPERV */
 
 ENTRY(page_fault)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index ff6f8022612c..18ed349b4f83 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -55,7 +55,7 @@ END(native_usergs_sysret64)
 
 .macro TRACE_IRQS_FLAGS flags:req
 #ifdef CONFIG_TRACE_IRQFLAGS
-	bt	$9, \flags		/* interrupts off? */
+	btl	$9, \flags		/* interrupts off? */
 	jnc	1f
 	TRACE_IRQS_ON
 1:
@@ -213,7 +213,7 @@ ENTRY(entry_SYSCALL_64)
 
 	swapgs
 	/*
-	 * This path is not taken when PAGE_TABLE_ISOLATION is disabled so it
+	 * This path is only taken when PAGE_TABLE_ISOLATION is disabled so it
 	 * is not required to switch CR3.
 	 */
 	movq	%rsp, PER_CPU_VAR(rsp_scratch)
@@ -227,100 +227,15 @@ ENTRY(entry_SYSCALL_64)
 	pushq	%rcx				/* pt_regs->ip */
 GLOBAL(entry_SYSCALL_64_after_hwframe)
 	pushq	%rax				/* pt_regs->orig_ax */
-	pushq	%rdi				/* pt_regs->di */
-	pushq	%rsi				/* pt_regs->si */
-	pushq	%rdx				/* pt_regs->dx */
-	pushq	%rcx				/* pt_regs->cx */
-	pushq	$-ENOSYS			/* pt_regs->ax */
-	pushq	%r8				/* pt_regs->r8 */
-	pushq	%r9				/* pt_regs->r9 */
-	pushq	%r10				/* pt_regs->r10 */
-	pushq	%r11				/* pt_regs->r11 */
-	sub	$(6*8), %rsp			/* pt_regs->bp, bx, r12-15 not saved */
-	UNWIND_HINT_REGS extra=0
 
-	TRACE_IRQS_OFF
-
-	/*
-	 * If we need to do entry work or if we guess we'll need to do
-	 * exit work, go straight to the slow path.
-	 */
-	movq	PER_CPU_VAR(current_task), %r11
-	testl	$_TIF_WORK_SYSCALL_ENTRY|_TIF_ALLWORK_MASK, TASK_TI_flags(%r11)
-	jnz	entry_SYSCALL64_slow_path
-
-entry_SYSCALL_64_fastpath:
-	/*
-	 * Easy case: enable interrupts and issue the syscall.  If the syscall
-	 * needs pt_regs, we'll call a stub that disables interrupts again
-	 * and jumps to the slow path.
-	 */
-	TRACE_IRQS_ON
-	ENABLE_INTERRUPTS(CLBR_NONE)
-#if __SYSCALL_MASK == ~0
-	cmpq	$__NR_syscall_max, %rax
-#else
-	andl	$__SYSCALL_MASK, %eax
-	cmpl	$__NR_syscall_max, %eax
-#endif
-	ja	1f				/* return -ENOSYS (already in pt_regs->ax) */
-	movq	%r10, %rcx
-
-	/*
-	 * This call instruction is handled specially in stub_ptregs_64.
-	 * It might end up jumping to the slow path.  If it jumps, RAX
-	 * and all argument registers are clobbered.
-	 */
-#ifdef CONFIG_RETPOLINE
-	movq	sys_call_table(, %rax, 8), %rax
-	call	__x86_indirect_thunk_rax
-#else
-	call	*sys_call_table(, %rax, 8)
-#endif
-.Lentry_SYSCALL_64_after_fastpath_call:
+	PUSH_AND_CLEAR_REGS rax=$-ENOSYS
 
-	movq	%rax, RAX(%rsp)
-1:
-
-	/*
-	 * If we get here, then we know that pt_regs is clean for SYSRET64.
-	 * If we see that no exit work is required (which we are required
-	 * to check with IRQs off), then we can go straight to SYSRET64.
-	 */
-	DISABLE_INTERRUPTS(CLBR_ANY)
 	TRACE_IRQS_OFF
-	movq	PER_CPU_VAR(current_task), %r11
-	testl	$_TIF_ALLWORK_MASK, TASK_TI_flags(%r11)
-	jnz	1f
-
-	LOCKDEP_SYS_EXIT
-	TRACE_IRQS_ON		/* user mode is traced as IRQs on */
-	movq	RIP(%rsp), %rcx
-	movq	EFLAGS(%rsp), %r11
-	addq	$6*8, %rsp	/* skip extra regs -- they were preserved */
-	UNWIND_HINT_EMPTY
-	jmp	.Lpop_c_regs_except_rcx_r11_and_sysret
-
-1:
-	/*
-	 * The fast path looked good when we started, but something changed
-	 * along the way and we need to switch to the slow path.  Calling
-	 * raise(3) will trigger this, for example.  IRQs are off.
-	 */
-	TRACE_IRQS_ON
-	ENABLE_INTERRUPTS(CLBR_ANY)
-	SAVE_EXTRA_REGS
-	movq	%rsp, %rdi
-	call	syscall_return_slowpath	/* returns with IRQs disabled */
-	jmp	return_from_SYSCALL_64
 
-entry_SYSCALL64_slow_path:
 	/* IRQs are off. */
-	SAVE_EXTRA_REGS
 	movq	%rsp, %rdi
 	call	do_syscall_64		/* returns with IRQs disabled */
 
-return_from_SYSCALL_64:
 	TRACE_IRQS_IRETQ		/* we're about to change IF */
 
 	/*
@@ -392,16 +307,7 @@ return_from_SYSCALL_64:
 syscall_return_via_sysret:
 	/* rcx and r11 are already restored (see code above) */
 	UNWIND_HINT_EMPTY
-	POP_EXTRA_REGS
-.Lpop_c_regs_except_rcx_r11_and_sysret:
-	popq	%rsi	/* skip r11 */
-	popq	%r10
-	popq	%r9
-	popq	%r8
-	popq	%rax
-	popq	%rsi	/* skip rcx */
-	popq	%rdx
-	popq	%rsi
+	POP_REGS pop_rdi=0 skip_r11rcx=1
 
 	/*
 	 * Now all regs are restored except RSP and RDI.
@@ -424,47 +330,6 @@ syscall_return_via_sysret:
 	USERGS_SYSRET64
 END(entry_SYSCALL_64)
 
-ENTRY(stub_ptregs_64)
-	/*
-	 * Syscalls marked as needing ptregs land here.
-	 * If we are on the fast path, we need to save the extra regs,
-	 * which we achieve by trying again on the slow path.  If we are on
-	 * the slow path, the extra regs are already saved.
-	 *
-	 * RAX stores a pointer to the C function implementing the syscall.
-	 * IRQs are on.
-	 */
-	cmpq	$.Lentry_SYSCALL_64_after_fastpath_call, (%rsp)
-	jne	1f
-
-	/*
-	 * Called from fast path -- disable IRQs again, pop return address
-	 * and jump to slow path
-	 */
-	DISABLE_INTERRUPTS(CLBR_ANY)
-	TRACE_IRQS_OFF
-	popq	%rax
-	UNWIND_HINT_REGS extra=0
-	jmp	entry_SYSCALL64_slow_path
-
-1:
-	JMP_NOSPEC %rax				/* Called from C */
-END(stub_ptregs_64)
-
-.macro ptregs_stub func
-ENTRY(ptregs_\func)
-	UNWIND_HINT_FUNC
-	leaq	\func(%rip), %rax
-	jmp	stub_ptregs_64
-END(ptregs_\func)
-.endm
-
-/* Instantiate ptregs_stub for each ptregs-using syscall */
-#define __SYSCALL_64_QUAL_(sym)
-#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_stub sym
-#define __SYSCALL_64(nr, sym, qual) __SYSCALL_64_QUAL_##qual(sym)
-#include <asm/syscalls_64.h>
-
 /*
  * %rdi: prev task
  * %rsi: next task
@@ -583,9 +448,19 @@ END(irq_entries_start)
  *
  * The invariant is that, if irq_count != -1, then the IRQ stack is in use.
  */
-.macro ENTER_IRQ_STACK regs=1 old_rsp
+.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0
 	DEBUG_ENTRY_ASSERT_IRQS_OFF
+
+	.if \save_ret
+	/*
+	 * If save_ret is set, the original stack contains one additional
+	 * entry -- the return address. Therefore, move the address one
+	 * entry below %rsp to \old_rsp.
+	 */
+	leaq	8(%rsp), \old_rsp
+	.else
 	movq	%rsp, \old_rsp
+	.endif
 
 	.if \regs
 	UNWIND_HINT_REGS base=\old_rsp
@@ -631,6 +506,15 @@ END(irq_entries_start)
 	.if \regs
 	UNWIND_HINT_REGS indirect=1
 	.endif
+
+	.if \save_ret
+	/*
+	 * Push the return address to the stack. This return address can
+	 * be found at the "real" original RSP, which was offset by 8 at
+	 * the beginning of this macro.
+	 */
+	pushq	-8(\old_rsp)
+	.endif
 .endm
 
 /*
@@ -654,29 +538,65 @@ END(irq_entries_start)
 .endm
 
 /*
- * Interrupt entry/exit.
+ * Interrupt entry helper function.
  *
- * Interrupt entry points save only callee clobbered registers in fast path.
- *
- * Entry runs with interrupts off.
+ * Entry runs with interrupts off. Stack layout at entry:
+ * +----------------------------------------------------+
+ * | regs->ss						|
+ * | regs->rsp						|
+ * | regs->eflags					|
+ * | regs->cs						|
+ * | regs->ip						|
+ * +----------------------------------------------------+
+ * | regs->orig_ax = ~(interrupt number)		|
+ * +----------------------------------------------------+
+ * | return address					|
+ * +----------------------------------------------------+
  */
-
-/* 0(%rsp): ~(interrupt number) */
-	.macro interrupt func
+ENTRY(interrupt_entry)
+	UNWIND_HINT_FUNC
+	ASM_CLAC
 	cld
 
-	testb	$3, CS-ORIG_RAX(%rsp)
+	testb	$3, CS-ORIG_RAX+8(%rsp)
 	jz	1f
 	SWAPGS
-	call	switch_to_thread_stack
+
+	/*
+	 * Switch to the thread stack. The IRET frame and orig_ax are
+	 * on the stack, as well as the return address. RDI..R12 are
+	 * not (yet) on the stack and space has not (yet) been
+	 * allocated for them.
+	 */
+	pushq	%rdi
+
+	/* Need to switch before accessing the thread stack. */
+	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi
+	movq	%rsp, %rdi
+	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
+
+	 /*
+	  * We have RDI, return address, and orig_ax on the stack on
+	  * top of the IRET frame. That means offset=24
+	  */
+	UNWIND_HINT_IRET_REGS base=%rdi offset=24
+
+	pushq	7*8(%rdi)		/* regs->ss */
+	pushq	6*8(%rdi)		/* regs->rsp */
+	pushq	5*8(%rdi)		/* regs->eflags */
+	pushq	4*8(%rdi)		/* regs->cs */
+	pushq	3*8(%rdi)		/* regs->ip */
+	pushq	2*8(%rdi)		/* regs->orig_ax */
+	pushq	8(%rdi)			/* return address */
+	UNWIND_HINT_FUNC
+
+	movq	(%rdi), %rdi
 1:
 
-	ALLOC_PT_GPREGS_ON_STACK
-	SAVE_C_REGS
-	SAVE_EXTRA_REGS
-	ENCODE_FRAME_POINTER
+	PUSH_AND_CLEAR_REGS save_ret=1
+	ENCODE_FRAME_POINTER 8
 
-	testb	$3, CS(%rsp)
+	testb	$3, CS+8(%rsp)
 	jz	1f
 
 	/*
@@ -684,7 +604,7 @@ END(irq_entries_start)
 	 *
 	 * We need to tell lockdep that IRQs are off.  We can't do this until
 	 * we fix gsbase, and we should do it before enter_from_user_mode
-	 * (which can take locks).  Since TRACE_IRQS_OFF idempotent,
+	 * (which can take locks).  Since TRACE_IRQS_OFF is idempotent,
 	 * the simplest way to handle it is to just call it twice if
 	 * we enter from user mode.  There's no reason to optimize this since
 	 * TRACE_IRQS_OFF is a no-op if lockdep is off.
@@ -694,12 +614,15 @@ END(irq_entries_start)
 	CALL_enter_from_user_mode
 
 1:
-	ENTER_IRQ_STACK old_rsp=%rdi
+	ENTER_IRQ_STACK old_rsp=%rdi save_ret=1
 	/* We entered an interrupt context - irqs are off: */
 	TRACE_IRQS_OFF
 
-	call	\func	/* rdi points to pt_regs */
-	.endm
+	ret
+END(interrupt_entry)
+
+
+/* Interrupt entry/exit. */
 
 	/*
 	 * The interrupt stubs push (~vector+0x80) onto the stack and
@@ -707,9 +630,10 @@ END(irq_entries_start)
 	 */
 	.p2align CONFIG_X86_L1_CACHE_SHIFT
 common_interrupt:
-	ASM_CLAC
 	addq	$-0x80, (%rsp)			/* Adjust vector to [-256, -1] range */
-	interrupt do_IRQ
+	call	interrupt_entry
+	UNWIND_HINT_REGS indirect=1
+	call	do_IRQ	/* rdi points to pt_regs */
 	/* 0(%rsp): old RSP */
 ret_from_intr:
 	DISABLE_INTERRUPTS(CLBR_ANY)
@@ -734,15 +658,7 @@ GLOBAL(swapgs_restore_regs_and_return_to_usermode)
 	ud2
 1:
 #endif
-	POP_EXTRA_REGS
-	popq	%r11
-	popq	%r10
-	popq	%r9
-	popq	%r8
-	popq	%rax
-	popq	%rcx
-	popq	%rdx
-	popq	%rsi
+	POP_REGS pop_rdi=0
 
 	/*
 	 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
@@ -800,9 +716,12 @@ GLOBAL(restore_regs_and_return_to_kernel)
 	ud2
 1:
 #endif
-	POP_EXTRA_REGS
-	POP_C_REGS
+	POP_REGS
 	addq	$8, %rsp	/* skip regs->orig_ax */
+	/*
+	 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
+	 * when returning from IPI handler.
+	 */
 	INTERRUPT_RETURN
 
 ENTRY(native_iret)
@@ -907,10 +826,11 @@ END(common_interrupt)
 .macro apicinterrupt3 num sym do_sym
 ENTRY(\sym)
 	UNWIND_HINT_IRET_REGS
-	ASM_CLAC
 	pushq	$~(\num)
 .Lcommon_\sym:
-	interrupt \do_sym
+	call	interrupt_entry
+	UNWIND_HINT_REGS indirect=1
+	call	\do_sym	/* rdi points to pt_regs */
 	jmp	ret_from_intr
 END(\sym)
 .endm
@@ -973,34 +893,6 @@ apicinterrupt IRQ_WORK_VECTOR			irq_work_interrupt		smp_irq_work_interrupt
  */
 #define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + ((x) - 1) * 8)
 
-/*
- * Switch to the thread stack.  This is called with the IRET frame and
- * orig_ax on the stack.  (That is, RDI..R12 are not on the stack and
- * space has not been allocated for them.)
- */
-ENTRY(switch_to_thread_stack)
-	UNWIND_HINT_FUNC
-
-	pushq	%rdi
-	/* Need to switch before accessing the thread stack. */
-	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi
-	movq	%rsp, %rdi
-	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
-	UNWIND_HINT sp_offset=16 sp_reg=ORC_REG_DI
-
-	pushq	7*8(%rdi)		/* regs->ss */
-	pushq	6*8(%rdi)		/* regs->rsp */
-	pushq	5*8(%rdi)		/* regs->eflags */
-	pushq	4*8(%rdi)		/* regs->cs */
-	pushq	3*8(%rdi)		/* regs->ip */
-	pushq	2*8(%rdi)		/* regs->orig_ax */
-	pushq	8(%rdi)			/* return address */
-	UNWIND_HINT_FUNC
-
-	movq	(%rdi), %rdi
-	ret
-END(switch_to_thread_stack)
-
 .macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1
 ENTRY(\sym)
 	UNWIND_HINT_IRET_REGS offset=\has_error_code*8
@@ -1016,10 +908,8 @@ ENTRY(\sym)
 	pushq	$-1				/* ORIG_RAX: no syscall to restart */
 	.endif
 
-	ALLOC_PT_GPREGS_ON_STACK
-
 	.if \paranoid < 2
-	testb	$3, CS(%rsp)			/* If coming from userspace, switch stacks */
+	testb	$3, CS-ORIG_RAX(%rsp)		/* If coming from userspace, switch stacks */
 	jnz	.Lfrom_usermode_switch_stack_\@
 	.endif
 
@@ -1229,9 +1119,7 @@ ENTRY(xen_failsafe_callback)
 	addq	$0x30, %rsp
 	UNWIND_HINT_IRET_REGS
 	pushq	$-1 /* orig_ax = -1 => not a system call */
-	ALLOC_PT_GPREGS_ON_STACK
-	SAVE_C_REGS
-	SAVE_EXTRA_REGS
+	PUSH_AND_CLEAR_REGS
 	ENCODE_FRAME_POINTER
 	jmp	error_exit
 END(xen_failsafe_callback)
@@ -1244,10 +1132,13 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
 #if IS_ENABLED(CONFIG_HYPERV)
 apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
 	hyperv_callback_vector hyperv_vector_handler
+
+apicinterrupt3 HYPERV_REENLIGHTENMENT_VECTOR \
+	hyperv_reenlightenment_vector hyperv_reenlightenment_intr
 #endif /* CONFIG_HYPERV */
 
 idtentry debug			do_debug		has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
-idtentry int3			do_int3			has_error_code=0	paranoid=1 shift_ist=DEBUG_STACK
+idtentry int3			do_int3			has_error_code=0
 idtentry stack_segment		do_stack_segment	has_error_code=1
 
 #ifdef CONFIG_XEN
@@ -1275,8 +1166,7 @@ idtentry machine_check		do_mce			has_error_code=0	paranoid=1
 ENTRY(paranoid_entry)
 	UNWIND_HINT_FUNC
 	cld
-	SAVE_C_REGS 8
-	SAVE_EXTRA_REGS 8
+	PUSH_AND_CLEAR_REGS save_ret=1
 	ENCODE_FRAME_POINTER 8
 	movl	$1, %ebx
 	movl	$MSR_GS_BASE, %ecx
@@ -1316,21 +1206,20 @@ ENTRY(paranoid_exit)
 	jmp	.Lparanoid_exit_restore
 .Lparanoid_exit_no_swapgs:
 	TRACE_IRQS_IRETQ_DEBUG
+	RESTORE_CR3	scratch_reg=%rbx save_reg=%r14
 .Lparanoid_exit_restore:
 	jmp restore_regs_and_return_to_kernel
 END(paranoid_exit)
 
 /*
- * Save all registers in pt_regs, and switch gs if needed.
+ * Save all registers in pt_regs, and switch GS if needed.
  * Return: EBX=0: came from user mode; EBX=1: otherwise
  */
 ENTRY(error_entry)
 	UNWIND_HINT_FUNC
 	cld
-	SAVE_C_REGS 8
-	SAVE_EXTRA_REGS 8
+	PUSH_AND_CLEAR_REGS save_ret=1
 	ENCODE_FRAME_POINTER 8
-	xorl	%ebx, %ebx
 	testb	$3, CS+8(%rsp)
 	jz	.Lerror_kernelspace
 
@@ -1511,22 +1400,7 @@ ENTRY(nmi)
 	pushq	1*8(%rdx)	/* pt_regs->rip */
 	UNWIND_HINT_IRET_REGS
 	pushq   $-1		/* pt_regs->orig_ax */
-	pushq   %rdi		/* pt_regs->di */
-	pushq   %rsi		/* pt_regs->si */
-	pushq   (%rdx)		/* pt_regs->dx */
-	pushq   %rcx		/* pt_regs->cx */
-	pushq   %rax		/* pt_regs->ax */
-	pushq   %r8		/* pt_regs->r8 */
-	pushq   %r9		/* pt_regs->r9 */
-	pushq   %r10		/* pt_regs->r10 */
-	pushq   %r11		/* pt_regs->r11 */
-	pushq	%rbx		/* pt_regs->rbx */
-	pushq	%rbp		/* pt_regs->rbp */
-	pushq	%r12		/* pt_regs->r12 */
-	pushq	%r13		/* pt_regs->r13 */
-	pushq	%r14		/* pt_regs->r14 */
-	pushq	%r15		/* pt_regs->r15 */
-	UNWIND_HINT_REGS
+	PUSH_AND_CLEAR_REGS rdx=(%rdx)
 	ENCODE_FRAME_POINTER
 
 	/*
@@ -1736,7 +1610,6 @@ end_repeat_nmi:
 	 * frame to point back to repeat_nmi.
 	 */
 	pushq	$-1				/* ORIG_RAX: no syscall to restart */
-	ALLOC_PT_GPREGS_ON_STACK
 
 	/*
 	 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
@@ -1760,8 +1633,7 @@ end_repeat_nmi:
 nmi_swapgs:
 	SWAPGS_UNSAFE_STACK
 nmi_restore:
-	POP_EXTRA_REGS
-	POP_C_REGS
+	POP_REGS
 
 	/*
 	 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S
index 98d5358e4041..08425c42f8b7 100644
--- a/arch/x86/entry/entry_64_compat.S
+++ b/arch/x86/entry/entry_64_compat.S
@@ -85,15 +85,25 @@ ENTRY(entry_SYSENTER_compat)
 	pushq	%rcx			/* pt_regs->cx */
 	pushq	$-ENOSYS		/* pt_regs->ax */
 	pushq   $0			/* pt_regs->r8  = 0 */
+	xorl	%r8d, %r8d		/* nospec   r8 */
 	pushq   $0			/* pt_regs->r9  = 0 */
+	xorl	%r9d, %r9d		/* nospec   r9 */
 	pushq   $0			/* pt_regs->r10 = 0 */
+	xorl	%r10d, %r10d		/* nospec   r10 */
 	pushq   $0			/* pt_regs->r11 = 0 */
+	xorl	%r11d, %r11d		/* nospec   r11 */
 	pushq   %rbx                    /* pt_regs->rbx */
+	xorl	%ebx, %ebx		/* nospec   rbx */
 	pushq   %rbp                    /* pt_regs->rbp (will be overwritten) */
+	xorl	%ebp, %ebp		/* nospec   rbp */
 	pushq   $0			/* pt_regs->r12 = 0 */
+	xorl	%r12d, %r12d		/* nospec   r12 */
 	pushq   $0			/* pt_regs->r13 = 0 */
+	xorl	%r13d, %r13d		/* nospec   r13 */
 	pushq   $0			/* pt_regs->r14 = 0 */
+	xorl	%r14d, %r14d		/* nospec   r14 */
 	pushq   $0			/* pt_regs->r15 = 0 */
+	xorl	%r15d, %r15d		/* nospec   r15 */
 	cld
 
 	/*
@@ -214,15 +224,25 @@ GLOBAL(entry_SYSCALL_compat_after_hwframe)
 	pushq	%rbp			/* pt_regs->cx (stashed in bp) */
 	pushq	$-ENOSYS		/* pt_regs->ax */
 	pushq   $0			/* pt_regs->r8  = 0 */
+	xorl	%r8d, %r8d		/* nospec   r8 */
 	pushq   $0			/* pt_regs->r9  = 0 */
+	xorl	%r9d, %r9d		/* nospec   r9 */
 	pushq   $0			/* pt_regs->r10 = 0 */
+	xorl	%r10d, %r10d		/* nospec   r10 */
 	pushq   $0			/* pt_regs->r11 = 0 */
+	xorl	%r11d, %r11d		/* nospec   r11 */
 	pushq   %rbx                    /* pt_regs->rbx */
+	xorl	%ebx, %ebx		/* nospec   rbx */
 	pushq   %rbp                    /* pt_regs->rbp (will be overwritten) */
+	xorl	%ebp, %ebp		/* nospec   rbp */
 	pushq   $0			/* pt_regs->r12 = 0 */
+	xorl	%r12d, %r12d		/* nospec   r12 */
 	pushq   $0			/* pt_regs->r13 = 0 */
+	xorl	%r13d, %r13d		/* nospec   r13 */
 	pushq   $0			/* pt_regs->r14 = 0 */
+	xorl	%r14d, %r14d		/* nospec   r14 */
 	pushq   $0			/* pt_regs->r15 = 0 */
+	xorl	%r15d, %r15d		/* nospec   r15 */
 
 	/*
 	 * User mode is traced as though IRQs are on, and SYSENTER
@@ -278,9 +298,9 @@ sysret32_from_system_call:
 	 */
 	SWITCH_TO_USER_CR3_NOSTACK scratch_reg=%r8 scratch_reg2=%r9
 
-	xorq	%r8, %r8
-	xorq	%r9, %r9
-	xorq	%r10, %r10
+	xorl	%r8d, %r8d
+	xorl	%r9d, %r9d
+	xorl	%r10d, %r10d
 	swapgs
 	sysretl
 END(entry_SYSCALL_compat)
@@ -327,26 +347,47 @@ ENTRY(entry_INT80_compat)
 	 */
 	movl	%eax, %eax
 
+	/* switch to thread stack expects orig_ax and rdi to be pushed */
 	pushq	%rax			/* pt_regs->orig_ax */
+	pushq	%rdi			/* pt_regs->di */
 
-	/* switch to thread stack expects orig_ax to be pushed */
-	call	switch_to_thread_stack
+	/* Need to switch before accessing the thread stack. */
+	SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi
+	movq	%rsp, %rdi
+	movq	PER_CPU_VAR(cpu_current_top_of_stack), %rsp
 
-	pushq	%rdi			/* pt_regs->di */
+	pushq	6*8(%rdi)		/* regs->ss */
+	pushq	5*8(%rdi)		/* regs->rsp */
+	pushq	4*8(%rdi)		/* regs->eflags */
+	pushq	3*8(%rdi)		/* regs->cs */
+	pushq	2*8(%rdi)		/* regs->ip */
+	pushq	1*8(%rdi)		/* regs->orig_ax */
+
+	pushq	(%rdi)			/* pt_regs->di */
 	pushq	%rsi			/* pt_regs->si */
 	pushq	%rdx			/* pt_regs->dx */
 	pushq	%rcx			/* pt_regs->cx */
 	pushq	$-ENOSYS		/* pt_regs->ax */
 	pushq   $0			/* pt_regs->r8  = 0 */
+	xorl	%r8d, %r8d		/* nospec   r8 */
 	pushq   $0			/* pt_regs->r9  = 0 */
+	xorl	%r9d, %r9d		/* nospec   r9 */
 	pushq   $0			/* pt_regs->r10 = 0 */
+	xorl	%r10d, %r10d		/* nospec   r10 */
 	pushq   $0			/* pt_regs->r11 = 0 */
+	xorl	%r11d, %r11d		/* nospec   r11 */
 	pushq   %rbx                    /* pt_regs->rbx */
+	xorl	%ebx, %ebx		/* nospec   rbx */
 	pushq   %rbp                    /* pt_regs->rbp */
+	xorl	%ebp, %ebp		/* nospec   rbp */
 	pushq   %r12                    /* pt_regs->r12 */
+	xorl	%r12d, %r12d		/* nospec   r12 */
 	pushq   %r13                    /* pt_regs->r13 */
+	xorl	%r13d, %r13d		/* nospec   r13 */
 	pushq   %r14                    /* pt_regs->r14 */
+	xorl	%r14d, %r14d		/* nospec   r14 */
 	pushq   %r15                    /* pt_regs->r15 */
+	xorl	%r15d, %r15d		/* nospec   r15 */
 	cld
 
 	/*
@@ -363,15 +404,3 @@ ENTRY(entry_INT80_compat)
 	TRACE_IRQS_ON
 	jmp	swapgs_restore_regs_and_return_to_usermode
 END(entry_INT80_compat)
-
-ENTRY(stub32_clone)
-	/*
-	 * The 32-bit clone ABI is: clone(..., int tls_val, int *child_tidptr).
-	 * The 64-bit clone ABI is: clone(..., int *child_tidptr, int tls_val).
-	 *
-	 * The native 64-bit kernel's sys_clone() implements the latter,
-	 * so we need to swap arguments here before calling it:
-	 */
-	xchg	%r8, %rcx
-	jmp	sys_clone
-ENDPROC(stub32_clone)
diff --git a/arch/x86/entry/syscall_64.c b/arch/x86/entry/syscall_64.c
index 9c09775e589d..c176d2fab1da 100644
--- a/arch/x86/entry/syscall_64.c
+++ b/arch/x86/entry/syscall_64.c
@@ -7,14 +7,11 @@
 #include <asm/asm-offsets.h>
 #include <asm/syscall.h>
 
-#define __SYSCALL_64_QUAL_(sym) sym
-#define __SYSCALL_64_QUAL_ptregs(sym) ptregs_##sym
-
-#define __SYSCALL_64(nr, sym, qual) extern asmlinkage long __SYSCALL_64_QUAL_##qual(sym)(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
+#define __SYSCALL_64(nr, sym, qual) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
 #include <asm/syscalls_64.h>
 #undef __SYSCALL_64
 
-#define __SYSCALL_64(nr, sym, qual) [nr] = __SYSCALL_64_QUAL_##qual(sym),
+#define __SYSCALL_64(nr, sym, qual) [nr] = sym,
 
 extern long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
 
diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl
index 448ac2161112..2a5e99cff859 100644
--- a/arch/x86/entry/syscalls/syscall_32.tbl
+++ b/arch/x86/entry/syscalls/syscall_32.tbl
@@ -8,12 +8,12 @@
 #
 0	i386	restart_syscall		sys_restart_syscall
 1	i386	exit			sys_exit
-2	i386	fork			sys_fork			sys_fork
+2	i386	fork			sys_fork
 3	i386	read			sys_read
 4	i386	write			sys_write
 5	i386	open			sys_open			compat_sys_open
 6	i386	close			sys_close
-7	i386	waitpid			sys_waitpid			sys32_waitpid
+7	i386	waitpid			sys_waitpid			compat_sys_x86_waitpid
 8	i386	creat			sys_creat
 9	i386	link			sys_link
 10	i386	unlink			sys_unlink
@@ -78,7 +78,7 @@
 69	i386	ssetmask		sys_ssetmask
 70	i386	setreuid		sys_setreuid16
 71	i386	setregid		sys_setregid16
-72	i386	sigsuspend		sys_sigsuspend			sys_sigsuspend
+72	i386	sigsuspend		sys_sigsuspend
 73	i386	sigpending		sys_sigpending			compat_sys_sigpending
 74	i386	sethostname		sys_sethostname
 75	i386	setrlimit		sys_setrlimit			compat_sys_setrlimit
@@ -96,7 +96,7 @@
 87	i386	swapon			sys_swapon
 88	i386	reboot			sys_reboot
 89	i386	readdir			sys_old_readdir			compat_sys_old_readdir
-90	i386	mmap			sys_old_mmap			sys32_mmap
+90	i386	mmap			sys_old_mmap			compat_sys_x86_mmap
 91	i386	munmap			sys_munmap
 92	i386	truncate		sys_truncate			compat_sys_truncate
 93	i386	ftruncate		sys_ftruncate			compat_sys_ftruncate
@@ -126,7 +126,7 @@
 117	i386	ipc			sys_ipc				compat_sys_ipc
 118	i386	fsync			sys_fsync
 119	i386	sigreturn		sys_sigreturn			sys32_sigreturn
-120	i386	clone			sys_clone			stub32_clone
+120	i386	clone			sys_clone			compat_sys_x86_clone
 121	i386	setdomainname		sys_setdomainname
 122	i386	uname			sys_newuname
 123	i386	modify_ldt		sys_modify_ldt
@@ -186,8 +186,8 @@
 177	i386	rt_sigtimedwait		sys_rt_sigtimedwait		compat_sys_rt_sigtimedwait
 178	i386	rt_sigqueueinfo		sys_rt_sigqueueinfo		compat_sys_rt_sigqueueinfo
 179	i386	rt_sigsuspend		sys_rt_sigsuspend
-180	i386	pread64			sys_pread64			sys32_pread
-181	i386	pwrite64		sys_pwrite64			sys32_pwrite
+180	i386	pread64			sys_pread64			compat_sys_x86_pread
+181	i386	pwrite64		sys_pwrite64			compat_sys_x86_pwrite
 182	i386	chown			sys_chown16
 183	i386	getcwd			sys_getcwd
 184	i386	capget			sys_capget
@@ -196,14 +196,14 @@
 187	i386	sendfile		sys_sendfile			compat_sys_sendfile
 188	i386	getpmsg
 189	i386	putpmsg
-190	i386	vfork			sys_vfork			sys_vfork
+190	i386	vfork			sys_vfork
 191	i386	ugetrlimit		sys_getrlimit			compat_sys_getrlimit
 192	i386	mmap2			sys_mmap_pgoff
-193	i386	truncate64		sys_truncate64			sys32_truncate64
-194	i386	ftruncate64		sys_ftruncate64			sys32_ftruncate64
-195	i386	stat64			sys_stat64			sys32_stat64
-196	i386	lstat64			sys_lstat64			sys32_lstat64
-197	i386	fstat64			sys_fstat64			sys32_fstat64
+193	i386	truncate64		sys_truncate64			compat_sys_x86_truncate64
+194	i386	ftruncate64		sys_ftruncate64			compat_sys_x86_ftruncate64
+195	i386	stat64			sys_stat64			compat_sys_x86_stat64
+196	i386	lstat64			sys_lstat64			compat_sys_x86_lstat64
+197	i386	fstat64			sys_fstat64			compat_sys_x86_fstat64
 198	i386	lchown32		sys_lchown
 199	i386	getuid32		sys_getuid
 200	i386	getgid32		sys_getgid
@@ -231,7 +231,7 @@
 # 222 is unused
 # 223 is unused
 224	i386	gettid			sys_gettid
-225	i386	readahead		sys_readahead			sys32_readahead
+225	i386	readahead		sys_readahead			compat_sys_x86_readahead
 226	i386	setxattr		sys_setxattr
 227	i386	lsetxattr		sys_lsetxattr
 228	i386	fsetxattr		sys_fsetxattr
@@ -256,7 +256,7 @@
 247	i386	io_getevents		sys_io_getevents		compat_sys_io_getevents
 248	i386	io_submit		sys_io_submit			compat_sys_io_submit
 249	i386	io_cancel		sys_io_cancel
-250	i386	fadvise64		sys_fadvise64			sys32_fadvise64
+250	i386	fadvise64		sys_fadvise64			compat_sys_x86_fadvise64
 # 251 is available for reuse (was briefly sys_set_zone_reclaim)
 252	i386	exit_group		sys_exit_group
 253	i386	lookup_dcookie		sys_lookup_dcookie		compat_sys_lookup_dcookie
@@ -278,7 +278,7 @@
 269	i386	fstatfs64		sys_fstatfs64			compat_sys_fstatfs64
 270	i386	tgkill			sys_tgkill
 271	i386	utimes			sys_utimes			compat_sys_utimes
-272	i386	fadvise64_64		sys_fadvise64_64		sys32_fadvise64_64
+272	i386	fadvise64_64		sys_fadvise64_64		compat_sys_x86_fadvise64_64
 273	i386	vserver
 274	i386	mbind			sys_mbind
 275	i386	get_mempolicy		sys_get_mempolicy		compat_sys_get_mempolicy
@@ -306,7 +306,7 @@
 297	i386	mknodat			sys_mknodat
 298	i386	fchownat		sys_fchownat
 299	i386	futimesat		sys_futimesat			compat_sys_futimesat
-300	i386	fstatat64		sys_fstatat64			sys32_fstatat
+300	i386	fstatat64		sys_fstatat64			compat_sys_x86_fstatat
 301	i386	unlinkat		sys_unlinkat
 302	i386	renameat		sys_renameat
 303	i386	linkat			sys_linkat
@@ -320,7 +320,7 @@
 311	i386	set_robust_list		sys_set_robust_list		compat_sys_set_robust_list
 312	i386	get_robust_list		sys_get_robust_list		compat_sys_get_robust_list
 313	i386	splice			sys_splice
-314	i386	sync_file_range		sys_sync_file_range		sys32_sync_file_range
+314	i386	sync_file_range		sys_sync_file_range		compat_sys_x86_sync_file_range
 315	i386	tee			sys_tee
 316	i386	vmsplice		sys_vmsplice			compat_sys_vmsplice
 317	i386	move_pages		sys_move_pages			compat_sys_move_pages
@@ -330,7 +330,7 @@
 321	i386	signalfd		sys_signalfd			compat_sys_signalfd
 322	i386	timerfd_create		sys_timerfd_create
 323	i386	eventfd			sys_eventfd
-324	i386	fallocate		sys_fallocate			sys32_fallocate
+324	i386	fallocate		sys_fallocate			compat_sys_x86_fallocate
 325	i386	timerfd_settime		sys_timerfd_settime		compat_sys_timerfd_settime
 326	i386	timerfd_gettime		sys_timerfd_gettime		compat_sys_timerfd_gettime
 327	i386	signalfd4		sys_signalfd4			compat_sys_signalfd4
diff --git a/arch/x86/entry/vdso/vdso32/vclock_gettime.c b/arch/x86/entry/vdso/vdso32/vclock_gettime.c
index 7780bbfb06ef..9242b28418d5 100644
--- a/arch/x86/entry/vdso/vdso32/vclock_gettime.c
+++ b/arch/x86/entry/vdso/vdso32/vclock_gettime.c
@@ -5,8 +5,6 @@
 #undef CONFIG_OPTIMIZE_INLINING
 #endif
 
-#undef CONFIG_X86_PPRO_FENCE
-
 #ifdef CONFIG_X86_64
 
 /*
diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c
index 577fa8adb785..317be365bce3 100644
--- a/arch/x86/entry/vsyscall/vsyscall_64.c
+++ b/arch/x86/entry/vsyscall/vsyscall_64.c
@@ -42,10 +42,8 @@
 #define CREATE_TRACE_POINTS
 #include "vsyscall_trace.h"
 
-static enum { EMULATE, NATIVE, NONE } vsyscall_mode =
-#if defined(CONFIG_LEGACY_VSYSCALL_NATIVE)
-	NATIVE;
-#elif defined(CONFIG_LEGACY_VSYSCALL_NONE)
+static enum { EMULATE, NONE } vsyscall_mode =
+#ifdef CONFIG_LEGACY_VSYSCALL_NONE
 	NONE;
 #else
 	EMULATE;
@@ -56,8 +54,6 @@ static int __init vsyscall_setup(char *str)
 	if (str) {
 		if (!strcmp("emulate", str))
 			vsyscall_mode = EMULATE;
-		else if (!strcmp("native", str))
-			vsyscall_mode = NATIVE;
 		else if (!strcmp("none", str))
 			vsyscall_mode = NONE;
 		else
@@ -139,10 +135,6 @@ bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
 
 	WARN_ON_ONCE(address != regs->ip);
 
-	/* This should be unreachable in NATIVE mode. */
-	if (WARN_ON(vsyscall_mode == NATIVE))
-		return false;
-
 	if (vsyscall_mode == NONE) {
 		warn_bad_vsyscall(KERN_INFO, regs,
 				  "vsyscall attempted with vsyscall=none");
@@ -355,7 +347,7 @@ void __init set_vsyscall_pgtable_user_bits(pgd_t *root)
 	set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER));
 	p4d = p4d_offset(pgd, VSYSCALL_ADDR);
 #if CONFIG_PGTABLE_LEVELS >= 5
-	p4d->p4d |= _PAGE_USER;
+	set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER));
 #endif
 	pud = pud_offset(p4d, VSYSCALL_ADDR);
 	set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER));
@@ -370,9 +362,7 @@ void __init map_vsyscall(void)
 
 	if (vsyscall_mode != NONE) {
 		__set_fixmap(VSYSCALL_PAGE, physaddr_vsyscall,
-			     vsyscall_mode == NATIVE
-			     ? PAGE_KERNEL_VSYSCALL
-			     : PAGE_KERNEL_VVAR);
+			     PAGE_KERNEL_VVAR);
 		set_vsyscall_pgtable_user_bits(swapper_pg_dir);
 	}
 
diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c
index 140d33288e78..88797c80b3e0 100644
--- a/arch/x86/events/core.c
+++ b/arch/x86/events/core.c
@@ -2118,7 +2118,8 @@ static int x86_pmu_event_init(struct perf_event *event)
 			event->destroy(event);
 	}
 
-	if (READ_ONCE(x86_pmu.attr_rdpmc))
+	if (READ_ONCE(x86_pmu.attr_rdpmc) &&
+	    !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
 		event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
 
 	return err;
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 731153a4681e..1e41d7508d99 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -2952,9 +2952,9 @@ static void intel_pebs_aliases_skl(struct perf_event *event)
 	return intel_pebs_aliases_precdist(event);
 }
 
-static unsigned long intel_pmu_free_running_flags(struct perf_event *event)
+static unsigned long intel_pmu_large_pebs_flags(struct perf_event *event)
 {
-	unsigned long flags = x86_pmu.free_running_flags;
+	unsigned long flags = x86_pmu.large_pebs_flags;
 
 	if (event->attr.use_clockid)
 		flags &= ~PERF_SAMPLE_TIME;
@@ -2976,8 +2976,8 @@ static int intel_pmu_hw_config(struct perf_event *event)
 		if (!event->attr.freq) {
 			event->hw.flags |= PERF_X86_EVENT_AUTO_RELOAD;
 			if (!(event->attr.sample_type &
-			      ~intel_pmu_free_running_flags(event)))
-				event->hw.flags |= PERF_X86_EVENT_FREERUNNING;
+			      ~intel_pmu_large_pebs_flags(event)))
+				event->hw.flags |= PERF_X86_EVENT_LARGE_PEBS;
 		}
 		if (x86_pmu.pebs_aliases)
 			x86_pmu.pebs_aliases(event);
@@ -3194,7 +3194,7 @@ static unsigned bdw_limit_period(struct perf_event *event, unsigned left)
 			X86_CONFIG(.event=0xc0, .umask=0x01)) {
 		if (left < 128)
 			left = 128;
-		left &= ~0x3fu;
+		left &= ~0x3fULL;
 	}
 	return left;
 }
@@ -3460,7 +3460,7 @@ static __initconst const struct x86_pmu core_pmu = {
 	.event_map		= intel_pmu_event_map,
 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
 	.apic			= 1,
-	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
+	.large_pebs_flags	= LARGE_PEBS_FLAGS,
 
 	/*
 	 * Intel PMCs cannot be accessed sanely above 32-bit width,
@@ -3502,7 +3502,7 @@ static __initconst const struct x86_pmu intel_pmu = {
 	.event_map		= intel_pmu_event_map,
 	.max_events		= ARRAY_SIZE(intel_perfmon_event_map),
 	.apic			= 1,
-	.free_running_flags	= PEBS_FREERUNNING_FLAGS,
+	.large_pebs_flags	= LARGE_PEBS_FLAGS,
 	/*
 	 * Intel PMCs cannot be accessed sanely above 32 bit width,
 	 * so we install an artificial 1<<31 period regardless of
@@ -3559,7 +3559,7 @@ static int intel_snb_pebs_broken(int cpu)
 		break;
 
 	case INTEL_FAM6_SANDYBRIDGE_X:
-		switch (cpu_data(cpu).x86_mask) {
+		switch (cpu_data(cpu).x86_stepping) {
 		case 6: rev = 0x618; break;
 		case 7: rev = 0x70c; break;
 		}
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index 18c25ab28557..5e526c54247e 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -935,7 +935,7 @@ void intel_pmu_pebs_add(struct perf_event *event)
 	bool needed_cb = pebs_needs_sched_cb(cpuc);
 
 	cpuc->n_pebs++;
-	if (hwc->flags & PERF_X86_EVENT_FREERUNNING)
+	if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
 		cpuc->n_large_pebs++;
 
 	pebs_update_state(needed_cb, cpuc, event->ctx->pmu);
@@ -975,7 +975,7 @@ void intel_pmu_pebs_del(struct perf_event *event)
 	bool needed_cb = pebs_needs_sched_cb(cpuc);
 
 	cpuc->n_pebs--;
-	if (hwc->flags & PERF_X86_EVENT_FREERUNNING)
+	if (hwc->flags & PERF_X86_EVENT_LARGE_PEBS)
 		cpuc->n_large_pebs--;
 
 	pebs_update_state(needed_cb, cpuc, event->ctx->pmu);
@@ -1153,6 +1153,7 @@ static void setup_pebs_sample_data(struct perf_event *event,
 	if (pebs == NULL)
 		return;
 
+	regs->flags &= ~PERF_EFLAGS_EXACT;
 	sample_type = event->attr.sample_type;
 	dsrc = sample_type & PERF_SAMPLE_DATA_SRC;
 
@@ -1197,7 +1198,6 @@ static void setup_pebs_sample_data(struct perf_event *event,
 	 */
 	*regs = *iregs;
 	regs->flags = pebs->flags;
-	set_linear_ip(regs, pebs->ip);
 
 	if (sample_type & PERF_SAMPLE_REGS_INTR) {
 		regs->ax = pebs->ax;
@@ -1233,13 +1233,22 @@ static void setup_pebs_sample_data(struct perf_event *event,
 #endif
 	}
 
-	if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format >= 2) {
-		regs->ip = pebs->real_ip;
-		regs->flags |= PERF_EFLAGS_EXACT;
-	} else if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(regs))
-		regs->flags |= PERF_EFLAGS_EXACT;
-	else
-		regs->flags &= ~PERF_EFLAGS_EXACT;
+	if (event->attr.precise_ip > 1) {
+		/* Haswell and later have the eventing IP, so use it: */
+		if (x86_pmu.intel_cap.pebs_format >= 2) {
+			set_linear_ip(regs, pebs->real_ip);
+			regs->flags |= PERF_EFLAGS_EXACT;
+		} else {
+			/* Otherwise use PEBS off-by-1 IP: */
+			set_linear_ip(regs, pebs->ip);
+
+			/* ... and try to fix it up using the LBR entries: */
+			if (intel_pmu_pebs_fixup_ip(regs))
+				regs->flags |= PERF_EFLAGS_EXACT;
+		}
+	} else
+		set_linear_ip(regs, pebs->ip);
+
 
 	if ((sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR)) &&
 	    x86_pmu.intel_cap.pebs_format >= 1)
@@ -1530,7 +1539,7 @@ void __init intel_ds_init(void)
 			x86_pmu.pebs_record_size =
 						sizeof(struct pebs_record_skl);
 			x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
-			x86_pmu.free_running_flags |= PERF_SAMPLE_TIME;
+			x86_pmu.large_pebs_flags |= PERF_SAMPLE_TIME;
 			break;
 
 		default:
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index ae64d0b69729..cf372b90557e 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -1186,7 +1186,7 @@ void __init intel_pmu_lbr_init_atom(void)
 	 * on PMU interrupt
 	 */
 	if (boot_cpu_data.x86_model == 28
-	    && boot_cpu_data.x86_mask < 10) {
+	    && boot_cpu_data.x86_stepping < 10) {
 		pr_cont("LBR disabled due to erratum");
 		return;
 	}
diff --git a/arch/x86/events/intel/p6.c b/arch/x86/events/intel/p6.c
index a5604c352930..408879b0c0d4 100644
--- a/arch/x86/events/intel/p6.c
+++ b/arch/x86/events/intel/p6.c
@@ -234,7 +234,7 @@ static __initconst const struct x86_pmu p6_pmu = {
 
 static __init void p6_pmu_rdpmc_quirk(void)
 {
-	if (boot_cpu_data.x86_mask < 9) {
+	if (boot_cpu_data.x86_stepping < 9) {
 		/*
 		 * PPro erratum 26; fixed in stepping 9 and above.
 		 */
diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c
index 6d8044ab1060..c98b943e58b4 100644
--- a/arch/x86/events/intel/uncore_snbep.c
+++ b/arch/x86/events/intel/uncore_snbep.c
@@ -3343,6 +3343,7 @@ static struct extra_reg skx_uncore_cha_extra_regs[] = {
 	SNBEP_CBO_EVENT_EXTRA_REG(0x9134, 0xffff, 0x4),
 	SNBEP_CBO_EVENT_EXTRA_REG(0x35, 0xff, 0x8),
 	SNBEP_CBO_EVENT_EXTRA_REG(0x36, 0xff, 0x8),
+	SNBEP_CBO_EVENT_EXTRA_REG(0x38, 0xff, 0x3),
 	EVENT_EXTRA_END
 };
 
@@ -3562,24 +3563,27 @@ static struct intel_uncore_type *skx_msr_uncores[] = {
 	NULL,
 };
 
+/*
+ * To determine the number of CHAs, it should read bits 27:0 in the CAPID6
+ * register which located at Device 30, Function 3, Offset 0x9C. PCI ID 0x2083.
+ */
+#define SKX_CAPID6		0x9c
+#define SKX_CHA_BIT_MASK	GENMASK(27, 0)
+
 static int skx_count_chabox(void)
 {
-	struct pci_dev *chabox_dev = NULL;
-	int bus, count = 0;
+	struct pci_dev *dev = NULL;
+	u32 val = 0;
 
-	while (1) {
-		chabox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x208d, chabox_dev);
-		if (!chabox_dev)
-			break;
-		if (count == 0)
-			bus = chabox_dev->bus->number;
-		if (bus != chabox_dev->bus->number)
-			break;
-		count++;
-	}
+	dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x2083, dev);
+	if (!dev)
+		goto out;
 
-	pci_dev_put(chabox_dev);
-	return count;
+	pci_read_config_dword(dev, SKX_CAPID6, &val);
+	val &= SKX_CHA_BIT_MASK;
+out:
+	pci_dev_put(dev);
+	return hweight32(val);
 }
 
 void skx_uncore_cpu_init(void)
@@ -3606,7 +3610,7 @@ static struct intel_uncore_type skx_uncore_imc = {
 };
 
 static struct attribute *skx_upi_uncore_formats_attr[] = {
-	&format_attr_event_ext.attr,
+	&format_attr_event.attr,
 	&format_attr_umask_ext.attr,
 	&format_attr_edge.attr,
 	&format_attr_inv.attr,
diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c
index 14efaa0e8684..18e2628e2d8f 100644
--- a/arch/x86/events/msr.c
+++ b/arch/x86/events/msr.c
@@ -10,7 +10,9 @@ enum perf_msr_id {
 	PERF_MSR_SMI			= 4,
 	PERF_MSR_PTSC			= 5,
 	PERF_MSR_IRPERF			= 6,
-
+	PERF_MSR_THERM			= 7,
+	PERF_MSR_THERM_SNAP		= 8,
+	PERF_MSR_THERM_UNIT		= 9,
 	PERF_MSR_EVENT_MAX,
 };
 
@@ -29,6 +31,11 @@ static bool test_irperf(int idx)
 	return boot_cpu_has(X86_FEATURE_IRPERF);
 }
 
+static bool test_therm_status(int idx)
+{
+	return boot_cpu_has(X86_FEATURE_DTHERM);
+}
+
 static bool test_intel(int idx)
 {
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
@@ -95,22 +102,28 @@ struct perf_msr {
 	bool	(*test)(int idx);
 };
 
-PMU_EVENT_ATTR_STRING(tsc,    evattr_tsc,    "event=0x00");
-PMU_EVENT_ATTR_STRING(aperf,  evattr_aperf,  "event=0x01");
-PMU_EVENT_ATTR_STRING(mperf,  evattr_mperf,  "event=0x02");
-PMU_EVENT_ATTR_STRING(pperf,  evattr_pperf,  "event=0x03");
-PMU_EVENT_ATTR_STRING(smi,    evattr_smi,    "event=0x04");
-PMU_EVENT_ATTR_STRING(ptsc,   evattr_ptsc,   "event=0x05");
-PMU_EVENT_ATTR_STRING(irperf, evattr_irperf, "event=0x06");
+PMU_EVENT_ATTR_STRING(tsc,				evattr_tsc,		"event=0x00"	);
+PMU_EVENT_ATTR_STRING(aperf,				evattr_aperf,		"event=0x01"	);
+PMU_EVENT_ATTR_STRING(mperf,				evattr_mperf,		"event=0x02"	);
+PMU_EVENT_ATTR_STRING(pperf,				evattr_pperf,		"event=0x03"	);
+PMU_EVENT_ATTR_STRING(smi,				evattr_smi,		"event=0x04"	);
+PMU_EVENT_ATTR_STRING(ptsc,				evattr_ptsc,		"event=0x05"	);
+PMU_EVENT_ATTR_STRING(irperf,				evattr_irperf,		"event=0x06"	);
+PMU_EVENT_ATTR_STRING(cpu_thermal_margin,		evattr_therm,		"event=0x07"	);
+PMU_EVENT_ATTR_STRING(cpu_thermal_margin.snapshot,	evattr_therm_snap,	"1"		);
+PMU_EVENT_ATTR_STRING(cpu_thermal_margin.unit,		evattr_therm_unit,	"C"		);
 
 static struct perf_msr msr[] = {
-	[PERF_MSR_TSC]    = { 0,		&evattr_tsc,	NULL,		 },
-	[PERF_MSR_APERF]  = { MSR_IA32_APERF,	&evattr_aperf,	test_aperfmperf, },
-	[PERF_MSR_MPERF]  = { MSR_IA32_MPERF,	&evattr_mperf,	test_aperfmperf, },
-	[PERF_MSR_PPERF]  = { MSR_PPERF,	&evattr_pperf,	test_intel,	 },
-	[PERF_MSR_SMI]    = { MSR_SMI_COUNT,	&evattr_smi,	test_intel,	 },
-	[PERF_MSR_PTSC]   = { MSR_F15H_PTSC,	&evattr_ptsc,	test_ptsc,	 },
-	[PERF_MSR_IRPERF] = { MSR_F17H_IRPERF,	&evattr_irperf,	test_irperf,	 },
+	[PERF_MSR_TSC]		= { 0,				&evattr_tsc,		NULL,			},
+	[PERF_MSR_APERF]	= { MSR_IA32_APERF,		&evattr_aperf,		test_aperfmperf,	},
+	[PERF_MSR_MPERF]	= { MSR_IA32_MPERF,		&evattr_mperf,		test_aperfmperf,	},
+	[PERF_MSR_PPERF]	= { MSR_PPERF,			&evattr_pperf,		test_intel,		},
+	[PERF_MSR_SMI]		= { MSR_SMI_COUNT,		&evattr_smi,		test_intel,		},
+	[PERF_MSR_PTSC]		= { MSR_F15H_PTSC,		&evattr_ptsc,		test_ptsc,		},
+	[PERF_MSR_IRPERF]	= { MSR_F17H_IRPERF,		&evattr_irperf,		test_irperf,		},
+	[PERF_MSR_THERM]	= { MSR_IA32_THERM_STATUS,	&evattr_therm,		test_therm_status,	},
+	[PERF_MSR_THERM_SNAP]	= { MSR_IA32_THERM_STATUS,	&evattr_therm_snap,	test_therm_status,	},
+	[PERF_MSR_THERM_UNIT]	= { MSR_IA32_THERM_STATUS,	&evattr_therm_unit,	test_therm_status,	},
 };
 
 static struct attribute *events_attrs[PERF_MSR_EVENT_MAX + 1] = {
@@ -161,9 +174,9 @@ static int msr_event_init(struct perf_event *event)
 	if (!msr[cfg].attr)
 		return -EINVAL;
 
-	event->hw.idx = -1;
-	event->hw.event_base = msr[cfg].msr;
-	event->hw.config = cfg;
+	event->hw.idx		= -1;
+	event->hw.event_base	= msr[cfg].msr;
+	event->hw.config	= cfg;
 
 	return 0;
 }
@@ -184,7 +197,7 @@ static void msr_event_update(struct perf_event *event)
 	u64 prev, now;
 	s64 delta;
 
-	/* Careful, an NMI might modify the previous event value. */
+	/* Careful, an NMI might modify the previous event value: */
 again:
 	prev = local64_read(&event->hw.prev_count);
 	now = msr_read_counter(event);
@@ -193,17 +206,22 @@ static void msr_event_update(struct perf_event *event)
 		goto again;
 
 	delta = now - prev;
-	if (unlikely(event->hw.event_base == MSR_SMI_COUNT))
+	if (unlikely(event->hw.event_base == MSR_SMI_COUNT)) {
 		delta = sign_extend64(delta, 31);
-
-	local64_add(delta, &event->count);
+		local64_add(delta, &event->count);
+	} else if (unlikely(event->hw.event_base == MSR_IA32_THERM_STATUS)) {
+		/* If valid, extract digital readout, otherwise set to -1: */
+		now = now & (1ULL << 31) ? (now >> 16) & 0x3f :  -1;
+		local64_set(&event->count, now);
+	} else {
+		local64_add(delta, &event->count);
+	}
 }
 
 static void msr_event_start(struct perf_event *event, int flags)
 {
-	u64 now;
+	u64 now = msr_read_counter(event);
 
-	now = msr_read_counter(event);
 	local64_set(&event->hw.prev_count, now);
 }
 
@@ -250,9 +268,7 @@ static int __init msr_init(void)
 	for (i = PERF_MSR_TSC + 1; i < PERF_MSR_EVENT_MAX; i++) {
 		u64 val;
 
-		/*
-		 * Virt sucks arse; you cannot tell if a R/O MSR is present :/
-		 */
+		/* Virt sucks; you cannot tell if a R/O MSR is present :/ */
 		if (!msr[i].test(i) || rdmsrl_safe(msr[i].msr, &val))
 			msr[i].attr = NULL;
 	}
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index 8e4ea143ed96..39cd0615f04f 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -69,7 +69,7 @@ struct event_constraint {
 #define PERF_X86_EVENT_RDPMC_ALLOWED	0x0100 /* grant rdpmc permission */
 #define PERF_X86_EVENT_EXCL_ACCT	0x0200 /* accounted EXCL event */
 #define PERF_X86_EVENT_AUTO_RELOAD	0x0400 /* use PEBS auto-reload */
-#define PERF_X86_EVENT_FREERUNNING	0x0800 /* use freerunning PEBS */
+#define PERF_X86_EVENT_LARGE_PEBS	0x0800 /* use large PEBS */
 
 
 struct amd_nb {
@@ -88,12 +88,13 @@ struct amd_nb {
  * REGS_USER can be handled for events limited to ring 3.
  *
  */
-#define PEBS_FREERUNNING_FLAGS \
+#define LARGE_PEBS_FLAGS \
 	(PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
 	PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
 	PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
 	PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
-	PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)
+	PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
+	PERF_SAMPLE_PERIOD)
 
 #define PEBS_REGS \
 	(PERF_REG_X86_AX | \
@@ -607,7 +608,7 @@ struct x86_pmu {
 	struct event_constraint *pebs_constraints;
 	void		(*pebs_aliases)(struct perf_event *event);
 	int 		max_pebs_events;
-	unsigned long	free_running_flags;
+	unsigned long	large_pebs_flags;
 
 	/*
 	 * Intel LBR
diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
index 189a398290db..2edc49e7409b 100644
--- a/arch/x86/hyperv/hv_init.c
+++ b/arch/x86/hyperv/hv_init.c
@@ -18,6 +18,8 @@
  */
 
 #include <linux/types.h>
+#include <asm/apic.h>
+#include <asm/desc.h>
 #include <asm/hypervisor.h>
 #include <asm/hyperv.h>
 #include <asm/mshyperv.h>
@@ -37,6 +39,7 @@ struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
 {
 	return tsc_pg;
 }
+EXPORT_SYMBOL_GPL(hv_get_tsc_page);
 
 static u64 read_hv_clock_tsc(struct clocksource *arg)
 {
@@ -101,6 +104,115 @@ static int hv_cpu_init(unsigned int cpu)
 	return 0;
 }
 
+static void (*hv_reenlightenment_cb)(void);
+
+static void hv_reenlightenment_notify(struct work_struct *dummy)
+{
+	struct hv_tsc_emulation_status emu_status;
+
+	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
+
+	/* Don't issue the callback if TSC accesses are not emulated */
+	if (hv_reenlightenment_cb && emu_status.inprogress)
+		hv_reenlightenment_cb();
+}
+static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify);
+
+void hyperv_stop_tsc_emulation(void)
+{
+	u64 freq;
+	struct hv_tsc_emulation_status emu_status;
+
+	rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
+	emu_status.inprogress = 0;
+	wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
+
+	rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq);
+	tsc_khz = div64_u64(freq, 1000);
+}
+EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation);
+
+static inline bool hv_reenlightenment_available(void)
+{
+	/*
+	 * Check for required features and priviliges to make TSC frequency
+	 * change notifications work.
+	 */
+	return ms_hyperv.features & HV_X64_ACCESS_FREQUENCY_MSRS &&
+		ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE &&
+		ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT;
+}
+
+__visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs)
+{
+	entering_ack_irq();
+
+	inc_irq_stat(irq_hv_reenlightenment_count);
+
+	schedule_delayed_work(&hv_reenlightenment_work, HZ/10);
+
+	exiting_irq();
+}
+
+void set_hv_tscchange_cb(void (*cb)(void))
+{
+	struct hv_reenlightenment_control re_ctrl = {
+		.vector = HYPERV_REENLIGHTENMENT_VECTOR,
+		.enabled = 1,
+		.target_vp = hv_vp_index[smp_processor_id()]
+	};
+	struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1};
+
+	if (!hv_reenlightenment_available()) {
+		pr_warn("Hyper-V: reenlightenment support is unavailable\n");
+		return;
+	}
+
+	hv_reenlightenment_cb = cb;
+
+	/* Make sure callback is registered before we write to MSRs */
+	wmb();
+
+	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
+	wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl));
+}
+EXPORT_SYMBOL_GPL(set_hv_tscchange_cb);
+
+void clear_hv_tscchange_cb(void)
+{
+	struct hv_reenlightenment_control re_ctrl;
+
+	if (!hv_reenlightenment_available())
+		return;
+
+	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
+	re_ctrl.enabled = 0;
+	wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
+
+	hv_reenlightenment_cb = NULL;
+}
+EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb);
+
+static int hv_cpu_die(unsigned int cpu)
+{
+	struct hv_reenlightenment_control re_ctrl;
+	unsigned int new_cpu;
+
+	if (hv_reenlightenment_cb == NULL)
+		return 0;
+
+	rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
+	if (re_ctrl.target_vp == hv_vp_index[cpu]) {
+		/* Reassign to some other online CPU */
+		new_cpu = cpumask_any_but(cpu_online_mask, cpu);
+
+		re_ctrl.target_vp = hv_vp_index[new_cpu];
+		wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
+	}
+
+	return 0;
+}
+
 /*
  * This function is to be invoked early in the boot sequence after the
  * hypervisor has been detected.
@@ -110,12 +222,19 @@ static int hv_cpu_init(unsigned int cpu)
  */
 void hyperv_init(void)
 {
-	u64 guest_id;
+	u64 guest_id, required_msrs;
 	union hv_x64_msr_hypercall_contents hypercall_msr;
 
 	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
 		return;
 
+	/* Absolutely required MSRs */
+	required_msrs = HV_X64_MSR_HYPERCALL_AVAILABLE |
+		HV_X64_MSR_VP_INDEX_AVAILABLE;
+
+	if ((ms_hyperv.features & required_msrs) != required_msrs)
+		return;
+
 	/* Allocate percpu VP index */
 	hv_vp_index = kmalloc_array(num_possible_cpus(), sizeof(*hv_vp_index),
 				    GFP_KERNEL);
@@ -123,7 +242,7 @@ void hyperv_init(void)
 		return;
 
 	if (cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online",
-			      hv_cpu_init, NULL) < 0)
+			      hv_cpu_init, hv_cpu_die) < 0)
 		goto free_vp_index;
 
 	/*
@@ -239,17 +358,24 @@ void hyperv_report_panic(struct pt_regs *regs, long err)
 }
 EXPORT_SYMBOL_GPL(hyperv_report_panic);
 
-bool hv_is_hypercall_page_setup(void)
+bool hv_is_hyperv_initialized(void)
 {
 	union hv_x64_msr_hypercall_contents hypercall_msr;
 
-	/* Check if the hypercall page is setup */
+	/*
+	 * Ensure that we're really on Hyper-V, and not a KVM or Xen
+	 * emulation of Hyper-V
+	 */
+	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
+		return false;
+
+	/*
+	 * Verify that earlier initialization succeeded by checking
+	 * that the hypercall page is setup
+	 */
 	hypercall_msr.as_uint64 = 0;
 	rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
-	if (!hypercall_msr.enable)
-		return false;
-
-	return true;
+	return hypercall_msr.enable;
 }
-EXPORT_SYMBOL_GPL(hv_is_hypercall_page_setup);
+EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized);
diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c
index 9cc9e1c1e2db..56c9ebac946f 100644
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -137,7 +137,12 @@ static void hyperv_flush_tlb_others(const struct cpumask *cpus,
 	}
 
 	if (info->mm) {
+		/*
+		 * AddressSpace argument must match the CR3 with PCID bits
+		 * stripped out.
+		 */
 		flush->address_space = virt_to_phys(info->mm->pgd);
+		flush->address_space &= CR3_ADDR_MASK;
 		flush->flags = 0;
 	} else {
 		flush->address_space = 0;
@@ -219,7 +224,12 @@ static void hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
 	}
 
 	if (info->mm) {
+		/*
+		 * AddressSpace argument must match the CR3 with PCID bits
+		 * stripped out.
+		 */
 		flush->address_space = virt_to_phys(info->mm->pgd);
+		flush->address_space &= CR3_ADDR_MASK;
 		flush->flags = 0;
 	} else {
 		flush->address_space = 0;
@@ -278,8 +288,6 @@ void hyperv_setup_mmu_ops(void)
 	if (!(ms_hyperv.hints & HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED))
 		return;
 
-	setup_clear_cpu_cap(X86_FEATURE_PCID);
-
 	if (!(ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) {
 		pr_info("Using hypercall for remote TLB flush\n");
 		pv_mmu_ops.flush_tlb_others = hyperv_flush_tlb_others;
diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c
index 96cd33bbfc85..6512498bbef6 100644
--- a/arch/x86/ia32/sys_ia32.c
+++ b/arch/x86/ia32/sys_ia32.c
@@ -51,15 +51,14 @@
 #define AA(__x)		((unsigned long)(__x))
 
 
-asmlinkage long sys32_truncate64(const char __user *filename,
-				 unsigned long offset_low,
-				 unsigned long offset_high)
+COMPAT_SYSCALL_DEFINE3(x86_truncate64, const char __user *, filename,
+		       unsigned long, offset_low, unsigned long, offset_high)
 {
        return sys_truncate(filename, ((loff_t) offset_high << 32) | offset_low);
 }
 
-asmlinkage long sys32_ftruncate64(unsigned int fd, unsigned long offset_low,
-				  unsigned long offset_high)
+COMPAT_SYSCALL_DEFINE3(x86_ftruncate64, unsigned int, fd,
+		       unsigned long, offset_low, unsigned long, offset_high)
 {
        return sys_ftruncate(fd, ((loff_t) offset_high << 32) | offset_low);
 }
@@ -96,8 +95,8 @@ static int cp_stat64(struct stat64 __user *ubuf, struct kstat *stat)
 	return 0;
 }
 
-asmlinkage long sys32_stat64(const char __user *filename,
-			     struct stat64 __user *statbuf)
+COMPAT_SYSCALL_DEFINE2(x86_stat64, const char __user *, filename,
+		       struct stat64 __user *, statbuf)
 {
 	struct kstat stat;
 	int ret = vfs_stat(filename, &stat);
@@ -107,8 +106,8 @@ asmlinkage long sys32_stat64(const char __user *filename,
 	return ret;
 }
 
-asmlinkage long sys32_lstat64(const char __user *filename,
-			      struct stat64 __user *statbuf)
+COMPAT_SYSCALL_DEFINE2(x86_lstat64, const char __user *, filename,
+		       struct stat64 __user *, statbuf)
 {
 	struct kstat stat;
 	int ret = vfs_lstat(filename, &stat);
@@ -117,7 +116,8 @@ asmlinkage long sys32_lstat64(const char __user *filename,
 	return ret;
 }
 
-asmlinkage long sys32_fstat64(unsigned int fd, struct stat64 __user *statbuf)
+COMPAT_SYSCALL_DEFINE2(x86_fstat64, unsigned int, fd,
+		       struct stat64 __user *, statbuf)
 {
 	struct kstat stat;
 	int ret = vfs_fstat(fd, &stat);
@@ -126,8 +126,9 @@ asmlinkage long sys32_fstat64(unsigned int fd, struct stat64 __user *statbuf)
 	return ret;
 }
 
-asmlinkage long sys32_fstatat(unsigned int dfd, const char __user *filename,
-			      struct stat64 __user *statbuf, int flag)
+COMPAT_SYSCALL_DEFINE4(x86_fstatat, unsigned int, dfd,
+		       const char __user *, filename,
+		       struct stat64 __user *, statbuf, int, flag)
 {
 	struct kstat stat;
 	int error;
@@ -153,7 +154,7 @@ struct mmap_arg_struct32 {
 	unsigned int offset;
 };
 
-asmlinkage long sys32_mmap(struct mmap_arg_struct32 __user *arg)
+COMPAT_SYSCALL_DEFINE1(x86_mmap, struct mmap_arg_struct32 __user *, arg)
 {
 	struct mmap_arg_struct32 a;
 
@@ -167,22 +168,22 @@ asmlinkage long sys32_mmap(struct mmap_arg_struct32 __user *arg)
 			       a.offset>>PAGE_SHIFT);
 }
 
-asmlinkage long sys32_waitpid(compat_pid_t pid, unsigned int __user *stat_addr,
-			      int options)
+COMPAT_SYSCALL_DEFINE3(x86_waitpid, compat_pid_t, pid, unsigned int __user *,
+		       stat_addr, int, options)
 {
 	return compat_sys_wait4(pid, stat_addr, options, NULL);
 }
 
 /* warning: next two assume little endian */
-asmlinkage long sys32_pread(unsigned int fd, char __user *ubuf, u32 count,
-			    u32 poslo, u32 poshi)
+COMPAT_SYSCALL_DEFINE5(x86_pread, unsigned int, fd, char __user *, ubuf,
+		       u32, count, u32, poslo, u32, poshi)
 {
 	return sys_pread64(fd, ubuf, count,
 			 ((loff_t)AA(poshi) << 32) | AA(poslo));
 }
 
-asmlinkage long sys32_pwrite(unsigned int fd, const char __user *ubuf,
-			     u32 count, u32 poslo, u32 poshi)
+COMPAT_SYSCALL_DEFINE5(x86_pwrite, unsigned int, fd, const char __user *, ubuf,
+		       u32, count, u32, poslo, u32, poshi)
 {
 	return sys_pwrite64(fd, ubuf, count,
 			  ((loff_t)AA(poshi) << 32) | AA(poslo));
@@ -193,8 +194,9 @@ asmlinkage long sys32_pwrite(unsigned int fd, const char __user *ubuf,
  * Some system calls that need sign extended arguments. This could be
  * done by a generic wrapper.
  */
-long sys32_fadvise64_64(int fd, __u32 offset_low, __u32 offset_high,
-			__u32 len_low, __u32 len_high, int advice)
+COMPAT_SYSCALL_DEFINE6(x86_fadvise64_64, int, fd, __u32, offset_low,
+		       __u32, offset_high, __u32, len_low, __u32, len_high,
+		       int, advice)
 {
 	return sys_fadvise64_64(fd,
 			       (((u64)offset_high)<<32) | offset_low,
@@ -202,31 +204,43 @@ long sys32_fadvise64_64(int fd, __u32 offset_low, __u32 offset_high,
 				advice);
 }
 
-asmlinkage ssize_t sys32_readahead(int fd, unsigned off_lo, unsigned off_hi,
-				   size_t count)
+COMPAT_SYSCALL_DEFINE4(x86_readahead, int, fd, unsigned int, off_lo,
+		       unsigned int, off_hi, size_t, count)
 {
 	return sys_readahead(fd, ((u64)off_hi << 32) | off_lo, count);
 }
 
-asmlinkage long sys32_sync_file_range(int fd, unsigned off_low, unsigned off_hi,
-				      unsigned n_low, unsigned n_hi,  int flags)
+COMPAT_SYSCALL_DEFINE6(x86_sync_file_range, int, fd, unsigned int, off_low,
+		       unsigned int, off_hi, unsigned int, n_low,
+		       unsigned int, n_hi, int, flags)
 {
 	return sys_sync_file_range(fd,
 				   ((u64)off_hi << 32) | off_low,
 				   ((u64)n_hi << 32) | n_low, flags);
 }
 
-asmlinkage long sys32_fadvise64(int fd, unsigned offset_lo, unsigned offset_hi,
-				size_t len, int advice)
+COMPAT_SYSCALL_DEFINE5(x86_fadvise64, int, fd, unsigned int, offset_lo,
+		       unsigned int, offset_hi, size_t, len, int, advice)
 {
 	return sys_fadvise64_64(fd, ((u64)offset_hi << 32) | offset_lo,
 				len, advice);
 }
 
-asmlinkage long sys32_fallocate(int fd, int mode, unsigned offset_lo,
-				unsigned offset_hi, unsigned len_lo,
-				unsigned len_hi)
+COMPAT_SYSCALL_DEFINE6(x86_fallocate, int, fd, int, mode,
+		       unsigned int, offset_lo, unsigned int, offset_hi,
+		       unsigned int, len_lo, unsigned int, len_hi)
 {
 	return sys_fallocate(fd, mode, ((u64)offset_hi << 32) | offset_lo,
 			     ((u64)len_hi << 32) | len_lo);
 }
+
+/*
+ * The 32-bit clone ABI is CONFIG_CLONE_BACKWARDS
+ */
+COMPAT_SYSCALL_DEFINE5(x86_clone, unsigned long, clone_flags,
+		       unsigned long, newsp, int __user *, parent_tidptr,
+		       unsigned long, tls_val, int __user *, child_tidptr)
+{
+	return sys_clone(clone_flags, newsp, parent_tidptr, child_tidptr,
+			tls_val);
+}
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index 5d6a53fd7521..de690c2d2e33 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -6,7 +6,6 @@ generated-y += unistd_32_ia32.h
 generated-y += unistd_64_x32.h
 generated-y += xen-hypercalls.h
 
-generic-y += clkdev.h
 generic-y += dma-contiguous.h
 generic-y += early_ioremap.h
 generic-y += mcs_spinlock.h
diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h
index 8d0ec9df1cbe..11881726ed37 100644
--- a/arch/x86/include/asm/acpi.h
+++ b/arch/x86/include/asm/acpi.h
@@ -49,7 +49,7 @@ extern int acpi_fix_pin2_polarity;
 extern int acpi_disable_cmcff;
 
 extern u8 acpi_sci_flags;
-extern int acpi_sci_override_gsi;
+extern u32 acpi_sci_override_gsi;
 void acpi_pic_sci_set_trigger(unsigned int, u16);
 
 struct device;
@@ -94,7 +94,7 @@ static inline unsigned int acpi_processor_cstate_check(unsigned int max_cstate)
 	if (boot_cpu_data.x86 == 0x0F &&
 	    boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
 	    boot_cpu_data.x86_model <= 0x05 &&
-	    boot_cpu_data.x86_mask < 0x0A)
+	    boot_cpu_data.x86_stepping < 0x0A)
 		return 1;
 	else if (boot_cpu_has(X86_BUG_AMD_APIC_C1E))
 		return 1;
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index cf5961ca8677..4cd6a3b71824 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -218,13 +218,11 @@ static inline int alternatives_text_reserved(void *start, void *end)
  */
 #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2,   \
 			   output, input...)				      \
-{									      \
 	asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
 		"call %P[new2]", feature2)				      \
 		: output, ASM_CALL_CONSTRAINT				      \
 		: [old] "i" (oldfunc), [new1] "i" (newfunc1),		      \
-		  [new2] "i" (newfunc2), ## input);			      \
-}
+		  [new2] "i" (newfunc2), ## input)
 
 /*
  * use this macro(s) if you need more than one output parameter
diff --git a/arch/x86/include/asm/apm.h b/arch/x86/include/asm/apm.h
index 4d4015ddcf26..c356098b6fb9 100644
--- a/arch/x86/include/asm/apm.h
+++ b/arch/x86/include/asm/apm.h
@@ -7,6 +7,8 @@
 #ifndef _ASM_X86_MACH_DEFAULT_APM_H
 #define _ASM_X86_MACH_DEFAULT_APM_H
 
+#include <asm/nospec-branch.h>
+
 #ifdef APM_ZERO_SEGS
 #	define APM_DO_ZERO_SEGS \
 		"pushl %%ds\n\t" \
@@ -32,6 +34,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
 	 * N.B. We do NOT need a cld after the BIOS call
 	 * because we always save and restore the flags.
 	 */
+	firmware_restrict_branch_speculation_start();
 	__asm__ __volatile__(APM_DO_ZERO_SEGS
 		"pushl %%edi\n\t"
 		"pushl %%ebp\n\t"
@@ -44,6 +47,7 @@ static inline void apm_bios_call_asm(u32 func, u32 ebx_in, u32 ecx_in,
 		  "=S" (*esi)
 		: "a" (func), "b" (ebx_in), "c" (ecx_in)
 		: "memory", "cc");
+	firmware_restrict_branch_speculation_end();
 }
 
 static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
@@ -56,6 +60,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
 	 * N.B. We do NOT need a cld after the BIOS call
 	 * because we always save and restore the flags.
 	 */
+	firmware_restrict_branch_speculation_start();
 	__asm__ __volatile__(APM_DO_ZERO_SEGS
 		"pushl %%edi\n\t"
 		"pushl %%ebp\n\t"
@@ -68,6 +73,7 @@ static inline bool apm_bios_call_simple_asm(u32 func, u32 ebx_in,
 		  "=S" (si)
 		: "a" (func), "b" (ebx_in), "c" (ecx_in)
 		: "memory", "cc");
+	firmware_restrict_branch_speculation_end();
 	return error;
 }
 
diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h
index 7fb336210e1b..042b5e892ed1 100644
--- a/arch/x86/include/asm/barrier.h
+++ b/arch/x86/include/asm/barrier.h
@@ -24,11 +24,35 @@
 #define wmb()	asm volatile("sfence" ::: "memory")
 #endif
 
-#ifdef CONFIG_X86_PPRO_FENCE
-#define dma_rmb()	rmb()
-#else
+/**
+ * array_index_mask_nospec() - generate a mask that is ~0UL when the
+ * 	bounds check succeeds and 0 otherwise
+ * @index: array element index
+ * @size: number of elements in array
+ *
+ * Returns:
+ *     0 - (index < size)
+ */
+static inline unsigned long array_index_mask_nospec(unsigned long index,
+		unsigned long size)
+{
+	unsigned long mask;
+
+	asm ("cmp %1,%2; sbb %0,%0;"
+			:"=r" (mask)
+			:"g"(size),"r" (index)
+			:"cc");
+	return mask;
+}
+
+/* Override the default implementation from linux/nospec.h. */
+#define array_index_mask_nospec array_index_mask_nospec
+
+/* Prevent speculative execution past this barrier. */
+#define barrier_nospec() alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC, \
+					   "lfence", X86_FEATURE_LFENCE_RDTSC)
+
 #define dma_rmb()	barrier()
-#endif
 #define dma_wmb()	barrier()
 
 #ifdef CONFIG_X86_32
@@ -40,30 +64,6 @@
 #define __smp_wmb()	barrier()
 #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
 
-#if defined(CONFIG_X86_PPRO_FENCE)
-
-/*
- * For this option x86 doesn't have a strong TSO memory
- * model and we should fall back to full barriers.
- */
-
-#define __smp_store_release(p, v)					\
-do {									\
-	compiletime_assert_atomic_type(*p);				\
-	__smp_mb();							\
-	WRITE_ONCE(*p, v);						\
-} while (0)
-
-#define __smp_load_acquire(p)						\
-({									\
-	typeof(*p) ___p1 = READ_ONCE(*p);				\
-	compiletime_assert_atomic_type(*p);				\
-	__smp_mb();							\
-	___p1;								\
-})
-
-#else /* regular x86 TSO memory ordering */
-
 #define __smp_store_release(p, v)					\
 do {									\
 	compiletime_assert_atomic_type(*p);				\
@@ -79,8 +79,6 @@ do {									\
 	___p1;								\
 })
 
-#endif
-
 /* Atomic operations are already serializing on x86 */
 #define __smp_mb__before_atomic()	barrier()
 #define __smp_mb__after_atomic()	barrier()
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 3fa039855b8f..9f645ba57dbb 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -78,7 +78,7 @@ set_bit(long nr, volatile unsigned long *addr)
 			: "iq" ((u8)CONST_MASK(nr))
 			: "memory");
 	} else {
-		asm volatile(LOCK_PREFIX "bts %1,%0"
+		asm volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0"
 			: BITOP_ADDR(addr) : "Ir" (nr) : "memory");
 	}
 }
@@ -94,7 +94,7 @@ set_bit(long nr, volatile unsigned long *addr)
  */
 static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
 {
-	asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory");
+	asm volatile(__ASM_SIZE(bts) " %1,%0" : ADDR : "Ir" (nr) : "memory");
 }
 
 /**
@@ -115,7 +115,7 @@ clear_bit(long nr, volatile unsigned long *addr)
 			: CONST_MASK_ADDR(nr, addr)
 			: "iq" ((u8)~CONST_MASK(nr)));
 	} else {
-		asm volatile(LOCK_PREFIX "btr %1,%0"
+		asm volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0"
 			: BITOP_ADDR(addr)
 			: "Ir" (nr));
 	}
@@ -137,7 +137,7 @@ static __always_inline void clear_bit_unlock(long nr, volatile unsigned long *ad
 
 static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
 {
-	asm volatile("btr %1,%0" : ADDR : "Ir" (nr));
+	asm volatile(__ASM_SIZE(btr) " %1,%0" : ADDR : "Ir" (nr));
 }
 
 static __always_inline bool clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
@@ -182,7 +182,7 @@ static __always_inline void __clear_bit_unlock(long nr, volatile unsigned long *
  */
 static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
 {
-	asm volatile("btc %1,%0" : ADDR : "Ir" (nr));
+	asm volatile(__ASM_SIZE(btc) " %1,%0" : ADDR : "Ir" (nr));
 }
 
 /**
@@ -201,7 +201,7 @@ static __always_inline void change_bit(long nr, volatile unsigned long *addr)
 			: CONST_MASK_ADDR(nr, addr)
 			: "iq" ((u8)CONST_MASK(nr)));
 	} else {
-		asm volatile(LOCK_PREFIX "btc %1,%0"
+		asm volatile(LOCK_PREFIX __ASM_SIZE(btc) " %1,%0"
 			: BITOP_ADDR(addr)
 			: "Ir" (nr));
 	}
@@ -217,7 +217,8 @@ static __always_inline void change_bit(long nr, volatile unsigned long *addr)
  */
 static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
 {
-	GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", c);
+	GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts),
+	                 *addr, "Ir", nr, "%0", c);
 }
 
 /**
@@ -246,7 +247,7 @@ static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *
 {
 	bool oldbit;
 
-	asm("bts %2,%1"
+	asm(__ASM_SIZE(bts) " %2,%1"
 	    CC_SET(c)
 	    : CC_OUT(c) (oldbit), ADDR
 	    : "Ir" (nr));
@@ -263,7 +264,8 @@ static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *
  */
 static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
 {
-	GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", c);
+	GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr),
+	                 *addr, "Ir", nr, "%0", c);
 }
 
 /**
@@ -286,7 +288,7 @@ static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long
 {
 	bool oldbit;
 
-	asm volatile("btr %2,%1"
+	asm volatile(__ASM_SIZE(btr) " %2,%1"
 		     CC_SET(c)
 		     : CC_OUT(c) (oldbit), ADDR
 		     : "Ir" (nr));
@@ -298,7 +300,7 @@ static __always_inline bool __test_and_change_bit(long nr, volatile unsigned lon
 {
 	bool oldbit;
 
-	asm volatile("btc %2,%1"
+	asm volatile(__ASM_SIZE(btc) " %2,%1"
 		     CC_SET(c)
 		     : CC_OUT(c) (oldbit), ADDR
 		     : "Ir" (nr) : "memory");
@@ -316,7 +318,8 @@ static __always_inline bool __test_and_change_bit(long nr, volatile unsigned lon
  */
 static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
 {
-	GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", c);
+	GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc),
+	                 *addr, "Ir", nr, "%0", c);
 }
 
 static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
@@ -329,7 +332,7 @@ static __always_inline bool variable_test_bit(long nr, volatile const unsigned l
 {
 	bool oldbit;
 
-	asm volatile("bt %2,%1"
+	asm volatile(__ASM_SIZE(bt) " %2,%1"
 		     CC_SET(c)
 		     : CC_OUT(c) (oldbit)
 		     : "m" (*(unsigned long *)addr), "Ir" (nr));
diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h
index 34d99af43994..6804d6642767 100644
--- a/arch/x86/include/asm/bug.h
+++ b/arch/x86/include/asm/bug.h
@@ -5,23 +5,20 @@
 #include <linux/stringify.h>
 
 /*
- * Since some emulators terminate on UD2, we cannot use it for WARN.
- * Since various instruction decoders disagree on the length of UD1,
- * we cannot use it either. So use UD0 for WARN.
+ * Despite that some emulators terminate on UD2, we use it for WARN().
  *
- * (binutils knows about "ud1" but {en,de}codes it as 2 bytes, whereas
- *  our kernel decoder thinks it takes a ModRM byte, which seems consistent
- *  with various things like the Intel SDM instruction encoding rules)
+ * Since various instruction decoders/specs disagree on the encoding of
+ * UD0/UD1.
  */
 
-#define ASM_UD0		".byte 0x0f, 0xff"
+#define ASM_UD0		".byte 0x0f, 0xff" /* + ModRM (for Intel) */
 #define ASM_UD1		".byte 0x0f, 0xb9" /* + ModRM */
 #define ASM_UD2		".byte 0x0f, 0x0b"
 
 #define INSN_UD0	0xff0f
 #define INSN_UD2	0x0b0f
 
-#define LEN_UD0		2
+#define LEN_UD2		2
 
 #ifdef CONFIG_GENERIC_BUG
 
@@ -77,7 +74,11 @@ do {								\
 	unreachable();						\
 } while (0)
 
-#define __WARN_FLAGS(flags)	_BUG_FLAGS(ASM_UD0, BUGFLAG_WARNING|(flags))
+#define __WARN_FLAGS(flags)					\
+do {								\
+	_BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags));		\
+	annotate_reachable();					\
+} while (0)
 
 #include <asm-generic/bug.h>
 
diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
index 2cbd75dd2fd3..e1c8dab86670 100644
--- a/arch/x86/include/asm/compat.h
+++ b/arch/x86/include/asm/compat.h
@@ -127,88 +127,6 @@ typedef u32		compat_old_sigset_t;	/* at least 32 bits */
 
 typedef u32               compat_sigset_word;
 
-typedef union compat_sigval {
-	compat_int_t	sival_int;
-	compat_uptr_t	sival_ptr;
-} compat_sigval_t;
-
-typedef struct compat_siginfo {
-	int si_signo;
-	int si_errno;
-	int si_code;
-
-	union {
-		int _pad[128/sizeof(int) - 3];
-
-		/* kill() */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-		} _kill;
-
-		/* POSIX.1b timers */
-		struct {
-			compat_timer_t _tid;	/* timer id */
-			int _overrun;		/* overrun count */
-			compat_sigval_t _sigval;	/* same as below */
-			int _sys_private;	/* not to be passed to user */
-			int _overrun_incr;	/* amount to add to overrun */
-		} _timer;
-
-		/* POSIX.1b signals */
-		struct {
-			unsigned int _pid;	/* sender's pid */
-			unsigned int _uid;	/* sender's uid */
-			compat_sigval_t _sigval;
-		} _rt;
-
-		/* SIGCHLD */
-		struct {
-			unsigned int _pid;	/* which child */
-			unsigned int _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_clock_t _utime;
-			compat_clock_t _stime;
-		} _sigchld;
-
-		/* SIGCHLD (x32 version) */
-		struct {
-			unsigned int _pid;	/* which child */
-			unsigned int _uid;	/* sender's uid */
-			int _status;		/* exit code */
-			compat_s64 _utime;
-			compat_s64 _stime;
-		} _sigchld_x32;
-
-		/* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
-		struct {
-			unsigned int _addr;	/* faulting insn/memory ref. */
-			short int _addr_lsb;	/* Valid LSB of the reported address. */
-			union {
-				/* used when si_code=SEGV_BNDERR */
-				struct {
-					compat_uptr_t _lower;
-					compat_uptr_t _upper;
-				} _addr_bnd;
-				/* used when si_code=SEGV_PKUERR */
-				compat_u32 _pkey;
-			};
-		} _sigfault;
-
-		/* SIGPOLL */
-		struct {
-			int _band;	/* POLL_IN, POLL_OUT, POLL_MSG */
-			int _fd;
-		} _sigpoll;
-
-		struct {
-			unsigned int _call_addr; /* calling insn */
-			int _syscall;	/* triggering system call number */
-			unsigned int _arch;	/* AUDIT_ARCH_* of syscall */
-		} _sigsys;
-	} _sifields;
-} compat_siginfo_t;
-
 #define COMPAT_OFF_T_MAX	0x7fffffff
 
 struct compat_ipc64_perm {
@@ -331,4 +249,8 @@ static inline bool in_compat_syscall(void)
 }
 #define in_compat_syscall in_compat_syscall	/* override the generic impl */
 
+struct compat_siginfo;
+int __copy_siginfo_to_user32(struct compat_siginfo __user *to,
+		const siginfo_t *from, bool x32_ABI);
+
 #endif /* _ASM_X86_COMPAT_H */
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index ea9a7dde62e5..736771c9822e 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -29,6 +29,7 @@ enum cpuid_leafs
 	CPUID_8000_000A_EDX,
 	CPUID_7_ECX,
 	CPUID_8000_0007_EBX,
+	CPUID_7_EDX,
 };
 
 #ifdef CONFIG_X86_FEATURE_NAMES
@@ -79,8 +80,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) ||	\
+	   CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) ||	\
 	   REQUIRED_MASK_CHECK					  ||	\
-	   BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+	   BUILD_BUG_ON_ZERO(NCAPINTS != 19))
 
 #define DISABLED_MASK_BIT_SET(feature_bit)				\
 	 ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||	\
@@ -101,8 +103,9 @@ extern const char * const x86_bug_flags[NBUGINTS*32];
 	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) ||	\
 	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) ||	\
+	   CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) ||	\
 	   DISABLED_MASK_CHECK					  ||	\
-	   BUILD_BUG_ON_ZERO(NCAPINTS != 18))
+	   BUILD_BUG_ON_ZERO(NCAPINTS != 19))
 
 #define cpu_has(c, bit)							\
 	(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :	\
@@ -145,45 +148,46 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
  */
 static __always_inline __pure bool _static_cpu_has(u16 bit)
 {
-		asm_volatile_goto("1: jmp 6f\n"
-			 "2:\n"
-			 ".skip -(((5f-4f) - (2b-1b)) > 0) * "
-			         "((5f-4f) - (2b-1b)),0x90\n"
-			 "3:\n"
-			 ".section .altinstructions,\"a\"\n"
-			 " .long 1b - .\n"		/* src offset */
-			 " .long 4f - .\n"		/* repl offset */
-			 " .word %P1\n"			/* always replace */
-			 " .byte 3b - 1b\n"		/* src len */
-			 " .byte 5f - 4f\n"		/* repl len */
-			 " .byte 3b - 2b\n"		/* pad len */
-			 ".previous\n"
-			 ".section .altinstr_replacement,\"ax\"\n"
-			 "4: jmp %l[t_no]\n"
-			 "5:\n"
-			 ".previous\n"
-			 ".section .altinstructions,\"a\"\n"
-			 " .long 1b - .\n"		/* src offset */
-			 " .long 0\n"			/* no replacement */
-			 " .word %P0\n"			/* feature bit */
-			 " .byte 3b - 1b\n"		/* src len */
-			 " .byte 0\n"			/* repl len */
-			 " .byte 0\n"			/* pad len */
-			 ".previous\n"
-			 ".section .altinstr_aux,\"ax\"\n"
-			 "6:\n"
-			 " testb %[bitnum],%[cap_byte]\n"
-			 " jnz %l[t_yes]\n"
-			 " jmp %l[t_no]\n"
-			 ".previous\n"
-			 : : "i" (bit), "i" (X86_FEATURE_ALWAYS),
-			     [bitnum] "i" (1 << (bit & 7)),
-			     [cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3])
-			 : : t_yes, t_no);
-	t_yes:
-		return true;
-	t_no:
-		return false;
+	asm_volatile_goto("1: jmp 6f\n"
+		 "2:\n"
+		 ".skip -(((5f-4f) - (2b-1b)) > 0) * "
+			 "((5f-4f) - (2b-1b)),0x90\n"
+		 "3:\n"
+		 ".section .altinstructions,\"a\"\n"
+		 " .long 1b - .\n"		/* src offset */
+		 " .long 4f - .\n"		/* repl offset */
+		 " .word %P[always]\n"		/* always replace */
+		 " .byte 3b - 1b\n"		/* src len */
+		 " .byte 5f - 4f\n"		/* repl len */
+		 " .byte 3b - 2b\n"		/* pad len */
+		 ".previous\n"
+		 ".section .altinstr_replacement,\"ax\"\n"
+		 "4: jmp %l[t_no]\n"
+		 "5:\n"
+		 ".previous\n"
+		 ".section .altinstructions,\"a\"\n"
+		 " .long 1b - .\n"		/* src offset */
+		 " .long 0\n"			/* no replacement */
+		 " .word %P[feature]\n"		/* feature bit */
+		 " .byte 3b - 1b\n"		/* src len */
+		 " .byte 0\n"			/* repl len */
+		 " .byte 0\n"			/* pad len */
+		 ".previous\n"
+		 ".section .altinstr_aux,\"ax\"\n"
+		 "6:\n"
+		 " testb %[bitnum],%[cap_byte]\n"
+		 " jnz %l[t_yes]\n"
+		 " jmp %l[t_no]\n"
+		 ".previous\n"
+		 : : [feature]  "i" (bit),
+		     [always]   "i" (X86_FEATURE_ALWAYS),
+		     [bitnum]   "i" (1 << (bit & 7)),
+		     [cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3])
+		 : : t_yes, t_no);
+t_yes:
+	return true;
+t_no:
+	return false;
 }
 
 #define static_cpu_has(bit)					\
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 25b9375c1484..d554c11e01ff 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -13,7 +13,7 @@
 /*
  * Defines x86 CPU feature bits
  */
-#define NCAPINTS			18	   /* N 32-bit words worth of info */
+#define NCAPINTS			19	   /* N 32-bit words worth of info */
 #define NBUGINTS			1	   /* N 32-bit bug flags */
 
 /*
@@ -203,14 +203,17 @@
 #define X86_FEATURE_PROC_FEEDBACK	( 7*32+ 9) /* AMD ProcFeedbackInterface */
 #define X86_FEATURE_SME			( 7*32+10) /* AMD Secure Memory Encryption */
 #define X86_FEATURE_PTI			( 7*32+11) /* Kernel Page Table Isolation enabled */
-#define X86_FEATURE_RETPOLINE		( 7*32+12) /* Generic Retpoline mitigation for Spectre variant 2 */
-#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* AMD Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE		( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */
+#define X86_FEATURE_RETPOLINE_AMD	( 7*32+13) /* "" AMD Retpoline mitigation for Spectre variant 2 */
 #define X86_FEATURE_INTEL_PPIN		( 7*32+14) /* Intel Processor Inventory Number */
-#define X86_FEATURE_AVX512_4VNNIW	( 7*32+16) /* AVX-512 Neural Network Instructions */
-#define X86_FEATURE_AVX512_4FMAPS	( 7*32+17) /* AVX-512 Multiply Accumulation Single precision */
+#define X86_FEATURE_CDP_L2		( 7*32+15) /* Code and Data Prioritization L2 */
 
 #define X86_FEATURE_MBA			( 7*32+18) /* Memory Bandwidth Allocation */
-#define X86_FEATURE_RSB_CTXSW		( 7*32+19) /* Fill RSB on context switches */
+#define X86_FEATURE_RSB_CTXSW		( 7*32+19) /* "" Fill RSB on context switches */
+#define X86_FEATURE_SEV			( 7*32+20) /* AMD Secure Encrypted Virtualization */
+
+#define X86_FEATURE_USE_IBPB		( 7*32+21) /* "" Indirect Branch Prediction Barrier enabled */
+#define X86_FEATURE_USE_IBRS_FW		( 7*32+22) /* "" Use IBRS during runtime firmware calls */
 
 /* Virtualization flags: Linux defined, word 8 */
 #define X86_FEATURE_TPR_SHADOW		( 8*32+ 0) /* Intel TPR Shadow */
@@ -271,6 +274,9 @@
 #define X86_FEATURE_CLZERO		(13*32+ 0) /* CLZERO instruction */
 #define X86_FEATURE_IRPERF		(13*32+ 1) /* Instructions Retired Count */
 #define X86_FEATURE_XSAVEERPTR		(13*32+ 2) /* Always save/restore FP error pointers */
+#define X86_FEATURE_IBPB		(13*32+12) /* Indirect Branch Prediction Barrier */
+#define X86_FEATURE_IBRS		(13*32+14) /* Indirect Branch Restricted Speculation */
+#define X86_FEATURE_STIBP		(13*32+15) /* Single Thread Indirect Branch Predictors */
 
 /* Thermal and Power Management Leaf, CPUID level 0x00000006 (EAX), word 14 */
 #define X86_FEATURE_DTHERM		(14*32+ 0) /* Digital Thermal Sensor */
@@ -310,6 +316,7 @@
 #define X86_FEATURE_VPCLMULQDQ		(16*32+10) /* Carry-Less Multiplication Double Quadword */
 #define X86_FEATURE_AVX512_VNNI		(16*32+11) /* Vector Neural Network Instructions */
 #define X86_FEATURE_AVX512_BITALG	(16*32+12) /* Support for VPOPCNT[B,W] and VPSHUF-BITQMB instructions */
+#define X86_FEATURE_TME			(16*32+13) /* Intel Total Memory Encryption */
 #define X86_FEATURE_AVX512_VPOPCNTDQ	(16*32+14) /* POPCNT for vectors of DW/QW */
 #define X86_FEATURE_LA57		(16*32+16) /* 5-level page tables */
 #define X86_FEATURE_RDPID		(16*32+22) /* RDPID instruction */
@@ -319,6 +326,14 @@
 #define X86_FEATURE_SUCCOR		(17*32+ 1) /* Uncorrectable error containment and recovery */
 #define X86_FEATURE_SMCA		(17*32+ 3) /* Scalable MCA */
 
+/* Intel-defined CPU features, CPUID level 0x00000007:0 (EDX), word 18 */
+#define X86_FEATURE_AVX512_4VNNIW	(18*32+ 2) /* AVX-512 Neural Network Instructions */
+#define X86_FEATURE_AVX512_4FMAPS	(18*32+ 3) /* AVX-512 Multiply Accumulation Single precision */
+#define X86_FEATURE_PCONFIG		(18*32+18) /* Intel PCONFIG */
+#define X86_FEATURE_SPEC_CTRL		(18*32+26) /* "" Speculation Control (IBRS + IBPB) */
+#define X86_FEATURE_INTEL_STIBP		(18*32+27) /* "" Single Thread Indirect Branch Predictors */
+#define X86_FEATURE_ARCH_CAPABILITIES	(18*32+29) /* IA32_ARCH_CAPABILITIES MSR (Intel) */
+
 /*
  * BUG word(s)
  */
diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h
index b027633e7300..33833d1909af 100644
--- a/arch/x86/include/asm/disabled-features.h
+++ b/arch/x86/include/asm/disabled-features.h
@@ -77,6 +77,7 @@
 #define DISABLED_MASK15	0
 #define DISABLED_MASK16	(DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP)
 #define DISABLED_MASK17	0
-#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 18)
+#define DISABLED_MASK18	0
+#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
 
 #endif /* _ASM_X86_DISABLED_FEATURES_H */
diff --git a/arch/x86/include/asm/dma-direct.h b/arch/x86/include/asm/dma-direct.h
new file mode 100644
index 000000000000..1295bc622ebe
--- /dev/null
+++ b/arch/x86/include/asm/dma-direct.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef ASM_X86_DMA_DIRECT_H
+#define ASM_X86_DMA_DIRECT_H 1
+
+#include <linux/mem_encrypt.h>
+
+#ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */
+bool dma_capable(struct device *dev, dma_addr_t addr, size_t size);
+dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
+phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr);
+#else
+static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
+{
+	if (!dev->dma_mask)
+		return 0;
+
+	return addr + size - 1 <= *dev->dma_mask;
+}
+
+static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
+{
+	return __sme_set(paddr);
+}
+
+static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
+{
+	return __sme_clr(daddr);
+}
+#endif /* CONFIG_X86_DMA_REMAP */
+#endif /* ASM_X86_DMA_DIRECT_H */
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index 0350d99bb8fd..6277c83c0eb1 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -12,7 +12,6 @@
 #include <asm/io.h>
 #include <asm/swiotlb.h>
 #include <linux/dma-contiguous.h>
-#include <linux/mem_encrypt.h>
 
 #ifdef CONFIG_ISA
 # define ISA_DMA_BIT_MASK DMA_BIT_MASK(24)
@@ -31,6 +30,9 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 	return dma_ops;
 }
 
+int arch_dma_supported(struct device *dev, u64 mask);
+#define arch_dma_supported arch_dma_supported
+
 bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp);
 #define arch_dma_alloc_attrs arch_dma_alloc_attrs
 
@@ -42,31 +44,6 @@ extern void dma_generic_free_coherent(struct device *dev, size_t size,
 				      void *vaddr, dma_addr_t dma_addr,
 				      unsigned long attrs);
 
-#ifdef CONFIG_X86_DMA_REMAP /* Platform code defines bridge-specific code */
-extern bool dma_capable(struct device *dev, dma_addr_t addr, size_t size);
-extern dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
-extern phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr);
-#else
-
-static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
-{
-	if (!dev->dma_mask)
-		return 0;
-
-	return addr + size - 1 <= *dev->dma_mask;
-}
-
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return __sme_set(paddr);
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
-{
-	return __sme_clr(daddr);
-}
-#endif /* CONFIG_X86_DMA_REMAP */
-
 static inline unsigned long dma_alloc_coherent_mask(struct device *dev,
 						    gfp_t gfp)
 {
diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h
index 85f6ccb80b91..a399c1ebf6f0 100644
--- a/arch/x86/include/asm/efi.h
+++ b/arch/x86/include/asm/efi.h
@@ -6,6 +6,7 @@
 #include <asm/pgtable.h>
 #include <asm/processor-flags.h>
 #include <asm/tlb.h>
+#include <asm/nospec-branch.h>
 
 /*
  * We map the EFI regions needed for runtime services non-contiguously,
@@ -36,8 +37,18 @@
 
 extern asmlinkage unsigned long efi_call_phys(void *, ...);
 
-#define arch_efi_call_virt_setup()	kernel_fpu_begin()
-#define arch_efi_call_virt_teardown()	kernel_fpu_end()
+#define arch_efi_call_virt_setup()					\
+({									\
+	kernel_fpu_begin();						\
+	firmware_restrict_branch_speculation_start();			\
+})
+
+#define arch_efi_call_virt_teardown()					\
+({									\
+	firmware_restrict_branch_speculation_end();			\
+	kernel_fpu_end();						\
+})
+
 
 /*
  * Wrap all the virtual calls in a way that forces the parameters on the stack.
@@ -73,6 +84,7 @@ struct efi_scratch {
 	efi_sync_low_kernel_mappings();					\
 	preempt_disable();						\
 	__kernel_fpu_begin();						\
+	firmware_restrict_branch_speculation_start();			\
 									\
 	if (efi_scratch.use_pgd) {					\
 		efi_scratch.prev_cr3 = __read_cr3();			\
@@ -91,6 +103,7 @@ struct efi_scratch {
 		__flush_tlb_all();					\
 	}								\
 									\
+	firmware_restrict_branch_speculation_end();			\
 	__kernel_fpu_end();						\
 	preempt_enable();						\
 })
diff --git a/arch/x86/include/asm/error-injection.h b/arch/x86/include/asm/error-injection.h
new file mode 100644
index 000000000000..47b7a1296245
--- /dev/null
+++ b/arch/x86/include/asm/error-injection.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_ERROR_INJECTION_H
+#define _ASM_ERROR_INJECTION_H
+
+#include <linux/compiler.h>
+#include <linux/linkage.h>
+#include <asm/ptrace.h>
+#include <asm-generic/error-injection.h>
+
+asmlinkage void just_return_func(void);
+void override_function_with_return(struct pt_regs *regs);
+
+#endif /* _ASM_ERROR_INJECTION_H */
diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h
index 64c4a30e0d39..e203169931c7 100644
--- a/arch/x86/include/asm/fixmap.h
+++ b/arch/x86/include/asm/fixmap.h
@@ -137,8 +137,10 @@ enum fixed_addresses {
 
 extern void reserve_top_address(unsigned long reserve);
 
-#define FIXADDR_SIZE	(__end_of_permanent_fixed_addresses << PAGE_SHIFT)
-#define FIXADDR_START	(FIXADDR_TOP - FIXADDR_SIZE)
+#define FIXADDR_SIZE		(__end_of_permanent_fixed_addresses << PAGE_SHIFT)
+#define FIXADDR_START		(FIXADDR_TOP - FIXADDR_SIZE)
+#define FIXADDR_TOT_SIZE	(__end_of_fixed_addresses << PAGE_SHIFT)
+#define FIXADDR_TOT_START	(FIXADDR_TOP - FIXADDR_TOT_SIZE)
 
 extern int fixmaps_set;
 
diff --git a/arch/x86/include/asm/fpu/signal.h b/arch/x86/include/asm/fpu/signal.h
index 4df2754ef380..44bbc39a57b3 100644
--- a/arch/x86/include/asm/fpu/signal.h
+++ b/arch/x86/include/asm/fpu/signal.h
@@ -20,12 +20,6 @@ int ia32_setup_frame(int sig, struct ksignal *ksig,
 # define ia32_setup_rt_frame	__setup_rt_frame
 #endif
 
-#ifdef CONFIG_COMPAT
-int __copy_siginfo_to_user32(compat_siginfo_t __user *to,
-		const siginfo_t *from, bool x32_ABI);
-#endif
-
-
 extern void convert_from_fxsr(struct user_i387_ia32_struct *env,
 			      struct task_struct *tsk);
 extern void convert_to_fxsr(struct task_struct *tsk,
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 51cc979dd364..7c341a74ec8c 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -38,6 +38,9 @@ typedef struct {
 #if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
 	unsigned int irq_hv_callback_count;
 #endif
+#if IS_ENABLED(CONFIG_HYPERV)
+	unsigned int irq_hv_reenlightenment_count;
+#endif
 } ____cacheline_aligned irq_cpustat_t;
 
 DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 2851077b6051..32e666e1231e 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -36,6 +36,7 @@ extern asmlinkage void kvm_posted_intr_wakeup_ipi(void);
 extern asmlinkage void kvm_posted_intr_nested_ipi(void);
 extern asmlinkage void error_interrupt(void);
 extern asmlinkage void irq_work_interrupt(void);
+extern asmlinkage void uv_bau_message_intr1(void);
 
 extern asmlinkage void spurious_interrupt(void);
 extern asmlinkage void thermal_interrupt(void);
diff --git a/arch/x86/include/asm/hypervisor.h b/arch/x86/include/asm/hypervisor.h
index 96aa6b9884dc..8c5aaba6633f 100644
--- a/arch/x86/include/asm/hypervisor.h
+++ b/arch/x86/include/asm/hypervisor.h
@@ -28,6 +28,7 @@ enum x86_hypervisor_type {
 	X86_HYPER_XEN_PV,
 	X86_HYPER_XEN_HVM,
 	X86_HYPER_KVM,
+	X86_HYPER_JAILHOUSE,
 };
 
 #ifdef CONFIG_HYPERVISOR_GUEST
diff --git a/arch/x86/include/asm/i8259.h b/arch/x86/include/asm/i8259.h
index c8376b40e882..5cdcdbd4d892 100644
--- a/arch/x86/include/asm/i8259.h
+++ b/arch/x86/include/asm/i8259.h
@@ -69,6 +69,11 @@ struct legacy_pic {
 extern struct legacy_pic *legacy_pic;
 extern struct legacy_pic null_legacy_pic;
 
+static inline bool has_legacy_pic(void)
+{
+	return legacy_pic != &null_legacy_pic;
+}
+
 static inline int nr_legacy_irqs(void)
 {
 	return legacy_pic->nr_legacy_irqs;
diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h
index 35a6bc4da8ad..cf090e584202 100644
--- a/arch/x86/include/asm/intel-family.h
+++ b/arch/x86/include/asm/intel-family.h
@@ -10,6 +10,10 @@
  *
  * Things ending in "2" are usually because we have no better
  * name for them.  There's no processor called "SILVERMONT2".
+ *
+ * While adding a new CPUID for a new microarchitecture, add a new
+ * group to keep logically sorted out in chronological order. Within
+ * that group keep the CPUID for the variants sorted by model number.
  */
 
 #define INTEL_FAM6_CORE_YONAH		0x0E
@@ -49,6 +53,8 @@
 #define INTEL_FAM6_KABYLAKE_MOBILE	0x8E
 #define INTEL_FAM6_KABYLAKE_DESKTOP	0x9E
 
+#define INTEL_FAM6_CANNONLAKE_MOBILE	0x66
+
 /* "Small Core" Processors (Atom) */
 
 #define INTEL_FAM6_ATOM_PINEVIEW	0x1C
diff --git a/arch/x86/include/asm/intel_pmc_ipc.h b/arch/x86/include/asm/intel_pmc_ipc.h
index 528ed4be4393..9e7adcdbe031 100644
--- a/arch/x86/include/asm/intel_pmc_ipc.h
+++ b/arch/x86/include/asm/intel_pmc_ipc.h
@@ -38,6 +38,7 @@ int intel_pmc_ipc_command(u32 cmd, u32 sub, u8 *in, u32 inlen,
 		u32 *out, u32 outlen);
 int intel_pmc_s0ix_counter_read(u64 *data);
 int intel_pmc_gcr_read(u32 offset, u32 *data);
+int intel_pmc_gcr_read64(u32 offset, u64 *data);
 int intel_pmc_gcr_write(u32 offset, u32 data);
 int intel_pmc_gcr_update(u32 offset, u32 mask, u32 val);
 
@@ -70,6 +71,11 @@ static inline int intel_pmc_gcr_read(u32 offset, u32 *data)
 	return -EINVAL;
 }
 
+static inline int intel_pmc_gcr_read64(u32 offset, u64 *data)
+{
+	return -EINVAL;
+}
+
 static inline int intel_pmc_gcr_write(u32 offset, u32 data)
 {
 	return -EINVAL;
diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h
index 95e948627fd0..f6e5b9375d8c 100644
--- a/arch/x86/include/asm/io.h
+++ b/arch/x86/include/asm/io.h
@@ -232,21 +232,6 @@ extern void set_iounmap_nonlazy(void);
  */
 #define __ISA_IO_base ((char __iomem *)(PAGE_OFFSET))
 
-/*
- *	Cache management
- *
- *	This needed for two cases
- *	1. Out of order aware processors
- *	2. Accidentally out of order processors (PPro errata #51)
- */
-
-static inline void flush_write_buffers(void)
-{
-#if defined(CONFIG_X86_PPRO_FENCE)
-	asm volatile("lock; addl $0,0(%%esp)": : :"memory");
-#endif
-}
-
 #endif /* __KERNEL__ */
 
 extern void native_io_delay(void);
diff --git a/arch/x86/include/asm/iosf_mbi.h b/arch/x86/include/asm/iosf_mbi.h
index 7d87437bd030..3de0489deade 100644
--- a/arch/x86/include/asm/iosf_mbi.h
+++ b/arch/x86/include/asm/iosf_mbi.h
@@ -146,6 +146,18 @@ int iosf_mbi_register_pmic_bus_access_notifier(struct notifier_block *nb);
  */
 int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb);
 
+/**
+ * iosf_mbi_unregister_pmic_bus_access_notifier_unlocked - Unregister PMIC bus
+ *                                                         notifier, unlocked
+ *
+ * Like iosf_mbi_unregister_pmic_bus_access_notifier(), but for use when the
+ * caller has already called iosf_mbi_punit_acquire() itself.
+ *
+ * @nb: notifier_block to unregister
+ */
+int iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
+	struct notifier_block *nb);
+
 /**
  * iosf_mbi_call_pmic_bus_access_notifier_chain - Call PMIC bus notifier chain
  *
@@ -154,6 +166,11 @@ int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb);
  */
 int iosf_mbi_call_pmic_bus_access_notifier_chain(unsigned long val, void *v);
 
+/**
+ * iosf_mbi_assert_punit_acquired - Assert that the P-Unit has been acquired.
+ */
+void iosf_mbi_assert_punit_acquired(void);
+
 #else /* CONFIG_IOSF_MBI is not enabled */
 static inline
 bool iosf_mbi_available(void)
@@ -197,12 +214,20 @@ int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb)
 	return 0;
 }
 
+static inline int
+iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(struct notifier_block *nb)
+{
+	return 0;
+}
+
 static inline
 int iosf_mbi_call_pmic_bus_access_notifier_chain(unsigned long val, void *v)
 {
 	return 0;
 }
 
+static inline void iosf_mbi_assert_punit_acquired(void) {}
+
 #endif /* CONFIG_IOSF_MBI */
 
 #endif /* IOSF_MBI_SYMS_H */
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 67421f649cfa..e71c1120426b 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -103,7 +103,12 @@
 #endif
 
 #define MANAGED_IRQ_SHUTDOWN_VECTOR	0xef
-#define LOCAL_TIMER_VECTOR		0xee
+
+#if IS_ENABLED(CONFIG_HYPERV)
+#define HYPERV_REENLIGHTENMENT_VECTOR	0xee
+#endif
+
+#define LOCAL_TIMER_VECTOR		0xed
 
 #define NR_VECTORS			 256
 
diff --git a/arch/x86/include/asm/jailhouse_para.h b/arch/x86/include/asm/jailhouse_para.h
new file mode 100644
index 000000000000..875b54376689
--- /dev/null
+++ b/arch/x86/include/asm/jailhouse_para.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL2.0 */
+
+/*
+ * Jailhouse paravirt_ops implementation
+ *
+ * Copyright (c) Siemens AG, 2015-2017
+ *
+ * Authors:
+ *  Jan Kiszka <jan.kiszka@siemens.com>
+ */
+
+#ifndef _ASM_X86_JAILHOUSE_PARA_H
+#define _ASM_X86_JAILHOUSE_PARA_H
+
+#include <linux/types.h>
+
+#ifdef CONFIG_JAILHOUSE_GUEST
+bool jailhouse_paravirt(void);
+#else
+static inline bool jailhouse_paravirt(void)
+{
+	return false;
+}
+#endif
+
+#endif /* _ASM_X86_JAILHOUSE_PARA_H */
diff --git a/arch/x86/include/asm/kasan.h b/arch/x86/include/asm/kasan.h
index b577dd0916aa..13e70da38bed 100644
--- a/arch/x86/include/asm/kasan.h
+++ b/arch/x86/include/asm/kasan.h
@@ -4,6 +4,7 @@
 
 #include <linux/const.h>
 #define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL)
+#define KASAN_SHADOW_SCALE_SHIFT 3
 
 /*
  * Compiler uses shadow offset assuming that addresses start
@@ -12,12 +13,15 @@
  * 'kernel address space start' >> KASAN_SHADOW_SCALE_SHIFT
  */
 #define KASAN_SHADOW_START      (KASAN_SHADOW_OFFSET + \
-					((-1UL << __VIRTUAL_MASK_SHIFT) >> 3))
+					((-1UL << __VIRTUAL_MASK_SHIFT) >> \
+						KASAN_SHADOW_SCALE_SHIFT))
 /*
- * 47 bits for kernel address -> (47 - 3) bits for shadow
- * 56 bits for kernel address -> (56 - 3) bits for shadow
+ * 47 bits for kernel address -> (47 - KASAN_SHADOW_SCALE_SHIFT) bits for shadow
+ * 56 bits for kernel address -> (56 - KASAN_SHADOW_SCALE_SHIFT) bits for shadow
  */
-#define KASAN_SHADOW_END        (KASAN_SHADOW_START + (1ULL << (__VIRTUAL_MASK_SHIFT - 3)))
+#define KASAN_SHADOW_END        (KASAN_SHADOW_START + \
+					(1ULL << (__VIRTUAL_MASK_SHIFT - \
+						  KASAN_SHADOW_SCALE_SHIFT)))
 
 #ifndef __ASSEMBLY__
 
diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h
index 9f2e3102e0bb..367d99cff426 100644
--- a/arch/x86/include/asm/kprobes.h
+++ b/arch/x86/include/asm/kprobes.h
@@ -67,6 +67,8 @@ extern const int kretprobe_blacklist_size;
 void arch_remove_kprobe(struct kprobe *p);
 asmlinkage void kretprobe_trampoline(void);
 
+extern void arch_kprobe_override_function(struct pt_regs *regs);
+
 /* Architecture specific copy of original instruction*/
 struct arch_specific_insn {
 	/* copy of the original instruction */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 516798431328..b605a5b6a30c 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -86,7 +86,7 @@
 			  | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \
 			  | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \
 			  | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \
-			  | X86_CR4_SMAP | X86_CR4_PKE))
+			  | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP))
 
 #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR)
 
@@ -504,8 +504,10 @@ struct kvm_vcpu_arch {
 	int mp_state;
 	u64 ia32_misc_enable_msr;
 	u64 smbase;
+	u64 smi_count;
 	bool tpr_access_reporting;
 	u64 ia32_xss;
+	u64 microcode_version;
 
 	/*
 	 * Paging state of the vcpu
@@ -760,6 +762,15 @@ enum kvm_irqchip_mode {
 	KVM_IRQCHIP_SPLIT,        /* created with KVM_CAP_SPLIT_IRQCHIP */
 };
 
+struct kvm_sev_info {
+	bool active;		/* SEV enabled guest */
+	unsigned int asid;	/* ASID used for this guest */
+	unsigned int handle;	/* SEV firmware handle */
+	int fd;			/* SEV device fd */
+	unsigned long pages_locked; /* Number of pages locked */
+	struct list_head regions_list;  /* List of registered regions */
+};
+
 struct kvm_arch {
 	unsigned int n_used_mmu_pages;
 	unsigned int n_requested_mmu_pages;
@@ -847,6 +858,8 @@ struct kvm_arch {
 
 	bool x2apic_format;
 	bool x2apic_broadcast_quirk_disabled;
+
+	struct kvm_sev_info sev_info;
 };
 
 struct kvm_vm_stat {
@@ -883,7 +896,6 @@ struct kvm_vcpu_stat {
 	u64 request_irq_exits;
 	u64 irq_exits;
 	u64 host_state_reload;
-	u64 efer_reload;
 	u64 fpu_reload;
 	u64 insn_emulation;
 	u64 insn_emulation_fail;
@@ -965,7 +977,7 @@ struct kvm_x86_ops {
 	unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
 	void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
 
-	void (*tlb_flush)(struct kvm_vcpu *vcpu);
+	void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa);
 
 	void (*run)(struct kvm_vcpu *vcpu);
 	int (*handle_exit)(struct kvm_vcpu *vcpu);
@@ -1017,6 +1029,7 @@ struct kvm_x86_ops {
 	void (*handle_external_intr)(struct kvm_vcpu *vcpu);
 	bool (*mpx_supported)(void);
 	bool (*xsaves_supported)(void);
+	bool (*umip_emulated)(void);
 
 	int (*check_nested_events)(struct kvm_vcpu *vcpu, bool external_intr);
 
@@ -1079,6 +1092,12 @@ struct kvm_x86_ops {
 	int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate);
 	int (*pre_leave_smm)(struct kvm_vcpu *vcpu, u64 smbase);
 	int (*enable_smi_window)(struct kvm_vcpu *vcpu);
+
+	int (*mem_enc_op)(struct kvm *kvm, void __user *argp);
+	int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
+	int (*mem_enc_unreg_region)(struct kvm *kvm, struct kvm_enc_region *argp);
+
+	int (*get_msr_feature)(struct kvm_msr_entry *entry);
 };
 
 struct kvm_arch_async_pf {
@@ -1448,7 +1467,4 @@ static inline int kvm_cpu_get_apicid(int mps_cpu)
 #define put_smstate(type, buf, offset, val)                      \
 	*(type *)((buf) + (offset) - 0x7e00) = val
 
-void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
-		unsigned long start, unsigned long end);
-
 #endif /* _ASM_X86_KVM_HOST_H */
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index b1e8d8db921f..96ea4b5ba658 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -376,6 +376,7 @@ struct smca_bank {
 extern struct smca_bank smca_banks[MAX_NR_BANKS];
 
 extern const char *smca_get_long_name(enum smca_bank_types t);
+extern bool amd_mce_is_memory_error(struct mce *m);
 
 extern int mce_threshold_create_device(unsigned int cpu);
 extern int mce_threshold_remove_device(unsigned int cpu);
@@ -384,6 +385,7 @@ extern int mce_threshold_remove_device(unsigned int cpu);
 
 static inline int mce_threshold_create_device(unsigned int cpu) { return 0; };
 static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; };
+static inline bool amd_mce_is_memory_error(struct mce *m) { return false; };
 
 #endif
 
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 55520cec8b27..6cf0e4cb7b97 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -37,7 +37,13 @@ struct cpu_signature {
 
 struct device;
 
-enum ucode_state { UCODE_ERROR, UCODE_OK, UCODE_NFOUND };
+enum ucode_state {
+	UCODE_OK	= 0,
+	UCODE_NEW,
+	UCODE_UPDATED,
+	UCODE_NFOUND,
+	UCODE_ERROR,
+};
 
 struct microcode_ops {
 	enum ucode_state (*request_microcode_user) (int cpu,
@@ -54,7 +60,7 @@ struct microcode_ops {
 	 * are being called.
 	 * See also the "Synchronization" section in microcode_core.c.
 	 */
-	int (*apply_microcode) (int cpu);
+	enum ucode_state (*apply_microcode) (int cpu);
 	int (*collect_cpu_info) (int cpu, struct cpu_signature *csig);
 };
 
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index c931b88982a0..1de72ce514cd 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -74,6 +74,7 @@ static inline void *ldt_slot_va(int slot)
 	return (void *)(LDT_BASE_ADDR + LDT_SLOT_STRIDE * slot);
 #else
 	BUG();
+	return (void *)fix_to_virt(FIX_HOLE);
 #endif
 }
 
diff --git a/arch/x86/include/asm/mpspec_def.h b/arch/x86/include/asm/mpspec_def.h
index a6bec8028480..6fb923a34309 100644
--- a/arch/x86/include/asm/mpspec_def.h
+++ b/arch/x86/include/asm/mpspec_def.h
@@ -128,9 +128,17 @@ enum mp_irq_source_types {
 	mp_ExtINT = 3
 };
 
-#define MP_IRQDIR_DEFAULT	0
-#define MP_IRQDIR_HIGH		1
-#define MP_IRQDIR_LOW		3
+#define MP_IRQPOL_DEFAULT	0x0
+#define MP_IRQPOL_ACTIVE_HIGH	0x1
+#define MP_IRQPOL_RESERVED	0x2
+#define MP_IRQPOL_ACTIVE_LOW	0x3
+#define MP_IRQPOL_MASK		0x3
+
+#define MP_IRQTRIG_DEFAULT	0x0
+#define MP_IRQTRIG_EDGE		0x4
+#define MP_IRQTRIG_RESERVED	0x8
+#define MP_IRQTRIG_LEVEL	0xc
+#define MP_IRQTRIG_MASK		0xc
 
 #define MP_APIC_ALL	0xFF
 
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 8bf450b13d9f..25283f7eb299 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -160,6 +160,7 @@ static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type)
 #define hv_set_synint_state(int_num, val) wrmsrl(int_num, val)
 
 void hyperv_callback_vector(void);
+void hyperv_reenlightenment_vector(void);
 #ifdef CONFIG_TRACING
 #define trace_hyperv_callback_vector hyperv_callback_vector
 #endif
@@ -314,20 +315,29 @@ void hyperv_init(void);
 void hyperv_setup_mmu_ops(void);
 void hyper_alloc_mmu(void);
 void hyperv_report_panic(struct pt_regs *regs, long err);
-bool hv_is_hypercall_page_setup(void);
+bool hv_is_hyperv_initialized(void);
 void hyperv_cleanup(void);
+
+void hyperv_reenlightenment_intr(struct pt_regs *regs);
+void set_hv_tscchange_cb(void (*cb)(void));
+void clear_hv_tscchange_cb(void);
+void hyperv_stop_tsc_emulation(void);
 #else /* CONFIG_HYPERV */
 static inline void hyperv_init(void) {}
-static inline bool hv_is_hypercall_page_setup(void) { return false; }
+static inline bool hv_is_hyperv_initialized(void) { return false; }
 static inline void hyperv_cleanup(void) {}
 static inline void hyperv_setup_mmu_ops(void) {}
+static inline void set_hv_tscchange_cb(void (*cb)(void)) {}
+static inline void clear_hv_tscchange_cb(void) {}
+static inline void hyperv_stop_tsc_emulation(void) {};
 #endif /* CONFIG_HYPERV */
 
 #ifdef CONFIG_HYPERV_TSCPAGE
 struct ms_hyperv_tsc_page *hv_get_tsc_page(void);
-static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg)
+static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg,
+				       u64 *cur_tsc)
 {
-	u64 scale, offset, cur_tsc;
+	u64 scale, offset;
 	u32 sequence;
 
 	/*
@@ -358,7 +368,7 @@ static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg)
 
 		scale = READ_ONCE(tsc_pg->tsc_scale);
 		offset = READ_ONCE(tsc_pg->tsc_offset);
-		cur_tsc = rdtsc_ordered();
+		*cur_tsc = rdtsc_ordered();
 
 		/*
 		 * Make sure we read sequence after we read all other values
@@ -368,7 +378,14 @@ static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg)
 
 	} while (READ_ONCE(tsc_pg->tsc_sequence) != sequence);
 
-	return mul_u64_u64_shr(cur_tsc, scale, 64) + offset;
+	return mul_u64_u64_shr(*cur_tsc, scale, 64) + offset;
+}
+
+static inline u64 hv_read_tsc_page(const struct ms_hyperv_tsc_page *tsc_pg)
+{
+	u64 cur_tsc;
+
+	return hv_read_tsc_page_tsc(tsc_pg, &cur_tsc);
 }
 
 #else
@@ -376,5 +393,12 @@ static inline struct ms_hyperv_tsc_page *hv_get_tsc_page(void)
 {
 	return NULL;
 }
+
+static inline u64 hv_read_tsc_page_tsc(const struct ms_hyperv_tsc_page *tsc_pg,
+				       u64 *cur_tsc)
+{
+	BUG();
+	return U64_MAX;
+}
 #endif
 #endif
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index e7b983a35506..c9084dedfcfa 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -39,6 +39,13 @@
 
 /* Intel MSRs. Some also available on other CPUs */
 
+#define MSR_IA32_SPEC_CTRL		0x00000048 /* Speculation Control */
+#define SPEC_CTRL_IBRS			(1 << 0)   /* Indirect Branch Restricted Speculation */
+#define SPEC_CTRL_STIBP			(1 << 1)   /* Single Thread Indirect Branch Predictors */
+
+#define MSR_IA32_PRED_CMD		0x00000049 /* Prediction Command */
+#define PRED_CMD_IBPB			(1 << 0)   /* Indirect Branch Prediction Barrier */
+
 #define MSR_PPIN_CTL			0x0000004e
 #define MSR_PPIN			0x0000004f
 
@@ -57,6 +64,11 @@
 #define SNB_C3_AUTO_UNDEMOTE		(1UL << 28)
 
 #define MSR_MTRRcap			0x000000fe
+
+#define MSR_IA32_ARCH_CAPABILITIES	0x0000010a
+#define ARCH_CAP_RDCL_NO		(1 << 0)   /* Not susceptible to Meltdown */
+#define ARCH_CAP_IBRS_ALL		(1 << 1)   /* Enhanced IBRS support */
+
 #define MSR_IA32_BBL_CR_CTL		0x00000119
 #define MSR_IA32_BBL_CR_CTL3		0x0000011e
 
@@ -385,6 +397,8 @@
 #define MSR_K7_PERFCTR3			0xc0010007
 #define MSR_K7_CLK_CTL			0xc001001b
 #define MSR_K7_HWCR			0xc0010015
+#define MSR_K7_HWCR_SMMLOCK_BIT		0
+#define MSR_K7_HWCR_SMMLOCK		BIT_ULL(MSR_K7_HWCR_SMMLOCK_BIT)
 #define MSR_K7_FID_VID_CTL		0xc0010041
 #define MSR_K7_FID_VID_STATUS		0xc0010042
 
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 07962f5f6fba..30df295f6d94 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -214,8 +214,7 @@ static __always_inline unsigned long long rdtsc_ordered(void)
 	 * that some other imaginary CPU is updating continuously with a
 	 * time stamp.
 	 */
-	alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC,
-			  "lfence", X86_FEATURE_LFENCE_RDTSC);
+	barrier_nospec();
 	return rdtsc();
 }
 
diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index 4ad41087ce0e..f928ad9b143f 100644
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -1,11 +1,12 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
-#ifndef __NOSPEC_BRANCH_H__
-#define __NOSPEC_BRANCH_H__
+#ifndef _ASM_X86_NOSPEC_BRANCH_H_
+#define _ASM_X86_NOSPEC_BRANCH_H_
 
 #include <asm/alternative.h>
 #include <asm/alternative-asm.h>
 #include <asm/cpufeatures.h>
+#include <asm/msr-index.h>
 
 /*
  * Fill the CPU return stack buffer.
@@ -66,6 +67,18 @@
 	.popsection
 .endm
 
+/*
+ * This should be used immediately before an indirect jump/call. It tells
+ * objtool the subsequent indirect jump/call is vouched safe for retpoline
+ * builds.
+ */
+.macro ANNOTATE_RETPOLINE_SAFE
+	.Lannotate_\@:
+	.pushsection .discard.retpoline_safe
+	_ASM_PTR .Lannotate_\@
+	.popsection
+.endm
+
 /*
  * These are the bare retpoline primitives for indirect jmp and call.
  * Do not use these directly; they only exist to make the ALTERNATIVE
@@ -102,9 +115,9 @@
 .macro JMP_NOSPEC reg:req
 #ifdef CONFIG_RETPOLINE
 	ANNOTATE_NOSPEC_ALTERNATIVE
-	ALTERNATIVE_2 __stringify(jmp *\reg),				\
+	ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; jmp *\reg),	\
 		__stringify(RETPOLINE_JMP \reg), X86_FEATURE_RETPOLINE,	\
-		__stringify(lfence; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
+		__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; jmp *\reg), X86_FEATURE_RETPOLINE_AMD
 #else
 	jmp	*\reg
 #endif
@@ -113,9 +126,9 @@
 .macro CALL_NOSPEC reg:req
 #ifdef CONFIG_RETPOLINE
 	ANNOTATE_NOSPEC_ALTERNATIVE
-	ALTERNATIVE_2 __stringify(call *\reg),				\
+	ALTERNATIVE_2 __stringify(ANNOTATE_RETPOLINE_SAFE; call *\reg),	\
 		__stringify(RETPOLINE_CALL \reg), X86_FEATURE_RETPOLINE,\
-		__stringify(lfence; call *\reg), X86_FEATURE_RETPOLINE_AMD
+		__stringify(lfence; ANNOTATE_RETPOLINE_SAFE; call *\reg), X86_FEATURE_RETPOLINE_AMD
 #else
 	call	*\reg
 #endif
@@ -143,6 +156,12 @@
 	".long 999b - .\n\t"					\
 	".popsection\n\t"
 
+#define ANNOTATE_RETPOLINE_SAFE					\
+	"999:\n\t"						\
+	".pushsection .discard.retpoline_safe\n\t"		\
+	_ASM_PTR " 999b\n\t"					\
+	".popsection\n\t"
+
 #if defined(CONFIG_X86_64) && defined(RETPOLINE)
 
 /*
@@ -152,6 +171,7 @@
 # define CALL_NOSPEC						\
 	ANNOTATE_NOSPEC_ALTERNATIVE				\
 	ALTERNATIVE(						\
+	ANNOTATE_RETPOLINE_SAFE					\
 	"call *%[thunk_target]\n",				\
 	"call __x86_indirect_thunk_%V[thunk_target]\n",		\
 	X86_FEATURE_RETPOLINE)
@@ -163,7 +183,10 @@
  * otherwise we'll run out of registers. We don't care about CET
  * here, anyway.
  */
-# define CALL_NOSPEC ALTERNATIVE("call *%[thunk_target]\n",	\
+# define CALL_NOSPEC						\
+	ALTERNATIVE(						\
+	ANNOTATE_RETPOLINE_SAFE					\
+	"call *%[thunk_target]\n",				\
 	"       jmp    904f;\n"					\
 	"       .align 16\n"					\
 	"901:	call   903f;\n"					\
@@ -201,7 +224,7 @@ extern char __indirect_thunk_end[];
  * On VMEXIT we must ensure that no RSB predictions learned in the guest
  * can be followed in the host, by overwriting the RSB completely. Both
  * retpoline and IBRS mitigations for Spectre v2 need this; only on future
- * CPUs with IBRS_ATT *might* it be avoided.
+ * CPUs with IBRS_ALL *might* it be avoided.
  */
 static inline void vmexit_fill_RSB(void)
 {
@@ -218,5 +241,78 @@ static inline void vmexit_fill_RSB(void)
 #endif
 }
 
+#define alternative_msr_write(_msr, _val, _feature)		\
+	asm volatile(ALTERNATIVE("",				\
+				 "movl %[msr], %%ecx\n\t"	\
+				 "movl %[val], %%eax\n\t"	\
+				 "movl $0, %%edx\n\t"		\
+				 "wrmsr",			\
+				 _feature)			\
+		     : : [msr] "i" (_msr), [val] "i" (_val)	\
+		     : "eax", "ecx", "edx", "memory")
+
+static inline void indirect_branch_prediction_barrier(void)
+{
+	alternative_msr_write(MSR_IA32_PRED_CMD, PRED_CMD_IBPB,
+			      X86_FEATURE_USE_IBPB);
+}
+
+/*
+ * With retpoline, we must use IBRS to restrict branch prediction
+ * before calling into firmware.
+ *
+ * (Implemented as CPP macros due to header hell.)
+ */
+#define firmware_restrict_branch_speculation_start()			\
+do {									\
+	preempt_disable();						\
+	alternative_msr_write(MSR_IA32_SPEC_CTRL, SPEC_CTRL_IBRS,	\
+			      X86_FEATURE_USE_IBRS_FW);			\
+} while (0)
+
+#define firmware_restrict_branch_speculation_end()			\
+do {									\
+	alternative_msr_write(MSR_IA32_SPEC_CTRL, 0,			\
+			      X86_FEATURE_USE_IBRS_FW);			\
+	preempt_enable();						\
+} while (0)
+
 #endif /* __ASSEMBLY__ */
-#endif /* __NOSPEC_BRANCH_H__ */
+
+/*
+ * Below is used in the eBPF JIT compiler and emits the byte sequence
+ * for the following assembly:
+ *
+ * With retpolines configured:
+ *
+ *    callq do_rop
+ *  spec_trap:
+ *    pause
+ *    lfence
+ *    jmp spec_trap
+ *  do_rop:
+ *    mov %rax,(%rsp)
+ *    retq
+ *
+ * Without retpolines configured:
+ *
+ *    jmp *%rax
+ */
+#ifdef CONFIG_RETPOLINE
+# define RETPOLINE_RAX_BPF_JIT_SIZE	17
+# define RETPOLINE_RAX_BPF_JIT()				\
+	EMIT1_off32(0xE8, 7);	 /* callq do_rop */		\
+	/* spec_trap: */					\
+	EMIT2(0xF3, 0x90);       /* pause */			\
+	EMIT3(0x0F, 0xAE, 0xE8); /* lfence */			\
+	EMIT2(0xEB, 0xF9);       /* jmp spec_trap */		\
+	/* do_rop: */						\
+	EMIT4(0x48, 0x89, 0x04, 0x24); /* mov %rax,(%rsp) */	\
+	EMIT1(0xC3);             /* retq */
+#else
+# define RETPOLINE_RAX_BPF_JIT_SIZE	2
+# define RETPOLINE_RAX_BPF_JIT()				\
+	EMIT2(0xFF, 0xE0);	 /* jmp *%rax */
+#endif
+
+#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */
diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
index 4baa6bceb232..d652a3808065 100644
--- a/arch/x86/include/asm/page_64.h
+++ b/arch/x86/include/asm/page_64.h
@@ -52,10 +52,6 @@ static inline void clear_page(void *page)
 
 void copy_page(void *to, void *from);
 
-#ifdef CONFIG_X86_MCE
-#define arch_unmap_kpfn arch_unmap_kpfn
-#endif
-
 #endif	/* !__ASSEMBLY__ */
 
 #ifdef CONFIG_X86_VSYSCALL_EMULATION
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 892df375b615..c83a2f418cea 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -7,6 +7,7 @@
 #ifdef CONFIG_PARAVIRT
 #include <asm/pgtable_types.h>
 #include <asm/asm.h>
+#include <asm/nospec-branch.h>
 
 #include <asm/paravirt_types.h>
 
@@ -297,9 +298,9 @@ static inline void __flush_tlb_global(void)
 {
 	PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
 }
-static inline void __flush_tlb_single(unsigned long addr)
+static inline void __flush_tlb_one_user(unsigned long addr)
 {
-	PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
+	PVOP_VCALL1(pv_mmu_ops.flush_tlb_one_user, addr);
 }
 
 static inline void flush_tlb_others(const struct cpumask *cpumask,
@@ -879,23 +880,27 @@ extern void default_banner(void);
 
 #define INTERRUPT_RETURN						\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,	\
-		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
+		  ANNOTATE_RETPOLINE_SAFE;					\
+		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret);)
 
 #define DISABLE_INTERRUPTS(clobbers)					\
 	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
 		  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);		\
+		  ANNOTATE_RETPOLINE_SAFE;					\
 		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);	\
 		  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 
 #define ENABLE_INTERRUPTS(clobbers)					\
 	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,	\
 		  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);		\
+		  ANNOTATE_RETPOLINE_SAFE;					\
 		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);	\
 		  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 
 #ifdef CONFIG_X86_32
 #define GET_CR0_INTO_EAX				\
 	push %ecx; push %edx;				\
+	ANNOTATE_RETPOLINE_SAFE;				\
 	call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0);	\
 	pop %edx; pop %ecx
 #else	/* !CONFIG_X86_32 */
@@ -917,21 +922,25 @@ extern void default_banner(void);
  */
 #define SWAPGS								\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,	\
-		  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)		\
+		  ANNOTATE_RETPOLINE_SAFE;					\
+		  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs);		\
 		 )
 
 #define GET_CR2_INTO_RAX				\
-	call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2)
+	ANNOTATE_RETPOLINE_SAFE;				\
+	call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2);
 
 #define USERGS_SYSRET64							\
 	PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),	\
 		  CLBR_NONE,						\
-		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
+		  ANNOTATE_RETPOLINE_SAFE;					\
+		  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64);)
 
 #ifdef CONFIG_DEBUG_ENTRY
 #define SAVE_FLAGS(clobbers)                                        \
 	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_save_fl), clobbers, \
 		  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);        \
+		  ANNOTATE_RETPOLINE_SAFE;				    \
 		  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_save_fl);    \
 		  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
 #endif
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 6ec54d01972d..180bc0bff0fb 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -43,6 +43,7 @@
 #include <asm/desc_defs.h>
 #include <asm/kmap_types.h>
 #include <asm/pgtable_types.h>
+#include <asm/nospec-branch.h>
 
 struct page;
 struct thread_struct;
@@ -217,7 +218,7 @@ struct pv_mmu_ops {
 	/* TLB operations */
 	void (*flush_tlb_user)(void);
 	void (*flush_tlb_kernel)(void);
-	void (*flush_tlb_single)(unsigned long addr);
+	void (*flush_tlb_one_user)(unsigned long addr);
 	void (*flush_tlb_others)(const struct cpumask *cpus,
 				 const struct flush_tlb_info *info);
 
@@ -392,7 +393,9 @@ int paravirt_disable_iospace(void);
  * offset into the paravirt_patch_template structure, and can therefore be
  * freely converted back into a structure offset.
  */
-#define PARAVIRT_CALL	"call *%c[paravirt_opptr];"
+#define PARAVIRT_CALL					\
+	ANNOTATE_RETPOLINE_SAFE				\
+	"call *%c[paravirt_opptr];"
 
 /*
  * These macros are intended to wrap calls through one of the paravirt
diff --git a/arch/x86/include/asm/pat.h b/arch/x86/include/asm/pat.h
index 8a3ee355b422..92015c65fa2a 100644
--- a/arch/x86/include/asm/pat.h
+++ b/arch/x86/include/asm/pat.h
@@ -22,4 +22,6 @@ int io_reserve_memtype(resource_size_t start, resource_size_t end,
 
 void io_free_memtype(resource_size_t start, resource_size_t end);
 
+bool pat_pfn_immune_to_uc_mtrr(unsigned long pfn);
+
 #endif /* _ASM_X86_PAT_H */
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index ba3c523aaf16..a06b07399d17 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -526,7 +526,7 @@ static inline bool x86_this_cpu_variable_test_bit(int nr,
 {
 	bool oldbit;
 
-	asm volatile("bt "__percpu_arg(2)",%1"
+	asm volatile("btl "__percpu_arg(2)",%1"
 			CC_SET(c)
 			: CC_OUT(c) (oldbit)
 			: "m" (*(unsigned long __percpu *)addr), "Ir" (nr));
diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h
index bc4af5453802..f24df59c40b2 100644
--- a/arch/x86/include/asm/pgtable-3level.h
+++ b/arch/x86/include/asm/pgtable-3level.h
@@ -158,7 +158,6 @@ static inline pte_t native_ptep_get_and_clear(pte_t *ptep)
 #define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp)
 #endif
 
-#ifdef CONFIG_SMP
 union split_pmd {
 	struct {
 		u32 pmd_low;
@@ -166,6 +165,8 @@ union split_pmd {
 	};
 	pmd_t pmd;
 };
+
+#ifdef CONFIG_SMP
 static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
 {
 	union split_pmd res, *orig = (union split_pmd *)pmdp;
@@ -181,6 +182,40 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *pmdp)
 #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp)
 #endif
 
+#ifndef pmdp_establish
+#define pmdp_establish pmdp_establish
+static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
+		unsigned long address, pmd_t *pmdp, pmd_t pmd)
+{
+	pmd_t old;
+
+	/*
+	 * If pmd has present bit cleared we can get away without expensive
+	 * cmpxchg64: we can update pmdp half-by-half without racing with
+	 * anybody.
+	 */
+	if (!(pmd_val(pmd) & _PAGE_PRESENT)) {
+		union split_pmd old, new, *ptr;
+
+		ptr = (union split_pmd *)pmdp;
+
+		new.pmd = pmd;
+
+		/* xchg acts as a barrier before setting of the high bits */
+		old.pmd_low = xchg(&ptr->pmd_low, new.pmd_low);
+		old.pmd_high = ptr->pmd_high;
+		ptr->pmd_high = new.pmd_high;
+		return old.pmd;
+	}
+
+	do {
+		old = *pmdp;
+	} while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd);
+
+	return old;
+}
+#endif
+
 #ifdef CONFIG_SMP
 union split_pud {
 	struct {
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index e42b8943cb1a..b444d83cfc95 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -350,14 +350,14 @@ static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
 {
 	pmdval_t v = native_pmd_val(pmd);
 
-	return __pmd(v | set);
+	return native_make_pmd(v | set);
 }
 
 static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
 {
 	pmdval_t v = native_pmd_val(pmd);
 
-	return __pmd(v & ~clear);
+	return native_make_pmd(v & ~clear);
 }
 
 static inline pmd_t pmd_mkold(pmd_t pmd)
@@ -409,14 +409,14 @@ static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
 {
 	pudval_t v = native_pud_val(pud);
 
-	return __pud(v | set);
+	return native_make_pud(v | set);
 }
 
 static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
 {
 	pudval_t v = native_pud_val(pud);
 
-	return __pud(v & ~clear);
+	return native_make_pud(v & ~clear);
 }
 
 static inline pud_t pud_mkold(pud_t pud)
@@ -1109,6 +1109,21 @@ static inline int pud_write(pud_t pud)
 	return pud_flags(pud) & _PAGE_RW;
 }
 
+#ifndef pmdp_establish
+#define pmdp_establish pmdp_establish
+static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
+		unsigned long address, pmd_t *pmdp, pmd_t pmd)
+{
+	if (IS_ENABLED(CONFIG_SMP)) {
+		return xchg(pmdp, pmd);
+	} else {
+		pmd_t old = *pmdp;
+		*pmdp = pmd;
+		return old;
+	}
+}
+#endif
+
 /*
  * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
  *
diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h
index e67c0620aec2..b3ec519e3982 100644
--- a/arch/x86/include/asm/pgtable_32.h
+++ b/arch/x86/include/asm/pgtable_32.h
@@ -32,6 +32,7 @@ extern pmd_t initial_pg_pmd[];
 static inline void pgtable_cache_init(void) { }
 static inline void check_pgt_cache(void) { }
 void paging_init(void);
+void sync_initial_page_table(void);
 
 /*
  * Define this if things work differently on an i386 and an i486:
@@ -61,7 +62,7 @@ void paging_init(void);
 #define kpte_clear_flush(ptep, vaddr)		\
 do {						\
 	pte_clear(&init_mm, (vaddr), (ptep));	\
-	__flush_tlb_one((vaddr));		\
+	__flush_tlb_one_kernel((vaddr));		\
 } while (0)
 
 #endif /* !__ASSEMBLY__ */
diff --git a/arch/x86/include/asm/pgtable_32_types.h b/arch/x86/include/asm/pgtable_32_types.h
index ce245b0cdfca..0777e18a1d23 100644
--- a/arch/x86/include/asm/pgtable_32_types.h
+++ b/arch/x86/include/asm/pgtable_32_types.h
@@ -44,8 +44,9 @@ extern bool __vmalloc_start_set; /* set once high_memory is set */
  */
 #define CPU_ENTRY_AREA_PAGES	(NR_CPUS * 40)
 
-#define CPU_ENTRY_AREA_BASE				\
-	((FIXADDR_START - PAGE_SIZE * (CPU_ENTRY_AREA_PAGES + 1)) & PMD_MASK)
+#define CPU_ENTRY_AREA_BASE						\
+	((FIXADDR_TOT_START - PAGE_SIZE * (CPU_ENTRY_AREA_PAGES + 1))   \
+	 & PMD_MASK)
 
 #define PKMAP_BASE		\
 	((CPU_ENTRY_AREA_BASE - PAGE_SIZE) & PMD_MASK)
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 81462e9a34f6..1149d2112b2e 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -28,6 +28,7 @@ extern pgd_t init_top_pgt[];
 #define swapper_pg_dir init_top_pgt
 
 extern void paging_init(void);
+static inline void sync_initial_page_table(void) { }
 
 #define pte_ERROR(e)					\
 	pr_err("%s:%d: bad pte %p(%016lx)\n",		\
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 3696398a9475..acfe755562a6 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -174,7 +174,6 @@ enum page_cache_mode {
 #define __PAGE_KERNEL_RO		(__PAGE_KERNEL & ~_PAGE_RW)
 #define __PAGE_KERNEL_RX		(__PAGE_KERNEL_EXEC & ~_PAGE_RW)
 #define __PAGE_KERNEL_NOCACHE		(__PAGE_KERNEL | _PAGE_NOCACHE)
-#define __PAGE_KERNEL_VSYSCALL		(__PAGE_KERNEL_RX | _PAGE_USER)
 #define __PAGE_KERNEL_VVAR		(__PAGE_KERNEL_RO | _PAGE_USER)
 #define __PAGE_KERNEL_LARGE		(__PAGE_KERNEL | _PAGE_PSE)
 #define __PAGE_KERNEL_LARGE_EXEC	(__PAGE_KERNEL_EXEC | _PAGE_PSE)
@@ -206,7 +205,6 @@ enum page_cache_mode {
 #define PAGE_KERNEL_NOCACHE	__pgprot(__PAGE_KERNEL_NOCACHE | _PAGE_ENC)
 #define PAGE_KERNEL_LARGE	__pgprot(__PAGE_KERNEL_LARGE | _PAGE_ENC)
 #define PAGE_KERNEL_LARGE_EXEC	__pgprot(__PAGE_KERNEL_LARGE_EXEC | _PAGE_ENC)
-#define PAGE_KERNEL_VSYSCALL	__pgprot(__PAGE_KERNEL_VSYSCALL | _PAGE_ENC)
 #define PAGE_KERNEL_VVAR	__pgprot(__PAGE_KERNEL_VVAR | _PAGE_ENC)
 
 #define PAGE_KERNEL_IO		__pgprot(__PAGE_KERNEL_IO)
@@ -323,6 +321,11 @@ static inline pudval_t native_pud_val(pud_t pud)
 #else
 #include <asm-generic/pgtable-nopud.h>
 
+static inline pud_t native_make_pud(pudval_t val)
+{
+	return (pud_t) { .p4d.pgd = native_make_pgd(val) };
+}
+
 static inline pudval_t native_pud_val(pud_t pud)
 {
 	return native_pgd_val(pud.p4d.pgd);
@@ -344,6 +347,11 @@ static inline pmdval_t native_pmd_val(pmd_t pmd)
 #else
 #include <asm-generic/pgtable-nopmd.h>
 
+static inline pmd_t native_make_pmd(pmdval_t val)
+{
+	return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) };
+}
+
 static inline pmdval_t native_pmd_val(pmd_t pmd)
 {
 	return native_pgd_val(pmd.pud.p4d.pgd);
diff --git a/arch/x86/include/asm/pmc_core.h b/arch/x86/include/asm/pmc_core.h
deleted file mode 100644
index d4855f11136d..000000000000
--- a/arch/x86/include/asm/pmc_core.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Intel Core SoC Power Management Controller Header File
- *
- * Copyright (c) 2016, Intel Corporation.
- * All Rights Reserved.
- *
- * Authors: Rajneesh Bhardwaj <rajneesh.bhardwaj@intel.com>
- *          Vishwanath Somayaji <vishwanath.somayaji@intel.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- */
-
-#ifndef _ASM_PMC_CORE_H
-#define _ASM_PMC_CORE_H
-
-/* API to read SLP_S0_RESIDENCY counter */
-int intel_pmc_slp_s0_counter_read(u32 *data);
-
-#endif /* _ASM_PMC_CORE_H */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index d3a67fba200a..b0ccd4847a58 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -91,7 +91,7 @@ struct cpuinfo_x86 {
 	__u8			x86;		/* CPU family */
 	__u8			x86_vendor;	/* CPU vendor */
 	__u8			x86_model;
-	__u8			x86_mask;
+	__u8			x86_stepping;
 #ifdef CONFIG_X86_64
 	/* Number of 4K pages in DTLB/ITLB combined(in pages): */
 	int			x86_tlbsize;
@@ -109,7 +109,7 @@ struct cpuinfo_x86 {
 	char			x86_vendor_id[16];
 	char			x86_model_id[64];
 	/* in KB - valid for CPUS which support this call: */
-	int			x86_cache_size;
+	unsigned int		x86_cache_size;
 	int			x86_cache_alignment;	/* In bytes */
 	/* Cache QoS architectural values: */
 	int			x86_cache_max_rmid;	/* max index */
@@ -460,8 +460,6 @@ struct thread_struct {
 	unsigned short		gsindex;
 #endif
 
-	u32			status;		/* thread synchronous flags */
-
 #ifdef CONFIG_X86_64
 	unsigned long		fsbase;
 	unsigned long		gsbase;
@@ -507,6 +505,14 @@ struct thread_struct {
 	 */
 };
 
+/* Whitelist the FPU state from the task_struct for hardened usercopy. */
+static inline void arch_thread_struct_whitelist(unsigned long *offset,
+						unsigned long *size)
+{
+	*offset = offsetof(struct thread_struct, fpu.state);
+	*size = fpu_kernel_xstate_size;
+}
+
 /*
  * Thread-synchronous status.
  *
@@ -971,4 +977,5 @@ bool xen_set_default_idle(void);
 
 void stop_this_cpu(void *dummy);
 void df_debug(struct pt_regs *regs, long error_code);
+void microcode_check(void);
 #endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h
index 14131dd06b29..6de1fd3d0097 100644
--- a/arch/x86/include/asm/ptrace.h
+++ b/arch/x86/include/asm/ptrace.h
@@ -109,6 +109,11 @@ static inline unsigned long regs_return_value(struct pt_regs *regs)
 	return regs->ax;
 }
 
+static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
+{
+	regs->ax = rc;
+}
+
 /*
  * user_mode(regs) determines whether a register set came from user
  * mode.  On x86_32, this is true if V8086 mode was enabled OR if the
diff --git a/arch/x86/include/asm/refcount.h b/arch/x86/include/asm/refcount.h
index 4e44250e7d0d..4cf11d88d3b3 100644
--- a/arch/x86/include/asm/refcount.h
+++ b/arch/x86/include/asm/refcount.h
@@ -17,7 +17,7 @@
 #define _REFCOUNT_EXCEPTION				\
 	".pushsection .text..refcount\n"		\
 	"111:\tlea %[counter], %%" _ASM_CX "\n"		\
-	"112:\t" ASM_UD0 "\n"				\
+	"112:\t" ASM_UD2 "\n"				\
 	ASM_UNREACHABLE					\
 	".popsection\n"					\
 	"113:\n"					\
@@ -67,13 +67,13 @@ static __always_inline __must_check
 bool refcount_sub_and_test(unsigned int i, refcount_t *r)
 {
 	GEN_BINARY_SUFFIXED_RMWcc(LOCK_PREFIX "subl", REFCOUNT_CHECK_LT_ZERO,
-				  r->refs.counter, "er", i, "%0", e);
+				  r->refs.counter, "er", i, "%0", e, "cx");
 }
 
 static __always_inline __must_check bool refcount_dec_and_test(refcount_t *r)
 {
 	GEN_UNARY_SUFFIXED_RMWcc(LOCK_PREFIX "decl", REFCOUNT_CHECK_LT_ZERO,
-				 r->refs.counter, "%0", e);
+				 r->refs.counter, "%0", e, "cx");
 }
 
 static __always_inline __must_check
diff --git a/arch/x86/include/asm/required-features.h b/arch/x86/include/asm/required-features.h
index d91ba04dd007..fb3a6de7440b 100644
--- a/arch/x86/include/asm/required-features.h
+++ b/arch/x86/include/asm/required-features.h
@@ -106,6 +106,7 @@
 #define REQUIRED_MASK15	0
 #define REQUIRED_MASK16	(NEED_LA57)
 #define REQUIRED_MASK17	0
-#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 18)
+#define REQUIRED_MASK18	0
+#define REQUIRED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
 
 #endif /* _ASM_X86_REQUIRED_FEATURES_H */
diff --git a/arch/x86/include/asm/rmwcc.h b/arch/x86/include/asm/rmwcc.h
index f91c365e57c3..4914a3e7c803 100644
--- a/arch/x86/include/asm/rmwcc.h
+++ b/arch/x86/include/asm/rmwcc.h
@@ -2,8 +2,7 @@
 #ifndef _ASM_X86_RMWcc
 #define _ASM_X86_RMWcc
 
-#define __CLOBBERS_MEM		"memory"
-#define __CLOBBERS_MEM_CC_CX	"memory", "cc", "cx"
+#define __CLOBBERS_MEM(clb...)	"memory", ## clb
 
 #if !defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(CC_HAVE_ASM_GOTO)
 
@@ -40,18 +39,19 @@ do {									\
 #endif /* defined(__GCC_ASM_FLAG_OUTPUTS__) || !defined(CC_HAVE_ASM_GOTO) */
 
 #define GEN_UNARY_RMWcc(op, var, arg0, cc)				\
-	__GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM)
+	__GEN_RMWcc(op " " arg0, var, cc, __CLOBBERS_MEM())
 
-#define GEN_UNARY_SUFFIXED_RMWcc(op, suffix, var, arg0, cc)		\
+#define GEN_UNARY_SUFFIXED_RMWcc(op, suffix, var, arg0, cc, clobbers...)\
 	__GEN_RMWcc(op " " arg0 "\n\t" suffix, var, cc,			\
-		    __CLOBBERS_MEM_CC_CX)
+		    __CLOBBERS_MEM(clobbers))
 
 #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc)			\
 	__GEN_RMWcc(op __BINARY_RMWcc_ARG arg0, var, cc,		\
-		    __CLOBBERS_MEM, vcon (val))
+		    __CLOBBERS_MEM(), vcon (val))
 
-#define GEN_BINARY_SUFFIXED_RMWcc(op, suffix, var, vcon, val, arg0, cc)	\
+#define GEN_BINARY_SUFFIXED_RMWcc(op, suffix, var, vcon, val, arg0, cc,	\
+				  clobbers...)				\
 	__GEN_RMWcc(op __BINARY_RMWcc_ARG arg0 "\n\t" suffix, var, cc,	\
-		    __CLOBBERS_MEM_CC_CX, vcon (val))
+		    __CLOBBERS_MEM(clobbers), vcon (val))
 
 #endif /* _ASM_X86_RMWcc */
diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h
index d6baf23782bc..5c019d23d06b 100644
--- a/arch/x86/include/asm/sections.h
+++ b/arch/x86/include/asm/sections.h
@@ -10,6 +10,7 @@ extern struct exception_table_entry __stop___ex_table[];
 
 #if defined(CONFIG_X86_64)
 extern char __end_rodata_hpage_align[];
+extern char __entry_trampoline_start[], __entry_trampoline_end[];
 #endif
 
 #endif	/* _ASM_X86_SECTIONS_H */
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 461f53d27708..a4189762b266 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -129,6 +129,7 @@ static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 void cpu_disable_common(void);
 void native_smp_prepare_boot_cpu(void);
 void native_smp_prepare_cpus(unsigned int max_cpus);
+void calculate_max_logical_packages(void);
 void native_smp_cpus_done(unsigned int max_cpus);
 void common_cpu_up(unsigned int cpunum, struct task_struct *tidle);
 int native_cpu_up(unsigned int cpunum, struct task_struct *tidle);
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index 78dd9df88157..0487ac054870 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -146,6 +146,9 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define SVM_VM_CR_SVM_LOCK_MASK 0x0008ULL
 #define SVM_VM_CR_SVM_DIS_MASK  0x0010ULL
 
+#define SVM_NESTED_CTL_NP_ENABLE	BIT(0)
+#define SVM_NESTED_CTL_SEV_ENABLE	BIT(1)
+
 struct __attribute__ ((__packed__)) vmcb_seg {
 	u16 selector;
 	u16 attrib;
diff --git a/arch/x86/include/asm/swiotlb.h b/arch/x86/include/asm/swiotlb.h
index bdf9aed40403..1c6a6cb230ff 100644
--- a/arch/x86/include/asm/swiotlb.h
+++ b/arch/x86/include/asm/swiotlb.h
@@ -28,8 +28,6 @@ static inline void pci_swiotlb_late_init(void)
 }
 #endif
 
-static inline void dma_mark_clean(void *addr, size_t size) {}
-
 extern void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
 					dma_addr_t *dma_handle, gfp_t flags,
 					unsigned long attrs);
diff --git a/arch/x86/include/asm/sync_core.h b/arch/x86/include/asm/sync_core.h
new file mode 100644
index 000000000000..c67caafd3381
--- /dev/null
+++ b/arch/x86/include/asm/sync_core.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_SYNC_CORE_H
+#define _ASM_X86_SYNC_CORE_H
+
+#include <linux/preempt.h>
+#include <asm/processor.h>
+#include <asm/cpufeature.h>
+
+/*
+ * Ensure that a core serializing instruction is issued before returning
+ * to user-mode. x86 implements return to user-space through sysexit,
+ * sysrel, and sysretq, which are not core serializing.
+ */
+static inline void sync_core_before_usermode(void)
+{
+	/* With PTI, we unconditionally serialize before running user code. */
+	if (static_cpu_has(X86_FEATURE_PTI))
+		return;
+	/*
+	 * Return from interrupt and NMI is done through iret, which is core
+	 * serializing.
+	 */
+	if (in_irq() || in_nmi())
+		return;
+	sync_core();
+}
+
+#endif /* _ASM_X86_SYNC_CORE_H */
diff --git a/arch/x86/include/asm/sys_ia32.h b/arch/x86/include/asm/sys_ia32.h
index 82c34ee25a65..906794aa034e 100644
--- a/arch/x86/include/asm/sys_ia32.h
+++ b/arch/x86/include/asm/sys_ia32.h
@@ -20,31 +20,43 @@
 #include <asm/ia32.h>
 
 /* ia32/sys_ia32.c */
-asmlinkage long sys32_truncate64(const char __user *, unsigned long, unsigned long);
-asmlinkage long sys32_ftruncate64(unsigned int, unsigned long, unsigned long);
+asmlinkage long compat_sys_x86_truncate64(const char __user *, unsigned long,
+					  unsigned long);
+asmlinkage long compat_sys_x86_ftruncate64(unsigned int, unsigned long,
+					   unsigned long);
 
-asmlinkage long sys32_stat64(const char __user *, struct stat64 __user *);
-asmlinkage long sys32_lstat64(const char __user *, struct stat64 __user *);
-asmlinkage long sys32_fstat64(unsigned int, struct stat64 __user *);
-asmlinkage long sys32_fstatat(unsigned int, const char __user *,
+asmlinkage long compat_sys_x86_stat64(const char __user *,
+				      struct stat64 __user *);
+asmlinkage long compat_sys_x86_lstat64(const char __user *,
+				       struct stat64 __user *);
+asmlinkage long compat_sys_x86_fstat64(unsigned int, struct stat64 __user *);
+asmlinkage long compat_sys_x86_fstatat(unsigned int, const char __user *,
 			      struct stat64 __user *, int);
 struct mmap_arg_struct32;
-asmlinkage long sys32_mmap(struct mmap_arg_struct32 __user *);
+asmlinkage long compat_sys_x86_mmap(struct mmap_arg_struct32 __user *);
 
-asmlinkage long sys32_waitpid(compat_pid_t, unsigned int __user *, int);
+asmlinkage long compat_sys_x86_waitpid(compat_pid_t, unsigned int __user *,
+				       int);
 
-asmlinkage long sys32_pread(unsigned int, char __user *, u32, u32, u32);
-asmlinkage long sys32_pwrite(unsigned int, const char __user *, u32, u32, u32);
+asmlinkage long compat_sys_x86_pread(unsigned int, char __user *, u32, u32,
+				     u32);
+asmlinkage long compat_sys_x86_pwrite(unsigned int, const char __user *, u32,
+				      u32, u32);
 
-long sys32_fadvise64_64(int, __u32, __u32, __u32, __u32, int);
-long sys32_vm86_warning(void);
+asmlinkage long compat_sys_x86_fadvise64_64(int, __u32, __u32, __u32, __u32,
+					    int);
 
-asmlinkage ssize_t sys32_readahead(int, unsigned, unsigned, size_t);
-asmlinkage long sys32_sync_file_range(int, unsigned, unsigned,
-				      unsigned, unsigned, int);
-asmlinkage long sys32_fadvise64(int, unsigned, unsigned, size_t, int);
-asmlinkage long sys32_fallocate(int, int, unsigned,
-				unsigned, unsigned, unsigned);
+asmlinkage ssize_t compat_sys_x86_readahead(int, unsigned int, unsigned int,
+					    size_t);
+asmlinkage long compat_sys_x86_sync_file_range(int, unsigned int, unsigned int,
+					       unsigned int, unsigned int,
+					       int);
+asmlinkage long compat_sys_x86_fadvise64(int, unsigned int, unsigned int,
+					 size_t, int);
+asmlinkage long compat_sys_x86_fallocate(int, int, unsigned int, unsigned int,
+					 unsigned int, unsigned int);
+asmlinkage long compat_sys_x86_clone(unsigned long, unsigned long, int __user *,
+				     unsigned long, int __user *);
 
 /* ia32/ia32_signal.c */
 asmlinkage long sys32_sigreturn(void);
diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h
index e3c95e8e61c5..03eedc21246d 100644
--- a/arch/x86/include/asm/syscall.h
+++ b/arch/x86/include/asm/syscall.h
@@ -60,7 +60,7 @@ static inline long syscall_get_error(struct task_struct *task,
 	 * TS_COMPAT is set for 32-bit syscall entries and then
 	 * remains set until we return to user mode.
 	 */
-	if (task->thread.status & (TS_COMPAT|TS_I386_REGS_POKED))
+	if (task->thread_info.status & (TS_COMPAT|TS_I386_REGS_POKED))
 		/*
 		 * Sign-extend the value so (int)-EFOO becomes (long)-EFOO
 		 * and will match correctly in comparisons.
@@ -116,7 +116,7 @@ static inline void syscall_get_arguments(struct task_struct *task,
 					 unsigned long *args)
 {
 # ifdef CONFIG_IA32_EMULATION
-	if (task->thread.status & TS_COMPAT)
+	if (task->thread_info.status & TS_COMPAT)
 		switch (i) {
 		case 0:
 			if (!n--) break;
@@ -177,7 +177,7 @@ static inline void syscall_set_arguments(struct task_struct *task,
 					 const unsigned long *args)
 {
 # ifdef CONFIG_IA32_EMULATION
-	if (task->thread.status & TS_COMPAT)
+	if (task->thread_info.status & TS_COMPAT)
 		switch (i) {
 		case 0:
 			if (!n--) break;
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 00223333821a..a5d9521bb2cb 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -55,6 +55,7 @@ struct task_struct;
 
 struct thread_info {
 	unsigned long		flags;		/* low level flags */
+	u32			status;		/* thread synchronous flags */
 };
 
 #define INIT_THREAD_INFO(tsk)			\
@@ -62,8 +63,6 @@ struct thread_info {
 	.flags		= 0,			\
 }
 
-#define init_stack		(init_thread_union.stack)
-
 #else /* !__ASSEMBLY__ */
 
 #include <asm/asm-offsets.h>
@@ -221,7 +220,7 @@ static inline int arch_within_stack_frames(const void * const stack,
 #define in_ia32_syscall() true
 #else
 #define in_ia32_syscall() (IS_ENABLED(CONFIG_IA32_EMULATION) && \
-			   current->thread.status & TS_COMPAT)
+			   current_thread_info()->status & TS_COMPAT)
 #endif
 
 /*
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index d33e4a26dc7e..84137c22fdfa 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -140,7 +140,7 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
 #else
 #define __flush_tlb() __native_flush_tlb()
 #define __flush_tlb_global() __native_flush_tlb_global()
-#define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
+#define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr)
 #endif
 
 static inline bool tlb_defer_switch_to_init_mm(void)
@@ -174,6 +174,8 @@ struct tlb_state {
 	struct mm_struct *loaded_mm;
 	u16 loaded_mm_asid;
 	u16 next_asid;
+	/* last user mm's ctx id */
+	u64 last_ctx_id;
 
 	/*
 	 * We can be in one of several states:
@@ -398,7 +400,7 @@ static inline void __native_flush_tlb_global(void)
 /*
  * flush one page in the user mapping
  */
-static inline void __native_flush_tlb_single(unsigned long addr)
+static inline void __native_flush_tlb_one_user(unsigned long addr)
 {
 	u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
 
@@ -435,18 +437,31 @@ static inline void __flush_tlb_all(void)
 /*
  * flush one page in the kernel mapping
  */
-static inline void __flush_tlb_one(unsigned long addr)
+static inline void __flush_tlb_one_kernel(unsigned long addr)
 {
 	count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
-	__flush_tlb_single(addr);
+
+	/*
+	 * If PTI is off, then __flush_tlb_one_user() is just INVLPG or its
+	 * paravirt equivalent.  Even with PCID, this is sufficient: we only
+	 * use PCID if we also use global PTEs for the kernel mapping, and
+	 * INVLPG flushes global translations across all address spaces.
+	 *
+	 * If PTI is on, then the kernel is mapped with non-global PTEs, and
+	 * __flush_tlb_one_user() will flush the given address for the current
+	 * kernel address space and for its usermode counterpart, but it does
+	 * not flush it for other address spaces.
+	 */
+	__flush_tlb_one_user(addr);
 
 	if (!static_cpu_has(X86_FEATURE_PTI))
 		return;
 
 	/*
-	 * __flush_tlb_single() will have cleared the TLB entry for this ASID,
-	 * but since kernel space is replicated across all, we must also
-	 * invalidate all others.
+	 * See above.  We need to propagate the flush to all other address
+	 * spaces.  In principle, we only need to propagate it to kernelmode
+	 * address spaces, but the extra bookkeeping we would need is not
+	 * worth it.
 	 */
 	invalidate_other_asid();
 }
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 574dff4d2913..aae77eb8491c 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -124,6 +124,11 @@ extern int __get_user_bad(void);
 
 #define __uaccess_begin() stac()
 #define __uaccess_end()   clac()
+#define __uaccess_begin_nospec()	\
+({					\
+	stac();				\
+	barrier_nospec();		\
+})
 
 /*
  * This is a type: either unsigned long, if the argument fits into
@@ -445,7 +450,7 @@ do {									\
 ({									\
 	int __gu_err;							\
 	__inttype(*(ptr)) __gu_val;					\
-	__uaccess_begin();						\
+	__uaccess_begin_nospec();					\
 	__get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);	\
 	__uaccess_end();						\
 	(x) = (__force __typeof__(*(ptr)))__gu_val;			\
@@ -487,6 +492,10 @@ struct __large_struct { unsigned long buf[100]; };
 	__uaccess_begin();						\
 	barrier();
 
+#define uaccess_try_nospec do {						\
+	current->thread.uaccess_err = 0;				\
+	__uaccess_begin_nospec();					\
+
 #define uaccess_catch(err)						\
 	__uaccess_end();						\
 	(err) |= (current->thread.uaccess_err ? -EFAULT : 0);		\
@@ -548,7 +557,7 @@ struct __large_struct { unsigned long buf[100]; };
  *	get_user_ex(...);
  * } get_user_catch(err)
  */
-#define get_user_try		uaccess_try
+#define get_user_try		uaccess_try_nospec
 #define get_user_catch(err)	uaccess_catch(err)
 
 #define get_user_ex(x, ptr)	do {					\
@@ -582,7 +591,7 @@ extern void __cmpxchg_wrong_size(void)
 	__typeof__(ptr) __uval = (uval);				\
 	__typeof__(*(ptr)) __old = (old);				\
 	__typeof__(*(ptr)) __new = (new);				\
-	__uaccess_begin();						\
+	__uaccess_begin_nospec();					\
 	switch (size) {							\
 	case 1:								\
 	{								\
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 72950401b223..ba2dc1930630 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -29,21 +29,21 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n)
 		switch (n) {
 		case 1:
 			ret = 0;
-			__uaccess_begin();
+			__uaccess_begin_nospec();
 			__get_user_asm_nozero(*(u8 *)to, from, ret,
 					      "b", "b", "=q", 1);
 			__uaccess_end();
 			return ret;
 		case 2:
 			ret = 0;
-			__uaccess_begin();
+			__uaccess_begin_nospec();
 			__get_user_asm_nozero(*(u16 *)to, from, ret,
 					      "w", "w", "=r", 2);
 			__uaccess_end();
 			return ret;
 		case 4:
 			ret = 0;
-			__uaccess_begin();
+			__uaccess_begin_nospec();
 			__get_user_asm_nozero(*(u32 *)to, from, ret,
 					      "l", "k", "=r", 4);
 			__uaccess_end();
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index f07ef3c575db..62546b3a398e 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -55,31 +55,31 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
 		return copy_user_generic(dst, (__force void *)src, size);
 	switch (size) {
 	case 1:
-		__uaccess_begin();
+		__uaccess_begin_nospec();
 		__get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src,
 			      ret, "b", "b", "=q", 1);
 		__uaccess_end();
 		return ret;
 	case 2:
-		__uaccess_begin();
+		__uaccess_begin_nospec();
 		__get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src,
 			      ret, "w", "w", "=r", 2);
 		__uaccess_end();
 		return ret;
 	case 4:
-		__uaccess_begin();
+		__uaccess_begin_nospec();
 		__get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src,
 			      ret, "l", "k", "=r", 4);
 		__uaccess_end();
 		return ret;
 	case 8:
-		__uaccess_begin();
+		__uaccess_begin_nospec();
 		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
 			      ret, "q", "", "=r", 8);
 		__uaccess_end();
 		return ret;
 	case 10:
-		__uaccess_begin();
+		__uaccess_begin_nospec();
 		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
 			       ret, "q", "", "=r", 10);
 		if (likely(!ret))
@@ -89,7 +89,7 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
 		__uaccess_end();
 		return ret;
 	case 16:
-		__uaccess_begin();
+		__uaccess_begin_nospec();
 		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
 			       ret, "q", "", "=r", 16);
 		if (likely(!ret))
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index 74f4c2ff6427..d8bfa98fca98 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -53,6 +53,10 @@ struct arch_uprobe {
 			u8	fixups;
 			u8	ilen;
 		} 			defparam;
+		struct {
+			u8	reg_offset;	/* to the start of pt_regs */
+			u8	ilen;
+		}			push;
 	};
 };
 
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index 7cac79802ad2..7803114aa140 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -48,7 +48,6 @@
 #define UV2_NET_ENDPOINT_INTD		0x28
 #define UV_NET_ENDPOINT_INTD		(is_uv1_hub() ?			\
 			UV1_NET_ENDPOINT_INTD : UV2_NET_ENDPOINT_INTD)
-#define UV_DESC_PSHIFT			49
 #define UV_PAYLOADQ_GNODE_SHIFT		49
 #define UV_PTC_BASENAME			"sgi_uv/ptc_statistics"
 #define UV_BAU_BASENAME			"sgi_uv/bau_tunables"
diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index 036e26d63d9a..44cf6d6deb7a 100644
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -241,6 +241,7 @@ static inline int uv_hub_info_check(int version)
 #define UV2_HUB_REVISION_BASE		3
 #define UV3_HUB_REVISION_BASE		5
 #define UV4_HUB_REVISION_BASE		7
+#define UV4A_HUB_REVISION_BASE		8	/* UV4 (fixed) rev 2 */
 
 #ifdef	UV1_HUB_IS_SUPPORTED
 static inline int is_uv1_hub(void)
@@ -280,6 +281,19 @@ static inline int is_uv3_hub(void)
 }
 #endif
 
+/* First test "is UV4A", then "is UV4" */
+#ifdef	UV4A_HUB_IS_SUPPORTED
+static inline int is_uv4a_hub(void)
+{
+	return (uv_hub_info->hub_revision >= UV4A_HUB_REVISION_BASE);
+}
+#else
+static inline int is_uv4a_hub(void)
+{
+	return 0;
+}
+#endif
+
 #ifdef	UV4_HUB_IS_SUPPORTED
 static inline int is_uv4_hub(void)
 {
diff --git a/arch/x86/include/asm/uv/uv_mmrs.h b/arch/x86/include/asm/uv/uv_mmrs.h
index 548d684a7960..62c79e26a59a 100644
--- a/arch/x86/include/asm/uv/uv_mmrs.h
+++ b/arch/x86/include/asm/uv/uv_mmrs.h
@@ -39,9 +39,11 @@
  *	#define UV2Hxxx	b
  *	#define UV3Hxxx	c
  *	#define UV4Hxxx	d
+ *	#define UV4AHxxx e
  *	#define UVHxxx	(is_uv1_hub() ? UV1Hxxx :
  *			(is_uv2_hub() ? UV2Hxxx :
  *			(is_uv3_hub() ? UV3Hxxx :
+ *			(is_uv4a_hub() ? UV4AHxxx :
  *					UV4Hxxx))
  *
  * If the MMR exists on all hub types > 1 but have different addresses, the
@@ -49,8 +51,10 @@
  *	#define UV2Hxxx	b
  *	#define UV3Hxxx	c
  *	#define UV4Hxxx	d
+ *	#define UV4AHxxx e
  *	#define UVHxxx	(is_uv2_hub() ? UV2Hxxx :
  *			(is_uv3_hub() ? UV3Hxxx :
+ *			(is_uv4a_hub() ? UV4AHxxx :
  *					UV4Hxxx))
  *
  *	union uvh_xxx {
@@ -63,6 +67,7 @@
  *		} s2;
  *		struct uv3h_xxx_s {	 # Full UV3 definition (*)
  *		} s3;
+ *		(NOTE: No struct uv4ah_xxx_s members exist)
  *		struct uv4h_xxx_s {	 # Full UV4 definition (*)
  *		} s4;
  *	};
@@ -99,6 +104,7 @@
 #define UV2_HUB_IS_SUPPORTED	1
 #define UV3_HUB_IS_SUPPORTED	1
 #define UV4_HUB_IS_SUPPORTED	1
+#define UV4A_HUB_IS_SUPPORTED	1
 
 /* Error function to catch undefined references */
 extern unsigned long uv_undefined(char *str);
@@ -2779,35 +2785,47 @@ union uvh_lb_bau_sb_activation_status_1_u {
 	/*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_BASE_32)
 
 #define UVH_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_SHFT	12
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT	49
-#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK	0x7ffe000000000000UL
 
+#define UV1H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT	49
 #define UV1H_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000007fffffff000UL
+#define UV1H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK	0x7ffe000000000000UL
 
-
+#define UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT	49
 #define UV2H_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000007fffffff000UL
+#define UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK	0x7ffe000000000000UL
 
+#define UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT	49
 #define UV3H_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000007fffffff000UL
+#define UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK	0x7ffe000000000000UL
 
+#define UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT	49
 #define UV4H_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x00003ffffffff000UL
-
-
-union uvh_lb_bau_sb_descriptor_base_u {
-	unsigned long	v;
-	struct uvh_lb_bau_sb_descriptor_base_s {
-		unsigned long	rsvd_0_11:12;
-		unsigned long	rsvd_12_48:37;
-		unsigned long	node_id:14;			/* RW */
-		unsigned long	rsvd_63:1;
-	} s;
-	struct uv4h_lb_bau_sb_descriptor_base_s {
-		unsigned long	rsvd_0_11:12;
-		unsigned long	page_address:34;		/* RW */
-		unsigned long	rsvd_46_48:3;
-		unsigned long	node_id:14;			/* RW */
-		unsigned long	rsvd_63:1;
-	} s4;
-};
+#define UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK	0x7ffe000000000000UL
+
+#define UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT	53
+#define UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_PAGE_ADDRESS_MASK 0x000ffffffffff000UL
+#define UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK	0xffe0000000000000UL
+
+#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT (			\
+	is_uv1_hub() ? UV1H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT :	\
+	is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT :	\
+	is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT :	\
+	is_uv4a_hub() ? UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT :	\
+	/*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT)
+
+#define UVH_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK (			\
+	is_uv1_hub() ? UV1H_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK :	\
+	is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK :	\
+	is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK :	\
+	is_uv4a_hub() ? UV4AH_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK :	\
+	/*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_PAGE_ADDRESS_MASK)
+
+#define UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK (			\
+	is_uv1_hub() ? UV1H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK :	\
+	is_uv2_hub() ? UV2H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK :	\
+	is_uv3_hub() ? UV3H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK :	\
+	is_uv4a_hub() ? UV4AH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK :	\
+	/*is_uv4_hub*/ UV4H_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_MASK)
 
 /* ========================================================================= */
 /*                               UVH_NODE_ID                                 */
@@ -3031,6 +3049,41 @@ union uvh_node_present_table_u {
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
 
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_SHFT 24
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_SHFT 48
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_SHFT 63
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_0_MMR_ENABLE_MASK 0x8000000000000000UL
+
 
 union uvh_rh_gam_alias210_overlay_config_0_mmr_u {
 	unsigned long	v;
@@ -3042,6 +3095,46 @@ union uvh_rh_gam_alias210_overlay_config_0_mmr_u {
 		unsigned long	rsvd_53_62:10;
 		unsigned long	enable:1;			/* RW */
 	} s;
+	struct uv1h_rh_gam_alias210_overlay_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s1;
+	struct uvxh_rh_gam_alias210_overlay_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} sx;
+	struct uv2h_rh_gam_alias210_overlay_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s2;
+	struct uv3h_rh_gam_alias210_overlay_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s3;
+	struct uv4h_rh_gam_alias210_overlay_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s4;
 };
 
 /* ========================================================================= */
@@ -3064,6 +3157,41 @@ union uvh_rh_gam_alias210_overlay_config_0_mmr_u {
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
 
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_SHFT 24
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_SHFT 48
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_SHFT 63
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_1_MMR_ENABLE_MASK 0x8000000000000000UL
+
 
 union uvh_rh_gam_alias210_overlay_config_1_mmr_u {
 	unsigned long	v;
@@ -3075,6 +3203,46 @@ union uvh_rh_gam_alias210_overlay_config_1_mmr_u {
 		unsigned long	rsvd_53_62:10;
 		unsigned long	enable:1;			/* RW */
 	} s;
+	struct uv1h_rh_gam_alias210_overlay_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s1;
+	struct uvxh_rh_gam_alias210_overlay_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} sx;
+	struct uv2h_rh_gam_alias210_overlay_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s2;
+	struct uv3h_rh_gam_alias210_overlay_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s3;
+	struct uv4h_rh_gam_alias210_overlay_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s4;
 };
 
 /* ========================================================================= */
@@ -3097,6 +3265,41 @@ union uvh_rh_gam_alias210_overlay_config_1_mmr_u {
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
 #define UVH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
 
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV1H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UVXH_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV2H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV3H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_SHFT 24
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_SHFT 48
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_SHFT 63
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_BASE_MASK 0x00000000ff000000UL
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_M_ALIAS_MASK 0x001f000000000000UL
+#define UV4H_RH_GAM_ALIAS210_OVERLAY_CONFIG_2_MMR_ENABLE_MASK 0x8000000000000000UL
+
 
 union uvh_rh_gam_alias210_overlay_config_2_mmr_u {
 	unsigned long	v;
@@ -3108,6 +3311,46 @@ union uvh_rh_gam_alias210_overlay_config_2_mmr_u {
 		unsigned long	rsvd_53_62:10;
 		unsigned long	enable:1;			/* RW */
 	} s;
+	struct uv1h_rh_gam_alias210_overlay_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s1;
+	struct uvxh_rh_gam_alias210_overlay_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} sx;
+	struct uv2h_rh_gam_alias210_overlay_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s2;
+	struct uv3h_rh_gam_alias210_overlay_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s3;
+	struct uv4h_rh_gam_alias210_overlay_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	base:8;				/* RW */
+		unsigned long	rsvd_32_47:16;
+		unsigned long	m_alias:5;			/* RW */
+		unsigned long	rsvd_53_62:10;
+		unsigned long	enable:1;			/* RW */
+	} s4;
 };
 
 /* ========================================================================= */
@@ -3126,6 +3369,21 @@ union uvh_rh_gam_alias210_overlay_config_2_mmr_u {
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
 
+#define UV1H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
+#define UV1H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
+#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
+#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_SHFT 24
+#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_0_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
 
 union uvh_rh_gam_alias210_redirect_config_0_mmr_u {
 	unsigned long	v;
@@ -3134,6 +3392,31 @@ union uvh_rh_gam_alias210_redirect_config_0_mmr_u {
 		unsigned long	dest_base:22;			/* RW */
 		unsigned long	rsvd_46_63:18;
 	} s;
+	struct uv1h_rh_gam_alias210_redirect_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s1;
+	struct uvxh_rh_gam_alias210_redirect_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} sx;
+	struct uv2h_rh_gam_alias210_redirect_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s2;
+	struct uv3h_rh_gam_alias210_redirect_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s3;
+	struct uv4h_rh_gam_alias210_redirect_config_0_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s4;
 };
 
 /* ========================================================================= */
@@ -3152,6 +3435,21 @@ union uvh_rh_gam_alias210_redirect_config_0_mmr_u {
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
 
+#define UV1H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
+#define UV1H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
+#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
+#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_SHFT 24
+#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_1_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
 
 union uvh_rh_gam_alias210_redirect_config_1_mmr_u {
 	unsigned long	v;
@@ -3160,6 +3458,31 @@ union uvh_rh_gam_alias210_redirect_config_1_mmr_u {
 		unsigned long	dest_base:22;			/* RW */
 		unsigned long	rsvd_46_63:18;
 	} s;
+	struct uv1h_rh_gam_alias210_redirect_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s1;
+	struct uvxh_rh_gam_alias210_redirect_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} sx;
+	struct uv2h_rh_gam_alias210_redirect_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s2;
+	struct uv3h_rh_gam_alias210_redirect_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s3;
+	struct uv4h_rh_gam_alias210_redirect_config_1_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s4;
 };
 
 /* ========================================================================= */
@@ -3178,6 +3501,21 @@ union uvh_rh_gam_alias210_redirect_config_1_mmr_u {
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
 #define UVH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
 
+#define UV1H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
+#define UV1H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
+#define UVXH_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
+#define UV2H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
+#define UV3H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
+#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_SHFT 24
+#define UV4H_RH_GAM_ALIAS210_REDIRECT_CONFIG_2_MMR_DEST_BASE_MASK 0x00003fffff000000UL
+
 
 union uvh_rh_gam_alias210_redirect_config_2_mmr_u {
 	unsigned long	v;
@@ -3186,6 +3524,31 @@ union uvh_rh_gam_alias210_redirect_config_2_mmr_u {
 		unsigned long	dest_base:22;			/* RW */
 		unsigned long	rsvd_46_63:18;
 	} s;
+	struct uv1h_rh_gam_alias210_redirect_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s1;
+	struct uvxh_rh_gam_alias210_redirect_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} sx;
+	struct uv2h_rh_gam_alias210_redirect_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s2;
+	struct uv3h_rh_gam_alias210_redirect_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s3;
+	struct uv4h_rh_gam_alias210_redirect_config_2_mmr_s {
+		unsigned long	rsvd_0_23:24;
+		unsigned long	dest_base:22;			/* RW */
+		unsigned long	rsvd_46_63:18;
+	} s4;
 };
 
 /* ========================================================================= */
@@ -3383,6 +3746,162 @@ union uvh_rh_gam_gru_overlay_config_mmr_u {
 	} s4;
 };
 
+/* ========================================================================= */
+/*                   UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR                    */
+/* ========================================================================= */
+#define UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR uv_undefined("UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR")
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR uv_undefined("UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR")
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR 0x1603000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR 0x483000UL
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR (				\
+	is_uv1_hub() ? UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR :		\
+	is_uv2_hub() ? UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR :		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR :		\
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR)
+
+
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT	26
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT	46
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_SHFT 63
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK	0x00003ffffc000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK	0x000fc00000000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT	26
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT	46
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_SHFT 63
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK	0x00003ffffc000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK	0x000fc00000000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT 52
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK 0x000ffffffc000000UL
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK 0x03f0000000000000UL
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT)
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK)
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK)
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK)
+
+union uvh_rh_gam_mmioh_overlay_config0_mmr_u {
+	unsigned long	v;
+	struct uv3h_rh_gam_mmioh_overlay_config0_mmr_s {
+		unsigned long	rsvd_0_25:26;
+		unsigned long	base:20;			/* RW */
+		unsigned long	m_io:6;				/* RW */
+		unsigned long	n_io:4;
+		unsigned long	rsvd_56_62:7;
+		unsigned long	enable:1;			/* RW */
+	} s3;
+	struct uv4h_rh_gam_mmioh_overlay_config0_mmr_s {
+		unsigned long	rsvd_0_25:26;
+		unsigned long	base:20;			/* RW */
+		unsigned long	m_io:6;				/* RW */
+		unsigned long	n_io:4;
+		unsigned long	rsvd_56_62:7;
+		unsigned long	enable:1;			/* RW */
+	} s4;
+	struct uv4ah_rh_gam_mmioh_overlay_config0_mmr_s {
+		unsigned long	rsvd_0_25:26;
+		unsigned long	base:26;			/* RW */
+		unsigned long	m_io:6;				/* RW */
+		unsigned long	n_io:4;
+		unsigned long	undef_62:1;			/* Undefined */
+		unsigned long	enable:1;			/* RW */
+	} s4a;
+};
+
+/* ========================================================================= */
+/*                   UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR                    */
+/* ========================================================================= */
+#define UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR uv_undefined("UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR")
+#define UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR uv_undefined("UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR")
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR 0x1603000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR 0x484000UL
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR (				\
+	is_uv1_hub() ? UV1H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR :		\
+	is_uv2_hub() ? UV2H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR :		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR :		\
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR)
+
+
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_SHFT	26
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT	46
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_SHFT 63
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK	0x00003ffffc000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK	0x000fc00000000000UL
+#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_SHFT	26
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT	46
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_SHFT 63
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK	0x00003ffffc000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK	0x000fc00000000000UL
+#define UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_MASK 0x8000000000000000UL
+
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT 52
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK 0x000ffffffc000000UL
+#define UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK 0x03f0000000000000UL
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT)
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK)
+
+#define UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK)
+
+union uvh_rh_gam_mmioh_overlay_config1_mmr_u {
+	unsigned long	v;
+	struct uv3h_rh_gam_mmioh_overlay_config1_mmr_s {
+		unsigned long	rsvd_0_25:26;
+		unsigned long	base:20;			/* RW */
+		unsigned long	m_io:6;				/* RW */
+		unsigned long	n_io:4;
+		unsigned long	rsvd_56_62:7;
+		unsigned long	enable:1;			/* RW */
+	} s3;
+	struct uv4h_rh_gam_mmioh_overlay_config1_mmr_s {
+		unsigned long	rsvd_0_25:26;
+		unsigned long	base:20;			/* RW */
+		unsigned long	m_io:6;				/* RW */
+		unsigned long	n_io:4;
+		unsigned long	rsvd_56_62:7;
+		unsigned long	enable:1;			/* RW */
+	} s4;
+	struct uv4ah_rh_gam_mmioh_overlay_config1_mmr_s {
+		unsigned long	rsvd_0_25:26;
+		unsigned long	base:26;			/* RW */
+		unsigned long	m_io:6;				/* RW */
+		unsigned long	n_io:4;
+		unsigned long	undef_62:1;			/* Undefined */
+		unsigned long	enable:1;			/* RW */
+	} s4a;
+};
+
 /* ========================================================================= */
 /*                   UVH_RH_GAM_MMIOH_OVERLAY_CONFIG_MMR                     */
 /* ========================================================================= */
@@ -3437,6 +3956,112 @@ union uvh_rh_gam_mmioh_overlay_config_mmr_u {
 	} s2;
 };
 
+/* ========================================================================= */
+/*                  UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR                    */
+/* ========================================================================= */
+#define UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR uv_undefined("UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR")
+#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR")
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR 0x1603800UL
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR 0x483800UL
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR (				\
+	is_uv1_hub() ? UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR :		\
+	is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR :		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR :		\
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR)
+
+#define UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH uv_undefined("UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH")
+#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH")
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH 128
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH 128
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH (			\
+	is_uv1_hub() ? UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH :	\
+	is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH :	\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH :	\
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH)
+
+
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_SHFT 0
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000007fffUL
+
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_SHFT 0
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000007fffUL
+
+#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000000fffUL
+
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK)
+
+union uvh_rh_gam_mmioh_redirect_config0_mmr_u {
+	unsigned long	v;
+	struct uv3h_rh_gam_mmioh_redirect_config0_mmr_s {
+		unsigned long	nasid:15;			/* RW */
+		unsigned long	rsvd_15_63:49;
+	} s3;
+	struct uv4h_rh_gam_mmioh_redirect_config0_mmr_s {
+		unsigned long	nasid:15;			/* RW */
+		unsigned long	rsvd_15_63:49;
+	} s4;
+	struct uv4ah_rh_gam_mmioh_redirect_config0_mmr_s {
+		unsigned long	nasid:12;			/* RW */
+		unsigned long	rsvd_12_63:52;
+	} s4a;
+};
+
+/* ========================================================================= */
+/*                  UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR                    */
+/* ========================================================================= */
+#define UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR uv_undefined("UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR")
+#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR")
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR 0x1604800UL
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR 0x484800UL
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR (				\
+	is_uv1_hub() ? UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR :		\
+	is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR :		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR :		\
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR)
+
+#define UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH uv_undefined("UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH")
+#define UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH uv_undefined("UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH")
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH 128
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH 128
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH (			\
+	is_uv1_hub() ? UV1H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH :	\
+	is_uv2_hub() ? UV2H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH :	\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH :	\
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH)
+
+
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_SHFT 0
+#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000007fffUL
+
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_SHFT 0
+#define UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000007fffUL
+
+#define UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000000fffUL
+
+#define UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK (		\
+	is_uv3_hub() ? UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK : \
+	is_uv4a_hub() ? UV4AH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK : \
+	/*is_uv4_hub*/ UV4H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK)
+
+union uvh_rh_gam_mmioh_redirect_config1_mmr_u {
+	unsigned long	v;
+	struct uv3h_rh_gam_mmioh_redirect_config1_mmr_s {
+		unsigned long	nasid:15;			/* RW */
+		unsigned long	rsvd_15_63:49;
+	} s3;
+	struct uv4h_rh_gam_mmioh_redirect_config1_mmr_s {
+		unsigned long	nasid:15;			/* RW */
+		unsigned long	rsvd_15_63:49;
+	} s4;
+	struct uv4ah_rh_gam_mmioh_redirect_config1_mmr_s {
+		unsigned long	nasid:12;			/* RW */
+		unsigned long	rsvd_12_63:52;
+	} s4a;
+};
+
 /* ========================================================================= */
 /*                    UVH_RH_GAM_MMR_OVERLAY_CONFIG_MMR                      */
 /* ========================================================================= */
@@ -4137,88 +4762,6 @@ union uv3h_gr0_gam_gr_config_u {
 	} s3;
 };
 
-/* ========================================================================= */
-/*                   UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR                   */
-/* ========================================================================= */
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR		0x1603000UL
-
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT	26
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT	46
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK	0x00003ffffc000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK	0x000fc00000000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK 0x8000000000000000UL
-
-union uv3h_rh_gam_mmioh_overlay_config0_mmr_u {
-	unsigned long	v;
-	struct uv3h_rh_gam_mmioh_overlay_config0_mmr_s {
-		unsigned long	rsvd_0_25:26;
-		unsigned long	base:20;			/* RW */
-		unsigned long	m_io:6;				/* RW */
-		unsigned long	n_io:4;
-		unsigned long	rsvd_56_62:7;
-		unsigned long	enable:1;			/* RW */
-	} s3;
-};
-
-/* ========================================================================= */
-/*                   UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR                   */
-/* ========================================================================= */
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR		0x1604000UL
-
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_SHFT	26
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT	46
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_SHFT 63
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK	0x00003ffffc000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK	0x000fc00000000000UL
-#define UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_ENABLE_MASK 0x8000000000000000UL
-
-union uv3h_rh_gam_mmioh_overlay_config1_mmr_u {
-	unsigned long	v;
-	struct uv3h_rh_gam_mmioh_overlay_config1_mmr_s {
-		unsigned long	rsvd_0_25:26;
-		unsigned long	base:20;			/* RW */
-		unsigned long	m_io:6;				/* RW */
-		unsigned long	n_io:4;
-		unsigned long	rsvd_56_62:7;
-		unsigned long	enable:1;			/* RW */
-	} s3;
-};
-
-/* ========================================================================= */
-/*                  UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR                   */
-/* ========================================================================= */
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR		0x1603800UL
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH	128
-
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_SHFT 0
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK 0x0000000000007fffUL
-
-union uv3h_rh_gam_mmioh_redirect_config0_mmr_u {
-	unsigned long	v;
-	struct uv3h_rh_gam_mmioh_redirect_config0_mmr_s {
-		unsigned long	nasid:15;			/* RW */
-		unsigned long	rsvd_15_63:49;
-	} s3;
-};
-
-/* ========================================================================= */
-/*                  UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR                   */
-/* ========================================================================= */
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR		0x1604800UL
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH	128
-
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_SHFT 0
-#define UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK 0x0000000000007fffUL
-
-union uv3h_rh_gam_mmioh_redirect_config1_mmr_u {
-	unsigned long	v;
-	struct uv3h_rh_gam_mmioh_redirect_config1_mmr_s {
-		unsigned long	nasid:15;			/* RW */
-		unsigned long	rsvd_15_63:49;
-	} s3;
-};
-
 /* ========================================================================= */
 /*                       UV4H_LB_PROC_INTD_QUEUE_FIRST                       */
 /* ========================================================================= */
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 8b6780751132..5db8b0b10766 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -352,6 +352,7 @@ enum vmcs_field {
 #define INTR_TYPE_NMI_INTR		(2 << 8) /* NMI */
 #define INTR_TYPE_HARD_EXCEPTION	(3 << 8) /* processor exception */
 #define INTR_TYPE_SOFT_INTR             (4 << 8) /* software interrupt */
+#define INTR_TYPE_PRIV_SW_EXCEPTION	(5 << 8) /* ICE breakpoint - undocumented */
 #define INTR_TYPE_SOFT_EXCEPTION	(6 << 8) /* software exception */
 
 /* GUEST_INTERRUPTIBILITY_INFO flags. */
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index aa4747569e23..fc2f082ac635 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -212,6 +212,7 @@ enum x86_legacy_i8042_state {
 struct x86_legacy_features {
 	enum x86_legacy_i8042_state i8042;
 	int rtc;
+	int warm_reset;
 	int no_vga;
 	int reserve_bios_regions;
 	struct x86_legacy_devices devices;
diff --git a/arch/x86/include/uapi/asm/Kbuild b/arch/x86/include/uapi/asm/Kbuild
index 1e901e421f2d..322681622d1e 100644
--- a/arch/x86/include/uapi/asm/Kbuild
+++ b/arch/x86/include/uapi/asm/Kbuild
@@ -5,3 +5,4 @@ generic-y += bpf_perf_event.h
 generated-y += unistd_32.h
 generated-y += unistd_64.h
 generated-y += unistd_x32.h
+generic-y += poll.h
diff --git a/arch/x86/include/uapi/asm/bootparam.h b/arch/x86/include/uapi/asm/bootparam.h
index afdd5ae0fcc4..aebf60357758 100644
--- a/arch/x86/include/uapi/asm/bootparam.h
+++ b/arch/x86/include/uapi/asm/bootparam.h
@@ -9,6 +9,7 @@
 #define SETUP_PCI			3
 #define SETUP_EFI			4
 #define SETUP_APPLE_PROPERTIES		5
+#define SETUP_JAILHOUSE			6
 
 /* ram_size flags */
 #define RAMDISK_IMAGE_START_MASK	0x07FF
@@ -126,6 +127,27 @@ struct boot_e820_entry {
 	__u32 type;
 } __attribute__((packed));
 
+/*
+ * Smallest compatible version of jailhouse_setup_data required by this kernel.
+ */
+#define JAILHOUSE_SETUP_REQUIRED_VERSION	1
+
+/*
+ * The boot loader is passing platform information via this Jailhouse-specific
+ * setup data structure.
+ */
+struct jailhouse_setup_data {
+	u16	version;
+	u16	compatible_version;
+	u16	pm_timer_address;
+	u16	num_cpus;
+	u64	pci_mmconfig_base;
+	u32	tsc_khz;
+	u32	apic_khz;
+	u8	standard_ioapic;
+	u8	cpu_ids[255];
+} __attribute__((packed));
+
 /* The so-called "zeropage" */
 struct boot_params {
 	struct screen_info screen_info;			/* 0x000 */
diff --git a/arch/x86/include/uapi/asm/hyperv.h b/arch/x86/include/uapi/asm/hyperv.h
index 1a5bfead93b4..099414345865 100644
--- a/arch/x86/include/uapi/asm/hyperv.h
+++ b/arch/x86/include/uapi/asm/hyperv.h
@@ -40,6 +40,9 @@
  */
 #define HV_X64_ACCESS_FREQUENCY_MSRS		(1 << 11)
 
+/* AccessReenlightenmentControls privilege */
+#define HV_X64_ACCESS_REENLIGHTENMENT		BIT(13)
+
 /*
  * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
  * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available
@@ -234,6 +237,30 @@
 #define HV_X64_MSR_CRASH_PARAMS		\
 		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
 
+/* TSC emulation after migration */
+#define HV_X64_MSR_REENLIGHTENMENT_CONTROL	0x40000106
+
+struct hv_reenlightenment_control {
+	__u64 vector:8;
+	__u64 reserved1:8;
+	__u64 enabled:1;
+	__u64 reserved2:15;
+	__u64 target_vp:32;
+};
+
+#define HV_X64_MSR_TSC_EMULATION_CONTROL	0x40000107
+#define HV_X64_MSR_TSC_EMULATION_STATUS		0x40000108
+
+struct hv_tsc_emulation_control {
+	__u64 enabled:1;
+	__u64 reserved:63;
+};
+
+struct hv_tsc_emulation_status {
+	__u64 inprogress:1;
+	__u64 reserved:63;
+};
+
 #define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
 #define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
 #define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h
index 09cc06483bed..6cfa9c8cb7d6 100644
--- a/arch/x86/include/uapi/asm/kvm_para.h
+++ b/arch/x86/include/uapi/asm/kvm_para.h
@@ -25,6 +25,8 @@
 #define KVM_FEATURE_STEAL_TIME		5
 #define KVM_FEATURE_PV_EOI		6
 #define KVM_FEATURE_PV_UNHALT		7
+#define KVM_FEATURE_PV_TLB_FLUSH	9
+#define KVM_FEATURE_ASYNC_PF_VMEXIT	10
 
 /* The last 8 bits are used to indicate how to interpret the flags field
  * in pvclock structure. If no bits are set, all flags are ignored.
@@ -51,6 +53,9 @@ struct kvm_steal_time {
 	__u32 pad[11];
 };
 
+#define KVM_VCPU_PREEMPTED          (1 << 0)
+#define KVM_VCPU_FLUSH_TLB          (1 << 1)
+
 #define KVM_CLOCK_PAIRING_WALLCLOCK 0
 struct kvm_clock_pairing {
 	__s64 sec;
diff --git a/arch/x86/include/uapi/asm/mce.h b/arch/x86/include/uapi/asm/mce.h
index 91723461dc1f..435db58a7bad 100644
--- a/arch/x86/include/uapi/asm/mce.h
+++ b/arch/x86/include/uapi/asm/mce.h
@@ -30,6 +30,7 @@ struct mce {
 	__u64 synd;	/* MCA_SYND MSR: only valid on SMCA systems */
 	__u64 ipid;	/* MCA_IPID MSR: only valid on SMCA systems */
 	__u64 ppin;	/* Protected Processor Inventory Number */
+	__u32 microcode;/* Microcode revision */
 };
 
 #define MCE_GET_RECORD_LEN   _IOR('M', 1, int)
diff --git a/arch/x86/include/uapi/asm/poll.h b/arch/x86/include/uapi/asm/poll.h
deleted file mode 100644
index c98509d3149e..000000000000
--- a/arch/x86/include/uapi/asm/poll.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <asm-generic/poll.h>
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 7e2baf7304ae..29786c87e864 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -115,6 +115,8 @@ obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o
 obj-$(CONFIG_PARAVIRT_CLOCK)	+= pvclock.o
 obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o
 
+obj-$(CONFIG_JAILHOUSE_GUEST)	+= jailhouse.o
+
 obj-$(CONFIG_EISA)		+= eisa.o
 obj-$(CONFIG_PCSPKR_PLATFORM)	+= pcspeaker.o
 
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index f4c463df8b08..2aa92094b59d 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -36,6 +36,7 @@
 #include <linux/ioport.h>
 #include <linux/pci.h>
 #include <linux/efi-bgrt.h>
+#include <linux/serial_core.h>
 
 #include <asm/e820/api.h>
 #include <asm/irqdomain.h>
@@ -68,8 +69,9 @@ int acpi_ioapic;
 int acpi_strict;
 int acpi_disable_cmcff;
 
+/* ACPI SCI override configuration */
 u8 acpi_sci_flags __initdata;
-int acpi_sci_override_gsi __initdata;
+u32 acpi_sci_override_gsi __initdata = INVALID_ACPI_IRQ;
 int acpi_skip_timer_override __initdata;
 int acpi_use_timer_override __initdata;
 int acpi_fix_pin2_polarity __initdata;
@@ -112,8 +114,6 @@ static u32 isa_irq_to_gsi[NR_IRQS_LEGACY] __read_mostly = {
 	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
 };
 
-#define	ACPI_INVALID_GSI		INT_MIN
-
 /*
  * This is just a simple wrapper around early_memremap(),
  * with sanity checks for phys == 0 and size == 0.
@@ -372,7 +372,7 @@ static void __init mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger,
 	 * and acpi_isa_irq_to_gsi() may give wrong result.
 	 */
 	if (gsi < nr_legacy_irqs() && isa_irq_to_gsi[gsi] == gsi)
-		isa_irq_to_gsi[gsi] = ACPI_INVALID_GSI;
+		isa_irq_to_gsi[gsi] = INVALID_ACPI_IRQ;
 	isa_irq_to_gsi[bus_irq] = gsi;
 }
 
@@ -620,24 +620,24 @@ int acpi_gsi_to_irq(u32 gsi, unsigned int *irqp)
 	}
 
 	rc = acpi_get_override_irq(gsi, &trigger, &polarity);
-	if (rc == 0) {
-		trigger = trigger ? ACPI_LEVEL_SENSITIVE : ACPI_EDGE_SENSITIVE;
-		polarity = polarity ? ACPI_ACTIVE_LOW : ACPI_ACTIVE_HIGH;
-		irq = acpi_register_gsi(NULL, gsi, trigger, polarity);
-		if (irq >= 0) {
-			*irqp = irq;
-			return 0;
-		}
-	}
+	if (rc)
+		return rc;
 
-	return -1;
+	trigger = trigger ? ACPI_LEVEL_SENSITIVE : ACPI_EDGE_SENSITIVE;
+	polarity = polarity ? ACPI_ACTIVE_LOW : ACPI_ACTIVE_HIGH;
+	irq = acpi_register_gsi(NULL, gsi, trigger, polarity);
+	if (irq < 0)
+		return irq;
+
+	*irqp = irq;
+	return 0;
 }
 EXPORT_SYMBOL_GPL(acpi_gsi_to_irq);
 
 int acpi_isa_irq_to_gsi(unsigned isa_irq, u32 *gsi)
 {
 	if (isa_irq < nr_legacy_irqs() &&
-	    isa_irq_to_gsi[isa_irq] != ACPI_INVALID_GSI) {
+	    isa_irq_to_gsi[isa_irq] != INVALID_ACPI_IRQ) {
 		*gsi = isa_irq_to_gsi[isa_irq];
 		return 0;
 	}
@@ -676,8 +676,7 @@ static int acpi_register_gsi_ioapic(struct device *dev, u32 gsi,
 	mutex_lock(&acpi_ioapic_lock);
 	irq = mp_map_gsi_to_irq(gsi, IOAPIC_MAP_ALLOC, &info);
 	/* Don't set up the ACPI SCI because it's already set up */
-	if (irq >= 0 && enable_update_mptable &&
-	    acpi_gbl_FADT.sci_interrupt != gsi)
+	if (irq >= 0 && enable_update_mptable && gsi != acpi_gbl_FADT.sci_interrupt)
 		mp_config_acpi_gsi(dev, gsi, trigger, polarity);
 	mutex_unlock(&acpi_ioapic_lock);
 #endif
@@ -1211,8 +1210,9 @@ static int __init acpi_parse_madt_ioapic_entries(void)
 	/*
 	 * If BIOS did not supply an INT_SRC_OVR for the SCI
 	 * pretend we got one so we can set the SCI flags.
+	 * But ignore setting up SCI on hardware reduced platforms.
 	 */
-	if (!acpi_sci_override_gsi)
+	if (acpi_sci_override_gsi == INVALID_ACPI_IRQ && !acpi_gbl_reduced_hardware)
 		acpi_sci_ioapic_setup(acpi_gbl_FADT.sci_interrupt, 0, 0,
 				      acpi_gbl_FADT.sci_interrupt);
 
@@ -1626,6 +1626,8 @@ int __init acpi_boot_init(void)
 	if (!acpi_noirq)
 		x86_init.pci.init = pci_acpi_init;
 
+	/* Do not enable ACPI SPCR console by default */
+	acpi_parse_spcr(earlycon_acpi_spcr_enable, false);
 	return 0;
 }
 
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 7188aea91549..f1915b744052 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -138,6 +138,8 @@ static int __init acpi_sleep_setup(char *str)
 			acpi_nvs_nosave_s3();
 		if (strncmp(str, "old_ordering", 12) == 0)
 			acpi_old_suspend_ordering();
+		if (strncmp(str, "nobl", 4) == 0)
+			acpi_sleep_no_blacklist();
 		str = strchr(str, ',');
 		if (str != NULL)
 			str += strspn(str, ", \t");
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 4817d743c263..a481763a3776 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -46,17 +46,6 @@ static int __init setup_noreplace_smp(char *str)
 }
 __setup("noreplace-smp", setup_noreplace_smp);
 
-#ifdef CONFIG_PARAVIRT
-static int __initdata_or_module noreplace_paravirt = 0;
-
-static int __init setup_noreplace_paravirt(char *str)
-{
-	noreplace_paravirt = 1;
-	return 1;
-}
-__setup("noreplace-paravirt", setup_noreplace_paravirt);
-#endif
-
 #define DPRINTK(fmt, args...)						\
 do {									\
 	if (debug_alternative)						\
@@ -298,7 +287,7 @@ recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insnbuf)
 	tgt_rip  = next_rip + o_dspl;
 	n_dspl = tgt_rip - orig_insn;
 
-	DPRINTK("target RIP: %p, new_displ: 0x%x", tgt_rip, n_dspl);
+	DPRINTK("target RIP: %px, new_displ: 0x%x", tgt_rip, n_dspl);
 
 	if (tgt_rip - orig_insn >= 0) {
 		if (n_dspl - 2 <= 127)
@@ -355,7 +344,7 @@ static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *ins
 	add_nops(instr + (a->instrlen - a->padlen), a->padlen);
 	local_irq_restore(flags);
 
-	DUMP_BYTES(instr, a->instrlen, "%p: [%d:%d) optimized NOPs: ",
+	DUMP_BYTES(instr, a->instrlen, "%px: [%d:%d) optimized NOPs: ",
 		   instr, a->instrlen - a->padlen, a->padlen);
 }
 
@@ -376,7 +365,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
 	u8 *instr, *replacement;
 	u8 insnbuf[MAX_PATCH_LEN];
 
-	DPRINTK("alt table %p -> %p", start, end);
+	DPRINTK("alt table %px, -> %px", start, end);
 	/*
 	 * The scan order should be from start to end. A later scanned
 	 * alternative code can overwrite previously scanned alternative code.
@@ -400,14 +389,14 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
 			continue;
 		}
 
-		DPRINTK("feat: %d*32+%d, old: (%p, len: %d), repl: (%p, len: %d), pad: %d",
+		DPRINTK("feat: %d*32+%d, old: (%px len: %d), repl: (%px, len: %d), pad: %d",
 			a->cpuid >> 5,
 			a->cpuid & 0x1f,
 			instr, a->instrlen,
 			replacement, a->replacementlen, a->padlen);
 
-		DUMP_BYTES(instr, a->instrlen, "%p: old_insn: ", instr);
-		DUMP_BYTES(replacement, a->replacementlen, "%p: rpl_insn: ", replacement);
+		DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
+		DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
 
 		memcpy(insnbuf, replacement, a->replacementlen);
 		insnbuf_sz = a->replacementlen;
@@ -433,7 +422,7 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
 				 a->instrlen - a->replacementlen);
 			insnbuf_sz += a->instrlen - a->replacementlen;
 		}
-		DUMP_BYTES(insnbuf, insnbuf_sz, "%p: final_insn: ", instr);
+		DUMP_BYTES(insnbuf, insnbuf_sz, "%px: final_insn: ", instr);
 
 		text_poke_early(instr, insnbuf, insnbuf_sz);
 	}
@@ -599,9 +588,6 @@ void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
 	struct paravirt_patch_site *p;
 	char insnbuf[MAX_PATCH_LEN];
 
-	if (noreplace_paravirt)
-		return;
-
 	for (p = start; p < end; p++) {
 		unsigned int used;
 
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index cc0e8bc0ea3f..ecd486cb06ab 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -31,6 +31,7 @@
 #include <linux/io.h>
 #include <linux/gfp.h>
 #include <linux/atomic.h>
+#include <linux/dma-direct.h>
 #include <asm/mtrr.h>
 #include <asm/pgtable.h>
 #include <asm/proto.h>
diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c
index 6db28f17ff28..c88e0b127810 100644
--- a/arch/x86/kernel/amd_nb.c
+++ b/arch/x86/kernel/amd_nb.c
@@ -235,7 +235,7 @@ int amd_cache_northbridges(void)
 	if (boot_cpu_data.x86 == 0x10 &&
 	    boot_cpu_data.x86_model >= 0x8 &&
 	    (boot_cpu_data.x86_model > 0x9 ||
-	     boot_cpu_data.x86_mask >= 0x1))
+	     boot_cpu_data.x86_stepping >= 0x1))
 		amd_northbridges.flags |= AMD_NB_L3_INDEX_DISABLE;
 
 	if (boot_cpu_data.x86 == 0x15)
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index f5d92bc3b884..2c4d5ece7456 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -30,6 +30,7 @@
 #include <asm/dma.h>
 #include <asm/amd_nb.h>
 #include <asm/x86_init.h>
+#include <linux/crash_dump.h>
 
 /*
  * Using 512M as goal, in case kexec will load kernel_big
@@ -56,6 +57,33 @@ int fallback_aper_force __initdata;
 
 int fix_aperture __initdata = 1;
 
+#ifdef CONFIG_PROC_VMCORE
+/*
+ * If the first kernel maps the aperture over e820 RAM, the kdump kernel will
+ * use the same range because it will remain configured in the northbridge.
+ * Trying to dump this area via /proc/vmcore may crash the machine, so exclude
+ * it from vmcore.
+ */
+static unsigned long aperture_pfn_start, aperture_page_count;
+
+static int gart_oldmem_pfn_is_ram(unsigned long pfn)
+{
+	return likely((pfn < aperture_pfn_start) ||
+		      (pfn >= aperture_pfn_start + aperture_page_count));
+}
+
+static void exclude_from_vmcore(u64 aper_base, u32 aper_order)
+{
+	aperture_pfn_start = aper_base >> PAGE_SHIFT;
+	aperture_page_count = (32 * 1024 * 1024) << aper_order >> PAGE_SHIFT;
+	WARN_ON(register_oldmem_pfn_is_ram(&gart_oldmem_pfn_is_ram));
+}
+#else
+static void exclude_from_vmcore(u64 aper_base, u32 aper_order)
+{
+}
+#endif
+
 /* This code runs before the PCI subsystem is initialized, so just
    access the northbridge directly. */
 
@@ -435,8 +463,16 @@ int __init gart_iommu_hole_init(void)
 
 out:
 	if (!fix && !fallback_aper_force) {
-		if (last_aper_base)
+		if (last_aper_base) {
+			/*
+			 * If this is the kdump kernel, the first kernel
+			 * may have allocated the range over its e820 RAM
+			 * and fixed up the northbridge
+			 */
+			exclude_from_vmcore(last_aper_base, last_aper_order);
+
 			return 1;
+		}
 		return 0;
 	}
 
@@ -473,6 +509,14 @@ int __init gart_iommu_hole_init(void)
 		return 0;
 	}
 
+	/*
+	 * If this is the kdump kernel _and_ the first kernel did not
+	 * configure the aperture in the northbridge, this range may
+	 * overlap with the first kernel's memory. We can't access the
+	 * range through vmcore even though it should be part of the dump.
+	 */
+	exclude_from_vmcore(aper_alloc, aper_order);
+
 	/* Fix up the north bridges */
 	for (i = 0; i < amd_nb_bus_dev_ranges[i].dev_limit; i++) {
 		int bus, dev_base, dev_limit;
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 25ddf02598d2..b203af0855b5 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -546,7 +546,7 @@ static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
 
 static u32 hsx_deadline_rev(void)
 {
-	switch (boot_cpu_data.x86_mask) {
+	switch (boot_cpu_data.x86_stepping) {
 	case 0x02: return 0x3a; /* EP */
 	case 0x04: return 0x0f; /* EX */
 	}
@@ -556,7 +556,7 @@ static u32 hsx_deadline_rev(void)
 
 static u32 bdx_deadline_rev(void)
 {
-	switch (boot_cpu_data.x86_mask) {
+	switch (boot_cpu_data.x86_stepping) {
 	case 0x02: return 0x00000011;
 	case 0x03: return 0x0700000e;
 	case 0x04: return 0x0f00000c;
@@ -568,7 +568,7 @@ static u32 bdx_deadline_rev(void)
 
 static u32 skx_deadline_rev(void)
 {
-	switch (boot_cpu_data.x86_mask) {
+	switch (boot_cpu_data.x86_stepping) {
 	case 0x03: return 0x01000136;
 	case 0x04: return 0x02000014;
 	}
diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c
index 25a87028cb3f..e84c9eb4e5b4 100644
--- a/arch/x86/kernel/apic/apic_flat_64.c
+++ b/arch/x86/kernel/apic/apic_flat_64.c
@@ -19,6 +19,7 @@
 #include <asm/smp.h>
 #include <asm/apic.h>
 #include <asm/ipi.h>
+#include <asm/jailhouse_para.h>
 
 #include <linux/acpi.h>
 
@@ -84,12 +85,8 @@ flat_send_IPI_mask_allbutself(const struct cpumask *cpumask, int vector)
 static void flat_send_IPI_allbutself(int vector)
 {
 	int cpu = smp_processor_id();
-#ifdef	CONFIG_HOTPLUG_CPU
-	int hotplug = 1;
-#else
-	int hotplug = 0;
-#endif
-	if (hotplug || vector == NMI_VECTOR) {
+
+	if (IS_ENABLED(CONFIG_HOTPLUG_CPU) || vector == NMI_VECTOR) {
 		if (!cpumask_equal(cpu_online_mask, cpumask_of(cpu))) {
 			unsigned long mask = cpumask_bits(cpu_online_mask)[0];
 
@@ -218,6 +215,15 @@ static int physflat_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 	return 0;
 }
 
+static void physflat_init_apic_ldr(void)
+{
+	/*
+	 * LDR and DFR are not involved in physflat mode, rather:
+	 * "In physical destination mode, the destination processor is
+	 * specified by its local APIC ID [...]." (Intel SDM, 10.6.2.1)
+	 */
+}
+
 static void physflat_send_IPI_allbutself(int vector)
 {
 	default_send_IPI_mask_allbutself_phys(cpu_online_mask, vector);
@@ -230,7 +236,8 @@ static void physflat_send_IPI_all(int vector)
 
 static int physflat_probe(void)
 {
-	if (apic == &apic_physflat || num_possible_cpus() > 8)
+	if (apic == &apic_physflat || num_possible_cpus() > 8 ||
+	    jailhouse_paravirt())
 		return 1;
 
 	return 0;
@@ -251,8 +258,7 @@ static struct apic apic_physflat __ro_after_init = {
 	.dest_logical			= 0,
 	.check_apicid_used		= NULL,
 
-	/* not needed, but shouldn't hurt: */
-	.init_apic_ldr			= flat_init_apic_ldr,
+	.init_apic_ldr			= physflat_init_apic_ldr,
 
 	.ioapic_phys_id_map		= NULL,
 	.setup_apic_routing		= NULL,
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 8a7963421460..7c5538769f7e 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -800,18 +800,18 @@ static int irq_polarity(int idx)
 	/*
 	 * Determine IRQ line polarity (high active or low active):
 	 */
-	switch (mp_irqs[idx].irqflag & 0x03) {
-	case 0:
+	switch (mp_irqs[idx].irqflag & MP_IRQPOL_MASK) {
+	case MP_IRQPOL_DEFAULT:
 		/* conforms to spec, ie. bus-type dependent polarity */
 		if (test_bit(bus, mp_bus_not_pci))
 			return default_ISA_polarity(idx);
 		else
 			return default_PCI_polarity(idx);
-	case 1:
+	case MP_IRQPOL_ACTIVE_HIGH:
 		return IOAPIC_POL_HIGH;
-	case 2:
+	case MP_IRQPOL_RESERVED:
 		pr_warn("IOAPIC: Invalid polarity: 2, defaulting to low\n");
-	case 3:
+	case MP_IRQPOL_ACTIVE_LOW:
 	default: /* Pointless default required due to do gcc stupidity */
 		return IOAPIC_POL_LOW;
 	}
@@ -845,8 +845,8 @@ static int irq_trigger(int idx)
 	/*
 	 * Determine IRQ trigger mode (edge or level sensitive):
 	 */
-	switch ((mp_irqs[idx].irqflag >> 2) & 0x03) {
-	case 0:
+	switch (mp_irqs[idx].irqflag & MP_IRQTRIG_MASK) {
+	case MP_IRQTRIG_DEFAULT:
 		/* conforms to spec, ie. bus-type dependent trigger mode */
 		if (test_bit(bus, mp_bus_not_pci))
 			trigger = default_ISA_trigger(idx);
@@ -854,11 +854,11 @@ static int irq_trigger(int idx)
 			trigger = default_PCI_trigger(idx);
 		/* Take EISA into account */
 		return eisa_irq_trigger(idx, bus, trigger);
-	case 1:
+	case MP_IRQTRIG_EDGE:
 		return IOAPIC_EDGE;
-	case 2:
+	case MP_IRQTRIG_RESERVED:
 		pr_warn("IOAPIC: Invalid trigger mode 2 defaulting to level\n");
-	case 3:
+	case MP_IRQTRIG_LEVEL:
 	default: /* Pointless default required due to do gcc stupidity */
 		return IOAPIC_LEVEL;
 	}
@@ -1603,7 +1603,7 @@ static void __init delay_with_tsc(void)
 	do {
 		rep_nop();
 		now = rdtsc();
-	} while ((now - start) < 40000000000UL / HZ &&
+	} while ((now - start) < 40000000000ULL / HZ &&
 		time_before_eq(jiffies, end));
 }
 
diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c
index 3cc471beb50b..bb6f7a2148d7 100644
--- a/arch/x86/kernel/apic/vector.c
+++ b/arch/x86/kernel/apic/vector.c
@@ -134,21 +134,40 @@ static void apic_update_vector(struct irq_data *irqd, unsigned int newvec,
 {
 	struct apic_chip_data *apicd = apic_chip_data(irqd);
 	struct irq_desc *desc = irq_data_to_desc(irqd);
+	bool managed = irqd_affinity_is_managed(irqd);
 
 	lockdep_assert_held(&vector_lock);
 
 	trace_vector_update(irqd->irq, newvec, newcpu, apicd->vector,
 			    apicd->cpu);
 
-	/* Setup the vector move, if required  */
-	if (apicd->vector && cpu_online(apicd->cpu)) {
+	/*
+	 * If there is no vector associated or if the associated vector is
+	 * the shutdown vector, which is associated to make PCI/MSI
+	 * shutdown mode work, then there is nothing to release. Clear out
+	 * prev_vector for this and the offlined target case.
+	 */
+	apicd->prev_vector = 0;
+	if (!apicd->vector || apicd->vector == MANAGED_IRQ_SHUTDOWN_VECTOR)
+		goto setnew;
+	/*
+	 * If the target CPU of the previous vector is online, then mark
+	 * the vector as move in progress and store it for cleanup when the
+	 * first interrupt on the new vector arrives. If the target CPU is
+	 * offline then the regular release mechanism via the cleanup
+	 * vector is not possible and the vector can be immediately freed
+	 * in the underlying matrix allocator.
+	 */
+	if (cpu_online(apicd->cpu)) {
 		apicd->move_in_progress = true;
 		apicd->prev_vector = apicd->vector;
 		apicd->prev_cpu = apicd->cpu;
 	} else {
-		apicd->prev_vector = 0;
+		irq_matrix_free(vector_matrix, apicd->cpu, apicd->vector,
+				managed);
 	}
 
+setnew:
 	apicd->vector = newvec;
 	apicd->cpu = newcpu;
 	BUG_ON(!IS_ERR_OR_NULL(per_cpu(vector_irq, newcpu)[newvec]));
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index e1b8e8bf6b3c..f11910b44638 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -137,6 +137,8 @@ static int __init early_get_pnodeid(void)
 	case UV3_HUB_PART_NUMBER_X:
 		uv_min_hub_revision_id += UV3_HUB_REVISION_BASE;
 		break;
+
+	/* Update: UV4A has only a modified revision to indicate HUB fixes */
 	case UV4_HUB_PART_NUMBER:
 		uv_min_hub_revision_id += UV4_HUB_REVISION_BASE - 1;
 		uv_cpuid.gnode_shift = 2; /* min partition is 4 sockets */
@@ -316,6 +318,7 @@ static int __init uv_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 	} else if (!strcmp(oem_table_id, "UVH")) {
 		/* Only UV1 systems: */
 		uv_system_type = UV_NON_UNIQUE_APIC;
+		x86_platform.legacy.warm_reset = 0;
 		__this_cpu_write(x2apic_extra_bits, pnodeid << uvh_apicid.s.pnode_shift);
 		uv_set_apicid_hibit();
 		uv_apic = 1;
@@ -767,6 +770,7 @@ static __init void map_gru_high(int max_pnode)
 		return;
 	}
 
+	/* Only UV3 has distributed GRU mode */
 	if (is_uv3_hub() && gru.s3.mode) {
 		map_gru_distributed(gru.v);
 		return;
@@ -790,63 +794,61 @@ static __init void map_mmr_high(int max_pnode)
 		pr_info("UV: MMR disabled\n");
 }
 
-/*
- * This commonality works because both 0 & 1 versions of the MMIOH OVERLAY
- * and REDIRECT MMR regs are exactly the same on UV3.
- */
-struct mmioh_config {
-	unsigned long overlay;
-	unsigned long redirect;
-	char *id;
-};
-
-static __initdata struct mmioh_config mmiohs[] = {
-	{
-		UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR,
-		UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR,
-		"MMIOH0"
-	},
-	{
-		UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR,
-		UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR,
-		"MMIOH1"
-	},
-};
-
-/* UV3 & UV4 have identical MMIOH overlay configs */
-static __init void map_mmioh_high_uv3(int index, int min_pnode, int max_pnode)
+/* UV3/4 have identical MMIOH overlay configs, UV4A is slightly different */
+static __init void map_mmioh_high_uv34(int index, int min_pnode, int max_pnode)
 {
-	union uv3h_rh_gam_mmioh_overlay_config0_mmr_u overlay;
+	unsigned long overlay;
 	unsigned long mmr;
 	unsigned long base;
+	unsigned long nasid_mask;
+	unsigned long m_overlay;
 	int i, n, shift, m_io, max_io;
 	int nasid, lnasid, fi, li;
 	char *id;
 
-	id = mmiohs[index].id;
-	overlay.v = uv_read_local_mmr(mmiohs[index].overlay);
-
-	pr_info("UV: %s overlay 0x%lx base:0x%x m_io:%d\n", id, overlay.v, overlay.s3.base, overlay.s3.m_io);
-	if (!overlay.s3.enable) {
+	if (index == 0) {
+		id = "MMIOH0";
+		m_overlay = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR;
+		overlay = uv_read_local_mmr(m_overlay);
+		base = overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_MASK;
+		mmr = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR;
+		m_io = (overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_MASK)
+			>> UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT;
+		shift = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_M_IO_SHFT;
+		n = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH;
+		nasid_mask = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_NASID_MASK;
+	} else {
+		id = "MMIOH1";
+		m_overlay = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR;
+		overlay = uv_read_local_mmr(m_overlay);
+		base = overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_BASE_MASK;
+		mmr = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR;
+		m_io = (overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_MASK)
+			>> UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT;
+		shift = UVH_RH_GAM_MMIOH_OVERLAY_CONFIG1_MMR_M_IO_SHFT;
+		n = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_DEPTH;
+		nasid_mask = UVH_RH_GAM_MMIOH_REDIRECT_CONFIG1_MMR_NASID_MASK;
+	}
+	pr_info("UV: %s overlay 0x%lx base:0x%lx m_io:%d\n", id, overlay, base, m_io);
+	if (!(overlay & UVH_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_ENABLE_MASK)) {
 		pr_info("UV: %s disabled\n", id);
 		return;
 	}
 
-	shift = UV3H_RH_GAM_MMIOH_OVERLAY_CONFIG0_MMR_BASE_SHFT;
-	base = (unsigned long)overlay.s3.base;
-	m_io = overlay.s3.m_io;
-	mmr = mmiohs[index].redirect;
-	n = UV3H_RH_GAM_MMIOH_REDIRECT_CONFIG0_MMR_DEPTH;
 	/* Convert to NASID: */
 	min_pnode *= 2;
 	max_pnode *= 2;
 	max_io = lnasid = fi = li = -1;
 
 	for (i = 0; i < n; i++) {
-		union uv3h_rh_gam_mmioh_redirect_config0_mmr_u redirect;
+		unsigned long m_redirect = mmr + i * 8;
+		unsigned long redirect = uv_read_local_mmr(m_redirect);
+
+		nasid = redirect & nasid_mask;
+		if (i == 0)
+			pr_info("UV: %s redirect base 0x%lx(@0x%lx) 0x%04x\n",
+				id, redirect, m_redirect, nasid);
 
-		redirect.v = uv_read_local_mmr(mmr + i * 8);
-		nasid = redirect.s3.nasid;
 		/* Invalid NASID: */
 		if (nasid < min_pnode || max_pnode < nasid)
 			nasid = -1;
@@ -894,8 +896,8 @@ static __init void map_mmioh_high(int min_pnode, int max_pnode)
 
 	if (is_uv3_hub() || is_uv4_hub()) {
 		/* Map both MMIOH regions: */
-		map_mmioh_high_uv3(0, min_pnode, max_pnode);
-		map_mmioh_high_uv3(1, min_pnode, max_pnode);
+		map_mmioh_high_uv34(0, min_pnode, max_pnode);
+		map_mmioh_high_uv34(1, min_pnode, max_pnode);
 		return;
 	}
 
@@ -1174,16 +1176,25 @@ static void __init decode_gam_rng_tbl(unsigned long ptr)
 
 	uv_gre_table = gre;
 	for (; gre->type != UV_GAM_RANGE_TYPE_UNUSED; gre++) {
+		unsigned long size = ((unsigned long)(gre->limit - lgre)
+					<< UV_GAM_RANGE_SHFT);
+		int order = 0;
+		char suffix[] = " KMGTPE";
+
+		while (size > 9999 && order < sizeof(suffix)) {
+			size /= 1024;
+			order++;
+		}
+
 		if (!index) {
 			pr_info("UV: GAM Range Table...\n");
 			pr_info("UV:  # %20s %14s %5s %4s %5s %3s %2s\n", "Range", "", "Size", "Type", "NASID", "SID", "PN");
 		}
-		pr_info("UV: %2d: 0x%014lx-0x%014lx %5luG %3d   %04x  %02x %02x\n",
+		pr_info("UV: %2d: 0x%014lx-0x%014lx %5lu%c %3d   %04x  %02x %02x\n",
 			index++,
 			(unsigned long)lgre << UV_GAM_RANGE_SHFT,
 			(unsigned long)gre->limit << UV_GAM_RANGE_SHFT,
-			((unsigned long)(gre->limit - lgre)) >>
-				(30 - UV_GAM_RANGE_SHFT), /* 64M -> 1G */
+			size, suffix[order],
 			gre->type, gre->nasid, gre->sockid, gre->pnode);
 
 		lgre = gre->limit;
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c
index e4b0d92b3ae0..dfcbe6924eaf 100644
--- a/arch/x86/kernel/apm_32.c
+++ b/arch/x86/kernel/apm_32.c
@@ -1506,7 +1506,7 @@ static ssize_t do_read(struct file *fp, char __user *buf, size_t count, loff_t *
 	return 0;
 }
 
-static unsigned int do_poll(struct file *fp, poll_table *wait)
+static __poll_t do_poll(struct file *fp, poll_table *wait)
 {
 	struct apm_user *as;
 
@@ -1515,7 +1515,7 @@ static unsigned int do_poll(struct file *fp, poll_table *wait)
 		return 0;
 	poll_wait(fp, &apm_waitqueue, wait);
 	if (!queue_empty(as))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 
@@ -2389,6 +2389,7 @@ static int __init apm_init(void)
 	if (HZ != 100)
 		idle_period = (idle_period * HZ) / 100;
 	if (idle_threshold < 100) {
+		cpuidle_poll_state_init(&apm_idle_driver);
 		if (!cpuidle_register_driver(&apm_idle_driver))
 			if (cpuidle_register_device(&apm_cpuidle_device))
 				cpuidle_unregister_driver(&apm_idle_driver);
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index fa1261eefa16..f91ba53e06c8 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -18,7 +18,7 @@ void foo(void)
 	OFFSET(CPUINFO_x86, cpuinfo_x86, x86);
 	OFFSET(CPUINFO_x86_vendor, cpuinfo_x86, x86_vendor);
 	OFFSET(CPUINFO_x86_model, cpuinfo_x86, x86_model);
-	OFFSET(CPUINFO_x86_mask, cpuinfo_x86, x86_mask);
+	OFFSET(CPUINFO_x86_stepping, cpuinfo_x86, x86_stepping);
 	OFFSET(CPUINFO_cpuid_level, cpuinfo_x86, cpuid_level);
 	OFFSET(CPUINFO_x86_capability, cpuinfo_x86, x86_capability);
 	OFFSET(CPUINFO_x86_vendor_id, cpuinfo_x86, x86_vendor_id);
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index ea831c858195..f0e6456ca7d3 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -119,7 +119,7 @@ static void init_amd_k6(struct cpuinfo_x86 *c)
 		return;
 	}
 
-	if (c->x86_model == 6 && c->x86_mask == 1) {
+	if (c->x86_model == 6 && c->x86_stepping == 1) {
 		const int K6_BUG_LOOP = 1000000;
 		int n;
 		void (*f_vide)(void);
@@ -149,7 +149,7 @@ static void init_amd_k6(struct cpuinfo_x86 *c)
 
 	/* K6 with old style WHCR */
 	if (c->x86_model < 8 ||
-	   (c->x86_model == 8 && c->x86_mask < 8)) {
+	   (c->x86_model == 8 && c->x86_stepping < 8)) {
 		/* We can only write allocate on the low 508Mb */
 		if (mbytes > 508)
 			mbytes = 508;
@@ -168,7 +168,7 @@ static void init_amd_k6(struct cpuinfo_x86 *c)
 		return;
 	}
 
-	if ((c->x86_model == 8 && c->x86_mask > 7) ||
+	if ((c->x86_model == 8 && c->x86_stepping > 7) ||
 	     c->x86_model == 9 || c->x86_model == 13) {
 		/* The more serious chips .. */
 
@@ -221,7 +221,7 @@ static void init_amd_k7(struct cpuinfo_x86 *c)
 	 * are more robust with CLK_CTL set to 200xxxxx instead of 600xxxxx
 	 * As per AMD technical note 27212 0.2
 	 */
-	if ((c->x86_model == 8 && c->x86_mask >= 1) || (c->x86_model > 8)) {
+	if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) {
 		rdmsr(MSR_K7_CLK_CTL, l, h);
 		if ((l & 0xfff00000) != 0x20000000) {
 			pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
@@ -241,12 +241,12 @@ static void init_amd_k7(struct cpuinfo_x86 *c)
 	 * but they are not certified as MP capable.
 	 */
 	/* Athlon 660/661 is valid. */
-	if ((c->x86_model == 6) && ((c->x86_mask == 0) ||
-	    (c->x86_mask == 1)))
+	if ((c->x86_model == 6) && ((c->x86_stepping == 0) ||
+	    (c->x86_stepping == 1)))
 		return;
 
 	/* Duron 670 is valid */
-	if ((c->x86_model == 7) && (c->x86_mask == 0))
+	if ((c->x86_model == 7) && (c->x86_stepping == 0))
 		return;
 
 	/*
@@ -256,8 +256,8 @@ static void init_amd_k7(struct cpuinfo_x86 *c)
 	 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for
 	 * more.
 	 */
-	if (((c->x86_model == 6) && (c->x86_mask >= 2)) ||
-	    ((c->x86_model == 7) && (c->x86_mask >= 1)) ||
+	if (((c->x86_model == 6) && (c->x86_stepping >= 2)) ||
+	    ((c->x86_model == 7) && (c->x86_stepping >= 1)) ||
 	     (c->x86_model > 7))
 		if (cpu_has(c, X86_FEATURE_MP))
 			return;
@@ -556,6 +556,51 @@ static void bsp_init_amd(struct cpuinfo_x86 *c)
 	}
 }
 
+static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
+{
+	u64 msr;
+
+	/*
+	 * BIOS support is required for SME and SEV.
+	 *   For SME: If BIOS has enabled SME then adjust x86_phys_bits by
+	 *	      the SME physical address space reduction value.
+	 *	      If BIOS has not enabled SME then don't advertise the
+	 *	      SME feature (set in scattered.c).
+	 *   For SEV: If BIOS has not enabled SEV then don't advertise the
+	 *            SEV feature (set in scattered.c).
+	 *
+	 *   In all cases, since support for SME and SEV requires long mode,
+	 *   don't advertise the feature under CONFIG_X86_32.
+	 */
+	if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) {
+		/* Check if memory encryption is enabled */
+		rdmsrl(MSR_K8_SYSCFG, msr);
+		if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT))
+			goto clear_all;
+
+		/*
+		 * Always adjust physical address bits. Even though this
+		 * will be a value above 32-bits this is still done for
+		 * CONFIG_X86_32 so that accurate values are reported.
+		 */
+		c->x86_phys_bits -= (cpuid_ebx(0x8000001f) >> 6) & 0x3f;
+
+		if (IS_ENABLED(CONFIG_X86_32))
+			goto clear_all;
+
+		rdmsrl(MSR_K7_HWCR, msr);
+		if (!(msr & MSR_K7_HWCR_SMMLOCK))
+			goto clear_sev;
+
+		return;
+
+clear_all:
+		clear_cpu_cap(c, X86_FEATURE_SME);
+clear_sev:
+		clear_cpu_cap(c, X86_FEATURE_SEV);
+	}
+}
+
 static void early_init_amd(struct cpuinfo_x86 *c)
 {
 	u32 dummy;
@@ -583,7 +628,7 @@ static void early_init_amd(struct cpuinfo_x86 *c)
 	/*  Set MTRR capability flag if appropriate */
 	if (c->x86 == 5)
 		if (c->x86_model == 13 || c->x86_model == 9 ||
-		    (c->x86_model == 8 && c->x86_mask >= 8))
+		    (c->x86_model == 8 && c->x86_stepping >= 8))
 			set_cpu_cap(c, X86_FEATURE_K6_MTRR);
 #endif
 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
@@ -627,26 +672,7 @@ static void early_init_amd(struct cpuinfo_x86 *c)
 	if (cpu_has_amd_erratum(c, amd_erratum_400))
 		set_cpu_bug(c, X86_BUG_AMD_E400);
 
-	/*
-	 * BIOS support is required for SME. If BIOS has enabled SME then
-	 * adjust x86_phys_bits by the SME physical address space reduction
-	 * value. If BIOS has not enabled SME then don't advertise the
-	 * feature (set in scattered.c). Also, since the SME support requires
-	 * long mode, don't advertise the feature under CONFIG_X86_32.
-	 */
-	if (cpu_has(c, X86_FEATURE_SME)) {
-		u64 msr;
-
-		/* Check if SME is enabled */
-		rdmsrl(MSR_K8_SYSCFG, msr);
-		if (msr & MSR_K8_SYSCFG_MEM_ENCRYPT) {
-			c->x86_phys_bits -= (cpuid_ebx(0x8000001f) >> 6) & 0x3f;
-			if (IS_ENABLED(CONFIG_X86_32))
-				clear_cpu_cap(c, X86_FEATURE_SME);
-		} else {
-			clear_cpu_cap(c, X86_FEATURE_SME);
-		}
-	}
+	early_detect_mem_encrypt(c);
 }
 
 static void init_amd_k8(struct cpuinfo_x86 *c)
@@ -769,7 +795,7 @@ static void init_amd_zn(struct cpuinfo_x86 *c)
 	 * Fix erratum 1076: CPB feature bit not being set in CPUID. It affects
 	 * all up to and including B1.
 	 */
-	if (c->x86_model <= 1 && c->x86_mask <= 1)
+	if (c->x86_model <= 1 && c->x86_stepping <= 1)
 		set_cpu_cap(c, X86_FEATURE_CPB);
 }
 
@@ -880,11 +906,11 @@ static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
 	/* AMD errata T13 (order #21922) */
 	if ((c->x86 == 6)) {
 		/* Duron Rev A0 */
-		if (c->x86_model == 3 && c->x86_mask == 0)
+		if (c->x86_model == 3 && c->x86_stepping == 0)
 			size = 64;
 		/* Tbird rev A1/A2 */
 		if (c->x86_model == 4 &&
-			(c->x86_mask == 0 || c->x86_mask == 1))
+			(c->x86_stepping == 0 || c->x86_stepping == 1))
 			size = 256;
 	}
 	return size;
@@ -1021,7 +1047,7 @@ static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
 	}
 
 	/* OSVW unavailable or ID unknown, match family-model-stepping range */
-	ms = (cpu->x86_model << 4) | cpu->x86_mask;
+	ms = (cpu->x86_model << 4) | cpu->x86_stepping;
 	while ((range = *erratum++))
 		if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
 		    (ms >= AMD_MODEL_RANGE_START(range)) &&
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 390b3dc3d438..bfca937bdcc3 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -11,6 +11,7 @@
 #include <linux/init.h>
 #include <linux/utsname.h>
 #include <linux/cpu.h>
+#include <linux/module.h>
 
 #include <asm/nospec-branch.h>
 #include <asm/cmdline.h>
@@ -90,20 +91,41 @@ static const char *spectre_v2_strings[] = {
 };
 
 #undef pr_fmt
-#define pr_fmt(fmt)     "Spectre V2 mitigation: " fmt
+#define pr_fmt(fmt)     "Spectre V2 : " fmt
 
 static enum spectre_v2_mitigation spectre_v2_enabled = SPECTRE_V2_NONE;
 
+#ifdef RETPOLINE
+static bool spectre_v2_bad_module;
+
+bool retpoline_module_ok(bool has_retpoline)
+{
+	if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
+		return true;
+
+	pr_err("System may be vulnerable to spectre v2\n");
+	spectre_v2_bad_module = true;
+	return false;
+}
+
+static inline const char *spectre_v2_module_string(void)
+{
+	return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
+}
+#else
+static inline const char *spectre_v2_module_string(void) { return ""; }
+#endif
+
 static void __init spec2_print_if_insecure(const char *reason)
 {
 	if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
-		pr_info("%s\n", reason);
+		pr_info("%s selected on command line.\n", reason);
 }
 
 static void __init spec2_print_if_secure(const char *reason)
 {
 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
-		pr_info("%s\n", reason);
+		pr_info("%s selected on command line.\n", reason);
 }
 
 static inline bool retp_compiler(void)
@@ -118,42 +140,65 @@ static inline bool match_option(const char *arg, int arglen, const char *opt)
 	return len == arglen && !strncmp(arg, opt, len);
 }
 
+static const struct {
+	const char *option;
+	enum spectre_v2_mitigation_cmd cmd;
+	bool secure;
+} mitigation_options[] = {
+	{ "off",               SPECTRE_V2_CMD_NONE,              false },
+	{ "on",                SPECTRE_V2_CMD_FORCE,             true },
+	{ "retpoline",         SPECTRE_V2_CMD_RETPOLINE,         false },
+	{ "retpoline,amd",     SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
+	{ "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
+	{ "auto",              SPECTRE_V2_CMD_AUTO,              false },
+};
+
 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 {
 	char arg[20];
-	int ret;
-
-	ret = cmdline_find_option(boot_command_line, "spectre_v2", arg,
-				  sizeof(arg));
-	if (ret > 0)  {
-		if (match_option(arg, ret, "off")) {
-			goto disable;
-		} else if (match_option(arg, ret, "on")) {
-			spec2_print_if_secure("force enabled on command line.");
-			return SPECTRE_V2_CMD_FORCE;
-		} else if (match_option(arg, ret, "retpoline")) {
-			spec2_print_if_insecure("retpoline selected on command line.");
-			return SPECTRE_V2_CMD_RETPOLINE;
-		} else if (match_option(arg, ret, "retpoline,amd")) {
-			if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
-				pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
-				return SPECTRE_V2_CMD_AUTO;
-			}
-			spec2_print_if_insecure("AMD retpoline selected on command line.");
-			return SPECTRE_V2_CMD_RETPOLINE_AMD;
-		} else if (match_option(arg, ret, "retpoline,generic")) {
-			spec2_print_if_insecure("generic retpoline selected on command line.");
-			return SPECTRE_V2_CMD_RETPOLINE_GENERIC;
-		} else if (match_option(arg, ret, "auto")) {
+	int ret, i;
+	enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
+
+	if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
+		return SPECTRE_V2_CMD_NONE;
+	else {
+		ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
+		if (ret < 0)
+			return SPECTRE_V2_CMD_AUTO;
+
+		for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
+			if (!match_option(arg, ret, mitigation_options[i].option))
+				continue;
+			cmd = mitigation_options[i].cmd;
+			break;
+		}
+
+		if (i >= ARRAY_SIZE(mitigation_options)) {
+			pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 			return SPECTRE_V2_CMD_AUTO;
 		}
 	}
 
-	if (!cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
+	if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
+	     cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
+	     cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
+	    !IS_ENABLED(CONFIG_RETPOLINE)) {
+		pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 		return SPECTRE_V2_CMD_AUTO;
-disable:
-	spec2_print_if_insecure("disabled on command line.");
-	return SPECTRE_V2_CMD_NONE;
+	}
+
+	if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
+	    boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
+		pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
+		return SPECTRE_V2_CMD_AUTO;
+	}
+
+	if (mitigation_options[i].secure)
+		spec2_print_if_secure(mitigation_options[i].option);
+	else
+		spec2_print_if_insecure(mitigation_options[i].option);
+
+	return cmd;
 }
 
 /* Check for Skylake-like CPUs (for RSB handling) */
@@ -191,10 +236,10 @@ static void __init spectre_v2_select_mitigation(void)
 		return;
 
 	case SPECTRE_V2_CMD_FORCE:
-		/* FALLTRHU */
 	case SPECTRE_V2_CMD_AUTO:
-		goto retpoline_auto;
-
+		if (IS_ENABLED(CONFIG_RETPOLINE))
+			goto retpoline_auto;
+		break;
 	case SPECTRE_V2_CMD_RETPOLINE_AMD:
 		if (IS_ENABLED(CONFIG_RETPOLINE))
 			goto retpoline_amd;
@@ -208,14 +253,14 @@ static void __init spectre_v2_select_mitigation(void)
 			goto retpoline_auto;
 		break;
 	}
-	pr_err("kernel not compiled with retpoline; no mitigation available!");
+	pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 	return;
 
 retpoline_auto:
 	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 	retpoline_amd:
 		if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
-			pr_err("LFENCE not serializing. Switching to generic retpoline\n");
+			pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 			goto retpoline_generic;
 		}
 		mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
@@ -233,7 +278,7 @@ static void __init spectre_v2_select_mitigation(void)
 	pr_info("%s\n", spectre_v2_strings[mode]);
 
 	/*
-	 * If neither SMEP or KPTI are available, there is a risk of
+	 * If neither SMEP nor PTI are available, there is a risk of
 	 * hitting userspace addresses in the RSB after a context switch
 	 * from a shallow call stack to a deeper one. To prevent this fill
 	 * the entire RSB, even when using IBRS.
@@ -247,15 +292,29 @@ static void __init spectre_v2_select_mitigation(void)
 	if ((!boot_cpu_has(X86_FEATURE_PTI) &&
 	     !boot_cpu_has(X86_FEATURE_SMEP)) || is_skylake_era()) {
 		setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
-		pr_info("Filling RSB on context switch\n");
+		pr_info("Spectre v2 mitigation: Filling RSB on context switch\n");
+	}
+
+	/* Initialize Indirect Branch Prediction Barrier if supported */
+	if (boot_cpu_has(X86_FEATURE_IBPB)) {
+		setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
+		pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
+	}
+
+	/*
+	 * Retpoline means the kernel is safe because it has no indirect
+	 * branches. But firmware isn't, so use IBRS to protect that.
+	 */
+	if (boot_cpu_has(X86_FEATURE_IBRS)) {
+		setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
+		pr_info("Enabling Restricted Speculation for firmware calls\n");
 	}
 }
 
 #undef pr_fmt
 
 #ifdef CONFIG_SYSFS
-ssize_t cpu_show_meltdown(struct device *dev,
-			  struct device_attribute *attr, char *buf)
+ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN))
 		return sprintf(buf, "Not affected\n");
@@ -264,20 +323,21 @@ ssize_t cpu_show_meltdown(struct device *dev,
 	return sprintf(buf, "Vulnerable\n");
 }
 
-ssize_t cpu_show_spectre_v1(struct device *dev,
-			    struct device_attribute *attr, char *buf)
+ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1))
 		return sprintf(buf, "Not affected\n");
-	return sprintf(buf, "Vulnerable\n");
+	return sprintf(buf, "Mitigation: __user pointer sanitization\n");
 }
 
-ssize_t cpu_show_spectre_v2(struct device *dev,
-			    struct device_attribute *attr, char *buf)
+ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
 		return sprintf(buf, "Not affected\n");
 
-	return sprintf(buf, "%s\n", spectre_v2_strings[spectre_v2_enabled]);
+	return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
+		       boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
+		       boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
+		       spectre_v2_module_string());
 }
 #endif
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index 68bc6d9b3132..e5ec0f11c0de 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -106,6 +106,10 @@ static void early_init_centaur(struct cpuinfo_x86 *c)
 #ifdef CONFIG_X86_64
 	set_cpu_cap(c, X86_FEATURE_SYSENTER32);
 #endif
+	if (c->x86_power & (1 << 8)) {
+		set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
+		set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
+	}
 }
 
 static void init_centaur(struct cpuinfo_x86 *c)
@@ -136,7 +140,7 @@ static void init_centaur(struct cpuinfo_x86 *c)
 			clear_cpu_cap(c, X86_FEATURE_TSC);
 			break;
 		case 8:
-			switch (c->x86_mask) {
+			switch (c->x86_stepping) {
 			default:
 			name = "2";
 				break;
@@ -211,7 +215,7 @@ centaur_size_cache(struct cpuinfo_x86 *c, unsigned int size)
 	 *  - Note, it seems this may only be in engineering samples.
 	 */
 	if ((c->x86 == 6) && (c->x86_model == 9) &&
-				(c->x86_mask == 1) && (size == 65))
+				(c->x86_stepping == 1) && (size == 65))
 		size -= 1;
 	return size;
 }
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index ef29ad001991..348cf4821240 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -47,6 +47,8 @@
 #include <asm/pat.h>
 #include <asm/microcode.h>
 #include <asm/microcode_intel.h>
+#include <asm/intel-family.h>
+#include <asm/cpu_device_id.h>
 
 #ifdef CONFIG_X86_LOCAL_APIC
 #include <asm/uv/uv.h>
@@ -729,7 +731,7 @@ void cpu_detect(struct cpuinfo_x86 *c)
 		cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
 		c->x86		= x86_family(tfms);
 		c->x86_model	= x86_model(tfms);
-		c->x86_mask	= x86_stepping(tfms);
+		c->x86_stepping	= x86_stepping(tfms);
 
 		if (cap0 & (1<<19)) {
 			c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
@@ -748,6 +750,26 @@ static void apply_forced_caps(struct cpuinfo_x86 *c)
 	}
 }
 
+static void init_speculation_control(struct cpuinfo_x86 *c)
+{
+	/*
+	 * The Intel SPEC_CTRL CPUID bit implies IBRS and IBPB support,
+	 * and they also have a different bit for STIBP support. Also,
+	 * a hypervisor might have set the individual AMD bits even on
+	 * Intel CPUs, for finer-grained selection of what's available.
+	 *
+	 * We use the AMD bits in 0x8000_0008 EBX as the generic hardware
+	 * features, which are visible in /proc/cpuinfo and used by the
+	 * kernel. So set those accordingly from the Intel bits.
+	 */
+	if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
+		set_cpu_cap(c, X86_FEATURE_IBRS);
+		set_cpu_cap(c, X86_FEATURE_IBPB);
+	}
+	if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
+		set_cpu_cap(c, X86_FEATURE_STIBP);
+}
+
 void get_cpu_cap(struct cpuinfo_x86 *c)
 {
 	u32 eax, ebx, ecx, edx;
@@ -769,6 +791,7 @@ void get_cpu_cap(struct cpuinfo_x86 *c)
 		cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
 		c->x86_capability[CPUID_7_0_EBX] = ebx;
 		c->x86_capability[CPUID_7_ECX] = ecx;
+		c->x86_capability[CPUID_7_EDX] = edx;
 	}
 
 	/* Extended state features: level 0x0000000d */
@@ -841,6 +864,7 @@ void get_cpu_cap(struct cpuinfo_x86 *c)
 		c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
 
 	init_scattered_cpuid_features(c);
+	init_speculation_control(c);
 
 	/*
 	 * Clear/Set all flags overridden by options, after probe.
@@ -876,6 +900,41 @@ static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
 #endif
 }
 
+static const __initconst struct x86_cpu_id cpu_no_speculation[] = {
+	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_CEDARVIEW,	X86_FEATURE_ANY },
+	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_CLOVERVIEW,	X86_FEATURE_ANY },
+	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_LINCROFT,	X86_FEATURE_ANY },
+	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_PENWELL,	X86_FEATURE_ANY },
+	{ X86_VENDOR_INTEL,	6, INTEL_FAM6_ATOM_PINEVIEW,	X86_FEATURE_ANY },
+	{ X86_VENDOR_CENTAUR,	5 },
+	{ X86_VENDOR_INTEL,	5 },
+	{ X86_VENDOR_NSC,	5 },
+	{ X86_VENDOR_ANY,	4 },
+	{}
+};
+
+static const __initconst struct x86_cpu_id cpu_no_meltdown[] = {
+	{ X86_VENDOR_AMD },
+	{}
+};
+
+static bool __init cpu_vulnerable_to_meltdown(struct cpuinfo_x86 *c)
+{
+	u64 ia32_cap = 0;
+
+	if (x86_match_cpu(cpu_no_meltdown))
+		return false;
+
+	if (cpu_has(c, X86_FEATURE_ARCH_CAPABILITIES))
+		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
+
+	/* Rogue Data Cache Load? No! */
+	if (ia32_cap & ARCH_CAP_RDCL_NO)
+		return false;
+
+	return true;
+}
+
 /*
  * Do minimum CPU detection early.
  * Fields really needed: vendor, cpuid_level, family, model, mask,
@@ -923,11 +982,12 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 
 	setup_force_cpu_cap(X86_FEATURE_ALWAYS);
 
-	if (c->x86_vendor != X86_VENDOR_AMD)
-		setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
-
-	setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
-	setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
+	if (!x86_match_cpu(cpu_no_speculation)) {
+		if (cpu_vulnerable_to_meltdown(c))
+			setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
+		setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
+		setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
+	}
 
 	fpu__init_system(c);
 
@@ -1124,9 +1184,9 @@ static void identify_cpu(struct cpuinfo_x86 *c)
 	int i;
 
 	c->loops_per_jiffy = loops_per_jiffy;
-	c->x86_cache_size = -1;
+	c->x86_cache_size = 0;
 	c->x86_vendor = X86_VENDOR_UNKNOWN;
-	c->x86_model = c->x86_mask = 0;	/* So far unknown... */
+	c->x86_model = c->x86_stepping = 0;	/* So far unknown... */
 	c->x86_vendor_id[0] = '\0'; /* Unset */
 	c->x86_model_id[0] = '\0';  /* Unset */
 	c->x86_max_cores = 1;
@@ -1318,8 +1378,8 @@ void print_cpu_info(struct cpuinfo_x86 *c)
 
 	pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
 
-	if (c->x86_mask || c->cpuid_level >= 0)
-		pr_cont(", stepping: 0x%x)\n", c->x86_mask);
+	if (c->x86_stepping || c->cpuid_level >= 0)
+		pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
 	else
 		pr_cont(")\n");
 }
@@ -1689,3 +1749,33 @@ static int __init init_cpu_syscore(void)
 	return 0;
 }
 core_initcall(init_cpu_syscore);
+
+/*
+ * The microcode loader calls this upon late microcode load to recheck features,
+ * only when microcode has been updated. Caller holds microcode_mutex and CPU
+ * hotplug lock.
+ */
+void microcode_check(void)
+{
+	struct cpuinfo_x86 info;
+
+	perf_check_microcode();
+
+	/* Reload CPUID max function as it might've changed. */
+	info.cpuid_level = cpuid_eax(0);
+
+	/*
+	 * Copy all capability leafs to pick up the synthetic ones so that
+	 * memcmp() below doesn't fail on that. The ones coming from CPUID will
+	 * get overwritten in get_cpu_cap().
+	 */
+	memcpy(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability));
+
+	get_cpu_cap(&info);
+
+	if (!memcmp(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability)))
+		return;
+
+	pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
+	pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
+}
diff --git a/arch/x86/kernel/cpu/cyrix.c b/arch/x86/kernel/cpu/cyrix.c
index 6b4bb335641f..8949b7ae6d92 100644
--- a/arch/x86/kernel/cpu/cyrix.c
+++ b/arch/x86/kernel/cpu/cyrix.c
@@ -215,7 +215,7 @@ static void init_cyrix(struct cpuinfo_x86 *c)
 
 	/* common case step number/rev -- exceptions handled below */
 	c->x86_model = (dir1 >> 4) + 1;
-	c->x86_mask = dir1 & 0xf;
+	c->x86_stepping = dir1 & 0xf;
 
 	/* Now cook; the original recipe is by Channing Corn, from Cyrix.
 	 * We do the same thing for each generation: we work out
diff --git a/arch/x86/kernel/cpu/hypervisor.c b/arch/x86/kernel/cpu/hypervisor.c
index bea8d3e24f50..479ca4728de0 100644
--- a/arch/x86/kernel/cpu/hypervisor.c
+++ b/arch/x86/kernel/cpu/hypervisor.c
@@ -31,6 +31,7 @@ extern const struct hypervisor_x86 x86_hyper_ms_hyperv;
 extern const struct hypervisor_x86 x86_hyper_xen_pv;
 extern const struct hypervisor_x86 x86_hyper_xen_hvm;
 extern const struct hypervisor_x86 x86_hyper_kvm;
+extern const struct hypervisor_x86 x86_hyper_jailhouse;
 
 static const __initconst struct hypervisor_x86 * const hypervisors[] =
 {
@@ -45,6 +46,9 @@ static const __initconst struct hypervisor_x86 * const hypervisors[] =
 #ifdef CONFIG_KVM_GUEST
 	&x86_hyper_kvm,
 #endif
+#ifdef CONFIG_JAILHOUSE_GUEST
+	&x86_hyper_jailhouse,
+#endif
 };
 
 enum x86_hypervisor_type x86_hyper_type;
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index b1af22073e28..c3af167d0a70 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -102,6 +102,62 @@ static void probe_xeon_phi_r3mwait(struct cpuinfo_x86 *c)
 		ELF_HWCAP2 |= HWCAP2_RING3MWAIT;
 }
 
+/*
+ * Early microcode releases for the Spectre v2 mitigation were broken.
+ * Information taken from;
+ * - https://newsroom.intel.com/wp-content/uploads/sites/11/2018/03/microcode-update-guidance.pdf
+ * - https://kb.vmware.com/s/article/52345
+ * - Microcode revisions observed in the wild
+ * - Release note from 20180108 microcode release
+ */
+struct sku_microcode {
+	u8 model;
+	u8 stepping;
+	u32 microcode;
+};
+static const struct sku_microcode spectre_bad_microcodes[] = {
+	{ INTEL_FAM6_KABYLAKE_DESKTOP,	0x0B,	0x80 },
+	{ INTEL_FAM6_KABYLAKE_DESKTOP,	0x0A,	0x80 },
+	{ INTEL_FAM6_KABYLAKE_DESKTOP,	0x09,	0x80 },
+	{ INTEL_FAM6_KABYLAKE_MOBILE,	0x0A,	0x80 },
+	{ INTEL_FAM6_KABYLAKE_MOBILE,	0x09,	0x80 },
+	{ INTEL_FAM6_SKYLAKE_X,		0x03,	0x0100013e },
+	{ INTEL_FAM6_SKYLAKE_X,		0x04,	0x0200003c },
+	{ INTEL_FAM6_BROADWELL_CORE,	0x04,	0x28 },
+	{ INTEL_FAM6_BROADWELL_GT3E,	0x01,	0x1b },
+	{ INTEL_FAM6_BROADWELL_XEON_D,	0x02,	0x14 },
+	{ INTEL_FAM6_BROADWELL_XEON_D,	0x03,	0x07000011 },
+	{ INTEL_FAM6_BROADWELL_X,	0x01,	0x0b000025 },
+	{ INTEL_FAM6_HASWELL_ULT,	0x01,	0x21 },
+	{ INTEL_FAM6_HASWELL_GT3E,	0x01,	0x18 },
+	{ INTEL_FAM6_HASWELL_CORE,	0x03,	0x23 },
+	{ INTEL_FAM6_HASWELL_X,		0x02,	0x3b },
+	{ INTEL_FAM6_HASWELL_X,		0x04,	0x10 },
+	{ INTEL_FAM6_IVYBRIDGE_X,	0x04,	0x42a },
+	/* Observed in the wild */
+	{ INTEL_FAM6_SANDYBRIDGE_X,	0x06,	0x61b },
+	{ INTEL_FAM6_SANDYBRIDGE_X,	0x07,	0x712 },
+};
+
+static bool bad_spectre_microcode(struct cpuinfo_x86 *c)
+{
+	int i;
+
+	/*
+	 * We know that the hypervisor lie to us on the microcode version so
+	 * we may as well hope that it is running the correct version.
+	 */
+	if (cpu_has(c, X86_FEATURE_HYPERVISOR))
+		return false;
+
+	for (i = 0; i < ARRAY_SIZE(spectre_bad_microcodes); i++) {
+		if (c->x86_model == spectre_bad_microcodes[i].model &&
+		    c->x86_stepping == spectre_bad_microcodes[i].stepping)
+			return (c->microcode <= spectre_bad_microcodes[i].microcode);
+	}
+	return false;
+}
+
 static void early_init_intel(struct cpuinfo_x86 *c)
 {
 	u64 misc_enable;
@@ -122,6 +178,19 @@ static void early_init_intel(struct cpuinfo_x86 *c)
 	if (c->x86 >= 6 && !cpu_has(c, X86_FEATURE_IA64))
 		c->microcode = intel_get_microcode_revision();
 
+	/* Now if any of them are set, check the blacklist and clear the lot */
+	if ((cpu_has(c, X86_FEATURE_SPEC_CTRL) ||
+	     cpu_has(c, X86_FEATURE_INTEL_STIBP) ||
+	     cpu_has(c, X86_FEATURE_IBRS) || cpu_has(c, X86_FEATURE_IBPB) ||
+	     cpu_has(c, X86_FEATURE_STIBP)) && bad_spectre_microcode(c)) {
+		pr_warn("Intel Spectre v2 broken microcode detected; disabling Speculation Control\n");
+		setup_clear_cpu_cap(X86_FEATURE_IBRS);
+		setup_clear_cpu_cap(X86_FEATURE_IBPB);
+		setup_clear_cpu_cap(X86_FEATURE_STIBP);
+		setup_clear_cpu_cap(X86_FEATURE_SPEC_CTRL);
+		setup_clear_cpu_cap(X86_FEATURE_INTEL_STIBP);
+	}
+
 	/*
 	 * Atom erratum AAE44/AAF40/AAG38/AAH41:
 	 *
@@ -130,7 +199,7 @@ static void early_init_intel(struct cpuinfo_x86 *c)
 	 * need the microcode to have already been loaded... so if it is
 	 * not, recommend a BIOS update and disable large pages.
 	 */
-	if (c->x86 == 6 && c->x86_model == 0x1c && c->x86_mask <= 2 &&
+	if (c->x86 == 6 && c->x86_model == 0x1c && c->x86_stepping <= 2 &&
 	    c->microcode < 0x20e) {
 		pr_warn("Atom PSE erratum detected, BIOS microcode update recommended\n");
 		clear_cpu_cap(c, X86_FEATURE_PSE);
@@ -146,7 +215,7 @@ static void early_init_intel(struct cpuinfo_x86 *c)
 
 	/* CPUID workaround for 0F33/0F34 CPU */
 	if (c->x86 == 0xF && c->x86_model == 0x3
-	    && (c->x86_mask == 0x3 || c->x86_mask == 0x4))
+	    && (c->x86_stepping == 0x3 || c->x86_stepping == 0x4))
 		c->x86_phys_bits = 36;
 
 	/*
@@ -244,7 +313,7 @@ int ppro_with_ram_bug(void)
 	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 	    boot_cpu_data.x86 == 6 &&
 	    boot_cpu_data.x86_model == 1 &&
-	    boot_cpu_data.x86_mask < 8) {
+	    boot_cpu_data.x86_stepping < 8) {
 		pr_info("Pentium Pro with Errata#50 detected. Taking evasive action.\n");
 		return 1;
 	}
@@ -261,7 +330,7 @@ static void intel_smp_check(struct cpuinfo_x86 *c)
 	 * Mask B, Pentium, but not Pentium MMX
 	 */
 	if (c->x86 == 5 &&
-	    c->x86_mask >= 1 && c->x86_mask <= 4 &&
+	    c->x86_stepping >= 1 && c->x86_stepping <= 4 &&
 	    c->x86_model <= 3) {
 		/*
 		 * Remember we have B step Pentia with bugs
@@ -304,7 +373,7 @@ static void intel_workarounds(struct cpuinfo_x86 *c)
 	 * SEP CPUID bug: Pentium Pro reports SEP but doesn't have it until
 	 * model 3 mask 3
 	 */
-	if ((c->x86<<8 | c->x86_model<<4 | c->x86_mask) < 0x633)
+	if ((c->x86<<8 | c->x86_model<<4 | c->x86_stepping) < 0x633)
 		clear_cpu_cap(c, X86_FEATURE_SEP);
 
 	/*
@@ -322,7 +391,7 @@ static void intel_workarounds(struct cpuinfo_x86 *c)
 	 * P4 Xeon erratum 037 workaround.
 	 * Hardware prefetcher may cause stale data to be loaded into the cache.
 	 */
-	if ((c->x86 == 15) && (c->x86_model == 1) && (c->x86_mask == 1)) {
+	if ((c->x86 == 15) && (c->x86_model == 1) && (c->x86_stepping == 1)) {
 		if (msr_set_bit(MSR_IA32_MISC_ENABLE,
 				MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE_BIT) > 0) {
 			pr_info("CPU: C0 stepping P4 Xeon detected.\n");
@@ -337,7 +406,7 @@ static void intel_workarounds(struct cpuinfo_x86 *c)
 	 * Specification Update").
 	 */
 	if (boot_cpu_has(X86_FEATURE_APIC) && (c->x86<<8 | c->x86_model<<4) == 0x520 &&
-	    (c->x86_mask < 0x6 || c->x86_mask == 0xb))
+	    (c->x86_stepping < 0x6 || c->x86_stepping == 0xb))
 		set_cpu_bug(c, X86_BUG_11AP);
 
 
@@ -584,7 +653,7 @@ static void init_intel(struct cpuinfo_x86 *c)
 		case 6:
 			if (l2 == 128)
 				p = "Celeron (Mendocino)";
-			else if (c->x86_mask == 0 || c->x86_mask == 5)
+			else if (c->x86_stepping == 0 || c->x86_stepping == 5)
 				p = "Celeron-A";
 			break;
 
diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c
index 99442370de40..589b948e6e01 100644
--- a/arch/x86/kernel/cpu/intel_rdt.c
+++ b/arch/x86/kernel/cpu/intel_rdt.c
@@ -135,6 +135,40 @@ struct rdt_resource rdt_resources_all[] = {
 		.format_str		= "%d=%0*x",
 		.fflags			= RFTYPE_RES_CACHE,
 	},
+	[RDT_RESOURCE_L2DATA] =
+	{
+		.rid			= RDT_RESOURCE_L2DATA,
+		.name			= "L2DATA",
+		.domains		= domain_init(RDT_RESOURCE_L2DATA),
+		.msr_base		= IA32_L2_CBM_BASE,
+		.msr_update		= cat_wrmsr,
+		.cache_level		= 2,
+		.cache = {
+			.min_cbm_bits	= 1,
+			.cbm_idx_mult	= 2,
+			.cbm_idx_offset	= 0,
+		},
+		.parse_ctrlval		= parse_cbm,
+		.format_str		= "%d=%0*x",
+		.fflags			= RFTYPE_RES_CACHE,
+	},
+	[RDT_RESOURCE_L2CODE] =
+	{
+		.rid			= RDT_RESOURCE_L2CODE,
+		.name			= "L2CODE",
+		.domains		= domain_init(RDT_RESOURCE_L2CODE),
+		.msr_base		= IA32_L2_CBM_BASE,
+		.msr_update		= cat_wrmsr,
+		.cache_level		= 2,
+		.cache = {
+			.min_cbm_bits	= 1,
+			.cbm_idx_mult	= 2,
+			.cbm_idx_offset	= 1,
+		},
+		.parse_ctrlval		= parse_cbm,
+		.format_str		= "%d=%0*x",
+		.fflags			= RFTYPE_RES_CACHE,
+	},
 	[RDT_RESOURCE_MBA] =
 	{
 		.rid			= RDT_RESOURCE_MBA,
@@ -259,15 +293,15 @@ static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r)
 	r->alloc_enabled = true;
 }
 
-static void rdt_get_cdp_l3_config(int type)
+static void rdt_get_cdp_config(int level, int type)
 {
-	struct rdt_resource *r_l3 = &rdt_resources_all[RDT_RESOURCE_L3];
+	struct rdt_resource *r_l = &rdt_resources_all[level];
 	struct rdt_resource *r = &rdt_resources_all[type];
 
-	r->num_closid = r_l3->num_closid / 2;
-	r->cache.cbm_len = r_l3->cache.cbm_len;
-	r->default_ctrl = r_l3->default_ctrl;
-	r->cache.shareable_bits = r_l3->cache.shareable_bits;
+	r->num_closid = r_l->num_closid / 2;
+	r->cache.cbm_len = r_l->cache.cbm_len;
+	r->default_ctrl = r_l->default_ctrl;
+	r->cache.shareable_bits = r_l->cache.shareable_bits;
 	r->data_width = (r->cache.cbm_len + 3) / 4;
 	r->alloc_capable = true;
 	/*
@@ -277,6 +311,18 @@ static void rdt_get_cdp_l3_config(int type)
 	r->alloc_enabled = false;
 }
 
+static void rdt_get_cdp_l3_config(void)
+{
+	rdt_get_cdp_config(RDT_RESOURCE_L3, RDT_RESOURCE_L3DATA);
+	rdt_get_cdp_config(RDT_RESOURCE_L3, RDT_RESOURCE_L3CODE);
+}
+
+static void rdt_get_cdp_l2_config(void)
+{
+	rdt_get_cdp_config(RDT_RESOURCE_L2, RDT_RESOURCE_L2DATA);
+	rdt_get_cdp_config(RDT_RESOURCE_L2, RDT_RESOURCE_L2CODE);
+}
+
 static int get_cache_id(int cpu, int level)
 {
 	struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu);
@@ -645,6 +691,7 @@ enum {
 	RDT_FLAG_L3_CAT,
 	RDT_FLAG_L3_CDP,
 	RDT_FLAG_L2_CAT,
+	RDT_FLAG_L2_CDP,
 	RDT_FLAG_MBA,
 };
 
@@ -667,6 +714,7 @@ static struct rdt_options rdt_options[]  __initdata = {
 	RDT_OPT(RDT_FLAG_L3_CAT,    "l3cat",	X86_FEATURE_CAT_L3),
 	RDT_OPT(RDT_FLAG_L3_CDP,    "l3cdp",	X86_FEATURE_CDP_L3),
 	RDT_OPT(RDT_FLAG_L2_CAT,    "l2cat",	X86_FEATURE_CAT_L2),
+	RDT_OPT(RDT_FLAG_L2_CDP,    "l2cdp",	X86_FEATURE_CDP_L2),
 	RDT_OPT(RDT_FLAG_MBA,	    "mba",	X86_FEATURE_MBA),
 };
 #define NUM_RDT_OPTIONS ARRAY_SIZE(rdt_options)
@@ -729,15 +777,15 @@ static __init bool get_rdt_alloc_resources(void)
 
 	if (rdt_cpu_has(X86_FEATURE_CAT_L3)) {
 		rdt_get_cache_alloc_cfg(1, &rdt_resources_all[RDT_RESOURCE_L3]);
-		if (rdt_cpu_has(X86_FEATURE_CDP_L3)) {
-			rdt_get_cdp_l3_config(RDT_RESOURCE_L3DATA);
-			rdt_get_cdp_l3_config(RDT_RESOURCE_L3CODE);
-		}
+		if (rdt_cpu_has(X86_FEATURE_CDP_L3))
+			rdt_get_cdp_l3_config();
 		ret = true;
 	}
 	if (rdt_cpu_has(X86_FEATURE_CAT_L2)) {
 		/* CPUID 0x10.2 fields are same format at 0x10.1 */
 		rdt_get_cache_alloc_cfg(2, &rdt_resources_all[RDT_RESOURCE_L2]);
+		if (rdt_cpu_has(X86_FEATURE_CDP_L2))
+			rdt_get_cdp_l2_config();
 		ret = true;
 	}
 
@@ -771,7 +819,7 @@ static __init void rdt_quirks(void)
 			cache_alloc_hsw_probe();
 		break;
 	case INTEL_FAM6_SKYLAKE_X:
-		if (boot_cpu_data.x86_mask <= 4)
+		if (boot_cpu_data.x86_stepping <= 4)
 			set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat");
 	}
 }
diff --git a/arch/x86/kernel/cpu/intel_rdt.h b/arch/x86/kernel/cpu/intel_rdt.h
index 3397244984f5..3fd7a70ee04a 100644
--- a/arch/x86/kernel/cpu/intel_rdt.h
+++ b/arch/x86/kernel/cpu/intel_rdt.h
@@ -7,12 +7,15 @@
 #include <linux/jump_label.h>
 
 #define IA32_L3_QOS_CFG		0xc81
+#define IA32_L2_QOS_CFG		0xc82
 #define IA32_L3_CBM_BASE	0xc90
 #define IA32_L2_CBM_BASE	0xd10
 #define IA32_MBA_THRTL_BASE	0xd50
 
 #define L3_QOS_CDP_ENABLE	0x01ULL
 
+#define L2_QOS_CDP_ENABLE	0x01ULL
+
 /*
  * Event IDs are used to program IA32_QM_EVTSEL before reading event
  * counter from IA32_QM_CTR
@@ -357,6 +360,8 @@ enum {
 	RDT_RESOURCE_L3DATA,
 	RDT_RESOURCE_L3CODE,
 	RDT_RESOURCE_L2,
+	RDT_RESOURCE_L2DATA,
+	RDT_RESOURCE_L2CODE,
 	RDT_RESOURCE_MBA,
 
 	/* Must be the last */
diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
index 64c5ff97ee0d..fca759d272a1 100644
--- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
+++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
@@ -990,6 +990,7 @@ mongroup_create_dir(struct kernfs_node *parent_kn, struct rdtgroup *prgrp,
 	kernfs_remove(kn);
 	return ret;
 }
+
 static void l3_qos_cfg_update(void *arg)
 {
 	bool *enable = arg;
@@ -997,8 +998,17 @@ static void l3_qos_cfg_update(void *arg)
 	wrmsrl(IA32_L3_QOS_CFG, *enable ? L3_QOS_CDP_ENABLE : 0ULL);
 }
 
-static int set_l3_qos_cfg(struct rdt_resource *r, bool enable)
+static void l2_qos_cfg_update(void *arg)
 {
+	bool *enable = arg;
+
+	wrmsrl(IA32_L2_QOS_CFG, *enable ? L2_QOS_CDP_ENABLE : 0ULL);
+}
+
+static int set_cache_qos_cfg(int level, bool enable)
+{
+	void (*update)(void *arg);
+	struct rdt_resource *r_l;
 	cpumask_var_t cpu_mask;
 	struct rdt_domain *d;
 	int cpu;
@@ -1006,16 +1016,24 @@ static int set_l3_qos_cfg(struct rdt_resource *r, bool enable)
 	if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
 		return -ENOMEM;
 
-	list_for_each_entry(d, &r->domains, list) {
+	if (level == RDT_RESOURCE_L3)
+		update = l3_qos_cfg_update;
+	else if (level == RDT_RESOURCE_L2)
+		update = l2_qos_cfg_update;
+	else
+		return -EINVAL;
+
+	r_l = &rdt_resources_all[level];
+	list_for_each_entry(d, &r_l->domains, list) {
 		/* Pick one CPU from each domain instance to update MSR */
 		cpumask_set_cpu(cpumask_any(&d->cpu_mask), cpu_mask);
 	}
 	cpu = get_cpu();
 	/* Update QOS_CFG MSR on this cpu if it's in cpu_mask. */
 	if (cpumask_test_cpu(cpu, cpu_mask))
-		l3_qos_cfg_update(&enable);
+		update(&enable);
 	/* Update QOS_CFG MSR on all other cpus in cpu_mask. */
-	smp_call_function_many(cpu_mask, l3_qos_cfg_update, &enable, 1);
+	smp_call_function_many(cpu_mask, update, &enable, 1);
 	put_cpu();
 
 	free_cpumask_var(cpu_mask);
@@ -1023,52 +1041,99 @@ static int set_l3_qos_cfg(struct rdt_resource *r, bool enable)
 	return 0;
 }
 
-static int cdp_enable(void)
+static int cdp_enable(int level, int data_type, int code_type)
 {
-	struct rdt_resource *r_l3data = &rdt_resources_all[RDT_RESOURCE_L3DATA];
-	struct rdt_resource *r_l3code = &rdt_resources_all[RDT_RESOURCE_L3CODE];
-	struct rdt_resource *r_l3 = &rdt_resources_all[RDT_RESOURCE_L3];
+	struct rdt_resource *r_ldata = &rdt_resources_all[data_type];
+	struct rdt_resource *r_lcode = &rdt_resources_all[code_type];
+	struct rdt_resource *r_l = &rdt_resources_all[level];
 	int ret;
 
-	if (!r_l3->alloc_capable || !r_l3data->alloc_capable ||
-	    !r_l3code->alloc_capable)
+	if (!r_l->alloc_capable || !r_ldata->alloc_capable ||
+	    !r_lcode->alloc_capable)
 		return -EINVAL;
 
-	ret = set_l3_qos_cfg(r_l3, true);
+	ret = set_cache_qos_cfg(level, true);
 	if (!ret) {
-		r_l3->alloc_enabled = false;
-		r_l3data->alloc_enabled = true;
-		r_l3code->alloc_enabled = true;
+		r_l->alloc_enabled = false;
+		r_ldata->alloc_enabled = true;
+		r_lcode->alloc_enabled = true;
 	}
 	return ret;
 }
 
-static void cdp_disable(void)
+static int cdpl3_enable(void)
 {
-	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3];
+	return cdp_enable(RDT_RESOURCE_L3, RDT_RESOURCE_L3DATA,
+			  RDT_RESOURCE_L3CODE);
+}
+
+static int cdpl2_enable(void)
+{
+	return cdp_enable(RDT_RESOURCE_L2, RDT_RESOURCE_L2DATA,
+			  RDT_RESOURCE_L2CODE);
+}
+
+static void cdp_disable(int level, int data_type, int code_type)
+{
+	struct rdt_resource *r = &rdt_resources_all[level];
 
 	r->alloc_enabled = r->alloc_capable;
 
-	if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled) {
-		rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled = false;
-		rdt_resources_all[RDT_RESOURCE_L3CODE].alloc_enabled = false;
-		set_l3_qos_cfg(r, false);
+	if (rdt_resources_all[data_type].alloc_enabled) {
+		rdt_resources_all[data_type].alloc_enabled = false;
+		rdt_resources_all[code_type].alloc_enabled = false;
+		set_cache_qos_cfg(level, false);
 	}
 }
 
+static void cdpl3_disable(void)
+{
+	cdp_disable(RDT_RESOURCE_L3, RDT_RESOURCE_L3DATA, RDT_RESOURCE_L3CODE);
+}
+
+static void cdpl2_disable(void)
+{
+	cdp_disable(RDT_RESOURCE_L2, RDT_RESOURCE_L2DATA, RDT_RESOURCE_L2CODE);
+}
+
+static void cdp_disable_all(void)
+{
+	if (rdt_resources_all[RDT_RESOURCE_L3DATA].alloc_enabled)
+		cdpl3_disable();
+	if (rdt_resources_all[RDT_RESOURCE_L2DATA].alloc_enabled)
+		cdpl2_disable();
+}
+
 static int parse_rdtgroupfs_options(char *data)
 {
 	char *token, *o = data;
 	int ret = 0;
 
 	while ((token = strsep(&o, ",")) != NULL) {
-		if (!*token)
-			return -EINVAL;
+		if (!*token) {
+			ret = -EINVAL;
+			goto out;
+		}
 
-		if (!strcmp(token, "cdp"))
-			ret = cdp_enable();
+		if (!strcmp(token, "cdp")) {
+			ret = cdpl3_enable();
+			if (ret)
+				goto out;
+		} else if (!strcmp(token, "cdpl2")) {
+			ret = cdpl2_enable();
+			if (ret)
+				goto out;
+		} else {
+			ret = -EINVAL;
+			goto out;
+		}
 	}
 
+	return 0;
+
+out:
+	pr_err("Invalid mount option \"%s\"\n", token);
+
 	return ret;
 }
 
@@ -1223,7 +1288,7 @@ static struct dentry *rdt_mount(struct file_system_type *fs_type,
 out_info:
 	kernfs_remove(kn_info);
 out_cdp:
-	cdp_disable();
+	cdp_disable_all();
 out:
 	rdt_last_cmd_clear();
 	mutex_unlock(&rdtgroup_mutex);
@@ -1383,7 +1448,7 @@ static void rdt_kill_sb(struct super_block *sb)
 	/*Put everything back to default values. */
 	for_each_alloc_enabled_rdt_resource(r)
 		reset_all_ctrls(r);
-	cdp_disable();
+	cdp_disable_all();
 	rmdir_all_sub();
 	static_branch_disable_cpuslocked(&rdt_alloc_enable_key);
 	static_branch_disable_cpuslocked(&rdt_mon_enable_key);
@@ -1739,6 +1804,7 @@ static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn,
 		goto out_common_fail;
 	}
 	closid = ret;
+	ret = 0;
 
 	rdtgrp->closid = closid;
 	list_add(&rdtgrp->rdtgroup_list, &rdt_all_groups);
diff --git a/arch/x86/kernel/cpu/mcheck/dev-mcelog.c b/arch/x86/kernel/cpu/mcheck/dev-mcelog.c
index 7f85b76f43bc..97685a0c3175 100644
--- a/arch/x86/kernel/cpu/mcheck/dev-mcelog.c
+++ b/arch/x86/kernel/cpu/mcheck/dev-mcelog.c
@@ -243,13 +243,13 @@ static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf,
 	return err ? err : buf - ubuf;
 }
 
-static unsigned int mce_chrdev_poll(struct file *file, poll_table *wait)
+static __poll_t mce_chrdev_poll(struct file *file, poll_table *wait)
 {
 	poll_wait(file, &mce_chrdev_wait, wait);
 	if (READ_ONCE(mcelog.next))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	if (!mce_apei_read_done && apei_check_mce())
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce-internal.h b/arch/x86/kernel/cpu/mcheck/mce-internal.h
index aa0d5df9dc60..e956eb267061 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-internal.h
+++ b/arch/x86/kernel/cpu/mcheck/mce-internal.h
@@ -115,4 +115,19 @@ static inline void mce_unregister_injector_chain(struct notifier_block *nb)	{ }
 
 extern struct mca_config mca_cfg;
 
+#ifndef CONFIG_X86_64
+/*
+ * On 32-bit systems it would be difficult to safely unmap a poison page
+ * from the kernel 1:1 map because there are no non-canonical addresses that
+ * we can use to refer to the address without risking a speculative access.
+ * However, this isn't much of an issue because:
+ * 1) Few unmappable pages are in the 1:1 map. Most are in HIGHMEM which
+ *    are only mapped into the kernel as needed
+ * 2) Few people would run a 32-bit kernel on a machine that supports
+ *    recoverable errors because they have too much memory to boot 32-bit.
+ */
+static inline void mce_unmap_kpfn(unsigned long pfn) {}
+#define mce_unmap_kpfn mce_unmap_kpfn
+#endif
+
 #endif /* __X86_MCE_INTERNAL_H__ */
diff --git a/arch/x86/kernel/cpu/mcheck/mce-severity.c b/arch/x86/kernel/cpu/mcheck/mce-severity.c
index 4ca632a06e0b..5bbd06f38ff6 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-severity.c
+++ b/arch/x86/kernel/cpu/mcheck/mce-severity.c
@@ -59,6 +59,7 @@ static struct severity {
 #define  MCGMASK(x, y)	.mcgmask = x, .mcgres = y
 #define  MASK(x, y)	.mask = x, .result = y
 #define MCI_UC_S (MCI_STATUS_UC|MCI_STATUS_S)
+#define MCI_UC_AR (MCI_STATUS_UC|MCI_STATUS_AR)
 #define MCI_UC_SAR (MCI_STATUS_UC|MCI_STATUS_S|MCI_STATUS_AR)
 #define	MCI_ADDR (MCI_STATUS_ADDRV|MCI_STATUS_MISCV)
 
@@ -101,6 +102,22 @@ static struct severity {
 		NOSER, BITCLR(MCI_STATUS_UC)
 		),
 
+	/*
+	 * known AO MCACODs reported via MCE or CMC:
+	 *
+	 * SRAO could be signaled either via a machine check exception or
+	 * CMCI with the corresponding bit S 1 or 0. So we don't need to
+	 * check bit S for SRAO.
+	 */
+	MCESEV(
+		AO, "Action optional: memory scrubbing error",
+		SER, MASK(MCI_STATUS_OVER|MCI_UC_AR|MCACOD_SCRUBMSK, MCI_STATUS_UC|MCACOD_SCRUB)
+		),
+	MCESEV(
+		AO, "Action optional: last level cache writeback error",
+		SER, MASK(MCI_STATUS_OVER|MCI_UC_AR|MCACOD, MCI_STATUS_UC|MCACOD_L3WB)
+		),
+
 	/* ignore OVER for UCNA */
 	MCESEV(
 		UCNA, "Uncorrected no action required",
@@ -149,15 +166,6 @@ static struct severity {
 		SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_SAR)
 		),
 
-	/* known AO MCACODs: */
-	MCESEV(
-		AO, "Action optional: memory scrubbing error",
-		SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR|MCACOD_SCRUBMSK, MCI_UC_S|MCACOD_SCRUB)
-		),
-	MCESEV(
-		AO, "Action optional: last level cache writeback error",
-		SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR|MCACOD, MCI_UC_S|MCACOD_L3WB)
-		),
 	MCESEV(
 		SOME, "Action optional: unknown MCACOD",
 		SER, MASK(MCI_STATUS_OVER|MCI_UC_SAR, MCI_UC_S)
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 868e412b4f0c..466f47301334 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -14,7 +14,6 @@
 #include <linux/capability.h>
 #include <linux/miscdevice.h>
 #include <linux/ratelimit.h>
-#include <linux/kallsyms.h>
 #include <linux/rcupdate.h>
 #include <linux/kobject.h>
 #include <linux/uaccess.h>
@@ -57,6 +56,9 @@
 
 static DEFINE_MUTEX(mce_log_mutex);
 
+/* sysfs synchronization */
+static DEFINE_MUTEX(mce_sysfs_mutex);
+
 #define CREATE_TRACE_POINTS
 #include <trace/events/mce.h>
 
@@ -106,6 +108,10 @@ static struct irq_work mce_irq_work;
 
 static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);
 
+#ifndef mce_unmap_kpfn
+static void mce_unmap_kpfn(unsigned long pfn);
+#endif
+
 /*
  * CPU/chipset specific EDAC code can register a notifier call here to print
  * MCE errors in a human-readable form.
@@ -127,6 +133,8 @@ void mce_setup(struct mce *m)
 
 	if (this_cpu_has(X86_FEATURE_INTEL_PPIN))
 		rdmsrl(MSR_PPIN, m->ppin);
+
+	m->microcode = boot_cpu_data.microcode;
 }
 
 DEFINE_PER_CPU(struct mce, injectm);
@@ -235,7 +243,7 @@ static void __print_mce(struct mce *m)
 			m->cs, m->ip);
 
 		if (m->cs == __KERNEL_CS)
-			print_symbol("{%s}", m->ip);
+			pr_cont("{%pS}", (void *)(unsigned long)m->ip);
 		pr_cont("\n");
 	}
 
@@ -259,7 +267,7 @@ static void __print_mce(struct mce *m)
 	 */
 	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
 		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
-		cpu_data(m->extcpu).microcode);
+		m->microcode);
 }
 
 static void print_mce(struct mce *m)
@@ -503,10 +511,8 @@ static int mce_usable_address(struct mce *m)
 bool mce_is_memory_error(struct mce *m)
 {
 	if (m->cpuvendor == X86_VENDOR_AMD) {
-		/* ErrCodeExt[20:16] */
-		u8 xec = (m->status >> 16) & 0x1f;
+		return amd_mce_is_memory_error(m);
 
-		return (xec == 0x0 || xec == 0x8);
 	} else if (m->cpuvendor == X86_VENDOR_INTEL) {
 		/*
 		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
@@ -530,6 +536,17 @@ bool mce_is_memory_error(struct mce *m)
 }
 EXPORT_SYMBOL_GPL(mce_is_memory_error);
 
+static bool mce_is_correctable(struct mce *m)
+{
+	if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
+		return false;
+
+	if (m->status & MCI_STATUS_UC)
+		return false;
+
+	return true;
+}
+
 static bool cec_add_mce(struct mce *m)
 {
 	if (!m)
@@ -537,7 +554,7 @@ static bool cec_add_mce(struct mce *m)
 
 	/* We eat only correctable DRAM errors with usable addresses. */
 	if (mce_is_memory_error(m) &&
-	    !(m->status & MCI_STATUS_UC) &&
+	    mce_is_correctable(m)  &&
 	    mce_usable_address(m))
 		if (!cec_add_elem(m->addr >> PAGE_SHIFT))
 			return true;
@@ -582,7 +599,8 @@ static int srao_decode_notifier(struct notifier_block *nb, unsigned long val,
 
 	if (mce_usable_address(mce) && (mce->severity == MCE_AO_SEVERITY)) {
 		pfn = mce->addr >> PAGE_SHIFT;
-		memory_failure(pfn, MCE_VECTOR, 0);
+		if (!memory_failure(pfn, 0))
+			mce_unmap_kpfn(pfn);
 	}
 
 	return NOTIFY_OK;
@@ -1046,15 +1064,16 @@ static int do_memory_failure(struct mce *m)
 	pr_err("Uncorrected hardware memory error in user-access at %llx", m->addr);
 	if (!(m->mcgstatus & MCG_STATUS_RIPV))
 		flags |= MF_MUST_KILL;
-	ret = memory_failure(m->addr >> PAGE_SHIFT, MCE_VECTOR, flags);
+	ret = memory_failure(m->addr >> PAGE_SHIFT, flags);
 	if (ret)
 		pr_err("Memory error not recovered");
+	else
+		mce_unmap_kpfn(m->addr >> PAGE_SHIFT);
 	return ret;
 }
 
-#if defined(arch_unmap_kpfn) && defined(CONFIG_MEMORY_FAILURE)
-
-void arch_unmap_kpfn(unsigned long pfn)
+#ifndef mce_unmap_kpfn
+static void mce_unmap_kpfn(unsigned long pfn)
 {
 	unsigned long decoy_addr;
 
@@ -1065,7 +1084,7 @@ void arch_unmap_kpfn(unsigned long pfn)
 	 * We would like to just call:
 	 *	set_memory_np((unsigned long)pfn_to_kaddr(pfn), 1);
 	 * but doing that would radically increase the odds of a
-	 * speculative access to the posion page because we'd have
+	 * speculative access to the poison page because we'd have
 	 * the virtual address of the kernel 1:1 mapping sitting
 	 * around in registers.
 	 * Instead we get tricky.  We create a non-canonical address
@@ -1090,7 +1109,6 @@ void arch_unmap_kpfn(unsigned long pfn)
 
 	if (set_memory_np(decoy_addr, 1))
 		pr_warn("Could not invalidate pfn=0x%lx from 1:1 map\n", pfn);
-
 }
 #endif
 
@@ -1325,7 +1343,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
 EXPORT_SYMBOL_GPL(do_machine_check);
 
 #ifndef CONFIG_MEMORY_FAILURE
-int memory_failure(unsigned long pfn, int vector, int flags)
+int memory_failure(unsigned long pfn, int flags)
 {
 	/* mce_severity() should not hand us an ACTION_REQUIRED error */
 	BUG_ON(flags & MF_ACTION_REQUIRED);
@@ -2073,6 +2091,7 @@ static ssize_t set_ignore_ce(struct device *s,
 	if (kstrtou64(buf, 0, &new) < 0)
 		return -EINVAL;
 
+	mutex_lock(&mce_sysfs_mutex);
 	if (mca_cfg.ignore_ce ^ !!new) {
 		if (new) {
 			/* disable ce features */
@@ -2085,6 +2104,8 @@ static ssize_t set_ignore_ce(struct device *s,
 			on_each_cpu(mce_enable_ce, (void *)1, 1);
 		}
 	}
+	mutex_unlock(&mce_sysfs_mutex);
+
 	return size;
 }
 
@@ -2097,6 +2118,7 @@ static ssize_t set_cmci_disabled(struct device *s,
 	if (kstrtou64(buf, 0, &new) < 0)
 		return -EINVAL;
 
+	mutex_lock(&mce_sysfs_mutex);
 	if (mca_cfg.cmci_disabled ^ !!new) {
 		if (new) {
 			/* disable cmci */
@@ -2108,6 +2130,8 @@ static ssize_t set_cmci_disabled(struct device *s,
 			on_each_cpu(mce_enable_ce, NULL, 1);
 		}
 	}
+	mutex_unlock(&mce_sysfs_mutex);
+
 	return size;
 }
 
@@ -2115,8 +2139,19 @@ static ssize_t store_int_with_restart(struct device *s,
 				      struct device_attribute *attr,
 				      const char *buf, size_t size)
 {
-	ssize_t ret = device_store_int(s, attr, buf, size);
+	unsigned long old_check_interval = check_interval;
+	ssize_t ret = device_store_ulong(s, attr, buf, size);
+
+	if (check_interval == old_check_interval)
+		return ret;
+
+	if (check_interval < 1)
+		check_interval = 1;
+
+	mutex_lock(&mce_sysfs_mutex);
 	mce_restart();
+	mutex_unlock(&mce_sysfs_mutex);
+
 	return ret;
 }
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 486f640b02ef..0f32ad242324 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -110,6 +110,20 @@ const char *smca_get_long_name(enum smca_bank_types t)
 }
 EXPORT_SYMBOL_GPL(smca_get_long_name);
 
+static enum smca_bank_types smca_get_bank_type(struct mce *m)
+{
+	struct smca_bank *b;
+
+	if (m->bank >= N_SMCA_BANK_TYPES)
+		return N_SMCA_BANK_TYPES;
+
+	b = &smca_banks[m->bank];
+	if (!b->hwid)
+		return N_SMCA_BANK_TYPES;
+
+	return b->hwid->bank_type;
+}
+
 static struct smca_hwid smca_hwid_mcatypes[] = {
 	/* { bank_type, hwid_mcatype, xec_bitmap } */
 
@@ -407,7 +421,9 @@ static void deferred_error_interrupt_enable(struct cpuinfo_x86 *c)
 	    (deferred_error_int_vector != amd_deferred_error_interrupt))
 		deferred_error_int_vector = amd_deferred_error_interrupt;
 
-	low = (low & ~MASK_DEF_INT_TYPE) | DEF_INT_TYPE_APIC;
+	if (!mce_flags.smca)
+		low = (low & ~MASK_DEF_INT_TYPE) | DEF_INT_TYPE_APIC;
+
 	wrmsr(MSR_CU_DEF_ERR, low, high);
 }
 
@@ -738,6 +754,17 @@ int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr)
 }
 EXPORT_SYMBOL_GPL(umc_normaddr_to_sysaddr);
 
+bool amd_mce_is_memory_error(struct mce *m)
+{
+	/* ErrCodeExt[20:16] */
+	u8 xec = (m->status >> 16) & 0x1f;
+
+	if (mce_flags.smca)
+		return smca_get_bank_type(m) == SMCA_UMC && xec == 0x0;
+
+	return m->bank == 4 && xec == 0x8;
+}
+
 static void __log_error(unsigned int bank, u64 status, u64 addr, u64 misc)
 {
 	struct mce m;
diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index 330b8462d426..48179928ff38 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -339,7 +339,7 @@ int __init save_microcode_in_initrd_amd(unsigned int cpuid_1_eax)
 		return -EINVAL;
 
 	ret = load_microcode_amd(true, x86_family(cpuid_1_eax), desc.data, desc.size);
-	if (ret != UCODE_OK)
+	if (ret > UCODE_UPDATED)
 		return -EINVAL;
 
 	return 0;
@@ -498,7 +498,7 @@ static unsigned int verify_patch_size(u8 family, u32 patch_size,
 	return patch_size;
 }
 
-static int apply_microcode_amd(int cpu)
+static enum ucode_state apply_microcode_amd(int cpu)
 {
 	struct cpuinfo_x86 *c = &cpu_data(cpu);
 	struct microcode_amd *mc_amd;
@@ -512,7 +512,7 @@ static int apply_microcode_amd(int cpu)
 
 	p = find_patch(cpu);
 	if (!p)
-		return 0;
+		return UCODE_NFOUND;
 
 	mc_amd  = p->data;
 	uci->mc = p->data;
@@ -523,13 +523,13 @@ static int apply_microcode_amd(int cpu)
 	if (rev >= mc_amd->hdr.patch_id) {
 		c->microcode = rev;
 		uci->cpu_sig.rev = rev;
-		return 0;
+		return UCODE_OK;
 	}
 
 	if (__apply_microcode_amd(mc_amd)) {
 		pr_err("CPU%d: update failed for patch_level=0x%08x\n",
 			cpu, mc_amd->hdr.patch_id);
-		return -1;
+		return UCODE_ERROR;
 	}
 	pr_info("CPU%d: new patch_level=0x%08x\n", cpu,
 		mc_amd->hdr.patch_id);
@@ -537,7 +537,7 @@ static int apply_microcode_amd(int cpu)
 	uci->cpu_sig.rev = mc_amd->hdr.patch_id;
 	c->microcode = mc_amd->hdr.patch_id;
 
-	return 0;
+	return UCODE_UPDATED;
 }
 
 static int install_equiv_cpu_table(const u8 *buf)
@@ -683,27 +683,35 @@ static enum ucode_state __load_microcode_amd(u8 family, const u8 *data,
 static enum ucode_state
 load_microcode_amd(bool save, u8 family, const u8 *data, size_t size)
 {
+	struct ucode_patch *p;
 	enum ucode_state ret;
 
 	/* free old equiv table */
 	free_equiv_cpu_table();
 
 	ret = __load_microcode_amd(family, data, size);
-
-	if (ret != UCODE_OK)
+	if (ret != UCODE_OK) {
 		cleanup();
+		return ret;
+	}
 
-#ifdef CONFIG_X86_32
-	/* save BSP's matching patch for early load */
-	if (save) {
-		struct ucode_patch *p = find_patch(0);
-		if (p) {
-			memset(amd_ucode_patch, 0, PATCH_MAX_SIZE);
-			memcpy(amd_ucode_patch, p->data, min_t(u32, ksize(p->data),
-							       PATCH_MAX_SIZE));
-		}
+	p = find_patch(0);
+	if (!p) {
+		return ret;
+	} else {
+		if (boot_cpu_data.microcode == p->patch_id)
+			return ret;
+
+		ret = UCODE_NEW;
 	}
-#endif
+
+	/* save BSP's matching patch for early load */
+	if (!save)
+		return ret;
+
+	memset(amd_ucode_patch, 0, PATCH_MAX_SIZE);
+	memcpy(amd_ucode_patch, p->data, min_t(u32, ksize(p->data), PATCH_MAX_SIZE));
+
 	return ret;
 }
 
diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
index e4fc595cd6ea..10c4fc2c91f8 100644
--- a/arch/x86/kernel/cpu/microcode/core.c
+++ b/arch/x86/kernel/cpu/microcode/core.c
@@ -22,13 +22,16 @@
 #define pr_fmt(fmt) "microcode: " fmt
 
 #include <linux/platform_device.h>
+#include <linux/stop_machine.h>
 #include <linux/syscore_ops.h>
 #include <linux/miscdevice.h>
 #include <linux/capability.h>
 #include <linux/firmware.h>
 #include <linux/kernel.h>
+#include <linux/delay.h>
 #include <linux/mutex.h>
 #include <linux/cpu.h>
+#include <linux/nmi.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 
@@ -64,6 +67,11 @@ LIST_HEAD(microcode_cache);
  */
 static DEFINE_MUTEX(microcode_mutex);
 
+/*
+ * Serialize late loading so that CPUs get updated one-by-one.
+ */
+static DEFINE_SPINLOCK(update_lock);
+
 struct ucode_cpu_info		ucode_cpu_info[NR_CPUS];
 
 struct cpu_info_ctx {
@@ -373,26 +381,23 @@ static int collect_cpu_info(int cpu)
 	return ret;
 }
 
-struct apply_microcode_ctx {
-	int err;
-};
-
 static void apply_microcode_local(void *arg)
 {
-	struct apply_microcode_ctx *ctx = arg;
+	enum ucode_state *err = arg;
 
-	ctx->err = microcode_ops->apply_microcode(smp_processor_id());
+	*err = microcode_ops->apply_microcode(smp_processor_id());
 }
 
 static int apply_microcode_on_target(int cpu)
 {
-	struct apply_microcode_ctx ctx = { .err = 0 };
+	enum ucode_state err;
 	int ret;
 
-	ret = smp_call_function_single(cpu, apply_microcode_local, &ctx, 1);
-	if (!ret)
-		ret = ctx.err;
-
+	ret = smp_call_function_single(cpu, apply_microcode_local, &err, 1);
+	if (!ret) {
+		if (err == UCODE_ERROR)
+			ret = 1;
+	}
 	return ret;
 }
 
@@ -489,31 +494,124 @@ static void __exit microcode_dev_exit(void)
 /* fake device for request_firmware */
 static struct platform_device	*microcode_pdev;
 
-static int reload_for_cpu(int cpu)
+/*
+ * Late loading dance. Why the heavy-handed stomp_machine effort?
+ *
+ * - HT siblings must be idle and not execute other code while the other sibling
+ *   is loading microcode in order to avoid any negative interactions caused by
+ *   the loading.
+ *
+ * - In addition, microcode update on the cores must be serialized until this
+ *   requirement can be relaxed in the future. Right now, this is conservative
+ *   and good.
+ */
+#define SPINUNIT 100 /* 100 nsec */
+
+static int check_online_cpus(void)
 {
-	struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
-	enum ucode_state ustate;
-	int err = 0;
+	if (num_online_cpus() == num_present_cpus())
+		return 0;
 
-	if (!uci->valid)
-		return err;
+	pr_err("Not all CPUs online, aborting microcode update.\n");
 
-	ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev, true);
-	if (ustate == UCODE_OK)
-		apply_microcode_on_target(cpu);
-	else
-		if (ustate == UCODE_ERROR)
-			err = -EINVAL;
-	return err;
+	return -EINVAL;
+}
+
+static atomic_t late_cpus_in;
+static atomic_t late_cpus_out;
+
+static int __wait_for_cpus(atomic_t *t, long long timeout)
+{
+	int all_cpus = num_online_cpus();
+
+	atomic_inc(t);
+
+	while (atomic_read(t) < all_cpus) {
+		if (timeout < SPINUNIT) {
+			pr_err("Timeout while waiting for CPUs rendezvous, remaining: %d\n",
+				all_cpus - atomic_read(t));
+			return 1;
+		}
+
+		ndelay(SPINUNIT);
+		timeout -= SPINUNIT;
+
+		touch_nmi_watchdog();
+	}
+	return 0;
+}
+
+/*
+ * Returns:
+ * < 0 - on error
+ *   0 - no update done
+ *   1 - microcode was updated
+ */
+static int __reload_late(void *info)
+{
+	int cpu = smp_processor_id();
+	enum ucode_state err;
+	int ret = 0;
+
+	/*
+	 * Wait for all CPUs to arrive. A load will not be attempted unless all
+	 * CPUs show up.
+	 * */
+	if (__wait_for_cpus(&late_cpus_in, NSEC_PER_SEC))
+		return -1;
+
+	spin_lock(&update_lock);
+	apply_microcode_local(&err);
+	spin_unlock(&update_lock);
+
+	if (err > UCODE_NFOUND) {
+		pr_warn("Error reloading microcode on CPU %d\n", cpu);
+		return -1;
+	/* siblings return UCODE_OK because their engine got updated already */
+	} else if (err == UCODE_UPDATED || err == UCODE_OK) {
+		ret = 1;
+	} else {
+		return ret;
+	}
+
+	/*
+	 * Increase the wait timeout to a safe value here since we're
+	 * serializing the microcode update and that could take a while on a
+	 * large number of CPUs. And that is fine as the *actual* timeout will
+	 * be determined by the last CPU finished updating and thus cut short.
+	 */
+	if (__wait_for_cpus(&late_cpus_out, NSEC_PER_SEC * num_online_cpus()))
+		panic("Timeout during microcode update!\n");
+
+	return ret;
+}
+
+/*
+ * Reload microcode late on all CPUs. Wait for a sec until they
+ * all gather together.
+ */
+static int microcode_reload_late(void)
+{
+	int ret;
+
+	atomic_set(&late_cpus_in,  0);
+	atomic_set(&late_cpus_out, 0);
+
+	ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask);
+	if (ret > 0)
+		microcode_check();
+
+	return ret;
 }
 
 static ssize_t reload_store(struct device *dev,
 			    struct device_attribute *attr,
 			    const char *buf, size_t size)
 {
+	enum ucode_state tmp_ret = UCODE_OK;
+	int bsp = boot_cpu_data.cpu_index;
 	unsigned long val;
-	int cpu;
-	ssize_t ret = 0, tmp_ret;
+	ssize_t ret = 0;
 
 	ret = kstrtoul(buf, 0, &val);
 	if (ret)
@@ -522,23 +620,24 @@ static ssize_t reload_store(struct device *dev,
 	if (val != 1)
 		return size;
 
+	tmp_ret = microcode_ops->request_microcode_fw(bsp, &microcode_pdev->dev, true);
+	if (tmp_ret != UCODE_NEW)
+		return size;
+
 	get_online_cpus();
-	mutex_lock(&microcode_mutex);
-	for_each_online_cpu(cpu) {
-		tmp_ret = reload_for_cpu(cpu);
-		if (tmp_ret != 0)
-			pr_warn("Error reloading microcode on CPU %d\n", cpu);
 
-		/* save retval of the first encountered reload error */
-		if (!ret)
-			ret = tmp_ret;
-	}
-	if (!ret)
-		perf_check_microcode();
+	ret = check_online_cpus();
+	if (ret)
+		goto put;
+
+	mutex_lock(&microcode_mutex);
+	ret = microcode_reload_late();
 	mutex_unlock(&microcode_mutex);
+
+put:
 	put_online_cpus();
 
-	if (!ret)
+	if (ret >= 0)
 		ret = size;
 
 	return ret;
@@ -560,7 +659,7 @@ static ssize_t pf_show(struct device *dev,
 	return sprintf(buf, "0x%x\n", uci->cpu_sig.pf);
 }
 
-static DEVICE_ATTR(reload, 0200, NULL, reload_store);
+static DEVICE_ATTR_WO(reload);
 static DEVICE_ATTR(version, 0400, version_show, NULL);
 static DEVICE_ATTR(processor_flags, 0400, pf_show, NULL);
 
@@ -606,10 +705,8 @@ static enum ucode_state microcode_init_cpu(int cpu, bool refresh_fw)
 	if (system_state != SYSTEM_RUNNING)
 		return UCODE_NFOUND;
 
-	ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev,
-						     refresh_fw);
-
-	if (ustate == UCODE_OK) {
+	ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev, refresh_fw);
+	if (ustate == UCODE_NEW) {
 		pr_debug("CPU%d updated upon init\n", cpu);
 		apply_microcode_on_target(cpu);
 	}
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index f7c55b0e753a..32b8e5724f96 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -589,6 +589,23 @@ static int apply_microcode_early(struct ucode_cpu_info *uci, bool early)
 	if (!mc)
 		return 0;
 
+	/*
+	 * Save us the MSR write below - which is a particular expensive
+	 * operation - when the other hyperthread has updated the microcode
+	 * already.
+	 */
+	rev = intel_get_microcode_revision();
+	if (rev >= mc->hdr.rev) {
+		uci->cpu_sig.rev = rev;
+		return UCODE_OK;
+	}
+
+	/*
+	 * Writeback and invalidate caches before updating microcode to avoid
+	 * internal issues depending on what the microcode is updating.
+	 */
+	native_wbinvd();
+
 	/* write microcode via MSR 0x79 */
 	native_wrmsrl(MSR_IA32_UCODE_WRITE, (unsigned long)mc->bits);
 
@@ -772,27 +789,44 @@ static int collect_cpu_info(int cpu_num, struct cpu_signature *csig)
 	return 0;
 }
 
-static int apply_microcode_intel(int cpu)
+static enum ucode_state apply_microcode_intel(int cpu)
 {
+	struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
+	struct cpuinfo_x86 *c = &cpu_data(cpu);
 	struct microcode_intel *mc;
-	struct ucode_cpu_info *uci;
-	struct cpuinfo_x86 *c;
 	static int prev_rev;
 	u32 rev;
 
 	/* We should bind the task to the CPU */
 	if (WARN_ON(raw_smp_processor_id() != cpu))
-		return -1;
+		return UCODE_ERROR;
 
-	uci = ucode_cpu_info + cpu;
-	mc = uci->mc;
+	/* Look for a newer patch in our cache: */
+	mc = find_patch(uci);
 	if (!mc) {
-		/* Look for a newer patch in our cache: */
-		mc = find_patch(uci);
+		mc = uci->mc;
 		if (!mc)
-			return 0;
+			return UCODE_NFOUND;
 	}
 
+	/*
+	 * Save us the MSR write below - which is a particular expensive
+	 * operation - when the other hyperthread has updated the microcode
+	 * already.
+	 */
+	rev = intel_get_microcode_revision();
+	if (rev >= mc->hdr.rev) {
+		uci->cpu_sig.rev = rev;
+		c->microcode = rev;
+		return UCODE_OK;
+	}
+
+	/*
+	 * Writeback and invalidate caches before updating microcode to avoid
+	 * internal issues depending on what the microcode is updating.
+	 */
+	native_wbinvd();
+
 	/* write microcode via MSR 0x79 */
 	wrmsrl(MSR_IA32_UCODE_WRITE, (unsigned long)mc->bits);
 
@@ -801,7 +835,7 @@ static int apply_microcode_intel(int cpu)
 	if (rev != mc->hdr.rev) {
 		pr_err("CPU%d update to revision 0x%x failed\n",
 		       cpu, mc->hdr.rev);
-		return -1;
+		return UCODE_ERROR;
 	}
 
 	if (rev != prev_rev) {
@@ -813,12 +847,10 @@ static int apply_microcode_intel(int cpu)
 		prev_rev = rev;
 	}
 
-	c = &cpu_data(cpu);
-
 	uci->cpu_sig.rev = rev;
 	c->microcode = rev;
 
-	return 0;
+	return UCODE_UPDATED;
 }
 
 static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
@@ -830,6 +862,7 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
 	unsigned int leftover = size;
 	unsigned int curr_mc_size = 0, new_mc_size = 0;
 	unsigned int csig, cpf;
+	enum ucode_state ret = UCODE_OK;
 
 	while (leftover) {
 		struct microcode_header_intel mc_header;
@@ -871,6 +904,7 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
 			new_mc  = mc;
 			new_mc_size = mc_size;
 			mc = NULL;	/* trigger new vmalloc */
+			ret = UCODE_NEW;
 		}
 
 		ucode_ptr += mc_size;
@@ -900,7 +934,7 @@ static enum ucode_state generic_load_microcode(int cpu, void *data, size_t size,
 	pr_debug("CPU%d found a matching microcode update with version 0x%x (current=0x%x)\n",
 		 cpu, new_rev, uci->cpu_sig.rev);
 
-	return UCODE_OK;
+	return ret;
 }
 
 static int get_ucode_fw(void *to, const void *from, size_t n)
@@ -921,7 +955,7 @@ static bool is_blacklisted(unsigned int cpu)
 	 */
 	if (c->x86 == 6 &&
 	    c->x86_model == INTEL_FAM6_BROADWELL_X &&
-	    c->x86_mask == 0x01 &&
+	    c->x86_stepping == 0x01 &&
 	    llc_size_per_core > 2621440 &&
 	    c->microcode < 0x0b000021) {
 		pr_err_once("Erratum BDF90: late loading with revision < 0x0b000021 (0x%x) disabled.\n", c->microcode);
@@ -944,7 +978,7 @@ static enum ucode_state request_microcode_fw(int cpu, struct device *device,
 		return UCODE_NFOUND;
 
 	sprintf(name, "intel-ucode/%02x-%02x-%02x",
-		c->x86, c->x86_model, c->x86_mask);
+		c->x86, c->x86_model, c->x86_stepping);
 
 	if (request_firmware_direct(&firmware, name, device)) {
 		pr_debug("data file %s load failed\n", name);
@@ -982,7 +1016,7 @@ static struct microcode_ops microcode_intel_ops = {
 
 static int __init calc_llc_size_per_core(struct cpuinfo_x86 *c)
 {
-	u64 llc_size = c->x86_cache_size * 1024;
+	u64 llc_size = c->x86_cache_size * 1024ULL;
 
 	do_div(llc_size, c->x86_max_cores);
 
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 85eb5fc180c8..9340f41ce8d3 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -251,6 +251,12 @@ static void __init ms_hyperv_init_platform(void)
 	hyperv_setup_mmu_ops();
 	/* Setup the IDT for hypervisor callback */
 	alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, hyperv_callback_vector);
+
+	/* Setup the IDT for reenlightenment notifications */
+	if (ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT)
+		alloc_intr_gate(HYPERV_REENLIGHTENMENT_VECTOR,
+				hyperv_reenlightenment_vector);
+
 #endif
 }
 
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
index fdc55215d44d..e12ee86906c6 100644
--- a/arch/x86/kernel/cpu/mtrr/generic.c
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
@@ -859,7 +859,7 @@ int generic_validate_add_page(unsigned long base, unsigned long size,
 	 */
 	if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
 	    boot_cpu_data.x86_model == 1 &&
-	    boot_cpu_data.x86_mask <= 7) {
+	    boot_cpu_data.x86_stepping <= 7) {
 		if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
 			pr_warn("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
 			return -EINVAL;
diff --git a/arch/x86/kernel/cpu/mtrr/main.c b/arch/x86/kernel/cpu/mtrr/main.c
index 40d5a8a75212..7468de429087 100644
--- a/arch/x86/kernel/cpu/mtrr/main.c
+++ b/arch/x86/kernel/cpu/mtrr/main.c
@@ -711,8 +711,8 @@ void __init mtrr_bp_init(void)
 			if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
 			    boot_cpu_data.x86 == 0xF &&
 			    boot_cpu_data.x86_model == 0x3 &&
-			    (boot_cpu_data.x86_mask == 0x3 ||
-			     boot_cpu_data.x86_mask == 0x4))
+			    (boot_cpu_data.x86_stepping == 0x3 ||
+			     boot_cpu_data.x86_stepping == 0x4))
 				phys_addr = 36;
 
 			size_or_mask = SIZE_OR_MASK_BITS(phys_addr);
diff --git a/arch/x86/kernel/cpu/proc.c b/arch/x86/kernel/cpu/proc.c
index e7ecedafa1c8..2c8522a39ed5 100644
--- a/arch/x86/kernel/cpu/proc.c
+++ b/arch/x86/kernel/cpu/proc.c
@@ -72,8 +72,8 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 		   c->x86_model,
 		   c->x86_model_id[0] ? c->x86_model_id : "unknown");
 
-	if (c->x86_mask || c->cpuid_level >= 0)
-		seq_printf(m, "stepping\t: %d\n", c->x86_mask);
+	if (c->x86_stepping || c->cpuid_level >= 0)
+		seq_printf(m, "stepping\t: %d\n", c->x86_stepping);
 	else
 		seq_puts(m, "stepping\t: unknown\n");
 	if (c->microcode)
@@ -91,8 +91,8 @@ static int show_cpuinfo(struct seq_file *m, void *v)
 	}
 
 	/* Cache size */
-	if (c->x86_cache_size >= 0)
-		seq_printf(m, "cache size\t: %d KB\n", c->x86_cache_size);
+	if (c->x86_cache_size)
+		seq_printf(m, "cache size\t: %u KB\n", c->x86_cache_size);
 
 	show_cpuinfo_core(m, c, cpu);
 	show_cpuinfo_misc(m, c);
diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index d0e69769abfd..772c219b6889 100644
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -21,16 +21,16 @@ struct cpuid_bit {
 static const struct cpuid_bit cpuid_bits[] = {
 	{ X86_FEATURE_APERFMPERF,       CPUID_ECX,  0, 0x00000006, 0 },
 	{ X86_FEATURE_EPB,		CPUID_ECX,  3, 0x00000006, 0 },
-	{ X86_FEATURE_AVX512_4VNNIW,    CPUID_EDX,  2, 0x00000007, 0 },
-	{ X86_FEATURE_AVX512_4FMAPS,    CPUID_EDX,  3, 0x00000007, 0 },
 	{ X86_FEATURE_CAT_L3,		CPUID_EBX,  1, 0x00000010, 0 },
 	{ X86_FEATURE_CAT_L2,		CPUID_EBX,  2, 0x00000010, 0 },
 	{ X86_FEATURE_CDP_L3,		CPUID_ECX,  2, 0x00000010, 1 },
+	{ X86_FEATURE_CDP_L2,		CPUID_ECX,  2, 0x00000010, 2 },
 	{ X86_FEATURE_MBA,		CPUID_EBX,  3, 0x00000010, 0 },
 	{ X86_FEATURE_HW_PSTATE,	CPUID_EDX,  7, 0x80000007, 0 },
 	{ X86_FEATURE_CPB,		CPUID_EDX,  9, 0x80000007, 0 },
 	{ X86_FEATURE_PROC_FEEDBACK,    CPUID_EDX, 11, 0x80000007, 0 },
 	{ X86_FEATURE_SME,		CPUID_EAX,  0, 0x8000001f, 0 },
+	{ X86_FEATURE_SEV,		CPUID_EAX,  1, 0x8000001f, 0 },
 	{ 0, 0, 0, 0, 0 }
 };
 
diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c
index 76e07698e6d1..25de5f6ca997 100644
--- a/arch/x86/kernel/devicetree.c
+++ b/arch/x86/kernel/devicetree.c
@@ -2,7 +2,6 @@
 /*
  * Architecture specific OF callbacks.
  */
-#include <linux/bootmem.h>
 #include <linux/export.h>
 #include <linux/io.h>
 #include <linux/interrupt.h>
@@ -39,11 +38,6 @@ void __init early_init_dt_add_memory_arch(u64 base, u64 size)
 	BUG();
 }
 
-void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
-{
-	return __alloc_bootmem(size, align, __pa(MAX_DMA_ADDRESS));
-}
-
 void __init add_dtb(u64 data)
 {
 	initial_dtb = data + offsetof(struct setup_data, data);
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index afbecff161d1..a2d8a3908670 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -109,7 +109,7 @@ void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs,
 	struct stack_info stack_info = {0};
 	unsigned long visit_mask = 0;
 	int graph_idx = 0;
-	bool partial;
+	bool partial = false;
 
 	printk("%sCall Trace:\n", log_lvl);
 
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index 1e82f787c160..bae0d32e327b 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -243,7 +243,7 @@ static void __init intel_remapping_check(int num, int slot, int func)
 #define KB(x)	((x) * 1024UL)
 #define MB(x)	(KB (KB (x)))
 
-static size_t __init i830_tseg_size(void)
+static resource_size_t __init i830_tseg_size(void)
 {
 	u8 esmramc = read_pci_config_byte(0, 0, 0, I830_ESMRAMC);
 
@@ -256,7 +256,7 @@ static size_t __init i830_tseg_size(void)
 		return KB(512);
 }
 
-static size_t __init i845_tseg_size(void)
+static resource_size_t __init i845_tseg_size(void)
 {
 	u8 esmramc = read_pci_config_byte(0, 0, 0, I845_ESMRAMC);
 	u8 tseg_size = esmramc & I845_TSEG_SIZE_MASK;
@@ -273,7 +273,7 @@ static size_t __init i845_tseg_size(void)
 	return 0;
 }
 
-static size_t __init i85x_tseg_size(void)
+static resource_size_t __init i85x_tseg_size(void)
 {
 	u8 esmramc = read_pci_config_byte(0, 0, 0, I85X_ESMRAMC);
 
@@ -283,12 +283,12 @@ static size_t __init i85x_tseg_size(void)
 	return MB(1);
 }
 
-static size_t __init i830_mem_size(void)
+static resource_size_t __init i830_mem_size(void)
 {
 	return read_pci_config_byte(0, 0, 0, I830_DRB3) * MB(32);
 }
 
-static size_t __init i85x_mem_size(void)
+static resource_size_t __init i85x_mem_size(void)
 {
 	return read_pci_config_byte(0, 0, 1, I85X_DRB3) * MB(32);
 }
@@ -297,36 +297,36 @@ static size_t __init i85x_mem_size(void)
  * On 830/845/85x the stolen memory base isn't available in any
  * register. We need to calculate it as TOM-TSEG_SIZE-stolen_size.
  */
-static phys_addr_t __init i830_stolen_base(int num, int slot, int func,
-					   size_t stolen_size)
+static resource_size_t __init i830_stolen_base(int num, int slot, int func,
+					       resource_size_t stolen_size)
 {
-	return (phys_addr_t)i830_mem_size() - i830_tseg_size() - stolen_size;
+	return i830_mem_size() - i830_tseg_size() - stolen_size;
 }
 
-static phys_addr_t __init i845_stolen_base(int num, int slot, int func,
-					   size_t stolen_size)
+static resource_size_t __init i845_stolen_base(int num, int slot, int func,
+					       resource_size_t stolen_size)
 {
-	return (phys_addr_t)i830_mem_size() - i845_tseg_size() - stolen_size;
+	return i830_mem_size() - i845_tseg_size() - stolen_size;
 }
 
-static phys_addr_t __init i85x_stolen_base(int num, int slot, int func,
-					   size_t stolen_size)
+static resource_size_t __init i85x_stolen_base(int num, int slot, int func,
+					       resource_size_t stolen_size)
 {
-	return (phys_addr_t)i85x_mem_size() - i85x_tseg_size() - stolen_size;
+	return i85x_mem_size() - i85x_tseg_size() - stolen_size;
 }
 
-static phys_addr_t __init i865_stolen_base(int num, int slot, int func,
-					   size_t stolen_size)
+static resource_size_t __init i865_stolen_base(int num, int slot, int func,
+					       resource_size_t stolen_size)
 {
 	u16 toud = 0;
 
 	toud = read_pci_config_16(0, 0, 0, I865_TOUD);
 
-	return (phys_addr_t)(toud << 16) + i845_tseg_size();
+	return toud * KB(64) + i845_tseg_size();
 }
 
-static phys_addr_t __init gen3_stolen_base(int num, int slot, int func,
-					   size_t stolen_size)
+static resource_size_t __init gen3_stolen_base(int num, int slot, int func,
+					       resource_size_t stolen_size)
 {
 	u32 bsm;
 
@@ -337,10 +337,10 @@ static phys_addr_t __init gen3_stolen_base(int num, int slot, int func,
 	 */
 	bsm = read_pci_config(num, slot, func, INTEL_BSM);
 
-	return (phys_addr_t)bsm & INTEL_BSM_MASK;
+	return bsm & INTEL_BSM_MASK;
 }
 
-static size_t __init i830_stolen_size(int num, int slot, int func)
+static resource_size_t __init i830_stolen_size(int num, int slot, int func)
 {
 	u16 gmch_ctrl;
 	u16 gms;
@@ -361,7 +361,7 @@ static size_t __init i830_stolen_size(int num, int slot, int func)
 	return 0;
 }
 
-static size_t __init gen3_stolen_size(int num, int slot, int func)
+static resource_size_t __init gen3_stolen_size(int num, int slot, int func)
 {
 	u16 gmch_ctrl;
 	u16 gms;
@@ -390,7 +390,7 @@ static size_t __init gen3_stolen_size(int num, int slot, int func)
 	return 0;
 }
 
-static size_t __init gen6_stolen_size(int num, int slot, int func)
+static resource_size_t __init gen6_stolen_size(int num, int slot, int func)
 {
 	u16 gmch_ctrl;
 	u16 gms;
@@ -398,10 +398,10 @@ static size_t __init gen6_stolen_size(int num, int slot, int func)
 	gmch_ctrl = read_pci_config_16(num, slot, func, SNB_GMCH_CTRL);
 	gms = (gmch_ctrl >> SNB_GMCH_GMS_SHIFT) & SNB_GMCH_GMS_MASK;
 
-	return (size_t)gms * MB(32);
+	return gms * MB(32);
 }
 
-static size_t __init gen8_stolen_size(int num, int slot, int func)
+static resource_size_t __init gen8_stolen_size(int num, int slot, int func)
 {
 	u16 gmch_ctrl;
 	u16 gms;
@@ -409,10 +409,10 @@ static size_t __init gen8_stolen_size(int num, int slot, int func)
 	gmch_ctrl = read_pci_config_16(num, slot, func, SNB_GMCH_CTRL);
 	gms = (gmch_ctrl >> BDW_GMCH_GMS_SHIFT) & BDW_GMCH_GMS_MASK;
 
-	return (size_t)gms * MB(32);
+	return gms * MB(32);
 }
 
-static size_t __init chv_stolen_size(int num, int slot, int func)
+static resource_size_t __init chv_stolen_size(int num, int slot, int func)
 {
 	u16 gmch_ctrl;
 	u16 gms;
@@ -426,14 +426,14 @@ static size_t __init chv_stolen_size(int num, int slot, int func)
 	 * 0x17 to 0x1d: 4MB increments start at 36MB
 	 */
 	if (gms < 0x11)
-		return (size_t)gms * MB(32);
+		return gms * MB(32);
 	else if (gms < 0x17)
-		return (size_t)(gms - 0x11 + 2) * MB(4);
+		return (gms - 0x11) * MB(4) + MB(8);
 	else
-		return (size_t)(gms - 0x17 + 9) * MB(4);
+		return (gms - 0x17) * MB(4) + MB(36);
 }
 
-static size_t __init gen9_stolen_size(int num, int slot, int func)
+static resource_size_t __init gen9_stolen_size(int num, int slot, int func)
 {
 	u16 gmch_ctrl;
 	u16 gms;
@@ -444,14 +444,15 @@ static size_t __init gen9_stolen_size(int num, int slot, int func)
 	/* 0x0  to 0xef: 32MB increments starting at 0MB */
 	/* 0xf0 to 0xfe: 4MB increments starting at 4MB */
 	if (gms < 0xf0)
-		return (size_t)gms * MB(32);
+		return gms * MB(32);
 	else
-		return (size_t)(gms - 0xf0 + 1) * MB(4);
+		return (gms - 0xf0) * MB(4) + MB(4);
 }
 
 struct intel_early_ops {
-	size_t (*stolen_size)(int num, int slot, int func);
-	phys_addr_t (*stolen_base)(int num, int slot, int func, size_t size);
+	resource_size_t (*stolen_size)(int num, int slot, int func);
+	resource_size_t (*stolen_base)(int num, int slot, int func,
+				       resource_size_t size);
 };
 
 static const struct intel_early_ops i830_early_ops __initconst = {
@@ -527,16 +528,20 @@ static const struct pci_device_id intel_early_ids[] __initconst = {
 	INTEL_SKL_IDS(&gen9_early_ops),
 	INTEL_BXT_IDS(&gen9_early_ops),
 	INTEL_KBL_IDS(&gen9_early_ops),
+	INTEL_CFL_IDS(&gen9_early_ops),
 	INTEL_GLK_IDS(&gen9_early_ops),
 	INTEL_CNL_IDS(&gen9_early_ops),
 };
 
+struct resource intel_graphics_stolen_res __ro_after_init = DEFINE_RES_MEM(0, 0);
+EXPORT_SYMBOL(intel_graphics_stolen_res);
+
 static void __init
 intel_graphics_stolen(int num, int slot, int func,
 		      const struct intel_early_ops *early_ops)
 {
-	phys_addr_t base, end;
-	size_t size;
+	resource_size_t base, size;
+	resource_size_t end;
 
 	size = early_ops->stolen_size(num, slot, func);
 	base = early_ops->stolen_base(num, slot, func, size);
@@ -545,8 +550,12 @@ intel_graphics_stolen(int num, int slot, int func,
 		return;
 
 	end = base + size - 1;
-	printk(KERN_INFO "Reserving Intel graphics memory at %pa-%pa\n",
-	       &base, &end);
+
+	intel_graphics_stolen_res.start = base;
+	intel_graphics_stolen_res.end = end;
+
+	printk(KERN_INFO "Reserving Intel graphics memory at %pR\n",
+	       &intel_graphics_stolen_res);
 
 	/* Mark this space as reserved */
 	e820__range_add(base, size, E820_TYPE_RESERVED);
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index c29020907886..b59e4fb40fd9 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -37,7 +37,7 @@
 #define X86		new_cpu_data+CPUINFO_x86
 #define X86_VENDOR	new_cpu_data+CPUINFO_x86_vendor
 #define X86_MODEL	new_cpu_data+CPUINFO_x86_model
-#define X86_MASK	new_cpu_data+CPUINFO_x86_mask
+#define X86_STEPPING	new_cpu_data+CPUINFO_x86_stepping
 #define X86_HARD_MATH	new_cpu_data+CPUINFO_hard_math
 #define X86_CPUID	new_cpu_data+CPUINFO_cpuid_level
 #define X86_CAPABILITY	new_cpu_data+CPUINFO_x86_capability
@@ -332,7 +332,7 @@ ENTRY(startup_32_smp)
 	shrb $4,%al
 	movb %al,X86_MODEL
 	andb $0x0f,%cl		# mask mask revision
-	movb %cl,X86_MASK
+	movb %cl,X86_STEPPING
 	movl %edx,X86_CAPABILITY
 
 .Lis486:
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 04a625f0fcda..0f545b3cf926 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -23,6 +23,7 @@
 #include <asm/nops.h>
 #include "../entry/calling.h"
 #include <asm/export.h>
+#include <asm/nospec-branch.h>
 
 #ifdef CONFIG_PARAVIRT
 #include <asm/asm-offsets.h>
@@ -134,6 +135,7 @@ ENTRY(secondary_startup_64)
 
 	/* Ensure I am executing from virtual addresses */
 	movq	$1f, %rax
+	ANNOTATE_RETPOLINE_SAFE
 	jmp	*%rax
 1:
 	UNWIND_HINT_EMPTY
diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index 56d99be3706a..2c3a1b4294eb 100644
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -140,6 +140,9 @@ static const __initconst struct idt_data apic_idts[] = {
 # ifdef CONFIG_IRQ_WORK
 	INTG(IRQ_WORK_VECTOR,		irq_work_interrupt),
 # endif
+#ifdef CONFIG_X86_UV
+	INTG(UV_BAU_MESSAGE,		uv_bau_message_intr1),
+#endif
 	INTG(SPURIOUS_APIC_VECTOR,	spurious_interrupt),
 	INTG(ERROR_APIC_VECTOR,		error_interrupt),
 #endif
@@ -160,7 +163,6 @@ static const __initconst struct idt_data early_pf_idts[] = {
  */
 static const __initconst struct idt_data dbg_idts[] = {
 	INTG(X86_TRAP_DB,	debug),
-	INTG(X86_TRAP_BP,	int3),
 };
 #endif
 
@@ -183,7 +185,6 @@ gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss;
 static const __initconst struct idt_data ist_idts[] = {
 	ISTG(X86_TRAP_DB,	debug,		DEBUG_STACK),
 	ISTG(X86_TRAP_NMI,	nmi,		NMI_STACK),
-	SISTG(X86_TRAP_BP,	int3,		DEBUG_STACK),
 	ISTG(X86_TRAP_DF,	double_fault,	DOUBLEFAULT_STACK),
 #ifdef CONFIG_X86_MCE
 	ISTG(X86_TRAP_MC,	&machine_check,	MCE_STACK),
diff --git a/arch/x86/kernel/ioport.c b/arch/x86/kernel/ioport.c
index 2f723301eb58..38deafebb21b 100644
--- a/arch/x86/kernel/ioport.c
+++ b/arch/x86/kernel/ioport.c
@@ -23,7 +23,7 @@
 /*
  * this changes the io permissions bitmap in the current task.
  */
-asmlinkage long sys_ioperm(unsigned long from, unsigned long num, int turn_on)
+SYSCALL_DEFINE3(ioperm, unsigned long, from, unsigned long, num, int, turn_on)
 {
 	struct thread_struct *t = &current->thread;
 	struct tss_struct *tss;
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 68e1867cca80..45fb4d2565f8 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -141,6 +141,15 @@ int arch_show_interrupts(struct seq_file *p, int prec)
 				   irq_stats(j)->irq_hv_callback_count);
 		seq_puts(p, "  Hypervisor callback interrupts\n");
 	}
+#endif
+#if IS_ENABLED(CONFIG_HYPERV)
+	if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) {
+		seq_printf(p, "%*s: ", prec, "HRE");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				   irq_stats(j)->irq_hv_reenlightenment_count);
+		seq_puts(p, "  Hyper-V reenlightenment interrupts\n");
+	}
 #endif
 	seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
 #if defined(CONFIG_X86_IO_APIC)
diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c
index f73f475d0573..d177940aa090 100644
--- a/arch/x86/kernel/itmt.c
+++ b/arch/x86/kernel/itmt.c
@@ -24,7 +24,6 @@
 #include <linux/cpumask.h>
 #include <linux/cpuset.h>
 #include <linux/mutex.h>
-#include <linux/sched.h>
 #include <linux/sysctl.h>
 #include <linux/nodemask.h>
 
diff --git a/arch/x86/kernel/jailhouse.c b/arch/x86/kernel/jailhouse.c
new file mode 100644
index 000000000000..b68fd895235a
--- /dev/null
+++ b/arch/x86/kernel/jailhouse.c
@@ -0,0 +1,211 @@
+// SPDX-License-Identifier: GPL2.0
+/*
+ * Jailhouse paravirt_ops implementation
+ *
+ * Copyright (c) Siemens AG, 2015-2017
+ *
+ * Authors:
+ *  Jan Kiszka <jan.kiszka@siemens.com>
+ */
+
+#include <linux/acpi_pmtmr.h>
+#include <linux/kernel.h>
+#include <linux/reboot.h>
+#include <asm/apic.h>
+#include <asm/cpu.h>
+#include <asm/hypervisor.h>
+#include <asm/i8259.h>
+#include <asm/irqdomain.h>
+#include <asm/pci_x86.h>
+#include <asm/reboot.h>
+#include <asm/setup.h>
+
+static __initdata struct jailhouse_setup_data setup_data;
+static unsigned int precalibrated_tsc_khz;
+
+static uint32_t jailhouse_cpuid_base(void)
+{
+	if (boot_cpu_data.cpuid_level < 0 ||
+	    !boot_cpu_has(X86_FEATURE_HYPERVISOR))
+		return 0;
+
+	return hypervisor_cpuid_base("Jailhouse\0\0\0", 0);
+}
+
+static uint32_t __init jailhouse_detect(void)
+{
+	return jailhouse_cpuid_base();
+}
+
+static void jailhouse_get_wallclock(struct timespec *now)
+{
+	memset(now, 0, sizeof(*now));
+}
+
+static void __init jailhouse_timer_init(void)
+{
+	lapic_timer_frequency = setup_data.apic_khz * (1000 / HZ);
+}
+
+static unsigned long jailhouse_get_tsc(void)
+{
+	return precalibrated_tsc_khz;
+}
+
+static void __init jailhouse_x2apic_init(void)
+{
+#ifdef CONFIG_X86_X2APIC
+	if (!x2apic_enabled())
+		return;
+	/*
+	 * We do not have access to IR inside Jailhouse non-root cells.  So
+	 * we have to run in physical mode.
+	 */
+	x2apic_phys = 1;
+	/*
+	 * This will trigger the switch to apic_x2apic_phys.  Empty OEM IDs
+	 * ensure that only this APIC driver picks up the call.
+	 */
+	default_acpi_madt_oem_check("", "");
+#endif
+}
+
+static void __init jailhouse_get_smp_config(unsigned int early)
+{
+	struct ioapic_domain_cfg ioapic_cfg = {
+		.type = IOAPIC_DOMAIN_STRICT,
+		.ops = &mp_ioapic_irqdomain_ops,
+	};
+	struct mpc_intsrc mp_irq = {
+		.type = MP_INTSRC,
+		.irqtype = mp_INT,
+		.irqflag = MP_IRQPOL_ACTIVE_HIGH | MP_IRQTRIG_EDGE,
+	};
+	unsigned int cpu;
+
+	jailhouse_x2apic_init();
+
+	register_lapic_address(0xfee00000);
+
+	for (cpu = 0; cpu < setup_data.num_cpus; cpu++) {
+		generic_processor_info(setup_data.cpu_ids[cpu],
+				       boot_cpu_apic_version);
+	}
+
+	smp_found_config = 1;
+
+	if (setup_data.standard_ioapic) {
+		mp_register_ioapic(0, 0xfec00000, gsi_top, &ioapic_cfg);
+
+		/* Register 1:1 mapping for legacy UART IRQs 3 and 4 */
+		mp_irq.srcbusirq = mp_irq.dstirq = 3;
+		mp_save_irq(&mp_irq);
+
+		mp_irq.srcbusirq = mp_irq.dstirq = 4;
+		mp_save_irq(&mp_irq);
+	}
+}
+
+static void jailhouse_no_restart(void)
+{
+	pr_notice("Jailhouse: Restart not supported, halting\n");
+	machine_halt();
+}
+
+static int __init jailhouse_pci_arch_init(void)
+{
+	pci_direct_init(1);
+
+	/*
+	 * There are no bridges on the virtual PCI root bus under Jailhouse,
+	 * thus no other way to discover all devices than a full scan.
+	 * Respect any overrides via the command line, though.
+	 */
+	if (pcibios_last_bus < 0)
+		pcibios_last_bus = 0xff;
+
+	return 0;
+}
+
+static void __init jailhouse_init_platform(void)
+{
+	u64 pa_data = boot_params.hdr.setup_data;
+	struct setup_data header;
+	void *mapping;
+
+	x86_init.irqs.pre_vector_init	= x86_init_noop;
+	x86_init.timers.timer_init	= jailhouse_timer_init;
+	x86_init.mpparse.get_smp_config	= jailhouse_get_smp_config;
+	x86_init.pci.arch_init		= jailhouse_pci_arch_init;
+
+	x86_platform.calibrate_cpu	= jailhouse_get_tsc;
+	x86_platform.calibrate_tsc	= jailhouse_get_tsc;
+	x86_platform.get_wallclock	= jailhouse_get_wallclock;
+	x86_platform.legacy.rtc		= 0;
+	x86_platform.legacy.warm_reset	= 0;
+	x86_platform.legacy.i8042	= X86_LEGACY_I8042_PLATFORM_ABSENT;
+
+	legacy_pic			= &null_legacy_pic;
+
+	machine_ops.emergency_restart	= jailhouse_no_restart;
+
+	while (pa_data) {
+		mapping = early_memremap(pa_data, sizeof(header));
+		memcpy(&header, mapping, sizeof(header));
+		early_memunmap(mapping, sizeof(header));
+
+		if (header.type == SETUP_JAILHOUSE &&
+		    header.len >= sizeof(setup_data)) {
+			pa_data += offsetof(struct setup_data, data);
+
+			mapping = early_memremap(pa_data, sizeof(setup_data));
+			memcpy(&setup_data, mapping, sizeof(setup_data));
+			early_memunmap(mapping, sizeof(setup_data));
+
+			break;
+		}
+
+		pa_data = header.next;
+	}
+
+	if (!pa_data)
+		panic("Jailhouse: No valid setup data found");
+
+	if (setup_data.compatible_version > JAILHOUSE_SETUP_REQUIRED_VERSION)
+		panic("Jailhouse: Unsupported setup data structure");
+
+	pmtmr_ioport = setup_data.pm_timer_address;
+	pr_debug("Jailhouse: PM-Timer IO Port: %#x\n", pmtmr_ioport);
+
+	precalibrated_tsc_khz = setup_data.tsc_khz;
+	setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
+
+	pci_probe = 0;
+
+	/*
+	 * Avoid that the kernel complains about missing ACPI tables - there
+	 * are none in a non-root cell.
+	 */
+	disable_acpi();
+}
+
+bool jailhouse_paravirt(void)
+{
+	return jailhouse_cpuid_base() != 0;
+}
+
+static bool jailhouse_x2apic_available(void)
+{
+	/*
+	 * The x2APIC is only available if the root cell enabled it. Jailhouse
+	 * does not support switching between xAPIC and x2APIC.
+	 */
+	return x2apic_enabled();
+}
+
+const struct hypervisor_x86 x86_hyper_jailhouse __refconst = {
+	.name			= "Jailhouse",
+	.detect			= jailhouse_detect,
+	.init.init_platform	= jailhouse_init_platform,
+	.init.x2apic_available	= jailhouse_x2apic_available,
+};
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index bd36f3c33cd0..0715f827607c 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -1168,10 +1168,18 @@ NOKPROBE_SYMBOL(longjmp_break_handler);
 
 bool arch_within_kprobe_blacklist(unsigned long addr)
 {
+	bool is_in_entry_trampoline_section = false;
+
+#ifdef CONFIG_X86_64
+	is_in_entry_trampoline_section =
+		(addr >= (unsigned long)__entry_trampoline_start &&
+		 addr < (unsigned long)__entry_trampoline_end);
+#endif
 	return  (addr >= (unsigned long)__kprobes_text_start &&
 		 addr < (unsigned long)__kprobes_text_end) ||
 		(addr >= (unsigned long)__entry_text_start &&
-		 addr < (unsigned long)__entry_text_end);
+		 addr < (unsigned long)__entry_text_end) ||
+		is_in_entry_trampoline_section;
 }
 
 int __init arch_init_kprobes(void)
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index b40ffbf156c1..fae86e36e399 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -49,7 +49,7 @@
 
 static int kvmapf = 1;
 
-static int parse_no_kvmapf(char *arg)
+static int __init parse_no_kvmapf(char *arg)
 {
         kvmapf = 0;
         return 0;
@@ -58,7 +58,7 @@ static int parse_no_kvmapf(char *arg)
 early_param("no-kvmapf", parse_no_kvmapf);
 
 static int steal_acc = 1;
-static int parse_no_stealacc(char *arg)
+static int __init parse_no_stealacc(char *arg)
 {
         steal_acc = 0;
         return 0;
@@ -67,7 +67,7 @@ static int parse_no_stealacc(char *arg)
 early_param("no-steal-acc", parse_no_stealacc);
 
 static int kvmclock_vsyscall = 1;
-static int parse_no_kvmclock_vsyscall(char *arg)
+static int __init parse_no_kvmclock_vsyscall(char *arg)
 {
         kvmclock_vsyscall = 0;
         return 0;
@@ -341,10 +341,10 @@ static void kvm_guest_cpu_init(void)
 #endif
 		pa |= KVM_ASYNC_PF_ENABLED;
 
-		/* Async page fault support for L1 hypervisor is optional */
-		if (wrmsr_safe(MSR_KVM_ASYNC_PF_EN,
-			(pa | KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT) & 0xffffffff, pa >> 32) < 0)
-			wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
+		if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT))
+			pa |= KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
+
+		wrmsrl(MSR_KVM_ASYNC_PF_EN, pa);
 		__this_cpu_write(apf_reason.enabled, 1);
 		printk(KERN_INFO"KVM setup async PF for cpu %d\n",
 		       smp_processor_id());
@@ -498,6 +498,34 @@ static void __init kvm_apf_trap_init(void)
 	update_intr_gate(X86_TRAP_PF, async_page_fault);
 }
 
+static DEFINE_PER_CPU(cpumask_var_t, __pv_tlb_mask);
+
+static void kvm_flush_tlb_others(const struct cpumask *cpumask,
+			const struct flush_tlb_info *info)
+{
+	u8 state;
+	int cpu;
+	struct kvm_steal_time *src;
+	struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_tlb_mask);
+
+	cpumask_copy(flushmask, cpumask);
+	/*
+	 * We have to call flush only on online vCPUs. And
+	 * queue flush_on_enter for pre-empted vCPUs
+	 */
+	for_each_cpu(cpu, flushmask) {
+		src = &per_cpu(steal_time, cpu);
+		state = READ_ONCE(src->preempted);
+		if ((state & KVM_VCPU_PREEMPTED)) {
+			if (try_cmpxchg(&src->preempted, &state,
+					state | KVM_VCPU_FLUSH_TLB))
+				__cpumask_clear_cpu(cpu, flushmask);
+		}
+	}
+
+	native_flush_tlb_others(flushmask, info);
+}
+
 static void __init kvm_guest_init(void)
 {
 	int i;
@@ -517,6 +545,10 @@ static void __init kvm_guest_init(void)
 		pv_time_ops.steal_clock = kvm_steal_clock;
 	}
 
+	if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
+	    kvm_para_has_feature(KVM_FEATURE_STEAL_TIME))
+		pv_mmu_ops.flush_tlb_others = kvm_flush_tlb_others;
+
 	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
 		apic_set_eoi_write(kvm_guest_apic_eoi_write);
 
@@ -598,6 +630,23 @@ static __init int activate_jump_labels(void)
 }
 arch_initcall(activate_jump_labels);
 
+static __init int kvm_setup_pv_tlb_flush(void)
+{
+	int cpu;
+
+	if (kvm_para_has_feature(KVM_FEATURE_PV_TLB_FLUSH) &&
+	    kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
+		for_each_possible_cpu(cpu) {
+			zalloc_cpumask_var_node(per_cpu_ptr(&__pv_tlb_mask, cpu),
+				GFP_KERNEL, cpu_to_node(cpu));
+		}
+		pr_info("KVM setup pv remote TLB flush\n");
+	}
+
+	return 0;
+}
+arch_initcall(kvm_setup_pv_tlb_flush);
+
 #ifdef CONFIG_PARAVIRT_SPINLOCKS
 
 /* Kick a cpu by its apicid. Used to wake up a halted vcpu */
@@ -643,7 +692,7 @@ __visible bool __kvm_vcpu_is_preempted(long cpu)
 {
 	struct kvm_steal_time *src = &per_cpu(steal_time, cpu);
 
-	return !!src->preempted;
+	return !!(src->preempted & KVM_VCPU_PREEMPTED);
 }
 PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted);
 
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 1f790cf9d38f..3b7427aa7d85 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -542,6 +542,7 @@ int arch_kexec_apply_relocations_add(const Elf64_Ehdr *ehdr,
 				goto overflow;
 			break;
 		case R_X86_64_PC32:
+		case R_X86_64_PLT32:
 			value -= (u64)address;
 			*(u32 *)location = value;
 			break;
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index da0c160e5589..f58336af095c 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -191,6 +191,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 				goto overflow;
 			break;
 		case R_X86_64_PC32:
+		case R_X86_64_PLT32:
 			if (*(u32 *)loc != 0)
 				goto invalid_relocation;
 			val -= (u64)loc;
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c
index 3a4b12809ab5..f1c5eb99d445 100644
--- a/arch/x86/kernel/mpparse.c
+++ b/arch/x86/kernel/mpparse.c
@@ -281,7 +281,7 @@ static void __init construct_default_ioirq_mptable(int mpc_default_type)
 	int ELCR_fallback = 0;
 
 	intsrc.type = MP_INTSRC;
-	intsrc.irqflag = 0;	/* conforming */
+	intsrc.irqflag = MP_IRQTRIG_DEFAULT | MP_IRQPOL_DEFAULT;
 	intsrc.srcbus = 0;
 	intsrc.dstapic = mpc_ioapic_id(0);
 
@@ -324,10 +324,13 @@ static void __init construct_default_ioirq_mptable(int mpc_default_type)
 			 *  copy that information over to the MP table in the
 			 *  irqflag field (level sensitive, active high polarity).
 			 */
-			if (ELCR_trigger(i))
-				intsrc.irqflag = 13;
-			else
-				intsrc.irqflag = 0;
+			if (ELCR_trigger(i)) {
+				intsrc.irqflag = MP_IRQTRIG_LEVEL |
+						 MP_IRQPOL_ACTIVE_HIGH;
+			} else {
+				intsrc.irqflag = MP_IRQTRIG_DEFAULT |
+						 MP_IRQPOL_DEFAULT;
+			}
 		}
 
 		intsrc.srcbusirq = i;
@@ -407,7 +410,7 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
 	processor.apicver = mpc_default_type > 4 ? 0x10 : 0x01;
 	processor.cpuflag = CPU_ENABLED;
 	processor.cpufeature = (boot_cpu_data.x86 << 8) |
-	    (boot_cpu_data.x86_model << 4) | boot_cpu_data.x86_mask;
+	    (boot_cpu_data.x86_model << 4) | boot_cpu_data.x86_stepping;
 	processor.featureflag = boot_cpu_data.x86_capability[CPUID_1_EDX];
 	processor.reserved[0] = 0;
 	processor.reserved[1] = 0;
@@ -419,7 +422,7 @@ static inline void __init construct_default_ISA_mptable(int mpc_default_type)
 	construct_ioapic_table(mpc_default_type);
 
 	lintsrc.type = MP_LINTSRC;
-	lintsrc.irqflag = 0;		/* conforming */
+	lintsrc.irqflag = MP_IRQTRIG_DEFAULT | MP_IRQPOL_DEFAULT;
 	lintsrc.srcbusid = 0;
 	lintsrc.srcbusirq = 0;
 	lintsrc.destapic = MP_APIC_ALL;
@@ -664,7 +667,7 @@ static int  __init get_MP_intsrc_index(struct mpc_intsrc *m)
 	if (m->irqtype != mp_INT)
 		return 0;
 
-	if (m->irqflag != 0x0f)
+	if (m->irqflag != (MP_IRQTRIG_LEVEL | MP_IRQPOL_ACTIVE_LOW))
 		return 0;
 
 	/* not legacy */
@@ -673,7 +676,8 @@ static int  __init get_MP_intsrc_index(struct mpc_intsrc *m)
 		if (mp_irqs[i].irqtype != mp_INT)
 			continue;
 
-		if (mp_irqs[i].irqflag != 0x0f)
+		if (mp_irqs[i].irqflag != (MP_IRQTRIG_LEVEL |
+					   MP_IRQPOL_ACTIVE_LOW))
 			continue;
 
 		if (mp_irqs[i].srcbus != m->srcbus)
@@ -784,7 +788,8 @@ static int  __init replace_intsrc_all(struct mpc_table *mpc,
 		if (mp_irqs[i].irqtype != mp_INT)
 			continue;
 
-		if (mp_irqs[i].irqflag != 0x0f)
+		if (mp_irqs[i].irqflag != (MP_IRQTRIG_LEVEL |
+					   MP_IRQPOL_ACTIVE_LOW))
 			continue;
 
 		if (nr_m_spare > 0) {
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 041096bdef86..99dc79e76bdc 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -200,9 +200,9 @@ static void native_flush_tlb_global(void)
 	__native_flush_tlb_global();
 }
 
-static void native_flush_tlb_single(unsigned long addr)
+static void native_flush_tlb_one_user(unsigned long addr)
 {
-	__native_flush_tlb_single(addr);
+	__native_flush_tlb_one_user(addr);
 }
 
 struct static_key paravirt_steal_enabled;
@@ -401,7 +401,7 @@ struct pv_mmu_ops pv_mmu_ops __ro_after_init = {
 
 	.flush_tlb_user = native_flush_tlb,
 	.flush_tlb_kernel = native_flush_tlb_global,
-	.flush_tlb_single = native_flush_tlb_single,
+	.flush_tlb_one_user = native_flush_tlb_one_user,
 	.flush_tlb_others = native_flush_tlb_others,
 
 	.pgd_alloc = __paravirt_pgd_alloc,
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 599d7462eccc..df7ab02f959f 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/dma-mapping.h>
+#include <linux/dma-direct.h>
 #include <linux/dma-debug.h>
 #include <linux/dmar.h>
 #include <linux/export.h>
@@ -87,7 +87,6 @@ void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 
 	dma_mask = dma_alloc_coherent_mask(dev, flag);
 
-	flag &= ~__GFP_ZERO;
 again:
 	page = NULL;
 	/* CMA can be used only in the context which permits sleeping */
@@ -139,7 +138,6 @@ bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)
 	if (!*dev)
 		*dev = &x86_dma_fallback_dev;
 
-	*gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
 	*gfp = dma_alloc_coherent_gfp_flags(*dev, *gfp);
 
 	if (!is_device_dma_capable(*dev))
@@ -217,7 +215,7 @@ static __init int iommu_setup(char *p)
 }
 early_param("iommu", iommu_setup);
 
-int x86_dma_supported(struct device *dev, u64 mask)
+int arch_dma_supported(struct device *dev, u64 mask)
 {
 #ifdef CONFIG_PCI
 	if (mask > 0xffffffff && forbid_dac > 0) {
@@ -226,12 +224,6 @@ int x86_dma_supported(struct device *dev, u64 mask)
 	}
 #endif
 
-	/* Copied from i386. Doesn't make much sense, because it will
-	   only work for pci_alloc_coherent.
-	   The caller just has to use GFP_DMA in this case. */
-	if (mask < DMA_BIT_MASK(24))
-		return 0;
-
 	/* Tell the device to use SAC when IOMMU force is on.  This
 	   allows the driver to use cheaper accesses in some cases.
 
@@ -251,6 +243,17 @@ int x86_dma_supported(struct device *dev, u64 mask)
 
 	return 1;
 }
+EXPORT_SYMBOL(arch_dma_supported);
+
+int x86_dma_supported(struct device *dev, u64 mask)
+{
+	/* Copied from i386. Doesn't make much sense, because it will
+	   only work for pci_alloc_coherent.
+	   The caller just has to use GFP_DMA in this case. */
+	if (mask < DMA_BIT_MASK(24))
+		return 0;
+	return 1;
+}
 
 static int __init pci_iommu_init(void)
 {
diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c
index b0caae27e1b7..ac7ea3a8242f 100644
--- a/arch/x86/kernel/pci-nommu.c
+++ b/arch/x86/kernel/pci-nommu.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Fallback functions when the main IOMMU code is not compiled in. This
    code is roughly equivalent to i386. */
-#include <linux/dma-mapping.h>
+#include <linux/dma-direct.h>
 #include <linux/scatterlist.h>
 #include <linux/string.h>
 #include <linux/gfp.h>
@@ -37,7 +37,6 @@ static dma_addr_t nommu_map_page(struct device *dev, struct page *page,
 	WARN_ON(size == 0);
 	if (!check_addr("map_single", dev, bus, size))
 		return NOMMU_MAPPING_ERROR;
-	flush_write_buffers();
 	return bus;
 }
 
@@ -72,25 +71,9 @@ static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg,
 			return 0;
 		s->dma_length = s->length;
 	}
-	flush_write_buffers();
 	return nents;
 }
 
-static void nommu_sync_single_for_device(struct device *dev,
-			dma_addr_t addr, size_t size,
-			enum dma_data_direction dir)
-{
-	flush_write_buffers();
-}
-
-
-static void nommu_sync_sg_for_device(struct device *dev,
-			struct scatterlist *sg, int nelems,
-			enum dma_data_direction dir)
-{
-	flush_write_buffers();
-}
-
 static int nommu_mapping_error(struct device *dev, dma_addr_t dma_addr)
 {
 	return dma_addr == NOMMU_MAPPING_ERROR;
@@ -101,8 +84,6 @@ const struct dma_map_ops nommu_dma_ops = {
 	.free			= dma_generic_free_coherent,
 	.map_sg			= nommu_map_sg,
 	.map_page		= nommu_map_page,
-	.sync_single_for_device = nommu_sync_single_for_device,
-	.sync_sg_for_device	= nommu_sync_sg_for_device,
 	.is_phys		= 1,
 	.mapping_error		= nommu_mapping_error,
 	.dma_supported		= x86_dma_supported,
diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index 53bd05ea90d8..0ee0f8f34251 100644
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -6,7 +6,7 @@
 #include <linux/init.h>
 #include <linux/swiotlb.h>
 #include <linux/bootmem.h>
-#include <linux/dma-mapping.h>
+#include <linux/dma-direct.h>
 #include <linux/mem_encrypt.h>
 
 #include <asm/iommu.h>
@@ -48,7 +48,7 @@ void x86_swiotlb_free_coherent(struct device *dev, size_t size,
 		dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs);
 }
 
-static const struct dma_map_ops swiotlb_dma_ops = {
+static const struct dma_map_ops x86_swiotlb_dma_ops = {
 	.mapping_error = swiotlb_dma_mapping_error,
 	.alloc = x86_swiotlb_alloc_coherent,
 	.free = x86_swiotlb_free_coherent,
@@ -112,7 +112,7 @@ void __init pci_swiotlb_init(void)
 {
 	if (swiotlb) {
 		swiotlb_init(0);
-		dma_ops = &swiotlb_dma_ops;
+		dma_ops = &x86_swiotlb_dma_ops;
 	}
 }
 
@@ -120,7 +120,7 @@ void __init pci_swiotlb_late_init(void)
 {
 	/* An IOMMU turned us off. */
 	if (!swiotlb)
-		swiotlb_free();
+		swiotlb_exit();
 	else {
 		printk(KERN_INFO "PCI-DMA: "
 		       "Using software bounce buffering for IO (SWIOTLB)\n");
diff --git a/arch/x86/kernel/platform-quirks.c b/arch/x86/kernel/platform-quirks.c
index 39a59299bfa0..235fe6008ac8 100644
--- a/arch/x86/kernel/platform-quirks.c
+++ b/arch/x86/kernel/platform-quirks.c
@@ -9,6 +9,7 @@ void __init x86_early_init_platform_quirks(void)
 {
 	x86_platform.legacy.i8042 = X86_LEGACY_I8042_EXPECTED_PRESENT;
 	x86_platform.legacy.rtc = 1;
+	x86_platform.legacy.warm_reset = 1;
 	x86_platform.legacy.reserve_bios_regions = 0;
 	x86_platform.legacy.devices.pnpbios = 1;
 
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index cb368c2a22ab..03408b942adb 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -21,7 +21,6 @@
 #include <linux/dmi.h>
 #include <linux/utsname.h>
 #include <linux/stackprotector.h>
-#include <linux/tick.h>
 #include <linux/cpuidle.h>
 #include <trace/events/power.h>
 #include <linux/hw_breakpoint.h>
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index c75466232016..9eb448c7859d 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -557,7 +557,7 @@ static void __set_personality_x32(void)
 	 * Pretend to come from a x32 execve.
 	 */
 	task_pt_regs(current)->orig_ax = __NR_x32_execve | __X32_SYSCALL_BIT;
-	current->thread.status &= ~TS_COMPAT;
+	current_thread_info()->status &= ~TS_COMPAT;
 #endif
 }
 
@@ -571,7 +571,7 @@ static void __set_personality_ia32(void)
 	current->personality |= force_personality32;
 	/* Prepare the first "return" to user space */
 	task_pt_regs(current)->orig_ax = __NR_ia32_execve;
-	current->thread.status |= TS_COMPAT;
+	current_thread_info()->status |= TS_COMPAT;
 #endif
 }
 
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index f37d18124648..ed5c4cdf0a34 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -935,7 +935,7 @@ static int putreg32(struct task_struct *child, unsigned regno, u32 value)
 		 */
 		regs->orig_ax = value;
 		if (syscall_get_nr(child, regs) >= 0)
-			child->thread.status |= TS_I386_REGS_POKED;
+			child->thread_info.status |= TS_I386_REGS_POKED;
 		break;
 
 	case offsetof(struct user32, regs.eflags):
diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
index 307d3bac5f04..11eda21eb697 100644
--- a/arch/x86/kernel/relocate_kernel_64.S
+++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -68,6 +68,9 @@ relocate_kernel:
 	movq	%cr4, %rax
 	movq	%rax, CR4(%r11)
 
+	/* Save CR4. Required to enable the right paging mode later. */
+	movq	%rax, %r13
+
 	/* zero out flags, and disable interrupts */
 	pushq $0
 	popfq
@@ -126,8 +129,13 @@ identity_mapped:
 	/*
 	 * Set cr4 to a known state:
 	 *  - physical address extension enabled
+	 *  - 5-level paging, if it was enabled before
 	 */
 	movl	$X86_CR4_PAE, %eax
+	testq	$X86_CR4_LA57, %r13
+	jz	1f
+	orl	$X86_CR4_LA57, %eax
+1:
 	movq	%rax, %cr4
 
 	jmp 1f
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 68d7ab81c62f..4c616be28506 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -114,7 +114,6 @@
 #include <asm/alternative.h>
 #include <asm/prom.h>
 #include <asm/microcode.h>
-#include <asm/mmu_context.h>
 #include <asm/kaslr.h>
 #include <asm/unwind.h>
 
@@ -1205,20 +1204,13 @@ void __init setup_arch(char **cmdline_p)
 
 	kasan_init();
 
-#ifdef CONFIG_X86_32
-	/* sync back kernel address range */
-	clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,
-			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
-			KERNEL_PGD_PTRS);
-
 	/*
-	 * sync back low identity map too.  It is used for example
-	 * in the 32-bit EFI stub.
+	 * Sync back kernel address range.
+	 *
+	 * FIXME: Can the later sync in setup_cpu_entry_areas() replace
+	 * this call?
 	 */
-	clone_pgd_range(initial_page_table,
-			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
-			min(KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
-#endif
+	sync_initial_page_table();
 
 	tboot_probe();
 
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 497aa766fab3..ea554f812ee1 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -287,24 +287,15 @@ void __init setup_per_cpu_areas(void)
 	/* Setup cpu initialized, callin, callout masks */
 	setup_cpu_local_masks();
 
-#ifdef CONFIG_X86_32
 	/*
 	 * Sync back kernel address range again.  We already did this in
 	 * setup_arch(), but percpu data also needs to be available in
 	 * the smpboot asm.  We can't reliably pick up percpu mappings
 	 * using vmalloc_fault(), because exception dispatch needs
 	 * percpu data.
+	 *
+	 * FIXME: Can the later sync in setup_cpu_entry_areas() replace
+	 * this call?
 	 */
-	clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,
-			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
-			KERNEL_PGD_PTRS);
-
-	/*
-	 * sync back low identity map too.  It is used for example
-	 * in the 32-bit EFI stub.
-	 */
-	clone_pgd_range(initial_page_table,
-			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
-			min(KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
-#endif
+	sync_initial_page_table();
 }
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index b9e00e8f1c9b..4cdc0b27ec82 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -787,7 +787,7 @@ static inline unsigned long get_nr_restart_syscall(const struct pt_regs *regs)
 	 * than the tracee.
 	 */
 #ifdef CONFIG_IA32_EMULATION
-	if (current->thread.status & (TS_COMPAT|TS_I386_REGS_POKED))
+	if (current_thread_info()->status & (TS_COMPAT|TS_I386_REGS_POKED))
 		return __NR_ia32_restart_syscall;
 #endif
 #ifdef CONFIG_X86_X32_ABI
diff --git a/arch/x86/kernel/signal_compat.c b/arch/x86/kernel/signal_compat.c
index 8c6da1a643da..0d930d8987cc 100644
--- a/arch/x86/kernel/signal_compat.c
+++ b/arch/x86/kernel/signal_compat.c
@@ -25,8 +25,8 @@ static inline void signal_compat_build_tests(void)
 	 * limits also have to look at this code.  Make sure any
 	 * new fields are handled in copy_siginfo_to_user32()!
 	 */
-	BUILD_BUG_ON(NSIGILL  != 8);
-	BUILD_BUG_ON(NSIGFPE  != 8);
+	BUILD_BUG_ON(NSIGILL  != 11);
+	BUILD_BUG_ON(NSIGFPE  != 13);
 	BUILD_BUG_ON(NSIGSEGV != 4);
 	BUILD_BUG_ON(NSIGBUS  != 5);
 	BUILD_BUG_ON(NSIGTRAP != 4);
@@ -43,6 +43,13 @@ static inline void signal_compat_build_tests(void)
 	BUILD_BUG_ON(offsetof(compat_siginfo_t, _sifields) != 3 * sizeof(int));
 #define CHECK_CSI_OFFSET(name)	  BUILD_BUG_ON(_sifields_offset != offsetof(compat_siginfo_t, _sifields.name))
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_signo) != 0);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_errno) != 4);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_code)  != 8);
+
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_signo) != 0);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_errno) != 4);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_code)  != 8);
 	 /*
 	 * Ensure that the size of each si_field never changes.
 	 * If it does, it is a sign that the
@@ -63,39 +70,101 @@ static inline void signal_compat_build_tests(void)
 	CHECK_CSI_SIZE  (_kill, 2*sizeof(int));
 	CHECK_SI_SIZE   (_kill, 2*sizeof(int));
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_pid) != 0x10);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_uid) != 0x14);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_pid) != 0xC);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_uid) != 0x10);
+
 	CHECK_CSI_OFFSET(_timer);
-	CHECK_CSI_SIZE  (_timer, 5*sizeof(int));
+	CHECK_CSI_SIZE  (_timer, 3*sizeof(int));
 	CHECK_SI_SIZE   (_timer, 6*sizeof(int));
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_tid)     != 0x10);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_overrun) != 0x14);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_value)   != 0x18);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_tid)     != 0x0C);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_overrun) != 0x10);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_value)   != 0x14);
+
 	CHECK_CSI_OFFSET(_rt);
 	CHECK_CSI_SIZE  (_rt, 3*sizeof(int));
 	CHECK_SI_SIZE   (_rt, 4*sizeof(int));
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_pid)   != 0x10);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_uid)   != 0x14);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_value) != 0x18);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_pid)   != 0x0C);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_uid)   != 0x10);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_value) != 0x14);
+
 	CHECK_CSI_OFFSET(_sigchld);
 	CHECK_CSI_SIZE  (_sigchld, 5*sizeof(int));
 	CHECK_SI_SIZE   (_sigchld, 8*sizeof(int));
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_pid)    != 0x10);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_uid)    != 0x14);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_status) != 0x18);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_utime)  != 0x20);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_stime)  != 0x28);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_pid)    != 0x0C);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_uid)    != 0x10);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_status) != 0x14);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_utime)  != 0x18);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_stime)  != 0x1C);
+
+#ifdef CONFIG_X86_X32_ABI
 	CHECK_CSI_OFFSET(_sigchld_x32);
 	CHECK_CSI_SIZE  (_sigchld_x32, 7*sizeof(int));
 	/* no _sigchld_x32 in the generic siginfo_t */
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, _sifields._sigchld_x32._utime)  != 0x18);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, _sifields._sigchld_x32._stime)  != 0x20);
+#endif
 
 	CHECK_CSI_OFFSET(_sigfault);
 	CHECK_CSI_SIZE  (_sigfault, 4*sizeof(int));
 	CHECK_SI_SIZE   (_sigfault, 8*sizeof(int));
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_addr) != 0x10);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_addr) != 0x0C);
+
+	BUILD_BUG_ON(offsetof(siginfo_t, si_addr_lsb) != 0x18);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_addr_lsb) != 0x10);
+
+	BUILD_BUG_ON(offsetof(siginfo_t, si_lower) != 0x20);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_upper) != 0x28);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_lower) != 0x14);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_upper) != 0x18);
+
+	BUILD_BUG_ON(offsetof(siginfo_t, si_pkey) != 0x20);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_pkey) != 0x14);
+
 	CHECK_CSI_OFFSET(_sigpoll);
 	CHECK_CSI_SIZE  (_sigpoll, 2*sizeof(int));
 	CHECK_SI_SIZE   (_sigpoll, 4*sizeof(int));
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_band)   != 0x10);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_fd)     != 0x18);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_band) != 0x0C);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_fd)   != 0x10);
+
 	CHECK_CSI_OFFSET(_sigsys);
 	CHECK_CSI_SIZE  (_sigsys, 3*sizeof(int));
 	CHECK_SI_SIZE   (_sigsys, 4*sizeof(int));
 
+	BUILD_BUG_ON(offsetof(siginfo_t, si_call_addr) != 0x10);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_syscall)   != 0x18);
+	BUILD_BUG_ON(offsetof(siginfo_t, si_arch)      != 0x1C);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_call_addr) != 0x0C);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_syscall)   != 0x10);
+	BUILD_BUG_ON(offsetof(compat_siginfo_t, si_arch)      != 0x14);
+
 	/* any new si_fields should be added here */
 }
 
 void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oact)
 {
+	signal_compat_build_tests();
+
 	/* Don't leak in-kernel non-uapi flags to user-space */
 	if (oact)
 		oact->sa.sa_flags &= ~(SA_IA32_ABI | SA_X32_ABI);
@@ -111,116 +180,3 @@ void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oact)
 	if (in_x32_syscall())
 		act->sa.sa_flags |= SA_X32_ABI;
 }
-
-int __copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from,
-		bool x32_ABI)
-{
-	int err = 0;
-
-	signal_compat_build_tests();
-
-	if (!access_ok(VERIFY_WRITE, to, sizeof(compat_siginfo_t)))
-		return -EFAULT;
-
-	put_user_try {
-		/* If you change siginfo_t structure, please make sure that
-		   this code is fixed accordingly.
-		   It should never copy any pad contained in the structure
-		   to avoid security leaks, but must copy the generic
-		   3 ints plus the relevant union member.  */
-		put_user_ex(from->si_signo, &to->si_signo);
-		put_user_ex(from->si_errno, &to->si_errno);
-		put_user_ex(from->si_code, &to->si_code);
-
-		if (from->si_code < 0) {
-			put_user_ex(from->si_pid, &to->si_pid);
-			put_user_ex(from->si_uid, &to->si_uid);
-			put_user_ex(ptr_to_compat(from->si_ptr), &to->si_ptr);
-		} else {
-			/*
-			 * First 32bits of unions are always present:
-			 * si_pid === si_band === si_tid === si_addr(LS half)
-			 */
-			put_user_ex(from->_sifields._pad[0],
-					  &to->_sifields._pad[0]);
-			switch (siginfo_layout(from->si_signo, from->si_code)) {
-			case SIL_FAULT:
-				if (from->si_signo == SIGBUS &&
-				    (from->si_code == BUS_MCEERR_AR ||
-				     from->si_code == BUS_MCEERR_AO))
-					put_user_ex(from->si_addr_lsb, &to->si_addr_lsb);
-
-				if (from->si_signo == SIGSEGV) {
-					if (from->si_code == SEGV_BNDERR) {
-						compat_uptr_t lower = (unsigned long)from->si_lower;
-						compat_uptr_t upper = (unsigned long)from->si_upper;
-						put_user_ex(lower, &to->si_lower);
-						put_user_ex(upper, &to->si_upper);
-					}
-					if (from->si_code == SEGV_PKUERR)
-						put_user_ex(from->si_pkey, &to->si_pkey);
-				}
-				break;
-			case SIL_SYS:
-				put_user_ex(from->si_syscall, &to->si_syscall);
-				put_user_ex(from->si_arch, &to->si_arch);
-				break;
-			case SIL_CHLD:
-				if (!x32_ABI) {
-					put_user_ex(from->si_utime, &to->si_utime);
-					put_user_ex(from->si_stime, &to->si_stime);
-				} else {
-					put_user_ex(from->si_utime, &to->_sifields._sigchld_x32._utime);
-					put_user_ex(from->si_stime, &to->_sifields._sigchld_x32._stime);
-				}
-				put_user_ex(from->si_status, &to->si_status);
-				/* FALL THROUGH */
-			case SIL_KILL:
-				put_user_ex(from->si_uid, &to->si_uid);
-				break;
-			case SIL_POLL:
-				put_user_ex(from->si_fd, &to->si_fd);
-				break;
-			case SIL_TIMER:
-				put_user_ex(from->si_overrun, &to->si_overrun);
-				put_user_ex(ptr_to_compat(from->si_ptr),
-					    &to->si_ptr);
-				break;
-			case SIL_RT:
-				put_user_ex(from->si_uid, &to->si_uid);
-				put_user_ex(from->si_int, &to->si_int);
-				break;
-			}
-		}
-	} put_user_catch(err);
-
-	return err;
-}
-
-/* from syscall's path, where we know the ABI */
-int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from)
-{
-	return __copy_siginfo_to_user32(to, from, in_x32_syscall());
-}
-
-int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from)
-{
-	int err = 0;
-	u32 ptr32;
-
-	if (!access_ok(VERIFY_READ, from, sizeof(compat_siginfo_t)))
-		return -EFAULT;
-
-	get_user_try {
-		get_user_ex(to->si_signo, &from->si_signo);
-		get_user_ex(to->si_errno, &from->si_errno);
-		get_user_ex(to->si_code, &from->si_code);
-
-		get_user_ex(to->si_pid, &from->si_pid);
-		get_user_ex(to->si_uid, &from->si_uid);
-		get_user_ex(ptr32, &from->si_ptr);
-		to->si_ptr = compat_ptr(ptr32);
-	} get_user_catch(err);
-
-	return err;
-}
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index ed556d50d7ed..ff99e2b6fc54 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -75,7 +75,6 @@
 #include <asm/uv/uv.h>
 #include <linux/mc146818rtc.h>
 #include <asm/i8259.h>
-#include <asm/realmode.h>
 #include <asm/misc.h>
 #include <asm/qspinlock.h>
 
@@ -934,7 +933,7 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
 	 * the targeted processor.
 	 */
 
-	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
+	if (x86_platform.legacy.warm_reset) {
 
 		pr_debug("Setting warm reset code and vector.\n");
 
@@ -1006,7 +1005,7 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
 	/* mark "stuck" area as not stuck */
 	*trampoline_status = 0;
 
-	if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
+	if (x86_platform.legacy.warm_reset) {
 		/*
 		 * Cleanup possible dangling ends...
 		 */
@@ -1282,11 +1281,10 @@ void __init native_smp_prepare_boot_cpu(void)
 	cpu_set_state_online(me);
 }
 
-void __init native_smp_cpus_done(unsigned int max_cpus)
+void __init calculate_max_logical_packages(void)
 {
 	int ncpus;
 
-	pr_debug("Boot done\n");
 	/*
 	 * Today neither Intel nor AMD support heterogenous systems so
 	 * extrapolate the boot cpu's data to all packages.
@@ -1294,6 +1292,13 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
 	ncpus = cpu_data(0).booted_cores * topology_max_smt_threads();
 	__max_logical_packages = DIV_ROUND_UP(nr_cpu_ids, ncpus);
 	pr_info("Max logical packages: %u\n", __max_logical_packages);
+}
+
+void __init native_smp_cpus_done(unsigned int max_cpus)
+{
+	pr_debug("Boot done\n");
+
+	calculate_max_logical_packages();
 
 	if (x86_has_numa_in_package)
 		set_sched_topology(x86_numa_in_package_topology);
@@ -1431,8 +1436,8 @@ static void remove_siblinginfo(int cpu)
 	cpumask_clear(cpu_llc_shared_mask(cpu));
 	cpumask_clear(topology_sibling_cpumask(cpu));
 	cpumask_clear(topology_core_cpumask(cpu));
-	c->phys_proc_id = 0;
 	c->cpu_core_id = 0;
+	c->booted_cores = 0;
 	cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
 	recompute_smt_state();
 }
diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c
index 749d189f8cd4..774ebafa97c4 100644
--- a/arch/x86/kernel/time.c
+++ b/arch/x86/kernel/time.c
@@ -69,9 +69,12 @@ static struct irqaction irq0  = {
 
 static void __init setup_default_timer_irq(void)
 {
-	if (!nr_legacy_irqs())
-		return;
-	setup_irq(0, &irq0);
+	/*
+	 * Unconditionally register the legacy timer; even without legacy
+	 * PIC/PIT we need this for the HPET0 in legacy replacement mode.
+	 */
+	if (setup_irq(0, &irq0))
+		pr_info("Failed to register legacy timer interrupt\n");
 }
 
 /* Default timer init function */
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 446c9ef8cfc3..03f3d7695dac 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -181,7 +181,7 @@ int fixup_bug(struct pt_regs *regs, int trapnr)
 		break;
 
 	case BUG_TRAP_TYPE_WARN:
-		regs->ip += LEN_UD0;
+		regs->ip += LEN_UD2;
 		return 1;
 	}
 
@@ -577,7 +577,6 @@ do_general_protection(struct pt_regs *regs, long error_code)
 }
 NOKPROBE_SYMBOL(do_general_protection);
 
-/* May run on IST stack. */
 dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
 {
 #ifdef CONFIG_DYNAMIC_FTRACE
@@ -592,6 +591,13 @@ dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
 	if (poke_int3_handler(regs))
 		return;
 
+	/*
+	 * Use ist_enter despite the fact that we don't use an IST stack.
+	 * We can be called from a kprobe in non-CONTEXT_KERNEL kernel
+	 * mode or even during context tracking state changes.
+	 *
+	 * This means that we can't schedule.  That's okay.
+	 */
 	ist_enter(regs);
 	RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU");
 #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
@@ -609,15 +615,10 @@ dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
 			SIGTRAP) == NOTIFY_STOP)
 		goto exit;
 
-	/*
-	 * Let others (NMI) know that the debug stack is in use
-	 * as we may switch to the interrupt stack.
-	 */
-	debug_stack_usage_inc();
 	cond_local_irq_enable(regs);
 	do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
 	cond_local_irq_disable(regs);
-	debug_stack_usage_dec();
+
 exit:
 	ist_exit(regs);
 }
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index e169e85db434..fb4302738410 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -25,6 +25,7 @@
 #include <asm/geode.h>
 #include <asm/apic.h>
 #include <asm/intel-family.h>
+#include <asm/i8259.h>
 
 unsigned int __read_mostly cpu_khz;	/* TSC clocks / usec, not used here */
 EXPORT_SYMBOL(cpu_khz);
@@ -363,6 +364,20 @@ static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin)
 	unsigned long tscmin, tscmax;
 	int pitcnt;
 
+	if (!has_legacy_pic()) {
+		/*
+		 * Relies on tsc_early_delay_calibrate() to have given us semi
+		 * usable udelay(), wait for the same 50ms we would have with
+		 * the PIT loop below.
+		 */
+		udelay(10 * USEC_PER_MSEC);
+		udelay(10 * USEC_PER_MSEC);
+		udelay(10 * USEC_PER_MSEC);
+		udelay(10 * USEC_PER_MSEC);
+		udelay(10 * USEC_PER_MSEC);
+		return ULONG_MAX;
+	}
+
 	/* Set the Gate high, disable speaker */
 	outb((inb(0x61) & ~0x02) | 0x01, 0x61);
 
@@ -487,6 +502,9 @@ static unsigned long quick_pit_calibrate(void)
 	u64 tsc, delta;
 	unsigned long d1, d2;
 
+	if (!has_legacy_pic())
+		return 0;
+
 	/* Set the Gate high, disable speaker */
 	outb((inb(0x61) & ~0x02) | 0x01, 0x61);
 
@@ -988,8 +1006,6 @@ static void __init detect_art(void)
 
 /* clocksource code */
 
-static struct clocksource clocksource_tsc;
-
 static void tsc_resume(struct clocksource *cs)
 {
 	tsc_verify_tsc_adjust(true);
@@ -1040,12 +1056,31 @@ static void tsc_cs_tick_stable(struct clocksource *cs)
 /*
  * .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc()
  */
+static struct clocksource clocksource_tsc_early = {
+	.name                   = "tsc-early",
+	.rating                 = 299,
+	.read                   = read_tsc,
+	.mask                   = CLOCKSOURCE_MASK(64),
+	.flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
+				  CLOCK_SOURCE_MUST_VERIFY,
+	.archdata               = { .vclock_mode = VCLOCK_TSC },
+	.resume			= tsc_resume,
+	.mark_unstable		= tsc_cs_mark_unstable,
+	.tick_stable		= tsc_cs_tick_stable,
+};
+
+/*
+ * Must mark VALID_FOR_HRES early such that when we unregister tsc_early
+ * this one will immediately take over. We will only register if TSC has
+ * been found good.
+ */
 static struct clocksource clocksource_tsc = {
 	.name                   = "tsc",
 	.rating                 = 300,
 	.read                   = read_tsc,
 	.mask                   = CLOCKSOURCE_MASK(64),
 	.flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
+				  CLOCK_SOURCE_VALID_FOR_HRES |
 				  CLOCK_SOURCE_MUST_VERIFY,
 	.archdata               = { .vclock_mode = VCLOCK_TSC },
 	.resume			= tsc_resume,
@@ -1169,8 +1204,8 @@ static void tsc_refine_calibration_work(struct work_struct *work)
 	int cpu;
 
 	/* Don't bother refining TSC on unstable systems */
-	if (check_tsc_unstable())
-		goto out;
+	if (tsc_unstable)
+		return;
 
 	/*
 	 * Since the work is started early in boot, we may be
@@ -1222,9 +1257,13 @@ static void tsc_refine_calibration_work(struct work_struct *work)
 		set_cyc2ns_scale(tsc_khz, cpu, tsc_stop);
 
 out:
+	if (tsc_unstable)
+		return;
+
 	if (boot_cpu_has(X86_FEATURE_ART))
 		art_related_clocksource = &clocksource_tsc;
 	clocksource_register_khz(&clocksource_tsc, tsc_khz);
+	clocksource_unregister(&clocksource_tsc_early);
 }
 
 
@@ -1233,13 +1272,11 @@ static int __init init_tsc_clocksource(void)
 	if (!boot_cpu_has(X86_FEATURE_TSC) || tsc_disabled > 0 || !tsc_khz)
 		return 0;
 
+	if (check_tsc_unstable())
+		return 0;
+
 	if (tsc_clocksource_reliable)
 		clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
-	/* lower the rating if we already know its unstable: */
-	if (check_tsc_unstable()) {
-		clocksource_tsc.rating = 0;
-		clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
-	}
 
 	if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3))
 		clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
@@ -1252,6 +1289,7 @@ static int __init init_tsc_clocksource(void)
 		if (boot_cpu_has(X86_FEATURE_ART))
 			art_related_clocksource = &clocksource_tsc;
 		clocksource_register_khz(&clocksource_tsc, tsc_khz);
+		clocksource_unregister(&clocksource_tsc_early);
 		return 0;
 	}
 
@@ -1356,9 +1394,12 @@ void __init tsc_init(void)
 
 	check_system_tsc_reliable();
 
-	if (unsynchronized_tsc())
+	if (unsynchronized_tsc()) {
 		mark_tsc_unstable("TSCs unsynchronized");
+		return;
+	}
 
+	clocksource_register_khz(&clocksource_tsc_early, tsc_khz);
 	detect_art();
 }
 
diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c
index 1f9188f5357c..feb28fee6cea 100644
--- a/arch/x86/kernel/unwind_orc.c
+++ b/arch/x86/kernel/unwind_orc.c
@@ -5,7 +5,6 @@
 #include <asm/unwind.h>
 #include <asm/orc_types.h>
 #include <asm/orc_lookup.h>
-#include <asm/sections.h>
 
 #define orc_warn(fmt, ...) \
 	printk_deferred_once(KERN_WARNING pr_fmt("WARNING: " fmt), ##__VA_ARGS__)
@@ -148,7 +147,7 @@ static struct orc_entry *orc_find(unsigned long ip)
 	}
 
 	/* vmlinux .init slow lookup: */
-	if (ip >= (unsigned long)_sinittext && ip < (unsigned long)_einittext)
+	if (init_kernel_text(ip))
 		return __orc_find(__start_orc_unwind_ip, __start_orc_unwind,
 				  __stop_orc_unwind_ip - __start_orc_unwind_ip, ip);
 
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index a3755d293a48..85c7ef23d99f 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -528,11 +528,11 @@ static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
 	return 0;
 }
 
-static int push_ret_address(struct pt_regs *regs, unsigned long ip)
+static int emulate_push_stack(struct pt_regs *regs, unsigned long val)
 {
 	unsigned long new_sp = regs->sp - sizeof_long();
 
-	if (copy_to_user((void __user *)new_sp, &ip, sizeof_long()))
+	if (copy_to_user((void __user *)new_sp, &val, sizeof_long()))
 		return -EFAULT;
 
 	regs->sp = new_sp;
@@ -566,7 +566,7 @@ static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs
 		regs->ip += correction;
 	} else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {
 		regs->sp += sizeof_long(); /* Pop incorrect return address */
-		if (push_ret_address(regs, utask->vaddr + auprobe->defparam.ilen))
+		if (emulate_push_stack(regs, utask->vaddr + auprobe->defparam.ilen))
 			return -ERESTART;
 	}
 	/* popf; tell the caller to not touch TF */
@@ -655,7 +655,7 @@ static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
 		 *
 		 * But there is corner case, see the comment in ->post_xol().
 		 */
-		if (push_ret_address(regs, new_ip))
+		if (emulate_push_stack(regs, new_ip))
 			return false;
 	} else if (!check_jmp_cond(auprobe, regs)) {
 		offs = 0;
@@ -665,6 +665,16 @@ static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
 	return true;
 }
 
+static bool push_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
+{
+	unsigned long *src_ptr = (void *)regs + auprobe->push.reg_offset;
+
+	if (emulate_push_stack(regs, *src_ptr))
+		return false;
+	regs->ip += auprobe->push.ilen;
+	return true;
+}
+
 static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
 {
 	BUG_ON(!branch_is_call(auprobe));
@@ -703,6 +713,10 @@ static const struct uprobe_xol_ops branch_xol_ops = {
 	.post_xol = branch_post_xol_op,
 };
 
+static const struct uprobe_xol_ops push_xol_ops = {
+	.emulate  = push_emulate_op,
+};
+
 /* Returns -ENOSYS if branch_xol_ops doesn't handle this insn */
 static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
 {
@@ -750,6 +764,87 @@ static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
 	return 0;
 }
 
+/* Returns -ENOSYS if push_xol_ops doesn't handle this insn */
+static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
+{
+	u8 opc1 = OPCODE1(insn), reg_offset = 0;
+
+	if (opc1 < 0x50 || opc1 > 0x57)
+		return -ENOSYS;
+
+	if (insn->length > 2)
+		return -ENOSYS;
+	if (insn->length == 2) {
+		/* only support rex_prefix 0x41 (x64 only) */
+#ifdef CONFIG_X86_64
+		if (insn->rex_prefix.nbytes != 1 ||
+		    insn->rex_prefix.bytes[0] != 0x41)
+			return -ENOSYS;
+
+		switch (opc1) {
+		case 0x50:
+			reg_offset = offsetof(struct pt_regs, r8);
+			break;
+		case 0x51:
+			reg_offset = offsetof(struct pt_regs, r9);
+			break;
+		case 0x52:
+			reg_offset = offsetof(struct pt_regs, r10);
+			break;
+		case 0x53:
+			reg_offset = offsetof(struct pt_regs, r11);
+			break;
+		case 0x54:
+			reg_offset = offsetof(struct pt_regs, r12);
+			break;
+		case 0x55:
+			reg_offset = offsetof(struct pt_regs, r13);
+			break;
+		case 0x56:
+			reg_offset = offsetof(struct pt_regs, r14);
+			break;
+		case 0x57:
+			reg_offset = offsetof(struct pt_regs, r15);
+			break;
+		}
+#else
+		return -ENOSYS;
+#endif
+	} else {
+		switch (opc1) {
+		case 0x50:
+			reg_offset = offsetof(struct pt_regs, ax);
+			break;
+		case 0x51:
+			reg_offset = offsetof(struct pt_regs, cx);
+			break;
+		case 0x52:
+			reg_offset = offsetof(struct pt_regs, dx);
+			break;
+		case 0x53:
+			reg_offset = offsetof(struct pt_regs, bx);
+			break;
+		case 0x54:
+			reg_offset = offsetof(struct pt_regs, sp);
+			break;
+		case 0x55:
+			reg_offset = offsetof(struct pt_regs, bp);
+			break;
+		case 0x56:
+			reg_offset = offsetof(struct pt_regs, si);
+			break;
+		case 0x57:
+			reg_offset = offsetof(struct pt_regs, di);
+			break;
+		}
+	}
+
+	auprobe->push.reg_offset = reg_offset;
+	auprobe->push.ilen = insn->length;
+	auprobe->ops = &push_xol_ops;
+	return 0;
+}
+
 /**
  * arch_uprobe_analyze_insn - instruction analysis including validity and fixups.
  * @mm: the probed address space.
@@ -771,6 +866,10 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm,
 	if (ret != -ENOSYS)
 		return ret;
 
+	ret = push_setup_xol_ops(auprobe, &insn);
+	if (ret != -ENOSYS)
+		return ret;
+
 	/*
 	 * Figure out which fixups default_post_xol_op() will need to perform,
 	 * and annotate defparam->fixups accordingly.
diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index 5edb27f1a2c4..9d0b5af7db91 100644
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -727,7 +727,8 @@ void handle_vm86_fault(struct kernel_vm86_regs *regs, long error_code)
 	return;
 
 check_vip:
-	if (VEFLAGS & X86_EFLAGS_VIP) {
+	if ((VEFLAGS & (X86_EFLAGS_VIP | X86_EFLAGS_VIF)) ==
+	    (X86_EFLAGS_VIP | X86_EFLAGS_VIF)) {
 		save_v86_state(regs, VM86_STI);
 		return;
 	}
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 9b138a06c1a4..b854ebf5851b 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -118,9 +118,11 @@ SECTIONS
 
 #ifdef CONFIG_X86_64
 		. = ALIGN(PAGE_SIZE);
+		VMLINUX_SYMBOL(__entry_trampoline_start) = .;
 		_entry_trampoline = .;
 		*(.entry_trampoline)
 		. = ALIGN(PAGE_SIZE);
+		VMLINUX_SYMBOL(__entry_trampoline_end) = .;
 		ASSERT(. - _entry_trampoline == PAGE_SIZE, "entry trampoline is too big");
 #endif
 
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index 3df51c287844..92fd433c50b9 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -81,6 +81,14 @@ config KVM_AMD
 	  To compile this as a module, choose M here: the module
 	  will be called kvm-amd.
 
+config KVM_AMD_SEV
+	def_bool y
+	bool "AMD Secure Encrypted Virtualization (SEV) support"
+	depends on KVM_AMD && X86_64
+	depends on CRYPTO_DEV_CCP && CRYPTO_DEV_CCP_DD && CRYPTO_DEV_SP_PSP
+	---help---
+	Provides support for launching Encrypted VMs on AMD processors.
+
 config KVM_MMU_AUDIT
 	bool "Audit KVM MMU"
 	depends on KVM && TRACEPOINTS
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 0099e10eb045..b671fc2d0422 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -67,9 +67,7 @@ u64 kvm_supported_xcr0(void)
 
 #define F(x) bit(X86_FEATURE_##x)
 
-/* These are scattered features in cpufeatures.h. */
-#define KVM_CPUID_BIT_AVX512_4VNNIW     2
-#define KVM_CPUID_BIT_AVX512_4FMAPS     3
+/* For scattered features from cpufeatures.h; we currently expose none */
 #define KF(x) bit(KVM_CPUID_BIT_##x)
 
 int kvm_update_cpuid(struct kvm_vcpu *vcpu)
@@ -293,13 +291,18 @@ static int __do_cpuid_ent_emulated(struct kvm_cpuid_entry2 *entry,
 {
 	switch (func) {
 	case 0:
-		entry->eax = 1;		/* only one leaf currently */
+		entry->eax = 7;
 		++*nent;
 		break;
 	case 1:
 		entry->ecx = F(MOVBE);
 		++*nent;
 		break;
+	case 7:
+		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
+		if (index == 0)
+			entry->ecx = F(RDPID);
+		++*nent;
 	default:
 		break;
 	}
@@ -327,6 +330,7 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 	unsigned f_invpcid = kvm_x86_ops->invpcid_supported() ? F(INVPCID) : 0;
 	unsigned f_mpx = kvm_mpx_supported() ? F(MPX) : 0;
 	unsigned f_xsaves = kvm_x86_ops->xsaves_supported() ? F(XSAVES) : 0;
+	unsigned f_umip = kvm_x86_ops->umip_emulated() ? F(UMIP) : 0;
 
 	/* cpuid 1.edx */
 	const u32 kvm_cpuid_1_edx_x86_features =
@@ -365,7 +369,12 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 		F(LAHF_LM) | F(CMP_LEGACY) | 0 /*SVM*/ | 0 /* ExtApicSpace */ |
 		F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
 		F(3DNOWPREFETCH) | F(OSVW) | 0 /* IBS */ | F(XOP) |
-		0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM);
+		0 /* SKINIT, WDT, LWP */ | F(FMA4) | F(TBM) |
+		F(TOPOEXT);
+
+	/* cpuid 0x80000008.ebx */
+	const u32 kvm_cpuid_8000_0008_ebx_x86_features =
+		F(IBPB) | F(IBRS);
 
 	/* cpuid 0xC0000001.edx */
 	const u32 kvm_cpuid_C000_0001_edx_x86_features =
@@ -387,12 +396,14 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 
 	/* cpuid 7.0.ecx*/
 	const u32 kvm_cpuid_7_0_ecx_x86_features =
-		F(AVX512VBMI) | F(LA57) | F(PKU) |
-		0 /*OSPKE*/ | F(AVX512_VPOPCNTDQ);
+		F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ |
+		F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
+		F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG);
 
 	/* cpuid 7.0.edx*/
 	const u32 kvm_cpuid_7_0_edx_x86_features =
-		KF(AVX512_4VNNIW) | KF(AVX512_4FMAPS);
+		F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
+		F(ARCH_CAPABILITIES);
 
 	/* all calls to cpuid_count() should be made on the same cpu */
 	get_cpu();
@@ -473,11 +484,12 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 			entry->ebx |= F(TSC_ADJUST);
 			entry->ecx &= kvm_cpuid_7_0_ecx_x86_features;
 			cpuid_mask(&entry->ecx, CPUID_7_ECX);
+			entry->ecx |= f_umip;
 			/* PKU is not yet implemented for shadow paging. */
 			if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE))
 				entry->ecx &= ~F(PKU);
 			entry->edx &= kvm_cpuid_7_0_edx_x86_features;
-			entry->edx &= get_scattered_cpuid_leaf(7, 0, CPUID_EDX);
+			cpuid_mask(&entry->edx, CPUID_7_EDX);
 		} else {
 			entry->ebx = 0;
 			entry->ecx = 0;
@@ -594,7 +606,9 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 			     (1 << KVM_FEATURE_ASYNC_PF) |
 			     (1 << KVM_FEATURE_PV_EOI) |
 			     (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
-			     (1 << KVM_FEATURE_PV_UNHALT);
+			     (1 << KVM_FEATURE_PV_UNHALT) |
+			     (1 << KVM_FEATURE_PV_TLB_FLUSH) |
+			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT);
 
 		if (sched_info_on())
 			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
@@ -604,7 +618,7 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 		entry->edx = 0;
 		break;
 	case 0x80000000:
-		entry->eax = min(entry->eax, 0x8000001a);
+		entry->eax = min(entry->eax, 0x8000001f);
 		break;
 	case 0x80000001:
 		entry->edx &= kvm_cpuid_8000_0001_edx_x86_features;
@@ -627,7 +641,14 @@ static inline int __do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
 		if (!g_phys_as)
 			g_phys_as = phys_as;
 		entry->eax = g_phys_as | (virt_as << 8);
-		entry->ebx = entry->edx = 0;
+		entry->edx = 0;
+		/* IBRS and IBPB aren't necessarily present in hardware cpuid */
+		if (boot_cpu_has(X86_FEATURE_IBPB))
+			entry->ebx |= F(IBPB);
+		if (boot_cpu_has(X86_FEATURE_IBRS))
+			entry->ebx |= F(IBRS);
+		entry->ebx &= kvm_cpuid_8000_0008_ebx_x86_features;
+		cpuid_mask(&entry->ebx, CPUID_8000_0008_EBX);
 		break;
 	}
 	case 0x80000019:
diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h
index c2cea6651279..9a327d5b6d1f 100644
--- a/arch/x86/kvm/cpuid.h
+++ b/arch/x86/kvm/cpuid.h
@@ -54,6 +54,7 @@ static const struct cpuid_reg reverse_cpuid[] = {
 	[CPUID_8000_000A_EDX] = {0x8000000a, 0, CPUID_EDX},
 	[CPUID_7_ECX]         = {         7, 0, CPUID_ECX},
 	[CPUID_8000_0007_EBX] = {0x80000007, 0, CPUID_EBX},
+	[CPUID_7_EDX]         = {         7, 0, CPUID_EDX},
 };
 
 static __always_inline struct cpuid_reg x86_feature_cpuid(unsigned x86_feature)
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index b514b2b2845a..d91eaeb01034 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -25,6 +25,7 @@
 #include <asm/kvm_emulate.h>
 #include <linux/stringify.h>
 #include <asm/debugreg.h>
+#include <asm/nospec-branch.h>
 
 #include "x86.h"
 #include "tss.h"
@@ -1021,8 +1022,8 @@ static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
 	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
 
 	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
-	asm("push %[flags]; popf; call *%[fastop]"
-	    : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
+	asm("push %[flags]; popf; " CALL_NOSPEC
+	    : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
 	return rc;
 }
 
@@ -3532,6 +3533,16 @@ static int em_cwd(struct x86_emulate_ctxt *ctxt)
 	return X86EMUL_CONTINUE;
 }
 
+static int em_rdpid(struct x86_emulate_ctxt *ctxt)
+{
+	u64 tsc_aux = 0;
+
+	if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
+		return emulate_gp(ctxt, 0);
+	ctxt->dst.val = tsc_aux;
+	return X86EMUL_CONTINUE;
+}
+
 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
 {
 	u64 tsc = 0;
@@ -3651,17 +3662,27 @@ static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
 	return X86EMUL_CONTINUE;
 }
 
-static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
+static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
 {
-	if (ctxt->modrm_reg > VCPU_SREG_GS)
-		return emulate_ud(ctxt);
+	if (segment > VCPU_SREG_GS &&
+	    (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
+	    ctxt->ops->cpl(ctxt) > 0)
+		return emulate_gp(ctxt, 0);
 
-	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
+	ctxt->dst.val = get_segment_selector(ctxt, segment);
 	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
 		ctxt->dst.bytes = 2;
 	return X86EMUL_CONTINUE;
 }
 
+static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
+{
+	if (ctxt->modrm_reg > VCPU_SREG_GS)
+		return emulate_ud(ctxt);
+
+	return em_store_sreg(ctxt, ctxt->modrm_reg);
+}
+
 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
 {
 	u16 sel = ctxt->src.val;
@@ -3677,6 +3698,11 @@ static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
 	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
 }
 
+static int em_sldt(struct x86_emulate_ctxt *ctxt)
+{
+	return em_store_sreg(ctxt, VCPU_SREG_LDTR);
+}
+
 static int em_lldt(struct x86_emulate_ctxt *ctxt)
 {
 	u16 sel = ctxt->src.val;
@@ -3686,6 +3712,11 @@ static int em_lldt(struct x86_emulate_ctxt *ctxt)
 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
 }
 
+static int em_str(struct x86_emulate_ctxt *ctxt)
+{
+	return em_store_sreg(ctxt, VCPU_SREG_TR);
+}
+
 static int em_ltr(struct x86_emulate_ctxt *ctxt)
 {
 	u16 sel = ctxt->src.val;
@@ -3738,6 +3769,10 @@ static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
 {
 	struct desc_ptr desc_ptr;
 
+	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
+	    ctxt->ops->cpl(ctxt) > 0)
+		return emulate_gp(ctxt, 0);
+
 	if (ctxt->mode == X86EMUL_MODE_PROT64)
 		ctxt->op_bytes = 8;
 	get(ctxt, &desc_ptr);
@@ -3797,6 +3832,10 @@ static int em_lidt(struct x86_emulate_ctxt *ctxt)
 
 static int em_smsw(struct x86_emulate_ctxt *ctxt)
 {
+	if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
+	    ctxt->ops->cpl(ctxt) > 0)
+		return emulate_gp(ctxt, 0);
+
 	if (ctxt->dst.type == OP_MEM)
 		ctxt->dst.bytes = 2;
 	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
@@ -4382,8 +4421,8 @@ static const struct opcode group5[] = {
 };
 
 static const struct opcode group6[] = {
-	DI(Prot | DstMem,	sldt),
-	DI(Prot | DstMem,	str),
+	II(Prot | DstMem,	   em_sldt, sldt),
+	II(Prot | DstMem,	   em_str, str),
 	II(Prot | Priv | SrcMem16, em_lldt, lldt),
 	II(Prot | Priv | SrcMem16, em_ltr, ltr),
 	N, N, N, N,
@@ -4414,10 +4453,20 @@ static const struct opcode group8[] = {
 	F(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
 };
 
+/*
+ * The "memory" destination is actually always a register, since we come
+ * from the register case of group9.
+ */
+static const struct gprefix pfx_0f_c7_7 = {
+	N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
+};
+
+
 static const struct group_dual group9 = { {
 	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
 }, {
-	N, N, N, N, N, N, N, N,
+	N, N, N, N, N, N, N,
+	GP(0, &pfx_0f_c7_7),
 } };
 
 static const struct opcode group11[] = {
@@ -5335,9 +5384,9 @@ static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
 	if (!(ctxt->d & ByteOp))
 		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
 
-	asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
+	asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
 	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
-	      [fastop]"+S"(fop), ASM_CALL_CONSTRAINT
+	      [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
 	    : "c"(ctxt->src2.val));
 
 	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c
index 5c24811e8b0b..f171051eecf3 100644
--- a/arch/x86/kvm/irq.c
+++ b/arch/x86/kvm/irq.c
@@ -79,7 +79,7 @@ int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v)
 	if (kvm_cpu_has_extint(v))
 		return 1;
 
-	if (kvm_vcpu_apicv_active(v))
+	if (!is_guest_mode(v) && kvm_vcpu_apicv_active(v))
 		return 0;
 
 	return kvm_apic_has_interrupt(v) != -1; /* LAPIC */
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index e2c1fb8d35ce..391dda8d43b7 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -364,32 +364,41 @@ static u8 count_vectors(void *bitmap)
 	return count;
 }
 
-int __kvm_apic_update_irr(u32 *pir, void *regs)
+bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
 {
 	u32 i, vec;
-	u32 pir_val, irr_val;
-	int max_irr = -1;
+	u32 pir_val, irr_val, prev_irr_val;
+	int max_updated_irr;
+
+	max_updated_irr = -1;
+	*max_irr = -1;
 
 	for (i = vec = 0; i <= 7; i++, vec += 32) {
 		pir_val = READ_ONCE(pir[i]);
 		irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
 		if (pir_val) {
+			prev_irr_val = irr_val;
 			irr_val |= xchg(&pir[i], 0);
 			*((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
+			if (prev_irr_val != irr_val) {
+				max_updated_irr =
+					__fls(irr_val ^ prev_irr_val) + vec;
+			}
 		}
 		if (irr_val)
-			max_irr = __fls(irr_val) + vec;
+			*max_irr = __fls(irr_val) + vec;
 	}
 
-	return max_irr;
+	return ((max_updated_irr != -1) &&
+		(max_updated_irr == *max_irr));
 }
 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
 
-int kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir)
+bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
 {
 	struct kvm_lapic *apic = vcpu->arch.apic;
 
-	return __kvm_apic_update_irr(pir, apic->regs);
+	return __kvm_apic_update_irr(pir, apic->regs, max_irr);
 }
 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
 
@@ -581,7 +590,7 @@ static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 {
 	int highest_irr;
-	if (kvm_x86_ops->sync_pir_to_irr && apic->vcpu->arch.apicv_active)
+	if (apic->vcpu->arch.apicv_active)
 		highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
 	else
 		highest_irr = apic_find_highest_irr(apic);
@@ -1993,14 +2002,13 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
 
 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
 {
-	struct kvm_lapic *apic;
+	struct kvm_lapic *apic = vcpu->arch.apic;
 	int i;
 
-	apic_debug("%s\n", __func__);
+	if (!apic)
+		return;
 
-	ASSERT(vcpu);
-	apic = vcpu->arch.apic;
-	ASSERT(apic != NULL);
+	apic_debug("%s\n", __func__);
 
 	/* Stop the timer in case it's a reset to an active apic */
 	hrtimer_cancel(&apic->lapic_timer.timer);
@@ -2156,7 +2164,6 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
 	 */
 	vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
 	static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
-	kvm_lapic_reset(vcpu, false);
 	kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
 
 	return 0;
@@ -2560,7 +2567,6 @@ void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
 
 	pe = xchg(&apic->pending_events, 0);
 	if (test_bit(KVM_APIC_INIT, &pe)) {
-		kvm_lapic_reset(vcpu, true);
 		kvm_vcpu_reset(vcpu, true);
 		if (kvm_vcpu_is_bsp(apic->vcpu))
 			vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 4b9935a38347..56c36014f7b7 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -75,8 +75,8 @@ int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 			   int short_hand, unsigned int dest, int dest_mode);
 
-int __kvm_apic_update_irr(u32 *pir, void *regs);
-int kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir);
+bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr);
+bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr);
 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu);
 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
 		     struct dest_map *dest_map);
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2b8eb4da4d08..763bb3bade63 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -42,6 +42,7 @@
 #include <linux/kern_levels.h>
 
 #include <asm/page.h>
+#include <asm/pat.h>
 #include <asm/cmpxchg.h>
 #include <asm/io.h>
 #include <asm/vmx.h>
@@ -381,7 +382,7 @@ void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
 }
 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
 
-void kvm_mmu_clear_all_pte_masks(void)
+static void kvm_mmu_clear_all_pte_masks(void)
 {
 	shadow_user_mask = 0;
 	shadow_accessed_mask = 0;
@@ -2708,7 +2709,18 @@ static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
 static bool kvm_is_mmio_pfn(kvm_pfn_t pfn)
 {
 	if (pfn_valid(pfn))
-		return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn));
+		return !is_zero_pfn(pfn) && PageReserved(pfn_to_page(pfn)) &&
+			/*
+			 * Some reserved pages, such as those from NVDIMM
+			 * DAX devices, are not for MMIO, and can be mapped
+			 * with cached memory type for better performance.
+			 * However, the above check misconceives those pages
+			 * as MMIO, and results in KVM mapping them with UC
+			 * memory type, which would hurt the performance.
+			 * Therefore, we check the host memory type in addition
+			 * and only treat UC/UC-/WC pages as MMIO.
+			 */
+			(!pat_enabled() || pat_pfn_immune_to_uc_mtrr(pfn));
 
 	return true;
 }
@@ -2758,8 +2770,10 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 	else
 		pte_access &= ~ACC_WRITE_MASK;
 
+	if (!kvm_is_mmio_pfn(pfn))
+		spte |= shadow_me_mask;
+
 	spte |= (u64)pfn << PAGE_SHIFT;
-	spte |= shadow_me_mask;
 
 	if (pte_access & ACC_WRITE_MASK) {
 
@@ -3017,7 +3031,7 @@ static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
 		return RET_PF_RETRY;
 	}
 
-	return -EFAULT;
+	return RET_PF_EMULATE;
 }
 
 static void transparent_hugepage_adjust(struct kvm_vcpu *vcpu,
@@ -4951,6 +4965,16 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u64 error_code,
 	if (mmio_info_in_cache(vcpu, cr2, direct))
 		emulation_type = 0;
 emulate:
+	/*
+	 * On AMD platforms, under certain conditions insn_len may be zero on #NPF.
+	 * This can happen if a guest gets a page-fault on data access but the HW
+	 * table walker is not able to read the instruction page (e.g instruction
+	 * page is not present in memory). In those cases we simply restart the
+	 * guest.
+	 */
+	if (unlikely(insn && !insn_len))
+		return 1;
+
 	er = x86_emulate_instruction(vcpu, cr2, emulation_type, insn, insn_len);
 
 	switch (er) {
@@ -5058,7 +5082,7 @@ void kvm_mmu_uninit_vm(struct kvm *kvm)
 typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head);
 
 /* The caller should hold mmu-lock before calling this function. */
-static bool
+static __always_inline bool
 slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
 			slot_level_handler fn, int start_level, int end_level,
 			gfn_t start_gfn, gfn_t end_gfn, bool lock_flush_tlb)
@@ -5088,7 +5112,7 @@ slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot,
 	return flush;
 }
 
-static bool
+static __always_inline bool
 slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 		  slot_level_handler fn, int start_level, int end_level,
 		  bool lock_flush_tlb)
@@ -5099,7 +5123,7 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 			lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 		      slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5107,7 +5131,7 @@ slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 			slot_level_handler fn, bool lock_flush_tlb)
 {
@@ -5115,7 +5139,7 @@ slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot,
 				 PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb);
 }
 
-static bool
+static __always_inline bool
 slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot,
 		 slot_level_handler fn, bool lock_flush_tlb)
 {
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index d22ddbdf5e6e..1272861e77b9 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -19,7 +19,7 @@
 
 #include <linux/ratelimit.h>
 
-char const *audit_point_name[] = {
+static char const *audit_point_name[] = {
 	"pre page fault",
 	"post page fault",
 	"pre pte write",
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index f40d0da1f1d3..be9c839e2c89 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -37,6 +37,10 @@
 #include <linux/amd-iommu.h>
 #include <linux/hashtable.h>
 #include <linux/frame.h>
+#include <linux/psp-sev.h>
+#include <linux/file.h>
+#include <linux/pagemap.h>
+#include <linux/swap.h>
 
 #include <asm/apic.h>
 #include <asm/perf_event.h>
@@ -45,6 +49,7 @@
 #include <asm/debugreg.h>
 #include <asm/kvm_para.h>
 #include <asm/irq_remapping.h>
+#include <asm/microcode.h>
 #include <asm/nospec-branch.h>
 
 #include <asm/virtext.h>
@@ -174,6 +179,8 @@ struct vcpu_svm {
 	uint64_t sysenter_eip;
 	uint64_t tsc_aux;
 
+	u64 msr_decfg;
+
 	u64 next_rip;
 
 	u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
@@ -184,6 +191,8 @@ struct vcpu_svm {
 		u64 gs_base;
 	} host;
 
+	u64 spec_ctrl;
+
 	u32 *msrpm;
 
 	ulong nmi_iret_rip;
@@ -212,6 +221,9 @@ struct vcpu_svm {
 	 */
 	struct list_head ir_list;
 	spinlock_t ir_list_lock;
+
+	/* which host CPU was used for running this vcpu */
+	unsigned int last_cpu;
 };
 
 /*
@@ -249,6 +261,8 @@ static const struct svm_direct_access_msrs {
 	{ .index = MSR_CSTAR,				.always = true  },
 	{ .index = MSR_SYSCALL_MASK,			.always = true  },
 #endif
+	{ .index = MSR_IA32_SPEC_CTRL,			.always = false },
+	{ .index = MSR_IA32_PRED_CMD,			.always = false },
 	{ .index = MSR_IA32_LASTBRANCHFROMIP,		.always = false },
 	{ .index = MSR_IA32_LASTBRANCHTOIP,		.always = false },
 	{ .index = MSR_IA32_LASTINTFROMIP,		.always = false },
@@ -285,8 +299,14 @@ module_param(vls, int, 0444);
 static int vgif = true;
 module_param(vgif, int, 0444);
 
+/* enable/disable SEV support */
+static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
+module_param(sev, int, 0444);
+
+static u8 rsm_ins_bytes[] = "\x0f\xaa";
+
 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
-static void svm_flush_tlb(struct kvm_vcpu *vcpu);
+static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
 static void svm_complete_interrupts(struct vcpu_svm *svm);
 
 static int nested_svm_exit_handled(struct vcpu_svm *svm);
@@ -320,6 +340,38 @@ enum {
 
 #define VMCB_AVIC_APIC_BAR_MASK		0xFFFFFFFFFF000ULL
 
+static unsigned int max_sev_asid;
+static unsigned int min_sev_asid;
+static unsigned long *sev_asid_bitmap;
+#define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
+
+struct enc_region {
+	struct list_head list;
+	unsigned long npages;
+	struct page **pages;
+	unsigned long uaddr;
+	unsigned long size;
+};
+
+static inline bool svm_sev_enabled(void)
+{
+	return max_sev_asid;
+}
+
+static inline bool sev_guest(struct kvm *kvm)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+
+	return sev->active;
+}
+
+static inline int sev_get_asid(struct kvm *kvm)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+
+	return sev->asid;
+}
+
 static inline void mark_all_dirty(struct vmcb *vmcb)
 {
 	vmcb->control.clean = 0;
@@ -526,9 +578,14 @@ struct svm_cpu_data {
 	u64 asid_generation;
 	u32 max_asid;
 	u32 next_asid;
+	u32 min_asid;
 	struct kvm_ldttss_desc *tss_desc;
 
 	struct page *save_area;
+	struct vmcb *current_vmcb;
+
+	/* index = sev_asid, value = vmcb pointer */
+	struct vmcb **sev_vmcbs;
 };
 
 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
@@ -783,6 +840,7 @@ static int svm_hardware_enable(void)
 	sd->asid_generation = 1;
 	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 	sd->next_asid = sd->max_asid + 1;
+	sd->min_asid = max_sev_asid + 1;
 
 	gdt = get_current_gdt_rw();
 	sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
@@ -841,6 +899,7 @@ static void svm_cpu_uninit(int cpu)
 		return;
 
 	per_cpu(svm_data, raw_smp_processor_id()) = NULL;
+	kfree(sd->sev_vmcbs);
 	__free_page(sd->save_area);
 	kfree(sd);
 }
@@ -854,11 +913,18 @@ static int svm_cpu_init(int cpu)
 	if (!sd)
 		return -ENOMEM;
 	sd->cpu = cpu;
-	sd->save_area = alloc_page(GFP_KERNEL);
 	r = -ENOMEM;
+	sd->save_area = alloc_page(GFP_KERNEL);
 	if (!sd->save_area)
 		goto err_1;
 
+	if (svm_sev_enabled()) {
+		r = -ENOMEM;
+		sd->sev_vmcbs = kmalloc((max_sev_asid + 1) * sizeof(void *), GFP_KERNEL);
+		if (!sd->sev_vmcbs)
+			goto err_1;
+	}
+
 	per_cpu(svm_data, cpu) = sd;
 
 	return 0;
@@ -880,6 +946,25 @@ static bool valid_msr_intercept(u32 index)
 	return false;
 }
 
+static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
+{
+	u8 bit_write;
+	unsigned long tmp;
+	u32 offset;
+	u32 *msrpm;
+
+	msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
+				      to_svm(vcpu)->msrpm;
+
+	offset    = svm_msrpm_offset(msr);
+	bit_write = 2 * (msr & 0x0f) + 1;
+	tmp       = msrpm[offset];
+
+	BUG_ON(offset == MSR_INVALID);
+
+	return !!test_bit(bit_write,  &tmp);
+}
+
 static void set_msr_interception(u32 *msrpm, unsigned msr,
 				 int read, int write)
 {
@@ -1046,6 +1131,48 @@ static int avic_ga_log_notifier(u32 ga_tag)
 	return 0;
 }
 
+static __init int sev_hardware_setup(void)
+{
+	struct sev_user_data_status *status;
+	int rc;
+
+	/* Maximum number of encrypted guests supported simultaneously */
+	max_sev_asid = cpuid_ecx(0x8000001F);
+
+	if (!max_sev_asid)
+		return 1;
+
+	/* Minimum ASID value that should be used for SEV guest */
+	min_sev_asid = cpuid_edx(0x8000001F);
+
+	/* Initialize SEV ASID bitmap */
+	sev_asid_bitmap = kcalloc(BITS_TO_LONGS(max_sev_asid),
+				sizeof(unsigned long), GFP_KERNEL);
+	if (!sev_asid_bitmap)
+		return 1;
+
+	status = kmalloc(sizeof(*status), GFP_KERNEL);
+	if (!status)
+		return 1;
+
+	/*
+	 * Check SEV platform status.
+	 *
+	 * PLATFORM_STATUS can be called in any state, if we failed to query
+	 * the PLATFORM status then either PSP firmware does not support SEV
+	 * feature or SEV firmware is dead.
+	 */
+	rc = sev_platform_status(status, NULL);
+	if (rc)
+		goto err;
+
+	pr_info("SEV supported\n");
+
+err:
+	kfree(status);
+	return rc;
+}
+
 static __init int svm_hardware_setup(void)
 {
 	int cpu;
@@ -1081,6 +1208,17 @@ static __init int svm_hardware_setup(void)
 		kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
 	}
 
+	if (sev) {
+		if (boot_cpu_has(X86_FEATURE_SEV) &&
+		    IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
+			r = sev_hardware_setup();
+			if (r)
+				sev = false;
+		} else {
+			sev = false;
+		}
+	}
+
 	for_each_possible_cpu(cpu) {
 		r = svm_cpu_init(cpu);
 		if (r)
@@ -1142,6 +1280,9 @@ static __exit void svm_hardware_unsetup(void)
 {
 	int cpu;
 
+	if (svm_sev_enabled())
+		kfree(sev_asid_bitmap);
+
 	for_each_possible_cpu(cpu)
 		svm_cpu_uninit(cpu);
 
@@ -1247,6 +1388,7 @@ static void init_vmcb(struct vcpu_svm *svm)
 	set_intercept(svm, INTERCEPT_SKINIT);
 	set_intercept(svm, INTERCEPT_WBINVD);
 	set_intercept(svm, INTERCEPT_XSETBV);
+	set_intercept(svm, INTERCEPT_RSM);
 
 	if (!kvm_mwait_in_guest()) {
 		set_intercept(svm, INTERCEPT_MONITOR);
@@ -1294,7 +1436,7 @@ static void init_vmcb(struct vcpu_svm *svm)
 
 	if (npt_enabled) {
 		/* Setup VMCB for Nested Paging */
-		control->nested_ctl = 1;
+		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
 		clr_intercept(svm, INTERCEPT_INVLPG);
 		clr_exception_intercept(svm, PF_VECTOR);
 		clr_cr_intercept(svm, INTERCEPT_CR3_READ);
@@ -1332,6 +1474,11 @@ static void init_vmcb(struct vcpu_svm *svm)
 		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
 	}
 
+	if (sev_guest(svm->vcpu.kvm)) {
+		svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
+		clr_exception_intercept(svm, UD_VECTOR);
+	}
+
 	mark_all_dirty(svm->vmcb);
 
 	enable_gif(svm);
@@ -1414,6 +1561,179 @@ static int avic_init_backing_page(struct kvm_vcpu *vcpu)
 	return 0;
 }
 
+static void __sev_asid_free(int asid)
+{
+	struct svm_cpu_data *sd;
+	int cpu, pos;
+
+	pos = asid - 1;
+	clear_bit(pos, sev_asid_bitmap);
+
+	for_each_possible_cpu(cpu) {
+		sd = per_cpu(svm_data, cpu);
+		sd->sev_vmcbs[pos] = NULL;
+	}
+}
+
+static void sev_asid_free(struct kvm *kvm)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+
+	__sev_asid_free(sev->asid);
+}
+
+static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
+{
+	struct sev_data_decommission *decommission;
+	struct sev_data_deactivate *data;
+
+	if (!handle)
+		return;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return;
+
+	/* deactivate handle */
+	data->handle = handle;
+	sev_guest_deactivate(data, NULL);
+
+	wbinvd_on_all_cpus();
+	sev_guest_df_flush(NULL);
+	kfree(data);
+
+	decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
+	if (!decommission)
+		return;
+
+	/* decommission handle */
+	decommission->handle = handle;
+	sev_guest_decommission(decommission, NULL);
+
+	kfree(decommission);
+}
+
+static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
+				    unsigned long ulen, unsigned long *n,
+				    int write)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	unsigned long npages, npinned, size;
+	unsigned long locked, lock_limit;
+	struct page **pages;
+	int first, last;
+
+	/* Calculate number of pages. */
+	first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
+	last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
+	npages = (last - first + 1);
+
+	locked = sev->pages_locked + npages;
+	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+	if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
+		pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
+		return NULL;
+	}
+
+	/* Avoid using vmalloc for smaller buffers. */
+	size = npages * sizeof(struct page *);
+	if (size > PAGE_SIZE)
+		pages = vmalloc(size);
+	else
+		pages = kmalloc(size, GFP_KERNEL);
+
+	if (!pages)
+		return NULL;
+
+	/* Pin the user virtual address. */
+	npinned = get_user_pages_fast(uaddr, npages, write ? FOLL_WRITE : 0, pages);
+	if (npinned != npages) {
+		pr_err("SEV: Failure locking %lu pages.\n", npages);
+		goto err;
+	}
+
+	*n = npages;
+	sev->pages_locked = locked;
+
+	return pages;
+
+err:
+	if (npinned > 0)
+		release_pages(pages, npinned);
+
+	kvfree(pages);
+	return NULL;
+}
+
+static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
+			     unsigned long npages)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+
+	release_pages(pages, npages);
+	kvfree(pages);
+	sev->pages_locked -= npages;
+}
+
+static void sev_clflush_pages(struct page *pages[], unsigned long npages)
+{
+	uint8_t *page_virtual;
+	unsigned long i;
+
+	if (npages == 0 || pages == NULL)
+		return;
+
+	for (i = 0; i < npages; i++) {
+		page_virtual = kmap_atomic(pages[i]);
+		clflush_cache_range(page_virtual, PAGE_SIZE);
+		kunmap_atomic(page_virtual);
+	}
+}
+
+static void __unregister_enc_region_locked(struct kvm *kvm,
+					   struct enc_region *region)
+{
+	/*
+	 * The guest may change the memory encryption attribute from C=0 -> C=1
+	 * or vice versa for this memory range. Lets make sure caches are
+	 * flushed to ensure that guest data gets written into memory with
+	 * correct C-bit.
+	 */
+	sev_clflush_pages(region->pages, region->npages);
+
+	sev_unpin_memory(kvm, region->pages, region->npages);
+	list_del(&region->list);
+	kfree(region);
+}
+
+static void sev_vm_destroy(struct kvm *kvm)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct list_head *head = &sev->regions_list;
+	struct list_head *pos, *q;
+
+	if (!sev_guest(kvm))
+		return;
+
+	mutex_lock(&kvm->lock);
+
+	/*
+	 * if userspace was terminated before unregistering the memory regions
+	 * then lets unpin all the registered memory.
+	 */
+	if (!list_empty(head)) {
+		list_for_each_safe(pos, q, head) {
+			__unregister_enc_region_locked(kvm,
+				list_entry(pos, struct enc_region, list));
+		}
+	}
+
+	mutex_unlock(&kvm->lock);
+
+	sev_unbind_asid(kvm, sev->handle);
+	sev_asid_free(kvm);
+}
+
 static void avic_vm_destroy(struct kvm *kvm)
 {
 	unsigned long flags;
@@ -1432,6 +1752,12 @@ static void avic_vm_destroy(struct kvm *kvm)
 	spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 }
 
+static void svm_vm_destroy(struct kvm *kvm)
+{
+	avic_vm_destroy(kvm);
+	sev_vm_destroy(kvm);
+}
+
 static int avic_vm_init(struct kvm *kvm)
 {
 	unsigned long flags;
@@ -1582,6 +1908,9 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	u32 dummy;
 	u32 eax = 1;
 
+	vcpu->arch.microcode_version = 0x01000065;
+	svm->spec_ctrl = 0;
+
 	if (!init_event) {
 		svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
 					   MSR_IA32_APICBASE_ENABLE;
@@ -1703,11 +2032,17 @@ static void svm_free_vcpu(struct kvm_vcpu *vcpu)
 	__free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
 	kvm_vcpu_uninit(vcpu);
 	kmem_cache_free(kvm_vcpu_cache, svm);
+	/*
+	 * The vmcb page can be recycled, causing a false negative in
+	 * svm_vcpu_load(). So do a full IBPB now.
+	 */
+	indirect_branch_prediction_barrier();
 }
 
 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
+	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
 	int i;
 
 	if (unlikely(cpu != vcpu->cpu)) {
@@ -1736,6 +2071,10 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 	if (static_cpu_has(X86_FEATURE_RDTSCP))
 		wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
 
+	if (sd->current_vmcb != svm->vmcb) {
+		sd->current_vmcb = svm->vmcb;
+		indirect_branch_prediction_barrier();
+	}
 	avic_vcpu_load(vcpu, cpu);
 }
 
@@ -2030,7 +2369,7 @@ static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 		return 1;
 
 	if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
-		svm_flush_tlb(vcpu);
+		svm_flush_tlb(vcpu, true);
 
 	vcpu->arch.cr4 = cr4;
 	if (!npt_enabled)
@@ -2089,7 +2428,7 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
 {
 	if (sd->next_asid > sd->max_asid) {
 		++sd->asid_generation;
-		sd->next_asid = 1;
+		sd->next_asid = sd->min_asid;
 		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
 	}
 
@@ -2137,22 +2476,24 @@ static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
 
 static int pf_interception(struct vcpu_svm *svm)
 {
-	u64 fault_address = svm->vmcb->control.exit_info_2;
+	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
 	u64 error_code = svm->vmcb->control.exit_info_1;
 
 	return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
-			svm->vmcb->control.insn_bytes,
+			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
+			svm->vmcb->control.insn_bytes : NULL,
 			svm->vmcb->control.insn_len);
 }
 
 static int npf_interception(struct vcpu_svm *svm)
 {
-	u64 fault_address = svm->vmcb->control.exit_info_2;
+	u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
 	u64 error_code = svm->vmcb->control.exit_info_1;
 
 	trace_kvm_page_fault(fault_address, error_code);
 	return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
-			svm->vmcb->control.insn_bytes,
+			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
+			svm->vmcb->control.insn_bytes : NULL,
 			svm->vmcb->control.insn_len);
 }
 
@@ -2379,7 +2720,7 @@ static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
 
 	svm->vmcb->control.nested_cr3 = __sme_set(root);
 	mark_dirty(svm->vmcb, VMCB_NPT);
-	svm_flush_tlb(vcpu);
+	svm_flush_tlb(vcpu, true);
 }
 
 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
@@ -2921,7 +3262,8 @@ static bool nested_vmcb_checks(struct vmcb *vmcb)
 	if (vmcb->control.asid == 0)
 		return false;
 
-	if (vmcb->control.nested_ctl && !npt_enabled)
+	if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
+	    !npt_enabled)
 		return false;
 
 	return true;
@@ -2935,7 +3277,7 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
 	else
 		svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
 
-	if (nested_vmcb->control.nested_ctl) {
+	if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
 		kvm_mmu_unload(&svm->vcpu);
 		svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
 		nested_svm_init_mmu_context(&svm->vcpu);
@@ -2983,7 +3325,7 @@ static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
 	svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
 	svm->nested.intercept            = nested_vmcb->control.intercept;
 
-	svm_flush_tlb(&svm->vcpu);
+	svm_flush_tlb(&svm->vcpu, true);
 	svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
 	if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
 		svm->vcpu.arch.hflags |= HF_VINTR_MASK;
@@ -3364,6 +3706,12 @@ static int emulate_on_interception(struct vcpu_svm *svm)
 	return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
 }
 
+static int rsm_interception(struct vcpu_svm *svm)
+{
+	return x86_emulate_instruction(&svm->vcpu, 0, 0,
+				       rsm_ins_bytes, 2) == EMULATE_DONE;
+}
+
 static int rdpmc_interception(struct vcpu_svm *svm)
 {
 	int err;
@@ -3525,6 +3873,22 @@ static int cr8_write_interception(struct vcpu_svm *svm)
 	return 0;
 }
 
+static int svm_get_msr_feature(struct kvm_msr_entry *msr)
+{
+	msr->data = 0;
+
+	switch (msr->index) {
+	case MSR_F10H_DECFG:
+		if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
+			msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
+		break;
+	default:
+		return 1;
+	}
+
+	return 0;
+}
+
 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
@@ -3593,8 +3957,12 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_VM_CR:
 		msr_info->data = svm->nested.vm_cr_msr;
 		break;
-	case MSR_IA32_UCODE_REV:
-		msr_info->data = 0x01000065;
+	case MSR_IA32_SPEC_CTRL:
+		if (!msr_info->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
+			return 1;
+
+		msr_info->data = svm->spec_ctrl;
 		break;
 	case MSR_F15H_IC_CFG: {
 
@@ -3613,6 +3981,9 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			msr_info->data = 0x1E;
 		}
 		break;
+	case MSR_F10H_DECFG:
+		msr_info->data = svm->msr_decfg;
+		break;
 	default:
 		return kvm_get_msr_common(vcpu, msr_info);
 	}
@@ -3684,6 +4055,49 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
 	case MSR_IA32_TSC:
 		kvm_write_tsc(vcpu, msr);
 		break;
+	case MSR_IA32_SPEC_CTRL:
+		if (!msr->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS))
+			return 1;
+
+		/* The STIBP bit doesn't fault even if it's not advertised */
+		if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP))
+			return 1;
+
+		svm->spec_ctrl = data;
+
+		if (!data)
+			break;
+
+		/*
+		 * For non-nested:
+		 * When it's written (to non-zero) for the first time, pass
+		 * it through.
+		 *
+		 * For nested:
+		 * The handling of the MSR bitmap for L2 guests is done in
+		 * nested_svm_vmrun_msrpm.
+		 * We update the L1 MSR bit as well since it will end up
+		 * touching the MSR anyway now.
+		 */
+		set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
+		break;
+	case MSR_IA32_PRED_CMD:
+		if (!msr->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_IBPB))
+			return 1;
+
+		if (data & ~PRED_CMD_IBPB)
+			return 1;
+
+		if (!data)
+			break;
+
+		wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
+		if (is_guest_mode(vcpu))
+			break;
+		set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
+		break;
 	case MSR_STAR:
 		svm->vmcb->save.star = data;
 		break;
@@ -3748,6 +4162,24 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
 	case MSR_VM_IGNNE:
 		vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
 		break;
+	case MSR_F10H_DECFG: {
+		struct kvm_msr_entry msr_entry;
+
+		msr_entry.index = msr->index;
+		if (svm_get_msr_feature(&msr_entry))
+			return 1;
+
+		/* Check the supported bits */
+		if (data & ~msr_entry.data)
+			return 1;
+
+		/* Don't allow the guest to change a bit, #GP */
+		if (!msr->host_initiated && (data ^ msr_entry.data))
+			return 1;
+
+		svm->msr_decfg = data;
+		break;
+	}
 	case MSR_IA32_APICBASE:
 		if (kvm_vcpu_apicv_active(vcpu))
 			avic_update_vapic_bar(to_svm(vcpu), data);
@@ -4156,7 +4588,7 @@ static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
 	[SVM_EXIT_MWAIT]			= mwait_interception,
 	[SVM_EXIT_XSETBV]			= xsetbv_interception,
 	[SVM_EXIT_NPF]				= npf_interception,
-	[SVM_EXIT_RSM]                          = emulate_on_interception,
+	[SVM_EXIT_RSM]                          = rsm_interception,
 	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
 	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
 };
@@ -4356,12 +4788,39 @@ static void reload_tss(struct kvm_vcpu *vcpu)
 	load_TR_desc();
 }
 
+static void pre_sev_run(struct vcpu_svm *svm, int cpu)
+{
+	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
+	int asid = sev_get_asid(svm->vcpu.kvm);
+
+	/* Assign the asid allocated with this SEV guest */
+	svm->vmcb->control.asid = asid;
+
+	/*
+	 * Flush guest TLB:
+	 *
+	 * 1) when different VMCB for the same ASID is to be run on the same host CPU.
+	 * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
+	 */
+	if (sd->sev_vmcbs[asid] == svm->vmcb &&
+	    svm->last_cpu == cpu)
+		return;
+
+	svm->last_cpu = cpu;
+	sd->sev_vmcbs[asid] = svm->vmcb;
+	svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
+	mark_dirty(svm->vmcb, VMCB_ASID);
+}
+
 static void pre_svm_run(struct vcpu_svm *svm)
 {
 	int cpu = raw_smp_processor_id();
 
 	struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
 
+	if (sev_guest(svm->vcpu.kvm))
+		return pre_sev_run(svm, cpu);
+
 	/* FIXME: handle wraparound of asid_generation */
 	if (svm->asid_generation != sd->asid_generation)
 		new_asid(svm, sd);
@@ -4779,7 +5238,7 @@ static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
 	return 0;
 }
 
-static void svm_flush_tlb(struct kvm_vcpu *vcpu)
+static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
@@ -4936,6 +5395,15 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 
 	local_irq_enable();
 
+	/*
+	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
+	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
+	 * is no need to worry about the conditional branch over the wrmsr
+	 * being speculatively taken.
+	 */
+	if (svm->spec_ctrl)
+		native_wrmsrl(MSR_IA32_SPEC_CTRL, svm->spec_ctrl);
+
 	asm volatile (
 		"push %%" _ASM_BP "; \n\t"
 		"mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
@@ -5028,6 +5496,27 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 		);
 
+	/*
+	 * We do not use IBRS in the kernel. If this vCPU has used the
+	 * SPEC_CTRL MSR it may have left it on; save the value and
+	 * turn it off. This is much more efficient than blindly adding
+	 * it to the atomic save/restore list. Especially as the former
+	 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
+	 *
+	 * For non-nested case:
+	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
+	 * save it.
+	 *
+	 * For nested case:
+	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
+	 * save it.
+	 */
+	if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
+		svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
+
+	if (svm->spec_ctrl)
+		native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
+
 	/* Eliminate branch target predictions from guest mode */
 	vmexit_fill_RSB();
 
@@ -5092,7 +5581,7 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
 
 	svm->vmcb->save.cr3 = __sme_set(root);
 	mark_dirty(svm->vmcb, VMCB_CR);
-	svm_flush_tlb(vcpu);
+	svm_flush_tlb(vcpu, true);
 }
 
 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
@@ -5106,7 +5595,7 @@ static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
 	svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
 	mark_dirty(svm->vmcb, VMCB_CR);
 
-	svm_flush_tlb(vcpu);
+	svm_flush_tlb(vcpu, true);
 }
 
 static int is_disabled(void)
@@ -5192,6 +5681,12 @@ static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
 			entry->edx |= SVM_FEATURE_NPT;
 
 		break;
+	case 0x8000001F:
+		/* Support memory encryption cpuid if host supports it */
+		if (boot_cpu_has(X86_FEATURE_SEV))
+			cpuid(0x8000001f, &entry->eax, &entry->ebx,
+				&entry->ecx, &entry->edx);
+
 	}
 }
 
@@ -5220,6 +5715,11 @@ static bool svm_xsaves_supported(void)
 	return false;
 }
 
+static bool svm_umip_emulated(void)
+{
+	return false;
+}
+
 static bool svm_has_wbinvd_exit(void)
 {
 	return true;
@@ -5521,46 +6021,871 @@ static int enable_smi_window(struct kvm_vcpu *vcpu)
 	return 0;
 }
 
-static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
-	.cpu_has_kvm_support = has_svm,
-	.disabled_by_bios = is_disabled,
-	.hardware_setup = svm_hardware_setup,
-	.hardware_unsetup = svm_hardware_unsetup,
-	.check_processor_compatibility = svm_check_processor_compat,
-	.hardware_enable = svm_hardware_enable,
-	.hardware_disable = svm_hardware_disable,
-	.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
-	.cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
+static int sev_asid_new(void)
+{
+	int pos;
 
-	.vcpu_create = svm_create_vcpu,
-	.vcpu_free = svm_free_vcpu,
-	.vcpu_reset = svm_vcpu_reset,
+	/*
+	 * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
+	 */
+	pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
+	if (pos >= max_sev_asid)
+		return -EBUSY;
 
-	.vm_init = avic_vm_init,
-	.vm_destroy = avic_vm_destroy,
+	set_bit(pos, sev_asid_bitmap);
+	return pos + 1;
+}
 
-	.prepare_guest_switch = svm_prepare_guest_switch,
-	.vcpu_load = svm_vcpu_load,
-	.vcpu_put = svm_vcpu_put,
-	.vcpu_blocking = svm_vcpu_blocking,
-	.vcpu_unblocking = svm_vcpu_unblocking,
+static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	int asid, ret;
 
-	.update_bp_intercept = update_bp_intercept,
-	.get_msr = svm_get_msr,
-	.set_msr = svm_set_msr,
-	.get_segment_base = svm_get_segment_base,
-	.get_segment = svm_get_segment,
-	.set_segment = svm_set_segment,
-	.get_cpl = svm_get_cpl,
-	.get_cs_db_l_bits = kvm_get_cs_db_l_bits,
-	.decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
-	.decache_cr3 = svm_decache_cr3,
-	.decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
-	.set_cr0 = svm_set_cr0,
-	.set_cr3 = svm_set_cr3,
-	.set_cr4 = svm_set_cr4,
-	.set_efer = svm_set_efer,
-	.get_idt = svm_get_idt,
+	ret = -EBUSY;
+	asid = sev_asid_new();
+	if (asid < 0)
+		return ret;
+
+	ret = sev_platform_init(&argp->error);
+	if (ret)
+		goto e_free;
+
+	sev->active = true;
+	sev->asid = asid;
+	INIT_LIST_HEAD(&sev->regions_list);
+
+	return 0;
+
+e_free:
+	__sev_asid_free(asid);
+	return ret;
+}
+
+static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
+{
+	struct sev_data_activate *data;
+	int asid = sev_get_asid(kvm);
+	int ret;
+
+	wbinvd_on_all_cpus();
+
+	ret = sev_guest_df_flush(error);
+	if (ret)
+		return ret;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* activate ASID on the given handle */
+	data->handle = handle;
+	data->asid   = asid;
+	ret = sev_guest_activate(data, error);
+	kfree(data);
+
+	return ret;
+}
+
+static int __sev_issue_cmd(int fd, int id, void *data, int *error)
+{
+	struct fd f;
+	int ret;
+
+	f = fdget(fd);
+	if (!f.file)
+		return -EBADF;
+
+	ret = sev_issue_cmd_external_user(f.file, id, data, error);
+
+	fdput(f);
+	return ret;
+}
+
+static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+
+	return __sev_issue_cmd(sev->fd, id, data, error);
+}
+
+static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct sev_data_launch_start *start;
+	struct kvm_sev_launch_start params;
+	void *dh_blob, *session_blob;
+	int *error = &argp->error;
+	int ret;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
+		return -EFAULT;
+
+	start = kzalloc(sizeof(*start), GFP_KERNEL);
+	if (!start)
+		return -ENOMEM;
+
+	dh_blob = NULL;
+	if (params.dh_uaddr) {
+		dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
+		if (IS_ERR(dh_blob)) {
+			ret = PTR_ERR(dh_blob);
+			goto e_free;
+		}
+
+		start->dh_cert_address = __sme_set(__pa(dh_blob));
+		start->dh_cert_len = params.dh_len;
+	}
+
+	session_blob = NULL;
+	if (params.session_uaddr) {
+		session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
+		if (IS_ERR(session_blob)) {
+			ret = PTR_ERR(session_blob);
+			goto e_free_dh;
+		}
+
+		start->session_address = __sme_set(__pa(session_blob));
+		start->session_len = params.session_len;
+	}
+
+	start->handle = params.handle;
+	start->policy = params.policy;
+
+	/* create memory encryption context */
+	ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
+	if (ret)
+		goto e_free_session;
+
+	/* Bind ASID to this guest */
+	ret = sev_bind_asid(kvm, start->handle, error);
+	if (ret)
+		goto e_free_session;
+
+	/* return handle to userspace */
+	params.handle = start->handle;
+	if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
+		sev_unbind_asid(kvm, start->handle);
+		ret = -EFAULT;
+		goto e_free_session;
+	}
+
+	sev->handle = start->handle;
+	sev->fd = argp->sev_fd;
+
+e_free_session:
+	kfree(session_blob);
+e_free_dh:
+	kfree(dh_blob);
+e_free:
+	kfree(start);
+	return ret;
+}
+
+static int get_num_contig_pages(int idx, struct page **inpages,
+				unsigned long npages)
+{
+	unsigned long paddr, next_paddr;
+	int i = idx + 1, pages = 1;
+
+	/* find the number of contiguous pages starting from idx */
+	paddr = __sme_page_pa(inpages[idx]);
+	while (i < npages) {
+		next_paddr = __sme_page_pa(inpages[i++]);
+		if ((paddr + PAGE_SIZE) == next_paddr) {
+			pages++;
+			paddr = next_paddr;
+			continue;
+		}
+		break;
+	}
+
+	return pages;
+}
+
+static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	unsigned long vaddr, vaddr_end, next_vaddr, npages, size;
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct kvm_sev_launch_update_data params;
+	struct sev_data_launch_update_data *data;
+	struct page **inpages;
+	int i, ret, pages;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
+		return -EFAULT;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	vaddr = params.uaddr;
+	size = params.len;
+	vaddr_end = vaddr + size;
+
+	/* Lock the user memory. */
+	inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
+	if (!inpages) {
+		ret = -ENOMEM;
+		goto e_free;
+	}
+
+	/*
+	 * The LAUNCH_UPDATE command will perform in-place encryption of the
+	 * memory content (i.e it will write the same memory region with C=1).
+	 * It's possible that the cache may contain the data with C=0, i.e.,
+	 * unencrypted so invalidate it first.
+	 */
+	sev_clflush_pages(inpages, npages);
+
+	for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
+		int offset, len;
+
+		/*
+		 * If the user buffer is not page-aligned, calculate the offset
+		 * within the page.
+		 */
+		offset = vaddr & (PAGE_SIZE - 1);
+
+		/* Calculate the number of pages that can be encrypted in one go. */
+		pages = get_num_contig_pages(i, inpages, npages);
+
+		len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
+
+		data->handle = sev->handle;
+		data->len = len;
+		data->address = __sme_page_pa(inpages[i]) + offset;
+		ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
+		if (ret)
+			goto e_unpin;
+
+		size -= len;
+		next_vaddr = vaddr + len;
+	}
+
+e_unpin:
+	/* content of memory is updated, mark pages dirty */
+	for (i = 0; i < npages; i++) {
+		set_page_dirty_lock(inpages[i]);
+		mark_page_accessed(inpages[i]);
+	}
+	/* unlock the user pages */
+	sev_unpin_memory(kvm, inpages, npages);
+e_free:
+	kfree(data);
+	return ret;
+}
+
+static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	void __user *measure = (void __user *)(uintptr_t)argp->data;
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct sev_data_launch_measure *data;
+	struct kvm_sev_launch_measure params;
+	void __user *p = NULL;
+	void *blob = NULL;
+	int ret;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	if (copy_from_user(&params, measure, sizeof(params)))
+		return -EFAULT;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* User wants to query the blob length */
+	if (!params.len)
+		goto cmd;
+
+	p = (void __user *)(uintptr_t)params.uaddr;
+	if (p) {
+		if (params.len > SEV_FW_BLOB_MAX_SIZE) {
+			ret = -EINVAL;
+			goto e_free;
+		}
+
+		ret = -ENOMEM;
+		blob = kmalloc(params.len, GFP_KERNEL);
+		if (!blob)
+			goto e_free;
+
+		data->address = __psp_pa(blob);
+		data->len = params.len;
+	}
+
+cmd:
+	data->handle = sev->handle;
+	ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
+
+	/*
+	 * If we query the session length, FW responded with expected data.
+	 */
+	if (!params.len)
+		goto done;
+
+	if (ret)
+		goto e_free_blob;
+
+	if (blob) {
+		if (copy_to_user(p, blob, params.len))
+			ret = -EFAULT;
+	}
+
+done:
+	params.len = data->len;
+	if (copy_to_user(measure, &params, sizeof(params)))
+		ret = -EFAULT;
+e_free_blob:
+	kfree(blob);
+e_free:
+	kfree(data);
+	return ret;
+}
+
+static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct sev_data_launch_finish *data;
+	int ret;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->handle = sev->handle;
+	ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
+
+	kfree(data);
+	return ret;
+}
+
+static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct kvm_sev_guest_status params;
+	struct sev_data_guest_status *data;
+	int ret;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->handle = sev->handle;
+	ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
+	if (ret)
+		goto e_free;
+
+	params.policy = data->policy;
+	params.state = data->state;
+	params.handle = data->handle;
+
+	if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
+		ret = -EFAULT;
+e_free:
+	kfree(data);
+	return ret;
+}
+
+static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
+			       unsigned long dst, int size,
+			       int *error, bool enc)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct sev_data_dbg *data;
+	int ret;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	data->handle = sev->handle;
+	data->dst_addr = dst;
+	data->src_addr = src;
+	data->len = size;
+
+	ret = sev_issue_cmd(kvm,
+			    enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
+			    data, error);
+	kfree(data);
+	return ret;
+}
+
+static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
+			     unsigned long dst_paddr, int sz, int *err)
+{
+	int offset;
+
+	/*
+	 * Its safe to read more than we are asked, caller should ensure that
+	 * destination has enough space.
+	 */
+	src_paddr = round_down(src_paddr, 16);
+	offset = src_paddr & 15;
+	sz = round_up(sz + offset, 16);
+
+	return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
+}
+
+static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
+				  unsigned long __user dst_uaddr,
+				  unsigned long dst_paddr,
+				  int size, int *err)
+{
+	struct page *tpage = NULL;
+	int ret, offset;
+
+	/* if inputs are not 16-byte then use intermediate buffer */
+	if (!IS_ALIGNED(dst_paddr, 16) ||
+	    !IS_ALIGNED(paddr,     16) ||
+	    !IS_ALIGNED(size,      16)) {
+		tpage = (void *)alloc_page(GFP_KERNEL);
+		if (!tpage)
+			return -ENOMEM;
+
+		dst_paddr = __sme_page_pa(tpage);
+	}
+
+	ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
+	if (ret)
+		goto e_free;
+
+	if (tpage) {
+		offset = paddr & 15;
+		if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
+				 page_address(tpage) + offset, size))
+			ret = -EFAULT;
+	}
+
+e_free:
+	if (tpage)
+		__free_page(tpage);
+
+	return ret;
+}
+
+static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
+				  unsigned long __user vaddr,
+				  unsigned long dst_paddr,
+				  unsigned long __user dst_vaddr,
+				  int size, int *error)
+{
+	struct page *src_tpage = NULL;
+	struct page *dst_tpage = NULL;
+	int ret, len = size;
+
+	/* If source buffer is not aligned then use an intermediate buffer */
+	if (!IS_ALIGNED(vaddr, 16)) {
+		src_tpage = alloc_page(GFP_KERNEL);
+		if (!src_tpage)
+			return -ENOMEM;
+
+		if (copy_from_user(page_address(src_tpage),
+				(void __user *)(uintptr_t)vaddr, size)) {
+			__free_page(src_tpage);
+			return -EFAULT;
+		}
+
+		paddr = __sme_page_pa(src_tpage);
+	}
+
+	/*
+	 *  If destination buffer or length is not aligned then do read-modify-write:
+	 *   - decrypt destination in an intermediate buffer
+	 *   - copy the source buffer in an intermediate buffer
+	 *   - use the intermediate buffer as source buffer
+	 */
+	if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
+		int dst_offset;
+
+		dst_tpage = alloc_page(GFP_KERNEL);
+		if (!dst_tpage) {
+			ret = -ENOMEM;
+			goto e_free;
+		}
+
+		ret = __sev_dbg_decrypt(kvm, dst_paddr,
+					__sme_page_pa(dst_tpage), size, error);
+		if (ret)
+			goto e_free;
+
+		/*
+		 *  If source is kernel buffer then use memcpy() otherwise
+		 *  copy_from_user().
+		 */
+		dst_offset = dst_paddr & 15;
+
+		if (src_tpage)
+			memcpy(page_address(dst_tpage) + dst_offset,
+			       page_address(src_tpage), size);
+		else {
+			if (copy_from_user(page_address(dst_tpage) + dst_offset,
+					   (void __user *)(uintptr_t)vaddr, size)) {
+				ret = -EFAULT;
+				goto e_free;
+			}
+		}
+
+		paddr = __sme_page_pa(dst_tpage);
+		dst_paddr = round_down(dst_paddr, 16);
+		len = round_up(size, 16);
+	}
+
+	ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
+
+e_free:
+	if (src_tpage)
+		__free_page(src_tpage);
+	if (dst_tpage)
+		__free_page(dst_tpage);
+	return ret;
+}
+
+static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
+{
+	unsigned long vaddr, vaddr_end, next_vaddr;
+	unsigned long dst_vaddr, dst_vaddr_end;
+	struct page **src_p, **dst_p;
+	struct kvm_sev_dbg debug;
+	unsigned long n;
+	int ret, size;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
+		return -EFAULT;
+
+	vaddr = debug.src_uaddr;
+	size = debug.len;
+	vaddr_end = vaddr + size;
+	dst_vaddr = debug.dst_uaddr;
+	dst_vaddr_end = dst_vaddr + size;
+
+	for (; vaddr < vaddr_end; vaddr = next_vaddr) {
+		int len, s_off, d_off;
+
+		/* lock userspace source and destination page */
+		src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
+		if (!src_p)
+			return -EFAULT;
+
+		dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
+		if (!dst_p) {
+			sev_unpin_memory(kvm, src_p, n);
+			return -EFAULT;
+		}
+
+		/*
+		 * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
+		 * memory content (i.e it will write the same memory region with C=1).
+		 * It's possible that the cache may contain the data with C=0, i.e.,
+		 * unencrypted so invalidate it first.
+		 */
+		sev_clflush_pages(src_p, 1);
+		sev_clflush_pages(dst_p, 1);
+
+		/*
+		 * Since user buffer may not be page aligned, calculate the
+		 * offset within the page.
+		 */
+		s_off = vaddr & ~PAGE_MASK;
+		d_off = dst_vaddr & ~PAGE_MASK;
+		len = min_t(size_t, (PAGE_SIZE - s_off), size);
+
+		if (dec)
+			ret = __sev_dbg_decrypt_user(kvm,
+						     __sme_page_pa(src_p[0]) + s_off,
+						     dst_vaddr,
+						     __sme_page_pa(dst_p[0]) + d_off,
+						     len, &argp->error);
+		else
+			ret = __sev_dbg_encrypt_user(kvm,
+						     __sme_page_pa(src_p[0]) + s_off,
+						     vaddr,
+						     __sme_page_pa(dst_p[0]) + d_off,
+						     dst_vaddr,
+						     len, &argp->error);
+
+		sev_unpin_memory(kvm, src_p, 1);
+		sev_unpin_memory(kvm, dst_p, 1);
+
+		if (ret)
+			goto err;
+
+		next_vaddr = vaddr + len;
+		dst_vaddr = dst_vaddr + len;
+		size -= len;
+	}
+err:
+	return ret;
+}
+
+static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct sev_data_launch_secret *data;
+	struct kvm_sev_launch_secret params;
+	struct page **pages;
+	void *blob, *hdr;
+	unsigned long n;
+	int ret, offset;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
+		return -EFAULT;
+
+	pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
+	if (!pages)
+		return -ENOMEM;
+
+	/*
+	 * The secret must be copied into contiguous memory region, lets verify
+	 * that userspace memory pages are contiguous before we issue command.
+	 */
+	if (get_num_contig_pages(0, pages, n) != n) {
+		ret = -EINVAL;
+		goto e_unpin_memory;
+	}
+
+	ret = -ENOMEM;
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		goto e_unpin_memory;
+
+	offset = params.guest_uaddr & (PAGE_SIZE - 1);
+	data->guest_address = __sme_page_pa(pages[0]) + offset;
+	data->guest_len = params.guest_len;
+
+	blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
+	if (IS_ERR(blob)) {
+		ret = PTR_ERR(blob);
+		goto e_free;
+	}
+
+	data->trans_address = __psp_pa(blob);
+	data->trans_len = params.trans_len;
+
+	hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
+	if (IS_ERR(hdr)) {
+		ret = PTR_ERR(hdr);
+		goto e_free_blob;
+	}
+	data->hdr_address = __psp_pa(hdr);
+	data->hdr_len = params.hdr_len;
+
+	data->handle = sev->handle;
+	ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
+
+	kfree(hdr);
+
+e_free_blob:
+	kfree(blob);
+e_free:
+	kfree(data);
+e_unpin_memory:
+	sev_unpin_memory(kvm, pages, n);
+	return ret;
+}
+
+static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
+{
+	struct kvm_sev_cmd sev_cmd;
+	int r;
+
+	if (!svm_sev_enabled())
+		return -ENOTTY;
+
+	if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
+		return -EFAULT;
+
+	mutex_lock(&kvm->lock);
+
+	switch (sev_cmd.id) {
+	case KVM_SEV_INIT:
+		r = sev_guest_init(kvm, &sev_cmd);
+		break;
+	case KVM_SEV_LAUNCH_START:
+		r = sev_launch_start(kvm, &sev_cmd);
+		break;
+	case KVM_SEV_LAUNCH_UPDATE_DATA:
+		r = sev_launch_update_data(kvm, &sev_cmd);
+		break;
+	case KVM_SEV_LAUNCH_MEASURE:
+		r = sev_launch_measure(kvm, &sev_cmd);
+		break;
+	case KVM_SEV_LAUNCH_FINISH:
+		r = sev_launch_finish(kvm, &sev_cmd);
+		break;
+	case KVM_SEV_GUEST_STATUS:
+		r = sev_guest_status(kvm, &sev_cmd);
+		break;
+	case KVM_SEV_DBG_DECRYPT:
+		r = sev_dbg_crypt(kvm, &sev_cmd, true);
+		break;
+	case KVM_SEV_DBG_ENCRYPT:
+		r = sev_dbg_crypt(kvm, &sev_cmd, false);
+		break;
+	case KVM_SEV_LAUNCH_SECRET:
+		r = sev_launch_secret(kvm, &sev_cmd);
+		break;
+	default:
+		r = -EINVAL;
+		goto out;
+	}
+
+	if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
+		r = -EFAULT;
+
+out:
+	mutex_unlock(&kvm->lock);
+	return r;
+}
+
+static int svm_register_enc_region(struct kvm *kvm,
+				   struct kvm_enc_region *range)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct enc_region *region;
+	int ret = 0;
+
+	if (!sev_guest(kvm))
+		return -ENOTTY;
+
+	region = kzalloc(sizeof(*region), GFP_KERNEL);
+	if (!region)
+		return -ENOMEM;
+
+	region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
+	if (!region->pages) {
+		ret = -ENOMEM;
+		goto e_free;
+	}
+
+	/*
+	 * The guest may change the memory encryption attribute from C=0 -> C=1
+	 * or vice versa for this memory range. Lets make sure caches are
+	 * flushed to ensure that guest data gets written into memory with
+	 * correct C-bit.
+	 */
+	sev_clflush_pages(region->pages, region->npages);
+
+	region->uaddr = range->addr;
+	region->size = range->size;
+
+	mutex_lock(&kvm->lock);
+	list_add_tail(&region->list, &sev->regions_list);
+	mutex_unlock(&kvm->lock);
+
+	return ret;
+
+e_free:
+	kfree(region);
+	return ret;
+}
+
+static struct enc_region *
+find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
+{
+	struct kvm_sev_info *sev = &kvm->arch.sev_info;
+	struct list_head *head = &sev->regions_list;
+	struct enc_region *i;
+
+	list_for_each_entry(i, head, list) {
+		if (i->uaddr == range->addr &&
+		    i->size == range->size)
+			return i;
+	}
+
+	return NULL;
+}
+
+
+static int svm_unregister_enc_region(struct kvm *kvm,
+				     struct kvm_enc_region *range)
+{
+	struct enc_region *region;
+	int ret;
+
+	mutex_lock(&kvm->lock);
+
+	if (!sev_guest(kvm)) {
+		ret = -ENOTTY;
+		goto failed;
+	}
+
+	region = find_enc_region(kvm, range);
+	if (!region) {
+		ret = -EINVAL;
+		goto failed;
+	}
+
+	__unregister_enc_region_locked(kvm, region);
+
+	mutex_unlock(&kvm->lock);
+	return 0;
+
+failed:
+	mutex_unlock(&kvm->lock);
+	return ret;
+}
+
+static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
+	.cpu_has_kvm_support = has_svm,
+	.disabled_by_bios = is_disabled,
+	.hardware_setup = svm_hardware_setup,
+	.hardware_unsetup = svm_hardware_unsetup,
+	.check_processor_compatibility = svm_check_processor_compat,
+	.hardware_enable = svm_hardware_enable,
+	.hardware_disable = svm_hardware_disable,
+	.cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
+	.cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
+
+	.vcpu_create = svm_create_vcpu,
+	.vcpu_free = svm_free_vcpu,
+	.vcpu_reset = svm_vcpu_reset,
+
+	.vm_init = avic_vm_init,
+	.vm_destroy = svm_vm_destroy,
+
+	.prepare_guest_switch = svm_prepare_guest_switch,
+	.vcpu_load = svm_vcpu_load,
+	.vcpu_put = svm_vcpu_put,
+	.vcpu_blocking = svm_vcpu_blocking,
+	.vcpu_unblocking = svm_vcpu_unblocking,
+
+	.update_bp_intercept = update_bp_intercept,
+	.get_msr_feature = svm_get_msr_feature,
+	.get_msr = svm_get_msr,
+	.set_msr = svm_set_msr,
+	.get_segment_base = svm_get_segment_base,
+	.get_segment = svm_get_segment,
+	.set_segment = svm_set_segment,
+	.get_cpl = svm_get_cpl,
+	.get_cs_db_l_bits = kvm_get_cs_db_l_bits,
+	.decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
+	.decache_cr3 = svm_decache_cr3,
+	.decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
+	.set_cr0 = svm_set_cr0,
+	.set_cr3 = svm_set_cr3,
+	.set_cr4 = svm_set_cr4,
+	.set_efer = svm_set_efer,
+	.get_idt = svm_get_idt,
 	.set_idt = svm_set_idt,
 	.get_gdt = svm_get_gdt,
 	.set_gdt = svm_set_gdt,
@@ -5597,6 +6922,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
 	.load_eoi_exitmap = svm_load_eoi_exitmap,
 	.hwapic_irr_update = svm_hwapic_irr_update,
 	.hwapic_isr_update = svm_hwapic_isr_update,
+	.sync_pir_to_irr = kvm_lapic_find_highest_irr,
 	.apicv_post_state_restore = avic_post_state_restore,
 
 	.set_tss_addr = svm_set_tss_addr,
@@ -5613,6 +6939,7 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
 	.invpcid_supported = svm_invpcid_supported,
 	.mpx_supported = svm_mpx_supported,
 	.xsaves_supported = svm_xsaves_supported,
+	.umip_emulated = svm_umip_emulated,
 
 	.set_supported_cpuid = svm_set_supported_cpuid,
 
@@ -5636,6 +6963,10 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
 	.pre_enter_smm = svm_pre_enter_smm,
 	.pre_leave_smm = svm_pre_leave_smm,
 	.enable_smi_window = enable_smi_window,
+
+	.mem_enc_op = svm_mem_enc_op,
+	.mem_enc_reg_region = svm_register_enc_region,
+	.mem_enc_unreg_region = svm_unregister_enc_region,
 };
 
 static int __init svm_init(void)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index c829d89e2e63..657c93409042 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -34,6 +34,7 @@
 #include <linux/tboot.h>
 #include <linux/hrtimer.h>
 #include <linux/frame.h>
+#include <linux/nospec.h>
 #include "kvm_cache_regs.h"
 #include "x86.h"
 
@@ -50,6 +51,7 @@
 #include <asm/apic.h>
 #include <asm/irq_remapping.h>
 #include <asm/mmu_context.h>
+#include <asm/microcode.h>
 #include <asm/nospec-branch.h>
 
 #include "trace.h"
@@ -111,6 +113,14 @@ static u64 __read_mostly host_xss;
 static bool __read_mostly enable_pml = 1;
 module_param_named(pml, enable_pml, bool, S_IRUGO);
 
+#define MSR_TYPE_R	1
+#define MSR_TYPE_W	2
+#define MSR_TYPE_RW	3
+
+#define MSR_BITMAP_MODE_X2APIC		1
+#define MSR_BITMAP_MODE_X2APIC_APICV	2
+#define MSR_BITMAP_MODE_LM		4
+
 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 
 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
@@ -185,7 +195,6 @@ module_param(ple_window_max, int, S_IRUGO);
 extern const ulong vmx_return;
 
 #define NR_AUTOLOAD_MSRS 8
-#define VMCS02_POOL_SIZE 1
 
 struct vmcs {
 	u32 revision_id;
@@ -210,6 +219,7 @@ struct loaded_vmcs {
 	int soft_vnmi_blocked;
 	ktime_t entry_time;
 	s64 vnmi_blocked_time;
+	unsigned long *msr_bitmap;
 	struct list_head loaded_vmcss_on_cpu_link;
 };
 
@@ -226,7 +236,7 @@ struct shared_msr_entry {
  * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
  * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
  * More than one of these structures may exist, if L1 runs multiple L2 guests.
- * nested_vmx_run() will use the data here to build a vmcs02: a VMCS for the
+ * nested_vmx_run() will use the data here to build the vmcs02: a VMCS for the
  * underlying hardware which will be used to run L2.
  * This structure is packed to ensure that its layout is identical across
  * machines (necessary for live migration).
@@ -409,12 +419,11 @@ struct __packed vmcs12 {
  */
 #define VMCS12_SIZE 0x1000
 
-/* Used to remember the last vmcs02 used for some recently used vmcs12s */
-struct vmcs02_list {
-	struct list_head list;
-	gpa_t vmptr;
-	struct loaded_vmcs vmcs02;
-};
+/*
+ * VMCS12_MAX_FIELD_INDEX is the highest index value used in any
+ * supported VMCS12 field encoding.
+ */
+#define VMCS12_MAX_FIELD_INDEX 0x17
 
 /*
  * The nested_vmx structure is part of vcpu_vmx, and holds information we need
@@ -439,16 +448,17 @@ struct nested_vmx {
 	 * data hold by vmcs12
 	 */
 	bool sync_shadow_vmcs;
+	bool dirty_vmcs12;
 
-	/* vmcs02_list cache of VMCSs recently used to run L2 guests */
-	struct list_head vmcs02_pool;
-	int vmcs02_num;
 	bool change_vmcs01_virtual_x2apic_mode;
 	/* L2 must run next, and mustn't decide to exit to L1. */
 	bool nested_run_pending;
+
+	struct loaded_vmcs vmcs02;
+
 	/*
-	 * Guest pages referred to in vmcs02 with host-physical pointers, so
-	 * we must keep them pinned while L2 runs.
+	 * Guest pages referred to in the vmcs02 with host-physical
+	 * pointers, so we must keep them pinned while L2 runs.
 	 */
 	struct page *apic_access_page;
 	struct page *virtual_apic_page;
@@ -457,8 +467,6 @@ struct nested_vmx {
 	bool pi_pending;
 	u16 posted_intr_nv;
 
-	unsigned long *msr_bitmap;
-
 	struct hrtimer preemption_timer;
 	bool preemption_timer_expired;
 
@@ -581,6 +589,7 @@ struct vcpu_vmx {
 	struct kvm_vcpu       vcpu;
 	unsigned long         host_rsp;
 	u8                    fail;
+	u8		      msr_bitmap_mode;
 	u32                   exit_intr_info;
 	u32                   idt_vectoring_info;
 	ulong                 rflags;
@@ -592,6 +601,10 @@ struct vcpu_vmx {
 	u64 		      msr_host_kernel_gs_base;
 	u64 		      msr_guest_kernel_gs_base;
 #endif
+
+	u64 		      arch_capabilities;
+	u64 		      spec_ctrl;
+
 	u32 vm_entry_controls_shadow;
 	u32 vm_exit_controls_shadow;
 	u32 secondary_exec_control;
@@ -659,6 +672,8 @@ struct vcpu_vmx {
 
 	u32 host_pkru;
 
+	unsigned long host_debugctlmsr;
+
 	/*
 	 * Only bits masked by msr_ia32_feature_control_valid_bits can be set in
 	 * msr_ia32_feature_control. FEATURE_CONTROL_LOCKED is always included
@@ -687,67 +702,24 @@ static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
 	return &(to_vmx(vcpu)->pi_desc);
 }
 
+#define ROL16(val, n) ((u16)(((u16)(val) << (n)) | ((u16)(val) >> (16 - (n)))))
 #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
-#define FIELD(number, name)	[number] = VMCS12_OFFSET(name)
-#define FIELD64(number, name)	[number] = VMCS12_OFFSET(name), \
-				[number##_HIGH] = VMCS12_OFFSET(name)+4
+#define FIELD(number, name)	[ROL16(number, 6)] = VMCS12_OFFSET(name)
+#define FIELD64(number, name)						\
+	FIELD(number, name),						\
+	[ROL16(number##_HIGH, 6)] = VMCS12_OFFSET(name) + sizeof(u32)
 
 
-static unsigned long shadow_read_only_fields[] = {
-	/*
-	 * We do NOT shadow fields that are modified when L0
-	 * traps and emulates any vmx instruction (e.g. VMPTRLD,
-	 * VMXON...) executed by L1.
-	 * For example, VM_INSTRUCTION_ERROR is read
-	 * by L1 if a vmx instruction fails (part of the error path).
-	 * Note the code assumes this logic. If for some reason
-	 * we start shadowing these fields then we need to
-	 * force a shadow sync when L0 emulates vmx instructions
-	 * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
-	 * by nested_vmx_failValid)
-	 */
-	VM_EXIT_REASON,
-	VM_EXIT_INTR_INFO,
-	VM_EXIT_INSTRUCTION_LEN,
-	IDT_VECTORING_INFO_FIELD,
-	IDT_VECTORING_ERROR_CODE,
-	VM_EXIT_INTR_ERROR_CODE,
-	EXIT_QUALIFICATION,
-	GUEST_LINEAR_ADDRESS,
-	GUEST_PHYSICAL_ADDRESS
+static u16 shadow_read_only_fields[] = {
+#define SHADOW_FIELD_RO(x) x,
+#include "vmx_shadow_fields.h"
 };
 static int max_shadow_read_only_fields =
 	ARRAY_SIZE(shadow_read_only_fields);
 
-static unsigned long shadow_read_write_fields[] = {
-	TPR_THRESHOLD,
-	GUEST_RIP,
-	GUEST_RSP,
-	GUEST_CR0,
-	GUEST_CR3,
-	GUEST_CR4,
-	GUEST_INTERRUPTIBILITY_INFO,
-	GUEST_RFLAGS,
-	GUEST_CS_SELECTOR,
-	GUEST_CS_AR_BYTES,
-	GUEST_CS_LIMIT,
-	GUEST_CS_BASE,
-	GUEST_ES_BASE,
-	GUEST_BNDCFGS,
-	CR0_GUEST_HOST_MASK,
-	CR0_READ_SHADOW,
-	CR4_READ_SHADOW,
-	TSC_OFFSET,
-	EXCEPTION_BITMAP,
-	CPU_BASED_VM_EXEC_CONTROL,
-	VM_ENTRY_EXCEPTION_ERROR_CODE,
-	VM_ENTRY_INTR_INFO_FIELD,
-	VM_ENTRY_INSTRUCTION_LEN,
-	VM_ENTRY_EXCEPTION_ERROR_CODE,
-	HOST_FS_BASE,
-	HOST_GS_BASE,
-	HOST_FS_SELECTOR,
-	HOST_GS_SELECTOR
+static u16 shadow_read_write_fields[] = {
+#define SHADOW_FIELD_RW(x) x,
+#include "vmx_shadow_fields.h"
 };
 static int max_shadow_read_write_fields =
 	ARRAY_SIZE(shadow_read_write_fields);
@@ -898,21 +870,22 @@ static const unsigned short vmcs_field_to_offset_table[] = {
 
 static inline short vmcs_field_to_offset(unsigned long field)
 {
-	BUILD_BUG_ON(ARRAY_SIZE(vmcs_field_to_offset_table) > SHRT_MAX);
+	const size_t size = ARRAY_SIZE(vmcs_field_to_offset_table);
+	unsigned short offset;
+	unsigned index;
 
-	if (field >= ARRAY_SIZE(vmcs_field_to_offset_table))
+	if (field >> 15)
 		return -ENOENT;
 
-	/*
-	 * FIXME: Mitigation for CVE-2017-5753.  To be replaced with a
-	 * generic mechanism.
-	 */
-	asm("lfence");
-
-	if (vmcs_field_to_offset_table[field] == 0)
+	index = ROL16(field, 6);
+	if (index >= size)
 		return -ENOENT;
 
-	return vmcs_field_to_offset_table[field];
+	index = array_index_nospec(index, size);
+	offset = vmcs_field_to_offset_table[index];
+	if (offset == 0)
+		return -ENOENT;
+	return offset;
 }
 
 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
@@ -935,6 +908,9 @@ static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
 					    u16 error_code);
+static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
+static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
+							  u32 msr, int type);
 
 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
@@ -952,14 +928,6 @@ static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
 
 enum {
-	VMX_IO_BITMAP_A,
-	VMX_IO_BITMAP_B,
-	VMX_MSR_BITMAP_LEGACY,
-	VMX_MSR_BITMAP_LONGMODE,
-	VMX_MSR_BITMAP_LEGACY_X2APIC_APICV,
-	VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV,
-	VMX_MSR_BITMAP_LEGACY_X2APIC,
-	VMX_MSR_BITMAP_LONGMODE_X2APIC,
 	VMX_VMREAD_BITMAP,
 	VMX_VMWRITE_BITMAP,
 	VMX_BITMAP_NR
@@ -967,14 +935,6 @@ enum {
 
 static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
 
-#define vmx_io_bitmap_a                      (vmx_bitmap[VMX_IO_BITMAP_A])
-#define vmx_io_bitmap_b                      (vmx_bitmap[VMX_IO_BITMAP_B])
-#define vmx_msr_bitmap_legacy                (vmx_bitmap[VMX_MSR_BITMAP_LEGACY])
-#define vmx_msr_bitmap_longmode              (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE])
-#define vmx_msr_bitmap_legacy_x2apic_apicv   (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC_APICV])
-#define vmx_msr_bitmap_longmode_x2apic_apicv (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC_APICV])
-#define vmx_msr_bitmap_legacy_x2apic         (vmx_bitmap[VMX_MSR_BITMAP_LEGACY_X2APIC])
-#define vmx_msr_bitmap_longmode_x2apic       (vmx_bitmap[VMX_MSR_BITMAP_LONGMODE_X2APIC])
 #define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
 #define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
 
@@ -1085,6 +1045,13 @@ static inline bool is_machine_check(u32 intr_info)
 		(INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
 }
 
+/* Undocumented: icebp/int1 */
+static inline bool is_icebp(u32 intr_info)
+{
+	return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
+		== (INTR_TYPE_PRIV_SW_EXCEPTION | INTR_INFO_VALID_MASK);
+}
+
 static inline bool cpu_has_vmx_msr_bitmap(void)
 {
 	return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
@@ -1918,6 +1885,52 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu)
 	vmcs_write32(EXCEPTION_BITMAP, eb);
 }
 
+/*
+ * Check if MSR is intercepted for currently loaded MSR bitmap.
+ */
+static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
+{
+	unsigned long *msr_bitmap;
+	int f = sizeof(unsigned long);
+
+	if (!cpu_has_vmx_msr_bitmap())
+		return true;
+
+	msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
+
+	if (msr <= 0x1fff) {
+		return !!test_bit(msr, msr_bitmap + 0x800 / f);
+	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
+		msr &= 0x1fff;
+		return !!test_bit(msr, msr_bitmap + 0xc00 / f);
+	}
+
+	return true;
+}
+
+/*
+ * Check if MSR is intercepted for L01 MSR bitmap.
+ */
+static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
+{
+	unsigned long *msr_bitmap;
+	int f = sizeof(unsigned long);
+
+	if (!cpu_has_vmx_msr_bitmap())
+		return true;
+
+	msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
+
+	if (msr <= 0x1fff) {
+		return !!test_bit(msr, msr_bitmap + 0x800 / f);
+	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
+		msr &= 0x1fff;
+		return !!test_bit(msr, msr_bitmap + 0xc00 / f);
+	}
+
+	return true;
+}
+
 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
 		unsigned long entry, unsigned long exit)
 {
@@ -2296,6 +2309,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 	if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
 		per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
 		vmcs_load(vmx->loaded_vmcs->vmcs);
+		indirect_branch_prediction_barrier();
 	}
 
 	if (!already_loaded) {
@@ -2333,6 +2347,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
 	vmx_vcpu_pi_load(vcpu, cpu);
 	vmx->host_pkru = read_pkru();
+	vmx->host_debugctlmsr = get_debugctlmsr();
 }
 
 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
@@ -2572,36 +2587,6 @@ static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
 	vmx->guest_msrs[from] = tmp;
 }
 
-static void vmx_set_msr_bitmap(struct kvm_vcpu *vcpu)
-{
-	unsigned long *msr_bitmap;
-
-	if (is_guest_mode(vcpu))
-		msr_bitmap = to_vmx(vcpu)->nested.msr_bitmap;
-	else if (cpu_has_secondary_exec_ctrls() &&
-		 (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &
-		  SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
-		if (enable_apicv && kvm_vcpu_apicv_active(vcpu)) {
-			if (is_long_mode(vcpu))
-				msr_bitmap = vmx_msr_bitmap_longmode_x2apic_apicv;
-			else
-				msr_bitmap = vmx_msr_bitmap_legacy_x2apic_apicv;
-		} else {
-			if (is_long_mode(vcpu))
-				msr_bitmap = vmx_msr_bitmap_longmode_x2apic;
-			else
-				msr_bitmap = vmx_msr_bitmap_legacy_x2apic;
-		}
-	} else {
-		if (is_long_mode(vcpu))
-			msr_bitmap = vmx_msr_bitmap_longmode;
-		else
-			msr_bitmap = vmx_msr_bitmap_legacy;
-	}
-
-	vmcs_write64(MSR_BITMAP, __pa(msr_bitmap));
-}
-
 /*
  * Set up the vmcs to automatically save and restore system
  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
@@ -2642,7 +2627,7 @@ static void setup_msrs(struct vcpu_vmx *vmx)
 	vmx->save_nmsrs = save_nmsrs;
 
 	if (cpu_has_vmx_msr_bitmap())
-		vmx_set_msr_bitmap(&vmx->vcpu);
+		vmx_update_msr_bitmap(&vmx->vcpu);
 }
 
 /*
@@ -2920,7 +2905,7 @@ static void nested_vmx_setup_ctls_msrs(struct vcpu_vmx *vmx)
 	rdmsrl(MSR_IA32_VMX_CR4_FIXED1, vmx->nested.nested_vmx_cr4_fixed1);
 
 	/* highest index: VMX_PREEMPTION_TIMER_VALUE */
-	vmx->nested.nested_vmx_vmcs_enum = 0x2e;
+	vmx->nested.nested_vmx_vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1;
 }
 
 /*
@@ -3249,6 +3234,11 @@ static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
 	return !(val & ~valid_bits);
 }
 
+static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
+{
+	return 1;
+}
+
 /*
  * Reads an msr value (of 'msr_index') into 'pdata'.
  * Returns 0 on success, non-0 otherwise.
@@ -3256,6 +3246,7 @@ static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
  */
 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 {
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct shared_msr_entry *msr;
 
 	switch (msr_info->index) {
@@ -3267,8 +3258,8 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		msr_info->data = vmcs_readl(GUEST_GS_BASE);
 		break;
 	case MSR_KERNEL_GS_BASE:
-		vmx_load_host_state(to_vmx(vcpu));
-		msr_info->data = to_vmx(vcpu)->msr_guest_kernel_gs_base;
+		vmx_load_host_state(vmx);
+		msr_info->data = vmx->msr_guest_kernel_gs_base;
 		break;
 #endif
 	case MSR_EFER:
@@ -3276,6 +3267,20 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_IA32_TSC:
 		msr_info->data = guest_read_tsc(vcpu);
 		break;
+	case MSR_IA32_SPEC_CTRL:
+		if (!msr_info->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS) &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
+			return 1;
+
+		msr_info->data = to_vmx(vcpu)->spec_ctrl;
+		break;
+	case MSR_IA32_ARCH_CAPABILITIES:
+		if (!msr_info->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES))
+			return 1;
+		msr_info->data = to_vmx(vcpu)->arch_capabilities;
+		break;
 	case MSR_IA32_SYSENTER_CS:
 		msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
 		break;
@@ -3294,13 +3299,13 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		break;
 	case MSR_IA32_MCG_EXT_CTL:
 		if (!msr_info->host_initiated &&
-		    !(to_vmx(vcpu)->msr_ia32_feature_control &
+		    !(vmx->msr_ia32_feature_control &
 		      FEATURE_CONTROL_LMCE))
 			return 1;
 		msr_info->data = vcpu->arch.mcg_ext_ctl;
 		break;
 	case MSR_IA32_FEATURE_CONTROL:
-		msr_info->data = to_vmx(vcpu)->msr_ia32_feature_control;
+		msr_info->data = vmx->msr_ia32_feature_control;
 		break;
 	case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
 		if (!nested_vmx_allowed(vcpu))
@@ -3317,7 +3322,7 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			return 1;
 		/* Otherwise falls through */
 	default:
-		msr = find_msr_entry(to_vmx(vcpu), msr_info->index);
+		msr = find_msr_entry(vmx, msr_info->index);
 		if (msr) {
 			msr_info->data = msr->data;
 			break;
@@ -3383,6 +3388,70 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_IA32_TSC:
 		kvm_write_tsc(vcpu, msr_info);
 		break;
+	case MSR_IA32_SPEC_CTRL:
+		if (!msr_info->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_IBRS) &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
+			return 1;
+
+		/* The STIBP bit doesn't fault even if it's not advertised */
+		if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP))
+			return 1;
+
+		vmx->spec_ctrl = data;
+
+		if (!data)
+			break;
+
+		/*
+		 * For non-nested:
+		 * When it's written (to non-zero) for the first time, pass
+		 * it through.
+		 *
+		 * For nested:
+		 * The handling of the MSR bitmap for L2 guests is done in
+		 * nested_vmx_merge_msr_bitmap. We should not touch the
+		 * vmcs02.msr_bitmap here since it gets completely overwritten
+		 * in the merging. We update the vmcs01 here for L1 as well
+		 * since it will end up touching the MSR anyway now.
+		 */
+		vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap,
+					      MSR_IA32_SPEC_CTRL,
+					      MSR_TYPE_RW);
+		break;
+	case MSR_IA32_PRED_CMD:
+		if (!msr_info->host_initiated &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_IBPB) &&
+		    !guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
+			return 1;
+
+		if (data & ~PRED_CMD_IBPB)
+			return 1;
+
+		if (!data)
+			break;
+
+		wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
+
+		/*
+		 * For non-nested:
+		 * When it's written (to non-zero) for the first time, pass
+		 * it through.
+		 *
+		 * For nested:
+		 * The handling of the MSR bitmap for L2 guests is done in
+		 * nested_vmx_merge_msr_bitmap. We should not touch the
+		 * vmcs02.msr_bitmap here since it gets completely overwritten
+		 * in the merging.
+		 */
+		vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD,
+					      MSR_TYPE_W);
+		break;
+	case MSR_IA32_ARCH_CAPABILITIES:
+		if (!msr_info->host_initiated)
+			return 1;
+		vmx->arch_capabilities = data;
+		break;
 	case MSR_IA32_CR_PAT:
 		if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
 			if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
@@ -3639,7 +3708,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 #endif
 	      CPU_BASED_CR3_LOAD_EXITING |
 	      CPU_BASED_CR3_STORE_EXITING |
-	      CPU_BASED_USE_IO_BITMAPS |
+	      CPU_BASED_UNCOND_IO_EXITING |
 	      CPU_BASED_MOV_DR_EXITING |
 	      CPU_BASED_USE_TSC_OFFSETING |
 	      CPU_BASED_INVLPG_EXITING |
@@ -3669,6 +3738,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 			SECONDARY_EXEC_ENABLE_EPT |
 			SECONDARY_EXEC_UNRESTRICTED_GUEST |
 			SECONDARY_EXEC_PAUSE_LOOP_EXITING |
+			SECONDARY_EXEC_DESC |
 			SECONDARY_EXEC_RDTSCP |
 			SECONDARY_EXEC_ENABLE_INVPCID |
 			SECONDARY_EXEC_APIC_REGISTER_VIRT |
@@ -3837,11 +3907,6 @@ static struct vmcs *alloc_vmcs_cpu(int cpu)
 	return vmcs;
 }
 
-static struct vmcs *alloc_vmcs(void)
-{
-	return alloc_vmcs_cpu(raw_smp_processor_id());
-}
-
 static void free_vmcs(struct vmcs *vmcs)
 {
 	free_pages((unsigned long)vmcs, vmcs_config.order);
@@ -3857,9 +3922,38 @@ static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
 	loaded_vmcs_clear(loaded_vmcs);
 	free_vmcs(loaded_vmcs->vmcs);
 	loaded_vmcs->vmcs = NULL;
+	if (loaded_vmcs->msr_bitmap)
+		free_page((unsigned long)loaded_vmcs->msr_bitmap);
 	WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
 }
 
+static struct vmcs *alloc_vmcs(void)
+{
+	return alloc_vmcs_cpu(raw_smp_processor_id());
+}
+
+static int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
+{
+	loaded_vmcs->vmcs = alloc_vmcs();
+	if (!loaded_vmcs->vmcs)
+		return -ENOMEM;
+
+	loaded_vmcs->shadow_vmcs = NULL;
+	loaded_vmcs_init(loaded_vmcs);
+
+	if (cpu_has_vmx_msr_bitmap()) {
+		loaded_vmcs->msr_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
+		if (!loaded_vmcs->msr_bitmap)
+			goto out_vmcs;
+		memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
+	}
+	return 0;
+
+out_vmcs:
+	free_loaded_vmcs(loaded_vmcs);
+	return -ENOMEM;
+}
+
 static void free_kvm_area(void)
 {
 	int cpu;
@@ -3870,17 +3964,17 @@ static void free_kvm_area(void)
 	}
 }
 
-enum vmcs_field_type {
-	VMCS_FIELD_TYPE_U16 = 0,
-	VMCS_FIELD_TYPE_U64 = 1,
-	VMCS_FIELD_TYPE_U32 = 2,
-	VMCS_FIELD_TYPE_NATURAL_WIDTH = 3
+enum vmcs_field_width {
+	VMCS_FIELD_WIDTH_U16 = 0,
+	VMCS_FIELD_WIDTH_U64 = 1,
+	VMCS_FIELD_WIDTH_U32 = 2,
+	VMCS_FIELD_WIDTH_NATURAL_WIDTH = 3
 };
 
-static inline int vmcs_field_type(unsigned long field)
+static inline int vmcs_field_width(unsigned long field)
 {
 	if (0x1 & field)	/* the *_HIGH fields are all 32 bit */
-		return VMCS_FIELD_TYPE_U32;
+		return VMCS_FIELD_WIDTH_U32;
 	return (field >> 13) & 0x3 ;
 }
 
@@ -3893,43 +3987,66 @@ static void init_vmcs_shadow_fields(void)
 {
 	int i, j;
 
-	/* No checks for read only fields yet */
+	for (i = j = 0; i < max_shadow_read_only_fields; i++) {
+		u16 field = shadow_read_only_fields[i];
+		if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
+		    (i + 1 == max_shadow_read_only_fields ||
+		     shadow_read_only_fields[i + 1] != field + 1))
+			pr_err("Missing field from shadow_read_only_field %x\n",
+			       field + 1);
+
+		clear_bit(field, vmx_vmread_bitmap);
+#ifdef CONFIG_X86_64
+		if (field & 1)
+			continue;
+#endif
+		if (j < i)
+			shadow_read_only_fields[j] = field;
+		j++;
+	}
+	max_shadow_read_only_fields = j;
 
 	for (i = j = 0; i < max_shadow_read_write_fields; i++) {
-		switch (shadow_read_write_fields[i]) {
-		case GUEST_BNDCFGS:
-			if (!kvm_mpx_supported())
+		u16 field = shadow_read_write_fields[i];
+		if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
+		    (i + 1 == max_shadow_read_write_fields ||
+		     shadow_read_write_fields[i + 1] != field + 1))
+			pr_err("Missing field from shadow_read_write_field %x\n",
+			       field + 1);
+
+		/*
+		 * PML and the preemption timer can be emulated, but the
+		 * processor cannot vmwrite to fields that don't exist
+		 * on bare metal.
+		 */
+		switch (field) {
+		case GUEST_PML_INDEX:
+			if (!cpu_has_vmx_pml())
+				continue;
+			break;
+		case VMX_PREEMPTION_TIMER_VALUE:
+			if (!cpu_has_vmx_preemption_timer())
+				continue;
+			break;
+		case GUEST_INTR_STATUS:
+			if (!cpu_has_vmx_apicv())
 				continue;
 			break;
 		default:
 			break;
 		}
 
+		clear_bit(field, vmx_vmwrite_bitmap);
+		clear_bit(field, vmx_vmread_bitmap);
+#ifdef CONFIG_X86_64
+		if (field & 1)
+			continue;
+#endif
 		if (j < i)
-			shadow_read_write_fields[j] =
-				shadow_read_write_fields[i];
+			shadow_read_write_fields[j] = field;
 		j++;
 	}
 	max_shadow_read_write_fields = j;
-
-	/* shadowed fields guest access without vmexit */
-	for (i = 0; i < max_shadow_read_write_fields; i++) {
-		unsigned long field = shadow_read_write_fields[i];
-
-		clear_bit(field, vmx_vmwrite_bitmap);
-		clear_bit(field, vmx_vmread_bitmap);
-		if (vmcs_field_type(field) == VMCS_FIELD_TYPE_U64) {
-			clear_bit(field + 1, vmx_vmwrite_bitmap);
-			clear_bit(field + 1, vmx_vmread_bitmap);
-		}
-	}
-	for (i = 0; i < max_shadow_read_only_fields; i++) {
-		unsigned long field = shadow_read_only_fields[i];
-
-		clear_bit(field, vmx_vmread_bitmap);
-		if (vmcs_field_type(field) == VMCS_FIELD_TYPE_U64)
-			clear_bit(field + 1, vmx_vmread_bitmap);
-	}
 }
 
 static __init int alloc_kvm_area(void)
@@ -4142,9 +4259,10 @@ static void exit_lmode(struct kvm_vcpu *vcpu)
 
 #endif
 
-static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid)
+static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid,
+				bool invalidate_gpa)
 {
-	if (enable_ept) {
+	if (enable_ept && (invalidate_gpa || !enable_vpid)) {
 		if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
 			return;
 		ept_sync_context(construct_eptp(vcpu, vcpu->arch.mmu.root_hpa));
@@ -4153,15 +4271,15 @@ static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid)
 	}
 }
 
-static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
+static void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
 {
-	__vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid);
+	__vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid, invalidate_gpa);
 }
 
 static void vmx_flush_tlb_ept_only(struct kvm_vcpu *vcpu)
 {
 	if (enable_ept)
-		vmx_flush_tlb(vcpu);
+		vmx_flush_tlb(vcpu, true);
 }
 
 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
@@ -4359,7 +4477,7 @@ static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 		ept_load_pdptrs(vcpu);
 	}
 
-	vmx_flush_tlb(vcpu);
+	vmx_flush_tlb(vcpu, true);
 	vmcs_writel(GUEST_CR3, guest_cr3);
 }
 
@@ -4376,6 +4494,15 @@ static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 		(to_vmx(vcpu)->rmode.vm86_active ?
 		 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON);
 
+	if ((cr4 & X86_CR4_UMIP) && !boot_cpu_has(X86_FEATURE_UMIP)) {
+		vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
+			      SECONDARY_EXEC_DESC);
+		hw_cr4 &= ~X86_CR4_UMIP;
+	} else if (!is_guest_mode(vcpu) ||
+	           !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC))
+		vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
+				SECONDARY_EXEC_DESC);
+
 	if (cr4 & X86_CR4_VMXE) {
 		/*
 		 * To use VMXON (and later other VMX instructions), a guest
@@ -4918,10 +5045,8 @@ static void free_vpid(int vpid)
 	spin_unlock(&vmx_vpid_lock);
 }
 
-#define MSR_TYPE_R	1
-#define MSR_TYPE_W	2
-static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
-						u32 msr, int type)
+static void __always_inline vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
+							  u32 msr, int type)
 {
 	int f = sizeof(unsigned long);
 
@@ -4955,6 +5080,50 @@ static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
 	}
 }
 
+static void __always_inline vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
+							 u32 msr, int type)
+{
+	int f = sizeof(unsigned long);
+
+	if (!cpu_has_vmx_msr_bitmap())
+		return;
+
+	/*
+	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
+	 * have the write-low and read-high bitmap offsets the wrong way round.
+	 * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
+	 */
+	if (msr <= 0x1fff) {
+		if (type & MSR_TYPE_R)
+			/* read-low */
+			__set_bit(msr, msr_bitmap + 0x000 / f);
+
+		if (type & MSR_TYPE_W)
+			/* write-low */
+			__set_bit(msr, msr_bitmap + 0x800 / f);
+
+	} else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
+		msr &= 0x1fff;
+		if (type & MSR_TYPE_R)
+			/* read-high */
+			__set_bit(msr, msr_bitmap + 0x400 / f);
+
+		if (type & MSR_TYPE_W)
+			/* write-high */
+			__set_bit(msr, msr_bitmap + 0xc00 / f);
+
+	}
+}
+
+static void __always_inline vmx_set_intercept_for_msr(unsigned long *msr_bitmap,
+			     			      u32 msr, int type, bool value)
+{
+	if (value)
+		vmx_enable_intercept_for_msr(msr_bitmap, msr, type);
+	else
+		vmx_disable_intercept_for_msr(msr_bitmap, msr, type);
+}
+
 /*
  * If a msr is allowed by L0, we should check whether it is allowed by L1.
  * The corresponding bit will be cleared unless both of L0 and L1 allow it.
@@ -4965,11 +5134,6 @@ static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
 {
 	int f = sizeof(unsigned long);
 
-	if (!cpu_has_vmx_msr_bitmap()) {
-		WARN_ON(1);
-		return;
-	}
-
 	/*
 	 * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
 	 * have the write-low and read-high bitmap offsets the wrong way round.
@@ -5001,30 +5165,70 @@ static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
 	}
 }
 
-static void vmx_disable_intercept_for_msr(u32 msr, bool longmode_only)
+static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu)
 {
-	if (!longmode_only)
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy,
-						msr, MSR_TYPE_R | MSR_TYPE_W);
-	__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode,
-						msr, MSR_TYPE_R | MSR_TYPE_W);
+	u8 mode = 0;
+
+	if (cpu_has_secondary_exec_ctrls() &&
+	    (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &
+	     SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
+		mode |= MSR_BITMAP_MODE_X2APIC;
+		if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
+			mode |= MSR_BITMAP_MODE_X2APIC_APICV;
+	}
+
+	if (is_long_mode(vcpu))
+		mode |= MSR_BITMAP_MODE_LM;
+
+	return mode;
 }
 
-static void vmx_disable_intercept_msr_x2apic(u32 msr, int type, bool apicv_active)
+#define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4))
+
+static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap,
+					 u8 mode)
 {
-	if (apicv_active) {
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic_apicv,
-				msr, type);
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic_apicv,
-				msr, type);
-	} else {
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_legacy_x2apic,
-				msr, type);
-		__vmx_disable_intercept_for_msr(vmx_msr_bitmap_longmode_x2apic,
-				msr, type);
+	int msr;
+
+	for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
+		unsigned word = msr / BITS_PER_LONG;
+		msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
+		msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
+	}
+
+	if (mode & MSR_BITMAP_MODE_X2APIC) {
+		/*
+		 * TPR reads and writes can be virtualized even if virtual interrupt
+		 * delivery is not in use.
+		 */
+		vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW);
+		if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
+			vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R);
+			vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
+			vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
+		}
 	}
 }
 
+static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu)
+{
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
+	u8 mode = vmx_msr_bitmap_mode(vcpu);
+	u8 changed = mode ^ vmx->msr_bitmap_mode;
+
+	if (!changed)
+		return;
+
+	vmx_set_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW,
+				  !(mode & MSR_BITMAP_MODE_LM));
+
+	if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV))
+		vmx_update_msr_bitmap_x2apic(msr_bitmap, mode);
+
+	vmx->msr_bitmap_mode = mode;
+}
+
 static bool vmx_get_enable_apicv(struct kvm_vcpu *vcpu)
 {
 	return enable_apicv;
@@ -5069,7 +5273,8 @@ static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
 	max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
 	if (max_irr != 256) {
 		vapic_page = kmap(vmx->nested.virtual_apic_page);
-		__kvm_apic_update_irr(vmx->nested.pi_desc->pir, vapic_page);
+		__kvm_apic_update_irr(vmx->nested.pi_desc->pir,
+			vapic_page, &max_irr);
 		kunmap(vmx->nested.virtual_apic_page);
 
 		status = vmcs_read16(GUEST_INTR_STATUS);
@@ -5129,14 +5334,15 @@ static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
 
 	if (is_guest_mode(vcpu) &&
 	    vector == vmx->nested.posted_intr_nv) {
-		/* the PIR and ON have been set by L1. */
-		kvm_vcpu_trigger_posted_interrupt(vcpu, true);
 		/*
 		 * If a posted intr is not recognized by hardware,
 		 * we will accomplish it in the next vmentry.
 		 */
 		vmx->nested.pi_pending = true;
 		kvm_make_request(KVM_REQ_EVENT, vcpu);
+		/* the PIR and ON have been set by L1. */
+		if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true))
+			kvm_vcpu_kick(vcpu);
 		return 0;
 	}
 	return -1;
@@ -5274,7 +5480,7 @@ static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
 	}
 
 	if (cpu_has_vmx_msr_bitmap())
-		vmx_set_msr_bitmap(vcpu);
+		vmx_update_msr_bitmap(vcpu);
 }
 
 static u32 vmx_exec_control(struct vcpu_vmx *vmx)
@@ -5315,6 +5521,7 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
 	struct kvm_vcpu *vcpu = &vmx->vcpu;
 
 	u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
+
 	if (!cpu_need_virtualize_apic_accesses(vcpu))
 		exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
 	if (vmx->vpid == 0)
@@ -5333,6 +5540,11 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
 		exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
 				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
 	exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
+
+	/* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP,
+	 * in vmx_set_cr4.  */
+	exec_control &= ~SECONDARY_EXEC_DESC;
+
 	/* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
 	   (handle_vmptrld).
 	   We can NOT enable shadow_vmcs here because we don't have yet
@@ -5452,16 +5664,12 @@ static void vmx_vcpu_setup(struct vcpu_vmx *vmx)
 #endif
 	int i;
 
-	/* I/O */
-	vmcs_write64(IO_BITMAP_A, __pa(vmx_io_bitmap_a));
-	vmcs_write64(IO_BITMAP_B, __pa(vmx_io_bitmap_b));
-
 	if (enable_shadow_vmcs) {
 		vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
 		vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
 	}
 	if (cpu_has_vmx_msr_bitmap())
-		vmcs_write64(MSR_BITMAP, __pa(vmx_msr_bitmap_legacy));
+		vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
 
 	vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
 
@@ -5539,6 +5747,8 @@ static void vmx_vcpu_setup(struct vcpu_vmx *vmx)
 		++vmx->nmsrs;
 	}
 
+	if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
+		rdmsrl(MSR_IA32_ARCH_CAPABILITIES, vmx->arch_capabilities);
 
 	vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl);
 
@@ -5567,7 +5777,9 @@ static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	u64 cr0;
 
 	vmx->rmode.vm86_active = 0;
+	vmx->spec_ctrl = 0;
 
+	vcpu->arch.microcode_version = 0x100000000ULL;
 	vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
 	kvm_set_cr8(vcpu, 0);
 
@@ -5974,7 +6186,7 @@ static int handle_exception(struct kvm_vcpu *vcpu)
 		      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
 			vcpu->arch.dr6 &= ~15;
 			vcpu->arch.dr6 |= dr6 | DR6_RTM;
-			if (!(dr6 & ~DR6_RESERVED)) /* icebp */
+			if (is_icebp(intr_info))
 				skip_emulated_instruction(vcpu);
 
 			kvm_queue_exception(vcpu, DB_VECTOR);
@@ -6107,6 +6319,12 @@ static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
 		return kvm_set_cr4(vcpu, val);
 }
 
+static int handle_desc(struct kvm_vcpu *vcpu)
+{
+	WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP));
+	return emulate_instruction(vcpu, 0) == EMULATE_DONE;
+}
+
 static int handle_cr(struct kvm_vcpu *vcpu)
 {
 	unsigned long exit_qualification, val;
@@ -6563,7 +6781,21 @@ static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
 	if (!is_guest_mode(vcpu) &&
 	    !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
 		trace_kvm_fast_mmio(gpa);
-		return kvm_skip_emulated_instruction(vcpu);
+		/*
+		 * Doing kvm_skip_emulated_instruction() depends on undefined
+		 * behavior: Intel's manual doesn't mandate
+		 * VM_EXIT_INSTRUCTION_LEN to be set in VMCS when EPT MISCONFIG
+		 * occurs and while on real hardware it was observed to be set,
+		 * other hypervisors (namely Hyper-V) don't set it, we end up
+		 * advancing IP with some random value. Disable fast mmio when
+		 * running nested and keep it for real hardware in hope that
+		 * VM_EXIT_INSTRUCTION_LEN will always be set correctly.
+		 */
+		if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
+			return kvm_skip_emulated_instruction(vcpu);
+		else
+			return x86_emulate_instruction(vcpu, gpa, EMULTYPE_SKIP,
+						       NULL, 0) == EMULATE_DONE;
 	}
 
 	ret = kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
@@ -6744,7 +6976,7 @@ void vmx_enable_tdp(void)
 
 static __init int hardware_setup(void)
 {
-	int r = -ENOMEM, i, msr;
+	int r = -ENOMEM, i;
 
 	rdmsrl_safe(MSR_EFER, &host_efer);
 
@@ -6760,13 +6992,6 @@ static __init int hardware_setup(void)
 	memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
 	memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
 
-	memset(vmx_io_bitmap_a, 0xff, PAGE_SIZE);
-
-	memset(vmx_io_bitmap_b, 0xff, PAGE_SIZE);
-
-	memset(vmx_msr_bitmap_legacy, 0xff, PAGE_SIZE);
-	memset(vmx_msr_bitmap_longmode, 0xff, PAGE_SIZE);
-
 	if (setup_vmcs_config(&vmcs_config) < 0) {
 		r = -EIO;
 		goto out;
@@ -6779,11 +7004,6 @@ static __init int hardware_setup(void)
 		!(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
 		enable_vpid = 0;
 
-	if (!cpu_has_vmx_shadow_vmcs())
-		enable_shadow_vmcs = 0;
-	if (enable_shadow_vmcs)
-		init_vmcs_shadow_fields();
-
 	if (!cpu_has_vmx_ept() ||
 	    !cpu_has_vmx_ept_4levels() ||
 	    !cpu_has_vmx_ept_mt_wb() ||
@@ -6835,42 +7055,8 @@ static __init int hardware_setup(void)
 		kvm_tsc_scaling_ratio_frac_bits = 48;
 	}
 
-	vmx_disable_intercept_for_msr(MSR_FS_BASE, false);
-	vmx_disable_intercept_for_msr(MSR_GS_BASE, false);
-	vmx_disable_intercept_for_msr(MSR_KERNEL_GS_BASE, true);
-	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_CS, false);
-	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_ESP, false);
-	vmx_disable_intercept_for_msr(MSR_IA32_SYSENTER_EIP, false);
-
-	memcpy(vmx_msr_bitmap_legacy_x2apic_apicv,
-			vmx_msr_bitmap_legacy, PAGE_SIZE);
-	memcpy(vmx_msr_bitmap_longmode_x2apic_apicv,
-			vmx_msr_bitmap_longmode, PAGE_SIZE);
-	memcpy(vmx_msr_bitmap_legacy_x2apic,
-			vmx_msr_bitmap_legacy, PAGE_SIZE);
-	memcpy(vmx_msr_bitmap_longmode_x2apic,
-			vmx_msr_bitmap_longmode, PAGE_SIZE);
-
 	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
 
-	for (msr = 0x800; msr <= 0x8ff; msr++) {
-		if (msr == 0x839 /* TMCCT */)
-			continue;
-		vmx_disable_intercept_msr_x2apic(msr, MSR_TYPE_R, true);
-	}
-
-	/*
-	 * TPR reads and writes can be virtualized even if virtual interrupt
-	 * delivery is not in use.
-	 */
-	vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_W, true);
-	vmx_disable_intercept_msr_x2apic(0x808, MSR_TYPE_R | MSR_TYPE_W, false);
-
-	/* EOI */
-	vmx_disable_intercept_msr_x2apic(0x80b, MSR_TYPE_W, true);
-	/* SELF-IPI */
-	vmx_disable_intercept_msr_x2apic(0x83f, MSR_TYPE_W, true);
-
 	if (enable_ept)
 		vmx_enable_tdp();
 	else
@@ -6903,6 +7089,11 @@ static __init int hardware_setup(void)
 		kvm_x86_ops->cancel_hv_timer = NULL;
 	}
 
+	if (!cpu_has_vmx_shadow_vmcs())
+		enable_shadow_vmcs = 0;
+	if (enable_shadow_vmcs)
+		init_vmcs_shadow_fields();
+
 	kvm_set_posted_intr_wakeup_handler(wakeup_handler);
 
 	kvm_mce_cap_supported |= MCG_LMCE_P;
@@ -6973,94 +7164,6 @@ static int handle_monitor(struct kvm_vcpu *vcpu)
 	return handle_nop(vcpu);
 }
 
-/*
- * To run an L2 guest, we need a vmcs02 based on the L1-specified vmcs12.
- * We could reuse a single VMCS for all the L2 guests, but we also want the
- * option to allocate a separate vmcs02 for each separate loaded vmcs12 - this
- * allows keeping them loaded on the processor, and in the future will allow
- * optimizations where prepare_vmcs02 doesn't need to set all the fields on
- * every entry if they never change.
- * So we keep, in vmx->nested.vmcs02_pool, a cache of size VMCS02_POOL_SIZE
- * (>=0) with a vmcs02 for each recently loaded vmcs12s, most recent first.
- *
- * The following functions allocate and free a vmcs02 in this pool.
- */
-
-/* Get a VMCS from the pool to use as vmcs02 for the current vmcs12. */
-static struct loaded_vmcs *nested_get_current_vmcs02(struct vcpu_vmx *vmx)
-{
-	struct vmcs02_list *item;
-	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
-		if (item->vmptr == vmx->nested.current_vmptr) {
-			list_move(&item->list, &vmx->nested.vmcs02_pool);
-			return &item->vmcs02;
-		}
-
-	if (vmx->nested.vmcs02_num >= max(VMCS02_POOL_SIZE, 1)) {
-		/* Recycle the least recently used VMCS. */
-		item = list_last_entry(&vmx->nested.vmcs02_pool,
-				       struct vmcs02_list, list);
-		item->vmptr = vmx->nested.current_vmptr;
-		list_move(&item->list, &vmx->nested.vmcs02_pool);
-		return &item->vmcs02;
-	}
-
-	/* Create a new VMCS */
-	item = kzalloc(sizeof(struct vmcs02_list), GFP_KERNEL);
-	if (!item)
-		return NULL;
-	item->vmcs02.vmcs = alloc_vmcs();
-	item->vmcs02.shadow_vmcs = NULL;
-	if (!item->vmcs02.vmcs) {
-		kfree(item);
-		return NULL;
-	}
-	loaded_vmcs_init(&item->vmcs02);
-	item->vmptr = vmx->nested.current_vmptr;
-	list_add(&(item->list), &(vmx->nested.vmcs02_pool));
-	vmx->nested.vmcs02_num++;
-	return &item->vmcs02;
-}
-
-/* Free and remove from pool a vmcs02 saved for a vmcs12 (if there is one) */
-static void nested_free_vmcs02(struct vcpu_vmx *vmx, gpa_t vmptr)
-{
-	struct vmcs02_list *item;
-	list_for_each_entry(item, &vmx->nested.vmcs02_pool, list)
-		if (item->vmptr == vmptr) {
-			free_loaded_vmcs(&item->vmcs02);
-			list_del(&item->list);
-			kfree(item);
-			vmx->nested.vmcs02_num--;
-			return;
-		}
-}
-
-/*
- * Free all VMCSs saved for this vcpu, except the one pointed by
- * vmx->loaded_vmcs. We must be running L1, so vmx->loaded_vmcs
- * must be &vmx->vmcs01.
- */
-static void nested_free_all_saved_vmcss(struct vcpu_vmx *vmx)
-{
-	struct vmcs02_list *item, *n;
-
-	WARN_ON(vmx->loaded_vmcs != &vmx->vmcs01);
-	list_for_each_entry_safe(item, n, &vmx->nested.vmcs02_pool, list) {
-		/*
-		 * Something will leak if the above WARN triggers.  Better than
-		 * a use-after-free.
-		 */
-		if (vmx->loaded_vmcs == &item->vmcs02)
-			continue;
-
-		free_loaded_vmcs(&item->vmcs02);
-		list_del(&item->list);
-		kfree(item);
-		vmx->nested.vmcs02_num--;
-	}
-}
-
 /*
  * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
  * set the success or error code of an emulated VMX instruction, as specified
@@ -7241,13 +7344,11 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct vmcs *shadow_vmcs;
+	int r;
 
-	if (cpu_has_vmx_msr_bitmap()) {
-		vmx->nested.msr_bitmap =
-				(unsigned long *)__get_free_page(GFP_KERNEL);
-		if (!vmx->nested.msr_bitmap)
-			goto out_msr_bitmap;
-	}
+	r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
+	if (r < 0)
+		goto out_vmcs02;
 
 	vmx->nested.cached_vmcs12 = kmalloc(VMCS12_SIZE, GFP_KERNEL);
 	if (!vmx->nested.cached_vmcs12)
@@ -7264,9 +7365,6 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu)
 		vmx->vmcs01.shadow_vmcs = shadow_vmcs;
 	}
 
-	INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool));
-	vmx->nested.vmcs02_num = 0;
-
 	hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
 		     HRTIMER_MODE_REL_PINNED);
 	vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
@@ -7278,9 +7376,9 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu)
 	kfree(vmx->nested.cached_vmcs12);
 
 out_cached_vmcs12:
-	free_page((unsigned long)vmx->nested.msr_bitmap);
+	free_loaded_vmcs(&vmx->nested.vmcs02);
 
-out_msr_bitmap:
+out_vmcs02:
 	return -ENOMEM;
 }
 
@@ -7423,10 +7521,6 @@ static void free_nested(struct vcpu_vmx *vmx)
 	free_vpid(vmx->nested.vpid02);
 	vmx->nested.posted_intr_nv = -1;
 	vmx->nested.current_vmptr = -1ull;
-	if (vmx->nested.msr_bitmap) {
-		free_page((unsigned long)vmx->nested.msr_bitmap);
-		vmx->nested.msr_bitmap = NULL;
-	}
 	if (enable_shadow_vmcs) {
 		vmx_disable_shadow_vmcs(vmx);
 		vmcs_clear(vmx->vmcs01.shadow_vmcs);
@@ -7434,7 +7528,7 @@ static void free_nested(struct vcpu_vmx *vmx)
 		vmx->vmcs01.shadow_vmcs = NULL;
 	}
 	kfree(vmx->nested.cached_vmcs12);
-	/* Unpin physical memory we referred to in current vmcs02 */
+	/* Unpin physical memory we referred to in the vmcs02 */
 	if (vmx->nested.apic_access_page) {
 		kvm_release_page_dirty(vmx->nested.apic_access_page);
 		vmx->nested.apic_access_page = NULL;
@@ -7450,7 +7544,7 @@ static void free_nested(struct vcpu_vmx *vmx)
 		vmx->nested.pi_desc = NULL;
 	}
 
-	nested_free_all_saved_vmcss(vmx);
+	free_loaded_vmcs(&vmx->nested.vmcs02);
 }
 
 /* Emulate the VMXOFF instruction */
@@ -7493,8 +7587,6 @@ static int handle_vmclear(struct kvm_vcpu *vcpu)
 			vmptr + offsetof(struct vmcs12, launch_state),
 			&zero, sizeof(zero));
 
-	nested_free_vmcs02(vmx, vmptr);
-
 	nested_vmx_succeed(vcpu);
 	return kvm_skip_emulated_instruction(vcpu);
 }
@@ -7532,17 +7624,17 @@ static inline int vmcs12_read_any(struct kvm_vcpu *vcpu,
 
 	p = ((char *)(get_vmcs12(vcpu))) + offset;
 
-	switch (vmcs_field_type(field)) {
-	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
+	switch (vmcs_field_width(field)) {
+	case VMCS_FIELD_WIDTH_NATURAL_WIDTH:
 		*ret = *((natural_width *)p);
 		return 0;
-	case VMCS_FIELD_TYPE_U16:
+	case VMCS_FIELD_WIDTH_U16:
 		*ret = *((u16 *)p);
 		return 0;
-	case VMCS_FIELD_TYPE_U32:
+	case VMCS_FIELD_WIDTH_U32:
 		*ret = *((u32 *)p);
 		return 0;
-	case VMCS_FIELD_TYPE_U64:
+	case VMCS_FIELD_WIDTH_U64:
 		*ret = *((u64 *)p);
 		return 0;
 	default:
@@ -7559,17 +7651,17 @@ static inline int vmcs12_write_any(struct kvm_vcpu *vcpu,
 	if (offset < 0)
 		return offset;
 
-	switch (vmcs_field_type(field)) {
-	case VMCS_FIELD_TYPE_U16:
+	switch (vmcs_field_width(field)) {
+	case VMCS_FIELD_WIDTH_U16:
 		*(u16 *)p = field_value;
 		return 0;
-	case VMCS_FIELD_TYPE_U32:
+	case VMCS_FIELD_WIDTH_U32:
 		*(u32 *)p = field_value;
 		return 0;
-	case VMCS_FIELD_TYPE_U64:
+	case VMCS_FIELD_WIDTH_U64:
 		*(u64 *)p = field_value;
 		return 0;
-	case VMCS_FIELD_TYPE_NATURAL_WIDTH:
+	case VMCS_FIELD_WIDTH_NATURAL_WIDTH:
 		*(natural_width *)p = field_value;
 		return 0;
 	default:
@@ -7585,7 +7677,7 @@ static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
 	unsigned long field;
 	u64 field_value;
 	struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
-	const unsigned long *fields = shadow_read_write_fields;
+	const u16 *fields = shadow_read_write_fields;
 	const int num_fields = max_shadow_read_write_fields;
 
 	preempt_disable();
@@ -7594,23 +7686,7 @@ static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
 
 	for (i = 0; i < num_fields; i++) {
 		field = fields[i];
-		switch (vmcs_field_type(field)) {
-		case VMCS_FIELD_TYPE_U16:
-			field_value = vmcs_read16(field);
-			break;
-		case VMCS_FIELD_TYPE_U32:
-			field_value = vmcs_read32(field);
-			break;
-		case VMCS_FIELD_TYPE_U64:
-			field_value = vmcs_read64(field);
-			break;
-		case VMCS_FIELD_TYPE_NATURAL_WIDTH:
-			field_value = vmcs_readl(field);
-			break;
-		default:
-			WARN_ON(1);
-			continue;
-		}
+		field_value = __vmcs_readl(field);
 		vmcs12_write_any(&vmx->vcpu, field, field_value);
 	}
 
@@ -7622,7 +7698,7 @@ static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
 
 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
 {
-	const unsigned long *fields[] = {
+	const u16 *fields[] = {
 		shadow_read_write_fields,
 		shadow_read_only_fields
 	};
@@ -7641,24 +7717,7 @@ static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
 		for (i = 0; i < max_fields[q]; i++) {
 			field = fields[q][i];
 			vmcs12_read_any(&vmx->vcpu, field, &field_value);
-
-			switch (vmcs_field_type(field)) {
-			case VMCS_FIELD_TYPE_U16:
-				vmcs_write16(field, (u16)field_value);
-				break;
-			case VMCS_FIELD_TYPE_U32:
-				vmcs_write32(field, (u32)field_value);
-				break;
-			case VMCS_FIELD_TYPE_U64:
-				vmcs_write64(field, (u64)field_value);
-				break;
-			case VMCS_FIELD_TYPE_NATURAL_WIDTH:
-				vmcs_writel(field, (long)field_value);
-				break;
-			default:
-				WARN_ON(1);
-				break;
-			}
+			__vmcs_writel(field, field_value);
 		}
 	}
 
@@ -7727,8 +7786,10 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
 {
 	unsigned long field;
 	gva_t gva;
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 	u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
+
 	/* The value to write might be 32 or 64 bits, depending on L1's long
 	 * mode, and eventually we need to write that into a field of several
 	 * possible lengths. The code below first zero-extends the value to 64
@@ -7771,6 +7832,20 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu)
 		return kvm_skip_emulated_instruction(vcpu);
 	}
 
+	switch (field) {
+#define SHADOW_FIELD_RW(x) case x:
+#include "vmx_shadow_fields.h"
+		/*
+		 * The fields that can be updated by L1 without a vmexit are
+		 * always updated in the vmcs02, the others go down the slow
+		 * path of prepare_vmcs02.
+		 */
+		break;
+	default:
+		vmx->nested.dirty_vmcs12 = true;
+		break;
+	}
+
 	nested_vmx_succeed(vcpu);
 	return kvm_skip_emulated_instruction(vcpu);
 }
@@ -7785,6 +7860,7 @@ static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
 			     __pa(vmx->vmcs01.shadow_vmcs));
 		vmx->nested.sync_shadow_vmcs = true;
 	}
+	vmx->nested.dirty_vmcs12 = true;
 }
 
 /* Emulate the VMPTRLD instruction */
@@ -8005,7 +8081,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
 		return kvm_skip_emulated_instruction(vcpu);
 	}
 
-	__vmx_flush_tlb(vcpu, vmx->nested.vpid02);
+	__vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
 	nested_vmx_succeed(vcpu);
 
 	return kvm_skip_emulated_instruction(vcpu);
@@ -8199,6 +8275,8 @@ static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
 	[EXIT_REASON_XSETBV]                  = handle_xsetbv,
 	[EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
 	[EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
+	[EXIT_REASON_GDTR_IDTR]		      = handle_desc,
+	[EXIT_REASON_LDTR_TR]		      = handle_desc,
 	[EXIT_REASON_EPT_VIOLATION]	      = handle_ept_violation,
 	[EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
 	[EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
@@ -8406,10 +8484,11 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
 
 	/*
 	 * The host physical addresses of some pages of guest memory
-	 * are loaded into VMCS02 (e.g. L1's Virtual APIC Page). The CPU
-	 * may write to these pages via their host physical address while
-	 * L2 is running, bypassing any address-translation-based dirty
-	 * tracking (e.g. EPT write protection).
+	 * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
+	 * Page). The CPU may write to these pages via their host
+	 * physical address while L2 is running, bypassing any
+	 * address-translation-based dirty tracking (e.g. EPT write
+	 * protection).
 	 *
 	 * Mark them dirty on every exit from L2 to prevent them from
 	 * getting out of sync with dirty tracking.
@@ -8943,7 +9022,7 @@ static void vmx_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
 	}
 	vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
 
-	vmx_set_msr_bitmap(vcpu);
+	vmx_update_msr_bitmap(vcpu);
 }
 
 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
@@ -9007,36 +9086,23 @@ static void vmx_set_rvi(int vector)
 
 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
 {
-	if (!is_guest_mode(vcpu)) {
-		vmx_set_rvi(max_irr);
-		return;
-	}
-
-	if (max_irr == -1)
-		return;
-
 	/*
-	 * In guest mode.  If a vmexit is needed, vmx_check_nested_events
-	 * handles it.
+	 * When running L2, updating RVI is only relevant when
+	 * vmcs12 virtual-interrupt-delivery enabled.
+	 * However, it can be enabled only when L1 also
+	 * intercepts external-interrupts and in that case
+	 * we should not update vmcs02 RVI but instead intercept
+	 * interrupt. Therefore, do nothing when running L2.
 	 */
-	if (nested_exit_on_intr(vcpu))
-		return;
-
-	/*
-	 * Else, fall back to pre-APICv interrupt injection since L2
-	 * is run without virtual interrupt delivery.
-	 */
-	if (!kvm_event_needs_reinjection(vcpu) &&
-	    vmx_interrupt_allowed(vcpu)) {
-		kvm_queue_interrupt(vcpu, max_irr, false);
-		vmx_inject_irq(vcpu);
-	}
+	if (!is_guest_mode(vcpu))
+		vmx_set_rvi(max_irr);
 }
 
 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	int max_irr;
+	bool max_irr_updated;
 
 	WARN_ON(!vcpu->arch.apicv_active);
 	if (pi_test_on(&vmx->pi_desc)) {
@@ -9046,7 +9112,23 @@ static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
 		 * But on x86 this is just a compiler barrier anyway.
 		 */
 		smp_mb__after_atomic();
-		max_irr = kvm_apic_update_irr(vcpu, vmx->pi_desc.pir);
+		max_irr_updated =
+			kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
+
+		/*
+		 * If we are running L2 and L1 has a new pending interrupt
+		 * which can be injected, we should re-evaluate
+		 * what should be done with this new L1 interrupt.
+		 * If L1 intercepts external-interrupts, we should
+		 * exit from L2 to L1. Otherwise, interrupt should be
+		 * delivered directly to L2.
+		 */
+		if (is_guest_mode(vcpu) && max_irr_updated) {
+			if (nested_exit_on_intr(vcpu))
+				kvm_vcpu_exiting_guest_mode(vcpu);
+			else
+				kvm_make_request(KVM_REQ_EVENT, vcpu);
+		}
 	} else {
 		max_irr = kvm_lapic_find_highest_irr(vcpu);
 	}
@@ -9129,14 +9211,14 @@ static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
 #endif
 			"pushf\n\t"
 			__ASM_SIZE(push) " $%c[cs]\n\t"
-			"call *%[entry]\n\t"
+			CALL_NOSPEC
 			:
 #ifdef CONFIG_X86_64
 			[sp]"=&r"(tmp),
 #endif
 			ASM_CALL_CONSTRAINT
 			:
-			[entry]"r"(entry),
+			THUNK_TARGET(entry),
 			[ss]"i"(__KERNEL_DS),
 			[cs]"i"(__KERNEL_CS)
 			);
@@ -9161,6 +9243,12 @@ static bool vmx_xsaves_supported(void)
 		SECONDARY_EXEC_XSAVES;
 }
 
+static bool vmx_umip_emulated(void)
+{
+	return vmcs_config.cpu_based_2nd_exec_ctrl &
+		SECONDARY_EXEC_DESC;
+}
+
 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
 {
 	u32 exit_intr_info;
@@ -9316,7 +9404,7 @@ static void vmx_arm_hv_timer(struct kvm_vcpu *vcpu)
 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	unsigned long debugctlmsr, cr3, cr4;
+	unsigned long cr3, cr4;
 
 	/* Record the guest's net vcpu time for enforced NMI injections. */
 	if (unlikely(!enable_vnmi &&
@@ -9369,10 +9457,18 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 		__write_pkru(vcpu->arch.pkru);
 
 	atomic_switch_perf_msrs(vmx);
-	debugctlmsr = get_debugctlmsr();
 
 	vmx_arm_hv_timer(vcpu);
 
+	/*
+	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
+	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
+	 * is no need to worry about the conditional branch over the wrmsr
+	 * being speculatively taken.
+	 */
+	if (vmx->spec_ctrl)
+		native_wrmsrl(MSR_IA32_SPEC_CTRL, vmx->spec_ctrl);
+
 	vmx->__launched = vmx->loaded_vmcs->launched;
 	asm(
 		/* Store host registers */
@@ -9491,12 +9587,33 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
 #endif
 	      );
 
+	/*
+	 * We do not use IBRS in the kernel. If this vCPU has used the
+	 * SPEC_CTRL MSR it may have left it on; save the value and
+	 * turn it off. This is much more efficient than blindly adding
+	 * it to the atomic save/restore list. Especially as the former
+	 * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
+	 *
+	 * For non-nested case:
+	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
+	 * save it.
+	 *
+	 * For nested case:
+	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
+	 * save it.
+	 */
+	if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
+		vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
+
+	if (vmx->spec_ctrl)
+		native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
+
 	/* Eliminate branch target predictions from guest mode */
 	vmexit_fill_RSB();
 
 	/* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
-	if (debugctlmsr)
-		update_debugctlmsr(debugctlmsr);
+	if (vmx->host_debugctlmsr)
+		update_debugctlmsr(vmx->host_debugctlmsr);
 
 #ifndef CONFIG_X86_64
 	/*
@@ -9576,10 +9693,8 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
 static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
 {
        struct vcpu_vmx *vmx = to_vmx(vcpu);
-       int r;
 
-       r = vcpu_load(vcpu);
-       BUG_ON(r);
+       vcpu_load(vcpu);
        vmx_switch_vmcs(vcpu, &vmx->vmcs01);
        free_nested(vmx);
        vcpu_put(vcpu);
@@ -9604,6 +9719,7 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
 {
 	int err;
 	struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
+	unsigned long *msr_bitmap;
 	int cpu;
 
 	if (!vmx)
@@ -9636,13 +9752,20 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
 	if (!vmx->guest_msrs)
 		goto free_pml;
 
-	vmx->loaded_vmcs = &vmx->vmcs01;
-	vmx->loaded_vmcs->vmcs = alloc_vmcs();
-	vmx->loaded_vmcs->shadow_vmcs = NULL;
-	if (!vmx->loaded_vmcs->vmcs)
+	err = alloc_loaded_vmcs(&vmx->vmcs01);
+	if (err < 0)
 		goto free_msrs;
-	loaded_vmcs_init(vmx->loaded_vmcs);
 
+	msr_bitmap = vmx->vmcs01.msr_bitmap;
+	vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW);
+	vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW);
+	vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
+	vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
+	vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
+	vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
+	vmx->msr_bitmap_mode = 0;
+
+	vmx->loaded_vmcs = &vmx->vmcs01;
 	cpu = get_cpu();
 	vmx_vcpu_load(&vmx->vcpu, cpu);
 	vmx->vcpu.cpu = cpu;
@@ -9771,7 +9894,8 @@ static void vmcs_set_secondary_exec_control(u32 new_ctl)
 	u32 mask =
 		SECONDARY_EXEC_SHADOW_VMCS |
 		SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
-		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
+		SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
+		SECONDARY_EXEC_DESC;
 
 	u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
 
@@ -9937,8 +10061,8 @@ static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
 	}
 }
 
-static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
-					       struct vmcs12 *vmcs12);
+static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
+						 struct vmcs12 *vmcs12);
 
 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
 					struct vmcs12 *vmcs12)
@@ -10027,10 +10151,9 @@ static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu,
 			(unsigned long)(vmcs12->posted_intr_desc_addr &
 			(PAGE_SIZE - 1)));
 	}
-	if (cpu_has_vmx_msr_bitmap() &&
-	    nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS) &&
-	    nested_vmx_merge_msr_bitmap(vcpu, vmcs12))
-		;
+	if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12))
+		vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
+			      CPU_BASED_USE_MSR_BITMAPS);
 	else
 		vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
 				CPU_BASED_USE_MSR_BITMAPS);
@@ -10099,48 +10222,90 @@ static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
  * Merge L0's and L1's MSR bitmap, return false to indicate that
  * we do not use the hardware.
  */
-static inline bool nested_vmx_merge_msr_bitmap(struct kvm_vcpu *vcpu,
-					       struct vmcs12 *vmcs12)
+static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
+						 struct vmcs12 *vmcs12)
 {
 	int msr;
 	struct page *page;
 	unsigned long *msr_bitmap_l1;
-	unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.msr_bitmap;
+	unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
+	/*
+	 * pred_cmd & spec_ctrl are trying to verify two things:
+	 *
+	 * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
+	 *    ensures that we do not accidentally generate an L02 MSR bitmap
+	 *    from the L12 MSR bitmap that is too permissive.
+	 * 2. That L1 or L2s have actually used the MSR. This avoids
+	 *    unnecessarily merging of the bitmap if the MSR is unused. This
+	 *    works properly because we only update the L01 MSR bitmap lazily.
+	 *    So even if L0 should pass L1 these MSRs, the L01 bitmap is only
+	 *    updated to reflect this when L1 (or its L2s) actually write to
+	 *    the MSR.
+	 */
+	bool pred_cmd = !msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD);
+	bool spec_ctrl = !msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL);
 
-	/* This shortcut is ok because we support only x2APIC MSRs so far. */
-	if (!nested_cpu_has_virt_x2apic_mode(vmcs12))
+	/* Nothing to do if the MSR bitmap is not in use.  */
+	if (!cpu_has_vmx_msr_bitmap() ||
+	    !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
+		return false;
+
+	if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
+	    !pred_cmd && !spec_ctrl)
 		return false;
 
 	page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap);
 	if (is_error_page(page))
 		return false;
+
 	msr_bitmap_l1 = (unsigned long *)kmap(page);
+	if (nested_cpu_has_apic_reg_virt(vmcs12)) {
+		/*
+		 * L0 need not intercept reads for MSRs between 0x800 and 0x8ff, it
+		 * just lets the processor take the value from the virtual-APIC page;
+		 * take those 256 bits directly from the L1 bitmap.
+		 */
+		for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
+			unsigned word = msr / BITS_PER_LONG;
+			msr_bitmap_l0[word] = msr_bitmap_l1[word];
+			msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0;
+		}
+	} else {
+		for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
+			unsigned word = msr / BITS_PER_LONG;
+			msr_bitmap_l0[word] = ~0;
+			msr_bitmap_l0[word + (0x800 / sizeof(long))] = ~0;
+		}
+	}
+
+	nested_vmx_disable_intercept_for_msr(
+		msr_bitmap_l1, msr_bitmap_l0,
+		X2APIC_MSR(APIC_TASKPRI),
+		MSR_TYPE_W);
 
-	memset(msr_bitmap_l0, 0xff, PAGE_SIZE);
+	if (nested_cpu_has_vid(vmcs12)) {
+		nested_vmx_disable_intercept_for_msr(
+			msr_bitmap_l1, msr_bitmap_l0,
+			X2APIC_MSR(APIC_EOI),
+			MSR_TYPE_W);
+		nested_vmx_disable_intercept_for_msr(
+			msr_bitmap_l1, msr_bitmap_l0,
+			X2APIC_MSR(APIC_SELF_IPI),
+			MSR_TYPE_W);
+	}
 
-	if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
-		if (nested_cpu_has_apic_reg_virt(vmcs12))
-			for (msr = 0x800; msr <= 0x8ff; msr++)
-				nested_vmx_disable_intercept_for_msr(
+	if (spec_ctrl)
+		nested_vmx_disable_intercept_for_msr(
 					msr_bitmap_l1, msr_bitmap_l0,
-					msr, MSR_TYPE_R);
+					MSR_IA32_SPEC_CTRL,
+					MSR_TYPE_R | MSR_TYPE_W);
 
+	if (pred_cmd)
 		nested_vmx_disable_intercept_for_msr(
-				msr_bitmap_l1, msr_bitmap_l0,
-				APIC_BASE_MSR + (APIC_TASKPRI >> 4),
-				MSR_TYPE_R | MSR_TYPE_W);
-
-		if (nested_cpu_has_vid(vmcs12)) {
-			nested_vmx_disable_intercept_for_msr(
-				msr_bitmap_l1, msr_bitmap_l0,
-				APIC_BASE_MSR + (APIC_EOI >> 4),
-				MSR_TYPE_W);
-			nested_vmx_disable_intercept_for_msr(
-				msr_bitmap_l1, msr_bitmap_l0,
-				APIC_BASE_MSR + (APIC_SELF_IPI >> 4),
-				MSR_TYPE_W);
-		}
-	}
+					msr_bitmap_l1, msr_bitmap_l0,
+					MSR_IA32_PRED_CMD,
+					MSR_TYPE_W);
+
 	kunmap(page);
 	kvm_release_page_clean(page);
 
@@ -10406,25 +10571,12 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne
 	return 0;
 }
 
-/*
- * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
- * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
- * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
- * guest in a way that will both be appropriate to L1's requests, and our
- * needs. In addition to modifying the active vmcs (which is vmcs02), this
- * function also has additional necessary side-effects, like setting various
- * vcpu->arch fields.
- * Returns 0 on success, 1 on failure. Invalid state exit qualification code
- * is assigned to entry_failure_code on failure.
- */
-static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
-			  bool from_vmentry, u32 *entry_failure_code)
+static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
+			       bool from_vmentry)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
-	u32 exec_control, vmcs12_exec_ctrl;
 
 	vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
-	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
 	vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
 	vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
 	vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
@@ -10432,7 +10584,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
 	vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
 	vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
-	vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
 	vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
 	vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
 	vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
@@ -10442,15 +10593,12 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
 	vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
 	vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
-	vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
 	vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
 	vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
 	vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
 	vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
 	vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
 	vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
-	vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
-	vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
 	vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
 	vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
 	vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
@@ -10460,6 +10608,130 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
 	vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
 
+	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
+	vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
+		vmcs12->guest_pending_dbg_exceptions);
+	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
+	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
+
+	if (nested_cpu_has_xsaves(vmcs12))
+		vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
+	vmcs_write64(VMCS_LINK_POINTER, -1ull);
+
+	if (cpu_has_vmx_posted_intr())
+		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
+
+	/*
+	 * Whether page-faults are trapped is determined by a combination of
+	 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
+	 * If enable_ept, L0 doesn't care about page faults and we should
+	 * set all of these to L1's desires. However, if !enable_ept, L0 does
+	 * care about (at least some) page faults, and because it is not easy
+	 * (if at all possible?) to merge L0 and L1's desires, we simply ask
+	 * to exit on each and every L2 page fault. This is done by setting
+	 * MASK=MATCH=0 and (see below) EB.PF=1.
+	 * Note that below we don't need special code to set EB.PF beyond the
+	 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
+	 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
+	 * !enable_ept, EB.PF is 1, so the "or" will always be 1.
+	 */
+	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
+		enable_ept ? vmcs12->page_fault_error_code_mask : 0);
+	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
+		enable_ept ? vmcs12->page_fault_error_code_match : 0);
+
+	/* All VMFUNCs are currently emulated through L0 vmexits.  */
+	if (cpu_has_vmx_vmfunc())
+		vmcs_write64(VM_FUNCTION_CONTROL, 0);
+
+	if (cpu_has_vmx_apicv()) {
+		vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0);
+		vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1);
+		vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2);
+		vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3);
+	}
+
+	/*
+	 * Set host-state according to L0's settings (vmcs12 is irrelevant here)
+	 * Some constant fields are set here by vmx_set_constant_host_state().
+	 * Other fields are different per CPU, and will be set later when
+	 * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
+	 */
+	vmx_set_constant_host_state(vmx);
+
+	/*
+	 * Set the MSR load/store lists to match L0's settings.
+	 */
+	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
+	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
+	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
+	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
+	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
+
+	set_cr4_guest_host_mask(vmx);
+
+	if (vmx_mpx_supported())
+		vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
+
+	if (enable_vpid) {
+		if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
+			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
+		else
+			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
+	}
+
+	/*
+	 * L1 may access the L2's PDPTR, so save them to construct vmcs12
+	 */
+	if (enable_ept) {
+		vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
+		vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
+		vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
+		vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
+	}
+
+	if (cpu_has_vmx_msr_bitmap())
+		vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
+}
+
+/*
+ * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
+ * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
+ * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
+ * guest in a way that will both be appropriate to L1's requests, and our
+ * needs. In addition to modifying the active vmcs (which is vmcs02), this
+ * function also has additional necessary side-effects, like setting various
+ * vcpu->arch fields.
+ * Returns 0 on success, 1 on failure. Invalid state exit qualification code
+ * is assigned to entry_failure_code on failure.
+ */
+static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
+			  bool from_vmentry, u32 *entry_failure_code)
+{
+	struct vcpu_vmx *vmx = to_vmx(vcpu);
+	u32 exec_control, vmcs12_exec_ctrl;
+
+	if (vmx->nested.dirty_vmcs12) {
+		prepare_vmcs02_full(vcpu, vmcs12, from_vmentry);
+		vmx->nested.dirty_vmcs12 = false;
+	}
+
+	/*
+	 * First, the fields that are shadowed.  This must be kept in sync
+	 * with vmx_shadow_fields.h.
+	 */
+
+	vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
+	vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
+	vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
+	vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
+	vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
+
+	/*
+	 * Not in vmcs02: GUEST_PML_INDEX, HOST_FS_SELECTOR, HOST_GS_SELECTOR,
+	 * HOST_FS_BASE, HOST_GS_BASE.
+	 */
+
 	if (from_vmentry &&
 	    (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
 		kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
@@ -10482,16 +10754,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	} else {
 		vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
 	}
-	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
 	vmx_set_rflags(vcpu, vmcs12->guest_rflags);
-	vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
-		vmcs12->guest_pending_dbg_exceptions);
-	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
-	vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
-
-	if (nested_cpu_has_xsaves(vmcs12))
-		vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
-	vmcs_write64(VMCS_LINK_POINTER, -1ull);
 
 	exec_control = vmcs12->pin_based_vm_exec_control;
 
@@ -10505,7 +10768,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	if (nested_cpu_has_posted_intr(vmcs12)) {
 		vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
 		vmx->nested.pi_pending = false;
-		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
 	} else {
 		exec_control &= ~PIN_BASED_POSTED_INTR;
 	}
@@ -10516,25 +10778,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	if (nested_cpu_has_preemption_timer(vmcs12))
 		vmx_start_preemption_timer(vcpu);
 
-	/*
-	 * Whether page-faults are trapped is determined by a combination of
-	 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
-	 * If enable_ept, L0 doesn't care about page faults and we should
-	 * set all of these to L1's desires. However, if !enable_ept, L0 does
-	 * care about (at least some) page faults, and because it is not easy
-	 * (if at all possible?) to merge L0 and L1's desires, we simply ask
-	 * to exit on each and every L2 page fault. This is done by setting
-	 * MASK=MATCH=0 and (see below) EB.PF=1.
-	 * Note that below we don't need special code to set EB.PF beyond the
-	 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
-	 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
-	 * !enable_ept, EB.PF is 1, so the "or" will always be 1.
-	 */
-	vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
-		enable_ept ? vmcs12->page_fault_error_code_mask : 0);
-	vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
-		enable_ept ? vmcs12->page_fault_error_code_match : 0);
-
 	if (cpu_has_secondary_exec_ctrls()) {
 		exec_control = vmx->secondary_exec_control;
 
@@ -10553,22 +10796,9 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 			exec_control |= vmcs12_exec_ctrl;
 		}
 
-		/* All VMFUNCs are currently emulated through L0 vmexits.  */
-		if (exec_control & SECONDARY_EXEC_ENABLE_VMFUNC)
-			vmcs_write64(VM_FUNCTION_CONTROL, 0);
-
-		if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) {
-			vmcs_write64(EOI_EXIT_BITMAP0,
-				vmcs12->eoi_exit_bitmap0);
-			vmcs_write64(EOI_EXIT_BITMAP1,
-				vmcs12->eoi_exit_bitmap1);
-			vmcs_write64(EOI_EXIT_BITMAP2,
-				vmcs12->eoi_exit_bitmap2);
-			vmcs_write64(EOI_EXIT_BITMAP3,
-				vmcs12->eoi_exit_bitmap3);
+		if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
 			vmcs_write16(GUEST_INTR_STATUS,
 				vmcs12->guest_intr_status);
-		}
 
 		/*
 		 * Write an illegal value to APIC_ACCESS_ADDR. Later,
@@ -10581,24 +10811,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 		vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
 	}
 
-
-	/*
-	 * Set host-state according to L0's settings (vmcs12 is irrelevant here)
-	 * Some constant fields are set here by vmx_set_constant_host_state().
-	 * Other fields are different per CPU, and will be set later when
-	 * vmx_vcpu_load() is called, and when vmx_save_host_state() is called.
-	 */
-	vmx_set_constant_host_state(vmx);
-
-	/*
-	 * Set the MSR load/store lists to match L0's settings.
-	 */
-	vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
-	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
-	vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host));
-	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
-	vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest));
-
 	/*
 	 * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
 	 * entry, but only if the current (host) sp changed from the value
@@ -10630,8 +10842,8 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	}
 
 	/*
-	 * Merging of IO bitmap not currently supported.
-	 * Rather, exit every time.
+	 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed
+	 * for I/O port accesses.
 	 */
 	exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
 	exec_control |= CPU_BASED_UNCOND_IO_EXITING;
@@ -10668,12 +10880,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 		vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
 	}
 
-	set_cr4_guest_host_mask(vmx);
-
-	if (from_vmentry &&
-	    vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)
-		vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
-
 	if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
 		vmcs_write64(TSC_OFFSET,
 			vcpu->arch.tsc_offset + vmcs12->tsc_offset);
@@ -10692,16 +10898,13 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 		 * even if spawn a lot of nested vCPUs.
 		 */
 		if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
-			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
 			if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
 				vmx->nested.last_vpid = vmcs12->virtual_processor_id;
-				__vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02);
+				__vmx_flush_tlb(vcpu, to_vmx(vcpu)->nested.vpid02, true);
 			}
 		} else {
-			vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
-			vmx_flush_tlb(vcpu);
+			vmx_flush_tlb(vcpu, true);
 		}
-
 	}
 
 	if (enable_pml) {
@@ -10758,16 +10961,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	if (!enable_ept)
 		vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
 
-	/*
-	 * L1 may access the L2's PDPTR, so save them to construct vmcs12
-	 */
-	if (enable_ept) {
-		vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
-		vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
-		vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
-		vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
-	}
-
 	kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
 	kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
 	return 0;
@@ -10903,20 +11096,15 @@ static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
-	struct loaded_vmcs *vmcs02;
 	u32 msr_entry_idx;
 	u32 exit_qual;
 
-	vmcs02 = nested_get_current_vmcs02(vmx);
-	if (!vmcs02)
-		return -ENOMEM;
-
 	enter_guest_mode(vcpu);
 
 	if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
 		vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
 
-	vmx_switch_vmcs(vcpu, vmcs02);
+	vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
 	vmx_segment_cache_clear(vmx);
 
 	if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &exit_qual)) {
@@ -11026,7 +11214,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
 	if (ret)
 		return ret;
 
-	if (vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT)
+	/*
+	 * If we're entering a halted L2 vcpu and the L2 vcpu won't be woken
+	 * by event injection, halt vcpu.
+	 */
+	if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
+	    !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK))
 		return kvm_vcpu_halt(vcpu);
 
 	vmx->nested.nested_run_pending = 1;
@@ -11128,7 +11321,6 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu, bool external_intr)
 		if (block_nested_events)
 			return -EBUSY;
 		nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
-		vcpu->arch.exception.pending = false;
 		return 0;
 	}
 
@@ -11409,11 +11601,8 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
 		 * L1's vpid. TODO: move to a more elaborate solution, giving
 		 * each L2 its own vpid and exposing the vpid feature to L1.
 		 */
-		vmx_flush_tlb(vcpu);
+		vmx_flush_tlb(vcpu, true);
 	}
-	/* Restore posted intr vector. */
-	if (nested_cpu_has_posted_intr(vmcs12))
-		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
 
 	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
 	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
@@ -11485,7 +11674,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
 	vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
 
 	if (cpu_has_vmx_msr_bitmap())
-		vmx_set_msr_bitmap(vcpu);
+		vmx_update_msr_bitmap(vcpu);
 
 	if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
 				vmcs12->vm_exit_msr_load_count))
@@ -11534,10 +11723,6 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
 	vm_exit_controls_reset_shadow(vmx);
 	vmx_segment_cache_clear(vmx);
 
-	/* if no vmcs02 cache requested, remove the one we used */
-	if (VMCS02_POOL_SIZE == 0)
-		nested_free_vmcs02(vmx, vmx->nested.current_vmptr);
-
 	/* Update any VMCS fields that might have changed while L2 ran */
 	vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
 	vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.nr);
@@ -11678,6 +11863,21 @@ static int vmx_check_intercept(struct kvm_vcpu *vcpu,
 			       struct x86_instruction_info *info,
 			       enum x86_intercept_stage stage)
 {
+	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
+	struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
+
+	/*
+	 * RDPID causes #UD if disabled through secondary execution controls.
+	 * Because it is marked as EmulateOnUD, we need to intercept it here.
+	 */
+	if (info->intercept == x86_intercept_rdtscp &&
+	    !nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDTSCP)) {
+		ctxt->exception.vector = UD_VECTOR;
+		ctxt->exception.error_code_valid = false;
+		return X86EMUL_PROPAGATE_FAULT;
+	}
+
+	/* TODO: check more intercepts... */
 	return X86EMUL_CONTINUE;
 }
 
@@ -12110,6 +12310,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
 	.vcpu_put = vmx_vcpu_put,
 
 	.update_bp_intercept = update_exception_bitmap,
+	.get_msr_feature = vmx_get_msr_feature,
 	.get_msr = vmx_get_msr,
 	.set_msr = vmx_set_msr,
 	.get_segment_base = vmx_get_segment_base,
@@ -12191,6 +12392,7 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
 	.handle_external_intr = vmx_handle_external_intr,
 	.mpx_supported = vmx_mpx_supported,
 	.xsaves_supported = vmx_xsaves_supported,
+	.umip_emulated = vmx_umip_emulated,
 
 	.check_nested_events = vmx_check_nested_events,
 
diff --git a/arch/x86/kvm/vmx_shadow_fields.h b/arch/x86/kvm/vmx_shadow_fields.h
new file mode 100644
index 000000000000..cd0c75f6d037
--- /dev/null
+++ b/arch/x86/kvm/vmx_shadow_fields.h
@@ -0,0 +1,77 @@
+#ifndef SHADOW_FIELD_RO
+#define SHADOW_FIELD_RO(x)
+#endif
+#ifndef SHADOW_FIELD_RW
+#define SHADOW_FIELD_RW(x)
+#endif
+
+/*
+ * We do NOT shadow fields that are modified when L0
+ * traps and emulates any vmx instruction (e.g. VMPTRLD,
+ * VMXON...) executed by L1.
+ * For example, VM_INSTRUCTION_ERROR is read
+ * by L1 if a vmx instruction fails (part of the error path).
+ * Note the code assumes this logic. If for some reason
+ * we start shadowing these fields then we need to
+ * force a shadow sync when L0 emulates vmx instructions
+ * (e.g. force a sync if VM_INSTRUCTION_ERROR is modified
+ * by nested_vmx_failValid)
+ *
+ * When adding or removing fields here, note that shadowed
+ * fields must always be synced by prepare_vmcs02, not just
+ * prepare_vmcs02_full.
+ */
+
+/*
+ * Keeping the fields ordered by size is an attempt at improving
+ * branch prediction in vmcs_read_any and vmcs_write_any.
+ */
+
+/* 16-bits */
+SHADOW_FIELD_RW(GUEST_CS_SELECTOR)
+SHADOW_FIELD_RW(GUEST_INTR_STATUS)
+SHADOW_FIELD_RW(GUEST_PML_INDEX)
+SHADOW_FIELD_RW(HOST_FS_SELECTOR)
+SHADOW_FIELD_RW(HOST_GS_SELECTOR)
+
+/* 32-bits */
+SHADOW_FIELD_RO(VM_EXIT_REASON)
+SHADOW_FIELD_RO(VM_EXIT_INTR_INFO)
+SHADOW_FIELD_RO(VM_EXIT_INSTRUCTION_LEN)
+SHADOW_FIELD_RO(IDT_VECTORING_INFO_FIELD)
+SHADOW_FIELD_RO(IDT_VECTORING_ERROR_CODE)
+SHADOW_FIELD_RO(VM_EXIT_INTR_ERROR_CODE)
+SHADOW_FIELD_RW(CPU_BASED_VM_EXEC_CONTROL)
+SHADOW_FIELD_RW(EXCEPTION_BITMAP)
+SHADOW_FIELD_RW(VM_ENTRY_EXCEPTION_ERROR_CODE)
+SHADOW_FIELD_RW(VM_ENTRY_INTR_INFO_FIELD)
+SHADOW_FIELD_RW(VM_ENTRY_INSTRUCTION_LEN)
+SHADOW_FIELD_RW(TPR_THRESHOLD)
+SHADOW_FIELD_RW(GUEST_CS_LIMIT)
+SHADOW_FIELD_RW(GUEST_CS_AR_BYTES)
+SHADOW_FIELD_RW(GUEST_INTERRUPTIBILITY_INFO)
+SHADOW_FIELD_RW(VMX_PREEMPTION_TIMER_VALUE)
+
+/* Natural width */
+SHADOW_FIELD_RO(EXIT_QUALIFICATION)
+SHADOW_FIELD_RO(GUEST_LINEAR_ADDRESS)
+SHADOW_FIELD_RW(GUEST_RIP)
+SHADOW_FIELD_RW(GUEST_RSP)
+SHADOW_FIELD_RW(GUEST_CR0)
+SHADOW_FIELD_RW(GUEST_CR3)
+SHADOW_FIELD_RW(GUEST_CR4)
+SHADOW_FIELD_RW(GUEST_RFLAGS)
+SHADOW_FIELD_RW(GUEST_CS_BASE)
+SHADOW_FIELD_RW(GUEST_ES_BASE)
+SHADOW_FIELD_RW(CR0_GUEST_HOST_MASK)
+SHADOW_FIELD_RW(CR0_READ_SHADOW)
+SHADOW_FIELD_RW(CR4_READ_SHADOW)
+SHADOW_FIELD_RW(HOST_FS_BASE)
+SHADOW_FIELD_RW(HOST_GS_BASE)
+
+/* 64-bit */
+SHADOW_FIELD_RO(GUEST_PHYSICAL_ADDRESS)
+SHADOW_FIELD_RO(GUEST_PHYSICAL_ADDRESS_HIGH)
+
+#undef SHADOW_FIELD_RO
+#undef SHADOW_FIELD_RW
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c53298dfbf50..18b5ca7a3197 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -67,6 +67,8 @@
 #include <asm/pvclock.h>
 #include <asm/div64.h>
 #include <asm/irq_remapping.h>
+#include <asm/mshyperv.h>
+#include <asm/hypervisor.h>
 
 #define CREATE_TRACE_POINTS
 #include "trace.h"
@@ -177,7 +179,6 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
 	{ "request_irq", VCPU_STAT(request_irq_exits) },
 	{ "irq_exits", VCPU_STAT(irq_exits) },
 	{ "host_state_reload", VCPU_STAT(host_state_reload) },
-	{ "efer_reload", VCPU_STAT(efer_reload) },
 	{ "fpu_reload", VCPU_STAT(fpu_reload) },
 	{ "insn_emulation", VCPU_STAT(insn_emulation) },
 	{ "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
@@ -702,7 +703,8 @@ static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
 	if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
 			!vcpu->guest_xcr0_loaded) {
 		/* kvm_set_xcr() also depends on this */
-		xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
+		if (vcpu->arch.xcr0 != host_xcr0)
+			xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
 		vcpu->guest_xcr0_loaded = 1;
 	}
 }
@@ -794,6 +796,9 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 	if (!guest_cpuid_has(vcpu, X86_FEATURE_LA57) && (cr4 & X86_CR4_LA57))
 		return 1;
 
+	if (!guest_cpuid_has(vcpu, X86_FEATURE_UMIP) && (cr4 & X86_CR4_UMIP))
+		return 1;
+
 	if (is_long_mode(vcpu)) {
 		if (!(cr4 & X86_CR4_PAE))
 			return 1;
@@ -1009,6 +1014,7 @@ static u32 msrs_to_save[] = {
 #endif
 	MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
 	MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS, MSR_TSC_AUX,
+	MSR_IA32_SPEC_CTRL, MSR_IA32_ARCH_CAPABILITIES
 };
 
 static unsigned num_msrs_to_save;
@@ -1036,12 +1042,52 @@ static u32 emulated_msrs[] = {
 	MSR_IA32_MCG_CTL,
 	MSR_IA32_MCG_EXT_CTL,
 	MSR_IA32_SMBASE,
+	MSR_SMI_COUNT,
 	MSR_PLATFORM_INFO,
 	MSR_MISC_FEATURES_ENABLES,
 };
 
 static unsigned num_emulated_msrs;
 
+/*
+ * List of msr numbers which are used to expose MSR-based features that
+ * can be used by a hypervisor to validate requested CPU features.
+ */
+static u32 msr_based_features[] = {
+	MSR_F10H_DECFG,
+	MSR_IA32_UCODE_REV,
+};
+
+static unsigned int num_msr_based_features;
+
+static int kvm_get_msr_feature(struct kvm_msr_entry *msr)
+{
+	switch (msr->index) {
+	case MSR_IA32_UCODE_REV:
+		rdmsrl(msr->index, msr->data);
+		break;
+	default:
+		if (kvm_x86_ops->get_msr_feature(msr))
+			return 1;
+	}
+	return 0;
+}
+
+static int do_get_msr_feature(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
+{
+	struct kvm_msr_entry msr;
+	int r;
+
+	msr.index = index;
+	r = kvm_get_msr_feature(&msr);
+	if (r)
+		return r;
+
+	*data = msr.data;
+
+	return 0;
+}
+
 bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
 {
 	if (efer & efer_reserved_bits)
@@ -1377,6 +1423,11 @@ static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
 	return tsc;
 }
 
+static inline int gtod_is_based_on_tsc(int mode)
+{
+	return mode == VCLOCK_TSC || mode == VCLOCK_HVCLOCK;
+}
+
 static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
 {
 #ifdef CONFIG_X86_64
@@ -1396,7 +1447,7 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
 	 * perform request to enable masterclock.
 	 */
 	if (ka->use_master_clock ||
-	    (gtod->clock.vclock_mode == VCLOCK_TSC && vcpus_matched))
+	    (gtod_is_based_on_tsc(gtod->clock.vclock_mode) && vcpus_matched))
 		kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);
 
 	trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
@@ -1459,6 +1510,19 @@ static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
 	vcpu->arch.tsc_offset = offset;
 }
 
+static inline bool kvm_check_tsc_unstable(void)
+{
+#ifdef CONFIG_X86_64
+	/*
+	 * TSC is marked unstable when we're running on Hyper-V,
+	 * 'TSC page' clocksource is good.
+	 */
+	if (pvclock_gtod_data.clock.vclock_mode == VCLOCK_HVCLOCK)
+		return false;
+#endif
+	return check_tsc_unstable();
+}
+
 void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
 {
 	struct kvm *kvm = vcpu->kvm;
@@ -1504,7 +1568,7 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
          */
 	if (synchronizing &&
 	    vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
-		if (!check_tsc_unstable()) {
+		if (!kvm_check_tsc_unstable()) {
 			offset = kvm->arch.cur_tsc_offset;
 			pr_debug("kvm: matched tsc offset for %llu\n", data);
 		} else {
@@ -1604,18 +1668,43 @@ static u64 read_tsc(void)
 	return last;
 }
 
-static inline u64 vgettsc(u64 *cycle_now)
+static inline u64 vgettsc(u64 *tsc_timestamp, int *mode)
 {
 	long v;
 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	u64 tsc_pg_val;
+
+	switch (gtod->clock.vclock_mode) {
+	case VCLOCK_HVCLOCK:
+		tsc_pg_val = hv_read_tsc_page_tsc(hv_get_tsc_page(),
+						  tsc_timestamp);
+		if (tsc_pg_val != U64_MAX) {
+			/* TSC page valid */
+			*mode = VCLOCK_HVCLOCK;
+			v = (tsc_pg_val - gtod->clock.cycle_last) &
+				gtod->clock.mask;
+		} else {
+			/* TSC page invalid */
+			*mode = VCLOCK_NONE;
+		}
+		break;
+	case VCLOCK_TSC:
+		*mode = VCLOCK_TSC;
+		*tsc_timestamp = read_tsc();
+		v = (*tsc_timestamp - gtod->clock.cycle_last) &
+			gtod->clock.mask;
+		break;
+	default:
+		*mode = VCLOCK_NONE;
+	}
 
-	*cycle_now = read_tsc();
+	if (*mode == VCLOCK_NONE)
+		*tsc_timestamp = v = 0;
 
-	v = (*cycle_now - gtod->clock.cycle_last) & gtod->clock.mask;
 	return v * gtod->clock.mult;
 }
 
-static int do_monotonic_boot(s64 *t, u64 *cycle_now)
+static int do_monotonic_boot(s64 *t, u64 *tsc_timestamp)
 {
 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
 	unsigned long seq;
@@ -1624,9 +1713,8 @@ static int do_monotonic_boot(s64 *t, u64 *cycle_now)
 
 	do {
 		seq = read_seqcount_begin(&gtod->seq);
-		mode = gtod->clock.vclock_mode;
 		ns = gtod->nsec_base;
-		ns += vgettsc(cycle_now);
+		ns += vgettsc(tsc_timestamp, &mode);
 		ns >>= gtod->clock.shift;
 		ns += gtod->boot_ns;
 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
@@ -1635,7 +1723,7 @@ static int do_monotonic_boot(s64 *t, u64 *cycle_now)
 	return mode;
 }
 
-static int do_realtime(struct timespec *ts, u64 *cycle_now)
+static int do_realtime(struct timespec *ts, u64 *tsc_timestamp)
 {
 	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
 	unsigned long seq;
@@ -1644,10 +1732,9 @@ static int do_realtime(struct timespec *ts, u64 *cycle_now)
 
 	do {
 		seq = read_seqcount_begin(&gtod->seq);
-		mode = gtod->clock.vclock_mode;
 		ts->tv_sec = gtod->wall_time_sec;
 		ns = gtod->nsec_base;
-		ns += vgettsc(cycle_now);
+		ns += vgettsc(tsc_timestamp, &mode);
 		ns >>= gtod->clock.shift;
 	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
 
@@ -1657,25 +1744,26 @@ static int do_realtime(struct timespec *ts, u64 *cycle_now)
 	return mode;
 }
 
-/* returns true if host is using tsc clocksource */
-static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *cycle_now)
+/* returns true if host is using TSC based clocksource */
+static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp)
 {
 	/* checked again under seqlock below */
-	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
 		return false;
 
-	return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC;
+	return gtod_is_based_on_tsc(do_monotonic_boot(kernel_ns,
+						      tsc_timestamp));
 }
 
-/* returns true if host is using tsc clocksource */
+/* returns true if host is using TSC based clocksource */
 static bool kvm_get_walltime_and_clockread(struct timespec *ts,
-					   u64 *cycle_now)
+					   u64 *tsc_timestamp)
 {
 	/* checked again under seqlock below */
-	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+	if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode))
 		return false;
 
-	return do_realtime(ts, cycle_now) == VCLOCK_TSC;
+	return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp));
 }
 #endif
 
@@ -2118,6 +2206,12 @@ static void kvmclock_reset(struct kvm_vcpu *vcpu)
 	vcpu->arch.pv_time_enabled = false;
 }
 
+static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
+{
+	++vcpu->stat.tlb_flush;
+	kvm_x86_ops->tlb_flush(vcpu, invalidate_gpa);
+}
+
 static void record_steal_time(struct kvm_vcpu *vcpu)
 {
 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
@@ -2127,7 +2221,12 @@ static void record_steal_time(struct kvm_vcpu *vcpu)
 		&vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
 		return;
 
-	vcpu->arch.st.steal.preempted = 0;
+	/*
+	 * Doing a TLB flush here, on the guest's behalf, can avoid
+	 * expensive IPIs.
+	 */
+	if (xchg(&vcpu->arch.st.steal.preempted, 0) & KVM_VCPU_FLUSH_TLB)
+		kvm_vcpu_flush_tlb(vcpu, false);
 
 	if (vcpu->arch.st.steal.version & 1)
 		vcpu->arch.st.steal.version += 1;  /* first time write, random junk */
@@ -2162,7 +2261,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 
 	switch (msr) {
 	case MSR_AMD64_NB_CFG:
-	case MSR_IA32_UCODE_REV:
 	case MSR_IA32_UCODE_WRITE:
 	case MSR_VM_HSAVE_PA:
 	case MSR_AMD64_PATCH_LOADER:
@@ -2170,6 +2268,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_AMD64_DC_CFG:
 		break;
 
+	case MSR_IA32_UCODE_REV:
+		if (msr_info->host_initiated)
+			vcpu->arch.microcode_version = data;
+		break;
 	case MSR_EFER:
 		return set_efer(vcpu, data);
 	case MSR_K7_HWCR:
@@ -2228,6 +2330,11 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			return 1;
 		vcpu->arch.smbase = data;
 		break;
+	case MSR_SMI_COUNT:
+		if (!msr_info->host_initiated)
+			return 1;
+		vcpu->arch.smi_count = data;
+		break;
 	case MSR_KVM_WALL_CLOCK_NEW:
 	case MSR_KVM_WALL_CLOCK:
 		vcpu->kvm->arch.wall_clock = data;
@@ -2460,7 +2567,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 		msr_info->data = 0;
 		break;
 	case MSR_IA32_UCODE_REV:
-		msr_info->data = 0x100000000ULL;
+		msr_info->data = vcpu->arch.microcode_version;
 		break;
 	case MSR_MTRRcap:
 	case 0x200 ... 0x2ff:
@@ -2502,6 +2609,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 			return 1;
 		msr_info->data = vcpu->arch.smbase;
 		break;
+	case MSR_SMI_COUNT:
+		msr_info->data = vcpu->arch.smi_count;
+		break;
 	case MSR_IA32_PERF_STATUS:
 		/* TSC increment by tick */
 		msr_info->data = 1000ULL;
@@ -2612,13 +2722,11 @@ static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
 		    int (*do_msr)(struct kvm_vcpu *vcpu,
 				  unsigned index, u64 *data))
 {
-	int i, idx;
+	int i;
 
-	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	for (i = 0; i < msrs->nmsrs; ++i)
 		if (do_msr(vcpu, entries[i].index, &entries[i].data))
 			break;
-	srcu_read_unlock(&vcpu->kvm->srcu, idx);
 
 	return i;
 }
@@ -2717,6 +2825,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_SET_BOOT_CPU_ID:
  	case KVM_CAP_SPLIT_IRQCHIP:
 	case KVM_CAP_IMMEDIATE_EXIT:
+	case KVM_CAP_GET_MSR_FEATURES:
 		r = 1;
 		break;
 	case KVM_CAP_ADJUST_CLOCK:
@@ -2831,6 +2940,31 @@ long kvm_arch_dev_ioctl(struct file *filp,
 			goto out;
 		r = 0;
 		break;
+	case KVM_GET_MSR_FEATURE_INDEX_LIST: {
+		struct kvm_msr_list __user *user_msr_list = argp;
+		struct kvm_msr_list msr_list;
+		unsigned int n;
+
+		r = -EFAULT;
+		if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list)))
+			goto out;
+		n = msr_list.nmsrs;
+		msr_list.nmsrs = num_msr_based_features;
+		if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list)))
+			goto out;
+		r = -E2BIG;
+		if (n < msr_list.nmsrs)
+			goto out;
+		r = -EFAULT;
+		if (copy_to_user(user_msr_list->indices, &msr_based_features,
+				 num_msr_based_features * sizeof(u32)))
+			goto out;
+		r = 0;
+		break;
+	}
+	case KVM_GET_MSRS:
+		r = msr_io(NULL, argp, do_get_msr_feature, 1);
+		break;
 	}
 	default:
 		r = -EINVAL;
@@ -2869,13 +3003,13 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
 	}
 
-	if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
+	if (unlikely(vcpu->cpu != cpu) || kvm_check_tsc_unstable()) {
 		s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
 				rdtsc() - vcpu->arch.last_host_tsc;
 		if (tsc_delta < 0)
 			mark_tsc_unstable("KVM discovered backwards TSC");
 
-		if (check_tsc_unstable()) {
+		if (kvm_check_tsc_unstable()) {
 			u64 offset = kvm_compute_tsc_offset(vcpu,
 						vcpu->arch.last_guest_tsc);
 			kvm_vcpu_write_tsc_offset(vcpu, offset);
@@ -2904,7 +3038,7 @@ static void kvm_steal_time_set_preempted(struct kvm_vcpu *vcpu)
 	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
 		return;
 
-	vcpu->arch.st.steal.preempted = 1;
+	vcpu->arch.st.steal.preempted = KVM_VCPU_PREEMPTED;
 
 	kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.st.stime,
 			&vcpu->arch.st.steal.preempted,
@@ -2938,12 +3072,18 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	pagefault_enable();
 	kvm_x86_ops->vcpu_put(vcpu);
 	vcpu->arch.last_host_tsc = rdtsc();
+	/*
+	 * If userspace has set any breakpoints or watchpoints, dr6 is restored
+	 * on every vmexit, but if not, we might have a stale dr6 from the
+	 * guest. do_debug expects dr6 to be cleared after it runs, do the same.
+	 */
+	set_debugreg(0, 6);
 }
 
 static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
 				    struct kvm_lapic_state *s)
 {
-	if (kvm_x86_ops->sync_pir_to_irr && vcpu->arch.apicv_active)
+	if (vcpu->arch.apicv_active)
 		kvm_x86_ops->sync_pir_to_irr(vcpu);
 
 	return kvm_apic_get_state(vcpu, s);
@@ -3472,6 +3612,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		void *buffer;
 	} u;
 
+	vcpu_load(vcpu);
+
 	u.buffer = NULL;
 	switch (ioctl) {
 	case KVM_GET_LAPIC: {
@@ -3497,8 +3639,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		if (!lapic_in_kernel(vcpu))
 			goto out;
 		u.lapic = memdup_user(argp, sizeof(*u.lapic));
-		if (IS_ERR(u.lapic))
-			return PTR_ERR(u.lapic);
+		if (IS_ERR(u.lapic)) {
+			r = PTR_ERR(u.lapic);
+			goto out_nofree;
+		}
 
 		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
 		break;
@@ -3558,12 +3702,18 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 		r = 0;
 		break;
 	}
-	case KVM_GET_MSRS:
+	case KVM_GET_MSRS: {
+		int idx = srcu_read_lock(&vcpu->kvm->srcu);
 		r = msr_io(vcpu, argp, do_get_msr, 1);
+		srcu_read_unlock(&vcpu->kvm->srcu, idx);
 		break;
-	case KVM_SET_MSRS:
+	}
+	case KVM_SET_MSRS: {
+		int idx = srcu_read_lock(&vcpu->kvm->srcu);
 		r = msr_io(vcpu, argp, do_set_msr, 0);
+		srcu_read_unlock(&vcpu->kvm->srcu, idx);
 		break;
+	}
 	case KVM_TPR_ACCESS_REPORTING: {
 		struct kvm_tpr_access_ctl tac;
 
@@ -3672,8 +3822,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 	}
 	case KVM_SET_XSAVE: {
 		u.xsave = memdup_user(argp, sizeof(*u.xsave));
-		if (IS_ERR(u.xsave))
-			return PTR_ERR(u.xsave);
+		if (IS_ERR(u.xsave)) {
+			r = PTR_ERR(u.xsave);
+			goto out_nofree;
+		}
 
 		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
 		break;
@@ -3695,8 +3847,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 	}
 	case KVM_SET_XCRS: {
 		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
-		if (IS_ERR(u.xcrs))
-			return PTR_ERR(u.xcrs);
+		if (IS_ERR(u.xcrs)) {
+			r = PTR_ERR(u.xcrs);
+			goto out_nofree;
+		}
 
 		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
 		break;
@@ -3740,6 +3894,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
 	}
 out:
 	kfree(u.buffer);
+out_nofree:
+	vcpu_put(vcpu);
 	return r;
 }
 
@@ -4237,13 +4393,14 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		mutex_unlock(&kvm->lock);
 		break;
 	case KVM_XEN_HVM_CONFIG: {
+		struct kvm_xen_hvm_config xhc;
 		r = -EFAULT;
-		if (copy_from_user(&kvm->arch.xen_hvm_config, argp,
-				   sizeof(struct kvm_xen_hvm_config)))
+		if (copy_from_user(&xhc, argp, sizeof(xhc)))
 			goto out;
 		r = -EINVAL;
-		if (kvm->arch.xen_hvm_config.flags)
+		if (xhc.flags)
 			goto out;
+		memcpy(&kvm->arch.xen_hvm_config, &xhc, sizeof(xhc));
 		r = 0;
 		break;
 	}
@@ -4295,6 +4452,36 @@ long kvm_arch_vm_ioctl(struct file *filp,
 		r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 		break;
 	}
+	case KVM_MEMORY_ENCRYPT_OP: {
+		r = -ENOTTY;
+		if (kvm_x86_ops->mem_enc_op)
+			r = kvm_x86_ops->mem_enc_op(kvm, argp);
+		break;
+	}
+	case KVM_MEMORY_ENCRYPT_REG_REGION: {
+		struct kvm_enc_region region;
+
+		r = -EFAULT;
+		if (copy_from_user(&region, argp, sizeof(region)))
+			goto out;
+
+		r = -ENOTTY;
+		if (kvm_x86_ops->mem_enc_reg_region)
+			r = kvm_x86_ops->mem_enc_reg_region(kvm, &region);
+		break;
+	}
+	case KVM_MEMORY_ENCRYPT_UNREG_REGION: {
+		struct kvm_enc_region region;
+
+		r = -EFAULT;
+		if (copy_from_user(&region, argp, sizeof(region)))
+			goto out;
+
+		r = -ENOTTY;
+		if (kvm_x86_ops->mem_enc_unreg_region)
+			r = kvm_x86_ops->mem_enc_unreg_region(kvm, &region);
+		break;
+	}
 	default:
 		r = -ENOTTY;
 	}
@@ -4349,6 +4536,19 @@ static void kvm_init_msr_list(void)
 		j++;
 	}
 	num_emulated_msrs = j;
+
+	for (i = j = 0; i < ARRAY_SIZE(msr_based_features); i++) {
+		struct kvm_msr_entry msr;
+
+		msr.index = msr_based_features[i];
+		if (kvm_get_msr_feature(&msr))
+			continue;
+
+		if (j < i)
+			msr_based_features[j] = msr_based_features[i];
+		j++;
+	}
+	num_msr_based_features = j;
 }
 
 static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len,
@@ -5703,7 +5903,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
 		 * handle watchpoints yet, those would be handled in
 		 * the emulate_ops.
 		 */
-		if (kvm_vcpu_check_breakpoint(vcpu, &r))
+		if (!(emulation_type & EMULTYPE_SKIP) &&
+		    kvm_vcpu_check_breakpoint(vcpu, &r))
 			return r;
 
 		ctxt->interruptibility = 0;
@@ -5889,6 +6090,43 @@ static void tsc_khz_changed(void *data)
 	__this_cpu_write(cpu_tsc_khz, khz);
 }
 
+#ifdef CONFIG_X86_64
+static void kvm_hyperv_tsc_notifier(void)
+{
+	struct kvm *kvm;
+	struct kvm_vcpu *vcpu;
+	int cpu;
+
+	spin_lock(&kvm_lock);
+	list_for_each_entry(kvm, &vm_list, vm_list)
+		kvm_make_mclock_inprogress_request(kvm);
+
+	hyperv_stop_tsc_emulation();
+
+	/* TSC frequency always matches when on Hyper-V */
+	for_each_present_cpu(cpu)
+		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
+	kvm_max_guest_tsc_khz = tsc_khz;
+
+	list_for_each_entry(kvm, &vm_list, vm_list) {
+		struct kvm_arch *ka = &kvm->arch;
+
+		spin_lock(&ka->pvclock_gtod_sync_lock);
+
+		pvclock_update_vm_gtod_copy(kvm);
+
+		kvm_for_each_vcpu(cpu, vcpu, kvm)
+			kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
+
+		kvm_for_each_vcpu(cpu, vcpu, kvm)
+			kvm_clear_request(KVM_REQ_MCLOCK_INPROGRESS, vcpu);
+
+		spin_unlock(&ka->pvclock_gtod_sync_lock);
+	}
+	spin_unlock(&kvm_lock);
+}
+#endif
+
 static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
 				     void *data)
 {
@@ -6110,9 +6348,9 @@ static int pvclock_gtod_notify(struct notifier_block *nb, unsigned long unused,
 	update_pvclock_gtod(tk);
 
 	/* disable master clock if host does not trust, or does not
-	 * use, TSC clocksource
+	 * use, TSC based clocksource.
 	 */
-	if (gtod->clock.vclock_mode != VCLOCK_TSC &&
+	if (!gtod_is_based_on_tsc(gtod->clock.vclock_mode) &&
 	    atomic_read(&kvm_guest_has_master_clock) != 0)
 		queue_work(system_long_wq, &pvclock_gtod_work);
 
@@ -6174,6 +6412,9 @@ int kvm_arch_init(void *opaque)
 	kvm_lapic_init();
 #ifdef CONFIG_X86_64
 	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);
+
+	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
+		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
 #endif
 
 	return 0;
@@ -6186,6 +6427,10 @@ int kvm_arch_init(void *opaque)
 
 void kvm_arch_exit(void)
 {
+#ifdef CONFIG_X86_64
+	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
+		clear_hv_tscchange_cb();
+#endif
 	kvm_lapic_exit();
 	perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
 
@@ -6448,6 +6693,7 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool req_int_win)
 		kvm_x86_ops->queue_exception(vcpu);
 	} else if (vcpu->arch.smi_pending && !is_smm(vcpu) && kvm_x86_ops->smi_allowed(vcpu)) {
 		vcpu->arch.smi_pending = false;
+		++vcpu->arch.smi_count;
 		enter_smm(vcpu);
 	} else if (vcpu->arch.nmi_pending && kvm_x86_ops->nmi_allowed(vcpu)) {
 		--vcpu->arch.nmi_pending;
@@ -6749,7 +6995,7 @@ static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
 	if (irqchip_split(vcpu->kvm))
 		kvm_scan_ioapic_routes(vcpu, vcpu->arch.ioapic_handled_vectors);
 	else {
-		if (kvm_x86_ops->sync_pir_to_irr && vcpu->arch.apicv_active)
+		if (vcpu->arch.apicv_active)
 			kvm_x86_ops->sync_pir_to_irr(vcpu);
 		kvm_ioapic_scan_entry(vcpu, vcpu->arch.ioapic_handled_vectors);
 	}
@@ -6758,12 +7004,6 @@ static void vcpu_scan_ioapic(struct kvm_vcpu *vcpu)
 	kvm_x86_ops->load_eoi_exitmap(vcpu, eoi_exit_bitmap);
 }
 
-static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu)
-{
-	++vcpu->stat.tlb_flush;
-	kvm_x86_ops->tlb_flush(vcpu);
-}
-
 void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm,
 		unsigned long start, unsigned long end)
 {
@@ -6832,7 +7072,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		if (kvm_check_request(KVM_REQ_MMU_SYNC, vcpu))
 			kvm_mmu_sync_roots(vcpu);
 		if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
-			kvm_vcpu_flush_tlb(vcpu);
+			kvm_vcpu_flush_tlb(vcpu, true);
 		if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) {
 			vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS;
 			r = 0;
@@ -6981,10 +7221,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	 * This handles the case where a posted interrupt was
 	 * notified with kvm_vcpu_kick.
 	 */
-	if (kvm_lapic_enabled(vcpu)) {
-		if (kvm_x86_ops->sync_pir_to_irr && vcpu->arch.apicv_active)
-			kvm_x86_ops->sync_pir_to_irr(vcpu);
-	}
+	if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active)
+		kvm_x86_ops->sync_pir_to_irr(vcpu);
 
 	if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu)
 	    || need_resched() || signal_pending(current)) {
@@ -7005,7 +7243,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	}
 
 	trace_kvm_entry(vcpu->vcpu_id);
-	wait_lapic_expire(vcpu);
+	if (lapic_timer_advance_ns)
+		wait_lapic_expire(vcpu);
 	guest_enter_irqoff();
 
 	if (unlikely(vcpu->arch.switch_db_regs)) {
@@ -7266,8 +7505,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int r;
 
+	vcpu_load(vcpu);
 	kvm_sigset_activate(vcpu);
-
 	kvm_load_guest_fpu(vcpu);
 
 	if (unlikely(vcpu->arch.mp_state == KVM_MP_STATE_UNINITIALIZED)) {
@@ -7314,11 +7553,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 	post_kvm_run_save(vcpu);
 	kvm_sigset_deactivate(vcpu);
 
+	vcpu_put(vcpu);
 	return r;
 }
 
 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
+	vcpu_load(vcpu);
+
 	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
 		/*
 		 * We are here if userspace calls get_regs() in the middle of
@@ -7352,11 +7594,14 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 	regs->rip = kvm_rip_read(vcpu);
 	regs->rflags = kvm_get_rflags(vcpu);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 {
+	vcpu_load(vcpu);
+
 	vcpu->arch.emulate_regs_need_sync_from_vcpu = true;
 	vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
 
@@ -7386,6 +7631,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
 
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -7404,6 +7650,8 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 {
 	struct desc_ptr dt;
 
+	vcpu_load(vcpu);
+
 	kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
 	kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
 	kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
@@ -7435,12 +7683,15 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
 		set_bit(vcpu->arch.interrupt.nr,
 			(unsigned long *)sregs->interrupt_bitmap);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
+	vcpu_load(vcpu);
+
 	kvm_apic_accept_events(vcpu);
 	if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED &&
 					vcpu->arch.pv.pv_unhalted)
@@ -7448,21 +7699,26 @@ int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 	else
 		mp_state->mp_state = vcpu->arch.mp_state;
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 				    struct kvm_mp_state *mp_state)
 {
+	int ret = -EINVAL;
+
+	vcpu_load(vcpu);
+
 	if (!lapic_in_kernel(vcpu) &&
 	    mp_state->mp_state != KVM_MP_STATE_RUNNABLE)
-		return -EINVAL;
+		goto out;
 
 	/* INITs are latched while in SMM */
 	if ((is_smm(vcpu) || vcpu->arch.smi_pending) &&
 	    (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED ||
 	     mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED))
-		return -EINVAL;
+		goto out;
 
 	if (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
 		vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
@@ -7470,7 +7726,11 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 	} else
 		vcpu->arch.mp_state = mp_state->mp_state;
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
-	return 0;
+
+	ret = 0;
+out:
+	vcpu_put(vcpu);
+	return ret;
 }
 
 int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
@@ -7524,18 +7784,21 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 	int mmu_reset_needed = 0;
 	int pending_vec, max_bits, idx;
 	struct desc_ptr dt;
+	int ret = -EINVAL;
+
+	vcpu_load(vcpu);
 
 	if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
 			(sregs->cr4 & X86_CR4_OSXSAVE))
-		return -EINVAL;
+		goto out;
 
 	if (kvm_valid_sregs(vcpu, sregs))
-		return -EINVAL;
+		goto out;
 
 	apic_base_msr.data = sregs->apic_base;
 	apic_base_msr.host_initiated = true;
 	if (kvm_set_apic_base(vcpu, &apic_base_msr))
-		return -EINVAL;
+		goto out;
 
 	dt.size = sregs->idt.limit;
 	dt.address = sregs->idt.base;
@@ -7601,7 +7864,10 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
 
 	kvm_make_request(KVM_REQ_EVENT, vcpu);
 
-	return 0;
+	ret = 0;
+out:
+	vcpu_put(vcpu);
+	return ret;
 }
 
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
@@ -7610,6 +7876,8 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 	unsigned long rflags;
 	int i, r;
 
+	vcpu_load(vcpu);
+
 	if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) {
 		r = -EBUSY;
 		if (vcpu->arch.exception.pending)
@@ -7655,7 +7923,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 	r = 0;
 
 out:
-
+	vcpu_put(vcpu);
 	return r;
 }
 
@@ -7669,6 +7937,8 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 	gpa_t gpa;
 	int idx;
 
+	vcpu_load(vcpu);
+
 	idx = srcu_read_lock(&vcpu->kvm->srcu);
 	gpa = kvm_mmu_gva_to_gpa_system(vcpu, vaddr, NULL);
 	srcu_read_unlock(&vcpu->kvm->srcu, idx);
@@ -7677,14 +7947,17 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
 	tr->writeable = 1;
 	tr->usermode = 0;
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
-	struct fxregs_state *fxsave =
-			&vcpu->arch.guest_fpu.state.fxsave;
+	struct fxregs_state *fxsave;
 
+	vcpu_load(vcpu);
+
+	fxsave = &vcpu->arch.guest_fpu.state.fxsave;
 	memcpy(fpu->fpr, fxsave->st_space, 128);
 	fpu->fcw = fxsave->cwd;
 	fpu->fsw = fxsave->swd;
@@ -7694,13 +7967,17 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 	fpu->last_dp = fxsave->rdp;
 	memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 {
-	struct fxregs_state *fxsave =
-			&vcpu->arch.guest_fpu.state.fxsave;
+	struct fxregs_state *fxsave;
+
+	vcpu_load(vcpu);
+
+	fxsave = &vcpu->arch.guest_fpu.state.fxsave;
 
 	memcpy(fxsave->st_space, fpu->fpr, 128);
 	fxsave->cwd = fpu->fcw;
@@ -7711,6 +7988,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
 	fxsave->rdp = fpu->last_dp;
 	memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space);
 
+	vcpu_put(vcpu);
 	return 0;
 }
 
@@ -7767,7 +8045,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 {
 	struct kvm_vcpu *vcpu;
 
-	if (check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
+	if (kvm_check_tsc_unstable() && atomic_read(&kvm->online_vcpus) != 0)
 		printk_once(KERN_WARNING
 		"kvm: SMP vm created on host with unstable TSC; "
 		"guest TSC will not be reliable\n");
@@ -7779,16 +8057,12 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
 
 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
 {
-	int r;
-
 	kvm_vcpu_mtrr_init(vcpu);
-	r = vcpu_load(vcpu);
-	if (r)
-		return r;
+	vcpu_load(vcpu);
 	kvm_vcpu_reset(vcpu, false);
 	kvm_mmu_setup(vcpu);
 	vcpu_put(vcpu);
-	return r;
+	return 0;
 }
 
 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
@@ -7798,13 +8072,15 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 
 	kvm_hv_vcpu_postcreate(vcpu);
 
-	if (vcpu_load(vcpu))
+	if (mutex_lock_killable(&vcpu->mutex))
 		return;
+	vcpu_load(vcpu);
 	msr.data = 0x0;
 	msr.index = MSR_IA32_TSC;
 	msr.host_initiated = true;
 	kvm_write_tsc(vcpu, &msr);
 	vcpu_put(vcpu);
+	mutex_unlock(&vcpu->mutex);
 
 	if (!kvmclock_periodic_sync)
 		return;
@@ -7815,11 +8091,9 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
-	int r;
 	vcpu->arch.apf.msr_val = 0;
 
-	r = vcpu_load(vcpu);
-	BUG_ON(r);
+	vcpu_load(vcpu);
 	kvm_mmu_unload(vcpu);
 	vcpu_put(vcpu);
 
@@ -7828,9 +8102,12 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 
 void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 {
+	kvm_lapic_reset(vcpu, init_event);
+
 	vcpu->arch.hflags = 0;
 
 	vcpu->arch.smi_pending = 0;
+	vcpu->arch.smi_count = 0;
 	atomic_set(&vcpu->arch.nmi_queued, 0);
 	vcpu->arch.nmi_pending = 0;
 	vcpu->arch.nmi_injected = false;
@@ -7924,7 +8201,7 @@ int kvm_arch_hardware_enable(void)
 		return ret;
 
 	local_tsc = rdtsc();
-	stable = !check_tsc_unstable();
+	stable = !kvm_check_tsc_unstable();
 	list_for_each_entry(kvm, &vm_list, vm_list) {
 		kvm_for_each_vcpu(i, vcpu, kvm) {
 			if (!stable && vcpu->cpu == smp_processor_id())
@@ -8190,9 +8467,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 
 static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
 {
-	int r;
-	r = vcpu_load(vcpu);
-	BUG_ON(r);
+	vcpu_load(vcpu);
 	kvm_mmu_unload(vcpu);
 	vcpu_put(vcpu);
 }
@@ -8272,10 +8547,8 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size)
 			return r;
 	}
 
-	if (!size) {
-		r = vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
-		WARN_ON(r < 0);
-	}
+	if (!size)
+		vm_munmap(old.userspace_addr, old.npages * PAGE_SIZE);
 
 	return 0;
 }
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index d0b95b7a90b4..b91215d1fd80 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -12,6 +12,7 @@
 
 static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu)
 {
+	vcpu->arch.exception.pending = false;
 	vcpu->arch.exception.injected = false;
 }
 
@@ -265,36 +266,8 @@ static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
 
 static inline bool kvm_mwait_in_guest(void)
 {
-	unsigned int eax, ebx, ecx, edx;
-
-	if (!cpu_has(&boot_cpu_data, X86_FEATURE_MWAIT))
-		return false;
-
-	switch (boot_cpu_data.x86_vendor) {
-	case X86_VENDOR_AMD:
-		/* All AMD CPUs have a working MWAIT implementation */
-		return true;
-	case X86_VENDOR_INTEL:
-		/* Handle Intel below */
-		break;
-	default:
-		return false;
-	}
-
-	/*
-	 * Intel CPUs without CPUID5_ECX_INTERRUPT_BREAK are problematic as
-	 * they would allow guest to stop the CPU completely by disabling
-	 * interrupts then invoking MWAIT.
-	 */
-	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
-		return false;
-
-	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
-
-	if (!(ecx & CPUID5_ECX_INTERRUPT_BREAK))
-		return false;
-
-	return true;
+	return boot_cpu_has(X86_FEATURE_MWAIT) &&
+		!boot_cpu_has_bug(X86_BUG_MONITOR);
 }
 
 #endif
diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile
index f23934bbaf4e..25a972c61b0a 100644
--- a/arch/x86/lib/Makefile
+++ b/arch/x86/lib/Makefile
@@ -26,6 +26,7 @@ lib-y += memcpy_$(BITS).o
 lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
 lib-$(CONFIG_INSTRUCTION_DECODER) += insn.o inat.o insn-eval.o
 lib-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
+lib-$(CONFIG_FUNCTION_ERROR_INJECTION)	+= error-inject.o
 lib-$(CONFIG_RETPOLINE) += retpoline.o
 
 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o
diff --git a/arch/x86/lib/cpu.c b/arch/x86/lib/cpu.c
index d6f848d1211d..2dd1fe13a37b 100644
--- a/arch/x86/lib/cpu.c
+++ b/arch/x86/lib/cpu.c
@@ -18,7 +18,7 @@ unsigned int x86_model(unsigned int sig)
 {
 	unsigned int fam, model;
 
-	 fam = x86_family(sig);
+	fam = x86_family(sig);
 
 	model = (sig >> 4) & 0xf;
 
diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c
index 4846eff7e4c8..f5b7f1b3b6d7 100644
--- a/arch/x86/lib/delay.c
+++ b/arch/x86/lib/delay.c
@@ -162,7 +162,7 @@ void __delay(unsigned long loops)
 }
 EXPORT_SYMBOL(__delay);
 
-inline void __const_udelay(unsigned long xloops)
+void __const_udelay(unsigned long xloops)
 {
 	unsigned long lpj = this_cpu_read(cpu_info.loops_per_jiffy) ? : loops_per_jiffy;
 	int d0;
diff --git a/arch/x86/lib/error-inject.c b/arch/x86/lib/error-inject.c
new file mode 100644
index 000000000000..3cdf06128d13
--- /dev/null
+++ b/arch/x86/lib/error-inject.c
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/error-injection.h>
+#include <linux/kprobes.h>
+
+asmlinkage void just_return_func(void);
+
+asm(
+	".type just_return_func, @function\n"
+	".globl just_return_func\n"
+	"just_return_func:\n"
+	"	ret\n"
+	".size just_return_func, .-just_return_func\n"
+);
+
+void override_function_with_return(struct pt_regs *regs)
+{
+	regs->ip = (unsigned long)&just_return_func;
+}
+NOKPROBE_SYMBOL(override_function_with_return);
diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S
index c97d935a29e8..49b167f73215 100644
--- a/arch/x86/lib/getuser.S
+++ b/arch/x86/lib/getuser.S
@@ -40,6 +40,8 @@ ENTRY(__get_user_1)
 	mov PER_CPU_VAR(current_task), %_ASM_DX
 	cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
 	jae bad_get_user
+	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
+	and %_ASM_DX, %_ASM_AX
 	ASM_STAC
 1:	movzbl (%_ASM_AX),%edx
 	xor %eax,%eax
@@ -54,6 +56,8 @@ ENTRY(__get_user_2)
 	mov PER_CPU_VAR(current_task), %_ASM_DX
 	cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
 	jae bad_get_user
+	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
+	and %_ASM_DX, %_ASM_AX
 	ASM_STAC
 2:	movzwl -1(%_ASM_AX),%edx
 	xor %eax,%eax
@@ -68,6 +72,8 @@ ENTRY(__get_user_4)
 	mov PER_CPU_VAR(current_task), %_ASM_DX
 	cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
 	jae bad_get_user
+	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
+	and %_ASM_DX, %_ASM_AX
 	ASM_STAC
 3:	movl -3(%_ASM_AX),%edx
 	xor %eax,%eax
@@ -83,6 +89,8 @@ ENTRY(__get_user_8)
 	mov PER_CPU_VAR(current_task), %_ASM_DX
 	cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
 	jae bad_get_user
+	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
+	and %_ASM_DX, %_ASM_AX
 	ASM_STAC
 4:	movq -7(%_ASM_AX),%rdx
 	xor %eax,%eax
@@ -94,6 +102,8 @@ ENTRY(__get_user_8)
 	mov PER_CPU_VAR(current_task), %_ASM_DX
 	cmp TASK_addr_limit(%_ASM_DX),%_ASM_AX
 	jae bad_get_user_8
+	sbb %_ASM_DX, %_ASM_DX		/* array_index_mask_nospec() */
+	and %_ASM_DX, %_ASM_AX
 	ASM_STAC
 4:	movl -7(%_ASM_AX),%edx
 5:	movl -3(%_ASM_AX),%ecx
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 1b377f734e64..7add8ba06887 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -331,12 +331,12 @@ do {									\
 
 unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
 {
-	stac();
+	__uaccess_begin_nospec();
 	if (movsl_is_ok(to, from, n))
 		__copy_user(to, from, n);
 	else
 		n = __copy_user_intel(to, from, n);
-	clac();
+	__uaccess_end();
 	return n;
 }
 EXPORT_SYMBOL(__copy_user_ll);
@@ -344,7 +344,7 @@ EXPORT_SYMBOL(__copy_user_ll);
 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
 					unsigned long n)
 {
-	stac();
+	__uaccess_begin_nospec();
 #ifdef CONFIG_X86_INTEL_USERCOPY
 	if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
 		n = __copy_user_intel_nocache(to, from, n);
@@ -353,7 +353,7 @@ unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *fr
 #else
 	__copy_user(to, from, n);
 #endif
-	clac();
+	__uaccess_end();
 	return n;
 }
 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c
index b9283cc27622..476d810639a8 100644
--- a/arch/x86/mm/cpu_entry_area.c
+++ b/arch/x86/mm/cpu_entry_area.c
@@ -163,4 +163,10 @@ void __init setup_cpu_entry_areas(void)
 
 	for_each_possible_cpu(cpu)
 		setup_cpu_entry_area(cpu);
+
+	/*
+	 * This is the last essential update to swapper_pgdir which needs
+	 * to be synchronized to initial_page_table on 32bit.
+	 */
+	sync_initial_page_table();
 }
diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c
index 9fe656c42aa5..45f5d6cf65ae 100644
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -21,16 +21,16 @@ ex_fixup_handler(const struct exception_table_entry *x)
 	return (ex_handler_t)((unsigned long)&x->handler + x->handler);
 }
 
-bool ex_handler_default(const struct exception_table_entry *fixup,
-		       struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_default(const struct exception_table_entry *fixup,
+				  struct pt_regs *regs, int trapnr)
 {
 	regs->ip = ex_fixup_addr(fixup);
 	return true;
 }
 EXPORT_SYMBOL(ex_handler_default);
 
-bool ex_handler_fault(const struct exception_table_entry *fixup,
-		     struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_fault(const struct exception_table_entry *fixup,
+				struct pt_regs *regs, int trapnr)
 {
 	regs->ip = ex_fixup_addr(fixup);
 	regs->ax = trapnr;
@@ -42,8 +42,8 @@ EXPORT_SYMBOL_GPL(ex_handler_fault);
  * Handler for UD0 exception following a failed test against the
  * result of a refcount inc/dec/add/sub.
  */
-bool ex_handler_refcount(const struct exception_table_entry *fixup,
-			 struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_refcount(const struct exception_table_entry *fixup,
+				   struct pt_regs *regs, int trapnr)
 {
 	/* First unconditionally saturate the refcount. */
 	*(int *)regs->cx = INT_MIN / 2;
@@ -95,8 +95,8 @@ EXPORT_SYMBOL(ex_handler_refcount);
  * of vulnerability by restoring from the initial state (essentially, zeroing
  * out all the FPU registers) if we can't restore from the task's FPU state.
  */
-bool ex_handler_fprestore(const struct exception_table_entry *fixup,
-			  struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_fprestore(const struct exception_table_entry *fixup,
+				    struct pt_regs *regs, int trapnr)
 {
 	regs->ip = ex_fixup_addr(fixup);
 
@@ -108,8 +108,8 @@ bool ex_handler_fprestore(const struct exception_table_entry *fixup,
 }
 EXPORT_SYMBOL_GPL(ex_handler_fprestore);
 
-bool ex_handler_ext(const struct exception_table_entry *fixup,
-		   struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_ext(const struct exception_table_entry *fixup,
+			      struct pt_regs *regs, int trapnr)
 {
 	/* Special hack for uaccess_err */
 	current->thread.uaccess_err = 1;
@@ -118,8 +118,8 @@ bool ex_handler_ext(const struct exception_table_entry *fixup,
 }
 EXPORT_SYMBOL(ex_handler_ext);
 
-bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup,
-			     struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup,
+				       struct pt_regs *regs, int trapnr)
 {
 	if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pF)\n",
 			 (unsigned int)regs->cx, regs->ip, (void *)regs->ip))
@@ -133,8 +133,8 @@ bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup,
 }
 EXPORT_SYMBOL(ex_handler_rdmsr_unsafe);
 
-bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup,
-			     struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup,
+				       struct pt_regs *regs, int trapnr)
 {
 	if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pF)\n",
 			 (unsigned int)regs->cx, (unsigned int)regs->dx,
@@ -147,8 +147,8 @@ bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup,
 }
 EXPORT_SYMBOL(ex_handler_wrmsr_unsafe);
 
-bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
-			 struct pt_regs *regs, int trapnr)
+__visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
+				   struct pt_regs *regs, int trapnr)
 {
 	if (static_cpu_has(X86_BUG_NULL_SEG))
 		asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS));
@@ -157,7 +157,7 @@ bool ex_handler_clear_fs(const struct exception_table_entry *fixup,
 }
 EXPORT_SYMBOL(ex_handler_clear_fs);
 
-bool ex_has_fault_handler(unsigned long ip)
+__visible bool ex_has_fault_handler(unsigned long ip)
 {
 	const struct exception_table_entry *e;
 	ex_handler_t handler;
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 800de815519c..25a30b5d6582 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -330,7 +330,7 @@ static noinline int vmalloc_fault(unsigned long address)
 	if (!pmd_k)
 		return -1;
 
-	if (pmd_huge(*pmd_k))
+	if (pmd_large(*pmd_k))
 		return 0;
 
 	pte_k = pte_offset_kernel(pmd_k, address);
@@ -475,7 +475,7 @@ static noinline int vmalloc_fault(unsigned long address)
 	if (pud_none(*pud) || pud_pfn(*pud) != pud_pfn(*pud_ref))
 		BUG();
 
-	if (pud_huge(*pud))
+	if (pud_large(*pud))
 		return 0;
 
 	pmd = pmd_offset(pud, address);
@@ -486,7 +486,7 @@ static noinline int vmalloc_fault(unsigned long address)
 	if (pmd_none(*pmd) || pmd_pfn(*pmd) != pmd_pfn(*pmd_ref))
 		BUG();
 
-	if (pmd_huge(*pmd))
+	if (pmd_large(*pmd))
 		return 0;
 
 	pte_ref = pte_offset_kernel(pmd_ref, address);
@@ -1248,10 +1248,6 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
 	tsk = current;
 	mm = tsk->mm;
 
-	/*
-	 * Detect and handle instructions that would cause a page fault for
-	 * both a tracked kernel page and a userspace page.
-	 */
 	prefetchw(&mm->mmap_sem);
 
 	if (unlikely(kmmio_fault(regs, address)))
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 135c9a7898c7..396e1f0151ac 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -453,6 +453,21 @@ static inline void permanent_kmaps_init(pgd_t *pgd_base)
 }
 #endif /* CONFIG_HIGHMEM */
 
+void __init sync_initial_page_table(void)
+{
+	clone_pgd_range(initial_page_table + KERNEL_PGD_BOUNDARY,
+			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			KERNEL_PGD_PTRS);
+
+	/*
+	 * sync back low identity map too.  It is used for example
+	 * in the 32-bit EFI stub.
+	 */
+	clone_pgd_range(initial_page_table,
+			swapper_pg_dir     + KERNEL_PGD_BOUNDARY,
+			min(KERNEL_PGD_PTRS, KERNEL_PGD_BOUNDARY));
+}
+
 void __init native_pagetable_init(void)
 {
 	unsigned long pfn, va;
@@ -829,23 +844,24 @@ void __init mem_init(void)
 }
 
 #ifdef CONFIG_MEMORY_HOTPLUG
-int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
+int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
+		bool want_memblock)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 
-	return __add_pages(nid, start_pfn, nr_pages, want_memblock);
+	return __add_pages(nid, start_pfn, nr_pages, altmap, want_memblock);
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
-int arch_remove_memory(u64 start, u64 size)
+int arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 	struct zone *zone;
 
 	zone = page_zone(pfn_to_page(start_pfn));
-	return __remove_pages(zone, start_pfn, nr_pages);
+	return __remove_pages(zone, start_pfn, nr_pages, altmap);
 }
 #endif
 #endif
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 4a837289f2ad..af11a2890235 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -256,7 +256,7 @@ static void __set_pte_vaddr(pud_t *pud, unsigned long vaddr, pte_t new_pte)
 	 * It's enough to flush this one mapping.
 	 * (PGE mappings get flushed as well)
 	 */
-	__flush_tlb_one(vaddr);
+	__flush_tlb_one_kernel(vaddr);
 }
 
 void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte)
@@ -772,12 +772,12 @@ static void update_end_of_memory_vars(u64 start, u64 size)
 	}
 }
 
-int add_pages(int nid, unsigned long start_pfn,
-	      unsigned long nr_pages, bool want_memblock)
+int add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages,
+		struct vmem_altmap *altmap, bool want_memblock)
 {
 	int ret;
 
-	ret = __add_pages(nid, start_pfn, nr_pages, want_memblock);
+	ret = __add_pages(nid, start_pfn, nr_pages, altmap, want_memblock);
 	WARN_ON_ONCE(ret);
 
 	/* update max_pfn, max_low_pfn and high_memory */
@@ -787,16 +787,16 @@ int add_pages(int nid, unsigned long start_pfn,
 	return ret;
 }
 
-int arch_add_memory(int nid, u64 start, u64 size, bool want_memblock)
+int arch_add_memory(int nid, u64 start, u64 size, struct vmem_altmap *altmap,
+		bool want_memblock)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 
 	init_memory_mapping(start, start + size);
 
-	return add_pages(nid, start_pfn, nr_pages, want_memblock);
+	return add_pages(nid, start_pfn, nr_pages, altmap, want_memblock);
 }
-EXPORT_SYMBOL_GPL(arch_add_memory);
 
 #define PAGE_INUSE 0xFD
 
@@ -804,12 +804,6 @@ static void __meminit free_pagetable(struct page *page, int order)
 {
 	unsigned long magic;
 	unsigned int nr_pages = 1 << order;
-	struct vmem_altmap *altmap = to_vmem_altmap((unsigned long) page);
-
-	if (altmap) {
-		vmem_altmap_free(altmap, nr_pages);
-		return;
-	}
 
 	/* bootmem page has reserved flag */
 	if (PageReserved(page)) {
@@ -826,6 +820,15 @@ static void __meminit free_pagetable(struct page *page, int order)
 		free_pages((unsigned long)page_address(page), order);
 }
 
+static void __meminit free_hugepage_table(struct page *page,
+		struct vmem_altmap *altmap)
+{
+	if (altmap)
+		vmem_altmap_free(altmap, PMD_SIZE / PAGE_SIZE);
+	else
+		free_pagetable(page, get_order(PMD_SIZE));
+}
+
 static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd)
 {
 	pte_t *pte;
@@ -953,7 +956,7 @@ remove_pte_table(pte_t *pte_start, unsigned long addr, unsigned long end,
 
 static void __meminit
 remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
-		 bool direct)
+		 bool direct, struct vmem_altmap *altmap)
 {
 	unsigned long next, pages = 0;
 	pte_t *pte_base;
@@ -971,8 +974,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
 			if (IS_ALIGNED(addr, PMD_SIZE) &&
 			    IS_ALIGNED(next, PMD_SIZE)) {
 				if (!direct)
-					free_pagetable(pmd_page(*pmd),
-						       get_order(PMD_SIZE));
+					free_hugepage_table(pmd_page(*pmd),
+							    altmap);
 
 				spin_lock(&init_mm.page_table_lock);
 				pmd_clear(pmd);
@@ -985,8 +988,8 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
 				page_addr = page_address(pmd_page(*pmd));
 				if (!memchr_inv(page_addr, PAGE_INUSE,
 						PMD_SIZE)) {
-					free_pagetable(pmd_page(*pmd),
-						       get_order(PMD_SIZE));
+					free_hugepage_table(pmd_page(*pmd),
+							    altmap);
 
 					spin_lock(&init_mm.page_table_lock);
 					pmd_clear(pmd);
@@ -1009,7 +1012,7 @@ remove_pmd_table(pmd_t *pmd_start, unsigned long addr, unsigned long end,
 
 static void __meminit
 remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
-		 bool direct)
+		 struct vmem_altmap *altmap, bool direct)
 {
 	unsigned long next, pages = 0;
 	pmd_t *pmd_base;
@@ -1054,7 +1057,7 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
 		}
 
 		pmd_base = pmd_offset(pud, 0);
-		remove_pmd_table(pmd_base, addr, next, direct);
+		remove_pmd_table(pmd_base, addr, next, direct, altmap);
 		free_pmd_table(pmd_base, pud);
 	}
 
@@ -1064,7 +1067,7 @@ remove_pud_table(pud_t *pud_start, unsigned long addr, unsigned long end,
 
 static void __meminit
 remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
-		 bool direct)
+		 struct vmem_altmap *altmap, bool direct)
 {
 	unsigned long next, pages = 0;
 	pud_t *pud_base;
@@ -1080,7 +1083,7 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
 		BUILD_BUG_ON(p4d_large(*p4d));
 
 		pud_base = pud_offset(p4d, 0);
-		remove_pud_table(pud_base, addr, next, direct);
+		remove_pud_table(pud_base, addr, next, altmap, direct);
 		/*
 		 * For 4-level page tables we do not want to free PUDs, but in the
 		 * 5-level case we should free them. This code will have to change
@@ -1096,7 +1099,8 @@ remove_p4d_table(p4d_t *p4d_start, unsigned long addr, unsigned long end,
 
 /* start and end are both virtual address. */
 static void __meminit
-remove_pagetable(unsigned long start, unsigned long end, bool direct)
+remove_pagetable(unsigned long start, unsigned long end, bool direct,
+		struct vmem_altmap *altmap)
 {
 	unsigned long next;
 	unsigned long addr;
@@ -1111,15 +1115,16 @@ remove_pagetable(unsigned long start, unsigned long end, bool direct)
 			continue;
 
 		p4d = p4d_offset(pgd, 0);
-		remove_p4d_table(p4d, addr, next, direct);
+		remove_p4d_table(p4d, addr, next, altmap, direct);
 	}
 
 	flush_tlb_all();
 }
 
-void __ref vmemmap_free(unsigned long start, unsigned long end)
+void __ref vmemmap_free(unsigned long start, unsigned long end,
+		struct vmem_altmap *altmap)
 {
-	remove_pagetable(start, end, false);
+	remove_pagetable(start, end, false, altmap);
 }
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
@@ -1129,24 +1134,22 @@ kernel_physical_mapping_remove(unsigned long start, unsigned long end)
 	start = (unsigned long)__va(start);
 	end = (unsigned long)__va(end);
 
-	remove_pagetable(start, end, true);
+	remove_pagetable(start, end, true, NULL);
 }
 
-int __ref arch_remove_memory(u64 start, u64 size)
+int __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 	struct page *page = pfn_to_page(start_pfn);
-	struct vmem_altmap *altmap;
 	struct zone *zone;
 	int ret;
 
 	/* With altmap the first mapped page is offset from @start */
-	altmap = to_vmem_altmap((unsigned long) page);
 	if (altmap)
 		page += vmem_altmap_offset(altmap);
 	zone = page_zone(page);
-	ret = __remove_pages(zone, start_pfn, nr_pages);
+	ret = __remove_pages(zone, start_pfn, nr_pages, altmap);
 	WARN_ON_ONCE(ret);
 	kernel_physical_mapping_remove(start, start + size);
 
@@ -1186,8 +1189,8 @@ void __init mem_init(void)
 	register_page_bootmem_info();
 
 	/* Register memory areas for /proc/kcore */
-	kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR,
-			 PAGE_SIZE, KCORE_OTHER);
+	if (get_gate_vma(&init_mm))
+		kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER);
 
 	mem_init_print_info(NULL);
 }
@@ -1378,7 +1381,10 @@ static int __meminit vmemmap_populate_hugepages(unsigned long start,
 		if (pmd_none(*pmd)) {
 			void *p;
 
-			p = __vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
+			if (altmap)
+				p = altmap_alloc_block_buf(PMD_SIZE, altmap);
+			else
+				p = vmemmap_alloc_block_buf(PMD_SIZE, node);
 			if (p) {
 				pte_t entry;
 
@@ -1411,9 +1417,9 @@ static int __meminit vmemmap_populate_hugepages(unsigned long start,
 	return 0;
 }
 
-int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
+int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
+		struct vmem_altmap *altmap)
 {
-	struct vmem_altmap *altmap = to_vmem_altmap(start);
 	int err;
 
 	if (boot_cpu_has(X86_FEATURE_PSE))
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index c45b6ec5357b..e2db83bebc3b 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -820,5 +820,5 @@ void __init __early_set_fixmap(enum fixed_addresses idx,
 		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
 	else
 		pte_clear(&init_mm, addr, pte);
-	__flush_tlb_one(addr);
+	__flush_tlb_one_kernel(addr);
 }
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
index 58477ec3d66d..7c8686709636 100644
--- a/arch/x86/mm/kmmio.c
+++ b/arch/x86/mm/kmmio.c
@@ -168,7 +168,7 @@ static int clear_page_presence(struct kmmio_fault_page *f, bool clear)
 		return -1;
 	}
 
-	__flush_tlb_one(f->addr);
+	__flush_tlb_one_kernel(f->addr);
 	return 0;
 }
 
diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c
index e1d61e8500f9..1a53071e2e17 100644
--- a/arch/x86/mm/mem_encrypt.c
+++ b/arch/x86/mm/mem_encrypt.c
@@ -15,7 +15,7 @@
 #include <linux/linkage.h>
 #include <linux/init.h>
 #include <linux/mm.h>
-#include <linux/dma-mapping.h>
+#include <linux/dma-direct.h>
 #include <linux/swiotlb.h>
 #include <linux/mem_encrypt.h>
 
diff --git a/arch/x86/mm/mem_encrypt_boot.S b/arch/x86/mm/mem_encrypt_boot.S
index 01f682cf77a8..40a6085063d6 100644
--- a/arch/x86/mm/mem_encrypt_boot.S
+++ b/arch/x86/mm/mem_encrypt_boot.S
@@ -15,6 +15,7 @@
 #include <asm/page.h>
 #include <asm/processor-flags.h>
 #include <asm/msr-index.h>
+#include <asm/nospec-branch.h>
 
 	.text
 	.code64
@@ -59,6 +60,7 @@ ENTRY(sme_encrypt_execute)
 	movq	%rax, %r8		/* Workarea encryption routine */
 	addq	$PAGE_SIZE, %r8		/* Workarea intermediate copy buffer */
 
+	ANNOTATE_RETPOLINE_SAFE
 	call	*%rax			/* Call the encryption routine */
 
 	pop	%r12
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c
index 4d434ddb75db..2c1ecf4763c4 100644
--- a/arch/x86/mm/mmio-mod.c
+++ b/arch/x86/mm/mmio-mod.c
@@ -29,7 +29,6 @@
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/io.h>
-#include <linux/kallsyms.h>
 #include <asm/pgtable.h>
 #include <linux/mmiotrace.h>
 #include <asm/e820/api.h> /* for ISA_START_ADDRESS */
@@ -123,8 +122,8 @@ static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
 	pr_emerg("unexpected fault for address: 0x%08lx, last fault for address: 0x%08lx\n",
 		 addr, my_reason->addr);
 	print_pte(addr);
-	print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip);
-	print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip);
+	pr_emerg("faulting IP is at %pS\n", (void *)regs->ip);
+	pr_emerg("last faulting IP was at %pS\n", (void *)my_reason->ip);
 #ifdef __i386__
 	pr_emerg("eax: %08lx   ebx: %08lx   ecx: %08lx   edx: %08lx\n",
 		 regs->ax, regs->bx, regs->cx, regs->dx);
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index fe7d57a8fb60..1555bd7d3449 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -677,6 +677,25 @@ static enum page_cache_mode lookup_memtype(u64 paddr)
 	return rettype;
 }
 
+/**
+ * pat_pfn_immune_to_uc_mtrr - Check whether the PAT memory type
+ * of @pfn cannot be overridden by UC MTRR memory type.
+ *
+ * Only to be called when PAT is enabled.
+ *
+ * Returns true, if the PAT memory type of @pfn is UC, UC-, or WC.
+ * Returns false in other cases.
+ */
+bool pat_pfn_immune_to_uc_mtrr(unsigned long pfn)
+{
+	enum page_cache_mode cm = lookup_memtype(PFN_PHYS(pfn));
+
+	return cm == _PAGE_CACHE_MODE_UC ||
+	       cm == _PAGE_CACHE_MODE_UC_MINUS ||
+	       cm == _PAGE_CACHE_MODE_WC;
+}
+EXPORT_SYMBOL_GPL(pat_pfn_immune_to_uc_mtrr);
+
 /**
  * io_reserve_memtype - Request a memory type mapping for a region of memory
  * @start: start (physical address) of the region
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 004abf9ebf12..34cda7e0551b 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -702,4 +702,52 @@ int pmd_clear_huge(pmd_t *pmd)
 
 	return 0;
 }
+
+/**
+ * pud_free_pmd_page - Clear pud entry and free pmd page.
+ * @pud: Pointer to a PUD.
+ *
+ * Context: The pud range has been unmaped and TLB purged.
+ * Return: 1 if clearing the entry succeeded. 0 otherwise.
+ */
+int pud_free_pmd_page(pud_t *pud)
+{
+	pmd_t *pmd;
+	int i;
+
+	if (pud_none(*pud))
+		return 1;
+
+	pmd = (pmd_t *)pud_page_vaddr(*pud);
+
+	for (i = 0; i < PTRS_PER_PMD; i++)
+		if (!pmd_free_pte_page(&pmd[i]))
+			return 0;
+
+	pud_clear(pud);
+	free_page((unsigned long)pmd);
+
+	return 1;
+}
+
+/**
+ * pmd_free_pte_page - Clear pmd entry and free pte page.
+ * @pmd: Pointer to a PMD.
+ *
+ * Context: The pmd range has been unmaped and TLB purged.
+ * Return: 1 if clearing the entry succeeded. 0 otherwise.
+ */
+int pmd_free_pte_page(pmd_t *pmd)
+{
+	pte_t *pte;
+
+	if (pmd_none(*pmd))
+		return 1;
+
+	pte = (pte_t *)pmd_page_vaddr(*pmd);
+	pmd_clear(pmd);
+	free_page((unsigned long)pte);
+
+	return 1;
+}
 #endif	/* CONFIG_HAVE_ARCH_HUGE_VMAP */
diff --git a/arch/x86/mm/pgtable_32.c b/arch/x86/mm/pgtable_32.c
index c3c5274410a9..9bb7f0ab9fe6 100644
--- a/arch/x86/mm/pgtable_32.c
+++ b/arch/x86/mm/pgtable_32.c
@@ -63,7 +63,7 @@ void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
 	 * It's enough to flush this one mapping.
 	 * (PGE mappings get flushed as well)
 	 */
-	__flush_tlb_one(vaddr);
+	__flush_tlb_one_kernel(vaddr);
 }
 
 unsigned long __FIXADDR_TOP = 0xfffff000;
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index ce38f165489b..631507f0c198 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -332,7 +332,7 @@ static void __init pti_clone_user_shared(void)
 }
 
 /*
- * Clone the ESPFIX P4D into the user space visinble page table
+ * Clone the ESPFIX P4D into the user space visible page table
  */
 static void __init pti_setup_espfix64(void)
 {
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5bfe61a5e8e3..7f1a51399674 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -6,13 +6,14 @@
 #include <linux/interrupt.h>
 #include <linux/export.h>
 #include <linux/cpu.h>
+#include <linux/debugfs.h>
 
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
+#include <asm/nospec-branch.h>
 #include <asm/cache.h>
 #include <asm/apic.h>
 #include <asm/uv/uv.h>
-#include <linux/debugfs.h>
 
 /*
  *	TLB flushing, formerly SMP-only
@@ -228,6 +229,12 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 #endif
 	this_cpu_write(cpu_tlbstate.is_lazy, false);
 
+	/*
+	 * The membarrier system call requires a full memory barrier and
+	 * core serialization before returning to user-space, after
+	 * storing to rq->curr. Writing to CR3 provides that full
+	 * memory barrier and core serializing instruction.
+	 */
 	if (real_prev == next) {
 		VM_WARN_ON(this_cpu_read(cpu_tlbstate.ctxs[prev_asid].ctx_id) !=
 			   next->context.ctx_id);
@@ -247,6 +254,27 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 	} else {
 		u16 new_asid;
 		bool need_flush;
+		u64 last_ctx_id = this_cpu_read(cpu_tlbstate.last_ctx_id);
+
+		/*
+		 * Avoid user/user BTB poisoning by flushing the branch
+		 * predictor when switching between processes. This stops
+		 * one process from doing Spectre-v2 attacks on another.
+		 *
+		 * As an optimization, flush indirect branches only when
+		 * switching into processes that disable dumping. This
+		 * protects high value processes like gpg, without having
+		 * too high performance overhead. IBPB is *expensive*!
+		 *
+		 * This will not flush branches when switching into kernel
+		 * threads. It will also not flush if we switch to idle
+		 * thread and back to the same process. It will flush if we
+		 * switch to a different non-dumpable process.
+		 */
+		if (tsk && tsk->mm &&
+		    tsk->mm->context.ctx_id != last_ctx_id &&
+		    get_dumpable(tsk->mm) != SUID_DUMP_USER)
+			indirect_branch_prediction_barrier();
 
 		if (IS_ENABLED(CONFIG_VMAP_STACK)) {
 			/*
@@ -292,6 +320,14 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 			trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, 0);
 		}
 
+		/*
+		 * Record last user mm's context id, so we can avoid
+		 * flushing branch buffer with IBPB if we switch back
+		 * to the same user.
+		 */
+		if (next != &init_mm)
+			this_cpu_write(cpu_tlbstate.last_ctx_id, next->context.ctx_id);
+
 		this_cpu_write(cpu_tlbstate.loaded_mm, next);
 		this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid);
 	}
@@ -369,6 +405,7 @@ void initialize_tlbstate_and_flush(void)
 	write_cr3(build_cr3(mm->pgd, 0));
 
 	/* Reinitialize tlbstate. */
+	this_cpu_write(cpu_tlbstate.last_ctx_id, mm->context.ctx_id);
 	this_cpu_write(cpu_tlbstate.loaded_mm_asid, 0);
 	this_cpu_write(cpu_tlbstate.next_asid, 1);
 	this_cpu_write(cpu_tlbstate.ctxs[0].ctx_id, mm->context.ctx_id);
@@ -461,7 +498,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 	 *    flush that changes context.tlb_gen from 2 to 3.  If they get
 	 *    processed on this CPU in reverse order, we'll see
 	 *     local_tlb_gen == 1, mm_tlb_gen == 3, and end != TLB_FLUSH_ALL.
-	 *    If we were to use __flush_tlb_single() and set local_tlb_gen to
+	 *    If we were to use __flush_tlb_one_user() and set local_tlb_gen to
 	 *    3, we'd be break the invariant: we'd update local_tlb_gen above
 	 *    1 without the full flush that's needed for tlb_gen 2.
 	 *
@@ -482,7 +519,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
 
 		addr = f->start;
 		while (addr < f->end) {
-			__flush_tlb_single(addr);
+			__flush_tlb_one_user(addr);
 			addr += PAGE_SIZE;
 		}
 		if (local)
@@ -629,7 +666,7 @@ static void do_kernel_range_flush(void *info)
 
 	/* flush range by one by one 'invlpg' */
 	for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
-		__flush_tlb_one(addr);
+		__flush_tlb_one_kernel(addr);
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 0554e8aef4d5..ce5b2ebd5701 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -13,10 +13,9 @@
 #include <linux/if_vlan.h>
 #include <asm/cacheflush.h>
 #include <asm/set_memory.h>
+#include <asm/nospec-branch.h>
 #include <linux/bpf.h>
 
-int bpf_jit_enable __read_mostly;
-
 /*
  * assembly code in arch/x86/net/bpf_jit.S
  */
@@ -154,6 +153,11 @@ static bool is_ereg(u32 reg)
 			     BIT(BPF_REG_AX));
 }
 
+static bool is_axreg(u32 reg)
+{
+	return reg == BPF_REG_0;
+}
+
 /* add modifiers if 'reg' maps to x64 registers r8..r15 */
 static u8 add_1mod(u8 byte, u32 reg)
 {
@@ -287,7 +291,7 @@ static void emit_bpf_tail_call(u8 **pprog)
 	EMIT2(0x89, 0xD2);                        /* mov edx, edx */
 	EMIT3(0x39, 0x56,                         /* cmp dword ptr [rsi + 16], edx */
 	      offsetof(struct bpf_array, map.max_entries));
-#define OFFSET1 43 /* number of bytes to jump */
+#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* number of bytes to jump */
 	EMIT2(X86_JBE, OFFSET1);                  /* jbe out */
 	label1 = cnt;
 
@@ -296,7 +300,7 @@ static void emit_bpf_tail_call(u8 **pprog)
 	 */
 	EMIT2_off32(0x8B, 0x85, 36);              /* mov eax, dword ptr [rbp + 36] */
 	EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT);     /* cmp eax, MAX_TAIL_CALL_CNT */
-#define OFFSET2 32
+#define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
 	EMIT2(X86_JA, OFFSET2);                   /* ja out */
 	label2 = cnt;
 	EMIT3(0x83, 0xC0, 0x01);                  /* add eax, 1 */
@@ -310,7 +314,7 @@ static void emit_bpf_tail_call(u8 **pprog)
 	 *   goto out;
 	 */
 	EMIT3(0x48, 0x85, 0xC0);		  /* test rax,rax */
-#define OFFSET3 10
+#define OFFSET3 (8 + RETPOLINE_RAX_BPF_JIT_SIZE)
 	EMIT2(X86_JE, OFFSET3);                   /* je out */
 	label3 = cnt;
 
@@ -323,7 +327,7 @@ static void emit_bpf_tail_call(u8 **pprog)
 	 * rdi == ctx (1st arg)
 	 * rax == prog->bpf_func + prologue_size
 	 */
-	EMIT2(0xFF, 0xE0);                        /* jmp rax */
+	RETPOLINE_RAX_BPF_JIT();
 
 	/* out: */
 	BUILD_BUG_ON(cnt - label1 != OFFSET1);
@@ -447,16 +451,36 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			else if (is_ereg(dst_reg))
 				EMIT1(add_1mod(0x40, dst_reg));
 
+			/* b3 holds 'normal' opcode, b2 short form only valid
+			 * in case dst is eax/rax.
+			 */
 			switch (BPF_OP(insn->code)) {
-			case BPF_ADD: b3 = 0xC0; break;
-			case BPF_SUB: b3 = 0xE8; break;
-			case BPF_AND: b3 = 0xE0; break;
-			case BPF_OR: b3 = 0xC8; break;
-			case BPF_XOR: b3 = 0xF0; break;
+			case BPF_ADD:
+				b3 = 0xC0;
+				b2 = 0x05;
+				break;
+			case BPF_SUB:
+				b3 = 0xE8;
+				b2 = 0x2D;
+				break;
+			case BPF_AND:
+				b3 = 0xE0;
+				b2 = 0x25;
+				break;
+			case BPF_OR:
+				b3 = 0xC8;
+				b2 = 0x0D;
+				break;
+			case BPF_XOR:
+				b3 = 0xF0;
+				b2 = 0x35;
+				break;
 			}
 
 			if (is_imm8(imm32))
 				EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
+			else if (is_axreg(dst_reg))
+				EMIT1_off32(b2, imm32);
 			else
 				EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
 			break;
@@ -545,26 +569,6 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 			 */
 			EMIT2(0x31, 0xd2);
 
-			if (BPF_SRC(insn->code) == BPF_X) {
-				/* if (src_reg == 0) return 0 */
-
-				/* cmp r11, 0 */
-				EMIT4(0x49, 0x83, 0xFB, 0x00);
-
-				/* jne .+9 (skip over pop, pop, xor and jmp) */
-				EMIT2(X86_JNE, 1 + 1 + 2 + 5);
-				EMIT1(0x5A); /* pop rdx */
-				EMIT1(0x58); /* pop rax */
-				EMIT2(0x31, 0xc0); /* xor eax, eax */
-
-				/* jmp cleanup_addr
-				 * addrs[i] - 11, because there are 11 bytes
-				 * after this insn: div, mov, pop, pop, mov
-				 */
-				jmp_offset = ctx->cleanup_addr - (addrs[i] - 11);
-				EMIT1_off32(0xE9, jmp_offset);
-			}
-
 			if (BPF_CLASS(insn->code) == BPF_ALU64)
 				/* div r11 */
 				EMIT3(0x49, 0xF7, 0xF3);
@@ -1109,19 +1113,29 @@ xadd:			if (is_imm8(insn->off))
 	return proglen;
 }
 
+struct x64_jit_data {
+	struct bpf_binary_header *header;
+	int *addrs;
+	u8 *image;
+	int proglen;
+	struct jit_context ctx;
+};
+
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 {
 	struct bpf_binary_header *header = NULL;
 	struct bpf_prog *tmp, *orig_prog = prog;
+	struct x64_jit_data *jit_data;
 	int proglen, oldproglen = 0;
 	struct jit_context ctx = {};
 	bool tmp_blinded = false;
+	bool extra_pass = false;
 	u8 *image = NULL;
 	int *addrs;
 	int pass;
 	int i;
 
-	if (!bpf_jit_enable)
+	if (!prog->jit_requested)
 		return orig_prog;
 
 	tmp = bpf_jit_blind_constants(prog);
@@ -1135,10 +1149,28 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		prog = tmp;
 	}
 
+	jit_data = prog->aux->jit_data;
+	if (!jit_data) {
+		jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
+		if (!jit_data) {
+			prog = orig_prog;
+			goto out;
+		}
+		prog->aux->jit_data = jit_data;
+	}
+	addrs = jit_data->addrs;
+	if (addrs) {
+		ctx = jit_data->ctx;
+		oldproglen = jit_data->proglen;
+		image = jit_data->image;
+		header = jit_data->header;
+		extra_pass = true;
+		goto skip_init_addrs;
+	}
 	addrs = kmalloc(prog->len * sizeof(*addrs), GFP_KERNEL);
 	if (!addrs) {
 		prog = orig_prog;
-		goto out;
+		goto out_addrs;
 	}
 
 	/* Before first pass, make a rough estimation of addrs[]
@@ -1149,13 +1181,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		addrs[i] = proglen;
 	}
 	ctx.cleanup_addr = proglen;
+skip_init_addrs:
 
 	/* JITed image shrinks with every pass and the loop iterates
 	 * until the image stops shrinking. Very large bpf programs
 	 * may converge on the last pass. In such case do one more
 	 * pass to emit the final image
 	 */
-	for (pass = 0; pass < 10 || image; pass++) {
+	for (pass = 0; pass < 20 || image; pass++) {
 		proglen = do_jit(prog, addrs, image, oldproglen, &ctx);
 		if (proglen <= 0) {
 			image = NULL;
@@ -1182,6 +1215,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 			}
 		}
 		oldproglen = proglen;
+		cond_resched();
 	}
 
 	if (bpf_jit_enable > 1)
@@ -1189,7 +1223,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 
 	if (image) {
 		bpf_flush_icache(header, image + proglen);
-		bpf_jit_binary_lock_ro(header);
+		if (!prog->is_func || extra_pass) {
+			bpf_jit_binary_lock_ro(header);
+		} else {
+			jit_data->addrs = addrs;
+			jit_data->ctx = ctx;
+			jit_data->proglen = proglen;
+			jit_data->image = image;
+			jit_data->header = header;
+		}
 		prog->bpf_func = (void *)image;
 		prog->jited = 1;
 		prog->jited_len = proglen;
@@ -1197,8 +1239,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		prog = orig_prog;
 	}
 
+	if (!prog->is_func || extra_pass) {
 out_addrs:
-	kfree(addrs);
+		kfree(addrs);
+		kfree(jit_data);
+		prog->aux->jit_data = NULL;
+	}
 out:
 	if (tmp_blinded)
 		bpf_jit_prog_release_other(prog, prog == orig_prog ?
diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 174c59774cc9..a7a7677265b6 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -460,7 +460,7 @@ static int nmi_setup(void)
 		goto fail;
 
 	for_each_possible_cpu(cpu) {
-		if (!cpu)
+		if (!IS_ENABLED(CONFIG_SMP) || !cpu)
 			continue;
 
 		memcpy(per_cpu(cpu_msrs, cpu).counters,
diff --git a/arch/x86/pci/intel_mid_pci.c b/arch/x86/pci/intel_mid_pci.c
index 511921045312..43867bc85368 100644
--- a/arch/x86/pci/intel_mid_pci.c
+++ b/arch/x86/pci/intel_mid_pci.c
@@ -300,6 +300,7 @@ int __init intel_mid_pci_init(void)
 	pci_root_ops = intel_mid_pci_ops;
 	pci_soc_mode = 1;
 	/* Continue with standard init */
+	acpi_noirq_set();
 	return 1;
 }
 
diff --git a/arch/x86/pci/irq.c b/arch/x86/pci/irq.c
index 0452629148be..52e55108404e 100644
--- a/arch/x86/pci/irq.c
+++ b/arch/x86/pci/irq.c
@@ -839,7 +839,8 @@ static void __init pirq_find_router(struct irq_router *r)
 	DBG(KERN_DEBUG "PCI: Attempting to find IRQ router for [%04x:%04x]\n",
 	    rt->rtr_vendor, rt->rtr_device);
 
-	pirq_router_dev = pci_get_bus_and_slot(rt->rtr_bus, rt->rtr_devfn);
+	pirq_router_dev = pci_get_domain_bus_and_slot(0, rt->rtr_bus,
+						      rt->rtr_devfn);
 	if (!pirq_router_dev) {
 		DBG(KERN_DEBUG "PCI: Interrupt router not found at "
 			"%02x:%02x\n", rt->rtr_bus, rt->rtr_devfn);
diff --git a/arch/x86/pci/sta2x11-fixup.c b/arch/x86/pci/sta2x11-fixup.c
index 53d600217973..75577c1490c4 100644
--- a/arch/x86/pci/sta2x11-fixup.c
+++ b/arch/x86/pci/sta2x11-fixup.c
@@ -26,6 +26,7 @@
 #include <linux/pci_ids.h>
 #include <linux/export.h>
 #include <linux/list.h>
+#include <linux/dma-direct.h>
 #include <asm/iommu.h>
 
 #define STA2X11_SWIOTLB_SIZE (4*1024*1024)
diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
index c4b3646bd04c..9542a746dc50 100644
--- a/arch/x86/pci/xen.c
+++ b/arch/x86/pci/xen.c
@@ -409,10 +409,8 @@ int __init pci_xen_init(void)
 	pcibios_enable_irq = xen_pcifront_enable_irq;
 	pcibios_disable_irq = NULL;
 
-#ifdef CONFIG_ACPI
 	/* Keep ACPI out of the picture */
-	acpi_noirq = 1;
-#endif
+	acpi_noirq_set();
 
 #ifdef CONFIG_PCI_MSI
 	x86_msi.setup_msi_irqs = xen_setup_msi_irqs;
diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c
index 2dd15e967c3f..f9cfbc0d1f33 100644
--- a/arch/x86/platform/efi/efi_64.c
+++ b/arch/x86/platform/efi/efi_64.c
@@ -25,7 +25,6 @@
 #include <linux/spinlock.h>
 #include <linux/bootmem.h>
 #include <linux/ioport.h>
-#include <linux/init.h>
 #include <linux/mc146818rtc.h>
 #include <linux/efi.h>
 #include <linux/uaccess.h>
@@ -228,7 +227,7 @@ int __init efi_alloc_page_tables(void)
 	if (!pud) {
 		if (CONFIG_PGTABLE_LEVELS > 4)
 			free_page((unsigned long) pgd_page_vaddr(*pgd));
-		free_page((unsigned long)efi_pgd);
+		free_pages((unsigned long)efi_pgd, PGD_ALLOCATION_ORDER);
 		return -ENOMEM;
 	}
 
diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c
index 86676cec99a1..fb1df9488e98 100644
--- a/arch/x86/platform/intel-mid/intel-mid.c
+++ b/arch/x86/platform/intel-mid/intel-mid.c
@@ -79,7 +79,7 @@ static void intel_mid_power_off(void)
 
 static void intel_mid_reboot(void)
 {
-	intel_scu_ipc_simple_command(IPCMSG_COLD_BOOT, 0);
+	intel_scu_ipc_simple_command(IPCMSG_COLD_RESET, 0);
 }
 
 static unsigned long __init intel_mid_calibrate_tsc(void)
@@ -194,7 +194,7 @@ void __init x86_intel_mid_early_setup(void)
 	x86_platform.calibrate_tsc = intel_mid_calibrate_tsc;
 	x86_platform.get_nmi_reason = intel_mid_get_nmi_reason;
 
-	x86_init.pci.init = intel_mid_pci_init;
+	x86_init.pci.arch_init = intel_mid_pci_init;
 	x86_init.pci.fixup_irqs = x86_init_noop;
 
 	legacy_pic = &null_legacy_pic;
diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c
index 19b43e3a9f0f..7be1e1fe9ae3 100644
--- a/arch/x86/platform/intel-mid/sfi.c
+++ b/arch/x86/platform/intel-mid/sfi.c
@@ -96,8 +96,7 @@ int __init sfi_parse_mtmr(struct sfi_table_header *table)
 			pentry->freq_hz, pentry->irq);
 		mp_irq.type = MP_INTSRC;
 		mp_irq.irqtype = mp_INT;
-		/* triggering mode edge bit 2-3, active high polarity bit 0-1 */
-		mp_irq.irqflag = 5;
+		mp_irq.irqflag = MP_IRQTRIG_EDGE | MP_IRQPOL_ACTIVE_HIGH;
 		mp_irq.srcbus = MP_BUS_ISA;
 		mp_irq.srcbusirq = pentry->irq;	/* IRQ */
 		mp_irq.dstapic = MP_APIC_ALL;
@@ -168,7 +167,7 @@ int __init sfi_parse_mrtc(struct sfi_table_header *table)
 			totallen, (u32)pentry->phys_addr, pentry->irq);
 		mp_irq.type = MP_INTSRC;
 		mp_irq.irqtype = mp_INT;
-		mp_irq.irqflag = 0xf;	/* level trigger and active low */
+		mp_irq.irqflag = MP_IRQTRIG_LEVEL | MP_IRQPOL_ACTIVE_LOW;
 		mp_irq.srcbus = MP_BUS_ISA;
 		mp_irq.srcbusirq = pentry->irq;	/* IRQ */
 		mp_irq.dstapic = MP_APIC_ALL;
diff --git a/arch/x86/platform/intel/iosf_mbi.c b/arch/x86/platform/intel/iosf_mbi.c
index a952ac199741..6f37a2137a79 100644
--- a/arch/x86/platform/intel/iosf_mbi.c
+++ b/arch/x86/platform/intel/iosf_mbi.c
@@ -218,14 +218,23 @@ int iosf_mbi_register_pmic_bus_access_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL(iosf_mbi_register_pmic_bus_access_notifier);
 
+int iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
+	struct notifier_block *nb)
+{
+	iosf_mbi_assert_punit_acquired();
+
+	return blocking_notifier_chain_unregister(
+				&iosf_mbi_pmic_bus_access_notifier, nb);
+}
+EXPORT_SYMBOL(iosf_mbi_unregister_pmic_bus_access_notifier_unlocked);
+
 int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb)
 {
 	int ret;
 
 	/* Wait for the bus to go inactive before unregistering */
 	mutex_lock(&iosf_mbi_punit_mutex);
-	ret = blocking_notifier_chain_unregister(
-				&iosf_mbi_pmic_bus_access_notifier, nb);
+	ret = iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(nb);
 	mutex_unlock(&iosf_mbi_punit_mutex);
 
 	return ret;
@@ -239,6 +248,12 @@ int iosf_mbi_call_pmic_bus_access_notifier_chain(unsigned long val, void *v)
 }
 EXPORT_SYMBOL(iosf_mbi_call_pmic_bus_access_notifier_chain);
 
+void iosf_mbi_assert_punit_acquired(void)
+{
+	WARN_ON(!mutex_is_locked(&iosf_mbi_punit_mutex));
+}
+EXPORT_SYMBOL(iosf_mbi_assert_punit_acquired);
+
 #ifdef CONFIG_IOSF_MBI_DEBUG
 static u32	dbg_mdr;
 static u32	dbg_mcr;
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 8538a6723171..b36caae0fb2f 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -299,7 +299,7 @@ static void bau_process_message(struct msg_desc *mdp, struct bau_control *bcp,
 		local_flush_tlb();
 		stat->d_alltlb++;
 	} else {
-		__flush_tlb_single(msg->address);
+		__flush_tlb_one_user(msg->address);
 		stat->d_onetlb++;
 	}
 	stat->d_requestee++;
@@ -1751,7 +1751,8 @@ static void activation_descriptor_init(int node, int pnode, int base_pnode)
 		uv1 = 1;
 
 	/* the 14-bit pnode */
-	write_mmr_descriptor_base(pnode, (n << UV_DESC_PSHIFT | m));
+	write_mmr_descriptor_base(pnode,
+		(n << UVH_LB_BAU_SB_DESCRIPTOR_BASE_NODE_ID_SHFT | m));
 	/*
 	 * Initializing all 8 (ITEMS_PER_DESC) descriptors for each
 	 * cpu even though we only use the first one; one descriptor can
@@ -2254,8 +2255,6 @@ static int __init uv_bau_init(void)
 			init_uvhub(uvhub, vector, uv_base_pnode);
 	}
 
-	alloc_intr_gate(vector, uv_bau_message_intr1);
-
 	for_each_possible_blade(uvhub) {
 		if (uv_blade_nr_possible_cpus(uvhub)) {
 			unsigned long val;
diff --git a/arch/x86/power/hibernate_32.c b/arch/x86/power/hibernate_32.c
index c35fdb585c68..afc4ed7b1578 100644
--- a/arch/x86/power/hibernate_32.c
+++ b/arch/x86/power/hibernate_32.c
@@ -145,7 +145,7 @@ static inline void resume_init_first_level_page_table(pgd_t *pg_dir)
 #endif
 }
 
-int swsusp_arch_resume(void)
+asmlinkage int swsusp_arch_resume(void)
 {
 	int error;
 
diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c
index f910c514438f..0ef5e5204968 100644
--- a/arch/x86/power/hibernate_64.c
+++ b/arch/x86/power/hibernate_64.c
@@ -174,7 +174,7 @@ static int relocate_restore_code(void)
 	return 0;
 }
 
-int swsusp_arch_resume(void)
+asmlinkage int swsusp_arch_resume(void)
 {
 	int error;
 
diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile
index 2f15a2ac4209..d70c15de417b 100644
--- a/arch/x86/purgatory/Makefile
+++ b/arch/x86/purgatory/Makefile
@@ -16,7 +16,7 @@ KCOV_INSTRUMENT := n
 # in turn leaves some undefined symbols like __fentry__ in purgatory and not
 # sure how to relocate those. Like kexec-tools, use custom flags.
 
-KBUILD_CFLAGS := -fno-strict-aliasing -Wall -Wstrict-prototypes -fno-zero-initialized-in-bss -fno-builtin -ffreestanding -c -MD -Os -mcmodel=large
+KBUILD_CFLAGS := -fno-strict-aliasing -Wall -Wstrict-prototypes -fno-zero-initialized-in-bss -fno-builtin -ffreestanding -c -Os -mcmodel=large
 KBUILD_CFLAGS += -m$(BITS)
 KBUILD_CFLAGS += $(call cc-option,-fno-PIE)
 
diff --git a/arch/x86/realmode/rm/trampoline_64.S b/arch/x86/realmode/rm/trampoline_64.S
index de53bd15df5a..24bb7598774e 100644
--- a/arch/x86/realmode/rm/trampoline_64.S
+++ b/arch/x86/realmode/rm/trampoline_64.S
@@ -102,7 +102,7 @@ ENTRY(startup_32)
 	 * don't we'll eventually crash trying to execute encrypted
 	 * instructions.
 	 */
-	bt	$TH_FLAGS_SME_ACTIVE_BIT, pa_tr_flags
+	btl	$TH_FLAGS_SME_ACTIVE_BIT, pa_tr_flags
 	jnc	.Ldone
 	movl	$MSR_K8_SYSCFG, %ecx
 	rdmsr
diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile
index 972b8e8d939c..09af7ff53044 100644
--- a/arch/x86/tools/Makefile
+++ b/arch/x86/tools/Makefile
@@ -13,28 +13,28 @@ else
   posttest_64bit = -n
 endif
 
-distill_awk = $(srctree)/arch/x86/tools/distill.awk
+reformatter = $(srctree)/arch/x86/tools/objdump_reformat.awk
 chkobjdump = $(srctree)/arch/x86/tools/chkobjdump.awk
 
 quiet_cmd_posttest = TEST    $@
-      cmd_posttest = ($(OBJDUMP) -v | $(AWK) -f $(chkobjdump)) || $(OBJDUMP) -d -j .text $(objtree)/vmlinux | $(AWK) -f $(distill_awk) | $(obj)/test_get_len $(posttest_64bit) $(posttest_verbose)
+      cmd_posttest = ($(OBJDUMP) -v | $(AWK) -f $(chkobjdump)) || $(OBJDUMP) -d -j .text $(objtree)/vmlinux | $(AWK) -f $(reformatter) | $(obj)/insn_decoder_test $(posttest_64bit) $(posttest_verbose)
 
 quiet_cmd_sanitytest = TEST    $@
       cmd_sanitytest = $(obj)/insn_sanity $(posttest_64bit) -m 1000000
 
-posttest: $(obj)/test_get_len vmlinux $(obj)/insn_sanity
+posttest: $(obj)/insn_decoder_test vmlinux $(obj)/insn_sanity
 	$(call cmd,posttest)
 	$(call cmd,sanitytest)
 
-hostprogs-y	+= test_get_len insn_sanity
+hostprogs-y	+= insn_decoder_test insn_sanity
 
 # -I needed for generated C source and C source which in the kernel tree.
-HOSTCFLAGS_test_get_len.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/
+HOSTCFLAGS_insn_decoder_test.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/
 
 HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/
 
 # Dependencies are also needed.
-$(obj)/test_get_len.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c
+$(obj)/insn_decoder_test.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c
 
 $(obj)/insn_sanity.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c
 
diff --git a/arch/x86/tools/test_get_len.c b/arch/x86/tools/insn_decoder_test.c
similarity index 81%
rename from arch/x86/tools/test_get_len.c
rename to arch/x86/tools/insn_decoder_test.c
index ecf31e0358c8..a3b4fd954931 100644
--- a/arch/x86/tools/test_get_len.c
+++ b/arch/x86/tools/insn_decoder_test.c
@@ -9,10 +9,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
  * Copyright (C) IBM Corporation, 2009
  */
 
@@ -21,6 +17,7 @@
 #include <string.h>
 #include <assert.h>
 #include <unistd.h>
+#include <stdarg.h>
 
 #define unlikely(cond) (cond)
 
@@ -33,7 +30,7 @@
  * particular.  See if insn_get_length() and the disassembler agree
  * on the length of each instruction in an elf disassembly.
  *
- * Usage: objdump -d a.out | awk -f distill.awk | ./test_get_len
+ * Usage: objdump -d a.out | awk -f objdump_reformat.awk | ./insn_decoder_test
  */
 
 const char *prog;
@@ -42,8 +39,8 @@ static int x86_64;
 
 static void usage(void)
 {
-	fprintf(stderr, "Usage: objdump -d a.out | awk -f distill.awk |"
-		" %s [-y|-n] [-v]\n", prog);
+	fprintf(stderr, "Usage: objdump -d a.out | awk -f objdump_reformat.awk"
+		" | %s [-y|-n] [-v]\n", prog);
 	fprintf(stderr, "\t-y	64bit mode\n");
 	fprintf(stderr, "\t-n	32bit mode\n");
 	fprintf(stderr, "\t-v	verbose mode\n");
@@ -52,10 +49,21 @@ static void usage(void)
 
 static void malformed_line(const char *line, int line_nr)
 {
-	fprintf(stderr, "%s: malformed line %d:\n%s", prog, line_nr, line);
+	fprintf(stderr, "%s: error: malformed line %d:\n%s",
+		prog, line_nr, line);
 	exit(3);
 }
 
+static void pr_warn(const char *fmt, ...)
+{
+	va_list ap;
+
+	fprintf(stderr, "%s: warning: ", prog);
+	va_start(ap, fmt);
+	vfprintf(stderr, fmt, ap);
+	va_end(ap);
+}
+
 static void dump_field(FILE *fp, const char *name, const char *indent,
 		       struct insn_field *field)
 {
@@ -153,21 +161,20 @@ int main(int argc, char **argv)
 		insn_get_length(&insn);
 		if (insn.length != nb) {
 			warnings++;
-			fprintf(stderr, "Warning: %s found difference at %s\n",
-				prog, sym);
-			fprintf(stderr, "Warning: %s", line);
-			fprintf(stderr, "Warning: objdump says %d bytes, but "
-				"insn_get_length() says %d\n", nb,
-				insn.length);
+			pr_warn("Found an x86 instruction decoder bug, "
+				"please report this.\n", sym);
+			pr_warn("%s", line);
+			pr_warn("objdump says %d bytes, but insn_get_length() "
+				"says %d\n", nb, insn.length);
 			if (verbose)
 				dump_insn(stderr, &insn);
 		}
 	}
 	if (warnings)
-		fprintf(stderr, "Warning: decoded and checked %d"
-			" instructions with %d warnings\n", insns, warnings);
+		pr_warn("Decoded and checked %d instructions with %d "
+			"failures\n", insns, warnings);
 	else
-		fprintf(stdout, "Success: decoded and checked %d"
-			" instructions\n", insns);
+		fprintf(stdout, "%s: success: Decoded and checked %d"
+			" instructions\n", prog, insns);
 	return 0;
 }
diff --git a/arch/x86/tools/distill.awk b/arch/x86/tools/objdump_reformat.awk
similarity index 91%
rename from arch/x86/tools/distill.awk
rename to arch/x86/tools/objdump_reformat.awk
index e0edeccc1429..f418c91b71f0 100644
--- a/arch/x86/tools/distill.awk
+++ b/arch/x86/tools/objdump_reformat.awk
@@ -1,7 +1,7 @@
 #!/bin/awk -f
 # SPDX-License-Identifier: GPL-2.0
-# Usage: objdump -d a.out | awk -f distill.awk | ./test_get_len
-# Distills the disassembly as follows:
+# Usage: objdump -d a.out | awk -f objdump_reformat.awk | ./insn_decoder_test
+# Reformats the disassembly as follows:
 # - Removes all lines except the disassembled instructions.
 # - For instructions that exceed 1 line (7 bytes), crams all the hex bytes
 # into a single line.
diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
index 5d73c443e778..220e97841e49 100644
--- a/arch/x86/tools/relocs.c
+++ b/arch/x86/tools/relocs.c
@@ -770,9 +770,12 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,
 		break;
 
 	case R_X86_64_PC32:
+	case R_X86_64_PLT32:
 		/*
 		 * PC relative relocations don't need to be adjusted unless
 		 * referencing a percpu symbol.
+		 *
+		 * NB: R_X86_64_PLT32 can be treated as R_X86_64_PC32.
 		 */
 		if (is_percpu_sym(sym, symname))
 			add_reloc(&relocs32neg, offset);
diff --git a/arch/x86/um/asm/barrier.h b/arch/x86/um/asm/barrier.h
index b7d73400ea29..f31e5d903161 100644
--- a/arch/x86/um/asm/barrier.h
+++ b/arch/x86/um/asm/barrier.h
@@ -30,11 +30,7 @@
 
 #endif /* CONFIG_X86_32 */
 
-#ifdef CONFIG_X86_PPRO_FENCE
-#define dma_rmb()	rmb()
-#else /* CONFIG_X86_PPRO_FENCE */
 #define dma_rmb()	barrier()
-#endif /* CONFIG_X86_PPRO_FENCE */
 #define dma_wmb()	barrier()
 
 #include <asm-generic/barrier.h>
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index c047f42552e1..3c2c2530737e 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1376,8 +1376,6 @@ asmlinkage __visible void __init xen_start_kernel(void)
 
 	if (!xen_initial_domain()) {
 		add_preferred_console("xenboot", 0, NULL);
-		add_preferred_console("tty", 0, NULL);
-		add_preferred_console("hvc", 0, NULL);
 		if (pci_xen)
 			x86_init.pci.arch_init = pci_xen_init;
 	} else {
@@ -1410,6 +1408,10 @@ asmlinkage __visible void __init xen_start_kernel(void)
 
 		xen_boot_params_init_edd();
 	}
+
+	add_preferred_console("tty", 0, NULL);
+	add_preferred_console("hvc", 0, NULL);
+
 #ifdef CONFIG_PCI
 	/* PCI BIOS service won't work from a PV guest. */
 	pci_probe &= ~PCI_PROBE_BIOS;
diff --git a/arch/x86/xen/mmu_hvm.c b/arch/x86/xen/mmu_hvm.c
index 2cfcfe4f6b2a..dd2ad82eee80 100644
--- a/arch/x86/xen/mmu_hvm.c
+++ b/arch/x86/xen/mmu_hvm.c
@@ -75,6 +75,6 @@ void __init xen_hvm_init_mmu_ops(void)
 	if (is_pagetable_dying_supported())
 		pv_mmu_ops.exit_mmap = xen_hvm_exit_mmap;
 #ifdef CONFIG_PROC_VMCORE
-	register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram);
+	WARN_ON(register_oldmem_pfn_is_ram(&xen_oldmem_pfn_is_ram));
 #endif
 }
diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c
index d85076223a69..aae88fec9941 100644
--- a/arch/x86/xen/mmu_pv.c
+++ b/arch/x86/xen/mmu_pv.c
@@ -1300,12 +1300,12 @@ static void xen_flush_tlb(void)
 	preempt_enable();
 }
 
-static void xen_flush_tlb_single(unsigned long addr)
+static void xen_flush_tlb_one_user(unsigned long addr)
 {
 	struct mmuext_op *op;
 	struct multicall_space mcs;
 
-	trace_xen_mmu_flush_tlb_single(addr);
+	trace_xen_mmu_flush_tlb_one_user(addr);
 
 	preempt_disable();
 
@@ -2370,7 +2370,7 @@ static const struct pv_mmu_ops xen_mmu_ops __initconst = {
 
 	.flush_tlb_user = xen_flush_tlb,
 	.flush_tlb_kernel = xen_flush_tlb,
-	.flush_tlb_single = xen_flush_tlb_single,
+	.flush_tlb_one_user = xen_flush_tlb_one_user,
 	.flush_tlb_others = xen_flush_tlb_others,
 
 	.pgd_alloc = xen_pgd_alloc,
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 13b4f19b9131..159a897151d6 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -694,6 +694,9 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
 	int i, ret = 0;
 	pte_t *pte;
 
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return 0;
+
 	if (kmap_ops) {
 		ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref,
 						kmap_ops, count);
@@ -736,6 +739,9 @@ int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
 {
 	int i, ret = 0;
 
+	if (xen_feature(XENFEAT_auto_translated_physmap))
+		return 0;
+
 	for (i = 0; i < count; i++) {
 		unsigned long mfn = __pfn_to_mfn(page_to_pfn(pages[i]));
 		unsigned long pfn = page_to_pfn(pages[i]);
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 77c959cf81e7..7a43b2ae19f1 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -122,6 +122,8 @@ void __init xen_smp_cpus_done(unsigned int max_cpus)
 
 	if (xen_hvm_domain())
 		native_smp_cpus_done(max_cpus);
+	else
+		calculate_max_logical_packages();
 
 	if (xen_have_vcpu_info_placement)
 		return;
diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 02f3445a2b5f..cd97a62394e7 100644
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -23,8 +23,6 @@ static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
 static DEFINE_PER_CPU(char *, irq_name);
 static bool xen_pvspin = true;
 
-#include <asm/qspinlock.h>
-
 static void xen_qlock_kick(int cpu)
 {
 	int irq = per_cpu(lock_kicker_irq, cpu);
diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c
index d9f96cc5d743..1d83152c761b 100644
--- a/arch/x86/xen/suspend.c
+++ b/arch/x86/xen/suspend.c
@@ -1,12 +1,15 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/types.h>
 #include <linux/tick.h>
+#include <linux/percpu-defs.h>
 
 #include <xen/xen.h>
 #include <xen/interface/xen.h>
 #include <xen/grant_table.h>
 #include <xen/events.h>
 
+#include <asm/cpufeatures.h>
+#include <asm/msr-index.h>
 #include <asm/xen/hypercall.h>
 #include <asm/xen/page.h>
 #include <asm/fixmap.h>
@@ -15,6 +18,8 @@
 #include "mmu.h"
 #include "pmu.h"
 
+static DEFINE_PER_CPU(u64, spec_ctrl);
+
 void xen_arch_pre_suspend(void)
 {
 	xen_save_time_memory_area();
@@ -35,6 +40,9 @@ void xen_arch_post_suspend(int cancelled)
 
 static void xen_vcpu_notify_restore(void *data)
 {
+	if (xen_pv_domain() && boot_cpu_has(X86_FEATURE_SPEC_CTRL))
+		wrmsrl(MSR_IA32_SPEC_CTRL, this_cpu_read(spec_ctrl));
+
 	/* Boot processor notified via generic timekeeping_resume() */
 	if (smp_processor_id() == 0)
 		return;
@@ -44,7 +52,15 @@ static void xen_vcpu_notify_restore(void *data)
 
 static void xen_vcpu_notify_suspend(void *data)
 {
+	u64 tmp;
+
 	tick_suspend_local();
+
+	if (xen_pv_domain() && boot_cpu_has(X86_FEATURE_SPEC_CTRL)) {
+		rdmsrl(MSR_IA32_SPEC_CTRL, tmp);
+		this_cpu_write(spec_ctrl, tmp);
+		wrmsrl(MSR_IA32_SPEC_CTRL, 0);
+	}
 }
 
 void xen_arch_resume(void)
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index 497cc55a0c16..96f26e026783 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -9,7 +9,9 @@
 
 #include <asm/boot.h>
 #include <asm/asm.h>
+#include <asm/msr.h>
 #include <asm/page_types.h>
+#include <asm/percpu.h>
 #include <asm/unwind_hints.h>
 
 #include <xen/interface/elfnote.h>
@@ -35,6 +37,20 @@ ENTRY(startup_xen)
 	mov %_ASM_SI, xen_start_info
 	mov $init_thread_union+THREAD_SIZE, %_ASM_SP
 
+#ifdef CONFIG_X86_64
+	/* Set up %gs.
+	 *
+	 * The base of %gs always points to the bottom of the irqstack
+	 * union.  If the stack protector canary is enabled, it is
+	 * located at %gs:40.  Note that, on SMP, the boot cpu uses
+	 * init data section till per cpu areas are set up.
+	 */
+	movl	$MSR_GS_BASE,%ecx
+	movq	$INIT_PER_CPU_VAR(irq_stack_union),%rax
+	cdq
+	wrmsr
+#endif
+
 	jmp xen_start_kernel
 END(startup_xen)
 	__FINIT
diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index 8bc52f749f20..c921e8bccdc8 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -15,6 +15,9 @@ config XTENSA
 	select GENERIC_IRQ_SHOW
 	select GENERIC_PCI_IOMAP
 	select GENERIC_SCHED_CLOCK
+	select GENERIC_STRNCPY_FROM_USER if KASAN
+	select HAVE_ARCH_KASAN if MMU
+	select HAVE_CC_STACKPROTECTOR
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
@@ -79,6 +82,10 @@ config VARIANT_IRQ_SWITCH
 config HAVE_XTENSA_GPIO32
 	def_bool n
 
+config KASAN_SHADOW_OFFSET
+	hex
+	default 0x6e400000
+
 menu "Processor type and features"
 
 choice
diff --git a/arch/xtensa/Makefile b/arch/xtensa/Makefile
index 7ee02fe4a63d..3a934b72a272 100644
--- a/arch/xtensa/Makefile
+++ b/arch/xtensa/Makefile
@@ -42,10 +42,11 @@ export PLATFORM
 
 # temporarily until string.h is fixed
 KBUILD_CFLAGS += -ffreestanding -D__linux__
-
-KBUILD_CFLAGS += -pipe -mlongcalls
-
+KBUILD_CFLAGS += -pipe -mlongcalls -mtext-section-literals
 KBUILD_CFLAGS += $(call cc-option,-mforce-no-pic,)
+KBUILD_CFLAGS += $(call cc-option,-mno-serialize-volatile,)
+
+KBUILD_AFLAGS += -mlongcalls -mtext-section-literals
 
 ifneq ($(CONFIG_LD_NO_RELAX),)
 LDFLAGS := --no-relax
diff --git a/arch/xtensa/boot/boot-redboot/bootstrap.S b/arch/xtensa/boot/boot-redboot/bootstrap.S
index bf7fabe6310d..bbf3b4b080cd 100644
--- a/arch/xtensa/boot/boot-redboot/bootstrap.S
+++ b/arch/xtensa/boot/boot-redboot/bootstrap.S
@@ -42,6 +42,7 @@ __start_a0:
 	.align 4
 
 	.section .text, "ax"
+	.literal_position
 	.begin literal_prefix .text
 
 	/* put literals in here! */
diff --git a/arch/xtensa/boot/lib/Makefile b/arch/xtensa/boot/lib/Makefile
index d2a7f48564a4..355127faade1 100644
--- a/arch/xtensa/boot/lib/Makefile
+++ b/arch/xtensa/boot/lib/Makefile
@@ -15,6 +15,12 @@ CFLAGS_REMOVE_inftrees.o = -pg
 CFLAGS_REMOVE_inffast.o = -pg
 endif
 
+KASAN_SANITIZE := n
+
+CFLAGS_REMOVE_inflate.o += -fstack-protector -fstack-protector-strong
+CFLAGS_REMOVE_zmem.o += -fstack-protector -fstack-protector-strong
+CFLAGS_REMOVE_inftrees.o += -fstack-protector -fstack-protector-strong
+CFLAGS_REMOVE_inffast.o += -fstack-protector -fstack-protector-strong
 
 quiet_cmd_copy_zlib = COPY    $@
       cmd_copy_zlib = cat $< > $@
diff --git a/arch/xtensa/configs/audio_kc705_defconfig b/arch/xtensa/configs/audio_kc705_defconfig
index 8d16925765cb..2bf964df37ba 100644
--- a/arch/xtensa/configs/audio_kc705_defconfig
+++ b/arch/xtensa/configs/audio_kc705_defconfig
@@ -1,7 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_IRQ_TIME_ACCOUNTING=y
diff --git a/arch/xtensa/configs/cadence_csp_defconfig b/arch/xtensa/configs/cadence_csp_defconfig
index f2d3094aa1d1..3221b7053fa3 100644
--- a/arch/xtensa/configs/cadence_csp_defconfig
+++ b/arch/xtensa/configs/cadence_csp_defconfig
@@ -1,7 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_USELIB=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_IRQ_TIME_ACCOUNTING=y
diff --git a/arch/xtensa/configs/generic_kc705_defconfig b/arch/xtensa/configs/generic_kc705_defconfig
index 744adeaf2945..985fa8546e4e 100644
--- a/arch/xtensa/configs/generic_kc705_defconfig
+++ b/arch/xtensa/configs/generic_kc705_defconfig
@@ -1,7 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_IRQ_TIME_ACCOUNTING=y
diff --git a/arch/xtensa/configs/nommu_kc705_defconfig b/arch/xtensa/configs/nommu_kc705_defconfig
index 78c2529d0459..624f9b3a3878 100644
--- a/arch/xtensa/configs/nommu_kc705_defconfig
+++ b/arch/xtensa/configs/nommu_kc705_defconfig
@@ -1,7 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_IRQ_TIME_ACCOUNTING=y
diff --git a/arch/xtensa/configs/smp_lx200_defconfig b/arch/xtensa/configs/smp_lx200_defconfig
index 14e3ca353ac8..11fed6c06a7c 100644
--- a/arch/xtensa/configs/smp_lx200_defconfig
+++ b/arch/xtensa/configs/smp_lx200_defconfig
@@ -1,7 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_FHANDLE=y
-CONFIG_IRQ_DOMAIN_DEBUG=y
 CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_IRQ_TIME_ACCOUNTING=y
diff --git a/arch/xtensa/include/asm/Kbuild b/arch/xtensa/include/asm/Kbuild
index dff7cc39437c..436b20337168 100644
--- a/arch/xtensa/include/asm/Kbuild
+++ b/arch/xtensa/include/asm/Kbuild
@@ -1,5 +1,4 @@
 generic-y += bug.h
-generic-y += clkdev.h
 generic-y += device.h
 generic-y += div64.h
 generic-y += dma-contiguous.h
diff --git a/arch/xtensa/include/asm/asmmacro.h b/arch/xtensa/include/asm/asmmacro.h
index 746dcc8b5abc..7f2ae5872151 100644
--- a/arch/xtensa/include/asm/asmmacro.h
+++ b/arch/xtensa/include/asm/asmmacro.h
@@ -150,5 +150,45 @@
 		__endl	\ar \as
 	.endm
 
+/* Load or store instructions that may cause exceptions use the EX macro. */
+
+#define EX(handler)				\
+	.section __ex_table, "a";		\
+	.word	97f, handler;			\
+	.previous				\
+97:
+
+
+/*
+ * Extract unaligned word that is split between two registers w0 and w1
+ * into r regardless of machine endianness. SAR must be loaded with the
+ * starting bit of the word (see __ssa8).
+ */
+
+	.macro __src_b	r, w0, w1
+#ifdef __XTENSA_EB__
+		src	\r, \w0, \w1
+#else
+		src	\r, \w1, \w0
+#endif
+	.endm
+
+/*
+ * Load 2 lowest address bits of r into SAR for __src_b to extract unaligned
+ * word starting at r from two registers loaded from consecutive aligned
+ * addresses covering r regardless of machine endianness.
+ *
+ *      r   0   1   2   3
+ * LE SAR   0   8  16  24
+ * BE SAR  32  24  16   8
+ */
+
+	.macro __ssa8	r
+#ifdef __XTENSA_EB__
+		ssa8b	\r
+#else
+		ssa8l	\r
+#endif
+	.endm
 
 #endif /* _XTENSA_ASMMACRO_H */
diff --git a/arch/xtensa/include/asm/current.h b/arch/xtensa/include/asm/current.h
index 47e46dcf5d49..5d98a7ad4251 100644
--- a/arch/xtensa/include/asm/current.h
+++ b/arch/xtensa/include/asm/current.h
@@ -11,6 +11,8 @@
 #ifndef _XTENSA_CURRENT_H
 #define _XTENSA_CURRENT_H
 
+#include <asm/thread_info.h>
+
 #ifndef __ASSEMBLY__
 
 #include <linux/thread_info.h>
@@ -26,8 +28,6 @@ static inline struct task_struct *get_current(void)
 
 #else
 
-#define CURRENT_SHIFT 13
-
 #define GET_CURRENT(reg,sp)		\
 	GET_THREAD_INFO(reg,sp);	\
 	l32i reg, reg, TI_TASK		\
diff --git a/arch/xtensa/include/asm/dma-mapping.h b/arch/xtensa/include/asm/dma-mapping.h
index 153bf2370988..44098800dad7 100644
--- a/arch/xtensa/include/asm/dma-mapping.h
+++ b/arch/xtensa/include/asm/dma-mapping.h
@@ -23,14 +23,4 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 	return &xtensa_dma_map_ops;
 }
 
-static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return (dma_addr_t)paddr;
-}
-
-static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
-{
-	return (phys_addr_t)daddr;
-}
-
 #endif	/* _XTENSA_DMA_MAPPING_H */
diff --git a/arch/xtensa/include/asm/fixmap.h b/arch/xtensa/include/asm/fixmap.h
index 0d30403b6c95..7e25c1b50ac0 100644
--- a/arch/xtensa/include/asm/fixmap.h
+++ b/arch/xtensa/include/asm/fixmap.h
@@ -44,7 +44,7 @@ enum fixed_addresses {
 	__end_of_fixed_addresses
 };
 
-#define FIXADDR_TOP     (VMALLOC_START - PAGE_SIZE)
+#define FIXADDR_TOP     (XCHAL_KSEG_CACHED_VADDR - PAGE_SIZE)
 #define FIXADDR_SIZE	(__end_of_fixed_addresses << PAGE_SHIFT)
 #define FIXADDR_START	((FIXADDR_TOP - FIXADDR_SIZE) & PMD_MASK)
 
@@ -63,7 +63,7 @@ static __always_inline unsigned long fix_to_virt(const unsigned int idx)
 	 * table.
 	 */
 	BUILD_BUG_ON(FIXADDR_START <
-		     XCHAL_PAGE_TABLE_VADDR + XCHAL_PAGE_TABLE_SIZE);
+		     TLBTEMP_BASE_1 + TLBTEMP_SIZE);
 	BUILD_BUG_ON(idx >= __end_of_fixed_addresses);
 	return __fix_to_virt(idx);
 }
diff --git a/arch/xtensa/include/asm/futex.h b/arch/xtensa/include/asm/futex.h
index eaaf1ebcc7a4..5bfbc1c401d4 100644
--- a/arch/xtensa/include/asm/futex.h
+++ b/arch/xtensa/include/asm/futex.h
@@ -92,7 +92,6 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 			      u32 oldval, u32 newval)
 {
 	int ret = 0;
-	u32 prev;
 
 	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
 		return -EFAULT;
@@ -103,26 +102,24 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
 
 	__asm__ __volatile__ (
 	"	# futex_atomic_cmpxchg_inatomic\n"
-	"1:	l32i	%1, %3, 0\n"
-	"	mov	%0, %5\n"
-	"	wsr	%1, scompare1\n"
-	"2:	s32c1i	%0, %3, 0\n"
-	"3:\n"
+	"	wsr	%5, scompare1\n"
+	"1:	s32c1i	%1, %4, 0\n"
+	"	s32i	%1, %6, 0\n"
+	"2:\n"
 	"	.section .fixup,\"ax\"\n"
 	"	.align 4\n"
-	"4:	.long	3b\n"
-	"5:	l32r	%1, 4b\n"
-	"	movi	%0, %6\n"
+	"3:	.long	2b\n"
+	"4:	l32r	%1, 3b\n"
+	"	movi	%0, %7\n"
 	"	jx	%1\n"
 	"	.previous\n"
 	"	.section __ex_table,\"a\"\n"
-	"	.long 1b,5b,2b,5b\n"
+	"	.long 1b,4b\n"
 	"	.previous\n"
-	: "+r" (ret), "=&r" (prev), "+m" (*uaddr)
-	: "r" (uaddr), "r" (oldval), "r" (newval), "I" (-EFAULT)
+	: "+r" (ret), "+r" (newval), "+m" (*uaddr), "+m" (*uval)
+	: "r" (uaddr), "r" (oldval), "r" (uval), "I" (-EFAULT)
 	: "memory");
 
-	*uval = prev;
 	return ret;
 }
 
diff --git a/arch/xtensa/include/asm/highmem.h b/arch/xtensa/include/asm/highmem.h
index 6e070db1022e..04e9340eac4b 100644
--- a/arch/xtensa/include/asm/highmem.h
+++ b/arch/xtensa/include/asm/highmem.h
@@ -72,7 +72,7 @@ static inline void *kmap(struct page *page)
 	 * page table.
 	 */
 	BUILD_BUG_ON(PKMAP_BASE <
-		     XCHAL_PAGE_TABLE_VADDR + XCHAL_PAGE_TABLE_SIZE);
+		     TLBTEMP_BASE_1 + TLBTEMP_SIZE);
 	BUG_ON(in_interrupt());
 	if (!PageHighMem(page))
 		return page_address(page);
diff --git a/arch/xtensa/include/asm/kasan.h b/arch/xtensa/include/asm/kasan.h
new file mode 100644
index 000000000000..216b6f32c375
--- /dev/null
+++ b/arch/xtensa/include/asm/kasan.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_KASAN_H
+#define __ASM_KASAN_H
+
+#ifndef __ASSEMBLY__
+
+#ifdef CONFIG_KASAN
+
+#include <linux/kernel.h>
+#include <linux/sizes.h>
+#include <asm/kmem_layout.h>
+
+#define KASAN_SHADOW_SCALE_SHIFT 3
+
+/* Start of area covered by KASAN */
+#define KASAN_START_VADDR __XTENSA_UL_CONST(0x90000000)
+/* Start of the shadow map */
+#define KASAN_SHADOW_START (XCHAL_PAGE_TABLE_VADDR + XCHAL_PAGE_TABLE_SIZE)
+/* Size of the shadow map */
+#define KASAN_SHADOW_SIZE (-KASAN_START_VADDR >> KASAN_SHADOW_SCALE_SHIFT)
+/* Offset for mem to shadow address transformation */
+#define KASAN_SHADOW_OFFSET __XTENSA_UL_CONST(CONFIG_KASAN_SHADOW_OFFSET)
+
+void __init kasan_early_init(void);
+void __init kasan_init(void);
+
+#else
+
+static inline void kasan_early_init(void)
+{
+}
+
+static inline void kasan_init(void)
+{
+}
+
+#endif
+#endif
+#endif
diff --git a/arch/xtensa/include/asm/kmem_layout.h b/arch/xtensa/include/asm/kmem_layout.h
index 561f8729bcde..2317c835a4db 100644
--- a/arch/xtensa/include/asm/kmem_layout.h
+++ b/arch/xtensa/include/asm/kmem_layout.h
@@ -71,4 +71,11 @@
 
 #endif
 
+#ifndef CONFIG_KASAN
+#define KERNEL_STACK_SHIFT	13
+#else
+#define KERNEL_STACK_SHIFT	15
+#endif
+#define KERNEL_STACK_SIZE	(1 << KERNEL_STACK_SHIFT)
+
 #endif
diff --git a/arch/xtensa/include/asm/linkage.h b/arch/xtensa/include/asm/linkage.h
new file mode 100644
index 000000000000..0ba9973235d9
--- /dev/null
+++ b/arch/xtensa/include/asm/linkage.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __ASM_LINKAGE_H
+#define __ASM_LINKAGE_H
+
+#define __ALIGN		.align 4
+#define __ALIGN_STR	".align 4"
+
+#endif
diff --git a/arch/xtensa/include/asm/mmu_context.h b/arch/xtensa/include/asm/mmu_context.h
index f7e186dfc4e4..de5e6cbbafe4 100644
--- a/arch/xtensa/include/asm/mmu_context.h
+++ b/arch/xtensa/include/asm/mmu_context.h
@@ -52,6 +52,7 @@ DECLARE_PER_CPU(unsigned long, asid_cache);
 #define ASID_INSERT(x)	(0x03020001 | (((x) & ASID_MASK) << 8))
 
 void init_mmu(void);
+void init_kio(void);
 
 static inline void set_rasid_register (unsigned long val)
 {
diff --git a/arch/xtensa/include/asm/nommu_context.h b/arch/xtensa/include/asm/nommu_context.h
index 2cebdbbdb633..37251b2ef871 100644
--- a/arch/xtensa/include/asm/nommu_context.h
+++ b/arch/xtensa/include/asm/nommu_context.h
@@ -3,6 +3,10 @@ static inline void init_mmu(void)
 {
 }
 
+static inline void init_kio(void)
+{
+}
+
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 }
diff --git a/arch/xtensa/include/asm/page.h b/arch/xtensa/include/asm/page.h
index 4ddbfd57a7c8..5d69c11c01b8 100644
--- a/arch/xtensa/include/asm/page.h
+++ b/arch/xtensa/include/asm/page.h
@@ -36,8 +36,6 @@
 #define MAX_LOW_PFN	PHYS_PFN(0xfffffffful)
 #endif
 
-#define PGTABLE_START	0x80000000
-
 /*
  * Cache aliasing:
  *
diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h
index 30dd5b2e4ad5..38802259978f 100644
--- a/arch/xtensa/include/asm/pgtable.h
+++ b/arch/xtensa/include/asm/pgtable.h
@@ -12,9 +12,9 @@
 #define _XTENSA_PGTABLE_H
 
 #define __ARCH_USE_5LEVEL_HACK
-#include <asm-generic/pgtable-nopmd.h>
 #include <asm/page.h>
 #include <asm/kmem_layout.h>
+#include <asm-generic/pgtable-nopmd.h>
 
 /*
  * We only use two ring levels, user and kernel space.
@@ -170,6 +170,7 @@
 #define PAGE_SHARED_EXEC \
 	__pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_WRITABLE | _PAGE_HW_EXEC)
 #define PAGE_KERNEL	   __pgprot(_PAGE_PRESENT | _PAGE_HW_WRITE)
+#define PAGE_KERNEL_RO	   __pgprot(_PAGE_PRESENT)
 #define PAGE_KERNEL_EXEC   __pgprot(_PAGE_PRESENT|_PAGE_HW_WRITE|_PAGE_HW_EXEC)
 
 #if (DCACHE_WAY_SIZE > PAGE_SIZE)
diff --git a/arch/xtensa/include/asm/ptrace.h b/arch/xtensa/include/asm/ptrace.h
index e2d9c5eb10bd..3a5c5918aea3 100644
--- a/arch/xtensa/include/asm/ptrace.h
+++ b/arch/xtensa/include/asm/ptrace.h
@@ -10,6 +10,7 @@
 #ifndef _XTENSA_PTRACE_H
 #define _XTENSA_PTRACE_H
 
+#include <asm/kmem_layout.h>
 #include <uapi/asm/ptrace.h>
 
 /*
@@ -38,20 +39,6 @@
  *		+-----------------------+ --------
  */
 
-#define KERNEL_STACK_SIZE (2 * PAGE_SIZE)
-
-/*  Offsets for exception_handlers[] (3 x 64-entries x 4-byte tables). */
-
-#define EXC_TABLE_KSTK		0x004	/* Kernel Stack */
-#define EXC_TABLE_DOUBLE_SAVE	0x008	/* Double exception save area for a0 */
-#define EXC_TABLE_FIXUP		0x00c	/* Fixup handler */
-#define EXC_TABLE_PARAM		0x010	/* For passing a parameter to fixup */
-#define EXC_TABLE_SYSCALL_SAVE	0x014	/* For fast syscall handler */
-#define EXC_TABLE_FAST_USER	0x100	/* Fast user exception handler */
-#define EXC_TABLE_FAST_KERNEL	0x200	/* Fast kernel exception handler */
-#define EXC_TABLE_DEFAULT	0x300	/* Default C-Handler */
-#define EXC_TABLE_SIZE		0x400
-
 #ifndef __ASSEMBLY__
 
 #include <asm/coprocessor.h>
diff --git a/arch/xtensa/include/asm/regs.h b/arch/xtensa/include/asm/regs.h
index 881a1134a4b4..477594e5817f 100644
--- a/arch/xtensa/include/asm/regs.h
+++ b/arch/xtensa/include/asm/regs.h
@@ -76,6 +76,7 @@
 #define EXCCAUSE_COPROCESSOR5_DISABLED		37
 #define EXCCAUSE_COPROCESSOR6_DISABLED		38
 #define EXCCAUSE_COPROCESSOR7_DISABLED		39
+#define EXCCAUSE_N				64
 
 /*  PS register fields.  */
 
diff --git a/arch/xtensa/include/asm/stackprotector.h b/arch/xtensa/include/asm/stackprotector.h
new file mode 100644
index 000000000000..e368f94fd2af
--- /dev/null
+++ b/arch/xtensa/include/asm/stackprotector.h
@@ -0,0 +1,40 @@
+/*
+ * GCC stack protector support.
+ *
+ * (This is directly adopted from the ARM implementation)
+ *
+ * Stack protector works by putting predefined pattern at the start of
+ * the stack frame and verifying that it hasn't been overwritten when
+ * returning from the function.  The pattern is called stack canary
+ * and gcc expects it to be defined by a global variable called
+ * "__stack_chk_guard" on Xtensa.  This unfortunately means that on SMP
+ * we cannot have a different canary value per task.
+ */
+
+#ifndef _ASM_STACKPROTECTOR_H
+#define _ASM_STACKPROTECTOR_H 1
+
+#include <linux/random.h>
+#include <linux/version.h>
+
+extern unsigned long __stack_chk_guard;
+
+/*
+ * Initialize the stackprotector canary value.
+ *
+ * NOTE: this must only be called from functions that never return,
+ * and it must always be inlined.
+ */
+static __always_inline void boot_init_stack_canary(void)
+{
+	unsigned long canary;
+
+	/* Try to get a semi random initial value. */
+	get_random_bytes(&canary, sizeof(canary));
+	canary ^= LINUX_VERSION_CODE;
+
+	current->stack_canary = canary;
+	__stack_chk_guard = current->stack_canary;
+}
+
+#endif	/* _ASM_STACKPROTECTOR_H */
diff --git a/arch/xtensa/include/asm/string.h b/arch/xtensa/include/asm/string.h
index 8d5d9dfadb09..89b51a0c752f 100644
--- a/arch/xtensa/include/asm/string.h
+++ b/arch/xtensa/include/asm/string.h
@@ -53,7 +53,7 @@ static inline char *strncpy(char *__dest, const char *__src, size_t __n)
 		"bne	%1, %5, 1b\n"
 		"2:"
 		: "=r" (__dest), "=r" (__src), "=&r" (__dummy)
-		: "0" (__dest), "1" (__src), "r" (__src+__n)
+		: "0" (__dest), "1" (__src), "r" ((uintptr_t)__src+__n)
 		: "memory");
 
 	return __xdest;
@@ -101,21 +101,40 @@ static inline int strncmp(const char *__cs, const char *__ct, size_t __n)
 		"2:\n\t"
 		"sub	%2, %2, %3"
 		: "=r" (__cs), "=r" (__ct), "=&r" (__res), "=&r" (__dummy)
-		: "0" (__cs), "1" (__ct), "r" (__cs+__n));
+		: "0" (__cs), "1" (__ct), "r" ((uintptr_t)__cs+__n));
 
 	return __res;
 }
 
 #define __HAVE_ARCH_MEMSET
 extern void *memset(void *__s, int __c, size_t __count);
+extern void *__memset(void *__s, int __c, size_t __count);
 
 #define __HAVE_ARCH_MEMCPY
 extern void *memcpy(void *__to, __const__ void *__from, size_t __n);
+extern void *__memcpy(void *__to, __const__ void *__from, size_t __n);
 
 #define __HAVE_ARCH_MEMMOVE
 extern void *memmove(void *__dest, __const__ void *__src, size_t __n);
+extern void *__memmove(void *__dest, __const__ void *__src, size_t __n);
 
 /* Don't build bcopy at all ...  */
 #define __HAVE_ARCH_BCOPY
 
+#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
+
+/*
+ * For files that are not instrumented (e.g. mm/slub.c) we
+ * should use not instrumented version of mem* functions.
+ */
+
+#define memcpy(dst, src, len) __memcpy(dst, src, len)
+#define memmove(dst, src, len) __memmove(dst, src, len)
+#define memset(s, c, n) __memset(s, c, n)
+
+#ifndef __NO_FORTIFY
+#define __NO_FORTIFY /* FORTIFY_SOURCE uses __builtin_memcpy, etc. */
+#endif
+#endif
+
 #endif	/* _XTENSA_STRING_H */
diff --git a/arch/xtensa/include/asm/thread_info.h b/arch/xtensa/include/asm/thread_info.h
index 7be2400f745a..2bd19ae61e47 100644
--- a/arch/xtensa/include/asm/thread_info.h
+++ b/arch/xtensa/include/asm/thread_info.h
@@ -11,7 +11,9 @@
 #ifndef _XTENSA_THREAD_INFO_H
 #define _XTENSA_THREAD_INFO_H
 
-#ifdef __KERNEL__
+#include <asm/kmem_layout.h>
+
+#define CURRENT_SHIFT KERNEL_STACK_SHIFT
 
 #ifndef __ASSEMBLY__
 # include <asm/processor.h>
@@ -77,14 +79,11 @@ struct thread_info {
 	.addr_limit	= KERNEL_DS,		\
 }
 
-#define init_thread_info	(init_thread_union.thread_info)
-#define init_stack		(init_thread_union.stack)
-
 /* how to get the thread information struct from C */
 static inline struct thread_info *current_thread_info(void)
 {
 	struct thread_info *ti;
-	 __asm__("extui %0,a1,0,13\n\t"
+	 __asm__("extui %0, a1, 0, "__stringify(CURRENT_SHIFT)"\n\t"
 	         "xor %0, a1, %0" : "=&r" (ti) : );
 	return ti;
 }
@@ -93,7 +92,7 @@ static inline struct thread_info *current_thread_info(void)
 
 /* how to get the thread information struct from ASM */
 #define GET_THREAD_INFO(reg,sp) \
-	extui reg, sp, 0, 13; \
+	extui reg, sp, 0, CURRENT_SHIFT; \
 	xor   reg, sp, reg
 #endif
 
@@ -130,8 +129,7 @@ static inline struct thread_info *current_thread_info(void)
  */
 #define TS_USEDFPU		0x0001	/* FPU was used by this task this quantum (SMP) */
 
-#define THREAD_SIZE 8192	//(2*PAGE_SIZE)
-#define THREAD_SIZE_ORDER 1
+#define THREAD_SIZE KERNEL_STACK_SIZE
+#define THREAD_SIZE_ORDER (KERNEL_STACK_SHIFT - PAGE_SHIFT)
 
-#endif	/* __KERNEL__ */
 #endif	/* _XTENSA_THREAD_INFO */
diff --git a/arch/xtensa/include/asm/traps.h b/arch/xtensa/include/asm/traps.h
index 2e69aa4b843f..f5cd7a7e65e0 100644
--- a/arch/xtensa/include/asm/traps.h
+++ b/arch/xtensa/include/asm/traps.h
@@ -12,6 +12,29 @@
 
 #include <asm/ptrace.h>
 
+/*
+ * Per-CPU exception handling data structure.
+ * EXCSAVE1 points to it.
+ */
+struct exc_table {
+	/* Kernel Stack */
+	void *kstk;
+	/* Double exception save area for a0 */
+	unsigned long double_save;
+	/* Fixup handler */
+	void *fixup;
+	/* For passing a parameter to fixup */
+	void *fixup_param;
+	/* For fast syscall handler */
+	unsigned long syscall_save;
+	/* Fast user exception handlers */
+	void *fast_user_handler[EXCCAUSE_N];
+	/* Fast kernel exception handlers */
+	void *fast_kernel_handler[EXCCAUSE_N];
+	/* Default C-Handlers */
+	void *default_handler[EXCCAUSE_N];
+};
+
 /*
  * handler must be either of the following:
  *  void (*)(struct pt_regs *regs);
@@ -19,6 +42,18 @@
  */
 extern void * __init trap_set_handler(int cause, void *handler);
 extern void do_unhandled(struct pt_regs *regs, unsigned long exccause);
+void fast_second_level_miss(void);
+
+/* Initialize minimal exc_table structure sufficient for basic paging */
+static inline void __init early_trap_init(void)
+{
+	static struct exc_table exc_table __initdata = {
+		.fast_kernel_handler[EXCCAUSE_DTLB_MISS] =
+			fast_second_level_miss,
+	};
+	__asm__ __volatile__("wsr  %0, excsave1\n" : : "a" (&exc_table));
+}
+
 void secondary_trap_init(void);
 
 static inline void spill_registers(void)
diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h
index b8f152b6aaa5..f1158b4c629c 100644
--- a/arch/xtensa/include/asm/uaccess.h
+++ b/arch/xtensa/include/asm/uaccess.h
@@ -44,6 +44,8 @@
 #define __access_ok(addr, size) (__kernel_ok || __user_ok((addr), (size)))
 #define access_ok(type, addr, size) __access_ok((unsigned long)(addr), (size))
 
+#define user_addr_max() (uaccess_kernel() ? ~0UL : TASK_SIZE)
+
 /*
  * These are the main single-value transfer routines.  They
  * automatically use the right size if we just have the right pointer
@@ -261,7 +263,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 static inline unsigned long
 __xtensa_clear_user(void *addr, unsigned long size)
 {
-	if ( ! memset(addr, 0, size) )
+	if (!__memset(addr, 0, size))
 		return size;
 	return 0;
 }
@@ -277,6 +279,8 @@ clear_user(void *addr, unsigned long size)
 #define __clear_user  __xtensa_clear_user
 
 
+#ifndef CONFIG_GENERIC_STRNCPY_FROM_USER
+
 extern long __strncpy_user(char *, const char *, long);
 
 static inline long
@@ -286,6 +290,9 @@ strncpy_from_user(char *dst, const char *src, long count)
 		return __strncpy_user(dst, src, count);
 	return -EFAULT;
 }
+#else
+long strncpy_from_user(char *dst, const char *src, long count);
+#endif
 
 /*
  * Return the size of a string (including the ending 0!)
diff --git a/arch/xtensa/kernel/Makefile b/arch/xtensa/kernel/Makefile
index bb8d55775a97..91907590d183 100644
--- a/arch/xtensa/kernel/Makefile
+++ b/arch/xtensa/kernel/Makefile
@@ -17,9 +17,6 @@ obj-$(CONFIG_XTENSA_VARIANT_HAVE_PERF_EVENTS) += perf_event.o
 obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
 obj-$(CONFIG_S32C1I_SELFTEST) += s32c1i_selftest.o
 
-AFLAGS_head.o += -mtext-section-literals
-AFLAGS_mxhead.o += -mtext-section-literals
-
 # In the Xtensa architecture, assembly generates literals which must always
 # precede the L32R instruction with a relative offset less than 256 kB.
 # Therefore, the .text and .literal section must be combined in parenthesis
diff --git a/arch/xtensa/kernel/align.S b/arch/xtensa/kernel/align.S
index 890004af03a9..9301452e521e 100644
--- a/arch/xtensa/kernel/align.S
+++ b/arch/xtensa/kernel/align.S
@@ -19,6 +19,7 @@
 #include <linux/linkage.h>
 #include <asm/current.h>
 #include <asm/asm-offsets.h>
+#include <asm/asmmacro.h>
 #include <asm/processor.h>
 
 #if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION
@@ -66,8 +67,6 @@
 #define	INSN_T		24
 #define	INSN_OP1	16
 
-.macro __src_b	r, w0, w1;	src	\r, \w0, \w1;	.endm
-.macro __ssa8	r;		ssa8b	\r;		.endm
 .macro __ssa8r	r;		ssa8l	\r;		.endm
 .macro __sh	r, s;		srl	\r, \s;		.endm
 .macro __sl	r, s;		sll	\r, \s;		.endm
@@ -81,8 +80,6 @@
 #define	INSN_T		4
 #define	INSN_OP1	12
 
-.macro __src_b	r, w0, w1;	src	\r, \w1, \w0;	.endm
-.macro __ssa8	r;		ssa8l	\r;		.endm
 .macro __ssa8r	r;		ssa8b	\r;		.endm
 .macro __sh	r, s;		sll	\r, \s;		.endm
 .macro __sl	r, s;		srl	\r, \s;		.endm
@@ -155,7 +152,7 @@
  *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
  */
 
-
+	.literal_position
 ENTRY(fast_unaligned)
 
 	/* Note: We don't expect the address to be aligned on a word
diff --git a/arch/xtensa/kernel/asm-offsets.c b/arch/xtensa/kernel/asm-offsets.c
index bcb5beb81177..022cf918ec20 100644
--- a/arch/xtensa/kernel/asm-offsets.c
+++ b/arch/xtensa/kernel/asm-offsets.c
@@ -76,6 +76,9 @@ int main(void)
 	DEFINE(TASK_PID, offsetof (struct task_struct, pid));
 	DEFINE(TASK_THREAD, offsetof (struct task_struct, thread));
 	DEFINE(TASK_THREAD_INFO, offsetof (struct task_struct, stack));
+#ifdef CONFIG_CC_STACKPROTECTOR
+	DEFINE(TASK_STACK_CANARY, offsetof(struct task_struct, stack_canary));
+#endif
 	DEFINE(TASK_STRUCT_SIZE, sizeof (struct task_struct));
 
 	/* offsets in thread_info struct */
@@ -129,5 +132,18 @@ int main(void)
 	       offsetof(struct debug_table, icount_level_save));
 #endif
 
+	/* struct exc_table */
+	DEFINE(EXC_TABLE_KSTK, offsetof(struct exc_table, kstk));
+	DEFINE(EXC_TABLE_DOUBLE_SAVE, offsetof(struct exc_table, double_save));
+	DEFINE(EXC_TABLE_FIXUP, offsetof(struct exc_table, fixup));
+	DEFINE(EXC_TABLE_PARAM, offsetof(struct exc_table, fixup_param));
+	DEFINE(EXC_TABLE_SYSCALL_SAVE,
+	       offsetof(struct exc_table, syscall_save));
+	DEFINE(EXC_TABLE_FAST_USER,
+	       offsetof(struct exc_table, fast_user_handler));
+	DEFINE(EXC_TABLE_FAST_KERNEL,
+	       offsetof(struct exc_table, fast_kernel_handler));
+	DEFINE(EXC_TABLE_DEFAULT, offsetof(struct exc_table, default_handler));
+
 	return 0;
 }
diff --git a/arch/xtensa/kernel/coprocessor.S b/arch/xtensa/kernel/coprocessor.S
index 3a98503ad11a..4f8b52d575a2 100644
--- a/arch/xtensa/kernel/coprocessor.S
+++ b/arch/xtensa/kernel/coprocessor.S
@@ -212,8 +212,7 @@ ENDPROC(coprocessor_restore)
 ENTRY(fast_coprocessor_double)
 
 	wsr	a0, excsave1
-	movi	a0, unrecoverable_exception
-	callx0	a0
+	call0	unrecoverable_exception
 
 ENDPROC(fast_coprocessor_double)
 
diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S
index 37a239556889..5caff0744f3c 100644
--- a/arch/xtensa/kernel/entry.S
+++ b/arch/xtensa/kernel/entry.S
@@ -14,6 +14,7 @@
 
 #include <linux/linkage.h>
 #include <asm/asm-offsets.h>
+#include <asm/asmmacro.h>
 #include <asm/processor.h>
 #include <asm/coprocessor.h>
 #include <asm/thread_info.h>
@@ -125,6 +126,7 @@
  *
  * Note: _user_exception might be at an odd address. Don't use call0..call12
  */
+	.literal_position
 
 ENTRY(user_exception)
 
@@ -475,8 +477,7 @@ common_exception_return:
 1:
 	irq_save a2, a3
 #ifdef CONFIG_TRACE_IRQFLAGS
-	movi	a4, trace_hardirqs_off
-	callx4	a4
+	call4	trace_hardirqs_off
 #endif
 
 	/* Jump if we are returning from kernel exceptions. */
@@ -503,24 +504,20 @@ common_exception_return:
 	/* Call do_signal() */
 
 #ifdef CONFIG_TRACE_IRQFLAGS
-	movi	a4, trace_hardirqs_on
-	callx4	a4
+	call4	trace_hardirqs_on
 #endif
 	rsil	a2, 0
-	movi	a4, do_notify_resume	# int do_notify_resume(struct pt_regs*)
 	mov	a6, a1
-	callx4	a4
+	call4	do_notify_resume	# int do_notify_resume(struct pt_regs*)
 	j	1b
 
 3:	/* Reschedule */
 
 #ifdef CONFIG_TRACE_IRQFLAGS
-	movi	a4, trace_hardirqs_on
-	callx4	a4
+	call4	trace_hardirqs_on
 #endif
 	rsil	a2, 0
-	movi	a4, schedule	# void schedule (void)
-	callx4	a4
+	call4	schedule	# void schedule (void)
 	j	1b
 
 #ifdef CONFIG_PREEMPT
@@ -531,8 +528,7 @@ common_exception_return:
 
 	l32i	a4, a2, TI_PRE_COUNT
 	bnez	a4, 4f
-	movi	a4, preempt_schedule_irq
-	callx4	a4
+	call4	preempt_schedule_irq
 	j	1b
 #endif
 
@@ -545,23 +541,20 @@ common_exception_return:
 5:
 #ifdef CONFIG_HAVE_HW_BREAKPOINT
 	_bbci.l	a4, TIF_DB_DISABLED, 7f
-	movi	a4, restore_dbreak
-	callx4	a4
+	call4	restore_dbreak
 7:
 #endif
 #ifdef CONFIG_DEBUG_TLB_SANITY
 	l32i	a4, a1, PT_DEPC
 	bgeui	a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
-	movi	a4, check_tlb_sanity
-	callx4	a4
+	call4	check_tlb_sanity
 #endif
 6:
 4:
 #ifdef CONFIG_TRACE_IRQFLAGS
 	extui	a4, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH
 	bgei	a4, LOCKLEVEL, 1f
-	movi	a4, trace_hardirqs_on
-	callx4	a4
+	call4	trace_hardirqs_on
 1:
 #endif
 	/* Restore optional registers. */
@@ -777,6 +770,8 @@ ENDPROC(kernel_exception)
  * When we get here,  a0 is trashed and saved to excsave[debuglevel]
  */
 
+	.literal_position
+
 ENTRY(debug_exception)
 
 	rsr	a0, SREG_EPS + XCHAL_DEBUGLEVEL
@@ -916,6 +911,8 @@ ENDPROC(debug_exception)
 unrecoverable_text:
 	.ascii "Unrecoverable error in exception handler\0"
 
+	.literal_position
+
 ENTRY(unrecoverable_exception)
 
 	movi	a0, 1
@@ -933,10 +930,8 @@ ENTRY(unrecoverable_exception)
 	movi	a0, 0
 	addi	a1, a1, PT_REGS_OFFSET
 
-	movi	a4, panic
 	movi	a6, unrecoverable_text
-
-	callx4	a4
+	call4	panic
 
 1:	j	1b
 
@@ -1073,8 +1068,7 @@ ENTRY(fast_syscall_unrecoverable)
 	xsr     a2, depc                # restore a2, depc
 
 	wsr     a0, excsave1
-	movi    a0, unrecoverable_exception
-	callx0  a0
+	call0	unrecoverable_exception
 
 ENDPROC(fast_syscall_unrecoverable)
 
@@ -1101,32 +1095,11 @@ ENDPROC(fast_syscall_unrecoverable)
  *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
  *
  * Note: we don't have to save a2; a2 holds the return value
- *
- * We use the two macros TRY and CATCH:
- *
- * TRY	 adds an entry to the __ex_table fixup table for the immediately
- *	 following instruction.
- *
- * CATCH catches any exception that occurred at one of the preceding TRY
- *       statements and continues from there
- *
- * Usage TRY	l32i	a0, a1, 0
- *		<other code>
- *	 done:	rfe
- *	 CATCH	<set return code>
- *		j done
  */
 
-#ifdef CONFIG_FAST_SYSCALL_XTENSA
-
-#define TRY								\
-	.section __ex_table, "a";					\
-	.word	66f, 67f;						\
-	.text;								\
-66:
+	.literal_position
 
-#define CATCH								\
-67:
+#ifdef CONFIG_FAST_SYSCALL_XTENSA
 
 ENTRY(fast_syscall_xtensa)
 
@@ -1141,9 +1114,9 @@ ENTRY(fast_syscall_xtensa)
 
 .Lswp:	/* Atomic compare and swap */
 
-TRY	l32i	a0, a3, 0		# read old value
+EX(.Leac) l32i	a0, a3, 0		# read old value
 	bne	a0, a4, 1f		# same as old value? jump
-TRY	s32i	a5, a3, 0		# different, modify value
+EX(.Leac) s32i	a5, a3, 0		# different, modify value
 	l32i	a7, a2, PT_AREG7	# restore a7
 	l32i	a0, a2, PT_AREG0	# restore a0
 	movi	a2, 1			# and return 1
@@ -1156,12 +1129,12 @@ TRY	s32i	a5, a3, 0		# different, modify value
 
 .Lnswp:	/* Atomic set, add, and exg_add. */
 
-TRY	l32i	a7, a3, 0		# orig
+EX(.Leac) l32i	a7, a3, 0		# orig
 	addi	a6, a6, -SYS_XTENSA_ATOMIC_SET
 	add	a0, a4, a7		# + arg
 	moveqz	a0, a4, a6		# set
 	addi	a6, a6, SYS_XTENSA_ATOMIC_SET
-TRY	s32i	a0, a3, 0		# write new value
+EX(.Leac) s32i	a0, a3, 0		# write new value
 
 	mov	a0, a2
 	mov	a2, a7
@@ -1169,7 +1142,6 @@ TRY	s32i	a0, a3, 0		# write new value
 	l32i	a0, a0, PT_AREG0	# restore a0
 	rfe
 
-CATCH
 .Leac:	l32i	a7, a2, PT_AREG7	# restore a7
 	l32i	a0, a2, PT_AREG0	# restore a0
 	movi	a2, -EFAULT
@@ -1411,14 +1383,12 @@ ENTRY(fast_syscall_spill_registers)
 	rsync
 
 	movi	a6, SIGSEGV
-	movi	a4, do_exit
-	callx4	a4
+	call4	do_exit
 
 	/* shouldn't return, so panic */
 
 	wsr	a0, excsave1
-	movi	a0, unrecoverable_exception
-	callx0	a0		# should not return
+	call0	unrecoverable_exception		# should not return
 1:	j	1b
 
 
@@ -1564,8 +1534,8 @@ ENDPROC(fast_syscall_spill_registers)
 
 ENTRY(fast_second_level_miss_double_kernel)
 
-1:	movi	a0, unrecoverable_exception
-	callx0	a0		# should not return
+1:
+	call0	unrecoverable_exception		# should not return
 1:	j	1b
 
 ENDPROC(fast_second_level_miss_double_kernel)
@@ -1887,6 +1857,7 @@ ENDPROC(fast_store_prohibited)
  * void system_call (struct pt_regs* regs, int exccause)
  *                            a2                 a3
  */
+	.literal_position
 
 ENTRY(system_call)
 
@@ -1896,9 +1867,8 @@ ENTRY(system_call)
 
 	l32i	a3, a2, PT_AREG2
 	mov	a6, a2
-	movi	a4, do_syscall_trace_enter
 	s32i	a3, a2, PT_SYSCALL
-	callx4	a4
+	call4	do_syscall_trace_enter
 	mov	a3, a6
 
 	/* syscall = sys_call_table[syscall_nr] */
@@ -1930,9 +1900,8 @@ ENTRY(system_call)
 1:	/* regs->areg[2] = return_value */
 
 	s32i	a6, a2, PT_AREG2
-	movi	a4, do_syscall_trace_leave
 	mov	a6, a2
-	callx4	a4
+	call4	do_syscall_trace_leave
 	retw
 
 ENDPROC(system_call)
@@ -2002,6 +1971,12 @@ ENTRY(_switch_to)
 	s32i	a1, a2, THREAD_SP	# save stack pointer
 #endif
 
+#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP)
+	movi	a6, __stack_chk_guard
+	l32i	a8, a3, TASK_STACK_CANARY
+	s32i	a8, a6, 0
+#endif
+
 	/* Disable ints while we manipulate the stack pointer. */
 
 	irq_save a14, a3
@@ -2048,12 +2023,10 @@ ENTRY(ret_from_fork)
 	/* void schedule_tail (struct task_struct *prev)
 	 * Note: prev is still in a6 (return value from fake call4 frame)
 	 */
-	movi	a4, schedule_tail
-	callx4	a4
+	call4	schedule_tail
 
-	movi	a4, do_syscall_trace_leave
 	mov	a6, a1
-	callx4	a4
+	call4	do_syscall_trace_leave
 
 	j	common_exception_return
 
diff --git a/arch/xtensa/kernel/head.S b/arch/xtensa/kernel/head.S
index 23ce62e60435..9c4e9433e536 100644
--- a/arch/xtensa/kernel/head.S
+++ b/arch/xtensa/kernel/head.S
@@ -264,11 +264,8 @@ ENTRY(_startup)
 
 	/* init_arch kick-starts the linux kernel */
 
-	movi	a4, init_arch
-	callx4	a4
-
-	movi	a4, start_kernel
-	callx4	a4
+	call4	init_arch
+	call4	start_kernel
 
 should_never_return:
 	j	should_never_return
@@ -294,8 +291,7 @@ should_never_return:
 	movi	a6, 0
 	wsr	a6, excsave1
 
-	movi	a4, secondary_start_kernel
-	callx4	a4
+	call4	secondary_start_kernel
 	j	should_never_return
 
 #endif  /* CONFIG_SMP */
diff --git a/arch/xtensa/kernel/module.c b/arch/xtensa/kernel/module.c
index b715237bae61..902845ddacb7 100644
--- a/arch/xtensa/kernel/module.c
+++ b/arch/xtensa/kernel/module.c
@@ -22,8 +22,6 @@
 #include <linux/kernel.h>
 #include <linux/cache.h>
 
-#undef DEBUG_RELOCATE
-
 static int
 decode_calln_opcode (unsigned char *location)
 {
@@ -58,10 +56,9 @@ int apply_relocate_add(Elf32_Shdr *sechdrs,
 	unsigned char *location;
 	uint32_t value;
 
-#ifdef DEBUG_RELOCATE
-	printk("Applying relocate section %u to %u\n", relsec,
-	       sechdrs[relsec].sh_info);
-#endif
+	pr_debug("Applying relocate section %u to %u\n", relsec,
+		 sechdrs[relsec].sh_info);
+
 	for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rela); i++) {
 		location = (char *)sechdrs[sechdrs[relsec].sh_info].sh_addr
 			+ rela[i].r_offset;
@@ -87,7 +84,7 @@ int apply_relocate_add(Elf32_Shdr *sechdrs,
 				value -= ((unsigned long)location & -4) + 4;
 				if ((value & 3) != 0 ||
 				    ((value + (1 << 19)) >> 20) != 0) {
-					printk("%s: relocation out of range, "
+					pr_err("%s: relocation out of range, "
 					       "section %d reloc %d "
 					       "sym '%s'\n",
 					       mod->name, relsec, i,
@@ -111,7 +108,7 @@ int apply_relocate_add(Elf32_Shdr *sechdrs,
 				value -= (((unsigned long)location + 3) & -4);
 				if ((value & 3) != 0 ||
 				    (signed int)value >> 18 != -1) {
-					printk("%s: relocation out of range, "
+					pr_err("%s: relocation out of range, "
 					       "section %d reloc %d "
 					       "sym '%s'\n",
 					       mod->name, relsec, i,
@@ -156,7 +153,7 @@ int apply_relocate_add(Elf32_Shdr *sechdrs,
 		case R_XTENSA_SLOT12_OP:
 		case R_XTENSA_SLOT13_OP:
 		case R_XTENSA_SLOT14_OP:
-			printk("%s: unexpected FLIX relocation: %u\n",
+			pr_err("%s: unexpected FLIX relocation: %u\n",
 			       mod->name,
 			       ELF32_R_TYPE(rela[i].r_info));
 			return -ENOEXEC;
@@ -176,13 +173,13 @@ int apply_relocate_add(Elf32_Shdr *sechdrs,
 		case R_XTENSA_SLOT12_ALT:
 		case R_XTENSA_SLOT13_ALT:
 		case R_XTENSA_SLOT14_ALT:
-			printk("%s: unexpected ALT relocation: %u\n",
+			pr_err("%s: unexpected ALT relocation: %u\n",
 			       mod->name,
 			       ELF32_R_TYPE(rela[i].r_info));
 			return -ENOEXEC;
 
 		default:
-			printk("%s: unexpected relocation: %u\n",
+			pr_err("%s: unexpected relocation: %u\n",
 			       mod->name,
 			       ELF32_R_TYPE(rela[i].r_info));
 			return -ENOEXEC;
diff --git a/arch/xtensa/kernel/pci-dma.c b/arch/xtensa/kernel/pci-dma.c
index 623720a11143..732631ce250f 100644
--- a/arch/xtensa/kernel/pci-dma.c
+++ b/arch/xtensa/kernel/pci-dma.c
@@ -16,6 +16,7 @@
  */
 
 #include <linux/dma-contiguous.h>
+#include <linux/dma-direct.h>
 #include <linux/gfp.h>
 #include <linux/highmem.h>
 #include <linux/mm.h>
@@ -123,7 +124,7 @@ static void *xtensa_dma_alloc(struct device *dev, size_t size,
 			      unsigned long attrs)
 {
 	unsigned long ret;
-	unsigned long uncached = 0;
+	unsigned long uncached;
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 	struct page *page = NULL;
 
@@ -144,15 +145,27 @@ static void *xtensa_dma_alloc(struct device *dev, size_t size,
 	if (!page)
 		return NULL;
 
-	ret = (unsigned long)page_address(page);
+	*handle = phys_to_dma(dev, page_to_phys(page));
 
-	/* We currently don't support coherent memory outside KSEG */
+#ifdef CONFIG_MMU
+	if (PageHighMem(page)) {
+		void *p;
 
+		p = dma_common_contiguous_remap(page, size, VM_MAP,
+						pgprot_noncached(PAGE_KERNEL),
+						__builtin_return_address(0));
+		if (!p) {
+			if (!dma_release_from_contiguous(dev, page, count))
+				__free_pages(page, get_order(size));
+		}
+		return p;
+	}
+#endif
+	ret = (unsigned long)page_address(page);
 	BUG_ON(ret < XCHAL_KSEG_CACHED_VADDR ||
 	       ret > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
 
 	uncached = ret + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR;
-	*handle = virt_to_bus((void *)ret);
 	__invalidate_dcache_range(ret, size);
 
 	return (void *)uncached;
@@ -161,13 +174,20 @@ static void *xtensa_dma_alloc(struct device *dev, size_t size,
 static void xtensa_dma_free(struct device *dev, size_t size, void *vaddr,
 			    dma_addr_t dma_handle, unsigned long attrs)
 {
-	unsigned long addr = (unsigned long)vaddr +
-		XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR;
-	struct page *page = virt_to_page(addr);
 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
-
-	BUG_ON(addr < XCHAL_KSEG_CACHED_VADDR ||
-	       addr > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
+	unsigned long addr = (unsigned long)vaddr;
+	struct page *page;
+
+	if (addr >= XCHAL_KSEG_BYPASS_VADDR &&
+	    addr - XCHAL_KSEG_BYPASS_VADDR < XCHAL_KSEG_SIZE) {
+		addr += XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR;
+		page = virt_to_page(addr);
+	} else {
+#ifdef CONFIG_MMU
+		dma_common_free_remap(vaddr, size, VM_MAP);
+#endif
+		page = pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_handle)));
+	}
 
 	if (!dma_release_from_contiguous(dev, page, count))
 		__free_pages(page, get_order(size));
diff --git a/arch/xtensa/kernel/pci.c b/arch/xtensa/kernel/pci.c
index 903963ee495d..d981f01c8d89 100644
--- a/arch/xtensa/kernel/pci.c
+++ b/arch/xtensa/kernel/pci.c
@@ -29,14 +29,6 @@
 #include <asm/pci-bridge.h>
 #include <asm/platform.h>
 
-#undef DEBUG
-
-#ifdef DEBUG
-#define DBG(x...) printk(x)
-#else
-#define DBG(x...)
-#endif
-
 /* PCI Controller */
 
 
@@ -101,8 +93,8 @@ pcibios_enable_resources(struct pci_dev *dev, int mask)
 	for(idx=0; idx<6; idx++) {
 		r = &dev->resource[idx];
 		if (!r->start && r->end) {
-			printk (KERN_ERR "PCI: Device %s not available because "
-				"of resource collisions\n", pci_name(dev));
+			pr_err("PCI: Device %s not available because "
+			       "of resource collisions\n", pci_name(dev));
 			return -EINVAL;
 		}
 		if (r->flags & IORESOURCE_IO)
@@ -113,7 +105,7 @@ pcibios_enable_resources(struct pci_dev *dev, int mask)
 	if (dev->resource[PCI_ROM_RESOURCE].start)
 		cmd |= PCI_COMMAND_MEMORY;
 	if (cmd != old_cmd) {
-		printk("PCI: Enabling device %s (%04x -> %04x)\n",
+		pr_info("PCI: Enabling device %s (%04x -> %04x)\n",
 			pci_name(dev), old_cmd, cmd);
 		pci_write_config_word(dev, PCI_COMMAND, cmd);
 	}
@@ -144,8 +136,8 @@ static void __init pci_controller_apertures(struct pci_controller *pci_ctrl,
 	res = &pci_ctrl->io_resource;
 	if (!res->flags) {
 		if (io_offset)
-			printk (KERN_ERR "I/O resource not set for host"
-				" bridge %d\n", pci_ctrl->index);
+			pr_err("I/O resource not set for host bridge %d\n",
+			       pci_ctrl->index);
 		res->start = 0;
 		res->end = IO_SPACE_LIMIT;
 		res->flags = IORESOURCE_IO;
@@ -159,8 +151,8 @@ static void __init pci_controller_apertures(struct pci_controller *pci_ctrl,
 		if (!res->flags) {
 			if (i > 0)
 				continue;
-			printk(KERN_ERR "Memory resource not set for "
-			       "host bridge %d\n", pci_ctrl->index);
+			pr_err("Memory resource not set for host bridge %d\n",
+			       pci_ctrl->index);
 			res->start = 0;
 			res->end = ~0U;
 			res->flags = IORESOURCE_MEM;
@@ -176,7 +168,7 @@ static int __init pcibios_init(void)
 	struct pci_bus *bus;
 	int next_busno = 0, ret;
 
-	printk("PCI: Probing PCI hardware\n");
+	pr_info("PCI: Probing PCI hardware\n");
 
 	/* Scan all of the recorded PCI controllers.  */
 	for (pci_ctrl = pci_ctrl_head; pci_ctrl; pci_ctrl = pci_ctrl->next) {
@@ -232,7 +224,7 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
 	for (idx=0; idx<6; idx++) {
 		r = &dev->resource[idx];
 		if (!r->start && r->end) {
-			printk(KERN_ERR "PCI: Device %s not available because "
+			pr_err("PCI: Device %s not available because "
 			       "of resource collisions\n", pci_name(dev));
 			return -EINVAL;
 		}
@@ -242,8 +234,8 @@ int pcibios_enable_device(struct pci_dev *dev, int mask)
 			cmd |= PCI_COMMAND_MEMORY;
 	}
 	if (cmd != old_cmd) {
-		printk("PCI: Enabling device %s (%04x -> %04x)\n",
-		       pci_name(dev), old_cmd, cmd);
+		pr_info("PCI: Enabling device %s (%04x -> %04x)\n",
+			pci_name(dev), old_cmd, cmd);
 		pci_write_config_word(dev, PCI_COMMAND, cmd);
 	}
 
diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c
index ff4f0ecb03dd..8dd0593fb2c4 100644
--- a/arch/xtensa/kernel/process.c
+++ b/arch/xtensa/kernel/process.c
@@ -58,6 +58,12 @@ void (*pm_power_off)(void) = NULL;
 EXPORT_SYMBOL(pm_power_off);
 
 
+#ifdef CONFIG_CC_STACKPROTECTOR
+#include <linux/stackprotector.h>
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
 #if XTENSA_HAVE_COPROCESSORS
 
 void coprocessor_release_all(struct thread_info *ti)
diff --git a/arch/xtensa/kernel/ptrace.c b/arch/xtensa/kernel/ptrace.c
index e2461968efb2..c0845cb1cbb9 100644
--- a/arch/xtensa/kernel/ptrace.c
+++ b/arch/xtensa/kernel/ptrace.c
@@ -278,7 +278,6 @@ static void ptrace_hbptriggered(struct perf_event *bp,
 				struct pt_regs *regs)
 {
 	int i;
-	siginfo_t info;
 	struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
 
 	if (bp->attr.bp_type & HW_BREAKPOINT_X) {
@@ -293,12 +292,7 @@ static void ptrace_hbptriggered(struct perf_event *bp,
 		i = (i << 1) | 1;
 	}
 
-	info.si_signo = SIGTRAP;
-	info.si_errno = i;
-	info.si_code = TRAP_HWBKPT;
-	info.si_addr = (void __user *)bkpt->address;
-
-	force_sig_info(SIGTRAP, &info, current);
+	force_sig_ptrace_errno_trap(i, (void __user *)bkpt->address);
 }
 
 static struct perf_event *ptrace_hbp_create(struct task_struct *tsk, int type)
diff --git a/arch/xtensa/kernel/setup.c b/arch/xtensa/kernel/setup.c
index 08175df7a69e..686a27444bba 100644
--- a/arch/xtensa/kernel/setup.c
+++ b/arch/xtensa/kernel/setup.c
@@ -20,7 +20,6 @@
 #include <linux/mm.h>
 #include <linux/proc_fs.h>
 #include <linux/screen_info.h>
-#include <linux/bootmem.h>
 #include <linux/kernel.h>
 #include <linux/percpu.h>
 #include <linux/cpu.h>
@@ -36,6 +35,7 @@
 #endif
 
 #include <asm/bootparam.h>
+#include <asm/kasan.h>
 #include <asm/mmu_context.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
@@ -156,7 +156,7 @@ static int __init parse_bootparam(const bp_tag_t* tag)
 	/* Boot parameters must start with a BP_TAG_FIRST tag. */
 
 	if (tag->id != BP_TAG_FIRST) {
-		printk(KERN_WARNING "Invalid boot parameters!\n");
+		pr_warn("Invalid boot parameters!\n");
 		return 0;
 	}
 
@@ -165,15 +165,14 @@ static int __init parse_bootparam(const bp_tag_t* tag)
 	/* Parse all tags. */
 
 	while (tag != NULL && tag->id != BP_TAG_LAST) {
-	 	for (t = &__tagtable_begin; t < &__tagtable_end; t++) {
+		for (t = &__tagtable_begin; t < &__tagtable_end; t++) {
 			if (tag->id == t->tag) {
 				t->parse(tag);
 				break;
 			}
 		}
 		if (t == &__tagtable_end)
-			printk(KERN_WARNING "Ignoring tag "
-			       "0x%08x\n", tag->id);
+			pr_warn("Ignoring tag 0x%08x\n", tag->id);
 		tag = (bp_tag_t*)((unsigned long)(tag + 1) + tag->size);
 	}
 
@@ -208,6 +207,8 @@ static int __init xtensa_dt_io_area(unsigned long node, const char *uname,
 	/* round down to nearest 256MB boundary */
 	xtensa_kio_paddr &= 0xf0000000;
 
+	init_kio();
+
 	return 1;
 }
 #else
@@ -218,17 +219,6 @@ static int __init xtensa_dt_io_area(unsigned long node, const char *uname,
 }
 #endif
 
-void __init early_init_dt_add_memory_arch(u64 base, u64 size)
-{
-	size &= PAGE_MASK;
-	memblock_add(base, size);
-}
-
-void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
-{
-	return __alloc_bootmem(size, align, 0);
-}
-
 void __init early_init_devtree(void *params)
 {
 	early_init_dt_scan(params);
@@ -246,6 +236,14 @@ void __init early_init_devtree(void *params)
 
 void __init init_arch(bp_tag_t *bp_start)
 {
+	/* Initialize MMU. */
+
+	init_mmu();
+
+	/* Initialize initial KASAN shadow map */
+
+	kasan_early_init();
+
 	/* Parse boot parameters */
 
 	if (bp_start)
@@ -263,10 +261,6 @@ void __init init_arch(bp_tag_t *bp_start)
 	/* Early hook for platforms */
 
 	platform_init(bp_start);
-
-	/* Initialize MMU. */
-
-	init_mmu();
 }
 
 /*
@@ -277,13 +271,13 @@ extern char _end[];
 extern char _stext[];
 extern char _WindowVectors_text_start;
 extern char _WindowVectors_text_end;
-extern char _DebugInterruptVector_literal_start;
+extern char _DebugInterruptVector_text_start;
 extern char _DebugInterruptVector_text_end;
-extern char _KernelExceptionVector_literal_start;
+extern char _KernelExceptionVector_text_start;
 extern char _KernelExceptionVector_text_end;
-extern char _UserExceptionVector_literal_start;
+extern char _UserExceptionVector_text_start;
 extern char _UserExceptionVector_text_end;
-extern char _DoubleExceptionVector_literal_start;
+extern char _DoubleExceptionVector_text_start;
 extern char _DoubleExceptionVector_text_end;
 #if XCHAL_EXCM_LEVEL >= 2
 extern char _Level2InterruptVector_text_start;
@@ -317,6 +311,13 @@ static inline int mem_reserve(unsigned long start, unsigned long end)
 
 void __init setup_arch(char **cmdline_p)
 {
+	pr_info("config ID: %08x:%08x\n",
+		get_sr(SREG_EPC), get_sr(SREG_EXCSAVE));
+	if (get_sr(SREG_EPC) != XCHAL_HW_CONFIGID0 ||
+	    get_sr(SREG_EXCSAVE) != XCHAL_HW_CONFIGID1)
+		pr_info("built for config ID: %08x:%08x\n",
+			XCHAL_HW_CONFIGID0, XCHAL_HW_CONFIGID1);
+
 	*cmdline_p = command_line;
 	platform_setup(cmdline_p);
 	strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE);
@@ -339,16 +340,16 @@ void __init setup_arch(char **cmdline_p)
 	mem_reserve(__pa(&_WindowVectors_text_start),
 		    __pa(&_WindowVectors_text_end));
 
-	mem_reserve(__pa(&_DebugInterruptVector_literal_start),
+	mem_reserve(__pa(&_DebugInterruptVector_text_start),
 		    __pa(&_DebugInterruptVector_text_end));
 
-	mem_reserve(__pa(&_KernelExceptionVector_literal_start),
+	mem_reserve(__pa(&_KernelExceptionVector_text_start),
 		    __pa(&_KernelExceptionVector_text_end));
 
-	mem_reserve(__pa(&_UserExceptionVector_literal_start),
+	mem_reserve(__pa(&_UserExceptionVector_text_start),
 		    __pa(&_UserExceptionVector_text_end));
 
-	mem_reserve(__pa(&_DoubleExceptionVector_literal_start),
+	mem_reserve(__pa(&_DoubleExceptionVector_text_start),
 		    __pa(&_DoubleExceptionVector_text_end));
 
 #if XCHAL_EXCM_LEVEL >= 2
@@ -380,7 +381,7 @@ void __init setup_arch(char **cmdline_p)
 #endif
 	parse_early_param();
 	bootmem_init();
-
+	kasan_init();
 	unflatten_and_copy_device_tree();
 
 #ifdef CONFIG_SMP
@@ -582,12 +583,14 @@ c_show(struct seq_file *f, void *slot)
 		      "model\t\t: Xtensa " XCHAL_HW_VERSION_NAME "\n"
 		      "core ID\t\t: " XCHAL_CORE_ID "\n"
 		      "build ID\t: 0x%x\n"
+		      "config ID\t: %08x:%08x\n"
 		      "byte order\t: %s\n"
 		      "cpu MHz\t\t: %lu.%02lu\n"
 		      "bogomips\t: %lu.%02lu\n",
 		      num_online_cpus(),
 		      cpumask_pr_args(cpu_online_mask),
 		      XCHAL_BUILD_UNIQUE_ID,
+		      get_sr(SREG_EPC), get_sr(SREG_EXCSAVE),
 		      XCHAL_HAVE_BE ?  "big" : "little",
 		      ccount_freq/1000000,
 		      (ccount_freq/10000) % 100,
diff --git a/arch/xtensa/kernel/signal.c b/arch/xtensa/kernel/signal.c
index d427e784ab44..f88e7a0b232c 100644
--- a/arch/xtensa/kernel/signal.c
+++ b/arch/xtensa/kernel/signal.c
@@ -28,8 +28,6 @@
 #include <asm/coprocessor.h>
 #include <asm/unistd.h>
 
-#define DEBUG_SIG  0
-
 extern struct task_struct *coproc_owners[];
 
 struct rt_sigframe
@@ -399,10 +397,8 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set,
 	regs->areg[8] = (unsigned long) &frame->uc;
 	regs->threadptr = tp;
 
-#if DEBUG_SIG
-	printk("SIG rt deliver (%s:%d): signal=%d sp=%p pc=%08x\n",
-		current->comm, current->pid, sig, frame, regs->pc);
-#endif
+	pr_debug("SIG rt deliver (%s:%d): signal=%d sp=%p pc=%08lx\n",
+		 current->comm, current->pid, sig, frame, regs->pc);
 
 	return 0;
 }
diff --git a/arch/xtensa/kernel/traps.c b/arch/xtensa/kernel/traps.c
index bae697a06a98..32c5207f1226 100644
--- a/arch/xtensa/kernel/traps.c
+++ b/arch/xtensa/kernel/traps.c
@@ -33,6 +33,7 @@
 #include <linux/kallsyms.h>
 #include <linux/delay.h>
 #include <linux/hardirq.h>
+#include <linux/ratelimit.h>
 
 #include <asm/stacktrace.h>
 #include <asm/ptrace.h>
@@ -158,8 +159,7 @@ COPROCESSOR(7),
  * 2. it is a temporary memory buffer for the exception handlers.
  */
 
-DEFINE_PER_CPU(unsigned long, exc_table[EXC_TABLE_SIZE/4]);
-
+DEFINE_PER_CPU(struct exc_table, exc_table);
 DEFINE_PER_CPU(struct debug_table, debug_table);
 
 void die(const char*, struct pt_regs*, long);
@@ -178,13 +178,14 @@ __die_if_kernel(const char *str, struct pt_regs *regs, long err)
 void do_unhandled(struct pt_regs *regs, unsigned long exccause)
 {
 	__die_if_kernel("Caught unhandled exception - should not happen",
-	    		regs, SIGKILL);
+			regs, SIGKILL);
 
 	/* If in user mode, send SIGILL signal to current process */
-	printk("Caught unhandled exception in '%s' "
-	       "(pid = %d, pc = %#010lx) - should not happen\n"
-	       "\tEXCCAUSE is %ld\n",
-	       current->comm, task_pid_nr(current), regs->pc, exccause);
+	pr_info_ratelimited("Caught unhandled exception in '%s' "
+			    "(pid = %d, pc = %#010lx) - should not happen\n"
+			    "\tEXCCAUSE is %ld\n",
+			    current->comm, task_pid_nr(current), regs->pc,
+			    exccause);
 	force_sig(SIGILL, current);
 }
 
@@ -305,8 +306,8 @@ do_illegal_instruction(struct pt_regs *regs)
 
 	/* If in user mode, send SIGILL signal to current process. */
 
-	printk("Illegal Instruction in '%s' (pid = %d, pc = %#010lx)\n",
-	    current->comm, task_pid_nr(current), regs->pc);
+	pr_info_ratelimited("Illegal Instruction in '%s' (pid = %d, pc = %#010lx)\n",
+			    current->comm, task_pid_nr(current), regs->pc);
 	force_sig(SIGILL, current);
 }
 
@@ -325,13 +326,14 @@ do_unaligned_user (struct pt_regs *regs)
 	siginfo_t info;
 
 	__die_if_kernel("Unhandled unaligned exception in kernel",
-	    		regs, SIGKILL);
+			regs, SIGKILL);
 
 	current->thread.bad_vaddr = regs->excvaddr;
 	current->thread.error_code = -3;
-	printk("Unaligned memory access to %08lx in '%s' "
-	       "(pid = %d, pc = %#010lx)\n",
-	       regs->excvaddr, current->comm, task_pid_nr(current), regs->pc);
+	pr_info_ratelimited("Unaligned memory access to %08lx in '%s' "
+			    "(pid = %d, pc = %#010lx)\n",
+			    regs->excvaddr, current->comm,
+			    task_pid_nr(current), regs->pc);
 	info.si_signo = SIGBUS;
 	info.si_errno = 0;
 	info.si_code = BUS_ADRALN;
@@ -365,28 +367,28 @@ do_debug(struct pt_regs *regs)
 }
 
 
-static void set_handler(int idx, void *handler)
-{
-	unsigned int cpu;
-
-	for_each_possible_cpu(cpu)
-		per_cpu(exc_table, cpu)[idx] = (unsigned long)handler;
-}
+#define set_handler(type, cause, handler)				\
+	do {								\
+		unsigned int cpu;					\
+									\
+		for_each_possible_cpu(cpu)				\
+			per_cpu(exc_table, cpu).type[cause] = (handler);\
+	} while (0)
 
 /* Set exception C handler - for temporary use when probing exceptions */
 
 void * __init trap_set_handler(int cause, void *handler)
 {
-	void *previous = (void *)per_cpu(exc_table, 0)[
-		EXC_TABLE_DEFAULT / 4 + cause];
-	set_handler(EXC_TABLE_DEFAULT / 4 + cause, handler);
+	void *previous = per_cpu(exc_table, 0).default_handler[cause];
+
+	set_handler(default_handler, cause, handler);
 	return previous;
 }
 
 
 static void trap_init_excsave(void)
 {
-	unsigned long excsave1 = (unsigned long)this_cpu_ptr(exc_table);
+	unsigned long excsave1 = (unsigned long)this_cpu_ptr(&exc_table);
 	__asm__ __volatile__("wsr  %0, excsave1\n" : : "a" (excsave1));
 }
 
@@ -418,10 +420,10 @@ void __init trap_init(void)
 
 	/* Setup default vectors. */
 
-	for(i = 0; i < 64; i++) {
-		set_handler(EXC_TABLE_FAST_USER/4   + i, user_exception);
-		set_handler(EXC_TABLE_FAST_KERNEL/4 + i, kernel_exception);
-		set_handler(EXC_TABLE_DEFAULT/4 + i, do_unhandled);
+	for (i = 0; i < EXCCAUSE_N; i++) {
+		set_handler(fast_user_handler, i, user_exception);
+		set_handler(fast_kernel_handler, i, kernel_exception);
+		set_handler(default_handler, i, do_unhandled);
 	}
 
 	/* Setup specific handlers. */
@@ -433,11 +435,11 @@ void __init trap_init(void)
 		void *handler = dispatch_init_table[i].handler;
 
 		if (fast == 0)
-			set_handler (EXC_TABLE_DEFAULT/4 + cause, handler);
+			set_handler(default_handler, cause, handler);
 		if (fast && fast & USER)
-			set_handler (EXC_TABLE_FAST_USER/4 + cause, handler);
+			set_handler(fast_user_handler, cause, handler);
 		if (fast && fast & KRNL)
-			set_handler (EXC_TABLE_FAST_KERNEL/4 + cause, handler);
+			set_handler(fast_kernel_handler, cause, handler);
 	}
 
 	/* Initialize EXCSAVE_1 to hold the address of the exception table. */
diff --git a/arch/xtensa/kernel/vectors.S b/arch/xtensa/kernel/vectors.S
index 332e9d635fb6..841503d3307c 100644
--- a/arch/xtensa/kernel/vectors.S
+++ b/arch/xtensa/kernel/vectors.S
@@ -205,9 +205,6 @@ ENDPROC(_KernelExceptionVector)
  */
 
 	.section .DoubleExceptionVector.text, "ax"
-	.begin literal_prefix .DoubleExceptionVector
-	.globl _DoubleExceptionVector_WindowUnderflow
-	.globl _DoubleExceptionVector_WindowOverflow
 
 ENTRY(_DoubleExceptionVector)
 
@@ -217,8 +214,12 @@ ENTRY(_DoubleExceptionVector)
 	/* Check for kernel double exception (usually fatal). */
 
 	rsr	a2, ps
-	_bbci.l	a2, PS_UM_BIT, .Lksp
+	_bbsi.l	a2, PS_UM_BIT, 1f
+	j	.Lksp
 
+	.align	4
+	.literal_position
+1:
 	/* Check if we are currently handling a window exception. */
 	/* Note: We don't need to indicate that we enter a critical section. */
 
@@ -304,8 +305,7 @@ _DoubleExceptionVector_WindowUnderflow:
 .Lunrecoverable:
 	rsr	a3, excsave1
 	wsr	a0, excsave1
-	movi	a0, unrecoverable_exception
-	callx0	a0
+	call0	unrecoverable_exception
 
 .Lfixup:/* Check for a fixup handler or if we were in a critical section. */
 
@@ -475,11 +475,8 @@ _DoubleExceptionVector_handle_exception:
 	rotw	-3
 	j	1b
 
-
 ENDPROC(_DoubleExceptionVector)
 
-	.end literal_prefix
-
 	.text
 /*
  * Fixup handler for TLB miss in double exception handler for window owerflow.
@@ -508,6 +505,8 @@ ENDPROC(_DoubleExceptionVector)
  * a3: exctable, original value in excsave1
  */
 
+	.literal_position
+
 ENTRY(window_overflow_restore_a0_fixup)
 
 	rsr	a0, ps
diff --git a/arch/xtensa/kernel/vmlinux.lds.S b/arch/xtensa/kernel/vmlinux.lds.S
index 162c77e53ca8..70b731edc7b8 100644
--- a/arch/xtensa/kernel/vmlinux.lds.S
+++ b/arch/xtensa/kernel/vmlinux.lds.S
@@ -45,24 +45,16 @@ jiffies = jiffies_64;
 	LONG(sym ## _end);			\
 	LONG(LOADADDR(section))
 
-/* Macro to define a section for a vector.
- *
- * Use of the MIN function catches the types of errors illustrated in
- * the following example:
- *
- * Assume the section .DoubleExceptionVector.literal is completely
- * full.  Then a programmer adds code to .DoubleExceptionVector.text
- * that produces another literal.  The final literal position will
- * overlay onto the first word of the adjacent code section
- * .DoubleExceptionVector.text.  (In practice, the literals will
- * overwrite the code, and the first few instructions will be
- * garbage.)
+/*
+ * Macro to define a section for a vector. When CONFIG_VECTORS_OFFSET is
+ * defined code for every vector is located with other init data. At startup
+ * time head.S copies code for every vector to its final position according
+ * to description recorded in the corresponding RELOCATE_ENTRY.
  */
 
 #ifdef CONFIG_VECTORS_OFFSET
-#define SECTION_VECTOR(sym, section, addr, max_prevsec_size, prevsec)       \
-  section addr : AT((MIN(LOADADDR(prevsec) + max_prevsec_size,		    \
-		         LOADADDR(prevsec) + SIZEOF(prevsec)) + 3) & ~ 3)   \
+#define SECTION_VECTOR(sym, section, addr, prevsec)                         \
+  section addr : AT(((LOADADDR(prevsec) + SIZEOF(prevsec)) + 3) & ~ 3)      \
   {									    \
     . = ALIGN(4);							    \
     sym ## _start = ABSOLUTE(.);		 			    \
@@ -112,26 +104,19 @@ SECTIONS
 #if XCHAL_EXCM_LEVEL >= 6
   SECTION_VECTOR (.Level6InterruptVector.text, INTLEVEL6_VECTOR_VADDR)
 #endif
-  SECTION_VECTOR (.DebugInterruptVector.literal, DEBUG_VECTOR_VADDR - 4)
   SECTION_VECTOR (.DebugInterruptVector.text, DEBUG_VECTOR_VADDR)
-  SECTION_VECTOR (.KernelExceptionVector.literal, KERNEL_VECTOR_VADDR - 4)
   SECTION_VECTOR (.KernelExceptionVector.text, KERNEL_VECTOR_VADDR)
-  SECTION_VECTOR (.UserExceptionVector.literal, USER_VECTOR_VADDR - 4)
   SECTION_VECTOR (.UserExceptionVector.text, USER_VECTOR_VADDR)
-  SECTION_VECTOR (.DoubleExceptionVector.literal, DOUBLEEXC_VECTOR_VADDR - 20)
   SECTION_VECTOR (.DoubleExceptionVector.text, DOUBLEEXC_VECTOR_VADDR)
 #endif
 
+    IRQENTRY_TEXT
+    SOFTIRQENTRY_TEXT
+    ENTRY_TEXT
     TEXT_TEXT
-    VMLINUX_SYMBOL(__sched_text_start) = .;
-    *(.sched.literal .sched.text)
-    VMLINUX_SYMBOL(__sched_text_end) = .;
-    VMLINUX_SYMBOL(__cpuidle_text_start) = .;
-    *(.cpuidle.literal .cpuidle.text)
-    VMLINUX_SYMBOL(__cpuidle_text_end) = .;
-    VMLINUX_SYMBOL(__lock_text_start) = .;
-    *(.spinlock.literal .spinlock.text)
-    VMLINUX_SYMBOL(__lock_text_end) = .;
+    SCHED_TEXT
+    CPUIDLE_TEXT
+    LOCK_TEXT
 
   }
   _etext = .;
@@ -196,8 +181,6 @@ SECTIONS
 		   .KernelExceptionVector.text);
     RELOCATE_ENTRY(_UserExceptionVector_text,
 		   .UserExceptionVector.text);
-    RELOCATE_ENTRY(_DoubleExceptionVector_literal,
-		   .DoubleExceptionVector.literal);
     RELOCATE_ENTRY(_DoubleExceptionVector_text,
 		   .DoubleExceptionVector.text);
     RELOCATE_ENTRY(_DebugInterruptVector_text,
@@ -230,25 +213,19 @@ SECTIONS
 
   SECTION_VECTOR (_WindowVectors_text,
 		  .WindowVectors.text,
-		  WINDOW_VECTORS_VADDR, 4,
+		  WINDOW_VECTORS_VADDR,
 		  .dummy)
-  SECTION_VECTOR (_DebugInterruptVector_literal,
-		  .DebugInterruptVector.literal,
-		  DEBUG_VECTOR_VADDR - 4,
-		  SIZEOF(.WindowVectors.text),
-		  .WindowVectors.text)
   SECTION_VECTOR (_DebugInterruptVector_text,
 		  .DebugInterruptVector.text,
 		  DEBUG_VECTOR_VADDR,
-		  4,
-		  .DebugInterruptVector.literal)
+		  .WindowVectors.text)
 #undef LAST
 #define LAST	.DebugInterruptVector.text
 #if XCHAL_EXCM_LEVEL >= 2
   SECTION_VECTOR (_Level2InterruptVector_text,
 		  .Level2InterruptVector.text,
 		  INTLEVEL2_VECTOR_VADDR,
-		  SIZEOF(LAST), LAST)
+		  LAST)
 # undef LAST
 # define LAST	.Level2InterruptVector.text
 #endif
@@ -256,7 +233,7 @@ SECTIONS
   SECTION_VECTOR (_Level3InterruptVector_text,
 		  .Level3InterruptVector.text,
 		  INTLEVEL3_VECTOR_VADDR,
-		  SIZEOF(LAST), LAST)
+		  LAST)
 # undef LAST
 # define LAST	.Level3InterruptVector.text
 #endif
@@ -264,7 +241,7 @@ SECTIONS
   SECTION_VECTOR (_Level4InterruptVector_text,
 		  .Level4InterruptVector.text,
 		  INTLEVEL4_VECTOR_VADDR,
-		  SIZEOF(LAST), LAST)
+		  LAST)
 # undef LAST
 # define LAST	.Level4InterruptVector.text
 #endif
@@ -272,7 +249,7 @@ SECTIONS
   SECTION_VECTOR (_Level5InterruptVector_text,
 		  .Level5InterruptVector.text,
 		  INTLEVEL5_VECTOR_VADDR,
-		  SIZEOF(LAST), LAST)
+		  LAST)
 # undef LAST
 # define LAST	.Level5InterruptVector.text
 #endif
@@ -280,40 +257,23 @@ SECTIONS
   SECTION_VECTOR (_Level6InterruptVector_text,
 		  .Level6InterruptVector.text,
 		  INTLEVEL6_VECTOR_VADDR,
-		  SIZEOF(LAST), LAST)
+		  LAST)
 # undef LAST
 # define LAST	.Level6InterruptVector.text
 #endif
-  SECTION_VECTOR (_KernelExceptionVector_literal,
-		  .KernelExceptionVector.literal,
-		  KERNEL_VECTOR_VADDR - 4,
-		  SIZEOF(LAST), LAST)
-#undef LAST
   SECTION_VECTOR (_KernelExceptionVector_text,
 		  .KernelExceptionVector.text,
 		  KERNEL_VECTOR_VADDR,
-		  4,
-		  .KernelExceptionVector.literal)
-  SECTION_VECTOR (_UserExceptionVector_literal,
-		  .UserExceptionVector.literal,
-		  USER_VECTOR_VADDR - 4,
-		  SIZEOF(.KernelExceptionVector.text),
-		  .KernelExceptionVector.text)
+		  LAST)
+#undef LAST
   SECTION_VECTOR (_UserExceptionVector_text,
 		  .UserExceptionVector.text,
 		  USER_VECTOR_VADDR,
-		  4,
-		  .UserExceptionVector.literal)
-  SECTION_VECTOR (_DoubleExceptionVector_literal,
-		  .DoubleExceptionVector.literal,
-		  DOUBLEEXC_VECTOR_VADDR - 20,
-		  SIZEOF(.UserExceptionVector.text),
-		  .UserExceptionVector.text)
+		  .KernelExceptionVector.text)
   SECTION_VECTOR (_DoubleExceptionVector_text,
 		  .DoubleExceptionVector.text,
 		  DOUBLEEXC_VECTOR_VADDR,
-		  20,
-		  .DoubleExceptionVector.literal)
+		  .UserExceptionVector.text)
 
   . = (LOADADDR( .DoubleExceptionVector.text ) + SIZEOF( .DoubleExceptionVector.text ) + 3) & ~ 3;
 
@@ -323,7 +283,6 @@ SECTIONS
   SECTION_VECTOR (_SecondaryResetVector_text,
 		  .SecondaryResetVector.text,
 		  RESET_VECTOR1_VADDR,
-		  SIZEOF(.DoubleExceptionVector.text),
 		  .DoubleExceptionVector.text)
 
   . = LOADADDR(.SecondaryResetVector.text)+SIZEOF(.SecondaryResetVector.text);
@@ -373,5 +332,4 @@ SECTIONS
 
   /* Sections to be discarded */
   DISCARDS
-  /DISCARD/ : { *(.exit.literal) }
 }
diff --git a/arch/xtensa/kernel/xtensa_ksyms.c b/arch/xtensa/kernel/xtensa_ksyms.c
index 672391003e40..04f19de46700 100644
--- a/arch/xtensa/kernel/xtensa_ksyms.c
+++ b/arch/xtensa/kernel/xtensa_ksyms.c
@@ -41,7 +41,12 @@
 EXPORT_SYMBOL(memset);
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(memmove);
+EXPORT_SYMBOL(__memset);
+EXPORT_SYMBOL(__memcpy);
+EXPORT_SYMBOL(__memmove);
+#ifndef CONFIG_GENERIC_STRNCPY_FROM_USER
 EXPORT_SYMBOL(__strncpy_user);
+#endif
 EXPORT_SYMBOL(clear_page);
 EXPORT_SYMBOL(copy_page);
 
diff --git a/arch/xtensa/lib/checksum.S b/arch/xtensa/lib/checksum.S
index 4eb573d2720e..528fe0dd9339 100644
--- a/arch/xtensa/lib/checksum.S
+++ b/arch/xtensa/lib/checksum.S
@@ -14,9 +14,10 @@
  *		2 of the License, or (at your option) any later version.
  */
 
-#include <asm/errno.h>
+#include <linux/errno.h>
 #include <linux/linkage.h>
 #include <variant/core.h>
+#include <asm/asmmacro.h>
 
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
@@ -175,23 +176,8 @@ ENDPROC(csum_partial)
 
 /*
  * Copy from ds while checksumming, otherwise like csum_partial
- *
- * The macros SRC and DST specify the type of access for the instruction.
- * thus we can call a custom exception handler for each access type.
  */
 
-#define SRC(y...)			\
-	9999: y;			\
-	.section __ex_table, "a";	\
-	.long 9999b, 6001f	;	\
-	.previous
-
-#define DST(y...)			\
-	9999: y;			\
-	.section __ex_table, "a";	\
-	.long 9999b, 6002f	;	\
-	.previous
-
 /*
 unsigned int csum_partial_copy_generic (const char *src, char *dst, int len,
 					int sum, int *src_err_ptr, int *dst_err_ptr)
@@ -244,28 +230,28 @@ ENTRY(csum_partial_copy_generic)
 	add	a10, a10, a2	/* a10 = end of last 32-byte src chunk */
 .Loop5:
 #endif
-SRC(	l32i	a9, a2, 0	)
-SRC(	l32i	a8, a2, 4	)
-DST(	s32i	a9, a3, 0	)
-DST(	s32i	a8, a3, 4	)
+EX(10f)	l32i	a9, a2, 0
+EX(10f)	l32i	a8, a2, 4
+EX(11f)	s32i	a9, a3, 0
+EX(11f)	s32i	a8, a3, 4
 	ONES_ADD(a5, a9)
 	ONES_ADD(a5, a8)
-SRC(	l32i	a9, a2, 8	)
-SRC(	l32i	a8, a2, 12	)
-DST(	s32i	a9, a3, 8	)
-DST(	s32i	a8, a3, 12	)
+EX(10f)	l32i	a9, a2, 8
+EX(10f)	l32i	a8, a2, 12
+EX(11f)	s32i	a9, a3, 8
+EX(11f)	s32i	a8, a3, 12
 	ONES_ADD(a5, a9)
 	ONES_ADD(a5, a8)
-SRC(	l32i	a9, a2, 16	)
-SRC(	l32i	a8, a2, 20	)
-DST(	s32i	a9, a3, 16	)
-DST(	s32i	a8, a3, 20	)
+EX(10f)	l32i	a9, a2, 16
+EX(10f)	l32i	a8, a2, 20
+EX(11f)	s32i	a9, a3, 16
+EX(11f)	s32i	a8, a3, 20
 	ONES_ADD(a5, a9)
 	ONES_ADD(a5, a8)
-SRC(	l32i	a9, a2, 24	)
-SRC(	l32i	a8, a2, 28	)
-DST(	s32i	a9, a3, 24	)
-DST(	s32i	a8, a3, 28	)
+EX(10f)	l32i	a9, a2, 24
+EX(10f)	l32i	a8, a2, 28
+EX(11f)	s32i	a9, a3, 24
+EX(11f)	s32i	a8, a3, 28
 	ONES_ADD(a5, a9)
 	ONES_ADD(a5, a8)
 	addi	a2, a2, 32
@@ -284,8 +270,8 @@ DST(	s32i	a8, a3, 28	)
 	add	a10, a10, a2	/* a10 = end of last 4-byte src chunk */
 .Loop6:
 #endif
-SRC(	l32i	a9, a2, 0	)
-DST(	s32i	a9, a3, 0	)
+EX(10f)	l32i	a9, a2, 0
+EX(11f)	s32i	a9, a3, 0
 	ONES_ADD(a5, a9)
 	addi	a2, a2, 4
 	addi	a3, a3, 4
@@ -315,8 +301,8 @@ DST(	s32i	a9, a3, 0	)
 	add	a10, a10, a2	/* a10 = end of last 2-byte src chunk */
 .Loop7:
 #endif
-SRC(	l16ui	a9, a2, 0	)
-DST(	s16i	a9, a3, 0	)
+EX(10f)	l16ui	a9, a2, 0
+EX(11f)	s16i	a9, a3, 0
 	ONES_ADD(a5, a9)
 	addi	a2, a2, 2
 	addi	a3, a3, 2
@@ -326,8 +312,8 @@ DST(	s16i	a9, a3, 0	)
 4:
 	/* This section processes a possible trailing odd byte. */
 	_bbci.l	a4, 0, 8f	/* 1-byte chunk */
-SRC(	l8ui	a9, a2, 0	)
-DST(	s8i	a9, a3, 0	)
+EX(10f)	l8ui	a9, a2, 0
+EX(11f)	s8i	a9, a3, 0
 #ifdef __XTENSA_EB__
 	slli	a9, a9, 8	/* shift byte to bits 8..15 */
 #endif
@@ -350,10 +336,10 @@ DST(	s8i	a9, a3, 0	)
 	add	a10, a10, a2	/* a10 = end of last odd-aligned, 2-byte src chunk */
 .Loop8:
 #endif
-SRC(	l8ui	a9, a2, 0	)
-SRC(	l8ui	a8, a2, 1	)
-DST(	s8i	a9, a3, 0	)
-DST(	s8i	a8, a3, 1	)
+EX(10f)	l8ui	a9, a2, 0
+EX(10f)	l8ui	a8, a2, 1
+EX(11f)	s8i	a9, a3, 0
+EX(11f)	s8i	a8, a3, 1
 #ifdef __XTENSA_EB__
 	slli	a9, a9, 8	/* combine into a single 16-bit value */
 #else				/* for checksum computation */
@@ -381,7 +367,7 @@ ENDPROC(csum_partial_copy_generic)
 	a12 = original dst for exception handling
 */
 
-6001:
+10:
 	_movi	a2, -EFAULT
 	s32i	a2, a6, 0	/* src_err_ptr */
 
@@ -403,7 +389,7 @@ ENDPROC(csum_partial_copy_generic)
 2:
 	retw
 
-6002:
+11:
 	movi	a2, -EFAULT
 	s32i	a2, a7, 0	/* dst_err_ptr */
 	movi	a2, 0
diff --git a/arch/xtensa/lib/memcopy.S b/arch/xtensa/lib/memcopy.S
index b1c219acabe7..c0f6981719d6 100644
--- a/arch/xtensa/lib/memcopy.S
+++ b/arch/xtensa/lib/memcopy.S
@@ -9,23 +9,9 @@
  * Copyright (C) 2002 - 2012 Tensilica Inc.
  */
 
+#include <linux/linkage.h>
 #include <variant/core.h>
-
-	.macro	src_b	r, w0, w1
-#ifdef __XTENSA_EB__
-	src	\r, \w0, \w1
-#else
-	src	\r, \w1, \w0
-#endif
-	.endm
-
-	.macro	ssa8	r
-#ifdef __XTENSA_EB__
-	ssa8b	\r
-#else
-	ssa8l	\r
-#endif
-	.endm
+#include <asm/asmmacro.h>
 
 /*
  * void *memcpy(void *dst, const void *src, size_t len);
@@ -123,10 +109,8 @@
 	addi	a5, a5,  2
 	j	.Ldstaligned	# dst is now aligned, return to main algorithm
 
-	.align	4
-	.global	memcpy
-	.type   memcpy,@function
-memcpy:
+ENTRY(__memcpy)
+WEAK(memcpy)
 
 	entry	sp, 16		# minimal stack frame
 	# a2/ dst, a3/ src, a4/ len
@@ -209,7 +193,7 @@ memcpy:
 .Lsrcunaligned:
 	_beqz	a4, .Ldone	# avoid loading anything for zero-length copies
 	# copy 16 bytes per iteration for word-aligned dst and unaligned src
-	ssa8	a3		# set shift amount from byte offset
+	__ssa8	a3		# set shift amount from byte offset
 
 /* set to 1 when running on ISS (simulator) with the
    lint or ferret client, or 0 to save a few cycles */
@@ -229,16 +213,16 @@ memcpy:
 .Loop2:
 	l32i	a7, a3,  4
 	l32i	a8, a3,  8
-	src_b	a6, a6, a7
+	__src_b	a6, a6, a7
 	s32i	a6, a5,  0
 	l32i	a9, a3, 12
-	src_b	a7, a7, a8
+	__src_b	a7, a7, a8
 	s32i	a7, a5,  4
 	l32i	a6, a3, 16
-	src_b	a8, a8, a9
+	__src_b	a8, a8, a9
 	s32i	a8, a5,  8
 	addi	a3, a3, 16
-	src_b	a9, a9, a6
+	__src_b	a9, a9, a6
 	s32i	a9, a5, 12
 	addi	a5, a5, 16
 #if !XCHAL_HAVE_LOOPS
@@ -249,10 +233,10 @@ memcpy:
 	# copy 8 bytes
 	l32i	a7, a3,  4
 	l32i	a8, a3,  8
-	src_b	a6, a6, a7
+	__src_b	a6, a6, a7
 	s32i	a6, a5,  0
 	addi	a3, a3,  8
-	src_b	a7, a7, a8
+	__src_b	a7, a7, a8
 	s32i	a7, a5,  4
 	addi	a5, a5,  8
 	mov	a6, a8
@@ -261,7 +245,7 @@ memcpy:
 	# copy 4 bytes
 	l32i	a7, a3,  4
 	addi	a3, a3,  4
-	src_b	a6, a6, a7
+	__src_b	a6, a6, a7
 	s32i	a6, a5,  0
 	addi	a5, a5,  4
 	mov	a6, a7
@@ -288,14 +272,14 @@ memcpy:
 	s8i	a6, a5,  0
 	retw
 
+ENDPROC(__memcpy)
 
 /*
  * void bcopy(const void *src, void *dest, size_t n);
  */
-	.align	4
-	.global	bcopy
-	.type   bcopy,@function
-bcopy:
+
+ENTRY(bcopy)
+
 	entry	sp, 16		# minimal stack frame
 	# a2=src, a3=dst, a4=len
 	mov	a5, a3
@@ -303,6 +287,8 @@ bcopy:
 	mov	a2, a5
 	j	.Lmovecommon	# go to common code for memmove+bcopy
 
+ENDPROC(bcopy)
+
 /*
  * void *memmove(void *dst, const void *src, size_t len);
  *
@@ -391,10 +377,8 @@ bcopy:
 	j	.Lbackdstaligned	# dst is now aligned,
 					# return to main algorithm
 
-	.align	4
-	.global	memmove
-	.type   memmove,@function
-memmove:
+ENTRY(__memmove)
+WEAK(memmove)
 
 	entry	sp, 16		# minimal stack frame
 	# a2/ dst, a3/ src, a4/ len
@@ -485,7 +469,7 @@ memmove:
 .Lbacksrcunaligned:
 	_beqz	a4, .Lbackdone	# avoid loading anything for zero-length copies
 	# copy 16 bytes per iteration for word-aligned dst and unaligned src
-	ssa8	a3		# set shift amount from byte offset
+	__ssa8	a3		# set shift amount from byte offset
 #define SIM_CHECKS_ALIGNMENT	1	/* set to 1 when running on ISS with
 					 * the lint or ferret client, or 0
 					 * to save a few cycles */
@@ -506,15 +490,15 @@ memmove:
 	l32i	a7, a3, 12
 	l32i	a8, a3,  8
 	addi	a5, a5, -16
-	src_b	a6, a7, a6
+	__src_b	a6, a7, a6
 	s32i	a6, a5, 12
 	l32i	a9, a3,  4
-	src_b	a7, a8, a7
+	__src_b	a7, a8, a7
 	s32i	a7, a5,  8
 	l32i	a6, a3,  0
-	src_b	a8, a9, a8
+	__src_b	a8, a9, a8
 	s32i	a8, a5,  4
-	src_b	a9, a6, a9
+	__src_b	a9, a6, a9
 	s32i	a9, a5,  0
 #if !XCHAL_HAVE_LOOPS
 	bne	a3, a10, .backLoop2 # continue loop if a3:src != a10:src_start
@@ -526,9 +510,9 @@ memmove:
 	l32i	a7, a3,  4
 	l32i	a8, a3,  0
 	addi	a5, a5, -8
-	src_b	a6, a7, a6
+	__src_b	a6, a7, a6
 	s32i	a6, a5,  4
-	src_b	a7, a8, a7
+	__src_b	a7, a8, a7
 	s32i	a7, a5,  0
 	mov	a6, a8
 .Lback12:
@@ -537,7 +521,7 @@ memmove:
 	addi	a3, a3, -4
 	l32i	a7, a3,  0
 	addi	a5, a5, -4
-	src_b	a6, a7, a6
+	__src_b	a6, a7, a6
 	s32i	a6, a5,  0
 	mov	a6, a7
 .Lback13:
@@ -566,11 +550,4 @@ memmove:
 	s8i	a6, a5,  0
 	retw
 
-
-/*
- * Local Variables:
- * mode:fundamental
- * comment-start: "# "
- * comment-start-skip: "# *"
- * End:
- */
+ENDPROC(__memmove)
diff --git a/arch/xtensa/lib/memset.S b/arch/xtensa/lib/memset.S
index 10b8c400f175..276747dec300 100644
--- a/arch/xtensa/lib/memset.S
+++ b/arch/xtensa/lib/memset.S
@@ -11,7 +11,9 @@
  *  Copyright (C) 2002 Tensilica Inc.
  */
 
+#include <linux/linkage.h>
 #include <variant/core.h>
+#include <asm/asmmacro.h>
 
 /*
  * void *memset(void *dst, int c, size_t length)
@@ -28,20 +30,10 @@
  *     the alignment labels).
  */
 
-/* Load or store instructions that may cause exceptions use the EX macro. */
-
-#define EX(insn,reg1,reg2,offset,handler)	\
-9:	insn	reg1, reg2, offset;		\
-	.section __ex_table, "a";		\
-	.word	9b, handler;			\
-	.previous
-
-
 .text
-.align	4
-.global	memset
-.type	memset,@function
-memset:
+ENTRY(__memset)
+WEAK(memset)
+
 	entry	sp, 16		# minimal stack frame
 	# a2/ dst, a3/ c, a4/ length
 	extui	a3, a3, 0, 8	# mask to just 8 bits
@@ -73,10 +65,10 @@ memset:
 	add	a6, a6, a5	# a6 = end of last 16B chunk
 #endif /* !XCHAL_HAVE_LOOPS */
 .Loop1:
-	EX(s32i, a3, a5,  0, memset_fixup)
-	EX(s32i, a3, a5,  4, memset_fixup)
-	EX(s32i, a3, a5,  8, memset_fixup)
-	EX(s32i, a3, a5, 12, memset_fixup)
+EX(10f) s32i	a3, a5,  0
+EX(10f) s32i	a3, a5,  4
+EX(10f) s32i	a3, a5,  8
+EX(10f) s32i	a3, a5, 12
 	addi	a5, a5, 16
 #if !XCHAL_HAVE_LOOPS
 	blt	a5, a6, .Loop1
@@ -84,23 +76,23 @@ memset:
 .Loop1done:
 	bbci.l	a4, 3, .L2
 	# set 8 bytes
-	EX(s32i, a3, a5,  0, memset_fixup)
-	EX(s32i, a3, a5,  4, memset_fixup)
+EX(10f) s32i	a3, a5,  0
+EX(10f) s32i	a3, a5,  4
 	addi	a5, a5,  8
 .L2:
 	bbci.l	a4, 2, .L3
 	# set 4 bytes
-	EX(s32i, a3, a5,  0, memset_fixup)
+EX(10f) s32i	a3, a5,  0
 	addi	a5, a5,  4
 .L3:
 	bbci.l	a4, 1, .L4
 	# set 2 bytes
-	EX(s16i, a3, a5,  0, memset_fixup)
+EX(10f) s16i	a3, a5,  0
 	addi	a5, a5,  2
 .L4:
 	bbci.l	a4, 0, .L5
 	# set 1 byte
-	EX(s8i, a3, a5,  0, memset_fixup)
+EX(10f) s8i	a3, a5,  0
 .L5:
 .Lret1:
 	retw
@@ -114,7 +106,7 @@ memset:
 	bbci.l	a5, 0, .L20		# branch if dst alignment half-aligned
 	# dst is only byte aligned
 	# set 1 byte
-	EX(s8i, a3, a5,  0, memset_fixup)
+EX(10f) s8i	a3, a5,  0
 	addi	a5, a5,  1
 	addi	a4, a4, -1
 	# now retest if dst aligned
@@ -122,7 +114,7 @@ memset:
 .L20:
 	# dst half-aligned
 	# set 2 bytes
-	EX(s16i, a3, a5,  0, memset_fixup)
+EX(10f) s16i	a3, a5,  0
 	addi	a5, a5,  2
 	addi	a4, a4, -2
 	j	.L0		# dst is now aligned, return to main algorithm
@@ -141,7 +133,7 @@ memset:
 	add	a6, a5, a4	# a6 = ending address
 #endif /* !XCHAL_HAVE_LOOPS */
 .Lbyteloop:
-	EX(s8i, a3, a5, 0, memset_fixup)
+EX(10f) s8i	a3, a5, 0
 	addi	a5, a5, 1
 #if !XCHAL_HAVE_LOOPS
 	blt	a5, a6, .Lbyteloop
@@ -149,12 +141,13 @@ memset:
 .Lbytesetdone:
 	retw
 
+ENDPROC(__memset)
 
 	.section .fixup, "ax"
 	.align	4
 
 /* We return zero if a failure occurred. */
 
-memset_fixup:
+10:
 	movi	a2, 0
 	retw
diff --git a/arch/xtensa/lib/pci-auto.c b/arch/xtensa/lib/pci-auto.c
index 34d05abbd921..a2b558161d6d 100644
--- a/arch/xtensa/lib/pci-auto.c
+++ b/arch/xtensa/lib/pci-auto.c
@@ -49,17 +49,6 @@
  *
  */
 
-
-/* define DEBUG to print some debugging messages. */
-
-#undef DEBUG
-
-#ifdef DEBUG
-# define DBG(x...) printk(x)
-#else
-# define DBG(x...)
-#endif
-
 static int pciauto_upper_iospc;
 static int pciauto_upper_memspc;
 
@@ -97,7 +86,7 @@ pciauto_setup_bars(struct pci_dev *dev, int bar_limit)
 		{
 			bar_size &= PCI_BASE_ADDRESS_IO_MASK;
 			upper_limit = &pciauto_upper_iospc;
-			DBG("PCI Autoconfig: BAR %d, I/O, ", bar_nr);
+			pr_debug("PCI Autoconfig: BAR %d, I/O, ", bar_nr);
 		}
 		else
 		{
@@ -107,7 +96,7 @@ pciauto_setup_bars(struct pci_dev *dev, int bar_limit)
 
 			bar_size &= PCI_BASE_ADDRESS_MEM_MASK;
 			upper_limit = &pciauto_upper_memspc;
-			DBG("PCI Autoconfig: BAR %d, Mem, ", bar_nr);
+			pr_debug("PCI Autoconfig: BAR %d, Mem, ", bar_nr);
 		}
 
 		/* Allocate a base address (bar_size is negative!) */
@@ -125,7 +114,8 @@ pciauto_setup_bars(struct pci_dev *dev, int bar_limit)
 		if (found_mem64)
 			pci_write_config_dword(dev, (bar+=4), 0x00000000);
 
-		DBG("size=0x%x, address=0x%x\n", ~bar_size + 1, *upper_limit);
+		pr_debug("size=0x%x, address=0x%x\n",
+			 ~bar_size + 1, *upper_limit);
 	}
 }
 
@@ -150,7 +140,7 @@ pciauto_setup_irq(struct pci_controller* pci_ctrl,struct pci_dev *dev,int devfn)
 	if (irq == -1)
 		irq = 0;
 
-	DBG("PCI Autoconfig: Interrupt %d, pin %d\n", irq, pin);
+	pr_debug("PCI Autoconfig: Interrupt %d, pin %d\n", irq, pin);
 
 	pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq);
 }
@@ -289,8 +279,8 @@ int __init pciauto_bus_scan(struct pci_controller *pci_ctrl, int current_bus)
 
 			int iosave, memsave;
 
-			DBG("PCI Autoconfig: Found P2P bridge, device %d\n",
-			    PCI_SLOT(pci_devfn));
+			pr_debug("PCI Autoconfig: Found P2P bridge, device %d\n",
+				 PCI_SLOT(pci_devfn));
 
 			/* Allocate PCI I/O and/or memory space */
 			pciauto_setup_bars(dev, PCI_BASE_ADDRESS_1);
@@ -306,23 +296,6 @@ int __init pciauto_bus_scan(struct pci_controller *pci_ctrl, int current_bus)
 
 		}
 
-
-#if 0
-		/* Skip legacy mode IDE controller */
-
-		if ((pci_class >> 16) == PCI_CLASS_STORAGE_IDE) {
-
-			unsigned char prg_iface;
-			pci_read_config_byte(dev, PCI_CLASS_PROG, &prg_iface);
-
-			if (!(prg_iface & PCIAUTO_IDE_MODE_MASK)) {
-				DBG("PCI Autoconfig: Skipping legacy mode "
-				    "IDE controller\n");
-				continue;
-			}
-		}
-#endif
-
 		/*
 		 * Found a peripheral, enable some standard
 		 * settings
@@ -337,8 +310,8 @@ int __init pciauto_bus_scan(struct pci_controller *pci_ctrl, int current_bus)
 		pci_write_config_byte(dev, PCI_LATENCY_TIMER, 0x80);
 
 		/* Allocate PCI I/O and/or memory space */
-		DBG("PCI Autoconfig: Found Bus %d, Device %d, Function %d\n",
-		    current_bus, PCI_SLOT(pci_devfn), PCI_FUNC(pci_devfn) );
+		pr_debug("PCI Autoconfig: Found Bus %d, Device %d, Function %d\n",
+			 current_bus, PCI_SLOT(pci_devfn), PCI_FUNC(pci_devfn));
 
 		pciauto_setup_bars(dev, PCI_BASE_ADDRESS_5);
 		pciauto_setup_irq(pci_ctrl, dev, pci_devfn);
diff --git a/arch/xtensa/lib/strncpy_user.S b/arch/xtensa/lib/strncpy_user.S
index 1ad0ecf45368..5fce16b67dca 100644
--- a/arch/xtensa/lib/strncpy_user.S
+++ b/arch/xtensa/lib/strncpy_user.S
@@ -11,16 +11,10 @@
  *  Copyright (C) 2002 Tensilica Inc.
  */
 
-#include <variant/core.h>
 #include <linux/errno.h>
-
-/* Load or store instructions that may cause exceptions use the EX macro. */
-
-#define EX(insn,reg1,reg2,offset,handler)	\
-9:	insn	reg1, reg2, offset;		\
-	.section __ex_table, "a";		\
-	.word	9b, handler;			\
-	.previous
+#include <linux/linkage.h>
+#include <variant/core.h>
+#include <asm/asmmacro.h>
 
 /*
  * char *__strncpy_user(char *dst, const char *src, size_t len)
@@ -54,10 +48,8 @@
 #   a12/ tmp
 
 .text
-.align	4
-.global	__strncpy_user
-.type	__strncpy_user,@function
-__strncpy_user:
+ENTRY(__strncpy_user)
+
 	entry	sp, 16		# minimal stack frame
 	# a2/ dst, a3/ src, a4/ len
 	mov	a11, a2		# leave dst in return value register
@@ -75,9 +67,9 @@ __strncpy_user:
 	j	.Ldstunaligned
 
 .Lsrc1mod2:	# src address is odd
-	EX(l8ui, a9, a3, 0, fixup_l)	# get byte 0
+EX(11f)	l8ui	a9, a3, 0		# get byte 0
 	addi	a3, a3, 1		# advance src pointer
-	EX(s8i, a9, a11, 0, fixup_s)	# store byte 0
+EX(10f)	s8i	a9, a11, 0		# store byte 0
 	beqz	a9, .Lret		# if byte 0 is zero
 	addi	a11, a11, 1		# advance dst pointer
 	addi	a4, a4, -1		# decrement len
@@ -85,16 +77,16 @@ __strncpy_user:
 	bbci.l	a3, 1, .Lsrcaligned	# if src is now word-aligned
 
 .Lsrc2mod4:	# src address is 2 mod 4
-	EX(l8ui, a9, a3, 0, fixup_l)	# get byte 0
+EX(11f)	l8ui	a9, a3, 0		# get byte 0
 	/* 1-cycle interlock */
-	EX(s8i, a9, a11, 0, fixup_s)	# store byte 0
+EX(10f)	s8i	a9, a11, 0		# store byte 0
 	beqz	a9, .Lret		# if byte 0 is zero
 	addi	a11, a11, 1		# advance dst pointer
 	addi	a4, a4, -1		# decrement len
 	beqz	a4, .Lret		# if len is zero
-	EX(l8ui, a9, a3, 1, fixup_l)	# get byte 0
+EX(11f)	l8ui	a9, a3, 1		# get byte 0
 	addi	a3, a3, 2		# advance src pointer
-	EX(s8i, a9, a11, 0, fixup_s)	# store byte 0
+EX(10f)	s8i	a9, a11, 0		# store byte 0
 	beqz	a9, .Lret		# if byte 0 is zero
 	addi	a11, a11, 1		# advance dst pointer
 	addi	a4, a4, -1		# decrement len
@@ -117,12 +109,12 @@ __strncpy_user:
 	add	a12, a12, a11	# a12 = end of last 4B chunck
 #endif
 .Loop1:
-	EX(l32i, a9, a3, 0, fixup_l)	# get word from src
+EX(11f)	l32i	a9, a3, 0		# get word from src
 	addi	a3, a3, 4		# advance src pointer
 	bnone	a9, a5, .Lz0		# if byte 0 is zero
 	bnone	a9, a6, .Lz1		# if byte 1 is zero
 	bnone	a9, a7, .Lz2		# if byte 2 is zero
-	EX(s32i, a9, a11, 0, fixup_s)	# store word to dst
+EX(10f)	s32i	a9, a11, 0		# store word to dst
 	bnone	a9, a8, .Lz3		# if byte 3 is zero
 	addi	a11, a11, 4		# advance dst pointer
 #if !XCHAL_HAVE_LOOPS
@@ -132,7 +124,7 @@ __strncpy_user:
 .Loop1done:
 	bbci.l	a4, 1, .L100
 	# copy 2 bytes
-	EX(l16ui, a9, a3, 0, fixup_l)
+EX(11f)	l16ui	a9, a3, 0
 	addi	a3, a3, 2		# advance src pointer
 #ifdef __XTENSA_EB__
 	bnone	a9, a7, .Lz0		# if byte 2 is zero
@@ -141,13 +133,13 @@ __strncpy_user:
 	bnone	a9, a5, .Lz0		# if byte 0 is zero
 	bnone	a9, a6, .Lz1		# if byte 1 is zero
 #endif
-	EX(s16i, a9, a11, 0, fixup_s)
+EX(10f)	s16i	a9, a11, 0
 	addi	a11, a11, 2		# advance dst pointer
 .L100:
 	bbci.l	a4, 0, .Lret
-	EX(l8ui, a9, a3, 0, fixup_l)
+EX(11f)	l8ui	a9, a3, 0
 	/* slot */
-	EX(s8i, a9, a11, 0, fixup_s)
+EX(10f)	s8i	a9, a11, 0
 	beqz	a9, .Lret		# if byte is zero
 	addi	a11, a11, 1-3		# advance dst ptr 1, but also cancel
 					# the effect of adding 3 in .Lz3 code
@@ -161,14 +153,14 @@ __strncpy_user:
 #ifdef __XTENSA_EB__
 	movi	a9, 0
 #endif /* __XTENSA_EB__ */
-	EX(s8i, a9, a11, 0, fixup_s)
+EX(10f)	s8i	a9, a11, 0
 	sub	a2, a11, a2		# compute strlen
 	retw
 .Lz1:	# byte 1 is zero
 #ifdef __XTENSA_EB__
 	extui   a9, a9, 16, 16
 #endif /* __XTENSA_EB__ */
-	EX(s16i, a9, a11, 0, fixup_s)
+EX(10f)	s16i	a9, a11, 0
 	addi	a11, a11, 1		# advance dst pointer
 	sub	a2, a11, a2		# compute strlen
 	retw
@@ -176,9 +168,9 @@ __strncpy_user:
 #ifdef __XTENSA_EB__
 	extui   a9, a9, 16, 16
 #endif /* __XTENSA_EB__ */
-	EX(s16i, a9, a11, 0, fixup_s)
+EX(10f)	s16i	a9, a11, 0
 	movi	a9, 0
-	EX(s8i, a9, a11, 2, fixup_s)
+EX(10f)	s8i	a9, a11, 2
 	addi	a11, a11, 2		# advance dst pointer
 	sub	a2, a11, a2		# compute strlen
 	retw
@@ -196,9 +188,9 @@ __strncpy_user:
 	add	a12, a11, a4		# a12 = ending address
 #endif /* XCHAL_HAVE_LOOPS */
 .Lnextbyte:
-	EX(l8ui, a9, a3, 0, fixup_l)
+EX(11f)	l8ui	a9, a3, 0
 	addi	a3, a3, 1
-	EX(s8i, a9, a11, 0, fixup_s)
+EX(10f)	s8i	a9, a11, 0
 	beqz	a9, .Lunalignedend
 	addi	a11, a11, 1
 #if !XCHAL_HAVE_LOOPS
@@ -209,6 +201,7 @@ __strncpy_user:
 	sub	a2, a11, a2		# compute strlen
 	retw
 
+ENDPROC(__strncpy_user)
 
 	.section .fixup, "ax"
 	.align	4
@@ -218,8 +211,7 @@ __strncpy_user:
 	 * implementation in memset().  Thus, we differentiate between
 	 * load/store fixups. */
 
-fixup_s:
-fixup_l:
+10:
+11:
 	movi	a2, -EFAULT
 	retw
-
diff --git a/arch/xtensa/lib/strnlen_user.S b/arch/xtensa/lib/strnlen_user.S
index 4c03b1e581e9..0b956ce7f386 100644
--- a/arch/xtensa/lib/strnlen_user.S
+++ b/arch/xtensa/lib/strnlen_user.S
@@ -11,15 +11,9 @@
  *  Copyright (C) 2002 Tensilica Inc.
  */
 
+#include <linux/linkage.h>
 #include <variant/core.h>
-
-/* Load or store instructions that may cause exceptions use the EX macro. */
-
-#define EX(insn,reg1,reg2,offset,handler)	\
-9:	insn	reg1, reg2, offset;		\
-	.section __ex_table, "a";		\
-	.word	9b, handler;			\
-	.previous
+#include <asm/asmmacro.h>
 
 /*
  * size_t __strnlen_user(const char *s, size_t len)
@@ -49,10 +43,8 @@
 #   a10/ tmp
 
 .text
-.align	4
-.global	__strnlen_user
-.type	__strnlen_user,@function
-__strnlen_user:
+ENTRY(__strnlen_user)
+
 	entry	sp, 16		# minimal stack frame
 	# a2/ s, a3/ len
 	addi	a4, a2, -4	# because we overincrement at the end;
@@ -77,7 +69,7 @@ __strnlen_user:
 	add	a10, a10, a4	# a10 = end of last 4B chunk
 #endif /* XCHAL_HAVE_LOOPS */
 .Loop:
-	EX(l32i, a9, a4, 4, lenfixup)	# get next word of string
+EX(10f)	l32i	a9, a4, 4		# get next word of string
 	addi	a4, a4, 4		# advance string pointer
 	bnone	a9, a5, .Lz0		# if byte 0 is zero
 	bnone	a9, a6, .Lz1		# if byte 1 is zero
@@ -88,7 +80,7 @@ __strnlen_user:
 #endif
 
 .Ldone:
-	EX(l32i, a9, a4, 4, lenfixup)	# load 4 bytes for remaining checks
+EX(10f)	l32i	a9, a4, 4	# load 4 bytes for remaining checks
 
 	bbci.l	a3, 1, .L100
 	# check two more bytes (bytes 0, 1 of word)
@@ -125,14 +117,14 @@ __strnlen_user:
 	retw
 
 .L1mod2:	# address is odd
-	EX(l8ui, a9, a4, 4, lenfixup)	# get byte 0
+EX(10f)	l8ui	a9, a4, 4		# get byte 0
 	addi	a4, a4, 1		# advance string pointer
 	beqz	a9, .Lz3		# if byte 0 is zero
 	bbci.l	a4, 1, .Laligned	# if string pointer is now word-aligned
 
 .L2mod4:	# address is 2 mod 4
 	addi	a4, a4, 2	# advance ptr for aligned access
-	EX(l32i, a9, a4, 0, lenfixup)	# get word with first two bytes of string
+EX(10f)	l32i	a9, a4, 0	# get word with first two bytes of string
 	bnone	a9, a7, .Lz2	# if byte 2 (of word, not string) is zero
 	bany	a9, a8, .Laligned # if byte 3 (of word, not string) is nonzero
 	# byte 3 is zero
@@ -140,8 +132,10 @@ __strnlen_user:
 	sub	a2, a4, a2	# subtract to get length
 	retw
 
+ENDPROC(__strnlen_user)
+
 	.section .fixup, "ax"
 	.align	4
-lenfixup:
+10:
 	movi	a2, 0
 	retw
diff --git a/arch/xtensa/lib/usercopy.S b/arch/xtensa/lib/usercopy.S
index d9cd766bde3e..64ab1971324f 100644
--- a/arch/xtensa/lib/usercopy.S
+++ b/arch/xtensa/lib/usercopy.S
@@ -53,30 +53,13 @@
  *	a11/ original length
  */
 
+#include <linux/linkage.h>
 #include <variant/core.h>
-
-#ifdef __XTENSA_EB__
-#define ALIGN(R, W0, W1) src	R, W0, W1
-#define SSA8(R)	ssa8b R
-#else
-#define ALIGN(R, W0, W1) src	R, W1, W0
-#define SSA8(R)	ssa8l R
-#endif
-
-/* Load or store instructions that may cause exceptions use the EX macro. */
-
-#define EX(insn,reg1,reg2,offset,handler)	\
-9:	insn	reg1, reg2, offset;		\
-	.section __ex_table, "a";		\
-	.word	9b, handler;			\
-	.previous
-
+#include <asm/asmmacro.h>
 
 	.text
-	.align	4
-	.global	__xtensa_copy_user
-	.type	__xtensa_copy_user,@function
-__xtensa_copy_user:
+ENTRY(__xtensa_copy_user)
+
 	entry	sp, 16		# minimal stack frame
 	# a2/ dst, a3/ src, a4/ len
 	mov	a5, a2		# copy dst so that a2 is return value
@@ -89,7 +72,7 @@ __xtensa_copy_user:
 				# per iteration
 	movi	a8, 3		  # if source is also aligned,
 	bnone	a3, a8, .Laligned # then use word copy
-	SSA8(	a3)		# set shift amount from byte offset
+	__ssa8	a3		# set shift amount from byte offset
 	bnez	a4, .Lsrcunaligned
 	movi	a2, 0		# return success for len==0
 	retw
@@ -102,9 +85,9 @@ __xtensa_copy_user:
 	bltui	a4, 7, .Lbytecopy	# do short copies byte by byte
 
 	# copy 1 byte
-	EX(l8ui, a6, a3, 0, fixup)
+EX(10f)	l8ui	a6, a3, 0
 	addi	a3, a3,  1
-	EX(s8i, a6, a5,  0, fixup)
+EX(10f)	s8i	a6, a5,  0
 	addi	a5, a5,  1
 	addi	a4, a4, -1
 	bbci.l	a5, 1, .Ldstaligned	# if dst is now aligned, then
@@ -112,11 +95,11 @@ __xtensa_copy_user:
 .Ldst2mod4:	# dst 16-bit aligned
 	# copy 2 bytes
 	bltui	a4, 6, .Lbytecopy	# do short copies byte by byte
-	EX(l8ui, a6, a3, 0, fixup)
-	EX(l8ui, a7, a3, 1, fixup)
+EX(10f)	l8ui	a6, a3, 0
+EX(10f)	l8ui	a7, a3, 1
 	addi	a3, a3,  2
-	EX(s8i, a6, a5,  0, fixup)
-	EX(s8i, a7, a5,  1, fixup)
+EX(10f)	s8i	a6, a5,  0
+EX(10f)	s8i	a7, a5,  1
 	addi	a5, a5,  2
 	addi	a4, a4, -2
 	j	.Ldstaligned	# dst is now aligned, return to main algorithm
@@ -135,9 +118,9 @@ __xtensa_copy_user:
 	add	a7, a3, a4	# a7 = end address for source
 #endif /* !XCHAL_HAVE_LOOPS */
 .Lnextbyte:
-	EX(l8ui, a6, a3, 0, fixup)
+EX(10f)	l8ui	a6, a3, 0
 	addi	a3, a3, 1
-	EX(s8i, a6, a5, 0, fixup)
+EX(10f)	s8i	a6, a5, 0
 	addi	a5, a5, 1
 #if !XCHAL_HAVE_LOOPS
 	blt	a3, a7, .Lnextbyte
@@ -161,15 +144,15 @@ __xtensa_copy_user:
 	add	a8, a8, a3	# a8 = end of last 16B source chunk
 #endif /* !XCHAL_HAVE_LOOPS */
 .Loop1:
-	EX(l32i, a6, a3,  0, fixup)
-	EX(l32i, a7, a3,  4, fixup)
-	EX(s32i, a6, a5,  0, fixup)
-	EX(l32i, a6, a3,  8, fixup)
-	EX(s32i, a7, a5,  4, fixup)
-	EX(l32i, a7, a3, 12, fixup)
-	EX(s32i, a6, a5,  8, fixup)
+EX(10f)	l32i	a6, a3,  0
+EX(10f)	l32i	a7, a3,  4
+EX(10f)	s32i	a6, a5,  0
+EX(10f)	l32i	a6, a3,  8
+EX(10f)	s32i	a7, a5,  4
+EX(10f)	l32i	a7, a3, 12
+EX(10f)	s32i	a6, a5,  8
 	addi	a3, a3, 16
-	EX(s32i, a7, a5, 12, fixup)
+EX(10f)	s32i	a7, a5, 12
 	addi	a5, a5, 16
 #if !XCHAL_HAVE_LOOPS
 	blt	a3, a8, .Loop1
@@ -177,31 +160,31 @@ __xtensa_copy_user:
 .Loop1done:
 	bbci.l	a4, 3, .L2
 	# copy 8 bytes
-	EX(l32i, a6, a3,  0, fixup)
-	EX(l32i, a7, a3,  4, fixup)
+EX(10f)	l32i	a6, a3,  0
+EX(10f)	l32i	a7, a3,  4
 	addi	a3, a3,  8
-	EX(s32i, a6, a5,  0, fixup)
-	EX(s32i, a7, a5,  4, fixup)
+EX(10f)	s32i	a6, a5,  0
+EX(10f)	s32i	a7, a5,  4
 	addi	a5, a5,  8
 .L2:
 	bbci.l	a4, 2, .L3
 	# copy 4 bytes
-	EX(l32i, a6, a3,  0, fixup)
+EX(10f)	l32i	a6, a3,  0
 	addi	a3, a3,  4
-	EX(s32i, a6, a5,  0, fixup)
+EX(10f)	s32i	a6, a5,  0
 	addi	a5, a5,  4
 .L3:
 	bbci.l	a4, 1, .L4
 	# copy 2 bytes
-	EX(l16ui, a6, a3,  0, fixup)
+EX(10f)	l16ui	a6, a3,  0
 	addi	a3, a3,  2
-	EX(s16i,  a6, a5,  0, fixup)
+EX(10f)	s16i	a6, a5,  0
 	addi	a5, a5,  2
 .L4:
 	bbci.l	a4, 0, .L5
 	# copy 1 byte
-	EX(l8ui, a6, a3,  0, fixup)
-	EX(s8i,  a6, a5,  0, fixup)
+EX(10f)	l8ui	a6, a3,  0
+EX(10f)	s8i	a6, a5,  0
 .L5:
 	movi	a2, 0		# return success for len bytes copied
 	retw
@@ -217,7 +200,7 @@ __xtensa_copy_user:
 	# copy 16 bytes per iteration for word-aligned dst and unaligned src
 	and	a10, a3, a8	# save unalignment offset for below
 	sub	a3, a3, a10	# align a3 (to avoid sim warnings only; not needed for hardware)
-	EX(l32i, a6, a3, 0, fixup)	# load first word
+EX(10f)	l32i	a6, a3, 0	# load first word
 #if XCHAL_HAVE_LOOPS
 	loopnez	a7, .Loop2done
 #else /* !XCHAL_HAVE_LOOPS */
@@ -226,19 +209,19 @@ __xtensa_copy_user:
 	add	a12, a12, a3	# a12 = end of last 16B source chunk
 #endif /* !XCHAL_HAVE_LOOPS */
 .Loop2:
-	EX(l32i, a7, a3,  4, fixup)
-	EX(l32i, a8, a3,  8, fixup)
-	ALIGN(	a6, a6, a7)
-	EX(s32i, a6, a5,  0, fixup)
-	EX(l32i, a9, a3, 12, fixup)
-	ALIGN(	a7, a7, a8)
-	EX(s32i, a7, a5,  4, fixup)
-	EX(l32i, a6, a3, 16, fixup)
-	ALIGN(	a8, a8, a9)
-	EX(s32i, a8, a5,  8, fixup)
+EX(10f)	l32i	a7, a3,  4
+EX(10f)	l32i	a8, a3,  8
+	__src_b	a6, a6, a7
+EX(10f)	s32i	a6, a5,  0
+EX(10f)	l32i	a9, a3, 12
+	__src_b	a7, a7, a8
+EX(10f)	s32i	a7, a5,  4
+EX(10f)	l32i	a6, a3, 16
+	__src_b	a8, a8, a9
+EX(10f)	s32i	a8, a5,  8
 	addi	a3, a3, 16
-	ALIGN(	a9, a9, a6)
-	EX(s32i, a9, a5, 12, fixup)
+	__src_b	a9, a9, a6
+EX(10f)	s32i	a9, a5, 12
 	addi	a5, a5, 16
 #if !XCHAL_HAVE_LOOPS
 	blt	a3, a12, .Loop2
@@ -246,43 +229,44 @@ __xtensa_copy_user:
 .Loop2done:
 	bbci.l	a4, 3, .L12
 	# copy 8 bytes
-	EX(l32i, a7, a3,  4, fixup)
-	EX(l32i, a8, a3,  8, fixup)
-	ALIGN(	a6, a6, a7)
-	EX(s32i, a6, a5,  0, fixup)
+EX(10f)	l32i	a7, a3,  4
+EX(10f)	l32i	a8, a3,  8
+	__src_b	a6, a6, a7
+EX(10f)	s32i	a6, a5,  0
 	addi	a3, a3,  8
-	ALIGN(	a7, a7, a8)
-	EX(s32i, a7, a5,  4, fixup)
+	__src_b	a7, a7, a8
+EX(10f)	s32i	a7, a5,  4
 	addi	a5, a5,  8
 	mov	a6, a8
 .L12:
 	bbci.l	a4, 2, .L13
 	# copy 4 bytes
-	EX(l32i, a7, a3,  4, fixup)
+EX(10f)	l32i	a7, a3,  4
 	addi	a3, a3,  4
-	ALIGN(	a6, a6, a7)
-	EX(s32i, a6, a5,  0, fixup)
+	__src_b	a6, a6, a7
+EX(10f)	s32i	a6, a5,  0
 	addi	a5, a5,  4
 	mov	a6, a7
 .L13:
 	add	a3, a3, a10	# readjust a3 with correct misalignment
 	bbci.l	a4, 1, .L14
 	# copy 2 bytes
-	EX(l8ui, a6, a3,  0, fixup)
-	EX(l8ui, a7, a3,  1, fixup)
+EX(10f)	l8ui	a6, a3,  0
+EX(10f)	l8ui	a7, a3,  1
 	addi	a3, a3,  2
-	EX(s8i, a6, a5,  0, fixup)
-	EX(s8i, a7, a5,  1, fixup)
+EX(10f)	s8i	a6, a5,  0
+EX(10f)	s8i	a7, a5,  1
 	addi	a5, a5,  2
 .L14:
 	bbci.l	a4, 0, .L15
 	# copy 1 byte
-	EX(l8ui, a6, a3,  0, fixup)
-	EX(s8i,  a6, a5,  0, fixup)
+EX(10f)	l8ui	a6, a3,  0
+EX(10f)	s8i	a6, a5,  0
 .L15:
 	movi	a2, 0		# return success for len bytes copied
 	retw
 
+ENDPROC(__xtensa_copy_user)
 
 	.section .fixup, "ax"
 	.align	4
@@ -294,7 +278,7 @@ __xtensa_copy_user:
  */
 
 
-fixup:
+10:
 	sub	a2, a5, a2	/* a2 <-- bytes copied */
 	sub	a2, a11, a2	/* a2 <-- bytes not copied */
 	retw
diff --git a/arch/xtensa/mm/Makefile b/arch/xtensa/mm/Makefile
index 0b3d296a016a..734888a00dc8 100644
--- a/arch/xtensa/mm/Makefile
+++ b/arch/xtensa/mm/Makefile
@@ -5,3 +5,8 @@
 obj-y			:= init.o misc.o
 obj-$(CONFIG_MMU)	+= cache.o fault.o ioremap.o mmu.o tlb.o
 obj-$(CONFIG_HIGHMEM)	+= highmem.o
+obj-$(CONFIG_KASAN)	+= kasan_init.o
+
+KASAN_SANITIZE_fault.o := n
+KASAN_SANITIZE_kasan_init.o := n
+KASAN_SANITIZE_mmu.o := n
diff --git a/arch/xtensa/mm/cache.c b/arch/xtensa/mm/cache.c
index 3c75c4e597da..57dc231a0709 100644
--- a/arch/xtensa/mm/cache.c
+++ b/arch/xtensa/mm/cache.c
@@ -33,9 +33,6 @@
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 
-//#define printd(x...) printk(x)
-#define printd(x...) do { } while(0)
-
 /* 
  * Note:
  * The kernel provides one architecture bit PG_arch_1 in the page flags that 
diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c
index a14df5aa98c8..8b9b6f44bb06 100644
--- a/arch/xtensa/mm/fault.c
+++ b/arch/xtensa/mm/fault.c
@@ -25,8 +25,6 @@
 DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST;
 void bad_page_fault(struct pt_regs*, unsigned long, int);
 
-#undef DEBUG_PAGE_FAULT
-
 /*
  * This routine handles page faults.  It determines the address,
  * and the problem, and then passes it off to one of the appropriate
@@ -68,10 +66,10 @@ void do_page_fault(struct pt_regs *regs)
 		    exccause == EXCCAUSE_ITLB_MISS ||
 		    exccause == EXCCAUSE_FETCH_CACHE_ATTRIBUTE) ? 1 : 0;
 
-#ifdef DEBUG_PAGE_FAULT
-	printk("[%s:%d:%08x:%d:%08x:%s%s]\n", current->comm, current->pid,
-	       address, exccause, regs->pc, is_write? "w":"", is_exec? "x":"");
-#endif
+	pr_debug("[%s:%d:%08x:%d:%08lx:%s%s]\n",
+		 current->comm, current->pid,
+		 address, exccause, regs->pc,
+		 is_write ? "w" : "", is_exec ? "x" : "");
 
 	if (user_mode(regs))
 		flags |= FAULT_FLAG_USER;
@@ -247,10 +245,8 @@ bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
 
 	/* Are we prepared to handle this kernel fault?  */
 	if ((entry = search_exception_tables(regs->pc)) != NULL) {
-#ifdef DEBUG_PAGE_FAULT
-		printk(KERN_DEBUG "%s: Exception at pc=%#010lx (%lx)\n",
-				current->comm, regs->pc, entry->fixup);
-#endif
+		pr_debug("%s: Exception at pc=%#010lx (%lx)\n",
+			 current->comm, regs->pc, entry->fixup);
 		current->thread.bad_uaddr = address;
 		regs->pc = entry->fixup;
 		return;
@@ -259,9 +255,9 @@ bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
 	/* Oops. The kernel tried to access some bad page. We'll have to
 	 * terminate things with extreme prejudice.
 	 */
-	printk(KERN_ALERT "Unable to handle kernel paging request at virtual "
-	       "address %08lx\n pc = %08lx, ra = %08lx\n",
-	       address, regs->pc, regs->areg[0]);
+	pr_alert("Unable to handle kernel paging request at virtual "
+		 "address %08lx\n pc = %08lx, ra = %08lx\n",
+		 address, regs->pc, regs->areg[0]);
 	die("Oops", regs, sig);
 	do_exit(sig);
 }
diff --git a/arch/xtensa/mm/init.c b/arch/xtensa/mm/init.c
index 720fe4e8b497..34aead7dcb48 100644
--- a/arch/xtensa/mm/init.c
+++ b/arch/xtensa/mm/init.c
@@ -79,19 +79,75 @@ void __init zones_init(void)
 	free_area_init_node(0, zones_size, ARCH_PFN_OFFSET, NULL);
 }
 
+#ifdef CONFIG_HIGHMEM
+static void __init free_area_high(unsigned long pfn, unsigned long end)
+{
+	for (; pfn < end; pfn++)
+		free_highmem_page(pfn_to_page(pfn));
+}
+
+static void __init free_highpages(void)
+{
+	unsigned long max_low = max_low_pfn;
+	struct memblock_region *mem, *res;
+
+	reset_all_zones_managed_pages();
+	/* set highmem page free */
+	for_each_memblock(memory, mem) {
+		unsigned long start = memblock_region_memory_base_pfn(mem);
+		unsigned long end = memblock_region_memory_end_pfn(mem);
+
+		/* Ignore complete lowmem entries */
+		if (end <= max_low)
+			continue;
+
+		if (memblock_is_nomap(mem))
+			continue;
+
+		/* Truncate partial highmem entries */
+		if (start < max_low)
+			start = max_low;
+
+		/* Find and exclude any reserved regions */
+		for_each_memblock(reserved, res) {
+			unsigned long res_start, res_end;
+
+			res_start = memblock_region_reserved_base_pfn(res);
+			res_end = memblock_region_reserved_end_pfn(res);
+
+			if (res_end < start)
+				continue;
+			if (res_start < start)
+				res_start = start;
+			if (res_start > end)
+				res_start = end;
+			if (res_end > end)
+				res_end = end;
+			if (res_start != start)
+				free_area_high(start, res_start);
+			start = res_end;
+			if (start == end)
+				break;
+		}
+
+		/* And now free anything which remains */
+		if (start < end)
+			free_area_high(start, end);
+	}
+}
+#else
+static void __init free_highpages(void)
+{
+}
+#endif
+
 /*
  * Initialize memory pages.
  */
 
 void __init mem_init(void)
 {
-#ifdef CONFIG_HIGHMEM
-	unsigned long tmp;
-
-	reset_all_zones_managed_pages();
-	for (tmp = max_low_pfn; tmp < max_pfn; tmp++)
-		free_highmem_page(pfn_to_page(tmp));
-#endif
+	free_highpages();
 
 	max_mapnr = max_pfn - ARCH_PFN_OFFSET;
 	high_memory = (void *)__va(max_low_pfn << PAGE_SHIFT);
@@ -100,29 +156,51 @@ void __init mem_init(void)
 
 	mem_init_print_info(NULL);
 	pr_info("virtual kernel memory layout:\n"
+#ifdef CONFIG_KASAN
+		"    kasan   : 0x%08lx - 0x%08lx  (%5lu MB)\n"
+#endif
+#ifdef CONFIG_MMU
+		"    vmalloc : 0x%08lx - 0x%08lx  (%5lu MB)\n"
+#endif
 #ifdef CONFIG_HIGHMEM
 		"    pkmap   : 0x%08lx - 0x%08lx  (%5lu kB)\n"
 		"    fixmap  : 0x%08lx - 0x%08lx  (%5lu kB)\n"
 #endif
-#ifdef CONFIG_MMU
-		"    vmalloc : 0x%08lx - 0x%08lx  (%5lu MB)\n"
+		"    lowmem  : 0x%08lx - 0x%08lx  (%5lu MB)\n"
+		"    .text   : 0x%08lx - 0x%08lx  (%5lu kB)\n"
+		"    .rodata : 0x%08lx - 0x%08lx  (%5lu kB)\n"
+		"    .data   : 0x%08lx - 0x%08lx  (%5lu kB)\n"
+		"    .init   : 0x%08lx - 0x%08lx  (%5lu kB)\n"
+		"    .bss    : 0x%08lx - 0x%08lx  (%5lu kB)\n",
+#ifdef CONFIG_KASAN
+		KASAN_SHADOW_START, KASAN_SHADOW_START + KASAN_SHADOW_SIZE,
+		KASAN_SHADOW_SIZE >> 20,
 #endif
-		"    lowmem  : 0x%08lx - 0x%08lx  (%5lu MB)\n",
+#ifdef CONFIG_MMU
+		VMALLOC_START, VMALLOC_END,
+		(VMALLOC_END - VMALLOC_START) >> 20,
 #ifdef CONFIG_HIGHMEM
 		PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE,
 		(LAST_PKMAP*PAGE_SIZE) >> 10,
 		FIXADDR_START, FIXADDR_TOP,
 		(FIXADDR_TOP - FIXADDR_START) >> 10,
 #endif
-#ifdef CONFIG_MMU
-		VMALLOC_START, VMALLOC_END,
-		(VMALLOC_END - VMALLOC_START) >> 20,
 		PAGE_OFFSET, PAGE_OFFSET +
 		(max_low_pfn - min_low_pfn) * PAGE_SIZE,
 #else
 		min_low_pfn * PAGE_SIZE, max_low_pfn * PAGE_SIZE,
 #endif
-		((max_low_pfn - min_low_pfn) * PAGE_SIZE) >> 20);
+		((max_low_pfn - min_low_pfn) * PAGE_SIZE) >> 20,
+		(unsigned long)_text, (unsigned long)_etext,
+		(unsigned long)(_etext - _text) >> 10,
+		(unsigned long)__start_rodata, (unsigned long)_sdata,
+		(unsigned long)(_sdata - __start_rodata) >> 10,
+		(unsigned long)_sdata, (unsigned long)_edata,
+		(unsigned long)(_edata - _sdata) >> 10,
+		(unsigned long)__init_begin, (unsigned long)__init_end,
+		(unsigned long)(__init_end - __init_begin) >> 10,
+		(unsigned long)__bss_start, (unsigned long)__bss_stop,
+		(unsigned long)(__bss_stop - __bss_start) >> 10);
 }
 
 #ifdef CONFIG_BLK_DEV_INITRD
diff --git a/arch/xtensa/mm/kasan_init.c b/arch/xtensa/mm/kasan_init.c
new file mode 100644
index 000000000000..6b532b6bd785
--- /dev/null
+++ b/arch/xtensa/mm/kasan_init.c
@@ -0,0 +1,95 @@
+/*
+ * Xtensa KASAN shadow map initialization
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2017 Cadence Design Systems Inc.
+ */
+
+#include <linux/bootmem.h>
+#include <linux/init_task.h>
+#include <linux/kasan.h>
+#include <linux/kernel.h>
+#include <linux/memblock.h>
+#include <asm/initialize_mmu.h>
+#include <asm/tlbflush.h>
+#include <asm/traps.h>
+
+void __init kasan_early_init(void)
+{
+	unsigned long vaddr = KASAN_SHADOW_START;
+	pgd_t *pgd = pgd_offset_k(vaddr);
+	pmd_t *pmd = pmd_offset(pgd, vaddr);
+	int i;
+
+	for (i = 0; i < PTRS_PER_PTE; ++i)
+		set_pte(kasan_zero_pte + i,
+			mk_pte(virt_to_page(kasan_zero_page), PAGE_KERNEL));
+
+	for (vaddr = 0; vaddr < KASAN_SHADOW_SIZE; vaddr += PMD_SIZE, ++pmd) {
+		BUG_ON(!pmd_none(*pmd));
+		set_pmd(pmd, __pmd((unsigned long)kasan_zero_pte));
+	}
+	early_trap_init();
+}
+
+static void __init populate(void *start, void *end)
+{
+	unsigned long n_pages = (end - start) / PAGE_SIZE;
+	unsigned long n_pmds = n_pages / PTRS_PER_PTE;
+	unsigned long i, j;
+	unsigned long vaddr = (unsigned long)start;
+	pgd_t *pgd = pgd_offset_k(vaddr);
+	pmd_t *pmd = pmd_offset(pgd, vaddr);
+	pte_t *pte = memblock_virt_alloc(n_pages * sizeof(pte_t), PAGE_SIZE);
+
+	pr_debug("%s: %p - %p\n", __func__, start, end);
+
+	for (i = j = 0; i < n_pmds; ++i) {
+		int k;
+
+		for (k = 0; k < PTRS_PER_PTE; ++k, ++j) {
+			phys_addr_t phys =
+				memblock_alloc_base(PAGE_SIZE, PAGE_SIZE,
+						    MEMBLOCK_ALLOC_ANYWHERE);
+
+			set_pte(pte + j, pfn_pte(PHYS_PFN(phys), PAGE_KERNEL));
+		}
+	}
+
+	for (i = 0; i < n_pmds ; ++i, pte += PTRS_PER_PTE)
+		set_pmd(pmd + i, __pmd((unsigned long)pte));
+
+	local_flush_tlb_all();
+	memset(start, 0, end - start);
+}
+
+void __init kasan_init(void)
+{
+	int i;
+
+	BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_START -
+		     (KASAN_START_VADDR >> KASAN_SHADOW_SCALE_SHIFT));
+	BUILD_BUG_ON(VMALLOC_START < KASAN_START_VADDR);
+
+	/*
+	 * Replace shadow map pages that cover addresses from VMALLOC area
+	 * start to the end of KSEG with clean writable pages.
+	 */
+	populate(kasan_mem_to_shadow((void *)VMALLOC_START),
+		 kasan_mem_to_shadow((void *)XCHAL_KSEG_BYPASS_VADDR));
+
+	/* Write protect kasan_zero_page and zero-initialize it again. */
+	for (i = 0; i < PTRS_PER_PTE; ++i)
+		set_pte(kasan_zero_pte + i,
+			mk_pte(virt_to_page(kasan_zero_page), PAGE_KERNEL_RO));
+
+	local_flush_tlb_all();
+	memset(kasan_zero_page, 0, PAGE_SIZE);
+
+	/* At this point kasan is fully initialized. Enable error messages. */
+	current->kasan_depth = 0;
+	pr_info("KernelAddressSanitizer initialized\n");
+}
diff --git a/arch/xtensa/mm/mmu.c b/arch/xtensa/mm/mmu.c
index 358d748d9083..9d1ecfc53670 100644
--- a/arch/xtensa/mm/mmu.c
+++ b/arch/xtensa/mm/mmu.c
@@ -56,7 +56,6 @@ static void __init fixedrange_init(void)
 
 void __init paging_init(void)
 {
-	memset(swapper_pg_dir, 0, PAGE_SIZE);
 #ifdef CONFIG_HIGHMEM
 	fixedrange_init();
 	pkmap_page_table = init_pmd(PKMAP_BASE, LAST_PKMAP);
@@ -82,6 +81,23 @@ void init_mmu(void)
 	set_itlbcfg_register(0);
 	set_dtlbcfg_register(0);
 #endif
+	init_kio();
+	local_flush_tlb_all();
+
+	/* Set rasid register to a known value. */
+
+	set_rasid_register(ASID_INSERT(ASID_USER_FIRST));
+
+	/* Set PTEVADDR special register to the start of the page
+	 * table, which is in kernel mappable space (ie. not
+	 * statically mapped).  This register's value is undefined on
+	 * reset.
+	 */
+	set_ptevaddr_register(XCHAL_PAGE_TABLE_VADDR);
+}
+
+void init_kio(void)
+{
 #if XCHAL_HAVE_PTP_MMU && XCHAL_HAVE_SPANNING_WAY && defined(CONFIG_OF)
 	/*
 	 * Update the IO area mapping in case xtensa_kio_paddr has changed
@@ -95,17 +111,4 @@ void init_mmu(void)
 	write_itlb_entry(__pte(xtensa_kio_paddr + CA_BYPASS),
 			XCHAL_KIO_BYPASS_VADDR + 6);
 #endif
-
-	local_flush_tlb_all();
-
-	/* Set rasid register to a known value. */
-
-	set_rasid_register(ASID_INSERT(ASID_USER_FIRST));
-
-	/* Set PTEVADDR special register to the start of the page
-	 * table, which is in kernel mappable space (ie. not
-	 * statically mapped).  This register's value is undefined on
-	 * reset.
-	 */
-	set_ptevaddr_register(PGTABLE_START);
 }
diff --git a/arch/xtensa/mm/tlb.c b/arch/xtensa/mm/tlb.c
index 35c822286bbe..59153d0aa890 100644
--- a/arch/xtensa/mm/tlb.c
+++ b/arch/xtensa/mm/tlb.c
@@ -95,10 +95,8 @@ void local_flush_tlb_range(struct vm_area_struct *vma,
 	if (mm->context.asid[cpu] == NO_CONTEXT)
 		return;
 
-#if 0
-	printk("[tlbrange<%02lx,%08lx,%08lx>]\n",
-			(unsigned long)mm->context.asid[cpu], start, end);
-#endif
+	pr_debug("[tlbrange<%02lx,%08lx,%08lx>]\n",
+		 (unsigned long)mm->context.asid[cpu], start, end);
 	local_irq_save(flags);
 
 	if (end-start + (PAGE_SIZE-1) <= _TLB_ENTRIES << PAGE_SHIFT) {
diff --git a/arch/xtensa/platforms/iss/console.c b/arch/xtensa/platforms/iss/console.c
index 464c2684c4f1..92f567f9a21e 100644
--- a/arch/xtensa/platforms/iss/console.c
+++ b/arch/xtensa/platforms/iss/console.c
@@ -185,7 +185,7 @@ int __init rs_init(void)
 
 	serial_driver = alloc_tty_driver(SERIAL_MAX_NUM_LINES);
 
-	printk ("%s %s\n", serial_name, serial_version);
+	pr_info("%s %s\n", serial_name, serial_version);
 
 	/* Initialize the tty_driver structure */
 
@@ -214,7 +214,7 @@ static __exit void rs_exit(void)
 	int error;
 
 	if ((error = tty_unregister_driver(serial_driver)))
-		printk("ISS_SERIAL: failed to unregister serial driver (%d)\n",
+		pr_err("ISS_SERIAL: failed to unregister serial driver (%d)\n",
 		       error);
 	put_tty_driver(serial_driver);
 	tty_port_destroy(&serial_port);
diff --git a/arch/xtensa/platforms/iss/network.c b/arch/xtensa/platforms/iss/network.c
index 6363b18e5b8c..d027dddc41ca 100644
--- a/arch/xtensa/platforms/iss/network.c
+++ b/arch/xtensa/platforms/iss/network.c
@@ -16,6 +16,8 @@
  *
  */
 
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
 #include <linux/list.h>
 #include <linux/irq.h>
 #include <linux/spinlock.h>
@@ -606,8 +608,6 @@ struct iss_net_init {
  * those fields. They will be later initialized in iss_net_init.
  */
 
-#define ERR KERN_ERR "iss_net_setup: "
-
 static int __init iss_net_setup(char *str)
 {
 	struct iss_net_private *device = NULL;
@@ -619,14 +619,14 @@ static int __init iss_net_setup(char *str)
 
 	end = strchr(str, '=');
 	if (!end) {
-		printk(ERR "Expected '=' after device number\n");
+		pr_err("Expected '=' after device number\n");
 		return 1;
 	}
 	*end = 0;
 	rc = kstrtouint(str, 0, &n);
 	*end = '=';
 	if (rc < 0) {
-		printk(ERR "Failed to parse '%s'\n", str);
+		pr_err("Failed to parse '%s'\n", str);
 		return 1;
 	}
 	str = end;
@@ -642,13 +642,13 @@ static int __init iss_net_setup(char *str)
 	spin_unlock(&devices_lock);
 
 	if (device && device->index == n) {
-		printk(ERR "Device %u already configured\n", n);
+		pr_err("Device %u already configured\n", n);
 		return 1;
 	}
 
 	new = alloc_bootmem(sizeof(*new));
 	if (new == NULL) {
-		printk(ERR "Alloc_bootmem failed\n");
+		pr_err("Alloc_bootmem failed\n");
 		return 1;
 	}
 
@@ -660,8 +660,6 @@ static int __init iss_net_setup(char *str)
 	return 1;
 }
 
-#undef ERR
-
 __setup("eth", iss_net_setup);
 
 /*
diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c
index da1525ec4c87..d819dc77fe65 100644
--- a/block/bfq-cgroup.c
+++ b/block/bfq-cgroup.c
@@ -775,10 +775,11 @@ static void bfq_pd_offline(struct blkg_policy_data *pd)
 	unsigned long flags;
 	int i;
 
+	spin_lock_irqsave(&bfqd->lock, flags);
+
 	if (!entity) /* root group */
-		return;
+		goto put_async_queues;
 
-	spin_lock_irqsave(&bfqd->lock, flags);
 	/*
 	 * Empty all service_trees belonging to this group before
 	 * deactivating the group itself.
@@ -809,6 +810,8 @@ static void bfq_pd_offline(struct blkg_policy_data *pd)
 	}
 
 	__bfq_deactivate_entity(entity, false);
+
+put_async_queues:
 	bfq_put_async_queues(bfqd, bfqg);
 
 	spin_unlock_irqrestore(&bfqd->lock, flags);
diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c
index bcb6d21baf12..aeca22d91101 100644
--- a/block/bfq-iosched.c
+++ b/block/bfq-iosched.c
@@ -166,6 +166,20 @@ static const int bfq_async_charge_factor = 10;
 /* Default timeout values, in jiffies, approximating CFQ defaults. */
 const int bfq_timeout = HZ / 8;
 
+/*
+ * Time limit for merging (see comments in bfq_setup_cooperator). Set
+ * to the slowest value that, in our tests, proved to be effective in
+ * removing false positives, while not causing true positives to miss
+ * queue merging.
+ *
+ * As can be deduced from the low time limit below, queue merging, if
+ * successful, happens at the very beggining of the I/O of the involved
+ * cooperating processes, as a consequence of the arrival of the very
+ * first requests from each cooperator.  After that, there is very
+ * little chance to find cooperators.
+ */
+static const unsigned long bfq_merge_time_limit = HZ/10;
+
 static struct kmem_cache *bfq_pool;
 
 /* Below this threshold (in ns), we consider thinktime immediate. */
@@ -178,7 +192,7 @@ static struct kmem_cache *bfq_pool;
 #define BFQQ_SEEK_THR		(sector_t)(8 * 100)
 #define BFQQ_SECT_THR_NONROT	(sector_t)(2 * 32)
 #define BFQQ_CLOSE_THR		(sector_t)(8 * 1024)
-#define BFQQ_SEEKY(bfqq)	(hweight32(bfqq->seek_history) > 32/8)
+#define BFQQ_SEEKY(bfqq)	(hweight32(bfqq->seek_history) > 19)
 
 /* Min number of samples required to perform peak-rate update */
 #define BFQ_RATE_MIN_SAMPLES	32
@@ -195,15 +209,17 @@ static struct kmem_cache *bfq_pool;
  * interactive applications automatically, using the following formula:
  * duration = (R / r) * T, where r is the peak rate of the device, and
  * R and T are two reference parameters.
- * In particular, R is the peak rate of the reference device (see below),
- * and T is a reference time: given the systems that are likely to be
- * installed on the reference device according to its speed class, T is
- * about the maximum time needed, under BFQ and while reading two files in
- * parallel, to load typical large applications on these systems.
- * In practice, the slower/faster the device at hand is, the more/less it
- * takes to load applications with respect to the reference device.
- * Accordingly, the longer/shorter BFQ grants weight raising to interactive
- * applications.
+ * In particular, R is the peak rate of the reference device (see
+ * below), and T is a reference time: given the systems that are
+ * likely to be installed on the reference device according to its
+ * speed class, T is about the maximum time needed, under BFQ and
+ * while reading two files in parallel, to load typical large
+ * applications on these systems (see the comments on
+ * max_service_from_wr below, for more details on how T is obtained).
+ * In practice, the slower/faster the device at hand is, the more/less
+ * it takes to load applications with respect to the reference device.
+ * Accordingly, the longer/shorter BFQ grants weight raising to
+ * interactive applications.
  *
  * BFQ uses four different reference pairs (R, T), depending on:
  * . whether the device is rotational or non-rotational;
@@ -240,6 +256,60 @@ static int T_slow[2];
 static int T_fast[2];
 static int device_speed_thresh[2];
 
+/*
+ * BFQ uses the above-detailed, time-based weight-raising mechanism to
+ * privilege interactive tasks. This mechanism is vulnerable to the
+ * following false positives: I/O-bound applications that will go on
+ * doing I/O for much longer than the duration of weight
+ * raising. These applications have basically no benefit from being
+ * weight-raised at the beginning of their I/O. On the opposite end,
+ * while being weight-raised, these applications
+ * a) unjustly steal throughput to applications that may actually need
+ * low latency;
+ * b) make BFQ uselessly perform device idling; device idling results
+ * in loss of device throughput with most flash-based storage, and may
+ * increase latencies when used purposelessly.
+ *
+ * BFQ tries to reduce these problems, by adopting the following
+ * countermeasure. To introduce this countermeasure, we need first to
+ * finish explaining how the duration of weight-raising for
+ * interactive tasks is computed.
+ *
+ * For a bfq_queue deemed as interactive, the duration of weight
+ * raising is dynamically adjusted, as a function of the estimated
+ * peak rate of the device, so as to be equal to the time needed to
+ * execute the 'largest' interactive task we benchmarked so far. By
+ * largest task, we mean the task for which each involved process has
+ * to do more I/O than for any of the other tasks we benchmarked. This
+ * reference interactive task is the start-up of LibreOffice Writer,
+ * and in this task each process/bfq_queue needs to have at most ~110K
+ * sectors transferred.
+ *
+ * This last piece of information enables BFQ to reduce the actual
+ * duration of weight-raising for at least one class of I/O-bound
+ * applications: those doing sequential or quasi-sequential I/O. An
+ * example is file copy. In fact, once started, the main I/O-bound
+ * processes of these applications usually consume the above 110K
+ * sectors in much less time than the processes of an application that
+ * is starting, because these I/O-bound processes will greedily devote
+ * almost all their CPU cycles only to their target,
+ * throughput-friendly I/O operations. This is even more true if BFQ
+ * happens to be underestimating the device peak rate, and thus
+ * overestimating the duration of weight raising. But, according to
+ * our measurements, once transferred 110K sectors, these processes
+ * have no right to be weight-raised any longer.
+ *
+ * Basing on the last consideration, BFQ ends weight-raising for a
+ * bfq_queue if the latter happens to have received an amount of
+ * service at least equal to the following constant. The constant is
+ * set to slightly more than 110K, to have a minimum safety margin.
+ *
+ * This early ending of weight-raising reduces the amount of time
+ * during which interactive false positives cause the two problems
+ * described at the beginning of these comments.
+ */
+static const unsigned long max_service_from_wr = 120000;
+
 #define RQ_BIC(rq)		icq_to_bic((rq)->elv.priv[0])
 #define RQ_BFQQ(rq)		((rq)->elv.priv[1])
 
@@ -403,6 +473,82 @@ static struct request *bfq_choose_req(struct bfq_data *bfqd,
 	}
 }
 
+/*
+ * See the comments on bfq_limit_depth for the purpose of
+ * the depths set in the function.
+ */
+static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt)
+{
+	bfqd->sb_shift = bt->sb.shift;
+
+	/*
+	 * In-word depths if no bfq_queue is being weight-raised:
+	 * leaving 25% of tags only for sync reads.
+	 *
+	 * In next formulas, right-shift the value
+	 * (1U<<bfqd->sb_shift), instead of computing directly
+	 * (1U<<(bfqd->sb_shift - something)), to be robust against
+	 * any possible value of bfqd->sb_shift, without having to
+	 * limit 'something'.
+	 */
+	/* no more than 50% of tags for async I/O */
+	bfqd->word_depths[0][0] = max((1U<<bfqd->sb_shift)>>1, 1U);
+	/*
+	 * no more than 75% of tags for sync writes (25% extra tags
+	 * w.r.t. async I/O, to prevent async I/O from starving sync
+	 * writes)
+	 */
+	bfqd->word_depths[0][1] = max(((1U<<bfqd->sb_shift) * 3)>>2, 1U);
+
+	/*
+	 * In-word depths in case some bfq_queue is being weight-
+	 * raised: leaving ~63% of tags for sync reads. This is the
+	 * highest percentage for which, in our tests, application
+	 * start-up times didn't suffer from any regression due to tag
+	 * shortage.
+	 */
+	/* no more than ~18% of tags for async I/O */
+	bfqd->word_depths[1][0] = max(((1U<<bfqd->sb_shift) * 3)>>4, 1U);
+	/* no more than ~37% of tags for sync writes (~20% extra tags) */
+	bfqd->word_depths[1][1] = max(((1U<<bfqd->sb_shift) * 6)>>4, 1U);
+}
+
+/*
+ * Async I/O can easily starve sync I/O (both sync reads and sync
+ * writes), by consuming all tags. Similarly, storms of sync writes,
+ * such as those that sync(2) may trigger, can starve sync reads.
+ * Limit depths of async I/O and sync writes so as to counter both
+ * problems.
+ */
+static void bfq_limit_depth(unsigned int op, struct blk_mq_alloc_data *data)
+{
+	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
+	struct bfq_data *bfqd = data->q->elevator->elevator_data;
+	struct sbitmap_queue *bt;
+
+	if (op_is_sync(op) && !op_is_write(op))
+		return;
+
+	if (data->flags & BLK_MQ_REQ_RESERVED) {
+		if (unlikely(!tags->nr_reserved_tags)) {
+			WARN_ON_ONCE(1);
+			return;
+		}
+		bt = &tags->breserved_tags;
+	} else
+		bt = &tags->bitmap_tags;
+
+	if (unlikely(bfqd->sb_shift != bt->sb.shift))
+		bfq_update_depths(bfqd, bt);
+
+	data->shallow_depth =
+		bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(op)];
+
+	bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u",
+			__func__, bfqd->wr_busy_queues, op_is_sync(op),
+			data->shallow_depth);
+}
+
 static struct bfq_queue *
 bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root,
 		     sector_t sector, struct rb_node **ret_parent,
@@ -444,6 +590,13 @@ bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root,
 	return bfqq;
 }
 
+static bool bfq_too_late_for_merging(struct bfq_queue *bfqq)
+{
+	return bfqq->service_from_backlogged > 0 &&
+		time_is_before_jiffies(bfqq->first_IO_time +
+				       bfq_merge_time_limit);
+}
+
 void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 {
 	struct rb_node **p, *parent;
@@ -454,6 +607,14 @@ void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 		bfqq->pos_root = NULL;
 	}
 
+	/*
+	 * bfqq cannot be merged any longer (see comments in
+	 * bfq_setup_cooperator): no point in adding bfqq into the
+	 * position tree.
+	 */
+	if (bfq_too_late_for_merging(bfqq))
+		return;
+
 	if (bfq_class_idle(bfqq))
 		return;
 	if (!bfqq->next_rq)
@@ -1247,6 +1408,7 @@ static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd,
 	if (old_wr_coeff == 1 && wr_or_deserves_wr) {
 		/* start a weight-raising period */
 		if (interactive) {
+			bfqq->service_from_wr = 0;
 			bfqq->wr_coeff = bfqd->bfq_wr_coeff;
 			bfqq->wr_cur_max_time = bfq_wr_duration(bfqd);
 		} else {
@@ -1627,6 +1789,8 @@ static void bfq_remove_request(struct request_queue *q,
 			rb_erase(&bfqq->pos_node, bfqq->pos_root);
 			bfqq->pos_root = NULL;
 		}
+	} else {
+		bfq_pos_tree_add_move(bfqd, bfqq);
 	}
 
 	if (rq->cmd_flags & REQ_META)
@@ -1933,6 +2097,9 @@ bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq)
 static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
 					struct bfq_queue *new_bfqq)
 {
+	if (bfq_too_late_for_merging(new_bfqq))
+		return false;
+
 	if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) ||
 	    (bfqq->ioprio_class != new_bfqq->ioprio_class))
 		return false;
@@ -1956,20 +2123,6 @@ static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq,
 	return true;
 }
 
-/*
- * If this function returns true, then bfqq cannot be merged. The idea
- * is that true cooperation happens very early after processes start
- * to do I/O. Usually, late cooperations are just accidental false
- * positives. In case bfqq is weight-raised, such false positives
- * would evidently degrade latency guarantees for bfqq.
- */
-static bool wr_from_too_long(struct bfq_queue *bfqq)
-{
-	return bfqq->wr_coeff > 1 &&
-		time_is_before_jiffies(bfqq->last_wr_start_finish +
-				       msecs_to_jiffies(100));
-}
-
 /*
  * Attempt to schedule a merge of bfqq with the currently in-service
  * queue or with a close queue among the scheduled queues.  Return
@@ -1983,11 +2136,6 @@ static bool wr_from_too_long(struct bfq_queue *bfqq)
  * to maintain. Besides, in such a critical condition as an out of memory,
  * the benefits of queue merging may be little relevant, or even negligible.
  *
- * Weight-raised queues can be merged only if their weight-raising
- * period has just started. In fact cooperating processes are usually
- * started together. Thus, with this filter we avoid false positives
- * that would jeopardize low-latency guarantees.
- *
  * WARNING: queue merging may impair fairness among non-weight raised
  * queues, for at least two reasons: 1) the original weight of a
  * merged queue may change during the merged state, 2) even being the
@@ -2001,12 +2149,24 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 {
 	struct bfq_queue *in_service_bfqq, *new_bfqq;
 
+	/*
+	 * Prevent bfqq from being merged if it has been created too
+	 * long ago. The idea is that true cooperating processes, and
+	 * thus their associated bfq_queues, are supposed to be
+	 * created shortly after each other. This is the case, e.g.,
+	 * for KVM/QEMU and dump I/O threads. Basing on this
+	 * assumption, the following filtering greatly reduces the
+	 * probability that two non-cooperating processes, which just
+	 * happen to do close I/O for some short time interval, have
+	 * their queues merged by mistake.
+	 */
+	if (bfq_too_late_for_merging(bfqq))
+		return NULL;
+
 	if (bfqq->new_bfqq)
 		return bfqq->new_bfqq;
 
-	if (!io_struct ||
-	    wr_from_too_long(bfqq) ||
-	    unlikely(bfqq == &bfqd->oom_bfqq))
+	if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq))
 		return NULL;
 
 	/* If there is only one backlogged queue, don't search. */
@@ -2015,12 +2175,9 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 
 	in_service_bfqq = bfqd->in_service_queue;
 
-	if (!in_service_bfqq || in_service_bfqq == bfqq
-	    || wr_from_too_long(in_service_bfqq) ||
-	    unlikely(in_service_bfqq == &bfqd->oom_bfqq))
-		goto check_scheduled;
-
-	if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) &&
+	if (in_service_bfqq && in_service_bfqq != bfqq &&
+	    likely(in_service_bfqq != &bfqd->oom_bfqq) &&
+	    bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) &&
 	    bfqq->entity.parent == in_service_bfqq->entity.parent &&
 	    bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) {
 		new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq);
@@ -2032,12 +2189,10 @@ bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 	 * queues. The only thing we need is that the bio/request is not
 	 * NULL, as we need it to establish whether a cooperator exists.
 	 */
-check_scheduled:
 	new_bfqq = bfq_find_close_cooperator(bfqd, bfqq,
 			bfq_io_struct_pos(io_struct, request));
 
-	if (new_bfqq && !wr_from_too_long(new_bfqq) &&
-	    likely(new_bfqq != &bfqd->oom_bfqq) &&
+	if (new_bfqq && likely(new_bfqq != &bfqd->oom_bfqq) &&
 	    bfq_may_be_close_cooperator(bfqq, new_bfqq))
 		return bfq_setup_merge(bfqq, new_bfqq);
 
@@ -2062,7 +2217,8 @@ static void bfq_bfqq_save_state(struct bfq_queue *bfqq)
 	bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq);
 	bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node);
 	if (unlikely(bfq_bfqq_just_created(bfqq) &&
-		     !bfq_bfqq_in_large_burst(bfqq))) {
+		     !bfq_bfqq_in_large_burst(bfqq) &&
+		     bfqq->bfqd->low_latency)) {
 		/*
 		 * bfqq being merged right after being created: bfqq
 		 * would have deserved interactive weight raising, but
@@ -2917,45 +3073,87 @@ static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  * whereas soft_rt_next_start is set to infinity for applications that do
  * not.
  *
- * Unfortunately, even a greedy application may happen to behave in an
- * isochronous way if the CPU load is high. In fact, the application may
- * stop issuing requests while the CPUs are busy serving other processes,
- * then restart, then stop again for a while, and so on. In addition, if
- * the disk achieves a low enough throughput with the request pattern
- * issued by the application (e.g., because the request pattern is random
- * and/or the device is slow), then the application may meet the above
- * bandwidth requirement too. To prevent such a greedy application to be
- * deemed as soft real-time, a further rule is used in the computation of
- * soft_rt_next_start: soft_rt_next_start must be higher than the current
- * time plus the maximum time for which the arrival of a request is waited
- * for when a sync queue becomes idle, namely bfqd->bfq_slice_idle.
- * This filters out greedy applications, as the latter issue instead their
- * next request as soon as possible after the last one has been completed
- * (in contrast, when a batch of requests is completed, a soft real-time
- * application spends some time processing data).
+ * Unfortunately, even a greedy (i.e., I/O-bound) application may
+ * happen to meet, occasionally or systematically, both the above
+ * bandwidth and isochrony requirements. This may happen at least in
+ * the following circumstances. First, if the CPU load is high. The
+ * application may stop issuing requests while the CPUs are busy
+ * serving other processes, then restart, then stop again for a while,
+ * and so on. The other circumstances are related to the storage
+ * device: the storage device is highly loaded or reaches a low-enough
+ * throughput with the I/O of the application (e.g., because the I/O
+ * is random and/or the device is slow). In all these cases, the
+ * I/O of the application may be simply slowed down enough to meet
+ * the bandwidth and isochrony requirements. To reduce the probability
+ * that greedy applications are deemed as soft real-time in these
+ * corner cases, a further rule is used in the computation of
+ * soft_rt_next_start: the return value of this function is forced to
+ * be higher than the maximum between the following two quantities.
  *
- * Unfortunately, the last filter may easily generate false positives if
- * only bfqd->bfq_slice_idle is used as a reference time interval and one
- * or both the following cases occur:
- * 1) HZ is so low that the duration of a jiffy is comparable to or higher
- *    than bfqd->bfq_slice_idle. This happens, e.g., on slow devices with
- *    HZ=100.
+ * (a) Current time plus: (1) the maximum time for which the arrival
+ *     of a request is waited for when a sync queue becomes idle,
+ *     namely bfqd->bfq_slice_idle, and (2) a few extra jiffies. We
+ *     postpone for a moment the reason for adding a few extra
+ *     jiffies; we get back to it after next item (b).  Lower-bounding
+ *     the return value of this function with the current time plus
+ *     bfqd->bfq_slice_idle tends to filter out greedy applications,
+ *     because the latter issue their next request as soon as possible
+ *     after the last one has been completed. In contrast, a soft
+ *     real-time application spends some time processing data, after a
+ *     batch of its requests has been completed.
+ *
+ * (b) Current value of bfqq->soft_rt_next_start. As pointed out
+ *     above, greedy applications may happen to meet both the
+ *     bandwidth and isochrony requirements under heavy CPU or
+ *     storage-device load. In more detail, in these scenarios, these
+ *     applications happen, only for limited time periods, to do I/O
+ *     slowly enough to meet all the requirements described so far,
+ *     including the filtering in above item (a). These slow-speed
+ *     time intervals are usually interspersed between other time
+ *     intervals during which these applications do I/O at a very high
+ *     speed. Fortunately, exactly because of the high speed of the
+ *     I/O in the high-speed intervals, the values returned by this
+ *     function happen to be so high, near the end of any such
+ *     high-speed interval, to be likely to fall *after* the end of
+ *     the low-speed time interval that follows. These high values are
+ *     stored in bfqq->soft_rt_next_start after each invocation of
+ *     this function. As a consequence, if the last value of
+ *     bfqq->soft_rt_next_start is constantly used to lower-bound the
+ *     next value that this function may return, then, from the very
+ *     beginning of a low-speed interval, bfqq->soft_rt_next_start is
+ *     likely to be constantly kept so high that any I/O request
+ *     issued during the low-speed interval is considered as arriving
+ *     to soon for the application to be deemed as soft
+ *     real-time. Then, in the high-speed interval that follows, the
+ *     application will not be deemed as soft real-time, just because
+ *     it will do I/O at a high speed. And so on.
+ *
+ * Getting back to the filtering in item (a), in the following two
+ * cases this filtering might be easily passed by a greedy
+ * application, if the reference quantity was just
+ * bfqd->bfq_slice_idle:
+ * 1) HZ is so low that the duration of a jiffy is comparable to or
+ *    higher than bfqd->bfq_slice_idle. This happens, e.g., on slow
+ *    devices with HZ=100. The time granularity may be so coarse
+ *    that the approximation, in jiffies, of bfqd->bfq_slice_idle
+ *    is rather lower than the exact value.
  * 2) jiffies, instead of increasing at a constant rate, may stop increasing
  *    for a while, then suddenly 'jump' by several units to recover the lost
  *    increments. This seems to happen, e.g., inside virtual machines.
- * To address this issue, we do not use as a reference time interval just
- * bfqd->bfq_slice_idle, but bfqd->bfq_slice_idle plus a few jiffies. In
- * particular we add the minimum number of jiffies for which the filter
- * seems to be quite precise also in embedded systems and KVM/QEMU virtual
- * machines.
+ * To address this issue, in the filtering in (a) we do not use as a
+ * reference time interval just bfqd->bfq_slice_idle, but
+ * bfqd->bfq_slice_idle plus a few jiffies. In particular, we add the
+ * minimum number of jiffies for which the filter seems to be quite
+ * precise also in embedded systems and KVM/QEMU virtual machines.
  */
 static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd,
 						struct bfq_queue *bfqq)
 {
-	return max(bfqq->last_idle_bklogged +
-		   HZ * bfqq->service_from_backlogged /
-		   bfqd->bfq_wr_max_softrt_rate,
-		   jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4);
+	return max3(bfqq->soft_rt_next_start,
+		    bfqq->last_idle_bklogged +
+		    HZ * bfqq->service_from_backlogged /
+		    bfqd->bfq_wr_max_softrt_rate,
+		    jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4);
 }
 
 /**
@@ -2999,17 +3197,6 @@ void bfq_bfqq_expire(struct bfq_data *bfqd,
 	 */
 	slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta);
 
-	/*
-	 * Increase service_from_backlogged before next statement,
-	 * because the possible next invocation of
-	 * bfq_bfqq_charge_time would likely inflate
-	 * entity->service. In contrast, service_from_backlogged must
-	 * contain real service, to enable the soft real-time
-	 * heuristic to correctly compute the bandwidth consumed by
-	 * bfqq.
-	 */
-	bfqq->service_from_backlogged += entity->service;
-
 	/*
 	 * As above explained, charge slow (typically seeky) and
 	 * timed-out queues with the time and not the service
@@ -3535,6 +3722,12 @@ static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq)
 				bfqq->entity.prio_changed = 1;
 			}
 		}
+		if (bfqq->wr_coeff > 1 &&
+		    bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time &&
+		    bfqq->service_from_wr > max_service_from_wr) {
+			/* see comments on max_service_from_wr */
+			bfq_bfqq_end_wr(bfqq);
+		}
 	}
 	/*
 	 * To improve latency (for this or other queues), immediately
@@ -3630,20 +3823,22 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
 		}
 
 		/*
-		 * We exploit the put_rq_private hook to decrement
-		 * rq_in_driver, but put_rq_private will not be
-		 * invoked on this request. So, to avoid unbalance,
-		 * just start this request, without incrementing
-		 * rq_in_driver. As a negative consequence,
-		 * rq_in_driver is deceptively lower than it should be
-		 * while this request is in service. This may cause
-		 * bfq_schedule_dispatch to be invoked uselessly.
+		 * We exploit the bfq_finish_requeue_request hook to
+		 * decrement rq_in_driver, but
+		 * bfq_finish_requeue_request will not be invoked on
+		 * this request. So, to avoid unbalance, just start
+		 * this request, without incrementing rq_in_driver. As
+		 * a negative consequence, rq_in_driver is deceptively
+		 * lower than it should be while this request is in
+		 * service. This may cause bfq_schedule_dispatch to be
+		 * invoked uselessly.
 		 *
 		 * As for implementing an exact solution, the
-		 * put_request hook, if defined, is probably invoked
-		 * also on this request. So, by exploiting this hook,
-		 * we could 1) increment rq_in_driver here, and 2)
-		 * decrement it in put_request. Such a solution would
+		 * bfq_finish_requeue_request hook, if defined, is
+		 * probably invoked also on this request. So, by
+		 * exploiting this hook, we could 1) increment
+		 * rq_in_driver here, and 2) decrement it in
+		 * bfq_finish_requeue_request. Such a solution would
 		 * let the value of the counter be always accurate,
 		 * but it would entail using an extra interface
 		 * function. This cost seems higher than the benefit,
@@ -3689,35 +3884,16 @@ static struct request *__bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
 	return rq;
 }
 
-static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
-{
-	struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
-	struct request *rq;
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
-	struct bfq_queue *in_serv_queue, *bfqq;
-	bool waiting_rq, idle_timer_disabled;
-#endif
-
-	spin_lock_irq(&bfqd->lock);
-
 #if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
-	in_serv_queue = bfqd->in_service_queue;
-	waiting_rq = in_serv_queue && bfq_bfqq_wait_request(in_serv_queue);
-
-	rq = __bfq_dispatch_request(hctx);
-
-	idle_timer_disabled =
-		waiting_rq && !bfq_bfqq_wait_request(in_serv_queue);
-
-#else
-	rq = __bfq_dispatch_request(hctx);
-#endif
-	spin_unlock_irq(&bfqd->lock);
+static void bfq_update_dispatch_stats(struct request_queue *q,
+				      struct request *rq,
+				      struct bfq_queue *in_serv_queue,
+				      bool idle_timer_disabled)
+{
+	struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL;
 
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
-	bfqq = rq ? RQ_BFQQ(rq) : NULL;
 	if (!idle_timer_disabled && !bfqq)
-		return rq;
+		return;
 
 	/*
 	 * rq and bfqq are guaranteed to exist until this function
@@ -3732,7 +3908,7 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
 	 * In addition, the following queue lock guarantees that
 	 * bfqq_group(bfqq) exists as well.
 	 */
-	spin_lock_irq(hctx->queue->queue_lock);
+	spin_lock_irq(q->queue_lock);
 	if (idle_timer_disabled)
 		/*
 		 * Since the idle timer has been disabled,
@@ -3751,9 +3927,37 @@ static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
 		bfqg_stats_set_start_empty_time(bfqg);
 		bfqg_stats_update_io_remove(bfqg, rq->cmd_flags);
 	}
-	spin_unlock_irq(hctx->queue->queue_lock);
+	spin_unlock_irq(q->queue_lock);
+}
+#else
+static inline void bfq_update_dispatch_stats(struct request_queue *q,
+					     struct request *rq,
+					     struct bfq_queue *in_serv_queue,
+					     bool idle_timer_disabled) {}
 #endif
 
+static struct request *bfq_dispatch_request(struct blk_mq_hw_ctx *hctx)
+{
+	struct bfq_data *bfqd = hctx->queue->elevator->elevator_data;
+	struct request *rq;
+	struct bfq_queue *in_serv_queue;
+	bool waiting_rq, idle_timer_disabled;
+
+	spin_lock_irq(&bfqd->lock);
+
+	in_serv_queue = bfqd->in_service_queue;
+	waiting_rq = in_serv_queue && bfq_bfqq_wait_request(in_serv_queue);
+
+	rq = __bfq_dispatch_request(hctx);
+
+	idle_timer_disabled =
+		waiting_rq && !bfq_bfqq_wait_request(in_serv_queue);
+
+	spin_unlock_irq(&bfqd->lock);
+
+	bfq_update_dispatch_stats(hctx->queue, rq, in_serv_queue,
+				  idle_timer_disabled);
+
 	return rq;
 }
 
@@ -4002,10 +4206,15 @@ static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq,
 	bfqq->split_time = bfq_smallest_from_now();
 
 	/*
-	 * Set to the value for which bfqq will not be deemed as
-	 * soft rt when it becomes backlogged.
+	 * To not forget the possibly high bandwidth consumed by a
+	 * process/queue in the recent past,
+	 * bfq_bfqq_softrt_next_start() returns a value at least equal
+	 * to the current value of bfqq->soft_rt_next_start (see
+	 * comments on bfq_bfqq_softrt_next_start).  Set
+	 * soft_rt_next_start to now, to mean that bfqq has consumed
+	 * no bandwidth so far.
 	 */
-	bfqq->soft_rt_next_start = bfq_greatest_from_now();
+	bfqq->soft_rt_next_start = jiffies;
 
 	/* first request is almost certainly seeky */
 	bfqq->seek_history = 1;
@@ -4276,16 +4485,48 @@ static bool __bfq_insert_request(struct bfq_data *bfqd, struct request *rq)
 	return idle_timer_disabled;
 }
 
+#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
+static void bfq_update_insert_stats(struct request_queue *q,
+				    struct bfq_queue *bfqq,
+				    bool idle_timer_disabled,
+				    unsigned int cmd_flags)
+{
+	if (!bfqq)
+		return;
+
+	/*
+	 * bfqq still exists, because it can disappear only after
+	 * either it is merged with another queue, or the process it
+	 * is associated with exits. But both actions must be taken by
+	 * the same process currently executing this flow of
+	 * instructions.
+	 *
+	 * In addition, the following queue lock guarantees that
+	 * bfqq_group(bfqq) exists as well.
+	 */
+	spin_lock_irq(q->queue_lock);
+	bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags);
+	if (idle_timer_disabled)
+		bfqg_stats_update_idle_time(bfqq_group(bfqq));
+	spin_unlock_irq(q->queue_lock);
+}
+#else
+static inline void bfq_update_insert_stats(struct request_queue *q,
+					   struct bfq_queue *bfqq,
+					   bool idle_timer_disabled,
+					   unsigned int cmd_flags) {}
+#endif
+
+static void bfq_prepare_request(struct request *rq, struct bio *bio);
+
 static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 			       bool at_head)
 {
 	struct request_queue *q = hctx->queue;
 	struct bfq_data *bfqd = q->elevator->elevator_data;
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
 	struct bfq_queue *bfqq = RQ_BFQQ(rq);
 	bool idle_timer_disabled = false;
 	unsigned int cmd_flags;
-#endif
 
 	spin_lock_irq(&bfqd->lock);
 	if (blk_mq_sched_try_insert_merge(q, rq)) {
@@ -4304,7 +4545,18 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 		else
 			list_add_tail(&rq->queuelist, &bfqd->dispatch);
 	} else {
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
+		if (WARN_ON_ONCE(!bfqq)) {
+			/*
+			 * This should never happen. Most likely rq is
+			 * a requeued regular request, being
+			 * re-inserted without being first
+			 * re-prepared. Do a prepare, to avoid
+			 * failure.
+			 */
+			bfq_prepare_request(rq, rq->bio);
+			bfqq = RQ_BFQQ(rq);
+		}
+
 		idle_timer_disabled = __bfq_insert_request(bfqd, rq);
 		/*
 		 * Update bfqq, because, if a queue merge has occurred
@@ -4312,9 +4564,6 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 		 * redirected into a new queue.
 		 */
 		bfqq = RQ_BFQQ(rq);
-#else
-		__bfq_insert_request(bfqd, rq);
-#endif
 
 		if (rq_mergeable(rq)) {
 			elv_rqhash_add(q, rq);
@@ -4323,35 +4572,17 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 		}
 	}
 
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
 	/*
 	 * Cache cmd_flags before releasing scheduler lock, because rq
 	 * may disappear afterwards (for example, because of a request
 	 * merge).
 	 */
 	cmd_flags = rq->cmd_flags;
-#endif
+
 	spin_unlock_irq(&bfqd->lock);
 
-#if defined(CONFIG_BFQ_GROUP_IOSCHED) && defined(CONFIG_DEBUG_BLK_CGROUP)
-	if (!bfqq)
-		return;
-	/*
-	 * bfqq still exists, because it can disappear only after
-	 * either it is merged with another queue, or the process it
-	 * is associated with exits. But both actions must be taken by
-	 * the same process currently executing this flow of
-	 * instruction.
-	 *
-	 * In addition, the following queue lock guarantees that
-	 * bfqq_group(bfqq) exists as well.
-	 */
-	spin_lock_irq(q->queue_lock);
-	bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags);
-	if (idle_timer_disabled)
-		bfqg_stats_update_idle_time(bfqq_group(bfqq));
-	spin_unlock_irq(q->queue_lock);
-#endif
+	bfq_update_insert_stats(q, bfqq, idle_timer_disabled,
+				cmd_flags);
 }
 
 static void bfq_insert_requests(struct blk_mq_hw_ctx *hctx,
@@ -4482,22 +4713,44 @@ static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd)
 		bfq_schedule_dispatch(bfqd);
 }
 
-static void bfq_put_rq_priv_body(struct bfq_queue *bfqq)
+static void bfq_finish_requeue_request_body(struct bfq_queue *bfqq)
 {
 	bfqq->allocated--;
 
 	bfq_put_queue(bfqq);
 }
 
-static void bfq_finish_request(struct request *rq)
+/*
+ * Handle either a requeue or a finish for rq. The things to do are
+ * the same in both cases: all references to rq are to be dropped. In
+ * particular, rq is considered completed from the point of view of
+ * the scheduler.
+ */
+static void bfq_finish_requeue_request(struct request *rq)
 {
-	struct bfq_queue *bfqq;
+	struct bfq_queue *bfqq = RQ_BFQQ(rq);
 	struct bfq_data *bfqd;
 
-	if (!rq->elv.icq)
+	/*
+	 * Requeue and finish hooks are invoked in blk-mq without
+	 * checking whether the involved request is actually still
+	 * referenced in the scheduler. To handle this fact, the
+	 * following two checks make this function exit in case of
+	 * spurious invocations, for which there is nothing to do.
+	 *
+	 * First, check whether rq has nothing to do with an elevator.
+	 */
+	if (unlikely(!(rq->rq_flags & RQF_ELVPRIV)))
+		return;
+
+	/*
+	 * rq either is not associated with any icq, or is an already
+	 * requeued request that has not (yet) been re-inserted into
+	 * a bfq_queue.
+	 */
+	if (!rq->elv.icq || !bfqq)
 		return;
 
-	bfqq = RQ_BFQQ(rq);
 	bfqd = bfqq->bfqd;
 
 	if (rq->rq_flags & RQF_STARTED)
@@ -4512,13 +4765,14 @@ static void bfq_finish_request(struct request *rq)
 		spin_lock_irqsave(&bfqd->lock, flags);
 
 		bfq_completed_request(bfqq, bfqd);
-		bfq_put_rq_priv_body(bfqq);
+		bfq_finish_requeue_request_body(bfqq);
 
 		spin_unlock_irqrestore(&bfqd->lock, flags);
 	} else {
 		/*
 		 * Request rq may be still/already in the scheduler,
-		 * in which case we need to remove it. And we cannot
+		 * in which case we need to remove it (this should
+		 * never happen in case of requeue). And we cannot
 		 * defer such a check and removal, to avoid
 		 * inconsistencies in the time interval from the end
 		 * of this function to the start of the deferred work.
@@ -4533,9 +4787,26 @@ static void bfq_finish_request(struct request *rq)
 			bfqg_stats_update_io_remove(bfqq_group(bfqq),
 						    rq->cmd_flags);
 		}
-		bfq_put_rq_priv_body(bfqq);
+		bfq_finish_requeue_request_body(bfqq);
 	}
 
+	/*
+	 * Reset private fields. In case of a requeue, this allows
+	 * this function to correctly do nothing if it is spuriously
+	 * invoked again on this same request (see the check at the
+	 * beginning of the function). Probably, a better general
+	 * design would be to prevent blk-mq from invoking the requeue
+	 * or finish hooks of an elevator, for a request that is not
+	 * referred by that elevator.
+	 *
+	 * Resetting the following fields would break the
+	 * request-insertion logic if rq is re-inserted into a bfq
+	 * internal queue, without a re-preparation. Here we assume
+	 * that re-insertions of requeued requests, without
+	 * re-preparation, can happen only for pass_through or at_head
+	 * requests (which are not re-inserted into bfq internal
+	 * queues).
+	 */
 	rq->elv.priv[0] = NULL;
 	rq->elv.priv[1] = NULL;
 }
@@ -4818,6 +5089,9 @@ static void bfq_exit_queue(struct elevator_queue *e)
 	hrtimer_cancel(&bfqd->idle_slice_timer);
 
 #ifdef CONFIG_BFQ_GROUP_IOSCHED
+	/* release oom-queue reference to root group */
+	bfqg_and_blkg_put(bfqd->root_group);
+
 	blkcg_deactivate_policy(bfqd->queue, &blkcg_policy_bfq);
 #else
 	spin_lock_irq(&bfqd->lock);
@@ -5206,8 +5480,10 @@ static struct elv_fs_entry bfq_attrs[] = {
 
 static struct elevator_type iosched_bfq_mq = {
 	.ops.mq = {
+		.limit_depth		= bfq_limit_depth,
 		.prepare_request	= bfq_prepare_request,
-		.finish_request		= bfq_finish_request,
+		.requeue_request        = bfq_finish_requeue_request,
+		.finish_request		= bfq_finish_requeue_request,
 		.exit_icq		= bfq_exit_icq,
 		.insert_requests	= bfq_insert_requests,
 		.dispatch_request	= bfq_dispatch_request,
diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h
index 91c4390903a1..350c39ae2896 100644
--- a/block/bfq-iosched.h
+++ b/block/bfq-iosched.h
@@ -337,6 +337,11 @@ struct bfq_queue {
 	 * last transition from idle to backlogged.
 	 */
 	unsigned long service_from_backlogged;
+	/*
+	 * Cumulative service received from the @bfq_queue since its
+	 * last transition to weight-raised state.
+	 */
+	unsigned long service_from_wr;
 
 	/*
 	 * Value of wr start time when switching to soft rt
@@ -344,6 +349,8 @@ struct bfq_queue {
 	unsigned long wr_start_at_switch_to_srt;
 
 	unsigned long split_time; /* time of last split */
+
+	unsigned long first_IO_time; /* time of first I/O for this queue */
 };
 
 /**
@@ -627,6 +634,18 @@ struct bfq_data {
 	struct bfq_io_cq *bio_bic;
 	/* bfqq associated with the task issuing current bio for merging */
 	struct bfq_queue *bio_bfqq;
+
+	/*
+	 * Cached sbitmap shift, used to compute depth limits in
+	 * bfq_update_depths.
+	 */
+	unsigned int sb_shift;
+
+	/*
+	 * Depth limits used in bfq_limit_depth (see comments on the
+	 * function)
+	 */
+	unsigned int word_depths[2][2];
 };
 
 enum bfqq_state_flags {
diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c
index e495d3f9b4b0..4498c43245e2 100644
--- a/block/bfq-wf2q.c
+++ b/block/bfq-wf2q.c
@@ -835,6 +835,13 @@ void bfq_bfqq_served(struct bfq_queue *bfqq, int served)
 	struct bfq_entity *entity = &bfqq->entity;
 	struct bfq_service_tree *st;
 
+	if (!bfqq->service_from_backlogged)
+		bfqq->first_IO_time = jiffies;
+
+	if (bfqq->wr_coeff > 1)
+		bfqq->service_from_wr += served;
+
+	bfqq->service_from_backlogged += served;
 	for_each_entity(entity) {
 		st = bfq_entity_service_tree(entity);
 
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 23b42e8aa03e..9cfdd6c83b5b 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -374,7 +374,6 @@ static void bio_integrity_verify_fn(struct work_struct *work)
 /**
  * __bio_integrity_endio - Integrity I/O completion function
  * @bio:	Protected bio
- * @error:	Pointer to errno
  *
  * Description: Completion for integrity I/O
  *
diff --git a/block/bio.c b/block/bio.c
index 9ef6cf3addb3..e1708db48258 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -970,34 +970,6 @@ void bio_advance(struct bio *bio, unsigned bytes)
 }
 EXPORT_SYMBOL(bio_advance);
 
-/**
- * bio_alloc_pages - allocates a single page for each bvec in a bio
- * @bio: bio to allocate pages for
- * @gfp_mask: flags for allocation
- *
- * Allocates pages up to @bio->bi_vcnt.
- *
- * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
- * freed.
- */
-int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
-{
-	int i;
-	struct bio_vec *bv;
-
-	bio_for_each_segment_all(bv, bio, i) {
-		bv->bv_page = alloc_page(gfp_mask);
-		if (!bv->bv_page) {
-			while (--bv >= bio->bi_io_vec)
-				__free_page(bv->bv_page);
-			return -ENOMEM;
-		}
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(bio_alloc_pages);
-
 /**
  * bio_copy_data - copy contents of data buffers from one chain of bios to
  * another
@@ -1838,7 +1810,7 @@ struct bio *bio_split(struct bio *bio, int sectors,
 	bio_advance(bio, split->bi_iter.bi_size);
 
 	if (bio_flagged(bio, BIO_TRACE_COMPLETION))
-		bio_set_flag(bio, BIO_TRACE_COMPLETION);
+		bio_set_flag(split, BIO_TRACE_COMPLETION);
 
 	return split;
 }
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 4117524ca45b..c2033a232a44 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -812,7 +812,6 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
 	struct gendisk *disk;
 	struct request_queue *q;
 	struct blkcg_gq *blkg;
-	struct module *owner;
 	unsigned int major, minor;
 	int key_len, part, ret;
 	char *body;
@@ -904,9 +903,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
 	spin_unlock_irq(q->queue_lock);
 	rcu_read_unlock();
 fail:
-	owner = disk->fops->owner;
-	put_disk(disk);
-	module_put(owner);
+	put_disk_and_module(disk);
 	/*
 	 * If queue was bypassing, we should retry.  Do so after a
 	 * short msleep().  It isn't strictly necessary but queue
@@ -931,13 +928,9 @@ EXPORT_SYMBOL_GPL(blkg_conf_prep);
 void blkg_conf_finish(struct blkg_conf_ctx *ctx)
 	__releases(ctx->disk->queue->queue_lock) __releases(rcu)
 {
-	struct module *owner;
-
 	spin_unlock_irq(ctx->disk->queue->queue_lock);
 	rcu_read_unlock();
-	owner = ctx->disk->fops->owner;
-	put_disk(ctx->disk);
-	module_put(owner);
+	put_disk_and_module(ctx->disk);
 }
 EXPORT_SYMBOL_GPL(blkg_conf_finish);
 
diff --git a/block/blk-core.c b/block/blk-core.c
index 3ba4326a63b5..6d82c4f7fadd 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -34,6 +34,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/blk-cgroup.h>
 #include <linux/debugfs.h>
+#include <linux/bpf.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/block.h>
@@ -126,6 +127,8 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
 	rq->start_time = jiffies;
 	set_start_time_ns(rq);
 	rq->part = NULL;
+	seqcount_init(&rq->gstate_seq);
+	u64_stats_init(&rq->aborted_gstate_sync);
 }
 EXPORT_SYMBOL(blk_rq_init);
 
@@ -143,6 +146,7 @@ static const struct {
 	[BLK_STS_MEDIUM]	= { -ENODATA,	"critical medium" },
 	[BLK_STS_PROTECTION]	= { -EILSEQ,	"protection" },
 	[BLK_STS_RESOURCE]	= { -ENOMEM,	"kernel resource" },
+	[BLK_STS_DEV_RESOURCE]	= { -EBUSY,	"device resource" },
 	[BLK_STS_AGAIN]		= { -EAGAIN,	"nonblocking retry" },
 
 	/* device mapper special case, should not leak out: */
@@ -699,6 +703,15 @@ void blk_cleanup_queue(struct request_queue *q)
 	queue_flag_set(QUEUE_FLAG_DEAD, q);
 	spin_unlock_irq(lock);
 
+	/*
+	 * make sure all in-progress dispatch are completed because
+	 * blk_freeze_queue() can only complete all requests, and
+	 * dispatch may still be in-progress since we dispatch requests
+	 * from more than one contexts
+	 */
+	if (q->mq_ops)
+		blk_mq_quiesce_queue(q);
+
 	/* for synchronous bio-based driver finish in-flight integrity i/o */
 	blk_flush_integrity();
 
@@ -1646,6 +1659,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)
 
 	lockdep_assert_held(q->queue_lock);
 
+	blk_req_zone_write_unlock(req);
 	blk_pm_put_request(req);
 
 	elv_completed_request(q, req);
@@ -2055,6 +2069,29 @@ static inline bool should_fail_request(struct hd_struct *part,
 
 #endif /* CONFIG_FAIL_MAKE_REQUEST */
 
+static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
+{
+	if (part->policy && op_is_write(bio_op(bio))) {
+		char b[BDEVNAME_SIZE];
+
+		printk(KERN_ERR
+		       "generic_make_request: Trying to write "
+			"to read-only block-device %s (partno %d)\n",
+			bio_devname(bio, b), part->partno);
+		return true;
+	}
+
+	return false;
+}
+
+static noinline int should_fail_bio(struct bio *bio)
+{
+	if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
+		return -EIO;
+	return 0;
+}
+ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
+
 /*
  * Remap block n of partition p to block n+start(p) of the disk.
  */
@@ -2063,27 +2100,28 @@ static inline int blk_partition_remap(struct bio *bio)
 	struct hd_struct *p;
 	int ret = 0;
 
+	rcu_read_lock();
+	p = __disk_get_part(bio->bi_disk, bio->bi_partno);
+	if (unlikely(!p || should_fail_request(p, bio->bi_iter.bi_size) ||
+		     bio_check_ro(bio, p))) {
+		ret = -EIO;
+		goto out;
+	}
+
 	/*
 	 * Zone reset does not include bi_size so bio_sectors() is always 0.
 	 * Include a test for the reset op code and perform the remap if needed.
 	 */
-	if (!bio->bi_partno ||
-	    (!bio_sectors(bio) && bio_op(bio) != REQ_OP_ZONE_RESET))
-		return 0;
+	if (!bio_sectors(bio) && bio_op(bio) != REQ_OP_ZONE_RESET)
+		goto out;
 
-	rcu_read_lock();
-	p = __disk_get_part(bio->bi_disk, bio->bi_partno);
-	if (likely(p && !should_fail_request(p, bio->bi_iter.bi_size))) {
-		bio->bi_iter.bi_sector += p->start_sect;
-		bio->bi_partno = 0;
-		trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
-				bio->bi_iter.bi_sector - p->start_sect);
-	} else {
-		printk("%s: fail for partition %d\n", __func__, bio->bi_partno);
-		ret = -EIO;
-	}
-	rcu_read_unlock();
+	bio->bi_iter.bi_sector += p->start_sect;
+	bio->bi_partno = 0;
+	trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
+			      bio->bi_iter.bi_sector - p->start_sect);
 
+out:
+	rcu_read_unlock();
 	return ret;
 }
 
@@ -2142,15 +2180,19 @@ generic_make_request_checks(struct bio *bio)
 	 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
 	 * if queue is not a request based queue.
 	 */
-
 	if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_rq_based(q))
 		goto not_supported;
 
-	if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
+	if (should_fail_bio(bio))
 		goto end_io;
 
-	if (blk_partition_remap(bio))
-		goto end_io;
+	if (!bio->bi_partno) {
+		if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
+			goto end_io;
+	} else {
+		if (blk_partition_remap(bio))
+			goto end_io;
+	}
 
 	if (bio_check_eod(bio, nr_sectors))
 		goto end_io;
@@ -2392,7 +2434,7 @@ blk_qc_t submit_bio(struct bio *bio)
 		unsigned int count;
 
 		if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
-			count = queue_logical_block_size(bio->bi_disk->queue);
+			count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
 		else
 			count = bio_sectors(bio);
 
@@ -2493,8 +2535,7 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *
 		 * bypass a potential scheduler on the bottom device for
 		 * insert.
 		 */
-		blk_mq_request_bypass_insert(rq, true);
-		return BLK_STS_OK;
+		return blk_mq_request_issue_directly(rq);
 	}
 
 	spin_lock_irqsave(q->queue_lock, flags);
@@ -2846,7 +2887,7 @@ void blk_start_request(struct request *req)
 		wbt_issue(req->q->rq_wb, &req->issue_stat);
 	}
 
-	BUG_ON(test_bit(REQ_ATOM_COMPLETE, &req->atomic_flags));
+	BUG_ON(blk_rq_is_complete(req));
 	blk_add_timer(req);
 }
 EXPORT_SYMBOL(blk_start_request);
@@ -3251,6 +3292,8 @@ void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
 {
 	if (bio_has_data(bio))
 		rq->nr_phys_segments = bio_phys_segments(q, bio);
+	else if (bio_op(bio) == REQ_OP_DISCARD)
+		rq->nr_phys_segments = 1;
 
 	rq->__data_len = bio->bi_iter.bi_size;
 	rq->bio = rq->biotail = bio;
@@ -3415,20 +3458,6 @@ int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
 }
 EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
 
-int kblockd_schedule_delayed_work(struct delayed_work *dwork,
-				  unsigned long delay)
-{
-	return queue_delayed_work(kblockd_workqueue, dwork, delay);
-}
-EXPORT_SYMBOL(kblockd_schedule_delayed_work);
-
-int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork,
-				     unsigned long delay)
-{
-	return queue_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
-}
-EXPORT_SYMBOL(kblockd_schedule_delayed_work_on);
-
 /**
  * blk_start_plug - initialize blk_plug and track it inside the task_struct
  * @plug:	The &struct blk_plug that needs to be initialized
diff --git a/block/blk-exec.c b/block/blk-exec.c
index 5c0f3dc446dc..f7b292f12449 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -61,7 +61,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
 	 * be reused after dying flag is set
 	 */
 	if (q->mq_ops) {
-		blk_mq_sched_insert_request(rq, at_head, true, false, false);
+		blk_mq_sched_insert_request(rq, at_head, true, false);
 		return;
 	}
 
diff --git a/block/blk-lib.c b/block/blk-lib.c
index 2bc544ce3d2e..a676084d4740 100644
--- a/block/blk-lib.c
+++ b/block/blk-lib.c
@@ -37,6 +37,9 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 	if (!q)
 		return -ENXIO;
 
+	if (bdev_read_only(bdev))
+		return -EPERM;
+
 	if (flags & BLKDEV_DISCARD_SECURE) {
 		if (!blk_queue_secure_erase(q))
 			return -EOPNOTSUPP;
@@ -156,6 +159,9 @@ static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
 	if (!q)
 		return -ENXIO;
 
+	if (bdev_read_only(bdev))
+		return -EPERM;
+
 	bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
 	if ((sector | nr_sects) & bs_mask)
 		return -EINVAL;
@@ -233,6 +239,9 @@ static int __blkdev_issue_write_zeroes(struct block_device *bdev,
 	if (!q)
 		return -ENXIO;
 
+	if (bdev_read_only(bdev))
+		return -EPERM;
+
 	/* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */
 	max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev);
 
@@ -287,6 +296,9 @@ static int __blkdev_issue_zero_pages(struct block_device *bdev,
 	if (!q)
 		return -ENXIO;
 
+	if (bdev_read_only(bdev))
+		return -EPERM;
+
 	while (nr_sects != 0) {
 		bio = next_bio(bio, __blkdev_sectors_to_bio_pages(nr_sects),
 			       gfp_mask);
diff --git a/block/blk-map.c b/block/blk-map.c
index d3a94719f03f..db9373bd31ac 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -119,7 +119,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
 	unsigned long align = q->dma_pad_mask | queue_dma_alignment(q);
 	struct bio *bio = NULL;
 	struct iov_iter i;
-	int ret;
+	int ret = -EINVAL;
 
 	if (!iter_is_iovec(iter))
 		goto fail;
@@ -148,7 +148,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
 	__blk_rq_unmap_user(bio);
 fail:
 	rq->bio = NULL;
-	return -EINVAL;
+	return ret;
 }
 EXPORT_SYMBOL(blk_rq_map_user_iov);
 
diff --git a/block/blk-merge.c b/block/blk-merge.c
index f5dedd57dff6..782940c65d8a 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -128,9 +128,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
 				nsegs++;
 				sectors = max_sectors;
 			}
-			if (sectors)
-				goto split;
-			/* Make this single bvec as the 1st segment */
+			goto split;
 		}
 
 		if (bvprvp && blk_queue_cluster(q)) {
@@ -146,22 +144,21 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
 			bvprvp = &bvprv;
 			sectors += bv.bv_len >> 9;
 
-			if (nsegs == 1 && seg_size > front_seg_size)
-				front_seg_size = seg_size;
 			continue;
 		}
 new_segment:
 		if (nsegs == queue_max_segments(q))
 			goto split;
 
+		if (nsegs == 1 && seg_size > front_seg_size)
+			front_seg_size = seg_size;
+
 		nsegs++;
 		bvprv = bv;
 		bvprvp = &bvprv;
 		seg_size = bv.bv_len;
 		sectors += bv.bv_len >> 9;
 
-		if (nsegs == 1 && seg_size > front_seg_size)
-			front_seg_size = seg_size;
 	}
 
 	do_split = false;
@@ -174,6 +171,8 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
 			bio = new;
 	}
 
+	if (nsegs == 1 && seg_size > front_seg_size)
+		front_seg_size = seg_size;
 	bio->bi_seg_front_size = front_seg_size;
 	if (seg_size > bio->bi_seg_back_size)
 		bio->bi_seg_back_size = seg_size;
@@ -551,6 +550,24 @@ static bool req_no_special_merge(struct request *req)
 	return !q->mq_ops && req->special;
 }
 
+static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
+		struct request *next)
+{
+	unsigned short segments = blk_rq_nr_discard_segments(req);
+
+	if (segments >= queue_max_discard_segments(q))
+		goto no_merge;
+	if (blk_rq_sectors(req) + bio_sectors(next->bio) >
+	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
+		goto no_merge;
+
+	req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
+	return true;
+no_merge:
+	req_set_nomerge(q, req);
+	return false;
+}
+
 static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
 				struct request *next)
 {
@@ -684,9 +701,13 @@ static struct request *attempt_merge(struct request_queue *q,
 	 * If we are allowed to merge, then append bio list
 	 * from next to rq and release next. merge_requests_fn
 	 * will have updated segment counts, update sector
-	 * counts here.
+	 * counts here. Handle DISCARDs separately, as they
+	 * have separate settings.
 	 */
-	if (!ll_merge_requests_fn(q, req, next))
+	if (req_op(req) == REQ_OP_DISCARD) {
+		if (!req_attempt_discard_merge(q, req, next))
+			return NULL;
+	} else if (!ll_merge_requests_fn(q, req, next))
 		return NULL;
 
 	/*
@@ -716,7 +737,8 @@ static struct request *attempt_merge(struct request_queue *q,
 
 	req->__data_len += blk_rq_bytes(next);
 
-	elv_merge_requests(q, req, next);
+	if (req_op(req) != REQ_OP_DISCARD)
+		elv_merge_requests(q, req, next);
 
 	/*
 	 * 'next' is going away, so update stats accordingly
diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
index b56a4f35720d..21cbc1f071c6 100644
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -289,17 +289,12 @@ static const char *const rqf_name[] = {
 	RQF_NAME(HASHED),
 	RQF_NAME(STATS),
 	RQF_NAME(SPECIAL_PAYLOAD),
+	RQF_NAME(ZONE_WRITE_LOCKED),
+	RQF_NAME(MQ_TIMEOUT_EXPIRED),
+	RQF_NAME(MQ_POLL_SLEPT),
 };
 #undef RQF_NAME
 
-#define RQAF_NAME(name) [REQ_ATOM_##name] = #name
-static const char *const rqaf_name[] = {
-	RQAF_NAME(COMPLETE),
-	RQAF_NAME(STARTED),
-	RQAF_NAME(POLL_SLEPT),
-};
-#undef RQAF_NAME
-
 int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq)
 {
 	const struct blk_mq_ops *const mq_ops = rq->q->mq_ops;
@@ -316,8 +311,7 @@ int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq)
 	seq_puts(m, ", .rq_flags=");
 	blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name,
 		       ARRAY_SIZE(rqf_name));
-	seq_puts(m, ", .atomic_flags=");
-	blk_flags_show(m, rq->atomic_flags, rqaf_name, ARRAY_SIZE(rqaf_name));
+	seq_printf(m, ", complete=%d", blk_rq_is_complete(rq));
 	seq_printf(m, ", .tag=%d, .internal_tag=%d", rq->tag,
 		   rq->internal_tag);
 	if (mq_ops->show_rq)
@@ -409,7 +403,7 @@ static void hctx_show_busy_rq(struct request *rq, void *data, bool reserved)
 	const struct show_busy_params *params = data;
 
 	if (blk_mq_map_queue(rq->q, rq->mq_ctx->cpu) == params->hctx &&
-	    test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
+	    blk_mq_rq_state(rq) != MQ_RQ_IDLE)
 		__blk_mq_debugfs_rq_show(params->m,
 					 list_entry_rq(&rq->queuelist));
 }
@@ -703,7 +697,11 @@ static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf,
 	const struct blk_mq_debugfs_attr *attr = m->private;
 	void *data = d_inode(file->f_path.dentry->d_parent)->i_private;
 
-	if (!attr->write)
+	/*
+	 * Attributes that only implement .seq_ops are read-only and 'attr' is
+	 * the same with 'data' in this case.
+	 */
+	if (attr == data || !attr->write)
 		return -EPERM;
 
 	return attr->write(data, buf, count, ppos);
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index c117bd8fd1f6..25c14c58385c 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -172,7 +172,6 @@ static void blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
 	WRITE_ONCE(hctx->dispatch_from, ctx);
 }
 
-/* return true if hw queue need to be run again */
 void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
 {
 	struct request_queue *q = hctx->queue;
@@ -260,6 +259,8 @@ bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
 		if (!*merged_request)
 			elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
 		return true;
+	case ELEVATOR_DISCARD_MERGE:
+		return bio_attempt_discard_merge(q, rq, bio);
 	default:
 		return false;
 	}
@@ -428,7 +429,7 @@ void blk_mq_sched_restart(struct blk_mq_hw_ctx *const hctx)
 }
 
 void blk_mq_sched_insert_request(struct request *rq, bool at_head,
-				 bool run_queue, bool async, bool can_block)
+				 bool run_queue, bool async)
 {
 	struct request_queue *q = rq->q;
 	struct elevator_queue *e = q->elevator;
diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h
index ba1d1418a96d..1e9c9018ace1 100644
--- a/block/blk-mq-sched.h
+++ b/block/blk-mq-sched.h
@@ -18,7 +18,7 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq);
 void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx);
 
 void blk_mq_sched_insert_request(struct request *rq, bool at_head,
-				 bool run_queue, bool async, bool can_block);
+				 bool run_queue, bool async);
 void blk_mq_sched_insert_requests(struct request_queue *q,
 				  struct blk_mq_ctx *ctx,
 				  struct list_head *list, bool run_queue_async);
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index 79969c3c234f..a54b4b070f1c 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -248,7 +248,7 @@ static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
 	return ret;
 }
 
-static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
+void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
 {
 	struct blk_mq_hw_ctx *hctx;
 	int i;
@@ -265,13 +265,6 @@ static void __blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
 	q->mq_sysfs_init_done = false;
 }
 
-void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
-{
-	mutex_lock(&q->sysfs_lock);
-	__blk_mq_unregister_dev(dev, q);
-	mutex_unlock(&q->sysfs_lock);
-}
-
 void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
 {
 	kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index c81b40ecd3f1..336dde07b230 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -134,12 +134,6 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
 	ws = bt_wait_ptr(bt, data->hctx);
 	drop_ctx = data->ctx == NULL;
 	do {
-		prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE);
-
-		tag = __blk_mq_get_tag(data, bt);
-		if (tag != -1)
-			break;
-
 		/*
 		 * We're out of tags on this hardware queue, kick any
 		 * pending IO submits before going to sleep waiting for
@@ -155,6 +149,13 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
 		if (tag != -1)
 			break;
 
+		prepare_to_wait_exclusive(&ws->wait, &wait,
+						TASK_UNINTERRUPTIBLE);
+
+		tag = __blk_mq_get_tag(data, bt);
+		if (tag != -1)
+			break;
+
 		if (data->ctx)
 			blk_mq_put_ctx(data->ctx);
 
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 3d3797327491..16e83e6df404 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -95,8 +95,7 @@ static void blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
 {
 	struct mq_inflight *mi = priv;
 
-	if (test_bit(REQ_ATOM_STARTED, &rq->atomic_flags) &&
-	    !test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) {
+	if (blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT) {
 		/*
 		 * index[0] counts the specific partition that was asked
 		 * for. index[1] counts the ones that are active on the
@@ -222,7 +221,7 @@ void blk_mq_quiesce_queue(struct request_queue *q)
 
 	queue_for_each_hw_ctx(q, hctx, i) {
 		if (hctx->flags & BLK_MQ_F_BLOCKING)
-			synchronize_srcu(hctx->queue_rq_srcu);
+			synchronize_srcu(hctx->srcu);
 		else
 			rcu = true;
 	}
@@ -272,15 +271,14 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 {
 	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
 	struct request *rq = tags->static_rqs[tag];
-
-	rq->rq_flags = 0;
+	req_flags_t rq_flags = 0;
 
 	if (data->flags & BLK_MQ_REQ_INTERNAL) {
 		rq->tag = -1;
 		rq->internal_tag = tag;
 	} else {
 		if (blk_mq_tag_busy(data->hctx)) {
-			rq->rq_flags = RQF_MQ_INFLIGHT;
+			rq_flags = RQF_MQ_INFLIGHT;
 			atomic_inc(&data->hctx->nr_active);
 		}
 		rq->tag = tag;
@@ -288,27 +286,22 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 		data->hctx->tags->rqs[rq->tag] = rq;
 	}
 
-	INIT_LIST_HEAD(&rq->queuelist);
 	/* csd/requeue_work/fifo_time is initialized before use */
 	rq->q = data->q;
 	rq->mq_ctx = data->ctx;
+	rq->rq_flags = rq_flags;
+	rq->cpu = -1;
 	rq->cmd_flags = op;
 	if (data->flags & BLK_MQ_REQ_PREEMPT)
 		rq->rq_flags |= RQF_PREEMPT;
 	if (blk_queue_io_stat(data->q))
 		rq->rq_flags |= RQF_IO_STAT;
-	/* do not touch atomic flags, it needs atomic ops against the timer */
-	rq->cpu = -1;
+	INIT_LIST_HEAD(&rq->queuelist);
 	INIT_HLIST_NODE(&rq->hash);
 	RB_CLEAR_NODE(&rq->rb_node);
 	rq->rq_disk = NULL;
 	rq->part = NULL;
 	rq->start_time = jiffies;
-#ifdef CONFIG_BLK_CGROUP
-	rq->rl = NULL;
-	set_start_time_ns(rq);
-	rq->io_start_time_ns = 0;
-#endif
 	rq->nr_phys_segments = 0;
 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 	rq->nr_integrity_segments = 0;
@@ -316,6 +309,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 	rq->special = NULL;
 	/* tag was already set */
 	rq->extra_len = 0;
+	rq->__deadline = 0;
 
 	INIT_LIST_HEAD(&rq->timeout_list);
 	rq->timeout = 0;
@@ -324,6 +318,12 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 	rq->end_io_data = NULL;
 	rq->next_rq = NULL;
 
+#ifdef CONFIG_BLK_CGROUP
+	rq->rl = NULL;
+	set_start_time_ns(rq);
+	rq->io_start_time_ns = 0;
+#endif
+
 	data->ctx->rq_dispatched[op_is_sync(op)]++;
 	return rq;
 }
@@ -443,7 +443,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
 		blk_queue_exit(q);
 		return ERR_PTR(-EXDEV);
 	}
-	cpu = cpumask_first(alloc_data.hctx->cpumask);
+	cpu = cpumask_first_and(alloc_data.hctx->cpumask, cpu_online_mask);
 	alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
 
 	rq = blk_mq_get_request(q, NULL, op, &alloc_data);
@@ -485,8 +485,7 @@ void blk_mq_free_request(struct request *rq)
 	if (blk_rq_rl(rq))
 		blk_put_rl(blk_rq_rl(rq));
 
-	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
-	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
+	blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
 	if (rq->tag != -1)
 		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
 	if (sched_tag != -1)
@@ -532,6 +531,9 @@ static void __blk_mq_complete_request(struct request *rq)
 	bool shared = false;
 	int cpu;
 
+	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT);
+	blk_mq_rq_update_state(rq, MQ_RQ_COMPLETE);
+
 	if (rq->internal_tag != -1)
 		blk_mq_sched_completed_request(rq);
 	if (rq->rq_flags & RQF_STATS) {
@@ -559,6 +561,56 @@ static void __blk_mq_complete_request(struct request *rq)
 	put_cpu();
 }
 
+static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
+	__releases(hctx->srcu)
+{
+	if (!(hctx->flags & BLK_MQ_F_BLOCKING))
+		rcu_read_unlock();
+	else
+		srcu_read_unlock(hctx->srcu, srcu_idx);
+}
+
+static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
+	__acquires(hctx->srcu)
+{
+	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
+		/* shut up gcc false positive */
+		*srcu_idx = 0;
+		rcu_read_lock();
+	} else
+		*srcu_idx = srcu_read_lock(hctx->srcu);
+}
+
+static void blk_mq_rq_update_aborted_gstate(struct request *rq, u64 gstate)
+{
+	unsigned long flags;
+
+	/*
+	 * blk_mq_rq_aborted_gstate() is used from the completion path and
+	 * can thus be called from irq context.  u64_stats_fetch in the
+	 * middle of update on the same CPU leads to lockup.  Disable irq
+	 * while updating.
+	 */
+	local_irq_save(flags);
+	u64_stats_update_begin(&rq->aborted_gstate_sync);
+	rq->aborted_gstate = gstate;
+	u64_stats_update_end(&rq->aborted_gstate_sync);
+	local_irq_restore(flags);
+}
+
+static u64 blk_mq_rq_aborted_gstate(struct request *rq)
+{
+	unsigned int start;
+	u64 aborted_gstate;
+
+	do {
+		start = u64_stats_fetch_begin(&rq->aborted_gstate_sync);
+		aborted_gstate = rq->aborted_gstate;
+	} while (u64_stats_fetch_retry(&rq->aborted_gstate_sync, start));
+
+	return aborted_gstate;
+}
+
 /**
  * blk_mq_complete_request - end I/O on a request
  * @rq:		the request being processed
@@ -570,17 +622,33 @@ static void __blk_mq_complete_request(struct request *rq)
 void blk_mq_complete_request(struct request *rq)
 {
 	struct request_queue *q = rq->q;
+	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu);
+	int srcu_idx;
 
 	if (unlikely(blk_should_fake_timeout(q)))
 		return;
-	if (!blk_mark_rq_complete(rq))
+
+	/*
+	 * If @rq->aborted_gstate equals the current instance, timeout is
+	 * claiming @rq and we lost.  This is synchronized through
+	 * hctx_lock().  See blk_mq_timeout_work() for details.
+	 *
+	 * Completion path never blocks and we can directly use RCU here
+	 * instead of hctx_lock() which can be either RCU or SRCU.
+	 * However, that would complicate paths which want to synchronize
+	 * against us.  Let stay in sync with the issue path so that
+	 * hctx_lock() covers both issue and completion paths.
+	 */
+	hctx_lock(hctx, &srcu_idx);
+	if (blk_mq_rq_aborted_gstate(rq) != rq->gstate)
 		__blk_mq_complete_request(rq);
+	hctx_unlock(hctx, srcu_idx);
 }
 EXPORT_SYMBOL(blk_mq_complete_request);
 
 int blk_mq_request_started(struct request *rq)
 {
-	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
+	return blk_mq_rq_state(rq) != MQ_RQ_IDLE;
 }
 EXPORT_SYMBOL_GPL(blk_mq_request_started);
 
@@ -598,34 +666,27 @@ void blk_mq_start_request(struct request *rq)
 		wbt_issue(q->rq_wb, &rq->issue_stat);
 	}
 
-	blk_add_timer(rq);
-
-	WARN_ON_ONCE(test_bit(REQ_ATOM_STARTED, &rq->atomic_flags));
+	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
 
 	/*
-	 * Mark us as started and clear complete. Complete might have been
-	 * set if requeue raced with timeout, which then marked it as
-	 * complete. So be sure to clear complete again when we start
-	 * the request, otherwise we'll ignore the completion event.
+	 * Mark @rq in-flight which also advances the generation number,
+	 * and register for timeout.  Protect with a seqcount to allow the
+	 * timeout path to read both @rq->gstate and @rq->deadline
+	 * coherently.
 	 *
-	 * Ensure that ->deadline is visible before we set STARTED, such that
-	 * blk_mq_check_expired() is guaranteed to observe our ->deadline when
-	 * it observes STARTED.
+	 * This is the only place where a request is marked in-flight.  If
+	 * the timeout path reads an in-flight @rq->gstate, the
+	 * @rq->deadline it reads together under @rq->gstate_seq is
+	 * guaranteed to be the matching one.
 	 */
-	smp_wmb();
-	set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
-	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) {
-		/*
-		 * Coherence order guarantees these consecutive stores to a
-		 * single variable propagate in the specified order. Thus the
-		 * clear_bit() is ordered _after_ the set bit. See
-		 * blk_mq_check_expired().
-		 *
-		 * (the bits must be part of the same byte for this to be
-		 * true).
-		 */
-		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
-	}
+	preempt_disable();
+	write_seqcount_begin(&rq->gstate_seq);
+
+	blk_mq_rq_update_state(rq, MQ_RQ_IN_FLIGHT);
+	blk_add_timer(rq);
+
+	write_seqcount_end(&rq->gstate_seq);
+	preempt_enable();
 
 	if (q->dma_drain_size && blk_rq_bytes(rq)) {
 		/*
@@ -639,13 +700,9 @@ void blk_mq_start_request(struct request *rq)
 EXPORT_SYMBOL(blk_mq_start_request);
 
 /*
- * When we reach here because queue is busy, REQ_ATOM_COMPLETE
- * flag isn't set yet, so there may be race with timeout handler,
- * but given rq->deadline is just set in .queue_rq() under
- * this situation, the race won't be possible in reality because
- * rq->timeout should be set as big enough to cover the window
- * between blk_mq_start_request() called from .queue_rq() and
- * clearing REQ_ATOM_STARTED here.
+ * When we reach here because queue is busy, it's safe to change the state
+ * to IDLE without checking @rq->aborted_gstate because we should still be
+ * holding the RCU read lock and thus protected against timeout.
  */
 static void __blk_mq_requeue_request(struct request *rq)
 {
@@ -655,9 +712,9 @@ static void __blk_mq_requeue_request(struct request *rq)
 
 	trace_block_rq_requeue(q, rq);
 	wbt_requeue(q->rq_wb, &rq->issue_stat);
-	blk_mq_sched_requeue_request(rq);
 
-	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
+	if (blk_mq_rq_state(rq) != MQ_RQ_IDLE) {
+		blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
 		if (q->dma_drain_size && blk_rq_bytes(rq))
 			rq->nr_phys_segments--;
 	}
@@ -667,6 +724,9 @@ void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
 {
 	__blk_mq_requeue_request(rq);
 
+	/* this request will be re-inserted to io scheduler queue */
+	blk_mq_sched_requeue_request(rq);
+
 	BUG_ON(blk_queued_rq(rq));
 	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
 }
@@ -689,13 +749,13 @@ static void blk_mq_requeue_work(struct work_struct *work)
 
 		rq->rq_flags &= ~RQF_SOFTBARRIER;
 		list_del_init(&rq->queuelist);
-		blk_mq_sched_insert_request(rq, true, false, false, true);
+		blk_mq_sched_insert_request(rq, true, false, false);
 	}
 
 	while (!list_empty(&rq_list)) {
 		rq = list_entry(rq_list.next, struct request, queuelist);
 		list_del_init(&rq->queuelist);
-		blk_mq_sched_insert_request(rq, false, false, false, true);
+		blk_mq_sched_insert_request(rq, false, false, false);
 	}
 
 	blk_mq_run_hw_queues(q, false);
@@ -729,7 +789,7 @@ EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 
 void blk_mq_kick_requeue_list(struct request_queue *q)
 {
-	kblockd_schedule_delayed_work(&q->requeue_work, 0);
+	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
 }
 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 
@@ -755,24 +815,15 @@ EXPORT_SYMBOL(blk_mq_tag_to_rq);
 struct blk_mq_timeout_data {
 	unsigned long next;
 	unsigned int next_set;
+	unsigned int nr_expired;
 };
 
-void blk_mq_rq_timed_out(struct request *req, bool reserved)
+static void blk_mq_rq_timed_out(struct request *req, bool reserved)
 {
 	const struct blk_mq_ops *ops = req->q->mq_ops;
 	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
 
-	/*
-	 * We know that complete is set at this point. If STARTED isn't set
-	 * anymore, then the request isn't active and the "timeout" should
-	 * just be ignored. This can happen due to the bitflag ordering.
-	 * Timeout first checks if STARTED is set, and if it is, assumes
-	 * the request is active. But if we race with completion, then
-	 * both flags will get cleared. So check here again, and ignore
-	 * a timeout event with a request that isn't active.
-	 */
-	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
-		return;
+	req->rq_flags |= RQF_MQ_TIMEOUT_EXPIRED;
 
 	if (ops->timeout)
 		ret = ops->timeout(req, reserved);
@@ -782,8 +833,13 @@ void blk_mq_rq_timed_out(struct request *req, bool reserved)
 		__blk_mq_complete_request(req);
 		break;
 	case BLK_EH_RESET_TIMER:
+		/*
+		 * As nothing prevents from completion happening while
+		 * ->aborted_gstate is set, this may lead to ignored
+		 * completions and further spurious timeouts.
+		 */
+		blk_mq_rq_update_aborted_gstate(req, 0);
 		blk_add_timer(req);
-		blk_clear_rq_complete(req);
 		break;
 	case BLK_EH_NOT_HANDLED:
 		break;
@@ -797,50 +853,51 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
 		struct request *rq, void *priv, bool reserved)
 {
 	struct blk_mq_timeout_data *data = priv;
-	unsigned long deadline;
+	unsigned long gstate, deadline;
+	int start;
 
-	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
-		return;
+	might_sleep();
 
-	/*
-	 * Ensures that if we see STARTED we must also see our
-	 * up-to-date deadline, see blk_mq_start_request().
-	 */
-	smp_rmb();
+	if (rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED)
+		return;
 
-	deadline = READ_ONCE(rq->deadline);
+	/* read coherent snapshots of @rq->state_gen and @rq->deadline */
+	while (true) {
+		start = read_seqcount_begin(&rq->gstate_seq);
+		gstate = READ_ONCE(rq->gstate);
+		deadline = blk_rq_deadline(rq);
+		if (!read_seqcount_retry(&rq->gstate_seq, start))
+			break;
+		cond_resched();
+	}
 
-	/*
-	 * The rq being checked may have been freed and reallocated
-	 * out already here, we avoid this race by checking rq->deadline
-	 * and REQ_ATOM_COMPLETE flag together:
-	 *
-	 * - if rq->deadline is observed as new value because of
-	 *   reusing, the rq won't be timed out because of timing.
-	 * - if rq->deadline is observed as previous value,
-	 *   REQ_ATOM_COMPLETE flag won't be cleared in reuse path
-	 *   because we put a barrier between setting rq->deadline
-	 *   and clearing the flag in blk_mq_start_request(), so
-	 *   this rq won't be timed out too.
-	 */
-	if (time_after_eq(jiffies, deadline)) {
-		if (!blk_mark_rq_complete(rq)) {
-			/*
-			 * Again coherence order ensures that consecutive reads
-			 * from the same variable must be in that order. This
-			 * ensures that if we see COMPLETE clear, we must then
-			 * see STARTED set and we'll ignore this timeout.
-			 *
-			 * (There's also the MB implied by the test_and_clear())
-			 */
-			blk_mq_rq_timed_out(rq, reserved);
-		}
+	/* if in-flight && overdue, mark for abortion */
+	if ((gstate & MQ_RQ_STATE_MASK) == MQ_RQ_IN_FLIGHT &&
+	    time_after_eq(jiffies, deadline)) {
+		blk_mq_rq_update_aborted_gstate(rq, gstate);
+		data->nr_expired++;
+		hctx->nr_expired++;
 	} else if (!data->next_set || time_after(data->next, deadline)) {
 		data->next = deadline;
 		data->next_set = 1;
 	}
 }
 
+static void blk_mq_terminate_expired(struct blk_mq_hw_ctx *hctx,
+		struct request *rq, void *priv, bool reserved)
+{
+	/*
+	 * We marked @rq->aborted_gstate and waited for RCU.  If there were
+	 * completions that we lost to, they would have finished and
+	 * updated @rq->gstate by now; otherwise, the completion path is
+	 * now guaranteed to see @rq->aborted_gstate and yield.  If
+	 * @rq->aborted_gstate still matches @rq->gstate, @rq is ours.
+	 */
+	if (!(rq->rq_flags & RQF_MQ_TIMEOUT_EXPIRED) &&
+	    READ_ONCE(rq->gstate) == rq->aborted_gstate)
+		blk_mq_rq_timed_out(rq, reserved);
+}
+
 static void blk_mq_timeout_work(struct work_struct *work)
 {
 	struct request_queue *q =
@@ -848,7 +905,9 @@ static void blk_mq_timeout_work(struct work_struct *work)
 	struct blk_mq_timeout_data data = {
 		.next		= 0,
 		.next_set	= 0,
+		.nr_expired	= 0,
 	};
+	struct blk_mq_hw_ctx *hctx;
 	int i;
 
 	/* A deadlock might occur if a request is stuck requiring a
@@ -867,14 +926,46 @@ static void blk_mq_timeout_work(struct work_struct *work)
 	if (!percpu_ref_tryget(&q->q_usage_counter))
 		return;
 
+	/* scan for the expired ones and set their ->aborted_gstate */
 	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
 
+	if (data.nr_expired) {
+		bool has_rcu = false;
+
+		/*
+		 * Wait till everyone sees ->aborted_gstate.  The
+		 * sequential waits for SRCUs aren't ideal.  If this ever
+		 * becomes a problem, we can add per-hw_ctx rcu_head and
+		 * wait in parallel.
+		 */
+		queue_for_each_hw_ctx(q, hctx, i) {
+			if (!hctx->nr_expired)
+				continue;
+
+			if (!(hctx->flags & BLK_MQ_F_BLOCKING))
+				has_rcu = true;
+			else
+				synchronize_srcu(hctx->srcu);
+
+			hctx->nr_expired = 0;
+		}
+		if (has_rcu)
+			synchronize_rcu();
+
+		/* terminate the ones we won */
+		blk_mq_queue_tag_busy_iter(q, blk_mq_terminate_expired, NULL);
+	}
+
 	if (data.next_set) {
 		data.next = blk_rq_timeout(round_jiffies_up(data.next));
 		mod_timer(&q->timeout, data.next);
 	} else {
-		struct blk_mq_hw_ctx *hctx;
-
+		/*
+		 * Request timeouts are handled as a forward rolling timer. If
+		 * we end up here it means that no requests are pending and
+		 * also that no request has been pending for a while. Mark
+		 * each hctx as idle.
+		 */
 		queue_for_each_hw_ctx(q, hctx, i) {
 			/* the hctx may be unmapped, so check it here */
 			if (blk_mq_hw_queue_mapped(hctx))
@@ -1010,68 +1101,71 @@ static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
 
 /*
  * Mark us waiting for a tag. For shared tags, this involves hooking us into
- * the tag wakeups. For non-shared tags, we can simply mark us nedeing a
- * restart. For both caes, take care to check the condition again after
+ * the tag wakeups. For non-shared tags, we can simply mark us needing a
+ * restart. For both cases, take care to check the condition again after
  * marking us as waiting.
  */
 static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx **hctx,
 				 struct request *rq)
 {
 	struct blk_mq_hw_ctx *this_hctx = *hctx;
-	bool shared_tags = (this_hctx->flags & BLK_MQ_F_TAG_SHARED) != 0;
 	struct sbq_wait_state *ws;
 	wait_queue_entry_t *wait;
 	bool ret;
 
-	if (!shared_tags) {
+	if (!(this_hctx->flags & BLK_MQ_F_TAG_SHARED)) {
 		if (!test_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state))
 			set_bit(BLK_MQ_S_SCHED_RESTART, &this_hctx->state);
-	} else {
-		wait = &this_hctx->dispatch_wait;
-		if (!list_empty_careful(&wait->entry))
-			return false;
 
-		spin_lock(&this_hctx->lock);
-		if (!list_empty(&wait->entry)) {
-			spin_unlock(&this_hctx->lock);
-			return false;
-		}
+		/*
+		 * It's possible that a tag was freed in the window between the
+		 * allocation failure and adding the hardware queue to the wait
+		 * queue.
+		 *
+		 * Don't clear RESTART here, someone else could have set it.
+		 * At most this will cost an extra queue run.
+		 */
+		return blk_mq_get_driver_tag(rq, hctx, false);
+	}
 
-		ws = bt_wait_ptr(&this_hctx->tags->bitmap_tags, this_hctx);
-		add_wait_queue(&ws->wait, wait);
+	wait = &this_hctx->dispatch_wait;
+	if (!list_empty_careful(&wait->entry))
+		return false;
+
+	spin_lock(&this_hctx->lock);
+	if (!list_empty(&wait->entry)) {
+		spin_unlock(&this_hctx->lock);
+		return false;
 	}
 
+	ws = bt_wait_ptr(&this_hctx->tags->bitmap_tags, this_hctx);
+	add_wait_queue(&ws->wait, wait);
+
 	/*
 	 * It's possible that a tag was freed in the window between the
 	 * allocation failure and adding the hardware queue to the wait
 	 * queue.
 	 */
 	ret = blk_mq_get_driver_tag(rq, hctx, false);
-
-	if (!shared_tags) {
-		/*
-		 * Don't clear RESTART here, someone else could have set it.
-		 * At most this will cost an extra queue run.
-		 */
-		return ret;
-	} else {
-		if (!ret) {
-			spin_unlock(&this_hctx->lock);
-			return false;
-		}
-
-		/*
-		 * We got a tag, remove ourselves from the wait queue to ensure
-		 * someone else gets the wakeup.
-		 */
-		spin_lock_irq(&ws->wait.lock);
-		list_del_init(&wait->entry);
-		spin_unlock_irq(&ws->wait.lock);
+	if (!ret) {
 		spin_unlock(&this_hctx->lock);
-		return true;
+		return false;
 	}
+
+	/*
+	 * We got a tag, remove ourselves from the wait queue to ensure
+	 * someone else gets the wakeup.
+	 */
+	spin_lock_irq(&ws->wait.lock);
+	list_del_init(&wait->entry);
+	spin_unlock_irq(&ws->wait.lock);
+	spin_unlock(&this_hctx->lock);
+
+	return true;
 }
 
+#define BLK_MQ_RESOURCE_DELAY	3		/* ms units */
+
 bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
 			     bool got_budget)
 {
@@ -1079,6 +1173,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
 	struct request *rq, *nxt;
 	bool no_tag = false;
 	int errors, queued;
+	blk_status_t ret = BLK_STS_OK;
 
 	if (list_empty(list))
 		return false;
@@ -1091,7 +1186,6 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
 	errors = queued = 0;
 	do {
 		struct blk_mq_queue_data bd;
-		blk_status_t ret;
 
 		rq = list_first_entry(list, struct request, queuelist);
 		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
@@ -1136,7 +1230,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
 		}
 
 		ret = q->mq_ops->queue_rq(hctx, &bd);
-		if (ret == BLK_STS_RESOURCE) {
+		if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) {
 			/*
 			 * If an I/O scheduler has been configured and we got a
 			 * driver tag for the next request already, free it
@@ -1167,6 +1261,8 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
 	 * that is where we will continue on next queue run.
 	 */
 	if (!list_empty(list)) {
+		bool needs_restart;
+
 		spin_lock(&hctx->lock);
 		list_splice_init(list, &hctx->dispatch);
 		spin_unlock(&hctx->lock);
@@ -1190,10 +1286,17 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
 		 * - Some but not all block drivers stop a queue before
 		 *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
 		 *   and dm-rq.
+		 *
+		 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
+		 * bit is set, run queue after a delay to avoid IO stalls
+		 * that could otherwise occur if the queue is idle.
 		 */
-		if (!blk_mq_sched_needs_restart(hctx) ||
+		needs_restart = blk_mq_sched_needs_restart(hctx);
+		if (!needs_restart ||
 		    (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
 			blk_mq_run_hw_queue(hctx, true);
+		else if (needs_restart && (ret == BLK_STS_RESOURCE))
+			blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
 	}
 
 	return (queued + errors) != 0;
@@ -1206,9 +1309,27 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 	/*
 	 * We should be running this queue from one of the CPUs that
 	 * are mapped to it.
+	 *
+	 * There are at least two related races now between setting
+	 * hctx->next_cpu from blk_mq_hctx_next_cpu() and running
+	 * __blk_mq_run_hw_queue():
+	 *
+	 * - hctx->next_cpu is found offline in blk_mq_hctx_next_cpu(),
+	 *   but later it becomes online, then this warning is harmless
+	 *   at all
+	 *
+	 * - hctx->next_cpu is found online in blk_mq_hctx_next_cpu(),
+	 *   but later it becomes offline, then the warning can't be
+	 *   triggered, and we depend on blk-mq timeout handler to
+	 *   handle dispatched requests to this hctx
 	 */
-	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
-		cpu_online(hctx->next_cpu));
+	if (!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
+		cpu_online(hctx->next_cpu)) {
+		printk(KERN_WARNING "run queue from wrong CPU %d, hctx %s\n",
+			raw_smp_processor_id(),
+			cpumask_empty(hctx->cpumask) ? "inactive": "active");
+		dump_stack();
+	}
 
 	/*
 	 * We can't run the queue inline with ints disabled. Ensure that
@@ -1216,17 +1337,11 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 	 */
 	WARN_ON_ONCE(in_interrupt());
 
-	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
-		rcu_read_lock();
-		blk_mq_sched_dispatch_requests(hctx);
-		rcu_read_unlock();
-	} else {
-		might_sleep();
+	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
 
-		srcu_idx = srcu_read_lock(hctx->queue_rq_srcu);
-		blk_mq_sched_dispatch_requests(hctx);
-		srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx);
-	}
+	hctx_lock(hctx, &srcu_idx);
+	blk_mq_sched_dispatch_requests(hctx);
+	hctx_unlock(hctx, srcu_idx);
 }
 
 /*
@@ -1237,20 +1352,47 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
  */
 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 {
+	bool tried = false;
+
 	if (hctx->queue->nr_hw_queues == 1)
 		return WORK_CPU_UNBOUND;
 
 	if (--hctx->next_cpu_batch <= 0) {
 		int next_cpu;
-
-		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
+select_cpu:
+		next_cpu = cpumask_next_and(hctx->next_cpu, hctx->cpumask,
+				cpu_online_mask);
 		if (next_cpu >= nr_cpu_ids)
-			next_cpu = cpumask_first(hctx->cpumask);
+			next_cpu = cpumask_first_and(hctx->cpumask,cpu_online_mask);
 
-		hctx->next_cpu = next_cpu;
+		/*
+		 * No online CPU is found, so have to make sure hctx->next_cpu
+		 * is set correctly for not breaking workqueue.
+		 */
+		if (next_cpu >= nr_cpu_ids)
+			hctx->next_cpu = cpumask_first(hctx->cpumask);
+		else
+			hctx->next_cpu = next_cpu;
 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 	}
 
+	/*
+	 * Do unbound schedule if we can't find a online CPU for this hctx,
+	 * and it should only happen in the path of handling CPU DEAD.
+	 */
+	if (!cpu_online(hctx->next_cpu)) {
+		if (!tried) {
+			tried = true;
+			goto select_cpu;
+		}
+
+		/*
+		 * Make sure to re-select CPU next time once after CPUs
+		 * in hctx->cpumask become online again.
+		 */
+		hctx->next_cpu_batch = 1;
+		return WORK_CPU_UNBOUND;
+	}
 	return hctx->next_cpu;
 }
 
@@ -1274,9 +1416,8 @@ static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
 		put_cpu();
 	}
 
-	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
-					 &hctx->run_work,
-					 msecs_to_jiffies(msecs));
+	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
+				    msecs_to_jiffies(msecs));
 }
 
 void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
@@ -1287,7 +1428,23 @@ EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);
 
 bool blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 {
-	if (blk_mq_hctx_has_pending(hctx)) {
+	int srcu_idx;
+	bool need_run;
+
+	/*
+	 * When queue is quiesced, we may be switching io scheduler, or
+	 * updating nr_hw_queues, or other things, and we can't run queue
+	 * any more, even __blk_mq_hctx_has_pending() can't be called safely.
+	 *
+	 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
+	 * quiesced.
+	 */
+	hctx_lock(hctx, &srcu_idx);
+	need_run = !blk_queue_quiesced(hctx->queue) &&
+		blk_mq_hctx_has_pending(hctx);
+	hctx_unlock(hctx, srcu_idx);
+
+	if (need_run) {
 		__blk_mq_delay_run_hw_queue(hctx, async, 0);
 		return true;
 	}
@@ -1595,9 +1752,9 @@ static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
 	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
 }
 
-static void __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
-					struct request *rq,
-					blk_qc_t *cookie, bool may_sleep)
+static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
+					    struct request *rq,
+					    blk_qc_t *cookie)
 {
 	struct request_queue *q = rq->q;
 	struct blk_mq_queue_data bd = {
@@ -1606,15 +1763,53 @@ static void __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
 	};
 	blk_qc_t new_cookie;
 	blk_status_t ret;
+
+	new_cookie = request_to_qc_t(hctx, rq);
+
+	/*
+	 * For OK queue, we are done. For error, caller may kill it.
+	 * Any other error (busy), just add it to our list as we
+	 * previously would have done.
+	 */
+	ret = q->mq_ops->queue_rq(hctx, &bd);
+	switch (ret) {
+	case BLK_STS_OK:
+		*cookie = new_cookie;
+		break;
+	case BLK_STS_RESOURCE:
+	case BLK_STS_DEV_RESOURCE:
+		__blk_mq_requeue_request(rq);
+		break;
+	default:
+		*cookie = BLK_QC_T_NONE;
+		break;
+	}
+
+	return ret;
+}
+
+static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
+						struct request *rq,
+						blk_qc_t *cookie,
+						bool bypass_insert)
+{
+	struct request_queue *q = rq->q;
 	bool run_queue = true;
 
-	/* RCU or SRCU read lock is needed before checking quiesced flag */
+	/*
+	 * RCU or SRCU read lock is needed before checking quiesced flag.
+	 *
+	 * When queue is stopped or quiesced, ignore 'bypass_insert' from
+	 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller,
+	 * and avoid driver to try to dispatch again.
+	 */
 	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) {
 		run_queue = false;
+		bypass_insert = false;
 		goto insert;
 	}
 
-	if (q->elevator)
+	if (q->elevator && !bypass_insert)
 		goto insert;
 
 	if (!blk_mq_get_driver_tag(rq, NULL, false))
@@ -1625,47 +1820,47 @@ static void __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
 		goto insert;
 	}
 
-	new_cookie = request_to_qc_t(hctx, rq);
-
-	/*
-	 * For OK queue, we are done. For error, kill it. Any other
-	 * error (busy), just add it to our list as we previously
-	 * would have done
-	 */
-	ret = q->mq_ops->queue_rq(hctx, &bd);
-	switch (ret) {
-	case BLK_STS_OK:
-		*cookie = new_cookie;
-		return;
-	case BLK_STS_RESOURCE:
-		__blk_mq_requeue_request(rq);
-		goto insert;
-	default:
-		*cookie = BLK_QC_T_NONE;
-		blk_mq_end_request(rq, ret);
-		return;
-	}
-
+	return __blk_mq_issue_directly(hctx, rq, cookie);
 insert:
-	blk_mq_sched_insert_request(rq, false, run_queue, false, may_sleep);
+	if (bypass_insert)
+		return BLK_STS_RESOURCE;
+
+	blk_mq_sched_insert_request(rq, false, run_queue, false);
+	return BLK_STS_OK;
 }
 
 static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
 		struct request *rq, blk_qc_t *cookie)
 {
-	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
-		rcu_read_lock();
-		__blk_mq_try_issue_directly(hctx, rq, cookie, false);
-		rcu_read_unlock();
-	} else {
-		unsigned int srcu_idx;
+	blk_status_t ret;
+	int srcu_idx;
 
-		might_sleep();
+	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
 
-		srcu_idx = srcu_read_lock(hctx->queue_rq_srcu);
-		__blk_mq_try_issue_directly(hctx, rq, cookie, true);
-		srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx);
-	}
+	hctx_lock(hctx, &srcu_idx);
+
+	ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false);
+	if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE)
+		blk_mq_sched_insert_request(rq, false, true, false);
+	else if (ret != BLK_STS_OK)
+		blk_mq_end_request(rq, ret);
+
+	hctx_unlock(hctx, srcu_idx);
+}
+
+blk_status_t blk_mq_request_issue_directly(struct request *rq)
+{
+	blk_status_t ret;
+	int srcu_idx;
+	blk_qc_t unused_cookie;
+	struct blk_mq_ctx *ctx = rq->mq_ctx;
+	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, ctx->cpu);
+
+	hctx_lock(hctx, &srcu_idx);
+	ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true);
+	hctx_unlock(hctx, srcu_idx);
+
+	return ret;
 }
 
 static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
@@ -1776,7 +1971,7 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
 	} else if (q->elevator) {
 		blk_mq_put_ctx(data.ctx);
 		blk_mq_bio_to_request(rq, bio);
-		blk_mq_sched_insert_request(rq, false, true, true, true);
+		blk_mq_sched_insert_request(rq, false, true, true);
 	} else {
 		blk_mq_put_ctx(data.ctx);
 		blk_mq_bio_to_request(rq, bio);
@@ -1869,6 +2064,22 @@ static size_t order_to_size(unsigned int order)
 	return (size_t)PAGE_SIZE << order;
 }
 
+static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
+			       unsigned int hctx_idx, int node)
+{
+	int ret;
+
+	if (set->ops->init_request) {
+		ret = set->ops->init_request(set, rq, hctx_idx, node);
+		if (ret)
+			return ret;
+	}
+
+	seqcount_init(&rq->gstate_seq);
+	u64_stats_init(&rq->aborted_gstate_sync);
+	return 0;
+}
+
 int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
 		     unsigned int hctx_idx, unsigned int depth)
 {
@@ -1930,12 +2141,9 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
 			struct request *rq = p;
 
 			tags->static_rqs[i] = rq;
-			if (set->ops->init_request) {
-				if (set->ops->init_request(set, rq, hctx_idx,
-						node)) {
-					tags->static_rqs[i] = NULL;
-					goto fail;
-				}
+			if (blk_mq_init_request(set, rq, hctx_idx, node)) {
+				tags->static_rqs[i] = NULL;
+				goto fail;
 			}
 
 			p += rq_size;
@@ -1994,7 +2202,8 @@ static void blk_mq_exit_hctx(struct request_queue *q,
 {
 	blk_mq_debugfs_unregister_hctx(hctx);
 
-	blk_mq_tag_idle(hctx);
+	if (blk_mq_hw_queue_mapped(hctx))
+		blk_mq_tag_idle(hctx);
 
 	if (set->ops->exit_request)
 		set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
@@ -2005,7 +2214,7 @@ static void blk_mq_exit_hctx(struct request_queue *q,
 		set->ops->exit_hctx(hctx, hctx_idx);
 
 	if (hctx->flags & BLK_MQ_F_BLOCKING)
-		cleanup_srcu_struct(hctx->queue_rq_srcu);
+		cleanup_srcu_struct(hctx->srcu);
 
 	blk_mq_remove_cpuhp(hctx);
 	blk_free_flush_queue(hctx->fq);
@@ -2074,13 +2283,11 @@ static int blk_mq_init_hctx(struct request_queue *q,
 	if (!hctx->fq)
 		goto sched_exit_hctx;
 
-	if (set->ops->init_request &&
-	    set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx,
-				   node))
+	if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, node))
 		goto free_fq;
 
 	if (hctx->flags & BLK_MQ_F_BLOCKING)
-		init_srcu_struct(hctx->queue_rq_srcu);
+		init_srcu_struct(hctx->srcu);
 
 	blk_mq_debugfs_register_hctx(q, hctx);
 
@@ -2116,16 +2323,11 @@ static void blk_mq_init_cpu_queues(struct request_queue *q,
 		INIT_LIST_HEAD(&__ctx->rq_list);
 		__ctx->queue = q;
 
-		/* If the cpu isn't present, the cpu is mapped to first hctx */
-		if (!cpu_present(i))
-			continue;
-
-		hctx = blk_mq_map_queue(q, i);
-
 		/*
 		 * Set local node, IFF we have more than one hw queue. If
 		 * not, we remain on the home node of the device
 		 */
+		hctx = blk_mq_map_queue(q, i);
 		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
 			hctx->numa_node = local_memory_node(cpu_to_node(i));
 	}
@@ -2182,7 +2384,7 @@ static void blk_mq_map_swqueue(struct request_queue *q)
 	 *
 	 * If the cpu isn't present, the cpu is mapped to first hctx.
 	 */
-	for_each_present_cpu(i) {
+	for_each_possible_cpu(i) {
 		hctx_idx = q->mq_map[i];
 		/* unmapped hw queue can be remapped after CPU topo changed */
 		if (!set->tags[hctx_idx] &&
@@ -2236,7 +2438,8 @@ static void blk_mq_map_swqueue(struct request_queue *q)
 		/*
 		 * Initialize batch roundrobin counts
 		 */
-		hctx->next_cpu = cpumask_first(hctx->cpumask);
+		hctx->next_cpu = cpumask_first_and(hctx->cpumask,
+				cpu_online_mask);
 		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 	}
 }
@@ -2369,7 +2572,7 @@ static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
 {
 	int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);
 
-	BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, queue_rq_srcu),
+	BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu),
 			   __alignof__(struct blk_mq_hw_ctx)) !=
 		     sizeof(struct blk_mq_hw_ctx));
 
@@ -2386,6 +2589,9 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
 	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
 
 	blk_mq_sysfs_unregister(q);
+
+	/* protect against switching io scheduler  */
+	mutex_lock(&q->sysfs_lock);
 	for (i = 0; i < set->nr_hw_queues; i++) {
 		int node;
 
@@ -2430,6 +2636,7 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
 		}
 	}
 	q->nr_hw_queues = i;
+	mutex_unlock(&q->sysfs_lock);
 	blk_mq_sysfs_register(q);
 }
 
@@ -2601,9 +2808,27 @@ static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
 
 static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
 {
-	if (set->ops->map_queues)
+	if (set->ops->map_queues) {
+		int cpu;
+		/*
+		 * transport .map_queues is usually done in the following
+		 * way:
+		 *
+		 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
+		 * 	mask = get_cpu_mask(queue)
+		 * 	for_each_cpu(cpu, mask)
+		 * 		set->mq_map[cpu] = queue;
+		 * }
+		 *
+		 * When we need to remap, the table has to be cleared for
+		 * killing stale mapping since one CPU may not be mapped
+		 * to any hw queue.
+		 */
+		for_each_possible_cpu(cpu)
+			set->mq_map[cpu] = 0;
+
 		return set->ops->map_queues(set);
-	else
+	} else
 		return blk_mq_map_queues(set);
 }
 
@@ -2712,6 +2937,7 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
 		return -EINVAL;
 
 	blk_mq_freeze_queue(q);
+	blk_mq_quiesce_queue(q);
 
 	ret = 0;
 	queue_for_each_hw_ctx(q, hctx, i) {
@@ -2735,6 +2961,7 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
 	if (!ret)
 		q->nr_requests = nr;
 
+	blk_mq_unquiesce_queue(q);
 	blk_mq_unfreeze_queue(q);
 
 	return ret;
@@ -2850,7 +3077,7 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
 	unsigned int nsecs;
 	ktime_t kt;
 
-	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
+	if (rq->rq_flags & RQF_MQ_POLL_SLEPT)
 		return false;
 
 	/*
@@ -2870,7 +3097,7 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
 	if (!nsecs)
 		return false;
 
-	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
+	rq->rq_flags |= RQF_MQ_POLL_SLEPT;
 
 	/*
 	 * This will be replaced with the stats tracking code, using
@@ -2884,7 +3111,7 @@ static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
 
 	hrtimer_init_sleeper(&hs, current);
 	do {
-		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
+		if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
 			break;
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		hrtimer_start_expires(&hs.timer, mode);
@@ -2939,6 +3166,7 @@ static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
 		cpu_relax();
 	}
 
+	__set_current_state(TASK_RUNNING);
 	return false;
 }
 
@@ -2970,12 +3198,6 @@ static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
 
 static int __init blk_mq_init(void)
 {
-	/*
-	 * See comment in block/blk.h rq_atomic_flags enum
-	 */
-	BUILD_BUG_ON((REQ_ATOM_STARTED / BITS_PER_BYTE) !=
-			(REQ_ATOM_COMPLETE / BITS_PER_BYTE));
-
 	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
 				blk_mq_hctx_notify_dead);
 	return 0;
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 6c7c3ff5bf62..88c558f71819 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -27,6 +27,20 @@ struct blk_mq_ctx {
 	struct kobject		kobj;
 } ____cacheline_aligned_in_smp;
 
+/*
+ * Bits for request->gstate.  The lower two bits carry MQ_RQ_* state value
+ * and the upper bits the generation number.
+ */
+enum mq_rq_state {
+	MQ_RQ_IDLE		= 0,
+	MQ_RQ_IN_FLIGHT		= 1,
+	MQ_RQ_COMPLETE		= 2,
+
+	MQ_RQ_STATE_BITS	= 2,
+	MQ_RQ_STATE_MASK	= (1 << MQ_RQ_STATE_BITS) - 1,
+	MQ_RQ_GEN_INC		= 1 << MQ_RQ_STATE_BITS,
+};
+
 void blk_mq_freeze_queue(struct request_queue *q);
 void blk_mq_free_queue(struct request_queue *q);
 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
@@ -60,6 +74,9 @@ void blk_mq_request_bypass_insert(struct request *rq, bool run_queue);
 void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
 				struct list_head *list);
 
+/* Used by blk_insert_cloned_request() to issue request directly */
+blk_status_t blk_mq_request_issue_directly(struct request *rq);
+
 /*
  * CPU -> queue mappings
  */
@@ -81,10 +98,41 @@ extern int blk_mq_sysfs_register(struct request_queue *q);
 extern void blk_mq_sysfs_unregister(struct request_queue *q);
 extern void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx);
 
-extern void blk_mq_rq_timed_out(struct request *req, bool reserved);
-
 void blk_mq_release(struct request_queue *q);
 
+/**
+ * blk_mq_rq_state() - read the current MQ_RQ_* state of a request
+ * @rq: target request.
+ */
+static inline int blk_mq_rq_state(struct request *rq)
+{
+	return READ_ONCE(rq->gstate) & MQ_RQ_STATE_MASK;
+}
+
+/**
+ * blk_mq_rq_update_state() - set the current MQ_RQ_* state of a request
+ * @rq: target request.
+ * @state: new state to set.
+ *
+ * Set @rq's state to @state.  The caller is responsible for ensuring that
+ * there are no other updaters.  A request can transition into IN_FLIGHT
+ * only from IDLE and doing so increments the generation number.
+ */
+static inline void blk_mq_rq_update_state(struct request *rq,
+					  enum mq_rq_state state)
+{
+	u64 old_val = READ_ONCE(rq->gstate);
+	u64 new_val = (old_val & ~MQ_RQ_STATE_MASK) | state;
+
+	if (state == MQ_RQ_IN_FLIGHT) {
+		WARN_ON_ONCE((old_val & MQ_RQ_STATE_MASK) != MQ_RQ_IDLE);
+		new_val += MQ_RQ_GEN_INC;
+	}
+
+	/* avoid exposing interim values */
+	WRITE_ONCE(rq->gstate, new_val);
+}
+
 static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
 					   unsigned int cpu)
 {
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 870484eaed1f..cbea895a5547 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -853,6 +853,10 @@ struct kobj_type blk_queue_ktype = {
 	.release	= blk_release_queue,
 };
 
+/**
+ * blk_register_queue - register a block layer queue with sysfs
+ * @disk: Disk of which the request queue should be registered with sysfs.
+ */
 int blk_register_queue(struct gendisk *disk)
 {
 	int ret;
@@ -909,11 +913,12 @@ int blk_register_queue(struct gendisk *disk)
 	if (q->request_fn || (q->mq_ops && q->elevator)) {
 		ret = elv_register_queue(q);
 		if (ret) {
+			mutex_unlock(&q->sysfs_lock);
 			kobject_uevent(&q->kobj, KOBJ_REMOVE);
 			kobject_del(&q->kobj);
 			blk_trace_remove_sysfs(dev);
 			kobject_put(&dev->kobj);
-			goto unlock;
+			return ret;
 		}
 	}
 	ret = 0;
@@ -921,7 +926,15 @@ int blk_register_queue(struct gendisk *disk)
 	mutex_unlock(&q->sysfs_lock);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(blk_register_queue);
 
+/**
+ * blk_unregister_queue - counterpart of blk_register_queue()
+ * @disk: Disk of which the request queue should be unregistered from sysfs.
+ *
+ * Note: the caller is responsible for guaranteeing that this function is called
+ * after blk_register_queue() has finished.
+ */
 void blk_unregister_queue(struct gendisk *disk)
 {
 	struct request_queue *q = disk->queue;
@@ -929,21 +942,39 @@ void blk_unregister_queue(struct gendisk *disk)
 	if (WARN_ON(!q))
 		return;
 
-	mutex_lock(&q->sysfs_lock);
-	queue_flag_clear_unlocked(QUEUE_FLAG_REGISTERED, q);
-	mutex_unlock(&q->sysfs_lock);
+	/* Return early if disk->queue was never registered. */
+	if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
+		return;
 
-	wbt_exit(q);
+	/*
+	 * Since sysfs_remove_dir() prevents adding new directory entries
+	 * before removal of existing entries starts, protect against
+	 * concurrent elv_iosched_store() calls.
+	 */
+	mutex_lock(&q->sysfs_lock);
 
+	spin_lock_irq(q->queue_lock);
+	queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
+	spin_unlock_irq(q->queue_lock);
 
+	/*
+	 * Remove the sysfs attributes before unregistering the queue data
+	 * structures that can be modified through sysfs.
+	 */
 	if (q->mq_ops)
 		blk_mq_unregister_dev(disk_to_dev(disk), q);
-
-	if (q->request_fn || (q->mq_ops && q->elevator))
-		elv_unregister_queue(q);
+	mutex_unlock(&q->sysfs_lock);
 
 	kobject_uevent(&q->kobj, KOBJ_REMOVE);
 	kobject_del(&q->kobj);
 	blk_trace_remove_sysfs(disk_to_dev(disk));
+
+	wbt_exit(q);
+
+	mutex_lock(&q->sysfs_lock);
+	if (q->request_fn || (q->mq_ops && q->elevator))
+		elv_unregister_queue(q);
+	mutex_unlock(&q->sysfs_lock);
+
 	kobject_put(&disk_to_dev(disk)->kobj);
 }
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index d19f416d6101..c5a131673733 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -216,9 +216,9 @@ struct throtl_data
 
 	unsigned int scale;
 
-	struct latency_bucket tmp_buckets[LATENCY_BUCKET_SIZE];
-	struct avg_latency_bucket avg_buckets[LATENCY_BUCKET_SIZE];
-	struct latency_bucket __percpu *latency_buckets;
+	struct latency_bucket tmp_buckets[2][LATENCY_BUCKET_SIZE];
+	struct avg_latency_bucket avg_buckets[2][LATENCY_BUCKET_SIZE];
+	struct latency_bucket __percpu *latency_buckets[2];
 	unsigned long last_calculate_time;
 	unsigned long filtered_latency;
 
@@ -1510,11 +1510,21 @@ static struct cftype throtl_legacy_files[] = {
 		.private = (unsigned long)&blkcg_policy_throtl,
 		.seq_show = blkg_print_stat_bytes,
 	},
+	{
+		.name = "throttle.io_service_bytes_recursive",
+		.private = (unsigned long)&blkcg_policy_throtl,
+		.seq_show = blkg_print_stat_bytes_recursive,
+	},
 	{
 		.name = "throttle.io_serviced",
 		.private = (unsigned long)&blkcg_policy_throtl,
 		.seq_show = blkg_print_stat_ios,
 	},
+	{
+		.name = "throttle.io_serviced_recursive",
+		.private = (unsigned long)&blkcg_policy_throtl,
+		.seq_show = blkg_print_stat_ios_recursive,
+	},
 	{ }	/* terminate */
 };
 
@@ -2040,10 +2050,10 @@ static void blk_throtl_update_idletime(struct throtl_grp *tg)
 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
 static void throtl_update_latency_buckets(struct throtl_data *td)
 {
-	struct avg_latency_bucket avg_latency[LATENCY_BUCKET_SIZE];
-	int i, cpu;
-	unsigned long last_latency = 0;
-	unsigned long latency;
+	struct avg_latency_bucket avg_latency[2][LATENCY_BUCKET_SIZE];
+	int i, cpu, rw;
+	unsigned long last_latency[2] = { 0 };
+	unsigned long latency[2];
 
 	if (!blk_queue_nonrot(td->queue))
 		return;
@@ -2052,56 +2062,67 @@ static void throtl_update_latency_buckets(struct throtl_data *td)
 	td->last_calculate_time = jiffies;
 
 	memset(avg_latency, 0, sizeof(avg_latency));
-	for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
-		struct latency_bucket *tmp = &td->tmp_buckets[i];
-
-		for_each_possible_cpu(cpu) {
-			struct latency_bucket *bucket;
-
-			/* this isn't race free, but ok in practice */
-			bucket = per_cpu_ptr(td->latency_buckets, cpu);
-			tmp->total_latency += bucket[i].total_latency;
-			tmp->samples += bucket[i].samples;
-			bucket[i].total_latency = 0;
-			bucket[i].samples = 0;
-		}
+	for (rw = READ; rw <= WRITE; rw++) {
+		for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
+			struct latency_bucket *tmp = &td->tmp_buckets[rw][i];
+
+			for_each_possible_cpu(cpu) {
+				struct latency_bucket *bucket;
+
+				/* this isn't race free, but ok in practice */
+				bucket = per_cpu_ptr(td->latency_buckets[rw],
+					cpu);
+				tmp->total_latency += bucket[i].total_latency;
+				tmp->samples += bucket[i].samples;
+				bucket[i].total_latency = 0;
+				bucket[i].samples = 0;
+			}
 
-		if (tmp->samples >= 32) {
-			int samples = tmp->samples;
+			if (tmp->samples >= 32) {
+				int samples = tmp->samples;
 
-			latency = tmp->total_latency;
+				latency[rw] = tmp->total_latency;
 
-			tmp->total_latency = 0;
-			tmp->samples = 0;
-			latency /= samples;
-			if (latency == 0)
-				continue;
-			avg_latency[i].latency = latency;
+				tmp->total_latency = 0;
+				tmp->samples = 0;
+				latency[rw] /= samples;
+				if (latency[rw] == 0)
+					continue;
+				avg_latency[rw][i].latency = latency[rw];
+			}
 		}
 	}
 
-	for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
-		if (!avg_latency[i].latency) {
-			if (td->avg_buckets[i].latency < last_latency)
-				td->avg_buckets[i].latency = last_latency;
-			continue;
-		}
+	for (rw = READ; rw <= WRITE; rw++) {
+		for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
+			if (!avg_latency[rw][i].latency) {
+				if (td->avg_buckets[rw][i].latency < last_latency[rw])
+					td->avg_buckets[rw][i].latency =
+						last_latency[rw];
+				continue;
+			}
 
-		if (!td->avg_buckets[i].valid)
-			latency = avg_latency[i].latency;
-		else
-			latency = (td->avg_buckets[i].latency * 7 +
-				avg_latency[i].latency) >> 3;
+			if (!td->avg_buckets[rw][i].valid)
+				latency[rw] = avg_latency[rw][i].latency;
+			else
+				latency[rw] = (td->avg_buckets[rw][i].latency * 7 +
+					avg_latency[rw][i].latency) >> 3;
 
-		td->avg_buckets[i].latency = max(latency, last_latency);
-		td->avg_buckets[i].valid = true;
-		last_latency = td->avg_buckets[i].latency;
+			td->avg_buckets[rw][i].latency = max(latency[rw],
+				last_latency[rw]);
+			td->avg_buckets[rw][i].valid = true;
+			last_latency[rw] = td->avg_buckets[rw][i].latency;
+		}
 	}
 
 	for (i = 0; i < LATENCY_BUCKET_SIZE; i++)
 		throtl_log(&td->service_queue,
-			"Latency bucket %d: latency=%ld, valid=%d", i,
-			td->avg_buckets[i].latency, td->avg_buckets[i].valid);
+			"Latency bucket %d: read latency=%ld, read valid=%d, "
+			"write latency=%ld, write valid=%d", i,
+			td->avg_buckets[READ][i].latency,
+			td->avg_buckets[READ][i].valid,
+			td->avg_buckets[WRITE][i].latency,
+			td->avg_buckets[WRITE][i].valid);
 }
 #else
 static inline void throtl_update_latency_buckets(struct throtl_data *td)
@@ -2242,16 +2263,17 @@ static void throtl_track_latency(struct throtl_data *td, sector_t size,
 	struct latency_bucket *latency;
 	int index;
 
-	if (!td || td->limit_index != LIMIT_LOW || op != REQ_OP_READ ||
+	if (!td || td->limit_index != LIMIT_LOW ||
+	    !(op == REQ_OP_READ || op == REQ_OP_WRITE) ||
 	    !blk_queue_nonrot(td->queue))
 		return;
 
 	index = request_bucket_index(size);
 
-	latency = get_cpu_ptr(td->latency_buckets);
+	latency = get_cpu_ptr(td->latency_buckets[op]);
 	latency[index].total_latency += time;
 	latency[index].samples++;
-	put_cpu_ptr(td->latency_buckets);
+	put_cpu_ptr(td->latency_buckets[op]);
 }
 
 void blk_throtl_stat_add(struct request *rq, u64 time_ns)
@@ -2270,6 +2292,7 @@ void blk_throtl_bio_endio(struct bio *bio)
 	unsigned long finish_time;
 	unsigned long start_time;
 	unsigned long lat;
+	int rw = bio_data_dir(bio);
 
 	tg = bio->bi_cg_private;
 	if (!tg)
@@ -2298,7 +2321,7 @@ void blk_throtl_bio_endio(struct bio *bio)
 
 		bucket = request_bucket_index(
 			blk_stat_size(&bio->bi_issue_stat));
-		threshold = tg->td->avg_buckets[bucket].latency +
+		threshold = tg->td->avg_buckets[rw][bucket].latency +
 			tg->latency_target;
 		if (lat > threshold)
 			tg->bad_bio_cnt++;
@@ -2391,9 +2414,16 @@ int blk_throtl_init(struct request_queue *q)
 	td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node);
 	if (!td)
 		return -ENOMEM;
-	td->latency_buckets = __alloc_percpu(sizeof(struct latency_bucket) *
+	td->latency_buckets[READ] = __alloc_percpu(sizeof(struct latency_bucket) *
 		LATENCY_BUCKET_SIZE, __alignof__(u64));
-	if (!td->latency_buckets) {
+	if (!td->latency_buckets[READ]) {
+		kfree(td);
+		return -ENOMEM;
+	}
+	td->latency_buckets[WRITE] = __alloc_percpu(sizeof(struct latency_bucket) *
+		LATENCY_BUCKET_SIZE, __alignof__(u64));
+	if (!td->latency_buckets[WRITE]) {
+		free_percpu(td->latency_buckets[READ]);
 		kfree(td);
 		return -ENOMEM;
 	}
@@ -2412,7 +2442,8 @@ int blk_throtl_init(struct request_queue *q)
 	/* activate policy */
 	ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
 	if (ret) {
-		free_percpu(td->latency_buckets);
+		free_percpu(td->latency_buckets[READ]);
+		free_percpu(td->latency_buckets[WRITE]);
 		kfree(td);
 	}
 	return ret;
@@ -2423,7 +2454,8 @@ void blk_throtl_exit(struct request_queue *q)
 	BUG_ON(!q->td);
 	throtl_shutdown_wq(q);
 	blkcg_deactivate_policy(q, &blkcg_policy_throtl);
-	free_percpu(q->td->latency_buckets);
+	free_percpu(q->td->latency_buckets[READ]);
+	free_percpu(q->td->latency_buckets[WRITE]);
 	kfree(q->td);
 }
 
@@ -2441,15 +2473,17 @@ void blk_throtl_register_queue(struct request_queue *q)
 	} else {
 		td->throtl_slice = DFL_THROTL_SLICE_HD;
 		td->filtered_latency = LATENCY_FILTERED_HD;
-		for (i = 0; i < LATENCY_BUCKET_SIZE; i++)
-			td->avg_buckets[i].latency = DFL_HD_BASELINE_LATENCY;
+		for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
+			td->avg_buckets[READ][i].latency = DFL_HD_BASELINE_LATENCY;
+			td->avg_buckets[WRITE][i].latency = DFL_HD_BASELINE_LATENCY;
+		}
 	}
 #ifndef CONFIG_BLK_DEV_THROTTLING_LOW
 	/* if no low limit, use previous default */
 	td->throtl_slice = DFL_THROTL_SLICE_HD;
 #endif
 
-	td->track_bio_latency = !q->mq_ops && !q->request_fn;
+	td->track_bio_latency = !queue_is_rq_based(q);
 	if (!td->track_bio_latency)
 		blk_stat_enable_accounting(q);
 }
diff --git a/block/blk-timeout.c b/block/blk-timeout.c
index 764ecf9aeb30..a05e3676d24a 100644
--- a/block/blk-timeout.c
+++ b/block/blk-timeout.c
@@ -112,7 +112,9 @@ static void blk_rq_timed_out(struct request *req)
 static void blk_rq_check_expired(struct request *rq, unsigned long *next_timeout,
 			  unsigned int *next_set)
 {
-	if (time_after_eq(jiffies, rq->deadline)) {
+	const unsigned long deadline = blk_rq_deadline(rq);
+
+	if (time_after_eq(jiffies, deadline)) {
 		list_del_init(&rq->timeout_list);
 
 		/*
@@ -120,8 +122,8 @@ static void blk_rq_check_expired(struct request *rq, unsigned long *next_timeout
 		 */
 		if (!blk_mark_rq_complete(rq))
 			blk_rq_timed_out(rq);
-	} else if (!*next_set || time_after(*next_timeout, rq->deadline)) {
-		*next_timeout = rq->deadline;
+	} else if (!*next_set || time_after(*next_timeout, deadline)) {
+		*next_timeout = deadline;
 		*next_set = 1;
 	}
 }
@@ -156,12 +158,17 @@ void blk_timeout_work(struct work_struct *work)
  */
 void blk_abort_request(struct request *req)
 {
-	if (blk_mark_rq_complete(req))
-		return;
-
 	if (req->q->mq_ops) {
-		blk_mq_rq_timed_out(req, false);
+		/*
+		 * All we need to ensure is that timeout scan takes place
+		 * immediately and that scan sees the new timeout value.
+		 * No need for fancy synchronizations.
+		 */
+		blk_rq_set_deadline(req, jiffies);
+		mod_timer(&req->q->timeout, 0);
 	} else {
+		if (blk_mark_rq_complete(req))
+			return;
 		blk_delete_timer(req);
 		blk_rq_timed_out(req);
 	}
@@ -208,7 +215,8 @@ void blk_add_timer(struct request *req)
 	if (!req->timeout)
 		req->timeout = q->rq_timeout;
 
-	WRITE_ONCE(req->deadline, jiffies + req->timeout);
+	blk_rq_set_deadline(req, jiffies + req->timeout);
+	req->rq_flags &= ~RQF_MQ_TIMEOUT_EXPIRED;
 
 	/*
 	 * Only the non-mq case needs to add the request to a protected list.
@@ -222,7 +230,7 @@ void blk_add_timer(struct request *req)
 	 * than an existing one, modify the timer. Round up to next nearest
 	 * second.
 	 */
-	expiry = blk_rq_timeout(round_jiffies_up(req->deadline));
+	expiry = blk_rq_timeout(round_jiffies_up(blk_rq_deadline(req)));
 
 	if (!timer_pending(&q->timeout) ||
 	    time_before(expiry, q->timeout.expires)) {
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index ae8de9780085..f92fc84b5e2c 100644
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -697,7 +697,15 @@ u64 wbt_default_latency_nsec(struct request_queue *q)
 
 static int wbt_data_dir(const struct request *rq)
 {
-	return rq_data_dir(rq);
+	const int op = req_op(rq);
+
+	if (op == REQ_OP_READ)
+		return READ;
+	else if (op == REQ_OP_WRITE || op == REQ_OP_FLUSH)
+		return WRITE;
+
+	/* don't account */
+	return -1;
 }
 
 int wbt_init(struct request_queue *q)
diff --git a/block/blk-zoned.c b/block/blk-zoned.c
index ff57fb51b338..acb7252c7e81 100644
--- a/block/blk-zoned.c
+++ b/block/blk-zoned.c
@@ -21,6 +21,48 @@ static inline sector_t blk_zone_start(struct request_queue *q,
 	return sector & ~zone_mask;
 }
 
+/*
+ * Return true if a request is a write requests that needs zone write locking.
+ */
+bool blk_req_needs_zone_write_lock(struct request *rq)
+{
+	if (!rq->q->seq_zones_wlock)
+		return false;
+
+	if (blk_rq_is_passthrough(rq))
+		return false;
+
+	switch (req_op(rq)) {
+	case REQ_OP_WRITE_ZEROES:
+	case REQ_OP_WRITE_SAME:
+	case REQ_OP_WRITE:
+		return blk_rq_zone_is_seq(rq);
+	default:
+		return false;
+	}
+}
+EXPORT_SYMBOL_GPL(blk_req_needs_zone_write_lock);
+
+void __blk_req_zone_write_lock(struct request *rq)
+{
+	if (WARN_ON_ONCE(test_and_set_bit(blk_rq_zone_no(rq),
+					  rq->q->seq_zones_wlock)))
+		return;
+
+	WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED);
+	rq->rq_flags |= RQF_ZONE_WRITE_LOCKED;
+}
+EXPORT_SYMBOL_GPL(__blk_req_zone_write_lock);
+
+void __blk_req_zone_write_unlock(struct request *rq)
+{
+	rq->rq_flags &= ~RQF_ZONE_WRITE_LOCKED;
+	if (rq->q->seq_zones_wlock)
+		WARN_ON_ONCE(!test_and_clear_bit(blk_rq_zone_no(rq),
+						 rq->q->seq_zones_wlock));
+}
+EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock);
+
 /*
  * Check that a zone report belongs to the partition.
  * If yes, fix its start sector and write pointer, copy it in the
diff --git a/block/blk.h b/block/blk.h
index 442098aa9463..46db5dc83dcb 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -119,34 +119,24 @@ void blk_account_io_start(struct request *req, bool new_io);
 void blk_account_io_completion(struct request *req, unsigned int bytes);
 void blk_account_io_done(struct request *req);
 
-/*
- * Internal atomic flags for request handling
- */
-enum rq_atomic_flags {
-	/*
-	 * Keep these two bits first - not because we depend on the
-	 * value of them, but we do depend on them being in the same
-	 * byte of storage to ensure ordering on writes. Keeping them
-	 * first will achieve that nicely.
-	 */
-	REQ_ATOM_COMPLETE = 0,
-	REQ_ATOM_STARTED,
-
-	REQ_ATOM_POLL_SLEPT,
-};
-
 /*
  * EH timer and IO completion will both attempt to 'grab' the request, make
- * sure that only one of them succeeds
+ * sure that only one of them succeeds. Steal the bottom bit of the
+ * __deadline field for this.
  */
 static inline int blk_mark_rq_complete(struct request *rq)
 {
-	return test_and_set_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
+	return test_and_set_bit(0, &rq->__deadline);
 }
 
 static inline void blk_clear_rq_complete(struct request *rq)
 {
-	clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
+	clear_bit(0, &rq->__deadline);
+}
+
+static inline bool blk_rq_is_complete(struct request *rq)
+{
+	return test_bit(0, &rq->__deadline);
 }
 
 /*
@@ -172,6 +162,9 @@ static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq
 		e->type->ops.sq.elevator_deactivate_req_fn(q, rq);
 }
 
+int elv_register_queue(struct request_queue *q);
+void elv_unregister_queue(struct request_queue *q);
+
 struct hd_struct *__disk_get_part(struct gendisk *disk, int partno);
 
 #ifdef CONFIG_FAIL_IO_TIMEOUT
@@ -245,6 +238,21 @@ static inline void req_set_nomerge(struct request_queue *q, struct request *req)
 		q->last_merge = NULL;
 }
 
+/*
+ * Steal a bit from this field for legacy IO path atomic IO marking. Note that
+ * setting the deadline clears the bottom bit, potentially clearing the
+ * completed bit. The user has to be OK with this (current ones are fine).
+ */
+static inline void blk_rq_set_deadline(struct request *rq, unsigned long time)
+{
+	rq->__deadline = time & ~0x1UL;
+}
+
+static inline unsigned long blk_rq_deadline(struct request *rq)
+{
+	return rq->__deadline & ~0x1UL;
+}
+
 /*
  * Internal io_context interface
  */
diff --git a/block/bounce.c b/block/bounce.c
index 1d05c422c932..6a3e68292273 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -113,45 +113,50 @@ int init_emergency_isa_pool(void)
 static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
 {
 	unsigned char *vfrom;
-	struct bio_vec tovec, *fromvec = from->bi_io_vec;
+	struct bio_vec tovec, fromvec;
 	struct bvec_iter iter;
+	/*
+	 * The bio of @from is created by bounce, so we can iterate
+	 * its bvec from start to end, but the @from->bi_iter can't be
+	 * trusted because it might be changed by splitting.
+	 */
+	struct bvec_iter from_iter = BVEC_ITER_ALL_INIT;
 
 	bio_for_each_segment(tovec, to, iter) {
-		if (tovec.bv_page != fromvec->bv_page) {
+		fromvec = bio_iter_iovec(from, from_iter);
+		if (tovec.bv_page != fromvec.bv_page) {
 			/*
 			 * fromvec->bv_offset and fromvec->bv_len might have
 			 * been modified by the block layer, so use the original
 			 * copy, bounce_copy_vec already uses tovec->bv_len
 			 */
-			vfrom = page_address(fromvec->bv_page) +
+			vfrom = page_address(fromvec.bv_page) +
 				tovec.bv_offset;
 
 			bounce_copy_vec(&tovec, vfrom);
 			flush_dcache_page(tovec.bv_page);
 		}
-
-		fromvec++;
+		bio_advance_iter(from, &from_iter, tovec.bv_len);
 	}
 }
 
 static void bounce_end_io(struct bio *bio, mempool_t *pool)
 {
 	struct bio *bio_orig = bio->bi_private;
-	struct bio_vec *bvec, *org_vec;
+	struct bio_vec *bvec, orig_vec;
 	int i;
-	int start = bio_orig->bi_iter.bi_idx;
+	struct bvec_iter orig_iter = bio_orig->bi_iter;
 
 	/*
 	 * free up bounce indirect pages used
 	 */
 	bio_for_each_segment_all(bvec, bio, i) {
-		org_vec = bio_orig->bi_io_vec + i + start;
-
-		if (bvec->bv_page == org_vec->bv_page)
-			continue;
-
-		dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
-		mempool_free(bvec->bv_page, pool);
+		orig_vec = bio_iter_iovec(bio_orig, orig_iter);
+		if (bvec->bv_page != orig_vec.bv_page) {
+			dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
+			mempool_free(bvec->bv_page, pool);
+		}
+		bio_advance_iter(bio_orig, &orig_iter, orig_vec.bv_len);
 	}
 
 	bio_orig->bi_status = bio->bi_status;
diff --git a/block/bsg-lib.c b/block/bsg-lib.c
index 15d25ccd51a5..1474153f73e3 100644
--- a/block/bsg-lib.c
+++ b/block/bsg-lib.c
@@ -30,7 +30,7 @@
 
 /**
  * bsg_teardown_job - routine to teardown a bsg job
- * @job: bsg_job that is to be torn down
+ * @kref: kref inside bsg_job that is to be torn down
  */
 static void bsg_teardown_job(struct kref *kref)
 {
@@ -251,6 +251,7 @@ static void bsg_exit_rq(struct request_queue *q, struct request *req)
  * @name: device to give bsg device
  * @job_fn: bsg job handler
  * @dd_job_size: size of LLD data needed for each job
+ * @release: @dev release function
  */
 struct request_queue *bsg_setup_queue(struct device *dev, const char *name,
 		bsg_job_fn *job_fn, int dd_job_size,
diff --git a/block/bsg.c b/block/bsg.c
index 452f94f1c5d4..06dc96e1f670 100644
--- a/block/bsg.c
+++ b/block/bsg.c
@@ -32,6 +32,9 @@
 #define BSG_DESCRIPTION	"Block layer SCSI generic (bsg) driver"
 #define BSG_VERSION	"0.4"
 
+#define bsg_dbg(bd, fmt, ...) \
+	pr_debug("%s: " fmt, (bd)->name, ##__VA_ARGS__)
+
 struct bsg_device {
 	struct request_queue *queue;
 	spinlock_t lock;
@@ -55,14 +58,6 @@ enum {
 #define BSG_DEFAULT_CMDS	64
 #define BSG_MAX_DEVS		32768
 
-#undef BSG_DEBUG
-
-#ifdef BSG_DEBUG
-#define dprintk(fmt, args...) printk(KERN_ERR "%s: " fmt, __func__, ##args)
-#else
-#define dprintk(fmt, args...)
-#endif
-
 static DEFINE_MUTEX(bsg_mutex);
 static DEFINE_IDR(bsg_minor_idr);
 
@@ -123,7 +118,7 @@ static struct bsg_command *bsg_alloc_command(struct bsg_device *bd)
 
 	bc->bd = bd;
 	INIT_LIST_HEAD(&bc->list);
-	dprintk("%s: returning free cmd %p\n", bd->name, bc);
+	bsg_dbg(bd, "returning free cmd %p\n", bc);
 	return bc;
 out:
 	spin_unlock_irq(&bd->lock);
@@ -222,7 +217,8 @@ bsg_map_hdr(struct bsg_device *bd, struct sg_io_v4 *hdr, fmode_t mode)
 	if (!bcd->class_dev)
 		return ERR_PTR(-ENXIO);
 
-	dprintk("map hdr %llx/%u %llx/%u\n", (unsigned long long) hdr->dout_xferp,
+	bsg_dbg(bd, "map hdr %llx/%u %llx/%u\n",
+		(unsigned long long) hdr->dout_xferp,
 		hdr->dout_xfer_len, (unsigned long long) hdr->din_xferp,
 		hdr->din_xfer_len);
 
@@ -299,8 +295,8 @@ static void bsg_rq_end_io(struct request *rq, blk_status_t status)
 	struct bsg_device *bd = bc->bd;
 	unsigned long flags;
 
-	dprintk("%s: finished rq %p bc %p, bio %p\n",
-		bd->name, rq, bc, bc->bio);
+	bsg_dbg(bd, "finished rq %p bc %p, bio %p\n",
+		rq, bc, bc->bio);
 
 	bc->hdr.duration = jiffies_to_msecs(jiffies - bc->hdr.duration);
 
@@ -333,7 +329,7 @@ static void bsg_add_command(struct bsg_device *bd, struct request_queue *q,
 	list_add_tail(&bc->list, &bd->busy_list);
 	spin_unlock_irq(&bd->lock);
 
-	dprintk("%s: queueing rq %p, bc %p\n", bd->name, rq, bc);
+	bsg_dbg(bd, "queueing rq %p, bc %p\n", rq, bc);
 
 	rq->end_io_data = bc;
 	blk_execute_rq_nowait(q, NULL, rq, at_head, bsg_rq_end_io);
@@ -379,7 +375,7 @@ static struct bsg_command *bsg_get_done_cmd(struct bsg_device *bd)
 		}
 	} while (1);
 
-	dprintk("%s: returning done %p\n", bd->name, bc);
+	bsg_dbg(bd, "returning done %p\n", bc);
 
 	return bc;
 }
@@ -390,7 +386,7 @@ static int blk_complete_sgv4_hdr_rq(struct request *rq, struct sg_io_v4 *hdr,
 	struct scsi_request *req = scsi_req(rq);
 	int ret = 0;
 
-	dprintk("rq %p bio %p 0x%x\n", rq, bio, req->result);
+	pr_debug("rq %p bio %p 0x%x\n", rq, bio, req->result);
 	/*
 	 * fill in all the output members
 	 */
@@ -469,7 +465,7 @@ static int bsg_complete_all_commands(struct bsg_device *bd)
 	struct bsg_command *bc;
 	int ret, tret;
 
-	dprintk("%s: entered\n", bd->name);
+	bsg_dbg(bd, "entered\n");
 
 	/*
 	 * wait for all commands to complete
@@ -572,7 +568,7 @@ bsg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 	int ret;
 	ssize_t bytes_read;
 
-	dprintk("%s: read %zd bytes\n", bd->name, count);
+	bsg_dbg(bd, "read %zd bytes\n", count);
 
 	bsg_set_block(bd, file);
 
@@ -646,7 +642,7 @@ bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
 	ssize_t bytes_written;
 	int ret;
 
-	dprintk("%s: write %zd bytes\n", bd->name, count);
+	bsg_dbg(bd, "write %zd bytes\n", count);
 
 	if (unlikely(uaccess_kernel()))
 		return -EINVAL;
@@ -664,7 +660,7 @@ bsg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
 	if (!bytes_written || err_block_err(ret))
 		bytes_written = ret;
 
-	dprintk("%s: returning %zd\n", bd->name, bytes_written);
+	bsg_dbg(bd, "returning %zd\n", bytes_written);
 	return bytes_written;
 }
 
@@ -717,7 +713,7 @@ static int bsg_put_device(struct bsg_device *bd)
 	hlist_del(&bd->dev_list);
 	mutex_unlock(&bsg_mutex);
 
-	dprintk("%s: tearing down\n", bd->name);
+	bsg_dbg(bd, "tearing down\n");
 
 	/*
 	 * close can always block
@@ -744,9 +740,7 @@ static struct bsg_device *bsg_add_device(struct inode *inode,
 					 struct file *file)
 {
 	struct bsg_device *bd;
-#ifdef BSG_DEBUG
 	unsigned char buf[32];
-#endif
 
 	if (!blk_queue_scsi_passthrough(rq)) {
 		WARN_ONCE(true, "Attempt to register a non-SCSI queue\n");
@@ -771,7 +765,7 @@ static struct bsg_device *bsg_add_device(struct inode *inode,
 	hlist_add_head(&bd->dev_list, bsg_dev_idx_hash(iminor(inode)));
 
 	strncpy(bd->name, dev_name(rq->bsg_dev.class_dev), sizeof(bd->name) - 1);
-	dprintk("bound to <%s>, max queue %d\n",
+	bsg_dbg(bd, "bound to <%s>, max queue %d\n",
 		format_dev_t(buf, inode->i_rdev), bd->max_queue);
 
 	mutex_unlock(&bsg_mutex);
@@ -845,19 +839,19 @@ static int bsg_release(struct inode *inode, struct file *file)
 	return bsg_put_device(bd);
 }
 
-static unsigned int bsg_poll(struct file *file, poll_table *wait)
+static __poll_t bsg_poll(struct file *file, poll_table *wait)
 {
 	struct bsg_device *bd = file->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &bd->wq_done, wait);
 	poll_wait(file, &bd->wq_free, wait);
 
 	spin_lock_irq(&bd->lock);
 	if (!list_empty(&bd->done_list))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	if (bd->queued_cmds < bd->max_queue)
-		mask |= POLLOUT;
+		mask |= EPOLLOUT;
 	spin_unlock_irq(&bd->lock);
 
 	return mask;
diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c
index b83f77460d28..9de9f156e203 100644
--- a/block/deadline-iosched.c
+++ b/block/deadline-iosched.c
@@ -50,8 +50,6 @@ struct deadline_data {
 	int front_merges;
 };
 
-static void deadline_move_request(struct deadline_data *, struct request *);
-
 static inline struct rb_root *
 deadline_rb_root(struct deadline_data *dd, struct request *rq)
 {
@@ -100,6 +98,12 @@ deadline_add_request(struct request_queue *q, struct request *rq)
 	struct deadline_data *dd = q->elevator->elevator_data;
 	const int data_dir = rq_data_dir(rq);
 
+	/*
+	 * This may be a requeue of a write request that has locked its
+	 * target zone. If it is the case, this releases the zone lock.
+	 */
+	blk_req_zone_write_unlock(rq);
+
 	deadline_add_rq_rb(dd, rq);
 
 	/*
@@ -190,6 +194,12 @@ deadline_move_to_dispatch(struct deadline_data *dd, struct request *rq)
 {
 	struct request_queue *q = rq->q;
 
+	/*
+	 * For a zoned block device, write requests must write lock their
+	 * target zone.
+	 */
+	blk_req_zone_write_lock(rq);
+
 	deadline_remove_request(q, rq);
 	elv_dispatch_add_tail(q, rq);
 }
@@ -230,6 +240,69 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)
 	return 0;
 }
 
+/*
+ * For the specified data direction, return the next request to dispatch using
+ * arrival ordered lists.
+ */
+static struct request *
+deadline_fifo_request(struct deadline_data *dd, int data_dir)
+{
+	struct request *rq;
+
+	if (WARN_ON_ONCE(data_dir != READ && data_dir != WRITE))
+		return NULL;
+
+	if (list_empty(&dd->fifo_list[data_dir]))
+		return NULL;
+
+	rq = rq_entry_fifo(dd->fifo_list[data_dir].next);
+	if (data_dir == READ || !blk_queue_is_zoned(rq->q))
+		return rq;
+
+	/*
+	 * Look for a write request that can be dispatched, that is one with
+	 * an unlocked target zone.
+	 */
+	list_for_each_entry(rq, &dd->fifo_list[WRITE], queuelist) {
+		if (blk_req_can_dispatch_to_zone(rq))
+			return rq;
+	}
+
+	return NULL;
+}
+
+/*
+ * For the specified data direction, return the next request to dispatch using
+ * sector position sorted lists.
+ */
+static struct request *
+deadline_next_request(struct deadline_data *dd, int data_dir)
+{
+	struct request *rq;
+
+	if (WARN_ON_ONCE(data_dir != READ && data_dir != WRITE))
+		return NULL;
+
+	rq = dd->next_rq[data_dir];
+	if (!rq)
+		return NULL;
+
+	if (data_dir == READ || !blk_queue_is_zoned(rq->q))
+		return rq;
+
+	/*
+	 * Look for a write request that can be dispatched, that is one with
+	 * an unlocked target zone.
+	 */
+	while (rq) {
+		if (blk_req_can_dispatch_to_zone(rq))
+			return rq;
+		rq = deadline_latter_request(rq);
+	}
+
+	return NULL;
+}
+
 /*
  * deadline_dispatch_requests selects the best request according to
  * read/write expire, fifo_batch, etc
@@ -239,16 +312,15 @@ static int deadline_dispatch_requests(struct request_queue *q, int force)
 	struct deadline_data *dd = q->elevator->elevator_data;
 	const int reads = !list_empty(&dd->fifo_list[READ]);
 	const int writes = !list_empty(&dd->fifo_list[WRITE]);
-	struct request *rq;
+	struct request *rq, *next_rq;
 	int data_dir;
 
 	/*
 	 * batches are currently reads XOR writes
 	 */
-	if (dd->next_rq[WRITE])
-		rq = dd->next_rq[WRITE];
-	else
-		rq = dd->next_rq[READ];
+	rq = deadline_next_request(dd, WRITE);
+	if (!rq)
+		rq = deadline_next_request(dd, READ);
 
 	if (rq && dd->batching < dd->fifo_batch)
 		/* we have a next request are still entitled to batch */
@@ -262,7 +334,8 @@ static int deadline_dispatch_requests(struct request_queue *q, int force)
 	if (reads) {
 		BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[READ]));
 
-		if (writes && (dd->starved++ >= dd->writes_starved))
+		if (deadline_fifo_request(dd, WRITE) &&
+		    (dd->starved++ >= dd->writes_starved))
 			goto dispatch_writes;
 
 		data_dir = READ;
@@ -291,21 +364,29 @@ static int deadline_dispatch_requests(struct request_queue *q, int force)
 	/*
 	 * we are not running a batch, find best request for selected data_dir
 	 */
-	if (deadline_check_fifo(dd, data_dir) || !dd->next_rq[data_dir]) {
+	next_rq = deadline_next_request(dd, data_dir);
+	if (deadline_check_fifo(dd, data_dir) || !next_rq) {
 		/*
 		 * A deadline has expired, the last request was in the other
 		 * direction, or we have run out of higher-sectored requests.
 		 * Start again from the request with the earliest expiry time.
 		 */
-		rq = rq_entry_fifo(dd->fifo_list[data_dir].next);
+		rq = deadline_fifo_request(dd, data_dir);
 	} else {
 		/*
 		 * The last req was the same dir and we have a next request in
 		 * sort order. No expired requests so continue on from here.
 		 */
-		rq = dd->next_rq[data_dir];
+		rq = next_rq;
 	}
 
+	/*
+	 * For a zoned block device, if we only have writes queued and none of
+	 * them can be dispatched, rq will be NULL.
+	 */
+	if (!rq)
+		return 0;
+
 	dd->batching = 0;
 
 dispatch_request:
@@ -318,6 +399,16 @@ static int deadline_dispatch_requests(struct request_queue *q, int force)
 	return 1;
 }
 
+/*
+ * For zoned block devices, write unlock the target zone of completed
+ * write requests.
+ */
+static void
+deadline_completed_request(struct request_queue *q, struct request *rq)
+{
+	blk_req_zone_write_unlock(rq);
+}
+
 static void deadline_exit_queue(struct elevator_queue *e)
 {
 	struct deadline_data *dd = e->elevator_data;
@@ -439,6 +530,7 @@ static struct elevator_type iosched_deadline = {
 		.elevator_merged_fn =		deadline_merged_request,
 		.elevator_merge_req_fn =	deadline_merged_requests,
 		.elevator_dispatch_fn =		deadline_dispatch_requests,
+		.elevator_completed_req_fn =	deadline_completed_request,
 		.elevator_add_req_fn =		deadline_add_request,
 		.elevator_former_req_fn =	elv_rb_former_request,
 		.elevator_latter_req_fn =	elv_rb_latter_request,
diff --git a/block/elevator.c b/block/elevator.c
index 7bda083d5968..e87e9b43aba0 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -869,6 +869,8 @@ int elv_register_queue(struct request_queue *q)
 	struct elevator_queue *e = q->elevator;
 	int error;
 
+	lockdep_assert_held(&q->sysfs_lock);
+
 	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
 	if (!error) {
 		struct elv_fs_entry *attr = e->type->elevator_attrs;
@@ -886,10 +888,11 @@ int elv_register_queue(struct request_queue *q)
 	}
 	return error;
 }
-EXPORT_SYMBOL(elv_register_queue);
 
 void elv_unregister_queue(struct request_queue *q)
 {
+	lockdep_assert_held(&q->sysfs_lock);
+
 	if (q) {
 		struct elevator_queue *e = q->elevator;
 
@@ -900,7 +903,6 @@ void elv_unregister_queue(struct request_queue *q)
 		wbt_enable_default(q);
 	}
 }
-EXPORT_SYMBOL(elv_unregister_queue);
 
 int elv_register(struct elevator_type *e)
 {
@@ -967,7 +969,10 @@ static int elevator_switch_mq(struct request_queue *q,
 {
 	int ret;
 
+	lockdep_assert_held(&q->sysfs_lock);
+
 	blk_mq_freeze_queue(q);
+	blk_mq_quiesce_queue(q);
 
 	if (q->elevator) {
 		if (q->elevator->registered)
@@ -994,6 +999,7 @@ static int elevator_switch_mq(struct request_queue *q,
 		blk_add_trace_msg(q, "elv switch: none");
 
 out:
+	blk_mq_unquiesce_queue(q);
 	blk_mq_unfreeze_queue(q);
 	return ret;
 }
@@ -1010,6 +1016,8 @@ static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
 	bool old_registered = false;
 	int err;
 
+	lockdep_assert_held(&q->sysfs_lock);
+
 	if (q->mq_ops)
 		return elevator_switch_mq(q, new_e);
 
diff --git a/block/genhd.c b/block/genhd.c
index 96a66f671720..9656f9e9f99e 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -547,7 +547,7 @@ static int exact_lock(dev_t devt, void *data)
 {
 	struct gendisk *p = data;
 
-	if (!get_disk(p))
+	if (!get_disk_and_module(p))
 		return -1;
 	return 0;
 }
@@ -629,16 +629,18 @@ static void register_disk(struct device *parent, struct gendisk *disk)
 }
 
 /**
- * device_add_disk - add partitioning information to kernel list
+ * __device_add_disk - add disk information to kernel list
  * @parent: parent device for the disk
  * @disk: per-device partitioning information
+ * @register_queue: register the queue if set to true
  *
  * This function registers the partitioning information in @disk
  * with the kernel.
  *
  * FIXME: error handling
  */
-void device_add_disk(struct device *parent, struct gendisk *disk)
+static void __device_add_disk(struct device *parent, struct gendisk *disk,
+			      bool register_queue)
 {
 	dev_t devt;
 	int retval;
@@ -682,7 +684,8 @@ void device_add_disk(struct device *parent, struct gendisk *disk)
 				    exact_match, exact_lock, disk);
 	}
 	register_disk(parent, disk);
-	blk_register_queue(disk);
+	if (register_queue)
+		blk_register_queue(disk);
 
 	/*
 	 * Take an extra ref on queue which will be put on disk_release()
@@ -693,8 +696,19 @@ void device_add_disk(struct device *parent, struct gendisk *disk)
 	disk_add_events(disk);
 	blk_integrity_add(disk);
 }
+
+void device_add_disk(struct device *parent, struct gendisk *disk)
+{
+	__device_add_disk(parent, disk, true);
+}
 EXPORT_SYMBOL(device_add_disk);
 
+void device_add_disk_no_queue_reg(struct device *parent, struct gendisk *disk)
+{
+	__device_add_disk(parent, disk, false);
+}
+EXPORT_SYMBOL(device_add_disk_no_queue_reg);
+
 void del_gendisk(struct gendisk *disk)
 {
 	struct disk_part_iter piter;
@@ -703,6 +717,11 @@ void del_gendisk(struct gendisk *disk)
 	blk_integrity_del(disk);
 	disk_del_events(disk);
 
+	/*
+	 * Block lookups of the disk until all bdevs are unhashed and the
+	 * disk is marked as dead (GENHD_FL_UP cleared).
+	 */
+	down_write(&disk->lookup_sem);
 	/* invalidate stuff */
 	disk_part_iter_init(&piter, disk,
 			     DISK_PITER_INCL_EMPTY | DISK_PITER_REVERSE);
@@ -717,6 +736,7 @@ void del_gendisk(struct gendisk *disk)
 	bdev_unhash_inode(disk_devt(disk));
 	set_capacity(disk, 0);
 	disk->flags &= ~GENHD_FL_UP;
+	up_write(&disk->lookup_sem);
 
 	if (!(disk->flags & GENHD_FL_HIDDEN))
 		sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi");
@@ -725,7 +745,8 @@ void del_gendisk(struct gendisk *disk)
 		 * Unregister bdi before releasing device numbers (as they can
 		 * get reused and we'd get clashes in sysfs).
 		 */
-		bdi_unregister(disk->queue->backing_dev_info);
+		if (!(disk->flags & GENHD_FL_HIDDEN))
+			bdi_unregister(disk->queue->backing_dev_info);
 		blk_unregister_queue(disk);
 	} else {
 		WARN_ON(1);
@@ -794,16 +815,28 @@ struct gendisk *get_gendisk(dev_t devt, int *partno)
 
 		spin_lock_bh(&ext_devt_lock);
 		part = idr_find(&ext_devt_idr, blk_mangle_minor(MINOR(devt)));
-		if (part && get_disk(part_to_disk(part))) {
+		if (part && get_disk_and_module(part_to_disk(part))) {
 			*partno = part->partno;
 			disk = part_to_disk(part);
 		}
 		spin_unlock_bh(&ext_devt_lock);
 	}
 
-	if (disk && unlikely(disk->flags & GENHD_FL_HIDDEN)) {
-		put_disk(disk);
+	if (!disk)
+		return NULL;
+
+	/*
+	 * Synchronize with del_gendisk() to not return disk that is being
+	 * destroyed.
+	 */
+	down_read(&disk->lookup_sem);
+	if (unlikely((disk->flags & GENHD_FL_HIDDEN) ||
+		     !(disk->flags & GENHD_FL_UP))) {
+		up_read(&disk->lookup_sem);
+		put_disk_and_module(disk);
 		disk = NULL;
+	} else {
+		up_read(&disk->lookup_sem);
 	}
 	return disk;
 }
@@ -1403,6 +1436,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id)
 			kfree(disk);
 			return NULL;
 		}
+		init_rwsem(&disk->lookup_sem);
 		disk->node_id = node_id;
 		if (disk_expand_part_tbl(disk, 0)) {
 			free_part_stats(&disk->part0);
@@ -1438,7 +1472,7 @@ struct gendisk *__alloc_disk_node(int minors, int node_id)
 }
 EXPORT_SYMBOL(__alloc_disk_node);
 
-struct kobject *get_disk(struct gendisk *disk)
+struct kobject *get_disk_and_module(struct gendisk *disk)
 {
 	struct module *owner;
 	struct kobject *kobj;
@@ -1456,17 +1490,30 @@ struct kobject *get_disk(struct gendisk *disk)
 	return kobj;
 
 }
-
-EXPORT_SYMBOL(get_disk);
+EXPORT_SYMBOL(get_disk_and_module);
 
 void put_disk(struct gendisk *disk)
 {
 	if (disk)
 		kobject_put(&disk_to_dev(disk)->kobj);
 }
-
 EXPORT_SYMBOL(put_disk);
 
+/*
+ * This is a counterpart of get_disk_and_module() and thus also of
+ * get_gendisk().
+ */
+void put_disk_and_module(struct gendisk *disk)
+{
+	if (disk) {
+		struct module *owner = disk->fops->owner;
+
+		put_disk(disk);
+		module_put(owner);
+	}
+}
+EXPORT_SYMBOL(put_disk_and_module);
+
 static void set_disk_ro_uevent(struct gendisk *gd, int ro)
 {
 	char event[] = "DISK_RO=1";
diff --git a/block/ioctl.c b/block/ioctl.c
index 1668506d8ed8..3884d810efd2 100644
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -225,7 +225,7 @@ static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode,
 
 	if (start + len > i_size_read(bdev->bd_inode))
 		return -EINVAL;
-	truncate_inode_pages_range(mapping, start, start + len);
+	truncate_inode_pages_range(mapping, start, start + len - 1);
 	return blkdev_issue_discard(bdev, start >> 9, len >> 9,
 				    GFP_KERNEL, flags);
 }
diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c
index f95c60774ce8..0d6d25e32e1f 100644
--- a/block/kyber-iosched.c
+++ b/block/kyber-iosched.c
@@ -833,6 +833,7 @@ static struct elevator_type kyber_sched = {
 		.limit_depth = kyber_limit_depth,
 		.prepare_request = kyber_prepare_request,
 		.finish_request = kyber_finish_request,
+		.requeue_request = kyber_finish_request,
 		.completed_request = kyber_completed_request,
 		.dispatch_request = kyber_dispatch_request,
 		.has_work = kyber_has_work,
diff --git a/block/mq-deadline.c b/block/mq-deadline.c
index 0179e484ec98..8ec0ba9f5386 100644
--- a/block/mq-deadline.c
+++ b/block/mq-deadline.c
@@ -59,6 +59,7 @@ struct deadline_data {
 	int front_merges;
 
 	spinlock_t lock;
+	spinlock_t zone_lock;
 	struct list_head dispatch;
 };
 
@@ -191,14 +192,84 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)
 	return 0;
 }
 
+/*
+ * For the specified data direction, return the next request to
+ * dispatch using arrival ordered lists.
+ */
+static struct request *
+deadline_fifo_request(struct deadline_data *dd, int data_dir)
+{
+	struct request *rq;
+	unsigned long flags;
+
+	if (WARN_ON_ONCE(data_dir != READ && data_dir != WRITE))
+		return NULL;
+
+	if (list_empty(&dd->fifo_list[data_dir]))
+		return NULL;
+
+	rq = rq_entry_fifo(dd->fifo_list[data_dir].next);
+	if (data_dir == READ || !blk_queue_is_zoned(rq->q))
+		return rq;
+
+	/*
+	 * Look for a write request that can be dispatched, that is one with
+	 * an unlocked target zone.
+	 */
+	spin_lock_irqsave(&dd->zone_lock, flags);
+	list_for_each_entry(rq, &dd->fifo_list[WRITE], queuelist) {
+		if (blk_req_can_dispatch_to_zone(rq))
+			goto out;
+	}
+	rq = NULL;
+out:
+	spin_unlock_irqrestore(&dd->zone_lock, flags);
+
+	return rq;
+}
+
+/*
+ * For the specified data direction, return the next request to
+ * dispatch using sector position sorted lists.
+ */
+static struct request *
+deadline_next_request(struct deadline_data *dd, int data_dir)
+{
+	struct request *rq;
+	unsigned long flags;
+
+	if (WARN_ON_ONCE(data_dir != READ && data_dir != WRITE))
+		return NULL;
+
+	rq = dd->next_rq[data_dir];
+	if (!rq)
+		return NULL;
+
+	if (data_dir == READ || !blk_queue_is_zoned(rq->q))
+		return rq;
+
+	/*
+	 * Look for a write request that can be dispatched, that is one with
+	 * an unlocked target zone.
+	 */
+	spin_lock_irqsave(&dd->zone_lock, flags);
+	while (rq) {
+		if (blk_req_can_dispatch_to_zone(rq))
+			break;
+		rq = deadline_latter_request(rq);
+	}
+	spin_unlock_irqrestore(&dd->zone_lock, flags);
+
+	return rq;
+}
+
 /*
  * deadline_dispatch_requests selects the best request according to
  * read/write expire, fifo_batch, etc
  */
-static struct request *__dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
+static struct request *__dd_dispatch_request(struct deadline_data *dd)
 {
-	struct deadline_data *dd = hctx->queue->elevator->elevator_data;
-	struct request *rq;
+	struct request *rq, *next_rq;
 	bool reads, writes;
 	int data_dir;
 
@@ -214,10 +285,9 @@ static struct request *__dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
 	/*
 	 * batches are currently reads XOR writes
 	 */
-	if (dd->next_rq[WRITE])
-		rq = dd->next_rq[WRITE];
-	else
-		rq = dd->next_rq[READ];
+	rq = deadline_next_request(dd, WRITE);
+	if (!rq)
+		rq = deadline_next_request(dd, READ);
 
 	if (rq && dd->batching < dd->fifo_batch)
 		/* we have a next request are still entitled to batch */
@@ -231,7 +301,8 @@ static struct request *__dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
 	if (reads) {
 		BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[READ]));
 
-		if (writes && (dd->starved++ >= dd->writes_starved))
+		if (deadline_fifo_request(dd, WRITE) &&
+		    (dd->starved++ >= dd->writes_starved))
 			goto dispatch_writes;
 
 		data_dir = READ;
@@ -260,21 +331,29 @@ static struct request *__dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
 	/*
 	 * we are not running a batch, find best request for selected data_dir
 	 */
-	if (deadline_check_fifo(dd, data_dir) || !dd->next_rq[data_dir]) {
+	next_rq = deadline_next_request(dd, data_dir);
+	if (deadline_check_fifo(dd, data_dir) || !next_rq) {
 		/*
 		 * A deadline has expired, the last request was in the other
 		 * direction, or we have run out of higher-sectored requests.
 		 * Start again from the request with the earliest expiry time.
 		 */
-		rq = rq_entry_fifo(dd->fifo_list[data_dir].next);
+		rq = deadline_fifo_request(dd, data_dir);
 	} else {
 		/*
 		 * The last req was the same dir and we have a next request in
 		 * sort order. No expired requests so continue on from here.
 		 */
-		rq = dd->next_rq[data_dir];
+		rq = next_rq;
 	}
 
+	/*
+	 * For a zoned block device, if we only have writes queued and none of
+	 * them can be dispatched, rq will be NULL.
+	 */
+	if (!rq)
+		return NULL;
+
 	dd->batching = 0;
 
 dispatch_request:
@@ -284,17 +363,27 @@ static struct request *__dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
 	dd->batching++;
 	deadline_move_request(dd, rq);
 done:
+	/*
+	 * If the request needs its target zone locked, do it.
+	 */
+	blk_req_zone_write_lock(rq);
 	rq->rq_flags |= RQF_STARTED;
 	return rq;
 }
 
+/*
+ * One confusing aspect here is that we get called for a specific
+ * hardware queue, but we return a request that may not be for a
+ * different hardware queue. This is because mq-deadline has shared
+ * state for all hardware queues, in terms of sorting, FIFOs, etc.
+ */
 static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx)
 {
 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;
 	struct request *rq;
 
 	spin_lock(&dd->lock);
-	rq = __dd_dispatch_request(hctx);
+	rq = __dd_dispatch_request(dd);
 	spin_unlock(&dd->lock);
 
 	return rq;
@@ -339,6 +428,7 @@ static int dd_init_queue(struct request_queue *q, struct elevator_type *e)
 	dd->front_merges = 1;
 	dd->fifo_batch = fifo_batch;
 	spin_lock_init(&dd->lock);
+	spin_lock_init(&dd->zone_lock);
 	INIT_LIST_HEAD(&dd->dispatch);
 
 	q->elevator = eq;
@@ -395,6 +485,12 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 	struct deadline_data *dd = q->elevator->elevator_data;
 	const int data_dir = rq_data_dir(rq);
 
+	/*
+	 * This may be a requeue of a write request that has locked its
+	 * target zone. If it is the case, this releases the zone lock.
+	 */
+	blk_req_zone_write_unlock(rq);
+
 	if (blk_mq_sched_try_insert_merge(q, rq))
 		return;
 
@@ -439,6 +535,35 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx,
 	spin_unlock(&dd->lock);
 }
 
+/*
+ * Nothing to do here. This is defined only to ensure that .finish_request
+ * method is called upon request completion.
+ */
+static void dd_prepare_request(struct request *rq, struct bio *bio)
+{
+}
+
+/*
+ * For zoned block devices, write unlock the target zone of
+ * completed write requests. Do this while holding the zone lock
+ * spinlock so that the zone is never unlocked while deadline_fifo_request()
+ * or deadline_next_request() are executing. This function is called for
+ * all requests, whether or not these requests complete successfully.
+ */
+static void dd_finish_request(struct request *rq)
+{
+	struct request_queue *q = rq->q;
+
+	if (blk_queue_is_zoned(q)) {
+		struct deadline_data *dd = q->elevator->elevator_data;
+		unsigned long flags;
+
+		spin_lock_irqsave(&dd->zone_lock, flags);
+		blk_req_zone_write_unlock(rq);
+		spin_unlock_irqrestore(&dd->zone_lock, flags);
+	}
+}
+
 static bool dd_has_work(struct blk_mq_hw_ctx *hctx)
 {
 	struct deadline_data *dd = hctx->queue->elevator->elevator_data;
@@ -640,6 +765,8 @@ static struct elevator_type mq_deadline = {
 	.ops.mq = {
 		.insert_requests	= dd_insert_requests,
 		.dispatch_request	= dd_dispatch_request,
+		.prepare_request	= dd_prepare_request,
+		.finish_request		= dd_finish_request,
 		.next_request		= elv_rb_latter_request,
 		.former_request		= elv_rb_former_request,
 		.bio_merge		= dd_bio_merge,
diff --git a/block/partition-generic.c b/block/partition-generic.c
index 91622db9aedf..08dabcd8b6ae 100644
--- a/block/partition-generic.c
+++ b/block/partition-generic.c
@@ -51,6 +51,12 @@ const char *bdevname(struct block_device *bdev, char *buf)
 
 EXPORT_SYMBOL(bdevname);
 
+const char *bio_devname(struct bio *bio, char *buf)
+{
+	return disk_name(bio->bi_disk, bio->bi_partno, buf);
+}
+EXPORT_SYMBOL(bio_devname);
+
 /*
  * There's very little reason to use this, you should really
  * have a struct block_device just about everywhere and use
diff --git a/block/partitions/msdos.c b/block/partitions/msdos.c
index 0af3a3db6fb0..82c44f7df911 100644
--- a/block/partitions/msdos.c
+++ b/block/partitions/msdos.c
@@ -301,7 +301,9 @@ static void parse_bsd(struct parsed_partitions *state,
 			continue;
 		bsd_start = le32_to_cpu(p->p_offset);
 		bsd_size = le32_to_cpu(p->p_size);
-		if (memcmp(flavour, "bsd\0", 4) == 0)
+		/* FreeBSD has relative offset if C partition offset is zero */
+		if (memcmp(flavour, "bsd\0", 4) == 0 &&
+		    le32_to_cpu(l->d_partitions[2].p_offset) == 0)
 			bsd_start += offset;
 		if (offset == bsd_start && size == bsd_size)
 			/* full parent partition, we have it already */
diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c
index edcfff974527..60b471f8621b 100644
--- a/block/scsi_ioctl.c
+++ b/block/scsi_ioctl.c
@@ -384,9 +384,10 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk,
 
 /**
  * sg_scsi_ioctl  --  handle deprecated SCSI_IOCTL_SEND_COMMAND ioctl
- * @file:	file this ioctl operates on (optional)
  * @q:		request queue to send scsi commands down
  * @disk:	gendisk to operate on (option)
+ * @mode:	mode used to open the file through which the ioctl has been
+ *		submitted
  * @sic:	userspace structure describing the command to perform
  *
  * Send down the scsi command described by @sic to the device below
@@ -415,10 +416,10 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk,
  *      Positive numbers returned are the compacted SCSI error codes (4
  *      bytes in one int) where the lowest byte is the SCSI status.
  */
-#define OMAX_SB_LEN 16          /* For backward compatibility */
 int sg_scsi_ioctl(struct request_queue *q, struct gendisk *disk, fmode_t mode,
 		struct scsi_ioctl_command __user *sic)
 {
+	enum { OMAX_SB_LEN = 16 };	/* For backward compatibility */
 	struct request *rq;
 	struct scsi_request *req;
 	int err;
@@ -692,38 +693,9 @@ int scsi_verify_blk_ioctl(struct block_device *bd, unsigned int cmd)
 	if (bd && bd == bd->bd_contains)
 		return 0;
 
-	/* Actually none of these is particularly useful on a partition,
-	 * but they are safe.
-	 */
-	switch (cmd) {
-	case SCSI_IOCTL_GET_IDLUN:
-	case SCSI_IOCTL_GET_BUS_NUMBER:
-	case SCSI_IOCTL_GET_PCI:
-	case SCSI_IOCTL_PROBE_HOST:
-	case SG_GET_VERSION_NUM:
-	case SG_SET_TIMEOUT:
-	case SG_GET_TIMEOUT:
-	case SG_GET_RESERVED_SIZE:
-	case SG_SET_RESERVED_SIZE:
-	case SG_EMULATED_HOST:
-		return 0;
-	case CDROM_GET_CAPABILITY:
-		/* Keep this until we remove the printk below.  udev sends it
-		 * and we do not want to spam dmesg about it.   CD-ROMs do
-		 * not have partitions, so we get here only for disks.
-		 */
-		return -ENOIOCTLCMD;
-	default:
-		break;
-	}
-
 	if (capable(CAP_SYS_RAWIO))
 		return 0;
 
-	/* In particular, rule out all resets and host-specific ioctls.  */
-	printk_ratelimited(KERN_WARNING
-			   "%s: sending ioctl %x to a partition!\n", current->comm, cmd);
-
 	return -ENOIOCTLCMD;
 }
 EXPORT_SYMBOL(scsi_verify_blk_ioctl);
diff --git a/block/sed-opal.c b/block/sed-opal.c
index 9ed51d0c6b1d..e4929eec547f 100644
--- a/block/sed-opal.c
+++ b/block/sed-opal.c
@@ -490,7 +490,7 @@ static int opal_discovery0_end(struct opal_dev *dev)
 
 	if (!found_com_id) {
 		pr_debug("Could not find OPAL comid for device. Returning early\n");
-		return -EOPNOTSUPP;;
+		return -EOPNOTSUPP;
 	}
 
 	dev->comid = comid;
diff --git a/certs/blacklist_nohashes.c b/certs/blacklist_nohashes.c
index 73fd99098ad7..753b703ef0ef 100644
--- a/certs/blacklist_nohashes.c
+++ b/certs/blacklist_nohashes.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 #include "blacklist.h"
 
-const char __initdata *const blacklist_hashes[] = {
+const char __initconst *const blacklist_hashes[] = {
 	NULL
 };
diff --git a/crypto/Kconfig b/crypto/Kconfig
index f7911963bb79..b75264b09a46 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -106,6 +106,7 @@ config CRYPTO_KPP
 config CRYPTO_ACOMP2
 	tristate
 	select CRYPTO_ALGAPI2
+	select SGL_ALLOC
 
 config CRYPTO_ACOMP
 	tristate
@@ -130,7 +131,7 @@ config CRYPTO_DH
 
 config CRYPTO_ECDH
 	tristate "ECDH algorithm"
-	select CRYTPO_KPP
+	select CRYPTO_KPP
 	select CRYPTO_RNG_DEFAULT
 	help
 	  Generic implementation of the ECDH algorithm
@@ -1339,6 +1340,7 @@ config CRYPTO_SALSA20_586
 	tristate "Salsa20 stream cipher algorithm (i586)"
 	depends on (X86 || UML_X86) && !64BIT
 	select CRYPTO_BLKCIPHER
+	select CRYPTO_SALSA20
 	help
 	  Salsa20 stream cipher algorithm.
 
@@ -1352,6 +1354,7 @@ config CRYPTO_SALSA20_X86_64
 	tristate "Salsa20 stream cipher algorithm (x86_64)"
 	depends on (X86 || UML_X86) && 64BIT
 	select CRYPTO_BLKCIPHER
+	select CRYPTO_SALSA20
 	help
 	  Salsa20 stream cipher algorithm.
 
diff --git a/crypto/Makefile b/crypto/Makefile
index d674884b2d51..cdbc03b35510 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
 obj-$(CONFIG_CRYPTO_SERPENT) += serpent_generic.o
 CFLAGS_serpent_generic.o := $(call cc-option,-fsched-pressure)  # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149
 obj-$(CONFIG_CRYPTO_AES) += aes_generic.o
+CFLAGS_aes_generic.o := $(call cc-option,-fno-code-hoisting) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83356
 obj-$(CONFIG_CRYPTO_AES_TI) += aes_ti.o
 obj-$(CONFIG_CRYPTO_CAMELLIA) += camellia_generic.o
 obj-$(CONFIG_CRYPTO_CAST_COMMON) += cast_common.o
diff --git a/crypto/ablk_helper.c b/crypto/ablk_helper.c
index 1441f07d0a19..09776bb1360e 100644
--- a/crypto/ablk_helper.c
+++ b/crypto/ablk_helper.c
@@ -18,9 +18,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
- * USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
@@ -28,7 +26,6 @@
 #include <linux/crypto.h>
 #include <linux/init.h>
 #include <linux/module.h>
-#include <linux/hardirq.h>
 #include <crypto/algapi.h>
 #include <crypto/cryptd.h>
 #include <crypto/ablk_helper.h>
diff --git a/crypto/aead.c b/crypto/aead.c
index f794b30a9407..60b3bbe973e7 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -54,11 +54,18 @@ int crypto_aead_setkey(struct crypto_aead *tfm,
 		       const u8 *key, unsigned int keylen)
 {
 	unsigned long alignmask = crypto_aead_alignmask(tfm);
+	int err;
 
 	if ((unsigned long)key & alignmask)
-		return setkey_unaligned(tfm, key, keylen);
+		err = setkey_unaligned(tfm, key, keylen);
+	else
+		err = crypto_aead_alg(tfm)->setkey(tfm, key, keylen);
+
+	if (err)
+		return err;
 
-	return crypto_aead_alg(tfm)->setkey(tfm, key, keylen);
+	crypto_aead_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
+	return 0;
 }
 EXPORT_SYMBOL_GPL(crypto_aead_setkey);
 
@@ -93,6 +100,8 @@ static int crypto_aead_init_tfm(struct crypto_tfm *tfm)
 	struct crypto_aead *aead = __crypto_aead_cast(tfm);
 	struct aead_alg *alg = crypto_aead_alg(aead);
 
+	crypto_aead_set_flags(aead, CRYPTO_TFM_NEED_KEY);
+
 	aead->authsize = alg->maxauthsize;
 
 	if (alg->exit)
@@ -295,7 +304,7 @@ int aead_init_geniv(struct crypto_aead *aead)
 	if (err)
 		goto out;
 
-	ctx->sknull = crypto_get_default_null_skcipher2();
+	ctx->sknull = crypto_get_default_null_skcipher();
 	err = PTR_ERR(ctx->sknull);
 	if (IS_ERR(ctx->sknull))
 		goto out;
@@ -315,7 +324,7 @@ int aead_init_geniv(struct crypto_aead *aead)
 	return err;
 
 drop_null:
-	crypto_put_default_null_skcipher2();
+	crypto_put_default_null_skcipher();
 	goto out;
 }
 EXPORT_SYMBOL_GPL(aead_init_geniv);
@@ -325,7 +334,7 @@ void aead_exit_geniv(struct crypto_aead *tfm)
 	struct aead_geniv_ctx *ctx = crypto_aead_ctx(tfm);
 
 	crypto_free_aead(ctx->child);
-	crypto_put_default_null_skcipher2();
+	crypto_put_default_null_skcipher();
 }
 EXPORT_SYMBOL_GPL(aead_exit_geniv);
 
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 35d4dcea381f..c49766b03165 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -150,7 +150,7 @@ EXPORT_SYMBOL_GPL(af_alg_release_parent);
 
 static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
 {
-	const u32 forbidden = CRYPTO_ALG_INTERNAL;
+	const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY;
 	struct sock *sk = sock->sk;
 	struct alg_sock *ask = alg_sk(sk);
 	struct sockaddr_alg *sa = (void *)uaddr;
@@ -158,6 +158,10 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
 	void *private;
 	int err;
 
+	/* If caller uses non-allowed flag, return error. */
+	if ((sa->salg_feat & ~allowed) || (sa->salg_mask & ~allowed))
+		return -EINVAL;
+
 	if (sock->state == SS_CONNECTED)
 		return -EINVAL;
 
@@ -176,9 +180,7 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
 	if (IS_ERR(type))
 		return PTR_ERR(type);
 
-	private = type->bind(sa->salg_name,
-			     sa->salg_feat & ~forbidden,
-			     sa->salg_mask & ~forbidden);
+	private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask);
 	if (IS_ERR(private)) {
 		module_put(type->owner);
 		return PTR_ERR(private);
@@ -733,9 +735,9 @@ void af_alg_wmem_wakeup(struct sock *sk)
 	rcu_read_lock();
 	wq = rcu_dereference(sk->sk_wq);
 	if (skwq_has_sleeper(wq))
-		wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
-							   POLLRDNORM |
-							   POLLRDBAND);
+		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
+							   EPOLLRDNORM |
+							   EPOLLRDBAND);
 	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
 	rcu_read_unlock();
 }
@@ -798,9 +800,9 @@ void af_alg_data_wakeup(struct sock *sk)
 	rcu_read_lock();
 	wq = rcu_dereference(sk->sk_wq);
 	if (skwq_has_sleeper(wq))
-		wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
-							   POLLRDNORM |
-							   POLLRDBAND);
+		wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
+							   EPOLLRDNORM |
+							   EPOLLRDBAND);
 	sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
 	rcu_read_unlock();
 }
@@ -1062,22 +1064,22 @@ EXPORT_SYMBOL_GPL(af_alg_async_cb);
 /**
  * af_alg_poll - poll system call handler
  */
-unsigned int af_alg_poll(struct file *file, struct socket *sock,
+__poll_t af_alg_poll(struct file *file, struct socket *sock,
 			 poll_table *wait)
 {
 	struct sock *sk = sock->sk;
 	struct alg_sock *ask = alg_sk(sk);
 	struct af_alg_ctx *ctx = ask->private;
-	unsigned int mask;
+	__poll_t mask;
 
 	sock_poll_wait(file, sk_sleep(sk), wait);
 	mask = 0;
 
 	if (!ctx->more || ctx->used)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	if (af_alg_writable(sk))
-		mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
+		mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
 
 	return mask;
 }
diff --git a/crypto/ahash.c b/crypto/ahash.c
index 3a35d67de7d9..266fc1d64f61 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -193,11 +193,18 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
 			unsigned int keylen)
 {
 	unsigned long alignmask = crypto_ahash_alignmask(tfm);
+	int err;
 
 	if ((unsigned long)key & alignmask)
-		return ahash_setkey_unaligned(tfm, key, keylen);
+		err = ahash_setkey_unaligned(tfm, key, keylen);
+	else
+		err = tfm->setkey(tfm, key, keylen);
+
+	if (err)
+		return err;
 
-	return tfm->setkey(tfm, key, keylen);
+	crypto_ahash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
+	return 0;
 }
 EXPORT_SYMBOL_GPL(crypto_ahash_setkey);
 
@@ -368,7 +375,12 @@ EXPORT_SYMBOL_GPL(crypto_ahash_finup);
 
 int crypto_ahash_digest(struct ahash_request *req)
 {
-	return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->digest);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+
+	if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
+		return -ENOKEY;
+
+	return crypto_ahash_op(req, tfm->digest);
 }
 EXPORT_SYMBOL_GPL(crypto_ahash_digest);
 
@@ -450,7 +462,6 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm)
 	struct ahash_alg *alg = crypto_ahash_alg(hash);
 
 	hash->setkey = ahash_nosetkey;
-	hash->has_setkey = false;
 	hash->export = ahash_no_export;
 	hash->import = ahash_no_import;
 
@@ -465,7 +476,8 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm)
 
 	if (alg->setkey) {
 		hash->setkey = alg->setkey;
-		hash->has_setkey = true;
+		if (!(alg->halg.base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY))
+			crypto_ahash_set_flags(hash, CRYPTO_TFM_NEED_KEY);
 	}
 	if (alg->export)
 		hash->export = alg->export;
@@ -649,5 +661,16 @@ struct hash_alg_common *ahash_attr_alg(struct rtattr *rta, u32 type, u32 mask)
 }
 EXPORT_SYMBOL_GPL(ahash_attr_alg);
 
+bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg)
+{
+	struct crypto_alg *alg = &halg->base;
+
+	if (alg->cra_type != &crypto_ahash_type)
+		return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg));
+
+	return __crypto_ahash_alg(alg)->setkey != NULL;
+}
+EXPORT_SYMBOL_GPL(crypto_hash_alg_has_setkey);
+
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Asynchronous cryptographic hash type");
diff --git a/crypto/algapi.c b/crypto/algapi.c
index 9a636f961572..395b082d03a9 100644
--- a/crypto/algapi.c
+++ b/crypto/algapi.c
@@ -62,7 +62,7 @@ static int crypto_check_alg(struct crypto_alg *alg)
 	if (alg->cra_priority < 0)
 		return -EINVAL;
 
-	atomic_set(&alg->cra_refcnt, 1);
+	refcount_set(&alg->cra_refcnt, 1);
 
 	return crypto_set_driver_name(alg);
 }
@@ -123,7 +123,6 @@ static void crypto_remove_instance(struct crypto_instance *inst,
 	if (!tmpl || !crypto_tmpl_get(tmpl))
 		return;
 
-	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, &inst->alg);
 	list_move(&inst->alg.cra_list, list);
 	hlist_del(&inst->list);
 	inst->alg.cra_destroy = crypto_destroy_instance;
@@ -236,7 +235,7 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)
 	if (!larval->adult)
 		goto free_larval;
 
-	atomic_set(&larval->alg.cra_refcnt, 1);
+	refcount_set(&larval->alg.cra_refcnt, 1);
 	memcpy(larval->alg.cra_driver_name, alg->cra_driver_name,
 	       CRYPTO_MAX_ALG_NAME);
 	larval->alg.cra_priority = alg->cra_priority;
@@ -392,7 +391,6 @@ static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
 
 	alg->cra_flags |= CRYPTO_ALG_DEAD;
 
-	crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg);
 	list_del_init(&alg->cra_list);
 	crypto_remove_spawns(alg, list, NULL);
 
@@ -411,7 +409,7 @@ int crypto_unregister_alg(struct crypto_alg *alg)
 	if (ret)
 		return ret;
 
-	BUG_ON(atomic_read(&alg->cra_refcnt) != 1);
+	BUG_ON(refcount_read(&alg->cra_refcnt) != 1);
 	if (alg->cra_destroy)
 		alg->cra_destroy(alg);
 
@@ -470,7 +468,6 @@ int crypto_register_template(struct crypto_template *tmpl)
 	}
 
 	list_add(&tmpl->list, &crypto_template_list);
-	crypto_notify(CRYPTO_MSG_TMPL_REGISTER, tmpl);
 	err = 0;
 out:
 	up_write(&crypto_alg_sem);
@@ -497,12 +494,10 @@ void crypto_unregister_template(struct crypto_template *tmpl)
 		BUG_ON(err);
 	}
 
-	crypto_notify(CRYPTO_MSG_TMPL_UNREGISTER, tmpl);
-
 	up_write(&crypto_alg_sem);
 
 	hlist_for_each_entry_safe(inst, n, list, list) {
-		BUG_ON(atomic_read(&inst->alg.cra_refcnt) != 1);
+		BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
 		crypto_free_instance(inst);
 	}
 	crypto_remove_final(&users);
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index e9885a35ef6e..4b07edd5a9ff 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -42,7 +42,6 @@
 
 struct aead_tfm {
 	struct crypto_aead *aead;
-	bool has_key;
 	struct crypto_skcipher *null_tfm;
 };
 
@@ -398,7 +397,7 @@ static int aead_check_key(struct socket *sock)
 
 	err = -ENOKEY;
 	lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
-	if (!tfm->has_key)
+	if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
 		goto unlock;
 
 	if (!pask->refcnt++)
@@ -491,7 +490,7 @@ static void *aead_bind(const char *name, u32 type, u32 mask)
 		return ERR_CAST(aead);
 	}
 
-	null_tfm = crypto_get_default_null_skcipher2();
+	null_tfm = crypto_get_default_null_skcipher();
 	if (IS_ERR(null_tfm)) {
 		crypto_free_aead(aead);
 		kfree(tfm);
@@ -509,7 +508,7 @@ static void aead_release(void *private)
 	struct aead_tfm *tfm = private;
 
 	crypto_free_aead(tfm->aead);
-	crypto_put_default_null_skcipher2();
+	crypto_put_default_null_skcipher();
 	kfree(tfm);
 }
 
@@ -523,12 +522,8 @@ static int aead_setauthsize(void *private, unsigned int authsize)
 static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
 {
 	struct aead_tfm *tfm = private;
-	int err;
-
-	err = crypto_aead_setkey(tfm->aead, key, keylen);
-	tfm->has_key = !err;
 
-	return err;
+	return crypto_aead_setkey(tfm->aead, key, keylen);
 }
 
 static void aead_sock_destruct(struct sock *sk)
@@ -589,7 +584,7 @@ static int aead_accept_parent(void *private, struct sock *sk)
 {
 	struct aead_tfm *tfm = private;
 
-	if (!tfm->has_key)
+	if (crypto_aead_get_flags(tfm->aead) & CRYPTO_TFM_NEED_KEY)
 		return -ENOKEY;
 
 	return aead_accept_parent_nokey(private, sk);
diff --git a/crypto/algif_hash.c b/crypto/algif_hash.c
index 76d2e716c792..6c9b1927a520 100644
--- a/crypto/algif_hash.c
+++ b/crypto/algif_hash.c
@@ -34,11 +34,6 @@ struct hash_ctx {
 	struct ahash_request req;
 };
 
-struct algif_hash_tfm {
-	struct crypto_ahash *hash;
-	bool has_key;
-};
-
 static int hash_alloc_result(struct sock *sk, struct hash_ctx *ctx)
 {
 	unsigned ds;
@@ -307,7 +302,7 @@ static int hash_check_key(struct socket *sock)
 	int err = 0;
 	struct sock *psk;
 	struct alg_sock *pask;
-	struct algif_hash_tfm *tfm;
+	struct crypto_ahash *tfm;
 	struct sock *sk = sock->sk;
 	struct alg_sock *ask = alg_sk(sk);
 
@@ -321,7 +316,7 @@ static int hash_check_key(struct socket *sock)
 
 	err = -ENOKEY;
 	lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
-	if (!tfm->has_key)
+	if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
 		goto unlock;
 
 	if (!pask->refcnt++)
@@ -412,41 +407,17 @@ static struct proto_ops algif_hash_ops_nokey = {
 
 static void *hash_bind(const char *name, u32 type, u32 mask)
 {
-	struct algif_hash_tfm *tfm;
-	struct crypto_ahash *hash;
-
-	tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
-	if (!tfm)
-		return ERR_PTR(-ENOMEM);
-
-	hash = crypto_alloc_ahash(name, type, mask);
-	if (IS_ERR(hash)) {
-		kfree(tfm);
-		return ERR_CAST(hash);
-	}
-
-	tfm->hash = hash;
-
-	return tfm;
+	return crypto_alloc_ahash(name, type, mask);
 }
 
 static void hash_release(void *private)
 {
-	struct algif_hash_tfm *tfm = private;
-
-	crypto_free_ahash(tfm->hash);
-	kfree(tfm);
+	crypto_free_ahash(private);
 }
 
 static int hash_setkey(void *private, const u8 *key, unsigned int keylen)
 {
-	struct algif_hash_tfm *tfm = private;
-	int err;
-
-	err = crypto_ahash_setkey(tfm->hash, key, keylen);
-	tfm->has_key = !err;
-
-	return err;
+	return crypto_ahash_setkey(private, key, keylen);
 }
 
 static void hash_sock_destruct(struct sock *sk)
@@ -461,11 +432,10 @@ static void hash_sock_destruct(struct sock *sk)
 
 static int hash_accept_parent_nokey(void *private, struct sock *sk)
 {
-	struct hash_ctx *ctx;
+	struct crypto_ahash *tfm = private;
 	struct alg_sock *ask = alg_sk(sk);
-	struct algif_hash_tfm *tfm = private;
-	struct crypto_ahash *hash = tfm->hash;
-	unsigned len = sizeof(*ctx) + crypto_ahash_reqsize(hash);
+	struct hash_ctx *ctx;
+	unsigned int len = sizeof(*ctx) + crypto_ahash_reqsize(tfm);
 
 	ctx = sock_kmalloc(sk, len, GFP_KERNEL);
 	if (!ctx)
@@ -478,7 +448,7 @@ static int hash_accept_parent_nokey(void *private, struct sock *sk)
 
 	ask->private = ctx;
 
-	ahash_request_set_tfm(&ctx->req, hash);
+	ahash_request_set_tfm(&ctx->req, tfm);
 	ahash_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 				   crypto_req_done, &ctx->wait);
 
@@ -489,9 +459,9 @@ static int hash_accept_parent_nokey(void *private, struct sock *sk)
 
 static int hash_accept_parent(void *private, struct sock *sk)
 {
-	struct algif_hash_tfm *tfm = private;
+	struct crypto_ahash *tfm = private;
 
-	if (!tfm->has_key && crypto_ahash_has_setkey(tfm->hash))
+	if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
 		return -ENOKEY;
 
 	return hash_accept_parent_nokey(private, sk);
diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c
index c5c47b680152..c4e885df4564 100644
--- a/crypto/algif_skcipher.c
+++ b/crypto/algif_skcipher.c
@@ -38,11 +38,6 @@
 #include <linux/net.h>
 #include <net/sock.h>
 
-struct skcipher_tfm {
-	struct crypto_skcipher *skcipher;
-	bool has_key;
-};
-
 static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
 			    size_t size)
 {
@@ -50,8 +45,7 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
 	struct alg_sock *ask = alg_sk(sk);
 	struct sock *psk = ask->parent;
 	struct alg_sock *pask = alg_sk(psk);
-	struct skcipher_tfm *skc = pask->private;
-	struct crypto_skcipher *tfm = skc->skcipher;
+	struct crypto_skcipher *tfm = pask->private;
 	unsigned ivsize = crypto_skcipher_ivsize(tfm);
 
 	return af_alg_sendmsg(sock, msg, size, ivsize);
@@ -65,8 +59,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
 	struct sock *psk = ask->parent;
 	struct alg_sock *pask = alg_sk(psk);
 	struct af_alg_ctx *ctx = ask->private;
-	struct skcipher_tfm *skc = pask->private;
-	struct crypto_skcipher *tfm = skc->skcipher;
+	struct crypto_skcipher *tfm = pask->private;
 	unsigned int bs = crypto_skcipher_blocksize(tfm);
 	struct af_alg_async_req *areq;
 	int err = 0;
@@ -193,7 +186,6 @@ static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
 	return ret;
 }
 
-
 static struct proto_ops algif_skcipher_ops = {
 	.family		=	PF_ALG,
 
@@ -221,7 +213,7 @@ static int skcipher_check_key(struct socket *sock)
 	int err = 0;
 	struct sock *psk;
 	struct alg_sock *pask;
-	struct skcipher_tfm *tfm;
+	struct crypto_skcipher *tfm;
 	struct sock *sk = sock->sk;
 	struct alg_sock *ask = alg_sk(sk);
 
@@ -235,7 +227,7 @@ static int skcipher_check_key(struct socket *sock)
 
 	err = -ENOKEY;
 	lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
-	if (!tfm->has_key)
+	if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
 		goto unlock;
 
 	if (!pask->refcnt++)
@@ -314,41 +306,17 @@ static struct proto_ops algif_skcipher_ops_nokey = {
 
 static void *skcipher_bind(const char *name, u32 type, u32 mask)
 {
-	struct skcipher_tfm *tfm;
-	struct crypto_skcipher *skcipher;
-
-	tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
-	if (!tfm)
-		return ERR_PTR(-ENOMEM);
-
-	skcipher = crypto_alloc_skcipher(name, type, mask);
-	if (IS_ERR(skcipher)) {
-		kfree(tfm);
-		return ERR_CAST(skcipher);
-	}
-
-	tfm->skcipher = skcipher;
-
-	return tfm;
+	return crypto_alloc_skcipher(name, type, mask);
 }
 
 static void skcipher_release(void *private)
 {
-	struct skcipher_tfm *tfm = private;
-
-	crypto_free_skcipher(tfm->skcipher);
-	kfree(tfm);
+	crypto_free_skcipher(private);
 }
 
 static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen)
 {
-	struct skcipher_tfm *tfm = private;
-	int err;
-
-	err = crypto_skcipher_setkey(tfm->skcipher, key, keylen);
-	tfm->has_key = !err;
-
-	return err;
+	return crypto_skcipher_setkey(private, key, keylen);
 }
 
 static void skcipher_sock_destruct(struct sock *sk)
@@ -357,8 +325,7 @@ static void skcipher_sock_destruct(struct sock *sk)
 	struct af_alg_ctx *ctx = ask->private;
 	struct sock *psk = ask->parent;
 	struct alg_sock *pask = alg_sk(psk);
-	struct skcipher_tfm *skc = pask->private;
-	struct crypto_skcipher *tfm = skc->skcipher;
+	struct crypto_skcipher *tfm = pask->private;
 
 	af_alg_pull_tsgl(sk, ctx->used, NULL, 0);
 	sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm));
@@ -370,22 +337,21 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
 {
 	struct af_alg_ctx *ctx;
 	struct alg_sock *ask = alg_sk(sk);
-	struct skcipher_tfm *tfm = private;
-	struct crypto_skcipher *skcipher = tfm->skcipher;
+	struct crypto_skcipher *tfm = private;
 	unsigned int len = sizeof(*ctx);
 
 	ctx = sock_kmalloc(sk, len, GFP_KERNEL);
 	if (!ctx)
 		return -ENOMEM;
 
-	ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(skcipher),
+	ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(tfm),
 			       GFP_KERNEL);
 	if (!ctx->iv) {
 		sock_kfree_s(sk, ctx, len);
 		return -ENOMEM;
 	}
 
-	memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher));
+	memset(ctx->iv, 0, crypto_skcipher_ivsize(tfm));
 
 	INIT_LIST_HEAD(&ctx->tsgl_list);
 	ctx->len = len;
@@ -405,9 +371,9 @@ static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
 
 static int skcipher_accept_parent(void *private, struct sock *sk)
 {
-	struct skcipher_tfm *tfm = private;
+	struct crypto_skcipher *tfm = private;
 
-	if (!tfm->has_key && crypto_skcipher_has_setkey(tfm->skcipher))
+	if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
 		return -ENOKEY;
 
 	return skcipher_accept_parent_nokey(private, sk);
diff --git a/crypto/api.c b/crypto/api.c
index 2a2479d168aa..70a894e52ff3 100644
--- a/crypto/api.c
+++ b/crypto/api.c
@@ -137,7 +137,7 @@ static struct crypto_alg *crypto_larval_add(const char *name, u32 type,
 	if (IS_ERR(larval))
 		return ERR_CAST(larval);
 
-	atomic_set(&larval->alg.cra_refcnt, 2);
+	refcount_set(&larval->alg.cra_refcnt, 2);
 
 	down_write(&crypto_alg_sem);
 	alg = __crypto_alg_lookup(name, type, mask);
@@ -205,7 +205,8 @@ struct crypto_alg *crypto_alg_lookup(const char *name, u32 type, u32 mask)
 }
 EXPORT_SYMBOL_GPL(crypto_alg_lookup);
 
-struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask)
+static struct crypto_alg *crypto_larval_lookup(const char *name, u32 type,
+					       u32 mask)
 {
 	struct crypto_alg *alg;
 
@@ -231,7 +232,6 @@ struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask)
 
 	return crypto_larval_add(name, type, mask);
 }
-EXPORT_SYMBOL_GPL(crypto_larval_lookup);
 
 int crypto_probing_notify(unsigned long val, void *v)
 {
diff --git a/crypto/asymmetric_keys/pkcs7_trust.c b/crypto/asymmetric_keys/pkcs7_trust.c
index 1f4e25f10049..598906b1e28d 100644
--- a/crypto/asymmetric_keys/pkcs7_trust.c
+++ b/crypto/asymmetric_keys/pkcs7_trust.c
@@ -106,6 +106,7 @@ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7,
 		pr_devel("sinfo %u: Direct signer is key %x\n",
 			 sinfo->index, key_serial(key));
 		x509 = NULL;
+		sig = sinfo->sig;
 		goto matched;
 	}
 	if (PTR_ERR(key) != -ENOKEY)
diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c
index 39e6de0c2761..97c77f66b20d 100644
--- a/crypto/asymmetric_keys/pkcs7_verify.c
+++ b/crypto/asymmetric_keys/pkcs7_verify.c
@@ -270,7 +270,7 @@ static int pkcs7_verify_sig_chain(struct pkcs7_message *pkcs7,
 				sinfo->index);
 			return 0;
 		}
-		ret = public_key_verify_signature(p->pub, p->sig);
+		ret = public_key_verify_signature(p->pub, x509->sig);
 		if (ret < 0)
 			return ret;
 		x509->signer = p;
@@ -366,8 +366,7 @@ static int pkcs7_verify_one(struct pkcs7_message *pkcs7,
  *
  *  (*) -EBADMSG if some part of the message was invalid, or:
  *
- *  (*) 0 if no signature chains were found to be blacklisted or to contain
- *	unsupported crypto, or:
+ *  (*) 0 if a signature chain passed verification, or:
  *
  *  (*) -EKEYREJECTED if a blacklisted key was encountered, or:
  *
@@ -423,8 +422,11 @@ int pkcs7_verify(struct pkcs7_message *pkcs7,
 
 	for (sinfo = pkcs7->signed_infos; sinfo; sinfo = sinfo->next) {
 		ret = pkcs7_verify_one(pkcs7, sinfo);
-		if (sinfo->blacklisted && actual_ret == -ENOPKG)
-			actual_ret = -EKEYREJECTED;
+		if (sinfo->blacklisted) {
+			if (actual_ret == -ENOPKG)
+				actual_ret = -EKEYREJECTED;
+			continue;
+		}
 		if (ret < 0) {
 			if (ret == -ENOPKG) {
 				sinfo->unsupported_crypto = true;
diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c
index de996586762a..e929fe1e4106 100644
--- a/crypto/asymmetric_keys/public_key.c
+++ b/crypto/asymmetric_keys/public_key.c
@@ -79,9 +79,11 @@ int public_key_verify_signature(const struct public_key *pkey,
 
 	BUG_ON(!pkey);
 	BUG_ON(!sig);
-	BUG_ON(!sig->digest);
 	BUG_ON(!sig->s);
 
+	if (!sig->digest)
+		return -ENOPKG;
+
 	alg_name = sig->pkey_algo;
 	if (strcmp(sig->pkey_algo, "rsa") == 0) {
 		/* The data wangled by the RSA algorithm is typically padded
diff --git a/crypto/asymmetric_keys/restrict.c b/crypto/asymmetric_keys/restrict.c
index 86fb68508952..7c93c7728454 100644
--- a/crypto/asymmetric_keys/restrict.c
+++ b/crypto/asymmetric_keys/restrict.c
@@ -67,8 +67,9 @@ __setup("ca_keys=", ca_keys_setup);
  *
  * Returns 0 if the new certificate was accepted, -ENOKEY if we couldn't find a
  * matching parent certificate in the trusted list, -EKEYREJECTED if the
- * signature check fails or the key is blacklisted and some other error if
- * there is a matching certificate but the signature check cannot be performed.
+ * signature check fails or the key is blacklisted, -ENOPKG if the signature
+ * uses unsupported crypto, or some other error if there is a matching
+ * certificate but the signature check cannot be performed.
  */
 int restrict_link_by_signature(struct key *dest_keyring,
 			       const struct key_type *type,
@@ -88,6 +89,8 @@ int restrict_link_by_signature(struct key *dest_keyring,
 		return -EOPNOTSUPP;
 
 	sig = payload->data[asym_auth];
+	if (!sig)
+		return -ENOPKG;
 	if (!sig->auth_ids[0] && !sig->auth_ids[1])
 		return -ENOKEY;
 
@@ -139,6 +142,8 @@ static int key_or_keyring_common(struct key *dest_keyring,
 		return -EOPNOTSUPP;
 
 	sig = payload->data[asym_auth];
+	if (!sig)
+		return -ENOPKG;
 	if (!sig->auth_ids[0] && !sig->auth_ids[1])
 		return -ENOKEY;
 
@@ -222,9 +227,9 @@ static int key_or_keyring_common(struct key *dest_keyring,
  *
  * Returns 0 if the new certificate was accepted, -ENOKEY if we
  * couldn't find a matching parent certificate in the trusted list,
- * -EKEYREJECTED if the signature check fails, and some other error if
- * there is a matching certificate but the signature check cannot be
- * performed.
+ * -EKEYREJECTED if the signature check fails, -ENOPKG if the signature uses
+ * unsupported crypto, or some other error if there is a matching certificate
+ * but the signature check cannot be performed.
  */
 int restrict_link_by_key_or_keyring(struct key *dest_keyring,
 				    const struct key_type *type,
@@ -249,9 +254,9 @@ int restrict_link_by_key_or_keyring(struct key *dest_keyring,
  *
  * Returns 0 if the new certificate was accepted, -ENOKEY if we
  * couldn't find a matching parent certificate in the trusted list,
- * -EKEYREJECTED if the signature check fails, and some other error if
- * there is a matching certificate but the signature check cannot be
- * performed.
+ * -EKEYREJECTED if the signature check fails, -ENOPKG if the signature uses
+ * unsupported crypto, or some other error if there is a matching certificate
+ * but the signature check cannot be performed.
  */
 int restrict_link_by_key_or_keyring_chain(struct key *dest_keyring,
 					  const struct key_type *type,
diff --git a/crypto/authenc.c b/crypto/authenc.c
index 875470b0e026..d3d6d72fe649 100644
--- a/crypto/authenc.c
+++ b/crypto/authenc.c
@@ -329,7 +329,7 @@ static int crypto_authenc_init_tfm(struct crypto_aead *tfm)
 	if (IS_ERR(enc))
 		goto err_free_ahash;
 
-	null = crypto_get_default_null_skcipher2();
+	null = crypto_get_default_null_skcipher();
 	err = PTR_ERR(null);
 	if (IS_ERR(null))
 		goto err_free_skcipher;
@@ -363,7 +363,7 @@ static void crypto_authenc_exit_tfm(struct crypto_aead *tfm)
 
 	crypto_free_ahash(ctx->auth);
 	crypto_free_skcipher(ctx->enc);
-	crypto_put_default_null_skcipher2();
+	crypto_put_default_null_skcipher();
 }
 
 static void crypto_authenc_free(struct aead_instance *inst)
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index 0cf5fefdb859..15f91ddd7f0e 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -352,7 +352,7 @@ static int crypto_authenc_esn_init_tfm(struct crypto_aead *tfm)
 	if (IS_ERR(enc))
 		goto err_free_ahash;
 
-	null = crypto_get_default_null_skcipher2();
+	null = crypto_get_default_null_skcipher();
 	err = PTR_ERR(null);
 	if (IS_ERR(null))
 		goto err_free_skcipher;
@@ -389,7 +389,7 @@ static void crypto_authenc_esn_exit_tfm(struct crypto_aead *tfm)
 
 	crypto_free_ahash(ctx->auth);
 	crypto_free_skcipher(ctx->enc);
-	crypto_put_default_null_skcipher2();
+	crypto_put_default_null_skcipher();
 }
 
 static void crypto_authenc_esn_free(struct aead_instance *inst)
diff --git a/crypto/blkcipher.c b/crypto/blkcipher.c
index 6c43a0a17a55..01c0d4aa2563 100644
--- a/crypto/blkcipher.c
+++ b/crypto/blkcipher.c
@@ -18,7 +18,6 @@
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <linux/errno.h>
-#include <linux/hardirq.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/seq_file.h>
diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c
index a02286bf319e..32ddd4836ff5 100644
--- a/crypto/camellia_generic.c
+++ b/crypto/camellia_generic.c
@@ -13,8 +13,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 /*
diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c
index df5c72629383..66169c178314 100644
--- a/crypto/cast5_generic.c
+++ b/crypto/cast5_generic.c
@@ -16,8 +16,7 @@
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
-* along with this program; if not, write to the Free Software
-* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 
diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c
index 058c8d755d03..c8e5ec69790e 100644
--- a/crypto/cast6_generic.c
+++ b/crypto/cast6_generic.c
@@ -13,8 +13,7 @@
  * any later version.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 
diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c
index 4a45fa4890c0..e451c3cb6a56 100644
--- a/crypto/chacha20_generic.c
+++ b/crypto/chacha20_generic.c
@@ -9,44 +9,38 @@
  * (at your option) any later version.
  */
 
+#include <asm/unaligned.h>
 #include <crypto/algapi.h>
 #include <crypto/chacha20.h>
 #include <crypto/internal/skcipher.h>
 #include <linux/module.h>
 
-static inline u32 le32_to_cpuvp(const void *p)
-{
-	return le32_to_cpup(p);
-}
-
 static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src,
 			     unsigned int bytes)
 {
-	u8 stream[CHACHA20_BLOCK_SIZE];
+	u32 stream[CHACHA20_BLOCK_WORDS];
 
 	if (dst != src)
 		memcpy(dst, src, bytes);
 
 	while (bytes >= CHACHA20_BLOCK_SIZE) {
 		chacha20_block(state, stream);
-		crypto_xor(dst, stream, CHACHA20_BLOCK_SIZE);
+		crypto_xor(dst, (const u8 *)stream, CHACHA20_BLOCK_SIZE);
 		bytes -= CHACHA20_BLOCK_SIZE;
 		dst += CHACHA20_BLOCK_SIZE;
 	}
 	if (bytes) {
 		chacha20_block(state, stream);
-		crypto_xor(dst, stream, bytes);
+		crypto_xor(dst, (const u8 *)stream, bytes);
 	}
 }
 
 void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv)
 {
-	static const char constant[16] = "expand 32-byte k";
-
-	state[0]  = le32_to_cpuvp(constant +  0);
-	state[1]  = le32_to_cpuvp(constant +  4);
-	state[2]  = le32_to_cpuvp(constant +  8);
-	state[3]  = le32_to_cpuvp(constant + 12);
+	state[0]  = 0x61707865; /* "expa" */
+	state[1]  = 0x3320646e; /* "nd 3" */
+	state[2]  = 0x79622d32; /* "2-by" */
+	state[3]  = 0x6b206574; /* "te k" */
 	state[4]  = ctx->key[0];
 	state[5]  = ctx->key[1];
 	state[6]  = ctx->key[2];
@@ -55,10 +49,10 @@ void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv)
 	state[9]  = ctx->key[5];
 	state[10] = ctx->key[6];
 	state[11] = ctx->key[7];
-	state[12] = le32_to_cpuvp(iv +  0);
-	state[13] = le32_to_cpuvp(iv +  4);
-	state[14] = le32_to_cpuvp(iv +  8);
-	state[15] = le32_to_cpuvp(iv + 12);
+	state[12] = get_unaligned_le32(iv +  0);
+	state[13] = get_unaligned_le32(iv +  4);
+	state[14] = get_unaligned_le32(iv +  8);
+	state[15] = get_unaligned_le32(iv + 12);
 }
 EXPORT_SYMBOL_GPL(crypto_chacha20_init);
 
@@ -72,7 +66,7 @@ int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key,
 		return -EINVAL;
 
 	for (i = 0; i < ARRAY_SIZE(ctx->key); i++)
-		ctx->key[i] = le32_to_cpuvp(key + i * sizeof(u32));
+		ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32));
 
 	return 0;
 }
@@ -111,7 +105,6 @@ static struct skcipher_alg alg = {
 	.base.cra_priority	= 100,
 	.base.cra_blocksize	= 1,
 	.base.cra_ctxsize	= sizeof(struct chacha20_ctx),
-	.base.cra_alignmask	= sizeof(u32) - 1,
 	.base.cra_module	= THIS_MODULE,
 
 	.min_keysize		= CHACHA20_KEY_SIZE,
diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c
index aa2a25fc7482..718cbce8d169 100644
--- a/crypto/crc32_generic.c
+++ b/crypto/crc32_generic.c
@@ -133,6 +133,7 @@ static struct shash_alg alg = {
 		.cra_name		= "crc32",
 		.cra_driver_name	= "crc32-generic",
 		.cra_priority		= 100,
+		.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 		.cra_blocksize		= CHKSUM_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(u32),
 		.cra_module		= THIS_MODULE,
diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c
index 4c0a0e271876..372320399622 100644
--- a/crypto/crc32c_generic.c
+++ b/crypto/crc32c_generic.c
@@ -146,6 +146,7 @@ static struct shash_alg alg = {
 		.cra_name		=	"crc32c",
 		.cra_driver_name	=	"crc32c-generic",
 		.cra_priority		=	100,
+		.cra_flags		=	CRYPTO_ALG_OPTIONAL_KEY,
 		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
 		.cra_alignmask		=	3,
 		.cra_ctxsize		=	sizeof(struct chksum_ctx),
diff --git a/crypto/cryptd.c b/crypto/cryptd.c
index bd43cf5be14c..addca7bae33f 100644
--- a/crypto/cryptd.c
+++ b/crypto/cryptd.c
@@ -32,7 +32,9 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 
-#define CRYPTD_MAX_CPU_QLEN 1000
+static unsigned int cryptd_max_cpu_qlen = 1000;
+module_param(cryptd_max_cpu_qlen, uint, 0);
+MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth");
 
 struct cryptd_cpu_queue {
 	struct crypto_queue queue;
@@ -116,6 +118,7 @@ static int cryptd_init_queue(struct cryptd_queue *queue,
 		crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
 		INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
 	}
+	pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen);
 	return 0;
 }
 
@@ -893,10 +896,9 @@ static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
 	if (err)
 		goto out_free_inst;
 
-	type = CRYPTO_ALG_ASYNC;
-	if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
-		type |= CRYPTO_ALG_INTERNAL;
-	inst->alg.halg.base.cra_flags = type;
+	inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC |
+		(alg->cra_flags & (CRYPTO_ALG_INTERNAL |
+				   CRYPTO_ALG_OPTIONAL_KEY));
 
 	inst->alg.halg.digestsize = salg->digestsize;
 	inst->alg.halg.statesize = salg->statesize;
@@ -911,7 +913,8 @@ static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
 	inst->alg.finup  = cryptd_hash_finup_enqueue;
 	inst->alg.export = cryptd_hash_export;
 	inst->alg.import = cryptd_hash_import;
-	inst->alg.setkey = cryptd_hash_setkey;
+	if (crypto_shash_alg_has_setkey(salg))
+		inst->alg.setkey = cryptd_hash_setkey;
 	inst->alg.digest = cryptd_hash_digest_enqueue;
 
 	err = ahash_register_instance(tmpl, inst);
@@ -1372,7 +1375,7 @@ static int __init cryptd_init(void)
 {
 	int err;
 
-	err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
+	err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen);
 	if (err)
 		return err;
 
diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c
index 0dbe2be7f783..5c291eedaa70 100644
--- a/crypto/crypto_user.c
+++ b/crypto/crypto_user.c
@@ -169,7 +169,7 @@ static int crypto_report_one(struct crypto_alg *alg,
 	ualg->cru_type = 0;
 	ualg->cru_mask = 0;
 	ualg->cru_flags = alg->cra_flags;
-	ualg->cru_refcnt = atomic_read(&alg->cra_refcnt);
+	ualg->cru_refcnt = refcount_read(&alg->cra_refcnt);
 
 	if (nla_put_u32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority))
 		goto nla_put_failure;
@@ -387,7 +387,7 @@ static int crypto_del_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
 		goto drop_alg;
 
 	err = -EBUSY;
-	if (atomic_read(&alg->cra_refcnt) > 2)
+	if (refcount_read(&alg->cra_refcnt) > 2)
 		goto drop_alg;
 
 	err = crypto_unregister_instance((struct crypto_instance *)alg);
diff --git a/crypto/ecc.c b/crypto/ecc.c
index 633a9bcdc574..18f32f2a5e1c 100644
--- a/crypto/ecc.c
+++ b/crypto/ecc.c
@@ -964,7 +964,7 @@ int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, u64 *privkey)
 	 * DRBG with a security strength of 256.
 	 */
 	if (crypto_get_default_rng())
-		err = -EFAULT;
+		return -EFAULT;
 
 	err = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes);
 	crypto_put_default_rng();
diff --git a/crypto/echainiv.c b/crypto/echainiv.c
index e3d889b122e0..45819e6015bf 100644
--- a/crypto/echainiv.c
+++ b/crypto/echainiv.c
@@ -118,8 +118,6 @@ static int echainiv_aead_create(struct crypto_template *tmpl,
 				struct rtattr **tb)
 {
 	struct aead_instance *inst;
-	struct crypto_aead_spawn *spawn;
-	struct aead_alg *alg;
 	int err;
 
 	inst = aead_geniv_alloc(tmpl, tb, 0, 0);
@@ -127,9 +125,6 @@ static int echainiv_aead_create(struct crypto_template *tmpl,
 	if (IS_ERR(inst))
 		return PTR_ERR(inst);
 
-	spawn = aead_instance_ctx(inst);
-	alg = crypto_spawn_aead_alg(spawn);
-
 	err = -EINVAL;
 	if (inst->alg.ivsize & (sizeof(u64) - 1) || !inst->alg.ivsize)
 		goto free_inst;
diff --git a/crypto/gcm.c b/crypto/gcm.c
index 8589681fb9f6..0ad879e1f9b2 100644
--- a/crypto/gcm.c
+++ b/crypto/gcm.c
@@ -1101,7 +1101,7 @@ static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
 	if (IS_ERR(aead))
 		return PTR_ERR(aead);
 
-	null = crypto_get_default_null_skcipher2();
+	null = crypto_get_default_null_skcipher();
 	err = PTR_ERR(null);
 	if (IS_ERR(null))
 		goto err_free_aead;
@@ -1129,7 +1129,7 @@ static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
 	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
 
 	crypto_free_aead(ctx->child);
-	crypto_put_default_null_skcipher2();
+	crypto_put_default_null_skcipher();
 }
 
 static void crypto_rfc4543_free(struct aead_instance *inst)
diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
index 24e601954c7a..a4b1c026aaee 100644
--- a/crypto/gf128mul.c
+++ b/crypto/gf128mul.c
@@ -160,8 +160,6 @@ void gf128mul_x8_ble(le128 *r, const le128 *x)
 {
 	u64 a = le64_to_cpu(x->a);
 	u64 b = le64_to_cpu(x->b);
-
-	/* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
 	u64 _tt = gf128mul_table_be[a >> 56];
 
 	r->a = cpu_to_le64((a << 8) | (b >> 56));
diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c
index 12ad3e3a84e3..1bffb3f712dd 100644
--- a/crypto/ghash-generic.c
+++ b/crypto/ghash-generic.c
@@ -56,9 +56,6 @@ static int ghash_update(struct shash_desc *desc,
 	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
 	u8 *dst = dctx->buffer;
 
-	if (!ctx->gf128)
-		return -ENOKEY;
-
 	if (dctx->bytes) {
 		int n = min(srclen, dctx->bytes);
 		u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);
@@ -111,9 +108,6 @@ static int ghash_final(struct shash_desc *desc, u8 *dst)
 	struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
 	u8 *buf = dctx->buffer;
 
-	if (!ctx->gf128)
-		return -ENOKEY;
-
 	ghash_flush(ctx, dctx);
 	memcpy(dst, buf, GHASH_BLOCK_SIZE);
 
diff --git a/crypto/internal.h b/crypto/internal.h
index f07320423191..5ac27fba10e8 100644
--- a/crypto/internal.h
+++ b/crypto/internal.h
@@ -30,9 +30,6 @@
 enum {
 	CRYPTO_MSG_ALG_REQUEST,
 	CRYPTO_MSG_ALG_REGISTER,
-	CRYPTO_MSG_ALG_UNREGISTER,
-	CRYPTO_MSG_TMPL_REGISTER,
-	CRYPTO_MSG_TMPL_UNREGISTER,
 };
 
 struct crypto_instance;
@@ -78,7 +75,6 @@ int crypto_init_compress_ops(struct crypto_tfm *tfm);
 
 struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask);
 void crypto_larval_kill(struct crypto_alg *alg);
-struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask);
 void crypto_alg_tested(const char *name, int err);
 
 void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
@@ -106,13 +102,13 @@ int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
 
 static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg)
 {
-	atomic_inc(&alg->cra_refcnt);
+	refcount_inc(&alg->cra_refcnt);
 	return alg;
 }
 
 static inline void crypto_alg_put(struct crypto_alg *alg)
 {
-	if (atomic_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy)
+	if (refcount_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy)
 		alg->cra_destroy(alg);
 }
 
diff --git a/crypto/keywrap.c b/crypto/keywrap.c
index 744e35134c45..ec5c6a087c90 100644
--- a/crypto/keywrap.c
+++ b/crypto/keywrap.c
@@ -188,7 +188,7 @@ static int crypto_kw_decrypt(struct blkcipher_desc *desc,
 	}
 
 	/* Perform authentication check */
-	if (block.A != cpu_to_be64(0xa6a6a6a6a6a6a6a6))
+	if (block.A != cpu_to_be64(0xa6a6a6a6a6a6a6a6ULL))
 		ret = -EBADMSG;
 
 	memzero_explicit(&block, sizeof(struct crypto_kw_block));
@@ -221,7 +221,7 @@ static int crypto_kw_encrypt(struct blkcipher_desc *desc,
 	 * Place the predefined IV into block A -- for encrypt, the caller
 	 * does not need to provide an IV, but he needs to fetch the final IV.
 	 */
-	block.A = cpu_to_be64(0xa6a6a6a6a6a6a6a6);
+	block.A = cpu_to_be64(0xa6a6a6a6a6a6a6a6ULL);
 
 	/*
 	 * src scatterlist is read-only. dst scatterlist is r/w. During the
diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c
index eca04d3729b3..fe5129d6ff4e 100644
--- a/crypto/mcryptd.c
+++ b/crypto/mcryptd.c
@@ -26,7 +26,6 @@
 #include <linux/sched.h>
 #include <linux/sched/stat.h>
 #include <linux/slab.h>
-#include <linux/hardirq.h>
 
 #define MCRYPTD_MAX_CPU_QLEN 100
 #define MCRYPTD_BATCH 9
@@ -517,10 +516,9 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
 	if (err)
 		goto out_free_inst;
 
-	type = CRYPTO_ALG_ASYNC;
-	if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
-		type |= CRYPTO_ALG_INTERNAL;
-	inst->alg.halg.base.cra_flags = type;
+	inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC |
+		(alg->cra_flags & (CRYPTO_ALG_INTERNAL |
+				   CRYPTO_ALG_OPTIONAL_KEY));
 
 	inst->alg.halg.digestsize = halg->digestsize;
 	inst->alg.halg.statesize = halg->statesize;
@@ -535,7 +533,8 @@ static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
 	inst->alg.finup  = mcryptd_hash_finup_enqueue;
 	inst->alg.export = mcryptd_hash_export;
 	inst->alg.import = mcryptd_hash_import;
-	inst->alg.setkey = mcryptd_hash_setkey;
+	if (crypto_hash_alg_has_setkey(halg))
+		inst->alg.setkey = mcryptd_hash_setkey;
 	inst->alg.digest = mcryptd_hash_digest_enqueue;
 
 	err = ahash_register_instance(tmpl, inst);
diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c
index b1c2d57dc734..b7a3a0613a30 100644
--- a/crypto/poly1305_generic.c
+++ b/crypto/poly1305_generic.c
@@ -47,17 +47,6 @@ int crypto_poly1305_init(struct shash_desc *desc)
 }
 EXPORT_SYMBOL_GPL(crypto_poly1305_init);
 
-int crypto_poly1305_setkey(struct crypto_shash *tfm,
-			   const u8 *key, unsigned int keylen)
-{
-	/* Poly1305 requires a unique key for each tag, which implies that
-	 * we can't set it on the tfm that gets accessed by multiple users
-	 * simultaneously. Instead we expect the key as the first 32 bytes in
-	 * the update() call. */
-	return -ENOTSUPP;
-}
-EXPORT_SYMBOL_GPL(crypto_poly1305_setkey);
-
 static void poly1305_setrkey(struct poly1305_desc_ctx *dctx, const u8 *key)
 {
 	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
@@ -76,6 +65,11 @@ static void poly1305_setskey(struct poly1305_desc_ctx *dctx, const u8 *key)
 	dctx->s[3] = get_unaligned_le32(key + 12);
 }
 
+/*
+ * Poly1305 requires a unique key for each tag, which implies that we can't set
+ * it on the tfm that gets accessed by multiple users simultaneously. Instead we
+ * expect the key as the first 32 bytes in the update() call.
+ */
 unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx,
 					const u8 *src, unsigned int srclen)
 {
@@ -210,7 +204,6 @@ EXPORT_SYMBOL_GPL(crypto_poly1305_update);
 int crypto_poly1305_final(struct shash_desc *desc, u8 *dst)
 {
 	struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
-	__le32 *mac = (__le32 *)dst;
 	u32 h0, h1, h2, h3, h4;
 	u32 g0, g1, g2, g3, g4;
 	u32 mask;
@@ -267,10 +260,10 @@ int crypto_poly1305_final(struct shash_desc *desc, u8 *dst)
 	h3 = (h3 >> 18) | (h4 <<  8);
 
 	/* mac = (h + s) % (2^128) */
-	f = (f >> 32) + h0 + dctx->s[0]; mac[0] = cpu_to_le32(f);
-	f = (f >> 32) + h1 + dctx->s[1]; mac[1] = cpu_to_le32(f);
-	f = (f >> 32) + h2 + dctx->s[2]; mac[2] = cpu_to_le32(f);
-	f = (f >> 32) + h3 + dctx->s[3]; mac[3] = cpu_to_le32(f);
+	f = (f >> 32) + h0 + dctx->s[0]; put_unaligned_le32(f, dst +  0);
+	f = (f >> 32) + h1 + dctx->s[1]; put_unaligned_le32(f, dst +  4);
+	f = (f >> 32) + h2 + dctx->s[2]; put_unaligned_le32(f, dst +  8);
+	f = (f >> 32) + h3 + dctx->s[3]; put_unaligned_le32(f, dst + 12);
 
 	return 0;
 }
@@ -281,14 +274,12 @@ static struct shash_alg poly1305_alg = {
 	.init		= crypto_poly1305_init,
 	.update		= crypto_poly1305_update,
 	.final		= crypto_poly1305_final,
-	.setkey		= crypto_poly1305_setkey,
 	.descsize	= sizeof(struct poly1305_desc_ctx),
 	.base		= {
 		.cra_name		= "poly1305",
 		.cra_driver_name	= "poly1305-generic",
 		.cra_priority		= 100,
 		.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
-		.cra_alignmask		= sizeof(u32) - 1,
 		.cra_blocksize		= POLY1305_BLOCK_SIZE,
 		.cra_module		= THIS_MODULE,
 	},
diff --git a/crypto/proc.c b/crypto/proc.c
index 2cc10c96d753..822fcef6d91c 100644
--- a/crypto/proc.c
+++ b/crypto/proc.c
@@ -46,7 +46,7 @@ static int c_show(struct seq_file *m, void *p)
 	seq_printf(m, "driver       : %s\n", alg->cra_driver_name);
 	seq_printf(m, "module       : %s\n", module_name(alg->cra_module));
 	seq_printf(m, "priority     : %d\n", alg->cra_priority);
-	seq_printf(m, "refcnt       : %d\n", atomic_read(&alg->cra_refcnt));
+	seq_printf(m, "refcnt       : %u\n", refcount_read(&alg->cra_refcnt));
 	seq_printf(m, "selftest     : %s\n",
 		   (alg->cra_flags & CRYPTO_ALG_TESTED) ?
 		   "passed" : "unknown");
diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c
index d7da0eea5622..5074006a56c3 100644
--- a/crypto/salsa20_generic.c
+++ b/crypto/salsa20_generic.c
@@ -19,49 +19,19 @@
  *
  */
 
-#include <linux/init.h>
+#include <asm/unaligned.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/salsa20.h>
 #include <linux/module.h>
-#include <linux/errno.h>
-#include <linux/crypto.h>
-#include <linux/types.h>
-#include <linux/bitops.h>
-#include <crypto/algapi.h>
-#include <asm/byteorder.h>
 
-#define SALSA20_IV_SIZE        8U
-#define SALSA20_MIN_KEY_SIZE  16U
-#define SALSA20_MAX_KEY_SIZE  32U
-
-/*
- * Start of code taken from D. J. Bernstein's reference implementation.
- * With some modifications and optimizations made to suit our needs.
- */
-
-/*
-salsa20-ref.c version 20051118
-D. J. Bernstein
-Public domain.
-*/
-
-#define U32TO8_LITTLE(p, v) \
-	{ (p)[0] = (v >>  0) & 0xff; (p)[1] = (v >>  8) & 0xff; \
-	  (p)[2] = (v >> 16) & 0xff; (p)[3] = (v >> 24) & 0xff; }
-#define U8TO32_LITTLE(p)   \
-	(((u32)((p)[0])      ) | ((u32)((p)[1]) <<  8) | \
-	 ((u32)((p)[2]) << 16) | ((u32)((p)[3]) << 24)   )
-
-struct salsa20_ctx
-{
-	u32 input[16];
-};
-
-static void salsa20_wordtobyte(u8 output[64], const u32 input[16])
+static void salsa20_block(u32 *state, __le32 *stream)
 {
 	u32 x[16];
 	int i;
 
-	memcpy(x, input, sizeof(x));
-	for (i = 20; i > 0; i -= 2) {
+	memcpy(x, state, sizeof(x));
+
+	for (i = 0; i < 20; i += 2) {
 		x[ 4] ^= rol32((x[ 0] + x[12]),  7);
 		x[ 8] ^= rol32((x[ 4] + x[ 0]),  9);
 		x[12] ^= rol32((x[ 8] + x[ 4]), 13);
@@ -95,145 +65,137 @@ static void salsa20_wordtobyte(u8 output[64], const u32 input[16])
 		x[14] ^= rol32((x[13] + x[12]), 13);
 		x[15] ^= rol32((x[14] + x[13]), 18);
 	}
-	for (i = 0; i < 16; ++i)
-		x[i] += input[i];
-	for (i = 0; i < 16; ++i)
-		U32TO8_LITTLE(output + 4 * i,x[i]);
-}
 
-static const char sigma[16] = "expand 32-byte k";
-static const char tau[16] = "expand 16-byte k";
+	for (i = 0; i < 16; i++)
+		stream[i] = cpu_to_le32(x[i] + state[i]);
+
+	if (++state[8] == 0)
+		state[9]++;
+}
 
-static void salsa20_keysetup(struct salsa20_ctx *ctx, const u8 *k, u32 kbytes)
+static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src,
+			    unsigned int bytes)
 {
-	const char *constants;
+	__le32 stream[SALSA20_BLOCK_SIZE / sizeof(__le32)];
 
-	ctx->input[1] = U8TO32_LITTLE(k + 0);
-	ctx->input[2] = U8TO32_LITTLE(k + 4);
-	ctx->input[3] = U8TO32_LITTLE(k + 8);
-	ctx->input[4] = U8TO32_LITTLE(k + 12);
-	if (kbytes == 32) { /* recommended */
-		k += 16;
-		constants = sigma;
-	} else { /* kbytes == 16 */
-		constants = tau;
+	if (dst != src)
+		memcpy(dst, src, bytes);
+
+	while (bytes >= SALSA20_BLOCK_SIZE) {
+		salsa20_block(state, stream);
+		crypto_xor(dst, (const u8 *)stream, SALSA20_BLOCK_SIZE);
+		bytes -= SALSA20_BLOCK_SIZE;
+		dst += SALSA20_BLOCK_SIZE;
+	}
+	if (bytes) {
+		salsa20_block(state, stream);
+		crypto_xor(dst, (const u8 *)stream, bytes);
 	}
-	ctx->input[11] = U8TO32_LITTLE(k + 0);
-	ctx->input[12] = U8TO32_LITTLE(k + 4);
-	ctx->input[13] = U8TO32_LITTLE(k + 8);
-	ctx->input[14] = U8TO32_LITTLE(k + 12);
-	ctx->input[0] = U8TO32_LITTLE(constants + 0);
-	ctx->input[5] = U8TO32_LITTLE(constants + 4);
-	ctx->input[10] = U8TO32_LITTLE(constants + 8);
-	ctx->input[15] = U8TO32_LITTLE(constants + 12);
 }
 
-static void salsa20_ivsetup(struct salsa20_ctx *ctx, const u8 *iv)
+void crypto_salsa20_init(u32 *state, const struct salsa20_ctx *ctx,
+			 const u8 *iv)
 {
-	ctx->input[6] = U8TO32_LITTLE(iv + 0);
-	ctx->input[7] = U8TO32_LITTLE(iv + 4);
-	ctx->input[8] = 0;
-	ctx->input[9] = 0;
+	memcpy(state, ctx->initial_state, sizeof(ctx->initial_state));
+	state[6] = get_unaligned_le32(iv + 0);
+	state[7] = get_unaligned_le32(iv + 4);
 }
+EXPORT_SYMBOL_GPL(crypto_salsa20_init);
 
-static void salsa20_encrypt_bytes(struct salsa20_ctx *ctx, u8 *dst,
-				  const u8 *src, unsigned int bytes)
+int crypto_salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key,
+			  unsigned int keysize)
 {
-	u8 buf[64];
-
-	if (dst != src)
-		memcpy(dst, src, bytes);
-
-	while (bytes) {
-		salsa20_wordtobyte(buf, ctx->input);
-
-		ctx->input[8]++;
-		if (!ctx->input[8])
-			ctx->input[9]++;
+	static const char sigma[16] = "expand 32-byte k";
+	static const char tau[16] = "expand 16-byte k";
+	struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm);
+	const char *constants;
 
-		if (bytes <= 64) {
-			crypto_xor(dst, buf, bytes);
-			return;
-		}
+	if (keysize != SALSA20_MIN_KEY_SIZE &&
+	    keysize != SALSA20_MAX_KEY_SIZE)
+		return -EINVAL;
 
-		crypto_xor(dst, buf, 64);
-		bytes -= 64;
-		dst += 64;
+	ctx->initial_state[1] = get_unaligned_le32(key + 0);
+	ctx->initial_state[2] = get_unaligned_le32(key + 4);
+	ctx->initial_state[3] = get_unaligned_le32(key + 8);
+	ctx->initial_state[4] = get_unaligned_le32(key + 12);
+	if (keysize == 32) { /* recommended */
+		key += 16;
+		constants = sigma;
+	} else { /* keysize == 16 */
+		constants = tau;
 	}
-}
-
-/*
- * End of code taken from D. J. Bernstein's reference implementation.
- */
+	ctx->initial_state[11] = get_unaligned_le32(key + 0);
+	ctx->initial_state[12] = get_unaligned_le32(key + 4);
+	ctx->initial_state[13] = get_unaligned_le32(key + 8);
+	ctx->initial_state[14] = get_unaligned_le32(key + 12);
+	ctx->initial_state[0]  = get_unaligned_le32(constants + 0);
+	ctx->initial_state[5]  = get_unaligned_le32(constants + 4);
+	ctx->initial_state[10] = get_unaligned_le32(constants + 8);
+	ctx->initial_state[15] = get_unaligned_le32(constants + 12);
+
+	/* space for the nonce; it will be overridden for each request */
+	ctx->initial_state[6] = 0;
+	ctx->initial_state[7] = 0;
+
+	/* initial block number */
+	ctx->initial_state[8] = 0;
+	ctx->initial_state[9] = 0;
 
-static int setkey(struct crypto_tfm *tfm, const u8 *key,
-		  unsigned int keysize)
-{
-	struct salsa20_ctx *ctx = crypto_tfm_ctx(tfm);
-	salsa20_keysetup(ctx, key, keysize);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(crypto_salsa20_setkey);
 
-static int encrypt(struct blkcipher_desc *desc,
-		   struct scatterlist *dst, struct scatterlist *src,
-		   unsigned int nbytes)
+static int salsa20_crypt(struct skcipher_request *req)
 {
-	struct blkcipher_walk walk;
-	struct crypto_blkcipher *tfm = desc->tfm;
-	struct salsa20_ctx *ctx = crypto_blkcipher_ctx(tfm);
+	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+	const struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm);
+	struct skcipher_walk walk;
+	u32 state[16];
 	int err;
 
-	blkcipher_walk_init(&walk, dst, src, nbytes);
-	err = blkcipher_walk_virt_block(desc, &walk, 64);
+	err = skcipher_walk_virt(&walk, req, true);
 
-	salsa20_ivsetup(ctx, walk.iv);
+	crypto_salsa20_init(state, ctx, walk.iv);
 
-	while (walk.nbytes >= 64) {
-		salsa20_encrypt_bytes(ctx, walk.dst.virt.addr,
-				      walk.src.virt.addr,
-				      walk.nbytes - (walk.nbytes % 64));
-		err = blkcipher_walk_done(desc, &walk, walk.nbytes % 64);
-	}
+	while (walk.nbytes > 0) {
+		unsigned int nbytes = walk.nbytes;
 
-	if (walk.nbytes) {
-		salsa20_encrypt_bytes(ctx, walk.dst.virt.addr,
-				      walk.src.virt.addr, walk.nbytes);
-		err = blkcipher_walk_done(desc, &walk, 0);
+		if (nbytes < walk.total)
+			nbytes = round_down(nbytes, walk.stride);
+
+		salsa20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr,
+				nbytes);
+		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
 	}
 
 	return err;
 }
 
-static struct crypto_alg alg = {
-	.cra_name           =   "salsa20",
-	.cra_driver_name    =   "salsa20-generic",
-	.cra_priority       =   100,
-	.cra_flags          =   CRYPTO_ALG_TYPE_BLKCIPHER,
-	.cra_type           =   &crypto_blkcipher_type,
-	.cra_blocksize      =   1,
-	.cra_ctxsize        =   sizeof(struct salsa20_ctx),
-	.cra_alignmask      =	3,
-	.cra_module         =   THIS_MODULE,
-	.cra_u              =   {
-		.blkcipher = {
-			.setkey         =   setkey,
-			.encrypt        =   encrypt,
-			.decrypt        =   encrypt,
-			.min_keysize    =   SALSA20_MIN_KEY_SIZE,
-			.max_keysize    =   SALSA20_MAX_KEY_SIZE,
-			.ivsize         =   SALSA20_IV_SIZE,
-		}
-	}
+static struct skcipher_alg alg = {
+	.base.cra_name		= "salsa20",
+	.base.cra_driver_name	= "salsa20-generic",
+	.base.cra_priority	= 100,
+	.base.cra_blocksize	= 1,
+	.base.cra_ctxsize	= sizeof(struct salsa20_ctx),
+	.base.cra_module	= THIS_MODULE,
+
+	.min_keysize		= SALSA20_MIN_KEY_SIZE,
+	.max_keysize		= SALSA20_MAX_KEY_SIZE,
+	.ivsize			= SALSA20_IV_SIZE,
+	.chunksize		= SALSA20_BLOCK_SIZE,
+	.setkey			= crypto_salsa20_setkey,
+	.encrypt		= salsa20_crypt,
+	.decrypt		= salsa20_crypt,
 };
 
 static int __init salsa20_generic_mod_init(void)
 {
-	return crypto_register_alg(&alg);
+	return crypto_register_skcipher(&alg);
 }
 
 static void __exit salsa20_generic_mod_fini(void)
 {
-	crypto_unregister_alg(&alg);
+	crypto_unregister_skcipher(&alg);
 }
 
 module_init(salsa20_generic_mod_init);
diff --git a/crypto/scompress.c b/crypto/scompress.c
index 2075e2c4e7df..968bbcf65c94 100644
--- a/crypto/scompress.c
+++ b/crypto/scompress.c
@@ -140,53 +140,6 @@ static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
 	return ret;
 }
 
-static void crypto_scomp_sg_free(struct scatterlist *sgl)
-{
-	int i, n;
-	struct page *page;
-
-	if (!sgl)
-		return;
-
-	n = sg_nents(sgl);
-	for_each_sg(sgl, sgl, n, i) {
-		page = sg_page(sgl);
-		if (page)
-			__free_page(page);
-	}
-
-	kfree(sgl);
-}
-
-static struct scatterlist *crypto_scomp_sg_alloc(size_t size, gfp_t gfp)
-{
-	struct scatterlist *sgl;
-	struct page *page;
-	int i, n;
-
-	n = ((size - 1) >> PAGE_SHIFT) + 1;
-
-	sgl = kmalloc_array(n, sizeof(struct scatterlist), gfp);
-	if (!sgl)
-		return NULL;
-
-	sg_init_table(sgl, n);
-
-	for (i = 0; i < n; i++) {
-		page = alloc_page(gfp);
-		if (!page)
-			goto err;
-		sg_set_page(sgl + i, page, PAGE_SIZE, 0);
-	}
-
-	return sgl;
-
-err:
-	sg_mark_end(sgl + i);
-	crypto_scomp_sg_free(sgl);
-	return NULL;
-}
-
 static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
 {
 	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
@@ -220,7 +173,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
 					      scratch_dst, &req->dlen, *ctx);
 	if (!ret) {
 		if (!req->dst) {
-			req->dst = crypto_scomp_sg_alloc(req->dlen, GFP_ATOMIC);
+			req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL);
 			if (!req->dst)
 				goto out;
 		}
@@ -274,7 +227,7 @@ int crypto_init_scomp_ops_async(struct crypto_tfm *tfm)
 
 	crt->compress = scomp_acomp_compress;
 	crt->decompress = scomp_acomp_decompress;
-	crt->dst_free = crypto_scomp_sg_free;
+	crt->dst_free = sgl_free;
 	crt->reqsize = sizeof(void *);
 
 	return 0;
diff --git a/crypto/seqiv.c b/crypto/seqiv.c
index 570b7d1aa0ca..39dbf2f7e5f5 100644
--- a/crypto/seqiv.c
+++ b/crypto/seqiv.c
@@ -144,8 +144,6 @@ static int seqiv_aead_decrypt(struct aead_request *req)
 static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb)
 {
 	struct aead_instance *inst;
-	struct crypto_aead_spawn *spawn;
-	struct aead_alg *alg;
 	int err;
 
 	inst = aead_geniv_alloc(tmpl, tb, 0, 0);
@@ -153,9 +151,6 @@ static int seqiv_aead_create(struct crypto_template *tmpl, struct rtattr **tb)
 	if (IS_ERR(inst))
 		return PTR_ERR(inst);
 
-	spawn = aead_instance_ctx(inst);
-	alg = crypto_spawn_aead_alg(spawn);
-
 	err = -EINVAL;
 	if (inst->alg.ivsize != sizeof(u64))
 		goto free_inst;
diff --git a/crypto/sha3_generic.c b/crypto/sha3_generic.c
index 7e8ed96236ce..ded148783303 100644
--- a/crypto/sha3_generic.c
+++ b/crypto/sha3_generic.c
@@ -5,6 +5,7 @@
  * http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf
  *
  * SHA-3 code by Jeff Garzik <jeff@garzik.org>
+ *               Ard Biesheuvel <ard.biesheuvel@linaro.org>
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
@@ -17,11 +18,23 @@
 #include <linux/module.h>
 #include <linux/types.h>
 #include <crypto/sha3.h>
-#include <asm/byteorder.h>
+#include <asm/unaligned.h>
 
-#define KECCAK_ROUNDS 24
+/*
+ * On some 32-bit architectures (mn10300 and h8300), GCC ends up using
+ * over 1 KB of stack if we inline the round calculation into the loop
+ * in keccakf(). On the other hand, on 64-bit architectures with plenty
+ * of [64-bit wide] general purpose registers, not inlining it severely
+ * hurts performance. So let's use 64-bitness as a heuristic to decide
+ * whether to inline or not.
+ */
+#ifdef CONFIG_64BIT
+#define SHA3_INLINE	inline
+#else
+#define SHA3_INLINE	noinline
+#endif
 
-#define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y))))
+#define KECCAK_ROUNDS 24
 
 static const u64 keccakf_rndc[24] = {
 	0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL,
@@ -34,100 +47,137 @@ static const u64 keccakf_rndc[24] = {
 	0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL
 };
 
-static const int keccakf_rotc[24] = {
-	1,  3,  6,  10, 15, 21, 28, 36, 45, 55, 2,  14,
-	27, 41, 56, 8,  25, 43, 62, 18, 39, 61, 20, 44
-};
-
-static const int keccakf_piln[24] = {
-	10, 7,  11, 17, 18, 3, 5,  16, 8,  21, 24, 4,
-	15, 23, 19, 13, 12, 2, 20, 14, 22, 9,  6,  1
-};
-
 /* update the state with given number of rounds */
 
-static void keccakf(u64 st[25])
+static SHA3_INLINE void keccakf_round(u64 st[25])
 {
-	int i, j, round;
-	u64 t, bc[5];
-
-	for (round = 0; round < KECCAK_ROUNDS; round++) {
-
-		/* Theta */
-		for (i = 0; i < 5; i++)
-			bc[i] = st[i] ^ st[i + 5] ^ st[i + 10] ^ st[i + 15]
-				^ st[i + 20];
-
-		for (i = 0; i < 5; i++) {
-			t = bc[(i + 4) % 5] ^ ROTL64(bc[(i + 1) % 5], 1);
-			for (j = 0; j < 25; j += 5)
-				st[j + i] ^= t;
-		}
-
-		/* Rho Pi */
-		t = st[1];
-		for (i = 0; i < 24; i++) {
-			j = keccakf_piln[i];
-			bc[0] = st[j];
-			st[j] = ROTL64(t, keccakf_rotc[i]);
-			t = bc[0];
-		}
+	u64 t[5], tt, bc[5];
+
+	/* Theta */
+	bc[0] = st[0] ^ st[5] ^ st[10] ^ st[15] ^ st[20];
+	bc[1] = st[1] ^ st[6] ^ st[11] ^ st[16] ^ st[21];
+	bc[2] = st[2] ^ st[7] ^ st[12] ^ st[17] ^ st[22];
+	bc[3] = st[3] ^ st[8] ^ st[13] ^ st[18] ^ st[23];
+	bc[4] = st[4] ^ st[9] ^ st[14] ^ st[19] ^ st[24];
+
+	t[0] = bc[4] ^ rol64(bc[1], 1);
+	t[1] = bc[0] ^ rol64(bc[2], 1);
+	t[2] = bc[1] ^ rol64(bc[3], 1);
+	t[3] = bc[2] ^ rol64(bc[4], 1);
+	t[4] = bc[3] ^ rol64(bc[0], 1);
+
+	st[0] ^= t[0];
+
+	/* Rho Pi */
+	tt = st[1];
+	st[ 1] = rol64(st[ 6] ^ t[1], 44);
+	st[ 6] = rol64(st[ 9] ^ t[4], 20);
+	st[ 9] = rol64(st[22] ^ t[2], 61);
+	st[22] = rol64(st[14] ^ t[4], 39);
+	st[14] = rol64(st[20] ^ t[0], 18);
+	st[20] = rol64(st[ 2] ^ t[2], 62);
+	st[ 2] = rol64(st[12] ^ t[2], 43);
+	st[12] = rol64(st[13] ^ t[3], 25);
+	st[13] = rol64(st[19] ^ t[4],  8);
+	st[19] = rol64(st[23] ^ t[3], 56);
+	st[23] = rol64(st[15] ^ t[0], 41);
+	st[15] = rol64(st[ 4] ^ t[4], 27);
+	st[ 4] = rol64(st[24] ^ t[4], 14);
+	st[24] = rol64(st[21] ^ t[1],  2);
+	st[21] = rol64(st[ 8] ^ t[3], 55);
+	st[ 8] = rol64(st[16] ^ t[1], 45);
+	st[16] = rol64(st[ 5] ^ t[0], 36);
+	st[ 5] = rol64(st[ 3] ^ t[3], 28);
+	st[ 3] = rol64(st[18] ^ t[3], 21);
+	st[18] = rol64(st[17] ^ t[2], 15);
+	st[17] = rol64(st[11] ^ t[1], 10);
+	st[11] = rol64(st[ 7] ^ t[2],  6);
+	st[ 7] = rol64(st[10] ^ t[0],  3);
+	st[10] = rol64(    tt ^ t[1],  1);
+
+	/* Chi */
+	bc[ 0] = ~st[ 1] & st[ 2];
+	bc[ 1] = ~st[ 2] & st[ 3];
+	bc[ 2] = ~st[ 3] & st[ 4];
+	bc[ 3] = ~st[ 4] & st[ 0];
+	bc[ 4] = ~st[ 0] & st[ 1];
+	st[ 0] ^= bc[ 0];
+	st[ 1] ^= bc[ 1];
+	st[ 2] ^= bc[ 2];
+	st[ 3] ^= bc[ 3];
+	st[ 4] ^= bc[ 4];
+
+	bc[ 0] = ~st[ 6] & st[ 7];
+	bc[ 1] = ~st[ 7] & st[ 8];
+	bc[ 2] = ~st[ 8] & st[ 9];
+	bc[ 3] = ~st[ 9] & st[ 5];
+	bc[ 4] = ~st[ 5] & st[ 6];
+	st[ 5] ^= bc[ 0];
+	st[ 6] ^= bc[ 1];
+	st[ 7] ^= bc[ 2];
+	st[ 8] ^= bc[ 3];
+	st[ 9] ^= bc[ 4];
+
+	bc[ 0] = ~st[11] & st[12];
+	bc[ 1] = ~st[12] & st[13];
+	bc[ 2] = ~st[13] & st[14];
+	bc[ 3] = ~st[14] & st[10];
+	bc[ 4] = ~st[10] & st[11];
+	st[10] ^= bc[ 0];
+	st[11] ^= bc[ 1];
+	st[12] ^= bc[ 2];
+	st[13] ^= bc[ 3];
+	st[14] ^= bc[ 4];
+
+	bc[ 0] = ~st[16] & st[17];
+	bc[ 1] = ~st[17] & st[18];
+	bc[ 2] = ~st[18] & st[19];
+	bc[ 3] = ~st[19] & st[15];
+	bc[ 4] = ~st[15] & st[16];
+	st[15] ^= bc[ 0];
+	st[16] ^= bc[ 1];
+	st[17] ^= bc[ 2];
+	st[18] ^= bc[ 3];
+	st[19] ^= bc[ 4];
+
+	bc[ 0] = ~st[21] & st[22];
+	bc[ 1] = ~st[22] & st[23];
+	bc[ 2] = ~st[23] & st[24];
+	bc[ 3] = ~st[24] & st[20];
+	bc[ 4] = ~st[20] & st[21];
+	st[20] ^= bc[ 0];
+	st[21] ^= bc[ 1];
+	st[22] ^= bc[ 2];
+	st[23] ^= bc[ 3];
+	st[24] ^= bc[ 4];
+}
 
-		/* Chi */
-		for (j = 0; j < 25; j += 5) {
-			for (i = 0; i < 5; i++)
-				bc[i] = st[j + i];
-			for (i = 0; i < 5; i++)
-				st[j + i] ^= (~bc[(i + 1) % 5]) &
-					     bc[(i + 2) % 5];
-		}
+static void __optimize("O3") keccakf(u64 st[25])
+{
+	int round;
 
+	for (round = 0; round < KECCAK_ROUNDS; round++) {
+		keccakf_round(st);
 		/* Iota */
 		st[0] ^= keccakf_rndc[round];
 	}
 }
 
-static void sha3_init(struct sha3_state *sctx, unsigned int digest_sz)
-{
-	memset(sctx, 0, sizeof(*sctx));
-	sctx->md_len = digest_sz;
-	sctx->rsiz = 200 - 2 * digest_sz;
-	sctx->rsizw = sctx->rsiz / 8;
-}
-
-static int sha3_224_init(struct shash_desc *desc)
-{
-	struct sha3_state *sctx = shash_desc_ctx(desc);
-
-	sha3_init(sctx, SHA3_224_DIGEST_SIZE);
-	return 0;
-}
-
-static int sha3_256_init(struct shash_desc *desc)
-{
-	struct sha3_state *sctx = shash_desc_ctx(desc);
-
-	sha3_init(sctx, SHA3_256_DIGEST_SIZE);
-	return 0;
-}
-
-static int sha3_384_init(struct shash_desc *desc)
+int crypto_sha3_init(struct shash_desc *desc)
 {
 	struct sha3_state *sctx = shash_desc_ctx(desc);
+	unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
 
-	sha3_init(sctx, SHA3_384_DIGEST_SIZE);
-	return 0;
-}
-
-static int sha3_512_init(struct shash_desc *desc)
-{
-	struct sha3_state *sctx = shash_desc_ctx(desc);
+	sctx->rsiz = 200 - 2 * digest_size;
+	sctx->rsizw = sctx->rsiz / 8;
+	sctx->partial = 0;
 
-	sha3_init(sctx, SHA3_512_DIGEST_SIZE);
+	memset(sctx->st, 0, sizeof(sctx->st));
 	return 0;
 }
+EXPORT_SYMBOL(crypto_sha3_init);
 
-static int sha3_update(struct shash_desc *desc, const u8 *data,
+int crypto_sha3_update(struct shash_desc *desc, const u8 *data,
 		       unsigned int len)
 {
 	struct sha3_state *sctx = shash_desc_ctx(desc);
@@ -149,7 +199,7 @@ static int sha3_update(struct shash_desc *desc, const u8 *data,
 			unsigned int i;
 
 			for (i = 0; i < sctx->rsizw; i++)
-				sctx->st[i] ^= ((u64 *) src)[i];
+				sctx->st[i] ^= get_unaligned_le64(src + 8 * i);
 			keccakf(sctx->st);
 
 			done += sctx->rsiz;
@@ -163,125 +213,89 @@ static int sha3_update(struct shash_desc *desc, const u8 *data,
 
 	return 0;
 }
+EXPORT_SYMBOL(crypto_sha3_update);
 
-static int sha3_final(struct shash_desc *desc, u8 *out)
+int crypto_sha3_final(struct shash_desc *desc, u8 *out)
 {
 	struct sha3_state *sctx = shash_desc_ctx(desc);
 	unsigned int i, inlen = sctx->partial;
+	unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
+	__le64 *digest = (__le64 *)out;
 
 	sctx->buf[inlen++] = 0x06;
 	memset(sctx->buf + inlen, 0, sctx->rsiz - inlen);
 	sctx->buf[sctx->rsiz - 1] |= 0x80;
 
 	for (i = 0; i < sctx->rsizw; i++)
-		sctx->st[i] ^= ((u64 *) sctx->buf)[i];
+		sctx->st[i] ^= get_unaligned_le64(sctx->buf + 8 * i);
 
 	keccakf(sctx->st);
 
-	for (i = 0; i < sctx->rsizw; i++)
-		sctx->st[i] = cpu_to_le64(sctx->st[i]);
+	for (i = 0; i < digest_size / 8; i++)
+		put_unaligned_le64(sctx->st[i], digest++);
 
-	memcpy(out, sctx->st, sctx->md_len);
+	if (digest_size & 4)
+		put_unaligned_le32(sctx->st[i], (__le32 *)digest);
 
 	memset(sctx, 0, sizeof(*sctx));
 	return 0;
 }
-
-static struct shash_alg sha3_224 = {
-	.digestsize	=	SHA3_224_DIGEST_SIZE,
-	.init		=	sha3_224_init,
-	.update		=	sha3_update,
-	.final		=	sha3_final,
-	.descsize	=	sizeof(struct sha3_state),
-	.base		=	{
-		.cra_name	=	"sha3-224",
-		.cra_driver_name =	"sha3-224-generic",
-		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
-		.cra_blocksize	=	SHA3_224_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static struct shash_alg sha3_256 = {
-	.digestsize	=	SHA3_256_DIGEST_SIZE,
-	.init		=	sha3_256_init,
-	.update		=	sha3_update,
-	.final		=	sha3_final,
-	.descsize	=	sizeof(struct sha3_state),
-	.base		=	{
-		.cra_name	=	"sha3-256",
-		.cra_driver_name =	"sha3-256-generic",
-		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
-		.cra_blocksize	=	SHA3_256_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static struct shash_alg sha3_384 = {
-	.digestsize	=	SHA3_384_DIGEST_SIZE,
-	.init		=	sha3_384_init,
-	.update		=	sha3_update,
-	.final		=	sha3_final,
-	.descsize	=	sizeof(struct sha3_state),
-	.base		=	{
-		.cra_name	=	"sha3-384",
-		.cra_driver_name =	"sha3-384-generic",
-		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
-		.cra_blocksize	=	SHA3_384_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
-
-static struct shash_alg sha3_512 = {
-	.digestsize	=	SHA3_512_DIGEST_SIZE,
-	.init		=	sha3_512_init,
-	.update		=	sha3_update,
-	.final		=	sha3_final,
-	.descsize	=	sizeof(struct sha3_state),
-	.base		=	{
-		.cra_name	=	"sha3-512",
-		.cra_driver_name =	"sha3-512-generic",
-		.cra_flags	=	CRYPTO_ALG_TYPE_SHASH,
-		.cra_blocksize	=	SHA3_512_BLOCK_SIZE,
-		.cra_module	=	THIS_MODULE,
-	}
-};
+EXPORT_SYMBOL(crypto_sha3_final);
+
+static struct shash_alg algs[] = { {
+	.digestsize		= SHA3_224_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= crypto_sha3_update,
+	.final			= crypto_sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-224",
+	.base.cra_driver_name	= "sha3-224-generic",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_224_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.digestsize		= SHA3_256_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= crypto_sha3_update,
+	.final			= crypto_sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-256",
+	.base.cra_driver_name	= "sha3-256-generic",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_256_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.digestsize		= SHA3_384_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= crypto_sha3_update,
+	.final			= crypto_sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-384",
+	.base.cra_driver_name	= "sha3-384-generic",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_384_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+}, {
+	.digestsize		= SHA3_512_DIGEST_SIZE,
+	.init			= crypto_sha3_init,
+	.update			= crypto_sha3_update,
+	.final			= crypto_sha3_final,
+	.descsize		= sizeof(struct sha3_state),
+	.base.cra_name		= "sha3-512",
+	.base.cra_driver_name	= "sha3-512-generic",
+	.base.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
+	.base.cra_blocksize	= SHA3_512_BLOCK_SIZE,
+	.base.cra_module	= THIS_MODULE,
+} };
 
 static int __init sha3_generic_mod_init(void)
 {
-	int ret;
-
-	ret = crypto_register_shash(&sha3_224);
-	if (ret < 0)
-		goto err_out;
-	ret = crypto_register_shash(&sha3_256);
-	if (ret < 0)
-		goto err_out_224;
-	ret = crypto_register_shash(&sha3_384);
-	if (ret < 0)
-		goto err_out_256;
-	ret = crypto_register_shash(&sha3_512);
-	if (ret < 0)
-		goto err_out_384;
-
-	return 0;
-
-err_out_384:
-	crypto_unregister_shash(&sha3_384);
-err_out_256:
-	crypto_unregister_shash(&sha3_256);
-err_out_224:
-	crypto_unregister_shash(&sha3_224);
-err_out:
-	return ret;
+	return crypto_register_shashes(algs, ARRAY_SIZE(algs));
 }
 
 static void __exit sha3_generic_mod_fini(void)
 {
-	crypto_unregister_shash(&sha3_224);
-	crypto_unregister_shash(&sha3_256);
-	crypto_unregister_shash(&sha3_384);
-	crypto_unregister_shash(&sha3_512);
+	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 }
 
 module_init(sha3_generic_mod_init);
diff --git a/crypto/shash.c b/crypto/shash.c
index e849d3ee2e27..5d732c6bb4b2 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -58,11 +58,18 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key,
 {
 	struct shash_alg *shash = crypto_shash_alg(tfm);
 	unsigned long alignmask = crypto_shash_alignmask(tfm);
+	int err;
 
 	if ((unsigned long)key & alignmask)
-		return shash_setkey_unaligned(tfm, key, keylen);
+		err = shash_setkey_unaligned(tfm, key, keylen);
+	else
+		err = shash->setkey(tfm, key, keylen);
+
+	if (err)
+		return err;
 
-	return shash->setkey(tfm, key, keylen);
+	crypto_shash_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
+	return 0;
 }
 EXPORT_SYMBOL_GPL(crypto_shash_setkey);
 
@@ -181,6 +188,9 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data,
 	struct shash_alg *shash = crypto_shash_alg(tfm);
 	unsigned long alignmask = crypto_shash_alignmask(tfm);
 
+	if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
+		return -ENOKEY;
+
 	if (((unsigned long)data | (unsigned long)out) & alignmask)
 		return shash_digest_unaligned(desc, data, len, out);
 
@@ -360,7 +370,8 @@ int crypto_init_shash_ops_async(struct crypto_tfm *tfm)
 	crt->digest = shash_async_digest;
 	crt->setkey = shash_async_setkey;
 
-	crt->has_setkey = alg->setkey != shash_no_setkey;
+	crypto_ahash_set_flags(crt, crypto_shash_get_flags(shash) &
+				    CRYPTO_TFM_NEED_KEY);
 
 	if (alg->export)
 		crt->export = shash_async_export;
@@ -375,8 +386,14 @@ int crypto_init_shash_ops_async(struct crypto_tfm *tfm)
 static int crypto_shash_init_tfm(struct crypto_tfm *tfm)
 {
 	struct crypto_shash *hash = __crypto_shash_cast(tfm);
+	struct shash_alg *alg = crypto_shash_alg(hash);
+
+	hash->descsize = alg->descsize;
+
+	if (crypto_shash_alg_has_setkey(alg) &&
+	    !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY))
+		crypto_shash_set_flags(hash, CRYPTO_TFM_NEED_KEY);
 
-	hash->descsize = crypto_shash_alg(hash)->descsize;
 	return 0;
 }
 
diff --git a/crypto/simd.c b/crypto/simd.c
index 88203370a62f..208226d7f908 100644
--- a/crypto/simd.c
+++ b/crypto/simd.c
@@ -19,9 +19,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
- * USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  */
 
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index 11af5fd6a443..0fe2a2923ad0 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -598,8 +598,11 @@ static int skcipher_setkey_blkcipher(struct crypto_skcipher *tfm,
 	err = crypto_blkcipher_setkey(blkcipher, key, keylen);
 	crypto_skcipher_set_flags(tfm, crypto_blkcipher_get_flags(blkcipher) &
 				       CRYPTO_TFM_RES_MASK);
+	if (err)
+		return err;
 
-	return err;
+	crypto_skcipher_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
+	return 0;
 }
 
 static int skcipher_crypt_blkcipher(struct skcipher_request *req,
@@ -674,6 +677,9 @@ static int crypto_init_skcipher_ops_blkcipher(struct crypto_tfm *tfm)
 	skcipher->ivsize = crypto_blkcipher_ivsize(blkcipher);
 	skcipher->keysize = calg->cra_blkcipher.max_keysize;
 
+	if (skcipher->keysize)
+		crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_NEED_KEY);
+
 	return 0;
 }
 
@@ -692,8 +698,11 @@ static int skcipher_setkey_ablkcipher(struct crypto_skcipher *tfm,
 	crypto_skcipher_set_flags(tfm,
 				  crypto_ablkcipher_get_flags(ablkcipher) &
 				  CRYPTO_TFM_RES_MASK);
+	if (err)
+		return err;
 
-	return err;
+	crypto_skcipher_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
+	return 0;
 }
 
 static int skcipher_crypt_ablkcipher(struct skcipher_request *req,
@@ -767,6 +776,9 @@ static int crypto_init_skcipher_ops_ablkcipher(struct crypto_tfm *tfm)
 			    sizeof(struct ablkcipher_request);
 	skcipher->keysize = calg->cra_ablkcipher.max_keysize;
 
+	if (skcipher->keysize)
+		crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_NEED_KEY);
+
 	return 0;
 }
 
@@ -796,6 +808,7 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 {
 	struct skcipher_alg *cipher = crypto_skcipher_alg(tfm);
 	unsigned long alignmask = crypto_skcipher_alignmask(tfm);
+	int err;
 
 	if (keylen < cipher->min_keysize || keylen > cipher->max_keysize) {
 		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
@@ -803,9 +816,15 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 	}
 
 	if ((unsigned long)key & alignmask)
-		return skcipher_setkey_unaligned(tfm, key, keylen);
+		err = skcipher_setkey_unaligned(tfm, key, keylen);
+	else
+		err = cipher->setkey(tfm, key, keylen);
+
+	if (err)
+		return err;
 
-	return cipher->setkey(tfm, key, keylen);
+	crypto_skcipher_clear_flags(tfm, CRYPTO_TFM_NEED_KEY);
+	return 0;
 }
 
 static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm)
@@ -834,6 +853,9 @@ static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm)
 	skcipher->ivsize = alg->ivsize;
 	skcipher->keysize = alg->max_keysize;
 
+	if (skcipher->keysize)
+		crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_NEED_KEY);
+
 	if (alg->exit)
 		skcipher->base.exit = crypto_skcipher_exit_tfm;
 
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 9267cbdb14d2..14213a096fd2 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -67,6 +67,7 @@ static char *alg = NULL;
 static u32 type;
 static u32 mask;
 static int mode;
+static u32 num_mb = 8;
 static char *tvmem[TVMEMSIZE];
 
 static char *check[] = {
@@ -79,6 +80,66 @@ static char *check[] = {
 	NULL
 };
 
+static u32 block_sizes[] = { 16, 64, 256, 1024, 8192, 0 };
+static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 };
+
+#define XBUFSIZE 8
+#define MAX_IVLEN 32
+
+static int testmgr_alloc_buf(char *buf[XBUFSIZE])
+{
+	int i;
+
+	for (i = 0; i < XBUFSIZE; i++) {
+		buf[i] = (void *)__get_free_page(GFP_KERNEL);
+		if (!buf[i])
+			goto err_free_buf;
+	}
+
+	return 0;
+
+err_free_buf:
+	while (i-- > 0)
+		free_page((unsigned long)buf[i]);
+
+	return -ENOMEM;
+}
+
+static void testmgr_free_buf(char *buf[XBUFSIZE])
+{
+	int i;
+
+	for (i = 0; i < XBUFSIZE; i++)
+		free_page((unsigned long)buf[i]);
+}
+
+static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
+			 unsigned int buflen, const void *assoc,
+			 unsigned int aad_size)
+{
+	int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
+	int k, rem;
+
+	if (np > XBUFSIZE) {
+		rem = PAGE_SIZE;
+		np = XBUFSIZE;
+	} else {
+		rem = buflen % PAGE_SIZE;
+	}
+
+	sg_init_table(sg, np + 1);
+
+	sg_set_buf(&sg[0], assoc, aad_size);
+
+	if (rem)
+		np--;
+	for (k = 0; k < np; k++)
+		sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
+
+	if (rem)
+		sg_set_buf(&sg[k + 1], xbuf[k], rem);
+}
+
 static inline int do_one_aead_op(struct aead_request *req, int ret)
 {
 	struct crypto_wait *wait = req->base.data;
@@ -86,8 +147,44 @@ static inline int do_one_aead_op(struct aead_request *req, int ret)
 	return crypto_wait_req(ret, wait);
 }
 
-static int test_aead_jiffies(struct aead_request *req, int enc,
-				int blen, int secs)
+struct test_mb_aead_data {
+	struct scatterlist sg[XBUFSIZE];
+	struct scatterlist sgout[XBUFSIZE];
+	struct aead_request *req;
+	struct crypto_wait wait;
+	char *xbuf[XBUFSIZE];
+	char *xoutbuf[XBUFSIZE];
+	char *axbuf[XBUFSIZE];
+};
+
+static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
+				u32 num_mb)
+{
+	int i, rc[num_mb], err = 0;
+
+	/* Fire up a bunch of concurrent requests */
+	for (i = 0; i < num_mb; i++) {
+		if (enc == ENCRYPT)
+			rc[i] = crypto_aead_encrypt(data[i].req);
+		else
+			rc[i] = crypto_aead_decrypt(data[i].req);
+	}
+
+	/* Wait for all requests to finish */
+	for (i = 0; i < num_mb; i++) {
+		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
+
+		if (rc[i]) {
+			pr_info("concurrent request %d error %d\n", i, rc[i]);
+			err = rc[i];
+		}
+	}
+
+	return err;
+}
+
+static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
+				int blen, int secs, u32 num_mb)
 {
 	unsigned long start, end;
 	int bcount;
@@ -95,21 +192,18 @@ static int test_aead_jiffies(struct aead_request *req, int enc,
 
 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
 	     time_before(jiffies, end); bcount++) {
-		if (enc)
-			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
-		else
-			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
-
+		ret = do_mult_aead_op(data, enc, num_mb);
 		if (ret)
 			return ret;
 	}
 
-	printk("%d operations in %d seconds (%ld bytes)\n",
-	       bcount, secs, (long)bcount * blen);
+	pr_cont("%d operations in %d seconds (%ld bytes)\n",
+		bcount * num_mb, secs, (long)bcount * blen * num_mb);
 	return 0;
 }
 
-static int test_aead_cycles(struct aead_request *req, int enc, int blen)
+static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
+			       int blen, u32 num_mb)
 {
 	unsigned long cycles = 0;
 	int ret = 0;
@@ -117,11 +211,7 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen)
 
 	/* Warm-up run. */
 	for (i = 0; i < 4; i++) {
-		if (enc)
-			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
-		else
-			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
-
+		ret = do_mult_aead_op(data, enc, num_mb);
 		if (ret)
 			goto out;
 	}
@@ -131,10 +221,7 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen)
 		cycles_t start, end;
 
 		start = get_cycles();
-		if (enc)
-			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
-		else
-			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
+		ret = do_mult_aead_op(data, enc, num_mb);
 		end = get_cycles();
 
 		if (ret)
@@ -145,64 +232,276 @@ static int test_aead_cycles(struct aead_request *req, int enc, int blen)
 
 out:
 	if (ret == 0)
-		printk("1 operation in %lu cycles (%d bytes)\n",
-		       (cycles + 4) / 8, blen);
+		pr_cont("1 operation in %lu cycles (%d bytes)\n",
+			(cycles + 4) / (8 * num_mb), blen);
 
 	return ret;
 }
 
-static u32 block_sizes[] = { 16, 64, 256, 1024, 8192, 0 };
-static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 };
+static void test_mb_aead_speed(const char *algo, int enc, int secs,
+			       struct aead_speed_template *template,
+			       unsigned int tcount, u8 authsize,
+			       unsigned int aad_size, u8 *keysize, u32 num_mb)
+{
+	struct test_mb_aead_data *data;
+	struct crypto_aead *tfm;
+	unsigned int i, j, iv_len;
+	const char *key;
+	const char *e;
+	void *assoc;
+	u32 *b_size;
+	char *iv;
+	int ret;
 
-#define XBUFSIZE 8
-#define MAX_IVLEN 32
 
-static int testmgr_alloc_buf(char *buf[XBUFSIZE])
-{
-	int i;
+	if (aad_size >= PAGE_SIZE) {
+		pr_err("associate data length (%u) too big\n", aad_size);
+		return;
+	}
 
-	for (i = 0; i < XBUFSIZE; i++) {
-		buf[i] = (void *)__get_free_page(GFP_KERNEL);
-		if (!buf[i])
-			goto err_free_buf;
+	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
+	if (!iv)
+		return;
+
+	if (enc == ENCRYPT)
+		e = "encryption";
+	else
+		e = "decryption";
+
+	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		goto out_free_iv;
+
+	tfm = crypto_alloc_aead(algo, 0, 0);
+	if (IS_ERR(tfm)) {
+		pr_err("failed to load transform for %s: %ld\n",
+			algo, PTR_ERR(tfm));
+		goto out_free_data;
 	}
 
-	return 0;
+	ret = crypto_aead_setauthsize(tfm, authsize);
 
-err_free_buf:
-	while (i-- > 0)
-		free_page((unsigned long)buf[i]);
+	for (i = 0; i < num_mb; ++i)
+		if (testmgr_alloc_buf(data[i].xbuf)) {
+			while (i--)
+				testmgr_free_buf(data[i].xbuf);
+			goto out_free_tfm;
+		}
 
-	return -ENOMEM;
+	for (i = 0; i < num_mb; ++i)
+		if (testmgr_alloc_buf(data[i].axbuf)) {
+			while (i--)
+				testmgr_free_buf(data[i].axbuf);
+			goto out_free_xbuf;
+		}
+
+	for (i = 0; i < num_mb; ++i)
+		if (testmgr_alloc_buf(data[i].xoutbuf)) {
+			while (i--)
+				testmgr_free_buf(data[i].xoutbuf);
+			goto out_free_axbuf;
+		}
+
+	for (i = 0; i < num_mb; ++i) {
+		data[i].req = aead_request_alloc(tfm, GFP_KERNEL);
+		if (!data[i].req) {
+			pr_err("alg: skcipher: Failed to allocate request for %s\n",
+			       algo);
+			while (i--)
+				aead_request_free(data[i].req);
+			goto out_free_xoutbuf;
+		}
+	}
+
+	for (i = 0; i < num_mb; ++i) {
+		crypto_init_wait(&data[i].wait);
+		aead_request_set_callback(data[i].req,
+					  CRYPTO_TFM_REQ_MAY_BACKLOG,
+					  crypto_req_done, &data[i].wait);
+	}
+
+	pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
+		get_driver_name(crypto_aead, tfm), e);
+
+	i = 0;
+	do {
+		b_size = aead_sizes;
+		do {
+			if (*b_size + authsize > XBUFSIZE * PAGE_SIZE) {
+				pr_err("template (%u) too big for buffer (%lu)\n",
+				       authsize + *b_size,
+				       XBUFSIZE * PAGE_SIZE);
+				goto out;
+			}
+
+			pr_info("test %u (%d bit key, %d byte blocks): ", i,
+				*keysize * 8, *b_size);
+
+			/* Set up tfm global state, i.e. the key */
+
+			memset(tvmem[0], 0xff, PAGE_SIZE);
+			key = tvmem[0];
+			for (j = 0; j < tcount; j++) {
+				if (template[j].klen == *keysize) {
+					key = template[j].key;
+					break;
+				}
+			}
+
+			crypto_aead_clear_flags(tfm, ~0);
+
+			ret = crypto_aead_setkey(tfm, key, *keysize);
+			if (ret) {
+				pr_err("setkey() failed flags=%x\n",
+				       crypto_aead_get_flags(tfm));
+				goto out;
+			}
+
+			iv_len = crypto_aead_ivsize(tfm);
+			if (iv_len)
+				memset(iv, 0xff, iv_len);
+
+			/* Now setup per request stuff, i.e. buffers */
+
+			for (j = 0; j < num_mb; ++j) {
+				struct test_mb_aead_data *cur = &data[j];
+
+				assoc = cur->axbuf[0];
+				memset(assoc, 0xff, aad_size);
+
+				sg_init_aead(cur->sg, cur->xbuf,
+					     *b_size + (enc ? 0 : authsize),
+					     assoc, aad_size);
+
+				sg_init_aead(cur->sgout, cur->xoutbuf,
+					     *b_size + (enc ? authsize : 0),
+					     assoc, aad_size);
+
+				aead_request_set_ad(cur->req, aad_size);
+
+				if (!enc) {
+
+					aead_request_set_crypt(cur->req,
+							       cur->sgout,
+							       cur->sg,
+							       *b_size, iv);
+					ret = crypto_aead_encrypt(cur->req);
+					ret = do_one_aead_op(cur->req, ret);
+
+					if (ret) {
+						pr_err("calculating auth failed failed (%d)\n",
+						       ret);
+						break;
+					}
+				}
+
+				aead_request_set_crypt(cur->req, cur->sg,
+						       cur->sgout, *b_size +
+						       (enc ? 0 : authsize),
+						       iv);
+
+			}
+
+			if (secs)
+				ret = test_mb_aead_jiffies(data, enc, *b_size,
+							   secs, num_mb);
+			else
+				ret = test_mb_aead_cycles(data, enc, *b_size,
+							  num_mb);
+
+			if (ret) {
+				pr_err("%s() failed return code=%d\n", e, ret);
+				break;
+			}
+			b_size++;
+			i++;
+		} while (*b_size);
+		keysize++;
+	} while (*keysize);
+
+out:
+	for (i = 0; i < num_mb; ++i)
+		aead_request_free(data[i].req);
+out_free_xoutbuf:
+	for (i = 0; i < num_mb; ++i)
+		testmgr_free_buf(data[i].xoutbuf);
+out_free_axbuf:
+	for (i = 0; i < num_mb; ++i)
+		testmgr_free_buf(data[i].axbuf);
+out_free_xbuf:
+	for (i = 0; i < num_mb; ++i)
+		testmgr_free_buf(data[i].xbuf);
+out_free_tfm:
+	crypto_free_aead(tfm);
+out_free_data:
+	kfree(data);
+out_free_iv:
+	kfree(iv);
 }
 
-static void testmgr_free_buf(char *buf[XBUFSIZE])
+static int test_aead_jiffies(struct aead_request *req, int enc,
+				int blen, int secs)
 {
-	int i;
+	unsigned long start, end;
+	int bcount;
+	int ret;
 
-	for (i = 0; i < XBUFSIZE; i++)
-		free_page((unsigned long)buf[i]);
+	for (start = jiffies, end = start + secs * HZ, bcount = 0;
+	     time_before(jiffies, end); bcount++) {
+		if (enc)
+			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
+		else
+			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
+
+		if (ret)
+			return ret;
+	}
+
+	printk("%d operations in %d seconds (%ld bytes)\n",
+	       bcount, secs, (long)bcount * blen);
+	return 0;
 }
 
-static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
-			unsigned int buflen)
+static int test_aead_cycles(struct aead_request *req, int enc, int blen)
 {
-	int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
-	int k, rem;
+	unsigned long cycles = 0;
+	int ret = 0;
+	int i;
 
-	if (np > XBUFSIZE) {
-		rem = PAGE_SIZE;
-		np = XBUFSIZE;
-	} else {
-		rem = buflen % PAGE_SIZE;
+	/* Warm-up run. */
+	for (i = 0; i < 4; i++) {
+		if (enc)
+			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
+		else
+			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
+
+		if (ret)
+			goto out;
 	}
 
-	sg_init_table(sg, np + 1);
-	np--;
-	for (k = 0; k < np; k++)
-		sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
+	/* The real thing. */
+	for (i = 0; i < 8; i++) {
+		cycles_t start, end;
+
+		start = get_cycles();
+		if (enc)
+			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
+		else
+			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
+		end = get_cycles();
+
+		if (ret)
+			goto out;
+
+		cycles += end - start;
+	}
+
+out:
+	if (ret == 0)
+		printk("1 operation in %lu cycles (%d bytes)\n",
+		       (cycles + 4) / 8, blen);
 
-	sg_set_buf(&sg[k + 1], xbuf[k], rem);
+	return ret;
 }
 
 static void test_aead_speed(const char *algo, int enc, unsigned int secs,
@@ -316,19 +615,37 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs,
 				goto out;
 			}
 
-			sg_init_aead(sg, xbuf,
-				    *b_size + (enc ? 0 : authsize));
+			sg_init_aead(sg, xbuf, *b_size + (enc ? 0 : authsize),
+				     assoc, aad_size);
 
 			sg_init_aead(sgout, xoutbuf,
-				    *b_size + (enc ? authsize : 0));
+				     *b_size + (enc ? authsize : 0), assoc,
+				     aad_size);
+
+			aead_request_set_ad(req, aad_size);
 
-			sg_set_buf(&sg[0], assoc, aad_size);
-			sg_set_buf(&sgout[0], assoc, aad_size);
+			if (!enc) {
+
+				/*
+				 * For decryption we need a proper auth so
+				 * we do the encryption path once with buffers
+				 * reversed (input <-> output) to calculate it
+				 */
+				aead_request_set_crypt(req, sgout, sg,
+						       *b_size, iv);
+				ret = do_one_aead_op(req,
+						     crypto_aead_encrypt(req));
+
+				if (ret) {
+					pr_err("calculating auth failed failed (%d)\n",
+					       ret);
+					break;
+				}
+			}
 
 			aead_request_set_crypt(req, sg, sgout,
 					       *b_size + (enc ? 0 : authsize),
 					       iv);
-			aead_request_set_ad(req, aad_size);
 
 			if (secs)
 				ret = test_aead_jiffies(req, enc, *b_size,
@@ -381,24 +698,98 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret)
 }
 
 struct test_mb_ahash_data {
-	struct scatterlist sg[TVMEMSIZE];
+	struct scatterlist sg[XBUFSIZE];
 	char result[64];
 	struct ahash_request *req;
 	struct crypto_wait wait;
 	char *xbuf[XBUFSIZE];
 };
 
-static void test_mb_ahash_speed(const char *algo, unsigned int sec,
-				struct hash_speed *speed)
+static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb)
+{
+	int i, rc[num_mb], err = 0;
+
+	/* Fire up a bunch of concurrent requests */
+	for (i = 0; i < num_mb; i++)
+		rc[i] = crypto_ahash_digest(data[i].req);
+
+	/* Wait for all requests to finish */
+	for (i = 0; i < num_mb; i++) {
+		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
+
+		if (rc[i]) {
+			pr_info("concurrent request %d error %d\n", i, rc[i]);
+			err = rc[i];
+		}
+	}
+
+	return err;
+}
+
+static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen,
+				 int secs, u32 num_mb)
+{
+	unsigned long start, end;
+	int bcount;
+	int ret;
+
+	for (start = jiffies, end = start + secs * HZ, bcount = 0;
+	     time_before(jiffies, end); bcount++) {
+		ret = do_mult_ahash_op(data, num_mb);
+		if (ret)
+			return ret;
+	}
+
+	pr_cont("%d operations in %d seconds (%ld bytes)\n",
+		bcount * num_mb, secs, (long)bcount * blen * num_mb);
+	return 0;
+}
+
+static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen,
+				u32 num_mb)
+{
+	unsigned long cycles = 0;
+	int ret = 0;
+	int i;
+
+	/* Warm-up run. */
+	for (i = 0; i < 4; i++) {
+		ret = do_mult_ahash_op(data, num_mb);
+		if (ret)
+			goto out;
+	}
+
+	/* The real thing. */
+	for (i = 0; i < 8; i++) {
+		cycles_t start, end;
+
+		start = get_cycles();
+		ret = do_mult_ahash_op(data, num_mb);
+		end = get_cycles();
+
+		if (ret)
+			goto out;
+
+		cycles += end - start;
+	}
+
+out:
+	if (ret == 0)
+		pr_cont("1 operation in %lu cycles (%d bytes)\n",
+			(cycles + 4) / (8 * num_mb), blen);
+
+	return ret;
+}
+
+static void test_mb_ahash_speed(const char *algo, unsigned int secs,
+				struct hash_speed *speed, u32 num_mb)
 {
 	struct test_mb_ahash_data *data;
 	struct crypto_ahash *tfm;
-	unsigned long start, end;
-	unsigned long cycles;
 	unsigned int i, j, k;
 	int ret;
 
-	data = kzalloc(sizeof(*data) * 8, GFP_KERNEL);
+	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return;
 
@@ -409,7 +800,7 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec,
 		goto free_data;
 	}
 
-	for (i = 0; i < 8; ++i) {
+	for (i = 0; i < num_mb; ++i) {
 		if (testmgr_alloc_buf(data[i].xbuf))
 			goto out;
 
@@ -424,7 +815,12 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec,
 
 		ahash_request_set_callback(data[i].req, 0, crypto_req_done,
 					   &data[i].wait);
-		test_hash_sg_init(data[i].sg);
+
+		sg_init_table(data[i].sg, XBUFSIZE);
+		for (j = 0; j < XBUFSIZE; j++) {
+			sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE);
+			memset(data[i].xbuf[j], 0xff, PAGE_SIZE);
+		}
 	}
 
 	pr_info("\ntesting speed of multibuffer %s (%s)\n", algo,
@@ -435,16 +831,16 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec,
 		if (speed[i].blen != speed[i].plen)
 			continue;
 
-		if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
+		if (speed[i].blen > XBUFSIZE * PAGE_SIZE) {
 			pr_err("template (%u) too big for tvmem (%lu)\n",
-			       speed[i].blen, TVMEMSIZE * PAGE_SIZE);
+			       speed[i].blen, XBUFSIZE * PAGE_SIZE);
 			goto out;
 		}
 
 		if (speed[i].klen)
 			crypto_ahash_setkey(tfm, tvmem[0], speed[i].klen);
 
-		for (k = 0; k < 8; k++)
+		for (k = 0; k < num_mb; k++)
 			ahash_request_set_crypt(data[k].req, data[k].sg,
 						data[k].result, speed[i].blen);
 
@@ -453,34 +849,12 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec,
 			i, speed[i].blen, speed[i].plen,
 			speed[i].blen / speed[i].plen);
 
-		start = get_cycles();
-
-		for (k = 0; k < 8; k++) {
-			ret = crypto_ahash_digest(data[k].req);
-			if (ret == -EINPROGRESS) {
-				ret = 0;
-				continue;
-			}
-
-			if (ret)
-				break;
-
-			crypto_req_done(&data[k].req->base, 0);
-		}
-
-		for (j = 0; j < k; j++) {
-			struct crypto_wait *wait = &data[j].wait;
-			int wait_ret;
-
-			wait_ret = crypto_wait_req(-EINPROGRESS, wait);
-			if (wait_ret)
-				ret = wait_ret;
-		}
+		if (secs)
+			ret = test_mb_ahash_jiffies(data, speed[i].blen, secs,
+						    num_mb);
+		else
+			ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb);
 
-		end = get_cycles();
-		cycles = end - start;
-		pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
-			cycles, cycles / (8 * speed[i].blen));
 
 		if (ret) {
 			pr_err("At least one hashing failed ret=%d\n", ret);
@@ -489,10 +863,10 @@ static void test_mb_ahash_speed(const char *algo, unsigned int sec,
 	}
 
 out:
-	for (k = 0; k < 8; ++k)
+	for (k = 0; k < num_mb; ++k)
 		ahash_request_free(data[k].req);
 
-	for (k = 0; k < 8; ++k)
+	for (k = 0; k < num_mb; ++k)
 		testmgr_free_buf(data[k].xbuf);
 
 	crypto_free_ahash(tfm);
@@ -736,6 +1110,254 @@ static void test_hash_speed(const char *algo, unsigned int secs,
 	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
 }
 
+struct test_mb_skcipher_data {
+	struct scatterlist sg[XBUFSIZE];
+	struct skcipher_request *req;
+	struct crypto_wait wait;
+	char *xbuf[XBUFSIZE];
+};
+
+static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
+				u32 num_mb)
+{
+	int i, rc[num_mb], err = 0;
+
+	/* Fire up a bunch of concurrent requests */
+	for (i = 0; i < num_mb; i++) {
+		if (enc == ENCRYPT)
+			rc[i] = crypto_skcipher_encrypt(data[i].req);
+		else
+			rc[i] = crypto_skcipher_decrypt(data[i].req);
+	}
+
+	/* Wait for all requests to finish */
+	for (i = 0; i < num_mb; i++) {
+		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
+
+		if (rc[i]) {
+			pr_info("concurrent request %d error %d\n", i, rc[i]);
+			err = rc[i];
+		}
+	}
+
+	return err;
+}
+
+static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
+				int blen, int secs, u32 num_mb)
+{
+	unsigned long start, end;
+	int bcount;
+	int ret;
+
+	for (start = jiffies, end = start + secs * HZ, bcount = 0;
+	     time_before(jiffies, end); bcount++) {
+		ret = do_mult_acipher_op(data, enc, num_mb);
+		if (ret)
+			return ret;
+	}
+
+	pr_cont("%d operations in %d seconds (%ld bytes)\n",
+		bcount * num_mb, secs, (long)bcount * blen * num_mb);
+	return 0;
+}
+
+static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
+			       int blen, u32 num_mb)
+{
+	unsigned long cycles = 0;
+	int ret = 0;
+	int i;
+
+	/* Warm-up run. */
+	for (i = 0; i < 4; i++) {
+		ret = do_mult_acipher_op(data, enc, num_mb);
+		if (ret)
+			goto out;
+	}
+
+	/* The real thing. */
+	for (i = 0; i < 8; i++) {
+		cycles_t start, end;
+
+		start = get_cycles();
+		ret = do_mult_acipher_op(data, enc, num_mb);
+		end = get_cycles();
+
+		if (ret)
+			goto out;
+
+		cycles += end - start;
+	}
+
+out:
+	if (ret == 0)
+		pr_cont("1 operation in %lu cycles (%d bytes)\n",
+			(cycles + 4) / (8 * num_mb), blen);
+
+	return ret;
+}
+
+static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
+				   struct cipher_speed_template *template,
+				   unsigned int tcount, u8 *keysize, u32 num_mb)
+{
+	struct test_mb_skcipher_data *data;
+	struct crypto_skcipher *tfm;
+	unsigned int i, j, iv_len;
+	const char *key;
+	const char *e;
+	u32 *b_size;
+	char iv[128];
+	int ret;
+
+	if (enc == ENCRYPT)
+		e = "encryption";
+	else
+		e = "decryption";
+
+	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return;
+
+	tfm = crypto_alloc_skcipher(algo, 0, 0);
+	if (IS_ERR(tfm)) {
+		pr_err("failed to load transform for %s: %ld\n",
+			algo, PTR_ERR(tfm));
+		goto out_free_data;
+	}
+
+	for (i = 0; i < num_mb; ++i)
+		if (testmgr_alloc_buf(data[i].xbuf)) {
+			while (i--)
+				testmgr_free_buf(data[i].xbuf);
+			goto out_free_tfm;
+		}
+
+
+	for (i = 0; i < num_mb; ++i)
+		if (testmgr_alloc_buf(data[i].xbuf)) {
+			while (i--)
+				testmgr_free_buf(data[i].xbuf);
+			goto out_free_tfm;
+		}
+
+
+	for (i = 0; i < num_mb; ++i) {
+		data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL);
+		if (!data[i].req) {
+			pr_err("alg: skcipher: Failed to allocate request for %s\n",
+			       algo);
+			while (i--)
+				skcipher_request_free(data[i].req);
+			goto out_free_xbuf;
+		}
+	}
+
+	for (i = 0; i < num_mb; ++i) {
+		skcipher_request_set_callback(data[i].req,
+					      CRYPTO_TFM_REQ_MAY_BACKLOG,
+					      crypto_req_done, &data[i].wait);
+		crypto_init_wait(&data[i].wait);
+	}
+
+	pr_info("\ntesting speed of multibuffer %s (%s) %s\n", algo,
+		get_driver_name(crypto_skcipher, tfm), e);
+
+	i = 0;
+	do {
+		b_size = block_sizes;
+		do {
+			if (*b_size > XBUFSIZE * PAGE_SIZE) {
+				pr_err("template (%u) too big for buffer (%lu)\n",
+				       *b_size, XBUFSIZE * PAGE_SIZE);
+				goto out;
+			}
+
+			pr_info("test %u (%d bit key, %d byte blocks): ", i,
+				*keysize * 8, *b_size);
+
+			/* Set up tfm global state, i.e. the key */
+
+			memset(tvmem[0], 0xff, PAGE_SIZE);
+			key = tvmem[0];
+			for (j = 0; j < tcount; j++) {
+				if (template[j].klen == *keysize) {
+					key = template[j].key;
+					break;
+				}
+			}
+
+			crypto_skcipher_clear_flags(tfm, ~0);
+
+			ret = crypto_skcipher_setkey(tfm, key, *keysize);
+			if (ret) {
+				pr_err("setkey() failed flags=%x\n",
+				       crypto_skcipher_get_flags(tfm));
+				goto out;
+			}
+
+			iv_len = crypto_skcipher_ivsize(tfm);
+			if (iv_len)
+				memset(&iv, 0xff, iv_len);
+
+			/* Now setup per request stuff, i.e. buffers */
+
+			for (j = 0; j < num_mb; ++j) {
+				struct test_mb_skcipher_data *cur = &data[j];
+				unsigned int k = *b_size;
+				unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE);
+				unsigned int p = 0;
+
+				sg_init_table(cur->sg, pages);
+
+				while (k > PAGE_SIZE) {
+					sg_set_buf(cur->sg + p, cur->xbuf[p],
+						   PAGE_SIZE);
+					memset(cur->xbuf[p], 0xff, PAGE_SIZE);
+					p++;
+					k -= PAGE_SIZE;
+				}
+
+				sg_set_buf(cur->sg + p, cur->xbuf[p], k);
+				memset(cur->xbuf[p], 0xff, k);
+
+				skcipher_request_set_crypt(cur->req, cur->sg,
+							   cur->sg, *b_size,
+							   iv);
+			}
+
+			if (secs)
+				ret = test_mb_acipher_jiffies(data, enc,
+							      *b_size, secs,
+							      num_mb);
+			else
+				ret = test_mb_acipher_cycles(data, enc,
+							     *b_size, num_mb);
+
+			if (ret) {
+				pr_err("%s() failed flags=%x\n", e,
+				       crypto_skcipher_get_flags(tfm));
+				break;
+			}
+			b_size++;
+			i++;
+		} while (*b_size);
+		keysize++;
+	} while (*keysize);
+
+out:
+	for (i = 0; i < num_mb; ++i)
+		skcipher_request_free(data[i].req);
+out_free_xbuf:
+	for (i = 0; i < num_mb; ++i)
+		testmgr_free_buf(data[i].xbuf);
+out_free_tfm:
+	crypto_free_skcipher(tfm);
+out_free_data:
+	kfree(data);
+}
+
 static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
 {
 	struct crypto_wait *wait = req->base.data;
@@ -1557,16 +2179,24 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
 				NULL, 0, 16, 16, aead_speed_template_20);
 		test_aead_speed("gcm(aes)", ENCRYPT, sec,
 				NULL, 0, 16, 8, speed_template_16_24_32);
+		test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec,
+				NULL, 0, 16, 16, aead_speed_template_20);
+		test_aead_speed("gcm(aes)", DECRYPT, sec,
+				NULL, 0, 16, 8, speed_template_16_24_32);
 		break;
 
 	case 212:
 		test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec,
 				NULL, 0, 16, 16, aead_speed_template_19);
+		test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec,
+				NULL, 0, 16, 16, aead_speed_template_19);
 		break;
 
 	case 213:
 		test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec,
 				NULL, 0, 16, 8, aead_speed_template_36);
+		test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec,
+				NULL, 0, 16, 8, aead_speed_template_36);
 		break;
 
 	case 214:
@@ -1574,6 +2204,33 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
 				  speed_template_32);
 		break;
 
+	case 215:
+		test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL,
+				   0, 16, 16, aead_speed_template_20, num_mb);
+		test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8,
+				   speed_template_16_24_32, num_mb);
+		test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL,
+				   0, 16, 16, aead_speed_template_20, num_mb);
+		test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8,
+				   speed_template_16_24_32, num_mb);
+		break;
+
+	case 216:
+		test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0,
+				   16, 16, aead_speed_template_19, num_mb);
+		test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0,
+				   16, 16, aead_speed_template_19, num_mb);
+		break;
+
+	case 217:
+		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT,
+				   sec, NULL, 0, 16, 8, aead_speed_template_36,
+				   num_mb);
+		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT,
+				   sec, NULL, 0, 16, 8, aead_speed_template_36,
+				   num_mb);
+		break;
+
 	case 300:
 		if (alg) {
 			test_hash_speed(alg, sec, generic_hash_speed_template);
@@ -1778,19 +2435,23 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
 		if (mode > 400 && mode < 500) break;
 		/* fall through */
 	case 422:
-		test_mb_ahash_speed("sha1", sec, generic_hash_speed_template);
+		test_mb_ahash_speed("sha1", sec, generic_hash_speed_template,
+				    num_mb);
 		if (mode > 400 && mode < 500) break;
 		/* fall through */
 	case 423:
-		test_mb_ahash_speed("sha256", sec, generic_hash_speed_template);
+		test_mb_ahash_speed("sha256", sec, generic_hash_speed_template,
+				    num_mb);
 		if (mode > 400 && mode < 500) break;
 		/* fall through */
 	case 424:
-		test_mb_ahash_speed("sha512", sec, generic_hash_speed_template);
+		test_mb_ahash_speed("sha512", sec, generic_hash_speed_template,
+				    num_mb);
 		if (mode > 400 && mode < 500) break;
 		/* fall through */
 	case 425:
-		test_mb_ahash_speed("sm3", sec, generic_hash_speed_template);
+		test_mb_ahash_speed("sm3", sec, generic_hash_speed_template,
+				    num_mb);
 		if (mode > 400 && mode < 500) break;
 		/* fall through */
 	case 499:
@@ -2008,6 +2669,218 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
 				   speed_template_8_32);
 		break;
 
+	case 600:
+		test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_40_48, num_mb);
+		test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_40_48, num_mb);
+		test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL,
+				       0, speed_template_20_28_36, num_mb);
+		test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL,
+				       0, speed_template_20_28_36, num_mb);
+		break;
+
+	case 601:
+		test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		test_mb_skcipher_speed("cfb(des3_ede)", ENCRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		test_mb_skcipher_speed("cfb(des3_ede)", DECRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		test_mb_skcipher_speed("ofb(des3_ede)", ENCRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		test_mb_skcipher_speed("ofb(des3_ede)", DECRYPT, sec,
+				       des3_speed_template, DES3_SPEED_VECTORS,
+				       speed_template_24, num_mb);
+		break;
+
+	case 602:
+		test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		test_mb_skcipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		test_mb_skcipher_speed("cfb(des)", DECRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		test_mb_skcipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		test_mb_skcipher_speed("ofb(des)", DECRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		break;
+
+	case 603:
+		test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_48, num_mb);
+		test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_48, num_mb);
+		test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		break;
+
+	case 604:
+		test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_24_32, num_mb);
+		test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_40_48, num_mb);
+		test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_40_48, num_mb);
+		test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_48_64, num_mb);
+		test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_48_64, num_mb);
+		break;
+
+	case 605:
+		test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8, num_mb);
+		break;
+
+	case 606:
+		test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8_16, num_mb);
+		test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
+				       speed_template_8_16, num_mb);
+		test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8_16, num_mb);
+		test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
+				       speed_template_8_16, num_mb);
+		test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8_16, num_mb);
+		test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
+				       speed_template_8_16, num_mb);
+		break;
+
+	case 607:
+		test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_48, num_mb);
+		test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_48, num_mb);
+		test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		break;
+
+	case 608:
+		test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
+				       speed_template_16_32, num_mb);
+		test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_48, num_mb);
+		test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_48, num_mb);
+		test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
+				       speed_template_32_64, num_mb);
+		break;
+
+	case 609:
+		test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8_32, num_mb);
+		test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
+				       speed_template_8_32, num_mb);
+		test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8_32, num_mb);
+		test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
+				       speed_template_8_32, num_mb);
+		test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
+				       speed_template_8_32, num_mb);
+		test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
+				       speed_template_8_32, num_mb);
+		break;
+
 	case 1000:
 		test_available();
 		break;
@@ -2069,6 +2942,8 @@ module_param(mode, int, 0);
 module_param(sec, uint, 0);
 MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
 		      "(defaults to zero which uses CPU cycles instead)");
+module_param(num_mb, uint, 0000);
+MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Quick & dirty crypto testing module");
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 29d7020b8826..d5e23a142a04 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -177,6 +177,18 @@ static void testmgr_free_buf(char *buf[XBUFSIZE])
 		free_page((unsigned long)buf[i]);
 }
 
+static int ahash_guard_result(char *result, char c, int size)
+{
+	int i;
+
+	for (i = 0; i < size; i++) {
+		if (result[i] != c)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int ahash_partial_update(struct ahash_request **preq,
 	struct crypto_ahash *tfm, const struct hash_testvec *template,
 	void *hash_buff, int k, int temp, struct scatterlist *sg,
@@ -185,7 +197,8 @@ static int ahash_partial_update(struct ahash_request **preq,
 	char *state;
 	struct ahash_request *req;
 	int statesize, ret = -EINVAL;
-	const char guard[] = { 0x00, 0xba, 0xad, 0x00 };
+	static const unsigned char guard[] = { 0x00, 0xba, 0xad, 0x00 };
+	int digestsize = crypto_ahash_digestsize(tfm);
 
 	req = *preq;
 	statesize = crypto_ahash_statesize(
@@ -196,12 +209,19 @@ static int ahash_partial_update(struct ahash_request **preq,
 		goto out_nostate;
 	}
 	memcpy(state + statesize, guard, sizeof(guard));
+	memset(result, 1, digestsize);
 	ret = crypto_ahash_export(req, state);
 	WARN_ON(memcmp(state + statesize, guard, sizeof(guard)));
 	if (ret) {
 		pr_err("alg: hash: Failed to export() for %s\n", algo);
 		goto out;
 	}
+	ret = ahash_guard_result(result, 1, digestsize);
+	if (ret) {
+		pr_err("alg: hash: Failed, export used req->result for %s\n",
+		       algo);
+		goto out;
+	}
 	ahash_request_free(req);
 	req = ahash_request_alloc(tfm, GFP_KERNEL);
 	if (!req) {
@@ -221,6 +241,12 @@ static int ahash_partial_update(struct ahash_request **preq,
 		pr_err("alg: hash: Failed to import() for %s\n", algo);
 		goto out;
 	}
+	ret = ahash_guard_result(result, 1, digestsize);
+	if (ret) {
+		pr_err("alg: hash: Failed, import used req->result for %s\n",
+		       algo);
+		goto out;
+	}
 	ret = crypto_wait_req(crypto_ahash_update(req), wait);
 	if (ret)
 		goto out;
@@ -316,18 +342,31 @@ static int __test_hash(struct crypto_ahash *tfm,
 				goto out;
 			}
 		} else {
+			memset(result, 1, digest_size);
 			ret = crypto_wait_req(crypto_ahash_init(req), &wait);
 			if (ret) {
 				pr_err("alg: hash: init failed on test %d "
 				       "for %s: ret=%d\n", j, algo, -ret);
 				goto out;
 			}
+			ret = ahash_guard_result(result, 1, digest_size);
+			if (ret) {
+				pr_err("alg: hash: init failed on test %d "
+				       "for %s: used req->result\n", j, algo);
+				goto out;
+			}
 			ret = crypto_wait_req(crypto_ahash_update(req), &wait);
 			if (ret) {
 				pr_err("alg: hash: update failed on test %d "
 				       "for %s: ret=%d\n", j, algo, -ret);
 				goto out;
 			}
+			ret = ahash_guard_result(result, 1, digest_size);
+			if (ret) {
+				pr_err("alg: hash: update failed on test %d "
+				       "for %s: used req->result\n", j, algo);
+				goto out;
+			}
 			ret = crypto_wait_req(crypto_ahash_final(req), &wait);
 			if (ret) {
 				pr_err("alg: hash: final failed on test %d "
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index a714b6293959..6044f6906bd6 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -1052,6 +1052,142 @@ static const struct hash_testvec sha3_224_tv_template[] = {
 				"\xc9\xfd\x55\x74\x49\x44\x79\xba"
 				"\x5c\x7e\x7a\xb7\x6e\xf2\x64\xea"
 				"\xd0\xfc\xce\x33",
+		.np	= 2,
+		.tap	= { 28, 28 },
+	}, {
+		.plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3"
+			     "\x7a\x11\x85\x1c\xb3\x27\xbe\x55"
+			     "\xec\x60\xf7\x8e\x02\x99\x30\xc7"
+			     "\x3b\xd2\x69\x00\x74\x0b\xa2\x16"
+			     "\xad\x44\xdb\x4f\xe6\x7d\x14\x88"
+			     "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa"
+			     "\x91\x05\x9c\x33\xca\x3e\xd5\x6c"
+			     "\x03\x77\x0e\xa5\x19\xb0\x47\xde"
+			     "\x52\xe9\x80\x17\x8b\x22\xb9\x2d"
+			     "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f"
+			     "\x36\xcd\x41\xd8\x6f\x06\x7a\x11"
+			     "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83"
+			     "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5"
+			     "\x69\x00\x97\x0b\xa2\x39\xd0\x44"
+			     "\xdb\x72\x09\x7d\x14\xab\x1f\xb6"
+			     "\x4d\xe4\x58\xef\x86\x1d\x91\x28"
+			     "\xbf\x33\xca\x61\xf8\x6c\x03\x9a"
+			     "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c"
+			     "\x80\x17\xae\x22\xb9\x50\xe7\x5b"
+			     "\xf2\x89\x20\x94\x2b\xc2\x36\xcd"
+			     "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f"
+			     "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1"
+			     "\x25\xbc\x53\xea\x5e\xf5\x8c\x00"
+			     "\x97\x2e\xc5\x39\xd0\x67\xfe\x72"
+			     "\x09\xa0\x14\xab\x42\xd9\x4d\xe4"
+			     "\x7b\x12\x86\x1d\xb4\x28\xbf\x56"
+			     "\xed\x61\xf8\x8f\x03\x9a\x31\xc8"
+			     "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17"
+			     "\xae\x45\xdc\x50\xe7\x7e\x15\x89"
+			     "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb"
+			     "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d"
+			     "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf"
+			     "\x53\xea\x81\x18\x8c\x23\xba\x2e"
+			     "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0"
+			     "\x37\xce\x42\xd9\x70\x07\x7b\x12"
+			     "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84"
+			     "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6"
+			     "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45"
+			     "\xdc\x73\x0a\x7e\x15\xac\x20\xb7"
+			     "\x4e\xe5\x59\xf0\x87\x1e\x92\x29"
+			     "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b"
+			     "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d"
+			     "\x81\x18\xaf\x23\xba\x51\xe8\x5c"
+			     "\xf3\x8a\x21\x95\x2c\xc3\x37\xce"
+			     "\x65\xfc\x70\x07\x9e\x12\xa9\x40"
+			     "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2"
+			     "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01"
+			     "\x98\x2f\xc6\x3a\xd1\x68\xff\x73"
+			     "\x0a\xa1\x15\xac\x43\xda\x4e\xe5"
+			     "\x7c\x13\x87\x1e\xb5\x29\xc0\x57"
+			     "\xee\x62\xf9\x90\x04\x9b\x32\xc9"
+			     "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18"
+			     "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a"
+			     "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc"
+			     "\x93\x07\x9e\x35\xcc\x40\xd7\x6e"
+			     "\x05\x79\x10\xa7\x1b\xb2\x49\xe0"
+			     "\x54\xeb\x82\x19\x8d\x24\xbb\x2f"
+			     "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1"
+			     "\x38\xcf\x43\xda\x71\x08\x7c\x13"
+			     "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85"
+			     "\x1c\x90\x27\xbe\x32\xc9\x60\xf7"
+			     "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46"
+			     "\xdd\x74\x0b\x7f\x16\xad\x21\xb8"
+			     "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a"
+			     "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c"
+			     "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e"
+			     "\x82\x19\xb0\x24\xbb\x52\xe9\x5d"
+			     "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf"
+			     "\x66\xfd\x71\x08\x9f\x13\xaa\x41"
+			     "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3"
+			     "\x27\xbe\x55\xec\x60\xf7\x8e\x02"
+			     "\x99\x30\xc7\x3b\xd2\x69\x00\x74"
+			     "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6"
+			     "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58"
+			     "\xef\x63\xfa\x91\x05\x9c\x33\xca"
+			     "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19"
+			     "\xb0\x47\xde\x52\xe9\x80\x17\x8b"
+			     "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd"
+			     "\x94\x08\x9f\x36\xcd\x41\xd8\x6f"
+			     "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1"
+			     "\x55\xec\x83\x1a\x8e\x25\xbc\x30"
+			     "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2"
+			     "\x39\xd0\x44\xdb\x72\x09\x7d\x14"
+			     "\xab\x1f\xb6\x4d\xe4\x58\xef\x86"
+			     "\x1d\x91\x28\xbf\x33\xca\x61\xf8"
+			     "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47"
+			     "\xde\x75\x0c\x80\x17\xae\x22\xb9"
+			     "\x50\xe7\x5b\xf2\x89\x20\x94\x2b"
+			     "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d"
+			     "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f"
+			     "\x83\x1a\xb1\x25\xbc\x53\xea\x5e"
+			     "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0"
+			     "\x67\xfe\x72\x09\xa0\x14\xab\x42"
+			     "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4"
+			     "\x28\xbf\x56\xed\x61\xf8\x8f\x03"
+			     "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75"
+			     "\x0c\xa3\x17\xae\x45\xdc\x50\xe7"
+			     "\x7e\x15\x89\x20\xb7\x2b\xc2\x59"
+			     "\xf0\x64\xfb\x92\x06\x9d\x34\xcb"
+			     "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a"
+			     "\xb1\x48\xdf\x53\xea\x81\x18\x8c"
+			     "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe"
+			     "\x95\x09\xa0\x37\xce\x42\xd9\x70"
+			     "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2"
+			     "\x56\xed\x84\x1b\x8f\x26\xbd\x31"
+			     "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3"
+			     "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15"
+			     "\xac\x20\xb7\x4e\xe5\x59\xf0\x87"
+			     "\x1e\x92\x29\xc0\x34\xcb\x62\xf9"
+			     "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48"
+			     "\xdf\x76\x0d\x81\x18\xaf\x23\xba"
+			     "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c"
+			     "\xc3\x37\xce\x65\xfc\x70\x07\x9e"
+			     "\x12\xa9\x40\xd7\x4b\xe2\x79\x10"
+			     "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f"
+			     "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1"
+			     "\x68\xff\x73\x0a\xa1\x15\xac\x43"
+			     "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5"
+			     "\x29\xc0\x57\xee\x62\xf9\x90\x04"
+			     "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76"
+			     "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8"
+			     "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a"
+			     "\xf1\x65\xfc\x93\x07\x9e\x35\xcc"
+			     "\x40\xd7\x6e\x05\x79\x10\xa7\x1b"
+			     "\xb2\x49\xe0\x54\xeb\x82\x19\x8d"
+			     "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff"
+			     "\x96\x0a\xa1\x38\xcf\x43\xda\x71"
+			     "\x08\x7c\x13\xaa\x1e\xb5\x4c",
+		.psize     = 1023,
+		.digest    = "\x7d\x0f\x2f\xb7\x65\x3b\xa7\x26"
+			     "\xc3\x88\x20\x71\x15\x06\xe8\x2d"
+			     "\xa3\x92\x44\xab\x3e\xe7\xff\x86"
+			     "\xb6\x79\x10\x72",
 	},
 };
 
@@ -1077,6 +1213,142 @@ static const struct hash_testvec sha3_256_tv_template[] = {
 				"\x49\x10\x03\x76\xa8\x23\x5e\x2c"
 				"\x82\xe1\xb9\x99\x8a\x99\x9e\x21"
 				"\xdb\x32\xdd\x97\x49\x6d\x33\x76",
+		.np	= 2,
+		.tap	= { 28, 28 },
+	}, {
+		.plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3"
+			     "\x7a\x11\x85\x1c\xb3\x27\xbe\x55"
+			     "\xec\x60\xf7\x8e\x02\x99\x30\xc7"
+			     "\x3b\xd2\x69\x00\x74\x0b\xa2\x16"
+			     "\xad\x44\xdb\x4f\xe6\x7d\x14\x88"
+			     "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa"
+			     "\x91\x05\x9c\x33\xca\x3e\xd5\x6c"
+			     "\x03\x77\x0e\xa5\x19\xb0\x47\xde"
+			     "\x52\xe9\x80\x17\x8b\x22\xb9\x2d"
+			     "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f"
+			     "\x36\xcd\x41\xd8\x6f\x06\x7a\x11"
+			     "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83"
+			     "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5"
+			     "\x69\x00\x97\x0b\xa2\x39\xd0\x44"
+			     "\xdb\x72\x09\x7d\x14\xab\x1f\xb6"
+			     "\x4d\xe4\x58\xef\x86\x1d\x91\x28"
+			     "\xbf\x33\xca\x61\xf8\x6c\x03\x9a"
+			     "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c"
+			     "\x80\x17\xae\x22\xb9\x50\xe7\x5b"
+			     "\xf2\x89\x20\x94\x2b\xc2\x36\xcd"
+			     "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f"
+			     "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1"
+			     "\x25\xbc\x53\xea\x5e\xf5\x8c\x00"
+			     "\x97\x2e\xc5\x39\xd0\x67\xfe\x72"
+			     "\x09\xa0\x14\xab\x42\xd9\x4d\xe4"
+			     "\x7b\x12\x86\x1d\xb4\x28\xbf\x56"
+			     "\xed\x61\xf8\x8f\x03\x9a\x31\xc8"
+			     "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17"
+			     "\xae\x45\xdc\x50\xe7\x7e\x15\x89"
+			     "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb"
+			     "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d"
+			     "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf"
+			     "\x53\xea\x81\x18\x8c\x23\xba\x2e"
+			     "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0"
+			     "\x37\xce\x42\xd9\x70\x07\x7b\x12"
+			     "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84"
+			     "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6"
+			     "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45"
+			     "\xdc\x73\x0a\x7e\x15\xac\x20\xb7"
+			     "\x4e\xe5\x59\xf0\x87\x1e\x92\x29"
+			     "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b"
+			     "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d"
+			     "\x81\x18\xaf\x23\xba\x51\xe8\x5c"
+			     "\xf3\x8a\x21\x95\x2c\xc3\x37\xce"
+			     "\x65\xfc\x70\x07\x9e\x12\xa9\x40"
+			     "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2"
+			     "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01"
+			     "\x98\x2f\xc6\x3a\xd1\x68\xff\x73"
+			     "\x0a\xa1\x15\xac\x43\xda\x4e\xe5"
+			     "\x7c\x13\x87\x1e\xb5\x29\xc0\x57"
+			     "\xee\x62\xf9\x90\x04\x9b\x32\xc9"
+			     "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18"
+			     "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a"
+			     "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc"
+			     "\x93\x07\x9e\x35\xcc\x40\xd7\x6e"
+			     "\x05\x79\x10\xa7\x1b\xb2\x49\xe0"
+			     "\x54\xeb\x82\x19\x8d\x24\xbb\x2f"
+			     "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1"
+			     "\x38\xcf\x43\xda\x71\x08\x7c\x13"
+			     "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85"
+			     "\x1c\x90\x27\xbe\x32\xc9\x60\xf7"
+			     "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46"
+			     "\xdd\x74\x0b\x7f\x16\xad\x21\xb8"
+			     "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a"
+			     "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c"
+			     "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e"
+			     "\x82\x19\xb0\x24\xbb\x52\xe9\x5d"
+			     "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf"
+			     "\x66\xfd\x71\x08\x9f\x13\xaa\x41"
+			     "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3"
+			     "\x27\xbe\x55\xec\x60\xf7\x8e\x02"
+			     "\x99\x30\xc7\x3b\xd2\x69\x00\x74"
+			     "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6"
+			     "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58"
+			     "\xef\x63\xfa\x91\x05\x9c\x33\xca"
+			     "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19"
+			     "\xb0\x47\xde\x52\xe9\x80\x17\x8b"
+			     "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd"
+			     "\x94\x08\x9f\x36\xcd\x41\xd8\x6f"
+			     "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1"
+			     "\x55\xec\x83\x1a\x8e\x25\xbc\x30"
+			     "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2"
+			     "\x39\xd0\x44\xdb\x72\x09\x7d\x14"
+			     "\xab\x1f\xb6\x4d\xe4\x58\xef\x86"
+			     "\x1d\x91\x28\xbf\x33\xca\x61\xf8"
+			     "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47"
+			     "\xde\x75\x0c\x80\x17\xae\x22\xb9"
+			     "\x50\xe7\x5b\xf2\x89\x20\x94\x2b"
+			     "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d"
+			     "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f"
+			     "\x83\x1a\xb1\x25\xbc\x53\xea\x5e"
+			     "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0"
+			     "\x67\xfe\x72\x09\xa0\x14\xab\x42"
+			     "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4"
+			     "\x28\xbf\x56\xed\x61\xf8\x8f\x03"
+			     "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75"
+			     "\x0c\xa3\x17\xae\x45\xdc\x50\xe7"
+			     "\x7e\x15\x89\x20\xb7\x2b\xc2\x59"
+			     "\xf0\x64\xfb\x92\x06\x9d\x34\xcb"
+			     "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a"
+			     "\xb1\x48\xdf\x53\xea\x81\x18\x8c"
+			     "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe"
+			     "\x95\x09\xa0\x37\xce\x42\xd9\x70"
+			     "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2"
+			     "\x56\xed\x84\x1b\x8f\x26\xbd\x31"
+			     "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3"
+			     "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15"
+			     "\xac\x20\xb7\x4e\xe5\x59\xf0\x87"
+			     "\x1e\x92\x29\xc0\x34\xcb\x62\xf9"
+			     "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48"
+			     "\xdf\x76\x0d\x81\x18\xaf\x23\xba"
+			     "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c"
+			     "\xc3\x37\xce\x65\xfc\x70\x07\x9e"
+			     "\x12\xa9\x40\xd7\x4b\xe2\x79\x10"
+			     "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f"
+			     "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1"
+			     "\x68\xff\x73\x0a\xa1\x15\xac\x43"
+			     "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5"
+			     "\x29\xc0\x57\xee\x62\xf9\x90\x04"
+			     "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76"
+			     "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8"
+			     "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a"
+			     "\xf1\x65\xfc\x93\x07\x9e\x35\xcc"
+			     "\x40\xd7\x6e\x05\x79\x10\xa7\x1b"
+			     "\xb2\x49\xe0\x54\xeb\x82\x19\x8d"
+			     "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff"
+			     "\x96\x0a\xa1\x38\xcf\x43\xda\x71"
+			     "\x08\x7c\x13\xaa\x1e\xb5\x4c",
+		.psize     = 1023,
+		.digest    = "\xde\x41\x04\xbd\xda\xda\xd9\x71"
+			     "\xf7\xfa\x80\xf5\xea\x11\x03\xb1"
+			     "\x3b\x6a\xbc\x5f\xb9\x66\x26\xf7"
+			     "\x8a\x97\xbb\xf2\x07\x08\x38\x30",
 	},
 };
 
@@ -1109,6 +1381,144 @@ static const struct hash_testvec sha3_384_tv_template[] = {
 				"\x9b\xfd\xbc\x32\xb9\xd4\xad\x5a"
 				"\xa0\x4a\x1f\x07\x6e\x62\xfe\xa1"
 				"\x9e\xef\x51\xac\xd0\x65\x7c\x22",
+		.np	= 2,
+		.tap	= { 28, 28 },
+	}, {
+		.plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3"
+			     "\x7a\x11\x85\x1c\xb3\x27\xbe\x55"
+			     "\xec\x60\xf7\x8e\x02\x99\x30\xc7"
+			     "\x3b\xd2\x69\x00\x74\x0b\xa2\x16"
+			     "\xad\x44\xdb\x4f\xe6\x7d\x14\x88"
+			     "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa"
+			     "\x91\x05\x9c\x33\xca\x3e\xd5\x6c"
+			     "\x03\x77\x0e\xa5\x19\xb0\x47\xde"
+			     "\x52\xe9\x80\x17\x8b\x22\xb9\x2d"
+			     "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f"
+			     "\x36\xcd\x41\xd8\x6f\x06\x7a\x11"
+			     "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83"
+			     "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5"
+			     "\x69\x00\x97\x0b\xa2\x39\xd0\x44"
+			     "\xdb\x72\x09\x7d\x14\xab\x1f\xb6"
+			     "\x4d\xe4\x58\xef\x86\x1d\x91\x28"
+			     "\xbf\x33\xca\x61\xf8\x6c\x03\x9a"
+			     "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c"
+			     "\x80\x17\xae\x22\xb9\x50\xe7\x5b"
+			     "\xf2\x89\x20\x94\x2b\xc2\x36\xcd"
+			     "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f"
+			     "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1"
+			     "\x25\xbc\x53\xea\x5e\xf5\x8c\x00"
+			     "\x97\x2e\xc5\x39\xd0\x67\xfe\x72"
+			     "\x09\xa0\x14\xab\x42\xd9\x4d\xe4"
+			     "\x7b\x12\x86\x1d\xb4\x28\xbf\x56"
+			     "\xed\x61\xf8\x8f\x03\x9a\x31\xc8"
+			     "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17"
+			     "\xae\x45\xdc\x50\xe7\x7e\x15\x89"
+			     "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb"
+			     "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d"
+			     "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf"
+			     "\x53\xea\x81\x18\x8c\x23\xba\x2e"
+			     "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0"
+			     "\x37\xce\x42\xd9\x70\x07\x7b\x12"
+			     "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84"
+			     "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6"
+			     "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45"
+			     "\xdc\x73\x0a\x7e\x15\xac\x20\xb7"
+			     "\x4e\xe5\x59\xf0\x87\x1e\x92\x29"
+			     "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b"
+			     "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d"
+			     "\x81\x18\xaf\x23\xba\x51\xe8\x5c"
+			     "\xf3\x8a\x21\x95\x2c\xc3\x37\xce"
+			     "\x65\xfc\x70\x07\x9e\x12\xa9\x40"
+			     "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2"
+			     "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01"
+			     "\x98\x2f\xc6\x3a\xd1\x68\xff\x73"
+			     "\x0a\xa1\x15\xac\x43\xda\x4e\xe5"
+			     "\x7c\x13\x87\x1e\xb5\x29\xc0\x57"
+			     "\xee\x62\xf9\x90\x04\x9b\x32\xc9"
+			     "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18"
+			     "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a"
+			     "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc"
+			     "\x93\x07\x9e\x35\xcc\x40\xd7\x6e"
+			     "\x05\x79\x10\xa7\x1b\xb2\x49\xe0"
+			     "\x54\xeb\x82\x19\x8d\x24\xbb\x2f"
+			     "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1"
+			     "\x38\xcf\x43\xda\x71\x08\x7c\x13"
+			     "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85"
+			     "\x1c\x90\x27\xbe\x32\xc9\x60\xf7"
+			     "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46"
+			     "\xdd\x74\x0b\x7f\x16\xad\x21\xb8"
+			     "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a"
+			     "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c"
+			     "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e"
+			     "\x82\x19\xb0\x24\xbb\x52\xe9\x5d"
+			     "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf"
+			     "\x66\xfd\x71\x08\x9f\x13\xaa\x41"
+			     "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3"
+			     "\x27\xbe\x55\xec\x60\xf7\x8e\x02"
+			     "\x99\x30\xc7\x3b\xd2\x69\x00\x74"
+			     "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6"
+			     "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58"
+			     "\xef\x63\xfa\x91\x05\x9c\x33\xca"
+			     "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19"
+			     "\xb0\x47\xde\x52\xe9\x80\x17\x8b"
+			     "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd"
+			     "\x94\x08\x9f\x36\xcd\x41\xd8\x6f"
+			     "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1"
+			     "\x55\xec\x83\x1a\x8e\x25\xbc\x30"
+			     "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2"
+			     "\x39\xd0\x44\xdb\x72\x09\x7d\x14"
+			     "\xab\x1f\xb6\x4d\xe4\x58\xef\x86"
+			     "\x1d\x91\x28\xbf\x33\xca\x61\xf8"
+			     "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47"
+			     "\xde\x75\x0c\x80\x17\xae\x22\xb9"
+			     "\x50\xe7\x5b\xf2\x89\x20\x94\x2b"
+			     "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d"
+			     "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f"
+			     "\x83\x1a\xb1\x25\xbc\x53\xea\x5e"
+			     "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0"
+			     "\x67\xfe\x72\x09\xa0\x14\xab\x42"
+			     "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4"
+			     "\x28\xbf\x56\xed\x61\xf8\x8f\x03"
+			     "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75"
+			     "\x0c\xa3\x17\xae\x45\xdc\x50\xe7"
+			     "\x7e\x15\x89\x20\xb7\x2b\xc2\x59"
+			     "\xf0\x64\xfb\x92\x06\x9d\x34\xcb"
+			     "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a"
+			     "\xb1\x48\xdf\x53\xea\x81\x18\x8c"
+			     "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe"
+			     "\x95\x09\xa0\x37\xce\x42\xd9\x70"
+			     "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2"
+			     "\x56\xed\x84\x1b\x8f\x26\xbd\x31"
+			     "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3"
+			     "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15"
+			     "\xac\x20\xb7\x4e\xe5\x59\xf0\x87"
+			     "\x1e\x92\x29\xc0\x34\xcb\x62\xf9"
+			     "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48"
+			     "\xdf\x76\x0d\x81\x18\xaf\x23\xba"
+			     "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c"
+			     "\xc3\x37\xce\x65\xfc\x70\x07\x9e"
+			     "\x12\xa9\x40\xd7\x4b\xe2\x79\x10"
+			     "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f"
+			     "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1"
+			     "\x68\xff\x73\x0a\xa1\x15\xac\x43"
+			     "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5"
+			     "\x29\xc0\x57\xee\x62\xf9\x90\x04"
+			     "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76"
+			     "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8"
+			     "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a"
+			     "\xf1\x65\xfc\x93\x07\x9e\x35\xcc"
+			     "\x40\xd7\x6e\x05\x79\x10\xa7\x1b"
+			     "\xb2\x49\xe0\x54\xeb\x82\x19\x8d"
+			     "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff"
+			     "\x96\x0a\xa1\x38\xcf\x43\xda\x71"
+			     "\x08\x7c\x13\xaa\x1e\xb5\x4c",
+		.psize     = 1023,
+		.digest    = "\x1b\x19\x4d\x8f\xd5\x36\x87\x71"
+			     "\xcf\xca\x30\x85\x9b\xc1\x25\xc7"
+			     "\x00\xcb\x73\x8a\x8e\xd4\xfe\x2b"
+			     "\x1a\xa2\xdc\x2e\x41\xfd\x52\x51"
+			     "\xd2\x21\xae\x2d\xc7\xae\x8c\x40"
+			     "\xb9\xe6\x56\x48\x03\xcd\x88\x6b",
 	},
 };
 
@@ -1147,6 +1557,146 @@ static const struct hash_testvec sha3_512_tv_template[] = {
 				"\xba\x1b\x0d\x8d\xc7\x8c\x08\x63"
 				"\x46\xb5\x33\xb4\x9c\x03\x0d\x99"
 				"\xa2\x7d\xaf\x11\x39\xd6\xe7\x5e",
+		.np	= 2,
+		.tap	= { 28, 28 },
+	}, {
+		.plaintext = "\x08\x9f\x13\xaa\x41\xd8\x4c\xe3"
+			     "\x7a\x11\x85\x1c\xb3\x27\xbe\x55"
+			     "\xec\x60\xf7\x8e\x02\x99\x30\xc7"
+			     "\x3b\xd2\x69\x00\x74\x0b\xa2\x16"
+			     "\xad\x44\xdb\x4f\xe6\x7d\x14\x88"
+			     "\x1f\xb6\x2a\xc1\x58\xef\x63\xfa"
+			     "\x91\x05\x9c\x33\xca\x3e\xd5\x6c"
+			     "\x03\x77\x0e\xa5\x19\xb0\x47\xde"
+			     "\x52\xe9\x80\x17\x8b\x22\xb9\x2d"
+			     "\xc4\x5b\xf2\x66\xfd\x94\x08\x9f"
+			     "\x36\xcd\x41\xd8\x6f\x06\x7a\x11"
+			     "\xa8\x1c\xb3\x4a\xe1\x55\xec\x83"
+			     "\x1a\x8e\x25\xbc\x30\xc7\x5e\xf5"
+			     "\x69\x00\x97\x0b\xa2\x39\xd0\x44"
+			     "\xdb\x72\x09\x7d\x14\xab\x1f\xb6"
+			     "\x4d\xe4\x58\xef\x86\x1d\x91\x28"
+			     "\xbf\x33\xca\x61\xf8\x6c\x03\x9a"
+			     "\x0e\xa5\x3c\xd3\x47\xde\x75\x0c"
+			     "\x80\x17\xae\x22\xb9\x50\xe7\x5b"
+			     "\xf2\x89\x20\x94\x2b\xc2\x36\xcd"
+			     "\x64\xfb\x6f\x06\x9d\x11\xa8\x3f"
+			     "\xd6\x4a\xe1\x78\x0f\x83\x1a\xb1"
+			     "\x25\xbc\x53\xea\x5e\xf5\x8c\x00"
+			     "\x97\x2e\xc5\x39\xd0\x67\xfe\x72"
+			     "\x09\xa0\x14\xab\x42\xd9\x4d\xe4"
+			     "\x7b\x12\x86\x1d\xb4\x28\xbf\x56"
+			     "\xed\x61\xf8\x8f\x03\x9a\x31\xc8"
+			     "\x3c\xd3\x6a\x01\x75\x0c\xa3\x17"
+			     "\xae\x45\xdc\x50\xe7\x7e\x15\x89"
+			     "\x20\xb7\x2b\xc2\x59\xf0\x64\xfb"
+			     "\x92\x06\x9d\x34\xcb\x3f\xd6\x6d"
+			     "\x04\x78\x0f\xa6\x1a\xb1\x48\xdf"
+			     "\x53\xea\x81\x18\x8c\x23\xba\x2e"
+			     "\xc5\x5c\xf3\x67\xfe\x95\x09\xa0"
+			     "\x37\xce\x42\xd9\x70\x07\x7b\x12"
+			     "\xa9\x1d\xb4\x4b\xe2\x56\xed\x84"
+			     "\x1b\x8f\x26\xbd\x31\xc8\x5f\xf6"
+			     "\x6a\x01\x98\x0c\xa3\x3a\xd1\x45"
+			     "\xdc\x73\x0a\x7e\x15\xac\x20\xb7"
+			     "\x4e\xe5\x59\xf0\x87\x1e\x92\x29"
+			     "\xc0\x34\xcb\x62\xf9\x6d\x04\x9b"
+			     "\x0f\xa6\x3d\xd4\x48\xdf\x76\x0d"
+			     "\x81\x18\xaf\x23\xba\x51\xe8\x5c"
+			     "\xf3\x8a\x21\x95\x2c\xc3\x37\xce"
+			     "\x65\xfc\x70\x07\x9e\x12\xa9\x40"
+			     "\xd7\x4b\xe2\x79\x10\x84\x1b\xb2"
+			     "\x26\xbd\x54\xeb\x5f\xf6\x8d\x01"
+			     "\x98\x2f\xc6\x3a\xd1\x68\xff\x73"
+			     "\x0a\xa1\x15\xac\x43\xda\x4e\xe5"
+			     "\x7c\x13\x87\x1e\xb5\x29\xc0\x57"
+			     "\xee\x62\xf9\x90\x04\x9b\x32\xc9"
+			     "\x3d\xd4\x6b\x02\x76\x0d\xa4\x18"
+			     "\xaf\x46\xdd\x51\xe8\x7f\x16\x8a"
+			     "\x21\xb8\x2c\xc3\x5a\xf1\x65\xfc"
+			     "\x93\x07\x9e\x35\xcc\x40\xd7\x6e"
+			     "\x05\x79\x10\xa7\x1b\xb2\x49\xe0"
+			     "\x54\xeb\x82\x19\x8d\x24\xbb\x2f"
+			     "\xc6\x5d\xf4\x68\xff\x96\x0a\xa1"
+			     "\x38\xcf\x43\xda\x71\x08\x7c\x13"
+			     "\xaa\x1e\xb5\x4c\xe3\x57\xee\x85"
+			     "\x1c\x90\x27\xbe\x32\xc9\x60\xf7"
+			     "\x6b\x02\x99\x0d\xa4\x3b\xd2\x46"
+			     "\xdd\x74\x0b\x7f\x16\xad\x21\xb8"
+			     "\x4f\xe6\x5a\xf1\x88\x1f\x93\x2a"
+			     "\xc1\x35\xcc\x63\xfa\x6e\x05\x9c"
+			     "\x10\xa7\x3e\xd5\x49\xe0\x77\x0e"
+			     "\x82\x19\xb0\x24\xbb\x52\xe9\x5d"
+			     "\xf4\x8b\x22\x96\x2d\xc4\x38\xcf"
+			     "\x66\xfd\x71\x08\x9f\x13\xaa\x41"
+			     "\xd8\x4c\xe3\x7a\x11\x85\x1c\xb3"
+			     "\x27\xbe\x55\xec\x60\xf7\x8e\x02"
+			     "\x99\x30\xc7\x3b\xd2\x69\x00\x74"
+			     "\x0b\xa2\x16\xad\x44\xdb\x4f\xe6"
+			     "\x7d\x14\x88\x1f\xb6\x2a\xc1\x58"
+			     "\xef\x63\xfa\x91\x05\x9c\x33\xca"
+			     "\x3e\xd5\x6c\x03\x77\x0e\xa5\x19"
+			     "\xb0\x47\xde\x52\xe9\x80\x17\x8b"
+			     "\x22\xb9\x2d\xc4\x5b\xf2\x66\xfd"
+			     "\x94\x08\x9f\x36\xcd\x41\xd8\x6f"
+			     "\x06\x7a\x11\xa8\x1c\xb3\x4a\xe1"
+			     "\x55\xec\x83\x1a\x8e\x25\xbc\x30"
+			     "\xc7\x5e\xf5\x69\x00\x97\x0b\xa2"
+			     "\x39\xd0\x44\xdb\x72\x09\x7d\x14"
+			     "\xab\x1f\xb6\x4d\xe4\x58\xef\x86"
+			     "\x1d\x91\x28\xbf\x33\xca\x61\xf8"
+			     "\x6c\x03\x9a\x0e\xa5\x3c\xd3\x47"
+			     "\xde\x75\x0c\x80\x17\xae\x22\xb9"
+			     "\x50\xe7\x5b\xf2\x89\x20\x94\x2b"
+			     "\xc2\x36\xcd\x64\xfb\x6f\x06\x9d"
+			     "\x11\xa8\x3f\xd6\x4a\xe1\x78\x0f"
+			     "\x83\x1a\xb1\x25\xbc\x53\xea\x5e"
+			     "\xf5\x8c\x00\x97\x2e\xc5\x39\xd0"
+			     "\x67\xfe\x72\x09\xa0\x14\xab\x42"
+			     "\xd9\x4d\xe4\x7b\x12\x86\x1d\xb4"
+			     "\x28\xbf\x56\xed\x61\xf8\x8f\x03"
+			     "\x9a\x31\xc8\x3c\xd3\x6a\x01\x75"
+			     "\x0c\xa3\x17\xae\x45\xdc\x50\xe7"
+			     "\x7e\x15\x89\x20\xb7\x2b\xc2\x59"
+			     "\xf0\x64\xfb\x92\x06\x9d\x34\xcb"
+			     "\x3f\xd6\x6d\x04\x78\x0f\xa6\x1a"
+			     "\xb1\x48\xdf\x53\xea\x81\x18\x8c"
+			     "\x23\xba\x2e\xc5\x5c\xf3\x67\xfe"
+			     "\x95\x09\xa0\x37\xce\x42\xd9\x70"
+			     "\x07\x7b\x12\xa9\x1d\xb4\x4b\xe2"
+			     "\x56\xed\x84\x1b\x8f\x26\xbd\x31"
+			     "\xc8\x5f\xf6\x6a\x01\x98\x0c\xa3"
+			     "\x3a\xd1\x45\xdc\x73\x0a\x7e\x15"
+			     "\xac\x20\xb7\x4e\xe5\x59\xf0\x87"
+			     "\x1e\x92\x29\xc0\x34\xcb\x62\xf9"
+			     "\x6d\x04\x9b\x0f\xa6\x3d\xd4\x48"
+			     "\xdf\x76\x0d\x81\x18\xaf\x23\xba"
+			     "\x51\xe8\x5c\xf3\x8a\x21\x95\x2c"
+			     "\xc3\x37\xce\x65\xfc\x70\x07\x9e"
+			     "\x12\xa9\x40\xd7\x4b\xe2\x79\x10"
+			     "\x84\x1b\xb2\x26\xbd\x54\xeb\x5f"
+			     "\xf6\x8d\x01\x98\x2f\xc6\x3a\xd1"
+			     "\x68\xff\x73\x0a\xa1\x15\xac\x43"
+			     "\xda\x4e\xe5\x7c\x13\x87\x1e\xb5"
+			     "\x29\xc0\x57\xee\x62\xf9\x90\x04"
+			     "\x9b\x32\xc9\x3d\xd4\x6b\x02\x76"
+			     "\x0d\xa4\x18\xaf\x46\xdd\x51\xe8"
+			     "\x7f\x16\x8a\x21\xb8\x2c\xc3\x5a"
+			     "\xf1\x65\xfc\x93\x07\x9e\x35\xcc"
+			     "\x40\xd7\x6e\x05\x79\x10\xa7\x1b"
+			     "\xb2\x49\xe0\x54\xeb\x82\x19\x8d"
+			     "\x24\xbb\x2f\xc6\x5d\xf4\x68\xff"
+			     "\x96\x0a\xa1\x38\xcf\x43\xda\x71"
+			     "\x08\x7c\x13\xaa\x1e\xb5\x4c",
+		.psize     = 1023,
+		.digest    = "\x59\xda\x30\xe3\x90\xe4\x3d\xde"
+			     "\xf0\xc6\x42\x17\xd7\xb2\x26\x47"
+			     "\x90\x28\xa6\x84\xe8\x49\x7a\x86"
+			     "\xd6\xb8\x9e\xf8\x07\x59\x21\x03"
+			     "\xad\xd2\xed\x48\xa3\xb9\xa5\xf0"
+			     "\xb3\xae\x02\x2b\xb8\xaf\xc3\x3b"
+			     "\xd6\xb0\x8f\xcb\x76\x8b\xa7\x41"
+			     "\x32\xc2\x8e\x50\x91\x86\x90\xfb",
 	},
 };
 
diff --git a/crypto/twofish_common.c b/crypto/twofish_common.c
index 5f62c4f9f6e0..f3a0dd25f871 100644
--- a/crypto/twofish_common.c
+++ b/crypto/twofish_common.c
@@ -24,9 +24,8 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
- * USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
  *
  * This code is a "clean room" implementation, written from the paper
  * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey,
diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c
index ebf7a3efb572..07e62433fbfb 100644
--- a/crypto/twofish_generic.c
+++ b/crypto/twofish_generic.c
@@ -23,9 +23,8 @@
  * GNU General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
- * USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
  *
  * This code is a "clean room" implementation, written from the paper
  * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey,
diff --git a/crypto/xcbc.c b/crypto/xcbc.c
index df90b332554c..25c75af50d3f 100644
--- a/crypto/xcbc.c
+++ b/crypto/xcbc.c
@@ -12,8 +12,7 @@
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  *
  * Author:
  * 	Kazunori Miyazawa <miyazawa@linux-ipv6.org>
diff --git a/drivers/Kconfig b/drivers/Kconfig
index 152744c5ef0f..879dc0604cba 100644
--- a/drivers/Kconfig
+++ b/drivers/Kconfig
@@ -153,6 +153,8 @@ source "drivers/remoteproc/Kconfig"
 
 source "drivers/rpmsg/Kconfig"
 
+source "drivers/soundwire/Kconfig"
+
 source "drivers/soc/Kconfig"
 
 source "drivers/devfreq/Kconfig"
@@ -211,4 +213,10 @@ source "drivers/mux/Kconfig"
 
 source "drivers/opp/Kconfig"
 
+source "drivers/visorbus/Kconfig"
+
+source "drivers/siox/Kconfig"
+
+source "drivers/slimbus/Kconfig"
+
 endmenu
diff --git a/drivers/Makefile b/drivers/Makefile
index e06f7f633f73..24cd47014657 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -16,10 +16,7 @@ obj-$(CONFIG_PINCTRL)		+= pinctrl/
 obj-$(CONFIG_GPIOLIB)		+= gpio/
 obj-y				+= pwm/
 
-obj-$(CONFIG_PCI)		+= pci/
-obj-$(CONFIG_PCI_ENDPOINT)	+= pci/endpoint/
-# PCI dwc controller drivers
-obj-y				+= pci/dwc/
+obj-y				+= pci/
 
 obj-$(CONFIG_PARISC)		+= parisc/
 obj-$(CONFIG_RAPIDIO)		+= rapidio/
@@ -87,6 +84,7 @@ obj-$(CONFIG_MTD)		+= mtd/
 obj-$(CONFIG_SPI)		+= spi/
 obj-$(CONFIG_SPMI)		+= spmi/
 obj-$(CONFIG_HSI)		+= hsi/
+obj-$(CONFIG_SLIMBUS)		+= slimbus/
 obj-y				+= net/
 obj-$(CONFIG_ATM)		+= atm/
 obj-$(CONFIG_FUSION)		+= message/
@@ -157,6 +155,7 @@ obj-$(CONFIG_MAILBOX)		+= mailbox/
 obj-$(CONFIG_HWSPINLOCK)	+= hwspinlock/
 obj-$(CONFIG_REMOTEPROC)	+= remoteproc/
 obj-$(CONFIG_RPMSG)		+= rpmsg/
+obj-$(CONFIG_SOUNDWIRE)		+= soundwire/
 
 # Virtualization drivers
 obj-$(CONFIG_VIRT_DRIVERS)	+= virt/
@@ -184,3 +183,5 @@ obj-$(CONFIG_FPGA)		+= fpga/
 obj-$(CONFIG_FSI)		+= fsi/
 obj-$(CONFIG_TEE)		+= tee/
 obj-$(CONFIG_MULTIPLEXER)	+= mux/
+obj-$(CONFIG_UNISYS_VISORBUS)	+= visorbus/
+obj-$(CONFIG_SIOX)		+= siox/
diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
index 46505396869e..f505e9a01b2d 100644
--- a/drivers/acpi/Kconfig
+++ b/drivers/acpi/Kconfig
@@ -79,7 +79,12 @@ config ACPI_DEBUGGER_USER
 endif
 
 config ACPI_SPCR_TABLE
-	bool
+	bool "ACPI Serial Port Console Redirection Support"
+	default y if X86
+	help
+	  Enable support for Serial Port Console Redirection (SPCR) Table.
+	  This table provides information about the configuration of the
+	  earlycon console.
 
 config ACPI_LPIT
 	bool
@@ -361,22 +366,6 @@ config ACPI_PCI_SLOT
 	  i.e., segment/bus/device/function tuples, with physical slots in
 	  the system.  If you are unsure, say N.
 
-config X86_PM_TIMER
-	bool "Power Management Timer Support" if EXPERT
-	depends on X86
-	default y
-	help
-	  The Power Management Timer is available on all ACPI-capable,
-	  in most cases even if ACPI is unusable or blacklisted.
-
-	  This timing source is not affected by power management features
-	  like aggressive processor idling, throttling, frequency and/or
-	  voltage scaling, unlike the commonly used Time Stamp Counter
-	  (TSC) timing source.
-
-	  You should nearly always say Y here because many modern
-	  systems require this timer. 
-
 config ACPI_CONTAINER
 	bool "Container and Module Devices"
 	default (ACPI_HOTPLUG_MEMORY || ACPI_HOTPLUG_CPU)
@@ -564,3 +553,19 @@ config TPS68470_PMIC_OPREGION
 	  using this, are probed.
 
 endif	# ACPI
+
+config X86_PM_TIMER
+	bool "Power Management Timer Support" if EXPERT
+	depends on X86 && (ACPI || JAILHOUSE_GUEST)
+	default y
+	help
+	  The Power Management Timer is available on all ACPI-capable,
+	  in most cases even if ACPI is unusable or blacklisted.
+
+	  This timing source is not affected by power management features
+	  like aggressive processor idling, throttling, frequency and/or
+	  voltage scaling, unlike the commonly used Time Stamp Counter
+	  (TSC) timing source.
+
+	  You should nearly always say Y here because many modern
+	  systems require this timer.
diff --git a/drivers/acpi/acpi_dbg.c b/drivers/acpi/acpi_dbg.c
index 3ec05aa1a903..f21c99ec46ee 100644
--- a/drivers/acpi/acpi_dbg.c
+++ b/drivers/acpi/acpi_dbg.c
@@ -718,15 +718,15 @@ static ssize_t acpi_aml_write(struct file *file, const char __user *buf,
 	return size > 0 ? size : ret;
 }
 
-static unsigned int acpi_aml_poll(struct file *file, poll_table *wait)
+static __poll_t acpi_aml_poll(struct file *file, poll_table *wait)
 {
-	int masks = 0;
+	__poll_t masks = 0;
 
 	poll_wait(file, &acpi_aml_io.wait, wait);
 	if (acpi_aml_user_readable())
-		masks |= POLLIN | POLLRDNORM;
+		masks |= EPOLLIN | EPOLLRDNORM;
 	if (acpi_aml_user_writable())
-		masks |= POLLOUT | POLLWRNORM;
+		masks |= EPOLLOUT | EPOLLWRNORM;
 
 	return masks;
 }
diff --git a/drivers/acpi/acpi_lpit.c b/drivers/acpi/acpi_lpit.c
index e94e478dd18b..cf4fc0161164 100644
--- a/drivers/acpi/acpi_lpit.c
+++ b/drivers/acpi/acpi_lpit.c
@@ -100,6 +100,7 @@ int lpit_read_residency_count_address(u64 *address)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(lpit_read_residency_count_address);
 
 static void lpit_update_residency(struct lpit_residency_info *info,
 				 struct acpi_lpit_native *lpit_native)
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index 7f2b02cc8ea1..2bcffec8dbf0 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -427,6 +427,142 @@ static int register_device_clock(struct acpi_device *adev,
 	return 0;
 }
 
+struct lpss_device_links {
+	const char *supplier_hid;
+	const char *supplier_uid;
+	const char *consumer_hid;
+	const char *consumer_uid;
+	u32 flags;
+};
+
+/*
+ * The _DEP method is used to identify dependencies but instead of creating
+ * device links for every handle in _DEP, only links in the following list are
+ * created. That is necessary because, in the general case, _DEP can refer to
+ * devices that might not have drivers, or that are on different buses, or where
+ * the supplier is not enumerated until after the consumer is probed.
+ */
+static const struct lpss_device_links lpss_device_links[] = {
+	{"808622C1", "7", "80860F14", "3", DL_FLAG_PM_RUNTIME},
+};
+
+static bool hid_uid_match(const char *hid1, const char *uid1,
+			  const char *hid2, const char *uid2)
+{
+	return !strcmp(hid1, hid2) && uid1 && uid2 && !strcmp(uid1, uid2);
+}
+
+static bool acpi_lpss_is_supplier(struct acpi_device *adev,
+				  const struct lpss_device_links *link)
+{
+	return hid_uid_match(acpi_device_hid(adev), acpi_device_uid(adev),
+			     link->supplier_hid, link->supplier_uid);
+}
+
+static bool acpi_lpss_is_consumer(struct acpi_device *adev,
+				  const struct lpss_device_links *link)
+{
+	return hid_uid_match(acpi_device_hid(adev), acpi_device_uid(adev),
+			     link->consumer_hid, link->consumer_uid);
+}
+
+struct hid_uid {
+	const char *hid;
+	const char *uid;
+};
+
+static int match_hid_uid(struct device *dev, void *data)
+{
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+	struct hid_uid *id = data;
+
+	if (!adev)
+		return 0;
+
+	return hid_uid_match(acpi_device_hid(adev), acpi_device_uid(adev),
+			     id->hid, id->uid);
+}
+
+static struct device *acpi_lpss_find_device(const char *hid, const char *uid)
+{
+	struct hid_uid data = {
+		.hid = hid,
+		.uid = uid,
+	};
+
+	return bus_find_device(&platform_bus_type, NULL, &data, match_hid_uid);
+}
+
+static bool acpi_lpss_dep(struct acpi_device *adev, acpi_handle handle)
+{
+	struct acpi_handle_list dep_devices;
+	acpi_status status;
+	int i;
+
+	if (!acpi_has_method(adev->handle, "_DEP"))
+		return false;
+
+	status = acpi_evaluate_reference(adev->handle, "_DEP", NULL,
+					 &dep_devices);
+	if (ACPI_FAILURE(status)) {
+		dev_dbg(&adev->dev, "Failed to evaluate _DEP.\n");
+		return false;
+	}
+
+	for (i = 0; i < dep_devices.count; i++) {
+		if (dep_devices.handles[i] == handle)
+			return true;
+	}
+
+	return false;
+}
+
+static void acpi_lpss_link_consumer(struct device *dev1,
+				    const struct lpss_device_links *link)
+{
+	struct device *dev2;
+
+	dev2 = acpi_lpss_find_device(link->consumer_hid, link->consumer_uid);
+	if (!dev2)
+		return;
+
+	if (acpi_lpss_dep(ACPI_COMPANION(dev2), ACPI_HANDLE(dev1)))
+		device_link_add(dev2, dev1, link->flags);
+
+	put_device(dev2);
+}
+
+static void acpi_lpss_link_supplier(struct device *dev1,
+				    const struct lpss_device_links *link)
+{
+	struct device *dev2;
+
+	dev2 = acpi_lpss_find_device(link->supplier_hid, link->supplier_uid);
+	if (!dev2)
+		return;
+
+	if (acpi_lpss_dep(ACPI_COMPANION(dev1), ACPI_HANDLE(dev2)))
+		device_link_add(dev1, dev2, link->flags);
+
+	put_device(dev2);
+}
+
+static void acpi_lpss_create_device_links(struct acpi_device *adev,
+					  struct platform_device *pdev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(lpss_device_links); i++) {
+		const struct lpss_device_links *link = &lpss_device_links[i];
+
+		if (acpi_lpss_is_supplier(adev, link))
+			acpi_lpss_link_consumer(&pdev->dev, link);
+
+		if (acpi_lpss_is_consumer(adev, link))
+			acpi_lpss_link_supplier(&pdev->dev, link);
+	}
+}
+
 static int acpi_lpss_create_device(struct acpi_device *adev,
 				   const struct acpi_device_id *id)
 {
@@ -465,6 +601,8 @@ static int acpi_lpss_create_device(struct acpi_device *adev,
 	acpi_dev_free_resource_list(&resource_list);
 
 	if (!pdata->mmio_base) {
+		/* Avoid acpi_bus_attach() instantiating a pdev for this dev. */
+		adev->pnp.type.platform_id = 0;
 		/* Skip the device, but continue the namespace scan. */
 		ret = 0;
 		goto err_out;
@@ -500,6 +638,7 @@ static int acpi_lpss_create_device(struct acpi_device *adev,
 	adev->driver_data = pdata;
 	pdev = acpi_create_platform_device(adev, dev_desc->properties);
 	if (!IS_ERR_OR_NULL(pdev)) {
+		acpi_lpss_create_device_links(adev, pdev);
 		return 1;
 	}
 
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index 0972ec0e2eb8..76fb96966f7b 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -53,7 +53,7 @@ MODULE_AUTHOR("Bruno Ducrot");
 MODULE_DESCRIPTION("ACPI Video Driver");
 MODULE_LICENSE("GPL");
 
-static bool brightness_switch_enabled = 1;
+static bool brightness_switch_enabled = true;
 module_param(brightness_switch_enabled, bool, 0644);
 
 /*
@@ -80,8 +80,8 @@ MODULE_PARM_DESC(report_key_events,
 static bool device_id_scheme = false;
 module_param(device_id_scheme, bool, 0444);
 
-static bool only_lcd = false;
-module_param(only_lcd, bool, 0444);
+static int only_lcd = -1;
+module_param(only_lcd, int, 0444);
 
 static int register_count;
 static DEFINE_MUTEX(register_count_mutex);
@@ -2136,6 +2136,16 @@ int acpi_video_register(void)
 		goto leave;
 	}
 
+	/*
+	 * We're seeing a lot of bogus backlight interfaces on newer machines
+	 * without a LCD such as desktops, servers and HDMI sticks. Checking
+	 * the lcd flag fixes this, so enable this on any machines which are
+	 * win8 ready (where we also prefer the native backlight driver, so
+	 * normally the acpi_video code should not register there anyways).
+	 */
+	if (only_lcd == -1)
+		only_lcd = acpi_osi_is_win8();
+
 	dmi_check_system(video_dmi_table);
 
 	ret = acpi_bus_register_driver(&acpi_video_bus);
diff --git a/drivers/acpi/acpi_watchdog.c b/drivers/acpi/acpi_watchdog.c
index 11b113f8e367..ebb626ffb5fa 100644
--- a/drivers/acpi/acpi_watchdog.c
+++ b/drivers/acpi/acpi_watchdog.c
@@ -74,10 +74,10 @@ void __init acpi_watchdog_init(void)
 		res.start = gas->address;
 		if (gas->space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
 			res.flags = IORESOURCE_MEM;
-			res.end = res.start + ALIGN(gas->access_width, 4);
+			res.end = res.start + ALIGN(gas->access_width, 4) - 1;
 		} else if (gas->space_id == ACPI_ADR_SPACE_SYSTEM_IO) {
 			res.flags = IORESOURCE_IO;
-			res.end = res.start + gas->access_width;
+			res.end = res.start + gas->access_width - 1;
 		} else {
 			pr_warn("Unsupported address space: %u\n",
 				gas->space_id);
diff --git a/drivers/acpi/acpica/acapps.h b/drivers/acpi/acpica/acapps.h
index 7a1a68b5ac5c..e65478593f9a 100644
--- a/drivers/acpi/acpica/acapps.h
+++ b/drivers/acpi/acpica/acapps.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -51,7 +51,7 @@
 /* Common info for tool signons */
 
 #define ACPICA_NAME                 "Intel ACPI Component Architecture"
-#define ACPICA_COPYRIGHT            "Copyright (c) 2000 - 2017 Intel Corporation"
+#define ACPICA_COPYRIGHT            "Copyright (c) 2000 - 2018 Intel Corporation"
 
 #if ACPI_MACHINE_WIDTH == 64
 #define ACPI_WIDTH          " (64-bit version)"
@@ -80,6 +80,9 @@
 	prefix, ACPICA_COPYRIGHT, \
 	prefix
 
+#define ACPI_COMMON_BUILD_TIME \
+	"Build date/time: %s %s\n", __DATE__, __TIME__
+
 /* Macros for usage messages */
 
 #define ACPI_USAGE_HEADER(usage) \
diff --git a/drivers/acpi/acpica/accommon.h b/drivers/acpi/acpica/accommon.h
index 49bf47ca5477..c349ffdf5557 100644
--- a/drivers/acpi/acpica/accommon.h
+++ b/drivers/acpi/acpica/accommon.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acconvert.h b/drivers/acpi/acpica/acconvert.h
index c84223b60b35..ce6e8db83e27 100644
--- a/drivers/acpi/acpica/acconvert.h
+++ b/drivers/acpi/acpica/acconvert.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acdebug.h b/drivers/acpi/acpica/acdebug.h
index 71743e5252f5..8b2cca5a717b 100644
--- a/drivers/acpi/acpica/acdebug.h
+++ b/drivers/acpi/acpica/acdebug.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -222,6 +222,10 @@ ACPI_DBR_DEPENDENT_RETURN_VOID(void
 void
 acpi_db_execute(char *name, char **args, acpi_object_type *types, u32 flags);
 
+void
+acpi_db_create_execution_thread(char *method_name_arg,
+				char **arguments, acpi_object_type *types);
+
 void
 acpi_db_create_execution_threads(char *num_threads_arg,
 				 char *num_loops_arg, char *method_name_arg);
diff --git a/drivers/acpi/acpica/acdispat.h b/drivers/acpi/acpica/acdispat.h
index f8f3a6e74128..fab590bc5fd3 100644
--- a/drivers/acpi/acpica/acdispat.h
+++ b/drivers/acpi/acpica/acdispat.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acevents.h b/drivers/acpi/acpica/acevents.h
index a2adfd42f85c..1b0269f6ac2d 100644
--- a/drivers/acpi/acpica/acevents.h
+++ b/drivers/acpi/acpica/acevents.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h
index 95eed442703f..27f322b2fed1 100644
--- a/drivers/acpi/acpica/acglobal.h
+++ b/drivers/acpi/acpica/acglobal.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -46,7 +46,7 @@
 
 /*****************************************************************************
  *
- * Globals related to the ACPI tables
+ * Globals related to the incoming ACPI tables
  *
  ****************************************************************************/
 
@@ -87,7 +87,7 @@ ACPI_GLOBAL(u8, acpi_gbl_integer_nybble_width);
 
 /*****************************************************************************
  *
- * Mutual exclusion within ACPICA subsystem
+ * Mutual exclusion within the ACPICA subsystem
  *
  ****************************************************************************/
 
@@ -167,7 +167,7 @@ ACPI_GLOBAL(u8, acpi_gbl_next_owner_id_offset);
 
 ACPI_INIT_GLOBAL(u8, acpi_gbl_namespace_initialized, FALSE);
 
-/* Misc */
+/* Miscellaneous */
 
 ACPI_GLOBAL(u32, acpi_gbl_original_mode);
 ACPI_GLOBAL(u32, acpi_gbl_ns_lookup_count);
@@ -191,10 +191,9 @@ extern const char acpi_gbl_lower_hex_digits[];
 extern const char acpi_gbl_upper_hex_digits[];
 extern const struct acpi_opcode_info acpi_gbl_aml_op_info[AML_NUM_OPCODES];
 
-#ifdef ACPI_DBG_TRACK_ALLOCATIONS
-
 /* Lists for tracking memory allocations (debug only) */
 
+#ifdef ACPI_DBG_TRACK_ALLOCATIONS
 ACPI_GLOBAL(struct acpi_memory_list *, acpi_gbl_global_list);
 ACPI_GLOBAL(struct acpi_memory_list *, acpi_gbl_ns_node_list);
 ACPI_GLOBAL(u8, acpi_gbl_display_final_mem_stats);
@@ -203,7 +202,7 @@ ACPI_GLOBAL(u8, acpi_gbl_disable_mem_tracking);
 
 /*****************************************************************************
  *
- * Namespace globals
+ * ACPI Namespace
  *
  ****************************************************************************/
 
@@ -234,15 +233,20 @@ ACPI_INIT_GLOBAL(u32, acpi_gbl_nesting_level, 0);
 
 /*****************************************************************************
  *
- * Interpreter globals
+ * Interpreter/Parser globals
  *
  ****************************************************************************/
 
-ACPI_GLOBAL(struct acpi_thread_state *, acpi_gbl_current_walk_list);
-
 /* Control method single step flag */
 
 ACPI_GLOBAL(u8, acpi_gbl_cm_single_step);
+ACPI_GLOBAL(struct acpi_thread_state *, acpi_gbl_current_walk_list);
+ACPI_INIT_GLOBAL(union acpi_parse_object, *acpi_gbl_current_scope, NULL);
+
+/* ASL/ASL+ converter */
+
+ACPI_INIT_GLOBAL(u8, acpi_gbl_capture_comments, FALSE);
+ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_last_list_head, NULL);
 
 /*****************************************************************************
  *
@@ -252,7 +256,6 @@ ACPI_GLOBAL(u8, acpi_gbl_cm_single_step);
 
 extern struct acpi_bit_register_info
     acpi_gbl_bit_register_info[ACPI_NUM_BITREG];
-
 ACPI_GLOBAL(u8, acpi_gbl_sleep_type_a);
 ACPI_GLOBAL(u8, acpi_gbl_sleep_type_b);
 
@@ -263,7 +266,6 @@ ACPI_GLOBAL(u8, acpi_gbl_sleep_type_b);
  ****************************************************************************/
 
 #if (!ACPI_REDUCED_HARDWARE)
-
 ACPI_GLOBAL(u8, acpi_gbl_all_gpes_initialized);
 ACPI_GLOBAL(struct acpi_gpe_xrupt_info *, acpi_gbl_gpe_xrupt_list_head);
 ACPI_GLOBAL(struct acpi_gpe_block_info *,
@@ -272,10 +274,8 @@ ACPI_GLOBAL(acpi_gbl_event_handler, acpi_gbl_global_event_handler);
 ACPI_GLOBAL(void *, acpi_gbl_global_event_handler_context);
 ACPI_GLOBAL(struct acpi_fixed_event_handler,
 	    acpi_gbl_fixed_event_handlers[ACPI_NUM_FIXED_EVENTS]);
-
 extern struct acpi_fixed_event_info
     acpi_gbl_fixed_event_info[ACPI_NUM_FIXED_EVENTS];
-
 #endif				/* !ACPI_REDUCED_HARDWARE */
 
 /*****************************************************************************
@@ -291,14 +291,14 @@ ACPI_GLOBAL(u32, acpi_gpe_count);
 ACPI_GLOBAL(u32, acpi_sci_count);
 ACPI_GLOBAL(u32, acpi_fixed_event_count[ACPI_NUM_FIXED_EVENTS]);
 
-/* Support for dynamic control method tracing mechanism */
+/* Dynamic control method tracing mechanism */
 
 ACPI_GLOBAL(u32, acpi_gbl_original_dbg_level);
 ACPI_GLOBAL(u32, acpi_gbl_original_dbg_layer);
 
 /*****************************************************************************
  *
- * Debugger and Disassembler globals
+ * Debugger and Disassembler
  *
  ****************************************************************************/
 
@@ -326,7 +326,6 @@ ACPI_GLOBAL(struct acpi_external_file *, acpi_gbl_external_file_list);
 #endif
 
 #ifdef ACPI_DEBUGGER
-
 ACPI_INIT_GLOBAL(u8, acpi_gbl_abort_method, FALSE);
 ACPI_INIT_GLOBAL(acpi_thread_id, acpi_gbl_db_thread_id, ACPI_INVALID_THREAD_ID);
 
@@ -340,7 +339,6 @@ ACPI_GLOBAL(u32, acpi_gbl_db_console_debug_level);
 ACPI_GLOBAL(struct acpi_namespace_node *, acpi_gbl_db_scope_node);
 ACPI_GLOBAL(u8, acpi_gbl_db_terminate_loop);
 ACPI_GLOBAL(u8, acpi_gbl_db_threads_terminated);
-
 ACPI_GLOBAL(char *, acpi_gbl_db_args[ACPI_DEBUGGER_MAX_ARGS]);
 ACPI_GLOBAL(acpi_object_type, acpi_gbl_db_arg_types[ACPI_DEBUGGER_MAX_ARGS]);
 
@@ -350,32 +348,33 @@ ACPI_GLOBAL(char, acpi_gbl_db_parsed_buf[ACPI_DB_LINE_BUFFER_SIZE]);
 ACPI_GLOBAL(char, acpi_gbl_db_scope_buf[ACPI_DB_LINE_BUFFER_SIZE]);
 ACPI_GLOBAL(char, acpi_gbl_db_debug_filename[ACPI_DB_LINE_BUFFER_SIZE]);
 
-/*
- * Statistic globals
- */
+/* Statistics globals */
+
 ACPI_GLOBAL(u16, acpi_gbl_obj_type_count[ACPI_TOTAL_TYPES]);
 ACPI_GLOBAL(u16, acpi_gbl_node_type_count[ACPI_TOTAL_TYPES]);
 ACPI_GLOBAL(u16, acpi_gbl_obj_type_count_misc);
 ACPI_GLOBAL(u16, acpi_gbl_node_type_count_misc);
 ACPI_GLOBAL(u32, acpi_gbl_num_nodes);
 ACPI_GLOBAL(u32, acpi_gbl_num_objects);
-
 #endif				/* ACPI_DEBUGGER */
 
 #if defined (ACPI_DISASSEMBLER) || defined (ACPI_ASL_COMPILER)
-
 ACPI_GLOBAL(const char, *acpi_gbl_pld_panel_list[]);
 ACPI_GLOBAL(const char, *acpi_gbl_pld_vertical_position_list[]);
 ACPI_GLOBAL(const char, *acpi_gbl_pld_horizontal_position_list[]);
 ACPI_GLOBAL(const char, *acpi_gbl_pld_shape_list[]);
-
 ACPI_INIT_GLOBAL(u8, acpi_gbl_disasm_flag, FALSE);
-
 #endif
 
-/*
- * Meant for the -ca option.
- */
+/*****************************************************************************
+ *
+ * ACPICA application-specific globals
+ *
+ ****************************************************************************/
+
+/* ASL-to-ASL+ conversion utility (implemented within the iASL compiler) */
+
+#ifdef ACPI_ASL_COMPILER
 ACPI_INIT_GLOBAL(char *, acpi_gbl_current_inline_comment, NULL);
 ACPI_INIT_GLOBAL(char *, acpi_gbl_current_end_node_comment, NULL);
 ACPI_INIT_GLOBAL(char *, acpi_gbl_current_open_brace_comment, NULL);
@@ -386,23 +385,18 @@ ACPI_INIT_GLOBAL(char *, acpi_gbl_current_filename, NULL);
 ACPI_INIT_GLOBAL(char *, acpi_gbl_current_parent_filename, NULL);
 ACPI_INIT_GLOBAL(char *, acpi_gbl_current_include_filename, NULL);
 
-ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_last_list_head, NULL);
-
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_def_blk_comment_list_head,
 		 NULL);
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_def_blk_comment_list_tail,
 		 NULL);
-
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_reg_comment_list_head,
 		 NULL);
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_reg_comment_list_tail,
 		 NULL);
-
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_inc_comment_list_head,
 		 NULL);
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_inc_comment_list_tail,
 		 NULL);
-
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_end_blk_comment_list_head,
 		 NULL);
 ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_end_blk_comment_list_tail,
@@ -410,30 +404,18 @@ ACPI_INIT_GLOBAL(struct acpi_comment_node, *acpi_gbl_end_blk_comment_list_tail,
 
 ACPI_INIT_GLOBAL(struct acpi_comment_addr_node,
 		 *acpi_gbl_comment_addr_list_head, NULL);
-
-ACPI_INIT_GLOBAL(union acpi_parse_object, *acpi_gbl_current_scope, NULL);
-
 ACPI_INIT_GLOBAL(struct acpi_file_node, *acpi_gbl_file_tree_root, NULL);
 
 ACPI_GLOBAL(acpi_cache_t *, acpi_gbl_reg_comment_cache);
 ACPI_GLOBAL(acpi_cache_t *, acpi_gbl_comment_addr_cache);
 ACPI_GLOBAL(acpi_cache_t *, acpi_gbl_file_cache);
 
-ACPI_INIT_GLOBAL(u8, gbl_capture_comments, FALSE);
-
 ACPI_INIT_GLOBAL(u8, acpi_gbl_debug_asl_conversion, FALSE);
 ACPI_INIT_GLOBAL(ACPI_FILE, acpi_gbl_conv_debug_file, NULL);
-
 ACPI_GLOBAL(char, acpi_gbl_table_sig[4]);
-
-/*****************************************************************************
- *
- * Application globals
- *
- ****************************************************************************/
+#endif
 
 #ifdef ACPI_APPLICATION
-
 ACPI_INIT_GLOBAL(ACPI_FILE, acpi_gbl_debug_file, NULL);
 ACPI_INIT_GLOBAL(ACPI_FILE, acpi_gbl_output_file, NULL);
 ACPI_INIT_GLOBAL(u8, acpi_gbl_debug_timeout, FALSE);
@@ -442,16 +424,6 @@ ACPI_INIT_GLOBAL(u8, acpi_gbl_debug_timeout, FALSE);
 
 ACPI_GLOBAL(acpi_spinlock, acpi_gbl_print_lock);	/* For print buffer */
 ACPI_GLOBAL(char, acpi_gbl_print_buffer[1024]);
-
 #endif				/* ACPI_APPLICATION */
 
-/*****************************************************************************
- *
- * Info/help support
- *
- ****************************************************************************/
-
-extern const struct ah_predefined_name asl_predefined_info[];
-extern const struct ah_device_id asl_device_ids[];
-
 #endif				/* __ACGLOBAL_H__ */
diff --git a/drivers/acpi/acpica/achware.h b/drivers/acpi/acpica/achware.h
index cd722d8edacb..3569aa3bf5ee 100644
--- a/drivers/acpi/acpica/achware.h
+++ b/drivers/acpi/acpica/achware.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acinterp.h b/drivers/acpi/acpica/acinterp.h
index 29555c8789a3..744374ab9285 100644
--- a/drivers/acpi/acpica/acinterp.h
+++ b/drivers/acpi/acpica/acinterp.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h
index 0d45b8bb1678..3ba3ff0f1c04 100644
--- a/drivers/acpi/acpica/aclocal.h
+++ b/drivers/acpi/acpica/aclocal.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -622,7 +622,7 @@ struct acpi_control_state {
 	union acpi_parse_object *predicate_op;
 	u8 *aml_predicate_start;	/* Start of if/while predicate */
 	u8 *package_end;	/* End of if/while block */
-	u32 loop_count;		/* While() loop counter */
+	u64 loop_timeout;	/* While() loop timeout */
 };
 
 /*
@@ -1218,16 +1218,17 @@ struct acpi_db_method_info {
 	acpi_object_type *types;
 
 	/*
-	 * Arguments to be passed to method for the command
-	 * Threads -
-	 *   the Number of threads, ID of current thread and
-	 *   Index of current thread inside all them created.
+	 * Arguments to be passed to method for the commands Threads and
+	 * Background. Note, ACPI specifies a maximum of 7 arguments (0 - 6).
+	 *
+	 * For the Threads command, the Number of threads, ID of current
+	 * thread and Index of current thread inside all them created.
 	 */
 	char init_args;
 #ifdef ACPI_DEBUGGER
-	acpi_object_type arg_types[4];
+	acpi_object_type arg_types[ACPI_METHOD_NUM_ARGS];
 #endif
-	char *arguments[4];
+	char *arguments[ACPI_METHOD_NUM_ARGS];
 	char num_threads_str[11];
 	char id_of_thread_str[11];
 	char index_of_thread_str[11];
diff --git a/drivers/acpi/acpica/acmacros.h b/drivers/acpi/acpica/acmacros.h
index c7f0c96cc00f..6463340c4522 100644
--- a/drivers/acpi/acpica/acmacros.h
+++ b/drivers/acpi/acpica/acmacros.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -455,7 +455,7 @@
  * the plist contains a set of parens to allow variable-length lists.
  * These macros are used for both the debug and non-debug versions of the code.
  */
-#define ACPI_ERROR_NAMESPACE(s, e)          acpi_ut_namespace_error (AE_INFO, s, e);
+#define ACPI_ERROR_NAMESPACE(s, p, e)       acpi_ut_prefixed_namespace_error (AE_INFO, s, p, e);
 #define ACPI_ERROR_METHOD(s, n, p, e)       acpi_ut_method_error (AE_INFO, s, n, p, e);
 #define ACPI_WARN_PREDEFINED(plist)         acpi_ut_predefined_warning plist
 #define ACPI_INFO_PREDEFINED(plist)         acpi_ut_predefined_info plist
diff --git a/drivers/acpi/acpica/acnamesp.h b/drivers/acpi/acpica/acnamesp.h
index 54a0c51b3e37..6c8f364fe2fc 100644
--- a/drivers/acpi/acpica/acnamesp.h
+++ b/drivers/acpi/acpica/acnamesp.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -289,6 +289,9 @@ acpi_ns_build_normalized_path(struct acpi_namespace_node *node,
 char *acpi_ns_get_normalized_pathname(struct acpi_namespace_node *node,
 				      u8 no_trailing);
 
+char *acpi_ns_build_prefixed_pathname(union acpi_generic_state *prefix_scope,
+				      const char *internal_path);
+
 char *acpi_ns_name_of_current_scope(struct acpi_walk_state *walk_state);
 
 acpi_status
diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h
index 5226146190bf..a1f4d3f385c8 100644
--- a/drivers/acpi/acpica/acobject.h
+++ b/drivers/acpi/acpica/acobject.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acopcode.h b/drivers/acpi/acpica/acopcode.h
index cbd59a302679..36c2c5825986 100644
--- a/drivers/acpi/acpica/acopcode.h
+++ b/drivers/acpi/acpica/acopcode.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acparser.h b/drivers/acpi/acpica/acparser.h
index c23c47328060..e25634951d03 100644
--- a/drivers/acpi/acpica/acparser.h
+++ b/drivers/acpi/acpica/acparser.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acpredef.h b/drivers/acpi/acpica/acpredef.h
index cdfcad8eb74c..7c27bcee6ac7 100644
--- a/drivers/acpi/acpica/acpredef.h
+++ b/drivers/acpi/acpica/acpredef.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acresrc.h b/drivers/acpi/acpica/acresrc.h
index 438f3098a093..20f36949928a 100644
--- a/drivers/acpi/acpica/acresrc.h
+++ b/drivers/acpi/acpica/acresrc.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acstruct.h b/drivers/acpi/acpica/acstruct.h
index 62134bdbeda6..0338ac32f9c6 100644
--- a/drivers/acpi/acpica/acstruct.h
+++ b/drivers/acpi/acpica/acstruct.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/actables.h b/drivers/acpi/acpica/actables.h
index 84a3ceb6e384..15b23414245a 100644
--- a/drivers/acpi/acpica/actables.h
+++ b/drivers/acpi/acpica/actables.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h
index 83b75e9db7ef..00d21d2f766e 100644
--- a/drivers/acpi/acpica/acutils.h
+++ b/drivers/acpi/acpica/acutils.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -118,9 +118,6 @@ extern const char *acpi_gbl_ptyp_decode[];
 #ifndef ACPI_MSG_ERROR
 #define ACPI_MSG_ERROR          "ACPI Error: "
 #endif
-#ifndef ACPI_MSG_EXCEPTION
-#define ACPI_MSG_EXCEPTION      "ACPI Exception: "
-#endif
 #ifndef ACPI_MSG_WARNING
 #define ACPI_MSG_WARNING        "ACPI Warning: "
 #endif
@@ -129,10 +126,10 @@ extern const char *acpi_gbl_ptyp_decode[];
 #endif
 
 #ifndef ACPI_MSG_BIOS_ERROR
-#define ACPI_MSG_BIOS_ERROR     "ACPI BIOS Error (bug): "
+#define ACPI_MSG_BIOS_ERROR     "Firmware Error (ACPI): "
 #endif
 #ifndef ACPI_MSG_BIOS_WARNING
-#define ACPI_MSG_BIOS_WARNING   "ACPI BIOS Warning (bug): "
+#define ACPI_MSG_BIOS_WARNING   "Firmware Warning (ACPI): "
 #endif
 
 /*
@@ -233,10 +230,10 @@ u64 acpi_ut_implicit_strtoul64(char *string);
  */
 acpi_status acpi_ut_init_globals(void);
 
-#if defined(ACPI_DEBUG_OUTPUT) || defined(ACPI_DEBUGGER)
-
 const char *acpi_ut_get_mutex_name(u32 mutex_id);
 
+#if defined(ACPI_DEBUG_OUTPUT) || defined(ACPI_DEBUGGER)
+
 const char *acpi_ut_get_notify_name(u32 notify_value, acpi_object_type type);
 #endif
 
@@ -641,9 +638,11 @@ void ut_convert_backslashes(char *pathname);
 
 void acpi_ut_repair_name(char *name);
 
-#if defined (ACPI_DEBUGGER) || defined (ACPI_APPLICATION)
+#if defined (ACPI_DEBUGGER) || defined (ACPI_APPLICATION) || defined (ACPI_DEBUG_OUTPUT)
 u8 acpi_ut_safe_strcpy(char *dest, acpi_size dest_size, char *source);
 
+void acpi_ut_safe_strncpy(char *dest, char *source, acpi_size dest_size);
+
 u8 acpi_ut_safe_strcat(char *dest, acpi_size dest_size, char *source);
 
 u8
@@ -737,9 +736,11 @@ acpi_ut_predefined_bios_error(const char *module_name,
 			      u8 node_flags, const char *format, ...);
 
 void
-acpi_ut_namespace_error(const char *module_name,
-			u32 line_number,
-			const char *internal_name, acpi_status lookup_status);
+acpi_ut_prefixed_namespace_error(const char *module_name,
+				 u32 line_number,
+				 union acpi_generic_state *prefix_scope,
+				 const char *internal_name,
+				 acpi_status lookup_status);
 
 void
 acpi_ut_method_error(const char *module_name,
diff --git a/drivers/acpi/acpica/amlcode.h b/drivers/acpi/acpica/amlcode.h
index f54dc5a34bdc..b0e9492a6297 100644
--- a/drivers/acpi/acpica/amlcode.h
+++ b/drivers/acpi/acpica/amlcode.h
@@ -7,7 +7,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/amlresrc.h b/drivers/acpi/acpica/amlresrc.h
index 1236e9a414e4..b680c229ddd5 100644
--- a/drivers/acpi/acpica/amlresrc.h
+++ b/drivers/acpi/acpica/amlresrc.h
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbcmds.c b/drivers/acpi/acpica/dbcmds.c
index 5984b90eb590..4112c85f2aab 100644
--- a/drivers/acpi/acpica/dbcmds.c
+++ b/drivers/acpi/acpica/dbcmds.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbconvert.c b/drivers/acpi/acpica/dbconvert.c
index 32d546f0db2f..27236a6c51ff 100644
--- a/drivers/acpi/acpica/dbconvert.c
+++ b/drivers/acpi/acpica/dbconvert.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbdisply.c b/drivers/acpi/acpica/dbdisply.c
index 5a606eac0c22..7df920cda77d 100644
--- a/drivers/acpi/acpica/dbdisply.c
+++ b/drivers/acpi/acpica/dbdisply.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbexec.c b/drivers/acpi/acpica/dbexec.c
index 3b30319752f0..8ad9e6d9e54b 100644
--- a/drivers/acpi/acpica/dbexec.c
+++ b/drivers/acpi/acpica/dbexec.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -67,6 +67,8 @@ static acpi_status
 acpi_db_execution_walk(acpi_handle obj_handle,
 		       u32 nesting_level, void *context, void **return_value);
 
+static void ACPI_SYSTEM_XFACE acpi_db_single_execution_thread(void *context);
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_db_delete_objects
@@ -229,7 +231,7 @@ static acpi_status acpi_db_execute_setup(struct acpi_db_method_info *info)
 
 	ACPI_FUNCTION_NAME(db_execute_setup);
 
-	/* Catenate the current scope to the supplied name */
+	/* Concatenate the current scope to the supplied name */
 
 	info->pathname[0] = 0;
 	if ((info->name[0] != '\\') && (info->name[0] != '/')) {
@@ -609,6 +611,112 @@ static void ACPI_SYSTEM_XFACE acpi_db_method_thread(void *context)
 	}
 }
 
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_db_single_execution_thread
+ *
+ * PARAMETERS:  context                 - Method info struct
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Create one thread and execute a method
+ *
+ ******************************************************************************/
+
+static void ACPI_SYSTEM_XFACE acpi_db_single_execution_thread(void *context)
+{
+	struct acpi_db_method_info *info = context;
+	acpi_status status;
+	struct acpi_buffer return_obj;
+
+	acpi_os_printf("\n");
+
+	status = acpi_db_execute_method(info, &return_obj);
+	if (ACPI_FAILURE(status)) {
+		acpi_os_printf("%s During evaluation of %s\n",
+			       acpi_format_exception(status), info->pathname);
+		return;
+	}
+
+	/* Display a return object, if any */
+
+	if (return_obj.length) {
+		acpi_os_printf("Evaluation of %s returned object %p, "
+			       "external buffer length %X\n",
+			       acpi_gbl_db_method_info.pathname,
+			       return_obj.pointer, (u32)return_obj.length);
+
+		acpi_db_dump_external_object(return_obj.pointer, 1);
+	}
+
+	acpi_os_printf("\nBackground thread completed\n%c ",
+		       ACPI_DEBUGGER_COMMAND_PROMPT);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_db_create_execution_thread
+ *
+ * PARAMETERS:  method_name_arg         - Control method to execute
+ *              arguments               - Array of arguments to the method
+ *              types                   - Corresponding array of object types
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Create a single thread to evaluate a namespace object. Handles
+ *              arguments passed on command line for control methods.
+ *
+ ******************************************************************************/
+
+void
+acpi_db_create_execution_thread(char *method_name_arg,
+				char **arguments, acpi_object_type *types)
+{
+	acpi_status status;
+	u32 i;
+
+	memset(&acpi_gbl_db_method_info, 0, sizeof(struct acpi_db_method_info));
+	acpi_gbl_db_method_info.name = method_name_arg;
+	acpi_gbl_db_method_info.init_args = 1;
+	acpi_gbl_db_method_info.args = acpi_gbl_db_method_info.arguments;
+	acpi_gbl_db_method_info.types = acpi_gbl_db_method_info.arg_types;
+
+	/* Setup method arguments, up to 7 (0-6) */
+
+	for (i = 0; (i < ACPI_METHOD_NUM_ARGS) && *arguments; i++) {
+		acpi_gbl_db_method_info.arguments[i] = *arguments;
+		arguments++;
+
+		acpi_gbl_db_method_info.arg_types[i] = *types;
+		types++;
+	}
+
+	status = acpi_db_execute_setup(&acpi_gbl_db_method_info);
+	if (ACPI_FAILURE(status)) {
+		return;
+	}
+
+	/* Get the NS node, determines existence also */
+
+	status = acpi_get_handle(NULL, acpi_gbl_db_method_info.pathname,
+				 &acpi_gbl_db_method_info.method);
+	if (ACPI_FAILURE(status)) {
+		acpi_os_printf("%s Could not get handle for %s\n",
+			       acpi_format_exception(status),
+			       acpi_gbl_db_method_info.pathname);
+		return;
+	}
+
+	status = acpi_os_execute(OSL_DEBUGGER_EXEC_THREAD,
+				 acpi_db_single_execution_thread,
+				 &acpi_gbl_db_method_info);
+	if (ACPI_FAILURE(status)) {
+		return;
+	}
+
+	acpi_os_printf("\nBackground thread started\n");
+}
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_db_create_execution_threads
diff --git a/drivers/acpi/acpica/dbfileio.c b/drivers/acpi/acpica/dbfileio.c
index 4d81ea291d93..084bb332f8e2 100644
--- a/drivers/acpi/acpica/dbfileio.c
+++ b/drivers/acpi/acpica/dbfileio.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -99,8 +99,8 @@ void acpi_db_open_debug_file(char *name)
 	}
 
 	acpi_os_printf("Debug output file %s opened\n", name);
-	strncpy(acpi_gbl_db_debug_filename, name,
-		sizeof(acpi_gbl_db_debug_filename));
+	acpi_ut_safe_strncpy(acpi_gbl_db_debug_filename, name,
+			     sizeof(acpi_gbl_db_debug_filename));
 	acpi_gbl_db_output_to_file = TRUE;
 }
 #endif
diff --git a/drivers/acpi/acpica/dbhistry.c b/drivers/acpi/acpica/dbhistry.c
index 7d08974c64c2..55c0f2742339 100644
--- a/drivers/acpi/acpica/dbhistry.c
+++ b/drivers/acpi/acpica/dbhistry.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbinput.c b/drivers/acpi/acpica/dbinput.c
index 2626d79db064..f7c661e06f37 100644
--- a/drivers/acpi/acpica/dbinput.c
+++ b/drivers/acpi/acpica/dbinput.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -136,6 +136,7 @@ enum acpi_ex_debugger_commands {
 	CMD_UNLOAD,
 
 	CMD_TERMINATE,
+	CMD_BACKGROUND,
 	CMD_THREADS,
 
 	CMD_TEST,
@@ -212,6 +213,7 @@ static const struct acpi_db_command_info acpi_gbl_db_commands[] = {
 	{"UNLOAD", 1},
 
 	{"TERMINATE", 0},
+	{"BACKGROUND", 1},
 	{"THREADS", 3},
 
 	{"TEST", 1},
@@ -222,9 +224,56 @@ static const struct acpi_db_command_info acpi_gbl_db_commands[] = {
 /*
  * Help for all debugger commands. First argument is the number of lines
  * of help to output for the command.
+ *
+ * Note: Some commands are not supported by the kernel-level version of
+ * the debugger.
  */
 static const struct acpi_db_command_help acpi_gbl_db_command_help[] = {
-	{0, "\nGeneral-Purpose Commands:", "\n"},
+	{0, "\nNamespace Access:", "\n"},
+	{1, "  Businfo", "Display system bus info\n"},
+	{1, "  Disassemble <Method>", "Disassemble a control method\n"},
+	{1, "  Find <AcpiName> (? is wildcard)",
+	 "Find ACPI name(s) with wildcards\n"},
+	{1, "  Integrity", "Validate namespace integrity\n"},
+	{1, "  Methods", "Display list of loaded control methods\n"},
+	{1, "  Namespace [Object] [Depth]",
+	 "Display loaded namespace tree/subtree\n"},
+	{1, "  Notify <Object> <Value>", "Send a notification on Object\n"},
+	{1, "  Objects [ObjectType]",
+	 "Display summary of all objects or just given type\n"},
+	{1, "  Owner <OwnerId> [Depth]",
+	 "Display loaded namespace by object owner\n"},
+	{1, "  Paths", "Display full pathnames of namespace objects\n"},
+	{1, "  Predefined", "Check all predefined names\n"},
+	{1, "  Prefix [<Namepath>]", "Set or Get current execution prefix\n"},
+	{1, "  References <Addr>", "Find all references to object at addr\n"},
+	{1, "  Resources [DeviceName]",
+	 "Display Device resources (no arg = all devices)\n"},
+	{1, "  Set N <NamedObject> <Value>", "Set value for named integer\n"},
+	{1, "  Template <Object>", "Format/dump a Buffer/ResourceTemplate\n"},
+	{1, "  Type <Object>", "Display object type\n"},
+
+	{0, "\nControl Method Execution:", "\n"},
+	{1, "  Evaluate <Namepath> [Arguments]",
+	 "Evaluate object or control method\n"},
+	{1, "  Execute <Namepath> [Arguments]", "Synonym for Evaluate\n"},
+#ifdef ACPI_APPLICATION
+	{1, "  Background <Namepath> [Arguments]",
+	 "Evaluate object/method in a separate thread\n"},
+	{1, "  Thread <Threads><Loops><NamePath>",
+	 "Spawn threads to execute method(s)\n"},
+#endif
+	{1, "  Debug <Namepath> [Arguments]", "Single-Step a control method\n"},
+	{7, "  [Arguments] formats:", "Control method argument formats\n"},
+	{1, "     Hex Integer", "Integer\n"},
+	{1, "     \"Ascii String\"", "String\n"},
+	{1, "     (Hex Byte List)", "Buffer\n"},
+	{1, "         (01 42 7A BF)", "Buffer example (4 bytes)\n"},
+	{1, "     [Package Element List]", "Package\n"},
+	{1, "         [0x01 0x1234 \"string\"]",
+	 "Package example (3 elements)\n"},
+
+	{0, "\nMiscellaneous:", "\n"},
 	{1, "  Allocations", "Display list of current memory allocations\n"},
 	{2, "  Dump <Address>|<Namepath>", "\n"},
 	{0, "       [Byte|Word|Dword|Qword]",
@@ -248,46 +297,30 @@ static const struct acpi_db_command_help acpi_gbl_db_command_help[] = {
 	{1, "     Stack", "Display CPU stack usage\n"},
 	{1, "     Tables", "Info about current ACPI table(s)\n"},
 	{1, "  Tables", "Display info about loaded ACPI tables\n"},
+#ifdef ACPI_APPLICATION
+	{1, "  Terminate", "Delete namespace and all internal objects\n"},
+#endif
 	{1, "  ! <CommandNumber>", "Execute command from history buffer\n"},
 	{1, "  !!", "Execute last command again\n"},
 
-	{0, "\nNamespace Access Commands:", "\n"},
-	{1, "  Businfo", "Display system bus info\n"},
-	{1, "  Disassemble <Method>", "Disassemble a control method\n"},
-	{1, "  Find <AcpiName> (? is wildcard)",
-	 "Find ACPI name(s) with wildcards\n"},
-	{1, "  Integrity", "Validate namespace integrity\n"},
-	{1, "  Methods", "Display list of loaded control methods\n"},
-	{1, "  Namespace [Object] [Depth]",
-	 "Display loaded namespace tree/subtree\n"},
-	{1, "  Notify <Object> <Value>", "Send a notification on Object\n"},
-	{1, "  Objects [ObjectType]",
-	 "Display summary of all objects or just given type\n"},
-	{1, "  Owner <OwnerId> [Depth]",
-	 "Display loaded namespace by object owner\n"},
-	{1, "  Paths", "Display full pathnames of namespace objects\n"},
-	{1, "  Predefined", "Check all predefined names\n"},
-	{1, "  Prefix [<Namepath>]", "Set or Get current execution prefix\n"},
-	{1, "  References <Addr>", "Find all references to object at addr\n"},
-	{1, "  Resources [DeviceName]",
-	 "Display Device resources (no arg = all devices)\n"},
-	{1, "  Set N <NamedObject> <Value>", "Set value for named integer\n"},
-	{1, "  Template <Object>", "Format/dump a Buffer/ResourceTemplate\n"},
-	{1, "  Type <Object>", "Display object type\n"},
+	{0, "\nMethod and Namespace Debugging:", "\n"},
+	{5, "  Trace <State> [<Namepath>] [Once]",
+	 "Trace control method execution\n"},
+	{1, "     Enable", "Enable all messages\n"},
+	{1, "     Disable", "Disable tracing\n"},
+	{1, "     Method", "Enable method execution messages\n"},
+	{1, "     Opcode", "Enable opcode execution messages\n"},
+	{3, "  Test <TestName>", "Invoke a debug test\n"},
+	{1, "     Objects", "Read/write/compare all namespace data objects\n"},
+	{1, "     Predefined",
+	 "Validate all ACPI predefined names (_STA, etc.)\n"},
+	{1, "  Execute predefined",
+	 "Execute all predefined (public) methods\n"},
 
-	{0, "\nControl Method Execution Commands:", "\n"},
+	{0, "\nControl Method Single-Step Execution:", "\n"},
 	{1, "  Arguments (or Args)", "Display method arguments\n"},
 	{1, "  Breakpoint <AmlOffset>", "Set an AML execution breakpoint\n"},
 	{1, "  Call", "Run to next control method invocation\n"},
-	{1, "  Debug <Namepath> [Arguments]", "Single Step a control method\n"},
-	{6, "  Evaluate", "Synonym for Execute\n"},
-	{5, "  Execute <Namepath> [Arguments]", "Execute control method\n"},
-	{1, "     Hex Integer", "Integer method argument\n"},
-	{1, "     \"Ascii String\"", "String method argument\n"},
-	{1, "     (Hex Byte List)", "Buffer method argument\n"},
-	{1, "     [Package Element List]", "Package method argument\n"},
-	{5, "  Execute predefined",
-	 "Execute all predefined (public) methods\n"},
 	{1, "  Go", "Allow method to run to completion\n"},
 	{1, "  Information", "Display info about the current method\n"},
 	{1, "  Into", "Step into (not over) a method call\n"},
@@ -296,41 +329,24 @@ static const struct acpi_db_command_help acpi_gbl_db_command_help[] = {
 	{1, "  Results", "Display method result stack\n"},
 	{1, "  Set <A|L> <#> <Value>", "Set method data (Arguments/Locals)\n"},
 	{1, "  Stop", "Terminate control method\n"},
-	{5, "  Trace <State> [<Namepath>] [Once]",
-	 "Trace control method execution\n"},
-	{1, "     Enable", "Enable all messages\n"},
-	{1, "     Disable", "Disable tracing\n"},
-	{1, "     Method", "Enable method execution messages\n"},
-	{1, "     Opcode", "Enable opcode execution messages\n"},
 	{1, "  Tree", "Display control method calling tree\n"},
 	{1, "  <Enter>", "Single step next AML opcode (over calls)\n"},
 
 #ifdef ACPI_APPLICATION
-	{0, "\nHardware Simulation Commands:", "\n"},
-	{1, "  EnableAcpi", "Enable ACPI (hardware) mode\n"},
-	{1, "  Event <F|G> <Value>", "Generate AcpiEvent (Fixed/GPE)\n"},
-	{1, "  Gpe <GpeNum> [GpeBlockDevice]", "Simulate a GPE\n"},
-	{1, "  Gpes", "Display info on all GPE devices\n"},
-	{1, "  Sci", "Generate an SCI\n"},
-	{1, "  Sleep [SleepState]", "Simulate sleep/wake sequence(s) (0-5)\n"},
-
-	{0, "\nFile I/O Commands:", "\n"},
+	{0, "\nFile Operations:", "\n"},
 	{1, "  Close", "Close debug output file\n"},
 	{1, "  Load <Input Filename>", "Load ACPI table from a file\n"},
 	{1, "  Open <Output Filename>", "Open a file for debug output\n"},
 	{1, "  Unload <Namepath>",
 	 "Unload an ACPI table via namespace object\n"},
 
-	{0, "\nUser Space Commands:", "\n"},
-	{1, "  Terminate", "Delete namespace and all internal objects\n"},
-	{1, "  Thread <Threads><Loops><NamePath>",
-	 "Spawn threads to execute method(s)\n"},
-
-	{0, "\nDebug Test Commands:", "\n"},
-	{3, "  Test <TestName>", "Invoke a debug test\n"},
-	{1, "     Objects", "Read/write/compare all namespace data objects\n"},
-	{1, "     Predefined",
-	 "Execute all ACPI predefined names (_STA, etc.)\n"},
+	{0, "\nHardware Simulation:", "\n"},
+	{1, "  EnableAcpi", "Enable ACPI (hardware) mode\n"},
+	{1, "  Event <F|G> <Value>", "Generate AcpiEvent (Fixed/GPE)\n"},
+	{1, "  Gpe <GpeNum> [GpeBlockDevice]", "Simulate a GPE\n"},
+	{1, "  Gpes", "Display info on all GPE devices\n"},
+	{1, "  Sci", "Generate an SCI\n"},
+	{1, "  Sleep [SleepState]", "Simulate sleep/wake sequence(s) (0-5)\n"},
 #endif
 	{0, NULL, NULL}
 };
@@ -442,11 +458,15 @@ static void acpi_db_display_help(char *command)
 
 		/* No argument to help, display help for all commands */
 
+		acpi_os_printf("\nSummary of AML Debugger Commands\n\n");
+
 		while (next->invocation) {
 			acpi_os_printf("%-38s%s", next->invocation,
 				       next->description);
 			next++;
 		}
+		acpi_os_printf("\n");
+
 	} else {
 		/* Display help for all commands that match the subtring */
 
@@ -1087,6 +1107,13 @@ acpi_db_command_dispatch(char *input_buffer,
 		/*  acpi_initialize (NULL); */
 		break;
 
+	case CMD_BACKGROUND:
+
+		acpi_db_create_execution_thread(acpi_gbl_db_args[1],
+						&acpi_gbl_db_args[2],
+						&acpi_gbl_db_arg_types[2]);
+		break;
+
 	case CMD_THREADS:
 
 		acpi_db_create_execution_threads(acpi_gbl_db_args[1],
diff --git a/drivers/acpi/acpica/dbmethod.c b/drivers/acpi/acpica/dbmethod.c
index df62c9245efc..2cda0bff6f2c 100644
--- a/drivers/acpi/acpica/dbmethod.c
+++ b/drivers/acpi/acpica/dbmethod.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbnames.c b/drivers/acpi/acpica/dbnames.c
index 8c207c772517..8796fc1e0360 100644
--- a/drivers/acpi/acpica/dbnames.c
+++ b/drivers/acpi/acpica/dbnames.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbobject.c b/drivers/acpi/acpica/dbobject.c
index e7b415c20aa8..d2063cbab39a 100644
--- a/drivers/acpi/acpica/dbobject.c
+++ b/drivers/acpi/acpica/dbobject.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbstats.c b/drivers/acpi/acpica/dbstats.c
index 99fb0160b8fb..d6aaef54e369 100644
--- a/drivers/acpi/acpica/dbstats.c
+++ b/drivers/acpi/acpica/dbstats.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbtest.c b/drivers/acpi/acpica/dbtest.c
index c6bee6143266..56e446b89d18 100644
--- a/drivers/acpi/acpica/dbtest.c
+++ b/drivers/acpi/acpica/dbtest.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbutils.c b/drivers/acpi/acpica/dbutils.c
index bfa972b64171..cd40854ee9be 100644
--- a/drivers/acpi/acpica/dbutils.c
+++ b/drivers/acpi/acpica/dbutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dbxface.c b/drivers/acpi/acpica/dbxface.c
index b6985323e7eb..77bbfa97cf91 100644
--- a/drivers/acpi/acpica/dbxface.c
+++ b/drivers/acpi/acpica/dbxface.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsargs.c b/drivers/acpi/acpica/dsargs.c
index 2873455c986d..04a9f60e7ad1 100644
--- a/drivers/acpi/acpica/dsargs.c
+++ b/drivers/acpi/acpica/dsargs.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dscontrol.c b/drivers/acpi/acpica/dscontrol.c
index f470e81b0499..606697e741a5 100644
--- a/drivers/acpi/acpica/dscontrol.c
+++ b/drivers/acpi/acpica/dscontrol.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -118,6 +118,8 @@ acpi_ds_exec_begin_control_op(struct acpi_walk_state *walk_state,
 		control_state->control.package_end =
 		    walk_state->parser_state.pkg_end;
 		control_state->control.opcode = op->common.aml_opcode;
+		control_state->control.loop_timeout = acpi_os_get_timer() +
+		    (u64)(acpi_gbl_max_loop_iterations * ACPI_100NSEC_PER_SEC);
 
 		/* Push the control state on this walk's control stack */
 
@@ -206,15 +208,15 @@ acpi_ds_exec_end_control_op(struct acpi_walk_state *walk_state,
 			/* Predicate was true, the body of the loop was just executed */
 
 			/*
-			 * This loop counter mechanism allows the interpreter to escape
-			 * possibly infinite loops. This can occur in poorly written AML
-			 * when the hardware does not respond within a while loop and the
-			 * loop does not implement a timeout.
+			 * This infinite loop detection mechanism allows the interpreter
+			 * to escape possibly infinite loops. This can occur in poorly
+			 * written AML when the hardware does not respond within a while
+			 * loop and the loop does not implement a timeout.
 			 */
-			control_state->control.loop_count++;
-			if (control_state->control.loop_count >
-			    acpi_gbl_max_loop_iterations) {
-				status = AE_AML_INFINITE_LOOP;
+			if (ACPI_TIME_AFTER(acpi_os_get_timer(),
+					    control_state->control.
+					    loop_timeout)) {
+				status = AE_AML_LOOP_TIMEOUT;
 				break;
 			}
 
diff --git a/drivers/acpi/acpica/dsdebug.c b/drivers/acpi/acpica/dsdebug.c
index d1f457eda980..14ec52eba408 100644
--- a/drivers/acpi/acpica/dsdebug.c
+++ b/drivers/acpi/acpica/dsdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsfield.c b/drivers/acpi/acpica/dsfield.c
index 7bcf5f5ea029..95ea639a9424 100644
--- a/drivers/acpi/acpica/dsfield.c
+++ b/drivers/acpi/acpica/dsfield.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -209,7 +209,8 @@ acpi_ds_create_buffer_field(union acpi_parse_object *op,
 					ACPI_IMODE_LOAD_PASS1, flags,
 					walk_state, &node);
 		if (ACPI_FAILURE(status)) {
-			ACPI_ERROR_NAMESPACE(arg->common.value.string, status);
+			ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+					     arg->common.value.string, status);
 			return_ACPI_STATUS(status);
 		}
 	}
@@ -383,7 +384,9 @@ acpi_ds_get_field_names(struct acpi_create_field_info *info,
 							walk_state,
 							&info->connection_node);
 				if (ACPI_FAILURE(status)) {
-					ACPI_ERROR_NAMESPACE(child->common.
+					ACPI_ERROR_NAMESPACE(walk_state->
+							     scope_info,
+							     child->common.
 							     value.name,
 							     status);
 					return_ACPI_STATUS(status);
@@ -402,7 +405,8 @@ acpi_ds_get_field_names(struct acpi_create_field_info *info,
 						ACPI_NS_DONT_OPEN_SCOPE,
 						walk_state, &info->field_node);
 			if (ACPI_FAILURE(status)) {
-				ACPI_ERROR_NAMESPACE((char *)&arg->named.name,
+				ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+						     (char *)&arg->named.name,
 						     status);
 				return_ACPI_STATUS(status);
 			} else {
@@ -498,7 +502,8 @@ acpi_ds_create_field(union acpi_parse_object *op,
 							&region_node);
 #endif
 		if (ACPI_FAILURE(status)) {
-			ACPI_ERROR_NAMESPACE(arg->common.value.name, status);
+			ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+					     arg->common.value.name, status);
 			return_ACPI_STATUS(status);
 		}
 	}
@@ -618,7 +623,8 @@ acpi_ds_init_field_objects(union acpi_parse_object *op,
 						ACPI_IMODE_LOAD_PASS1, flags,
 						walk_state, &node);
 			if (ACPI_FAILURE(status)) {
-				ACPI_ERROR_NAMESPACE((char *)&arg->named.name,
+				ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+						     (char *)&arg->named.name,
 						     status);
 				if (status != AE_ALREADY_EXISTS) {
 					return_ACPI_STATUS(status);
@@ -681,7 +687,8 @@ acpi_ds_create_bank_field(union acpi_parse_object *op,
 							&region_node);
 #endif
 		if (ACPI_FAILURE(status)) {
-			ACPI_ERROR_NAMESPACE(arg->common.value.name, status);
+			ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+					     arg->common.value.name, status);
 			return_ACPI_STATUS(status);
 		}
 	}
@@ -695,7 +702,8 @@ acpi_ds_create_bank_field(union acpi_parse_object *op,
 			   ACPI_NS_SEARCH_PARENT, walk_state,
 			   &info.register_node);
 	if (ACPI_FAILURE(status)) {
-		ACPI_ERROR_NAMESPACE(arg->common.value.string, status);
+		ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+				     arg->common.value.string, status);
 		return_ACPI_STATUS(status);
 	}
 
@@ -765,7 +773,8 @@ acpi_ds_create_index_field(union acpi_parse_object *op,
 			   ACPI_NS_SEARCH_PARENT, walk_state,
 			   &info.register_node);
 	if (ACPI_FAILURE(status)) {
-		ACPI_ERROR_NAMESPACE(arg->common.value.string, status);
+		ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+				     arg->common.value.string, status);
 		return_ACPI_STATUS(status);
 	}
 
@@ -778,7 +787,8 @@ acpi_ds_create_index_field(union acpi_parse_object *op,
 			   ACPI_NS_SEARCH_PARENT, walk_state,
 			   &info.data_register_node);
 	if (ACPI_FAILURE(status)) {
-		ACPI_ERROR_NAMESPACE(arg->common.value.string, status);
+		ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+				     arg->common.value.string, status);
 		return_ACPI_STATUS(status);
 	}
 
diff --git a/drivers/acpi/acpica/dsinit.c b/drivers/acpi/acpica/dsinit.c
index b1842dd4edf7..946ff2e130d9 100644
--- a/drivers/acpi/acpica/dsinit.c
+++ b/drivers/acpi/acpica/dsinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c
index d7fc36917c67..b9c460c2d763 100644
--- a/drivers/acpi/acpica/dsmethod.c
+++ b/drivers/acpi/acpica/dsmethod.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsmthdat.c b/drivers/acpi/acpica/dsmthdat.c
index 27a7de95f7b0..157f1645d91a 100644
--- a/drivers/acpi/acpica/dsmthdat.c
+++ b/drivers/acpi/acpica/dsmthdat.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dsobject.c b/drivers/acpi/acpica/dsobject.c
index 82448551781b..4fa3400a95ba 100644
--- a/drivers/acpi/acpica/dsobject.c
+++ b/drivers/acpi/acpica/dsobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -112,7 +112,9 @@ acpi_ds_build_internal_object(struct acpi_walk_state *walk_state,
 							 acpi_namespace_node,
 							 &(op->common.node)));
 				if (ACPI_FAILURE(status)) {
-					ACPI_ERROR_NAMESPACE(op->common.value.
+					ACPI_ERROR_NAMESPACE(walk_state->
+							     scope_info,
+							     op->common.value.
 							     string, status);
 					return_ACPI_STATUS(status);
 				}
diff --git a/drivers/acpi/acpica/dsopcode.c b/drivers/acpi/acpica/dsopcode.c
index 0336df7ac47d..0181cd317751 100644
--- a/drivers/acpi/acpica/dsopcode.c
+++ b/drivers/acpi/acpica/dsopcode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dspkginit.c b/drivers/acpi/acpica/dspkginit.c
index 6d487edfe2de..902bee78036c 100644
--- a/drivers/acpi/acpica/dspkginit.c
+++ b/drivers/acpi/acpica/dspkginit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -297,8 +297,10 @@ acpi_ds_init_package_element(u8 object_type,
 {
 	union acpi_operand_object **element_ptr;
 
+	ACPI_FUNCTION_TRACE(ds_init_package_element);
+
 	if (!source_object) {
-		return (AE_OK);
+		return_ACPI_STATUS(AE_OK);
 	}
 
 	/*
@@ -329,7 +331,7 @@ acpi_ds_init_package_element(u8 object_type,
 		source_object->package.flags |= AOPOBJ_DATA_VALID;
 	}
 
-	return (AE_OK);
+	return_ACPI_STATUS(AE_OK);
 }
 
 /*******************************************************************************
@@ -352,6 +354,7 @@ acpi_ds_resolve_package_element(union acpi_operand_object **element_ptr)
 	union acpi_generic_state scope_info;
 	union acpi_operand_object *element = *element_ptr;
 	struct acpi_namespace_node *resolved_node;
+	struct acpi_namespace_node *original_node;
 	char *external_path = NULL;
 	acpi_object_type type;
 
@@ -441,6 +444,7 @@ acpi_ds_resolve_package_element(union acpi_operand_object **element_ptr)
 	 * will remain as named references. This behavior is not described
 	 * in the ACPI spec, but it appears to be an oversight.
 	 */
+	original_node = resolved_node;
 	status = acpi_ex_resolve_node_to_value(&resolved_node, NULL);
 	if (ACPI_FAILURE(status)) {
 		return_VOID;
@@ -468,26 +472,27 @@ acpi_ds_resolve_package_element(union acpi_operand_object **element_ptr)
 		 */
 	case ACPI_TYPE_DEVICE:
 	case ACPI_TYPE_THERMAL:
-
-		/* TBD: This may not be necesssary */
-
-		acpi_ut_add_reference(resolved_node->object);
+	case ACPI_TYPE_METHOD:
 		break;
 
 	case ACPI_TYPE_MUTEX:
-	case ACPI_TYPE_METHOD:
 	case ACPI_TYPE_POWER:
 	case ACPI_TYPE_PROCESSOR:
 	case ACPI_TYPE_EVENT:
 	case ACPI_TYPE_REGION:
 
+		/* acpi_ex_resolve_node_to_value gave these an extra reference */
+
+		acpi_ut_remove_reference(original_node->object);
 		break;
 
 	default:
 		/*
 		 * For all other types - the node was resolved to an actual
-		 * operand object with a value, return the object
+		 * operand object with a value, return the object. Remove
+		 * a reference on the existing object.
 		 */
+		acpi_ut_remove_reference(element);
 		*element_ptr = (union acpi_operand_object *)resolved_node;
 		break;
 	}
diff --git a/drivers/acpi/acpica/dsutils.c b/drivers/acpi/acpica/dsutils.c
index 0dabd9b95684..a4ce0b4a55a6 100644
--- a/drivers/acpi/acpica/dsutils.c
+++ b/drivers/acpi/acpica/dsutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -583,7 +583,8 @@ acpi_ds_create_operand(struct acpi_walk_state *walk_state,
 			}
 
 			if (ACPI_FAILURE(status)) {
-				ACPI_ERROR_NAMESPACE(name_string, status);
+				ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+						     name_string, status);
 			}
 		}
 
diff --git a/drivers/acpi/acpica/dswexec.c b/drivers/acpi/acpica/dswexec.c
index 22f45d090733..2c07d220a50f 100644
--- a/drivers/acpi/acpica/dswexec.c
+++ b/drivers/acpi/acpica/dswexec.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dswload.c b/drivers/acpi/acpica/dswload.c
index eaa859a89702..fa4ef9229e17 100644
--- a/drivers/acpi/acpica/dswload.c
+++ b/drivers/acpi/acpica/dswload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -207,7 +207,8 @@ acpi_ds_load1_begin_op(struct acpi_walk_state *walk_state,
 		}
 #endif
 		if (ACPI_FAILURE(status)) {
-			ACPI_ERROR_NAMESPACE(path, status);
+			ACPI_ERROR_NAMESPACE(walk_state->scope_info, path,
+					     status);
 			return_ACPI_STATUS(status);
 		}
 
@@ -375,7 +376,8 @@ acpi_ds_load1_begin_op(struct acpi_walk_state *walk_state,
 			}
 
 			if (ACPI_FAILURE(status)) {
-				ACPI_ERROR_NAMESPACE(path, status);
+				ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+						     path, status);
 				return_ACPI_STATUS(status);
 			}
 		}
diff --git a/drivers/acpi/acpica/dswload2.c b/drivers/acpi/acpica/dswload2.c
index aad83ef5a4ec..3b1313ba60d0 100644
--- a/drivers/acpi/acpica/dswload2.c
+++ b/drivers/acpi/acpica/dswload2.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -184,11 +184,14 @@ acpi_ds_load2_begin_op(struct acpi_walk_state *walk_state,
 				if (status == AE_NOT_FOUND) {
 					status = AE_OK;
 				} else {
-					ACPI_ERROR_NAMESPACE(buffer_ptr,
+					ACPI_ERROR_NAMESPACE(walk_state->
+							     scope_info,
+							     buffer_ptr,
 							     status);
 				}
 #else
-				ACPI_ERROR_NAMESPACE(buffer_ptr, status);
+				ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+						     buffer_ptr, status);
 #endif
 				return_ACPI_STATUS(status);
 			}
@@ -343,7 +346,8 @@ acpi_ds_load2_begin_op(struct acpi_walk_state *walk_state,
 	}
 
 	if (ACPI_FAILURE(status)) {
-		ACPI_ERROR_NAMESPACE(buffer_ptr, status);
+		ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+				     buffer_ptr, status);
 		return_ACPI_STATUS(status);
 	}
 
@@ -719,7 +723,8 @@ acpi_status acpi_ds_load2_end_op(struct acpi_walk_state *walk_state)
 			 */
 			op->common.node = new_node;
 		} else {
-			ACPI_ERROR_NAMESPACE(arg->common.value.string, status);
+			ACPI_ERROR_NAMESPACE(walk_state->scope_info,
+					     arg->common.value.string, status);
 		}
 		break;
 
diff --git a/drivers/acpi/acpica/dswscope.c b/drivers/acpi/acpica/dswscope.c
index 3e081983d2ee..8b5c3613c060 100644
--- a/drivers/acpi/acpica/dswscope.c
+++ b/drivers/acpi/acpica/dswscope.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/dswstate.c b/drivers/acpi/acpica/dswstate.c
index da111a1f5bfb..ee002d17526e 100644
--- a/drivers/acpi/acpica/dswstate.c
+++ b/drivers/acpi/acpica/dswstate.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evevent.c b/drivers/acpi/acpica/evevent.c
index d3b6b314fa50..4b2b0b44a16b 100644
--- a/drivers/acpi/acpica/evevent.c
+++ b/drivers/acpi/acpica/evevent.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evglock.c b/drivers/acpi/acpica/evglock.c
index 0ce33b0f430c..012b80de1501 100644
--- a/drivers/acpi/acpica/evglock.c
+++ b/drivers/acpi/acpica/evglock.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpe.c b/drivers/acpi/acpica/evgpe.c
index 263d8fc4a9e2..410a3907c051 100644
--- a/drivers/acpi/acpica/evgpe.c
+++ b/drivers/acpi/acpica/evgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpeblk.c b/drivers/acpi/acpica/evgpeblk.c
index 3a3cb8624f41..7ce756cc28ab 100644
--- a/drivers/acpi/acpica/evgpeblk.c
+++ b/drivers/acpi/acpica/evgpeblk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpeinit.c b/drivers/acpi/acpica/evgpeinit.c
index 8649c6242478..8ad4816c9950 100644
--- a/drivers/acpi/acpica/evgpeinit.c
+++ b/drivers/acpi/acpica/evgpeinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evgpeutil.c b/drivers/acpi/acpica/evgpeutil.c
index c8adb400330a..729a8960a3af 100644
--- a/drivers/acpi/acpica/evgpeutil.c
+++ b/drivers/acpi/acpica/evgpeutil.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evhandler.c b/drivers/acpi/acpica/evhandler.c
index 2db61ef1b4a3..20fb51c06b8d 100644
--- a/drivers/acpi/acpica/evhandler.c
+++ b/drivers/acpi/acpica/evhandler.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evmisc.c b/drivers/acpi/acpica/evmisc.c
index 4f6bb3f016ab..40d0b1f541a0 100644
--- a/drivers/acpi/acpica/evmisc.c
+++ b/drivers/acpi/acpica/evmisc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c
index 28b447ff92df..de196c8e3f30 100644
--- a/drivers/acpi/acpica/evregion.c
+++ b/drivers/acpi/acpica/evregion.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -298,6 +298,16 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj,
 		ACPI_EXCEPTION((AE_INFO, status, "Returned by Handler for [%s]",
 				acpi_ut_get_region_name(region_obj->region.
 							space_id)));
+
+		/*
+		 * Special case for an EC timeout. These are seen so frequently
+		 * that an additional error message is helpful
+		 */
+		if ((region_obj->region.space_id == ACPI_ADR_SPACE_EC) &&
+		    (status == AE_TIME)) {
+			ACPI_ERROR((AE_INFO,
+				    "Timeout from EC hardware or EC device driver"));
+		}
 	}
 
 	if (!(handler_desc->address_space.handler_flags &
diff --git a/drivers/acpi/acpica/evrgnini.c b/drivers/acpi/acpica/evrgnini.c
index 93ec528bcd9a..4187f563fede 100644
--- a/drivers/acpi/acpica/evrgnini.c
+++ b/drivers/acpi/acpica/evrgnini.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evsci.c b/drivers/acpi/acpica/evsci.c
index 8ce73b962006..d5594f79f877 100644
--- a/drivers/acpi/acpica/evsci.c
+++ b/drivers/acpi/acpica/evsci.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxface.c b/drivers/acpi/acpica/evxface.c
index dd1b9dd64cef..9b3c01bf1438 100644
--- a/drivers/acpi/acpica/evxface.c
+++ b/drivers/acpi/acpica/evxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxfevnt.c b/drivers/acpi/acpica/evxfevnt.c
index c773ac4892cb..96c2520f9570 100644
--- a/drivers/acpi/acpica/evxfevnt.c
+++ b/drivers/acpi/acpica/evxfevnt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxfgpe.c b/drivers/acpi/acpica/evxfgpe.c
index 67c7c4ce276c..cbb1598df9dc 100644
--- a/drivers/acpi/acpica/evxfgpe.c
+++ b/drivers/acpi/acpica/evxfgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/evxfregn.c b/drivers/acpi/acpica/evxfregn.c
index beba9d56a0d8..705fcd86151a 100644
--- a/drivers/acpi/acpica/evxfregn.c
+++ b/drivers/acpi/acpica/evxfregn.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exconcat.c b/drivers/acpi/acpica/exconcat.c
index 59b8de2f07d3..ea20e10dd1f2 100644
--- a/drivers/acpi/acpica/exconcat.c
+++ b/drivers/acpi/acpica/exconcat.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c
index 61813bd43f9e..827f47b72663 100644
--- a/drivers/acpi/acpica/exconfig.c
+++ b/drivers/acpi/acpica/exconfig.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exconvrt.c b/drivers/acpi/acpica/exconvrt.c
index 23ebadb06a95..9abcc41a573b 100644
--- a/drivers/acpi/acpica/exconvrt.c
+++ b/drivers/acpi/acpica/exconvrt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/excreate.c b/drivers/acpi/acpica/excreate.c
index b8adb11f1b07..3dece45dd997 100644
--- a/drivers/acpi/acpica/excreate.c
+++ b/drivers/acpi/acpica/excreate.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exdebug.c b/drivers/acpi/acpica/exdebug.c
index a8191d2ca5e3..d931a66a16e3 100644
--- a/drivers/acpi/acpica/exdebug.c
+++ b/drivers/acpi/acpica/exdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exdump.c b/drivers/acpi/acpica/exdump.c
index 83398dc4b7c2..4989ce9591ae 100644
--- a/drivers/acpi/acpica/exdump.c
+++ b/drivers/acpi/acpica/exdump.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -617,10 +617,11 @@ void acpi_ex_dump_operand(union acpi_operand_object *obj_desc, u32 depth)
 	u32 length;
 	u32 index;
 
-	ACPI_FUNCTION_NAME(ex_dump_operand)
+	ACPI_FUNCTION_NAME(ex_dump_operand);
 
-	    /* Check if debug output enabled */
-	    if (!ACPI_IS_DEBUG_ENABLED(ACPI_LV_EXEC, _COMPONENT)) {
+	/* Check if debug output enabled */
+
+	if (!ACPI_IS_DEBUG_ENABLED(ACPI_LV_EXEC, _COMPONENT)) {
 		return;
 	}
 
@@ -904,7 +905,7 @@ void
 acpi_ex_dump_operands(union acpi_operand_object **operands,
 		      const char *opcode_name, u32 num_operands)
 {
-	ACPI_FUNCTION_NAME(ex_dump_operands);
+	ACPI_FUNCTION_TRACE(ex_dump_operands);
 
 	if (!opcode_name) {
 		opcode_name = "UNKNOWN";
@@ -928,7 +929,7 @@ acpi_ex_dump_operands(union acpi_operand_object **operands,
 
 	ACPI_DEBUG_PRINT((ACPI_DB_EXEC,
 			  "**** End operand dump for [%s]\n", opcode_name));
-	return;
+	return_VOID;
 }
 
 /*******************************************************************************
diff --git a/drivers/acpi/acpica/exfield.c b/drivers/acpi/acpica/exfield.c
index 5fda981f6498..e3b0650e5bb6 100644
--- a/drivers/acpi/acpica/exfield.c
+++ b/drivers/acpi/acpica/exfield.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exfldio.c b/drivers/acpi/acpica/exfldio.c
index a656608dca84..3d0f274be88b 100644
--- a/drivers/acpi/acpica/exfldio.c
+++ b/drivers/acpi/acpica/exfldio.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exmisc.c b/drivers/acpi/acpica/exmisc.c
index dbad3ebd7df5..1518fcb22ae1 100644
--- a/drivers/acpi/acpica/exmisc.c
+++ b/drivers/acpi/acpica/exmisc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exmutex.c b/drivers/acpi/acpica/exmutex.c
index ecd95b3f35f1..24c9741dee48 100644
--- a/drivers/acpi/acpica/exmutex.c
+++ b/drivers/acpi/acpica/exmutex.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exnames.c b/drivers/acpi/acpica/exnames.c
index caa5ed1f65ec..c7b249cda5c0 100644
--- a/drivers/acpi/acpica/exnames.c
+++ b/drivers/acpi/acpica/exnames.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg1.c b/drivers/acpi/acpica/exoparg1.c
index f787651348c1..dae01c93e480 100644
--- a/drivers/acpi/acpica/exoparg1.c
+++ b/drivers/acpi/acpica/exoparg1.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg2.c b/drivers/acpi/acpica/exoparg2.c
index 57980b7d3594..3cafa1d6f31a 100644
--- a/drivers/acpi/acpica/exoparg2.c
+++ b/drivers/acpi/acpica/exoparg2.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg3.c b/drivers/acpi/acpica/exoparg3.c
index ce857addc8db..f16c655121ff 100644
--- a/drivers/acpi/acpica/exoparg3.c
+++ b/drivers/acpi/acpica/exoparg3.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exoparg6.c b/drivers/acpi/acpica/exoparg6.c
index 688032b58a21..8b39fffce6dc 100644
--- a/drivers/acpi/acpica/exoparg6.c
+++ b/drivers/acpi/acpica/exoparg6.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exprep.c b/drivers/acpi/acpica/exprep.c
index 8de060664204..1d1040f2e3f8 100644
--- a/drivers/acpi/acpica/exprep.c
+++ b/drivers/acpi/acpica/exprep.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exregion.c b/drivers/acpi/acpica/exregion.c
index 7bcc9d809b7e..387c438aa485 100644
--- a/drivers/acpi/acpica/exregion.c
+++ b/drivers/acpi/acpica/exregion.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exresnte.c b/drivers/acpi/acpica/exresnte.c
index 91c1de046442..77fa8d9aa5bf 100644
--- a/drivers/acpi/acpica/exresnte.c
+++ b/drivers/acpi/acpica/exresnte.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exresolv.c b/drivers/acpi/acpica/exresolv.c
index 5e1854ea85f6..b104bc3ca809 100644
--- a/drivers/acpi/acpica/exresolv.c
+++ b/drivers/acpi/acpica/exresolv.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exresop.c b/drivers/acpi/acpica/exresop.c
index 1c7c9962b0de..2643d34f194d 100644
--- a/drivers/acpi/acpica/exresop.c
+++ b/drivers/acpi/acpica/exresop.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exstore.c b/drivers/acpi/acpica/exstore.c
index bdd43cde8f36..8f106bdcad5f 100644
--- a/drivers/acpi/acpica/exstore.c
+++ b/drivers/acpi/acpica/exstore.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exstoren.c b/drivers/acpi/acpica/exstoren.c
index 56f59cf5da29..3d458d1996b0 100644
--- a/drivers/acpi/acpica/exstoren.c
+++ b/drivers/acpi/acpica/exstoren.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exstorob.c b/drivers/acpi/acpica/exstorob.c
index 4ba7fcbf23b0..905443a3c28f 100644
--- a/drivers/acpi/acpica/exstorob.c
+++ b/drivers/acpi/acpica/exstorob.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exsystem.c b/drivers/acpi/acpica/exsystem.c
index ad3b610057f3..420d9b145d2e 100644
--- a/drivers/acpi/acpica/exsystem.c
+++ b/drivers/acpi/acpica/exsystem.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/extrace.c b/drivers/acpi/acpica/extrace.c
index ae9df8672d9e..9a67d507a132 100644
--- a/drivers/acpi/acpica/extrace.c
+++ b/drivers/acpi/acpica/extrace.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/exutils.c b/drivers/acpi/acpica/exutils.c
index 34d608358eaf..fb80d3f55d63 100644
--- a/drivers/acpi/acpica/exutils.c
+++ b/drivers/acpi/acpica/exutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwacpi.c b/drivers/acpi/acpica/hwacpi.c
index fad249e774b4..68e958d4c25f 100644
--- a/drivers/acpi/acpica/hwacpi.c
+++ b/drivers/acpi/acpica/hwacpi.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwesleep.c b/drivers/acpi/acpica/hwesleep.c
index 12626d021a9b..64855b62a5ae 100644
--- a/drivers/acpi/acpica/hwesleep.c
+++ b/drivers/acpi/acpica/hwesleep.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwgpe.c b/drivers/acpi/acpica/hwgpe.c
index 09b6822aa5cc..c1c54af148d0 100644
--- a/drivers/acpi/acpica/hwgpe.c
+++ b/drivers/acpi/acpica/hwgpe.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwpci.c b/drivers/acpi/acpica/hwpci.c
index 283819930be6..faa2fa45eb1c 100644
--- a/drivers/acpi/acpica/hwpci.c
+++ b/drivers/acpi/acpica/hwpci.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwregs.c b/drivers/acpi/acpica/hwregs.c
index aa6e00081915..f3e7b7851a3a 100644
--- a/drivers/acpi/acpica/hwregs.c
+++ b/drivers/acpi/acpica/hwregs.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwsleep.c b/drivers/acpi/acpica/hwsleep.c
index 1fe7387a00e6..c85c373ecbc4 100644
--- a/drivers/acpi/acpica/hwsleep.c
+++ b/drivers/acpi/acpica/hwsleep.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwtimer.c b/drivers/acpi/acpica/hwtimer.c
index a2f4e25d45b1..511e3b8ffc6d 100644
--- a/drivers/acpi/acpica/hwtimer.c
+++ b/drivers/acpi/acpica/hwtimer.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -150,10 +150,10 @@ ACPI_EXPORT_SYMBOL(acpi_get_timer)
  *
  ******************************************************************************/
 acpi_status
-acpi_get_timer_duration(u32 start_ticks, u32 end_ticks, u32 * time_elapsed)
+acpi_get_timer_duration(u32 start_ticks, u32 end_ticks, u32 *time_elapsed)
 {
 	acpi_status status;
-	u32 delta_ticks;
+	u64 delta_ticks;
 	u64 quotient;
 
 	ACPI_FUNCTION_TRACE(acpi_get_timer_duration);
@@ -168,30 +168,29 @@ acpi_get_timer_duration(u32 start_ticks, u32 end_ticks, u32 * time_elapsed)
 		return_ACPI_STATUS(AE_SUPPORT);
 	}
 
+	if (start_ticks == end_ticks) {
+		*time_elapsed = 0;
+		return_ACPI_STATUS(AE_OK);
+	}
+
 	/*
 	 * Compute Tick Delta:
 	 * Handle (max one) timer rollovers on 24-bit versus 32-bit timers.
 	 */
-	if (start_ticks < end_ticks) {
-		delta_ticks = end_ticks - start_ticks;
-	} else if (start_ticks > end_ticks) {
+	delta_ticks = end_ticks;
+	if (start_ticks > end_ticks) {
 		if ((acpi_gbl_FADT.flags & ACPI_FADT_32BIT_TIMER) == 0) {
 
 			/* 24-bit Timer */
 
-			delta_ticks =
-			    (((0x00FFFFFF - start_ticks) +
-			      end_ticks) & 0x00FFFFFF);
+			delta_ticks |= (u64)1 << 24;
 		} else {
 			/* 32-bit Timer */
 
-			delta_ticks = (0xFFFFFFFF - start_ticks) + end_ticks;
+			delta_ticks |= (u64)1 << 32;
 		}
-	} else {		/* start_ticks == end_ticks */
-
-		*time_elapsed = 0;
-		return_ACPI_STATUS(AE_OK);
 	}
+	delta_ticks -= start_ticks;
 
 	/*
 	 * Compute Duration (Requires a 64-bit multiply and divide):
@@ -199,10 +198,10 @@ acpi_get_timer_duration(u32 start_ticks, u32 end_ticks, u32 * time_elapsed)
 	 * time_elapsed (microseconds) =
 	 *  (delta_ticks * ACPI_USEC_PER_SEC) / ACPI_PM_TIMER_FREQUENCY;
 	 */
-	status = acpi_ut_short_divide(((u64)delta_ticks) * ACPI_USEC_PER_SEC,
+	status = acpi_ut_short_divide(delta_ticks * ACPI_USEC_PER_SEC,
 				      ACPI_PM_TIMER_FREQUENCY, &quotient, NULL);
 
-	*time_elapsed = (u32) quotient;
+	*time_elapsed = (u32)quotient;
 	return_ACPI_STATUS(status);
 }
 
diff --git a/drivers/acpi/acpica/hwvalid.c b/drivers/acpi/acpica/hwvalid.c
index 3094cec4eab4..65d82e6add0b 100644
--- a/drivers/acpi/acpica/hwvalid.c
+++ b/drivers/acpi/acpica/hwvalid.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -128,14 +128,14 @@ acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width)
 	acpi_io_address last_address;
 	const struct acpi_port_info *port_info;
 
-	ACPI_FUNCTION_NAME(hw_validate_io_request);
+	ACPI_FUNCTION_TRACE(hw_validate_io_request);
 
 	/* Supported widths are 8/16/32 */
 
 	if ((bit_width != 8) && (bit_width != 16) && (bit_width != 32)) {
 		ACPI_ERROR((AE_INFO,
 			    "Bad BitWidth parameter: %8.8X", bit_width));
-		return (AE_BAD_PARAMETER);
+		return_ACPI_STATUS(AE_BAD_PARAMETER);
 	}
 
 	port_info = acpi_protected_ports;
@@ -153,13 +153,13 @@ acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width)
 		ACPI_ERROR((AE_INFO,
 			    "Illegal I/O port address/length above 64K: %8.8X%8.8X/0x%X",
 			    ACPI_FORMAT_UINT64(address), byte_width));
-		return (AE_LIMIT);
+		return_ACPI_STATUS(AE_LIMIT);
 	}
 
 	/* Exit if requested address is not within the protected port table */
 
 	if (address > acpi_protected_ports[ACPI_PORT_INFO_ENTRIES - 1].end) {
-		return (AE_OK);
+		return_ACPI_STATUS(AE_OK);
 	}
 
 	/* Check request against the list of protected I/O ports */
@@ -180,8 +180,8 @@ acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width)
 			/* Port illegality may depend on the _OSI calls made by the BIOS */
 
 			if (acpi_gbl_osi_data >= port_info->osi_dependency) {
-				ACPI_DEBUG_PRINT((ACPI_DB_IO,
-						  "Denied AML access to port 0x%8.8X%8.8X/%X (%s 0x%.4X-0x%.4X)",
+				ACPI_DEBUG_PRINT((ACPI_DB_VALUES,
+						  "Denied AML access to port 0x%8.8X%8.8X/%X (%s 0x%.4X-0x%.4X)\n",
 						  ACPI_FORMAT_UINT64(address),
 						  byte_width, port_info->name,
 						  port_info->start,
@@ -198,7 +198,7 @@ acpi_hw_validate_io_request(acpi_io_address address, u32 bit_width)
 		}
 	}
 
-	return (AE_OK);
+	return_ACPI_STATUS(AE_OK);
 }
 
 /******************************************************************************
diff --git a/drivers/acpi/acpica/hwxface.c b/drivers/acpi/acpica/hwxface.c
index b3c6e439933c..d320b129b7d7 100644
--- a/drivers/acpi/acpica/hwxface.c
+++ b/drivers/acpi/acpica/hwxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/hwxfsleep.c b/drivers/acpi/acpica/hwxfsleep.c
index e5c095ca6083..589c774bbf9a 100644
--- a/drivers/acpi/acpica/hwxfsleep.c
+++ b/drivers/acpi/acpica/hwxfsleep.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsaccess.c b/drivers/acpi/acpica/nsaccess.c
index f2733f51ca8d..07f672b5a1d1 100644
--- a/drivers/acpi/acpica/nsaccess.c
+++ b/drivers/acpi/acpica/nsaccess.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -644,17 +644,18 @@ acpi_ns_lookup(union acpi_generic_state *scope_info,
 					    this_node->object;
 				}
 			}
-#ifdef ACPI_ASL_COMPILER
-			if (!acpi_gbl_disasm_flag &&
-			    (this_node->flags & ANOBJ_IS_EXTERNAL)) {
-				this_node->flags |= IMPLICIT_EXTERNAL;
-			}
-#endif
 		}
 
 		/* Special handling for the last segment (num_segments == 0) */
 
 		else {
+#ifdef ACPI_ASL_COMPILER
+			if (!acpi_gbl_disasm_flag
+			    && (this_node->flags & ANOBJ_IS_EXTERNAL)) {
+				this_node->flags &= ~IMPLICIT_EXTERNAL;
+			}
+#endif
+
 			/*
 			 * Sanity typecheck of the target object:
 			 *
diff --git a/drivers/acpi/acpica/nsalloc.c b/drivers/acpi/acpica/nsalloc.c
index 8ba5b32c9f71..ce57ccf4c1bf 100644
--- a/drivers/acpi/acpica/nsalloc.c
+++ b/drivers/acpi/acpica/nsalloc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsarguments.c b/drivers/acpi/acpica/nsarguments.c
index 67b7370dcae5..ce296ac14cf0 100644
--- a/drivers/acpi/acpica/nsarguments.c
+++ b/drivers/acpi/acpica/nsarguments.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsconvert.c b/drivers/acpi/acpica/nsconvert.c
index 539d775bbc92..2f9d5d190fa9 100644
--- a/drivers/acpi/acpica/nsconvert.c
+++ b/drivers/acpi/acpica/nsconvert.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -495,7 +495,8 @@ acpi_ns_convert_to_reference(struct acpi_namespace_node *scope,
 
 		/* Check if we are resolving a named reference within a package */
 
-		ACPI_ERROR_NAMESPACE(original_object->string.pointer, status);
+		ACPI_ERROR_NAMESPACE(&scope_info,
+				     original_object->string.pointer, status);
 		goto error_exit;
 	}
 
diff --git a/drivers/acpi/acpica/nsdump.c b/drivers/acpi/acpica/nsdump.c
index 4123b5077a7d..e2ac16818dc3 100644
--- a/drivers/acpi/acpica/nsdump.c
+++ b/drivers/acpi/acpica/nsdump.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsdumpdv.c b/drivers/acpi/acpica/nsdumpdv.c
index 5026594763ea..09ac00dee450 100644
--- a/drivers/acpi/acpica/nsdumpdv.c
+++ b/drivers/acpi/acpica/nsdumpdv.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nseval.c b/drivers/acpi/acpica/nseval.c
index d22167cbd0ca..c2d883b8c45e 100644
--- a/drivers/acpi/acpica/nseval.c
+++ b/drivers/acpi/acpica/nseval.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsinit.c b/drivers/acpi/acpica/nsinit.c
index 9c6297949712..c17af4a3ab67 100644
--- a/drivers/acpi/acpica/nsinit.c
+++ b/drivers/acpi/acpica/nsinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsload.c b/drivers/acpi/acpica/nsload.c
index d2915e186ae1..fdfe9309bd33 100644
--- a/drivers/acpi/acpica/nsload.c
+++ b/drivers/acpi/acpica/nsload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsnames.c b/drivers/acpi/acpica/nsnames.c
index a410760a0308..c686eda7ca66 100644
--- a/drivers/acpi/acpica/nsnames.c
+++ b/drivers/acpi/acpica/nsnames.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -49,6 +49,9 @@
 #define _COMPONENT          ACPI_NAMESPACE
 ACPI_MODULE_NAME("nsnames")
 
+/* Local Prototypes */
+static void acpi_ns_normalize_pathname(char *original_path);
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_ns_get_external_pathname
@@ -63,6 +66,7 @@ ACPI_MODULE_NAME("nsnames")
  *              for error and debug statements.
  *
  ******************************************************************************/
+
 char *acpi_ns_get_external_pathname(struct acpi_namespace_node *node)
 {
 	char *name_buffer;
@@ -352,3 +356,148 @@ char *acpi_ns_get_normalized_pathname(struct acpi_namespace_node *node,
 
 	return_PTR(name_buffer);
 }
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ns_build_prefixed_pathname
+ *
+ * PARAMETERS:  prefix_scope        - Scope/Path that prefixes the internal path
+ *              internal_path       - Name or path of the namespace node
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Construct a fully qualified pathname from a concatenation of:
+ *              1) Path associated with the prefix_scope namespace node
+ *              2) External path representation of the Internal path
+ *
+ ******************************************************************************/
+
+char *acpi_ns_build_prefixed_pathname(union acpi_generic_state *prefix_scope,
+				      const char *internal_path)
+{
+	acpi_status status;
+	char *full_path = NULL;
+	char *external_path = NULL;
+	char *prefix_path = NULL;
+	u32 prefix_path_length = 0;
+
+	/* If there is a prefix, get the pathname to it */
+
+	if (prefix_scope && prefix_scope->scope.node) {
+		prefix_path =
+		    acpi_ns_get_normalized_pathname(prefix_scope->scope.node,
+						    TRUE);
+		if (prefix_path) {
+			prefix_path_length = strlen(prefix_path);
+		}
+	}
+
+	status = acpi_ns_externalize_name(ACPI_UINT32_MAX, internal_path,
+					  NULL, &external_path);
+	if (ACPI_FAILURE(status)) {
+		goto cleanup;
+	}
+
+	/* Merge the prefix path and the path. 2 is for one dot and trailing null */
+
+	full_path =
+	    ACPI_ALLOCATE_ZEROED(prefix_path_length + strlen(external_path) +
+				 2);
+	if (!full_path) {
+		goto cleanup;
+	}
+
+	/* Don't merge if the External path is already fully qualified */
+
+	if (prefix_path && (*external_path != '\\') && (*external_path != '^')) {
+		strcat(full_path, prefix_path);
+		if (prefix_path[1]) {
+			strcat(full_path, ".");
+		}
+	}
+
+	acpi_ns_normalize_pathname(external_path);
+	strcat(full_path, external_path);
+
+cleanup:
+	if (prefix_path) {
+		ACPI_FREE(prefix_path);
+	}
+	if (external_path) {
+		ACPI_FREE(external_path);
+	}
+
+	return (full_path);
+}
+
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ns_normalize_pathname
+ *
+ * PARAMETERS:  original_path       - Path to be normalized, in External format
+ *
+ * RETURN:      The original path is processed in-place
+ *
+ * DESCRIPTION: Remove trailing underscores from each element of a path.
+ *
+ *              For example:  \A___.B___.C___ becomes \A.B.C
+ *
+ ******************************************************************************/
+
+static void acpi_ns_normalize_pathname(char *original_path)
+{
+	char *input_path = original_path;
+	char *new_path_buffer;
+	char *new_path;
+	u32 i;
+
+	/* Allocate a temp buffer in which to construct the new path */
+
+	new_path_buffer = ACPI_ALLOCATE_ZEROED(strlen(input_path) + 1);
+	new_path = new_path_buffer;
+	if (!new_path_buffer) {
+		return;
+	}
+
+	/* Special characters may appear at the beginning of the path */
+
+	if (*input_path == '\\') {
+		*new_path = *input_path;
+		new_path++;
+		input_path++;
+	}
+
+	while (*input_path == '^') {
+		*new_path = *input_path;
+		new_path++;
+		input_path++;
+	}
+
+	/* Remainder of the path */
+
+	while (*input_path) {
+
+		/* Do one nameseg at a time */
+
+		for (i = 0; (i < ACPI_NAME_SIZE) && *input_path; i++) {
+			if ((i == 0) || (*input_path != '_')) {	/* First char is allowed to be underscore */
+				*new_path = *input_path;
+				new_path++;
+			}
+
+			input_path++;
+		}
+
+		/* Dot means that there are more namesegs to come */
+
+		if (*input_path == '.') {
+			*new_path = *input_path;
+			new_path++;
+			input_path++;
+		}
+	}
+
+	*new_path = 0;
+	strcpy(original_path, new_path_buffer);
+	ACPI_FREE(new_path_buffer);
+}
diff --git a/drivers/acpi/acpica/nsobject.c b/drivers/acpi/acpica/nsobject.c
index 707b2aa501e1..757e44555ec3 100644
--- a/drivers/acpi/acpica/nsobject.c
+++ b/drivers/acpi/acpica/nsobject.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsparse.c b/drivers/acpi/acpica/nsparse.c
index 2fc33a5203f4..c5b22ea5b369 100644
--- a/drivers/acpi/acpica/nsparse.c
+++ b/drivers/acpi/acpica/nsparse.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nspredef.c b/drivers/acpi/acpica/nspredef.c
index 9d14b509529e..4f1f6d6d9ddf 100644
--- a/drivers/acpi/acpica/nspredef.c
+++ b/drivers/acpi/acpica/nspredef.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsprepkg.c b/drivers/acpi/acpica/nsprepkg.c
index a8ea8fb1d299..7805d5ce8127 100644
--- a/drivers/acpi/acpica/nsprepkg.c
+++ b/drivers/acpi/acpica/nsprepkg.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsrepair.c b/drivers/acpi/acpica/nsrepair.c
index 418ef2ac82ab..7b6b6d281f1c 100644
--- a/drivers/acpi/acpica/nsrepair.c
+++ b/drivers/acpi/acpica/nsrepair.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsrepair2.c b/drivers/acpi/acpica/nsrepair2.c
index 06037e044694..29c3973c7815 100644
--- a/drivers/acpi/acpica/nsrepair2.c
+++ b/drivers/acpi/acpica/nsrepair2.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nssearch.c b/drivers/acpi/acpica/nssearch.c
index 5de8957f5ef0..a469447f5c02 100644
--- a/drivers/acpi/acpica/nssearch.c
+++ b/drivers/acpi/acpica/nssearch.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -417,6 +417,7 @@ acpi_ns_search_and_enter(u32 target_name,
 	if (flags & ACPI_NS_EXTERNAL ||
 	    (walk_state && walk_state->opcode == AML_SCOPE_OP)) {
 		new_node->flags |= ANOBJ_IS_EXTERNAL;
+		new_node->flags |= IMPLICIT_EXTERNAL;
 	}
 #endif
 
diff --git a/drivers/acpi/acpica/nsutils.c b/drivers/acpi/acpica/nsutils.c
index b43fe5fce64b..0487fdb59b0e 100644
--- a/drivers/acpi/acpica/nsutils.c
+++ b/drivers/acpi/acpica/nsutils.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nswalk.c b/drivers/acpi/acpica/nswalk.c
index 6b6e6f498cff..dd7ae1bc8af8 100644
--- a/drivers/acpi/acpica/nswalk.c
+++ b/drivers/acpi/acpica/nswalk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsxfeval.c b/drivers/acpi/acpica/nsxfeval.c
index 783f4c838aee..1075bd9541f5 100644
--- a/drivers/acpi/acpica/nsxfeval.c
+++ b/drivers/acpi/acpica/nsxfeval.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -61,10 +61,10 @@ static void acpi_ns_resolve_references(struct acpi_evaluate_info *info);
  *
  * PARAMETERS:  handle              - Object handle (optional)
  *              pathname            - Object pathname (optional)
- *              external_params     - List of parameters to pass to method,
+ *              external_params     - List of parameters to pass to a method,
  *                                    terminated by NULL. May be NULL
  *                                    if no parameters are being passed.
- *              return_buffer       - Where to put method's return value (if
+ *              return_buffer       - Where to put the object's return value (if
  *                                    any). If NULL, no value is returned.
  *              return_type         - Expected type of return object
  *
@@ -100,13 +100,14 @@ acpi_evaluate_object_typed(acpi_handle handle,
 		free_buffer_on_error = TRUE;
 	}
 
+	/* Get a handle here, in order to build an error message if needed */
+
+	target_handle = handle;
 	if (pathname) {
 		status = acpi_get_handle(handle, pathname, &target_handle);
 		if (ACPI_FAILURE(status)) {
 			return_ACPI_STATUS(status);
 		}
-	} else {
-		target_handle = handle;
 	}
 
 	full_pathname = acpi_ns_get_external_pathname(target_handle);
diff --git a/drivers/acpi/acpica/nsxfname.c b/drivers/acpi/acpica/nsxfname.c
index 106966235805..e9603fc9586c 100644
--- a/drivers/acpi/acpica/nsxfname.c
+++ b/drivers/acpi/acpica/nsxfname.c
@@ -6,7 +6,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/nsxfobj.c b/drivers/acpi/acpica/nsxfobj.c
index 47f689ec3fcb..ac1fbf767cac 100644
--- a/drivers/acpi/acpica/nsxfobj.c
+++ b/drivers/acpi/acpica/nsxfobj.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psargs.c b/drivers/acpi/acpica/psargs.c
index eb9dfaca555f..dbc51bc5fdd6 100644
--- a/drivers/acpi/acpica/psargs.c
+++ b/drivers/acpi/acpica/psargs.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -361,7 +361,7 @@ acpi_ps_get_next_namepath(struct acpi_walk_state *walk_state,
 	/* Final exception check (may have been changed from code above) */
 
 	if (ACPI_FAILURE(status)) {
-		ACPI_ERROR_NAMESPACE(path, status);
+		ACPI_ERROR_NAMESPACE(walk_state->scope_info, path, status);
 
 		if ((walk_state->parse_flags & ACPI_PARSE_MODE_MASK) ==
 		    ACPI_PARSE_EXECUTE) {
diff --git a/drivers/acpi/acpica/psloop.c b/drivers/acpi/acpica/psloop.c
index bb04dec168ad..7dca287d7690 100644
--- a/drivers/acpi/acpica/psloop.c
+++ b/drivers/acpi/acpica/psloop.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psobject.c b/drivers/acpi/acpica/psobject.c
index 0bef6df71bba..b18f1e048985 100644
--- a/drivers/acpi/acpica/psobject.c
+++ b/drivers/acpi/acpica/psobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -372,16 +372,10 @@ acpi_ps_create_op(struct acpi_walk_state *walk_state,
 			 * external declaration opcode. Setting walk_state->Aml to
 			 * walk_state->parser_state.Aml + 2 moves increments the
 			 * walk_state->Aml past the object type and the paramcount of the
-			 * external opcode. For the error message, only print the AML
-			 * offset. We could attempt to print the name but this may cause
-			 * a segmentation fault when printing the namepath because the
-			 * AML may be incorrect.
+			 * external opcode.
 			 */
-			acpi_os_printf
-			    ("// Invalid external declaration at AML offset 0x%x.\n",
-			     walk_state->aml -
-			     walk_state->parser_state.aml_start);
 			walk_state->aml = walk_state->parser_state.aml + 2;
+			walk_state->parser_state.aml = walk_state->aml;
 			return_ACPI_STATUS(AE_CTRL_PARSE_CONTINUE);
 		}
 #endif
diff --git a/drivers/acpi/acpica/psopcode.c b/drivers/acpi/acpica/psopcode.c
index a402ad772a1e..d31f3eb23225 100644
--- a/drivers/acpi/acpica/psopcode.c
+++ b/drivers/acpi/acpica/psopcode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psopinfo.c b/drivers/acpi/acpica/psopinfo.c
index eff22950232b..1dc1fc79297e 100644
--- a/drivers/acpi/acpica/psopinfo.c
+++ b/drivers/acpi/acpica/psopinfo.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psparse.c b/drivers/acpi/acpica/psparse.c
index ac88319dc111..2474ff961294 100644
--- a/drivers/acpi/acpica/psparse.c
+++ b/drivers/acpi/acpica/psparse.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psscope.c b/drivers/acpi/acpica/psscope.c
index 22d7f1d6849b..f49cdcc65700 100644
--- a/drivers/acpi/acpica/psscope.c
+++ b/drivers/acpi/acpica/psscope.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/pstree.c b/drivers/acpi/acpica/pstree.c
index c06d6e2fc7a5..f9fa88c79b32 100644
--- a/drivers/acpi/acpica/pstree.c
+++ b/drivers/acpi/acpica/pstree.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psutils.c b/drivers/acpi/acpica/psutils.c
index 02642760cb93..fe151f42de3a 100644
--- a/drivers/acpi/acpica/psutils.c
+++ b/drivers/acpi/acpica/psutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -94,9 +94,11 @@ void acpi_ps_init_op(union acpi_parse_object *op, u16 opcode)
 	op->common.descriptor_type = ACPI_DESC_TYPE_PARSER;
 	op->common.aml_opcode = opcode;
 
-	ACPI_DISASM_ONLY_MEMBERS(strncpy(op->common.aml_op_name,
-					 (acpi_ps_get_opcode_info(opcode))->
-					 name, sizeof(op->common.aml_op_name)));
+	ACPI_DISASM_ONLY_MEMBERS(acpi_ut_safe_strncpy(op->common.aml_op_name,
+						      (acpi_ps_get_opcode_info
+						       (opcode))->name,
+						      sizeof(op->common.
+							     aml_op_name)));
 }
 
 /*******************************************************************************
@@ -158,10 +160,10 @@ union acpi_parse_object *acpi_ps_alloc_op(u16 opcode, u8 *aml)
 		if (opcode == AML_SCOPE_OP) {
 			acpi_gbl_current_scope = op;
 		}
-	}
 
-	if (gbl_capture_comments) {
-		ASL_CV_TRANSFER_COMMENTS(op);
+		if (acpi_gbl_capture_comments) {
+			ASL_CV_TRANSFER_COMMENTS(op);
+		}
 	}
 
 	return (op);
diff --git a/drivers/acpi/acpica/pswalk.c b/drivers/acpi/acpica/pswalk.c
index 22a37c82af19..bc5c779e54e8 100644
--- a/drivers/acpi/acpica/pswalk.c
+++ b/drivers/acpi/acpica/pswalk.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/psxface.c b/drivers/acpi/acpica/psxface.c
index c88a681586bf..d2270ade5cf8 100644
--- a/drivers/acpi/acpica/psxface.c
+++ b/drivers/acpi/acpica/psxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsaddr.c b/drivers/acpi/acpica/rsaddr.c
index a131a28bb09d..213bad89675b 100644
--- a/drivers/acpi/acpica/rsaddr.c
+++ b/drivers/acpi/acpica/rsaddr.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rscalc.c b/drivers/acpi/acpica/rscalc.c
index 659fb718504a..576f7aae162b 100644
--- a/drivers/acpi/acpica/rscalc.c
+++ b/drivers/acpi/acpica/rscalc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rscreate.c b/drivers/acpi/acpica/rscreate.c
index f72ff0b54a63..fe07001ea865 100644
--- a/drivers/acpi/acpica/rscreate.c
+++ b/drivers/acpi/acpica/rscreate.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsdump.c b/drivers/acpi/acpica/rsdump.c
index 55fd1880efbe..bc4c4755aeb9 100644
--- a/drivers/acpi/acpica/rsdump.c
+++ b/drivers/acpi/acpica/rsdump.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsdumpinfo.c b/drivers/acpi/acpica/rsdumpinfo.c
index da150e17795b..c4a2a08e31ac 100644
--- a/drivers/acpi/acpica/rsdumpinfo.c
+++ b/drivers/acpi/acpica/rsdumpinfo.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsinfo.c b/drivers/acpi/acpica/rsinfo.c
index b0e50518d766..e819bb0f45af 100644
--- a/drivers/acpi/acpica/rsinfo.c
+++ b/drivers/acpi/acpica/rsinfo.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsio.c b/drivers/acpi/acpica/rsio.c
index b7a47fbc519b..eafd993592f6 100644
--- a/drivers/acpi/acpica/rsio.c
+++ b/drivers/acpi/acpica/rsio.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsirq.c b/drivers/acpi/acpica/rsirq.c
index 092a733c42b8..aabd73298eb8 100644
--- a/drivers/acpi/acpica/rsirq.c
+++ b/drivers/acpi/acpica/rsirq.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rslist.c b/drivers/acpi/acpica/rslist.c
index 36a6657dd34d..11214780ea8f 100644
--- a/drivers/acpi/acpica/rslist.c
+++ b/drivers/acpi/acpica/rslist.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsmemory.c b/drivers/acpi/acpica/rsmemory.c
index 273eecb3001b..05e375abc6b5 100644
--- a/drivers/acpi/acpica/rsmemory.c
+++ b/drivers/acpi/acpica/rsmemory.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsmisc.c b/drivers/acpi/acpica/rsmisc.c
index cc4b5486c4bc..7b4627181cc6 100644
--- a/drivers/acpi/acpica/rsmisc.c
+++ b/drivers/acpi/acpica/rsmisc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsserial.c b/drivers/acpi/acpica/rsserial.c
index 14d12d6eb716..87dac2812072 100644
--- a/drivers/acpi/acpica/rsserial.c
+++ b/drivers/acpi/acpica/rsserial.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsutils.c b/drivers/acpi/acpica/rsutils.c
index b2aeca01204a..49ff7f851d58 100644
--- a/drivers/acpi/acpica/rsutils.c
+++ b/drivers/acpi/acpica/rsutils.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/rsxface.c b/drivers/acpi/acpica/rsxface.c
index be65e65e216e..3b481f0b81c5 100644
--- a/drivers/acpi/acpica/rsxface.c
+++ b/drivers/acpi/acpica/rsxface.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbdata.c b/drivers/acpi/acpica/tbdata.c
index b19a2f0ea331..ec69267f1447 100644
--- a/drivers/acpi/acpica/tbdata.c
+++ b/drivers/acpi/acpica/tbdata.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbfadt.c b/drivers/acpi/acpica/tbfadt.c
index 5f051d82188d..d1763c5e4e91 100644
--- a/drivers/acpi/acpica/tbfadt.c
+++ b/drivers/acpi/acpica/tbfadt.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbfind.c b/drivers/acpi/acpica/tbfind.c
index fea89c8d305c..999a64a48e1a 100644
--- a/drivers/acpi/acpica/tbfind.c
+++ b/drivers/acpi/acpica/tbfind.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbinstal.c b/drivers/acpi/acpica/tbinstal.c
index 0dfc0ac3c141..00be16da1ee2 100644
--- a/drivers/acpi/acpica/tbinstal.c
+++ b/drivers/acpi/acpica/tbinstal.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbprint.c b/drivers/acpi/acpica/tbprint.c
index edfd7b10be19..8cdcdd2c4697 100644
--- a/drivers/acpi/acpica/tbprint.c
+++ b/drivers/acpi/acpica/tbprint.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbutils.c b/drivers/acpi/acpica/tbutils.c
index 0c6768d20395..30d40ff8992b 100644
--- a/drivers/acpi/acpica/tbutils.c
+++ b/drivers/acpi/acpica/tbutils.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbxface.c b/drivers/acpi/acpica/tbxface.c
index 5ecb8d2e6834..dca91b6f8cc2 100644
--- a/drivers/acpi/acpica/tbxface.c
+++ b/drivers/acpi/acpica/tbxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c
index d81f442228b8..e09b4b26300e 100644
--- a/drivers/acpi/acpica/tbxfload.c
+++ b/drivers/acpi/acpica/tbxfload.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/tbxfroot.c b/drivers/acpi/acpica/tbxfroot.c
index f9f9a7da2cad..abf3c62e1e80 100644
--- a/drivers/acpi/acpica/tbxfroot.c
+++ b/drivers/acpi/acpica/tbxfroot.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utaddress.c b/drivers/acpi/acpica/utaddress.c
index 26a0633115be..d8540f380ae5 100644
--- a/drivers/acpi/acpica/utaddress.c
+++ b/drivers/acpi/acpica/utaddress.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utalloc.c b/drivers/acpi/acpica/utalloc.c
index 5594a359dbf1..12fbaddbfb0d 100644
--- a/drivers/acpi/acpica/utalloc.c
+++ b/drivers/acpi/acpica/utalloc.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utascii.c b/drivers/acpi/acpica/utascii.c
index 909bdb198651..95565e46a695 100644
--- a/drivers/acpi/acpica/utascii.c
+++ b/drivers/acpi/acpica/utascii.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utbuffer.c b/drivers/acpi/acpica/utbuffer.c
index f17eaa009dde..2c5a14c2f46b 100644
--- a/drivers/acpi/acpica/utbuffer.c
+++ b/drivers/acpi/acpica/utbuffer.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utcache.c b/drivers/acpi/acpica/utcache.c
index 531493306dee..08e6944404b3 100644
--- a/drivers/acpi/acpica/utcache.c
+++ b/drivers/acpi/acpica/utcache.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utcopy.c b/drivers/acpi/acpica/utcopy.c
index e9382255d6c6..01434af99035 100644
--- a/drivers/acpi/acpica/utcopy.c
+++ b/drivers/acpi/acpica/utcopy.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utdebug.c b/drivers/acpi/acpica/utdebug.c
index 615a885e2ca3..2201be1bf4c2 100644
--- a/drivers/acpi/acpica/utdebug.c
+++ b/drivers/acpi/acpica/utdebug.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -163,6 +163,9 @@ acpi_debug_print(u32 requested_debug_level,
 {
 	acpi_thread_id thread_id;
 	va_list args;
+#ifdef ACPI_APPLICATION
+	int fill_count;
+#endif
 
 	/* Check if debug output enabled */
 
@@ -202,10 +205,21 @@ acpi_debug_print(u32 requested_debug_level,
 		acpi_os_printf("[%u] ", (u32)thread_id);
 	}
 
-	acpi_os_printf("[%02ld] ", acpi_gbl_nesting_level);
-#endif
+	fill_count = 48 - acpi_gbl_nesting_level -
+	    strlen(acpi_ut_trim_function_name(function_name));
+	if (fill_count < 0) {
+		fill_count = 0;
+	}
+
+	acpi_os_printf("[%02ld] %*s",
+		       acpi_gbl_nesting_level, acpi_gbl_nesting_level + 1, " ");
+	acpi_os_printf("%s%*s: ",
+		       acpi_ut_trim_function_name(function_name), fill_count,
+		       " ");
 
+#else
 	acpi_os_printf("%-22.22s: ", acpi_ut_trim_function_name(function_name));
+#endif
 
 	va_start(args, format);
 	acpi_os_vprintf(format, args);
diff --git a/drivers/acpi/acpica/utdecode.c b/drivers/acpi/acpica/utdecode.c
index 02cd2c2d961a..1a3f316a18a8 100644
--- a/drivers/acpi/acpica/utdecode.c
+++ b/drivers/acpi/acpica/utdecode.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -395,11 +395,6 @@ const char *acpi_ut_get_reference_name(union acpi_operand_object *object)
 	return (acpi_gbl_ref_class_names[object->reference.class]);
 }
 
-#if defined(ACPI_DEBUG_OUTPUT) || defined(ACPI_DEBUGGER)
-/*
- * Strings and procedures used for debug only
- */
-
 /*******************************************************************************
  *
  * FUNCTION:    acpi_ut_get_mutex_name
@@ -433,6 +428,12 @@ const char *acpi_ut_get_mutex_name(u32 mutex_id)
 	return (acpi_gbl_mutex_names[mutex_id]);
 }
 
+#if defined(ACPI_DEBUG_OUTPUT) || defined(ACPI_DEBUGGER)
+
+/*
+ * Strings and procedures used for debug only
+ */
+
 /*******************************************************************************
  *
  * FUNCTION:    acpi_ut_get_notify_name
diff --git a/drivers/acpi/acpica/utdelete.c b/drivers/acpi/acpica/utdelete.c
index c6eb9fae70f9..7d8d0208f0a3 100644
--- a/drivers/acpi/acpica/utdelete.c
+++ b/drivers/acpi/acpica/utdelete.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/uterror.c b/drivers/acpi/acpica/uterror.c
index e3368186e1c1..ce5e891291bf 100644
--- a/drivers/acpi/acpica/uterror.c
+++ b/drivers/acpi/acpica/uterror.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -180,6 +180,78 @@ acpi_ut_predefined_bios_error(const char *module_name,
 	va_end(arg_list);
 }
 
+/*******************************************************************************
+ *
+ * FUNCTION:    acpi_ut_prefixed_namespace_error
+ *
+ * PARAMETERS:  module_name         - Caller's module name (for error output)
+ *              line_number         - Caller's line number (for error output)
+ *              prefix_scope        - Scope/Path that prefixes the internal path
+ *              internal_path       - Name or path of the namespace node
+ *              lookup_status       - Exception code from NS lookup
+ *
+ * RETURN:      None
+ *
+ * DESCRIPTION: Print error message with the full pathname constructed this way:
+ *
+ *                  prefix_scope_node_full_path.externalized_internal_path
+ *
+ * NOTE:        10/2017: Treat the major ns_lookup errors as firmware errors
+ *
+ ******************************************************************************/
+
+void
+acpi_ut_prefixed_namespace_error(const char *module_name,
+				 u32 line_number,
+				 union acpi_generic_state *prefix_scope,
+				 const char *internal_path,
+				 acpi_status lookup_status)
+{
+	char *full_path;
+	const char *message;
+
+	/*
+	 * Main cases:
+	 * 1) Object creation, object must not already exist
+	 * 2) Object lookup, object must exist
+	 */
+	switch (lookup_status) {
+	case AE_ALREADY_EXISTS:
+
+		acpi_os_printf(ACPI_MSG_BIOS_ERROR);
+		message = "Failure creating";
+		break;
+
+	case AE_NOT_FOUND:
+
+		acpi_os_printf(ACPI_MSG_BIOS_ERROR);
+		message = "Failure looking up";
+		break;
+
+	default:
+
+		acpi_os_printf(ACPI_MSG_ERROR);
+		message = "Failure looking up";
+		break;
+	}
+
+	/* Concatenate the prefix path and the internal path */
+
+	full_path =
+	    acpi_ns_build_prefixed_pathname(prefix_scope, internal_path);
+
+	acpi_os_printf("%s [%s], %s", message,
+		       full_path ? full_path : "Could not get pathname",
+		       acpi_format_exception(lookup_status));
+
+	if (full_path) {
+		ACPI_FREE(full_path);
+	}
+
+	ACPI_MSG_SUFFIX;
+}
+
+#ifdef __OBSOLETE_FUNCTION
 /*******************************************************************************
  *
  * FUNCTION:    acpi_ut_namespace_error
@@ -240,6 +312,7 @@ acpi_ut_namespace_error(const char *module_name,
 	ACPI_MSG_SUFFIX;
 	ACPI_MSG_REDIRECT_END;
 }
+#endif
 
 /*******************************************************************************
  *
diff --git a/drivers/acpi/acpica/uteval.c b/drivers/acpi/acpica/uteval.c
index 3fce7519c690..b8be0b82a130 100644
--- a/drivers/acpi/acpica/uteval.c
+++ b/drivers/acpi/acpica/uteval.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utexcep.c b/drivers/acpi/acpica/utexcep.c
index eb6dcab33d2f..e3dbad8b73e5 100644
--- a/drivers/acpi/acpica/utexcep.c
+++ b/drivers/acpi/acpica/utexcep.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utglobal.c b/drivers/acpi/acpica/utglobal.c
index 230a50c82f22..933595b0e594 100644
--- a/drivers/acpi/acpica/utglobal.c
+++ b/drivers/acpi/acpica/utglobal.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/uthex.c b/drivers/acpi/acpica/uthex.c
index fb406daf47fa..f5886d557a94 100644
--- a/drivers/acpi/acpica/uthex.c
+++ b/drivers/acpi/acpica/uthex.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utids.c b/drivers/acpi/acpica/utids.c
index a6eb580ee21d..db3c3c1d33da 100644
--- a/drivers/acpi/acpica/utids.c
+++ b/drivers/acpi/acpica/utids.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utinit.c b/drivers/acpi/acpica/utinit.c
index 23e766d1691d..a2005b030347 100644
--- a/drivers/acpi/acpica/utinit.c
+++ b/drivers/acpi/acpica/utinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -206,7 +206,6 @@ acpi_status acpi_ut_init_globals(void)
 	acpi_gbl_next_owner_id_offset = 0;
 	acpi_gbl_debugger_configuration = DEBUGGER_THREADING;
 	acpi_gbl_osi_mutex = NULL;
-	acpi_gbl_max_loop_iterations = ACPI_MAX_LOOP_COUNT;
 
 	/* Hardware oriented */
 
diff --git a/drivers/acpi/acpica/utlock.c b/drivers/acpi/acpica/utlock.c
index db2d9910866e..0636074a4c23 100644
--- a/drivers/acpi/acpica/utlock.c
+++ b/drivers/acpi/acpica/utlock.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utmath.c b/drivers/acpi/acpica/utmath.c
index 5f9c680076c4..eddf71990433 100644
--- a/drivers/acpi/acpica/utmath.c
+++ b/drivers/acpi/acpica/utmath.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -134,7 +134,7 @@ acpi_status acpi_ut_short_shift_left(u64 operand, u32 count, u64 *out_result)
 
 	if ((count & 63) >= 32) {
 		operand_ovl.part.hi = operand_ovl.part.lo;
-		operand_ovl.part.lo ^= operand_ovl.part.lo;
+		operand_ovl.part.lo = 0;
 		count = (count & 63) - 32;
 	}
 	ACPI_SHIFT_LEFT_64_BY_32(operand_ovl.part.hi,
@@ -171,7 +171,7 @@ acpi_status acpi_ut_short_shift_right(u64 operand, u32 count, u64 *out_result)
 
 	if ((count & 63) >= 32) {
 		operand_ovl.part.lo = operand_ovl.part.hi;
-		operand_ovl.part.hi ^= operand_ovl.part.hi;
+		operand_ovl.part.hi = 0;
 		count = (count & 63) - 32;
 	}
 	ACPI_SHIFT_RIGHT_64_BY_32(operand_ovl.part.hi,
diff --git a/drivers/acpi/acpica/utmisc.c b/drivers/acpi/acpica/utmisc.c
index 45c78c2adbf0..a331313ad5fa 100644
--- a/drivers/acpi/acpica/utmisc.c
+++ b/drivers/acpi/acpica/utmisc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utmutex.c b/drivers/acpi/acpica/utmutex.c
index 586354788018..6767bd1626f7 100644
--- a/drivers/acpi/acpica/utmutex.c
+++ b/drivers/acpi/acpica/utmutex.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -286,8 +286,9 @@ acpi_status acpi_ut_acquire_mutex(acpi_mutex_handle mutex_id)
 		acpi_gbl_mutex_info[mutex_id].thread_id = this_thread_id;
 	} else {
 		ACPI_EXCEPTION((AE_INFO, status,
-				"Thread %u could not acquire Mutex [0x%X]",
-				(u32)this_thread_id, mutex_id));
+				"Thread %u could not acquire Mutex [%s] (0x%X)",
+				(u32)this_thread_id,
+				acpi_ut_get_mutex_name(mutex_id), mutex_id));
 	}
 
 	return (status);
@@ -322,8 +323,8 @@ acpi_status acpi_ut_release_mutex(acpi_mutex_handle mutex_id)
 	 */
 	if (acpi_gbl_mutex_info[mutex_id].thread_id == ACPI_MUTEX_NOT_ACQUIRED) {
 		ACPI_ERROR((AE_INFO,
-			    "Mutex [0x%X] is not acquired, cannot release",
-			    mutex_id));
+			    "Mutex [%s] (0x%X) is not acquired, cannot release",
+			    acpi_ut_get_mutex_name(mutex_id), mutex_id));
 
 		return (AE_NOT_ACQUIRED);
 	}
diff --git a/drivers/acpi/acpica/utnonansi.c b/drivers/acpi/acpica/utnonansi.c
index 792664982ea3..94219610e259 100644
--- a/drivers/acpi/acpica/utnonansi.c
+++ b/drivers/acpi/acpica/utnonansi.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -140,7 +140,7 @@ int acpi_ut_stricmp(char *string1, char *string2)
 	return (c1 - c2);
 }
 
-#if defined (ACPI_DEBUGGER) || defined (ACPI_APPLICATION)
+#if defined (ACPI_DEBUGGER) || defined (ACPI_APPLICATION) || defined (ACPI_DEBUG_OUTPUT)
 /*******************************************************************************
  *
  * FUNCTION:    acpi_ut_safe_strcpy, acpi_ut_safe_strcat, acpi_ut_safe_strncat
@@ -199,4 +199,13 @@ acpi_ut_safe_strncat(char *dest,
 	strncat(dest, source, max_transfer_length);
 	return (FALSE);
 }
+
+void acpi_ut_safe_strncpy(char *dest, char *source, acpi_size dest_size)
+{
+	/* Always terminate destination string */
+
+	strncpy(dest, source, dest_size);
+	dest[dest_size - 1] = 0;
+}
+
 #endif
diff --git a/drivers/acpi/acpica/utobject.c b/drivers/acpi/acpica/utobject.c
index cb3db9fed50d..375901c0a596 100644
--- a/drivers/acpi/acpica/utobject.c
+++ b/drivers/acpi/acpica/utobject.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utosi.c b/drivers/acpi/acpica/utosi.c
index 3175b133c0e4..00ea104f6a0a 100644
--- a/drivers/acpi/acpica/utosi.c
+++ b/drivers/acpi/acpica/utosi.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -101,6 +101,8 @@ static struct acpi_interface_info acpi_default_supported_interfaces[] = {
 	{"Windows 2012", NULL, 0, ACPI_OSI_WIN_8},	/* Windows 8 and Server 2012 - Added 08/2012 */
 	{"Windows 2013", NULL, 0, ACPI_OSI_WIN_8},	/* Windows 8.1 and Server 2012 R2 - Added 01/2014 */
 	{"Windows 2015", NULL, 0, ACPI_OSI_WIN_10},	/* Windows 10 - Added 03/2015 */
+	{"Windows 2016", NULL, 0, ACPI_OSI_WIN_10_RS1},	/* Windows 10 version 1607 - Added 12/2017 */
+	{"Windows 2017", NULL, 0, ACPI_OSI_WIN_10_RS2},	/* Windows 10 version 1703 - Added 12/2017 */
 
 	/* Feature Group Strings */
 
diff --git a/drivers/acpi/acpica/utownerid.c b/drivers/acpi/acpica/utownerid.c
index 1b3ee74a87eb..9923dfa708be 100644
--- a/drivers/acpi/acpica/utownerid.c
+++ b/drivers/acpi/acpica/utownerid.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utpredef.c b/drivers/acpi/acpica/utpredef.c
index 350709f23e4c..ae6fef02b692 100644
--- a/drivers/acpi/acpica/utpredef.c
+++ b/drivers/acpi/acpica/utpredef.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utprint.c b/drivers/acpi/acpica/utprint.c
index c008589b41bd..ac07700f5b79 100644
--- a/drivers/acpi/acpica/utprint.c
+++ b/drivers/acpi/acpica/utprint.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utresdecode.c b/drivers/acpi/acpica/utresdecode.c
index e15a2538558b..93fa3450ca88 100644
--- a/drivers/acpi/acpica/utresdecode.c
+++ b/drivers/acpi/acpica/utresdecode.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utresrc.c b/drivers/acpi/acpica/utresrc.c
index f9801d13547f..4d289d9c734c 100644
--- a/drivers/acpi/acpica/utresrc.c
+++ b/drivers/acpi/acpica/utresrc.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utstate.c b/drivers/acpi/acpica/utstate.c
index eafabcd2fada..7750c48739d8 100644
--- a/drivers/acpi/acpica/utstate.c
+++ b/drivers/acpi/acpica/utstate.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utstring.c b/drivers/acpi/acpica/utstring.c
index 9eacbcb9e4f4..a9507d1976ff 100644
--- a/drivers/acpi/acpica/utstring.c
+++ b/drivers/acpi/acpica/utstring.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utstrsuppt.c b/drivers/acpi/acpica/utstrsuppt.c
index 965fb5cec94f..6fc76f0b60e9 100644
--- a/drivers/acpi/acpica/utstrsuppt.c
+++ b/drivers/acpi/acpica/utstrsuppt.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -52,10 +52,9 @@ static acpi_status
 acpi_ut_insert_digit(u64 *accumulated_value, u32 base, int ascii_digit);
 
 static acpi_status
-acpi_ut_strtoul_multiply64(u64 multiplicand, u64 multiplier, u64 *out_product);
+acpi_ut_strtoul_multiply64(u64 multiplicand, u32 base, u64 *out_product);
 
-static acpi_status
-acpi_ut_strtoul_add64(u64 addend1, u64 addend2, u64 *out_sum);
+static acpi_status acpi_ut_strtoul_add64(u64 addend1, u32 digit, u64 *out_sum);
 
 /*******************************************************************************
  *
@@ -357,7 +356,7 @@ acpi_ut_insert_digit(u64 *accumulated_value, u32 base, int ascii_digit)
  * FUNCTION:    acpi_ut_strtoul_multiply64
  *
  * PARAMETERS:  multiplicand            - Current accumulated converted integer
- *              multiplier              - Base/Radix
+ *              base                    - Base/Radix
  *              out_product             - Where the product is returned
  *
  * RETURN:      Status and 64-bit product
@@ -369,33 +368,40 @@ acpi_ut_insert_digit(u64 *accumulated_value, u32 base, int ascii_digit)
  ******************************************************************************/
 
 static acpi_status
-acpi_ut_strtoul_multiply64(u64 multiplicand, u64 multiplier, u64 *out_product)
+acpi_ut_strtoul_multiply64(u64 multiplicand, u32 base, u64 *out_product)
 {
-	u64 val;
+	u64 product;
+	u64 quotient;
 
 	/* Exit if either operand is zero */
 
 	*out_product = 0;
-	if (!multiplicand || !multiplier) {
+	if (!multiplicand || !base) {
 		return (AE_OK);
 	}
 
-	/* Check for 64-bit overflow before the actual multiplication */
-
-	acpi_ut_short_divide(ACPI_UINT64_MAX, (u32)multiplier, &val, NULL);
-	if (multiplicand > val) {
+	/*
+	 * Check for 64-bit overflow before the actual multiplication.
+	 *
+	 * Notes: 64-bit division is often not supported on 32-bit platforms
+	 * (it requires a library function), Therefore ACPICA has a local
+	 * 64-bit divide function. Also, Multiplier is currently only used
+	 * as the radix (8/10/16), to the 64/32 divide will always work.
+	 */
+	acpi_ut_short_divide(ACPI_UINT64_MAX, base, &quotient, NULL);
+	if (multiplicand > quotient) {
 		return (AE_NUMERIC_OVERFLOW);
 	}
 
-	val = multiplicand * multiplier;
+	product = multiplicand * base;
 
 	/* Check for 32-bit overflow if necessary */
 
-	if ((acpi_gbl_integer_bit_width == 32) && (val > ACPI_UINT32_MAX)) {
+	if ((acpi_gbl_integer_bit_width == 32) && (product > ACPI_UINT32_MAX)) {
 		return (AE_NUMERIC_OVERFLOW);
 	}
 
-	*out_product = val;
+	*out_product = product;
 	return (AE_OK);
 }
 
@@ -404,7 +410,7 @@ acpi_ut_strtoul_multiply64(u64 multiplicand, u64 multiplier, u64 *out_product)
  * FUNCTION:    acpi_ut_strtoul_add64
  *
  * PARAMETERS:  addend1                 - Current accumulated converted integer
- *              addend2                 - New hex value/char
+ *              digit                   - New hex value/char
  *              out_sum                 - Where sum is returned (Accumulator)
  *
  * RETURN:      Status and 64-bit sum
@@ -415,17 +421,17 @@ acpi_ut_strtoul_multiply64(u64 multiplicand, u64 multiplier, u64 *out_product)
  *
  ******************************************************************************/
 
-static acpi_status acpi_ut_strtoul_add64(u64 addend1, u64 addend2, u64 *out_sum)
+static acpi_status acpi_ut_strtoul_add64(u64 addend1, u32 digit, u64 *out_sum)
 {
 	u64 sum;
 
 	/* Check for 64-bit overflow before the actual addition */
 
-	if ((addend1 > 0) && (addend2 > (ACPI_UINT64_MAX - addend1))) {
+	if ((addend1 > 0) && (digit > (ACPI_UINT64_MAX - addend1))) {
 		return (AE_NUMERIC_OVERFLOW);
 	}
 
-	sum = addend1 + addend2;
+	sum = addend1 + digit;
 
 	/* Check for 32-bit overflow if necessary */
 
diff --git a/drivers/acpi/acpica/utstrtoul64.c b/drivers/acpi/acpica/utstrtoul64.c
index e2067dcb9389..9f7cef1de34a 100644
--- a/drivers/acpi/acpica/utstrtoul64.c
+++ b/drivers/acpi/acpica/utstrtoul64.c
@@ -6,7 +6,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/uttrack.c b/drivers/acpi/acpica/uttrack.c
index 3c8de88ecbd5..8cc70ca4e0fb 100644
--- a/drivers/acpi/acpica/uttrack.c
+++ b/drivers/acpi/acpica/uttrack.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -402,8 +402,8 @@ acpi_ut_track_allocation(struct acpi_debug_mem_block *allocation,
 	allocation->component = component;
 	allocation->line = line;
 
-	strncpy(allocation->module, module, ACPI_MAX_MODULE_NAME);
-	allocation->module[ACPI_MAX_MODULE_NAME - 1] = 0;
+	acpi_ut_safe_strncpy(allocation->module, (char *)module,
+			     ACPI_MAX_MODULE_NAME);
 
 	if (!element) {
 
@@ -717,7 +717,7 @@ void acpi_ut_dump_allocations(u32 component, const char *module)
 	if (!num_outstanding) {
 		ACPI_INFO(("No outstanding allocations"));
 	} else {
-		ACPI_ERROR((AE_INFO, "%u(0x%X) Outstanding allocations",
+		ACPI_ERROR((AE_INFO, "%u (0x%X) Outstanding cache allocations",
 			    num_outstanding, num_outstanding));
 	}
 
diff --git a/drivers/acpi/acpica/utuuid.c b/drivers/acpi/acpica/utuuid.c
index 5028e06718b1..95946fdb55d5 100644
--- a/drivers/acpi/acpica/utuuid.c
+++ b/drivers/acpi/acpica/utuuid.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utxface.c b/drivers/acpi/acpica/utxface.c
index 6b9ba4029f8e..25ef2ce64603 100644
--- a/drivers/acpi/acpica/utxface.c
+++ b/drivers/acpi/acpica/utxface.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utxferror.c b/drivers/acpi/acpica/utxferror.c
index 950a1e500bfa..a78861ded894 100644
--- a/drivers/acpi/acpica/utxferror.c
+++ b/drivers/acpi/acpica/utxferror.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -96,8 +96,8 @@ ACPI_EXPORT_SYMBOL(acpi_error)
  *
  * RETURN:      None
  *
- * DESCRIPTION: Print "ACPI Exception" message with module/line/version info
- *              and decoded acpi_status.
+ * DESCRIPTION: Print an "ACPI Error" message with module/line/version
+ *              info as well as decoded acpi_status.
  *
  ******************************************************************************/
 void ACPI_INTERNAL_VAR_XFACE
@@ -111,10 +111,10 @@ acpi_exception(const char *module_name,
 	/* For AE_OK, just print the message */
 
 	if (ACPI_SUCCESS(status)) {
-		acpi_os_printf(ACPI_MSG_EXCEPTION);
+		acpi_os_printf(ACPI_MSG_ERROR);
 
 	} else {
-		acpi_os_printf(ACPI_MSG_EXCEPTION "%s, ",
+		acpi_os_printf(ACPI_MSG_ERROR "%s, ",
 			       acpi_format_exception(status));
 	}
 
diff --git a/drivers/acpi/acpica/utxfinit.c b/drivers/acpi/acpica/utxfinit.c
index 6d5180601cf2..e727db52a55e 100644
--- a/drivers/acpi/acpica/utxfinit.c
+++ b/drivers/acpi/acpica/utxfinit.c
@@ -5,7 +5,7 @@
  *****************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/acpica/utxfmutex.c b/drivers/acpi/acpica/utxfmutex.c
index 0b85f113f726..764782fcf1bd 100644
--- a/drivers/acpi/acpica/utxfmutex.c
+++ b/drivers/acpi/acpica/utxfmutex.c
@@ -5,7 +5,7 @@
  ******************************************************************************/
 
 /*
- * Copyright (C) 2000 - 2017, Intel Corp.
+ * Copyright (C) 2000 - 2018, Intel Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index 6402f7fad3bb..1efefe919555 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -410,7 +410,52 @@ static void ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata, int
 		flags = 0;
 
 	if (flags != -1)
-		memory_failure_queue(pfn, 0, flags);
+		memory_failure_queue(pfn, flags);
+#endif
+}
+
+/*
+ * PCIe AER errors need to be sent to the AER driver for reporting and
+ * recovery. The GHES severities map to the following AER severities and
+ * require the following handling:
+ *
+ * GHES_SEV_CORRECTABLE -> AER_CORRECTABLE
+ *     These need to be reported by the AER driver but no recovery is
+ *     necessary.
+ * GHES_SEV_RECOVERABLE -> AER_NONFATAL
+ * GHES_SEV_RECOVERABLE && CPER_SEC_RESET -> AER_FATAL
+ *     These both need to be reported and recovered from by the AER driver.
+ * GHES_SEV_PANIC does not make it to this handling since the kernel must
+ *     panic.
+ */
+static void ghes_handle_aer(struct acpi_hest_generic_data *gdata)
+{
+#ifdef CONFIG_ACPI_APEI_PCIEAER
+	struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata);
+
+	if (pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID &&
+	    pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) {
+		unsigned int devfn;
+		int aer_severity;
+
+		devfn = PCI_DEVFN(pcie_err->device_id.device,
+				  pcie_err->device_id.function);
+		aer_severity = cper_severity_to_aer(gdata->error_severity);
+
+		/*
+		 * If firmware reset the component to contain
+		 * the error, we must reinitialize it before
+		 * use, so treat it as a fatal AER error.
+		 */
+		if (gdata->flags & CPER_SEC_RESET)
+			aer_severity = AER_FATAL;
+
+		aer_recover_queue(pcie_err->device_id.segment,
+				  pcie_err->device_id.bus,
+				  devfn, aer_severity,
+				  (struct aer_capability_regs *)
+				  pcie_err->aer_info);
+	}
 #endif
 }
 
@@ -441,38 +486,9 @@ static void ghes_do_proc(struct ghes *ghes,
 			arch_apei_report_mem_error(sev, mem_err);
 			ghes_handle_memory_failure(gdata, sev);
 		}
-#ifdef CONFIG_ACPI_APEI_PCIEAER
 		else if (guid_equal(sec_type, &CPER_SEC_PCIE)) {
-			struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata);
-
-			if (sev == GHES_SEV_RECOVERABLE &&
-			    sec_sev == GHES_SEV_RECOVERABLE &&
-			    pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID &&
-			    pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) {
-				unsigned int devfn;
-				int aer_severity;
-
-				devfn = PCI_DEVFN(pcie_err->device_id.device,
-						  pcie_err->device_id.function);
-				aer_severity = cper_severity_to_aer(gdata->error_severity);
-
-				/*
-				 * If firmware reset the component to contain
-				 * the error, we must reinitialize it before
-				 * use, so treat it as a fatal AER error.
-				 */
-				if (gdata->flags & CPER_SEC_RESET)
-					aer_severity = AER_FATAL;
-
-				aer_recover_queue(pcie_err->device_id.segment,
-						  pcie_err->device_id.bus,
-						  devfn, aer_severity,
-						  (struct aer_capability_regs *)
-						  pcie_err->aer_info);
-			}
-
+			ghes_handle_aer(gdata);
 		}
-#endif
 		else if (guid_equal(sec_type, &CPER_SEC_PROC_ARM)) {
 			struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata);
 
@@ -870,7 +886,6 @@ static void ghes_print_queued_estatus(void)
 	struct ghes_estatus_node *estatus_node;
 	struct acpi_hest_generic *generic;
 	struct acpi_hest_generic_status *estatus;
-	u32 len, node_len;
 
 	llnode = llist_del_all(&ghes_estatus_llist);
 	/*
@@ -882,8 +897,6 @@ static void ghes_print_queued_estatus(void)
 		estatus_node = llist_entry(llnode, struct ghes_estatus_node,
 					   llnode);
 		estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
-		len = cper_estatus_len(estatus);
-		node_len = GHES_ESTATUS_NODE_LEN(len);
 		generic = estatus_node->generic;
 		ghes_print_estatus(NULL, generic, estatus);
 		llnode = llnode->next;
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index 13e7b56e33ae..f2eb6c37ea0a 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -1237,13 +1237,11 @@ static int acpi_battery_add(struct acpi_device *device)
 
 #ifdef CONFIG_ACPI_PROCFS_POWER
 	result = acpi_battery_add_fs(device);
-#endif
 	if (result) {
-#ifdef CONFIG_ACPI_PROCFS_POWER
 		acpi_battery_remove_fs(device);
-#endif
 		goto fail;
 	}
+#endif
 
 	printk(KERN_INFO PREFIX "%s Slot [%s] (battery %s)\n",
 		ACPI_BATTERY_DEVICE_NAME, acpi_device_bid(device),
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index 4d0979e02a28..0dad0bd9327b 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -66,10 +66,37 @@ static int set_copy_dsdt(const struct dmi_system_id *id)
 	return 0;
 }
 #endif
+static int set_gbl_term_list(const struct dmi_system_id *id)
+{
+	acpi_gbl_parse_table_as_term_list = 1;
+	return 0;
+}
 
-static const struct dmi_system_id dsdt_dmi_table[] __initconst = {
+static const struct dmi_system_id acpi_quirks_dmi_table[] __initconst = {
+	/*
+	 * Touchpad on Dell XPS 9570/Precision M5530 doesn't work under I2C
+	 * mode.
+	 * https://bugzilla.kernel.org/show_bug.cgi?id=198515
+	 */
+	{
+		.callback = set_gbl_term_list,
+		.ident = "Dell Precision M5530",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Precision M5530"),
+		},
+	},
+	{
+		.callback = set_gbl_term_list,
+		.ident = "Dell XPS 15 9570",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "XPS 15 9570"),
+		},
+	},
 	/*
 	 * Invoke DSDT corruption work-around on all Toshiba Satellite.
+	 * DSDT will be copied to memory.
 	 * https://bugzilla.kernel.org/show_bug.cgi?id=14679
 	 */
 	{
@@ -83,7 +110,7 @@ static const struct dmi_system_id dsdt_dmi_table[] __initconst = {
 	{}
 };
 #else
-static const struct dmi_system_id dsdt_dmi_table[] __initconst = {
+static const struct dmi_system_id acpi_quirks_dmi_table[] __initconst = {
 	{}
 };
 #endif
@@ -108,6 +135,7 @@ acpi_status acpi_bus_get_status_handle(acpi_handle handle,
 	}
 	return status;
 }
+EXPORT_SYMBOL_GPL(acpi_bus_get_status_handle);
 
 int acpi_bus_get_status(struct acpi_device *device)
 {
@@ -119,6 +147,12 @@ int acpi_bus_get_status(struct acpi_device *device)
 		return 0;
 	}
 
+	/* Battery devices must have their deps met before calling _STA */
+	if (acpi_device_is_battery(device) && device->dep_unmet) {
+		acpi_set_device_status(device, 0);
+		return 0;
+	}
+
 	status = acpi_bus_get_status_handle(device->handle, &sta);
 	if (ACPI_FAILURE(status))
 		return -ENODEV;
@@ -626,13 +660,15 @@ struct acpi_device *acpi_companion_match(const struct device *dev)
  * acpi_of_match_device - Match device object using the "compatible" property.
  * @adev: ACPI device object to match.
  * @of_match_table: List of device IDs to match against.
+ * @of_id: OF ID if matched
  *
  * If @dev has an ACPI companion which has ACPI_DT_NAMESPACE_HID in its list of
  * identifiers and a _DSD object with the "compatible" property, use that
  * property to match against the given list of identifiers.
  */
 static bool acpi_of_match_device(struct acpi_device *adev,
-				 const struct of_device_id *of_match_table)
+				 const struct of_device_id *of_match_table,
+				 const struct of_device_id **of_id)
 {
 	const union acpi_object *of_compatible, *obj;
 	int i, nval;
@@ -656,8 +692,11 @@ static bool acpi_of_match_device(struct acpi_device *adev,
 		const struct of_device_id *id;
 
 		for (id = of_match_table; id->compatible[0]; id++)
-			if (!strcasecmp(obj->string.pointer, id->compatible))
+			if (!strcasecmp(obj->string.pointer, id->compatible)) {
+				if (of_id)
+					*of_id = id;
 				return true;
+			}
 	}
 
 	return false;
@@ -728,10 +767,11 @@ static bool __acpi_match_device_cls(const struct acpi_device_id *id,
 	return true;
 }
 
-static const struct acpi_device_id *__acpi_match_device(
-	struct acpi_device *device,
-	const struct acpi_device_id *ids,
-	const struct of_device_id *of_ids)
+static bool __acpi_match_device(struct acpi_device *device,
+				const struct acpi_device_id *acpi_ids,
+				const struct of_device_id *of_ids,
+				const struct acpi_device_id **acpi_id,
+				const struct of_device_id **of_id)
 {
 	const struct acpi_device_id *id;
 	struct acpi_hardware_id *hwid;
@@ -741,30 +781,32 @@ static const struct acpi_device_id *__acpi_match_device(
 	 * driver for it.
 	 */
 	if (!device || !device->status.present)
-		return NULL;
+		return false;
 
 	list_for_each_entry(hwid, &device->pnp.ids, list) {
 		/* First, check the ACPI/PNP IDs provided by the caller. */
-		for (id = ids; id->id[0] || id->cls; id++) {
-			if (id->id[0] && !strcmp((char *) id->id, hwid->id))
-				return id;
-			else if (id->cls && __acpi_match_device_cls(id, hwid))
-				return id;
+		if (acpi_ids) {
+			for (id = acpi_ids; id->id[0] || id->cls; id++) {
+				if (id->id[0] && !strcmp((char *)id->id, hwid->id))
+					goto out_acpi_match;
+				if (id->cls && __acpi_match_device_cls(id, hwid))
+					goto out_acpi_match;
+			}
 		}
 
 		/*
 		 * Next, check ACPI_DT_NAMESPACE_HID and try to match the
 		 * "compatible" property if found.
-		 *
-		 * The id returned by the below is not valid, but the only
-		 * caller passing non-NULL of_ids here is only interested in
-		 * whether or not the return value is NULL.
 		 */
-		if (!strcmp(ACPI_DT_NAMESPACE_HID, hwid->id)
-		    && acpi_of_match_device(device, of_ids))
-			return id;
+		if (!strcmp(ACPI_DT_NAMESPACE_HID, hwid->id))
+			return acpi_of_match_device(device, of_ids, of_id);
 	}
-	return NULL;
+	return false;
+
+out_acpi_match:
+	if (acpi_id)
+		*acpi_id = id;
+	return true;
 }
 
 /**
@@ -781,14 +823,29 @@ static const struct acpi_device_id *__acpi_match_device(
 const struct acpi_device_id *acpi_match_device(const struct acpi_device_id *ids,
 					       const struct device *dev)
 {
-	return __acpi_match_device(acpi_companion_match(dev), ids, NULL);
+	const struct acpi_device_id *id = NULL;
+
+	__acpi_match_device(acpi_companion_match(dev), ids, NULL, &id, NULL);
+	return id;
 }
 EXPORT_SYMBOL_GPL(acpi_match_device);
 
+const void *acpi_device_get_match_data(const struct device *dev)
+{
+	const struct acpi_device_id *match;
+
+	match = acpi_match_device(dev->driver->acpi_match_table, dev);
+	if (!match)
+		return NULL;
+
+	return (const void *)match->driver_data;
+}
+EXPORT_SYMBOL_GPL(acpi_device_get_match_data);
+
 int acpi_match_device_ids(struct acpi_device *device,
 			  const struct acpi_device_id *ids)
 {
-	return __acpi_match_device(device, ids, NULL) ? 0 : -ENOENT;
+	return __acpi_match_device(device, ids, NULL, NULL, NULL) ? 0 : -ENOENT;
 }
 EXPORT_SYMBOL(acpi_match_device_ids);
 
@@ -797,10 +854,12 @@ bool acpi_driver_match_device(struct device *dev,
 {
 	if (!drv->acpi_match_table)
 		return acpi_of_match_device(ACPI_COMPANION(dev),
-					    drv->of_match_table);
+					    drv->of_match_table,
+					    NULL);
 
-	return !!__acpi_match_device(acpi_companion_match(dev),
-				     drv->acpi_match_table, drv->of_match_table);
+	return __acpi_match_device(acpi_companion_match(dev),
+				   drv->acpi_match_table, drv->of_match_table,
+				   NULL, NULL);
 }
 EXPORT_SYMBOL_GPL(acpi_driver_match_device);
 
@@ -1001,11 +1060,8 @@ void __init acpi_early_init(void)
 
 	acpi_permanent_mmap = true;
 
-	/*
-	 * If the machine falls into the DMI check table,
-	 * DSDT will be copied to memory
-	 */
-	dmi_check_system(dsdt_dmi_table);
+	/* Check machine-specific quirks */
+	dmi_check_system(acpi_quirks_dmi_table);
 
 	status = acpi_reallocate_root_table();
 	if (ACPI_FAILURE(status)) {
diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c
index bf8e4d371fa7..e1eee7a60fad 100644
--- a/drivers/acpi/button.c
+++ b/drivers/acpi/button.c
@@ -30,6 +30,7 @@
 #include <linux/input.h>
 #include <linux/slab.h>
 #include <linux/acpi.h>
+#include <linux/dmi.h>
 #include <acpi/button.h>
 
 #define PREFIX "ACPI: "
@@ -76,6 +77,22 @@ static const struct acpi_device_id button_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, button_device_ids);
 
+/*
+ * Some devices which don't even have a lid in anyway have a broken _LID
+ * method (e.g. pointing to a floating gpio pin) causing spurious LID events.
+ */
+static const struct dmi_system_id lid_blacklst[] = {
+	{
+		/* GP-electronic T701 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Insyde"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "T701"),
+			DMI_MATCH(DMI_BIOS_VERSION, "BYT70A.YNCHENG.WIN.007"),
+		},
+	},
+	{}
+};
+
 static int acpi_button_add(struct acpi_device *device);
 static int acpi_button_remove(struct acpi_device *device);
 static void acpi_button_notify(struct acpi_device *device, u32 event);
@@ -210,6 +227,8 @@ static int acpi_lid_notify_state(struct acpi_device *device, int state)
 	}
 	/* Send the platform triggered reliable event */
 	if (do_update) {
+		acpi_handle_debug(device->handle, "ACPI LID %s\n",
+				  state ? "open" : "closed");
 		input_report_switch(button->input, SW_LID, !state);
 		input_sync(button->input);
 		button->last_state = !!state;
@@ -473,6 +492,9 @@ static int acpi_button_add(struct acpi_device *device)
 	char *name, *class;
 	int error;
 
+	if (!strcmp(hid, ACPI_BUTTON_HID_LID) && dmi_check_system(lid_blacklst))
+		return -ENODEV;
+
 	button = kzalloc(sizeof(struct acpi_button), GFP_KERNEL);
 	if (!button)
 		return -ENOMEM;
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index 06ea4749ebd9..0afbb2658cbc 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -119,7 +119,7 @@ static DEFINE_PER_CPU(struct cpc_desc *, cpc_desc_ptr);
  * to PCC commands. Keeping it high enough to cover emulators where
  * the processors run painfully slow.
  */
-#define NUM_RETRIES 500
+#define NUM_RETRIES 500ULL
 
 struct cppc_attr {
 	struct attribute attr;
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index a4c8ad98560d..c4d0a1c912f0 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -990,7 +990,7 @@ void acpi_subsys_complete(struct device *dev)
 	 * the sleep state it is going out of and it has never been resumed till
 	 * now, resume it in case the firmware powered it up.
 	 */
-	if (dev->power.direct_complete && pm_resume_via_firmware())
+	if (pm_runtime_suspended(dev) && pm_resume_via_firmware())
 		pm_request_resume(dev);
 }
 EXPORT_SYMBOL_GPL(acpi_subsys_complete);
@@ -1039,10 +1039,28 @@ EXPORT_SYMBOL_GPL(acpi_subsys_suspend_late);
  */
 int acpi_subsys_suspend_noirq(struct device *dev)
 {
-	if (dev_pm_smart_suspend_and_suspended(dev))
+	int ret;
+
+	if (dev_pm_smart_suspend_and_suspended(dev)) {
+		dev->power.may_skip_resume = true;
 		return 0;
+	}
+
+	ret = pm_generic_suspend_noirq(dev);
+	if (ret)
+		return ret;
+
+	/*
+	 * If the target system sleep state is suspend-to-idle, it is sufficient
+	 * to check whether or not the device's wakeup settings are good for
+	 * runtime PM.  Otherwise, the pm_resume_via_firmware() check will cause
+	 * acpi_subsys_complete() to take care of fixing up the device's state
+	 * anyway, if need be.
+	 */
+	dev->power.may_skip_resume = device_may_wakeup(dev) ||
+					!device_can_wakeup(dev);
 
-	return pm_generic_suspend_noirq(dev);
+	return 0;
 }
 EXPORT_SYMBOL_GPL(acpi_subsys_suspend_noirq);
 
@@ -1052,6 +1070,9 @@ EXPORT_SYMBOL_GPL(acpi_subsys_suspend_noirq);
  */
 int acpi_subsys_resume_noirq(struct device *dev)
 {
+	if (dev_pm_may_skip_resume(dev))
+		return 0;
+
 	/*
 	 * Devices with DPM_FLAG_SMART_SUSPEND may be left in runtime suspend
 	 * during system suspend, so update their runtime PM status to "active"
diff --git a/drivers/acpi/device_sysfs.c b/drivers/acpi/device_sysfs.c
index a041689e5701..545e91420cde 100644
--- a/drivers/acpi/device_sysfs.c
+++ b/drivers/acpi/device_sysfs.c
@@ -357,7 +357,7 @@ static ssize_t real_power_state_show(struct device *dev,
 	return sprintf(buf, "%s\n", acpi_power_state_string(state));
 }
 
-static DEVICE_ATTR(real_power_state, 0444, real_power_state_show, NULL);
+static DEVICE_ATTR_RO(real_power_state);
 
 static ssize_t power_state_show(struct device *dev,
 				struct device_attribute *attr, char *buf)
@@ -367,7 +367,7 @@ static ssize_t power_state_show(struct device *dev,
 	return sprintf(buf, "%s\n", acpi_power_state_string(adev->power.state));
 }
 
-static DEVICE_ATTR(power_state, 0444, power_state_show, NULL);
+static DEVICE_ATTR_RO(power_state);
 
 static ssize_t
 acpi_eject_store(struct device *d, struct device_attribute *attr,
@@ -462,7 +462,7 @@ static ssize_t description_show(struct device *dev,
 
 	return result;
 }
-static DEVICE_ATTR(description, 0444, description_show, NULL);
+static DEVICE_ATTR_RO(description);
 
 static ssize_t
 acpi_device_sun_show(struct device *dev, struct device_attribute *attr,
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index 0252c9b9af3d..30a572956557 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -1516,7 +1516,7 @@ static int acpi_ec_setup(struct acpi_ec *ec, bool handle_events)
 	}
 
 	acpi_handle_info(ec->handle,
-			 "GPE=0x%lx, EC_CMD/EC_SC=0x%lx, EC_DATA=0x%lx\n",
+			 "GPE=0x%x, EC_CMD/EC_SC=0x%lx, EC_DATA=0x%lx\n",
 			 ec->gpe, ec->command_addr, ec->data_addr);
 	return ret;
 }
@@ -1927,6 +1927,9 @@ static int acpi_ec_suspend_noirq(struct device *dev)
 	    ec->reference_count >= 1)
 		acpi_set_gpe(NULL, ec->gpe, ACPI_GPE_DISABLE);
 
+	if (acpi_sleep_no_ec_events())
+		acpi_ec_enter_noirq(ec);
+
 	return 0;
 }
 
@@ -1934,6 +1937,9 @@ static int acpi_ec_resume_noirq(struct device *dev)
 {
 	struct acpi_ec *ec = acpi_driver_data(to_acpi_device(dev));
 
+	if (acpi_sleep_no_ec_events())
+		acpi_ec_leave_noirq(ec);
+
 	if (ec_no_wakeup && test_bit(EC_FLAGS_STARTED, &ec->flags) &&
 	    ec->reference_count >= 1)
 		acpi_set_gpe(NULL, ec->gpe, ACPI_GPE_ENABLE);
diff --git a/drivers/acpi/ec_sys.c b/drivers/acpi/ec_sys.c
index 6c7dd7af789e..dd70d6c2bca0 100644
--- a/drivers/acpi/ec_sys.c
+++ b/drivers/acpi/ec_sys.c
@@ -128,7 +128,7 @@ static int acpi_ec_add_debugfs(struct acpi_ec *ec, unsigned int ec_device_count)
 		return -ENOMEM;
 	}
 
-	if (!debugfs_create_x32("gpe", 0444, dev_dir, (u32 *)&first_ec->gpe))
+	if (!debugfs_create_x32("gpe", 0444, dev_dir, &first_ec->gpe))
 		goto error;
 	if (!debugfs_create_bool("use_global_lock", 0444, dev_dir,
 				 &first_ec->global_lock))
diff --git a/drivers/acpi/evged.c b/drivers/acpi/evged.c
index 46f060356a22..f13ba2c07667 100644
--- a/drivers/acpi/evged.c
+++ b/drivers/acpi/evged.c
@@ -49,6 +49,11 @@
 
 #define MODULE_NAME	"acpi-ged"
 
+struct acpi_ged_device {
+	struct device *dev;
+	struct list_head event_list;
+};
+
 struct acpi_ged_event {
 	struct list_head node;
 	struct device *dev;
@@ -76,7 +81,8 @@ static acpi_status acpi_ged_request_interrupt(struct acpi_resource *ares,
 	unsigned int irq;
 	unsigned int gsi;
 	unsigned int irqflags = IRQF_ONESHOT;
-	struct device *dev = context;
+	struct acpi_ged_device *geddev = context;
+	struct device *dev = geddev->dev;
 	acpi_handle handle = ACPI_HANDLE(dev);
 	acpi_handle evt_handle;
 	struct resource r;
@@ -102,8 +108,6 @@ static acpi_status acpi_ged_request_interrupt(struct acpi_resource *ares,
 		return AE_ERROR;
 	}
 
-	dev_info(dev, "GED listening GSI %u @ IRQ %u\n", gsi, irq);
-
 	event = devm_kzalloc(dev, sizeof(*event), GFP_KERNEL);
 	if (!event)
 		return AE_ERROR;
@@ -116,29 +120,58 @@ static acpi_status acpi_ged_request_interrupt(struct acpi_resource *ares,
 	if (r.flags & IORESOURCE_IRQ_SHAREABLE)
 		irqflags |= IRQF_SHARED;
 
-	if (devm_request_threaded_irq(dev, irq, NULL, acpi_ged_irq_handler,
-				      irqflags, "ACPI:Ged", event)) {
+	if (request_threaded_irq(irq, NULL, acpi_ged_irq_handler,
+				 irqflags, "ACPI:Ged", event)) {
 		dev_err(dev, "failed to setup event handler for irq %u\n", irq);
 		return AE_ERROR;
 	}
 
+	dev_dbg(dev, "GED listening GSI %u @ IRQ %u\n", gsi, irq);
+	list_add_tail(&event->node, &geddev->event_list);
 	return AE_OK;
 }
 
 static int ged_probe(struct platform_device *pdev)
 {
+	struct acpi_ged_device *geddev;
 	acpi_status acpi_ret;
 
+	geddev = devm_kzalloc(&pdev->dev, sizeof(*geddev), GFP_KERNEL);
+	if (!geddev)
+		return -ENOMEM;
+
+	geddev->dev = &pdev->dev;
+	INIT_LIST_HEAD(&geddev->event_list);
 	acpi_ret = acpi_walk_resources(ACPI_HANDLE(&pdev->dev), "_CRS",
-				       acpi_ged_request_interrupt, &pdev->dev);
+				       acpi_ged_request_interrupt, geddev);
 	if (ACPI_FAILURE(acpi_ret)) {
 		dev_err(&pdev->dev, "unable to parse the _CRS record\n");
 		return -EINVAL;
 	}
+	platform_set_drvdata(pdev, geddev);
 
 	return 0;
 }
 
+static void ged_shutdown(struct platform_device *pdev)
+{
+	struct acpi_ged_device *geddev = platform_get_drvdata(pdev);
+	struct acpi_ged_event *event, *next;
+
+	list_for_each_entry_safe(event, next, &geddev->event_list, node) {
+		free_irq(event->irq, event);
+		list_del(&event->node);
+		dev_dbg(geddev->dev, "GED releasing GSI %u @ IRQ %u\n",
+			 event->gsi, event->irq);
+	}
+}
+
+static int ged_remove(struct platform_device *pdev)
+{
+	ged_shutdown(pdev);
+	return 0;
+}
+
 static const struct acpi_device_id ged_acpi_ids[] = {
 	{"ACPI0013"},
 	{},
@@ -146,6 +179,8 @@ static const struct acpi_device_id ged_acpi_ids[] = {
 
 static struct platform_driver ged_driver = {
 	.probe = ged_probe,
+	.remove = ged_remove,
+	.shutdown = ged_shutdown,
 	.driver = {
 		.name = MODULE_NAME,
 		.acpi_match_table = ACPI_PTR(ged_acpi_ids),
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index 7f43423de43c..1d0a501bc7f0 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -159,7 +159,7 @@ static inline void acpi_early_processor_osc(void) {}
    -------------------------------------------------------------------------- */
 struct acpi_ec {
 	acpi_handle handle;
-	unsigned long gpe;
+	u32 gpe;
 	unsigned long command_addr;
 	unsigned long data_addr;
 	bool global_lock;
diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c
index abeb4df4f22e..eb09ef55c38a 100644
--- a/drivers/acpi/nfit/core.c
+++ b/drivers/acpi/nfit/core.c
@@ -838,6 +838,18 @@ static bool add_flush(struct acpi_nfit_desc *acpi_desc,
 	return true;
 }
 
+static bool add_platform_cap(struct acpi_nfit_desc *acpi_desc,
+		struct acpi_nfit_capabilities *pcap)
+{
+	struct device *dev = acpi_desc->dev;
+	u32 mask;
+
+	mask = (1 << (pcap->highest_capability + 1)) - 1;
+	acpi_desc->platform_cap = pcap->capabilities & mask;
+	dev_dbg(dev, "%s: cap: %#x\n", __func__, acpi_desc->platform_cap);
+	return true;
+}
+
 static void *add_table(struct acpi_nfit_desc *acpi_desc,
 		struct nfit_table_prev *prev, void *table, const void *end)
 {
@@ -883,6 +895,10 @@ static void *add_table(struct acpi_nfit_desc *acpi_desc,
 	case ACPI_NFIT_TYPE_SMBIOS:
 		dev_dbg(dev, "%s: smbios\n", __func__);
 		break;
+	case ACPI_NFIT_TYPE_CAPABILITIES:
+		if (!add_platform_cap(acpi_desc, table))
+			return err;
+		break;
 	default:
 		dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
 		break;
@@ -1867,6 +1883,9 @@ static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
 		struct kernfs_node *nfit_kernfs;
 
 		nvdimm = nfit_mem->nvdimm;
+		if (!nvdimm)
+			continue;
+
 		nfit_kernfs = sysfs_get_dirent(nvdimm_kobj(nvdimm)->sd, "nfit");
 		if (nfit_kernfs)
 			nfit_mem->flags_attr = sysfs_get_dirent(nfit_kernfs,
@@ -2656,6 +2675,16 @@ static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
 	else
 		ndr_desc->numa_node = NUMA_NO_NODE;
 
+	/*
+	 * Persistence domain bits are hierarchical, if
+	 * ACPI_NFIT_CAPABILITY_CACHE_FLUSH is set then
+	 * ACPI_NFIT_CAPABILITY_MEM_FLUSH is implied.
+	 */
+	if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_CACHE_FLUSH)
+		set_bit(ND_REGION_PERSIST_CACHE, &ndr_desc->flags);
+	else if (acpi_desc->platform_cap & ACPI_NFIT_CAPABILITY_MEM_FLUSH)
+		set_bit(ND_REGION_PERSIST_MEMCTRL, &ndr_desc->flags);
+
 	list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 		struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
 		struct nd_mapping_desc *mapping;
@@ -3464,6 +3493,7 @@ static __init int nfit_init(void)
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
 	BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
+	BUILD_BUG_ON(sizeof(struct acpi_nfit_capabilities) != 16);
 
 	guid_parse(UUID_VOLATILE_MEMORY, &nfit_uuid[NFIT_SPA_VOLATILE]);
 	guid_parse(UUID_PERSISTENT_MEMORY, &nfit_uuid[NFIT_SPA_PM]);
diff --git a/drivers/acpi/nfit/nfit.h b/drivers/acpi/nfit/nfit.h
index f0cf18b2da8b..50d36e166d70 100644
--- a/drivers/acpi/nfit/nfit.h
+++ b/drivers/acpi/nfit/nfit.h
@@ -202,6 +202,7 @@ struct acpi_nfit_desc {
 	unsigned long dimm_cmd_force_en;
 	unsigned long bus_cmd_force_en;
 	unsigned long bus_nfit_cmd_force_en;
+	unsigned int platform_cap;
 	int (*blk_do_io)(struct nd_blk_region *ndbr, resource_size_t dpa,
 			void *iobuf, u64 len, int rw);
 };
diff --git a/drivers/acpi/numa.c b/drivers/acpi/numa.c
index 917f1cc0fda4..85167603b9c9 100644
--- a/drivers/acpi/numa.c
+++ b/drivers/acpi/numa.c
@@ -103,25 +103,27 @@ int acpi_map_pxm_to_node(int pxm)
  */
 int acpi_map_pxm_to_online_node(int pxm)
 {
-	int node, n, dist, min_dist;
+	int node, min_node;
 
 	node = acpi_map_pxm_to_node(pxm);
 
 	if (node == NUMA_NO_NODE)
 		node = 0;
 
+	min_node = node;
 	if (!node_online(node)) {
-		min_dist = INT_MAX;
+		int min_dist = INT_MAX, dist, n;
+
 		for_each_online_node(n) {
 			dist = node_distance(node, n);
 			if (dist < min_dist) {
 				min_dist = dist;
-				node = n;
+				min_node = n;
 			}
 		}
 	}
 
-	return node;
+	return min_node;
 }
 EXPORT_SYMBOL(acpi_map_pxm_to_online_node);
 
@@ -460,8 +462,7 @@ int __init acpi_numa_init(void)
 					srat_proc, ARRAY_SIZE(srat_proc), 0);
 
 		cnt = acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY,
-					    acpi_parse_memory_affinity,
-					    NR_NODE_MEMBLKS);
+					    acpi_parse_memory_affinity, 0);
 	}
 
 	/* SLIT: System Locality Information Table */
diff --git a/drivers/acpi/pci_link.c b/drivers/acpi/pci_link.c
index bc3d914dfc3e..85ad679390e3 100644
--- a/drivers/acpi/pci_link.c
+++ b/drivers/acpi/pci_link.c
@@ -612,7 +612,7 @@ static int acpi_pci_link_allocate(struct acpi_pci_link *link)
 			acpi_isa_irq_penalty[link->irq.active] +=
 				PIRQ_PENALTY_PCI_USING;
 
-		printk(KERN_WARNING PREFIX "%s [%s] enabled at IRQ %d\n",
+		pr_info("%s [%s] enabled at IRQ %d\n",
 		       acpi_device_name(link->device),
 		       acpi_device_bid(link->device), link->irq.active);
 	}
diff --git a/drivers/acpi/pmic/intel_pmic_bxtwc.c b/drivers/acpi/pmic/intel_pmic_bxtwc.c
index 90011aad4d20..886ac8b93cd0 100644
--- a/drivers/acpi/pmic/intel_pmic_bxtwc.c
+++ b/drivers/acpi/pmic/intel_pmic_bxtwc.c
@@ -400,7 +400,7 @@ static int intel_bxtwc_pmic_opregion_probe(struct platform_device *pdev)
 			&intel_bxtwc_pmic_opregion_data);
 }
 
-static struct platform_device_id bxt_wc_opregion_id_table[] = {
+static const struct platform_device_id bxt_wc_opregion_id_table[] = {
 	{ .name = "bxt_wcove_region" },
 	{},
 };
@@ -412,9 +412,4 @@ static struct platform_driver intel_bxtwc_pmic_opregion_driver = {
 	},
 	.id_table = bxt_wc_opregion_id_table,
 };
-
-static int __init intel_bxtwc_pmic_opregion_driver_init(void)
-{
-	return platform_driver_register(&intel_bxtwc_pmic_opregion_driver);
-}
-device_initcall(intel_bxtwc_pmic_opregion_driver_init);
+builtin_platform_driver(intel_bxtwc_pmic_opregion_driver);
diff --git a/drivers/acpi/pmic/intel_pmic_chtdc_ti.c b/drivers/acpi/pmic/intel_pmic_chtdc_ti.c
index 109c1e9c9c7a..f6d73a243d80 100644
--- a/drivers/acpi/pmic/intel_pmic_chtdc_ti.c
+++ b/drivers/acpi/pmic/intel_pmic_chtdc_ti.c
@@ -131,7 +131,4 @@ static struct platform_driver chtdc_ti_pmic_opregion_driver = {
 	},
 	.id_table = chtdc_ti_pmic_opregion_id_table,
 };
-module_platform_driver(chtdc_ti_pmic_opregion_driver);
-
-MODULE_DESCRIPTION("Dollar Cove TI PMIC opregion driver");
-MODULE_LICENSE("GPL v2");
+builtin_platform_driver(chtdc_ti_pmic_opregion_driver);
diff --git a/drivers/acpi/pmic/intel_pmic_chtwc.c b/drivers/acpi/pmic/intel_pmic_chtwc.c
index 85636d7a9d39..9912422c8185 100644
--- a/drivers/acpi/pmic/intel_pmic_chtwc.c
+++ b/drivers/acpi/pmic/intel_pmic_chtwc.c
@@ -260,11 +260,10 @@ static int intel_cht_wc_pmic_opregion_probe(struct platform_device *pdev)
 			&intel_cht_wc_pmic_opregion_data);
 }
 
-static struct platform_device_id cht_wc_opregion_id_table[] = {
+static const struct platform_device_id cht_wc_opregion_id_table[] = {
 	{ .name = "cht_wcove_region" },
 	{},
 };
-MODULE_DEVICE_TABLE(platform, cht_wc_opregion_id_table);
 
 static struct platform_driver intel_cht_wc_pmic_opregion_driver = {
 	.probe = intel_cht_wc_pmic_opregion_probe,
@@ -273,8 +272,4 @@ static struct platform_driver intel_cht_wc_pmic_opregion_driver = {
 	},
 	.id_table = cht_wc_opregion_id_table,
 };
-module_platform_driver(intel_cht_wc_pmic_opregion_driver);
-
-MODULE_DESCRIPTION("Intel CHT Whiskey Cove PMIC operation region driver");
-MODULE_AUTHOR("Hans de Goede <hdegoede@redhat.com>");
-MODULE_LICENSE("GPL");
+builtin_platform_driver(intel_cht_wc_pmic_opregion_driver);
diff --git a/drivers/acpi/pmic/intel_pmic_crc.c b/drivers/acpi/pmic/intel_pmic_crc.c
index d7f1761ab1bc..7ffa74048107 100644
--- a/drivers/acpi/pmic/intel_pmic_crc.c
+++ b/drivers/acpi/pmic/intel_pmic_crc.c
@@ -201,9 +201,4 @@ static struct platform_driver intel_crc_pmic_opregion_driver = {
 		.name = "crystal_cove_pmic",
 	},
 };
-
-static int __init intel_crc_pmic_opregion_driver_init(void)
-{
-	return platform_driver_register(&intel_crc_pmic_opregion_driver);
-}
-device_initcall(intel_crc_pmic_opregion_driver_init);
+builtin_platform_driver(intel_crc_pmic_opregion_driver);
diff --git a/drivers/acpi/pmic/intel_pmic_xpower.c b/drivers/acpi/pmic/intel_pmic_xpower.c
index 6c99d3f81095..316e55174aa9 100644
--- a/drivers/acpi/pmic/intel_pmic_xpower.c
+++ b/drivers/acpi/pmic/intel_pmic_xpower.c
@@ -278,9 +278,4 @@ static struct platform_driver intel_xpower_pmic_opregion_driver = {
 		.name = "axp288_pmic_acpi",
 	},
 };
-
-static int __init intel_xpower_pmic_opregion_driver_init(void)
-{
-	return platform_driver_register(&intel_xpower_pmic_opregion_driver);
-}
-device_initcall(intel_xpower_pmic_opregion_driver_init);
+builtin_platform_driver(intel_xpower_pmic_opregion_driver);
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index d50a7b6ccddd..abb559cd28d7 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -207,6 +207,7 @@ static void tsc_check_state(int state)
 	switch (boot_cpu_data.x86_vendor) {
 	case X86_VENDOR_AMD:
 	case X86_VENDOR_INTEL:
+	case X86_VENDOR_CENTAUR:
 		/*
 		 * AMD Fam10h TSC will tick in all
 		 * C/P/S0/S1 states when this bit is set.
@@ -291,6 +292,9 @@ static int acpi_processor_get_power_info_default(struct acpi_processor *pr)
 		pr->power.states[ACPI_STATE_C1].type = ACPI_STATE_C1;
 		pr->power.states[ACPI_STATE_C1].valid = 1;
 		pr->power.states[ACPI_STATE_C1].entry_method = ACPI_CSTATE_HALT;
+
+		snprintf(pr->power.states[ACPI_STATE_C1].desc,
+			 ACPI_CX_DESC_LEN, "ACPI HLT");
 	}
 	/* the C0 state only exists as a filler in our array */
 	pr->power.states[ACPI_STATE_C0].valid = 1;
diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c
index 18b72eec3507..c7cf48ad5cb9 100644
--- a/drivers/acpi/processor_perflib.c
+++ b/drivers/acpi/processor_perflib.c
@@ -159,7 +159,7 @@ void acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag)
 {
 	int ret;
 
-	if (ignore_ppc) {
+	if (ignore_ppc || !pr->performance) {
 		/*
 		 * Only when it is notification event, the _OST object
 		 * will be evaluated. Otherwise it is skipped.
diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
index e26ea209b63e..5815356ea6ad 100644
--- a/drivers/acpi/property.c
+++ b/drivers/acpi/property.c
@@ -1271,9 +1271,17 @@ static int acpi_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
 	return 0;
 }
 
+static const void *
+acpi_fwnode_device_get_match_data(const struct fwnode_handle *fwnode,
+				  const struct device *dev)
+{
+	return acpi_device_get_match_data(dev);
+}
+
 #define DECLARE_ACPI_FWNODE_OPS(ops) \
 	const struct fwnode_operations ops = {				\
 		.device_is_available = acpi_fwnode_device_is_available, \
+		.device_get_match_data = acpi_fwnode_device_get_match_data, \
 		.property_present = acpi_fwnode_property_present,	\
 		.property_read_int_array =				\
 			acpi_fwnode_property_read_int_array,		\
diff --git a/drivers/acpi/sbshc.c b/drivers/acpi/sbshc.c
index 2fa8304171e0..7a3431018e0a 100644
--- a/drivers/acpi/sbshc.c
+++ b/drivers/acpi/sbshc.c
@@ -275,8 +275,8 @@ static int acpi_smbus_hc_add(struct acpi_device *device)
 	device->driver_data = hc;
 
 	acpi_ec_add_query_handler(hc->ec, hc->query_bit, NULL, smbus_alarm, hc);
-	printk(KERN_INFO PREFIX "SBS HC: EC = 0x%p, offset = 0x%0x, query_bit = 0x%0x\n",
-		hc->ec, hc->offset, hc->query_bit);
+	dev_info(&device->dev, "SBS HC: offset = 0x%0x, query_bit = 0x%0x\n",
+		 hc->offset, hc->query_bit);
 
 	return 0;
 }
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index b0fe5272c76a..8e63d937babb 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -1565,6 +1565,8 @@ void acpi_init_device_object(struct acpi_device *device, acpi_handle handle,
 	device_initialize(&device->dev);
 	dev_set_uevent_suppress(&device->dev, true);
 	acpi_init_coherency(device);
+	/* Assume there are unmet deps until acpi_device_dep_initialize() runs */
+	device->dep_unmet = 1;
 }
 
 void acpi_device_add_finalize(struct acpi_device *device)
@@ -1588,6 +1590,14 @@ static int acpi_add_single_object(struct acpi_device **child,
 	}
 
 	acpi_init_device_object(device, handle, type, sta);
+	/*
+	 * For ACPI_BUS_TYPE_DEVICE getting the status is delayed till here so
+	 * that we can call acpi_bus_get_status() and use its quirk handling.
+	 * Note this must be done before the get power-/wakeup_dev-flags calls.
+	 */
+	if (type == ACPI_BUS_TYPE_DEVICE)
+		acpi_bus_get_status(device);
+
 	acpi_bus_get_power_flags(device);
 	acpi_bus_get_wakeup_device_flags(device);
 
@@ -1660,9 +1670,11 @@ static int acpi_bus_type_and_status(acpi_handle handle, int *type,
 			return -ENODEV;
 
 		*type = ACPI_BUS_TYPE_DEVICE;
-		status = acpi_bus_get_status_handle(handle, sta);
-		if (ACPI_FAILURE(status))
-			*sta = 0;
+		/*
+		 * acpi_add_single_object updates this once we've an acpi_device
+		 * so that acpi_bus_get_status' quirk handling can be used.
+		 */
+		*sta = 0;
 		break;
 	case ACPI_TYPE_PROCESSOR:
 		*type = ACPI_BUS_TYPE_PROCESSOR;
@@ -1760,6 +1772,8 @@ static void acpi_device_dep_initialize(struct acpi_device *adev)
 	acpi_status status;
 	int i;
 
+	adev->dep_unmet = 0;
+
 	if (!acpi_has_method(adev->handle, "_DEP"))
 		return;
 
diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c
index 8082871b409a..46cde0912762 100644
--- a/drivers/acpi/sleep.c
+++ b/drivers/acpi/sleep.c
@@ -367,10 +367,20 @@ static const struct dmi_system_id acpisleep_dmi_table[] __initconst = {
 	{},
 };
 
+static bool ignore_blacklist;
+
+void __init acpi_sleep_no_blacklist(void)
+{
+	ignore_blacklist = true;
+}
+
 static void __init acpi_sleep_dmi_check(void)
 {
 	int year;
 
+	if (ignore_blacklist)
+		return;
+
 	if (dmi_get_date(DMI_BIOS_DATE, &year, NULL, NULL) && year >= 2012)
 		acpi_nvs_nosave_s3();
 
@@ -697,7 +707,8 @@ static const struct acpi_device_id lps0_device_ids[] = {
 #define ACPI_LPS0_ENTRY		5
 #define ACPI_LPS0_EXIT		6
 
-#define ACPI_S2IDLE_FUNC_MASK	((1 << ACPI_LPS0_ENTRY) | (1 << ACPI_LPS0_EXIT))
+#define ACPI_LPS0_SCREEN_MASK	((1 << ACPI_LPS0_SCREEN_OFF) | (1 << ACPI_LPS0_SCREEN_ON))
+#define ACPI_LPS0_PLATFORM_MASK	((1 << ACPI_LPS0_ENTRY) | (1 << ACPI_LPS0_EXIT))
 
 static acpi_handle lps0_device_handle;
 static guid_t lps0_dsm_guid;
@@ -900,7 +911,8 @@ static int lps0_device_attach(struct acpi_device *adev,
 	if (out_obj && out_obj->type == ACPI_TYPE_BUFFER) {
 		char bitmask = *(char *)out_obj->buffer.pointer;
 
-		if ((bitmask & ACPI_S2IDLE_FUNC_MASK) == ACPI_S2IDLE_FUNC_MASK) {
+		if ((bitmask & ACPI_LPS0_PLATFORM_MASK) == ACPI_LPS0_PLATFORM_MASK ||
+		    (bitmask & ACPI_LPS0_SCREEN_MASK) == ACPI_LPS0_SCREEN_MASK) {
 			lps0_dsm_func_mask = bitmask;
 			lps0_device_handle = adev->handle;
 			/*
diff --git a/drivers/acpi/spcr.c b/drivers/acpi/spcr.c
index 324b35bfe781..9d52743080a4 100644
--- a/drivers/acpi/spcr.c
+++ b/drivers/acpi/spcr.c
@@ -21,7 +21,7 @@
  * occasionally getting stuck as 1. To avoid the potential for a hang, check
  * TXFE == 0 instead of BUSY == 1. This may not be suitable for all UART
  * implementations, so only do so if an affected platform is detected in
- * parse_spcr().
+ * acpi_parse_spcr().
  */
 bool qdf2400_e44_present;
 EXPORT_SYMBOL(qdf2400_e44_present);
@@ -74,19 +74,21 @@ static bool xgene_8250_erratum_present(struct acpi_table_spcr *tb)
 }
 
 /**
- * parse_spcr() - parse ACPI SPCR table and add preferred console
+ * acpi_parse_spcr() - parse ACPI SPCR table and add preferred console
  *
- * @earlycon: set up earlycon for the console specified by the table
+ * @enable_earlycon: set up earlycon for the console specified by the table
+ * @enable_console: setup the console specified by the table.
  *
  * For the architectures with support for ACPI, CONFIG_ACPI_SPCR_TABLE may be
  * defined to parse ACPI SPCR table.  As a result of the parsing preferred
- * console is registered and if @earlycon is true, earlycon is set up.
+ * console is registered and if @enable_earlycon is true, earlycon is set up.
+ * If @enable_console is true the system console is also configured.
  *
  * When CONFIG_ACPI_SPCR_TABLE is defined, this function should be called
  * from arch initialization code as soon as the DT/ACPI decision is made.
  *
  */
-int __init parse_spcr(bool earlycon)
+int __init acpi_parse_spcr(bool enable_earlycon, bool enable_console)
 {
 	static char opts[64];
 	struct acpi_table_spcr *table;
@@ -105,17 +107,15 @@ int __init parse_spcr(bool earlycon)
 	if (ACPI_FAILURE(status))
 		return -ENOENT;
 
-	if (table->header.revision < 2) {
-		err = -ENOENT;
-		pr_err("wrong table version\n");
-		goto done;
-	}
+	if (table->header.revision < 2)
+		pr_info("SPCR table version %d\n", table->header.revision);
 
 	if (table->serial_port.space_id == ACPI_ADR_SPACE_SYSTEM_MEMORY) {
 		switch (ACPI_ACCESS_BIT_WIDTH((
 			table->serial_port.access_width))) {
 		default:
 			pr_err("Unexpected SPCR Access Width.  Defaulting to byte size\n");
+			/* fall through */
 		case 8:
 			iotype = "mmio";
 			break;
@@ -185,7 +185,7 @@ int __init parse_spcr(bool earlycon)
 	 */
 	if (qdf2400_erratum_44_present(&table->header)) {
 		qdf2400_e44_present = true;
-		if (earlycon)
+		if (enable_earlycon)
 			uart = "qdf2400_e44";
 	}
 
@@ -205,11 +205,13 @@ int __init parse_spcr(bool earlycon)
 
 	pr_info("console: %s\n", opts);
 
-	if (earlycon)
+	if (enable_earlycon)
 		setup_earlycon(opts);
 
-	err = add_preferred_console(uart, 0, opts + strlen(uart) + 1);
-
+	if (enable_console)
+		err = add_preferred_console(uart, 0, opts + strlen(uart) + 1);
+	else
+		err = 0;
 done:
 	acpi_put_table((struct acpi_table_header *)table);
 	return err;
diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c
index 06a150bb35bf..4fc59c3bc673 100644
--- a/drivers/acpi/sysfs.c
+++ b/drivers/acpi/sysfs.c
@@ -816,14 +816,8 @@ static ssize_t counter_set(struct kobject *kobj,
  * interface:
  *   echo unmask > /sys/firmware/acpi/interrupts/gpe00
  */
-
-/*
- * Currently, the GPE flooding prevention only supports to mask the GPEs
- * numbered from 00 to 7f.
- */
-#define ACPI_MASKABLE_GPE_MAX	0x80
-
-static u64 __initdata acpi_masked_gpes;
+#define ACPI_MASKABLE_GPE_MAX	0xFF
+static DECLARE_BITMAP(acpi_masked_gpes_map, ACPI_MASKABLE_GPE_MAX) __initdata;
 
 static int __init acpi_gpe_set_masked_gpes(char *val)
 {
@@ -831,7 +825,7 @@ static int __init acpi_gpe_set_masked_gpes(char *val)
 
 	if (kstrtou8(val, 0, &gpe) || gpe > ACPI_MASKABLE_GPE_MAX)
 		return -EINVAL;
-	acpi_masked_gpes |= ((u64)1<<gpe);
+	set_bit(gpe, acpi_masked_gpes_map);
 
 	return 1;
 }
@@ -843,15 +837,11 @@ void __init acpi_gpe_apply_masked_gpes(void)
 	acpi_status status;
 	u8 gpe;
 
-	for (gpe = 0;
-	     gpe < min_t(u8, ACPI_MASKABLE_GPE_MAX, acpi_current_gpe_count);
-	     gpe++) {
-		if (acpi_masked_gpes & ((u64)1<<gpe)) {
-			status = acpi_get_gpe_device(gpe, &handle);
-			if (ACPI_SUCCESS(status)) {
-				pr_info("Masking GPE 0x%x.\n", gpe);
-				(void)acpi_mask_gpe(handle, gpe, TRUE);
-			}
+	for_each_set_bit(gpe, acpi_masked_gpes_map, ACPI_MASKABLE_GPE_MAX) {
+		status = acpi_get_gpe_device(gpe, &handle);
+		if (ACPI_SUCCESS(status)) {
+			pr_info("Masking GPE 0x%x.\n", gpe);
+			(void)acpi_mask_gpe(handle, gpe, TRUE);
 		}
 	}
 }
diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c
index 80ce2a7d224b..7bcb66ccccf3 100644
--- a/drivers/acpi/tables.c
+++ b/drivers/acpi/tables.c
@@ -456,7 +456,8 @@ static const char * const table_sigs[] = {
 	ACPI_SIG_SLIC, ACPI_SIG_SPCR, ACPI_SIG_SPMI, ACPI_SIG_TCPA,
 	ACPI_SIG_UEFI, ACPI_SIG_WAET, ACPI_SIG_WDAT, ACPI_SIG_WDDT,
 	ACPI_SIG_WDRT, ACPI_SIG_DSDT, ACPI_SIG_FADT, ACPI_SIG_PSDT,
-	ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT, NULL };
+	ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT, ACPI_SIG_IORT,
+	NULL };
 
 #define ACPI_HEADER_SIZE sizeof(struct acpi_table_header)
 
diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c
index 9d49a1acebe3..78db97687f26 100644
--- a/drivers/acpi/utils.c
+++ b/drivers/acpi/utils.c
@@ -737,16 +737,17 @@ bool acpi_dev_found(const char *hid)
 }
 EXPORT_SYMBOL(acpi_dev_found);
 
-struct acpi_dev_present_info {
+struct acpi_dev_match_info {
+	const char *dev_name;
 	struct acpi_device_id hid[2];
 	const char *uid;
 	s64 hrv;
 };
 
-static int acpi_dev_present_cb(struct device *dev, void *data)
+static int acpi_dev_match_cb(struct device *dev, void *data)
 {
 	struct acpi_device *adev = to_acpi_device(dev);
-	struct acpi_dev_present_info *match = data;
+	struct acpi_dev_match_info *match = data;
 	unsigned long long hrv;
 	acpi_status status;
 
@@ -757,6 +758,8 @@ static int acpi_dev_present_cb(struct device *dev, void *data)
 	    strcmp(adev->pnp.unique_id, match->uid)))
 		return 0;
 
+	match->dev_name = acpi_dev_name(adev);
+
 	if (match->hrv == -1)
 		return 1;
 
@@ -789,20 +792,44 @@ static int acpi_dev_present_cb(struct device *dev, void *data)
  */
 bool acpi_dev_present(const char *hid, const char *uid, s64 hrv)
 {
-	struct acpi_dev_present_info match = {};
+	struct acpi_dev_match_info match = {};
 	struct device *dev;
 
 	strlcpy(match.hid[0].id, hid, sizeof(match.hid[0].id));
 	match.uid = uid;
 	match.hrv = hrv;
 
-	dev = bus_find_device(&acpi_bus_type, NULL, &match,
-			      acpi_dev_present_cb);
-
+	dev = bus_find_device(&acpi_bus_type, NULL, &match, acpi_dev_match_cb);
 	return !!dev;
 }
 EXPORT_SYMBOL(acpi_dev_present);
 
+/**
+ * acpi_dev_get_first_match_name - Return name of first match of ACPI device
+ * @hid: Hardware ID of the device.
+ * @uid: Unique ID of the device, pass NULL to not check _UID
+ * @hrv: Hardware Revision of the device, pass -1 to not check _HRV
+ *
+ * Return device name if a matching device was present
+ * at the moment of invocation, or NULL otherwise.
+ *
+ * See additional information in acpi_dev_present() as well.
+ */
+const char *
+acpi_dev_get_first_match_name(const char *hid, const char *uid, s64 hrv)
+{
+	struct acpi_dev_match_info match = {};
+	struct device *dev;
+
+	strlcpy(match.hid[0].id, hid, sizeof(match.hid[0].id));
+	match.uid = uid;
+	match.hrv = hrv;
+
+	dev = bus_find_device(&acpi_bus_type, NULL, &match, acpi_dev_match_cb);
+	return dev ? match.dev_name : NULL;
+}
+EXPORT_SYMBOL(acpi_dev_get_first_match_name);
+
 /*
  * acpi_backlight= handling, this is done here rather then in video_detect.c
  * because __setup cannot be used in modules.
diff --git a/drivers/android/binder.c b/drivers/android/binder.c
index a7ecfde66b7b..764b63a5aade 100644
--- a/drivers/android/binder.c
+++ b/drivers/android/binder.c
@@ -141,7 +141,7 @@ enum {
 };
 static uint32_t binder_debug_mask = BINDER_DEBUG_USER_ERROR |
 	BINDER_DEBUG_FAILED_TRANSACTION | BINDER_DEBUG_DEAD_TRANSACTION;
-module_param_named(debug_mask, binder_debug_mask, uint, S_IWUSR | S_IRUGO);
+module_param_named(debug_mask, binder_debug_mask, uint, 0644);
 
 static char *binder_devices_param = CONFIG_ANDROID_BINDER_DEVICES;
 module_param_named(devices, binder_devices_param, charp, 0444);
@@ -160,7 +160,7 @@ static int binder_set_stop_on_user_error(const char *val,
 	return ret;
 }
 module_param_call(stop_on_user_error, binder_set_stop_on_user_error,
-	param_get_int, &binder_stop_on_user_error, S_IWUSR | S_IRUGO);
+	param_get_int, &binder_stop_on_user_error, 0644);
 
 #define binder_debug(mask, x...) \
 	do { \
@@ -249,7 +249,7 @@ static struct binder_transaction_log_entry *binder_transaction_log_add(
 	unsigned int cur = atomic_inc_return(&log->cur);
 
 	if (cur >= ARRAY_SIZE(log->entry))
-		log->full = 1;
+		log->full = true;
 	e = &log->entry[cur % ARRAY_SIZE(log->entry)];
 	WRITE_ONCE(e->debug_id_done, 0);
 	/*
@@ -493,8 +493,6 @@ enum binder_deferred_state {
  *                        (protected by @inner_lock)
  * @todo:                 list of work for this process
  *                        (protected by @inner_lock)
- * @wait:                 wait queue head to wait for proc work
- *                        (invariant after initialized)
  * @stats:                per-process binder statistics
  *                        (atomics, no lock needed)
  * @delivered_death:      list of delivered death notification
@@ -537,7 +535,6 @@ struct binder_proc {
 	bool is_dead;
 
 	struct list_head todo;
-	wait_queue_head_t wait;
 	struct binder_stats stats;
 	struct list_head delivered_death;
 	int max_threads;
@@ -579,6 +576,8 @@ enum {
  *                        (protected by @proc->inner_lock)
  * @todo:                 list of work to do for this thread
  *                        (protected by @proc->inner_lock)
+ * @process_todo:         whether work in @todo should be processed
+ *                        (protected by @proc->inner_lock)
  * @return_error:         transaction errors reported by this thread
  *                        (only accessed by this thread)
  * @reply_error:          transaction errors reported by target thread
@@ -604,6 +603,7 @@ struct binder_thread {
 	bool looper_need_return; /* can be written by other thread */
 	struct binder_transaction *transaction_stack;
 	struct list_head todo;
+	bool process_todo;
 	struct binder_error return_error;
 	struct binder_error reply_error;
 	wait_queue_head_t wait;
@@ -789,6 +789,16 @@ static bool binder_worklist_empty(struct binder_proc *proc,
 	return ret;
 }
 
+/**
+ * binder_enqueue_work_ilocked() - Add an item to the work list
+ * @work:         struct binder_work to add to list
+ * @target_list:  list to add work to
+ *
+ * Adds the work to the specified list. Asserts that work
+ * is not already on a list.
+ *
+ * Requires the proc->inner_lock to be held.
+ */
 static void
 binder_enqueue_work_ilocked(struct binder_work *work,
 			   struct list_head *target_list)
@@ -799,22 +809,56 @@ binder_enqueue_work_ilocked(struct binder_work *work,
 }
 
 /**
- * binder_enqueue_work() - Add an item to the work list
- * @proc:         binder_proc associated with list
+ * binder_enqueue_deferred_thread_work_ilocked() - Add deferred thread work
+ * @thread:       thread to queue work to
  * @work:         struct binder_work to add to list
- * @target_list:  list to add work to
  *
- * Adds the work to the specified list. Asserts that work
- * is not already on a list.
+ * Adds the work to the todo list of the thread. Doesn't set the process_todo
+ * flag, which means that (if it wasn't already set) the thread will go to
+ * sleep without handling this work when it calls read.
+ *
+ * Requires the proc->inner_lock to be held.
  */
 static void
-binder_enqueue_work(struct binder_proc *proc,
-		    struct binder_work *work,
-		    struct list_head *target_list)
+binder_enqueue_deferred_thread_work_ilocked(struct binder_thread *thread,
+					    struct binder_work *work)
 {
-	binder_inner_proc_lock(proc);
-	binder_enqueue_work_ilocked(work, target_list);
-	binder_inner_proc_unlock(proc);
+	binder_enqueue_work_ilocked(work, &thread->todo);
+}
+
+/**
+ * binder_enqueue_thread_work_ilocked() - Add an item to the thread work list
+ * @thread:       thread to queue work to
+ * @work:         struct binder_work to add to list
+ *
+ * Adds the work to the todo list of the thread, and enables processing
+ * of the todo queue.
+ *
+ * Requires the proc->inner_lock to be held.
+ */
+static void
+binder_enqueue_thread_work_ilocked(struct binder_thread *thread,
+				   struct binder_work *work)
+{
+	binder_enqueue_work_ilocked(work, &thread->todo);
+	thread->process_todo = true;
+}
+
+/**
+ * binder_enqueue_thread_work() - Add an item to the thread work list
+ * @thread:       thread to queue work to
+ * @work:         struct binder_work to add to list
+ *
+ * Adds the work to the todo list of the thread, and enables processing
+ * of the todo queue.
+ */
+static void
+binder_enqueue_thread_work(struct binder_thread *thread,
+			   struct binder_work *work)
+{
+	binder_inner_proc_lock(thread->proc);
+	binder_enqueue_thread_work_ilocked(thread, work);
+	binder_inner_proc_unlock(thread->proc);
 }
 
 static void
@@ -940,7 +984,7 @@ static long task_close_fd(struct binder_proc *proc, unsigned int fd)
 static bool binder_has_work_ilocked(struct binder_thread *thread,
 				    bool do_proc_work)
 {
-	return !binder_worklist_empty_ilocked(&thread->todo) ||
+	return thread->process_todo ||
 		thread->looper_need_return ||
 		(do_proc_work &&
 		 !binder_worklist_empty_ilocked(&thread->proc->todo));
@@ -1228,6 +1272,17 @@ static int binder_inc_node_nilocked(struct binder_node *node, int strong,
 			node->local_strong_refs++;
 		if (!node->has_strong_ref && target_list) {
 			binder_dequeue_work_ilocked(&node->work);
+			/*
+			 * Note: this function is the only place where we queue
+			 * directly to a thread->todo without using the
+			 * corresponding binder_enqueue_thread_work() helper
+			 * functions; in this case it's ok to not set the
+			 * process_todo flag, since we know this node work will
+			 * always be followed by other work that starts queue
+			 * processing: in case of synchronous transactions, a
+			 * BR_REPLY or BR_ERROR; in case of oneway
+			 * transactions, a BR_TRANSACTION_COMPLETE.
+			 */
 			binder_enqueue_work_ilocked(&node->work, target_list);
 		}
 	} else {
@@ -1239,6 +1294,9 @@ static int binder_inc_node_nilocked(struct binder_node *node, int strong,
 					node->debug_id);
 				return -EINVAL;
 			}
+			/*
+			 * See comment above
+			 */
 			binder_enqueue_work_ilocked(&node->work, target_list);
 		}
 	}
@@ -1928,13 +1986,19 @@ static void binder_send_failed_reply(struct binder_transaction *t,
 			binder_pop_transaction_ilocked(target_thread, t);
 			if (target_thread->reply_error.cmd == BR_OK) {
 				target_thread->reply_error.cmd = error_code;
-				binder_enqueue_work_ilocked(
-					&target_thread->reply_error.work,
-					&target_thread->todo);
+				binder_enqueue_thread_work_ilocked(
+					target_thread,
+					&target_thread->reply_error.work);
 				wake_up_interruptible(&target_thread->wait);
 			} else {
-				WARN(1, "Unexpected reply error: %u\n",
-						target_thread->reply_error.cmd);
+				/*
+				 * Cannot get here for normal operation, but
+				 * we can if multiple synchronous transactions
+				 * are sent without blocking for responses.
+				 * Just ignore the 2nd error in this case.
+				 */
+				pr_warn("Unexpected reply error: %u\n",
+					target_thread->reply_error.cmd);
 			}
 			binder_inner_proc_unlock(target_thread->proc);
 			binder_thread_dec_tmpref(target_thread);
@@ -2135,7 +2199,7 @@ static void binder_transaction_buffer_release(struct binder_proc *proc,
 	int debug_id = buffer->debug_id;
 
 	binder_debug(BINDER_DEBUG_TRANSACTION,
-		     "%d buffer release %d, size %zd-%zd, failed at %p\n",
+		     "%d buffer release %d, size %zd-%zd, failed at %pK\n",
 		     proc->pid, buffer->debug_id,
 		     buffer->data_size, buffer->offsets_size, failed_at);
 
@@ -2569,20 +2633,18 @@ static bool binder_proc_transaction(struct binder_transaction *t,
 				    struct binder_proc *proc,
 				    struct binder_thread *thread)
 {
-	struct list_head *target_list = NULL;
 	struct binder_node *node = t->buffer->target_node;
 	bool oneway = !!(t->flags & TF_ONE_WAY);
-	bool wakeup = true;
+	bool pending_async = false;
 
 	BUG_ON(!node);
 	binder_node_lock(node);
 	if (oneway) {
 		BUG_ON(thread);
 		if (node->has_async_transaction) {
-			target_list = &node->async_todo;
-			wakeup = false;
+			pending_async = true;
 		} else {
-			node->has_async_transaction = 1;
+			node->has_async_transaction = true;
 		}
 	}
 
@@ -2594,19 +2656,17 @@ static bool binder_proc_transaction(struct binder_transaction *t,
 		return false;
 	}
 
-	if (!thread && !target_list)
+	if (!thread && !pending_async)
 		thread = binder_select_thread_ilocked(proc);
 
 	if (thread)
-		target_list = &thread->todo;
-	else if (!target_list)
-		target_list = &proc->todo;
+		binder_enqueue_thread_work_ilocked(thread, &t->work);
+	else if (!pending_async)
+		binder_enqueue_work_ilocked(&t->work, &proc->todo);
 	else
-		BUG_ON(target_list != &node->async_todo);
-
-	binder_enqueue_work_ilocked(&t->work, target_list);
+		binder_enqueue_work_ilocked(&t->work, &node->async_todo);
 
-	if (wakeup)
+	if (!pending_async)
 		binder_wakeup_thread_ilocked(proc, thread, !oneway /* sync */);
 
 	binder_inner_proc_unlock(proc);
@@ -3101,10 +3161,10 @@ static void binder_transaction(struct binder_proc *proc,
 		}
 	}
 	tcomplete->type = BINDER_WORK_TRANSACTION_COMPLETE;
-	binder_enqueue_work(proc, tcomplete, &thread->todo);
 	t->work.type = BINDER_WORK_TRANSACTION;
 
 	if (reply) {
+		binder_enqueue_thread_work(thread, tcomplete);
 		binder_inner_proc_lock(target_proc);
 		if (target_thread->is_dead) {
 			binder_inner_proc_unlock(target_proc);
@@ -3112,13 +3172,21 @@ static void binder_transaction(struct binder_proc *proc,
 		}
 		BUG_ON(t->buffer->async_transaction != 0);
 		binder_pop_transaction_ilocked(target_thread, in_reply_to);
-		binder_enqueue_work_ilocked(&t->work, &target_thread->todo);
+		binder_enqueue_thread_work_ilocked(target_thread, &t->work);
 		binder_inner_proc_unlock(target_proc);
 		wake_up_interruptible_sync(&target_thread->wait);
 		binder_free_transaction(in_reply_to);
 	} else if (!(t->flags & TF_ONE_WAY)) {
 		BUG_ON(t->buffer->async_transaction != 0);
 		binder_inner_proc_lock(proc);
+		/*
+		 * Defer the TRANSACTION_COMPLETE, so we don't return to
+		 * userspace immediately; this allows the target process to
+		 * immediately start processing this transaction, reducing
+		 * latency. We will then return the TRANSACTION_COMPLETE when
+		 * the target replies (or there is an error).
+		 */
+		binder_enqueue_deferred_thread_work_ilocked(thread, tcomplete);
 		t->need_reply = 1;
 		t->from_parent = thread->transaction_stack;
 		thread->transaction_stack = t;
@@ -3132,6 +3200,7 @@ static void binder_transaction(struct binder_proc *proc,
 	} else {
 		BUG_ON(target_node == NULL);
 		BUG_ON(t->buffer->async_transaction != 1);
+		binder_enqueue_thread_work(thread, tcomplete);
 		if (!binder_proc_transaction(t, target_proc, NULL))
 			goto err_dead_proc_or_thread;
 	}
@@ -3210,15 +3279,11 @@ static void binder_transaction(struct binder_proc *proc,
 	BUG_ON(thread->return_error.cmd != BR_OK);
 	if (in_reply_to) {
 		thread->return_error.cmd = BR_TRANSACTION_COMPLETE;
-		binder_enqueue_work(thread->proc,
-				    &thread->return_error.work,
-				    &thread->todo);
+		binder_enqueue_thread_work(thread, &thread->return_error.work);
 		binder_send_failed_reply(in_reply_to, return_error);
 	} else {
 		thread->return_error.cmd = return_error;
-		binder_enqueue_work(thread->proc,
-				    &thread->return_error.work,
-				    &thread->todo);
+		binder_enqueue_thread_work(thread, &thread->return_error.work);
 	}
 }
 
@@ -3424,7 +3489,7 @@ static int binder_thread_write(struct binder_proc *proc,
 				w = binder_dequeue_work_head_ilocked(
 						&buf_node->async_todo);
 				if (!w) {
-					buf_node->has_async_transaction = 0;
+					buf_node->has_async_transaction = false;
 				} else {
 					binder_enqueue_work_ilocked(
 							w, &proc->todo);
@@ -3522,10 +3587,9 @@ static int binder_thread_write(struct binder_proc *proc,
 					WARN_ON(thread->return_error.cmd !=
 						BR_OK);
 					thread->return_error.cmd = BR_ERROR;
-					binder_enqueue_work(
-						thread->proc,
-						&thread->return_error.work,
-						&thread->todo);
+					binder_enqueue_thread_work(
+						thread,
+						&thread->return_error.work);
 					binder_debug(
 						BINDER_DEBUG_FAILED_TRANSACTION,
 						"%d:%d BC_REQUEST_DEATH_NOTIFICATION failed\n",
@@ -3605,9 +3669,9 @@ static int binder_thread_write(struct binder_proc *proc,
 					if (thread->looper &
 					    (BINDER_LOOPER_STATE_REGISTERED |
 					     BINDER_LOOPER_STATE_ENTERED))
-						binder_enqueue_work_ilocked(
-								&death->work,
-								&thread->todo);
+						binder_enqueue_thread_work_ilocked(
+								thread,
+								&death->work);
 					else {
 						binder_enqueue_work_ilocked(
 								&death->work,
@@ -3647,7 +3711,7 @@ static int binder_thread_write(struct binder_proc *proc,
 				}
 			}
 			binder_debug(BINDER_DEBUG_DEAD_BINDER,
-				     "%d:%d BC_DEAD_BINDER_DONE %016llx found %p\n",
+				     "%d:%d BC_DEAD_BINDER_DONE %016llx found %pK\n",
 				     proc->pid, thread->pid, (u64)cookie,
 				     death);
 			if (death == NULL) {
@@ -3662,8 +3726,8 @@ static int binder_thread_write(struct binder_proc *proc,
 				if (thread->looper &
 					(BINDER_LOOPER_STATE_REGISTERED |
 					 BINDER_LOOPER_STATE_ENTERED))
-					binder_enqueue_work_ilocked(
-						&death->work, &thread->todo);
+					binder_enqueue_thread_work_ilocked(
+						thread, &death->work);
 				else {
 					binder_enqueue_work_ilocked(
 							&death->work,
@@ -3837,6 +3901,8 @@ static int binder_thread_read(struct binder_proc *proc,
 			break;
 		}
 		w = binder_dequeue_work_head_ilocked(list);
+		if (binder_worklist_empty_ilocked(&thread->todo))
+			thread->process_todo = false;
 
 		switch (w->type) {
 		case BINDER_WORK_TRANSACTION: {
@@ -4302,8 +4368,29 @@ static int binder_thread_release(struct binder_proc *proc,
 		if (t)
 			spin_lock(&t->lock);
 	}
+
+	/*
+	 * If this thread used poll, make sure we remove the waitqueue
+	 * from any epoll data structures holding it with POLLFREE.
+	 * waitqueue_active() is safe to use here because we're holding
+	 * the inner lock.
+	 */
+	if ((thread->looper & BINDER_LOOPER_STATE_POLL) &&
+	    waitqueue_active(&thread->wait)) {
+		wake_up_poll(&thread->wait, EPOLLHUP | POLLFREE);
+	}
+
 	binder_inner_proc_unlock(thread->proc);
 
+	/*
+	 * This is needed to avoid races between wake_up_poll() above and
+	 * and ep_remove_waitqueue() called for other reasons (eg the epoll file
+	 * descriptor being closed); ep_remove_waitqueue() holds an RCU read
+	 * lock, so we can be sure it's done after calling synchronize_rcu().
+	 */
+	if (thread->looper & BINDER_LOOPER_STATE_POLL)
+		synchronize_rcu();
+
 	if (send_reply)
 		binder_send_failed_reply(send_reply, BR_DEAD_REPLY);
 	binder_release_work(proc, &thread->todo);
@@ -4311,7 +4398,7 @@ static int binder_thread_release(struct binder_proc *proc,
 	return active_transactions;
 }
 
-static unsigned int binder_poll(struct file *filp,
+static __poll_t binder_poll(struct file *filp,
 				struct poll_table_struct *wait)
 {
 	struct binder_proc *proc = filp->private_data;
@@ -4319,6 +4406,8 @@ static unsigned int binder_poll(struct file *filp,
 	bool wait_for_proc_work;
 
 	thread = binder_get_thread(proc);
+	if (!thread)
+		return POLLERR;
 
 	binder_inner_proc_lock(thread->proc);
 	thread->looper |= BINDER_LOOPER_STATE_POLL;
@@ -4329,7 +4418,7 @@ static unsigned int binder_poll(struct file *filp,
 	poll_wait(filp, &thread->wait, wait);
 
 	if (binder_has_work(thread, wait_for_proc_work))
-		return POLLIN;
+		return EPOLLIN;
 
 	return 0;
 }
@@ -4646,7 +4735,7 @@ static int binder_mmap(struct file *filp, struct vm_area_struct *vma)
 	return 0;
 
 err_bad_arg:
-	pr_err("binder_mmap: %d %lx-%lx %s failed %d\n",
+	pr_err("%s: %d %lx-%lx %s failed %d\n", __func__,
 	       proc->pid, vma->vm_start, vma->vm_end, failure_string, ret);
 	return ret;
 }
@@ -4656,7 +4745,7 @@ static int binder_open(struct inode *nodp, struct file *filp)
 	struct binder_proc *proc;
 	struct binder_device *binder_dev;
 
-	binder_debug(BINDER_DEBUG_OPEN_CLOSE, "binder_open: %d:%d\n",
+	binder_debug(BINDER_DEBUG_OPEN_CLOSE, "%s: %d:%d\n", __func__,
 		     current->group_leader->pid, current->pid);
 
 	proc = kzalloc(sizeof(*proc), GFP_KERNEL);
@@ -4695,7 +4784,7 @@ static int binder_open(struct inode *nodp, struct file *filp)
 		 * anyway print all contexts that a given PID has, so this
 		 * is not a problem.
 		 */
-		proc->debugfs_entry = debugfs_create_file(strbuf, S_IRUGO,
+		proc->debugfs_entry = debugfs_create_file(strbuf, 0444,
 			binder_debugfs_dir_entry_proc,
 			(void *)(unsigned long)proc->pid,
 			&binder_proc_fops);
@@ -4962,7 +5051,7 @@ static void print_binder_transaction_ilocked(struct seq_file *m,
 	spin_lock(&t->lock);
 	to_proc = t->to_proc;
 	seq_printf(m,
-		   "%s %d: %p from %d:%d to %d:%d code %x flags %x pri %ld r%d",
+		   "%s %d: %pK from %d:%d to %d:%d code %x flags %x pri %ld r%d",
 		   prefix, t->debug_id, t,
 		   t->from ? t->from->proc->pid : 0,
 		   t->from ? t->from->pid : 0,
@@ -4986,7 +5075,7 @@ static void print_binder_transaction_ilocked(struct seq_file *m,
 	}
 	if (buffer->target_node)
 		seq_printf(m, " node %d", buffer->target_node->debug_id);
-	seq_printf(m, " size %zd:%zd data %p\n",
+	seq_printf(m, " size %zd:%zd data %pK\n",
 		   buffer->data_size, buffer->offsets_size,
 		   buffer->data);
 }
@@ -5524,7 +5613,9 @@ static int __init binder_init(void)
 	struct binder_device *device;
 	struct hlist_node *tmp;
 
-	binder_alloc_shrinker_init();
+	ret = binder_alloc_shrinker_init();
+	if (ret)
+		return ret;
 
 	atomic_set(&binder_transaction_log.cur, ~0U);
 	atomic_set(&binder_transaction_log_failed.cur, ~0U);
@@ -5536,27 +5627,27 @@ static int __init binder_init(void)
 
 	if (binder_debugfs_dir_entry_root) {
 		debugfs_create_file("state",
-				    S_IRUGO,
+				    0444,
 				    binder_debugfs_dir_entry_root,
 				    NULL,
 				    &binder_state_fops);
 		debugfs_create_file("stats",
-				    S_IRUGO,
+				    0444,
 				    binder_debugfs_dir_entry_root,
 				    NULL,
 				    &binder_stats_fops);
 		debugfs_create_file("transactions",
-				    S_IRUGO,
+				    0444,
 				    binder_debugfs_dir_entry_root,
 				    NULL,
 				    &binder_transactions_fops);
 		debugfs_create_file("transaction_log",
-				    S_IRUGO,
+				    0444,
 				    binder_debugfs_dir_entry_root,
 				    &binder_transaction_log,
 				    &binder_transaction_log_fops);
 		debugfs_create_file("failed_transaction_log",
-				    S_IRUGO,
+				    0444,
 				    binder_debugfs_dir_entry_root,
 				    &binder_transaction_log_failed,
 				    &binder_transaction_log_fops);
diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
index 6f6f745605af..5a426c877dfb 100644
--- a/drivers/android/binder_alloc.c
+++ b/drivers/android/binder_alloc.c
@@ -281,6 +281,9 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
 			goto err_vm_insert_page_failed;
 		}
 
+		if (index + 1 > alloc->pages_high)
+			alloc->pages_high = index + 1;
+
 		trace_binder_alloc_page_end(alloc, index);
 		/* vm_insert_page does not seem to increment the refcount */
 	}
@@ -324,11 +327,12 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
 	return vma ? -ENOMEM : -ESRCH;
 }
 
-struct binder_buffer *binder_alloc_new_buf_locked(struct binder_alloc *alloc,
-						  size_t data_size,
-						  size_t offsets_size,
-						  size_t extra_buffers_size,
-						  int is_async)
+static struct binder_buffer *binder_alloc_new_buf_locked(
+				struct binder_alloc *alloc,
+				size_t data_size,
+				size_t offsets_size,
+				size_t extra_buffers_size,
+				int is_async)
 {
 	struct rb_node *n = alloc->free_buffers.rb_node;
 	struct binder_buffer *buffer;
@@ -666,7 +670,7 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc,
 		goto err_already_mapped;
 	}
 
-	area = get_vm_area(vma->vm_end - vma->vm_start, VM_IOREMAP);
+	area = get_vm_area(vma->vm_end - vma->vm_start, VM_ALLOC);
 	if (area == NULL) {
 		ret = -ENOMEM;
 		failure_string = "get_vm_area";
@@ -853,6 +857,7 @@ void binder_alloc_print_pages(struct seq_file *m,
 	}
 	mutex_unlock(&alloc->mutex);
 	seq_printf(m, "  pages: %d:%d:%d\n", active, lru, free);
+	seq_printf(m, "  pages high watermark: %zu\n", alloc->pages_high);
 }
 
 /**
@@ -1002,8 +1007,14 @@ void binder_alloc_init(struct binder_alloc *alloc)
 	INIT_LIST_HEAD(&alloc->buffers);
 }
 
-void binder_alloc_shrinker_init(void)
+int binder_alloc_shrinker_init(void)
 {
-	list_lru_init(&binder_alloc_lru);
-	register_shrinker(&binder_shrinker);
+	int ret = list_lru_init(&binder_alloc_lru);
+
+	if (ret == 0) {
+		ret = register_shrinker(&binder_shrinker);
+		if (ret)
+			list_lru_destroy(&binder_alloc_lru);
+	}
+	return ret;
 }
diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h
index 2dd33b6df104..9ef64e563856 100644
--- a/drivers/android/binder_alloc.h
+++ b/drivers/android/binder_alloc.h
@@ -92,6 +92,7 @@ struct binder_lru_page {
  * @pages:              array of binder_lru_page
  * @buffer_size:        size of address space specified via mmap
  * @pid:                pid for associated binder_proc (invariant after init)
+ * @pages_high:         high watermark of offset in @pages
  *
  * Bookkeeping structure for per-proc address space management for binder
  * buffers. It is normally initialized during binder_init() and binder_mmap()
@@ -112,6 +113,7 @@ struct binder_alloc {
 	size_t buffer_size;
 	uint32_t buffer_free;
 	int pid;
+	size_t pages_high;
 };
 
 #ifdef CONFIG_ANDROID_BINDER_IPC_SELFTEST
@@ -128,7 +130,7 @@ extern struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,
 						  size_t extra_buffers_size,
 						  int is_async);
 extern void binder_alloc_init(struct binder_alloc *alloc);
-void binder_alloc_shrinker_init(void);
+extern int binder_alloc_shrinker_init(void);
 extern void binder_alloc_vma_close(struct binder_alloc *alloc);
 extern struct binder_buffer *
 binder_alloc_prepare_to_free(struct binder_alloc *alloc,
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig
index cb5339166563..a7120d621154 100644
--- a/drivers/ata/Kconfig
+++ b/drivers/ata/Kconfig
@@ -92,6 +92,25 @@ config SATA_AHCI
 
 	  If unsure, say N.
 
+config SATA_MOBILE_LPM_POLICY
+	int "Default SATA Link Power Management policy for mobile chipsets"
+	range 0 4
+	default 0
+	depends on SATA_AHCI
+	help
+	  Select the Default SATA Link Power Management (LPM) policy to use
+	  for mobile / laptop variants of chipsets / "South Bridges".
+
+	  The value set has the following meanings:
+		0 => Keep firmware settings
+		1 => Maximum performance
+		2 => Medium power
+		3 => Medium power with Device Initiated PM enabled
+		4 => Minimum power
+
+	  Note "Minimum power" is known to cause issues, including disk
+	  corruption, with some disks and should not be used.
+
 config SATA_AHCI_PLATFORM
 	tristate "Platform AHCI SATA support"
 	help
@@ -925,15 +944,6 @@ endif # ATA_BMDMA
 
 comment "PIO-only SFF controllers"
 
-config PATA_AT32
-	tristate "Atmel AVR32 PATA support (Experimental)"
-	depends on AVR32 && PLATFORM_AT32AP
-	help
-	  This option enables support for the IDE devices on the
-	  Atmel AT32AP platform.
-
-	  If unsure, say N.
-
 config PATA_CMD640_PCI
 	tristate "CMD640 PCI PATA support (Experimental)"
 	depends on PCI
diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile
index 8daec3e657f8..f1f5a3fbc777 100644
--- a/drivers/ata/Makefile
+++ b/drivers/ata/Makefile
@@ -96,7 +96,6 @@ obj-$(CONFIG_PATA_VIA)		+= pata_via.o
 obj-$(CONFIG_PATA_WINBOND)	+= pata_sl82c105.o
 
 # SFF PIO only
-obj-$(CONFIG_PATA_AT32)		+= pata_at32.o
 obj-$(CONFIG_PATA_CMD640_PCI)	+= pata_cmd640.o
 obj-$(CONFIG_PATA_FALCON)	+= pata_falcon.o
 obj-$(CONFIG_PATA_ISAPNP)	+= pata_isapnp.o
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index 5443cb71d7ba..1ff17799769d 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -65,6 +65,7 @@ enum board_ids {
 	/* board IDs by feature in alphabetical order */
 	board_ahci,
 	board_ahci_ign_iferr,
+	board_ahci_mobile,
 	board_ahci_nomsi,
 	board_ahci_noncq,
 	board_ahci_nosntf,
@@ -140,6 +141,13 @@ static const struct ata_port_info ahci_port_info[] = {
 		.udma_mask	= ATA_UDMA6,
 		.port_ops	= &ahci_ops,
 	},
+	[board_ahci_mobile] = {
+		AHCI_HFLAGS	(AHCI_HFLAG_IS_MOBILE),
+		.flags		= AHCI_FLAG_COMMON,
+		.pio_mask	= ATA_PIO4,
+		.udma_mask	= ATA_UDMA6,
+		.port_ops	= &ahci_ops,
+	},
 	[board_ahci_nomsi] = {
 		AHCI_HFLAGS	(AHCI_HFLAG_NO_MSI),
 		.flags		= AHCI_FLAG_COMMON,
@@ -252,13 +260,13 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
 	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
 	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
-	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
-	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
-	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
-	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
-	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
+	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci_mobile }, /* ICH9M */
+	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci_mobile }, /* ICH9M */
+	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci_mobile }, /* ICH9M */
+	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci_mobile }, /* ICH9M */
+	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci_mobile }, /* ICH9M */
 	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
-	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
+	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci_mobile }, /* ICH9M */
 	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
 	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
 	{ PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
@@ -268,9 +276,9 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0x3b23), board_ahci }, /* PCH AHCI */
 	{ PCI_VDEVICE(INTEL, 0x3b24), board_ahci }, /* PCH RAID */
 	{ PCI_VDEVICE(INTEL, 0x3b25), board_ahci }, /* PCH RAID */
-	{ PCI_VDEVICE(INTEL, 0x3b29), board_ahci }, /* PCH AHCI */
+	{ PCI_VDEVICE(INTEL, 0x3b29), board_ahci_mobile }, /* PCH M AHCI */
 	{ PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */
-	{ PCI_VDEVICE(INTEL, 0x3b2c), board_ahci }, /* PCH RAID */
+	{ PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_mobile }, /* PCH M RAID */
 	{ PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */
 	{ PCI_VDEVICE(INTEL, 0x19b0), board_ahci }, /* DNV AHCI */
 	{ PCI_VDEVICE(INTEL, 0x19b1), board_ahci }, /* DNV AHCI */
@@ -293,9 +301,9 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0x19cE), board_ahci }, /* DNV AHCI */
 	{ PCI_VDEVICE(INTEL, 0x19cF), board_ahci }, /* DNV AHCI */
 	{ PCI_VDEVICE(INTEL, 0x1c02), board_ahci }, /* CPT AHCI */
-	{ PCI_VDEVICE(INTEL, 0x1c03), board_ahci }, /* CPT AHCI */
+	{ PCI_VDEVICE(INTEL, 0x1c03), board_ahci_mobile }, /* CPT M AHCI */
 	{ PCI_VDEVICE(INTEL, 0x1c04), board_ahci }, /* CPT RAID */
-	{ PCI_VDEVICE(INTEL, 0x1c05), board_ahci }, /* CPT RAID */
+	{ PCI_VDEVICE(INTEL, 0x1c05), board_ahci_mobile }, /* CPT M RAID */
 	{ PCI_VDEVICE(INTEL, 0x1c06), board_ahci }, /* CPT RAID */
 	{ PCI_VDEVICE(INTEL, 0x1c07), board_ahci }, /* CPT RAID */
 	{ PCI_VDEVICE(INTEL, 0x1d02), board_ahci }, /* PBG AHCI */
@@ -304,28 +312,28 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0x2826), board_ahci }, /* PBG RAID */
 	{ PCI_VDEVICE(INTEL, 0x2323), board_ahci }, /* DH89xxCC AHCI */
 	{ PCI_VDEVICE(INTEL, 0x1e02), board_ahci }, /* Panther Point AHCI */
-	{ PCI_VDEVICE(INTEL, 0x1e03), board_ahci }, /* Panther Point AHCI */
+	{ PCI_VDEVICE(INTEL, 0x1e03), board_ahci_mobile }, /* Panther M AHCI */
 	{ PCI_VDEVICE(INTEL, 0x1e04), board_ahci }, /* Panther Point RAID */
 	{ PCI_VDEVICE(INTEL, 0x1e05), board_ahci }, /* Panther Point RAID */
 	{ PCI_VDEVICE(INTEL, 0x1e06), board_ahci }, /* Panther Point RAID */
-	{ PCI_VDEVICE(INTEL, 0x1e07), board_ahci }, /* Panther Point RAID */
+	{ PCI_VDEVICE(INTEL, 0x1e07), board_ahci_mobile }, /* Panther M RAID */
 	{ PCI_VDEVICE(INTEL, 0x1e0e), board_ahci }, /* Panther Point RAID */
 	{ PCI_VDEVICE(INTEL, 0x8c02), board_ahci }, /* Lynx Point AHCI */
-	{ PCI_VDEVICE(INTEL, 0x8c03), board_ahci }, /* Lynx Point AHCI */
+	{ PCI_VDEVICE(INTEL, 0x8c03), board_ahci_mobile }, /* Lynx M AHCI */
 	{ PCI_VDEVICE(INTEL, 0x8c04), board_ahci }, /* Lynx Point RAID */
-	{ PCI_VDEVICE(INTEL, 0x8c05), board_ahci }, /* Lynx Point RAID */
+	{ PCI_VDEVICE(INTEL, 0x8c05), board_ahci_mobile }, /* Lynx M RAID */
 	{ PCI_VDEVICE(INTEL, 0x8c06), board_ahci }, /* Lynx Point RAID */
-	{ PCI_VDEVICE(INTEL, 0x8c07), board_ahci }, /* Lynx Point RAID */
+	{ PCI_VDEVICE(INTEL, 0x8c07), board_ahci_mobile }, /* Lynx M RAID */
 	{ PCI_VDEVICE(INTEL, 0x8c0e), board_ahci }, /* Lynx Point RAID */
-	{ PCI_VDEVICE(INTEL, 0x8c0f), board_ahci }, /* Lynx Point RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c02), board_ahci }, /* Lynx Point-LP AHCI */
-	{ PCI_VDEVICE(INTEL, 0x9c03), board_ahci }, /* Lynx Point-LP AHCI */
-	{ PCI_VDEVICE(INTEL, 0x9c04), board_ahci }, /* Lynx Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c05), board_ahci }, /* Lynx Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c06), board_ahci }, /* Lynx Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c07), board_ahci }, /* Lynx Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c0e), board_ahci }, /* Lynx Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c0f), board_ahci }, /* Lynx Point-LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x8c0f), board_ahci_mobile }, /* Lynx M RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c02), board_ahci_mobile }, /* Lynx LP AHCI */
+	{ PCI_VDEVICE(INTEL, 0x9c03), board_ahci_mobile }, /* Lynx LP AHCI */
+	{ PCI_VDEVICE(INTEL, 0x9c04), board_ahci_mobile }, /* Lynx LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c05), board_ahci_mobile }, /* Lynx LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c06), board_ahci_mobile }, /* Lynx LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c07), board_ahci_mobile }, /* Lynx LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c0e), board_ahci_mobile }, /* Lynx LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c0f), board_ahci_mobile }, /* Lynx LP RAID */
 	{ PCI_VDEVICE(INTEL, 0x1f22), board_ahci }, /* Avoton AHCI */
 	{ PCI_VDEVICE(INTEL, 0x1f23), board_ahci }, /* Avoton AHCI */
 	{ PCI_VDEVICE(INTEL, 0x1f24), board_ahci }, /* Avoton RAID */
@@ -353,26 +361,26 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0x8d66), board_ahci }, /* Wellsburg RAID */
 	{ PCI_VDEVICE(INTEL, 0x8d6e), board_ahci }, /* Wellsburg RAID */
 	{ PCI_VDEVICE(INTEL, 0x23a3), board_ahci }, /* Coleto Creek AHCI */
-	{ PCI_VDEVICE(INTEL, 0x9c83), board_ahci }, /* Wildcat Point-LP AHCI */
-	{ PCI_VDEVICE(INTEL, 0x9c85), board_ahci }, /* Wildcat Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c87), board_ahci }, /* Wildcat Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9c8f), board_ahci }, /* Wildcat Point-LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c83), board_ahci_mobile }, /* Wildcat LP AHCI */
+	{ PCI_VDEVICE(INTEL, 0x9c85), board_ahci_mobile }, /* Wildcat LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c87), board_ahci_mobile }, /* Wildcat LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9c8f), board_ahci_mobile }, /* Wildcat LP RAID */
 	{ PCI_VDEVICE(INTEL, 0x8c82), board_ahci }, /* 9 Series AHCI */
-	{ PCI_VDEVICE(INTEL, 0x8c83), board_ahci }, /* 9 Series AHCI */
+	{ PCI_VDEVICE(INTEL, 0x8c83), board_ahci_mobile }, /* 9 Series M AHCI */
 	{ PCI_VDEVICE(INTEL, 0x8c84), board_ahci }, /* 9 Series RAID */
-	{ PCI_VDEVICE(INTEL, 0x8c85), board_ahci }, /* 9 Series RAID */
+	{ PCI_VDEVICE(INTEL, 0x8c85), board_ahci_mobile }, /* 9 Series M RAID */
 	{ PCI_VDEVICE(INTEL, 0x8c86), board_ahci }, /* 9 Series RAID */
-	{ PCI_VDEVICE(INTEL, 0x8c87), board_ahci }, /* 9 Series RAID */
+	{ PCI_VDEVICE(INTEL, 0x8c87), board_ahci_mobile }, /* 9 Series M RAID */
 	{ PCI_VDEVICE(INTEL, 0x8c8e), board_ahci }, /* 9 Series RAID */
-	{ PCI_VDEVICE(INTEL, 0x8c8f), board_ahci }, /* 9 Series RAID */
-	{ PCI_VDEVICE(INTEL, 0x9d03), board_ahci }, /* Sunrise Point-LP AHCI */
-	{ PCI_VDEVICE(INTEL, 0x9d05), board_ahci }, /* Sunrise Point-LP RAID */
-	{ PCI_VDEVICE(INTEL, 0x9d07), board_ahci }, /* Sunrise Point-LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x8c8f), board_ahci_mobile }, /* 9 Series M RAID */
+	{ PCI_VDEVICE(INTEL, 0x9d03), board_ahci_mobile }, /* Sunrise LP AHCI */
+	{ PCI_VDEVICE(INTEL, 0x9d05), board_ahci_mobile }, /* Sunrise LP RAID */
+	{ PCI_VDEVICE(INTEL, 0x9d07), board_ahci_mobile }, /* Sunrise LP RAID */
 	{ PCI_VDEVICE(INTEL, 0xa102), board_ahci }, /* Sunrise Point-H AHCI */
-	{ PCI_VDEVICE(INTEL, 0xa103), board_ahci }, /* Sunrise Point-H AHCI */
+	{ PCI_VDEVICE(INTEL, 0xa103), board_ahci_mobile }, /* Sunrise M AHCI */
 	{ PCI_VDEVICE(INTEL, 0xa105), board_ahci }, /* Sunrise Point-H RAID */
 	{ PCI_VDEVICE(INTEL, 0xa106), board_ahci }, /* Sunrise Point-H RAID */
-	{ PCI_VDEVICE(INTEL, 0xa107), board_ahci }, /* Sunrise Point-H RAID */
+	{ PCI_VDEVICE(INTEL, 0xa107), board_ahci_mobile }, /* Sunrise M RAID */
 	{ PCI_VDEVICE(INTEL, 0xa10f), board_ahci }, /* Sunrise Point-H RAID */
 	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* Lewisburg RAID*/
 	{ PCI_VDEVICE(INTEL, 0x2823), board_ahci }, /* Lewisburg AHCI*/
@@ -386,6 +394,11 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, 0xa206), board_ahci }, /* Lewisburg RAID*/
 	{ PCI_VDEVICE(INTEL, 0xa252), board_ahci }, /* Lewisburg RAID*/
 	{ PCI_VDEVICE(INTEL, 0xa256), board_ahci }, /* Lewisburg RAID*/
+	{ PCI_VDEVICE(INTEL, 0xa356), board_ahci }, /* Cannon Lake PCH-H RAID */
+	{ PCI_VDEVICE(INTEL, 0x0f22), board_ahci_mobile }, /* Bay Trail AHCI */
+	{ PCI_VDEVICE(INTEL, 0x0f23), board_ahci_mobile }, /* Bay Trail AHCI */
+	{ PCI_VDEVICE(INTEL, 0x22a3), board_ahci_mobile }, /* Cherry Tr. AHCI */
+	{ PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_mobile }, /* ApolloLake AHCI */
 
 	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
 	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
@@ -537,7 +550,9 @@ static const struct pci_device_id ahci_pci_tbl[] = {
 	  .driver_data = board_ahci_yes_fbs },
 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9230),
 	  .driver_data = board_ahci_yes_fbs },
-	{ PCI_DEVICE(PCI_VENDOR_ID_TTI, 0x0642),
+	{ PCI_DEVICE(PCI_VENDOR_ID_TTI, 0x0642), /* highpoint rocketraid 642L */
+	  .driver_data = board_ahci_yes_fbs },
+	{ PCI_DEVICE(PCI_VENDOR_ID_TTI, 0x0645), /* highpoint rocketraid 644L */
 	  .driver_data = board_ahci_yes_fbs },
 
 	/* Promise */
@@ -593,6 +608,9 @@ static int marvell_enable = 1;
 module_param(marvell_enable, int, 0644);
 MODULE_PARM_DESC(marvell_enable, "Marvell SATA via AHCI (1 = enabled)");
 
+static int mobile_lpm_policy = CONFIG_SATA_MOBILE_LPM_POLICY;
+module_param(mobile_lpm_policy, int, 0644);
+MODULE_PARM_DESC(mobile_lpm_policy, "Default LPM policy for mobile chipsets");
 
 static void ahci_pci_save_initial_config(struct pci_dev *pdev,
 					 struct ahci_host_priv *hpriv)
@@ -1728,6 +1746,10 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		if (ap->flags & ATA_FLAG_EM)
 			ap->em_message_type = hpriv->em_msg_type;
 
+		if ((hpriv->flags & AHCI_HFLAG_IS_MOBILE) &&
+		    mobile_lpm_policy >= ATA_LPM_UNKNOWN &&
+		    mobile_lpm_policy <= ATA_LPM_MIN_POWER)
+			ap->target_lpm_policy = mobile_lpm_policy;
 
 		/* disabled/not-implemented port */
 		if (!(hpriv->port_map & (1 << i)))
diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h
index 749fd94441b0..a9d996e17d75 100644
--- a/drivers/ata/ahci.h
+++ b/drivers/ata/ahci.h
@@ -251,6 +251,9 @@ enum {
 	AHCI_HFLAG_YES_ALPM		= (1 << 23), /* force ALPM cap on */
 	AHCI_HFLAG_NO_WRITE_TO_RO	= (1 << 24), /* don't write to read
 							only registers */
+	AHCI_HFLAG_IS_MOBILE		= (1 << 25), /* mobile chipset, use
+							SATA_MOBILE_LPM_POLICY
+							as default lpm_policy */
 
 	/* ap->flags bits */
 
diff --git a/drivers/ata/ahci_brcm.c b/drivers/ata/ahci_brcm.c
index 5936d1679bf3..ea430819c80b 100644
--- a/drivers/ata/ahci_brcm.c
+++ b/drivers/ata/ahci_brcm.c
@@ -70,6 +70,13 @@
 	(DATA_ENDIAN << DMADESC_ENDIAN_SHIFT) |		\
 	(MMIO_ENDIAN << MMIO_ENDIAN_SHIFT))
 
+#define BUS_CTRL_ENDIAN_NSP_CONF			\
+	(0x02 << DMADATA_ENDIAN_SHIFT | 0x02 << DMADESC_ENDIAN_SHIFT)
+
+#define BUS_CTRL_ENDIAN_CONF_MASK			\
+	(0x3 << MMIO_ENDIAN_SHIFT | 0x3 << DMADESC_ENDIAN_SHIFT |	\
+	 0x3 << DMADATA_ENDIAN_SHIFT | 0x3 << PIODATA_ENDIAN_SHIFT)
+
 enum brcm_ahci_version {
 	BRCM_SATA_BCM7425 = 1,
 	BRCM_SATA_BCM7445,
@@ -89,14 +96,6 @@ struct brcm_ahci_priv {
 	enum brcm_ahci_version version;
 };
 
-static const struct ata_port_info ahci_brcm_port_info = {
-	.flags		= AHCI_FLAG_COMMON | ATA_FLAG_NO_DIPM,
-	.link_flags	= ATA_LFLAG_NO_DB_DELAY,
-	.pio_mask	= ATA_PIO4,
-	.udma_mask	= ATA_UDMA6,
-	.port_ops	= &ahci_platform_ops,
-};
-
 static inline u32 brcm_sata_readreg(void __iomem *addr)
 {
 	/*
@@ -250,20 +249,105 @@ static u32 brcm_ahci_get_portmask(struct platform_device *pdev,
 static void brcm_sata_init(struct brcm_ahci_priv *priv)
 {
 	void __iomem *ctrl = priv->top_ctrl + SATA_TOP_CTRL_BUS_CTRL;
+	u32 data;
 
 	/* Configure endianness */
-	if (priv->version ==  BRCM_SATA_NSP) {
-		u32 data = brcm_sata_readreg(ctrl);
-
-		data &= ~((0x03 << DMADATA_ENDIAN_SHIFT) |
-			(0x03 << DMADESC_ENDIAN_SHIFT));
-		data |= (0x02 << DMADATA_ENDIAN_SHIFT) |
-			(0x02 << DMADESC_ENDIAN_SHIFT);
-		brcm_sata_writereg(data, ctrl);
-	} else
-		brcm_sata_writereg(BUS_CTRL_ENDIAN_CONF, ctrl);
+	data = brcm_sata_readreg(ctrl);
+	data &= ~BUS_CTRL_ENDIAN_CONF_MASK;
+	if (priv->version == BRCM_SATA_NSP)
+		data |= BUS_CTRL_ENDIAN_NSP_CONF;
+	else
+		data |= BUS_CTRL_ENDIAN_CONF;
+	brcm_sata_writereg(data, ctrl);
+}
+
+static unsigned int brcm_ahci_read_id(struct ata_device *dev,
+				      struct ata_taskfile *tf, u16 *id)
+{
+	struct ata_port *ap = dev->link->ap;
+	struct ata_host *host = ap->host;
+	struct ahci_host_priv *hpriv = host->private_data;
+	struct brcm_ahci_priv *priv = hpriv->plat_data;
+	void __iomem *mmio = hpriv->mmio;
+	unsigned int err_mask;
+	unsigned long flags;
+	int i, rc;
+	u32 ctl;
+
+	/* Try to read the device ID and, if this fails, proceed with the
+	 * recovery sequence below
+	 */
+	err_mask = ata_do_dev_read_id(dev, tf, id);
+	if (likely(!err_mask))
+		return err_mask;
+
+	/* Disable host interrupts */
+	spin_lock_irqsave(&host->lock, flags);
+	ctl = readl(mmio + HOST_CTL);
+	ctl &= ~HOST_IRQ_EN;
+	writel(ctl, mmio + HOST_CTL);
+	readl(mmio + HOST_CTL); /* flush */
+	spin_unlock_irqrestore(&host->lock, flags);
+
+	/* Perform the SATA PHY reset sequence */
+	brcm_sata_phy_disable(priv, ap->port_no);
+
+	/* Bring the PHY back on */
+	brcm_sata_phy_enable(priv, ap->port_no);
+
+	/* Re-initialize and calibrate the PHY */
+	for (i = 0; i < hpriv->nports; i++) {
+		rc = phy_init(hpriv->phys[i]);
+		if (rc)
+			goto disable_phys;
+
+		rc = phy_calibrate(hpriv->phys[i]);
+		if (rc) {
+			phy_exit(hpriv->phys[i]);
+			goto disable_phys;
+		}
+	}
+
+	/* Re-enable host interrupts */
+	spin_lock_irqsave(&host->lock, flags);
+	ctl = readl(mmio + HOST_CTL);
+	ctl |= HOST_IRQ_EN;
+	writel(ctl, mmio + HOST_CTL);
+	readl(mmio + HOST_CTL); /* flush */
+	spin_unlock_irqrestore(&host->lock, flags);
+
+	return ata_do_dev_read_id(dev, tf, id);
+
+disable_phys:
+	while (--i >= 0) {
+		phy_power_off(hpriv->phys[i]);
+		phy_exit(hpriv->phys[i]);
+	}
+
+	return AC_ERR_OTHER;
+}
+
+static void brcm_ahci_host_stop(struct ata_host *host)
+{
+	struct ahci_host_priv *hpriv = host->private_data;
+
+	ahci_platform_disable_resources(hpriv);
 }
 
+static struct ata_port_operations ahci_brcm_platform_ops = {
+	.inherits	= &ahci_ops,
+	.host_stop	= brcm_ahci_host_stop,
+	.read_id	= brcm_ahci_read_id,
+};
+
+static const struct ata_port_info ahci_brcm_port_info = {
+	.flags		= AHCI_FLAG_COMMON | ATA_FLAG_NO_DIPM,
+	.link_flags	= ATA_LFLAG_NO_DB_DELAY,
+	.pio_mask	= ATA_PIO4,
+	.udma_mask	= ATA_UDMA6,
+	.port_ops	= &ahci_brcm_platform_ops,
+};
+
 #ifdef CONFIG_PM_SLEEP
 static int brcm_ahci_suspend(struct device *dev)
 {
diff --git a/drivers/ata/ata_piix.c b/drivers/ata/ata_piix.c
index b702c20fbc2b..7ecb1322a514 100644
--- a/drivers/ata/ata_piix.c
+++ b/drivers/ata/ata_piix.c
@@ -458,7 +458,7 @@ static const struct piix_map_db *piix_map_db_table[] = {
 	[ich8_2port_sata_byt]	= &ich8_2port_map_db,
 };
 
-static struct pci_bits piix_enable_bits[] = {
+static const struct pci_bits piix_enable_bits[] = {
 	{ 0x41U, 1U, 0x80UL, 0x80UL },	/* port 0 */
 	{ 0x43U, 1U, 0x80UL, 0x80UL },	/* port 1 */
 };
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index a0de7a38430c..7adcf3caabd0 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -665,6 +665,16 @@ int ahci_stop_engine(struct ata_port *ap)
 	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
 		return 0;
 
+	/*
+	 * Don't try to issue commands but return with ENODEV if the
+	 * AHCI controller not available anymore (e.g. due to PCIe hot
+	 * unplugging). Otherwise a 500ms delay for each port is added.
+	 */
+	if (tmp == 0xffffffff) {
+		dev_err(ap->host->dev, "AHCI controller unavailable!\n");
+		return -ENODEV;
+	}
+
 	/* setting HBA to idle */
 	tmp &= ~PORT_CMD_START;
 	writel(tmp, port_mmio + PORT_CMD);
diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c
index 341d0ef82cbd..30cc8f1a31e1 100644
--- a/drivers/ata/libahci_platform.c
+++ b/drivers/ata/libahci_platform.c
@@ -340,7 +340,7 @@ static int ahci_platform_get_regulator(struct ahci_host_priv *hpriv, u32 port,
  * 2) regulator for controlling the targets power (optional)
  * 3) 0 - AHCI_MAX_CLKS clocks, as specified in the devs devicetree node,
  *    or for non devicetree enabled platforms a single clock
- *	4) phys (optional)
+ * 4) phys (optional)
  *
  * RETURNS:
  * The allocated ahci_host_priv on success, otherwise an ERR_PTR value
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 3c09122bf038..7431ccd03316 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -4530,6 +4530,25 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = {
 	{ "PIONEER DVD-RW  DVR-212D",	NULL,	ATA_HORKAGE_NOSETXFER },
 	{ "PIONEER DVD-RW  DVR-216D",	NULL,	ATA_HORKAGE_NOSETXFER },
 
+	/* Crucial BX100 SSD 500GB has broken LPM support */
+	{ "CT500BX100SSD1",		NULL,	ATA_HORKAGE_NOLPM },
+
+	/* 512GB MX100 with MU01 firmware has both queued TRIM and LPM issues */
+	{ "Crucial_CT512MX100*",	"MU01",	ATA_HORKAGE_NO_NCQ_TRIM |
+						ATA_HORKAGE_ZERO_AFTER_TRIM |
+						ATA_HORKAGE_NOLPM, },
+	/* 512GB MX100 with newer firmware has only LPM issues */
+	{ "Crucial_CT512MX100*",	NULL,	ATA_HORKAGE_ZERO_AFTER_TRIM |
+						ATA_HORKAGE_NOLPM, },
+
+	/* 480GB+ M500 SSDs have both queued TRIM and LPM issues */
+	{ "Crucial_CT480M500*",		NULL,	ATA_HORKAGE_NO_NCQ_TRIM |
+						ATA_HORKAGE_ZERO_AFTER_TRIM |
+						ATA_HORKAGE_NOLPM, },
+	{ "Crucial_CT960M500*",		NULL,	ATA_HORKAGE_NO_NCQ_TRIM |
+						ATA_HORKAGE_ZERO_AFTER_TRIM |
+						ATA_HORKAGE_NOLPM, },
+
 	/* devices that don't properly handle queued TRIM commands */
 	{ "Micron_M500_*",		NULL,	ATA_HORKAGE_NO_NCQ_TRIM |
 						ATA_HORKAGE_ZERO_AFTER_TRIM, },
@@ -4541,7 +4560,9 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = {
 						ATA_HORKAGE_ZERO_AFTER_TRIM, },
 	{ "Crucial_CT*MX100*",		"MU01",	ATA_HORKAGE_NO_NCQ_TRIM |
 						ATA_HORKAGE_ZERO_AFTER_TRIM, },
-	{ "Samsung SSD 8*",		NULL,	ATA_HORKAGE_NO_NCQ_TRIM |
+	{ "Samsung SSD 840*",		NULL,	ATA_HORKAGE_NO_NCQ_TRIM |
+						ATA_HORKAGE_ZERO_AFTER_TRIM, },
+	{ "Samsung SSD 850*",		NULL,	ATA_HORKAGE_NO_NCQ_TRIM |
 						ATA_HORKAGE_ZERO_AFTER_TRIM, },
 	{ "FCCT*M500*",			NULL,	ATA_HORKAGE_NO_NCQ_TRIM |
 						ATA_HORKAGE_ZERO_AFTER_TRIM, },
@@ -5401,8 +5422,7 @@ void ata_qc_issue(struct ata_queued_cmd *qc)
 	 * We guarantee to LLDs that they will have at least one
 	 * non-zero sg if the command is a data command.
 	 */
-	if (WARN_ON_ONCE(ata_is_data(prot) &&
-			 (!qc->sg || !qc->n_elem || !qc->nbytes)))
+	if (ata_is_data(prot) && (!qc->sg || !qc->n_elem || !qc->nbytes))
 		goto sys_err;
 
 	if (ata_is_dma(prot) || (ata_is_pio(prot) &&
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 11c3137d7b0a..c016829a38fd 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -815,7 +815,8 @@ void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap)
 
 	if (ap->pflags & ATA_PFLAG_LOADING)
 		ap->pflags &= ~ATA_PFLAG_LOADING;
-	else if (ap->pflags & ATA_PFLAG_SCSI_HOTPLUG)
+	else if ((ap->pflags & ATA_PFLAG_SCSI_HOTPLUG) &&
+		!(ap->flags & ATA_FLAG_SAS_HOST))
 		schedule_delayed_work(&ap->hotplug_task, 0);
 
 	if (ap->pflags & ATA_PFLAG_RECOVERED)
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 66be961c93a4..89a9d4a2efc8 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -3316,6 +3316,12 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc)
 		goto invalid_fld;
 	}
 
+	/* We may not issue NCQ commands to devices not supporting NCQ */
+	if (ata_is_ncq(tf->protocol) && !ata_ncq_enabled(dev)) {
+		fp = 1;
+		goto invalid_fld;
+	}
+
 	/* sanity check for pio multi commands */
 	if ((cdb[1] & 0xe0) && !is_multi_taskfile(tf)) {
 		fp = 1;
@@ -4282,7 +4288,7 @@ static inline void ata_scsi_dump_cdb(struct ata_port *ap,
 #ifdef ATA_DEBUG
 	struct scsi_device *scsidev = cmd->device;
 
-	DPRINTK("CDB (%u:%d,%d,%d) %9ph\n",
+	DPRINTK("CDB (%u:%d,%d,%lld) %9ph\n",
 		ap->print_id,
 		scsidev->channel, scsidev->id, scsidev->lun,
 		cmd->cmnd);
@@ -4309,7 +4315,9 @@ static inline int __ata_scsi_queuecmd(struct scsi_cmnd *scmd,
 		if (likely((scsi_op != ATA_16) || !atapi_passthru16)) {
 			/* relay SCSI command to ATAPI device */
 			int len = COMMAND_SIZE(scsi_op);
-			if (unlikely(len > scmd->cmd_len || len > dev->cdb_len))
+			if (unlikely(len > scmd->cmd_len ||
+				     len > dev->cdb_len ||
+				     scmd->cmd_len > ATAPI_CDB_LEN))
 				goto bad_cdb_len;
 
 			xlat_func = atapi_xlat;
diff --git a/drivers/ata/pata_ali.c b/drivers/ata/pata_ali.c
index d19cd88ed2d3..0b122f903b8a 100644
--- a/drivers/ata/pata_ali.c
+++ b/drivers/ata/pata_ali.c
@@ -466,7 +466,8 @@ static void ali_init_chipset(struct pci_dev *pdev)
 			tmp |= 0x01;	/* CD_ROM enable for DMA */
 		pci_write_config_byte(pdev, 0x53, tmp);
 	}
-	north = pci_get_bus_and_slot(0, PCI_DEVFN(0,0));
+	north = pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus), 0,
+					    PCI_DEVFN(0, 0));
 	if (north && north->vendor == PCI_VENDOR_ID_AL && ali_isa_bridge) {
 		/* Configure the ALi bridge logic. For non ALi rely on BIOS.
 		   Set the south bridge enable bit */
diff --git a/drivers/ata/pata_at32.c b/drivers/ata/pata_at32.c
deleted file mode 100644
index 9aeb7a6dd4d4..000000000000
--- a/drivers/ata/pata_at32.c
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * AVR32 SMC/CFC PATA Driver
- *
- * Copyright (C) 2007 Atmel Norway
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- */
-
-#define DEBUG
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/device.h>
-#include <linux/platform_device.h>
-#include <linux/delay.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/slab.h>
-#include <scsi/scsi_host.h>
-#include <linux/ata.h>
-#include <linux/libata.h>
-#include <linux/err.h>
-#include <linux/io.h>
-
-#include <mach/board.h>
-#include <mach/smc.h>
-
-#define DRV_NAME "pata_at32"
-#define DRV_VERSION "0.0.3"
-
-/*
- * CompactFlash controller memory layout relative to the base address:
- *
- *	Attribute memory:  0000 0000 -> 003f ffff
- *	Common memory:	   0040 0000 -> 007f ffff
- *	I/O memory:	   0080 0000 -> 00bf ffff
- *	True IDE Mode:	   00c0 0000 -> 00df ffff
- *	Alt IDE Mode:	   00e0 0000 -> 00ff ffff
- *
- * Only True IDE and Alt True IDE mode are needed for this driver.
- *
- *	True IDE mode	  => CS0 = 0, CS1 = 1 (cmd, error, stat, etc)
- *	Alt True IDE mode => CS0 = 1, CS1 = 0 (ctl, alt_stat)
- */
-#define CF_IDE_OFFSET	  0x00c00000
-#define CF_ALT_IDE_OFFSET 0x00e00000
-#define CF_RES_SIZE	  2048
-
-/*
- * Define DEBUG_BUS if you are doing debugging of your own EBI -> PATA
- * adaptor with a logic analyzer or similar.
- */
-#undef DEBUG_BUS
-
-/*
- * ATA PIO modes
- *
- *	Name	| Mb/s	| Min cycle time | Mask
- *	--------+-------+----------------+--------
- *	Mode 0	| 3.3	| 600 ns	 | 0x01
- *	Mode 1	| 5.2	| 383 ns	 | 0x03
- *	Mode 2	| 8.3	| 240 ns	 | 0x07
- *	Mode 3	| 11.1	| 180 ns	 | 0x0f
- *	Mode 4	| 16.7	| 120 ns	 | 0x1f
- *
- * Alter PIO_MASK below according to table to set maximal PIO mode.
- */
-enum {
-  PIO_MASK = ATA_PIO4,
-};
-
-/*
- * Struct containing private information about device.
- */
-struct at32_ide_info {
-	unsigned int		irq;
-	struct resource		res_ide;
-	struct resource		res_alt;
-	void __iomem		*ide_addr;
-	void __iomem		*alt_addr;
-	unsigned int		cs;
-	struct smc_config	smc;
-};
-
-/*
- * Setup SMC for the given ATA timing.
- */
-static int pata_at32_setup_timing(struct device *dev,
-				  struct at32_ide_info *info,
-				  const struct ata_timing *ata)
-{
-	struct smc_config *smc = &info->smc;
-	struct smc_timing timing;
-
-	int active;
-	int recover;
-
-	memset(&timing, 0, sizeof(struct smc_timing));
-
-	/* Total cycle time */
-	timing.read_cycle  = ata->cyc8b;
-
-	/* DIOR <= CFIOR timings */
-	timing.nrd_setup   = ata->setup;
-	timing.nrd_pulse   = ata->act8b;
-	timing.nrd_recover = ata->rec8b;
-
-	/* Convert nanosecond timing to clock cycles */
-	smc_set_timing(smc, &timing);
-
-	/* Add one extra cycle setup due to signal ring */
-	smc->nrd_setup = smc->nrd_setup + 1;
-
-	active  = smc->nrd_setup + smc->nrd_pulse;
-	recover = smc->read_cycle - active;
-
-	/* Need at least two cycles recovery */
-	if (recover < 2)
-	  smc->read_cycle = active + 2;
-
-	/* (CS0, CS1, DIR, OE) <= (CFCE1, CFCE2, CFRNW, NCSX) timings */
-	smc->ncs_read_setup = 1;
-	smc->ncs_read_pulse = smc->read_cycle - 2;
-
-	/* Write timings same as read timings */
-	smc->write_cycle = smc->read_cycle;
-	smc->nwe_setup = smc->nrd_setup;
-	smc->nwe_pulse = smc->nrd_pulse;
-	smc->ncs_write_setup = smc->ncs_read_setup;
-	smc->ncs_write_pulse = smc->ncs_read_pulse;
-
-	/* Do some debugging output of ATA and SMC timings */
-	dev_dbg(dev, "ATA: C=%d S=%d P=%d R=%d\n",
-		ata->cyc8b, ata->setup, ata->act8b, ata->rec8b);
-
-	dev_dbg(dev, "SMC: C=%d S=%d P=%d NS=%d NP=%d\n",
-		smc->read_cycle, smc->nrd_setup, smc->nrd_pulse,
-		smc->ncs_read_setup, smc->ncs_read_pulse);
-
-	/* Finally, configure the SMC */
-	return smc_set_configuration(info->cs, smc);
-}
-
-/*
- * Procedures for libATA.
- */
-static void pata_at32_set_piomode(struct ata_port *ap, struct ata_device *adev)
-{
-	struct ata_timing timing;
-	struct at32_ide_info *info = ap->host->private_data;
-
-	int ret;
-
-	/* Compute ATA timing */
-	ret = ata_timing_compute(adev, adev->pio_mode, &timing, 1000, 0);
-	if (ret) {
-		dev_warn(ap->dev, "Failed to compute ATA timing %d\n", ret);
-		return;
-	}
-
-	/* Setup SMC to ATA timing */
-	ret = pata_at32_setup_timing(ap->dev, info, &timing);
-	if (ret) {
-		dev_warn(ap->dev, "Failed to setup ATA timing %d\n", ret);
-		return;
-	}
-}
-
-static struct scsi_host_template at32_sht = {
-	ATA_PIO_SHT(DRV_NAME),
-};
-
-static struct ata_port_operations at32_port_ops = {
-	.inherits		= &ata_sff_port_ops,
-	.cable_detect		= ata_cable_40wire,
-	.set_piomode		= pata_at32_set_piomode,
-};
-
-static int __init pata_at32_init_one(struct device *dev,
-				     struct at32_ide_info *info)
-{
-	struct ata_host *host;
-	struct ata_port *ap;
-
-	host = ata_host_alloc(dev, 1);
-	if (!host)
-		return -ENOMEM;
-
-	ap = host->ports[0];
-
-	/* Setup ATA bindings */
-	ap->ops	     = &at32_port_ops;
-	ap->pio_mask = PIO_MASK;
-	ap->flags   |= ATA_FLAG_SLAVE_POSS;
-
-	/*
-	 * Since all 8-bit taskfile transfers has to go on the lower
-	 * byte of the data bus and there is a bug in the SMC that
-	 * makes it impossible to alter the bus width during runtime,
-	 * we need to hardwire the address signals as follows:
-	 *
-	 *	A_IDE(2:0) <= A_EBI(3:1)
-	 *
-	 * This makes all addresses on the EBI even, thus all data
-	 * will be on the lower byte of the data bus.  All addresses
-	 * used by libATA need to be altered according to this.
-	 */
-	ap->ioaddr.altstatus_addr = info->alt_addr + (0x06 << 1);
-	ap->ioaddr.ctl_addr	  = info->alt_addr + (0x06 << 1);
-
-	ap->ioaddr.data_addr	  = info->ide_addr + (ATA_REG_DATA << 1);
-	ap->ioaddr.error_addr	  = info->ide_addr + (ATA_REG_ERR << 1);
-	ap->ioaddr.feature_addr	  = info->ide_addr + (ATA_REG_FEATURE << 1);
-	ap->ioaddr.nsect_addr	  = info->ide_addr + (ATA_REG_NSECT << 1);
-	ap->ioaddr.lbal_addr	  = info->ide_addr + (ATA_REG_LBAL << 1);
-	ap->ioaddr.lbam_addr	  = info->ide_addr + (ATA_REG_LBAM << 1);
-	ap->ioaddr.lbah_addr	  = info->ide_addr + (ATA_REG_LBAH << 1);
-	ap->ioaddr.device_addr	  = info->ide_addr + (ATA_REG_DEVICE << 1);
-	ap->ioaddr.status_addr	  = info->ide_addr + (ATA_REG_STATUS << 1);
-	ap->ioaddr.command_addr	  = info->ide_addr + (ATA_REG_CMD << 1);
-
-	/* Set info as private data of ATA host */
-	host->private_data = info;
-
-	/* Register ATA device and return */
-	return ata_host_activate(host, info->irq, ata_sff_interrupt,
-				 IRQF_SHARED | IRQF_TRIGGER_RISING,
-				 &at32_sht);
-}
-
-/*
- * This function may come in handy for people analyzing their own
- * EBI -> PATA adaptors.
- */
-#ifdef DEBUG_BUS
-
-static void __init pata_at32_debug_bus(struct device *dev,
-				       struct at32_ide_info *info)
-{
-	const int d1 = 0xff;
-	const int d2 = 0x00;
-
-	int i;
-
-	/* Write 8-bit values (registers) */
-	iowrite8(d1, info->alt_addr + (0x06 << 1));
-	iowrite8(d2, info->alt_addr + (0x06 << 1));
-
-	for (i = 0; i < 8; i++) {
-		iowrite8(d1, info->ide_addr + (i << 1));
-		iowrite8(d2, info->ide_addr + (i << 1));
-	}
-
-	/* Write 16 bit values (data) */
-	iowrite16(d1,	   info->ide_addr);
-	iowrite16(d1 << 8, info->ide_addr);
-
-	iowrite16(d1,	   info->ide_addr);
-	iowrite16(d1 << 8, info->ide_addr);
-}
-
-#endif
-
-static int __init pata_at32_probe(struct platform_device *pdev)
-{
-	const struct ata_timing initial_timing =
-		{XFER_PIO_0, 70, 290, 240, 600, 165, 150, 600, 0};
-
-	struct device		 *dev = &pdev->dev;
-	struct at32_ide_info	 *info;
-	struct ide_platform_data *board = dev_get_platdata(&pdev->dev);
-	struct resource		 *res;
-
-	int irq;
-	int ret;
-
-	if (!board)
-		return -ENXIO;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENXIO;
-
-	/* Retrive IRQ */
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	/* Setup struct containing private information */
-	info = kzalloc(sizeof(struct at32_ide_info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	info->irq = irq;
-	info->cs  = board->cs;
-
-	/* Request memory resources */
-	info->res_ide.start = res->start + CF_IDE_OFFSET;
-	info->res_ide.end   = info->res_ide.start + CF_RES_SIZE - 1;
-	info->res_ide.name  = "ide";
-	info->res_ide.flags = IORESOURCE_MEM;
-
-	ret = request_resource(res, &info->res_ide);
-	if (ret)
-		goto err_req_res_ide;
-
-	info->res_alt.start = res->start + CF_ALT_IDE_OFFSET;
-	info->res_alt.end   = info->res_alt.start + CF_RES_SIZE - 1;
-	info->res_alt.name  = "alt";
-	info->res_alt.flags = IORESOURCE_MEM;
-
-	ret = request_resource(res, &info->res_alt);
-	if (ret)
-		goto err_req_res_alt;
-
-	/* Setup non-timing elements of SMC */
-	info->smc.bus_width	 = 2; /* 16 bit data bus */
-	info->smc.nrd_controlled = 1; /* Sample data on rising edge of NRD */
-	info->smc.nwe_controlled = 0; /* Drive data on falling edge of NCS */
-	info->smc.nwait_mode	 = 3; /* NWAIT is in READY mode */
-	info->smc.byte_write	 = 0; /* Byte select access type */
-	info->smc.tdf_mode	 = 0; /* TDF optimization disabled */
-	info->smc.tdf_cycles	 = 0; /* No TDF wait cycles */
-
-	/* Setup SMC to ATA timing */
-	ret = pata_at32_setup_timing(dev, info, &initial_timing);
-	if (ret)
-		goto err_setup_timing;
-
-	/* Map ATA address space */
-	ret = -ENOMEM;
-	info->ide_addr = devm_ioremap(dev, info->res_ide.start, 16);
-	info->alt_addr = devm_ioremap(dev, info->res_alt.start, 16);
-	if (!info->ide_addr || !info->alt_addr)
-		goto err_ioremap;
-
-#ifdef DEBUG_BUS
-	pata_at32_debug_bus(dev, info);
-#endif
-
-	/* Setup and register ATA device */
-	ret = pata_at32_init_one(dev, info);
-	if (ret)
-		goto err_ata_device;
-
-	return 0;
-
- err_ata_device:
- err_ioremap:
- err_setup_timing:
-	release_resource(&info->res_alt);
- err_req_res_alt:
-	release_resource(&info->res_ide);
- err_req_res_ide:
-	kfree(info);
-
-	return ret;
-}
-
-static int __exit pata_at32_remove(struct platform_device *pdev)
-{
-	struct ata_host *host = platform_get_drvdata(pdev);
-	struct at32_ide_info *info;
-
-	if (!host)
-		return 0;
-
-	info = host->private_data;
-	ata_host_detach(host);
-
-	if (!info)
-		return 0;
-
-	release_resource(&info->res_ide);
-	release_resource(&info->res_alt);
-
-	kfree(info);
-
-	return 0;
-}
-
-/* work with hotplug and coldplug */
-MODULE_ALIAS("platform:at32_ide");
-
-static struct platform_driver pata_at32_driver = {
-	.remove	       = __exit_p(pata_at32_remove),
-	.driver	       = {
-		.name  = "at32_ide",
-	},
-};
-
-module_platform_driver_probe(pata_at32_driver, pata_at32_probe);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("AVR32 SMC/CFC PATA Driver");
-MODULE_AUTHOR("Kristoffer Nyborg Gregertsen <kngregertsen@norway.atmel.com>");
-MODULE_VERSION(DRV_VERSION);
diff --git a/drivers/ata/pata_atiixp.c b/drivers/ata/pata_atiixp.c
index 49d705c9f0f7..4d49fd3c927b 100644
--- a/drivers/ata/pata_atiixp.c
+++ b/drivers/ata/pata_atiixp.c
@@ -278,6 +278,10 @@ static int atiixp_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	};
 	const struct ata_port_info *ppi[] = { &info, &info };
 
+	/* SB600 doesn't have secondary port wired */
+	if((pdev->device == PCI_DEVICE_ID_ATI_IXP600_IDE))
+		ppi[1] = &ata_dummy_port_info;
+
 	return ata_pci_bmdma_init_one(pdev, ppi, &atiixp_sht, NULL,
 				      ATA_HOST_PARALLEL_SCAN);
 }
diff --git a/drivers/ata/pata_it821x.c b/drivers/ata/pata_it821x.c
index 7a21edf89e72..8468b300193b 100644
--- a/drivers/ata/pata_it821x.c
+++ b/drivers/ata/pata_it821x.c
@@ -683,7 +683,7 @@ static u8 *it821x_firmware_command(struct ata_port *ap, u8 cmd, int len)
 			ioread16_rep(ap->ioaddr.data_addr, buf, len/2);
 			return (u8 *)buf;
 		}
-		mdelay(1);
+		usleep_range(500, 1000);
 	}
 	kfree(buf);
 	printk(KERN_ERR "it821x_firmware_command: timeout\n");
diff --git a/drivers/ata/pata_pdc2027x.c b/drivers/ata/pata_pdc2027x.c
index 6db2e34bd52f..1a18e675ba9f 100644
--- a/drivers/ata/pata_pdc2027x.c
+++ b/drivers/ata/pata_pdc2027x.c
@@ -580,7 +580,7 @@ static void pdc_adjust_pll(struct ata_host *host, long pll_clock, unsigned int b
 	ioread16(mmio_base + PDC_PLL_CTL); /* flush */
 
 	/* Wait the PLL circuit to be stable */
-	mdelay(30);
+	msleep(30);
 
 #ifdef PDC_DEBUG
 	/*
@@ -620,7 +620,7 @@ static long pdc_detect_pll_input_clock(struct ata_host *host)
 	start_time = ktime_get();
 
 	/* Let the counter run for 100 ms. */
-	mdelay(100);
+	msleep(100);
 
 	/* Read the counter values again */
 	end_count = pdc_read_counter(host);
diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c
index cc208b72b199..42d4589b43d4 100644
--- a/drivers/ata/sata_mv.c
+++ b/drivers/ata/sata_mv.c
@@ -3596,7 +3596,7 @@ static void mv_reset_channel(struct mv_host_priv *hpriv, void __iomem *mmio,
 	hpriv->ops->phy_errata(hpriv, mmio, port_no);
 
 	if (IS_GEN_I(hpriv))
-		mdelay(1);
+		usleep_range(500, 1000);
 }
 
 static void mv_pmp_select(struct ata_port *ap, int pmp)
diff --git a/drivers/ata/sata_rcar.c b/drivers/ata/sata_rcar.c
index 80ee2f2a50d0..6456e07db72a 100644
--- a/drivers/ata/sata_rcar.c
+++ b/drivers/ata/sata_rcar.c
@@ -146,6 +146,7 @@
 enum sata_rcar_type {
 	RCAR_GEN1_SATA,
 	RCAR_GEN2_SATA,
+	RCAR_GEN3_SATA,
 	RCAR_R8A7790_ES1_SATA,
 };
 
@@ -784,26 +785,11 @@ static void sata_rcar_setup_port(struct ata_host *host)
 	ioaddr->command_addr	= ioaddr->cmd_addr + (ATA_REG_CMD << 2);
 }
 
-static void sata_rcar_init_controller(struct ata_host *host)
+static void sata_rcar_init_module(struct sata_rcar_priv *priv)
 {
-	struct sata_rcar_priv *priv = host->private_data;
 	void __iomem *base = priv->base;
 	u32 val;
 
-	/* reset and setup phy */
-	switch (priv->type) {
-	case RCAR_GEN1_SATA:
-		sata_rcar_gen1_phy_init(priv);
-		break;
-	case RCAR_GEN2_SATA:
-	case RCAR_R8A7790_ES1_SATA:
-		sata_rcar_gen2_phy_init(priv);
-		break;
-	default:
-		dev_warn(host->dev, "SATA phy is not initialized\n");
-		break;
-	}
-
 	/* SATA-IP reset state */
 	val = ioread32(base + ATAPI_CONTROL1_REG);
 	val |= ATAPI_CONTROL1_RESET;
@@ -824,10 +810,33 @@ static void sata_rcar_init_controller(struct ata_host *host)
 	/* ack and mask */
 	iowrite32(0, base + SATAINTSTAT_REG);
 	iowrite32(0x7ff, base + SATAINTMASK_REG);
+
 	/* enable interrupts */
 	iowrite32(ATAPI_INT_ENABLE_SATAINT, base + ATAPI_INT_ENABLE_REG);
 }
 
+static void sata_rcar_init_controller(struct ata_host *host)
+{
+	struct sata_rcar_priv *priv = host->private_data;
+
+	/* reset and setup phy */
+	switch (priv->type) {
+	case RCAR_GEN1_SATA:
+		sata_rcar_gen1_phy_init(priv);
+		break;
+	case RCAR_GEN2_SATA:
+	case RCAR_GEN3_SATA:
+	case RCAR_R8A7790_ES1_SATA:
+		sata_rcar_gen2_phy_init(priv);
+		break;
+	default:
+		dev_warn(host->dev, "SATA phy is not initialized\n");
+		break;
+	}
+
+	sata_rcar_init_module(priv);
+}
+
 static const struct of_device_id sata_rcar_match[] = {
 	{
 		/* Deprecated by "renesas,sata-r8a7779" */
@@ -856,7 +865,7 @@ static const struct of_device_id sata_rcar_match[] = {
 	},
 	{
 		.compatible = "renesas,sata-r8a7795",
-		.data = (void *)RCAR_GEN2_SATA
+		.data = (void *)RCAR_GEN3_SATA
 	},
 	{
 		.compatible = "renesas,rcar-gen2-sata",
@@ -864,7 +873,7 @@ static const struct of_device_id sata_rcar_match[] = {
 	},
 	{
 		.compatible = "renesas,rcar-gen3-sata",
-		.data = (void *)RCAR_GEN2_SATA
+		.data = (void *)RCAR_GEN3_SATA
 	},
 	{ },
 };
@@ -982,11 +991,18 @@ static int sata_rcar_resume(struct device *dev)
 	if (ret)
 		return ret;
 
-	/* ack and mask */
-	iowrite32(0, base + SATAINTSTAT_REG);
-	iowrite32(0x7ff, base + SATAINTMASK_REG);
-	/* enable interrupts */
-	iowrite32(ATAPI_INT_ENABLE_SATAINT, base + ATAPI_INT_ENABLE_REG);
+	if (priv->type == RCAR_GEN3_SATA) {
+		sata_rcar_gen2_phy_init(priv);
+		sata_rcar_init_module(priv);
+	} else {
+		/* ack and mask */
+		iowrite32(0, base + SATAINTSTAT_REG);
+		iowrite32(0x7ff, base + SATAINTMASK_REG);
+
+		/* enable interrupts */
+		iowrite32(ATAPI_INT_ENABLE_SATAINT,
+			  base + ATAPI_INT_ENABLE_REG);
+	}
 
 	ata_host_resume(host);
 
diff --git a/drivers/atm/eni.c b/drivers/atm/eni.c
index ce47eb17901d..6470e3c4c990 100644
--- a/drivers/atm/eni.c
+++ b/drivers/atm/eni.c
@@ -473,7 +473,7 @@ static int do_rx_dma(struct atm_vcc *vcc,struct sk_buff *skb,
 		ENI_PRV_POS(skb) = eni_vcc->descr+size+1;
 		skb_queue_tail(&eni_dev->rx_queue,skb);
 		eni_vcc->last = skb;
-rx_enqueued++;
+		rx_enqueued++;
 	}
 	eni_vcc->descr = here;
 	eni_out(dma_wr,MID_DMA_WR_RX);
@@ -715,7 +715,7 @@ static void get_service(struct atm_dev *dev)
 			else eni_dev->slow = vcc;
 			eni_dev->last_slow = vcc;
 		}
-putting++;
+		putting++;
 		ENI_VCC(vcc)->servicing++;
 	}
 }
@@ -744,7 +744,7 @@ static void dequeue_rx(struct atm_dev *dev)
 		}
 		EVENT("dequeued (size=%ld,pos=0x%lx)\n",ENI_PRV_SIZE(skb),
 		    ENI_PRV_POS(skb));
-rx_dequeued++;
+		rx_dequeued++;
 		vcc = ATM_SKB(skb)->vcc;
 		eni_vcc = ENI_VCC(vcc);
 		first = 0;
@@ -1174,7 +1174,7 @@ DPRINTK("doing direct send\n"); /* @@@ well, this doesn't work anyway */
 	DPRINTK("dma_wr set to %d, tx_pos is now %ld\n",dma_wr,tx->tx_pos);
 	eni_out(dma_wr,MID_DMA_WR_TX);
 	skb_queue_tail(&eni_dev->tx_queue,skb);
-queued++;
+	queued++;
 	return enq_ok;
 }
 
@@ -1195,7 +1195,7 @@ static void poll_tx(struct atm_dev *dev)
 				if (res == enq_ok) continue;
 				DPRINTK("re-queuing TX PDU\n");
 				skb_queue_head(&tx->backlog,skb);
-requeued++;
+				requeued++;
 				if (res == enq_jam) return;
 				break;
 			}
@@ -1232,7 +1232,7 @@ static void dequeue_tx(struct atm_dev *dev)
 		else dev_kfree_skb_irq(skb);
 		atomic_inc(&vcc->stats->tx);
 		wake_up(&eni_dev->tx_wait);
-dma_complete++;
+		dma_complete++;
 	}
 }
 
@@ -1555,7 +1555,7 @@ static void eni_tasklet(unsigned long data)
 	}
 	if (events & MID_TX_COMPLETE) {
 		EVENT("INT: TX COMPLETE\n",0,0);
-tx_complete++;
+		tx_complete++;
 		wake_up(&eni_dev->tx_wait);
 		/* poll_rx ? */
 	}
@@ -2069,14 +2069,14 @@ static int eni_send(struct atm_vcc *vcc,struct sk_buff *skb)
 		}
 		*(u32 *) skb->data = htonl(*(u32 *) skb->data);
 	}
-submitted++;
+	submitted++;
 	ATM_SKB(skb)->vcc = vcc;
 	tasklet_disable(&ENI_DEV(vcc->dev)->task);
 	res = do_tx(skb);
 	tasklet_enable(&ENI_DEV(vcc->dev)->task);
 	if (res == enq_ok) return 0;
 	skb_queue_tail(&ENI_VCC(vcc)->tx->backlog,skb);
-backlogged++;
+	backlogged++;
 	tasklet_schedule(&ENI_DEV(vcc->dev)->task);
 	return 0;
 }
diff --git a/drivers/atm/he.c b/drivers/atm/he.c
index e58538c29377..29f102dcfec4 100644
--- a/drivers/atm/he.c
+++ b/drivers/atm/he.c
@@ -738,13 +738,13 @@ static int he_init_cs_block_rcm(struct he_dev *he_dev)
 #else
 		/* this is pretty, but avoids _divdu3 and is mostly correct */
 		mult = he_dev->atm_dev->link_rate / ATM_OC3_PCR;
-		if (rate_cps > (272 * mult))
+		if (rate_cps > (272ULL * mult))
 			buf = 4;
-		else if (rate_cps > (204 * mult))
+		else if (rate_cps > (204ULL * mult))
 			buf = 3;
-		else if (rate_cps > (136 * mult))
+		else if (rate_cps > (136ULL * mult))
 			buf = 2;
-		else if (rate_cps > (68 * mult))
+		else if (rate_cps > (68ULL * mult))
 			buf = 1;
 		else
 			buf = 0;
diff --git a/drivers/atm/iphase.c b/drivers/atm/iphase.c
index 98a3a43484c8..44abb8a0a5e5 100644
--- a/drivers/atm/iphase.c
+++ b/drivers/atm/iphase.c
@@ -3147,7 +3147,7 @@ static int ia_proc_read(struct atm_dev *dev,loff_t *pos,char *page)
                            "  Size of Tx Buffer  :  %u\n"
                            "  Number of Rx Buffer:  %u\n"
                            "  Size of Rx Buffer  :  %u\n"
-                           "  Packets Receiverd  :  %u\n"
+                           "  Packets Received   :  %u\n"
                            "  Packets Transmitted:  %u\n"
                            "  Cells Received     :  %u\n"
                            "  Cells Transmitted  :  %u\n"
diff --git a/drivers/auxdisplay/img-ascii-lcd.c b/drivers/auxdisplay/img-ascii-lcd.c
index db040b378224..834509506ef6 100644
--- a/drivers/auxdisplay/img-ascii-lcd.c
+++ b/drivers/auxdisplay/img-ascii-lcd.c
@@ -97,7 +97,7 @@ static struct img_ascii_lcd_config boston_config = {
 static void malta_update(struct img_ascii_lcd_ctx *ctx)
 {
 	unsigned int i;
-	int err;
+	int err = 0;
 
 	for (i = 0; i < ctx->cfg->num_chars; i++) {
 		err = regmap_write(ctx->regmap,
@@ -180,7 +180,7 @@ static int sead3_wait_lcd_idle(struct img_ascii_lcd_ctx *ctx)
 static void sead3_update(struct img_ascii_lcd_ctx *ctx)
 {
 	unsigned int i;
-	int err;
+	int err = 0;
 
 	for (i = 0; i < ctx->cfg->num_chars; i++) {
 		err = sead3_wait_lcd_idle(ctx);
@@ -224,7 +224,7 @@ MODULE_DEVICE_TABLE(of, img_ascii_lcd_matches);
 
 /**
  * img_ascii_lcd_scroll() - scroll the display by a character
- * @arg: really a pointer to the private data structure
+ * @t: really a pointer to the private data structure
  *
  * Scroll the current message along the LCD by one character, rearming the
  * timer if required.
@@ -441,3 +441,7 @@ static struct platform_driver img_ascii_lcd_driver = {
 	.remove	= img_ascii_lcd_remove,
 };
 module_platform_driver(img_ascii_lcd_driver);
+
+MODULE_DESCRIPTION("Imagination Technologies ASCII LCD Display");
+MODULE_AUTHOR("Paul Burton <paul.burton@mips.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/auxdisplay/panel.c b/drivers/auxdisplay/panel.c
index ea7869c0d7f9..ec5e8800f8ad 100644
--- a/drivers/auxdisplay/panel.c
+++ b/drivers/auxdisplay/panel.c
@@ -1372,7 +1372,7 @@ static void panel_process_inputs(void)
 				break;
 			input->rise_timer = 0;
 			input->state = INPUT_ST_RISING;
-			/* no break here, fall through */
+			/* fall through */
 		case INPUT_ST_RISING:
 			if ((phys_curr & input->mask) != input->value) {
 				input->state = INPUT_ST_LOW;
@@ -1385,11 +1385,11 @@ static void panel_process_inputs(void)
 			}
 			input->high_timer = 0;
 			input->state = INPUT_ST_HIGH;
-			/* no break here, fall through */
+			/* fall through */
 		case INPUT_ST_HIGH:
 			if (input_state_high(input))
 				break;
-			/* no break here, fall through */
+			/* fall through */
 		case INPUT_ST_FALLING:
 			input_state_falling(input);
 		}
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index 2415ad9f6dd4..29b0eb452b3a 100644
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -86,10 +86,9 @@ config FW_LOADER
 	  require userspace firmware loading support, but a module built
 	  out-of-tree does.
 
-config FIRMWARE_IN_KERNEL
-	bool "Include in-kernel firmware blobs in kernel binary"
+config EXTRA_FIRMWARE
+	string "External firmware blobs to build into the kernel binary"
 	depends on FW_LOADER
-	default y
 	help
 	  Various drivers in the kernel source tree may require firmware,
 	  which is generally available in your distribution's linux-firmware
@@ -99,23 +98,6 @@ config FIRMWARE_IN_KERNEL
 	  /lib/firmware/ on your system, so they can be loaded by userspace
 	  helpers on request.
 
-	  Enabling this option will build each required firmware blob
-	  specified by EXTRA_FIRMWARE into the kernel directly, where
-	  request_firmware() will find them without having to call out to
-	  userspace. This may be useful if your root file system requires a
-	  device that uses such firmware and you do not wish to use an
-	  initrd.
-
-	  This single option controls the inclusion of firmware for
-	  every driver that uses request_firmware(), which avoids a
-	  proliferation of 'Include firmware for xxx device' options.
-
-	  Say 'N' and let firmware be loaded from userspace.
-
-config EXTRA_FIRMWARE
-	string "External firmware blobs to build into the kernel binary"
-	depends on FW_LOADER
-	help
 	  This option allows firmware to be built into the kernel for the case
 	  where the user either cannot or doesn't want to provide it from
 	  userspace at runtime (for example, when the firmware in question is
@@ -126,11 +108,11 @@ config EXTRA_FIRMWARE
 	  firmware files -- the same names that appear in MODULE_FIRMWARE()
 	  and request_firmware() in the source. These files should exist under
 	  the directory specified by the EXTRA_FIRMWARE_DIR option, which is
-	  by default the firmware subdirectory of the kernel source tree.
+	  /lib/firmware by default.
 
 	  For example, you might set CONFIG_EXTRA_FIRMWARE="usb8388.bin", copy
-	  the usb8388.bin file into the firmware directory, and build the kernel.
-	  Then any request_firmware("usb8388.bin") will be satisfied internally
+	  the usb8388.bin file into /lib/firmware, and build the kernel. Then
+	  any request_firmware("usb8388.bin") will be satisfied internally
 	  without needing to call out to userspace.
 
 	  WARNING: If you include additional firmware files into your binary
@@ -249,6 +231,7 @@ config DMA_SHARED_BUFFER
 	bool
 	default n
 	select ANON_INODES
+	select IRQ_WORK
 	help
 	  This option enables the framework for buffer-sharing between
 	  multiple drivers. A buffer is associated with a file using driver
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index 4de87b0b53c8..52ec5174bcb1 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -1,15 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Arch specific cpu topology information
  *
  * Copyright (C) 2016, ARM Ltd.
  * Written by: Juri Lelli, ARM Ltd.
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Released under the GPLv2 only.
- * SPDX-License-Identifier: GPL-2.0
  */
 
 #include <linux/acpi.h>
diff --git a/drivers/base/attribute_container.c b/drivers/base/attribute_container.c
index 95e3ef82f3b7..20736aaa0e69 100644
--- a/drivers/base/attribute_container.c
+++ b/drivers/base/attribute_container.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * attribute_container.c - implementation of a simple container for classes
  *
  * Copyright (c) 2005 - James Bottomley <James.Bottomley@steeleye.com>
  *
- * This file is licensed under GPLv2
- *
  * The basic idea here is to enable a device to be attached to an
  * aritrary numer of classes without having to allocate storage for them.
  * Instead, the contained classes select the devices they need to attach
diff --git a/drivers/base/bus.c b/drivers/base/bus.c
index 22a64fd3309b..ef6183306b40 100644
--- a/drivers/base/bus.c
+++ b/drivers/base/bus.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * bus.c - bus driver management
  *
@@ -5,9 +6,6 @@
  * Copyright (c) 2002-3 Open Source Development Labs
  * Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de>
  * Copyright (c) 2007 Novell Inc.
- *
- * This file is released under the GPLv2
- *
  */
 
 #include <linux/async.h>
@@ -309,7 +307,7 @@ int bus_for_each_dev(struct bus_type *bus, struct device *start,
 
 	klist_iter_init_node(&bus->p->klist_devices, &i,
 			     (start ? &start->p->knode_bus : NULL));
-	while ((dev = next_device(&i)) && !error)
+	while (!error && (dev = next_device(&i)))
 		error = fn(dev, data);
 	klist_iter_exit(&i);
 	return error;
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index 07532d83be0b..edf726267282 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * cacheinfo support - processor cache information via sysfs
  *
  * Based on arch/x86/kernel/cpu/intel_cacheinfo.c
  * Author: Sudeep Holla <sudeep.holla@arm.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
diff --git a/drivers/base/class.c b/drivers/base/class.c
index 52eb8e644acd..54def4e02f00 100644
--- a/drivers/base/class.c
+++ b/drivers/base/class.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * class.c - basic device class management
  *
@@ -5,9 +6,6 @@
  * Copyright (c) 2002-3 Open Source Development Labs
  * Copyright (c) 2003-2004 Greg Kroah-Hartman
  * Copyright (c) 2003-2004 IBM Corp.
- *
- * This file is released under the GPLv2
- *
  */
 
 #include <linux/device.h>
diff --git a/drivers/base/component.c b/drivers/base/component.c
index 89b032f2ffd2..8946dfee4768 100644
--- a/drivers/base/component.c
+++ b/drivers/base/component.c
@@ -1,10 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Componentized device handling.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * This is work in progress.  We gather up the component devices into a list,
  * and bind them when instructed.  At the moment, we're specific to the DRM
  * subsystem, and only handles one master device, but this doesn't have to be
@@ -17,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
+#include <linux/debugfs.h>
 
 struct component;
 
@@ -41,6 +39,7 @@ struct master {
 	const struct component_master_ops *ops;
 	struct device *dev;
 	struct component_match *match;
+	struct dentry *dentry;
 };
 
 struct component {
@@ -56,6 +55,80 @@ static DEFINE_MUTEX(component_mutex);
 static LIST_HEAD(component_list);
 static LIST_HEAD(masters);
 
+#ifdef CONFIG_DEBUG_FS
+
+static struct dentry *component_debugfs_dir;
+
+static int component_devices_show(struct seq_file *s, void *data)
+{
+	struct master *m = s->private;
+	struct component_match *match = m->match;
+	size_t i;
+
+	mutex_lock(&component_mutex);
+	seq_printf(s, "%-40s %20s\n", "master name", "status");
+	seq_puts(s, "-------------------------------------------------------------\n");
+	seq_printf(s, "%-40s %20s\n\n",
+		   dev_name(m->dev), m->bound ? "bound" : "not bound");
+
+	seq_printf(s, "%-40s %20s\n", "device name", "status");
+	seq_puts(s, "-------------------------------------------------------------\n");
+	for (i = 0; i < match->num; i++) {
+		struct device *d = (struct device *)match->compare[i].data;
+
+		seq_printf(s, "%-40s %20s\n", dev_name(d),
+			   match->compare[i].component ?
+			   "registered" : "not registered");
+	}
+	mutex_unlock(&component_mutex);
+
+	return 0;
+}
+
+static int component_devices_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, component_devices_show, inode->i_private);
+}
+
+static const struct file_operations component_devices_fops = {
+	.open = component_devices_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static int __init component_debug_init(void)
+{
+	component_debugfs_dir = debugfs_create_dir("device_component", NULL);
+
+	return 0;
+}
+
+core_initcall(component_debug_init);
+
+static void component_master_debugfs_add(struct master *m)
+{
+	m->dentry = debugfs_create_file(dev_name(m->dev), 0444,
+					component_debugfs_dir,
+					m, &component_devices_fops);
+}
+
+static void component_master_debugfs_del(struct master *m)
+{
+	debugfs_remove(m->dentry);
+	m->dentry = NULL;
+}
+
+#else
+
+static void component_master_debugfs_add(struct master *m)
+{ }
+
+static void component_master_debugfs_del(struct master *m)
+{ }
+
+#endif
+
 static struct master *__master_find(struct device *dev,
 	const struct component_master_ops *ops)
 {
@@ -290,6 +363,7 @@ static void free_master(struct master *master)
 	struct component_match *match = master->match;
 	int i;
 
+	component_master_debugfs_del(master);
 	list_del(&master->node);
 
 	if (match) {
@@ -323,6 +397,7 @@ int component_master_add_with_match(struct device *dev,
 	master->ops = ops;
 	master->match = match;
 
+	component_master_debugfs_add(master);
 	/* Add to the list of available masters. */
 	mutex_lock(&component_mutex);
 	list_add(&master->node, &masters);
diff --git a/drivers/base/container.c b/drivers/base/container.c
index ecbfbe2e908f..1ba42d2d3532 100644
--- a/drivers/base/container.c
+++ b/drivers/base/container.c
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * System bus type for containers.
  *
  * Copyright (C) 2013, Intel Corporation
  * Author: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/container.h>
diff --git a/drivers/base/core.c b/drivers/base/core.c
index 110230d86527..5847364f25d9 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * drivers/base/core.c - core driver model code (device registration, etc)
  *
@@ -5,9 +6,6 @@
  * Copyright (c) 2002-3 Open Source Development Labs
  * Copyright (c) 2006 Greg Kroah-Hartman <gregkh@suse.de>
  * Copyright (c) 2006 Novell, Inc.
- *
- * This file is released under the GPLv2
- *
  */
 
 #include <linux/device.h>
@@ -22,7 +20,6 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/genhd.h>
-#include <linux/kallsyms.h>
 #include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 #include <linux/netdevice.h>
@@ -313,6 +310,9 @@ static void __device_link_del(struct device_link *link)
 	dev_info(link->consumer, "Dropping the link to %s\n",
 		 dev_name(link->supplier));
 
+	if (link->flags & DL_FLAG_PM_RUNTIME)
+		pm_runtime_drop_link(link->consumer);
+
 	list_del(&link->s_node);
 	list_del(&link->c_node);
 	device_link_free(link);
@@ -687,8 +687,8 @@ static ssize_t dev_attr_show(struct kobject *kobj, struct attribute *attr,
 	if (dev_attr->show)
 		ret = dev_attr->show(dev, dev_attr, buf);
 	if (ret >= (ssize_t)PAGE_SIZE) {
-		print_symbol("dev_attr_show: %s returned bad count\n",
-				(unsigned long)dev_attr->show);
+		printk("dev_attr_show: %pS returned bad count\n",
+				dev_attr->show);
 	}
 	return ret;
 }
@@ -2116,7 +2116,7 @@ int device_for_each_child(struct device *parent, void *data,
 		return 0;
 
 	klist_iter_init(&parent->p->klist_children, &i);
-	while ((child = next_device(&i)) && !error)
+	while (!error && (child = next_device(&i)))
 		error = fn(child, data);
 	klist_iter_exit(&i);
 	return error;
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index d99038487a0d..d21a2d913107 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * CPU subsystem support
  */
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 2c964f56dafe..de6fd092bf2f 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * drivers/base/dd.c - The core device/driver interactions.
  *
@@ -13,8 +14,6 @@
  * Copyright (c) 2002-3 Open Source Development Labs
  * Copyright (c) 2007-2009 Greg Kroah-Hartman <gregkh@suse.de>
  * Copyright (c) 2007-2009 Novell Inc.
- *
- * This file is released under the GPLv2
  */
 
 #include <linux/device.h>
@@ -289,6 +288,18 @@ static void driver_bound(struct device *dev)
 	kobject_uevent(&dev->kobj, KOBJ_BIND);
 }
 
+static ssize_t coredump_store(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	device_lock(dev);
+	if (dev->driver->coredump)
+		dev->driver->coredump(dev);
+	device_unlock(dev);
+
+	return count;
+}
+static DEVICE_ATTR_WO(coredump);
+
 static int driver_sysfs_add(struct device *dev)
 {
 	int ret;
@@ -298,14 +309,26 @@ static int driver_sysfs_add(struct device *dev)
 					     BUS_NOTIFY_BIND_DRIVER, dev);
 
 	ret = sysfs_create_link(&dev->driver->p->kobj, &dev->kobj,
+				kobject_name(&dev->kobj));
+	if (ret)
+		goto fail;
+
+	ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj,
+				"driver");
+	if (ret)
+		goto rm_dev;
+
+	if (!IS_ENABLED(CONFIG_DEV_COREDUMP) || !dev->driver->coredump ||
+	    !device_create_file(dev, &dev_attr_coredump))
+		return 0;
+
+	sysfs_remove_link(&dev->kobj, "driver");
+
+rm_dev:
+	sysfs_remove_link(&dev->driver->p->kobj,
 			  kobject_name(&dev->kobj));
-	if (ret == 0) {
-		ret = sysfs_create_link(&dev->kobj, &dev->driver->p->kobj,
-					"driver");
-		if (ret)
-			sysfs_remove_link(&dev->driver->p->kobj,
-					kobject_name(&dev->kobj));
-	}
+
+fail:
 	return ret;
 }
 
@@ -314,6 +337,8 @@ static void driver_sysfs_remove(struct device *dev)
 	struct device_driver *drv = dev->driver;
 
 	if (drv) {
+		if (drv->coredump)
+			device_remove_file(dev, &dev_attr_coredump);
 		sysfs_remove_link(&drv->p->kobj, kobject_name(&dev->kobj));
 		sysfs_remove_link(&dev->kobj, "driver");
 	}
diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c
index 7be310f7db73..f1a3353f3494 100644
--- a/drivers/base/devcoredump.c
+++ b/drivers/base/devcoredump.c
@@ -1,23 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * This file is provided under the GPLv2 license.
- *
- * GPL LICENSE SUMMARY
- *
  * Copyright(c) 2014 Intel Mobile Communications GmbH
  * Copyright(c) 2015 Intel Deutschland GmbH
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * The full GNU General Public License is included in this distribution
- * in the file called COPYING.
- *
  * Contact Information:
  *  Intel Linux Wireless <ilw@linux.intel.com>
  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
diff --git a/drivers/base/devres.c b/drivers/base/devres.c
index 71d577025285..95b67281cd2a 100644
--- a/drivers/base/devres.c
+++ b/drivers/base/devres.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * drivers/base/devres.c - device resource management
  *
  * Copyright (c) 2006  SUSE Linux Products GmbH
  * Copyright (c) 2006  Tejun Heo <teheo@suse.de>
- *
- * This file is released under the GPLv2.
  */
 
 #include <linux/device.h>
diff --git a/drivers/base/dma-contiguous.c b/drivers/base/dma-contiguous.c
index ea9726e71468..d987dcd1bd56 100644
--- a/drivers/base/dma-contiguous.c
+++ b/drivers/base/dma-contiguous.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Contiguous Memory Allocator for DMA mapping framework
  * Copyright (c) 2010-2011 by Samsung Electronics.
  * Written by:
  *	Marek Szyprowski <m.szyprowski@samsung.com>
  *	Michal Nazarewicz <mina86@mina86.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License or (at your optional) any later version of the license.
  */
 
 #define pr_fmt(fmt) "cma: " fmt
diff --git a/drivers/base/dma-mapping.c b/drivers/base/dma-mapping.c
index e584eddef0a7..3b118353ea17 100644
--- a/drivers/base/dma-mapping.c
+++ b/drivers/base/dma-mapping.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * drivers/base/dma-mapping.c - arch-independent dma-mapping routines
  *
  * Copyright (c) 2006  SUSE Linux Products GmbH
  * Copyright (c) 2006  Tejun Heo <teheo@suse.de>
- *
- * This file is released under the GPLv2.
  */
 
 #include <linux/acpi.h>
diff --git a/drivers/base/driver.c b/drivers/base/driver.c
index 4eabfe28d2b3..ba912558a510 100644
--- a/drivers/base/driver.c
+++ b/drivers/base/driver.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * driver.c - centralized device driver management
  *
@@ -5,9 +6,6 @@
  * Copyright (c) 2002-3 Open Source Development Labs
  * Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de>
  * Copyright (c) 2007 Novell Inc.
- *
- * This file is released under the GPLv2
- *
  */
 
 #include <linux/device.h>
@@ -52,7 +50,7 @@ int driver_for_each_device(struct device_driver *drv, struct device *start,
 
 	klist_iter_init_node(&drv->p->klist_devices, &i,
 			     start ? &start->p->knode_driver : NULL);
-	while ((dev = next_device(&i)) && !error)
+	while (!error && (dev = next_device(&i)))
 		error = fn(dev, data);
 	klist_iter_exit(&i);
 	return error;
diff --git a/drivers/base/firmware.c b/drivers/base/firmware.c
index 113815556809..8dff940e0db9 100644
--- a/drivers/base/firmware.c
+++ b/drivers/base/firmware.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * firmware.c - firmware subsystem hoohaw.
  *
@@ -5,8 +6,6 @@
  * Copyright (c) 2002-3 Open Source Development Labs
  * Copyright (c) 2007 Greg Kroah-Hartman <gregkh@suse.de>
  * Copyright (c) 2007 Novell Inc.
- *
- * This file is released under the GPLv2
  */
 #include <linux/kobject.h>
 #include <linux/module.h>
diff --git a/drivers/base/firmware_class.c b/drivers/base/firmware_class.c
index 4b57cf5bc81d..7dd36ace6152 100644
--- a/drivers/base/firmware_class.c
+++ b/drivers/base/firmware_class.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * firmware_class.c - Multi purpose firmware loading support
  *
@@ -41,6 +42,96 @@ MODULE_AUTHOR("Manuel Estrada Sainz");
 MODULE_DESCRIPTION("Multi purpose firmware loading support");
 MODULE_LICENSE("GPL");
 
+enum fw_status {
+	FW_STATUS_UNKNOWN,
+	FW_STATUS_LOADING,
+	FW_STATUS_DONE,
+	FW_STATUS_ABORTED,
+};
+
+/*
+ * Concurrent request_firmware() for the same firmware need to be
+ * serialized.  struct fw_state is simple state machine which hold the
+ * state of the firmware loading.
+ */
+struct fw_state {
+	struct completion completion;
+	enum fw_status status;
+};
+
+/* firmware behavior options */
+#define FW_OPT_UEVENT	(1U << 0)
+#define FW_OPT_NOWAIT	(1U << 1)
+#define FW_OPT_USERHELPER	(1U << 2)
+#define FW_OPT_NO_WARN	(1U << 3)
+#define FW_OPT_NOCACHE	(1U << 4)
+#define FW_OPT_NOFALLBACK (1U << 5)
+
+struct firmware_cache {
+	/* firmware_buf instance will be added into the below list */
+	spinlock_t lock;
+	struct list_head head;
+	int state;
+
+#ifdef CONFIG_PM_SLEEP
+	/*
+	 * Names of firmware images which have been cached successfully
+	 * will be added into the below list so that device uncache
+	 * helper can trace which firmware images have been cached
+	 * before.
+	 */
+	spinlock_t name_lock;
+	struct list_head fw_names;
+
+	struct delayed_work work;
+
+	struct notifier_block   pm_notify;
+#endif
+};
+
+struct fw_priv {
+	struct kref ref;
+	struct list_head list;
+	struct firmware_cache *fwc;
+	struct fw_state fw_st;
+	void *data;
+	size_t size;
+	size_t allocated_size;
+#ifdef CONFIG_FW_LOADER_USER_HELPER
+	bool is_paged_buf;
+	bool need_uevent;
+	struct page **pages;
+	int nr_pages;
+	int page_array_size;
+	struct list_head pending_list;
+#endif
+	const char *fw_name;
+};
+
+struct fw_cache_entry {
+	struct list_head list;
+	const char *name;
+};
+
+struct fw_name_devm {
+	unsigned long magic;
+	const char *name;
+};
+
+static inline struct fw_priv *to_fw_priv(struct kref *ref)
+{
+	return container_of(ref, struct fw_priv, ref);
+}
+
+#define	FW_LOADER_NO_CACHE	0
+#define	FW_LOADER_START_CACHE	1
+
+/* fw_lock could be moved to 'struct fw_sysfs' but since it is just
+ * guarding for corner cases a global lock should be OK */
+static DEFINE_MUTEX(fw_lock);
+
+static struct firmware_cache fw_cache;
+
 /* Builtin firmware support */
 
 #ifdef CONFIG_FW_LOADER
@@ -48,6 +139,14 @@ MODULE_LICENSE("GPL");
 extern struct builtin_fw __start_builtin_fw[];
 extern struct builtin_fw __end_builtin_fw[];
 
+static void fw_copy_to_prealloc_buf(struct firmware *fw,
+				    void *buf, size_t size)
+{
+	if (!buf || size < fw->size)
+		return;
+	memcpy(buf, fw->data, fw->size);
+}
+
 static bool fw_get_builtin_firmware(struct firmware *fw, const char *name,
 				    void *buf, size_t size)
 {
@@ -57,9 +156,8 @@ static bool fw_get_builtin_firmware(struct firmware *fw, const char *name,
 		if (strcmp(name, b_fw->name) == 0) {
 			fw->size = b_fw->size;
 			fw->data = b_fw->data;
+			fw_copy_to_prealloc_buf(fw, buf, size);
 
-			if (buf && fw->size <= size)
-				memcpy(buf, fw->data, fw->size);
 			return true;
 		}
 	}
@@ -93,13 +191,6 @@ static inline bool fw_is_builtin_firmware(const struct firmware *fw)
 }
 #endif
 
-enum fw_status {
-	FW_STATUS_UNKNOWN,
-	FW_STATUS_LOADING,
-	FW_STATUS_DONE,
-	FW_STATUS_ABORTED,
-};
-
 static int loading_timeout = 60;	/* In seconds */
 
 static inline long firmware_loading_timeout(void)
@@ -107,29 +198,17 @@ static inline long firmware_loading_timeout(void)
 	return loading_timeout > 0 ? loading_timeout * HZ : MAX_JIFFY_OFFSET;
 }
 
-/*
- * Concurrent request_firmware() for the same firmware need to be
- * serialized.  struct fw_state is simple state machine which hold the
- * state of the firmware loading.
- */
-struct fw_state {
-	struct completion completion;
-	enum fw_status status;
-};
-
-static void fw_state_init(struct fw_state *fw_st)
+static void fw_state_init(struct fw_priv *fw_priv)
 {
+	struct fw_state *fw_st = &fw_priv->fw_st;
+
 	init_completion(&fw_st->completion);
 	fw_st->status = FW_STATUS_UNKNOWN;
 }
 
-static inline bool __fw_state_is_done(enum fw_status status)
-{
-	return status == FW_STATUS_DONE || status == FW_STATUS_ABORTED;
-}
-
-static int __fw_state_wait_common(struct fw_state *fw_st, long timeout)
+static int __fw_state_wait_common(struct fw_priv *fw_priv, long timeout)
 {
+	struct fw_state *fw_st = &fw_priv->fw_st;
 	long ret;
 
 	ret = wait_for_completion_killable_timeout(&fw_st->completion, timeout);
@@ -141,226 +220,172 @@ static int __fw_state_wait_common(struct fw_state *fw_st, long timeout)
 	return ret < 0 ? ret : 0;
 }
 
-static void __fw_state_set(struct fw_state *fw_st,
+static void __fw_state_set(struct fw_priv *fw_priv,
 			   enum fw_status status)
 {
+	struct fw_state *fw_st = &fw_priv->fw_st;
+
 	WRITE_ONCE(fw_st->status, status);
 
 	if (status == FW_STATUS_DONE || status == FW_STATUS_ABORTED)
 		complete_all(&fw_st->completion);
 }
 
-#define fw_state_start(fw_st)					\
-	__fw_state_set(fw_st, FW_STATUS_LOADING)
-#define fw_state_done(fw_st)					\
-	__fw_state_set(fw_st, FW_STATUS_DONE)
-#define fw_state_aborted(fw_st)					\
-	__fw_state_set(fw_st, FW_STATUS_ABORTED)
-#define fw_state_wait(fw_st)					\
-	__fw_state_wait_common(fw_st, MAX_SCHEDULE_TIMEOUT)
-
-static int __fw_state_check(struct fw_state *fw_st, enum fw_status status)
+static inline void fw_state_start(struct fw_priv *fw_priv)
 {
-	return fw_st->status == status;
+	__fw_state_set(fw_priv, FW_STATUS_LOADING);
 }
 
-#define fw_state_is_aborted(fw_st)				\
-	__fw_state_check(fw_st, FW_STATUS_ABORTED)
-
-#ifdef CONFIG_FW_LOADER_USER_HELPER
-
-#define fw_state_aborted(fw_st)					\
-	__fw_state_set(fw_st, FW_STATUS_ABORTED)
-#define fw_state_is_done(fw_st)					\
-	__fw_state_check(fw_st, FW_STATUS_DONE)
-#define fw_state_is_loading(fw_st)				\
-	__fw_state_check(fw_st, FW_STATUS_LOADING)
-#define fw_state_wait_timeout(fw_st, timeout)			\
-	__fw_state_wait_common(fw_st, timeout)
-
-#endif /* CONFIG_FW_LOADER_USER_HELPER */
+static inline void fw_state_done(struct fw_priv *fw_priv)
+{
+	__fw_state_set(fw_priv, FW_STATUS_DONE);
+}
 
-/* firmware behavior options */
-#define FW_OPT_UEVENT	(1U << 0)
-#define FW_OPT_NOWAIT	(1U << 1)
-#ifdef CONFIG_FW_LOADER_USER_HELPER
-#define FW_OPT_USERHELPER	(1U << 2)
-#else
-#define FW_OPT_USERHELPER	0
-#endif
-#ifdef CONFIG_FW_LOADER_USER_HELPER_FALLBACK
-#define FW_OPT_FALLBACK		FW_OPT_USERHELPER
-#else
-#define FW_OPT_FALLBACK		0
-#endif
-#define FW_OPT_NO_WARN	(1U << 3)
-#define FW_OPT_NOCACHE	(1U << 4)
+static inline void fw_state_aborted(struct fw_priv *fw_priv)
+{
+	__fw_state_set(fw_priv, FW_STATUS_ABORTED);
+}
 
-struct firmware_cache {
-	/* firmware_buf instance will be added into the below list */
-	spinlock_t lock;
-	struct list_head head;
-	int state;
+static inline int fw_state_wait(struct fw_priv *fw_priv)
+{
+	return __fw_state_wait_common(fw_priv, MAX_SCHEDULE_TIMEOUT);
+}
 
-#ifdef CONFIG_PM_SLEEP
-	/*
-	 * Names of firmware images which have been cached successfully
-	 * will be added into the below list so that device uncache
-	 * helper can trace which firmware images have been cached
-	 * before.
-	 */
-	spinlock_t name_lock;
-	struct list_head fw_names;
+static bool __fw_state_check(struct fw_priv *fw_priv,
+			     enum fw_status status)
+{
+	struct fw_state *fw_st = &fw_priv->fw_st;
 
-	struct delayed_work work;
+	return fw_st->status == status;
+}
 
-	struct notifier_block   pm_notify;
-#endif
-};
+static inline bool fw_state_is_aborted(struct fw_priv *fw_priv)
+{
+	return __fw_state_check(fw_priv, FW_STATUS_ABORTED);
+}
 
-struct firmware_buf {
-	struct kref ref;
-	struct list_head list;
-	struct firmware_cache *fwc;
-	struct fw_state fw_st;
-	void *data;
-	size_t size;
-	size_t allocated_size;
 #ifdef CONFIG_FW_LOADER_USER_HELPER
-	bool is_paged_buf;
-	bool need_uevent;
-	struct page **pages;
-	int nr_pages;
-	int page_array_size;
-	struct list_head pending_list;
-#endif
-	const char *fw_id;
-};
 
-struct fw_cache_entry {
-	struct list_head list;
-	const char *name;
-};
+static inline bool fw_sysfs_done(struct fw_priv *fw_priv)
+{
+	return __fw_state_check(fw_priv, FW_STATUS_DONE);
+}
 
-struct fw_name_devm {
-	unsigned long magic;
-	const char *name;
-};
+static inline bool fw_sysfs_loading(struct fw_priv *fw_priv)
+{
+	return __fw_state_check(fw_priv, FW_STATUS_LOADING);
+}
 
-#define to_fwbuf(d) container_of(d, struct firmware_buf, ref)
+static inline int fw_sysfs_wait_timeout(struct fw_priv *fw_priv,  long timeout)
+{
+	return __fw_state_wait_common(fw_priv, timeout);
+}
 
-#define	FW_LOADER_NO_CACHE	0
-#define	FW_LOADER_START_CACHE	1
+#endif /* CONFIG_FW_LOADER_USER_HELPER */
 
 static int fw_cache_piggyback_on_request(const char *name);
 
-/* fw_lock could be moved to 'struct firmware_priv' but since it is just
- * guarding for corner cases a global lock should be OK */
-static DEFINE_MUTEX(fw_lock);
-
-static struct firmware_cache fw_cache;
-
-static struct firmware_buf *__allocate_fw_buf(const char *fw_name,
-					      struct firmware_cache *fwc,
-					      void *dbuf, size_t size)
+static struct fw_priv *__allocate_fw_priv(const char *fw_name,
+					  struct firmware_cache *fwc,
+					  void *dbuf, size_t size)
 {
-	struct firmware_buf *buf;
+	struct fw_priv *fw_priv;
 
-	buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
-	if (!buf)
+	fw_priv = kzalloc(sizeof(*fw_priv), GFP_ATOMIC);
+	if (!fw_priv)
 		return NULL;
 
-	buf->fw_id = kstrdup_const(fw_name, GFP_ATOMIC);
-	if (!buf->fw_id) {
-		kfree(buf);
+	fw_priv->fw_name = kstrdup_const(fw_name, GFP_ATOMIC);
+	if (!fw_priv->fw_name) {
+		kfree(fw_priv);
 		return NULL;
 	}
 
-	kref_init(&buf->ref);
-	buf->fwc = fwc;
-	buf->data = dbuf;
-	buf->allocated_size = size;
-	fw_state_init(&buf->fw_st);
+	kref_init(&fw_priv->ref);
+	fw_priv->fwc = fwc;
+	fw_priv->data = dbuf;
+	fw_priv->allocated_size = size;
+	fw_state_init(fw_priv);
 #ifdef CONFIG_FW_LOADER_USER_HELPER
-	INIT_LIST_HEAD(&buf->pending_list);
+	INIT_LIST_HEAD(&fw_priv->pending_list);
 #endif
 
-	pr_debug("%s: fw-%s buf=%p\n", __func__, fw_name, buf);
+	pr_debug("%s: fw-%s fw_priv=%p\n", __func__, fw_name, fw_priv);
 
-	return buf;
+	return fw_priv;
 }
 
-static struct firmware_buf *__fw_lookup_buf(const char *fw_name)
+static struct fw_priv *__lookup_fw_priv(const char *fw_name)
 {
-	struct firmware_buf *tmp;
+	struct fw_priv *tmp;
 	struct firmware_cache *fwc = &fw_cache;
 
 	list_for_each_entry(tmp, &fwc->head, list)
-		if (!strcmp(tmp->fw_id, fw_name))
+		if (!strcmp(tmp->fw_name, fw_name))
 			return tmp;
 	return NULL;
 }
 
 /* Returns 1 for batching firmware requests with the same name */
-static int fw_lookup_and_allocate_buf(const char *fw_name,
-				      struct firmware_cache *fwc,
-				      struct firmware_buf **buf, void *dbuf,
-				      size_t size)
+static int alloc_lookup_fw_priv(const char *fw_name,
+				struct firmware_cache *fwc,
+				struct fw_priv **fw_priv, void *dbuf,
+				size_t size)
 {
-	struct firmware_buf *tmp;
+	struct fw_priv *tmp;
 
 	spin_lock(&fwc->lock);
-	tmp = __fw_lookup_buf(fw_name);
+	tmp = __lookup_fw_priv(fw_name);
 	if (tmp) {
 		kref_get(&tmp->ref);
 		spin_unlock(&fwc->lock);
-		*buf = tmp;
-		pr_debug("batched request - sharing the same struct firmware_buf and lookup for multiple requests\n");
+		*fw_priv = tmp;
+		pr_debug("batched request - sharing the same struct fw_priv and lookup for multiple requests\n");
 		return 1;
 	}
-	tmp = __allocate_fw_buf(fw_name, fwc, dbuf, size);
+	tmp = __allocate_fw_priv(fw_name, fwc, dbuf, size);
 	if (tmp)
 		list_add(&tmp->list, &fwc->head);
 	spin_unlock(&fwc->lock);
 
-	*buf = tmp;
+	*fw_priv = tmp;
 
 	return tmp ? 0 : -ENOMEM;
 }
 
-static void __fw_free_buf(struct kref *ref)
+static void __free_fw_priv(struct kref *ref)
 	__releases(&fwc->lock)
 {
-	struct firmware_buf *buf = to_fwbuf(ref);
-	struct firmware_cache *fwc = buf->fwc;
+	struct fw_priv *fw_priv = to_fw_priv(ref);
+	struct firmware_cache *fwc = fw_priv->fwc;
 
-	pr_debug("%s: fw-%s buf=%p data=%p size=%u\n",
-		 __func__, buf->fw_id, buf, buf->data,
-		 (unsigned int)buf->size);
+	pr_debug("%s: fw-%s fw_priv=%p data=%p size=%u\n",
+		 __func__, fw_priv->fw_name, fw_priv, fw_priv->data,
+		 (unsigned int)fw_priv->size);
 
-	list_del(&buf->list);
+	list_del(&fw_priv->list);
 	spin_unlock(&fwc->lock);
 
 #ifdef CONFIG_FW_LOADER_USER_HELPER
-	if (buf->is_paged_buf) {
+	if (fw_priv->is_paged_buf) {
 		int i;
-		vunmap(buf->data);
-		for (i = 0; i < buf->nr_pages; i++)
-			__free_page(buf->pages[i]);
-		vfree(buf->pages);
+		vunmap(fw_priv->data);
+		for (i = 0; i < fw_priv->nr_pages; i++)
+			__free_page(fw_priv->pages[i]);
+		vfree(fw_priv->pages);
 	} else
 #endif
-	if (!buf->allocated_size)
-		vfree(buf->data);
-	kfree_const(buf->fw_id);
-	kfree(buf);
+	if (!fw_priv->allocated_size)
+		vfree(fw_priv->data);
+	kfree_const(fw_priv->fw_name);
+	kfree(fw_priv);
 }
 
-static void fw_free_buf(struct firmware_buf *buf)
+static void free_fw_priv(struct fw_priv *fw_priv)
 {
-	struct firmware_cache *fwc = buf->fwc;
+	struct firmware_cache *fwc = fw_priv->fwc;
 	spin_lock(&fwc->lock);
-	if (!kref_put(&buf->ref, __fw_free_buf))
+	if (!kref_put(&fw_priv->ref, __free_fw_priv))
 		spin_unlock(&fwc->lock);
 }
 
@@ -383,7 +408,7 @@ module_param_string(path, fw_path_para, sizeof(fw_path_para), 0644);
 MODULE_PARM_DESC(path, "customized firmware image search path with a higher priority than default path");
 
 static int
-fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf)
+fw_get_filesystem_firmware(struct device *device, struct fw_priv *fw_priv)
 {
 	loff_t size;
 	int i, len;
@@ -393,9 +418,9 @@ fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf)
 	size_t msize = INT_MAX;
 
 	/* Already populated data member means we're loading into a buffer */
-	if (buf->data) {
+	if (fw_priv->data) {
 		id = READING_FIRMWARE_PREALLOC_BUFFER;
-		msize = buf->allocated_size;
+		msize = fw_priv->allocated_size;
 	}
 
 	path = __getname();
@@ -408,15 +433,15 @@ fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf)
 			continue;
 
 		len = snprintf(path, PATH_MAX, "%s/%s",
-			       fw_path[i], buf->fw_id);
+			       fw_path[i], fw_priv->fw_name);
 		if (len >= PATH_MAX) {
 			rc = -ENAMETOOLONG;
 			break;
 		}
 
-		buf->size = 0;
-		rc = kernel_read_file_from_path(path, &buf->data, &size, msize,
-						id);
+		fw_priv->size = 0;
+		rc = kernel_read_file_from_path(path, &fw_priv->data, &size,
+						msize, id);
 		if (rc) {
 			if (rc == -ENOENT)
 				dev_dbg(device, "loading %s failed with error %d\n",
@@ -426,9 +451,9 @@ fw_get_filesystem_firmware(struct device *device, struct firmware_buf *buf)
 					 path, rc);
 			continue;
 		}
-		dev_dbg(device, "direct-loading %s\n", buf->fw_id);
-		buf->size = size;
-		fw_state_done(&buf->fw_st);
+		dev_dbg(device, "direct-loading %s\n", fw_priv->fw_name);
+		fw_priv->size = size;
+		fw_state_done(fw_priv);
 		break;
 	}
 	__putname(path);
@@ -444,22 +469,22 @@ static void firmware_free_data(const struct firmware *fw)
 		vfree(fw->data);
 		return;
 	}
-	fw_free_buf(fw->priv);
+	free_fw_priv(fw->priv);
 }
 
 /* store the pages buffer info firmware from buf */
-static void fw_set_page_data(struct firmware_buf *buf, struct firmware *fw)
+static void fw_set_page_data(struct fw_priv *fw_priv, struct firmware *fw)
 {
-	fw->priv = buf;
+	fw->priv = fw_priv;
 #ifdef CONFIG_FW_LOADER_USER_HELPER
-	fw->pages = buf->pages;
+	fw->pages = fw_priv->pages;
 #endif
-	fw->size = buf->size;
-	fw->data = buf->data;
+	fw->size = fw_priv->size;
+	fw->data = fw_priv->data;
 
-	pr_debug("%s: fw-%s buf=%p data=%p size=%u\n",
-		 __func__, buf->fw_id, buf, buf->data,
-		 (unsigned int)buf->size);
+	pr_debug("%s: fw-%s fw_priv=%p data=%p size=%u\n",
+		 __func__, fw_priv->fw_name, fw_priv, fw_priv->data,
+		 (unsigned int)fw_priv->size);
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -523,13 +548,13 @@ static int fw_add_devm_name(struct device *dev, const char *name)
 }
 #endif
 
-static int assign_firmware_buf(struct firmware *fw, struct device *device,
-			       unsigned int opt_flags)
+static int assign_fw(struct firmware *fw, struct device *device,
+		     unsigned int opt_flags)
 {
-	struct firmware_buf *buf = fw->priv;
+	struct fw_priv *fw_priv = fw->priv;
 
 	mutex_lock(&fw_lock);
-	if (!buf->size || fw_state_is_aborted(&buf->fw_st)) {
+	if (!fw_priv->size || fw_state_is_aborted(fw_priv)) {
 		mutex_unlock(&fw_lock);
 		return -ENOENT;
 	}
@@ -544,20 +569,20 @@ static int assign_firmware_buf(struct firmware *fw, struct device *device,
 	/* don't cache firmware handled without uevent */
 	if (device && (opt_flags & FW_OPT_UEVENT) &&
 	    !(opt_flags & FW_OPT_NOCACHE))
-		fw_add_devm_name(device, buf->fw_id);
+		fw_add_devm_name(device, fw_priv->fw_name);
 
 	/*
 	 * After caching firmware image is started, let it piggyback
 	 * on request firmware.
 	 */
 	if (!(opt_flags & FW_OPT_NOCACHE) &&
-	    buf->fwc->state == FW_LOADER_START_CACHE) {
-		if (fw_cache_piggyback_on_request(buf->fw_id))
-			kref_get(&buf->ref);
+	    fw_priv->fwc->state == FW_LOADER_START_CACHE) {
+		if (fw_cache_piggyback_on_request(fw_priv->fw_name))
+			kref_get(&fw_priv->ref);
 	}
 
 	/* pass the pages buffer to driver at the last minute */
-	fw_set_page_data(buf, fw);
+	fw_set_page_data(fw_priv, fw);
 	mutex_unlock(&fw_lock);
 	return 0;
 }
@@ -566,49 +591,50 @@ static int assign_firmware_buf(struct firmware *fw, struct device *device,
  * user-mode helper code
  */
 #ifdef CONFIG_FW_LOADER_USER_HELPER
-struct firmware_priv {
+struct fw_sysfs {
 	bool nowait;
 	struct device dev;
-	struct firmware_buf *buf;
+	struct fw_priv *fw_priv;
 	struct firmware *fw;
 };
 
-static struct firmware_priv *to_firmware_priv(struct device *dev)
+static struct fw_sysfs *to_fw_sysfs(struct device *dev)
 {
-	return container_of(dev, struct firmware_priv, dev);
+	return container_of(dev, struct fw_sysfs, dev);
 }
 
-static void __fw_load_abort(struct firmware_buf *buf)
+static void __fw_load_abort(struct fw_priv *fw_priv)
 {
 	/*
 	 * There is a small window in which user can write to 'loading'
 	 * between loading done and disappearance of 'loading'
 	 */
-	if (fw_state_is_done(&buf->fw_st))
+	if (fw_sysfs_done(fw_priv))
 		return;
 
-	list_del_init(&buf->pending_list);
-	fw_state_aborted(&buf->fw_st);
+	list_del_init(&fw_priv->pending_list);
+	fw_state_aborted(fw_priv);
 }
 
-static void fw_load_abort(struct firmware_priv *fw_priv)
+static void fw_load_abort(struct fw_sysfs *fw_sysfs)
 {
-	struct firmware_buf *buf = fw_priv->buf;
+	struct fw_priv *fw_priv = fw_sysfs->fw_priv;
 
-	__fw_load_abort(buf);
+	__fw_load_abort(fw_priv);
 }
 
 static LIST_HEAD(pending_fw_head);
 
 static void kill_pending_fw_fallback_reqs(bool only_kill_custom)
 {
-	struct firmware_buf *buf;
-	struct firmware_buf *next;
+	struct fw_priv *fw_priv;
+	struct fw_priv *next;
 
 	mutex_lock(&fw_lock);
-	list_for_each_entry_safe(buf, next, &pending_fw_head, pending_list) {
-		if (!buf->need_uevent || !only_kill_custom)
-			 __fw_load_abort(buf);
+	list_for_each_entry_safe(fw_priv, next, &pending_fw_head,
+				 pending_list) {
+		if (!fw_priv->need_uevent || !only_kill_custom)
+			 __fw_load_abort(fw_priv);
 	}
 	mutex_unlock(&fw_lock);
 }
@@ -651,18 +677,18 @@ ATTRIBUTE_GROUPS(firmware_class);
 
 static void fw_dev_release(struct device *dev)
 {
-	struct firmware_priv *fw_priv = to_firmware_priv(dev);
+	struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev);
 
-	kfree(fw_priv);
+	kfree(fw_sysfs);
 }
 
-static int do_firmware_uevent(struct firmware_priv *fw_priv, struct kobj_uevent_env *env)
+static int do_firmware_uevent(struct fw_sysfs *fw_sysfs, struct kobj_uevent_env *env)
 {
-	if (add_uevent_var(env, "FIRMWARE=%s", fw_priv->buf->fw_id))
+	if (add_uevent_var(env, "FIRMWARE=%s", fw_sysfs->fw_priv->fw_name))
 		return -ENOMEM;
 	if (add_uevent_var(env, "TIMEOUT=%i", loading_timeout))
 		return -ENOMEM;
-	if (add_uevent_var(env, "ASYNC=%d", fw_priv->nowait))
+	if (add_uevent_var(env, "ASYNC=%d", fw_sysfs->nowait))
 		return -ENOMEM;
 
 	return 0;
@@ -670,12 +696,12 @@ static int do_firmware_uevent(struct firmware_priv *fw_priv, struct kobj_uevent_
 
 static int firmware_uevent(struct device *dev, struct kobj_uevent_env *env)
 {
-	struct firmware_priv *fw_priv = to_firmware_priv(dev);
+	struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev);
 	int err = 0;
 
 	mutex_lock(&fw_lock);
-	if (fw_priv->buf)
-		err = do_firmware_uevent(fw_priv, env);
+	if (fw_sysfs->fw_priv)
+		err = do_firmware_uevent(fw_sysfs, env);
 	mutex_unlock(&fw_lock);
 	return err;
 }
@@ -687,15 +713,25 @@ static struct class firmware_class = {
 	.dev_release	= fw_dev_release,
 };
 
+static inline int register_sysfs_loader(void)
+{
+	return class_register(&firmware_class);
+}
+
+static inline void unregister_sysfs_loader(void)
+{
+	class_unregister(&firmware_class);
+}
+
 static ssize_t firmware_loading_show(struct device *dev,
 				     struct device_attribute *attr, char *buf)
 {
-	struct firmware_priv *fw_priv = to_firmware_priv(dev);
+	struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev);
 	int loading = 0;
 
 	mutex_lock(&fw_lock);
-	if (fw_priv->buf)
-		loading = fw_state_is_loading(&fw_priv->buf->fw_st);
+	if (fw_sysfs->fw_priv)
+		loading = fw_sysfs_loading(fw_sysfs->fw_priv);
 	mutex_unlock(&fw_lock);
 
 	return sprintf(buf, "%d\n", loading);
@@ -707,14 +743,15 @@ static ssize_t firmware_loading_show(struct device *dev,
 #endif
 
 /* one pages buffer should be mapped/unmapped only once */
-static int fw_map_pages_buf(struct firmware_buf *buf)
+static int map_fw_priv_pages(struct fw_priv *fw_priv)
 {
-	if (!buf->is_paged_buf)
+	if (!fw_priv->is_paged_buf)
 		return 0;
 
-	vunmap(buf->data);
-	buf->data = vmap(buf->pages, buf->nr_pages, 0, PAGE_KERNEL_RO);
-	if (!buf->data)
+	vunmap(fw_priv->data);
+	fw_priv->data = vmap(fw_priv->pages, fw_priv->nr_pages, 0,
+			     PAGE_KERNEL_RO);
+	if (!fw_priv->data)
 		return -ENOMEM;
 	return 0;
 }
@@ -736,32 +773,32 @@ static ssize_t firmware_loading_store(struct device *dev,
 				      struct device_attribute *attr,
 				      const char *buf, size_t count)
 {
-	struct firmware_priv *fw_priv = to_firmware_priv(dev);
-	struct firmware_buf *fw_buf;
+	struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev);
+	struct fw_priv *fw_priv;
 	ssize_t written = count;
 	int loading = simple_strtol(buf, NULL, 10);
 	int i;
 
 	mutex_lock(&fw_lock);
-	fw_buf = fw_priv->buf;
-	if (fw_state_is_aborted(&fw_buf->fw_st))
+	fw_priv = fw_sysfs->fw_priv;
+	if (fw_state_is_aborted(fw_priv))
 		goto out;
 
 	switch (loading) {
 	case 1:
 		/* discarding any previous partial load */
-		if (!fw_state_is_done(&fw_buf->fw_st)) {
-			for (i = 0; i < fw_buf->nr_pages; i++)
-				__free_page(fw_buf->pages[i]);
-			vfree(fw_buf->pages);
-			fw_buf->pages = NULL;
-			fw_buf->page_array_size = 0;
-			fw_buf->nr_pages = 0;
-			fw_state_start(&fw_buf->fw_st);
+		if (!fw_sysfs_done(fw_priv)) {
+			for (i = 0; i < fw_priv->nr_pages; i++)
+				__free_page(fw_priv->pages[i]);
+			vfree(fw_priv->pages);
+			fw_priv->pages = NULL;
+			fw_priv->page_array_size = 0;
+			fw_priv->nr_pages = 0;
+			fw_state_start(fw_priv);
 		}
 		break;
 	case 0:
-		if (fw_state_is_loading(&fw_buf->fw_st)) {
+		if (fw_sysfs_loading(fw_priv)) {
 			int rc;
 
 			/*
@@ -770,25 +807,25 @@ static ssize_t firmware_loading_store(struct device *dev,
 			 * see the mapped 'buf->data' once the loading
 			 * is completed.
 			 * */
-			rc = fw_map_pages_buf(fw_buf);
+			rc = map_fw_priv_pages(fw_priv);
 			if (rc)
 				dev_err(dev, "%s: map pages failed\n",
 					__func__);
 			else
 				rc = security_kernel_post_read_file(NULL,
-						fw_buf->data, fw_buf->size,
+						fw_priv->data, fw_priv->size,
 						READING_FIRMWARE);
 
 			/*
 			 * Same logic as fw_load_abort, only the DONE bit
 			 * is ignored and we set ABORT only on failure.
 			 */
-			list_del_init(&fw_buf->pending_list);
+			list_del_init(&fw_priv->pending_list);
 			if (rc) {
-				fw_state_aborted(&fw_buf->fw_st);
+				fw_state_aborted(fw_priv);
 				written = rc;
 			} else {
-				fw_state_done(&fw_buf->fw_st);
+				fw_state_done(fw_priv);
 			}
 			break;
 		}
@@ -797,7 +834,7 @@ static ssize_t firmware_loading_store(struct device *dev,
 		dev_err(dev, "%s: unexpected value (%d)\n", __func__, loading);
 		/* fallthrough */
 	case -1:
-		fw_load_abort(fw_priv);
+		fw_load_abort(fw_sysfs);
 		break;
 	}
 out:
@@ -807,16 +844,16 @@ static ssize_t firmware_loading_store(struct device *dev,
 
 static DEVICE_ATTR(loading, 0644, firmware_loading_show, firmware_loading_store);
 
-static void firmware_rw_buf(struct firmware_buf *buf, char *buffer,
+static void firmware_rw_data(struct fw_priv *fw_priv, char *buffer,
 			   loff_t offset, size_t count, bool read)
 {
 	if (read)
-		memcpy(buffer, buf->data + offset, count);
+		memcpy(buffer, fw_priv->data + offset, count);
 	else
-		memcpy(buf->data + offset, buffer, count);
+		memcpy(fw_priv->data + offset, buffer, count);
 }
 
-static void firmware_rw(struct firmware_buf *buf, char *buffer,
+static void firmware_rw(struct fw_priv *fw_priv, char *buffer,
 			loff_t offset, size_t count, bool read)
 {
 	while (count) {
@@ -825,14 +862,14 @@ static void firmware_rw(struct firmware_buf *buf, char *buffer,
 		int page_ofs = offset & (PAGE_SIZE-1);
 		int page_cnt = min_t(size_t, PAGE_SIZE - page_ofs, count);
 
-		page_data = kmap(buf->pages[page_nr]);
+		page_data = kmap(fw_priv->pages[page_nr]);
 
 		if (read)
 			memcpy(buffer, page_data + page_ofs, page_cnt);
 		else
 			memcpy(page_data + page_ofs, buffer, page_cnt);
 
-		kunmap(buf->pages[page_nr]);
+		kunmap(fw_priv->pages[page_nr]);
 		buffer += page_cnt;
 		offset += page_cnt;
 		count -= page_cnt;
@@ -844,69 +881,69 @@ static ssize_t firmware_data_read(struct file *filp, struct kobject *kobj,
 				  char *buffer, loff_t offset, size_t count)
 {
 	struct device *dev = kobj_to_dev(kobj);
-	struct firmware_priv *fw_priv = to_firmware_priv(dev);
-	struct firmware_buf *buf;
+	struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev);
+	struct fw_priv *fw_priv;
 	ssize_t ret_count;
 
 	mutex_lock(&fw_lock);
-	buf = fw_priv->buf;
-	if (!buf || fw_state_is_done(&buf->fw_st)) {
+	fw_priv = fw_sysfs->fw_priv;
+	if (!fw_priv || fw_sysfs_done(fw_priv)) {
 		ret_count = -ENODEV;
 		goto out;
 	}
-	if (offset > buf->size) {
+	if (offset > fw_priv->size) {
 		ret_count = 0;
 		goto out;
 	}
-	if (count > buf->size - offset)
-		count = buf->size - offset;
+	if (count > fw_priv->size - offset)
+		count = fw_priv->size - offset;
 
 	ret_count = count;
 
-	if (buf->data)
-		firmware_rw_buf(buf, buffer, offset, count, true);
+	if (fw_priv->data)
+		firmware_rw_data(fw_priv, buffer, offset, count, true);
 	else
-		firmware_rw(buf, buffer, offset, count, true);
+		firmware_rw(fw_priv, buffer, offset, count, true);
 
 out:
 	mutex_unlock(&fw_lock);
 	return ret_count;
 }
 
-static int fw_realloc_buffer(struct firmware_priv *fw_priv, int min_size)
+static int fw_realloc_pages(struct fw_sysfs *fw_sysfs, int min_size)
 {
-	struct firmware_buf *buf = fw_priv->buf;
+	struct fw_priv *fw_priv= fw_sysfs->fw_priv;
 	int pages_needed = PAGE_ALIGN(min_size) >> PAGE_SHIFT;
 
 	/* If the array of pages is too small, grow it... */
-	if (buf->page_array_size < pages_needed) {
+	if (fw_priv->page_array_size < pages_needed) {
 		int new_array_size = max(pages_needed,
-					 buf->page_array_size * 2);
+					 fw_priv->page_array_size * 2);
 		struct page **new_pages;
 
 		new_pages = vmalloc(new_array_size * sizeof(void *));
 		if (!new_pages) {
-			fw_load_abort(fw_priv);
+			fw_load_abort(fw_sysfs);
 			return -ENOMEM;
 		}
-		memcpy(new_pages, buf->pages,
-		       buf->page_array_size * sizeof(void *));
-		memset(&new_pages[buf->page_array_size], 0, sizeof(void *) *
-		       (new_array_size - buf->page_array_size));
-		vfree(buf->pages);
-		buf->pages = new_pages;
-		buf->page_array_size = new_array_size;
+		memcpy(new_pages, fw_priv->pages,
+		       fw_priv->page_array_size * sizeof(void *));
+		memset(&new_pages[fw_priv->page_array_size], 0, sizeof(void *) *
+		       (new_array_size - fw_priv->page_array_size));
+		vfree(fw_priv->pages);
+		fw_priv->pages = new_pages;
+		fw_priv->page_array_size = new_array_size;
 	}
 
-	while (buf->nr_pages < pages_needed) {
-		buf->pages[buf->nr_pages] =
+	while (fw_priv->nr_pages < pages_needed) {
+		fw_priv->pages[fw_priv->nr_pages] =
 			alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
 
-		if (!buf->pages[buf->nr_pages]) {
-			fw_load_abort(fw_priv);
+		if (!fw_priv->pages[fw_priv->nr_pages]) {
+			fw_load_abort(fw_sysfs);
 			return -ENOMEM;
 		}
-		buf->nr_pages++;
+		fw_priv->nr_pages++;
 	}
 	return 0;
 }
@@ -928,37 +965,37 @@ static ssize_t firmware_data_write(struct file *filp, struct kobject *kobj,
 				   char *buffer, loff_t offset, size_t count)
 {
 	struct device *dev = kobj_to_dev(kobj);
-	struct firmware_priv *fw_priv = to_firmware_priv(dev);
-	struct firmware_buf *buf;
+	struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev);
+	struct fw_priv *fw_priv;
 	ssize_t retval;
 
 	if (!capable(CAP_SYS_RAWIO))
 		return -EPERM;
 
 	mutex_lock(&fw_lock);
-	buf = fw_priv->buf;
-	if (!buf || fw_state_is_done(&buf->fw_st)) {
+	fw_priv = fw_sysfs->fw_priv;
+	if (!fw_priv || fw_sysfs_done(fw_priv)) {
 		retval = -ENODEV;
 		goto out;
 	}
 
-	if (buf->data) {
-		if (offset + count > buf->allocated_size) {
+	if (fw_priv->data) {
+		if (offset + count > fw_priv->allocated_size) {
 			retval = -ENOMEM;
 			goto out;
 		}
-		firmware_rw_buf(buf, buffer, offset, count, false);
+		firmware_rw_data(fw_priv, buffer, offset, count, false);
 		retval = count;
 	} else {
-		retval = fw_realloc_buffer(fw_priv, offset + count);
+		retval = fw_realloc_pages(fw_sysfs, offset + count);
 		if (retval)
 			goto out;
 
 		retval = count;
-		firmware_rw(buf, buffer, offset, count, false);
+		firmware_rw(fw_priv, buffer, offset, count, false);
 	}
 
-	buf->size = max_t(size_t, offset + count, buf->size);
+	fw_priv->size = max_t(size_t, offset + count, fw_priv->size);
 out:
 	mutex_unlock(&fw_lock);
 	return retval;
@@ -991,22 +1028,22 @@ static const struct attribute_group *fw_dev_attr_groups[] = {
 	NULL
 };
 
-static struct firmware_priv *
+static struct fw_sysfs *
 fw_create_instance(struct firmware *firmware, const char *fw_name,
 		   struct device *device, unsigned int opt_flags)
 {
-	struct firmware_priv *fw_priv;
+	struct fw_sysfs *fw_sysfs;
 	struct device *f_dev;
 
-	fw_priv = kzalloc(sizeof(*fw_priv), GFP_KERNEL);
-	if (!fw_priv) {
-		fw_priv = ERR_PTR(-ENOMEM);
+	fw_sysfs = kzalloc(sizeof(*fw_sysfs), GFP_KERNEL);
+	if (!fw_sysfs) {
+		fw_sysfs = ERR_PTR(-ENOMEM);
 		goto exit;
 	}
 
-	fw_priv->nowait = !!(opt_flags & FW_OPT_NOWAIT);
-	fw_priv->fw = firmware;
-	f_dev = &fw_priv->dev;
+	fw_sysfs->nowait = !!(opt_flags & FW_OPT_NOWAIT);
+	fw_sysfs->fw = firmware;
+	f_dev = &fw_sysfs->dev;
 
 	device_initialize(f_dev);
 	dev_set_name(f_dev, "%s", fw_name);
@@ -1014,20 +1051,20 @@ fw_create_instance(struct firmware *firmware, const char *fw_name,
 	f_dev->class = &firmware_class;
 	f_dev->groups = fw_dev_attr_groups;
 exit:
-	return fw_priv;
+	return fw_sysfs;
 }
 
 /* load a firmware via user helper */
-static int _request_firmware_load(struct firmware_priv *fw_priv,
+static int _request_firmware_load(struct fw_sysfs *fw_sysfs,
 				  unsigned int opt_flags, long timeout)
 {
 	int retval = 0;
-	struct device *f_dev = &fw_priv->dev;
-	struct firmware_buf *buf = fw_priv->buf;
+	struct device *f_dev = &fw_sysfs->dev;
+	struct fw_priv *fw_priv = fw_sysfs->fw_priv;
 
 	/* fall back on userspace loading */
-	if (!buf->data)
-		buf->is_paged_buf = true;
+	if (!fw_priv->data)
+		fw_priv->is_paged_buf = true;
 
 	dev_set_uevent_suppress(f_dev, true);
 
@@ -1038,31 +1075,31 @@ static int _request_firmware_load(struct firmware_priv *fw_priv,
 	}
 
 	mutex_lock(&fw_lock);
-	list_add(&buf->pending_list, &pending_fw_head);
+	list_add(&fw_priv->pending_list, &pending_fw_head);
 	mutex_unlock(&fw_lock);
 
 	if (opt_flags & FW_OPT_UEVENT) {
-		buf->need_uevent = true;
+		fw_priv->need_uevent = true;
 		dev_set_uevent_suppress(f_dev, false);
-		dev_dbg(f_dev, "firmware: requesting %s\n", buf->fw_id);
-		kobject_uevent(&fw_priv->dev.kobj, KOBJ_ADD);
+		dev_dbg(f_dev, "firmware: requesting %s\n", fw_priv->fw_name);
+		kobject_uevent(&fw_sysfs->dev.kobj, KOBJ_ADD);
 	} else {
 		timeout = MAX_JIFFY_OFFSET;
 	}
 
-	retval = fw_state_wait_timeout(&buf->fw_st, timeout);
+	retval = fw_sysfs_wait_timeout(fw_priv, timeout);
 	if (retval < 0) {
 		mutex_lock(&fw_lock);
-		fw_load_abort(fw_priv);
+		fw_load_abort(fw_sysfs);
 		mutex_unlock(&fw_lock);
 	}
 
-	if (fw_state_is_aborted(&buf->fw_st)) {
+	if (fw_state_is_aborted(fw_priv)) {
 		if (retval == -ERESTARTSYS)
 			retval = -EINTR;
 		else
 			retval = -EAGAIN;
-	} else if (buf->is_paged_buf && !buf->data)
+	} else if (fw_priv->is_paged_buf && !fw_priv->data)
 		retval = -ENOMEM;
 
 	device_del(f_dev);
@@ -1075,7 +1112,7 @@ static int fw_load_from_user_helper(struct firmware *firmware,
 				    const char *name, struct device *device,
 				    unsigned int opt_flags)
 {
-	struct firmware_priv *fw_priv;
+	struct fw_sysfs *fw_sysfs;
 	long timeout;
 	int ret;
 
@@ -1096,17 +1133,17 @@ static int fw_load_from_user_helper(struct firmware *firmware,
 		}
 	}
 
-	fw_priv = fw_create_instance(firmware, name, device, opt_flags);
-	if (IS_ERR(fw_priv)) {
-		ret = PTR_ERR(fw_priv);
+	fw_sysfs = fw_create_instance(firmware, name, device, opt_flags);
+	if (IS_ERR(fw_sysfs)) {
+		ret = PTR_ERR(fw_sysfs);
 		goto out_unlock;
 	}
 
-	fw_priv->buf = firmware->priv;
-	ret = _request_firmware_load(fw_priv, opt_flags, timeout);
+	fw_sysfs->fw_priv = firmware->priv;
+	ret = _request_firmware_load(fw_sysfs, opt_flags, timeout);
 
 	if (!ret)
-		ret = assign_firmware_buf(firmware, device, opt_flags);
+		ret = assign_fw(firmware, device, opt_flags);
 
 out_unlock:
 	usermodehelper_read_unlock();
@@ -1114,16 +1151,60 @@ static int fw_load_from_user_helper(struct firmware *firmware,
 	return ret;
 }
 
+#ifdef CONFIG_FW_LOADER_USER_HELPER_FALLBACK
+static bool fw_force_sysfs_fallback(unsigned int opt_flags)
+{
+	return true;
+}
+#else
+static bool fw_force_sysfs_fallback(unsigned int opt_flags)
+{
+	if (!(opt_flags & FW_OPT_USERHELPER))
+		return false;
+	return true;
+}
+#endif
+
+static bool fw_run_sysfs_fallback(unsigned int opt_flags)
+{
+	if ((opt_flags & FW_OPT_NOFALLBACK))
+		return false;
+
+	return fw_force_sysfs_fallback(opt_flags);
+}
+
+static int fw_sysfs_fallback(struct firmware *fw, const char *name,
+			    struct device *device,
+			    unsigned int opt_flags,
+			    int ret)
+{
+	if (!fw_run_sysfs_fallback(opt_flags))
+		return ret;
+
+	dev_warn(device, "Falling back to user helper\n");
+	return fw_load_from_user_helper(fw, name, device, opt_flags);
+}
 #else /* CONFIG_FW_LOADER_USER_HELPER */
-static inline int
-fw_load_from_user_helper(struct firmware *firmware, const char *name,
-			 struct device *device, unsigned int opt_flags)
+static int fw_sysfs_fallback(struct firmware *fw, const char *name,
+			     struct device *device,
+			     unsigned int opt_flags,
+			     int ret)
 {
-	return -ENOENT;
+	/* Keep carrying over the same error */
+	return ret;
 }
 
 static inline void kill_pending_fw_fallback_reqs(bool only_kill_custom) { }
 
+static inline int register_sysfs_loader(void)
+{
+	return 0;
+}
+
+static inline void unregister_sysfs_loader(void)
+{
+}
+
 #endif /* CONFIG_FW_LOADER_USER_HELPER */
 
 /* prepare firmware and firmware_buf structs;
@@ -1135,7 +1216,7 @@ _request_firmware_prepare(struct firmware **firmware_p, const char *name,
 			  struct device *device, void *dbuf, size_t size)
 {
 	struct firmware *firmware;
-	struct firmware_buf *buf;
+	struct fw_priv *fw_priv;
 	int ret;
 
 	*firmware_p = firmware = kzalloc(sizeof(*firmware), GFP_KERNEL);
@@ -1150,18 +1231,18 @@ _request_firmware_prepare(struct firmware **firmware_p, const char *name,
 		return 0; /* assigned */
 	}
 
-	ret = fw_lookup_and_allocate_buf(name, &fw_cache, &buf, dbuf, size);
+	ret = alloc_lookup_fw_priv(name, &fw_cache, &fw_priv, dbuf, size);
 
 	/*
-	 * bind with 'buf' now to avoid warning in failure path
+	 * bind with 'priv' now to avoid warning in failure path
 	 * of requesting firmware.
 	 */
-	firmware->priv = buf;
+	firmware->priv = fw_priv;
 
 	if (ret > 0) {
-		ret = fw_state_wait(&buf->fw_st);
+		ret = fw_state_wait(fw_priv);
 		if (!ret) {
-			fw_set_page_data(buf, firmware);
+			fw_set_page_data(fw_priv, firmware);
 			return 0; /* assigned */
 		}
 	}
@@ -1177,20 +1258,20 @@ _request_firmware_prepare(struct firmware **firmware_p, const char *name,
  * released until the last user calls release_firmware().
  *
  * Failed batched requests are possible as well, in such cases we just share
- * the struct firmware_buf and won't release it until all requests are woken
+ * the struct fw_priv and won't release it until all requests are woken
  * and have gone through this same path.
  */
 static void fw_abort_batch_reqs(struct firmware *fw)
 {
-	struct firmware_buf *buf;
+	struct fw_priv *fw_priv;
 
 	/* Loaded directly? */
 	if (!fw || !fw->priv)
 		return;
 
-	buf = fw->priv;
-	if (!fw_state_is_aborted(&buf->fw_st))
-		fw_state_aborted(&buf->fw_st);
+	fw_priv = fw->priv;
+	if (!fw_state_is_aborted(fw_priv))
+		fw_state_aborted(fw_priv);
 }
 
 /* called from request_firmware() and request_firmware_work_func() */
@@ -1220,13 +1301,9 @@ _request_firmware(const struct firmware **firmware_p, const char *name,
 			dev_warn(device,
 				 "Direct firmware load for %s failed with error %d\n",
 				 name, ret);
-		if (opt_flags & FW_OPT_USERHELPER) {
-			dev_warn(device, "Falling back to user helper\n");
-			ret = fw_load_from_user_helper(fw, name, device,
-						       opt_flags);
-		}
+		ret = fw_sysfs_fallback(fw, name, device, opt_flags, ret);
 	} else
-		ret = assign_firmware_buf(fw, device, opt_flags);
+		ret = assign_fw(fw, device, opt_flags);
 
  out:
 	if (ret < 0) {
@@ -1268,7 +1345,7 @@ request_firmware(const struct firmware **firmware_p, const char *name,
 	/* Need to pin this module until return */
 	__module_get(THIS_MODULE);
 	ret = _request_firmware(firmware_p, name, device, NULL, 0,
-				FW_OPT_UEVENT | FW_OPT_FALLBACK);
+				FW_OPT_UEVENT);
 	module_put(THIS_MODULE);
 	return ret;
 }
@@ -1292,7 +1369,8 @@ int request_firmware_direct(const struct firmware **firmware_p,
 
 	__module_get(THIS_MODULE);
 	ret = _request_firmware(firmware_p, name, device, NULL, 0,
-				FW_OPT_UEVENT | FW_OPT_NO_WARN);
+				FW_OPT_UEVENT | FW_OPT_NO_WARN |
+				FW_OPT_NOFALLBACK);
 	module_put(THIS_MODULE);
 	return ret;
 }
@@ -1321,8 +1399,7 @@ request_firmware_into_buf(const struct firmware **firmware_p, const char *name,
 
 	__module_get(THIS_MODULE);
 	ret = _request_firmware(firmware_p, name, device, buf, size,
-				FW_OPT_UEVENT | FW_OPT_FALLBACK |
-				FW_OPT_NOCACHE);
+				FW_OPT_UEVENT | FW_OPT_NOCACHE);
 	module_put(THIS_MODULE);
 	return ret;
 }
@@ -1414,7 +1491,7 @@ request_firmware_nowait(
 	fw_work->device = device;
 	fw_work->context = context;
 	fw_work->cont = cont;
-	fw_work->opt_flags = FW_OPT_NOWAIT | FW_OPT_FALLBACK |
+	fw_work->opt_flags = FW_OPT_NOWAIT |
 		(uevent ? FW_OPT_UEVENT : FW_OPT_USERHELPER);
 
 	if (!try_module_get(module)) {
@@ -1463,13 +1540,13 @@ static int cache_firmware(const char *fw_name)
 	return ret;
 }
 
-static struct firmware_buf *fw_lookup_buf(const char *fw_name)
+static struct fw_priv *lookup_fw_priv(const char *fw_name)
 {
-	struct firmware_buf *tmp;
+	struct fw_priv *tmp;
 	struct firmware_cache *fwc = &fw_cache;
 
 	spin_lock(&fwc->lock);
-	tmp = __fw_lookup_buf(fw_name);
+	tmp = __lookup_fw_priv(fw_name);
 	spin_unlock(&fwc->lock);
 
 	return tmp;
@@ -1488,7 +1565,7 @@ static struct firmware_buf *fw_lookup_buf(const char *fw_name)
  */
 static int uncache_firmware(const char *fw_name)
 {
-	struct firmware_buf *buf;
+	struct fw_priv *fw_priv;
 	struct firmware fw;
 
 	pr_debug("%s: %s\n", __func__, fw_name);
@@ -1496,9 +1573,9 @@ static int uncache_firmware(const char *fw_name)
 	if (fw_get_builtin_firmware(&fw, fw_name, NULL, 0))
 		return 0;
 
-	buf = fw_lookup_buf(fw_name);
-	if (buf) {
-		fw_free_buf(buf);
+	fw_priv = lookup_fw_priv(fw_name);
+	if (fw_priv) {
+		free_fw_priv(fw_priv);
 		return 0;
 	}
 
@@ -1767,20 +1844,11 @@ static int fw_suspend(void)
 static struct syscore_ops fw_syscore_ops = {
 	.suspend = fw_suspend,
 };
-#else
-static int fw_cache_piggyback_on_request(const char *name)
-{
-	return 0;
-}
-#endif
 
-static void __init fw_cache_init(void)
+static int __init register_fw_pm_ops(void)
 {
-	spin_lock_init(&fw_cache.lock);
-	INIT_LIST_HEAD(&fw_cache.head);
-	fw_cache.state = FW_LOADER_NO_CACHE;
+	int ret;
 
-#ifdef CONFIG_PM_SLEEP
 	spin_lock_init(&fw_cache.name_lock);
 	INIT_LIST_HEAD(&fw_cache.fw_names);
 
@@ -1788,10 +1856,39 @@ static void __init fw_cache_init(void)
 			  device_uncache_fw_images_work);
 
 	fw_cache.pm_notify.notifier_call = fw_pm_notify;
-	register_pm_notifier(&fw_cache.pm_notify);
+	ret = register_pm_notifier(&fw_cache.pm_notify);
+	if (ret)
+		return ret;
 
 	register_syscore_ops(&fw_syscore_ops);
+
+	return ret;
+}
+
+static inline void unregister_fw_pm_ops(void)
+{
+	unregister_syscore_ops(&fw_syscore_ops);
+	unregister_pm_notifier(&fw_cache.pm_notify);
+}
+#else
+static int fw_cache_piggyback_on_request(const char *name)
+{
+	return 0;
+}
+static inline int register_fw_pm_ops(void)
+{
+	return 0;
+}
+static inline void unregister_fw_pm_ops(void)
+{
+}
 #endif
+
+static void __init fw_cache_init(void)
+{
+	spin_lock_init(&fw_cache.lock);
+	INIT_LIST_HEAD(&fw_cache.head);
+	fw_cache.state = FW_LOADER_NO_CACHE;
 }
 
 static int fw_shutdown_notify(struct notifier_block *unused1,
@@ -1812,25 +1909,31 @@ static struct notifier_block fw_shutdown_nb = {
 
 static int __init firmware_class_init(void)
 {
+	int ret;
+
+	/* No need to unfold these on exit */
 	fw_cache_init();
-	register_reboot_notifier(&fw_shutdown_nb);
-#ifdef CONFIG_FW_LOADER_USER_HELPER
-	return class_register(&firmware_class);
-#else
-	return 0;
-#endif
+
+	ret = register_fw_pm_ops();
+	if (ret)
+		return ret;
+
+	ret = register_reboot_notifier(&fw_shutdown_nb);
+	if (ret)
+		goto out;
+
+	return register_sysfs_loader();
+
+out:
+	unregister_fw_pm_ops();
+	return ret;
 }
 
 static void __exit firmware_class_exit(void)
 {
-#ifdef CONFIG_PM_SLEEP
-	unregister_syscore_ops(&fw_syscore_ops);
-	unregister_pm_notifier(&fw_cache.pm_notify);
-#endif
+	unregister_fw_pm_ops();
 	unregister_reboot_notifier(&fw_shutdown_nb);
-#ifdef CONFIG_FW_LOADER_USER_HELPER
-	class_unregister(&firmware_class);
-#endif
+	unregister_sysfs_loader();
 }
 
 fs_initcall(firmware_class_init);
diff --git a/drivers/base/hypervisor.c b/drivers/base/hypervisor.c
index 4f8b741f4615..1ce59b4b53ce 100644
--- a/drivers/base/hypervisor.c
+++ b/drivers/base/hypervisor.c
@@ -1,11 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * hypervisor.c - /sys/hypervisor subsystem.
  *
  * Copyright (C) IBM Corp. 2006
  * Copyright (C) 2007 Greg Kroah-Hartman <gregkh@suse.de>
  * Copyright (C) 2007 Novell Inc.
- *
- * This file is released under the GPLv2
  */
 
 #include <linux/kobject.h>
diff --git a/drivers/base/init.c b/drivers/base/init.c
index 48c0e220acc0..dd85b05a6a16 100644
--- a/drivers/base/init.c
+++ b/drivers/base/init.c
@@ -1,8 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2002-3 Patrick Mochel
  * Copyright (c) 2002-3 Open Source Development Labs
- *
- * This file is released under the GPLv2
  */
 
 #include <linux/device.h>
diff --git a/drivers/base/isa.c b/drivers/base/isa.c
index 372d10af2600..2772f5d1948a 100644
--- a/drivers/base/isa.c
+++ b/drivers/base/isa.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * ISA bus.
  */
diff --git a/drivers/base/map.c b/drivers/base/map.c
index c1d38234d725..5650ab2b247a 100644
--- a/drivers/base/map.c
+++ b/drivers/base/map.c
@@ -1,8 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *  linux/drivers/base/map.c
  *
  * (C) Copyright Al Viro 2002,2003
- *	Released under GPL v2.
  *
  * NOTE: data structure needs to be changed.  It works, but for large dev_t
  * it will be too slow.  It is isolated, though, so these changes will be
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index 1d60b58a8c19..fe4b24f05f6a 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -569,7 +569,7 @@ store_hard_offline_page(struct device *dev,
 	if (kstrtoull(buf, 0, &pfn) < 0)
 		return -EINVAL;
 	pfn >>= PAGE_SHIFT;
-	ret = memory_failure(pfn, 0, 0);
+	ret = memory_failure(pfn, 0);
 	return ret ? ret : count;
 }
 
diff --git a/drivers/base/module.c b/drivers/base/module.c
index 2a215780eda2..46ad4d636731 100644
--- a/drivers/base/module.c
+++ b/drivers/base/module.c
@@ -1,8 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * module.c - module sysfs fun for drivers
- *
- * This file is released under the GPLv2
- *
  */
 #include <linux/device.h>
 #include <linux/module.h>
diff --git a/drivers/base/pinctrl.c b/drivers/base/pinctrl.c
index eb929dd6ef1e..c22864458511 100644
--- a/drivers/base/pinctrl.c
+++ b/drivers/base/pinctrl.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Driver core interface to the pinctrl subsystem.
  *
@@ -6,8 +7,6 @@
  * Based on bits of regulator core, gpio core and clk core
  *
  * Author: Linus Walleij <linus.walleij@linaro.org>
- *
- * License terms: GNU General Public License (GPL) version 2
  */
 
 #include <linux/device.h>
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index e5473525e7b2..8e22073aeeed 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * MSI framework for platform devices
  *
  * Copyright (C) 2015 ARM Limited, All Rights Reserved.
  * Author: Marc Zyngier <marc.zyngier@arm.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/device.h>
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index c203fb90c1a0..f1bf7b38d91c 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -1,11 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * platform.c - platform 'pseudo' bus for legacy devices
  *
  * Copyright (c) 2002-3 Patrick Mochel
  * Copyright (c) 2002-3 Open Source Development Labs
  *
- * This file is released under the GPLv2
- *
  * Please see Documentation/driver-model/platform.txt for more
  * information.
  */
diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index 0c80bea05bcb..1ea0e2502e8e 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -1032,15 +1032,12 @@ static int genpd_prepare(struct device *dev)
 static int genpd_finish_suspend(struct device *dev, bool poweroff)
 {
 	struct generic_pm_domain *genpd;
-	int ret;
+	int ret = 0;
 
 	genpd = dev_to_genpd(dev);
 	if (IS_ERR(genpd))
 		return -EINVAL;
 
-	if (dev->power.wakeup_path && genpd_is_active_wakeup(genpd))
-		return 0;
-
 	if (poweroff)
 		ret = pm_generic_poweroff_noirq(dev);
 	else
@@ -1048,10 +1045,19 @@ static int genpd_finish_suspend(struct device *dev, bool poweroff)
 	if (ret)
 		return ret;
 
-	if (genpd->dev_ops.stop && genpd->dev_ops.start) {
-		ret = pm_runtime_force_suspend(dev);
-		if (ret)
+	if (dev->power.wakeup_path && genpd_is_active_wakeup(genpd))
+		return 0;
+
+	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
+	    !pm_runtime_status_suspended(dev)) {
+		ret = genpd_stop_dev(genpd, dev);
+		if (ret) {
+			if (poweroff)
+				pm_generic_restore_noirq(dev);
+			else
+				pm_generic_resume_noirq(dev);
 			return ret;
+		}
 	}
 
 	genpd_lock(genpd);
@@ -1085,7 +1091,7 @@ static int genpd_suspend_noirq(struct device *dev)
 static int genpd_resume_noirq(struct device *dev)
 {
 	struct generic_pm_domain *genpd;
-	int ret = 0;
+	int ret;
 
 	dev_dbg(dev, "%s()\n", __func__);
 
@@ -1094,21 +1100,21 @@ static int genpd_resume_noirq(struct device *dev)
 		return -EINVAL;
 
 	if (dev->power.wakeup_path && genpd_is_active_wakeup(genpd))
-		return 0;
+		return pm_generic_resume_noirq(dev);
 
 	genpd_lock(genpd);
 	genpd_sync_power_on(genpd, true, 0);
 	genpd->suspended_count--;
 	genpd_unlock(genpd);
 
-	if (genpd->dev_ops.stop && genpd->dev_ops.start)
-		ret = pm_runtime_force_resume(dev);
-
-	ret = pm_generic_resume_noirq(dev);
-	if (ret)
-		return ret;
+	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
+	    !pm_runtime_status_suspended(dev)) {
+		ret = genpd_start_dev(genpd, dev);
+		if (ret)
+			return ret;
+	}
 
-	return ret;
+	return pm_generic_resume_noirq(dev);
 }
 
 /**
@@ -1135,8 +1141,9 @@ static int genpd_freeze_noirq(struct device *dev)
 	if (ret)
 		return ret;
 
-	if (genpd->dev_ops.stop && genpd->dev_ops.start)
-		ret = pm_runtime_force_suspend(dev);
+	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
+	    !pm_runtime_status_suspended(dev))
+		ret = genpd_stop_dev(genpd, dev);
 
 	return ret;
 }
@@ -1159,8 +1166,9 @@ static int genpd_thaw_noirq(struct device *dev)
 	if (IS_ERR(genpd))
 		return -EINVAL;
 
-	if (genpd->dev_ops.stop && genpd->dev_ops.start) {
-		ret = pm_runtime_force_resume(dev);
+	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
+	    !pm_runtime_status_suspended(dev)) {
+		ret = genpd_start_dev(genpd, dev);
 		if (ret)
 			return ret;
 	}
@@ -1217,8 +1225,9 @@ static int genpd_restore_noirq(struct device *dev)
 	genpd_sync_power_on(genpd, true, 0);
 	genpd_unlock(genpd);
 
-	if (genpd->dev_ops.stop && genpd->dev_ops.start) {
-		ret = pm_runtime_force_resume(dev);
+	if (genpd->dev_ops.stop && genpd->dev_ops.start &&
+	    !pm_runtime_status_suspended(dev)) {
+		ret = genpd_start_dev(genpd, dev);
 		if (ret)
 			return ret;
 	}
@@ -2199,20 +2208,8 @@ int genpd_dev_pm_attach(struct device *dev)
 
 	ret = of_parse_phandle_with_args(dev->of_node, "power-domains",
 					"#power-domain-cells", 0, &pd_args);
-	if (ret < 0) {
-		if (ret != -ENOENT)
-			return ret;
-
-		/*
-		 * Try legacy Samsung-specific bindings
-		 * (for backwards compatibility of DT ABI)
-		 */
-		pd_args.args_count = 0;
-		pd_args.np = of_parse_phandle(dev->of_node,
-						"samsung,power-domain", 0);
-		if (!pd_args.np)
-			return -ENOENT;
-	}
+	if (ret < 0)
+		return ret;
 
 	mutex_lock(&gpd_list_lock);
 	pd = genpd_get_from_provider(&pd_args);
@@ -2293,6 +2290,38 @@ static int genpd_parse_state(struct genpd_power_state *genpd_state,
 	return 0;
 }
 
+static int genpd_iterate_idle_states(struct device_node *dn,
+				     struct genpd_power_state *states)
+{
+	int ret;
+	struct of_phandle_iterator it;
+	struct device_node *np;
+	int i = 0;
+
+	ret = of_count_phandle_with_args(dn, "domain-idle-states", NULL);
+	if (ret <= 0)
+		return ret;
+
+	/* Loop over the phandles until all the requested entry is found */
+	of_for_each_phandle(&it, ret, dn, "domain-idle-states", NULL, 0) {
+		np = it.node;
+		if (!of_match_node(idle_state_match, np))
+			continue;
+		if (states) {
+			ret = genpd_parse_state(&states[i], np);
+			if (ret) {
+				pr_err("Parsing idle state node %pOF failed with err %d\n",
+				       np, ret);
+				of_node_put(np);
+				return ret;
+			}
+		}
+		i++;
+	}
+
+	return i;
+}
+
 /**
  * of_genpd_parse_idle_states: Return array of idle states for the genpd.
  *
@@ -2302,49 +2331,31 @@ static int genpd_parse_state(struct genpd_power_state *genpd_state,
  *
  * Returns the device states parsed from the OF node. The memory for the states
  * is allocated by this function and is the responsibility of the caller to
- * free the memory after use.
+ * free the memory after use. If no domain idle states is found it returns
+ * -EINVAL and in case of errors, a negative error code.
  */
 int of_genpd_parse_idle_states(struct device_node *dn,
 			struct genpd_power_state **states, int *n)
 {
 	struct genpd_power_state *st;
-	struct device_node *np;
-	int i = 0;
-	int err, ret;
-	int count;
-	struct of_phandle_iterator it;
-	const struct of_device_id *match_id;
+	int ret;
 
-	count = of_count_phandle_with_args(dn, "domain-idle-states", NULL);
-	if (count <= 0)
-		return -EINVAL;
+	ret = genpd_iterate_idle_states(dn, NULL);
+	if (ret <= 0)
+		return ret < 0 ? ret : -EINVAL;
 
-	st = kcalloc(count, sizeof(*st), GFP_KERNEL);
+	st = kcalloc(ret, sizeof(*st), GFP_KERNEL);
 	if (!st)
 		return -ENOMEM;
 
-	/* Loop over the phandles until all the requested entry is found */
-	of_for_each_phandle(&it, err, dn, "domain-idle-states", NULL, 0) {
-		np = it.node;
-		match_id = of_match_node(idle_state_match, np);
-		if (!match_id)
-			continue;
-		ret = genpd_parse_state(&st[i++], np);
-		if (ret) {
-			pr_err
-			("Parsing idle state node %pOF failed with err %d\n",
-							np, ret);
-			of_node_put(np);
-			kfree(st);
-			return ret;
-		}
+	ret = genpd_iterate_idle_states(dn, st);
+	if (ret <= 0) {
+		kfree(st);
+		return ret < 0 ? ret : -EINVAL;
 	}
 
-	*n = i;
-	if (!i)
-		kfree(st);
-	else
-		*states = st;
+	*states = st;
+	*n = ret;
 
 	return 0;
 }
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index 08744b572af6..02a497e7c785 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -18,7 +18,6 @@
  */
 
 #include <linux/device.h>
-#include <linux/kallsyms.h>
 #include <linux/export.h>
 #include <linux/mutex.h>
 #include <linux/pm.h>
@@ -540,6 +539,73 @@ void dev_pm_skip_next_resume_phases(struct device *dev)
 	dev->power.is_suspended = false;
 }
 
+/**
+ * suspend_event - Return a "suspend" message for given "resume" one.
+ * @resume_msg: PM message representing a system-wide resume transition.
+ */
+static pm_message_t suspend_event(pm_message_t resume_msg)
+{
+	switch (resume_msg.event) {
+	case PM_EVENT_RESUME:
+		return PMSG_SUSPEND;
+	case PM_EVENT_THAW:
+	case PM_EVENT_RESTORE:
+		return PMSG_FREEZE;
+	case PM_EVENT_RECOVER:
+		return PMSG_HIBERNATE;
+	}
+	return PMSG_ON;
+}
+
+/**
+ * dev_pm_may_skip_resume - System-wide device resume optimization check.
+ * @dev: Target device.
+ *
+ * Checks whether or not the device may be left in suspend after a system-wide
+ * transition to the working state.
+ */
+bool dev_pm_may_skip_resume(struct device *dev)
+{
+	return !dev->power.must_resume && pm_transition.event != PM_EVENT_RESTORE;
+}
+
+static pm_callback_t dpm_subsys_resume_noirq_cb(struct device *dev,
+						pm_message_t state,
+						const char **info_p)
+{
+	pm_callback_t callback;
+	const char *info;
+
+	if (dev->pm_domain) {
+		info = "noirq power domain ";
+		callback = pm_noirq_op(&dev->pm_domain->ops, state);
+	} else if (dev->type && dev->type->pm) {
+		info = "noirq type ";
+		callback = pm_noirq_op(dev->type->pm, state);
+	} else if (dev->class && dev->class->pm) {
+		info = "noirq class ";
+		callback = pm_noirq_op(dev->class->pm, state);
+	} else if (dev->bus && dev->bus->pm) {
+		info = "noirq bus ";
+		callback = pm_noirq_op(dev->bus->pm, state);
+	} else {
+		return NULL;
+	}
+
+	if (info_p)
+		*info_p = info;
+
+	return callback;
+}
+
+static pm_callback_t dpm_subsys_suspend_noirq_cb(struct device *dev,
+						 pm_message_t state,
+						 const char **info_p);
+
+static pm_callback_t dpm_subsys_suspend_late_cb(struct device *dev,
+						pm_message_t state,
+						const char **info_p);
+
 /**
  * device_resume_noirq - Execute a "noirq resume" callback for given device.
  * @dev: Device to handle.
@@ -551,8 +617,9 @@ void dev_pm_skip_next_resume_phases(struct device *dev)
  */
 static int device_resume_noirq(struct device *dev, pm_message_t state, bool async)
 {
-	pm_callback_t callback = NULL;
-	const char *info = NULL;
+	pm_callback_t callback;
+	const char *info;
+	bool skip_resume;
 	int error = 0;
 
 	TRACE_DEVICE(dev);
@@ -566,29 +633,61 @@ static int device_resume_noirq(struct device *dev, pm_message_t state, bool asyn
 
 	dpm_wait_for_superior(dev, async);
 
-	if (dev->pm_domain) {
-		info = "noirq power domain ";
-		callback = pm_noirq_op(&dev->pm_domain->ops, state);
-	} else if (dev->type && dev->type->pm) {
-		info = "noirq type ";
-		callback = pm_noirq_op(dev->type->pm, state);
-	} else if (dev->class && dev->class->pm) {
-		info = "noirq class ";
-		callback = pm_noirq_op(dev->class->pm, state);
-	} else if (dev->bus && dev->bus->pm) {
-		info = "noirq bus ";
-		callback = pm_noirq_op(dev->bus->pm, state);
+	skip_resume = dev_pm_may_skip_resume(dev);
+
+	callback = dpm_subsys_resume_noirq_cb(dev, state, &info);
+	if (callback)
+		goto Run;
+
+	if (skip_resume)
+		goto Skip;
+
+	if (dev_pm_smart_suspend_and_suspended(dev)) {
+		pm_message_t suspend_msg = suspend_event(state);
+
+		/*
+		 * If "freeze" callbacks have been skipped during a transition
+		 * related to hibernation, the subsequent "thaw" callbacks must
+		 * be skipped too or bad things may happen.  Otherwise, resume
+		 * callbacks are going to be run for the device, so its runtime
+		 * PM status must be changed to reflect the new state after the
+		 * transition under way.
+		 */
+		if (!dpm_subsys_suspend_late_cb(dev, suspend_msg, NULL) &&
+		    !dpm_subsys_suspend_noirq_cb(dev, suspend_msg, NULL)) {
+			if (state.event == PM_EVENT_THAW) {
+				skip_resume = true;
+				goto Skip;
+			} else {
+				pm_runtime_set_active(dev);
+			}
+		}
 	}
 
-	if (!callback && dev->driver && dev->driver->pm) {
+	if (dev->driver && dev->driver->pm) {
 		info = "noirq driver ";
 		callback = pm_noirq_op(dev->driver->pm, state);
 	}
 
+Run:
 	error = dpm_run_callback(callback, dev, state, info);
+
+Skip:
 	dev->power.is_noirq_suspended = false;
 
- Out:
+	if (skip_resume) {
+		/*
+		 * The device is going to be left in suspend, but it might not
+		 * have been in runtime suspend before the system suspended, so
+		 * its runtime PM status needs to be updated to avoid confusing
+		 * the runtime PM framework when runtime PM is enabled for the
+		 * device again.
+		 */
+		pm_runtime_set_suspended(dev);
+		dev_pm_skip_next_resume_phases(dev);
+	}
+
+Out:
 	complete_all(&dev->power.completion);
 	TRACE_RESUME(error);
 	return error;
@@ -681,6 +780,35 @@ void dpm_resume_noirq(pm_message_t state)
 	dpm_noirq_end();
 }
 
+static pm_callback_t dpm_subsys_resume_early_cb(struct device *dev,
+						pm_message_t state,
+						const char **info_p)
+{
+	pm_callback_t callback;
+	const char *info;
+
+	if (dev->pm_domain) {
+		info = "early power domain ";
+		callback = pm_late_early_op(&dev->pm_domain->ops, state);
+	} else if (dev->type && dev->type->pm) {
+		info = "early type ";
+		callback = pm_late_early_op(dev->type->pm, state);
+	} else if (dev->class && dev->class->pm) {
+		info = "early class ";
+		callback = pm_late_early_op(dev->class->pm, state);
+	} else if (dev->bus && dev->bus->pm) {
+		info = "early bus ";
+		callback = pm_late_early_op(dev->bus->pm, state);
+	} else {
+		return NULL;
+	}
+
+	if (info_p)
+		*info_p = info;
+
+	return callback;
+}
+
 /**
  * device_resume_early - Execute an "early resume" callback for given device.
  * @dev: Device to handle.
@@ -691,8 +819,8 @@ void dpm_resume_noirq(pm_message_t state)
  */
 static int device_resume_early(struct device *dev, pm_message_t state, bool async)
 {
-	pm_callback_t callback = NULL;
-	const char *info = NULL;
+	pm_callback_t callback;
+	const char *info;
 	int error = 0;
 
 	TRACE_DEVICE(dev);
@@ -706,19 +834,7 @@ static int device_resume_early(struct device *dev, pm_message_t state, bool asyn
 
 	dpm_wait_for_superior(dev, async);
 
-	if (dev->pm_domain) {
-		info = "early power domain ";
-		callback = pm_late_early_op(&dev->pm_domain->ops, state);
-	} else if (dev->type && dev->type->pm) {
-		info = "early type ";
-		callback = pm_late_early_op(dev->type->pm, state);
-	} else if (dev->class && dev->class->pm) {
-		info = "early class ";
-		callback = pm_late_early_op(dev->class->pm, state);
-	} else if (dev->bus && dev->bus->pm) {
-		info = "early bus ";
-		callback = pm_late_early_op(dev->bus->pm, state);
-	}
+	callback = dpm_subsys_resume_early_cb(dev, state, &info);
 
 	if (!callback && dev->driver && dev->driver->pm) {
 		info = "early driver ";
@@ -1089,6 +1205,77 @@ static pm_message_t resume_event(pm_message_t sleep_state)
 	return PMSG_ON;
 }
 
+static void dpm_superior_set_must_resume(struct device *dev)
+{
+	struct device_link *link;
+	int idx;
+
+	if (dev->parent)
+		dev->parent->power.must_resume = true;
+
+	idx = device_links_read_lock();
+
+	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node)
+		link->supplier->power.must_resume = true;
+
+	device_links_read_unlock(idx);
+}
+
+static pm_callback_t dpm_subsys_suspend_noirq_cb(struct device *dev,
+						 pm_message_t state,
+						 const char **info_p)
+{
+	pm_callback_t callback;
+	const char *info;
+
+	if (dev->pm_domain) {
+		info = "noirq power domain ";
+		callback = pm_noirq_op(&dev->pm_domain->ops, state);
+	} else if (dev->type && dev->type->pm) {
+		info = "noirq type ";
+		callback = pm_noirq_op(dev->type->pm, state);
+	} else if (dev->class && dev->class->pm) {
+		info = "noirq class ";
+		callback = pm_noirq_op(dev->class->pm, state);
+	} else if (dev->bus && dev->bus->pm) {
+		info = "noirq bus ";
+		callback = pm_noirq_op(dev->bus->pm, state);
+	} else {
+		return NULL;
+	}
+
+	if (info_p)
+		*info_p = info;
+
+	return callback;
+}
+
+static bool device_must_resume(struct device *dev, pm_message_t state,
+			       bool no_subsys_suspend_noirq)
+{
+	pm_message_t resume_msg = resume_event(state);
+
+	/*
+	 * If all of the device driver's "noirq", "late" and "early" callbacks
+	 * are invoked directly by the core, the decision to allow the device to
+	 * stay in suspend can be based on its current runtime PM status and its
+	 * wakeup settings.
+	 */
+	if (no_subsys_suspend_noirq &&
+	    !dpm_subsys_suspend_late_cb(dev, state, NULL) &&
+	    !dpm_subsys_resume_early_cb(dev, resume_msg, NULL) &&
+	    !dpm_subsys_resume_noirq_cb(dev, resume_msg, NULL))
+		return !pm_runtime_status_suspended(dev) &&
+			(resume_msg.event != PM_EVENT_RESUME ||
+			 (device_can_wakeup(dev) && !device_may_wakeup(dev)));
+
+	/*
+	 * The only safe strategy here is to require that if the device may not
+	 * be left in suspend, resume callbacks must be invoked for it.
+	 */
+	return !dev->power.may_skip_resume;
+}
+
 /**
  * __device_suspend_noirq - Execute a "noirq suspend" callback for given device.
  * @dev: Device to handle.
@@ -1100,8 +1287,9 @@ static pm_message_t resume_event(pm_message_t sleep_state)
  */
 static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool async)
 {
-	pm_callback_t callback = NULL;
-	const char *info = NULL;
+	pm_callback_t callback;
+	const char *info;
+	bool no_subsys_cb = false;
 	int error = 0;
 
 	TRACE_DEVICE(dev);
@@ -1120,30 +1308,40 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 	if (dev->power.syscore || dev->power.direct_complete)
 		goto Complete;
 
-	if (dev->pm_domain) {
-		info = "noirq power domain ";
-		callback = pm_noirq_op(&dev->pm_domain->ops, state);
-	} else if (dev->type && dev->type->pm) {
-		info = "noirq type ";
-		callback = pm_noirq_op(dev->type->pm, state);
-	} else if (dev->class && dev->class->pm) {
-		info = "noirq class ";
-		callback = pm_noirq_op(dev->class->pm, state);
-	} else if (dev->bus && dev->bus->pm) {
-		info = "noirq bus ";
-		callback = pm_noirq_op(dev->bus->pm, state);
-	}
+	callback = dpm_subsys_suspend_noirq_cb(dev, state, &info);
+	if (callback)
+		goto Run;
 
-	if (!callback && dev->driver && dev->driver->pm) {
+	no_subsys_cb = !dpm_subsys_suspend_late_cb(dev, state, NULL);
+
+	if (dev_pm_smart_suspend_and_suspended(dev) && no_subsys_cb)
+		goto Skip;
+
+	if (dev->driver && dev->driver->pm) {
 		info = "noirq driver ";
 		callback = pm_noirq_op(dev->driver->pm, state);
 	}
 
+Run:
 	error = dpm_run_callback(callback, dev, state, info);
-	if (!error)
-		dev->power.is_noirq_suspended = true;
-	else
+	if (error) {
 		async_error = error;
+		goto Complete;
+	}
+
+Skip:
+	dev->power.is_noirq_suspended = true;
+
+	if (dev_pm_test_driver_flags(dev, DPM_FLAG_LEAVE_SUSPENDED)) {
+		dev->power.must_resume = dev->power.must_resume ||
+				atomic_read(&dev->power.usage_count) > 1 ||
+				device_must_resume(dev, state, no_subsys_cb);
+	} else {
+		dev->power.must_resume = true;
+	}
+
+	if (dev->power.must_resume)
+		dpm_superior_set_must_resume(dev);
 
 Complete:
 	complete_all(&dev->power.completion);
@@ -1249,6 +1447,50 @@ int dpm_suspend_noirq(pm_message_t state)
 	return ret;
 }
 
+static void dpm_propagate_wakeup_to_parent(struct device *dev)
+{
+	struct device *parent = dev->parent;
+
+	if (!parent)
+		return;
+
+	spin_lock_irq(&parent->power.lock);
+
+	if (dev->power.wakeup_path && !parent->power.ignore_children)
+		parent->power.wakeup_path = true;
+
+	spin_unlock_irq(&parent->power.lock);
+}
+
+static pm_callback_t dpm_subsys_suspend_late_cb(struct device *dev,
+						pm_message_t state,
+						const char **info_p)
+{
+	pm_callback_t callback;
+	const char *info;
+
+	if (dev->pm_domain) {
+		info = "late power domain ";
+		callback = pm_late_early_op(&dev->pm_domain->ops, state);
+	} else if (dev->type && dev->type->pm) {
+		info = "late type ";
+		callback = pm_late_early_op(dev->type->pm, state);
+	} else if (dev->class && dev->class->pm) {
+		info = "late class ";
+		callback = pm_late_early_op(dev->class->pm, state);
+	} else if (dev->bus && dev->bus->pm) {
+		info = "late bus ";
+		callback = pm_late_early_op(dev->bus->pm, state);
+	} else {
+		return NULL;
+	}
+
+	if (info_p)
+		*info_p = info;
+
+	return callback;
+}
+
 /**
  * __device_suspend_late - Execute a "late suspend" callback for given device.
  * @dev: Device to handle.
@@ -1259,8 +1501,8 @@ int dpm_suspend_noirq(pm_message_t state)
  */
 static int __device_suspend_late(struct device *dev, pm_message_t state, bool async)
 {
-	pm_callback_t callback = NULL;
-	const char *info = NULL;
+	pm_callback_t callback;
+	const char *info;
 	int error = 0;
 
 	TRACE_DEVICE(dev);
@@ -1281,30 +1523,29 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 	if (dev->power.syscore || dev->power.direct_complete)
 		goto Complete;
 
-	if (dev->pm_domain) {
-		info = "late power domain ";
-		callback = pm_late_early_op(&dev->pm_domain->ops, state);
-	} else if (dev->type && dev->type->pm) {
-		info = "late type ";
-		callback = pm_late_early_op(dev->type->pm, state);
-	} else if (dev->class && dev->class->pm) {
-		info = "late class ";
-		callback = pm_late_early_op(dev->class->pm, state);
-	} else if (dev->bus && dev->bus->pm) {
-		info = "late bus ";
-		callback = pm_late_early_op(dev->bus->pm, state);
-	}
+	callback = dpm_subsys_suspend_late_cb(dev, state, &info);
+	if (callback)
+		goto Run;
 
-	if (!callback && dev->driver && dev->driver->pm) {
+	if (dev_pm_smart_suspend_and_suspended(dev) &&
+	    !dpm_subsys_suspend_noirq_cb(dev, state, NULL))
+		goto Skip;
+
+	if (dev->driver && dev->driver->pm) {
 		info = "late driver ";
 		callback = pm_late_early_op(dev->driver->pm, state);
 	}
 
+Run:
 	error = dpm_run_callback(callback, dev, state, info);
-	if (!error)
-		dev->power.is_late_suspended = true;
-	else
+	if (error) {
 		async_error = error;
+		goto Complete;
+	}
+	dpm_propagate_wakeup_to_parent(dev);
+
+Skip:
+	dev->power.is_late_suspended = true;
 
 Complete:
 	TRACE_SUSPEND(error);
@@ -1435,11 +1676,17 @@ static int legacy_suspend(struct device *dev, pm_message_t state,
 	return error;
 }
 
-static void dpm_clear_suppliers_direct_complete(struct device *dev)
+static void dpm_clear_superiors_direct_complete(struct device *dev)
 {
 	struct device_link *link;
 	int idx;
 
+	if (dev->parent) {
+		spin_lock_irq(&dev->parent->power.lock);
+		dev->parent->power.direct_complete = false;
+		spin_unlock_irq(&dev->parent->power.lock);
+	}
+
 	idx = device_links_read_lock();
 
 	list_for_each_entry_rcu(link, &dev->links.suppliers, c_node) {
@@ -1500,6 +1747,9 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 		dev->power.direct_complete = false;
 	}
 
+	dev->power.may_skip_resume = false;
+	dev->power.must_resume = false;
+
 	dpm_watchdog_set(&wd, dev);
 	device_lock(dev);
 
@@ -1543,20 +1793,12 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 
  End:
 	if (!error) {
-		struct device *parent = dev->parent;
-
 		dev->power.is_suspended = true;
-		if (parent) {
-			spin_lock_irq(&parent->power.lock);
-
-			dev->parent->power.direct_complete = false;
-			if (dev->power.wakeup_path
-			    && !dev->parent->power.ignore_children)
-				dev->parent->power.wakeup_path = true;
+		if (device_may_wakeup(dev))
+			dev->power.wakeup_path = true;
 
-			spin_unlock_irq(&parent->power.lock);
-		}
-		dpm_clear_suppliers_direct_complete(dev);
+		dpm_propagate_wakeup_to_parent(dev);
+		dpm_clear_superiors_direct_complete(dev);
 	}
 
 	device_unlock(dev);
@@ -1665,8 +1907,9 @@ static int device_prepare(struct device *dev, pm_message_t state)
 	if (dev->power.syscore)
 		return 0;
 
-	WARN_ON(dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) &&
-		!pm_runtime_enabled(dev));
+	WARN_ON(!pm_runtime_enabled(dev) &&
+		dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND |
+					      DPM_FLAG_LEAVE_SUSPENDED));
 
 	/*
 	 * If a device's parent goes into runtime suspend at the wrong time,
@@ -1678,7 +1921,7 @@ static int device_prepare(struct device *dev, pm_message_t state)
 
 	device_lock(dev);
 
-	dev->power.wakeup_path = device_may_wakeup(dev);
+	dev->power.wakeup_path = false;
 
 	if (dev->power.no_pm_callbacks) {
 		ret = 1;	/* Let device go direct_complete */
diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h
index 7beee75399d4..21244c53e377 100644
--- a/drivers/base/power/power.h
+++ b/drivers/base/power/power.h
@@ -41,20 +41,15 @@ extern void dev_pm_disable_wake_irq_check(struct device *dev);
 
 #ifdef CONFIG_PM_SLEEP
 
-extern int device_wakeup_attach_irq(struct device *dev,
-				    struct wake_irq *wakeirq);
+extern void device_wakeup_attach_irq(struct device *dev, struct wake_irq *wakeirq);
 extern void device_wakeup_detach_irq(struct device *dev);
 extern void device_wakeup_arm_wake_irqs(void);
 extern void device_wakeup_disarm_wake_irqs(void);
 
 #else
 
-static inline int
-device_wakeup_attach_irq(struct device *dev,
-			 struct wake_irq *wakeirq)
-{
-	return 0;
-}
+static inline void device_wakeup_attach_irq(struct device *dev,
+					    struct wake_irq *wakeirq) {}
 
 static inline void device_wakeup_detach_irq(struct device *dev)
 {
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 6e89b51ea3d9..8bef3cb2424d 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -1613,22 +1613,34 @@ void pm_runtime_drop_link(struct device *dev)
 	spin_unlock_irq(&dev->power.lock);
 }
 
+static bool pm_runtime_need_not_resume(struct device *dev)
+{
+	return atomic_read(&dev->power.usage_count) <= 1 &&
+		(atomic_read(&dev->power.child_count) == 0 ||
+		 dev->power.ignore_children);
+}
+
 /**
  * pm_runtime_force_suspend - Force a device into suspend state if needed.
  * @dev: Device to suspend.
  *
  * Disable runtime PM so we safely can check the device's runtime PM status and
- * if it is active, invoke it's .runtime_suspend callback to bring it into
- * suspend state. Keep runtime PM disabled to preserve the state unless we
- * encounter errors.
+ * if it is active, invoke its ->runtime_suspend callback to suspend it and
+ * change its runtime PM status field to RPM_SUSPENDED.  Also, if the device's
+ * usage and children counters don't indicate that the device was in use before
+ * the system-wide transition under way, decrement its parent's children counter
+ * (if there is a parent).  Keep runtime PM disabled to preserve the state
+ * unless we encounter errors.
  *
  * Typically this function may be invoked from a system suspend callback to make
- * sure the device is put into low power state.
+ * sure the device is put into low power state and it should only be used during
+ * system-wide PM transitions to sleep states.  It assumes that the analogous
+ * pm_runtime_force_resume() will be used to resume the device.
  */
 int pm_runtime_force_suspend(struct device *dev)
 {
 	int (*callback)(struct device *);
-	int ret = 0;
+	int ret;
 
 	pm_runtime_disable(dev);
 	if (pm_runtime_status_suspended(dev))
@@ -1636,27 +1648,23 @@ int pm_runtime_force_suspend(struct device *dev)
 
 	callback = RPM_GET_CALLBACK(dev, runtime_suspend);
 
-	if (!callback) {
-		ret = -ENOSYS;
-		goto err;
-	}
-
-	ret = callback(dev);
+	ret = callback ? callback(dev) : 0;
 	if (ret)
 		goto err;
 
 	/*
-	 * Increase the runtime PM usage count for the device's parent, in case
-	 * when we find the device being used when system suspend was invoked.
-	 * This informs pm_runtime_force_resume() to resume the parent
-	 * immediately, which is needed to be able to resume its children,
-	 * when not deferring the resume to be managed via runtime PM.
+	 * If the device can stay in suspend after the system-wide transition
+	 * to the working state that will follow, drop the children counter of
+	 * its parent, but set its status to RPM_SUSPENDED anyway in case this
+	 * function will be called again for it in the meantime.
 	 */
-	if (dev->parent && atomic_read(&dev->power.usage_count) > 1)
-		pm_runtime_get_noresume(dev->parent);
+	if (pm_runtime_need_not_resume(dev))
+		pm_runtime_set_suspended(dev);
+	else
+		__update_runtime_status(dev, RPM_SUSPENDED);
 
-	pm_runtime_set_suspended(dev);
 	return 0;
+
 err:
 	pm_runtime_enable(dev);
 	return ret;
@@ -1669,13 +1677,9 @@ EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
  *
  * Prior invoking this function we expect the user to have brought the device
  * into low power state by a call to pm_runtime_force_suspend(). Here we reverse
- * those actions and brings the device into full power, if it is expected to be
- * used on system resume. To distinguish that, we check whether the runtime PM
- * usage count is greater than 1 (the PM core increases the usage count in the
- * system PM prepare phase), as that indicates a real user (such as a subsystem,
- * driver, userspace, etc.) is using it. If that is the case, the device is
- * expected to be used on system resume as well, so then we resume it. In the
- * other case, we defer the resume to be managed via runtime PM.
+ * those actions and bring the device into full power, if it is expected to be
+ * used on system resume.  In the other case, we defer the resume to be managed
+ * via runtime PM.
  *
  * Typically this function may be invoked from a system resume callback.
  */
@@ -1684,32 +1688,18 @@ int pm_runtime_force_resume(struct device *dev)
 	int (*callback)(struct device *);
 	int ret = 0;
 
-	callback = RPM_GET_CALLBACK(dev, runtime_resume);
-
-	if (!callback) {
-		ret = -ENOSYS;
-		goto out;
-	}
-
-	if (!pm_runtime_status_suspended(dev))
+	if (!pm_runtime_status_suspended(dev) || pm_runtime_need_not_resume(dev))
 		goto out;
 
 	/*
-	 * Decrease the parent's runtime PM usage count, if we increased it
-	 * during system suspend in pm_runtime_force_suspend().
-	*/
-	if (atomic_read(&dev->power.usage_count) > 1) {
-		if (dev->parent)
-			pm_runtime_put_noidle(dev->parent);
-	} else {
-		goto out;
-	}
+	 * The value of the parent's children counter is correct already, so
+	 * just update the status of the device.
+	 */
+	__update_runtime_status(dev, RPM_ACTIVE);
 
-	ret = pm_runtime_set_active(dev);
-	if (ret)
-		goto out;
+	callback = RPM_GET_CALLBACK(dev, runtime_resume);
 
-	ret = callback(dev);
+	ret = callback ? callback(dev) : 0;
 	if (ret) {
 		pm_runtime_set_suspended(dev);
 		goto out;
diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c
index e153e28b1857..0f651efc58a1 100644
--- a/drivers/base/power/sysfs.c
+++ b/drivers/base/power/sysfs.c
@@ -108,16 +108,10 @@ static ssize_t control_show(struct device *dev, struct device_attribute *attr,
 static ssize_t control_store(struct device * dev, struct device_attribute *attr,
 			     const char * buf, size_t n)
 {
-	char *cp;
-	int len = n;
-
-	cp = memchr(buf, '\n', n);
-	if (cp)
-		len = cp - buf;
 	device_lock(dev);
-	if (len == sizeof ctrl_auto - 1 && strncmp(buf, ctrl_auto, len) == 0)
+	if (sysfs_streq(buf, ctrl_auto))
 		pm_runtime_allow(dev);
-	else if (len == sizeof ctrl_on - 1 && strncmp(buf, ctrl_on, len) == 0)
+	else if (sysfs_streq(buf, ctrl_on))
 		pm_runtime_forbid(dev);
 	else
 		n = -EINVAL;
@@ -125,9 +119,9 @@ static ssize_t control_store(struct device * dev, struct device_attribute *attr,
 	return n;
 }
 
-static DEVICE_ATTR(control, 0644, control_show, control_store);
+static DEVICE_ATTR_RW(control);
 
-static ssize_t rtpm_active_time_show(struct device *dev,
+static ssize_t runtime_active_time_show(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
 	int ret;
@@ -138,9 +132,9 @@ static ssize_t rtpm_active_time_show(struct device *dev,
 	return ret;
 }
 
-static DEVICE_ATTR(runtime_active_time, 0444, rtpm_active_time_show, NULL);
+static DEVICE_ATTR_RO(runtime_active_time);
 
-static ssize_t rtpm_suspended_time_show(struct device *dev,
+static ssize_t runtime_suspended_time_show(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
 	int ret;
@@ -152,9 +146,9 @@ static ssize_t rtpm_suspended_time_show(struct device *dev,
 	return ret;
 }
 
-static DEVICE_ATTR(runtime_suspended_time, 0444, rtpm_suspended_time_show, NULL);
+static DEVICE_ATTR_RO(runtime_suspended_time);
 
-static ssize_t rtpm_status_show(struct device *dev,
+static ssize_t runtime_status_show(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
 	const char *p;
@@ -184,7 +178,7 @@ static ssize_t rtpm_status_show(struct device *dev,
 	return sprintf(buf, p);
 }
 
-static DEVICE_ATTR(runtime_status, 0444, rtpm_status_show, NULL);
+static DEVICE_ATTR_RO(runtime_status);
 
 static ssize_t autosuspend_delay_ms_show(struct device *dev,
 		struct device_attribute *attr, char *buf)
@@ -211,26 +205,25 @@ static ssize_t autosuspend_delay_ms_store(struct device *dev,
 	return n;
 }
 
-static DEVICE_ATTR(autosuspend_delay_ms, 0644, autosuspend_delay_ms_show,
-		autosuspend_delay_ms_store);
+static DEVICE_ATTR_RW(autosuspend_delay_ms);
 
-static ssize_t pm_qos_resume_latency_show(struct device *dev,
-					  struct device_attribute *attr,
-					  char *buf)
+static ssize_t pm_qos_resume_latency_us_show(struct device *dev,
+					     struct device_attribute *attr,
+					     char *buf)
 {
 	s32 value = dev_pm_qos_requested_resume_latency(dev);
 
 	if (value == 0)
 		return sprintf(buf, "n/a\n");
-	else if (value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
+	if (value == PM_QOS_RESUME_LATENCY_NO_CONSTRAINT)
 		value = 0;
 
 	return sprintf(buf, "%d\n", value);
 }
 
-static ssize_t pm_qos_resume_latency_store(struct device *dev,
-					   struct device_attribute *attr,
-					   const char *buf, size_t n)
+static ssize_t pm_qos_resume_latency_us_store(struct device *dev,
+					      struct device_attribute *attr,
+					      const char *buf, size_t n)
 {
 	s32 value;
 	int ret;
@@ -245,7 +238,7 @@ static ssize_t pm_qos_resume_latency_store(struct device *dev,
 
 		if (value == 0)
 			value = PM_QOS_RESUME_LATENCY_NO_CONSTRAINT;
-	} else if (!strcmp(buf, "n/a") || !strcmp(buf, "n/a\n")) {
+	} else if (sysfs_streq(buf, "n/a")) {
 		value = 0;
 	} else {
 		return -EINVAL;
@@ -256,26 +249,25 @@ static ssize_t pm_qos_resume_latency_store(struct device *dev,
 	return ret < 0 ? ret : n;
 }
 
-static DEVICE_ATTR(pm_qos_resume_latency_us, 0644,
-		   pm_qos_resume_latency_show, pm_qos_resume_latency_store);
+static DEVICE_ATTR_RW(pm_qos_resume_latency_us);
 
-static ssize_t pm_qos_latency_tolerance_show(struct device *dev,
-					     struct device_attribute *attr,
-					     char *buf)
+static ssize_t pm_qos_latency_tolerance_us_show(struct device *dev,
+						struct device_attribute *attr,
+						char *buf)
 {
 	s32 value = dev_pm_qos_get_user_latency_tolerance(dev);
 
 	if (value < 0)
 		return sprintf(buf, "auto\n");
-	else if (value == PM_QOS_LATENCY_ANY)
+	if (value == PM_QOS_LATENCY_ANY)
 		return sprintf(buf, "any\n");
 
 	return sprintf(buf, "%d\n", value);
 }
 
-static ssize_t pm_qos_latency_tolerance_store(struct device *dev,
-					      struct device_attribute *attr,
-					      const char *buf, size_t n)
+static ssize_t pm_qos_latency_tolerance_us_store(struct device *dev,
+						 struct device_attribute *attr,
+						 const char *buf, size_t n)
 {
 	s32 value;
 	int ret;
@@ -285,9 +277,9 @@ static ssize_t pm_qos_latency_tolerance_store(struct device *dev,
 		if (value < 0)
 			return -EINVAL;
 	} else {
-		if (!strcmp(buf, "auto") || !strcmp(buf, "auto\n"))
+		if (sysfs_streq(buf, "auto"))
 			value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT;
-		else if (!strcmp(buf, "any") || !strcmp(buf, "any\n"))
+		else if (sysfs_streq(buf, "any"))
 			value = PM_QOS_LATENCY_ANY;
 		else
 			return -EINVAL;
@@ -296,8 +288,7 @@ static ssize_t pm_qos_latency_tolerance_store(struct device *dev,
 	return ret < 0 ? ret : n;
 }
 
-static DEVICE_ATTR(pm_qos_latency_tolerance_us, 0644,
-		   pm_qos_latency_tolerance_show, pm_qos_latency_tolerance_store);
+static DEVICE_ATTR_RW(pm_qos_latency_tolerance_us);
 
 static ssize_t pm_qos_no_power_off_show(struct device *dev,
 					struct device_attribute *attr,
@@ -323,49 +314,39 @@ static ssize_t pm_qos_no_power_off_store(struct device *dev,
 	return ret < 0 ? ret : n;
 }
 
-static DEVICE_ATTR(pm_qos_no_power_off, 0644,
-		   pm_qos_no_power_off_show, pm_qos_no_power_off_store);
+static DEVICE_ATTR_RW(pm_qos_no_power_off);
 
 #ifdef CONFIG_PM_SLEEP
 static const char _enabled[] = "enabled";
 static const char _disabled[] = "disabled";
 
-static ssize_t
-wake_show(struct device * dev, struct device_attribute *attr, char * buf)
+static ssize_t wakeup_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
 {
 	return sprintf(buf, "%s\n", device_can_wakeup(dev)
 		? (device_may_wakeup(dev) ? _enabled : _disabled)
 		: "");
 }
 
-static ssize_t
-wake_store(struct device * dev, struct device_attribute *attr,
-	const char * buf, size_t n)
+static ssize_t wakeup_store(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t n)
 {
-	char *cp;
-	int len = n;
-
 	if (!device_can_wakeup(dev))
 		return -EINVAL;
 
-	cp = memchr(buf, '\n', n);
-	if (cp)
-		len = cp - buf;
-	if (len == sizeof _enabled - 1
-			&& strncmp(buf, _enabled, sizeof _enabled - 1) == 0)
+	if (sysfs_streq(buf, _enabled))
 		device_set_wakeup_enable(dev, 1);
-	else if (len == sizeof _disabled - 1
-			&& strncmp(buf, _disabled, sizeof _disabled - 1) == 0)
+	else if (sysfs_streq(buf, _disabled))
 		device_set_wakeup_enable(dev, 0);
 	else
 		return -EINVAL;
 	return n;
 }
 
-static DEVICE_ATTR(wakeup, 0644, wake_show, wake_store);
+static DEVICE_ATTR_RW(wakeup);
 
 static ssize_t wakeup_count_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
+				 struct device_attribute *attr, char *buf)
 {
 	unsigned long count = 0;
 	bool enabled = false;
@@ -379,10 +360,11 @@ static ssize_t wakeup_count_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_count, 0444, wakeup_count_show, NULL);
+static DEVICE_ATTR_RO(wakeup_count);
 
 static ssize_t wakeup_active_count_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
+					struct device_attribute *attr,
+					char *buf)
 {
 	unsigned long count = 0;
 	bool enabled = false;
@@ -396,11 +378,11 @@ static ssize_t wakeup_active_count_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_active_count, 0444, wakeup_active_count_show, NULL);
+static DEVICE_ATTR_RO(wakeup_active_count);
 
 static ssize_t wakeup_abort_count_show(struct device *dev,
-					struct device_attribute *attr,
-					char *buf)
+				       struct device_attribute *attr,
+				       char *buf)
 {
 	unsigned long count = 0;
 	bool enabled = false;
@@ -414,7 +396,7 @@ static ssize_t wakeup_abort_count_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_abort_count, 0444, wakeup_abort_count_show, NULL);
+static DEVICE_ATTR_RO(wakeup_abort_count);
 
 static ssize_t wakeup_expire_count_show(struct device *dev,
 					struct device_attribute *attr,
@@ -432,10 +414,10 @@ static ssize_t wakeup_expire_count_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lu\n", count) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_expire_count, 0444, wakeup_expire_count_show, NULL);
+static DEVICE_ATTR_RO(wakeup_expire_count);
 
 static ssize_t wakeup_active_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
+				  struct device_attribute *attr, char *buf)
 {
 	unsigned int active = 0;
 	bool enabled = false;
@@ -449,10 +431,11 @@ static ssize_t wakeup_active_show(struct device *dev,
 	return enabled ? sprintf(buf, "%u\n", active) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_active, 0444, wakeup_active_show, NULL);
+static DEVICE_ATTR_RO(wakeup_active);
 
-static ssize_t wakeup_total_time_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
+static ssize_t wakeup_total_time_ms_show(struct device *dev,
+					 struct device_attribute *attr,
+					 char *buf)
 {
 	s64 msec = 0;
 	bool enabled = false;
@@ -466,10 +449,10 @@ static ssize_t wakeup_total_time_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_total_time_ms, 0444, wakeup_total_time_show, NULL);
+static DEVICE_ATTR_RO(wakeup_total_time_ms);
 
-static ssize_t wakeup_max_time_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
+static ssize_t wakeup_max_time_ms_show(struct device *dev,
+				       struct device_attribute *attr, char *buf)
 {
 	s64 msec = 0;
 	bool enabled = false;
@@ -483,10 +466,11 @@ static ssize_t wakeup_max_time_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_max_time_ms, 0444, wakeup_max_time_show, NULL);
+static DEVICE_ATTR_RO(wakeup_max_time_ms);
 
-static ssize_t wakeup_last_time_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
+static ssize_t wakeup_last_time_ms_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
 {
 	s64 msec = 0;
 	bool enabled = false;
@@ -500,12 +484,12 @@ static ssize_t wakeup_last_time_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_last_time_ms, 0444, wakeup_last_time_show, NULL);
+static DEVICE_ATTR_RO(wakeup_last_time_ms);
 
 #ifdef CONFIG_PM_AUTOSLEEP
-static ssize_t wakeup_prevent_sleep_time_show(struct device *dev,
-					      struct device_attribute *attr,
-					      char *buf)
+static ssize_t wakeup_prevent_sleep_time_ms_show(struct device *dev,
+						 struct device_attribute *attr,
+						 char *buf)
 {
 	s64 msec = 0;
 	bool enabled = false;
@@ -519,40 +503,39 @@ static ssize_t wakeup_prevent_sleep_time_show(struct device *dev,
 	return enabled ? sprintf(buf, "%lld\n", msec) : sprintf(buf, "\n");
 }
 
-static DEVICE_ATTR(wakeup_prevent_sleep_time_ms, 0444,
-		   wakeup_prevent_sleep_time_show, NULL);
+static DEVICE_ATTR_RO(wakeup_prevent_sleep_time_ms);
 #endif /* CONFIG_PM_AUTOSLEEP */
 #endif /* CONFIG_PM_SLEEP */
 
 #ifdef CONFIG_PM_ADVANCED_DEBUG
-static ssize_t rtpm_usagecount_show(struct device *dev,
-				    struct device_attribute *attr, char *buf)
+static ssize_t runtime_usage_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
 {
 	return sprintf(buf, "%d\n", atomic_read(&dev->power.usage_count));
 }
+static DEVICE_ATTR_RO(runtime_usage);
 
-static ssize_t rtpm_children_show(struct device *dev,
-				  struct device_attribute *attr, char *buf)
+static ssize_t runtime_active_kids_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
 {
 	return sprintf(buf, "%d\n", dev->power.ignore_children ?
 		0 : atomic_read(&dev->power.child_count));
 }
+static DEVICE_ATTR_RO(runtime_active_kids);
 
-static ssize_t rtpm_enabled_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
+static ssize_t runtime_enabled_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
 {
-	if ((dev->power.disable_depth) && (dev->power.runtime_auto == false))
+	if (dev->power.disable_depth && (dev->power.runtime_auto == false))
 		return sprintf(buf, "disabled & forbidden\n");
-	else if (dev->power.disable_depth)
+	if (dev->power.disable_depth)
 		return sprintf(buf, "disabled\n");
-	else if (dev->power.runtime_auto == false)
+	if (dev->power.runtime_auto == false)
 		return sprintf(buf, "forbidden\n");
 	return sprintf(buf, "enabled\n");
 }
-
-static DEVICE_ATTR(runtime_usage, 0444, rtpm_usagecount_show, NULL);
-static DEVICE_ATTR(runtime_active_kids, 0444, rtpm_children_show, NULL);
-static DEVICE_ATTR(runtime_enabled, 0444, rtpm_enabled_show, NULL);
+static DEVICE_ATTR_RO(runtime_enabled);
 
 #ifdef CONFIG_PM_SLEEP
 static ssize_t async_show(struct device *dev, struct device_attribute *attr,
@@ -566,23 +549,16 @@ static ssize_t async_show(struct device *dev, struct device_attribute *attr,
 static ssize_t async_store(struct device *dev, struct device_attribute *attr,
 			   const char *buf, size_t n)
 {
-	char *cp;
-	int len = n;
-
-	cp = memchr(buf, '\n', n);
-	if (cp)
-		len = cp - buf;
-	if (len == sizeof _enabled - 1 && strncmp(buf, _enabled, len) == 0)
+	if (sysfs_streq(buf, _enabled))
 		device_enable_async_suspend(dev);
-	else if (len == sizeof _disabled - 1 &&
-		 strncmp(buf, _disabled, len) == 0)
+	else if (sysfs_streq(buf, _disabled))
 		device_disable_async_suspend(dev);
 	else
 		return -EINVAL;
 	return n;
 }
 
-static DEVICE_ATTR(async, 0644, async_show, async_store);
+static DEVICE_ATTR_RW(async);
 
 #endif /* CONFIG_PM_SLEEP */
 #endif /* CONFIG_PM_ADVANCED_DEBUG */
diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c
index ae0429827f31..6637fc319269 100644
--- a/drivers/base/power/wakeirq.c
+++ b/drivers/base/power/wakeirq.c
@@ -33,7 +33,6 @@ static int dev_pm_attach_wake_irq(struct device *dev, int irq,
 				  struct wake_irq *wirq)
 {
 	unsigned long flags;
-	int err;
 
 	if (!dev || !wirq)
 		return -EINVAL;
@@ -45,12 +44,11 @@ static int dev_pm_attach_wake_irq(struct device *dev, int irq,
 		return -EEXIST;
 	}
 
-	err = device_wakeup_attach_irq(dev, wirq);
-	if (!err)
-		dev->power.wakeirq = wirq;
+	dev->power.wakeirq = wirq;
+	device_wakeup_attach_irq(dev, wirq);
 
 	spin_unlock_irqrestore(&dev->power.lock, flags);
-	return err;
+	return 0;
 }
 
 /**
@@ -323,7 +321,8 @@ void dev_pm_arm_wake_irq(struct wake_irq *wirq)
 		return;
 
 	if (device_may_wakeup(wirq->dev)) {
-		if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)
+		if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED &&
+		    !pm_runtime_status_suspended(wirq->dev))
 			enable_irq(wirq->irq);
 
 		enable_irq_wake(wirq->irq);
@@ -345,7 +344,8 @@ void dev_pm_disarm_wake_irq(struct wake_irq *wirq)
 	if (device_may_wakeup(wirq->dev)) {
 		disable_irq_wake(wirq->irq);
 
-		if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)
+		if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED &&
+		    !pm_runtime_status_suspended(wirq->dev))
 			disable_irq_nosync(wirq->irq);
 	}
 }
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index 38559f04db2c..ea01621ed769 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -19,6 +19,11 @@
 
 #include "power.h"
 
+#ifndef CONFIG_SUSPEND
+suspend_state_t pm_suspend_target_state;
+#define pm_suspend_target_state	(PM_SUSPEND_ON)
+#endif
+
 /*
  * If set, the suspend/hibernate code will abort transitions to a sleep state
  * if wakeup events are registered during or immediately before the transition.
@@ -268,6 +273,9 @@ int device_wakeup_enable(struct device *dev)
 	if (!dev || !dev->power.can_wakeup)
 		return -EINVAL;
 
+	if (pm_suspend_target_state != PM_SUSPEND_ON)
+		dev_dbg(dev, "Suspicious %s() during system transition!\n", __func__);
+
 	ws = wakeup_source_register(dev_name(dev));
 	if (!ws)
 		return -ENOMEM;
@@ -291,22 +299,19 @@ EXPORT_SYMBOL_GPL(device_wakeup_enable);
  *
  * Call under the device's power.lock lock.
  */
-int device_wakeup_attach_irq(struct device *dev,
+void device_wakeup_attach_irq(struct device *dev,
 			     struct wake_irq *wakeirq)
 {
 	struct wakeup_source *ws;
 
 	ws = dev->power.wakeup;
-	if (!ws) {
-		dev_err(dev, "forgot to call call device_init_wakeup?\n");
-		return -EINVAL;
-	}
+	if (!ws)
+		return;
 
 	if (ws->wakeirq)
-		return -EEXIST;
+		dev_err(dev, "Leftover wakeup IRQ found, overriding\n");
 
 	ws->wakeirq = wakeirq;
-	return 0;
 }
 
 /**
@@ -448,9 +453,7 @@ int device_init_wakeup(struct device *dev, bool enable)
 		device_set_wakeup_capable(dev, true);
 		ret = device_wakeup_enable(dev);
 	} else {
-		if (dev->power.can_wakeup)
-			device_wakeup_disable(dev);
-
+		device_wakeup_disable(dev);
 		device_set_wakeup_capable(dev, false);
 	}
 
@@ -464,9 +467,6 @@ EXPORT_SYMBOL_GPL(device_init_wakeup);
  */
 int device_set_wakeup_enable(struct device *dev, bool enable)
 {
-	if (!dev || !dev->power.can_wakeup)
-		return -EINVAL;
-
 	return enable ? device_wakeup_enable(dev) : device_wakeup_disable(dev);
 }
 EXPORT_SYMBOL_GPL(device_set_wakeup_enable);
diff --git a/drivers/base/property.c b/drivers/base/property.c
index 851b1b6596a4..8f205f6461ed 100644
--- a/drivers/base/property.c
+++ b/drivers/base/property.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * property.c - Unified device property interface.
  *
  * Copyright (C) 2014, Intel Corporation
  * Authors: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
  *          Mika Westerberg <mika.westerberg@linux.intel.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/acpi.h>
@@ -16,6 +13,7 @@
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_graph.h>
+#include <linux/of_irq.h>
 #include <linux/property.h>
 #include <linux/etherdevice.h>
 #include <linux/phy.h>
@@ -698,6 +696,23 @@ int fwnode_property_get_reference_args(const struct fwnode_handle *fwnode,
 }
 EXPORT_SYMBOL_GPL(fwnode_property_get_reference_args);
 
+static void property_entry_free_data(const struct property_entry *p)
+{
+	size_t i, nval;
+
+	if (p->is_array) {
+		if (p->is_string && p->pointer.str) {
+			nval = p->length / sizeof(const char *);
+			for (i = 0; i < nval; i++)
+				kfree(p->pointer.str[i]);
+		}
+		kfree(p->pointer.raw_data);
+	} else if (p->is_string) {
+		kfree(p->value.str);
+	}
+	kfree(p->name);
+}
+
 static int property_copy_string_array(struct property_entry *dst,
 				      const struct property_entry *src)
 {
@@ -728,34 +743,24 @@ static int property_entry_copy_data(struct property_entry *dst,
 {
 	int error;
 
-	dst->name = kstrdup(src->name, GFP_KERNEL);
-	if (!dst->name)
-		return -ENOMEM;
-
 	if (src->is_array) {
-		if (!src->length) {
-			error = -ENODATA;
-			goto out_free_name;
-		}
+		if (!src->length)
+			return -ENODATA;
 
 		if (src->is_string) {
 			error = property_copy_string_array(dst, src);
 			if (error)
-				goto out_free_name;
+				return error;
 		} else {
 			dst->pointer.raw_data = kmemdup(src->pointer.raw_data,
 							src->length, GFP_KERNEL);
-			if (!dst->pointer.raw_data) {
-				error = -ENOMEM;
-				goto out_free_name;
-			}
+			if (!dst->pointer.raw_data)
+				return -ENOMEM;
 		}
 	} else if (src->is_string) {
 		dst->value.str = kstrdup(src->value.str, GFP_KERNEL);
-		if (!dst->value.str && src->value.str) {
-			error = -ENOMEM;
-			goto out_free_name;
-		}
+		if (!dst->value.str && src->value.str)
+			return -ENOMEM;
 	} else {
 		dst->value.raw_data = src->value.raw_data;
 	}
@@ -764,28 +769,15 @@ static int property_entry_copy_data(struct property_entry *dst,
 	dst->is_array = src->is_array;
 	dst->is_string = src->is_string;
 
+	dst->name = kstrdup(src->name, GFP_KERNEL);
+	if (!dst->name)
+		goto out_free_data;
+
 	return 0;
 
-out_free_name:
-	kfree(dst->name);
-	return error;
-}
-
-static void property_entry_free_data(const struct property_entry *p)
-{
-	size_t i, nval;
-
-	if (p->is_array) {
-		if (p->is_string && p->pointer.str) {
-			nval = p->length / sizeof(const char *);
-			for (i = 0; i < nval; i++)
-				kfree(p->pointer.str[i]);
-		}
-		kfree(p->pointer.raw_data);
-	} else if (p->is_string) {
-		kfree(p->value.str);
-	}
-	kfree(p->name);
+out_free_data:
+	property_entry_free_data(dst);
+	return -ENOMEM;
 }
 
 /**
@@ -996,6 +988,32 @@ fwnode_get_next_child_node(const struct fwnode_handle *fwnode,
 }
 EXPORT_SYMBOL_GPL(fwnode_get_next_child_node);
 
+/**
+ * fwnode_get_next_available_child_node - Return the next
+ * available child node handle for a node
+ * @fwnode: Firmware node to find the next child node for.
+ * @child: Handle to one of the node's child nodes or a %NULL handle.
+ */
+struct fwnode_handle *
+fwnode_get_next_available_child_node(const struct fwnode_handle *fwnode,
+				     struct fwnode_handle *child)
+{
+	struct fwnode_handle *next_child = child;
+
+	if (!fwnode)
+		return NULL;
+
+	do {
+		next_child = fwnode_get_next_child_node(fwnode, next_child);
+
+		if (!next_child || fwnode_device_is_available(next_child))
+			break;
+	} while (next_child);
+
+	return next_child;
+}
+EXPORT_SYMBOL_GPL(fwnode_get_next_available_child_node);
+
 /**
  * device_get_next_child_node - Return the next child node handle for a device
  * @dev: Device to find the next child node for.
@@ -1126,21 +1144,21 @@ enum dev_dma_attr device_get_dma_attr(struct device *dev)
 EXPORT_SYMBOL_GPL(device_get_dma_attr);
 
 /**
- * device_get_phy_mode - Get phy mode for given device
- * @dev:	Pointer to the given device
+ * fwnode_get_phy_mode - Get phy mode for given firmware node
+ * @fwnode:	Pointer to the given node
  *
  * The function gets phy interface string from property 'phy-mode' or
  * 'phy-connection-type', and return its index in phy_modes table, or errno in
  * error case.
  */
-int device_get_phy_mode(struct device *dev)
+int fwnode_get_phy_mode(struct fwnode_handle *fwnode)
 {
 	const char *pm;
 	int err, i;
 
-	err = device_property_read_string(dev, "phy-mode", &pm);
+	err = fwnode_property_read_string(fwnode, "phy-mode", &pm);
 	if (err < 0)
-		err = device_property_read_string(dev,
+		err = fwnode_property_read_string(fwnode,
 						  "phy-connection-type", &pm);
 	if (err < 0)
 		return err;
@@ -1151,13 +1169,27 @@ int device_get_phy_mode(struct device *dev)
 
 	return -ENODEV;
 }
+EXPORT_SYMBOL_GPL(fwnode_get_phy_mode);
+
+/**
+ * device_get_phy_mode - Get phy mode for given device
+ * @dev:	Pointer to the given device
+ *
+ * The function gets phy interface string from property 'phy-mode' or
+ * 'phy-connection-type', and return its index in phy_modes table, or errno in
+ * error case.
+ */
+int device_get_phy_mode(struct device *dev)
+{
+	return fwnode_get_phy_mode(dev_fwnode(dev));
+}
 EXPORT_SYMBOL_GPL(device_get_phy_mode);
 
-static void *device_get_mac_addr(struct device *dev,
+static void *fwnode_get_mac_addr(struct fwnode_handle *fwnode,
 				 const char *name, char *addr,
 				 int alen)
 {
-	int ret = device_property_read_u8_array(dev, name, addr, alen);
+	int ret = fwnode_property_read_u8_array(fwnode, name, addr, alen);
 
 	if (ret == 0 && alen == ETH_ALEN && is_valid_ether_addr(addr))
 		return addr;
@@ -1165,8 +1197,8 @@ static void *device_get_mac_addr(struct device *dev,
 }
 
 /**
- * device_get_mac_address - Get the MAC for a given device
- * @dev:	Pointer to the device
+ * fwnode_get_mac_address - Get the MAC from the firmware node
+ * @fwnode:	Pointer to the firmware node
  * @addr:	Address of buffer to store the MAC in
  * @alen:	Length of the buffer pointed to by addr, should be ETH_ALEN
  *
@@ -1187,22 +1219,59 @@ static void *device_get_mac_addr(struct device *dev,
  * In this case, the real MAC is in 'local-mac-address', and 'mac-address'
  * exists but is all zeros.
 */
-void *device_get_mac_address(struct device *dev, char *addr, int alen)
+void *fwnode_get_mac_address(struct fwnode_handle *fwnode, char *addr, int alen)
 {
 	char *res;
 
-	res = device_get_mac_addr(dev, "mac-address", addr, alen);
+	res = fwnode_get_mac_addr(fwnode, "mac-address", addr, alen);
 	if (res)
 		return res;
 
-	res = device_get_mac_addr(dev, "local-mac-address", addr, alen);
+	res = fwnode_get_mac_addr(fwnode, "local-mac-address", addr, alen);
 	if (res)
 		return res;
 
-	return device_get_mac_addr(dev, "address", addr, alen);
+	return fwnode_get_mac_addr(fwnode, "address", addr, alen);
+}
+EXPORT_SYMBOL(fwnode_get_mac_address);
+
+/**
+ * device_get_mac_address - Get the MAC for a given device
+ * @dev:	Pointer to the device
+ * @addr:	Address of buffer to store the MAC in
+ * @alen:	Length of the buffer pointed to by addr, should be ETH_ALEN
+ */
+void *device_get_mac_address(struct device *dev, char *addr, int alen)
+{
+	return fwnode_get_mac_address(dev_fwnode(dev), addr, alen);
 }
 EXPORT_SYMBOL(device_get_mac_address);
 
+/**
+ * fwnode_irq_get - Get IRQ directly from a fwnode
+ * @fwnode:	Pointer to the firmware node
+ * @index:	Zero-based index of the IRQ
+ *
+ * Returns Linux IRQ number on success. Other values are determined
+ * accordingly to acpi_/of_ irq_get() operation.
+ */
+int fwnode_irq_get(struct fwnode_handle *fwnode, unsigned int index)
+{
+	struct device_node *of_node = to_of_node(fwnode);
+	struct resource res;
+	int ret;
+
+	if (IS_ENABLED(CONFIG_OF) && of_node)
+		return of_irq_get(of_node, index);
+
+	ret = acpi_irq_get(ACPI_HANDLE_FWNODE(fwnode), index, &res);
+	if (ret)
+		return ret;
+
+	return res.start;
+}
+EXPORT_SYMBOL(fwnode_irq_get);
+
 /**
  * device_graph_get_next_endpoint - Get next endpoint firmware node
  * @fwnode: Pointer to the parent firmware node
@@ -1340,3 +1409,9 @@ int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode,
 	return fwnode_call_int_op(fwnode, graph_parse_endpoint, endpoint);
 }
 EXPORT_SYMBOL(fwnode_graph_parse_endpoint);
+
+const void *device_get_match_data(struct device *dev)
+{
+	return fwnode_call_ptr_op(dev_fwnode(dev), device_get_match_data, dev);
+}
+EXPORT_SYMBOL_GPL(device_get_match_data);
diff --git a/drivers/base/regmap/Kconfig b/drivers/base/regmap/Kconfig
index 3a1535d812d8..aff34c0c2a3e 100644
--- a/drivers/base/regmap/Kconfig
+++ b/drivers/base/regmap/Kconfig
@@ -6,7 +6,6 @@
 config REGMAP
 	default y if (REGMAP_I2C || REGMAP_SPI || REGMAP_SPMI || REGMAP_W1 || REGMAP_AC97 || REGMAP_MMIO || REGMAP_IRQ)
 	select IRQ_DOMAIN if REGMAP_IRQ
-	select REGMAP_HWSPINLOCK if HWSPINLOCK=y
 	bool
 
 config REGCACHE_COMPRESSED
@@ -21,6 +20,10 @@ config REGMAP_I2C
 	tristate
 	depends on I2C
 
+config REGMAP_SLIMBUS
+	tristate
+	depends on SLIMBUS
+
 config REGMAP_SPI
 	tristate
 	depends on SPI
@@ -39,5 +42,6 @@ config REGMAP_MMIO
 config REGMAP_IRQ
 	bool
 
-config REGMAP_HWSPINLOCK
-	bool
+config REGMAP_SOUNDWIRE
+	tristate
+	depends on SOUNDWIRE_BUS
diff --git a/drivers/base/regmap/Makefile b/drivers/base/regmap/Makefile
index 0d298c446108..5ed0023fabda 100644
--- a/drivers/base/regmap/Makefile
+++ b/drivers/base/regmap/Makefile
@@ -8,8 +8,10 @@ obj-$(CONFIG_REGCACHE_COMPRESSED) += regcache-lzo.o
 obj-$(CONFIG_DEBUG_FS) += regmap-debugfs.o
 obj-$(CONFIG_REGMAP_AC97) += regmap-ac97.o
 obj-$(CONFIG_REGMAP_I2C) += regmap-i2c.o
+obj-$(CONFIG_REGMAP_SLIMBUS) += regmap-slimbus.o
 obj-$(CONFIG_REGMAP_SPI) += regmap-spi.o
 obj-$(CONFIG_REGMAP_SPMI) += regmap-spmi.o
 obj-$(CONFIG_REGMAP_MMIO) += regmap-mmio.o
 obj-$(CONFIG_REGMAP_IRQ) += regmap-irq.o
 obj-$(CONFIG_REGMAP_W1) += regmap-w1.o
+obj-$(CONFIG_REGMAP_SOUNDWIRE) += regmap-sdw.o
diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h
index 8641183cac2f..53785e0e297a 100644
--- a/drivers/base/regmap/internal.h
+++ b/drivers/base/regmap/internal.h
@@ -77,6 +77,7 @@ struct regmap {
 	int async_ret;
 
 #ifdef CONFIG_DEBUG_FS
+	bool debugfs_disable;
 	struct dentry *debugfs;
 	const char *debugfs_name;
 
@@ -215,10 +216,17 @@ struct regmap_field {
 extern void regmap_debugfs_initcall(void);
 extern void regmap_debugfs_init(struct regmap *map, const char *name);
 extern void regmap_debugfs_exit(struct regmap *map);
+
+static inline void regmap_debugfs_disable(struct regmap *map)
+{
+	map->debugfs_disable = true;
+}
+
 #else
 static inline void regmap_debugfs_initcall(void) { }
 static inline void regmap_debugfs_init(struct regmap *map, const char *name) { }
 static inline void regmap_debugfs_exit(struct regmap *map) { }
+static inline void regmap_debugfs_disable(struct regmap *map) { }
 #endif
 
 /* regcache core declarations */
diff --git a/drivers/base/regmap/regcache-flat.c b/drivers/base/regmap/regcache-flat.c
index 4d2e50bfc726..bc6cd88b8cc6 100644
--- a/drivers/base/regmap/regcache-flat.c
+++ b/drivers/base/regmap/regcache-flat.c
@@ -37,9 +37,12 @@ static int regcache_flat_init(struct regmap *map)
 
 	cache = map->cache;
 
-	for (i = 0; i < map->num_reg_defaults; i++)
-		cache[regcache_flat_get_index(map, map->reg_defaults[i].reg)] =
-				map->reg_defaults[i].def;
+	for (i = 0; i < map->num_reg_defaults; i++) {
+		unsigned int reg = map->reg_defaults[i].reg;
+		unsigned int index = regcache_flat_get_index(map, reg);
+
+		cache[index] = map->reg_defaults[i].def;
+	}
 
 	return 0;
 }
@@ -56,8 +59,9 @@ static int regcache_flat_read(struct regmap *map,
 			      unsigned int reg, unsigned int *value)
 {
 	unsigned int *cache = map->cache;
+	unsigned int index = regcache_flat_get_index(map, reg);
 
-	*value = cache[regcache_flat_get_index(map, reg)];
+	*value = cache[index];
 
 	return 0;
 }
@@ -66,8 +70,9 @@ static int regcache_flat_write(struct regmap *map, unsigned int reg,
 			       unsigned int value)
 {
 	unsigned int *cache = map->cache;
+	unsigned int index = regcache_flat_get_index(map, reg);
 
-	cache[regcache_flat_get_index(map, reg)] = value;
+	cache[index] = value;
 
 	return 0;
 }
diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c
index 36ce3511c733..f3266334063e 100644
--- a/drivers/base/regmap/regmap-debugfs.c
+++ b/drivers/base/regmap/regmap-debugfs.c
@@ -529,6 +529,18 @@ void regmap_debugfs_init(struct regmap *map, const char *name)
 	struct regmap_range_node *range_node;
 	const char *devname = "dummy";
 
+	/*
+	 * Userspace can initiate reads from the hardware over debugfs.
+	 * Normally internal regmap structures and buffers are protected with
+	 * a mutex or a spinlock, but if the regmap owner decided to disable
+	 * all locking mechanisms, this is no longer the case. For safety:
+	 * don't create the debugfs entries if locking is disabled.
+	 */
+	if (map->debugfs_disable) {
+		dev_dbg(map->dev, "regmap locking disabled - not creating debugfs entries\n");
+		return;
+	}
+
 	/* If we don't have the debugfs root yet, postpone init */
 	if (!regmap_debugfs_root) {
 		struct regmap_debugfs_node *node;
diff --git a/drivers/base/regmap/regmap-sdw.c b/drivers/base/regmap/regmap-sdw.c
new file mode 100644
index 000000000000..50a66382d87d
--- /dev/null
+++ b/drivers/base/regmap/regmap-sdw.c
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright(c) 2015-17 Intel Corporation.
+
+#include <linux/device.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/soundwire/sdw.h>
+#include "internal.h"
+
+static int regmap_sdw_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct device *dev = context;
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+
+	return sdw_write(slave, reg, val);
+}
+
+static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct device *dev = context;
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+	int read;
+
+	read = sdw_read(slave, reg);
+	if (read < 0)
+		return read;
+
+	*val = read;
+	return 0;
+}
+
+static struct regmap_bus regmap_sdw = {
+	.reg_read = regmap_sdw_read,
+	.reg_write = regmap_sdw_write,
+	.reg_format_endian_default = REGMAP_ENDIAN_LITTLE,
+	.val_format_endian_default = REGMAP_ENDIAN_LITTLE,
+};
+
+static int regmap_sdw_config_check(const struct regmap_config *config)
+{
+	/* All register are 8-bits wide as per MIPI Soundwire 1.0 Spec */
+	if (config->val_bits != 8)
+		return -ENOTSUPP;
+
+	/* Registers are 32 bits wide */
+	if (config->reg_bits != 32)
+		return -ENOTSUPP;
+
+	if (config->pad_bits != 0)
+		return -ENOTSUPP;
+
+	return 0;
+}
+
+struct regmap *__regmap_init_sdw(struct sdw_slave *sdw,
+				 const struct regmap_config *config,
+				 struct lock_class_key *lock_key,
+				 const char *lock_name)
+{
+	int ret;
+
+	ret = regmap_sdw_config_check(config);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return __regmap_init(&sdw->dev, &regmap_sdw,
+			&sdw->dev, config, lock_key, lock_name);
+}
+EXPORT_SYMBOL_GPL(__regmap_init_sdw);
+
+struct regmap *__devm_regmap_init_sdw(struct sdw_slave *sdw,
+				      const struct regmap_config *config,
+				      struct lock_class_key *lock_key,
+				      const char *lock_name)
+{
+	int ret;
+
+	ret = regmap_sdw_config_check(config);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return __devm_regmap_init(&sdw->dev, &regmap_sdw,
+			&sdw->dev, config, lock_key, lock_name);
+}
+EXPORT_SYMBOL_GPL(__devm_regmap_init_sdw);
+
+MODULE_DESCRIPTION("Regmap SoundWire Module");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/base/regmap/regmap-slimbus.c b/drivers/base/regmap/regmap-slimbus.c
new file mode 100644
index 000000000000..c90bee81d954
--- /dev/null
+++ b/drivers/base/regmap/regmap-slimbus.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017, Linaro Ltd.
+
+#include <linux/regmap.h>
+#include <linux/slimbus.h>
+#include <linux/module.h>
+
+#include "internal.h"
+
+static int regmap_slimbus_byte_reg_read(void *context, unsigned int reg,
+					unsigned int *val)
+{
+	struct slim_device *sdev = context;
+	int v;
+
+	v = slim_readb(sdev, reg);
+
+	if (v < 0)
+		return v;
+
+	*val = v;
+
+	return 0;
+}
+
+static int regmap_slimbus_byte_reg_write(void *context, unsigned int reg,
+					 unsigned int val)
+{
+	struct slim_device *sdev = context;
+
+	return slim_writeb(sdev, reg, val);
+}
+
+static struct regmap_bus regmap_slimbus_bus = {
+	.reg_write = regmap_slimbus_byte_reg_write,
+	.reg_read = regmap_slimbus_byte_reg_read,
+	.reg_format_endian_default = REGMAP_ENDIAN_LITTLE,
+	.val_format_endian_default = REGMAP_ENDIAN_LITTLE,
+};
+
+static const struct regmap_bus *regmap_get_slimbus(struct slim_device *slim,
+					const struct regmap_config *config)
+{
+	if (config->val_bits == 8 && config->reg_bits == 8)
+		return &regmap_slimbus_bus;
+
+	return ERR_PTR(-ENOTSUPP);
+}
+
+struct regmap *__regmap_init_slimbus(struct slim_device *slimbus,
+				     const struct regmap_config *config,
+				     struct lock_class_key *lock_key,
+				     const char *lock_name)
+{
+	const struct regmap_bus *bus = regmap_get_slimbus(slimbus, config);
+
+	if (IS_ERR(bus))
+		return ERR_CAST(bus);
+
+	return __regmap_init(&slimbus->dev, bus, &slimbus->dev, config,
+			     lock_key, lock_name);
+}
+EXPORT_SYMBOL_GPL(__regmap_init_slimbus);
+
+struct regmap *__devm_regmap_init_slimbus(struct slim_device *slimbus,
+					  const struct regmap_config *config,
+					  struct lock_class_key *lock_key,
+					  const char *lock_name)
+{
+	const struct regmap_bus *bus = regmap_get_slimbus(slimbus, config);
+
+	if (IS_ERR(bus))
+		return ERR_CAST(bus);
+
+	return __devm_regmap_init(&slimbus->dev, bus, &slimbus, config,
+				  lock_key, lock_name);
+}
+EXPORT_SYMBOL_GPL(__devm_regmap_init_slimbus);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index 8d516a9bfc01..ee302ccdfbc8 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -414,7 +414,6 @@ static unsigned int regmap_parse_64_native(const void *buf)
 }
 #endif
 
-#ifdef REGMAP_HWSPINLOCK
 static void regmap_lock_hwlock(void *__map)
 {
 	struct regmap *map = __map;
@@ -457,7 +456,11 @@ static void regmap_unlock_hwlock_irqrestore(void *__map)
 
 	hwspin_unlock_irqrestore(map->hwlock, &map->spinlock_flags);
 }
-#endif
+
+static void regmap_lock_unlock_none(void *__map)
+{
+
+}
 
 static void regmap_lock_mutex(void *__map)
 {
@@ -669,16 +672,26 @@ struct regmap *__regmap_init(struct device *dev,
 		goto err;
 	}
 
-	if (config->lock && config->unlock) {
+	if (config->name) {
+		map->name = kstrdup_const(config->name, GFP_KERNEL);
+		if (!map->name) {
+			ret = -ENOMEM;
+			goto err_map;
+		}
+	}
+
+	if (config->disable_locking) {
+		map->lock = map->unlock = regmap_lock_unlock_none;
+		regmap_debugfs_disable(map);
+	} else if (config->lock && config->unlock) {
 		map->lock = config->lock;
 		map->unlock = config->unlock;
 		map->lock_arg = config->lock_arg;
-	} else if (config->hwlock_id) {
-#ifdef REGMAP_HWSPINLOCK
+	} else if (config->use_hwlock) {
 		map->hwlock = hwspin_lock_request_specific(config->hwlock_id);
 		if (!map->hwlock) {
 			ret = -ENXIO;
-			goto err_map;
+			goto err_name;
 		}
 
 		switch (config->hwlock_mode) {
@@ -697,10 +710,6 @@ struct regmap *__regmap_init(struct device *dev,
 		}
 
 		map->lock_arg = map;
-#else
-		ret = -EINVAL;
-		goto err_map;
-#endif
 	} else {
 		if ((bus && bus->fast_io) ||
 		    config->fast_io) {
@@ -762,14 +771,15 @@ struct regmap *__regmap_init(struct device *dev,
 	map->volatile_reg = config->volatile_reg;
 	map->precious_reg = config->precious_reg;
 	map->cache_type = config->cache_type;
-	map->name = config->name;
 
 	spin_lock_init(&map->async_lock);
 	INIT_LIST_HEAD(&map->async_list);
 	INIT_LIST_HEAD(&map->async_free);
 	init_waitqueue_head(&map->async_waitq);
 
-	if (config->read_flag_mask || config->write_flag_mask) {
+	if (config->read_flag_mask ||
+	    config->write_flag_mask ||
+	    config->zero_flag_mask) {
 		map->read_flag_mask = config->read_flag_mask;
 		map->write_flag_mask = config->write_flag_mask;
 	} else if (bus) {
@@ -1116,8 +1126,10 @@ struct regmap *__regmap_init(struct device *dev,
 	regmap_range_exit(map);
 	kfree(map->work_buf);
 err_hwlock:
-	if (IS_ENABLED(REGMAP_HWSPINLOCK) && map->hwlock)
+	if (map->hwlock)
 		hwspin_lock_free(map->hwlock);
+err_name:
+	kfree_const(map->name);
 err_map:
 	kfree(map);
 err:
@@ -1305,8 +1317,9 @@ void regmap_exit(struct regmap *map)
 		kfree(async->work_buf);
 		kfree(async);
 	}
-	if (IS_ENABLED(REGMAP_HWSPINLOCK) && map->hwlock)
+	if (map->hwlock)
 		hwspin_lock_free(map->hwlock);
+	kfree_const(map->name);
 	kfree(map);
 }
 EXPORT_SYMBOL_GPL(regmap_exit);
@@ -2423,13 +2436,15 @@ static int _regmap_bus_read(void *context, unsigned int reg,
 {
 	int ret;
 	struct regmap *map = context;
+	void *work_val = map->work_buf + map->format.reg_bytes +
+		map->format.pad_bytes;
 
 	if (!map->format.parse_val)
 		return -EINVAL;
 
-	ret = _regmap_raw_read(map, reg, map->work_buf, map->format.val_bytes);
+	ret = _regmap_raw_read(map, reg, work_val, map->format.val_bytes);
 	if (ret == 0)
-		*val = map->format.parse_val(map->work_buf);
+		*val = map->format.parse_val(work_val);
 
 	return ret;
 }
diff --git a/drivers/base/soc.c b/drivers/base/soc.c
index 909dedae4c4e..4e80f48ad5d6 100644
--- a/drivers/base/soc.c
+++ b/drivers/base/soc.c
@@ -1,8 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) ST-Ericsson SA 2011
  *
  * Author: Lee Jones <lee.jones@linaro.org> for ST-Ericsson.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/sysfs.h>
diff --git a/drivers/base/syscore.c b/drivers/base/syscore.c
index 8d98a329f6ea..6e076f359dcc 100644
--- a/drivers/base/syscore.c
+++ b/drivers/base/syscore.c
@@ -1,9 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *  syscore.c - Execution of system core operations.
  *
  *  Copyright (C) 2011 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
- *
- *  This file is released under the GPLv2.
  */
 
 #include <linux/syscore_ops.h>
diff --git a/drivers/base/test/test_async_driver_probe.c b/drivers/base/test/test_async_driver_probe.c
index a3355d66bc12..e7f145d662f0 100644
--- a/drivers/base/test/test_async_driver_probe.c
+++ b/drivers/base/test/test_async_driver_probe.c
@@ -1,14 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2014 Google, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
diff --git a/drivers/base/topology.c b/drivers/base/topology.c
index d936fcf9f1fb..5fd9f167ecc1 100644
--- a/drivers/base/topology.c
+++ b/drivers/base/topology.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * driver/base/topology.c - Populate sysfs with cpu topology information
  *
@@ -6,22 +7,6 @@
  * Copyright (C) 2006, Intel Corp.
  *
  * All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- * NON INFRINGEMENT.  See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
  */
 #include <linux/mm.h>
 #include <linux/cpu.h>
diff --git a/drivers/base/transport_class.c b/drivers/base/transport_class.c
index f6c453c3816e..5ed86ded6e6b 100644
--- a/drivers/base/transport_class.c
+++ b/drivers/base/transport_class.c
@@ -1,11 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * transport_class.c - implementation of generic transport classes
  *                     using attribute_containers
  *
  * Copyright (c) 2005 - James Bottomley <James.Bottomley@steeleye.com>
  *
- * This file is licensed under GPLv2
- *
  * The basic idea here is to allow any "device controller" (which
  * would most often be a Host Bus Adapter to use the services of one
  * or more tranport classes for performing transport specific
diff --git a/drivers/bcma/driver_pcie2.c b/drivers/bcma/driver_pcie2.c
index b1a6e327cb23..cf889fc62ac7 100644
--- a/drivers/bcma/driver_pcie2.c
+++ b/drivers/bcma/driver_pcie2.c
@@ -83,7 +83,8 @@ static void bcma_core_pcie2_hw_ltr_war(struct bcma_drv_pcie2 *pcie2)
 		bcma_core_pcie2_set_ltr_vals(pcie2);
 
 		/* TODO:
-		si_core_wrapperreg(pcie2, 3, 0x60, 0x8080, 0); */
+		 *si_core_wrapperreg(pcie2, 3, 0x60, 0x8080, 0);
+		 */
 
 		/* enable the LTR */
 		devstsctr2 |= PCIE2_CAP_DEVSTSCTRL2_LTRENAB;
diff --git a/drivers/block/DAC960.c b/drivers/block/DAC960.c
index 442e777bdfb2..f781eff7d23e 100644
--- a/drivers/block/DAC960.c
+++ b/drivers/block/DAC960.c
@@ -268,17 +268,17 @@ static bool DAC960_CreateAuxiliaryStructures(DAC960_Controller_T *Controller)
   void *AllocationPointer = NULL;
   void *ScatterGatherCPU = NULL;
   dma_addr_t ScatterGatherDMA;
-  struct pci_pool *ScatterGatherPool;
+  struct dma_pool *ScatterGatherPool;
   void *RequestSenseCPU = NULL;
   dma_addr_t RequestSenseDMA;
-  struct pci_pool *RequestSensePool = NULL;
+  struct dma_pool *RequestSensePool = NULL;
 
   if (Controller->FirmwareType == DAC960_V1_Controller)
     {
       CommandAllocationLength = offsetof(DAC960_Command_T, V1.EndMarker);
       CommandAllocationGroupSize = DAC960_V1_CommandAllocationGroupSize;
-      ScatterGatherPool = pci_pool_create("DAC960_V1_ScatterGather",
-		Controller->PCIDevice,
+      ScatterGatherPool = dma_pool_create("DAC960_V1_ScatterGather",
+		&Controller->PCIDevice->dev,
 	DAC960_V1_ScatterGatherLimit * sizeof(DAC960_V1_ScatterGatherSegment_T),
 	sizeof(DAC960_V1_ScatterGatherSegment_T), 0);
       if (ScatterGatherPool == NULL)
@@ -290,18 +290,18 @@ static bool DAC960_CreateAuxiliaryStructures(DAC960_Controller_T *Controller)
     {
       CommandAllocationLength = offsetof(DAC960_Command_T, V2.EndMarker);
       CommandAllocationGroupSize = DAC960_V2_CommandAllocationGroupSize;
-      ScatterGatherPool = pci_pool_create("DAC960_V2_ScatterGather",
-		Controller->PCIDevice,
+      ScatterGatherPool = dma_pool_create("DAC960_V2_ScatterGather",
+		&Controller->PCIDevice->dev,
 	DAC960_V2_ScatterGatherLimit * sizeof(DAC960_V2_ScatterGatherSegment_T),
 	sizeof(DAC960_V2_ScatterGatherSegment_T), 0);
       if (ScatterGatherPool == NULL)
 	    return DAC960_Failure(Controller,
 			"AUXILIARY STRUCTURE CREATION (SG)");
-      RequestSensePool = pci_pool_create("DAC960_V2_RequestSense",
-		Controller->PCIDevice, sizeof(DAC960_SCSI_RequestSense_T),
+      RequestSensePool = dma_pool_create("DAC960_V2_RequestSense",
+		&Controller->PCIDevice->dev, sizeof(DAC960_SCSI_RequestSense_T),
 		sizeof(int), 0);
       if (RequestSensePool == NULL) {
-	    pci_pool_destroy(ScatterGatherPool);
+	    dma_pool_destroy(ScatterGatherPool);
 	    return DAC960_Failure(Controller,
 			"AUXILIARY STRUCTURE CREATION (SG)");
       }
@@ -335,16 +335,16 @@ static bool DAC960_CreateAuxiliaryStructures(DAC960_Controller_T *Controller)
       Command->Next = Controller->FreeCommands;
       Controller->FreeCommands = Command;
       Controller->Commands[CommandIdentifier-1] = Command;
-      ScatterGatherCPU = pci_pool_alloc(ScatterGatherPool, GFP_ATOMIC,
+      ScatterGatherCPU = dma_pool_alloc(ScatterGatherPool, GFP_ATOMIC,
 							&ScatterGatherDMA);
       if (ScatterGatherCPU == NULL)
 	  return DAC960_Failure(Controller, "AUXILIARY STRUCTURE CREATION");
 
       if (RequestSensePool != NULL) {
-  	  RequestSenseCPU = pci_pool_alloc(RequestSensePool, GFP_ATOMIC,
+	  RequestSenseCPU = dma_pool_alloc(RequestSensePool, GFP_ATOMIC,
 						&RequestSenseDMA);
   	  if (RequestSenseCPU == NULL) {
-                pci_pool_free(ScatterGatherPool, ScatterGatherCPU,
+                dma_pool_free(ScatterGatherPool, ScatterGatherCPU,
                                 ScatterGatherDMA);
     		return DAC960_Failure(Controller,
 					"AUXILIARY STRUCTURE CREATION");
@@ -379,8 +379,8 @@ static bool DAC960_CreateAuxiliaryStructures(DAC960_Controller_T *Controller)
 static void DAC960_DestroyAuxiliaryStructures(DAC960_Controller_T *Controller)
 {
   int i;
-  struct pci_pool *ScatterGatherPool = Controller->ScatterGatherPool;
-  struct pci_pool *RequestSensePool = NULL;
+  struct dma_pool *ScatterGatherPool = Controller->ScatterGatherPool;
+  struct dma_pool *RequestSensePool = NULL;
   void *ScatterGatherCPU;
   dma_addr_t ScatterGatherDMA;
   void *RequestSenseCPU;
@@ -411,9 +411,9 @@ static void DAC960_DestroyAuxiliaryStructures(DAC960_Controller_T *Controller)
 	  RequestSenseDMA = Command->V2.RequestSenseDMA;
       }
       if (ScatterGatherCPU != NULL)
-          pci_pool_free(ScatterGatherPool, ScatterGatherCPU, ScatterGatherDMA);
+          dma_pool_free(ScatterGatherPool, ScatterGatherCPU, ScatterGatherDMA);
       if (RequestSenseCPU != NULL)
-          pci_pool_free(RequestSensePool, RequestSenseCPU, RequestSenseDMA);
+          dma_pool_free(RequestSensePool, RequestSenseCPU, RequestSenseDMA);
 
       if ((Command->CommandIdentifier
 	   % Controller->CommandAllocationGroupSize) == 1) {
@@ -437,13 +437,11 @@ static void DAC960_DestroyAuxiliaryStructures(DAC960_Controller_T *Controller)
       Controller->CurrentStatusBuffer = NULL;
     }
 
-  if (ScatterGatherPool != NULL)
-  	pci_pool_destroy(ScatterGatherPool);
+  dma_pool_destroy(ScatterGatherPool);
   if (Controller->FirmwareType == DAC960_V1_Controller)
   	return;
 
-  if (RequestSensePool != NULL)
-	pci_pool_destroy(RequestSensePool);
+  dma_pool_destroy(RequestSensePool);
 
   for (i = 0; i < DAC960_MaxLogicalDrives; i++) {
 	kfree(Controller->V2.LogicalDeviceInformation[i]);
@@ -6619,43 +6617,27 @@ static void DAC960_DestroyProcEntries(DAC960_Controller_T *Controller)
 
 #ifdef DAC960_GAM_MINOR
 
-/*
- * DAC960_gam_ioctl is the ioctl function for performing RAID operations.
-*/
-
-static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
-						unsigned long Argument)
+static long DAC960_gam_get_controller_info(DAC960_ControllerInfo_T __user *UserSpaceControllerInfo)
 {
-  long ErrorCode = 0;
-  if (!capable(CAP_SYS_ADMIN)) return -EACCES;
-
-  mutex_lock(&DAC960_mutex);
-  switch (Request)
-    {
-    case DAC960_IOCTL_GET_CONTROLLER_COUNT:
-      ErrorCode = DAC960_ControllerCount;
-      break;
-    case DAC960_IOCTL_GET_CONTROLLER_INFO:
-      {
-	DAC960_ControllerInfo_T __user *UserSpaceControllerInfo =
-	  (DAC960_ControllerInfo_T __user *) Argument;
 	DAC960_ControllerInfo_T ControllerInfo;
 	DAC960_Controller_T *Controller;
 	int ControllerNumber;
+	long ErrorCode;
+
 	if (UserSpaceControllerInfo == NULL)
 		ErrorCode = -EINVAL;
 	else ErrorCode = get_user(ControllerNumber,
 			     &UserSpaceControllerInfo->ControllerNumber);
 	if (ErrorCode != 0)
-		break;
+		goto out;
 	ErrorCode = -ENXIO;
 	if (ControllerNumber < 0 ||
 	    ControllerNumber > DAC960_ControllerCount - 1) {
-	  break;
+		goto out;
 	}
 	Controller = DAC960_Controllers[ControllerNumber];
 	if (Controller == NULL)
-		break;
+		goto out;
 	memset(&ControllerInfo, 0, sizeof(DAC960_ControllerInfo_T));
 	ControllerInfo.ControllerNumber = ControllerNumber;
 	ControllerInfo.FirmwareType = Controller->FirmwareType;
@@ -6670,12 +6652,12 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 	strcpy(ControllerInfo.FirmwareVersion, Controller->FirmwareVersion);
 	ErrorCode = (copy_to_user(UserSpaceControllerInfo, &ControllerInfo,
 			     sizeof(DAC960_ControllerInfo_T)) ? -EFAULT : 0);
-	break;
-      }
-    case DAC960_IOCTL_V1_EXECUTE_COMMAND:
-      {
-	DAC960_V1_UserCommand_T __user *UserSpaceUserCommand =
-	  (DAC960_V1_UserCommand_T __user *) Argument;
+out:
+	return ErrorCode;
+}
+
+static long DAC960_gam_v1_execute_command(DAC960_V1_UserCommand_T __user *UserSpaceUserCommand)
+{
 	DAC960_V1_UserCommand_T UserCommand;
 	DAC960_Controller_T *Controller;
 	DAC960_Command_T *Command = NULL;
@@ -6688,39 +6670,41 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 	int ControllerNumber, DataTransferLength;
 	unsigned char *DataTransferBuffer = NULL;
 	dma_addr_t DataTransferBufferDMA;
+        long ErrorCode;
+
 	if (UserSpaceUserCommand == NULL) {
 		ErrorCode = -EINVAL;
-		break;
+		goto out;
 	}
 	if (copy_from_user(&UserCommand, UserSpaceUserCommand,
 				   sizeof(DAC960_V1_UserCommand_T))) {
 		ErrorCode = -EFAULT;
-		break;
+		goto out;
 	}
 	ControllerNumber = UserCommand.ControllerNumber;
     	ErrorCode = -ENXIO;
 	if (ControllerNumber < 0 ||
 	    ControllerNumber > DAC960_ControllerCount - 1)
-	    	break;
+		goto out;
 	Controller = DAC960_Controllers[ControllerNumber];
 	if (Controller == NULL)
-		break;
+		goto out;
 	ErrorCode = -EINVAL;
 	if (Controller->FirmwareType != DAC960_V1_Controller)
-		break;
+		goto out;
 	CommandOpcode = UserCommand.CommandMailbox.Common.CommandOpcode;
 	DataTransferLength = UserCommand.DataTransferLength;
 	if (CommandOpcode & 0x80)
-		break;
+		goto out;
 	if (CommandOpcode == DAC960_V1_DCDB)
 	  {
 	    if (copy_from_user(&DCDB, UserCommand.DCDB,
 			       sizeof(DAC960_V1_DCDB_T))) {
 		ErrorCode = -EFAULT;
-		break;
+		goto out;
 	    }
 	    if (DCDB.Channel >= DAC960_V1_MaxChannels)
-	    		break;
+		goto out;
 	    if (!((DataTransferLength == 0 &&
 		   DCDB.Direction
 		   == DAC960_V1_DCDB_NoDataTransfer) ||
@@ -6730,15 +6714,15 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 		  (DataTransferLength < 0 &&
 		   DCDB.Direction
 		   == DAC960_V1_DCDB_DataTransferSystemToDevice)))
-		   	break;
+			goto out;
 	    if (((DCDB.TransferLengthHigh4 << 16) | DCDB.TransferLength)
 		!= abs(DataTransferLength))
-			break;
+			goto out;
 	    DCDB_IOBUF = pci_alloc_consistent(Controller->PCIDevice,
 			sizeof(DAC960_V1_DCDB_T), &DCDB_IOBUFDMA);
 	    if (DCDB_IOBUF == NULL) {
 	    		ErrorCode = -ENOMEM;
-			break;
+			goto out;
 		}
 	  }
 	ErrorCode = -ENOMEM;
@@ -6748,19 +6732,19 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
                                                        DataTransferLength,
                                                        &DataTransferBufferDMA);
 	    if (DataTransferBuffer == NULL)
-	    	break;
+		goto out;
 	  }
 	else if (DataTransferLength < 0)
 	  {
 	    DataTransferBuffer = pci_alloc_consistent(Controller->PCIDevice,
 				-DataTransferLength, &DataTransferBufferDMA);
 	    if (DataTransferBuffer == NULL)
-	    	break;
+		goto out;
 	    if (copy_from_user(DataTransferBuffer,
 			       UserCommand.DataTransferBuffer,
 			       -DataTransferLength)) {
 		ErrorCode = -EFAULT;
-		break;
+		goto out;
 	    }
 	  }
 	if (CommandOpcode == DAC960_V1_DCDB)
@@ -6837,12 +6821,12 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 	if (DCDB_IOBUF != NULL)
 	  pci_free_consistent(Controller->PCIDevice, sizeof(DAC960_V1_DCDB_T),
 			DCDB_IOBUF, DCDB_IOBUFDMA);
-      	break;
-      }
-    case DAC960_IOCTL_V2_EXECUTE_COMMAND:
-      {
-	DAC960_V2_UserCommand_T __user *UserSpaceUserCommand =
-	  (DAC960_V2_UserCommand_T __user *) Argument;
+	out:
+	return ErrorCode;
+}
+
+static long DAC960_gam_v2_execute_command(DAC960_V2_UserCommand_T __user *UserSpaceUserCommand)
+{
 	DAC960_V2_UserCommand_T UserCommand;
 	DAC960_Controller_T *Controller;
 	DAC960_Command_T *Command = NULL;
@@ -6855,26 +6839,26 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 	dma_addr_t DataTransferBufferDMA;
 	unsigned char *RequestSenseBuffer = NULL;
 	dma_addr_t RequestSenseBufferDMA;
+	long ErrorCode = -EINVAL;
 
-	ErrorCode = -EINVAL;
 	if (UserSpaceUserCommand == NULL)
-		break;
+		goto out;
 	if (copy_from_user(&UserCommand, UserSpaceUserCommand,
 			   sizeof(DAC960_V2_UserCommand_T))) {
 		ErrorCode = -EFAULT;
-		break;
+		goto out;
 	}
 	ErrorCode = -ENXIO;
 	ControllerNumber = UserCommand.ControllerNumber;
 	if (ControllerNumber < 0 ||
 	    ControllerNumber > DAC960_ControllerCount - 1)
-	    	break;
+		goto out;
 	Controller = DAC960_Controllers[ControllerNumber];
 	if (Controller == NULL)
-		break;
+		goto out;
 	if (Controller->FirmwareType != DAC960_V2_Controller){
 		ErrorCode = -EINVAL;
-		break;
+		goto out;
 	}
 	DataTransferLength = UserCommand.DataTransferLength;
     	ErrorCode = -ENOMEM;
@@ -6884,14 +6868,14 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
                                                        DataTransferLength,
                                                        &DataTransferBufferDMA);
 	    if (DataTransferBuffer == NULL)
-	    	break;
+		goto out;
 	  }
 	else if (DataTransferLength < 0)
 	  {
 	    DataTransferBuffer = pci_alloc_consistent(Controller->PCIDevice,
 				-DataTransferLength, &DataTransferBufferDMA);
 	    if (DataTransferBuffer == NULL)
-	    	break;
+		goto out;
 	    if (copy_from_user(DataTransferBuffer,
 			       UserCommand.DataTransferBuffer,
 			       -DataTransferLength)) {
@@ -7001,42 +6985,44 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 	if (RequestSenseBuffer != NULL)
 	  pci_free_consistent(Controller->PCIDevice, RequestSenseLength,
 		RequestSenseBuffer, RequestSenseBufferDMA);
-        break;
-      }
-    case DAC960_IOCTL_V2_GET_HEALTH_STATUS:
-      {
-	DAC960_V2_GetHealthStatus_T __user *UserSpaceGetHealthStatus =
-	  (DAC960_V2_GetHealthStatus_T __user *) Argument;
+out:
+        return ErrorCode;
+}
+
+static long DAC960_gam_v2_get_health_status(DAC960_V2_GetHealthStatus_T __user *UserSpaceGetHealthStatus)
+{
 	DAC960_V2_GetHealthStatus_T GetHealthStatus;
 	DAC960_V2_HealthStatusBuffer_T HealthStatusBuffer;
 	DAC960_Controller_T *Controller;
 	int ControllerNumber;
+	long ErrorCode;
+
 	if (UserSpaceGetHealthStatus == NULL) {
 		ErrorCode = -EINVAL;
-		break;
+		goto out;
 	}
 	if (copy_from_user(&GetHealthStatus, UserSpaceGetHealthStatus,
 			   sizeof(DAC960_V2_GetHealthStatus_T))) {
 		ErrorCode = -EFAULT;
-		break;
+		goto out;
 	}
 	ErrorCode = -ENXIO;
 	ControllerNumber = GetHealthStatus.ControllerNumber;
 	if (ControllerNumber < 0 ||
 	    ControllerNumber > DAC960_ControllerCount - 1)
-		    break;
+		goto out;
 	Controller = DAC960_Controllers[ControllerNumber];
 	if (Controller == NULL)
-		break;
+		goto out;
 	if (Controller->FirmwareType != DAC960_V2_Controller) {
 		ErrorCode = -EINVAL;
-		break;
+		goto out;
 	}
 	if (copy_from_user(&HealthStatusBuffer,
 			   GetHealthStatus.HealthStatusBuffer,
 			   sizeof(DAC960_V2_HealthStatusBuffer_T))) {
 		ErrorCode = -EFAULT;
-		break;
+		goto out;
 	}
 	ErrorCode = wait_event_interruptible_timeout(Controller->HealthStatusWaitQueue,
 			!(Controller->V2.HealthStatusBuffer->StatusChangeCounter
@@ -7046,7 +7032,7 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 			DAC960_MonitoringTimerInterval);
 	if (ErrorCode == -ERESTARTSYS) {
 		ErrorCode = -EINTR;
-		break;
+		goto out;
 	}
 	if (copy_to_user(GetHealthStatus.HealthStatusBuffer,
 			 Controller->V2.HealthStatusBuffer,
@@ -7054,7 +7040,39 @@ static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
 		ErrorCode = -EFAULT;
 	else
 		ErrorCode =  0;
-      }
+
+out:
+	return ErrorCode;
+}
+
+/*
+ * DAC960_gam_ioctl is the ioctl function for performing RAID operations.
+*/
+
+static long DAC960_gam_ioctl(struct file *file, unsigned int Request,
+						unsigned long Argument)
+{
+  long ErrorCode = 0;
+  void __user *argp = (void __user *)Argument;
+  if (!capable(CAP_SYS_ADMIN)) return -EACCES;
+
+  mutex_lock(&DAC960_mutex);
+  switch (Request)
+    {
+    case DAC960_IOCTL_GET_CONTROLLER_COUNT:
+      ErrorCode = DAC960_ControllerCount;
+      break;
+    case DAC960_IOCTL_GET_CONTROLLER_INFO:
+      ErrorCode = DAC960_gam_get_controller_info(argp);
+      break;
+    case DAC960_IOCTL_V1_EXECUTE_COMMAND:
+      ErrorCode = DAC960_gam_v1_execute_command(argp);
+      break;
+    case DAC960_IOCTL_V2_EXECUTE_COMMAND:
+      ErrorCode = DAC960_gam_v2_execute_command(argp);
+      break;
+    case DAC960_IOCTL_V2_GET_HEALTH_STATUS:
+      ErrorCode = DAC960_gam_v2_get_health_status(argp);
       break;
       default:
 	ErrorCode = -ENOTTY;
diff --git a/drivers/block/DAC960.h b/drivers/block/DAC960.h
index 6a6226a2b932..21aff470d268 100644
--- a/drivers/block/DAC960.h
+++ b/drivers/block/DAC960.h
@@ -2316,7 +2316,7 @@ typedef struct DAC960_Controller
   bool SuppressEnclosureMessages;
   struct timer_list MonitoringTimer;
   struct gendisk *disks[DAC960_MaxLogicalDrives];
-  struct pci_pool *ScatterGatherPool;
+  struct dma_pool *ScatterGatherPool;
   DAC960_Command_T *FreeCommands;
   unsigned char *CombinedStatusBuffer;
   unsigned char *CurrentStatusBuffer;
@@ -2429,7 +2429,7 @@ typedef struct DAC960_Controller
       bool NeedDeviceSerialNumberInformation;
       bool StartLogicalDeviceInformationScan;
       bool StartPhysicalDeviceInformationScan;
-      struct pci_pool *RequestSensePool;
+      struct dma_pool *RequestSensePool;
 
       dma_addr_t	FirstCommandMailboxDMA;
       DAC960_V2_CommandMailbox_T *FirstCommandMailbox;
diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig
index 40579d0cb3d1..ad9b687a236a 100644
--- a/drivers/block/Kconfig
+++ b/drivers/block/Kconfig
@@ -20,6 +20,10 @@ config BLK_DEV_NULL_BLK
 	tristate "Null test block driver"
 	select CONFIGFS_FS
 
+config BLK_DEV_NULL_BLK_FAULT_INJECTION
+	bool "Support fault injection for Null test block driver"
+	depends on BLK_DEV_NULL_BLK && FAULT_INJECTION
+
 config BLK_DEV_FD
 	tristate "Normal floppy disk support"
 	depends on ARCH_MAY_HAVE_PC_FDC
diff --git a/drivers/block/amiflop.c b/drivers/block/amiflop.c
index e5aa62fcf5a8..3aaf6af3ec23 100644
--- a/drivers/block/amiflop.c
+++ b/drivers/block/amiflop.c
@@ -1758,7 +1758,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)
 	if (unit[drive].type->code == FD_NODRIVE)
 		return NULL;
 	*part = 0;
-	return get_disk(unit[drive].gendisk);
+	return get_disk_and_module(unit[drive].gendisk);
 }
 
 static int __init amiga_floppy_probe(struct platform_device *pdev)
diff --git a/drivers/block/aoe/aoe.h b/drivers/block/aoe/aoe.h
index 9220f8e833d0..c0ebda1283cc 100644
--- a/drivers/block/aoe/aoe.h
+++ b/drivers/block/aoe/aoe.h
@@ -112,8 +112,7 @@ enum frame_flags {
 struct frame {
 	struct list_head head;
 	u32 tag;
-	struct timeval sent;	/* high-res time packet was sent */
-	u32 sent_jiffs;		/* low-res jiffies-based sent time */
+	ktime_t sent;			/* high-res time packet was sent */
 	ulong waited;
 	ulong waited_total;
 	struct aoetgt *t;		/* parent target I belong to */
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 812fed069708..540bb60cd071 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -398,8 +398,7 @@ aoecmd_ata_rw(struct aoedev *d)
 
 	skb = skb_clone(f->skb, GFP_ATOMIC);
 	if (skb) {
-		do_gettimeofday(&f->sent);
-		f->sent_jiffs = (u32) jiffies;
+		f->sent = ktime_get();
 		__skb_queue_head_init(&queue);
 		__skb_queue_tail(&queue, skb);
 		aoenet_xmit(&queue);
@@ -489,8 +488,7 @@ resend(struct aoedev *d, struct frame *f)
 	skb = skb_clone(skb, GFP_ATOMIC);
 	if (skb == NULL)
 		return;
-	do_gettimeofday(&f->sent);
-	f->sent_jiffs = (u32) jiffies;
+	f->sent = ktime_get();
 	__skb_queue_head_init(&queue);
 	__skb_queue_tail(&queue, skb);
 	aoenet_xmit(&queue);
@@ -499,33 +497,17 @@ resend(struct aoedev *d, struct frame *f)
 static int
 tsince_hr(struct frame *f)
 {
-	struct timeval now;
-	int n;
+	u64 delta = ktime_to_ns(ktime_sub(ktime_get(), f->sent));
 
-	do_gettimeofday(&now);
-	n = now.tv_usec - f->sent.tv_usec;
-	n += (now.tv_sec - f->sent.tv_sec) * USEC_PER_SEC;
+	/* delta is normally under 4.2 seconds, avoid 64-bit division */
+	if (likely(delta <= UINT_MAX))
+		return (u32)delta / NSEC_PER_USEC;
 
-	if (n < 0)
-		n = -n;
+	/* avoid overflow after 71 minutes */
+	if (delta > ((u64)INT_MAX * NSEC_PER_USEC))
+		return INT_MAX;
 
-	/* For relatively long periods, use jiffies to avoid
-	 * discrepancies caused by updates to the system time.
-	 *
-	 * On system with HZ of 1000, 32-bits is over 49 days
-	 * worth of jiffies, or over 71 minutes worth of usecs.
-	 *
-	 * Jiffies overflow is handled by subtraction of unsigned ints:
-	 * (gdb) print (unsigned) 2 - (unsigned) 0xfffffffe
-	 * $3 = 4
-	 * (gdb)
-	 */
-	if (n > USEC_PER_SEC / 4) {
-		n = ((u32) jiffies) - f->sent_jiffs;
-		n *= USEC_PER_SEC / HZ;
-	}
-
-	return n;
+	return div_u64(delta, NSEC_PER_USEC);
 }
 
 static int
@@ -589,7 +571,6 @@ reassign_frame(struct frame *f)
 	nf->waited = 0;
 	nf->waited_total = f->waited_total;
 	nf->sent = f->sent;
-	nf->sent_jiffs = f->sent_jiffs;
 	f->skb = skb;
 
 	return nf;
@@ -633,8 +614,7 @@ probe(struct aoetgt *t)
 
 	skb = skb_clone(f->skb, GFP_ATOMIC);
 	if (skb) {
-		do_gettimeofday(&f->sent);
-		f->sent_jiffs = (u32) jiffies;
+		f->sent = ktime_get();
 		__skb_queue_head_init(&queue);
 		__skb_queue_tail(&queue, skb);
 		aoenet_xmit(&queue);
@@ -1432,10 +1412,8 @@ aoecmd_ata_id(struct aoedev *d)
 	d->timer.function = rexmit_timer;
 
 	skb = skb_clone(skb, GFP_ATOMIC);
-	if (skb) {
-		do_gettimeofday(&f->sent);
-		f->sent_jiffs = (u32) jiffies;
-	}
+	if (skb)
+		f->sent = ktime_get();
 
 	return skb;
 }
diff --git a/drivers/block/ataflop.c b/drivers/block/ataflop.c
index 8bc3b9fd8dd2..dfb2c2622e5a 100644
--- a/drivers/block/ataflop.c
+++ b/drivers/block/ataflop.c
@@ -1917,7 +1917,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)
 	if (drive >= FD_MAX_UNITS || type > NUM_DISK_MINORS)
 		return NULL;
 	*part = 0;
-	return get_disk(unit[drive].disk);
+	return get_disk_and_module(unit[drive].disk);
 }
 
 static int __init atari_floppy_init (void)
diff --git a/drivers/block/brd.c b/drivers/block/brd.c
index 8028a3a7e7fd..deea78e485da 100644
--- a/drivers/block/brd.c
+++ b/drivers/block/brd.c
@@ -456,7 +456,7 @@ static struct kobject *brd_probe(dev_t dev, int *part, void *data)
 
 	mutex_lock(&brd_devices_mutex);
 	brd = brd_init_one(MINOR(dev) / max_part, &new);
-	kobj = brd ? get_disk(brd->brd_disk) : NULL;
+	kobj = brd ? get_disk_and_module(brd->brd_disk) : NULL;
 	mutex_unlock(&brd_devices_mutex);
 
 	if (new)
diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c
index bd97908c766f..9f4e6f502b84 100644
--- a/drivers/block/drbd/drbd_bitmap.c
+++ b/drivers/block/drbd/drbd_bitmap.c
@@ -953,7 +953,7 @@ static void drbd_bm_endio(struct bio *bio)
 	struct drbd_bm_aio_ctx *ctx = bio->bi_private;
 	struct drbd_device *device = ctx->device;
 	struct drbd_bitmap *b = device->bitmap;
-	unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page);
+	unsigned int idx = bm_page_to_idx(bio_first_page_all(bio));
 
 	if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 &&
 	    !bm_test_page_unchanged(b->bm_pages[idx]))
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 4b4697a1f963..0a0394aa1b9c 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -1847,7 +1847,7 @@ int drbd_send(struct drbd_connection *connection, struct socket *sock,
 	      void *buf, size_t size, unsigned msg_flags)
 {
 	struct kvec iov = {.iov_base = buf, .iov_len = size};
-	struct msghdr msg;
+	struct msghdr msg = {.msg_flags = msg_flags | MSG_NOSIGNAL};
 	int rv, sent = 0;
 
 	if (!sock)
@@ -1855,12 +1855,6 @@ int drbd_send(struct drbd_connection *connection, struct socket *sock,
 
 	/* THINK  if (signal_pending) return ... ? */
 
-	msg.msg_name       = NULL;
-	msg.msg_namelen    = 0;
-	msg.msg_control    = NULL;
-	msg.msg_controllen = 0;
-	msg.msg_flags      = msg_flags | MSG_NOSIGNAL;
-
 	iov_iter_kvec(&msg.msg_iter, WRITE | ITER_KVEC, &iov, 1, size);
 
 	if (sock == connection->data.socket) {
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index cb2fa63f6bc0..c72dee0ef083 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -516,7 +516,8 @@ static int drbd_recv_short(struct socket *sock, void *buf, size_t size, int flag
 	struct msghdr msg = {
 		.msg_flags = (flags ? flags : MSG_WAITALL | MSG_NOSIGNAL)
 	};
-	return kernel_recvmsg(sock, &msg, &iov, 1, size, msg.msg_flags);
+	iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC, &iov, 1, size);
+	return sock_recvmsg(sock, &msg, msg.msg_flags);
 }
 
 static int drbd_recv(struct drbd_connection *connection, void *buf, size_t size)
diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c
index eae484acfbbc..8ec7235fc93b 100644
--- a/drivers/block/floppy.c
+++ b/drivers/block/floppy.c
@@ -4505,7 +4505,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)
 	if (((*part >> 2) & 0x1f) >= ARRAY_SIZE(floppy_type))
 		return NULL;
 	*part = 0;
-	return get_disk(disks[drive]);
+	return get_disk_and_module(disks[drive]);
 }
 
 static int __init do_floppy_init(void)
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index d5fe720cf149..ee62d2d517bf 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -266,7 +266,7 @@ static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
 	struct iov_iter i;
 	ssize_t bw;
 
-	iov_iter_bvec(&i, ITER_BVEC, bvec, 1, bvec->bv_len);
+	iov_iter_bvec(&i, ITER_BVEC | WRITE, bvec, 1, bvec->bv_len);
 
 	file_start_write(file);
 	bw = vfs_iter_write(file, &i, ppos, 0);
@@ -1922,7 +1922,7 @@ static struct kobject *loop_probe(dev_t dev, int *part, void *data)
 	if (err < 0)
 		kobj = NULL;
 	else
-		kobj = get_disk(lo->lo_disk);
+		kobj = get_disk_and_module(lo->lo_disk);
 	mutex_unlock(&loop_index_mutex);
 
 	*part = 0;
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 5f2a4240a204..86258b00a1d4 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -1591,7 +1591,7 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info)
 			if (new_index < 0) {
 				mutex_unlock(&nbd_index_mutex);
 				printk(KERN_ERR "nbd: failed to add new device\n");
-				return ret;
+				return new_index;
 			}
 			nbd = idr_find(&nbd_index_idr, new_index);
 		}
diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c
index ad0477ae820f..287a09611c0f 100644
--- a/drivers/block/null_blk.c
+++ b/drivers/block/null_blk.c
@@ -12,9 +12,9 @@
 #include <linux/slab.h>
 #include <linux/blk-mq.h>
 #include <linux/hrtimer.h>
-#include <linux/lightnvm.h>
 #include <linux/configfs.h>
 #include <linux/badblocks.h>
+#include <linux/fault-inject.h>
 
 #define SECTOR_SHIFT		9
 #define PAGE_SECTORS_SHIFT	(PAGE_SHIFT - SECTOR_SHIFT)
@@ -27,6 +27,10 @@
 #define TICKS_PER_SEC		50ULL
 #define TIMER_INTERVAL		(NSEC_PER_SEC / TICKS_PER_SEC)
 
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+static DECLARE_FAULT_ATTR(null_timeout_attr);
+#endif
+
 static inline u64 mb_per_tick(int mbps)
 {
 	return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
@@ -107,7 +111,6 @@ struct nullb_device {
 	unsigned int hw_queue_depth; /* queue depth */
 	unsigned int index; /* index of the disk, only valid with a disk */
 	unsigned int mbps; /* Bandwidth throttle cap (in MB/s) */
-	bool use_lightnvm; /* register as a LightNVM device */
 	bool blocking; /* blocking blk-mq device */
 	bool use_per_node_hctx; /* use per-node allocation for hardware context */
 	bool power; /* power on/off the device */
@@ -121,7 +124,6 @@ struct nullb {
 	unsigned int index;
 	struct request_queue *q;
 	struct gendisk *disk;
-	struct nvm_dev *ndev;
 	struct blk_mq_tag_set *tag_set;
 	struct blk_mq_tag_set __tag_set;
 	unsigned int queue_depth;
@@ -139,7 +141,6 @@ static LIST_HEAD(nullb_list);
 static struct mutex lock;
 static int null_major;
 static DEFINE_IDA(nullb_indexes);
-static struct kmem_cache *ppa_cache;
 static struct blk_mq_tag_set tag_set;
 
 enum {
@@ -166,6 +167,11 @@ static int g_home_node = NUMA_NO_NODE;
 module_param_named(home_node, g_home_node, int, S_IRUGO);
 MODULE_PARM_DESC(home_node, "Home node for the device");
 
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+static char g_timeout_str[80];
+module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), S_IRUGO);
+#endif
+
 static int g_queue_mode = NULL_Q_MQ;
 
 static int null_param_store_val(const char *str, int *val, int min, int max)
@@ -208,10 +214,6 @@ static int nr_devices = 1;
 module_param(nr_devices, int, S_IRUGO);
 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 
-static bool g_use_lightnvm;
-module_param_named(use_lightnvm, g_use_lightnvm, bool, S_IRUGO);
-MODULE_PARM_DESC(use_lightnvm, "Register as a LightNVM device");
-
 static bool g_blocking;
 module_param_named(blocking, g_blocking, bool, S_IRUGO);
 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
@@ -345,7 +347,6 @@ NULLB_DEVICE_ATTR(blocksize, uint);
 NULLB_DEVICE_ATTR(irqmode, uint);
 NULLB_DEVICE_ATTR(hw_queue_depth, uint);
 NULLB_DEVICE_ATTR(index, uint);
-NULLB_DEVICE_ATTR(use_lightnvm, bool);
 NULLB_DEVICE_ATTR(blocking, bool);
 NULLB_DEVICE_ATTR(use_per_node_hctx, bool);
 NULLB_DEVICE_ATTR(memory_backed, bool);
@@ -455,7 +456,6 @@ static struct configfs_attribute *nullb_device_attrs[] = {
 	&nullb_device_attr_irqmode,
 	&nullb_device_attr_hw_queue_depth,
 	&nullb_device_attr_index,
-	&nullb_device_attr_use_lightnvm,
 	&nullb_device_attr_blocking,
 	&nullb_device_attr_use_per_node_hctx,
 	&nullb_device_attr_power,
@@ -573,7 +573,6 @@ static struct nullb_device *null_alloc_dev(void)
 	dev->blocksize = g_bs;
 	dev->irqmode = g_irqmode;
 	dev->hw_queue_depth = g_hw_queue_depth;
-	dev->use_lightnvm = g_use_lightnvm;
 	dev->blocking = g_blocking;
 	dev->use_per_node_hctx = g_use_per_node_hctx;
 	return dev;
@@ -1231,7 +1230,7 @@ static blk_status_t null_handle_cmd(struct nullb_cmd *cmd)
 				return BLK_STS_OK;
 			} else
 				/* requeue request */
-				return BLK_STS_RESOURCE;
+				return BLK_STS_DEV_RESOURCE;
 		}
 	}
 
@@ -1352,6 +1351,12 @@ static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
 	return BLK_QC_T_NONE;
 }
 
+static enum blk_eh_timer_return null_rq_timed_out_fn(struct request *rq)
+{
+	pr_info("null: rq %p timed out\n", rq);
+	return BLK_EH_HANDLED;
+}
+
 static int null_rq_prep_fn(struct request_queue *q, struct request *req)
 {
 	struct nullb *nullb = q->queuedata;
@@ -1369,6 +1374,16 @@ static int null_rq_prep_fn(struct request_queue *q, struct request *req)
 	return BLKPREP_DEFER;
 }
 
+static bool should_timeout_request(struct request *rq)
+{
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+	if (g_timeout_str[0])
+		return should_fail(&null_timeout_attr, 1);
+#endif
+
+	return false;
+}
+
 static void null_request_fn(struct request_queue *q)
 {
 	struct request *rq;
@@ -1376,12 +1391,20 @@ static void null_request_fn(struct request_queue *q)
 	while ((rq = blk_fetch_request(q)) != NULL) {
 		struct nullb_cmd *cmd = rq->special;
 
-		spin_unlock_irq(q->queue_lock);
-		null_handle_cmd(cmd);
-		spin_lock_irq(q->queue_lock);
+		if (!should_timeout_request(rq)) {
+			spin_unlock_irq(q->queue_lock);
+			null_handle_cmd(cmd);
+			spin_lock_irq(q->queue_lock);
+		}
 	}
 }
 
+static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
+{
+	pr_info("null: rq %p timed out\n", rq);
+	return BLK_EH_HANDLED;
+}
+
 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
 			 const struct blk_mq_queue_data *bd)
 {
@@ -1399,12 +1422,16 @@ static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
 
 	blk_mq_start_request(bd->rq);
 
-	return null_handle_cmd(cmd);
+	if (!should_timeout_request(bd->rq))
+		return null_handle_cmd(cmd);
+
+	return BLK_STS_OK;
 }
 
 static const struct blk_mq_ops null_mq_ops = {
 	.queue_rq       = null_queue_rq,
 	.complete	= null_softirq_done_fn,
+	.timeout	= null_timeout_rq,
 };
 
 static void cleanup_queue(struct nullb_queue *nq)
@@ -1423,170 +1450,6 @@ static void cleanup_queues(struct nullb *nullb)
 	kfree(nullb->queues);
 }
 
-#ifdef CONFIG_NVM
-
-static void null_lnvm_end_io(struct request *rq, blk_status_t status)
-{
-	struct nvm_rq *rqd = rq->end_io_data;
-
-	/* XXX: lighnvm core seems to expect NVM_RSP_* values here.. */
-	rqd->error = status ? -EIO : 0;
-	nvm_end_io(rqd);
-
-	blk_put_request(rq);
-}
-
-static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
-{
-	struct request_queue *q = dev->q;
-	struct request *rq;
-	struct bio *bio = rqd->bio;
-
-	rq = blk_mq_alloc_request(q,
-		op_is_write(bio_op(bio)) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
-	if (IS_ERR(rq))
-		return -ENOMEM;
-
-	blk_init_request_from_bio(rq, bio);
-
-	rq->end_io_data = rqd;
-
-	blk_execute_rq_nowait(q, NULL, rq, 0, null_lnvm_end_io);
-
-	return 0;
-}
-
-static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
-{
-	struct nullb *nullb = dev->q->queuedata;
-	sector_t size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
-	sector_t blksize;
-	struct nvm_id_group *grp;
-
-	id->ver_id = 0x1;
-	id->vmnt = 0;
-	id->cap = 0x2;
-	id->dom = 0x1;
-
-	id->ppaf.blk_offset = 0;
-	id->ppaf.blk_len = 16;
-	id->ppaf.pg_offset = 16;
-	id->ppaf.pg_len = 16;
-	id->ppaf.sect_offset = 32;
-	id->ppaf.sect_len = 8;
-	id->ppaf.pln_offset = 40;
-	id->ppaf.pln_len = 8;
-	id->ppaf.lun_offset = 48;
-	id->ppaf.lun_len = 8;
-	id->ppaf.ch_offset = 56;
-	id->ppaf.ch_len = 8;
-
-	sector_div(size, nullb->dev->blocksize); /* convert size to pages */
-	size >>= 8; /* concert size to pgs pr blk */
-	grp = &id->grp;
-	grp->mtype = 0;
-	grp->fmtype = 0;
-	grp->num_ch = 1;
-	grp->num_pg = 256;
-	blksize = size;
-	size >>= 16;
-	grp->num_lun = size + 1;
-	sector_div(blksize, grp->num_lun);
-	grp->num_blk = blksize;
-	grp->num_pln = 1;
-
-	grp->fpg_sz = nullb->dev->blocksize;
-	grp->csecs = nullb->dev->blocksize;
-	grp->trdt = 25000;
-	grp->trdm = 25000;
-	grp->tprt = 500000;
-	grp->tprm = 500000;
-	grp->tbet = 1500000;
-	grp->tbem = 1500000;
-	grp->mpos = 0x010101; /* single plane rwe */
-	grp->cpar = nullb->dev->hw_queue_depth;
-
-	return 0;
-}
-
-static void *null_lnvm_create_dma_pool(struct nvm_dev *dev, char *name)
-{
-	mempool_t *virtmem_pool;
-
-	virtmem_pool = mempool_create_slab_pool(64, ppa_cache);
-	if (!virtmem_pool) {
-		pr_err("null_blk: Unable to create virtual memory pool\n");
-		return NULL;
-	}
-
-	return virtmem_pool;
-}
-
-static void null_lnvm_destroy_dma_pool(void *pool)
-{
-	mempool_destroy(pool);
-}
-
-static void *null_lnvm_dev_dma_alloc(struct nvm_dev *dev, void *pool,
-				gfp_t mem_flags, dma_addr_t *dma_handler)
-{
-	return mempool_alloc(pool, mem_flags);
-}
-
-static void null_lnvm_dev_dma_free(void *pool, void *entry,
-							dma_addr_t dma_handler)
-{
-	mempool_free(entry, pool);
-}
-
-static struct nvm_dev_ops null_lnvm_dev_ops = {
-	.identity		= null_lnvm_id,
-	.submit_io		= null_lnvm_submit_io,
-
-	.create_dma_pool	= null_lnvm_create_dma_pool,
-	.destroy_dma_pool	= null_lnvm_destroy_dma_pool,
-	.dev_dma_alloc		= null_lnvm_dev_dma_alloc,
-	.dev_dma_free		= null_lnvm_dev_dma_free,
-
-	/* Simulate nvme protocol restriction */
-	.max_phys_sect		= 64,
-};
-
-static int null_nvm_register(struct nullb *nullb)
-{
-	struct nvm_dev *dev;
-	int rv;
-
-	dev = nvm_alloc_dev(0);
-	if (!dev)
-		return -ENOMEM;
-
-	dev->q = nullb->q;
-	memcpy(dev->name, nullb->disk_name, DISK_NAME_LEN);
-	dev->ops = &null_lnvm_dev_ops;
-
-	rv = nvm_register(dev);
-	if (rv) {
-		kfree(dev);
-		return rv;
-	}
-	nullb->ndev = dev;
-	return 0;
-}
-
-static void null_nvm_unregister(struct nullb *nullb)
-{
-	nvm_unregister(nullb->ndev);
-}
-#else
-static int null_nvm_register(struct nullb *nullb)
-{
-	pr_err("null_blk: CONFIG_NVM needs to be enabled for LightNVM\n");
-	return -EINVAL;
-}
-static void null_nvm_unregister(struct nullb *nullb) {}
-#endif /* CONFIG_NVM */
-
 static void null_del_dev(struct nullb *nullb)
 {
 	struct nullb_device *dev = nullb->dev;
@@ -1595,10 +1458,7 @@ static void null_del_dev(struct nullb *nullb)
 
 	list_del_init(&nullb->list);
 
-	if (dev->use_lightnvm)
-		null_nvm_unregister(nullb);
-	else
-		del_gendisk(nullb->disk);
+	del_gendisk(nullb->disk);
 
 	if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
 		hrtimer_cancel(&nullb->bw_timer);
@@ -1610,8 +1470,7 @@ static void null_del_dev(struct nullb *nullb)
 	if (dev->queue_mode == NULL_Q_MQ &&
 	    nullb->tag_set == &nullb->__tag_set)
 		blk_mq_free_tag_set(nullb->tag_set);
-	if (!dev->use_lightnvm)
-		put_disk(nullb->disk);
+	put_disk(nullb->disk);
 	cleanup_queues(nullb);
 	if (null_cache_active(nullb))
 		null_free_device_storage(nullb->dev, true);
@@ -1775,11 +1634,6 @@ static void null_validate_conf(struct nullb_device *dev)
 {
 	dev->blocksize = round_down(dev->blocksize, 512);
 	dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
-	if (dev->use_lightnvm && dev->blocksize != 4096)
-		dev->blocksize = 4096;
-
-	if (dev->use_lightnvm && dev->queue_mode != NULL_Q_MQ)
-		dev->queue_mode = NULL_Q_MQ;
 
 	if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
 		if (dev->submit_queues != nr_online_nodes)
@@ -1805,6 +1659,20 @@ static void null_validate_conf(struct nullb_device *dev)
 		dev->mbps = 0;
 }
 
+static bool null_setup_fault(void)
+{
+#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
+	if (!g_timeout_str[0])
+		return true;
+
+	if (!setup_fault_attr(&null_timeout_attr, g_timeout_str))
+		return false;
+
+	null_timeout_attr.verbose = 0;
+#endif
+	return true;
+}
+
 static int null_add_dev(struct nullb_device *dev)
 {
 	struct nullb *nullb;
@@ -1838,6 +1706,10 @@ static int null_add_dev(struct nullb_device *dev)
 		if (rv)
 			goto out_cleanup_queues;
 
+		if (!null_setup_fault())
+			goto out_cleanup_queues;
+
+		nullb->tag_set->timeout = 5 * HZ;
 		nullb->q = blk_mq_init_queue(nullb->tag_set);
 		if (IS_ERR(nullb->q)) {
 			rv = -ENOMEM;
@@ -1861,8 +1733,14 @@ static int null_add_dev(struct nullb_device *dev)
 			rv = -ENOMEM;
 			goto out_cleanup_queues;
 		}
+
+		if (!null_setup_fault())
+			goto out_cleanup_blk_queue;
+
 		blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
 		blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
+		blk_queue_rq_timed_out(nullb->q, null_rq_timed_out_fn);
+		nullb->q->rq_timeout = 5 * HZ;
 		rv = init_driver_queues(nullb);
 		if (rv)
 			goto out_cleanup_blk_queue;
@@ -1895,11 +1773,7 @@ static int null_add_dev(struct nullb_device *dev)
 
 	sprintf(nullb->disk_name, "nullb%d", nullb->index);
 
-	if (dev->use_lightnvm)
-		rv = null_nvm_register(nullb);
-	else
-		rv = null_gendisk_register(nullb);
-
+	rv = null_gendisk_register(nullb);
 	if (rv)
 		goto out_cleanup_blk_queue;
 
@@ -1938,18 +1812,6 @@ static int __init null_init(void)
 		g_bs = PAGE_SIZE;
 	}
 
-	if (g_use_lightnvm && g_bs != 4096) {
-		pr_warn("null_blk: LightNVM only supports 4k block size\n");
-		pr_warn("null_blk: defaults block size to 4k\n");
-		g_bs = 4096;
-	}
-
-	if (g_use_lightnvm && g_queue_mode != NULL_Q_MQ) {
-		pr_warn("null_blk: LightNVM only supported for blk-mq\n");
-		pr_warn("null_blk: defaults queue mode to blk-mq\n");
-		g_queue_mode = NULL_Q_MQ;
-	}
-
 	if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
 		if (g_submit_queues != nr_online_nodes) {
 			pr_warn("null_blk: submit_queues param is set to %u.\n",
@@ -1982,16 +1844,6 @@ static int __init null_init(void)
 		goto err_conf;
 	}
 
-	if (g_use_lightnvm) {
-		ppa_cache = kmem_cache_create("ppa_cache", 64 * sizeof(u64),
-								0, 0, NULL);
-		if (!ppa_cache) {
-			pr_err("null_blk: unable to create ppa cache\n");
-			ret = -ENOMEM;
-			goto err_ppa;
-		}
-	}
-
 	for (i = 0; i < nr_devices; i++) {
 		dev = null_alloc_dev();
 		if (!dev) {
@@ -2015,8 +1867,6 @@ static int __init null_init(void)
 		null_del_dev(nullb);
 		null_free_dev(dev);
 	}
-	kmem_cache_destroy(ppa_cache);
-err_ppa:
 	unregister_blkdev(null_major, "nullb");
 err_conf:
 	configfs_unregister_subsystem(&nullb_subsys);
@@ -2047,8 +1897,6 @@ static void __exit null_exit(void)
 
 	if (g_queue_mode == NULL_Q_MQ && shared_tags)
 		blk_mq_free_tag_set(&tag_set);
-
-	kmem_cache_destroy(ppa_cache);
 }
 
 module_init(null_init);
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 67974796c350..c61d20c9f3f8 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -1122,7 +1122,7 @@ static int pkt_start_recovery(struct packet_data *pkt)
 	pkt->sector = new_sector;
 
 	bio_reset(pkt->bio);
-	bio_set_set(pkt->bio, pd->bdev);
+	bio_set_dev(pkt->bio, pd->bdev);
 	bio_set_op_attrs(pkt->bio, REQ_OP_WRITE, 0);
 	pkt->bio->bi_iter.bi_sector = new_sector;
 	pkt->bio->bi_iter.bi_size = pkt->frames * CD_FRAMESIZE;
@@ -2579,14 +2579,14 @@ static int pkt_new_dev(struct pktcdvd_device *pd, dev_t dev)
 	bdev = bdget(dev);
 	if (!bdev)
 		return -ENOMEM;
+	ret = blkdev_get(bdev, FMODE_READ | FMODE_NDELAY, NULL);
+	if (ret)
+		return ret;
 	if (!blk_queue_scsi_passthrough(bdev_get_queue(bdev))) {
 		WARN_ONCE(true, "Attempt to register a non-SCSI queue\n");
-		bdput(bdev);
+		blkdev_put(bdev, FMODE_READ | FMODE_NDELAY);
 		return -EINVAL;
 	}
-	ret = blkdev_get(bdev, FMODE_READ | FMODE_NDELAY, NULL);
-	if (ret)
-		return ret;
 
 	/* This is safe, since we have a reference from open(). */
 	__module_get(THIS_MODULE);
@@ -2745,7 +2745,7 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev)
 	pd->pkt_dev = MKDEV(pktdev_major, idx);
 	ret = pkt_new_dev(pd, dev);
 	if (ret)
-		goto out_new_dev;
+		goto out_mem2;
 
 	/* inherit events of the host device */
 	disk->events = pd->bdev->bd_disk->events;
@@ -2763,8 +2763,6 @@ static int pkt_setup_dev(dev_t dev, dev_t* pkt_dev)
 	mutex_unlock(&ctl_mutex);
 	return 0;
 
-out_new_dev:
-	blk_cleanup_queue(disk->queue);
 out_mem2:
 	put_disk(disk);
 out_mem:
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index cc93522a6d41..8e40da093766 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -124,11 +124,13 @@ static int atomic_dec_return_safe(atomic_t *v)
 #define RBD_FEATURE_STRIPINGV2		(1ULL<<1)
 #define RBD_FEATURE_EXCLUSIVE_LOCK	(1ULL<<2)
 #define RBD_FEATURE_DATA_POOL		(1ULL<<7)
+#define RBD_FEATURE_OPERATIONS		(1ULL<<8)
 
 #define RBD_FEATURES_ALL	(RBD_FEATURE_LAYERING |		\
 				 RBD_FEATURE_STRIPINGV2 |	\
 				 RBD_FEATURE_EXCLUSIVE_LOCK |	\
-				 RBD_FEATURE_DATA_POOL)
+				 RBD_FEATURE_DATA_POOL |	\
+				 RBD_FEATURE_OPERATIONS)
 
 /* Features supported by this (client software) implementation. */
 
@@ -281,7 +283,6 @@ struct rbd_obj_request {
 	int			result;
 
 	rbd_obj_callback_t	callback;
-	struct completion	completion;
 
 	struct kref		kref;
 };
@@ -1734,10 +1735,7 @@ static void rbd_obj_request_complete(struct rbd_obj_request *obj_request)
 {
 	dout("%s: obj %p cb %p\n", __func__, obj_request,
 		obj_request->callback);
-	if (obj_request->callback)
-		obj_request->callback(obj_request);
-	else
-		complete_all(&obj_request->completion);
+	obj_request->callback(obj_request);
 }
 
 static void rbd_obj_request_error(struct rbd_obj_request *obj_request, int err)
@@ -2013,7 +2011,6 @@ rbd_obj_request_create(enum obj_request_type type)
 	obj_request->which = BAD_WHICH;
 	obj_request->type = type;
 	INIT_LIST_HEAD(&obj_request->links);
-	init_completion(&obj_request->completion);
 	kref_init(&obj_request->kref);
 
 	dout("%s %p\n", __func__, obj_request);
@@ -2129,15 +2126,13 @@ static struct rbd_img_request *rbd_img_request_create(
 {
 	struct rbd_img_request *img_request;
 
-	img_request = kmem_cache_alloc(rbd_img_request_cache, GFP_NOIO);
+	img_request = kmem_cache_zalloc(rbd_img_request_cache, GFP_NOIO);
 	if (!img_request)
 		return NULL;
 
-	img_request->rq = NULL;
 	img_request->rbd_dev = rbd_dev;
 	img_request->offset = offset;
 	img_request->length = length;
-	img_request->flags = 0;
 	if (op_type == OBJ_OP_DISCARD) {
 		img_request_discard_set(img_request);
 		img_request->snapc = snapc;
@@ -2149,11 +2144,8 @@ static struct rbd_img_request *rbd_img_request_create(
 	}
 	if (rbd_dev_parent_get(rbd_dev))
 		img_request_layered_set(img_request);
+
 	spin_lock_init(&img_request->completion_lock);
-	img_request->next_completion = 0;
-	img_request->callback = NULL;
-	img_request->result = 0;
-	img_request->obj_request_count = 0;
 	INIT_LIST_HEAD(&img_request->obj_requests);
 	kref_init(&img_request->kref);
 
@@ -2692,8 +2684,6 @@ static int rbd_img_obj_parent_read_full(struct rbd_obj_request *obj_request)
 
 	parent_request->copyup_pages = NULL;
 	parent_request->copyup_page_count = 0;
-	parent_request->obj_request = NULL;
-	rbd_obj_request_put(obj_request);
 out_err:
 	if (pages)
 		ceph_release_page_vector(pages, page_count);
diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c
index de0d08133c7e..e41935ab41ef 100644
--- a/drivers/block/skd_main.c
+++ b/drivers/block/skd_main.c
@@ -32,7 +32,6 @@
 #include <linux/aer.h>
 #include <linux/wait.h>
 #include <linux/stringify.h>
-#include <linux/slab_def.h>
 #include <scsi/scsi.h>
 #include <scsi/sg.h>
 #include <linux/io.h>
@@ -2603,7 +2602,8 @@ static void *skd_alloc_dma(struct skd_device *skdev, struct kmem_cache *s,
 	buf = kmem_cache_alloc(s, gfp);
 	if (!buf)
 		return NULL;
-	*dma_handle = dma_map_single(dev, buf, s->size, dir);
+	*dma_handle = dma_map_single(dev, buf,
+				     kmem_cache_size(s), dir);
 	if (dma_mapping_error(dev, *dma_handle)) {
 		kmem_cache_free(s, buf);
 		buf = NULL;
@@ -2618,7 +2618,8 @@ static void skd_free_dma(struct skd_device *skdev, struct kmem_cache *s,
 	if (!vaddr)
 		return;
 
-	dma_unmap_single(&skdev->pdev->dev, dma_handle, s->size, dir);
+	dma_unmap_single(&skdev->pdev->dev, dma_handle,
+			 kmem_cache_size(s), dir);
 	kmem_cache_free(s, vaddr);
 }
 
diff --git a/drivers/block/smart1,2.h b/drivers/block/smart1,2.h
deleted file mode 100644
index e5565fbaeb30..000000000000
--- a/drivers/block/smart1,2.h
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- *    Disk Array driver for Compaq SMART2 Controllers
- *    Copyright 1998 Compaq Computer Corporation
- *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; either version 2 of the License, or
- *    (at your option) any later version.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- *    NON INFRINGEMENT.  See the GNU General Public License for more details.
- *
- *    You should have received a copy of the GNU General Public License
- *    along with this program; if not, write to the Free Software
- *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- *
- *    Questions/Comments/Bugfixes to iss_storagedev@hp.com
- *
- *    If you want to make changes, improve or add functionality to this
- *    driver, you'll probably need the Compaq Array Controller Interface
- *    Specificiation (Document number ECG086/1198)
- */
-
-/*
- * This file contains the controller communication implementation for
- * Compaq SMART-1 and SMART-2 controllers.  To the best of my knowledge,
- * this should support:
- *
- *  PCI:
- *  SMART-2/P, SMART-2DH, SMART-2SL, SMART-221, SMART-3100ES, SMART-3200
- *  Integerated SMART Array Controller, SMART-4200, SMART-4250ES
- *
- *  EISA:
- *  SMART-2/E, SMART, IAES, IDA-2, IDA
- */
-
-/*
- * Memory mapped FIFO interface (SMART 42xx cards)
- */
-static void smart4_submit_command(ctlr_info_t *h, cmdlist_t *c)
-{
-        writel(c->busaddr, h->vaddr + S42XX_REQUEST_PORT_OFFSET);
-}
-
-/*  
- *  This card is the opposite of the other cards.  
- *   0 turns interrupts on... 
- *   0x08 turns them off... 
- */
-static void smart4_intr_mask(ctlr_info_t *h, unsigned long val)
-{
-	if (val) 
-	{ /* Turn interrupts on */
-		writel(0, h->vaddr + S42XX_REPLY_INTR_MASK_OFFSET);
-	} else /* Turn them off */
-	{
-        	writel( S42XX_INTR_OFF, 
-			h->vaddr + S42XX_REPLY_INTR_MASK_OFFSET);
-	}
-}
-
-/*
- *  For older cards FIFO Full = 0. 
- *  On this card 0 means there is room, anything else FIFO Full. 
- * 
- */ 
-static unsigned long smart4_fifo_full(ctlr_info_t *h)
-{
-	
-        return (!readl(h->vaddr + S42XX_REQUEST_PORT_OFFSET));
-}
-
-/* This type of controller returns -1 if the fifo is empty, 
- *    Not 0 like the others.
- *    And we need to let it know we read a value out 
- */ 
-static unsigned long smart4_completed(ctlr_info_t *h)
-{
-	long register_value 
-		= readl(h->vaddr + S42XX_REPLY_PORT_OFFSET);
-
-	/* Fifo is empty */
-	if( register_value == 0xffffffff)
-		return 0; 	
-
-	/* Need to let it know we got the reply */
-	/* We do this by writing a 0 to the port we just read from */
-	writel(0, h->vaddr + S42XX_REPLY_PORT_OFFSET);
-
-	return ((unsigned long) register_value); 
-}
-
- /*
- *  This hardware returns interrupt pending at a different place and 
- *  it does not tell us if the fifo is empty, we will have check  
- *  that by getting a 0 back from the command_completed call. 
- */
-static unsigned long smart4_intr_pending(ctlr_info_t *h)
-{
-	unsigned long register_value  = 
-		readl(h->vaddr + S42XX_INTR_STATUS);
-
-	if( register_value &  S42XX_INTR_PENDING) 
-		return  FIFO_NOT_EMPTY;	
-	return 0 ;
-}
-
-static struct access_method smart4_access = {
-	smart4_submit_command,
-	smart4_intr_mask,
-	smart4_fifo_full,
-	smart4_intr_pending,
-	smart4_completed,
-};
-
-/*
- * Memory mapped FIFO interface (PCI SMART2 and SMART 3xxx cards)
- */
-static void smart2_submit_command(ctlr_info_t *h, cmdlist_t *c)
-{
-	writel(c->busaddr, h->vaddr + COMMAND_FIFO);
-}
-
-static void smart2_intr_mask(ctlr_info_t *h, unsigned long val)
-{
-	writel(val, h->vaddr + INTR_MASK);
-}
-
-static unsigned long smart2_fifo_full(ctlr_info_t *h)
-{
-	return readl(h->vaddr + COMMAND_FIFO);
-}
-
-static unsigned long smart2_completed(ctlr_info_t *h)
-{
-	return readl(h->vaddr + COMMAND_COMPLETE_FIFO);
-}
-
-static unsigned long smart2_intr_pending(ctlr_info_t *h)
-{
-	return readl(h->vaddr + INTR_PENDING);
-}
-
-static struct access_method smart2_access = {
-	smart2_submit_command,
-	smart2_intr_mask,
-	smart2_fifo_full,
-	smart2_intr_pending,
-	smart2_completed,
-};
-
-/*
- *  IO access for SMART-2/E cards
- */
-static void smart2e_submit_command(ctlr_info_t *h, cmdlist_t *c)
-{
-	outl(c->busaddr, h->io_mem_addr + COMMAND_FIFO);
-}
-
-static void smart2e_intr_mask(ctlr_info_t *h, unsigned long val)
-{
-	outl(val, h->io_mem_addr + INTR_MASK);
-}
-
-static unsigned long smart2e_fifo_full(ctlr_info_t *h)
-{
-	return inl(h->io_mem_addr + COMMAND_FIFO);
-}
-
-static unsigned long smart2e_completed(ctlr_info_t *h)
-{
-	return inl(h->io_mem_addr + COMMAND_COMPLETE_FIFO);
-}
-
-static unsigned long smart2e_intr_pending(ctlr_info_t *h)
-{
-	return inl(h->io_mem_addr + INTR_PENDING);
-}
-
-static struct access_method smart2e_access = {
-	smart2e_submit_command,
-	smart2e_intr_mask,
-	smart2e_fifo_full,
-	smart2e_intr_pending,
-	smart2e_completed,
-};
-
-/*
- *  IO access for older SMART-1 type cards
- */
-#define SMART1_SYSTEM_MASK		0xC8E
-#define SMART1_SYSTEM_DOORBELL		0xC8F
-#define SMART1_LOCAL_MASK		0xC8C
-#define SMART1_LOCAL_DOORBELL		0xC8D
-#define SMART1_INTR_MASK		0xC89
-#define SMART1_LISTADDR			0xC90
-#define SMART1_LISTLEN			0xC94
-#define SMART1_TAG			0xC97
-#define SMART1_COMPLETE_ADDR		0xC98
-#define SMART1_LISTSTATUS		0xC9E
-
-#define CHANNEL_BUSY			0x01
-#define CHANNEL_CLEAR			0x02
-
-static void smart1_submit_command(ctlr_info_t *h, cmdlist_t *c)
-{
-	/*
-	 * This __u16 is actually a bunch of control flags on SMART
-	 * and below.  We want them all to be zero.
-	 */
-	c->hdr.size = 0;
-
-	outb(CHANNEL_CLEAR, h->io_mem_addr + SMART1_SYSTEM_DOORBELL);
-
-	outl(c->busaddr, h->io_mem_addr + SMART1_LISTADDR);
-	outw(c->size, h->io_mem_addr + SMART1_LISTLEN);
-
-	outb(CHANNEL_BUSY, h->io_mem_addr + SMART1_LOCAL_DOORBELL);
-}
-
-static void smart1_intr_mask(ctlr_info_t *h, unsigned long val)
-{
-	if (val == 1) {
-		outb(0xFD, h->io_mem_addr + SMART1_SYSTEM_DOORBELL);
-		outb(CHANNEL_BUSY, h->io_mem_addr + SMART1_LOCAL_DOORBELL);
-		outb(0x01, h->io_mem_addr + SMART1_INTR_MASK);
-		outb(0x01, h->io_mem_addr + SMART1_SYSTEM_MASK);
-	} else {
-		outb(0, h->io_mem_addr + 0xC8E);
-	}
-}
-
-static unsigned long smart1_fifo_full(ctlr_info_t *h)
-{
-	unsigned char chan;
-	chan = inb(h->io_mem_addr + SMART1_SYSTEM_DOORBELL) & CHANNEL_CLEAR;
-	return chan;
-}
-
-static unsigned long smart1_completed(ctlr_info_t *h)
-{
-	unsigned char status;
-	unsigned long cmd;
-
-	if (inb(h->io_mem_addr + SMART1_SYSTEM_DOORBELL) & CHANNEL_BUSY) {
-		outb(CHANNEL_BUSY, h->io_mem_addr + SMART1_SYSTEM_DOORBELL);
-
-		cmd = inl(h->io_mem_addr + SMART1_COMPLETE_ADDR);
-		status = inb(h->io_mem_addr + SMART1_LISTSTATUS);
-
-		outb(CHANNEL_CLEAR, h->io_mem_addr + SMART1_LOCAL_DOORBELL);
-
-		/*
-		 * this is x86 (actually compaq x86) only, so it's ok
-		 */
-		if (cmd) ((cmdlist_t*)bus_to_virt(cmd))->req.hdr.rcode = status;
-	} else {
-		cmd = 0;
-	}
-	return cmd;
-}
-
-static unsigned long smart1_intr_pending(ctlr_info_t *h)
-{
-	unsigned char chan;
-	chan = inb(h->io_mem_addr + SMART1_SYSTEM_DOORBELL) & CHANNEL_BUSY;
-	return chan;
-}
-
-static struct access_method smart1_access = {
-	smart1_submit_command,
-	smart1_intr_mask,
-	smart1_fifo_full,
-	smart1_intr_pending,
-	smart1_completed,
-};
diff --git a/drivers/block/swim.c b/drivers/block/swim.c
index 84434d3ea19b..64e066eba72e 100644
--- a/drivers/block/swim.c
+++ b/drivers/block/swim.c
@@ -799,7 +799,7 @@ static struct kobject *floppy_find(dev_t dev, int *part, void *data)
 		return NULL;
 
 	*part = 0;
-	return get_disk(swd->unit[drive].disk);
+	return get_disk_and_module(swd->unit[drive].disk);
 }
 
 static int swim_add_floppy(struct swim_priv *swd, enum drive_location location)
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 68846897d213..4a07593c2efd 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -276,7 +276,7 @@ static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,
 		/* Out of mem doesn't actually happen, since we fall back
 		 * to direct descriptors */
 		if (err == -ENOMEM || err == -ENOSPC)
-			return BLK_STS_RESOURCE;
+			return BLK_STS_DEV_RESOURCE;
 		return BLK_STS_IOERR;
 	}
 
@@ -373,14 +373,12 @@ static ssize_t virtblk_serial_show(struct device *dev,
 
 static DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL);
 
-static void virtblk_config_changed_work(struct work_struct *work)
+/* The queue's logical block size must be set before calling this */
+static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)
 {
-	struct virtio_blk *vblk =
-		container_of(work, struct virtio_blk, config_work);
 	struct virtio_device *vdev = vblk->vdev;
 	struct request_queue *q = vblk->disk->queue;
 	char cap_str_2[10], cap_str_10[10];
-	char *envp[] = { "RESIZE=1", NULL };
 	unsigned long long nblocks;
 	u64 capacity;
 
@@ -402,13 +400,24 @@ static void virtblk_config_changed_work(struct work_struct *work)
 			STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
 
 	dev_notice(&vdev->dev,
-		   "new size: %llu %d-byte logical blocks (%s/%s)\n",
+		   "[%s] %s%llu %d-byte logical blocks (%s/%s)\n",
+		   vblk->disk->disk_name,
+		   resize ? "new size: " : "",
 		   nblocks,
 		   queue_logical_block_size(q),
 		   cap_str_10,
 		   cap_str_2);
 
 	set_capacity(vblk->disk, capacity);
+}
+
+static void virtblk_config_changed_work(struct work_struct *work)
+{
+	struct virtio_blk *vblk =
+		container_of(work, struct virtio_blk, config_work);
+	char *envp[] = { "RESIZE=1", NULL };
+
+	virtblk_update_capacity(vblk, true);
 	revalidate_disk(vblk->disk);
 	kobject_uevent_env(&disk_to_dev(vblk->disk)->kobj, KOBJ_CHANGE, envp);
 }
@@ -621,7 +630,6 @@ static int virtblk_probe(struct virtio_device *vdev)
 	struct request_queue *q;
 	int err, index;
 
-	u64 cap;
 	u32 v, blk_size, sg_elems, opt_io_size;
 	u16 min_io_size;
 	u8 physical_block_exp, alignment_offset;
@@ -719,17 +727,6 @@ static int virtblk_probe(struct virtio_device *vdev)
 	if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO))
 		set_disk_ro(vblk->disk, 1);
 
-	/* Host must always specify the capacity. */
-	virtio_cread(vdev, struct virtio_blk_config, capacity, &cap);
-
-	/* If capacity is too big, truncate with warning. */
-	if ((sector_t)cap != cap) {
-		dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n",
-			 (unsigned long long)cap);
-		cap = (sector_t)-1;
-	}
-	set_capacity(vblk->disk, cap);
-
 	/* We can handle whatever the host told us to handle. */
 	blk_queue_max_segments(q, vblk->sg_elems-2);
 
@@ -780,6 +777,7 @@ static int virtblk_probe(struct virtio_device *vdev)
 	if (!err && opt_io_size)
 		blk_queue_io_opt(q, blk_size * opt_io_size);
 
+	virtblk_update_capacity(vblk, false);
 	virtio_device_ready(vdev);
 
 	device_add_disk(&vdev->dev, vblk->disk);
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index 891265acb10e..92ec1bbece51 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -262,6 +262,7 @@ static DEFINE_SPINLOCK(minor_lock);
 
 static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo);
 static void blkfront_gather_backend_features(struct blkfront_info *info);
+static int negotiate_mq(struct blkfront_info *info);
 
 static int get_id_from_freelist(struct blkfront_ring_info *rinfo)
 {
@@ -911,7 +912,7 @@ static blk_status_t blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
 out_busy:
 	blk_mq_stop_hw_queue(hctx);
 	spin_unlock_irqrestore(&rinfo->ring_lock, flags);
-	return BLK_STS_RESOURCE;
+	return BLK_STS_DEV_RESOURCE;
 }
 
 static void blkif_complete_rq(struct request *rq)
@@ -1774,11 +1775,18 @@ static int talk_to_blkback(struct xenbus_device *dev,
 	unsigned int i, max_page_order;
 	unsigned int ring_page_order;
 
+	if (!info)
+		return -ENODEV;
+
 	max_page_order = xenbus_read_unsigned(info->xbdev->otherend,
 					      "max-ring-page-order", 0);
 	ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
 	info->nr_ring_pages = 1 << ring_page_order;
 
+	err = negotiate_mq(info);
+	if (err)
+		goto destroy_blkring;
+
 	for (i = 0; i < info->nr_rings; i++) {
 		struct blkfront_ring_info *rinfo = &info->rinfo[i];
 
@@ -1978,11 +1986,6 @@ static int blkfront_probe(struct xenbus_device *dev,
 	}
 
 	info->xbdev = dev;
-	err = negotiate_mq(info);
-	if (err) {
-		kfree(info);
-		return err;
-	}
 
 	mutex_init(&info->mutex);
 	info->vdevice = vdevice;
@@ -2099,10 +2102,6 @@ static int blkfront_resume(struct xenbus_device *dev)
 
 	blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
 
-	err = negotiate_mq(info);
-	if (err)
-		return err;
-
 	err = talk_to_blkback(dev, info);
 	if (!err)
 		blk_mq_update_nr_hw_queues(&info->tag_set, info->nr_rings);
diff --git a/drivers/block/z2ram.c b/drivers/block/z2ram.c
index 41c95c9b2ab4..8f9130ab5887 100644
--- a/drivers/block/z2ram.c
+++ b/drivers/block/z2ram.c
@@ -332,7 +332,7 @@ static const struct block_device_operations z2_fops =
 static struct kobject *z2_find(dev_t dev, int *part, void *data)
 {
 	*part = 0;
-	return get_disk(z2ram_gendisk);
+	return get_disk_and_module(z2ram_gendisk);
 }
 
 static struct request_queue *z2_queue;
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index d70eba30003a..0afa6c8c3857 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -430,7 +430,7 @@ static void put_entry_bdev(struct zram *zram, unsigned long entry)
 
 static void zram_page_end_io(struct bio *bio)
 {
-	struct page *page = bio->bi_io_vec[0].bv_page;
+	struct page *page = bio_first_page_all(bio);
 
 	page_endio(page, op_is_write(bio_op(bio)),
 			blk_status_to_errno(bio->bi_status));
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index 60e1c7d6986d..07e55cd8f8c8 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -31,6 +31,16 @@ config BT_HCIBTUSB
 	  Say Y here to compile support for Bluetooth USB devices into the
 	  kernel or say M to compile it as module (btusb).
 
+config BT_HCIBTUSB_AUTOSUSPEND
+	bool "Enable USB autosuspend for Bluetooth USB devices by default"
+	depends on BT_HCIBTUSB
+	help
+	  Say Y here to enable USB autosuspend for Bluetooth USB devices by
+	  default.
+
+	  This can be overridden by passing btusb.enable_autosuspend=[y|n]
+	  on the kernel commandline.
+
 config BT_HCIBTUSB_BCM
 	bool "Broadcom protocol support"
 	depends on BT_HCIBTUSB
@@ -67,6 +77,7 @@ config BT_HCIBTSDIO
 config BT_HCIUART
 	tristate "HCI UART driver"
 	depends on SERIAL_DEV_BUS || !SERIAL_DEV_BUS
+	depends on NVMEM || !NVMEM
 	depends on TTY
 	help
 	  Bluetooth HCI UART driver.
@@ -97,6 +108,7 @@ config BT_HCIUART_NOKIA
 	tristate "UART Nokia H4+ protocol support"
 	depends on BT_HCIUART
 	depends on BT_HCIUART_SERDEV
+	depends on GPIOLIB
 	depends on PM
 	select BT_HCIUART_H4
 	select BT_BCM
@@ -158,6 +170,7 @@ config BT_HCIUART_3WIRE
 config BT_HCIUART_INTEL
 	bool "Intel protocol support"
 	depends on BT_HCIUART
+	depends on GPIOLIB
 	select BT_HCIUART_H4
 	select BT_INTEL
 	help
@@ -171,6 +184,7 @@ config BT_HCIUART_BCM
 	depends on BT_HCIUART
 	depends on BT_HCIUART_SERDEV
 	depends on (!ACPI || SERIAL_DEV_CTRL_TTYPORT)
+	depends on GPIOLIB
 	select BT_HCIUART_H4
 	select BT_BCM
 	help
diff --git a/drivers/bluetooth/bluecard_cs.c b/drivers/bluetooth/bluecard_cs.c
index d513ef4743dc..82437a69f99c 100644
--- a/drivers/bluetooth/bluecard_cs.c
+++ b/drivers/bluetooth/bluecard_cs.c
@@ -302,9 +302,7 @@ static void bluecard_write_wakeup(struct bluecard_info *info)
 			}
 
 			/* Wait until the command reaches the baseband */
-			prepare_to_wait(&wq, &wait, TASK_INTERRUPTIBLE);
-			schedule_timeout(HZ/10);
-			finish_wait(&wq, &wait);
+			mdelay(100);
 
 			/* Set baud on baseband */
 			info->ctrl_reg &= ~0x03;
@@ -316,9 +314,7 @@ static void bluecard_write_wakeup(struct bluecard_info *info)
 			outb(info->ctrl_reg, iobase + REG_CONTROL);
 
 			/* Wait before the next HCI packet can be send */
-			prepare_to_wait(&wq, &wait, TASK_INTERRUPTIBLE);
-			schedule_timeout(HZ);
-			finish_wait(&wq, &wait);
+			mdelay(1000);
 		}
 
 		if (len == skb->len) {
diff --git a/drivers/bluetooth/bpa10x.c b/drivers/bluetooth/bpa10x.c
index 7971bfbd4321..801ea4ca65e4 100644
--- a/drivers/bluetooth/bpa10x.c
+++ b/drivers/bluetooth/bpa10x.c
@@ -259,7 +259,7 @@ static int bpa10x_flush(struct hci_dev *hdev)
 
 static int bpa10x_setup(struct hci_dev *hdev)
 {
-	const u8 req[] = { 0x07 };
+	static const u8 req[] = { 0x07 };
 	struct sk_buff *skb;
 
 	BT_DBG("%s", hdev->name);
diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c
index afa4cb3b16e3..6659f113042c 100644
--- a/drivers/bluetooth/btbcm.c
+++ b/drivers/bluetooth/btbcm.c
@@ -323,6 +323,7 @@ static const struct {
 	{ 0x410e, "BCM43341B0"	},	/* 002.001.014 */
 	{ 0x4406, "BCM4324B3"	},	/* 002.004.006 */
 	{ 0x610c, "BCM4354"	},	/* 003.001.012 */
+	{ 0x2122, "BCM4343A0"	},	/* 001.001.034 */
 	{ 0x2209, "BCM43430A1"  },	/* 001.002.009 */
 	{ 0x6119, "BCM4345C0"	},	/* 003.001.025 */
 	{ 0x230f, "BCM4356A2"	},	/* 001.003.015 */
diff --git a/drivers/bluetooth/btbcm.h b/drivers/bluetooth/btbcm.h
index d9e6b41658e5..cfe6ad4cc621 100644
--- a/drivers/bluetooth/btbcm.h
+++ b/drivers/bluetooth/btbcm.h
@@ -44,8 +44,8 @@ struct bcm_set_sleep_mode {
 	__u8 tristate_control;
 	__u8 usb_auto_sleep;
 	__u8 usb_resume_timeout;
-	__u8 pulsed_host_wake;
 	__u8 break_to_host;
+	__u8 pulsed_host_wake;
 } __packed;
 
 struct bcm_set_pcm_int_params {
diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c
index 4459555c9d88..5270d5513201 100644
--- a/drivers/bluetooth/btintel.c
+++ b/drivers/bluetooth/btintel.c
@@ -24,6 +24,7 @@
 #include <linux/module.h>
 #include <linux/firmware.h>
 #include <linux/regmap.h>
+#include <asm/unaligned.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -75,7 +76,7 @@ EXPORT_SYMBOL_GPL(btintel_check_bdaddr);
 
 int btintel_enter_mfg(struct hci_dev *hdev)
 {
-	const u8 param[] = { 0x01, 0x00 };
+	static const u8 param[] = { 0x01, 0x00 };
 	struct sk_buff *skb;
 
 	skb = __hci_cmd_sync(hdev, 0xfc11, 2, param, HCI_CMD_TIMEOUT);
@@ -569,6 +570,160 @@ struct regmap *btintel_regmap_init(struct hci_dev *hdev, u16 opcode_read,
 }
 EXPORT_SYMBOL_GPL(btintel_regmap_init);
 
+int btintel_send_intel_reset(struct hci_dev *hdev, u32 boot_param)
+{
+	struct intel_reset params = { 0x00, 0x01, 0x00, 0x01, 0x00000000 };
+	struct sk_buff *skb;
+
+	params.boot_param = cpu_to_le32(boot_param);
+
+	skb = __hci_cmd_sync(hdev, 0xfc01, sizeof(params), &params,
+			     HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		bt_dev_err(hdev, "Failed to send Intel Reset command");
+		return PTR_ERR(skb);
+	}
+
+	kfree_skb(skb);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(btintel_send_intel_reset);
+
+int btintel_read_boot_params(struct hci_dev *hdev,
+			     struct intel_boot_params *params)
+{
+	struct sk_buff *skb;
+
+	skb = __hci_cmd_sync(hdev, 0xfc0d, 0, NULL, HCI_INIT_TIMEOUT);
+	if (IS_ERR(skb)) {
+		bt_dev_err(hdev, "Reading Intel boot parameters failed (%ld)",
+			   PTR_ERR(skb));
+		return PTR_ERR(skb);
+	}
+
+	if (skb->len != sizeof(*params)) {
+		bt_dev_err(hdev, "Intel boot parameters size mismatch");
+		kfree_skb(skb);
+		return -EILSEQ;
+	}
+
+	memcpy(params, skb->data, sizeof(*params));
+
+	kfree_skb(skb);
+
+	if (params->status) {
+		bt_dev_err(hdev, "Intel boot parameters command failed (%02x)",
+			   params->status);
+		return -bt_to_errno(params->status);
+	}
+
+	bt_dev_info(hdev, "Device revision is %u",
+		    le16_to_cpu(params->dev_revid));
+
+	bt_dev_info(hdev, "Secure boot is %s",
+		    params->secure_boot ? "enabled" : "disabled");
+
+	bt_dev_info(hdev, "OTP lock is %s",
+		    params->otp_lock ? "enabled" : "disabled");
+
+	bt_dev_info(hdev, "API lock is %s",
+		    params->api_lock ? "enabled" : "disabled");
+
+	bt_dev_info(hdev, "Debug lock is %s",
+		    params->debug_lock ? "enabled" : "disabled");
+
+	bt_dev_info(hdev, "Minimum firmware build %u week %u %u",
+		    params->min_fw_build_nn, params->min_fw_build_cw,
+		    2000 + params->min_fw_build_yy);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(btintel_read_boot_params);
+
+int btintel_download_firmware(struct hci_dev *hdev, const struct firmware *fw,
+			      u32 *boot_param)
+{
+	int err;
+	const u8 *fw_ptr;
+	u32 frag_len;
+
+	/* Start the firmware download transaction with the Init fragment
+	 * represented by the 128 bytes of CSS header.
+	 */
+	err = btintel_secure_send(hdev, 0x00, 128, fw->data);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware header (%d)", err);
+		goto done;
+	}
+
+	/* Send the 256 bytes of public key information from the firmware
+	 * as the PKey fragment.
+	 */
+	err = btintel_secure_send(hdev, 0x03, 256, fw->data + 128);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware pkey (%d)", err);
+		goto done;
+	}
+
+	/* Send the 256 bytes of signature information from the firmware
+	 * as the Sign fragment.
+	 */
+	err = btintel_secure_send(hdev, 0x02, 256, fw->data + 388);
+	if (err < 0) {
+		bt_dev_err(hdev, "Failed to send firmware signature (%d)", err);
+		goto done;
+	}
+
+	fw_ptr = fw->data + 644;
+	frag_len = 0;
+
+	while (fw_ptr - fw->data < fw->size) {
+		struct hci_command_hdr *cmd = (void *)(fw_ptr + frag_len);
+
+		/* Each SKU has a different reset parameter to use in the
+		 * HCI_Intel_Reset command and it is embedded in the firmware
+		 * data. So, instead of using static value per SKU, check
+		 * the firmware data and save it for later use.
+		 */
+		if (le16_to_cpu(cmd->opcode) == 0xfc0e) {
+			/* The boot parameter is the first 32-bit value
+			 * and rest of 3 octets are reserved.
+			 */
+			*boot_param = get_unaligned_le32(fw_ptr + sizeof(*cmd));
+
+			bt_dev_dbg(hdev, "boot_param=0x%x", *boot_param);
+		}
+
+		frag_len += sizeof(*cmd) + cmd->plen;
+
+		/* The parameter length of the secure send command requires
+		 * a 4 byte alignment. It happens so that the firmware file
+		 * contains proper Intel_NOP commands to align the fragments
+		 * as needed.
+		 *
+		 * Send set of commands with 4 byte alignment from the
+		 * firmware data buffer as a single Data fragement.
+		 */
+		if (!(frag_len % 4)) {
+			err = btintel_secure_send(hdev, 0x01, frag_len, fw_ptr);
+			if (err < 0) {
+				bt_dev_err(hdev,
+					   "Failed to send firmware data (%d)",
+					   err);
+				goto done;
+			}
+
+			fw_ptr += frag_len;
+			frag_len = 0;
+		}
+	}
+
+done:
+	return err;
+}
+EXPORT_SYMBOL_GPL(btintel_download_firmware);
+
 MODULE_AUTHOR("Marcel Holtmann <marcel@holtmann.org>");
 MODULE_DESCRIPTION("Bluetooth support for Intel devices ver " VERSION);
 MODULE_VERSION(VERSION);
diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h
index 1e8955aaafed..41c642cc523f 100644
--- a/drivers/bluetooth/btintel.h
+++ b/drivers/bluetooth/btintel.h
@@ -69,6 +69,14 @@ struct intel_secure_send_result {
 	__u8     status;
 } __packed;
 
+struct intel_reset {
+	__u8     reset_type;
+	__u8     patch_enable;
+	__u8     ddc_reload;
+	__u8     boot_option;
+	__le32   boot_param;
+} __packed;
+
 #if IS_ENABLED(CONFIG_BT_INTEL)
 
 int btintel_check_bdaddr(struct hci_dev *hdev);
@@ -89,7 +97,11 @@ int btintel_read_version(struct hci_dev *hdev, struct intel_version *ver);
 
 struct regmap *btintel_regmap_init(struct hci_dev *hdev, u16 opcode_read,
 				   u16 opcode_write);
-
+int btintel_send_intel_reset(struct hci_dev *hdev, u32 boot_param);
+int btintel_read_boot_params(struct hci_dev *hdev,
+			     struct intel_boot_params *params);
+int btintel_download_firmware(struct hci_dev *dev, const struct firmware *fw,
+			      u32 *boot_param);
 #else
 
 static inline int btintel_check_bdaddr(struct hci_dev *hdev)
@@ -165,4 +177,23 @@ static inline struct regmap *btintel_regmap_init(struct hci_dev *hdev,
 {
 	return ERR_PTR(-EINVAL);
 }
+
+static inline int btintel_send_intel_reset(struct hci_dev *hdev,
+					   u32 reset_param)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int btintel_read_boot_params(struct hci_dev *hdev,
+					   struct intel_boot_params *params)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int btintel_download_firmware(struct hci_dev *dev,
+					    const struct firmware *fw,
+					    u32 *boot_param)
+{
+	return -EOPNOTSUPP;
+}
 #endif
diff --git a/drivers/bluetooth/btqcomsmd.c b/drivers/bluetooth/btqcomsmd.c
index 663bed63b871..2c9a5fc9137d 100644
--- a/drivers/bluetooth/btqcomsmd.c
+++ b/drivers/bluetooth/btqcomsmd.c
@@ -88,7 +88,8 @@ static int btqcomsmd_send(struct hci_dev *hdev, struct sk_buff *skb)
 		break;
 	}
 
-	kfree_skb(skb);
+	if (!ret)
+		kfree_skb(skb);
 
 	return ret;
 }
diff --git a/drivers/bluetooth/btsdio.c b/drivers/bluetooth/btsdio.c
index c8e945d19ffe..20142bc77554 100644
--- a/drivers/bluetooth/btsdio.c
+++ b/drivers/bluetooth/btsdio.c
@@ -31,6 +31,7 @@
 #include <linux/errno.h>
 #include <linux/skbuff.h>
 
+#include <linux/mmc/host.h>
 #include <linux/mmc/sdio_ids.h>
 #include <linux/mmc/sdio_func.h>
 
@@ -292,6 +293,14 @@ static int btsdio_probe(struct sdio_func *func,
 		tuple = tuple->next;
 	}
 
+	/* BCM43341 devices soldered onto the PCB (non-removable) use an
+	 * uart connection for bluetooth, ignore the BT SDIO interface.
+	 */
+	if (func->vendor == SDIO_VENDOR_ID_BROADCOM &&
+	    func->device == SDIO_DEVICE_ID_BROADCOM_43341 &&
+	    !mmc_card_is_removable(func->card->host))
+		return -ENODEV;
+
 	data = devm_kzalloc(&func->dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index f7120c9eb9bd..366a49c7c08f 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -21,8 +21,10 @@
  *
  */
 
+#include <linux/dmi.h>
 #include <linux/module.h>
 #include <linux/usb.h>
+#include <linux/usb/quirks.h>
 #include <linux/firmware.h>
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
@@ -40,6 +42,7 @@
 
 static bool disable_scofix;
 static bool force_scofix;
+static bool enable_autosuspend = IS_ENABLED(CONFIG_BT_HCIBTUSB_AUTOSUSPEND);
 
 static bool reset = true;
 
@@ -228,7 +231,6 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0930, 0x0227), .driver_info = BTUSB_ATH3012 },
 	{ USB_DEVICE(0x0b05, 0x17d0), .driver_info = BTUSB_ATH3012 },
 	{ USB_DEVICE(0x0cf3, 0x0036), .driver_info = BTUSB_ATH3012 },
-	{ USB_DEVICE(0x0cf3, 0x3004), .driver_info = BTUSB_ATH3012 },
 	{ USB_DEVICE(0x0cf3, 0x3008), .driver_info = BTUSB_ATH3012 },
 	{ USB_DEVICE(0x0cf3, 0x311d), .driver_info = BTUSB_ATH3012 },
 	{ USB_DEVICE(0x0cf3, 0x311e), .driver_info = BTUSB_ATH3012 },
@@ -261,8 +263,10 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0489, 0xe03c), .driver_info = BTUSB_ATH3012 },
 
 	/* QCA ROME chipset */
+	{ USB_DEVICE(0x0cf3, 0x3004), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0cf3, 0xe007), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0cf3, 0xe009), .driver_info = BTUSB_QCA_ROME },
+	{ USB_DEVICE(0x0cf3, 0xe010), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0cf3, 0xe300), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0cf3, 0xe301), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0cf3, 0xe360), .driver_info = BTUSB_QCA_ROME },
@@ -270,6 +274,7 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0489, 0xe09f), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x0489, 0xe0a2), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x04ca, 0x3011), .driver_info = BTUSB_QCA_ROME },
+	{ USB_DEVICE(0x04ca, 0x3015), .driver_info = BTUSB_QCA_ROME },
 	{ USB_DEVICE(0x04ca, 0x3016), .driver_info = BTUSB_QCA_ROME },
 
 	/* Broadcom BCM2035 */
@@ -375,6 +380,21 @@ static const struct usb_device_id blacklist_table[] = {
 	{ }	/* Terminating entry */
 };
 
+/* The Bluetooth USB module build into some devices needs to be reset on resume,
+ * this is a problem with the platform (likely shutting off all power) not with
+ * the module itself. So we use a DMI list to match known broken platforms.
+ */
+static const struct dmi_system_id btusb_needs_reset_resume_table[] = {
+	{
+		/* Dell OptiPlex 3060 (QCA ROME device 0cf3:e007) */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 3060"),
+		},
+	},
+	{}
+};
+
 #define BTUSB_MAX_ISOC_FRAMES	10
 
 #define BTUSB_INTR_RUNNING	0
@@ -387,9 +407,8 @@ static const struct usb_device_id blacklist_table[] = {
 #define BTUSB_FIRMWARE_LOADED	7
 #define BTUSB_FIRMWARE_FAILED	8
 #define BTUSB_BOOTING		9
-#define BTUSB_RESET_RESUME	10
-#define BTUSB_DIAG_RUNNING	11
-#define BTUSB_OOB_WAKE_ENABLED	12
+#define BTUSB_DIAG_RUNNING	10
+#define BTUSB_OOB_WAKE_ENABLED	11
 
 struct btusb_data {
 	struct hci_dev       *hdev;
@@ -2006,15 +2025,11 @@ static int btusb_send_frame_intel(struct hci_dev *hdev, struct sk_buff *skb)
 
 static int btusb_setup_intel_new(struct hci_dev *hdev)
 {
-	static const u8 reset_param[] = { 0x00, 0x01, 0x00, 0x01,
-					  0x00, 0x08, 0x04, 0x00 };
 	struct btusb_data *data = hci_get_drvdata(hdev);
-	struct sk_buff *skb;
 	struct intel_version ver;
-	struct intel_boot_params *params;
+	struct intel_boot_params params;
 	const struct firmware *fw;
-	const u8 *fw_ptr;
-	u32 frag_len;
+	u32 boot_param;
 	char fwname[64];
 	ktime_t calltime, delta, rettime;
 	unsigned long long duration;
@@ -2022,6 +2037,12 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 
 	BT_DBG("%s", hdev->name);
 
+	/* Set the default boot parameter to 0x0 and it is updated to
+	 * SKU specific boot parameter after reading Intel_Write_Boot_Params
+	 * command while downloading the firmware.
+	 */
+	boot_param = 0x00000000;
+
 	calltime = ktime_get();
 
 	/* Read the Intel version information to determine if the device
@@ -2092,55 +2113,24 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 	/* Read the secure boot parameters to identify the operating
 	 * details of the bootloader.
 	 */
-	skb = __hci_cmd_sync(hdev, 0xfc0d, 0, NULL, HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb)) {
-		BT_ERR("%s: Reading Intel boot parameters failed (%ld)",
-		       hdev->name, PTR_ERR(skb));
-		return PTR_ERR(skb);
-	}
-
-	if (skb->len != sizeof(*params)) {
-		BT_ERR("%s: Intel boot parameters size mismatch", hdev->name);
-		kfree_skb(skb);
-		return -EILSEQ;
-	}
-
-	params = (struct intel_boot_params *)skb->data;
-
-	bt_dev_info(hdev, "Device revision is %u",
-		    le16_to_cpu(params->dev_revid));
-
-	bt_dev_info(hdev, "Secure boot is %s",
-		    params->secure_boot ? "enabled" : "disabled");
-
-	bt_dev_info(hdev, "OTP lock is %s",
-		    params->otp_lock ? "enabled" : "disabled");
-
-	bt_dev_info(hdev, "API lock is %s",
-		    params->api_lock ? "enabled" : "disabled");
-
-	bt_dev_info(hdev, "Debug lock is %s",
-		    params->debug_lock ? "enabled" : "disabled");
-
-	bt_dev_info(hdev, "Minimum firmware build %u week %u %u",
-		    params->min_fw_build_nn, params->min_fw_build_cw,
-		    2000 + params->min_fw_build_yy);
+	err = btintel_read_boot_params(hdev, &params);
+	if (err)
+		return err;
 
 	/* It is required that every single firmware fragment is acknowledged
 	 * with a command complete event. If the boot parameters indicate
 	 * that this bootloader does not send them, then abort the setup.
 	 */
-	if (params->limited_cce != 0x00) {
+	if (params.limited_cce != 0x00) {
 		BT_ERR("%s: Unsupported Intel firmware loading method (%u)",
-		       hdev->name, params->limited_cce);
-		kfree_skb(skb);
+		       hdev->name, params.limited_cce);
 		return -EINVAL;
 	}
 
 	/* If the OTP has no valid Bluetooth device address, then there will
 	 * also be no valid address for the operational firmware.
 	 */
-	if (!bacmp(&params->otp_bdaddr, BDADDR_ANY)) {
+	if (!bacmp(&params.otp_bdaddr, BDADDR_ANY)) {
 		bt_dev_info(hdev, "No device address configured");
 		set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks);
 	}
@@ -2171,7 +2161,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 	case 0x0c:	/* WsP */
 		snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.sfi",
 			 le16_to_cpu(ver.hw_variant),
-			 le16_to_cpu(params->dev_revid));
+			 le16_to_cpu(params.dev_revid));
 		break;
 	case 0x11:	/* JfP */
 	case 0x12:	/* ThP */
@@ -2189,7 +2179,6 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 	if (err < 0) {
 		BT_ERR("%s: Failed to load Intel firmware file (%d)",
 		       hdev->name, err);
-		kfree_skb(skb);
 		return err;
 	}
 
@@ -2203,7 +2192,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 	case 0x0c:	/* WsP */
 		snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.ddc",
 			 le16_to_cpu(ver.hw_variant),
-			 le16_to_cpu(params->dev_revid));
+			 le16_to_cpu(params.dev_revid));
 		break;
 	case 0x11:	/* JfP */
 	case 0x12:	/* ThP */
@@ -2217,8 +2206,6 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 		return -EINVAL;
 	}
 
-	kfree_skb(skb);
-
 	if (fw->size < 644) {
 		BT_ERR("%s: Invalid size of firmware file (%zu)",
 		       hdev->name, fw->size);
@@ -2228,64 +2215,10 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 
 	set_bit(BTUSB_DOWNLOADING, &data->flags);
 
-	/* Start the firmware download transaction with the Init fragment
-	 * represented by the 128 bytes of CSS header.
-	 */
-	err = btintel_secure_send(hdev, 0x00, 128, fw->data);
-	if (err < 0) {
-		BT_ERR("%s: Failed to send firmware header (%d)",
-		       hdev->name, err);
-		goto done;
-	}
-
-	/* Send the 256 bytes of public key information from the firmware
-	 * as the PKey fragment.
-	 */
-	err = btintel_secure_send(hdev, 0x03, 256, fw->data + 128);
-	if (err < 0) {
-		BT_ERR("%s: Failed to send firmware public key (%d)",
-		       hdev->name, err);
-		goto done;
-	}
-
-	/* Send the 256 bytes of signature information from the firmware
-	 * as the Sign fragment.
-	 */
-	err = btintel_secure_send(hdev, 0x02, 256, fw->data + 388);
-	if (err < 0) {
-		BT_ERR("%s: Failed to send firmware signature (%d)",
-		       hdev->name, err);
+	/* Start firmware downloading and get boot parameter */
+	err = btintel_download_firmware(hdev, fw, &boot_param);
+	if (err < 0)
 		goto done;
-	}
-
-	fw_ptr = fw->data + 644;
-	frag_len = 0;
-
-	while (fw_ptr - fw->data < fw->size) {
-		struct hci_command_hdr *cmd = (void *)(fw_ptr + frag_len);
-
-		frag_len += sizeof(*cmd) + cmd->plen;
-
-		/* The parameter length of the secure send command requires
-		 * a 4 byte alignment. It happens so that the firmware file
-		 * contains proper Intel_NOP commands to align the fragments
-		 * as needed.
-		 *
-		 * Send set of commands with 4 byte alignment from the
-		 * firmware data buffer as a single Data fragement.
-		 */
-		if (!(frag_len % 4)) {
-			err = btintel_secure_send(hdev, 0x01, frag_len, fw_ptr);
-			if (err < 0) {
-				BT_ERR("%s: Failed to send firmware data (%d)",
-				       hdev->name, err);
-				goto done;
-			}
-
-			fw_ptr += frag_len;
-			frag_len = 0;
-		}
-	}
 
 	set_bit(BTUSB_FIRMWARE_LOADED, &data->flags);
 
@@ -2338,12 +2271,9 @@ static int btusb_setup_intel_new(struct hci_dev *hdev)
 
 	set_bit(BTUSB_BOOTING, &data->flags);
 
-	skb = __hci_cmd_sync(hdev, 0xfc01, sizeof(reset_param), reset_param,
-			     HCI_INIT_TIMEOUT);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	kfree_skb(skb);
+	err = btintel_send_intel_reset(hdev, boot_param);
+	if (err)
+		return err;
 
 	/* The bootloader will not indicate when the device is ready. This
 	 * is done by the operational firmware sending bootup notification.
@@ -3031,6 +2961,9 @@ static int btusb_probe(struct usb_interface *intf,
 	hdev->send   = btusb_send_frame;
 	hdev->notify = btusb_notify;
 
+	if (dmi_check_system(btusb_needs_reset_resume_table))
+		interface_to_usbdev(intf)->quirks |= USB_QUIRK_RESET_RESUME;
+
 #ifdef CONFIG_PM
 	err = btusb_config_oob_wake(hdev);
 	if (err)
@@ -3117,12 +3050,6 @@ static int btusb_probe(struct usb_interface *intf,
 	if (id->driver_info & BTUSB_QCA_ROME) {
 		data->setup_on_usb = btusb_setup_qca;
 		hdev->set_bdaddr = btusb_set_bdaddr_ath3012;
-
-		/* QCA Rome devices lose their updated firmware over suspend,
-		 * but the USB hub doesn't notice any status change.
-		 * Explicitly request a device reset on resume.
-		 */
-		set_bit(BTUSB_RESET_RESUME, &data->flags);
 	}
 
 #ifdef CONFIG_BT_HCIBTUSB_RTL
@@ -3133,7 +3060,7 @@ static int btusb_probe(struct usb_interface *intf,
 		 * but the USB hub doesn't notice any status change.
 		 * Explicitly request a device reset on resume.
 		 */
-		set_bit(BTUSB_RESET_RESUME, &data->flags);
+		interface_to_usbdev(intf)->quirks |= USB_QUIRK_RESET_RESUME;
 	}
 #endif
 
@@ -3213,6 +3140,9 @@ static int btusb_probe(struct usb_interface *intf,
 	}
 #endif
 
+	if (enable_autosuspend)
+		usb_enable_autosuspend(data->udev);
+
 	err = hci_register_dev(hdev);
 	if (err < 0)
 		goto out_free_dev;
@@ -3299,14 +3229,6 @@ static int btusb_suspend(struct usb_interface *intf, pm_message_t message)
 		enable_irq(data->oob_wake_irq);
 	}
 
-	/* Optionally request a device reset on resume, but only when
-	 * wakeups are disabled. If wakeups are enabled we assume the
-	 * device will stay powered up throughout suspend.
-	 */
-	if (test_bit(BTUSB_RESET_RESUME, &data->flags) &&
-	    !device_may_wakeup(&data->udev->dev))
-		data->udev->reset_resume = 1;
-
 	return 0;
 }
 
@@ -3425,6 +3347,9 @@ MODULE_PARM_DESC(disable_scofix, "Disable fixup of wrong SCO buffer size");
 module_param(force_scofix, bool, 0644);
 MODULE_PARM_DESC(force_scofix, "Force fixup of wrong SCO buffers size");
 
+module_param(enable_autosuspend, bool, 0644);
+MODULE_PARM_DESC(enable_autosuspend, "Enable USB autosuspend by default");
+
 module_param(reset, bool, 0644);
 MODULE_PARM_DESC(reset, "Send HCI reset command on initialization");
 
diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c
index 707c2d1b84c7..40b9fb247010 100644
--- a/drivers/bluetooth/hci_bcm.c
+++ b/drivers/bluetooth/hci_bcm.c
@@ -29,6 +29,7 @@
 #include <linux/acpi.h>
 #include <linux/of.h>
 #include <linux/property.h>
+#include <linux/platform_data/x86/apple.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/gpio/consumer.h>
@@ -52,7 +53,37 @@
 
 #define BCM_AUTOSUSPEND_DELAY	5000 /* default autosleep delay */
 
-/* device driver resources */
+/**
+ * struct bcm_device - device driver resources
+ * @serdev_hu: HCI UART controller struct
+ * @list: bcm_device_list node
+ * @dev: physical UART slave
+ * @name: device name logged by bt_dev_*() functions
+ * @device_wakeup: BT_WAKE pin,
+ *	assert = Bluetooth device must wake up or remain awake,
+ *	deassert = Bluetooth device may sleep when sleep criteria are met
+ * @shutdown: BT_REG_ON pin,
+ *	power up or power down Bluetooth device internal regulators
+ * @set_device_wakeup: callback to toggle BT_WAKE pin
+ *	either by accessing @device_wakeup or by calling @btlp
+ * @set_shutdown: callback to toggle BT_REG_ON pin
+ *	either by accessing @shutdown or by calling @btpu/@btpd
+ * @btlp: Apple ACPI method to toggle BT_WAKE pin ("Bluetooth Low Power")
+ * @btpu: Apple ACPI method to drive BT_REG_ON pin high ("Bluetooth Power Up")
+ * @btpd: Apple ACPI method to drive BT_REG_ON pin low ("Bluetooth Power Down")
+ * @clk: clock used by Bluetooth device
+ * @clk_enabled: whether @clk is prepared and enabled
+ * @init_speed: default baudrate of Bluetooth device;
+ *	the host UART is initially set to this baudrate so that
+ *	it can configure the Bluetooth device for @oper_speed
+ * @oper_speed: preferred baudrate of Bluetooth device;
+ *	set to 0 if @init_speed is already the preferred baudrate
+ * @irq: interrupt triggered by HOST_WAKE_BT pin
+ * @irq_active_low: whether @irq is active low
+ * @hu: pointer to HCI UART controller struct,
+ *	used to disable flow control during runtime suspend and system sleep
+ * @is_suspended: whether flow control is currently disabled
+ */
 struct bcm_device {
 	/* Must be the first member, hci_serdev.c expects this. */
 	struct hci_uart		serdev_hu;
@@ -63,6 +94,11 @@ struct bcm_device {
 	const char		*name;
 	struct gpio_desc	*device_wakeup;
 	struct gpio_desc	*shutdown;
+	int			(*set_device_wakeup)(struct bcm_device *, bool);
+	int			(*set_shutdown)(struct bcm_device *, bool);
+#ifdef CONFIG_ACPI
+	acpi_handle		btlp, btpu, btpd;
+#endif
 
 	struct clk		*clk;
 	bool			clk_enabled;
@@ -74,7 +110,7 @@ struct bcm_device {
 
 #ifdef CONFIG_PM
 	struct hci_uart		*hu;
-	bool			is_suspended; /* suspend/resume flag */
+	bool			is_suspended;
 #endif
 };
 
@@ -170,11 +206,21 @@ static bool bcm_device_exists(struct bcm_device *device)
 
 static int bcm_gpio_set_power(struct bcm_device *dev, bool powered)
 {
-	if (powered && !IS_ERR(dev->clk) && !dev->clk_enabled)
-		clk_prepare_enable(dev->clk);
+	int err;
 
-	gpiod_set_value(dev->shutdown, powered);
-	gpiod_set_value(dev->device_wakeup, powered);
+	if (powered && !IS_ERR(dev->clk) && !dev->clk_enabled) {
+		err = clk_prepare_enable(dev->clk);
+		if (err)
+			return err;
+	}
+
+	err = dev->set_shutdown(dev, powered);
+	if (err)
+		goto err_clk_disable;
+
+	err = dev->set_device_wakeup(dev, powered);
+	if (err)
+		goto err_revert_shutdown;
 
 	if (!powered && !IS_ERR(dev->clk) && dev->clk_enabled)
 		clk_disable_unprepare(dev->clk);
@@ -182,6 +228,13 @@ static int bcm_gpio_set_power(struct bcm_device *dev, bool powered)
 	dev->clk_enabled = powered;
 
 	return 0;
+
+err_revert_shutdown:
+	dev->set_shutdown(dev, !powered);
+err_clk_disable:
+	if (powered && !IS_ERR(dev->clk) && !dev->clk_enabled)
+		clk_disable_unprepare(dev->clk);
+	return err;
 }
 
 #ifdef CONFIG_PM
@@ -218,8 +271,10 @@ static int bcm_request_irq(struct bcm_data *bcm)
 			       bdev->irq_active_low ? IRQF_TRIGGER_FALLING :
 						      IRQF_TRIGGER_RISING,
 			       "host_wake", bdev);
-	if (err)
+	if (err) {
+		bdev->irq = err;
 		goto unlock;
+	}
 
 	device_init_wakeup(bdev->dev, true);
 
@@ -247,8 +302,8 @@ static const struct bcm_set_sleep_mode default_sleep_params = {
 	/* Irrelevant USB flags */
 	.usb_auto_sleep = 0,
 	.usb_resume_timeout = 0,
-	.pulsed_host_wake = 0,
-	.break_to_host = 0
+	.break_to_host = 0,
+	.pulsed_host_wake = 1,
 };
 
 static int bcm_setup_sleep(struct hci_uart *hu)
@@ -304,6 +359,7 @@ static int bcm_open(struct hci_uart *hu)
 {
 	struct bcm_data *bcm;
 	struct list_head *p;
+	int err;
 
 	bt_dev_dbg(hu->hdev, "hu %p", hu);
 
@@ -318,7 +374,10 @@ static int bcm_open(struct hci_uart *hu)
 	mutex_lock(&bcm_device_lock);
 
 	if (hu->serdev) {
-		serdev_device_open(hu->serdev);
+		err = serdev_device_open(hu->serdev);
+		if (err)
+			goto err_free;
+
 		bcm->dev = serdev_device_get_drvdata(hu->serdev);
 		goto out;
 	}
@@ -346,17 +405,33 @@ static int bcm_open(struct hci_uart *hu)
 	if (bcm->dev) {
 		hu->init_speed = bcm->dev->init_speed;
 		hu->oper_speed = bcm->dev->oper_speed;
-		bcm_gpio_set_power(bcm->dev, true);
+		err = bcm_gpio_set_power(bcm->dev, true);
+		if (err)
+			goto err_unset_hu;
 	}
 
 	mutex_unlock(&bcm_device_lock);
 	return 0;
+
+err_unset_hu:
+	if (hu->serdev)
+		serdev_device_close(hu->serdev);
+#ifdef CONFIG_PM
+	else
+		bcm->dev->hu = NULL;
+#endif
+err_free:
+	mutex_unlock(&bcm_device_lock);
+	hu->priv = NULL;
+	kfree(bcm);
+	return err;
 }
 
 static int bcm_close(struct hci_uart *hu)
 {
 	struct bcm_data *bcm = hu->priv;
 	struct bcm_device *bdev = NULL;
+	int err;
 
 	bt_dev_dbg(hu->hdev, "hu %p", hu);
 
@@ -374,16 +449,17 @@ static int bcm_close(struct hci_uart *hu)
 	}
 
 	if (bdev) {
-		bcm_gpio_set_power(bdev, false);
-#ifdef CONFIG_PM
-		pm_runtime_disable(bdev->dev);
-		pm_runtime_set_suspended(bdev->dev);
-
-		if (device_can_wakeup(bdev->dev)) {
+		if (IS_ENABLED(CONFIG_PM) && bdev->irq > 0) {
 			devm_free_irq(bdev->dev, bdev->irq, bdev);
 			device_init_wakeup(bdev->dev, false);
+			pm_runtime_disable(bdev->dev);
 		}
-#endif
+
+		err = bcm_gpio_set_power(bdev, false);
+		if (err)
+			bt_dev_err(hu->hdev, "Failed to power down");
+		else
+			pm_runtime_set_suspended(bdev->dev);
 	}
 	mutex_unlock(&bcm_device_lock);
 
@@ -566,6 +642,7 @@ static struct sk_buff *bcm_dequeue(struct hci_uart *hu)
 static int bcm_suspend_device(struct device *dev)
 {
 	struct bcm_device *bdev = dev_get_drvdata(dev);
+	int err;
 
 	bt_dev_dbg(bdev, "");
 
@@ -577,27 +654,37 @@ static int bcm_suspend_device(struct device *dev)
 	}
 
 	/* Suspend the device */
-	if (bdev->device_wakeup) {
-		gpiod_set_value(bdev->device_wakeup, false);
-		bt_dev_dbg(bdev, "suspend, delaying 15 ms");
-		mdelay(15);
+	err = bdev->set_device_wakeup(bdev, false);
+	if (err) {
+		if (bdev->is_suspended && bdev->hu) {
+			bdev->is_suspended = false;
+			hci_uart_set_flow_control(bdev->hu, false);
+		}
+		return -EBUSY;
 	}
 
+	bt_dev_dbg(bdev, "suspend, delaying 15 ms");
+	msleep(15);
+
 	return 0;
 }
 
 static int bcm_resume_device(struct device *dev)
 {
 	struct bcm_device *bdev = dev_get_drvdata(dev);
+	int err;
 
 	bt_dev_dbg(bdev, "");
 
-	if (bdev->device_wakeup) {
-		gpiod_set_value(bdev->device_wakeup, true);
-		bt_dev_dbg(bdev, "resume, delaying 15 ms");
-		mdelay(15);
+	err = bdev->set_device_wakeup(bdev, true);
+	if (err) {
+		dev_err(dev, "Failed to power up\n");
+		return err;
 	}
 
+	bt_dev_dbg(bdev, "resume, delaying 15 ms");
+	msleep(15);
+
 	/* When this executes, the device has woken up already */
 	if (bdev->is_suspended && bdev->hu) {
 		bdev->is_suspended = false;
@@ -632,7 +719,7 @@ static int bcm_suspend(struct device *dev)
 	if (pm_runtime_active(dev))
 		bcm_suspend_device(dev);
 
-	if (device_may_wakeup(dev)) {
+	if (device_may_wakeup(dev) && bdev->irq > 0) {
 		error = enable_irq_wake(bdev->irq);
 		if (!error)
 			bt_dev_dbg(bdev, "BCM irq: enabled");
@@ -648,6 +735,7 @@ static int bcm_suspend(struct device *dev)
 static int bcm_resume(struct device *dev)
 {
 	struct bcm_device *bdev = dev_get_drvdata(dev);
+	int err = 0;
 
 	bt_dev_dbg(bdev, "resume: is_suspended %d", bdev->is_suspended);
 
@@ -662,19 +750,21 @@ static int bcm_resume(struct device *dev)
 	if (!bdev->hu)
 		goto unlock;
 
-	if (device_may_wakeup(dev)) {
+	if (device_may_wakeup(dev) && bdev->irq > 0) {
 		disable_irq_wake(bdev->irq);
 		bt_dev_dbg(bdev, "BCM irq: disabled");
 	}
 
-	bcm_resume_device(dev);
+	err = bcm_resume_device(dev);
 
 unlock:
 	mutex_unlock(&bcm_device_lock);
 
-	pm_runtime_disable(dev);
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
+	if (!err) {
+		pm_runtime_disable(dev);
+		pm_runtime_set_active(dev);
+		pm_runtime_enable(dev);
+	}
 
 	return 0;
 }
@@ -771,16 +861,73 @@ static int bcm_resource(struct acpi_resource *ares, void *data)
 
 	return 0;
 }
+
+static int bcm_apple_set_device_wakeup(struct bcm_device *dev, bool awake)
+{
+	if (ACPI_FAILURE(acpi_execute_simple_method(dev->btlp, NULL, !awake)))
+		return -EIO;
+
+	return 0;
+}
+
+static int bcm_apple_set_shutdown(struct bcm_device *dev, bool powered)
+{
+	if (ACPI_FAILURE(acpi_evaluate_object(powered ? dev->btpu : dev->btpd,
+					      NULL, NULL, NULL)))
+		return -EIO;
+
+	return 0;
+}
+
+static int bcm_apple_get_resources(struct bcm_device *dev)
+{
+	struct acpi_device *adev = ACPI_COMPANION(dev->dev);
+	const union acpi_object *obj;
+
+	if (!adev ||
+	    ACPI_FAILURE(acpi_get_handle(adev->handle, "BTLP", &dev->btlp)) ||
+	    ACPI_FAILURE(acpi_get_handle(adev->handle, "BTPU", &dev->btpu)) ||
+	    ACPI_FAILURE(acpi_get_handle(adev->handle, "BTPD", &dev->btpd)))
+		return -ENODEV;
+
+	if (!acpi_dev_get_property(adev, "baud", ACPI_TYPE_BUFFER, &obj) &&
+	    obj->buffer.length == 8)
+		dev->init_speed = *(u64 *)obj->buffer.pointer;
+
+	dev->set_device_wakeup = bcm_apple_set_device_wakeup;
+	dev->set_shutdown = bcm_apple_set_shutdown;
+
+	return 0;
+}
+#else
+static inline int bcm_apple_get_resources(struct bcm_device *dev)
+{
+	return -EOPNOTSUPP;
+}
 #endif /* CONFIG_ACPI */
 
+static int bcm_gpio_set_device_wakeup(struct bcm_device *dev, bool awake)
+{
+	gpiod_set_value(dev->device_wakeup, awake);
+	return 0;
+}
+
+static int bcm_gpio_set_shutdown(struct bcm_device *dev, bool powered)
+{
+	gpiod_set_value(dev->shutdown, powered);
+	return 0;
+}
+
 static int bcm_get_resources(struct bcm_device *dev)
 {
 	dev->name = dev_name(dev->dev);
 
+	if (x86_apple_machine && !bcm_apple_get_resources(dev))
+		return 0;
+
 	dev->clk = devm_clk_get(dev->dev, NULL);
 
-	dev->device_wakeup = devm_gpiod_get_optional(dev->dev,
-						     "device-wakeup",
+	dev->device_wakeup = devm_gpiod_get_optional(dev->dev, "device-wakeup",
 						     GPIOD_OUT_LOW);
 	if (IS_ERR(dev->device_wakeup))
 		return PTR_ERR(dev->device_wakeup);
@@ -790,6 +937,9 @@ static int bcm_get_resources(struct bcm_device *dev)
 	if (IS_ERR(dev->shutdown))
 		return PTR_ERR(dev->shutdown);
 
+	dev->set_device_wakeup = bcm_gpio_set_device_wakeup;
+	dev->set_shutdown = bcm_gpio_set_shutdown;
+
 	/* IRQ can be declared in ACPI table as Interrupt or GpioInt */
 	if (dev->irq <= 0) {
 		struct gpio_desc *gpio;
@@ -802,7 +952,7 @@ static int bcm_get_resources(struct bcm_device *dev)
 		dev->irq = gpiod_to_irq(gpio);
 	}
 
-	dev_info(dev->dev, "BCM irq: %d\n", dev->irq);
+	dev_dbg(dev->dev, "BCM irq: %d\n", dev->irq);
 	return 0;
 }
 
@@ -892,7 +1042,9 @@ static int bcm_probe(struct platform_device *pdev)
 	list_add_tail(&dev->list, &bcm_device_list);
 	mutex_unlock(&bcm_device_lock);
 
-	bcm_gpio_set_power(dev, false);
+	ret = bcm_gpio_set_power(dev, false);
+	if (ret)
+		dev_err(&pdev->dev, "Failed to power down\n");
 
 	return 0;
 }
@@ -939,6 +1091,7 @@ static const struct acpi_device_id bcm_acpi_match[] = {
 	{ "BCM2E65", (kernel_ulong_t)&acpi_bcm_int_last_gpios },
 	{ "BCM2E67", (kernel_ulong_t)&acpi_bcm_int_last_gpios },
 	{ "BCM2E71", (kernel_ulong_t)&acpi_bcm_int_last_gpios },
+	{ "BCM2E72", (kernel_ulong_t)&acpi_bcm_int_last_gpios },
 	{ "BCM2E7B", (kernel_ulong_t)&acpi_bcm_int_last_gpios },
 	{ "BCM2E7C", (kernel_ulong_t)&acpi_bcm_int_last_gpios },
 	{ "BCM2E7E", (kernel_ulong_t)&acpi_bcm_int_first_gpios },
@@ -993,7 +1146,9 @@ static int bcm_serdev_probe(struct serdev_device *serdev)
 	if (err)
 		return err;
 
-	bcm_gpio_set_power(bcmdev, false);
+	err = bcm_gpio_set_power(bcmdev, false);
+	if (err)
+		dev_err(&serdev->dev, "Failed to power down\n");
 
 	return hci_uart_register_device(&bcmdev->serdev_hu, &bcm_proto);
 }
diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c
index aad07e40ea4f..7c166e3b308b 100644
--- a/drivers/bluetooth/hci_intel.c
+++ b/drivers/bluetooth/hci_intel.c
@@ -540,18 +540,15 @@ static int intel_set_baudrate(struct hci_uart *hu, unsigned int speed)
 
 static int intel_setup(struct hci_uart *hu)
 {
-	static const u8 reset_param[] = { 0x00, 0x01, 0x00, 0x01,
-					  0x00, 0x08, 0x04, 0x00 };
 	struct intel_data *intel = hu->priv;
 	struct hci_dev *hdev = hu->hdev;
 	struct sk_buff *skb;
 	struct intel_version ver;
-	struct intel_boot_params *params;
+	struct intel_boot_params params;
 	struct list_head *p;
 	const struct firmware *fw;
-	const u8 *fw_ptr;
 	char fwname[64];
-	u32 frag_len;
+	u32 boot_param;
 	ktime_t calltime, delta, rettime;
 	unsigned long long duration;
 	unsigned int init_speed, oper_speed;
@@ -563,6 +560,12 @@ static int intel_setup(struct hci_uart *hu)
 	hu->hdev->set_diag = btintel_set_diag;
 	hu->hdev->set_bdaddr = btintel_set_bdaddr;
 
+	/* Set the default boot parameter to 0x0 and it is updated to
+	 * SKU specific boot parameter after reading Intel_Write_Boot_Params
+	 * command while downloading the firmware.
+	 */
+	boot_param = 0x00000000;
+
 	calltime = ktime_get();
 
 	if (hu->init_speed)
@@ -656,85 +659,95 @@ static int intel_setup(struct hci_uart *hu)
 	/* Read the secure boot parameters to identify the operating
 	 * details of the bootloader.
 	 */
-	skb = __hci_cmd_sync(hdev, 0xfc0d, 0, NULL, HCI_CMD_TIMEOUT);
-	if (IS_ERR(skb)) {
-		bt_dev_err(hdev, "Reading Intel boot parameters failed (%ld)",
-			   PTR_ERR(skb));
-		return PTR_ERR(skb);
-	}
-
-	if (skb->len != sizeof(*params)) {
-		bt_dev_err(hdev, "Intel boot parameters size mismatch");
-		kfree_skb(skb);
-		return -EILSEQ;
-	}
-
-	params = (struct intel_boot_params *)skb->data;
-	if (params->status) {
-		bt_dev_err(hdev, "Intel boot parameters command failure (%02x)",
-			   params->status);
-		err = -bt_to_errno(params->status);
-		kfree_skb(skb);
+	err = btintel_read_boot_params(hdev, &params);
+	if (err)
 		return err;
-	}
-
-	bt_dev_info(hdev, "Device revision is %u",
-		    le16_to_cpu(params->dev_revid));
-
-	bt_dev_info(hdev, "Secure boot is %s",
-		    params->secure_boot ? "enabled" : "disabled");
-
-	bt_dev_info(hdev, "Minimum firmware build %u week %u %u",
-		params->min_fw_build_nn, params->min_fw_build_cw,
-		2000 + params->min_fw_build_yy);
 
 	/* It is required that every single firmware fragment is acknowledged
 	 * with a command complete event. If the boot parameters indicate
 	 * that this bootloader does not send them, then abort the setup.
 	 */
-	if (params->limited_cce != 0x00) {
+	if (params.limited_cce != 0x00) {
 		bt_dev_err(hdev, "Unsupported Intel firmware loading method (%u)",
-			   params->limited_cce);
-		kfree_skb(skb);
+			   params.limited_cce);
 		return -EINVAL;
 	}
 
 	/* If the OTP has no valid Bluetooth device address, then there will
 	 * also be no valid address for the operational firmware.
 	 */
-	if (!bacmp(&params->otp_bdaddr, BDADDR_ANY)) {
+	if (!bacmp(&params.otp_bdaddr, BDADDR_ANY)) {
 		bt_dev_info(hdev, "No device address configured");
 		set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks);
 	}
 
 	/* With this Intel bootloader only the hardware variant and device
-	 * revision information are used to select the right firmware.
+	 * revision information are used to select the right firmware for SfP
+	 * and WsP.
 	 *
 	 * The firmware filename is ibt-<hw_variant>-<dev_revid>.sfi.
 	 *
 	 * Currently the supported hardware variants are:
 	 *   11 (0x0b) for iBT 3.0 (LnP/SfP)
+	 *   12 (0x0c) for iBT 3.5 (WsP)
+	 *
+	 * For ThP/JfP and for future SKU's, the FW name varies based on HW
+	 * variant, HW revision and FW revision, as these are dependent on CNVi
+	 * and RF Combination.
+	 *
+	 *   18 (0x12) for iBT3.5 (ThP/JfP)
+	 *
+	 * The firmware file name for these will be
+	 * ibt-<hw_variant>-<hw_revision>-<fw_revision>.sfi.
+	 *
 	 */
-	snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.sfi",
-		le16_to_cpu(ver.hw_variant),
-		le16_to_cpu(params->dev_revid));
+	switch (ver.hw_variant) {
+	case 0x0b:      /* SfP */
+	case 0x0c:      /* WsP */
+		snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.sfi",
+			 le16_to_cpu(ver.hw_variant),
+			 le16_to_cpu(params.dev_revid));
+		break;
+	case 0x12:      /* ThP */
+		snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u-%u.sfi",
+			 le16_to_cpu(ver.hw_variant),
+			 le16_to_cpu(ver.hw_revision),
+			 le16_to_cpu(ver.fw_revision));
+		break;
+	default:
+		bt_dev_err(hdev, "Unsupported Intel hardware variant (%u)",
+			   ver.hw_variant);
+		return -EINVAL;
+	}
 
 	err = request_firmware(&fw, fwname, &hdev->dev);
 	if (err < 0) {
 		bt_dev_err(hdev, "Failed to load Intel firmware file (%d)",
 			   err);
-		kfree_skb(skb);
 		return err;
 	}
 
 	bt_dev_info(hdev, "Found device firmware: %s", fwname);
 
 	/* Save the DDC file name for later */
-	snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.ddc",
-		le16_to_cpu(ver.hw_variant),
-		le16_to_cpu(params->dev_revid));
-
-	kfree_skb(skb);
+	switch (ver.hw_variant) {
+	case 0x0b:      /* SfP */
+	case 0x0c:      /* WsP */
+		snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u.ddc",
+			 le16_to_cpu(ver.hw_variant),
+			 le16_to_cpu(params.dev_revid));
+		break;
+	case 0x12:      /* ThP */
+		snprintf(fwname, sizeof(fwname), "intel/ibt-%u-%u-%u.ddc",
+			 le16_to_cpu(ver.hw_variant),
+			 le16_to_cpu(ver.hw_revision),
+			 le16_to_cpu(ver.fw_revision));
+		break;
+	default:
+		bt_dev_err(hdev, "Unsupported Intel hardware variant (%u)",
+			   ver.hw_variant);
+		return -EINVAL;
+	}
 
 	if (fw->size < 644) {
 		bt_dev_err(hdev, "Invalid size of firmware file (%zu)",
@@ -745,70 +758,10 @@ static int intel_setup(struct hci_uart *hu)
 
 	set_bit(STATE_DOWNLOADING, &intel->flags);
 
-	/* Start the firmware download transaction with the Init fragment
-	 * represented by the 128 bytes of CSS header.
-	 */
-	err = btintel_secure_send(hdev, 0x00, 128, fw->data);
-	if (err < 0) {
-		bt_dev_err(hdev, "Failed to send firmware header (%d)", err);
-		goto done;
-	}
-
-	/* Send the 256 bytes of public key information from the firmware
-	 * as the PKey fragment.
-	 */
-	err = btintel_secure_send(hdev, 0x03, 256, fw->data + 128);
-	if (err < 0) {
-		bt_dev_err(hdev, "Failed to send firmware public key (%d)",
-			   err);
-		goto done;
-	}
-
-	/* Send the 256 bytes of signature information from the firmware
-	 * as the Sign fragment.
-	 */
-	err = btintel_secure_send(hdev, 0x02, 256, fw->data + 388);
-	if (err < 0) {
-		bt_dev_err(hdev, "Failed to send firmware signature (%d)",
-			   err);
+	/* Start firmware downloading and get boot parameter */
+	err = btintel_download_firmware(hdev, fw, &boot_param);
+	if (err < 0)
 		goto done;
-	}
-
-	fw_ptr = fw->data + 644;
-	frag_len = 0;
-
-	while (fw_ptr - fw->data < fw->size) {
-		struct hci_command_hdr *cmd = (void *)(fw_ptr + frag_len);
-
-		frag_len += sizeof(*cmd) + cmd->plen;
-
-		bt_dev_dbg(hdev, "Patching %td/%zu", (fw_ptr - fw->data),
-			   fw->size);
-
-		/* The parameter length of the secure send command requires
-		 * a 4 byte alignment. It happens so that the firmware file
-		 * contains proper Intel_NOP commands to align the fragments
-		 * as needed.
-		 *
-		 * Send set of commands with 4 byte alignment from the
-		 * firmware data buffer as a single Data fragement.
-		 */
-		if (frag_len % 4)
-			continue;
-
-		/* Send each command from the firmware data buffer as
-		 * a single Data fragment.
-		 */
-		err = btintel_secure_send(hdev, 0x01, frag_len, fw_ptr);
-		if (err < 0) {
-			bt_dev_err(hdev, "Failed to send firmware data (%d)",
-				   err);
-			goto done;
-		}
-
-		fw_ptr += frag_len;
-		frag_len = 0;
-	}
 
 	set_bit(STATE_FIRMWARE_LOADED, &intel->flags);
 
@@ -869,12 +822,9 @@ static int intel_setup(struct hci_uart *hu)
 
 	set_bit(STATE_BOOTING, &intel->flags);
 
-	skb = __hci_cmd_sync(hdev, 0xfc01, sizeof(reset_param), reset_param,
-			     HCI_CMD_TIMEOUT);
-	if (IS_ERR(skb))
-		return PTR_ERR(skb);
-
-	kfree_skb(skb);
+	err = btintel_send_intel_reset(hdev, boot_param);
+	if (err)
+		return err;
 
 	/* The bootloader will not indicate when the device is ready. This
 	 * is done by the operational firmware sending bootup notification.
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index c823914b3a80..b6a71705b7d6 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -794,7 +794,7 @@ static ssize_t hci_uart_tty_write(struct tty_struct *tty, struct file *file,
 	return 0;
 }
 
-static unsigned int hci_uart_tty_poll(struct tty_struct *tty,
+static __poll_t hci_uart_tty_poll(struct tty_struct *tty,
 				      struct file *filp, poll_table *wait)
 {
 	return 0;
diff --git a/drivers/bluetooth/hci_ll.c b/drivers/bluetooth/hci_ll.c
index e2c078d61730..1b4417a623a4 100644
--- a/drivers/bluetooth/hci_ll.c
+++ b/drivers/bluetooth/hci_ll.c
@@ -53,9 +53,14 @@
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
 #include <linux/gpio/consumer.h>
+#include <linux/nvmem-consumer.h>
 
 #include "hci_uart.h"
 
+/* Vendor-specific HCI commands */
+#define HCI_VS_WRITE_BD_ADDR			0xfc06
+#define HCI_VS_UPDATE_UART_HCI_BAUDRATE		0xff36
+
 /* HCILL commands */
 #define HCILL_GO_TO_SLEEP_IND	0x30
 #define HCILL_GO_TO_SLEEP_ACK	0x31
@@ -86,6 +91,7 @@ struct ll_device {
 	struct serdev_device *serdev;
 	struct gpio_desc *enable_gpio;
 	struct clk *ext_clk;
+	bdaddr_t bdaddr;
 };
 
 struct ll_struct {
@@ -620,7 +626,7 @@ static int download_firmware(struct ll_device *lldev)
 		case ACTION_SEND_COMMAND:	/* action send */
 			bt_dev_dbg(lldev->hu.hdev, "S");
 			cmd = (struct hci_command *)action_ptr;
-			if (cmd->opcode == 0xff36) {
+			if (cmd->opcode == HCI_VS_UPDATE_UART_HCI_BAUDRATE) {
 				/* ignore remote change
 				 * baud rate HCI VS command
 				 */
@@ -628,11 +634,11 @@ static int download_firmware(struct ll_device *lldev)
 				break;
 			}
 			if (cmd->prefix != 1)
-				bt_dev_dbg(lldev->hu.hdev, "command type %d\n", cmd->prefix);
+				bt_dev_dbg(lldev->hu.hdev, "command type %d", cmd->prefix);
 
 			skb = __hci_cmd_sync(lldev->hu.hdev, cmd->opcode, cmd->plen, &cmd->speed, HCI_INIT_TIMEOUT);
 			if (IS_ERR(skb)) {
-				bt_dev_err(lldev->hu.hdev, "send command failed\n");
+				bt_dev_err(lldev->hu.hdev, "send command failed");
 				err = PTR_ERR(skb);
 				goto out_rel_fw;
 			}
@@ -659,6 +665,24 @@ static int download_firmware(struct ll_device *lldev)
 	return err;
 }
 
+static int ll_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
+{
+	bdaddr_t bdaddr_swapped;
+	struct sk_buff *skb;
+
+	/* HCI_VS_WRITE_BD_ADDR (at least on a CC2560A chip) expects the BD
+	 * address to be MSB first, but bdaddr_t has the convention of being
+	 * LSB first.
+	 */
+	baswap(&bdaddr_swapped, bdaddr);
+	skb = __hci_cmd_sync(hdev, HCI_VS_WRITE_BD_ADDR, sizeof(bdaddr_t),
+			     &bdaddr_swapped, HCI_INIT_TIMEOUT);
+	if (!IS_ERR(skb))
+		kfree_skb(skb);
+
+	return PTR_ERR_OR_ZERO(skb);
+}
+
 static int ll_setup(struct hci_uart *hu)
 {
 	int err, retry = 3;
@@ -671,14 +695,20 @@ static int ll_setup(struct hci_uart *hu)
 
 	lldev = serdev_device_get_drvdata(serdev);
 
+	hu->hdev->set_bdaddr = ll_set_bdaddr;
+
 	serdev_device_set_flow_control(serdev, true);
 
 	do {
-		/* Configure BT_EN to HIGH state */
+		/* Reset the Bluetooth device */
 		gpiod_set_value_cansleep(lldev->enable_gpio, 0);
 		msleep(5);
 		gpiod_set_value_cansleep(lldev->enable_gpio, 1);
-		msleep(100);
+		err = serdev_device_wait_for_cts(serdev, true, 200);
+		if (err) {
+			bt_dev_err(hu->hdev, "Failed to get CTS");
+			return err;
+		}
 
 		err = download_firmware(lldev);
 		if (!err)
@@ -691,6 +721,18 @@ static int ll_setup(struct hci_uart *hu)
 	if (err)
 		return err;
 
+	/* Set BD address if one was specified at probe */
+	if (!bacmp(&lldev->bdaddr, BDADDR_NONE)) {
+		/* This means that there was an error getting the BD address
+		 * during probe, so mark the device as having a bad address.
+		 */
+		set_bit(HCI_QUIRK_INVALID_BDADDR, &hu->hdev->quirks);
+	} else if (bacmp(&lldev->bdaddr, BDADDR_ANY)) {
+		err = ll_set_bdaddr(hu->hdev, &lldev->bdaddr);
+		if (err)
+			set_bit(HCI_QUIRK_INVALID_BDADDR, &hu->hdev->quirks);
+	}
+
 	/* Operational speed if any */
 	if (hu->oper_speed)
 		speed = hu->oper_speed;
@@ -700,7 +742,12 @@ static int ll_setup(struct hci_uart *hu)
 		speed = 0;
 
 	if (speed) {
-		struct sk_buff *skb = __hci_cmd_sync(hu->hdev, 0xff36, sizeof(speed), &speed, HCI_INIT_TIMEOUT);
+		__le32 speed_le = cpu_to_le32(speed);
+		struct sk_buff *skb;
+
+		skb = __hci_cmd_sync(hu->hdev, HCI_VS_UPDATE_UART_HCI_BAUDRATE,
+				     sizeof(speed_le), &speed_le,
+				     HCI_INIT_TIMEOUT);
 		if (!IS_ERR(skb)) {
 			kfree_skb(skb);
 			serdev_device_set_baudrate(serdev, speed);
@@ -716,6 +763,7 @@ static int hci_ti_probe(struct serdev_device *serdev)
 {
 	struct hci_uart *hu;
 	struct ll_device *lldev;
+	struct nvmem_cell *bdaddr_cell;
 	u32 max_speed = 3000000;
 
 	lldev = devm_kzalloc(&serdev->dev, sizeof(struct ll_device), GFP_KERNEL);
@@ -737,6 +785,52 @@ static int hci_ti_probe(struct serdev_device *serdev)
 	of_property_read_u32(serdev->dev.of_node, "max-speed", &max_speed);
 	hci_uart_set_speeds(hu, 115200, max_speed);
 
+	/* optional BD address from nvram */
+	bdaddr_cell = nvmem_cell_get(&serdev->dev, "bd-address");
+	if (IS_ERR(bdaddr_cell)) {
+		int err = PTR_ERR(bdaddr_cell);
+
+		if (err == -EPROBE_DEFER)
+			return err;
+
+		/* ENOENT means there is no matching nvmem cell and ENOSYS
+		 * means that nvmem is not enabled in the kernel configuration.
+		 */
+		if (err != -ENOENT && err != -ENOSYS) {
+			/* If there was some other error, give userspace a
+			 * chance to fix the problem instead of failing to load
+			 * the driver. Using BDADDR_NONE as a flag that is
+			 * tested later in the setup function.
+			 */
+			dev_warn(&serdev->dev,
+				 "Failed to get \"bd-address\" nvmem cell (%d)\n",
+				 err);
+			bacpy(&lldev->bdaddr, BDADDR_NONE);
+		}
+	} else {
+		bdaddr_t *bdaddr;
+		size_t len;
+
+		bdaddr = nvmem_cell_read(bdaddr_cell, &len);
+		nvmem_cell_put(bdaddr_cell);
+		if (IS_ERR(bdaddr)) {
+			dev_err(&serdev->dev, "Failed to read nvmem bd-address\n");
+			return PTR_ERR(bdaddr);
+		}
+		if (len != sizeof(bdaddr_t)) {
+			dev_err(&serdev->dev, "Invalid nvmem bd-address length\n");
+			kfree(bdaddr);
+			return -EINVAL;
+		}
+
+		/* As per the device tree bindings, the value from nvmem is
+		 * expected to be MSB first, but in the kernel it is expected
+		 * that bdaddr_t is LSB first.
+		 */
+		baswap(&lldev->bdaddr, bdaddr);
+		kfree(bdaddr);
+	}
+
 	return hci_uart_register_device(hu, &llp);
 }
 
@@ -748,6 +842,7 @@ static void hci_ti_remove(struct serdev_device *serdev)
 }
 
 static const struct of_device_id hci_ti_of_match[] = {
+	{ .compatible = "ti,cc2560" },
 	{ .compatible = "ti,wl1271-st" },
 	{ .compatible = "ti,wl1273-st" },
 	{ .compatible = "ti,wl1281-st" },
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index bbd7db7384e6..05ec530b8a3a 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -932,6 +932,9 @@ static int qca_setup(struct hci_uart *hu)
 	if (!ret) {
 		set_bit(STATE_IN_BAND_SLEEP_ENABLED, &qca->flags);
 		qca_debugfs_init(hdev);
+	} else if (ret == -ENOENT) {
+		/* No patch/nvm-config found, run with original fw/config */
+		ret = 0;
 	}
 
 	/* Setup bdaddr */
diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c
index 71664b22ec9d..e0e6461b9200 100644
--- a/drivers/bluetooth/hci_serdev.c
+++ b/drivers/bluetooth/hci_serdev.c
@@ -303,6 +303,7 @@ int hci_uart_register_device(struct hci_uart *hu,
 	hci_set_drvdata(hdev, hu);
 
 	INIT_WORK(&hu->write_work, hci_uart_write_work);
+	percpu_init_rwsem(&hu->proto_lock);
 
 	/* Only when vendor specific setup callback is provided, consider
 	 * the manufacturer information valid. This avoids filling in the
diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c
index e6f6dbc04131..22f9145a426f 100644
--- a/drivers/bluetooth/hci_vhci.c
+++ b/drivers/bluetooth/hci_vhci.c
@@ -299,16 +299,16 @@ static ssize_t vhci_write(struct kiocb *iocb, struct iov_iter *from)
 	return vhci_get_user(data, from);
 }
 
-static unsigned int vhci_poll(struct file *file, poll_table *wait)
+static __poll_t vhci_poll(struct file *file, poll_table *wait)
 {
 	struct vhci_data *data = file->private_data;
 
 	poll_wait(file, &data->read_wait, wait);
 
 	if (!skb_queue_empty(&data->readq))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
-	return POLLOUT | POLLWRNORM;
+	return EPOLLOUT | EPOLLWRNORM;
 }
 
 static void vhci_open_timeout(struct work_struct *work)
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index dc7b3c7b7d42..57e011d36a79 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -120,7 +120,7 @@ config QCOM_EBI2
 	  SRAM, ethernet adapters, FPGAs and LCD displays.
 
 config SIMPLE_PM_BUS
-	bool "Simple Power-Managed Bus Driver"
+	tristate "Simple Power-Managed Bus Driver"
 	depends on OF && PM
 	help
 	  Driver for transparent busses that don't need a real driver, but
diff --git a/drivers/bus/omap_l3_noc.c b/drivers/bus/omap_l3_noc.c
index 5012e3ad1225..b040447575ad 100644
--- a/drivers/bus/omap_l3_noc.c
+++ b/drivers/bus/omap_l3_noc.c
@@ -375,3 +375,8 @@ static void __exit omap_l3_exit(void)
 	platform_driver_unregister(&omap_l3_driver);
 }
 module_exit(omap_l3_exit);
+
+MODULE_AUTHOR("Santosh Shilimkar");
+MODULE_AUTHOR("Sricharan R");
+MODULE_DESCRIPTION("OMAP L3 Interconnect error handling driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/bus/omap_l3_smx.c b/drivers/bus/omap_l3_smx.c
index 360a5c0a4ee0..b853a729537a 100644
--- a/drivers/bus/omap_l3_smx.c
+++ b/drivers/bus/omap_l3_smx.c
@@ -309,3 +309,9 @@ static void __exit omap3_l3_exit(void)
 	platform_driver_unregister(&omap3_l3_driver);
 }
 module_exit(omap3_l3_exit);
+
+MODULE_AUTHOR("Felipe Balbi");
+MODULE_AUTHOR("Santosh Shilimkar");
+MODULE_AUTHOR("Sricharan R");
+MODULE_DESCRIPTION("OMAP3XXX L3 Interconnect Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c
index c3c76a1ea8a8..cdaeeea7999c 100644
--- a/drivers/bus/ti-sysc.c
+++ b/drivers/bus/ti-sysc.c
@@ -18,6 +18,9 @@
 #include <linux/pm_runtime.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
+#include <linux/platform_data/ti-sysc.h>
+
+#include <dt-bindings/bus/ti-sysc.h>
 
 enum sysc_registers {
 	SYSC_REVISION,
@@ -36,6 +39,9 @@ enum sysc_clocks {
 
 static const char * const clock_names[] = { "fck", "ick", };
 
+#define SYSC_IDLEMODE_MASK		3
+#define SYSC_CLOCKACTIVITY_MASK		3
+
 /**
  * struct sysc - TI sysc interconnect target module registers and capabilities
  * @dev: struct device pointer
@@ -45,6 +51,10 @@ static const char * const clock_names[] = { "fck", "ick", };
  * @offsets: register offsets from module base
  * @clocks: clocks used by the interconnect target module
  * @legacy_mode: configured for legacy mode if set
+ * @cap: interconnect target module capabilities
+ * @cfg: interconnect target module configuration
+ * @name: name if available
+ * @revision: interconnect target module revision
  */
 struct sysc {
 	struct device *dev;
@@ -54,12 +64,34 @@ struct sysc {
 	int offsets[SYSC_MAX_REGS];
 	struct clk *clocks[SYSC_MAX_CLOCKS];
 	const char *legacy_mode;
+	const struct sysc_capabilities *cap;
+	struct sysc_config cfg;
+	const char *name;
+	u32 revision;
 };
 
+static u32 sysc_read(struct sysc *ddata, int offset)
+{
+	if (ddata->cfg.quirks & SYSC_QUIRK_16BIT) {
+		u32 val;
+
+		val = readw_relaxed(ddata->module_va + offset);
+		val |= (readw_relaxed(ddata->module_va + offset + 4) << 16);
+
+		return val;
+	}
+
+	return readl_relaxed(ddata->module_va + offset);
+}
+
 static u32 sysc_read_revision(struct sysc *ddata)
 {
-	return readl_relaxed(ddata->module_va +
-			     ddata->offsets[SYSC_REVISION]);
+	int offset = ddata->offsets[SYSC_REVISION];
+
+	if (offset < 0)
+		return 0;
+
+	return sysc_read(ddata, offset);
 }
 
 static int sysc_get_one_clock(struct sysc *ddata,
@@ -206,6 +238,21 @@ static int sysc_check_children(struct sysc *ddata)
 	return 0;
 }
 
+/*
+ * So far only I2C uses 16-bit read access with clockactivity with revision
+ * in two registers with stride of 4. We can detect this based on the rev
+ * register size to configure things far enough to be able to properly read
+ * the revision register.
+ */
+static void sysc_check_quirk_16bit(struct sysc *ddata, struct resource *res)
+{
+	if (resource_size(res) == 8) {
+		dev_dbg(ddata->dev,
+			"enabling 16-bit and clockactivity quirks\n");
+		ddata->cfg.quirks |= SYSC_QUIRK_16BIT | SYSC_QUIRK_USE_CLOCKACT;
+	}
+}
+
 /**
  * sysc_parse_one - parses the interconnect target module registers
  * @ddata: device driver data
@@ -236,6 +283,8 @@ static int sysc_parse_one(struct sysc *ddata, enum sysc_registers reg)
 	}
 
 	ddata->offsets[reg] = res->start - ddata->module_pa;
+	if (reg == SYSC_REVISION)
+		sysc_check_quirk_16bit(ddata, res);
 
 	return 0;
 }
@@ -369,22 +418,12 @@ static int sysc_map_and_check_registers(struct sysc *ddata)
  */
 static int sysc_show_rev(char *bufp, struct sysc *ddata)
 {
-	int error, len;
+	int len;
 
 	if (ddata->offsets[SYSC_REVISION] < 0)
 		return sprintf(bufp, ":NA");
 
-	error = pm_runtime_get_sync(ddata->dev);
-	if (error < 0) {
-		pm_runtime_put_noidle(ddata->dev);
-
-		return 0;
-	}
-
-	len = sprintf(bufp, ":%08x", sysc_read_revision(ddata));
-
-	pm_runtime_mark_last_busy(ddata->dev);
-	pm_runtime_put_autosuspend(ddata->dev);
+	len = sprintf(bufp, ":%08x", ddata->revision);
 
 	return len;
 }
@@ -464,6 +503,151 @@ static const struct dev_pm_ops sysc_pm_ops = {
 			   NULL)
 };
 
+/* At this point the module is configured enough to read the revision */
+static int sysc_init_module(struct sysc *ddata)
+{
+	int error;
+
+	error = pm_runtime_get_sync(ddata->dev);
+	if (error < 0) {
+		pm_runtime_put_noidle(ddata->dev);
+
+		return 0;
+	}
+	ddata->revision = sysc_read_revision(ddata);
+	pm_runtime_put_sync(ddata->dev);
+
+	return 0;
+}
+
+static int sysc_init_sysc_mask(struct sysc *ddata)
+{
+	struct device_node *np = ddata->dev->of_node;
+	int error;
+	u32 val;
+
+	error = of_property_read_u32(np, "ti,sysc-mask", &val);
+	if (error)
+		return 0;
+
+	if (val)
+		ddata->cfg.sysc_val = val & ddata->cap->sysc_mask;
+	else
+		ddata->cfg.sysc_val = ddata->cap->sysc_mask;
+
+	return 0;
+}
+
+static int sysc_init_idlemode(struct sysc *ddata, u8 *idlemodes,
+			      const char *name)
+{
+	struct device_node *np = ddata->dev->of_node;
+	struct property *prop;
+	const __be32 *p;
+	u32 val;
+
+	of_property_for_each_u32(np, name, prop, p, val) {
+		if (val >= SYSC_NR_IDLEMODES) {
+			dev_err(ddata->dev, "invalid idlemode: %i\n", val);
+			return -EINVAL;
+		}
+		*idlemodes |=  (1 << val);
+	}
+
+	return 0;
+}
+
+static int sysc_init_idlemodes(struct sysc *ddata)
+{
+	int error;
+
+	error = sysc_init_idlemode(ddata, &ddata->cfg.midlemodes,
+				   "ti,sysc-midle");
+	if (error)
+		return error;
+
+	error = sysc_init_idlemode(ddata, &ddata->cfg.sidlemodes,
+				   "ti,sysc-sidle");
+	if (error)
+		return error;
+
+	return 0;
+}
+
+/*
+ * Only some devices on omap4 and later have SYSCONFIG reset done
+ * bit. We can detect this if there is no SYSSTATUS at all, or the
+ * SYSTATUS bit 0 is not used. Note that some SYSSTATUS registers
+ * have multiple bits for the child devices like OHCI and EHCI.
+ * Depends on SYSC being parsed first.
+ */
+static int sysc_init_syss_mask(struct sysc *ddata)
+{
+	struct device_node *np = ddata->dev->of_node;
+	int error;
+	u32 val;
+
+	error = of_property_read_u32(np, "ti,syss-mask", &val);
+	if (error) {
+		if ((ddata->cap->type == TI_SYSC_OMAP4 ||
+		     ddata->cap->type == TI_SYSC_OMAP4_TIMER) &&
+		    (ddata->cfg.sysc_val & SYSC_OMAP4_SOFTRESET))
+			ddata->cfg.quirks |= SYSC_QUIRK_RESET_STATUS;
+
+		return 0;
+	}
+
+	if (!(val & 1) && (ddata->cfg.sysc_val & SYSC_OMAP4_SOFTRESET))
+		ddata->cfg.quirks |= SYSC_QUIRK_RESET_STATUS;
+
+	ddata->cfg.syss_mask = val;
+
+	return 0;
+}
+
+/* Device tree configured quirks */
+struct sysc_dts_quirk {
+	const char *name;
+	u32 mask;
+};
+
+static const struct sysc_dts_quirk sysc_dts_quirks[] = {
+	{ .name = "ti,no-idle-on-init",
+	  .mask = SYSC_QUIRK_NO_IDLE_ON_INIT, },
+	{ .name = "ti,no-reset-on-init",
+	  .mask = SYSC_QUIRK_NO_RESET_ON_INIT, },
+};
+
+static int sysc_init_dts_quirks(struct sysc *ddata)
+{
+	struct device_node *np = ddata->dev->of_node;
+	const struct property *prop;
+	int i, len, error;
+	u32 val;
+
+	ddata->legacy_mode = of_get_property(np, "ti,hwmods", NULL);
+
+	for (i = 0; i < ARRAY_SIZE(sysc_dts_quirks); i++) {
+		prop = of_get_property(np, sysc_dts_quirks[i].name, &len);
+		if (!prop)
+			continue;
+
+		ddata->cfg.quirks |= sysc_dts_quirks[i].mask;
+	}
+
+	error = of_property_read_u32(np, "ti,sysc-delay-us", &val);
+	if (!error) {
+		if (val > 255) {
+			dev_warn(ddata->dev, "bad ti,sysc-delay-us: %i\n",
+				 val);
+		}
+
+		ddata->cfg.srst_udelay = (u8)val;
+	}
+
+	return 0;
+}
+
 static void sysc_unprepare(struct sysc *ddata)
 {
 	int i;
@@ -474,9 +658,230 @@ static void sysc_unprepare(struct sysc *ddata)
 	}
 }
 
+/*
+ * Common sysc register bits found on omap2, also known as type1
+ */
+static const struct sysc_regbits sysc_regbits_omap2 = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = 12,
+	.sidle_shift = 3,
+	.clkact_shift = 8,
+	.emufree_shift = 5,
+	.enwkup_shift = 2,
+	.srst_shift = 1,
+	.autoidle_shift = 0,
+};
+
+static const struct sysc_capabilities sysc_omap2 = {
+	.type = TI_SYSC_OMAP2,
+	.sysc_mask = SYSC_OMAP2_CLOCKACTIVITY | SYSC_OMAP2_EMUFREE |
+		     SYSC_OMAP2_ENAWAKEUP | SYSC_OMAP2_SOFTRESET |
+		     SYSC_OMAP2_AUTOIDLE,
+	.regbits = &sysc_regbits_omap2,
+};
+
+/* All omap2 and 3 timers, and timers 1, 2 & 10 on omap 4 and 5 */
+static const struct sysc_capabilities sysc_omap2_timer = {
+	.type = TI_SYSC_OMAP2_TIMER,
+	.sysc_mask = SYSC_OMAP2_CLOCKACTIVITY | SYSC_OMAP2_EMUFREE |
+		     SYSC_OMAP2_ENAWAKEUP | SYSC_OMAP2_SOFTRESET |
+		     SYSC_OMAP2_AUTOIDLE,
+	.regbits = &sysc_regbits_omap2,
+	.mod_quirks = SYSC_QUIRK_USE_CLOCKACT,
+};
+
+/*
+ * SHAM2 (SHA1/MD5) sysc found on omap3, a variant of sysc_regbits_omap2
+ * with different sidle position
+ */
+static const struct sysc_regbits sysc_regbits_omap3_sham = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = -ENODEV,
+	.sidle_shift = 4,
+	.clkact_shift = -ENODEV,
+	.enwkup_shift = -ENODEV,
+	.srst_shift = 1,
+	.autoidle_shift = 0,
+	.emufree_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_omap3_sham = {
+	.type = TI_SYSC_OMAP3_SHAM,
+	.sysc_mask = SYSC_OMAP2_SOFTRESET | SYSC_OMAP2_AUTOIDLE,
+	.regbits = &sysc_regbits_omap3_sham,
+};
+
+/*
+ * AES register bits found on omap3 and later, a variant of
+ * sysc_regbits_omap2 with different sidle position
+ */
+static const struct sysc_regbits sysc_regbits_omap3_aes = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = -ENODEV,
+	.sidle_shift = 6,
+	.clkact_shift = -ENODEV,
+	.enwkup_shift = -ENODEV,
+	.srst_shift = 1,
+	.autoidle_shift = 0,
+	.emufree_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_omap3_aes = {
+	.type = TI_SYSC_OMAP3_AES,
+	.sysc_mask = SYSC_OMAP2_SOFTRESET | SYSC_OMAP2_AUTOIDLE,
+	.regbits = &sysc_regbits_omap3_aes,
+};
+
+/*
+ * Common sysc register bits found on omap4, also known as type2
+ */
+static const struct sysc_regbits sysc_regbits_omap4 = {
+	.dmadisable_shift = 16,
+	.midle_shift = 4,
+	.sidle_shift = 2,
+	.clkact_shift = -ENODEV,
+	.enwkup_shift = -ENODEV,
+	.emufree_shift = 1,
+	.srst_shift = 0,
+	.autoidle_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_omap4 = {
+	.type = TI_SYSC_OMAP4,
+	.sysc_mask = SYSC_OMAP4_DMADISABLE | SYSC_OMAP4_FREEEMU |
+		     SYSC_OMAP4_SOFTRESET,
+	.regbits = &sysc_regbits_omap4,
+};
+
+static const struct sysc_capabilities sysc_omap4_timer = {
+	.type = TI_SYSC_OMAP4_TIMER,
+	.sysc_mask = SYSC_OMAP4_DMADISABLE | SYSC_OMAP4_FREEEMU |
+		     SYSC_OMAP4_SOFTRESET,
+	.regbits = &sysc_regbits_omap4,
+};
+
+/*
+ * Common sysc register bits found on omap4, also known as type3
+ */
+static const struct sysc_regbits sysc_regbits_omap4_simple = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = 2,
+	.sidle_shift = 0,
+	.clkact_shift = -ENODEV,
+	.enwkup_shift = -ENODEV,
+	.srst_shift = -ENODEV,
+	.emufree_shift = -ENODEV,
+	.autoidle_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_omap4_simple = {
+	.type = TI_SYSC_OMAP4_SIMPLE,
+	.regbits = &sysc_regbits_omap4_simple,
+};
+
+/*
+ * SmartReflex sysc found on omap34xx
+ */
+static const struct sysc_regbits sysc_regbits_omap34xx_sr = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = -ENODEV,
+	.sidle_shift = -ENODEV,
+	.clkact_shift = 20,
+	.enwkup_shift = -ENODEV,
+	.srst_shift = -ENODEV,
+	.emufree_shift = -ENODEV,
+	.autoidle_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_34xx_sr = {
+	.type = TI_SYSC_OMAP34XX_SR,
+	.sysc_mask = SYSC_OMAP2_CLOCKACTIVITY,
+	.regbits = &sysc_regbits_omap34xx_sr,
+	.mod_quirks = SYSC_QUIRK_USE_CLOCKACT | SYSC_QUIRK_UNCACHED,
+};
+
+/*
+ * SmartReflex sysc found on omap36xx and later
+ */
+static const struct sysc_regbits sysc_regbits_omap36xx_sr = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = -ENODEV,
+	.sidle_shift = 24,
+	.clkact_shift = -ENODEV,
+	.enwkup_shift = 26,
+	.srst_shift = -ENODEV,
+	.emufree_shift = -ENODEV,
+	.autoidle_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_36xx_sr = {
+	.type = TI_SYSC_OMAP36XX_SR,
+	.sysc_mask = SYSC_OMAP3_SR_ENAWAKEUP,
+	.regbits = &sysc_regbits_omap36xx_sr,
+	.mod_quirks = SYSC_QUIRK_UNCACHED,
+};
+
+static const struct sysc_capabilities sysc_omap4_sr = {
+	.type = TI_SYSC_OMAP4_SR,
+	.regbits = &sysc_regbits_omap36xx_sr,
+};
+
+/*
+ * McASP register bits found on omap4 and later
+ */
+static const struct sysc_regbits sysc_regbits_omap4_mcasp = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = -ENODEV,
+	.sidle_shift = 0,
+	.clkact_shift = -ENODEV,
+	.enwkup_shift = -ENODEV,
+	.srst_shift = -ENODEV,
+	.emufree_shift = -ENODEV,
+	.autoidle_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_omap4_mcasp = {
+	.type = TI_SYSC_OMAP4_MCASP,
+	.regbits = &sysc_regbits_omap4_mcasp,
+};
+
+/*
+ * FS USB host found on omap4 and later
+ */
+static const struct sysc_regbits sysc_regbits_omap4_usb_host_fs = {
+	.dmadisable_shift = -ENODEV,
+	.midle_shift = -ENODEV,
+	.sidle_shift = 24,
+	.clkact_shift = -ENODEV,
+	.enwkup_shift = 26,
+	.srst_shift = -ENODEV,
+	.emufree_shift = -ENODEV,
+	.autoidle_shift = -ENODEV,
+};
+
+static const struct sysc_capabilities sysc_omap4_usb_host_fs = {
+	.type = TI_SYSC_OMAP4_USB_HOST_FS,
+	.sysc_mask = SYSC_OMAP2_ENAWAKEUP,
+	.regbits = &sysc_regbits_omap4_usb_host_fs,
+};
+
+static int sysc_init_match(struct sysc *ddata)
+{
+	const struct sysc_capabilities *cap;
+
+	cap = of_device_get_match_data(ddata->dev);
+	if (!cap)
+		return -EINVAL;
+
+	ddata->cap = cap;
+	if (ddata->cap)
+		ddata->cfg.quirks |= ddata->cap->mod_quirks;
+
+	return 0;
+}
+
 static int sysc_probe(struct platform_device *pdev)
 {
-	struct device_node *np = pdev->dev.of_node;
 	struct sysc *ddata;
 	int error;
 
@@ -485,7 +890,15 @@ static int sysc_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ddata->dev = &pdev->dev;
-	ddata->legacy_mode = of_get_property(np, "ti,hwmods", NULL);
+	platform_set_drvdata(pdev, ddata);
+
+	error = sysc_init_match(ddata);
+	if (error)
+		return error;
+
+	error = sysc_init_dts_quirks(ddata);
+	if (error)
+		goto unprepare;
 
 	error = sysc_get_clocks(ddata);
 	if (error)
@@ -495,9 +908,24 @@ static int sysc_probe(struct platform_device *pdev)
 	if (error)
 		goto unprepare;
 
-	platform_set_drvdata(pdev, ddata);
+	error = sysc_init_sysc_mask(ddata);
+	if (error)
+		goto unprepare;
+
+	error = sysc_init_idlemodes(ddata);
+	if (error)
+		goto unprepare;
+
+	error = sysc_init_syss_mask(ddata);
+	if (error)
+		goto unprepare;
 
 	pm_runtime_enable(ddata->dev);
+
+	error = sysc_init_module(ddata);
+	if (error)
+		goto unprepare;
+
 	error = pm_runtime_get_sync(ddata->dev);
 	if (error < 0) {
 		pm_runtime_put_noidle(ddata->dev);
@@ -554,16 +982,19 @@ static int sysc_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id sysc_match[] = {
-	{ .compatible = "ti,sysc-omap2" },
-	{ .compatible = "ti,sysc-omap4" },
-	{ .compatible = "ti,sysc-omap4-simple" },
-	{ .compatible = "ti,sysc-omap3430-sr" },
-	{ .compatible = "ti,sysc-omap3630-sr" },
-	{ .compatible = "ti,sysc-omap4-sr" },
-	{ .compatible = "ti,sysc-omap3-sham" },
-	{ .compatible = "ti,sysc-omap-aes" },
-	{ .compatible = "ti,sysc-mcasp" },
-	{ .compatible = "ti,sysc-usb-host-fs" },
+	{ .compatible = "ti,sysc-omap2", .data = &sysc_omap2, },
+	{ .compatible = "ti,sysc-omap2-timer", .data = &sysc_omap2_timer, },
+	{ .compatible = "ti,sysc-omap4", .data = &sysc_omap4, },
+	{ .compatible = "ti,sysc-omap4-timer", .data = &sysc_omap4_timer, },
+	{ .compatible = "ti,sysc-omap4-simple", .data = &sysc_omap4_simple, },
+	{ .compatible = "ti,sysc-omap3430-sr", .data = &sysc_34xx_sr, },
+	{ .compatible = "ti,sysc-omap3630-sr", .data = &sysc_36xx_sr, },
+	{ .compatible = "ti,sysc-omap4-sr", .data = &sysc_omap4_sr, },
+	{ .compatible = "ti,sysc-omap3-sham", .data = &sysc_omap3_sham, },
+	{ .compatible = "ti,sysc-omap-aes", .data = &sysc_omap3_aes, },
+	{ .compatible = "ti,sysc-mcasp", .data = &sysc_omap4_mcasp, },
+	{ .compatible = "ti,sysc-usb-host-fs",
+	  .data = &sysc_omap4_usb_host_fs, },
 	{  },
 };
 MODULE_DEVICE_TABLE(of, sysc_match);
diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index 9b6b6023193b..c6271ce250b3 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -80,7 +80,7 @@ static struct _intel_private {
 	unsigned int needs_dmar : 1;
 	phys_addr_t gma_bus_addr;
 	/*  Size of memory reserved for graphics by the BIOS */
-	unsigned int stolen_size;
+	resource_size_t stolen_size;
 	/* Total number of gtt entries. */
 	unsigned int gtt_total_entries;
 	/* Part of the gtt that is mappable by the cpu, for those chips where
@@ -333,13 +333,13 @@ static void i810_write_entry(dma_addr_t addr, unsigned int entry,
 	writel_relaxed(addr | pte_flags, intel_private.gtt + entry);
 }
 
-static unsigned int intel_gtt_stolen_size(void)
+static resource_size_t intel_gtt_stolen_size(void)
 {
 	u16 gmch_ctrl;
 	u8 rdct;
 	int local = 0;
 	static const int ddt[4] = { 0, 16, 32, 64 };
-	unsigned int stolen_size = 0;
+	resource_size_t stolen_size = 0;
 
 	if (INTEL_GTT_GEN == 1)
 		return 0; /* no stolen mem on i81x */
@@ -417,8 +417,8 @@ static unsigned int intel_gtt_stolen_size(void)
 	}
 
 	if (stolen_size > 0) {
-		dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
-		       stolen_size / KB(1), local ? "local" : "stolen");
+		dev_info(&intel_private.bridge_dev->dev, "detected %lluK %s memory\n",
+		       (u64)stolen_size / KB(1), local ? "local" : "stolen");
 	} else {
 		dev_info(&intel_private.bridge_dev->dev,
 		       "no pre-allocated video memory detected\n");
@@ -872,6 +872,8 @@ void intel_gtt_insert_sg_entries(struct sg_table *st,
 		}
 	}
 	wmb();
+	if (intel_private.driver->chipset_flush)
+		intel_private.driver->chipset_flush();
 }
 EXPORT_SYMBOL(intel_gtt_insert_sg_entries);
 
@@ -1422,12 +1424,10 @@ int intel_gmch_probe(struct pci_dev *bridge_pdev, struct pci_dev *gpu_pdev,
 EXPORT_SYMBOL(intel_gmch_probe);
 
 void intel_gtt_get(u64 *gtt_total,
-		   u32 *stolen_size,
 		   phys_addr_t *mappable_base,
-		   u64 *mappable_end)
+		   resource_size_t *mappable_end)
 {
 	*gtt_total = intel_private.gtt_total_entries << PAGE_SHIFT;
-	*stolen_size = intel_private.stolen_size;
 	*mappable_base = intel_private.gma_bus_addr;
 	*mappable_end = intel_private.gtt_mappable_entries << PAGE_SHIFT;
 }
diff --git a/drivers/char/agp/nvidia-agp.c b/drivers/char/agp/nvidia-agp.c
index 828b34445203..623205bcd04a 100644
--- a/drivers/char/agp/nvidia-agp.c
+++ b/drivers/char/agp/nvidia-agp.c
@@ -340,11 +340,17 @@ static int agp_nvidia_probe(struct pci_dev *pdev,
 	u8 cap_ptr;
 
 	nvidia_private.dev_1 =
-		pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 1));
+		pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
+					    (unsigned int)pdev->bus->number,
+					    PCI_DEVFN(0, 1));
 	nvidia_private.dev_2 =
-		pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(0, 2));
+		pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
+					    (unsigned int)pdev->bus->number,
+					    PCI_DEVFN(0, 2));
 	nvidia_private.dev_3 =
-		pci_get_bus_and_slot((unsigned int)pdev->bus->number, PCI_DEVFN(30, 0));
+		pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
+					    (unsigned int)pdev->bus->number,
+					    PCI_DEVFN(30, 0));
 
 	if (!nvidia_private.dev_1 || !nvidia_private.dev_2 || !nvidia_private.dev_3) {
 		printk(KERN_INFO PFX "Detected an NVIDIA nForce/nForce2 "
diff --git a/drivers/char/agp/sworks-agp.c b/drivers/char/agp/sworks-agp.c
index 03be4ac79b0d..4dbdd3bc9bb8 100644
--- a/drivers/char/agp/sworks-agp.c
+++ b/drivers/char/agp/sworks-agp.c
@@ -474,7 +474,8 @@ static int agp_serverworks_probe(struct pci_dev *pdev,
 	}
 
 	/* Everything is on func 1 here so we are hardcoding function one */
-	bridge_dev = pci_get_bus_and_slot((unsigned int)pdev->bus->number,
+	bridge_dev = pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
+			(unsigned int)pdev->bus->number,
 			PCI_DEVFN(0, 1));
 	if (!bridge_dev) {
 		dev_info(&pdev->dev, "can't find secondary device\n");
diff --git a/drivers/char/apm-emulation.c b/drivers/char/apm-emulation.c
index 1dfb9f8de171..a5e2f9e557ea 100644
--- a/drivers/char/apm-emulation.c
+++ b/drivers/char/apm-emulation.c
@@ -236,12 +236,12 @@ static ssize_t apm_read(struct file *fp, char __user *buf, size_t count, loff_t
 	return ret;
 }
 
-static unsigned int apm_poll(struct file *fp, poll_table * wait)
+static __poll_t apm_poll(struct file *fp, poll_table * wait)
 {
 	struct apm_user *as = fp->private_data;
 
 	poll_wait(fp, &apm_waitqueue, wait);
-	return queue_empty(&as->queue) ? 0 : POLLIN | POLLRDNORM;
+	return queue_empty(&as->queue) ? 0 : EPOLLIN | EPOLLRDNORM;
 }
 
 /*
diff --git a/drivers/char/dsp56k.c b/drivers/char/dsp56k.c
index 0d7b577e0ff0..06749e295ada 100644
--- a/drivers/char/dsp56k.c
+++ b/drivers/char/dsp56k.c
@@ -406,7 +406,7 @@ static long dsp56k_ioctl(struct file *file, unsigned int cmd,
  * Do I need this function at all???
  */
 #if 0
-static unsigned int dsp56k_poll(struct file *file, poll_table *wait)
+static __poll_t dsp56k_poll(struct file *file, poll_table *wait)
 {
 	int dev = iminor(file_inode(file)) & 0x0f;
 
@@ -414,7 +414,7 @@ static unsigned int dsp56k_poll(struct file *file, poll_table *wait)
 	{
 	case DSP56K_DEV_56001:
 		/* poll_wait(file, ???, wait); */
-		return POLLIN | POLLRDNORM | POLLOUT;
+		return EPOLLIN | EPOLLRDNORM | EPOLLOUT;
 
 	default:
 		printk("DSP56k driver: Unknown minor device: %d\n", dev);
diff --git a/drivers/char/dtlk.c b/drivers/char/dtlk.c
index 839ee61d352a..f882460b5a44 100644
--- a/drivers/char/dtlk.c
+++ b/drivers/char/dtlk.c
@@ -62,7 +62,7 @@
 #include <linux/uaccess.h>	/* for get_user, etc. */
 #include <linux/wait.h>		/* for wait_queue */
 #include <linux/init.h>		/* for __init, module_{init,exit} */
-#include <linux/poll.h>		/* for POLLIN, etc. */
+#include <linux/poll.h>		/* for EPOLLIN, etc. */
 #include <linux/dtlk.h>		/* local header file for DoubleTalk values */
 
 #ifdef TRACING
@@ -91,7 +91,7 @@ static ssize_t dtlk_read(struct file *, char __user *,
 			 size_t nbytes, loff_t * ppos);
 static ssize_t dtlk_write(struct file *, const char __user *,
 			  size_t nbytes, loff_t * ppos);
-static unsigned int dtlk_poll(struct file *, poll_table *);
+static __poll_t dtlk_poll(struct file *, poll_table *);
 static int dtlk_open(struct inode *, struct file *);
 static int dtlk_release(struct inode *, struct file *);
 static long dtlk_ioctl(struct file *file,
@@ -228,9 +228,9 @@ static ssize_t dtlk_write(struct file *file, const char __user *buf,
 	return -EAGAIN;
 }
 
-static unsigned int dtlk_poll(struct file *file, poll_table * wait)
+static __poll_t dtlk_poll(struct file *file, poll_table * wait)
 {
-	int mask = 0;
+	__poll_t mask = 0;
 	unsigned long expires;
 
 	TRACE_TEXT(" dtlk_poll");
@@ -244,11 +244,11 @@ static unsigned int dtlk_poll(struct file *file, poll_table * wait)
 
 	if (dtlk_has_indexing && dtlk_readable()) {
 	        del_timer(&dtlk_timer);
-		mask = POLLIN | POLLRDNORM;
+		mask = EPOLLIN | EPOLLRDNORM;
 	}
 	if (dtlk_writeable()) {
 	        del_timer(&dtlk_timer);
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 	}
 	/* there are no exception conditions */
 
diff --git a/drivers/char/hpet.c b/drivers/char/hpet.c
index b941e6d59fd6..be426eb2a353 100644
--- a/drivers/char/hpet.c
+++ b/drivers/char/hpet.c
@@ -342,7 +342,7 @@ hpet_read(struct file *file, char __user *buf, size_t count, loff_t * ppos)
 	return retval;
 }
 
-static unsigned int hpet_poll(struct file *file, poll_table * wait)
+static __poll_t hpet_poll(struct file *file, poll_table * wait)
 {
 	unsigned long v;
 	struct hpet_dev *devp;
@@ -359,7 +359,7 @@ static unsigned int hpet_poll(struct file *file, poll_table * wait)
 	spin_unlock_irq(&hpet_lock);
 
 	if (v != 0)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index f6e3e5abc117..4d0f571c15f9 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -73,26 +73,14 @@ config HW_RANDOM_ATMEL
 
 	  If unsure, say Y.
 
-config HW_RANDOM_BCM63XX
-	tristate "Broadcom BCM63xx Random Number Generator support"
-	depends on BCM63XX || BMIPS_GENERIC
-	default HW_RANDOM
-	---help---
-	  This driver provides kernel-side support for the Random Number
-	  Generator hardware found on the Broadcom BCM63xx SoCs.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called bcm63xx-rng
-
-	  If unusure, say Y.
-
 config HW_RANDOM_BCM2835
-	tristate "Broadcom BCM2835 Random Number Generator support"
-	depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X
+	tristate "Broadcom BCM2835/BCM63xx Random Number Generator support"
+	depends on ARCH_BCM2835 || ARCH_BCM_NSP || ARCH_BCM_5301X || \
+		   ARCH_BCM_63XX || BCM63XX || BMIPS_GENERIC
 	default HW_RANDOM
 	---help---
 	  This driver provides kernel-side support for the Random Number
-	  Generator hardware found on the Broadcom BCM2835 SoCs.
+	  Generator hardware found on the Broadcom BCM2835 and BCM63xx SoCs.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called bcm2835-rng
@@ -306,19 +294,6 @@ config HW_RANDOM_POWERNV
 
 	  If unsure, say Y.
 
-config HW_RANDOM_TPM
-	tristate "TPM HW Random Number Generator support"
-	depends on TCG_TPM
-	default HW_RANDOM
-	---help---
-	  This driver provides kernel-side support for the Random Number
-	  Generator in the Trusted Platform Module
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called tpm-rng.
-
-	  If unsure, say Y.
-
 config HW_RANDOM_HISI
 	tristate "Hisilicon Random Number Generator support"
 	depends on HW_RANDOM && ARCH_HISI
@@ -449,6 +424,18 @@ config HW_RANDOM_S390
 
 	  If unsure, say Y.
 
+config HW_RANDOM_EXYNOS
+	tristate "Samsung Exynos True Random Number Generator support"
+	depends on ARCH_EXYNOS || COMPILE_TEST
+	default HW_RANDOM
+	---help---
+	  This driver provides support for the True Random Number
+	  Generator available in Exynos SoCs.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called exynos-trng.
+
+	  If unsure, say Y.
 endif # HW_RANDOM
 
 config UML_RANDOM
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index f3728d008fff..b780370bd4eb 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -9,11 +9,11 @@ obj-$(CONFIG_HW_RANDOM_TIMERIOMEM) += timeriomem-rng.o
 obj-$(CONFIG_HW_RANDOM_INTEL) += intel-rng.o
 obj-$(CONFIG_HW_RANDOM_AMD) += amd-rng.o
 obj-$(CONFIG_HW_RANDOM_ATMEL) += atmel-rng.o
-obj-$(CONFIG_HW_RANDOM_BCM63XX)	+= bcm63xx-rng.o
 obj-$(CONFIG_HW_RANDOM_GEODE) += geode-rng.o
 obj-$(CONFIG_HW_RANDOM_N2RNG) += n2-rng.o
 n2-rng-y := n2-drv.o n2-asm.o
 obj-$(CONFIG_HW_RANDOM_VIA) += via-rng.o
+obj-$(CONFIG_HW_RANDOM_EXYNOS) += exynos-trng.o
 obj-$(CONFIG_HW_RANDOM_IXP4XX) += ixp4xx-rng.o
 obj-$(CONFIG_HW_RANDOM_OMAP) += omap-rng.o
 obj-$(CONFIG_HW_RANDOM_OMAP3_ROM) += omap3-rom-rng.o
@@ -27,7 +27,6 @@ obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o
 obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o
 obj-$(CONFIG_HW_RANDOM_POWERNV) += powernv-rng.o
 obj-$(CONFIG_HW_RANDOM_HISI)	+= hisi-rng.o
-obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o
 obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o
 obj-$(CONFIG_HW_RANDOM_IPROC_RNG200) += iproc-rng200.o
 obj-$(CONFIG_HW_RANDOM_MSM) += msm-rng.o
diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c
index 574211a49549..7a84cec30c3a 100644
--- a/drivers/char/hw_random/bcm2835-rng.c
+++ b/drivers/char/hw_random/bcm2835-rng.c
@@ -15,6 +15,7 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/printk.h>
+#include <linux/clk.h>
 
 #define RNG_CTRL	0x0
 #define RNG_STATUS	0x4
@@ -29,116 +30,180 @@
 
 #define RNG_INT_OFF	0x1
 
-static void __init nsp_rng_init(void __iomem *base)
+struct bcm2835_rng_priv {
+	struct hwrng rng;
+	void __iomem *base;
+	bool mask_interrupts;
+	struct clk *clk;
+};
+
+static inline struct bcm2835_rng_priv *to_rng_priv(struct hwrng *rng)
 {
-	u32 val;
+	return container_of(rng, struct bcm2835_rng_priv, rng);
+}
+
+static inline u32 rng_readl(struct bcm2835_rng_priv *priv, u32 offset)
+{
+	/* MIPS chips strapped for BE will automagically configure the
+	 * peripheral registers for CPU-native byte order.
+	 */
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		return __raw_readl(priv->base + offset);
+	else
+		return readl(priv->base + offset);
+}
 
-	/* mask the interrupt */
-	val = readl(base + RNG_INT_MASK);
-	val |= RNG_INT_OFF;
-	writel(val, base + RNG_INT_MASK);
+static inline void rng_writel(struct bcm2835_rng_priv *priv, u32 val,
+			      u32 offset)
+{
+	if (IS_ENABLED(CONFIG_MIPS) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
+		__raw_writel(val, priv->base + offset);
+	else
+		writel(val, priv->base + offset);
 }
 
 static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max,
 			       bool wait)
 {
-	void __iomem *rng_base = (void __iomem *)rng->priv;
+	struct bcm2835_rng_priv *priv = to_rng_priv(rng);
 	u32 max_words = max / sizeof(u32);
 	u32 num_words, count;
 
-	while ((__raw_readl(rng_base + RNG_STATUS) >> 24) == 0) {
+	while ((rng_readl(priv, RNG_STATUS) >> 24) == 0) {
 		if (!wait)
 			return 0;
 		cpu_relax();
 	}
 
-	num_words = readl(rng_base + RNG_STATUS) >> 24;
+	num_words = rng_readl(priv, RNG_STATUS) >> 24;
 	if (num_words > max_words)
 		num_words = max_words;
 
 	for (count = 0; count < num_words; count++)
-		((u32 *)buf)[count] = readl(rng_base + RNG_DATA);
+		((u32 *)buf)[count] = rng_readl(priv, RNG_DATA);
 
 	return num_words * sizeof(u32);
 }
 
-static struct hwrng bcm2835_rng_ops = {
-	.name	= "bcm2835",
-	.read	= bcm2835_rng_read,
+static int bcm2835_rng_init(struct hwrng *rng)
+{
+	struct bcm2835_rng_priv *priv = to_rng_priv(rng);
+	int ret = 0;
+	u32 val;
+
+	if (!IS_ERR(priv->clk)) {
+		ret = clk_prepare_enable(priv->clk);
+		if (ret)
+			return ret;
+	}
+
+	if (priv->mask_interrupts) {
+		/* mask the interrupt */
+		val = rng_readl(priv, RNG_INT_MASK);
+		val |= RNG_INT_OFF;
+		rng_writel(priv, val, RNG_INT_MASK);
+	}
+
+	/* set warm-up count & enable */
+	rng_writel(priv, RNG_WARMUP_COUNT, RNG_STATUS);
+	rng_writel(priv, RNG_RBGEN, RNG_CTRL);
+
+	return ret;
+}
+
+static void bcm2835_rng_cleanup(struct hwrng *rng)
+{
+	struct bcm2835_rng_priv *priv = to_rng_priv(rng);
+
+	/* disable rng hardware */
+	rng_writel(priv, 0, RNG_CTRL);
+
+	if (!IS_ERR(priv->clk))
+		clk_disable_unprepare(priv->clk);
+}
+
+struct bcm2835_rng_of_data {
+	bool mask_interrupts;
+};
+
+static const struct bcm2835_rng_of_data nsp_rng_of_data = {
+	.mask_interrupts = true,
 };
 
 static const struct of_device_id bcm2835_rng_of_match[] = {
 	{ .compatible = "brcm,bcm2835-rng"},
-	{ .compatible = "brcm,bcm-nsp-rng", .data = nsp_rng_init},
-	{ .compatible = "brcm,bcm5301x-rng", .data = nsp_rng_init},
+	{ .compatible = "brcm,bcm-nsp-rng", .data = &nsp_rng_of_data },
+	{ .compatible = "brcm,bcm5301x-rng", .data = &nsp_rng_of_data },
+	{ .compatible = "brcm,bcm6368-rng"},
 	{},
 };
 
 static int bcm2835_rng_probe(struct platform_device *pdev)
 {
+	const struct bcm2835_rng_of_data *of_data;
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
-	void (*rng_setup)(void __iomem *base);
 	const struct of_device_id *rng_id;
-	void __iomem *rng_base;
+	struct bcm2835_rng_priv *priv;
+	struct resource *r;
 	int err;
 
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, priv);
+
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
 	/* map peripheral */
-	rng_base = of_iomap(np, 0);
-	if (!rng_base) {
-		dev_err(dev, "failed to remap rng regs");
-		return -ENODEV;
-	}
-	bcm2835_rng_ops.priv = (unsigned long)rng_base;
+	priv->base = devm_ioremap_resource(dev, r);
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	/* Clock is optional on most platforms */
+	priv->clk = devm_clk_get(dev, NULL);
+
+	priv->rng.name = pdev->name;
+	priv->rng.init = bcm2835_rng_init;
+	priv->rng.read = bcm2835_rng_read;
+	priv->rng.cleanup = bcm2835_rng_cleanup;
 
 	rng_id = of_match_node(bcm2835_rng_of_match, np);
-	if (!rng_id) {
-		iounmap(rng_base);
+	if (!rng_id)
 		return -EINVAL;
-	}
-	/* Check for rng init function, execute it */
-	rng_setup = rng_id->data;
-	if (rng_setup)
-		rng_setup(rng_base);
 
-	/* set warm-up count & enable */
-	__raw_writel(RNG_WARMUP_COUNT, rng_base + RNG_STATUS);
-	__raw_writel(RNG_RBGEN, rng_base + RNG_CTRL);
+	/* Check for rng init function, execute it */
+	of_data = rng_id->data;
+	if (of_data)
+		priv->mask_interrupts = of_data->mask_interrupts;
 
 	/* register driver */
-	err = hwrng_register(&bcm2835_rng_ops);
-	if (err) {
+	err = devm_hwrng_register(dev, &priv->rng);
+	if (err)
 		dev_err(dev, "hwrng registration failed\n");
-		iounmap(rng_base);
-	} else
+	else
 		dev_info(dev, "hwrng registered\n");
 
 	return err;
 }
 
-static int bcm2835_rng_remove(struct platform_device *pdev)
-{
-	void __iomem *rng_base = (void __iomem *)bcm2835_rng_ops.priv;
-
-	/* disable rng hardware */
-	__raw_writel(0, rng_base + RNG_CTRL);
-
-	/* unregister driver */
-	hwrng_unregister(&bcm2835_rng_ops);
-	iounmap(rng_base);
-
-	return 0;
-}
-
 MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match);
 
+static struct platform_device_id bcm2835_rng_devtype[] = {
+	{ .name = "bcm2835-rng" },
+	{ .name = "bcm63xx-rng" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(platform, bcm2835_rng_devtype);
+
 static struct platform_driver bcm2835_rng_driver = {
 	.driver = {
 		.name = "bcm2835-rng",
 		.of_match_table = bcm2835_rng_of_match,
 	},
 	.probe		= bcm2835_rng_probe,
-	.remove		= bcm2835_rng_remove,
+	.id_table	= bcm2835_rng_devtype,
 };
 module_platform_driver(bcm2835_rng_driver);
 
diff --git a/drivers/char/hw_random/bcm63xx-rng.c b/drivers/char/hw_random/bcm63xx-rng.c
deleted file mode 100644
index 5132c9cde50d..000000000000
--- a/drivers/char/hw_random/bcm63xx-rng.c
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Broadcom BCM63xx Random Number Generator support
- *
- * Copyright (C) 2011, Florian Fainelli <florian@openwrt.org>
- * Copyright (C) 2009, Broadcom Corporation
- *
- */
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/io.h>
-#include <linux/err.h>
-#include <linux/clk.h>
-#include <linux/platform_device.h>
-#include <linux/hw_random.h>
-#include <linux/of.h>
-
-#define RNG_CTRL			0x00
-#define RNG_EN				(1 << 0)
-
-#define RNG_STAT			0x04
-#define RNG_AVAIL_MASK			(0xff000000)
-
-#define RNG_DATA			0x08
-#define RNG_THRES			0x0c
-#define RNG_MASK			0x10
-
-struct bcm63xx_rng_priv {
-	struct hwrng rng;
-	struct clk *clk;
-	void __iomem *regs;
-};
-
-#define to_rng_priv(rng)	container_of(rng, struct bcm63xx_rng_priv, rng)
-
-static int bcm63xx_rng_init(struct hwrng *rng)
-{
-	struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
-	u32 val;
-	int error;
-
-	error = clk_prepare_enable(priv->clk);
-	if (error)
-		return error;
-
-	val = __raw_readl(priv->regs + RNG_CTRL);
-	val |= RNG_EN;
-	__raw_writel(val, priv->regs + RNG_CTRL);
-
-	return 0;
-}
-
-static void bcm63xx_rng_cleanup(struct hwrng *rng)
-{
-	struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
-	u32 val;
-
-	val = __raw_readl(priv->regs + RNG_CTRL);
-	val &= ~RNG_EN;
-	__raw_writel(val, priv->regs + RNG_CTRL);
-
-	clk_disable_unprepare(priv->clk);
-}
-
-static int bcm63xx_rng_data_present(struct hwrng *rng, int wait)
-{
-	struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
-
-	return __raw_readl(priv->regs + RNG_STAT) & RNG_AVAIL_MASK;
-}
-
-static int bcm63xx_rng_data_read(struct hwrng *rng, u32 *data)
-{
-	struct bcm63xx_rng_priv *priv = to_rng_priv(rng);
-
-	*data = __raw_readl(priv->regs + RNG_DATA);
-
-	return 4;
-}
-
-static int bcm63xx_rng_probe(struct platform_device *pdev)
-{
-	struct resource *r;
-	int ret;
-	struct bcm63xx_rng_priv *priv;
-
-	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!r) {
-		dev_err(&pdev->dev, "no iomem resource\n");
-		return -ENXIO;
-	}
-
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->rng.name = pdev->name;
-	priv->rng.init = bcm63xx_rng_init;
-	priv->rng.cleanup = bcm63xx_rng_cleanup;
-	priv->rng.data_present = bcm63xx_rng_data_present;
-	priv->rng.data_read = bcm63xx_rng_data_read;
-
-	priv->clk = devm_clk_get(&pdev->dev, "ipsec");
-	if (IS_ERR(priv->clk)) {
-		ret = PTR_ERR(priv->clk);
-		dev_err(&pdev->dev, "no clock for device: %d\n", ret);
-		return ret;
-	}
-
-	if (!devm_request_mem_region(&pdev->dev, r->start,
-					resource_size(r), pdev->name)) {
-		dev_err(&pdev->dev, "request mem failed");
-		return -EBUSY;
-	}
-
-	priv->regs = devm_ioremap_nocache(&pdev->dev, r->start,
-					resource_size(r));
-	if (!priv->regs) {
-		dev_err(&pdev->dev, "ioremap failed");
-		return -ENOMEM;
-	}
-
-	ret = devm_hwrng_register(&pdev->dev, &priv->rng);
-	if (ret) {
-		dev_err(&pdev->dev, "failed to register rng device: %d\n",
-			ret);
-		return ret;
-	}
-
-	dev_info(&pdev->dev, "registered RNG driver\n");
-
-	return 0;
-}
-
-#ifdef CONFIG_OF
-static const struct of_device_id bcm63xx_rng_of_match[] = {
-	{ .compatible = "brcm,bcm6368-rng", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, bcm63xx_rng_of_match);
-#endif
-
-static struct platform_driver bcm63xx_rng_driver = {
-	.probe		= bcm63xx_rng_probe,
-	.driver		= {
-		.name	= "bcm63xx-rng",
-		.of_match_table = of_match_ptr(bcm63xx_rng_of_match),
-	},
-};
-
-module_platform_driver(bcm63xx_rng_driver);
-
-MODULE_AUTHOR("Florian Fainelli <florian@openwrt.org>");
-MODULE_DESCRIPTION("Broadcom BCM63xx RNG driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index 657b8770b6b9..91bb98c42a1c 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -306,6 +306,10 @@ static int enable_best_rng(void)
 		ret = ((new_rng == current_rng) ? 0 : set_current_rng(new_rng));
 		if (!ret)
 			cur_rng_set_by_user = 0;
+	} else {
+		drop_current_rng();
+		cur_rng_set_by_user = 0;
+		ret = 0;
 	}
 
 	return ret;
diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c
new file mode 100644
index 000000000000..1947aed7c044
--- /dev/null
+++ b/drivers/char/hw_random/exynos-trng.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RNG driver for Exynos TRNGs
+ *
+ * Author: Łukasz Stelmach <l.stelmach@samsung.com>
+ *
+ * Copyright 2017 (c) Samsung Electronics Software, Inc.
+ *
+ * Based on the Exynos PRNG driver drivers/crypto/exynos-rng by
+ * Krzysztof Kozłowski <krzk@kernel.org>
+ */
+
+#include <linux/clk.h>
+#include <linux/crypto.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/hw_random.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+
+#define EXYNOS_TRNG_CLKDIV         (0x0)
+
+#define EXYNOS_TRNG_CTRL           (0x20)
+#define EXYNOS_TRNG_CTRL_RNGEN     BIT(31)
+
+#define EXYNOS_TRNG_POST_CTRL      (0x30)
+#define EXYNOS_TRNG_ONLINE_CTRL    (0x40)
+#define EXYNOS_TRNG_ONLINE_STAT    (0x44)
+#define EXYNOS_TRNG_ONLINE_MAXCHI2 (0x48)
+#define EXYNOS_TRNG_FIFO_CTRL      (0x50)
+#define EXYNOS_TRNG_FIFO_0         (0x80)
+#define EXYNOS_TRNG_FIFO_1         (0x84)
+#define EXYNOS_TRNG_FIFO_2         (0x88)
+#define EXYNOS_TRNG_FIFO_3         (0x8c)
+#define EXYNOS_TRNG_FIFO_4         (0x90)
+#define EXYNOS_TRNG_FIFO_5         (0x94)
+#define EXYNOS_TRNG_FIFO_6         (0x98)
+#define EXYNOS_TRNG_FIFO_7         (0x9c)
+#define EXYNOS_TRNG_FIFO_LEN       (8)
+#define EXYNOS_TRNG_CLOCK_RATE     (500000)
+
+
+struct exynos_trng_dev {
+	struct device    *dev;
+	void __iomem     *mem;
+	struct clk       *clk;
+	struct hwrng rng;
+};
+
+static int exynos_trng_do_read(struct hwrng *rng, void *data, size_t max,
+			       bool wait)
+{
+	struct exynos_trng_dev *trng;
+	int val;
+
+	max = min_t(size_t, max, (EXYNOS_TRNG_FIFO_LEN * 4));
+
+	trng = (struct exynos_trng_dev *)rng->priv;
+
+	writel_relaxed(max * 8, trng->mem + EXYNOS_TRNG_FIFO_CTRL);
+	val = readl_poll_timeout(trng->mem + EXYNOS_TRNG_FIFO_CTRL, val,
+				 val == 0, 200, 1000000);
+	if (val < 0)
+		return val;
+
+	memcpy_fromio(data, trng->mem + EXYNOS_TRNG_FIFO_0, max);
+
+	return max;
+}
+
+static int exynos_trng_init(struct hwrng *rng)
+{
+	struct exynos_trng_dev *trng = (struct exynos_trng_dev *)rng->priv;
+	unsigned long sss_rate;
+	u32 val;
+
+	sss_rate = clk_get_rate(trng->clk);
+
+	/*
+	 * For most TRNG circuits the clock frequency of under 500 kHz
+	 * is safe.
+	 */
+	val = sss_rate / (EXYNOS_TRNG_CLOCK_RATE * 2);
+	if (val > 0x7fff) {
+		dev_err(trng->dev, "clock divider too large: %d", val);
+		return -ERANGE;
+	}
+	val = val << 1;
+	writel_relaxed(val, trng->mem + EXYNOS_TRNG_CLKDIV);
+
+	/* Enable the generator. */
+	val = EXYNOS_TRNG_CTRL_RNGEN;
+	writel_relaxed(val, trng->mem + EXYNOS_TRNG_CTRL);
+
+	/*
+	 * Disable post-processing. /dev/hwrng is supposed to deliver
+	 * unprocessed data.
+	 */
+	writel_relaxed(0, trng->mem + EXYNOS_TRNG_POST_CTRL);
+
+	return 0;
+}
+
+static int exynos_trng_probe(struct platform_device *pdev)
+{
+	struct exynos_trng_dev *trng;
+	struct resource *res;
+	int ret = -ENOMEM;
+
+	trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL);
+	if (!trng)
+		return ret;
+
+	trng->rng.name = devm_kstrdup(&pdev->dev, dev_name(&pdev->dev),
+				      GFP_KERNEL);
+	if (!trng->rng.name)
+		return ret;
+
+	trng->rng.init = exynos_trng_init;
+	trng->rng.read = exynos_trng_do_read;
+	trng->rng.priv = (unsigned long) trng;
+
+	platform_set_drvdata(pdev, trng);
+	trng->dev = &pdev->dev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	trng->mem = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(trng->mem))
+		return PTR_ERR(trng->mem);
+
+	pm_runtime_enable(&pdev->dev);
+	ret = pm_runtime_get_sync(&pdev->dev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Could not get runtime PM.\n");
+		goto err_pm_get;
+	}
+
+	trng->clk = devm_clk_get(&pdev->dev, "secss");
+	if (IS_ERR(trng->clk)) {
+		ret = PTR_ERR(trng->clk);
+		dev_err(&pdev->dev, "Could not get clock.\n");
+		goto err_clock;
+	}
+
+	ret = clk_prepare_enable(trng->clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Could not enable the clk.\n");
+		goto err_clock;
+	}
+
+	ret = hwrng_register(&trng->rng);
+	if (ret) {
+		dev_err(&pdev->dev, "Could not register hwrng device.\n");
+		goto err_register;
+	}
+
+	dev_info(&pdev->dev, "Exynos True Random Number Generator.\n");
+
+	return 0;
+
+err_register:
+	clk_disable_unprepare(trng->clk);
+
+err_clock:
+	pm_runtime_put_sync(&pdev->dev);
+
+err_pm_get:
+	pm_runtime_disable(&pdev->dev);
+
+	return ret;
+}
+
+static int exynos_trng_remove(struct platform_device *pdev)
+{
+	struct exynos_trng_dev *trng =  platform_get_drvdata(pdev);
+
+	hwrng_unregister(&trng->rng);
+	clk_disable_unprepare(trng->clk);
+
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static int __maybe_unused exynos_trng_suspend(struct device *dev)
+{
+	pm_runtime_put_sync(dev);
+
+	return 0;
+}
+
+static int __maybe_unused exynos_trng_resume(struct device *dev)
+{
+	int ret;
+
+	ret = pm_runtime_get_sync(dev);
+	if (ret < 0) {
+		dev_err(dev, "Could not get runtime PM.\n");
+		pm_runtime_put_noidle(dev);
+		return ret;
+	}
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(exynos_trng_pm_ops, exynos_trng_suspend,
+			 exynos_trng_resume);
+
+static const struct of_device_id exynos_trng_dt_match[] = {
+	{
+		.compatible = "samsung,exynos5250-trng",
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, exynos_trng_dt_match);
+
+static struct platform_driver exynos_trng_driver = {
+	.driver = {
+		.name = "exynos-trng",
+		.pm = &exynos_trng_pm_ops,
+		.of_match_table = exynos_trng_dt_match,
+	},
+	.probe = exynos_trng_probe,
+	.remove = exynos_trng_remove,
+};
+
+module_platform_driver(exynos_trng_driver);
+MODULE_AUTHOR("Łukasz Stelmach");
+MODULE_DESCRIPTION("H/W TRNG driver for Exynos chips");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c
index 88db42d30760..eca87249bcff 100644
--- a/drivers/char/hw_random/imx-rngc.c
+++ b/drivers/char/hw_random/imx-rngc.c
@@ -282,8 +282,7 @@ static int __exit imx_rngc_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM
-static int imx_rngc_suspend(struct device *dev)
+static int __maybe_unused imx_rngc_suspend(struct device *dev)
 {
 	struct imx_rngc *rngc = dev_get_drvdata(dev);
 
@@ -292,7 +291,7 @@ static int imx_rngc_suspend(struct device *dev)
 	return 0;
 }
 
-static int imx_rngc_resume(struct device *dev)
+static int __maybe_unused imx_rngc_resume(struct device *dev)
 {
 	struct imx_rngc *rngc = dev_get_drvdata(dev);
 
@@ -301,11 +300,7 @@ static int imx_rngc_resume(struct device *dev)
 	return 0;
 }
 
-static const struct dev_pm_ops imx_rngc_pm_ops = {
-	.suspend	= imx_rngc_suspend,
-	.resume		= imx_rngc_resume,
-};
-#endif
+SIMPLE_DEV_PM_OPS(imx_rngc_pm_ops, imx_rngc_suspend, imx_rngc_resume);
 
 static const struct of_device_id imx_rngc_dt_ids[] = {
 	{ .compatible = "fsl,imx25-rngb", .data = NULL, },
@@ -316,9 +311,7 @@ MODULE_DEVICE_TABLE(of, imx_rngc_dt_ids);
 static struct platform_driver imx_rngc_driver = {
 	.driver = {
 		.name = "imx_rngc",
-#ifdef CONFIG_PM
 		.pm = &imx_rngc_pm_ops,
-#endif
 		.of_match_table = imx_rngc_dt_ids,
 	},
 	.remove = __exit_p(imx_rngc_remove),
diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c
index 8da7bcf54105..7f99cd52b40e 100644
--- a/drivers/char/hw_random/mtk-rng.c
+++ b/drivers/char/hw_random/mtk-rng.c
@@ -135,6 +135,7 @@ static int mtk_rng_probe(struct platform_device *pdev)
 #endif
 	priv->rng.read = mtk_rng_read;
 	priv->rng.priv = (unsigned long)&pdev->dev;
+	priv->rng.quality = 900;
 
 	priv->clk = devm_clk_get(&pdev->dev, "rng");
 	if (IS_ERR(priv->clk)) {
diff --git a/drivers/char/hw_random/tpm-rng.c b/drivers/char/hw_random/tpm-rng.c
deleted file mode 100644
index d6d448266f07..000000000000
--- a/drivers/char/hw_random/tpm-rng.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright (C) 2012 Kent Yoder IBM Corporation
- *
- * HWRNG interfaces to pull RNG data from a TPM
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- */
-
-#include <linux/module.h>
-#include <linux/hw_random.h>
-#include <linux/tpm.h>
-
-#define MODULE_NAME "tpm-rng"
-
-static int tpm_rng_read(struct hwrng *rng, void *data, size_t max, bool wait)
-{
-	return tpm_get_random(TPM_ANY_NUM, data, max);
-}
-
-static struct hwrng tpm_rng = {
-	.name = MODULE_NAME,
-	.read = tpm_rng_read,
-};
-
-static int __init rng_init(void)
-{
-	return hwrng_register(&tpm_rng);
-}
-module_init(rng_init);
-
-static void __exit rng_exit(void)
-{
-	hwrng_unregister(&tpm_rng);
-}
-module_exit(rng_exit);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Kent Yoder <key@linux.vnet.ibm.com>");
-MODULE_DESCRIPTION("RNG driver for TPM devices");
diff --git a/drivers/char/hw_random/via-rng.c b/drivers/char/hw_random/via-rng.c
index d1f5bb534e0e..6e9df558325b 100644
--- a/drivers/char/hw_random/via-rng.c
+++ b/drivers/char/hw_random/via-rng.c
@@ -162,7 +162,7 @@ static int via_rng_init(struct hwrng *rng)
 	/* Enable secondary noise source on CPUs where it is present. */
 
 	/* Nehemiah stepping 8 and higher */
-	if ((c->x86_model == 9) && (c->x86_mask > 7))
+	if ((c->x86_model == 9) && (c->x86_stepping > 7))
 		lo |= VIA_NOISESRC2;
 
 	/* Esther */
diff --git a/drivers/char/ipmi/bt-bmc.c b/drivers/char/ipmi/bt-bmc.c
index 6edfaa72b98b..c95b93b7598b 100644
--- a/drivers/char/ipmi/bt-bmc.c
+++ b/drivers/char/ipmi/bt-bmc.c
@@ -338,10 +338,10 @@ static int bt_bmc_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int bt_bmc_poll(struct file *file, poll_table *wait)
+static __poll_t bt_bmc_poll(struct file *file, poll_table *wait)
 {
 	struct bt_bmc *bt_bmc = file_bt_bmc(file);
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	u8 ctrl;
 
 	poll_wait(file, &bt_bmc->queue, wait);
@@ -349,10 +349,10 @@ static unsigned int bt_bmc_poll(struct file *file, poll_table *wait)
 	ctrl = bt_inb(bt_bmc, BT_CTRL);
 
 	if (ctrl & BT_CTRL_H2B_ATN)
-		mask |= POLLIN;
+		mask |= EPOLLIN;
 
 	if (!(ctrl & (BT_CTRL_H_BUSY | BT_CTRL_B2H_ATN)))
-		mask |= POLLOUT;
+		mask |= EPOLLOUT;
 
 	return mask;
 }
diff --git a/drivers/char/ipmi/ipmi_devintf.c b/drivers/char/ipmi/ipmi_devintf.c
index 2ffca4232686..5f1bc9174735 100644
--- a/drivers/char/ipmi/ipmi_devintf.c
+++ b/drivers/char/ipmi/ipmi_devintf.c
@@ -78,10 +78,10 @@ static void file_receive_handler(struct ipmi_recv_msg *msg,
 	spin_unlock_irqrestore(&(priv->recv_msg_lock), flags);
 }
 
-static unsigned int ipmi_poll(struct file *file, poll_table *wait)
+static __poll_t ipmi_poll(struct file *file, poll_table *wait)
 {
 	struct ipmi_file_private *priv = file->private_data;
-	unsigned int             mask = 0;
+	__poll_t             mask = 0;
 	unsigned long            flags;
 
 	poll_wait(file, &priv->wait, wait);
@@ -89,7 +89,7 @@ static unsigned int ipmi_poll(struct file *file, poll_table *wait)
 	spin_lock_irqsave(&priv->recv_msg_lock, flags);
 
 	if (!list_empty(&(priv->recv_msgs)))
-		mask |= (POLLIN | POLLRDNORM);
+		mask |= (EPOLLIN | EPOLLRDNORM);
 
 	spin_unlock_irqrestore(&priv->recv_msg_lock, flags);
 
diff --git a/drivers/char/ipmi/ipmi_dmi.c b/drivers/char/ipmi/ipmi_dmi.c
index ab78b3be7e33..c5112b17d7ea 100644
--- a/drivers/char/ipmi/ipmi_dmi.c
+++ b/drivers/char/ipmi/ipmi_dmi.c
@@ -106,7 +106,10 @@ static void __init dmi_add_platform_ipmi(unsigned long base_addr,
 		pr_err("ipmi:dmi: Error allocation IPMI platform device\n");
 		return;
 	}
-	pdev->driver_override = override;
+	pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
+					  override);
+	if (!pdev->driver_override)
+		goto err;
 
 	if (type == IPMI_DMI_TYPE_SSIF) {
 		set_prop_entry(p[pidx++], "i2c-addr", u16, base_addr);
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index f45732a2cb3e..e0b0d7e2d976 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -84,7 +84,7 @@ static int panic_op_write_handler(const char *val,
 	char valcp[16];
 	char *s;
 
-	strncpy(valcp, val, 16);
+	strncpy(valcp, val, 15);
 	valcp[15] = '\0';
 
 	s = strstrip(valcp);
@@ -2588,7 +2588,7 @@ static ssize_t device_id_show(struct device *dev,
 
 	return snprintf(buf, 10, "%u\n", id.device_id);
 }
-static DEVICE_ATTR(device_id, S_IRUGO, device_id_show, NULL);
+static DEVICE_ATTR_RO(device_id);
 
 static ssize_t provides_device_sdrs_show(struct device *dev,
 					 struct device_attribute *attr,
@@ -2604,8 +2604,7 @@ static ssize_t provides_device_sdrs_show(struct device *dev,
 
 	return snprintf(buf, 10, "%u\n", (id.device_revision & 0x80) >> 7);
 }
-static DEVICE_ATTR(provides_device_sdrs, S_IRUGO, provides_device_sdrs_show,
-		   NULL);
+static DEVICE_ATTR_RO(provides_device_sdrs);
 
 static ssize_t revision_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
@@ -2620,7 +2619,7 @@ static ssize_t revision_show(struct device *dev, struct device_attribute *attr,
 
 	return snprintf(buf, 20, "%u\n", id.device_revision & 0x0F);
 }
-static DEVICE_ATTR(revision, S_IRUGO, revision_show, NULL);
+static DEVICE_ATTR_RO(revision);
 
 static ssize_t firmware_revision_show(struct device *dev,
 				      struct device_attribute *attr,
@@ -2637,7 +2636,7 @@ static ssize_t firmware_revision_show(struct device *dev,
 	return snprintf(buf, 20, "%u.%x\n", id.firmware_revision_1,
 			id.firmware_revision_2);
 }
-static DEVICE_ATTR(firmware_revision, S_IRUGO, firmware_revision_show, NULL);
+static DEVICE_ATTR_RO(firmware_revision);
 
 static ssize_t ipmi_version_show(struct device *dev,
 				 struct device_attribute *attr,
@@ -2655,7 +2654,7 @@ static ssize_t ipmi_version_show(struct device *dev,
 			ipmi_version_major(&id),
 			ipmi_version_minor(&id));
 }
-static DEVICE_ATTR(ipmi_version, S_IRUGO, ipmi_version_show, NULL);
+static DEVICE_ATTR_RO(ipmi_version);
 
 static ssize_t add_dev_support_show(struct device *dev,
 				    struct device_attribute *attr,
@@ -2688,7 +2687,7 @@ static ssize_t manufacturer_id_show(struct device *dev,
 
 	return snprintf(buf, 20, "0x%6.6x\n", id.manufacturer_id);
 }
-static DEVICE_ATTR(manufacturer_id, S_IRUGO, manufacturer_id_show, NULL);
+static DEVICE_ATTR_RO(manufacturer_id);
 
 static ssize_t product_id_show(struct device *dev,
 			       struct device_attribute *attr,
@@ -2704,7 +2703,7 @@ static ssize_t product_id_show(struct device *dev,
 
 	return snprintf(buf, 10, "0x%4.4x\n", id.product_id);
 }
-static DEVICE_ATTR(product_id, S_IRUGO, product_id_show, NULL);
+static DEVICE_ATTR_RO(product_id);
 
 static ssize_t aux_firmware_rev_show(struct device *dev,
 				     struct device_attribute *attr,
@@ -2742,7 +2741,7 @@ static ssize_t guid_show(struct device *dev, struct device_attribute *attr,
 
 	return snprintf(buf, 38, "%pUl\n", guid.b);
 }
-static DEVICE_ATTR(guid, S_IRUGO, guid_show, NULL);
+static DEVICE_ATTR_RO(guid);
 
 static struct attribute *bmc_dev_attrs[] = {
 	&dev_attr_device_id.attr,
diff --git a/drivers/char/ipmi/ipmi_powernv.c b/drivers/char/ipmi/ipmi_powernv.c
index 07fddbefefe4..bcf493d8e238 100644
--- a/drivers/char/ipmi/ipmi_powernv.c
+++ b/drivers/char/ipmi/ipmi_powernv.c
@@ -250,8 +250,9 @@ static int ipmi_powernv_probe(struct platform_device *pdev)
 		ipmi->irq = opal_event_request(prop);
 	}
 
-	if (request_irq(ipmi->irq, ipmi_opal_event, IRQ_TYPE_LEVEL_HIGH,
-				"opal-ipmi", ipmi)) {
+	rc = request_irq(ipmi->irq, ipmi_opal_event, IRQ_TYPE_LEVEL_HIGH,
+			 "opal-ipmi", ipmi);
+	if (rc) {
 		dev_warn(dev, "Unable to request irq\n");
 		goto err_dispose;
 	}
@@ -264,7 +265,6 @@ static int ipmi_powernv_probe(struct platform_device *pdev)
 		goto err_unregister;
 	}
 
-	/* todo: query actual ipmi_device_id */
 	rc = ipmi_register_smi(&ipmi_powernv_smi_handlers, ipmi, dev, 0);
 	if (rc) {
 		dev_warn(dev, "IPMI SMI registration failed (%d)\n", rc);
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index 71fad747c0c7..6768cb2dd740 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -1938,8 +1938,10 @@ static void check_for_broken_irqs(struct smi_info *smi_info)
 
 static inline void stop_timer_and_thread(struct smi_info *smi_info)
 {
-	if (smi_info->thread != NULL)
+	if (smi_info->thread != NULL) {
 		kthread_stop(smi_info->thread);
+		smi_info->thread = NULL;
+	}
 
 	smi_info->timer_can_start = false;
 	if (smi_info->timer_running)
@@ -2045,6 +2047,7 @@ static int try_smi_init(struct smi_info *new_smi)
 	int rv = 0;
 	int i;
 	char *init_name = NULL;
+	bool platform_device_registered = false;
 
 	pr_info(PFX "Trying %s-specified %s state machine at %s address 0x%lx, slave address 0x%x, irq %d\n",
 		ipmi_addr_src_to_str(new_smi->io.addr_source),
@@ -2173,6 +2176,7 @@ static int try_smi_init(struct smi_info *new_smi)
 				rv);
 			goto out_err;
 		}
+		platform_device_registered = true;
 	}
 
 	dev_set_drvdata(new_smi->io.dev, new_smi);
@@ -2279,10 +2283,11 @@ static int try_smi_init(struct smi_info *new_smi)
 	}
 
 	if (new_smi->pdev) {
-		platform_device_unregister(new_smi->pdev);
+		if (platform_device_registered)
+			platform_device_unregister(new_smi->pdev);
+		else
+			platform_device_put(new_smi->pdev);
 		new_smi->pdev = NULL;
-	} else if (new_smi->pdev) {
-		platform_device_put(new_smi->pdev);
 	}
 
 	kfree(init_name);
diff --git a/drivers/char/ipmi/ipmi_si_platform.c b/drivers/char/ipmi/ipmi_si_platform.c
index 9573f1116450..f4214870d726 100644
--- a/drivers/char/ipmi/ipmi_si_platform.c
+++ b/drivers/char/ipmi/ipmi_si_platform.c
@@ -40,7 +40,7 @@ MODULE_PARM_DESC(tryacpi, "Setting this to zero will disable the"
 #endif
 #ifdef CONFIG_OF
 module_param_named(tryopenfirmware, si_tryopenfirmware, bool, 0);
-MODULE_PARM_DESC(tryacpi, "Setting this to zero will disable the"
+MODULE_PARM_DESC(tryopenfirmware, "Setting this to zero will disable the"
 		 " default scan of the interfaces identified via OpenFirmware");
 #endif
 #ifdef CONFIG_DMI
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index 3cfaec728604..f929e72bdac8 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -2071,8 +2071,7 @@ static int ssif_platform_remove(struct platform_device *dev)
 		return 0;
 
 	mutex_lock(&ssif_infos_mutex);
-	if (addr_info->client)
-		i2c_unregister_device(addr_info->client);
+	i2c_unregister_device(addr_info->client);
 
 	list_del(&addr_info->link);
 	kfree(addr_info);
diff --git a/drivers/char/ipmi/ipmi_watchdog.c b/drivers/char/ipmi/ipmi_watchdog.c
index 76b270678b50..a58acdcf7414 100644
--- a/drivers/char/ipmi/ipmi_watchdog.c
+++ b/drivers/char/ipmi/ipmi_watchdog.c
@@ -232,7 +232,7 @@ static int set_param_str(const char *val, const struct kernel_param *kp)
 	char       valcp[16];
 	char       *s;
 
-	strncpy(valcp, val, 16);
+	strncpy(valcp, val, 15);
 	valcp[15] = '\0';
 
 	s = strstrip(valcp);
@@ -298,7 +298,7 @@ module_param(pretimeout, timeout, 0644);
 MODULE_PARM_DESC(pretimeout, "Pretimeout value in seconds.");
 
 module_param(panic_wdt_timeout, timeout, 0644);
-MODULE_PARM_DESC(timeout, "Timeout value on kernel panic in seconds.");
+MODULE_PARM_DESC(panic_wdt_timeout, "Timeout value on kernel panic in seconds.");
 
 module_param_cb(action, &param_ops_str, action_op, 0644);
 MODULE_PARM_DESC(action, "Timeout action. One of: "
@@ -887,15 +887,15 @@ static int ipmi_open(struct inode *ino, struct file *filep)
 	}
 }
 
-static unsigned int ipmi_poll(struct file *file, poll_table *wait)
+static __poll_t ipmi_poll(struct file *file, poll_table *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &read_q, wait);
 
 	spin_lock(&ipmi_read_lock);
 	if (data_to_read)
-		mask |= (POLLIN | POLLRDNORM);
+		mask |= (EPOLLIN | EPOLLRDNORM);
 	spin_unlock(&ipmi_read_lock);
 
 	return mask;
diff --git a/drivers/char/lp.c b/drivers/char/lp.c
index 8249762192d5..8c4dd1a3bb6a 100644
--- a/drivers/char/lp.c
+++ b/drivers/char/lp.c
@@ -659,17 +659,31 @@ static int lp_do_ioctl(unsigned int minor, unsigned int cmd,
 	return retval;
 }
 
-static int lp_set_timeout(unsigned int minor, struct timeval *par_timeout)
+static int lp_set_timeout(unsigned int minor, s64 tv_sec, long tv_usec)
 {
 	long to_jiffies;
 
 	/* Convert to jiffies, place in lp_table */
-	if ((par_timeout->tv_sec < 0) ||
-	    (par_timeout->tv_usec < 0)) {
+	if (tv_sec < 0 || tv_usec < 0)
 		return -EINVAL;
+
+	/*
+	 * we used to not check, so let's not make this fatal,
+	 * but deal with user space passing a 32-bit tv_nsec in
+	 * a 64-bit field, capping the timeout to 1 second
+	 * worth of microseconds, and capping the total at
+	 * MAX_JIFFY_OFFSET.
+	 */
+	if (tv_usec > 999999)
+		tv_usec = 999999;
+
+	if (tv_sec >= MAX_SEC_IN_JIFFIES - 1) {
+		to_jiffies = MAX_JIFFY_OFFSET;
+	} else {
+		to_jiffies = DIV_ROUND_UP(tv_usec, 1000000/HZ);
+		to_jiffies += tv_sec * (long) HZ;
 	}
-	to_jiffies = DIV_ROUND_UP(par_timeout->tv_usec, 1000000/HZ);
-	to_jiffies += par_timeout->tv_sec * (long) HZ;
+
 	if (to_jiffies <= 0) {
 		return -EINVAL;
 	}
@@ -677,23 +691,43 @@ static int lp_set_timeout(unsigned int minor, struct timeval *par_timeout)
 	return 0;
 }
 
+static int lp_set_timeout32(unsigned int minor, void __user *arg)
+{
+	s32 karg[2];
+
+	if (copy_from_user(karg, arg, sizeof(karg)))
+		return -EFAULT;
+
+	return lp_set_timeout(minor, karg[0], karg[1]);
+}
+
+static int lp_set_timeout64(unsigned int minor, void __user *arg)
+{
+	s64 karg[2];
+
+	if (copy_from_user(karg, arg, sizeof(karg)))
+		return -EFAULT;
+
+	return lp_set_timeout(minor, karg[0], karg[1]);
+}
+
 static long lp_ioctl(struct file *file, unsigned int cmd,
 			unsigned long arg)
 {
 	unsigned int minor;
-	struct timeval par_timeout;
 	int ret;
 
 	minor = iminor(file_inode(file));
 	mutex_lock(&lp_mutex);
 	switch (cmd) {
-	case LPSETTIMEOUT:
-		if (copy_from_user(&par_timeout, (void __user *)arg,
-					sizeof (struct timeval))) {
-			ret = -EFAULT;
+	case LPSETTIMEOUT_OLD:
+		if (BITS_PER_LONG == 32) {
+			ret = lp_set_timeout32(minor, (void __user *)arg);
 			break;
 		}
-		ret = lp_set_timeout(minor, &par_timeout);
+		/* fallthrough for 64-bit */
+	case LPSETTIMEOUT_NEW:
+		ret = lp_set_timeout64(minor, (void __user *)arg);
 		break;
 	default:
 		ret = lp_do_ioctl(minor, cmd, arg, (void __user *)arg);
@@ -709,18 +743,19 @@ static long lp_compat_ioctl(struct file *file, unsigned int cmd,
 			unsigned long arg)
 {
 	unsigned int minor;
-	struct timeval par_timeout;
 	int ret;
 
 	minor = iminor(file_inode(file));
 	mutex_lock(&lp_mutex);
 	switch (cmd) {
-	case LPSETTIMEOUT:
-		if (compat_get_timeval(&par_timeout, compat_ptr(arg))) {
-			ret = -EFAULT;
+	case LPSETTIMEOUT_OLD:
+		if (!COMPAT_USE_64BIT_TIME) {
+			ret = lp_set_timeout32(minor, (void __user *)arg);
 			break;
 		}
-		ret = lp_set_timeout(minor, &par_timeout);
+		/* fallthrough for x32 mode */
+	case LPSETTIMEOUT_NEW:
+		ret = lp_set_timeout64(minor, (void __user *)arg);
 		break;
 #ifdef LP_STATS
 	case LPGETSTATS:
@@ -865,7 +900,7 @@ static int __init lp_setup (char *str)
 			printk(KERN_INFO "lp: too many ports, %s ignored.\n",
 			       str);
 	} else if (!strcmp(str, "reset")) {
-		reset = 1;
+		reset = true;
 	}
 	return 1;
 }
diff --git a/drivers/char/mem.c b/drivers/char/mem.c
index 6aefe5370e5b..052011bcf100 100644
--- a/drivers/char/mem.c
+++ b/drivers/char/mem.c
@@ -107,6 +107,8 @@ static ssize_t read_mem(struct file *file, char __user *buf,
 	phys_addr_t p = *ppos;
 	ssize_t read, sz;
 	void *ptr;
+	char *bounce;
+	int err;
 
 	if (p != *ppos)
 		return 0;
@@ -129,15 +131,22 @@ static ssize_t read_mem(struct file *file, char __user *buf,
 	}
 #endif
 
+	bounce = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!bounce)
+		return -ENOMEM;
+
 	while (count > 0) {
 		unsigned long remaining;
 		int allowed;
 
 		sz = size_inside_page(p, count);
 
+		err = -EPERM;
 		allowed = page_is_allowed(p >> PAGE_SHIFT);
 		if (!allowed)
-			return -EPERM;
+			goto failed;
+
+		err = -EFAULT;
 		if (allowed == 2) {
 			/* Show zeros for restricted memory. */
 			remaining = clear_user(buf, sz);
@@ -149,24 +158,32 @@ static ssize_t read_mem(struct file *file, char __user *buf,
 			 */
 			ptr = xlate_dev_mem_ptr(p);
 			if (!ptr)
-				return -EFAULT;
-
-			remaining = copy_to_user(buf, ptr, sz);
+				goto failed;
 
+			err = probe_kernel_read(bounce, ptr, sz);
 			unxlate_dev_mem_ptr(p, ptr);
+			if (err)
+				goto failed;
+
+			remaining = copy_to_user(buf, bounce, sz);
 		}
 
 		if (remaining)
-			return -EFAULT;
+			goto failed;
 
 		buf += sz;
 		p += sz;
 		count -= sz;
 		read += sz;
 	}
+	kfree(bounce);
 
 	*ppos += read;
 	return read;
+
+failed:
+	kfree(bounce);
+	return err;
 }
 
 static ssize_t write_mem(struct file *file, const char __user *buf,
diff --git a/drivers/char/pcmcia/cm4040_cs.c b/drivers/char/pcmcia/cm4040_cs.c
index 9a1aaf538758..f80965407d3c 100644
--- a/drivers/char/pcmcia/cm4040_cs.c
+++ b/drivers/char/pcmcia/cm4040_cs.c
@@ -415,17 +415,17 @@ static ssize_t cm4040_write(struct file *filp, const char __user *buf,
 	return count;
 }
 
-static unsigned int cm4040_poll(struct file *filp, poll_table *wait)
+static __poll_t cm4040_poll(struct file *filp, poll_table *wait)
 {
 	struct reader_dev *dev = filp->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(filp, &dev->poll_wait, wait);
 
 	if (test_and_clear_bit(BS_READABLE, &dev->buffer_status))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	if (test_and_clear_bit(BS_WRITABLE, &dev->buffer_status))
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 
 	DEBUGP(2, dev, "<- cm4040_poll(%u)\n", mask);
 
diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c
index d256110ba672..1ae77b41050a 100644
--- a/drivers/char/ppdev.c
+++ b/drivers/char/ppdev.c
@@ -769,14 +769,14 @@ static int pp_release(struct inode *inode, struct file *file)
 }
 
 /* No kernel lock held - fine */
-static unsigned int pp_poll(struct file *file, poll_table *wait)
+static __poll_t pp_poll(struct file *file, poll_table *wait)
 {
 	struct pp_struct *pp = file->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &pp->irq_wait, wait);
 	if (atomic_read(&pp->irqc))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/char/random.c b/drivers/char/random.c
index ec42c8bb9b0d..e5b3d3ba4660 100644
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -431,9 +431,9 @@ static int crng_init = 0;
 static int crng_init_cnt = 0;
 #define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE)
 static void _extract_crng(struct crng_state *crng,
-			  __u8 out[CHACHA20_BLOCK_SIZE]);
+			  __u32 out[CHACHA20_BLOCK_WORDS]);
 static void _crng_backtrack_protect(struct crng_state *crng,
-				    __u8 tmp[CHACHA20_BLOCK_SIZE], int used);
+				    __u32 tmp[CHACHA20_BLOCK_WORDS], int used);
 static void process_random_ready_list(void);
 static void _get_random_bytes(void *buf, int nbytes);
 
@@ -817,7 +817,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
 	unsigned long	flags;
 	int		i, num;
 	union {
-		__u8	block[CHACHA20_BLOCK_SIZE];
+		__u32	block[CHACHA20_BLOCK_WORDS];
 		__u32	key[8];
 	} buf;
 
@@ -851,7 +851,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
 }
 
 static void _extract_crng(struct crng_state *crng,
-			  __u8 out[CHACHA20_BLOCK_SIZE])
+			  __u32 out[CHACHA20_BLOCK_WORDS])
 {
 	unsigned long v, flags;
 
@@ -867,7 +867,7 @@ static void _extract_crng(struct crng_state *crng,
 	spin_unlock_irqrestore(&crng->lock, flags);
 }
 
-static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE])
+static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS])
 {
 	struct crng_state *crng = NULL;
 
@@ -885,7 +885,7 @@ static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE])
  * enough) to mutate the CRNG key to provide backtracking protection.
  */
 static void _crng_backtrack_protect(struct crng_state *crng,
-				    __u8 tmp[CHACHA20_BLOCK_SIZE], int used)
+				    __u32 tmp[CHACHA20_BLOCK_WORDS], int used)
 {
 	unsigned long	flags;
 	__u32		*s, *d;
@@ -897,14 +897,14 @@ static void _crng_backtrack_protect(struct crng_state *crng,
 		used = 0;
 	}
 	spin_lock_irqsave(&crng->lock, flags);
-	s = (__u32 *) &tmp[used];
+	s = &tmp[used / sizeof(__u32)];
 	d = &crng->state[4];
 	for (i=0; i < 8; i++)
 		*d++ ^= *s++;
 	spin_unlock_irqrestore(&crng->lock, flags);
 }
 
-static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used)
+static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used)
 {
 	struct crng_state *crng = NULL;
 
@@ -920,7 +920,7 @@ static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used)
 static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
 {
 	ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE;
-	__u8 tmp[CHACHA20_BLOCK_SIZE];
+	__u32 tmp[CHACHA20_BLOCK_WORDS];
 	int large_request = (nbytes > 256);
 
 	while (nbytes) {
@@ -1507,7 +1507,7 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller,
  */
 static void _get_random_bytes(void *buf, int nbytes)
 {
-	__u8 tmp[CHACHA20_BLOCK_SIZE];
+	__u32 tmp[CHACHA20_BLOCK_WORDS];
 
 	trace_get_random_bytes(nbytes, _RET_IP_);
 
@@ -1784,18 +1784,18 @@ urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
 	return ret;
 }
 
-static unsigned int
+static __poll_t
 random_poll(struct file *file, poll_table * wait)
 {
-	unsigned int mask;
+	__poll_t mask;
 
 	poll_wait(file, &random_read_wait, wait);
 	poll_wait(file, &random_write_wait, wait);
 	mask = 0;
 	if (ENTROPY_BITS(&input_pool) >= random_read_wakeup_bits)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits)
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 	return mask;
 }
 
@@ -2114,7 +2114,7 @@ u64 get_random_u64(void)
 	if (use_lock)
 		read_lock_irqsave(&batched_entropy_reset_lock, flags);
 	if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
-		extract_crng((u8 *)batch->entropy_u64);
+		extract_crng((__u32 *)batch->entropy_u64);
 		batch->position = 0;
 	}
 	ret = batch->entropy_u64[batch->position++];
@@ -2144,7 +2144,7 @@ u32 get_random_u32(void)
 	if (use_lock)
 		read_lock_irqsave(&batched_entropy_reset_lock, flags);
 	if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
-		extract_crng((u8 *)batch->entropy_u32);
+		extract_crng(batch->entropy_u32);
 		batch->position = 0;
 	}
 	ret = batch->entropy_u32[batch->position++];
diff --git a/drivers/char/rtc.c b/drivers/char/rtc.c
index 5542a438bbd0..0c858d027bf3 100644
--- a/drivers/char/rtc.c
+++ b/drivers/char/rtc.c
@@ -147,7 +147,7 @@ static long rtc_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 static void rtc_get_rtc_time(struct rtc_time *rtc_tm);
 
 #ifdef RTC_IRQ
-static unsigned int rtc_poll(struct file *file, poll_table *wait);
+static __poll_t rtc_poll(struct file *file, poll_table *wait);
 #endif
 
 static void get_rtc_alm_time(struct rtc_time *alm_tm);
@@ -790,7 +790,7 @@ static int rtc_release(struct inode *inode, struct file *file)
 }
 
 #ifdef RTC_IRQ
-static unsigned int rtc_poll(struct file *file, poll_table *wait)
+static __poll_t rtc_poll(struct file *file, poll_table *wait)
 {
 	unsigned long l;
 
@@ -804,7 +804,7 @@ static unsigned int rtc_poll(struct file *file, poll_table *wait)
 	spin_unlock_irq(&rtc_lock);
 
 	if (l != 0)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 #endif
diff --git a/drivers/char/snsc.c b/drivers/char/snsc.c
index 6aa32679fd58..5918ea7499bb 100644
--- a/drivers/char/snsc.c
+++ b/drivers/char/snsc.c
@@ -321,10 +321,10 @@ scdrv_write(struct file *file, const char __user *buf,
 	return status;
 }
 
-static unsigned int
+static __poll_t
 scdrv_poll(struct file *file, struct poll_table_struct *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	int status = 0;
 	struct subch_data_s *sd = (struct subch_data_s *) file->private_data;
 	unsigned long flags;
@@ -340,10 +340,10 @@ scdrv_poll(struct file *file, struct poll_table_struct *wait)
 
 	if (status > 0) {
 		if (status & SAL_IROUTER_INTR_RECV) {
-			mask |= POLLIN | POLLRDNORM;
+			mask |= EPOLLIN | EPOLLRDNORM;
 		}
 		if (status & SAL_IROUTER_INTR_XMIT) {
-			mask |= POLLOUT | POLLWRNORM;
+			mask |= EPOLLOUT | EPOLLWRNORM;
 		}
 	}
 
diff --git a/drivers/char/sonypi.c b/drivers/char/sonypi.c
index d3a979e25724..186689833231 100644
--- a/drivers/char/sonypi.c
+++ b/drivers/char/sonypi.c
@@ -940,11 +940,11 @@ static ssize_t sonypi_misc_read(struct file *file, char __user *buf,
 	return ret;
 }
 
-static unsigned int sonypi_misc_poll(struct file *file, poll_table *wait)
+static __poll_t sonypi_misc_poll(struct file *file, poll_table *wait)
 {
 	poll_wait(file, &sonypi_device.fifo_proc_list, wait);
 	if (kfifo_len(&sonypi_device.fifo))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 
diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig
index a30352202f1f..18c81cbe4704 100644
--- a/drivers/char/tpm/Kconfig
+++ b/drivers/char/tpm/Kconfig
@@ -26,6 +26,17 @@ menuconfig TCG_TPM
 
 if TCG_TPM
 
+config HW_RANDOM_TPM
+	bool "TPM HW Random Number Generator support"
+	depends on TCG_TPM && HW_RANDOM && !(TCG_TPM=y && HW_RANDOM=m)
+	default y
+	---help---
+	  This setting exposes the TPM's Random Number Generator as a hwrng
+	  device. This allows the kernel to collect randomness from the TPM at
+	  boot, and provides the TPM randomines in /dev/hwrng.
+
+	  If unsure, say Y.
+
 config TCG_TIS_CORE
 	tristate
 	---help---
diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile
index 34b4bcf46f43..acd758381c58 100644
--- a/drivers/char/tpm/Makefile
+++ b/drivers/char/tpm/Makefile
@@ -6,8 +6,9 @@ obj-$(CONFIG_TCG_TPM) += tpm.o
 tpm-y := tpm-interface.o tpm-dev.o tpm-sysfs.o tpm-chip.o tpm2-cmd.o \
 	 tpm-dev-common.o tpmrm-dev.o tpm1_eventlog.o tpm2_eventlog.o \
          tpm2-space.o
-tpm-$(CONFIG_ACPI) += tpm_ppi.o tpm_acpi.o
-tpm-$(CONFIG_OF) += tpm_of.o
+tpm-$(CONFIG_ACPI) += tpm_ppi.o tpm_eventlog_acpi.o
+tpm-$(CONFIG_EFI) += tpm_eventlog_efi.o
+tpm-$(CONFIG_OF) += tpm_eventlog_of.o
 obj-$(CONFIG_TCG_TIS_CORE) += tpm_tis_core.o
 obj-$(CONFIG_TCG_TIS) += tpm_tis.o
 obj-$(CONFIG_TCG_TIS_SPI) += tpm_tis_spi.o
diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c
index 4d1dc8b46877..f95b9c75175b 100644
--- a/drivers/char/tpm/st33zp24/st33zp24.c
+++ b/drivers/char/tpm/st33zp24/st33zp24.c
@@ -457,7 +457,7 @@ static int st33zp24_recv(struct tpm_chip *chip, unsigned char *buf,
 			    size_t count)
 {
 	int size = 0;
-	int expected;
+	u32 expected;
 
 	if (!chip)
 		return -EBUSY;
@@ -474,7 +474,7 @@ static int st33zp24_recv(struct tpm_chip *chip, unsigned char *buf,
 	}
 
 	expected = be32_to_cpu(*(__be32 *)(buf + 2));
-	if (expected > count) {
+	if (expected > count || expected < TPM_HEADER_SIZE) {
 		size = -EIO;
 		goto out;
 	}
diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c
index 0eca20c5a80c..0a62c19937b6 100644
--- a/drivers/char/tpm/tpm-chip.c
+++ b/drivers/char/tpm/tpm-chip.c
@@ -26,8 +26,9 @@
 #include <linux/spinlock.h>
 #include <linux/freezer.h>
 #include <linux/major.h>
+#include <linux/tpm_eventlog.h>
+#include <linux/hw_random.h>
 #include "tpm.h"
-#include "tpm_eventlog.h"
 
 DEFINE_IDR(dev_nums_idr);
 static DEFINE_MUTEX(idr_lock);
@@ -80,21 +81,26 @@ void tpm_put_ops(struct tpm_chip *chip)
 EXPORT_SYMBOL_GPL(tpm_put_ops);
 
 /**
- * tpm_chip_find_get() - return tpm_chip for a given chip number
- * @chip_num: id to find
+ * tpm_chip_find_get() - find and reserve a TPM chip
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
  *
- * The return'd chip has been tpm_try_get_ops'd and must be released via
- * tpm_put_ops
+ * Finds a TPM chip and reserves its class device and operations. The chip must
+ * be released with tpm_chip_put_ops() after use.
+ *
+ * Return:
+ * A reserved &struct tpm_chip instance.
+ * %NULL if a chip is not found.
+ * %NULL if the chip is not available.
  */
-struct tpm_chip *tpm_chip_find_get(int chip_num)
+struct tpm_chip *tpm_chip_find_get(struct tpm_chip *chip)
 {
-	struct tpm_chip *chip, *res = NULL;
+	struct tpm_chip *res = NULL;
+	int chip_num = 0;
 	int chip_prev;
 
 	mutex_lock(&idr_lock);
 
-	if (chip_num == TPM_ANY_NUM) {
-		chip_num = 0;
+	if (!chip) {
 		do {
 			chip_prev = chip_num;
 			chip = idr_get_next(&dev_nums_idr, &chip_num);
@@ -104,8 +110,7 @@ struct tpm_chip *tpm_chip_find_get(int chip_num)
 			}
 		} while (chip_prev != chip_num);
 	} else {
-		chip = idr_find(&dev_nums_idr, chip_num);
-		if (chip && !tpm_try_get_ops(chip))
+		if (!tpm_try_get_ops(chip))
 			res = chip;
 	}
 
@@ -387,6 +392,26 @@ static int tpm_add_legacy_sysfs(struct tpm_chip *chip)
 
 	return 0;
 }
+
+static int tpm_hwrng_read(struct hwrng *rng, void *data, size_t max, bool wait)
+{
+	struct tpm_chip *chip = container_of(rng, struct tpm_chip, hwrng);
+
+	return tpm_get_random(chip, data, max);
+}
+
+static int tpm_add_hwrng(struct tpm_chip *chip)
+{
+	if (!IS_ENABLED(CONFIG_HW_RANDOM_TPM))
+		return 0;
+
+	snprintf(chip->hwrng_name, sizeof(chip->hwrng_name),
+		 "tpm-rng-%d", chip->dev_num);
+	chip->hwrng.name = chip->hwrng_name;
+	chip->hwrng.read = tpm_hwrng_read;
+	return hwrng_register(&chip->hwrng);
+}
+
 /*
  * tpm_chip_register() - create a character device for the TPM chip
  * @chip: TPM chip to use.
@@ -419,11 +444,13 @@ int tpm_chip_register(struct tpm_chip *chip)
 
 	tpm_add_ppi(chip);
 
+	rc = tpm_add_hwrng(chip);
+	if (rc)
+		goto out_ppi;
+
 	rc = tpm_add_char_device(chip);
-	if (rc) {
-		tpm_bios_log_teardown(chip);
-		return rc;
-	}
+	if (rc)
+		goto out_hwrng;
 
 	rc = tpm_add_legacy_sysfs(chip);
 	if (rc) {
@@ -432,6 +459,14 @@ int tpm_chip_register(struct tpm_chip *chip)
 	}
 
 	return 0;
+
+out_hwrng:
+	if (IS_ENABLED(CONFIG_HW_RANDOM_TPM))
+		hwrng_unregister(&chip->hwrng);
+out_ppi:
+	tpm_bios_log_teardown(chip);
+
+	return rc;
 }
 EXPORT_SYMBOL_GPL(tpm_chip_register);
 
@@ -451,6 +486,8 @@ EXPORT_SYMBOL_GPL(tpm_chip_register);
 void tpm_chip_unregister(struct tpm_chip *chip)
 {
 	tpm_del_legacy_sysfs(chip);
+	if (IS_ENABLED(CONFIG_HW_RANDOM_TPM))
+		hwrng_unregister(&chip->hwrng);
 	tpm_bios_log_teardown(chip);
 	if (chip->flags & TPM_CHIP_FLAG_TPM2)
 		cdev_device_del(&chip->cdevs, &chip->devs);
diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c
index 1d6729be4cd6..9e80a953d693 100644
--- a/drivers/char/tpm/tpm-interface.c
+++ b/drivers/char/tpm/tpm-interface.c
@@ -30,9 +30,9 @@
 #include <linux/spinlock.h>
 #include <linux/freezer.h>
 #include <linux/pm_runtime.h>
+#include <linux/tpm_eventlog.h>
 
 #include "tpm.h"
-#include "tpm_eventlog.h"
 
 #define TPM_MAX_ORDINAL 243
 #define TSC_MAX_ORDINAL 12
@@ -328,7 +328,7 @@ unsigned long tpm_calc_ordinal_duration(struct tpm_chip *chip,
 }
 EXPORT_SYMBOL_GPL(tpm_calc_ordinal_duration);
 
-static bool tpm_validate_command(struct tpm_chip *chip,
+static int tpm_validate_command(struct tpm_chip *chip,
 				 struct tpm_space *space,
 				 const u8 *cmd,
 				 size_t len)
@@ -340,10 +340,10 @@ static bool tpm_validate_command(struct tpm_chip *chip,
 	unsigned int nr_handles;
 
 	if (len < TPM_HEADER_SIZE)
-		return false;
+		return -EINVAL;
 
 	if (!space)
-		return true;
+		return 0;
 
 	if (chip->flags & TPM_CHIP_FLAG_TPM2 && chip->nr_commands) {
 		cc = be32_to_cpu(header->ordinal);
@@ -352,7 +352,7 @@ static bool tpm_validate_command(struct tpm_chip *chip,
 		if (i < 0) {
 			dev_dbg(&chip->dev, "0x%04X is an invalid command\n",
 				cc);
-			return false;
+			return -EOPNOTSUPP;
 		}
 
 		attrs = chip->cc_attrs_tbl[i];
@@ -362,11 +362,11 @@ static bool tpm_validate_command(struct tpm_chip *chip,
 			goto err_len;
 	}
 
-	return true;
+	return 0;
 err_len:
 	dev_dbg(&chip->dev,
 		"%s: insufficient command length %zu", __func__, len);
-	return false;
+	return -EINVAL;
 }
 
 /**
@@ -391,8 +391,20 @@ ssize_t tpm_transmit(struct tpm_chip *chip, struct tpm_space *space,
 	unsigned long stop;
 	bool need_locality;
 
-	if (!tpm_validate_command(chip, space, buf, bufsiz))
-		return -EINVAL;
+	rc = tpm_validate_command(chip, space, buf, bufsiz);
+	if (rc == -EINVAL)
+		return rc;
+	/*
+	 * If the command is not implemented by the TPM, synthesize a
+	 * response with a TPM2_RC_COMMAND_CODE return for user-space.
+	 */
+	if (rc == -EOPNOTSUPP) {
+		header->length = cpu_to_be32(sizeof(*header));
+		header->tag = cpu_to_be16(TPM2_ST_NO_SESSIONS);
+		header->return_code = cpu_to_be32(TPM2_RC_COMMAND_CODE |
+						  TSS2_RESMGR_TPM_RC_LAYER);
+		return bufsiz;
+	}
 
 	if (bufsiz > TPM_BUFSIZE)
 		bufsiz = TPM_BUFSIZE;
@@ -413,6 +425,9 @@ ssize_t tpm_transmit(struct tpm_chip *chip, struct tpm_space *space,
 	if (chip->dev.parent)
 		pm_runtime_get_sync(chip->dev.parent);
 
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, true);
+
 	/* Store the decision as chip->locality will be changed. */
 	need_locality = chip->locality == -1;
 
@@ -489,6 +504,9 @@ ssize_t tpm_transmit(struct tpm_chip *chip, struct tpm_space *space,
 		chip->locality = -1;
 	}
 out_no_locality:
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, false);
+
 	if (chip->dev.parent)
 		pm_runtime_put_sync(chip->dev.parent);
 
@@ -809,19 +827,20 @@ int tpm_pcr_read_dev(struct tpm_chip *chip, int pcr_idx, u8 *res_buf)
 }
 
 /**
- * tpm_is_tpm2 - is the chip a TPM2 chip?
- * @chip_num:	tpm idx # or ANY
+ * tpm_is_tpm2 - do we a have a TPM2 chip?
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
  *
- * Returns < 0 on error, and 1 or 0 on success depending whether the chip
- * is a TPM2 chip.
+ * Return:
+ * 1 if we have a TPM2 chip.
+ * 0 if we don't have a TPM2 chip.
+ * A negative number for system errors (errno).
  */
-int tpm_is_tpm2(u32 chip_num)
+int tpm_is_tpm2(struct tpm_chip *chip)
 {
-	struct tpm_chip *chip;
 	int rc;
 
-	chip = tpm_chip_find_get(chip_num);
-	if (chip == NULL)
+	chip = tpm_chip_find_get(chip);
+	if (!chip)
 		return -ENODEV;
 
 	rc = (chip->flags & TPM_CHIP_FLAG_TPM2) != 0;
@@ -833,23 +852,19 @@ int tpm_is_tpm2(u32 chip_num)
 EXPORT_SYMBOL_GPL(tpm_is_tpm2);
 
 /**
- * tpm_pcr_read - read a pcr value
- * @chip_num:	tpm idx # or ANY
- * @pcr_idx:	pcr idx to retrieve
- * @res_buf:	TPM_PCR value
- *		size of res_buf is 20 bytes (or NULL if you don't care)
+ * tpm_pcr_read - read a PCR value from SHA1 bank
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
+ * @pcr_idx:	the PCR to be retrieved
+ * @res_buf:	the value of the PCR
  *
- * The TPM driver should be built-in, but for whatever reason it
- * isn't, protect against the chip disappearing, by incrementing
- * the module usage count.
+ * Return: same as with tpm_transmit_cmd()
  */
-int tpm_pcr_read(u32 chip_num, int pcr_idx, u8 *res_buf)
+int tpm_pcr_read(struct tpm_chip *chip, int pcr_idx, u8 *res_buf)
 {
-	struct tpm_chip *chip;
 	int rc;
 
-	chip = tpm_chip_find_get(chip_num);
-	if (chip == NULL)
+	chip = tpm_chip_find_get(chip);
+	if (!chip)
 		return -ENODEV;
 	if (chip->flags & TPM_CHIP_FLAG_TPM2)
 		rc = tpm2_pcr_read(chip, pcr_idx, res_buf);
@@ -889,25 +904,26 @@ static int tpm1_pcr_extend(struct tpm_chip *chip, int pcr_idx, const u8 *hash,
 }
 
 /**
- * tpm_pcr_extend - extend pcr value with hash
- * @chip_num:	tpm idx # or AN&
- * @pcr_idx:	pcr idx to extend
- * @hash:	hash value used to extend pcr value
+ * tpm_pcr_extend - extend a PCR value in SHA1 bank.
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
+ * @pcr_idx:	the PCR to be retrieved
+ * @hash:	the hash value used to extend the PCR value
+ *
+ * Note: with TPM 2.0 extends also those banks with a known digest size to the
+ * cryto subsystem in order to prevent malicious use of those PCR banks. In the
+ * future we should dynamically determine digest sizes.
  *
- * The TPM driver should be built-in, but for whatever reason it
- * isn't, protect against the chip disappearing, by incrementing
- * the module usage count.
+ * Return: same as with tpm_transmit_cmd()
  */
-int tpm_pcr_extend(u32 chip_num, int pcr_idx, const u8 *hash)
+int tpm_pcr_extend(struct tpm_chip *chip, int pcr_idx, const u8 *hash)
 {
 	int rc;
-	struct tpm_chip *chip;
 	struct tpm2_digest digest_list[ARRAY_SIZE(chip->active_banks)];
 	u32 count = 0;
 	int i;
 
-	chip = tpm_chip_find_get(chip_num);
-	if (chip == NULL)
+	chip = tpm_chip_find_get(chip);
+	if (!chip)
 		return -ENODEV;
 
 	if (chip->flags & TPM_CHIP_FLAG_TPM2) {
@@ -1019,82 +1035,29 @@ int tpm1_auto_startup(struct tpm_chip *chip)
 	return rc;
 }
 
-int tpm_send(u32 chip_num, void *cmd, size_t buflen)
+/**
+ * tpm_send - send a TPM command
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
+ * @cmd:	a TPM command buffer
+ * @buflen:	the length of the TPM command buffer
+ *
+ * Return: same as with tpm_transmit_cmd()
+ */
+int tpm_send(struct tpm_chip *chip, void *cmd, size_t buflen)
 {
-	struct tpm_chip *chip;
 	int rc;
 
-	chip = tpm_chip_find_get(chip_num);
-	if (chip == NULL)
+	chip = tpm_chip_find_get(chip);
+	if (!chip)
 		return -ENODEV;
 
 	rc = tpm_transmit_cmd(chip, NULL, cmd, buflen, 0, 0,
-			      "attempting tpm_cmd");
+			      "attempting to a send a command");
 	tpm_put_ops(chip);
 	return rc;
 }
 EXPORT_SYMBOL_GPL(tpm_send);
 
-static bool wait_for_tpm_stat_cond(struct tpm_chip *chip, u8 mask,
-					bool check_cancel, bool *canceled)
-{
-	u8 status = chip->ops->status(chip);
-
-	*canceled = false;
-	if ((status & mask) == mask)
-		return true;
-	if (check_cancel && chip->ops->req_canceled(chip, status)) {
-		*canceled = true;
-		return true;
-	}
-	return false;
-}
-
-int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout,
-		      wait_queue_head_t *queue, bool check_cancel)
-{
-	unsigned long stop;
-	long rc;
-	u8 status;
-	bool canceled = false;
-
-	/* check current status */
-	status = chip->ops->status(chip);
-	if ((status & mask) == mask)
-		return 0;
-
-	stop = jiffies + timeout;
-
-	if (chip->flags & TPM_CHIP_FLAG_IRQ) {
-again:
-		timeout = stop - jiffies;
-		if ((long)timeout <= 0)
-			return -ETIME;
-		rc = wait_event_interruptible_timeout(*queue,
-			wait_for_tpm_stat_cond(chip, mask, check_cancel,
-					       &canceled),
-			timeout);
-		if (rc > 0) {
-			if (canceled)
-				return -ECANCELED;
-			return 0;
-		}
-		if (rc == -ERESTARTSYS && freezing(current)) {
-			clear_thread_flag(TIF_SIGPENDING);
-			goto again;
-		}
-	} else {
-		do {
-			tpm_msleep(TPM_TIMEOUT);
-			status = chip->ops->status(chip);
-			if ((status & mask) == mask)
-				return 0;
-		} while (time_before(jiffies, stop));
-	}
-	return -ETIME;
-}
-EXPORT_SYMBOL_GPL(wait_for_tpm_stat);
-
 #define TPM_ORD_SAVESTATE 152
 #define SAVESTATE_RESULT_SIZE 10
 
@@ -1187,16 +1150,15 @@ static const struct tpm_input_header tpm_getrandom_header = {
 };
 
 /**
- * tpm_get_random() - Get random bytes from the tpm's RNG
- * @chip_num: A specific chip number for the request or TPM_ANY_NUM
- * @out: destination buffer for the random bytes
- * @max: the max number of bytes to write to @out
+ * tpm_get_random() - get random bytes from the TPM's RNG
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
+ * @out:	destination buffer for the random bytes
+ * @max:	the max number of bytes to write to @out
  *
- * Returns < 0 on error and the number of bytes read on success
+ * Return: same as with tpm_transmit_cmd()
  */
-int tpm_get_random(u32 chip_num, u8 *out, size_t max)
+int tpm_get_random(struct tpm_chip *chip, u8 *out, size_t max)
 {
-	struct tpm_chip *chip;
 	struct tpm_cmd_t tpm_cmd;
 	u32 recd, num_bytes = min_t(u32, max, TPM_MAX_RNG_DATA), rlength;
 	int err, total = 0, retries = 5;
@@ -1205,8 +1167,8 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max)
 	if (!out || !num_bytes || max > TPM_MAX_RNG_DATA)
 		return -EINVAL;
 
-	chip = tpm_chip_find_get(chip_num);
-	if (chip == NULL)
+	chip = tpm_chip_find_get(chip);
+	if (!chip)
 		return -ENODEV;
 
 	if (chip->flags & TPM_CHIP_FLAG_TPM2) {
@@ -1228,6 +1190,10 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max)
 			break;
 
 		recd = be32_to_cpu(tpm_cmd.params.getrandom_out.rng_data_len);
+		if (recd > num_bytes) {
+			total = -EFAULT;
+			break;
+		}
 
 		rlength = be32_to_cpu(tpm_cmd.header.out.length);
 		if (rlength < offsetof(struct tpm_getrandom_out, rng_data) +
@@ -1248,22 +1214,23 @@ int tpm_get_random(u32 chip_num, u8 *out, size_t max)
 EXPORT_SYMBOL_GPL(tpm_get_random);
 
 /**
- * tpm_seal_trusted() - seal a trusted key
- * @chip_num: A specific chip number for the request or TPM_ANY_NUM
- * @options: authentication values and other options
- * @payload: the key data in clear and encrypted form
+ * tpm_seal_trusted() - seal a trusted key payload
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
+ * @options:	authentication values and other options
+ * @payload:	the key data in clear and encrypted form
  *
- * Returns < 0 on error and 0 on success. At the moment, only TPM 2.0 chips
- * are supported.
+ * Note: only TPM 2.0 chip are supported. TPM 1.x implementation is located in
+ * the keyring subsystem.
+ *
+ * Return: same as with tpm_transmit_cmd()
  */
-int tpm_seal_trusted(u32 chip_num, struct trusted_key_payload *payload,
+int tpm_seal_trusted(struct tpm_chip *chip, struct trusted_key_payload *payload,
 		     struct trusted_key_options *options)
 {
-	struct tpm_chip *chip;
 	int rc;
 
-	chip = tpm_chip_find_get(chip_num);
-	if (chip == NULL || !(chip->flags & TPM_CHIP_FLAG_TPM2))
+	chip = tpm_chip_find_get(chip);
+	if (!chip || !(chip->flags & TPM_CHIP_FLAG_TPM2))
 		return -ENODEV;
 
 	rc = tpm2_seal_trusted(chip, payload, options);
@@ -1275,21 +1242,23 @@ EXPORT_SYMBOL_GPL(tpm_seal_trusted);
 
 /**
  * tpm_unseal_trusted() - unseal a trusted key
- * @chip_num: A specific chip number for the request or TPM_ANY_NUM
- * @options: authentication values and other options
- * @payload: the key data in clear and encrypted form
+ * @chip:	a &struct tpm_chip instance, %NULL for the default chip
+ * @options:	authentication values and other options
+ * @payload:	the key data in clear and encrypted form
+ *
+ * Note: only TPM 2.0 chip are supported. TPM 1.x implementation is located in
+ * the keyring subsystem.
  *
- * Returns < 0 on error and 0 on success. At the moment, only TPM 2.0 chips
- * are supported.
+ * Return: same as with tpm_transmit_cmd()
  */
-int tpm_unseal_trusted(u32 chip_num, struct trusted_key_payload *payload,
+int tpm_unseal_trusted(struct tpm_chip *chip,
+		       struct trusted_key_payload *payload,
 		       struct trusted_key_options *options)
 {
-	struct tpm_chip *chip;
 	int rc;
 
-	chip = tpm_chip_find_get(chip_num);
-	if (chip == NULL || !(chip->flags & TPM_CHIP_FLAG_TPM2))
+	chip = tpm_chip_find_get(chip);
+	if (!chip || !(chip->flags & TPM_CHIP_FLAG_TPM2))
 		return -ENODEV;
 
 	rc = tpm2_unseal_trusted(chip, payload, options);
diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h
index 528cffbd49d3..f895fba4e20d 100644
--- a/drivers/char/tpm/tpm.h
+++ b/drivers/char/tpm/tpm.h
@@ -26,6 +26,7 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/fs.h>
+#include <linux/hw_random.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
 #include <linux/platform_device.h>
@@ -34,6 +35,7 @@
 #include <linux/acpi.h>
 #include <linux/cdev.h>
 #include <linux/highmem.h>
+#include <linux/tpm_eventlog.h>
 #include <crypto/hash_info.h>
 
 #ifdef CONFIG_X86
@@ -93,12 +95,17 @@ enum tpm2_structures {
 	TPM2_ST_SESSIONS	= 0x8002,
 };
 
+/* Indicates from what layer of the software stack the error comes from */
+#define TSS2_RC_LAYER_SHIFT	 16
+#define TSS2_RESMGR_TPM_RC_LAYER (11 << TSS2_RC_LAYER_SHIFT)
+
 enum tpm2_return_codes {
 	TPM2_RC_SUCCESS		= 0x0000,
 	TPM2_RC_HASH		= 0x0083, /* RC_FMT1 */
 	TPM2_RC_HANDLE		= 0x008B,
 	TPM2_RC_INITIALIZE	= 0x0100, /* RC_VER1 */
 	TPM2_RC_DISABLED	= 0x0120,
+	TPM2_RC_COMMAND_CODE    = 0x0143,
 	TPM2_RC_TESTING		= 0x090A, /* RC_WARN */
 	TPM2_RC_REFERENCE_H0	= 0x0910,
 };
@@ -210,6 +217,9 @@ struct tpm_chip {
 	int dev_num;		/* /dev/tpm# */
 	unsigned long is_open;	/* only one allowed */
 
+	char hwrng_name[64];
+	struct hwrng hwrng;
+
 	struct mutex tpm_mutex;	/* tpm is processing */
 
 	unsigned long timeout_a; /* jiffies */
@@ -385,10 +395,6 @@ struct tpm_cmd_t {
 	tpm_cmd_params	params;
 } __packed;
 
-struct tpm2_digest {
-	u16 alg_id;
-	u8 digest[SHA512_DIGEST_SIZE];
-} __packed;
 
 /* A string buffer type for constructing TPM commands. This is based on the
  * ideas of string buffer code in security/keys/trusted.h but is heap based
@@ -512,16 +518,14 @@ int tpm_do_selftest(struct tpm_chip *chip);
 unsigned long tpm_calc_ordinal_duration(struct tpm_chip *chip, u32 ordinal);
 int tpm_pm_suspend(struct device *dev);
 int tpm_pm_resume(struct device *dev);
-int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask, unsigned long timeout,
-		      wait_queue_head_t *queue, bool check_cancel);
 
 static inline void tpm_msleep(unsigned int delay_msec)
 {
-	usleep_range(delay_msec * 1000,
-		     (delay_msec * 1000) + TPM_TIMEOUT_RANGE_US);
+	usleep_range((delay_msec * 1000) - TPM_TIMEOUT_RANGE_US,
+		     delay_msec * 1000);
 };
 
-struct tpm_chip *tpm_chip_find_get(int chip_num);
+struct tpm_chip *tpm_chip_find_get(struct tpm_chip *chip);
 __must_check int tpm_try_get_ops(struct tpm_chip *chip);
 void tpm_put_ops(struct tpm_chip *chip);
 
@@ -575,4 +579,34 @@ int tpm2_prepare_space(struct tpm_chip *chip, struct tpm_space *space, u32 cc,
 		       u8 *cmd);
 int tpm2_commit_space(struct tpm_chip *chip, struct tpm_space *space,
 		      u32 cc, u8 *buf, size_t *bufsiz);
+
+extern const struct seq_operations tpm2_binary_b_measurements_seqops;
+
+#if defined(CONFIG_ACPI)
+int tpm_read_log_acpi(struct tpm_chip *chip);
+#else
+static inline int tpm_read_log_acpi(struct tpm_chip *chip)
+{
+	return -ENODEV;
+}
+#endif
+#if defined(CONFIG_OF)
+int tpm_read_log_of(struct tpm_chip *chip);
+#else
+static inline int tpm_read_log_of(struct tpm_chip *chip)
+{
+	return -ENODEV;
+}
+#endif
+#if defined(CONFIG_EFI)
+int tpm_read_log_efi(struct tpm_chip *chip);
+#else
+static inline int tpm_read_log_efi(struct tpm_chip *chip)
+{
+	return -ENODEV;
+}
+#endif
+
+int tpm_bios_log_setup(struct tpm_chip *chip);
+void tpm_bios_log_teardown(struct tpm_chip *chip);
 #endif
diff --git a/drivers/char/tpm/tpm1_eventlog.c b/drivers/char/tpm/tpm1_eventlog.c
index 9a8605e500b5..add798bd69d0 100644
--- a/drivers/char/tpm/tpm1_eventlog.c
+++ b/drivers/char/tpm/tpm1_eventlog.c
@@ -21,13 +21,14 @@
  */
 
 #include <linux/seq_file.h>
+#include <linux/efi.h>
 #include <linux/fs.h>
 #include <linux/security.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/tpm_eventlog.h>
 
 #include "tpm.h"
-#include "tpm_eventlog.h"
 
 
 static const char* tcpa_event_type_strings[] = {
@@ -371,6 +372,10 @@ static int tpm_read_log(struct tpm_chip *chip)
 	if (rc != -ENODEV)
 		return rc;
 
+	rc = tpm_read_log_efi(chip);
+	if (rc != -ENODEV)
+		return rc;
+
 	return tpm_read_log_of(chip);
 }
 
@@ -388,11 +393,13 @@ int tpm_bios_log_setup(struct tpm_chip *chip)
 {
 	const char *name = dev_name(&chip->dev);
 	unsigned int cnt;
+	int log_version;
 	int rc = 0;
 
 	rc = tpm_read_log(chip);
-	if (rc)
+	if (rc < 0)
 		return rc;
+	log_version = rc;
 
 	cnt = 0;
 	chip->bios_dir[cnt] = securityfs_create_dir(name, NULL);
@@ -404,7 +411,7 @@ int tpm_bios_log_setup(struct tpm_chip *chip)
 	cnt++;
 
 	chip->bin_log_seqops.chip = chip;
-	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+	if (log_version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2)
 		chip->bin_log_seqops.seqops =
 			&tpm2_binary_b_measurements_seqops;
 	else
diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c
index f40d20671a78..a700f8f9ead7 100644
--- a/drivers/char/tpm/tpm2-cmd.c
+++ b/drivers/char/tpm/tpm2-cmd.c
@@ -683,6 +683,10 @@ static int tpm2_unseal_cmd(struct tpm_chip *chip,
 	if (!rc) {
 		data_len = be16_to_cpup(
 			(__be16 *) &buf.data[TPM_HEADER_SIZE + 4]);
+		if (data_len < MIN_KEY_SIZE ||  data_len > MAX_KEY_SIZE + 1) {
+			rc = -EFAULT;
+			goto out;
+		}
 
 		rlength = be32_to_cpu(((struct tpm2_cmd *)&buf)
 					->header.out.length);
@@ -849,28 +853,26 @@ static const struct tpm_input_header tpm2_selftest_header = {
 static int tpm2_do_selftest(struct tpm_chip *chip)
 {
 	int rc;
-	unsigned int delay_msec = 20;
+	unsigned int delay_msec = 10;
 	long duration;
 	struct tpm2_cmd cmd;
 
 	duration = jiffies_to_msecs(
 		tpm2_calc_ordinal_duration(chip, TPM2_CC_SELF_TEST));
 
-	while (duration > 0) {
+	while (1) {
 		cmd.header.in = tpm2_selftest_header;
 		cmd.params.selftest_in.full_test = 0;
 
 		rc = tpm_transmit_cmd(chip, NULL, &cmd, TPM2_SELF_TEST_IN_SIZE,
 				      0, 0, "continue selftest");
 
-		if (rc != TPM2_RC_TESTING)
+		if (rc != TPM2_RC_TESTING || delay_msec >= duration)
 			break;
 
-		tpm_msleep(delay_msec);
-		duration -= delay_msec;
-
-		/* wait longer the next round */
+		/* wait longer than before */
 		delay_msec *= 2;
+		tpm_msleep(delay_msec);
 	}
 
 	return rc;
diff --git a/drivers/char/tpm/tpm2_eventlog.c b/drivers/char/tpm/tpm2_eventlog.c
index 34a8afa69138..1ce4411292ba 100644
--- a/drivers/char/tpm/tpm2_eventlog.c
+++ b/drivers/char/tpm/tpm2_eventlog.c
@@ -21,9 +21,9 @@
 #include <linux/security.h>
 #include <linux/module.h>
 #include <linux/slab.h>
+#include <linux/tpm_eventlog.h>
 
 #include "tpm.h"
-#include "tpm_eventlog.h"
 
 /*
  * calc_tpm2_event_size() - calculate the event size, where event
diff --git a/drivers/char/tpm/tpm_acpi.c b/drivers/char/tpm/tpm_eventlog_acpi.c
similarity index 97%
rename from drivers/char/tpm/tpm_acpi.c
rename to drivers/char/tpm/tpm_eventlog_acpi.c
index 169edf3ce86d..66f19e93c216 100644
--- a/drivers/char/tpm/tpm_acpi.c
+++ b/drivers/char/tpm/tpm_eventlog_acpi.c
@@ -25,9 +25,9 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/acpi.h>
+#include <linux/tpm_eventlog.h>
 
 #include "tpm.h"
-#include "tpm_eventlog.h"
 
 struct acpi_tcpa {
 	struct acpi_table_header hdr;
@@ -102,7 +102,7 @@ int tpm_read_log_acpi(struct tpm_chip *chip)
 	memcpy_fromio(log->bios_event_log, virt, len);
 
 	acpi_os_unmap_iomem(virt, len);
-	return 0;
+	return EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
 
 err:
 	kfree(log->bios_event_log);
diff --git a/drivers/char/tpm/tpm_eventlog_efi.c b/drivers/char/tpm/tpm_eventlog_efi.c
new file mode 100644
index 000000000000..e3f9ffd341d2
--- /dev/null
+++ b/drivers/char/tpm/tpm_eventlog_efi.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2017 Google
+ *
+ * Authors:
+ *      Thiebaud Weksteen <tweek@google.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ */
+
+#include <linux/efi.h>
+#include <linux/tpm_eventlog.h>
+
+#include "tpm.h"
+
+/* read binary bios log from EFI configuration table */
+int tpm_read_log_efi(struct tpm_chip *chip)
+{
+
+	struct linux_efi_tpm_eventlog *log_tbl;
+	struct tpm_bios_log *log;
+	u32 log_size;
+	u8 tpm_log_version;
+
+	if (!(chip->flags & TPM_CHIP_FLAG_TPM2))
+		return -ENODEV;
+
+	if (efi.tpm_log == EFI_INVALID_TABLE_ADDR)
+		return -ENODEV;
+
+	log = &chip->log;
+
+	log_tbl = memremap(efi.tpm_log, sizeof(*log_tbl), MEMREMAP_WB);
+	if (!log_tbl) {
+		pr_err("Could not map UEFI TPM log table !\n");
+		return -ENOMEM;
+	}
+
+	log_size = log_tbl->size;
+	memunmap(log_tbl);
+
+	log_tbl = memremap(efi.tpm_log, sizeof(*log_tbl) + log_size,
+			   MEMREMAP_WB);
+	if (!log_tbl) {
+		pr_err("Could not map UEFI TPM log table payload!\n");
+		return -ENOMEM;
+	}
+
+	/* malloc EventLog space */
+	log->bios_event_log = kmalloc(log_size, GFP_KERNEL);
+	if (!log->bios_event_log)
+		goto err_memunmap;
+	memcpy(log->bios_event_log, log_tbl->log, log_size);
+	log->bios_event_log_end = log->bios_event_log + log_size;
+
+	tpm_log_version = log_tbl->version;
+	memunmap(log_tbl);
+	return tpm_log_version;
+
+err_memunmap:
+	memunmap(log_tbl);
+	return -ENOMEM;
+}
diff --git a/drivers/char/tpm/tpm_of.c b/drivers/char/tpm/tpm_eventlog_of.c
similarity index 93%
rename from drivers/char/tpm/tpm_of.c
rename to drivers/char/tpm/tpm_eventlog_of.c
index aadb7f464076..96fd5646f866 100644
--- a/drivers/char/tpm/tpm_of.c
+++ b/drivers/char/tpm/tpm_eventlog_of.c
@@ -17,9 +17,9 @@
 
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/tpm_eventlog.h>
 
 #include "tpm.h"
-#include "tpm_eventlog.h"
 
 int tpm_read_log_of(struct tpm_chip *chip)
 {
@@ -76,5 +76,7 @@ int tpm_read_log_of(struct tpm_chip *chip)
 
 	memcpy(log->bios_event_log, __va(base), size);
 
-	return 0;
+	if (chip->flags & TPM_CHIP_FLAG_TPM2)
+		return EFI_TCG2_EVENT_LOG_FORMAT_TCG_2;
+	return EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
 }
diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c
index 79d6bbb58e39..6116cd05e228 100644
--- a/drivers/char/tpm/tpm_i2c_infineon.c
+++ b/drivers/char/tpm/tpm_i2c_infineon.c
@@ -473,7 +473,8 @@ static int recv_data(struct tpm_chip *chip, u8 *buf, size_t count)
 static int tpm_tis_i2c_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 {
 	int size = 0;
-	int expected, status;
+	int status;
+	u32 expected;
 
 	if (count < TPM_HEADER_SIZE) {
 		size = -EIO;
@@ -488,7 +489,7 @@ static int tpm_tis_i2c_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 	}
 
 	expected = be32_to_cpu(*(__be32 *)(buf + 2));
-	if ((size_t) expected > count) {
+	if (((size_t) expected > count) || (expected < TPM_HEADER_SIZE)) {
 		size = -EIO;
 		goto out;
 	}
@@ -665,9 +666,9 @@ static int tpm_tis_i2c_init(struct device *dev)
 }
 
 static const struct i2c_device_id tpm_tis_i2c_table[] = {
-	{"tpm_i2c_infineon", 0},
-	{"slb9635tt", 0},
-	{"slb9645tt", 1},
+	{"tpm_i2c_infineon"},
+	{"slb9635tt"},
+	{"slb9645tt"},
 	{},
 };
 
@@ -675,24 +676,9 @@ MODULE_DEVICE_TABLE(i2c, tpm_tis_i2c_table);
 
 #ifdef CONFIG_OF
 static const struct of_device_id tpm_tis_i2c_of_match[] = {
-	{
-		.name = "tpm_i2c_infineon",
-		.type = "tpm",
-		.compatible = "infineon,tpm_i2c_infineon",
-		.data = (void *)0
-	},
-	{
-		.name = "slb9635tt",
-		.type = "tpm",
-		.compatible = "infineon,slb9635tt",
-		.data = (void *)0
-	},
-	{
-		.name = "slb9645tt",
-		.type = "tpm",
-		.compatible = "infineon,slb9645tt",
-		.data = (void *)1
-	},
+	{.compatible = "infineon,tpm_i2c_infineon"},
+	{.compatible = "infineon,slb9635tt"},
+	{.compatible = "infineon,slb9645tt"},
 	{},
 };
 MODULE_DEVICE_TABLE(of, tpm_tis_i2c_of_match);
diff --git a/drivers/char/tpm/tpm_i2c_nuvoton.c b/drivers/char/tpm/tpm_i2c_nuvoton.c
index c6428771841f..caa86b19c76d 100644
--- a/drivers/char/tpm/tpm_i2c_nuvoton.c
+++ b/drivers/char/tpm/tpm_i2c_nuvoton.c
@@ -281,7 +281,11 @@ static int i2c_nuvoton_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 	struct device *dev = chip->dev.parent;
 	struct i2c_client *client = to_i2c_client(dev);
 	s32 rc;
-	int expected, status, burst_count, retries, size = 0;
+	int status;
+	int burst_count;
+	int retries;
+	int size = 0;
+	u32 expected;
 
 	if (count < TPM_HEADER_SIZE) {
 		i2c_nuvoton_ready(chip);    /* return to idle */
@@ -323,7 +327,7 @@ static int i2c_nuvoton_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 		 * to machine native
 		 */
 		expected = be32_to_cpu(*(__be32 *) (buf + 2));
-		if (expected > count) {
+		if (expected > count || expected < size) {
 			dev_err(dev, "%s() expected > count\n", __func__);
 			size = -EIO;
 			continue;
diff --git a/drivers/char/tpm/tpm_tis.c b/drivers/char/tpm/tpm_tis.c
index e2d1055fb814..f08949a5f678 100644
--- a/drivers/char/tpm/tpm_tis.c
+++ b/drivers/char/tpm/tpm_tis.c
@@ -133,93 +133,14 @@ static int check_acpi_tpm2(struct device *dev)
 }
 #endif
 
-#ifdef CONFIG_X86
-#define INTEL_LEGACY_BLK_BASE_ADDR      0xFED08000
-#define ILB_REMAP_SIZE			0x100
-#define LPC_CNTRL_REG_OFFSET            0x84
-#define LPC_CLKRUN_EN                   (1 << 2)
-
-static void __iomem *ilb_base_addr;
-
-static inline bool is_bsw(void)
-{
-	return ((boot_cpu_data.x86_model == INTEL_FAM6_ATOM_AIRMONT) ? 1 : 0);
-}
-
-/**
- * tpm_platform_begin_xfer() - clear LPC CLKRUN_EN i.e. clocks will be running
- */
-static void tpm_platform_begin_xfer(void)
-{
-	u32 clkrun_val;
-
-	if (!is_bsw())
-		return;
-
-	clkrun_val = ioread32(ilb_base_addr + LPC_CNTRL_REG_OFFSET);
-
-	/* Disable LPC CLKRUN# */
-	clkrun_val &= ~LPC_CLKRUN_EN;
-	iowrite32(clkrun_val, ilb_base_addr + LPC_CNTRL_REG_OFFSET);
-
-	/*
-	 * Write any random value on port 0x80 which is on LPC, to make
-	 * sure LPC clock is running before sending any TPM command.
-	 */
-	outb(0xCC, 0x80);
-
-}
-
-/**
- * tpm_platform_end_xfer() - set LPC CLKRUN_EN i.e. clocks can be turned off
- */
-static void tpm_platform_end_xfer(void)
-{
-	u32 clkrun_val;
-
-	if (!is_bsw())
-		return;
-
-	clkrun_val = ioread32(ilb_base_addr + LPC_CNTRL_REG_OFFSET);
-
-	/* Enable LPC CLKRUN# */
-	clkrun_val |= LPC_CLKRUN_EN;
-	iowrite32(clkrun_val, ilb_base_addr + LPC_CNTRL_REG_OFFSET);
-
-	/*
-	 * Write any random value on port 0x80 which is on LPC, to make
-	 * sure LPC clock is running before sending any TPM command.
-	 */
-	outb(0xCC, 0x80);
-
-}
-#else
-static inline bool is_bsw(void)
-{
-	return false;
-}
-
-static void tpm_platform_begin_xfer(void)
-{
-}
-
-static void tpm_platform_end_xfer(void)
-{
-}
-#endif
-
 static int tpm_tcg_read_bytes(struct tpm_tis_data *data, u32 addr, u16 len,
 			      u8 *result)
 {
 	struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data);
 
-	tpm_platform_begin_xfer();
-
 	while (len--)
 		*result++ = ioread8(phy->iobase + addr);
 
-	tpm_platform_end_xfer();
-
 	return 0;
 }
 
@@ -228,13 +149,9 @@ static int tpm_tcg_write_bytes(struct tpm_tis_data *data, u32 addr, u16 len,
 {
 	struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data);
 
-	tpm_platform_begin_xfer();
-
 	while (len--)
 		iowrite8(*value++, phy->iobase + addr);
 
-	tpm_platform_end_xfer();
-
 	return 0;
 }
 
@@ -242,12 +159,8 @@ static int tpm_tcg_read16(struct tpm_tis_data *data, u32 addr, u16 *result)
 {
 	struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data);
 
-	tpm_platform_begin_xfer();
-
 	*result = ioread16(phy->iobase + addr);
 
-	tpm_platform_end_xfer();
-
 	return 0;
 }
 
@@ -255,12 +168,8 @@ static int tpm_tcg_read32(struct tpm_tis_data *data, u32 addr, u32 *result)
 {
 	struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data);
 
-	tpm_platform_begin_xfer();
-
 	*result = ioread32(phy->iobase + addr);
 
-	tpm_platform_end_xfer();
-
 	return 0;
 }
 
@@ -268,12 +177,8 @@ static int tpm_tcg_write32(struct tpm_tis_data *data, u32 addr, u32 value)
 {
 	struct tpm_tis_tcg_phy *phy = to_tpm_tis_tcg_phy(data);
 
-	tpm_platform_begin_xfer();
-
 	iowrite32(value, phy->iobase + addr);
 
-	tpm_platform_end_xfer();
-
 	return 0;
 }
 
@@ -461,11 +366,6 @@ static int __init init_tis(void)
 	if (rc)
 		goto err_force;
 
-#ifdef CONFIG_X86
-	if (is_bsw())
-		ilb_base_addr = ioremap(INTEL_LEGACY_BLK_BASE_ADDR,
-					ILB_REMAP_SIZE);
-#endif
 	rc = platform_driver_register(&tis_drv);
 	if (rc)
 		goto err_platform;
@@ -484,10 +384,6 @@ static int __init init_tis(void)
 err_platform:
 	if (force_pdev)
 		platform_device_unregister(force_pdev);
-#ifdef CONFIG_X86
-	if (is_bsw())
-		iounmap(ilb_base_addr);
-#endif
 err_force:
 	return rc;
 }
@@ -497,10 +393,6 @@ static void __exit cleanup_tis(void)
 	pnp_unregister_driver(&tis_pnp_driver);
 	platform_driver_unregister(&tis_drv);
 
-#ifdef CONFIG_X86
-	if (is_bsw())
-		iounmap(ilb_base_addr);
-#endif
 	if (force_pdev)
 		platform_device_unregister(force_pdev);
 }
diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
index fdde971bc810..da074e3db19b 100644
--- a/drivers/char/tpm/tpm_tis_core.c
+++ b/drivers/char/tpm/tpm_tis_core.c
@@ -31,6 +31,74 @@
 #include "tpm.h"
 #include "tpm_tis_core.h"
 
+/* This is a polling delay to check for status and burstcount.
+ * As per ddwg input, expectation is that status check and burstcount
+ * check should return within few usecs.
+ */
+#define TPM_POLL_SLEEP	1  /* msec */
+
+static void tpm_tis_clkrun_enable(struct tpm_chip *chip, bool value);
+
+static bool wait_for_tpm_stat_cond(struct tpm_chip *chip, u8 mask,
+					bool check_cancel, bool *canceled)
+{
+	u8 status = chip->ops->status(chip);
+
+	*canceled = false;
+	if ((status & mask) == mask)
+		return true;
+	if (check_cancel && chip->ops->req_canceled(chip, status)) {
+		*canceled = true;
+		return true;
+	}
+	return false;
+}
+
+static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
+		unsigned long timeout, wait_queue_head_t *queue,
+		bool check_cancel)
+{
+	unsigned long stop;
+	long rc;
+	u8 status;
+	bool canceled = false;
+
+	/* check current status */
+	status = chip->ops->status(chip);
+	if ((status & mask) == mask)
+		return 0;
+
+	stop = jiffies + timeout;
+
+	if (chip->flags & TPM_CHIP_FLAG_IRQ) {
+again:
+		timeout = stop - jiffies;
+		if ((long)timeout <= 0)
+			return -ETIME;
+		rc = wait_event_interruptible_timeout(*queue,
+			wait_for_tpm_stat_cond(chip, mask, check_cancel,
+					       &canceled),
+			timeout);
+		if (rc > 0) {
+			if (canceled)
+				return -ECANCELED;
+			return 0;
+		}
+		if (rc == -ERESTARTSYS && freezing(current)) {
+			clear_thread_flag(TIF_SIGPENDING);
+			goto again;
+		}
+	} else {
+		do {
+			tpm_msleep(TPM_POLL_SLEEP);
+			status = chip->ops->status(chip);
+			if ((status & mask) == mask)
+				return 0;
+		} while (time_before(jiffies, stop));
+	}
+	return -ETIME;
+}
+
 /* Before we attempt to access the TPM we must see that the valid bit is set.
  * The specification says that this bit is 0 at reset and remains 0 until the
  * 'TPM has gone through its self test and initialization and has established
@@ -164,7 +232,7 @@ static int get_burstcount(struct tpm_chip *chip)
 		burstcnt = (value >> 8) & 0xFFFF;
 		if (burstcnt)
 			return burstcnt;
-		tpm_msleep(TPM_TIMEOUT);
+		tpm_msleep(TPM_POLL_SLEEP);
 	} while (time_before(jiffies, stop));
 	return -EBUSY;
 }
@@ -202,7 +270,8 @@ static int tpm_tis_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 {
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
 	int size = 0;
-	int expected, status;
+	int status;
+	u32 expected;
 
 	if (count < TPM_HEADER_SIZE) {
 		size = -EIO;
@@ -217,7 +286,7 @@ static int tpm_tis_recv(struct tpm_chip *chip, u8 *buf, size_t count)
 	}
 
 	expected = be32_to_cpu(*(__be32 *) (buf + 2));
-	if (expected > count) {
+	if (expected > count || expected < TPM_HEADER_SIZE) {
 		size = -EIO;
 		goto out;
 	}
@@ -421,19 +490,28 @@ static bool tpm_tis_update_timeouts(struct tpm_chip *chip,
 	int i, rc;
 	u32 did_vid;
 
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, true);
+
 	rc = tpm_tis_read32(priv, TPM_DID_VID(0), &did_vid);
 	if (rc < 0)
-		return rc;
+		goto out;
 
 	for (i = 0; i != ARRAY_SIZE(vendor_timeout_overrides); i++) {
 		if (vendor_timeout_overrides[i].did_vid != did_vid)
 			continue;
 		memcpy(timeout_cap, vendor_timeout_overrides[i].timeout_us,
 		       sizeof(vendor_timeout_overrides[i].timeout_us));
-		return true;
+		rc = true;
 	}
 
-	return false;
+	rc = false;
+
+out:
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, false);
+
+	return rc;
 }
 
 /*
@@ -653,14 +731,73 @@ void tpm_tis_remove(struct tpm_chip *chip)
 	u32 interrupt;
 	int rc;
 
+	tpm_tis_clkrun_enable(chip, true);
+
 	rc = tpm_tis_read32(priv, reg, &interrupt);
 	if (rc < 0)
 		interrupt = 0;
 
 	tpm_tis_write32(priv, reg, ~TPM_GLOBAL_INT_ENABLE & interrupt);
+
+	tpm_tis_clkrun_enable(chip, false);
+
+	if (priv->ilb_base_addr)
+		iounmap(priv->ilb_base_addr);
 }
 EXPORT_SYMBOL_GPL(tpm_tis_remove);
 
+/**
+ * tpm_tis_clkrun_enable() - Keep clkrun protocol disabled for entire duration
+ *                           of a single TPM command
+ * @chip:	TPM chip to use
+ * @value:	1 - Disable CLKRUN protocol, so that clocks are free running
+ *		0 - Enable CLKRUN protocol
+ * Call this function directly in tpm_tis_remove() in error or driver removal
+ * path, since the chip->ops is set to NULL in tpm_chip_unregister().
+ */
+static void tpm_tis_clkrun_enable(struct tpm_chip *chip, bool value)
+{
+	struct tpm_tis_data *data = dev_get_drvdata(&chip->dev);
+	u32 clkrun_val;
+
+	if (!IS_ENABLED(CONFIG_X86) || !is_bsw() ||
+	    !data->ilb_base_addr)
+		return;
+
+	if (value) {
+		data->clkrun_enabled++;
+		if (data->clkrun_enabled > 1)
+			return;
+		clkrun_val = ioread32(data->ilb_base_addr + LPC_CNTRL_OFFSET);
+
+		/* Disable LPC CLKRUN# */
+		clkrun_val &= ~LPC_CLKRUN_EN;
+		iowrite32(clkrun_val, data->ilb_base_addr + LPC_CNTRL_OFFSET);
+
+		/*
+		 * Write any random value on port 0x80 which is on LPC, to make
+		 * sure LPC clock is running before sending any TPM command.
+		 */
+		outb(0xCC, 0x80);
+	} else {
+		data->clkrun_enabled--;
+		if (data->clkrun_enabled)
+			return;
+
+		clkrun_val = ioread32(data->ilb_base_addr + LPC_CNTRL_OFFSET);
+
+		/* Enable LPC CLKRUN# */
+		clkrun_val |= LPC_CLKRUN_EN;
+		iowrite32(clkrun_val, data->ilb_base_addr + LPC_CNTRL_OFFSET);
+
+		/*
+		 * Write any random value on port 0x80 which is on LPC, to make
+		 * sure LPC clock is running before sending any TPM command.
+		 */
+		outb(0xCC, 0x80);
+	}
+}
+
 static const struct tpm_class_ops tpm_tis = {
 	.flags = TPM_OPS_AUTO_STARTUP,
 	.status = tpm_tis_status,
@@ -673,13 +810,17 @@ static const struct tpm_class_ops tpm_tis = {
 	.req_canceled = tpm_tis_req_canceled,
 	.request_locality = request_locality,
 	.relinquish_locality = release_locality,
+	.clk_enable = tpm_tis_clkrun_enable,
 };
 
 int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 		      const struct tpm_tis_phy_ops *phy_ops,
 		      acpi_handle acpi_dev_handle)
 {
-	u32 vendor, intfcaps, intmask;
+	u32 vendor;
+	u32 intfcaps;
+	u32 intmask;
+	u32 clkrun_val;
 	u8 rid;
 	int rc, probe;
 	struct tpm_chip *chip;
@@ -700,6 +841,23 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 	priv->phy_ops = phy_ops;
 	dev_set_drvdata(&chip->dev, priv);
 
+	if (is_bsw()) {
+		priv->ilb_base_addr = ioremap(INTEL_LEGACY_BLK_BASE_ADDR,
+					ILB_REMAP_SIZE);
+		if (!priv->ilb_base_addr)
+			return -ENOMEM;
+
+		clkrun_val = ioread32(priv->ilb_base_addr + LPC_CNTRL_OFFSET);
+		/* Check if CLKRUN# is already not enabled in the LPC bus */
+		if (!(clkrun_val & LPC_CLKRUN_EN)) {
+			iounmap(priv->ilb_base_addr);
+			priv->ilb_base_addr = NULL;
+		}
+	}
+
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, true);
+
 	if (wait_startup(chip, 0) != 0) {
 		rc = -ENODEV;
 		goto out_err;
@@ -790,9 +948,20 @@ int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 		}
 	}
 
-	return tpm_chip_register(chip);
+	rc = tpm_chip_register(chip);
+	if (rc)
+		goto out_err;
+
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, false);
+
+	return 0;
 out_err:
+	if ((chip->ops != NULL) && (chip->ops->clk_enable != NULL))
+		chip->ops->clk_enable(chip, false);
+
 	tpm_tis_remove(chip);
+
 	return rc;
 }
 EXPORT_SYMBOL_GPL(tpm_tis_core_init);
@@ -804,22 +973,31 @@ static void tpm_tis_reenable_interrupts(struct tpm_chip *chip)
 	u32 intmask;
 	int rc;
 
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, true);
+
 	/* reenable interrupts that device may have lost or
 	 * BIOS/firmware may have disabled
 	 */
 	rc = tpm_tis_write8(priv, TPM_INT_VECTOR(priv->locality), priv->irq);
 	if (rc < 0)
-		return;
+		goto out;
 
 	rc = tpm_tis_read32(priv, TPM_INT_ENABLE(priv->locality), &intmask);
 	if (rc < 0)
-		return;
+		goto out;
 
 	intmask |= TPM_INTF_CMD_READY_INT
 	    | TPM_INTF_LOCALITY_CHANGE_INT | TPM_INTF_DATA_AVAIL_INT
 	    | TPM_INTF_STS_VALID_INT | TPM_GLOBAL_INT_ENABLE;
 
 	tpm_tis_write32(priv, TPM_INT_ENABLE(priv->locality), intmask);
+
+out:
+	if (chip->ops->clk_enable != NULL)
+		chip->ops->clk_enable(chip, false);
+
+	return;
 }
 
 int tpm_tis_resume(struct device *dev)
diff --git a/drivers/char/tpm/tpm_tis_core.h b/drivers/char/tpm/tpm_tis_core.h
index 6bbac319ff3b..d5c6a2e952b3 100644
--- a/drivers/char/tpm/tpm_tis_core.h
+++ b/drivers/char/tpm/tpm_tis_core.h
@@ -79,6 +79,11 @@ enum tis_defaults {
 #define	TPM_DID_VID(l)			(0x0F00 | ((l) << 12))
 #define	TPM_RID(l)			(0x0F04 | ((l) << 12))
 
+#define LPC_CNTRL_OFFSET		0x84
+#define LPC_CLKRUN_EN			(1 << 2)
+#define INTEL_LEGACY_BLK_BASE_ADDR	0xFED08000
+#define ILB_REMAP_SIZE			0x100
+
 enum tpm_tis_flags {
 	TPM_TIS_ITPM_WORKAROUND		= BIT(0),
 };
@@ -89,6 +94,8 @@ struct tpm_tis_data {
 	int irq;
 	bool irq_tested;
 	unsigned int flags;
+	void __iomem *ilb_base_addr;
+	u16 clkrun_enabled;
 	wait_queue_head_t int_queue;
 	wait_queue_head_t read_queue;
 	const struct tpm_tis_phy_ops *phy_ops;
@@ -144,6 +151,15 @@ static inline int tpm_tis_write32(struct tpm_tis_data *data, u32 addr,
 	return data->phy_ops->write32(data, addr, value);
 }
 
+static inline bool is_bsw(void)
+{
+#ifdef CONFIG_X86
+	return ((boot_cpu_data.x86_model == INTEL_FAM6_ATOM_AIRMONT) ? 1 : 0);
+#else
+	return false;
+#endif
+}
+
 void tpm_tis_remove(struct tpm_chip *chip);
 int tpm_tis_core_init(struct device *dev, struct tpm_tis_data *priv, int irq,
 		      const struct tpm_tis_phy_ops *phy_ops,
diff --git a/drivers/char/tpm/tpm_vtpm_proxy.c b/drivers/char/tpm/tpm_vtpm_proxy.c
index 1d877cc9af97..e4f79f920450 100644
--- a/drivers/char/tpm/tpm_vtpm_proxy.c
+++ b/drivers/char/tpm/tpm_vtpm_proxy.c
@@ -173,22 +173,22 @@ static ssize_t vtpm_proxy_fops_write(struct file *filp, const char __user *buf,
  *
  * Return: Poll flags
  */
-static unsigned int vtpm_proxy_fops_poll(struct file *filp, poll_table *wait)
+static __poll_t vtpm_proxy_fops_poll(struct file *filp, poll_table *wait)
 {
 	struct proxy_dev *proxy_dev = filp->private_data;
-	unsigned ret;
+	__poll_t ret;
 
 	poll_wait(filp, &proxy_dev->wq, wait);
 
-	ret = POLLOUT;
+	ret = EPOLLOUT;
 
 	mutex_lock(&proxy_dev->buf_lock);
 
 	if (proxy_dev->req_len)
-		ret |= POLLIN | POLLRDNORM;
+		ret |= EPOLLIN | EPOLLRDNORM;
 
 	if (!(proxy_dev->state & STATE_OPENED_FLAG))
-		ret |= POLLHUP;
+		ret |= EPOLLHUP;
 
 	mutex_unlock(&proxy_dev->buf_lock);
 
diff --git a/drivers/char/tpm/xen-tpmfront.c b/drivers/char/tpm/xen-tpmfront.c
index 656e8af95d52..911475d36800 100644
--- a/drivers/char/tpm/xen-tpmfront.c
+++ b/drivers/char/tpm/xen-tpmfront.c
@@ -10,6 +10,7 @@
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/interrupt.h>
+#include <linux/freezer.h>
 #include <xen/xen.h>
 #include <xen/events.h>
 #include <xen/interface/io/tpmif.h>
@@ -39,6 +40,66 @@ enum status_bits {
 	VTPM_STATUS_CANCELED = 0x8,
 };
 
+static bool wait_for_tpm_stat_cond(struct tpm_chip *chip, u8 mask,
+					bool check_cancel, bool *canceled)
+{
+	u8 status = chip->ops->status(chip);
+
+	*canceled = false;
+	if ((status & mask) == mask)
+		return true;
+	if (check_cancel && chip->ops->req_canceled(chip, status)) {
+		*canceled = true;
+		return true;
+	}
+	return false;
+}
+
+static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
+		unsigned long timeout, wait_queue_head_t *queue,
+		bool check_cancel)
+{
+	unsigned long stop;
+	long rc;
+	u8 status;
+	bool canceled = false;
+
+	/* check current status */
+	status = chip->ops->status(chip);
+	if ((status & mask) == mask)
+		return 0;
+
+	stop = jiffies + timeout;
+
+	if (chip->flags & TPM_CHIP_FLAG_IRQ) {
+again:
+		timeout = stop - jiffies;
+		if ((long)timeout <= 0)
+			return -ETIME;
+		rc = wait_event_interruptible_timeout(*queue,
+			wait_for_tpm_stat_cond(chip, mask, check_cancel,
+					       &canceled),
+			timeout);
+		if (rc > 0) {
+			if (canceled)
+				return -ECANCELED;
+			return 0;
+		}
+		if (rc == -ERESTARTSYS && freezing(current)) {
+			clear_thread_flag(TIF_SIGPENDING);
+			goto again;
+		}
+	} else {
+		do {
+			tpm_msleep(TPM_TIMEOUT);
+			status = chip->ops->status(chip);
+			if ((status & mask) == mask)
+				return 0;
+		} while (time_before(jiffies, stop));
+	}
+	return -ETIME;
+}
+
 static u8 vtpm_status(struct tpm_chip *chip)
 {
 	struct tpm_private *priv = dev_get_drvdata(&chip->dev);
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index d1aed2513bd9..468f06134012 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -982,25 +982,25 @@ static ssize_t port_fops_splice_write(struct pipe_inode_info *pipe,
 	return ret;
 }
 
-static unsigned int port_fops_poll(struct file *filp, poll_table *wait)
+static __poll_t port_fops_poll(struct file *filp, poll_table *wait)
 {
 	struct port *port;
-	unsigned int ret;
+	__poll_t ret;
 
 	port = filp->private_data;
 	poll_wait(filp, &port->waitqueue, wait);
 
 	if (!port->guest_connected) {
 		/* Port got unplugged */
-		return POLLHUP;
+		return EPOLLHUP;
 	}
 	ret = 0;
 	if (!will_read_block(port))
-		ret |= POLLIN | POLLRDNORM;
+		ret |= EPOLLIN | EPOLLRDNORM;
 	if (!will_write_block(port))
-		ret |= POLLOUT;
+		ret |= EPOLLOUT;
 	if (!port->host_connected)
-		ret |= POLLHUP;
+		ret |= EPOLLHUP;
 
 	return ret;
 }
diff --git a/drivers/char/xillybus/Kconfig b/drivers/char/xillybus/Kconfig
index b302684d86c1..a1f16df08d32 100644
--- a/drivers/char/xillybus/Kconfig
+++ b/drivers/char/xillybus/Kconfig
@@ -4,7 +4,7 @@
 
 config XILLYBUS
 	tristate "Xillybus generic FPGA interface"
-	depends on PCI || (OF_ADDRESS && OF_IRQ)
+	depends on PCI || OF
 	select CRC32
 	help
 	  Xillybus is a generic interface for peripherals designed on
@@ -24,7 +24,7 @@ config XILLYBUS_PCIE
 
 config XILLYBUS_OF
 	tristate "Xillybus over Device Tree"
-	depends on OF_ADDRESS && OF_IRQ && HAS_DMA
+	depends on OF && HAS_DMA
 	help
 	  Set to M if you want Xillybus to find its resources from the
 	  Open Firmware Flattened Device Tree. If the target is an embedded
diff --git a/drivers/char/xillybus/xillybus_core.c b/drivers/char/xillybus/xillybus_core.c
index b6c9cdead7f3..a11af94e2e65 100644
--- a/drivers/char/xillybus/xillybus_core.c
+++ b/drivers/char/xillybus/xillybus_core.c
@@ -1736,10 +1736,10 @@ static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
 	return pos;
 }
 
-static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
+static __poll_t xillybus_poll(struct file *filp, poll_table *wait)
 {
 	struct xilly_channel *channel = filp->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	unsigned long flags;
 
 	poll_wait(filp, &channel->endpoint->ep_wait, wait);
@@ -1758,15 +1758,15 @@ static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
 
 		spin_lock_irqsave(&channel->wr_spinlock, flags);
 		if (!channel->wr_empty || channel->wr_ready)
-			mask |= POLLIN | POLLRDNORM;
+			mask |= EPOLLIN | EPOLLRDNORM;
 
 		if (channel->wr_hangup)
 			/*
-			 * Not POLLHUP, because its behavior is in the
-			 * mist, and POLLIN does what we want: Wake up
+			 * Not EPOLLHUP, because its behavior is in the
+			 * mist, and EPOLLIN does what we want: Wake up
 			 * the read file descriptor so it sees EOF.
 			 */
-			mask |=  POLLIN | POLLRDNORM;
+			mask |=  EPOLLIN | EPOLLRDNORM;
 		spin_unlock_irqrestore(&channel->wr_spinlock, flags);
 	}
 
@@ -1781,12 +1781,12 @@ static unsigned int xillybus_poll(struct file *filp, poll_table *wait)
 
 		spin_lock_irqsave(&channel->rd_spinlock, flags);
 		if (!channel->rd_full)
-			mask |= POLLOUT | POLLWRNORM;
+			mask |= EPOLLOUT | EPOLLWRNORM;
 		spin_unlock_irqrestore(&channel->rd_spinlock, flags);
 	}
 
 	if (channel->endpoint->fatal_error)
-		mask |= POLLERR;
+		mask |= EPOLLERR;
 
 	return mask;
 }
diff --git a/drivers/char/xillybus/xillybus_of.c b/drivers/char/xillybus/xillybus_of.c
index 78a492f5acfb..4d6625ccb48f 100644
--- a/drivers/char/xillybus/xillybus_of.c
+++ b/drivers/char/xillybus/xillybus_of.c
@@ -15,10 +15,6 @@
 #include <linux/slab.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
-#include <linux/of_irq.h>
-#include <linux/of_address.h>
-#include <linux/of_device.h>
-#include <linux/of_platform.h>
 #include <linux/err.h>
 #include "xillybus.h"
 
@@ -123,7 +119,7 @@ static int xilly_drv_probe(struct platform_device *op)
 	struct xilly_endpoint *endpoint;
 	int rc;
 	int irq;
-	struct resource res;
+	struct resource *res;
 	struct xilly_endpoint_hardware *ephw = &of_hw;
 
 	if (of_property_read_bool(dev->of_node, "dma-coherent"))
@@ -136,13 +132,13 @@ static int xilly_drv_probe(struct platform_device *op)
 
 	dev_set_drvdata(dev, endpoint);
 
-	rc = of_address_to_resource(dev->of_node, 0, &res);
-	endpoint->registers = devm_ioremap_resource(dev, &res);
+	res = platform_get_resource(op, IORESOURCE_MEM, 0);
+	endpoint->registers = devm_ioremap_resource(dev, res);
 
 	if (IS_ERR(endpoint->registers))
 		return PTR_ERR(endpoint->registers);
 
-	irq = irq_of_parse_and_map(dev->of_node, 0);
+	irq = platform_get_irq(op, 0);
 
 	rc = devm_request_irq(dev, irq, xillybus_isr, 0, xillyname, endpoint);
 
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index 1c4e1aa6767e..98ce9fc6e6c0 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -142,6 +142,18 @@ config COMMON_CLK_GEMINI
 	  This driver supports the SoC clocks on the Cortina Systems Gemini
 	  platform, also known as SL3516 or CS3516.
 
+config COMMON_CLK_ASPEED
+	bool "Clock driver for Aspeed BMC SoCs"
+	depends on ARCH_ASPEED || COMPILE_TEST
+	default ARCH_ASPEED
+	select MFD_SYSCON
+	select RESET_CONTROLLER
+	---help---
+	  This driver supports the SoC clocks on the Aspeed BMC platforms.
+
+	  The G4 and G5 series, including the ast2400 and ast2500, are supported
+	  by this driver.
+
 config COMMON_CLK_S2MPS11
 	tristate "Clock driver for S2MPS1X/S5M8767 MFD"
 	depends on MFD_SEC_CORE || COMPILE_TEST
@@ -236,6 +248,7 @@ source "drivers/clk/mvebu/Kconfig"
 source "drivers/clk/qcom/Kconfig"
 source "drivers/clk/renesas/Kconfig"
 source "drivers/clk/samsung/Kconfig"
+source "drivers/clk/sprd/Kconfig"
 source "drivers/clk/sunxi-ng/Kconfig"
 source "drivers/clk/tegra/Kconfig"
 source "drivers/clk/ti/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index f7f761b02bed..71ec41e6364f 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -27,6 +27,7 @@ obj-$(CONFIG_ARCH_CLPS711X)		+= clk-clps711x.o
 obj-$(CONFIG_COMMON_CLK_CS2000_CP)	+= clk-cs2000-cp.o
 obj-$(CONFIG_ARCH_EFM32)		+= clk-efm32gg.o
 obj-$(CONFIG_COMMON_CLK_GEMINI)		+= clk-gemini.o
+obj-$(CONFIG_COMMON_CLK_ASPEED)		+= clk-aspeed.o
 obj-$(CONFIG_ARCH_HIGHBANK)		+= clk-highbank.o
 obj-$(CONFIG_CLK_HSDK)			+= clk-hsdk-pll.o
 obj-$(CONFIG_COMMON_CLK_MAX77686)	+= clk-max77686.o
@@ -67,7 +68,7 @@ obj-$(CONFIG_ARCH_MXC)			+= imx/
 obj-$(CONFIG_MACH_INGENIC)		+= ingenic/
 obj-$(CONFIG_ARCH_KEYSTONE)		+= keystone/
 obj-$(CONFIG_MACH_LOONGSON32)		+= loongson1/
-obj-$(CONFIG_ARCH_MEDIATEK)		+= mediatek/
+obj-y					+= mediatek/
 obj-$(CONFIG_COMMON_CLK_AMLOGIC)	+= meson/
 obj-$(CONFIG_MACH_PIC32)		+= microchip/
 ifeq ($(CONFIG_COMMON_CLK), y)
@@ -85,6 +86,7 @@ obj-$(CONFIG_COMMON_CLK_SAMSUNG)	+= samsung/
 obj-$(CONFIG_ARCH_SIRF)			+= sirf/
 obj-$(CONFIG_ARCH_SOCFPGA)		+= socfpga/
 obj-$(CONFIG_PLAT_SPEAR)		+= spear/
+obj-$(CONFIG_ARCH_SPRD)			+= sprd/
 obj-$(CONFIG_ARCH_STI)			+= st/
 obj-$(CONFIG_ARCH_SUNXI)		+= sunxi/
 obj-$(CONFIG_ARCH_SUNXI)		+= sunxi-ng/
diff --git a/drivers/clk/at91/clk-programmable.c b/drivers/clk/at91/clk-programmable.c
index 85a449cf61e3..0e6aab1252fc 100644
--- a/drivers/clk/at91/clk-programmable.c
+++ b/drivers/clk/at91/clk-programmable.c
@@ -204,6 +204,8 @@ at91_clk_register_programmable(struct regmap *regmap,
 	if (ret) {
 		kfree(prog);
 		hw = ERR_PTR(ret);
+	} else {
+		pmc_register_pck(id);
 	}
 
 	return hw;
diff --git a/drivers/clk/at91/pmc.c b/drivers/clk/at91/pmc.c
index 775af473fe11..1fa27f4ea538 100644
--- a/drivers/clk/at91/pmc.c
+++ b/drivers/clk/at91/pmc.c
@@ -22,6 +22,7 @@
 #include "pmc.h"
 
 #define PMC_MAX_IDS 128
+#define PMC_MAX_PCKS 8
 
 int of_at91_get_clk_range(struct device_node *np, const char *propname,
 			  struct clk_range *range)
@@ -50,6 +51,7 @@ EXPORT_SYMBOL_GPL(of_at91_get_clk_range);
 static struct regmap *pmcreg;
 
 static u8 registered_ids[PMC_MAX_IDS];
+static u8 registered_pcks[PMC_MAX_PCKS];
 
 static struct
 {
@@ -66,8 +68,13 @@ static struct
 	u32 pcr[PMC_MAX_IDS];
 	u32 audio_pll0;
 	u32 audio_pll1;
+	u32 pckr[PMC_MAX_PCKS];
 } pmc_cache;
 
+/*
+ * As Peripheral ID 0 is invalid on AT91 chips, the identifier is stored
+ * without alteration in the table, and 0 is for unused clocks.
+ */
 void pmc_register_id(u8 id)
 {
 	int i;
@@ -82,11 +89,30 @@ void pmc_register_id(u8 id)
 	}
 }
 
+/*
+ * As Programmable Clock 0 is valid on AT91 chips, there is an offset
+ * of 1 between the stored value and the real clock ID.
+ */
+void pmc_register_pck(u8 pck)
+{
+	int i;
+
+	for (i = 0; i < PMC_MAX_PCKS; i++) {
+		if (registered_pcks[i] == 0) {
+			registered_pcks[i] = pck + 1;
+			break;
+		}
+		if (registered_pcks[i] == (pck + 1))
+			break;
+	}
+}
+
 static int pmc_suspend(void)
 {
 	int i;
+	u8 num;
 
-	regmap_read(pmcreg, AT91_PMC_IMR, &pmc_cache.scsr);
+	regmap_read(pmcreg, AT91_PMC_SCSR, &pmc_cache.scsr);
 	regmap_read(pmcreg, AT91_PMC_PCSR, &pmc_cache.pcsr0);
 	regmap_read(pmcreg, AT91_CKGR_UCKR, &pmc_cache.uckr);
 	regmap_read(pmcreg, AT91_CKGR_MOR, &pmc_cache.mor);
@@ -103,14 +129,29 @@ static int pmc_suspend(void)
 		regmap_read(pmcreg, AT91_PMC_PCR,
 			    &pmc_cache.pcr[registered_ids[i]]);
 	}
+	for (i = 0; registered_pcks[i]; i++) {
+		num = registered_pcks[i] - 1;
+		regmap_read(pmcreg, AT91_PMC_PCKR(num), &pmc_cache.pckr[num]);
+	}
 
 	return 0;
 }
 
+static bool pmc_ready(unsigned int mask)
+{
+	unsigned int status;
+
+	regmap_read(pmcreg, AT91_PMC_SR, &status);
+
+	return ((status & mask) == mask) ? 1 : 0;
+}
+
 static void pmc_resume(void)
 {
-	int i, ret = 0;
+	int i;
+	u8 num;
 	u32 tmp;
+	u32 mask = AT91_PMC_MCKRDY | AT91_PMC_LOCKA;
 
 	regmap_read(pmcreg, AT91_PMC_MCKR, &tmp);
 	if (pmc_cache.mckr != tmp)
@@ -119,7 +160,7 @@ static void pmc_resume(void)
 	if (pmc_cache.pllar != tmp)
 		pr_warn("PLLAR was not configured properly by the firmware\n");
 
-	regmap_write(pmcreg, AT91_PMC_IMR, pmc_cache.scsr);
+	regmap_write(pmcreg, AT91_PMC_SCER, pmc_cache.scsr);
 	regmap_write(pmcreg, AT91_PMC_PCER, pmc_cache.pcsr0);
 	regmap_write(pmcreg, AT91_CKGR_UCKR, pmc_cache.uckr);
 	regmap_write(pmcreg, AT91_CKGR_MOR, pmc_cache.mor);
@@ -133,14 +174,16 @@ static void pmc_resume(void)
 			     pmc_cache.pcr[registered_ids[i]] |
 			     AT91_PMC_PCR_CMD);
 	}
-
-	if (pmc_cache.uckr & AT91_PMC_UPLLEN) {
-		ret = regmap_read_poll_timeout(pmcreg, AT91_PMC_SR, tmp,
-					       !(tmp & AT91_PMC_LOCKU),
-					       10, 5000);
-		if (ret)
-			pr_crit("USB PLL didn't lock when resuming\n");
+	for (i = 0; registered_pcks[i]; i++) {
+		num = registered_pcks[i] - 1;
+		regmap_write(pmcreg, AT91_PMC_PCKR(num), pmc_cache.pckr[num]);
 	}
+
+	if (pmc_cache.uckr & AT91_PMC_UPLLEN)
+		mask |= AT91_PMC_LOCKU;
+
+	while (!pmc_ready(mask))
+		cpu_relax();
 }
 
 static struct syscore_ops pmc_syscore_ops = {
diff --git a/drivers/clk/at91/pmc.h b/drivers/clk/at91/pmc.h
index 858e8ef7e8db..d22b1fa9ecdc 100644
--- a/drivers/clk/at91/pmc.h
+++ b/drivers/clk/at91/pmc.h
@@ -31,8 +31,10 @@ int of_at91_get_clk_range(struct device_node *np, const char *propname,
 
 #ifdef CONFIG_PM
 void pmc_register_id(u8 id);
+void pmc_register_pck(u8 pck);
 #else
 static inline void pmc_register_id(u8 id) {}
+static inline void pmc_register_pck(u8 pck) {}
 #endif
 
 #endif /* __PMC_H_ */
diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c
index 44301a3d9963..a07f6451694a 100644
--- a/drivers/clk/bcm/clk-bcm2835.c
+++ b/drivers/clk/bcm/clk-bcm2835.c
@@ -449,17 +449,17 @@ struct bcm2835_pll_ana_bits {
 static const struct bcm2835_pll_ana_bits bcm2835_ana_default = {
 	.mask0 = 0,
 	.set0 = 0,
-	.mask1 = (u32)~(A2W_PLL_KI_MASK | A2W_PLL_KP_MASK),
+	.mask1 = A2W_PLL_KI_MASK | A2W_PLL_KP_MASK,
 	.set1 = (2 << A2W_PLL_KI_SHIFT) | (8 << A2W_PLL_KP_SHIFT),
-	.mask3 = (u32)~A2W_PLL_KA_MASK,
+	.mask3 = A2W_PLL_KA_MASK,
 	.set3 = (2 << A2W_PLL_KA_SHIFT),
 	.fb_prediv_mask = BIT(14),
 };
 
 static const struct bcm2835_pll_ana_bits bcm2835_ana_pllh = {
-	.mask0 = (u32)~(A2W_PLLH_KA_MASK | A2W_PLLH_KI_LOW_MASK),
+	.mask0 = A2W_PLLH_KA_MASK | A2W_PLLH_KI_LOW_MASK,
 	.set0 = (2 << A2W_PLLH_KA_SHIFT) | (2 << A2W_PLLH_KI_LOW_SHIFT),
-	.mask1 = (u32)~(A2W_PLLH_KI_HIGH_MASK | A2W_PLLH_KP_MASK),
+	.mask1 = A2W_PLLH_KI_HIGH_MASK | A2W_PLLH_KP_MASK,
 	.set1 = (6 << A2W_PLLH_KP_SHIFT),
 	.mask3 = 0,
 	.set3 = 0,
@@ -623,8 +623,10 @@ static int bcm2835_pll_on(struct clk_hw *hw)
 		     ~A2W_PLL_CTRL_PWRDN);
 
 	/* Take the PLL out of reset. */
+	spin_lock(&cprman->regs_lock);
 	cprman_write(cprman, data->cm_ctrl_reg,
 		     cprman_read(cprman, data->cm_ctrl_reg) & ~CM_PLL_ANARST);
+	spin_unlock(&cprman->regs_lock);
 
 	/* Wait for the PLL to lock. */
 	timeout = ktime_add_ns(ktime_get(), LOCK_TIMEOUT_NS);
@@ -701,9 +703,11 @@ static int bcm2835_pll_set_rate(struct clk_hw *hw,
 	}
 
 	/* Unmask the reference clock from the oscillator. */
+	spin_lock(&cprman->regs_lock);
 	cprman_write(cprman, A2W_XOSC_CTRL,
 		     cprman_read(cprman, A2W_XOSC_CTRL) |
 		     data->reference_enable_mask);
+	spin_unlock(&cprman->regs_lock);
 
 	if (do_ana_setup_first)
 		bcm2835_pll_write_ana(cprman, data->ana_reg_base, ana);
diff --git a/drivers/clk/bcm/clk-cygnus.c b/drivers/clk/bcm/clk-cygnus.c
index 464fdc4bc66b..b8d073e4855f 100644
--- a/drivers/clk/bcm/clk-cygnus.c
+++ b/drivers/clk/bcm/clk-cygnus.c
@@ -269,23 +269,10 @@ static void __init cygnus_asiu_init(struct device_node *node)
 }
 CLK_OF_DECLARE(cygnus_asiu_clk, "brcm,cygnus-asiu-clk", cygnus_asiu_init);
 
-/*
- * AUDIO PLL VCO frequency parameter table
- *
- * PLL output frequency = ((ndiv_int + ndiv_frac / 2^20) *
- * (parent clock rate / pdiv)
- *
- * On Cygnus, parent is the 25MHz oscillator
- */
-static const struct iproc_pll_vco_param audiopll_vco_params[] = {
-	/* rate (Hz) ndiv_int ndiv_frac pdiv */
-	{ 1354750204UL,  54,     199238,   1 },
-	{ 1769470191UL,  70,     816639,   1 },
-};
-
 static const struct iproc_pll_ctrl audiopll = {
 	.flags = IPROC_CLK_PLL_NEEDS_SW_CFG | IPROC_CLK_PLL_HAS_NDIV_FRAC |
-		IPROC_CLK_PLL_USER_MODE_ON | IPROC_CLK_PLL_RESET_ACTIVE_LOW,
+		IPROC_CLK_PLL_USER_MODE_ON | IPROC_CLK_PLL_RESET_ACTIVE_LOW |
+		IPROC_CLK_PLL_CALC_PARAM,
 	.reset = RESET_VAL(0x5c, 0, 1),
 	.dig_filter = DF_VAL(0x48, 0, 3, 6, 4, 3, 3),
 	.sw_ctrl = SW_CTRL_VAL(0x4, 0),
@@ -300,8 +287,7 @@ static const struct iproc_pll_ctrl audiopll = {
 static const struct iproc_clk_ctrl audiopll_clk[] = {
 	[BCM_CYGNUS_AUDIOPLL_CH0] = {
 		.channel = BCM_CYGNUS_AUDIOPLL_CH0,
-		.flags = IPROC_CLK_AON |
-				IPROC_CLK_MCLK_DIV_BY_2,
+		.flags = IPROC_CLK_AON | IPROC_CLK_MCLK_DIV_BY_2,
 		.enable = ENABLE_VAL(0x14, 8, 10, 9),
 		.mdiv = REG_VAL(0x14, 0, 8),
 	},
@@ -321,9 +307,8 @@ static const struct iproc_clk_ctrl audiopll_clk[] = {
 
 static void __init cygnus_audiopll_clk_init(struct device_node *node)
 {
-	iproc_pll_clk_setup(node, &audiopll, audiopll_vco_params,
-			    ARRAY_SIZE(audiopll_vco_params), audiopll_clk,
-			    ARRAY_SIZE(audiopll_clk));
+	iproc_pll_clk_setup(node, &audiopll, NULL, 0,
+			    audiopll_clk,  ARRAY_SIZE(audiopll_clk));
 }
 CLK_OF_DECLARE(cygnus_audiopll, "brcm,cygnus-audiopll",
 			cygnus_audiopll_clk_init);
diff --git a/drivers/clk/bcm/clk-iproc-pll.c b/drivers/clk/bcm/clk-iproc-pll.c
index 375d8dd80d45..43a58ae5a89d 100644
--- a/drivers/clk/bcm/clk-iproc-pll.c
+++ b/drivers/clk/bcm/clk-iproc-pll.c
@@ -69,16 +69,6 @@ enum vco_freq_range {
 	VCO_MAX       = 4000000000U,
 };
 
-struct iproc_pll;
-
-struct iproc_clk {
-	struct clk_hw hw;
-	const char *name;
-	struct iproc_pll *pll;
-	unsigned long rate;
-	const struct iproc_clk_ctrl *ctrl;
-};
-
 struct iproc_pll {
 	void __iomem *status_base;
 	void __iomem *control_base;
@@ -88,13 +78,49 @@ struct iproc_pll {
 	const struct iproc_pll_ctrl *ctrl;
 	const struct iproc_pll_vco_param *vco_param;
 	unsigned int num_vco_entries;
+};
 
-	struct clk_hw_onecell_data *clk_data;
-	struct iproc_clk *clks;
+struct iproc_clk {
+	struct clk_hw hw;
+	struct iproc_pll *pll;
+	const struct iproc_clk_ctrl *ctrl;
 };
 
 #define to_iproc_clk(hw) container_of(hw, struct iproc_clk, hw)
 
+static int pll_calc_param(unsigned long target_rate,
+			unsigned long parent_rate,
+			struct iproc_pll_vco_param *vco_out)
+{
+	u64 ndiv_int, ndiv_frac, residual;
+
+	ndiv_int = target_rate / parent_rate;
+
+	if (!ndiv_int || (ndiv_int > 255))
+		return -EINVAL;
+
+	residual = target_rate - (ndiv_int * parent_rate);
+	residual <<= 20;
+
+	/*
+	 * Add half of the divisor so the result will be rounded to closest
+	 * instead of rounded down.
+	 */
+	residual += (parent_rate / 2);
+	ndiv_frac = div64_u64((u64)residual, (u64)parent_rate);
+
+	vco_out->ndiv_int = ndiv_int;
+	vco_out->ndiv_frac = ndiv_frac;
+	vco_out->pdiv = 1;
+
+	vco_out->rate = vco_out->ndiv_int * parent_rate;
+	residual = (u64)vco_out->ndiv_frac * (u64)parent_rate;
+	residual >>= 20;
+	vco_out->rate += residual;
+
+	return 0;
+}
+
 /*
  * Based on the target frequency, find a match from the VCO frequency parameter
  * table and return its index
@@ -252,17 +278,51 @@ static void __pll_bring_out_reset(struct iproc_pll *pll, unsigned int kp,
 	iproc_pll_write(pll, pll->control_base, reset->offset, val);
 }
 
-static int pll_set_rate(struct iproc_clk *clk, unsigned int rate_index,
+/*
+ * Determines if the change to be applied to the PLL is minor (just an update
+ * or the fractional divider). If so, then we can avoid going through a
+ * disruptive reset and lock sequence.
+ */
+static bool pll_fractional_change_only(struct iproc_pll *pll,
+				       struct iproc_pll_vco_param *vco)
+{
+	const struct iproc_pll_ctrl *ctrl = pll->ctrl;
+	u32 val;
+	u32 ndiv_int;
+	unsigned int pdiv;
+
+	/* PLL needs to be locked */
+	val = readl(pll->status_base + ctrl->status.offset);
+	if ((val & (1 << ctrl->status.shift)) == 0)
+		return false;
+
+	val = readl(pll->control_base + ctrl->ndiv_int.offset);
+	ndiv_int = (val >> ctrl->ndiv_int.shift) &
+		bit_mask(ctrl->ndiv_int.width);
+
+	if (ndiv_int != vco->ndiv_int)
+		return false;
+
+	val = readl(pll->control_base + ctrl->pdiv.offset);
+	pdiv = (val >> ctrl->pdiv.shift) & bit_mask(ctrl->pdiv.width);
+
+	if (pdiv != vco->pdiv)
+		return false;
+
+	return true;
+}
+
+static int pll_set_rate(struct iproc_clk *clk, struct iproc_pll_vco_param *vco,
 			unsigned long parent_rate)
 {
 	struct iproc_pll *pll = clk->pll;
-	const struct iproc_pll_vco_param *vco = &pll->vco_param[rate_index];
 	const struct iproc_pll_ctrl *ctrl = pll->ctrl;
 	int ka = 0, ki, kp, ret;
 	unsigned long rate = vco->rate;
 	u32 val;
 	enum kp_band kp_index;
 	unsigned long ref_freq;
+	const char *clk_name = clk_hw_get_name(&clk->hw);
 
 	/*
 	 * reference frequency = parent frequency / PDIV
@@ -285,22 +345,35 @@ static int pll_set_rate(struct iproc_clk *clk, unsigned int rate_index,
 		kp_index = KP_BAND_HIGH_HIGH;
 	} else {
 		pr_err("%s: pll: %s has invalid rate: %lu\n", __func__,
-				clk->name, rate);
+				clk_name, rate);
 		return -EINVAL;
 	}
 
 	kp = get_kp(ref_freq, kp_index);
 	if (kp < 0) {
-		pr_err("%s: pll: %s has invalid kp\n", __func__, clk->name);
+		pr_err("%s: pll: %s has invalid kp\n", __func__, clk_name);
 		return kp;
 	}
 
 	ret = __pll_enable(pll);
 	if (ret) {
-		pr_err("%s: pll: %s fails to enable\n", __func__, clk->name);
+		pr_err("%s: pll: %s fails to enable\n", __func__, clk_name);
 		return ret;
 	}
 
+	if (pll_fractional_change_only(clk->pll, vco)) {
+		/* program fractional part of NDIV */
+		if (ctrl->flags & IPROC_CLK_PLL_HAS_NDIV_FRAC) {
+			val = readl(pll->control_base + ctrl->ndiv_frac.offset);
+			val &= ~(bit_mask(ctrl->ndiv_frac.width) <<
+				 ctrl->ndiv_frac.shift);
+			val |= vco->ndiv_frac << ctrl->ndiv_frac.shift;
+			iproc_pll_write(pll, pll->control_base,
+					ctrl->ndiv_frac.offset, val);
+			return 0;
+		}
+	}
+
 	/* put PLL in reset */
 	__pll_put_in_reset(pll);
 
@@ -354,7 +427,7 @@ static int pll_set_rate(struct iproc_clk *clk, unsigned int rate_index,
 
 	ret = pll_wait_for_lock(pll);
 	if (ret < 0) {
-		pr_err("%s: pll: %s failed to lock\n", __func__, clk->name);
+		pr_err("%s: pll: %s failed to lock\n", __func__, clk_name);
 		return ret;
 	}
 
@@ -390,16 +463,15 @@ static unsigned long iproc_pll_recalc_rate(struct clk_hw *hw,
 	u32 val;
 	u64 ndiv, ndiv_int, ndiv_frac;
 	unsigned int pdiv;
+	unsigned long rate;
 
 	if (parent_rate == 0)
 		return 0;
 
 	/* PLL needs to be locked */
 	val = readl(pll->status_base + ctrl->status.offset);
-	if ((val & (1 << ctrl->status.shift)) == 0) {
-		clk->rate = 0;
+	if ((val & (1 << ctrl->status.shift)) == 0)
 		return 0;
-	}
 
 	/*
 	 * PLL output frequency =
@@ -421,35 +493,60 @@ static unsigned long iproc_pll_recalc_rate(struct clk_hw *hw,
 	val = readl(pll->control_base + ctrl->pdiv.offset);
 	pdiv = (val >> ctrl->pdiv.shift) & bit_mask(ctrl->pdiv.width);
 
-	clk->rate = (ndiv * parent_rate) >> 20;
+	rate = (ndiv * parent_rate) >> 20;
 
 	if (pdiv == 0)
-		clk->rate *= 2;
+		rate *= 2;
 	else
-		clk->rate /= pdiv;
+		rate /= pdiv;
 
-	return clk->rate;
+	return rate;
 }
 
-static long iproc_pll_round_rate(struct clk_hw *hw, unsigned long rate,
-				 unsigned long *parent_rate)
+static int iproc_pll_determine_rate(struct clk_hw *hw,
+		struct clk_rate_request *req)
 {
-	unsigned i;
+	unsigned int  i;
 	struct iproc_clk *clk = to_iproc_clk(hw);
 	struct iproc_pll *pll = clk->pll;
+	const struct iproc_pll_ctrl *ctrl = pll->ctrl;
+	unsigned long  diff, best_diff;
+	unsigned int  best_idx = 0;
+	int ret;
+
+	if (req->rate == 0 || req->best_parent_rate == 0)
+		return -EINVAL;
+
+	if (ctrl->flags & IPROC_CLK_PLL_CALC_PARAM) {
+		struct iproc_pll_vco_param vco_param;
 
-	if (rate == 0 || *parent_rate == 0 || !pll->vco_param)
+		ret = pll_calc_param(req->rate, req->best_parent_rate,
+					&vco_param);
+		if (ret)
+			return ret;
+
+		req->rate = vco_param.rate;
+		return 0;
+	}
+
+	if (!pll->vco_param)
 		return -EINVAL;
 
+	best_diff = ULONG_MAX;
 	for (i = 0; i < pll->num_vco_entries; i++) {
-		if (rate <= pll->vco_param[i].rate)
+		diff = abs(req->rate - pll->vco_param[i].rate);
+		if (diff <= best_diff) {
+			best_diff = diff;
+			best_idx = i;
+		}
+		/* break now if perfect match */
+		if (diff == 0)
 			break;
 	}
 
-	if (i == pll->num_vco_entries)
-		i--;
+	req->rate = pll->vco_param[best_idx].rate;
 
-	return pll->vco_param[i].rate;
+	return 0;
 }
 
 static int iproc_pll_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -457,13 +554,23 @@ static int iproc_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 {
 	struct iproc_clk *clk = to_iproc_clk(hw);
 	struct iproc_pll *pll = clk->pll;
+	const struct iproc_pll_ctrl *ctrl = pll->ctrl;
+	struct iproc_pll_vco_param vco_param;
 	int rate_index, ret;
 
-	rate_index = pll_get_rate_index(pll, rate);
-	if (rate_index < 0)
-		return rate_index;
+	if (ctrl->flags & IPROC_CLK_PLL_CALC_PARAM) {
+		ret = pll_calc_param(rate, parent_rate, &vco_param);
+		if (ret)
+			return ret;
+	} else {
+		rate_index = pll_get_rate_index(pll, rate);
+		if (rate_index < 0)
+			return rate_index;
 
-	ret = pll_set_rate(clk, rate_index, parent_rate);
+		vco_param = pll->vco_param[rate_index];
+	}
+
+	ret = pll_set_rate(clk, &vco_param, parent_rate);
 	return ret;
 }
 
@@ -471,7 +578,7 @@ static const struct clk_ops iproc_pll_ops = {
 	.enable = iproc_pll_enable,
 	.disable = iproc_pll_disable,
 	.recalc_rate = iproc_pll_recalc_rate,
-	.round_rate = iproc_pll_round_rate,
+	.determine_rate = iproc_pll_determine_rate,
 	.set_rate = iproc_pll_set_rate,
 };
 
@@ -518,6 +625,7 @@ static unsigned long iproc_clk_recalc_rate(struct clk_hw *hw,
 	struct iproc_pll *pll = clk->pll;
 	u32 val;
 	unsigned int mdiv;
+	unsigned long rate;
 
 	if (parent_rate == 0)
 		return 0;
@@ -528,32 +636,33 @@ static unsigned long iproc_clk_recalc_rate(struct clk_hw *hw,
 		mdiv = 256;
 
 	if (ctrl->flags & IPROC_CLK_MCLK_DIV_BY_2)
-		clk->rate = parent_rate / (mdiv * 2);
+		rate = parent_rate / (mdiv * 2);
 	else
-		clk->rate = parent_rate / mdiv;
+		rate = parent_rate / mdiv;
 
-	return clk->rate;
+	return rate;
 }
 
-static long iproc_clk_round_rate(struct clk_hw *hw, unsigned long rate,
-		unsigned long *parent_rate)
+static int iproc_clk_determine_rate(struct clk_hw *hw,
+		struct clk_rate_request *req)
 {
-	unsigned int div;
+	unsigned int bestdiv;
 
-	if (rate == 0 || *parent_rate == 0)
+	if (req->rate == 0)
 		return -EINVAL;
+	if (req->rate == req->best_parent_rate)
+		return 0;
 
-	if (rate == *parent_rate)
-		return *parent_rate;
+	bestdiv = DIV_ROUND_CLOSEST(req->best_parent_rate, req->rate);
+	if (bestdiv < 2)
+		req->rate = req->best_parent_rate;
 
-	div = DIV_ROUND_UP(*parent_rate, rate);
-	if (div < 2)
-		return *parent_rate;
+	if (bestdiv > 256)
+		bestdiv = 256;
 
-	if (div > 256)
-		div = 256;
+	req->rate = req->best_parent_rate / bestdiv;
 
-	return *parent_rate / div;
+	return 0;
 }
 
 static int iproc_clk_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -568,10 +677,10 @@ static int iproc_clk_set_rate(struct clk_hw *hw, unsigned long rate,
 	if (rate == 0 || parent_rate == 0)
 		return -EINVAL;
 
+	div = DIV_ROUND_CLOSEST(parent_rate, rate);
 	if (ctrl->flags & IPROC_CLK_MCLK_DIV_BY_2)
-		div = DIV_ROUND_UP(parent_rate, rate * 2);
-	else
-		div = DIV_ROUND_UP(parent_rate, rate);
+		div /=  2;
+
 	if (div > 256)
 		return -EINVAL;
 
@@ -583,10 +692,6 @@ static int iproc_clk_set_rate(struct clk_hw *hw, unsigned long rate,
 		val |= div << ctrl->mdiv.shift;
 	}
 	iproc_pll_write(pll, pll->control_base, ctrl->mdiv.offset, val);
-	if (ctrl->flags & IPROC_CLK_MCLK_DIV_BY_2)
-		clk->rate = parent_rate / (div * 2);
-	else
-		clk->rate = parent_rate / div;
 
 	return 0;
 }
@@ -595,7 +700,7 @@ static const struct clk_ops iproc_clk_ops = {
 	.enable = iproc_clk_enable,
 	.disable = iproc_clk_disable,
 	.recalc_rate = iproc_clk_recalc_rate,
-	.round_rate = iproc_clk_round_rate,
+	.determine_rate = iproc_clk_determine_rate,
 	.set_rate = iproc_clk_set_rate,
 };
 
@@ -629,6 +734,8 @@ void iproc_pll_clk_setup(struct device_node *node,
 	struct iproc_clk *iclk;
 	struct clk_init_data init;
 	const char *parent_name;
+	struct iproc_clk *iclk_array;
+	struct clk_hw_onecell_data *clk_data;
 
 	if (WARN_ON(!pll_ctrl) || WARN_ON(!clk_ctrl))
 		return;
@@ -637,14 +744,14 @@ void iproc_pll_clk_setup(struct device_node *node,
 	if (WARN_ON(!pll))
 		return;
 
-	pll->clk_data = kzalloc(sizeof(*pll->clk_data->hws) * num_clks +
-				sizeof(*pll->clk_data), GFP_KERNEL);
-	if (WARN_ON(!pll->clk_data))
+	clk_data = kzalloc(sizeof(*clk_data->hws) * num_clks +
+				sizeof(*clk_data), GFP_KERNEL);
+	if (WARN_ON(!clk_data))
 		goto err_clk_data;
-	pll->clk_data->num = num_clks;
+	clk_data->num = num_clks;
 
-	pll->clks = kcalloc(num_clks, sizeof(*pll->clks), GFP_KERNEL);
-	if (WARN_ON(!pll->clks))
+	iclk_array = kcalloc(num_clks, sizeof(struct iproc_clk), GFP_KERNEL);
+	if (WARN_ON(!iclk_array))
 		goto err_clks;
 
 	pll->control_base = of_iomap(node, 0);
@@ -674,9 +781,8 @@ void iproc_pll_clk_setup(struct device_node *node,
 	/* initialize and register the PLL itself */
 	pll->ctrl = pll_ctrl;
 
-	iclk = &pll->clks[0];
+	iclk = &iclk_array[0];
 	iclk->pll = pll;
-	iclk->name = node->name;
 
 	init.name = node->name;
 	init.ops = &iproc_pll_ops;
@@ -697,7 +803,7 @@ void iproc_pll_clk_setup(struct device_node *node,
 	if (WARN_ON(ret))
 		goto err_pll_register;
 
-	pll->clk_data->hws[0] = &iclk->hw;
+	clk_data->hws[0] = &iclk->hw;
 
 	/* now initialize and register all leaf clocks */
 	for (i = 1; i < num_clks; i++) {
@@ -711,8 +817,7 @@ void iproc_pll_clk_setup(struct device_node *node,
 		if (WARN_ON(ret))
 			goto err_clk_register;
 
-		iclk = &pll->clks[i];
-		iclk->name = clk_name;
+		iclk = &iclk_array[i];
 		iclk->pll = pll;
 		iclk->ctrl = &clk_ctrl[i];
 
@@ -727,11 +832,10 @@ void iproc_pll_clk_setup(struct device_node *node,
 		if (WARN_ON(ret))
 			goto err_clk_register;
 
-		pll->clk_data->hws[i] = &iclk->hw;
+		clk_data->hws[i] = &iclk->hw;
 	}
 
-	ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get,
-				     pll->clk_data);
+	ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data);
 	if (WARN_ON(ret))
 		goto err_clk_register;
 
@@ -739,7 +843,7 @@ void iproc_pll_clk_setup(struct device_node *node,
 
 err_clk_register:
 	while (--i >= 0)
-		clk_hw_unregister(pll->clk_data->hws[i]);
+		clk_hw_unregister(clk_data->hws[i]);
 
 err_pll_register:
 	if (pll->status_base != pll->control_base)
@@ -756,10 +860,10 @@ void iproc_pll_clk_setup(struct device_node *node,
 	iounmap(pll->control_base);
 
 err_pll_iomap:
-	kfree(pll->clks);
+	kfree(iclk_array);
 
 err_clks:
-	kfree(pll->clk_data);
+	kfree(clk_data);
 
 err_clk_data:
 	kfree(pll);
diff --git a/drivers/clk/bcm/clk-iproc.h b/drivers/clk/bcm/clk-iproc.h
index 2148b4ea9f28..a48ddd3e0b28 100644
--- a/drivers/clk/bcm/clk-iproc.h
+++ b/drivers/clk/bcm/clk-iproc.h
@@ -80,6 +80,11 @@
  */
 #define IPROC_CLK_PLL_RESET_ACTIVE_LOW BIT(9)
 
+/*
+ * Calculate the PLL parameters are runtime, instead of using table
+ */
+#define IPROC_CLK_PLL_CALC_PARAM BIT(10)
+
 /*
  * Parameters for VCO frequency configuration
  *
diff --git a/drivers/clk/clk-aspeed.c b/drivers/clk/clk-aspeed.c
new file mode 100644
index 000000000000..5eb50c31e455
--- /dev/null
+++ b/drivers/clk/clk-aspeed.c
@@ -0,0 +1,673 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#define pr_fmt(fmt) "clk-aspeed: " fmt
+
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset-controller.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include <dt-bindings/clock/aspeed-clock.h>
+
+#define ASPEED_NUM_CLKS		35
+
+#define ASPEED_RESET_CTRL	0x04
+#define ASPEED_CLK_SELECTION	0x08
+#define ASPEED_CLK_STOP_CTRL	0x0c
+#define ASPEED_MPLL_PARAM	0x20
+#define ASPEED_HPLL_PARAM	0x24
+#define  AST2500_HPLL_BYPASS_EN	BIT(20)
+#define  AST2400_HPLL_STRAPPED	BIT(18)
+#define  AST2400_HPLL_BYPASS_EN	BIT(17)
+#define ASPEED_MISC_CTRL	0x2c
+#define  UART_DIV13_EN		BIT(12)
+#define ASPEED_STRAP		0x70
+#define  CLKIN_25MHZ_EN		BIT(23)
+#define  AST2400_CLK_SOURCE_SEL	BIT(18)
+#define ASPEED_CLK_SELECTION_2	0xd8
+
+/* Globally visible clocks */
+static DEFINE_SPINLOCK(aspeed_clk_lock);
+
+/* Keeps track of all clocks */
+static struct clk_hw_onecell_data *aspeed_clk_data;
+
+static void __iomem *scu_base;
+
+/**
+ * struct aspeed_gate_data - Aspeed gated clocks
+ * @clock_idx: bit used to gate this clock in the clock register
+ * @reset_idx: bit used to reset this IP in the reset register. -1 if no
+ *             reset is required when enabling the clock
+ * @name: the clock name
+ * @parent_name: the name of the parent clock
+ * @flags: standard clock framework flags
+ */
+struct aspeed_gate_data {
+	u8		clock_idx;
+	s8		reset_idx;
+	const char	*name;
+	const char	*parent_name;
+	unsigned long	flags;
+};
+
+/**
+ * struct aspeed_clk_gate - Aspeed specific clk_gate structure
+ * @hw:		handle between common and hardware-specific interfaces
+ * @reg:	register controlling gate
+ * @clock_idx:	bit used to gate this clock in the clock register
+ * @reset_idx:	bit used to reset this IP in the reset register. -1 if no
+ *		reset is required when enabling the clock
+ * @flags:	hardware-specific flags
+ * @lock:	register lock
+ *
+ * Some of the clocks in the Aspeed SoC must be put in reset before enabling.
+ * This modified version of clk_gate allows an optional reset bit to be
+ * specified.
+ */
+struct aspeed_clk_gate {
+	struct clk_hw	hw;
+	struct regmap	*map;
+	u8		clock_idx;
+	s8		reset_idx;
+	u8		flags;
+	spinlock_t	*lock;
+};
+
+#define to_aspeed_clk_gate(_hw) container_of(_hw, struct aspeed_clk_gate, hw)
+
+/* TODO: ask Aspeed about the actual parent data */
+static const struct aspeed_gate_data aspeed_gates[] = {
+	/*				 clk rst   name			parent	flags */
+	[ASPEED_CLK_GATE_ECLK] =	{  0, -1, "eclk-gate",		"eclk",	0 }, /* Video Engine */
+	[ASPEED_CLK_GATE_GCLK] =	{  1,  7, "gclk-gate",		NULL,	0 }, /* 2D engine */
+	[ASPEED_CLK_GATE_MCLK] =	{  2, -1, "mclk-gate",		"mpll",	CLK_IS_CRITICAL }, /* SDRAM */
+	[ASPEED_CLK_GATE_VCLK] =	{  3,  6, "vclk-gate",		NULL,	0 }, /* Video Capture */
+	[ASPEED_CLK_GATE_BCLK] =	{  4, 10, "bclk-gate",		"bclk",	0 }, /* PCIe/PCI */
+	[ASPEED_CLK_GATE_DCLK] =	{  5, -1, "dclk-gate",		NULL,	0 }, /* DAC */
+	[ASPEED_CLK_GATE_REFCLK] =	{  6, -1, "refclk-gate",	"clkin", CLK_IS_CRITICAL },
+	[ASPEED_CLK_GATE_USBPORT2CLK] =	{  7,  3, "usb-port2-gate",	NULL,	0 }, /* USB2.0 Host port 2 */
+	[ASPEED_CLK_GATE_LCLK] =	{  8,  5, "lclk-gate",		NULL,	0 }, /* LPC */
+	[ASPEED_CLK_GATE_USBUHCICLK] =	{  9, 15, "usb-uhci-gate",	NULL,	0 }, /* USB1.1 (requires port 2 enabled) */
+	[ASPEED_CLK_GATE_D1CLK] =	{ 10, 13, "d1clk-gate",		NULL,	0 }, /* GFX CRT */
+	[ASPEED_CLK_GATE_YCLK] =	{ 13,  4, "yclk-gate",		NULL,	0 }, /* HAC */
+	[ASPEED_CLK_GATE_USBPORT1CLK] = { 14, 14, "usb-port1-gate",	NULL,	0 }, /* USB2 hub/USB2 host port 1/USB1.1 dev */
+	[ASPEED_CLK_GATE_UART1CLK] =	{ 15, -1, "uart1clk-gate",	"uart",	0 }, /* UART1 */
+	[ASPEED_CLK_GATE_UART2CLK] =	{ 16, -1, "uart2clk-gate",	"uart",	0 }, /* UART2 */
+	[ASPEED_CLK_GATE_UART5CLK] =	{ 17, -1, "uart5clk-gate",	"uart",	0 }, /* UART5 */
+	[ASPEED_CLK_GATE_ESPICLK] =	{ 19, -1, "espiclk-gate",	NULL,	0 }, /* eSPI */
+	[ASPEED_CLK_GATE_MAC1CLK] =	{ 20, 11, "mac1clk-gate",	"mac",	0 }, /* MAC1 */
+	[ASPEED_CLK_GATE_MAC2CLK] =	{ 21, 12, "mac2clk-gate",	"mac",	0 }, /* MAC2 */
+	[ASPEED_CLK_GATE_RSACLK] =	{ 24, -1, "rsaclk-gate",	NULL,	0 }, /* RSA */
+	[ASPEED_CLK_GATE_UART3CLK] =	{ 25, -1, "uart3clk-gate",	"uart",	0 }, /* UART3 */
+	[ASPEED_CLK_GATE_UART4CLK] =	{ 26, -1, "uart4clk-gate",	"uart",	0 }, /* UART4 */
+	[ASPEED_CLK_GATE_SDCLKCLK] =	{ 27, 16, "sdclk-gate",		NULL,	0 }, /* SDIO/SD */
+	[ASPEED_CLK_GATE_LHCCLK] =	{ 28, -1, "lhclk-gate",		"lhclk", 0 }, /* LPC master/LPC+ */
+};
+
+static const struct clk_div_table ast2500_mac_div_table[] = {
+	{ 0x0, 4 }, /* Yep, really. Aspeed confirmed this is correct */
+	{ 0x1, 4 },
+	{ 0x2, 6 },
+	{ 0x3, 8 },
+	{ 0x4, 10 },
+	{ 0x5, 12 },
+	{ 0x6, 14 },
+	{ 0x7, 16 },
+	{ 0 }
+};
+
+static const struct clk_div_table ast2400_div_table[] = {
+	{ 0x0, 2 },
+	{ 0x1, 4 },
+	{ 0x2, 6 },
+	{ 0x3, 8 },
+	{ 0x4, 10 },
+	{ 0x5, 12 },
+	{ 0x6, 14 },
+	{ 0x7, 16 },
+	{ 0 }
+};
+
+static const struct clk_div_table ast2500_div_table[] = {
+	{ 0x0, 4 },
+	{ 0x1, 8 },
+	{ 0x2, 12 },
+	{ 0x3, 16 },
+	{ 0x4, 20 },
+	{ 0x5, 24 },
+	{ 0x6, 28 },
+	{ 0x7, 32 },
+	{ 0 }
+};
+
+static struct clk_hw *aspeed_ast2400_calc_pll(const char *name, u32 val)
+{
+	unsigned int mult, div;
+
+	if (val & AST2400_HPLL_BYPASS_EN) {
+		/* Pass through mode */
+		mult = div = 1;
+	} else {
+		/* F = 24Mhz * (2-OD) * [(N + 2) / (D + 1)] */
+		u32 n = (val >> 5) & 0x3f;
+		u32 od = (val >> 4) & 0x1;
+		u32 d = val & 0xf;
+
+		mult = (2 - od) * (n + 2);
+		div = d + 1;
+	}
+	return clk_hw_register_fixed_factor(NULL, name, "clkin", 0,
+			mult, div);
+};
+
+static struct clk_hw *aspeed_ast2500_calc_pll(const char *name, u32 val)
+{
+	unsigned int mult, div;
+
+	if (val & AST2500_HPLL_BYPASS_EN) {
+		/* Pass through mode */
+		mult = div = 1;
+	} else {
+		/* F = clkin * [(M+1) / (N+1)] / (P + 1) */
+		u32 p = (val >> 13) & 0x3f;
+		u32 m = (val >> 5) & 0xff;
+		u32 n = val & 0x1f;
+
+		mult = (m + 1) / (n + 1);
+		div = p + 1;
+	}
+
+	return clk_hw_register_fixed_factor(NULL, name, "clkin", 0,
+			mult, div);
+}
+
+struct aspeed_clk_soc_data {
+	const struct clk_div_table *div_table;
+	const struct clk_div_table *mac_div_table;
+	struct clk_hw *(*calc_pll)(const char *name, u32 val);
+};
+
+static const struct aspeed_clk_soc_data ast2500_data = {
+	.div_table = ast2500_div_table,
+	.mac_div_table = ast2500_mac_div_table,
+	.calc_pll = aspeed_ast2500_calc_pll,
+};
+
+static const struct aspeed_clk_soc_data ast2400_data = {
+	.div_table = ast2400_div_table,
+	.mac_div_table = ast2400_div_table,
+	.calc_pll = aspeed_ast2400_calc_pll,
+};
+
+static int aspeed_clk_is_enabled(struct clk_hw *hw)
+{
+	struct aspeed_clk_gate *gate = to_aspeed_clk_gate(hw);
+	u32 clk = BIT(gate->clock_idx);
+	u32 enval = (gate->flags & CLK_GATE_SET_TO_DISABLE) ? 0 : clk;
+	u32 reg;
+
+	regmap_read(gate->map, ASPEED_CLK_STOP_CTRL, &reg);
+
+	return ((reg & clk) == enval) ? 1 : 0;
+}
+
+static int aspeed_clk_enable(struct clk_hw *hw)
+{
+	struct aspeed_clk_gate *gate = to_aspeed_clk_gate(hw);
+	unsigned long flags;
+	u32 clk = BIT(gate->clock_idx);
+	u32 rst = BIT(gate->reset_idx);
+	u32 enval;
+
+	spin_lock_irqsave(gate->lock, flags);
+
+	if (aspeed_clk_is_enabled(hw)) {
+		spin_unlock_irqrestore(gate->lock, flags);
+		return 0;
+	}
+
+	if (gate->reset_idx >= 0) {
+		/* Put IP in reset */
+		regmap_update_bits(gate->map, ASPEED_RESET_CTRL, rst, rst);
+
+		/* Delay 100us */
+		udelay(100);
+	}
+
+	/* Enable clock */
+	enval = (gate->flags & CLK_GATE_SET_TO_DISABLE) ? 0 : clk;
+	regmap_update_bits(gate->map, ASPEED_CLK_STOP_CTRL, clk, enval);
+
+	if (gate->reset_idx >= 0) {
+		/* A delay of 10ms is specified by the ASPEED docs */
+		mdelay(10);
+
+		/* Take IP out of reset */
+		regmap_update_bits(gate->map, ASPEED_RESET_CTRL, rst, 0);
+	}
+
+	spin_unlock_irqrestore(gate->lock, flags);
+
+	return 0;
+}
+
+static void aspeed_clk_disable(struct clk_hw *hw)
+{
+	struct aspeed_clk_gate *gate = to_aspeed_clk_gate(hw);
+	unsigned long flags;
+	u32 clk = BIT(gate->clock_idx);
+	u32 enval;
+
+	spin_lock_irqsave(gate->lock, flags);
+
+	enval = (gate->flags & CLK_GATE_SET_TO_DISABLE) ? clk : 0;
+	regmap_update_bits(gate->map, ASPEED_CLK_STOP_CTRL, clk, enval);
+
+	spin_unlock_irqrestore(gate->lock, flags);
+}
+
+static const struct clk_ops aspeed_clk_gate_ops = {
+	.enable = aspeed_clk_enable,
+	.disable = aspeed_clk_disable,
+	.is_enabled = aspeed_clk_is_enabled,
+};
+
+/**
+ * struct aspeed_reset - Aspeed reset controller
+ * @map: regmap to access the containing system controller
+ * @rcdev: reset controller device
+ */
+struct aspeed_reset {
+	struct regmap			*map;
+	struct reset_controller_dev	rcdev;
+};
+
+#define to_aspeed_reset(p) container_of((p), struct aspeed_reset, rcdev)
+
+static const u8 aspeed_resets[] = {
+	[ASPEED_RESET_XDMA]	= 25,
+	[ASPEED_RESET_MCTP]	= 24,
+	[ASPEED_RESET_ADC]	= 23,
+	[ASPEED_RESET_JTAG_MASTER] = 22,
+	[ASPEED_RESET_MIC]	= 18,
+	[ASPEED_RESET_PWM]	=  9,
+	[ASPEED_RESET_PCIVGA]	=  8,
+	[ASPEED_RESET_I2C]	=  2,
+	[ASPEED_RESET_AHB]	=  1,
+};
+
+static int aspeed_reset_deassert(struct reset_controller_dev *rcdev,
+				 unsigned long id)
+{
+	struct aspeed_reset *ar = to_aspeed_reset(rcdev);
+	u32 rst = BIT(aspeed_resets[id]);
+
+	return regmap_update_bits(ar->map, ASPEED_RESET_CTRL, rst, 0);
+}
+
+static int aspeed_reset_assert(struct reset_controller_dev *rcdev,
+			       unsigned long id)
+{
+	struct aspeed_reset *ar = to_aspeed_reset(rcdev);
+	u32 rst = BIT(aspeed_resets[id]);
+
+	return regmap_update_bits(ar->map, ASPEED_RESET_CTRL, rst, rst);
+}
+
+static int aspeed_reset_status(struct reset_controller_dev *rcdev,
+			       unsigned long id)
+{
+	struct aspeed_reset *ar = to_aspeed_reset(rcdev);
+	u32 val, rst = BIT(aspeed_resets[id]);
+	int ret;
+
+	ret = regmap_read(ar->map, ASPEED_RESET_CTRL, &val);
+	if (ret)
+		return ret;
+
+	return !!(val & rst);
+}
+
+static const struct reset_control_ops aspeed_reset_ops = {
+	.assert = aspeed_reset_assert,
+	.deassert = aspeed_reset_deassert,
+	.status = aspeed_reset_status,
+};
+
+static struct clk_hw *aspeed_clk_hw_register_gate(struct device *dev,
+		const char *name, const char *parent_name, unsigned long flags,
+		struct regmap *map, u8 clock_idx, u8 reset_idx,
+		u8 clk_gate_flags, spinlock_t *lock)
+{
+	struct aspeed_clk_gate *gate;
+	struct clk_init_data init;
+	struct clk_hw *hw;
+	int ret;
+
+	gate = kzalloc(sizeof(*gate), GFP_KERNEL);
+	if (!gate)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &aspeed_clk_gate_ops;
+	init.flags = flags;
+	init.parent_names = parent_name ? &parent_name : NULL;
+	init.num_parents = parent_name ? 1 : 0;
+
+	gate->map = map;
+	gate->clock_idx = clock_idx;
+	gate->reset_idx = reset_idx;
+	gate->flags = clk_gate_flags;
+	gate->lock = lock;
+	gate->hw.init = &init;
+
+	hw = &gate->hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret) {
+		kfree(gate);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+static int aspeed_clk_probe(struct platform_device *pdev)
+{
+	const struct aspeed_clk_soc_data *soc_data;
+	struct device *dev = &pdev->dev;
+	struct aspeed_reset *ar;
+	struct regmap *map;
+	struct clk_hw *hw;
+	u32 val, rate;
+	int i, ret;
+
+	map = syscon_node_to_regmap(dev->of_node);
+	if (IS_ERR(map)) {
+		dev_err(dev, "no syscon regmap\n");
+		return PTR_ERR(map);
+	}
+
+	ar = devm_kzalloc(dev, sizeof(*ar), GFP_KERNEL);
+	if (!ar)
+		return -ENOMEM;
+
+	ar->map = map;
+	ar->rcdev.owner = THIS_MODULE;
+	ar->rcdev.nr_resets = ARRAY_SIZE(aspeed_resets);
+	ar->rcdev.ops = &aspeed_reset_ops;
+	ar->rcdev.of_node = dev->of_node;
+
+	ret = devm_reset_controller_register(dev, &ar->rcdev);
+	if (ret) {
+		dev_err(dev, "could not register reset controller\n");
+		return ret;
+	}
+
+	/* SoC generations share common layouts but have different divisors */
+	soc_data = of_device_get_match_data(dev);
+	if (!soc_data) {
+		dev_err(dev, "no match data for platform\n");
+		return -EINVAL;
+	}
+
+	/* UART clock div13 setting */
+	regmap_read(map, ASPEED_MISC_CTRL, &val);
+	if (val & UART_DIV13_EN)
+		rate = 24000000 / 13;
+	else
+		rate = 24000000;
+	/* TODO: Find the parent data for the uart clock */
+	hw = clk_hw_register_fixed_rate(dev, "uart", NULL, 0, rate);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_UART] = hw;
+
+	/*
+	 * Memory controller (M-PLL) PLL. This clock is configured by the
+	 * bootloader, and is exposed to Linux as a read-only clock rate.
+	 */
+	regmap_read(map, ASPEED_MPLL_PARAM, &val);
+	hw = soc_data->calc_pll("mpll", val);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_MPLL] =	hw;
+
+	/* SD/SDIO clock divider (TODO: There's a gate too) */
+	hw = clk_hw_register_divider_table(dev, "sdio", "hpll", 0,
+			scu_base + ASPEED_CLK_SELECTION, 12, 3, 0,
+			soc_data->div_table,
+			&aspeed_clk_lock);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_SDIO] = hw;
+
+	/* MAC AHB bus clock divider */
+	hw = clk_hw_register_divider_table(dev, "mac", "hpll", 0,
+			scu_base + ASPEED_CLK_SELECTION, 16, 3, 0,
+			soc_data->mac_div_table,
+			&aspeed_clk_lock);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_MAC] = hw;
+
+	/* LPC Host (LHCLK) clock divider */
+	hw = clk_hw_register_divider_table(dev, "lhclk", "hpll", 0,
+			scu_base + ASPEED_CLK_SELECTION, 20, 3, 0,
+			soc_data->div_table,
+			&aspeed_clk_lock);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_LHCLK] = hw;
+
+	/* P-Bus (BCLK) clock divider */
+	hw = clk_hw_register_divider_table(dev, "bclk", "hpll", 0,
+			scu_base + ASPEED_CLK_SELECTION_2, 0, 2, 0,
+			soc_data->div_table,
+			&aspeed_clk_lock);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_BCLK] = hw;
+
+	/*
+	 * TODO: There are a number of clocks that not included in this driver
+	 * as more information is required:
+	 *   D2-PLL
+	 *   D-PLL
+	 *   YCLK
+	 *   RGMII
+	 *   RMII
+	 *   UART[1..5] clock source mux
+	 *   Video Engine (ECLK) mux and clock divider
+	 */
+
+	for (i = 0; i < ARRAY_SIZE(aspeed_gates); i++) {
+		const struct aspeed_gate_data *gd = &aspeed_gates[i];
+		u32 gate_flags;
+
+		/* Special case: the USB port 1 clock (bit 14) is always
+		 * working the opposite way from the other ones.
+		 */
+		gate_flags = (gd->clock_idx == 14) ? 0 : CLK_GATE_SET_TO_DISABLE;
+		hw = aspeed_clk_hw_register_gate(dev,
+				gd->name,
+				gd->parent_name,
+				gd->flags,
+				map,
+				gd->clock_idx,
+				gd->reset_idx,
+				gate_flags,
+				&aspeed_clk_lock);
+		if (IS_ERR(hw))
+			return PTR_ERR(hw);
+		aspeed_clk_data->hws[i] = hw;
+	}
+
+	return 0;
+};
+
+static const struct of_device_id aspeed_clk_dt_ids[] = {
+	{ .compatible = "aspeed,ast2400-scu", .data = &ast2400_data },
+	{ .compatible = "aspeed,ast2500-scu", .data = &ast2500_data },
+	{ }
+};
+
+static struct platform_driver aspeed_clk_driver = {
+	.probe  = aspeed_clk_probe,
+	.driver = {
+		.name = "aspeed-clk",
+		.of_match_table = aspeed_clk_dt_ids,
+		.suppress_bind_attrs = true,
+	},
+};
+builtin_platform_driver(aspeed_clk_driver);
+
+static void __init aspeed_ast2400_cc(struct regmap *map)
+{
+	struct clk_hw *hw;
+	u32 val, freq, div;
+
+	/*
+	 * CLKIN is the crystal oscillator, 24, 48 or 25MHz selected by
+	 * strapping
+	 */
+	regmap_read(map, ASPEED_STRAP, &val);
+	if (val & CLKIN_25MHZ_EN)
+		freq = 25000000;
+	else if (val & AST2400_CLK_SOURCE_SEL)
+		freq = 48000000;
+	else
+		freq = 24000000;
+	hw = clk_hw_register_fixed_rate(NULL, "clkin", NULL, 0, freq);
+	pr_debug("clkin @%u MHz\n", freq / 1000000);
+
+	/*
+	 * High-speed PLL clock derived from the crystal. This the CPU clock,
+	 * and we assume that it is enabled
+	 */
+	regmap_read(map, ASPEED_HPLL_PARAM, &val);
+	WARN(val & AST2400_HPLL_STRAPPED, "hpll is strapped not configured");
+	aspeed_clk_data->hws[ASPEED_CLK_HPLL] = aspeed_ast2400_calc_pll("hpll", val);
+
+	/*
+	 * Strap bits 11:10 define the CPU/AHB clock frequency ratio (aka HCLK)
+	 *   00: Select CPU:AHB = 1:1
+	 *   01: Select CPU:AHB = 2:1
+	 *   10: Select CPU:AHB = 4:1
+	 *   11: Select CPU:AHB = 3:1
+	 */
+	regmap_read(map, ASPEED_STRAP, &val);
+	val = (val >> 10) & 0x3;
+	div = val + 1;
+	if (div == 3)
+		div = 4;
+	else if (div == 4)
+		div = 3;
+	hw = clk_hw_register_fixed_factor(NULL, "ahb", "hpll", 0, 1, div);
+	aspeed_clk_data->hws[ASPEED_CLK_AHB] = hw;
+
+	/* APB clock clock selection register SCU08 (aka PCLK) */
+	hw = clk_hw_register_divider_table(NULL, "apb", "hpll", 0,
+			scu_base + ASPEED_CLK_SELECTION, 23, 3, 0,
+			ast2400_div_table,
+			&aspeed_clk_lock);
+	aspeed_clk_data->hws[ASPEED_CLK_APB] = hw;
+}
+
+static void __init aspeed_ast2500_cc(struct regmap *map)
+{
+	struct clk_hw *hw;
+	u32 val, freq, div;
+
+	/* CLKIN is the crystal oscillator, 24 or 25MHz selected by strapping */
+	regmap_read(map, ASPEED_STRAP, &val);
+	if (val & CLKIN_25MHZ_EN)
+		freq = 25000000;
+	else
+		freq = 24000000;
+	hw = clk_hw_register_fixed_rate(NULL, "clkin", NULL, 0, freq);
+	pr_debug("clkin @%u MHz\n", freq / 1000000);
+
+	/*
+	 * High-speed PLL clock derived from the crystal. This the CPU clock,
+	 * and we assume that it is enabled
+	 */
+	regmap_read(map, ASPEED_HPLL_PARAM, &val);
+	aspeed_clk_data->hws[ASPEED_CLK_HPLL] = aspeed_ast2500_calc_pll("hpll", val);
+
+	/* Strap bits 11:9 define the AXI/AHB clock frequency ratio (aka HCLK)*/
+	regmap_read(map, ASPEED_STRAP, &val);
+	val = (val >> 9) & 0x7;
+	WARN(val == 0, "strapping is zero: cannot determine ahb clock");
+	div = 2 * (val + 1);
+	hw = clk_hw_register_fixed_factor(NULL, "ahb", "hpll", 0, 1, div);
+	aspeed_clk_data->hws[ASPEED_CLK_AHB] = hw;
+
+	/* APB clock clock selection register SCU08 (aka PCLK) */
+	regmap_read(map, ASPEED_CLK_SELECTION, &val);
+	val = (val >> 23) & 0x7;
+	div = 4 * (val + 1);
+	hw = clk_hw_register_fixed_factor(NULL, "apb", "hpll", 0, 1, div);
+	aspeed_clk_data->hws[ASPEED_CLK_APB] = hw;
+};
+
+static void __init aspeed_cc_init(struct device_node *np)
+{
+	struct regmap *map;
+	u32 val;
+	int ret;
+	int i;
+
+	scu_base = of_iomap(np, 0);
+	if (!scu_base)
+		return;
+
+	aspeed_clk_data = kzalloc(sizeof(*aspeed_clk_data) +
+			sizeof(*aspeed_clk_data->hws) * ASPEED_NUM_CLKS,
+			GFP_KERNEL);
+	if (!aspeed_clk_data)
+		return;
+
+	/*
+	 * This way all clocks fetched before the platform device probes,
+	 * except those we assign here for early use, will be deferred.
+	 */
+	for (i = 0; i < ASPEED_NUM_CLKS; i++)
+		aspeed_clk_data->hws[i] = ERR_PTR(-EPROBE_DEFER);
+
+	map = syscon_node_to_regmap(np);
+	if (IS_ERR(map)) {
+		pr_err("no syscon regmap\n");
+		return;
+	}
+	/*
+	 * We check that the regmap works on this very first access,
+	 * but as this is an MMIO-backed regmap, subsequent regmap
+	 * access is not going to fail and we skip error checks from
+	 * this point.
+	 */
+	ret = regmap_read(map, ASPEED_STRAP, &val);
+	if (ret) {
+		pr_err("failed to read strapping register\n");
+		return;
+	}
+
+	if (of_device_is_compatible(np, "aspeed,ast2400-scu"))
+		aspeed_ast2400_cc(map);
+	else if (of_device_is_compatible(np, "aspeed,ast2500-scu"))
+		aspeed_ast2500_cc(map);
+	else
+		pr_err("unknown platform, failed to add clocks\n");
+
+	aspeed_clk_data->num = ASPEED_NUM_CLKS;
+	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, aspeed_clk_data);
+	if (ret)
+		pr_err("failed to add DT provider: %d\n", ret);
+};
+CLK_OF_DECLARE_DRIVER(aspeed_cc_g5, "aspeed,ast2500-scu", aspeed_cc_init);
+CLK_OF_DECLARE_DRIVER(aspeed_cc_g4, "aspeed,ast2400-scu", aspeed_cc_init);
diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c
index 5e918e7afaba..48d11f2598e8 100644
--- a/drivers/clk/clk-axi-clkgen.c
+++ b/drivers/clk/clk-axi-clkgen.c
@@ -40,6 +40,10 @@
 #define MMCM_REG_FILTER1	0x4e
 #define MMCM_REG_FILTER2	0x4f
 
+#define MMCM_CLKOUT_NOCOUNT	BIT(6)
+
+#define MMCM_CLK_DIV_NOCOUNT	BIT(12)
+
 struct axi_clkgen {
 	void __iomem *base;
 	struct clk_hw clk_hw;
@@ -298,13 +302,17 @@ static long axi_clkgen_round_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long *parent_rate)
 {
 	unsigned int d, m, dout;
+	unsigned long long tmp;
 
 	axi_clkgen_calc_params(*parent_rate, rate, &d, &m, &dout);
 
 	if (d == 0 || dout == 0 || m == 0)
 		return -EINVAL;
 
-	return *parent_rate / d * m / dout;
+	tmp = (unsigned long long)*parent_rate * m;
+	tmp = DIV_ROUND_CLOSEST_ULL(tmp, dout * d);
+
+	return min_t(unsigned long long, tmp, LONG_MAX);
 }
 
 static unsigned long axi_clkgen_recalc_rate(struct clk_hw *clk_hw,
@@ -315,18 +323,33 @@ static unsigned long axi_clkgen_recalc_rate(struct clk_hw *clk_hw,
 	unsigned int reg;
 	unsigned long long tmp;
 
-	axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLKOUT0_1, &reg);
-	dout = (reg & 0x3f) + ((reg >> 6) & 0x3f);
+	axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLKOUT0_2, &reg);
+	if (reg & MMCM_CLKOUT_NOCOUNT) {
+		dout = 1;
+	} else {
+		axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLKOUT0_1, &reg);
+		dout = (reg & 0x3f) + ((reg >> 6) & 0x3f);
+	}
+
 	axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_DIV, &reg);
-	d = (reg & 0x3f) + ((reg >> 6) & 0x3f);
-	axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_FB1, &reg);
-	m = (reg & 0x3f) + ((reg >> 6) & 0x3f);
+	if (reg & MMCM_CLK_DIV_NOCOUNT)
+		d = 1;
+	else
+		d = (reg & 0x3f) + ((reg >> 6) & 0x3f);
+
+	axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_FB2, &reg);
+	if (reg & MMCM_CLKOUT_NOCOUNT) {
+		m = 1;
+	} else {
+		axi_clkgen_mmcm_read(axi_clkgen, MMCM_REG_CLK_FB1, &reg);
+		m = (reg & 0x3f) + ((reg >> 6) & 0x3f);
+	}
 
 	if (d == 0 || dout == 0)
 		return 0;
 
-	tmp = (unsigned long long)(parent_rate / d) * m;
-	do_div(tmp, dout);
+	tmp = (unsigned long long)parent_rate * m;
+	tmp = DIV_ROUND_CLOSEST_ULL(tmp, dout * d);
 
 	return min_t(unsigned long long, tmp, ULONG_MAX);
 }
diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c
index 4ed516cb7276..b49942b9fe50 100644
--- a/drivers/clk/clk-divider.c
+++ b/drivers/clk/clk-divider.c
@@ -118,12 +118,11 @@ static unsigned int _get_val(const struct clk_div_table *table,
 unsigned long divider_recalc_rate(struct clk_hw *hw, unsigned long parent_rate,
 				  unsigned int val,
 				  const struct clk_div_table *table,
-				  unsigned long flags)
+				  unsigned long flags, unsigned long width)
 {
-	struct clk_divider *divider = to_clk_divider(hw);
 	unsigned int div;
 
-	div = _get_div(table, val, flags, divider->width);
+	div = _get_div(table, val, flags, width);
 	if (!div) {
 		WARN(!(flags & CLK_DIVIDER_ALLOW_ZERO),
 			"%s: Zero divisor and CLK_DIVIDER_ALLOW_ZERO not set\n",
@@ -145,7 +144,7 @@ static unsigned long clk_divider_recalc_rate(struct clk_hw *hw,
 	val &= div_mask(divider->width);
 
 	return divider_recalc_rate(hw, parent_rate, val, divider->table,
-				   divider->flags);
+				   divider->flags, divider->width);
 }
 
 static bool _is_valid_table_div(const struct clk_div_table *table,
diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c
index b0ea753b8709..3a1812f65e5d 100644
--- a/drivers/clk/clk-qoriq.c
+++ b/drivers/clk/clk-qoriq.c
@@ -41,7 +41,7 @@ struct clockgen_pll_div {
 };
 
 struct clockgen_pll {
-	struct clockgen_pll_div div[4];
+	struct clockgen_pll_div div[8];
 };
 
 #define CLKSEL_VALID	1
@@ -1127,6 +1127,13 @@ static void __init create_one_pll(struct clockgen *cg, int idx)
 		struct clk *clk;
 		int ret;
 
+		/*
+		 * For platform PLL, there are 8 divider clocks.
+		 * For core PLL, there are 4 divider clocks at most.
+		 */
+		if (idx != PLATFORM_PLL && i >= 4)
+			break;
+
 		snprintf(pll->div[i].name, sizeof(pll->div[i].name),
 			 "cg-pll%d-div%d", idx, i + 1);
 
diff --git a/drivers/clk/clk-si5351.c b/drivers/clk/clk-si5351.c
index 20d90769cced..50e7c341e97e 100644
--- a/drivers/clk/clk-si5351.c
+++ b/drivers/clk/clk-si5351.c
@@ -72,7 +72,7 @@ static const char * const si5351_input_names[] = {
 	"xtal", "clkin"
 };
 static const char * const si5351_pll_names[] = {
-	"plla", "pllb", "vxco"
+	"si5351_plla", "si5351_pllb", "si5351_vxco"
 };
 static const char * const si5351_msynth_names[] = {
 	"ms0", "ms1", "ms2", "ms3", "ms4", "ms5", "ms6", "ms7"
@@ -903,13 +903,42 @@ static int _si5351_clkout_set_disable_state(
 	return 0;
 }
 
+static void _si5351_clkout_reset_pll(struct si5351_driver_data *drvdata, int num)
+{
+	u8 val = si5351_reg_read(drvdata, SI5351_CLK0_CTRL + num);
+
+	switch (val & SI5351_CLK_INPUT_MASK) {
+	case SI5351_CLK_INPUT_XTAL:
+	case SI5351_CLK_INPUT_CLKIN:
+		return;  /* pll not used, no need to reset */
+	}
+
+	si5351_reg_write(drvdata, SI5351_PLL_RESET,
+			 val & SI5351_CLK_PLL_SELECT ? SI5351_PLL_RESET_B :
+						       SI5351_PLL_RESET_A);
+
+	dev_dbg(&drvdata->client->dev, "%s - %s: pll = %d\n",
+		__func__, clk_hw_get_name(&drvdata->clkout[num].hw),
+		(val & SI5351_CLK_PLL_SELECT) ? 1 : 0);
+}
+
 static int si5351_clkout_prepare(struct clk_hw *hw)
 {
 	struct si5351_hw_data *hwdata =
 		container_of(hw, struct si5351_hw_data, hw);
+	struct si5351_platform_data *pdata =
+		hwdata->drvdata->client->dev.platform_data;
 
 	si5351_set_bits(hwdata->drvdata, SI5351_CLK0_CTRL + hwdata->num,
 			SI5351_CLK_POWERDOWN, 0);
+
+	/*
+	 * Do a pll soft reset on the parent pll -- needed to get a
+	 * deterministic phase relationship between the output clocks.
+	 */
+	if (pdata->clkout[hwdata->num].pll_reset)
+		_si5351_clkout_reset_pll(hwdata->drvdata, hwdata->num);
+
 	si5351_set_bits(hwdata->drvdata, SI5351_OUTPUT_ENABLE_CTRL,
 			(1 << hwdata->num), 0);
 	return 0;
@@ -1297,6 +1326,9 @@ static int si5351_dt_parse(struct i2c_client *client,
 
 		pdata->clkout[num].pll_master =
 			of_property_read_bool(child, "silabs,pll-master");
+
+		pdata->clkout[num].pll_reset =
+			of_property_read_bool(child, "silabs,pll-reset");
 	}
 	client->dev.platform_data = pdata;
 
@@ -1437,11 +1469,6 @@ static int si5351_i2c_probe(struct i2c_client *client,
 		}
 	}
 
-	if (!IS_ERR(drvdata->pxtal))
-		clk_prepare_enable(drvdata->pxtal);
-	if (!IS_ERR(drvdata->pclkin))
-		clk_prepare_enable(drvdata->pclkin);
-
 	/* register xtal input clock gate */
 	memset(&init, 0, sizeof(init));
 	init.name = si5351_input_names[0];
@@ -1456,7 +1483,7 @@ static int si5351_i2c_probe(struct i2c_client *client,
 	ret = devm_clk_hw_register(&client->dev, &drvdata->xtal);
 	if (ret) {
 		dev_err(&client->dev, "unable to register %s\n", init.name);
-		goto err_clk;
+		return ret;
 	}
 
 	/* register clkin input clock gate */
@@ -1474,7 +1501,7 @@ static int si5351_i2c_probe(struct i2c_client *client,
 		if (ret) {
 			dev_err(&client->dev, "unable to register %s\n",
 				init.name);
-			goto err_clk;
+			return ret;
 		}
 	}
 
@@ -1496,7 +1523,7 @@ static int si5351_i2c_probe(struct i2c_client *client,
 	ret = devm_clk_hw_register(&client->dev, &drvdata->pll[0].hw);
 	if (ret) {
 		dev_err(&client->dev, "unable to register %s\n", init.name);
-		goto err_clk;
+		return ret;
 	}
 
 	/* register PLLB or VXCO (Si5351B) */
@@ -1520,7 +1547,7 @@ static int si5351_i2c_probe(struct i2c_client *client,
 	ret = devm_clk_hw_register(&client->dev, &drvdata->pll[1].hw);
 	if (ret) {
 		dev_err(&client->dev, "unable to register %s\n", init.name);
-		goto err_clk;
+		return ret;
 	}
 
 	/* register clk multisync and clk out divider */
@@ -1539,7 +1566,7 @@ static int si5351_i2c_probe(struct i2c_client *client,
 
 	if (WARN_ON(!drvdata->msynth || !drvdata->clkout)) {
 		ret = -ENOMEM;
-		goto err_clk;
+		return ret;
 	}
 
 	for (n = 0; n < num_clocks; n++) {
@@ -1559,7 +1586,7 @@ static int si5351_i2c_probe(struct i2c_client *client,
 		if (ret) {
 			dev_err(&client->dev, "unable to register %s\n",
 				init.name);
-			goto err_clk;
+			return ret;
 		}
 	}
 
@@ -1587,7 +1614,7 @@ static int si5351_i2c_probe(struct i2c_client *client,
 		if (ret) {
 			dev_err(&client->dev, "unable to register %s\n",
 				init.name);
-			goto err_clk;
+			return ret;
 		}
 
 		/* set initial clkout rate */
@@ -1606,17 +1633,17 @@ static int si5351_i2c_probe(struct i2c_client *client,
 				     drvdata);
 	if (ret) {
 		dev_err(&client->dev, "unable to add clk provider\n");
-		goto err_clk;
+		return ret;
 	}
 
 	return 0;
+}
 
-err_clk:
-	if (!IS_ERR(drvdata->pxtal))
-		clk_disable_unprepare(drvdata->pxtal);
-	if (!IS_ERR(drvdata->pclkin))
-		clk_disable_unprepare(drvdata->pclkin);
-	return ret;
+static int si5351_i2c_remove(struct i2c_client *client)
+{
+	of_clk_del_provider(client->dev.of_node);
+
+	return 0;
 }
 
 static const struct i2c_device_id si5351_i2c_ids[] = {
@@ -1634,6 +1661,7 @@ static struct i2c_driver si5351_driver = {
 		.of_match_table = of_match_ptr(si5351_dt_ids),
 	},
 	.probe = si5351_i2c_probe,
+	.remove = si5351_i2c_remove,
 	.id_table = si5351_i2c_ids,
 };
 module_i2c_driver(si5351_driver);
diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c
index 96c6b6bc8f0e..da44f8dc1d29 100644
--- a/drivers/clk/clk-stm32f4.c
+++ b/drivers/clk/clk-stm32f4.c
@@ -1424,7 +1424,7 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 
 	base = of_iomap(np, 0);
 	if (!base) {
-		pr_err("%s: unable to map resource", np->name);
+		pr_err("%s: unable to map resource\n", np->name);
 		return;
 	}
 
diff --git a/drivers/clk/clk-stm32h7.c b/drivers/clk/clk-stm32h7.c
index 61c3e40507d3..db2b162c0d4c 100644
--- a/drivers/clk/clk-stm32h7.c
+++ b/drivers/clk/clk-stm32h7.c
@@ -1,20 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (C) Gabriel Fernandez 2017
- * Author: Gabriel Fernandez <gabriel.fernandez@st.com>
- *
- * License terms: GPL V2.0.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program.  If not, see <http://www.gnu.org/licenses/>.
+ * Copyright (C) STMicroelectronics 2017
+ * Author: Gabriel Fernandez <gabriel.fernandez@st.com> for STMicroelectronics.
  */
 
 #include <linux/clk.h>
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index b56c11f51baf..076d4244d672 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -24,6 +24,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/sched.h>
 #include <linux/clkdev.h>
+#include <linux/stringify.h>
 
 #include "clk.h"
 
@@ -62,6 +63,7 @@ struct clk_core {
 	bool			orphan;
 	unsigned int		enable_count;
 	unsigned int		prepare_count;
+	unsigned int		protect_count;
 	unsigned long		min_rate;
 	unsigned long		max_rate;
 	unsigned long		accuracy;
@@ -86,6 +88,7 @@ struct clk {
 	const char *con_id;
 	unsigned long min_rate;
 	unsigned long max_rate;
+	unsigned int exclusive_count;
 	struct hlist_node clks_node;
 };
 
@@ -141,10 +144,18 @@ static unsigned long clk_enable_lock(void)
 {
 	unsigned long flags;
 
-	if (!spin_trylock_irqsave(&enable_lock, flags)) {
+	/*
+	 * On UP systems, spin_trylock_irqsave() always returns true, even if
+	 * we already hold the lock. So, in that case, we rely only on
+	 * reference counting.
+	 */
+	if (!IS_ENABLED(CONFIG_SMP) ||
+	    !spin_trylock_irqsave(&enable_lock, flags)) {
 		if (enable_owner == current) {
 			enable_refcnt++;
 			__acquire(enable_lock);
+			if (!IS_ENABLED(CONFIG_SMP))
+				local_save_flags(flags);
 			return flags;
 		}
 		spin_lock_irqsave(&enable_lock, flags);
@@ -170,6 +181,11 @@ static void clk_enable_unlock(unsigned long flags)
 	spin_unlock_irqrestore(&enable_lock, flags);
 }
 
+static bool clk_core_rate_is_protected(struct clk_core *core)
+{
+	return core->protect_count;
+}
+
 static bool clk_core_is_prepared(struct clk_core *core)
 {
 	bool ret = false;
@@ -382,6 +398,11 @@ bool clk_hw_is_prepared(const struct clk_hw *hw)
 	return clk_core_is_prepared(hw->core);
 }
 
+bool clk_hw_rate_is_protected(const struct clk_hw *hw)
+{
+	return clk_core_rate_is_protected(hw->core);
+}
+
 bool clk_hw_is_enabled(const struct clk_hw *hw)
 {
 	return clk_core_is_enabled(hw->core);
@@ -520,6 +541,139 @@ EXPORT_SYMBOL_GPL(__clk_mux_determine_rate_closest);
 
 /***        clk api        ***/
 
+static void clk_core_rate_unprotect(struct clk_core *core)
+{
+	lockdep_assert_held(&prepare_lock);
+
+	if (!core)
+		return;
+
+	if (WARN_ON(core->protect_count == 0))
+		return;
+
+	if (--core->protect_count > 0)
+		return;
+
+	clk_core_rate_unprotect(core->parent);
+}
+
+static int clk_core_rate_nuke_protect(struct clk_core *core)
+{
+	int ret;
+
+	lockdep_assert_held(&prepare_lock);
+
+	if (!core)
+		return -EINVAL;
+
+	if (core->protect_count == 0)
+		return 0;
+
+	ret = core->protect_count;
+	core->protect_count = 1;
+	clk_core_rate_unprotect(core);
+
+	return ret;
+}
+
+/**
+ * clk_rate_exclusive_put - release exclusivity over clock rate control
+ * @clk: the clk over which the exclusivity is released
+ *
+ * clk_rate_exclusive_put() completes a critical section during which a clock
+ * consumer cannot tolerate any other consumer making any operation on the
+ * clock which could result in a rate change or rate glitch. Exclusive clocks
+ * cannot have their rate changed, either directly or indirectly due to changes
+ * further up the parent chain of clocks. As a result, clocks up parent chain
+ * also get under exclusive control of the calling consumer.
+ *
+ * If exlusivity is claimed more than once on clock, even by the same consumer,
+ * the rate effectively gets locked as exclusivity can't be preempted.
+ *
+ * Calls to clk_rate_exclusive_put() must be balanced with calls to
+ * clk_rate_exclusive_get(). Calls to this function may sleep, and do not return
+ * error status.
+ */
+void clk_rate_exclusive_put(struct clk *clk)
+{
+	if (!clk)
+		return;
+
+	clk_prepare_lock();
+
+	/*
+	 * if there is something wrong with this consumer protect count, stop
+	 * here before messing with the provider
+	 */
+	if (WARN_ON(clk->exclusive_count <= 0))
+		goto out;
+
+	clk_core_rate_unprotect(clk->core);
+	clk->exclusive_count--;
+out:
+	clk_prepare_unlock();
+}
+EXPORT_SYMBOL_GPL(clk_rate_exclusive_put);
+
+static void clk_core_rate_protect(struct clk_core *core)
+{
+	lockdep_assert_held(&prepare_lock);
+
+	if (!core)
+		return;
+
+	if (core->protect_count == 0)
+		clk_core_rate_protect(core->parent);
+
+	core->protect_count++;
+}
+
+static void clk_core_rate_restore_protect(struct clk_core *core, int count)
+{
+	lockdep_assert_held(&prepare_lock);
+
+	if (!core)
+		return;
+
+	if (count == 0)
+		return;
+
+	clk_core_rate_protect(core);
+	core->protect_count = count;
+}
+
+/**
+ * clk_rate_exclusive_get - get exclusivity over the clk rate control
+ * @clk: the clk over which the exclusity of rate control is requested
+ *
+ * clk_rate_exlusive_get() begins a critical section during which a clock
+ * consumer cannot tolerate any other consumer making any operation on the
+ * clock which could result in a rate change or rate glitch. Exclusive clocks
+ * cannot have their rate changed, either directly or indirectly due to changes
+ * further up the parent chain of clocks. As a result, clocks up parent chain
+ * also get under exclusive control of the calling consumer.
+ *
+ * If exlusivity is claimed more than once on clock, even by the same consumer,
+ * the rate effectively gets locked as exclusivity can't be preempted.
+ *
+ * Calls to clk_rate_exclusive_get() should be balanced with calls to
+ * clk_rate_exclusive_put(). Calls to this function may sleep.
+ * Returns 0 on success, -EERROR otherwise
+ */
+int clk_rate_exclusive_get(struct clk *clk)
+{
+	if (!clk)
+		return 0;
+
+	clk_prepare_lock();
+	clk_core_rate_protect(clk->core);
+	clk->exclusive_count++;
+	clk_prepare_unlock();
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(clk_rate_exclusive_get);
+
 static void clk_core_unprepare(struct clk_core *core)
 {
 	lockdep_assert_held(&prepare_lock);
@@ -906,10 +1060,9 @@ static int clk_disable_unused(void)
 }
 late_initcall_sync(clk_disable_unused);
 
-static int clk_core_round_rate_nolock(struct clk_core *core,
-				      struct clk_rate_request *req)
+static int clk_core_determine_round_nolock(struct clk_core *core,
+					   struct clk_rate_request *req)
 {
-	struct clk_core *parent;
 	long rate;
 
 	lockdep_assert_held(&prepare_lock);
@@ -917,6 +1070,38 @@ static int clk_core_round_rate_nolock(struct clk_core *core,
 	if (!core)
 		return 0;
 
+	/*
+	 * At this point, core protection will be disabled if
+	 * - if the provider is not protected at all
+	 * - if the calling consumer is the only one which has exclusivity
+	 *   over the provider
+	 */
+	if (clk_core_rate_is_protected(core)) {
+		req->rate = core->rate;
+	} else if (core->ops->determine_rate) {
+		return core->ops->determine_rate(core->hw, req);
+	} else if (core->ops->round_rate) {
+		rate = core->ops->round_rate(core->hw, req->rate,
+					     &req->best_parent_rate);
+		if (rate < 0)
+			return rate;
+
+		req->rate = rate;
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void clk_core_init_rate_req(struct clk_core * const core,
+				   struct clk_rate_request *req)
+{
+	struct clk_core *parent;
+
+	if (WARN_ON(!core || !req))
+		return;
+
 	parent = core->parent;
 	if (parent) {
 		req->best_parent_hw = parent->hw;
@@ -925,22 +1110,34 @@ static int clk_core_round_rate_nolock(struct clk_core *core,
 		req->best_parent_hw = NULL;
 		req->best_parent_rate = 0;
 	}
+}
 
-	if (core->ops->determine_rate) {
-		return core->ops->determine_rate(core->hw, req);
-	} else if (core->ops->round_rate) {
-		rate = core->ops->round_rate(core->hw, req->rate,
-					     &req->best_parent_rate);
-		if (rate < 0)
-			return rate;
+static bool clk_core_can_round(struct clk_core * const core)
+{
+	if (core->ops->determine_rate || core->ops->round_rate)
+		return true;
 
-		req->rate = rate;
-	} else if (core->flags & CLK_SET_RATE_PARENT) {
-		return clk_core_round_rate_nolock(parent, req);
-	} else {
-		req->rate = core->rate;
+	return false;
+}
+
+static int clk_core_round_rate_nolock(struct clk_core *core,
+				      struct clk_rate_request *req)
+{
+	lockdep_assert_held(&prepare_lock);
+
+	if (!core) {
+		req->rate = 0;
+		return 0;
 	}
 
+	clk_core_init_rate_req(core, req);
+
+	if (clk_core_can_round(core))
+		return clk_core_determine_round_nolock(core, req);
+	else if (core->flags & CLK_SET_RATE_PARENT)
+		return clk_core_round_rate_nolock(core->parent, req);
+
+	req->rate = core->rate;
 	return 0;
 }
 
@@ -997,10 +1194,17 @@ long clk_round_rate(struct clk *clk, unsigned long rate)
 
 	clk_prepare_lock();
 
+	if (clk->exclusive_count)
+		clk_core_rate_unprotect(clk->core);
+
 	clk_core_get_boundaries(clk->core, &req.min_rate, &req.max_rate);
 	req.rate = rate;
 
 	ret = clk_core_round_rate_nolock(clk->core, &req);
+
+	if (clk->exclusive_count)
+		clk_core_rate_protect(clk->core);
+
 	clk_prepare_unlock();
 
 	if (ret)
@@ -1433,34 +1637,23 @@ static struct clk_core *clk_calc_new_rates(struct clk_core *core,
 	clk_core_get_boundaries(core, &min_rate, &max_rate);
 
 	/* find the closest rate and parent clk/rate */
-	if (core->ops->determine_rate) {
+	if (clk_core_can_round(core)) {
 		struct clk_rate_request req;
 
 		req.rate = rate;
 		req.min_rate = min_rate;
 		req.max_rate = max_rate;
-		if (parent) {
-			req.best_parent_hw = parent->hw;
-			req.best_parent_rate = parent->rate;
-		} else {
-			req.best_parent_hw = NULL;
-			req.best_parent_rate = 0;
-		}
 
-		ret = core->ops->determine_rate(core->hw, &req);
+		clk_core_init_rate_req(core, &req);
+
+		ret = clk_core_determine_round_nolock(core, &req);
 		if (ret < 0)
 			return NULL;
 
 		best_parent_rate = req.best_parent_rate;
 		new_rate = req.rate;
 		parent = req.best_parent_hw ? req.best_parent_hw->core : NULL;
-	} else if (core->ops->round_rate) {
-		ret = core->ops->round_rate(core->hw, rate,
-					    &best_parent_rate);
-		if (ret < 0)
-			return NULL;
 
-		new_rate = ret;
 		if (new_rate < min_rate || new_rate > max_rate)
 			return NULL;
 	} else if (!parent || !(core->flags & CLK_SET_RATE_PARENT)) {
@@ -1642,25 +1835,58 @@ static void clk_change_rate(struct clk_core *core)
 	clk_pm_runtime_put(core);
 }
 
+static unsigned long clk_core_req_round_rate_nolock(struct clk_core *core,
+						     unsigned long req_rate)
+{
+	int ret, cnt;
+	struct clk_rate_request req;
+
+	lockdep_assert_held(&prepare_lock);
+
+	if (!core)
+		return 0;
+
+	/* simulate what the rate would be if it could be freely set */
+	cnt = clk_core_rate_nuke_protect(core);
+	if (cnt < 0)
+		return cnt;
+
+	clk_core_get_boundaries(core, &req.min_rate, &req.max_rate);
+	req.rate = req_rate;
+
+	ret = clk_core_round_rate_nolock(core, &req);
+
+	/* restore the protection */
+	clk_core_rate_restore_protect(core, cnt);
+
+	return ret ? 0 : req.rate;
+}
+
 static int clk_core_set_rate_nolock(struct clk_core *core,
 				    unsigned long req_rate)
 {
 	struct clk_core *top, *fail_clk;
-	unsigned long rate = req_rate;
+	unsigned long rate;
 	int ret = 0;
 
 	if (!core)
 		return 0;
 
+	rate = clk_core_req_round_rate_nolock(core, req_rate);
+
 	/* bail early if nothing to do */
 	if (rate == clk_core_get_rate_nolock(core))
 		return 0;
 
+	/* fail on a direct rate set of a protected provider */
+	if (clk_core_rate_is_protected(core))
+		return -EBUSY;
+
 	if ((core->flags & CLK_SET_RATE_GATE) && core->prepare_count)
 		return -EBUSY;
 
 	/* calculate new rates and get the topmost changed clock */
-	top = clk_calc_new_rates(core, rate);
+	top = clk_calc_new_rates(core, req_rate);
 	if (!top)
 		return -EINVAL;
 
@@ -1719,14 +1945,67 @@ int clk_set_rate(struct clk *clk, unsigned long rate)
 	/* prevent racing with updates to the clock topology */
 	clk_prepare_lock();
 
+	if (clk->exclusive_count)
+		clk_core_rate_unprotect(clk->core);
+
 	ret = clk_core_set_rate_nolock(clk->core, rate);
 
+	if (clk->exclusive_count)
+		clk_core_rate_protect(clk->core);
+
 	clk_prepare_unlock();
 
 	return ret;
 }
 EXPORT_SYMBOL_GPL(clk_set_rate);
 
+/**
+ * clk_set_rate_exclusive - specify a new rate get exclusive control
+ * @clk: the clk whose rate is being changed
+ * @rate: the new rate for clk
+ *
+ * This is a combination of clk_set_rate() and clk_rate_exclusive_get()
+ * within a critical section
+ *
+ * This can be used initially to ensure that at least 1 consumer is
+ * statisfied when several consumers are competing for exclusivity over the
+ * same clock provider.
+ *
+ * The exclusivity is not applied if setting the rate failed.
+ *
+ * Calls to clk_rate_exclusive_get() should be balanced with calls to
+ * clk_rate_exclusive_put().
+ *
+ * Returns 0 on success, -EERROR otherwise.
+ */
+int clk_set_rate_exclusive(struct clk *clk, unsigned long rate)
+{
+	int ret;
+
+	if (!clk)
+		return 0;
+
+	/* prevent racing with updates to the clock topology */
+	clk_prepare_lock();
+
+	/*
+	 * The temporary protection removal is not here, on purpose
+	 * This function is meant to be used instead of clk_rate_protect,
+	 * so before the consumer code path protect the clock provider
+	 */
+
+	ret = clk_core_set_rate_nolock(clk->core, rate);
+	if (!ret) {
+		clk_core_rate_protect(clk->core);
+		clk->exclusive_count++;
+	}
+
+	clk_prepare_unlock();
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(clk_set_rate_exclusive);
+
 /**
  * clk_set_rate_range - set a rate range for a clock source
  * @clk: clock source
@@ -1738,6 +2017,7 @@ EXPORT_SYMBOL_GPL(clk_set_rate);
 int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max)
 {
 	int ret = 0;
+	unsigned long old_min, old_max, rate;
 
 	if (!clk)
 		return 0;
@@ -1751,12 +2031,46 @@ int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max)
 
 	clk_prepare_lock();
 
-	if (min != clk->min_rate || max != clk->max_rate) {
-		clk->min_rate = min;
-		clk->max_rate = max;
-		ret = clk_core_set_rate_nolock(clk->core, clk->core->req_rate);
+	if (clk->exclusive_count)
+		clk_core_rate_unprotect(clk->core);
+
+	/* Save the current values in case we need to rollback the change */
+	old_min = clk->min_rate;
+	old_max = clk->max_rate;
+	clk->min_rate = min;
+	clk->max_rate = max;
+
+	rate = clk_core_get_rate_nolock(clk->core);
+	if (rate < min || rate > max) {
+		/*
+		 * FIXME:
+		 * We are in bit of trouble here, current rate is outside the
+		 * the requested range. We are going try to request appropriate
+		 * range boundary but there is a catch. It may fail for the
+		 * usual reason (clock broken, clock protected, etc) but also
+		 * because:
+		 * - round_rate() was not favorable and fell on the wrong
+		 *   side of the boundary
+		 * - the determine_rate() callback does not really check for
+		 *   this corner case when determining the rate
+		 */
+
+		if (rate < min)
+			rate = min;
+		else
+			rate = max;
+
+		ret = clk_core_set_rate_nolock(clk->core, rate);
+		if (ret) {
+			/* rollback the changes */
+			clk->min_rate = old_min;
+			clk->max_rate = old_max;
+		}
 	}
 
+	if (clk->exclusive_count)
+		clk_core_rate_protect(clk->core);
+
 	clk_prepare_unlock();
 
 	return ret;
@@ -1877,32 +2191,31 @@ bool clk_has_parent(struct clk *clk, struct clk *parent)
 }
 EXPORT_SYMBOL_GPL(clk_has_parent);
 
-static int clk_core_set_parent(struct clk_core *core, struct clk_core *parent)
+static int clk_core_set_parent_nolock(struct clk_core *core,
+				      struct clk_core *parent)
 {
 	int ret = 0;
 	int p_index = 0;
 	unsigned long p_rate = 0;
 
+	lockdep_assert_held(&prepare_lock);
+
 	if (!core)
 		return 0;
 
-	/* prevent racing with updates to the clock topology */
-	clk_prepare_lock();
-
 	if (core->parent == parent)
-		goto out;
+		return 0;
 
 	/* verify ops for for multi-parent clks */
-	if ((core->num_parents > 1) && (!core->ops->set_parent)) {
-		ret = -ENOSYS;
-		goto out;
-	}
+	if (core->num_parents > 1 && !core->ops->set_parent)
+		return -EPERM;
 
 	/* check that we are allowed to re-parent if the clock is in use */
-	if ((core->flags & CLK_SET_PARENT_GATE) && core->prepare_count) {
-		ret = -EBUSY;
-		goto out;
-	}
+	if ((core->flags & CLK_SET_PARENT_GATE) && core->prepare_count)
+		return -EBUSY;
+
+	if (clk_core_rate_is_protected(core))
+		return -EBUSY;
 
 	/* try finding the new parent index */
 	if (parent) {
@@ -1910,15 +2223,14 @@ static int clk_core_set_parent(struct clk_core *core, struct clk_core *parent)
 		if (p_index < 0) {
 			pr_debug("%s: clk %s can not be parent of clk %s\n",
 					__func__, parent->name, core->name);
-			ret = p_index;
-			goto out;
+			return p_index;
 		}
 		p_rate = parent->rate;
 	}
 
 	ret = clk_pm_runtime_get(core);
 	if (ret)
-		goto out;
+		return ret;
 
 	/* propagate PRE_RATE_CHANGE notifications */
 	ret = __clk_speculate_rates(core, p_rate);
@@ -1940,8 +2252,6 @@ static int clk_core_set_parent(struct clk_core *core, struct clk_core *parent)
 
 runtime_put:
 	clk_pm_runtime_put(core);
-out:
-	clk_prepare_unlock();
 
 	return ret;
 }
@@ -1965,13 +2275,53 @@ static int clk_core_set_parent(struct clk_core *core, struct clk_core *parent)
  */
 int clk_set_parent(struct clk *clk, struct clk *parent)
 {
+	int ret;
+
 	if (!clk)
 		return 0;
 
-	return clk_core_set_parent(clk->core, parent ? parent->core : NULL);
+	clk_prepare_lock();
+
+	if (clk->exclusive_count)
+		clk_core_rate_unprotect(clk->core);
+
+	ret = clk_core_set_parent_nolock(clk->core,
+					 parent ? parent->core : NULL);
+
+	if (clk->exclusive_count)
+		clk_core_rate_protect(clk->core);
+
+	clk_prepare_unlock();
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(clk_set_parent);
 
+static int clk_core_set_phase_nolock(struct clk_core *core, int degrees)
+{
+	int ret = -EINVAL;
+
+	lockdep_assert_held(&prepare_lock);
+
+	if (!core)
+		return 0;
+
+	if (clk_core_rate_is_protected(core))
+		return -EBUSY;
+
+	trace_clk_set_phase(core, degrees);
+
+	if (core->ops->set_phase) {
+		ret = core->ops->set_phase(core->hw, degrees);
+		if (!ret)
+			core->phase = degrees;
+	}
+
+	trace_clk_set_phase_complete(core, degrees);
+
+	return ret;
+}
+
 /**
  * clk_set_phase - adjust the phase shift of a clock signal
  * @clk: clock signal source
@@ -1994,7 +2344,7 @@ EXPORT_SYMBOL_GPL(clk_set_parent);
  */
 int clk_set_phase(struct clk *clk, int degrees)
 {
-	int ret = -EINVAL;
+	int ret;
 
 	if (!clk)
 		return 0;
@@ -2006,15 +2356,13 @@ int clk_set_phase(struct clk *clk, int degrees)
 
 	clk_prepare_lock();
 
-	trace_clk_set_phase(clk->core, degrees);
+	if (clk->exclusive_count)
+		clk_core_rate_unprotect(clk->core);
 
-	if (clk->core->ops->set_phase)
-		ret = clk->core->ops->set_phase(clk->core->hw, degrees);
+	ret = clk_core_set_phase_nolock(clk->core, degrees);
 
-	trace_clk_set_phase_complete(clk->core, degrees);
-
-	if (!ret)
-		clk->core->phase = degrees;
+	if (clk->exclusive_count)
+		clk_core_rate_protect(clk->core);
 
 	clk_prepare_unlock();
 
@@ -2102,11 +2450,12 @@ static void clk_summary_show_one(struct seq_file *s, struct clk_core *c,
 	if (!c)
 		return;
 
-	seq_printf(s, "%*s%-*s %11d %12d %11lu %10lu %-3d\n",
+	seq_printf(s, "%*s%-*s %7d %8d %8d %11lu %10lu %-3d\n",
 		   level * 3 + 1, "",
 		   30 - level * 3, c->name,
-		   c->enable_count, c->prepare_count, clk_core_get_rate(c),
-		   clk_core_get_accuracy(c), clk_core_get_phase(c));
+		   c->enable_count, c->prepare_count, c->protect_count,
+		   clk_core_get_rate(c), clk_core_get_accuracy(c),
+		   clk_core_get_phase(c));
 }
 
 static void clk_summary_show_subtree(struct seq_file *s, struct clk_core *c,
@@ -2128,7 +2477,8 @@ static int clk_summary_show(struct seq_file *s, void *data)
 	struct clk_core *c;
 	struct hlist_head **lists = (struct hlist_head **)s->private;
 
-	seq_puts(s, "   clock                         enable_cnt  prepare_cnt        rate   accuracy   phase\n");
+	seq_puts(s, "                                 enable  prepare  protect                               \n");
+	seq_puts(s, "   clock                          count    count    count        rate   accuracy   phase\n");
 	seq_puts(s, "----------------------------------------------------------------------------------------\n");
 
 	clk_prepare_lock();
@@ -2164,6 +2514,7 @@ static void clk_dump_one(struct seq_file *s, struct clk_core *c, int level)
 	seq_printf(s, "\"%s\": { ", c->name);
 	seq_printf(s, "\"enable_count\": %d,", c->enable_count);
 	seq_printf(s, "\"prepare_count\": %d,", c->prepare_count);
+	seq_printf(s, "\"protect_count\": %d,", c->protect_count);
 	seq_printf(s, "\"rate\": %lu,", clk_core_get_rate(c));
 	seq_printf(s, "\"accuracy\": %lu,", clk_core_get_accuracy(c));
 	seq_printf(s, "\"phase\": %d", clk_core_get_phase(c));
@@ -2223,6 +2574,58 @@ static const struct file_operations clk_dump_fops = {
 	.release	= single_release,
 };
 
+static const struct {
+	unsigned long flag;
+	const char *name;
+} clk_flags[] = {
+#define ENTRY(f) { f, __stringify(f) }
+	ENTRY(CLK_SET_RATE_GATE),
+	ENTRY(CLK_SET_PARENT_GATE),
+	ENTRY(CLK_SET_RATE_PARENT),
+	ENTRY(CLK_IGNORE_UNUSED),
+	ENTRY(CLK_IS_BASIC),
+	ENTRY(CLK_GET_RATE_NOCACHE),
+	ENTRY(CLK_SET_RATE_NO_REPARENT),
+	ENTRY(CLK_GET_ACCURACY_NOCACHE),
+	ENTRY(CLK_RECALC_NEW_RATES),
+	ENTRY(CLK_SET_RATE_UNGATE),
+	ENTRY(CLK_IS_CRITICAL),
+	ENTRY(CLK_OPS_PARENT_ENABLE),
+#undef ENTRY
+};
+
+static int clk_flags_dump(struct seq_file *s, void *data)
+{
+	struct clk_core *core = s->private;
+	unsigned long flags = core->flags;
+	unsigned int i;
+
+	for (i = 0; flags && i < ARRAY_SIZE(clk_flags); i++) {
+		if (flags & clk_flags[i].flag) {
+			seq_printf(s, "%s\n", clk_flags[i].name);
+			flags &= ~clk_flags[i].flag;
+		}
+	}
+	if (flags) {
+		/* Unknown flags */
+		seq_printf(s, "0x%lx\n", flags);
+	}
+
+	return 0;
+}
+
+static int clk_flags_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, clk_flags_dump, inode->i_private);
+}
+
+static const struct file_operations clk_flags_fops = {
+	.open		= clk_flags_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
 static int possible_parents_dump(struct seq_file *s, void *data)
 {
 	struct clk_core *core = s->private;
@@ -2264,43 +2667,46 @@ static int clk_debug_create_one(struct clk_core *core, struct dentry *pdentry)
 
 	core->dentry = d;
 
-	d = debugfs_create_u32("clk_rate", S_IRUGO, core->dentry,
-			(u32 *)&core->rate);
+	d = debugfs_create_ulong("clk_rate", 0444, core->dentry, &core->rate);
 	if (!d)
 		goto err_out;
 
-	d = debugfs_create_u32("clk_accuracy", S_IRUGO, core->dentry,
-			(u32 *)&core->accuracy);
+	d = debugfs_create_ulong("clk_accuracy", 0444, core->dentry,
+				 &core->accuracy);
 	if (!d)
 		goto err_out;
 
-	d = debugfs_create_u32("clk_phase", S_IRUGO, core->dentry,
-			(u32 *)&core->phase);
+	d = debugfs_create_u32("clk_phase", 0444, core->dentry, &core->phase);
 	if (!d)
 		goto err_out;
 
-	d = debugfs_create_x32("clk_flags", S_IRUGO, core->dentry,
-			(u32 *)&core->flags);
+	d = debugfs_create_file("clk_flags", 0444, core->dentry, core,
+				&clk_flags_fops);
 	if (!d)
 		goto err_out;
 
-	d = debugfs_create_u32("clk_prepare_count", S_IRUGO, core->dentry,
-			(u32 *)&core->prepare_count);
+	d = debugfs_create_u32("clk_prepare_count", 0444, core->dentry,
+			       &core->prepare_count);
 	if (!d)
 		goto err_out;
 
-	d = debugfs_create_u32("clk_enable_count", S_IRUGO, core->dentry,
-			(u32 *)&core->enable_count);
+	d = debugfs_create_u32("clk_enable_count", 0444, core->dentry,
+			       &core->enable_count);
 	if (!d)
 		goto err_out;
 
-	d = debugfs_create_u32("clk_notifier_count", S_IRUGO, core->dentry,
-			(u32 *)&core->notifier_count);
+	d = debugfs_create_u32("clk_protect_count", 0444, core->dentry,
+			       &core->protect_count);
+	if (!d)
+		goto err_out;
+
+	d = debugfs_create_u32("clk_notifier_count", 0444, core->dentry,
+			       &core->notifier_count);
 	if (!d)
 		goto err_out;
 
 	if (core->num_parents > 1) {
-		d = debugfs_create_file("clk_possible_parents", S_IRUGO,
+		d = debugfs_create_file("clk_possible_parents", 0444,
 				core->dentry, core, &possible_parents_fops);
 		if (!d)
 			goto err_out;
@@ -2336,12 +2742,8 @@ static int clk_debug_register(struct clk_core *core)
 
 	mutex_lock(&clk_debug_lock);
 	hlist_add_head(&core->debug_node, &clk_debug_list);
-
-	if (!inited)
-		goto unlock;
-
-	ret = clk_debug_create_one(core, rootdir);
-unlock:
+	if (inited)
+		ret = clk_debug_create_one(core, rootdir);
 	mutex_unlock(&clk_debug_lock);
 
 	return ret;
@@ -2396,22 +2798,22 @@ static int __init clk_debug_init(void)
 	if (!rootdir)
 		return -ENOMEM;
 
-	d = debugfs_create_file("clk_summary", S_IRUGO, rootdir, &all_lists,
+	d = debugfs_create_file("clk_summary", 0444, rootdir, &all_lists,
 				&clk_summary_fops);
 	if (!d)
 		return -ENOMEM;
 
-	d = debugfs_create_file("clk_dump", S_IRUGO, rootdir, &all_lists,
+	d = debugfs_create_file("clk_dump", 0444, rootdir, &all_lists,
 				&clk_dump_fops);
 	if (!d)
 		return -ENOMEM;
 
-	d = debugfs_create_file("clk_orphan_summary", S_IRUGO, rootdir,
+	d = debugfs_create_file("clk_orphan_summary", 0444, rootdir,
 				&orphan_list, &clk_summary_fops);
 	if (!d)
 		return -ENOMEM;
 
-	d = debugfs_create_file("clk_orphan_dump", S_IRUGO, rootdir,
+	d = debugfs_create_file("clk_orphan_dump", 0444, rootdir,
 				&orphan_list, &clk_dump_fops);
 	if (!d)
 		return -ENOMEM;
@@ -2570,6 +2972,21 @@ static int __clk_core_init(struct clk_core *core)
 		rate = 0;
 	core->rate = core->req_rate = rate;
 
+	/*
+	 * Enable CLK_IS_CRITICAL clocks so newly added critical clocks
+	 * don't get accidentally disabled when walking the orphan tree and
+	 * reparenting clocks
+	 */
+	if (core->flags & CLK_IS_CRITICAL) {
+		unsigned long flags;
+
+		clk_core_prepare(core);
+
+		flags = clk_enable_lock();
+		clk_core_enable(core);
+		clk_enable_unlock(flags);
+	}
+
 	/*
 	 * walk the list of orphan clocks and reparent any that newly finds a
 	 * parent.
@@ -2578,10 +2995,13 @@ static int __clk_core_init(struct clk_core *core)
 		struct clk_core *parent = __clk_init_parent(orphan);
 
 		/*
-		 * we could call __clk_set_parent, but that would result in a
-		 * redundant call to the .set_rate op, if it exists
+		 * We need to use __clk_set_parent_before() and _after() to
+		 * to properly migrate any prepare/enable count of the orphan
+		 * clock. This is important for CLK_IS_CRITICAL clocks, which
+		 * are enabled during init but might not have a parent yet.
 		 */
 		if (parent) {
+			/* update the clk tree topology */
 			__clk_set_parent_before(orphan, parent);
 			__clk_set_parent_after(orphan, parent, NULL);
 			__clk_recalc_accuracies(orphan);
@@ -2600,16 +3020,6 @@ static int __clk_core_init(struct clk_core *core)
 	if (core->ops->init)
 		core->ops->init(core->hw);
 
-	if (core->flags & CLK_IS_CRITICAL) {
-		unsigned long flags;
-
-		clk_core_prepare(core);
-
-		flags = clk_enable_lock();
-		clk_core_enable(core);
-		clk_enable_unlock(flags);
-	}
-
 	kref_init(&core->ref);
 out:
 	clk_pm_runtime_put(core);
@@ -2684,7 +3094,13 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw)
 		ret = -ENOMEM;
 		goto fail_name;
 	}
+
+	if (WARN_ON(!hw->init->ops)) {
+		ret = -EINVAL;
+		goto fail_ops;
+	}
 	core->ops = hw->init->ops;
+
 	if (dev && pm_runtime_enabled(dev))
 		core->dev = dev;
 	if (dev && dev->driver)
@@ -2746,6 +3162,7 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw)
 		kfree_const(core->parent_names[i]);
 	kfree(core->parent_names);
 fail_parent_names:
+fail_ops:
 	kfree_const(core->name);
 fail_name:
 	kfree(core);
@@ -2857,7 +3274,7 @@ void clk_unregister(struct clk *clk)
 		/* Reparent all children to the orphan list. */
 		hlist_for_each_entry_safe(child, t, &clk->core->children,
 					  child_node)
-			clk_core_set_parent(child, NULL);
+			clk_core_set_parent_nolock(child, NULL);
 	}
 
 	hlist_del_init(&clk->core->child_node);
@@ -2865,6 +3282,11 @@ void clk_unregister(struct clk *clk)
 	if (clk->core->prepare_count)
 		pr_warn("%s: unregistering prepared clock: %s\n",
 					__func__, clk->core->name);
+
+	if (clk->core->protect_count)
+		pr_warn("%s: unregistering protected clock: %s\n",
+					__func__, clk->core->name);
+
 	kref_put(&clk->core->ref, __clk_release);
 unlock:
 	clk_prepare_unlock();
@@ -3023,6 +3445,18 @@ void __clk_put(struct clk *clk)
 
 	clk_prepare_lock();
 
+	/*
+	 * Before calling clk_put, all calls to clk_rate_exclusive_get() from a
+	 * given user should be balanced with calls to clk_rate_exclusive_put()
+	 * and by that same consumer
+	 */
+	if (WARN_ON(clk->exclusive_count)) {
+		/* We voiced our concern, let's sanitize the situation */
+		clk->core->protect_count -= (clk->exclusive_count - 1);
+		clk_core_rate_unprotect(clk->core);
+		clk->exclusive_count = 0;
+	}
+
 	hlist_del(&clk->clks_node);
 	if (clk->min_rate > clk->core->req_rate ||
 	    clk->max_rate < clk->core->req_rate)
@@ -3559,7 +3993,7 @@ static int parent_ready(struct device_node *np)
  * of_clk_detect_critical() - set CLK_IS_CRITICAL flag from Device Tree
  * @np: Device node pointer associated with clock provider
  * @index: clock index
- * @flags: pointer to clk_core->flags
+ * @flags: pointer to top-level framework flags
  *
  * Detects if the clock-critical property exists and, if so, sets the
  * corresponding CLK_IS_CRITICAL flag.
diff --git a/drivers/clk/clk.h b/drivers/clk/clk.h
index 00b35a13cdf3..70c0ba6336c1 100644
--- a/drivers/clk/clk.h
+++ b/drivers/clk/clk.h
@@ -20,6 +20,8 @@ struct clk *__of_clk_get_from_provider(struct of_phandle_args *clkspec,
 struct clk *__clk_create_clk(struct clk_hw *hw, const char *dev_id,
 			     const char *con_id);
 void __clk_free_clk(struct clk *clk);
+int __clk_get(struct clk *clk);
+void __clk_put(struct clk *clk);
 #else
 /* All these casts to avoid ifdefs in clkdev... */
 static inline struct clk *
@@ -32,5 +34,7 @@ static struct clk_hw *__clk_get_hw(struct clk *clk)
 {
 	return (struct clk_hw *)clk;
 }
+static inline int __clk_get(struct clk *clk) { return 1; }
+static inline void __clk_put(struct clk *clk) { }
 
 #endif
diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c
index 6b2f29df3f70..7513411140b6 100644
--- a/drivers/clk/clkdev.c
+++ b/drivers/clk/clkdev.c
@@ -256,7 +256,7 @@ vclkdev_alloc(struct clk_hw *hw, const char *con_id, const char *dev_fmt,
 {
 	struct clk_lookup_alloc *cla;
 
-	cla = __clkdev_alloc(sizeof(*cla));
+	cla = kzalloc(sizeof(*cla), GFP_KERNEL);
 	if (!cla)
 		return NULL;
 
diff --git a/drivers/clk/h8300/clk-div.c b/drivers/clk/h8300/clk-div.c
index 4ae624425e9d..d413ade95c99 100644
--- a/drivers/clk/h8300/clk-div.c
+++ b/drivers/clk/h8300/clk-div.c
@@ -24,13 +24,13 @@ static void __init h8300_div_clk_setup(struct device_node *node)
 
 	num_parents = of_clk_get_parent_count(node);
 	if (!num_parents) {
-		pr_err("%s: no parent found", clk_name);
+		pr_err("%s: no parent found\n", clk_name);
 		return;
 	}
 
 	divcr = of_iomap(node, 0);
 	if (divcr == NULL) {
-		pr_err("%s: failed to map divide register", clk_name);
+		pr_err("%s: failed to map divide register\n", clk_name);
 		goto error;
 	}
 	offset = (unsigned long)divcr & 3;
diff --git a/drivers/clk/h8300/clk-h8s2678.c b/drivers/clk/h8300/clk-h8s2678.c
index fc24b0b55a3d..b68045d8b921 100644
--- a/drivers/clk/h8300/clk-h8s2678.c
+++ b/drivers/clk/h8300/clk-h8s2678.c
@@ -93,7 +93,7 @@ static void __init h8s2678_pll_clk_setup(struct device_node *node)
 
 	num_parents = of_clk_get_parent_count(node);
 	if (!num_parents) {
-		pr_err("%s: no parent found", clk_name);
+		pr_err("%s: no parent found\n", clk_name);
 		return;
 	}
 
@@ -104,13 +104,13 @@ static void __init h8s2678_pll_clk_setup(struct device_node *node)
 
 	pll_clock->sckcr = of_iomap(node, 0);
 	if (pll_clock->sckcr == NULL) {
-		pr_err("%s: failed to map divide register", clk_name);
+		pr_err("%s: failed to map divide register\n", clk_name);
 		goto free_clock;
 	}
 
 	pll_clock->pllcr = of_iomap(node, 1);
 	if (pll_clock->pllcr == NULL) {
-		pr_err("%s: failed to map multiply register", clk_name);
+		pr_err("%s: failed to map multiply register\n", clk_name);
 		goto unmap_sckcr;
 	}
 
diff --git a/drivers/clk/hisilicon/Kconfig b/drivers/clk/hisilicon/Kconfig
index 7098bfd32b1b..1bd43550e4c8 100644
--- a/drivers/clk/hisilicon/Kconfig
+++ b/drivers/clk/hisilicon/Kconfig
@@ -49,3 +49,9 @@ config STUB_CLK_HI6220
 	default ARCH_HISI
 	help
 	  Build the Hisilicon Hi6220 stub clock driver.
+
+config STUB_CLK_HI3660
+	bool "Hi3660 Stub Clock Driver"
+	depends on COMMON_CLK_HI3660 && MAILBOX
+	help
+	  Build the Hisilicon Hi3660 stub clock driver.
diff --git a/drivers/clk/hisilicon/Makefile b/drivers/clk/hisilicon/Makefile
index 0e55612112af..4806fc2cb4ac 100644
--- a/drivers/clk/hisilicon/Makefile
+++ b/drivers/clk/hisilicon/Makefile
@@ -15,3 +15,4 @@ obj-$(CONFIG_COMMON_CLK_HI3798CV200)	+= crg-hi3798cv200.o
 obj-$(CONFIG_COMMON_CLK_HI6220)	+= clk-hi6220.o
 obj-$(CONFIG_RESET_HISI)	+= reset.o
 obj-$(CONFIG_STUB_CLK_HI6220)	+= clk-hi6220-stub.o
+obj-$(CONFIG_STUB_CLK_HI3660)	+= clk-hi3660-stub.o
diff --git a/drivers/clk/hisilicon/clk-hi3660-stub.c b/drivers/clk/hisilicon/clk-hi3660-stub.c
new file mode 100644
index 000000000000..e8b2c43b1bb8
--- /dev/null
+++ b/drivers/clk/hisilicon/clk-hi3660-stub.c
@@ -0,0 +1,187 @@
+/*
+ * Hisilicon clock driver
+ *
+ * Copyright (c) 2013-2017 Hisilicon Limited.
+ * Copyright (c) 2017 Linaro Limited.
+ *
+ * Author: Kai Zhao <zhaokai1@hisilicon.com>
+ *	    Tao Wang <kevin.wangtao@hisilicon.com>
+ *	    Leo Yan <leo.yan@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/clock/hi3660-clock.h>
+
+#define HI3660_STUB_CLOCK_DATA		(0x70)
+#define MHZ				(1000 * 1000)
+
+#define DEFINE_CLK_STUB(_id, _cmd, _name)			\
+	{							\
+		.id = (_id),					\
+		.cmd = (_cmd),					\
+		.hw.init = &(struct clk_init_data) {		\
+			.name = #_name,				\
+			.ops = &hi3660_stub_clk_ops,		\
+			.num_parents = 0,			\
+			.flags = CLK_GET_RATE_NOCACHE,		\
+		},						\
+	},
+
+#define to_stub_clk(_hw) container_of(_hw, struct hi3660_stub_clk, hw)
+
+struct hi3660_stub_clk_chan {
+	struct mbox_client cl;
+	struct mbox_chan *mbox;
+};
+
+struct hi3660_stub_clk {
+	unsigned int id;
+	struct clk_hw hw;
+	unsigned int cmd;
+	unsigned int msg[8];
+	unsigned int rate;
+};
+
+static void __iomem *freq_reg;
+static struct hi3660_stub_clk_chan stub_clk_chan;
+
+static unsigned long hi3660_stub_clk_recalc_rate(struct clk_hw *hw,
+						 unsigned long parent_rate)
+{
+	struct hi3660_stub_clk *stub_clk = to_stub_clk(hw);
+
+	/*
+	 * LPM3 writes back the CPU frequency in shared SRAM so read
+	 * back the frequency.
+	 */
+	stub_clk->rate = readl(freq_reg + (stub_clk->id << 2)) * MHZ;
+	return stub_clk->rate;
+}
+
+static long hi3660_stub_clk_round_rate(struct clk_hw *hw, unsigned long rate,
+				       unsigned long *prate)
+{
+	/*
+	 * LPM3 handles rate rounding so just return whatever
+	 * rate is requested.
+	 */
+	return rate;
+}
+
+static int hi3660_stub_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long parent_rate)
+{
+	struct hi3660_stub_clk *stub_clk = to_stub_clk(hw);
+
+	stub_clk->msg[0] = stub_clk->cmd;
+	stub_clk->msg[1] = rate / MHZ;
+
+	dev_dbg(stub_clk_chan.cl.dev, "set rate msg[0]=0x%x msg[1]=0x%x\n",
+		stub_clk->msg[0], stub_clk->msg[1]);
+
+	mbox_send_message(stub_clk_chan.mbox, stub_clk->msg);
+	mbox_client_txdone(stub_clk_chan.mbox, 0);
+
+	stub_clk->rate = rate;
+	return 0;
+}
+
+static const struct clk_ops hi3660_stub_clk_ops = {
+	.recalc_rate    = hi3660_stub_clk_recalc_rate,
+	.round_rate     = hi3660_stub_clk_round_rate,
+	.set_rate       = hi3660_stub_clk_set_rate,
+};
+
+static struct hi3660_stub_clk hi3660_stub_clks[HI3660_CLK_STUB_NUM] = {
+	DEFINE_CLK_STUB(HI3660_CLK_STUB_CLUSTER0, 0x0001030A, "cpu-cluster.0")
+	DEFINE_CLK_STUB(HI3660_CLK_STUB_CLUSTER1, 0x0002030A, "cpu-cluster.1")
+	DEFINE_CLK_STUB(HI3660_CLK_STUB_GPU, 0x0003030A, "clk-g3d")
+	DEFINE_CLK_STUB(HI3660_CLK_STUB_DDR, 0x00040309, "clk-ddrc")
+};
+
+static struct clk_hw *hi3660_stub_clk_hw_get(struct of_phandle_args *clkspec,
+					     void *data)
+{
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= HI3660_CLK_STUB_NUM) {
+		pr_err("%s: invalid index %u\n", __func__, idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &hi3660_stub_clks[idx].hw;
+}
+
+static int hi3660_stub_clk_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	unsigned int i;
+	int ret;
+
+	/* Use mailbox client without blocking */
+	stub_clk_chan.cl.dev = dev;
+	stub_clk_chan.cl.tx_done = NULL;
+	stub_clk_chan.cl.tx_block = false;
+	stub_clk_chan.cl.knows_txdone = false;
+
+	/* Allocate mailbox channel */
+	stub_clk_chan.mbox = mbox_request_channel(&stub_clk_chan.cl, 0);
+	if (IS_ERR(stub_clk_chan.mbox))
+		return PTR_ERR(stub_clk_chan.mbox);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -EINVAL;
+	freq_reg = devm_ioremap(dev, res->start, resource_size(res));
+	if (!freq_reg)
+		return -ENOMEM;
+
+	freq_reg += HI3660_STUB_CLOCK_DATA;
+
+	for (i = 0; i < HI3660_CLK_STUB_NUM; i++) {
+		ret = devm_clk_hw_register(&pdev->dev, &hi3660_stub_clks[i].hw);
+		if (ret)
+			return ret;
+	}
+
+	return devm_of_clk_add_hw_provider(&pdev->dev, hi3660_stub_clk_hw_get,
+					   hi3660_stub_clks);
+}
+
+static const struct of_device_id hi3660_stub_clk_of_match[] = {
+	{ .compatible = "hisilicon,hi3660-stub-clk", },
+	{}
+};
+
+static struct platform_driver hi3660_stub_clk_driver = {
+	.probe	= hi3660_stub_clk_probe,
+	.driver = {
+		.name = "hi3660-stub-clk",
+		.of_match_table = hi3660_stub_clk_of_match,
+	},
+};
+
+static int __init hi3660_stub_clk_init(void)
+{
+	return platform_driver_register(&hi3660_stub_clk_driver);
+}
+subsys_initcall(hi3660_stub_clk_init);
diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c
index a1c1f684ad58..9f46cf9dcc65 100644
--- a/drivers/clk/hisilicon/clkdivider-hi6220.c
+++ b/drivers/clk/hisilicon/clkdivider-hi6220.c
@@ -56,7 +56,7 @@ static unsigned long hi6220_clkdiv_recalc_rate(struct clk_hw *hw,
 	val &= div_mask(dclk->width);
 
 	return divider_recalc_rate(hw, parent_rate, val, dclk->table,
-				   CLK_DIVIDER_ROUND_CLOSEST);
+				   CLK_DIVIDER_ROUND_CLOSEST, dclk->width);
 }
 
 static long hi6220_clkdiv_round_rate(struct clk_hw *hw, unsigned long rate,
diff --git a/drivers/clk/imx/clk-imx51-imx53.c b/drivers/clk/imx/clk-imx51-imx53.c
index 7bcaf270db11..caa8bd40692c 100644
--- a/drivers/clk/imx/clk-imx51-imx53.c
+++ b/drivers/clk/imx/clk-imx51-imx53.c
@@ -131,7 +131,17 @@ static const char *ieee1588_sels[] = { "pll3_sw", "pll4_sw", "dummy" /* usbphy2_
 static struct clk *clk[IMX5_CLK_END];
 static struct clk_onecell_data clk_data;
 
-static struct clk ** const uart_clks[] __initconst = {
+static struct clk ** const uart_clks_mx51[] __initconst = {
+	&clk[IMX5_CLK_UART1_IPG_GATE],
+	&clk[IMX5_CLK_UART1_PER_GATE],
+	&clk[IMX5_CLK_UART2_IPG_GATE],
+	&clk[IMX5_CLK_UART2_PER_GATE],
+	&clk[IMX5_CLK_UART3_IPG_GATE],
+	&clk[IMX5_CLK_UART3_PER_GATE],
+	NULL
+};
+
+static struct clk ** const uart_clks_mx50_mx53[] __initconst = {
 	&clk[IMX5_CLK_UART1_IPG_GATE],
 	&clk[IMX5_CLK_UART1_PER_GATE],
 	&clk[IMX5_CLK_UART2_IPG_GATE],
@@ -257,10 +267,6 @@ static void __init mx5_clocks_common_init(void __iomem *ccm_base)
 	clk[IMX5_CLK_VPU_SEL]		= imx_clk_mux("vpu_sel", MXC_CCM_CBCMR, 14, 2, vpu_sel, ARRAY_SIZE(vpu_sel));
 	clk[IMX5_CLK_VPU_GATE]		= imx_clk_gate2("vpu_gate", "vpu_sel", MXC_CCM_CCGR5, 6);
 	clk[IMX5_CLK_VPU_REFERENCE_GATE] = imx_clk_gate2("vpu_reference_gate", "osc", MXC_CCM_CCGR5, 8);
-	clk[IMX5_CLK_UART4_IPG_GATE]	= imx_clk_gate2("uart4_ipg_gate", "ipg", MXC_CCM_CCGR7, 8);
-	clk[IMX5_CLK_UART4_PER_GATE]	= imx_clk_gate2("uart4_per_gate", "uart_root", MXC_CCM_CCGR7, 10);
-	clk[IMX5_CLK_UART5_IPG_GATE]	= imx_clk_gate2("uart5_ipg_gate", "ipg", MXC_CCM_CCGR7, 12);
-	clk[IMX5_CLK_UART5_PER_GATE]	= imx_clk_gate2("uart5_per_gate", "uart_root", MXC_CCM_CCGR7, 14);
 	clk[IMX5_CLK_GPC_DVFS]		= imx_clk_gate2("gpc_dvfs", "dummy", MXC_CCM_CCGR5, 24);
 
 	clk[IMX5_CLK_SSI_APM]		= imx_clk_mux("ssi_apm", MXC_CCM_CSCMR1, 8, 2, ssi_apm_sels, ARRAY_SIZE(ssi_apm_sels));
@@ -325,8 +331,6 @@ static void __init mx5_clocks_common_init(void __iomem *ccm_base)
 	clk_prepare_enable(clk[IMX5_CLK_TMAX1]);
 	clk_prepare_enable(clk[IMX5_CLK_TMAX2]); /* esdhc2, fec */
 	clk_prepare_enable(clk[IMX5_CLK_TMAX3]); /* esdhc1, esdhc4 */
-
-	imx_register_uart_clocks(uart_clks);
 }
 
 static void __init mx50_clocks_init(struct device_node *np)
@@ -361,6 +365,10 @@ static void __init mx50_clocks_init(struct device_node *np)
 	clk[IMX5_CLK_USB_PHY1_GATE]	= imx_clk_gate2("usb_phy1_gate", "usb_phy_sel", MXC_CCM_CCGR4, 10);
 	clk[IMX5_CLK_USB_PHY2_GATE]	= imx_clk_gate2("usb_phy2_gate", "usb_phy_sel", MXC_CCM_CCGR4, 12);
 	clk[IMX5_CLK_I2C3_GATE]		= imx_clk_gate2("i2c3_gate", "per_root", MXC_CCM_CCGR1, 22);
+	clk[IMX5_CLK_UART4_IPG_GATE]	= imx_clk_gate2("uart4_ipg_gate", "ipg", MXC_CCM_CCGR7, 8);
+	clk[IMX5_CLK_UART4_PER_GATE]	= imx_clk_gate2("uart4_per_gate", "uart_root", MXC_CCM_CCGR7, 10);
+	clk[IMX5_CLK_UART5_IPG_GATE]	= imx_clk_gate2("uart5_ipg_gate", "ipg", MXC_CCM_CCGR7, 12);
+	clk[IMX5_CLK_UART5_PER_GATE]	= imx_clk_gate2("uart5_per_gate", "uart_root", MXC_CCM_CCGR7, 14);
 
 	clk[IMX5_CLK_CKO1_SEL]		= imx_clk_mux("cko1_sel", MXC_CCM_CCOSR, 0, 4,
 						mx53_cko1_sel, ARRAY_SIZE(mx53_cko1_sel));
@@ -388,6 +396,8 @@ static void __init mx50_clocks_init(struct device_node *np)
 
 	r = clk_round_rate(clk[IMX5_CLK_USBOH3_PER_GATE], 54000000);
 	clk_set_rate(clk[IMX5_CLK_USBOH3_PER_GATE], r);
+
+	imx_register_uart_clocks(uart_clks_mx50_mx53);
 }
 CLK_OF_DECLARE(imx50_ccm, "fsl,imx50-ccm", mx50_clocks_init);
 
@@ -477,6 +487,8 @@ static void __init mx51_clocks_init(struct device_node *np)
 	val = readl(MXC_CCM_CLPCR);
 	val |= 1 << 23;
 	writel(val, MXC_CCM_CLPCR);
+
+	imx_register_uart_clocks(uart_clks_mx51);
 }
 CLK_OF_DECLARE(imx51_ccm, "fsl,imx51-ccm", mx51_clocks_init);
 
@@ -562,6 +574,10 @@ static void __init mx53_clocks_init(struct device_node *np)
 	clk[IMX5_CLK_IEEE1588_PRED]	= imx_clk_divider("ieee1588_pred", "ieee1588_sel", MXC_CCM_CSCDR2, 6, 3);
 	clk[IMX5_CLK_IEEE1588_PODF]	= imx_clk_divider("ieee1588_podf", "ieee1588_pred", MXC_CCM_CSCDR2, 0, 6);
 	clk[IMX5_CLK_IEEE1588_GATE]	= imx_clk_gate2("ieee1588_serial_gate", "ieee1588_podf", MXC_CCM_CCGR7, 6);
+	clk[IMX5_CLK_UART4_IPG_GATE]	= imx_clk_gate2("uart4_ipg_gate", "ipg", MXC_CCM_CCGR7, 8);
+	clk[IMX5_CLK_UART4_PER_GATE]	= imx_clk_gate2("uart4_per_gate", "uart_root", MXC_CCM_CCGR7, 10);
+	clk[IMX5_CLK_UART5_IPG_GATE]	= imx_clk_gate2("uart5_ipg_gate", "ipg", MXC_CCM_CCGR7, 12);
+	clk[IMX5_CLK_UART5_PER_GATE]	= imx_clk_gate2("uart5_per_gate", "uart_root", MXC_CCM_CCGR7, 14);
 
 	clk[IMX5_CLK_CKO1_SEL]		= imx_clk_mux("cko1_sel", MXC_CCM_CCOSR, 0, 4,
 						mx53_cko1_sel, ARRAY_SIZE(mx53_cko1_sel));
@@ -602,5 +618,7 @@ static void __init mx53_clocks_init(struct device_node *np)
 
 	r = clk_round_rate(clk[IMX5_CLK_USBOH3_PER_GATE], 54000000);
 	clk_set_rate(clk[IMX5_CLK_USBOH3_PER_GATE], r);
+
+	imx_register_uart_clocks(uart_clks_mx50_mx53);
 }
 CLK_OF_DECLARE(imx53_ccm, "fsl,imx53-ccm", mx53_clocks_init);
diff --git a/drivers/clk/ingenic/Makefile b/drivers/clk/ingenic/Makefile
index cd47b0664c2b..1456e4cdb562 100644
--- a/drivers/clk/ingenic/Makefile
+++ b/drivers/clk/ingenic/Makefile
@@ -1,3 +1,4 @@
 obj-y				+= cgu.o
 obj-$(CONFIG_MACH_JZ4740)	+= jz4740-cgu.o
+obj-$(CONFIG_MACH_JZ4770)	+= jz4770-cgu.o
 obj-$(CONFIG_MACH_JZ4780)	+= jz4780-cgu.o
diff --git a/drivers/clk/ingenic/cgu.c b/drivers/clk/ingenic/cgu.c
index ab393637f7b0..56a712c9075f 100644
--- a/drivers/clk/ingenic/cgu.c
+++ b/drivers/clk/ingenic/cgu.c
@@ -100,15 +100,13 @@ ingenic_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	n += pll_info->n_offset;
 	od_enc = ctl >> pll_info->od_shift;
 	od_enc &= GENMASK(pll_info->od_bits - 1, 0);
-	bypass = !!(ctl & BIT(pll_info->bypass_bit));
+	bypass = !pll_info->no_bypass_bit &&
+		 !!(ctl & BIT(pll_info->bypass_bit));
 	enable = !!(ctl & BIT(pll_info->enable_bit));
 
 	if (bypass)
 		return parent_rate;
 
-	if (!enable)
-		return 0;
-
 	for (od = 0; od < pll_info->od_max; od++) {
 		if (pll_info->od_encoding[od] == od_enc)
 			break;
@@ -152,17 +150,25 @@ ingenic_pll_calc(const struct ingenic_cgu_clk_info *clk_info,
 	return div_u64((u64)parent_rate * m, n * od);
 }
 
-static long
-ingenic_pll_round_rate(struct clk_hw *hw, unsigned long req_rate,
-		       unsigned long *prate)
+static inline const struct ingenic_cgu_clk_info *to_clk_info(
+		struct ingenic_clk *ingenic_clk)
 {
-	struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw);
 	struct ingenic_cgu *cgu = ingenic_clk->cgu;
 	const struct ingenic_cgu_clk_info *clk_info;
 
 	clk_info = &cgu->clock_info[ingenic_clk->idx];
 	BUG_ON(clk_info->type != CGU_CLK_PLL);
 
+	return clk_info;
+}
+
+static long
+ingenic_pll_round_rate(struct clk_hw *hw, unsigned long req_rate,
+		       unsigned long *prate)
+{
+	struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw);
+	const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk);
+
 	return ingenic_pll_calc(clk_info, req_rate, *prate, NULL, NULL, NULL);
 }
 
@@ -170,19 +176,14 @@ static int
 ingenic_pll_set_rate(struct clk_hw *hw, unsigned long req_rate,
 		     unsigned long parent_rate)
 {
-	const unsigned timeout = 100;
 	struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw);
 	struct ingenic_cgu *cgu = ingenic_clk->cgu;
-	const struct ingenic_cgu_clk_info *clk_info;
-	const struct ingenic_cgu_pll_info *pll_info;
+	const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk);
+	const struct ingenic_cgu_pll_info *pll_info = &clk_info->pll;
 	unsigned long rate, flags;
-	unsigned m, n, od, i;
+	unsigned int m, n, od;
 	u32 ctl;
 
-	clk_info = &cgu->clock_info[ingenic_clk->idx];
-	BUG_ON(clk_info->type != CGU_CLK_PLL);
-	pll_info = &clk_info->pll;
-
 	rate = ingenic_pll_calc(clk_info, req_rate, parent_rate,
 			       &m, &n, &od);
 	if (rate != req_rate)
@@ -201,6 +202,26 @@ ingenic_pll_set_rate(struct clk_hw *hw, unsigned long req_rate,
 	ctl &= ~(GENMASK(pll_info->od_bits - 1, 0) << pll_info->od_shift);
 	ctl |= pll_info->od_encoding[od - 1] << pll_info->od_shift;
 
+	writel(ctl, cgu->base + pll_info->reg);
+	spin_unlock_irqrestore(&cgu->lock, flags);
+
+	return 0;
+}
+
+static int ingenic_pll_enable(struct clk_hw *hw)
+{
+	struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw);
+	struct ingenic_cgu *cgu = ingenic_clk->cgu;
+	const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk);
+	const struct ingenic_cgu_pll_info *pll_info = &clk_info->pll;
+	const unsigned int timeout = 100;
+	unsigned long flags;
+	unsigned int i;
+	u32 ctl;
+
+	spin_lock_irqsave(&cgu->lock, flags);
+	ctl = readl(cgu->base + pll_info->reg);
+
 	ctl &= ~BIT(pll_info->bypass_bit);
 	ctl |= BIT(pll_info->enable_bit);
 
@@ -222,10 +243,48 @@ ingenic_pll_set_rate(struct clk_hw *hw, unsigned long req_rate,
 	return 0;
 }
 
+static void ingenic_pll_disable(struct clk_hw *hw)
+{
+	struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw);
+	struct ingenic_cgu *cgu = ingenic_clk->cgu;
+	const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk);
+	const struct ingenic_cgu_pll_info *pll_info = &clk_info->pll;
+	unsigned long flags;
+	u32 ctl;
+
+	spin_lock_irqsave(&cgu->lock, flags);
+	ctl = readl(cgu->base + pll_info->reg);
+
+	ctl &= ~BIT(pll_info->enable_bit);
+
+	writel(ctl, cgu->base + pll_info->reg);
+	spin_unlock_irqrestore(&cgu->lock, flags);
+}
+
+static int ingenic_pll_is_enabled(struct clk_hw *hw)
+{
+	struct ingenic_clk *ingenic_clk = to_ingenic_clk(hw);
+	struct ingenic_cgu *cgu = ingenic_clk->cgu;
+	const struct ingenic_cgu_clk_info *clk_info = to_clk_info(ingenic_clk);
+	const struct ingenic_cgu_pll_info *pll_info = &clk_info->pll;
+	unsigned long flags;
+	u32 ctl;
+
+	spin_lock_irqsave(&cgu->lock, flags);
+	ctl = readl(cgu->base + pll_info->reg);
+	spin_unlock_irqrestore(&cgu->lock, flags);
+
+	return !!(ctl & BIT(pll_info->enable_bit));
+}
+
 static const struct clk_ops ingenic_pll_ops = {
 	.recalc_rate = ingenic_pll_recalc_rate,
 	.round_rate = ingenic_pll_round_rate,
 	.set_rate = ingenic_pll_set_rate,
+
+	.enable = ingenic_pll_enable,
+	.disable = ingenic_pll_disable,
+	.is_enabled = ingenic_pll_is_enabled,
 };
 
 /*
@@ -328,6 +387,8 @@ ingenic_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 		div *= clk_info->div.div;
 
 		rate /= div;
+	} else if (clk_info->type & CGU_CLK_FIXDIV) {
+		rate /= clk_info->fixdiv.div;
 	}
 
 	return rate;
@@ -598,6 +659,7 @@ static int ingenic_register_clock(struct ingenic_cgu *cgu, unsigned idx)
 		}
 	} else if (caps & CGU_CLK_PLL) {
 		clk_init.ops = &ingenic_pll_ops;
+		clk_init.flags |= CLK_SET_RATE_GATE;
 
 		caps &= ~CGU_CLK_PLL;
 
diff --git a/drivers/clk/ingenic/cgu.h b/drivers/clk/ingenic/cgu.h
index e78b586536ea..9da34910bd80 100644
--- a/drivers/clk/ingenic/cgu.h
+++ b/drivers/clk/ingenic/cgu.h
@@ -48,6 +48,7 @@
  * @bypass_bit: the index of the bypass bit in the PLL control register
  * @enable_bit: the index of the enable bit in the PLL control register
  * @stable_bit: the index of the stable bit in the PLL control register
+ * @no_bypass_bit: if set, the PLL has no bypass functionality
  */
 struct ingenic_cgu_pll_info {
 	unsigned reg;
@@ -58,6 +59,7 @@ struct ingenic_cgu_pll_info {
 	u8 bypass_bit;
 	u8 enable_bit;
 	u8 stable_bit;
+	bool no_bypass_bit;
 };
 
 /**
@@ -120,7 +122,7 @@ struct ingenic_cgu_gate_info {
  * @clk_ops: custom clock operation callbacks
  */
 struct ingenic_cgu_custom_info {
-	struct clk_ops *clk_ops;
+	const struct clk_ops *clk_ops;
 };
 
 /**
diff --git a/drivers/clk/ingenic/jz4770-cgu.c b/drivers/clk/ingenic/jz4770-cgu.c
new file mode 100644
index 000000000000..c78d369b9403
--- /dev/null
+++ b/drivers/clk/ingenic/jz4770-cgu.c
@@ -0,0 +1,483 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * JZ4770 SoC CGU driver
+ * Copyright 2018, Paul Cercueil <paul@crapouillou.net>
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/syscore_ops.h>
+#include <dt-bindings/clock/jz4770-cgu.h>
+#include "cgu.h"
+
+/*
+ * CPM registers offset address definition
+ */
+#define CGU_REG_CPCCR		0x00
+#define CGU_REG_LCR		0x04
+#define CGU_REG_CPPCR0		0x10
+#define CGU_REG_CLKGR0		0x20
+#define CGU_REG_OPCR		0x24
+#define CGU_REG_CLKGR1		0x28
+#define CGU_REG_CPPCR1		0x30
+#define CGU_REG_USBPCR1		0x48
+#define CGU_REG_USBCDR		0x50
+#define CGU_REG_I2SCDR		0x60
+#define CGU_REG_LPCDR		0x64
+#define CGU_REG_MSC0CDR		0x68
+#define CGU_REG_UHCCDR		0x6c
+#define CGU_REG_SSICDR		0x74
+#define CGU_REG_CIMCDR		0x7c
+#define CGU_REG_GPSCDR		0x80
+#define CGU_REG_PCMCDR		0x84
+#define CGU_REG_GPUCDR		0x88
+#define CGU_REG_MSC1CDR		0xA4
+#define CGU_REG_MSC2CDR		0xA8
+#define CGU_REG_BCHCDR		0xAC
+
+/* bits within the LCR register */
+#define LCR_LPM			BIT(0)		/* Low Power Mode */
+
+/* bits within the OPCR register */
+#define OPCR_SPENDH		BIT(5)		/* UHC PHY suspend */
+#define OPCR_SPENDN		BIT(7)		/* OTG PHY suspend */
+
+/* bits within the USBPCR1 register */
+#define USBPCR1_UHC_POWER	BIT(5)		/* UHC PHY power down */
+
+static struct ingenic_cgu *cgu;
+
+static int jz4770_uhc_phy_enable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr1	= cgu->base + CGU_REG_USBPCR1;
+
+	writel(readl(reg_opcr) & ~OPCR_SPENDH, reg_opcr);
+	writel(readl(reg_usbpcr1) | USBPCR1_UHC_POWER, reg_usbpcr1);
+	return 0;
+}
+
+static void jz4770_uhc_phy_disable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr1	= cgu->base + CGU_REG_USBPCR1;
+
+	writel(readl(reg_usbpcr1) & ~USBPCR1_UHC_POWER, reg_usbpcr1);
+	writel(readl(reg_opcr) | OPCR_SPENDH, reg_opcr);
+}
+
+static int jz4770_uhc_phy_is_enabled(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+	void __iomem *reg_usbpcr1	= cgu->base + CGU_REG_USBPCR1;
+
+	return !(readl(reg_opcr) & OPCR_SPENDH) &&
+		(readl(reg_usbpcr1) & USBPCR1_UHC_POWER);
+}
+
+static const struct clk_ops jz4770_uhc_phy_ops = {
+	.enable = jz4770_uhc_phy_enable,
+	.disable = jz4770_uhc_phy_disable,
+	.is_enabled = jz4770_uhc_phy_is_enabled,
+};
+
+static int jz4770_otg_phy_enable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+
+	writel(readl(reg_opcr) | OPCR_SPENDN, reg_opcr);
+
+	/* Wait for the clock to be stable */
+	udelay(50);
+	return 0;
+}
+
+static void jz4770_otg_phy_disable(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+
+	writel(readl(reg_opcr) & ~OPCR_SPENDN, reg_opcr);
+}
+
+static int jz4770_otg_phy_is_enabled(struct clk_hw *hw)
+{
+	void __iomem *reg_opcr		= cgu->base + CGU_REG_OPCR;
+
+	return !!(readl(reg_opcr) & OPCR_SPENDN);
+}
+
+static const struct clk_ops jz4770_otg_phy_ops = {
+	.enable = jz4770_otg_phy_enable,
+	.disable = jz4770_otg_phy_disable,
+	.is_enabled = jz4770_otg_phy_is_enabled,
+};
+
+static const s8 pll_od_encoding[8] = {
+	0x0, 0x1, -1, 0x2, -1, -1, -1, 0x3,
+};
+
+static const struct ingenic_cgu_clk_info jz4770_cgu_clocks[] = {
+
+	/* External clocks */
+
+	[JZ4770_CLK_EXT] = { "ext", CGU_CLK_EXT },
+	[JZ4770_CLK_OSC32K] = { "osc32k", CGU_CLK_EXT },
+
+	/* PLLs */
+
+	[JZ4770_CLK_PLL0] = {
+		"pll0", CGU_CLK_PLL,
+		.parents = { JZ4770_CLK_EXT },
+		.pll = {
+			.reg = CGU_REG_CPPCR0,
+			.m_shift = 24,
+			.m_bits = 7,
+			.m_offset = 1,
+			.n_shift = 18,
+			.n_bits = 5,
+			.n_offset = 1,
+			.od_shift = 16,
+			.od_bits = 2,
+			.od_max = 8,
+			.od_encoding = pll_od_encoding,
+			.bypass_bit = 9,
+			.enable_bit = 8,
+			.stable_bit = 10,
+		},
+	},
+
+	[JZ4770_CLK_PLL1] = {
+		/* TODO: PLL1 can depend on PLL0 */
+		"pll1", CGU_CLK_PLL,
+		.parents = { JZ4770_CLK_EXT },
+		.pll = {
+			.reg = CGU_REG_CPPCR1,
+			.m_shift = 24,
+			.m_bits = 7,
+			.m_offset = 1,
+			.n_shift = 18,
+			.n_bits = 5,
+			.n_offset = 1,
+			.od_shift = 16,
+			.od_bits = 2,
+			.od_max = 8,
+			.od_encoding = pll_od_encoding,
+			.enable_bit = 7,
+			.stable_bit = 6,
+			.no_bypass_bit = true,
+		},
+	},
+
+	/* Main clocks */
+
+	[JZ4770_CLK_CCLK] = {
+		"cclk", CGU_CLK_DIV,
+		.parents = { JZ4770_CLK_PLL0, },
+		.div = { CGU_REG_CPCCR, 0, 1, 4, 22, -1, -1 },
+	},
+	[JZ4770_CLK_H0CLK] = {
+		"h0clk", CGU_CLK_DIV,
+		.parents = { JZ4770_CLK_PLL0, },
+		.div = { CGU_REG_CPCCR, 4, 1, 4, 22, -1, -1 },
+	},
+	[JZ4770_CLK_H1CLK] = {
+		"h1clk", CGU_CLK_DIV | CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_PLL0, },
+		.div = { CGU_REG_CPCCR, 24, 1, 4, 22, -1, -1 },
+		.gate = { CGU_REG_LCR, 30 },
+	},
+	[JZ4770_CLK_H2CLK] = {
+		"h2clk", CGU_CLK_DIV,
+		.parents = { JZ4770_CLK_PLL0, },
+		.div = { CGU_REG_CPCCR, 16, 1, 4, 22, -1, -1 },
+	},
+	[JZ4770_CLK_C1CLK] = {
+		"c1clk", CGU_CLK_DIV,
+		.parents = { JZ4770_CLK_PLL0, },
+		.div = { CGU_REG_CPCCR, 12, 1, 4, 22, -1, -1 },
+	},
+	[JZ4770_CLK_PCLK] = {
+		"pclk", CGU_CLK_DIV,
+		.parents = { JZ4770_CLK_PLL0, },
+		.div = { CGU_REG_CPCCR, 8, 1, 4, 22, -1, -1 },
+	},
+
+	/* Those divided clocks can connect to PLL0 or PLL1 */
+
+	[JZ4770_CLK_MMC0_MUX] = {
+		"mmc0_mux", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_MSC0CDR, 30, 1 },
+		.div = { CGU_REG_MSC0CDR, 0, 1, 7, -1, -1, 31 },
+		.gate = { CGU_REG_MSC0CDR, 31 },
+	},
+	[JZ4770_CLK_MMC1_MUX] = {
+		"mmc1_mux", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_MSC1CDR, 30, 1 },
+		.div = { CGU_REG_MSC1CDR, 0, 1, 7, -1, -1, 31 },
+		.gate = { CGU_REG_MSC1CDR, 31 },
+	},
+	[JZ4770_CLK_MMC2_MUX] = {
+		"mmc2_mux", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_MSC2CDR, 30, 1 },
+		.div = { CGU_REG_MSC2CDR, 0, 1, 7, -1, -1, 31 },
+		.gate = { CGU_REG_MSC2CDR, 31 },
+	},
+	[JZ4770_CLK_CIM] = {
+		"cim", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_CIMCDR, 31, 1 },
+		.div = { CGU_REG_CIMCDR, 0, 1, 8, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 26 },
+	},
+	[JZ4770_CLK_UHC] = {
+		"uhc", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_UHCCDR, 29, 1 },
+		.div = { CGU_REG_UHCCDR, 0, 1, 4, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 24 },
+	},
+	[JZ4770_CLK_GPU] = {
+		"gpu", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, -1 },
+		.mux = { CGU_REG_GPUCDR, 31, 1 },
+		.div = { CGU_REG_GPUCDR, 0, 1, 3, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR1, 9 },
+	},
+	[JZ4770_CLK_BCH] = {
+		"bch", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_BCHCDR, 31, 1 },
+		.div = { CGU_REG_BCHCDR, 0, 1, 3, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 1 },
+	},
+	[JZ4770_CLK_LPCLK_MUX] = {
+		"lpclk", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_LPCDR, 29, 1 },
+		.div = { CGU_REG_LPCDR, 0, 1, 11, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 28 },
+	},
+	[JZ4770_CLK_GPS] = {
+		"gps", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_PLL0, JZ4770_CLK_PLL1, },
+		.mux = { CGU_REG_GPSCDR, 31, 1 },
+		.div = { CGU_REG_GPSCDR, 0, 1, 4, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 22 },
+	},
+
+	/* Those divided clocks can connect to EXT, PLL0 or PLL1 */
+
+	[JZ4770_CLK_SSI_MUX] = {
+		"ssi_mux", CGU_CLK_DIV | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_EXT, -1,
+			JZ4770_CLK_PLL0, JZ4770_CLK_PLL1 },
+		.mux = { CGU_REG_SSICDR, 30, 2 },
+		.div = { CGU_REG_SSICDR, 0, 1, 6, -1, -1, -1 },
+	},
+	[JZ4770_CLK_PCM_MUX] = {
+		"pcm_mux", CGU_CLK_DIV | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_EXT, -1,
+			JZ4770_CLK_PLL0, JZ4770_CLK_PLL1 },
+		.mux = { CGU_REG_PCMCDR, 30, 2 },
+		.div = { CGU_REG_PCMCDR, 0, 1, 9, -1, -1, -1 },
+	},
+	[JZ4770_CLK_I2S] = {
+		"i2s", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_EXT, -1,
+			JZ4770_CLK_PLL0, JZ4770_CLK_PLL1 },
+		.mux = { CGU_REG_I2SCDR, 30, 2 },
+		.div = { CGU_REG_I2SCDR, 0, 1, 9, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR1, 13 },
+	},
+	[JZ4770_CLK_OTG] = {
+		"usb", CGU_CLK_DIV | CGU_CLK_GATE | CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_EXT, -1,
+			JZ4770_CLK_PLL0, JZ4770_CLK_PLL1 },
+		.mux = { CGU_REG_USBCDR, 30, 2 },
+		.div = { CGU_REG_USBCDR, 0, 1, 8, -1, -1, -1 },
+		.gate = { CGU_REG_CLKGR0, 2 },
+	},
+
+	/* Gate-only clocks */
+
+	[JZ4770_CLK_SSI0] = {
+		"ssi0", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_SSI_MUX, },
+		.gate = { CGU_REG_CLKGR0, 4 },
+	},
+	[JZ4770_CLK_SSI1] = {
+		"ssi1", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_SSI_MUX, },
+		.gate = { CGU_REG_CLKGR0, 19 },
+	},
+	[JZ4770_CLK_SSI2] = {
+		"ssi2", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_SSI_MUX, },
+		.gate = { CGU_REG_CLKGR0, 20 },
+	},
+	[JZ4770_CLK_PCM0] = {
+		"pcm0", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_PCM_MUX, },
+		.gate = { CGU_REG_CLKGR1, 8 },
+	},
+	[JZ4770_CLK_PCM1] = {
+		"pcm1", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_PCM_MUX, },
+		.gate = { CGU_REG_CLKGR1, 10 },
+	},
+	[JZ4770_CLK_DMA] = {
+		"dma", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_H2CLK, },
+		.gate = { CGU_REG_CLKGR0, 21 },
+	},
+	[JZ4770_CLK_I2C0] = {
+		"i2c0", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 5 },
+	},
+	[JZ4770_CLK_I2C1] = {
+		"i2c1", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 6 },
+	},
+	[JZ4770_CLK_I2C2] = {
+		"i2c2", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR1, 15 },
+	},
+	[JZ4770_CLK_UART0] = {
+		"uart0", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 15 },
+	},
+	[JZ4770_CLK_UART1] = {
+		"uart1", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 16 },
+	},
+	[JZ4770_CLK_UART2] = {
+		"uart2", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 17 },
+	},
+	[JZ4770_CLK_UART3] = {
+		"uart3", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 18 },
+	},
+	[JZ4770_CLK_IPU] = {
+		"ipu", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_H0CLK, },
+		.gate = { CGU_REG_CLKGR0, 29 },
+	},
+	[JZ4770_CLK_ADC] = {
+		"adc", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 14 },
+	},
+	[JZ4770_CLK_AIC] = {
+		"aic", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_EXT, },
+		.gate = { CGU_REG_CLKGR0, 8 },
+	},
+	[JZ4770_CLK_AUX] = {
+		"aux", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_C1CLK, },
+		.gate = { CGU_REG_CLKGR1, 14 },
+	},
+	[JZ4770_CLK_VPU] = {
+		"vpu", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_H1CLK, },
+		.gate = { CGU_REG_CLKGR1, 7 },
+	},
+	[JZ4770_CLK_MMC0] = {
+		"mmc0", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_MMC0_MUX, },
+		.gate = { CGU_REG_CLKGR0, 3 },
+	},
+	[JZ4770_CLK_MMC1] = {
+		"mmc1", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_MMC1_MUX, },
+		.gate = { CGU_REG_CLKGR0, 11 },
+	},
+	[JZ4770_CLK_MMC2] = {
+		"mmc2", CGU_CLK_GATE,
+		.parents = { JZ4770_CLK_MMC2_MUX, },
+		.gate = { CGU_REG_CLKGR0, 12 },
+	},
+
+	/* Custom clocks */
+
+	[JZ4770_CLK_UHC_PHY] = {
+		"uhc_phy", CGU_CLK_CUSTOM,
+		.parents = { JZ4770_CLK_UHC, -1, -1, -1 },
+		.custom = { &jz4770_uhc_phy_ops },
+	},
+	[JZ4770_CLK_OTG_PHY] = {
+		"usb_phy", CGU_CLK_CUSTOM,
+		.parents = { JZ4770_CLK_OTG, -1, -1, -1 },
+		.custom = { &jz4770_otg_phy_ops },
+	},
+
+	[JZ4770_CLK_EXT512] = {
+		"ext/512", CGU_CLK_FIXDIV,
+		.parents = { JZ4770_CLK_EXT },
+		.fixdiv = { 512 },
+	},
+
+	[JZ4770_CLK_RTC] = {
+		"rtc", CGU_CLK_MUX,
+		.parents = { JZ4770_CLK_EXT512, JZ4770_CLK_OSC32K, },
+		.mux = { CGU_REG_OPCR, 2, 1},
+	},
+};
+
+#if IS_ENABLED(CONFIG_PM_SLEEP)
+static int jz4770_cgu_pm_suspend(void)
+{
+	u32 val;
+
+	val = readl(cgu->base + CGU_REG_LCR);
+	writel(val | LCR_LPM, cgu->base + CGU_REG_LCR);
+	return 0;
+}
+
+static void jz4770_cgu_pm_resume(void)
+{
+	u32 val;
+
+	val = readl(cgu->base + CGU_REG_LCR);
+	writel(val & ~LCR_LPM, cgu->base + CGU_REG_LCR);
+}
+
+static struct syscore_ops jz4770_cgu_pm_ops = {
+	.suspend = jz4770_cgu_pm_suspend,
+	.resume = jz4770_cgu_pm_resume,
+};
+#endif /* CONFIG_PM_SLEEP */
+
+static void __init jz4770_cgu_init(struct device_node *np)
+{
+	int retval;
+
+	cgu = ingenic_cgu_new(jz4770_cgu_clocks,
+			      ARRAY_SIZE(jz4770_cgu_clocks), np);
+	if (!cgu)
+		pr_err("%s: failed to initialise CGU\n", __func__);
+
+	retval = ingenic_cgu_register_clocks(cgu);
+	if (retval)
+		pr_err("%s: failed to register CGU Clocks\n", __func__);
+
+#if IS_ENABLED(CONFIG_PM_SLEEP)
+	register_syscore_ops(&jz4770_cgu_pm_ops);
+#endif
+}
+
+/* We only probe via devicetree, no need for a platform driver */
+CLK_OF_DECLARE(jz4770_cgu, "ingenic,jz4770-cgu", jz4770_cgu_init);
diff --git a/drivers/clk/ingenic/jz4780-cgu.c b/drivers/clk/ingenic/jz4780-cgu.c
index ac3585ed8228..6427be117ff1 100644
--- a/drivers/clk/ingenic/jz4780-cgu.c
+++ b/drivers/clk/ingenic/jz4780-cgu.c
@@ -203,7 +203,7 @@ static int jz4780_otg_phy_set_rate(struct clk_hw *hw, unsigned long req_rate,
 	return 0;
 }
 
-static struct clk_ops jz4780_otg_phy_ops = {
+static const struct clk_ops jz4780_otg_phy_ops = {
 	.get_parent = jz4780_otg_phy_get_parent,
 	.set_parent = jz4780_otg_phy_set_parent,
 
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
index 59dc0aad553c..1f9ea0f21df1 100644
--- a/drivers/clk/mediatek/Kconfig
+++ b/drivers/clk/mediatek/Kconfig
@@ -1,136 +1,140 @@
 #
-# MediaTek SoC drivers
+# MediaTek Clock Drivers
 #
+menu "Clock driver for MediaTek SoC"
+	depends on ARCH_MEDIATEK || COMPILE_TEST
+
 config COMMON_CLK_MEDIATEK
 	bool
+	select RESET_CONTROLLER
 	---help---
-	  Mediatek SoCs' clock support.
+	  MediaTek SoCs' clock support.
 
 config COMMON_CLK_MT2701
-	bool "Clock driver for Mediatek MT2701"
+	bool "Clock driver for MediaTek MT2701"
 	depends on (ARCH_MEDIATEK && ARM) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM
 	---help---
-	  This driver supports Mediatek MT2701 basic clocks.
+	  This driver supports MediaTek MT2701 basic clocks.
 
 config COMMON_CLK_MT2701_MMSYS
-	bool "Clock driver for Mediatek MT2701 mmsys"
+	bool "Clock driver for MediaTek MT2701 mmsys"
 	depends on COMMON_CLK_MT2701
 	---help---
-	  This driver supports Mediatek MT2701 mmsys clocks.
+	  This driver supports MediaTek MT2701 mmsys clocks.
 
 config COMMON_CLK_MT2701_IMGSYS
-	bool "Clock driver for Mediatek MT2701 imgsys"
+	bool "Clock driver for MediaTek MT2701 imgsys"
 	depends on COMMON_CLK_MT2701
 	---help---
-	  This driver supports Mediatek MT2701 imgsys clocks.
+	  This driver supports MediaTek MT2701 imgsys clocks.
 
 config COMMON_CLK_MT2701_VDECSYS
-	bool "Clock driver for Mediatek MT2701 vdecsys"
+	bool "Clock driver for MediaTek MT2701 vdecsys"
 	depends on COMMON_CLK_MT2701
 	---help---
-	  This driver supports Mediatek MT2701 vdecsys clocks.
+	  This driver supports MediaTek MT2701 vdecsys clocks.
 
 config COMMON_CLK_MT2701_HIFSYS
-	bool "Clock driver for Mediatek MT2701 hifsys"
+	bool "Clock driver for MediaTek MT2701 hifsys"
 	depends on COMMON_CLK_MT2701
 	---help---
-	  This driver supports Mediatek MT2701 hifsys clocks.
+	  This driver supports MediaTek MT2701 hifsys clocks.
 
 config COMMON_CLK_MT2701_ETHSYS
-	bool "Clock driver for Mediatek MT2701 ethsys"
+	bool "Clock driver for MediaTek MT2701 ethsys"
 	depends on COMMON_CLK_MT2701
 	---help---
-	  This driver supports Mediatek MT2701 ethsys clocks.
+	  This driver supports MediaTek MT2701 ethsys clocks.
 
 config COMMON_CLK_MT2701_BDPSYS
-	bool "Clock driver for Mediatek MT2701 bdpsys"
+	bool "Clock driver for MediaTek MT2701 bdpsys"
 	depends on COMMON_CLK_MT2701
 	---help---
-	  This driver supports Mediatek MT2701 bdpsys clocks.
+	  This driver supports MediaTek MT2701 bdpsys clocks.
 
 config COMMON_CLK_MT2712
-	bool "Clock driver for Mediatek MT2712"
+	bool "Clock driver for MediaTek MT2712"
 	depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM64
 	---help---
-	  This driver supports Mediatek MT2712 basic clocks.
+	  This driver supports MediaTek MT2712 basic clocks.
 
 config COMMON_CLK_MT2712_BDPSYS
-	bool "Clock driver for Mediatek MT2712 bdpsys"
+	bool "Clock driver for MediaTek MT2712 bdpsys"
 	depends on COMMON_CLK_MT2712
 	---help---
-	  This driver supports Mediatek MT2712 bdpsys clocks.
+	  This driver supports MediaTek MT2712 bdpsys clocks.
 
 config COMMON_CLK_MT2712_IMGSYS
-	bool "Clock driver for Mediatek MT2712 imgsys"
+	bool "Clock driver for MediaTek MT2712 imgsys"
 	depends on COMMON_CLK_MT2712
 	---help---
-	  This driver supports Mediatek MT2712 imgsys clocks.
+	  This driver supports MediaTek MT2712 imgsys clocks.
 
 config COMMON_CLK_MT2712_JPGDECSYS
-	bool "Clock driver for Mediatek MT2712 jpgdecsys"
+	bool "Clock driver for MediaTek MT2712 jpgdecsys"
 	depends on COMMON_CLK_MT2712
 	---help---
-	  This driver supports Mediatek MT2712 jpgdecsys clocks.
+	  This driver supports MediaTek MT2712 jpgdecsys clocks.
 
 config COMMON_CLK_MT2712_MFGCFG
-	bool "Clock driver for Mediatek MT2712 mfgcfg"
+	bool "Clock driver for MediaTek MT2712 mfgcfg"
 	depends on COMMON_CLK_MT2712
 	---help---
-	  This driver supports Mediatek MT2712 mfgcfg clocks.
+	  This driver supports MediaTek MT2712 mfgcfg clocks.
 
 config COMMON_CLK_MT2712_MMSYS
-	bool "Clock driver for Mediatek MT2712 mmsys"
+	bool "Clock driver for MediaTek MT2712 mmsys"
 	depends on COMMON_CLK_MT2712
 	---help---
-	  This driver supports Mediatek MT2712 mmsys clocks.
+	  This driver supports MediaTek MT2712 mmsys clocks.
 
 config COMMON_CLK_MT2712_VDECSYS
-	bool "Clock driver for Mediatek MT2712 vdecsys"
+	bool "Clock driver for MediaTek MT2712 vdecsys"
 	depends on COMMON_CLK_MT2712
 	---help---
-	  This driver supports Mediatek MT2712 vdecsys clocks.
+	  This driver supports MediaTek MT2712 vdecsys clocks.
 
 config COMMON_CLK_MT2712_VENCSYS
-	bool "Clock driver for Mediatek MT2712 vencsys"
+	bool "Clock driver for MediaTek MT2712 vencsys"
 	depends on COMMON_CLK_MT2712
 	---help---
-	  This driver supports Mediatek MT2712 vencsys clocks.
+	  This driver supports MediaTek MT2712 vencsys clocks.
 
 config COMMON_CLK_MT6797
-       bool "Clock driver for Mediatek MT6797"
+       bool "Clock driver for MediaTek MT6797"
        depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
        select COMMON_CLK_MEDIATEK
        default ARCH_MEDIATEK && ARM64
        ---help---
-         This driver supports Mediatek MT6797 basic clocks.
+         This driver supports MediaTek MT6797 basic clocks.
 
 config COMMON_CLK_MT6797_MMSYS
-       bool "Clock driver for Mediatek MT6797 mmsys"
+       bool "Clock driver for MediaTek MT6797 mmsys"
        depends on COMMON_CLK_MT6797
        ---help---
-         This driver supports Mediatek MT6797 mmsys clocks.
+         This driver supports MediaTek MT6797 mmsys clocks.
 
 config COMMON_CLK_MT6797_IMGSYS
-       bool "Clock driver for Mediatek MT6797 imgsys"
+       bool "Clock driver for MediaTek MT6797 imgsys"
        depends on COMMON_CLK_MT6797
        ---help---
-         This driver supports Mediatek MT6797 imgsys clocks.
+         This driver supports MediaTek MT6797 imgsys clocks.
 
 config COMMON_CLK_MT6797_VDECSYS
-       bool "Clock driver for Mediatek MT6797 vdecsys"
+       bool "Clock driver for MediaTek MT6797 vdecsys"
        depends on COMMON_CLK_MT6797
        ---help---
-         This driver supports Mediatek MT6797 vdecsys clocks.
+         This driver supports MediaTek MT6797 vdecsys clocks.
 
 config COMMON_CLK_MT6797_VENCSYS
-       bool "Clock driver for Mediatek MT6797 vencsys"
+       bool "Clock driver for MediaTek MT6797 vencsys"
        depends on COMMON_CLK_MT6797
        ---help---
-         This driver supports Mediatek MT6797 vencsys clocks.
+         This driver supports MediaTek MT6797 vencsys clocks.
 
 config COMMON_CLK_MT7622
 	bool "Clock driver for MediaTek MT7622"
@@ -163,17 +167,18 @@ config COMMON_CLK_MT7622_AUDSYS
 	  to audio consumers such as I2S and TDM.
 
 config COMMON_CLK_MT8135
-	bool "Clock driver for Mediatek MT8135"
+	bool "Clock driver for MediaTek MT8135"
 	depends on (ARCH_MEDIATEK && ARM) || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK && ARM
 	---help---
-	  This driver supports Mediatek MT8135 clocks.
+	  This driver supports MediaTek MT8135 clocks.
 
 config COMMON_CLK_MT8173
-	bool "Clock driver for Mediatek MT8173"
+	bool "Clock driver for MediaTek MT8173"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select COMMON_CLK_MEDIATEK
 	default ARCH_MEDIATEK
 	---help---
-	  This driver supports Mediatek MT8173 clocks.
+	  This driver supports MediaTek MT8173 clocks.
+endmenu
diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile
index c421ffcd49ff..5160fdc4bbb8 100644
--- a/drivers/clk/mediatek/Makefile
+++ b/drivers/clk/mediatek/Makefile
@@ -1,6 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_COMMON_CLK_MEDIATEK) += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o clk-cpumux.o
-obj-$(CONFIG_RESET_CONTROLLER) += reset.o
+obj-$(CONFIG_COMMON_CLK_MEDIATEK) += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o clk-cpumux.o reset.o
 obj-$(CONFIG_COMMON_CLK_MT6797) += clk-mt6797.o
 obj-$(CONFIG_COMMON_CLK_MT6797_IMGSYS) += clk-mt6797-img.o
 obj-$(CONFIG_COMMON_CLK_MT6797_MMSYS) += clk-mt6797-mm.o
diff --git a/drivers/clk/mediatek/clk-mtk.h b/drivers/clk/mediatek/clk-mtk.h
index f10250dcece4..f83c2bbb677e 100644
--- a/drivers/clk/mediatek/clk-mtk.h
+++ b/drivers/clk/mediatek/clk-mtk.h
@@ -20,6 +20,7 @@
 #include <linux/clk-provider.h>
 
 struct clk;
+struct clk_onecell_data;
 
 #define MAX_MUX_GATE_BIT	31
 #define INVALID_MUX_GATE_BIT	(MAX_MUX_GATE_BIT + 1)
@@ -228,14 +229,7 @@ void mtk_clk_register_plls(struct device_node *node,
 struct clk *mtk_clk_register_ref2usb_tx(const char *name,
 			const char *parent_name, void __iomem *reg);
 
-#ifdef CONFIG_RESET_CONTROLLER
 void mtk_register_reset_controller(struct device_node *np,
 			unsigned int num_regs, int regofs);
-#else
-static inline void mtk_register_reset_controller(struct device_node *np,
-			unsigned int num_regs, int regofs)
-{
-}
-#endif
 
 #endif /* __DRV_CLK_MTK_H */
diff --git a/drivers/clk/meson/Kconfig b/drivers/clk/meson/Kconfig
index d2d0174a6eca..7694302c70a4 100644
--- a/drivers/clk/meson/Kconfig
+++ b/drivers/clk/meson/Kconfig
@@ -19,3 +19,11 @@ config COMMON_CLK_GXBB
 	help
 	  Support for the clock controller on AmLogic S905 devices, aka gxbb.
 	  Say Y if you want peripherals and CPU frequency scaling to work.
+
+config COMMON_CLK_AXG
+	bool
+	depends on COMMON_CLK_AMLOGIC
+	select RESET_CONTROLLER
+	help
+	  Support for the clock controller on AmLogic A113D devices, aka axg.
+	  Say Y if you want peripherals and CPU frequency scaling to work.
diff --git a/drivers/clk/meson/Makefile b/drivers/clk/meson/Makefile
index b139d41b25da..3c03ce583798 100644
--- a/drivers/clk/meson/Makefile
+++ b/drivers/clk/meson/Makefile
@@ -5,3 +5,4 @@
 obj-$(CONFIG_COMMON_CLK_AMLOGIC) += clk-pll.o clk-cpu.o clk-mpll.o clk-audio-divider.o
 obj-$(CONFIG_COMMON_CLK_MESON8B) += meson8b.o
 obj-$(CONFIG_COMMON_CLK_GXBB)	 += gxbb.o gxbb-aoclk.o gxbb-aoclk-regmap.o gxbb-aoclk-32k.o
+obj-$(CONFIG_COMMON_CLK_AXG)	 += axg.o
diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c
new file mode 100644
index 000000000000..1294f3ad7cd5
--- /dev/null
+++ b/drivers/clk/meson/axg.c
@@ -0,0 +1,938 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * AmLogic Meson-AXG Clock Controller Driver
+ *
+ * Copyright (c) 2016 Baylibre SAS.
+ * Author: Michael Turquette <mturquette@baylibre.com>
+ *
+ * Copyright (c) 2017 Amlogic, inc.
+ * Author: Qiufang Dai <qiufang.dai@amlogic.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/init.h>
+
+#include "clkc.h"
+#include "axg.h"
+
+static DEFINE_SPINLOCK(meson_clk_lock);
+
+static const struct pll_rate_table sys_pll_rate_table[] = {
+	PLL_RATE(24000000, 56, 1, 2),
+	PLL_RATE(48000000, 64, 1, 2),
+	PLL_RATE(72000000, 72, 1, 2),
+	PLL_RATE(96000000, 64, 1, 2),
+	PLL_RATE(120000000, 80, 1, 2),
+	PLL_RATE(144000000, 96, 1, 2),
+	PLL_RATE(168000000, 56, 1, 1),
+	PLL_RATE(192000000, 64, 1, 1),
+	PLL_RATE(216000000, 72, 1, 1),
+	PLL_RATE(240000000, 80, 1, 1),
+	PLL_RATE(264000000, 88, 1, 1),
+	PLL_RATE(288000000, 96, 1, 1),
+	PLL_RATE(312000000, 52, 1, 2),
+	PLL_RATE(336000000, 56, 1, 2),
+	PLL_RATE(360000000, 60, 1, 2),
+	PLL_RATE(384000000, 64, 1, 2),
+	PLL_RATE(408000000, 68, 1, 2),
+	PLL_RATE(432000000, 72, 1, 2),
+	PLL_RATE(456000000, 76, 1, 2),
+	PLL_RATE(480000000, 80, 1, 2),
+	PLL_RATE(504000000, 84, 1, 2),
+	PLL_RATE(528000000, 88, 1, 2),
+	PLL_RATE(552000000, 92, 1, 2),
+	PLL_RATE(576000000, 96, 1, 2),
+	PLL_RATE(600000000, 50, 1, 1),
+	PLL_RATE(624000000, 52, 1, 1),
+	PLL_RATE(648000000, 54, 1, 1),
+	PLL_RATE(672000000, 56, 1, 1),
+	PLL_RATE(696000000, 58, 1, 1),
+	PLL_RATE(720000000, 60, 1, 1),
+	PLL_RATE(744000000, 62, 1, 1),
+	PLL_RATE(768000000, 64, 1, 1),
+	PLL_RATE(792000000, 66, 1, 1),
+	PLL_RATE(816000000, 68, 1, 1),
+	PLL_RATE(840000000, 70, 1, 1),
+	PLL_RATE(864000000, 72, 1, 1),
+	PLL_RATE(888000000, 74, 1, 1),
+	PLL_RATE(912000000, 76, 1, 1),
+	PLL_RATE(936000000, 78, 1, 1),
+	PLL_RATE(960000000, 80, 1, 1),
+	PLL_RATE(984000000, 82, 1, 1),
+	PLL_RATE(1008000000, 84, 1, 1),
+	PLL_RATE(1032000000, 86, 1, 1),
+	PLL_RATE(1056000000, 88, 1, 1),
+	PLL_RATE(1080000000, 90, 1, 1),
+	PLL_RATE(1104000000, 92, 1, 1),
+	PLL_RATE(1128000000, 94, 1, 1),
+	PLL_RATE(1152000000, 96, 1, 1),
+	PLL_RATE(1176000000, 98, 1, 1),
+	PLL_RATE(1200000000, 50, 1, 0),
+	PLL_RATE(1224000000, 51, 1, 0),
+	PLL_RATE(1248000000, 52, 1, 0),
+	PLL_RATE(1272000000, 53, 1, 0),
+	PLL_RATE(1296000000, 54, 1, 0),
+	PLL_RATE(1320000000, 55, 1, 0),
+	PLL_RATE(1344000000, 56, 1, 0),
+	PLL_RATE(1368000000, 57, 1, 0),
+	PLL_RATE(1392000000, 58, 1, 0),
+	PLL_RATE(1416000000, 59, 1, 0),
+	PLL_RATE(1440000000, 60, 1, 0),
+	PLL_RATE(1464000000, 61, 1, 0),
+	PLL_RATE(1488000000, 62, 1, 0),
+	PLL_RATE(1512000000, 63, 1, 0),
+	PLL_RATE(1536000000, 64, 1, 0),
+	PLL_RATE(1560000000, 65, 1, 0),
+	PLL_RATE(1584000000, 66, 1, 0),
+	PLL_RATE(1608000000, 67, 1, 0),
+	PLL_RATE(1632000000, 68, 1, 0),
+	PLL_RATE(1656000000, 68, 1, 0),
+	PLL_RATE(1680000000, 68, 1, 0),
+	PLL_RATE(1704000000, 68, 1, 0),
+	PLL_RATE(1728000000, 69, 1, 0),
+	PLL_RATE(1752000000, 69, 1, 0),
+	PLL_RATE(1776000000, 69, 1, 0),
+	PLL_RATE(1800000000, 69, 1, 0),
+	PLL_RATE(1824000000, 70, 1, 0),
+	PLL_RATE(1848000000, 70, 1, 0),
+	PLL_RATE(1872000000, 70, 1, 0),
+	PLL_RATE(1896000000, 70, 1, 0),
+	PLL_RATE(1920000000, 71, 1, 0),
+	PLL_RATE(1944000000, 71, 1, 0),
+	PLL_RATE(1968000000, 71, 1, 0),
+	PLL_RATE(1992000000, 71, 1, 0),
+	PLL_RATE(2016000000, 72, 1, 0),
+	PLL_RATE(2040000000, 72, 1, 0),
+	PLL_RATE(2064000000, 72, 1, 0),
+	PLL_RATE(2088000000, 72, 1, 0),
+	PLL_RATE(2112000000, 73, 1, 0),
+	{ /* sentinel */ },
+};
+
+static struct meson_clk_pll axg_fixed_pll = {
+	.m = {
+		.reg_off = HHI_MPLL_CNTL,
+		.shift   = 0,
+		.width   = 9,
+	},
+	.n = {
+		.reg_off = HHI_MPLL_CNTL,
+		.shift   = 9,
+		.width   = 5,
+	},
+	.od = {
+		.reg_off = HHI_MPLL_CNTL,
+		.shift   = 16,
+		.width   = 2,
+	},
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "fixed_pll",
+		.ops = &meson_clk_pll_ro_ops,
+		.parent_names = (const char *[]){ "xtal" },
+		.num_parents = 1,
+	},
+};
+
+static struct meson_clk_pll axg_sys_pll = {
+	.m = {
+		.reg_off = HHI_SYS_PLL_CNTL,
+		.shift   = 0,
+		.width   = 9,
+	},
+	.n = {
+		.reg_off = HHI_SYS_PLL_CNTL,
+		.shift   = 9,
+		.width   = 5,
+	},
+	.od = {
+		.reg_off = HHI_SYS_PLL_CNTL,
+		.shift   = 10,
+		.width   = 2,
+	},
+	.rate_table = sys_pll_rate_table,
+	.rate_count = ARRAY_SIZE(sys_pll_rate_table),
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "sys_pll",
+		.ops = &meson_clk_pll_ro_ops,
+		.parent_names = (const char *[]){ "xtal" },
+		.num_parents = 1,
+		.flags = CLK_GET_RATE_NOCACHE,
+	},
+};
+
+static const struct pll_rate_table axg_gp0_pll_rate_table[] = {
+	PLL_RATE(240000000, 40, 1, 2),
+	PLL_RATE(246000000, 41, 1, 2),
+	PLL_RATE(252000000, 42, 1, 2),
+	PLL_RATE(258000000, 43, 1, 2),
+	PLL_RATE(264000000, 44, 1, 2),
+	PLL_RATE(270000000, 45, 1, 2),
+	PLL_RATE(276000000, 46, 1, 2),
+	PLL_RATE(282000000, 47, 1, 2),
+	PLL_RATE(288000000, 48, 1, 2),
+	PLL_RATE(294000000, 49, 1, 2),
+	PLL_RATE(300000000, 50, 1, 2),
+	PLL_RATE(306000000, 51, 1, 2),
+	PLL_RATE(312000000, 52, 1, 2),
+	PLL_RATE(318000000, 53, 1, 2),
+	PLL_RATE(324000000, 54, 1, 2),
+	PLL_RATE(330000000, 55, 1, 2),
+	PLL_RATE(336000000, 56, 1, 2),
+	PLL_RATE(342000000, 57, 1, 2),
+	PLL_RATE(348000000, 58, 1, 2),
+	PLL_RATE(354000000, 59, 1, 2),
+	PLL_RATE(360000000, 60, 1, 2),
+	PLL_RATE(366000000, 61, 1, 2),
+	PLL_RATE(372000000, 62, 1, 2),
+	PLL_RATE(378000000, 63, 1, 2),
+	PLL_RATE(384000000, 64, 1, 2),
+	PLL_RATE(390000000, 65, 1, 3),
+	PLL_RATE(396000000, 66, 1, 3),
+	PLL_RATE(402000000, 67, 1, 3),
+	PLL_RATE(408000000, 68, 1, 3),
+	PLL_RATE(480000000, 40, 1, 1),
+	PLL_RATE(492000000, 41, 1, 1),
+	PLL_RATE(504000000, 42, 1, 1),
+	PLL_RATE(516000000, 43, 1, 1),
+	PLL_RATE(528000000, 44, 1, 1),
+	PLL_RATE(540000000, 45, 1, 1),
+	PLL_RATE(552000000, 46, 1, 1),
+	PLL_RATE(564000000, 47, 1, 1),
+	PLL_RATE(576000000, 48, 1, 1),
+	PLL_RATE(588000000, 49, 1, 1),
+	PLL_RATE(600000000, 50, 1, 1),
+	PLL_RATE(612000000, 51, 1, 1),
+	PLL_RATE(624000000, 52, 1, 1),
+	PLL_RATE(636000000, 53, 1, 1),
+	PLL_RATE(648000000, 54, 1, 1),
+	PLL_RATE(660000000, 55, 1, 1),
+	PLL_RATE(672000000, 56, 1, 1),
+	PLL_RATE(684000000, 57, 1, 1),
+	PLL_RATE(696000000, 58, 1, 1),
+	PLL_RATE(708000000, 59, 1, 1),
+	PLL_RATE(720000000, 60, 1, 1),
+	PLL_RATE(732000000, 61, 1, 1),
+	PLL_RATE(744000000, 62, 1, 1),
+	PLL_RATE(756000000, 63, 1, 1),
+	PLL_RATE(768000000, 64, 1, 1),
+	PLL_RATE(780000000, 65, 1, 1),
+	PLL_RATE(792000000, 66, 1, 1),
+	PLL_RATE(804000000, 67, 1, 1),
+	PLL_RATE(816000000, 68, 1, 1),
+	PLL_RATE(960000000, 40, 1, 0),
+	PLL_RATE(984000000, 41, 1, 0),
+	PLL_RATE(1008000000, 42, 1, 0),
+	PLL_RATE(1032000000, 43, 1, 0),
+	PLL_RATE(1056000000, 44, 1, 0),
+	PLL_RATE(1080000000, 45, 1, 0),
+	PLL_RATE(1104000000, 46, 1, 0),
+	PLL_RATE(1128000000, 47, 1, 0),
+	PLL_RATE(1152000000, 48, 1, 0),
+	PLL_RATE(1176000000, 49, 1, 0),
+	PLL_RATE(1200000000, 50, 1, 0),
+	PLL_RATE(1224000000, 51, 1, 0),
+	PLL_RATE(1248000000, 52, 1, 0),
+	PLL_RATE(1272000000, 53, 1, 0),
+	PLL_RATE(1296000000, 54, 1, 0),
+	PLL_RATE(1320000000, 55, 1, 0),
+	PLL_RATE(1344000000, 56, 1, 0),
+	PLL_RATE(1368000000, 57, 1, 0),
+	PLL_RATE(1392000000, 58, 1, 0),
+	PLL_RATE(1416000000, 59, 1, 0),
+	PLL_RATE(1440000000, 60, 1, 0),
+	PLL_RATE(1464000000, 61, 1, 0),
+	PLL_RATE(1488000000, 62, 1, 0),
+	PLL_RATE(1512000000, 63, 1, 0),
+	PLL_RATE(1536000000, 64, 1, 0),
+	PLL_RATE(1560000000, 65, 1, 0),
+	PLL_RATE(1584000000, 66, 1, 0),
+	PLL_RATE(1608000000, 67, 1, 0),
+	PLL_RATE(1632000000, 68, 1, 0),
+	{ /* sentinel */ },
+};
+
+static struct pll_params_table axg_gp0_params_table[] = {
+	PLL_PARAM(HHI_GP0_PLL_CNTL, 0x40010250),
+	PLL_PARAM(HHI_GP0_PLL_CNTL1, 0xc084a000),
+	PLL_PARAM(HHI_GP0_PLL_CNTL2, 0xb75020be),
+	PLL_PARAM(HHI_GP0_PLL_CNTL3, 0x0a59a288),
+	PLL_PARAM(HHI_GP0_PLL_CNTL4, 0xc000004d),
+	PLL_PARAM(HHI_GP0_PLL_CNTL5, 0x00078000),
+};
+
+static struct meson_clk_pll axg_gp0_pll = {
+	.m = {
+		.reg_off = HHI_GP0_PLL_CNTL,
+		.shift   = 0,
+		.width   = 9,
+	},
+	.n = {
+		.reg_off = HHI_GP0_PLL_CNTL,
+		.shift   = 9,
+		.width   = 5,
+	},
+	.od = {
+		.reg_off = HHI_GP0_PLL_CNTL,
+		.shift   = 16,
+		.width   = 2,
+	},
+	.params = {
+		.params_table = axg_gp0_params_table,
+		.params_count =	ARRAY_SIZE(axg_gp0_params_table),
+		.no_init_reset = true,
+		.reset_lock_loop = true,
+	},
+	.rate_table = axg_gp0_pll_rate_table,
+	.rate_count = ARRAY_SIZE(axg_gp0_pll_rate_table),
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "gp0_pll",
+		.ops = &meson_clk_pll_ops,
+		.parent_names = (const char *[]){ "xtal" },
+		.num_parents = 1,
+	},
+};
+
+
+static struct clk_fixed_factor axg_fclk_div2 = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(struct clk_init_data){
+		.name = "fclk_div2",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_fixed_factor axg_fclk_div3 = {
+	.mult = 1,
+	.div = 3,
+	.hw.init = &(struct clk_init_data){
+		.name = "fclk_div3",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_fixed_factor axg_fclk_div4 = {
+	.mult = 1,
+	.div = 4,
+	.hw.init = &(struct clk_init_data){
+		.name = "fclk_div4",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_fixed_factor axg_fclk_div5 = {
+	.mult = 1,
+	.div = 5,
+	.hw.init = &(struct clk_init_data){
+		.name = "fclk_div5",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_fixed_factor axg_fclk_div7 = {
+	.mult = 1,
+	.div = 7,
+	.hw.init = &(struct clk_init_data){
+		.name = "fclk_div7",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct meson_clk_mpll axg_mpll0 = {
+	.sdm = {
+		.reg_off = HHI_MPLL_CNTL7,
+		.shift   = 0,
+		.width   = 14,
+	},
+	.sdm_en = {
+		.reg_off = HHI_MPLL_CNTL7,
+		.shift   = 15,
+		.width	 = 1,
+	},
+	.n2 = {
+		.reg_off = HHI_MPLL_CNTL7,
+		.shift   = 16,
+		.width   = 9,
+	},
+	.en = {
+		.reg_off = HHI_MPLL_CNTL7,
+		.shift   = 14,
+		.width	 = 1,
+	},
+	.ssen = {
+		.reg_off = HHI_MPLL_CNTL,
+		.shift   = 25,
+		.width	 = 1,
+	},
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "mpll0",
+		.ops = &meson_clk_mpll_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct meson_clk_mpll axg_mpll1 = {
+	.sdm = {
+		.reg_off = HHI_MPLL_CNTL8,
+		.shift   = 0,
+		.width   = 14,
+	},
+	.sdm_en = {
+		.reg_off = HHI_MPLL_CNTL8,
+		.shift   = 15,
+		.width	 = 1,
+	},
+	.n2 = {
+		.reg_off = HHI_MPLL_CNTL8,
+		.shift   = 16,
+		.width   = 9,
+	},
+	.en = {
+		.reg_off = HHI_MPLL_CNTL8,
+		.shift   = 14,
+		.width	 = 1,
+	},
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "mpll1",
+		.ops = &meson_clk_mpll_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct meson_clk_mpll axg_mpll2 = {
+	.sdm = {
+		.reg_off = HHI_MPLL_CNTL9,
+		.shift   = 0,
+		.width   = 14,
+	},
+	.sdm_en = {
+		.reg_off = HHI_MPLL_CNTL9,
+		.shift   = 15,
+		.width	 = 1,
+	},
+	.n2 = {
+		.reg_off = HHI_MPLL_CNTL9,
+		.shift   = 16,
+		.width   = 9,
+	},
+	.en = {
+		.reg_off = HHI_MPLL_CNTL9,
+		.shift   = 14,
+		.width	 = 1,
+	},
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "mpll2",
+		.ops = &meson_clk_mpll_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+static struct meson_clk_mpll axg_mpll3 = {
+	.sdm = {
+		.reg_off = HHI_MPLL3_CNTL0,
+		.shift   = 12,
+		.width   = 14,
+	},
+	.sdm_en = {
+		.reg_off = HHI_MPLL3_CNTL0,
+		.shift   = 11,
+		.width	 = 1,
+	},
+	.n2 = {
+		.reg_off = HHI_MPLL3_CNTL0,
+		.shift   = 2,
+		.width   = 9,
+	},
+	.en = {
+		.reg_off = HHI_MPLL3_CNTL0,
+		.shift   = 0,
+		.width	 = 1,
+	},
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "mpll3",
+		.ops = &meson_clk_mpll_ops,
+		.parent_names = (const char *[]){ "fixed_pll" },
+		.num_parents = 1,
+	},
+};
+
+/*
+ * FIXME The legacy composite clocks (e.g. clk81) are both PLL post-dividers
+ * and should be modeled with their respective PLLs via the forthcoming
+ * coordinated clock rates feature
+ */
+static u32 mux_table_clk81[]	= { 0, 2, 3, 4, 5, 6, 7 };
+static const char * const clk81_parent_names[] = {
+	"xtal", "fclk_div7", "mpll1", "mpll2", "fclk_div4",
+	"fclk_div3", "fclk_div5"
+};
+
+static struct clk_mux axg_mpeg_clk_sel = {
+	.reg = (void *)HHI_MPEG_CLK_CNTL,
+	.mask = 0x7,
+	.shift = 12,
+	.flags = CLK_MUX_READ_ONLY,
+	.table = mux_table_clk81,
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "mpeg_clk_sel",
+		.ops = &clk_mux_ro_ops,
+		.parent_names = clk81_parent_names,
+		.num_parents = ARRAY_SIZE(clk81_parent_names),
+	},
+};
+
+static struct clk_divider axg_mpeg_clk_div = {
+	.reg = (void *)HHI_MPEG_CLK_CNTL,
+	.shift = 0,
+	.width = 7,
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "mpeg_clk_div",
+		.ops = &clk_divider_ops,
+		.parent_names = (const char *[]){ "mpeg_clk_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_gate axg_clk81 = {
+	.reg = (void *)HHI_MPEG_CLK_CNTL,
+	.bit_idx = 7,
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "clk81",
+		.ops = &clk_gate_ops,
+		.parent_names = (const char *[]){ "mpeg_clk_div" },
+		.num_parents = 1,
+		.flags = (CLK_SET_RATE_PARENT | CLK_IS_CRITICAL),
+	},
+};
+
+static const char * const axg_sd_emmc_clk0_parent_names[] = {
+	"xtal", "fclk_div2", "fclk_div3", "fclk_div5", "fclk_div7",
+
+	/*
+	 * Following these parent clocks, we should also have had mpll2, mpll3
+	 * and gp0_pll but these clocks are too precious to be used here. All
+	 * the necessary rates for MMC and NAND operation can be acheived using
+	 * xtal or fclk_div clocks
+	 */
+};
+
+/* SDcard clock */
+static struct clk_mux axg_sd_emmc_b_clk0_sel = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.mask = 0x7,
+	.shift = 25,
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_b_clk0_sel",
+		.ops = &clk_mux_ops,
+		.parent_names = axg_sd_emmc_clk0_parent_names,
+		.num_parents = ARRAY_SIZE(axg_sd_emmc_clk0_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_divider axg_sd_emmc_b_clk0_div = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.shift = 16,
+	.width = 7,
+	.lock = &meson_clk_lock,
+	.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_b_clk0_div",
+		.ops = &clk_divider_ops,
+		.parent_names = (const char *[]){ "sd_emmc_b_clk0_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_gate axg_sd_emmc_b_clk0 = {
+	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
+	.bit_idx = 23,
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "sd_emmc_b_clk0",
+		.ops = &clk_gate_ops,
+		.parent_names = (const char *[]){ "sd_emmc_b_clk0_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+/* EMMC/NAND clock */
+static struct clk_mux axg_sd_emmc_c_clk0_sel = {
+	.reg = (void *)HHI_NAND_CLK_CNTL,
+	.mask = 0x7,
+	.shift = 9,
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_c_clk0_sel",
+		.ops = &clk_mux_ops,
+		.parent_names = axg_sd_emmc_clk0_parent_names,
+		.num_parents = ARRAY_SIZE(axg_sd_emmc_clk0_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_divider axg_sd_emmc_c_clk0_div = {
+	.reg = (void *)HHI_NAND_CLK_CNTL,
+	.shift = 0,
+	.width = 7,
+	.lock = &meson_clk_lock,
+	.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	.hw.init = &(struct clk_init_data) {
+		.name = "sd_emmc_c_clk0_div",
+		.ops = &clk_divider_ops,
+		.parent_names = (const char *[]){ "sd_emmc_c_clk0_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_gate axg_sd_emmc_c_clk0 = {
+	.reg = (void *)HHI_NAND_CLK_CNTL,
+	.bit_idx = 7,
+	.lock = &meson_clk_lock,
+	.hw.init = &(struct clk_init_data){
+		.name = "sd_emmc_c_clk0",
+		.ops = &clk_gate_ops,
+		.parent_names = (const char *[]){ "sd_emmc_c_clk0_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+/* Everything Else (EE) domain gates */
+static MESON_GATE(axg_ddr, HHI_GCLK_MPEG0, 0);
+static MESON_GATE(axg_audio_locker, HHI_GCLK_MPEG0, 2);
+static MESON_GATE(axg_mipi_dsi_host, HHI_GCLK_MPEG0, 3);
+static MESON_GATE(axg_isa, HHI_GCLK_MPEG0, 5);
+static MESON_GATE(axg_pl301, HHI_GCLK_MPEG0, 6);
+static MESON_GATE(axg_periphs, HHI_GCLK_MPEG0, 7);
+static MESON_GATE(axg_spicc_0, HHI_GCLK_MPEG0, 8);
+static MESON_GATE(axg_i2c, HHI_GCLK_MPEG0, 9);
+static MESON_GATE(axg_rng0, HHI_GCLK_MPEG0, 12);
+static MESON_GATE(axg_uart0, HHI_GCLK_MPEG0, 13);
+static MESON_GATE(axg_mipi_dsi_phy, HHI_GCLK_MPEG0, 14);
+static MESON_GATE(axg_spicc_1, HHI_GCLK_MPEG0, 15);
+static MESON_GATE(axg_pcie_a, HHI_GCLK_MPEG0, 16);
+static MESON_GATE(axg_pcie_b, HHI_GCLK_MPEG0, 17);
+static MESON_GATE(axg_hiu_reg, HHI_GCLK_MPEG0, 19);
+static MESON_GATE(axg_assist_misc, HHI_GCLK_MPEG0, 23);
+static MESON_GATE(axg_emmc_b, HHI_GCLK_MPEG0, 25);
+static MESON_GATE(axg_emmc_c, HHI_GCLK_MPEG0, 26);
+static MESON_GATE(axg_dma, HHI_GCLK_MPEG0, 27);
+static MESON_GATE(axg_spi, HHI_GCLK_MPEG0, 30);
+
+static MESON_GATE(axg_audio, HHI_GCLK_MPEG1, 0);
+static MESON_GATE(axg_eth_core, HHI_GCLK_MPEG1, 3);
+static MESON_GATE(axg_uart1, HHI_GCLK_MPEG1, 16);
+static MESON_GATE(axg_g2d, HHI_GCLK_MPEG1, 20);
+static MESON_GATE(axg_usb0, HHI_GCLK_MPEG1, 21);
+static MESON_GATE(axg_usb1, HHI_GCLK_MPEG1, 22);
+static MESON_GATE(axg_reset, HHI_GCLK_MPEG1, 23);
+static MESON_GATE(axg_usb_general, HHI_GCLK_MPEG1, 26);
+static MESON_GATE(axg_ahb_arb0, HHI_GCLK_MPEG1, 29);
+static MESON_GATE(axg_efuse, HHI_GCLK_MPEG1, 30);
+static MESON_GATE(axg_boot_rom, HHI_GCLK_MPEG1, 31);
+
+static MESON_GATE(axg_ahb_data_bus, HHI_GCLK_MPEG2, 1);
+static MESON_GATE(axg_ahb_ctrl_bus, HHI_GCLK_MPEG2, 2);
+static MESON_GATE(axg_usb1_to_ddr, HHI_GCLK_MPEG2, 8);
+static MESON_GATE(axg_usb0_to_ddr, HHI_GCLK_MPEG2, 9);
+static MESON_GATE(axg_mmc_pclk, HHI_GCLK_MPEG2, 11);
+static MESON_GATE(axg_vpu_intr, HHI_GCLK_MPEG2, 25);
+static MESON_GATE(axg_sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26);
+static MESON_GATE(axg_gic, HHI_GCLK_MPEG2, 30);
+
+/* Always On (AO) domain gates */
+
+static MESON_GATE(axg_ao_media_cpu, HHI_GCLK_AO, 0);
+static MESON_GATE(axg_ao_ahb_sram, HHI_GCLK_AO, 1);
+static MESON_GATE(axg_ao_ahb_bus, HHI_GCLK_AO, 2);
+static MESON_GATE(axg_ao_iface, HHI_GCLK_AO, 3);
+static MESON_GATE(axg_ao_i2c, HHI_GCLK_AO, 4);
+
+/* Array of all clocks provided by this provider */
+
+static struct clk_hw_onecell_data axg_hw_onecell_data = {
+	.hws = {
+		[CLKID_SYS_PLL]			= &axg_sys_pll.hw,
+		[CLKID_FIXED_PLL]		= &axg_fixed_pll.hw,
+		[CLKID_FCLK_DIV2]		= &axg_fclk_div2.hw,
+		[CLKID_FCLK_DIV3]		= &axg_fclk_div3.hw,
+		[CLKID_FCLK_DIV4]		= &axg_fclk_div4.hw,
+		[CLKID_FCLK_DIV5]		= &axg_fclk_div5.hw,
+		[CLKID_FCLK_DIV7]		= &axg_fclk_div7.hw,
+		[CLKID_GP0_PLL]			= &axg_gp0_pll.hw,
+		[CLKID_MPEG_SEL]		= &axg_mpeg_clk_sel.hw,
+		[CLKID_MPEG_DIV]		= &axg_mpeg_clk_div.hw,
+		[CLKID_CLK81]			= &axg_clk81.hw,
+		[CLKID_MPLL0]			= &axg_mpll0.hw,
+		[CLKID_MPLL1]			= &axg_mpll1.hw,
+		[CLKID_MPLL2]			= &axg_mpll2.hw,
+		[CLKID_MPLL3]			= &axg_mpll3.hw,
+		[CLKID_DDR]			= &axg_ddr.hw,
+		[CLKID_AUDIO_LOCKER]		= &axg_audio_locker.hw,
+		[CLKID_MIPI_DSI_HOST]		= &axg_mipi_dsi_host.hw,
+		[CLKID_ISA]			= &axg_isa.hw,
+		[CLKID_PL301]			= &axg_pl301.hw,
+		[CLKID_PERIPHS]			= &axg_periphs.hw,
+		[CLKID_SPICC0]			= &axg_spicc_0.hw,
+		[CLKID_I2C]			= &axg_i2c.hw,
+		[CLKID_RNG0]			= &axg_rng0.hw,
+		[CLKID_UART0]			= &axg_uart0.hw,
+		[CLKID_MIPI_DSI_PHY]		= &axg_mipi_dsi_phy.hw,
+		[CLKID_SPICC1]			= &axg_spicc_1.hw,
+		[CLKID_PCIE_A]			= &axg_pcie_a.hw,
+		[CLKID_PCIE_B]			= &axg_pcie_b.hw,
+		[CLKID_HIU_IFACE]		= &axg_hiu_reg.hw,
+		[CLKID_ASSIST_MISC]		= &axg_assist_misc.hw,
+		[CLKID_SD_EMMC_B]		= &axg_emmc_b.hw,
+		[CLKID_SD_EMMC_C]		= &axg_emmc_c.hw,
+		[CLKID_DMA]			= &axg_dma.hw,
+		[CLKID_SPI]			= &axg_spi.hw,
+		[CLKID_AUDIO]			= &axg_audio.hw,
+		[CLKID_ETH]			= &axg_eth_core.hw,
+		[CLKID_UART1]			= &axg_uart1.hw,
+		[CLKID_G2D]			= &axg_g2d.hw,
+		[CLKID_USB0]			= &axg_usb0.hw,
+		[CLKID_USB1]			= &axg_usb1.hw,
+		[CLKID_RESET]			= &axg_reset.hw,
+		[CLKID_USB]			= &axg_usb_general.hw,
+		[CLKID_AHB_ARB0]		= &axg_ahb_arb0.hw,
+		[CLKID_EFUSE]			= &axg_efuse.hw,
+		[CLKID_BOOT_ROM]		= &axg_boot_rom.hw,
+		[CLKID_AHB_DATA_BUS]		= &axg_ahb_data_bus.hw,
+		[CLKID_AHB_CTRL_BUS]		= &axg_ahb_ctrl_bus.hw,
+		[CLKID_USB1_DDR_BRIDGE]		= &axg_usb1_to_ddr.hw,
+		[CLKID_USB0_DDR_BRIDGE]		= &axg_usb0_to_ddr.hw,
+		[CLKID_MMC_PCLK]		= &axg_mmc_pclk.hw,
+		[CLKID_VPU_INTR]		= &axg_vpu_intr.hw,
+		[CLKID_SEC_AHB_AHB3_BRIDGE]	= &axg_sec_ahb_ahb3_bridge.hw,
+		[CLKID_GIC]			= &axg_gic.hw,
+		[CLKID_AO_MEDIA_CPU]		= &axg_ao_media_cpu.hw,
+		[CLKID_AO_AHB_SRAM]		= &axg_ao_ahb_sram.hw,
+		[CLKID_AO_AHB_BUS]		= &axg_ao_ahb_bus.hw,
+		[CLKID_AO_IFACE]		= &axg_ao_iface.hw,
+		[CLKID_AO_I2C]			= &axg_ao_i2c.hw,
+		[CLKID_SD_EMMC_B_CLK0_SEL]	= &axg_sd_emmc_b_clk0_sel.hw,
+		[CLKID_SD_EMMC_B_CLK0_DIV]	= &axg_sd_emmc_b_clk0_div.hw,
+		[CLKID_SD_EMMC_B_CLK0]		= &axg_sd_emmc_b_clk0.hw,
+		[CLKID_SD_EMMC_C_CLK0_SEL]	= &axg_sd_emmc_c_clk0_sel.hw,
+		[CLKID_SD_EMMC_C_CLK0_DIV]	= &axg_sd_emmc_c_clk0_div.hw,
+		[CLKID_SD_EMMC_C_CLK0]		= &axg_sd_emmc_c_clk0.hw,
+		[NR_CLKS]			= NULL,
+	},
+	.num = NR_CLKS,
+};
+
+/* Convenience tables to populate base addresses in .probe */
+
+static struct meson_clk_pll *const axg_clk_plls[] = {
+	&axg_fixed_pll,
+	&axg_sys_pll,
+	&axg_gp0_pll,
+};
+
+static struct meson_clk_mpll *const axg_clk_mplls[] = {
+	&axg_mpll0,
+	&axg_mpll1,
+	&axg_mpll2,
+	&axg_mpll3,
+};
+
+static struct clk_gate *const axg_clk_gates[] = {
+	&axg_clk81,
+	&axg_ddr,
+	&axg_audio_locker,
+	&axg_mipi_dsi_host,
+	&axg_isa,
+	&axg_pl301,
+	&axg_periphs,
+	&axg_spicc_0,
+	&axg_i2c,
+	&axg_rng0,
+	&axg_uart0,
+	&axg_mipi_dsi_phy,
+	&axg_spicc_1,
+	&axg_pcie_a,
+	&axg_pcie_b,
+	&axg_hiu_reg,
+	&axg_assist_misc,
+	&axg_emmc_b,
+	&axg_emmc_c,
+	&axg_dma,
+	&axg_spi,
+	&axg_audio,
+	&axg_eth_core,
+	&axg_uart1,
+	&axg_g2d,
+	&axg_usb0,
+	&axg_usb1,
+	&axg_reset,
+	&axg_usb_general,
+	&axg_ahb_arb0,
+	&axg_efuse,
+	&axg_boot_rom,
+	&axg_ahb_data_bus,
+	&axg_ahb_ctrl_bus,
+	&axg_usb1_to_ddr,
+	&axg_usb0_to_ddr,
+	&axg_mmc_pclk,
+	&axg_vpu_intr,
+	&axg_sec_ahb_ahb3_bridge,
+	&axg_gic,
+	&axg_ao_media_cpu,
+	&axg_ao_ahb_sram,
+	&axg_ao_ahb_bus,
+	&axg_ao_iface,
+	&axg_ao_i2c,
+	&axg_sd_emmc_b_clk0,
+	&axg_sd_emmc_c_clk0,
+};
+
+static struct clk_mux *const axg_clk_muxes[] = {
+	&axg_mpeg_clk_sel,
+	&axg_sd_emmc_b_clk0_sel,
+	&axg_sd_emmc_c_clk0_sel,
+};
+
+static struct clk_divider *const axg_clk_dividers[] = {
+	&axg_mpeg_clk_div,
+	&axg_sd_emmc_b_clk0_div,
+	&axg_sd_emmc_c_clk0_div,
+};
+
+struct clkc_data {
+	struct clk_gate *const *clk_gates;
+	unsigned int clk_gates_count;
+	struct meson_clk_mpll *const *clk_mplls;
+	unsigned int clk_mplls_count;
+	struct meson_clk_pll *const *clk_plls;
+	unsigned int clk_plls_count;
+	struct clk_mux *const *clk_muxes;
+	unsigned int clk_muxes_count;
+	struct clk_divider *const *clk_dividers;
+	unsigned int clk_dividers_count;
+	struct clk_hw_onecell_data *hw_onecell_data;
+};
+
+static const struct clkc_data axg_clkc_data = {
+	.clk_gates = axg_clk_gates,
+	.clk_gates_count = ARRAY_SIZE(axg_clk_gates),
+	.clk_mplls = axg_clk_mplls,
+	.clk_mplls_count = ARRAY_SIZE(axg_clk_mplls),
+	.clk_plls = axg_clk_plls,
+	.clk_plls_count = ARRAY_SIZE(axg_clk_plls),
+	.clk_muxes = axg_clk_muxes,
+	.clk_muxes_count = ARRAY_SIZE(axg_clk_muxes),
+	.clk_dividers = axg_clk_dividers,
+	.clk_dividers_count = ARRAY_SIZE(axg_clk_dividers),
+	.hw_onecell_data = &axg_hw_onecell_data,
+};
+
+static const struct of_device_id clkc_match_table[] = {
+	{ .compatible = "amlogic,axg-clkc", .data = &axg_clkc_data },
+	{}
+};
+
+static int axg_clkc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const struct clkc_data *clkc_data;
+	struct resource *res;
+	void __iomem *clk_base;
+	int ret, clkid, i;
+
+	clkc_data = of_device_get_match_data(&pdev->dev);
+	if (!clkc_data)
+		return -EINVAL;
+
+	/*  Generic clocks and PLLs */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -EINVAL;
+	clk_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+	if (!clk_base) {
+		dev_err(&pdev->dev, "Unable to map clk base\n");
+		return -ENXIO;
+	}
+
+	/* Populate base address for PLLs */
+	for (i = 0; i < clkc_data->clk_plls_count; i++)
+		clkc_data->clk_plls[i]->base = clk_base;
+
+	/* Populate base address for MPLLs */
+	for (i = 0; i < clkc_data->clk_mplls_count; i++)
+		clkc_data->clk_mplls[i]->base = clk_base;
+
+	/* Populate base address for gates */
+	for (i = 0; i < clkc_data->clk_gates_count; i++)
+		clkc_data->clk_gates[i]->reg = clk_base +
+			(u64)clkc_data->clk_gates[i]->reg;
+
+	/* Populate base address for muxes */
+	for (i = 0; i < clkc_data->clk_muxes_count; i++)
+		clkc_data->clk_muxes[i]->reg = clk_base +
+			(u64)clkc_data->clk_muxes[i]->reg;
+
+	/* Populate base address for dividers */
+	for (i = 0; i < clkc_data->clk_dividers_count; i++)
+		clkc_data->clk_dividers[i]->reg = clk_base +
+			(u64)clkc_data->clk_dividers[i]->reg;
+
+	for (clkid = 0; clkid < clkc_data->hw_onecell_data->num; clkid++) {
+		/* array might be sparse */
+		if (!clkc_data->hw_onecell_data->hws[clkid])
+			continue;
+
+		ret = devm_clk_hw_register(dev,
+					clkc_data->hw_onecell_data->hws[clkid]);
+		if (ret) {
+			dev_err(&pdev->dev, "Clock registration failed\n");
+			return ret;
+		}
+	}
+
+	return of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get,
+			clkc_data->hw_onecell_data);
+}
+
+static struct platform_driver axg_driver = {
+	.probe		= axg_clkc_probe,
+	.driver		= {
+		.name	= "axg-clkc",
+		.of_match_table = clkc_match_table,
+	},
+};
+
+builtin_platform_driver(axg_driver);
diff --git a/drivers/clk/meson/axg.h b/drivers/clk/meson/axg.h
new file mode 100644
index 000000000000..ce0bafdb6b28
--- /dev/null
+++ b/drivers/clk/meson/axg.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */
+/*
+ * Copyright (c) 2016 AmLogic, Inc.
+ * Author: Michael Turquette <mturquette@baylibre.com>
+ *
+ * Copyright (c) 2017 Amlogic, inc.
+ * Author: Qiufang Dai <qiufang.dai@amlogic.com>
+ *
+ */
+#ifndef __AXG_H
+#define __AXG_H
+
+/*
+ * Clock controller register offsets
+ *
+ * Register offsets from the data sheet must be multiplied by 4 before
+ * adding them to the base address to get the right value.
+ */
+#define HHI_GP0_PLL_CNTL		0x40
+#define HHI_GP0_PLL_CNTL2		0x44
+#define HHI_GP0_PLL_CNTL3		0x48
+#define HHI_GP0_PLL_CNTL4		0x4c
+#define HHI_GP0_PLL_CNTL5		0x50
+#define HHI_GP0_PLL_STS			0x54
+#define HHI_GP0_PLL_CNTL1		0x58
+#define HHI_HIFI_PLL_CNTL		0x80
+#define HHI_HIFI_PLL_CNTL2		0x84
+#define HHI_HIFI_PLL_CNTL3		0x88
+#define HHI_HIFI_PLL_CNTL4		0x8C
+#define HHI_HIFI_PLL_CNTL5		0x90
+#define HHI_HIFI_PLL_STS		0x94
+#define HHI_HIFI_PLL_CNTL1		0x98
+
+#define HHI_XTAL_DIVN_CNTL		0xbc
+#define HHI_GCLK2_MPEG0			0xc0
+#define HHI_GCLK2_MPEG1			0xc4
+#define HHI_GCLK2_MPEG2			0xc8
+#define HHI_GCLK2_OTHER			0xd0
+#define HHI_GCLK2_AO			0xd4
+#define HHI_PCIE_PLL_CNTL		0xd8
+#define HHI_PCIE_PLL_CNTL1		0xdC
+#define HHI_PCIE_PLL_CNTL2		0xe0
+#define HHI_PCIE_PLL_CNTL3		0xe4
+#define HHI_PCIE_PLL_CNTL4		0xe8
+#define HHI_PCIE_PLL_CNTL5		0xec
+#define HHI_PCIE_PLL_CNTL6		0xf0
+#define HHI_PCIE_PLL_STS		0xf4
+
+#define HHI_MEM_PD_REG0			0x100
+#define HHI_VPU_MEM_PD_REG0		0x104
+#define HHI_VIID_CLK_DIV		0x128
+#define HHI_VIID_CLK_CNTL		0x12c
+
+#define HHI_GCLK_MPEG0			0x140
+#define HHI_GCLK_MPEG1			0x144
+#define HHI_GCLK_MPEG2			0x148
+#define HHI_GCLK_OTHER			0x150
+#define HHI_GCLK_AO			0x154
+#define HHI_SYS_CPU_CLK_CNTL1		0x15c
+#define HHI_SYS_CPU_RESET_CNTL		0x160
+#define HHI_VID_CLK_DIV			0x164
+#define HHI_SPICC_HCLK_CNTL		0x168
+
+#define HHI_MPEG_CLK_CNTL		0x174
+#define HHI_VID_CLK_CNTL		0x17c
+#define HHI_TS_CLK_CNTL			0x190
+#define HHI_VID_CLK_CNTL2		0x194
+#define HHI_SYS_CPU_CLK_CNTL0		0x19c
+#define HHI_VID_PLL_CLK_DIV		0x1a0
+#define HHI_VPU_CLK_CNTL		0x1bC
+
+#define HHI_VAPBCLK_CNTL		0x1F4
+
+#define HHI_GEN_CLK_CNTL		0x228
+
+#define HHI_VDIN_MEAS_CLK_CNTL		0x250
+#define HHI_NAND_CLK_CNTL		0x25C
+#define HHI_SD_EMMC_CLK_CNTL		0x264
+
+#define HHI_MPLL_CNTL			0x280
+#define HHI_MPLL_CNTL2			0x284
+#define HHI_MPLL_CNTL3			0x288
+#define HHI_MPLL_CNTL4			0x28C
+#define HHI_MPLL_CNTL5			0x290
+#define HHI_MPLL_CNTL6			0x294
+#define HHI_MPLL_CNTL7			0x298
+#define HHI_MPLL_CNTL8			0x29C
+#define HHI_MPLL_CNTL9			0x2A0
+#define HHI_MPLL_CNTL10			0x2A4
+
+#define HHI_MPLL3_CNTL0			0x2E0
+#define HHI_MPLL3_CNTL1			0x2E4
+#define HHI_PLL_TOP_MISC		0x2E8
+
+#define HHI_SYS_PLL_CNTL1		0x2FC
+#define HHI_SYS_PLL_CNTL		0x300
+#define HHI_SYS_PLL_CNTL2		0x304
+#define HHI_SYS_PLL_CNTL3		0x308
+#define HHI_SYS_PLL_CNTL4		0x30c
+#define HHI_SYS_PLL_CNTL5		0x310
+#define HHI_SYS_PLL_STS			0x314
+#define HHI_DPLL_TOP_I			0x318
+#define HHI_DPLL_TOP2_I			0x31C
+
+/*
+ * CLKID index values
+ *
+ * These indices are entirely contrived and do not map onto the hardware.
+ * It has now been decided to expose everything by default in the DT header:
+ * include/dt-bindings/clock/axg-clkc.h. Only the clocks ids we don't want
+ * to expose, such as the internal muxes and dividers of composite clocks,
+ * will remain defined here.
+ */
+#define CLKID_MPEG_SEL				8
+#define CLKID_MPEG_DIV				9
+#define CLKID_SD_EMMC_B_CLK0_SEL		61
+#define CLKID_SD_EMMC_B_CLK0_DIV		62
+#define CLKID_SD_EMMC_C_CLK0_SEL		63
+#define CLKID_SD_EMMC_C_CLK0_DIV		64
+
+#define NR_CLKS					65
+
+/* include the CLKIDs that have been made part of the DT binding */
+#include <dt-bindings/clock/axg-clkc.h>
+
+#endif /* __AXG_H */
diff --git a/drivers/clk/meson/clk-mpll.c b/drivers/clk/meson/clk-mpll.c
index 44a5a535ca63..5144360e2c80 100644
--- a/drivers/clk/meson/clk-mpll.c
+++ b/drivers/clk/meson/clk-mpll.c
@@ -98,7 +98,7 @@ static void params_from_rate(unsigned long requested_rate,
 		*sdm = SDM_DEN - 1;
 	} else {
 		*n2 = div;
-		*sdm = DIV_ROUND_UP(rem * SDM_DEN, requested_rate);
+		*sdm = DIV_ROUND_UP_ULL((u64)rem * SDM_DEN, requested_rate);
 	}
 }
 
diff --git a/drivers/clk/meson/clkc.h b/drivers/clk/meson/clkc.h
index 1629da9b4141..c2ff0520ce53 100644
--- a/drivers/clk/meson/clkc.h
+++ b/drivers/clk/meson/clkc.h
@@ -134,7 +134,7 @@ struct meson_clk_audio_divider {
 struct clk_gate _name = { 						\
 	.reg = (void __iomem *) _reg, 					\
 	.bit_idx = (_bit), 						\
-	.lock = &clk_lock,						\
+	.lock = &meson_clk_lock,					\
 	.hw.init = &(struct clk_init_data) { 				\
 		.name = #_name,					\
 		.ops = &clk_gate_ops,					\
diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c
index ae385310e980..af24455af5b4 100644
--- a/drivers/clk/meson/gxbb.c
+++ b/drivers/clk/meson/gxbb.c
@@ -27,7 +27,7 @@
 #include "clkc.h"
 #include "gxbb.h"
 
-static DEFINE_SPINLOCK(clk_lock);
+static DEFINE_SPINLOCK(meson_clk_lock);
 
 static const struct pll_rate_table sys_pll_rate_table[] = {
 	PLL_RATE(24000000, 56, 1, 2),
@@ -294,7 +294,7 @@ static struct meson_clk_pll gxbb_fixed_pll = {
 		.shift   = 16,
 		.width   = 2,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "fixed_pll",
 		.ops = &meson_clk_pll_ro_ops,
@@ -330,7 +330,7 @@ static struct meson_clk_pll gxbb_hdmi_pll = {
 		.shift   = 22,
 		.width   = 2,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "hdmi_pll",
 		.ops = &meson_clk_pll_ro_ops,
@@ -358,7 +358,7 @@ static struct meson_clk_pll gxbb_sys_pll = {
 	},
 	.rate_table = sys_pll_rate_table,
 	.rate_count = ARRAY_SIZE(sys_pll_rate_table),
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sys_pll",
 		.ops = &meson_clk_pll_ro_ops,
@@ -399,7 +399,7 @@ static struct meson_clk_pll gxbb_gp0_pll = {
 	},
 	.rate_table = gxbb_gp0_pll_rate_table,
 	.rate_count = ARRAY_SIZE(gxbb_gp0_pll_rate_table),
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "gp0_pll",
 		.ops = &meson_clk_pll_ops,
@@ -442,7 +442,7 @@ static struct meson_clk_pll gxl_gp0_pll = {
 	},
 	.rate_table = gxl_gp0_pll_rate_table,
 	.rate_count = ARRAY_SIZE(gxl_gp0_pll_rate_table),
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "gp0_pll",
 		.ops = &meson_clk_pll_ops,
@@ -533,7 +533,7 @@ static struct meson_clk_mpll gxbb_mpll0 = {
 		.shift   = 25,
 		.width	 = 1,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpll0",
 		.ops = &meson_clk_mpll_ops,
@@ -563,7 +563,7 @@ static struct meson_clk_mpll gxbb_mpll1 = {
 		.shift   = 14,
 		.width	 = 1,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpll1",
 		.ops = &meson_clk_mpll_ops,
@@ -593,7 +593,7 @@ static struct meson_clk_mpll gxbb_mpll2 = {
 		.shift   = 14,
 		.width	 = 1,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpll2",
 		.ops = &meson_clk_mpll_ops,
@@ -620,7 +620,7 @@ static struct clk_mux gxbb_mpeg_clk_sel = {
 	.shift = 12,
 	.flags = CLK_MUX_READ_ONLY,
 	.table = mux_table_clk81,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpeg_clk_sel",
 		.ops = &clk_mux_ro_ops,
@@ -639,7 +639,7 @@ static struct clk_divider gxbb_mpeg_clk_div = {
 	.reg = (void *)HHI_MPEG_CLK_CNTL,
 	.shift = 0,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpeg_clk_div",
 		.ops = &clk_divider_ops,
@@ -653,7 +653,7 @@ static struct clk_divider gxbb_mpeg_clk_div = {
 static struct clk_gate gxbb_clk81 = {
 	.reg = (void *)HHI_MPEG_CLK_CNTL,
 	.bit_idx = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "clk81",
 		.ops = &clk_gate_ops,
@@ -667,7 +667,7 @@ static struct clk_mux gxbb_sar_adc_clk_sel = {
 	.reg = (void *)HHI_SAR_CLK_CNTL,
 	.mask = 0x3,
 	.shift = 9,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sar_adc_clk_sel",
 		.ops = &clk_mux_ops,
@@ -681,7 +681,7 @@ static struct clk_divider gxbb_sar_adc_clk_div = {
 	.reg = (void *)HHI_SAR_CLK_CNTL,
 	.shift = 0,
 	.width = 8,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sar_adc_clk_div",
 		.ops = &clk_divider_ops,
@@ -693,7 +693,7 @@ static struct clk_divider gxbb_sar_adc_clk_div = {
 static struct clk_gate gxbb_sar_adc_clk = {
 	.reg = (void *)HHI_SAR_CLK_CNTL,
 	.bit_idx = 8,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sar_adc_clk",
 		.ops = &clk_gate_ops,
@@ -719,7 +719,7 @@ static struct clk_mux gxbb_mali_0_sel = {
 	.mask = 0x7,
 	.shift = 9,
 	.table = mux_table_mali_0_1,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_0_sel",
 		.ops = &clk_mux_ops,
@@ -738,7 +738,7 @@ static struct clk_divider gxbb_mali_0_div = {
 	.reg = (void *)HHI_MALI_CLK_CNTL,
 	.shift = 0,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_0_div",
 		.ops = &clk_divider_ops,
@@ -751,7 +751,7 @@ static struct clk_divider gxbb_mali_0_div = {
 static struct clk_gate gxbb_mali_0 = {
 	.reg = (void *)HHI_MALI_CLK_CNTL,
 	.bit_idx = 8,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_0",
 		.ops = &clk_gate_ops,
@@ -766,7 +766,7 @@ static struct clk_mux gxbb_mali_1_sel = {
 	.mask = 0x7,
 	.shift = 25,
 	.table = mux_table_mali_0_1,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_1_sel",
 		.ops = &clk_mux_ops,
@@ -785,7 +785,7 @@ static struct clk_divider gxbb_mali_1_div = {
 	.reg = (void *)HHI_MALI_CLK_CNTL,
 	.shift = 16,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_1_div",
 		.ops = &clk_divider_ops,
@@ -798,7 +798,7 @@ static struct clk_divider gxbb_mali_1_div = {
 static struct clk_gate gxbb_mali_1 = {
 	.reg = (void *)HHI_MALI_CLK_CNTL,
 	.bit_idx = 24,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mali_1",
 		.ops = &clk_gate_ops,
@@ -818,7 +818,7 @@ static struct clk_mux gxbb_mali = {
 	.mask = 1,
 	.shift = 31,
 	.table = mux_table_mali,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mali",
 		.ops = &clk_mux_ops,
@@ -834,7 +834,7 @@ static struct clk_mux gxbb_cts_amclk_sel = {
 	.shift = 9,
 	/* Default parent unknown (register reset value: 0) */
 	.table = (u32[]){ 1, 2, 3 },
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 		.hw.init = &(struct clk_init_data){
 		.name = "cts_amclk_sel",
 		.ops = &clk_mux_ops,
@@ -851,7 +851,7 @@ static struct meson_clk_audio_divider gxbb_cts_amclk_div = {
 		.width   = 8,
 	},
 	.flags = CLK_DIVIDER_ROUND_CLOSEST,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "cts_amclk_div",
 		.ops = &meson_clk_audio_divider_ops,
@@ -864,7 +864,7 @@ static struct meson_clk_audio_divider gxbb_cts_amclk_div = {
 static struct clk_gate gxbb_cts_amclk = {
 	.reg = (void *) HHI_AUD_CLK_CNTL,
 	.bit_idx = 8,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "cts_amclk",
 		.ops = &clk_gate_ops,
@@ -880,7 +880,7 @@ static struct clk_mux gxbb_cts_mclk_i958_sel = {
 	.shift = 25,
 	/* Default parent unknown (register reset value: 0) */
 	.table = (u32[]){ 1, 2, 3 },
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "cts_mclk_i958_sel",
 		.ops = &clk_mux_ops,
@@ -894,7 +894,7 @@ static struct clk_divider gxbb_cts_mclk_i958_div = {
 	.reg = (void *)HHI_AUD_CLK_CNTL2,
 	.shift = 16,
 	.width = 8,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.flags = CLK_DIVIDER_ROUND_CLOSEST,
 	.hw.init = &(struct clk_init_data) {
 		.name = "cts_mclk_i958_div",
@@ -908,7 +908,7 @@ static struct clk_divider gxbb_cts_mclk_i958_div = {
 static struct clk_gate gxbb_cts_mclk_i958 = {
 	.reg = (void *)HHI_AUD_CLK_CNTL2,
 	.bit_idx = 24,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "cts_mclk_i958",
 		.ops = &clk_gate_ops,
@@ -922,7 +922,7 @@ static struct clk_mux gxbb_cts_i958 = {
 	.reg = (void *)HHI_AUD_CLK_CNTL2,
 	.mask = 0x1,
 	.shift = 27,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 		.hw.init = &(struct clk_init_data){
 		.name = "cts_i958",
 		.ops = &clk_mux_ops,
@@ -940,7 +940,7 @@ static struct clk_divider gxbb_32k_clk_div = {
 	.reg = (void *)HHI_32K_CLK_CNTL,
 	.shift = 0,
 	.width = 14,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "32k_clk_div",
 		.ops = &clk_divider_ops,
@@ -953,7 +953,7 @@ static struct clk_divider gxbb_32k_clk_div = {
 static struct clk_gate gxbb_32k_clk = {
 	.reg = (void *)HHI_32K_CLK_CNTL,
 	.bit_idx = 15,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "32k_clk",
 		.ops = &clk_gate_ops,
@@ -971,7 +971,7 @@ static struct clk_mux gxbb_32k_clk_sel = {
 	.reg = (void *)HHI_32K_CLK_CNTL,
 	.mask = 0x3,
 	.shift = 16,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 		.hw.init = &(struct clk_init_data){
 		.name = "32k_clk_sel",
 		.ops = &clk_mux_ops,
@@ -997,7 +997,7 @@ static struct clk_mux gxbb_sd_emmc_a_clk0_sel = {
 	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
 	.mask = 0x7,
 	.shift = 9,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "sd_emmc_a_clk0_sel",
 		.ops = &clk_mux_ops,
@@ -1011,7 +1011,7 @@ static struct clk_divider gxbb_sd_emmc_a_clk0_div = {
 	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
 	.shift = 0,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.flags = CLK_DIVIDER_ROUND_CLOSEST,
 	.hw.init = &(struct clk_init_data) {
 		.name = "sd_emmc_a_clk0_div",
@@ -1025,23 +1025,13 @@ static struct clk_divider gxbb_sd_emmc_a_clk0_div = {
 static struct clk_gate gxbb_sd_emmc_a_clk0 = {
 	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
 	.bit_idx = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sd_emmc_a_clk0",
 		.ops = &clk_gate_ops,
 		.parent_names = (const char *[]){ "sd_emmc_a_clk0_div" },
 		.num_parents = 1,
-
-		/*
-		 * FIXME:
-		 * We need CLK_IGNORE_UNUSED because mmc DT node point to xtal
-		 * instead of this clock. CCF would gate this on boot, killing
-		 * the mmc controller. Please remove this flag once DT properly
-		 * point to this clock instead of xtal
-		 *
-		 * Same goes for emmc B and C clocks
-		 */
-		.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -1050,7 +1040,7 @@ static struct clk_mux gxbb_sd_emmc_b_clk0_sel = {
 	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
 	.mask = 0x7,
 	.shift = 25,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "sd_emmc_b_clk0_sel",
 		.ops = &clk_mux_ops,
@@ -1064,7 +1054,7 @@ static struct clk_divider gxbb_sd_emmc_b_clk0_div = {
 	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
 	.shift = 16,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.flags = CLK_DIVIDER_ROUND_CLOSEST,
 	.hw.init = &(struct clk_init_data) {
 		.name = "sd_emmc_b_clk0_div",
@@ -1078,13 +1068,13 @@ static struct clk_divider gxbb_sd_emmc_b_clk0_div = {
 static struct clk_gate gxbb_sd_emmc_b_clk0 = {
 	.reg = (void *)HHI_SD_EMMC_CLK_CNTL,
 	.bit_idx = 23,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sd_emmc_b_clk0",
 		.ops = &clk_gate_ops,
 		.parent_names = (const char *[]){ "sd_emmc_b_clk0_div" },
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -1093,7 +1083,7 @@ static struct clk_mux gxbb_sd_emmc_c_clk0_sel = {
 	.reg = (void *)HHI_NAND_CLK_CNTL,
 	.mask = 0x7,
 	.shift = 9,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "sd_emmc_c_clk0_sel",
 		.ops = &clk_mux_ops,
@@ -1107,7 +1097,7 @@ static struct clk_divider gxbb_sd_emmc_c_clk0_div = {
 	.reg = (void *)HHI_NAND_CLK_CNTL,
 	.shift = 0,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.flags = CLK_DIVIDER_ROUND_CLOSEST,
 	.hw.init = &(struct clk_init_data) {
 		.name = "sd_emmc_c_clk0_div",
@@ -1121,13 +1111,13 @@ static struct clk_divider gxbb_sd_emmc_c_clk0_div = {
 static struct clk_gate gxbb_sd_emmc_c_clk0 = {
 	.reg = (void *)HHI_NAND_CLK_CNTL,
 	.bit_idx = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sd_emmc_c_clk0",
 		.ops = &clk_gate_ops,
 		.parent_names = (const char *[]){ "sd_emmc_c_clk0_div" },
 		.num_parents = 1,
-		.flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
@@ -1142,7 +1132,7 @@ static struct clk_mux gxbb_vpu_0_sel = {
 	.reg = (void *)HHI_VPU_CLK_CNTL,
 	.mask = 0x3,
 	.shift = 9,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.table = mux_table_vpu,
 	.hw.init = &(struct clk_init_data){
 		.name = "vpu_0_sel",
@@ -1161,7 +1151,7 @@ static struct clk_divider gxbb_vpu_0_div = {
 	.reg = (void *)HHI_VPU_CLK_CNTL,
 	.shift = 0,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "vpu_0_div",
 		.ops = &clk_divider_ops,
@@ -1174,7 +1164,7 @@ static struct clk_divider gxbb_vpu_0_div = {
 static struct clk_gate gxbb_vpu_0 = {
 	.reg = (void *)HHI_VPU_CLK_CNTL,
 	.bit_idx = 8,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "vpu_0",
 		.ops = &clk_gate_ops,
@@ -1188,7 +1178,7 @@ static struct clk_mux gxbb_vpu_1_sel = {
 	.reg = (void *)HHI_VPU_CLK_CNTL,
 	.mask = 0x3,
 	.shift = 25,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.table = mux_table_vpu,
 	.hw.init = &(struct clk_init_data){
 		.name = "vpu_1_sel",
@@ -1207,7 +1197,7 @@ static struct clk_divider gxbb_vpu_1_div = {
 	.reg = (void *)HHI_VPU_CLK_CNTL,
 	.shift = 16,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "vpu_1_div",
 		.ops = &clk_divider_ops,
@@ -1220,7 +1210,7 @@ static struct clk_divider gxbb_vpu_1_div = {
 static struct clk_gate gxbb_vpu_1 = {
 	.reg = (void *)HHI_VPU_CLK_CNTL,
 	.bit_idx = 24,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "vpu_1",
 		.ops = &clk_gate_ops,
@@ -1234,7 +1224,7 @@ static struct clk_mux gxbb_vpu = {
 	.reg = (void *)HHI_VPU_CLK_CNTL,
 	.mask = 1,
 	.shift = 31,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "vpu",
 		.ops = &clk_mux_ops,
@@ -1259,7 +1249,7 @@ static struct clk_mux gxbb_vapb_0_sel = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.mask = 0x3,
 	.shift = 9,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.table = mux_table_vapb,
 	.hw.init = &(struct clk_init_data){
 		.name = "vapb_0_sel",
@@ -1278,7 +1268,7 @@ static struct clk_divider gxbb_vapb_0_div = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.shift = 0,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "vapb_0_div",
 		.ops = &clk_divider_ops,
@@ -1291,7 +1281,7 @@ static struct clk_divider gxbb_vapb_0_div = {
 static struct clk_gate gxbb_vapb_0 = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.bit_idx = 8,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "vapb_0",
 		.ops = &clk_gate_ops,
@@ -1305,7 +1295,7 @@ static struct clk_mux gxbb_vapb_1_sel = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.mask = 0x3,
 	.shift = 25,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.table = mux_table_vapb,
 	.hw.init = &(struct clk_init_data){
 		.name = "vapb_1_sel",
@@ -1324,7 +1314,7 @@ static struct clk_divider gxbb_vapb_1_div = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.shift = 16,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "vapb_1_div",
 		.ops = &clk_divider_ops,
@@ -1337,7 +1327,7 @@ static struct clk_divider gxbb_vapb_1_div = {
 static struct clk_gate gxbb_vapb_1 = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.bit_idx = 24,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "vapb_1",
 		.ops = &clk_gate_ops,
@@ -1351,7 +1341,7 @@ static struct clk_mux gxbb_vapb_sel = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.mask = 1,
 	.shift = 31,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "vapb_sel",
 		.ops = &clk_mux_ops,
@@ -1368,7 +1358,7 @@ static struct clk_mux gxbb_vapb_sel = {
 static struct clk_gate gxbb_vapb = {
 	.reg = (void *)HHI_VAPBCLK_CNTL,
 	.bit_idx = 30,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data) {
 		.name = "vapb",
 		.ops = &clk_gate_ops,
@@ -1386,7 +1376,7 @@ static MESON_GATE(gxbb_pl301, HHI_GCLK_MPEG0, 6);
 static MESON_GATE(gxbb_periphs, HHI_GCLK_MPEG0, 7);
 static MESON_GATE(gxbb_spicc, HHI_GCLK_MPEG0, 8);
 static MESON_GATE(gxbb_i2c, HHI_GCLK_MPEG0, 9);
-static MESON_GATE(gxbb_sar_adc, HHI_GCLK_MPEG0, 10);
+static MESON_GATE(gxbb_sana, HHI_GCLK_MPEG0, 10);
 static MESON_GATE(gxbb_smart_card, HHI_GCLK_MPEG0, 11);
 static MESON_GATE(gxbb_rng0, HHI_GCLK_MPEG0, 12);
 static MESON_GATE(gxbb_uart0, HHI_GCLK_MPEG0, 13);
@@ -1437,7 +1427,7 @@ static MESON_GATE(gxbb_usb0_ddr_bridge, HHI_GCLK_MPEG2, 9);
 static MESON_GATE(gxbb_mmc_pclk, HHI_GCLK_MPEG2, 11);
 static MESON_GATE(gxbb_dvin, HHI_GCLK_MPEG2, 12);
 static MESON_GATE(gxbb_uart2, HHI_GCLK_MPEG2, 15);
-static MESON_GATE(gxbb_sana, HHI_GCLK_MPEG2, 22);
+static MESON_GATE(gxbb_sar_adc, HHI_GCLK_MPEG2, 22);
 static MESON_GATE(gxbb_vpu_intr, HHI_GCLK_MPEG2, 25);
 static MESON_GATE(gxbb_sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26);
 static MESON_GATE(gxbb_clk81_a53, HHI_GCLK_MPEG2, 29);
diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c
index 20ab7190d328..3ffea80c1308 100644
--- a/drivers/clk/meson/meson8b.c
+++ b/drivers/clk/meson/meson8b.c
@@ -32,7 +32,7 @@
 #include "clkc.h"
 #include "meson8b.h"
 
-static DEFINE_SPINLOCK(clk_lock);
+static DEFINE_SPINLOCK(meson_clk_lock);
 
 static void __iomem *clk_base;
 
@@ -136,7 +136,7 @@ static struct meson_clk_pll meson8b_fixed_pll = {
 		.shift   = 16,
 		.width   = 2,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "fixed_pll",
 		.ops = &meson_clk_pll_ro_ops,
@@ -162,7 +162,7 @@ static struct meson_clk_pll meson8b_vid_pll = {
 		.shift   = 16,
 		.width   = 2,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "vid_pll",
 		.ops = &meson_clk_pll_ro_ops,
@@ -190,7 +190,7 @@ static struct meson_clk_pll meson8b_sys_pll = {
 	},
 	.rate_table = sys_pll_rate_table,
 	.rate_count = ARRAY_SIZE(sys_pll_rate_table),
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "sys_pll",
 		.ops = &meson_clk_pll_ops,
@@ -281,7 +281,7 @@ static struct meson_clk_mpll meson8b_mpll0 = {
 		.shift   = 25,
 		.width   = 1,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpll0",
 		.ops = &meson_clk_mpll_ops,
@@ -311,7 +311,7 @@ static struct meson_clk_mpll meson8b_mpll1 = {
 		.shift   = 14,
 		.width   = 1,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpll1",
 		.ops = &meson_clk_mpll_ops,
@@ -341,7 +341,7 @@ static struct meson_clk_mpll meson8b_mpll2 = {
 		.shift   = 14,
 		.width   = 1,
 	},
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpll2",
 		.ops = &meson_clk_mpll_ops,
@@ -375,7 +375,7 @@ struct clk_mux meson8b_mpeg_clk_sel = {
 	.shift = 12,
 	.flags = CLK_MUX_READ_ONLY,
 	.table = mux_table_clk81,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpeg_clk_sel",
 		.ops = &clk_mux_ro_ops,
@@ -395,7 +395,7 @@ struct clk_divider meson8b_mpeg_clk_div = {
 	.reg = (void *)HHI_MPEG_CLK_CNTL,
 	.shift = 0,
 	.width = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "mpeg_clk_div",
 		.ops = &clk_divider_ops,
@@ -408,7 +408,7 @@ struct clk_divider meson8b_mpeg_clk_div = {
 struct clk_gate meson8b_clk81 = {
 	.reg = (void *)HHI_MPEG_CLK_CNTL,
 	.bit_idx = 7,
-	.lock = &clk_lock,
+	.lock = &meson_clk_lock,
 	.hw.init = &(struct clk_init_data){
 		.name = "clk81",
 		.ops = &clk_gate_ops,
@@ -773,7 +773,7 @@ static int meson8b_clk_reset_update(struct reset_controller_dev *rcdev,
 
 	reset = &meson8b_clk_reset_bits[id];
 
-	spin_lock_irqsave(&clk_lock, flags);
+	spin_lock_irqsave(&meson_clk_lock, flags);
 
 	val = readl(meson8b_clk_reset->base + reset->reg);
 	if (assert)
@@ -782,7 +782,7 @@ static int meson8b_clk_reset_update(struct reset_controller_dev *rcdev,
 		val &= ~BIT(reset->bit_idx);
 	writel(val, meson8b_clk_reset->base + reset->reg);
 
-	spin_unlock_irqrestore(&clk_lock, flags);
+	spin_unlock_irqrestore(&meson_clk_lock, flags);
 
 	return 0;
 }
diff --git a/drivers/clk/mvebu/armada-37xx-periph.c b/drivers/clk/mvebu/armada-37xx-periph.c
index cecb0fdfaef6..87213ea7fc84 100644
--- a/drivers/clk/mvebu/armada-37xx-periph.c
+++ b/drivers/clk/mvebu/armada-37xx-periph.c
@@ -21,9 +21,11 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/slab.h>
 
 #define TBG_SEL		0x0
@@ -33,6 +35,26 @@
 #define CLK_SEL		0x10
 #define CLK_DIS		0x14
 
+#define LOAD_LEVEL_NR	4
+
+#define ARMADA_37XX_NB_L0L1	0x18
+#define ARMADA_37XX_NB_L2L3	0x1C
+#define		ARMADA_37XX_NB_TBG_DIV_OFF	13
+#define		ARMADA_37XX_NB_TBG_DIV_MASK	0x7
+#define		ARMADA_37XX_NB_CLK_SEL_OFF	11
+#define		ARMADA_37XX_NB_CLK_SEL_MASK	0x1
+#define		ARMADA_37XX_NB_TBG_SEL_OFF	9
+#define		ARMADA_37XX_NB_TBG_SEL_MASK	0x3
+#define		ARMADA_37XX_NB_CONFIG_SHIFT	16
+#define ARMADA_37XX_NB_DYN_MOD	0x24
+#define		ARMADA_37XX_NB_DFS_EN	31
+#define ARMADA_37XX_NB_CPU_LOAD	0x30
+#define		ARMADA_37XX_NB_CPU_LOAD_MASK	0x3
+#define		ARMADA_37XX_DVFS_LOAD_0		0
+#define		ARMADA_37XX_DVFS_LOAD_1		1
+#define		ARMADA_37XX_DVFS_LOAD_2		2
+#define		ARMADA_37XX_DVFS_LOAD_3		3
+
 struct clk_periph_driver_data {
 	struct clk_hw_onecell_data *hw_data;
 	spinlock_t lock;
@@ -46,7 +68,18 @@ struct clk_double_div {
 	u8 shift2;
 };
 
+struct clk_pm_cpu {
+	struct clk_hw hw;
+	void __iomem *reg_mux;
+	u8 shift_mux;
+	u32 mask_mux;
+	void __iomem *reg_div;
+	u8 shift_div;
+	struct regmap *nb_pm_base;
+};
+
 #define to_clk_double_div(_hw) container_of(_hw, struct clk_double_div, hw)
+#define to_clk_pm_cpu(_hw) container_of(_hw, struct clk_pm_cpu, hw)
 
 struct clk_periph_data {
 	const char *name;
@@ -55,6 +88,7 @@ struct clk_periph_data {
 	struct clk_hw *mux_hw;
 	struct clk_hw *rate_hw;
 	struct clk_hw *gate_hw;
+	struct clk_hw *muxrate_hw;
 	bool is_double_div;
 };
 
@@ -79,7 +113,9 @@ static const struct clk_div_table clk_table2[] = {
 	{ .val = 1, .div = 4, },
 	{ .val = 0, .div = 0, }, /* last entry */
 };
+
 static const struct clk_ops clk_double_div_ops;
+static const struct clk_ops clk_pm_cpu_ops;
 
 #define PERIPH_GATE(_name, _bit)		\
 struct clk_gate gate_##_name = {		\
@@ -121,6 +157,18 @@ struct clk_divider rate_##_name = {		\
 	}					\
 };
 
+#define PERIPH_PM_CPU(_name, _shift1, _reg, _shift2)	\
+struct clk_pm_cpu muxrate_##_name = {		\
+	.reg_mux = (void *)TBG_SEL,		\
+	.mask_mux = 3,				\
+	.shift_mux = _shift1,			\
+	.reg_div = (void *)_reg,		\
+	.shift_div = _shift2,			\
+	.hw.init = &(struct clk_init_data){	\
+		.ops =  &clk_pm_cpu_ops,	\
+	}					\
+};
+
 #define PERIPH_CLK_FULL_DD(_name, _bit, _shift, _reg1, _reg2, _shift1, _shift2)\
 static PERIPH_GATE(_name, _bit);			    \
 static PERIPH_MUX(_name, _shift);			    \
@@ -135,10 +183,6 @@ static PERIPH_DIV(_name, _reg, _shift1, _table);
 static PERIPH_GATE(_name, _bit);			\
 static PERIPH_DIV(_name, _reg, _shift, _table);
 
-#define PERIPH_CLK_MUX_DIV(_name, _shift,  _reg, _shift_div, _table)	\
-static PERIPH_MUX(_name, _shift);			    \
-static PERIPH_DIV(_name, _reg, _shift_div, _table);
-
 #define PERIPH_CLK_MUX_DD(_name, _shift, _reg1, _reg2, _shift1, _shift2)\
 static PERIPH_MUX(_name, _shift);			    \
 static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
@@ -179,13 +223,12 @@ static PERIPH_DOUBLEDIV(_name, _reg1, _reg2, _shift1, _shift2);
 	  .rate_hw = &rate_##_name.hw,				\
 	}
 
-#define REF_CLK_MUX_DIV(_name)				\
+#define REF_CLK_PM_CPU(_name)				\
 	{ .name = #_name,				\
 	  .parent_names = (const char *[]){ "TBG-A-P",	\
 	      "TBG-B-P", "TBG-A-S", "TBG-B-S"},		\
 	  .num_parents = 4,				\
-	  .mux_hw = &mux_##_name.hw,			\
-	  .rate_hw = &rate_##_name.hw,			\
+	  .muxrate_hw = &muxrate_##_name.hw,		\
 	}
 
 #define REF_CLK_MUX_DD(_name)				\
@@ -215,9 +258,9 @@ PERIPH_CLK_FULL_DD(ddr_fclk, 21, 16, DIV_SEL0, DIV_SEL0, 15, 12);
 PERIPH_CLK_FULL(trace, 22, 18, DIV_SEL0, 20, clk_table6);
 PERIPH_CLK_FULL(counter, 23, 20, DIV_SEL0, 23, clk_table6);
 PERIPH_CLK_FULL_DD(eip97, 24, 24, DIV_SEL2, DIV_SEL2, 22, 19);
-PERIPH_CLK_MUX_DIV(cpu, 22, DIV_SEL0, 28, clk_table6);
+static PERIPH_PM_CPU(cpu, 22, DIV_SEL0, 28);
 
-static struct clk_periph_data data_nb[] ={
+static struct clk_periph_data data_nb[] = {
 	REF_CLK_FULL_DD(mmc),
 	REF_CLK_FULL_DD(sata_host),
 	REF_CLK_FULL_DD(sec_at),
@@ -234,7 +277,7 @@ static struct clk_periph_data data_nb[] ={
 	REF_CLK_FULL(trace),
 	REF_CLK_FULL(counter),
 	REF_CLK_FULL_DD(eip97),
-	REF_CLK_MUX_DIV(cpu),
+	REF_CLK_PM_CPU(cpu),
 	{ },
 };
 
@@ -281,7 +324,7 @@ static unsigned int get_div(void __iomem *reg, int shift)
 }
 
 static unsigned long clk_double_div_recalc_rate(struct clk_hw *hw,
-		unsigned long parent_rate)
+						unsigned long parent_rate)
 {
 	struct clk_double_div *double_div = to_clk_double_div(hw);
 	unsigned int div;
@@ -296,6 +339,222 @@ static const struct clk_ops clk_double_div_ops = {
 	.recalc_rate = clk_double_div_recalc_rate,
 };
 
+static void armada_3700_pm_dvfs_update_regs(unsigned int load_level,
+					    unsigned int *reg,
+					    unsigned int *offset)
+{
+	if (load_level <= ARMADA_37XX_DVFS_LOAD_1)
+		*reg = ARMADA_37XX_NB_L0L1;
+	else
+		*reg = ARMADA_37XX_NB_L2L3;
+
+	if (load_level == ARMADA_37XX_DVFS_LOAD_0 ||
+	    load_level ==  ARMADA_37XX_DVFS_LOAD_2)
+		*offset += ARMADA_37XX_NB_CONFIG_SHIFT;
+}
+
+static bool armada_3700_pm_dvfs_is_enabled(struct regmap *base)
+{
+	unsigned int val, reg = ARMADA_37XX_NB_DYN_MOD;
+
+	if (IS_ERR(base))
+		return false;
+
+	regmap_read(base, reg, &val);
+
+	return !!(val & BIT(ARMADA_37XX_NB_DFS_EN));
+}
+
+static unsigned int armada_3700_pm_dvfs_get_cpu_div(struct regmap *base)
+{
+	unsigned int reg = ARMADA_37XX_NB_CPU_LOAD;
+	unsigned int offset = ARMADA_37XX_NB_TBG_DIV_OFF;
+	unsigned int load_level, div;
+
+	/*
+	 * This function is always called after the function
+	 * armada_3700_pm_dvfs_is_enabled, so no need to check again
+	 * if the base is valid.
+	 */
+	regmap_read(base, reg, &load_level);
+
+	/*
+	 * The register and the offset inside this register accessed to
+	 * read the current divider depend on the load level
+	 */
+	load_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
+	armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
+
+	regmap_read(base, reg, &div);
+
+	return (div >> offset) & ARMADA_37XX_NB_TBG_DIV_MASK;
+}
+
+static unsigned int armada_3700_pm_dvfs_get_cpu_parent(struct regmap *base)
+{
+	unsigned int reg = ARMADA_37XX_NB_CPU_LOAD;
+	unsigned int offset = ARMADA_37XX_NB_TBG_SEL_OFF;
+	unsigned int load_level, sel;
+
+	/*
+	 * This function is always called after the function
+	 * armada_3700_pm_dvfs_is_enabled, so no need to check again
+	 * if the base is valid
+	 */
+	regmap_read(base, reg, &load_level);
+
+	/*
+	 * The register and the offset inside this register accessed to
+	 * read the current divider depend on the load level
+	 */
+	load_level &= ARMADA_37XX_NB_CPU_LOAD_MASK;
+	armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
+
+	regmap_read(base, reg, &sel);
+
+	return (sel >> offset) & ARMADA_37XX_NB_TBG_SEL_MASK;
+}
+
+static u8 clk_pm_cpu_get_parent(struct clk_hw *hw)
+{
+	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
+	int num_parents = clk_hw_get_num_parents(hw);
+	u32 val;
+
+	if (armada_3700_pm_dvfs_is_enabled(pm_cpu->nb_pm_base)) {
+		val = armada_3700_pm_dvfs_get_cpu_parent(pm_cpu->nb_pm_base);
+	} else {
+		val = readl(pm_cpu->reg_mux) >> pm_cpu->shift_mux;
+		val &= pm_cpu->mask_mux;
+	}
+
+	if (val >= num_parents)
+		return -EINVAL;
+
+	return val;
+}
+
+static int clk_pm_cpu_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
+	struct regmap *base = pm_cpu->nb_pm_base;
+	int load_level;
+
+	/*
+	 * We set the clock parent only if the DVFS is available but
+	 * not enabled.
+	 */
+	if (IS_ERR(base) || armada_3700_pm_dvfs_is_enabled(base))
+		return -EINVAL;
+
+	/* Set the parent clock for all the load level */
+	for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
+		unsigned int reg, mask,  val,
+			offset = ARMADA_37XX_NB_TBG_SEL_OFF;
+
+		armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
+
+		val = index << offset;
+		mask = ARMADA_37XX_NB_TBG_SEL_MASK << offset;
+		regmap_update_bits(base, reg, mask, val);
+	}
+	return 0;
+}
+
+static unsigned long clk_pm_cpu_recalc_rate(struct clk_hw *hw,
+					    unsigned long parent_rate)
+{
+	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
+	unsigned int div;
+
+	if (armada_3700_pm_dvfs_is_enabled(pm_cpu->nb_pm_base))
+		div = armada_3700_pm_dvfs_get_cpu_div(pm_cpu->nb_pm_base);
+	else
+		div = get_div(pm_cpu->reg_div, pm_cpu->shift_div);
+	return DIV_ROUND_UP_ULL((u64)parent_rate, div);
+}
+
+static long clk_pm_cpu_round_rate(struct clk_hw *hw, unsigned long rate,
+				  unsigned long *parent_rate)
+{
+	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
+	struct regmap *base = pm_cpu->nb_pm_base;
+	unsigned int div = *parent_rate / rate;
+	unsigned int load_level;
+	/* only available when DVFS is enabled */
+	if (!armada_3700_pm_dvfs_is_enabled(base))
+		return -EINVAL;
+
+	for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
+		unsigned int reg, val, offset = ARMADA_37XX_NB_TBG_DIV_OFF;
+
+		armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
+
+		regmap_read(base, reg, &val);
+
+		val >>= offset;
+		val &= ARMADA_37XX_NB_TBG_DIV_MASK;
+		if (val == div)
+			/*
+			 * We found a load level matching the target
+			 * divider, switch to this load level and
+			 * return.
+			 */
+			return *parent_rate / div;
+	}
+
+	/* We didn't find any valid divider */
+	return -EINVAL;
+}
+
+static int clk_pm_cpu_set_rate(struct clk_hw *hw, unsigned long rate,
+			       unsigned long parent_rate)
+{
+	struct clk_pm_cpu *pm_cpu = to_clk_pm_cpu(hw);
+	struct regmap *base = pm_cpu->nb_pm_base;
+	unsigned int div = parent_rate / rate;
+	unsigned int load_level;
+
+	/* only available when DVFS is enabled */
+	if (!armada_3700_pm_dvfs_is_enabled(base))
+		return -EINVAL;
+
+	for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
+		unsigned int reg, mask, val,
+			offset = ARMADA_37XX_NB_TBG_DIV_OFF;
+
+		armada_3700_pm_dvfs_update_regs(load_level, &reg, &offset);
+
+		regmap_read(base, reg, &val);
+		val >>= offset;
+		val &= ARMADA_37XX_NB_TBG_DIV_MASK;
+
+		if (val == div) {
+			/*
+			 * We found a load level matching the target
+			 * divider, switch to this load level and
+			 * return.
+			 */
+			reg = ARMADA_37XX_NB_CPU_LOAD;
+			mask = ARMADA_37XX_NB_CPU_LOAD_MASK;
+			regmap_update_bits(base, reg, mask, load_level);
+
+			return rate;
+		}
+	}
+
+	/* We didn't find any valid divider */
+	return -EINVAL;
+}
+
+static const struct clk_ops clk_pm_cpu_ops = {
+	.get_parent = clk_pm_cpu_get_parent,
+	.set_parent = clk_pm_cpu_set_parent,
+	.round_rate = clk_pm_cpu_round_rate,
+	.set_rate = clk_pm_cpu_set_rate,
+	.recalc_rate = clk_pm_cpu_recalc_rate,
+};
+
 static const struct of_device_id armada_3700_periph_clock_of_match[] = {
 	{ .compatible = "marvell,armada-3700-periph-clock-nb",
 	  .data = data_nb, },
@@ -303,6 +562,7 @@ static const struct of_device_id armada_3700_periph_clock_of_match[] = {
 	.data = data_sb, },
 	{ }
 };
+
 static int armada_3700_add_composite_clk(const struct clk_periph_data *data,
 					 void __iomem *reg, spinlock_t *lock,
 					 struct device *dev, struct clk_hw **hw)
@@ -354,15 +614,31 @@ static int armada_3700_add_composite_clk(const struct clk_periph_data *data,
 		}
 	}
 
-	*hw = clk_hw_register_composite(dev, data->name, data->parent_names,
-				       data->num_parents, mux_hw,
-				       mux_ops, rate_hw, rate_ops,
-				       gate_hw, gate_ops, CLK_IGNORE_UNUSED);
+	if (data->muxrate_hw) {
+		struct clk_pm_cpu *pmcpu_clk;
+		struct clk_hw *muxrate_hw = data->muxrate_hw;
+		struct regmap *map;
 
-	if (IS_ERR(*hw))
-		return PTR_ERR(*hw);
+		pmcpu_clk =  to_clk_pm_cpu(muxrate_hw);
+		pmcpu_clk->reg_mux = reg + (u64)pmcpu_clk->reg_mux;
+		pmcpu_clk->reg_div = reg + (u64)pmcpu_clk->reg_div;
 
-	return 0;
+		mux_hw = muxrate_hw;
+		rate_hw = muxrate_hw;
+		mux_ops = muxrate_hw->init->ops;
+		rate_ops = muxrate_hw->init->ops;
+
+		map = syscon_regmap_lookup_by_compatible(
+				"marvell,armada-3700-nb-pm");
+		pmcpu_clk->nb_pm_base = map;
+	}
+
+	*hw = clk_hw_register_composite(dev, data->name, data->parent_names,
+					data->num_parents, mux_hw,
+					mux_ops, rate_hw, rate_ops,
+					gate_hw, gate_ops, CLK_IGNORE_UNUSED);
+
+	return PTR_ERR_OR_ZERO(*hw);
 }
 
 static int armada_3700_periph_clock_probe(struct platform_device *pdev)
@@ -406,12 +682,11 @@ static int armada_3700_periph_clock_probe(struct platform_device *pdev)
 		if (armada_3700_add_composite_clk(&data[i], reg,
 						  &driver_data->lock, dev, hw))
 			dev_err(dev, "Can't register periph clock %s\n",
-			       data[i].name);
-
+				data[i].name);
 	}
 
 	ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get,
-				  driver_data->hw_data);
+				     driver_data->hw_data);
 	if (ret) {
 		for (i = 0; i < num_periph; i++)
 			clk_hw_unregister(driver_data->hw_data->hws[i]);
diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c
index 7b359afd620e..f5d815f577e0 100644
--- a/drivers/clk/nxp/clk-lpc32xx.c
+++ b/drivers/clk/nxp/clk-lpc32xx.c
@@ -526,7 +526,7 @@ static unsigned long clk_pll_recalc_rate(struct clk_hw *hw,
 	    !(pll_is_valid(parent_rate, 1, 1000000, 20000000)
 	      && pll_is_valid(cco_rate, 1, 156000000, 320000000)
 	      && pll_is_valid(ref_rate, 1, 1000000, 27000000)))
-		pr_err("%s: PLL clocks are not in valid ranges: %lu/%lu/%lu",
+		pr_err("%s: PLL clocks are not in valid ranges: %lu/%lu/%lu\n",
 		       clk_hw_get_name(hw),
 		       parent_rate, cco_rate, ref_rate);
 
@@ -956,7 +956,7 @@ static unsigned long clk_divider_recalc_rate(struct clk_hw *hw,
 	val &= div_mask(divider->width);
 
 	return divider_recalc_rate(hw, parent_rate, val, divider->table,
-				   divider->flags);
+				   divider->flags, divider->width);
 }
 
 static long clk_divider_round_rate(struct clk_hw *hw, unsigned long rate,
@@ -1505,7 +1505,7 @@ static void __init lpc32xx_clk_init(struct device_node *np)
 		return;
 	}
 	if (clk_get_rate(clk_32k) != 32768) {
-		pr_err("invalid clock rate of external 32KHz oscillator");
+		pr_err("invalid clock rate of external 32KHz oscillator\n");
 		return;
 	}
 
diff --git a/drivers/clk/pxa/clk-pxa3xx.c b/drivers/clk/pxa/clk-pxa3xx.c
index 42bdaa772be0..2d126df2bccd 100644
--- a/drivers/clk/pxa/clk-pxa3xx.c
+++ b/drivers/clk/pxa/clk-pxa3xx.c
@@ -329,12 +329,16 @@ static void __init pxa3xx_dummy_clocks_init(void)
 
 static void __init pxa3xx_base_clocks_init(void)
 {
+	struct clk *clk;
+
 	pxa3xx_register_plls();
 	pxa3xx_register_core();
 	clk_register_clk_pxa3xx_system_bus();
 	clk_register_clk_pxa3xx_ac97();
 	clk_register_clk_pxa3xx_smemc();
-	clk_register_gate(NULL, "CLK_POUT", "osc_13mhz", 0, OSCC, 11, 0, NULL);
+	clk = clk_register_gate(NULL, "CLK_POUT",
+				"osc_13mhz", 0, OSCC, 11, 0, NULL);
+	clk_register_clkdev(clk, "CLK_POUT", NULL);
 	clkdev_pxa_register(CLK_OSTIMER, "OSTIMER0", NULL,
 			    clk_register_fixed_factor(NULL, "os-timer0",
 						      "osc_13mhz", 0, 1, 4));
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index 9f6c278deead..fbf4532f94b8 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -12,6 +12,27 @@ config COMMON_CLK_QCOM
 	select REGMAP_MMIO
 	select RESET_CONTROLLER
 
+config QCOM_A53PLL
+	tristate "MSM8916 A53 PLL"
+	depends on COMMON_CLK_QCOM
+	default ARCH_QCOM
+	help
+	  Support for the A53 PLL on MSM8916 devices. It provides
+	  the CPU with frequencies above 1GHz.
+	  Say Y if you want to support higher CPU frequencies on MSM8916
+	  devices.
+
+config QCOM_CLK_APCS_MSM8916
+	tristate "MSM8916 APCS Clock Controller"
+	depends on COMMON_CLK_QCOM
+	depends on QCOM_APCS_IPC || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  Support for the APCS Clock Controller on msm8916 devices. The
+	  APCS is managing the mux and divider which feeds the CPUs.
+	  Say Y if you want to support CPU frequency scaling on devices
+	  such as msm8916.
+
 config QCOM_CLK_RPM
 	tristate "RPM based Clock Controller"
 	depends on COMMON_CLK_QCOM && MFD_QCOM_RPM
@@ -196,3 +217,12 @@ config MSM_MMCC_8996
 	  Support for the multimedia clock controller on msm8996 devices.
 	  Say Y if you want to support multimedia devices such as display,
 	  graphics, video encode/decode, camera, etc.
+
+config SPMI_PMIC_CLKDIV
+	tristate "SPMI PMIC clkdiv Support"
+	depends on (COMMON_CLK_QCOM && SPMI) || COMPILE_TEST
+	help
+	  This driver supports the clkdiv functionality on the Qualcomm
+	  Technologies, Inc. SPMI PMIC. It configures the frequency of
+	  clkdiv outputs of the PMIC. These clocks are typically wired
+	  through alternate functions on GPIO pins.
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 26410d31446b..230332cf317e 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -10,6 +10,7 @@ clk-qcom-y += clk-rcg2.o
 clk-qcom-y += clk-branch.o
 clk-qcom-y += clk-regmap-divider.o
 clk-qcom-y += clk-regmap-mux.o
+clk-qcom-y += clk-regmap-mux-div.o
 clk-qcom-y += reset.o
 clk-qcom-$(CONFIG_QCOM_GDSC) += gdsc.o
 
@@ -32,5 +33,8 @@ obj-$(CONFIG_MSM_LCC_8960) += lcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8960) += mmcc-msm8960.o
 obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o
 obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o
+obj-$(CONFIG_QCOM_A53PLL) += a53-pll.o
+obj-$(CONFIG_QCOM_CLK_APCS_MSM8916) += apcs-msm8916.o
 obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o
 obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o
+obj-$(CONFIG_SPMI_PMIC_CLKDIV) += clk-spmi-pmic-div.o
diff --git a/drivers/clk/qcom/a53-pll.c b/drivers/clk/qcom/a53-pll.c
new file mode 100644
index 000000000000..45cfc57bff92
--- /dev/null
+++ b/drivers/clk/qcom/a53-pll.c
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Qualcomm A53 PLL driver
+ *
+ * Copyright (c) 2017, Linaro Limited
+ * Author: Georgi Djakov <georgi.djakov@linaro.org>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/module.h>
+
+#include "clk-pll.h"
+#include "clk-regmap.h"
+
+static const struct pll_freq_tbl a53pll_freq[] = {
+	{  998400000, 52, 0x0, 0x1, 0 },
+	{ 1094400000, 57, 0x0, 0x1, 0 },
+	{ 1152000000, 62, 0x0, 0x1, 0 },
+	{ 1209600000, 63, 0x0, 0x1, 0 },
+	{ 1248000000, 65, 0x0, 0x1, 0 },
+	{ 1363200000, 71, 0x0, 0x1, 0 },
+	{ 1401600000, 73, 0x0, 0x1, 0 },
+	{ }
+};
+
+static const struct regmap_config a53pll_regmap_config = {
+	.reg_bits		= 32,
+	.reg_stride		= 4,
+	.val_bits		= 32,
+	.max_register		= 0x40,
+	.fast_io		= true,
+};
+
+static int qcom_a53pll_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct regmap *regmap;
+	struct resource *res;
+	struct clk_pll *pll;
+	void __iomem *base;
+	struct clk_init_data init = { };
+	int ret;
+
+	pll = devm_kzalloc(dev, sizeof(*pll), GFP_KERNEL);
+	if (!pll)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	regmap = devm_regmap_init_mmio(dev, base, &a53pll_regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	pll->l_reg = 0x04;
+	pll->m_reg = 0x08;
+	pll->n_reg = 0x0c;
+	pll->config_reg = 0x14;
+	pll->mode_reg = 0x00;
+	pll->status_reg = 0x1c;
+	pll->status_bit = 16;
+	pll->freq_tbl = a53pll_freq;
+
+	init.name = "a53pll";
+	init.parent_names = (const char *[]){ "xo" };
+	init.num_parents = 1;
+	init.ops = &clk_pll_sr2_ops;
+	init.flags = CLK_IS_CRITICAL;
+	pll->clkr.hw.init = &init;
+
+	ret = devm_clk_register_regmap(dev, &pll->clkr);
+	if (ret) {
+		dev_err(dev, "failed to register regmap clock: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get,
+					  &pll->clkr.hw);
+	if (ret) {
+		dev_err(dev, "failed to add clock provider: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id qcom_a53pll_match_table[] = {
+	{ .compatible = "qcom,msm8916-a53pll" },
+	{ }
+};
+
+static struct platform_driver qcom_a53pll_driver = {
+	.probe = qcom_a53pll_probe,
+	.driver = {
+		.name = "qcom-a53pll",
+		.of_match_table = qcom_a53pll_match_table,
+	},
+};
+module_platform_driver(qcom_a53pll_driver);
+
+MODULE_DESCRIPTION("Qualcomm A53 PLL Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/apcs-msm8916.c b/drivers/clk/qcom/apcs-msm8916.c
new file mode 100644
index 000000000000..b1cc8dbcd327
--- /dev/null
+++ b/drivers/clk/qcom/apcs-msm8916.c
@@ -0,0 +1,137 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Qualcomm APCS clock controller driver
+ *
+ * Copyright (c) 2017, Linaro Limited
+ * Author: Georgi Djakov <georgi.djakov@linaro.org>
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include "clk-regmap.h"
+#include "clk-regmap-mux-div.h"
+
+static const u32 gpll0_a53cc_map[] = { 4, 5 };
+
+static const char * const gpll0_a53cc[] = {
+	"gpll0_vote",
+	"a53pll",
+};
+
+/*
+ * We use the notifier function for switching to a temporary safe configuration
+ * (mux and divider), while the A53 PLL is reconfigured.
+ */
+static int a53cc_notifier_cb(struct notifier_block *nb, unsigned long event,
+			     void *data)
+{
+	int ret = 0;
+	struct clk_regmap_mux_div *md = container_of(nb,
+						     struct clk_regmap_mux_div,
+						     clk_nb);
+	if (event == PRE_RATE_CHANGE)
+		/* set the mux and divider to safe frequency (400mhz) */
+		ret = mux_div_set_src_div(md, 4, 3);
+
+	return notifier_from_errno(ret);
+}
+
+static int qcom_apcs_msm8916_clk_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device *parent = dev->parent;
+	struct clk_regmap_mux_div *a53cc;
+	struct regmap *regmap;
+	struct clk_init_data init = { };
+	int ret = -ENODEV;
+
+	regmap = dev_get_regmap(parent, NULL);
+	if (!regmap) {
+		dev_err(dev, "failed to get regmap: %d\n", ret);
+		return ret;
+	}
+
+	a53cc = devm_kzalloc(dev, sizeof(*a53cc), GFP_KERNEL);
+	if (!a53cc)
+		return -ENOMEM;
+
+	init.name = "a53mux";
+	init.parent_names = gpll0_a53cc;
+	init.num_parents = ARRAY_SIZE(gpll0_a53cc);
+	init.ops = &clk_regmap_mux_div_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+
+	a53cc->clkr.hw.init = &init;
+	a53cc->clkr.regmap = regmap;
+	a53cc->reg_offset = 0x50;
+	a53cc->hid_width = 5;
+	a53cc->hid_shift = 0;
+	a53cc->src_width = 3;
+	a53cc->src_shift = 8;
+	a53cc->parent_map = gpll0_a53cc_map;
+
+	a53cc->pclk = devm_clk_get(parent, NULL);
+	if (IS_ERR(a53cc->pclk)) {
+		ret = PTR_ERR(a53cc->pclk);
+		dev_err(dev, "failed to get clk: %d\n", ret);
+		return ret;
+	}
+
+	a53cc->clk_nb.notifier_call = a53cc_notifier_cb;
+	ret = clk_notifier_register(a53cc->pclk, &a53cc->clk_nb);
+	if (ret) {
+		dev_err(dev, "failed to register clock notifier: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_clk_register_regmap(dev, &a53cc->clkr);
+	if (ret) {
+		dev_err(dev, "failed to register regmap clock: %d\n", ret);
+		goto err;
+	}
+
+	ret = of_clk_add_hw_provider(parent->of_node, of_clk_hw_simple_get,
+				     &a53cc->clkr.hw);
+	if (ret) {
+		dev_err(dev, "failed to add clock provider: %d\n", ret);
+		goto err;
+	}
+
+	platform_set_drvdata(pdev, a53cc);
+
+	return 0;
+
+err:
+	clk_notifier_unregister(a53cc->pclk, &a53cc->clk_nb);
+	return ret;
+}
+
+static int qcom_apcs_msm8916_clk_remove(struct platform_device *pdev)
+{
+	struct clk_regmap_mux_div *a53cc = platform_get_drvdata(pdev);
+	struct device *parent = pdev->dev.parent;
+
+	clk_notifier_unregister(a53cc->pclk, &a53cc->clk_nb);
+	of_clk_del_provider(parent->of_node);
+
+	return 0;
+}
+
+static struct platform_driver qcom_apcs_msm8916_clk_driver = {
+	.probe = qcom_apcs_msm8916_clk_probe,
+	.remove = qcom_apcs_msm8916_clk_remove,
+	.driver = {
+		.name = "qcom-apcs-msm8916-clk",
+	},
+};
+module_platform_driver(qcom_apcs_msm8916_clk_driver);
+
+MODULE_AUTHOR("Georgi Djakov <georgi.djakov@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Qualcomm MSM8916 APCS clock driver");
diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c
index 47a1da3739ce..6d04cd96482a 100644
--- a/drivers/clk/qcom/clk-alpha-pll.c
+++ b/drivers/clk/qcom/clk-alpha-pll.c
@@ -20,7 +20,7 @@
 #include "clk-alpha-pll.h"
 #include "common.h"
 
-#define PLL_MODE		0x00
+#define PLL_MODE(p)		((p)->offset + 0x0)
 # define PLL_OUTCTRL		BIT(0)
 # define PLL_BYPASSNL		BIT(1)
 # define PLL_RESET_N		BIT(2)
@@ -32,35 +32,87 @@
 # define PLL_VOTE_FSM_ENA	BIT(20)
 # define PLL_FSM_ENA		BIT(20)
 # define PLL_VOTE_FSM_RESET	BIT(21)
+# define PLL_UPDATE		BIT(22)
+# define PLL_UPDATE_BYPASS	BIT(23)
 # define PLL_OFFLINE_ACK	BIT(28)
+# define ALPHA_PLL_ACK_LATCH	BIT(29)
 # define PLL_ACTIVE_FLAG	BIT(30)
 # define PLL_LOCK_DET		BIT(31)
 
-#define PLL_L_VAL		0x04
-#define PLL_ALPHA_VAL		0x08
-#define PLL_ALPHA_VAL_U		0x0c
+#define PLL_L_VAL(p)		((p)->offset + (p)->regs[PLL_OFF_L_VAL])
+#define PLL_ALPHA_VAL(p)	((p)->offset + (p)->regs[PLL_OFF_ALPHA_VAL])
+#define PLL_ALPHA_VAL_U(p)	((p)->offset + (p)->regs[PLL_OFF_ALPHA_VAL_U])
 
-#define PLL_USER_CTL		0x10
+#define PLL_USER_CTL(p)		((p)->offset + (p)->regs[PLL_OFF_USER_CTL])
 # define PLL_POST_DIV_SHIFT	8
-# define PLL_POST_DIV_MASK	0xf
+# define PLL_POST_DIV_MASK(p)	GENMASK((p)->width, 0)
 # define PLL_ALPHA_EN		BIT(24)
+# define PLL_ALPHA_MODE		BIT(25)
 # define PLL_VCO_SHIFT		20
 # define PLL_VCO_MASK		0x3
 
-#define PLL_USER_CTL_U		0x14
-
-#define PLL_CONFIG_CTL		0x18
-#define PLL_CONFIG_CTL_U	0x20
-#define PLL_TEST_CTL		0x1c
-#define PLL_TEST_CTL_U		0x20
-#define PLL_STATUS		0x24
+#define PLL_USER_CTL_U(p)	((p)->offset + (p)->regs[PLL_OFF_USER_CTL_U])
+
+#define PLL_CONFIG_CTL(p)	((p)->offset + (p)->regs[PLL_OFF_CONFIG_CTL])
+#define PLL_CONFIG_CTL_U(p)	((p)->offset + (p)->regs[PLL_OFF_CONFIG_CTL_U])
+#define PLL_TEST_CTL(p)		((p)->offset + (p)->regs[PLL_OFF_TEST_CTL])
+#define PLL_TEST_CTL_U(p)	((p)->offset + (p)->regs[PLL_OFF_TEST_CTL_U])
+#define PLL_STATUS(p)		((p)->offset + (p)->regs[PLL_OFF_STATUS])
+
+const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = {
+	[CLK_ALPHA_PLL_TYPE_DEFAULT] =  {
+		[PLL_OFF_L_VAL] = 0x04,
+		[PLL_OFF_ALPHA_VAL] = 0x08,
+		[PLL_OFF_ALPHA_VAL_U] = 0x0c,
+		[PLL_OFF_USER_CTL] = 0x10,
+		[PLL_OFF_USER_CTL_U] = 0x14,
+		[PLL_OFF_CONFIG_CTL] = 0x18,
+		[PLL_OFF_TEST_CTL] = 0x1c,
+		[PLL_OFF_TEST_CTL_U] = 0x20,
+		[PLL_OFF_STATUS] = 0x24,
+	},
+	[CLK_ALPHA_PLL_TYPE_HUAYRA] =  {
+		[PLL_OFF_L_VAL] = 0x04,
+		[PLL_OFF_ALPHA_VAL] = 0x08,
+		[PLL_OFF_USER_CTL] = 0x10,
+		[PLL_OFF_CONFIG_CTL] = 0x14,
+		[PLL_OFF_CONFIG_CTL_U] = 0x18,
+		[PLL_OFF_TEST_CTL] = 0x1c,
+		[PLL_OFF_TEST_CTL_U] = 0x20,
+		[PLL_OFF_STATUS] = 0x24,
+	},
+	[CLK_ALPHA_PLL_TYPE_BRAMMO] =  {
+		[PLL_OFF_L_VAL] = 0x04,
+		[PLL_OFF_ALPHA_VAL] = 0x08,
+		[PLL_OFF_ALPHA_VAL_U] = 0x0c,
+		[PLL_OFF_USER_CTL] = 0x10,
+		[PLL_OFF_CONFIG_CTL] = 0x18,
+		[PLL_OFF_TEST_CTL] = 0x1c,
+		[PLL_OFF_STATUS] = 0x24,
+	},
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_regs);
 
 /*
  * Even though 40 bits are present, use only 32 for ease of calculation.
  */
 #define ALPHA_REG_BITWIDTH	40
-#define ALPHA_BITWIDTH		32
-#define ALPHA_16BIT_MASK	0xffff
+#define ALPHA_REG_16BIT_WIDTH	16
+#define ALPHA_BITWIDTH		32U
+#define ALPHA_SHIFT(w)		min(w, ALPHA_BITWIDTH)
+
+#define PLL_HUAYRA_M_WIDTH		8
+#define PLL_HUAYRA_M_SHIFT		8
+#define PLL_HUAYRA_M_MASK		0xff
+#define PLL_HUAYRA_N_SHIFT		0
+#define PLL_HUAYRA_N_MASK		0xff
+#define PLL_HUAYRA_ALPHA_WIDTH		16
+
+#define pll_alpha_width(p)					\
+		((PLL_ALPHA_VAL_U(p) - PLL_ALPHA_VAL(p) == 4) ?	\
+				 ALPHA_REG_BITWIDTH : ALPHA_REG_16BIT_WIDTH)
+
+#define pll_has_64bit_config(p)	((PLL_CONFIG_CTL_U(p) - PLL_CONFIG_CTL(p)) == 4)
 
 #define to_clk_alpha_pll(_hw) container_of(to_clk_regmap(_hw), \
 					   struct clk_alpha_pll, clkr)
@@ -71,18 +123,17 @@
 static int wait_for_pll(struct clk_alpha_pll *pll, u32 mask, bool inverse,
 			const char *action)
 {
-	u32 val, off;
+	u32 val;
 	int count;
 	int ret;
 	const char *name = clk_hw_get_name(&pll->clkr.hw);
 
-	off = pll->offset;
-	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
 	if (ret)
 		return ret;
 
 	for (count = 100; count > 0; count--) {
-		ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+		ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
 		if (ret)
 			return ret;
 		if (inverse && !(val & mask))
@@ -109,16 +160,30 @@ static int wait_for_pll(struct clk_alpha_pll *pll, u32 mask, bool inverse,
 #define wait_for_pll_offline(pll) \
 	wait_for_pll(pll, PLL_OFFLINE_ACK, 0, "offline")
 
+#define wait_for_pll_update(pll) \
+	wait_for_pll(pll, PLL_UPDATE, 1, "update")
+
+#define wait_for_pll_update_ack_set(pll) \
+	wait_for_pll(pll, ALPHA_PLL_ACK_LATCH, 0, "update_ack_set")
+
+#define wait_for_pll_update_ack_clear(pll) \
+	wait_for_pll(pll, ALPHA_PLL_ACK_LATCH, 1, "update_ack_clear")
+
 void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
 			     const struct alpha_pll_config *config)
 {
 	u32 val, mask;
-	u32 off = pll->offset;
 
-	regmap_write(regmap, off + PLL_L_VAL, config->l);
-	regmap_write(regmap, off + PLL_ALPHA_VAL, config->alpha);
-	regmap_write(regmap, off + PLL_CONFIG_CTL, config->config_ctl_val);
-	regmap_write(regmap, off + PLL_CONFIG_CTL_U, config->config_ctl_hi_val);
+	regmap_write(regmap, PLL_L_VAL(pll), config->l);
+	regmap_write(regmap, PLL_ALPHA_VAL(pll), config->alpha);
+	regmap_write(regmap, PLL_CONFIG_CTL(pll), config->config_ctl_val);
+
+	if (pll_has_64bit_config(pll))
+		regmap_write(regmap, PLL_CONFIG_CTL_U(pll),
+			     config->config_ctl_hi_val);
+
+	if (pll_alpha_width(pll) > 32)
+		regmap_write(regmap, PLL_ALPHA_VAL_U(pll), config->alpha_hi);
 
 	val = config->main_output_mask;
 	val |= config->aux_output_mask;
@@ -127,6 +192,8 @@ void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
 	val |= config->pre_div_val;
 	val |= config->post_div_val;
 	val |= config->vco_val;
+	val |= config->alpha_en_mask;
+	val |= config->alpha_mode_mask;
 
 	mask = config->main_output_mask;
 	mask |= config->aux_output_mask;
@@ -136,20 +203,19 @@ void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
 	mask |= config->post_div_mask;
 	mask |= config->vco_mask;
 
-	regmap_update_bits(regmap, off + PLL_USER_CTL, mask, val);
+	regmap_update_bits(regmap, PLL_USER_CTL(pll), mask, val);
 
 	if (pll->flags & SUPPORTS_FSM_MODE)
-		qcom_pll_set_fsm_mode(regmap, off + PLL_MODE, 6, 0);
+		qcom_pll_set_fsm_mode(regmap, PLL_MODE(pll), 6, 0);
 }
 
 static int clk_alpha_pll_hwfsm_enable(struct clk_hw *hw)
 {
 	int ret;
-	u32 val, off;
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val;
 
-	off = pll->offset;
-	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
 	if (ret)
 		return ret;
 
@@ -158,7 +224,7 @@ static int clk_alpha_pll_hwfsm_enable(struct clk_hw *hw)
 	if (pll->flags & SUPPORTS_OFFLINE_REQ)
 		val &= ~PLL_OFFLINE_REQ;
 
-	ret = regmap_write(pll->clkr.regmap, off + PLL_MODE, val);
+	ret = regmap_write(pll->clkr.regmap, PLL_MODE(pll), val);
 	if (ret)
 		return ret;
 
@@ -171,16 +237,15 @@ static int clk_alpha_pll_hwfsm_enable(struct clk_hw *hw)
 static void clk_alpha_pll_hwfsm_disable(struct clk_hw *hw)
 {
 	int ret;
-	u32 val, off;
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val;
 
-	off = pll->offset;
-	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
 	if (ret)
 		return;
 
 	if (pll->flags & SUPPORTS_OFFLINE_REQ) {
-		ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+		ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll),
 					 PLL_OFFLINE_REQ, PLL_OFFLINE_REQ);
 		if (ret)
 			return;
@@ -191,7 +256,7 @@ static void clk_alpha_pll_hwfsm_disable(struct clk_hw *hw)
 	}
 
 	/* Disable hwfsm */
-	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+	ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll),
 				 PLL_FSM_ENA, 0);
 	if (ret)
 		return;
@@ -202,11 +267,10 @@ static void clk_alpha_pll_hwfsm_disable(struct clk_hw *hw)
 static int pll_is_enabled(struct clk_hw *hw, u32 mask)
 {
 	int ret;
-	u32 val, off;
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 val;
 
-	off = pll->offset;
-	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
 	if (ret)
 		return ret;
 
@@ -227,12 +291,10 @@ static int clk_alpha_pll_enable(struct clk_hw *hw)
 {
 	int ret;
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
-	u32 val, mask, off;
-
-	off = pll->offset;
+	u32 val, mask;
 
 	mask = PLL_OUTCTRL | PLL_RESET_N | PLL_BYPASSNL;
-	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
 	if (ret)
 		return ret;
 
@@ -248,7 +310,7 @@ static int clk_alpha_pll_enable(struct clk_hw *hw)
 	if ((val & mask) == mask)
 		return 0;
 
-	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+	ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll),
 				 PLL_BYPASSNL, PLL_BYPASSNL);
 	if (ret)
 		return ret;
@@ -260,7 +322,7 @@ static int clk_alpha_pll_enable(struct clk_hw *hw)
 	mb();
 	udelay(5);
 
-	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+	ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll),
 				 PLL_RESET_N, PLL_RESET_N);
 	if (ret)
 		return ret;
@@ -269,7 +331,7 @@ static int clk_alpha_pll_enable(struct clk_hw *hw)
 	if (ret)
 		return ret;
 
-	ret = regmap_update_bits(pll->clkr.regmap, off + PLL_MODE,
+	ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll),
 				 PLL_OUTCTRL, PLL_OUTCTRL);
 
 	/* Ensure that the write above goes through before returning. */
@@ -281,11 +343,9 @@ static void clk_alpha_pll_disable(struct clk_hw *hw)
 {
 	int ret;
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
-	u32 val, mask, off;
-
-	off = pll->offset;
+	u32 val, mask;
 
-	ret = regmap_read(pll->clkr.regmap, off + PLL_MODE, &val);
+	ret = regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val);
 	if (ret)
 		return;
 
@@ -296,23 +356,25 @@ static void clk_alpha_pll_disable(struct clk_hw *hw)
 	}
 
 	mask = PLL_OUTCTRL;
-	regmap_update_bits(pll->clkr.regmap, off + PLL_MODE, mask, 0);
+	regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), mask, 0);
 
 	/* Delay of 2 output clock ticks required until output is disabled */
 	mb();
 	udelay(1);
 
 	mask = PLL_RESET_N | PLL_BYPASSNL;
-	regmap_update_bits(pll->clkr.regmap, off + PLL_MODE, mask, 0);
+	regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), mask, 0);
 }
 
-static unsigned long alpha_pll_calc_rate(u64 prate, u32 l, u32 a)
+static unsigned long
+alpha_pll_calc_rate(u64 prate, u32 l, u32 a, u32 alpha_width)
 {
-	return (prate * l) + ((prate * a) >> ALPHA_BITWIDTH);
+	return (prate * l) + ((prate * a) >> ALPHA_SHIFT(alpha_width));
 }
 
 static unsigned long
-alpha_pll_round_rate(unsigned long rate, unsigned long prate, u32 *l, u64 *a)
+alpha_pll_round_rate(unsigned long rate, unsigned long prate, u32 *l, u64 *a,
+		     u32 alpha_width)
 {
 	u64 remainder;
 	u64 quotient;
@@ -327,14 +389,15 @@ alpha_pll_round_rate(unsigned long rate, unsigned long prate, u32 *l, u64 *a)
 	}
 
 	/* Upper ALPHA_BITWIDTH bits of Alpha */
-	quotient = remainder << ALPHA_BITWIDTH;
+	quotient = remainder << ALPHA_SHIFT(alpha_width);
+
 	remainder = do_div(quotient, prate);
 
 	if (remainder)
 		quotient++;
 
 	*a = quotient;
-	return alpha_pll_calc_rate(prate, *l, *a);
+	return alpha_pll_calc_rate(prate, *l, *a, alpha_width);
 }
 
 static const struct pll_vco *
@@ -356,71 +419,138 @@ clk_alpha_pll_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	u32 l, low, high, ctl;
 	u64 a = 0, prate = parent_rate;
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
-	u32 off = pll->offset;
+	u32 alpha_width = pll_alpha_width(pll);
 
-	regmap_read(pll->clkr.regmap, off + PLL_L_VAL, &l);
+	regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l);
 
-	regmap_read(pll->clkr.regmap, off + PLL_USER_CTL, &ctl);
+	regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl);
 	if (ctl & PLL_ALPHA_EN) {
-		regmap_read(pll->clkr.regmap, off + PLL_ALPHA_VAL, &low);
-		if (pll->flags & SUPPORTS_16BIT_ALPHA) {
-			a = low & ALPHA_16BIT_MASK;
-		} else {
-			regmap_read(pll->clkr.regmap, off + PLL_ALPHA_VAL_U,
+		regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &low);
+		if (alpha_width > 32) {
+			regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL_U(pll),
 				    &high);
 			a = (u64)high << 32 | low;
-			a >>= ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH;
+		} else {
+			a = low & GENMASK(alpha_width - 1, 0);
 		}
+
+		if (alpha_width > ALPHA_BITWIDTH)
+			a >>= alpha_width - ALPHA_BITWIDTH;
 	}
 
-	return alpha_pll_calc_rate(prate, l, a);
+	return alpha_pll_calc_rate(prate, l, a, alpha_width);
 }
 
-static int clk_alpha_pll_set_rate(struct clk_hw *hw, unsigned long rate,
-				  unsigned long prate)
+static int clk_alpha_pll_update_latch(struct clk_alpha_pll *pll,
+				      int (*is_enabled)(struct clk_hw *))
+{
+	int ret;
+	u32 mode;
+
+	if (!is_enabled(&pll->clkr.hw) ||
+	    !(pll->flags & SUPPORTS_DYNAMIC_UPDATE))
+		return 0;
+
+	regmap_read(pll->clkr.regmap, PLL_MODE(pll), &mode);
+
+	/* Latch the input to the PLL */
+	regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_UPDATE,
+			   PLL_UPDATE);
+
+	/* Wait for 2 reference cycle before checking ACK bit */
+	udelay(1);
+
+	/*
+	 * PLL will latch the new L, Alpha and freq control word.
+	 * PLL will respond by raising PLL_ACK_LATCH output when new programming
+	 * has been latched in and PLL is being updated. When
+	 * UPDATE_LOGIC_BYPASS bit is not set, PLL_UPDATE will be cleared
+	 * automatically by hardware when PLL_ACK_LATCH is asserted by PLL.
+	 */
+	if (mode & PLL_UPDATE_BYPASS) {
+		ret = wait_for_pll_update_ack_set(pll);
+		if (ret)
+			return ret;
+
+		regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_UPDATE, 0);
+	} else {
+		ret = wait_for_pll_update(pll);
+		if (ret)
+			return ret;
+	}
+
+	ret = wait_for_pll_update_ack_clear(pll);
+	if (ret)
+		return ret;
+
+	/* Wait for PLL output to stabilize */
+	udelay(10);
+
+	return 0;
+}
+
+static int __clk_alpha_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long prate,
+				    int (*is_enabled)(struct clk_hw *))
 {
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
 	const struct pll_vco *vco;
-	u32 l, off = pll->offset;
+	u32 l, alpha_width = pll_alpha_width(pll);
 	u64 a;
 
-	rate = alpha_pll_round_rate(rate, prate, &l, &a);
+	rate = alpha_pll_round_rate(rate, prate, &l, &a, alpha_width);
 	vco = alpha_pll_find_vco(pll, rate);
-	if (!vco) {
+	if (pll->vco_table && !vco) {
 		pr_err("alpha pll not in a valid vco range\n");
 		return -EINVAL;
 	}
 
-	regmap_write(pll->clkr.regmap, off + PLL_L_VAL, l);
+	regmap_write(pll->clkr.regmap, PLL_L_VAL(pll), l);
 
-	if (pll->flags & SUPPORTS_16BIT_ALPHA) {
-		regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL,
-			     a & ALPHA_16BIT_MASK);
-	} else {
-		a <<= (ALPHA_REG_BITWIDTH - ALPHA_BITWIDTH);
-		regmap_write(pll->clkr.regmap, off + PLL_ALPHA_VAL_U, a >> 32);
+	if (alpha_width > ALPHA_BITWIDTH)
+		a <<= alpha_width - ALPHA_BITWIDTH;
+
+	if (alpha_width > 32)
+		regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL_U(pll), a >> 32);
+
+	regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL(pll), a);
+
+	if (vco) {
+		regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll),
+				   PLL_VCO_MASK << PLL_VCO_SHIFT,
+				   vco->val << PLL_VCO_SHIFT);
 	}
 
-	regmap_update_bits(pll->clkr.regmap, off + PLL_USER_CTL,
-			   PLL_VCO_MASK << PLL_VCO_SHIFT,
-			   vco->val << PLL_VCO_SHIFT);
+	regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll),
+			   PLL_ALPHA_EN, PLL_ALPHA_EN);
 
-	regmap_update_bits(pll->clkr.regmap, off + PLL_USER_CTL, PLL_ALPHA_EN,
-			   PLL_ALPHA_EN);
+	return clk_alpha_pll_update_latch(pll, is_enabled);
+}
 
-	return 0;
+static int clk_alpha_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+				  unsigned long prate)
+{
+	return __clk_alpha_pll_set_rate(hw, rate, prate,
+					clk_alpha_pll_is_enabled);
+}
+
+static int clk_alpha_pll_hwfsm_set_rate(struct clk_hw *hw, unsigned long rate,
+					unsigned long prate)
+{
+	return __clk_alpha_pll_set_rate(hw, rate, prate,
+					clk_alpha_pll_hwfsm_is_enabled);
 }
 
 static long clk_alpha_pll_round_rate(struct clk_hw *hw, unsigned long rate,
 				     unsigned long *prate)
 {
 	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
-	u32 l;
+	u32 l, alpha_width = pll_alpha_width(pll);
 	u64 a;
 	unsigned long min_freq, max_freq;
 
-	rate = alpha_pll_round_rate(rate, *prate, &l, &a);
-	if (alpha_pll_find_vco(pll, rate))
+	rate = alpha_pll_round_rate(rate, *prate, &l, &a, alpha_width);
+	if (!pll->vco_table || alpha_pll_find_vco(pll, rate))
 		return rate;
 
 	min_freq = pll->vco_table[0].min_freq;
@@ -429,6 +559,158 @@ static long clk_alpha_pll_round_rate(struct clk_hw *hw, unsigned long rate,
 	return clamp(rate, min_freq, max_freq);
 }
 
+static unsigned long
+alpha_huayra_pll_calc_rate(u64 prate, u32 l, u32 a)
+{
+	/*
+	 * a contains 16 bit alpha_val in two’s compliment number in the range
+	 * of [-0.5, 0.5).
+	 */
+	if (a >= BIT(PLL_HUAYRA_ALPHA_WIDTH - 1))
+		l -= 1;
+
+	return (prate * l) + (prate * a >> PLL_HUAYRA_ALPHA_WIDTH);
+}
+
+static unsigned long
+alpha_huayra_pll_round_rate(unsigned long rate, unsigned long prate,
+			    u32 *l, u32 *a)
+{
+	u64 remainder;
+	u64 quotient;
+
+	quotient = rate;
+	remainder = do_div(quotient, prate);
+	*l = quotient;
+
+	if (!remainder) {
+		*a = 0;
+		return rate;
+	}
+
+	quotient = remainder << PLL_HUAYRA_ALPHA_WIDTH;
+	remainder = do_div(quotient, prate);
+
+	if (remainder)
+		quotient++;
+
+	/*
+	 * alpha_val should be in two’s compliment number in the range
+	 * of [-0.5, 0.5) so if quotient >= 0.5 then increment the l value
+	 * since alpha value will be subtracted in this case.
+	 */
+	if (quotient >= BIT(PLL_HUAYRA_ALPHA_WIDTH - 1))
+		*l += 1;
+
+	*a = quotient;
+	return alpha_huayra_pll_calc_rate(prate, *l, *a);
+}
+
+static unsigned long
+alpha_pll_huayra_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	u64 rate = parent_rate, tmp;
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 l, alpha = 0, ctl, alpha_m, alpha_n;
+
+	regmap_read(pll->clkr.regmap, PLL_L_VAL(pll), &l);
+	regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl);
+
+	if (ctl & PLL_ALPHA_EN) {
+		regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &alpha);
+		/*
+		 * Depending upon alpha_mode, it can be treated as M/N value or
+		 * as a two’s compliment number. When alpha_mode=1,
+		 * pll_alpha_val<15:8>=M and pll_apla_val<7:0>=N
+		 *
+		 *		Fout=FIN*(L+(M/N))
+		 *
+		 * M is a signed number (-128 to 127) and N is unsigned
+		 * (0 to 255). M/N has to be within +/-0.5.
+		 *
+		 * When alpha_mode=0, it is a two’s compliment number in the
+		 * range [-0.5, 0.5).
+		 *
+		 *		Fout=FIN*(L+(alpha_val)/2^16)
+		 *
+		 * where alpha_val is two’s compliment number.
+		 */
+		if (!(ctl & PLL_ALPHA_MODE))
+			return alpha_huayra_pll_calc_rate(rate, l, alpha);
+
+		alpha_m = alpha >> PLL_HUAYRA_M_SHIFT & PLL_HUAYRA_M_MASK;
+		alpha_n = alpha >> PLL_HUAYRA_N_SHIFT & PLL_HUAYRA_N_MASK;
+
+		rate *= l;
+		tmp = parent_rate;
+		if (alpha_m >= BIT(PLL_HUAYRA_M_WIDTH - 1)) {
+			alpha_m = BIT(PLL_HUAYRA_M_WIDTH) - alpha_m;
+			tmp *= alpha_m;
+			do_div(tmp, alpha_n);
+			rate -= tmp;
+		} else {
+			tmp *= alpha_m;
+			do_div(tmp, alpha_n);
+			rate += tmp;
+		}
+
+		return rate;
+	}
+
+	return alpha_huayra_pll_calc_rate(rate, l, alpha);
+}
+
+static int alpha_pll_huayra_set_rate(struct clk_hw *hw, unsigned long rate,
+				     unsigned long prate)
+{
+	struct clk_alpha_pll *pll = to_clk_alpha_pll(hw);
+	u32 l, a, ctl, cur_alpha = 0;
+
+	rate = alpha_huayra_pll_round_rate(rate, prate, &l, &a);
+
+	regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl);
+
+	if (ctl & PLL_ALPHA_EN)
+		regmap_read(pll->clkr.regmap, PLL_ALPHA_VAL(pll), &cur_alpha);
+
+	/*
+	 * Huayra PLL supports PLL dynamic programming. User can change L_VAL,
+	 * without having to go through the power on sequence.
+	 */
+	if (clk_alpha_pll_is_enabled(hw)) {
+		if (cur_alpha != a) {
+			pr_err("clock needs to be gated %s\n",
+			       clk_hw_get_name(hw));
+			return -EBUSY;
+		}
+
+		regmap_write(pll->clkr.regmap, PLL_L_VAL(pll), l);
+		/* Ensure that the write above goes to detect L val change. */
+		mb();
+		return wait_for_pll_enable_lock(pll);
+	}
+
+	regmap_write(pll->clkr.regmap, PLL_L_VAL(pll), l);
+	regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL(pll), a);
+
+	if (a == 0)
+		regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll),
+				   PLL_ALPHA_EN, 0x0);
+	else
+		regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll),
+				   PLL_ALPHA_EN | PLL_ALPHA_MODE, PLL_ALPHA_EN);
+
+	return 0;
+}
+
+static long alpha_pll_huayra_round_rate(struct clk_hw *hw, unsigned long rate,
+					unsigned long *prate)
+{
+	u32 l, a;
+
+	return alpha_huayra_pll_round_rate(rate, *prate, &l, &a);
+}
+
 const struct clk_ops clk_alpha_pll_ops = {
 	.enable = clk_alpha_pll_enable,
 	.disable = clk_alpha_pll_disable,
@@ -439,13 +721,23 @@ const struct clk_ops clk_alpha_pll_ops = {
 };
 EXPORT_SYMBOL_GPL(clk_alpha_pll_ops);
 
+const struct clk_ops clk_alpha_pll_huayra_ops = {
+	.enable = clk_alpha_pll_enable,
+	.disable = clk_alpha_pll_disable,
+	.is_enabled = clk_alpha_pll_is_enabled,
+	.recalc_rate = alpha_pll_huayra_recalc_rate,
+	.round_rate = alpha_pll_huayra_round_rate,
+	.set_rate = alpha_pll_huayra_set_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_huayra_ops);
+
 const struct clk_ops clk_alpha_pll_hwfsm_ops = {
 	.enable = clk_alpha_pll_hwfsm_enable,
 	.disable = clk_alpha_pll_hwfsm_disable,
 	.is_enabled = clk_alpha_pll_hwfsm_is_enabled,
 	.recalc_rate = clk_alpha_pll_recalc_rate,
 	.round_rate = clk_alpha_pll_round_rate,
-	.set_rate = clk_alpha_pll_set_rate,
+	.set_rate = clk_alpha_pll_hwfsm_set_rate,
 };
 EXPORT_SYMBOL_GPL(clk_alpha_pll_hwfsm_ops);
 
@@ -455,10 +747,10 @@ clk_alpha_pll_postdiv_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
 	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
 	u32 ctl;
 
-	regmap_read(pll->clkr.regmap, pll->offset + PLL_USER_CTL, &ctl);
+	regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl);
 
 	ctl >>= PLL_POST_DIV_SHIFT;
-	ctl &= PLL_POST_DIV_MASK;
+	ctl &= PLL_POST_DIV_MASK(pll);
 
 	return parent_rate >> fls(ctl);
 }
@@ -472,16 +764,48 @@ static const struct clk_div_table clk_alpha_div_table[] = {
 	{ }
 };
 
+static const struct clk_div_table clk_alpha_2bit_div_table[] = {
+	{ 0x0, 1 },
+	{ 0x1, 2 },
+	{ 0x3, 4 },
+	{ }
+};
+
 static long
 clk_alpha_pll_postdiv_round_rate(struct clk_hw *hw, unsigned long rate,
 				 unsigned long *prate)
 {
 	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+	const struct clk_div_table *table;
 
-	return divider_round_rate(hw, rate, prate, clk_alpha_div_table,
+	if (pll->width == 2)
+		table = clk_alpha_2bit_div_table;
+	else
+		table = clk_alpha_div_table;
+
+	return divider_round_rate(hw, rate, prate, table,
 				  pll->width, CLK_DIVIDER_POWER_OF_TWO);
 }
 
+static long
+clk_alpha_pll_postdiv_round_ro_rate(struct clk_hw *hw, unsigned long rate,
+				    unsigned long *prate)
+{
+	struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw);
+	u32 ctl, div;
+
+	regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &ctl);
+
+	ctl >>= PLL_POST_DIV_SHIFT;
+	ctl &= BIT(pll->width) - 1;
+	div = 1 << fls(ctl);
+
+	if (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT)
+		*prate = clk_hw_round_rate(clk_hw_get_parent(hw), div * rate);
+
+	return DIV_ROUND_UP_ULL((u64)*prate, div);
+}
+
 static int clk_alpha_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate,
 					  unsigned long parent_rate)
 {
@@ -491,8 +815,8 @@ static int clk_alpha_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate,
 	/* 16 -> 0xf, 8 -> 0x7, 4 -> 0x3, 2 -> 0x1, 1 -> 0x0 */
 	div = DIV_ROUND_UP_ULL((u64)parent_rate, rate) - 1;
 
-	return regmap_update_bits(pll->clkr.regmap, pll->offset + PLL_USER_CTL,
-				  PLL_POST_DIV_MASK << PLL_POST_DIV_SHIFT,
+	return regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll),
+				  PLL_POST_DIV_MASK(pll) << PLL_POST_DIV_SHIFT,
 				  div << PLL_POST_DIV_SHIFT);
 }
 
@@ -502,3 +826,9 @@ const struct clk_ops clk_alpha_pll_postdiv_ops = {
 	.set_rate = clk_alpha_pll_postdiv_set_rate,
 };
 EXPORT_SYMBOL_GPL(clk_alpha_pll_postdiv_ops);
+
+const struct clk_ops clk_alpha_pll_postdiv_ro_ops = {
+	.round_rate = clk_alpha_pll_postdiv_round_ro_rate,
+	.recalc_rate = clk_alpha_pll_postdiv_recalc_rate,
+};
+EXPORT_SYMBOL_GPL(clk_alpha_pll_postdiv_ro_ops);
diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h
index d6e1ee2c7348..7593e8a56cf2 100644
--- a/drivers/clk/qcom/clk-alpha-pll.h
+++ b/drivers/clk/qcom/clk-alpha-pll.h
@@ -17,6 +17,30 @@
 #include <linux/clk-provider.h>
 #include "clk-regmap.h"
 
+/* Alpha PLL types */
+enum {
+	CLK_ALPHA_PLL_TYPE_DEFAULT,
+	CLK_ALPHA_PLL_TYPE_HUAYRA,
+	CLK_ALPHA_PLL_TYPE_BRAMMO,
+	CLK_ALPHA_PLL_TYPE_MAX,
+};
+
+enum {
+	PLL_OFF_L_VAL,
+	PLL_OFF_ALPHA_VAL,
+	PLL_OFF_ALPHA_VAL_U,
+	PLL_OFF_USER_CTL,
+	PLL_OFF_USER_CTL_U,
+	PLL_OFF_CONFIG_CTL,
+	PLL_OFF_CONFIG_CTL_U,
+	PLL_OFF_TEST_CTL,
+	PLL_OFF_TEST_CTL_U,
+	PLL_OFF_STATUS,
+	PLL_OFF_MAX_REGS
+};
+
+extern const u8 clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_MAX][PLL_OFF_MAX_REGS];
+
 struct pll_vco {
 	unsigned long min_freq;
 	unsigned long max_freq;
@@ -27,16 +51,18 @@ struct pll_vco {
  * struct clk_alpha_pll - phase locked loop (PLL)
  * @offset: base address of registers
  * @vco_table: array of VCO settings
+ * @regs: alpha pll register map (see @clk_alpha_pll_regs)
  * @clkr: regmap clock handle
  */
 struct clk_alpha_pll {
 	u32 offset;
+	const u8 *regs;
 
 	const struct pll_vco *vco_table;
 	size_t num_vco;
 #define SUPPORTS_OFFLINE_REQ	BIT(0)
-#define SUPPORTS_16BIT_ALPHA	BIT(1)
 #define SUPPORTS_FSM_MODE	BIT(2)
+#define SUPPORTS_DYNAMIC_UPDATE	BIT(3)
 	u8 flags;
 
 	struct clk_regmap clkr;
@@ -45,12 +71,14 @@ struct clk_alpha_pll {
 /**
  * struct clk_alpha_pll_postdiv - phase locked loop (PLL) post-divider
  * @offset: base address of registers
+ * @regs: alpha pll register map (see @clk_alpha_pll_regs)
  * @width: width of post-divider
  * @clkr: regmap clock handle
  */
 struct clk_alpha_pll_postdiv {
 	u32 offset;
 	u8 width;
+	const u8 *regs;
 
 	struct clk_regmap clkr;
 };
@@ -58,12 +86,15 @@ struct clk_alpha_pll_postdiv {
 struct alpha_pll_config {
 	u32 l;
 	u32 alpha;
+	u32 alpha_hi;
 	u32 config_ctl_val;
 	u32 config_ctl_hi_val;
 	u32 main_output_mask;
 	u32 aux_output_mask;
 	u32 aux2_output_mask;
 	u32 early_output_mask;
+	u32 alpha_en_mask;
+	u32 alpha_mode_mask;
 	u32 pre_div_val;
 	u32 pre_div_mask;
 	u32 post_div_val;
@@ -75,6 +106,8 @@ struct alpha_pll_config {
 extern const struct clk_ops clk_alpha_pll_ops;
 extern const struct clk_ops clk_alpha_pll_hwfsm_ops;
 extern const struct clk_ops clk_alpha_pll_postdiv_ops;
+extern const struct clk_ops clk_alpha_pll_huayra_ops;
+extern const struct clk_ops clk_alpha_pll_postdiv_ro_ops;
 
 void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
 			     const struct alpha_pll_config *config);
diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h
index a2495457e564..2a7489a84e69 100644
--- a/drivers/clk/qcom/clk-rcg.h
+++ b/drivers/clk/qcom/clk-rcg.h
@@ -25,16 +25,6 @@ struct freq_tbl {
 	u16 n;
 };
 
-/**
- * struct parent_map - map table for PLL source select configuration values
- * @src: source PLL
- * @cfg: configuration value
- */
-struct parent_map {
-	u8 src;
-	u8 cfg;
-};
-
 /**
  * struct mn - M/N:D counter
  * @mnctr_en_bit: bit to enable mn counter
diff --git a/drivers/clk/qcom/clk-regmap-divider.c b/drivers/clk/qcom/clk-regmap-divider.c
index 53484912301e..4e9b8c2c8980 100644
--- a/drivers/clk/qcom/clk-regmap-divider.c
+++ b/drivers/clk/qcom/clk-regmap-divider.c
@@ -23,6 +23,29 @@ static inline struct clk_regmap_div *to_clk_regmap_div(struct clk_hw *hw)
 	return container_of(to_clk_regmap(hw), struct clk_regmap_div, clkr);
 }
 
+static long div_round_ro_rate(struct clk_hw *hw, unsigned long rate,
+			      unsigned long *prate)
+{
+	struct clk_regmap_div *divider = to_clk_regmap_div(hw);
+	struct clk_regmap *clkr = &divider->clkr;
+	u32 div;
+	struct clk_hw *hw_parent = clk_hw_get_parent(hw);
+
+	regmap_read(clkr->regmap, divider->reg, &div);
+	div >>= divider->shift;
+	div &= BIT(divider->width) - 1;
+	div += 1;
+
+	if (clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) {
+		if (!hw_parent)
+			return -EINVAL;
+
+		*prate = clk_hw_round_rate(hw_parent, rate * div);
+	}
+
+	return DIV_ROUND_UP_ULL((u64)*prate, div);
+}
+
 static long div_round_rate(struct clk_hw *hw, unsigned long rate,
 			   unsigned long *prate)
 {
@@ -59,7 +82,7 @@ static unsigned long div_recalc_rate(struct clk_hw *hw,
 	div &= BIT(divider->width) - 1;
 
 	return divider_recalc_rate(hw, parent_rate, div, NULL,
-				   CLK_DIVIDER_ROUND_CLOSEST);
+				   CLK_DIVIDER_ROUND_CLOSEST, divider->width);
 }
 
 const struct clk_ops clk_regmap_div_ops = {
@@ -68,3 +91,9 @@ const struct clk_ops clk_regmap_div_ops = {
 	.recalc_rate = div_recalc_rate,
 };
 EXPORT_SYMBOL_GPL(clk_regmap_div_ops);
+
+const struct clk_ops clk_regmap_div_ro_ops = {
+	.round_rate = div_round_ro_rate,
+	.recalc_rate = div_recalc_rate,
+};
+EXPORT_SYMBOL_GPL(clk_regmap_div_ro_ops);
diff --git a/drivers/clk/qcom/clk-regmap-divider.h b/drivers/clk/qcom/clk-regmap-divider.h
index fc4492e3a827..8c39c2703caf 100644
--- a/drivers/clk/qcom/clk-regmap-divider.h
+++ b/drivers/clk/qcom/clk-regmap-divider.h
@@ -25,5 +25,6 @@ struct clk_regmap_div {
 };
 
 extern const struct clk_ops clk_regmap_div_ops;
+extern const struct clk_ops clk_regmap_div_ro_ops;
 
 #endif
diff --git a/drivers/clk/qcom/clk-regmap-mux-div.c b/drivers/clk/qcom/clk-regmap-mux-div.c
new file mode 100644
index 000000000000..6044839da85a
--- /dev/null
+++ b/drivers/clk/qcom/clk-regmap-mux-div.c
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017, Linaro Limited
+ * Author: Georgi Djakov <georgi.djakov@linaro.org>
+ */
+
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/regmap.h>
+
+#include "clk-regmap-mux-div.h"
+
+#define CMD_RCGR			0x0
+#define CMD_RCGR_UPDATE			BIT(0)
+#define CMD_RCGR_DIRTY_CFG		BIT(4)
+#define CMD_RCGR_ROOT_OFF		BIT(31)
+#define CFG_RCGR			0x4
+
+#define to_clk_regmap_mux_div(_hw) \
+	container_of(to_clk_regmap(_hw), struct clk_regmap_mux_div, clkr)
+
+int mux_div_set_src_div(struct clk_regmap_mux_div *md, u32 src, u32 div)
+{
+	int ret, count;
+	u32 val, mask;
+	const char *name = clk_hw_get_name(&md->clkr.hw);
+
+	val = (div << md->hid_shift) | (src << md->src_shift);
+	mask = ((BIT(md->hid_width) - 1) << md->hid_shift) |
+	       ((BIT(md->src_width) - 1) << md->src_shift);
+
+	ret = regmap_update_bits(md->clkr.regmap, CFG_RCGR + md->reg_offset,
+				 mask, val);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(md->clkr.regmap, CMD_RCGR + md->reg_offset,
+				 CMD_RCGR_UPDATE, CMD_RCGR_UPDATE);
+	if (ret)
+		return ret;
+
+	/* Wait for update to take effect */
+	for (count = 500; count > 0; count--) {
+		ret = regmap_read(md->clkr.regmap, CMD_RCGR + md->reg_offset,
+				  &val);
+		if (ret)
+			return ret;
+		if (!(val & CMD_RCGR_UPDATE))
+			return 0;
+		udelay(1);
+	}
+
+	pr_err("%s: RCG did not update its configuration", name);
+	return -EBUSY;
+}
+EXPORT_SYMBOL_GPL(mux_div_set_src_div);
+
+static void mux_div_get_src_div(struct clk_regmap_mux_div *md, u32 *src,
+				u32 *div)
+{
+	u32 val, d, s;
+	const char *name = clk_hw_get_name(&md->clkr.hw);
+
+	regmap_read(md->clkr.regmap, CMD_RCGR + md->reg_offset, &val);
+
+	if (val & CMD_RCGR_DIRTY_CFG) {
+		pr_err("%s: RCG configuration is pending\n", name);
+		return;
+	}
+
+	regmap_read(md->clkr.regmap, CFG_RCGR + md->reg_offset, &val);
+	s = (val >> md->src_shift);
+	s &= BIT(md->src_width) - 1;
+	*src = s;
+
+	d = (val >> md->hid_shift);
+	d &= BIT(md->hid_width) - 1;
+	*div = d;
+}
+
+static inline bool is_better_rate(unsigned long req, unsigned long best,
+				  unsigned long new)
+{
+	return (req <= new && new < best) || (best < req && best < new);
+}
+
+static int mux_div_determine_rate(struct clk_hw *hw,
+				  struct clk_rate_request *req)
+{
+	struct clk_regmap_mux_div *md = to_clk_regmap_mux_div(hw);
+	unsigned int i, div, max_div;
+	unsigned long actual_rate, best_rate = 0;
+	unsigned long req_rate = req->rate;
+
+	for (i = 0; i < clk_hw_get_num_parents(hw); i++) {
+		struct clk_hw *parent = clk_hw_get_parent_by_index(hw, i);
+		unsigned long parent_rate = clk_hw_get_rate(parent);
+
+		max_div = BIT(md->hid_width) - 1;
+		for (div = 1; div < max_div; div++) {
+			parent_rate = mult_frac(req_rate, div, 2);
+			parent_rate = clk_hw_round_rate(parent, parent_rate);
+			actual_rate = mult_frac(parent_rate, 2, div);
+
+			if (is_better_rate(req_rate, best_rate, actual_rate)) {
+				best_rate = actual_rate;
+				req->rate = best_rate;
+				req->best_parent_rate = parent_rate;
+				req->best_parent_hw = parent;
+			}
+
+			if (actual_rate < req_rate || best_rate <= req_rate)
+				break;
+		}
+	}
+
+	if (!best_rate)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int __mux_div_set_rate_and_parent(struct clk_hw *hw, unsigned long rate,
+					 unsigned long prate, u32 src)
+{
+	struct clk_regmap_mux_div *md = to_clk_regmap_mux_div(hw);
+	int ret;
+	u32 div, max_div, best_src = 0, best_div = 0;
+	unsigned int i;
+	unsigned long actual_rate, best_rate = 0;
+
+	for (i = 0; i < clk_hw_get_num_parents(hw); i++) {
+		struct clk_hw *parent = clk_hw_get_parent_by_index(hw, i);
+		unsigned long parent_rate = clk_hw_get_rate(parent);
+
+		max_div = BIT(md->hid_width) - 1;
+		for (div = 1; div < max_div; div++) {
+			parent_rate = mult_frac(rate, div, 2);
+			parent_rate = clk_hw_round_rate(parent, parent_rate);
+			actual_rate = mult_frac(parent_rate, 2, div);
+
+			if (is_better_rate(rate, best_rate, actual_rate)) {
+				best_rate = actual_rate;
+				best_src = md->parent_map[i];
+				best_div = div - 1;
+			}
+
+			if (actual_rate < rate || best_rate <= rate)
+				break;
+		}
+	}
+
+	ret = mux_div_set_src_div(md, best_src, best_div);
+	if (!ret) {
+		md->div = best_div;
+		md->src = best_src;
+	}
+
+	return ret;
+}
+
+static u8 mux_div_get_parent(struct clk_hw *hw)
+{
+	struct clk_regmap_mux_div *md = to_clk_regmap_mux_div(hw);
+	const char *name = clk_hw_get_name(hw);
+	u32 i, div, src = 0;
+
+	mux_div_get_src_div(md, &src, &div);
+
+	for (i = 0; i < clk_hw_get_num_parents(hw); i++)
+		if (src == md->parent_map[i])
+			return i;
+
+	pr_err("%s: Can't find parent with src %d\n", name, src);
+	return 0;
+}
+
+static int mux_div_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_regmap_mux_div *md = to_clk_regmap_mux_div(hw);
+
+	return mux_div_set_src_div(md, md->parent_map[index], md->div);
+}
+
+static int mux_div_set_rate(struct clk_hw *hw,
+			    unsigned long rate, unsigned long prate)
+{
+	struct clk_regmap_mux_div *md = to_clk_regmap_mux_div(hw);
+
+	return __mux_div_set_rate_and_parent(hw, rate, prate, md->src);
+}
+
+static int mux_div_set_rate_and_parent(struct clk_hw *hw,  unsigned long rate,
+				       unsigned long prate, u8 index)
+{
+	struct clk_regmap_mux_div *md = to_clk_regmap_mux_div(hw);
+
+	return __mux_div_set_rate_and_parent(hw, rate, prate,
+					     md->parent_map[index]);
+}
+
+static unsigned long mux_div_recalc_rate(struct clk_hw *hw, unsigned long prate)
+{
+	struct clk_regmap_mux_div *md = to_clk_regmap_mux_div(hw);
+	u32 div, src;
+	int i, num_parents = clk_hw_get_num_parents(hw);
+	const char *name = clk_hw_get_name(hw);
+
+	mux_div_get_src_div(md, &src, &div);
+	for (i = 0; i < num_parents; i++)
+		if (src == md->parent_map[i]) {
+			struct clk_hw *p = clk_hw_get_parent_by_index(hw, i);
+			unsigned long parent_rate = clk_hw_get_rate(p);
+
+			return mult_frac(parent_rate, 2, div + 1);
+		}
+
+	pr_err("%s: Can't find parent %d\n", name, src);
+	return 0;
+}
+
+const struct clk_ops clk_regmap_mux_div_ops = {
+	.get_parent = mux_div_get_parent,
+	.set_parent = mux_div_set_parent,
+	.set_rate = mux_div_set_rate,
+	.set_rate_and_parent = mux_div_set_rate_and_parent,
+	.determine_rate = mux_div_determine_rate,
+	.recalc_rate = mux_div_recalc_rate,
+};
+EXPORT_SYMBOL_GPL(clk_regmap_mux_div_ops);
diff --git a/drivers/clk/qcom/clk-regmap-mux-div.h b/drivers/clk/qcom/clk-regmap-mux-div.h
new file mode 100644
index 000000000000..6cd6261be7ac
--- /dev/null
+++ b/drivers/clk/qcom/clk-regmap-mux-div.h
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2017, Linaro Limited
+ * Author: Georgi Djakov <georgi.djakov@linaro.org>
+ */
+
+#ifndef __QCOM_CLK_REGMAP_MUX_DIV_H__
+#define __QCOM_CLK_REGMAP_MUX_DIV_H__
+
+#include <linux/clk-provider.h>
+#include "clk-regmap.h"
+
+/**
+ * struct mux_div_clk - combined mux/divider clock
+ * @reg_offset: offset of the mux/divider register
+ * @hid_width:	number of bits in half integer divider
+ * @hid_shift:	lowest bit of hid value field
+ * @src_width:	number of bits in source select
+ * @src_shift:	lowest bit of source select field
+ * @div:	the divider raw configuration value
+ * @src:	the mux index which will be used if the clock is enabled
+ * @parent_map: map from parent_names index to src_sel field
+ * @clkr:	handle between common and hardware-specific interfaces
+ * @pclk:	the input PLL clock
+ * @clk_nb:	clock notifier for rate changes of the input PLL
+ */
+struct clk_regmap_mux_div {
+	u32				reg_offset;
+	u32				hid_width;
+	u32				hid_shift;
+	u32				src_width;
+	u32				src_shift;
+	u32				div;
+	u32				src;
+	const u32			*parent_map;
+	struct clk_regmap		clkr;
+	struct clk			*pclk;
+	struct notifier_block		clk_nb;
+};
+
+extern const struct clk_ops clk_regmap_mux_div_ops;
+extern int mux_div_set_src_div(struct clk_regmap_mux_div *md, u32 src, u32 div);
+
+#endif
diff --git a/drivers/clk/qcom/clk-regmap-mux.c b/drivers/clk/qcom/clk-regmap-mux.c
index cae3071f384c..0f3a1bda3e91 100644
--- a/drivers/clk/qcom/clk-regmap-mux.c
+++ b/drivers/clk/qcom/clk-regmap-mux.c
@@ -35,6 +35,9 @@ static u8 mux_get_parent(struct clk_hw *hw)
 	val >>= mux->shift;
 	val &= mask;
 
+	if (mux->parent_map)
+		return qcom_find_src_index(hw, mux->parent_map, val);
+
 	return val;
 }
 
@@ -45,6 +48,9 @@ static int mux_set_parent(struct clk_hw *hw, u8 index)
 	unsigned int mask = GENMASK(mux->width + mux->shift - 1, mux->shift);
 	unsigned int val;
 
+	if (mux->parent_map)
+		index = mux->parent_map[index].cfg;
+
 	val = index;
 	val <<= mux->shift;
 
diff --git a/drivers/clk/qcom/clk-regmap-mux.h b/drivers/clk/qcom/clk-regmap-mux.h
index 5cec76154fda..7797cddabe6b 100644
--- a/drivers/clk/qcom/clk-regmap-mux.h
+++ b/drivers/clk/qcom/clk-regmap-mux.h
@@ -16,11 +16,13 @@
 
 #include <linux/clk-provider.h>
 #include "clk-regmap.h"
+#include "common.h"
 
 struct clk_regmap_mux {
 	u32			reg;
 	u32			shift;
 	u32			width;
+	const struct parent_map	*parent_map;
 	struct clk_regmap	clkr;
 };
 
diff --git a/drivers/clk/qcom/clk-spmi-pmic-div.c b/drivers/clk/qcom/clk-spmi-pmic-div.c
new file mode 100644
index 000000000000..8672ab84746f
--- /dev/null
+++ b/drivers/clk/qcom/clk-spmi-pmic-div.c
@@ -0,0 +1,302 @@
+/* Copyright (c) 2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/log2.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#define REG_DIV_CTL1			0x43
+#define DIV_CTL1_DIV_FACTOR_MASK	GENMASK(2, 0)
+
+#define REG_EN_CTL			0x46
+#define REG_EN_MASK			BIT(7)
+
+struct clkdiv {
+	struct regmap		*regmap;
+	u16			base;
+	spinlock_t		lock;
+
+	struct clk_hw		hw;
+	unsigned int		cxo_period_ns;
+};
+
+static inline struct clkdiv *to_clkdiv(struct clk_hw *hw)
+{
+	return container_of(hw, struct clkdiv, hw);
+}
+
+static inline unsigned int div_factor_to_div(unsigned int div_factor)
+{
+	if (!div_factor)
+		div_factor = 1;
+
+	return 1 << (div_factor - 1);
+}
+
+static inline unsigned int div_to_div_factor(unsigned int div)
+{
+	return min(ilog2(div) + 1, 7);
+}
+
+static bool is_spmi_pmic_clkdiv_enabled(struct clkdiv *clkdiv)
+{
+	unsigned int val = 0;
+
+	regmap_read(clkdiv->regmap, clkdiv->base + REG_EN_CTL, &val);
+
+	return val & REG_EN_MASK;
+}
+
+static int
+__spmi_pmic_clkdiv_set_enable_state(struct clkdiv *clkdiv, bool enable,
+				    unsigned int div_factor)
+{
+	int ret;
+	unsigned int ns = clkdiv->cxo_period_ns;
+	unsigned int div = div_factor_to_div(div_factor);
+
+	ret = regmap_update_bits(clkdiv->regmap, clkdiv->base + REG_EN_CTL,
+				 REG_EN_MASK, enable ? REG_EN_MASK : 0);
+	if (ret)
+		return ret;
+
+	if (enable)
+		ndelay((2 + 3 * div) * ns);
+	else
+		ndelay(3 * div * ns);
+
+	return 0;
+}
+
+static int spmi_pmic_clkdiv_set_enable_state(struct clkdiv *clkdiv, bool enable)
+{
+	unsigned int div_factor;
+
+	regmap_read(clkdiv->regmap, clkdiv->base + REG_DIV_CTL1, &div_factor);
+	div_factor &= DIV_CTL1_DIV_FACTOR_MASK;
+
+	return __spmi_pmic_clkdiv_set_enable_state(clkdiv, enable, div_factor);
+}
+
+static int clk_spmi_pmic_div_enable(struct clk_hw *hw)
+{
+	struct clkdiv *clkdiv = to_clkdiv(hw);
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&clkdiv->lock, flags);
+	ret = spmi_pmic_clkdiv_set_enable_state(clkdiv, true);
+	spin_unlock_irqrestore(&clkdiv->lock, flags);
+
+	return ret;
+}
+
+static void clk_spmi_pmic_div_disable(struct clk_hw *hw)
+{
+	struct clkdiv *clkdiv = to_clkdiv(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(&clkdiv->lock, flags);
+	spmi_pmic_clkdiv_set_enable_state(clkdiv, false);
+	spin_unlock_irqrestore(&clkdiv->lock, flags);
+}
+
+static long clk_spmi_pmic_div_round_rate(struct clk_hw *hw, unsigned long rate,
+					 unsigned long *parent_rate)
+{
+	unsigned int div, div_factor;
+
+	div = DIV_ROUND_UP(*parent_rate, rate);
+	div_factor = div_to_div_factor(div);
+	div = div_factor_to_div(div_factor);
+
+	return *parent_rate / div;
+}
+
+static unsigned long
+clk_spmi_pmic_div_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct clkdiv *clkdiv = to_clkdiv(hw);
+	unsigned int div_factor;
+
+	regmap_read(clkdiv->regmap, clkdiv->base + REG_DIV_CTL1, &div_factor);
+	div_factor &= DIV_CTL1_DIV_FACTOR_MASK;
+
+	return parent_rate / div_factor_to_div(div_factor);
+}
+
+static int clk_spmi_pmic_div_set_rate(struct clk_hw *hw, unsigned long rate,
+				      unsigned long parent_rate)
+{
+	struct clkdiv *clkdiv = to_clkdiv(hw);
+	unsigned int div_factor = div_to_div_factor(parent_rate / rate);
+	unsigned long flags;
+	bool enabled;
+	int ret;
+
+	spin_lock_irqsave(&clkdiv->lock, flags);
+	enabled = is_spmi_pmic_clkdiv_enabled(clkdiv);
+	if (enabled) {
+		ret = spmi_pmic_clkdiv_set_enable_state(clkdiv, false);
+		if (ret)
+			goto unlock;
+	}
+
+	ret = regmap_update_bits(clkdiv->regmap, clkdiv->base + REG_DIV_CTL1,
+				 DIV_CTL1_DIV_FACTOR_MASK, div_factor);
+	if (ret)
+		goto unlock;
+
+	if (enabled)
+		ret = __spmi_pmic_clkdiv_set_enable_state(clkdiv, true,
+							  div_factor);
+
+unlock:
+	spin_unlock_irqrestore(&clkdiv->lock, flags);
+
+	return ret;
+}
+
+static const struct clk_ops clk_spmi_pmic_div_ops = {
+	.enable = clk_spmi_pmic_div_enable,
+	.disable = clk_spmi_pmic_div_disable,
+	.set_rate = clk_spmi_pmic_div_set_rate,
+	.recalc_rate = clk_spmi_pmic_div_recalc_rate,
+	.round_rate = clk_spmi_pmic_div_round_rate,
+};
+
+struct spmi_pmic_div_clk_cc {
+	int		nclks;
+	struct clkdiv	clks[];
+};
+
+static struct clk_hw *
+spmi_pmic_div_clk_hw_get(struct of_phandle_args *clkspec, void *data)
+{
+	struct spmi_pmic_div_clk_cc *cc = data;
+	int idx = clkspec->args[0] - 1; /* Start at 1 instead of 0 */
+
+	if (idx < 0 || idx >= cc->nclks) {
+		pr_err("%s: index value %u is invalid; allowed range [1, %d]\n",
+		       __func__, clkspec->args[0], cc->nclks);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &cc->clks[idx].hw;
+}
+
+static int spmi_pmic_clkdiv_probe(struct platform_device *pdev)
+{
+	struct spmi_pmic_div_clk_cc *cc;
+	struct clk_init_data init = {};
+	struct clkdiv *clkdiv;
+	struct clk *cxo;
+	struct regmap *regmap;
+	struct device *dev = &pdev->dev;
+	struct device_node *of_node = dev->of_node;
+	const char *parent_name;
+	int nclks, i, ret, cxo_hz;
+	char name[20];
+	u32 start;
+
+	ret = of_property_read_u32(of_node, "reg", &start);
+	if (ret < 0) {
+		dev_err(dev, "reg property reading failed\n");
+		return ret;
+	}
+
+	regmap = dev_get_regmap(dev->parent, NULL);
+	if (!regmap) {
+		dev_err(dev, "Couldn't get parent's regmap\n");
+		return -EINVAL;
+	}
+
+	ret = of_property_read_u32(of_node, "qcom,num-clkdivs", &nclks);
+	if (ret < 0) {
+		dev_err(dev, "qcom,num-clkdivs property reading failed, ret=%d\n",
+			ret);
+		return ret;
+	}
+
+	if (!nclks)
+		return -EINVAL;
+
+	cc = devm_kzalloc(dev, sizeof(*cc) + sizeof(*cc->clks) * nclks,
+			  GFP_KERNEL);
+	if (!cc)
+		return -ENOMEM;
+	cc->nclks = nclks;
+
+	cxo = clk_get(dev, "xo");
+	if (IS_ERR(cxo)) {
+		ret = PTR_ERR(cxo);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to get xo clock\n");
+		return ret;
+	}
+	cxo_hz = clk_get_rate(cxo);
+	clk_put(cxo);
+
+	parent_name = of_clk_get_parent_name(of_node, 0);
+	if (!parent_name) {
+		dev_err(dev, "missing parent clock\n");
+		return -ENODEV;
+	}
+
+	init.name = name;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+	init.ops = &clk_spmi_pmic_div_ops;
+
+	for (i = 0, clkdiv = cc->clks; i < nclks; i++) {
+		snprintf(name, sizeof(name), "div_clk%d", i + 1);
+
+		spin_lock_init(&clkdiv[i].lock);
+		clkdiv[i].base = start + i * 0x100;
+		clkdiv[i].regmap = regmap;
+		clkdiv[i].cxo_period_ns = NSEC_PER_SEC / cxo_hz;
+		clkdiv[i].hw.init = &init;
+
+		ret = devm_clk_hw_register(dev, &clkdiv[i].hw);
+		if (ret)
+			return ret;
+	}
+
+	return devm_of_clk_add_hw_provider(dev, spmi_pmic_div_clk_hw_get, cc);
+}
+
+static const struct of_device_id spmi_pmic_clkdiv_match_table[] = {
+	{ .compatible = "qcom,spmi-clkdiv" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, spmi_pmic_clkdiv_match_table);
+
+static struct platform_driver spmi_pmic_clkdiv_driver = {
+	.driver		= {
+		.name	= "qcom,spmi-pmic-clkdiv",
+		.of_match_table = spmi_pmic_clkdiv_match_table,
+	},
+	.probe		= spmi_pmic_clkdiv_probe,
+};
+module_platform_driver(spmi_pmic_clkdiv_driver);
+
+MODULE_DESCRIPTION("QCOM SPMI PMIC clkdiv driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h
index 23c1927669ba..00196ee15e73 100644
--- a/drivers/clk/qcom/common.h
+++ b/drivers/clk/qcom/common.h
@@ -20,7 +20,6 @@ struct qcom_reset_map;
 struct regmap;
 struct freq_tbl;
 struct clk_hw;
-struct parent_map;
 
 #define PLL_LOCK_COUNT_SHIFT	8
 #define PLL_LOCK_COUNT_MASK	0x3f
@@ -39,6 +38,16 @@ struct qcom_cc_desc {
 	size_t num_gdscs;
 };
 
+/**
+ * struct parent_map - map table for source select configuration values
+ * @src: source
+ * @cfg: configuration value
+ */
+struct parent_map {
+	u8 src;
+	u8 cfg;
+};
+
 extern const struct freq_tbl *qcom_find_freq(const struct freq_tbl *f,
 					     unsigned long rate);
 extern const struct freq_tbl *qcom_find_freq_floor(const struct freq_tbl *f,
diff --git a/drivers/clk/qcom/gcc-ipq8074.c b/drivers/clk/qcom/gcc-ipq8074.c
index 0f735d37690f..0462f4a8c932 100644
--- a/drivers/clk/qcom/gcc-ipq8074.c
+++ b/drivers/clk/qcom/gcc-ipq8074.c
@@ -28,6 +28,8 @@
 #include "clk-rcg.h"
 #include "clk-branch.h"
 #include "clk-alpha-pll.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
 #include "reset.h"
 
 #define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) }
@@ -36,6 +38,24 @@ enum {
 	P_XO,
 	P_GPLL0,
 	P_GPLL0_DIV2,
+	P_GPLL2,
+	P_GPLL4,
+	P_GPLL6,
+	P_SLEEP_CLK,
+	P_PCIE20_PHY0_PIPE,
+	P_PCIE20_PHY1_PIPE,
+	P_USB3PHY_0_PIPE,
+	P_USB3PHY_1_PIPE,
+	P_UBI32_PLL,
+	P_NSS_CRYPTO_PLL,
+	P_BIAS_PLL,
+	P_BIAS_PLL_NSS_NOC,
+	P_UNIPHY0_RX,
+	P_UNIPHY0_TX,
+	P_UNIPHY1_RX,
+	P_UNIPHY1_TX,
+	P_UNIPHY2_RX,
+	P_UNIPHY2_TX,
 };
 
 static const char * const gcc_xo_gpll0_gpll0_out_main_div2[] = {
@@ -50,8 +70,345 @@ static const struct parent_map gcc_xo_gpll0_gpll0_out_main_div2_map[] = {
 	{ P_GPLL0_DIV2, 4 },
 };
 
+static const char * const gcc_xo_gpll0[] = {
+	"xo",
+	"gpll0",
+};
+
+static const struct parent_map gcc_xo_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+};
+
+static const char * const gcc_xo_gpll0_gpll2_gpll0_out_main_div2[] = {
+	"xo",
+	"gpll0",
+	"gpll2",
+	"gpll0_out_main_div2",
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll2_gpll0_out_main_div2_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL2, 2 },
+	{ P_GPLL0_DIV2, 4 },
+};
+
+static const char * const gcc_xo_gpll0_sleep_clk[] = {
+	"xo",
+	"gpll0",
+	"sleep_clk",
+};
+
+static const struct parent_map gcc_xo_gpll0_sleep_clk_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 2 },
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const char * const gcc_xo_gpll6_gpll0_gpll0_out_main_div2[] = {
+	"xo",
+	"gpll6",
+	"gpll0",
+	"gpll0_out_main_div2",
+};
+
+static const struct parent_map gcc_xo_gpll6_gpll0_gpll0_out_main_div2_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL6, 1 },
+	{ P_GPLL0, 3 },
+	{ P_GPLL0_DIV2, 4 },
+};
+
+static const char * const gcc_xo_gpll0_out_main_div2_gpll0[] = {
+	"xo",
+	"gpll0_out_main_div2",
+	"gpll0",
+};
+
+static const struct parent_map gcc_xo_gpll0_out_main_div2_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_DIV2, 2 },
+	{ P_GPLL0, 1 },
+};
+
+static const char * const gcc_usb3phy_0_cc_pipe_clk_xo[] = {
+	"usb3phy_0_cc_pipe_clk",
+	"xo",
+};
+
+static const struct parent_map gcc_usb3phy_0_cc_pipe_clk_xo_map[] = {
+	{ P_USB3PHY_0_PIPE, 0 },
+	{ P_XO, 2 },
+};
+
+static const char * const gcc_usb3phy_1_cc_pipe_clk_xo[] = {
+	"usb3phy_1_cc_pipe_clk",
+	"xo",
+};
+
+static const struct parent_map gcc_usb3phy_1_cc_pipe_clk_xo_map[] = {
+	{ P_USB3PHY_1_PIPE, 0 },
+	{ P_XO, 2 },
+};
+
+static const char * const gcc_pcie20_phy0_pipe_clk_xo[] = {
+	"pcie20_phy0_pipe_clk",
+	"xo",
+};
+
+static const struct parent_map gcc_pcie20_phy0_pipe_clk_xo_map[] = {
+	{ P_PCIE20_PHY0_PIPE, 0 },
+	{ P_XO, 2 },
+};
+
+static const char * const gcc_pcie20_phy1_pipe_clk_xo[] = {
+	"pcie20_phy1_pipe_clk",
+	"xo",
+};
+
+static const struct parent_map gcc_pcie20_phy1_pipe_clk_xo_map[] = {
+	{ P_PCIE20_PHY1_PIPE, 0 },
+	{ P_XO, 2 },
+};
+
+static const char * const gcc_xo_gpll0_gpll6_gpll0_div2[] = {
+	"xo",
+	"gpll0",
+	"gpll6",
+	"gpll0_out_main_div2",
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll6_gpll0_div2_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL6, 2 },
+	{ P_GPLL0_DIV2, 4 },
+};
+
+static const char * const gcc_xo_gpll0_gpll6_gpll0_out_main_div2[] = {
+	"xo",
+	"gpll0",
+	"gpll6",
+	"gpll0_out_main_div2",
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll6_gpll0_out_main_div2_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL6, 2 },
+	{ P_GPLL0_DIV2, 3 },
+};
+
+static const char * const gcc_xo_bias_pll_nss_noc_clk_gpll0_gpll2[] = {
+	"xo",
+	"bias_pll_nss_noc_clk",
+	"gpll0",
+	"gpll2",
+};
+
+static const struct parent_map gcc_xo_bias_pll_nss_noc_clk_gpll0_gpll2_map[] = {
+	{ P_XO, 0 },
+	{ P_BIAS_PLL_NSS_NOC, 1 },
+	{ P_GPLL0, 2 },
+	{ P_GPLL2, 3 },
+};
+
+static const char * const gcc_xo_nss_crypto_pll_gpll0[] = {
+	"xo",
+	"nss_crypto_pll",
+	"gpll0",
+};
+
+static const struct parent_map gcc_xo_nss_crypto_pll_gpll0_map[] = {
+	{ P_XO, 0 },
+	{ P_NSS_CRYPTO_PLL, 1 },
+	{ P_GPLL0, 2 },
+};
+
+static const char * const gcc_xo_ubi32_pll_gpll0_gpll2_gpll4_gpll6[] = {
+	"xo",
+	"ubi32_pll",
+	"gpll0",
+	"gpll2",
+	"gpll4",
+	"gpll6",
+};
+
+static const struct parent_map gcc_xo_ubi32_gpll0_gpll2_gpll4_gpll6_map[] = {
+	{ P_XO, 0 },
+	{ P_UBI32_PLL, 1 },
+	{ P_GPLL0, 2 },
+	{ P_GPLL2, 3 },
+	{ P_GPLL4, 4 },
+	{ P_GPLL6, 5 },
+};
+
+static const char * const gcc_xo_gpll0_out_main_div2[] = {
+	"xo",
+	"gpll0_out_main_div2",
+};
+
+static const struct parent_map gcc_xo_gpll0_out_main_div2_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_DIV2, 1 },
+};
+
+static const char * const gcc_xo_bias_gpll0_gpll4_nss_ubi32[] = {
+	"xo",
+	"bias_pll_cc_clk",
+	"gpll0",
+	"gpll4",
+	"nss_crypto_pll",
+	"ubi32_pll",
+};
+
+static const struct parent_map gcc_xo_bias_gpll0_gpll4_nss_ubi32_map[] = {
+	{ P_XO, 0 },
+	{ P_BIAS_PLL, 1 },
+	{ P_GPLL0, 2 },
+	{ P_GPLL4, 3 },
+	{ P_NSS_CRYPTO_PLL, 4 },
+	{ P_UBI32_PLL, 5 },
+};
+
+static const char * const gcc_xo_gpll0_gpll4[] = {
+	"xo",
+	"gpll0",
+	"gpll4",
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll4_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL4, 2 },
+};
+
+static const char * const gcc_xo_uniphy0_rx_tx_ubi32_bias[] = {
+	"xo",
+	"uniphy0_gcc_rx_clk",
+	"uniphy0_gcc_tx_clk",
+	"ubi32_pll",
+	"bias_pll_cc_clk",
+};
+
+static const struct parent_map gcc_xo_uniphy0_rx_tx_ubi32_bias_map[] = {
+	{ P_XO, 0 },
+	{ P_UNIPHY0_RX, 1 },
+	{ P_UNIPHY0_TX, 2 },
+	{ P_UBI32_PLL, 5 },
+	{ P_BIAS_PLL, 6 },
+};
+
+static const char * const gcc_xo_uniphy0_tx_rx_ubi32_bias[] = {
+	"xo",
+	"uniphy0_gcc_tx_clk",
+	"uniphy0_gcc_rx_clk",
+	"ubi32_pll",
+	"bias_pll_cc_clk",
+};
+
+static const struct parent_map gcc_xo_uniphy0_tx_rx_ubi32_bias_map[] = {
+	{ P_XO, 0 },
+	{ P_UNIPHY0_TX, 1 },
+	{ P_UNIPHY0_RX, 2 },
+	{ P_UBI32_PLL, 5 },
+	{ P_BIAS_PLL, 6 },
+};
+
+static const char * const gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias[] = {
+	"xo",
+	"uniphy0_gcc_rx_clk",
+	"uniphy0_gcc_tx_clk",
+	"uniphy1_gcc_rx_clk",
+	"uniphy1_gcc_tx_clk",
+	"ubi32_pll",
+	"bias_pll_cc_clk",
+};
+
+static const struct parent_map
+gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map[] = {
+	{ P_XO, 0 },
+	{ P_UNIPHY0_RX, 1 },
+	{ P_UNIPHY0_TX, 2 },
+	{ P_UNIPHY1_RX, 3 },
+	{ P_UNIPHY1_TX, 4 },
+	{ P_UBI32_PLL, 5 },
+	{ P_BIAS_PLL, 6 },
+};
+
+static const char * const gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias[] = {
+	"xo",
+	"uniphy0_gcc_tx_clk",
+	"uniphy0_gcc_rx_clk",
+	"uniphy1_gcc_tx_clk",
+	"uniphy1_gcc_rx_clk",
+	"ubi32_pll",
+	"bias_pll_cc_clk",
+};
+
+static const struct parent_map
+gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map[] = {
+	{ P_XO, 0 },
+	{ P_UNIPHY0_TX, 1 },
+	{ P_UNIPHY0_RX, 2 },
+	{ P_UNIPHY1_TX, 3 },
+	{ P_UNIPHY1_RX, 4 },
+	{ P_UBI32_PLL, 5 },
+	{ P_BIAS_PLL, 6 },
+};
+
+static const char * const gcc_xo_uniphy2_rx_tx_ubi32_bias[] = {
+	"xo",
+	"uniphy2_gcc_rx_clk",
+	"uniphy2_gcc_tx_clk",
+	"ubi32_pll",
+	"bias_pll_cc_clk",
+};
+
+static const struct parent_map gcc_xo_uniphy2_rx_tx_ubi32_bias_map[] = {
+	{ P_XO, 0 },
+	{ P_UNIPHY2_RX, 1 },
+	{ P_UNIPHY2_TX, 2 },
+	{ P_UBI32_PLL, 5 },
+	{ P_BIAS_PLL, 6 },
+};
+
+static const char * const gcc_xo_uniphy2_tx_rx_ubi32_bias[] = {
+	"xo",
+	"uniphy2_gcc_tx_clk",
+	"uniphy2_gcc_rx_clk",
+	"ubi32_pll",
+	"bias_pll_cc_clk",
+};
+
+static const struct parent_map gcc_xo_uniphy2_tx_rx_ubi32_bias_map[] = {
+	{ P_XO, 0 },
+	{ P_UNIPHY2_TX, 1 },
+	{ P_UNIPHY2_RX, 2 },
+	{ P_UBI32_PLL, 5 },
+	{ P_BIAS_PLL, 6 },
+};
+
+static const char * const gcc_xo_gpll0_gpll6_gpll0_sleep_clk[] = {
+	"xo",
+	"gpll0",
+	"gpll6",
+	"gpll0_out_main_div2",
+	"sleep_clk",
+};
+
+static const struct parent_map gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0, 1 },
+	{ P_GPLL6, 2 },
+	{ P_GPLL0_DIV2, 4 },
+	{ P_SLEEP_CLK, 6 },
+};
+
 static struct clk_alpha_pll gpll0_main = {
 	.offset = 0x21000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr = {
 		.enable_reg = 0x0b000,
 		.enable_mask = BIT(0),
@@ -82,93 +439,274 @@ static struct clk_fixed_factor gpll0_out_main_div2 = {
 
 static struct clk_alpha_pll_postdiv gpll0 = {
 	.offset = 0x21000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "gpll0",
 		.parent_names = (const char *[]){
 			"gpll0_main"
 		},
 		.num_parents = 1,
-		.ops = &clk_alpha_pll_postdiv_ops,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
 	},
 };
 
-static const struct freq_tbl ftbl_pcnoc_bfdcd_clk_src[] = {
-	F(19200000, P_XO, 1, 0, 0),
-	F(50000000, P_GPLL0, 16, 0, 0),
-	F(100000000, P_GPLL0, 8, 0, 0),
-	{ }
+static struct clk_alpha_pll gpll2_main = {
+	.offset = 0x4a000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr = {
+		.enable_reg = 0x0b000,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll2_main",
+			.parent_names = (const char *[]){
+				"xo"
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+			.flags = CLK_IS_CRITICAL,
+		},
+	},
 };
 
-static struct clk_rcg2 pcnoc_bfdcd_clk_src = {
-	.cmd_rcgr = 0x27000,
-	.freq_tbl = ftbl_pcnoc_bfdcd_clk_src,
-	.hid_width = 5,
-	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+static struct clk_alpha_pll_postdiv gpll2 = {
+	.offset = 0x4a000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
-		.name = "pcnoc_bfdcd_clk_src",
-		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
-		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
-		.flags = CLK_IS_CRITICAL,
+		.name = "gpll2",
+		.parent_names = (const char *[]){
+			"gpll2_main"
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
-static struct clk_fixed_factor pcnoc_clk_src = {
-	.mult = 1,
-	.div = 1,
-	.hw.init = &(struct clk_init_data){
-		.name = "pcnoc_clk_src",
+static struct clk_alpha_pll gpll4_main = {
+	.offset = 0x24000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr = {
+		.enable_reg = 0x0b000,
+		.enable_mask = BIT(5),
+		.hw.init = &(struct clk_init_data){
+			.name = "gpll4_main",
+			.parent_names = (const char *[]){
+				"xo"
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+			.flags = CLK_IS_CRITICAL,
+		},
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll4 = {
+	.offset = 0x24000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gpll4",
 		.parent_names = (const char *[]){
-			"pcnoc_bfdcd_clk_src"
+			"gpll4_main"
 		},
 		.num_parents = 1,
-		.ops = &clk_fixed_factor_ops,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
 		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
-static struct clk_branch gcc_sleep_clk_src = {
-	.halt_reg = 0x30000,
+static struct clk_alpha_pll gpll6_main = {
+	.offset = 0x37000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO],
+	.flags = SUPPORTS_DYNAMIC_UPDATE,
 	.clkr = {
-		.enable_reg = 0x30000,
-		.enable_mask = BIT(1),
+		.enable_reg = 0x0b000,
+		.enable_mask = BIT(7),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_sleep_clk_src",
+			.name = "gpll6_main",
 			.parent_names = (const char *[]){
-				"sleep_clk"
+				"xo"
 			},
 			.num_parents = 1,
-			.ops = &clk_branch2_ops,
+			.ops = &clk_alpha_pll_ops,
+			.flags = CLK_IS_CRITICAL,
 		},
 	},
 };
 
-static const struct freq_tbl ftbl_blsp1_qup_i2c_apps_clk_src[] = {
-	F(19200000, P_XO, 1, 0, 0),
-	F(25000000, P_GPLL0_DIV2, 16, 0, 0),
-	F(50000000, P_GPLL0, 16, 0, 0),
-	{ }
-};
-
-static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = {
-	.cmd_rcgr = 0x0200c,
-	.freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src,
-	.hid_width = 5,
-	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+static struct clk_alpha_pll_postdiv gpll6 = {
+	.offset = 0x37000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO],
+	.width = 2,
 	.clkr.hw.init = &(struct clk_init_data){
-		.name = "blsp1_qup1_i2c_apps_clk_src",
-		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
-		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+		.name = "gpll6",
+		.parent_names = (const char *[]){
+			"gpll6_main"
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+		.flags = CLK_SET_RATE_PARENT,
 	},
 };
 
-static const struct freq_tbl ftbl_blsp1_qup_spi_apps_clk_src[] = {
-	F(960000, P_XO, 10, 1, 2),
-	F(4800000, P_XO, 4, 0, 0),
-	F(9600000, P_XO, 2, 0, 0),
-	F(12500000, P_GPLL0_DIV2, 16, 1, 2),
-	F(16000000, P_GPLL0, 10, 1, 5),
+static struct clk_fixed_factor gpll6_out_main_div2 = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(struct clk_init_data){
+		.name = "gpll6_out_main_div2",
+		.parent_names = (const char *[]){
+			"gpll6_main"
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll ubi32_pll_main = {
+	.offset = 0x25000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA],
+	.flags = SUPPORTS_DYNAMIC_UPDATE,
+	.clkr = {
+		.enable_reg = 0x0b000,
+		.enable_mask = BIT(6),
+		.hw.init = &(struct clk_init_data){
+			.name = "ubi32_pll_main",
+			.parent_names = (const char *[]){
+				"xo"
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_huayra_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll_postdiv ubi32_pll = {
+	.offset = 0x25000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_HUAYRA],
+	.width = 2,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ubi32_pll",
+		.parent_names = (const char *[]){
+			"ubi32_pll_main"
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_alpha_pll nss_crypto_pll_main = {
+	.offset = 0x22000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.clkr = {
+		.enable_reg = 0x0b000,
+		.enable_mask = BIT(4),
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_crypto_pll_main",
+			.parent_names = (const char *[]){
+				"xo"
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll_postdiv nss_crypto_pll = {
+	.offset = 0x22000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
+	.width = 4,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_crypto_pll",
+		.parent_names = (const char *[]){
+			"nss_crypto_pll_main"
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_pcnoc_bfdcd_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 pcnoc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x27000,
+	.freq_tbl = ftbl_pcnoc_bfdcd_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcnoc_bfdcd_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+		.flags = CLK_IS_CRITICAL,
+	},
+};
+
+static struct clk_fixed_factor pcnoc_clk_src = {
+	.mult = 1,
+	.div = 1,
+	.hw.init = &(struct clk_init_data){
+		.name = "pcnoc_clk_src",
+		.parent_names = (const char *[]){
+			"pcnoc_bfdcd_clk_src"
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_branch gcc_sleep_clk_src = {
+	.halt_reg = 0x30000,
+	.clkr = {
+		.enable_reg = 0x30000,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sleep_clk_src",
+			.parent_names = (const char *[]){
+				"sleep_clk"
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_qup_i2c_apps_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0_DIV2, 16, 0, 0),
+	F(50000000, P_GPLL0, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = {
+	.cmd_rcgr = 0x0200c,
+	.freq_tbl = ftbl_blsp1_qup_i2c_apps_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_qup1_i2c_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_blsp1_qup_spi_apps_clk_src[] = {
+	F(960000, P_XO, 10, 1, 2),
+	F(4800000, P_XO, 4, 0, 0),
+	F(9600000, P_XO, 2, 0, 0),
+	F(12500000, P_GPLL0_DIV2, 16, 1, 2),
+	F(16000000, P_GPLL0, 10, 1, 5),
 	F(19200000, P_XO, 1, 0, 0),
 	F(25000000, P_GPLL0, 16, 1, 2),
 	F(50000000, P_GPLL0, 16, 0, 0),
@@ -358,85 +896,3017 @@ static struct clk_rcg2 blsp1_uart1_apps_clk_src = {
 	},
 };
 
-static struct clk_rcg2 blsp1_uart2_apps_clk_src = {
-	.cmd_rcgr = 0x03034,
-	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
-	.mnd_width = 16,
-	.hid_width = 5,
-	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
-	.clkr.hw.init = &(struct clk_init_data){
-		.name = "blsp1_uart2_apps_clk_src",
-		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
-		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+static struct clk_rcg2 blsp1_uart2_apps_clk_src = {
+	.cmd_rcgr = 0x03034,
+	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart3_apps_clk_src = {
+	.cmd_rcgr = 0x04034,
+	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart3_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart4_apps_clk_src = {
+	.cmd_rcgr = 0x05034,
+	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart4_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart5_apps_clk_src = {
+	.cmd_rcgr = 0x06034,
+	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart5_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 blsp1_uart6_apps_clk_src = {
+	.cmd_rcgr = 0x07034,
+	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "blsp1_uart6_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_pcie_axi_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 pcie0_axi_clk_src = {
+	.cmd_rcgr = 0x75054,
+	.freq_tbl = ftbl_pcie_axi_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcie0_axi_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_pcie_aux_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+};
+
+static struct clk_rcg2 pcie0_aux_clk_src = {
+	.cmd_rcgr = 0x75024,
+	.freq_tbl = ftbl_pcie_aux_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcie0_aux_clk_src",
+		.parent_names = gcc_xo_gpll0_sleep_clk,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_mux pcie0_pipe_clk_src = {
+	.reg = 0x7501c,
+	.shift = 8,
+	.width = 2,
+	.parent_map = gcc_pcie20_phy0_pipe_clk_xo_map,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "pcie0_pipe_clk_src",
+			.parent_names = gcc_pcie20_phy0_pipe_clk_xo,
+			.num_parents = 2,
+			.ops = &clk_regmap_mux_closest_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 pcie1_axi_clk_src = {
+	.cmd_rcgr = 0x76054,
+	.freq_tbl = ftbl_pcie_axi_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcie1_axi_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 pcie1_aux_clk_src = {
+	.cmd_rcgr = 0x76024,
+	.freq_tbl = ftbl_pcie_aux_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "pcie1_aux_clk_src",
+		.parent_names = gcc_xo_gpll0_sleep_clk,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_mux pcie1_pipe_clk_src = {
+	.reg = 0x7601c,
+	.shift = 8,
+	.width = 2,
+	.parent_map = gcc_pcie20_phy1_pipe_clk_xo_map,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "pcie1_pipe_clk_src",
+			.parent_names = gcc_pcie20_phy1_pipe_clk_xo,
+			.num_parents = 2,
+			.ops = &clk_regmap_mux_closest_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_sdcc_apps_clk_src[] = {
+	F(144000, P_XO, 16, 3, 25),
+	F(400000, P_XO, 12, 1, 4),
+	F(24000000, P_GPLL2, 12, 1, 4),
+	F(48000000, P_GPLL2, 12, 1, 2),
+	F(96000000, P_GPLL2, 12, 0, 0),
+	F(177777778, P_GPLL0, 4.5, 0, 0),
+	F(192000000, P_GPLL2, 6, 0, 0),
+	F(384000000, P_GPLL2, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x42004,
+	.freq_tbl = ftbl_sdcc_apps_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll2_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc1_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll2_gpll0_out_main_div2,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_sdcc_ice_core_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(308570000, P_GPLL6, 3.5, 0, 0),
+};
+
+static struct clk_rcg2 sdcc1_ice_core_clk_src = {
+	.cmd_rcgr = 0x5d000,
+	.freq_tbl = ftbl_sdcc_ice_core_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_gpll0_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc1_ice_core_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll6_gpll0_div2,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x43004,
+	.freq_tbl = ftbl_sdcc_apps_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll2_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "sdcc2_apps_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll2_gpll0_out_main_div2,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb_master_clk_src[] = {
+	F(80000000, P_GPLL0_DIV2, 5, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(133330000, P_GPLL0, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb0_master_clk_src = {
+	.cmd_rcgr = 0x3e00c,
+	.freq_tbl = ftbl_usb_master_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_out_main_div2_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb0_master_clk_src",
+		.parent_names = gcc_xo_gpll0_out_main_div2_gpll0,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb_aux_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 usb0_aux_clk_src = {
+	.cmd_rcgr = 0x3e05c,
+	.freq_tbl = ftbl_usb_aux_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb0_aux_clk_src",
+		.parent_names = gcc_xo_gpll0_sleep_clk,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_usb_mock_utmi_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(20000000, P_GPLL6, 6, 1, 9),
+	F(60000000, P_GPLL6, 6, 1, 3),
+	{ }
+};
+
+static struct clk_rcg2 usb0_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x3e020,
+	.freq_tbl = ftbl_usb_mock_utmi_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll6_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb0_mock_utmi_clk_src",
+		.parent_names = gcc_xo_gpll6_gpll0_gpll0_out_main_div2,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_mux usb0_pipe_clk_src = {
+	.reg = 0x3e048,
+	.shift = 8,
+	.width = 2,
+	.parent_map = gcc_usb3phy_0_cc_pipe_clk_xo_map,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "usb0_pipe_clk_src",
+			.parent_names = gcc_usb3phy_0_cc_pipe_clk_xo,
+			.num_parents = 2,
+			.ops = &clk_regmap_mux_closest_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 usb1_master_clk_src = {
+	.cmd_rcgr = 0x3f00c,
+	.freq_tbl = ftbl_usb_master_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_out_main_div2_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb1_master_clk_src",
+		.parent_names = gcc_xo_gpll0_out_main_div2_gpll0,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 usb1_aux_clk_src = {
+	.cmd_rcgr = 0x3f05c,
+	.freq_tbl = ftbl_usb_aux_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb1_aux_clk_src",
+		.parent_names = gcc_xo_gpll0_sleep_clk,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 usb1_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x3f020,
+	.freq_tbl = ftbl_usb_mock_utmi_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll6_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "usb1_mock_utmi_clk_src",
+		.parent_names = gcc_xo_gpll6_gpll0_gpll0_out_main_div2,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_mux usb1_pipe_clk_src = {
+	.reg = 0x3f048,
+	.shift = 8,
+	.width = 2,
+	.parent_map = gcc_usb3phy_1_cc_pipe_clk_xo_map,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "usb1_pipe_clk_src",
+			.parent_names = gcc_usb3phy_1_cc_pipe_clk_xo,
+			.num_parents = 2,
+			.ops = &clk_regmap_mux_closest_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_branch gcc_xo_clk_src = {
+	.halt_reg = 0x30018,
+	.clkr = {
+		.enable_reg = 0x30018,
+		.enable_mask = BIT(1),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_xo_clk_src",
+			.parent_names = (const char *[]){
+				"xo"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_fixed_factor gcc_xo_div4_clk_src = {
+	.mult = 1,
+	.div = 4,
+	.hw.init = &(struct clk_init_data){
+		.name = "gcc_xo_div4_clk_src",
+		.parent_names = (const char *[]){
+			"gcc_xo_clk_src"
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_system_noc_bfdcd_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0_DIV2, 8, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(133333333, P_GPLL0, 6, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	F(266666667, P_GPLL0, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 system_noc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x26004,
+	.freq_tbl = ftbl_system_noc_bfdcd_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "system_noc_bfdcd_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll6_gpll0_out_main_div2,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+		.flags = CLK_IS_CRITICAL,
+	},
+};
+
+static struct clk_fixed_factor system_noc_clk_src = {
+	.mult = 1,
+	.div = 1,
+	.hw.init = &(struct clk_init_data){
+		.name = "system_noc_clk_src",
+		.parent_names = (const char *[]){
+			"system_noc_bfdcd_clk_src"
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_nss_ce_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(200000000, P_GPLL0, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_ce_clk_src = {
+	.cmd_rcgr = 0x68098,
+	.freq_tbl = ftbl_nss_ce_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_ce_clk_src",
+		.parent_names = gcc_xo_gpll0,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_nss_noc_bfdcd_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(461500000, P_BIAS_PLL_NSS_NOC, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_noc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x68088,
+	.freq_tbl = ftbl_nss_noc_bfdcd_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_bias_pll_nss_noc_clk_gpll0_gpll2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_noc_bfdcd_clk_src",
+		.parent_names = gcc_xo_bias_pll_nss_noc_clk_gpll0_gpll2,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_fixed_factor nss_noc_clk_src = {
+	.mult = 1,
+	.div = 1,
+	.hw.init = &(struct clk_init_data){
+		.name = "nss_noc_clk_src",
+		.parent_names = (const char *[]){
+			"nss_noc_bfdcd_clk_src"
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_nss_crypto_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(600000000, P_NSS_CRYPTO_PLL, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_crypto_clk_src = {
+	.cmd_rcgr = 0x68144,
+	.freq_tbl = ftbl_nss_crypto_clk_src,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_xo_nss_crypto_pll_gpll0_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_crypto_clk_src",
+		.parent_names = gcc_xo_nss_crypto_pll_gpll0,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_nss_ubi_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(187200000, P_UBI32_PLL, 8, 0, 0),
+	F(748800000, P_UBI32_PLL, 2, 0, 0),
+	F(1497600000, P_UBI32_PLL, 1, 0, 0),
+	F(1689600000, P_UBI32_PLL, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_ubi0_clk_src = {
+	.cmd_rcgr = 0x68104,
+	.freq_tbl = ftbl_nss_ubi_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_ubi32_gpll0_gpll2_gpll4_gpll6_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_ubi0_clk_src",
+		.parent_names = gcc_xo_ubi32_pll_gpll0_gpll2_gpll4_gpll6,
+		.num_parents = 6,
+		.ops = &clk_rcg2_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap_div nss_ubi0_div_clk_src = {
+	.reg = 0x68118,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_ubi0_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_ubi0_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ro_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 nss_ubi1_clk_src = {
+	.cmd_rcgr = 0x68124,
+	.freq_tbl = ftbl_nss_ubi_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_ubi32_gpll0_gpll2_gpll4_gpll6_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_ubi1_clk_src",
+		.parent_names = gcc_xo_ubi32_pll_gpll0_gpll2_gpll4_gpll6,
+		.num_parents = 6,
+		.ops = &clk_rcg2_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap_div nss_ubi1_div_clk_src = {
+	.reg = 0x68138,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_ubi1_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_ubi1_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ro_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_ubi_mpt_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0_DIV2, 16, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 ubi_mpt_clk_src = {
+	.cmd_rcgr = 0x68090,
+	.freq_tbl = ftbl_ubi_mpt_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "ubi_mpt_clk_src",
+		.parent_names = gcc_xo_gpll0_out_main_div2,
+		.num_parents = 2,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_nss_imem_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(400000000, P_GPLL0, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_imem_clk_src = {
+	.cmd_rcgr = 0x68158,
+	.freq_tbl = ftbl_nss_imem_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll4_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_imem_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll4,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_nss_ppe_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(300000000, P_BIAS_PLL, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_ppe_clk_src = {
+	.cmd_rcgr = 0x68080,
+	.freq_tbl = ftbl_nss_ppe_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_bias_gpll0_gpll4_nss_ubi32_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_ppe_clk_src",
+		.parent_names = gcc_xo_bias_gpll0_gpll4_nss_ubi32,
+		.num_parents = 6,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_fixed_factor nss_ppe_cdiv_clk_src = {
+	.mult = 1,
+	.div = 4,
+	.hw.init = &(struct clk_init_data){
+		.name = "nss_ppe_cdiv_clk_src",
+		.parent_names = (const char *[]){
+			"nss_ppe_clk_src"
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const struct freq_tbl ftbl_nss_port1_rx_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_UNIPHY0_RX, 5, 0, 0),
+	F(125000000, P_UNIPHY0_RX, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_port1_rx_clk_src = {
+	.cmd_rcgr = 0x68020,
+	.freq_tbl = ftbl_nss_port1_rx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port1_rx_clk_src",
+		.parent_names = gcc_xo_uniphy0_rx_tx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port1_rx_div_clk_src = {
+	.reg = 0x68400,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port1_rx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port1_rx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_nss_port1_tx_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_UNIPHY0_TX, 5, 0, 0),
+	F(125000000, P_UNIPHY0_TX, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_port1_tx_clk_src = {
+	.cmd_rcgr = 0x68028,
+	.freq_tbl = ftbl_nss_port1_tx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port1_tx_clk_src",
+		.parent_names = gcc_xo_uniphy0_tx_rx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port1_tx_div_clk_src = {
+	.reg = 0x68404,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port1_tx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port1_tx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 nss_port2_rx_clk_src = {
+	.cmd_rcgr = 0x68030,
+	.freq_tbl = ftbl_nss_port1_rx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port2_rx_clk_src",
+		.parent_names = gcc_xo_uniphy0_rx_tx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port2_rx_div_clk_src = {
+	.reg = 0x68410,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port2_rx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port2_rx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 nss_port2_tx_clk_src = {
+	.cmd_rcgr = 0x68038,
+	.freq_tbl = ftbl_nss_port1_tx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port2_tx_clk_src",
+		.parent_names = gcc_xo_uniphy0_tx_rx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port2_tx_div_clk_src = {
+	.reg = 0x68414,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port2_tx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port2_tx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 nss_port3_rx_clk_src = {
+	.cmd_rcgr = 0x68040,
+	.freq_tbl = ftbl_nss_port1_rx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port3_rx_clk_src",
+		.parent_names = gcc_xo_uniphy0_rx_tx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port3_rx_div_clk_src = {
+	.reg = 0x68420,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port3_rx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port3_rx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 nss_port3_tx_clk_src = {
+	.cmd_rcgr = 0x68048,
+	.freq_tbl = ftbl_nss_port1_tx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port3_tx_clk_src",
+		.parent_names = gcc_xo_uniphy0_tx_rx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port3_tx_div_clk_src = {
+	.reg = 0x68424,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port3_tx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port3_tx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 nss_port4_rx_clk_src = {
+	.cmd_rcgr = 0x68050,
+	.freq_tbl = ftbl_nss_port1_rx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_rx_tx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port4_rx_clk_src",
+		.parent_names = gcc_xo_uniphy0_rx_tx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port4_rx_div_clk_src = {
+	.reg = 0x68430,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port4_rx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port4_rx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct clk_rcg2 nss_port4_tx_clk_src = {
+	.cmd_rcgr = 0x68058,
+	.freq_tbl = ftbl_nss_port1_tx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_tx_rx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port4_tx_clk_src",
+		.parent_names = gcc_xo_uniphy0_tx_rx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port4_tx_div_clk_src = {
+	.reg = 0x68434,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port4_tx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port4_tx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_nss_port5_rx_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_UNIPHY1_RX, 12.5, 0, 0),
+	F(78125000, P_UNIPHY1_RX, 4, 0, 0),
+	F(125000000, P_UNIPHY1_RX, 2.5, 0, 0),
+	F(156250000, P_UNIPHY1_RX, 2, 0, 0),
+	F(312500000, P_UNIPHY1_RX, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_port5_rx_clk_src = {
+	.cmd_rcgr = 0x68060,
+	.freq_tbl = ftbl_nss_port5_rx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port5_rx_clk_src",
+		.parent_names = gcc_xo_uniphy0_rx_tx_uniphy1_rx_tx_ubi32_bias,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port5_rx_div_clk_src = {
+	.reg = 0x68440,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port5_rx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port5_rx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_nss_port5_tx_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_UNIPHY1_TX, 12.5, 0, 0),
+	F(78125000, P_UNIPHY1_TX, 4, 0, 0),
+	F(125000000, P_UNIPHY1_TX, 2.5, 0, 0),
+	F(156250000, P_UNIPHY1_TX, 2, 0, 0),
+	F(312500000, P_UNIPHY1_TX, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_port5_tx_clk_src = {
+	.cmd_rcgr = 0x68068,
+	.freq_tbl = ftbl_nss_port5_tx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port5_tx_clk_src",
+		.parent_names = gcc_xo_uniphy0_tx_rx_uniphy1_tx_rx_ubi32_bias,
+		.num_parents = 7,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port5_tx_div_clk_src = {
+	.reg = 0x68444,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port5_tx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port5_tx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_nss_port6_rx_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_UNIPHY2_RX, 12.5, 0, 0),
+	F(78125000, P_UNIPHY2_RX, 4, 0, 0),
+	F(125000000, P_UNIPHY2_RX, 2.5, 0, 0),
+	F(156250000, P_UNIPHY2_RX, 2, 0, 0),
+	F(312500000, P_UNIPHY2_RX, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_port6_rx_clk_src = {
+	.cmd_rcgr = 0x68070,
+	.freq_tbl = ftbl_nss_port6_rx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy2_rx_tx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port6_rx_clk_src",
+		.parent_names = gcc_xo_uniphy2_rx_tx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port6_rx_div_clk_src = {
+	.reg = 0x68450,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port6_rx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port6_rx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_nss_port6_tx_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(25000000, P_UNIPHY2_TX, 12.5, 0, 0),
+	F(78125000, P_UNIPHY2_TX, 4, 0, 0),
+	F(125000000, P_UNIPHY2_TX, 2.5, 0, 0),
+	F(156250000, P_UNIPHY2_TX, 2, 0, 0),
+	F(312500000, P_UNIPHY2_TX, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 nss_port6_tx_clk_src = {
+	.cmd_rcgr = 0x68078,
+	.freq_tbl = ftbl_nss_port6_tx_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_uniphy2_tx_rx_ubi32_bias_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "nss_port6_tx_clk_src",
+		.parent_names = gcc_xo_uniphy2_tx_rx_ubi32_bias,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div nss_port6_tx_div_clk_src = {
+	.reg = 0x68454,
+	.shift = 0,
+	.width = 4,
+	.clkr = {
+		.hw.init = &(struct clk_init_data){
+			.name = "nss_port6_tx_div_clk_src",
+			.parent_names = (const char *[]){
+				"nss_port6_tx_clk_src"
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_div_ops,
+			.flags = CLK_SET_RATE_PARENT,
+		},
+	},
+};
+
+static struct freq_tbl ftbl_crypto_clk_src[] = {
+	F(40000000, P_GPLL0_DIV2, 10, 0, 0),
+	F(80000000, P_GPLL0, 10, 0, 0),
+	F(100000000, P_GPLL0, 8, 0, 0),
+	F(160000000, P_GPLL0, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 crypto_clk_src = {
+	.cmd_rcgr = 0x16004,
+	.freq_tbl = ftbl_crypto_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "crypto_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
+		.num_parents = 3,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct freq_tbl ftbl_gp_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gp1_clk_src = {
+	.cmd_rcgr = 0x08004,
+	.freq_tbl = ftbl_gp_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp1_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll6_gpll0_sleep_clk,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp2_clk_src = {
+	.cmd_rcgr = 0x09004,
+	.freq_tbl = ftbl_gp_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp2_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll6_gpll0_sleep_clk,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gp3_clk_src = {
+	.cmd_rcgr = 0x0a004,
+	.freq_tbl = ftbl_gp_clk_src,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_xo_gpll0_gpll6_gpll0_sleep_clk_map,
+	.clkr.hw.init = &(struct clk_init_data){
+		.name = "gp3_clk_src",
+		.parent_names = gcc_xo_gpll0_gpll6_gpll0_sleep_clk,
+		.num_parents = 5,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_branch gcc_blsp1_ahb_clk = {
+	.halt_reg = 0x01008,
+	.clkr = {
+		.enable_reg = 0x01008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
+	.halt_reg = 0x02008,
+	.clkr = {
+		.enable_reg = 0x02008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup1_i2c_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
+	.halt_reg = 0x02004,
+	.clkr = {
+		.enable_reg = 0x02004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup1_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup1_spi_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
+	.halt_reg = 0x03010,
+	.clkr = {
+		.enable_reg = 0x03010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup2_i2c_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
+	.halt_reg = 0x0300c,
+	.clkr = {
+		.enable_reg = 0x0300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup2_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup2_spi_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
+	.halt_reg = 0x04010,
+	.clkr = {
+		.enable_reg = 0x04010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup3_i2c_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
+	.halt_reg = 0x0400c,
+	.clkr = {
+		.enable_reg = 0x0400c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup3_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup3_spi_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
+	.halt_reg = 0x05010,
+	.clkr = {
+		.enable_reg = 0x05010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup4_i2c_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
+	.halt_reg = 0x0500c,
+	.clkr = {
+		.enable_reg = 0x0500c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup4_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup4_spi_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = {
+	.halt_reg = 0x06010,
+	.clkr = {
+		.enable_reg = 0x06010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup5_i2c_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = {
+	.halt_reg = 0x0600c,
+	.clkr = {
+		.enable_reg = 0x0600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup5_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup5_spi_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = {
+	.halt_reg = 0x07010,
+	.clkr = {
+		.enable_reg = 0x07010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_i2c_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup6_i2c_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = {
+	.halt_reg = 0x0700c,
+	.clkr = {
+		.enable_reg = 0x0700c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_qup6_spi_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_qup6_spi_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart1_apps_clk = {
+	.halt_reg = 0x0203c,
+	.clkr = {
+		.enable_reg = 0x0203c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart1_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart1_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart2_apps_clk = {
+	.halt_reg = 0x0302c,
+	.clkr = {
+		.enable_reg = 0x0302c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart2_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart2_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart3_apps_clk = {
+	.halt_reg = 0x0402c,
+	.clkr = {
+		.enable_reg = 0x0402c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart3_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart3_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart4_apps_clk = {
+	.halt_reg = 0x0502c,
+	.clkr = {
+		.enable_reg = 0x0502c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart4_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart4_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart5_apps_clk = {
+	.halt_reg = 0x0602c,
+	.clkr = {
+		.enable_reg = 0x0602c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart5_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart5_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_blsp1_uart6_apps_clk = {
+	.halt_reg = 0x0702c,
+	.clkr = {
+		.enable_reg = 0x0702c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_blsp1_uart6_apps_clk",
+			.parent_names = (const char *[]){
+				"blsp1_uart6_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_prng_ahb_clk = {
+	.halt_reg = 0x13004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x0b004,
+		.enable_mask = BIT(8),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_prng_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_ahb_clk = {
+	.halt_reg = 0x57024,
+	.clkr = {
+		.enable_reg = 0x57024,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qpic_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_clk = {
+	.halt_reg = 0x57020,
+	.clkr = {
+		.enable_reg = 0x57020,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_qpic_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_ahb_clk = {
+	.halt_reg = 0x75010,
+	.clkr = {
+		.enable_reg = 0x75010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie0_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_aux_clk = {
+	.halt_reg = 0x75014,
+	.clkr = {
+		.enable_reg = 0x75014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie0_aux_clk",
+			.parent_names = (const char *[]){
+				"pcie0_aux_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_m_clk = {
+	.halt_reg = 0x75008,
+	.clkr = {
+		.enable_reg = 0x75008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie0_axi_m_clk",
+			.parent_names = (const char *[]){
+				"pcie0_axi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_s_clk = {
+	.halt_reg = 0x7500c,
+	.clkr = {
+		.enable_reg = 0x7500c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie0_axi_s_clk",
+			.parent_names = (const char *[]){
+				"pcie0_axi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_pipe_clk = {
+	.halt_reg = 0x75018,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x75018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie0_pipe_clk",
+			.parent_names = (const char *[]){
+				"pcie0_pipe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_pcie0_axi_clk = {
+	.halt_reg = 0x26048,
+	.clkr = {
+		.enable_reg = 0x26048,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sys_noc_pcie0_axi_clk",
+			.parent_names = (const char *[]){
+				"pcie0_axi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_ahb_clk = {
+	.halt_reg = 0x76010,
+	.clkr = {
+		.enable_reg = 0x76010,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie1_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_aux_clk = {
+	.halt_reg = 0x76014,
+	.clkr = {
+		.enable_reg = 0x76014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie1_aux_clk",
+			.parent_names = (const char *[]){
+				"pcie1_aux_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_m_clk = {
+	.halt_reg = 0x76008,
+	.clkr = {
+		.enable_reg = 0x76008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie1_axi_m_clk",
+			.parent_names = (const char *[]){
+				"pcie1_axi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_s_clk = {
+	.halt_reg = 0x7600c,
+	.clkr = {
+		.enable_reg = 0x7600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie1_axi_s_clk",
+			.parent_names = (const char *[]){
+				"pcie1_axi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_pipe_clk = {
+	.halt_reg = 0x76018,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x76018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_pcie1_pipe_clk",
+			.parent_names = (const char *[]){
+				"pcie1_pipe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_pcie1_axi_clk = {
+	.halt_reg = 0x2604c,
+	.clkr = {
+		.enable_reg = 0x2604c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sys_noc_pcie1_axi_clk",
+			.parent_names = (const char *[]){
+				"pcie1_axi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_aux_clk = {
+	.halt_reg = 0x3e044,
+	.clkr = {
+		.enable_reg = 0x3e044,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb0_aux_clk",
+			.parent_names = (const char *[]){
+				"usb0_aux_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_usb0_axi_clk = {
+	.halt_reg = 0x26040,
+	.clkr = {
+		.enable_reg = 0x26040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sys_noc_usb0_axi_clk",
+			.parent_names = (const char *[]){
+				"usb0_master_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_master_clk = {
+	.halt_reg = 0x3e000,
+	.clkr = {
+		.enable_reg = 0x3e000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb0_master_clk",
+			.parent_names = (const char *[]){
+				"usb0_master_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_mock_utmi_clk = {
+	.halt_reg = 0x3e008,
+	.clkr = {
+		.enable_reg = 0x3e008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb0_mock_utmi_clk",
+			.parent_names = (const char *[]){
+				"usb0_mock_utmi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_phy_cfg_ahb_clk = {
+	.halt_reg = 0x3e080,
+	.clkr = {
+		.enable_reg = 0x3e080,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb0_phy_cfg_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_pipe_clk = {
+	.halt_reg = 0x3e040,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x3e040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb0_pipe_clk",
+			.parent_names = (const char *[]){
+				"usb0_pipe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_sleep_clk = {
+	.halt_reg = 0x3e004,
+	.clkr = {
+		.enable_reg = 0x3e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb0_sleep_clk",
+			.parent_names = (const char *[]){
+				"gcc_sleep_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_aux_clk = {
+	.halt_reg = 0x3f044,
+	.clkr = {
+		.enable_reg = 0x3f044,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb1_aux_clk",
+			.parent_names = (const char *[]){
+				"usb1_aux_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sys_noc_usb1_axi_clk = {
+	.halt_reg = 0x26044,
+	.clkr = {
+		.enable_reg = 0x26044,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sys_noc_usb1_axi_clk",
+			.parent_names = (const char *[]){
+				"usb1_master_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_master_clk = {
+	.halt_reg = 0x3f000,
+	.clkr = {
+		.enable_reg = 0x3f000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb1_master_clk",
+			.parent_names = (const char *[]){
+				"usb1_master_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_mock_utmi_clk = {
+	.halt_reg = 0x3f008,
+	.clkr = {
+		.enable_reg = 0x3f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb1_mock_utmi_clk",
+			.parent_names = (const char *[]){
+				"usb1_mock_utmi_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_phy_cfg_ahb_clk = {
+	.halt_reg = 0x3f080,
+	.clkr = {
+		.enable_reg = 0x3f080,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb1_phy_cfg_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_pipe_clk = {
+	.halt_reg = 0x3f040,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x3f040,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb1_pipe_clk",
+			.parent_names = (const char *[]){
+				"usb1_pipe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_sleep_clk = {
+	.halt_reg = 0x3f004,
+	.clkr = {
+		.enable_reg = 0x3f004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_usb1_sleep_clk",
+			.parent_names = (const char *[]){
+				"gcc_sleep_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0x4201c,
+	.clkr = {
+		.enable_reg = 0x4201c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x42018,
+	.clkr = {
+		.enable_reg = 0x42018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_names = (const char *[]){
+				"sdcc1_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ice_core_clk = {
+	.halt_reg = 0x5d014,
+	.clkr = {
+		.enable_reg = 0x5d014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc1_ice_core_clk",
+			.parent_names = (const char *[]){
+				"sdcc1_ice_core_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x4301c,
+	.clkr = {
+		.enable_reg = 0x4301c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x43018,
+	.clkr = {
+		.enable_reg = 0x43018,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_names = (const char *[]){
+				"sdcc2_apps_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mem_noc_nss_axi_clk = {
+	.halt_reg = 0x1d03c,
+	.clkr = {
+		.enable_reg = 0x1d03c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mem_noc_nss_axi_clk",
+			.parent_names = (const char *[]){
+				"nss_noc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ce_apb_clk = {
+	.halt_reg = 0x68174,
+	.clkr = {
+		.enable_reg = 0x68174,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_ce_apb_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ce_axi_clk = {
+	.halt_reg = 0x68170,
+	.clkr = {
+		.enable_reg = 0x68170,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_ce_axi_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_cfg_clk = {
+	.halt_reg = 0x68160,
+	.clkr = {
+		.enable_reg = 0x68160,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_cfg_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_crypto_clk = {
+	.halt_reg = 0x68164,
+	.clkr = {
+		.enable_reg = 0x68164,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_crypto_clk",
+			.parent_names = (const char *[]){
+				"nss_crypto_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_csr_clk = {
+	.halt_reg = 0x68318,
+	.clkr = {
+		.enable_reg = 0x68318,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_csr_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_edma_cfg_clk = {
+	.halt_reg = 0x6819c,
+	.clkr = {
+		.enable_reg = 0x6819c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_edma_cfg_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_edma_clk = {
+	.halt_reg = 0x68198,
+	.clkr = {
+		.enable_reg = 0x68198,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_edma_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_imem_clk = {
+	.halt_reg = 0x68178,
+	.clkr = {
+		.enable_reg = 0x68178,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_imem_clk",
+			.parent_names = (const char *[]){
+				"nss_imem_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_noc_clk = {
+	.halt_reg = 0x68168,
+	.clkr = {
+		.enable_reg = 0x68168,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_noc_clk",
+			.parent_names = (const char *[]){
+				"nss_noc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ppe_btq_clk = {
+	.halt_reg = 0x6833c,
+	.clkr = {
+		.enable_reg = 0x6833c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_ppe_btq_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ppe_cfg_clk = {
+	.halt_reg = 0x68194,
+	.clkr = {
+		.enable_reg = 0x68194,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_ppe_cfg_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ppe_clk = {
+	.halt_reg = 0x68190,
+	.clkr = {
+		.enable_reg = 0x68190,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_ppe_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ppe_ipe_clk = {
+	.halt_reg = 0x68338,
+	.clkr = {
+		.enable_reg = 0x68338,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_ppe_ipe_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ptp_ref_clk = {
+	.halt_reg = 0x6816c,
+	.clkr = {
+		.enable_reg = 0x6816c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_ptp_ref_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_cdiv_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_ce_apb_clk = {
+	.halt_reg = 0x6830c,
+	.clkr = {
+		.enable_reg = 0x6830c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_ce_apb_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_ce_axi_clk = {
+	.halt_reg = 0x68308,
+	.clkr = {
+		.enable_reg = 0x68308,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_ce_axi_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_crypto_clk = {
+	.halt_reg = 0x68314,
+	.clkr = {
+		.enable_reg = 0x68314,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_crypto_clk",
+			.parent_names = (const char *[]){
+				"nss_crypto_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_ppe_cfg_clk = {
+	.halt_reg = 0x68304,
+	.clkr = {
+		.enable_reg = 0x68304,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_ppe_cfg_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_ppe_clk = {
+	.halt_reg = 0x68300,
+	.clkr = {
+		.enable_reg = 0x68300,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_ppe_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_qosgen_ref_clk = {
+	.halt_reg = 0x68180,
+	.clkr = {
+		.enable_reg = 0x68180,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_qosgen_ref_clk",
+			.parent_names = (const char *[]){
+				"gcc_xo_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_snoc_clk = {
+	.halt_reg = 0x68188,
+	.clkr = {
+		.enable_reg = 0x68188,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_snoc_clk",
+			.parent_names = (const char *[]){
+				"system_noc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_timeout_ref_clk = {
+	.halt_reg = 0x68184,
+	.clkr = {
+		.enable_reg = 0x68184,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_timeout_ref_clk",
+			.parent_names = (const char *[]){
+				"gcc_xo_div4_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_ubi0_ahb_clk = {
+	.halt_reg = 0x68270,
+	.clkr = {
+		.enable_reg = 0x68270,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_ubi0_ahb_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_ubi1_ahb_clk = {
+	.halt_reg = 0x68274,
+	.clkr = {
+		.enable_reg = 0x68274,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nssnoc_ubi1_ahb_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi0_ahb_clk = {
+	.halt_reg = 0x6820c,
+	.clkr = {
+		.enable_reg = 0x6820c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi0_ahb_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi0_axi_clk = {
+	.halt_reg = 0x68200,
+	.clkr = {
+		.enable_reg = 0x68200,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi0_axi_clk",
+			.parent_names = (const char *[]){
+				"nss_noc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi0_nc_axi_clk = {
+	.halt_reg = 0x68204,
+	.clkr = {
+		.enable_reg = 0x68204,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi0_nc_axi_clk",
+			.parent_names = (const char *[]){
+				"nss_noc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi0_core_clk = {
+	.halt_reg = 0x68210,
+	.clkr = {
+		.enable_reg = 0x68210,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi0_core_clk",
+			.parent_names = (const char *[]){
+				"nss_ubi0_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi0_mpt_clk = {
+	.halt_reg = 0x68208,
+	.clkr = {
+		.enable_reg = 0x68208,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi0_mpt_clk",
+			.parent_names = (const char *[]){
+				"ubi_mpt_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi1_ahb_clk = {
+	.halt_reg = 0x6822c,
+	.clkr = {
+		.enable_reg = 0x6822c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi1_ahb_clk",
+			.parent_names = (const char *[]){
+				"nss_ce_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi1_axi_clk = {
+	.halt_reg = 0x68220,
+	.clkr = {
+		.enable_reg = 0x68220,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi1_axi_clk",
+			.parent_names = (const char *[]){
+				"nss_noc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi1_nc_axi_clk = {
+	.halt_reg = 0x68224,
+	.clkr = {
+		.enable_reg = 0x68224,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi1_nc_axi_clk",
+			.parent_names = (const char *[]){
+				"nss_noc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi1_core_clk = {
+	.halt_reg = 0x68230,
+	.clkr = {
+		.enable_reg = 0x68230,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi1_core_clk",
+			.parent_names = (const char *[]){
+				"nss_ubi1_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ubi1_mpt_clk = {
+	.halt_reg = 0x68228,
+	.clkr = {
+		.enable_reg = 0x68228,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_ubi1_mpt_clk",
+			.parent_names = (const char *[]){
+				"ubi_mpt_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cmn_12gpll_ahb_clk = {
+	.halt_reg = 0x56308,
+	.clkr = {
+		.enable_reg = 0x56308,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cmn_12gpll_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cmn_12gpll_sys_clk = {
+	.halt_reg = 0x5630c,
+	.clkr = {
+		.enable_reg = 0x5630c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_cmn_12gpll_sys_clk",
+			.parent_names = (const char *[]){
+				"gcc_xo_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdio_ahb_clk = {
+	.halt_reg = 0x58004,
+	.clkr = {
+		.enable_reg = 0x58004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_mdio_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy0_ahb_clk = {
+	.halt_reg = 0x56008,
+	.clkr = {
+		.enable_reg = 0x56008,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_uniphy0_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy0_sys_clk = {
+	.halt_reg = 0x5600c,
+	.clkr = {
+		.enable_reg = 0x5600c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_uniphy0_sys_clk",
+			.parent_names = (const char *[]){
+				"gcc_xo_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy1_ahb_clk = {
+	.halt_reg = 0x56108,
+	.clkr = {
+		.enable_reg = 0x56108,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_uniphy1_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy1_sys_clk = {
+	.halt_reg = 0x5610c,
+	.clkr = {
+		.enable_reg = 0x5610c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_uniphy1_sys_clk",
+			.parent_names = (const char *[]){
+				"gcc_xo_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy2_ahb_clk = {
+	.halt_reg = 0x56208,
+	.clkr = {
+		.enable_reg = 0x56208,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_uniphy2_ahb_clk",
+			.parent_names = (const char *[]){
+				"pcnoc_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy2_sys_clk = {
+	.halt_reg = 0x5620c,
+	.clkr = {
+		.enable_reg = 0x5620c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_uniphy2_sys_clk",
+			.parent_names = (const char *[]){
+				"gcc_xo_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port1_rx_clk = {
+	.halt_reg = 0x68240,
+	.clkr = {
+		.enable_reg = 0x68240,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port1_rx_clk",
+			.parent_names = (const char *[]){
+				"nss_port1_rx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port1_tx_clk = {
+	.halt_reg = 0x68244,
+	.clkr = {
+		.enable_reg = 0x68244,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port1_tx_clk",
+			.parent_names = (const char *[]){
+				"nss_port1_tx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port2_rx_clk = {
+	.halt_reg = 0x68248,
+	.clkr = {
+		.enable_reg = 0x68248,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port2_rx_clk",
+			.parent_names = (const char *[]){
+				"nss_port2_rx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port2_tx_clk = {
+	.halt_reg = 0x6824c,
+	.clkr = {
+		.enable_reg = 0x6824c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port2_tx_clk",
+			.parent_names = (const char *[]){
+				"nss_port2_tx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port3_rx_clk = {
+	.halt_reg = 0x68250,
+	.clkr = {
+		.enable_reg = 0x68250,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port3_rx_clk",
+			.parent_names = (const char *[]){
+				"nss_port3_rx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port3_tx_clk = {
+	.halt_reg = 0x68254,
+	.clkr = {
+		.enable_reg = 0x68254,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port3_tx_clk",
+			.parent_names = (const char *[]){
+				"nss_port3_tx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port4_rx_clk = {
+	.halt_reg = 0x68258,
+	.clkr = {
+		.enable_reg = 0x68258,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port4_rx_clk",
+			.parent_names = (const char *[]){
+				"nss_port4_rx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port4_tx_clk = {
+	.halt_reg = 0x6825c,
+	.clkr = {
+		.enable_reg = 0x6825c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port4_tx_clk",
+			.parent_names = (const char *[]){
+				"nss_port4_tx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_port5_rx_clk = {
+	.halt_reg = 0x68260,
+	.clkr = {
+		.enable_reg = 0x68260,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port5_rx_clk",
+			.parent_names = (const char *[]){
+				"nss_port5_rx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
 	},
 };
 
-static struct clk_rcg2 blsp1_uart3_apps_clk_src = {
-	.cmd_rcgr = 0x04034,
-	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
-	.mnd_width = 16,
-	.hid_width = 5,
-	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
-	.clkr.hw.init = &(struct clk_init_data){
-		.name = "blsp1_uart3_apps_clk_src",
-		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
-		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+static struct clk_branch gcc_nss_port5_tx_clk = {
+	.halt_reg = 0x68264,
+	.clkr = {
+		.enable_reg = 0x68264,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port5_tx_clk",
+			.parent_names = (const char *[]){
+				"nss_port5_tx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
 	},
 };
 
-static struct clk_rcg2 blsp1_uart4_apps_clk_src = {
-	.cmd_rcgr = 0x05034,
-	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
-	.mnd_width = 16,
-	.hid_width = 5,
-	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
-	.clkr.hw.init = &(struct clk_init_data){
-		.name = "blsp1_uart4_apps_clk_src",
-		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
-		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+static struct clk_branch gcc_nss_port6_rx_clk = {
+	.halt_reg = 0x68268,
+	.clkr = {
+		.enable_reg = 0x68268,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port6_rx_clk",
+			.parent_names = (const char *[]){
+				"nss_port6_rx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
 	},
 };
 
-static struct clk_rcg2 blsp1_uart5_apps_clk_src = {
-	.cmd_rcgr = 0x06034,
-	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
-	.mnd_width = 16,
-	.hid_width = 5,
-	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
-	.clkr.hw.init = &(struct clk_init_data){
-		.name = "blsp1_uart5_apps_clk_src",
-		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
-		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+static struct clk_branch gcc_nss_port6_tx_clk = {
+	.halt_reg = 0x6826c,
+	.clkr = {
+		.enable_reg = 0x6826c,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_nss_port6_tx_clk",
+			.parent_names = (const char *[]){
+				"nss_port6_tx_div_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
 	},
 };
 
-static struct clk_rcg2 blsp1_uart6_apps_clk_src = {
-	.cmd_rcgr = 0x07034,
-	.freq_tbl = ftbl_blsp1_uart_apps_clk_src,
-	.mnd_width = 16,
-	.hid_width = 5,
-	.parent_map = gcc_xo_gpll0_gpll0_out_main_div2_map,
-	.clkr.hw.init = &(struct clk_init_data){
-		.name = "blsp1_uart6_apps_clk_src",
-		.parent_names = gcc_xo_gpll0_gpll0_out_main_div2,
-		.num_parents = 3,
-		.ops = &clk_rcg2_ops,
+static struct clk_branch gcc_port1_mac_clk = {
+	.halt_reg = 0x68320,
+	.clkr = {
+		.enable_reg = 0x68320,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_port1_mac_clk",
+			.parent_names = (const char *[]){
+				"nss_ppe_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
 	},
 };
 
-static struct clk_branch gcc_blsp1_ahb_clk = {
-	.halt_reg = 0x01008,
+static struct clk_branch gcc_port2_mac_clk = {
+	.halt_reg = 0x68324,
 	.clkr = {
-		.enable_reg = 0x01008,
+		.enable_reg = 0x68324,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_ahb_clk",
+			.name = "gcc_port2_mac_clk",
 			.parent_names = (const char *[]){
-				"pcnoc_clk_src"
+				"nss_ppe_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -445,15 +3915,15 @@ static struct clk_branch gcc_blsp1_ahb_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
-	.halt_reg = 0x02008,
+static struct clk_branch gcc_port3_mac_clk = {
+	.halt_reg = 0x68328,
 	.clkr = {
-		.enable_reg = 0x02008,
+		.enable_reg = 0x68328,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup1_i2c_apps_clk",
+			.name = "gcc_port3_mac_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup1_i2c_apps_clk_src"
+				"nss_ppe_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -462,15 +3932,15 @@ static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
-	.halt_reg = 0x02004,
+static struct clk_branch gcc_port4_mac_clk = {
+	.halt_reg = 0x6832c,
 	.clkr = {
-		.enable_reg = 0x02004,
+		.enable_reg = 0x6832c,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup1_spi_apps_clk",
+			.name = "gcc_port4_mac_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup1_spi_apps_clk_src"
+				"nss_ppe_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -479,15 +3949,15 @@ static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
-	.halt_reg = 0x03010,
+static struct clk_branch gcc_port5_mac_clk = {
+	.halt_reg = 0x68330,
 	.clkr = {
-		.enable_reg = 0x03010,
+		.enable_reg = 0x68330,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup2_i2c_apps_clk",
+			.name = "gcc_port5_mac_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup2_i2c_apps_clk_src"
+				"nss_ppe_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -496,15 +3966,15 @@ static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
-	.halt_reg = 0x0300c,
+static struct clk_branch gcc_port6_mac_clk = {
+	.halt_reg = 0x68334,
 	.clkr = {
-		.enable_reg = 0x0300c,
+		.enable_reg = 0x68334,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup2_spi_apps_clk",
+			.name = "gcc_port6_mac_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup2_spi_apps_clk_src"
+				"nss_ppe_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -513,15 +3983,15 @@ static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
-	.halt_reg = 0x04010,
+static struct clk_branch gcc_uniphy0_port1_rx_clk = {
+	.halt_reg = 0x56010,
 	.clkr = {
-		.enable_reg = 0x04010,
+		.enable_reg = 0x56010,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup3_i2c_apps_clk",
+			.name = "gcc_uniphy0_port1_rx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup3_i2c_apps_clk_src"
+				"nss_port1_rx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -530,15 +4000,15 @@ static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
-	.halt_reg = 0x0400c,
+static struct clk_branch gcc_uniphy0_port1_tx_clk = {
+	.halt_reg = 0x56014,
 	.clkr = {
-		.enable_reg = 0x0400c,
+		.enable_reg = 0x56014,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup3_spi_apps_clk",
+			.name = "gcc_uniphy0_port1_tx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup3_spi_apps_clk_src"
+				"nss_port1_tx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -547,15 +4017,15 @@ static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
-	.halt_reg = 0x05010,
+static struct clk_branch gcc_uniphy0_port2_rx_clk = {
+	.halt_reg = 0x56018,
 	.clkr = {
-		.enable_reg = 0x05010,
+		.enable_reg = 0x56018,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup4_i2c_apps_clk",
+			.name = "gcc_uniphy0_port2_rx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup4_i2c_apps_clk_src"
+				"nss_port2_rx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -564,15 +4034,15 @@ static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
-	.halt_reg = 0x0500c,
+static struct clk_branch gcc_uniphy0_port2_tx_clk = {
+	.halt_reg = 0x5601c,
 	.clkr = {
-		.enable_reg = 0x0500c,
+		.enable_reg = 0x5601c,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup4_spi_apps_clk",
+			.name = "gcc_uniphy0_port2_tx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup4_spi_apps_clk_src"
+				"nss_port2_tx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -581,15 +4051,15 @@ static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = {
-	.halt_reg = 0x06010,
+static struct clk_branch gcc_uniphy0_port3_rx_clk = {
+	.halt_reg = 0x56020,
 	.clkr = {
-		.enable_reg = 0x06010,
+		.enable_reg = 0x56020,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup5_i2c_apps_clk",
+			.name = "gcc_uniphy0_port3_rx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup5_i2c_apps_clk_src"
+				"nss_port3_rx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -598,15 +4068,15 @@ static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = {
-	.halt_reg = 0x0600c,
+static struct clk_branch gcc_uniphy0_port3_tx_clk = {
+	.halt_reg = 0x56024,
 	.clkr = {
-		.enable_reg = 0x0600c,
+		.enable_reg = 0x56024,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup5_spi_apps_clk",
+			.name = "gcc_uniphy0_port3_tx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup5_spi_apps_clk_src"
+				"nss_port3_tx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -615,15 +4085,15 @@ static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = {
-	.halt_reg = 0x07010,
+static struct clk_branch gcc_uniphy0_port4_rx_clk = {
+	.halt_reg = 0x56028,
 	.clkr = {
-		.enable_reg = 0x07010,
+		.enable_reg = 0x56028,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup6_i2c_apps_clk",
+			.name = "gcc_uniphy0_port4_rx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup6_i2c_apps_clk_src"
+				"nss_port4_rx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -632,15 +4102,15 @@ static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = {
-	.halt_reg = 0x0700c,
+static struct clk_branch gcc_uniphy0_port4_tx_clk = {
+	.halt_reg = 0x5602c,
 	.clkr = {
-		.enable_reg = 0x0700c,
+		.enable_reg = 0x5602c,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_qup6_spi_apps_clk",
+			.name = "gcc_uniphy0_port4_tx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_qup6_spi_apps_clk_src"
+				"nss_port4_tx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -649,15 +4119,15 @@ static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_uart1_apps_clk = {
-	.halt_reg = 0x0203c,
+static struct clk_branch gcc_uniphy0_port5_rx_clk = {
+	.halt_reg = 0x56030,
 	.clkr = {
-		.enable_reg = 0x0203c,
+		.enable_reg = 0x56030,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_uart1_apps_clk",
+			.name = "gcc_uniphy0_port5_rx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_uart1_apps_clk_src"
+				"nss_port5_rx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -666,15 +4136,15 @@ static struct clk_branch gcc_blsp1_uart1_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_uart2_apps_clk = {
-	.halt_reg = 0x0302c,
+static struct clk_branch gcc_uniphy0_port5_tx_clk = {
+	.halt_reg = 0x56034,
 	.clkr = {
-		.enable_reg = 0x0302c,
+		.enable_reg = 0x56034,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_uart2_apps_clk",
+			.name = "gcc_uniphy0_port5_tx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_uart2_apps_clk_src"
+				"nss_port5_tx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -683,15 +4153,15 @@ static struct clk_branch gcc_blsp1_uart2_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_uart3_apps_clk = {
-	.halt_reg = 0x0402c,
+static struct clk_branch gcc_uniphy1_port5_rx_clk = {
+	.halt_reg = 0x56110,
 	.clkr = {
-		.enable_reg = 0x0402c,
+		.enable_reg = 0x56110,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_uart3_apps_clk",
+			.name = "gcc_uniphy1_port5_rx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_uart3_apps_clk_src"
+				"nss_port5_rx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -700,15 +4170,15 @@ static struct clk_branch gcc_blsp1_uart3_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_uart4_apps_clk = {
-	.halt_reg = 0x0502c,
+static struct clk_branch gcc_uniphy1_port5_tx_clk = {
+	.halt_reg = 0x56114,
 	.clkr = {
-		.enable_reg = 0x0502c,
+		.enable_reg = 0x56114,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_uart4_apps_clk",
+			.name = "gcc_uniphy1_port5_tx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_uart4_apps_clk_src"
+				"nss_port5_tx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -717,15 +4187,15 @@ static struct clk_branch gcc_blsp1_uart4_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_uart5_apps_clk = {
-	.halt_reg = 0x0602c,
+static struct clk_branch gcc_uniphy2_port6_rx_clk = {
+	.halt_reg = 0x56210,
 	.clkr = {
-		.enable_reg = 0x0602c,
+		.enable_reg = 0x56210,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_uart5_apps_clk",
+			.name = "gcc_uniphy2_port6_rx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_uart5_apps_clk_src"
+				"nss_port6_rx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -734,15 +4204,15 @@ static struct clk_branch gcc_blsp1_uart5_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_blsp1_uart6_apps_clk = {
-	.halt_reg = 0x0702c,
+static struct clk_branch gcc_uniphy2_port6_tx_clk = {
+	.halt_reg = 0x56214,
 	.clkr = {
-		.enable_reg = 0x0702c,
+		.enable_reg = 0x56214,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_blsp1_uart6_apps_clk",
+			.name = "gcc_uniphy2_port6_tx_clk",
 			.parent_names = (const char *[]){
-				"blsp1_uart6_apps_clk_src"
+				"nss_port6_tx_div_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -751,14 +4221,14 @@ static struct clk_branch gcc_blsp1_uart6_apps_clk = {
 	},
 };
 
-static struct clk_branch gcc_prng_ahb_clk = {
-	.halt_reg = 0x13004,
+static struct clk_branch gcc_crypto_ahb_clk = {
+	.halt_reg = 0x16024,
 	.halt_check = BRANCH_HALT_VOTED,
 	.clkr = {
 		.enable_reg = 0x0b004,
-		.enable_mask = BIT(8),
+		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_prng_ahb_clk",
+			.name = "gcc_crypto_ahb_clk",
 			.parent_names = (const char *[]){
 				"pcnoc_clk_src"
 			},
@@ -769,13 +4239,14 @@ static struct clk_branch gcc_prng_ahb_clk = {
 	},
 };
 
-static struct clk_branch gcc_qpic_ahb_clk = {
-	.halt_reg = 0x57024,
+static struct clk_branch gcc_crypto_axi_clk = {
+	.halt_reg = 0x16020,
+	.halt_check = BRANCH_HALT_VOTED,
 	.clkr = {
-		.enable_reg = 0x57024,
-		.enable_mask = BIT(0),
+		.enable_reg = 0x0b004,
+		.enable_mask = BIT(1),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_qpic_ahb_clk",
+			.name = "gcc_crypto_axi_clk",
 			.parent_names = (const char *[]){
 				"pcnoc_clk_src"
 			},
@@ -786,15 +4257,67 @@ static struct clk_branch gcc_qpic_ahb_clk = {
 	},
 };
 
-static struct clk_branch gcc_qpic_clk = {
-	.halt_reg = 0x57020,
+static struct clk_branch gcc_crypto_clk = {
+	.halt_reg = 0x1601c,
+	.halt_check = BRANCH_HALT_VOTED,
 	.clkr = {
-		.enable_reg = 0x57020,
+		.enable_reg = 0x0b004,
+		.enable_mask = BIT(2),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_crypto_clk",
+			.parent_names = (const char *[]){
+				"crypto_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x08000,
+	.clkr = {
+		.enable_reg = 0x08000,
 		.enable_mask = BIT(0),
 		.hw.init = &(struct clk_init_data){
-			.name = "gcc_qpic_clk",
+			.name = "gcc_gp1_clk",
 			.parent_names = (const char *[]){
-				"pcnoc_clk_src"
+				"gp1_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x09000,
+	.clkr = {
+		.enable_reg = 0x09000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp2_clk",
+			.parent_names = (const char *[]){
+				"gp2_clk_src"
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x0a000,
+	.clkr = {
+		.enable_reg = 0x0a000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data){
+			.name = "gcc_gp3_clk",
+			.parent_names = (const char *[]){
+				"gp3_clk_src"
 			},
 			.num_parents = 1,
 			.flags = CLK_SET_RATE_PARENT,
@@ -805,12 +4328,27 @@ static struct clk_branch gcc_qpic_clk = {
 
 static struct clk_hw *gcc_ipq8074_hws[] = {
 	&gpll0_out_main_div2.hw,
+	&gpll6_out_main_div2.hw,
 	&pcnoc_clk_src.hw,
+	&system_noc_clk_src.hw,
+	&gcc_xo_div4_clk_src.hw,
+	&nss_noc_clk_src.hw,
+	&nss_ppe_cdiv_clk_src.hw,
 };
 
 static struct clk_regmap *gcc_ipq8074_clks[] = {
 	[GPLL0_MAIN] = &gpll0_main.clkr,
 	[GPLL0] = &gpll0.clkr,
+	[GPLL2_MAIN] = &gpll2_main.clkr,
+	[GPLL2] = &gpll2.clkr,
+	[GPLL4_MAIN] = &gpll4_main.clkr,
+	[GPLL4] = &gpll4.clkr,
+	[GPLL6_MAIN] = &gpll6_main.clkr,
+	[GPLL6] = &gpll6.clkr,
+	[UBI32_PLL_MAIN] = &ubi32_pll_main.clkr,
+	[UBI32_PLL] = &ubi32_pll.clkr,
+	[NSS_CRYPTO_PLL_MAIN] = &nss_crypto_pll_main.clkr,
+	[NSS_CRYPTO_PLL] = &nss_crypto_pll.clkr,
 	[PCNOC_BFDCD_CLK_SRC] = &pcnoc_bfdcd_clk_src.clkr,
 	[GCC_SLEEP_CLK_SRC] = &gcc_sleep_clk_src.clkr,
 	[BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr,
@@ -831,6 +4369,63 @@ static struct clk_regmap *gcc_ipq8074_clks[] = {
 	[BLSP1_UART4_APPS_CLK_SRC] = &blsp1_uart4_apps_clk_src.clkr,
 	[BLSP1_UART5_APPS_CLK_SRC] = &blsp1_uart5_apps_clk_src.clkr,
 	[BLSP1_UART6_APPS_CLK_SRC] = &blsp1_uart6_apps_clk_src.clkr,
+	[PCIE0_AXI_CLK_SRC] = &pcie0_axi_clk_src.clkr,
+	[PCIE0_AUX_CLK_SRC] = &pcie0_aux_clk_src.clkr,
+	[PCIE0_PIPE_CLK_SRC] = &pcie0_pipe_clk_src.clkr,
+	[PCIE1_AXI_CLK_SRC] = &pcie1_axi_clk_src.clkr,
+	[PCIE1_AUX_CLK_SRC] = &pcie1_aux_clk_src.clkr,
+	[PCIE1_PIPE_CLK_SRC] = &pcie1_pipe_clk_src.clkr,
+	[SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr,
+	[SDCC1_ICE_CORE_CLK_SRC] = &sdcc1_ice_core_clk_src.clkr,
+	[SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr,
+	[USB0_MASTER_CLK_SRC] = &usb0_master_clk_src.clkr,
+	[USB0_AUX_CLK_SRC] = &usb0_aux_clk_src.clkr,
+	[USB0_MOCK_UTMI_CLK_SRC] = &usb0_mock_utmi_clk_src.clkr,
+	[USB0_PIPE_CLK_SRC] = &usb0_pipe_clk_src.clkr,
+	[USB1_MASTER_CLK_SRC] = &usb1_master_clk_src.clkr,
+	[USB1_AUX_CLK_SRC] = &usb1_aux_clk_src.clkr,
+	[USB1_MOCK_UTMI_CLK_SRC] = &usb1_mock_utmi_clk_src.clkr,
+	[USB1_PIPE_CLK_SRC] = &usb1_pipe_clk_src.clkr,
+	[GCC_XO_CLK_SRC] = &gcc_xo_clk_src.clkr,
+	[SYSTEM_NOC_BFDCD_CLK_SRC] = &system_noc_bfdcd_clk_src.clkr,
+	[NSS_CE_CLK_SRC] = &nss_ce_clk_src.clkr,
+	[NSS_NOC_BFDCD_CLK_SRC] = &nss_noc_bfdcd_clk_src.clkr,
+	[NSS_CRYPTO_CLK_SRC] = &nss_crypto_clk_src.clkr,
+	[NSS_UBI0_CLK_SRC] = &nss_ubi0_clk_src.clkr,
+	[NSS_UBI0_DIV_CLK_SRC] = &nss_ubi0_div_clk_src.clkr,
+	[NSS_UBI1_CLK_SRC] = &nss_ubi1_clk_src.clkr,
+	[NSS_UBI1_DIV_CLK_SRC] = &nss_ubi1_div_clk_src.clkr,
+	[UBI_MPT_CLK_SRC] = &ubi_mpt_clk_src.clkr,
+	[NSS_IMEM_CLK_SRC] = &nss_imem_clk_src.clkr,
+	[NSS_PPE_CLK_SRC] = &nss_ppe_clk_src.clkr,
+	[NSS_PORT1_RX_CLK_SRC] = &nss_port1_rx_clk_src.clkr,
+	[NSS_PORT1_RX_DIV_CLK_SRC] = &nss_port1_rx_div_clk_src.clkr,
+	[NSS_PORT1_TX_CLK_SRC] = &nss_port1_tx_clk_src.clkr,
+	[NSS_PORT1_TX_DIV_CLK_SRC] = &nss_port1_tx_div_clk_src.clkr,
+	[NSS_PORT2_RX_CLK_SRC] = &nss_port2_rx_clk_src.clkr,
+	[NSS_PORT2_RX_DIV_CLK_SRC] = &nss_port2_rx_div_clk_src.clkr,
+	[NSS_PORT2_TX_CLK_SRC] = &nss_port2_tx_clk_src.clkr,
+	[NSS_PORT2_TX_DIV_CLK_SRC] = &nss_port2_tx_div_clk_src.clkr,
+	[NSS_PORT3_RX_CLK_SRC] = &nss_port3_rx_clk_src.clkr,
+	[NSS_PORT3_RX_DIV_CLK_SRC] = &nss_port3_rx_div_clk_src.clkr,
+	[NSS_PORT3_TX_CLK_SRC] = &nss_port3_tx_clk_src.clkr,
+	[NSS_PORT3_TX_DIV_CLK_SRC] = &nss_port3_tx_div_clk_src.clkr,
+	[NSS_PORT4_RX_CLK_SRC] = &nss_port4_rx_clk_src.clkr,
+	[NSS_PORT4_RX_DIV_CLK_SRC] = &nss_port4_rx_div_clk_src.clkr,
+	[NSS_PORT4_TX_CLK_SRC] = &nss_port4_tx_clk_src.clkr,
+	[NSS_PORT4_TX_DIV_CLK_SRC] = &nss_port4_tx_div_clk_src.clkr,
+	[NSS_PORT5_RX_CLK_SRC] = &nss_port5_rx_clk_src.clkr,
+	[NSS_PORT5_RX_DIV_CLK_SRC] = &nss_port5_rx_div_clk_src.clkr,
+	[NSS_PORT5_TX_CLK_SRC] = &nss_port5_tx_clk_src.clkr,
+	[NSS_PORT5_TX_DIV_CLK_SRC] = &nss_port5_tx_div_clk_src.clkr,
+	[NSS_PORT6_RX_CLK_SRC] = &nss_port6_rx_clk_src.clkr,
+	[NSS_PORT6_RX_DIV_CLK_SRC] = &nss_port6_rx_div_clk_src.clkr,
+	[NSS_PORT6_TX_CLK_SRC] = &nss_port6_tx_clk_src.clkr,
+	[NSS_PORT6_TX_DIV_CLK_SRC] = &nss_port6_tx_div_clk_src.clkr,
+	[CRYPTO_CLK_SRC] = &crypto_clk_src.clkr,
+	[GP1_CLK_SRC] = &gp1_clk_src.clkr,
+	[GP2_CLK_SRC] = &gp2_clk_src.clkr,
+	[GP3_CLK_SRC] = &gp3_clk_src.clkr,
 	[GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr,
 	[GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr,
 	[GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr,
@@ -853,6 +4448,119 @@ static struct clk_regmap *gcc_ipq8074_clks[] = {
 	[GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr,
 	[GCC_QPIC_AHB_CLK] = &gcc_qpic_ahb_clk.clkr,
 	[GCC_QPIC_CLK] = &gcc_qpic_clk.clkr,
+	[GCC_PCIE0_AHB_CLK] = &gcc_pcie0_ahb_clk.clkr,
+	[GCC_PCIE0_AUX_CLK] = &gcc_pcie0_aux_clk.clkr,
+	[GCC_PCIE0_AXI_M_CLK] = &gcc_pcie0_axi_m_clk.clkr,
+	[GCC_PCIE0_AXI_S_CLK] = &gcc_pcie0_axi_s_clk.clkr,
+	[GCC_PCIE0_PIPE_CLK] = &gcc_pcie0_pipe_clk.clkr,
+	[GCC_SYS_NOC_PCIE0_AXI_CLK] = &gcc_sys_noc_pcie0_axi_clk.clkr,
+	[GCC_PCIE1_AHB_CLK] = &gcc_pcie1_ahb_clk.clkr,
+	[GCC_PCIE1_AUX_CLK] = &gcc_pcie1_aux_clk.clkr,
+	[GCC_PCIE1_AXI_M_CLK] = &gcc_pcie1_axi_m_clk.clkr,
+	[GCC_PCIE1_AXI_S_CLK] = &gcc_pcie1_axi_s_clk.clkr,
+	[GCC_PCIE1_PIPE_CLK] = &gcc_pcie1_pipe_clk.clkr,
+	[GCC_SYS_NOC_PCIE1_AXI_CLK] = &gcc_sys_noc_pcie1_axi_clk.clkr,
+	[GCC_USB0_AUX_CLK] = &gcc_usb0_aux_clk.clkr,
+	[GCC_SYS_NOC_USB0_AXI_CLK] = &gcc_sys_noc_usb0_axi_clk.clkr,
+	[GCC_USB0_MASTER_CLK] = &gcc_usb0_master_clk.clkr,
+	[GCC_USB0_MOCK_UTMI_CLK] = &gcc_usb0_mock_utmi_clk.clkr,
+	[GCC_USB0_PHY_CFG_AHB_CLK] = &gcc_usb0_phy_cfg_ahb_clk.clkr,
+	[GCC_USB0_PIPE_CLK] = &gcc_usb0_pipe_clk.clkr,
+	[GCC_USB0_SLEEP_CLK] = &gcc_usb0_sleep_clk.clkr,
+	[GCC_USB1_AUX_CLK] = &gcc_usb1_aux_clk.clkr,
+	[GCC_SYS_NOC_USB1_AXI_CLK] = &gcc_sys_noc_usb1_axi_clk.clkr,
+	[GCC_USB1_MASTER_CLK] = &gcc_usb1_master_clk.clkr,
+	[GCC_USB1_MOCK_UTMI_CLK] = &gcc_usb1_mock_utmi_clk.clkr,
+	[GCC_USB1_PHY_CFG_AHB_CLK] = &gcc_usb1_phy_cfg_ahb_clk.clkr,
+	[GCC_USB1_PIPE_CLK] = &gcc_usb1_pipe_clk.clkr,
+	[GCC_USB1_SLEEP_CLK] = &gcc_usb1_sleep_clk.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_MEM_NOC_NSS_AXI_CLK] = &gcc_mem_noc_nss_axi_clk.clkr,
+	[GCC_NSS_CE_APB_CLK] = &gcc_nss_ce_apb_clk.clkr,
+	[GCC_NSS_CE_AXI_CLK] = &gcc_nss_ce_axi_clk.clkr,
+	[GCC_NSS_CFG_CLK] = &gcc_nss_cfg_clk.clkr,
+	[GCC_NSS_CRYPTO_CLK] = &gcc_nss_crypto_clk.clkr,
+	[GCC_NSS_CSR_CLK] = &gcc_nss_csr_clk.clkr,
+	[GCC_NSS_EDMA_CFG_CLK] = &gcc_nss_edma_cfg_clk.clkr,
+	[GCC_NSS_EDMA_CLK] = &gcc_nss_edma_clk.clkr,
+	[GCC_NSS_IMEM_CLK] = &gcc_nss_imem_clk.clkr,
+	[GCC_NSS_NOC_CLK] = &gcc_nss_noc_clk.clkr,
+	[GCC_NSS_PPE_BTQ_CLK] = &gcc_nss_ppe_btq_clk.clkr,
+	[GCC_NSS_PPE_CFG_CLK] = &gcc_nss_ppe_cfg_clk.clkr,
+	[GCC_NSS_PPE_CLK] = &gcc_nss_ppe_clk.clkr,
+	[GCC_NSS_PPE_IPE_CLK] = &gcc_nss_ppe_ipe_clk.clkr,
+	[GCC_NSS_PTP_REF_CLK] = &gcc_nss_ptp_ref_clk.clkr,
+	[GCC_NSSNOC_CE_APB_CLK] = &gcc_nssnoc_ce_apb_clk.clkr,
+	[GCC_NSSNOC_CE_AXI_CLK] = &gcc_nssnoc_ce_axi_clk.clkr,
+	[GCC_NSSNOC_CRYPTO_CLK] = &gcc_nssnoc_crypto_clk.clkr,
+	[GCC_NSSNOC_PPE_CFG_CLK] = &gcc_nssnoc_ppe_cfg_clk.clkr,
+	[GCC_NSSNOC_PPE_CLK] = &gcc_nssnoc_ppe_clk.clkr,
+	[GCC_NSSNOC_QOSGEN_REF_CLK] = &gcc_nssnoc_qosgen_ref_clk.clkr,
+	[GCC_NSSNOC_SNOC_CLK] = &gcc_nssnoc_snoc_clk.clkr,
+	[GCC_NSSNOC_TIMEOUT_REF_CLK] = &gcc_nssnoc_timeout_ref_clk.clkr,
+	[GCC_NSSNOC_UBI0_AHB_CLK] = &gcc_nssnoc_ubi0_ahb_clk.clkr,
+	[GCC_NSSNOC_UBI1_AHB_CLK] = &gcc_nssnoc_ubi1_ahb_clk.clkr,
+	[GCC_UBI0_AHB_CLK] = &gcc_ubi0_ahb_clk.clkr,
+	[GCC_UBI0_AXI_CLK] = &gcc_ubi0_axi_clk.clkr,
+	[GCC_UBI0_NC_AXI_CLK] = &gcc_ubi0_nc_axi_clk.clkr,
+	[GCC_UBI0_CORE_CLK] = &gcc_ubi0_core_clk.clkr,
+	[GCC_UBI0_MPT_CLK] = &gcc_ubi0_mpt_clk.clkr,
+	[GCC_UBI1_AHB_CLK] = &gcc_ubi1_ahb_clk.clkr,
+	[GCC_UBI1_AXI_CLK] = &gcc_ubi1_axi_clk.clkr,
+	[GCC_UBI1_NC_AXI_CLK] = &gcc_ubi1_nc_axi_clk.clkr,
+	[GCC_UBI1_CORE_CLK] = &gcc_ubi1_core_clk.clkr,
+	[GCC_UBI1_MPT_CLK] = &gcc_ubi1_mpt_clk.clkr,
+	[GCC_CMN_12GPLL_AHB_CLK] = &gcc_cmn_12gpll_ahb_clk.clkr,
+	[GCC_CMN_12GPLL_SYS_CLK] = &gcc_cmn_12gpll_sys_clk.clkr,
+	[GCC_MDIO_AHB_CLK] = &gcc_mdio_ahb_clk.clkr,
+	[GCC_UNIPHY0_AHB_CLK] = &gcc_uniphy0_ahb_clk.clkr,
+	[GCC_UNIPHY0_SYS_CLK] = &gcc_uniphy0_sys_clk.clkr,
+	[GCC_UNIPHY1_AHB_CLK] = &gcc_uniphy1_ahb_clk.clkr,
+	[GCC_UNIPHY1_SYS_CLK] = &gcc_uniphy1_sys_clk.clkr,
+	[GCC_UNIPHY2_AHB_CLK] = &gcc_uniphy2_ahb_clk.clkr,
+	[GCC_UNIPHY2_SYS_CLK] = &gcc_uniphy2_sys_clk.clkr,
+	[GCC_NSS_PORT1_RX_CLK] = &gcc_nss_port1_rx_clk.clkr,
+	[GCC_NSS_PORT1_TX_CLK] = &gcc_nss_port1_tx_clk.clkr,
+	[GCC_NSS_PORT2_RX_CLK] = &gcc_nss_port2_rx_clk.clkr,
+	[GCC_NSS_PORT2_TX_CLK] = &gcc_nss_port2_tx_clk.clkr,
+	[GCC_NSS_PORT3_RX_CLK] = &gcc_nss_port3_rx_clk.clkr,
+	[GCC_NSS_PORT3_TX_CLK] = &gcc_nss_port3_tx_clk.clkr,
+	[GCC_NSS_PORT4_RX_CLK] = &gcc_nss_port4_rx_clk.clkr,
+	[GCC_NSS_PORT4_TX_CLK] = &gcc_nss_port4_tx_clk.clkr,
+	[GCC_NSS_PORT5_RX_CLK] = &gcc_nss_port5_rx_clk.clkr,
+	[GCC_NSS_PORT5_TX_CLK] = &gcc_nss_port5_tx_clk.clkr,
+	[GCC_NSS_PORT6_RX_CLK] = &gcc_nss_port6_rx_clk.clkr,
+	[GCC_NSS_PORT6_TX_CLK] = &gcc_nss_port6_tx_clk.clkr,
+	[GCC_PORT1_MAC_CLK] = &gcc_port1_mac_clk.clkr,
+	[GCC_PORT2_MAC_CLK] = &gcc_port2_mac_clk.clkr,
+	[GCC_PORT3_MAC_CLK] = &gcc_port3_mac_clk.clkr,
+	[GCC_PORT4_MAC_CLK] = &gcc_port4_mac_clk.clkr,
+	[GCC_PORT5_MAC_CLK] = &gcc_port5_mac_clk.clkr,
+	[GCC_PORT6_MAC_CLK] = &gcc_port6_mac_clk.clkr,
+	[GCC_UNIPHY0_PORT1_RX_CLK] = &gcc_uniphy0_port1_rx_clk.clkr,
+	[GCC_UNIPHY0_PORT1_TX_CLK] = &gcc_uniphy0_port1_tx_clk.clkr,
+	[GCC_UNIPHY0_PORT2_RX_CLK] = &gcc_uniphy0_port2_rx_clk.clkr,
+	[GCC_UNIPHY0_PORT2_TX_CLK] = &gcc_uniphy0_port2_tx_clk.clkr,
+	[GCC_UNIPHY0_PORT3_RX_CLK] = &gcc_uniphy0_port3_rx_clk.clkr,
+	[GCC_UNIPHY0_PORT3_TX_CLK] = &gcc_uniphy0_port3_tx_clk.clkr,
+	[GCC_UNIPHY0_PORT4_RX_CLK] = &gcc_uniphy0_port4_rx_clk.clkr,
+	[GCC_UNIPHY0_PORT4_TX_CLK] = &gcc_uniphy0_port4_tx_clk.clkr,
+	[GCC_UNIPHY0_PORT5_RX_CLK] = &gcc_uniphy0_port5_rx_clk.clkr,
+	[GCC_UNIPHY0_PORT5_TX_CLK] = &gcc_uniphy0_port5_tx_clk.clkr,
+	[GCC_UNIPHY1_PORT5_RX_CLK] = &gcc_uniphy1_port5_rx_clk.clkr,
+	[GCC_UNIPHY1_PORT5_TX_CLK] = &gcc_uniphy1_port5_tx_clk.clkr,
+	[GCC_UNIPHY2_PORT6_RX_CLK] = &gcc_uniphy2_port6_rx_clk.clkr,
+	[GCC_UNIPHY2_PORT6_TX_CLK] = &gcc_uniphy2_port6_tx_clk.clkr,
+	[GCC_CRYPTO_AHB_CLK] = &gcc_crypto_ahb_clk.clkr,
+	[GCC_CRYPTO_AXI_CLK] = &gcc_crypto_axi_clk.clkr,
+	[GCC_CRYPTO_CLK] = &gcc_crypto_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
 };
 
 static const struct qcom_reset_map gcc_ipq8074_resets[] = {
@@ -945,6 +4653,48 @@ static const struct qcom_reset_map gcc_ipq8074_resets[] = {
 	[GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR] = { 0x78000, 0 },
 	[GCC_APC1_VOLTAGE_DROOP_DETECTOR_BCR] = { 0x79000, 0 },
 	[GCC_SMMU_CATS_BCR] = { 0x7c000, 0 },
+	[GCC_UBI0_AXI_ARES] = { 0x68010, 0 },
+	[GCC_UBI0_AHB_ARES] = { 0x68010, 1 },
+	[GCC_UBI0_NC_AXI_ARES] = { 0x68010, 2 },
+	[GCC_UBI0_DBG_ARES] = { 0x68010, 3 },
+	[GCC_UBI0_CORE_CLAMP_ENABLE] = { 0x68010, 4 },
+	[GCC_UBI0_CLKRST_CLAMP_ENABLE] = { 0x68010, 5 },
+	[GCC_UBI1_AXI_ARES] = { 0x68010, 8 },
+	[GCC_UBI1_AHB_ARES] = { 0x68010, 9 },
+	[GCC_UBI1_NC_AXI_ARES] = { 0x68010, 10 },
+	[GCC_UBI1_DBG_ARES] = { 0x68010, 11 },
+	[GCC_UBI1_CORE_CLAMP_ENABLE] = { 0x68010, 12 },
+	[GCC_UBI1_CLKRST_CLAMP_ENABLE] = { 0x68010, 13 },
+	[GCC_NSS_CFG_ARES] = { 0x68010, 16 },
+	[GCC_NSS_IMEM_ARES] = { 0x68010, 17 },
+	[GCC_NSS_NOC_ARES] = { 0x68010, 18 },
+	[GCC_NSS_CRYPTO_ARES] = { 0x68010, 19 },
+	[GCC_NSS_CSR_ARES] = { 0x68010, 20 },
+	[GCC_NSS_CE_APB_ARES] = { 0x68010, 21 },
+	[GCC_NSS_CE_AXI_ARES] = { 0x68010, 22 },
+	[GCC_NSSNOC_CE_APB_ARES] = { 0x68010, 23 },
+	[GCC_NSSNOC_CE_AXI_ARES] = { 0x68010, 24 },
+	[GCC_NSSNOC_UBI0_AHB_ARES] = { 0x68010, 25 },
+	[GCC_NSSNOC_UBI1_AHB_ARES] = { 0x68010, 26 },
+	[GCC_NSSNOC_SNOC_ARES] = { 0x68010, 27 },
+	[GCC_NSSNOC_CRYPTO_ARES] = { 0x68010, 28 },
+	[GCC_NSSNOC_ATB_ARES] = { 0x68010, 29 },
+	[GCC_NSSNOC_QOSGEN_REF_ARES] = { 0x68010, 30 },
+	[GCC_NSSNOC_TIMEOUT_REF_ARES] = { 0x68010, 31 },
+	[GCC_PCIE0_PIPE_ARES] = { 0x75040, 0 },
+	[GCC_PCIE0_SLEEP_ARES] = { 0x75040, 1 },
+	[GCC_PCIE0_CORE_STICKY_ARES] = { 0x75040, 2 },
+	[GCC_PCIE0_AXI_MASTER_ARES] = { 0x75040, 3 },
+	[GCC_PCIE0_AXI_SLAVE_ARES] = { 0x75040, 4 },
+	[GCC_PCIE0_AHB_ARES] = { 0x75040, 5 },
+	[GCC_PCIE0_AXI_MASTER_STICKY_ARES] = { 0x75040, 6 },
+	[GCC_PCIE1_PIPE_ARES] = { 0x76040, 0 },
+	[GCC_PCIE1_SLEEP_ARES] = { 0x76040, 1 },
+	[GCC_PCIE1_CORE_STICKY_ARES] = { 0x76040, 2 },
+	[GCC_PCIE1_AXI_MASTER_ARES] = { 0x76040, 3 },
+	[GCC_PCIE1_AXI_SLAVE_ARES] = { 0x76040, 4 },
+	[GCC_PCIE1_AHB_ARES] = { 0x76040, 5 },
+	[GCC_PCIE1_AXI_MASTER_STICKY_ARES] = { 0x76040, 6 },
 };
 
 static const struct of_device_id gcc_ipq8074_match_table[] = {
diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c
index 3410ee68d4bc..d6c7f50ba86a 100644
--- a/drivers/clk/qcom/gcc-msm8916.c
+++ b/drivers/clk/qcom/gcc-msm8916.c
@@ -1259,20 +1259,25 @@ static struct clk_branch gcc_ultaudio_ahbfabric_ixfabric_lpm_clk = {
 };
 
 static const struct freq_tbl ftbl_gcc_ultaudio_lpaif_i2s_clk[] = {
+	F(128000, P_XO, 10, 1, 15),
 	F(256000, P_XO, 5, 1, 15),
+	F(384000, P_XO, 5, 1, 10),
 	F(512000, P_XO, 5, 2, 15),
+	F(576000, P_XO, 5, 3, 20),
 	F(705600, P_GPLL1, 16, 1, 80),
 	F(768000, P_XO, 5, 1, 5),
 	F(800000, P_XO, 5, 5, 24),
-	F(1024000, P_GPLL1, 14, 1, 63),
+	F(1024000, P_XO, 5, 4, 15),
 	F(1152000, P_XO, 1, 3, 50),
 	F(1411200, P_GPLL1, 16, 1, 40),
 	F(1536000, P_XO, 1, 2, 25),
 	F(1600000, P_XO, 12, 0, 0),
-	F(2048000, P_GPLL1, 9, 1, 49),
+	F(1728000, P_XO, 5, 9, 20),
+	F(2048000, P_XO, 5, 8, 15),
+	F(2304000, P_XO, 5, 3, 5),
 	F(2400000, P_XO, 8, 0, 0),
 	F(2822400, P_GPLL1, 16, 1, 20),
-	F(3072000, P_GPLL1, 14, 1, 21),
+	F(3072000, P_XO, 5, 4, 5),
 	F(4096000, P_GPLL1, 9, 2, 49),
 	F(4800000, P_XO, 4, 0, 0),
 	F(5644800, P_GPLL1, 16, 1, 10),
@@ -1431,6 +1436,7 @@ static struct clk_branch gcc_ultaudio_stc_xo_clk = {
 
 static const struct freq_tbl ftbl_codec_clk[] = {
 	F(9600000, P_XO, 2, 0, 0),
+	F(12288000, P_XO, 1, 16, 25),
 	F(19200000, P_XO, 1, 0, 0),
 	F(11289600, P_EXT_MCLK, 1, 0, 0),
 	{ }
@@ -1438,6 +1444,7 @@ static const struct freq_tbl ftbl_codec_clk[] = {
 
 static struct clk_rcg2 codec_digcodec_clk_src = {
 	.cmd_rcgr = 0x1c09c,
+	.mnd_width = 8,
 	.hid_width = 5,
 	.parent_map = gcc_xo_gpll1_emclk_sleep_map,
 	.freq_tbl = ftbl_codec_clk,
diff --git a/drivers/clk/qcom/gcc-msm8994.c b/drivers/clk/qcom/gcc-msm8994.c
index 7983288d9141..1e38efc37180 100644
--- a/drivers/clk/qcom/gcc-msm8994.c
+++ b/drivers/clk/qcom/gcc-msm8994.c
@@ -73,6 +73,7 @@ static struct clk_fixed_factor xo = {
 
 static struct clk_alpha_pll gpll0_early = {
 	.offset = 0x00000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr = {
 		.enable_reg = 0x1480,
 		.enable_mask = BIT(0),
@@ -88,6 +89,7 @@ static struct clk_alpha_pll gpll0_early = {
 
 static struct clk_alpha_pll_postdiv gpll0 = {
 	.offset = 0x00000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr.hw.init = &(struct clk_init_data)
 	{
 		.name = "gpll0",
@@ -99,6 +101,7 @@ static struct clk_alpha_pll_postdiv gpll0 = {
 
 static struct clk_alpha_pll gpll4_early = {
 	.offset = 0x1dc0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr = {
 		.enable_reg = 0x1480,
 		.enable_mask = BIT(4),
@@ -114,6 +117,7 @@ static struct clk_alpha_pll gpll4_early = {
 
 static struct clk_alpha_pll_postdiv gpll4 = {
 	.offset = 0x1dc0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr.hw.init = &(struct clk_init_data)
 	{
 		.name = "gpll4",
diff --git a/drivers/clk/qcom/gcc-msm8996.c b/drivers/clk/qcom/gcc-msm8996.c
index 7ddec886fcd3..5d7451209206 100644
--- a/drivers/clk/qcom/gcc-msm8996.c
+++ b/drivers/clk/qcom/gcc-msm8996.c
@@ -227,6 +227,7 @@ static struct clk_fixed_factor xo = {
 
 static struct clk_alpha_pll gpll0_early = {
 	.offset = 0x00000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr = {
 		.enable_reg = 0x52000,
 		.enable_mask = BIT(0),
@@ -252,6 +253,7 @@ static struct clk_fixed_factor gpll0_early_div = {
 
 static struct clk_alpha_pll_postdiv gpll0 = {
 	.offset = 0x00000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "gpll0",
 		.parent_names = (const char *[]){ "gpll0_early" },
@@ -262,6 +264,7 @@ static struct clk_alpha_pll_postdiv gpll0 = {
 
 static struct clk_alpha_pll gpll4_early = {
 	.offset = 0x77000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr = {
 		.enable_reg = 0x52000,
 		.enable_mask = BIT(4),
@@ -276,6 +279,7 @@ static struct clk_alpha_pll gpll4_early = {
 
 static struct clk_alpha_pll_postdiv gpll4 = {
 	.offset = 0x77000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "gpll4",
 		.parent_names = (const char *[]){ "gpll4_early" },
diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c
index 352394d8fd8c..66a2fa4ec93c 100644
--- a/drivers/clk/qcom/mmcc-msm8996.c
+++ b/drivers/clk/qcom/mmcc-msm8996.c
@@ -267,6 +267,7 @@ static struct pll_vco mmpll_t_vco[] = {
 
 static struct clk_alpha_pll mmpll0_early = {
 	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_p_vco,
 	.num_vco = ARRAY_SIZE(mmpll_p_vco),
 	.clkr = {
@@ -283,6 +284,7 @@ static struct clk_alpha_pll mmpll0_early = {
 
 static struct clk_alpha_pll_postdiv mmpll0 = {
 	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll0",
@@ -295,6 +297,7 @@ static struct clk_alpha_pll_postdiv mmpll0 = {
 
 static struct clk_alpha_pll mmpll1_early = {
 	.offset = 0x30,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_p_vco,
 	.num_vco = ARRAY_SIZE(mmpll_p_vco),
 	.clkr = {
@@ -311,6 +314,7 @@ static struct clk_alpha_pll mmpll1_early = {
 
 static struct clk_alpha_pll_postdiv mmpll1 = {
 	.offset = 0x30,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll1",
@@ -323,6 +327,7 @@ static struct clk_alpha_pll_postdiv mmpll1 = {
 
 static struct clk_alpha_pll mmpll2_early = {
 	.offset = 0x4100,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_gfx_vco,
 	.num_vco = ARRAY_SIZE(mmpll_gfx_vco),
 	.clkr.hw.init = &(struct clk_init_data){
@@ -335,6 +340,7 @@ static struct clk_alpha_pll mmpll2_early = {
 
 static struct clk_alpha_pll_postdiv mmpll2 = {
 	.offset = 0x4100,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll2",
@@ -347,6 +353,7 @@ static struct clk_alpha_pll_postdiv mmpll2 = {
 
 static struct clk_alpha_pll mmpll3_early = {
 	.offset = 0x60,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_p_vco,
 	.num_vco = ARRAY_SIZE(mmpll_p_vco),
 	.clkr.hw.init = &(struct clk_init_data){
@@ -359,6 +366,7 @@ static struct clk_alpha_pll mmpll3_early = {
 
 static struct clk_alpha_pll_postdiv mmpll3 = {
 	.offset = 0x60,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll3",
@@ -371,6 +379,7 @@ static struct clk_alpha_pll_postdiv mmpll3 = {
 
 static struct clk_alpha_pll mmpll4_early = {
 	.offset = 0x90,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_t_vco,
 	.num_vco = ARRAY_SIZE(mmpll_t_vco),
 	.clkr.hw.init = &(struct clk_init_data){
@@ -383,6 +392,7 @@ static struct clk_alpha_pll mmpll4_early = {
 
 static struct clk_alpha_pll_postdiv mmpll4 = {
 	.offset = 0x90,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 2,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll4",
@@ -395,6 +405,7 @@ static struct clk_alpha_pll_postdiv mmpll4 = {
 
 static struct clk_alpha_pll mmpll5_early = {
 	.offset = 0xc0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_p_vco,
 	.num_vco = ARRAY_SIZE(mmpll_p_vco),
 	.clkr.hw.init = &(struct clk_init_data){
@@ -407,6 +418,7 @@ static struct clk_alpha_pll mmpll5_early = {
 
 static struct clk_alpha_pll_postdiv mmpll5 = {
 	.offset = 0xc0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll5",
@@ -419,6 +431,7 @@ static struct clk_alpha_pll_postdiv mmpll5 = {
 
 static struct clk_alpha_pll mmpll8_early = {
 	.offset = 0x4130,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_gfx_vco,
 	.num_vco = ARRAY_SIZE(mmpll_gfx_vco),
 	.clkr.hw.init = &(struct clk_init_data){
@@ -431,6 +444,7 @@ static struct clk_alpha_pll mmpll8_early = {
 
 static struct clk_alpha_pll_postdiv mmpll8 = {
 	.offset = 0x4130,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 4,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll8",
@@ -443,6 +457,7 @@ static struct clk_alpha_pll_postdiv mmpll8 = {
 
 static struct clk_alpha_pll mmpll9_early = {
 	.offset = 0x4200,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.vco_table = mmpll_t_vco,
 	.num_vco = ARRAY_SIZE(mmpll_t_vco),
 	.clkr.hw.init = &(struct clk_init_data){
@@ -455,6 +470,7 @@ static struct clk_alpha_pll mmpll9_early = {
 
 static struct clk_alpha_pll_postdiv mmpll9 = {
 	.offset = 0x4200,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT],
 	.width = 2,
 	.clkr.hw.init = &(struct clk_init_data){
 		.name = "mmpll9",
diff --git a/drivers/clk/renesas/clk-mstp.c b/drivers/clk/renesas/clk-mstp.c
index c944cc421e30..858c24d4da8f 100644
--- a/drivers/clk/renesas/clk-mstp.c
+++ b/drivers/clk/renesas/clk-mstp.c
@@ -341,7 +341,7 @@ void __init cpg_mstp_add_clk_domain(struct device_node *np)
 		return;
 
 	pd->name = np->name;
-	pd->flags = GENPD_FLAG_PM_CLK;
+	pd->flags = GENPD_FLAG_PM_CLK | GENPD_FLAG_ACTIVE_WAKEUP;
 	pd->attach_dev = cpg_mstp_attach_dev;
 	pd->detach_dev = cpg_mstp_detach_dev;
 	pm_genpd_init(pd, &pm_domain_always_on_gov, false);
diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c
index b3767472088a..41e29734126b 100644
--- a/drivers/clk/renesas/r8a7796-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c
@@ -115,6 +115,7 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = {
 };
 
 static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
+	DEF_MOD("fdp1-0",		 119,	R8A7796_CLK_S0D1),
 	DEF_MOD("scif5",		 202,	R8A7796_CLK_S3D4),
 	DEF_MOD("scif4",		 203,	R8A7796_CLK_S3D4),
 	DEF_MOD("scif3",		 204,	R8A7796_CLK_S3D4),
diff --git a/drivers/clk/renesas/r8a77970-cpg-mssr.c b/drivers/clk/renesas/r8a77970-cpg-mssr.c
index 72f98527473a..f55842917e8d 100644
--- a/drivers/clk/renesas/r8a77970-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77970-cpg-mssr.c
@@ -105,6 +105,7 @@ static const struct mssr_mod_clk r8a77970_mod_clks[] __initconst = {
 	DEF_MOD("vspd0",		 623,	R8A77970_CLK_S2D1),
 	DEF_MOD("csi40",		 716,	R8A77970_CLK_CSI0),
 	DEF_MOD("du0",			 724,	R8A77970_CLK_S2D1),
+	DEF_MOD("lvds",			 727,	R8A77970_CLK_S2D1),
 	DEF_MOD("vin3",			 808,	R8A77970_CLK_S2D1),
 	DEF_MOD("vin2",			 809,	R8A77970_CLK_S2D1),
 	DEF_MOD("vin1",			 810,	R8A77970_CLK_S2D1),
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index e3d03ffea4bc..e3cc72c81311 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -513,7 +513,7 @@ static int __init cpg_mssr_add_clk_domain(struct device *dev,
 
 	genpd = &pd->genpd;
 	genpd->name = np->name;
-	genpd->flags = GENPD_FLAG_PM_CLK;
+	genpd->flags = GENPD_FLAG_PM_CLK | GENPD_FLAG_ACTIVE_WAKEUP;
 	genpd->attach_dev = cpg_mssr_attach_dev;
 	genpd->detach_dev = cpg_mssr_detach_dev;
 	pm_genpd_init(genpd, &pm_domain_always_on_gov, false);
diff --git a/drivers/clk/spear/clk-frac-synth.c b/drivers/clk/spear/clk-frac-synth.c
index 229c96daece6..f5be02205ac6 100644
--- a/drivers/clk/spear/clk-frac-synth.c
+++ b/drivers/clk/spear/clk-frac-synth.c
@@ -131,7 +131,7 @@ struct clk *clk_register_frac(const char *name, const char *parent_name,
 	struct clk *clk;
 
 	if (!name || !parent_name || !reg || !rtbl || !rtbl_cnt) {
-		pr_err("Invalid arguments passed");
+		pr_err("Invalid arguments passed\n");
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/clk/spear/clk-gpt-synth.c b/drivers/clk/spear/clk-gpt-synth.c
index 28262f422562..6ed406d943ba 100644
--- a/drivers/clk/spear/clk-gpt-synth.c
+++ b/drivers/clk/spear/clk-gpt-synth.c
@@ -120,7 +120,7 @@ struct clk *clk_register_gpt(const char *name, const char *parent_name, unsigned
 	struct clk *clk;
 
 	if (!name || !parent_name || !reg || !rtbl || !rtbl_cnt) {
-		pr_err("Invalid arguments passed");
+		pr_err("Invalid arguments passed\n");
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/clk/sprd/Kconfig b/drivers/clk/sprd/Kconfig
new file mode 100644
index 000000000000..87892471eb96
--- /dev/null
+++ b/drivers/clk/sprd/Kconfig
@@ -0,0 +1,14 @@
+config SPRD_COMMON_CLK
+	tristate "Clock support for Spreadtrum SoCs"
+	depends on ARCH_SPRD || COMPILE_TEST
+	default ARCH_SPRD
+
+if SPRD_COMMON_CLK
+
+# SoC Drivers
+
+config SPRD_SC9860_CLK
+	tristate "Support for the Spreadtrum SC9860 clocks"
+	depends on (ARM64 && ARCH_SPRD) || COMPILE_TEST
+	default ARM64 && ARCH_SPRD
+endif
diff --git a/drivers/clk/sprd/Makefile b/drivers/clk/sprd/Makefile
new file mode 100644
index 000000000000..b0d81e541ebd
--- /dev/null
+++ b/drivers/clk/sprd/Makefile
@@ -0,0 +1,11 @@
+obj-$(CONFIG_SPRD_COMMON_CLK)	+= clk-sprd.o
+
+clk-sprd-y	+= common.o
+clk-sprd-y	+= gate.o
+clk-sprd-y	+= mux.o
+clk-sprd-y	+= div.o
+clk-sprd-y	+= composite.o
+clk-sprd-y	+= pll.o
+
+## SoC support
+obj-$(CONFIG_SPRD_SC9860_CLK)	+= sc9860-clk.o
diff --git a/drivers/clk/sprd/common.c b/drivers/clk/sprd/common.c
new file mode 100644
index 000000000000..e038b0447206
--- /dev/null
+++ b/drivers/clk/sprd/common.c
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum clock infrastructure
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+
+#include "common.h"
+
+static const struct regmap_config sprdclk_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0xffff,
+	.fast_io	= true,
+};
+
+static void sprd_clk_set_regmap(const struct sprd_clk_desc *desc,
+			 struct regmap *regmap)
+{
+	int i;
+	struct sprd_clk_common *cclk;
+
+	for (i = 0; i < desc->num_clk_clks; i++) {
+		cclk = desc->clk_clks[i];
+		if (!cclk)
+			continue;
+
+		cclk->regmap = regmap;
+	}
+}
+
+int sprd_clk_regmap_init(struct platform_device *pdev,
+			 const struct sprd_clk_desc *desc)
+{
+	void __iomem *base;
+	struct device_node *node = pdev->dev.of_node;
+	struct regmap *regmap;
+
+	if (of_find_property(node, "sprd,syscon", NULL)) {
+		regmap = syscon_regmap_lookup_by_phandle(node, "sprd,syscon");
+		if (IS_ERR_OR_NULL(regmap)) {
+			pr_err("%s: failed to get syscon regmap\n", __func__);
+			return PTR_ERR(regmap);
+		}
+	} else {
+		base = of_iomap(node, 0);
+		regmap = devm_regmap_init_mmio(&pdev->dev, base,
+					       &sprdclk_regmap_config);
+		if (IS_ERR_OR_NULL(regmap)) {
+			pr_err("failed to init regmap\n");
+			return PTR_ERR(regmap);
+		}
+	}
+
+	sprd_clk_set_regmap(desc, regmap);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(sprd_clk_regmap_init);
+
+int sprd_clk_probe(struct device *dev, struct clk_hw_onecell_data *clkhw)
+{
+	int i, ret;
+	struct clk_hw *hw;
+
+	for (i = 0; i < clkhw->num; i++) {
+
+		hw = clkhw->hws[i];
+
+		if (!hw)
+			continue;
+
+		ret = devm_clk_hw_register(dev, hw);
+		if (ret) {
+			dev_err(dev, "Couldn't register clock %d - %s\n",
+				i, hw->init->name);
+			return ret;
+		}
+	}
+
+	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clkhw);
+	if (ret)
+		dev_err(dev, "Failed to add clock provider\n");
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(sprd_clk_probe);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/sprd/common.h b/drivers/clk/sprd/common.h
new file mode 100644
index 000000000000..abd9ff5ef448
--- /dev/null
+++ b/drivers/clk/sprd/common.h
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum clock infrastructure
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#ifndef _SPRD_CLK_COMMON_H_
+#define _SPRD_CLK_COMMON_H_
+
+#include <linux/clk-provider.h>
+#include <linux/of_platform.h>
+#include <linux/regmap.h>
+
+struct device_node;
+
+struct sprd_clk_common {
+	struct regmap	*regmap;
+	u32		reg;
+	struct clk_hw	hw;
+};
+
+struct sprd_clk_desc {
+	struct sprd_clk_common		**clk_clks;
+	unsigned long			num_clk_clks;
+	struct clk_hw_onecell_data      *hw_clks;
+};
+
+static inline struct sprd_clk_common *
+	hw_to_sprd_clk_common(const struct clk_hw *hw)
+{
+	return container_of(hw, struct sprd_clk_common, hw);
+}
+int sprd_clk_regmap_init(struct platform_device *pdev,
+			 const struct sprd_clk_desc *desc);
+int sprd_clk_probe(struct device *dev, struct clk_hw_onecell_data *clkhw);
+
+#endif /* _SPRD_CLK_COMMON_H_ */
diff --git a/drivers/clk/sprd/composite.c b/drivers/clk/sprd/composite.c
new file mode 100644
index 000000000000..ebb644820b1e
--- /dev/null
+++ b/drivers/clk/sprd/composite.c
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum composite clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#include <linux/clk-provider.h>
+
+#include "composite.h"
+
+static long sprd_comp_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *parent_rate)
+{
+	struct sprd_comp *cc = hw_to_sprd_comp(hw);
+
+	return sprd_div_helper_round_rate(&cc->common, &cc->div,
+					 rate, parent_rate);
+}
+
+static unsigned long sprd_comp_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct sprd_comp *cc = hw_to_sprd_comp(hw);
+
+	return sprd_div_helper_recalc_rate(&cc->common, &cc->div, parent_rate);
+}
+
+static int sprd_comp_set_rate(struct clk_hw *hw, unsigned long rate,
+			     unsigned long parent_rate)
+{
+	struct sprd_comp *cc = hw_to_sprd_comp(hw);
+
+	return sprd_div_helper_set_rate(&cc->common, &cc->div,
+				       rate, parent_rate);
+}
+
+static u8 sprd_comp_get_parent(struct clk_hw *hw)
+{
+	struct sprd_comp *cc = hw_to_sprd_comp(hw);
+
+	return sprd_mux_helper_get_parent(&cc->common, &cc->mux);
+}
+
+static int sprd_comp_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct sprd_comp *cc = hw_to_sprd_comp(hw);
+
+	return sprd_mux_helper_set_parent(&cc->common, &cc->mux, index);
+}
+
+const struct clk_ops sprd_comp_ops = {
+	.get_parent	= sprd_comp_get_parent,
+	.set_parent	= sprd_comp_set_parent,
+
+	.round_rate	= sprd_comp_round_rate,
+	.recalc_rate	= sprd_comp_recalc_rate,
+	.set_rate	= sprd_comp_set_rate,
+};
+EXPORT_SYMBOL_GPL(sprd_comp_ops);
diff --git a/drivers/clk/sprd/composite.h b/drivers/clk/sprd/composite.h
new file mode 100644
index 000000000000..0984e9e252dc
--- /dev/null
+++ b/drivers/clk/sprd/composite.h
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum composite clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#ifndef _SPRD_COMPOSITE_H_
+#define _SPRD_COMPOSITE_H_
+
+#include "common.h"
+#include "mux.h"
+#include "div.h"
+
+struct sprd_comp {
+	struct sprd_mux_ssel	mux;
+	struct sprd_div_internal	div;
+	struct sprd_clk_common	common;
+};
+
+#define SPRD_COMP_CLK_TABLE(_struct, _name, _parent, _reg, _table,	\
+			_mshift, _mwidth, _dshift, _dwidth, _flags)	\
+	struct sprd_comp _struct = {					\
+		.mux	= _SPRD_MUX_CLK(_mshift, _mwidth, _table),	\
+		.div	= _SPRD_DIV_CLK(_dshift, _dwidth),		\
+		.common = {						\
+			.regmap		= NULL,				\
+			.reg		= _reg,				\
+			.hw.init = CLK_HW_INIT_PARENTS(_name,		\
+						       _parent,		\
+						       &sprd_comp_ops,	\
+						       _flags),		\
+			 }						\
+	}
+
+#define SPRD_COMP_CLK(_struct, _name, _parent, _reg, _mshift,	\
+			_mwidth, _dshift, _dwidth, _flags)	\
+	SPRD_COMP_CLK_TABLE(_struct, _name, _parent, _reg,	\
+			    NULL, _mshift, _mwidth,		\
+			    _dshift, _dwidth, _flags)
+
+static inline struct sprd_comp *hw_to_sprd_comp(const struct clk_hw *hw)
+{
+	struct sprd_clk_common *common = hw_to_sprd_clk_common(hw);
+
+	return container_of(common, struct sprd_comp, common);
+}
+
+extern const struct clk_ops sprd_comp_ops;
+
+#endif /* _SPRD_COMPOSITE_H_ */
diff --git a/drivers/clk/sprd/div.c b/drivers/clk/sprd/div.c
new file mode 100644
index 000000000000..7621a1d1ab9c
--- /dev/null
+++ b/drivers/clk/sprd/div.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum divider clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#include <linux/clk-provider.h>
+
+#include "div.h"
+
+long sprd_div_helper_round_rate(struct sprd_clk_common *common,
+				const struct sprd_div_internal *div,
+				unsigned long rate,
+				unsigned long *parent_rate)
+{
+	return divider_round_rate(&common->hw, rate, parent_rate,
+				  NULL, div->width, 0);
+}
+EXPORT_SYMBOL_GPL(sprd_div_helper_round_rate);
+
+static long sprd_div_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *parent_rate)
+{
+	struct sprd_div *cd = hw_to_sprd_div(hw);
+
+	return sprd_div_helper_round_rate(&cd->common, &cd->div,
+					  rate, parent_rate);
+}
+
+unsigned long sprd_div_helper_recalc_rate(struct sprd_clk_common *common,
+					  const struct sprd_div_internal *div,
+					  unsigned long parent_rate)
+{
+	unsigned long val;
+	unsigned int reg;
+
+	regmap_read(common->regmap, common->reg, &reg);
+	val = reg >> div->shift;
+	val &= (1 << div->width) - 1;
+
+	return divider_recalc_rate(&common->hw, parent_rate, val, NULL, 0,
+				   div->width);
+}
+EXPORT_SYMBOL_GPL(sprd_div_helper_recalc_rate);
+
+static unsigned long sprd_div_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct sprd_div *cd = hw_to_sprd_div(hw);
+
+	return sprd_div_helper_recalc_rate(&cd->common, &cd->div, parent_rate);
+}
+
+int sprd_div_helper_set_rate(const struct sprd_clk_common *common,
+			     const struct sprd_div_internal *div,
+			     unsigned long rate,
+			     unsigned long parent_rate)
+{
+	unsigned long val;
+	unsigned int reg;
+
+	val = divider_get_val(rate, parent_rate, NULL,
+			      div->width, 0);
+
+	regmap_read(common->regmap, common->reg, &reg);
+	reg &= ~GENMASK(div->width + div->shift - 1, div->shift);
+
+	regmap_write(common->regmap, common->reg,
+			  reg | (val << div->shift));
+
+	return 0;
+
+}
+EXPORT_SYMBOL_GPL(sprd_div_helper_set_rate);
+
+static int sprd_div_set_rate(struct clk_hw *hw, unsigned long rate,
+			     unsigned long parent_rate)
+{
+	struct sprd_div *cd = hw_to_sprd_div(hw);
+
+	return sprd_div_helper_set_rate(&cd->common, &cd->div,
+					rate, parent_rate);
+}
+
+const struct clk_ops sprd_div_ops = {
+	.recalc_rate = sprd_div_recalc_rate,
+	.round_rate = sprd_div_round_rate,
+	.set_rate = sprd_div_set_rate,
+};
+EXPORT_SYMBOL_GPL(sprd_div_ops);
diff --git a/drivers/clk/sprd/div.h b/drivers/clk/sprd/div.h
new file mode 100644
index 000000000000..b3033d24d431
--- /dev/null
+++ b/drivers/clk/sprd/div.h
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum divider clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#ifndef _SPRD_DIV_H_
+#define _SPRD_DIV_H_
+
+#include "common.h"
+
+/**
+ * struct sprd_div_internal - Internal divider description
+ * @shift: Bit offset of the divider in its register
+ * @width: Width of the divider field in its register
+ *
+ * That structure represents a single divider, and is meant to be
+ * embedded in other structures representing the various clock
+ * classes.
+ */
+struct sprd_div_internal {
+	u8	shift;
+	u8	width;
+};
+
+#define _SPRD_DIV_CLK(_shift, _width)	\
+	{				\
+		.shift	= _shift,	\
+		.width	= _width,	\
+	}
+
+struct sprd_div {
+	struct sprd_div_internal	div;
+	struct sprd_clk_common	common;
+};
+
+#define SPRD_DIV_CLK(_struct, _name, _parent, _reg,			\
+			_shift, _width, _flags)				\
+	struct sprd_div _struct = {					\
+		.div	= _SPRD_DIV_CLK(_shift, _width),		\
+		.common	= {						\
+			.regmap		= NULL,				\
+			.reg		= _reg,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						      _parent,		\
+						      &sprd_div_ops,	\
+						      _flags),		\
+		}							\
+	}
+
+static inline struct sprd_div *hw_to_sprd_div(const struct clk_hw *hw)
+{
+	struct sprd_clk_common *common = hw_to_sprd_clk_common(hw);
+
+	return container_of(common, struct sprd_div, common);
+}
+
+long sprd_div_helper_round_rate(struct sprd_clk_common *common,
+				const struct sprd_div_internal *div,
+				unsigned long rate,
+				unsigned long *parent_rate);
+
+unsigned long sprd_div_helper_recalc_rate(struct sprd_clk_common *common,
+					  const struct sprd_div_internal *div,
+					  unsigned long parent_rate);
+
+int sprd_div_helper_set_rate(const struct sprd_clk_common *common,
+			     const struct sprd_div_internal *div,
+			     unsigned long rate,
+			     unsigned long parent_rate);
+
+extern const struct clk_ops sprd_div_ops;
+
+#endif /* _SPRD_DIV_H_ */
diff --git a/drivers/clk/sprd/gate.c b/drivers/clk/sprd/gate.c
new file mode 100644
index 000000000000..f59d1936b412
--- /dev/null
+++ b/drivers/clk/sprd/gate.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum gate clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+
+#include "gate.h"
+
+static void clk_gate_toggle(const struct sprd_gate *sg, bool en)
+{
+	const struct sprd_clk_common *common = &sg->common;
+	unsigned int reg;
+	bool set = sg->flags & CLK_GATE_SET_TO_DISABLE ? true : false;
+
+	set ^= en;
+
+	regmap_read(common->regmap, common->reg, &reg);
+
+	if (set)
+		reg |= sg->enable_mask;
+	else
+		reg &= ~sg->enable_mask;
+
+	regmap_write(common->regmap, common->reg, reg);
+}
+
+static void clk_sc_gate_toggle(const struct sprd_gate *sg, bool en)
+{
+	const struct sprd_clk_common *common = &sg->common;
+	bool set = sg->flags & CLK_GATE_SET_TO_DISABLE ? 1 : 0;
+	unsigned int offset;
+
+	set ^= en;
+
+	/*
+	 * Each set/clear gate clock has three registers:
+	 * common->reg			- base register
+	 * common->reg + offset		- set register
+	 * common->reg + 2 * offset	- clear register
+	 */
+	offset = set ? sg->sc_offset : sg->sc_offset * 2;
+
+	regmap_write(common->regmap, common->reg + offset,
+			  sg->enable_mask);
+}
+
+static void sprd_gate_disable(struct clk_hw *hw)
+{
+	struct sprd_gate *sg = hw_to_sprd_gate(hw);
+
+	clk_gate_toggle(sg, false);
+}
+
+static int sprd_gate_enable(struct clk_hw *hw)
+{
+	struct sprd_gate *sg = hw_to_sprd_gate(hw);
+
+	clk_gate_toggle(sg, true);
+
+	return 0;
+}
+
+static void sprd_sc_gate_disable(struct clk_hw *hw)
+{
+	struct sprd_gate *sg = hw_to_sprd_gate(hw);
+
+	clk_sc_gate_toggle(sg, false);
+}
+
+static int sprd_sc_gate_enable(struct clk_hw *hw)
+{
+	struct sprd_gate *sg = hw_to_sprd_gate(hw);
+
+	clk_sc_gate_toggle(sg, true);
+
+	return 0;
+}
+static int sprd_gate_is_enabled(struct clk_hw *hw)
+{
+	struct sprd_gate *sg = hw_to_sprd_gate(hw);
+	struct sprd_clk_common *common = &sg->common;
+	unsigned int reg;
+
+	regmap_read(common->regmap, common->reg, &reg);
+
+	if (sg->flags & CLK_GATE_SET_TO_DISABLE)
+		reg ^= sg->enable_mask;
+
+	reg &= sg->enable_mask;
+
+	return reg ? 1 : 0;
+}
+
+const struct clk_ops sprd_gate_ops = {
+	.disable	= sprd_gate_disable,
+	.enable		= sprd_gate_enable,
+	.is_enabled	= sprd_gate_is_enabled,
+};
+EXPORT_SYMBOL_GPL(sprd_gate_ops);
+
+const struct clk_ops sprd_sc_gate_ops = {
+	.disable	= sprd_sc_gate_disable,
+	.enable		= sprd_sc_gate_enable,
+	.is_enabled	= sprd_gate_is_enabled,
+};
+EXPORT_SYMBOL_GPL(sprd_sc_gate_ops);
+
diff --git a/drivers/clk/sprd/gate.h b/drivers/clk/sprd/gate.h
new file mode 100644
index 000000000000..2e582c68a08b
--- /dev/null
+++ b/drivers/clk/sprd/gate.h
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum gate clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#ifndef _SPRD_GATE_H_
+#define _SPRD_GATE_H_
+
+#include "common.h"
+
+struct sprd_gate {
+	u32			enable_mask;
+	u16			flags;
+	u16			sc_offset;
+
+	struct sprd_clk_common	common;
+};
+
+#define SPRD_SC_GATE_CLK_OPS(_struct, _name, _parent, _reg, _sc_offset,	\
+			     _enable_mask, _flags, _gate_flags, _ops)	\
+	struct sprd_gate _struct = {					\
+		.enable_mask	= _enable_mask,				\
+		.sc_offset	= _sc_offset,				\
+		.flags		= _gate_flags,				\
+		.common	= {						\
+			.regmap		= NULL,				\
+			.reg		= _reg,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						      _parent,		\
+						      _ops,		\
+						      _flags),		\
+		}							\
+	}
+
+#define SPRD_GATE_CLK(_struct, _name, _parent, _reg,			\
+		      _enable_mask, _flags, _gate_flags)		\
+	SPRD_SC_GATE_CLK_OPS(_struct, _name, _parent, _reg, 0,		\
+			     _enable_mask, _flags, _gate_flags,		\
+			     &sprd_gate_ops)
+
+#define SPRD_SC_GATE_CLK(_struct, _name, _parent, _reg, _sc_offset,	\
+			 _enable_mask, _flags, _gate_flags)		\
+	SPRD_SC_GATE_CLK_OPS(_struct, _name, _parent, _reg, _sc_offset,	\
+			     _enable_mask, _flags, _gate_flags,		\
+			     &sprd_sc_gate_ops)
+
+static inline struct sprd_gate *hw_to_sprd_gate(const struct clk_hw *hw)
+{
+	struct sprd_clk_common *common = hw_to_sprd_clk_common(hw);
+
+	return container_of(common, struct sprd_gate, common);
+}
+
+extern const struct clk_ops sprd_gate_ops;
+extern const struct clk_ops sprd_sc_gate_ops;
+
+#endif /* _SPRD_GATE_H_ */
diff --git a/drivers/clk/sprd/mux.c b/drivers/clk/sprd/mux.c
new file mode 100644
index 000000000000..624041b60358
--- /dev/null
+++ b/drivers/clk/sprd/mux.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum multiplexer clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/regmap.h>
+
+#include "mux.h"
+
+u8 sprd_mux_helper_get_parent(const struct sprd_clk_common *common,
+			      const struct sprd_mux_ssel *mux)
+{
+	unsigned int reg;
+	u8 parent;
+	int num_parents;
+	int i;
+
+	regmap_read(common->regmap, common->reg, &reg);
+	parent = reg >> mux->shift;
+	parent &= (1 << mux->width) - 1;
+
+	if (!mux->table)
+		return parent;
+
+	num_parents = clk_hw_get_num_parents(&common->hw);
+
+	for (i = 0; i < num_parents - 1; i++)
+		if (parent >= mux->table[i] && parent < mux->table[i + 1])
+			return i;
+
+	return num_parents - 1;
+}
+EXPORT_SYMBOL_GPL(sprd_mux_helper_get_parent);
+
+static u8 sprd_mux_get_parent(struct clk_hw *hw)
+{
+	struct sprd_mux *cm = hw_to_sprd_mux(hw);
+
+	return sprd_mux_helper_get_parent(&cm->common, &cm->mux);
+}
+
+int sprd_mux_helper_set_parent(const struct sprd_clk_common *common,
+			       const struct sprd_mux_ssel *mux,
+			       u8 index)
+{
+	unsigned int reg;
+
+	if (mux->table)
+		index = mux->table[index];
+
+	regmap_read(common->regmap, common->reg, &reg);
+	reg &= ~GENMASK(mux->width + mux->shift - 1, mux->shift);
+	regmap_write(common->regmap, common->reg,
+			  reg | (index << mux->shift));
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(sprd_mux_helper_set_parent);
+
+static int sprd_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct sprd_mux *cm = hw_to_sprd_mux(hw);
+
+	return sprd_mux_helper_set_parent(&cm->common, &cm->mux, index);
+}
+
+const struct clk_ops sprd_mux_ops = {
+	.get_parent = sprd_mux_get_parent,
+	.set_parent = sprd_mux_set_parent,
+	.determine_rate = __clk_mux_determine_rate,
+};
+EXPORT_SYMBOL_GPL(sprd_mux_ops);
diff --git a/drivers/clk/sprd/mux.h b/drivers/clk/sprd/mux.h
new file mode 100644
index 000000000000..548cfa0f145c
--- /dev/null
+++ b/drivers/clk/sprd/mux.h
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum multiplexer clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#ifndef _SPRD_MUX_H_
+#define _SPRD_MUX_H_
+
+#include "common.h"
+
+/**
+ * struct sprd_mux_ssel - Mux clock's source select bits in its register
+ * @shift: Bit offset of the divider in its register
+ * @width: Width of the divider field in its register
+ * @table: For some mux clocks, not all sources are used on some special
+ *	   chips, this matches the value of mux clock's register and the
+ *	   sources which are used for this mux clock
+ */
+struct sprd_mux_ssel {
+	u8		shift;
+	u8		width;
+	const u8	*table;
+};
+
+struct sprd_mux {
+	struct sprd_mux_ssel mux;
+	struct sprd_clk_common	common;
+};
+
+#define _SPRD_MUX_CLK(_shift, _width, _table)		\
+	{						\
+		.shift	= _shift,			\
+		.width	= _width,			\
+		.table	= _table,			\
+	}
+
+#define SPRD_MUX_CLK_TABLE(_struct, _name, _parents, _table,		\
+				     _reg, _shift, _width,		\
+				     _flags)				\
+	struct sprd_mux _struct = {					\
+		.mux	= _SPRD_MUX_CLK(_shift, _width, _table),	\
+		.common	= {						\
+			.regmap		= NULL,				\
+			.reg		= _reg,				\
+			.hw.init = CLK_HW_INIT_PARENTS(_name,		\
+						       _parents,	\
+						       &sprd_mux_ops,	\
+						       _flags),		\
+		}							\
+	}
+
+#define SPRD_MUX_CLK(_struct, _name, _parents, _reg,		\
+		     _shift, _width, _flags)			\
+	SPRD_MUX_CLK_TABLE(_struct, _name, _parents, NULL,	\
+			   _reg, _shift, _width, _flags)
+
+static inline struct sprd_mux *hw_to_sprd_mux(const struct clk_hw *hw)
+{
+	struct sprd_clk_common *common = hw_to_sprd_clk_common(hw);
+
+	return container_of(common, struct sprd_mux, common);
+}
+
+extern const struct clk_ops sprd_mux_ops;
+
+u8 sprd_mux_helper_get_parent(const struct sprd_clk_common *common,
+			      const struct sprd_mux_ssel *mux);
+int sprd_mux_helper_set_parent(const struct sprd_clk_common *common,
+			       const struct sprd_mux_ssel *mux,
+			       u8 index);
+
+#endif /* _SPRD_MUX_H_ */
diff --git a/drivers/clk/sprd/pll.c b/drivers/clk/sprd/pll.c
new file mode 100644
index 000000000000..36b4402bf09e
--- /dev/null
+++ b/drivers/clk/sprd/pll.c
@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum pll clock driver
+//
+// Copyright (C) 2015~2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include "pll.h"
+
+#define CLK_PLL_1M	1000000
+#define CLK_PLL_10M	(CLK_PLL_1M * 10)
+
+#define pindex(pll, member)		\
+	(pll->factors[member].shift / (8 * sizeof(pll->regs_num)))
+
+#define pshift(pll, member)		\
+	(pll->factors[member].shift % (8 * sizeof(pll->regs_num)))
+
+#define pwidth(pll, member)		\
+	pll->factors[member].width
+
+#define pmask(pll, member)					\
+	((pwidth(pll, member)) ?				\
+	GENMASK(pwidth(pll, member) + pshift(pll, member) - 1,	\
+	pshift(pll, member)) : 0)
+
+#define pinternal(pll, cfg, member)	\
+	(cfg[pindex(pll, member)] & pmask(pll, member))
+
+#define pinternal_val(pll, cfg, member)	\
+	(pinternal(pll, cfg, member) >> pshift(pll, member))
+
+static inline unsigned int
+sprd_pll_read(const struct sprd_pll *pll, u8 index)
+{
+	const struct sprd_clk_common *common = &pll->common;
+	unsigned int val = 0;
+
+	if (WARN_ON(index >= pll->regs_num))
+		return 0;
+
+	regmap_read(common->regmap, common->reg + index * 4, &val);
+
+	return val;
+}
+
+static inline void
+sprd_pll_write(const struct sprd_pll *pll, u8 index,
+				  u32 msk, u32 val)
+{
+	const struct sprd_clk_common *common = &pll->common;
+	unsigned int offset, reg;
+	int ret = 0;
+
+	if (WARN_ON(index >= pll->regs_num))
+		return;
+
+	offset = common->reg + index * 4;
+	ret = regmap_read(common->regmap, offset, &reg);
+	if (!ret)
+		regmap_write(common->regmap, offset, (reg & ~msk) | val);
+}
+
+static unsigned long pll_get_refin(const struct sprd_pll *pll)
+{
+	u32 shift, mask, index, refin_id = 3;
+	const unsigned long refin[4] = { 2, 4, 13, 26 };
+
+	if (pwidth(pll, PLL_REFIN)) {
+		index = pindex(pll, PLL_REFIN);
+		shift = pshift(pll, PLL_REFIN);
+		mask = pmask(pll, PLL_REFIN);
+		refin_id = (sprd_pll_read(pll, index) & mask) >> shift;
+		if (refin_id > 3)
+			refin_id = 3;
+	}
+
+	return refin[refin_id];
+}
+
+static u32 pll_get_ibias(u64 rate, const u64 *table)
+{
+	u32 i, num = table[0];
+
+	for (i = 1; i < num + 1; i++)
+		if (rate <= table[i])
+			break;
+
+	return (i == num + 1) ? num : i;
+}
+
+static unsigned long _sprd_pll_recalc_rate(const struct sprd_pll *pll,
+					   unsigned long parent_rate)
+{
+	u32 *cfg;
+	u32 i, mask, regs_num = pll->regs_num;
+	unsigned long rate, nint, kint = 0;
+	u64 refin;
+	u16 k1, k2;
+
+	cfg = kcalloc(regs_num, sizeof(*cfg), GFP_KERNEL);
+	if (!cfg)
+		return -ENOMEM;
+
+	for (i = 0; i < regs_num; i++)
+		cfg[i] = sprd_pll_read(pll, i);
+
+	refin = pll_get_refin(pll);
+
+	if (pinternal(pll, cfg, PLL_PREDIV))
+		refin = refin * 2;
+
+	if (pwidth(pll, PLL_POSTDIV) &&
+	    ((pll->fflag == 1 && pinternal(pll, cfg, PLL_POSTDIV)) ||
+	     (!pll->fflag && !pinternal(pll, cfg, PLL_POSTDIV))))
+		refin = refin / 2;
+
+	if (!pinternal(pll, cfg, PLL_DIV_S)) {
+		rate = refin * pinternal_val(pll, cfg, PLL_N) * CLK_PLL_10M;
+	} else {
+		nint = pinternal_val(pll, cfg, PLL_NINT);
+		if (pinternal(pll, cfg, PLL_SDM_EN))
+			kint = pinternal_val(pll, cfg, PLL_KINT);
+
+		mask = pmask(pll, PLL_KINT);
+
+		k1 = pll->k1;
+		k2 = pll->k2;
+		rate = DIV_ROUND_CLOSEST_ULL(refin * kint * k1,
+					 ((mask >> __ffs(mask)) + 1)) *
+					 k2 + refin * nint * CLK_PLL_1M;
+	}
+
+	return rate;
+}
+
+#define SPRD_PLL_WRITE_CHECK(pll, i, mask, val)		\
+	(((sprd_pll_read(pll, i) & mask) == val) ? 0 : (-EFAULT))
+
+static int _sprd_pll_set_rate(const struct sprd_pll *pll,
+			      unsigned long rate,
+			      unsigned long parent_rate)
+{
+	struct reg_cfg *cfg;
+	int ret = 0;
+	u32 mask, shift, width, ibias_val, index;
+	u32 regs_num = pll->regs_num, i = 0;
+	unsigned long kint, nint;
+	u64 tmp, refin, fvco = rate;
+
+	cfg = kcalloc(regs_num, sizeof(*cfg), GFP_KERNEL);
+	if (!cfg)
+		return -ENOMEM;
+
+	refin = pll_get_refin(pll);
+
+	mask = pmask(pll, PLL_PREDIV);
+	index = pindex(pll, PLL_PREDIV);
+	width = pwidth(pll, PLL_PREDIV);
+	if (width && (sprd_pll_read(pll, index) & mask))
+		refin = refin * 2;
+
+	mask = pmask(pll, PLL_POSTDIV);
+	index = pindex(pll, PLL_POSTDIV);
+	width = pwidth(pll, PLL_POSTDIV);
+	cfg[index].msk = mask;
+	if (width && ((pll->fflag == 1 && fvco <= pll->fvco) ||
+		      (pll->fflag == 0 && fvco > pll->fvco)))
+		cfg[index].val |= mask;
+
+	if (width && fvco <= pll->fvco)
+		fvco = fvco * 2;
+
+	mask = pmask(pll, PLL_DIV_S);
+	index = pindex(pll, PLL_DIV_S);
+	cfg[index].val |= mask;
+	cfg[index].msk |= mask;
+
+	mask = pmask(pll, PLL_SDM_EN);
+	index = pindex(pll, PLL_SDM_EN);
+	cfg[index].val |= mask;
+	cfg[index].msk |= mask;
+
+	nint = do_div(fvco, refin * CLK_PLL_1M);
+	mask = pmask(pll, PLL_NINT);
+	index = pindex(pll, PLL_NINT);
+	shift = pshift(pll, PLL_NINT);
+	cfg[index].val |= (nint << shift) & mask;
+	cfg[index].msk |= mask;
+
+	mask = pmask(pll, PLL_KINT);
+	index = pindex(pll, PLL_KINT);
+	width = pwidth(pll, PLL_KINT);
+	shift = pshift(pll, PLL_KINT);
+	tmp = fvco - refin * nint * CLK_PLL_1M;
+	tmp = do_div(tmp, 10000) * ((mask >> shift) + 1);
+	kint = DIV_ROUND_CLOSEST_ULL(tmp, refin * 100);
+	cfg[index].val |= (kint << shift) & mask;
+	cfg[index].msk |= mask;
+
+	ibias_val = pll_get_ibias(fvco, pll->itable);
+
+	mask = pmask(pll, PLL_IBIAS);
+	index = pindex(pll, PLL_IBIAS);
+	shift = pshift(pll, PLL_IBIAS);
+	cfg[index].val |= ibias_val << shift & mask;
+	cfg[index].msk |= mask;
+
+	for (i = 0; i < regs_num; i++) {
+		if (cfg[i].msk) {
+			sprd_pll_write(pll, i, cfg[i].msk, cfg[i].val);
+			ret |= SPRD_PLL_WRITE_CHECK(pll, i, cfg[i].msk,
+						   cfg[i].val);
+		}
+	}
+
+	if (!ret)
+		udelay(pll->udelay);
+
+	return ret;
+}
+
+static unsigned long sprd_pll_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct sprd_pll *pll = hw_to_sprd_pll(hw);
+
+	return _sprd_pll_recalc_rate(pll, parent_rate);
+}
+
+static int sprd_pll_set_rate(struct clk_hw *hw,
+			     unsigned long rate,
+			     unsigned long parent_rate)
+{
+	struct sprd_pll *pll = hw_to_sprd_pll(hw);
+
+	return _sprd_pll_set_rate(pll, rate, parent_rate);
+}
+
+static int sprd_pll_clk_prepare(struct clk_hw *hw)
+{
+	struct sprd_pll *pll = hw_to_sprd_pll(hw);
+
+	udelay(pll->udelay);
+
+	return 0;
+}
+
+static long sprd_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *prate)
+{
+	return rate;
+}
+
+const struct clk_ops sprd_pll_ops = {
+	.prepare = sprd_pll_clk_prepare,
+	.recalc_rate = sprd_pll_recalc_rate,
+	.round_rate = sprd_pll_round_rate,
+	.set_rate = sprd_pll_set_rate,
+};
+EXPORT_SYMBOL_GPL(sprd_pll_ops);
diff --git a/drivers/clk/sprd/pll.h b/drivers/clk/sprd/pll.h
new file mode 100644
index 000000000000..514175621099
--- /dev/null
+++ b/drivers/clk/sprd/pll.h
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreadtrum pll clock driver
+//
+// Copyright (C) 2015~2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#ifndef _SPRD_PLL_H_
+#define _SPRD_PLL_H_
+
+#include "common.h"
+
+struct reg_cfg {
+	u32 val;
+	u32 msk;
+};
+
+struct clk_bit_field {
+	u8 shift;
+	u8 width;
+};
+
+enum {
+	PLL_LOCK_DONE,
+	PLL_DIV_S,
+	PLL_MOD_EN,
+	PLL_SDM_EN,
+	PLL_REFIN,
+	PLL_IBIAS,
+	PLL_N,
+	PLL_NINT,
+	PLL_KINT,
+	PLL_PREDIV,
+	PLL_POSTDIV,
+
+	PLL_FACT_MAX
+};
+
+/*
+ * struct sprd_pll - definition of adjustable pll clock
+ *
+ * @reg:	registers used to set the configuration of pll clock,
+ *		reg[0] shows how many registers this pll clock uses.
+ * @itable:	pll ibias table, itable[0] means how many items this
+ *		table includes
+ * @udelay	delay time after setting rate
+ * @factors	used to calculate the pll clock rate
+ * @fvco:	fvco threshold rate
+ * @fflag:	fvco flag
+ */
+struct sprd_pll {
+	u32 regs_num;
+	const u64 *itable;
+	const struct clk_bit_field *factors;
+	u16 udelay;
+	u16 k1;
+	u16 k2;
+	u16 fflag;
+	u64 fvco;
+
+	struct sprd_clk_common	common;
+};
+
+#define SPRD_PLL_WITH_ITABLE_K_FVCO(_struct, _name, _parent, _reg,	\
+				    _regs_num, _itable, _factors,	\
+				    _udelay, _k1, _k2, _fflag, _fvco)	\
+	struct sprd_pll _struct = {					\
+		.regs_num	= _regs_num,				\
+		.itable		= _itable,				\
+		.factors	= _factors,				\
+		.udelay		= _udelay,				\
+		.k1		= _k1,					\
+		.k2		= _k2,					\
+		.fflag		= _fflag,				\
+		.fvco		= _fvco,				\
+		.common		= {					\
+			.regmap		= NULL,				\
+			.reg		= _reg,				\
+			.hw.init	= CLK_HW_INIT(_name,		\
+						      _parent,		\
+						      &sprd_pll_ops,	\
+						      0),		\
+		},							\
+	}
+
+#define SPRD_PLL_WITH_ITABLE_K(_struct, _name, _parent, _reg,		\
+			       _regs_num, _itable, _factors,		\
+			       _udelay, _k1, _k2)			\
+	SPRD_PLL_WITH_ITABLE_K_FVCO(_struct, _name, _parent, _reg,	\
+				    _regs_num, _itable, _factors,	\
+				    _udelay, _k1, _k2, 0, 0)
+
+#define SPRD_PLL_WITH_ITABLE_1K(_struct, _name, _parent, _reg,		\
+				_regs_num, _itable, _factors, _udelay)	\
+	SPRD_PLL_WITH_ITABLE_K_FVCO(_struct, _name, _parent, _reg,	\
+				    _regs_num, _itable, _factors,	\
+				    _udelay, 1000, 1000, 0, 0)
+
+static inline struct sprd_pll *hw_to_sprd_pll(struct clk_hw *hw)
+{
+	struct sprd_clk_common *common = hw_to_sprd_clk_common(hw);
+
+	return container_of(common, struct sprd_pll, common);
+}
+
+extern const struct clk_ops sprd_pll_ops;
+
+#endif /* _SPRD_PLL_H_ */
diff --git a/drivers/clk/sprd/sc9860-clk.c b/drivers/clk/sprd/sc9860-clk.c
new file mode 100644
index 000000000000..ed5c027df0f4
--- /dev/null
+++ b/drivers/clk/sprd/sc9860-clk.c
@@ -0,0 +1,1974 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Spreatrum SC9860 clock driver
+//
+// Copyright (C) 2017 Spreadtrum, Inc.
+// Author: Chunyan Zhang <chunyan.zhang@spreadtrum.com>
+
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include <dt-bindings/clock/sprd,sc9860-clk.h>
+
+#include "common.h"
+#include "composite.h"
+#include "div.h"
+#include "gate.h"
+#include "mux.h"
+#include "pll.h"
+
+static CLK_FIXED_FACTOR(fac_4m,		"fac-4m",	"ext-26m",
+			6, 1, 0);
+static CLK_FIXED_FACTOR(fac_2m,		"fac-2m",	"ext-26m",
+			13, 1, 0);
+static CLK_FIXED_FACTOR(fac_1m,		"fac-1m",	"ext-26m",
+			26, 1, 0);
+static CLK_FIXED_FACTOR(fac_250k,	"fac-250k",	"ext-26m",
+			104, 1, 0);
+static CLK_FIXED_FACTOR(fac_rpll0_26m,	"rpll0-26m",	"ext-26m",
+			1, 1, 0);
+static CLK_FIXED_FACTOR(fac_rpll1_26m,	"rpll1-26m",	"ext-26m",
+			1, 1, 0);
+static CLK_FIXED_FACTOR(fac_rco_25m,	"rco-25m",	"ext-rc0-100m",
+			4, 1, 0);
+static CLK_FIXED_FACTOR(fac_rco_4m,	"rco-4m",	"ext-rc0-100m",
+			25, 1, 0);
+static CLK_FIXED_FACTOR(fac_rco_2m,	"rco-2m",	"ext-rc0-100m",
+			50, 1, 0);
+static CLK_FIXED_FACTOR(fac_3k2,	"fac-3k2",	"ext-32k",
+			10, 1, 0);
+static CLK_FIXED_FACTOR(fac_1k,		"fac-1k",	"ext-32k",
+			32, 1, 0);
+
+static SPRD_SC_GATE_CLK(mpll0_gate,	"mpll0-gate",	"ext-26m", 0xb0,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(mpll1_gate,	"mpll1-gate",	"ext-26m", 0xb0,
+		     0x1000, BIT(18), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(dpll0_gate,	"dpll0-gate",	"ext-26m", 0xb4,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(dpll1_gate,	"dpll1-gate",	"ext-26m", 0xb4,
+		     0x1000, BIT(18), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ltepll0_gate,	"ltepll0-gate",	"ext-26m", 0xb8,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(twpll_gate,	"twpll-gate",	"ext-26m", 0xbc,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ltepll1_gate,	"ltepll1-gate",	"ext-26m", 0x10c,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(rpll0_gate,	"rpll0-gate",	"ext-26m", 0x16c,
+		     0x1000, BIT(2), 0, 0);
+static SPRD_SC_GATE_CLK(rpll1_gate,	"rpll1-gate",	"ext-26m", 0x16c,
+		     0x1000, BIT(18), 0, 0);
+static SPRD_SC_GATE_CLK(cppll_gate,	"cppll-gate",	"ext-26m", 0x2b4,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(gpll_gate,	"gpll-gate",	"ext-26m", 0x32c,
+		0x1000, BIT(0), CLK_IGNORE_UNUSED, CLK_GATE_SET_TO_DISABLE);
+
+static struct sprd_clk_common *sc9860_pmu_gate_clks[] = {
+	/* address base is 0x402b0000 */
+	&mpll0_gate.common,
+	&mpll1_gate.common,
+	&dpll0_gate.common,
+	&dpll1_gate.common,
+	&ltepll0_gate.common,
+	&twpll_gate.common,
+	&ltepll1_gate.common,
+	&rpll0_gate.common,
+	&rpll1_gate.common,
+	&cppll_gate.common,
+	&gpll_gate.common,
+};
+
+static struct clk_hw_onecell_data sc9860_pmu_gate_hws = {
+	.hws	= {
+		[CLK_FAC_4M]		= &fac_4m.hw,
+		[CLK_FAC_2M]		= &fac_2m.hw,
+		[CLK_FAC_1M]		= &fac_1m.hw,
+		[CLK_FAC_250K]		= &fac_250k.hw,
+		[CLK_FAC_RPLL0_26M]	= &fac_rpll0_26m.hw,
+		[CLK_FAC_RPLL1_26M]	= &fac_rpll1_26m.hw,
+		[CLK_FAC_RCO25M]	= &fac_rco_25m.hw,
+		[CLK_FAC_RCO4M]		= &fac_rco_4m.hw,
+		[CLK_FAC_RCO2M]		= &fac_rco_2m.hw,
+		[CLK_FAC_3K2]		= &fac_3k2.hw,
+		[CLK_FAC_1K]		= &fac_1k.hw,
+		[CLK_MPLL0_GATE]	= &mpll0_gate.common.hw,
+		[CLK_MPLL1_GATE]	= &mpll1_gate.common.hw,
+		[CLK_DPLL0_GATE]	= &dpll0_gate.common.hw,
+		[CLK_DPLL1_GATE]	= &dpll1_gate.common.hw,
+		[CLK_LTEPLL0_GATE]	= &ltepll0_gate.common.hw,
+		[CLK_TWPLL_GATE]	= &twpll_gate.common.hw,
+		[CLK_LTEPLL1_GATE]	= &ltepll1_gate.common.hw,
+		[CLK_RPLL0_GATE]	= &rpll0_gate.common.hw,
+		[CLK_RPLL1_GATE]	= &rpll1_gate.common.hw,
+		[CLK_CPPLL_GATE]	= &cppll_gate.common.hw,
+		[CLK_GPLL_GATE]		= &gpll_gate.common.hw,
+	},
+	.num	= CLK_PMU_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_pmu_gate_desc = {
+	.clk_clks	= sc9860_pmu_gate_clks,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_pmu_gate_clks),
+	.hw_clks        = &sc9860_pmu_gate_hws,
+};
+
+/* GPLL/LPLL/DPLL/RPLL/CPLL */
+static const u64 itable1[4] = {3, 780000000, 988000000, 1196000000};
+
+/* TWPLL/MPLL0/MPLL1 */
+static const u64 itable2[4] = {3, 1638000000, 2080000000, 2600000000UL};
+
+static const struct clk_bit_field f_mpll0[PLL_FACT_MAX] = {
+	{ .shift = 20,	.width = 1 },	/* lock_done	*/
+	{ .shift = 19,	.width = 1 },	/* div_s	*/
+	{ .shift = 18,	.width = 1 },	/* mod_en	*/
+	{ .shift = 17,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 11,	.width = 2 },	/* ibias	*/
+	{ .shift = 0,	.width = 7 },	/* n		*/
+	{ .shift = 57,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 0,	.width = 0 },	/* prediv	*/
+	{ .shift = 56,	.width = 1 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_K_FVCO(mpll0_clk, "mpll0", "mpll0-gate", 0x24,
+				   2, itable2, f_mpll0, 200,
+				   1000, 1000, 1, 1300000000);
+
+static const struct clk_bit_field f_mpll1[PLL_FACT_MAX] = {
+	{ .shift = 20,	.width = 1 },	/* lock_done	*/
+	{ .shift = 19,	.width = 1 },	/* div_s	*/
+	{ .shift = 18,	.width = 1 },	/* mod_en	*/
+	{ .shift = 17,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 11,	.width = 2 },	/* ibias	*/
+	{ .shift = 0,	.width = 7 },	/* n		*/
+	{ .shift = 57,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 56,	.width = 1 },	/* prediv	*/
+	{ .shift = 0,	.width = 0 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_1K(mpll1_clk, "mpll1", "mpll1-gate", 0x2c,
+			       2, itable2, f_mpll1, 200);
+
+static const struct clk_bit_field f_dpll[PLL_FACT_MAX] = {
+	{ .shift = 16,	.width = 1 },	/* lock_done	*/
+	{ .shift = 15,	.width = 1 },	/* div_s	*/
+	{ .shift = 14,	.width = 1 },	/* mod_en	*/
+	{ .shift = 13,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 8,	.width = 2 },	/* ibias	*/
+	{ .shift = 0,	.width = 7 },	/* n		*/
+	{ .shift = 57,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 0,	.width = 0 },	/* prediv	*/
+	{ .shift = 0,	.width = 0 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_1K(dpll0_clk, "dpll0", "dpll0-gate", 0x34,
+			       2, itable1, f_dpll, 200);
+
+static SPRD_PLL_WITH_ITABLE_1K(dpll1_clk, "dpll1", "dpll1-gate", 0x3c,
+			       2, itable1, f_dpll, 200);
+
+static const struct clk_bit_field f_rpll[PLL_FACT_MAX] = {
+	{ .shift = 0,	.width = 1 },	/* lock_done	*/
+	{ .shift = 3,	.width = 1 },	/* div_s	*/
+	{ .shift = 80,	.width = 1 },	/* mod_en	*/
+	{ .shift = 81,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 14,	.width = 2 },	/* ibias	*/
+	{ .shift = 16,	.width = 7 },	/* n		*/
+	{ .shift = 4,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 0,	.width = 0 },	/* prediv	*/
+	{ .shift = 0,	.width = 0 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_1K(rpll0_clk, "rpll0", "rpll0-gate", 0x44,
+			       3, itable1, f_rpll, 200);
+
+static SPRD_PLL_WITH_ITABLE_1K(rpll1_clk, "rpll1", "rpll1-gate", 0x50,
+			       3, itable1, f_rpll, 200);
+
+static const struct clk_bit_field f_twpll[PLL_FACT_MAX] = {
+	{ .shift = 21,	.width = 1 },	/* lock_done	*/
+	{ .shift = 20,	.width = 1 },	/* div_s	*/
+	{ .shift = 19,	.width = 1 },	/* mod_en	*/
+	{ .shift = 18,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 13,	.width = 2 },	/* ibias	*/
+	{ .shift = 0,	.width = 7 },	/* n		*/
+	{ .shift = 57,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 0,	.width = 0 },	/* prediv	*/
+	{ .shift = 0,	.width = 0 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_1K(twpll_clk, "twpll", "twpll-gate", 0x5c,
+			       2, itable2, f_twpll, 200);
+
+static const struct clk_bit_field f_ltepll[PLL_FACT_MAX] = {
+	{ .shift = 31,	.width = 1 },	/* lock_done	*/
+	{ .shift = 27,	.width = 1 },	/* div_s	*/
+	{ .shift = 26,	.width = 1 },	/* mod_en	*/
+	{ .shift = 25,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 20,	.width = 2 },	/* ibias	*/
+	{ .shift = 0,	.width = 7 },	/* n		*/
+	{ .shift = 57,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 0,	.width = 0 },	/* prediv	*/
+	{ .shift = 0,	.width = 0 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_1K(ltepll0_clk, "ltepll0", "ltepll0-gate",
+			       0x64, 2, itable1,
+			       f_ltepll, 200);
+static SPRD_PLL_WITH_ITABLE_1K(ltepll1_clk, "ltepll1", "ltepll1-gate",
+			       0x6c, 2, itable1,
+			       f_ltepll, 200);
+
+static const struct clk_bit_field f_gpll[PLL_FACT_MAX] = {
+	{ .shift = 18,	.width = 1 },	/* lock_done	*/
+	{ .shift = 15,	.width = 1 },	/* div_s	*/
+	{ .shift = 14,	.width = 1 },	/* mod_en	*/
+	{ .shift = 13,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 8,	.width = 2 },	/* ibias	*/
+	{ .shift = 0,	.width = 7 },	/* n		*/
+	{ .shift = 57,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 0,	.width = 0 },	/* prediv	*/
+	{ .shift = 17,	.width = 1 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_K_FVCO(gpll_clk, "gpll", "gpll-gate", 0x9c,
+				   2, itable1, f_gpll, 200,
+				   1000, 1000, 1, 600000000);
+
+static const struct clk_bit_field f_cppll[PLL_FACT_MAX] = {
+	{ .shift = 17,	.width = 1 },	/* lock_done	*/
+	{ .shift = 15,	.width = 1 },	/* div_s	*/
+	{ .shift = 14,	.width = 1 },	/* mod_en	*/
+	{ .shift = 13,	.width = 1 },	/* sdm_en	*/
+	{ .shift = 0,	.width = 0 },	/* refin	*/
+	{ .shift = 8,	.width = 2 },	/* ibias	*/
+	{ .shift = 0,	.width = 7 },	/* n		*/
+	{ .shift = 57,	.width = 7 },	/* nint		*/
+	{ .shift = 32,	.width = 23},	/* kint		*/
+	{ .shift = 0,	.width = 0 },	/* prediv	*/
+	{ .shift = 0,	.width = 0 },	/* postdiv	*/
+};
+static SPRD_PLL_WITH_ITABLE_1K(cppll_clk, "cppll", "cppll-gate", 0xc4,
+			       2, itable1, f_cppll, 200);
+
+static CLK_FIXED_FACTOR(gpll_42m5, "gpll-42m5", "gpll", 20, 1, 0);
+static CLK_FIXED_FACTOR(twpll_768m, "twpll-768m", "twpll", 2, 1, 0);
+static CLK_FIXED_FACTOR(twpll_384m, "twpll-384m", "twpll", 4, 1, 0);
+static CLK_FIXED_FACTOR(twpll_192m, "twpll-192m", "twpll", 8, 1, 0);
+static CLK_FIXED_FACTOR(twpll_96m, "twpll-96m", "twpll", 16, 1, 0);
+static CLK_FIXED_FACTOR(twpll_48m, "twpll-48m", "twpll", 32, 1, 0);
+static CLK_FIXED_FACTOR(twpll_24m, "twpll-24m", "twpll", 64, 1, 0);
+static CLK_FIXED_FACTOR(twpll_12m, "twpll-12m", "twpll", 128, 1, 0);
+static CLK_FIXED_FACTOR(twpll_512m, "twpll-512m", "twpll", 3, 1, 0);
+static CLK_FIXED_FACTOR(twpll_256m, "twpll-256m", "twpll", 6, 1, 0);
+static CLK_FIXED_FACTOR(twpll_128m, "twpll-128m", "twpll", 12, 1, 0);
+static CLK_FIXED_FACTOR(twpll_64m, "twpll-64m", "twpll", 24, 1, 0);
+static CLK_FIXED_FACTOR(twpll_307m2, "twpll-307m2", "twpll", 5, 1, 0);
+static CLK_FIXED_FACTOR(twpll_153m6, "twpll-153m6", "twpll", 10, 1, 0);
+static CLK_FIXED_FACTOR(twpll_76m8, "twpll-76m8", "twpll", 20, 1, 0);
+static CLK_FIXED_FACTOR(twpll_51m2, "twpll-51m2", "twpll", 30, 1, 0);
+static CLK_FIXED_FACTOR(twpll_38m4, "twpll-38m4", "twpll", 40, 1, 0);
+static CLK_FIXED_FACTOR(twpll_19m2, "twpll-19m2", "twpll", 80, 1, 0);
+static CLK_FIXED_FACTOR(l0_614m4, "l0-614m4", "ltepll0", 2, 1, 0);
+static CLK_FIXED_FACTOR(l0_409m6, "l0-409m6", "ltepll0", 3, 1, 0);
+static CLK_FIXED_FACTOR(l0_38m, "l0-38m", "ltepll0", 32, 1, 0);
+static CLK_FIXED_FACTOR(l1_38m, "l1-38m", "ltepll1", 32, 1, 0);
+static CLK_FIXED_FACTOR(rpll0_192m, "rpll0-192m", "rpll0", 6, 1, 0);
+static CLK_FIXED_FACTOR(rpll0_96m, "rpll0-96m", "rpll0", 12, 1, 0);
+static CLK_FIXED_FACTOR(rpll0_48m, "rpll0-48m", "rpll0", 24, 1, 0);
+static CLK_FIXED_FACTOR(rpll1_468m, "rpll1-468m", "rpll1", 2, 1, 0);
+static CLK_FIXED_FACTOR(rpll1_192m, "rpll1-192m", "rpll1", 6, 1, 0);
+static CLK_FIXED_FACTOR(rpll1_96m, "rpll1-96m", "rpll1", 12, 1, 0);
+static CLK_FIXED_FACTOR(rpll1_64m, "rpll1-64m", "rpll1", 18, 1, 0);
+static CLK_FIXED_FACTOR(rpll1_48m, "rpll1-48m", "rpll1", 24, 1, 0);
+static CLK_FIXED_FACTOR(dpll0_50m, "dpll0-50m", "dpll0", 16, 1, 0);
+static CLK_FIXED_FACTOR(dpll1_50m, "dpll1-50m", "dpll1", 16, 1, 0);
+static CLK_FIXED_FACTOR(cppll_50m, "cppll-50m", "cppll", 18, 1, 0);
+static CLK_FIXED_FACTOR(m0_39m, "m0-39m", "mpll0", 32, 1, 0);
+static CLK_FIXED_FACTOR(m1_63m, "m1-63m", "mpll1", 32, 1, 0);
+
+static struct sprd_clk_common *sc9860_pll_clks[] = {
+	/* address base is 0x40400000 */
+	&mpll0_clk.common,
+	&mpll1_clk.common,
+	&dpll0_clk.common,
+	&dpll1_clk.common,
+	&rpll0_clk.common,
+	&rpll1_clk.common,
+	&twpll_clk.common,
+	&ltepll0_clk.common,
+	&ltepll1_clk.common,
+	&gpll_clk.common,
+	&cppll_clk.common,
+};
+
+static struct clk_hw_onecell_data sc9860_pll_hws = {
+	.hws	= {
+		[CLK_MPLL0]		= &mpll0_clk.common.hw,
+		[CLK_MPLL1]		= &mpll1_clk.common.hw,
+		[CLK_DPLL0]		= &dpll0_clk.common.hw,
+		[CLK_DPLL1]		= &dpll1_clk.common.hw,
+		[CLK_RPLL0]		= &rpll0_clk.common.hw,
+		[CLK_RPLL1]		= &rpll1_clk.common.hw,
+		[CLK_TWPLL]		= &twpll_clk.common.hw,
+		[CLK_LTEPLL0]		= &ltepll0_clk.common.hw,
+		[CLK_LTEPLL1]		= &ltepll1_clk.common.hw,
+		[CLK_GPLL]		= &gpll_clk.common.hw,
+		[CLK_CPPLL]		= &cppll_clk.common.hw,
+		[CLK_GPLL_42M5]		= &gpll_42m5.hw,
+		[CLK_TWPLL_768M]	= &twpll_768m.hw,
+		[CLK_TWPLL_384M]	= &twpll_384m.hw,
+		[CLK_TWPLL_192M]	= &twpll_192m.hw,
+		[CLK_TWPLL_96M]		= &twpll_96m.hw,
+		[CLK_TWPLL_48M]		= &twpll_48m.hw,
+		[CLK_TWPLL_24M]		= &twpll_24m.hw,
+		[CLK_TWPLL_12M]		= &twpll_12m.hw,
+		[CLK_TWPLL_512M]	= &twpll_512m.hw,
+		[CLK_TWPLL_256M]	= &twpll_256m.hw,
+		[CLK_TWPLL_128M]	= &twpll_128m.hw,
+		[CLK_TWPLL_64M]		= &twpll_64m.hw,
+		[CLK_TWPLL_307M2]	= &twpll_307m2.hw,
+		[CLK_TWPLL_153M6]	= &twpll_153m6.hw,
+		[CLK_TWPLL_76M8]	= &twpll_76m8.hw,
+		[CLK_TWPLL_51M2]	= &twpll_51m2.hw,
+		[CLK_TWPLL_38M4]	= &twpll_38m4.hw,
+		[CLK_TWPLL_19M2]	= &twpll_19m2.hw,
+		[CLK_L0_614M4]		= &l0_614m4.hw,
+		[CLK_L0_409M6]		= &l0_409m6.hw,
+		[CLK_L0_38M]		= &l0_38m.hw,
+		[CLK_L1_38M]		= &l1_38m.hw,
+		[CLK_RPLL0_192M]	= &rpll0_192m.hw,
+		[CLK_RPLL0_96M]		= &rpll0_96m.hw,
+		[CLK_RPLL0_48M]		= &rpll0_48m.hw,
+		[CLK_RPLL1_468M]	= &rpll1_468m.hw,
+		[CLK_RPLL1_192M]	= &rpll1_192m.hw,
+		[CLK_RPLL1_96M]		= &rpll1_96m.hw,
+		[CLK_RPLL1_64M]		= &rpll1_64m.hw,
+		[CLK_RPLL1_48M]		= &rpll1_48m.hw,
+		[CLK_DPLL0_50M]		= &dpll0_50m.hw,
+		[CLK_DPLL1_50M]		= &dpll1_50m.hw,
+		[CLK_CPPLL_50M]		= &cppll_50m.hw,
+		[CLK_M0_39M]		= &m0_39m.hw,
+		[CLK_M1_63M]		= &m1_63m.hw,
+	},
+	.num	= CLK_PLL_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_pll_desc = {
+	.clk_clks	= sc9860_pll_clks,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_pll_clks),
+	.hw_clks	= &sc9860_pll_hws,
+};
+
+#define SC9860_MUX_FLAG	\
+	(CLK_GET_RATE_NOCACHE | CLK_SET_RATE_NO_REPARENT)
+
+static const char * const ap_apb_parents[] = { "ext-26m", "twpll-64m",
+					       "twpll-96m", "twpll-128m" };
+static SPRD_MUX_CLK(ap_apb, "ap-apb", ap_apb_parents,
+		    0x20, 0, 1, SC9860_MUX_FLAG);
+
+static const char * const ap_apb_usb3[] = { "ext-32k", "twpll-24m" };
+static SPRD_MUX_CLK(ap_usb3, "ap-usb3", ap_apb_usb3,
+		    0x2c, 0, 1, SC9860_MUX_FLAG);
+
+static const char * const uart_parents[] = {	"ext-26m",	"twpll-48m",
+						"twpll-51m2",	"twpll-96m" };
+static SPRD_COMP_CLK(uart0_clk,	"uart0",	uart_parents, 0x30,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(uart1_clk,	"uart1",	uart_parents, 0x34,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(uart2_clk,	"uart2",	uart_parents, 0x38,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(uart3_clk,	"uart3",	uart_parents, 0x3c,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(uart4_clk,	"uart4",	uart_parents, 0x40,
+		     0, 2, 8, 3, 0);
+
+static const char * const i2c_parents[] = { "ext-26m", "twpll-48m",
+					    "twpll-51m2", "twpll-153m6" };
+static SPRD_COMP_CLK(i2c0_clk,	"i2c0", i2c_parents, 0x44,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(i2c1_clk,	"i2c1", i2c_parents, 0x48,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(i2c2_clk,	"i2c2", i2c_parents, 0x4c,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(i2c3_clk,	"i2c3", i2c_parents, 0x50,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(i2c4_clk,	"i2c4", i2c_parents, 0x54,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(i2c5_clk,	"i2c5", i2c_parents, 0x58,
+		     0, 2, 8, 3, 0);
+
+static const char * const spi_parents[] = {	"ext-26m",	"twpll-128m",
+						"twpll-153m6",	"twpll-192m" };
+static SPRD_COMP_CLK(spi0_clk,	"spi0",	spi_parents, 0x5c,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(spi1_clk,	"spi1",	spi_parents, 0x60,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(spi2_clk,	"spi2",	spi_parents, 0x64,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(spi3_clk,	"spi3",	spi_parents, 0x68,
+		     0, 2, 8, 3, 0);
+
+static const char * const iis_parents[] = { "ext-26m",
+					    "twpll-128m",
+					    "twpll-153m6" };
+static SPRD_COMP_CLK(iis0_clk,	"iis0",	iis_parents, 0x6c,
+		     0, 2, 8, 6, 0);
+static SPRD_COMP_CLK(iis1_clk,	"iis1",	iis_parents, 0x70,
+		     0, 2, 8, 6, 0);
+static SPRD_COMP_CLK(iis2_clk,	"iis2",	iis_parents, 0x74,
+		     0, 2, 8, 6, 0);
+static SPRD_COMP_CLK(iis3_clk,	"iis3",	iis_parents, 0x78,
+		     0, 2, 8, 6, 0);
+
+static struct sprd_clk_common *sc9860_ap_clks[] = {
+	/* address base is 0x20000000 */
+	&ap_apb.common,
+	&ap_usb3.common,
+	&uart0_clk.common,
+	&uart1_clk.common,
+	&uart2_clk.common,
+	&uart3_clk.common,
+	&uart4_clk.common,
+	&i2c0_clk.common,
+	&i2c1_clk.common,
+	&i2c2_clk.common,
+	&i2c3_clk.common,
+	&i2c4_clk.common,
+	&i2c5_clk.common,
+	&spi0_clk.common,
+	&spi1_clk.common,
+	&spi2_clk.common,
+	&spi3_clk.common,
+	&iis0_clk.common,
+	&iis1_clk.common,
+	&iis2_clk.common,
+	&iis3_clk.common,
+};
+
+static struct clk_hw_onecell_data sc9860_ap_clk_hws = {
+	.hws	= {
+		[CLK_AP_APB]	= &ap_apb.common.hw,
+		[CLK_AP_USB3]	= &ap_usb3.common.hw,
+		[CLK_UART0]	= &uart0_clk.common.hw,
+		[CLK_UART1]	= &uart1_clk.common.hw,
+		[CLK_UART2]	= &uart2_clk.common.hw,
+		[CLK_UART3]	= &uart3_clk.common.hw,
+		[CLK_UART4]	= &uart4_clk.common.hw,
+		[CLK_I2C0]	= &i2c0_clk.common.hw,
+		[CLK_I2C1]	= &i2c1_clk.common.hw,
+		[CLK_I2C2]	= &i2c2_clk.common.hw,
+		[CLK_I2C3]	= &i2c3_clk.common.hw,
+		[CLK_I2C4]	= &i2c4_clk.common.hw,
+		[CLK_I2C5]	= &i2c5_clk.common.hw,
+		[CLK_SPI0]	= &spi0_clk.common.hw,
+		[CLK_SPI1]	= &spi1_clk.common.hw,
+		[CLK_SPI2]	= &spi2_clk.common.hw,
+		[CLK_SPI3]	= &spi3_clk.common.hw,
+		[CLK_IIS0]	= &iis0_clk.common.hw,
+		[CLK_IIS1]	= &iis1_clk.common.hw,
+		[CLK_IIS2]	= &iis2_clk.common.hw,
+		[CLK_IIS3]	= &iis3_clk.common.hw,
+	},
+	.num	= CLK_AP_CLK_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_ap_clk_desc = {
+	.clk_clks	= sc9860_ap_clks,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_ap_clks),
+	.hw_clks	= &sc9860_ap_clk_hws,
+};
+
+static const char * const aon_apb_parents[] = { "rco-25m",	"ext-26m",
+						"ext-rco-100m",	"twpll-96m",
+						"twpll-128m",
+						"twpll-153m6" };
+static SPRD_COMP_CLK(aon_apb, "aon-apb", aon_apb_parents, 0x230,
+		     0, 3, 8, 2, 0);
+
+static const char * const aux_parents[] = { "ext-32k",		"rpll0-26m",
+					    "rpll1-26m",	"ext-26m",
+					    "cppll-50m",	"rco-25m",
+					    "dpll0-50m",	"dpll1-50m",
+					    "gpll-42m5",	"twpll-48m",
+					    "m0-39m",		"m1-63m",
+					    "l0-38m",		"l1-38m" };
+
+static SPRD_COMP_CLK(aux0_clk,	"aux0",		aux_parents, 0x238,
+		     0, 5, 8, 4, 0);
+static SPRD_COMP_CLK(aux1_clk,	"aux1",		aux_parents, 0x23c,
+		     0, 5, 8, 4, 0);
+static SPRD_COMP_CLK(aux2_clk,	"aux2",		aux_parents, 0x240,
+		     0, 5, 8, 4, 0);
+static SPRD_COMP_CLK(probe_clk,	"probe",	aux_parents, 0x244,
+		     0, 5, 8, 4, 0);
+
+static const char * const sp_ahb_parents[] = {	"rco-4m",	"ext-26m",
+						"ext-rco-100m",	"twpll-96m",
+						"twpll-128m",
+						"twpll-153m6" };
+static SPRD_COMP_CLK(sp_ahb,	"sp-ahb",	sp_ahb_parents, 0x2d0,
+		     0, 3, 8, 2, 0);
+
+static const char * const cci_parents[] = {	"ext-26m",	"twpll-384m",
+						"l0-614m4",	"twpll-768m" };
+static SPRD_COMP_CLK(cci_clk,	"cci",		cci_parents, 0x300,
+		     0, 2, 8, 2, 0);
+static SPRD_COMP_CLK(gic_clk,	"gic",		cci_parents, 0x304,
+		     0, 2, 8, 2, 0);
+static SPRD_COMP_CLK(cssys_clk,	"cssys",	cci_parents, 0x310,
+		     0, 2, 8, 2, 0);
+
+static const char * const sdio_2x_parents[] = {	"fac-1m",	"ext-26m",
+						"twpll-307m2",	"twpll-384m",
+						"l0-409m6" };
+static SPRD_COMP_CLK(sdio0_2x,	"sdio0-2x",	sdio_2x_parents, 0x328,
+		     0, 3, 8, 4, 0);
+static SPRD_COMP_CLK(sdio1_2x,	"sdio1-2x",	sdio_2x_parents, 0x330,
+		     0, 3, 8, 4, 0);
+static SPRD_COMP_CLK(sdio2_2x,	"sdio2-2x",	sdio_2x_parents, 0x338,
+		     0, 3, 8, 4, 0);
+static SPRD_COMP_CLK(emmc_2x,	"emmc-2x",	sdio_2x_parents, 0x340,
+		     0, 3, 8, 4, 0);
+
+static SPRD_DIV_CLK(sdio0_1x,	"sdio0-1x",	"sdio0-2x",	0x32c,
+		    8, 1, 0);
+static SPRD_DIV_CLK(sdio1_1x,	"sdio1-1x",	"sdio1-2x",	0x334,
+		    8, 1, 0);
+static SPRD_DIV_CLK(sdio2_1x,	"sdio2-1x",	"sdio2-2x",	0x33c,
+		    8, 1, 0);
+static SPRD_DIV_CLK(emmc_1x,	"emmc-1x",	"emmc-2x",	0x344,
+		    8, 1, 0);
+
+static const char * const adi_parents[] = {	"rco-4m",	"ext-26m",
+						"rco-25m",	"twpll-38m4",
+						"twpll-51m2" };
+static SPRD_MUX_CLK(adi_clk,	"adi",	adi_parents, 0x234,
+		    0, 3, SC9860_MUX_FLAG);
+
+static const char * const pwm_parents[] = {	"ext-32k",	"ext-26m",
+						"rco-4m",	"rco-25m",
+						"twpll-48m" };
+static SPRD_MUX_CLK(pwm0_clk,	"pwm0",	pwm_parents, 0x248,
+		    0, 3, SC9860_MUX_FLAG);
+static SPRD_MUX_CLK(pwm1_clk,	"pwm1",	pwm_parents, 0x24c,
+		    0, 3, SC9860_MUX_FLAG);
+static SPRD_MUX_CLK(pwm2_clk,	"pwm2",	pwm_parents, 0x250,
+		    0, 3, SC9860_MUX_FLAG);
+static SPRD_MUX_CLK(pwm3_clk,	"pwm3",	pwm_parents, 0x254,
+		    0, 3, SC9860_MUX_FLAG);
+
+static const char * const efuse_parents[] = { "rco-25m", "ext-26m" };
+static SPRD_MUX_CLK(efuse_clk, "efuse", efuse_parents, 0x258,
+		    0, 1, SC9860_MUX_FLAG);
+
+static const char * const cm3_uart_parents[] = { "rco-4m",	"ext-26m",
+						 "rco-100m",	"twpll-48m",
+						 "twpll-51m2",	"twpll-96m",
+						 "twpll-128m" };
+static SPRD_MUX_CLK(cm3_uart0, "cm3-uart0", cm3_uart_parents, 0x25c,
+		    0, 3, SC9860_MUX_FLAG);
+static SPRD_MUX_CLK(cm3_uart1, "cm3-uart1", cm3_uart_parents, 0x260,
+		    0, 3, SC9860_MUX_FLAG);
+
+static const char * const thm_parents[] = { "ext-32k", "fac-250k" };
+static SPRD_MUX_CLK(thm_clk,	"thm",	thm_parents, 0x270,
+		    0, 1, SC9860_MUX_FLAG);
+
+static const char * const cm3_i2c_parents[] = {	"rco-4m",
+						"ext-26m",
+						"rco-100m",
+						"twpll-48m",
+						"twpll-51m2",
+						"twpll-153m6" };
+static SPRD_MUX_CLK(cm3_i2c0, "cm3-i2c0", cm3_i2c_parents, 0x274,
+		    0, 3, SC9860_MUX_FLAG);
+static SPRD_MUX_CLK(cm3_i2c1, "cm3-i2c1", cm3_i2c_parents, 0x278,
+		    0, 3, SC9860_MUX_FLAG);
+static SPRD_MUX_CLK(aon_i2c, "aon-i2c",	cm3_i2c_parents, 0x280,
+		    0, 3, SC9860_MUX_FLAG);
+
+static const char * const cm4_spi_parents[] = {	"ext-26m",	"twpll-96m",
+						"rco-100m",	"twpll-128m",
+						"twpll-153m6",	"twpll-192m" };
+static SPRD_MUX_CLK(cm4_spi, "cm4-spi", cm4_spi_parents, 0x27c,
+		    0, 3, SC9860_MUX_FLAG);
+
+static SPRD_MUX_CLK(avs_clk, "avs", uart_parents, 0x284,
+		    0, 2, SC9860_MUX_FLAG);
+
+static const char * const ca53_dap_parents[] = { "ext-26m",	"rco-4m",
+						 "rco-100m",	"twpll-76m8",
+						 "twpll-128m",	"twpll-153m6" };
+static SPRD_MUX_CLK(ca53_dap, "ca53-dap", ca53_dap_parents, 0x288,
+		    0, 3, SC9860_MUX_FLAG);
+
+static const char * const ca53_ts_parents[] = {	"ext-32k", "ext-26m",
+						"clk-twpll-128m",
+						"clk-twpll-153m6" };
+static SPRD_MUX_CLK(ca53_ts, "ca53-ts", ca53_ts_parents, 0x290,
+		    0, 2, SC9860_MUX_FLAG);
+
+static const char * const djtag_tck_parents[] = { "rco-4m", "ext-26m" };
+static SPRD_MUX_CLK(djtag_tck, "djtag-tck", djtag_tck_parents, 0x2c8,
+		    0, 1, SC9860_MUX_FLAG);
+
+static const char * const pmu_parents[] = { "ext-32k", "rco-4m", "clk-4m" };
+static SPRD_MUX_CLK(pmu_clk, "pmu", pmu_parents, 0x2e0,
+		    0, 2, SC9860_MUX_FLAG);
+
+static const char * const pmu_26m_parents[] = { "rco-25m", "ext-26m" };
+static SPRD_MUX_CLK(pmu_26m, "pmu-26m", pmu_26m_parents, 0x2e4,
+		    0, 1, SC9860_MUX_FLAG);
+
+static const char * const debounce_parents[] = { "ext-32k", "rco-4m",
+						 "rco-25m", "ext-26m" };
+static SPRD_MUX_CLK(debounce_clk, "debounce", debounce_parents, 0x2e8,
+		    0, 2, SC9860_MUX_FLAG);
+
+static const char * const otg2_ref_parents[] = { "twpll-12m", "twpll-24m" };
+static SPRD_MUX_CLK(otg2_ref, "otg2-ref", otg2_ref_parents, 0x2f4,
+		    0, 1, SC9860_MUX_FLAG);
+
+static const char * const usb3_ref_parents[] = { "twpll-24m", "twpll-19m2",
+						 "twpll-48m" };
+static SPRD_MUX_CLK(usb3_ref, "usb3-ref", usb3_ref_parents, 0x2f8,
+		    0, 2, SC9860_MUX_FLAG);
+
+static const char * const ap_axi_parents[] = { "ext-26m", "twpll-76m8",
+					       "twpll-128m", "twpll-256m" };
+static SPRD_MUX_CLK(ap_axi, "ap-axi", ap_axi_parents, 0x324,
+		    0, 2, SC9860_MUX_FLAG);
+
+static struct sprd_clk_common *sc9860_aon_prediv[] = {
+	/* address base is 0x402d0000 */
+	&aon_apb.common,
+	&aux0_clk.common,
+	&aux1_clk.common,
+	&aux2_clk.common,
+	&probe_clk.common,
+	&sp_ahb.common,
+	&cci_clk.common,
+	&gic_clk.common,
+	&cssys_clk.common,
+	&sdio0_2x.common,
+	&sdio1_2x.common,
+	&sdio2_2x.common,
+	&emmc_2x.common,
+	&sdio0_1x.common,
+	&sdio1_1x.common,
+	&sdio2_1x.common,
+	&emmc_1x.common,
+	&adi_clk.common,
+	&pwm0_clk.common,
+	&pwm1_clk.common,
+	&pwm2_clk.common,
+	&pwm3_clk.common,
+	&efuse_clk.common,
+	&cm3_uart0.common,
+	&cm3_uart1.common,
+	&thm_clk.common,
+	&cm3_i2c0.common,
+	&cm3_i2c1.common,
+	&cm4_spi.common,
+	&aon_i2c.common,
+	&avs_clk.common,
+	&ca53_dap.common,
+	&ca53_ts.common,
+	&djtag_tck.common,
+	&pmu_clk.common,
+	&pmu_26m.common,
+	&debounce_clk.common,
+	&otg2_ref.common,
+	&usb3_ref.common,
+	&ap_axi.common,
+};
+
+static struct clk_hw_onecell_data sc9860_aon_prediv_hws = {
+	.hws	= {
+		[CLK_AON_APB]		= &aon_apb.common.hw,
+		[CLK_AUX0]		= &aux0_clk.common.hw,
+		[CLK_AUX1]		= &aux1_clk.common.hw,
+		[CLK_AUX2]		= &aux2_clk.common.hw,
+		[CLK_PROBE]		= &probe_clk.common.hw,
+		[CLK_SP_AHB]		= &sp_ahb.common.hw,
+		[CLK_CCI]		= &cci_clk.common.hw,
+		[CLK_GIC]		= &gic_clk.common.hw,
+		[CLK_CSSYS]		= &cssys_clk.common.hw,
+		[CLK_SDIO0_2X]		= &sdio0_2x.common.hw,
+		[CLK_SDIO1_2X]		= &sdio1_2x.common.hw,
+		[CLK_SDIO2_2X]		= &sdio2_2x.common.hw,
+		[CLK_EMMC_2X]		= &emmc_2x.common.hw,
+		[CLK_SDIO0_1X]		= &sdio0_1x.common.hw,
+		[CLK_SDIO1_1X]		= &sdio1_1x.common.hw,
+		[CLK_SDIO2_1X]		= &sdio2_1x.common.hw,
+		[CLK_EMMC_1X]		= &emmc_1x.common.hw,
+		[CLK_ADI]		= &adi_clk.common.hw,
+		[CLK_PWM0]		= &pwm0_clk.common.hw,
+		[CLK_PWM1]		= &pwm1_clk.common.hw,
+		[CLK_PWM2]		= &pwm2_clk.common.hw,
+		[CLK_PWM3]		= &pwm3_clk.common.hw,
+		[CLK_EFUSE]		= &efuse_clk.common.hw,
+		[CLK_CM3_UART0]		= &cm3_uart0.common.hw,
+		[CLK_CM3_UART1]		= &cm3_uart1.common.hw,
+		[CLK_THM]		= &thm_clk.common.hw,
+		[CLK_CM3_I2C0]		= &cm3_i2c0.common.hw,
+		[CLK_CM3_I2C1]		= &cm3_i2c1.common.hw,
+		[CLK_CM4_SPI]		= &cm4_spi.common.hw,
+		[CLK_AON_I2C]		= &aon_i2c.common.hw,
+		[CLK_AVS]		= &avs_clk.common.hw,
+		[CLK_CA53_DAP]		= &ca53_dap.common.hw,
+		[CLK_CA53_TS]		= &ca53_ts.common.hw,
+		[CLK_DJTAG_TCK]		= &djtag_tck.common.hw,
+		[CLK_PMU]		= &pmu_clk.common.hw,
+		[CLK_PMU_26M]		= &pmu_26m.common.hw,
+		[CLK_DEBOUNCE]		= &debounce_clk.common.hw,
+		[CLK_OTG2_REF]		= &otg2_ref.common.hw,
+		[CLK_USB3_REF]		= &usb3_ref.common.hw,
+		[CLK_AP_AXI]		= &ap_axi.common.hw,
+	},
+	.num	= CLK_AON_PREDIV_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_aon_prediv_desc = {
+	.clk_clks	= sc9860_aon_prediv,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_aon_prediv),
+	.hw_clks	= &sc9860_aon_prediv_hws,
+};
+
+static SPRD_SC_GATE_CLK(usb3_eb,		"usb3-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(usb3_suspend,	"usb3-suspend", "ap-axi", 0x0,
+		     0x1000, BIT(3), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(usb3_ref_eb,	"usb3-ref-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(4), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(dma_eb,		"dma-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(5), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(sdio0_eb,		"sdio0-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(7), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(sdio1_eb,		"sdio1-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(8), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(sdio2_eb,		"sdio2-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(9), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(emmc_eb,		"emmc-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(10), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(rom_eb,		"rom-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(12), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(busmon_eb,		"busmon-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(13), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(cc63s_eb,		"cc63s-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(22), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(cc63p_eb,		"cc63p-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(23), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ce0_eb,		"ce0-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(24), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ce1_eb,		"ce1-eb",	"ap-axi", 0x0,
+		     0x1000, BIT(25), CLK_IGNORE_UNUSED, 0);
+
+static struct sprd_clk_common *sc9860_apahb_gate[] = {
+	/* address base is 0x20210000 */
+	&usb3_eb.common,
+	&usb3_suspend.common,
+	&usb3_ref_eb.common,
+	&dma_eb.common,
+	&sdio0_eb.common,
+	&sdio1_eb.common,
+	&sdio2_eb.common,
+	&emmc_eb.common,
+	&rom_eb.common,
+	&busmon_eb.common,
+	&cc63s_eb.common,
+	&cc63p_eb.common,
+	&ce0_eb.common,
+	&ce1_eb.common,
+};
+
+static struct clk_hw_onecell_data sc9860_apahb_gate_hws = {
+	.hws	= {
+		[CLK_USB3_EB]		= &usb3_eb.common.hw,
+		[CLK_USB3_SUSPEND_EB]	= &usb3_suspend.common.hw,
+		[CLK_USB3_REF_EB]	= &usb3_ref_eb.common.hw,
+		[CLK_DMA_EB]		= &dma_eb.common.hw,
+		[CLK_SDIO0_EB]		= &sdio0_eb.common.hw,
+		[CLK_SDIO1_EB]		= &sdio1_eb.common.hw,
+		[CLK_SDIO2_EB]		= &sdio2_eb.common.hw,
+		[CLK_EMMC_EB]		= &emmc_eb.common.hw,
+		[CLK_ROM_EB]		= &rom_eb.common.hw,
+		[CLK_BUSMON_EB]		= &busmon_eb.common.hw,
+		[CLK_CC63S_EB]		= &cc63s_eb.common.hw,
+		[CLK_CC63P_EB]		= &cc63p_eb.common.hw,
+		[CLK_CE0_EB]		= &ce0_eb.common.hw,
+		[CLK_CE1_EB]		= &ce1_eb.common.hw,
+	},
+	.num	= CLK_APAHB_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_apahb_gate_desc = {
+	.clk_clks	= sc9860_apahb_gate,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_apahb_gate),
+	.hw_clks	= &sc9860_apahb_gate_hws,
+};
+
+static SPRD_SC_GATE_CLK(avs_lit_eb,	"avs-lit-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(0), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(avs_big_eb,	"avs-big-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(1), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_intc5_eb,	"ap-intc5-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(gpio_eb,		"gpio-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(3), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pwm0_eb,		"pwm0-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(4), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pwm1_eb,		"pwm1-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(5), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pwm2_eb,		"pwm2-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(6), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pwm3_eb,		"pwm3-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(7), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(kpd_eb,		"kpd-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(8), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aon_sys_eb,	"aon-sys-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(9), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_sys_eb,	"ap-sys-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(10), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aon_tmr_eb,	"aon-tmr-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(11), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_tmr0_eb,	"ap-tmr0-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(12), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(efuse_eb,	"efuse-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(13), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(eic_eb,		"eic-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(14), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pub1_reg_eb,	"pub1-reg-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(15), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(adi_eb,		"adi-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(16), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_intc0_eb,	"ap-intc0-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(17), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_intc1_eb,	"ap-intc1-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(18), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_intc2_eb,	"ap-intc2-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(19), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_intc3_eb,	"ap-intc3-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(20), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_intc4_eb,	"ap-intc4-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(21), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(splk_eb,		"splk-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(22), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(mspi_eb,		"mspi-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(23), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pub0_reg_eb,	"pub0-reg-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(24), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pin_eb,		"pin-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(25), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aon_ckg_eb,	"aon-ckg-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(26), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(gpu_eb,		"gpu-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(27), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(apcpu_ts0_eb,	"apcpu-ts0-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(28), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(apcpu_ts1_eb,	"apcpu-ts1-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(29), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(dap_eb,		"dap-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(30), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(i2c_eb,		"i2c-eb",	"aon-apb", 0x0,
+		     0x1000, BIT(31), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(pmu_eb,		"pmu-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(0), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(thm_eb,		"thm-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(1), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aux0_eb,		"aux0-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aux1_eb,		"aux1-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(3), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aux2_eb,		"aux2-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(4), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(probe_eb,		"probe-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(5), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(gpu0_avs_eb,	"gpu0-avs-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(6), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(gpu1_avs_eb,	"gpu1-avs-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(7), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(apcpu_wdg_eb,	"apcpu-wdg-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(8), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_tmr1_eb,	"ap-tmr1-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(9), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_tmr2_eb,	"ap-tmr2-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(10), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(disp_emc_eb,	"disp-emc-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(11), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(zip_emc_eb,	"zip-emc-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(12), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(gsp_emc_eb,	"gsp-emc-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(13), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(osc_aon_eb,	"osc-aon-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(14), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(lvds_trx_eb,	"lvds-trx-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(15), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(lvds_tcxo_eb,	"lvds-tcxo-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(16), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(mdar_eb,		"mdar-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(17), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(rtc4m0_cal_eb, "rtc4m0-cal-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(18), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(rct100m_cal_eb, "rct100m-cal-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(19), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(djtag_eb,		"djtag-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(20), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(mbox_eb,		"mbox-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(21), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aon_dma_eb,	"aon-dma-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(22), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(dbg_emc_eb,	"dbg-emc-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(23), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(lvds_pll_div_en, "lvds-pll-div-en", "aon-apb", 0x4,
+		     0x1000, BIT(24), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(def_eb,		"def-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(25), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(aon_apb_rsv0,	"aon-apb-rsv0",	"aon-apb", 0x4,
+		     0x1000, BIT(26), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(orp_jtag_eb,	"orp-jtag-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(27), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(vsp_eb,		"vsp-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(28), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(cam_eb,		"cam-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(29), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(disp_eb,		"disp-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(30), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(dbg_axi_if_eb, "dbg-axi-if-eb",	"aon-apb", 0x4,
+		     0x1000, BIT(31), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(sdio0_2x_en,	"sdio0-2x-en",	"aon-apb", 0x13c,
+			       0x1000, BIT(2), 0, 0);
+static SPRD_SC_GATE_CLK(sdio1_2x_en,	"sdio1-2x-en",	"aon-apb", 0x13c,
+			       0x1000, BIT(4), 0, 0);
+static SPRD_SC_GATE_CLK(sdio2_2x_en,	"sdio2-2x-en",	"aon-apb", 0x13c,
+			       0x1000, BIT(6), 0, 0);
+static SPRD_SC_GATE_CLK(emmc_2x_en,	"emmc-2x-en",	"aon-apb", 0x13c,
+			       0x1000, BIT(9), 0, 0);
+
+static struct sprd_clk_common *sc9860_aon_gate[] = {
+	/* address base is 0x402e0000 */
+	&avs_lit_eb.common,
+	&avs_big_eb.common,
+	&ap_intc5_eb.common,
+	&gpio_eb.common,
+	&pwm0_eb.common,
+	&pwm1_eb.common,
+	&pwm2_eb.common,
+	&pwm3_eb.common,
+	&kpd_eb.common,
+	&aon_sys_eb.common,
+	&ap_sys_eb.common,
+	&aon_tmr_eb.common,
+	&ap_tmr0_eb.common,
+	&efuse_eb.common,
+	&eic_eb.common,
+	&pub1_reg_eb.common,
+	&adi_eb.common,
+	&ap_intc0_eb.common,
+	&ap_intc1_eb.common,
+	&ap_intc2_eb.common,
+	&ap_intc3_eb.common,
+	&ap_intc4_eb.common,
+	&splk_eb.common,
+	&mspi_eb.common,
+	&pub0_reg_eb.common,
+	&pin_eb.common,
+	&aon_ckg_eb.common,
+	&gpu_eb.common,
+	&apcpu_ts0_eb.common,
+	&apcpu_ts1_eb.common,
+	&dap_eb.common,
+	&i2c_eb.common,
+	&pmu_eb.common,
+	&thm_eb.common,
+	&aux0_eb.common,
+	&aux1_eb.common,
+	&aux2_eb.common,
+	&probe_eb.common,
+	&gpu0_avs_eb.common,
+	&gpu1_avs_eb.common,
+	&apcpu_wdg_eb.common,
+	&ap_tmr1_eb.common,
+	&ap_tmr2_eb.common,
+	&disp_emc_eb.common,
+	&zip_emc_eb.common,
+	&gsp_emc_eb.common,
+	&osc_aon_eb.common,
+	&lvds_trx_eb.common,
+	&lvds_tcxo_eb.common,
+	&mdar_eb.common,
+	&rtc4m0_cal_eb.common,
+	&rct100m_cal_eb.common,
+	&djtag_eb.common,
+	&mbox_eb.common,
+	&aon_dma_eb.common,
+	&dbg_emc_eb.common,
+	&lvds_pll_div_en.common,
+	&def_eb.common,
+	&aon_apb_rsv0.common,
+	&orp_jtag_eb.common,
+	&vsp_eb.common,
+	&cam_eb.common,
+	&disp_eb.common,
+	&dbg_axi_if_eb.common,
+	&sdio0_2x_en.common,
+	&sdio1_2x_en.common,
+	&sdio2_2x_en.common,
+	&emmc_2x_en.common,
+};
+
+static struct clk_hw_onecell_data sc9860_aon_gate_hws = {
+	.hws	= {
+		[CLK_AVS_LIT_EB]	= &avs_lit_eb.common.hw,
+		[CLK_AVS_BIG_EB]	= &avs_big_eb.common.hw,
+		[CLK_AP_INTC5_EB]	= &ap_intc5_eb.common.hw,
+		[CLK_GPIO_EB]		= &gpio_eb.common.hw,
+		[CLK_PWM0_EB]		= &pwm0_eb.common.hw,
+		[CLK_PWM1_EB]		= &pwm1_eb.common.hw,
+		[CLK_PWM2_EB]		= &pwm2_eb.common.hw,
+		[CLK_PWM3_EB]		= &pwm3_eb.common.hw,
+		[CLK_KPD_EB]		= &kpd_eb.common.hw,
+		[CLK_AON_SYS_EB]	= &aon_sys_eb.common.hw,
+		[CLK_AP_SYS_EB]		= &ap_sys_eb.common.hw,
+		[CLK_AON_TMR_EB]	= &aon_tmr_eb.common.hw,
+		[CLK_AP_TMR0_EB]	= &ap_tmr0_eb.common.hw,
+		[CLK_EFUSE_EB]		= &efuse_eb.common.hw,
+		[CLK_EIC_EB]		= &eic_eb.common.hw,
+		[CLK_PUB1_REG_EB]	= &pub1_reg_eb.common.hw,
+		[CLK_ADI_EB]		= &adi_eb.common.hw,
+		[CLK_AP_INTC0_EB]	= &ap_intc0_eb.common.hw,
+		[CLK_AP_INTC1_EB]	= &ap_intc1_eb.common.hw,
+		[CLK_AP_INTC2_EB]	= &ap_intc2_eb.common.hw,
+		[CLK_AP_INTC3_EB]	= &ap_intc3_eb.common.hw,
+		[CLK_AP_INTC4_EB]	= &ap_intc4_eb.common.hw,
+		[CLK_SPLK_EB]		= &splk_eb.common.hw,
+		[CLK_MSPI_EB]		= &mspi_eb.common.hw,
+		[CLK_PUB0_REG_EB]	= &pub0_reg_eb.common.hw,
+		[CLK_PIN_EB]		= &pin_eb.common.hw,
+		[CLK_AON_CKG_EB]	= &aon_ckg_eb.common.hw,
+		[CLK_GPU_EB]		= &gpu_eb.common.hw,
+		[CLK_APCPU_TS0_EB]	= &apcpu_ts0_eb.common.hw,
+		[CLK_APCPU_TS1_EB]	= &apcpu_ts1_eb.common.hw,
+		[CLK_DAP_EB]		= &dap_eb.common.hw,
+		[CLK_I2C_EB]		= &i2c_eb.common.hw,
+		[CLK_PMU_EB]		= &pmu_eb.common.hw,
+		[CLK_THM_EB]		= &thm_eb.common.hw,
+		[CLK_AUX0_EB]		= &aux0_eb.common.hw,
+		[CLK_AUX1_EB]		= &aux1_eb.common.hw,
+		[CLK_AUX2_EB]		= &aux2_eb.common.hw,
+		[CLK_PROBE_EB]		= &probe_eb.common.hw,
+		[CLK_GPU0_AVS_EB]	= &gpu0_avs_eb.common.hw,
+		[CLK_GPU1_AVS_EB]	= &gpu1_avs_eb.common.hw,
+		[CLK_APCPU_WDG_EB]	= &apcpu_wdg_eb.common.hw,
+		[CLK_AP_TMR1_EB]	= &ap_tmr1_eb.common.hw,
+		[CLK_AP_TMR2_EB]	= &ap_tmr2_eb.common.hw,
+		[CLK_DISP_EMC_EB]	= &disp_emc_eb.common.hw,
+		[CLK_ZIP_EMC_EB]	= &zip_emc_eb.common.hw,
+		[CLK_GSP_EMC_EB]	= &gsp_emc_eb.common.hw,
+		[CLK_OSC_AON_EB]	= &osc_aon_eb.common.hw,
+		[CLK_LVDS_TRX_EB]	= &lvds_trx_eb.common.hw,
+		[CLK_LVDS_TCXO_EB]	= &lvds_tcxo_eb.common.hw,
+		[CLK_MDAR_EB]		= &mdar_eb.common.hw,
+		[CLK_RTC4M0_CAL_EB]	= &rtc4m0_cal_eb.common.hw,
+		[CLK_RCT100M_CAL_EB]	= &rct100m_cal_eb.common.hw,
+		[CLK_DJTAG_EB]		= &djtag_eb.common.hw,
+		[CLK_MBOX_EB]		= &mbox_eb.common.hw,
+		[CLK_AON_DMA_EB]	= &aon_dma_eb.common.hw,
+		[CLK_DBG_EMC_EB]	= &dbg_emc_eb.common.hw,
+		[CLK_LVDS_PLL_DIV_EN]	= &lvds_pll_div_en.common.hw,
+		[CLK_DEF_EB]		= &def_eb.common.hw,
+		[CLK_AON_APB_RSV0]	= &aon_apb_rsv0.common.hw,
+		[CLK_ORP_JTAG_EB]	= &orp_jtag_eb.common.hw,
+		[CLK_VSP_EB]		= &vsp_eb.common.hw,
+		[CLK_CAM_EB]		= &cam_eb.common.hw,
+		[CLK_DISP_EB]		= &disp_eb.common.hw,
+		[CLK_DBG_AXI_IF_EB]	= &dbg_axi_if_eb.common.hw,
+		[CLK_SDIO0_2X_EN]	= &sdio0_2x_en.common.hw,
+		[CLK_SDIO1_2X_EN]	= &sdio1_2x_en.common.hw,
+		[CLK_SDIO2_2X_EN]	= &sdio2_2x_en.common.hw,
+		[CLK_EMMC_2X_EN]	= &emmc_2x_en.common.hw,
+	},
+	.num	= CLK_AON_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_aon_gate_desc = {
+	.clk_clks	= sc9860_aon_gate,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_aon_gate),
+	.hw_clks	= &sc9860_aon_gate_hws,
+};
+
+static const u8 mcu_table[] = { 0, 1, 2, 3, 4, 8 };
+static const char * const lit_mcu_parents[] = {	"ext-26m",	"twpll-512m",
+						"twpll-768m",	"ltepll0",
+						"twpll",	"mpll0" };
+static SPRD_COMP_CLK_TABLE(lit_mcu, "lit-mcu", lit_mcu_parents, 0x20,
+			   mcu_table, 0, 4, 4, 3, 0);
+
+static const char * const big_mcu_parents[] = {	"ext-26m",	"twpll-512m",
+						"twpll-768m",	"ltepll0",
+						"twpll",	"mpll1" };
+static SPRD_COMP_CLK_TABLE(big_mcu, "big-mcu", big_mcu_parents, 0x24,
+			   mcu_table, 0, 4, 4, 3, 0);
+
+static struct sprd_clk_common *sc9860_aonsecure_clk[] = {
+	/* address base is 0x40880000 */
+	&lit_mcu.common,
+	&big_mcu.common,
+};
+
+static struct clk_hw_onecell_data sc9860_aonsecure_clk_hws = {
+	.hws	= {
+		[CLK_LIT_MCU]		= &lit_mcu.common.hw,
+		[CLK_BIG_MCU]		= &big_mcu.common.hw,
+	},
+	.num	= CLK_AONSECURE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_aonsecure_clk_desc = {
+	.clk_clks	= sc9860_aonsecure_clk,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_aonsecure_clk),
+	.hw_clks	= &sc9860_aonsecure_clk_hws,
+};
+
+static SPRD_SC_GATE_CLK(agcp_iis0_eb,	"agcp-iis0-eb",		"aon-apb",
+		     0x0, 0x100, BIT(0), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_iis1_eb,	"agcp-iis1-eb",		"aon-apb",
+		     0x0, 0x100, BIT(1), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_iis2_eb,	"agcp-iis2-eb",		"aon-apb",
+		     0x0, 0x100, BIT(2), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_iis3_eb,	"agcp-iis3-eb",		"aon-apb",
+		     0x0, 0x100, BIT(3), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_uart_eb,	"agcp-uart-eb",		"aon-apb",
+		     0x0, 0x100, BIT(4), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_dmacp_eb,	"agcp-dmacp-eb",	"aon-apb",
+		     0x0, 0x100, BIT(5), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_dmaap_eb,	"agcp-dmaap-eb",	"aon-apb",
+		     0x0, 0x100, BIT(6), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_arc48k_eb,	"agcp-arc48k-eb",	"aon-apb",
+		     0x0, 0x100, BIT(10), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_src44p1k_eb, "agcp-src44p1k-eb",	"aon-apb",
+		     0x0, 0x100, BIT(11), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_mcdt_eb,	"agcp-mcdt-eb",		"aon-apb",
+		     0x0, 0x100, BIT(12), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_vbcifd_eb,	"agcp-vbcifd-eb",	"aon-apb",
+		     0x0, 0x100, BIT(13), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_vbc_eb,	"agcp-vbc-eb",		"aon-apb",
+		     0x0, 0x100, BIT(14), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_spinlock_eb, "agcp-spinlock-eb",	"aon-apb",
+		     0x0, 0x100, BIT(15), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_icu_eb,	"agcp-icu-eb",		"aon-apb",
+		     0x0, 0x100, BIT(16), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(agcp_ap_ashb_eb, "agcp-ap-ashb-eb",	"aon-apb",
+		     0x0, 0x100, BIT(17), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_cp_ashb_eb, "agcp-cp-ashb-eb",	"aon-apb",
+		     0x0, 0x100, BIT(18), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_aud_eb,	"agcp-aud-eb",		"aon-apb",
+		     0x0, 0x100, BIT(19), 0, 0);
+static SPRD_SC_GATE_CLK(agcp_audif_eb,	"agcp-audif-eb",	"aon-apb",
+		     0x0, 0x100, BIT(20), 0, 0);
+
+static struct sprd_clk_common *sc9860_agcp_gate[] = {
+	/* address base is 0x415e0000 */
+	&agcp_iis0_eb.common,
+	&agcp_iis1_eb.common,
+	&agcp_iis2_eb.common,
+	&agcp_iis3_eb.common,
+	&agcp_uart_eb.common,
+	&agcp_dmacp_eb.common,
+	&agcp_dmaap_eb.common,
+	&agcp_arc48k_eb.common,
+	&agcp_src44p1k_eb.common,
+	&agcp_mcdt_eb.common,
+	&agcp_vbcifd_eb.common,
+	&agcp_vbc_eb.common,
+	&agcp_spinlock_eb.common,
+	&agcp_icu_eb.common,
+	&agcp_ap_ashb_eb.common,
+	&agcp_cp_ashb_eb.common,
+	&agcp_aud_eb.common,
+	&agcp_audif_eb.common,
+};
+
+static struct clk_hw_onecell_data sc9860_agcp_gate_hws = {
+	.hws	= {
+		[CLK_AGCP_IIS0_EB]	= &agcp_iis0_eb.common.hw,
+		[CLK_AGCP_IIS1_EB]	= &agcp_iis1_eb.common.hw,
+		[CLK_AGCP_IIS2_EB]	= &agcp_iis2_eb.common.hw,
+		[CLK_AGCP_IIS3_EB]	= &agcp_iis3_eb.common.hw,
+		[CLK_AGCP_UART_EB]	= &agcp_uart_eb.common.hw,
+		[CLK_AGCP_DMACP_EB]	= &agcp_dmacp_eb.common.hw,
+		[CLK_AGCP_DMAAP_EB]	= &agcp_dmaap_eb.common.hw,
+		[CLK_AGCP_ARC48K_EB]	= &agcp_arc48k_eb.common.hw,
+		[CLK_AGCP_SRC44P1K_EB]	= &agcp_src44p1k_eb.common.hw,
+		[CLK_AGCP_MCDT_EB]	= &agcp_mcdt_eb.common.hw,
+		[CLK_AGCP_VBCIFD_EB]	= &agcp_vbcifd_eb.common.hw,
+		[CLK_AGCP_VBC_EB]	= &agcp_vbc_eb.common.hw,
+		[CLK_AGCP_SPINLOCK_EB]	= &agcp_spinlock_eb.common.hw,
+		[CLK_AGCP_ICU_EB]	= &agcp_icu_eb.common.hw,
+		[CLK_AGCP_AP_ASHB_EB]	= &agcp_ap_ashb_eb.common.hw,
+		[CLK_AGCP_CP_ASHB_EB]	= &agcp_cp_ashb_eb.common.hw,
+		[CLK_AGCP_AUD_EB]	= &agcp_aud_eb.common.hw,
+		[CLK_AGCP_AUDIF_EB]	= &agcp_audif_eb.common.hw,
+	},
+	.num	= CLK_AGCP_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_agcp_gate_desc = {
+	.clk_clks	= sc9860_agcp_gate,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_agcp_gate),
+	.hw_clks	= &sc9860_agcp_gate_hws,
+};
+
+static const char * const gpu_parents[] = { "twpll-512m",
+					    "twpll-768m",
+					    "gpll" };
+static SPRD_COMP_CLK(gpu_clk,	"gpu",	gpu_parents, 0x20,
+		     0, 2, 8, 4, 0);
+
+static struct sprd_clk_common *sc9860_gpu_clk[] = {
+	/* address base is 0x60200000 */
+	&gpu_clk.common,
+};
+
+static struct clk_hw_onecell_data sc9860_gpu_clk_hws = {
+	.hws	= {
+		[CLK_GPU]	= &gpu_clk.common.hw,
+	},
+	.num	= CLK_GPU_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_gpu_clk_desc = {
+	.clk_clks	= sc9860_gpu_clk,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_gpu_clk),
+	.hw_clks	= &sc9860_gpu_clk_hws,
+};
+
+static const char * const ahb_parents[] = { "ext-26m", "twpll-96m",
+					    "twpll-128m", "twpll-153m6" };
+static SPRD_MUX_CLK(ahb_vsp, "ahb-vsp", ahb_parents, 0x20,
+		    0, 2, SC9860_MUX_FLAG);
+
+static const char * const vsp_parents[] = {	"twpll-76m8",	"twpll-128m",
+						"twpll-256m",	"twpll-307m2",
+						"twpll-384m" };
+static SPRD_COMP_CLK(vsp_clk, "vsp", vsp_parents, 0x24, 0, 3, 8, 2, 0);
+
+static const char * const dispc_parents[] = {	"twpll-76m8",	"twpll-128m",
+						"twpll-256m",	"twpll-307m2" };
+static SPRD_COMP_CLK(vsp_enc, "vsp-enc", dispc_parents, 0x28, 0, 2, 8, 2, 0);
+
+static const char * const vpp_parents[] = { "twpll-96m", "twpll-153m6",
+					    "twpll-192m", "twpll-256m" };
+static SPRD_MUX_CLK(vpp_clk, "vpp", vpp_parents, 0x2c,
+		    0, 2, SC9860_MUX_FLAG);
+static const char * const vsp_26m_parents[] = { "ext-26m" };
+static SPRD_MUX_CLK(vsp_26m, "vsp-26m", vsp_26m_parents, 0x30,
+		    0, 1, SC9860_MUX_FLAG);
+
+static struct sprd_clk_common *sc9860_vsp_clk[] = {
+	/* address base is 0x61000000 */
+	&ahb_vsp.common,
+	&vsp_clk.common,
+	&vsp_enc.common,
+	&vpp_clk.common,
+	&vsp_26m.common,
+};
+
+static struct clk_hw_onecell_data sc9860_vsp_clk_hws = {
+	.hws	= {
+		[CLK_AHB_VSP]	= &ahb_vsp.common.hw,
+		[CLK_VSP]	= &vsp_clk.common.hw,
+		[CLK_VSP_ENC]	= &vsp_enc.common.hw,
+		[CLK_VPP]	= &vpp_clk.common.hw,
+		[CLK_VSP_26M]	= &vsp_26m.common.hw,
+	},
+	.num	= CLK_VSP_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_vsp_clk_desc = {
+	.clk_clks	= sc9860_vsp_clk,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_vsp_clk),
+	.hw_clks	= &sc9860_vsp_clk_hws,
+};
+
+static SPRD_SC_GATE_CLK(vsp_dec_eb,	"vsp-dec-eb",	"ahb-vsp", 0x0,
+		     0x1000, BIT(0), 0, 0);
+static SPRD_SC_GATE_CLK(vsp_ckg_eb,	"vsp-ckg-eb",	"ahb-vsp", 0x0,
+		     0x1000, BIT(1), 0, 0);
+static SPRD_SC_GATE_CLK(vsp_mmu_eb,	"vsp-mmu-eb",	"ahb-vsp", 0x0,
+		     0x1000, BIT(2), 0, 0);
+static SPRD_SC_GATE_CLK(vsp_enc_eb,	"vsp-enc-eb",	"ahb-vsp", 0x0,
+		     0x1000, BIT(3), 0, 0);
+static SPRD_SC_GATE_CLK(vpp_eb,		"vpp-eb",	"ahb-vsp", 0x0,
+		     0x1000, BIT(4), 0, 0);
+static SPRD_SC_GATE_CLK(vsp_26m_eb,	"vsp-26m-eb",	"ahb-vsp", 0x0,
+		     0x1000, BIT(5), 0, 0);
+static SPRD_GATE_CLK(vsp_axi_gate,	"vsp-axi-gate",	"ahb-vsp", 0x8,
+		     BIT(0), 0, 0);
+static SPRD_GATE_CLK(vsp_enc_gate,	"vsp-enc-gate",	"ahb-vsp", 0x8,
+		     BIT(1), 0, 0);
+static SPRD_GATE_CLK(vpp_axi_gate,	"vpp-axi-gate",	"ahb-vsp", 0x8,
+		     BIT(2), 0, 0);
+static SPRD_GATE_CLK(vsp_bm_gate,	"vsp-bm-gate",	"ahb-vsp", 0x8,
+		     BIT(8), 0, 0);
+static SPRD_GATE_CLK(vsp_enc_bm_gate, "vsp-enc-bm-gate", "ahb-vsp", 0x8,
+		     BIT(9), 0, 0);
+static SPRD_GATE_CLK(vpp_bm_gate,	"vpp-bm-gate",	"ahb-vsp", 0x8,
+		     BIT(10), 0, 0);
+
+static struct sprd_clk_common *sc9860_vsp_gate[] = {
+	/* address base is 0x61100000 */
+	&vsp_dec_eb.common,
+	&vsp_ckg_eb.common,
+	&vsp_mmu_eb.common,
+	&vsp_enc_eb.common,
+	&vpp_eb.common,
+	&vsp_26m_eb.common,
+	&vsp_axi_gate.common,
+	&vsp_enc_gate.common,
+	&vpp_axi_gate.common,
+	&vsp_bm_gate.common,
+	&vsp_enc_bm_gate.common,
+	&vpp_bm_gate.common,
+};
+
+static struct clk_hw_onecell_data sc9860_vsp_gate_hws = {
+	.hws	= {
+		[CLK_VSP_DEC_EB]	= &vsp_dec_eb.common.hw,
+		[CLK_VSP_CKG_EB]	= &vsp_ckg_eb.common.hw,
+		[CLK_VSP_MMU_EB]	= &vsp_mmu_eb.common.hw,
+		[CLK_VSP_ENC_EB]	= &vsp_enc_eb.common.hw,
+		[CLK_VPP_EB]		= &vpp_eb.common.hw,
+		[CLK_VSP_26M_EB]	= &vsp_26m_eb.common.hw,
+		[CLK_VSP_AXI_GATE]	= &vsp_axi_gate.common.hw,
+		[CLK_VSP_ENC_GATE]	= &vsp_enc_gate.common.hw,
+		[CLK_VPP_AXI_GATE]	= &vpp_axi_gate.common.hw,
+		[CLK_VSP_BM_GATE]	= &vsp_bm_gate.common.hw,
+		[CLK_VSP_ENC_BM_GATE]	= &vsp_enc_bm_gate.common.hw,
+		[CLK_VPP_BM_GATE]	= &vpp_bm_gate.common.hw,
+	},
+	.num	= CLK_VSP_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_vsp_gate_desc = {
+	.clk_clks	= sc9860_vsp_gate,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_vsp_gate),
+	.hw_clks	= &sc9860_vsp_gate_hws,
+};
+
+static SPRD_MUX_CLK(ahb_cam, "ahb-cam", ahb_parents, 0x20,
+		    0, 2, SC9860_MUX_FLAG);
+static const char * const sensor_parents[] = {	"ext-26m",	"twpll-48m",
+						"twpll-76m8",	"twpll-96m" };
+static SPRD_COMP_CLK(sensor0_clk, "sensor0", sensor_parents, 0x24,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(sensor1_clk, "sensor1", sensor_parents, 0x28,
+		     0, 2, 8, 3, 0);
+static SPRD_COMP_CLK(sensor2_clk, "sensor2", sensor_parents, 0x2c,
+		     0, 2, 8, 3, 0);
+static SPRD_GATE_CLK(mipi_csi0_eb, "mipi-csi0-eb", "ahb-cam", 0x4c,
+		     BIT(16), 0, 0);
+static SPRD_GATE_CLK(mipi_csi1_eb, "mipi-csi1-eb", "ahb-cam", 0x50,
+		     BIT(16), 0, 0);
+
+static struct sprd_clk_common *sc9860_cam_clk[] = {
+	/* address base is 0x62000000 */
+	&ahb_cam.common,
+	&sensor0_clk.common,
+	&sensor1_clk.common,
+	&sensor2_clk.common,
+	&mipi_csi0_eb.common,
+	&mipi_csi1_eb.common,
+};
+
+static struct clk_hw_onecell_data sc9860_cam_clk_hws = {
+	.hws	= {
+		[CLK_AHB_CAM]		= &ahb_cam.common.hw,
+		[CLK_SENSOR0]		= &sensor0_clk.common.hw,
+		[CLK_SENSOR1]		= &sensor1_clk.common.hw,
+		[CLK_SENSOR2]		= &sensor2_clk.common.hw,
+		[CLK_MIPI_CSI0_EB]	= &mipi_csi0_eb.common.hw,
+		[CLK_MIPI_CSI1_EB]	= &mipi_csi1_eb.common.hw,
+	},
+	.num	= CLK_CAM_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_cam_clk_desc = {
+	.clk_clks	= sc9860_cam_clk,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_cam_clk),
+	.hw_clks	= &sc9860_cam_clk_hws,
+};
+
+static SPRD_SC_GATE_CLK(dcam0_eb,		"dcam0-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(0), 0, 0);
+static SPRD_SC_GATE_CLK(dcam1_eb,		"dcam1-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(1), 0, 0);
+static SPRD_SC_GATE_CLK(isp0_eb,		"isp0-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(2), 0, 0);
+static SPRD_SC_GATE_CLK(csi0_eb,		"csi0-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(3), 0, 0);
+static SPRD_SC_GATE_CLK(csi1_eb,		"csi1-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(4), 0, 0);
+static SPRD_SC_GATE_CLK(jpg0_eb,		"jpg0-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(5), 0, 0);
+static SPRD_SC_GATE_CLK(jpg1_eb,		"jpg1-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(6), 0, 0);
+static SPRD_SC_GATE_CLK(cam_ckg_eb,	"cam-ckg-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(7), 0, 0);
+static SPRD_SC_GATE_CLK(cam_mmu_eb,	"cam-mmu-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(8), 0, 0);
+static SPRD_SC_GATE_CLK(isp1_eb,		"isp1-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(9), 0, 0);
+static SPRD_SC_GATE_CLK(cpp_eb,		"cpp-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(10), 0, 0);
+static SPRD_SC_GATE_CLK(mmu_pf_eb,		"mmu-pf-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(11), 0, 0);
+static SPRD_SC_GATE_CLK(isp2_eb,		"isp2-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(12), 0, 0);
+static SPRD_SC_GATE_CLK(dcam2isp_if_eb, "dcam2isp-if-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(13), 0, 0);
+static SPRD_SC_GATE_CLK(isp2dcam_if_eb, "isp2dcam-if-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(14), 0, 0);
+static SPRD_SC_GATE_CLK(isp_lclk_eb,	"isp-lclk-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(15), 0, 0);
+static SPRD_SC_GATE_CLK(isp_iclk_eb,	"isp-iclk-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(16), 0, 0);
+static SPRD_SC_GATE_CLK(isp_mclk_eb,	"isp-mclk-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(17), 0, 0);
+static SPRD_SC_GATE_CLK(isp_pclk_eb,	"isp-pclk-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(18), 0, 0);
+static SPRD_SC_GATE_CLK(isp_isp2dcam_eb, "isp-isp2dcam-eb", "ahb-cam", 0x0,
+		     0x1000, BIT(19), 0, 0);
+static SPRD_SC_GATE_CLK(dcam0_if_eb,	"dcam0-if-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(20), 0, 0);
+static SPRD_SC_GATE_CLK(clk26m_if_eb,	"clk26m-if-eb",	"ahb-cam", 0x0,
+		     0x1000, BIT(21), 0, 0);
+static SPRD_GATE_CLK(cphy0_gate, "cphy0-gate", "ahb-cam", 0x8,
+		     BIT(0), 0, 0);
+static SPRD_GATE_CLK(mipi_csi0_gate, "mipi-csi0-gate", "ahb-cam", 0x8,
+		     BIT(1), 0, 0);
+static SPRD_GATE_CLK(cphy1_gate,	"cphy1-gate",	"ahb-cam", 0x8,
+		     BIT(2), 0, 0);
+static SPRD_GATE_CLK(mipi_csi1,		"mipi-csi1",	"ahb-cam", 0x8,
+		     BIT(3), 0, 0);
+static SPRD_GATE_CLK(dcam0_axi_gate,	"dcam0-axi-gate", "ahb-cam", 0x8,
+		     BIT(4), 0, 0);
+static SPRD_GATE_CLK(dcam1_axi_gate,	"dcam1-axi-gate", "ahb-cam", 0x8,
+		     BIT(5), 0, 0);
+static SPRD_GATE_CLK(sensor0_gate,	"sensor0-gate",	"ahb-cam", 0x8,
+		     BIT(6), 0, 0);
+static SPRD_GATE_CLK(sensor1_gate,	"sensor1-gate",	"ahb-cam", 0x8,
+		     BIT(7), 0, 0);
+static SPRD_GATE_CLK(jpg0_axi_gate,	"jpg0-axi-gate", "ahb-cam", 0x8,
+		     BIT(8), 0, 0);
+static SPRD_GATE_CLK(gpg1_axi_gate,	"gpg1-axi-gate", "ahb-cam", 0x8,
+		     BIT(9), 0, 0);
+static SPRD_GATE_CLK(isp0_axi_gate,	"isp0-axi-gate", "ahb-cam", 0x8,
+		     BIT(10), 0, 0);
+static SPRD_GATE_CLK(isp1_axi_gate,	"isp1-axi-gate", "ahb-cam", 0x8,
+		     BIT(11), 0, 0);
+static SPRD_GATE_CLK(isp2_axi_gate,	"isp2-axi-gate", "ahb-cam", 0x8,
+		     BIT(12), 0, 0);
+static SPRD_GATE_CLK(cpp_axi_gate,	"cpp-axi-gate",	"ahb-cam", 0x8,
+		     BIT(13), 0, 0);
+static SPRD_GATE_CLK(d0_if_axi_gate,	"d0-if-axi-gate", "ahb-cam", 0x8,
+		     BIT(14), 0, 0);
+static SPRD_GATE_CLK(d2i_if_axi_gate, "d2i-if-axi-gate", "ahb-cam", 0x8,
+		     BIT(15), 0, 0);
+static SPRD_GATE_CLK(i2d_if_axi_gate, "i2d-if-axi-gate", "ahb-cam", 0x8,
+		     BIT(16), 0, 0);
+static SPRD_GATE_CLK(spare_axi_gate, "spare-axi-gate",	"ahb-cam", 0x8,
+		     BIT(17), 0, 0);
+static SPRD_GATE_CLK(sensor2_gate, "sensor2-gate",	"ahb-cam", 0x8,
+		     BIT(18), 0, 0);
+static SPRD_SC_GATE_CLK(d0if_in_d_en, "d0if-in-d-en", "ahb-cam", 0x28,
+		     0x1000, BIT(0), 0, 0);
+static SPRD_SC_GATE_CLK(d1if_in_d_en, "d1if-in-d-en", "ahb-cam", 0x28,
+		     0x1000, BIT(1), 0, 0);
+static SPRD_SC_GATE_CLK(d0if_in_d2i_en, "d0if-in-d2i-en", "ahb-cam", 0x28,
+		     0x1000, BIT(2), 0, 0);
+static SPRD_SC_GATE_CLK(d1if_in_d2i_en, "d1if-in-d2i-en",	"ahb-cam", 0x28,
+		     0x1000, BIT(3), 0, 0);
+static SPRD_SC_GATE_CLK(ia_in_d2i_en, "ia-in-d2i-en",	"ahb-cam", 0x28,
+		     0x1000, BIT(4), 0, 0);
+static SPRD_SC_GATE_CLK(ib_in_d2i_en,	"ib-in-d2i-en",	"ahb-cam", 0x28,
+		     0x1000, BIT(5), 0, 0);
+static SPRD_SC_GATE_CLK(ic_in_d2i_en,	"ic-in-d2i-en",	"ahb-cam", 0x28,
+		     0x1000, BIT(6), 0, 0);
+static SPRD_SC_GATE_CLK(ia_in_i_en,	"ia-in-i-en",	"ahb-cam", 0x28,
+		     0x1000, BIT(7), 0, 0);
+static SPRD_SC_GATE_CLK(ib_in_i_en,	"ib-in-i-en",	"ahb-cam", 0x28,
+		     0x1000, BIT(8), 0, 0);
+static SPRD_SC_GATE_CLK(ic_in_i_en,	"ic-in-i-en",	"ahb-cam", 0x28,
+		     0x1000, BIT(9), 0, 0);
+
+static struct sprd_clk_common *sc9860_cam_gate[] = {
+	/* address base is 0x62100000 */
+	&dcam0_eb.common,
+	&dcam1_eb.common,
+	&isp0_eb.common,
+	&csi0_eb.common,
+	&csi1_eb.common,
+	&jpg0_eb.common,
+	&jpg1_eb.common,
+	&cam_ckg_eb.common,
+	&cam_mmu_eb.common,
+	&isp1_eb.common,
+	&cpp_eb.common,
+	&mmu_pf_eb.common,
+	&isp2_eb.common,
+	&dcam2isp_if_eb.common,
+	&isp2dcam_if_eb.common,
+	&isp_lclk_eb.common,
+	&isp_iclk_eb.common,
+	&isp_mclk_eb.common,
+	&isp_pclk_eb.common,
+	&isp_isp2dcam_eb.common,
+	&dcam0_if_eb.common,
+	&clk26m_if_eb.common,
+	&cphy0_gate.common,
+	&mipi_csi0_gate.common,
+	&cphy1_gate.common,
+	&mipi_csi1.common,
+	&dcam0_axi_gate.common,
+	&dcam1_axi_gate.common,
+	&sensor0_gate.common,
+	&sensor1_gate.common,
+	&jpg0_axi_gate.common,
+	&gpg1_axi_gate.common,
+	&isp0_axi_gate.common,
+	&isp1_axi_gate.common,
+	&isp2_axi_gate.common,
+	&cpp_axi_gate.common,
+	&d0_if_axi_gate.common,
+	&d2i_if_axi_gate.common,
+	&i2d_if_axi_gate.common,
+	&spare_axi_gate.common,
+	&sensor2_gate.common,
+	&d0if_in_d_en.common,
+	&d1if_in_d_en.common,
+	&d0if_in_d2i_en.common,
+	&d1if_in_d2i_en.common,
+	&ia_in_d2i_en.common,
+	&ib_in_d2i_en.common,
+	&ic_in_d2i_en.common,
+	&ia_in_i_en.common,
+	&ib_in_i_en.common,
+	&ic_in_i_en.common,
+};
+
+static struct clk_hw_onecell_data sc9860_cam_gate_hws = {
+	.hws	= {
+		[CLK_DCAM0_EB]		= &dcam0_eb.common.hw,
+		[CLK_DCAM1_EB]		= &dcam1_eb.common.hw,
+		[CLK_ISP0_EB]		= &isp0_eb.common.hw,
+		[CLK_CSI0_EB]		= &csi0_eb.common.hw,
+		[CLK_CSI1_EB]		= &csi1_eb.common.hw,
+		[CLK_JPG0_EB]		= &jpg0_eb.common.hw,
+		[CLK_JPG1_EB]		= &jpg1_eb.common.hw,
+		[CLK_CAM_CKG_EB]	= &cam_ckg_eb.common.hw,
+		[CLK_CAM_MMU_EB]	= &cam_mmu_eb.common.hw,
+		[CLK_ISP1_EB]		= &isp1_eb.common.hw,
+		[CLK_CPP_EB]		= &cpp_eb.common.hw,
+		[CLK_MMU_PF_EB]		= &mmu_pf_eb.common.hw,
+		[CLK_ISP2_EB]		= &isp2_eb.common.hw,
+		[CLK_DCAM2ISP_IF_EB]	= &dcam2isp_if_eb.common.hw,
+		[CLK_ISP2DCAM_IF_EB]	= &isp2dcam_if_eb.common.hw,
+		[CLK_ISP_LCLK_EB]	= &isp_lclk_eb.common.hw,
+		[CLK_ISP_ICLK_EB]	= &isp_iclk_eb.common.hw,
+		[CLK_ISP_MCLK_EB]	= &isp_mclk_eb.common.hw,
+		[CLK_ISP_PCLK_EB]	= &isp_pclk_eb.common.hw,
+		[CLK_ISP_ISP2DCAM_EB]	= &isp_isp2dcam_eb.common.hw,
+		[CLK_DCAM0_IF_EB]	= &dcam0_if_eb.common.hw,
+		[CLK_CLK26M_IF_EB]	= &clk26m_if_eb.common.hw,
+		[CLK_CPHY0_GATE]	= &cphy0_gate.common.hw,
+		[CLK_MIPI_CSI0_GATE]	= &mipi_csi0_gate.common.hw,
+		[CLK_CPHY1_GATE]	= &cphy1_gate.common.hw,
+		[CLK_MIPI_CSI1]		= &mipi_csi1.common.hw,
+		[CLK_DCAM0_AXI_GATE]	= &dcam0_axi_gate.common.hw,
+		[CLK_DCAM1_AXI_GATE]	= &dcam1_axi_gate.common.hw,
+		[CLK_SENSOR0_GATE]	= &sensor0_gate.common.hw,
+		[CLK_SENSOR1_GATE]	= &sensor1_gate.common.hw,
+		[CLK_JPG0_AXI_GATE]	= &jpg0_axi_gate.common.hw,
+		[CLK_GPG1_AXI_GATE]	= &gpg1_axi_gate.common.hw,
+		[CLK_ISP0_AXI_GATE]	= &isp0_axi_gate.common.hw,
+		[CLK_ISP1_AXI_GATE]	= &isp1_axi_gate.common.hw,
+		[CLK_ISP2_AXI_GATE]	= &isp2_axi_gate.common.hw,
+		[CLK_CPP_AXI_GATE]	= &cpp_axi_gate.common.hw,
+		[CLK_D0_IF_AXI_GATE]	= &d0_if_axi_gate.common.hw,
+		[CLK_D2I_IF_AXI_GATE]	= &d2i_if_axi_gate.common.hw,
+		[CLK_I2D_IF_AXI_GATE]	= &i2d_if_axi_gate.common.hw,
+		[CLK_SPARE_AXI_GATE]	= &spare_axi_gate.common.hw,
+		[CLK_SENSOR2_GATE]	= &sensor2_gate.common.hw,
+		[CLK_D0IF_IN_D_EN]	= &d0if_in_d_en.common.hw,
+		[CLK_D1IF_IN_D_EN]	= &d1if_in_d_en.common.hw,
+		[CLK_D0IF_IN_D2I_EN]	= &d0if_in_d2i_en.common.hw,
+		[CLK_D1IF_IN_D2I_EN]	= &d1if_in_d2i_en.common.hw,
+		[CLK_IA_IN_D2I_EN]	= &ia_in_d2i_en.common.hw,
+		[CLK_IB_IN_D2I_EN]	= &ib_in_d2i_en.common.hw,
+		[CLK_IC_IN_D2I_EN]	= &ic_in_d2i_en.common.hw,
+		[CLK_IA_IN_I_EN]	= &ia_in_i_en.common.hw,
+		[CLK_IB_IN_I_EN]	= &ib_in_i_en.common.hw,
+		[CLK_IC_IN_I_EN]	= &ic_in_i_en.common.hw,
+	},
+	.num	= CLK_CAM_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_cam_gate_desc = {
+	.clk_clks	= sc9860_cam_gate,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_cam_gate),
+	.hw_clks	= &sc9860_cam_gate_hws,
+};
+
+static SPRD_MUX_CLK(ahb_disp, "ahb-disp", ahb_parents, 0x20,
+		    0, 2, SC9860_MUX_FLAG);
+static SPRD_COMP_CLK(dispc0_dpi, "dispc0-dpi", dispc_parents,	0x34,
+		     0, 2, 8, 2, 0);
+static SPRD_COMP_CLK(dispc1_dpi, "dispc1-dpi", dispc_parents,	0x40,
+		     0, 2, 8, 2, 0);
+
+static struct sprd_clk_common *sc9860_disp_clk[] = {
+	/* address base is 0x63000000 */
+	&ahb_disp.common,
+	&dispc0_dpi.common,
+	&dispc1_dpi.common,
+};
+
+static struct clk_hw_onecell_data sc9860_disp_clk_hws = {
+	.hws	= {
+		[CLK_AHB_DISP]		= &ahb_disp.common.hw,
+		[CLK_DISPC0_DPI]	= &dispc0_dpi.common.hw,
+		[CLK_DISPC1_DPI]	= &dispc1_dpi.common.hw,
+	},
+	.num	= CLK_DISP_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_disp_clk_desc = {
+	.clk_clks	= sc9860_disp_clk,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_disp_clk),
+	.hw_clks	= &sc9860_disp_clk_hws,
+};
+
+static SPRD_SC_GATE_CLK(dispc0_eb,	"dispc0-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(0), 0, 0);
+static SPRD_SC_GATE_CLK(dispc1_eb,	"dispc1-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(1), 0, 0);
+static SPRD_SC_GATE_CLK(dispc_mmu_eb,	"dispc-mmu-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(2), 0, 0);
+static SPRD_SC_GATE_CLK(gsp0_eb,		"gsp0-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(3), 0, 0);
+static SPRD_SC_GATE_CLK(gsp1_eb,		"gsp1-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(4), 0, 0);
+static SPRD_SC_GATE_CLK(gsp0_mmu_eb,	"gsp0-mmu-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(5), 0, 0);
+static SPRD_SC_GATE_CLK(gsp1_mmu_eb,	"gsp1-mmu-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(6), 0, 0);
+static SPRD_SC_GATE_CLK(dsi0_eb,		"dsi0-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(7), 0, 0);
+static SPRD_SC_GATE_CLK(dsi1_eb,		"dsi1-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(8), 0, 0);
+static SPRD_SC_GATE_CLK(disp_ckg_eb,	"disp-ckg-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(9), 0, 0);
+static SPRD_SC_GATE_CLK(disp_gpu_eb,	"disp-gpu-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(10), 0, 0);
+static SPRD_SC_GATE_CLK(gpu_mtx_eb,	"gpu-mtx-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(13), 0, 0);
+static SPRD_SC_GATE_CLK(gsp_mtx_eb,	"gsp-mtx-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(14), 0, 0);
+static SPRD_SC_GATE_CLK(tmc_mtx_eb,	"tmc-mtx-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(15), 0, 0);
+static SPRD_SC_GATE_CLK(dispc_mtx_eb,	"dispc-mtx-eb",	"ahb-disp", 0x0,
+		     0x1000, BIT(16), 0, 0);
+static SPRD_GATE_CLK(dphy0_gate,	"dphy0-gate",	"ahb-disp", 0x8,
+		     BIT(0), 0, 0);
+static SPRD_GATE_CLK(dphy1_gate,	"dphy1-gate",	"ahb-disp", 0x8,
+		     BIT(1), 0, 0);
+static SPRD_GATE_CLK(gsp0_a_gate,	"gsp0-a-gate",	"ahb-disp", 0x8,
+		     BIT(2), 0, 0);
+static SPRD_GATE_CLK(gsp1_a_gate,	"gsp1-a-gate",	"ahb-disp", 0x8,
+		     BIT(3), 0, 0);
+static SPRD_GATE_CLK(gsp0_f_gate,	"gsp0-f-gate",	"ahb-disp", 0x8,
+		     BIT(4), 0, 0);
+static SPRD_GATE_CLK(gsp1_f_gate,	"gsp1-f-gate",	"ahb-disp", 0x8,
+		     BIT(5), 0, 0);
+static SPRD_GATE_CLK(d_mtx_f_gate,	"d-mtx-f-gate",	"ahb-disp", 0x8,
+		     BIT(6), 0, 0);
+static SPRD_GATE_CLK(d_mtx_a_gate,	"d-mtx-a-gate",	"ahb-disp", 0x8,
+		     BIT(7), 0, 0);
+static SPRD_GATE_CLK(d_noc_f_gate,	"d-noc-f-gate",	"ahb-disp", 0x8,
+		     BIT(8), 0, 0);
+static SPRD_GATE_CLK(d_noc_a_gate,	"d-noc-a-gate",	"ahb-disp", 0x8,
+		     BIT(9), 0, 0);
+static SPRD_GATE_CLK(gsp_mtx_f_gate, "gsp-mtx-f-gate", "ahb-disp",  0x8,
+		     BIT(10), 0, 0);
+static SPRD_GATE_CLK(gsp_mtx_a_gate, "gsp-mtx-a-gate", "ahb-disp",  0x8,
+		     BIT(11), 0, 0);
+static SPRD_GATE_CLK(gsp_noc_f_gate, "gsp-noc-f-gate", "ahb-disp",  0x8,
+		     BIT(12), 0, 0);
+static SPRD_GATE_CLK(gsp_noc_a_gate, "gsp-noc-a-gate", "ahb-disp",  0x8,
+		     BIT(13), 0, 0);
+static SPRD_GATE_CLK(dispm0idle_gate, "dispm0idle-gate", "ahb-disp", 0x8,
+		     BIT(14), 0, 0);
+static SPRD_GATE_CLK(gspm0idle_gate, "gspm0idle-gate", "ahb-disp",  0x8,
+		     BIT(15), 0, 0);
+
+static struct sprd_clk_common *sc9860_disp_gate[] = {
+	/* address base is 0x63100000 */
+	&dispc0_eb.common,
+	&dispc1_eb.common,
+	&dispc_mmu_eb.common,
+	&gsp0_eb.common,
+	&gsp1_eb.common,
+	&gsp0_mmu_eb.common,
+	&gsp1_mmu_eb.common,
+	&dsi0_eb.common,
+	&dsi1_eb.common,
+	&disp_ckg_eb.common,
+	&disp_gpu_eb.common,
+	&gpu_mtx_eb.common,
+	&gsp_mtx_eb.common,
+	&tmc_mtx_eb.common,
+	&dispc_mtx_eb.common,
+	&dphy0_gate.common,
+	&dphy1_gate.common,
+	&gsp0_a_gate.common,
+	&gsp1_a_gate.common,
+	&gsp0_f_gate.common,
+	&gsp1_f_gate.common,
+	&d_mtx_f_gate.common,
+	&d_mtx_a_gate.common,
+	&d_noc_f_gate.common,
+	&d_noc_a_gate.common,
+	&gsp_mtx_f_gate.common,
+	&gsp_mtx_a_gate.common,
+	&gsp_noc_f_gate.common,
+	&gsp_noc_a_gate.common,
+	&dispm0idle_gate.common,
+	&gspm0idle_gate.common,
+};
+
+static struct clk_hw_onecell_data sc9860_disp_gate_hws = {
+	.hws	= {
+		[CLK_DISPC0_EB]		= &dispc0_eb.common.hw,
+		[CLK_DISPC1_EB]		= &dispc1_eb.common.hw,
+		[CLK_DISPC_MMU_EB]	= &dispc_mmu_eb.common.hw,
+		[CLK_GSP0_EB]		= &gsp0_eb.common.hw,
+		[CLK_GSP1_EB]		= &gsp1_eb.common.hw,
+		[CLK_GSP0_MMU_EB]	= &gsp0_mmu_eb.common.hw,
+		[CLK_GSP1_MMU_EB]	= &gsp1_mmu_eb.common.hw,
+		[CLK_DSI0_EB]		= &dsi0_eb.common.hw,
+		[CLK_DSI1_EB]		= &dsi1_eb.common.hw,
+		[CLK_DISP_CKG_EB]	= &disp_ckg_eb.common.hw,
+		[CLK_DISP_GPU_EB]	= &disp_gpu_eb.common.hw,
+		[CLK_GPU_MTX_EB]	= &gpu_mtx_eb.common.hw,
+		[CLK_GSP_MTX_EB]	= &gsp_mtx_eb.common.hw,
+		[CLK_TMC_MTX_EB]	= &tmc_mtx_eb.common.hw,
+		[CLK_DISPC_MTX_EB]	= &dispc_mtx_eb.common.hw,
+		[CLK_DPHY0_GATE]	= &dphy0_gate.common.hw,
+		[CLK_DPHY1_GATE]	= &dphy1_gate.common.hw,
+		[CLK_GSP0_A_GATE]	= &gsp0_a_gate.common.hw,
+		[CLK_GSP1_A_GATE]	= &gsp1_a_gate.common.hw,
+		[CLK_GSP0_F_GATE]	= &gsp0_f_gate.common.hw,
+		[CLK_GSP1_F_GATE]	= &gsp1_f_gate.common.hw,
+		[CLK_D_MTX_F_GATE]	= &d_mtx_f_gate.common.hw,
+		[CLK_D_MTX_A_GATE]	= &d_mtx_a_gate.common.hw,
+		[CLK_D_NOC_F_GATE]	= &d_noc_f_gate.common.hw,
+		[CLK_D_NOC_A_GATE]	= &d_noc_a_gate.common.hw,
+		[CLK_GSP_MTX_F_GATE]	= &gsp_mtx_f_gate.common.hw,
+		[CLK_GSP_MTX_A_GATE]	= &gsp_mtx_a_gate.common.hw,
+		[CLK_GSP_NOC_F_GATE]	= &gsp_noc_f_gate.common.hw,
+		[CLK_GSP_NOC_A_GATE]	= &gsp_noc_a_gate.common.hw,
+		[CLK_DISPM0IDLE_GATE]	= &dispm0idle_gate.common.hw,
+		[CLK_GSPM0IDLE_GATE]	= &gspm0idle_gate.common.hw,
+	},
+	.num	= CLK_DISP_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_disp_gate_desc = {
+	.clk_clks	= sc9860_disp_gate,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_disp_gate),
+	.hw_clks	= &sc9860_disp_gate_hws,
+};
+
+static SPRD_SC_GATE_CLK(sim0_eb,	"sim0-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(0), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(iis0_eb,	"iis0-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(1), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(iis1_eb,	"iis1-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(2), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(iis2_eb,	"iis2-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(3), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(iis3_eb,	"iis3-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(4), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(spi0_eb,	"spi0-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(5), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(spi1_eb,	"spi1-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(6), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(spi2_eb,	"spi2-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(7), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(i2c0_eb,	"i2c0-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(8), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(i2c1_eb,	"i2c1-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(9), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(i2c2_eb,	"i2c2-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(10), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(i2c3_eb,	"i2c3-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(11), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(i2c4_eb,	"i2c4-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(12), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(i2c5_eb,	"i2c5-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(13), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(uart0_eb,	"uart0-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(14), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(uart1_eb,	"uart1-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(15), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(uart2_eb,	"uart2-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(16), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(uart3_eb,	"uart3-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(17), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(uart4_eb,	"uart4-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(18), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(ap_ckg_eb,	"ap-ckg-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(19), CLK_IGNORE_UNUSED, 0);
+static SPRD_SC_GATE_CLK(spi3_eb,	"spi3-eb",	"ap-apb", 0x0,
+		     0x1000, BIT(20), CLK_IGNORE_UNUSED, 0);
+
+static struct sprd_clk_common *sc9860_apapb_gate[] = {
+	/* address base is 0x70b00000 */
+	&sim0_eb.common,
+	&iis0_eb.common,
+	&iis1_eb.common,
+	&iis2_eb.common,
+	&iis3_eb.common,
+	&spi0_eb.common,
+	&spi1_eb.common,
+	&spi2_eb.common,
+	&i2c0_eb.common,
+	&i2c1_eb.common,
+	&i2c2_eb.common,
+	&i2c3_eb.common,
+	&i2c4_eb.common,
+	&i2c5_eb.common,
+	&uart0_eb.common,
+	&uart1_eb.common,
+	&uart2_eb.common,
+	&uart3_eb.common,
+	&uart4_eb.common,
+	&ap_ckg_eb.common,
+	&spi3_eb.common,
+};
+
+static struct clk_hw_onecell_data sc9860_apapb_gate_hws = {
+	.hws	= {
+		[CLK_SIM0_EB]		= &sim0_eb.common.hw,
+		[CLK_IIS0_EB]		= &iis0_eb.common.hw,
+		[CLK_IIS1_EB]		= &iis1_eb.common.hw,
+		[CLK_IIS2_EB]		= &iis2_eb.common.hw,
+		[CLK_IIS3_EB]		= &iis3_eb.common.hw,
+		[CLK_SPI0_EB]		= &spi0_eb.common.hw,
+		[CLK_SPI1_EB]		= &spi1_eb.common.hw,
+		[CLK_SPI2_EB]		= &spi2_eb.common.hw,
+		[CLK_I2C0_EB]		= &i2c0_eb.common.hw,
+		[CLK_I2C1_EB]		= &i2c1_eb.common.hw,
+		[CLK_I2C2_EB]		= &i2c2_eb.common.hw,
+		[CLK_I2C3_EB]		= &i2c3_eb.common.hw,
+		[CLK_I2C4_EB]		= &i2c4_eb.common.hw,
+		[CLK_I2C5_EB]		= &i2c5_eb.common.hw,
+		[CLK_UART0_EB]		= &uart0_eb.common.hw,
+		[CLK_UART1_EB]		= &uart1_eb.common.hw,
+		[CLK_UART2_EB]		= &uart2_eb.common.hw,
+		[CLK_UART3_EB]		= &uart3_eb.common.hw,
+		[CLK_UART4_EB]		= &uart4_eb.common.hw,
+		[CLK_AP_CKG_EB]		= &ap_ckg_eb.common.hw,
+		[CLK_SPI3_EB]		= &spi3_eb.common.hw,
+	},
+	.num	= CLK_APAPB_GATE_NUM,
+};
+
+static const struct sprd_clk_desc sc9860_apapb_gate_desc = {
+	.clk_clks	= sc9860_apapb_gate,
+	.num_clk_clks	= ARRAY_SIZE(sc9860_apapb_gate),
+	.hw_clks	= &sc9860_apapb_gate_hws,
+};
+
+static const struct of_device_id sprd_sc9860_clk_ids[] = {
+	{ .compatible = "sprd,sc9860-pmu-gate",		/* 0x402b */
+	  .data = &sc9860_pmu_gate_desc },
+	{ .compatible = "sprd,sc9860-pll",		/* 0x4040 */
+	  .data = &sc9860_pll_desc },
+	{ .compatible = "sprd,sc9860-ap-clk",		/* 0x2000 */
+	  .data = &sc9860_ap_clk_desc },
+	{ .compatible = "sprd,sc9860-aon-prediv",	/* 0x402d */
+	  .data = &sc9860_aon_prediv_desc },
+	{ .compatible = "sprd,sc9860-apahb-gate",	/* 0x2021 */
+	  .data = &sc9860_apahb_gate_desc },
+	{ .compatible = "sprd,sc9860-aon-gate",		/* 0x402e */
+	  .data = &sc9860_aon_gate_desc },
+	{ .compatible = "sprd,sc9860-aonsecure-clk",	/* 0x4088 */
+	  .data = &sc9860_aonsecure_clk_desc },
+	{ .compatible = "sprd,sc9860-agcp-gate",	/* 0x415e */
+	  .data = &sc9860_agcp_gate_desc },
+	{ .compatible = "sprd,sc9860-gpu-clk",		/* 0x6020 */
+	  .data = &sc9860_gpu_clk_desc },
+	{ .compatible = "sprd,sc9860-vsp-clk",		/* 0x6100 */
+	  .data = &sc9860_vsp_clk_desc },
+	{ .compatible = "sprd,sc9860-vsp-gate",		/* 0x6110 */
+	  .data = &sc9860_vsp_gate_desc },
+	{ .compatible = "sprd,sc9860-cam-clk",		/* 0x6200 */
+	  .data = &sc9860_cam_clk_desc },
+	{ .compatible = "sprd,sc9860-cam-gate",		/* 0x6210 */
+	  .data = &sc9860_cam_gate_desc },
+	{ .compatible = "sprd,sc9860-disp-clk",		/* 0x6300 */
+	  .data = &sc9860_disp_clk_desc },
+	{ .compatible = "sprd,sc9860-disp-gate",	/* 0x6310 */
+	  .data = &sc9860_disp_gate_desc },
+	{ .compatible = "sprd,sc9860-apapb-gate",	/* 0x70b0 */
+	  .data = &sc9860_apapb_gate_desc },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sprd_sc9860_clk_ids);
+
+static int sc9860_clk_probe(struct platform_device *pdev)
+{
+	const struct of_device_id *match;
+	const struct sprd_clk_desc *desc;
+
+	match = of_match_node(sprd_sc9860_clk_ids, pdev->dev.of_node);
+	if (!match) {
+		pr_err("%s: of_match_node() failed", __func__);
+		return -ENODEV;
+	}
+
+	desc = match->data;
+	sprd_clk_regmap_init(pdev, desc);
+
+	return sprd_clk_probe(&pdev->dev, desc->hw_clks);
+}
+
+static struct platform_driver sc9860_clk_driver = {
+	.probe	= sc9860_clk_probe,
+	.driver	= {
+		.name	= "sc9860-clk",
+		.of_match_table	= sprd_sc9860_clk_ids,
+	},
+};
+module_platform_driver(sc9860_clk_driver);
+
+MODULE_DESCRIPTION("Spreadtrum SC9860 Clock Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:sc9860-clk");
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
index 2bb4cabf802f..ee9c12cf3f08 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
@@ -400,28 +400,45 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(nand_clk, "nand", mod0_default_parents, 0x080,
 				  BIT(31),	/* gate */
 				  0);
 
+/*
+ * MMC clocks are the new timing mode (see A83T & H3) variety, but without
+ * the mode switch. This means they have a 2x post divider between the clock
+ * and the MMC module. This is not documented in the manual, but is taken
+ * into consideration when setting the mmc module clocks in the BSP kernel.
+ * Without it, MMC performance is degraded.
+ *
+ * We model it here to be consistent with other SoCs supporting this mode.
+ * The alternative would be to add the 2x multiplier when setting the MMC
+ * module clock in the MMC driver, just for the A64.
+ */
 static const char * const mmc_default_parents[] = { "osc24M", "pll-periph0-2x",
 						    "pll-periph1-2x" };
-static SUNXI_CCU_MP_WITH_MUX_GATE(mmc0_clk, "mmc0", mmc_default_parents, 0x088,
-				  0, 4,		/* M */
-				  16, 2,	/* P */
-				  24, 2,	/* mux */
-				  BIT(31),	/* gate */
-				  0);
-
-static SUNXI_CCU_MP_WITH_MUX_GATE(mmc1_clk, "mmc1", mmc_default_parents, 0x08c,
-				  0, 4,		/* M */
-				  16, 2,	/* P */
-				  24, 2,	/* mux */
-				  BIT(31),	/* gate */
-				  0);
-
-static SUNXI_CCU_MP_WITH_MUX_GATE(mmc2_clk, "mmc2", mmc_default_parents, 0x090,
-				  0, 4,		/* M */
-				  16, 2,	/* P */
-				  24, 2,	/* mux */
-				  BIT(31),	/* gate */
-				  0);
+static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc0_clk, "mmc0",
+					  mmc_default_parents, 0x088,
+					  0, 4,		/* M */
+					  16, 2,	/* P */
+					  24, 2,	/* mux */
+					  BIT(31),	/* gate */
+					  2,		/* post-div */
+					  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc1_clk, "mmc1",
+					  mmc_default_parents, 0x08c,
+					  0, 4,		/* M */
+					  16, 2,	/* P */
+					  24, 2,	/* mux */
+					  BIT(31),	/* gate */
+					  2,		/* post-div */
+					  0);
+
+static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc2_clk, "mmc2",
+					  mmc_default_parents, 0x090,
+					  0, 4,		/* M */
+					  16, 2,	/* P */
+					  24, 2,	/* mux */
+					  BIT(31),	/* gate */
+					  2,		/* post-div */
+					  0);
 
 static const char * const ts_parents[] = { "osc24M", "pll-periph0", };
 static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", ts_parents, 0x098,
diff --git a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
index 72b16ed1012b..3b97f60540ad 100644
--- a/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
+++ b/drivers/clk/sunxi-ng/ccu-sun6i-a31.c
@@ -762,7 +762,7 @@ static struct ccu_mp out_a_clk = {
 		.features	= CCU_FEATURE_FIXED_PREDIV,
 		.hw.init	= CLK_HW_INIT_PARENTS("out-a",
 						      clk_out_parents,
-						      &ccu_div_ops,
+						      &ccu_mp_ops,
 						      0),
 	},
 };
@@ -783,7 +783,7 @@ static struct ccu_mp out_b_clk = {
 		.features	= CCU_FEATURE_FIXED_PREDIV,
 		.hw.init	= CLK_HW_INIT_PARENTS("out-b",
 						      clk_out_parents,
-						      &ccu_div_ops,
+						      &ccu_mp_ops,
 						      0),
 	},
 };
@@ -804,7 +804,7 @@ static struct ccu_mp out_c_clk = {
 		.features	= CCU_FEATURE_FIXED_PREDIV,
 		.hw.init	= CLK_HW_INIT_PARENTS("out-c",
 						      clk_out_parents,
-						      &ccu_div_ops,
+						      &ccu_mp_ops,
 						      0),
 	},
 };
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
index 5cedcd0d8be8..7d08015b980d 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
@@ -76,15 +76,26 @@ static struct ccu_mult pll_c1cpux_clk = {
  */
 #define SUN8I_A83T_PLL_AUDIO_REG	0x008
 
+/* clock rates doubled for post divider */
+static struct ccu_sdm_setting pll_audio_sdm_table[] = {
+	{ .rate = 45158400, .pattern = 0xc00121ff, .m = 29, .n = 54 },
+	{ .rate = 49152000, .pattern = 0xc000e147, .m = 30, .n = 61 },
+};
+
 static struct ccu_nm pll_audio_clk = {
 	.enable		= BIT(31),
 	.lock		= BIT(2),
 	.n		= _SUNXI_CCU_MULT_OFFSET_MIN_MAX(8, 8, 0, 12, 0),
 	.m		= _SUNXI_CCU_DIV(0, 6),
+	.fixed_post_div	= 2,
+	.sdm		= _SUNXI_CCU_SDM(pll_audio_sdm_table, BIT(24),
+					 0x284, BIT(31)),
 	.common		= {
 		.reg		= SUN8I_A83T_PLL_AUDIO_REG,
 		.lock_reg	= CCU_SUN8I_A83T_LOCK_REG,
-		.features	= CCU_FEATURE_LOCK_REG,
+		.features	= CCU_FEATURE_LOCK_REG |
+				  CCU_FEATURE_FIXED_POSTDIV |
+				  CCU_FEATURE_SIGMA_DELTA_MOD,
 		.hw.init	= CLK_HW_INIT("pll-audio", "osc24M",
 					      &ccu_nm_ops, CLK_SET_RATE_UNGATE),
 	},
@@ -493,8 +504,8 @@ static SUNXI_CCU_MUX_WITH_GATE(tcon0_clk, "tcon0", tcon0_parents,
 				 0x118, 24, 3, BIT(31), CLK_SET_RATE_PARENT);
 
 static const char * const tcon1_parents[] = { "pll-video1" };
-static SUNXI_CCU_MUX_WITH_GATE(tcon1_clk, "tcon1", tcon1_parents,
-				 0x11c, 24, 3, BIT(31), CLK_SET_RATE_PARENT);
+static SUNXI_CCU_M_WITH_MUX_GATE(tcon1_clk, "tcon1", tcon1_parents,
+				 0x11c, 0, 4, 24, 2, BIT(31), CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_GATE(csi_misc_clk, "csi-misc", "osc24M", 0x130, BIT(16), 0);
 
@@ -889,9 +900,10 @@ static int sun8i_a83t_ccu_probe(struct platform_device *pdev)
 	if (IS_ERR(reg))
 		return PTR_ERR(reg);
 
-	/* Enforce d1 = 0, d2 = 0 for Audio PLL */
+	/* Enforce d1 = 0, d2 = 1 for Audio PLL */
 	val = readl(reg + SUN8I_A83T_PLL_AUDIO_REG);
-	val &= ~(BIT(16) | BIT(18));
+	val &= ~BIT(16);
+	val |= BIT(18);
 	writel(val, reg + SUN8I_A83T_PLL_AUDIO_REG);
 
 	/* Enforce P = 1 for both CPU cluster PLLs */
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c
index 5cc9d9952121..468d1abaf0ee 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c
@@ -41,6 +41,8 @@ static SUNXI_CCU_GATE(wb_clk,		"wb",		"wb-div",
 
 static SUNXI_CCU_M(mixer0_div_clk, "mixer0-div", "de", 0x0c, 0, 4,
 		   CLK_SET_RATE_PARENT);
+static SUNXI_CCU_M(mixer1_div_clk, "mixer1-div", "de", 0x0c, 4, 4,
+		   CLK_SET_RATE_PARENT);
 static SUNXI_CCU_M(wb_div_clk, "wb-div", "de", 0x0c, 8, 4,
 		   CLK_SET_RATE_PARENT);
 
@@ -65,6 +67,20 @@ static struct ccu_common *sun8i_a83t_de2_clks[] = {
 	&wb_div_a83_clk.common,
 };
 
+static struct ccu_common *sun8i_h3_de2_clks[] = {
+	&mixer0_clk.common,
+	&mixer1_clk.common,
+	&wb_clk.common,
+
+	&bus_mixer0_clk.common,
+	&bus_mixer1_clk.common,
+	&bus_wb_clk.common,
+
+	&mixer0_div_clk.common,
+	&mixer1_div_clk.common,
+	&wb_div_clk.common,
+};
+
 static struct ccu_common *sun8i_v3s_de2_clks[] = {
 	&mixer0_clk.common,
 	&wb_clk.common,
@@ -93,6 +109,23 @@ static struct clk_hw_onecell_data sun8i_a83t_de2_hw_clks = {
 	.num	= CLK_NUMBER,
 };
 
+static struct clk_hw_onecell_data sun8i_h3_de2_hw_clks = {
+	.hws	= {
+		[CLK_MIXER0]		= &mixer0_clk.common.hw,
+		[CLK_MIXER1]		= &mixer1_clk.common.hw,
+		[CLK_WB]		= &wb_clk.common.hw,
+
+		[CLK_BUS_MIXER0]	= &bus_mixer0_clk.common.hw,
+		[CLK_BUS_MIXER1]	= &bus_mixer1_clk.common.hw,
+		[CLK_BUS_WB]		= &bus_wb_clk.common.hw,
+
+		[CLK_MIXER0_DIV]	= &mixer0_div_clk.common.hw,
+		[CLK_MIXER1_DIV]	= &mixer1_div_clk.common.hw,
+		[CLK_WB_DIV]		= &wb_div_clk.common.hw,
+	},
+	.num	= CLK_NUMBER,
+};
+
 static struct clk_hw_onecell_data sun8i_v3s_de2_hw_clks = {
 	.hws	= {
 		[CLK_MIXER0]		= &mixer0_clk.common.hw,
@@ -133,11 +166,21 @@ static const struct sunxi_ccu_desc sun8i_a83t_de2_clk_desc = {
 	.num_resets	= ARRAY_SIZE(sun8i_a83t_de2_resets),
 };
 
+static const struct sunxi_ccu_desc sun8i_h3_de2_clk_desc = {
+	.ccu_clks	= sun8i_h3_de2_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun8i_h3_de2_clks),
+
+	.hw_clks	= &sun8i_h3_de2_hw_clks,
+
+	.resets		= sun8i_a83t_de2_resets,
+	.num_resets	= ARRAY_SIZE(sun8i_a83t_de2_resets),
+};
+
 static const struct sunxi_ccu_desc sun50i_a64_de2_clk_desc = {
-	.ccu_clks	= sun8i_a83t_de2_clks,
-	.num_ccu_clks	= ARRAY_SIZE(sun8i_a83t_de2_clks),
+	.ccu_clks	= sun8i_h3_de2_clks,
+	.num_ccu_clks	= ARRAY_SIZE(sun8i_h3_de2_clks),
 
-	.hw_clks	= &sun8i_a83t_de2_hw_clks,
+	.hw_clks	= &sun8i_h3_de2_hw_clks,
 
 	.resets		= sun50i_a64_de2_resets,
 	.num_resets	= ARRAY_SIZE(sun50i_a64_de2_resets),
@@ -237,6 +280,10 @@ static const struct of_device_id sunxi_de2_clk_ids[] = {
 		.compatible = "allwinner,sun8i-a83t-de2-clk",
 		.data = &sun8i_a83t_de2_clk_desc,
 	},
+	{
+		.compatible = "allwinner,sun8i-h3-de2-clk",
+		.data = &sun8i_h3_de2_clk_desc,
+	},
 	{
 		.compatible = "allwinner,sun8i-v3s-de2-clk",
 		.data = &sun8i_v3s_de2_clk_desc,
diff --git a/drivers/clk/sunxi-ng/ccu_common.h b/drivers/clk/sunxi-ng/ccu_common.h
index 5d684ce77c54..568cfaed0813 100644
--- a/drivers/clk/sunxi-ng/ccu_common.h
+++ b/drivers/clk/sunxi-ng/ccu_common.h
@@ -31,35 +31,6 @@
 
 struct device_node;
 
-#define CLK_HW_INIT(_name, _parent, _ops, _flags)			\
-	&(struct clk_init_data) {					\
-		.flags		= _flags,				\
-		.name		= _name,				\
-		.parent_names	= (const char *[]) { _parent },		\
-		.num_parents	= 1,					\
-		.ops 		= _ops,					\
-	}
-
-#define CLK_HW_INIT_PARENTS(_name, _parents, _ops, _flags)		\
-	&(struct clk_init_data) {					\
-		.flags		= _flags,				\
-		.name		= _name,				\
-		.parent_names	= _parents,				\
-		.num_parents	= ARRAY_SIZE(_parents),			\
-		.ops 		= _ops,					\
-	}
-
-#define CLK_FIXED_FACTOR(_struct, _name, _parent,			\
-			_div, _mult, _flags)				\
-	struct clk_fixed_factor _struct = {				\
-		.div		= _div,					\
-		.mult		= _mult,				\
-		.hw.init	= CLK_HW_INIT(_name,			\
-					      _parent,			\
-					      &clk_fixed_factor_ops,	\
-					      _flags),			\
-	}
-
 struct ccu_common {
 	void __iomem	*base;
 	u16		reg;
diff --git a/drivers/clk/sunxi-ng/ccu_div.c b/drivers/clk/sunxi-ng/ccu_div.c
index baa3cf96507b..302a18efd39f 100644
--- a/drivers/clk/sunxi-ng/ccu_div.c
+++ b/drivers/clk/sunxi-ng/ccu_div.c
@@ -71,7 +71,7 @@ static unsigned long ccu_div_recalc_rate(struct clk_hw *hw,
 						  parent_rate);
 
 	val = divider_recalc_rate(hw, parent_rate, val, cd->div.table,
-				  cd->div.flags);
+				  cd->div.flags, cd->div.width);
 
 	if (cd->common.features & CCU_FEATURE_FIXED_POSTDIV)
 		val /= cd->fixed_post_div;
diff --git a/drivers/clk/sunxi-ng/ccu_mp.c b/drivers/clk/sunxi-ng/ccu_mp.c
index 688855e7dc8c..5d0af4051737 100644
--- a/drivers/clk/sunxi-ng/ccu_mp.c
+++ b/drivers/clk/sunxi-ng/ccu_mp.c
@@ -50,12 +50,19 @@ static unsigned long ccu_mp_round_rate(struct ccu_mux_internal *mux,
 	unsigned int max_m, max_p;
 	unsigned int m, p;
 
+	if (cmp->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate *= cmp->fixed_post_div;
+
 	max_m = cmp->m.max ?: 1 << cmp->m.width;
 	max_p = cmp->p.max ?: 1 << ((1 << cmp->p.width) - 1);
 
 	ccu_mp_find_best(*parent_rate, rate, max_m, max_p, &m, &p);
+	rate = *parent_rate / p / m;
+
+	if (cmp->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate /= cmp->fixed_post_div;
 
-	return *parent_rate / p / m;
+	return rate;
 }
 
 static void ccu_mp_disable(struct clk_hw *hw)
@@ -83,6 +90,7 @@ static unsigned long ccu_mp_recalc_rate(struct clk_hw *hw,
 					unsigned long parent_rate)
 {
 	struct ccu_mp *cmp = hw_to_ccu_mp(hw);
+	unsigned long rate;
 	unsigned int m, p;
 	u32 reg;
 
@@ -101,7 +109,11 @@ static unsigned long ccu_mp_recalc_rate(struct clk_hw *hw,
 	p = reg >> cmp->p.shift;
 	p &= (1 << cmp->p.width) - 1;
 
-	return (parent_rate >> p) / m;
+	rate = (parent_rate >> p) / m;
+	if (cmp->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate /= cmp->fixed_post_div;
+
+	return rate;
 }
 
 static int ccu_mp_determine_rate(struct clk_hw *hw,
@@ -129,6 +141,10 @@ static int ccu_mp_set_rate(struct clk_hw *hw, unsigned long rate,
 	max_m = cmp->m.max ?: 1 << cmp->m.width;
 	max_p = cmp->p.max ?: 1 << ((1 << cmp->p.width) - 1);
 
+	/* Adjust target rate according to post-dividers */
+	if (cmp->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate = rate * cmp->fixed_post_div;
+
 	ccu_mp_find_best(parent_rate, rate, max_m, max_p, &m, &p);
 
 	spin_lock_irqsave(cmp->common.lock, flags);
diff --git a/drivers/clk/sunxi-ng/ccu_mp.h b/drivers/clk/sunxi-ng/ccu_mp.h
index aaef11d747ea..5107635e61de 100644
--- a/drivers/clk/sunxi-ng/ccu_mp.h
+++ b/drivers/clk/sunxi-ng/ccu_mp.h
@@ -33,9 +33,33 @@ struct ccu_mp {
 	struct ccu_div_internal		m;
 	struct ccu_div_internal		p;
 	struct ccu_mux_internal	mux;
+
+	unsigned int		fixed_post_div;
+
 	struct ccu_common	common;
 };
 
+#define SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(_struct, _name, _parents, _reg, \
+					   _mshift, _mwidth,		\
+					   _pshift, _pwidth,		\
+					   _muxshift, _muxwidth,	\
+					   _gate, _postdiv, _flags)	\
+	struct ccu_mp _struct = {					\
+		.enable	= _gate,					\
+		.m	= _SUNXI_CCU_DIV(_mshift, _mwidth),		\
+		.p	= _SUNXI_CCU_DIV(_pshift, _pwidth),		\
+		.mux	= _SUNXI_CCU_MUX(_muxshift, _muxwidth),		\
+		.fixed_post_div	= _postdiv,				\
+		.common	= {						\
+			.reg		= _reg,				\
+			.features	= CCU_FEATURE_FIXED_POSTDIV,	\
+			.hw.init	= CLK_HW_INIT_PARENTS(_name,	\
+							      _parents, \
+							      &ccu_mp_ops, \
+							      _flags),	\
+		}							\
+	}
+
 #define SUNXI_CCU_MP_WITH_MUX_GATE(_struct, _name, _parents, _reg,	\
 				   _mshift, _mwidth,			\
 				   _pshift, _pwidth,			\
diff --git a/drivers/clk/sunxi-ng/ccu_nm.c b/drivers/clk/sunxi-ng/ccu_nm.c
index 7620aa973a6e..a16de092bf94 100644
--- a/drivers/clk/sunxi-ng/ccu_nm.c
+++ b/drivers/clk/sunxi-ng/ccu_nm.c
@@ -70,11 +70,18 @@ static unsigned long ccu_nm_recalc_rate(struct clk_hw *hw,
 					unsigned long parent_rate)
 {
 	struct ccu_nm *nm = hw_to_ccu_nm(hw);
+	unsigned long rate;
 	unsigned long n, m;
 	u32 reg;
 
-	if (ccu_frac_helper_is_enabled(&nm->common, &nm->frac))
-		return ccu_frac_helper_read_rate(&nm->common, &nm->frac);
+	if (ccu_frac_helper_is_enabled(&nm->common, &nm->frac)) {
+		rate = ccu_frac_helper_read_rate(&nm->common, &nm->frac);
+
+		if (nm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+			rate /= nm->fixed_post_div;
+
+		return rate;
+	}
 
 	reg = readl(nm->common.base + nm->common.reg);
 
@@ -90,15 +97,15 @@ static unsigned long ccu_nm_recalc_rate(struct clk_hw *hw,
 	if (!m)
 		m++;
 
-	if (ccu_sdm_helper_is_enabled(&nm->common, &nm->sdm)) {
-		unsigned long rate =
-			ccu_sdm_helper_read_rate(&nm->common, &nm->sdm,
-						 m, n);
-		if (rate)
-			return rate;
-	}
+	if (ccu_sdm_helper_is_enabled(&nm->common, &nm->sdm))
+		rate = ccu_sdm_helper_read_rate(&nm->common, &nm->sdm, m, n);
+	else
+		rate = parent_rate * n / m;
+
+	if (nm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate /= nm->fixed_post_div;
 
-	return parent_rate * n / m;
+	return rate;
 }
 
 static long ccu_nm_round_rate(struct clk_hw *hw, unsigned long rate,
@@ -107,11 +114,20 @@ static long ccu_nm_round_rate(struct clk_hw *hw, unsigned long rate,
 	struct ccu_nm *nm = hw_to_ccu_nm(hw);
 	struct _ccu_nm _nm;
 
-	if (ccu_frac_helper_has_rate(&nm->common, &nm->frac, rate))
+	if (nm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate *= nm->fixed_post_div;
+
+	if (ccu_frac_helper_has_rate(&nm->common, &nm->frac, rate)) {
+		if (nm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+			rate /= nm->fixed_post_div;
 		return rate;
+	}
 
-	if (ccu_sdm_helper_has_rate(&nm->common, &nm->sdm, rate))
+	if (ccu_sdm_helper_has_rate(&nm->common, &nm->sdm, rate)) {
+		if (nm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+			rate /= nm->fixed_post_div;
 		return rate;
+	}
 
 	_nm.min_n = nm->n.min ?: 1;
 	_nm.max_n = nm->n.max ?: 1 << nm->n.width;
@@ -119,8 +135,12 @@ static long ccu_nm_round_rate(struct clk_hw *hw, unsigned long rate,
 	_nm.max_m = nm->m.max ?: 1 << nm->m.width;
 
 	ccu_nm_find_best(*parent_rate, rate, &_nm);
+	rate = *parent_rate * _nm.n / _nm.m;
+
+	if (nm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate /= nm->fixed_post_div;
 
-	return *parent_rate * _nm.n / _nm.m;
+	return rate;
 }
 
 static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -131,6 +151,10 @@ static int ccu_nm_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned long flags;
 	u32 reg;
 
+	/* Adjust target rate according to post-dividers */
+	if (nm->common.features & CCU_FEATURE_FIXED_POSTDIV)
+		rate = rate * nm->fixed_post_div;
+
 	if (ccu_frac_helper_has_rate(&nm->common, &nm->frac, rate)) {
 		spin_lock_irqsave(nm->common.lock, flags);
 
diff --git a/drivers/clk/sunxi-ng/ccu_nm.h b/drivers/clk/sunxi-ng/ccu_nm.h
index c623b0c7a23c..eba586b4c7d0 100644
--- a/drivers/clk/sunxi-ng/ccu_nm.h
+++ b/drivers/clk/sunxi-ng/ccu_nm.h
@@ -36,6 +36,8 @@ struct ccu_nm {
 	struct ccu_frac_internal	frac;
 	struct ccu_sdm_internal		sdm;
 
+	unsigned int		fixed_post_div;
+
 	struct ccu_common	common;
 };
 
diff --git a/drivers/clk/sunxi/clk-factors.c b/drivers/clk/sunxi/clk-factors.c
index 856fef65433b..661a73284e9f 100644
--- a/drivers/clk/sunxi/clk-factors.c
+++ b/drivers/clk/sunxi/clk-factors.c
@@ -176,10 +176,10 @@ static const struct clk_ops clk_factors_ops = {
 	.set_rate = clk_factors_set_rate,
 };
 
-struct clk *sunxi_factors_register(struct device_node *node,
-				   const struct factors_data *data,
-				   spinlock_t *lock,
-				   void __iomem *reg)
+static struct clk *__sunxi_factors_register(struct device_node *node,
+					    const struct factors_data *data,
+					    spinlock_t *lock, void __iomem *reg,
+					    unsigned long flags)
 {
 	struct clk *clk;
 	struct clk_factors *factors;
@@ -249,7 +249,7 @@ struct clk *sunxi_factors_register(struct device_node *node,
 			parents, i,
 			mux_hw, &clk_mux_ops,
 			&factors->hw, &clk_factors_ops,
-			gate_hw, &clk_gate_ops, 0);
+			gate_hw, &clk_gate_ops, CLK_IS_CRITICAL);
 	if (IS_ERR(clk))
 		goto err_register;
 
@@ -272,6 +272,22 @@ struct clk *sunxi_factors_register(struct device_node *node,
 	return NULL;
 }
 
+struct clk *sunxi_factors_register(struct device_node *node,
+				   const struct factors_data *data,
+				   spinlock_t *lock,
+				   void __iomem *reg)
+{
+	return __sunxi_factors_register(node, data, lock, reg, 0);
+}
+
+struct clk *sunxi_factors_register_critical(struct device_node *node,
+					    const struct factors_data *data,
+					    spinlock_t *lock,
+					    void __iomem *reg)
+{
+	return __sunxi_factors_register(node, data, lock, reg, CLK_IS_CRITICAL);
+}
+
 void sunxi_factors_unregister(struct device_node *node, struct clk *clk)
 {
 	struct clk_hw *hw = __clk_get_hw(clk);
diff --git a/drivers/clk/sunxi/clk-factors.h b/drivers/clk/sunxi/clk-factors.h
index 824f746b2567..7ad2ca924d0d 100644
--- a/drivers/clk/sunxi/clk-factors.h
+++ b/drivers/clk/sunxi/clk-factors.h
@@ -55,6 +55,10 @@ struct clk *sunxi_factors_register(struct device_node *node,
 				   const struct factors_data *data,
 				   spinlock_t *lock,
 				   void __iomem *reg);
+struct clk *sunxi_factors_register_critical(struct device_node *node,
+					    const struct factors_data *data,
+					    spinlock_t *lock,
+					    void __iomem *reg);
 
 void sunxi_factors_unregister(struct device_node *node, struct clk *clk);
 
diff --git a/drivers/clk/sunxi/clk-mod0.c b/drivers/clk/sunxi/clk-mod0.c
index 4417ae129ac7..a27c264cc9b4 100644
--- a/drivers/clk/sunxi/clk-mod0.c
+++ b/drivers/clk/sunxi/clk-mod0.c
@@ -15,7 +15,6 @@
  */
 
 #include <linux/clk.h>
-#include <linux/clkdev.h>
 #include <linux/clk-provider.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
@@ -155,7 +154,6 @@ static DEFINE_SPINLOCK(sun5i_a13_mbus_lock);
 
 static void __init sun5i_a13_mbus_setup(struct device_node *node)
 {
-	struct clk *mbus;
 	void __iomem *reg;
 
 	reg = of_iomap(node, 0);
@@ -164,12 +162,9 @@ static void __init sun5i_a13_mbus_setup(struct device_node *node)
 		return;
 	}
 
-	mbus = sunxi_factors_register(node, &sun4i_a10_mod0_data,
-				      &sun5i_a13_mbus_lock, reg);
-
 	/* The MBUS clocks needs to be always enabled */
-	__clk_get(mbus);
-	clk_prepare_enable(mbus);
+	sunxi_factors_register_critical(node, &sun4i_a10_mod0_data,
+					&sun5i_a13_mbus_lock, reg);
 }
 CLK_OF_DECLARE(sun5i_a13_mbus, "allwinner,sun5i-a13-mbus-clk", sun5i_a13_mbus_setup);
 
diff --git a/drivers/clk/sunxi/clk-sun8i-apb0.c b/drivers/clk/sunxi/clk-sun8i-apb0.c
index ea1eed24778c..d5c31804ee54 100644
--- a/drivers/clk/sunxi/clk-sun8i-apb0.c
+++ b/drivers/clk/sunxi/clk-sun8i-apb0.c
@@ -98,10 +98,7 @@ static int sun8i_a23_apb0_clk_probe(struct platform_device *pdev)
 		return PTR_ERR(reg);
 
 	clk = sun8i_a23_apb0_register(np, reg);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(clk);
 }
 
 static const struct of_device_id sun8i_a23_apb0_clk_dt_ids[] = {
diff --git a/drivers/clk/sunxi/clk-sun8i-mbus.c b/drivers/clk/sunxi/clk-sun8i-mbus.c
index b200ebf159ee..56db89b6979f 100644
--- a/drivers/clk/sunxi/clk-sun8i-mbus.c
+++ b/drivers/clk/sunxi/clk-sun8i-mbus.c
@@ -15,7 +15,6 @@
  */
 
 #include <linux/clk.h>
-#include <linux/clkdev.h>
 #include <linux/clk-provider.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
@@ -82,11 +81,12 @@ static void __init sun8i_a23_mbus_setup(struct device_node *node)
 	mux->mask = SUN8I_MBUS_MUX_MASK;
 	mux->lock = &sun8i_a23_mbus_lock;
 
+	/* The MBUS clocks needs to be always enabled */
 	clk = clk_register_composite(NULL, clk_name, parents, num_parents,
 				     &mux->hw, &clk_mux_ops,
 				     &div->hw, &clk_divider_ops,
 				     &gate->hw, &clk_gate_ops,
-				     0);
+				     CLK_IS_CRITICAL);
 	if (IS_ERR(clk))
 		goto err_free_gate;
 
@@ -95,9 +95,6 @@ static void __init sun8i_a23_mbus_setup(struct device_node *node)
 		goto err_unregister_clk;
 
 	kfree(parents); /* parents is deep copied */
-	/* The MBUS clocks needs to be always enabled */
-	__clk_get(clk);
-	clk_prepare_enable(clk);
 
 	return;
 
diff --git a/drivers/clk/sunxi/clk-sun9i-core.c b/drivers/clk/sunxi/clk-sun9i-core.c
index 43f014f85803..e9295c286d5d 100644
--- a/drivers/clk/sunxi/clk-sun9i-core.c
+++ b/drivers/clk/sunxi/clk-sun9i-core.c
@@ -15,7 +15,6 @@
  */
 
 #include <linux/clk.h>
-#include <linux/clkdev.h>
 #include <linux/clk-provider.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
@@ -140,7 +139,6 @@ static DEFINE_SPINLOCK(sun9i_a80_gt_lock);
 static void __init sun9i_a80_gt_setup(struct device_node *node)
 {
 	void __iomem *reg;
-	struct clk *gt;
 
 	reg = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(reg)) {
@@ -149,12 +147,9 @@ static void __init sun9i_a80_gt_setup(struct device_node *node)
 		return;
 	}
 
-	gt = sunxi_factors_register(node, &sun9i_a80_gt_data,
-				    &sun9i_a80_gt_lock, reg);
-
 	/* The GT bus clock needs to be always enabled */
-	__clk_get(gt);
-	clk_prepare_enable(gt);
+	sunxi_factors_register_critical(node, &sun9i_a80_gt_data,
+					&sun9i_a80_gt_lock, reg);
 }
 CLK_OF_DECLARE(sun9i_a80_gt, "allwinner,sun9i-a80-gt-clk", sun9i_a80_gt_setup);
 
diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c
index aa4add580516..012714d94b42 100644
--- a/drivers/clk/sunxi/clk-sunxi.c
+++ b/drivers/clk/sunxi/clk-sunxi.c
@@ -656,7 +656,8 @@ static const struct mux_data sun8i_h3_ahb2_mux_data __initconst = {
 };
 
 static struct clk * __init sunxi_mux_clk_setup(struct device_node *node,
-					       const struct mux_data *data)
+					       const struct mux_data *data,
+					       unsigned long flags)
 {
 	struct clk *clk;
 	const char *clk_name = node->name;
@@ -678,7 +679,7 @@ static struct clk * __init sunxi_mux_clk_setup(struct device_node *node,
 	}
 
 	clk = clk_register_mux(NULL, clk_name, parents, i,
-			       CLK_SET_RATE_PARENT, reg,
+			       CLK_SET_RATE_PARENT | flags, reg,
 			       data->shift, SUNXI_MUX_GATE_WIDTH,
 			       0, &clk_lock);
 
@@ -703,29 +704,22 @@ static struct clk * __init sunxi_mux_clk_setup(struct device_node *node,
 
 static void __init sun4i_cpu_clk_setup(struct device_node *node)
 {
-	struct clk *clk;
-
-	clk = sunxi_mux_clk_setup(node, &sun4i_cpu_mux_data);
-	if (!clk)
-		return;
-
 	/* Protect CPU clock */
-	__clk_get(clk);
-	clk_prepare_enable(clk);
+	sunxi_mux_clk_setup(node, &sun4i_cpu_mux_data, CLK_IS_CRITICAL);
 }
 CLK_OF_DECLARE(sun4i_cpu, "allwinner,sun4i-a10-cpu-clk",
 	       sun4i_cpu_clk_setup);
 
 static void __init sun6i_ahb1_mux_clk_setup(struct device_node *node)
 {
-	sunxi_mux_clk_setup(node, &sun6i_a31_ahb1_mux_data);
+	sunxi_mux_clk_setup(node, &sun6i_a31_ahb1_mux_data, 0);
 }
 CLK_OF_DECLARE(sun6i_ahb1_mux, "allwinner,sun6i-a31-ahb1-mux-clk",
 	       sun6i_ahb1_mux_clk_setup);
 
 static void __init sun8i_ahb2_clk_setup(struct device_node *node)
 {
-	sunxi_mux_clk_setup(node, &sun8i_h3_ahb2_mux_data);
+	sunxi_mux_clk_setup(node, &sun8i_h3_ahb2_mux_data, 0);
 }
 CLK_OF_DECLARE(sun8i_ahb2, "allwinner,sun8i-h3-ahb2-clk",
 	       sun8i_ahb2_clk_setup);
@@ -900,6 +894,7 @@ struct divs_data {
 		u8 shift; /* otherwise it's a normal divisor with this shift */
 		u8 pow;   /* is it power-of-two based? */
 		u8 gate;  /* is it independently gateable? */
+		bool critical;
 	} div[SUNXI_DIVS_MAX_QTY];
 };
 
@@ -915,7 +910,8 @@ static const struct divs_data pll5_divs_data __initconst = {
 	.factors = &sun4i_pll5_data,
 	.ndivs = 2,
 	.div = {
-		{ .shift = 0, .pow = 0, }, /* M, DDR */
+		/* Protect PLL5_DDR */
+		{ .shift = 0, .pow = 0, .critical = true }, /* M, DDR */
 		{ .shift = 16, .pow = 1, }, /* P, other */
 		/* No output for the base factor clock */
 	}
@@ -1089,7 +1085,9 @@ static struct clk ** __init sunxi_divs_clk_setup(struct device_node *node,
 						 NULL, NULL,
 						 rate_hw, rate_ops,
 						 gate_hw, &clk_gate_ops,
-						 clkflags);
+						 clkflags |
+						 data->div[i].critical ?
+							CLK_IS_CRITICAL : 0);
 
 		WARN_ON(IS_ERR(clk_data->clks[i]));
 	}
@@ -1117,15 +1115,7 @@ static struct clk ** __init sunxi_divs_clk_setup(struct device_node *node,
 
 static void __init sun4i_pll5_clk_setup(struct device_node *node)
 {
-	struct clk **clks;
-
-	clks = sunxi_divs_clk_setup(node, &pll5_divs_data);
-	if (!clks)
-		return;
-
-	/* Protect PLL5_DDR */
-	__clk_get(clks[0]);
-	clk_prepare_enable(clks[0]);
+	sunxi_divs_clk_setup(node, &pll5_divs_data);
 }
 CLK_OF_DECLARE(sun4i_pll5, "allwinner,sun4i-a10-pll5-clk",
 	       sun4i_pll5_clk_setup);
diff --git a/drivers/clk/ti/Makefile b/drivers/clk/ti/Makefile
index a2293ee09440..5ab295d2a3cb 100644
--- a/drivers/clk/ti/Makefile
+++ b/drivers/clk/ti/Makefile
@@ -19,10 +19,6 @@ obj-$(CONFIG_SOC_DRA7XX)		+= $(clk-common) clk-7xx.o \
 					   clk-dra7-atl.o dpll3xxx.o dpll44xx.o
 obj-$(CONFIG_SOC_AM43XX)		+= $(clk-common) dpll3xxx.o clk-43xx.o
 
-ifdef CONFIG_ATAGS
-obj-$(CONFIG_ARCH_OMAP3)                += clk-3xxx-legacy.o
-endif
-
 endif	# CONFIG_ARCH_OMAP2PLUS
 
 obj-$(CONFIG_COMMON_CLK_TI_ADPLL)	+= adpll.o
diff --git a/drivers/clk/ti/apll.c b/drivers/clk/ti/apll.c
index 83b148f8037c..9498e9363b57 100644
--- a/drivers/clk/ti/apll.c
+++ b/drivers/clk/ti/apll.c
@@ -133,9 +133,10 @@ static const struct clk_ops apll_ck_ops = {
 	.get_parent	= &dra7_init_apll_parent,
 };
 
-static void __init omap_clk_register_apll(struct clk_hw *hw,
+static void __init omap_clk_register_apll(void *user,
 					  struct device_node *node)
 {
+	struct clk_hw *hw = user;
 	struct clk_hw_omap *clk_hw = to_clk_hw_omap(hw);
 	struct dpll_data *ad = clk_hw->dpll_data;
 	struct clk *clk;
diff --git a/drivers/clk/ti/clk-33xx.c b/drivers/clk/ti/clk-33xx.c
index 0e47d95faf49..12e0a2d19911 100644
--- a/drivers/clk/ti/clk-33xx.c
+++ b/drivers/clk/ti/clk-33xx.c
@@ -19,98 +19,201 @@
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/clk/ti.h>
+#include <dt-bindings/clock/am3.h>
 
 #include "clock.h"
 
+static const char * const am3_gpio1_dbclk_parents[] __initconst = {
+	"l4_per_cm:clk:0138:0",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data am3_gpio2_bit_data[] __initconst = {
+	{ 18, TI_CLK_GATE, am3_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data am3_gpio3_bit_data[] __initconst = {
+	{ 18, TI_CLK_GATE, am3_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data am3_gpio4_bit_data[] __initconst = {
+	{ 18, TI_CLK_GATE, am3_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am3_l4_per_clkctrl_regs[] __initconst = {
+	{ AM3_CPGMAC0_CLKCTRL, NULL, CLKF_SW_SUP, "cpsw_125mhz_gclk", "cpsw_125mhz_clkdm" },
+	{ AM3_LCDC_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_SET_RATE_PARENT, "lcd_gclk", "lcdc_clkdm" },
+	{ AM3_USB_OTG_HS_CLKCTRL, NULL, CLKF_SW_SUP, "usbotg_fck", "l3s_clkdm" },
+	{ AM3_TPTC0_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_EMIF_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_ddr_m2_div2_ck", "l3_clkdm" },
+	{ AM3_OCMCRAM_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_GPMC_CLKCTRL, NULL, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
+	{ AM3_MCASP0_CLKCTRL, NULL, CLKF_SW_SUP, "mcasp0_fck", "l3s_clkdm" },
+	{ AM3_UART6_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_MMC1_CLKCTRL, NULL, CLKF_SW_SUP, "mmc_clk" },
+	{ AM3_ELM_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_I2C3_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_I2C2_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_SPI0_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_SPI1_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_L4_LS_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_MCASP1_CLKCTRL, NULL, CLKF_SW_SUP, "mcasp1_fck", "l3s_clkdm" },
+	{ AM3_UART2_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_UART3_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_UART4_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_UART5_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM3_TIMER7_CLKCTRL, NULL, CLKF_SW_SUP, "timer7_fck" },
+	{ AM3_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "timer2_fck" },
+	{ AM3_TIMER3_CLKCTRL, NULL, CLKF_SW_SUP, "timer3_fck" },
+	{ AM3_TIMER4_CLKCTRL, NULL, CLKF_SW_SUP, "timer4_fck" },
+	{ AM3_RNG_CLKCTRL, NULL, CLKF_SW_SUP, "rng_fck" },
+	{ AM3_AES_CLKCTRL, NULL, CLKF_SW_SUP, "aes0_fck", "l3_clkdm" },
+	{ AM3_SHAM_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_GPIO2_CLKCTRL, am3_gpio2_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_GPIO3_CLKCTRL, am3_gpio3_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_GPIO4_CLKCTRL, am3_gpio4_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_TPCC_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_D_CAN0_CLKCTRL, NULL, CLKF_SW_SUP, "dcan0_fck" },
+	{ AM3_D_CAN1_CLKCTRL, NULL, CLKF_SW_SUP, "dcan1_fck" },
+	{ AM3_EPWMSS1_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_EPWMSS0_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_EPWMSS2_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_L3_INSTR_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_L3_MAIN_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_PRUSS_CLKCTRL, NULL, CLKF_SW_SUP, "pruss_ocp_gclk", "pruss_ocp_clkdm" },
+	{ AM3_TIMER5_CLKCTRL, NULL, CLKF_SW_SUP, "timer5_fck" },
+	{ AM3_TIMER6_CLKCTRL, NULL, CLKF_SW_SUP, "timer6_fck" },
+	{ AM3_MMC2_CLKCTRL, NULL, CLKF_SW_SUP, "mmc_clk" },
+	{ AM3_MMC3_CLKCTRL, NULL, CLKF_SW_SUP, "mmc_clk", "l3s_clkdm" },
+	{ AM3_TPTC1_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_TPTC2_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM3_SPINLOCK_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_MAILBOX_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_L4_HS_CLKCTRL, NULL, CLKF_SW_SUP, "l4hs_gclk", "l4hs_clkdm" },
+	{ AM3_OCPWP_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM3_CLKDIV32K_CLKCTRL, NULL, CLKF_SW_SUP, "clkdiv32k_ck", "clk_24mhz_clkdm" },
+	{ 0 },
+};
+
+static const char * const am3_gpio0_dbclk_parents[] __initconst = {
+	"gpio0_dbclk_mux_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data am3_gpio1_bit_data[] __initconst = {
+	{ 18, TI_CLK_GATE, am3_gpio0_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const char * const am3_dbg_sysclk_ck_parents[] __initconst = {
+	"sys_clkin_ck",
+	NULL,
+};
+
+static const char * const am3_trace_pmd_clk_mux_ck_parents[] __initconst = {
+	"l4_wkup_cm:clk:0010:19",
+	"l4_wkup_cm:clk:0010:30",
+	NULL,
+};
+
+static const char * const am3_trace_clk_div_ck_parents[] __initconst = {
+	"l4_wkup_cm:clk:0010:20",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data am3_trace_clk_div_ck_data __initconst = {
+	.max_div = 64,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
+};
+
+static const char * const am3_stm_clk_div_ck_parents[] __initconst = {
+	"l4_wkup_cm:clk:0010:22",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data am3_stm_clk_div_ck_data __initconst = {
+	.max_div = 64,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
+};
+
+static const char * const am3_dbg_clka_ck_parents[] __initconst = {
+	"dpll_core_m4_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data am3_debugss_bit_data[] __initconst = {
+	{ 19, TI_CLK_GATE, am3_dbg_sysclk_ck_parents, NULL },
+	{ 20, TI_CLK_MUX, am3_trace_pmd_clk_mux_ck_parents, NULL },
+	{ 22, TI_CLK_MUX, am3_trace_pmd_clk_mux_ck_parents, NULL },
+	{ 24, TI_CLK_DIVIDER, am3_trace_clk_div_ck_parents, &am3_trace_clk_div_ck_data },
+	{ 27, TI_CLK_DIVIDER, am3_stm_clk_div_ck_parents, &am3_stm_clk_div_ck_data },
+	{ 30, TI_CLK_GATE, am3_dbg_clka_ck_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am3_l4_wkup_clkctrl_regs[] __initconst = {
+	{ AM3_CONTROL_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_core_m4_div2_ck" },
+	{ AM3_GPIO1_CLKCTRL, am3_gpio1_bit_data, CLKF_SW_SUP, "dpll_core_m4_div2_ck" },
+	{ AM3_L4_WKUP_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_core_m4_div2_ck" },
+	{ AM3_DEBUGSS_CLKCTRL, am3_debugss_bit_data, CLKF_SW_SUP, "l4_wkup_cm:clk:0010:24", "l3_aon_clkdm" },
+	{ AM3_WKUP_M3_CLKCTRL, NULL, CLKF_NO_IDLEST, "dpll_core_m4_div2_ck", "l4_wkup_aon_clkdm" },
+	{ AM3_UART1_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_wkupdm_ck" },
+	{ AM3_I2C1_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_wkupdm_ck" },
+	{ AM3_ADC_TSC_CLKCTRL, NULL, CLKF_SW_SUP, "adc_tsc_fck" },
+	{ AM3_SMARTREFLEX0_CLKCTRL, NULL, CLKF_SW_SUP, "smartreflex0_fck" },
+	{ AM3_TIMER1_CLKCTRL, NULL, CLKF_SW_SUP, "timer1_fck" },
+	{ AM3_SMARTREFLEX1_CLKCTRL, NULL, CLKF_SW_SUP, "smartreflex1_fck" },
+	{ AM3_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "wdt1_fck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am3_mpu_clkctrl_regs[] __initconst = {
+	{ AM3_MPU_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_mpu_m2_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am3_l4_rtc_clkctrl_regs[] __initconst = {
+	{ AM3_RTC_CLKCTRL, NULL, CLKF_SW_SUP, "clk_32768_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am3_gfx_l3_clkctrl_regs[] __initconst = {
+	{ AM3_GFX_CLKCTRL, NULL, CLKF_SW_SUP, "gfx_fck_div_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am3_l4_cefuse_clkctrl_regs[] __initconst = {
+	{ AM3_CEFUSE_CLKCTRL, NULL, CLKF_SW_SUP, "sys_clkin_ck" },
+	{ 0 },
+};
+
+const struct omap_clkctrl_data am3_clkctrl_data[] __initconst = {
+	{ 0x44e00014, am3_l4_per_clkctrl_regs },
+	{ 0x44e00404, am3_l4_wkup_clkctrl_regs },
+	{ 0x44e00604, am3_mpu_clkctrl_regs },
+	{ 0x44e00800, am3_l4_rtc_clkctrl_regs },
+	{ 0x44e00904, am3_gfx_l3_clkctrl_regs },
+	{ 0x44e00a20, am3_l4_cefuse_clkctrl_regs },
+	{ 0 },
+};
+
 static struct ti_dt_clk am33xx_clks[] = {
-	DT_CLK(NULL, "clk_32768_ck", "clk_32768_ck"),
-	DT_CLK(NULL, "clk_rc32k_ck", "clk_rc32k_ck"),
-	DT_CLK(NULL, "virt_19200000_ck", "virt_19200000_ck"),
-	DT_CLK(NULL, "virt_24000000_ck", "virt_24000000_ck"),
-	DT_CLK(NULL, "virt_25000000_ck", "virt_25000000_ck"),
-	DT_CLK(NULL, "virt_26000000_ck", "virt_26000000_ck"),
-	DT_CLK(NULL, "sys_clkin_ck", "sys_clkin_ck"),
-	DT_CLK(NULL, "tclkin_ck", "tclkin_ck"),
-	DT_CLK(NULL, "dpll_core_ck", "dpll_core_ck"),
-	DT_CLK(NULL, "dpll_core_x2_ck", "dpll_core_x2_ck"),
-	DT_CLK(NULL, "dpll_core_m4_ck", "dpll_core_m4_ck"),
-	DT_CLK(NULL, "dpll_core_m5_ck", "dpll_core_m5_ck"),
-	DT_CLK(NULL, "dpll_core_m6_ck", "dpll_core_m6_ck"),
-	DT_CLK(NULL, "dpll_mpu_ck", "dpll_mpu_ck"),
-	DT_CLK(NULL, "dpll_mpu_m2_ck", "dpll_mpu_m2_ck"),
-	DT_CLK(NULL, "dpll_ddr_ck", "dpll_ddr_ck"),
-	DT_CLK(NULL, "dpll_ddr_m2_ck", "dpll_ddr_m2_ck"),
-	DT_CLK(NULL, "dpll_ddr_m2_div2_ck", "dpll_ddr_m2_div2_ck"),
-	DT_CLK(NULL, "dpll_disp_ck", "dpll_disp_ck"),
-	DT_CLK(NULL, "dpll_disp_m2_ck", "dpll_disp_m2_ck"),
-	DT_CLK(NULL, "dpll_per_ck", "dpll_per_ck"),
-	DT_CLK(NULL, "dpll_per_m2_ck", "dpll_per_m2_ck"),
-	DT_CLK(NULL, "dpll_per_m2_div4_wkupdm_ck", "dpll_per_m2_div4_wkupdm_ck"),
-	DT_CLK(NULL, "dpll_per_m2_div4_ck", "dpll_per_m2_div4_ck"),
-	DT_CLK(NULL, "adc_tsc_fck", "adc_tsc_fck"),
-	DT_CLK(NULL, "cefuse_fck", "cefuse_fck"),
-	DT_CLK(NULL, "clkdiv32k_ck", "clkdiv32k_ck"),
-	DT_CLK(NULL, "clkdiv32k_ick", "clkdiv32k_ick"),
-	DT_CLK(NULL, "dcan0_fck", "dcan0_fck"),
-	DT_CLK("481cc000.d_can", NULL, "dcan0_fck"),
-	DT_CLK(NULL, "dcan1_fck", "dcan1_fck"),
-	DT_CLK("481d0000.d_can", NULL, "dcan1_fck"),
-	DT_CLK(NULL, "pruss_ocp_gclk", "pruss_ocp_gclk"),
-	DT_CLK(NULL, "mcasp0_fck", "mcasp0_fck"),
-	DT_CLK(NULL, "mcasp1_fck", "mcasp1_fck"),
-	DT_CLK(NULL, "mmu_fck", "mmu_fck"),
-	DT_CLK(NULL, "smartreflex0_fck", "smartreflex0_fck"),
-	DT_CLK(NULL, "smartreflex1_fck", "smartreflex1_fck"),
-	DT_CLK(NULL, "sha0_fck", "sha0_fck"),
-	DT_CLK(NULL, "aes0_fck", "aes0_fck"),
-	DT_CLK(NULL, "rng_fck", "rng_fck"),
-	DT_CLK(NULL, "timer1_fck", "timer1_fck"),
-	DT_CLK(NULL, "timer2_fck", "timer2_fck"),
-	DT_CLK(NULL, "timer3_fck", "timer3_fck"),
-	DT_CLK(NULL, "timer4_fck", "timer4_fck"),
-	DT_CLK(NULL, "timer5_fck", "timer5_fck"),
-	DT_CLK(NULL, "timer6_fck", "timer6_fck"),
-	DT_CLK(NULL, "timer7_fck", "timer7_fck"),
-	DT_CLK(NULL, "usbotg_fck", "usbotg_fck"),
-	DT_CLK(NULL, "ieee5000_fck", "ieee5000_fck"),
-	DT_CLK(NULL, "wdt1_fck", "wdt1_fck"),
-	DT_CLK(NULL, "l4_rtc_gclk", "l4_rtc_gclk"),
-	DT_CLK(NULL, "l3_gclk", "l3_gclk"),
-	DT_CLK(NULL, "dpll_core_m4_div2_ck", "dpll_core_m4_div2_ck"),
-	DT_CLK(NULL, "l4hs_gclk", "l4hs_gclk"),
-	DT_CLK(NULL, "l3s_gclk", "l3s_gclk"),
-	DT_CLK(NULL, "l4fw_gclk", "l4fw_gclk"),
-	DT_CLK(NULL, "l4ls_gclk", "l4ls_gclk"),
-	DT_CLK(NULL, "clk_24mhz", "clk_24mhz"),
-	DT_CLK(NULL, "sysclk_div_ck", "sysclk_div_ck"),
-	DT_CLK(NULL, "cpsw_125mhz_gclk", "cpsw_125mhz_gclk"),
-	DT_CLK(NULL, "cpsw_cpts_rft_clk", "cpsw_cpts_rft_clk"),
-	DT_CLK(NULL, "gpio0_dbclk_mux_ck", "gpio0_dbclk_mux_ck"),
-	DT_CLK(NULL, "gpio0_dbclk", "gpio0_dbclk"),
-	DT_CLK(NULL, "gpio1_dbclk", "gpio1_dbclk"),
-	DT_CLK(NULL, "gpio2_dbclk", "gpio2_dbclk"),
-	DT_CLK(NULL, "gpio3_dbclk", "gpio3_dbclk"),
-	DT_CLK(NULL, "lcd_gclk", "lcd_gclk"),
-	DT_CLK(NULL, "mmc_clk", "mmc_clk"),
-	DT_CLK(NULL, "gfx_fclk_clksel_ck", "gfx_fclk_clksel_ck"),
-	DT_CLK(NULL, "gfx_fck_div_ck", "gfx_fck_div_ck"),
-	DT_CLK(NULL, "sysclkout_pre_ck", "sysclkout_pre_ck"),
-	DT_CLK(NULL, "clkout2_div_ck", "clkout2_div_ck"),
-	DT_CLK(NULL, "timer_32k_ck", "clkdiv32k_ick"),
+	DT_CLK(NULL, "timer_32k_ck", "l4_per_cm:0138:0"),
 	DT_CLK(NULL, "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK(NULL, "dbg_sysclk_ck", "dbg_sysclk_ck"),
-	DT_CLK(NULL, "dbg_clka_ck", "dbg_clka_ck"),
-	DT_CLK(NULL, "stm_pmd_clock_mux_ck", "stm_pmd_clock_mux_ck"),
-	DT_CLK(NULL, "trace_pmd_clk_mux_ck", "trace_pmd_clk_mux_ck"),
-	DT_CLK(NULL, "stm_clk_div_ck", "stm_clk_div_ck"),
-	DT_CLK(NULL, "trace_clk_div_ck", "trace_clk_div_ck"),
-	DT_CLK(NULL, "clkout2_ck", "clkout2_ck"),
-	DT_CLK("48300200.ehrpwm", "tbclk", "ehrpwm0_tbclk"),
-	DT_CLK("48302200.ehrpwm", "tbclk", "ehrpwm1_tbclk"),
-	DT_CLK("48304200.ehrpwm", "tbclk", "ehrpwm2_tbclk"),
-	DT_CLK("48300200.pwm", "tbclk", "ehrpwm0_tbclk"),
-	DT_CLK("48302200.pwm", "tbclk", "ehrpwm1_tbclk"),
-	DT_CLK("48304200.pwm", "tbclk", "ehrpwm2_tbclk"),
+	DT_CLK(NULL, "clkdiv32k_ick", "l4_per_cm:0138:0"),
+	DT_CLK(NULL, "dbg_clka_ck", "l4_wkup_cm:0010:30"),
+	DT_CLK(NULL, "dbg_sysclk_ck", "l4_wkup_cm:0010:19"),
+	DT_CLK(NULL, "gpio0_dbclk", "l4_wkup_cm:0004:18"),
+	DT_CLK(NULL, "gpio1_dbclk", "l4_per_cm:0098:18"),
+	DT_CLK(NULL, "gpio2_dbclk", "l4_per_cm:009c:18"),
+	DT_CLK(NULL, "gpio3_dbclk", "l4_per_cm:00a0:18"),
+	DT_CLK(NULL, "stm_clk_div_ck", "l4_wkup_cm:0010:27"),
+	DT_CLK(NULL, "stm_pmd_clock_mux_ck", "l4_wkup_cm:0010:22"),
+	DT_CLK(NULL, "trace_clk_div_ck", "l4_wkup_cm:0010:24"),
+	DT_CLK(NULL, "trace_pmd_clk_mux_ck", "l4_wkup_cm:0010:20"),
 	{ .node_name = NULL },
 };
 
@@ -133,6 +236,8 @@ int __init am33xx_dt_clk_init(void)
 
 	omap2_clk_disable_autoidle_all();
 
+	ti_clk_add_aliases();
+
 	omap2_clk_enable_init_clocks(enable_init_clks,
 				     ARRAY_SIZE(enable_init_clks));
 
diff --git a/drivers/clk/ti/clk-3xxx-legacy.c b/drivers/clk/ti/clk-3xxx-legacy.c
deleted file mode 100644
index 0fbf8a917955..000000000000
--- a/drivers/clk/ti/clk-3xxx-legacy.c
+++ /dev/null
@@ -1,4656 +0,0 @@
-/*
- * OMAP3 Legacy clock data
- *
- * Copyright (C) 2014 Texas Instruments, Inc
- *     Tero Kristo (t-kristo@ti.com)
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation version 2.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
-
-#include <linux/kernel.h>
-#include <linux/clk.h>
-#include <linux/clk-provider.h>
-#include <linux/clk/ti.h>
-
-#include "clock.h"
-
-static struct ti_clk_fixed virt_12m_ck_data = {
-	.frequency = 12000000,
-};
-
-static struct ti_clk virt_12m_ck = {
-	.name = "virt_12m_ck",
-	.type = TI_CLK_FIXED,
-	.data = &virt_12m_ck_data,
-};
-
-static struct ti_clk_fixed virt_13m_ck_data = {
-	.frequency = 13000000,
-};
-
-static struct ti_clk virt_13m_ck = {
-	.name = "virt_13m_ck",
-	.type = TI_CLK_FIXED,
-	.data = &virt_13m_ck_data,
-};
-
-static struct ti_clk_fixed virt_19200000_ck_data = {
-	.frequency = 19200000,
-};
-
-static struct ti_clk virt_19200000_ck = {
-	.name = "virt_19200000_ck",
-	.type = TI_CLK_FIXED,
-	.data = &virt_19200000_ck_data,
-};
-
-static struct ti_clk_fixed virt_26000000_ck_data = {
-	.frequency = 26000000,
-};
-
-static struct ti_clk virt_26000000_ck = {
-	.name = "virt_26000000_ck",
-	.type = TI_CLK_FIXED,
-	.data = &virt_26000000_ck_data,
-};
-
-static struct ti_clk_fixed virt_38_4m_ck_data = {
-	.frequency = 38400000,
-};
-
-static struct ti_clk virt_38_4m_ck = {
-	.name = "virt_38_4m_ck",
-	.type = TI_CLK_FIXED,
-	.data = &virt_38_4m_ck_data,
-};
-
-static struct ti_clk_fixed virt_16_8m_ck_data = {
-	.frequency = 16800000,
-};
-
-static struct ti_clk virt_16_8m_ck = {
-	.name = "virt_16_8m_ck",
-	.type = TI_CLK_FIXED,
-	.data = &virt_16_8m_ck_data,
-};
-
-static const char *osc_sys_ck_parents[] = {
-	"virt_12m_ck",
-	"virt_13m_ck",
-	"virt_19200000_ck",
-	"virt_26000000_ck",
-	"virt_38_4m_ck",
-	"virt_16_8m_ck",
-};
-
-static struct ti_clk_mux osc_sys_ck_data = {
-	.num_parents = ARRAY_SIZE(osc_sys_ck_parents),
-	.reg = 0xd40,
-	.module = TI_CLKM_PRM,
-	.parents = osc_sys_ck_parents,
-};
-
-static struct ti_clk osc_sys_ck = {
-	.name = "osc_sys_ck",
-	.type = TI_CLK_MUX,
-	.data = &osc_sys_ck_data,
-};
-
-static struct ti_clk_divider sys_ck_data = {
-	.parent = "osc_sys_ck",
-	.bit_shift = 6,
-	.max_div = 3,
-	.reg = 0x1270,
-	.module = TI_CLKM_PRM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk sys_ck = {
-	.name = "sys_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &sys_ck_data,
-};
-
-static const char *dpll3_ck_parents[] = {
-	"sys_ck",
-	"sys_ck",
-};
-
-static struct ti_clk_dpll dpll3_ck_data = {
-	.num_parents = ARRAY_SIZE(dpll3_ck_parents),
-	.control_reg = 0xd00,
-	.idlest_reg = 0xd20,
-	.mult_div1_reg = 0xd40,
-	.autoidle_reg = 0xd30,
-	.module = TI_CLKM_CM,
-	.parents = dpll3_ck_parents,
-	.flags = CLKF_CORE,
-	.freqsel_mask = 0xf0,
-	.div1_mask = 0x7f00,
-	.idlest_mask = 0x1,
-	.auto_recal_bit = 0x3,
-	.max_divider = 0x80,
-	.min_divider = 0x1,
-	.recal_en_bit = 0x5,
-	.max_multiplier = 0x7ff,
-	.enable_mask = 0x7,
-	.mult_mask = 0x7ff0000,
-	.recal_st_bit = 0x5,
-	.autoidle_mask = 0x7,
-};
-
-static struct ti_clk dpll3_ck = {
-	.name = "dpll3_ck",
-	.clkdm_name = "dpll3_clkdm",
-	.type = TI_CLK_DPLL,
-	.data = &dpll3_ck_data,
-};
-
-static struct ti_clk_divider dpll3_m2_ck_data = {
-	.parent = "dpll3_ck",
-	.bit_shift = 27,
-	.max_div = 31,
-	.reg = 0xd40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll3_m2_ck = {
-	.name = "dpll3_m2_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll3_m2_ck_data,
-};
-
-static struct ti_clk_fixed_factor core_ck_data = {
-	.parent = "dpll3_m2_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk core_ck = {
-	.name = "core_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_ck_data,
-};
-
-static struct ti_clk_divider l3_ick_data = {
-	.parent = "core_ck",
-	.max_div = 3,
-	.reg = 0xa40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk l3_ick = {
-	.name = "l3_ick",
-	.type = TI_CLK_DIVIDER,
-	.data = &l3_ick_data,
-};
-
-static struct ti_clk_fixed_factor security_l3_ick_data = {
-	.parent = "l3_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk security_l3_ick = {
-	.name = "security_l3_ick",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &security_l3_ick_data,
-};
-
-static struct ti_clk_fixed_factor wkup_l4_ick_data = {
-	.parent = "sys_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk wkup_l4_ick = {
-	.name = "wkup_l4_ick",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &wkup_l4_ick_data,
-};
-
-static struct ti_clk_gate usim_ick_data = {
-	.parent = "wkup_l4_ick",
-	.bit_shift = 9,
-	.reg = 0xc10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk usim_ick = {
-	.name = "usim_ick",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &usim_ick_data,
-};
-
-static struct ti_clk_gate dss2_alwon_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 1,
-	.reg = 0xe00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk dss2_alwon_fck = {
-	.name = "dss2_alwon_fck",
-	.clkdm_name = "dss_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &dss2_alwon_fck_data,
-};
-
-static struct ti_clk_divider l4_ick_data = {
-	.parent = "l3_ick",
-	.bit_shift = 2,
-	.max_div = 3,
-	.reg = 0xa40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk l4_ick = {
-	.name = "l4_ick",
-	.type = TI_CLK_DIVIDER,
-	.data = &l4_ick_data,
-};
-
-static struct ti_clk_fixed_factor core_l4_ick_data = {
-	.parent = "l4_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk core_l4_ick = {
-	.name = "core_l4_ick",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_l4_ick_data,
-};
-
-static struct ti_clk_gate mmchs2_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 25,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mmchs2_ick = {
-	.name = "mmchs2_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mmchs2_ick_data,
-};
-
-static const char *dpll4_ck_parents[] = {
-	"sys_ck",
-	"sys_ck",
-};
-
-static struct ti_clk_dpll dpll4_ck_data = {
-	.num_parents = ARRAY_SIZE(dpll4_ck_parents),
-	.control_reg = 0xd00,
-	.idlest_reg = 0xd20,
-	.mult_div1_reg = 0xd44,
-	.autoidle_reg = 0xd30,
-	.module = TI_CLKM_CM,
-	.parents = dpll4_ck_parents,
-	.flags = CLKF_PER,
-	.freqsel_mask = 0xf00000,
-	.modes = 0x82,
-	.div1_mask = 0x7f,
-	.idlest_mask = 0x2,
-	.auto_recal_bit = 0x13,
-	.max_divider = 0x80,
-	.min_divider = 0x1,
-	.recal_en_bit = 0x6,
-	.max_multiplier = 0x7ff,
-	.enable_mask = 0x70000,
-	.mult_mask = 0x7ff00,
-	.recal_st_bit = 0x6,
-	.autoidle_mask = 0x38,
-};
-
-static struct ti_clk dpll4_ck = {
-	.name = "dpll4_ck",
-	.clkdm_name = "dpll4_clkdm",
-	.type = TI_CLK_DPLL,
-	.data = &dpll4_ck_data,
-};
-
-static struct ti_clk_divider dpll4_m2_ck_data = {
-	.parent = "dpll4_ck",
-	.max_div = 63,
-	.reg = 0xd48,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll4_m2_ck = {
-	.name = "dpll4_m2_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll4_m2_ck_data,
-};
-
-static struct ti_clk_fixed_factor dpll4_m2x2_mul_ck_data = {
-	.parent = "dpll4_m2_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll4_m2x2_mul_ck = {
-	.name = "dpll4_m2x2_mul_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll4_m2x2_mul_ck_data,
-};
-
-static struct ti_clk_gate dpll4_m2x2_ck_data = {
-	.parent = "dpll4_m2x2_mul_ck",
-	.bit_shift = 0x1b,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m2x2_ck = {
-	.name = "dpll4_m2x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m2x2_ck_data,
-};
-
-static struct ti_clk_fixed_factor omap_96m_alwon_fck_data = {
-	.parent = "dpll4_m2x2_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk omap_96m_alwon_fck = {
-	.name = "omap_96m_alwon_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &omap_96m_alwon_fck_data,
-};
-
-static struct ti_clk_fixed_factor cm_96m_fck_data = {
-	.parent = "omap_96m_alwon_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk cm_96m_fck = {
-	.name = "cm_96m_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &cm_96m_fck_data,
-};
-
-static const char *omap_96m_fck_parents[] = {
-	"cm_96m_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux omap_96m_fck_data = {
-	.bit_shift = 6,
-	.num_parents = ARRAY_SIZE(omap_96m_fck_parents),
-	.reg = 0xd40,
-	.module = TI_CLKM_CM,
-	.parents = omap_96m_fck_parents,
-};
-
-static struct ti_clk omap_96m_fck = {
-	.name = "omap_96m_fck",
-	.type = TI_CLK_MUX,
-	.data = &omap_96m_fck_data,
-};
-
-static struct ti_clk_fixed_factor core_96m_fck_data = {
-	.parent = "omap_96m_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk core_96m_fck = {
-	.name = "core_96m_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_96m_fck_data,
-};
-
-static struct ti_clk_gate mspro_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 23,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mspro_fck = {
-	.name = "mspro_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mspro_fck_data,
-};
-
-static struct ti_clk_gate dss_ick_3430es2_data = {
-	.parent = "l4_ick",
-	.bit_shift = 0,
-	.reg = 0xe10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_DSS | CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk dss_ick_3430es2 = {
-	.name = "dss_ick",
-	.clkdm_name = "dss_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &dss_ick_3430es2_data,
-};
-
-static struct ti_clk_gate uart4_ick_am35xx_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 23,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk uart4_ick_am35xx = {
-	.name = "uart4_ick_am35xx",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart4_ick_am35xx_data,
-};
-
-static struct ti_clk_fixed_factor security_l4_ick2_data = {
-	.parent = "l4_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk security_l4_ick2 = {
-	.name = "security_l4_ick2",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &security_l4_ick2_data,
-};
-
-static struct ti_clk_gate aes1_ick_data = {
-	.parent = "security_l4_ick2",
-	.bit_shift = 3,
-	.reg = 0xa14,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk aes1_ick = {
-	.name = "aes1_ick",
-	.type = TI_CLK_GATE,
-	.data = &aes1_ick_data,
-};
-
-static const char *dpll5_ck_parents[] = {
-	"sys_ck",
-	"sys_ck",
-};
-
-static struct ti_clk_dpll dpll5_ck_data = {
-	.num_parents = ARRAY_SIZE(dpll5_ck_parents),
-	.control_reg = 0xd04,
-	.idlest_reg = 0xd24,
-	.mult_div1_reg = 0xd4c,
-	.autoidle_reg = 0xd34,
-	.module = TI_CLKM_CM,
-	.parents = dpll5_ck_parents,
-	.freqsel_mask = 0xf0,
-	.modes = 0x82,
-	.div1_mask = 0x7f,
-	.idlest_mask = 0x1,
-	.auto_recal_bit = 0x3,
-	.max_divider = 0x80,
-	.min_divider = 0x1,
-	.recal_en_bit = 0x19,
-	.max_multiplier = 0x7ff,
-	.enable_mask = 0x7,
-	.mult_mask = 0x7ff00,
-	.recal_st_bit = 0x19,
-	.autoidle_mask = 0x7,
-};
-
-static struct ti_clk dpll5_ck = {
-	.name = "dpll5_ck",
-	.clkdm_name = "dpll5_clkdm",
-	.type = TI_CLK_DPLL,
-	.data = &dpll5_ck_data,
-};
-
-static struct ti_clk_divider dpll5_m2_ck_data = {
-	.parent = "dpll5_ck",
-	.max_div = 31,
-	.reg = 0xd50,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll5_m2_ck = {
-	.name = "dpll5_m2_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll5_m2_ck_data,
-};
-
-static struct ti_clk_gate usbhost_120m_fck_data = {
-	.parent = "dpll5_m2_ck",
-	.bit_shift = 1,
-	.reg = 0x1400,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk usbhost_120m_fck = {
-	.name = "usbhost_120m_fck",
-	.clkdm_name = "usbhost_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &usbhost_120m_fck_data,
-};
-
-static struct ti_clk_fixed_factor cm_96m_d2_fck_data = {
-	.parent = "cm_96m_fck",
-	.div = 2,
-	.mult = 1,
-};
-
-static struct ti_clk cm_96m_d2_fck = {
-	.name = "cm_96m_d2_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &cm_96m_d2_fck_data,
-};
-
-static struct ti_clk_fixed sys_altclk_data = {
-	.frequency = 0x0,
-};
-
-static struct ti_clk sys_altclk = {
-	.name = "sys_altclk",
-	.type = TI_CLK_FIXED,
-	.data = &sys_altclk_data,
-};
-
-static const char *omap_48m_fck_parents[] = {
-	"cm_96m_d2_fck",
-	"sys_altclk",
-};
-
-static struct ti_clk_mux omap_48m_fck_data = {
-	.bit_shift = 3,
-	.num_parents = ARRAY_SIZE(omap_48m_fck_parents),
-	.reg = 0xd40,
-	.module = TI_CLKM_CM,
-	.parents = omap_48m_fck_parents,
-};
-
-static struct ti_clk omap_48m_fck = {
-	.name = "omap_48m_fck",
-	.type = TI_CLK_MUX,
-	.data = &omap_48m_fck_data,
-};
-
-static struct ti_clk_fixed_factor core_48m_fck_data = {
-	.parent = "omap_48m_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk core_48m_fck = {
-	.name = "core_48m_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_48m_fck_data,
-};
-
-static struct ti_clk_fixed mcbsp_clks_data = {
-	.frequency = 0x0,
-};
-
-static struct ti_clk mcbsp_clks = {
-	.name = "mcbsp_clks",
-	.type = TI_CLK_FIXED,
-	.data = &mcbsp_clks_data,
-};
-
-static struct ti_clk_gate mcbsp2_gate_fck_data = {
-	.parent = "mcbsp_clks",
-	.bit_shift = 0,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_fixed_factor per_96m_fck_data = {
-	.parent = "omap_96m_alwon_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk per_96m_fck = {
-	.name = "per_96m_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &per_96m_fck_data,
-};
-
-static const char *mcbsp2_mux_fck_parents[] = {
-	"per_96m_fck",
-	"mcbsp_clks",
-};
-
-static struct ti_clk_mux mcbsp2_mux_fck_data = {
-	.bit_shift = 6,
-	.num_parents = ARRAY_SIZE(mcbsp2_mux_fck_parents),
-	.reg = 0x274,
-	.module = TI_CLKM_SCRM,
-	.parents = mcbsp2_mux_fck_parents,
-};
-
-static struct ti_clk_composite mcbsp2_fck_data = {
-	.mux = &mcbsp2_mux_fck_data,
-	.gate = &mcbsp2_gate_fck_data,
-};
-
-static struct ti_clk mcbsp2_fck = {
-	.name = "mcbsp2_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &mcbsp2_fck_data,
-};
-
-static struct ti_clk_fixed_factor dpll3_m2x2_ck_data = {
-	.parent = "dpll3_m2_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll3_m2x2_ck = {
-	.name = "dpll3_m2x2_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll3_m2x2_ck_data,
-};
-
-static struct ti_clk_fixed_factor corex2_fck_data = {
-	.parent = "dpll3_m2x2_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk corex2_fck = {
-	.name = "corex2_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &corex2_fck_data,
-};
-
-static struct ti_clk_gate ssi_ssr_gate_fck_3430es1_data = {
-	.parent = "corex2_fck",
-	.bit_shift = 0,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_NO_WAIT,
-};
-
-static int ssi_ssr_div_fck_3430es1_divs[] = {
-	0,
-	1,
-	2,
-	3,
-	4,
-	0,
-	6,
-	0,
-	8,
-};
-
-static struct ti_clk_divider ssi_ssr_div_fck_3430es1_data = {
-	.num_dividers = ARRAY_SIZE(ssi_ssr_div_fck_3430es1_divs),
-	.parent = "corex2_fck",
-	.bit_shift = 8,
-	.dividers = ssi_ssr_div_fck_3430es1_divs,
-	.reg = 0xa40,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_composite ssi_ssr_fck_3430es1_data = {
-	.gate = &ssi_ssr_gate_fck_3430es1_data,
-	.divider = &ssi_ssr_div_fck_3430es1_data,
-};
-
-static struct ti_clk ssi_ssr_fck_3430es1 = {
-	.name = "ssi_ssr_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &ssi_ssr_fck_3430es1_data,
-};
-
-static struct ti_clk_fixed_factor ssi_sst_fck_3430es1_data = {
-	.parent = "ssi_ssr_fck",
-	.div = 2,
-	.mult = 1,
-};
-
-static struct ti_clk ssi_sst_fck_3430es1 = {
-	.name = "ssi_sst_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &ssi_sst_fck_3430es1_data,
-};
-
-static struct ti_clk_fixed omap_32k_fck_data = {
-	.frequency = 32768,
-};
-
-static struct ti_clk omap_32k_fck = {
-	.name = "omap_32k_fck",
-	.type = TI_CLK_FIXED,
-	.data = &omap_32k_fck_data,
-};
-
-static struct ti_clk_fixed_factor per_32k_alwon_fck_data = {
-	.parent = "omap_32k_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk per_32k_alwon_fck = {
-	.name = "per_32k_alwon_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &per_32k_alwon_fck_data,
-};
-
-static struct ti_clk_gate gpio5_dbck_data = {
-	.parent = "per_32k_alwon_fck",
-	.bit_shift = 16,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk gpio5_dbck = {
-	.name = "gpio5_dbck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio5_dbck_data,
-};
-
-static struct ti_clk_gate gpt1_ick_data = {
-	.parent = "wkup_l4_ick",
-	.bit_shift = 0,
-	.reg = 0xc10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt1_ick = {
-	.name = "gpt1_ick",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt1_ick_data,
-};
-
-static struct ti_clk_gate mcspi3_fck_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 20,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mcspi3_fck = {
-	.name = "mcspi3_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi3_fck_data,
-};
-
-static struct ti_clk_gate gpt2_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 3,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static const char *gpt2_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt2_mux_fck_data = {
-	.num_parents = ARRAY_SIZE(gpt2_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt2_mux_fck_parents,
-};
-
-static struct ti_clk_composite gpt2_fck_data = {
-	.mux = &gpt2_mux_fck_data,
-	.gate = &gpt2_gate_fck_data,
-};
-
-static struct ti_clk gpt2_fck = {
-	.name = "gpt2_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt2_fck_data,
-};
-
-static struct ti_clk_gate gpt10_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 11,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt10_ick = {
-	.name = "gpt10_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt10_ick_data,
-};
-
-static struct ti_clk_gate uart2_fck_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 14,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk uart2_fck = {
-	.name = "uart2_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart2_fck_data,
-};
-
-static struct ti_clk_fixed_factor sr_l4_ick_data = {
-	.parent = "l4_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk sr_l4_ick = {
-	.name = "sr_l4_ick",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &sr_l4_ick_data,
-};
-
-static struct ti_clk_fixed_factor omap_96m_d8_fck_data = {
-	.parent = "omap_96m_fck",
-	.div = 8,
-	.mult = 1,
-};
-
-static struct ti_clk omap_96m_d8_fck = {
-	.name = "omap_96m_d8_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &omap_96m_d8_fck_data,
-};
-
-static struct ti_clk_divider dpll4_m5_ck_data = {
-	.parent = "dpll4_ck",
-	.max_div = 63,
-	.reg = 0xf40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll4_m5_ck = {
-	.name = "dpll4_m5_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll4_m5_ck_data,
-};
-
-static struct ti_clk_fixed_factor dpll4_m5x2_mul_ck_data = {
-	.parent = "dpll4_m5_ck",
-	.div = 1,
-	.mult = 2,
-	.flags = CLKF_SET_RATE_PARENT,
-};
-
-static struct ti_clk dpll4_m5x2_mul_ck = {
-	.name = "dpll4_m5x2_mul_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll4_m5x2_mul_ck_data,
-};
-
-static struct ti_clk_gate dpll4_m5x2_ck_data = {
-	.parent = "dpll4_m5x2_mul_ck",
-	.bit_shift = 0x1e,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m5x2_ck = {
-	.name = "dpll4_m5x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m5x2_ck_data,
-};
-
-static struct ti_clk_gate cam_mclk_data = {
-	.parent = "dpll4_m5x2_ck",
-	.bit_shift = 0,
-	.reg = 0xf00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_RATE_PARENT,
-};
-
-static struct ti_clk cam_mclk = {
-	.name = "cam_mclk",
-	.type = TI_CLK_GATE,
-	.data = &cam_mclk_data,
-};
-
-static struct ti_clk_gate mcbsp3_gate_fck_data = {
-	.parent = "mcbsp_clks",
-	.bit_shift = 1,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static const char *mcbsp3_mux_fck_parents[] = {
-	"per_96m_fck",
-	"mcbsp_clks",
-};
-
-static struct ti_clk_mux mcbsp3_mux_fck_data = {
-	.num_parents = ARRAY_SIZE(mcbsp3_mux_fck_parents),
-	.reg = 0x2d8,
-	.module = TI_CLKM_SCRM,
-	.parents = mcbsp3_mux_fck_parents,
-};
-
-static struct ti_clk_composite mcbsp3_fck_data = {
-	.mux = &mcbsp3_mux_fck_data,
-	.gate = &mcbsp3_gate_fck_data,
-};
-
-static struct ti_clk mcbsp3_fck = {
-	.name = "mcbsp3_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &mcbsp3_fck_data,
-};
-
-static struct ti_clk_gate csi2_96m_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 1,
-	.reg = 0xf00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk csi2_96m_fck = {
-	.name = "csi2_96m_fck",
-	.clkdm_name = "cam_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &csi2_96m_fck_data,
-};
-
-static struct ti_clk_gate gpt9_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 10,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static const char *gpt9_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt9_mux_fck_data = {
-	.bit_shift = 7,
-	.num_parents = ARRAY_SIZE(gpt9_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt9_mux_fck_parents,
-};
-
-static struct ti_clk_composite gpt9_fck_data = {
-	.mux = &gpt9_mux_fck_data,
-	.gate = &gpt9_gate_fck_data,
-};
-
-static struct ti_clk gpt9_fck = {
-	.name = "gpt9_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt9_fck_data,
-};
-
-static struct ti_clk_divider dpll3_m3_ck_data = {
-	.parent = "dpll3_ck",
-	.bit_shift = 16,
-	.max_div = 31,
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll3_m3_ck = {
-	.name = "dpll3_m3_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll3_m3_ck_data,
-};
-
-static struct ti_clk_fixed_factor dpll3_m3x2_mul_ck_data = {
-	.parent = "dpll3_m3_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll3_m3x2_mul_ck = {
-	.name = "dpll3_m3x2_mul_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll3_m3x2_mul_ck_data,
-};
-
-static struct ti_clk_gate sr2_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 7,
-	.reg = 0xc00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk sr2_fck = {
-	.name = "sr2_fck",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &sr2_fck_data,
-};
-
-static struct ti_clk_fixed pclk_ck_data = {
-	.frequency = 27000000,
-};
-
-static struct ti_clk pclk_ck = {
-	.name = "pclk_ck",
-	.type = TI_CLK_FIXED,
-	.data = &pclk_ck_data,
-};
-
-static struct ti_clk_gate wdt2_ick_data = {
-	.parent = "wkup_l4_ick",
-	.bit_shift = 5,
-	.reg = 0xc10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk wdt2_ick = {
-	.name = "wdt2_ick",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &wdt2_ick_data,
-};
-
-static struct ti_clk_fixed_factor core_l3_ick_data = {
-	.parent = "l3_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk core_l3_ick = {
-	.name = "core_l3_ick",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_l3_ick_data,
-};
-
-static struct ti_clk_gate mcspi4_fck_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 21,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mcspi4_fck = {
-	.name = "mcspi4_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi4_fck_data,
-};
-
-static struct ti_clk_fixed_factor per_48m_fck_data = {
-	.parent = "omap_48m_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk per_48m_fck = {
-	.name = "per_48m_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &per_48m_fck_data,
-};
-
-static struct ti_clk_gate uart4_fck_data = {
-	.parent = "per_48m_fck",
-	.bit_shift = 18,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk uart4_fck = {
-	.name = "uart4_fck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart4_fck_data,
-};
-
-static struct ti_clk_fixed_factor omap_96m_d10_fck_data = {
-	.parent = "omap_96m_fck",
-	.div = 10,
-	.mult = 1,
-};
-
-static struct ti_clk omap_96m_d10_fck = {
-	.name = "omap_96m_d10_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &omap_96m_d10_fck_data,
-};
-
-static struct ti_clk_gate usim_gate_fck_data = {
-	.parent = "omap_96m_fck",
-	.bit_shift = 9,
-	.reg = 0xc00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_fixed_factor per_l4_ick_data = {
-	.parent = "l4_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk per_l4_ick = {
-	.name = "per_l4_ick",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &per_l4_ick_data,
-};
-
-static struct ti_clk_gate gpt5_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 6,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt5_ick = {
-	.name = "gpt5_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt5_ick_data,
-};
-
-static struct ti_clk_gate mcspi2_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 19,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcspi2_ick = {
-	.name = "mcspi2_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi2_ick_data,
-};
-
-static struct ti_clk_fixed_factor ssi_l4_ick_data = {
-	.parent = "l4_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk ssi_l4_ick = {
-	.name = "ssi_l4_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &ssi_l4_ick_data,
-};
-
-static struct ti_clk_gate ssi_ick_3430es1_data = {
-	.parent = "ssi_l4_ick",
-	.bit_shift = 0,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_NO_WAIT | CLKF_INTERFACE,
-};
-
-static struct ti_clk ssi_ick_3430es1 = {
-	.name = "ssi_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &ssi_ick_3430es1_data,
-};
-
-static struct ti_clk_gate i2c2_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 16,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk i2c2_fck = {
-	.name = "i2c2_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &i2c2_fck_data,
-};
-
-static struct ti_clk_divider dpll1_fck_data = {
-	.parent = "core_ck",
-	.bit_shift = 19,
-	.max_div = 7,
-	.reg = 0x940,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll1_fck = {
-	.name = "dpll1_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll1_fck_data,
-};
-
-static const char *dpll1_ck_parents[] = {
-	"sys_ck",
-	"dpll1_fck",
-};
-
-static struct ti_clk_dpll dpll1_ck_data = {
-	.num_parents = ARRAY_SIZE(dpll1_ck_parents),
-	.control_reg = 0x904,
-	.idlest_reg = 0x924,
-	.mult_div1_reg = 0x940,
-	.autoidle_reg = 0x934,
-	.module = TI_CLKM_CM,
-	.parents = dpll1_ck_parents,
-	.freqsel_mask = 0xf0,
-	.modes = 0xa0,
-	.div1_mask = 0x7f,
-	.idlest_mask = 0x1,
-	.auto_recal_bit = 0x3,
-	.max_divider = 0x80,
-	.min_divider = 0x1,
-	.recal_en_bit = 0x7,
-	.max_multiplier = 0x7ff,
-	.enable_mask = 0x7,
-	.mult_mask = 0x7ff00,
-	.recal_st_bit = 0x7,
-	.autoidle_mask = 0x7,
-};
-
-static struct ti_clk dpll1_ck = {
-	.name = "dpll1_ck",
-	.clkdm_name = "dpll1_clkdm",
-	.type = TI_CLK_DPLL,
-	.data = &dpll1_ck_data,
-};
-
-static struct ti_clk_fixed secure_32k_fck_data = {
-	.frequency = 32768,
-};
-
-static struct ti_clk secure_32k_fck = {
-	.name = "secure_32k_fck",
-	.type = TI_CLK_FIXED,
-	.data = &secure_32k_fck_data,
-};
-
-static struct ti_clk_gate gpio5_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 16,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpio5_ick = {
-	.name = "gpio5_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio5_ick_data,
-};
-
-static struct ti_clk_divider dpll4_m4_ck_data = {
-	.parent = "dpll4_ck",
-	.max_div = 32,
-	.reg = 0xe40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll4_m4_ck = {
-	.name = "dpll4_m4_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll4_m4_ck_data,
-};
-
-static struct ti_clk_fixed_factor dpll4_m4x2_mul_ck_data = {
-	.parent = "dpll4_m4_ck",
-	.div = 1,
-	.mult = 2,
-	.flags = CLKF_SET_RATE_PARENT,
-};
-
-static struct ti_clk dpll4_m4x2_mul_ck = {
-	.name = "dpll4_m4x2_mul_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll4_m4x2_mul_ck_data,
-};
-
-static struct ti_clk_gate dpll4_m4x2_ck_data = {
-	.parent = "dpll4_m4x2_mul_ck",
-	.bit_shift = 0x1d,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_RATE_PARENT | CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m4x2_ck = {
-	.name = "dpll4_m4x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m4x2_ck_data,
-};
-
-static struct ti_clk_gate dss1_alwon_fck_3430es2_data = {
-	.parent = "dpll4_m4x2_ck",
-	.bit_shift = 0,
-	.reg = 0xe00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_DSS | CLKF_SET_RATE_PARENT,
-};
-
-static struct ti_clk dss1_alwon_fck_3430es2 = {
-	.name = "dss1_alwon_fck",
-	.clkdm_name = "dss_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &dss1_alwon_fck_3430es2_data,
-};
-
-static struct ti_clk_gate uart3_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 11,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk uart3_ick = {
-	.name = "uart3_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart3_ick_data,
-};
-
-static struct ti_clk_divider dpll4_m3_ck_data = {
-	.parent = "dpll4_ck",
-	.bit_shift = 8,
-	.max_div = 32,
-	.reg = 0xe40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll4_m3_ck = {
-	.name = "dpll4_m3_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll4_m3_ck_data,
-};
-
-static struct ti_clk_gate mcbsp3_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 1,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcbsp3_ick = {
-	.name = "mcbsp3_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcbsp3_ick_data,
-};
-
-static struct ti_clk_gate gpio3_dbck_data = {
-	.parent = "per_32k_alwon_fck",
-	.bit_shift = 14,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk gpio3_dbck = {
-	.name = "gpio3_dbck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio3_dbck_data,
-};
-
-static struct ti_clk_gate fac_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 8,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk fac_ick = {
-	.name = "fac_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &fac_ick_data,
-};
-
-static struct ti_clk_gate clkout2_src_gate_ck_data = {
-	.parent = "core_ck",
-	.bit_shift = 7,
-	.reg = 0xd70,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_NO_WAIT,
-};
-
-static struct ti_clk_fixed_factor dpll4_m3x2_mul_ck_data = {
-	.parent = "dpll4_m3_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll4_m3x2_mul_ck = {
-	.name = "dpll4_m3x2_mul_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll4_m3x2_mul_ck_data,
-};
-
-static struct ti_clk_gate dpll4_m3x2_ck_data = {
-	.parent = "dpll4_m3x2_mul_ck",
-	.bit_shift = 0x1c,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m3x2_ck = {
-	.name = "dpll4_m3x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m3x2_ck_data,
-};
-
-static const char *omap_54m_fck_parents[] = {
-	"dpll4_m3x2_ck",
-	"sys_altclk",
-};
-
-static struct ti_clk_mux omap_54m_fck_data = {
-	.bit_shift = 5,
-	.num_parents = ARRAY_SIZE(omap_54m_fck_parents),
-	.reg = 0xd40,
-	.module = TI_CLKM_CM,
-	.parents = omap_54m_fck_parents,
-};
-
-static struct ti_clk omap_54m_fck = {
-	.name = "omap_54m_fck",
-	.type = TI_CLK_MUX,
-	.data = &omap_54m_fck_data,
-};
-
-static const char *clkout2_src_mux_ck_parents[] = {
-	"core_ck",
-	"sys_ck",
-	"cm_96m_fck",
-	"omap_54m_fck",
-};
-
-static struct ti_clk_mux clkout2_src_mux_ck_data = {
-	.num_parents = ARRAY_SIZE(clkout2_src_mux_ck_parents),
-	.reg = 0xd70,
-	.module = TI_CLKM_CM,
-	.parents = clkout2_src_mux_ck_parents,
-};
-
-static struct ti_clk_composite clkout2_src_ck_data = {
-	.mux = &clkout2_src_mux_ck_data,
-	.gate = &clkout2_src_gate_ck_data,
-};
-
-static struct ti_clk clkout2_src_ck = {
-	.name = "clkout2_src_ck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &clkout2_src_ck_data,
-};
-
-static struct ti_clk_gate i2c1_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 15,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk i2c1_fck = {
-	.name = "i2c1_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &i2c1_fck_data,
-};
-
-static struct ti_clk_gate wdt3_fck_data = {
-	.parent = "per_32k_alwon_fck",
-	.bit_shift = 12,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk wdt3_fck = {
-	.name = "wdt3_fck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &wdt3_fck_data,
-};
-
-static struct ti_clk_gate gpt7_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 8,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static const char *gpt7_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt7_mux_fck_data = {
-	.bit_shift = 5,
-	.num_parents = ARRAY_SIZE(gpt7_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt7_mux_fck_parents,
-};
-
-static struct ti_clk_composite gpt7_fck_data = {
-	.mux = &gpt7_mux_fck_data,
-	.gate = &gpt7_gate_fck_data,
-};
-
-static struct ti_clk gpt7_fck = {
-	.name = "gpt7_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt7_fck_data,
-};
-
-static struct ti_clk_gate usb_l4_gate_ick_data = {
-	.parent = "l4_ick",
-	.bit_shift = 5,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INTERFACE,
-};
-
-static struct ti_clk_divider usb_l4_div_ick_data = {
-	.parent = "l4_ick",
-	.bit_shift = 4,
-	.max_div = 1,
-	.reg = 0xa40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk_composite usb_l4_ick_data = {
-	.gate = &usb_l4_gate_ick_data,
-	.divider = &usb_l4_div_ick_data,
-};
-
-static struct ti_clk usb_l4_ick = {
-	.name = "usb_l4_ick",
-	.type = TI_CLK_COMPOSITE,
-	.data = &usb_l4_ick_data,
-};
-
-static struct ti_clk_gate uart4_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 18,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk uart4_ick = {
-	.name = "uart4_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart4_ick_data,
-};
-
-static struct ti_clk_fixed dummy_ck_data = {
-	.frequency = 0,
-};
-
-static struct ti_clk dummy_ck = {
-	.name = "dummy_ck",
-	.type = TI_CLK_FIXED,
-	.data = &dummy_ck_data,
-};
-
-static const char *gpt3_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt3_mux_fck_data = {
-	.bit_shift = 1,
-	.num_parents = ARRAY_SIZE(gpt3_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt3_mux_fck_parents,
-};
-
-static struct ti_clk_gate gpt9_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 10,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt9_ick = {
-	.name = "gpt9_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt9_ick_data,
-};
-
-static struct ti_clk_gate gpt10_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 11,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_gate dss_ick_3430es1_data = {
-	.parent = "l4_ick",
-	.bit_shift = 0,
-	.reg = 0xe10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_NO_WAIT | CLKF_INTERFACE,
-};
-
-static struct ti_clk dss_ick_3430es1 = {
-	.name = "dss_ick",
-	.clkdm_name = "dss_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &dss_ick_3430es1_data,
-};
-
-static struct ti_clk_gate gpt11_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 12,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt11_ick = {
-	.name = "gpt11_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt11_ick_data,
-};
-
-static struct ti_clk_divider dpll2_fck_data = {
-	.parent = "core_ck",
-	.bit_shift = 19,
-	.max_div = 7,
-	.reg = 0x40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll2_fck = {
-	.name = "dpll2_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll2_fck_data,
-};
-
-static struct ti_clk_gate uart1_fck_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 13,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk uart1_fck = {
-	.name = "uart1_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart1_fck_data,
-};
-
-static struct ti_clk_gate hsotgusb_ick_3430es1_data = {
-	.parent = "core_l3_ick",
-	.bit_shift = 4,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_NO_WAIT | CLKF_INTERFACE,
-};
-
-static struct ti_clk hsotgusb_ick_3430es1 = {
-	.name = "hsotgusb_ick_3430es1",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &hsotgusb_ick_3430es1_data,
-};
-
-static struct ti_clk_gate gpio2_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 13,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpio2_ick = {
-	.name = "gpio2_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio2_ick_data,
-};
-
-static struct ti_clk_gate mmchs1_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 24,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mmchs1_ick = {
-	.name = "mmchs1_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mmchs1_ick_data,
-};
-
-static struct ti_clk_gate modem_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 31,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk modem_fck = {
-	.name = "modem_fck",
-	.clkdm_name = "d2d_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &modem_fck_data,
-};
-
-static struct ti_clk_gate mcbsp4_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 2,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcbsp4_ick = {
-	.name = "mcbsp4_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcbsp4_ick_data,
-};
-
-static struct ti_clk_gate gpio1_ick_data = {
-	.parent = "wkup_l4_ick",
-	.bit_shift = 3,
-	.reg = 0xc10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpio1_ick = {
-	.name = "gpio1_ick",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio1_ick_data,
-};
-
-static const char *gpt6_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt6_mux_fck_data = {
-	.bit_shift = 4,
-	.num_parents = ARRAY_SIZE(gpt6_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt6_mux_fck_parents,
-};
-
-static struct ti_clk_fixed_factor dpll1_x2_ck_data = {
-	.parent = "dpll1_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll1_x2_ck = {
-	.name = "dpll1_x2_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll1_x2_ck_data,
-};
-
-static struct ti_clk_divider dpll1_x2m2_ck_data = {
-	.parent = "dpll1_x2_ck",
-	.max_div = 31,
-	.reg = 0x944,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll1_x2m2_ck = {
-	.name = "dpll1_x2m2_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll1_x2m2_ck_data,
-};
-
-static struct ti_clk_fixed_factor mpu_ck_data = {
-	.parent = "dpll1_x2m2_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk mpu_ck = {
-	.name = "mpu_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &mpu_ck_data,
-};
-
-static struct ti_clk_divider arm_fck_data = {
-	.parent = "mpu_ck",
-	.max_div = 2,
-	.reg = 0x924,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk arm_fck = {
-	.name = "arm_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &arm_fck_data,
-};
-
-static struct ti_clk_fixed_factor core_d3_ck_data = {
-	.parent = "core_ck",
-	.div = 3,
-	.mult = 1,
-};
-
-static struct ti_clk core_d3_ck = {
-	.name = "core_d3_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_d3_ck_data,
-};
-
-static struct ti_clk_gate gpt11_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 12,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-};
-
-static const char *gpt11_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt11_mux_fck_data = {
-	.bit_shift = 7,
-	.num_parents = ARRAY_SIZE(gpt11_mux_fck_parents),
-	.reg = 0xa40,
-	.module = TI_CLKM_CM,
-	.parents = gpt11_mux_fck_parents,
-};
-
-static struct ti_clk_composite gpt11_fck_data = {
-	.mux = &gpt11_mux_fck_data,
-	.gate = &gpt11_gate_fck_data,
-};
-
-static struct ti_clk gpt11_fck = {
-	.name = "gpt11_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt11_fck_data,
-};
-
-static struct ti_clk_fixed_factor core_d6_ck_data = {
-	.parent = "core_ck",
-	.div = 6,
-	.mult = 1,
-};
-
-static struct ti_clk core_d6_ck = {
-	.name = "core_d6_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_d6_ck_data,
-};
-
-static struct ti_clk_gate uart4_fck_am35xx_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 23,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk uart4_fck_am35xx = {
-	.name = "uart4_fck_am35xx",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart4_fck_am35xx_data,
-};
-
-static struct ti_clk_gate dpll3_m3x2_ck_data = {
-	.parent = "dpll3_m3x2_mul_ck",
-	.bit_shift = 0xc,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll3_m3x2_ck = {
-	.name = "dpll3_m3x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll3_m3x2_ck_data,
-};
-
-static struct ti_clk_fixed_factor emu_core_alwon_ck_data = {
-	.parent = "dpll3_m3x2_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk emu_core_alwon_ck = {
-	.name = "emu_core_alwon_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &emu_core_alwon_ck_data,
-};
-
-static struct ti_clk_divider dpll4_m6_ck_data = {
-	.parent = "dpll4_ck",
-	.bit_shift = 24,
-	.max_div = 63,
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll4_m6_ck = {
-	.name = "dpll4_m6_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll4_m6_ck_data,
-};
-
-static struct ti_clk_fixed_factor dpll4_m6x2_mul_ck_data = {
-	.parent = "dpll4_m6_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll4_m6x2_mul_ck = {
-	.name = "dpll4_m6x2_mul_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll4_m6x2_mul_ck_data,
-};
-
-static struct ti_clk_gate dpll4_m6x2_ck_data = {
-	.parent = "dpll4_m6x2_mul_ck",
-	.bit_shift = 0x1f,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m6x2_ck = {
-	.name = "dpll4_m6x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m6x2_ck_data,
-};
-
-static struct ti_clk_fixed_factor emu_per_alwon_ck_data = {
-	.parent = "dpll4_m6x2_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk emu_per_alwon_ck = {
-	.name = "emu_per_alwon_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &emu_per_alwon_ck_data,
-};
-
-static struct ti_clk_fixed_factor emu_mpu_alwon_ck_data = {
-	.parent = "mpu_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk emu_mpu_alwon_ck = {
-	.name = "emu_mpu_alwon_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &emu_mpu_alwon_ck_data,
-};
-
-static const char *emu_src_mux_ck_parents[] = {
-	"sys_ck",
-	"emu_core_alwon_ck",
-	"emu_per_alwon_ck",
-	"emu_mpu_alwon_ck",
-};
-
-static struct ti_clk_mux emu_src_mux_ck_data = {
-	.num_parents = ARRAY_SIZE(emu_src_mux_ck_parents),
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.parents = emu_src_mux_ck_parents,
-};
-
-static struct ti_clk emu_src_mux_ck = {
-	.name = "emu_src_mux_ck",
-	.type = TI_CLK_MUX,
-	.data = &emu_src_mux_ck_data,
-};
-
-static struct ti_clk_gate emu_src_ck_data = {
-	.parent = "emu_src_mux_ck",
-	.flags = CLKF_CLKDM,
-};
-
-static struct ti_clk emu_src_ck = {
-	.name = "emu_src_ck",
-	.clkdm_name = "emu_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &emu_src_ck_data,
-};
-
-static struct ti_clk_divider atclk_fck_data = {
-	.parent = "emu_src_ck",
-	.bit_shift = 4,
-	.max_div = 3,
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk atclk_fck = {
-	.name = "atclk_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &atclk_fck_data,
-};
-
-static struct ti_clk_gate ipss_ick_data = {
-	.parent = "core_l3_ick",
-	.bit_shift = 4,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_AM35XX | CLKF_INTERFACE,
-};
-
-static struct ti_clk ipss_ick = {
-	.name = "ipss_ick",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &ipss_ick_data,
-};
-
-static struct ti_clk_gate emac_ick_data = {
-	.parent = "ipss_ick",
-	.bit_shift = 1,
-	.reg = 0x59c,
-	.module = TI_CLKM_SCRM,
-	.flags = CLKF_AM35XX,
-};
-
-static struct ti_clk emac_ick = {
-	.name = "emac_ick",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &emac_ick_data,
-};
-
-static struct ti_clk_gate vpfe_ick_data = {
-	.parent = "ipss_ick",
-	.bit_shift = 2,
-	.reg = 0x59c,
-	.module = TI_CLKM_SCRM,
-	.flags = CLKF_AM35XX,
-};
-
-static struct ti_clk vpfe_ick = {
-	.name = "vpfe_ick",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &vpfe_ick_data,
-};
-
-static const char *dpll2_ck_parents[] = {
-	"sys_ck",
-	"dpll2_fck",
-};
-
-static struct ti_clk_dpll dpll2_ck_data = {
-	.num_parents = ARRAY_SIZE(dpll2_ck_parents),
-	.control_reg = 0x4,
-	.idlest_reg = 0x24,
-	.mult_div1_reg = 0x40,
-	.autoidle_reg = 0x34,
-	.module = TI_CLKM_CM,
-	.parents = dpll2_ck_parents,
-	.freqsel_mask = 0xf0,
-	.modes = 0xa2,
-	.div1_mask = 0x7f,
-	.idlest_mask = 0x1,
-	.auto_recal_bit = 0x3,
-	.max_divider = 0x80,
-	.min_divider = 0x1,
-	.recal_en_bit = 0x8,
-	.max_multiplier = 0x7ff,
-	.enable_mask = 0x7,
-	.mult_mask = 0x7ff00,
-	.recal_st_bit = 0x8,
-	.autoidle_mask = 0x7,
-};
-
-static struct ti_clk dpll2_ck = {
-	.name = "dpll2_ck",
-	.clkdm_name = "dpll2_clkdm",
-	.type = TI_CLK_DPLL,
-	.data = &dpll2_ck_data,
-};
-
-static struct ti_clk_divider dpll2_m2_ck_data = {
-	.parent = "dpll2_ck",
-	.max_div = 31,
-	.reg = 0x44,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk dpll2_m2_ck = {
-	.name = "dpll2_m2_ck",
-	.type = TI_CLK_DIVIDER,
-	.data = &dpll2_m2_ck_data,
-};
-
-static const char *mcbsp4_mux_fck_parents[] = {
-	"per_96m_fck",
-	"mcbsp_clks",
-};
-
-static struct ti_clk_mux mcbsp4_mux_fck_data = {
-	.bit_shift = 2,
-	.num_parents = ARRAY_SIZE(mcbsp4_mux_fck_parents),
-	.reg = 0x2d8,
-	.module = TI_CLKM_SCRM,
-	.parents = mcbsp4_mux_fck_parents,
-};
-
-static const char *mcbsp1_mux_fck_parents[] = {
-	"core_96m_fck",
-	"mcbsp_clks",
-};
-
-static struct ti_clk_mux mcbsp1_mux_fck_data = {
-	.bit_shift = 2,
-	.num_parents = ARRAY_SIZE(mcbsp1_mux_fck_parents),
-	.reg = 0x274,
-	.module = TI_CLKM_SCRM,
-	.parents = mcbsp1_mux_fck_parents,
-};
-
-static struct ti_clk_gate gpt8_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 9,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_gate gpt8_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 9,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt8_ick = {
-	.name = "gpt8_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt8_ick_data,
-};
-
-static const char *gpt10_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt10_mux_fck_data = {
-	.bit_shift = 6,
-	.num_parents = ARRAY_SIZE(gpt10_mux_fck_parents),
-	.reg = 0xa40,
-	.module = TI_CLKM_CM,
-	.parents = gpt10_mux_fck_parents,
-};
-
-static struct ti_clk_gate mmchs3_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 30,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mmchs3_ick = {
-	.name = "mmchs3_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mmchs3_ick_data,
-};
-
-static struct ti_clk_gate gpio3_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 14,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpio3_ick = {
-	.name = "gpio3_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio3_ick_data,
-};
-
-static const char *traceclk_src_fck_parents[] = {
-	"sys_ck",
-	"emu_core_alwon_ck",
-	"emu_per_alwon_ck",
-	"emu_mpu_alwon_ck",
-};
-
-static struct ti_clk_mux traceclk_src_fck_data = {
-	.bit_shift = 2,
-	.num_parents = ARRAY_SIZE(traceclk_src_fck_parents),
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.parents = traceclk_src_fck_parents,
-};
-
-static struct ti_clk traceclk_src_fck = {
-	.name = "traceclk_src_fck",
-	.type = TI_CLK_MUX,
-	.data = &traceclk_src_fck_data,
-};
-
-static struct ti_clk_divider traceclk_fck_data = {
-	.parent = "traceclk_src_fck",
-	.bit_shift = 11,
-	.max_div = 7,
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk traceclk_fck = {
-	.name = "traceclk_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &traceclk_fck_data,
-};
-
-static struct ti_clk_gate mcbsp5_gate_fck_data = {
-	.parent = "mcbsp_clks",
-	.bit_shift = 10,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_gate sad2d_ick_data = {
-	.parent = "l3_ick",
-	.bit_shift = 3,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk sad2d_ick = {
-	.name = "sad2d_ick",
-	.clkdm_name = "d2d_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &sad2d_ick_data,
-};
-
-static const char *gpt1_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt1_mux_fck_data = {
-	.num_parents = ARRAY_SIZE(gpt1_mux_fck_parents),
-	.reg = 0xc40,
-	.module = TI_CLKM_CM,
-	.parents = gpt1_mux_fck_parents,
-};
-
-static struct ti_clk_gate hecc_ck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 3,
-	.reg = 0x59c,
-	.module = TI_CLKM_SCRM,
-	.flags = CLKF_AM35XX,
-};
-
-static struct ti_clk hecc_ck = {
-	.name = "hecc_ck",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &hecc_ck_data,
-};
-
-static struct ti_clk_gate gpt1_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 0,
-	.reg = 0xc00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_composite gpt1_fck_data = {
-	.mux = &gpt1_mux_fck_data,
-	.gate = &gpt1_gate_fck_data,
-};
-
-static struct ti_clk gpt1_fck = {
-	.name = "gpt1_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt1_fck_data,
-};
-
-static struct ti_clk_gate dpll4_m2x2_ck_omap36xx_data = {
-	.parent = "dpll4_m2x2_mul_ck",
-	.bit_shift = 0x1b,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_HSDIV | CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m2x2_ck_omap36xx = {
-	.name = "dpll4_m2x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m2x2_ck_omap36xx_data,
-	.patch = &dpll4_m2x2_ck,
-};
-
-static struct ti_clk_divider gfx_l3_fck_data = {
-	.parent = "l3_ick",
-	.max_div = 7,
-	.reg = 0xb40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk gfx_l3_fck = {
-	.name = "gfx_l3_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &gfx_l3_fck_data,
-};
-
-static struct ti_clk_gate gfx_cg1_ck_data = {
-	.parent = "gfx_l3_fck",
-	.bit_shift = 1,
-	.reg = 0xb00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk gfx_cg1_ck = {
-	.name = "gfx_cg1_ck",
-	.clkdm_name = "gfx_3430es1_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gfx_cg1_ck_data,
-};
-
-static struct ti_clk_gate mailboxes_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 7,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mailboxes_ick = {
-	.name = "mailboxes_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mailboxes_ick_data,
-};
-
-static struct ti_clk_gate sha11_ick_data = {
-	.parent = "security_l4_ick2",
-	.bit_shift = 1,
-	.reg = 0xa14,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk sha11_ick = {
-	.name = "sha11_ick",
-	.type = TI_CLK_GATE,
-	.data = &sha11_ick_data,
-};
-
-static struct ti_clk_gate hsotgusb_ick_am35xx_data = {
-	.parent = "ipss_ick",
-	.bit_shift = 0,
-	.reg = 0x59c,
-	.module = TI_CLKM_SCRM,
-	.flags = CLKF_AM35XX,
-};
-
-static struct ti_clk hsotgusb_ick_am35xx = {
-	.name = "hsotgusb_ick_am35xx",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &hsotgusb_ick_am35xx_data,
-};
-
-static struct ti_clk_gate mmchs3_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 30,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mmchs3_fck = {
-	.name = "mmchs3_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mmchs3_fck_data,
-};
-
-static struct ti_clk_divider pclk_fck_data = {
-	.parent = "emu_src_ck",
-	.bit_shift = 8,
-	.max_div = 7,
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk pclk_fck = {
-	.name = "pclk_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &pclk_fck_data,
-};
-
-static const char *dpll4_ck_omap36xx_parents[] = {
-	"sys_ck",
-	"sys_ck",
-};
-
-static struct ti_clk_dpll dpll4_ck_omap36xx_data = {
-	.num_parents = ARRAY_SIZE(dpll4_ck_omap36xx_parents),
-	.control_reg = 0xd00,
-	.idlest_reg = 0xd20,
-	.mult_div1_reg = 0xd44,
-	.autoidle_reg = 0xd30,
-	.module = TI_CLKM_CM,
-	.parents = dpll4_ck_omap36xx_parents,
-	.modes = 0x82,
-	.div1_mask = 0x7f,
-	.idlest_mask = 0x2,
-	.auto_recal_bit = 0x13,
-	.max_divider = 0x80,
-	.min_divider = 0x1,
-	.recal_en_bit = 0x6,
-	.max_multiplier = 0xfff,
-	.enable_mask = 0x70000,
-	.mult_mask = 0xfff00,
-	.recal_st_bit = 0x6,
-	.autoidle_mask = 0x38,
-	.sddiv_mask = 0xff000000,
-	.dco_mask = 0xe00000,
-	.flags = CLKF_PER | CLKF_J_TYPE,
-};
-
-static struct ti_clk dpll4_ck_omap36xx = {
-	.name = "dpll4_ck",
-	.type = TI_CLK_DPLL,
-	.data = &dpll4_ck_omap36xx_data,
-	.patch = &dpll4_ck,
-};
-
-static struct ti_clk_gate uart3_fck_data = {
-	.parent = "per_48m_fck",
-	.bit_shift = 11,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk uart3_fck = {
-	.name = "uart3_fck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart3_fck_data,
-};
-
-static struct ti_clk_fixed_factor wkup_32k_fck_data = {
-	.parent = "omap_32k_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk wkup_32k_fck = {
-	.name = "wkup_32k_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &wkup_32k_fck_data,
-};
-
-static struct ti_clk_gate sys_clkout1_data = {
-	.parent = "osc_sys_ck",
-	.bit_shift = 7,
-	.reg = 0xd70,
-	.module = TI_CLKM_PRM,
-};
-
-static struct ti_clk sys_clkout1 = {
-	.name = "sys_clkout1",
-	.type = TI_CLK_GATE,
-	.data = &sys_clkout1_data,
-};
-
-static struct ti_clk_fixed_factor gpmc_fck_data = {
-	.parent = "core_l3_ick",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk gpmc_fck = {
-	.name = "gpmc_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &gpmc_fck_data,
-};
-
-static struct ti_clk_fixed_factor dpll5_m2_d20_ck_data = {
-	.parent = "dpll5_m2_ck",
-	.div = 20,
-	.mult = 1,
-};
-
-static struct ti_clk dpll5_m2_d20_ck = {
-	.name = "dpll5_m2_d20_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll5_m2_d20_ck_data,
-};
-
-static struct ti_clk_gate dpll4_m5x2_ck_omap36xx_data = {
-	.parent = "dpll4_m5x2_mul_ck",
-	.bit_shift = 0x1e,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_HSDIV | CLKF_SET_RATE_PARENT | CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m5x2_ck_omap36xx = {
-	.name = "dpll4_m5x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m5x2_ck_omap36xx_data,
-	.patch = &dpll4_m5x2_ck,
-};
-
-static struct ti_clk_gate ssi_ssr_gate_fck_3430es2_data = {
-	.parent = "corex2_fck",
-	.bit_shift = 0,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_NO_WAIT,
-};
-
-static struct ti_clk_gate uart1_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 13,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk uart1_ick = {
-	.name = "uart1_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart1_ick_data,
-};
-
-static struct ti_clk_gate iva2_ck_data = {
-	.parent = "dpll2_m2_ck",
-	.bit_shift = 0,
-	.reg = 0x0,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk iva2_ck = {
-	.name = "iva2_ck",
-	.clkdm_name = "iva2_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &iva2_ck_data,
-};
-
-static struct ti_clk_gate pka_ick_data = {
-	.parent = "security_l3_ick",
-	.bit_shift = 4,
-	.reg = 0xa14,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk pka_ick = {
-	.name = "pka_ick",
-	.type = TI_CLK_GATE,
-	.data = &pka_ick_data,
-};
-
-static struct ti_clk_gate gpt12_ick_data = {
-	.parent = "wkup_l4_ick",
-	.bit_shift = 1,
-	.reg = 0xc10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt12_ick = {
-	.name = "gpt12_ick",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt12_ick_data,
-};
-
-static const char *mcbsp5_mux_fck_parents[] = {
-	"core_96m_fck",
-	"mcbsp_clks",
-};
-
-static struct ti_clk_mux mcbsp5_mux_fck_data = {
-	.bit_shift = 4,
-	.num_parents = ARRAY_SIZE(mcbsp5_mux_fck_parents),
-	.reg = 0x2d8,
-	.module = TI_CLKM_SCRM,
-	.parents = mcbsp5_mux_fck_parents,
-};
-
-static struct ti_clk_composite mcbsp5_fck_data = {
-	.mux = &mcbsp5_mux_fck_data,
-	.gate = &mcbsp5_gate_fck_data,
-};
-
-static struct ti_clk mcbsp5_fck = {
-	.name = "mcbsp5_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &mcbsp5_fck_data,
-};
-
-static struct ti_clk_gate usbhost_48m_fck_data = {
-	.parent = "omap_48m_fck",
-	.bit_shift = 0,
-	.reg = 0x1400,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_DSS,
-};
-
-static struct ti_clk usbhost_48m_fck = {
-	.name = "usbhost_48m_fck",
-	.clkdm_name = "usbhost_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &usbhost_48m_fck_data,
-};
-
-static struct ti_clk_gate des1_ick_data = {
-	.parent = "security_l4_ick2",
-	.bit_shift = 0,
-	.reg = 0xa14,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk des1_ick = {
-	.name = "des1_ick",
-	.type = TI_CLK_GATE,
-	.data = &des1_ick_data,
-};
-
-static struct ti_clk_gate sgx_gate_fck_data = {
-	.parent = "core_ck",
-	.bit_shift = 1,
-	.reg = 0xb00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_fixed_factor core_d4_ck_data = {
-	.parent = "core_ck",
-	.div = 4,
-	.mult = 1,
-};
-
-static struct ti_clk core_d4_ck = {
-	.name = "core_d4_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_d4_ck_data,
-};
-
-static struct ti_clk_fixed_factor omap_192m_alwon_fck_data = {
-	.parent = "dpll4_m2x2_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk omap_192m_alwon_fck = {
-	.name = "omap_192m_alwon_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &omap_192m_alwon_fck_data,
-};
-
-static struct ti_clk_fixed_factor core_d2_ck_data = {
-	.parent = "core_ck",
-	.div = 2,
-	.mult = 1,
-};
-
-static struct ti_clk core_d2_ck = {
-	.name = "core_d2_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_d2_ck_data,
-};
-
-static struct ti_clk_fixed_factor corex2_d3_fck_data = {
-	.parent = "corex2_fck",
-	.div = 3,
-	.mult = 1,
-};
-
-static struct ti_clk corex2_d3_fck = {
-	.name = "corex2_d3_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &corex2_d3_fck_data,
-};
-
-static struct ti_clk_fixed_factor corex2_d5_fck_data = {
-	.parent = "corex2_fck",
-	.div = 5,
-	.mult = 1,
-};
-
-static struct ti_clk corex2_d5_fck = {
-	.name = "corex2_d5_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &corex2_d5_fck_data,
-};
-
-static const char *sgx_mux_fck_parents[] = {
-	"core_d3_ck",
-	"core_d4_ck",
-	"core_d6_ck",
-	"cm_96m_fck",
-	"omap_192m_alwon_fck",
-	"core_d2_ck",
-	"corex2_d3_fck",
-	"corex2_d5_fck",
-};
-
-static struct ti_clk_mux sgx_mux_fck_data = {
-	.num_parents = ARRAY_SIZE(sgx_mux_fck_parents),
-	.reg = 0xb40,
-	.module = TI_CLKM_CM,
-	.parents = sgx_mux_fck_parents,
-};
-
-static struct ti_clk_composite sgx_fck_data = {
-	.mux = &sgx_mux_fck_data,
-	.gate = &sgx_gate_fck_data,
-};
-
-static struct ti_clk sgx_fck = {
-	.name = "sgx_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &sgx_fck_data,
-};
-
-static struct ti_clk_gate mcspi1_fck_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 18,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mcspi1_fck = {
-	.name = "mcspi1_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi1_fck_data,
-};
-
-static struct ti_clk_gate mmchs2_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 25,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mmchs2_fck = {
-	.name = "mmchs2_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mmchs2_fck_data,
-};
-
-static struct ti_clk_gate mcspi2_fck_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 19,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mcspi2_fck = {
-	.name = "mcspi2_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi2_fck_data,
-};
-
-static struct ti_clk_gate vpfe_fck_data = {
-	.parent = "pclk_ck",
-	.bit_shift = 10,
-	.reg = 0x59c,
-	.module = TI_CLKM_SCRM,
-};
-
-static struct ti_clk vpfe_fck = {
-	.name = "vpfe_fck",
-	.type = TI_CLK_GATE,
-	.data = &vpfe_fck_data,
-};
-
-static struct ti_clk_gate gpt4_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 5,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_gate mcbsp1_gate_fck_data = {
-	.parent = "mcbsp_clks",
-	.bit_shift = 9,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_gate gpt5_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 6,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static const char *gpt5_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt5_mux_fck_data = {
-	.bit_shift = 3,
-	.num_parents = ARRAY_SIZE(gpt5_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt5_mux_fck_parents,
-};
-
-static struct ti_clk_composite gpt5_fck_data = {
-	.mux = &gpt5_mux_fck_data,
-	.gate = &gpt5_gate_fck_data,
-};
-
-static struct ti_clk gpt5_fck = {
-	.name = "gpt5_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt5_fck_data,
-};
-
-static struct ti_clk_gate ts_fck_data = {
-	.parent = "omap_32k_fck",
-	.bit_shift = 1,
-	.reg = 0xa08,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk ts_fck = {
-	.name = "ts_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &ts_fck_data,
-};
-
-static struct ti_clk_fixed_factor wdt1_fck_data = {
-	.parent = "secure_32k_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk wdt1_fck = {
-	.name = "wdt1_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &wdt1_fck_data,
-};
-
-static struct ti_clk_gate dpll4_m6x2_ck_omap36xx_data = {
-	.parent = "dpll4_m6x2_mul_ck",
-	.bit_shift = 0x1f,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_HSDIV | CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m6x2_ck_omap36xx = {
-	.name = "dpll4_m6x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m6x2_ck_omap36xx_data,
-	.patch = &dpll4_m6x2_ck,
-};
-
-static const char *gpt4_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt4_mux_fck_data = {
-	.bit_shift = 2,
-	.num_parents = ARRAY_SIZE(gpt4_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt4_mux_fck_parents,
-};
-
-static struct ti_clk_gate usbhost_ick_data = {
-	.parent = "l4_ick",
-	.bit_shift = 0,
-	.reg = 0x1410,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_DSS | CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk usbhost_ick = {
-	.name = "usbhost_ick",
-	.clkdm_name = "usbhost_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &usbhost_ick_data,
-};
-
-static struct ti_clk_gate mcbsp2_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 0,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcbsp2_ick = {
-	.name = "mcbsp2_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcbsp2_ick_data,
-};
-
-static struct ti_clk_gate omapctrl_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 6,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk omapctrl_ick = {
-	.name = "omapctrl_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &omapctrl_ick_data,
-};
-
-static struct ti_clk_fixed_factor omap_96m_d4_fck_data = {
-	.parent = "omap_96m_fck",
-	.div = 4,
-	.mult = 1,
-};
-
-static struct ti_clk omap_96m_d4_fck = {
-	.name = "omap_96m_d4_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &omap_96m_d4_fck_data,
-};
-
-static struct ti_clk_gate gpt6_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 7,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt6_ick = {
-	.name = "gpt6_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt6_ick_data,
-};
-
-static struct ti_clk_gate dpll3_m3x2_ck_omap36xx_data = {
-	.parent = "dpll3_m3x2_mul_ck",
-	.bit_shift = 0xc,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_HSDIV | CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll3_m3x2_ck_omap36xx = {
-	.name = "dpll3_m3x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll3_m3x2_ck_omap36xx_data,
-	.patch = &dpll3_m3x2_ck,
-};
-
-static struct ti_clk_gate i2c3_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 17,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk i2c3_ick = {
-	.name = "i2c3_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &i2c3_ick_data,
-};
-
-static struct ti_clk_gate gpio6_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 17,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpio6_ick = {
-	.name = "gpio6_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio6_ick_data,
-};
-
-static struct ti_clk_gate mspro_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 23,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mspro_ick = {
-	.name = "mspro_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mspro_ick_data,
-};
-
-static struct ti_clk_composite mcbsp1_fck_data = {
-	.mux = &mcbsp1_mux_fck_data,
-	.gate = &mcbsp1_gate_fck_data,
-};
-
-static struct ti_clk mcbsp1_fck = {
-	.name = "mcbsp1_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &mcbsp1_fck_data,
-};
-
-static struct ti_clk_gate gpt3_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 4,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_fixed rmii_ck_data = {
-	.frequency = 50000000,
-};
-
-static struct ti_clk rmii_ck = {
-	.name = "rmii_ck",
-	.type = TI_CLK_FIXED,
-	.data = &rmii_ck_data,
-};
-
-static struct ti_clk_gate gpt6_gate_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 7,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_composite gpt6_fck_data = {
-	.mux = &gpt6_mux_fck_data,
-	.gate = &gpt6_gate_fck_data,
-};
-
-static struct ti_clk gpt6_fck = {
-	.name = "gpt6_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt6_fck_data,
-};
-
-static struct ti_clk_fixed_factor dpll5_m2_d4_ck_data = {
-	.parent = "dpll5_m2_ck",
-	.div = 4,
-	.mult = 1,
-};
-
-static struct ti_clk dpll5_m2_d4_ck = {
-	.name = "dpll5_m2_d4_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll5_m2_d4_ck_data,
-};
-
-static struct ti_clk_fixed_factor sys_d2_ck_data = {
-	.parent = "sys_ck",
-	.div = 2,
-	.mult = 1,
-};
-
-static struct ti_clk sys_d2_ck = {
-	.name = "sys_d2_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &sys_d2_ck_data,
-};
-
-static struct ti_clk_fixed_factor omap_96m_d2_fck_data = {
-	.parent = "omap_96m_fck",
-	.div = 2,
-	.mult = 1,
-};
-
-static struct ti_clk omap_96m_d2_fck = {
-	.name = "omap_96m_d2_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &omap_96m_d2_fck_data,
-};
-
-static struct ti_clk_fixed_factor dpll5_m2_d8_ck_data = {
-	.parent = "dpll5_m2_ck",
-	.div = 8,
-	.mult = 1,
-};
-
-static struct ti_clk dpll5_m2_d8_ck = {
-	.name = "dpll5_m2_d8_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll5_m2_d8_ck_data,
-};
-
-static struct ti_clk_fixed_factor dpll5_m2_d16_ck_data = {
-	.parent = "dpll5_m2_ck",
-	.div = 16,
-	.mult = 1,
-};
-
-static struct ti_clk dpll5_m2_d16_ck = {
-	.name = "dpll5_m2_d16_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll5_m2_d16_ck_data,
-};
-
-static const char *usim_mux_fck_parents[] = {
-	"sys_ck",
-	"sys_d2_ck",
-	"omap_96m_d2_fck",
-	"omap_96m_d4_fck",
-	"omap_96m_d8_fck",
-	"omap_96m_d10_fck",
-	"dpll5_m2_d4_ck",
-	"dpll5_m2_d8_ck",
-	"dpll5_m2_d16_ck",
-	"dpll5_m2_d20_ck",
-};
-
-static struct ti_clk_mux usim_mux_fck_data = {
-	.bit_shift = 3,
-	.num_parents = ARRAY_SIZE(usim_mux_fck_parents),
-	.reg = 0xc40,
-	.module = TI_CLKM_CM,
-	.parents = usim_mux_fck_parents,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk_composite usim_fck_data = {
-	.mux = &usim_mux_fck_data,
-	.gate = &usim_gate_fck_data,
-};
-
-static struct ti_clk usim_fck = {
-	.name = "usim_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &usim_fck_data,
-};
-
-static int ssi_ssr_div_fck_3430es2_divs[] = {
-	0,
-	1,
-	2,
-	3,
-	4,
-	0,
-	6,
-	0,
-	8,
-};
-
-static struct ti_clk_divider ssi_ssr_div_fck_3430es2_data = {
-	.num_dividers = ARRAY_SIZE(ssi_ssr_div_fck_3430es2_divs),
-	.parent = "corex2_fck",
-	.bit_shift = 8,
-	.dividers = ssi_ssr_div_fck_3430es2_divs,
-	.reg = 0xa40,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_composite ssi_ssr_fck_3430es2_data = {
-	.gate = &ssi_ssr_gate_fck_3430es2_data,
-	.divider = &ssi_ssr_div_fck_3430es2_data,
-};
-
-static struct ti_clk ssi_ssr_fck_3430es2 = {
-	.name = "ssi_ssr_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &ssi_ssr_fck_3430es2_data,
-};
-
-static struct ti_clk_gate dss1_alwon_fck_3430es1_data = {
-	.parent = "dpll4_m4x2_ck",
-	.bit_shift = 0,
-	.reg = 0xe00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SET_RATE_PARENT,
-};
-
-static struct ti_clk dss1_alwon_fck_3430es1 = {
-	.name = "dss1_alwon_fck",
-	.clkdm_name = "dss_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &dss1_alwon_fck_3430es1_data,
-};
-
-static struct ti_clk_gate gpt3_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 4,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt3_ick = {
-	.name = "gpt3_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt3_ick_data,
-};
-
-static struct ti_clk_fixed_factor omap_12m_fck_data = {
-	.parent = "omap_48m_fck",
-	.div = 4,
-	.mult = 1,
-};
-
-static struct ti_clk omap_12m_fck = {
-	.name = "omap_12m_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &omap_12m_fck_data,
-};
-
-static struct ti_clk_fixed_factor core_12m_fck_data = {
-	.parent = "omap_12m_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk core_12m_fck = {
-	.name = "core_12m_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &core_12m_fck_data,
-};
-
-static struct ti_clk_gate hdq_fck_data = {
-	.parent = "core_12m_fck",
-	.bit_shift = 22,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk hdq_fck = {
-	.name = "hdq_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &hdq_fck_data,
-};
-
-static struct ti_clk_gate usbtll_fck_data = {
-	.parent = "dpll5_m2_ck",
-	.bit_shift = 2,
-	.reg = 0xa08,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk usbtll_fck = {
-	.name = "usbtll_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &usbtll_fck_data,
-};
-
-static struct ti_clk_gate hsotgusb_fck_am35xx_data = {
-	.parent = "sys_ck",
-	.bit_shift = 8,
-	.reg = 0x59c,
-	.module = TI_CLKM_SCRM,
-};
-
-static struct ti_clk hsotgusb_fck_am35xx = {
-	.name = "hsotgusb_fck_am35xx",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &hsotgusb_fck_am35xx_data,
-};
-
-static struct ti_clk_gate hsotgusb_ick_3430es2_data = {
-	.parent = "core_l3_ick",
-	.bit_shift = 4,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_HSOTGUSB | CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk hsotgusb_ick_3430es2 = {
-	.name = "hsotgusb_ick_3430es2",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &hsotgusb_ick_3430es2_data,
-};
-
-static struct ti_clk_gate gfx_l3_ck_data = {
-	.parent = "l3_ick",
-	.bit_shift = 0,
-	.reg = 0xb10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk gfx_l3_ck = {
-	.name = "gfx_l3_ck",
-	.clkdm_name = "gfx_3430es1_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gfx_l3_ck_data,
-};
-
-static struct ti_clk_fixed_factor gfx_l3_ick_data = {
-	.parent = "gfx_l3_ck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk gfx_l3_ick = {
-	.name = "gfx_l3_ick",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &gfx_l3_ick_data,
-};
-
-static struct ti_clk_gate mcbsp1_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 9,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcbsp1_ick = {
-	.name = "mcbsp1_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcbsp1_ick_data,
-};
-
-static struct ti_clk_fixed_factor gpt12_fck_data = {
-	.parent = "secure_32k_fck",
-	.div = 1,
-	.mult = 1,
-};
-
-static struct ti_clk gpt12_fck = {
-	.name = "gpt12_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &gpt12_fck_data,
-};
-
-static struct ti_clk_gate gfx_cg2_ck_data = {
-	.parent = "gfx_l3_fck",
-	.bit_shift = 2,
-	.reg = 0xb00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk gfx_cg2_ck = {
-	.name = "gfx_cg2_ck",
-	.clkdm_name = "gfx_3430es1_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gfx_cg2_ck_data,
-};
-
-static struct ti_clk_gate i2c2_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 16,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk i2c2_ick = {
-	.name = "i2c2_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &i2c2_ick_data,
-};
-
-static struct ti_clk_gate gpio4_dbck_data = {
-	.parent = "per_32k_alwon_fck",
-	.bit_shift = 15,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk gpio4_dbck = {
-	.name = "gpio4_dbck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio4_dbck_data,
-};
-
-static struct ti_clk_gate i2c3_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 17,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk i2c3_fck = {
-	.name = "i2c3_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &i2c3_fck_data,
-};
-
-static struct ti_clk_composite gpt3_fck_data = {
-	.mux = &gpt3_mux_fck_data,
-	.gate = &gpt3_gate_fck_data,
-};
-
-static struct ti_clk gpt3_fck = {
-	.name = "gpt3_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt3_fck_data,
-};
-
-static struct ti_clk_gate i2c1_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 15,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk i2c1_ick = {
-	.name = "i2c1_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &i2c1_ick_data,
-};
-
-static struct ti_clk_gate omap_32ksync_ick_data = {
-	.parent = "wkup_l4_ick",
-	.bit_shift = 2,
-	.reg = 0xc10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk omap_32ksync_ick = {
-	.name = "omap_32ksync_ick",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &omap_32ksync_ick_data,
-};
-
-static struct ti_clk_gate aes2_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 28,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk aes2_ick = {
-	.name = "aes2_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &aes2_ick_data,
-};
-
-static const char *gpt8_mux_fck_parents[] = {
-	"omap_32k_fck",
-	"sys_ck",
-};
-
-static struct ti_clk_mux gpt8_mux_fck_data = {
-	.bit_shift = 6,
-	.num_parents = ARRAY_SIZE(gpt8_mux_fck_parents),
-	.reg = 0x1040,
-	.module = TI_CLKM_CM,
-	.parents = gpt8_mux_fck_parents,
-};
-
-static struct ti_clk_composite gpt8_fck_data = {
-	.mux = &gpt8_mux_fck_data,
-	.gate = &gpt8_gate_fck_data,
-};
-
-static struct ti_clk gpt8_fck = {
-	.name = "gpt8_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt8_fck_data,
-};
-
-static struct ti_clk_gate mcbsp4_gate_fck_data = {
-	.parent = "mcbsp_clks",
-	.bit_shift = 2,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk_composite mcbsp4_fck_data = {
-	.mux = &mcbsp4_mux_fck_data,
-	.gate = &mcbsp4_gate_fck_data,
-};
-
-static struct ti_clk mcbsp4_fck = {
-	.name = "mcbsp4_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &mcbsp4_fck_data,
-};
-
-static struct ti_clk_gate gpio2_dbck_data = {
-	.parent = "per_32k_alwon_fck",
-	.bit_shift = 13,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk gpio2_dbck = {
-	.name = "gpio2_dbck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio2_dbck_data,
-};
-
-static struct ti_clk_gate usbtll_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 2,
-	.reg = 0xa18,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk usbtll_ick = {
-	.name = "usbtll_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &usbtll_ick_data,
-};
-
-static struct ti_clk_gate mcspi4_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 21,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcspi4_ick = {
-	.name = "mcspi4_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi4_ick_data,
-};
-
-static struct ti_clk_gate dss_96m_fck_data = {
-	.parent = "omap_96m_fck",
-	.bit_shift = 2,
-	.reg = 0xe00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk dss_96m_fck = {
-	.name = "dss_96m_fck",
-	.clkdm_name = "dss_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &dss_96m_fck_data,
-};
-
-static struct ti_clk_divider rm_ick_data = {
-	.parent = "l4_ick",
-	.bit_shift = 1,
-	.max_div = 3,
-	.reg = 0xc40,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk rm_ick = {
-	.name = "rm_ick",
-	.type = TI_CLK_DIVIDER,
-	.data = &rm_ick_data,
-};
-
-static struct ti_clk_gate hdq_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 22,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk hdq_ick = {
-	.name = "hdq_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &hdq_ick_data,
-};
-
-static struct ti_clk_fixed_factor dpll3_x2_ck_data = {
-	.parent = "dpll3_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll3_x2_ck = {
-	.name = "dpll3_x2_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll3_x2_ck_data,
-};
-
-static struct ti_clk_gate mad2d_ick_data = {
-	.parent = "l3_ick",
-	.bit_shift = 3,
-	.reg = 0xa18,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mad2d_ick = {
-	.name = "mad2d_ick",
-	.clkdm_name = "d2d_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mad2d_ick_data,
-};
-
-static struct ti_clk_gate fshostusb_fck_data = {
-	.parent = "core_48m_fck",
-	.bit_shift = 5,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk fshostusb_fck = {
-	.name = "fshostusb_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &fshostusb_fck_data,
-};
-
-static struct ti_clk_gate sr1_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 6,
-	.reg = 0xc00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk sr1_fck = {
-	.name = "sr1_fck",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &sr1_fck_data,
-};
-
-static struct ti_clk_gate des2_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 26,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk des2_ick = {
-	.name = "des2_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &des2_ick_data,
-};
-
-static struct ti_clk_gate sdrc_ick_data = {
-	.parent = "core_l3_ick",
-	.bit_shift = 1,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk sdrc_ick = {
-	.name = "sdrc_ick",
-	.clkdm_name = "core_l3_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &sdrc_ick_data,
-};
-
-static struct ti_clk_composite gpt4_fck_data = {
-	.mux = &gpt4_mux_fck_data,
-	.gate = &gpt4_gate_fck_data,
-};
-
-static struct ti_clk gpt4_fck = {
-	.name = "gpt4_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt4_fck_data,
-};
-
-static struct ti_clk_gate dpll4_m3x2_ck_omap36xx_data = {
-	.parent = "dpll4_m3x2_mul_ck",
-	.bit_shift = 0x1c,
-	.reg = 0xd00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_HSDIV | CLKF_SET_BIT_TO_DISABLE,
-};
-
-static struct ti_clk dpll4_m3x2_ck_omap36xx = {
-	.name = "dpll4_m3x2_ck",
-	.type = TI_CLK_GATE,
-	.data = &dpll4_m3x2_ck_omap36xx_data,
-	.patch = &dpll4_m3x2_ck,
-};
-
-static struct ti_clk_gate cpefuse_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 0,
-	.reg = 0xa08,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk cpefuse_fck = {
-	.name = "cpefuse_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &cpefuse_fck_data,
-};
-
-static struct ti_clk_gate mcspi3_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 20,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcspi3_ick = {
-	.name = "mcspi3_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi3_ick_data,
-};
-
-static struct ti_clk_fixed_factor ssi_sst_fck_3430es2_data = {
-	.parent = "ssi_ssr_fck",
-	.div = 2,
-	.mult = 1,
-};
-
-static struct ti_clk ssi_sst_fck_3430es2 = {
-	.name = "ssi_sst_fck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &ssi_sst_fck_3430es2_data,
-};
-
-static struct ti_clk_gate gpio1_dbck_data = {
-	.parent = "wkup_32k_fck",
-	.bit_shift = 3,
-	.reg = 0xc00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk gpio1_dbck = {
-	.name = "gpio1_dbck",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio1_dbck_data,
-};
-
-static struct ti_clk_gate gpt4_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 5,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt4_ick = {
-	.name = "gpt4_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt4_ick_data,
-};
-
-static struct ti_clk_gate gpt2_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 3,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt2_ick = {
-	.name = "gpt2_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt2_ick_data,
-};
-
-static struct ti_clk_gate mmchs1_fck_data = {
-	.parent = "core_96m_fck",
-	.bit_shift = 24,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk mmchs1_fck = {
-	.name = "mmchs1_fck",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mmchs1_fck_data,
-};
-
-static struct ti_clk_fixed dummy_apb_pclk_data = {
-	.frequency = 0x0,
-};
-
-static struct ti_clk dummy_apb_pclk = {
-	.name = "dummy_apb_pclk",
-	.type = TI_CLK_FIXED,
-	.data = &dummy_apb_pclk_data,
-};
-
-static struct ti_clk_gate gpio6_dbck_data = {
-	.parent = "per_32k_alwon_fck",
-	.bit_shift = 17,
-	.reg = 0x1000,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk gpio6_dbck = {
-	.name = "gpio6_dbck",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio6_dbck_data,
-};
-
-static struct ti_clk_gate uart2_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 14,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk uart2_ick = {
-	.name = "uart2_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &uart2_ick_data,
-};
-
-static struct ti_clk_fixed_factor dpll4_x2_ck_data = {
-	.parent = "dpll4_ck",
-	.div = 1,
-	.mult = 2,
-};
-
-static struct ti_clk dpll4_x2_ck = {
-	.name = "dpll4_x2_ck",
-	.type = TI_CLK_FIXED_FACTOR,
-	.data = &dpll4_x2_ck_data,
-};
-
-static struct ti_clk_gate gpt7_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 8,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpt7_ick = {
-	.name = "gpt7_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpt7_ick_data,
-};
-
-static struct ti_clk_gate dss_tv_fck_data = {
-	.parent = "omap_54m_fck",
-	.bit_shift = 2,
-	.reg = 0xe00,
-	.module = TI_CLKM_CM,
-};
-
-static struct ti_clk dss_tv_fck = {
-	.name = "dss_tv_fck",
-	.clkdm_name = "dss_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &dss_tv_fck_data,
-};
-
-static struct ti_clk_gate mcbsp5_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 10,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcbsp5_ick = {
-	.name = "mcbsp5_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcbsp5_ick_data,
-};
-
-static struct ti_clk_gate mcspi1_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 18,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk mcspi1_ick = {
-	.name = "mcspi1_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &mcspi1_ick_data,
-};
-
-static struct ti_clk_gate d2d_26m_fck_data = {
-	.parent = "sys_ck",
-	.bit_shift = 3,
-	.reg = 0xa00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk d2d_26m_fck = {
-	.name = "d2d_26m_fck",
-	.clkdm_name = "d2d_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &d2d_26m_fck_data,
-};
-
-static struct ti_clk_gate wdt3_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 12,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk wdt3_ick = {
-	.name = "wdt3_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &wdt3_ick_data,
-};
-
-static struct ti_clk_divider pclkx2_fck_data = {
-	.parent = "emu_src_ck",
-	.bit_shift = 6,
-	.max_div = 3,
-	.reg = 0x1140,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_STARTS_AT_ONE,
-};
-
-static struct ti_clk pclkx2_fck = {
-	.name = "pclkx2_fck",
-	.type = TI_CLK_DIVIDER,
-	.data = &pclkx2_fck_data,
-};
-
-static struct ti_clk_gate sha12_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 27,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk sha12_ick = {
-	.name = "sha12_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &sha12_ick_data,
-};
-
-static struct ti_clk_gate emac_fck_data = {
-	.parent = "rmii_ck",
-	.bit_shift = 9,
-	.reg = 0x59c,
-	.module = TI_CLKM_SCRM,
-};
-
-static struct ti_clk emac_fck = {
-	.name = "emac_fck",
-	.type = TI_CLK_GATE,
-	.data = &emac_fck_data,
-};
-
-static struct ti_clk_composite gpt10_fck_data = {
-	.mux = &gpt10_mux_fck_data,
-	.gate = &gpt10_gate_fck_data,
-};
-
-static struct ti_clk gpt10_fck = {
-	.name = "gpt10_fck",
-	.type = TI_CLK_COMPOSITE,
-	.data = &gpt10_fck_data,
-};
-
-static struct ti_clk_gate wdt2_fck_data = {
-	.parent = "wkup_32k_fck",
-	.bit_shift = 5,
-	.reg = 0xc00,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk wdt2_fck = {
-	.name = "wdt2_fck",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &wdt2_fck_data,
-};
-
-static struct ti_clk_gate cam_ick_data = {
-	.parent = "l4_ick",
-	.bit_shift = 0,
-	.reg = 0xf10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_NO_WAIT | CLKF_INTERFACE,
-};
-
-static struct ti_clk cam_ick = {
-	.name = "cam_ick",
-	.clkdm_name = "cam_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &cam_ick_data,
-};
-
-static struct ti_clk_gate ssi_ick_3430es2_data = {
-	.parent = "ssi_l4_ick",
-	.bit_shift = 0,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_SSI | CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk ssi_ick_3430es2 = {
-	.name = "ssi_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &ssi_ick_3430es2_data,
-};
-
-static struct ti_clk_gate gpio4_ick_data = {
-	.parent = "per_l4_ick",
-	.bit_shift = 15,
-	.reg = 0x1010,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk gpio4_ick = {
-	.name = "gpio4_ick",
-	.clkdm_name = "per_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &gpio4_ick_data,
-};
-
-static struct ti_clk_gate wdt1_ick_data = {
-	.parent = "wkup_l4_ick",
-	.bit_shift = 4,
-	.reg = 0xc10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk wdt1_ick = {
-	.name = "wdt1_ick",
-	.clkdm_name = "wkup_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &wdt1_ick_data,
-};
-
-static struct ti_clk_gate rng_ick_data = {
-	.parent = "security_l4_ick2",
-	.bit_shift = 2,
-	.reg = 0xa14,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk rng_ick = {
-	.name = "rng_ick",
-	.type = TI_CLK_GATE,
-	.data = &rng_ick_data,
-};
-
-static struct ti_clk_gate icr_ick_data = {
-	.parent = "core_l4_ick",
-	.bit_shift = 29,
-	.reg = 0xa10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_OMAP3 | CLKF_INTERFACE,
-};
-
-static struct ti_clk icr_ick = {
-	.name = "icr_ick",
-	.clkdm_name = "core_l4_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &icr_ick_data,
-};
-
-static struct ti_clk_gate sgx_ick_data = {
-	.parent = "l3_ick",
-	.bit_shift = 0,
-	.reg = 0xb10,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_WAIT,
-};
-
-static struct ti_clk sgx_ick = {
-	.name = "sgx_ick",
-	.clkdm_name = "sgx_clkdm",
-	.type = TI_CLK_GATE,
-	.data = &sgx_ick_data,
-};
-
-static struct ti_clk_divider sys_clkout2_data = {
-	.parent = "clkout2_src_ck",
-	.bit_shift = 3,
-	.max_div = 64,
-	.reg = 0xd70,
-	.module = TI_CLKM_CM,
-	.flags = CLKF_INDEX_POWER_OF_TWO,
-};
-
-static struct ti_clk sys_clkout2 = {
-	.name = "sys_clkout2",
-	.type = TI_CLK_DIVIDER,
-	.data = &sys_clkout2_data,
-};
-
-static struct ti_clk_alias omap34xx_omap36xx_clks[] = {
-	CLK(NULL, "security_l4_ick2", &security_l4_ick2),
-	CLK(NULL, "aes1_ick", &aes1_ick),
-	CLK("omap_rng", "ick", &rng_ick),
-	CLK("omap3-rom-rng", "ick", &rng_ick),
-	CLK(NULL, "sha11_ick", &sha11_ick),
-	CLK(NULL, "des1_ick", &des1_ick),
-	CLK(NULL, "cam_mclk", &cam_mclk),
-	CLK(NULL, "cam_ick", &cam_ick),
-	CLK(NULL, "csi2_96m_fck", &csi2_96m_fck),
-	CLK(NULL, "security_l3_ick", &security_l3_ick),
-	CLK(NULL, "pka_ick", &pka_ick),
-	CLK(NULL, "icr_ick", &icr_ick),
-	CLK(NULL, "des2_ick", &des2_ick),
-	CLK(NULL, "mspro_ick", &mspro_ick),
-	CLK(NULL, "mailboxes_ick", &mailboxes_ick),
-	CLK(NULL, "ssi_l4_ick", &ssi_l4_ick),
-	CLK(NULL, "sr1_fck", &sr1_fck),
-	CLK(NULL, "sr2_fck", &sr2_fck),
-	CLK(NULL, "sr_l4_ick", &sr_l4_ick),
-	CLK(NULL, "dpll2_fck", &dpll2_fck),
-	CLK(NULL, "dpll2_ck", &dpll2_ck),
-	CLK(NULL, "dpll2_m2_ck", &dpll2_m2_ck),
-	CLK(NULL, "iva2_ck", &iva2_ck),
-	CLK(NULL, "modem_fck", &modem_fck),
-	CLK(NULL, "sad2d_ick", &sad2d_ick),
-	CLK(NULL, "mad2d_ick", &mad2d_ick),
-	CLK(NULL, "mspro_fck", &mspro_fck),
-	{ NULL },
-};
-
-static struct ti_clk_alias omap36xx_omap3430es2plus_clks[] = {
-	CLK(NULL, "ssi_ssr_fck", &ssi_ssr_fck_3430es2),
-	CLK(NULL, "ssi_sst_fck", &ssi_sst_fck_3430es2),
-	CLK("musb-omap2430", "ick", &hsotgusb_ick_3430es2),
-	CLK(NULL, "hsotgusb_ick", &hsotgusb_ick_3430es2),
-	CLK(NULL, "ssi_ick", &ssi_ick_3430es2),
-	CLK(NULL, "sys_d2_ck", &sys_d2_ck),
-	CLK(NULL, "omap_96m_d2_fck", &omap_96m_d2_fck),
-	CLK(NULL, "omap_96m_d4_fck", &omap_96m_d4_fck),
-	CLK(NULL, "omap_96m_d8_fck", &omap_96m_d8_fck),
-	CLK(NULL, "omap_96m_d10_fck", &omap_96m_d10_fck),
-	CLK(NULL, "dpll5_m2_d4_ck", &dpll5_m2_d4_ck),
-	CLK(NULL, "dpll5_m2_d8_ck", &dpll5_m2_d8_ck),
-	CLK(NULL, "dpll5_m2_d16_ck", &dpll5_m2_d16_ck),
-	CLK(NULL, "dpll5_m2_d20_ck", &dpll5_m2_d20_ck),
-	CLK(NULL, "usim_fck", &usim_fck),
-	CLK(NULL, "usim_ick", &usim_ick),
-	{ NULL },
-};
-
-static struct ti_clk_alias omap3xxx_clks[] = {
-	CLK(NULL, "apb_pclk", &dummy_apb_pclk),
-	CLK(NULL, "omap_32k_fck", &omap_32k_fck),
-	CLK(NULL, "virt_12m_ck", &virt_12m_ck),
-	CLK(NULL, "virt_13m_ck", &virt_13m_ck),
-	CLK(NULL, "virt_19200000_ck", &virt_19200000_ck),
-	CLK(NULL, "virt_26000000_ck", &virt_26000000_ck),
-	CLK(NULL, "virt_38_4m_ck", &virt_38_4m_ck),
-	CLK(NULL, "virt_16_8m_ck", &virt_16_8m_ck),
-	CLK(NULL, "osc_sys_ck", &osc_sys_ck),
-	CLK("twl", "fck", &osc_sys_ck),
-	CLK(NULL, "sys_ck", &sys_ck),
-	CLK(NULL, "timer_sys_ck", &sys_ck),
-	CLK(NULL, "dpll4_ck", &dpll4_ck),
-	CLK(NULL, "dpll4_m2_ck", &dpll4_m2_ck),
-	CLK(NULL, "dpll4_m2x2_mul_ck", &dpll4_m2x2_mul_ck),
-	CLK(NULL, "dpll4_m2x2_ck", &dpll4_m2x2_ck),
-	CLK(NULL, "omap_96m_alwon_fck", &omap_96m_alwon_fck),
-	CLK(NULL, "dpll3_ck", &dpll3_ck),
-	CLK(NULL, "dpll3_m3_ck", &dpll3_m3_ck),
-	CLK(NULL, "dpll3_m3x2_mul_ck", &dpll3_m3x2_mul_ck),
-	CLK(NULL, "dpll3_m3x2_ck", &dpll3_m3x2_ck),
-	CLK("etb", "emu_core_alwon_ck", &emu_core_alwon_ck),
-	CLK(NULL, "sys_altclk", &sys_altclk),
-	CLK(NULL, "sys_clkout1", &sys_clkout1),
-	CLK(NULL, "dpll3_m2_ck", &dpll3_m2_ck),
-	CLK(NULL, "core_ck", &core_ck),
-	CLK(NULL, "dpll1_fck", &dpll1_fck),
-	CLK(NULL, "dpll1_ck", &dpll1_ck),
-	CLK(NULL, "cpufreq_ck", &dpll1_ck),
-	CLK(NULL, "dpll1_x2_ck", &dpll1_x2_ck),
-	CLK(NULL, "dpll1_x2m2_ck", &dpll1_x2m2_ck),
-	CLK(NULL, "dpll3_x2_ck", &dpll3_x2_ck),
-	CLK(NULL, "dpll3_m2x2_ck", &dpll3_m2x2_ck),
-	CLK(NULL, "dpll4_x2_ck", &dpll4_x2_ck),
-	CLK(NULL, "cm_96m_fck", &cm_96m_fck),
-	CLK(NULL, "omap_96m_fck", &omap_96m_fck),
-	CLK(NULL, "dpll4_m3_ck", &dpll4_m3_ck),
-	CLK(NULL, "dpll4_m3x2_mul_ck", &dpll4_m3x2_mul_ck),
-	CLK(NULL, "dpll4_m3x2_ck", &dpll4_m3x2_ck),
-	CLK(NULL, "omap_54m_fck", &omap_54m_fck),
-	CLK(NULL, "cm_96m_d2_fck", &cm_96m_d2_fck),
-	CLK(NULL, "omap_48m_fck", &omap_48m_fck),
-	CLK(NULL, "omap_12m_fck", &omap_12m_fck),
-	CLK(NULL, "dpll4_m4_ck", &dpll4_m4_ck),
-	CLK(NULL, "dpll4_m4x2_mul_ck", &dpll4_m4x2_mul_ck),
-	CLK(NULL, "dpll4_m4x2_ck", &dpll4_m4x2_ck),
-	CLK(NULL, "dpll4_m5_ck", &dpll4_m5_ck),
-	CLK(NULL, "dpll4_m5x2_mul_ck", &dpll4_m5x2_mul_ck),
-	CLK(NULL, "dpll4_m5x2_ck", &dpll4_m5x2_ck),
-	CLK(NULL, "dpll4_m6_ck", &dpll4_m6_ck),
-	CLK(NULL, "dpll4_m6x2_mul_ck", &dpll4_m6x2_mul_ck),
-	CLK(NULL, "dpll4_m6x2_ck", &dpll4_m6x2_ck),
-	CLK("etb", "emu_per_alwon_ck", &emu_per_alwon_ck),
-	CLK(NULL, "clkout2_src_ck", &clkout2_src_ck),
-	CLK(NULL, "sys_clkout2", &sys_clkout2),
-	CLK(NULL, "corex2_fck", &corex2_fck),
-	CLK(NULL, "mpu_ck", &mpu_ck),
-	CLK(NULL, "arm_fck", &arm_fck),
-	CLK("etb", "emu_mpu_alwon_ck", &emu_mpu_alwon_ck),
-	CLK(NULL, "l3_ick", &l3_ick),
-	CLK(NULL, "l4_ick", &l4_ick),
-	CLK(NULL, "rm_ick", &rm_ick),
-	CLK(NULL, "timer_32k_ck", &omap_32k_fck),
-	CLK(NULL, "gpt10_fck", &gpt10_fck),
-	CLK(NULL, "gpt11_fck", &gpt11_fck),
-	CLK(NULL, "core_96m_fck", &core_96m_fck),
-	CLK(NULL, "mmchs2_fck", &mmchs2_fck),
-	CLK(NULL, "mmchs1_fck", &mmchs1_fck),
-	CLK(NULL, "i2c3_fck", &i2c3_fck),
-	CLK(NULL, "i2c2_fck", &i2c2_fck),
-	CLK(NULL, "i2c1_fck", &i2c1_fck),
-	CLK(NULL, "core_48m_fck", &core_48m_fck),
-	CLK(NULL, "mcspi4_fck", &mcspi4_fck),
-	CLK(NULL, "mcspi3_fck", &mcspi3_fck),
-	CLK(NULL, "mcspi2_fck", &mcspi2_fck),
-	CLK(NULL, "mcspi1_fck", &mcspi1_fck),
-	CLK(NULL, "uart2_fck", &uart2_fck),
-	CLK(NULL, "uart1_fck", &uart1_fck),
-	CLK(NULL, "core_12m_fck", &core_12m_fck),
-	CLK("omap_hdq.0", "fck", &hdq_fck),
-	CLK(NULL, "hdq_fck", &hdq_fck),
-	CLK(NULL, "core_l3_ick", &core_l3_ick),
-	CLK(NULL, "sdrc_ick", &sdrc_ick),
-	CLK(NULL, "gpmc_fck", &gpmc_fck),
-	CLK(NULL, "core_l4_ick", &core_l4_ick),
-	CLK("omap_hsmmc.1", "ick", &mmchs2_ick),
-	CLK("omap_hsmmc.0", "ick", &mmchs1_ick),
-	CLK(NULL, "mmchs2_ick", &mmchs2_ick),
-	CLK(NULL, "mmchs1_ick", &mmchs1_ick),
-	CLK("omap_hdq.0", "ick", &hdq_ick),
-	CLK(NULL, "hdq_ick", &hdq_ick),
-	CLK("omap2_mcspi.4", "ick", &mcspi4_ick),
-	CLK("omap2_mcspi.3", "ick", &mcspi3_ick),
-	CLK("omap2_mcspi.2", "ick", &mcspi2_ick),
-	CLK("omap2_mcspi.1", "ick", &mcspi1_ick),
-	CLK(NULL, "mcspi4_ick", &mcspi4_ick),
-	CLK(NULL, "mcspi3_ick", &mcspi3_ick),
-	CLK(NULL, "mcspi2_ick", &mcspi2_ick),
-	CLK(NULL, "mcspi1_ick", &mcspi1_ick),
-	CLK("omap_i2c.3", "ick", &i2c3_ick),
-	CLK("omap_i2c.2", "ick", &i2c2_ick),
-	CLK("omap_i2c.1", "ick", &i2c1_ick),
-	CLK(NULL, "i2c3_ick", &i2c3_ick),
-	CLK(NULL, "i2c2_ick", &i2c2_ick),
-	CLK(NULL, "i2c1_ick", &i2c1_ick),
-	CLK(NULL, "uart2_ick", &uart2_ick),
-	CLK(NULL, "uart1_ick", &uart1_ick),
-	CLK(NULL, "gpt11_ick", &gpt11_ick),
-	CLK(NULL, "gpt10_ick", &gpt10_ick),
-	CLK(NULL, "mcbsp5_ick", &mcbsp5_ick),
-	CLK(NULL, "mcbsp1_ick", &mcbsp1_ick),
-	CLK(NULL, "omapctrl_ick", &omapctrl_ick),
-	CLK(NULL, "dss_tv_fck", &dss_tv_fck),
-	CLK(NULL, "dss_96m_fck", &dss_96m_fck),
-	CLK(NULL, "dss2_alwon_fck", &dss2_alwon_fck),
-	CLK(NULL, "init_60m_fclk", &dummy_ck),
-	CLK(NULL, "gpt1_fck", &gpt1_fck),
-	CLK(NULL, "aes2_ick", &aes2_ick),
-	CLK(NULL, "wkup_32k_fck", &wkup_32k_fck),
-	CLK(NULL, "gpio1_dbck", &gpio1_dbck),
-	CLK(NULL, "sha12_ick", &sha12_ick),
-	CLK(NULL, "wdt2_fck", &wdt2_fck),
-	CLK(NULL, "wkup_l4_ick", &wkup_l4_ick),
-	CLK("omap_wdt", "ick", &wdt2_ick),
-	CLK(NULL, "wdt2_ick", &wdt2_ick),
-	CLK(NULL, "wdt1_ick", &wdt1_ick),
-	CLK(NULL, "gpio1_ick", &gpio1_ick),
-	CLK(NULL, "omap_32ksync_ick", &omap_32ksync_ick),
-	CLK(NULL, "gpt12_ick", &gpt12_ick),
-	CLK(NULL, "gpt1_ick", &gpt1_ick),
-	CLK(NULL, "per_96m_fck", &per_96m_fck),
-	CLK(NULL, "per_48m_fck", &per_48m_fck),
-	CLK(NULL, "uart3_fck", &uart3_fck),
-	CLK(NULL, "gpt2_fck", &gpt2_fck),
-	CLK(NULL, "gpt3_fck", &gpt3_fck),
-	CLK(NULL, "gpt4_fck", &gpt4_fck),
-	CLK(NULL, "gpt5_fck", &gpt5_fck),
-	CLK(NULL, "gpt6_fck", &gpt6_fck),
-	CLK(NULL, "gpt7_fck", &gpt7_fck),
-	CLK(NULL, "gpt8_fck", &gpt8_fck),
-	CLK(NULL, "gpt9_fck", &gpt9_fck),
-	CLK(NULL, "per_32k_alwon_fck", &per_32k_alwon_fck),
-	CLK(NULL, "gpio6_dbck", &gpio6_dbck),
-	CLK(NULL, "gpio5_dbck", &gpio5_dbck),
-	CLK(NULL, "gpio4_dbck", &gpio4_dbck),
-	CLK(NULL, "gpio3_dbck", &gpio3_dbck),
-	CLK(NULL, "gpio2_dbck", &gpio2_dbck),
-	CLK(NULL, "wdt3_fck", &wdt3_fck),
-	CLK(NULL, "per_l4_ick", &per_l4_ick),
-	CLK(NULL, "gpio6_ick", &gpio6_ick),
-	CLK(NULL, "gpio5_ick", &gpio5_ick),
-	CLK(NULL, "gpio4_ick", &gpio4_ick),
-	CLK(NULL, "gpio3_ick", &gpio3_ick),
-	CLK(NULL, "gpio2_ick", &gpio2_ick),
-	CLK(NULL, "wdt3_ick", &wdt3_ick),
-	CLK(NULL, "uart3_ick", &uart3_ick),
-	CLK(NULL, "uart4_ick", &uart4_ick),
-	CLK(NULL, "gpt9_ick", &gpt9_ick),
-	CLK(NULL, "gpt8_ick", &gpt8_ick),
-	CLK(NULL, "gpt7_ick", &gpt7_ick),
-	CLK(NULL, "gpt6_ick", &gpt6_ick),
-	CLK(NULL, "gpt5_ick", &gpt5_ick),
-	CLK(NULL, "gpt4_ick", &gpt4_ick),
-	CLK(NULL, "gpt3_ick", &gpt3_ick),
-	CLK(NULL, "gpt2_ick", &gpt2_ick),
-	CLK(NULL, "mcbsp_clks", &mcbsp_clks),
-	CLK("omap-mcbsp.1", "ick", &mcbsp1_ick),
-	CLK("omap-mcbsp.2", "ick", &mcbsp2_ick),
-	CLK("omap-mcbsp.3", "ick", &mcbsp3_ick),
-	CLK("omap-mcbsp.4", "ick", &mcbsp4_ick),
-	CLK("omap-mcbsp.5", "ick", &mcbsp5_ick),
-	CLK(NULL, "mcbsp1_ick", &mcbsp1_ick),
-	CLK(NULL, "mcbsp2_ick", &mcbsp2_ick),
-	CLK(NULL, "mcbsp3_ick", &mcbsp3_ick),
-	CLK(NULL, "mcbsp4_ick", &mcbsp4_ick),
-	CLK(NULL, "mcbsp5_ick", &mcbsp5_ick),
-	CLK(NULL, "mcbsp1_fck", &mcbsp1_fck),
-	CLK(NULL, "mcbsp2_fck", &mcbsp2_fck),
-	CLK(NULL, "mcbsp3_fck", &mcbsp3_fck),
-	CLK(NULL, "mcbsp4_fck", &mcbsp4_fck),
-	CLK(NULL, "mcbsp5_fck", &mcbsp5_fck),
-	CLK(NULL, "emu_src_mux_ck", &emu_src_mux_ck),
-	CLK("etb", "emu_src_ck", &emu_src_ck),
-	CLK(NULL, "emu_src_mux_ck", &emu_src_mux_ck),
-	CLK(NULL, "emu_src_ck", &emu_src_ck),
-	CLK(NULL, "pclk_fck", &pclk_fck),
-	CLK(NULL, "pclkx2_fck", &pclkx2_fck),
-	CLK(NULL, "atclk_fck", &atclk_fck),
-	CLK(NULL, "traceclk_src_fck", &traceclk_src_fck),
-	CLK(NULL, "traceclk_fck", &traceclk_fck),
-	CLK(NULL, "secure_32k_fck", &secure_32k_fck),
-	CLK(NULL, "gpt12_fck", &gpt12_fck),
-	CLK(NULL, "wdt1_fck", &wdt1_fck),
-	{ NULL },
-};
-
-static struct ti_clk_alias omap36xx_am35xx_omap3430es2plus_clks[] = {
-	CLK(NULL, "dpll5_ck", &dpll5_ck),
-	CLK(NULL, "dpll5_m2_ck", &dpll5_m2_ck),
-	CLK(NULL, "core_d3_ck", &core_d3_ck),
-	CLK(NULL, "core_d4_ck", &core_d4_ck),
-	CLK(NULL, "core_d6_ck", &core_d6_ck),
-	CLK(NULL, "omap_192m_alwon_fck", &omap_192m_alwon_fck),
-	CLK(NULL, "core_d2_ck", &core_d2_ck),
-	CLK(NULL, "corex2_d3_fck", &corex2_d3_fck),
-	CLK(NULL, "corex2_d5_fck", &corex2_d5_fck),
-	CLK(NULL, "sgx_fck", &sgx_fck),
-	CLK(NULL, "sgx_ick", &sgx_ick),
-	CLK(NULL, "cpefuse_fck", &cpefuse_fck),
-	CLK(NULL, "ts_fck", &ts_fck),
-	CLK(NULL, "usbtll_fck", &usbtll_fck),
-	CLK(NULL, "usbtll_ick", &usbtll_ick),
-	CLK("omap_hsmmc.2", "ick", &mmchs3_ick),
-	CLK(NULL, "mmchs3_ick", &mmchs3_ick),
-	CLK(NULL, "mmchs3_fck", &mmchs3_fck),
-	CLK(NULL, "dss1_alwon_fck", &dss1_alwon_fck_3430es2),
-	CLK("omapdss_dss", "ick", &dss_ick_3430es2),
-	CLK(NULL, "dss_ick", &dss_ick_3430es2),
-	CLK(NULL, "usbhost_120m_fck", &usbhost_120m_fck),
-	CLK(NULL, "usbhost_48m_fck", &usbhost_48m_fck),
-	CLK(NULL, "usbhost_ick", &usbhost_ick),
-	{ NULL },
-};
-
-static struct ti_clk_alias omap3430es1_clks[] = {
-	CLK(NULL, "gfx_l3_ck", &gfx_l3_ck),
-	CLK(NULL, "gfx_l3_fck", &gfx_l3_fck),
-	CLK(NULL, "gfx_l3_ick", &gfx_l3_ick),
-	CLK(NULL, "gfx_cg1_ck", &gfx_cg1_ck),
-	CLK(NULL, "gfx_cg2_ck", &gfx_cg2_ck),
-	CLK(NULL, "d2d_26m_fck", &d2d_26m_fck),
-	CLK(NULL, "fshostusb_fck", &fshostusb_fck),
-	CLK(NULL, "ssi_ssr_fck", &ssi_ssr_fck_3430es1),
-	CLK(NULL, "ssi_sst_fck", &ssi_sst_fck_3430es1),
-	CLK("musb-omap2430", "ick", &hsotgusb_ick_3430es1),
-	CLK(NULL, "hsotgusb_ick", &hsotgusb_ick_3430es1),
-	CLK(NULL, "fac_ick", &fac_ick),
-	CLK(NULL, "ssi_ick", &ssi_ick_3430es1),
-	CLK(NULL, "usb_l4_ick", &usb_l4_ick),
-	CLK(NULL, "dss1_alwon_fck", &dss1_alwon_fck_3430es1),
-	CLK("omapdss_dss", "ick", &dss_ick_3430es1),
-	CLK(NULL, "dss_ick", &dss_ick_3430es1),
-	{ NULL },
-};
-
-static struct ti_clk_alias omap36xx_clks[] = {
-	CLK(NULL, "uart4_fck", &uart4_fck),
-	{ NULL },
-};
-
-static struct ti_clk_alias am35xx_clks[] = {
-	CLK(NULL, "ipss_ick", &ipss_ick),
-	CLK(NULL, "rmii_ck", &rmii_ck),
-	CLK(NULL, "pclk_ck", &pclk_ck),
-	CLK(NULL, "emac_ick", &emac_ick),
-	CLK(NULL, "emac_fck", &emac_fck),
-	CLK("davinci_emac.0", NULL, &emac_ick),
-	CLK("davinci_mdio.0", NULL, &emac_fck),
-	CLK("vpfe-capture", "master", &vpfe_ick),
-	CLK("vpfe-capture", "slave", &vpfe_fck),
-	CLK(NULL, "hsotgusb_ick", &hsotgusb_ick_am35xx),
-	CLK(NULL, "hsotgusb_fck", &hsotgusb_fck_am35xx),
-	CLK(NULL, "hecc_ck", &hecc_ck),
-	CLK(NULL, "uart4_ick", &uart4_ick_am35xx),
-	CLK(NULL, "uart4_fck", &uart4_fck_am35xx),
-	{ NULL },
-};
-
-static struct ti_clk *omap36xx_clk_patches[] = {
-	&dpll4_m3x2_ck_omap36xx,
-	&dpll3_m3x2_ck_omap36xx,
-	&dpll4_m6x2_ck_omap36xx,
-	&dpll4_m2x2_ck_omap36xx,
-	&dpll4_m5x2_ck_omap36xx,
-	&dpll4_ck_omap36xx,
-	NULL,
-};
-
-static const char *enable_init_clks[] = {
-	"sdrc_ick",
-	"gpmc_fck",
-	"omapctrl_ick",
-};
-
-static void __init omap3_clk_legacy_common_init(void)
-{
-	omap2_clk_disable_autoidle_all();
-
-	omap2_clk_enable_init_clocks(enable_init_clks,
-				     ARRAY_SIZE(enable_init_clks));
-
-	pr_info("Clocking rate (Crystal/Core/MPU): %ld.%01ld/%ld/%ld MHz\n",
-		(clk_get_rate(osc_sys_ck.clk) / 1000000),
-		(clk_get_rate(osc_sys_ck.clk) / 100000) % 10,
-		(clk_get_rate(core_ck.clk) / 1000000),
-		(clk_get_rate(arm_fck.clk) / 1000000));
-}
-
-int __init omap3430es1_clk_legacy_init(void)
-{
-	int r;
-
-	r = ti_clk_register_legacy_clks(omap3430es1_clks);
-	r |= ti_clk_register_legacy_clks(omap34xx_omap36xx_clks);
-	r |= ti_clk_register_legacy_clks(omap3xxx_clks);
-
-	omap3_clk_legacy_common_init();
-
-	return r;
-}
-
-int __init omap3430_clk_legacy_init(void)
-{
-	int r;
-
-	r = ti_clk_register_legacy_clks(omap34xx_omap36xx_clks);
-	r |= ti_clk_register_legacy_clks(omap36xx_omap3430es2plus_clks);
-	r |= ti_clk_register_legacy_clks(omap36xx_am35xx_omap3430es2plus_clks);
-	r |= ti_clk_register_legacy_clks(omap3xxx_clks);
-
-	omap3_clk_legacy_common_init();
-	omap3_clk_lock_dpll5();
-
-	return r;
-}
-
-int __init omap36xx_clk_legacy_init(void)
-{
-	int r;
-
-	ti_clk_patch_legacy_clks(omap36xx_clk_patches);
-	r = ti_clk_register_legacy_clks(omap36xx_clks);
-	r |= ti_clk_register_legacy_clks(omap36xx_omap3430es2plus_clks);
-	r |= ti_clk_register_legacy_clks(omap34xx_omap36xx_clks);
-	r |= ti_clk_register_legacy_clks(omap36xx_am35xx_omap3430es2plus_clks);
-	r |= ti_clk_register_legacy_clks(omap3xxx_clks);
-
-	omap3_clk_legacy_common_init();
-	omap3_clk_lock_dpll5();
-
-	return r;
-}
-
-int __init am35xx_clk_legacy_init(void)
-{
-	int r;
-
-	r = ti_clk_register_legacy_clks(am35xx_clks);
-	r |= ti_clk_register_legacy_clks(omap36xx_am35xx_omap3430es2plus_clks);
-	r |= ti_clk_register_legacy_clks(omap3xxx_clks);
-
-	omap3_clk_legacy_common_init();
-	omap3_clk_lock_dpll5();
-
-	return r;
-}
diff --git a/drivers/clk/ti/clk-3xxx.c b/drivers/clk/ti/clk-3xxx.c
index b1251cae98b8..8aa5f5793835 100644
--- a/drivers/clk/ti/clk-3xxx.c
+++ b/drivers/clk/ti/clk-3xxx.c
@@ -224,296 +224,43 @@ const struct clk_hw_omap_ops clkhwops_am35xx_ipss_wait = {
 };
 
 static struct ti_dt_clk omap3xxx_clks[] = {
-	DT_CLK(NULL, "apb_pclk", "dummy_apb_pclk"),
-	DT_CLK(NULL, "omap_32k_fck", "omap_32k_fck"),
-	DT_CLK(NULL, "virt_12m_ck", "virt_12m_ck"),
-	DT_CLK(NULL, "virt_13m_ck", "virt_13m_ck"),
-	DT_CLK(NULL, "virt_19200000_ck", "virt_19200000_ck"),
-	DT_CLK(NULL, "virt_26000000_ck", "virt_26000000_ck"),
-	DT_CLK(NULL, "virt_38_4m_ck", "virt_38_4m_ck"),
-	DT_CLK(NULL, "osc_sys_ck", "osc_sys_ck"),
-	DT_CLK("twl", "fck", "osc_sys_ck"),
-	DT_CLK(NULL, "sys_ck", "sys_ck"),
-	DT_CLK(NULL, "omap_96m_alwon_fck", "omap_96m_alwon_fck"),
-	DT_CLK("etb", "emu_core_alwon_ck", "emu_core_alwon_ck"),
-	DT_CLK(NULL, "sys_altclk", "sys_altclk"),
-	DT_CLK(NULL, "sys_clkout1", "sys_clkout1"),
-	DT_CLK(NULL, "dpll1_ck", "dpll1_ck"),
-	DT_CLK(NULL, "dpll1_x2_ck", "dpll1_x2_ck"),
-	DT_CLK(NULL, "dpll1_x2m2_ck", "dpll1_x2m2_ck"),
-	DT_CLK(NULL, "dpll3_ck", "dpll3_ck"),
-	DT_CLK(NULL, "core_ck", "core_ck"),
-	DT_CLK(NULL, "dpll3_x2_ck", "dpll3_x2_ck"),
-	DT_CLK(NULL, "dpll3_m2_ck", "dpll3_m2_ck"),
-	DT_CLK(NULL, "dpll3_m2x2_ck", "dpll3_m2x2_ck"),
-	DT_CLK(NULL, "dpll3_m3_ck", "dpll3_m3_ck"),
-	DT_CLK(NULL, "dpll3_m3x2_ck", "dpll3_m3x2_ck"),
-	DT_CLK(NULL, "dpll4_ck", "dpll4_ck"),
-	DT_CLK(NULL, "dpll4_x2_ck", "dpll4_x2_ck"),
-	DT_CLK(NULL, "omap_96m_fck", "omap_96m_fck"),
-	DT_CLK(NULL, "cm_96m_fck", "cm_96m_fck"),
-	DT_CLK(NULL, "omap_54m_fck", "omap_54m_fck"),
-	DT_CLK(NULL, "omap_48m_fck", "omap_48m_fck"),
-	DT_CLK(NULL, "omap_12m_fck", "omap_12m_fck"),
-	DT_CLK(NULL, "dpll4_m2_ck", "dpll4_m2_ck"),
-	DT_CLK(NULL, "dpll4_m2x2_ck", "dpll4_m2x2_ck"),
-	DT_CLK(NULL, "dpll4_m3_ck", "dpll4_m3_ck"),
-	DT_CLK(NULL, "dpll4_m3x2_ck", "dpll4_m3x2_ck"),
-	DT_CLK(NULL, "dpll4_m4_ck", "dpll4_m4_ck"),
-	DT_CLK(NULL, "dpll4_m4x2_ck", "dpll4_m4x2_ck"),
-	DT_CLK(NULL, "dpll4_m5_ck", "dpll4_m5_ck"),
-	DT_CLK(NULL, "dpll4_m5x2_ck", "dpll4_m5x2_ck"),
-	DT_CLK(NULL, "dpll4_m6_ck", "dpll4_m6_ck"),
-	DT_CLK(NULL, "dpll4_m6x2_ck", "dpll4_m6x2_ck"),
-	DT_CLK("etb", "emu_per_alwon_ck", "emu_per_alwon_ck"),
-	DT_CLK(NULL, "clkout2_src_ck", "clkout2_src_ck"),
-	DT_CLK(NULL, "sys_clkout2", "sys_clkout2"),
-	DT_CLK(NULL, "corex2_fck", "corex2_fck"),
-	DT_CLK(NULL, "dpll1_fck", "dpll1_fck"),
-	DT_CLK(NULL, "mpu_ck", "mpu_ck"),
-	DT_CLK(NULL, "arm_fck", "arm_fck"),
-	DT_CLK("etb", "emu_mpu_alwon_ck", "emu_mpu_alwon_ck"),
-	DT_CLK(NULL, "l3_ick", "l3_ick"),
-	DT_CLK(NULL, "l4_ick", "l4_ick"),
-	DT_CLK(NULL, "rm_ick", "rm_ick"),
-	DT_CLK(NULL, "gpt10_fck", "gpt10_fck"),
-	DT_CLK(NULL, "gpt11_fck", "gpt11_fck"),
-	DT_CLK(NULL, "core_96m_fck", "core_96m_fck"),
-	DT_CLK(NULL, "mmchs2_fck", "mmchs2_fck"),
-	DT_CLK(NULL, "mmchs1_fck", "mmchs1_fck"),
-	DT_CLK(NULL, "i2c3_fck", "i2c3_fck"),
-	DT_CLK(NULL, "i2c2_fck", "i2c2_fck"),
-	DT_CLK(NULL, "i2c1_fck", "i2c1_fck"),
-	DT_CLK(NULL, "core_48m_fck", "core_48m_fck"),
-	DT_CLK(NULL, "mcspi4_fck", "mcspi4_fck"),
-	DT_CLK(NULL, "mcspi3_fck", "mcspi3_fck"),
-	DT_CLK(NULL, "mcspi2_fck", "mcspi2_fck"),
-	DT_CLK(NULL, "mcspi1_fck", "mcspi1_fck"),
-	DT_CLK(NULL, "uart2_fck", "uart2_fck"),
-	DT_CLK(NULL, "uart1_fck", "uart1_fck"),
-	DT_CLK(NULL, "core_12m_fck", "core_12m_fck"),
-	DT_CLK("omap_hdq.0", "fck", "hdq_fck"),
-	DT_CLK(NULL, "hdq_fck", "hdq_fck"),
-	DT_CLK(NULL, "core_l3_ick", "core_l3_ick"),
-	DT_CLK(NULL, "sdrc_ick", "sdrc_ick"),
-	DT_CLK(NULL, "gpmc_fck", "gpmc_fck"),
-	DT_CLK(NULL, "core_l4_ick", "core_l4_ick"),
-	DT_CLK("omap_hsmmc.1", "ick", "mmchs2_ick"),
-	DT_CLK("omap_hsmmc.0", "ick", "mmchs1_ick"),
-	DT_CLK(NULL, "mmchs2_ick", "mmchs2_ick"),
-	DT_CLK(NULL, "mmchs1_ick", "mmchs1_ick"),
-	DT_CLK("omap_hdq.0", "ick", "hdq_ick"),
-	DT_CLK(NULL, "hdq_ick", "hdq_ick"),
-	DT_CLK("omap2_mcspi.4", "ick", "mcspi4_ick"),
-	DT_CLK("omap2_mcspi.3", "ick", "mcspi3_ick"),
-	DT_CLK("omap2_mcspi.2", "ick", "mcspi2_ick"),
-	DT_CLK("omap2_mcspi.1", "ick", "mcspi1_ick"),
-	DT_CLK(NULL, "mcspi4_ick", "mcspi4_ick"),
-	DT_CLK(NULL, "mcspi3_ick", "mcspi3_ick"),
-	DT_CLK(NULL, "mcspi2_ick", "mcspi2_ick"),
-	DT_CLK(NULL, "mcspi1_ick", "mcspi1_ick"),
-	DT_CLK("omap_i2c.3", "ick", "i2c3_ick"),
-	DT_CLK("omap_i2c.2", "ick", "i2c2_ick"),
-	DT_CLK("omap_i2c.1", "ick", "i2c1_ick"),
-	DT_CLK(NULL, "i2c3_ick", "i2c3_ick"),
-	DT_CLK(NULL, "i2c2_ick", "i2c2_ick"),
-	DT_CLK(NULL, "i2c1_ick", "i2c1_ick"),
-	DT_CLK(NULL, "uart2_ick", "uart2_ick"),
-	DT_CLK(NULL, "uart1_ick", "uart1_ick"),
-	DT_CLK(NULL, "gpt11_ick", "gpt11_ick"),
-	DT_CLK(NULL, "gpt10_ick", "gpt10_ick"),
-	DT_CLK(NULL, "omapctrl_ick", "omapctrl_ick"),
-	DT_CLK(NULL, "dss_tv_fck", "dss_tv_fck"),
-	DT_CLK(NULL, "dss_96m_fck", "dss_96m_fck"),
-	DT_CLK(NULL, "dss2_alwon_fck", "dss2_alwon_fck"),
-	DT_CLK(NULL, "init_60m_fclk", "dummy_ck"),
-	DT_CLK(NULL, "gpt1_fck", "gpt1_fck"),
-	DT_CLK(NULL, "aes2_ick", "aes2_ick"),
-	DT_CLK(NULL, "wkup_32k_fck", "wkup_32k_fck"),
-	DT_CLK(NULL, "gpio1_dbck", "gpio1_dbck"),
-	DT_CLK(NULL, "sha12_ick", "sha12_ick"),
-	DT_CLK(NULL, "wdt2_fck", "wdt2_fck"),
-	DT_CLK("omap_wdt", "ick", "wdt2_ick"),
-	DT_CLK(NULL, "wdt2_ick", "wdt2_ick"),
-	DT_CLK(NULL, "wdt1_ick", "wdt1_ick"),
-	DT_CLK(NULL, "gpio1_ick", "gpio1_ick"),
-	DT_CLK(NULL, "omap_32ksync_ick", "omap_32ksync_ick"),
-	DT_CLK(NULL, "gpt12_ick", "gpt12_ick"),
-	DT_CLK(NULL, "gpt1_ick", "gpt1_ick"),
-	DT_CLK(NULL, "per_96m_fck", "per_96m_fck"),
-	DT_CLK(NULL, "per_48m_fck", "per_48m_fck"),
-	DT_CLK(NULL, "uart3_fck", "uart3_fck"),
-	DT_CLK(NULL, "gpt2_fck", "gpt2_fck"),
-	DT_CLK(NULL, "gpt3_fck", "gpt3_fck"),
-	DT_CLK(NULL, "gpt4_fck", "gpt4_fck"),
-	DT_CLK(NULL, "gpt5_fck", "gpt5_fck"),
-	DT_CLK(NULL, "gpt6_fck", "gpt6_fck"),
-	DT_CLK(NULL, "gpt7_fck", "gpt7_fck"),
-	DT_CLK(NULL, "gpt8_fck", "gpt8_fck"),
-	DT_CLK(NULL, "gpt9_fck", "gpt9_fck"),
-	DT_CLK(NULL, "per_32k_alwon_fck", "per_32k_alwon_fck"),
-	DT_CLK(NULL, "gpio6_dbck", "gpio6_dbck"),
-	DT_CLK(NULL, "gpio5_dbck", "gpio5_dbck"),
-	DT_CLK(NULL, "gpio4_dbck", "gpio4_dbck"),
-	DT_CLK(NULL, "gpio3_dbck", "gpio3_dbck"),
-	DT_CLK(NULL, "gpio2_dbck", "gpio2_dbck"),
-	DT_CLK(NULL, "wdt3_fck", "wdt3_fck"),
-	DT_CLK(NULL, "per_l4_ick", "per_l4_ick"),
-	DT_CLK(NULL, "gpio6_ick", "gpio6_ick"),
-	DT_CLK(NULL, "gpio5_ick", "gpio5_ick"),
-	DT_CLK(NULL, "gpio4_ick", "gpio4_ick"),
-	DT_CLK(NULL, "gpio3_ick", "gpio3_ick"),
-	DT_CLK(NULL, "gpio2_ick", "gpio2_ick"),
-	DT_CLK(NULL, "wdt3_ick", "wdt3_ick"),
-	DT_CLK(NULL, "uart3_ick", "uart3_ick"),
-	DT_CLK(NULL, "gpt9_ick", "gpt9_ick"),
-	DT_CLK(NULL, "gpt8_ick", "gpt8_ick"),
-	DT_CLK(NULL, "gpt7_ick", "gpt7_ick"),
-	DT_CLK(NULL, "gpt6_ick", "gpt6_ick"),
-	DT_CLK(NULL, "gpt5_ick", "gpt5_ick"),
-	DT_CLK(NULL, "gpt4_ick", "gpt4_ick"),
-	DT_CLK(NULL, "gpt3_ick", "gpt3_ick"),
-	DT_CLK(NULL, "gpt2_ick", "gpt2_ick"),
-	DT_CLK(NULL, "mcbsp_clks", "mcbsp_clks"),
-	DT_CLK(NULL, "mcbsp1_ick", "mcbsp1_ick"),
-	DT_CLK(NULL, "mcbsp2_ick", "mcbsp2_ick"),
-	DT_CLK(NULL, "mcbsp3_ick", "mcbsp3_ick"),
-	DT_CLK(NULL, "mcbsp4_ick", "mcbsp4_ick"),
-	DT_CLK(NULL, "mcbsp5_ick", "mcbsp5_ick"),
-	DT_CLK(NULL, "mcbsp1_fck", "mcbsp1_fck"),
-	DT_CLK(NULL, "mcbsp2_fck", "mcbsp2_fck"),
-	DT_CLK(NULL, "mcbsp3_fck", "mcbsp3_fck"),
-	DT_CLK(NULL, "mcbsp4_fck", "mcbsp4_fck"),
-	DT_CLK(NULL, "mcbsp5_fck", "mcbsp5_fck"),
-	DT_CLK("etb", "emu_src_ck", "emu_src_ck"),
-	DT_CLK(NULL, "emu_src_ck", "emu_src_ck"),
-	DT_CLK(NULL, "pclk_fck", "pclk_fck"),
-	DT_CLK(NULL, "pclkx2_fck", "pclkx2_fck"),
-	DT_CLK(NULL, "atclk_fck", "atclk_fck"),
-	DT_CLK(NULL, "traceclk_src_fck", "traceclk_src_fck"),
-	DT_CLK(NULL, "traceclk_fck", "traceclk_fck"),
-	DT_CLK(NULL, "secure_32k_fck", "secure_32k_fck"),
-	DT_CLK(NULL, "gpt12_fck", "gpt12_fck"),
-	DT_CLK(NULL, "wdt1_fck", "wdt1_fck"),
 	DT_CLK(NULL, "timer_32k_ck", "omap_32k_fck"),
 	DT_CLK(NULL, "timer_sys_ck", "sys_ck"),
-	DT_CLK(NULL, "cpufreq_ck", "dpll1_ck"),
-	{ .node_name = NULL },
-};
-
-static struct ti_dt_clk omap34xx_omap36xx_clks[] = {
-	DT_CLK(NULL, "aes1_ick", "aes1_ick"),
-	DT_CLK("omap_rng", "ick", "rng_ick"),
-	DT_CLK("omap3-rom-rng", "ick", "rng_ick"),
-	DT_CLK(NULL, "sha11_ick", "sha11_ick"),
-	DT_CLK(NULL, "des1_ick", "des1_ick"),
-	DT_CLK(NULL, "cam_mclk", "cam_mclk"),
-	DT_CLK(NULL, "cam_ick", "cam_ick"),
-	DT_CLK(NULL, "csi2_96m_fck", "csi2_96m_fck"),
-	DT_CLK(NULL, "security_l3_ick", "security_l3_ick"),
-	DT_CLK(NULL, "pka_ick", "pka_ick"),
-	DT_CLK(NULL, "icr_ick", "icr_ick"),
-	DT_CLK("omap-aes", "ick", "aes2_ick"),
-	DT_CLK("omap-sham", "ick", "sha12_ick"),
-	DT_CLK(NULL, "des2_ick", "des2_ick"),
-	DT_CLK(NULL, "mspro_ick", "mspro_ick"),
-	DT_CLK(NULL, "mailboxes_ick", "mailboxes_ick"),
-	DT_CLK(NULL, "ssi_l4_ick", "ssi_l4_ick"),
-	DT_CLK(NULL, "sr1_fck", "sr1_fck"),
-	DT_CLK(NULL, "sr2_fck", "sr2_fck"),
-	DT_CLK(NULL, "sr_l4_ick", "sr_l4_ick"),
-	DT_CLK(NULL, "security_l4_ick2", "security_l4_ick2"),
-	DT_CLK(NULL, "wkup_l4_ick", "wkup_l4_ick"),
-	DT_CLK(NULL, "dpll2_fck", "dpll2_fck"),
-	DT_CLK(NULL, "iva2_ck", "iva2_ck"),
-	DT_CLK(NULL, "modem_fck", "modem_fck"),
-	DT_CLK(NULL, "sad2d_ick", "sad2d_ick"),
-	DT_CLK(NULL, "mad2d_ick", "mad2d_ick"),
-	DT_CLK(NULL, "mspro_fck", "mspro_fck"),
-	DT_CLK(NULL, "dpll2_ck", "dpll2_ck"),
-	DT_CLK(NULL, "dpll2_m2_ck", "dpll2_m2_ck"),
 	{ .node_name = NULL },
 };
 
 static struct ti_dt_clk omap36xx_omap3430es2plus_clks[] = {
 	DT_CLK(NULL, "ssi_ssr_fck", "ssi_ssr_fck_3430es2"),
 	DT_CLK(NULL, "ssi_sst_fck", "ssi_sst_fck_3430es2"),
-	DT_CLK("musb-omap2430", "ick", "hsotgusb_ick_3430es2"),
 	DT_CLK(NULL, "hsotgusb_ick", "hsotgusb_ick_3430es2"),
 	DT_CLK(NULL, "ssi_ick", "ssi_ick_3430es2"),
-	DT_CLK(NULL, "usim_fck", "usim_fck"),
-	DT_CLK(NULL, "usim_ick", "usim_ick"),
 	{ .node_name = NULL },
 };
 
 static struct ti_dt_clk omap3430es1_clks[] = {
-	DT_CLK(NULL, "gfx_l3_ck", "gfx_l3_ck"),
-	DT_CLK(NULL, "gfx_l3_fck", "gfx_l3_fck"),
-	DT_CLK(NULL, "gfx_l3_ick", "gfx_l3_ick"),
-	DT_CLK(NULL, "gfx_cg1_ck", "gfx_cg1_ck"),
-	DT_CLK(NULL, "gfx_cg2_ck", "gfx_cg2_ck"),
-	DT_CLK(NULL, "d2d_26m_fck", "d2d_26m_fck"),
-	DT_CLK(NULL, "fshostusb_fck", "fshostusb_fck"),
 	DT_CLK(NULL, "ssi_ssr_fck", "ssi_ssr_fck_3430es1"),
 	DT_CLK(NULL, "ssi_sst_fck", "ssi_sst_fck_3430es1"),
-	DT_CLK("musb-omap2430", "ick", "hsotgusb_ick_3430es1"),
 	DT_CLK(NULL, "hsotgusb_ick", "hsotgusb_ick_3430es1"),
-	DT_CLK(NULL, "fac_ick", "fac_ick"),
 	DT_CLK(NULL, "ssi_ick", "ssi_ick_3430es1"),
-	DT_CLK(NULL, "usb_l4_ick", "usb_l4_ick"),
 	DT_CLK(NULL, "dss1_alwon_fck", "dss1_alwon_fck_3430es1"),
-	DT_CLK("omapdss_dss", "ick", "dss_ick_3430es1"),
 	DT_CLK(NULL, "dss_ick", "dss_ick_3430es1"),
 	{ .node_name = NULL },
 };
 
 static struct ti_dt_clk omap36xx_am35xx_omap3430es2plus_clks[] = {
-	DT_CLK(NULL, "virt_16_8m_ck", "virt_16_8m_ck"),
-	DT_CLK(NULL, "dpll5_ck", "dpll5_ck"),
-	DT_CLK(NULL, "dpll5_m2_ck", "dpll5_m2_ck"),
-	DT_CLK(NULL, "sgx_fck", "sgx_fck"),
-	DT_CLK(NULL, "sgx_ick", "sgx_ick"),
-	DT_CLK(NULL, "cpefuse_fck", "cpefuse_fck"),
-	DT_CLK(NULL, "ts_fck", "ts_fck"),
-	DT_CLK(NULL, "usbtll_fck", "usbtll_fck"),
-	DT_CLK(NULL, "usbtll_ick", "usbtll_ick"),
-	DT_CLK("omap_hsmmc.2", "ick", "mmchs3_ick"),
-	DT_CLK(NULL, "mmchs3_ick", "mmchs3_ick"),
-	DT_CLK(NULL, "mmchs3_fck", "mmchs3_fck"),
 	DT_CLK(NULL, "dss1_alwon_fck", "dss1_alwon_fck_3430es2"),
-	DT_CLK("omapdss_dss", "ick", "dss_ick_3430es2"),
 	DT_CLK(NULL, "dss_ick", "dss_ick_3430es2"),
-	DT_CLK(NULL, "usbhost_120m_fck", "usbhost_120m_fck"),
-	DT_CLK(NULL, "usbhost_48m_fck", "usbhost_48m_fck"),
-	DT_CLK(NULL, "usbhost_ick", "usbhost_ick"),
 	{ .node_name = NULL },
 };
 
 static struct ti_dt_clk am35xx_clks[] = {
-	DT_CLK(NULL, "ipss_ick", "ipss_ick"),
-	DT_CLK(NULL, "rmii_ck", "rmii_ck"),
-	DT_CLK(NULL, "pclk_ck", "pclk_ck"),
-	DT_CLK(NULL, "emac_ick", "emac_ick"),
-	DT_CLK(NULL, "emac_fck", "emac_fck"),
-	DT_CLK("davinci_emac.0", NULL, "emac_ick"),
-	DT_CLK("davinci_mdio.0", NULL, "emac_fck"),
-	DT_CLK("vpfe-capture", "master", "vpfe_ick"),
-	DT_CLK("vpfe-capture", "slave", "vpfe_fck"),
 	DT_CLK(NULL, "hsotgusb_ick", "hsotgusb_ick_am35xx"),
 	DT_CLK(NULL, "hsotgusb_fck", "hsotgusb_fck_am35xx"),
-	DT_CLK(NULL, "hecc_ck", "hecc_ck"),
 	DT_CLK(NULL, "uart4_ick", "uart4_ick_am35xx"),
 	DT_CLK(NULL, "uart4_fck", "uart4_fck_am35xx"),
 	{ .node_name = NULL },
 };
 
-static struct ti_dt_clk omap36xx_clks[] = {
-	DT_CLK(NULL, "omap_192m_alwon_fck", "omap_192m_alwon_fck"),
-	DT_CLK(NULL, "uart4_fck", "uart4_fck"),
-	DT_CLK(NULL, "uart4_ick", "uart4_ick"),
-	{ .node_name = NULL },
-};
-
 static const char *enable_init_clks[] = {
 	"sdrc_ick",
 	"gpmc_fck",
@@ -579,16 +326,10 @@ static int __init omap3xxx_dt_clk_init(int soc_type)
 	    soc_type == OMAP3_SOC_OMAP3630)
 		ti_dt_clocks_register(omap36xx_omap3430es2plus_clks);
 
-	if (soc_type == OMAP3_SOC_OMAP3430_ES1 ||
-	    soc_type == OMAP3_SOC_OMAP3430_ES2_PLUS ||
-	    soc_type == OMAP3_SOC_OMAP3630)
-		ti_dt_clocks_register(omap34xx_omap36xx_clks);
-
-	if (soc_type == OMAP3_SOC_OMAP3630)
-		ti_dt_clocks_register(omap36xx_clks);
-
 	omap2_clk_disable_autoidle_all();
 
+	ti_clk_add_aliases();
+
 	omap2_clk_enable_init_clocks(enable_init_clks,
 				     ARRAY_SIZE(enable_init_clks));
 
diff --git a/drivers/clk/ti/clk-43xx.c b/drivers/clk/ti/clk-43xx.c
index e816a7500e43..63c5ddb50187 100644
--- a/drivers/clk/ti/clk-43xx.c
+++ b/drivers/clk/ti/clk-43xx.c
@@ -19,109 +19,208 @@
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/clk/ti.h>
+#include <dt-bindings/clock/am4.h>
 
 #include "clock.h"
 
+static const char * const am4_synctimer_32kclk_parents[] __initconst = {
+	"mux_synctimer32k_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data am4_counter_32k_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_synctimer_32kclk_parents, NULL },
+	{ 0 },
+};
+
+static const char * const am4_gpio0_dbclk_parents[] __initconst = {
+	"gpio0_dbclk_mux_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data am4_gpio1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_gpio0_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am4_l4_wkup_clkctrl_regs[] __initconst = {
+	{ AM4_ADC_TSC_CLKCTRL, NULL, CLKF_SW_SUP, "adc_tsc_fck", "l3s_tsc_clkdm" },
+	{ AM4_L4_WKUP_CLKCTRL, NULL, CLKF_SW_SUP, "sys_clkin_ck", "l4_wkup_clkdm" },
+	{ AM4_WKUP_M3_CLKCTRL, NULL, CLKF_NO_IDLEST, "sys_clkin_ck" },
+	{ AM4_COUNTER_32K_CLKCTRL, am4_counter_32k_bit_data, CLKF_SW_SUP, "l4_wkup_cm:clk:0210:8" },
+	{ AM4_TIMER1_CLKCTRL, NULL, CLKF_SW_SUP, "timer1_fck", "l4_wkup_clkdm" },
+	{ AM4_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "wdt1_fck", "l4_wkup_clkdm" },
+	{ AM4_I2C1_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_wkupdm_ck", "l4_wkup_clkdm" },
+	{ AM4_UART1_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_wkupdm_ck", "l4_wkup_clkdm" },
+	{ AM4_SMARTREFLEX0_CLKCTRL, NULL, CLKF_SW_SUP, "smartreflex0_fck", "l4_wkup_clkdm" },
+	{ AM4_SMARTREFLEX1_CLKCTRL, NULL, CLKF_SW_SUP, "smartreflex1_fck", "l4_wkup_clkdm" },
+	{ AM4_CONTROL_CLKCTRL, NULL, CLKF_SW_SUP, "sys_clkin_ck", "l4_wkup_clkdm" },
+	{ AM4_GPIO1_CLKCTRL, am4_gpio1_bit_data, CLKF_SW_SUP, "sys_clkin_ck", "l4_wkup_clkdm" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am4_mpu_clkctrl_regs[] __initconst = {
+	{ AM4_MPU_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_mpu_m2_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am4_gfx_l3_clkctrl_regs[] __initconst = {
+	{ AM4_GFX_CLKCTRL, NULL, CLKF_SW_SUP, "gfx_fck_div_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am4_l4_rtc_clkctrl_regs[] __initconst = {
+	{ AM4_RTC_CLKCTRL, NULL, CLKF_SW_SUP, "clk_32768_ck" },
+	{ 0 },
+};
+
+static const char * const am4_usb_otg_ss0_refclk960m_parents[] __initconst = {
+	"dpll_per_clkdcoldo",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data am4_usb_otg_ss0_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_usb_otg_ss0_refclk960m_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data am4_usb_otg_ss1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_usb_otg_ss0_refclk960m_parents, NULL },
+	{ 0 },
+};
+
+static const char * const am4_gpio1_dbclk_parents[] __initconst = {
+	"clkdiv32k_ick",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data am4_gpio2_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data am4_gpio3_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data am4_gpio4_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data am4_gpio5_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data am4_gpio6_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, am4_gpio1_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data am4_l4_per_clkctrl_regs[] __initconst = {
+	{ AM4_L3_MAIN_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_AES_CLKCTRL, NULL, CLKF_SW_SUP, "aes0_fck", "l3_clkdm" },
+	{ AM4_DES_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_L3_INSTR_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_OCMCRAM_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_SHAM_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_VPFE0_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3s_clkdm" },
+	{ AM4_VPFE1_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3s_clkdm" },
+	{ AM4_TPCC_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_TPTC0_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_TPTC1_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_TPTC2_CLKCTRL, NULL, CLKF_SW_SUP, "l3_gclk", "l3_clkdm" },
+	{ AM4_L4_HS_CLKCTRL, NULL, CLKF_SW_SUP, "l4hs_gclk", "l3_clkdm" },
+	{ AM4_GPMC_CLKCTRL, NULL, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
+	{ AM4_MCASP0_CLKCTRL, NULL, CLKF_SW_SUP, "mcasp0_fck", "l3s_clkdm" },
+	{ AM4_MCASP1_CLKCTRL, NULL, CLKF_SW_SUP, "mcasp1_fck", "l3s_clkdm" },
+	{ AM4_MMC3_CLKCTRL, NULL, CLKF_SW_SUP, "mmc_clk", "l3s_clkdm" },
+	{ AM4_QSPI_CLKCTRL, NULL, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
+	{ AM4_USB_OTG_SS0_CLKCTRL, am4_usb_otg_ss0_bit_data, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
+	{ AM4_USB_OTG_SS1_CLKCTRL, am4_usb_otg_ss1_bit_data, CLKF_SW_SUP, "l3s_gclk", "l3s_clkdm" },
+	{ AM4_PRUSS_CLKCTRL, NULL, CLKF_SW_SUP, "pruss_ocp_gclk", "pruss_ocp_clkdm" },
+	{ AM4_L4_LS_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_D_CAN0_CLKCTRL, NULL, CLKF_SW_SUP, "dcan0_fck" },
+	{ AM4_D_CAN1_CLKCTRL, NULL, CLKF_SW_SUP, "dcan1_fck" },
+	{ AM4_EPWMSS0_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_EPWMSS1_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_EPWMSS2_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_EPWMSS3_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_EPWMSS4_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_EPWMSS5_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_ELM_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_GPIO2_CLKCTRL, am4_gpio2_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_GPIO3_CLKCTRL, am4_gpio3_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_GPIO4_CLKCTRL, am4_gpio4_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_GPIO5_CLKCTRL, am4_gpio5_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_GPIO6_CLKCTRL, am4_gpio6_bit_data, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_HDQ1W_CLKCTRL, NULL, CLKF_SW_SUP, "func_12m_clk" },
+	{ AM4_I2C2_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_I2C3_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_MAILBOX_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_MMC1_CLKCTRL, NULL, CLKF_SW_SUP, "mmc_clk" },
+	{ AM4_MMC2_CLKCTRL, NULL, CLKF_SW_SUP, "mmc_clk" },
+	{ AM4_RNG_CLKCTRL, NULL, CLKF_SW_SUP, "rng_fck" },
+	{ AM4_SPI0_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_SPI1_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_SPI2_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_SPI3_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_SPI4_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_SPINLOCK_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "timer2_fck" },
+	{ AM4_TIMER3_CLKCTRL, NULL, CLKF_SW_SUP, "timer3_fck" },
+	{ AM4_TIMER4_CLKCTRL, NULL, CLKF_SW_SUP, "timer4_fck" },
+	{ AM4_TIMER5_CLKCTRL, NULL, CLKF_SW_SUP, "timer5_fck" },
+	{ AM4_TIMER6_CLKCTRL, NULL, CLKF_SW_SUP, "timer6_fck" },
+	{ AM4_TIMER7_CLKCTRL, NULL, CLKF_SW_SUP, "timer7_fck" },
+	{ AM4_TIMER8_CLKCTRL, NULL, CLKF_SW_SUP, "timer8_fck" },
+	{ AM4_TIMER9_CLKCTRL, NULL, CLKF_SW_SUP, "timer9_fck" },
+	{ AM4_TIMER10_CLKCTRL, NULL, CLKF_SW_SUP, "timer10_fck" },
+	{ AM4_TIMER11_CLKCTRL, NULL, CLKF_SW_SUP, "timer11_fck" },
+	{ AM4_UART2_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_UART3_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_UART4_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_UART5_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_UART6_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_per_m2_div4_ck" },
+	{ AM4_OCP2SCP0_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_OCP2SCP1_CLKCTRL, NULL, CLKF_SW_SUP, "l4ls_gclk" },
+	{ AM4_EMIF_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_ddr_m2_ck", "emif_clkdm" },
+	{ AM4_DSS_CORE_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_SET_RATE_PARENT, "disp_clk", "dss_clkdm" },
+	{ AM4_CPGMAC0_CLKCTRL, NULL, CLKF_SW_SUP, "cpsw_125mhz_gclk", "cpsw_125mhz_clkdm" },
+	{ 0 },
+};
+
+const struct omap_clkctrl_data am4_clkctrl_data[] __initconst = {
+	{ 0x44df2820, am4_l4_wkup_clkctrl_regs },
+	{ 0x44df8320, am4_mpu_clkctrl_regs },
+	{ 0x44df8420, am4_gfx_l3_clkctrl_regs },
+	{ 0x44df8520, am4_l4_rtc_clkctrl_regs },
+	{ 0x44df8820, am4_l4_per_clkctrl_regs },
+	{ 0 },
+};
+
+const struct omap_clkctrl_data am438x_clkctrl_data[] __initconst = {
+	{ 0x44df2820, am4_l4_wkup_clkctrl_regs },
+	{ 0x44df8320, am4_mpu_clkctrl_regs },
+	{ 0x44df8420, am4_gfx_l3_clkctrl_regs },
+	{ 0x44df8820, am4_l4_per_clkctrl_regs },
+	{ 0 },
+};
+
 static struct ti_dt_clk am43xx_clks[] = {
-	DT_CLK(NULL, "clk_32768_ck", "clk_32768_ck"),
-	DT_CLK(NULL, "clk_rc32k_ck", "clk_rc32k_ck"),
-	DT_CLK(NULL, "virt_19200000_ck", "virt_19200000_ck"),
-	DT_CLK(NULL, "virt_24000000_ck", "virt_24000000_ck"),
-	DT_CLK(NULL, "virt_25000000_ck", "virt_25000000_ck"),
-	DT_CLK(NULL, "virt_26000000_ck", "virt_26000000_ck"),
-	DT_CLK(NULL, "sys_clkin_ck", "sys_clkin_ck"),
-	DT_CLK(NULL, "tclkin_ck", "tclkin_ck"),
-	DT_CLK(NULL, "dpll_core_ck", "dpll_core_ck"),
-	DT_CLK(NULL, "dpll_core_x2_ck", "dpll_core_x2_ck"),
-	DT_CLK(NULL, "dpll_core_m4_ck", "dpll_core_m4_ck"),
-	DT_CLK(NULL, "dpll_core_m5_ck", "dpll_core_m5_ck"),
-	DT_CLK(NULL, "dpll_core_m6_ck", "dpll_core_m6_ck"),
-	DT_CLK(NULL, "dpll_mpu_ck", "dpll_mpu_ck"),
-	DT_CLK(NULL, "dpll_mpu_m2_ck", "dpll_mpu_m2_ck"),
-	DT_CLK(NULL, "dpll_ddr_ck", "dpll_ddr_ck"),
-	DT_CLK(NULL, "dpll_ddr_m2_ck", "dpll_ddr_m2_ck"),
-	DT_CLK(NULL, "dpll_disp_ck", "dpll_disp_ck"),
-	DT_CLK(NULL, "dpll_disp_m2_ck", "dpll_disp_m2_ck"),
-	DT_CLK(NULL, "dpll_per_ck", "dpll_per_ck"),
-	DT_CLK(NULL, "dpll_per_m2_ck", "dpll_per_m2_ck"),
-	DT_CLK(NULL, "dpll_per_m2_div4_wkupdm_ck", "dpll_per_m2_div4_wkupdm_ck"),
-	DT_CLK(NULL, "dpll_per_m2_div4_ck", "dpll_per_m2_div4_ck"),
-	DT_CLK(NULL, "adc_tsc_fck", "adc_tsc_fck"),
-	DT_CLK(NULL, "clkdiv32k_ck", "clkdiv32k_ck"),
-	DT_CLK(NULL, "clkdiv32k_ick", "clkdiv32k_ick"),
-	DT_CLK(NULL, "dcan0_fck", "dcan0_fck"),
-	DT_CLK(NULL, "dcan1_fck", "dcan1_fck"),
-	DT_CLK(NULL, "pruss_ocp_gclk", "pruss_ocp_gclk"),
-	DT_CLK(NULL, "mcasp0_fck", "mcasp0_fck"),
-	DT_CLK(NULL, "mcasp1_fck", "mcasp1_fck"),
-	DT_CLK(NULL, "smartreflex0_fck", "smartreflex0_fck"),
-	DT_CLK(NULL, "smartreflex1_fck", "smartreflex1_fck"),
-	DT_CLK(NULL, "sha0_fck", "sha0_fck"),
-	DT_CLK(NULL, "aes0_fck", "aes0_fck"),
-	DT_CLK(NULL, "rng_fck", "rng_fck"),
-	DT_CLK(NULL, "timer1_fck", "timer1_fck"),
-	DT_CLK(NULL, "timer2_fck", "timer2_fck"),
-	DT_CLK(NULL, "timer3_fck", "timer3_fck"),
-	DT_CLK(NULL, "timer4_fck", "timer4_fck"),
-	DT_CLK(NULL, "timer5_fck", "timer5_fck"),
-	DT_CLK(NULL, "timer6_fck", "timer6_fck"),
-	DT_CLK(NULL, "timer7_fck", "timer7_fck"),
-	DT_CLK(NULL, "wdt1_fck", "wdt1_fck"),
-	DT_CLK(NULL, "l3_gclk", "l3_gclk"),
-	DT_CLK(NULL, "dpll_core_m4_div2_ck", "dpll_core_m4_div2_ck"),
-	DT_CLK(NULL, "l4hs_gclk", "l4hs_gclk"),
-	DT_CLK(NULL, "l3s_gclk", "l3s_gclk"),
-	DT_CLK(NULL, "l4ls_gclk", "l4ls_gclk"),
-	DT_CLK(NULL, "clk_24mhz", "clk_24mhz"),
-	DT_CLK(NULL, "cpsw_125mhz_gclk", "cpsw_125mhz_gclk"),
-	DT_CLK(NULL, "cpsw_cpts_rft_clk", "cpsw_cpts_rft_clk"),
-	DT_CLK(NULL, "dpll_clksel_mac_clk", "dpll_clksel_mac_clk"),
-	DT_CLK(NULL, "gpio0_dbclk_mux_ck", "gpio0_dbclk_mux_ck"),
-	DT_CLK(NULL, "gpio0_dbclk", "gpio0_dbclk"),
-	DT_CLK(NULL, "gpio1_dbclk", "gpio1_dbclk"),
-	DT_CLK(NULL, "gpio2_dbclk", "gpio2_dbclk"),
-	DT_CLK(NULL, "gpio3_dbclk", "gpio3_dbclk"),
-	DT_CLK(NULL, "gpio4_dbclk", "gpio4_dbclk"),
-	DT_CLK(NULL, "gpio5_dbclk", "gpio5_dbclk"),
-	DT_CLK(NULL, "mmc_clk", "mmc_clk"),
-	DT_CLK(NULL, "gfx_fclk_clksel_ck", "gfx_fclk_clksel_ck"),
-	DT_CLK(NULL, "gfx_fck_div_ck", "gfx_fck_div_ck"),
 	DT_CLK(NULL, "timer_32k_ck", "clkdiv32k_ick"),
 	DT_CLK(NULL, "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK(NULL, "sysclk_div", "sysclk_div"),
-	DT_CLK(NULL, "disp_clk", "disp_clk"),
-	DT_CLK(NULL, "clk_32k_mosc_ck", "clk_32k_mosc_ck"),
-	DT_CLK(NULL, "clk_32k_tpm_ck", "clk_32k_tpm_ck"),
-	DT_CLK(NULL, "dpll_extdev_ck", "dpll_extdev_ck"),
-	DT_CLK(NULL, "dpll_extdev_m2_ck", "dpll_extdev_m2_ck"),
-	DT_CLK(NULL, "mux_synctimer32k_ck", "mux_synctimer32k_ck"),
-	DT_CLK(NULL, "synctimer_32kclk", "synctimer_32kclk"),
-	DT_CLK(NULL, "timer8_fck", "timer8_fck"),
-	DT_CLK(NULL, "timer9_fck", "timer9_fck"),
-	DT_CLK(NULL, "timer10_fck", "timer10_fck"),
-	DT_CLK(NULL, "timer11_fck", "timer11_fck"),
-	DT_CLK(NULL, "cpsw_50m_clkdiv", "cpsw_50m_clkdiv"),
-	DT_CLK(NULL, "cpsw_5m_clkdiv", "cpsw_5m_clkdiv"),
-	DT_CLK(NULL, "dpll_ddr_x2_ck", "dpll_ddr_x2_ck"),
-	DT_CLK(NULL, "dpll_ddr_m4_ck", "dpll_ddr_m4_ck"),
-	DT_CLK(NULL, "dpll_per_clkdcoldo", "dpll_per_clkdcoldo"),
-	DT_CLK(NULL, "dll_aging_clk_div", "dll_aging_clk_div"),
-	DT_CLK(NULL, "div_core_25m_ck", "div_core_25m_ck"),
-	DT_CLK(NULL, "func_12m_clk", "func_12m_clk"),
-	DT_CLK(NULL, "vtp_clk_div", "vtp_clk_div"),
-	DT_CLK(NULL, "usbphy_32khz_clkmux", "usbphy_32khz_clkmux"),
-	DT_CLK("48300200.ehrpwm", "tbclk", "ehrpwm0_tbclk"),
-	DT_CLK("48302200.ehrpwm", "tbclk", "ehrpwm1_tbclk"),
-	DT_CLK("48304200.ehrpwm", "tbclk", "ehrpwm2_tbclk"),
-	DT_CLK("48306200.ehrpwm", "tbclk", "ehrpwm3_tbclk"),
-	DT_CLK("48308200.ehrpwm", "tbclk", "ehrpwm4_tbclk"),
-	DT_CLK("4830a200.ehrpwm", "tbclk", "ehrpwm5_tbclk"),
-	DT_CLK("48300200.pwm", "tbclk", "ehrpwm0_tbclk"),
-	DT_CLK("48302200.pwm", "tbclk", "ehrpwm1_tbclk"),
-	DT_CLK("48304200.pwm", "tbclk", "ehrpwm2_tbclk"),
-	DT_CLK("48306200.pwm", "tbclk", "ehrpwm3_tbclk"),
-	DT_CLK("48308200.pwm", "tbclk", "ehrpwm4_tbclk"),
-	DT_CLK("4830a200.pwm", "tbclk", "ehrpwm5_tbclk"),
+	DT_CLK(NULL, "gpio0_dbclk", "l4_wkup_cm:0348:8"),
+	DT_CLK(NULL, "gpio1_dbclk", "l4_per_cm:0458:8"),
+	DT_CLK(NULL, "gpio2_dbclk", "l4_per_cm:0460:8"),
+	DT_CLK(NULL, "gpio3_dbclk", "l4_per_cm:0468:8"),
+	DT_CLK(NULL, "gpio4_dbclk", "l4_per_cm:0470:8"),
+	DT_CLK(NULL, "gpio5_dbclk", "l4_per_cm:0478:8"),
+	DT_CLK(NULL, "synctimer_32kclk", "l4_wkup_cm:0210:8"),
+	DT_CLK(NULL, "usb_otg_ss0_refclk960m", "l4_per_cm:0240:8"),
+	DT_CLK(NULL, "usb_otg_ss1_refclk960m", "l4_per_cm:0248:8"),
 	{ .node_name = NULL },
 };
 
@@ -133,6 +232,8 @@ int __init am43xx_dt_clk_init(void)
 
 	omap2_clk_disable_autoidle_all();
 
+	ti_clk_add_aliases();
+
 	/*
 	 * cpsw_cpts_rft_clk  has got the choice of 3 clocksources
 	 * dpll_core_m4_ck, dpll_core_m5_ck and dpll_disp_m2_ck.
diff --git a/drivers/clk/ti/clk-44xx.c b/drivers/clk/ti/clk-44xx.c
index 2005f032c02f..339d30d64ebb 100644
--- a/drivers/clk/ti/clk-44xx.c
+++ b/drivers/clk/ti/clk-44xx.c
@@ -35,7 +35,7 @@
 #define OMAP4_DPLL_USB_DEFFREQ				960000000
 
 static const struct omap_clkctrl_reg_data omap4_mpuss_clkctrl_regs[] __initconst = {
-	{ OMAP4_MPU_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_mpu_m2_ck" },
+	{ OMAP4_MPU_CLKCTRL, NULL, 0, "dpll_mpu_m2_ck" },
 	{ 0 },
 };
 
@@ -59,7 +59,7 @@ static const struct omap_clkctrl_bit_data omap4_aess_bit_data[] __initconst = {
 };
 
 static const char * const omap4_func_dmic_abe_gfclk_parents[] __initconst = {
-	"dmic_sync_mux_ck",
+	"abe_cm:clk:0018:26",
 	"pad_clks_ck",
 	"slimbus_clk",
 	NULL,
@@ -79,7 +79,7 @@ static const struct omap_clkctrl_bit_data omap4_dmic_bit_data[] __initconst = {
 };
 
 static const char * const omap4_func_mcasp_abe_gfclk_parents[] __initconst = {
-	"mcasp_sync_mux_ck",
+	"abe_cm:clk:0020:26",
 	"pad_clks_ck",
 	"slimbus_clk",
 	NULL,
@@ -92,7 +92,7 @@ static const struct omap_clkctrl_bit_data omap4_mcasp_bit_data[] __initconst = {
 };
 
 static const char * const omap4_func_mcbsp1_gfclk_parents[] __initconst = {
-	"mcbsp1_sync_mux_ck",
+	"abe_cm:clk:0028:26",
 	"pad_clks_ck",
 	"slimbus_clk",
 	NULL,
@@ -105,7 +105,7 @@ static const struct omap_clkctrl_bit_data omap4_mcbsp1_bit_data[] __initconst =
 };
 
 static const char * const omap4_func_mcbsp2_gfclk_parents[] __initconst = {
-	"mcbsp2_sync_mux_ck",
+	"abe_cm:clk:0030:26",
 	"pad_clks_ck",
 	"slimbus_clk",
 	NULL,
@@ -118,7 +118,7 @@ static const struct omap_clkctrl_bit_data omap4_mcbsp2_bit_data[] __initconst =
 };
 
 static const char * const omap4_func_mcbsp3_gfclk_parents[] __initconst = {
-	"mcbsp3_sync_mux_ck",
+	"abe_cm:clk:0038:26",
 	"pad_clks_ck",
 	"slimbus_clk",
 	NULL,
@@ -186,18 +186,18 @@ static const struct omap_clkctrl_bit_data omap4_timer8_bit_data[] __initconst =
 
 static const struct omap_clkctrl_reg_data omap4_abe_clkctrl_regs[] __initconst = {
 	{ OMAP4_L4_ABE_CLKCTRL, NULL, 0, "ocp_abe_iclk" },
-	{ OMAP4_AESS_CLKCTRL, omap4_aess_bit_data, CLKF_SW_SUP, "aess_fclk" },
+	{ OMAP4_AESS_CLKCTRL, omap4_aess_bit_data, CLKF_SW_SUP, "abe_cm:clk:0008:24" },
 	{ OMAP4_MCPDM_CLKCTRL, NULL, CLKF_SW_SUP, "pad_clks_ck" },
-	{ OMAP4_DMIC_CLKCTRL, omap4_dmic_bit_data, CLKF_SW_SUP, "func_dmic_abe_gfclk" },
-	{ OMAP4_MCASP_CLKCTRL, omap4_mcasp_bit_data, CLKF_SW_SUP, "func_mcasp_abe_gfclk" },
-	{ OMAP4_MCBSP1_CLKCTRL, omap4_mcbsp1_bit_data, CLKF_SW_SUP, "func_mcbsp1_gfclk" },
-	{ OMAP4_MCBSP2_CLKCTRL, omap4_mcbsp2_bit_data, CLKF_SW_SUP, "func_mcbsp2_gfclk" },
-	{ OMAP4_MCBSP3_CLKCTRL, omap4_mcbsp3_bit_data, CLKF_SW_SUP, "func_mcbsp3_gfclk" },
-	{ OMAP4_SLIMBUS1_CLKCTRL, omap4_slimbus1_bit_data, CLKF_SW_SUP, "slimbus1_fclk_0" },
-	{ OMAP4_TIMER5_CLKCTRL, omap4_timer5_bit_data, CLKF_SW_SUP, "timer5_sync_mux" },
-	{ OMAP4_TIMER6_CLKCTRL, omap4_timer6_bit_data, CLKF_SW_SUP, "timer6_sync_mux" },
-	{ OMAP4_TIMER7_CLKCTRL, omap4_timer7_bit_data, CLKF_SW_SUP, "timer7_sync_mux" },
-	{ OMAP4_TIMER8_CLKCTRL, omap4_timer8_bit_data, CLKF_SW_SUP, "timer8_sync_mux" },
+	{ OMAP4_DMIC_CLKCTRL, omap4_dmic_bit_data, CLKF_SW_SUP, "abe_cm:clk:0018:24" },
+	{ OMAP4_MCASP_CLKCTRL, omap4_mcasp_bit_data, CLKF_SW_SUP, "abe_cm:clk:0020:24" },
+	{ OMAP4_MCBSP1_CLKCTRL, omap4_mcbsp1_bit_data, CLKF_SW_SUP, "abe_cm:clk:0028:24" },
+	{ OMAP4_MCBSP2_CLKCTRL, omap4_mcbsp2_bit_data, CLKF_SW_SUP, "abe_cm:clk:0030:24" },
+	{ OMAP4_MCBSP3_CLKCTRL, omap4_mcbsp3_bit_data, CLKF_SW_SUP, "abe_cm:clk:0038:24" },
+	{ OMAP4_SLIMBUS1_CLKCTRL, omap4_slimbus1_bit_data, CLKF_SW_SUP, "abe_cm:clk:0040:8" },
+	{ OMAP4_TIMER5_CLKCTRL, omap4_timer5_bit_data, CLKF_SW_SUP, "abe_cm:clk:0048:24" },
+	{ OMAP4_TIMER6_CLKCTRL, omap4_timer6_bit_data, CLKF_SW_SUP, "abe_cm:clk:0050:24" },
+	{ OMAP4_TIMER7_CLKCTRL, omap4_timer7_bit_data, CLKF_SW_SUP, "abe_cm:clk:0058:24" },
+	{ OMAP4_TIMER8_CLKCTRL, omap4_timer8_bit_data, CLKF_SW_SUP, "abe_cm:clk:0060:24" },
 	{ OMAP4_WD_TIMER3_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
 	{ 0 },
 };
@@ -280,6 +280,7 @@ static const char * const omap4_fdif_fck_parents[] __initconst = {
 
 static const struct omap_clkctrl_div_data omap4_fdif_fck_data __initconst = {
 	.max_div = 4,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
 };
 
 static const struct omap_clkctrl_bit_data omap4_fdif_bit_data[] __initconst = {
@@ -289,7 +290,7 @@ static const struct omap_clkctrl_bit_data omap4_fdif_bit_data[] __initconst = {
 
 static const struct omap_clkctrl_reg_data omap4_iss_clkctrl_regs[] __initconst = {
 	{ OMAP4_ISS_CLKCTRL, omap4_iss_bit_data, CLKF_SW_SUP, "ducati_clk_mux_ck" },
-	{ OMAP4_FDIF_CLKCTRL, omap4_fdif_bit_data, CLKF_SW_SUP, "fdif_fck" },
+	{ OMAP4_FDIF_CLKCTRL, omap4_fdif_bit_data, CLKF_SW_SUP, "iss_cm:clk:0008:24" },
 	{ 0 },
 };
 
@@ -322,7 +323,7 @@ static const struct omap_clkctrl_bit_data omap4_dss_core_bit_data[] __initconst
 };
 
 static const struct omap_clkctrl_reg_data omap4_l3_dss_clkctrl_regs[] __initconst = {
-	{ OMAP4_DSS_CORE_CLKCTRL, omap4_dss_core_bit_data, CLKF_SW_SUP, "dss_dss_clk" },
+	{ OMAP4_DSS_CORE_CLKCTRL, omap4_dss_core_bit_data, CLKF_SW_SUP, "l3_dss_cm:clk:0000:8" },
 	{ 0 },
 };
 
@@ -338,7 +339,7 @@ static const struct omap_clkctrl_bit_data omap4_gpu_bit_data[] __initconst = {
 };
 
 static const struct omap_clkctrl_reg_data omap4_l3_gfx_clkctrl_regs[] __initconst = {
-	{ OMAP4_GPU_CLKCTRL, omap4_gpu_bit_data, CLKF_SW_SUP, "sgx_clk_mux" },
+	{ OMAP4_GPU_CLKCTRL, omap4_gpu_bit_data, CLKF_SW_SUP, "l3_gfx_cm:clk:0000:24" },
 	{ 0 },
 };
 
@@ -365,6 +366,7 @@ static const char * const omap4_hsi_fck_parents[] __initconst = {
 
 static const struct omap_clkctrl_div_data omap4_hsi_fck_data __initconst = {
 	.max_div = 4,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
 };
 
 static const struct omap_clkctrl_bit_data omap4_hsi_bit_data[] __initconst = {
@@ -373,12 +375,12 @@ static const struct omap_clkctrl_bit_data omap4_hsi_bit_data[] __initconst = {
 };
 
 static const char * const omap4_usb_host_hs_utmi_p1_clk_parents[] __initconst = {
-	"utmi_p1_gfclk",
+	"l3_init_cm:clk:0038:24",
 	NULL,
 };
 
 static const char * const omap4_usb_host_hs_utmi_p2_clk_parents[] __initconst = {
-	"utmi_p2_gfclk",
+	"l3_init_cm:clk:0038:25",
 	NULL,
 };
 
@@ -419,7 +421,7 @@ static const struct omap_clkctrl_bit_data omap4_usb_host_hs_bit_data[] __initcon
 };
 
 static const char * const omap4_usb_otg_hs_xclk_parents[] __initconst = {
-	"otg_60m_gfclk",
+	"l3_init_cm:clk:0040:24",
 	NULL,
 };
 
@@ -453,14 +455,14 @@ static const struct omap_clkctrl_bit_data omap4_ocp2scp_usb_phy_bit_data[] __ini
 };
 
 static const struct omap_clkctrl_reg_data omap4_l3_init_clkctrl_regs[] __initconst = {
-	{ OMAP4_MMC1_CLKCTRL, omap4_mmc1_bit_data, CLKF_SW_SUP, "hsmmc1_fclk" },
-	{ OMAP4_MMC2_CLKCTRL, omap4_mmc2_bit_data, CLKF_SW_SUP, "hsmmc2_fclk" },
-	{ OMAP4_HSI_CLKCTRL, omap4_hsi_bit_data, CLKF_HW_SUP, "hsi_fck" },
+	{ OMAP4_MMC1_CLKCTRL, omap4_mmc1_bit_data, CLKF_SW_SUP, "l3_init_cm:clk:0008:24" },
+	{ OMAP4_MMC2_CLKCTRL, omap4_mmc2_bit_data, CLKF_SW_SUP, "l3_init_cm:clk:0010:24" },
+	{ OMAP4_HSI_CLKCTRL, omap4_hsi_bit_data, CLKF_HW_SUP, "l3_init_cm:clk:0018:24" },
 	{ OMAP4_USB_HOST_HS_CLKCTRL, omap4_usb_host_hs_bit_data, CLKF_SW_SUP, "init_60m_fclk" },
 	{ OMAP4_USB_OTG_HS_CLKCTRL, omap4_usb_otg_hs_bit_data, CLKF_HW_SUP, "l3_div_ck" },
 	{ OMAP4_USB_TLL_HS_CLKCTRL, omap4_usb_tll_hs_bit_data, CLKF_HW_SUP, "l4_div_ck" },
 	{ OMAP4_USB_HOST_FS_CLKCTRL, NULL, CLKF_SW_SUP, "func_48mc_fclk" },
-	{ OMAP4_OCP2SCP_USB_PHY_CLKCTRL, omap4_ocp2scp_usb_phy_bit_data, CLKF_HW_SUP, "ocp2scp_usb_phy_phy_48m" },
+	{ OMAP4_OCP2SCP_USB_PHY_CLKCTRL, omap4_ocp2scp_usb_phy_bit_data, CLKF_HW_SUP, "l3_init_cm:clk:00c0:8" },
 	{ 0 },
 };
 
@@ -531,7 +533,7 @@ static const struct omap_clkctrl_bit_data omap4_gpio6_bit_data[] __initconst = {
 };
 
 static const char * const omap4_per_mcbsp4_gfclk_parents[] __initconst = {
-	"mcbsp4_sync_mux_ck",
+	"l4_per_cm:clk:00c0:26",
 	"pad_clks_ck",
 	NULL,
 };
@@ -544,7 +546,7 @@ static const char * const omap4_mcbsp4_sync_mux_ck_parents[] __initconst = {
 
 static const struct omap_clkctrl_bit_data omap4_mcbsp4_bit_data[] __initconst = {
 	{ 24, TI_CLK_MUX, omap4_per_mcbsp4_gfclk_parents, NULL },
-	{ 25, TI_CLK_MUX, omap4_mcbsp4_sync_mux_ck_parents, NULL },
+	{ 26, TI_CLK_MUX, omap4_mcbsp4_sync_mux_ck_parents, NULL },
 	{ 0 },
 };
 
@@ -571,12 +573,12 @@ static const struct omap_clkctrl_bit_data omap4_slimbus2_bit_data[] __initconst
 };
 
 static const struct omap_clkctrl_reg_data omap4_l4_per_clkctrl_regs[] __initconst = {
-	{ OMAP4_TIMER10_CLKCTRL, omap4_timer10_bit_data, CLKF_SW_SUP, "cm2_dm10_mux" },
-	{ OMAP4_TIMER11_CLKCTRL, omap4_timer11_bit_data, CLKF_SW_SUP, "cm2_dm11_mux" },
-	{ OMAP4_TIMER2_CLKCTRL, omap4_timer2_bit_data, CLKF_SW_SUP, "cm2_dm2_mux" },
-	{ OMAP4_TIMER3_CLKCTRL, omap4_timer3_bit_data, CLKF_SW_SUP, "cm2_dm3_mux" },
-	{ OMAP4_TIMER4_CLKCTRL, omap4_timer4_bit_data, CLKF_SW_SUP, "cm2_dm4_mux" },
-	{ OMAP4_TIMER9_CLKCTRL, omap4_timer9_bit_data, CLKF_SW_SUP, "cm2_dm9_mux" },
+	{ OMAP4_TIMER10_CLKCTRL, omap4_timer10_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:0008:24" },
+	{ OMAP4_TIMER11_CLKCTRL, omap4_timer11_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:0010:24" },
+	{ OMAP4_TIMER2_CLKCTRL, omap4_timer2_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:0018:24" },
+	{ OMAP4_TIMER3_CLKCTRL, omap4_timer3_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:0020:24" },
+	{ OMAP4_TIMER4_CLKCTRL, omap4_timer4_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:0028:24" },
+	{ OMAP4_TIMER9_CLKCTRL, omap4_timer9_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:0030:24" },
 	{ OMAP4_ELM_CLKCTRL, NULL, 0, "l4_div_ck" },
 	{ OMAP4_GPIO2_CLKCTRL, omap4_gpio2_bit_data, CLKF_HW_SUP, "l4_div_ck" },
 	{ OMAP4_GPIO3_CLKCTRL, omap4_gpio3_bit_data, CLKF_HW_SUP, "l4_div_ck" },
@@ -589,14 +591,14 @@ static const struct omap_clkctrl_reg_data omap4_l4_per_clkctrl_regs[] __initcons
 	{ OMAP4_I2C3_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
 	{ OMAP4_I2C4_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
 	{ OMAP4_L4_PER_CLKCTRL, NULL, 0, "l4_div_ck" },
-	{ OMAP4_MCBSP4_CLKCTRL, omap4_mcbsp4_bit_data, CLKF_SW_SUP, "per_mcbsp4_gfclk" },
+	{ OMAP4_MCBSP4_CLKCTRL, omap4_mcbsp4_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:00c0:24" },
 	{ OMAP4_MCSPI1_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
 	{ OMAP4_MCSPI2_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
 	{ OMAP4_MCSPI3_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
 	{ OMAP4_MCSPI4_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
 	{ OMAP4_MMC3_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
 	{ OMAP4_MMC4_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
-	{ OMAP4_SLIMBUS2_CLKCTRL, omap4_slimbus2_bit_data, CLKF_SW_SUP, "slimbus2_fclk_0" },
+	{ OMAP4_SLIMBUS2_CLKCTRL, omap4_slimbus2_bit_data, CLKF_SW_SUP, "l4_per_cm:clk:0118:8" },
 	{ OMAP4_UART1_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
 	{ OMAP4_UART2_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
 	{ OMAP4_UART3_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
@@ -619,7 +621,7 @@ static const struct omap_clkctrl_reg_data omap4_l4_wkup_clkctrl_regs[] __initcon
 	{ OMAP4_L4_WKUP_CLKCTRL, NULL, 0, "l4_wkup_clk_mux_ck" },
 	{ OMAP4_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
 	{ OMAP4_GPIO1_CLKCTRL, omap4_gpio1_bit_data, CLKF_HW_SUP, "l4_wkup_clk_mux_ck" },
-	{ OMAP4_TIMER1_CLKCTRL, omap4_timer1_bit_data, CLKF_SW_SUP, "dmt1_clk_mux" },
+	{ OMAP4_TIMER1_CLKCTRL, omap4_timer1_bit_data, CLKF_SW_SUP, "l4_wkup_cm:clk:0020:24" },
 	{ OMAP4_COUNTER_32K_CLKCTRL, NULL, 0, "sys_32k_ck" },
 	{ OMAP4_KBD_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
 	{ 0 },
@@ -633,7 +635,7 @@ static const char * const omap4_pmd_stm_clock_mux_ck_parents[] __initconst = {
 };
 
 static const char * const omap4_trace_clk_div_div_ck_parents[] __initconst = {
-	"pmd_trace_clk_mux_ck",
+	"emu_sys_cm:clk:0000:22",
 	NULL,
 };
 
@@ -651,12 +653,13 @@ static const struct omap_clkctrl_div_data omap4_trace_clk_div_div_ck_data __init
 };
 
 static const char * const omap4_stm_clk_div_ck_parents[] __initconst = {
-	"pmd_stm_clock_mux_ck",
+	"emu_sys_cm:clk:0000:20",
 	NULL,
 };
 
 static const struct omap_clkctrl_div_data omap4_stm_clk_div_ck_data __initconst = {
 	.max_div = 64,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
 };
 
 static const struct omap_clkctrl_bit_data omap4_debugss_bit_data[] __initconst = {
@@ -697,52 +700,79 @@ const struct omap_clkctrl_data omap4_clkctrl_data[] __initconst = {
 };
 
 static struct ti_dt_clk omap44xx_clks[] = {
-	DT_CLK("smp_twd", NULL, "mpu_periphclk"),
-	DT_CLK("omapdss_dss", "ick", "dss_fck"),
-	DT_CLK("usbhs_omap", "fs_fck", "usb_host_fs_fck"),
-	DT_CLK("usbhs_omap", "hs_fck", "usb_host_hs_fck"),
-	DT_CLK("musb-omap2430", "ick", "usb_otg_hs_ick"),
-	DT_CLK("usbhs_omap", "usbtll_ick", "usb_tll_hs_ick"),
-	DT_CLK("usbhs_tll", "usbtll_ick", "usb_tll_hs_ick"),
-	DT_CLK("omap_i2c.1", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.2", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.3", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.4", "ick", "dummy_ck"),
-	DT_CLK(NULL, "mailboxes_ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.0", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.1", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.2", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.3", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.4", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.1", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.2", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.3", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.4", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.1", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.2", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.3", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.4", "ick", "dummy_ck"),
-	DT_CLK(NULL, "uart1_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart2_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart3_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart4_ick", "dummy_ck"),
-	DT_CLK("usbhs_omap", "usbhost_ick", "dummy_ck"),
-	DT_CLK("usbhs_omap", "usbtll_fck", "dummy_ck"),
-	DT_CLK("usbhs_tll", "usbtll_fck", "dummy_ck"),
-	DT_CLK("omap_wdt", "ick", "dummy_ck"),
 	DT_CLK(NULL, "timer_32k_ck", "sys_32k_ck"),
-	DT_CLK("4a318000.timer", "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK("48032000.timer", "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK("48034000.timer", "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK("48036000.timer", "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK("4803e000.timer", "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK("48086000.timer", "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK("48088000.timer", "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK("40138000.timer", "timer_sys_ck", "syc_clk_div_ck"),
-	DT_CLK("4013a000.timer", "timer_sys_ck", "syc_clk_div_ck"),
-	DT_CLK("4013c000.timer", "timer_sys_ck", "syc_clk_div_ck"),
-	DT_CLK("4013e000.timer", "timer_sys_ck", "syc_clk_div_ck"),
-	DT_CLK(NULL, "cpufreq_ck", "dpll_mpu_ck"),
+	/*
+	 * XXX: All the clock aliases below are only needed for legacy
+	 * hwmod support. Once hwmod is removed, these can be removed
+	 * also.
+	 */
+	DT_CLK(NULL, "aess_fclk", "abe_cm:0008:24"),
+	DT_CLK(NULL, "cm2_dm10_mux", "l4_per_cm:0008:24"),
+	DT_CLK(NULL, "cm2_dm11_mux", "l4_per_cm:0010:24"),
+	DT_CLK(NULL, "cm2_dm2_mux", "l4_per_cm:0018:24"),
+	DT_CLK(NULL, "cm2_dm3_mux", "l4_per_cm:0020:24"),
+	DT_CLK(NULL, "cm2_dm4_mux", "l4_per_cm:0028:24"),
+	DT_CLK(NULL, "cm2_dm9_mux", "l4_per_cm:0030:24"),
+	DT_CLK(NULL, "dmic_sync_mux_ck", "abe_cm:0018:26"),
+	DT_CLK(NULL, "dmt1_clk_mux", "l4_wkup_cm:0020:24"),
+	DT_CLK(NULL, "dss_48mhz_clk", "l3_dss_cm:0000:9"),
+	DT_CLK(NULL, "dss_dss_clk", "l3_dss_cm:0000:8"),
+	DT_CLK(NULL, "dss_sys_clk", "l3_dss_cm:0000:10"),
+	DT_CLK(NULL, "dss_tv_clk", "l3_dss_cm:0000:11"),
+	DT_CLK(NULL, "fdif_fck", "iss_cm:0008:24"),
+	DT_CLK(NULL, "func_dmic_abe_gfclk", "abe_cm:0018:24"),
+	DT_CLK(NULL, "func_mcasp_abe_gfclk", "abe_cm:0020:24"),
+	DT_CLK(NULL, "func_mcbsp1_gfclk", "abe_cm:0028:24"),
+	DT_CLK(NULL, "func_mcbsp2_gfclk", "abe_cm:0030:24"),
+	DT_CLK(NULL, "func_mcbsp3_gfclk", "abe_cm:0038:24"),
+	DT_CLK(NULL, "gpio1_dbclk", "l4_wkup_cm:0018:8"),
+	DT_CLK(NULL, "gpio2_dbclk", "l4_per_cm:0040:8"),
+	DT_CLK(NULL, "gpio3_dbclk", "l4_per_cm:0048:8"),
+	DT_CLK(NULL, "gpio4_dbclk", "l4_per_cm:0050:8"),
+	DT_CLK(NULL, "gpio5_dbclk", "l4_per_cm:0058:8"),
+	DT_CLK(NULL, "gpio6_dbclk", "l4_per_cm:0060:8"),
+	DT_CLK(NULL, "hsi_fck", "l3_init_cm:0018:24"),
+	DT_CLK(NULL, "hsmmc1_fclk", "l3_init_cm:0008:24"),
+	DT_CLK(NULL, "hsmmc2_fclk", "l3_init_cm:0010:24"),
+	DT_CLK(NULL, "iss_ctrlclk", "iss_cm:0000:8"),
+	DT_CLK(NULL, "mcasp_sync_mux_ck", "abe_cm:0020:26"),
+	DT_CLK(NULL, "mcbsp1_sync_mux_ck", "abe_cm:0028:26"),
+	DT_CLK(NULL, "mcbsp2_sync_mux_ck", "abe_cm:0030:26"),
+	DT_CLK(NULL, "mcbsp3_sync_mux_ck", "abe_cm:0038:26"),
+	DT_CLK(NULL, "mcbsp4_sync_mux_ck", "l4_per_cm:00c0:26"),
+	DT_CLK(NULL, "ocp2scp_usb_phy_phy_48m", "l3_init_cm:00c0:8"),
+	DT_CLK(NULL, "otg_60m_gfclk", "l3_init_cm:0040:24"),
+	DT_CLK(NULL, "per_mcbsp4_gfclk", "l4_per_cm:00c0:24"),
+	DT_CLK(NULL, "pmd_stm_clock_mux_ck", "emu_sys_cm:0000:20"),
+	DT_CLK(NULL, "pmd_trace_clk_mux_ck", "emu_sys_cm:0000:22"),
+	DT_CLK(NULL, "sgx_clk_mux", "l3_gfx_cm:0000:24"),
+	DT_CLK(NULL, "slimbus1_fclk_0", "abe_cm:0040:8"),
+	DT_CLK(NULL, "slimbus1_fclk_1", "abe_cm:0040:9"),
+	DT_CLK(NULL, "slimbus1_fclk_2", "abe_cm:0040:10"),
+	DT_CLK(NULL, "slimbus1_slimbus_clk", "abe_cm:0040:11"),
+	DT_CLK(NULL, "slimbus2_fclk_0", "l4_per_cm:0118:8"),
+	DT_CLK(NULL, "slimbus2_fclk_1", "l4_per_cm:0118:9"),
+	DT_CLK(NULL, "slimbus2_slimbus_clk", "l4_per_cm:0118:10"),
+	DT_CLK(NULL, "stm_clk_div_ck", "emu_sys_cm:0000:27"),
+	DT_CLK(NULL, "timer5_sync_mux", "abe_cm:0048:24"),
+	DT_CLK(NULL, "timer6_sync_mux", "abe_cm:0050:24"),
+	DT_CLK(NULL, "timer7_sync_mux", "abe_cm:0058:24"),
+	DT_CLK(NULL, "timer8_sync_mux", "abe_cm:0060:24"),
+	DT_CLK(NULL, "trace_clk_div_div_ck", "emu_sys_cm:0000:24"),
+	DT_CLK(NULL, "usb_host_hs_func48mclk", "l3_init_cm:0038:15"),
+	DT_CLK(NULL, "usb_host_hs_hsic480m_p1_clk", "l3_init_cm:0038:13"),
+	DT_CLK(NULL, "usb_host_hs_hsic480m_p2_clk", "l3_init_cm:0038:14"),
+	DT_CLK(NULL, "usb_host_hs_hsic60m_p1_clk", "l3_init_cm:0038:11"),
+	DT_CLK(NULL, "usb_host_hs_hsic60m_p2_clk", "l3_init_cm:0038:12"),
+	DT_CLK(NULL, "usb_host_hs_utmi_p1_clk", "l3_init_cm:0038:8"),
+	DT_CLK(NULL, "usb_host_hs_utmi_p2_clk", "l3_init_cm:0038:9"),
+	DT_CLK(NULL, "usb_host_hs_utmi_p3_clk", "l3_init_cm:0038:10"),
+	DT_CLK(NULL, "usb_otg_hs_xclk", "l3_init_cm:0040:8"),
+	DT_CLK(NULL, "usb_tll_hs_usb_ch0_clk", "l3_init_cm:0048:8"),
+	DT_CLK(NULL, "usb_tll_hs_usb_ch1_clk", "l3_init_cm:0048:9"),
+	DT_CLK(NULL, "usb_tll_hs_usb_ch2_clk", "l3_init_cm:0048:10"),
+	DT_CLK(NULL, "utmi_p1_gfclk", "l3_init_cm:0038:24"),
+	DT_CLK(NULL, "utmi_p2_gfclk", "l3_init_cm:0038:25"),
 	{ .node_name = NULL },
 };
 
diff --git a/drivers/clk/ti/clk-54xx.c b/drivers/clk/ti/clk-54xx.c
index 294bc03ec067..a17b0c4646a1 100644
--- a/drivers/clk/ti/clk-54xx.c
+++ b/drivers/clk/ti/clk-54xx.c
@@ -16,6 +16,7 @@
 #include <linux/clkdev.h>
 #include <linux/io.h>
 #include <linux/clk/ti.h>
+#include <dt-bindings/clock/omap5.h>
 
 #include "clock.h"
 
@@ -27,201 +28,511 @@
  */
 #define OMAP5_DPLL_USB_DEFFREQ				960000000
 
+static const struct omap_clkctrl_reg_data omap5_mpu_clkctrl_regs[] __initconst = {
+	{ OMAP5_MPU_CLKCTRL, NULL, 0, "dpll_mpu_m2_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_dsp_clkctrl_regs[] __initconst = {
+	{ OMAP5_MMU_DSP_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_iva_h11x2_ck" },
+	{ 0 },
+};
+
+static const char * const omap5_dmic_gfclk_parents[] __initconst = {
+	"abe_cm:clk:0018:26",
+	"pad_clks_ck",
+	"slimbus_clk",
+	NULL,
+};
+
+static const char * const omap5_dmic_sync_mux_ck_parents[] __initconst = {
+	"abe_24m_fclk",
+	"dss_syc_gfclk_div",
+	"func_24m_clk",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_dmic_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_dmic_gfclk_parents, NULL },
+	{ 26, TI_CLK_MUX, omap5_dmic_sync_mux_ck_parents, NULL },
+	{ 0 },
+};
+
+static const char * const omap5_mcbsp1_gfclk_parents[] __initconst = {
+	"abe_cm:clk:0028:26",
+	"pad_clks_ck",
+	"slimbus_clk",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_mcbsp1_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_mcbsp1_gfclk_parents, NULL },
+	{ 26, TI_CLK_MUX, omap5_dmic_sync_mux_ck_parents, NULL },
+	{ 0 },
+};
+
+static const char * const omap5_mcbsp2_gfclk_parents[] __initconst = {
+	"abe_cm:clk:0030:26",
+	"pad_clks_ck",
+	"slimbus_clk",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_mcbsp2_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_mcbsp2_gfclk_parents, NULL },
+	{ 26, TI_CLK_MUX, omap5_dmic_sync_mux_ck_parents, NULL },
+	{ 0 },
+};
+
+static const char * const omap5_mcbsp3_gfclk_parents[] __initconst = {
+	"abe_cm:clk:0038:26",
+	"pad_clks_ck",
+	"slimbus_clk",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_mcbsp3_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_mcbsp3_gfclk_parents, NULL },
+	{ 26, TI_CLK_MUX, omap5_dmic_sync_mux_ck_parents, NULL },
+	{ 0 },
+};
+
+static const char * const omap5_timer5_gfclk_mux_parents[] __initconst = {
+	"dss_syc_gfclk_div",
+	"sys_32k_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer5_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer6_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer7_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer8_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_abe_clkctrl_regs[] __initconst = {
+	{ OMAP5_L4_ABE_CLKCTRL, NULL, 0, "abe_iclk" },
+	{ OMAP5_MCPDM_CLKCTRL, NULL, CLKF_SW_SUP, "pad_clks_ck" },
+	{ OMAP5_DMIC_CLKCTRL, omap5_dmic_bit_data, CLKF_SW_SUP, "abe_cm:clk:0018:24" },
+	{ OMAP5_MCBSP1_CLKCTRL, omap5_mcbsp1_bit_data, CLKF_SW_SUP, "abe_cm:clk:0028:24" },
+	{ OMAP5_MCBSP2_CLKCTRL, omap5_mcbsp2_bit_data, CLKF_SW_SUP, "abe_cm:clk:0030:24" },
+	{ OMAP5_MCBSP3_CLKCTRL, omap5_mcbsp3_bit_data, CLKF_SW_SUP, "abe_cm:clk:0038:24" },
+	{ OMAP5_TIMER5_CLKCTRL, omap5_timer5_bit_data, CLKF_SW_SUP, "abe_cm:clk:0048:24" },
+	{ OMAP5_TIMER6_CLKCTRL, omap5_timer6_bit_data, CLKF_SW_SUP, "abe_cm:clk:0050:24" },
+	{ OMAP5_TIMER7_CLKCTRL, omap5_timer7_bit_data, CLKF_SW_SUP, "abe_cm:clk:0058:24" },
+	{ OMAP5_TIMER8_CLKCTRL, omap5_timer8_bit_data, CLKF_SW_SUP, "abe_cm:clk:0060:24" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_l3main1_clkctrl_regs[] __initconst = {
+	{ OMAP5_L3_MAIN_1_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_l3main2_clkctrl_regs[] __initconst = {
+	{ OMAP5_L3_MAIN_2_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_ipu_clkctrl_regs[] __initconst = {
+	{ OMAP5_MMU_IPU_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h22x2_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_dma_clkctrl_regs[] __initconst = {
+	{ OMAP5_DMA_SYSTEM_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_emif_clkctrl_regs[] __initconst = {
+	{ OMAP5_DMM_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ OMAP5_EMIF1_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h11x2_ck" },
+	{ OMAP5_EMIF2_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h11x2_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_l4cfg_clkctrl_regs[] __initconst = {
+	{ OMAP5_L4_CFG_CLKCTRL, NULL, 0, "l4_root_clk_div" },
+	{ OMAP5_SPINLOCK_CLKCTRL, NULL, 0, "l4_root_clk_div" },
+	{ OMAP5_MAILBOX_CLKCTRL, NULL, 0, "l4_root_clk_div" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_l3instr_clkctrl_regs[] __initconst = {
+	{ OMAP5_L3_MAIN_3_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ OMAP5_L3_INSTR_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const char * const omap5_timer10_gfclk_mux_parents[] __initconst = {
+	"sys_clkin",
+	"sys_32k_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer10_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer11_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer2_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer3_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer4_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer9_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const char * const omap5_gpio2_dbclk_parents[] __initconst = {
+	"sys_32k_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio2_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio3_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio4_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio5_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio6_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio7_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio8_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_l4per_clkctrl_regs[] __initconst = {
+	{ OMAP5_TIMER10_CLKCTRL, omap5_timer10_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0008:24" },
+	{ OMAP5_TIMER11_CLKCTRL, omap5_timer11_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0010:24" },
+	{ OMAP5_TIMER2_CLKCTRL, omap5_timer2_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0018:24" },
+	{ OMAP5_TIMER3_CLKCTRL, omap5_timer3_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0020:24" },
+	{ OMAP5_TIMER4_CLKCTRL, omap5_timer4_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0028:24" },
+	{ OMAP5_TIMER9_CLKCTRL, omap5_timer9_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0030:24" },
+	{ OMAP5_GPIO2_CLKCTRL, omap5_gpio2_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_GPIO3_CLKCTRL, omap5_gpio3_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_GPIO4_CLKCTRL, omap5_gpio4_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_GPIO5_CLKCTRL, omap5_gpio5_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_GPIO6_CLKCTRL, omap5_gpio6_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_I2C1_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ OMAP5_I2C2_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ OMAP5_I2C3_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ OMAP5_I2C4_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ OMAP5_L4_PER_CLKCTRL, NULL, 0, "l4_root_clk_div" },
+	{ OMAP5_MCSPI1_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_MCSPI2_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_MCSPI3_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_MCSPI4_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_GPIO7_CLKCTRL, omap5_gpio7_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_GPIO8_CLKCTRL, omap5_gpio8_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_MMC3_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_MMC4_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_UART1_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_UART2_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_UART3_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_UART4_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_MMC5_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ OMAP5_I2C5_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ OMAP5_UART5_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_UART6_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ 0 },
+};
+
+static const char * const omap5_dss_dss_clk_parents[] __initconst = {
+	"dpll_per_h12x2_ck",
+	NULL,
+};
+
+static const char * const omap5_dss_48mhz_clk_parents[] __initconst = {
+	"func_48m_fclk",
+	NULL,
+};
+
+static const char * const omap5_dss_sys_clk_parents[] __initconst = {
+	"dss_syc_gfclk_div",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_dss_core_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_dss_dss_clk_parents, NULL },
+	{ 9, TI_CLK_GATE, omap5_dss_48mhz_clk_parents, NULL },
+	{ 10, TI_CLK_GATE, omap5_dss_sys_clk_parents, NULL },
+	{ 11, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_dss_clkctrl_regs[] __initconst = {
+	{ OMAP5_DSS_CORE_CLKCTRL, omap5_dss_core_bit_data, CLKF_SW_SUP, "dss_cm:clk:0000:8" },
+	{ 0 },
+};
+
+static const char * const omap5_mmc1_fclk_mux_parents[] __initconst = {
+	"func_128m_clk",
+	"dpll_per_m2x2_ck",
+	NULL,
+};
+
+static const char * const omap5_mmc1_fclk_parents[] __initconst = {
+	"l3init_cm:clk:0008:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data omap5_mmc1_fclk_data __initconst = {
+	.max_div = 2,
+};
+
+static const struct omap_clkctrl_bit_data omap5_mmc1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 24, TI_CLK_MUX, omap5_mmc1_fclk_mux_parents, NULL },
+	{ 25, TI_CLK_DIVIDER, omap5_mmc1_fclk_parents, &omap5_mmc1_fclk_data },
+	{ 0 },
+};
+
+static const char * const omap5_mmc2_fclk_parents[] __initconst = {
+	"l3init_cm:clk:0010:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data omap5_mmc2_fclk_data __initconst = {
+	.max_div = 2,
+};
+
+static const struct omap_clkctrl_bit_data omap5_mmc2_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_mmc1_fclk_mux_parents, NULL },
+	{ 25, TI_CLK_DIVIDER, omap5_mmc2_fclk_parents, &omap5_mmc2_fclk_data },
+	{ 0 },
+};
+
+static const char * const omap5_usb_host_hs_hsic60m_p3_clk_parents[] __initconst = {
+	"l3init_60m_fclk",
+	NULL,
+};
+
+static const char * const omap5_usb_host_hs_hsic480m_p3_clk_parents[] __initconst = {
+	"dpll_usb_m2_ck",
+	NULL,
+};
+
+static const char * const omap5_usb_host_hs_utmi_p1_clk_parents[] __initconst = {
+	"l3init_cm:clk:0038:24",
+	NULL,
+};
+
+static const char * const omap5_usb_host_hs_utmi_p2_clk_parents[] __initconst = {
+	"l3init_cm:clk:0038:25",
+	NULL,
+};
+
+static const char * const omap5_utmi_p1_gfclk_parents[] __initconst = {
+	"l3init_60m_fclk",
+	"xclk60mhsp1_ck",
+	NULL,
+};
+
+static const char * const omap5_utmi_p2_gfclk_parents[] __initconst = {
+	"l3init_60m_fclk",
+	"xclk60mhsp2_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_usb_host_hs_bit_data[] __initconst = {
+	{ 6, TI_CLK_GATE, omap5_usb_host_hs_hsic60m_p3_clk_parents, NULL },
+	{ 7, TI_CLK_GATE, omap5_usb_host_hs_hsic480m_p3_clk_parents, NULL },
+	{ 8, TI_CLK_GATE, omap5_usb_host_hs_utmi_p1_clk_parents, NULL },
+	{ 9, TI_CLK_GATE, omap5_usb_host_hs_utmi_p2_clk_parents, NULL },
+	{ 10, TI_CLK_GATE, omap5_usb_host_hs_hsic60m_p3_clk_parents, NULL },
+	{ 11, TI_CLK_GATE, omap5_usb_host_hs_hsic60m_p3_clk_parents, NULL },
+	{ 12, TI_CLK_GATE, omap5_usb_host_hs_hsic60m_p3_clk_parents, NULL },
+	{ 13, TI_CLK_GATE, omap5_usb_host_hs_hsic480m_p3_clk_parents, NULL },
+	{ 14, TI_CLK_GATE, omap5_usb_host_hs_hsic480m_p3_clk_parents, NULL },
+	{ 24, TI_CLK_MUX, omap5_utmi_p1_gfclk_parents, NULL },
+	{ 25, TI_CLK_MUX, omap5_utmi_p2_gfclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_usb_tll_hs_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_usb_host_hs_hsic60m_p3_clk_parents, NULL },
+	{ 9, TI_CLK_GATE, omap5_usb_host_hs_hsic60m_p3_clk_parents, NULL },
+	{ 10, TI_CLK_GATE, omap5_usb_host_hs_hsic60m_p3_clk_parents, NULL },
+	{ 0 },
+};
+
+static const char * const omap5_sata_ref_clk_parents[] __initconst = {
+	"sys_clkin",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_sata_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_sata_ref_clk_parents, NULL },
+	{ 0 },
+};
+
+static const char * const omap5_usb_otg_ss_refclk960m_parents[] __initconst = {
+	"dpll_usb_clkdcoldo",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data omap5_usb_otg_ss_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_usb_otg_ss_refclk960m_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_l3init_clkctrl_regs[] __initconst = {
+	{ OMAP5_MMC1_CLKCTRL, omap5_mmc1_bit_data, CLKF_SW_SUP, "l3init_cm:clk:0008:25" },
+	{ OMAP5_MMC2_CLKCTRL, omap5_mmc2_bit_data, CLKF_SW_SUP, "l3init_cm:clk:0010:25" },
+	{ OMAP5_USB_HOST_HS_CLKCTRL, omap5_usb_host_hs_bit_data, CLKF_SW_SUP, "l3init_60m_fclk" },
+	{ OMAP5_USB_TLL_HS_CLKCTRL, omap5_usb_tll_hs_bit_data, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_SATA_CLKCTRL, omap5_sata_bit_data, CLKF_SW_SUP, "func_48m_fclk" },
+	{ OMAP5_OCP2SCP1_CLKCTRL, NULL, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_OCP2SCP3_CLKCTRL, NULL, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ OMAP5_USB_OTG_SS_CLKCTRL, omap5_usb_otg_ss_bit_data, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_gpio1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, omap5_gpio2_dbclk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data omap5_timer1_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, omap5_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data omap5_wkupaon_clkctrl_regs[] __initconst = {
+	{ OMAP5_L4_WKUP_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
+	{ OMAP5_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
+	{ OMAP5_GPIO1_CLKCTRL, omap5_gpio1_bit_data, CLKF_HW_SUP, "wkupaon_iclk_mux" },
+	{ OMAP5_TIMER1_CLKCTRL, omap5_timer1_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0020:24" },
+	{ OMAP5_COUNTER_32K_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
+	{ OMAP5_KBD_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
+	{ 0 },
+};
+
+const struct omap_clkctrl_data omap5_clkctrl_data[] __initconst = {
+	{ 0x4a004320, omap5_mpu_clkctrl_regs },
+	{ 0x4a004420, omap5_dsp_clkctrl_regs },
+	{ 0x4a004520, omap5_abe_clkctrl_regs },
+	{ 0x4a008720, omap5_l3main1_clkctrl_regs },
+	{ 0x4a008820, omap5_l3main2_clkctrl_regs },
+	{ 0x4a008920, omap5_ipu_clkctrl_regs },
+	{ 0x4a008a20, omap5_dma_clkctrl_regs },
+	{ 0x4a008b20, omap5_emif_clkctrl_regs },
+	{ 0x4a008d20, omap5_l4cfg_clkctrl_regs },
+	{ 0x4a008e20, omap5_l3instr_clkctrl_regs },
+	{ 0x4a009020, omap5_l4per_clkctrl_regs },
+	{ 0x4a009420, omap5_dss_clkctrl_regs },
+	{ 0x4a009620, omap5_l3init_clkctrl_regs },
+	{ 0x4ae07920, omap5_wkupaon_clkctrl_regs },
+	{ 0 },
+};
+
 static struct ti_dt_clk omap54xx_clks[] = {
-	DT_CLK(NULL, "pad_clks_src_ck", "pad_clks_src_ck"),
-	DT_CLK(NULL, "pad_clks_ck", "pad_clks_ck"),
-	DT_CLK(NULL, "secure_32k_clk_src_ck", "secure_32k_clk_src_ck"),
-	DT_CLK(NULL, "slimbus_src_clk", "slimbus_src_clk"),
-	DT_CLK(NULL, "slimbus_clk", "slimbus_clk"),
-	DT_CLK(NULL, "sys_32k_ck", "sys_32k_ck"),
-	DT_CLK(NULL, "virt_12000000_ck", "virt_12000000_ck"),
-	DT_CLK(NULL, "virt_13000000_ck", "virt_13000000_ck"),
-	DT_CLK(NULL, "virt_16800000_ck", "virt_16800000_ck"),
-	DT_CLK(NULL, "virt_19200000_ck", "virt_19200000_ck"),
-	DT_CLK(NULL, "virt_26000000_ck", "virt_26000000_ck"),
-	DT_CLK(NULL, "virt_27000000_ck", "virt_27000000_ck"),
-	DT_CLK(NULL, "virt_38400000_ck", "virt_38400000_ck"),
-	DT_CLK(NULL, "sys_clkin", "sys_clkin"),
-	DT_CLK(NULL, "xclk60mhsp1_ck", "xclk60mhsp1_ck"),
-	DT_CLK(NULL, "xclk60mhsp2_ck", "xclk60mhsp2_ck"),
-	DT_CLK(NULL, "abe_dpll_bypass_clk_mux", "abe_dpll_bypass_clk_mux"),
-	DT_CLK(NULL, "abe_dpll_clk_mux", "abe_dpll_clk_mux"),
-	DT_CLK(NULL, "dpll_abe_ck", "dpll_abe_ck"),
-	DT_CLK(NULL, "dpll_abe_x2_ck", "dpll_abe_x2_ck"),
-	DT_CLK(NULL, "dpll_abe_m2x2_ck", "dpll_abe_m2x2_ck"),
-	DT_CLK(NULL, "abe_24m_fclk", "abe_24m_fclk"),
-	DT_CLK(NULL, "abe_clk", "abe_clk"),
-	DT_CLK(NULL, "abe_iclk", "abe_iclk"),
-	DT_CLK(NULL, "abe_lp_clk_div", "abe_lp_clk_div"),
-	DT_CLK(NULL, "dpll_abe_m3x2_ck", "dpll_abe_m3x2_ck"),
-	DT_CLK(NULL, "dpll_core_ck", "dpll_core_ck"),
-	DT_CLK(NULL, "dpll_core_x2_ck", "dpll_core_x2_ck"),
-	DT_CLK(NULL, "dpll_core_h21x2_ck", "dpll_core_h21x2_ck"),
-	DT_CLK(NULL, "c2c_fclk", "c2c_fclk"),
-	DT_CLK(NULL, "c2c_iclk", "c2c_iclk"),
-	DT_CLK(NULL, "custefuse_sys_gfclk_div", "custefuse_sys_gfclk_div"),
-	DT_CLK(NULL, "dpll_core_h11x2_ck", "dpll_core_h11x2_ck"),
-	DT_CLK(NULL, "dpll_core_h12x2_ck", "dpll_core_h12x2_ck"),
-	DT_CLK(NULL, "dpll_core_h13x2_ck", "dpll_core_h13x2_ck"),
-	DT_CLK(NULL, "dpll_core_h14x2_ck", "dpll_core_h14x2_ck"),
-	DT_CLK(NULL, "dpll_core_h22x2_ck", "dpll_core_h22x2_ck"),
-	DT_CLK(NULL, "dpll_core_h23x2_ck", "dpll_core_h23x2_ck"),
-	DT_CLK(NULL, "dpll_core_h24x2_ck", "dpll_core_h24x2_ck"),
-	DT_CLK(NULL, "dpll_core_m2_ck", "dpll_core_m2_ck"),
-	DT_CLK(NULL, "dpll_core_m3x2_ck", "dpll_core_m3x2_ck"),
-	DT_CLK(NULL, "iva_dpll_hs_clk_div", "iva_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_iva_ck", "dpll_iva_ck"),
-	DT_CLK(NULL, "dpll_iva_x2_ck", "dpll_iva_x2_ck"),
-	DT_CLK(NULL, "dpll_iva_h11x2_ck", "dpll_iva_h11x2_ck"),
-	DT_CLK(NULL, "dpll_iva_h12x2_ck", "dpll_iva_h12x2_ck"),
-	DT_CLK(NULL, "mpu_dpll_hs_clk_div", "mpu_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_mpu_ck", "dpll_mpu_ck"),
-	DT_CLK(NULL, "dpll_mpu_m2_ck", "dpll_mpu_m2_ck"),
-	DT_CLK(NULL, "per_dpll_hs_clk_div", "per_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_per_ck", "dpll_per_ck"),
-	DT_CLK(NULL, "dpll_per_x2_ck", "dpll_per_x2_ck"),
-	DT_CLK(NULL, "dpll_per_h11x2_ck", "dpll_per_h11x2_ck"),
-	DT_CLK(NULL, "dpll_per_h12x2_ck", "dpll_per_h12x2_ck"),
-	DT_CLK(NULL, "dpll_per_h14x2_ck", "dpll_per_h14x2_ck"),
-	DT_CLK(NULL, "dpll_per_m2_ck", "dpll_per_m2_ck"),
-	DT_CLK(NULL, "dpll_per_m2x2_ck", "dpll_per_m2x2_ck"),
-	DT_CLK(NULL, "dpll_per_m3x2_ck", "dpll_per_m3x2_ck"),
-	DT_CLK(NULL, "dpll_unipro1_ck", "dpll_unipro1_ck"),
-	DT_CLK(NULL, "dpll_unipro1_clkdcoldo", "dpll_unipro1_clkdcoldo"),
-	DT_CLK(NULL, "dpll_unipro1_m2_ck", "dpll_unipro1_m2_ck"),
-	DT_CLK(NULL, "dpll_unipro2_ck", "dpll_unipro2_ck"),
-	DT_CLK(NULL, "dpll_unipro2_clkdcoldo", "dpll_unipro2_clkdcoldo"),
-	DT_CLK(NULL, "dpll_unipro2_m2_ck", "dpll_unipro2_m2_ck"),
-	DT_CLK(NULL, "usb_dpll_hs_clk_div", "usb_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_usb_ck", "dpll_usb_ck"),
-	DT_CLK(NULL, "dpll_usb_clkdcoldo", "dpll_usb_clkdcoldo"),
-	DT_CLK(NULL, "dpll_usb_m2_ck", "dpll_usb_m2_ck"),
-	DT_CLK(NULL, "dss_syc_gfclk_div", "dss_syc_gfclk_div"),
-	DT_CLK(NULL, "func_128m_clk", "func_128m_clk"),
-	DT_CLK(NULL, "func_12m_fclk", "func_12m_fclk"),
-	DT_CLK(NULL, "func_24m_clk", "func_24m_clk"),
-	DT_CLK(NULL, "func_48m_fclk", "func_48m_fclk"),
-	DT_CLK(NULL, "func_96m_fclk", "func_96m_fclk"),
-	DT_CLK(NULL, "l3_iclk_div", "l3_iclk_div"),
-	DT_CLK(NULL, "gpu_l3_iclk", "gpu_l3_iclk"),
-	DT_CLK(NULL, "l3init_60m_fclk", "l3init_60m_fclk"),
-	DT_CLK(NULL, "wkupaon_iclk_mux", "wkupaon_iclk_mux"),
-	DT_CLK(NULL, "l3instr_ts_gclk_div", "l3instr_ts_gclk_div"),
-	DT_CLK(NULL, "l4_root_clk_div", "l4_root_clk_div"),
-	DT_CLK(NULL, "dss_32khz_clk", "dss_32khz_clk"),
-	DT_CLK(NULL, "dss_48mhz_clk", "dss_48mhz_clk"),
-	DT_CLK(NULL, "dss_dss_clk", "dss_dss_clk"),
-	DT_CLK(NULL, "dss_sys_clk", "dss_sys_clk"),
-	DT_CLK(NULL, "gpio1_dbclk", "gpio1_dbclk"),
-	DT_CLK(NULL, "gpio2_dbclk", "gpio2_dbclk"),
-	DT_CLK(NULL, "gpio3_dbclk", "gpio3_dbclk"),
-	DT_CLK(NULL, "gpio4_dbclk", "gpio4_dbclk"),
-	DT_CLK(NULL, "gpio5_dbclk", "gpio5_dbclk"),
-	DT_CLK(NULL, "gpio6_dbclk", "gpio6_dbclk"),
-	DT_CLK(NULL, "gpio7_dbclk", "gpio7_dbclk"),
-	DT_CLK(NULL, "gpio8_dbclk", "gpio8_dbclk"),
-	DT_CLK(NULL, "iss_ctrlclk", "iss_ctrlclk"),
-	DT_CLK(NULL, "lli_txphy_clk", "lli_txphy_clk"),
-	DT_CLK(NULL, "lli_txphy_ls_clk", "lli_txphy_ls_clk"),
-	DT_CLK(NULL, "mmc1_32khz_clk", "mmc1_32khz_clk"),
-	DT_CLK(NULL, "sata_ref_clk", "sata_ref_clk"),
-	DT_CLK(NULL, "slimbus1_slimbus_clk", "slimbus1_slimbus_clk"),
-	DT_CLK(NULL, "usb_host_hs_hsic480m_p1_clk", "usb_host_hs_hsic480m_p1_clk"),
-	DT_CLK(NULL, "usb_host_hs_hsic480m_p2_clk", "usb_host_hs_hsic480m_p2_clk"),
-	DT_CLK(NULL, "usb_host_hs_hsic480m_p3_clk", "usb_host_hs_hsic480m_p3_clk"),
-	DT_CLK(NULL, "usb_host_hs_hsic60m_p1_clk", "usb_host_hs_hsic60m_p1_clk"),
-	DT_CLK(NULL, "usb_host_hs_hsic60m_p2_clk", "usb_host_hs_hsic60m_p2_clk"),
-	DT_CLK(NULL, "usb_host_hs_hsic60m_p3_clk", "usb_host_hs_hsic60m_p3_clk"),
-	DT_CLK(NULL, "usb_host_hs_utmi_p1_clk", "usb_host_hs_utmi_p1_clk"),
-	DT_CLK(NULL, "usb_host_hs_utmi_p2_clk", "usb_host_hs_utmi_p2_clk"),
-	DT_CLK(NULL, "usb_host_hs_utmi_p3_clk", "usb_host_hs_utmi_p3_clk"),
-	DT_CLK(NULL, "usb_otg_ss_refclk960m", "usb_otg_ss_refclk960m"),
-	DT_CLK(NULL, "usb_phy_cm_clk32k", "usb_phy_cm_clk32k"),
-	DT_CLK(NULL, "usb_tll_hs_usb_ch0_clk", "usb_tll_hs_usb_ch0_clk"),
-	DT_CLK(NULL, "usb_tll_hs_usb_ch1_clk", "usb_tll_hs_usb_ch1_clk"),
-	DT_CLK(NULL, "usb_tll_hs_usb_ch2_clk", "usb_tll_hs_usb_ch2_clk"),
-	DT_CLK(NULL, "aess_fclk", "aess_fclk"),
-	DT_CLK(NULL, "dmic_sync_mux_ck", "dmic_sync_mux_ck"),
-	DT_CLK(NULL, "dmic_gfclk", "dmic_gfclk"),
-	DT_CLK(NULL, "fdif_fclk", "fdif_fclk"),
-	DT_CLK(NULL, "gpu_core_gclk_mux", "gpu_core_gclk_mux"),
-	DT_CLK(NULL, "gpu_hyd_gclk_mux", "gpu_hyd_gclk_mux"),
-	DT_CLK(NULL, "hsi_fclk", "hsi_fclk"),
-	DT_CLK(NULL, "mcasp_sync_mux_ck", "mcasp_sync_mux_ck"),
-	DT_CLK(NULL, "mcasp_gfclk", "mcasp_gfclk"),
-	DT_CLK(NULL, "mcbsp1_sync_mux_ck", "mcbsp1_sync_mux_ck"),
-	DT_CLK(NULL, "mcbsp1_gfclk", "mcbsp1_gfclk"),
-	DT_CLK(NULL, "mcbsp2_sync_mux_ck", "mcbsp2_sync_mux_ck"),
-	DT_CLK(NULL, "mcbsp2_gfclk", "mcbsp2_gfclk"),
-	DT_CLK(NULL, "mcbsp3_sync_mux_ck", "mcbsp3_sync_mux_ck"),
-	DT_CLK(NULL, "mcbsp3_gfclk", "mcbsp3_gfclk"),
-	DT_CLK(NULL, "mmc1_fclk_mux", "mmc1_fclk_mux"),
-	DT_CLK(NULL, "mmc1_fclk", "mmc1_fclk"),
-	DT_CLK(NULL, "mmc2_fclk_mux", "mmc2_fclk_mux"),
-	DT_CLK(NULL, "mmc2_fclk", "mmc2_fclk"),
-	DT_CLK(NULL, "timer10_gfclk_mux", "timer10_gfclk_mux"),
-	DT_CLK(NULL, "timer11_gfclk_mux", "timer11_gfclk_mux"),
-	DT_CLK(NULL, "timer1_gfclk_mux", "timer1_gfclk_mux"),
-	DT_CLK(NULL, "timer2_gfclk_mux", "timer2_gfclk_mux"),
-	DT_CLK(NULL, "timer3_gfclk_mux", "timer3_gfclk_mux"),
-	DT_CLK(NULL, "timer4_gfclk_mux", "timer4_gfclk_mux"),
-	DT_CLK(NULL, "timer5_gfclk_mux", "timer5_gfclk_mux"),
-	DT_CLK(NULL, "timer6_gfclk_mux", "timer6_gfclk_mux"),
-	DT_CLK(NULL, "timer7_gfclk_mux", "timer7_gfclk_mux"),
-	DT_CLK(NULL, "timer8_gfclk_mux", "timer8_gfclk_mux"),
-	DT_CLK(NULL, "timer9_gfclk_mux", "timer9_gfclk_mux"),
-	DT_CLK(NULL, "utmi_p1_gfclk", "utmi_p1_gfclk"),
-	DT_CLK(NULL, "utmi_p2_gfclk", "utmi_p2_gfclk"),
-	DT_CLK(NULL, "auxclk0_src_ck", "auxclk0_src_ck"),
-	DT_CLK(NULL, "auxclk0_ck", "auxclk0_ck"),
-	DT_CLK(NULL, "auxclkreq0_ck", "auxclkreq0_ck"),
-	DT_CLK(NULL, "auxclk1_src_ck", "auxclk1_src_ck"),
-	DT_CLK(NULL, "auxclk1_ck", "auxclk1_ck"),
-	DT_CLK(NULL, "auxclkreq1_ck", "auxclkreq1_ck"),
-	DT_CLK(NULL, "auxclk2_src_ck", "auxclk2_src_ck"),
-	DT_CLK(NULL, "auxclk2_ck", "auxclk2_ck"),
-	DT_CLK(NULL, "auxclkreq2_ck", "auxclkreq2_ck"),
-	DT_CLK(NULL, "auxclk3_src_ck", "auxclk3_src_ck"),
-	DT_CLK(NULL, "auxclk3_ck", "auxclk3_ck"),
-	DT_CLK(NULL, "auxclkreq3_ck", "auxclkreq3_ck"),
-	DT_CLK("omap_i2c.1", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.2", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.3", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.4", "ick", "dummy_ck"),
-	DT_CLK(NULL, "mailboxes_ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.0", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.1", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.2", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.3", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.4", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.1", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.2", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.3", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.4", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.1", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.2", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.3", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.4", "ick", "dummy_ck"),
-	DT_CLK(NULL, "uart1_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart2_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart3_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart4_ick", "dummy_ck"),
-	DT_CLK("usbhs_omap", "usbhost_ick", "dummy_ck"),
-	DT_CLK("usbhs_omap", "usbtll_fck", "dummy_ck"),
-	DT_CLK("omap_wdt", "ick", "dummy_ck"),
 	DT_CLK(NULL, "timer_32k_ck", "sys_32k_ck"),
 	DT_CLK(NULL, "sys_clkin_ck", "sys_clkin"),
-	DT_CLK("4ae18000.timer", "timer_sys_ck", "sys_clkin"),
-	DT_CLK("48032000.timer", "timer_sys_ck", "sys_clkin"),
-	DT_CLK("48034000.timer", "timer_sys_ck", "sys_clkin"),
-	DT_CLK("48036000.timer", "timer_sys_ck", "sys_clkin"),
-	DT_CLK("4803e000.timer", "timer_sys_ck", "sys_clkin"),
-	DT_CLK("48086000.timer", "timer_sys_ck", "sys_clkin"),
-	DT_CLK("48088000.timer", "timer_sys_ck", "sys_clkin"),
-	DT_CLK("40138000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
-	DT_CLK("4013a000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
-	DT_CLK("4013c000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
-	DT_CLK("4013e000.timer", "timer_sys_ck", "dss_syc_gfclk_div"),
+	DT_CLK(NULL, "dmic_gfclk", "abe_cm:0018:24"),
+	DT_CLK(NULL, "dmic_sync_mux_ck", "abe_cm:0018:26"),
+	DT_CLK(NULL, "dss_32khz_clk", "dss_cm:0000:11"),
+	DT_CLK(NULL, "dss_48mhz_clk", "dss_cm:0000:9"),
+	DT_CLK(NULL, "dss_dss_clk", "dss_cm:0000:8"),
+	DT_CLK(NULL, "dss_sys_clk", "dss_cm:0000:10"),
+	DT_CLK(NULL, "gpio1_dbclk", "wkupaon_cm:0018:8"),
+	DT_CLK(NULL, "gpio2_dbclk", "l4per_cm:0040:8"),
+	DT_CLK(NULL, "gpio3_dbclk", "l4per_cm:0048:8"),
+	DT_CLK(NULL, "gpio4_dbclk", "l4per_cm:0050:8"),
+	DT_CLK(NULL, "gpio5_dbclk", "l4per_cm:0058:8"),
+	DT_CLK(NULL, "gpio6_dbclk", "l4per_cm:0060:8"),
+	DT_CLK(NULL, "gpio7_dbclk", "l4per_cm:00f0:8"),
+	DT_CLK(NULL, "gpio8_dbclk", "l4per_cm:00f8:8"),
+	DT_CLK(NULL, "mcbsp1_gfclk", "abe_cm:0028:24"),
+	DT_CLK(NULL, "mcbsp1_sync_mux_ck", "abe_cm:0028:26"),
+	DT_CLK(NULL, "mcbsp2_gfclk", "abe_cm:0030:24"),
+	DT_CLK(NULL, "mcbsp2_sync_mux_ck", "abe_cm:0030:26"),
+	DT_CLK(NULL, "mcbsp3_gfclk", "abe_cm:0038:24"),
+	DT_CLK(NULL, "mcbsp3_sync_mux_ck", "abe_cm:0038:26"),
+	DT_CLK(NULL, "mmc1_32khz_clk", "l3init_cm:0008:8"),
+	DT_CLK(NULL, "mmc1_fclk", "l3init_cm:0008:25"),
+	DT_CLK(NULL, "mmc1_fclk_mux", "l3init_cm:0008:24"),
+	DT_CLK(NULL, "mmc2_fclk", "l3init_cm:0010:25"),
+	DT_CLK(NULL, "mmc2_fclk_mux", "l3init_cm:0010:24"),
+	DT_CLK(NULL, "sata_ref_clk", "l3init_cm:0068:8"),
+	DT_CLK(NULL, "timer10_gfclk_mux", "l4per_cm:0008:24"),
+	DT_CLK(NULL, "timer11_gfclk_mux", "l4per_cm:0010:24"),
+	DT_CLK(NULL, "timer1_gfclk_mux", "wkupaon_cm:0020:24"),
+	DT_CLK(NULL, "timer2_gfclk_mux", "l4per_cm:0018:24"),
+	DT_CLK(NULL, "timer3_gfclk_mux", "l4per_cm:0020:24"),
+	DT_CLK(NULL, "timer4_gfclk_mux", "l4per_cm:0028:24"),
+	DT_CLK(NULL, "timer5_gfclk_mux", "abe_cm:0048:24"),
+	DT_CLK(NULL, "timer6_gfclk_mux", "abe_cm:0050:24"),
+	DT_CLK(NULL, "timer7_gfclk_mux", "abe_cm:0058:24"),
+	DT_CLK(NULL, "timer8_gfclk_mux", "abe_cm:0060:24"),
+	DT_CLK(NULL, "timer9_gfclk_mux", "l4per_cm:0030:24"),
+	DT_CLK(NULL, "usb_host_hs_hsic480m_p1_clk", "l3init_cm:0038:13"),
+	DT_CLK(NULL, "usb_host_hs_hsic480m_p2_clk", "l3init_cm:0038:14"),
+	DT_CLK(NULL, "usb_host_hs_hsic480m_p3_clk", "l3init_cm:0038:7"),
+	DT_CLK(NULL, "usb_host_hs_hsic60m_p1_clk", "l3init_cm:0038:11"),
+	DT_CLK(NULL, "usb_host_hs_hsic60m_p2_clk", "l3init_cm:0038:12"),
+	DT_CLK(NULL, "usb_host_hs_hsic60m_p3_clk", "l3init_cm:0038:6"),
+	DT_CLK(NULL, "usb_host_hs_utmi_p1_clk", "l3init_cm:0038:8"),
+	DT_CLK(NULL, "usb_host_hs_utmi_p2_clk", "l3init_cm:0038:9"),
+	DT_CLK(NULL, "usb_host_hs_utmi_p3_clk", "l3init_cm:0038:10"),
+	DT_CLK(NULL, "usb_otg_ss_refclk960m", "l3init_cm:00d0:8"),
+	DT_CLK(NULL, "usb_tll_hs_usb_ch0_clk", "l3init_cm:0048:8"),
+	DT_CLK(NULL, "usb_tll_hs_usb_ch1_clk", "l3init_cm:0048:9"),
+	DT_CLK(NULL, "usb_tll_hs_usb_ch2_clk", "l3init_cm:0048:10"),
+	DT_CLK(NULL, "utmi_p1_gfclk", "l3init_cm:0038:24"),
+	DT_CLK(NULL, "utmi_p2_gfclk", "l3init_cm:0038:25"),
 	{ .node_name = NULL },
 };
 
@@ -234,6 +545,8 @@ int __init omap5xxx_dt_clk_init(void)
 
 	omap2_clk_disable_autoidle_all();
 
+	ti_clk_add_aliases();
+
 	abe_dpll_ref = clk_get_sys(NULL, "abe_dpll_clk_mux");
 	sys_32k_ck = clk_get_sys(NULL, "sys_32k_ck");
 	rc = clk_set_parent(abe_dpll_ref, sys_32k_ck);
diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c
index 9fd6043314eb..fb249a1637a5 100644
--- a/drivers/clk/ti/clk-7xx.c
+++ b/drivers/clk/ti/clk-7xx.c
@@ -15,297 +15,809 @@
 #include <linux/clk.h>
 #include <linux/clkdev.h>
 #include <linux/clk/ti.h>
+#include <dt-bindings/clock/dra7.h>
 
 #include "clock.h"
 
 #define DRA7_DPLL_GMAC_DEFFREQ				1000000000
 #define DRA7_DPLL_USB_DEFFREQ				960000000
 
+static const struct omap_clkctrl_reg_data dra7_mpu_clkctrl_regs[] __initconst = {
+	{ DRA7_MPU_CLKCTRL, NULL, 0, "dpll_mpu_m2_ck" },
+	{ 0 },
+};
+
+static const char * const dra7_mcasp1_aux_gfclk_mux_parents[] __initconst = {
+	"per_abe_x1_gfclk2_div",
+	"video1_clk2_div",
+	"video2_clk2_div",
+	"hdmi_clk2_div",
+	NULL,
+};
+
+static const char * const dra7_mcasp1_ahclkx_mux_parents[] __initconst = {
+	"abe_24m_fclk",
+	"abe_sys_clk_div",
+	"func_24m_clk",
+	"atl_clkin3_ck",
+	"atl_clkin2_ck",
+	"atl_clkin1_ck",
+	"atl_clkin0_ck",
+	"sys_clkin2",
+	"ref_clkin0_ck",
+	"ref_clkin1_ck",
+	"ref_clkin2_ck",
+	"ref_clkin3_ck",
+	"mlb_clk",
+	"mlbp_clk",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp1_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 28, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_timer5_gfclk_mux_parents[] __initconst = {
+	"timer_sys_clk_div",
+	"sys_32k_ck",
+	"sys_clkin2",
+	"ref_clkin0_ck",
+	"ref_clkin1_ck",
+	"ref_clkin2_ck",
+	"ref_clkin3_ck",
+	"abe_giclk_div",
+	"video1_div_clk",
+	"video2_div_clk",
+	"hdmi_div_clk",
+	"clkoutmux0_clk_mux",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer5_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer6_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer7_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer8_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer5_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_uart6_gfclk_mux_parents[] __initconst = {
+	"func_48m_fclk",
+	"dpll_per_m2x2_ck",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart6_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_ipu_clkctrl_regs[] __initconst = {
+	{ DRA7_MCASP1_CLKCTRL, dra7_mcasp1_bit_data, CLKF_SW_SUP, "ipu_cm:clk:0010:22" },
+	{ DRA7_TIMER5_CLKCTRL, dra7_timer5_bit_data, CLKF_SW_SUP, "ipu_cm:clk:0018:24" },
+	{ DRA7_TIMER6_CLKCTRL, dra7_timer6_bit_data, CLKF_SW_SUP, "ipu_cm:clk:0020:24" },
+	{ DRA7_TIMER7_CLKCTRL, dra7_timer7_bit_data, CLKF_SW_SUP, "ipu_cm:clk:0028:24" },
+	{ DRA7_TIMER8_CLKCTRL, dra7_timer8_bit_data, CLKF_SW_SUP, "ipu_cm:clk:0030:24" },
+	{ DRA7_I2C5_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ DRA7_UART6_CLKCTRL, dra7_uart6_bit_data, CLKF_SW_SUP, "ipu_cm:clk:0040:24" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_rtc_clkctrl_regs[] __initconst = {
+	{ DRA7_RTCSS_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_coreaon_clkctrl_regs[] __initconst = {
+	{ DRA7_SMARTREFLEX_MPU_CLKCTRL, NULL, CLKF_SW_SUP, "wkupaon_iclk_mux" },
+	{ DRA7_SMARTREFLEX_CORE_CLKCTRL, NULL, CLKF_SW_SUP, "wkupaon_iclk_mux" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_l3main1_clkctrl_regs[] __initconst = {
+	{ DRA7_L3_MAIN_1_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_GPMC_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_TPCC_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_TPTC0_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_TPTC1_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_VCP1_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_VCP2_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_dma_clkctrl_regs[] __initconst = {
+	{ DRA7_DMA_SYSTEM_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_emif_clkctrl_regs[] __initconst = {
+	{ DRA7_DMM_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const char * const dra7_atl_dpll_clk_mux_parents[] __initconst = {
+	"sys_32k_ck",
+	"video1_clkin_ck",
+	"video2_clkin_ck",
+	"hdmi_clkin_ck",
+	NULL,
+};
+
+static const char * const dra7_atl_gfclk_mux_parents[] __initconst = {
+	"l3_iclk_div",
+	"dpll_abe_m2_ck",
+	"atl_cm:clk:0000:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_atl_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_atl_dpll_clk_mux_parents, NULL },
+	{ 26, TI_CLK_MUX, dra7_atl_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_atl_clkctrl_regs[] __initconst = {
+	{ DRA7_ATL_CLKCTRL, dra7_atl_bit_data, CLKF_SW_SUP, "atl_cm:clk:0000:26" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_l4cfg_clkctrl_regs[] __initconst = {
+	{ DRA7_L4_CFG_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_SPINLOCK_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX1_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX2_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX3_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX4_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX5_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX6_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX7_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX8_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX9_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX10_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX11_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX12_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_MAILBOX13_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_l3instr_clkctrl_regs[] __initconst = {
+	{ DRA7_L3_MAIN_2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_L3_INSTR_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
+	{ 0 },
+};
+
+static const char * const dra7_dss_dss_clk_parents[] __initconst = {
+	"dpll_per_h12x2_ck",
+	NULL,
+};
+
+static const char * const dra7_dss_48mhz_clk_parents[] __initconst = {
+	"func_48m_fclk",
+	NULL,
+};
+
+static const char * const dra7_dss_hdmi_clk_parents[] __initconst = {
+	"hdmi_dpll_clk_mux",
+	NULL,
+};
+
+static const char * const dra7_dss_32khz_clk_parents[] __initconst = {
+	"sys_32k_ck",
+	NULL,
+};
+
+static const char * const dra7_dss_video1_clk_parents[] __initconst = {
+	"video1_dpll_clk_mux",
+	NULL,
+};
+
+static const char * const dra7_dss_video2_clk_parents[] __initconst = {
+	"video2_dpll_clk_mux",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_dss_core_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_dss_clk_parents, NULL },
+	{ 9, TI_CLK_GATE, dra7_dss_48mhz_clk_parents, NULL },
+	{ 10, TI_CLK_GATE, dra7_dss_hdmi_clk_parents, NULL },
+	{ 11, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 12, TI_CLK_GATE, dra7_dss_video1_clk_parents, NULL },
+	{ 13, TI_CLK_GATE, dra7_dss_video2_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_dss_clkctrl_regs[] __initconst = {
+	{ DRA7_DSS_CORE_CLKCTRL, dra7_dss_core_bit_data, CLKF_SW_SUP, "dss_cm:clk:0000:8" },
+	{ DRA7_BB2D_CLKCTRL, NULL, CLKF_SW_SUP, "dpll_core_h24x2_ck" },
+	{ 0 },
+};
+
+static const char * const dra7_mmc1_fclk_mux_parents[] __initconst = {
+	"func_128m_clk",
+	"dpll_per_m2x2_ck",
+	NULL,
+};
+
+static const char * const dra7_mmc1_fclk_div_parents[] __initconst = {
+	"l3init_cm:clk:0008:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data dra7_mmc1_fclk_div_data __initconst = {
+	.max_div = 4,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
+};
+
+static const struct omap_clkctrl_bit_data dra7_mmc1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mmc1_fclk_mux_parents, NULL },
+	{ 25, TI_CLK_DIVIDER, dra7_mmc1_fclk_div_parents, &dra7_mmc1_fclk_div_data },
+	{ 0 },
+};
+
+static const char * const dra7_mmc2_fclk_div_parents[] __initconst = {
+	"l3init_cm:clk:0010:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data dra7_mmc2_fclk_div_data __initconst = {
+	.max_div = 4,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
+};
+
+static const struct omap_clkctrl_bit_data dra7_mmc2_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mmc1_fclk_mux_parents, NULL },
+	{ 25, TI_CLK_DIVIDER, dra7_mmc2_fclk_div_parents, &dra7_mmc2_fclk_div_data },
+	{ 0 },
+};
+
+static const char * const dra7_usb_otg_ss2_refclk960m_parents[] __initconst = {
+	"l3init_960m_gfclk",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_usb_otg_ss2_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_usb_otg_ss2_refclk960m_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_sata_ref_clk_parents[] __initconst = {
+	"sys_clkin1",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_sata_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_sata_ref_clk_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_optfclk_pciephy1_clk_parents[] __initconst = {
+	"apll_pcie_ck",
+	NULL,
+};
+
+static const char * const dra7_optfclk_pciephy1_div_clk_parents[] __initconst = {
+	"optfclk_pciephy_div",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_pcie1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 9, TI_CLK_GATE, dra7_optfclk_pciephy1_clk_parents, NULL },
+	{ 10, TI_CLK_GATE, dra7_optfclk_pciephy1_div_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_pcie2_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 9, TI_CLK_GATE, dra7_optfclk_pciephy1_clk_parents, NULL },
+	{ 10, TI_CLK_GATE, dra7_optfclk_pciephy1_div_clk_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_rmii_50mhz_clk_mux_parents[] __initconst = {
+	"dpll_gmac_h11x2_ck",
+	"rmii_clk_ck",
+	NULL,
+};
+
+static const char * const dra7_gmac_rft_clk_mux_parents[] __initconst = {
+	"video1_clkin_ck",
+	"video2_clkin_ck",
+	"dpll_abe_m2_ck",
+	"hdmi_clkin_ck",
+	"l3_iclk_div",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_gmac_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_rmii_50mhz_clk_mux_parents, NULL },
+	{ 25, TI_CLK_MUX, dra7_gmac_rft_clk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_usb_otg_ss1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_usb_otg_ss2_refclk960m_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_l3init_clkctrl_regs[] __initconst = {
+	{ DRA7_MMC1_CLKCTRL, dra7_mmc1_bit_data, CLKF_SW_SUP, "l3init_cm:clk:0008:25" },
+	{ DRA7_MMC2_CLKCTRL, dra7_mmc2_bit_data, CLKF_SW_SUP, "l3init_cm:clk:0010:25" },
+	{ DRA7_USB_OTG_SS2_CLKCTRL, dra7_usb_otg_ss2_bit_data, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ DRA7_USB_OTG_SS3_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ DRA7_USB_OTG_SS4_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ DRA7_SATA_CLKCTRL, dra7_sata_bit_data, CLKF_SW_SUP, "func_48m_fclk" },
+	{ DRA7_PCIE1_CLKCTRL, dra7_pcie1_bit_data, CLKF_SW_SUP, "l4_root_clk_div", "pcie_clkdm" },
+	{ DRA7_PCIE2_CLKCTRL, dra7_pcie2_bit_data, CLKF_SW_SUP, "l4_root_clk_div", "pcie_clkdm" },
+	{ DRA7_GMAC_CLKCTRL, dra7_gmac_bit_data, CLKF_SW_SUP, "dpll_gmac_ck", "gmac_clkdm" },
+	{ DRA7_OCP2SCP1_CLKCTRL, NULL, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ DRA7_OCP2SCP3_CLKCTRL, NULL, CLKF_HW_SUP, "l4_root_clk_div" },
+	{ DRA7_USB_OTG_SS1_CLKCTRL, dra7_usb_otg_ss1_bit_data, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ 0 },
+};
+
+static const char * const dra7_timer10_gfclk_mux_parents[] __initconst = {
+	"timer_sys_clk_div",
+	"sys_32k_ck",
+	"sys_clkin2",
+	"ref_clkin0_ck",
+	"ref_clkin1_ck",
+	"ref_clkin2_ck",
+	"ref_clkin3_ck",
+	"abe_giclk_div",
+	"video1_div_clk",
+	"video2_div_clk",
+	"hdmi_div_clk",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer10_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer11_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer2_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer3_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer4_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer9_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio2_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio3_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio4_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio5_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio6_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer13_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer14_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer15_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio7_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio8_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_mmc3_gfclk_div_parents[] __initconst = {
+	"l4per_cm:clk:0120:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data dra7_mmc3_gfclk_div_data __initconst = {
+	.max_div = 4,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
+};
+
+static const struct omap_clkctrl_bit_data dra7_mmc3_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 25, TI_CLK_DIVIDER, dra7_mmc3_gfclk_div_parents, &dra7_mmc3_gfclk_div_data },
+	{ 0 },
+};
+
+static const char * const dra7_mmc4_gfclk_div_parents[] __initconst = {
+	"l4per_cm:clk:0128:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data dra7_mmc4_gfclk_div_data __initconst = {
+	.max_div = 4,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
+};
+
+static const struct omap_clkctrl_bit_data dra7_mmc4_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 25, TI_CLK_DIVIDER, dra7_mmc4_gfclk_div_parents, &dra7_mmc4_gfclk_div_data },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer16_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_qspi_gfclk_mux_parents[] __initconst = {
+	"func_128m_clk",
+	"dpll_per_h13x2_ck",
+	NULL,
+};
+
+static const char * const dra7_qspi_gfclk_div_parents[] __initconst = {
+	"l4per_cm:clk:0138:24",
+	NULL,
+};
+
+static const struct omap_clkctrl_div_data dra7_qspi_gfclk_div_data __initconst = {
+	.max_div = 4,
+	.flags = CLK_DIVIDER_POWER_OF_TWO,
+};
+
+static const struct omap_clkctrl_bit_data dra7_qspi_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_qspi_gfclk_mux_parents, NULL },
+	{ 25, TI_CLK_DIVIDER, dra7_qspi_gfclk_div_parents, &dra7_qspi_gfclk_div_data },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart1_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart2_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart3_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart4_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp2_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 28, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp3_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart5_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp5_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp8_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp4_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart7_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart8_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart9_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp6_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_mcasp7_bit_data[] __initconst = {
+	{ 22, TI_CLK_MUX, dra7_mcasp1_aux_gfclk_mux_parents, NULL },
+	{ 24, TI_CLK_MUX, dra7_mcasp1_ahclkx_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_l4per_clkctrl_regs[] __initconst = {
+	{ DRA7_L4_PER2_CLKCTRL, NULL, 0, "l3_iclk_div", "l4per2_clkdm" },
+	{ DRA7_L4_PER3_CLKCTRL, NULL, 0, "l3_iclk_div", "l4per3_clkdm" },
+	{ DRA7_TIMER10_CLKCTRL, dra7_timer10_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0028:24" },
+	{ DRA7_TIMER11_CLKCTRL, dra7_timer11_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0030:24" },
+	{ DRA7_TIMER2_CLKCTRL, dra7_timer2_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0038:24" },
+	{ DRA7_TIMER3_CLKCTRL, dra7_timer3_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0040:24" },
+	{ DRA7_TIMER4_CLKCTRL, dra7_timer4_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0048:24" },
+	{ DRA7_TIMER9_CLKCTRL, dra7_timer9_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0050:24" },
+	{ DRA7_ELM_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_GPIO2_CLKCTRL, dra7_gpio2_bit_data, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_GPIO3_CLKCTRL, dra7_gpio3_bit_data, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_GPIO4_CLKCTRL, dra7_gpio4_bit_data, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_GPIO5_CLKCTRL, dra7_gpio5_bit_data, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_GPIO6_CLKCTRL, dra7_gpio6_bit_data, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_HDQ1W_CLKCTRL, NULL, CLKF_SW_SUP, "func_12m_fclk" },
+	{ DRA7_EPWMSS1_CLKCTRL, NULL, CLKF_SW_SUP, "l4_root_clk_div", "l4per2_clkdm" },
+	{ DRA7_EPWMSS2_CLKCTRL, NULL, CLKF_SW_SUP, "l4_root_clk_div", "l4per2_clkdm" },
+	{ DRA7_I2C1_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ DRA7_I2C2_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ DRA7_I2C3_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ DRA7_I2C4_CLKCTRL, NULL, CLKF_SW_SUP, "func_96m_fclk" },
+	{ DRA7_L4_PER1_CLKCTRL, NULL, 0, "l3_iclk_div" },
+	{ DRA7_EPWMSS0_CLKCTRL, NULL, CLKF_SW_SUP, "l4_root_clk_div", "l4per2_clkdm" },
+	{ DRA7_TIMER13_CLKCTRL, dra7_timer13_bit_data, CLKF_SW_SUP, "l4per_cm:clk:00c8:24", "l4per3_clkdm" },
+	{ DRA7_TIMER14_CLKCTRL, dra7_timer14_bit_data, CLKF_SW_SUP, "l4per_cm:clk:00d0:24", "l4per3_clkdm" },
+	{ DRA7_TIMER15_CLKCTRL, dra7_timer15_bit_data, CLKF_SW_SUP, "l4per_cm:clk:00d8:24", "l4per3_clkdm" },
+	{ DRA7_MCSPI1_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ DRA7_MCSPI2_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ DRA7_MCSPI3_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ DRA7_MCSPI4_CLKCTRL, NULL, CLKF_SW_SUP, "func_48m_fclk" },
+	{ DRA7_GPIO7_CLKCTRL, dra7_gpio7_bit_data, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_GPIO8_CLKCTRL, dra7_gpio8_bit_data, CLKF_HW_SUP, "l3_iclk_div" },
+	{ DRA7_MMC3_CLKCTRL, dra7_mmc3_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0120:25" },
+	{ DRA7_MMC4_CLKCTRL, dra7_mmc4_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0128:25" },
+	{ DRA7_TIMER16_CLKCTRL, dra7_timer16_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0130:24", "l4per3_clkdm" },
+	{ DRA7_QSPI_CLKCTRL, dra7_qspi_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0138:25", "l4per2_clkdm" },
+	{ DRA7_UART1_CLKCTRL, dra7_uart1_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0140:24" },
+	{ DRA7_UART2_CLKCTRL, dra7_uart2_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0148:24" },
+	{ DRA7_UART3_CLKCTRL, dra7_uart3_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0150:24" },
+	{ DRA7_UART4_CLKCTRL, dra7_uart4_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0158:24" },
+	{ DRA7_MCASP2_CLKCTRL, dra7_mcasp2_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0160:22", "l4per2_clkdm" },
+	{ DRA7_MCASP3_CLKCTRL, dra7_mcasp3_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0168:22", "l4per2_clkdm" },
+	{ DRA7_UART5_CLKCTRL, dra7_uart5_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0170:24" },
+	{ DRA7_MCASP5_CLKCTRL, dra7_mcasp5_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0178:22", "l4per2_clkdm" },
+	{ DRA7_MCASP8_CLKCTRL, dra7_mcasp8_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0190:24", "l4per2_clkdm" },
+	{ DRA7_MCASP4_CLKCTRL, dra7_mcasp4_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0198:22", "l4per2_clkdm" },
+	{ DRA7_AES1_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
+	{ DRA7_AES2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
+	{ DRA7_DES_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
+	{ DRA7_RNG_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
+	{ DRA7_SHAM_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
+	{ DRA7_UART7_CLKCTRL, dra7_uart7_bit_data, CLKF_SW_SUP, "l4per_cm:clk:01d0:24", "l4per2_clkdm" },
+	{ DRA7_UART8_CLKCTRL, dra7_uart8_bit_data, CLKF_SW_SUP, "l4per_cm:clk:01e0:24", "l4per2_clkdm" },
+	{ DRA7_UART9_CLKCTRL, dra7_uart9_bit_data, CLKF_SW_SUP, "l4per_cm:clk:01e8:24", "l4per2_clkdm" },
+	{ DRA7_DCAN2_CLKCTRL, NULL, CLKF_SW_SUP, "sys_clkin1", "l4per2_clkdm" },
+	{ DRA7_MCASP6_CLKCTRL, dra7_mcasp6_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0204:22", "l4per2_clkdm" },
+	{ DRA7_MCASP7_CLKCTRL, dra7_mcasp7_bit_data, CLKF_SW_SUP, "l4per_cm:clk:0208:22", "l4per2_clkdm" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_gpio1_bit_data[] __initconst = {
+	{ 8, TI_CLK_GATE, dra7_dss_32khz_clk_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_timer1_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_timer10_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_bit_data dra7_uart10_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_uart6_gfclk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const char * const dra7_dcan1_sys_clk_mux_parents[] __initconst = {
+	"sys_clkin1",
+	"sys_clkin2",
+	NULL,
+};
+
+static const struct omap_clkctrl_bit_data dra7_dcan1_bit_data[] __initconst = {
+	{ 24, TI_CLK_MUX, dra7_dcan1_sys_clk_mux_parents, NULL },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dra7_wkupaon_clkctrl_regs[] __initconst = {
+	{ DRA7_L4_WKUP_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
+	{ DRA7_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
+	{ DRA7_GPIO1_CLKCTRL, dra7_gpio1_bit_data, CLKF_HW_SUP, "wkupaon_iclk_mux" },
+	{ DRA7_TIMER1_CLKCTRL, dra7_timer1_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0020:24" },
+	{ DRA7_TIMER12_CLKCTRL, NULL, 0, "secure_32k_clk_src_ck" },
+	{ DRA7_COUNTER_32K_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
+	{ DRA7_UART10_CLKCTRL, dra7_uart10_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0060:24" },
+	{ DRA7_DCAN1_CLKCTRL, dra7_dcan1_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0068:24" },
+	{ 0 },
+};
+
+const struct omap_clkctrl_data dra7_clkctrl_data[] __initconst = {
+	{ 0x4a005320, dra7_mpu_clkctrl_regs },
+	{ 0x4a005540, dra7_ipu_clkctrl_regs },
+	{ 0x4a005740, dra7_rtc_clkctrl_regs },
+	{ 0x4a008620, dra7_coreaon_clkctrl_regs },
+	{ 0x4a008720, dra7_l3main1_clkctrl_regs },
+	{ 0x4a008a20, dra7_dma_clkctrl_regs },
+	{ 0x4a008b20, dra7_emif_clkctrl_regs },
+	{ 0x4a008c00, dra7_atl_clkctrl_regs },
+	{ 0x4a008d20, dra7_l4cfg_clkctrl_regs },
+	{ 0x4a008e20, dra7_l3instr_clkctrl_regs },
+	{ 0x4a009120, dra7_dss_clkctrl_regs },
+	{ 0x4a009320, dra7_l3init_clkctrl_regs },
+	{ 0x4a009700, dra7_l4per_clkctrl_regs },
+	{ 0x4ae07820, dra7_wkupaon_clkctrl_regs },
+	{ 0 },
+};
+
 static struct ti_dt_clk dra7xx_clks[] = {
-	DT_CLK(NULL, "atl_clkin0_ck", "atl_clkin0_ck"),
-	DT_CLK(NULL, "atl_clkin1_ck", "atl_clkin1_ck"),
-	DT_CLK(NULL, "atl_clkin2_ck", "atl_clkin2_ck"),
-	DT_CLK(NULL, "atl_clkin3_ck", "atl_clkin3_ck"),
-	DT_CLK(NULL, "hdmi_clkin_ck", "hdmi_clkin_ck"),
-	DT_CLK(NULL, "mlb_clkin_ck", "mlb_clkin_ck"),
-	DT_CLK(NULL, "mlbp_clkin_ck", "mlbp_clkin_ck"),
-	DT_CLK(NULL, "pciesref_acs_clk_ck", "pciesref_acs_clk_ck"),
-	DT_CLK(NULL, "ref_clkin0_ck", "ref_clkin0_ck"),
-	DT_CLK(NULL, "ref_clkin1_ck", "ref_clkin1_ck"),
-	DT_CLK(NULL, "ref_clkin2_ck", "ref_clkin2_ck"),
-	DT_CLK(NULL, "ref_clkin3_ck", "ref_clkin3_ck"),
-	DT_CLK(NULL, "rmii_clk_ck", "rmii_clk_ck"),
-	DT_CLK(NULL, "sdvenc_clkin_ck", "sdvenc_clkin_ck"),
-	DT_CLK(NULL, "secure_32k_clk_src_ck", "secure_32k_clk_src_ck"),
-	DT_CLK(NULL, "sys_32k_ck", "sys_32k_ck"),
-	DT_CLK(NULL, "virt_12000000_ck", "virt_12000000_ck"),
-	DT_CLK(NULL, "virt_13000000_ck", "virt_13000000_ck"),
-	DT_CLK(NULL, "virt_16800000_ck", "virt_16800000_ck"),
-	DT_CLK(NULL, "virt_19200000_ck", "virt_19200000_ck"),
-	DT_CLK(NULL, "virt_20000000_ck", "virt_20000000_ck"),
-	DT_CLK(NULL, "virt_26000000_ck", "virt_26000000_ck"),
-	DT_CLK(NULL, "virt_27000000_ck", "virt_27000000_ck"),
-	DT_CLK(NULL, "virt_38400000_ck", "virt_38400000_ck"),
-	DT_CLK(NULL, "sys_clkin1", "sys_clkin1"),
-	DT_CLK(NULL, "sys_clkin2", "sys_clkin2"),
-	DT_CLK(NULL, "usb_otg_clkin_ck", "usb_otg_clkin_ck"),
-	DT_CLK(NULL, "video1_clkin_ck", "video1_clkin_ck"),
-	DT_CLK(NULL, "video1_m2_clkin_ck", "video1_m2_clkin_ck"),
-	DT_CLK(NULL, "video2_clkin_ck", "video2_clkin_ck"),
-	DT_CLK(NULL, "video2_m2_clkin_ck", "video2_m2_clkin_ck"),
-	DT_CLK(NULL, "abe_dpll_sys_clk_mux", "abe_dpll_sys_clk_mux"),
-	DT_CLK(NULL, "abe_dpll_bypass_clk_mux", "abe_dpll_bypass_clk_mux"),
-	DT_CLK(NULL, "abe_dpll_clk_mux", "abe_dpll_clk_mux"),
-	DT_CLK(NULL, "dpll_abe_ck", "dpll_abe_ck"),
-	DT_CLK(NULL, "dpll_abe_x2_ck", "dpll_abe_x2_ck"),
-	DT_CLK(NULL, "dpll_abe_m2x2_ck", "dpll_abe_m2x2_ck"),
-	DT_CLK(NULL, "abe_24m_fclk", "abe_24m_fclk"),
-	DT_CLK(NULL, "abe_clk", "abe_clk"),
-	DT_CLK(NULL, "aess_fclk", "aess_fclk"),
-	DT_CLK(NULL, "abe_giclk_div", "abe_giclk_div"),
-	DT_CLK(NULL, "abe_lp_clk_div", "abe_lp_clk_div"),
-	DT_CLK(NULL, "abe_sys_clk_div", "abe_sys_clk_div"),
-	DT_CLK(NULL, "adc_gfclk_mux", "adc_gfclk_mux"),
-	DT_CLK(NULL, "dpll_pcie_ref_ck", "dpll_pcie_ref_ck"),
-	DT_CLK(NULL, "dpll_pcie_ref_m2ldo_ck", "dpll_pcie_ref_m2ldo_ck"),
-	DT_CLK(NULL, "apll_pcie_ck", "apll_pcie_ck"),
-	DT_CLK(NULL, "apll_pcie_clkvcoldo", "apll_pcie_clkvcoldo"),
-	DT_CLK(NULL, "apll_pcie_clkvcoldo_div", "apll_pcie_clkvcoldo_div"),
-	DT_CLK(NULL, "apll_pcie_m2_ck", "apll_pcie_m2_ck"),
-	DT_CLK(NULL, "sys_clk1_dclk_div", "sys_clk1_dclk_div"),
-	DT_CLK(NULL, "sys_clk2_dclk_div", "sys_clk2_dclk_div"),
-	DT_CLK(NULL, "dpll_abe_m2_ck", "dpll_abe_m2_ck"),
-	DT_CLK(NULL, "per_abe_x1_dclk_div", "per_abe_x1_dclk_div"),
-	DT_CLK(NULL, "dpll_abe_m3x2_ck", "dpll_abe_m3x2_ck"),
-	DT_CLK(NULL, "dpll_core_ck", "dpll_core_ck"),
-	DT_CLK(NULL, "dpll_core_x2_ck", "dpll_core_x2_ck"),
-	DT_CLK(NULL, "dpll_core_h12x2_ck", "dpll_core_h12x2_ck"),
-	DT_CLK(NULL, "mpu_dpll_hs_clk_div", "mpu_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_mpu_ck", "dpll_mpu_ck"),
-	DT_CLK(NULL, "dpll_mpu_m2_ck", "dpll_mpu_m2_ck"),
-	DT_CLK(NULL, "mpu_dclk_div", "mpu_dclk_div"),
-	DT_CLK(NULL, "dsp_dpll_hs_clk_div", "dsp_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_dsp_ck", "dpll_dsp_ck"),
-	DT_CLK(NULL, "dpll_dsp_m2_ck", "dpll_dsp_m2_ck"),
-	DT_CLK(NULL, "dsp_gclk_div", "dsp_gclk_div"),
-	DT_CLK(NULL, "iva_dpll_hs_clk_div", "iva_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_iva_ck", "dpll_iva_ck"),
-	DT_CLK(NULL, "dpll_iva_m2_ck", "dpll_iva_m2_ck"),
-	DT_CLK(NULL, "iva_dclk", "iva_dclk"),
-	DT_CLK(NULL, "dpll_gpu_ck", "dpll_gpu_ck"),
-	DT_CLK(NULL, "dpll_gpu_m2_ck", "dpll_gpu_m2_ck"),
-	DT_CLK(NULL, "gpu_dclk", "gpu_dclk"),
-	DT_CLK(NULL, "dpll_core_m2_ck", "dpll_core_m2_ck"),
-	DT_CLK(NULL, "core_dpll_out_dclk_div", "core_dpll_out_dclk_div"),
-	DT_CLK(NULL, "dpll_ddr_ck", "dpll_ddr_ck"),
-	DT_CLK(NULL, "dpll_ddr_m2_ck", "dpll_ddr_m2_ck"),
-	DT_CLK(NULL, "emif_phy_dclk_div", "emif_phy_dclk_div"),
-	DT_CLK(NULL, "dpll_gmac_ck", "dpll_gmac_ck"),
-	DT_CLK(NULL, "dpll_gmac_m2_ck", "dpll_gmac_m2_ck"),
-	DT_CLK(NULL, "gmac_250m_dclk_div", "gmac_250m_dclk_div"),
-	DT_CLK(NULL, "video2_dclk_div", "video2_dclk_div"),
-	DT_CLK(NULL, "video1_dclk_div", "video1_dclk_div"),
-	DT_CLK(NULL, "hdmi_dclk_div", "hdmi_dclk_div"),
-	DT_CLK(NULL, "per_dpll_hs_clk_div", "per_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_per_ck", "dpll_per_ck"),
-	DT_CLK(NULL, "dpll_per_m2_ck", "dpll_per_m2_ck"),
-	DT_CLK(NULL, "func_96m_aon_dclk_div", "func_96m_aon_dclk_div"),
-	DT_CLK(NULL, "usb_dpll_hs_clk_div", "usb_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_usb_ck", "dpll_usb_ck"),
-	DT_CLK(NULL, "dpll_usb_m2_ck", "dpll_usb_m2_ck"),
-	DT_CLK(NULL, "l3init_480m_dclk_div", "l3init_480m_dclk_div"),
-	DT_CLK(NULL, "usb_otg_dclk_div", "usb_otg_dclk_div"),
-	DT_CLK(NULL, "sata_dclk_div", "sata_dclk_div"),
-	DT_CLK(NULL, "dpll_pcie_ref_m2_ck", "dpll_pcie_ref_m2_ck"),
-	DT_CLK(NULL, "pcie2_dclk_div", "pcie2_dclk_div"),
-	DT_CLK(NULL, "pcie_dclk_div", "pcie_dclk_div"),
-	DT_CLK(NULL, "emu_dclk_div", "emu_dclk_div"),
-	DT_CLK(NULL, "secure_32k_dclk_div", "secure_32k_dclk_div"),
-	DT_CLK(NULL, "eve_dpll_hs_clk_div", "eve_dpll_hs_clk_div"),
-	DT_CLK(NULL, "dpll_eve_ck", "dpll_eve_ck"),
-	DT_CLK(NULL, "dpll_eve_m2_ck", "dpll_eve_m2_ck"),
-	DT_CLK(NULL, "eve_dclk_div", "eve_dclk_div"),
-	DT_CLK(NULL, "clkoutmux0_clk_mux", "clkoutmux0_clk_mux"),
-	DT_CLK(NULL, "clkoutmux1_clk_mux", "clkoutmux1_clk_mux"),
-	DT_CLK(NULL, "clkoutmux2_clk_mux", "clkoutmux2_clk_mux"),
-	DT_CLK(NULL, "custefuse_sys_gfclk_div", "custefuse_sys_gfclk_div"),
-	DT_CLK(NULL, "dpll_core_h13x2_ck", "dpll_core_h13x2_ck"),
-	DT_CLK(NULL, "dpll_core_h14x2_ck", "dpll_core_h14x2_ck"),
-	DT_CLK(NULL, "dpll_core_h22x2_ck", "dpll_core_h22x2_ck"),
-	DT_CLK(NULL, "dpll_core_h23x2_ck", "dpll_core_h23x2_ck"),
-	DT_CLK(NULL, "dpll_core_h24x2_ck", "dpll_core_h24x2_ck"),
-	DT_CLK(NULL, "dpll_ddr_x2_ck", "dpll_ddr_x2_ck"),
-	DT_CLK(NULL, "dpll_ddr_h11x2_ck", "dpll_ddr_h11x2_ck"),
-	DT_CLK(NULL, "dpll_dsp_x2_ck", "dpll_dsp_x2_ck"),
-	DT_CLK(NULL, "dpll_dsp_m3x2_ck", "dpll_dsp_m3x2_ck"),
-	DT_CLK(NULL, "dpll_gmac_x2_ck", "dpll_gmac_x2_ck"),
-	DT_CLK(NULL, "dpll_gmac_h11x2_ck", "dpll_gmac_h11x2_ck"),
-	DT_CLK(NULL, "dpll_gmac_h12x2_ck", "dpll_gmac_h12x2_ck"),
-	DT_CLK(NULL, "dpll_gmac_h13x2_ck", "dpll_gmac_h13x2_ck"),
-	DT_CLK(NULL, "dpll_gmac_m3x2_ck", "dpll_gmac_m3x2_ck"),
-	DT_CLK(NULL, "dpll_per_x2_ck", "dpll_per_x2_ck"),
-	DT_CLK(NULL, "dpll_per_h11x2_ck", "dpll_per_h11x2_ck"),
-	DT_CLK(NULL, "dpll_per_h12x2_ck", "dpll_per_h12x2_ck"),
-	DT_CLK(NULL, "dpll_per_h13x2_ck", "dpll_per_h13x2_ck"),
-	DT_CLK(NULL, "dpll_per_h14x2_ck", "dpll_per_h14x2_ck"),
-	DT_CLK(NULL, "dpll_per_m2x2_ck", "dpll_per_m2x2_ck"),
-	DT_CLK(NULL, "dpll_usb_clkdcoldo", "dpll_usb_clkdcoldo"),
-	DT_CLK(NULL, "eve_clk", "eve_clk"),
-	DT_CLK(NULL, "func_128m_clk", "func_128m_clk"),
-	DT_CLK(NULL, "func_12m_fclk", "func_12m_fclk"),
-	DT_CLK(NULL, "func_24m_clk", "func_24m_clk"),
-	DT_CLK(NULL, "func_48m_fclk", "func_48m_fclk"),
-	DT_CLK(NULL, "func_96m_fclk", "func_96m_fclk"),
-	DT_CLK(NULL, "gmii_m_clk_div", "gmii_m_clk_div"),
-	DT_CLK(NULL, "hdmi_clk2_div", "hdmi_clk2_div"),
-	DT_CLK(NULL, "hdmi_div_clk", "hdmi_div_clk"),
-	DT_CLK(NULL, "hdmi_dpll_clk_mux", "hdmi_dpll_clk_mux"),
-	DT_CLK(NULL, "l3_iclk_div", "l3_iclk_div"),
-	DT_CLK(NULL, "l3init_60m_fclk", "l3init_60m_fclk"),
-	DT_CLK(NULL, "l4_root_clk_div", "l4_root_clk_div"),
-	DT_CLK(NULL, "mlb_clk", "mlb_clk"),
-	DT_CLK(NULL, "mlbp_clk", "mlbp_clk"),
-	DT_CLK(NULL, "per_abe_x1_gfclk2_div", "per_abe_x1_gfclk2_div"),
-	DT_CLK(NULL, "timer_sys_clk_div", "timer_sys_clk_div"),
-	DT_CLK(NULL, "video1_clk2_div", "video1_clk2_div"),
-	DT_CLK(NULL, "video1_div_clk", "video1_div_clk"),
-	DT_CLK(NULL, "video1_dpll_clk_mux", "video1_dpll_clk_mux"),
-	DT_CLK(NULL, "video2_clk2_div", "video2_clk2_div"),
-	DT_CLK(NULL, "video2_div_clk", "video2_div_clk"),
-	DT_CLK(NULL, "video2_dpll_clk_mux", "video2_dpll_clk_mux"),
-	DT_CLK(NULL, "wkupaon_iclk_mux", "wkupaon_iclk_mux"),
-	DT_CLK(NULL, "dss_32khz_clk", "dss_32khz_clk"),
-	DT_CLK(NULL, "dss_48mhz_clk", "dss_48mhz_clk"),
-	DT_CLK(NULL, "dss_dss_clk", "dss_dss_clk"),
-	DT_CLK(NULL, "dss_hdmi_clk", "dss_hdmi_clk"),
-	DT_CLK(NULL, "dss_video1_clk", "dss_video1_clk"),
-	DT_CLK(NULL, "dss_video2_clk", "dss_video2_clk"),
-	DT_CLK(NULL, "gpio1_dbclk", "gpio1_dbclk"),
-	DT_CLK(NULL, "gpio2_dbclk", "gpio2_dbclk"),
-	DT_CLK(NULL, "gpio3_dbclk", "gpio3_dbclk"),
-	DT_CLK(NULL, "gpio4_dbclk", "gpio4_dbclk"),
-	DT_CLK(NULL, "gpio5_dbclk", "gpio5_dbclk"),
-	DT_CLK(NULL, "gpio6_dbclk", "gpio6_dbclk"),
-	DT_CLK(NULL, "gpio7_dbclk", "gpio7_dbclk"),
-	DT_CLK(NULL, "gpio8_dbclk", "gpio8_dbclk"),
-	DT_CLK(NULL, "mmc1_clk32k", "mmc1_clk32k"),
-	DT_CLK(NULL, "mmc2_clk32k", "mmc2_clk32k"),
-	DT_CLK(NULL, "mmc3_clk32k", "mmc3_clk32k"),
-	DT_CLK(NULL, "mmc4_clk32k", "mmc4_clk32k"),
-	DT_CLK(NULL, "sata_ref_clk", "sata_ref_clk"),
-	DT_CLK(NULL, "usb_otg_ss1_refclk960m", "usb_otg_ss1_refclk960m"),
-	DT_CLK(NULL, "usb_otg_ss2_refclk960m", "usb_otg_ss2_refclk960m"),
-	DT_CLK(NULL, "usb_phy1_always_on_clk32k", "usb_phy1_always_on_clk32k"),
-	DT_CLK(NULL, "usb_phy2_always_on_clk32k", "usb_phy2_always_on_clk32k"),
-	DT_CLK(NULL, "usb_phy3_always_on_clk32k", "usb_phy3_always_on_clk32k"),
-	DT_CLK(NULL, "atl_dpll_clk_mux", "atl_dpll_clk_mux"),
-	DT_CLK(NULL, "atl_gfclk_mux", "atl_gfclk_mux"),
-	DT_CLK(NULL, "dcan1_sys_clk_mux", "dcan1_sys_clk_mux"),
-	DT_CLK(NULL, "gmac_rft_clk_mux", "gmac_rft_clk_mux"),
-	DT_CLK(NULL, "gpu_core_gclk_mux", "gpu_core_gclk_mux"),
-	DT_CLK(NULL, "gpu_hyd_gclk_mux", "gpu_hyd_gclk_mux"),
-	DT_CLK(NULL, "ipu1_gfclk_mux", "ipu1_gfclk_mux"),
-	DT_CLK(NULL, "l3instr_ts_gclk_div", "l3instr_ts_gclk_div"),
-	DT_CLK(NULL, "mcasp1_ahclkr_mux", "mcasp1_ahclkr_mux"),
-	DT_CLK(NULL, "mcasp1_ahclkx_mux", "mcasp1_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp1_aux_gfclk_mux", "mcasp1_aux_gfclk_mux"),
-	DT_CLK(NULL, "mcasp2_ahclkr_mux", "mcasp2_ahclkr_mux"),
-	DT_CLK(NULL, "mcasp2_ahclkx_mux", "mcasp2_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp2_aux_gfclk_mux", "mcasp2_aux_gfclk_mux"),
-	DT_CLK(NULL, "mcasp3_ahclkx_mux", "mcasp3_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp3_aux_gfclk_mux", "mcasp3_aux_gfclk_mux"),
-	DT_CLK(NULL, "mcasp4_ahclkx_mux", "mcasp4_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp4_aux_gfclk_mux", "mcasp4_aux_gfclk_mux"),
-	DT_CLK(NULL, "mcasp5_ahclkx_mux", "mcasp5_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp5_aux_gfclk_mux", "mcasp5_aux_gfclk_mux"),
-	DT_CLK(NULL, "mcasp6_ahclkx_mux", "mcasp6_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp6_aux_gfclk_mux", "mcasp6_aux_gfclk_mux"),
-	DT_CLK(NULL, "mcasp7_ahclkx_mux", "mcasp7_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp7_aux_gfclk_mux", "mcasp7_aux_gfclk_mux"),
-	DT_CLK(NULL, "mcasp8_ahclkx_mux", "mcasp8_ahclkx_mux"),
-	DT_CLK(NULL, "mcasp8_aux_gfclk_mux", "mcasp8_aux_gfclk_mux"),
-	DT_CLK(NULL, "mmc1_fclk_mux", "mmc1_fclk_mux"),
-	DT_CLK(NULL, "mmc1_fclk_div", "mmc1_fclk_div"),
-	DT_CLK(NULL, "mmc2_fclk_mux", "mmc2_fclk_mux"),
-	DT_CLK(NULL, "mmc2_fclk_div", "mmc2_fclk_div"),
-	DT_CLK(NULL, "mmc3_gfclk_mux", "mmc3_gfclk_mux"),
-	DT_CLK(NULL, "mmc3_gfclk_div", "mmc3_gfclk_div"),
-	DT_CLK(NULL, "mmc4_gfclk_mux", "mmc4_gfclk_mux"),
-	DT_CLK(NULL, "mmc4_gfclk_div", "mmc4_gfclk_div"),
-	DT_CLK(NULL, "qspi_gfclk_mux", "qspi_gfclk_mux"),
-	DT_CLK(NULL, "qspi_gfclk_div", "qspi_gfclk_div"),
-	DT_CLK(NULL, "timer10_gfclk_mux", "timer10_gfclk_mux"),
-	DT_CLK(NULL, "timer11_gfclk_mux", "timer11_gfclk_mux"),
-	DT_CLK(NULL, "timer13_gfclk_mux", "timer13_gfclk_mux"),
-	DT_CLK(NULL, "timer14_gfclk_mux", "timer14_gfclk_mux"),
-	DT_CLK(NULL, "timer15_gfclk_mux", "timer15_gfclk_mux"),
-	DT_CLK(NULL, "timer16_gfclk_mux", "timer16_gfclk_mux"),
-	DT_CLK(NULL, "timer1_gfclk_mux", "timer1_gfclk_mux"),
-	DT_CLK(NULL, "timer2_gfclk_mux", "timer2_gfclk_mux"),
-	DT_CLK(NULL, "timer3_gfclk_mux", "timer3_gfclk_mux"),
-	DT_CLK(NULL, "timer4_gfclk_mux", "timer4_gfclk_mux"),
-	DT_CLK(NULL, "timer5_gfclk_mux", "timer5_gfclk_mux"),
-	DT_CLK(NULL, "timer6_gfclk_mux", "timer6_gfclk_mux"),
-	DT_CLK(NULL, "timer7_gfclk_mux", "timer7_gfclk_mux"),
-	DT_CLK(NULL, "timer8_gfclk_mux", "timer8_gfclk_mux"),
-	DT_CLK(NULL, "timer9_gfclk_mux", "timer9_gfclk_mux"),
-	DT_CLK(NULL, "uart10_gfclk_mux", "uart10_gfclk_mux"),
-	DT_CLK(NULL, "uart1_gfclk_mux", "uart1_gfclk_mux"),
-	DT_CLK(NULL, "uart2_gfclk_mux", "uart2_gfclk_mux"),
-	DT_CLK(NULL, "uart3_gfclk_mux", "uart3_gfclk_mux"),
-	DT_CLK(NULL, "uart4_gfclk_mux", "uart4_gfclk_mux"),
-	DT_CLK(NULL, "uart5_gfclk_mux", "uart5_gfclk_mux"),
-	DT_CLK(NULL, "uart6_gfclk_mux", "uart6_gfclk_mux"),
-	DT_CLK(NULL, "uart7_gfclk_mux", "uart7_gfclk_mux"),
-	DT_CLK(NULL, "uart8_gfclk_mux", "uart8_gfclk_mux"),
-	DT_CLK(NULL, "uart9_gfclk_mux", "uart9_gfclk_mux"),
-	DT_CLK(NULL, "vip1_gclk_mux", "vip1_gclk_mux"),
-	DT_CLK(NULL, "vip2_gclk_mux", "vip2_gclk_mux"),
-	DT_CLK(NULL, "vip3_gclk_mux", "vip3_gclk_mux"),
-	DT_CLK("omap_i2c.1", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.2", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.3", "ick", "dummy_ck"),
-	DT_CLK("omap_i2c.4", "ick", "dummy_ck"),
-	DT_CLK(NULL, "mailboxes_ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.0", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.1", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.2", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.3", "ick", "dummy_ck"),
-	DT_CLK("omap_hsmmc.4", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.1", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.2", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.3", "ick", "dummy_ck"),
-	DT_CLK("omap-mcbsp.4", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.1", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.2", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.3", "ick", "dummy_ck"),
-	DT_CLK("omap2_mcspi.4", "ick", "dummy_ck"),
-	DT_CLK(NULL, "uart1_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart2_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart3_ick", "dummy_ck"),
-	DT_CLK(NULL, "uart4_ick", "dummy_ck"),
-	DT_CLK("usbhs_omap", "usbhost_ick", "dummy_ck"),
-	DT_CLK("usbhs_omap", "usbtll_fck", "dummy_ck"),
-	DT_CLK("omap_wdt", "ick", "dummy_ck"),
 	DT_CLK(NULL, "timer_32k_ck", "sys_32k_ck"),
 	DT_CLK(NULL, "sys_clkin_ck", "timer_sys_clk_div"),
-	DT_CLK("4ae18000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48032000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48034000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48036000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("4803e000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48086000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48088000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48820000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48822000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48824000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48826000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("48828000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("4882a000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("4882c000.timer", "timer_sys_ck", "timer_sys_clk_div"),
-	DT_CLK("4882e000.timer", "timer_sys_ck", "timer_sys_clk_div"),
 	DT_CLK(NULL, "sys_clkin", "sys_clkin1"),
-	DT_CLK(NULL, "dss_deshdcp_clk", "dss_deshdcp_clk"),
+	DT_CLK(NULL, "atl_dpll_clk_mux", "atl_cm:0000:24"),
+	DT_CLK(NULL, "atl_gfclk_mux", "atl_cm:0000:26"),
+	DT_CLK(NULL, "dcan1_sys_clk_mux", "wkupaon_cm:0068:24"),
+	DT_CLK(NULL, "dss_32khz_clk", "dss_cm:0000:11"),
+	DT_CLK(NULL, "dss_48mhz_clk", "dss_cm:0000:9"),
+	DT_CLK(NULL, "dss_dss_clk", "dss_cm:0000:8"),
+	DT_CLK(NULL, "dss_hdmi_clk", "dss_cm:0000:10"),
+	DT_CLK(NULL, "dss_video1_clk", "dss_cm:0000:12"),
+	DT_CLK(NULL, "dss_video2_clk", "dss_cm:0000:13"),
+	DT_CLK(NULL, "gmac_rft_clk_mux", "l3init_cm:00b0:25"),
+	DT_CLK(NULL, "gpio1_dbclk", "wkupaon_cm:0018:8"),
+	DT_CLK(NULL, "gpio2_dbclk", "l4per_cm:0060:8"),
+	DT_CLK(NULL, "gpio3_dbclk", "l4per_cm:0068:8"),
+	DT_CLK(NULL, "gpio4_dbclk", "l4per_cm:0070:8"),
+	DT_CLK(NULL, "gpio5_dbclk", "l4per_cm:0078:8"),
+	DT_CLK(NULL, "gpio6_dbclk", "l4per_cm:0080:8"),
+	DT_CLK(NULL, "gpio7_dbclk", "l4per_cm:0110:8"),
+	DT_CLK(NULL, "gpio8_dbclk", "l4per_cm:0118:8"),
+	DT_CLK(NULL, "mcasp1_ahclkr_mux", "ipu_cm:0010:28"),
+	DT_CLK(NULL, "mcasp1_ahclkx_mux", "ipu_cm:0010:24"),
+	DT_CLK(NULL, "mcasp1_aux_gfclk_mux", "ipu_cm:0010:22"),
+	DT_CLK(NULL, "mcasp2_ahclkr_mux", "l4per_cm:0160:28"),
+	DT_CLK(NULL, "mcasp2_ahclkx_mux", "l4per_cm:0160:24"),
+	DT_CLK(NULL, "mcasp2_aux_gfclk_mux", "l4per_cm:0160:22"),
+	DT_CLK(NULL, "mcasp3_ahclkx_mux", "l4per_cm:0168:24"),
+	DT_CLK(NULL, "mcasp3_aux_gfclk_mux", "l4per_cm:0168:22"),
+	DT_CLK(NULL, "mcasp4_ahclkx_mux", "l4per_cm:0198:24"),
+	DT_CLK(NULL, "mcasp4_aux_gfclk_mux", "l4per_cm:0198:22"),
+	DT_CLK(NULL, "mcasp5_ahclkx_mux", "l4per_cm:0178:24"),
+	DT_CLK(NULL, "mcasp5_aux_gfclk_mux", "l4per_cm:0178:22"),
+	DT_CLK(NULL, "mcasp6_ahclkx_mux", "l4per_cm:0204:24"),
+	DT_CLK(NULL, "mcasp6_aux_gfclk_mux", "l4per_cm:0204:22"),
+	DT_CLK(NULL, "mcasp7_ahclkx_mux", "l4per_cm:0208:24"),
+	DT_CLK(NULL, "mcasp7_aux_gfclk_mux", "l4per_cm:0208:22"),
+	DT_CLK(NULL, "mcasp8_ahclkx_mux", "l4per_cm:0190:22"),
+	DT_CLK(NULL, "mcasp8_aux_gfclk_mux", "l4per_cm:0190:24"),
+	DT_CLK(NULL, "mmc1_clk32k", "l3init_cm:0008:8"),
+	DT_CLK(NULL, "mmc1_fclk_div", "l3init_cm:0008:25"),
+	DT_CLK(NULL, "mmc1_fclk_mux", "l3init_cm:0008:24"),
+	DT_CLK(NULL, "mmc2_clk32k", "l3init_cm:0010:8"),
+	DT_CLK(NULL, "mmc2_fclk_div", "l3init_cm:0010:25"),
+	DT_CLK(NULL, "mmc2_fclk_mux", "l3init_cm:0010:24"),
+	DT_CLK(NULL, "mmc3_clk32k", "l4per_cm:0120:8"),
+	DT_CLK(NULL, "mmc3_gfclk_div", "l4per_cm:0120:25"),
+	DT_CLK(NULL, "mmc3_gfclk_mux", "l4per_cm:0120:24"),
+	DT_CLK(NULL, "mmc4_clk32k", "l4per_cm:0128:8"),
+	DT_CLK(NULL, "mmc4_gfclk_div", "l4per_cm:0128:25"),
+	DT_CLK(NULL, "mmc4_gfclk_mux", "l4per_cm:0128:24"),
+	DT_CLK(NULL, "optfclk_pciephy1_32khz", "l3init_cm:0090:8"),
+	DT_CLK(NULL, "optfclk_pciephy1_clk", "l3init_cm:0090:9"),
+	DT_CLK(NULL, "optfclk_pciephy1_div_clk", "l3init_cm:0090:10"),
+	DT_CLK(NULL, "optfclk_pciephy2_32khz", "l3init_cm:0098:8"),
+	DT_CLK(NULL, "optfclk_pciephy2_clk", "l3init_cm:0098:9"),
+	DT_CLK(NULL, "optfclk_pciephy2_div_clk", "l3init_cm:0098:10"),
+	DT_CLK(NULL, "qspi_gfclk_div", "l4per_cm:0138:25"),
+	DT_CLK(NULL, "qspi_gfclk_mux", "l4per_cm:0138:24"),
+	DT_CLK(NULL, "rmii_50mhz_clk_mux", "l3init_cm:00b0:24"),
+	DT_CLK(NULL, "sata_ref_clk", "l3init_cm:0068:8"),
+	DT_CLK(NULL, "timer10_gfclk_mux", "l4per_cm:0028:24"),
+	DT_CLK(NULL, "timer11_gfclk_mux", "l4per_cm:0030:24"),
+	DT_CLK(NULL, "timer13_gfclk_mux", "l4per_cm:00c8:24"),
+	DT_CLK(NULL, "timer14_gfclk_mux", "l4per_cm:00d0:24"),
+	DT_CLK(NULL, "timer15_gfclk_mux", "l4per_cm:00d8:24"),
+	DT_CLK(NULL, "timer16_gfclk_mux", "l4per_cm:0130:24"),
+	DT_CLK(NULL, "timer1_gfclk_mux", "wkupaon_cm:0020:24"),
+	DT_CLK(NULL, "timer2_gfclk_mux", "l4per_cm:0038:24"),
+	DT_CLK(NULL, "timer3_gfclk_mux", "l4per_cm:0040:24"),
+	DT_CLK(NULL, "timer4_gfclk_mux", "l4per_cm:0048:24"),
+	DT_CLK(NULL, "timer5_gfclk_mux", "ipu_cm:0018:24"),
+	DT_CLK(NULL, "timer6_gfclk_mux", "ipu_cm:0020:24"),
+	DT_CLK(NULL, "timer7_gfclk_mux", "ipu_cm:0028:24"),
+	DT_CLK(NULL, "timer8_gfclk_mux", "ipu_cm:0030:24"),
+	DT_CLK(NULL, "timer9_gfclk_mux", "l4per_cm:0050:24"),
+	DT_CLK(NULL, "uart10_gfclk_mux", "wkupaon_cm:0060:24"),
+	DT_CLK(NULL, "uart1_gfclk_mux", "l4per_cm:0140:24"),
+	DT_CLK(NULL, "uart2_gfclk_mux", "l4per_cm:0148:24"),
+	DT_CLK(NULL, "uart3_gfclk_mux", "l4per_cm:0150:24"),
+	DT_CLK(NULL, "uart4_gfclk_mux", "l4per_cm:0158:24"),
+	DT_CLK(NULL, "uart5_gfclk_mux", "l4per_cm:0170:24"),
+	DT_CLK(NULL, "uart6_gfclk_mux", "ipu_cm:0040:24"),
+	DT_CLK(NULL, "uart7_gfclk_mux", "l4per_cm:01d0:24"),
+	DT_CLK(NULL, "uart8_gfclk_mux", "l4per_cm:01e0:24"),
+	DT_CLK(NULL, "uart9_gfclk_mux", "l4per_cm:01e8:24"),
+	DT_CLK(NULL, "usb_otg_ss1_refclk960m", "l3init_cm:00d0:8"),
+	DT_CLK(NULL, "usb_otg_ss2_refclk960m", "l3init_cm:0020:8"),
 	{ .node_name = NULL },
 };
 
@@ -318,6 +830,8 @@ int __init dra7xx_dt_clk_init(void)
 
 	omap2_clk_disable_autoidle_all();
 
+	ti_clk_add_aliases();
+
 	dpll_ck = clk_get_sys(NULL, "dpll_gmac_ck");
 	rc = clk_set_rate(dpll_ck, DRA7_DPLL_GMAC_DEFFREQ);
 	if (rc)
diff --git a/drivers/clk/ti/clk-814x.c b/drivers/clk/ti/clk-814x.c
index 52c6efc53731..f688fdd2cb59 100644
--- a/drivers/clk/ti/clk-814x.c
+++ b/drivers/clk/ti/clk-814x.c
@@ -9,23 +9,48 @@
 #include <linux/clk-provider.h>
 #include <linux/clk/ti.h>
 #include <linux/of_platform.h>
+#include <dt-bindings/clock/dm814.h>
 
 #include "clock.h"
 
+static const struct omap_clkctrl_reg_data dm814_default_clkctrl_regs[] __initconst = {
+	{ DM814_USB_OTG_HS_CLKCTRL, NULL, CLKF_SW_SUP, "pll260dcoclkldo" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dm814_alwon_clkctrl_regs[] __initconst = {
+	{ DM814_UART1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM814_UART2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM814_UART3_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM814_GPIO1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM814_GPIO2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM814_I2C1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM814_I2C2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM814_WD_TIMER_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "sysclk18_ck" },
+	{ DM814_MCSPI1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM814_GPMC_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM814_CPGMAC0_CLKCTRL, NULL, CLKF_SW_SUP, "cpsw_125mhz_gclk" },
+	{ DM814_MPU_CLKCTRL, NULL, CLKF_SW_SUP, "mpu_ck" },
+	{ DM814_RTC_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "sysclk18_ck" },
+	{ DM814_TPCC_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM814_TPTC0_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM814_TPTC1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM814_TPTC2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM814_TPTC3_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM814_MMC1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk8_ck" },
+	{ DM814_MMC2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk8_ck" },
+	{ DM814_MMC3_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk8_ck" },
+	{ 0 },
+};
+
+const struct omap_clkctrl_data dm814_clkctrl_data[] __initconst = {
+	{ 0x48180500, dm814_default_clkctrl_regs },
+	{ 0x48181400, dm814_alwon_clkctrl_regs },
+	{ 0 },
+};
+
 static struct ti_dt_clk dm814_clks[] = {
-	DT_CLK(NULL, "devosc_ck", "devosc_ck"),
-	DT_CLK(NULL, "mpu_ck", "mpu_ck"),
-	DT_CLK(NULL, "sysclk4_ck", "sysclk4_ck"),
-	DT_CLK(NULL, "sysclk5_ck", "sysclk5_ck"),
-	DT_CLK(NULL, "sysclk6_ck", "sysclk6_ck"),
-	DT_CLK(NULL, "sysclk8_ck", "sysclk8_ck"),
-	DT_CLK(NULL, "sysclk10_ck", "sysclk10_ck"),
-	DT_CLK(NULL, "sysclk18_ck", "sysclk18_ck"),
 	DT_CLK(NULL, "timer_sys_ck", "devosc_ck"),
-	DT_CLK(NULL, "timer1_fck", "timer1_fck"),
-	DT_CLK(NULL, "timer2_fck", "timer2_fck"),
-	DT_CLK(NULL, "cpsw_125mhz_gclk", "cpsw_125mhz_gclk"),
-	DT_CLK(NULL, "cpsw_cpts_rft_clk", "cpsw_cpts_rft_clk"),
 	{ .node_name = NULL },
 };
 
@@ -83,6 +108,7 @@ int __init dm814x_dt_clk_init(void)
 {
 	ti_dt_clocks_register(dm814_clks);
 	omap2_clk_disable_autoidle_all();
+	ti_clk_add_aliases();
 	omap2_clk_enable_init_clocks(NULL, 0);
 	timer_clocks_initialized = true;
 
diff --git a/drivers/clk/ti/clk-816x.c b/drivers/clk/ti/clk-816x.c
index 2a5d84fdddc5..7d215cdf9dda 100644
--- a/drivers/clk/ti/clk-816x.c
+++ b/drivers/clk/ti/clk-816x.c
@@ -13,30 +13,59 @@
 #include <linux/list.h>
 #include <linux/clk-provider.h>
 #include <linux/clk/ti.h>
+#include <dt-bindings/clock/dm816.h>
 
 #include "clock.h"
 
+static const struct omap_clkctrl_reg_data dm816_default_clkctrl_regs[] __initconst = {
+	{ DM816_USB_OTG_HS_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ 0 },
+};
+
+static const struct omap_clkctrl_reg_data dm816_alwon_clkctrl_regs[] __initconst = {
+	{ DM816_UART1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM816_UART2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM816_UART3_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM816_GPIO1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM816_GPIO2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM816_I2C1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM816_I2C2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM816_TIMER1_CLKCTRL, NULL, CLKF_SW_SUP, "timer1_fck" },
+	{ DM816_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "timer2_fck" },
+	{ DM816_TIMER3_CLKCTRL, NULL, CLKF_SW_SUP, "timer3_fck" },
+	{ DM816_TIMER4_CLKCTRL, NULL, CLKF_SW_SUP, "timer4_fck" },
+	{ DM816_TIMER5_CLKCTRL, NULL, CLKF_SW_SUP, "timer5_fck" },
+	{ DM816_TIMER6_CLKCTRL, NULL, CLKF_SW_SUP, "timer6_fck" },
+	{ DM816_TIMER7_CLKCTRL, NULL, CLKF_SW_SUP, "timer7_fck" },
+	{ DM816_WD_TIMER_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "sysclk18_ck" },
+	{ DM816_MCSPI1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM816_MAILBOX_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM816_SPINBOX_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM816_MMC1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk10_ck" },
+	{ DM816_GPMC_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk6_ck" },
+	{ DM816_DAVINCI_MDIO_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "sysclk24_ck" },
+	{ DM816_EMAC1_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "sysclk24_ck" },
+	{ DM816_MPU_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk2_ck" },
+	{ DM816_RTC_CLKCTRL, NULL, CLKF_SW_SUP | CLKF_NO_IDLEST, "sysclk18_ck" },
+	{ DM816_TPCC_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM816_TPTC0_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM816_TPTC1_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM816_TPTC2_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ DM816_TPTC3_CLKCTRL, NULL, CLKF_SW_SUP, "sysclk4_ck" },
+	{ 0 },
+};
+
+const struct omap_clkctrl_data dm816_clkctrl_data[] __initconst = {
+	{ 0x48180500, dm816_default_clkctrl_regs },
+	{ 0x48181400, dm816_alwon_clkctrl_regs },
+	{ 0 },
+};
+
 static struct ti_dt_clk dm816x_clks[] = {
 	DT_CLK(NULL, "sys_clkin", "sys_clkin_ck"),
 	DT_CLK(NULL, "timer_sys_ck", "sys_clkin_ck"),
-	DT_CLK(NULL, "sys_32k_ck", "sys_32k_ck"),
 	DT_CLK(NULL, "timer_32k_ck", "sysclk18_ck"),
 	DT_CLK(NULL, "timer_ext_ck", "tclkin_ck"),
-	DT_CLK(NULL, "mpu_ck", "mpu_ck"),
-	DT_CLK(NULL, "timer1_fck", "timer1_fck"),
-	DT_CLK(NULL, "timer2_fck", "timer2_fck"),
-	DT_CLK(NULL, "timer3_fck", "timer3_fck"),
-	DT_CLK(NULL, "timer4_fck", "timer4_fck"),
-	DT_CLK(NULL, "timer5_fck", "timer5_fck"),
-	DT_CLK(NULL, "timer6_fck", "timer6_fck"),
-	DT_CLK(NULL, "timer7_fck", "timer7_fck"),
-	DT_CLK(NULL, "sysclk4_ck", "sysclk4_ck"),
-	DT_CLK(NULL, "sysclk5_ck", "sysclk5_ck"),
-	DT_CLK(NULL, "sysclk6_ck", "sysclk6_ck"),
-	DT_CLK(NULL, "sysclk10_ck", "sysclk10_ck"),
-	DT_CLK(NULL, "sysclk18_ck", "sysclk18_ck"),
-	DT_CLK(NULL, "sysclk24_ck", "sysclk24_ck"),
-	DT_CLK("4a100000.ethernet", "sysclk24_ck", "sysclk24_ck"),
 	{ .node_name = NULL },
 };
 
@@ -50,6 +79,7 @@ int __init dm816x_dt_clk_init(void)
 {
 	ti_dt_clocks_register(dm816x_clks);
 	omap2_clk_disable_autoidle_all();
+	ti_clk_add_aliases();
 	omap2_clk_enable_init_clocks(enable_init_clks,
 				     ARRAY_SIZE(enable_init_clks));
 
diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c
index e5a1c8297a1d..f4d6802a8544 100644
--- a/drivers/clk/ti/clk.c
+++ b/drivers/clk/ti/clk.c
@@ -108,25 +108,77 @@ void __init ti_dt_clocks_register(struct ti_dt_clk oclks[])
 	struct device_node *node;
 	struct clk *clk;
 	struct of_phandle_args clkspec;
+	char buf[64];
+	char *ptr;
+	char *tags[2];
+	int i;
+	int num_args;
+	int ret;
+	static bool clkctrl_nodes_missing;
+	static bool has_clkctrl_data;
 
 	for (c = oclks; c->node_name != NULL; c++) {
-		node = of_find_node_by_name(NULL, c->node_name);
+		strcpy(buf, c->node_name);
+		ptr = buf;
+		for (i = 0; i < 2; i++)
+			tags[i] = NULL;
+		num_args = 0;
+		while (*ptr) {
+			if (*ptr == ':') {
+				if (num_args >= 2) {
+					pr_warn("Bad number of tags on %s\n",
+						c->node_name);
+					return;
+				}
+				tags[num_args++] = ptr + 1;
+				*ptr = 0;
+			}
+			ptr++;
+		}
+
+		if (num_args && clkctrl_nodes_missing)
+			continue;
+
+		node = of_find_node_by_name(NULL, buf);
+		if (num_args)
+			node = of_find_node_by_name(node, "clk");
 		clkspec.np = node;
+		clkspec.args_count = num_args;
+		for (i = 0; i < num_args; i++) {
+			ret = kstrtoint(tags[i], i ? 10 : 16, clkspec.args + i);
+			if (ret) {
+				pr_warn("Bad tag in %s at %d: %s\n",
+					c->node_name, i, tags[i]);
+				return;
+			}
+		}
 		clk = of_clk_get_from_provider(&clkspec);
 
 		if (!IS_ERR(clk)) {
 			c->lk.clk = clk;
 			clkdev_add(&c->lk);
 		} else {
-			pr_warn("failed to lookup clock node %s\n",
-				c->node_name);
+			if (num_args && !has_clkctrl_data) {
+				if (of_find_compatible_node(NULL, NULL,
+							    "ti,clkctrl")) {
+					has_clkctrl_data = true;
+				} else {
+					clkctrl_nodes_missing = true;
+
+					pr_warn("missing clkctrl nodes, please update your dts.\n");
+					continue;
+				}
+			}
+
+			pr_warn("failed to lookup clock node %s, ret=%ld\n",
+				c->node_name, PTR_ERR(clk));
 		}
 	}
 }
 
 struct clk_init_item {
 	struct device_node *node;
-	struct clk_hw *hw;
+	void *user;
 	ti_of_clk_init_cb_t func;
 	struct list_head link;
 };
@@ -136,14 +188,14 @@ static LIST_HEAD(retry_list);
 /**
  * ti_clk_retry_init - retries a failed clock init at later phase
  * @node: device not for the clock
- * @hw: partially initialized clk_hw struct for the clock
+ * @user: user data pointer
  * @func: init function to be called for the clock
  *
  * Adds a failed clock init to the retry list. The retry list is parsed
  * once all the other clocks have been initialized.
  */
-int __init ti_clk_retry_init(struct device_node *node, struct clk_hw *hw,
-			      ti_of_clk_init_cb_t func)
+int __init ti_clk_retry_init(struct device_node *node, void *user,
+			     ti_of_clk_init_cb_t func)
 {
 	struct clk_init_item *retry;
 
@@ -154,7 +206,7 @@ int __init ti_clk_retry_init(struct device_node *node, struct clk_hw *hw,
 
 	retry->node = node;
 	retry->func = func;
-	retry->hw = hw;
+	retry->user = user;
 	list_add(&retry->link, &retry_list);
 
 	return 0;
@@ -276,7 +328,7 @@ void ti_dt_clk_init_retry_clks(void)
 	while (!list_empty(&retry_list) && retries) {
 		list_for_each_entry_safe(retry, tmp, &retry_list, link) {
 			pr_debug("retry-init: %s\n", retry->node->name);
-			retry->func(retry->hw, retry->node);
+			retry->func(retry->user, retry->node);
 			list_del(&retry->link);
 			kfree(retry);
 		}
@@ -284,141 +336,6 @@ void ti_dt_clk_init_retry_clks(void)
 	}
 }
 
-#if defined(CONFIG_ARCH_OMAP3) && defined(CONFIG_ATAGS)
-void __init ti_clk_patch_legacy_clks(struct ti_clk **patch)
-{
-	while (*patch) {
-		memcpy((*patch)->patch, *patch, sizeof(**patch));
-		patch++;
-	}
-}
-
-struct clk __init *ti_clk_register_clk(struct ti_clk *setup)
-{
-	struct clk *clk;
-	struct ti_clk_fixed *fixed;
-	struct ti_clk_fixed_factor *fixed_factor;
-	struct clk_hw *clk_hw;
-	int ret;
-
-	if (setup->clk)
-		return setup->clk;
-
-	switch (setup->type) {
-	case TI_CLK_FIXED:
-		fixed = setup->data;
-
-		clk = clk_register_fixed_rate(NULL, setup->name, NULL, 0,
-					      fixed->frequency);
-		if (!IS_ERR(clk)) {
-			ret = ti_clk_add_alias(NULL, clk, setup->name);
-			if (ret) {
-				clk_unregister(clk);
-				clk = ERR_PTR(ret);
-			}
-		}
-		break;
-	case TI_CLK_MUX:
-		clk = ti_clk_register_mux(setup);
-		break;
-	case TI_CLK_DIVIDER:
-		clk = ti_clk_register_divider(setup);
-		break;
-	case TI_CLK_COMPOSITE:
-		clk = ti_clk_register_composite(setup);
-		break;
-	case TI_CLK_FIXED_FACTOR:
-		fixed_factor = setup->data;
-
-		clk = clk_register_fixed_factor(NULL, setup->name,
-						fixed_factor->parent,
-						0, fixed_factor->mult,
-						fixed_factor->div);
-		if (!IS_ERR(clk)) {
-			ret = ti_clk_add_alias(NULL, clk, setup->name);
-			if (ret) {
-				clk_unregister(clk);
-				clk = ERR_PTR(ret);
-			}
-		}
-		break;
-	case TI_CLK_GATE:
-		clk = ti_clk_register_gate(setup);
-		break;
-	case TI_CLK_DPLL:
-		clk = ti_clk_register_dpll(setup);
-		break;
-	default:
-		pr_err("bad type for %s!\n", setup->name);
-		clk = ERR_PTR(-EINVAL);
-	}
-
-	if (!IS_ERR(clk)) {
-		setup->clk = clk;
-		if (setup->clkdm_name) {
-			clk_hw = __clk_get_hw(clk);
-			if (clk_hw_get_flags(clk_hw) & CLK_IS_BASIC) {
-				pr_warn("can't setup clkdm for basic clk %s\n",
-					setup->name);
-			} else {
-				to_clk_hw_omap(clk_hw)->clkdm_name =
-					setup->clkdm_name;
-				omap2_init_clk_clkdm(clk_hw);
-			}
-		}
-	}
-
-	return clk;
-}
-
-int __init ti_clk_register_legacy_clks(struct ti_clk_alias *clks)
-{
-	struct clk *clk;
-	bool retry;
-	struct ti_clk_alias *retry_clk;
-	struct ti_clk_alias *tmp;
-
-	while (clks->clk) {
-		clk = ti_clk_register_clk(clks->clk);
-		if (IS_ERR(clk)) {
-			if (PTR_ERR(clk) == -EAGAIN) {
-				list_add(&clks->link, &retry_list);
-			} else {
-				pr_err("register for %s failed: %ld\n",
-				       clks->clk->name, PTR_ERR(clk));
-				return PTR_ERR(clk);
-			}
-		}
-		clks++;
-	}
-
-	retry = true;
-
-	while (!list_empty(&retry_list) && retry) {
-		retry = false;
-		list_for_each_entry_safe(retry_clk, tmp, &retry_list, link) {
-			pr_debug("retry-init: %s\n", retry_clk->clk->name);
-			clk = ti_clk_register_clk(retry_clk->clk);
-			if (IS_ERR(clk)) {
-				if (PTR_ERR(clk) == -EAGAIN) {
-					continue;
-				} else {
-					pr_err("register for %s failed: %ld\n",
-					       retry_clk->clk->name,
-					       PTR_ERR(clk));
-					return PTR_ERR(clk);
-				}
-			} else {
-				retry = true;
-				list_del(&retry_clk->link);
-			}
-		}
-	}
-
-	return 0;
-}
-#endif
-
 static const struct of_device_id simple_clk_match_table[] __initconst = {
 	{ .compatible = "fixed-clock" },
 	{ .compatible = "fixed-factor-clock" },
diff --git a/drivers/clk/ti/clkctrl.c b/drivers/clk/ti/clkctrl.c
index 53e71d0503ec..421b05392220 100644
--- a/drivers/clk/ti/clkctrl.c
+++ b/drivers/clk/ti/clkctrl.c
@@ -21,6 +21,7 @@
 #include <linux/of_address.h>
 #include <linux/clk/ti.h>
 #include <linux/delay.h>
+#include <linux/timekeeping.h>
 #include "clock.h"
 
 #define NO_IDLEST			0x1
@@ -46,6 +47,7 @@ static bool _early_timeout = true;
 struct omap_clkctrl_provider {
 	void __iomem *base;
 	struct list_head clocks;
+	char *clkdm_name;
 };
 
 struct omap_clkctrl_clk {
@@ -89,7 +91,18 @@ static bool _omap4_is_ready(u32 val)
 
 static bool _omap4_is_timeout(union omap4_timeout *time, u32 timeout)
 {
-	if (unlikely(_early_timeout)) {
+	/*
+	 * There are two special cases where ktime_to_ns() can't be
+	 * used to track the timeouts. First one is during early boot
+	 * when the timers haven't been initialized yet. The second
+	 * one is during suspend-resume cycle while timekeeping is
+	 * being suspended / resumed. Clocksource for the system
+	 * can be from a timer that requires pm_runtime access, which
+	 * will eventually bring us here with timekeeping_suspended,
+	 * during both suspend entry and resume paths. This happens
+	 * at least on am43xx platform.
+	 */
+	if (unlikely(_early_timeout || timekeeping_suspended)) {
 		if (time->cycles++ < timeout) {
 			udelay(1);
 			return false;
@@ -208,6 +221,7 @@ static const struct clk_ops omap4_clkctrl_clk_ops = {
 	.enable		= _omap4_clkctrl_clk_enable,
 	.disable	= _omap4_clkctrl_clk_disable,
 	.is_enabled	= _omap4_clkctrl_clk_is_enabled,
+	.init		= omap2_init_clk_clkdm,
 };
 
 static struct clk_hw *_ti_omap4_clkctrl_xlate(struct of_phandle_args *clkspec,
@@ -321,6 +335,9 @@ _ti_clkctrl_setup_mux(struct omap_clkctrl_provider *provider,
 	}
 
 	mux->mask = num_parents;
+	if (!(mux->flags & CLK_MUX_INDEX_ONE))
+		mux->mask--;
+
 	mux->mask = (1 << fls(mux->mask)) - 1;
 
 	mux->shift = data->bit;
@@ -340,6 +357,7 @@ _ti_clkctrl_setup_div(struct omap_clkctrl_provider *provider,
 {
 	struct clk_omap_divider *div;
 	const struct omap_clkctrl_div_data *div_data = data->data;
+	u8 div_flags = 0;
 
 	div = kzalloc(sizeof(*div), GFP_KERNEL);
 	if (!div)
@@ -347,12 +365,16 @@ _ti_clkctrl_setup_div(struct omap_clkctrl_provider *provider,
 
 	div->reg.ptr = reg;
 	div->shift = data->bit;
+	div->flags = div_data->flags;
+
+	if (div->flags & CLK_DIVIDER_POWER_OF_TWO)
+		div_flags |= CLKF_INDEX_POWER_OF_TWO;
 
-	if (ti_clk_parse_divider_data((int *)div_data->dividers,
-				      div_data->max_div, 0, 0,
+	if (ti_clk_parse_divider_data((int *)div_data->dividers, 0,
+				      div_data->max_div, div_flags,
 				      &div->width, &div->table)) {
-		pr_err("%s: Data parsing for %s:%04x:%d failed\n", __func__,
-		       node->name, offset, data->bit);
+		pr_err("%s: Data parsing for %pOF:%04x:%d failed\n", __func__,
+		       node, offset, data->bit);
 		kfree(div);
 		return;
 	}
@@ -400,6 +422,12 @@ _ti_clkctrl_setup_subclks(struct omap_clkctrl_provider *provider,
 	}
 }
 
+static void __init _clkctrl_add_provider(void *data,
+					 struct device_node *np)
+{
+	of_clk_add_hw_provider(np, _ti_omap4_clkctrl_xlate, data);
+}
+
 static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 {
 	struct omap_clkctrl_provider *provider;
@@ -411,6 +439,7 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 	struct omap_clkctrl_clk *clkctrl_clk;
 	const __be32 *addrp;
 	u32 addr;
+	int ret;
 
 	addrp = of_get_address(node, 0, NULL, NULL);
 	addr = (u32)of_translate_address(node, addrp);
@@ -419,6 +448,31 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 	if (of_machine_is_compatible("ti,omap4"))
 		data = omap4_clkctrl_data;
 #endif
+#ifdef CONFIG_SOC_OMAP5
+	if (of_machine_is_compatible("ti,omap5"))
+		data = omap5_clkctrl_data;
+#endif
+#ifdef CONFIG_SOC_DRA7XX
+	if (of_machine_is_compatible("ti,dra7"))
+		data = dra7_clkctrl_data;
+#endif
+#ifdef CONFIG_SOC_AM33XX
+	if (of_machine_is_compatible("ti,am33xx"))
+		data = am3_clkctrl_data;
+#endif
+#ifdef CONFIG_SOC_AM43XX
+	if (of_machine_is_compatible("ti,am4372"))
+		data = am4_clkctrl_data;
+	if (of_machine_is_compatible("ti,am438x"))
+		data = am438x_clkctrl_data;
+#endif
+#ifdef CONFIG_SOC_TI81XX
+	if (of_machine_is_compatible("ti,dm814"))
+		data = dm814_clkctrl_data;
+
+	if (of_machine_is_compatible("ti,dm816"))
+		data = dm816_clkctrl_data;
+#endif
 
 	while (data->addr) {
 		if (addr == data->addr)
@@ -428,7 +482,7 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 	}
 
 	if (!data->addr) {
-		pr_err("%s not found from clkctrl data.\n", node->name);
+		pr_err("%pOF not found from clkctrl data.\n", node);
 		return;
 	}
 
@@ -438,6 +492,21 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 
 	provider->base = of_iomap(node, 0);
 
+	provider->clkdm_name = kmalloc(strlen(node->parent->name) + 3,
+				       GFP_KERNEL);
+	if (!provider->clkdm_name) {
+		kfree(provider);
+		return;
+	}
+
+	/*
+	 * Create default clkdm name, replace _cm from end of parent node
+	 * name with _clkdm
+	 */
+	strcpy(provider->clkdm_name, node->parent->name);
+	provider->clkdm_name[strlen(provider->clkdm_name) - 2] = 0;
+	strcat(provider->clkdm_name, "clkdm");
+
 	INIT_LIST_HEAD(&provider->clocks);
 
 	/* Generate clocks */
@@ -460,9 +529,16 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 		if (reg_data->flags & CLKF_NO_IDLEST)
 			hw->flags |= NO_IDLEST;
 
+		if (reg_data->clkdm_name)
+			hw->clkdm_name = reg_data->clkdm_name;
+		else
+			hw->clkdm_name = provider->clkdm_name;
+
 		init.parent_names = &reg_data->parent;
 		init.num_parents = 1;
 		init.flags = 0;
+		if (reg_data->flags & CLKF_SET_RATE_PARENT)
+			init.flags |= CLK_SET_RATE_PARENT;
 		init.name = kasprintf(GFP_KERNEL, "%s:%s:%04x:%d",
 				      node->parent->name, node->name,
 				      reg_data->offset, 0);
@@ -485,7 +561,10 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 		reg_data++;
 	}
 
-	of_clk_add_hw_provider(node, _ti_omap4_clkctrl_xlate, provider);
+	ret = of_clk_add_hw_provider(node, _ti_omap4_clkctrl_xlate, provider);
+	if (ret == -EPROBE_DEFER)
+		ti_clk_retry_init(node, provider, _clkctrl_add_provider);
+
 	return;
 
 cleanup:
diff --git a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h
index 561dbe99ced7..d9b43bfc2532 100644
--- a/drivers/clk/ti/clock.h
+++ b/drivers/clk/ti/clock.h
@@ -92,17 +92,6 @@ struct ti_clk {
 	struct clk *clk;
 };
 
-struct ti_clk_alias {
-	struct ti_clk *clk;
-	struct clk_lookup lk;
-	struct list_head link;
-};
-
-struct ti_clk_fixed {
-	u32 frequency;
-	u16 flags;
-};
-
 struct ti_clk_mux {
 	u8 bit_shift;
 	int num_parents;
@@ -123,13 +112,6 @@ struct ti_clk_divider {
 	u16 flags;
 };
 
-struct ti_clk_fixed_factor {
-	const char *parent;
-	u16 div;
-	u16 mult;
-	u16 flags;
-};
-
 struct ti_clk_gate {
 	const char *parent;
 	u8 bit_shift;
@@ -138,44 +120,6 @@ struct ti_clk_gate {
 	u16 flags;
 };
 
-struct ti_clk_composite {
-	struct ti_clk_divider *divider;
-	struct ti_clk_mux *mux;
-	struct ti_clk_gate *gate;
-	u16 flags;
-};
-
-struct ti_clk_clkdm_gate {
-	const char *parent;
-	u16 flags;
-};
-
-struct ti_clk_dpll {
-	int num_parents;
-	u16 control_reg;
-	u16 idlest_reg;
-	u16 autoidle_reg;
-	u16 mult_div1_reg;
-	u8 module;
-	const char **parents;
-	u16 flags;
-	u8 modes;
-	u32 mult_mask;
-	u32 div1_mask;
-	u32 enable_mask;
-	u32 autoidle_mask;
-	u32 freqsel_mask;
-	u32 idlest_mask;
-	u32 dco_mask;
-	u32 sddiv_mask;
-	u16 max_multiplier;
-	u16 max_divider;
-	u8 min_divider;
-	u8 auto_recal_bit;
-	u8 recal_en_bit;
-	u8 recal_st_bit;
-};
-
 /* Composite clock component types */
 enum {
 	CLK_COMPONENT_TYPE_GATE = 0,
@@ -207,6 +151,7 @@ struct ti_dt_clk {
 struct omap_clkctrl_div_data {
 	const int *dividers;
 	int max_div;
+	u32 flags;
 };
 
 struct omap_clkctrl_bit_data {
@@ -221,6 +166,7 @@ struct omap_clkctrl_reg_data {
 	const struct omap_clkctrl_bit_data *bit_data;
 	u16 flags;
 	const char *parent;
+	const char *clkdm_name;
 };
 
 struct omap_clkctrl_data {
@@ -229,40 +175,35 @@ struct omap_clkctrl_data {
 };
 
 extern const struct omap_clkctrl_data omap4_clkctrl_data[];
+extern const struct omap_clkctrl_data omap5_clkctrl_data[];
+extern const struct omap_clkctrl_data dra7_clkctrl_data[];
+extern const struct omap_clkctrl_data am3_clkctrl_data[];
+extern const struct omap_clkctrl_data am4_clkctrl_data[];
+extern const struct omap_clkctrl_data am438x_clkctrl_data[];
+extern const struct omap_clkctrl_data dm814_clkctrl_data[];
+extern const struct omap_clkctrl_data dm816_clkctrl_data[];
 
 #define CLKF_SW_SUP	BIT(0)
 #define CLKF_HW_SUP	BIT(1)
 #define CLKF_NO_IDLEST	BIT(2)
 
-typedef void (*ti_of_clk_init_cb_t)(struct clk_hw *, struct device_node *);
+typedef void (*ti_of_clk_init_cb_t)(void *, struct device_node *);
 
-struct clk *ti_clk_register_gate(struct ti_clk *setup);
-struct clk *ti_clk_register_interface(struct ti_clk *setup);
-struct clk *ti_clk_register_mux(struct ti_clk *setup);
-struct clk *ti_clk_register_divider(struct ti_clk *setup);
-struct clk *ti_clk_register_composite(struct ti_clk *setup);
-struct clk *ti_clk_register_dpll(struct ti_clk *setup);
 struct clk *ti_clk_register(struct device *dev, struct clk_hw *hw,
 			    const char *con);
 int ti_clk_add_alias(struct device *dev, struct clk *clk, const char *con);
 void ti_clk_add_aliases(void);
 
-struct clk_hw *ti_clk_build_component_div(struct ti_clk_divider *setup);
-struct clk_hw *ti_clk_build_component_gate(struct ti_clk_gate *setup);
 struct clk_hw *ti_clk_build_component_mux(struct ti_clk_mux *setup);
 
 int ti_clk_parse_divider_data(int *div_table, int num_dividers, int max_div,
 			      u8 flags, u8 *width,
 			      const struct clk_div_table **table);
 
-void ti_clk_patch_legacy_clks(struct ti_clk **patch);
-struct clk *ti_clk_register_clk(struct ti_clk *setup);
-int ti_clk_register_legacy_clks(struct ti_clk_alias *clks);
-
 int ti_clk_get_reg_addr(struct device_node *node, int index,
 			struct clk_omap_reg *reg);
 void ti_dt_clocks_register(struct ti_dt_clk *oclks);
-int ti_clk_retry_init(struct device_node *node, struct clk_hw *hw,
+int ti_clk_retry_init(struct device_node *node, void *user,
 		      ti_of_clk_init_cb_t func);
 int ti_clk_add_component(struct device_node *node, struct clk_hw *hw, int type);
 
diff --git a/drivers/clk/ti/composite.c b/drivers/clk/ti/composite.c
index beea89463ca2..030e8b2c1050 100644
--- a/drivers/clk/ti/composite.c
+++ b/drivers/clk/ti/composite.c
@@ -116,54 +116,10 @@ static inline struct clk_hw *_get_hw(struct clk_hw_omap_comp *clk, int idx)
 
 #define to_clk_hw_comp(_hw) container_of(_hw, struct clk_hw_omap_comp, hw)
 
-#if defined(CONFIG_ARCH_OMAP3) && defined(CONFIG_ATAGS)
-struct clk *ti_clk_register_composite(struct ti_clk *setup)
-{
-	struct ti_clk_composite *comp;
-	struct clk_hw *gate;
-	struct clk_hw *mux;
-	struct clk_hw *div;
-	int num_parents = 1;
-	const char * const *parent_names = NULL;
-	struct clk *clk;
-	int ret;
-
-	comp = setup->data;
-
-	div = ti_clk_build_component_div(comp->divider);
-	gate = ti_clk_build_component_gate(comp->gate);
-	mux = ti_clk_build_component_mux(comp->mux);
-
-	if (div)
-		parent_names = &comp->divider->parent;
-
-	if (gate)
-		parent_names = &comp->gate->parent;
-
-	if (mux) {
-		num_parents = comp->mux->num_parents;
-		parent_names = comp->mux->parents;
-	}
-
-	clk = clk_register_composite(NULL, setup->name,
-				     parent_names, num_parents, mux,
-				     &ti_clk_mux_ops, div,
-				     &ti_composite_divider_ops, gate,
-				     &ti_composite_gate_ops, 0);
-
-	ret = ti_clk_add_alias(NULL, clk, setup->name);
-	if (ret) {
-		clk_unregister(clk);
-		return ERR_PTR(ret);
-	}
-
-	return clk;
-}
-#endif
-
-static void __init _register_composite(struct clk_hw *hw,
+static void __init _register_composite(void *user,
 				       struct device_node *node)
 {
+	struct clk_hw *hw = user;
 	struct clk *clk;
 	struct clk_hw_omap_comp *cclk = to_clk_hw_comp(hw);
 	struct component_clk *comp;
diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c
index d4e4444bc5ca..7d33ca9042cb 100644
--- a/drivers/clk/ti/dpll.c
+++ b/drivers/clk/ti/dpll.c
@@ -152,9 +152,10 @@ static const struct clk_ops dpll_x2_ck_ops = {
  * clk-bypass is missing), the clock is added to retry list and
  * the initialization is retried on later stage.
  */
-static void __init _register_dpll(struct clk_hw *hw,
+static void __init _register_dpll(void *user,
 				  struct device_node *node)
 {
+	struct clk_hw *hw = user;
 	struct clk_hw_omap *clk_hw = to_clk_hw_omap(hw);
 	struct dpll_data *dd = clk_hw->dpll_data;
 	struct clk *clk;
@@ -202,96 +203,6 @@ static void __init _register_dpll(struct clk_hw *hw,
 	kfree(clk_hw);
 }
 
-#if defined(CONFIG_ARCH_OMAP3) && defined(CONFIG_ATAGS)
-void _get_reg(u8 module, u16 offset, struct clk_omap_reg *reg)
-{
-	reg->index = module;
-	reg->offset = offset;
-}
-
-struct clk *ti_clk_register_dpll(struct ti_clk *setup)
-{
-	struct clk_hw_omap *clk_hw;
-	struct clk_init_data init = { NULL };
-	struct dpll_data *dd;
-	struct clk *clk;
-	struct ti_clk_dpll *dpll;
-	const struct clk_ops *ops = &omap3_dpll_ck_ops;
-	struct clk *clk_ref;
-	struct clk *clk_bypass;
-
-	dpll = setup->data;
-
-	if (dpll->num_parents < 2)
-		return ERR_PTR(-EINVAL);
-
-	clk_ref = clk_get_sys(NULL, dpll->parents[0]);
-	clk_bypass = clk_get_sys(NULL, dpll->parents[1]);
-
-	if (IS_ERR_OR_NULL(clk_ref) || IS_ERR_OR_NULL(clk_bypass))
-		return ERR_PTR(-EAGAIN);
-
-	dd = kzalloc(sizeof(*dd), GFP_KERNEL);
-	clk_hw = kzalloc(sizeof(*clk_hw), GFP_KERNEL);
-	if (!dd || !clk_hw) {
-		clk = ERR_PTR(-ENOMEM);
-		goto cleanup;
-	}
-
-	clk_hw->dpll_data = dd;
-	clk_hw->ops = &clkhwops_omap3_dpll;
-	clk_hw->hw.init = &init;
-
-	init.name = setup->name;
-	init.ops = ops;
-
-	init.num_parents = dpll->num_parents;
-	init.parent_names = dpll->parents;
-
-	_get_reg(dpll->module, dpll->control_reg, &dd->control_reg);
-	_get_reg(dpll->module, dpll->idlest_reg, &dd->idlest_reg);
-	_get_reg(dpll->module, dpll->mult_div1_reg, &dd->mult_div1_reg);
-	_get_reg(dpll->module, dpll->autoidle_reg, &dd->autoidle_reg);
-
-	dd->modes = dpll->modes;
-	dd->div1_mask = dpll->div1_mask;
-	dd->idlest_mask = dpll->idlest_mask;
-	dd->mult_mask = dpll->mult_mask;
-	dd->autoidle_mask = dpll->autoidle_mask;
-	dd->enable_mask = dpll->enable_mask;
-	dd->sddiv_mask = dpll->sddiv_mask;
-	dd->dco_mask = dpll->dco_mask;
-	dd->max_divider = dpll->max_divider;
-	dd->min_divider = dpll->min_divider;
-	dd->max_multiplier = dpll->max_multiplier;
-	dd->auto_recal_bit = dpll->auto_recal_bit;
-	dd->recal_en_bit = dpll->recal_en_bit;
-	dd->recal_st_bit = dpll->recal_st_bit;
-
-	dd->clk_ref = __clk_get_hw(clk_ref);
-	dd->clk_bypass = __clk_get_hw(clk_bypass);
-
-	if (dpll->flags & CLKF_CORE)
-		ops = &omap3_dpll_core_ck_ops;
-
-	if (dpll->flags & CLKF_PER)
-		ops = &omap3_dpll_per_ck_ops;
-
-	if (dpll->flags & CLKF_J_TYPE)
-		dd->flags |= DPLL_J_TYPE;
-
-	clk = ti_clk_register(NULL, &clk_hw->hw, setup->name);
-
-	if (!IS_ERR(clk))
-		return clk;
-
-cleanup:
-	kfree(dd);
-	kfree(clk_hw);
-	return clk;
-}
-#endif
-
 #if defined(CONFIG_ARCH_OMAP4) || defined(CONFIG_SOC_OMAP5) || \
 	defined(CONFIG_SOC_DRA7XX) || defined(CONFIG_SOC_AM33XX) || \
 	defined(CONFIG_SOC_AM43XX)
diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c
index 7151ec3a1b07..935b2de5fb88 100644
--- a/drivers/clk/ti/gate.c
+++ b/drivers/clk/ti/gate.c
@@ -128,53 +128,6 @@ static struct clk *_register_gate(struct device *dev, const char *name,
 	return clk;
 }
 
-#if defined(CONFIG_ARCH_OMAP3) && defined(CONFIG_ATAGS)
-struct clk *ti_clk_register_gate(struct ti_clk *setup)
-{
-	const struct clk_ops *ops = &omap_gate_clk_ops;
-	const struct clk_hw_omap_ops *hw_ops = NULL;
-	struct clk_omap_reg reg;
-	u32 flags = 0;
-	u8 clk_gate_flags = 0;
-	struct ti_clk_gate *gate;
-
-	gate = setup->data;
-
-	if (gate->flags & CLKF_INTERFACE)
-		return ti_clk_register_interface(setup);
-
-	if (gate->flags & CLKF_SET_RATE_PARENT)
-		flags |= CLK_SET_RATE_PARENT;
-
-	if (gate->flags & CLKF_SET_BIT_TO_DISABLE)
-		clk_gate_flags |= INVERT_ENABLE;
-
-	if (gate->flags & CLKF_HSDIV) {
-		ops = &omap_gate_clk_hsdiv_restore_ops;
-		hw_ops = &clkhwops_wait;
-	}
-
-	if (gate->flags & CLKF_DSS)
-		hw_ops = &clkhwops_omap3430es2_dss_usbhost_wait;
-
-	if (gate->flags & CLKF_WAIT)
-		hw_ops = &clkhwops_wait;
-
-	if (gate->flags & CLKF_CLKDM)
-		ops = &omap_gate_clkdm_clk_ops;
-
-	if (gate->flags & CLKF_AM35XX)
-		hw_ops = &clkhwops_am35xx_ipss_module_wait;
-
-	reg.index = gate->module;
-	reg.offset = gate->reg;
-	reg.ptr = NULL;
-
-	return _register_gate(NULL, setup->name, gate->parent, flags,
-			      &reg, gate->bit_shift,
-			      clk_gate_flags, ops, hw_ops);
-}
-
 struct clk_hw *ti_clk_build_component_gate(struct ti_clk_gate *setup)
 {
 	struct clk_hw_omap *gate;
@@ -204,7 +157,6 @@ struct clk_hw *ti_clk_build_component_gate(struct ti_clk_gate *setup)
 
 	return &gate->hw;
 }
-#endif
 
 static void __init _of_ti_gate_clk_setup(struct device_node *node,
 					 const struct clk_ops *ops,
diff --git a/drivers/clk/ti/interface.c b/drivers/clk/ti/interface.c
index 62cf50c1e1e3..41ae7021670e 100644
--- a/drivers/clk/ti/interface.c
+++ b/drivers/clk/ti/interface.c
@@ -67,38 +67,6 @@ static struct clk *_register_interface(struct device *dev, const char *name,
 	return clk;
 }
 
-#if defined(CONFIG_ARCH_OMAP3) && defined(CONFIG_ATAGS)
-struct clk *ti_clk_register_interface(struct ti_clk *setup)
-{
-	const struct clk_hw_omap_ops *ops = &clkhwops_iclk_wait;
-	struct clk_omap_reg reg;
-	struct ti_clk_gate *gate;
-
-	gate = setup->data;
-	reg.index = gate->module;
-	reg.offset = gate->reg;
-	reg.ptr = NULL;
-
-	if (gate->flags & CLKF_NO_WAIT)
-		ops = &clkhwops_iclk;
-
-	if (gate->flags & CLKF_HSOTGUSB)
-		ops = &clkhwops_omap3430es2_iclk_hsotgusb_wait;
-
-	if (gate->flags & CLKF_DSS)
-		ops = &clkhwops_omap3430es2_iclk_dss_usbhost_wait;
-
-	if (gate->flags & CLKF_SSI)
-		ops = &clkhwops_omap3430es2_iclk_ssi_wait;
-
-	if (gate->flags & CLKF_AM35XX)
-		ops = &clkhwops_am35xx_ipss_wait;
-
-	return _register_interface(NULL, setup->name, gate->parent,
-				   &reg, gate->bit_shift, ops);
-}
-#endif
-
 static void __init _of_ti_interface_clk_setup(struct device_node *node,
 					      const struct clk_hw_omap_ops *ops)
 {
diff --git a/drivers/clk/zte/clk.h b/drivers/clk/zte/clk.h
index 4df0f121b56d..f1041e36bcf1 100644
--- a/drivers/clk/zte/clk.h
+++ b/drivers/clk/zte/clk.h
@@ -14,24 +14,6 @@
 
 #define PNAME(x) static const char *x[]
 
-#define CLK_HW_INIT(_name, _parent, _ops, _flags)			\
-	&(struct clk_init_data) {					\
-		.flags		= _flags,				\
-		.name		= _name,				\
-		.parent_names	= (const char *[]) { _parent },		\
-		.num_parents	= 1,					\
-		.ops		= _ops,					\
-	}
-
-#define CLK_HW_INIT_PARENTS(_name, _parents, _ops, _flags)		\
-	&(struct clk_init_data) {					\
-		.flags		= _flags,				\
-		.name		= _name,				\
-		.parent_names	= _parents,				\
-		.num_parents	= ARRAY_SIZE(_parents),			\
-		.ops		= _ops,					\
-	}
-
 struct zx_pll_config {
 	unsigned long rate;
 	u32 cfg0;
diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig
index c729a88007d0..d2e5382821a4 100644
--- a/drivers/clocksource/Kconfig
+++ b/drivers/clocksource/Kconfig
@@ -269,6 +269,7 @@ config CLKSRC_STM32
 	bool "Clocksource for STM32 SoCs" if !ARCH_STM32
 	depends on OF && ARM && (ARCH_STM32 || COMPILE_TEST)
 	select CLKSRC_MMIO
+	select TIMER_OF
 
 config CLKSRC_MPS2
 	bool "Clocksource for MPS2 SoCs" if COMPILE_TEST
@@ -385,6 +386,7 @@ config ATMEL_PIT
 
 config ATMEL_ST
 	bool "Atmel ST timer support" if COMPILE_TEST
+	depends on HAS_IOMEM
 	select TIMER_OF
 	select MFD_SYSCON
 	help
@@ -441,6 +443,13 @@ config MTK_TIMER
 	help
 	  Support for Mediatek timer driver.
 
+config SPRD_TIMER
+	bool "Spreadtrum timer driver" if COMPILE_TEST
+	depends on HAS_IOMEM
+	select TIMER_OF
+	help
+	  Enables support for the Spreadtrum timer driver.
+
 config SYS_SUPPORTS_SH_MTU2
         bool
 
diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile
index 72711f1491e3..d6dec4489d66 100644
--- a/drivers/clocksource/Makefile
+++ b/drivers/clocksource/Makefile
@@ -54,6 +54,7 @@ obj-$(CONFIG_CLKSRC_TI_32K)	+= timer-ti-32k.o
 obj-$(CONFIG_CLKSRC_NPS)	+= timer-nps.o
 obj-$(CONFIG_OXNAS_RPS_TIMER)	+= timer-oxnas-rps.o
 obj-$(CONFIG_OWL_TIMER)		+= owl-timer.o
+obj-$(CONFIG_SPRD_TIMER)	+= timer-sprd.o
 
 obj-$(CONFIG_ARC_TIMERS)		+= arc_timer.o
 obj-$(CONFIG_ARM_ARCH_TIMER)		+= arm_arch_timer.o
diff --git a/drivers/clocksource/arc_timer.c b/drivers/clocksource/arc_timer.c
index 4927355f9cbe..471b428d8034 100644
--- a/drivers/clocksource/arc_timer.c
+++ b/drivers/clocksource/arc_timer.c
@@ -251,9 +251,14 @@ static irqreturn_t timer_irq_handler(int irq, void *dev_id)
 	int irq_reenable = clockevent_state_periodic(evt);
 
 	/*
-	 * Any write to CTRL reg ACks the interrupt, we rewrite the
-	 * Count when [N]ot [H]alted bit.
-	 * And re-arm it if perioid by [I]nterrupt [E]nable bit
+	 * 1. ACK the interrupt
+	 *    - For ARC700, any write to CTRL reg ACKs it, so just rewrite
+	 *      Count when [N]ot [H]alted bit.
+	 *    - For HS3x, it is a bit subtle. On taken count-down interrupt,
+	 *      IP bit [3] is set, which needs to be cleared for ACK'ing.
+	 *      The write below can only update the other two bits, hence
+	 *      explicitly clears IP bit
+	 * 2. Re-arm interrupt if periodic by writing to IE bit [0]
 	 */
 	write_aux_reg(ARC_REG_TIMER0_CTRL, irq_reenable | TIMER_CTRL_NH);
 
diff --git a/drivers/clocksource/fsl_ftm_timer.c b/drivers/clocksource/fsl_ftm_timer.c
index 3ee7e6fea621..846d18daf893 100644
--- a/drivers/clocksource/fsl_ftm_timer.c
+++ b/drivers/clocksource/fsl_ftm_timer.c
@@ -281,7 +281,7 @@ static int __init __ftm_clk_init(struct device_node *np, char *cnt_name,
 
 static unsigned long __init ftm_clk_init(struct device_node *np)
 {
-	unsigned long freq;
+	long freq;
 
 	freq = __ftm_clk_init(np, "ftm-evt-counter-en", "ftm-evt");
 	if (freq <= 0)
diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c
index a04808a21d4e..986b6796b631 100644
--- a/drivers/clocksource/mips-gic-timer.c
+++ b/drivers/clocksource/mips-gic-timer.c
@@ -166,7 +166,7 @@ static int __init __gic_clocksource_init(void)
 
 	/* Set clocksource mask. */
 	count_width = read_gic_config() & GIC_CONFIG_COUNTBITS;
-	count_width >>= __fls(GIC_CONFIG_COUNTBITS);
+	count_width >>= __ffs(GIC_CONFIG_COUNTBITS);
 	count_width *= 4;
 	count_width += 32;
 	gic_clocksource.mask = CLOCKSOURCE_MASK(count_width);
@@ -205,12 +205,12 @@ static int __init gic_clocksource_of_init(struct device_node *node)
 	} else if (of_property_read_u32(node, "clock-frequency",
 					&gic_frequency)) {
 		pr_err("GIC frequency not specified.\n");
-		return -EINVAL;;
+		return -EINVAL;
 	}
 	gic_timer_irq = irq_of_parse_and_map(node, 0);
 	if (!gic_timer_irq) {
 		pr_err("GIC timer IRQ not specified.\n");
-		return -EINVAL;;
+		return -EINVAL;
 	}
 
 	ret = __gic_clocksource_init();
diff --git a/drivers/clocksource/owl-timer.c b/drivers/clocksource/owl-timer.c
index c68630565079..ea00a5e8f95d 100644
--- a/drivers/clocksource/owl-timer.c
+++ b/drivers/clocksource/owl-timer.c
@@ -168,5 +168,6 @@ static int __init owl_timer_init(struct device_node *node)
 
 	return 0;
 }
-CLOCKSOURCE_OF_DECLARE(owl_s500, "actions,s500-timer", owl_timer_init);
-CLOCKSOURCE_OF_DECLARE(owl_s900, "actions,s900-timer", owl_timer_init);
+TIMER_OF_DECLARE(owl_s500, "actions,s500-timer", owl_timer_init);
+TIMER_OF_DECLARE(owl_s700, "actions,s700-timer", owl_timer_init);
+TIMER_OF_DECLARE(owl_s900, "actions,s900-timer", owl_timer_init);
diff --git a/drivers/clocksource/tcb_clksrc.c b/drivers/clocksource/tcb_clksrc.c
index 9de47d4d2d9e..43f4d5c4d6fa 100644
--- a/drivers/clocksource/tcb_clksrc.c
+++ b/drivers/clocksource/tcb_clksrc.c
@@ -384,7 +384,7 @@ static int __init tcb_clksrc_init(void)
 
 	printk(bootinfo, clksrc.name, CONFIG_ATMEL_TCB_CLKSRC_BLOCK,
 			divided_rate / 1000000,
-			((divided_rate + 500000) % 1000000) / 1000);
+			((divided_rate % 1000000) + 500) / 1000);
 
 	if (tc->tcb_config && tc->tcb_config->counter_width == 32) {
 		/* use apropriate function to read 32 bit counter */
diff --git a/drivers/clocksource/timer-of.c b/drivers/clocksource/timer-of.c
index a31990408153..06ed88a2a8a0 100644
--- a/drivers/clocksource/timer-of.c
+++ b/drivers/clocksource/timer-of.c
@@ -24,7 +24,13 @@
 
 #include "timer-of.h"
 
-static __init void timer_irq_exit(struct of_timer_irq *of_irq)
+/**
+ * timer_of_irq_exit - Release the interrupt
+ * @of_irq: an of_timer_irq structure pointer
+ *
+ * Free the irq resource
+ */
+static __init void timer_of_irq_exit(struct of_timer_irq *of_irq)
 {
 	struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
 
@@ -34,8 +40,24 @@ static __init void timer_irq_exit(struct of_timer_irq *of_irq)
 		free_irq(of_irq->irq, clkevt);
 }
 
-static __init int timer_irq_init(struct device_node *np,
-				 struct of_timer_irq *of_irq)
+/**
+ * timer_of_irq_init - Request the interrupt
+ * @np: a device tree node pointer
+ * @of_irq: an of_timer_irq structure pointer
+ *
+ * Get the interrupt number from the DT from its definition and
+ * request it. The interrupt is gotten by falling back the following way:
+ *
+ * - Get interrupt number by name
+ * - Get interrupt number by index
+ *
+ * When the interrupt is per CPU, 'request_percpu_irq()' is called,
+ * otherwise 'request_irq()' is used.
+ *
+ * Returns 0 on success, < 0 otherwise
+ */
+static __init int timer_of_irq_init(struct device_node *np,
+				    struct of_timer_irq *of_irq)
 {
 	int ret;
 	struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
@@ -72,15 +94,30 @@ static __init int timer_irq_init(struct device_node *np,
 	return 0;
 }
 
-static __init void timer_clk_exit(struct of_timer_clk *of_clk)
+/**
+ * timer_of_clk_exit - Release the clock resources
+ * @of_clk: a of_timer_clk structure pointer
+ *
+ * Disables and releases the refcount on the clk
+ */
+static __init void timer_of_clk_exit(struct of_timer_clk *of_clk)
 {
 	of_clk->rate = 0;
 	clk_disable_unprepare(of_clk->clk);
 	clk_put(of_clk->clk);
 }
 
-static __init int timer_clk_init(struct device_node *np,
-				 struct of_timer_clk *of_clk)
+/**
+ * timer_of_clk_init - Initialize the clock resources
+ * @np: a device tree node pointer
+ * @of_clk: a of_timer_clk structure pointer
+ *
+ * Get the clock by name or by index, enable it and get the rate
+ *
+ * Returns 0 on success, < 0 otherwise
+ */
+static __init int timer_of_clk_init(struct device_node *np,
+				    struct of_timer_clk *of_clk)
 {
 	int ret;
 
@@ -116,19 +153,19 @@ static __init int timer_clk_init(struct device_node *np,
 	goto out;
 }
 
-static __init void timer_base_exit(struct of_timer_base *of_base)
+static __init void timer_of_base_exit(struct of_timer_base *of_base)
 {
 	iounmap(of_base->base);
 }
 
-static __init int timer_base_init(struct device_node *np,
-				  struct of_timer_base *of_base)
+static __init int timer_of_base_init(struct device_node *np,
+				     struct of_timer_base *of_base)
 {
-	const char *name = of_base->name ? of_base->name : np->full_name;
-
-	of_base->base = of_io_request_and_map(np, of_base->index, name);
+	of_base->base = of_base->name ?
+		of_io_request_and_map(np, of_base->index, of_base->name) :
+		of_iomap(np, of_base->index);
 	if (IS_ERR(of_base->base)) {
-		pr_err("Failed to iomap (%s)\n", name);
+		pr_err("Failed to iomap (%s)\n", of_base->name);
 		return PTR_ERR(of_base->base);
 	}
 
@@ -141,21 +178,21 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
 	int flags = 0;
 
 	if (to->flags & TIMER_OF_BASE) {
-		ret = timer_base_init(np, &to->of_base);
+		ret = timer_of_base_init(np, &to->of_base);
 		if (ret)
 			goto out_fail;
 		flags |= TIMER_OF_BASE;
 	}
 
 	if (to->flags & TIMER_OF_CLOCK) {
-		ret = timer_clk_init(np, &to->of_clk);
+		ret = timer_of_clk_init(np, &to->of_clk);
 		if (ret)
 			goto out_fail;
 		flags |= TIMER_OF_CLOCK;
 	}
 
 	if (to->flags & TIMER_OF_IRQ) {
-		ret = timer_irq_init(np, &to->of_irq);
+		ret = timer_of_irq_init(np, &to->of_irq);
 		if (ret)
 			goto out_fail;
 		flags |= TIMER_OF_IRQ;
@@ -163,17 +200,20 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
 
 	if (!to->clkevt.name)
 		to->clkevt.name = np->name;
+
+	to->np = np;
+
 	return ret;
 
 out_fail:
 	if (flags & TIMER_OF_IRQ)
-		timer_irq_exit(&to->of_irq);
+		timer_of_irq_exit(&to->of_irq);
 
 	if (flags & TIMER_OF_CLOCK)
-		timer_clk_exit(&to->of_clk);
+		timer_of_clk_exit(&to->of_clk);
 
 	if (flags & TIMER_OF_BASE)
-		timer_base_exit(&to->of_base);
+		timer_of_base_exit(&to->of_base);
 	return ret;
 }
 
@@ -187,11 +227,11 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
 void __init timer_of_cleanup(struct timer_of *to)
 {
 	if (to->flags & TIMER_OF_IRQ)
-		timer_irq_exit(&to->of_irq);
+		timer_of_irq_exit(&to->of_irq);
 
 	if (to->flags & TIMER_OF_CLOCK)
-		timer_clk_exit(&to->of_clk);
+		timer_of_clk_exit(&to->of_clk);
 
 	if (to->flags & TIMER_OF_BASE)
-		timer_base_exit(&to->of_base);
+		timer_of_base_exit(&to->of_base);
 }
diff --git a/drivers/clocksource/timer-of.h b/drivers/clocksource/timer-of.h
index 3f708f1be43d..a5478f3e8589 100644
--- a/drivers/clocksource/timer-of.h
+++ b/drivers/clocksource/timer-of.h
@@ -33,6 +33,7 @@ struct of_timer_clk {
 
 struct timer_of {
 	unsigned int flags;
+	struct device_node *np;
 	struct clock_event_device clkevt;
 	struct of_timer_base of_base;
 	struct of_timer_irq  of_irq;
diff --git a/drivers/clocksource/timer-sprd.c b/drivers/clocksource/timer-sprd.c
new file mode 100644
index 000000000000..ef9ebeafb3ed
--- /dev/null
+++ b/drivers/clocksource/timer-sprd.c
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 Spreadtrum Communications Inc.
+ */
+
+#include <linux/init.h>
+#include <linux/interrupt.h>
+
+#include "timer-of.h"
+
+#define TIMER_NAME		"sprd_timer"
+
+#define TIMER_LOAD_LO		0x0
+#define TIMER_LOAD_HI		0x4
+#define TIMER_VALUE_LO		0x8
+#define TIMER_VALUE_HI		0xc
+
+#define TIMER_CTL		0x10
+#define TIMER_CTL_PERIOD_MODE	BIT(0)
+#define TIMER_CTL_ENABLE	BIT(1)
+#define TIMER_CTL_64BIT_WIDTH	BIT(16)
+
+#define TIMER_INT		0x14
+#define TIMER_INT_EN		BIT(0)
+#define TIMER_INT_RAW_STS	BIT(1)
+#define TIMER_INT_MASK_STS	BIT(2)
+#define TIMER_INT_CLR		BIT(3)
+
+#define TIMER_VALUE_SHDW_LO	0x18
+#define TIMER_VALUE_SHDW_HI	0x1c
+
+#define TIMER_VALUE_LO_MASK	GENMASK(31, 0)
+
+static void sprd_timer_enable(void __iomem *base, u32 flag)
+{
+	u32 val = readl_relaxed(base + TIMER_CTL);
+
+	val |= TIMER_CTL_ENABLE;
+	if (flag & TIMER_CTL_64BIT_WIDTH)
+		val |= TIMER_CTL_64BIT_WIDTH;
+	else
+		val &= ~TIMER_CTL_64BIT_WIDTH;
+
+	if (flag & TIMER_CTL_PERIOD_MODE)
+		val |= TIMER_CTL_PERIOD_MODE;
+	else
+		val &= ~TIMER_CTL_PERIOD_MODE;
+
+	writel_relaxed(val, base + TIMER_CTL);
+}
+
+static void sprd_timer_disable(void __iomem *base)
+{
+	u32 val = readl_relaxed(base + TIMER_CTL);
+
+	val &= ~TIMER_CTL_ENABLE;
+	writel_relaxed(val, base + TIMER_CTL);
+}
+
+static void sprd_timer_update_counter(void __iomem *base, unsigned long cycles)
+{
+	writel_relaxed(cycles & TIMER_VALUE_LO_MASK, base + TIMER_LOAD_LO);
+	writel_relaxed(0, base + TIMER_LOAD_HI);
+}
+
+static void sprd_timer_enable_interrupt(void __iomem *base)
+{
+	writel_relaxed(TIMER_INT_EN, base + TIMER_INT);
+}
+
+static void sprd_timer_clear_interrupt(void __iomem *base)
+{
+	u32 val = readl_relaxed(base + TIMER_INT);
+
+	val |= TIMER_INT_CLR;
+	writel_relaxed(val, base + TIMER_INT);
+}
+
+static int sprd_timer_set_next_event(unsigned long cycles,
+				     struct clock_event_device *ce)
+{
+	struct timer_of *to = to_timer_of(ce);
+
+	sprd_timer_disable(timer_of_base(to));
+	sprd_timer_update_counter(timer_of_base(to), cycles);
+	sprd_timer_enable(timer_of_base(to), 0);
+
+	return 0;
+}
+
+static int sprd_timer_set_periodic(struct clock_event_device *ce)
+{
+	struct timer_of *to = to_timer_of(ce);
+
+	sprd_timer_disable(timer_of_base(to));
+	sprd_timer_update_counter(timer_of_base(to), timer_of_period(to));
+	sprd_timer_enable(timer_of_base(to), TIMER_CTL_PERIOD_MODE);
+
+	return 0;
+}
+
+static int sprd_timer_shutdown(struct clock_event_device *ce)
+{
+	struct timer_of *to = to_timer_of(ce);
+
+	sprd_timer_disable(timer_of_base(to));
+	return 0;
+}
+
+static irqreturn_t sprd_timer_interrupt(int irq, void *dev_id)
+{
+	struct clock_event_device *ce = (struct clock_event_device *)dev_id;
+	struct timer_of *to = to_timer_of(ce);
+
+	sprd_timer_clear_interrupt(timer_of_base(to));
+
+	if (clockevent_state_oneshot(ce))
+		sprd_timer_disable(timer_of_base(to));
+
+	ce->event_handler(ce);
+	return IRQ_HANDLED;
+}
+
+static struct timer_of to = {
+	.flags = TIMER_OF_IRQ | TIMER_OF_BASE | TIMER_OF_CLOCK,
+
+	.clkevt = {
+		.name = TIMER_NAME,
+		.rating = 300,
+		.features = CLOCK_EVT_FEAT_DYNIRQ | CLOCK_EVT_FEAT_PERIODIC |
+			CLOCK_EVT_FEAT_ONESHOT,
+		.set_state_shutdown = sprd_timer_shutdown,
+		.set_state_periodic = sprd_timer_set_periodic,
+		.set_next_event = sprd_timer_set_next_event,
+		.cpumask = cpu_possible_mask,
+	},
+
+	.of_irq = {
+		.handler = sprd_timer_interrupt,
+		.flags = IRQF_TIMER | IRQF_IRQPOLL,
+	},
+};
+
+static int __init sprd_timer_init(struct device_node *np)
+{
+	int ret;
+
+	ret = timer_of_init(np, &to);
+	if (ret)
+		return ret;
+
+	sprd_timer_enable_interrupt(timer_of_base(&to));
+	clockevents_config_and_register(&to.clkevt, timer_of_rate(&to),
+					1, UINT_MAX);
+
+	return 0;
+}
+
+TIMER_OF_DECLARE(sc9860_timer, "sprd,sc9860-timer", sprd_timer_init);
diff --git a/drivers/clocksource/timer-stm32.c b/drivers/clocksource/timer-stm32.c
index 8f2423789ba9..e5cdc3af684c 100644
--- a/drivers/clocksource/timer-stm32.c
+++ b/drivers/clocksource/timer-stm32.c
@@ -9,6 +9,7 @@
 #include <linux/kernel.h>
 #include <linux/clocksource.h>
 #include <linux/clockchips.h>
+#include <linux/delay.h>
 #include <linux/irq.h>
 #include <linux/interrupt.h>
 #include <linux/of.h>
@@ -16,175 +17,318 @@
 #include <linux/of_irq.h>
 #include <linux/clk.h>
 #include <linux/reset.h>
+#include <linux/sched_clock.h>
+#include <linux/slab.h>
+
+#include "timer-of.h"
 
 #define TIM_CR1		0x00
 #define TIM_DIER	0x0c
 #define TIM_SR		0x10
 #define TIM_EGR		0x14
+#define TIM_CNT		0x24
 #define TIM_PSC		0x28
 #define TIM_ARR		0x2c
+#define TIM_CCR1	0x34
 
 #define TIM_CR1_CEN	BIT(0)
+#define TIM_CR1_UDIS	BIT(1)
 #define TIM_CR1_OPM	BIT(3)
 #define TIM_CR1_ARPE	BIT(7)
 
 #define TIM_DIER_UIE	BIT(0)
+#define TIM_DIER_CC1IE	BIT(1)
 
 #define TIM_SR_UIF	BIT(0)
 
 #define TIM_EGR_UG	BIT(0)
 
-struct stm32_clock_event_ddata {
-	struct clock_event_device evtdev;
-	unsigned periodic_top;
-	void __iomem *base;
+#define TIM_PSC_MAX	USHRT_MAX
+#define TIM_PSC_CLKRATE	10000
+
+struct stm32_timer_private {
+	int bits;
 };
 
-static int stm32_clock_event_shutdown(struct clock_event_device *evtdev)
+/**
+ * stm32_timer_of_bits_set - set accessor helper
+ * @to: a timer_of structure pointer
+ * @bits: the number of bits (16 or 32)
+ *
+ * Accessor helper to set the number of bits in the timer-of private
+ * structure.
+ *
+ */
+static void stm32_timer_of_bits_set(struct timer_of *to, int bits)
 {
-	struct stm32_clock_event_ddata *data =
-		container_of(evtdev, struct stm32_clock_event_ddata, evtdev);
-	void *base = data->base;
+	struct stm32_timer_private *pd = to->private_data;
 
-	writel_relaxed(0, base + TIM_CR1);
-	return 0;
+	pd->bits = bits;
+}
+
+/**
+ * stm32_timer_of_bits_get - get accessor helper
+ * @to: a timer_of structure pointer
+ *
+ * Accessor helper to get the number of bits in the timer-of private
+ * structure.
+ *
+ * Returns an integer corresponding to the number of bits.
+ */
+static int stm32_timer_of_bits_get(struct timer_of *to)
+{
+	struct stm32_timer_private *pd = to->private_data;
+
+	return pd->bits;
+}
+
+static void __iomem *stm32_timer_cnt __read_mostly;
+
+static u64 notrace stm32_read_sched_clock(void)
+{
+	return readl_relaxed(stm32_timer_cnt);
+}
+
+static struct delay_timer stm32_timer_delay;
+
+static unsigned long stm32_read_delay(void)
+{
+	return readl_relaxed(stm32_timer_cnt);
 }
 
-static int stm32_clock_event_set_periodic(struct clock_event_device *evtdev)
+static void stm32_clock_event_disable(struct timer_of *to)
 {
-	struct stm32_clock_event_ddata *data =
-		container_of(evtdev, struct stm32_clock_event_ddata, evtdev);
-	void *base = data->base;
+	writel_relaxed(0, timer_of_base(to) + TIM_DIER);
+}
+
+/**
+ * stm32_timer_start - Start the counter without event
+ * @to: a timer_of structure pointer
+ *
+ * Start the timer in order to have the counter reset and start
+ * incrementing but disable interrupt event when there is a counter
+ * overflow. By default, the counter direction is used as upcounter.
+ */
+static void stm32_timer_start(struct timer_of *to)
+{
+	writel_relaxed(TIM_CR1_UDIS | TIM_CR1_CEN, timer_of_base(to) + TIM_CR1);
+}
+
+static int stm32_clock_event_shutdown(struct clock_event_device *clkevt)
+{
+	struct timer_of *to = to_timer_of(clkevt);
+
+	stm32_clock_event_disable(to);
 
-	writel_relaxed(data->periodic_top, base + TIM_ARR);
-	writel_relaxed(TIM_CR1_ARPE | TIM_CR1_CEN, base + TIM_CR1);
 	return 0;
 }
 
 static int stm32_clock_event_set_next_event(unsigned long evt,
-					    struct clock_event_device *evtdev)
+					    struct clock_event_device *clkevt)
 {
-	struct stm32_clock_event_ddata *data =
-		container_of(evtdev, struct stm32_clock_event_ddata, evtdev);
+	struct timer_of *to = to_timer_of(clkevt);
+	unsigned long now, next;
+
+	next = readl_relaxed(timer_of_base(to) + TIM_CNT) + evt;
+	writel_relaxed(next, timer_of_base(to) + TIM_CCR1);
+	now = readl_relaxed(timer_of_base(to) + TIM_CNT);
+
+	if ((next - now) > evt)
+		return -ETIME;
 
-	writel_relaxed(evt, data->base + TIM_ARR);
-	writel_relaxed(TIM_CR1_ARPE | TIM_CR1_OPM | TIM_CR1_CEN,
-		       data->base + TIM_CR1);
+	writel_relaxed(TIM_DIER_CC1IE, timer_of_base(to) + TIM_DIER);
+
+	return 0;
+}
+
+static int stm32_clock_event_set_periodic(struct clock_event_device *clkevt)
+{
+	struct timer_of *to = to_timer_of(clkevt);
+
+	stm32_timer_start(to);
+
+	return stm32_clock_event_set_next_event(timer_of_period(to), clkevt);
+}
+
+static int stm32_clock_event_set_oneshot(struct clock_event_device *clkevt)
+{
+	struct timer_of *to = to_timer_of(clkevt);
+
+	stm32_timer_start(to);
 
 	return 0;
 }
 
 static irqreturn_t stm32_clock_event_handler(int irq, void *dev_id)
 {
-	struct stm32_clock_event_ddata *data = dev_id;
+	struct clock_event_device *clkevt = (struct clock_event_device *)dev_id;
+	struct timer_of *to = to_timer_of(clkevt);
+
+	writel_relaxed(0, timer_of_base(to) + TIM_SR);
 
-	writel_relaxed(0, data->base + TIM_SR);
+	if (clockevent_state_periodic(clkevt))
+		stm32_clock_event_set_periodic(clkevt);
+	else
+		stm32_clock_event_shutdown(clkevt);
 
-	data->evtdev.event_handler(&data->evtdev);
+	clkevt->event_handler(clkevt);
 
 	return IRQ_HANDLED;
 }
 
-static struct stm32_clock_event_ddata clock_event_ddata = {
-	.evtdev = {
-		.name = "stm32 clockevent",
-		.features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC,
-		.set_state_shutdown = stm32_clock_event_shutdown,
-		.set_state_periodic = stm32_clock_event_set_periodic,
-		.set_state_oneshot = stm32_clock_event_shutdown,
-		.tick_resume = stm32_clock_event_shutdown,
-		.set_next_event = stm32_clock_event_set_next_event,
-		.rating = 200,
-	},
-};
+/**
+ * stm32_timer_width - Sort out the timer width (32/16)
+ * @to: a pointer to a timer-of structure
+ *
+ * Write the 32-bit max value and read/return the result. If the timer
+ * is 32 bits wide, the result will be UINT_MAX, otherwise it will
+ * be truncated by the 16-bit register to USHRT_MAX.
+ *
+ */
+static void __init stm32_timer_set_width(struct timer_of *to)
+{
+	u32 width;
+
+	writel_relaxed(UINT_MAX, timer_of_base(to) + TIM_ARR);
+
+	width = readl_relaxed(timer_of_base(to) + TIM_ARR);
+
+	stm32_timer_of_bits_set(to, width == UINT_MAX ? 32 : 16);
+}
 
-static int __init stm32_clockevent_init(struct device_node *np)
+/**
+ * stm32_timer_set_prescaler - Compute and set the prescaler register
+ * @to: a pointer to a timer-of structure
+ *
+ * Depending on the timer width, compute the prescaler to always
+ * target a 10MHz timer rate for 16 bits. 32-bit timers are
+ * considered precise and long enough to not use the prescaler.
+ */
+static void __init stm32_timer_set_prescaler(struct timer_of *to)
 {
-	struct stm32_clock_event_ddata *data = &clock_event_ddata;
-	struct clk *clk;
-	struct reset_control *rstc;
-	unsigned long rate, max_delta;
-	int irq, ret, bits, prescaler = 1;
-
-	clk = of_clk_get(np, 0);
-	if (IS_ERR(clk)) {
-		ret = PTR_ERR(clk);
-		pr_err("failed to get clock for clockevent (%d)\n", ret);
-		goto err_clk_get;
+	int prescaler = 1;
+
+	if (stm32_timer_of_bits_get(to) != 32) {
+		prescaler = DIV_ROUND_CLOSEST(timer_of_rate(to),
+					      TIM_PSC_CLKRATE);
+		/*
+		 * The prescaler register is an u16, the variable
+		 * can't be greater than TIM_PSC_MAX, let's cap it in
+		 * this case.
+		 */
+		prescaler = prescaler < TIM_PSC_MAX ? prescaler : TIM_PSC_MAX;
 	}
 
-	ret = clk_prepare_enable(clk);
-	if (ret) {
-		pr_err("failed to enable timer clock for clockevent (%d)\n",
-		       ret);
-		goto err_clk_enable;
-	}
+	writel_relaxed(prescaler - 1, timer_of_base(to) + TIM_PSC);
+	writel_relaxed(TIM_EGR_UG, timer_of_base(to) + TIM_EGR);
+	writel_relaxed(0, timer_of_base(to) + TIM_SR);
 
-	rate = clk_get_rate(clk);
+	/* Adjust rate and period given the prescaler value */
+	to->of_clk.rate = DIV_ROUND_CLOSEST(to->of_clk.rate, prescaler);
+	to->of_clk.period = DIV_ROUND_UP(to->of_clk.rate, HZ);
+}
 
-	rstc = of_reset_control_get(np, NULL);
-	if (!IS_ERR(rstc)) {
-		reset_control_assert(rstc);
-		reset_control_deassert(rstc);
+static int __init stm32_clocksource_init(struct timer_of *to)
+{
+        u32 bits = stm32_timer_of_bits_get(to);
+	const char *name = to->np->full_name;
+
+	/*
+	 * This driver allows to register several timers and relies on
+	 * the generic time framework to select the right one.
+	 * However, nothing allows to do the same for the
+	 * sched_clock. We are not interested in a sched_clock for the
+	 * 16-bit timers but only for the 32-bit one, so if no 32-bit
+	 * timer is registered yet, we select this 32-bit timer as a
+	 * sched_clock.
+	 */
+	if (bits == 32 && !stm32_timer_cnt) {
+
+		/*
+		 * Start immediately the counter as we will be using
+		 * it right after.
+		 */
+		stm32_timer_start(to);
+
+		stm32_timer_cnt = timer_of_base(to) + TIM_CNT;
+		sched_clock_register(stm32_read_sched_clock, bits, timer_of_rate(to));
+		pr_info("%s: STM32 sched_clock registered\n", name);
+
+		stm32_timer_delay.read_current_timer = stm32_read_delay;
+		stm32_timer_delay.freq = timer_of_rate(to);
+		register_current_timer_delay(&stm32_timer_delay);
+		pr_info("%s: STM32 delay timer registered\n", name);
 	}
 
-	data->base = of_iomap(np, 0);
-	if (!data->base) {
-		ret = -ENXIO;
-		pr_err("failed to map registers for clockevent\n");
-		goto err_iomap;
-	}
+	return clocksource_mmio_init(timer_of_base(to) + TIM_CNT, name,
+				     timer_of_rate(to), bits == 32 ? 250 : 100,
+				     bits, clocksource_mmio_readl_up);
+}
 
-	irq = irq_of_parse_and_map(np, 0);
-	if (!irq) {
-		ret = -EINVAL;
-		pr_err("%pOF: failed to get irq.\n", np);
-		goto err_get_irq;
-	}
+static void __init stm32_clockevent_init(struct timer_of *to)
+{
+	u32 bits = stm32_timer_of_bits_get(to);
 
-	/* Detect whether the timer is 16 or 32 bits */
-	writel_relaxed(~0U, data->base + TIM_ARR);
-	max_delta = readl_relaxed(data->base + TIM_ARR);
-	if (max_delta == ~0U) {
-		prescaler = 1;
-		bits = 32;
-	} else {
-		prescaler = 1024;
-		bits = 16;
-	}
-	writel_relaxed(0, data->base + TIM_ARR);
+	to->clkevt.name = to->np->full_name;
+	to->clkevt.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
+	to->clkevt.set_state_shutdown = stm32_clock_event_shutdown;
+	to->clkevt.set_state_periodic = stm32_clock_event_set_periodic;
+	to->clkevt.set_state_oneshot = stm32_clock_event_set_oneshot;
+	to->clkevt.tick_resume = stm32_clock_event_shutdown;
+	to->clkevt.set_next_event = stm32_clock_event_set_next_event;
+	to->clkevt.rating = bits == 32 ? 250 : 100;
+
+	clockevents_config_and_register(&to->clkevt, timer_of_rate(to), 0x1,
+					(1 <<  bits) - 1);
+
+	pr_info("%pOF: STM32 clockevent driver initialized (%d bits)\n",
+		to->np, bits);
+}
+
+static int __init stm32_timer_init(struct device_node *node)
+{
+	struct reset_control *rstc;
+	struct timer_of *to;
+	int ret;
+
+	to = kzalloc(sizeof(*to), GFP_KERNEL);
+	if (!to)
+		return -ENOMEM;
 
-	writel_relaxed(prescaler - 1, data->base + TIM_PSC);
-	writel_relaxed(TIM_EGR_UG, data->base + TIM_EGR);
-	writel_relaxed(TIM_DIER_UIE, data->base + TIM_DIER);
-	writel_relaxed(0, data->base + TIM_SR);
+	to->flags = TIMER_OF_IRQ | TIMER_OF_CLOCK | TIMER_OF_BASE;
+	to->of_irq.handler = stm32_clock_event_handler;
 
-	data->periodic_top = DIV_ROUND_CLOSEST(rate, prescaler * HZ);
+	ret = timer_of_init(node, to);
+	if (ret)
+		goto err;
 
-	clockevents_config_and_register(&data->evtdev,
-					DIV_ROUND_CLOSEST(rate, prescaler),
-					0x1, max_delta);
+	to->private_data = kzalloc(sizeof(struct stm32_timer_private),
+				   GFP_KERNEL);
+	if (!to->private_data)
+		goto deinit;
 
-	ret = request_irq(irq, stm32_clock_event_handler, IRQF_TIMER,
-			"stm32 clockevent", data);
-	if (ret) {
-		pr_err("%pOF: failed to request irq.\n", np);
-		goto err_get_irq;
+	rstc = of_reset_control_get(node, NULL);
+	if (!IS_ERR(rstc)) {
+		reset_control_assert(rstc);
+		reset_control_deassert(rstc);
 	}
 
-	pr_info("%pOF: STM32 clockevent driver initialized (%d bits)\n",
-			np, bits);
+	stm32_timer_set_width(to);
 
-	return ret;
+	stm32_timer_set_prescaler(to);
+
+	ret = stm32_clocksource_init(to);
+	if (ret)
+		goto deinit;
+
+	stm32_clockevent_init(to);
+	return 0;
 
-err_get_irq:
-	iounmap(data->base);
-err_iomap:
-	clk_disable_unprepare(clk);
-err_clk_enable:
-	clk_put(clk);
-err_clk_get:
+deinit:
+	timer_of_cleanup(to);
+err:
+	kfree(to);
 	return ret;
 }
 
-TIMER_OF_DECLARE(stm32, "st,stm32-timer", stm32_clockevent_init);
+TIMER_OF_DECLARE(stm32, "st,stm32-timer", stm32_timer_init);
diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c
index 2a3fe83ec337..3b56ea3f52af 100644
--- a/drivers/clocksource/timer-sun5i.c
+++ b/drivers/clocksource/timer-sun5i.c
@@ -334,7 +334,7 @@ static int __init sun5i_timer_init(struct device_node *node)
 	timer_base = of_io_request_and_map(node, 0, of_node_full_name(node));
 	if (IS_ERR(timer_base)) {
 		pr_err("Can't map registers\n");
-		return PTR_ERR(timer_base);;
+		return PTR_ERR(timer_base);
 	}
 
 	irq = irq_of_parse_and_map(node, 0);
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index d8addbce40bc..608af20a3494 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -239,16 +239,6 @@ if PPC32 || PPC64
 source "drivers/cpufreq/Kconfig.powerpc"
 endif
 
-if AVR32
-config AVR32_AT32AP_CPUFREQ
-	bool "CPU frequency driver for AT32AP"
-	depends on PLATFORM_AT32AP
-	default n
-	help
-	  This enables the CPU frequency driver for AT32AP processors.
-	  If in doubt, say N.
-endif
-
 if IA64
 config IA64_ACPI_CPUFREQ
 	tristate "ACPI Processor P-States driver"
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index bdce4488ded1..fb586e09682d 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -2,6 +2,29 @@
 # ARM CPU Frequency scaling drivers
 #
 
+config ACPI_CPPC_CPUFREQ
+	tristate "CPUFreq driver based on the ACPI CPPC spec"
+	depends on ACPI_PROCESSOR
+	select ACPI_CPPC_LIB
+	help
+	  This adds a CPUFreq driver which uses CPPC methods
+	  as described in the ACPIv5.1 spec. CPPC stands for
+	  Collaborative Processor Performance Controls. It
+	  is based on an abstract continuous scale of CPU
+	  performance values which allows the remote power
+	  processor to flexibly optimize for power and
+	  performance. CPPC relies on power management firmware
+	  support for its operation.
+
+	  If in doubt, say N.
+
+config ARM_ARMADA_37XX_CPUFREQ
+	tristate "Armada 37xx CPUFreq support"
+	depends on ARCH_MVEBU
+	help
+	  This adds the CPUFreq driver support for Marvell Armada 37xx SoCs.
+	  The Armada 37xx PMU supports 4 frequency and VDD levels.
+
 # big LITTLE core layer and glue drivers
 config ARM_BIG_LITTLE_CPUFREQ
 	tristate "Generic ARM big LITTLE CPUfreq driver"
@@ -12,6 +35,30 @@ config ARM_BIG_LITTLE_CPUFREQ
 	help
 	  This enables the Generic CPUfreq driver for ARM big.LITTLE platforms.
 
+config ARM_DT_BL_CPUFREQ
+	tristate "Generic probing via DT for ARM big LITTLE CPUfreq driver"
+	depends on ARM_BIG_LITTLE_CPUFREQ && OF
+	help
+	  This enables probing via DT for Generic CPUfreq driver for ARM
+	  big.LITTLE platform. This gets frequency tables from DT.
+
+config ARM_SCPI_CPUFREQ
+	tristate "SCPI based CPUfreq driver"
+	depends on ARM_SCPI_PROTOCOL && COMMON_CLK_SCPI
+	help
+	  This adds the CPUfreq driver support for ARM platforms using SCPI
+	  protocol for CPU power management.
+
+	  This driver uses SCPI Message Protocol driver to interact with the
+	  firmware providing the CPU DVFS functionality.
+
+config ARM_VEXPRESS_SPC_CPUFREQ
+	tristate "Versatile Express SPC based CPUfreq driver"
+	depends on ARM_BIG_LITTLE_CPUFREQ && ARCH_VEXPRESS_SPC
+	help
+	  This add the CPUfreq driver support for Versatile Express
+	  big.LITTLE platforms using SPC for power management.
+
 config ARM_BRCMSTB_AVS_CPUFREQ
 	tristate "Broadcom STB AVS CPUfreq driver"
 	depends on ARCH_BRCMSTB || COMPILE_TEST
@@ -33,20 +80,6 @@ config ARM_BRCMSTB_AVS_CPUFREQ_DEBUG
 
 	  If in doubt, say N.
 
-config ARM_DT_BL_CPUFREQ
-	tristate "Generic probing via DT for ARM big LITTLE CPUfreq driver"
-	depends on ARM_BIG_LITTLE_CPUFREQ && OF
-	help
-	  This enables probing via DT for Generic CPUfreq driver for ARM
-	  big.LITTLE platform. This gets frequency tables from DT.
-
-config ARM_VEXPRESS_SPC_CPUFREQ
-        tristate "Versatile Express SPC based CPUfreq driver"
-	depends on ARM_BIG_LITTLE_CPUFREQ && ARCH_VEXPRESS_SPC
-        help
-          This add the CPUfreq driver support for Versatile Express
-	  big.LITTLE platforms using SPC for power management.
-
 config ARM_EXYNOS5440_CPUFREQ
 	tristate "SAMSUNG EXYNOS5440"
 	depends on SOC_EXYNOS5440
@@ -205,16 +238,6 @@ config ARM_SA1100_CPUFREQ
 config ARM_SA1110_CPUFREQ
 	bool
 
-config ARM_SCPI_CPUFREQ
-        tristate "SCPI based CPUfreq driver"
-	depends on ARM_BIG_LITTLE_CPUFREQ && ARM_SCPI_PROTOCOL && COMMON_CLK_SCPI
-        help
-	  This adds the CPUfreq driver support for ARM big.LITTLE platforms
-	  using SCPI protocol for CPU power management.
-
-	  This driver uses SCPI Message Protocol driver to interact with the
-	  firmware providing the CPU DVFS functionality.
-
 config ARM_SPEAR_CPUFREQ
 	bool "SPEAr CPUFreq support"
 	depends on PLAT_SPEAR
@@ -275,20 +298,3 @@ config ARM_PXA2xx_CPUFREQ
 	  This add the CPUFreq driver support for Intel PXA2xx SOCs.
 
 	  If in doubt, say N.
-
-config ACPI_CPPC_CPUFREQ
-	tristate "CPUFreq driver based on the ACPI CPPC spec"
-	depends on ACPI_PROCESSOR
-	select ACPI_CPPC_LIB
-	default n
-	help
-	  This adds a CPUFreq driver which uses CPPC methods
-	  as described in the ACPIv5.1 spec. CPPC stands for
-	  Collaborative Processor Performance Controls. It
-	  is based on an abstract continuous scale of CPU
-	  performance values which allows the remote power
-	  processor to flexibly optimize for power and
-	  performance. CPPC relies on power management firmware
-	  support for its operation.
-
-	  If in doubt, say N.
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 812f9e0d01a3..c60c1e141d9d 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -52,23 +52,26 @@ obj-$(CONFIG_ARM_BIG_LITTLE_CPUFREQ)	+= arm_big_little.o
 # LITTLE drivers, so that it is probed last.
 obj-$(CONFIG_ARM_DT_BL_CPUFREQ)		+= arm_big_little_dt.o
 
+obj-$(CONFIG_ARM_ARMADA_37XX_CPUFREQ)	+= armada-37xx-cpufreq.o
 obj-$(CONFIG_ARM_BRCMSTB_AVS_CPUFREQ)	+= brcmstb-avs-cpufreq.o
+obj-$(CONFIG_ACPI_CPPC_CPUFREQ)		+= cppc_cpufreq.o
 obj-$(CONFIG_ARCH_DAVINCI)		+= davinci-cpufreq.o
 obj-$(CONFIG_ARM_EXYNOS5440_CPUFREQ)	+= exynos5440-cpufreq.o
 obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ)	+= highbank-cpufreq.o
 obj-$(CONFIG_ARM_IMX6Q_CPUFREQ)		+= imx6q-cpufreq.o
 obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ)	+= kirkwood-cpufreq.o
 obj-$(CONFIG_ARM_MEDIATEK_CPUFREQ)	+= mediatek-cpufreq.o
+obj-$(CONFIG_MACH_MVEBU_V7)		+= mvebu-cpufreq.o
 obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ)	+= omap-cpufreq.o
 obj-$(CONFIG_ARM_PXA2xx_CPUFREQ)	+= pxa2xx-cpufreq.o
 obj-$(CONFIG_PXA3xx)			+= pxa3xx-cpufreq.o
-obj-$(CONFIG_ARM_S3C24XX_CPUFREQ)	+= s3c24xx-cpufreq.o
-obj-$(CONFIG_ARM_S3C24XX_CPUFREQ_DEBUGFS) += s3c24xx-cpufreq-debugfs.o
 obj-$(CONFIG_ARM_S3C2410_CPUFREQ)	+= s3c2410-cpufreq.o
 obj-$(CONFIG_ARM_S3C2412_CPUFREQ)	+= s3c2412-cpufreq.o
 obj-$(CONFIG_ARM_S3C2416_CPUFREQ)	+= s3c2416-cpufreq.o
 obj-$(CONFIG_ARM_S3C2440_CPUFREQ)	+= s3c2440-cpufreq.o
 obj-$(CONFIG_ARM_S3C64XX_CPUFREQ)	+= s3c64xx-cpufreq.o
+obj-$(CONFIG_ARM_S3C24XX_CPUFREQ)	+= s3c24xx-cpufreq.o
+obj-$(CONFIG_ARM_S3C24XX_CPUFREQ_DEBUGFS) += s3c24xx-cpufreq-debugfs.o
 obj-$(CONFIG_ARM_S5PV210_CPUFREQ)	+= s5pv210-cpufreq.o
 obj-$(CONFIG_ARM_SA1100_CPUFREQ)	+= sa1100-cpufreq.o
 obj-$(CONFIG_ARM_SA1110_CPUFREQ)	+= sa1110-cpufreq.o
@@ -81,8 +84,6 @@ obj-$(CONFIG_ARM_TEGRA124_CPUFREQ)	+= tegra124-cpufreq.o
 obj-$(CONFIG_ARM_TEGRA186_CPUFREQ)	+= tegra186-cpufreq.o
 obj-$(CONFIG_ARM_TI_CPUFREQ)		+= ti-cpufreq.o
 obj-$(CONFIG_ARM_VEXPRESS_SPC_CPUFREQ)	+= vexpress-spc-cpufreq.o
-obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o
-obj-$(CONFIG_MACH_MVEBU_V7)		+= mvebu-cpufreq.o
 
 
 ##################################################################################
@@ -99,7 +100,6 @@ obj-$(CONFIG_POWERNV_CPUFREQ)		+= powernv-cpufreq.o
 
 ##################################################################################
 # Other platform drivers
-obj-$(CONFIG_AVR32_AT32AP_CPUFREQ)	+= at32ap-cpufreq.o
 obj-$(CONFIG_BFIN_CPU_FREQ)		+= blackfin-cpufreq.o
 obj-$(CONFIG_BMIPS_CPUFREQ)		+= bmips-cpufreq.o
 obj-$(CONFIG_CRIS_MACH_ARTPEC3)		+= cris-artpec3-cpufreq.o
diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c
index 3a2ca0f79daf..d0c34df0529c 100644
--- a/drivers/cpufreq/acpi-cpufreq.c
+++ b/drivers/cpufreq/acpi-cpufreq.c
@@ -629,7 +629,7 @@ static int acpi_cpufreq_blacklist(struct cpuinfo_x86 *c)
 	if (c->x86_vendor == X86_VENDOR_INTEL) {
 		if ((c->x86 == 15) &&
 		    (c->x86_model == 6) &&
-		    (c->x86_mask == 8)) {
+		    (c->x86_stepping == 8)) {
 			pr_info("Intel(R) Xeon(R) 7100 Errata AL30, processors may lock up on frequency changes: disabling acpi-cpufreq\n");
 			return -ENODEV;
 		    }
diff --git a/drivers/cpufreq/amd_freq_sensitivity.c b/drivers/cpufreq/amd_freq_sensitivity.c
index 042023bbbf62..be926d9a66e5 100644
--- a/drivers/cpufreq/amd_freq_sensitivity.c
+++ b/drivers/cpufreq/amd_freq_sensitivity.c
@@ -14,6 +14,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/types.h>
+#include <linux/pci.h>
 #include <linux/percpu-defs.h>
 #include <linux/init.h>
 #include <linux/mod_devicetable.h>
@@ -109,12 +110,18 @@ static unsigned int amd_powersave_bias_target(struct cpufreq_policy *policy,
 static int __init amd_freq_sensitivity_init(void)
 {
 	u64 val;
+	struct pci_dev *pcidev;
 
 	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
 		return -ENODEV;
 
-	if (!static_cpu_has(X86_FEATURE_PROC_FEEDBACK))
-		return -ENODEV;
+	pcidev = pci_get_device(PCI_VENDOR_ID_AMD,
+			PCI_DEVICE_ID_AMD_KERNCZ_SMBUS, NULL);
+
+	if (!pcidev) {
+		if (!static_cpu_has(X86_FEATURE_PROC_FEEDBACK))
+			return -ENODEV;
+	}
 
 	if (rdmsrl_safe(MSR_AMD64_FREQ_SENSITIVITY_ACTUAL, &val))
 		return -ENODEV;
diff --git a/drivers/cpufreq/arm_big_little.c b/drivers/cpufreq/arm_big_little.c
index 65ec5f01aa8d..c56b57dcfda5 100644
--- a/drivers/cpufreq/arm_big_little.c
+++ b/drivers/cpufreq/arm_big_little.c
@@ -526,34 +526,13 @@ static int bL_cpufreq_exit(struct cpufreq_policy *policy)
 
 static void bL_cpufreq_ready(struct cpufreq_policy *policy)
 {
-	struct device *cpu_dev = get_cpu_device(policy->cpu);
 	int cur_cluster = cpu_to_cluster(policy->cpu);
-	struct device_node *np;
 
 	/* Do not register a cpu_cooling device if we are in IKS mode */
 	if (cur_cluster >= MAX_CLUSTERS)
 		return;
 
-	np = of_node_get(cpu_dev->of_node);
-	if (WARN_ON(!np))
-		return;
-
-	if (of_find_property(np, "#cooling-cells", NULL)) {
-		u32 power_coefficient = 0;
-
-		of_property_read_u32(np, "dynamic-power-coefficient",
-				     &power_coefficient);
-
-		cdev[cur_cluster] = of_cpufreq_power_cooling_register(np,
-				policy, power_coefficient, NULL);
-		if (IS_ERR(cdev[cur_cluster])) {
-			dev_err(cpu_dev,
-				"running cpufreq without cooling device: %ld\n",
-				PTR_ERR(cdev[cur_cluster]));
-			cdev[cur_cluster] = NULL;
-		}
-	}
-	of_node_put(np);
+	cdev[cur_cluster] = of_cpufreq_cooling_register(policy);
 }
 
 static struct cpufreq_driver bL_cpufreq_driver = {
diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c
new file mode 100644
index 000000000000..c6ebc88a7d8d
--- /dev/null
+++ b/drivers/cpufreq/armada-37xx-cpufreq.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * CPU frequency scaling support for Armada 37xx platform.
+ *
+ * Copyright (C) 2017 Marvell
+ *
+ * Gregory CLEMENT <gregory.clement@free-electrons.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+/* Power management in North Bridge register set */
+#define ARMADA_37XX_NB_L0L1	0x18
+#define ARMADA_37XX_NB_L2L3	0x1C
+#define  ARMADA_37XX_NB_TBG_DIV_OFF	13
+#define  ARMADA_37XX_NB_TBG_DIV_MASK	0x7
+#define  ARMADA_37XX_NB_CLK_SEL_OFF	11
+#define  ARMADA_37XX_NB_CLK_SEL_MASK	0x1
+#define  ARMADA_37XX_NB_CLK_SEL_TBG	0x1
+#define  ARMADA_37XX_NB_TBG_SEL_OFF	9
+#define  ARMADA_37XX_NB_TBG_SEL_MASK	0x3
+#define  ARMADA_37XX_NB_VDD_SEL_OFF	6
+#define  ARMADA_37XX_NB_VDD_SEL_MASK	0x3
+#define  ARMADA_37XX_NB_CONFIG_SHIFT	16
+#define ARMADA_37XX_NB_DYN_MOD	0x24
+#define  ARMADA_37XX_NB_CLK_SEL_EN	BIT(26)
+#define  ARMADA_37XX_NB_TBG_EN		BIT(28)
+#define  ARMADA_37XX_NB_DIV_EN		BIT(29)
+#define  ARMADA_37XX_NB_VDD_EN		BIT(30)
+#define  ARMADA_37XX_NB_DFS_EN		BIT(31)
+#define ARMADA_37XX_NB_CPU_LOAD 0x30
+#define  ARMADA_37XX_NB_CPU_LOAD_MASK	0x3
+#define  ARMADA_37XX_DVFS_LOAD_0	0
+#define  ARMADA_37XX_DVFS_LOAD_1	1
+#define  ARMADA_37XX_DVFS_LOAD_2	2
+#define  ARMADA_37XX_DVFS_LOAD_3	3
+
+/*
+ * On Armada 37xx the Power management manages 4 level of CPU load,
+ * each level can be associated with a CPU clock source, a CPU
+ * divider, a VDD level, etc...
+ */
+#define LOAD_LEVEL_NR	4
+
+struct armada_37xx_dvfs {
+	u32 cpu_freq_max;
+	u8 divider[LOAD_LEVEL_NR];
+};
+
+static struct armada_37xx_dvfs armada_37xx_dvfs[] = {
+	{.cpu_freq_max = 1200*1000*1000, .divider = {1, 2, 4, 6} },
+	{.cpu_freq_max = 1000*1000*1000, .divider = {1, 2, 4, 5} },
+	{.cpu_freq_max = 800*1000*1000,  .divider = {1, 2, 3, 4} },
+	{.cpu_freq_max = 600*1000*1000,  .divider = {2, 4, 5, 6} },
+};
+
+static struct armada_37xx_dvfs *armada_37xx_cpu_freq_info_get(u32 freq)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(armada_37xx_dvfs); i++) {
+		if (freq == armada_37xx_dvfs[i].cpu_freq_max)
+			return &armada_37xx_dvfs[i];
+	}
+
+	pr_err("Unsupported CPU frequency %d MHz\n", freq/1000000);
+	return NULL;
+}
+
+/*
+ * Setup the four level managed by the hardware. Once the four level
+ * will be configured then the DVFS will be enabled.
+ */
+static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base,
+						 struct clk *clk, u8 *divider)
+{
+	int load_lvl;
+	struct clk *parent;
+
+	for (load_lvl = 0; load_lvl < LOAD_LEVEL_NR; load_lvl++) {
+		unsigned int reg, mask, val, offset = 0;
+
+		if (load_lvl <= ARMADA_37XX_DVFS_LOAD_1)
+			reg = ARMADA_37XX_NB_L0L1;
+		else
+			reg = ARMADA_37XX_NB_L2L3;
+
+		if (load_lvl == ARMADA_37XX_DVFS_LOAD_0 ||
+		    load_lvl == ARMADA_37XX_DVFS_LOAD_2)
+			offset += ARMADA_37XX_NB_CONFIG_SHIFT;
+
+		/* Set cpu clock source, for all the level we use TBG */
+		val = ARMADA_37XX_NB_CLK_SEL_TBG << ARMADA_37XX_NB_CLK_SEL_OFF;
+		mask = (ARMADA_37XX_NB_CLK_SEL_MASK
+			<< ARMADA_37XX_NB_CLK_SEL_OFF);
+
+		/*
+		 * Set cpu divider based on the pre-computed array in
+		 * order to have balanced step.
+		 */
+		val |= divider[load_lvl] << ARMADA_37XX_NB_TBG_DIV_OFF;
+		mask |= (ARMADA_37XX_NB_TBG_DIV_MASK
+			<< ARMADA_37XX_NB_TBG_DIV_OFF);
+
+		/* Set VDD divider which is actually the load level. */
+		val |= load_lvl << ARMADA_37XX_NB_VDD_SEL_OFF;
+		mask |= (ARMADA_37XX_NB_VDD_SEL_MASK
+			<< ARMADA_37XX_NB_VDD_SEL_OFF);
+
+		val <<= offset;
+		mask <<= offset;
+
+		regmap_update_bits(base, reg, mask, val);
+	}
+
+	/*
+	 * Set cpu clock source, for all the level we keep the same
+	 * clock source that the one already configured. For this one
+	 * we need to use the clock framework
+	 */
+	parent = clk_get_parent(clk);
+	clk_set_parent(clk, parent);
+}
+
+static void __init armada37xx_cpufreq_disable_dvfs(struct regmap *base)
+{
+	unsigned int reg = ARMADA_37XX_NB_DYN_MOD,
+		mask = ARMADA_37XX_NB_DFS_EN;
+
+	regmap_update_bits(base, reg, mask, 0);
+}
+
+static void __init armada37xx_cpufreq_enable_dvfs(struct regmap *base)
+{
+	unsigned int val, reg = ARMADA_37XX_NB_CPU_LOAD,
+		mask = ARMADA_37XX_NB_CPU_LOAD_MASK;
+
+	/* Start with the highest load (0) */
+	val = ARMADA_37XX_DVFS_LOAD_0;
+	regmap_update_bits(base, reg, mask, val);
+
+	/* Now enable DVFS for the CPUs */
+	reg = ARMADA_37XX_NB_DYN_MOD;
+	mask =	ARMADA_37XX_NB_CLK_SEL_EN | ARMADA_37XX_NB_TBG_EN |
+		ARMADA_37XX_NB_DIV_EN | ARMADA_37XX_NB_VDD_EN |
+		ARMADA_37XX_NB_DFS_EN;
+
+	regmap_update_bits(base, reg, mask, mask);
+}
+
+static int __init armada37xx_cpufreq_driver_init(void)
+{
+	struct armada_37xx_dvfs *dvfs;
+	struct platform_device *pdev;
+	unsigned int cur_frequency;
+	struct regmap *nb_pm_base;
+	struct device *cpu_dev;
+	int load_lvl, ret;
+	struct clk *clk;
+
+	nb_pm_base =
+		syscon_regmap_lookup_by_compatible("marvell,armada-3700-nb-pm");
+
+	if (IS_ERR(nb_pm_base))
+		return -ENODEV;
+
+	/* Before doing any configuration on the DVFS first, disable it */
+	armada37xx_cpufreq_disable_dvfs(nb_pm_base);
+
+	/*
+	 * On CPU 0 register the operating points supported (which are
+	 * the nominal CPU frequency and full integer divisions of
+	 * it).
+	 */
+	cpu_dev = get_cpu_device(0);
+	if (!cpu_dev) {
+		dev_err(cpu_dev, "Cannot get CPU\n");
+		return -ENODEV;
+	}
+
+	clk = clk_get(cpu_dev, 0);
+	if (IS_ERR(clk)) {
+		dev_err(cpu_dev, "Cannot get clock for CPU0\n");
+		return PTR_ERR(clk);
+	}
+
+	/* Get nominal (current) CPU frequency */
+	cur_frequency = clk_get_rate(clk);
+	if (!cur_frequency) {
+		dev_err(cpu_dev, "Failed to get clock rate for CPU\n");
+		return -EINVAL;
+	}
+
+	dvfs = armada_37xx_cpu_freq_info_get(cur_frequency);
+	if (!dvfs)
+		return -EINVAL;
+
+	armada37xx_cpufreq_dvfs_setup(nb_pm_base, clk, dvfs->divider);
+
+	for (load_lvl = ARMADA_37XX_DVFS_LOAD_0; load_lvl < LOAD_LEVEL_NR;
+	     load_lvl++) {
+		unsigned long freq = cur_frequency / dvfs->divider[load_lvl];
+
+		ret = dev_pm_opp_add(cpu_dev, freq, 0);
+		if (ret) {
+			/* clean-up the already added opp before leaving */
+			while (load_lvl-- > ARMADA_37XX_DVFS_LOAD_0) {
+				freq = cur_frequency / dvfs->divider[load_lvl];
+				dev_pm_opp_remove(cpu_dev, freq);
+			}
+			return ret;
+		}
+	}
+
+	/* Now that everything is setup, enable the DVFS at hardware level */
+	armada37xx_cpufreq_enable_dvfs(nb_pm_base);
+
+	pdev = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
+
+	return PTR_ERR_OR_ZERO(pdev);
+}
+/* late_initcall, to guarantee the driver is loaded after A37xx clock driver */
+late_initcall(armada37xx_cpufreq_driver_init);
+
+MODULE_AUTHOR("Gregory CLEMENT <gregory.clement@free-electrons.com>");
+MODULE_DESCRIPTION("Armada 37xx cpufreq driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/at32ap-cpufreq.c b/drivers/cpufreq/at32ap-cpufreq.c
deleted file mode 100644
index 7b612c8bb09e..000000000000
--- a/drivers/cpufreq/at32ap-cpufreq.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright (C) 2004-2007 Atmel Corporation
- *
- * Based on MIPS implementation arch/mips/kernel/time.c
- *   Copyright 2001 MontaVista Software Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-/*#define DEBUG*/
-
-#include <linux/kernel.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/cpufreq.h>
-#include <linux/io.h>
-#include <linux/clk.h>
-#include <linux/err.h>
-#include <linux/export.h>
-#include <linux/slab.h>
-
-static struct cpufreq_frequency_table *freq_table;
-
-static unsigned int	ref_freq;
-static unsigned long	loops_per_jiffy_ref;
-
-static int at32_set_target(struct cpufreq_policy *policy, unsigned int index)
-{
-	unsigned int old_freq, new_freq;
-
-	old_freq = policy->cur;
-	new_freq = freq_table[index].frequency;
-
-	if (!ref_freq) {
-		ref_freq = old_freq;
-		loops_per_jiffy_ref = boot_cpu_data.loops_per_jiffy;
-	}
-
-	if (old_freq < new_freq)
-		boot_cpu_data.loops_per_jiffy = cpufreq_scale(
-				loops_per_jiffy_ref, ref_freq, new_freq);
-	clk_set_rate(policy->clk, new_freq * 1000);
-	if (new_freq < old_freq)
-		boot_cpu_data.loops_per_jiffy = cpufreq_scale(
-				loops_per_jiffy_ref, ref_freq, new_freq);
-
-	return 0;
-}
-
-static int at32_cpufreq_driver_init(struct cpufreq_policy *policy)
-{
-	unsigned int frequency, rate, min_freq;
-	struct clk *cpuclk;
-	int retval, steps, i;
-
-	if (policy->cpu != 0)
-		return -EINVAL;
-
-	cpuclk = clk_get(NULL, "cpu");
-	if (IS_ERR(cpuclk)) {
-		pr_debug("cpufreq: could not get CPU clk\n");
-		retval = PTR_ERR(cpuclk);
-		goto out_err;
-	}
-
-	min_freq = (clk_round_rate(cpuclk, 1) + 500) / 1000;
-	frequency = (clk_round_rate(cpuclk, ~0UL) + 500) / 1000;
-	policy->cpuinfo.transition_latency = 0;
-
-	/*
-	 * AVR32 CPU frequency rate scales in power of two between maximum and
-	 * minimum, also add space for the table end marker.
-	 *
-	 * Further validate that the frequency is usable, and append it to the
-	 * frequency table.
-	 */
-	steps = fls(frequency / min_freq) + 1;
-	freq_table = kzalloc(steps * sizeof(struct cpufreq_frequency_table),
-			GFP_KERNEL);
-	if (!freq_table) {
-		retval = -ENOMEM;
-		goto out_err_put_clk;
-	}
-
-	for (i = 0; i < (steps - 1); i++) {
-		rate = clk_round_rate(cpuclk, frequency * 1000) / 1000;
-
-		if (rate != frequency)
-			freq_table[i].frequency = CPUFREQ_ENTRY_INVALID;
-		else
-			freq_table[i].frequency = frequency;
-
-		frequency /= 2;
-	}
-
-	policy->clk = cpuclk;
-	freq_table[steps - 1].frequency = CPUFREQ_TABLE_END;
-
-	retval = cpufreq_table_validate_and_show(policy, freq_table);
-	if (!retval) {
-		printk("cpufreq: AT32AP CPU frequency driver\n");
-		return 0;
-	}
-
-	kfree(freq_table);
-out_err_put_clk:
-	clk_put(cpuclk);
-out_err:
-	return retval;
-}
-
-static struct cpufreq_driver at32_driver = {
-	.name		= "at32ap",
-	.init		= at32_cpufreq_driver_init,
-	.verify		= cpufreq_generic_frequency_table_verify,
-	.target_index	= at32_set_target,
-	.get		= cpufreq_generic_get,
-	.flags		= CPUFREQ_STICKY,
-};
-
-static int __init at32_cpufreq_init(void)
-{
-	return cpufreq_register_driver(&at32_driver);
-}
-late_initcall(at32_cpufreq_init);
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index ecc56e26f8f6..3b585e4bfac5 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -108,6 +108,14 @@ static const struct of_device_id blacklist[] __initconst = {
 
 	{ .compatible = "marvell,armadaxp", },
 
+	{ .compatible = "mediatek,mt2701", },
+	{ .compatible = "mediatek,mt2712", },
+	{ .compatible = "mediatek,mt7622", },
+	{ .compatible = "mediatek,mt7623", },
+	{ .compatible = "mediatek,mt817x", },
+	{ .compatible = "mediatek,mt8173", },
+	{ .compatible = "mediatek,mt8176", },
+
 	{ .compatible = "nvidia,tegra124", },
 
 	{ .compatible = "st,stih407", },
diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c
index 545946ad0752..de3d104c25d7 100644
--- a/drivers/cpufreq/cpufreq-dt.c
+++ b/drivers/cpufreq/cpufreq-dt.c
@@ -319,33 +319,8 @@ static int cpufreq_exit(struct cpufreq_policy *policy)
 static void cpufreq_ready(struct cpufreq_policy *policy)
 {
 	struct private_data *priv = policy->driver_data;
-	struct device_node *np = of_node_get(priv->cpu_dev->of_node);
 
-	if (WARN_ON(!np))
-		return;
-
-	/*
-	 * For now, just loading the cooling device;
-	 * thermal DT code takes care of matching them.
-	 */
-	if (of_find_property(np, "#cooling-cells", NULL)) {
-		u32 power_coefficient = 0;
-
-		of_property_read_u32(np, "dynamic-power-coefficient",
-				     &power_coefficient);
-
-		priv->cdev = of_cpufreq_power_cooling_register(np,
-				policy, power_coefficient, NULL);
-		if (IS_ERR(priv->cdev)) {
-			dev_err(priv->cpu_dev,
-				"running cpufreq without cooling device: %ld\n",
-				PTR_ERR(priv->cdev));
-
-			priv->cdev = NULL;
-		}
-	}
-
-	of_node_put(np);
+	priv->cdev = of_cpufreq_cooling_register(policy);
 }
 
 static struct cpufreq_driver dt_cpufreq_driver = {
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 41d148af7748..de33ebf008ad 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -601,19 +601,18 @@ static struct cpufreq_governor *find_governor(const char *str_governor)
 /**
  * cpufreq_parse_governor - parse a governor string
  */
-static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
-				struct cpufreq_governor **governor)
+static int cpufreq_parse_governor(char *str_governor,
+				  struct cpufreq_policy *policy)
 {
-	int err = -EINVAL;
-
 	if (cpufreq_driver->setpolicy) {
 		if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
-			*policy = CPUFREQ_POLICY_PERFORMANCE;
-			err = 0;
-		} else if (!strncasecmp(str_governor, "powersave",
-						CPUFREQ_NAME_LEN)) {
-			*policy = CPUFREQ_POLICY_POWERSAVE;
-			err = 0;
+			policy->policy = CPUFREQ_POLICY_PERFORMANCE;
+			return 0;
+		}
+
+		if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) {
+			policy->policy = CPUFREQ_POLICY_POWERSAVE;
+			return 0;
 		}
 	} else {
 		struct cpufreq_governor *t;
@@ -621,26 +620,31 @@ static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 		mutex_lock(&cpufreq_governor_mutex);
 
 		t = find_governor(str_governor);
-
-		if (t == NULL) {
+		if (!t) {
 			int ret;
 
 			mutex_unlock(&cpufreq_governor_mutex);
+
 			ret = request_module("cpufreq_%s", str_governor);
-			mutex_lock(&cpufreq_governor_mutex);
+			if (ret)
+				return -EINVAL;
 
-			if (ret == 0)
-				t = find_governor(str_governor);
-		}
+			mutex_lock(&cpufreq_governor_mutex);
 
-		if (t != NULL) {
-			*governor = t;
-			err = 0;
+			t = find_governor(str_governor);
 		}
+		if (t && !try_module_get(t->owner))
+			t = NULL;
 
 		mutex_unlock(&cpufreq_governor_mutex);
+
+		if (t) {
+			policy->governor = t;
+			return 0;
+		}
 	}
-	return err;
+
+	return -EINVAL;
 }
 
 /**
@@ -760,11 +764,14 @@ static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 	if (ret != 1)
 		return -EINVAL;
 
-	if (cpufreq_parse_governor(str_governor, &new_policy.policy,
-						&new_policy.governor))
+	if (cpufreq_parse_governor(str_governor, &new_policy))
 		return -EINVAL;
 
 	ret = cpufreq_set_policy(policy, &new_policy);
+
+	if (new_policy.governor)
+		module_put(new_policy.governor->owner);
+
 	return ret ? ret : count;
 }
 
@@ -1044,8 +1051,7 @@ static int cpufreq_init_policy(struct cpufreq_policy *policy)
 		if (policy->last_policy)
 			new_policy.policy = policy->last_policy;
 		else
-			cpufreq_parse_governor(gov->name, &new_policy.policy,
-					       NULL);
+			cpufreq_parse_governor(gov->name, &new_policy);
 	}
 	/* set default policy */
 	return cpufreq_set_policy(policy, &new_policy);
@@ -1680,6 +1686,9 @@ void cpufreq_resume(void)
 	if (!cpufreq_driver)
 		return;
 
+	if (unlikely(!cpufreq_suspended))
+		return;
+
 	cpufreq_suspended = false;
 
 	if (!has_target() && !cpufreq_driver->resume)
@@ -2160,7 +2169,6 @@ void cpufreq_unregister_governor(struct cpufreq_governor *governor)
 	mutex_lock(&cpufreq_governor_mutex);
 	list_del(&governor->governor_list);
 	mutex_unlock(&cpufreq_governor_mutex);
-	return;
 }
 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
 
diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c
index 1e55b5790853..1572129844a5 100644
--- a/drivers/cpufreq/cpufreq_stats.c
+++ b/drivers/cpufreq/cpufreq_stats.c
@@ -27,7 +27,7 @@ struct cpufreq_stats {
 	unsigned int *trans_table;
 };
 
-static int cpufreq_stats_update(struct cpufreq_stats *stats)
+static void cpufreq_stats_update(struct cpufreq_stats *stats)
 {
 	unsigned long long cur_time = get_jiffies_64();
 
@@ -35,7 +35,6 @@ static int cpufreq_stats_update(struct cpufreq_stats *stats)
 	stats->time_in_state[stats->last_index] += cur_time - stats->last_time;
 	stats->last_time = cur_time;
 	spin_unlock(&cpufreq_stats_lock);
-	return 0;
 }
 
 static void cpufreq_stats_clear_table(struct cpufreq_stats *stats)
diff --git a/drivers/cpufreq/exynos5440-cpufreq.c b/drivers/cpufreq/exynos5440-cpufreq.c
index b6b369c22272..932caa386ece 100644
--- a/drivers/cpufreq/exynos5440-cpufreq.c
+++ b/drivers/cpufreq/exynos5440-cpufreq.c
@@ -115,10 +115,10 @@ static struct cpufreq_freqs freqs;
 static int init_div_table(void)
 {
 	struct cpufreq_frequency_table *pos, *freq_tbl = dvfs_info->freq_table;
-	unsigned int tmp, clk_div, ema_div, freq, volt_id;
+	unsigned int tmp, clk_div, ema_div, freq, volt_id, idx;
 	struct dev_pm_opp *opp;
 
-	cpufreq_for_each_entry(pos, freq_tbl) {
+	cpufreq_for_each_entry_idx(pos, freq_tbl, idx) {
 		opp = dev_pm_opp_find_freq_exact(dvfs_info->dev,
 					pos->frequency * 1000, true);
 		if (IS_ERR(opp)) {
@@ -154,8 +154,7 @@ static int init_div_table(void)
 		tmp = (clk_div | ema_div | (volt_id << P0_7_VDD_SHIFT)
 			| ((freq / FREQ_UNIT) << P0_7_FREQ_SHIFT));
 
-		__raw_writel(tmp, dvfs_info->base + XMU_PMU_P0_7 + 4 *
-						(pos - freq_tbl));
+		__raw_writel(tmp, dvfs_info->base + XMU_PMU_P0_7 + 4 * idx);
 		dev_pm_opp_put(opp);
 	}
 
diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c
index 3bbbf9e6960c..6d007f824ca7 100644
--- a/drivers/cpufreq/freq_table.c
+++ b/drivers/cpufreq/freq_table.c
@@ -143,10 +143,9 @@ int cpufreq_table_index_unsorted(struct cpufreq_policy *policy,
 		break;
 	}
 
-	cpufreq_for_each_valid_entry(pos, table) {
+	cpufreq_for_each_valid_entry_idx(pos, table, i) {
 		freq = pos->frequency;
 
-		i = pos - table;
 		if ((freq < policy->min) || (freq > policy->max))
 			continue;
 		if (freq == target_freq) {
@@ -211,15 +210,16 @@ int cpufreq_frequency_table_get_index(struct cpufreq_policy *policy,
 		unsigned int freq)
 {
 	struct cpufreq_frequency_table *pos, *table = policy->freq_table;
+	int idx;
 
 	if (unlikely(!table)) {
 		pr_debug("%s: Unable to find frequency table\n", __func__);
 		return -ENOENT;
 	}
 
-	cpufreq_for_each_valid_entry(pos, table)
+	cpufreq_for_each_valid_entry_idx(pos, table, idx)
 		if (pos->frequency == freq)
-			return pos - table;
+			return idx;
 
 	return -EINVAL;
 }
diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c
index d9b2c2de49c4..ff67859948b3 100644
--- a/drivers/cpufreq/imx6q-cpufreq.c
+++ b/drivers/cpufreq/imx6q-cpufreq.c
@@ -25,15 +25,29 @@ static struct regulator *arm_reg;
 static struct regulator *pu_reg;
 static struct regulator *soc_reg;
 
-static struct clk *arm_clk;
-static struct clk *pll1_sys_clk;
-static struct clk *pll1_sw_clk;
-static struct clk *step_clk;
-static struct clk *pll2_pfd2_396m_clk;
-
-/* clk used by i.MX6UL */
-static struct clk *pll2_bus_clk;
-static struct clk *secondary_sel_clk;
+enum IMX6_CPUFREQ_CLKS {
+	ARM,
+	PLL1_SYS,
+	STEP,
+	PLL1_SW,
+	PLL2_PFD2_396M,
+	/* MX6UL requires two more clks */
+	PLL2_BUS,
+	SECONDARY_SEL,
+};
+#define IMX6Q_CPUFREQ_CLK_NUM		5
+#define IMX6UL_CPUFREQ_CLK_NUM		7
+
+static int num_clks;
+static struct clk_bulk_data clks[] = {
+	{ .id = "arm" },
+	{ .id = "pll1_sys" },
+	{ .id = "step" },
+	{ .id = "pll1_sw" },
+	{ .id = "pll2_pfd2_396m" },
+	{ .id = "pll2_bus" },
+	{ .id = "secondary_sel" },
+};
 
 static struct device *cpu_dev;
 static bool free_opp;
@@ -53,7 +67,7 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
 
 	new_freq = freq_table[index].frequency;
 	freq_hz = new_freq * 1000;
-	old_freq = clk_get_rate(arm_clk) / 1000;
+	old_freq = clk_get_rate(clks[ARM].clk) / 1000;
 
 	opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_hz);
 	if (IS_ERR(opp)) {
@@ -112,29 +126,35 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
 		 * voltage of 528MHz, so lower the CPU frequency to one
 		 * half before changing CPU frequency.
 		 */
-		clk_set_rate(arm_clk, (old_freq >> 1) * 1000);
-		clk_set_parent(pll1_sw_clk, pll1_sys_clk);
-		if (freq_hz > clk_get_rate(pll2_pfd2_396m_clk))
-			clk_set_parent(secondary_sel_clk, pll2_bus_clk);
+		clk_set_rate(clks[ARM].clk, (old_freq >> 1) * 1000);
+		clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk);
+		if (freq_hz > clk_get_rate(clks[PLL2_PFD2_396M].clk))
+			clk_set_parent(clks[SECONDARY_SEL].clk,
+				       clks[PLL2_BUS].clk);
 		else
-			clk_set_parent(secondary_sel_clk, pll2_pfd2_396m_clk);
-		clk_set_parent(step_clk, secondary_sel_clk);
-		clk_set_parent(pll1_sw_clk, step_clk);
+			clk_set_parent(clks[SECONDARY_SEL].clk,
+				       clks[PLL2_PFD2_396M].clk);
+		clk_set_parent(clks[STEP].clk, clks[SECONDARY_SEL].clk);
+		clk_set_parent(clks[PLL1_SW].clk, clks[STEP].clk);
+		if (freq_hz > clk_get_rate(clks[PLL2_BUS].clk)) {
+			clk_set_rate(clks[PLL1_SYS].clk, new_freq * 1000);
+			clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk);
+		}
 	} else {
-		clk_set_parent(step_clk, pll2_pfd2_396m_clk);
-		clk_set_parent(pll1_sw_clk, step_clk);
-		if (freq_hz > clk_get_rate(pll2_pfd2_396m_clk)) {
-			clk_set_rate(pll1_sys_clk, new_freq * 1000);
-			clk_set_parent(pll1_sw_clk, pll1_sys_clk);
+		clk_set_parent(clks[STEP].clk, clks[PLL2_PFD2_396M].clk);
+		clk_set_parent(clks[PLL1_SW].clk, clks[STEP].clk);
+		if (freq_hz > clk_get_rate(clks[PLL2_PFD2_396M].clk)) {
+			clk_set_rate(clks[PLL1_SYS].clk, new_freq * 1000);
+			clk_set_parent(clks[PLL1_SW].clk, clks[PLL1_SYS].clk);
 		} else {
 			/* pll1_sys needs to be enabled for divider rate change to work. */
 			pll1_sys_temp_enabled = true;
-			clk_prepare_enable(pll1_sys_clk);
+			clk_prepare_enable(clks[PLL1_SYS].clk);
 		}
 	}
 
 	/* Ensure the arm clock divider is what we expect */
-	ret = clk_set_rate(arm_clk, new_freq * 1000);
+	ret = clk_set_rate(clks[ARM].clk, new_freq * 1000);
 	if (ret) {
 		dev_err(cpu_dev, "failed to set clock rate: %d\n", ret);
 		regulator_set_voltage_tol(arm_reg, volt_old, 0);
@@ -143,7 +163,7 @@ static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index)
 
 	/* PLL1 is only needed until after ARM-PODF is set. */
 	if (pll1_sys_temp_enabled)
-		clk_disable_unprepare(pll1_sys_clk);
+		clk_disable_unprepare(clks[PLL1_SYS].clk);
 
 	/* scaling down?  scale voltage after frequency */
 	if (new_freq < old_freq) {
@@ -174,7 +194,7 @@ static int imx6q_cpufreq_init(struct cpufreq_policy *policy)
 {
 	int ret;
 
-	policy->clk = arm_clk;
+	policy->clk = clks[ARM].clk;
 	ret = cpufreq_generic_init(policy, freq_table, transition_latency);
 	policy->suspend_freq = policy->max;
 
@@ -244,6 +264,43 @@ static void imx6q_opp_check_speed_grading(struct device *dev)
 	of_node_put(np);
 }
 
+#define OCOTP_CFG3_6UL_SPEED_696MHZ	0x2
+
+static void imx6ul_opp_check_speed_grading(struct device *dev)
+{
+	struct device_node *np;
+	void __iomem *base;
+	u32 val;
+
+	np = of_find_compatible_node(NULL, NULL, "fsl,imx6ul-ocotp");
+	if (!np)
+		return;
+
+	base = of_iomap(np, 0);
+	if (!base) {
+		dev_err(dev, "failed to map ocotp\n");
+		goto put_node;
+	}
+
+	/*
+	 * Speed GRADING[1:0] defines the max speed of ARM:
+	 * 2b'00: Reserved;
+	 * 2b'01: 528000000Hz;
+	 * 2b'10: 696000000Hz;
+	 * 2b'11: Reserved;
+	 * We need to set the max speed of ARM according to fuse map.
+	 */
+	val = readl_relaxed(base + OCOTP_CFG3);
+	val >>= OCOTP_CFG3_SPEED_SHIFT;
+	val &= 0x3;
+	if (val != OCOTP_CFG3_6UL_SPEED_696MHZ)
+		if (dev_pm_opp_disable(dev, 696000000))
+			dev_warn(dev, "failed to disable 696MHz OPP\n");
+	iounmap(base);
+put_node:
+	of_node_put(np);
+}
+
 static int imx6q_cpufreq_probe(struct platform_device *pdev)
 {
 	struct device_node *np;
@@ -266,28 +323,15 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev)
 		return -ENOENT;
 	}
 
-	arm_clk = clk_get(cpu_dev, "arm");
-	pll1_sys_clk = clk_get(cpu_dev, "pll1_sys");
-	pll1_sw_clk = clk_get(cpu_dev, "pll1_sw");
-	step_clk = clk_get(cpu_dev, "step");
-	pll2_pfd2_396m_clk = clk_get(cpu_dev, "pll2_pfd2_396m");
-	if (IS_ERR(arm_clk) || IS_ERR(pll1_sys_clk) || IS_ERR(pll1_sw_clk) ||
-	    IS_ERR(step_clk) || IS_ERR(pll2_pfd2_396m_clk)) {
-		dev_err(cpu_dev, "failed to get clocks\n");
-		ret = -ENOENT;
-		goto put_clk;
-	}
-
 	if (of_machine_is_compatible("fsl,imx6ul") ||
-	    of_machine_is_compatible("fsl,imx6ull")) {
-		pll2_bus_clk = clk_get(cpu_dev, "pll2_bus");
-		secondary_sel_clk = clk_get(cpu_dev, "secondary_sel");
-		if (IS_ERR(pll2_bus_clk) || IS_ERR(secondary_sel_clk)) {
-			dev_err(cpu_dev, "failed to get clocks specific to imx6ul\n");
-			ret = -ENOENT;
-			goto put_clk;
-		}
-	}
+	    of_machine_is_compatible("fsl,imx6ull"))
+		num_clks = IMX6UL_CPUFREQ_CLK_NUM;
+	else
+		num_clks = IMX6Q_CPUFREQ_CLK_NUM;
+
+	ret = clk_bulk_get(cpu_dev, num_clks, clks);
+	if (ret)
+		goto put_node;
 
 	arm_reg = regulator_get(cpu_dev, "arm");
 	pu_reg = regulator_get_optional(cpu_dev, "pu");
@@ -311,7 +355,10 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev)
 		goto put_reg;
 	}
 
-	imx6q_opp_check_speed_grading(cpu_dev);
+	if (of_machine_is_compatible("fsl,imx6ul"))
+		imx6ul_opp_check_speed_grading(cpu_dev);
+	else
+		imx6q_opp_check_speed_grading(cpu_dev);
 
 	/* Because we have added the OPPs here, we must free them */
 	free_opp = true;
@@ -424,22 +471,11 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev)
 		regulator_put(pu_reg);
 	if (!IS_ERR(soc_reg))
 		regulator_put(soc_reg);
-put_clk:
-	if (!IS_ERR(arm_clk))
-		clk_put(arm_clk);
-	if (!IS_ERR(pll1_sys_clk))
-		clk_put(pll1_sys_clk);
-	if (!IS_ERR(pll1_sw_clk))
-		clk_put(pll1_sw_clk);
-	if (!IS_ERR(step_clk))
-		clk_put(step_clk);
-	if (!IS_ERR(pll2_pfd2_396m_clk))
-		clk_put(pll2_pfd2_396m_clk);
-	if (!IS_ERR(pll2_bus_clk))
-		clk_put(pll2_bus_clk);
-	if (!IS_ERR(secondary_sel_clk))
-		clk_put(secondary_sel_clk);
+
+	clk_bulk_put(num_clks, clks);
+put_node:
 	of_node_put(np);
+
 	return ret;
 }
 
@@ -453,13 +489,8 @@ static int imx6q_cpufreq_remove(struct platform_device *pdev)
 	if (!IS_ERR(pu_reg))
 		regulator_put(pu_reg);
 	regulator_put(soc_reg);
-	clk_put(arm_clk);
-	clk_put(pll1_sys_clk);
-	clk_put(pll1_sw_clk);
-	clk_put(step_clk);
-	clk_put(pll2_pfd2_396m_clk);
-	clk_put(pll2_bus_clk);
-	clk_put(secondary_sel_clk);
+
+	clk_bulk_put(num_clks, clks);
 
 	return 0;
 }
@@ -473,6 +504,7 @@ static struct platform_driver imx6q_cpufreq_platdrv = {
 };
 module_platform_driver(imx6q_cpufreq_platdrv);
 
+MODULE_ALIAS("platform:imx6q-cpufreq");
 MODULE_AUTHOR("Shawn Guo <shawn.guo@linaro.org>");
 MODULE_DESCRIPTION("Freescale i.MX6Q cpufreq driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 93a0e88bef76..6d084c61ee25 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -779,6 +779,8 @@ static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
 	return 0;
 }
 
+static void intel_pstate_hwp_enable(struct cpudata *cpudata);
+
 static int intel_pstate_resume(struct cpufreq_policy *policy)
 {
 	if (!hwp_active)
@@ -786,6 +788,9 @@ static int intel_pstate_resume(struct cpufreq_policy *policy)
 
 	mutex_lock(&intel_pstate_limits_lock);
 
+	if (policy->cpu == 0)
+		intel_pstate_hwp_enable(all_cpu_data[policy->cpu]);
+
 	all_cpu_data[policy->cpu]->epp_policy = 0;
 	intel_pstate_hwp_set(policy->cpu);
 
@@ -1595,15 +1600,6 @@ static const struct pstate_funcs knl_funcs = {
 	.get_val = core_get_val,
 };
 
-static const struct pstate_funcs bxt_funcs = {
-	.get_max = core_get_max_pstate,
-	.get_max_physical = core_get_max_pstate_physical,
-	.get_min = core_get_min_pstate,
-	.get_turbo = core_get_turbo_pstate,
-	.get_scaling = core_get_scaling,
-	.get_val = core_get_val,
-};
-
 #define ICPU(model, policy) \
 	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
 			(unsigned long)&policy }
@@ -1627,8 +1623,9 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
 	ICPU(INTEL_FAM6_BROADWELL_XEON_D,	core_funcs),
 	ICPU(INTEL_FAM6_XEON_PHI_KNL,		knl_funcs),
 	ICPU(INTEL_FAM6_XEON_PHI_KNM,		knl_funcs),
-	ICPU(INTEL_FAM6_ATOM_GOLDMONT,		bxt_funcs),
-	ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE,       bxt_funcs),
+	ICPU(INTEL_FAM6_ATOM_GOLDMONT,		core_funcs),
+	ICPU(INTEL_FAM6_ATOM_GEMINI_LAKE,       core_funcs),
+	ICPU(INTEL_FAM6_SKYLAKE_X,		core_funcs),
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
diff --git a/drivers/cpufreq/longhaul.c b/drivers/cpufreq/longhaul.c
index c46a12df40dd..f730b6528c18 100644
--- a/drivers/cpufreq/longhaul.c
+++ b/drivers/cpufreq/longhaul.c
@@ -600,7 +600,7 @@ static void longhaul_setup_voltagescaling(void)
 	/* Calculate kHz for one voltage step */
 	kHz_step = (highest_speed - min_vid_speed) / numvscales;
 
-	cpufreq_for_each_entry(freq_pos, longhaul_table) {
+	cpufreq_for_each_entry_idx(freq_pos, longhaul_table, j) {
 		speed = freq_pos->frequency;
 		if (speed > min_vid_speed)
 			pos = (speed - min_vid_speed) / kHz_step + minvid.pos;
@@ -609,7 +609,7 @@ static void longhaul_setup_voltagescaling(void)
 		freq_pos->driver_data |= mV_vrm_table[pos] << 8;
 		vid = vrm_mV_table[mV_vrm_table[pos]];
 		pr_info("f: %d kHz, index: %d, vid: %d mV\n",
-			speed, (int)(freq_pos - longhaul_table), vid.mV);
+			speed, j, vid.mV);
 	}
 
 	can_scale_voltage = 1;
@@ -775,7 +775,7 @@ static int longhaul_cpu_init(struct cpufreq_policy *policy)
 		break;
 
 	case 7:
-		switch (c->x86_mask) {
+		switch (c->x86_stepping) {
 		case 0:
 			longhaul_version = TYPE_LONGHAUL_V1;
 			cpu_model = CPU_SAMUEL2;
@@ -787,7 +787,7 @@ static int longhaul_cpu_init(struct cpufreq_policy *policy)
 			break;
 		case 1 ... 15:
 			longhaul_version = TYPE_LONGHAUL_V2;
-			if (c->x86_mask < 8) {
+			if (c->x86_stepping < 8) {
 				cpu_model = CPU_SAMUEL2;
 				cpuname = "C3 'Samuel 2' [C5B]";
 			} else {
@@ -814,7 +814,7 @@ static int longhaul_cpu_init(struct cpufreq_policy *policy)
 		numscales = 32;
 		memcpy(mults, nehemiah_mults, sizeof(nehemiah_mults));
 		memcpy(eblcr, nehemiah_eblcr, sizeof(nehemiah_eblcr));
-		switch (c->x86_mask) {
+		switch (c->x86_stepping) {
 		case 0 ... 1:
 			cpu_model = CPU_NEHEMIAH;
 			cpuname = "C3 'Nehemiah A' [C5XLOE]";
@@ -894,7 +894,7 @@ static int longhaul_cpu_init(struct cpufreq_policy *policy)
 	if ((longhaul_version != TYPE_LONGHAUL_V1) && (scale_voltage != 0))
 		longhaul_setup_voltagescaling();
 
-	policy->cpuinfo.transition_latency = 200000;	/* nsec */
+	policy->transition_delay_us = 200000;	/* usec */
 
 	return cpufreq_table_validate_and_show(policy, longhaul_table);
 }
diff --git a/drivers/cpufreq/mediatek-cpufreq.c b/drivers/cpufreq/mediatek-cpufreq.c
index e0d5090b303d..8c04dddd3c28 100644
--- a/drivers/cpufreq/mediatek-cpufreq.c
+++ b/drivers/cpufreq/mediatek-cpufreq.c
@@ -310,28 +310,8 @@ static int mtk_cpufreq_set_target(struct cpufreq_policy *policy,
 static void mtk_cpufreq_ready(struct cpufreq_policy *policy)
 {
 	struct mtk_cpu_dvfs_info *info = policy->driver_data;
-	struct device_node *np = of_node_get(info->cpu_dev->of_node);
-	u32 capacitance = 0;
 
-	if (WARN_ON(!np))
-		return;
-
-	if (of_find_property(np, "#cooling-cells", NULL)) {
-		of_property_read_u32(np, DYNAMIC_POWER, &capacitance);
-
-		info->cdev = of_cpufreq_power_cooling_register(np,
-						policy, capacitance, NULL);
-
-		if (IS_ERR(info->cdev)) {
-			dev_err(info->cpu_dev,
-				"running cpufreq without cooling device: %ld\n",
-				PTR_ERR(info->cdev));
-
-			info->cdev = NULL;
-		}
-	}
-
-	of_node_put(np);
+	info->cdev = of_cpufreq_cooling_register(policy);
 }
 
 static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu)
@@ -574,6 +554,7 @@ static struct platform_driver mtk_cpufreq_platdrv = {
 /* List of machines supported by this driver */
 static const struct of_device_id mtk_cpufreq_machines[] __initconst = {
 	{ .compatible = "mediatek,mt2701", },
+	{ .compatible = "mediatek,mt2712", },
 	{ .compatible = "mediatek,mt7622", },
 	{ .compatible = "mediatek,mt7623", },
 	{ .compatible = "mediatek,mt817x", },
diff --git a/drivers/cpufreq/mvebu-cpufreq.c b/drivers/cpufreq/mvebu-cpufreq.c
index ed915ee85dd9..31513bd42705 100644
--- a/drivers/cpufreq/mvebu-cpufreq.c
+++ b/drivers/cpufreq/mvebu-cpufreq.c
@@ -76,12 +76,6 @@ static int __init armada_xp_pmsu_cpufreq_init(void)
 			return PTR_ERR(clk);
 		}
 
-		/*
-		 * In case of a failure of dev_pm_opp_add(), we don't
-		 * bother with cleaning up the registered OPP (there's
-		 * no function to do so), and simply cancel the
-		 * registration of the cpufreq device.
-		 */
 		ret = dev_pm_opp_add(cpu_dev, clk_get_rate(clk), 0);
 		if (ret) {
 			clk_put(clk);
@@ -91,7 +85,8 @@ static int __init armada_xp_pmsu_cpufreq_init(void)
 		ret = dev_pm_opp_add(cpu_dev, clk_get_rate(clk) / 2, 0);
 		if (ret) {
 			clk_put(clk);
-			return ret;
+			dev_err(cpu_dev, "Failed to register OPPs\n");
+			goto opp_register_failed;
 		}
 
 		ret = dev_pm_opp_set_sharing_cpus(cpu_dev,
@@ -99,9 +94,16 @@ static int __init armada_xp_pmsu_cpufreq_init(void)
 		if (ret)
 			dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
 				__func__, ret);
+		clk_put(clk);
 	}
 
 	platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
 	return 0;
+
+opp_register_failed:
+	/* As registering has failed remove all the opp for all cpus */
+	dev_pm_opp_cpumask_remove_table(cpu_possible_mask);
+
+	return ret;
 }
 device_initcall(armada_xp_pmsu_cpufreq_init);
diff --git a/drivers/cpufreq/p4-clockmod.c b/drivers/cpufreq/p4-clockmod.c
index fd77812313f3..a25741b1281b 100644
--- a/drivers/cpufreq/p4-clockmod.c
+++ b/drivers/cpufreq/p4-clockmod.c
@@ -168,7 +168,7 @@ static int cpufreq_p4_cpu_init(struct cpufreq_policy *policy)
 #endif
 
 	/* Errata workaround */
-	cpuid = (c->x86 << 8) | (c->x86_model << 4) | c->x86_mask;
+	cpuid = (c->x86 << 8) | (c->x86_model << 4) | c->x86_stepping;
 	switch (cpuid) {
 	case 0x0f07:
 	case 0x0f0a:
diff --git a/drivers/cpufreq/pasemi-cpufreq.c b/drivers/cpufreq/pasemi-cpufreq.c
index b257fc7d5204..75dfbd2a58ea 100644
--- a/drivers/cpufreq/pasemi-cpufreq.c
+++ b/drivers/cpufreq/pasemi-cpufreq.c
@@ -139,7 +139,7 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	struct cpufreq_frequency_table *pos;
 	const u32 *max_freqp;
 	u32 max_freq;
-	int cur_astate;
+	int cur_astate, idx;
 	struct resource res;
 	struct device_node *cpu, *dn;
 	int err = -ENODEV;
@@ -198,9 +198,9 @@ static int pas_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	pr_debug("initializing frequency table\n");
 
 	/* initialize frequency table */
-	cpufreq_for_each_entry(pos, pas_freqs) {
+	cpufreq_for_each_entry_idx(pos, pas_freqs, idx) {
 		pos->frequency = get_astate_freq(pos->driver_data) * 100000;
-		pr_debug("%d: %d\n", (int)(pos - pas_freqs), pos->frequency);
+		pr_debug("%d: %d\n", idx, pos->frequency);
 	}
 
 	cur_astate = get_cur_astate(policy->cpu);
diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c
index 80ac313e6c59..302e9ce793a0 100644
--- a/drivers/cpufreq/powernow-k7.c
+++ b/drivers/cpufreq/powernow-k7.c
@@ -131,7 +131,7 @@ static int check_powernow(void)
 		return 0;
 	}
 
-	if ((c->x86_model == 6) && (c->x86_mask == 0)) {
+	if ((c->x86_model == 6) && (c->x86_stepping == 0)) {
 		pr_info("K7 660[A0] core detected, enabling errata workarounds\n");
 		have_a0 = 1;
 	}
diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c
index b6d7c4c98d0a..29cdec198657 100644
--- a/drivers/cpufreq/powernv-cpufreq.c
+++ b/drivers/cpufreq/powernv-cpufreq.c
@@ -29,6 +29,7 @@
 #include <linux/reboot.h>
 #include <linux/slab.h>
 #include <linux/cpu.h>
+#include <linux/hashtable.h>
 #include <trace/events/power.h>
 
 #include <asm/cputhreads.h>
@@ -38,14 +39,13 @@
 #include <asm/opal.h>
 #include <linux/timer.h>
 
-#define POWERNV_MAX_PSTATES	256
+#define POWERNV_MAX_PSTATES_ORDER  8
+#define POWERNV_MAX_PSTATES	(1UL << (POWERNV_MAX_PSTATES_ORDER))
 #define PMSR_PSAFE_ENABLE	(1UL << 30)
 #define PMSR_SPR_EM_DISABLE	(1UL << 31)
-#define PMSR_MAX(x)		((x >> 32) & 0xFF)
+#define MAX_PSTATE_SHIFT	32
 #define LPSTATE_SHIFT		48
 #define GPSTATE_SHIFT		56
-#define GET_LPSTATE(x)		(((x) >> LPSTATE_SHIFT) & 0xFF)
-#define GET_GPSTATE(x)		(((x) >> GPSTATE_SHIFT) & 0xFF)
 
 #define MAX_RAMP_DOWN_TIME				5120
 /*
@@ -94,6 +94,27 @@ struct global_pstate_info {
 };
 
 static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
+
+DEFINE_HASHTABLE(pstate_revmap, POWERNV_MAX_PSTATES_ORDER);
+/**
+ * struct pstate_idx_revmap_data: Entry in the hashmap pstate_revmap
+ *				  indexed by a function of pstate id.
+ *
+ * @pstate_id: pstate id for this entry.
+ *
+ * @cpufreq_table_idx: Index into the powernv_freqs
+ *		       cpufreq_frequency_table for frequency
+ *		       corresponding to pstate_id.
+ *
+ * @hentry: hlist_node that hooks this entry into the pstate_revmap
+ *	    hashtable
+ */
+struct pstate_idx_revmap_data {
+	u8 pstate_id;
+	unsigned int cpufreq_table_idx;
+	struct hlist_node hentry;
+};
+
 static bool rebooting, throttled, occ_reset;
 
 static const char * const throttle_reason[] = {
@@ -148,39 +169,56 @@ static struct powernv_pstate_info {
 	bool wof_enabled;
 } powernv_pstate_info;
 
-/* Use following macros for conversions between pstate_id and index */
-static inline int idx_to_pstate(unsigned int i)
+static inline u8 extract_pstate(u64 pmsr_val, unsigned int shift)
+{
+	return ((pmsr_val >> shift) & 0xFF);
+}
+
+#define extract_local_pstate(x) extract_pstate(x, LPSTATE_SHIFT)
+#define extract_global_pstate(x) extract_pstate(x, GPSTATE_SHIFT)
+#define extract_max_pstate(x)  extract_pstate(x, MAX_PSTATE_SHIFT)
+
+/* Use following functions for conversions between pstate_id and index */
+
+/**
+ * idx_to_pstate : Returns the pstate id corresponding to the
+ *		   frequency in the cpufreq frequency table
+ *		   powernv_freqs indexed by @i.
+ *
+ *		   If @i is out of bound, this will return the pstate
+ *		   corresponding to the nominal frequency.
+ */
+static inline u8 idx_to_pstate(unsigned int i)
 {
 	if (unlikely(i >= powernv_pstate_info.nr_pstates)) {
-		pr_warn_once("index %u is out of bound\n", i);
+		pr_warn_once("idx_to_pstate: index %u is out of bound\n", i);
 		return powernv_freqs[powernv_pstate_info.nominal].driver_data;
 	}
 
 	return powernv_freqs[i].driver_data;
 }
 
-static inline unsigned int pstate_to_idx(int pstate)
+/**
+ * pstate_to_idx : Returns the index in the cpufreq frequencytable
+ *		   powernv_freqs for the frequency whose corresponding
+ *		   pstate id is @pstate.
+ *
+ *		   If no frequency corresponding to @pstate is found,
+ *		   this will return the index of the nominal
+ *		   frequency.
+ */
+static unsigned int pstate_to_idx(u8 pstate)
 {
-	int min = powernv_freqs[powernv_pstate_info.min].driver_data;
-	int max = powernv_freqs[powernv_pstate_info.max].driver_data;
+	unsigned int key = pstate % POWERNV_MAX_PSTATES;
+	struct pstate_idx_revmap_data *revmap_data;
 
-	if (min > 0) {
-		if (unlikely((pstate < max) || (pstate > min))) {
-			pr_warn_once("pstate %d is out of bound\n", pstate);
-			return powernv_pstate_info.nominal;
-		}
-	} else {
-		if (unlikely((pstate > max) || (pstate < min))) {
-			pr_warn_once("pstate %d is out of bound\n", pstate);
-			return powernv_pstate_info.nominal;
-		}
+	hash_for_each_possible(pstate_revmap, revmap_data, hentry, key) {
+		if (revmap_data->pstate_id == pstate)
+			return revmap_data->cpufreq_table_idx;
 	}
-	/*
-	 * abs() is deliberately used so that is works with
-	 * both monotonically increasing and decreasing
-	 * pstate values
-	 */
-	return abs(pstate - idx_to_pstate(powernv_pstate_info.max));
+
+	pr_warn_once("pstate_to_idx: pstate 0x%x not found\n", pstate);
+	return powernv_pstate_info.nominal;
 }
 
 static inline void reset_gpstates(struct cpufreq_policy *policy)
@@ -247,7 +285,7 @@ static int init_powernv_pstates(void)
 		powernv_pstate_info.wof_enabled = true;
 
 next:
-	pr_info("cpufreq pstate min %d nominal %d max %d\n", pstate_min,
+	pr_info("cpufreq pstate min 0x%x nominal 0x%x max 0x%x\n", pstate_min,
 		pstate_nominal, pstate_max);
 	pr_info("Workload Optimized Frequency is %s in the platform\n",
 		(powernv_pstate_info.wof_enabled) ? "enabled" : "disabled");
@@ -278,19 +316,30 @@ static int init_powernv_pstates(void)
 
 	powernv_pstate_info.nr_pstates = nr_pstates;
 	pr_debug("NR PStates %d\n", nr_pstates);
+
 	for (i = 0; i < nr_pstates; i++) {
 		u32 id = be32_to_cpu(pstate_ids[i]);
 		u32 freq = be32_to_cpu(pstate_freqs[i]);
+		struct pstate_idx_revmap_data *revmap_data;
+		unsigned int key;
 
 		pr_debug("PState id %d freq %d MHz\n", id, freq);
 		powernv_freqs[i].frequency = freq * 1000; /* kHz */
-		powernv_freqs[i].driver_data = id;
+		powernv_freqs[i].driver_data = id & 0xFF;
+
+		revmap_data = (struct pstate_idx_revmap_data *)
+			      kmalloc(sizeof(*revmap_data), GFP_KERNEL);
+
+		revmap_data->pstate_id = id & 0xFF;
+		revmap_data->cpufreq_table_idx = i;
+		key = (revmap_data->pstate_id) % POWERNV_MAX_PSTATES;
+		hash_add(pstate_revmap, &revmap_data->hentry, key);
 
 		if (id == pstate_max)
 			powernv_pstate_info.max = i;
-		else if (id == pstate_nominal)
+		if (id == pstate_nominal)
 			powernv_pstate_info.nominal = i;
-		else if (id == pstate_min)
+		if (id == pstate_min)
 			powernv_pstate_info.min = i;
 
 		if (powernv_pstate_info.wof_enabled && id == pstate_turbo) {
@@ -307,14 +356,13 @@ static int init_powernv_pstates(void)
 }
 
 /* Returns the CPU frequency corresponding to the pstate_id. */
-static unsigned int pstate_id_to_freq(int pstate_id)
+static unsigned int pstate_id_to_freq(u8 pstate_id)
 {
 	int i;
 
 	i = pstate_to_idx(pstate_id);
 	if (i >= powernv_pstate_info.nr_pstates || i < 0) {
-		pr_warn("PState id %d outside of PState table, "
-			"reporting nominal id %d instead\n",
+		pr_warn("PState id 0x%x outside of PState table, reporting nominal id 0x%x instead\n",
 			pstate_id, idx_to_pstate(powernv_pstate_info.nominal));
 		i = powernv_pstate_info.nominal;
 	}
@@ -420,8 +468,8 @@ static inline void set_pmspr(unsigned long sprn, unsigned long val)
  */
 struct powernv_smp_call_data {
 	unsigned int freq;
-	int pstate_id;
-	int gpstate_id;
+	u8 pstate_id;
+	u8 gpstate_id;
 };
 
 /*
@@ -438,22 +486,15 @@ struct powernv_smp_call_data {
 static void powernv_read_cpu_freq(void *arg)
 {
 	unsigned long pmspr_val;
-	s8 local_pstate_id;
 	struct powernv_smp_call_data *freq_data = arg;
 
 	pmspr_val = get_pmspr(SPRN_PMSR);
-
-	/*
-	 * The local pstate id corresponds bits 48..55 in the PMSR.
-	 * Note: Watch out for the sign!
-	 */
-	local_pstate_id = (pmspr_val >> 48) & 0xFF;
-	freq_data->pstate_id = local_pstate_id;
+	freq_data->pstate_id = extract_local_pstate(pmspr_val);
 	freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);
 
-	pr_debug("cpu %d pmsr %016lX pstate_id %d frequency %d kHz\n",
-		raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
-		freq_data->freq);
+	pr_debug("cpu %d pmsr %016lX pstate_id 0x%x frequency %d kHz\n",
+		 raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
+		 freq_data->freq);
 }
 
 /*
@@ -515,21 +556,21 @@ static void powernv_cpufreq_throttle_check(void *data)
 	struct chip *chip;
 	unsigned int cpu = smp_processor_id();
 	unsigned long pmsr;
-	int pmsr_pmax;
+	u8 pmsr_pmax;
 	unsigned int pmsr_pmax_idx;
 
 	pmsr = get_pmspr(SPRN_PMSR);
 	chip = this_cpu_read(chip_info);
 
 	/* Check for Pmax Capping */
-	pmsr_pmax = (s8)PMSR_MAX(pmsr);
+	pmsr_pmax = extract_max_pstate(pmsr);
 	pmsr_pmax_idx = pstate_to_idx(pmsr_pmax);
 	if (pmsr_pmax_idx != powernv_pstate_info.max) {
 		if (chip->throttled)
 			goto next;
 		chip->throttled = true;
 		if (pmsr_pmax_idx > powernv_pstate_info.nominal) {
-			pr_warn_once("CPU %d on Chip %u has Pmax(%d) reduced below nominal frequency(%d)\n",
+			pr_warn_once("CPU %d on Chip %u has Pmax(0x%x) reduced below that of nominal frequency(0x%x)\n",
 				     cpu, chip->id, pmsr_pmax,
 				     idx_to_pstate(powernv_pstate_info.nominal));
 			chip->throttle_sub_turbo++;
@@ -645,8 +686,8 @@ void gpstate_timer_handler(struct timer_list *t)
 	 * value. Hence, read from PMCR to get correct data.
 	 */
 	val = get_pmspr(SPRN_PMCR);
-	freq_data.gpstate_id = (s8)GET_GPSTATE(val);
-	freq_data.pstate_id = (s8)GET_LPSTATE(val);
+	freq_data.gpstate_id = extract_global_pstate(val);
+	freq_data.pstate_id = extract_local_pstate(val);
 	if (freq_data.gpstate_id  == freq_data.pstate_id) {
 		reset_gpstates(policy);
 		spin_unlock(&gpstates->gpstate_lock);
diff --git a/drivers/cpufreq/qoriq-cpufreq.c b/drivers/cpufreq/qoriq-cpufreq.c
index 4ada55b8856e..0562761a3dec 100644
--- a/drivers/cpufreq/qoriq-cpufreq.c
+++ b/drivers/cpufreq/qoriq-cpufreq.c
@@ -275,20 +275,8 @@ static int qoriq_cpufreq_target(struct cpufreq_policy *policy,
 static void qoriq_cpufreq_ready(struct cpufreq_policy *policy)
 {
 	struct cpu_data *cpud = policy->driver_data;
-	struct device_node *np = of_get_cpu_node(policy->cpu, NULL);
 
-	if (of_find_property(np, "#cooling-cells", NULL)) {
-		cpud->cdev = of_cpufreq_cooling_register(np, policy);
-
-		if (IS_ERR(cpud->cdev) && PTR_ERR(cpud->cdev) != -ENOSYS) {
-			pr_err("cpu%d is not running as cooling device: %ld\n",
-					policy->cpu, PTR_ERR(cpud->cdev));
-
-			cpud->cdev = NULL;
-		}
-	}
-
-	of_node_put(np);
+	cpud->cdev = of_cpufreq_cooling_register(policy);
 }
 
 static struct cpufreq_driver qoriq_cpufreq_driver = {
diff --git a/drivers/cpufreq/s3c24xx-cpufreq.c b/drivers/cpufreq/s3c24xx-cpufreq.c
index 7b596fa38ad2..6bebc1f9f55a 100644
--- a/drivers/cpufreq/s3c24xx-cpufreq.c
+++ b/drivers/cpufreq/s3c24xx-cpufreq.c
@@ -351,7 +351,13 @@ struct clk *s3c_cpufreq_clk_get(struct device *dev, const char *name)
 static int s3c_cpufreq_init(struct cpufreq_policy *policy)
 {
 	policy->clk = clk_arm;
-	return cpufreq_generic_init(policy, ftab, cpu_cur.info->latency);
+
+	policy->cpuinfo.transition_latency = cpu_cur.info->latency;
+
+	if (ftab)
+		return cpufreq_table_validate_and_show(policy, ftab);
+
+	return 0;
 }
 
 static int __init s3c_cpufreq_initclks(void)
diff --git a/drivers/cpufreq/scpi-cpufreq.c b/drivers/cpufreq/scpi-cpufreq.c
index 05d299052c5c..d300a163945f 100644
--- a/drivers/cpufreq/scpi-cpufreq.c
+++ b/drivers/cpufreq/scpi-cpufreq.c
@@ -18,27 +18,97 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/clk.h>
 #include <linux/cpu.h>
 #include <linux/cpufreq.h>
+#include <linux/cpumask.h>
+#include <linux/cpu_cooling.h>
+#include <linux/export.h>
 #include <linux/module.h>
-#include <linux/platform_device.h>
+#include <linux/of_platform.h>
 #include <linux/pm_opp.h>
 #include <linux/scpi_protocol.h>
+#include <linux/slab.h>
 #include <linux/types.h>
 
-#include "arm_big_little.h"
+struct scpi_data {
+	struct clk *clk;
+	struct device *cpu_dev;
+	struct thermal_cooling_device *cdev;
+};
 
 static struct scpi_ops *scpi_ops;
 
-static int scpi_get_transition_latency(struct device *cpu_dev)
+static unsigned int scpi_cpufreq_get_rate(unsigned int cpu)
 {
-	return scpi_ops->get_transition_latency(cpu_dev);
+	struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
+	struct scpi_data *priv = policy->driver_data;
+	unsigned long rate = clk_get_rate(priv->clk);
+
+	return rate / 1000;
 }
 
-static int scpi_init_opp_table(const struct cpumask *cpumask)
+static int
+scpi_cpufreq_set_target(struct cpufreq_policy *policy, unsigned int index)
 {
+	unsigned long freq = policy->freq_table[index].frequency;
+	struct scpi_data *priv = policy->driver_data;
+	u64 rate = freq * 1000;
 	int ret;
-	struct device *cpu_dev = get_cpu_device(cpumask_first(cpumask));
+
+	ret = clk_set_rate(priv->clk, rate);
+
+	if (ret)
+		return ret;
+
+	if (clk_get_rate(priv->clk) != rate)
+		return -EIO;
+
+	arch_set_freq_scale(policy->related_cpus, freq,
+			    policy->cpuinfo.max_freq);
+
+	return 0;
+}
+
+static int
+scpi_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask)
+{
+	int cpu, domain, tdomain;
+	struct device *tcpu_dev;
+
+	domain = scpi_ops->device_domain_id(cpu_dev);
+	if (domain < 0)
+		return domain;
+
+	for_each_possible_cpu(cpu) {
+		if (cpu == cpu_dev->id)
+			continue;
+
+		tcpu_dev = get_cpu_device(cpu);
+		if (!tcpu_dev)
+			continue;
+
+		tdomain = scpi_ops->device_domain_id(tcpu_dev);
+		if (tdomain == domain)
+			cpumask_set_cpu(cpu, cpumask);
+	}
+
+	return 0;
+}
+
+static int scpi_cpufreq_init(struct cpufreq_policy *policy)
+{
+	int ret;
+	unsigned int latency;
+	struct device *cpu_dev;
+	struct scpi_data *priv;
+	struct cpufreq_frequency_table *freq_table;
+
+	cpu_dev = get_cpu_device(policy->cpu);
+	if (!cpu_dev) {
+		pr_err("failed to get cpu%d device\n", policy->cpu);
+		return -ENODEV;
+	}
 
 	ret = scpi_ops->add_opps_to_device(cpu_dev);
 	if (ret) {
@@ -46,32 +116,131 @@ static int scpi_init_opp_table(const struct cpumask *cpumask)
 		return ret;
 	}
 
-	ret = dev_pm_opp_set_sharing_cpus(cpu_dev, cpumask);
-	if (ret)
+	ret = scpi_get_sharing_cpus(cpu_dev, policy->cpus);
+	if (ret) {
+		dev_warn(cpu_dev, "failed to get sharing cpumask\n");
+		return ret;
+	}
+
+	ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
+	if (ret) {
 		dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n",
 			__func__, ret);
+		return ret;
+	}
+
+	ret = dev_pm_opp_get_opp_count(cpu_dev);
+	if (ret <= 0) {
+		dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
+		ret = -EPROBE_DEFER;
+		goto out_free_opp;
+	}
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv) {
+		ret = -ENOMEM;
+		goto out_free_opp;
+	}
+
+	ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
+	if (ret) {
+		dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
+		goto out_free_priv;
+	}
+
+	priv->cpu_dev = cpu_dev;
+	priv->clk = clk_get(cpu_dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d\n",
+			__func__, cpu_dev->id);
+		ret = PTR_ERR(priv->clk);
+		goto out_free_cpufreq_table;
+	}
+
+	policy->driver_data = priv;
+
+	ret = cpufreq_table_validate_and_show(policy, freq_table);
+	if (ret) {
+		dev_err(cpu_dev, "%s: invalid frequency table: %d\n", __func__,
+			ret);
+		goto out_put_clk;
+	}
+
+	/* scpi allows DVFS request for any domain from any CPU */
+	policy->dvfs_possible_from_any_cpu = true;
+
+	latency = scpi_ops->get_transition_latency(cpu_dev);
+	if (!latency)
+		latency = CPUFREQ_ETERNAL;
+
+	policy->cpuinfo.transition_latency = latency;
+
+	policy->fast_switch_possible = false;
+	return 0;
+
+out_put_clk:
+	clk_put(priv->clk);
+out_free_cpufreq_table:
+	dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
+out_free_priv:
+	kfree(priv);
+out_free_opp:
+	dev_pm_opp_cpumask_remove_table(policy->cpus);
+
 	return ret;
 }
 
-static const struct cpufreq_arm_bL_ops scpi_cpufreq_ops = {
-	.name	= "scpi",
-	.get_transition_latency = scpi_get_transition_latency,
-	.init_opp_table = scpi_init_opp_table,
-	.free_opp_table = dev_pm_opp_cpumask_remove_table,
+static int scpi_cpufreq_exit(struct cpufreq_policy *policy)
+{
+	struct scpi_data *priv = policy->driver_data;
+
+	cpufreq_cooling_unregister(priv->cdev);
+	clk_put(priv->clk);
+	dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
+	kfree(priv);
+	dev_pm_opp_cpumask_remove_table(policy->related_cpus);
+
+	return 0;
+}
+
+static void scpi_cpufreq_ready(struct cpufreq_policy *policy)
+{
+	struct scpi_data *priv = policy->driver_data;
+
+	priv->cdev = of_cpufreq_cooling_register(policy);
+}
+
+static struct cpufreq_driver scpi_cpufreq_driver = {
+	.name	= "scpi-cpufreq",
+	.flags	= CPUFREQ_STICKY | CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
+		  CPUFREQ_NEED_INITIAL_FREQ_CHECK,
+	.verify	= cpufreq_generic_frequency_table_verify,
+	.attr	= cpufreq_generic_attr,
+	.get	= scpi_cpufreq_get_rate,
+	.init	= scpi_cpufreq_init,
+	.exit	= scpi_cpufreq_exit,
+	.ready	= scpi_cpufreq_ready,
+	.target_index	= scpi_cpufreq_set_target,
 };
 
 static int scpi_cpufreq_probe(struct platform_device *pdev)
 {
+	int ret;
+
 	scpi_ops = get_scpi_ops();
 	if (!scpi_ops)
 		return -EIO;
 
-	return bL_cpufreq_register(&scpi_cpufreq_ops);
+	ret = cpufreq_register_driver(&scpi_cpufreq_driver);
+	if (ret)
+		dev_err(&pdev->dev, "%s: registering cpufreq failed, err: %d\n",
+			__func__, ret);
+	return ret;
 }
 
 static int scpi_cpufreq_remove(struct platform_device *pdev)
 {
-	bL_cpufreq_unregister(&scpi_cpufreq_ops);
+	cpufreq_unregister_driver(&scpi_cpufreq_driver);
 	scpi_ops = NULL;
 	return 0;
 }
diff --git a/drivers/cpufreq/speedstep-centrino.c b/drivers/cpufreq/speedstep-centrino.c
index 41bc5397f4bb..4fa5adf16c70 100644
--- a/drivers/cpufreq/speedstep-centrino.c
+++ b/drivers/cpufreq/speedstep-centrino.c
@@ -37,7 +37,7 @@ struct cpu_id
 {
 	__u8	x86;            /* CPU family */
 	__u8	x86_model;	/* model */
-	__u8	x86_mask;	/* stepping */
+	__u8	x86_stepping;	/* stepping */
 };
 
 enum {
@@ -277,7 +277,7 @@ static int centrino_verify_cpu_id(const struct cpuinfo_x86 *c,
 {
 	if ((c->x86 == x->x86) &&
 	    (c->x86_model == x->x86_model) &&
-	    (c->x86_mask == x->x86_mask))
+	    (c->x86_stepping == x->x86_stepping))
 		return 1;
 	return 0;
 }
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
index 8085ec9000d1..e3a9962ee410 100644
--- a/drivers/cpufreq/speedstep-lib.c
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -272,9 +272,9 @@ unsigned int speedstep_detect_processor(void)
 		ebx = cpuid_ebx(0x00000001);
 		ebx &= 0x000000FF;
 
-		pr_debug("ebx value is %x, x86_mask is %x\n", ebx, c->x86_mask);
+		pr_debug("ebx value is %x, x86_stepping is %x\n", ebx, c->x86_stepping);
 
-		switch (c->x86_mask) {
+		switch (c->x86_stepping) {
 		case 4:
 			/*
 			 * B-stepping [M-P4-M]
@@ -361,7 +361,7 @@ unsigned int speedstep_detect_processor(void)
 				msr_lo, msr_hi);
 		if ((msr_hi & (1<<18)) &&
 		    (relaxed_check ? 1 : (msr_hi & (3<<24)))) {
-			if (c->x86_mask == 0x01) {
+			if (c->x86_stepping == 0x01) {
 				pr_debug("early PIII version\n");
 				return SPEEDSTEP_CPU_PIII_C_EARLY;
 			} else
diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c
index 923317f03b4b..a099b7bf74cd 100644
--- a/drivers/cpufreq/ti-cpufreq.c
+++ b/drivers/cpufreq/ti-cpufreq.c
@@ -17,6 +17,7 @@
 #include <linux/cpu.h>
 #include <linux/io.h>
 #include <linux/mfd/syscon.h>
+#include <linux/module.h>
 #include <linux/init.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
@@ -50,6 +51,7 @@ struct ti_cpufreq_soc_data {
 	unsigned long efuse_mask;
 	unsigned long efuse_shift;
 	unsigned long rev_offset;
+	bool multi_regulator;
 };
 
 struct ti_cpufreq_data {
@@ -57,6 +59,7 @@ struct ti_cpufreq_data {
 	struct device_node *opp_node;
 	struct regmap *syscon;
 	const struct ti_cpufreq_soc_data *soc_data;
+	struct opp_table *opp_table;
 };
 
 static unsigned long amx3_efuse_xlate(struct ti_cpufreq_data *opp_data,
@@ -95,6 +98,7 @@ static struct ti_cpufreq_soc_data am3x_soc_data = {
 	.efuse_offset = 0x07fc,
 	.efuse_mask = 0x1fff,
 	.rev_offset = 0x600,
+	.multi_regulator = false,
 };
 
 static struct ti_cpufreq_soc_data am4x_soc_data = {
@@ -103,6 +107,7 @@ static struct ti_cpufreq_soc_data am4x_soc_data = {
 	.efuse_offset = 0x0610,
 	.efuse_mask = 0x3f,
 	.rev_offset = 0x600,
+	.multi_regulator = false,
 };
 
 static struct ti_cpufreq_soc_data dra7_soc_data = {
@@ -111,6 +116,7 @@ static struct ti_cpufreq_soc_data dra7_soc_data = {
 	.efuse_mask = 0xf80000,
 	.efuse_shift = 19,
 	.rev_offset = 0x204,
+	.multi_regulator = true,
 };
 
 /**
@@ -195,12 +201,14 @@ static const struct of_device_id ti_cpufreq_of_match[] = {
 	{},
 };
 
-static int ti_cpufreq_init(void)
+static int ti_cpufreq_probe(struct platform_device *pdev)
 {
 	u32 version[VERSION_COUNT];
 	struct device_node *np;
 	const struct of_device_id *match;
+	struct opp_table *ti_opp_table;
 	struct ti_cpufreq_data *opp_data;
+	const char * const reg_names[] = {"vdd", "vbb"};
 	int ret;
 
 	np = of_find_node_by_path("/");
@@ -247,16 +255,29 @@ static int ti_cpufreq_init(void)
 	if (ret)
 		goto fail_put_node;
 
-	ret = PTR_ERR_OR_ZERO(dev_pm_opp_set_supported_hw(opp_data->cpu_dev,
-							  version, VERSION_COUNT));
-	if (ret) {
+	ti_opp_table = dev_pm_opp_set_supported_hw(opp_data->cpu_dev,
+						   version, VERSION_COUNT);
+	if (IS_ERR(ti_opp_table)) {
 		dev_err(opp_data->cpu_dev,
 			"Failed to set supported hardware\n");
+		ret = PTR_ERR(ti_opp_table);
 		goto fail_put_node;
 	}
 
-	of_node_put(opp_data->opp_node);
+	opp_data->opp_table = ti_opp_table;
+
+	if (opp_data->soc_data->multi_regulator) {
+		ti_opp_table = dev_pm_opp_set_regulators(opp_data->cpu_dev,
+							 reg_names,
+							 ARRAY_SIZE(reg_names));
+		if (IS_ERR(ti_opp_table)) {
+			dev_pm_opp_put_supported_hw(opp_data->opp_table);
+			ret =  PTR_ERR(ti_opp_table);
+			goto fail_put_node;
+		}
+	}
 
+	of_node_put(opp_data->opp_node);
 register_cpufreq_dt:
 	platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
 
@@ -269,4 +290,22 @@ static int ti_cpufreq_init(void)
 
 	return ret;
 }
-device_initcall(ti_cpufreq_init);
+
+static int ti_cpufreq_init(void)
+{
+	platform_device_register_simple("ti-cpufreq", -1, NULL, 0);
+	return 0;
+}
+module_init(ti_cpufreq_init);
+
+static struct platform_driver ti_cpufreq_driver = {
+	.probe = ti_cpufreq_probe,
+	.driver = {
+		.name = "ti-cpufreq",
+	},
+};
+module_platform_driver(ti_cpufreq_driver);
+
+MODULE_DESCRIPTION("TI CPUFreq/OPP hw-supported driver");
+MODULE_AUTHOR("Dave Gerlach <d-gerlach@ti.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c
index e06605b21841..1a8234e706bc 100644
--- a/drivers/cpuidle/cpuidle-powernv.c
+++ b/drivers/cpuidle/cpuidle-powernv.c
@@ -76,6 +76,8 @@ static int snooze_loop(struct cpuidle_device *dev,
 	ppc64_runlatch_on();
 	clear_thread_flag(TIF_POLLING_NRFLAG);
 
+	local_irq_disable();
+
 	return index;
 }
 
diff --git a/drivers/cpuidle/cpuidle-pseries.c b/drivers/cpuidle/cpuidle-pseries.c
index a187a39fb866..9e56bc411061 100644
--- a/drivers/cpuidle/cpuidle-pseries.c
+++ b/drivers/cpuidle/cpuidle-pseries.c
@@ -51,8 +51,6 @@ static inline void idle_loop_epilog(unsigned long in_purr)
 	get_lppaca()->wait_state_cycles = cpu_to_be64(wait_cycles);
 	get_lppaca()->idle = 0;
 
-	if (irqs_disabled())
-		local_irq_enable();
 	ppc64_runlatch_on();
 }
 
@@ -87,6 +85,8 @@ static int snooze_loop(struct cpuidle_device *dev,
 	HMT_medium();
 	clear_thread_flag(TIF_POLLING_NRFLAG);
 
+	local_irq_disable();
+
 	idle_loop_epilog(in_purr);
 
 	return index;
@@ -121,6 +121,7 @@ static int dedicated_cede_loop(struct cpuidle_device *dev,
 	HMT_medium();
 	check_and_cede_processor();
 
+	local_irq_disable();
 	get_lppaca()->donate_dedicated_cpu = 0;
 
 	idle_loop_epilog(in_purr);
@@ -145,6 +146,7 @@ static int shared_cede_loop(struct cpuidle_device *dev,
 	 */
 	check_and_cede_processor();
 
+	local_irq_disable();
 	idle_loop_epilog(in_purr);
 
 	return index;
@@ -172,11 +174,17 @@ static struct cpuidle_state dedicated_states[] = {
  * States for shared partition case.
  */
 static struct cpuidle_state shared_states[] = {
+	{ /* Snooze */
+		.name = "snooze",
+		.desc = "snooze",
+		.exit_latency = 0,
+		.target_residency = 0,
+		.enter = &snooze_loop },
 	{ /* Shared Cede */
 		.name = "Shared Cede",
 		.desc = "Shared Cede",
-		.exit_latency = 0,
-		.target_residency = 0,
+		.exit_latency = 10,
+		.target_residency = 100,
 		.enter = &shared_cede_loop },
 };
 
diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
index 4e78263e34a4..5d359aff3cc5 100644
--- a/drivers/cpuidle/governor.c
+++ b/drivers/cpuidle/governor.c
@@ -36,14 +36,15 @@ static struct cpuidle_governor * __cpuidle_find_governor(const char *str)
 /**
  * cpuidle_switch_governor - changes the governor
  * @gov: the new target governor
- *
- * NOTE: "gov" can be NULL to specify disabled
  * Must be called with cpuidle_lock acquired.
  */
 int cpuidle_switch_governor(struct cpuidle_governor *gov)
 {
 	struct cpuidle_device *dev;
 
+	if (!gov)
+		return -EINVAL;
+
 	if (gov == cpuidle_curr_governor)
 		return 0;
 
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 47ec920d5b71..4b741b83e23f 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -723,7 +723,6 @@ config CRYPTO_DEV_ARTPEC6
 	select CRYPTO_HASH
 	select CRYPTO_SHA1
 	select CRYPTO_SHA256
-	select CRYPTO_SHA384
 	select CRYPTO_SHA512
 	help
 	  Enables the driver for the on-chip crypto accelerator
diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c
index eeaf27859d80..ea83d0bff0e9 100644
--- a/drivers/crypto/amcc/crypto4xx_alg.c
+++ b/drivers/crypto/amcc/crypto4xx_alg.c
@@ -256,10 +256,6 @@ static inline bool crypto4xx_aead_need_fallback(struct aead_request *req,
 	if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3))
 		return true;
 
-	/* CCM - fix CBC MAC mismatch in special case */
-	if (is_ccm && decrypt && !req->assoclen)
-		return true;
-
 	return false;
 }
 
@@ -330,7 +326,7 @@ int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key,
 	sa = (struct dynamic_sa_ctl *) ctx->sa_in;
 	sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2);
 
-	set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV,
+	set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
 				 SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
 				 SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC,
 				 SA_CIPHER_ALG_AES,
diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c
index c44954e274bc..76f459ad2821 100644
--- a/drivers/crypto/amcc/crypto4xx_core.c
+++ b/drivers/crypto/amcc/crypto4xx_core.c
@@ -128,7 +128,14 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev)
 	writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT);
 	writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT);
 	writel(PPC4XX_INT_CFG, dev->ce_base + CRYPTO4XX_INT_CFG);
-	writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN);
+	if (dev->is_revb) {
+		writel(PPC4XX_INT_TIMEOUT_CNT_REVB << 10,
+		       dev->ce_base + CRYPTO4XX_INT_TIMEOUT_CNT);
+		writel(PPC4XX_PD_DONE_INT | PPC4XX_TMO_ERR_INT,
+		       dev->ce_base + CRYPTO4XX_INT_EN);
+	} else {
+		writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN);
+	}
 }
 
 int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size)
@@ -275,14 +282,12 @@ static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx)
  */
 static u32 crypto4xx_build_gdr(struct crypto4xx_device *dev)
 {
-	dev->gdr = dma_alloc_coherent(dev->core_dev->device,
-				      sizeof(struct ce_gd) * PPC4XX_NUM_GD,
-				      &dev->gdr_pa, GFP_ATOMIC);
+	dev->gdr = dma_zalloc_coherent(dev->core_dev->device,
+				       sizeof(struct ce_gd) * PPC4XX_NUM_GD,
+				       &dev->gdr_pa, GFP_ATOMIC);
 	if (!dev->gdr)
 		return -ENOMEM;
 
-	memset(dev->gdr, 0, sizeof(struct ce_gd) * PPC4XX_NUM_GD);
-
 	return 0;
 }
 
@@ -570,15 +575,14 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev,
 				struct pd_uinfo *pd_uinfo,
 				struct ce_pd *pd)
 {
-	struct aead_request *aead_req;
-	struct crypto4xx_ctx *ctx;
+	struct aead_request *aead_req = container_of(pd_uinfo->async_req,
+		struct aead_request, base);
 	struct scatterlist *dst = pd_uinfo->dest_va;
+	size_t cp_len = crypto_aead_authsize(
+		crypto_aead_reqtfm(aead_req));
+	u32 icv[cp_len];
 	int err = 0;
 
-	aead_req = container_of(pd_uinfo->async_req, struct aead_request,
-				base);
-	ctx  = crypto_tfm_ctx(aead_req->base.tfm);
-
 	if (pd_uinfo->using_sd) {
 		crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo,
 					  pd->pd_ctl_len.bf.pkt_len,
@@ -590,38 +594,39 @@ static void crypto4xx_aead_done(struct crypto4xx_device *dev,
 
 	if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) {
 		/* append icv at the end */
-		size_t cp_len = crypto_aead_authsize(
-			crypto_aead_reqtfm(aead_req));
-		u32 icv[cp_len];
-
 		crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest,
 					   cp_len);
 
 		scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen,
 					 cp_len, 1);
+	} else {
+		/* check icv at the end */
+		scatterwalk_map_and_copy(icv, aead_req->src,
+			aead_req->assoclen + aead_req->cryptlen -
+			cp_len, cp_len, 0);
+
+		crypto4xx_memcpy_from_le32(icv, icv, cp_len);
+
+		if (crypto_memneq(icv, pd_uinfo->sr_va->save_digest, cp_len))
+			err = -EBADMSG;
 	}
 
 	crypto4xx_ret_sg_desc(dev, pd_uinfo);
 
 	if (pd->pd_ctl.bf.status & 0xff) {
-		if (pd->pd_ctl.bf.status & 0x1) {
-			/* authentication error */
-			err = -EBADMSG;
-		} else {
-			if (!__ratelimit(&dev->aead_ratelimit)) {
-				if (pd->pd_ctl.bf.status & 2)
-					pr_err("pad fail error\n");
-				if (pd->pd_ctl.bf.status & 4)
-					pr_err("seqnum fail\n");
-				if (pd->pd_ctl.bf.status & 8)
-					pr_err("error _notify\n");
-				pr_err("aead return err status = 0x%02x\n",
-					pd->pd_ctl.bf.status & 0xff);
-				pr_err("pd pad_ctl = 0x%08x\n",
-					pd->pd_ctl.bf.pd_pad_ctl);
-			}
-			err = -EINVAL;
+		if (!__ratelimit(&dev->aead_ratelimit)) {
+			if (pd->pd_ctl.bf.status & 2)
+				pr_err("pad fail error\n");
+			if (pd->pd_ctl.bf.status & 4)
+				pr_err("seqnum fail\n");
+			if (pd->pd_ctl.bf.status & 8)
+				pr_err("error _notify\n");
+			pr_err("aead return err status = 0x%02x\n",
+				pd->pd_ctl.bf.status & 0xff);
+			pr_err("pd pad_ctl = 0x%08x\n",
+				pd->pd_ctl.bf.pd_pad_ctl);
 		}
+		err = -EINVAL;
 	}
 
 	if (pd_uinfo->state & PD_ENTRY_BUSY)
@@ -1070,21 +1075,29 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data)
 /**
  * Top Half of isr.
  */
-static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data)
+static inline irqreturn_t crypto4xx_interrupt_handler(int irq, void *data,
+						      u32 clr_val)
 {
 	struct device *dev = (struct device *)data;
 	struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
 
-	if (!core_dev->dev->ce_base)
-		return 0;
-
-	writel(PPC4XX_INTERRUPT_CLR,
-	       core_dev->dev->ce_base + CRYPTO4XX_INT_CLR);
+	writel(clr_val, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR);
 	tasklet_schedule(&core_dev->tasklet);
 
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data)
+{
+	return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR);
+}
+
+static irqreturn_t crypto4xx_ce_interrupt_handler_revb(int irq, void *data)
+{
+	return crypto4xx_interrupt_handler(irq, data, PPC4XX_INTERRUPT_CLR |
+		PPC4XX_TMO_ERR_INT);
+}
+
 /**
  * Supported Crypto Algorithms
  */
@@ -1266,6 +1279,8 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	struct resource res;
 	struct device *dev = &ofdev->dev;
 	struct crypto4xx_core_device *core_dev;
+	u32 pvr;
+	bool is_revb = true;
 
 	rc = of_address_to_resource(ofdev->dev.of_node, 0, &res);
 	if (rc)
@@ -1282,6 +1297,7 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 		       mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET);
 		mtdcri(SDR0, PPC405EX_SDR0_SRST,
 		       mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET);
+		is_revb = false;
 	} else if (of_find_compatible_node(NULL, NULL,
 			"amcc,ppc460sx-crypto")) {
 		mtdcri(SDR0, PPC460SX_SDR0_SRST,
@@ -1304,7 +1320,22 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	if (!core_dev->dev)
 		goto err_alloc_dev;
 
+	/*
+	 * Older version of 460EX/GT have a hardware bug.
+	 * Hence they do not support H/W based security intr coalescing
+	 */
+	pvr = mfspr(SPRN_PVR);
+	if (is_revb && ((pvr >> 4) == 0x130218A)) {
+		u32 min = PVR_MIN(pvr);
+
+		if (min < 4) {
+			dev_info(dev, "RevA detected - disable interrupt coalescing\n");
+			is_revb = false;
+		}
+	}
+
 	core_dev->dev->core_dev = core_dev;
+	core_dev->dev->is_revb = is_revb;
 	core_dev->device = dev;
 	spin_lock_init(&core_dev->lock);
 	INIT_LIST_HEAD(&core_dev->dev->alg_list);
@@ -1325,13 +1356,6 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb,
 		     (unsigned long) dev);
 
-	/* Register for Crypto isr, Crypto Engine IRQ */
-	core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0);
-	rc = request_irq(core_dev->irq, crypto4xx_ce_interrupt_handler, 0,
-			 core_dev->dev->name, dev);
-	if (rc)
-		goto err_request_irq;
-
 	core_dev->dev->ce_base = of_iomap(ofdev->dev.of_node, 0);
 	if (!core_dev->dev->ce_base) {
 		dev_err(dev, "failed to of_iomap\n");
@@ -1339,6 +1363,15 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 		goto err_iomap;
 	}
 
+	/* Register for Crypto isr, Crypto Engine IRQ */
+	core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0);
+	rc = request_irq(core_dev->irq, is_revb ?
+			 crypto4xx_ce_interrupt_handler_revb :
+			 crypto4xx_ce_interrupt_handler, 0,
+			 KBUILD_MODNAME, dev);
+	if (rc)
+		goto err_request_irq;
+
 	/* need to setup pdr, rdr, gdr and sdr before this */
 	crypto4xx_hw_init(core_dev->dev);
 
@@ -1352,11 +1385,11 @@ static int crypto4xx_probe(struct platform_device *ofdev)
 	return 0;
 
 err_start_dev:
-	iounmap(core_dev->dev->ce_base);
-err_iomap:
 	free_irq(core_dev->irq, dev);
 err_request_irq:
 	irq_dispose_mapping(core_dev->irq);
+	iounmap(core_dev->dev->ce_base);
+err_iomap:
 	tasklet_kill(&core_dev->tasklet);
 err_build_sdr:
 	crypto4xx_destroy_sdr(core_dev->dev);
@@ -1397,7 +1430,7 @@ MODULE_DEVICE_TABLE(of, crypto4xx_match);
 
 static struct platform_driver crypto4xx_driver = {
 	.driver = {
-		.name = MODULE_NAME,
+		.name = KBUILD_MODNAME,
 		.of_match_table = crypto4xx_match,
 	},
 	.probe		= crypto4xx_probe,
diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h
index 8ac3bd37203b..23b726da6534 100644
--- a/drivers/crypto/amcc/crypto4xx_core.h
+++ b/drivers/crypto/amcc/crypto4xx_core.h
@@ -28,8 +28,6 @@
 #include "crypto4xx_reg_def.h"
 #include "crypto4xx_sa.h"
 
-#define MODULE_NAME "crypto4xx"
-
 #define PPC460SX_SDR0_SRST                      0x201
 #define PPC405EX_SDR0_SRST                      0x200
 #define PPC460EX_SDR0_SRST                      0x201
@@ -82,7 +80,6 @@ struct pd_uinfo {
 
 struct crypto4xx_device {
 	struct crypto4xx_core_device *core_dev;
-	char *name;
 	void __iomem *ce_base;
 	void __iomem *trng_base;
 
@@ -109,6 +106,7 @@ struct crypto4xx_device {
 	struct list_head alg_list;	/* List of algorithm supported
 					by this device */
 	struct ratelimit_state aead_ratelimit;
+	bool is_revb;
 };
 
 struct crypto4xx_core_device {
diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h
index 0a22ec5d1a96..472331787e04 100644
--- a/drivers/crypto/amcc/crypto4xx_reg_def.h
+++ b/drivers/crypto/amcc/crypto4xx_reg_def.h
@@ -121,13 +121,15 @@
 #define PPC4XX_PD_SIZE				6
 #define PPC4XX_CTX_DONE_INT			0x2000
 #define PPC4XX_PD_DONE_INT			0x8000
+#define PPC4XX_TMO_ERR_INT			0x40000
 #define PPC4XX_BYTE_ORDER			0x22222
 #define PPC4XX_INTERRUPT_CLR			0x3ffff
 #define PPC4XX_PRNG_CTRL_AUTO_EN		0x3
 #define PPC4XX_DC_3DES_EN			1
 #define PPC4XX_TRNG_EN				0x00020000
-#define PPC4XX_INT_DESCR_CNT			4
+#define PPC4XX_INT_DESCR_CNT			7
 #define PPC4XX_INT_TIMEOUT_CNT			0
+#define PPC4XX_INT_TIMEOUT_CNT_REVB		0x3FF
 #define PPC4XX_INT_CFG				1
 /**
  * all follow define are ad hoc
diff --git a/drivers/crypto/amcc/crypto4xx_trng.c b/drivers/crypto/amcc/crypto4xx_trng.c
index 677ca17fd223..5e63742b0d22 100644
--- a/drivers/crypto/amcc/crypto4xx_trng.c
+++ b/drivers/crypto/amcc/crypto4xx_trng.c
@@ -92,7 +92,7 @@ void ppc4xx_trng_probe(struct crypto4xx_core_device *core_dev)
 	if (!rng)
 		goto err_out;
 
-	rng->name = MODULE_NAME;
+	rng->name = KBUILD_MODNAME;
 	rng->data_present = ppc4xx_trng_data_present;
 	rng->data_read = ppc4xx_trng_data_read;
 	rng->priv = (unsigned long) dev;
diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c
index 456278440863..0fb8bbf41a8d 100644
--- a/drivers/crypto/axis/artpec6_crypto.c
+++ b/drivers/crypto/axis/artpec6_crypto.c
@@ -22,6 +22,7 @@
 #include <linux/slab.h>
 
 #include <crypto/aes.h>
+#include <crypto/gcm.h>
 #include <crypto/internal/aead.h>
 #include <crypto/internal/hash.h>
 #include <crypto/internal/skcipher.h>
@@ -1934,7 +1935,7 @@ static int artpec6_crypto_prepare_aead(struct aead_request *areq)
 
 	memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher));
 	// The HW omits the initial increment of the counter field.
-	crypto_inc(req_ctx->hw_ctx.J0+12, 4);
+	memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4);
 
 	ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx,
 		sizeof(struct artpec6_crypto_aead_hw_ctx), false, false);
@@ -2956,7 +2957,7 @@ static struct aead_alg aead_algos[] = {
 		.setkey = artpec6_crypto_aead_set_key,
 		.encrypt = artpec6_crypto_aead_encrypt,
 		.decrypt = artpec6_crypto_aead_decrypt,
-		.ivsize = AES_BLOCK_SIZE,
+		.ivsize = GCM_AES_IV_SIZE,
 		.maxauthsize = AES_BLOCK_SIZE,
 
 		.base = {
@@ -3041,9 +3042,6 @@ static int artpec6_crypto_probe(struct platform_device *pdev)
 	variant = (enum artpec6_crypto_variant)match->data;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
 	base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c
index ce70b44d0fb6..2b75f95bbe1b 100644
--- a/drivers/crypto/bcm/cipher.c
+++ b/drivers/crypto/bcm/cipher.c
@@ -42,7 +42,6 @@
 #include <crypto/authenc.h>
 #include <crypto/skcipher.h>
 #include <crypto/hash.h>
-#include <crypto/aes.h>
 #include <crypto/sha3.h>
 
 #include "util.h"
diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c
index a118b9bed669..bfbf8bf77f03 100644
--- a/drivers/crypto/bfin_crc.c
+++ b/drivers/crypto/bfin_crc.c
@@ -494,7 +494,8 @@ static struct ahash_alg algs = {
 		.cra_driver_name	= DRIVER_NAME,
 		.cra_priority		= 100,
 		.cra_flags		= CRYPTO_ALG_TYPE_AHASH |
-						CRYPTO_ALG_ASYNC,
+						CRYPTO_ALG_ASYNC |
+						CRYPTO_ALG_OPTIONAL_KEY,
 		.cra_blocksize		= CHKSUM_BLOCK_SIZE,
 		.cra_ctxsize		= sizeof(struct bfin_crypto_crc_ctx),
 		.cra_alignmask		= 3,
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index baa8dd52472d..2188235be02d 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -108,6 +108,7 @@ struct caam_ctx {
 	dma_addr_t sh_desc_dec_dma;
 	dma_addr_t sh_desc_givenc_dma;
 	dma_addr_t key_dma;
+	enum dma_data_direction dir;
 	struct device *jrdev;
 	struct alginfo adata;
 	struct alginfo cdata;
@@ -118,6 +119,7 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
 {
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 	u32 *desc;
 	int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
 			ctx->adata.keylen_pad;
@@ -136,9 +138,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
 
 	/* aead_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
-	cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize);
+	cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
+				    ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	/*
 	 * Job Descriptor and Shared Descriptors
@@ -154,9 +157,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead)
 
 	/* aead_decrypt shared descriptor */
 	desc = ctx->sh_desc_dec;
-	cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize);
+	cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
+				    ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	return 0;
 }
@@ -168,6 +172,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	unsigned int ivsize = crypto_aead_ivsize(aead);
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 	u32 ctx1_iv_off = 0;
 	u32 *desc, *nonce = NULL;
 	u32 inl_mask;
@@ -234,9 +239,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_enc;
 	cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
 			       ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
-			       false);
+			       false, ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 skip_enc:
 	/*
@@ -266,9 +271,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_dec;
 	cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
 			       ctx->authsize, alg->caam.geniv, is_rfc3686,
-			       nonce, ctx1_iv_off, false);
+			       nonce, ctx1_iv_off, false, ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	if (!alg->caam.geniv)
 		goto skip_givenc;
@@ -300,9 +305,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_enc;
 	cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
 				  ctx->authsize, is_rfc3686, nonce,
-				  ctx1_iv_off, false);
+				  ctx1_iv_off, false, ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 skip_givenc:
 	return 0;
@@ -346,7 +351,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_enc;
 	cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	/*
 	 * Job Descriptor and Shared Descriptors
@@ -363,7 +368,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_dec;
 	cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	return 0;
 }
@@ -405,7 +410,7 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_enc;
 	cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	/*
 	 * Job Descriptor and Shared Descriptors
@@ -422,7 +427,7 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_dec;
 	cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	return 0;
 }
@@ -465,7 +470,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_enc;
 	cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	/*
 	 * Job Descriptor and Shared Descriptors
@@ -482,7 +487,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead)
 	desc = ctx->sh_desc_dec;
 	cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	return 0;
 }
@@ -503,6 +508,7 @@ static int aead_setkey(struct crypto_aead *aead,
 {
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 	struct crypto_authenc_keys keys;
 	int ret = 0;
 
@@ -517,6 +523,27 @@ static int aead_setkey(struct crypto_aead *aead,
 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 #endif
 
+	/*
+	 * If DKP is supported, use it in the shared descriptor to generate
+	 * the split key.
+	 */
+	if (ctrlpriv->era >= 6) {
+		ctx->adata.keylen = keys.authkeylen;
+		ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
+						      OP_ALG_ALGSEL_MASK);
+
+		if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
+			goto badkey;
+
+		memcpy(ctx->key, keys.authkey, keys.authkeylen);
+		memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
+		       keys.enckeylen);
+		dma_sync_single_for_device(jrdev, ctx->key_dma,
+					   ctx->adata.keylen_pad +
+					   keys.enckeylen, ctx->dir);
+		goto skip_split_key;
+	}
+
 	ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
 			    keys.authkeylen, CAAM_MAX_KEY_SIZE -
 			    keys.enckeylen);
@@ -527,12 +554,14 @@ static int aead_setkey(struct crypto_aead *aead,
 	/* postpend encryption key to auth split key */
 	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
-				   keys.enckeylen, DMA_TO_DEVICE);
+				   keys.enckeylen, ctx->dir);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 		       ctx->adata.keylen_pad + keys.enckeylen, 1);
 #endif
+
+skip_split_key:
 	ctx->cdata.keylen = keys.enckeylen;
 	return aead_set_sh_desc(aead);
 badkey:
@@ -552,7 +581,7 @@ static int gcm_setkey(struct crypto_aead *aead,
 #endif
 
 	memcpy(ctx->key, key, keylen);
-	dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
+	dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
 	ctx->cdata.keylen = keylen;
 
 	return gcm_set_sh_desc(aead);
@@ -580,7 +609,7 @@ static int rfc4106_setkey(struct crypto_aead *aead,
 	 */
 	ctx->cdata.keylen = keylen - 4;
 	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
-				   DMA_TO_DEVICE);
+				   ctx->dir);
 	return rfc4106_set_sh_desc(aead);
 }
 
@@ -606,7 +635,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
 	 */
 	ctx->cdata.keylen = keylen - 4;
 	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
-				   DMA_TO_DEVICE);
+				   ctx->dir);
 	return rfc4543_set_sh_desc(aead);
 }
 
@@ -625,7 +654,6 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 	const bool is_rfc3686 = (ctr_mode &&
 				 (strstr(alg_name, "rfc3686") != NULL));
 
-	memcpy(ctx->key, key, keylen);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@@ -648,9 +676,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		keylen -= CTR_RFC3686_NONCE_SIZE;
 	}
 
-	dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
 	ctx->cdata.keylen = keylen;
-	ctx->cdata.key_virt = ctx->key;
+	ctx->cdata.key_virt = key;
 	ctx->cdata.key_inline = true;
 
 	/* ablkcipher_encrypt shared descriptor */
@@ -658,21 +685,21 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 	cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
 				     ctx1_iv_off);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	/* ablkcipher_decrypt shared descriptor */
 	desc = ctx->sh_desc_dec;
 	cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
 				     ctx1_iv_off);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	/* ablkcipher_givencrypt shared descriptor */
 	desc = ctx->sh_desc_givenc;
 	cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686,
 					ctx1_iv_off);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	return 0;
 }
@@ -691,23 +718,21 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		return -EINVAL;
 	}
 
-	memcpy(ctx->key, key, keylen);
-	dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
 	ctx->cdata.keylen = keylen;
-	ctx->cdata.key_virt = ctx->key;
+	ctx->cdata.key_virt = key;
 	ctx->cdata.key_inline = true;
 
 	/* xts_ablkcipher_encrypt shared descriptor */
 	desc = ctx->sh_desc_enc;
 	cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	/* xts_ablkcipher_decrypt shared descriptor */
 	desc = ctx->sh_desc_dec;
 	cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 
 	return 0;
 }
@@ -979,9 +1004,6 @@ static void init_aead_job(struct aead_request *req,
 		append_seq_out_ptr(desc, dst_dma,
 				   req->assoclen + req->cryptlen - authsize,
 				   out_options);
-
-	/* REG3 = assoclen */
-	append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
 }
 
 static void init_gcm_job(struct aead_request *req,
@@ -996,6 +1018,7 @@ static void init_gcm_job(struct aead_request *req,
 	unsigned int last;
 
 	init_aead_job(req, edesc, all_contig, encrypt);
+	append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
 
 	/* BUG This should not be specific to generic GCM. */
 	last = 0;
@@ -1022,6 +1045,7 @@ static void init_authenc_job(struct aead_request *req,
 						 struct caam_aead_alg, aead);
 	unsigned int ivsize = crypto_aead_ivsize(aead);
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
 	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 			       OP_ALG_AAI_CTR_MOD128);
 	const bool is_rfc3686 = alg->caam.rfc3686;
@@ -1045,6 +1069,15 @@ static void init_authenc_job(struct aead_request *req,
 
 	init_aead_job(req, edesc, all_contig, encrypt);
 
+	/*
+	 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
+	 * having DPOVRD as destination.
+	 */
+	if (ctrlpriv->era < 3)
+		append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
+	else
+		append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
+
 	if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
 		append_load_as_imm(desc, req->iv, ivsize,
 				   LDST_CLASS_1_CCB |
@@ -3228,9 +3261,11 @@ struct caam_crypto_alg {
 	struct caam_alg_entry caam;
 };
 
-static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
+static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
+			    bool uses_dkp)
 {
 	dma_addr_t dma_addr;
+	struct caam_drv_private *priv;
 
 	ctx->jrdev = caam_jr_alloc();
 	if (IS_ERR(ctx->jrdev)) {
@@ -3238,10 +3273,16 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
 		return PTR_ERR(ctx->jrdev);
 	}
 
+	priv = dev_get_drvdata(ctx->jrdev->parent);
+	if (priv->era >= 6 && uses_dkp)
+		ctx->dir = DMA_BIDIRECTIONAL;
+	else
+		ctx->dir = DMA_TO_DEVICE;
+
 	dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
 					offsetof(struct caam_ctx,
 						 sh_desc_enc_dma),
-					DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
+					ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
 	if (dma_mapping_error(ctx->jrdev, dma_addr)) {
 		dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
 		caam_jr_free(ctx->jrdev);
@@ -3269,7 +3310,7 @@ static int caam_cra_init(struct crypto_tfm *tfm)
 		 container_of(alg, struct caam_crypto_alg, crypto_alg);
 	struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	return caam_init_common(ctx, &caam_alg->caam);
+	return caam_init_common(ctx, &caam_alg->caam, false);
 }
 
 static int caam_aead_init(struct crypto_aead *tfm)
@@ -3279,14 +3320,15 @@ static int caam_aead_init(struct crypto_aead *tfm)
 		 container_of(alg, struct caam_aead_alg, aead);
 	struct caam_ctx *ctx = crypto_aead_ctx(tfm);
 
-	return caam_init_common(ctx, &caam_alg->caam);
+	return caam_init_common(ctx, &caam_alg->caam,
+				alg->setkey == aead_setkey);
 }
 
 static void caam_exit_common(struct caam_ctx *ctx)
 {
 	dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
 			       offsetof(struct caam_ctx, sh_desc_enc_dma),
-			       DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
+			       ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
 	caam_jr_free(ctx->jrdev);
 }
 
diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c
index 530c14ee32de..ceb93fbb76e6 100644
--- a/drivers/crypto/caam/caamalg_desc.c
+++ b/drivers/crypto/caam/caamalg_desc.c
@@ -45,16 +45,16 @@ static inline void append_dec_op1(u32 *desc, u32 type)
  * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
  *                               (non-protocol) with no (null) encryption.
  * @desc: pointer to buffer used for descriptor construction
- * @adata: pointer to authentication transform definitions. Note that since a
- *         split key is to be used, the size of the split key itself is
- *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
- *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @adata: pointer to authentication transform definitions.
+ *         A split key is required for SEC Era < 6; the size of the split key
+ *         is specified in this case. Valid algorithm values - one of
+ *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
+ *         with OP_ALG_AAI_HMAC_PRECOMP.
  * @icvsize: integrity check value (ICV) size (truncated or full)
- *
- * Note: Requires an MDHA split key.
+ * @era: SEC Era
  */
 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
-				 unsigned int icvsize)
+				 unsigned int icvsize, int era)
 {
 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
 
@@ -63,13 +63,18 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
 	/* Skip if already shared */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 				   JUMP_COND_SHRD);
-	if (adata->key_inline)
-		append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
-				  adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT |
-				  KEY_ENC);
-	else
-		append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
-			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	if (era < 6) {
+		if (adata->key_inline)
+			append_key_as_imm(desc, adata->key_virt,
+					  adata->keylen_pad, adata->keylen,
+					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
+					  KEY_ENC);
+		else
+			append_key(desc, adata->key_dma, adata->keylen,
+				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	} else {
+		append_proto_dkp(desc, adata);
+	}
 	set_jump_tgt_here(desc, key_jump_cmd);
 
 	/* assoclen + cryptlen = seqinlen */
@@ -121,16 +126,16 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
  * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
  *                               (non-protocol) with no (null) decryption.
  * @desc: pointer to buffer used for descriptor construction
- * @adata: pointer to authentication transform definitions. Note that since a
- *         split key is to be used, the size of the split key itself is
- *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
- *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @adata: pointer to authentication transform definitions.
+ *         A split key is required for SEC Era < 6; the size of the split key
+ *         is specified in this case. Valid algorithm values - one of
+ *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
+ *         with OP_ALG_AAI_HMAC_PRECOMP.
  * @icvsize: integrity check value (ICV) size (truncated or full)
- *
- * Note: Requires an MDHA split key.
+ * @era: SEC Era
  */
 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
-				 unsigned int icvsize)
+				 unsigned int icvsize, int era)
 {
 	u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
 
@@ -139,13 +144,18 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
 	/* Skip if already shared */
 	key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 				   JUMP_COND_SHRD);
-	if (adata->key_inline)
-		append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
-				  adata->keylen, CLASS_2 |
-				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
-	else
-		append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
-			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	if (era < 6) {
+		if (adata->key_inline)
+			append_key_as_imm(desc, adata->key_virt,
+					  adata->keylen_pad, adata->keylen,
+					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
+					  KEY_ENC);
+		else
+			append_key(desc, adata->key_dma, adata->keylen,
+				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	} else {
+		append_proto_dkp(desc, adata);
+	}
 	set_jump_tgt_here(desc, key_jump_cmd);
 
 	/* Class 2 operation */
@@ -204,7 +214,7 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
 static void init_sh_desc_key_aead(u32 * const desc,
 				  struct alginfo * const cdata,
 				  struct alginfo * const adata,
-				  const bool is_rfc3686, u32 *nonce)
+				  const bool is_rfc3686, u32 *nonce, int era)
 {
 	u32 *key_jump_cmd;
 	unsigned int enckeylen = cdata->keylen;
@@ -224,13 +234,18 @@ static void init_sh_desc_key_aead(u32 * const desc,
 	if (is_rfc3686)
 		enckeylen -= CTR_RFC3686_NONCE_SIZE;
 
-	if (adata->key_inline)
-		append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
-				  adata->keylen, CLASS_2 |
-				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
-	else
-		append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
-			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	if (era < 6) {
+		if (adata->key_inline)
+			append_key_as_imm(desc, adata->key_virt,
+					  adata->keylen_pad, adata->keylen,
+					  CLASS_2 | KEY_DEST_MDHA_SPLIT |
+					  KEY_ENC);
+		else
+			append_key(desc, adata->key_dma, adata->keylen,
+				   CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
+	} else {
+		append_proto_dkp(desc, adata);
+	}
 
 	if (cdata->key_inline)
 		append_key_as_imm(desc, cdata->key_virt, enckeylen,
@@ -261,26 +276,27 @@ static void init_sh_desc_key_aead(u32 * const desc,
  * @cdata: pointer to block cipher transform definitions
  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
- * @adata: pointer to authentication transform definitions. Note that since a
- *         split key is to be used, the size of the split key itself is
- *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
- *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @adata: pointer to authentication transform definitions.
+ *         A split key is required for SEC Era < 6; the size of the split key
+ *         is specified in this case. Valid algorithm values - one of
+ *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
+ *         with OP_ALG_AAI_HMAC_PRECOMP.
  * @ivsize: initialization vector size
  * @icvsize: integrity check value (ICV) size (truncated or full)
  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
  * @nonce: pointer to rfc3686 nonce
  * @ctx1_iv_off: IV offset in CONTEXT1 register
  * @is_qi: true when called from caam/qi
- *
- * Note: Requires an MDHA split key.
+ * @era: SEC Era
  */
 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
 			    struct alginfo *adata, unsigned int ivsize,
 			    unsigned int icvsize, const bool is_rfc3686,
-			    u32 *nonce, const u32 ctx1_iv_off, const bool is_qi)
+			    u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
+			    int era)
 {
 	/* Note: Context registers are saved. */
-	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
+	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 
 	/* Class 2 operation */
 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
@@ -306,8 +322,13 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
 	}
 
 	/* Read and write assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+	if (is_qi || era < 3) {
+		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+	} else {
+		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
+		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
+	}
 
 	/* Skip assoc data */
 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
@@ -350,27 +371,27 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
  * @cdata: pointer to block cipher transform definitions
  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
- * @adata: pointer to authentication transform definitions. Note that since a
- *         split key is to be used, the size of the split key itself is
- *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
- *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @adata: pointer to authentication transform definitions.
+ *         A split key is required for SEC Era < 6; the size of the split key
+ *         is specified in this case. Valid algorithm values - one of
+ *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
+ *         with OP_ALG_AAI_HMAC_PRECOMP.
  * @ivsize: initialization vector size
  * @icvsize: integrity check value (ICV) size (truncated or full)
  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
  * @nonce: pointer to rfc3686 nonce
  * @ctx1_iv_off: IV offset in CONTEXT1 register
  * @is_qi: true when called from caam/qi
- *
- * Note: Requires an MDHA split key.
+ * @era: SEC Era
  */
 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
 			    struct alginfo *adata, unsigned int ivsize,
 			    unsigned int icvsize, const bool geniv,
 			    const bool is_rfc3686, u32 *nonce,
-			    const u32 ctx1_iv_off, const bool is_qi)
+			    const u32 ctx1_iv_off, const bool is_qi, int era)
 {
 	/* Note: Context registers are saved. */
-	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
+	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 
 	/* Class 2 operation */
 	append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
@@ -397,11 +418,23 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
 	}
 
 	/* Read and write assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	if (geniv)
-		append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
-	else
-		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+	if (is_qi || era < 3) {
+		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+		if (geniv)
+			append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
+						ivsize);
+		else
+			append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
+					CAAM_CMD_SZ);
+	} else {
+		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
+		if (geniv)
+			append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
+						ivsize);
+		else
+			append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
+					CAAM_CMD_SZ);
+	}
 
 	/* Skip assoc data */
 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
@@ -456,29 +489,29 @@ EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
  * @cdata: pointer to block cipher transform definitions
  *         Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
  *         with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
- * @adata: pointer to authentication transform definitions. Note that since a
- *         split key is to be used, the size of the split key itself is
- *         specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
- *         SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
+ * @adata: pointer to authentication transform definitions.
+ *         A split key is required for SEC Era < 6; the size of the split key
+ *         is specified in this case. Valid algorithm values - one of
+ *         OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
+ *         with OP_ALG_AAI_HMAC_PRECOMP.
  * @ivsize: initialization vector size
  * @icvsize: integrity check value (ICV) size (truncated or full)
  * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
  * @nonce: pointer to rfc3686 nonce
  * @ctx1_iv_off: IV offset in CONTEXT1 register
  * @is_qi: true when called from caam/qi
- *
- * Note: Requires an MDHA split key.
+ * @era: SEC Era
  */
 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
 			       struct alginfo *adata, unsigned int ivsize,
 			       unsigned int icvsize, const bool is_rfc3686,
 			       u32 *nonce, const u32 ctx1_iv_off,
-			       const bool is_qi)
+			       const bool is_qi, int era)
 {
 	u32 geniv, moveiv;
 
 	/* Note: Context registers are saved. */
-	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
+	init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
 
 	if (is_qi) {
 		u32 *wait_load_cmd;
@@ -528,8 +561,13 @@ void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
 			 OP_ALG_ENCRYPT);
 
 	/* Read and write assoclen bytes */
-	append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
-	append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+	if (is_qi || era < 3) {
+		append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+		append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+	} else {
+		append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
+		append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
+	}
 
 	/* Skip assoc data */
 	append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
@@ -1075,7 +1113,7 @@ void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
 
 	/* Load nonce into CONTEXT1 reg */
 	if (is_rfc3686) {
-		u8 *nonce = cdata->key_virt + cdata->keylen;
+		const u8 *nonce = cdata->key_virt + cdata->keylen;
 
 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
 				   LDST_CLASS_IND_CCB |
@@ -1140,7 +1178,7 @@ void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
 
 	/* Load nonce into CONTEXT1 reg */
 	if (is_rfc3686) {
-		u8 *nonce = cdata->key_virt + cdata->keylen;
+		const u8 *nonce = cdata->key_virt + cdata->keylen;
 
 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
 				   LDST_CLASS_IND_CCB |
@@ -1209,7 +1247,7 @@ void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
 
 	/* Load Nonce into CONTEXT1 reg */
 	if (is_rfc3686) {
-		u8 *nonce = cdata->key_virt + cdata->keylen;
+		const u8 *nonce = cdata->key_virt + cdata->keylen;
 
 		append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
 				   LDST_CLASS_IND_CCB |
diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h
index e412ec8f7005..5f9445ae2114 100644
--- a/drivers/crypto/caam/caamalg_desc.h
+++ b/drivers/crypto/caam/caamalg_desc.h
@@ -43,28 +43,28 @@
 					 15 * CAAM_CMD_SZ)
 
 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
-				 unsigned int icvsize);
+				 unsigned int icvsize, int era);
 
 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
-				 unsigned int icvsize);
+				 unsigned int icvsize, int era);
 
 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
 			    struct alginfo *adata, unsigned int ivsize,
 			    unsigned int icvsize, const bool is_rfc3686,
 			    u32 *nonce, const u32 ctx1_iv_off,
-			    const bool is_qi);
+			    const bool is_qi, int era);
 
 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
 			    struct alginfo *adata, unsigned int ivsize,
 			    unsigned int icvsize, const bool geniv,
 			    const bool is_rfc3686, u32 *nonce,
-			    const u32 ctx1_iv_off, const bool is_qi);
+			    const u32 ctx1_iv_off, const bool is_qi, int era);
 
 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
 			       struct alginfo *adata, unsigned int ivsize,
 			       unsigned int icvsize, const bool is_rfc3686,
 			       u32 *nonce, const u32 ctx1_iv_off,
-			       const bool is_qi);
+			       const bool is_qi, int era);
 
 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
 			   unsigned int icvsize);
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index f9f08fce4356..4aecc9435f69 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -53,6 +53,7 @@ struct caam_ctx {
 	u32 sh_desc_givenc[DESC_MAX_USED_LEN];
 	u8 key[CAAM_MAX_KEY_SIZE];
 	dma_addr_t key_dma;
+	enum dma_data_direction dir;
 	struct alginfo adata;
 	struct alginfo cdata;
 	unsigned int authsize;
@@ -74,6 +75,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 	const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
 			       OP_ALG_AAI_CTR_MOD128);
 	const bool is_rfc3686 = alg->caam.rfc3686;
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
 
 	if (!ctx->cdata.keylen || !ctx->authsize)
 		return 0;
@@ -124,7 +126,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 
 	cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 			       ivsize, ctx->authsize, is_rfc3686, nonce,
-			       ctx1_iv_off, true);
+			       ctx1_iv_off, true, ctrlpriv->era);
 
 skip_enc:
 	/* aead_decrypt shared descriptor */
@@ -149,7 +151,8 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 
 	cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
 			       ivsize, ctx->authsize, alg->caam.geniv,
-			       is_rfc3686, nonce, ctx1_iv_off, true);
+			       is_rfc3686, nonce, ctx1_iv_off, true,
+			       ctrlpriv->era);
 
 	if (!alg->caam.geniv)
 		goto skip_givenc;
@@ -176,7 +179,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead)
 
 	cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
 				  ivsize, ctx->authsize, is_rfc3686, nonce,
-				  ctx1_iv_off, true);
+				  ctx1_iv_off, true, ctrlpriv->era);
 
 skip_givenc:
 	return 0;
@@ -197,6 +200,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 {
 	struct caam_ctx *ctx = crypto_aead_ctx(aead);
 	struct device *jrdev = ctx->jrdev;
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 	struct crypto_authenc_keys keys;
 	int ret = 0;
 
@@ -211,6 +215,27 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 #endif
 
+	/*
+	 * If DKP is supported, use it in the shared descriptor to generate
+	 * the split key.
+	 */
+	if (ctrlpriv->era >= 6) {
+		ctx->adata.keylen = keys.authkeylen;
+		ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
+						      OP_ALG_ALGSEL_MASK);
+
+		if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
+			goto badkey;
+
+		memcpy(ctx->key, keys.authkey, keys.authkeylen);
+		memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
+		       keys.enckeylen);
+		dma_sync_single_for_device(jrdev, ctx->key_dma,
+					   ctx->adata.keylen_pad +
+					   keys.enckeylen, ctx->dir);
+		goto skip_split_key;
+	}
+
 	ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
 			    keys.authkeylen, CAAM_MAX_KEY_SIZE -
 			    keys.enckeylen);
@@ -220,13 +245,14 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 	/* postpend encryption key to auth split key */
 	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
-				   keys.enckeylen, DMA_TO_DEVICE);
+				   keys.enckeylen, ctx->dir);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 		       ctx->adata.keylen_pad + keys.enckeylen, 1);
 #endif
 
+skip_split_key:
 	ctx->cdata.keylen = keys.enckeylen;
 
 	ret = aead_set_sh_desc(aead);
@@ -272,7 +298,6 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 	const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686"));
 	int ret = 0;
 
-	memcpy(ctx->key, key, keylen);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
@@ -295,9 +320,8 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		keylen -= CTR_RFC3686_NONCE_SIZE;
 	}
 
-	dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
 	ctx->cdata.keylen = keylen;
-	ctx->cdata.key_virt = ctx->key;
+	ctx->cdata.key_virt = key;
 	ctx->cdata.key_inline = true;
 
 	/* ablkcipher encrypt, decrypt, givencrypt shared descriptors */
@@ -356,10 +380,8 @@ static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 		return -EINVAL;
 	}
 
-	memcpy(ctx->key, key, keylen);
-	dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE);
 	ctx->cdata.keylen = keylen;
-	ctx->cdata.key_virt = ctx->key;
+	ctx->cdata.key_virt = key;
 	ctx->cdata.key_inline = true;
 
 	/* xts ablkcipher encrypt, decrypt shared descriptors */
@@ -668,7 +690,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
 	qm_sg_ents = 1 + !!ivsize + mapped_src_nents +
 		     (mapped_dst_nents > 1 ? mapped_dst_nents : 0);
 	if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) {
-		dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
+		dev_err(qidev, "Insufficient S/G entries: %d > %zu\n",
 			qm_sg_ents, CAAM_QI_MAX_AEAD_SG);
 		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
 			   iv_dma, ivsize, op_type, 0, 0);
@@ -905,7 +927,7 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
 
 	qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
 	if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
-		dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
+		dev_err(qidev, "Insufficient S/G entries: %d > %zu\n",
 			qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
 		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
 			   iv_dma, ivsize, op_type, 0, 0);
@@ -1058,7 +1080,7 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
 	}
 
 	if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) {
-		dev_err(qidev, "Insufficient S/G entries: %d > %lu\n",
+		dev_err(qidev, "Insufficient S/G entries: %d > %zu\n",
 			qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG);
 		caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
 			   iv_dma, ivsize, GIVENCRYPT, 0, 0);
@@ -2123,7 +2145,8 @@ struct caam_crypto_alg {
 	struct caam_alg_entry caam;
 };
 
-static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
+static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
+			    bool uses_dkp)
 {
 	struct caam_drv_private *priv;
 
@@ -2137,8 +2160,14 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
 		return PTR_ERR(ctx->jrdev);
 	}
 
+	priv = dev_get_drvdata(ctx->jrdev->parent);
+	if (priv->era >= 6 && uses_dkp)
+		ctx->dir = DMA_BIDIRECTIONAL;
+	else
+		ctx->dir = DMA_TO_DEVICE;
+
 	ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key),
-				      DMA_TO_DEVICE);
+				      ctx->dir);
 	if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) {
 		dev_err(ctx->jrdev, "unable to map key\n");
 		caam_jr_free(ctx->jrdev);
@@ -2149,7 +2178,6 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
 	ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
 	ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
 
-	priv = dev_get_drvdata(ctx->jrdev->parent);
 	ctx->qidev = priv->qidev;
 
 	spin_lock_init(&ctx->lock);
@@ -2167,7 +2195,7 @@ static int caam_cra_init(struct crypto_tfm *tfm)
 							crypto_alg);
 	struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
 
-	return caam_init_common(ctx, &caam_alg->caam);
+	return caam_init_common(ctx, &caam_alg->caam, false);
 }
 
 static int caam_aead_init(struct crypto_aead *tfm)
@@ -2177,7 +2205,8 @@ static int caam_aead_init(struct crypto_aead *tfm)
 						      aead);
 	struct caam_ctx *ctx = crypto_aead_ctx(tfm);
 
-	return caam_init_common(ctx, &caam_alg->caam);
+	return caam_init_common(ctx, &caam_alg->caam,
+				alg->setkey == aead_setkey);
 }
 
 static void caam_exit_common(struct caam_ctx *ctx)
@@ -2186,8 +2215,7 @@ static void caam_exit_common(struct caam_ctx *ctx)
 	caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
 	caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]);
 
-	dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key),
-			 DMA_TO_DEVICE);
+	dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir);
 
 	caam_jr_free(ctx->jrdev);
 }
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 616720a04e7a..0beb28196e20 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -107,6 +107,7 @@ struct caam_hash_ctx {
 	dma_addr_t sh_desc_update_first_dma;
 	dma_addr_t sh_desc_fin_dma;
 	dma_addr_t sh_desc_digest_dma;
+	enum dma_data_direction dir;
 	struct device *jrdev;
 	u8 key[CAAM_MAX_HASH_KEY_SIZE];
 	int ctx_len;
@@ -241,7 +242,8 @@ static inline int ctx_map_to_sec4_sg(struct device *jrdev,
  *     read and write to seqout
  */
 static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
-				     struct caam_hash_ctx *ctx, bool import_ctx)
+				     struct caam_hash_ctx *ctx, bool import_ctx,
+				     int era)
 {
 	u32 op = ctx->adata.algtype;
 	u32 *skip_key_load;
@@ -254,9 +256,12 @@ static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
 		skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 					    JUMP_COND_SHRD);
 
-		append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
-				  ctx->adata.keylen, CLASS_2 |
-				  KEY_DEST_MDHA_SPLIT | KEY_ENC);
+		if (era < 6)
+			append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
+					  ctx->adata.keylen, CLASS_2 |
+					  KEY_DEST_MDHA_SPLIT | KEY_ENC);
+		else
+			append_proto_dkp(desc, &ctx->adata);
 
 		set_jump_tgt_here(desc, skip_key_load);
 
@@ -289,13 +294,17 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct device *jrdev = ctx->jrdev;
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
 	u32 *desc;
 
+	ctx->adata.key_virt = ctx->key;
+
 	/* ahash_update shared descriptor */
 	desc = ctx->sh_desc_update;
-	ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
+	ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true,
+			  ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR,
 		       "ahash update shdesc@"__stringify(__LINE__)": ",
@@ -304,9 +313,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 
 	/* ahash_update_first shared descriptor */
 	desc = ctx->sh_desc_update_first;
-	ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
+	ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false,
+			  ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR,
 		       "ahash update first shdesc@"__stringify(__LINE__)": ",
@@ -315,9 +325,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 
 	/* ahash_final shared descriptor */
 	desc = ctx->sh_desc_fin;
-	ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
+	ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true,
+			  ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
 		       DUMP_PREFIX_ADDRESS, 16, 4, desc,
@@ -326,9 +337,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash)
 
 	/* ahash_digest shared descriptor */
 	desc = ctx->sh_desc_digest;
-	ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
+	ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false,
+			  ctrlpriv->era);
 	dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,
-				   desc_bytes(desc), DMA_TO_DEVICE);
+				   desc_bytes(desc), ctx->dir);
 #ifdef DEBUG
 	print_hex_dump(KERN_ERR,
 		       "ahash digest shdesc@"__stringify(__LINE__)": ",
@@ -421,6 +433,7 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 	struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
 	int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
 	int digestsize = crypto_ahash_digestsize(ahash);
+	struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
 	int ret;
 	u8 *hashed_key = NULL;
 
@@ -441,16 +454,26 @@ static int ahash_setkey(struct crypto_ahash *ahash,
 		key = hashed_key;
 	}
 
-	ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
-			    CAAM_MAX_HASH_KEY_SIZE);
-	if (ret)
-		goto bad_free_key;
+	/*
+	 * If DKP is supported, use it in the shared descriptor to generate
+	 * the split key.
+	 */
+	if (ctrlpriv->era >= 6) {
+		ctx->adata.key_inline = true;
+		ctx->adata.keylen = keylen;
+		ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
+						      OP_ALG_ALGSEL_MASK);
 
-#ifdef DEBUG
-	print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
-		       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
-		       ctx->adata.keylen_pad, 1);
-#endif
+		if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE)
+			goto bad_free_key;
+
+		memcpy(ctx->key, key, keylen);
+	} else {
+		ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,
+				    keylen, CAAM_MAX_HASH_KEY_SIZE);
+		if (ret)
+			goto bad_free_key;
+	}
 
 	kfree(hashed_key);
 	return ahash_set_sh_desc(ahash);
@@ -1715,6 +1738,7 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
 					 HASH_MSG_LEN + 64,
 					 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
 	dma_addr_t dma_addr;
+	struct caam_drv_private *priv;
 
 	/*
 	 * Get a Job ring from Job Ring driver to ensure in-order
@@ -1726,10 +1750,13 @@ static int caam_hash_cra_init(struct crypto_tfm *tfm)
 		return PTR_ERR(ctx->jrdev);
 	}
 
+	priv = dev_get_drvdata(ctx->jrdev->parent);
+	ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+
 	dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,
 					offsetof(struct caam_hash_ctx,
 						 sh_desc_update_dma),
-					DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
+					ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
 	if (dma_mapping_error(ctx->jrdev, dma_addr)) {
 		dev_err(ctx->jrdev, "unable to map shared descriptors\n");
 		caam_jr_free(ctx->jrdev);
@@ -1764,7 +1791,7 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm)
 	dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,
 			       offsetof(struct caam_hash_ctx,
 					sh_desc_update_dma),
-			       DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC);
+			       ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
 	caam_jr_free(ctx->jrdev);
 }
 
diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c
index 027e121c6f70..e843cf410373 100644
--- a/drivers/crypto/caam/ctrl.c
+++ b/drivers/crypto/caam/ctrl.c
@@ -228,12 +228,16 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask,
 		 * without any error (HW optimizations for later
 		 * CAAM eras), then try again.
 		 */
+		if (ret)
+			break;
+
 		rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK;
 		if ((status && status != JRSTA_SSRC_JUMP_HALT_CC) ||
-		    !(rdsta_val & (1 << sh_idx)))
+		    !(rdsta_val & (1 << sh_idx))) {
 			ret = -EAGAIN;
-		if (ret)
 			break;
+		}
+
 		dev_info(ctrldev, "Instantiated RNG4 SH%d\n", sh_idx);
 		/* Clear the contents before recreating the descriptor */
 		memset(desc, 0x00, CAAM_CMD_SZ * 7);
@@ -611,6 +615,8 @@ static int caam_probe(struct platform_device *pdev)
 		goto iounmap_ctrl;
 	}
 
+	ctrlpriv->era = caam_get_era();
+
 	ret = of_platform_populate(nprop, caam_match, NULL, dev);
 	if (ret) {
 		dev_err(dev, "JR platform devices creation error\n");
@@ -742,7 +748,7 @@ static int caam_probe(struct platform_device *pdev)
 
 	/* Report "alive" for developer to see */
 	dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id,
-		 caam_get_era());
+		 ctrlpriv->era);
 	dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n",
 		 ctrlpriv->total_jobrs, ctrlpriv->qi_present,
 		 caam_dpaa2 ? "yes" : "no");
diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h
index 8142de7ba050..f76ff160a02c 100644
--- a/drivers/crypto/caam/desc.h
+++ b/drivers/crypto/caam/desc.h
@@ -444,6 +444,18 @@
 #define OP_PCLID_DSAVERIFY	(0x16 << OP_PCLID_SHIFT)
 #define OP_PCLID_RSAENC_PUBKEY  (0x18 << OP_PCLID_SHIFT)
 #define OP_PCLID_RSADEC_PRVKEY  (0x19 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_MD5	(0x20 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_SHA1	(0x21 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_SHA224	(0x22 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_SHA256	(0x23 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_SHA384	(0x24 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_SHA512	(0x25 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_RIF_MD5	(0x60 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_RIF_SHA1	(0x61 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_RIF_SHA224	(0x62 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_RIF_SHA256	(0x63 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_RIF_SHA384	(0x64 << OP_PCLID_SHIFT)
+#define OP_PCLID_DKP_RIF_SHA512	(0x65 << OP_PCLID_SHIFT)
 
 /* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */
 #define OP_PCLID_IPSEC		(0x01 << OP_PCLID_SHIFT)
@@ -1093,6 +1105,22 @@
 /* MacSec protinfos */
 #define OP_PCL_MACSEC				 0x0001
 
+/* Derived Key Protocol (DKP) Protinfo */
+#define OP_PCL_DKP_SRC_SHIFT	14
+#define OP_PCL_DKP_SRC_MASK	(3 << OP_PCL_DKP_SRC_SHIFT)
+#define OP_PCL_DKP_SRC_IMM	(0 << OP_PCL_DKP_SRC_SHIFT)
+#define OP_PCL_DKP_SRC_SEQ	(1 << OP_PCL_DKP_SRC_SHIFT)
+#define OP_PCL_DKP_SRC_PTR	(2 << OP_PCL_DKP_SRC_SHIFT)
+#define OP_PCL_DKP_SRC_SGF	(3 << OP_PCL_DKP_SRC_SHIFT)
+#define OP_PCL_DKP_DST_SHIFT	12
+#define OP_PCL_DKP_DST_MASK	(3 << OP_PCL_DKP_DST_SHIFT)
+#define OP_PCL_DKP_DST_IMM	(0 << OP_PCL_DKP_DST_SHIFT)
+#define OP_PCL_DKP_DST_SEQ	(1 << OP_PCL_DKP_DST_SHIFT)
+#define OP_PCL_DKP_DST_PTR	(2 << OP_PCL_DKP_DST_SHIFT)
+#define OP_PCL_DKP_DST_SGF	(3 << OP_PCL_DKP_DST_SHIFT)
+#define OP_PCL_DKP_KEY_SHIFT	0
+#define OP_PCL_DKP_KEY_MASK	(0xfff << OP_PCL_DKP_KEY_SHIFT)
+
 /* PKI unidirectional protocol protinfo bits */
 #define OP_PCL_PKPROT_TEST			 0x0008
 #define OP_PCL_PKPROT_DECRYPT			 0x0004
@@ -1452,6 +1480,7 @@
 #define MATH_DEST_REG1		(0x01 << MATH_DEST_SHIFT)
 #define MATH_DEST_REG2		(0x02 << MATH_DEST_SHIFT)
 #define MATH_DEST_REG3		(0x03 << MATH_DEST_SHIFT)
+#define MATH_DEST_DPOVRD	(0x07 << MATH_DEST_SHIFT)
 #define MATH_DEST_SEQINLEN	(0x08 << MATH_DEST_SHIFT)
 #define MATH_DEST_SEQOUTLEN	(0x09 << MATH_DEST_SHIFT)
 #define MATH_DEST_VARSEQINLEN	(0x0a << MATH_DEST_SHIFT)
diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h
index ba1ca0806f0a..d4256fa4a1d6 100644
--- a/drivers/crypto/caam/desc_constr.h
+++ b/drivers/crypto/caam/desc_constr.h
@@ -109,7 +109,7 @@ static inline void init_job_desc_shared(u32 * const desc, dma_addr_t ptr,
 	append_ptr(desc, ptr);
 }
 
-static inline void append_data(u32 * const desc, void *data, int len)
+static inline void append_data(u32 * const desc, const void *data, int len)
 {
 	u32 *offset = desc_end(desc);
 
@@ -172,7 +172,7 @@ static inline void append_cmd_ptr_extlen(u32 * const desc, dma_addr_t ptr,
 	append_cmd(desc, len);
 }
 
-static inline void append_cmd_data(u32 * const desc, void *data, int len,
+static inline void append_cmd_data(u32 * const desc, const void *data, int len,
 				   u32 command)
 {
 	append_cmd(desc, command | IMMEDIATE | len);
@@ -271,7 +271,7 @@ APPEND_SEQ_PTR_INTLEN(in, IN)
 APPEND_SEQ_PTR_INTLEN(out, OUT)
 
 #define APPEND_CMD_PTR_TO_IMM(cmd, op) \
-static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \
+static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \
 					 unsigned int len, u32 options) \
 { \
 	PRINT_POS; \
@@ -312,7 +312,7 @@ APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_PTR, u32)
  * from length of immediate data provided, e.g., split keys
  */
 #define APPEND_CMD_PTR_TO_IMM2(cmd, op) \
-static inline void append_##cmd##_as_imm(u32 * const desc, void *data, \
+static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \
 					 unsigned int data_len, \
 					 unsigned int len, u32 options) \
 { \
@@ -452,7 +452,7 @@ struct alginfo {
 	unsigned int keylen_pad;
 	union {
 		dma_addr_t key_dma;
-		void *key_virt;
+		const void *key_virt;
 	};
 	bool key_inline;
 };
@@ -496,4 +496,45 @@ static inline int desc_inline_query(unsigned int sd_base_len,
 	return (rem_bytes >= 0) ? 0 : -1;
 }
 
+/**
+ * append_proto_dkp - Derived Key Protocol (DKP): key -> split key
+ * @desc: pointer to buffer used for descriptor construction
+ * @adata: pointer to authentication transform definitions.
+ *         keylen should be the length of initial key, while keylen_pad
+ *         the length of the derived (split) key.
+ *         Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224,
+ *         SHA256, SHA384, SHA512}.
+ */
+static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata)
+{
+	u32 protid;
+
+	/*
+	 * Quick & dirty translation from OP_ALG_ALGSEL_{MD5, SHA*}
+	 * to OP_PCLID_DKP_{MD5, SHA*}
+	 */
+	protid = (adata->algtype & OP_ALG_ALGSEL_SUBMASK) |
+		 (0x20 << OP_ALG_ALGSEL_SHIFT);
+
+	if (adata->key_inline) {
+		int words;
+
+		append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
+				 OP_PCL_DKP_SRC_IMM | OP_PCL_DKP_DST_IMM |
+				 adata->keylen);
+		append_data(desc, adata->key_virt, adata->keylen);
+
+		/* Reserve space in descriptor buffer for the derived key */
+		words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) -
+			 ALIGN(adata->keylen, CAAM_CMD_SZ)) / CAAM_CMD_SZ;
+		if (words)
+			(*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words);
+	} else {
+		append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid |
+				 OP_PCL_DKP_SRC_PTR | OP_PCL_DKP_DST_PTR |
+				 adata->keylen);
+		append_ptr(desc, adata->key_dma);
+	}
+}
+
 #endif /* DESC_CONSTR_H */
diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h
index 91f1107276e5..7696a774a362 100644
--- a/drivers/crypto/caam/intern.h
+++ b/drivers/crypto/caam/intern.h
@@ -84,6 +84,7 @@ struct caam_drv_private {
 	u8 qi_present;		/* Nonzero if QI present in device */
 	int secvio_irq;		/* Security violation interrupt number */
 	int virt_en;		/* Virtualization enabled in CAAM */
+	int era;		/* CAAM Era (internal HW revision) */
 
 #define	RNG4_MAX_HANDLES 2
 	/* RNG4 block */
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index 8c79c3a153dc..312b5f042f31 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -11,36 +11,6 @@
 #include "desc_constr.h"
 #include "key_gen.h"
 
-/**
- * split_key_len - Compute MDHA split key length for a given algorithm
- * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
- *        SHA224, SHA384, SHA512.
- *
- * Return: MDHA split key length
- */
-static inline u32 split_key_len(u32 hash)
-{
-	/* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
-	static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
-	u32 idx;
-
-	idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT;
-
-	return (u32)(mdpadlen[idx] * 2);
-}
-
-/**
- * split_key_pad_len - Compute MDHA split key pad length for a given algorithm
- * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
- *        SHA224, SHA384, SHA512.
- *
- * Return: MDHA split key pad length
- */
-static inline u32 split_key_pad_len(u32 hash)
-{
-	return ALIGN(split_key_len(hash), 16);
-}
-
 void split_key_done(struct device *dev, u32 *desc, u32 err,
 			   void *context)
 {
diff --git a/drivers/crypto/caam/key_gen.h b/drivers/crypto/caam/key_gen.h
index 5db055c25bd2..818f78f6fc1a 100644
--- a/drivers/crypto/caam/key_gen.h
+++ b/drivers/crypto/caam/key_gen.h
@@ -6,6 +6,36 @@
  *
  */
 
+/**
+ * split_key_len - Compute MDHA split key length for a given algorithm
+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
+ *        SHA224, SHA384, SHA512.
+ *
+ * Return: MDHA split key length
+ */
+static inline u32 split_key_len(u32 hash)
+{
+	/* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
+	static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
+	u32 idx;
+
+	idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT;
+
+	return (u32)(mdpadlen[idx] * 2);
+}
+
+/**
+ * split_key_pad_len - Compute MDHA split key pad length for a given algorithm
+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1,
+ *        SHA224, SHA384, SHA512.
+ *
+ * Return: MDHA split key pad length
+ */
+static inline u32 split_key_pad_len(u32 hash)
+{
+	return ALIGN(split_key_len(hash), 16);
+}
+
 struct split_key_result {
 	struct completion completion;
 	int err;
diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
index 169e66231bcf..b0ba4331944b 100644
--- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
+++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c
@@ -459,7 +459,8 @@ int process_request(struct cpt_vf *cptvf, struct cpt_request_info *req)
 	info->completion_addr = kzalloc(sizeof(union cpt_res_s), GFP_KERNEL);
 	if (unlikely(!info->completion_addr)) {
 		dev_err(&pdev->dev, "Unable to allocate memory for completion_addr\n");
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto request_cleanup;
 	}
 
 	result = (union cpt_res_s *)info->completion_addr;
diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
index 4addc238a6ef..deaefd532aaa 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c
@@ -6,7 +6,6 @@
 #include "nitrox_dev.h"
 #include "nitrox_req.h"
 #include "nitrox_csr.h"
-#include "nitrox_req.h"
 
 /* SLC_STORE_INFO */
 #define MIN_UDD_LEN 16
diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig
index 6d626606b9c5..b9dfae47aefd 100644
--- a/drivers/crypto/ccp/Kconfig
+++ b/drivers/crypto/ccp/Kconfig
@@ -1,5 +1,6 @@
 config CRYPTO_DEV_CCP_DD
 	tristate "Secure Processor device driver"
+	depends on CPU_SUP_AMD || ARM64
 	default m
 	help
 	  Provides AMD Secure Processor device driver.
@@ -32,3 +33,14 @@ config CRYPTO_DEV_CCP_CRYPTO
 	  Support for using the cryptographic API with the AMD Cryptographic
 	  Coprocessor. This module supports offload of SHA and AES algorithms.
 	  If you choose 'M' here, this module will be called ccp_crypto.
+
+config CRYPTO_DEV_SP_PSP
+	bool "Platform Security Processor (PSP) device"
+	default y
+	depends on CRYPTO_DEV_CCP_DD && X86_64
+	help
+	 Provide support for the AMD Platform Security Processor (PSP).
+	 The PSP is a dedicated processor that provides support for key
+	 management commands in Secure Encrypted Virtualization (SEV) mode,
+	 along with software-based Trusted Execution Environment (TEE) to
+	 enable third-party trusted applications.
diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile
index c4ce726b931e..51d1c0cf66c7 100644
--- a/drivers/crypto/ccp/Makefile
+++ b/drivers/crypto/ccp/Makefile
@@ -8,6 +8,7 @@ ccp-$(CONFIG_CRYPTO_DEV_SP_CCP) += ccp-dev.o \
 	    ccp-dmaengine.o \
 	    ccp-debugfs.o
 ccp-$(CONFIG_PCI) += sp-pci.o
+ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += psp-dev.o
 
 obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o
 ccp-crypto-objs := ccp-crypto-main.o \
diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
index ff02b713c6f6..ca1f0d780b61 100644
--- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c
+++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c
@@ -21,7 +21,6 @@
 #include <crypto/ctr.h>
 #include <crypto/gcm.h>
 #include <crypto/scatterwalk.h>
-#include <linux/delay.h>
 
 #include "ccp-crypto.h"
 
diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c
new file mode 100644
index 000000000000..b3afb6cc9d72
--- /dev/null
+++ b/drivers/crypto/ccp/psp-dev.c
@@ -0,0 +1,805 @@
+/*
+ * AMD Platform Security Processor (PSP) interface
+ *
+ * Copyright (C) 2016-2017 Advanced Micro Devices, Inc.
+ *
+ * Author: Brijesh Singh <brijesh.singh@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/kthread.h>
+#include <linux/sched.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/hw_random.h>
+#include <linux/ccp.h>
+
+#include "sp-dev.h"
+#include "psp-dev.h"
+
+#define DEVICE_NAME	"sev"
+
+static DEFINE_MUTEX(sev_cmd_mutex);
+static struct sev_misc_dev *misc_dev;
+static struct psp_device *psp_master;
+
+static struct psp_device *psp_alloc_struct(struct sp_device *sp)
+{
+	struct device *dev = sp->dev;
+	struct psp_device *psp;
+
+	psp = devm_kzalloc(dev, sizeof(*psp), GFP_KERNEL);
+	if (!psp)
+		return NULL;
+
+	psp->dev = dev;
+	psp->sp = sp;
+
+	snprintf(psp->name, sizeof(psp->name), "psp-%u", sp->ord);
+
+	return psp;
+}
+
+static irqreturn_t psp_irq_handler(int irq, void *data)
+{
+	struct psp_device *psp = data;
+	unsigned int status;
+	int reg;
+
+	/* Read the interrupt status: */
+	status = ioread32(psp->io_regs + PSP_P2CMSG_INTSTS);
+
+	/* Check if it is command completion: */
+	if (!(status & BIT(PSP_CMD_COMPLETE_REG)))
+		goto done;
+
+	/* Check if it is SEV command completion: */
+	reg = ioread32(psp->io_regs + PSP_CMDRESP);
+	if (reg & PSP_CMDRESP_RESP) {
+		psp->sev_int_rcvd = 1;
+		wake_up(&psp->sev_int_queue);
+	}
+
+done:
+	/* Clear the interrupt status by writing the same value we read. */
+	iowrite32(status, psp->io_regs + PSP_P2CMSG_INTSTS);
+
+	return IRQ_HANDLED;
+}
+
+static void sev_wait_cmd_ioc(struct psp_device *psp, unsigned int *reg)
+{
+	psp->sev_int_rcvd = 0;
+
+	wait_event(psp->sev_int_queue, psp->sev_int_rcvd);
+	*reg = ioread32(psp->io_regs + PSP_CMDRESP);
+}
+
+static int sev_cmd_buffer_len(int cmd)
+{
+	switch (cmd) {
+	case SEV_CMD_INIT:			return sizeof(struct sev_data_init);
+	case SEV_CMD_PLATFORM_STATUS:		return sizeof(struct sev_user_data_status);
+	case SEV_CMD_PEK_CSR:			return sizeof(struct sev_data_pek_csr);
+	case SEV_CMD_PEK_CERT_IMPORT:		return sizeof(struct sev_data_pek_cert_import);
+	case SEV_CMD_PDH_CERT_EXPORT:		return sizeof(struct sev_data_pdh_cert_export);
+	case SEV_CMD_LAUNCH_START:		return sizeof(struct sev_data_launch_start);
+	case SEV_CMD_LAUNCH_UPDATE_DATA:	return sizeof(struct sev_data_launch_update_data);
+	case SEV_CMD_LAUNCH_UPDATE_VMSA:	return sizeof(struct sev_data_launch_update_vmsa);
+	case SEV_CMD_LAUNCH_FINISH:		return sizeof(struct sev_data_launch_finish);
+	case SEV_CMD_LAUNCH_MEASURE:		return sizeof(struct sev_data_launch_measure);
+	case SEV_CMD_ACTIVATE:			return sizeof(struct sev_data_activate);
+	case SEV_CMD_DEACTIVATE:		return sizeof(struct sev_data_deactivate);
+	case SEV_CMD_DECOMMISSION:		return sizeof(struct sev_data_decommission);
+	case SEV_CMD_GUEST_STATUS:		return sizeof(struct sev_data_guest_status);
+	case SEV_CMD_DBG_DECRYPT:		return sizeof(struct sev_data_dbg);
+	case SEV_CMD_DBG_ENCRYPT:		return sizeof(struct sev_data_dbg);
+	case SEV_CMD_SEND_START:		return sizeof(struct sev_data_send_start);
+	case SEV_CMD_SEND_UPDATE_DATA:		return sizeof(struct sev_data_send_update_data);
+	case SEV_CMD_SEND_UPDATE_VMSA:		return sizeof(struct sev_data_send_update_vmsa);
+	case SEV_CMD_SEND_FINISH:		return sizeof(struct sev_data_send_finish);
+	case SEV_CMD_RECEIVE_START:		return sizeof(struct sev_data_receive_start);
+	case SEV_CMD_RECEIVE_FINISH:		return sizeof(struct sev_data_receive_finish);
+	case SEV_CMD_RECEIVE_UPDATE_DATA:	return sizeof(struct sev_data_receive_update_data);
+	case SEV_CMD_RECEIVE_UPDATE_VMSA:	return sizeof(struct sev_data_receive_update_vmsa);
+	case SEV_CMD_LAUNCH_UPDATE_SECRET:	return sizeof(struct sev_data_launch_secret);
+	default:				return 0;
+	}
+
+	return 0;
+}
+
+static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
+{
+	struct psp_device *psp = psp_master;
+	unsigned int phys_lsb, phys_msb;
+	unsigned int reg, ret = 0;
+
+	if (!psp)
+		return -ENODEV;
+
+	/* Get the physical address of the command buffer */
+	phys_lsb = data ? lower_32_bits(__psp_pa(data)) : 0;
+	phys_msb = data ? upper_32_bits(__psp_pa(data)) : 0;
+
+	dev_dbg(psp->dev, "sev command id %#x buffer 0x%08x%08x\n",
+		cmd, phys_msb, phys_lsb);
+
+	print_hex_dump_debug("(in):  ", DUMP_PREFIX_OFFSET, 16, 2, data,
+			     sev_cmd_buffer_len(cmd), false);
+
+	iowrite32(phys_lsb, psp->io_regs + PSP_CMDBUFF_ADDR_LO);
+	iowrite32(phys_msb, psp->io_regs + PSP_CMDBUFF_ADDR_HI);
+
+	reg = cmd;
+	reg <<= PSP_CMDRESP_CMD_SHIFT;
+	reg |= PSP_CMDRESP_IOC;
+	iowrite32(reg, psp->io_regs + PSP_CMDRESP);
+
+	/* wait for command completion */
+	sev_wait_cmd_ioc(psp, &reg);
+
+	if (psp_ret)
+		*psp_ret = reg & PSP_CMDRESP_ERR_MASK;
+
+	if (reg & PSP_CMDRESP_ERR_MASK) {
+		dev_dbg(psp->dev, "sev command %#x failed (%#010x)\n",
+			cmd, reg & PSP_CMDRESP_ERR_MASK);
+		ret = -EIO;
+	}
+
+	print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
+			     sev_cmd_buffer_len(cmd), false);
+
+	return ret;
+}
+
+static int sev_do_cmd(int cmd, void *data, int *psp_ret)
+{
+	int rc;
+
+	mutex_lock(&sev_cmd_mutex);
+	rc = __sev_do_cmd_locked(cmd, data, psp_ret);
+	mutex_unlock(&sev_cmd_mutex);
+
+	return rc;
+}
+
+static int __sev_platform_init_locked(int *error)
+{
+	struct psp_device *psp = psp_master;
+	int rc = 0;
+
+	if (!psp)
+		return -ENODEV;
+
+	if (psp->sev_state == SEV_STATE_INIT)
+		return 0;
+
+	rc = __sev_do_cmd_locked(SEV_CMD_INIT, &psp->init_cmd_buf, error);
+	if (rc)
+		return rc;
+
+	psp->sev_state = SEV_STATE_INIT;
+	dev_dbg(psp->dev, "SEV firmware initialized\n");
+
+	return rc;
+}
+
+int sev_platform_init(int *error)
+{
+	int rc;
+
+	mutex_lock(&sev_cmd_mutex);
+	rc = __sev_platform_init_locked(error);
+	mutex_unlock(&sev_cmd_mutex);
+
+	return rc;
+}
+EXPORT_SYMBOL_GPL(sev_platform_init);
+
+static int __sev_platform_shutdown_locked(int *error)
+{
+	int ret;
+
+	ret = __sev_do_cmd_locked(SEV_CMD_SHUTDOWN, NULL, error);
+	if (ret)
+		return ret;
+
+	psp_master->sev_state = SEV_STATE_UNINIT;
+	dev_dbg(psp_master->dev, "SEV firmware shutdown\n");
+
+	return ret;
+}
+
+static int sev_platform_shutdown(int *error)
+{
+	int rc;
+
+	mutex_lock(&sev_cmd_mutex);
+	rc = __sev_platform_shutdown_locked(NULL);
+	mutex_unlock(&sev_cmd_mutex);
+
+	return rc;
+}
+
+static int sev_get_platform_state(int *state, int *error)
+{
+	int rc;
+
+	rc = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS,
+				 &psp_master->status_cmd_buf, error);
+	if (rc)
+		return rc;
+
+	*state = psp_master->status_cmd_buf.state;
+	return rc;
+}
+
+static int sev_ioctl_do_reset(struct sev_issue_cmd *argp)
+{
+	int state, rc;
+
+	/*
+	 * The SEV spec requires that FACTORY_RESET must be issued in
+	 * UNINIT state. Before we go further lets check if any guest is
+	 * active.
+	 *
+	 * If FW is in WORKING state then deny the request otherwise issue
+	 * SHUTDOWN command do INIT -> UNINIT before issuing the FACTORY_RESET.
+	 *
+	 */
+	rc = sev_get_platform_state(&state, &argp->error);
+	if (rc)
+		return rc;
+
+	if (state == SEV_STATE_WORKING)
+		return -EBUSY;
+
+	if (state == SEV_STATE_INIT) {
+		rc = __sev_platform_shutdown_locked(&argp->error);
+		if (rc)
+			return rc;
+	}
+
+	return __sev_do_cmd_locked(SEV_CMD_FACTORY_RESET, NULL, &argp->error);
+}
+
+static int sev_ioctl_do_platform_status(struct sev_issue_cmd *argp)
+{
+	struct sev_user_data_status *data = &psp_master->status_cmd_buf;
+	int ret;
+
+	ret = __sev_do_cmd_locked(SEV_CMD_PLATFORM_STATUS, data, &argp->error);
+	if (ret)
+		return ret;
+
+	if (copy_to_user((void __user *)argp->data, data, sizeof(*data)))
+		ret = -EFAULT;
+
+	return ret;
+}
+
+static int sev_ioctl_do_pek_pdh_gen(int cmd, struct sev_issue_cmd *argp)
+{
+	int rc;
+
+	if (psp_master->sev_state == SEV_STATE_UNINIT) {
+		rc = __sev_platform_init_locked(&argp->error);
+		if (rc)
+			return rc;
+	}
+
+	return __sev_do_cmd_locked(cmd, NULL, &argp->error);
+}
+
+static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp)
+{
+	struct sev_user_data_pek_csr input;
+	struct sev_data_pek_csr *data;
+	void *blob = NULL;
+	int ret;
+
+	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
+		return -EFAULT;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* userspace wants to query CSR length */
+	if (!input.address || !input.length)
+		goto cmd;
+
+	/* allocate a physically contiguous buffer to store the CSR blob */
+	if (!access_ok(VERIFY_WRITE, input.address, input.length) ||
+	    input.length > SEV_FW_BLOB_MAX_SIZE) {
+		ret = -EFAULT;
+		goto e_free;
+	}
+
+	blob = kmalloc(input.length, GFP_KERNEL);
+	if (!blob) {
+		ret = -ENOMEM;
+		goto e_free;
+	}
+
+	data->address = __psp_pa(blob);
+	data->len = input.length;
+
+cmd:
+	if (psp_master->sev_state == SEV_STATE_UNINIT) {
+		ret = __sev_platform_init_locked(&argp->error);
+		if (ret)
+			goto e_free_blob;
+	}
+
+	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, data, &argp->error);
+
+	 /* If we query the CSR length, FW responded with expected data. */
+	input.length = data->len;
+
+	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
+		ret = -EFAULT;
+		goto e_free_blob;
+	}
+
+	if (blob) {
+		if (copy_to_user((void __user *)input.address, blob, input.length))
+			ret = -EFAULT;
+	}
+
+e_free_blob:
+	kfree(blob);
+e_free:
+	kfree(data);
+	return ret;
+}
+
+void *psp_copy_user_blob(u64 __user uaddr, u32 len)
+{
+	void *data;
+
+	if (!uaddr || !len)
+		return ERR_PTR(-EINVAL);
+
+	/* verify that blob length does not exceed our limit */
+	if (len > SEV_FW_BLOB_MAX_SIZE)
+		return ERR_PTR(-EINVAL);
+
+	data = kmalloc(len, GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
+
+	if (copy_from_user(data, (void __user *)(uintptr_t)uaddr, len))
+		goto e_free;
+
+	return data;
+
+e_free:
+	kfree(data);
+	return ERR_PTR(-EFAULT);
+}
+EXPORT_SYMBOL_GPL(psp_copy_user_blob);
+
+static int sev_ioctl_do_pek_import(struct sev_issue_cmd *argp)
+{
+	struct sev_user_data_pek_cert_import input;
+	struct sev_data_pek_cert_import *data;
+	void *pek_blob, *oca_blob;
+	int ret;
+
+	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
+		return -EFAULT;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* copy PEK certificate blobs from userspace */
+	pek_blob = psp_copy_user_blob(input.pek_cert_address, input.pek_cert_len);
+	if (IS_ERR(pek_blob)) {
+		ret = PTR_ERR(pek_blob);
+		goto e_free;
+	}
+
+	data->pek_cert_address = __psp_pa(pek_blob);
+	data->pek_cert_len = input.pek_cert_len;
+
+	/* copy PEK certificate blobs from userspace */
+	oca_blob = psp_copy_user_blob(input.oca_cert_address, input.oca_cert_len);
+	if (IS_ERR(oca_blob)) {
+		ret = PTR_ERR(oca_blob);
+		goto e_free_pek;
+	}
+
+	data->oca_cert_address = __psp_pa(oca_blob);
+	data->oca_cert_len = input.oca_cert_len;
+
+	/* If platform is not in INIT state then transition it to INIT */
+	if (psp_master->sev_state != SEV_STATE_INIT) {
+		ret = __sev_platform_init_locked(&argp->error);
+		if (ret)
+			goto e_free_oca;
+	}
+
+	ret = __sev_do_cmd_locked(SEV_CMD_PEK_CERT_IMPORT, data, &argp->error);
+
+e_free_oca:
+	kfree(oca_blob);
+e_free_pek:
+	kfree(pek_blob);
+e_free:
+	kfree(data);
+	return ret;
+}
+
+static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp)
+{
+	struct sev_user_data_pdh_cert_export input;
+	void *pdh_blob = NULL, *cert_blob = NULL;
+	struct sev_data_pdh_cert_export *data;
+	int ret;
+
+	if (copy_from_user(&input, (void __user *)argp->data, sizeof(input)))
+		return -EFAULT;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* Userspace wants to query the certificate length. */
+	if (!input.pdh_cert_address ||
+	    !input.pdh_cert_len ||
+	    !input.cert_chain_address)
+		goto cmd;
+
+	/* Allocate a physically contiguous buffer to store the PDH blob. */
+	if ((input.pdh_cert_len > SEV_FW_BLOB_MAX_SIZE) ||
+	    !access_ok(VERIFY_WRITE, input.pdh_cert_address, input.pdh_cert_len)) {
+		ret = -EFAULT;
+		goto e_free;
+	}
+
+	/* Allocate a physically contiguous buffer to store the cert chain blob. */
+	if ((input.cert_chain_len > SEV_FW_BLOB_MAX_SIZE) ||
+	    !access_ok(VERIFY_WRITE, input.cert_chain_address, input.cert_chain_len)) {
+		ret = -EFAULT;
+		goto e_free;
+	}
+
+	pdh_blob = kmalloc(input.pdh_cert_len, GFP_KERNEL);
+	if (!pdh_blob) {
+		ret = -ENOMEM;
+		goto e_free;
+	}
+
+	data->pdh_cert_address = __psp_pa(pdh_blob);
+	data->pdh_cert_len = input.pdh_cert_len;
+
+	cert_blob = kmalloc(input.cert_chain_len, GFP_KERNEL);
+	if (!cert_blob) {
+		ret = -ENOMEM;
+		goto e_free_pdh;
+	}
+
+	data->cert_chain_address = __psp_pa(cert_blob);
+	data->cert_chain_len = input.cert_chain_len;
+
+cmd:
+	/* If platform is not in INIT state then transition it to INIT. */
+	if (psp_master->sev_state != SEV_STATE_INIT) {
+		ret = __sev_platform_init_locked(&argp->error);
+		if (ret)
+			goto e_free_cert;
+	}
+
+	ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, data, &argp->error);
+
+	/* If we query the length, FW responded with expected data. */
+	input.cert_chain_len = data->cert_chain_len;
+	input.pdh_cert_len = data->pdh_cert_len;
+
+	if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) {
+		ret = -EFAULT;
+		goto e_free_cert;
+	}
+
+	if (pdh_blob) {
+		if (copy_to_user((void __user *)input.pdh_cert_address,
+				 pdh_blob, input.pdh_cert_len)) {
+			ret = -EFAULT;
+			goto e_free_cert;
+		}
+	}
+
+	if (cert_blob) {
+		if (copy_to_user((void __user *)input.cert_chain_address,
+				 cert_blob, input.cert_chain_len))
+			ret = -EFAULT;
+	}
+
+e_free_cert:
+	kfree(cert_blob);
+e_free_pdh:
+	kfree(pdh_blob);
+e_free:
+	kfree(data);
+	return ret;
+}
+
+static long sev_ioctl(struct file *file, unsigned int ioctl, unsigned long arg)
+{
+	void __user *argp = (void __user *)arg;
+	struct sev_issue_cmd input;
+	int ret = -EFAULT;
+
+	if (!psp_master)
+		return -ENODEV;
+
+	if (ioctl != SEV_ISSUE_CMD)
+		return -EINVAL;
+
+	if (copy_from_user(&input, argp, sizeof(struct sev_issue_cmd)))
+		return -EFAULT;
+
+	if (input.cmd > SEV_MAX)
+		return -EINVAL;
+
+	mutex_lock(&sev_cmd_mutex);
+
+	switch (input.cmd) {
+
+	case SEV_FACTORY_RESET:
+		ret = sev_ioctl_do_reset(&input);
+		break;
+	case SEV_PLATFORM_STATUS:
+		ret = sev_ioctl_do_platform_status(&input);
+		break;
+	case SEV_PEK_GEN:
+		ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PEK_GEN, &input);
+		break;
+	case SEV_PDH_GEN:
+		ret = sev_ioctl_do_pek_pdh_gen(SEV_CMD_PDH_GEN, &input);
+		break;
+	case SEV_PEK_CSR:
+		ret = sev_ioctl_do_pek_csr(&input);
+		break;
+	case SEV_PEK_CERT_IMPORT:
+		ret = sev_ioctl_do_pek_import(&input);
+		break;
+	case SEV_PDH_CERT_EXPORT:
+		ret = sev_ioctl_do_pdh_export(&input);
+		break;
+	default:
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (copy_to_user(argp, &input, sizeof(struct sev_issue_cmd)))
+		ret = -EFAULT;
+out:
+	mutex_unlock(&sev_cmd_mutex);
+
+	return ret;
+}
+
+static const struct file_operations sev_fops = {
+	.owner	= THIS_MODULE,
+	.unlocked_ioctl = sev_ioctl,
+};
+
+int sev_platform_status(struct sev_user_data_status *data, int *error)
+{
+	return sev_do_cmd(SEV_CMD_PLATFORM_STATUS, data, error);
+}
+EXPORT_SYMBOL_GPL(sev_platform_status);
+
+int sev_guest_deactivate(struct sev_data_deactivate *data, int *error)
+{
+	return sev_do_cmd(SEV_CMD_DEACTIVATE, data, error);
+}
+EXPORT_SYMBOL_GPL(sev_guest_deactivate);
+
+int sev_guest_activate(struct sev_data_activate *data, int *error)
+{
+	return sev_do_cmd(SEV_CMD_ACTIVATE, data, error);
+}
+EXPORT_SYMBOL_GPL(sev_guest_activate);
+
+int sev_guest_decommission(struct sev_data_decommission *data, int *error)
+{
+	return sev_do_cmd(SEV_CMD_DECOMMISSION, data, error);
+}
+EXPORT_SYMBOL_GPL(sev_guest_decommission);
+
+int sev_guest_df_flush(int *error)
+{
+	return sev_do_cmd(SEV_CMD_DF_FLUSH, NULL, error);
+}
+EXPORT_SYMBOL_GPL(sev_guest_df_flush);
+
+static void sev_exit(struct kref *ref)
+{
+	struct sev_misc_dev *misc_dev = container_of(ref, struct sev_misc_dev, refcount);
+
+	misc_deregister(&misc_dev->misc);
+}
+
+static int sev_misc_init(struct psp_device *psp)
+{
+	struct device *dev = psp->dev;
+	int ret;
+
+	/*
+	 * SEV feature support can be detected on multiple devices but the SEV
+	 * FW commands must be issued on the master. During probe, we do not
+	 * know the master hence we create /dev/sev on the first device probe.
+	 * sev_do_cmd() finds the right master device to which to issue the
+	 * command to the firmware.
+	 */
+	if (!misc_dev) {
+		struct miscdevice *misc;
+
+		misc_dev = devm_kzalloc(dev, sizeof(*misc_dev), GFP_KERNEL);
+		if (!misc_dev)
+			return -ENOMEM;
+
+		misc = &misc_dev->misc;
+		misc->minor = MISC_DYNAMIC_MINOR;
+		misc->name = DEVICE_NAME;
+		misc->fops = &sev_fops;
+
+		ret = misc_register(misc);
+		if (ret)
+			return ret;
+
+		kref_init(&misc_dev->refcount);
+	} else {
+		kref_get(&misc_dev->refcount);
+	}
+
+	init_waitqueue_head(&psp->sev_int_queue);
+	psp->sev_misc = misc_dev;
+	dev_dbg(dev, "registered SEV device\n");
+
+	return 0;
+}
+
+static int sev_init(struct psp_device *psp)
+{
+	/* Check if device supports SEV feature */
+	if (!(ioread32(psp->io_regs + PSP_FEATURE_REG) & 1)) {
+		dev_dbg(psp->dev, "device does not support SEV\n");
+		return 1;
+	}
+
+	return sev_misc_init(psp);
+}
+
+int psp_dev_init(struct sp_device *sp)
+{
+	struct device *dev = sp->dev;
+	struct psp_device *psp;
+	int ret;
+
+	ret = -ENOMEM;
+	psp = psp_alloc_struct(sp);
+	if (!psp)
+		goto e_err;
+
+	sp->psp_data = psp;
+
+	psp->vdata = (struct psp_vdata *)sp->dev_vdata->psp_vdata;
+	if (!psp->vdata) {
+		ret = -ENODEV;
+		dev_err(dev, "missing driver data\n");
+		goto e_err;
+	}
+
+	psp->io_regs = sp->io_map + psp->vdata->offset;
+
+	/* Disable and clear interrupts until ready */
+	iowrite32(0, psp->io_regs + PSP_P2CMSG_INTEN);
+	iowrite32(-1, psp->io_regs + PSP_P2CMSG_INTSTS);
+
+	/* Request an irq */
+	ret = sp_request_psp_irq(psp->sp, psp_irq_handler, psp->name, psp);
+	if (ret) {
+		dev_err(dev, "psp: unable to allocate an IRQ\n");
+		goto e_err;
+	}
+
+	ret = sev_init(psp);
+	if (ret)
+		goto e_irq;
+
+	if (sp->set_psp_master_device)
+		sp->set_psp_master_device(sp);
+
+	/* Enable interrupt */
+	iowrite32(-1, psp->io_regs + PSP_P2CMSG_INTEN);
+
+	return 0;
+
+e_irq:
+	sp_free_psp_irq(psp->sp, psp);
+e_err:
+	sp->psp_data = NULL;
+
+	dev_notice(dev, "psp initialization failed\n");
+
+	return ret;
+}
+
+void psp_dev_destroy(struct sp_device *sp)
+{
+	struct psp_device *psp = sp->psp_data;
+
+	if (psp->sev_misc)
+		kref_put(&misc_dev->refcount, sev_exit);
+
+	sp_free_psp_irq(sp, psp);
+}
+
+int sev_issue_cmd_external_user(struct file *filep, unsigned int cmd,
+				void *data, int *error)
+{
+	if (!filep || filep->f_op != &sev_fops)
+		return -EBADF;
+
+	return  sev_do_cmd(cmd, data, error);
+}
+EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
+
+void psp_pci_init(void)
+{
+	struct sev_user_data_status *status;
+	struct sp_device *sp;
+	int error, rc;
+
+	sp = sp_get_psp_master_device();
+	if (!sp)
+		return;
+
+	psp_master = sp->psp_data;
+
+	/* Initialize the platform */
+	rc = sev_platform_init(&error);
+	if (rc) {
+		dev_err(sp->dev, "SEV: failed to INIT error %#x\n", error);
+		goto err;
+	}
+
+	/* Display SEV firmware version */
+	status = &psp_master->status_cmd_buf;
+	rc = sev_platform_status(status, &error);
+	if (rc) {
+		dev_err(sp->dev, "SEV: failed to get status error %#x\n", error);
+		goto err;
+	}
+
+	dev_info(sp->dev, "SEV API:%d.%d build:%d\n", status->api_major,
+		 status->api_minor, status->build);
+	return;
+
+err:
+	psp_master = NULL;
+}
+
+void psp_pci_exit(void)
+{
+	if (!psp_master)
+		return;
+
+	sev_platform_shutdown(NULL);
+}
diff --git a/drivers/crypto/ccp/psp-dev.h b/drivers/crypto/ccp/psp-dev.h
new file mode 100644
index 000000000000..c81f0b11287a
--- /dev/null
+++ b/drivers/crypto/ccp/psp-dev.h
@@ -0,0 +1,83 @@
+/*
+ * AMD Platform Security Processor (PSP) interface driver
+ *
+ * Copyright (C) 2017 Advanced Micro Devices, Inc.
+ *
+ * Author: Brijesh Singh <brijesh.singh@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef __PSP_DEV_H__
+#define __PSP_DEV_H__
+
+#include <linux/device.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/wait.h>
+#include <linux/dmapool.h>
+#include <linux/hw_random.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/irqreturn.h>
+#include <linux/dmaengine.h>
+#include <linux/psp-sev.h>
+#include <linux/miscdevice.h>
+
+#include "sp-dev.h"
+
+#define PSP_C2PMSG(_num)		((_num) << 2)
+#define PSP_CMDRESP			PSP_C2PMSG(32)
+#define PSP_CMDBUFF_ADDR_LO		PSP_C2PMSG(56)
+#define PSP_CMDBUFF_ADDR_HI             PSP_C2PMSG(57)
+#define PSP_FEATURE_REG			PSP_C2PMSG(63)
+
+#define PSP_P2CMSG(_num)		((_num) << 2)
+#define PSP_CMD_COMPLETE_REG		1
+#define PSP_CMD_COMPLETE		PSP_P2CMSG(PSP_CMD_COMPLETE_REG)
+
+#define PSP_P2CMSG_INTEN		0x0110
+#define PSP_P2CMSG_INTSTS		0x0114
+
+#define PSP_C2PMSG_ATTR_0		0x0118
+#define PSP_C2PMSG_ATTR_1		0x011c
+#define PSP_C2PMSG_ATTR_2		0x0120
+#define PSP_C2PMSG_ATTR_3		0x0124
+#define PSP_P2CMSG_ATTR_0		0x0128
+
+#define PSP_CMDRESP_CMD_SHIFT		16
+#define PSP_CMDRESP_IOC			BIT(0)
+#define PSP_CMDRESP_RESP		BIT(31)
+#define PSP_CMDRESP_ERR_MASK		0xffff
+
+#define MAX_PSP_NAME_LEN		16
+
+struct sev_misc_dev {
+	struct kref refcount;
+	struct miscdevice misc;
+};
+
+struct psp_device {
+	struct list_head entry;
+
+	struct psp_vdata *vdata;
+	char name[MAX_PSP_NAME_LEN];
+
+	struct device *dev;
+	struct sp_device *sp;
+
+	void __iomem *io_regs;
+
+	int sev_state;
+	unsigned int sev_int_rcvd;
+	wait_queue_head_t sev_int_queue;
+	struct sev_misc_dev *sev_misc;
+	struct sev_user_data_status status_cmd_buf;
+	struct sev_data_init init_cmd_buf;
+};
+
+#endif /* __PSP_DEV_H */
diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c
index bef387c8abfd..eb0da6572720 100644
--- a/drivers/crypto/ccp/sp-dev.c
+++ b/drivers/crypto/ccp/sp-dev.c
@@ -198,6 +198,8 @@ int sp_init(struct sp_device *sp)
 	if (sp->dev_vdata->ccp_vdata)
 		ccp_dev_init(sp);
 
+	if (sp->dev_vdata->psp_vdata)
+		psp_dev_init(sp);
 	return 0;
 }
 
@@ -206,6 +208,9 @@ void sp_destroy(struct sp_device *sp)
 	if (sp->dev_vdata->ccp_vdata)
 		ccp_dev_destroy(sp);
 
+	if (sp->dev_vdata->psp_vdata)
+		psp_dev_destroy(sp);
+
 	sp_del_device(sp);
 }
 
@@ -237,6 +242,27 @@ int sp_resume(struct sp_device *sp)
 }
 #endif
 
+struct sp_device *sp_get_psp_master_device(void)
+{
+	struct sp_device *i, *ret = NULL;
+	unsigned long flags;
+
+	write_lock_irqsave(&sp_unit_lock, flags);
+	if (list_empty(&sp_units))
+		goto unlock;
+
+	list_for_each_entry(i, &sp_units, entry) {
+		if (i->psp_data)
+			break;
+	}
+
+	if (i->get_psp_master_device)
+		ret = i->get_psp_master_device();
+unlock:
+	write_unlock_irqrestore(&sp_unit_lock, flags);
+	return ret;
+}
+
 static int __init sp_mod_init(void)
 {
 #ifdef CONFIG_X86
@@ -246,6 +272,10 @@ static int __init sp_mod_init(void)
 	if (ret)
 		return ret;
 
+#ifdef CONFIG_CRYPTO_DEV_SP_PSP
+	psp_pci_init();
+#endif
+
 	return 0;
 #endif
 
@@ -265,6 +295,11 @@ static int __init sp_mod_init(void)
 static void __exit sp_mod_exit(void)
 {
 #ifdef CONFIG_X86
+
+#ifdef CONFIG_CRYPTO_DEV_SP_PSP
+	psp_pci_exit();
+#endif
+
 	sp_pci_exit();
 #endif
 
diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h
index 5ab486ade1ad..acb197b66ced 100644
--- a/drivers/crypto/ccp/sp-dev.h
+++ b/drivers/crypto/ccp/sp-dev.h
@@ -42,12 +42,17 @@ struct ccp_vdata {
 	const unsigned int offset;
 	const unsigned int rsamax;
 };
+
+struct psp_vdata {
+	const unsigned int offset;
+};
+
 /* Structure to hold SP device data */
 struct sp_dev_vdata {
 	const unsigned int bar;
 
 	const struct ccp_vdata *ccp_vdata;
-	void *psp_vdata;
+	const struct psp_vdata *psp_vdata;
 };
 
 struct sp_device {
@@ -68,6 +73,10 @@ struct sp_device {
 	/* DMA caching attribute support */
 	unsigned int axcache;
 
+	/* get and set master device */
+	struct sp_device*(*get_psp_master_device)(void);
+	void (*set_psp_master_device)(struct sp_device *);
+
 	bool irq_registered;
 	bool use_tasklet;
 
@@ -103,6 +112,7 @@ void sp_free_ccp_irq(struct sp_device *sp, void *data);
 int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler,
 		       const char *name, void *data);
 void sp_free_psp_irq(struct sp_device *sp, void *data);
+struct sp_device *sp_get_psp_master_device(void);
 
 #ifdef CONFIG_CRYPTO_DEV_SP_CCP
 
@@ -130,4 +140,20 @@ static inline int ccp_dev_resume(struct sp_device *sp)
 }
 #endif	/* CONFIG_CRYPTO_DEV_SP_CCP */
 
+#ifdef CONFIG_CRYPTO_DEV_SP_PSP
+
+int psp_dev_init(struct sp_device *sp);
+void psp_pci_init(void);
+void psp_dev_destroy(struct sp_device *sp);
+void psp_pci_exit(void);
+
+#else /* !CONFIG_CRYPTO_DEV_SP_PSP */
+
+static inline int psp_dev_init(struct sp_device *sp) { return 0; }
+static inline void psp_pci_init(void) { }
+static inline void psp_dev_destroy(struct sp_device *sp) { }
+static inline void psp_pci_exit(void) { }
+
+#endif /* CONFIG_CRYPTO_DEV_SP_PSP */
+
 #endif
diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c
index 9859aa683a28..f5f43c50698a 100644
--- a/drivers/crypto/ccp/sp-pci.c
+++ b/drivers/crypto/ccp/sp-pci.c
@@ -25,6 +25,7 @@
 #include <linux/ccp.h>
 
 #include "ccp-dev.h"
+#include "psp-dev.h"
 
 #define MSIX_VECTORS			2
 
@@ -32,6 +33,7 @@ struct sp_pci {
 	int msix_count;
 	struct msix_entry msix_entry[MSIX_VECTORS];
 };
+static struct sp_device *sp_dev_master;
 
 static int sp_get_msix_irqs(struct sp_device *sp)
 {
@@ -108,6 +110,45 @@ static void sp_free_irqs(struct sp_device *sp)
 	sp->psp_irq = 0;
 }
 
+static bool sp_pci_is_master(struct sp_device *sp)
+{
+	struct device *dev_cur, *dev_new;
+	struct pci_dev *pdev_cur, *pdev_new;
+
+	dev_new = sp->dev;
+	dev_cur = sp_dev_master->dev;
+
+	pdev_new = to_pci_dev(dev_new);
+	pdev_cur = to_pci_dev(dev_cur);
+
+	if (pdev_new->bus->number < pdev_cur->bus->number)
+		return true;
+
+	if (PCI_SLOT(pdev_new->devfn) < PCI_SLOT(pdev_cur->devfn))
+		return true;
+
+	if (PCI_FUNC(pdev_new->devfn) < PCI_FUNC(pdev_cur->devfn))
+		return true;
+
+	return false;
+}
+
+static void psp_set_master(struct sp_device *sp)
+{
+	if (!sp_dev_master) {
+		sp_dev_master = sp;
+		return;
+	}
+
+	if (sp_pci_is_master(sp))
+		sp_dev_master = sp;
+}
+
+static struct sp_device *psp_get_master(void)
+{
+	return sp_dev_master;
+}
+
 static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	struct sp_device *sp;
@@ -166,6 +207,8 @@ static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto e_err;
 
 	pci_set_master(pdev);
+	sp->set_psp_master_device = psp_set_master;
+	sp->get_psp_master_device = psp_get_master;
 
 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
 	if (ret) {
@@ -225,6 +268,12 @@ static int sp_pci_resume(struct pci_dev *pdev)
 }
 #endif
 
+#ifdef CONFIG_CRYPTO_DEV_SP_PSP
+static const struct psp_vdata psp_entry = {
+	.offset = 0x10500,
+};
+#endif
+
 static const struct sp_dev_vdata dev_vdata[] = {
 	{
 		.bar = 2,
@@ -236,6 +285,9 @@ static const struct sp_dev_vdata dev_vdata[] = {
 		.bar = 2,
 #ifdef CONFIG_CRYPTO_DEV_SP_CCP
 		.ccp_vdata = &ccpv5a,
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SP_PSP
+		.psp_vdata = &psp_entry
 #endif
 	},
 	{
diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig
index b56b3f711d94..5ae9f8706f17 100644
--- a/drivers/crypto/chelsio/Kconfig
+++ b/drivers/crypto/chelsio/Kconfig
@@ -19,3 +19,13 @@ config CRYPTO_DEV_CHELSIO
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called chcr.
+
+config CHELSIO_IPSEC_INLINE
+        bool "Chelsio IPSec XFRM Tx crypto offload"
+        depends on CHELSIO_T4
+	depends on CRYPTO_DEV_CHELSIO
+        depends on XFRM_OFFLOAD
+        depends on INET_ESP_OFFLOAD || INET6_ESP_OFFLOAD
+        default n
+        ---help---
+          Enable support for IPSec Tx Inline.
diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile
index bebdf06687ad..eaecaf1ebcf3 100644
--- a/drivers/crypto/chelsio/Makefile
+++ b/drivers/crypto/chelsio/Makefile
@@ -2,3 +2,4 @@ ccflags-y := -Idrivers/net/ethernet/chelsio/cxgb4
 
 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o
 chcr-objs :=  chcr_core.o chcr_algo.o
+chcr-$(CONFIG_CHELSIO_IPSEC_INLINE) += chcr_ipsec.o
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 4eed7171e2ae..34a02d690548 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -73,6 +73,29 @@
 
 #define IV AES_BLOCK_SIZE
 
+static unsigned int sgl_ent_len[] = {
+	0, 0, 16, 24, 40, 48, 64, 72, 88,
+	96, 112, 120, 136, 144, 160, 168, 184,
+	192, 208, 216, 232, 240, 256, 264, 280,
+	288, 304, 312, 328, 336, 352, 360, 376
+};
+
+static unsigned int dsgl_ent_len[] = {
+	0, 32, 32, 48, 48, 64, 64, 80, 80,
+	112, 112, 128, 128, 144, 144, 160, 160,
+	192, 192, 208, 208, 224, 224, 240, 240,
+	272, 272, 288, 288, 304, 304, 320, 320
+};
+
+static u32 round_constant[11] = {
+	0x01000000, 0x02000000, 0x04000000, 0x08000000,
+	0x10000000, 0x20000000, 0x40000000, 0x80000000,
+	0x1B000000, 0x36000000, 0x6C000000
+};
+
+static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
+				   unsigned char *input, int err);
+
 static inline  struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
 {
 	return ctx->crypto_ctx->aeadctx;
@@ -108,18 +131,6 @@ static inline int is_ofld_imm(const struct sk_buff *skb)
 	return (skb->len <= SGE_MAX_WR_LEN);
 }
 
-/*
- *	sgl_len - calculates the size of an SGL of the given capacity
- *	@n: the number of SGL entries
- *	Calculates the number of flits needed for a scatter/gather list that
- *	can hold the given number of entries.
- */
-static inline unsigned int sgl_len(unsigned int n)
-{
-	n--;
-	return (3 * n) / 2 + (n & 1) + 2;
-}
-
 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
 			 unsigned int entlen,
 			 unsigned int skip)
@@ -160,7 +171,6 @@ static inline void chcr_handle_ahash_resp(struct ahash_request *req,
 
 	if (input == NULL)
 		goto out;
-	reqctx = ahash_request_ctx(req);
 	digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
 	if (reqctx->is_sg_map)
 		chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
@@ -183,30 +193,17 @@ static inline void chcr_handle_ahash_resp(struct ahash_request *req,
 	}
 out:
 	req->base.complete(&req->base, err);
+}
 
-	}
-
-static inline void chcr_handle_aead_resp(struct aead_request *req,
-					 unsigned char *input,
-					 int err)
+static inline int get_aead_subtype(struct crypto_aead *aead)
 {
-	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
-	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-	struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
-
-
-	chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
-	if (reqctx->b0_dma)
-		dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
-				 reqctx->b0_len, DMA_BIDIRECTIONAL);
-	if (reqctx->verify == VERIFY_SW) {
-		chcr_verify_tag(req, input, &err);
-		reqctx->verify = VERIFY_HW;
+	struct aead_alg *alg = crypto_aead_alg(aead);
+	struct chcr_alg_template *chcr_crypto_alg =
+		container_of(alg, struct chcr_alg_template, alg.aead);
+	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
 }
-	req->base.complete(&req->base, err);
 
-}
-static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
+void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
 {
 	u8 temp[SHA512_DIGEST_SIZE];
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@@ -231,6 +228,25 @@ static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
 		*err = 0;
 }
 
+static inline void chcr_handle_aead_resp(struct aead_request *req,
+					 unsigned char *input,
+					 int err)
+{
+	struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
+
+	chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
+	if (reqctx->b0_dma)
+		dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
+				 reqctx->b0_len, DMA_BIDIRECTIONAL);
+	if (reqctx->verify == VERIFY_SW) {
+		chcr_verify_tag(req, input, &err);
+		reqctx->verify = VERIFY_HW;
+	}
+	req->base.complete(&req->base, err);
+}
+
 /*
  *	chcr_handle_resp - Unmap the DMA buffers associated with the request
  *	@req: crypto request
@@ -558,7 +574,8 @@ static void  ulptx_walk_add_sg(struct ulptx_walk *walk,
 			skip = 0;
 		}
 	}
-	if (walk->nents == 0) {
+	WARN(!sg, "SG should not be null here\n");
+	if (sg && (walk->nents == 0)) {
 		small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
 		sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
 		walk->sgl->len0 = cpu_to_be32(sgmin);
@@ -595,14 +612,6 @@ static void  ulptx_walk_add_sg(struct ulptx_walk *walk,
 	}
 }
 
-static inline int get_aead_subtype(struct crypto_aead *aead)
-{
-	struct aead_alg *alg = crypto_aead_alg(aead);
-	struct chcr_alg_template *chcr_crypto_alg =
-		container_of(alg, struct chcr_alg_template, alg.aead);
-	return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
-}
-
 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
 {
 	struct crypto_alg *alg = tfm->__crt_alg;
@@ -675,7 +684,7 @@ static int chcr_sg_ent_in_wr(struct scatterlist *src,
 			if (srclen <= dstlen)
 				break;
 			less = min_t(unsigned int, sg_dma_len(dst) - offset -
-				dstskip, CHCR_DST_SG_SIZE);
+				     dstskip, CHCR_DST_SG_SIZE);
 			dstlen += less;
 			offset += less;
 			if (offset == sg_dma_len(dst)) {
@@ -686,7 +695,7 @@ static int chcr_sg_ent_in_wr(struct scatterlist *src,
 			dstskip = 0;
 		}
 		src = sg_next(src);
-		 srcskip = 0;
+		srcskip = 0;
 	}
 	return min(srclen, dstlen);
 }
@@ -1008,7 +1017,8 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
 	return bytes;
 }
 
-static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
+static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
+			     u32 isfinal)
 {
 	struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 	struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
@@ -1035,7 +1045,8 @@ static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
 	for (i = 0; i < (round % 8); i++)
 		gf128mul_x_ble((le128 *)iv, (le128 *)iv);
 
-	crypto_cipher_decrypt_one(cipher, iv, iv);
+	if (!isfinal)
+		crypto_cipher_decrypt_one(cipher, iv, iv);
 out:
 	return ret;
 }
@@ -1056,7 +1067,7 @@ static int chcr_update_cipher_iv(struct ablkcipher_request *req,
 			CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
 						AES_BLOCK_SIZE) + 1);
 	else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
-		ret = chcr_update_tweak(req, iv);
+		ret = chcr_update_tweak(req, iv, 0);
 	else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
 		if (reqctx->op)
 			sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
@@ -1087,7 +1098,7 @@ static int chcr_final_cipher_iv(struct ablkcipher_request *req,
 		ctr_add_iv(iv, req->info, (reqctx->processed /
 			   AES_BLOCK_SIZE));
 	else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
-		ret = chcr_update_tweak(req, iv);
+		ret = chcr_update_tweak(req, iv, 1);
 	else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
 		if (reqctx->op)
 			sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
@@ -1101,7 +1112,6 @@ static int chcr_final_cipher_iv(struct ablkcipher_request *req,
 
 }
 
-
 static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
 				   unsigned char *input, int err)
 {
@@ -1135,10 +1145,10 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
 		bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1,
 					  SPACE_LEFT(ablkctx->enckey_len),
 					  reqctx->src_ofst, reqctx->dst_ofst);
-	if ((bytes + reqctx->processed) >= req->nbytes)
-		bytes  = req->nbytes - reqctx->processed;
-	else
-		bytes = ROUND_16(bytes);
+		if ((bytes + reqctx->processed) >= req->nbytes)
+			bytes  = req->nbytes - reqctx->processed;
+		else
+			bytes = ROUND_16(bytes);
 	} else {
 		/*CTR mode counter overfloa*/
 		bytes  = req->nbytes - reqctx->processed;
@@ -1239,15 +1249,15 @@ static int process_cipher(struct ablkcipher_request *req,
 					  MIN_CIPHER_SG,
 					  SPACE_LEFT(ablkctx->enckey_len),
 					  0, 0);
-	if ((bytes + reqctx->processed) >= req->nbytes)
-		bytes  = req->nbytes - reqctx->processed;
-	else
-		bytes = ROUND_16(bytes);
+		if ((bytes + reqctx->processed) >= req->nbytes)
+			bytes  = req->nbytes - reqctx->processed;
+		else
+			bytes = ROUND_16(bytes);
 	} else {
 		bytes = req->nbytes;
 	}
 	if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
-				  CRYPTO_ALG_SUB_TYPE_CTR) {
+	    CRYPTO_ALG_SUB_TYPE_CTR) {
 		bytes = adjust_ctr_overflow(req->info, bytes);
 	}
 	if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
@@ -2014,11 +2024,8 @@ static int chcr_aead_common_init(struct aead_request *req,
 	struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
 	int error = -EINVAL;
-	unsigned int dst_size;
 	unsigned int authsize = crypto_aead_authsize(tfm);
 
-	dst_size = req->assoclen + req->cryptlen + (op_type ?
-					-authsize : authsize);
 	/* validate key size */
 	if (aeadctx->enckey_len == 0)
 		goto err;
@@ -2083,7 +2090,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 	struct cpl_rx_phys_dsgl *phys_cpl;
 	struct ulptx_sgl *ulptx;
 	unsigned int transhdr_len;
-	unsigned int dst_size = 0, temp;
+	unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm);
 	unsigned int   kctx_len = 0, dnents;
 	unsigned int  assoclen = req->assoclen;
 	unsigned int  authsize = crypto_aead_authsize(tfm);
@@ -2097,24 +2104,19 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 		return NULL;
 
 	reqctx->b0_dma = 0;
-	if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
+	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
+	subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
 		null = 1;
 		assoclen = 0;
 	}
-	dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
-						    authsize);
 	error = chcr_aead_common_init(req, op_type);
 	if (error)
 		return ERR_PTR(error);
-	if (dst_size) {
-		dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
-		dnents += sg_nents_xlen(req->dst, req->cryptlen +
-			(op_type ? -authsize : authsize), CHCR_DST_SG_SIZE,
-			req->assoclen);
-		dnents += MIN_AUTH_SG; // For IV
-	} else {
-		dnents = 0;
-	}
+	dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
+	dnents += sg_nents_xlen(req->dst, req->cryptlen +
+		(op_type ? -authsize : authsize), CHCR_DST_SG_SIZE,
+		req->assoclen);
+	dnents += MIN_AUTH_SG; // For IV
 
 	dst_size = get_space_for_phys_dsgl(dnents);
 	kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
@@ -2162,16 +2164,23 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 					temp & 0xF,
 					null ? 0 : assoclen + IV + 1,
 					temp, temp);
+	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
+	    subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
+		temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
+	else
+		temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
 	chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
 					(op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
-					CHCR_SCMD_CIPHER_MODE_AES_CBC,
+					temp,
 					actx->auth_mode, aeadctx->hmac_ctrl,
 					IV >> 1);
 	chcr_req->sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
 					 0, 0, dst_size);
 
 	chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
-	if (op_type == CHCR_ENCRYPT_OP)
+	if (op_type == CHCR_ENCRYPT_OP ||
+		subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
+		subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
 		memcpy(chcr_req->key_ctx.key, aeadctx->key,
 		       aeadctx->enckey_len);
 	else
@@ -2181,7 +2190,16 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 	memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
 					4), actx->h_iopad, kctx_len -
 				(DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
-	memcpy(reqctx->iv, req->iv, IV);
+	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
+	    subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
+		memcpy(reqctx->iv, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
+		memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
+				CTR_RFC3686_IV_SIZE);
+		*(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
+			CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
+	} else {
+		memcpy(reqctx->iv, req->iv, IV);
+	}
 	phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
 	ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
 	chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
@@ -2202,9 +2220,9 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
 	return ERR_PTR(error);
 }
 
-static int chcr_aead_dma_map(struct device *dev,
-			     struct aead_request *req,
-			     unsigned short op_type)
+int chcr_aead_dma_map(struct device *dev,
+		      struct aead_request *req,
+		      unsigned short op_type)
 {
 	int error;
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
@@ -2246,9 +2264,9 @@ static int chcr_aead_dma_map(struct device *dev,
 	return -ENOMEM;
 }
 
-static void chcr_aead_dma_unmap(struct device *dev,
-			     struct aead_request *req,
-			     unsigned short op_type)
+void chcr_aead_dma_unmap(struct device *dev,
+			 struct aead_request *req,
+			 unsigned short op_type)
 {
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@@ -2273,10 +2291,10 @@ static void chcr_aead_dma_unmap(struct device *dev,
 	}
 }
 
-static inline void chcr_add_aead_src_ent(struct aead_request *req,
-			       struct ulptx_sgl *ulptx,
-			       unsigned int assoclen,
-			       unsigned short op_type)
+void chcr_add_aead_src_ent(struct aead_request *req,
+			   struct ulptx_sgl *ulptx,
+			   unsigned int assoclen,
+			   unsigned short op_type)
 {
 	struct ulptx_walk ulp_walk;
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
@@ -2308,11 +2326,11 @@ static inline void chcr_add_aead_src_ent(struct aead_request *req,
 	}
 }
 
-static inline void chcr_add_aead_dst_ent(struct aead_request *req,
-			       struct cpl_rx_phys_dsgl *phys_cpl,
-			       unsigned int assoclen,
-			       unsigned short op_type,
-			       unsigned short qid)
+void chcr_add_aead_dst_ent(struct aead_request *req,
+			   struct cpl_rx_phys_dsgl *phys_cpl,
+			   unsigned int assoclen,
+			   unsigned short op_type,
+			   unsigned short qid)
 {
 	struct chcr_aead_reqctx  *reqctx = aead_request_ctx(req);
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@@ -2330,9 +2348,9 @@ static inline void chcr_add_aead_dst_ent(struct aead_request *req,
 	dsgl_walk_end(&dsgl_walk, qid);
 }
 
-static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
-					   struct ulptx_sgl *ulptx,
-					   struct  cipher_wr_param *wrparam)
+void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
+			     struct ulptx_sgl *ulptx,
+			     struct  cipher_wr_param *wrparam)
 {
 	struct ulptx_walk ulp_walk;
 	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
@@ -2355,10 +2373,10 @@ static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
 	}
 }
 
-static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
-					   struct cpl_rx_phys_dsgl *phys_cpl,
-					   struct  cipher_wr_param *wrparam,
-					   unsigned short qid)
+void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
+			     struct cpl_rx_phys_dsgl *phys_cpl,
+			     struct  cipher_wr_param *wrparam,
+			     unsigned short qid)
 {
 	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
 	struct dsgl_walk dsgl_walk;
@@ -2373,9 +2391,9 @@ static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
 	dsgl_walk_end(&dsgl_walk, qid);
 }
 
-static inline void chcr_add_hash_src_ent(struct ahash_request *req,
-					   struct ulptx_sgl *ulptx,
-					   struct hash_wr_param *param)
+void chcr_add_hash_src_ent(struct ahash_request *req,
+			   struct ulptx_sgl *ulptx,
+			   struct hash_wr_param *param)
 {
 	struct ulptx_walk ulp_walk;
 	struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
@@ -2395,16 +2413,13 @@ static inline void chcr_add_hash_src_ent(struct ahash_request *req,
 			ulptx_walk_add_page(&ulp_walk, param->bfr_len,
 					    &reqctx->dma_addr);
 		ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len,
-					  0);
-//	       reqctx->srcsg = ulp_walk.last_sg;
-//	       reqctx->src_ofst = ulp_walk.last_sg_len;
-			ulptx_walk_end(&ulp_walk);
+				  0);
+		ulptx_walk_end(&ulp_walk);
 	}
 }
 
-
-static inline int chcr_hash_dma_map(struct device *dev,
-			     struct ahash_request *req)
+int chcr_hash_dma_map(struct device *dev,
+		      struct ahash_request *req)
 {
 	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
 	int error = 0;
@@ -2414,13 +2429,13 @@ static inline int chcr_hash_dma_map(struct device *dev,
 	error = dma_map_sg(dev, req->src, sg_nents(req->src),
 			   DMA_TO_DEVICE);
 	if (!error)
-		return error;
+		return -ENOMEM;
 	req_ctx->is_sg_map = 1;
 	return 0;
 }
 
-static inline void chcr_hash_dma_unmap(struct device *dev,
-			     struct ahash_request *req)
+void chcr_hash_dma_unmap(struct device *dev,
+			 struct ahash_request *req)
 {
 	struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
 
@@ -2433,9 +2448,8 @@ static inline void chcr_hash_dma_unmap(struct device *dev,
 
 }
 
-
-static int chcr_cipher_dma_map(struct device *dev,
-			     struct ablkcipher_request *req)
+int chcr_cipher_dma_map(struct device *dev,
+			struct ablkcipher_request *req)
 {
 	int error;
 	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
@@ -2469,8 +2483,9 @@ static int chcr_cipher_dma_map(struct device *dev,
 	dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
 	return -ENOMEM;
 }
-static void chcr_cipher_dma_unmap(struct device *dev,
-				  struct ablkcipher_request *req)
+
+void chcr_cipher_dma_unmap(struct device *dev,
+			   struct ablkcipher_request *req)
 {
 	struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
 
@@ -2666,8 +2681,6 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 	sub_type = get_aead_subtype(tfm);
 	if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
 		assoclen -= 8;
-	dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
-						   authsize);
 	error = chcr_aead_common_init(req, op_type);
 	if (error)
 		return ERR_PTR(error);
@@ -2677,15 +2690,11 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
 	error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
 	if (error)
 		goto err;
-	if (dst_size) {
-		dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
-		dnents += sg_nents_xlen(req->dst, req->cryptlen
-				+ (op_type ? -authsize : authsize),
-				CHCR_DST_SG_SIZE, req->assoclen);
-		dnents += MIN_CCM_SG; // For IV and B0
-	} else {
-		dnents = 0;
-	}
+	dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
+	dnents += sg_nents_xlen(req->dst, req->cryptlen
+			+ (op_type ? -authsize : authsize),
+			CHCR_DST_SG_SIZE, req->assoclen);
+	dnents += MIN_CCM_SG; // For IV and B0
 	dst_size = get_space_for_phys_dsgl(dnents);
 	kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
 	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
@@ -2780,19 +2789,14 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 		assoclen = req->assoclen - 8;
 
 	reqctx->b0_dma = 0;
-	dst_size = assoclen + req->cryptlen + (op_type ? -authsize :  authsize);
 	error = chcr_aead_common_init(req, op_type);
-		if (error)
-			return	ERR_PTR(error);
-	if (dst_size) {
-		dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
-		dnents += sg_nents_xlen(req->dst,
-			req->cryptlen + (op_type ? -authsize : authsize),
+	if (error)
+		return ERR_PTR(error);
+	dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
+	dnents += sg_nents_xlen(req->dst, req->cryptlen +
+				(op_type ? -authsize : authsize),
 				CHCR_DST_SG_SIZE, req->assoclen);
-		dnents += MIN_GCM_SG; // For IV
-	} else {
-		dnents = 0;
-	}
+	dnents += MIN_GCM_SG; // For IV
 	dst_size = get_space_for_phys_dsgl(dnents);
 	kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
 		AEAD_H_SIZE;
@@ -2829,10 +2833,10 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
 	chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
 					assoclen ? 1 : 0, assoclen,
 					assoclen + IV + 1, 0);
-		chcr_req->sec_cpl.cipherstop_lo_authinsert =
+	chcr_req->sec_cpl.cipherstop_lo_authinsert =
 			FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1,
 						temp, temp);
-		chcr_req->sec_cpl.seqno_numivs =
+	chcr_req->sec_cpl.seqno_numivs =
 			FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
 					CHCR_ENCRYPT_OP) ? 1 : 0,
 					CHCR_SCMD_CIPHER_MODE_AES_GCM,
@@ -3212,7 +3216,7 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
 	struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
 	/* it contains auth and cipher key both*/
 	struct crypto_authenc_keys keys;
-	unsigned int bs;
+	unsigned int bs, subtype;
 	unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
 	int err = 0, i, key_ctx_len = 0;
 	unsigned char ck_size = 0;
@@ -3241,6 +3245,15 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
 		pr_err("chcr : Unsupported digest size\n");
 		goto out;
 	}
+	subtype = get_aead_subtype(authenc);
+	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
+		subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
+		if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
+			goto out;
+		memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
+		- CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
+		keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
+	}
 	if (keys.enckeylen == AES_KEYSIZE_128) {
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
 	} else if (keys.enckeylen == AES_KEYSIZE_192) {
@@ -3258,9 +3271,12 @@ static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
 	 */
 	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
 	aeadctx->enckey_len = keys.enckeylen;
-	get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
-			    aeadctx->enckey_len << 3);
+	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
+		subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
 
+		get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
+			    aeadctx->enckey_len << 3);
+	}
 	base_hash  = chcr_alloc_shash(max_authsize);
 	if (IS_ERR(base_hash)) {
 		pr_err("chcr : Base driver cannot be loaded\n");
@@ -3333,6 +3349,7 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
 	struct crypto_authenc_keys keys;
 	int err;
 	/* it contains auth and cipher key both*/
+	unsigned int subtype;
 	int key_ctx_len = 0;
 	unsigned char ck_size = 0;
 
@@ -3350,6 +3367,15 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
 		crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 		goto out;
 	}
+	subtype = get_aead_subtype(authenc);
+	if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
+	    subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
+		if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
+			goto out;
+		memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
+			- CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
+		keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
+	}
 	if (keys.enckeylen == AES_KEYSIZE_128) {
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
 	} else if (keys.enckeylen == AES_KEYSIZE_192) {
@@ -3357,13 +3383,16 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
 	} else if (keys.enckeylen == AES_KEYSIZE_256) {
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
 	} else {
-		pr_err("chcr : Unsupported cipher key\n");
+		pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen);
 		goto out;
 	}
 	memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
 	aeadctx->enckey_len = keys.enckeylen;
-	get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
-				    aeadctx->enckey_len << 3);
+	if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
+	    subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
+		get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
+				aeadctx->enckey_len << 3);
+	}
 	key_ctx_len =  sizeof(struct _key_ctx)
 		+ ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
 
@@ -3375,6 +3404,40 @@ static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
 	aeadctx->enckey_len = 0;
 	return -EINVAL;
 }
+
+static int chcr_aead_op(struct aead_request *req,
+			unsigned short op_type,
+			int size,
+			create_wr_t create_wr_fn)
+{
+	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+	struct uld_ctx *u_ctx;
+	struct sk_buff *skb;
+
+	if (!a_ctx(tfm)->dev) {
+		pr_err("chcr : %s : No crypto device.\n", __func__);
+		return -ENXIO;
+	}
+	u_ctx = ULD_CTX(a_ctx(tfm));
+	if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
+				   a_ctx(tfm)->tx_qidx)) {
+		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
+			return -EBUSY;
+	}
+
+	/* Form a WR from req */
+	skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
+			   op_type);
+
+	if (IS_ERR(skb) || !skb)
+		return PTR_ERR(skb);
+
+	skb->dev = u_ctx->lldi.ports[0];
+	set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
+	chcr_send_wr(skb);
+	return -EINPROGRESS;
+}
+
 static int chcr_aead_encrypt(struct aead_request *req)
 {
 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
@@ -3383,8 +3446,10 @@ static int chcr_aead_encrypt(struct aead_request *req)
 	reqctx->verify = VERIFY_HW;
 
 	switch (get_aead_subtype(tfm)) {
-	case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
-	case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
+	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
+	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
+	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
+	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
 		return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
 				    create_authenc_wr);
 	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
@@ -3413,8 +3478,10 @@ static int chcr_aead_decrypt(struct aead_request *req)
 	}
 
 	switch (get_aead_subtype(tfm)) {
-	case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
-	case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
+	case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
+	case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
+	case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
+	case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
 		return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
 				    create_authenc_wr);
 	case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
@@ -3427,38 +3494,6 @@ static int chcr_aead_decrypt(struct aead_request *req)
 	}
 }
 
-static int chcr_aead_op(struct aead_request *req,
-			  unsigned short op_type,
-			  int size,
-			  create_wr_t create_wr_fn)
-{
-	struct crypto_aead *tfm = crypto_aead_reqtfm(req);
-	struct uld_ctx *u_ctx;
-	struct sk_buff *skb;
-
-	if (!a_ctx(tfm)->dev) {
-		pr_err("chcr : %s : No crypto device.\n", __func__);
-		return -ENXIO;
-	}
-	u_ctx = ULD_CTX(a_ctx(tfm));
-	if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
-				   a_ctx(tfm)->tx_qidx)) {
-		if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
-			return -EBUSY;
-	}
-
-	/* Form a WR from req */
-	skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
-			   op_type);
-
-	if (IS_ERR(skb) || !skb)
-		return PTR_ERR(skb);
-
-	skb->dev = u_ctx->lldi.ports[0];
-	set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
-	chcr_send_wr(skb);
-	return -EINPROGRESS;
-}
 static struct chcr_alg_template driver_algs[] = {
 	/* AES-CBC */
 	{
@@ -3742,7 +3777,7 @@ static struct chcr_alg_template driver_algs[] = {
 		}
 	},
 	{
-		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
 		.is_registered = 0,
 		.alg.aead = {
 			.base = {
@@ -3763,7 +3798,7 @@ static struct chcr_alg_template driver_algs[] = {
 		}
 	},
 	{
-		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
 		.is_registered = 0,
 		.alg.aead = {
 			.base = {
@@ -3785,7 +3820,7 @@ static struct chcr_alg_template driver_algs[] = {
 		}
 	},
 	{
-		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
 		.is_registered = 0,
 		.alg.aead = {
 			.base = {
@@ -3805,7 +3840,7 @@ static struct chcr_alg_template driver_algs[] = {
 		}
 	},
 	{
-		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
 		.is_registered = 0,
 		.alg.aead = {
 			.base = {
@@ -3826,7 +3861,7 @@ static struct chcr_alg_template driver_algs[] = {
 		}
 	},
 	{
-		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
 		.is_registered = 0,
 		.alg.aead = {
 			.base = {
@@ -3847,7 +3882,7 @@ static struct chcr_alg_template driver_algs[] = {
 		}
 	},
 	{
-		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
 		.is_registered = 0,
 		.alg.aead = {
 			.base = {
@@ -3867,6 +3902,133 @@ static struct chcr_alg_template driver_algs[] = {
 			.setauthsize = chcr_authenc_null_setauthsize,
 		}
 	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
+				.cra_driver_name =
+				"authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_priority = CHCR_AEAD_PRIORITY,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA1_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+
+				.cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
+				.cra_driver_name =
+				"authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_priority = CHCR_AEAD_PRIORITY,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize	= SHA256_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
+				.cra_driver_name =
+				"authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_priority = CHCR_AEAD_PRIORITY,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+			},
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA224_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
+				.cra_driver_name =
+				"authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_priority = CHCR_AEAD_PRIORITY,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA384_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
+				.cra_driver_name =
+				"authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_priority = CHCR_AEAD_PRIORITY,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = SHA512_DIGEST_SIZE,
+			.setkey = chcr_authenc_setkey,
+			.setauthsize = chcr_authenc_setauthsize,
+		}
+	},
+	{
+		.type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
+		.is_registered = 0,
+		.alg.aead = {
+			.base = {
+				.cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
+				.cra_driver_name =
+				"authenc-digest_null-rfc3686-ctr-aes-chcr",
+				.cra_blocksize	 = 1,
+				.cra_priority = CHCR_AEAD_PRIORITY,
+				.cra_ctxsize =	sizeof(struct chcr_context) +
+						sizeof(struct chcr_aead_ctx) +
+						sizeof(struct chcr_authenc_ctx),
+
+			},
+			.ivsize  = CTR_RFC3686_IV_SIZE,
+			.maxauthsize = 0,
+			.setkey  = chcr_aead_digest_null_setkey,
+			.setauthsize = chcr_authenc_null_setauthsize,
+		}
+	},
+
 };
 
 /*
diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h
index 96c9335ee728..d1673a5d4bf5 100644
--- a/drivers/crypto/chelsio/chcr_algo.h
+++ b/drivers/crypto/chelsio/chcr_algo.h
@@ -226,15 +226,6 @@
 #define SPACE_LEFT(len) \
 	((SGE_MAX_WR_LEN - WR_MIN_LEN - (len)))
 
-unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88,
-				96, 112, 120, 136, 144, 160, 168, 184,
-				192, 208, 216, 232, 240, 256, 264, 280,
-				288, 304, 312, 328, 336, 352, 360, 376};
-unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80,
-				112, 112, 128, 128, 144, 144, 160, 160,
-				192, 192, 208, 208, 224, 224, 240, 240,
-				272, 272, 288, 288, 304, 304, 320, 320};
-
 struct algo_param {
 	unsigned int auth_mode;
 	unsigned int mk_size;
@@ -404,10 +395,4 @@ static inline u32 aes_ks_subword(const u32 w)
 	return *(u32 *)(&bytes[0]);
 }
 
-static u32 round_constant[11] = {
-	0x01000000, 0x02000000, 0x04000000, 0x08000000,
-	0x10000000, 0x20000000, 0x40000000, 0x80000000,
-	0x1B000000, 0x36000000, 0x6C000000
-};
-
 #endif /* __CHCR_ALGO_H__ */
diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c
index f5a2624081dc..04f277cade7c 100644
--- a/drivers/crypto/chelsio/chcr_core.c
+++ b/drivers/crypto/chelsio/chcr_core.c
@@ -48,6 +48,9 @@ static struct cxgb4_uld_info chcr_uld_info = {
 	.add = chcr_uld_add,
 	.state_change = chcr_uld_state_change,
 	.rx_handler = chcr_uld_rx_handler,
+#ifdef CONFIG_CHELSIO_IPSEC_INLINE
+	.tx_handler = chcr_uld_tx_handler,
+#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
 };
 
 struct uld_ctx *assign_chcr_device(void)
@@ -164,6 +167,10 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld)
 		goto out;
 	}
 	u_ctx->lldi = *lld;
+#ifdef CONFIG_CHELSIO_IPSEC_INLINE
+	if (lld->crypto & ULP_CRYPTO_IPSEC_INLINE)
+		chcr_add_xfrmops(lld);
+#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
 out:
 	return u_ctx;
 }
@@ -187,6 +194,13 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
 	return 0;
 }
 
+#ifdef CONFIG_CHELSIO_IPSEC_INLINE
+int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev)
+{
+	return chcr_ipsec_xmit(skb, dev);
+}
+#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
+
 static int chcr_uld_state_change(void *handle, enum cxgb4_state state)
 {
 	struct uld_ctx *u_ctx = handle;
diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h
index 94e7412f6164..3c29ee09b8b5 100644
--- a/drivers/crypto/chelsio/chcr_core.h
+++ b/drivers/crypto/chelsio/chcr_core.h
@@ -39,6 +39,7 @@
 #include <crypto/algapi.h>
 #include "t4_hw.h"
 #include "cxgb4.h"
+#include "t4_msg.h"
 #include "cxgb4_uld.h"
 
 #define DRV_MODULE_NAME "chcr"
@@ -89,12 +90,49 @@ struct uld_ctx {
 	struct chcr_dev *dev;
 };
 
+struct chcr_ipsec_req {
+	struct ulp_txpkt ulptx;
+	struct ulptx_idata sc_imm;
+	struct cpl_tx_sec_pdu sec_cpl;
+	struct _key_ctx key_ctx;
+};
+
+struct chcr_ipsec_wr {
+	struct fw_ulptx_wr wreq;
+	struct chcr_ipsec_req req;
+};
+
+struct ipsec_sa_entry {
+	int hmac_ctrl;
+	unsigned int enckey_len;
+	unsigned int kctx_len;
+	unsigned int authsize;
+	__be32 key_ctx_hdr;
+	char salt[MAX_SALT];
+	char key[2 * AES_MAX_KEY_SIZE];
+};
+
+/*
+ *      sgl_len - calculates the size of an SGL of the given capacity
+ *      @n: the number of SGL entries
+ *      Calculates the number of flits needed for a scatter/gather list that
+ *      can hold the given number of entries.
+ */
+static inline unsigned int sgl_len(unsigned int n)
+{
+	n--;
+	return (3 * n) / 2 + (n & 1) + 2;
+}
+
 struct uld_ctx *assign_chcr_device(void);
 int chcr_send_wr(struct sk_buff *skb);
 int start_crypto(void);
 int stop_crypto(void);
 int chcr_uld_rx_handler(void *handle, const __be64 *rsp,
 			const struct pkt_gl *pgl);
+int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev);
 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
 		     int err);
+int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev);
+void chcr_add_xfrmops(const struct cxgb4_lld_info *lld);
 #endif /* __CHCR_CORE_H__ */
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index 94a87e3ad9bc..7daf0a17a7d2 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -134,14 +134,16 @@
 #define CRYPTO_ALG_SUB_TYPE_HASH_HMAC       0x01000000
 #define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106    0x02000000
 #define CRYPTO_ALG_SUB_TYPE_AEAD_GCM	    0x03000000
-#define CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC    0x04000000
+#define CRYPTO_ALG_SUB_TYPE_CBC_SHA	    0x04000000
 #define CRYPTO_ALG_SUB_TYPE_AEAD_CCM        0x05000000
 #define CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309    0x06000000
-#define CRYPTO_ALG_SUB_TYPE_AEAD_NULL       0x07000000
+#define CRYPTO_ALG_SUB_TYPE_CBC_NULL	    0x07000000
 #define CRYPTO_ALG_SUB_TYPE_CTR             0x08000000
 #define CRYPTO_ALG_SUB_TYPE_CTR_RFC3686     0x09000000
 #define CRYPTO_ALG_SUB_TYPE_XTS		    0x0a000000
 #define CRYPTO_ALG_SUB_TYPE_CBC		    0x0b000000
+#define CRYPTO_ALG_SUB_TYPE_CTR_SHA	    0x0c000000
+#define CRYPTO_ALG_SUB_TYPE_CTR_NULL   0x0d000000
 #define CRYPTO_ALG_TYPE_HMAC (CRYPTO_ALG_TYPE_AHASH |\
 			      CRYPTO_ALG_SUB_TYPE_HASH_HMAC)
 
@@ -210,8 +212,6 @@ struct dsgl_walk {
 	struct phys_sge_pairs *to;
 };
 
-
-
 struct chcr_gcm_ctx {
 	u8 ghash_h[AEAD_H_SIZE];
 };
@@ -227,21 +227,18 @@ struct __aead_ctx {
 	struct chcr_authenc_ctx authenc[0];
 };
 
-
-
 struct chcr_aead_ctx {
 	__be32 key_ctx_hdr;
 	unsigned int enckey_len;
 	struct crypto_aead *sw_cipher;
 	u8 salt[MAX_SALT];
 	u8 key[CHCR_AES_MAX_KEY_LEN];
+	u8 nonce[4];
 	u16 hmac_ctrl;
 	u16 mayverify;
 	struct	__aead_ctx ctx[0];
 };
 
-
-
 struct hmac_ctx {
 	struct crypto_shash *base_hash;
 	u8 ipad[CHCR_HASH_MAX_BLOCK_SIZE_128];
@@ -307,44 +304,29 @@ typedef struct sk_buff *(*create_wr_t)(struct aead_request *req,
 				       int size,
 				       unsigned short op_type);
 
-static int chcr_aead_op(struct aead_request *req_base,
-			  unsigned short op_type,
-			  int size,
-			  create_wr_t create_wr_fn);
-static inline int get_aead_subtype(struct crypto_aead *aead);
-static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
-				   unsigned char *input, int err);
-static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err);
-static int chcr_aead_dma_map(struct device *dev, struct aead_request *req,
-			     unsigned short op_type);
-static void chcr_aead_dma_unmap(struct device *dev, struct aead_request
-				*req, unsigned short op_type);
-static inline void chcr_add_aead_dst_ent(struct aead_request *req,
-				    struct cpl_rx_phys_dsgl *phys_cpl,
-				    unsigned int assoclen,
-				    unsigned short op_type,
-				    unsigned short qid);
-static inline void chcr_add_aead_src_ent(struct aead_request *req,
-				    struct ulptx_sgl *ulptx,
-				    unsigned int assoclen,
-				    unsigned short op_type);
-static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
-					   struct ulptx_sgl *ulptx,
-					   struct  cipher_wr_param *wrparam);
-static int chcr_cipher_dma_map(struct device *dev,
-			       struct ablkcipher_request *req);
-static void chcr_cipher_dma_unmap(struct device *dev,
-				  struct ablkcipher_request *req);
-static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
-					   struct cpl_rx_phys_dsgl *phys_cpl,
-					   struct  cipher_wr_param *wrparam,
-					   unsigned short qid);
+void chcr_verify_tag(struct aead_request *req, u8 *input, int *err);
+int chcr_aead_dma_map(struct device *dev, struct aead_request *req,
+		      unsigned short op_type);
+void chcr_aead_dma_unmap(struct device *dev, struct aead_request *req,
+			 unsigned short op_type);
+void chcr_add_aead_dst_ent(struct aead_request *req,
+			   struct cpl_rx_phys_dsgl *phys_cpl,
+			   unsigned int assoclen, unsigned short op_type,
+			   unsigned short qid);
+void chcr_add_aead_src_ent(struct aead_request *req, struct ulptx_sgl *ulptx,
+			   unsigned int assoclen, unsigned short op_type);
+void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
+			     struct ulptx_sgl *ulptx,
+			     struct  cipher_wr_param *wrparam);
+int chcr_cipher_dma_map(struct device *dev, struct ablkcipher_request *req);
+void chcr_cipher_dma_unmap(struct device *dev, struct ablkcipher_request *req);
+void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
+			     struct cpl_rx_phys_dsgl *phys_cpl,
+			     struct  cipher_wr_param *wrparam,
+			     unsigned short qid);
 int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip);
-static inline void chcr_add_hash_src_ent(struct ahash_request *req,
-					 struct ulptx_sgl *ulptx,
-					 struct hash_wr_param *param);
-static inline int chcr_hash_dma_map(struct device *dev,
-				    struct ahash_request *req);
-static inline void chcr_hash_dma_unmap(struct device *dev,
-				       struct ahash_request *req);
+void chcr_add_hash_src_ent(struct ahash_request *req, struct ulptx_sgl *ulptx,
+			   struct hash_wr_param *param);
+int chcr_hash_dma_map(struct device *dev, struct ahash_request *req);
+void chcr_hash_dma_unmap(struct device *dev, struct ahash_request *req);
 #endif /* __CHCR_CRYPTO_H__ */
diff --git a/drivers/crypto/chelsio/chcr_ipsec.c b/drivers/crypto/chelsio/chcr_ipsec.c
new file mode 100644
index 000000000000..db1e241104ed
--- /dev/null
+++ b/drivers/crypto/chelsio/chcr_ipsec.c
@@ -0,0 +1,654 @@
+/*
+ * This file is part of the Chelsio T6 Crypto driver for Linux.
+ *
+ * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Written and Maintained by:
+ *	Atul Gupta (atul.gupta@chelsio.com)
+ */
+
+#define pr_fmt(fmt) "chcr:" fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <linux/cryptohash.h>
+#include <linux/skbuff.h>
+#include <linux/rtnetlink.h>
+#include <linux/highmem.h>
+#include <linux/if_vlan.h>
+#include <linux/ip.h>
+#include <linux/netdevice.h>
+#include <net/esp.h>
+#include <net/xfrm.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/hash.h>
+#include <crypto/sha.h>
+#include <crypto/authenc.h>
+#include <crypto/internal/aead.h>
+#include <crypto/null.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/aead.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/hash.h>
+
+#include "chcr_core.h"
+#include "chcr_algo.h"
+#include "chcr_crypto.h"
+
+/*
+ * Max Tx descriptor space we allow for an Ethernet packet to be inlined
+ * into a WR.
+ */
+#define MAX_IMM_TX_PKT_LEN 256
+#define GCM_ESP_IV_SIZE     8
+
+static int chcr_xfrm_add_state(struct xfrm_state *x);
+static void chcr_xfrm_del_state(struct xfrm_state *x);
+static void chcr_xfrm_free_state(struct xfrm_state *x);
+static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
+
+static const struct xfrmdev_ops chcr_xfrmdev_ops = {
+	.xdo_dev_state_add      = chcr_xfrm_add_state,
+	.xdo_dev_state_delete   = chcr_xfrm_del_state,
+	.xdo_dev_state_free     = chcr_xfrm_free_state,
+	.xdo_dev_offload_ok     = chcr_ipsec_offload_ok,
+};
+
+/* Add offload xfrms to Chelsio Interface */
+void chcr_add_xfrmops(const struct cxgb4_lld_info *lld)
+{
+	struct net_device *netdev = NULL;
+	int i;
+
+	for (i = 0; i < lld->nports; i++) {
+		netdev = lld->ports[i];
+		if (!netdev)
+			continue;
+		netdev->xfrmdev_ops = &chcr_xfrmdev_ops;
+		netdev->hw_enc_features |= NETIF_F_HW_ESP;
+		netdev->features |= NETIF_F_HW_ESP;
+		rtnl_lock();
+		netdev_change_features(netdev);
+		rtnl_unlock();
+	}
+}
+
+static inline int chcr_ipsec_setauthsize(struct xfrm_state *x,
+					 struct ipsec_sa_entry *sa_entry)
+{
+	int hmac_ctrl;
+	int authsize = x->aead->alg_icv_len / 8;
+
+	sa_entry->authsize = authsize;
+
+	switch (authsize) {
+	case ICV_8:
+		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
+		break;
+	case ICV_12:
+		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
+		break;
+	case ICV_16:
+		hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return hmac_ctrl;
+}
+
+static inline int chcr_ipsec_setkey(struct xfrm_state *x,
+				    struct ipsec_sa_entry *sa_entry)
+{
+	struct crypto_cipher *cipher;
+	int keylen = (x->aead->alg_key_len + 7) / 8;
+	unsigned char *key = x->aead->alg_key;
+	int ck_size, key_ctx_size = 0;
+	unsigned char ghash_h[AEAD_H_SIZE];
+	int ret = 0;
+
+	if (keylen > 3) {
+		keylen -= 4;  /* nonce/salt is present in the last 4 bytes */
+		memcpy(sa_entry->salt, key + keylen, 4);
+	}
+
+	if (keylen == AES_KEYSIZE_128) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
+	} else if (keylen == AES_KEYSIZE_192) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
+	} else if (keylen == AES_KEYSIZE_256) {
+		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
+	} else {
+		pr_err("GCM: Invalid key length %d\n", keylen);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	memcpy(sa_entry->key, key, keylen);
+	sa_entry->enckey_len = keylen;
+	key_ctx_size = sizeof(struct _key_ctx) +
+			      ((DIV_ROUND_UP(keylen, 16)) << 4) +
+			      AEAD_H_SIZE;
+
+	sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
+						 CHCR_KEYCTX_MAC_KEY_SIZE_128,
+						 0, 0,
+						 key_ctx_size >> 4);
+
+	/* Calculate the H = CIPH(K, 0 repeated 16 times).
+	 * It will go in key context
+	 */
+	cipher = crypto_alloc_cipher("aes-generic", 0, 0);
+	if (IS_ERR(cipher)) {
+		sa_entry->enckey_len = 0;
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = crypto_cipher_setkey(cipher, key, keylen);
+	if (ret) {
+		sa_entry->enckey_len = 0;
+		goto out1;
+	}
+	memset(ghash_h, 0, AEAD_H_SIZE);
+	crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
+	memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
+	       16), ghash_h, AEAD_H_SIZE);
+	sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
+			      AEAD_H_SIZE;
+out1:
+	crypto_free_cipher(cipher);
+out:
+	return ret;
+}
+
+/*
+ * chcr_xfrm_add_state
+ * returns 0 on success, negative error if failed to send message to FPGA
+ * positive error if FPGA returned a bad response
+ */
+static int chcr_xfrm_add_state(struct xfrm_state *x)
+{
+	struct ipsec_sa_entry *sa_entry;
+	int res = 0;
+
+	if (x->props.aalgo != SADB_AALG_NONE) {
+		pr_debug("CHCR: Cannot offload authenticated xfrm states\n");
+		return -EINVAL;
+	}
+	if (x->props.calgo != SADB_X_CALG_NONE) {
+		pr_debug("CHCR: Cannot offload compressed xfrm states\n");
+		return -EINVAL;
+	}
+	if (x->props.flags & XFRM_STATE_ESN) {
+		pr_debug("CHCR: Cannot offload ESN xfrm states\n");
+		return -EINVAL;
+	}
+	if (x->props.family != AF_INET &&
+	    x->props.family != AF_INET6) {
+		pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n");
+		return -EINVAL;
+	}
+	if (x->props.mode != XFRM_MODE_TRANSPORT &&
+	    x->props.mode != XFRM_MODE_TUNNEL) {
+		pr_debug("CHCR: Only transport and tunnel xfrm offload\n");
+		return -EINVAL;
+	}
+	if (x->id.proto != IPPROTO_ESP) {
+		pr_debug("CHCR: Only ESP xfrm state offloaded\n");
+		return -EINVAL;
+	}
+	if (x->encap) {
+		pr_debug("CHCR: Encapsulated xfrm state not offloaded\n");
+		return -EINVAL;
+	}
+	if (!x->aead) {
+		pr_debug("CHCR: Cannot offload xfrm states without aead\n");
+		return -EINVAL;
+	}
+	if (x->aead->alg_icv_len != 128 &&
+	    x->aead->alg_icv_len != 96) {
+		pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n");
+	return -EINVAL;
+	}
+	if ((x->aead->alg_key_len != 128 + 32) &&
+	    (x->aead->alg_key_len != 256 + 32)) {
+		pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
+		return -EINVAL;
+	}
+	if (x->tfcpad) {
+		pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n");
+		return -EINVAL;
+	}
+	if (!x->geniv) {
+		pr_debug("CHCR: Cannot offload xfrm states without geniv\n");
+		return -EINVAL;
+	}
+	if (strcmp(x->geniv, "seqiv")) {
+		pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n");
+		return -EINVAL;
+	}
+
+	sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
+	if (!sa_entry) {
+		res = -ENOMEM;
+		goto out;
+	}
+
+	sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry);
+	chcr_ipsec_setkey(x, sa_entry);
+	x->xso.offload_handle = (unsigned long)sa_entry;
+	try_module_get(THIS_MODULE);
+out:
+	return res;
+}
+
+static void chcr_xfrm_del_state(struct xfrm_state *x)
+{
+	/* do nothing */
+	if (!x->xso.offload_handle)
+		return;
+}
+
+static void chcr_xfrm_free_state(struct xfrm_state *x)
+{
+	struct ipsec_sa_entry *sa_entry;
+
+	if (!x->xso.offload_handle)
+		return;
+
+	sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
+	kfree(sa_entry);
+	module_put(THIS_MODULE);
+}
+
+static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
+{
+	/* Offload with IP options is not supported yet */
+	if (ip_hdr(skb)->ihl > 5)
+		return false;
+
+	return true;
+}
+
+static inline int is_eth_imm(const struct sk_buff *skb, unsigned int kctx_len)
+{
+	int hdrlen = sizeof(struct chcr_ipsec_req) + kctx_len;
+
+	hdrlen += sizeof(struct cpl_tx_pkt);
+	if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
+		return hdrlen;
+	return 0;
+}
+
+static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb,
+					     unsigned int kctx_len)
+{
+	unsigned int flits;
+	int hdrlen = is_eth_imm(skb, kctx_len);
+
+	/* If the skb is small enough, we can pump it out as a work request
+	 * with only immediate data.  In that case we just have to have the
+	 * TX Packet header plus the skb data in the Work Request.
+	 */
+
+	if (hdrlen)
+		return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
+
+	flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
+
+	/* Otherwise, we're going to have to construct a Scatter gather list
+	 * of the skb body and fragments.  We also include the flits necessary
+	 * for the TX Packet Work Request and CPL.  We always have a firmware
+	 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
+	 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
+	 * message or, if we're doing a Large Send Offload, an LSO CPL message
+	 * with an embedded TX Packet Write CPL message.
+	 */
+	flits += (sizeof(struct fw_ulptx_wr) +
+		  sizeof(struct chcr_ipsec_req) +
+		  kctx_len +
+		  sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
+	return flits;
+}
+
+inline void *copy_cpltx_pktxt(struct sk_buff *skb,
+				struct net_device *dev,
+				void *pos)
+{
+	struct adapter *adap;
+	struct port_info *pi;
+	struct sge_eth_txq *q;
+	struct cpl_tx_pkt_core *cpl;
+	u64 cntrl = 0;
+	u32 ctrl0, qidx;
+
+	pi = netdev_priv(dev);
+	adap = pi->adapter;
+	qidx = skb->queue_mapping;
+	q = &adap->sge.ethtxq[qidx + pi->first_qset];
+
+	cpl = (struct cpl_tx_pkt_core *)pos;
+
+	if (skb->ip_summed == CHECKSUM_PARTIAL)
+		cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
+	ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
+			       TXPKT_PF_V(adap->pf);
+	if (skb_vlan_tag_present(skb)) {
+		q->vlan_ins++;
+		cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
+	}
+
+	cpl->ctrl0 = htonl(ctrl0);
+	cpl->pack = htons(0);
+	cpl->len = htons(skb->len);
+	cpl->ctrl1 = cpu_to_be64(cntrl);
+
+	pos += sizeof(struct cpl_tx_pkt_core);
+	return pos;
+}
+
+inline void *copy_key_cpltx_pktxt(struct sk_buff *skb,
+				struct net_device *dev,
+				void *pos,
+				struct ipsec_sa_entry *sa_entry)
+{
+	struct adapter *adap;
+	struct port_info *pi;
+	struct sge_eth_txq *q;
+	unsigned int len, qidx;
+	struct _key_ctx *key_ctx;
+	int left, eoq, key_len;
+
+	pi = netdev_priv(dev);
+	adap = pi->adapter;
+	qidx = skb->queue_mapping;
+	q = &adap->sge.ethtxq[qidx + pi->first_qset];
+	len = sa_entry->enckey_len + sizeof(struct cpl_tx_pkt_core);
+	key_len = sa_entry->kctx_len;
+
+	/* end of queue, reset pos to start of queue */
+	eoq = (void *)q->q.stat - pos;
+	left = eoq;
+	if (!eoq) {
+		pos = q->q.desc;
+		left = 64 * q->q.size;
+	}
+
+	/* Copy the Key context header */
+	key_ctx = (struct _key_ctx *)pos;
+	key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;
+	memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);
+	pos += sizeof(struct _key_ctx);
+	left -= sizeof(struct _key_ctx);
+
+	if (likely(len <= left)) {
+		memcpy(key_ctx->key, sa_entry->key, key_len);
+		pos += key_len;
+	} else {
+		if (key_len <= left) {
+			memcpy(pos, sa_entry->key, key_len);
+			pos += key_len;
+		} else {
+			memcpy(pos, sa_entry->key, left);
+			memcpy(q->q.desc, sa_entry->key + left,
+			       key_len - left);
+			pos = (u8 *)q->q.desc + (key_len - left);
+		}
+	}
+	/* Copy CPL TX PKT XT */
+	pos = copy_cpltx_pktxt(skb, dev, pos);
+
+	return pos;
+}
+
+inline void *chcr_crypto_wreq(struct sk_buff *skb,
+			       struct net_device *dev,
+			       void *pos,
+			       int credits,
+			       struct ipsec_sa_entry *sa_entry)
+{
+	struct port_info *pi = netdev_priv(dev);
+	struct adapter *adap = pi->adapter;
+	unsigned int immdatalen = 0;
+	unsigned int ivsize = GCM_ESP_IV_SIZE;
+	struct chcr_ipsec_wr *wr;
+	unsigned int flits;
+	u32 wr_mid;
+	int qidx = skb_get_queue_mapping(skb);
+	struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
+	unsigned int kctx_len = sa_entry->kctx_len;
+	int qid = q->q.cntxt_id;
+
+	atomic_inc(&adap->chcr_stats.ipsec_cnt);
+
+	flits = calc_tx_sec_flits(skb, kctx_len);
+
+	if (is_eth_imm(skb, kctx_len))
+		immdatalen = skb->len;
+
+	/* WR Header */
+	wr = (struct chcr_ipsec_wr *)pos;
+	wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
+	wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(flits, 2));
+
+	if (unlikely(credits < ETHTXQ_STOP_THRES)) {
+		netif_tx_stop_queue(q->txq);
+		q->q.stops++;
+		wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
+	}
+	wr_mid |= FW_ULPTX_WR_DATA_F;
+	wr->wreq.flowid_len16 = htonl(wr_mid);
+
+	/* ULPTX */
+	wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
+	wr->req.ulptx.len = htonl(DIV_ROUND_UP(flits, 2)  - 1);
+
+	/* Sub-command */
+	wr->req.sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
+	wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
+					 sizeof(wr->req.key_ctx) +
+					 kctx_len +
+					 sizeof(struct cpl_tx_pkt_core) +
+					 immdatalen);
+
+	/* CPL_SEC_PDU */
+	wr->req.sec_cpl.op_ivinsrtofst = htonl(
+				CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
+				CPL_TX_SEC_PDU_CPLLEN_V(2) |
+				CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
+				CPL_TX_SEC_PDU_IVINSRTOFST_V(
+				(skb_transport_offset(skb) +
+				sizeof(struct ip_esp_hdr) + 1)));
+
+	wr->req.sec_cpl.pldlen = htonl(skb->len);
+
+	wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
+				(skb_transport_offset(skb) + 1),
+				(skb_transport_offset(skb) +
+				 sizeof(struct ip_esp_hdr)),
+				(skb_transport_offset(skb) +
+				 sizeof(struct ip_esp_hdr) +
+				 GCM_ESP_IV_SIZE + 1), 0);
+
+	wr->req.sec_cpl.cipherstop_lo_authinsert =
+		FILL_SEC_CPL_AUTHINSERT(0, skb_transport_offset(skb) +
+					   sizeof(struct ip_esp_hdr) +
+					   GCM_ESP_IV_SIZE + 1,
+					   sa_entry->authsize,
+					   sa_entry->authsize);
+	wr->req.sec_cpl.seqno_numivs =
+		FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
+					 CHCR_SCMD_CIPHER_MODE_AES_GCM,
+					 CHCR_SCMD_AUTH_MODE_GHASH,
+					 sa_entry->hmac_ctrl,
+					 ivsize >> 1);
+	wr->req.sec_cpl.ivgen_hdrlen =  FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
+								  0, 0, 0);
+
+	pos += sizeof(struct fw_ulptx_wr) +
+	       sizeof(struct ulp_txpkt) +
+	       sizeof(struct ulptx_idata) +
+	       sizeof(struct cpl_tx_sec_pdu);
+
+	pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
+
+	return pos;
+}
+
+/**
+ *      flits_to_desc - returns the num of Tx descriptors for the given flits
+ *      @n: the number of flits
+ *
+ *      Returns the number of Tx descriptors needed for the supplied number
+ *      of flits.
+ */
+static inline unsigned int flits_to_desc(unsigned int n)
+{
+	WARN_ON(n > SGE_MAX_WR_LEN / 8);
+	return DIV_ROUND_UP(n, 8);
+}
+
+static inline unsigned int txq_avail(const struct sge_txq *q)
+{
+	return q->size - 1 - q->in_use;
+}
+
+static void eth_txq_stop(struct sge_eth_txq *q)
+{
+	netif_tx_stop_queue(q->txq);
+	q->q.stops++;
+}
+
+static inline void txq_advance(struct sge_txq *q, unsigned int n)
+{
+	q->in_use += n;
+	q->pidx += n;
+	if (q->pidx >= q->size)
+		q->pidx -= q->size;
+}
+
+/*
+ *      chcr_ipsec_xmit called from ULD Tx handler
+ */
+int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct xfrm_state *x = xfrm_input_state(skb);
+	struct ipsec_sa_entry *sa_entry;
+	u64 *pos, *end, *before, *sgl;
+	int qidx, left, credits;
+	unsigned int flits = 0, ndesc, kctx_len;
+	struct adapter *adap;
+	struct sge_eth_txq *q;
+	struct port_info *pi;
+	dma_addr_t addr[MAX_SKB_FRAGS + 1];
+	bool immediate = false;
+
+	if (!x->xso.offload_handle)
+		return NETDEV_TX_BUSY;
+
+	sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
+	kctx_len = sa_entry->kctx_len;
+
+	if (skb->sp->len != 1) {
+out_free:       dev_kfree_skb_any(skb);
+		return NETDEV_TX_OK;
+	}
+
+	pi = netdev_priv(dev);
+	adap = pi->adapter;
+	qidx = skb->queue_mapping;
+	q = &adap->sge.ethtxq[qidx + pi->first_qset];
+
+	cxgb4_reclaim_completed_tx(adap, &q->q, true);
+
+	flits = calc_tx_sec_flits(skb, sa_entry->kctx_len);
+	ndesc = flits_to_desc(flits);
+	credits = txq_avail(&q->q) - ndesc;
+
+	if (unlikely(credits < 0)) {
+		eth_txq_stop(q);
+		dev_err(adap->pdev_dev,
+			"%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",
+			dev->name, qidx, credits, ndesc, txq_avail(&q->q),
+			flits);
+		return NETDEV_TX_BUSY;
+	}
+
+	if (is_eth_imm(skb, kctx_len))
+		immediate = true;
+
+	if (!immediate &&
+	    unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
+		q->mapping_err++;
+		goto out_free;
+	}
+
+	pos = (u64 *)&q->q.desc[q->q.pidx];
+	before = (u64 *)pos;
+	end = (u64 *)pos + flits;
+	/* Setup IPSec CPL */
+	pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos,
+				       credits, sa_entry);
+	if (before > (u64 *)pos) {
+		left = (u8 *)end - (u8 *)q->q.stat;
+		end = (void *)q->q.desc + left;
+	}
+	if (pos == (u64 *)q->q.stat) {
+		left = (u8 *)end - (u8 *)q->q.stat;
+		end = (void *)q->q.desc + left;
+		pos = (void *)q->q.desc;
+	}
+
+	sgl = (void *)pos;
+	if (immediate) {
+		cxgb4_inline_tx_skb(skb, &q->q, sgl);
+		dev_consume_skb_any(skb);
+	} else {
+		int last_desc;
+
+		cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,
+				0, addr);
+		skb_orphan(skb);
+
+		last_desc = q->q.pidx + ndesc - 1;
+		if (last_desc >= q->q.size)
+			last_desc -= q->q.size;
+		q->q.sdesc[last_desc].skb = skb;
+		q->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)sgl;
+	}
+	txq_advance(&q->q, ndesc);
+
+	cxgb4_ring_tx_db(adap, &q->q, ndesc);
+	return NETDEV_TX_OK;
+}
diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c
index 451620b475a0..86f5f459762e 100644
--- a/drivers/crypto/exynos-rng.c
+++ b/drivers/crypto/exynos-rng.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * exynos-rng.c - Random Number Generator driver for the Exynos
  *
@@ -6,15 +7,6 @@
  * Loosely based on old driver from drivers/char/hw_random/exynos-rng.c:
  * Copyright (C) 2012 Samsung Electronics
  * Jonghwa Lee <jonghwa3.lee@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation;
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/clk.h>
@@ -22,12 +14,18 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 
 #include <crypto/internal/rng.h>
 
 #define EXYNOS_RNG_CONTROL		0x0
 #define EXYNOS_RNG_STATUS		0x10
+
+#define EXYNOS_RNG_SEED_CONF		0x14
+#define EXYNOS_RNG_GEN_PRNG	        BIT(1)
+
 #define EXYNOS_RNG_SEED_BASE		0x140
 #define EXYNOS_RNG_SEED(n)		(EXYNOS_RNG_SEED_BASE + (n * 0x4))
 #define EXYNOS_RNG_OUT_BASE		0x160
@@ -43,13 +41,21 @@
 #define EXYNOS_RNG_SEED_REGS		5
 #define EXYNOS_RNG_SEED_SIZE		(EXYNOS_RNG_SEED_REGS * 4)
 
+enum exynos_prng_type {
+	EXYNOS_PRNG_UNKNOWN = 0,
+	EXYNOS_PRNG_EXYNOS4,
+	EXYNOS_PRNG_EXYNOS5,
+};
+
 /*
- * Driver re-seeds itself with generated random numbers to increase
- * the randomness.
+ * Driver re-seeds itself with generated random numbers to hinder
+ * backtracking of the original seed.
  *
  * Time for next re-seed in ms.
  */
-#define EXYNOS_RNG_RESEED_TIME		100
+#define EXYNOS_RNG_RESEED_TIME		1000
+#define EXYNOS_RNG_RESEED_BYTES		65536
+
 /*
  * In polling mode, do not wait infinitely for the engine to finish the work.
  */
@@ -63,13 +69,17 @@ struct exynos_rng_ctx {
 /* Device associated memory */
 struct exynos_rng_dev {
 	struct device			*dev;
+	enum exynos_prng_type		type;
 	void __iomem			*mem;
 	struct clk			*clk;
+	struct mutex 			lock;
 	/* Generated numbers stored for seeding during resume */
 	u8				seed_save[EXYNOS_RNG_SEED_SIZE];
 	unsigned int			seed_save_len;
 	/* Time of last seeding in jiffies */
 	unsigned long			last_seeding;
+	/* Bytes generated since last seeding */
+	unsigned long			bytes_seeding;
 };
 
 static struct exynos_rng_dev *exynos_rng_dev;
@@ -114,38 +124,11 @@ static int exynos_rng_set_seed(struct exynos_rng_dev *rng,
 	}
 
 	rng->last_seeding = jiffies;
+	rng->bytes_seeding = 0;
 
 	return 0;
 }
 
-/*
- * Read from output registers and put the data under 'dst' array,
- * up to dlen bytes.
- *
- * Returns number of bytes actually stored in 'dst' (dlen
- * or EXYNOS_RNG_SEED_SIZE).
- */
-static unsigned int exynos_rng_copy_random(struct exynos_rng_dev *rng,
-					   u8 *dst, unsigned int dlen)
-{
-	unsigned int cnt = 0;
-	int i, j;
-	u32 val;
-
-	for (j = 0; j < EXYNOS_RNG_SEED_REGS; j++) {
-		val = exynos_rng_readl(rng, EXYNOS_RNG_OUT(j));
-
-		for (i = 0; i < 4; i++) {
-			dst[cnt] = val & 0xff;
-			val >>= 8;
-			if (++cnt >= dlen)
-				return cnt;
-		}
-	}
-
-	return cnt;
-}
-
 /*
  * Start the engine and poll for finish.  Then read from output registers
  * filling the 'dst' buffer up to 'dlen' bytes or up to size of generated
@@ -160,8 +143,13 @@ static int exynos_rng_get_random(struct exynos_rng_dev *rng,
 {
 	int retry = EXYNOS_RNG_WAIT_RETRIES;
 
-	exynos_rng_writel(rng, EXYNOS_RNG_CONTROL_START,
-			  EXYNOS_RNG_CONTROL);
+	if (rng->type == EXYNOS_PRNG_EXYNOS4) {
+		exynos_rng_writel(rng, EXYNOS_RNG_CONTROL_START,
+				  EXYNOS_RNG_CONTROL);
+	} else if (rng->type == EXYNOS_PRNG_EXYNOS5) {
+		exynos_rng_writel(rng, EXYNOS_RNG_GEN_PRNG,
+				  EXYNOS_RNG_SEED_CONF);
+	}
 
 	while (!(exynos_rng_readl(rng,
 			EXYNOS_RNG_STATUS) & EXYNOS_RNG_STATUS_RNG_DONE) && --retry)
@@ -173,7 +161,9 @@ static int exynos_rng_get_random(struct exynos_rng_dev *rng,
 	/* Clear status bit */
 	exynos_rng_writel(rng, EXYNOS_RNG_STATUS_RNG_DONE,
 			  EXYNOS_RNG_STATUS);
-	*read = exynos_rng_copy_random(rng, dst, dlen);
+	*read = min_t(size_t, dlen, EXYNOS_RNG_SEED_SIZE);
+	memcpy_fromio(dst, rng->mem + EXYNOS_RNG_OUT_BASE, *read);
+	rng->bytes_seeding += *read;
 
 	return 0;
 }
@@ -187,13 +177,18 @@ static void exynos_rng_reseed(struct exynos_rng_dev *rng)
 	unsigned int read = 0;
 	u8 seed[EXYNOS_RNG_SEED_SIZE];
 
-	if (time_before(now, next_seeding))
+	if (time_before(now, next_seeding) &&
+	    rng->bytes_seeding < EXYNOS_RNG_RESEED_BYTES)
 		return;
 
 	if (exynos_rng_get_random(rng, seed, sizeof(seed), &read))
 		return;
 
 	exynos_rng_set_seed(rng, seed, read);
+
+	/* Let others do some of their job. */
+	mutex_unlock(&rng->lock);
+	mutex_lock(&rng->lock);
 }
 
 static int exynos_rng_generate(struct crypto_rng *tfm,
@@ -209,6 +204,7 @@ static int exynos_rng_generate(struct crypto_rng *tfm,
 	if (ret)
 		return ret;
 
+	mutex_lock(&rng->lock);
 	do {
 		ret = exynos_rng_get_random(rng, dst, dlen, &read);
 		if (ret)
@@ -219,6 +215,7 @@ static int exynos_rng_generate(struct crypto_rng *tfm,
 
 		exynos_rng_reseed(rng);
 	} while (dlen > 0);
+	mutex_unlock(&rng->lock);
 
 	clk_disable_unprepare(rng->clk);
 
@@ -236,7 +233,9 @@ static int exynos_rng_seed(struct crypto_rng *tfm, const u8 *seed,
 	if (ret)
 		return ret;
 
+	mutex_lock(&rng->lock);
 	ret = exynos_rng_set_seed(ctx->rng, seed, slen);
+	mutex_unlock(&rng->lock);
 
 	clk_disable_unprepare(rng->clk);
 
@@ -259,7 +258,7 @@ static struct rng_alg exynos_rng_alg = {
 	.base			= {
 		.cra_name		= "stdrng",
 		.cra_driver_name	= "exynos_rng",
-		.cra_priority		= 100,
+		.cra_priority		= 300,
 		.cra_ctxsize		= sizeof(struct exynos_rng_ctx),
 		.cra_module		= THIS_MODULE,
 		.cra_init		= exynos_rng_kcapi_init,
@@ -279,6 +278,10 @@ static int exynos_rng_probe(struct platform_device *pdev)
 	if (!rng)
 		return -ENOMEM;
 
+	rng->type = (enum exynos_prng_type)of_device_get_match_data(&pdev->dev);
+
+	mutex_init(&rng->lock);
+
 	rng->dev = &pdev->dev;
 	rng->clk = devm_clk_get(&pdev->dev, "secss");
 	if (IS_ERR(rng->clk)) {
@@ -329,9 +332,14 @@ static int __maybe_unused exynos_rng_suspend(struct device *dev)
 	if (ret)
 		return ret;
 
+	mutex_lock(&rng->lock);
+
 	/* Get new random numbers and store them for seeding on resume. */
 	exynos_rng_get_random(rng, rng->seed_save, sizeof(rng->seed_save),
 			      &(rng->seed_save_len));
+
+	mutex_unlock(&rng->lock);
+
 	dev_dbg(rng->dev, "Stored %u bytes for seeding on system resume\n",
 		rng->seed_save_len);
 
@@ -354,8 +362,12 @@ static int __maybe_unused exynos_rng_resume(struct device *dev)
 	if (ret)
 		return ret;
 
+	mutex_lock(&rng->lock);
+
 	ret = exynos_rng_set_seed(rng, rng->seed_save, rng->seed_save_len);
 
+	mutex_unlock(&rng->lock);
+
 	clk_disable_unprepare(rng->clk);
 
 	return ret;
@@ -367,6 +379,10 @@ static SIMPLE_DEV_PM_OPS(exynos_rng_pm_ops, exynos_rng_suspend,
 static const struct of_device_id exynos_rng_dt_match[] = {
 	{
 		.compatible = "samsung,exynos4-rng",
+		.data = (const void *)EXYNOS_PRNG_EXYNOS4,
+	}, {
+		.compatible = "samsung,exynos5250-prng",
+		.data = (const void *)EXYNOS_PRNG_EXYNOS5,
 	},
 	{ },
 };
@@ -386,4 +402,4 @@ module_platform_driver(exynos_rng_driver);
 
 MODULE_DESCRIPTION("Exynos H/W Random Number Generator driver");
 MODULE_AUTHOR("Krzysztof Kozlowski <krzk@kernel.org>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c
index e09d4055b19e..a5a36fe7bf2c 100644
--- a/drivers/crypto/hifn_795x.c
+++ b/drivers/crypto/hifn_795x.c
@@ -2579,6 +2579,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	for (i = 0; i < 3; ++i)
 		if (dev->bar[i])
 			iounmap(dev->bar[i]);
+	kfree(dev);
 
 err_out_free_regions:
 	pci_release_regions(pdev);
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index 4bcef78a08aa..225e74a7f724 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -108,10 +108,10 @@ static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
 	writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
 	       EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
 	       EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
-	       priv->base + ctrl);
+	       EIP197_PE(priv) + ctrl);
 
 	/* Enable access to the program memory */
-	writel(prog_en, priv->base + EIP197_PE_ICE_RAM_CTRL);
+	writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL);
 
 	/* Write the firmware */
 	for (i = 0; i < fw->size / sizeof(u32); i++)
@@ -119,12 +119,12 @@ static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
 		       priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
 
 	/* Disable access to the program memory */
-	writel(0, priv->base + EIP197_PE_ICE_RAM_CTRL);
+	writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL);
 
 	/* Release engine from reset */
-	val = readl(priv->base + ctrl);
+	val = readl(EIP197_PE(priv) + ctrl);
 	val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET;
-	writel(val, priv->base + ctrl);
+	writel(val, EIP197_PE(priv) + ctrl);
 }
 
 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
@@ -145,14 +145,14 @@ static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
 	 }
 
 	/* Clear the scratchpad memory */
-	val = readl(priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
+	val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL);
 	val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
 	       EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
 	       EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
 	       EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
-	writel(val, priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
+	writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL);
 
-	memset(priv->base + EIP197_PE_ICE_SCRATCH_RAM, 0,
+	memset(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM, 0,
 	       EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
 
 	eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL,
@@ -173,7 +173,7 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
 	u32 hdw, cd_size_rnd, val;
 	int i;
 
-	hdw = readl(priv->base + EIP197_HIA_OPTIONS);
+	hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
 	hdw &= GENMASK(27, 25);
 	hdw >>= 25;
 
@@ -182,26 +182,25 @@ static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
 	for (i = 0; i < priv->config.rings; i++) {
 		/* ring base address */
 		writel(lower_32_bits(priv->ring[i].cdr.base_dma),
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
 		writel(upper_32_bits(priv->ring[i].cdr.base_dma),
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
 
 		writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
 		       priv->config.cd_size,
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DESC_SIZE);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
 		writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) |
 		       (EIP197_FETCH_COUNT * priv->config.cd_offset),
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
 
 		/* Configure DMA tx control */
 		val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
 		val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
-		writel(val,
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DMA_CFG);
+		writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
 
 		/* clear any pending interrupt */
 		writel(GENMASK(5, 0),
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_STAT);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
 	}
 
 	return 0;
@@ -212,7 +211,7 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
 	u32 hdw, rd_size_rnd, val;
 	int i;
 
-	hdw = readl(priv->base + EIP197_HIA_OPTIONS);
+	hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
 	hdw &= GENMASK(27, 25);
 	hdw >>= 25;
 
@@ -221,33 +220,33 @@ static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
 	for (i = 0; i < priv->config.rings; i++) {
 		/* ring base address */
 		writel(lower_32_bits(priv->ring[i].rdr.base_dma),
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
 		writel(upper_32_bits(priv->ring[i].rdr.base_dma),
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
 
 		writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
 		       priv->config.rd_size,
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DESC_SIZE);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
 
 		writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) |
 		       (EIP197_FETCH_COUNT * priv->config.rd_offset),
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
 
 		/* Configure DMA tx control */
 		val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
 		val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
 		val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG;
 		writel(val,
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DMA_CFG);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
 
 		/* clear any pending interrupt */
 		writel(GENMASK(7, 0),
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_STAT);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
 
 		/* enable ring interrupt */
-		val = readl(priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
+		val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
 		val |= EIP197_RDR_IRQ(i);
-		writel(val, priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
+		writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
 	}
 
 	return 0;
@@ -259,39 +258,40 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 	int i, ret;
 
 	/* Determine endianess and configure byte swap */
-	version = readl(priv->base + EIP197_HIA_VERSION);
-	val = readl(priv->base + EIP197_HIA_MST_CTRL);
+	version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION);
+	val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
 
 	if ((version & 0xffff) == EIP197_HIA_VERSION_BE)
 		val |= EIP197_MST_CTRL_BYTE_SWAP;
 	else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
 		val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
 
-	writel(val, priv->base + EIP197_HIA_MST_CTRL);
-
+	writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
 
 	/* Configure wr/rd cache values */
 	writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
 	       EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
-	       priv->base + EIP197_MST_CTRL);
+	       EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
 
 	/* Interrupts reset */
 
 	/* Disable all global interrupts */
-	writel(0, priv->base + EIP197_HIA_AIC_G_ENABLE_CTRL);
+	writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
 
 	/* Clear any pending interrupt */
-	writel(GENMASK(31, 0), priv->base + EIP197_HIA_AIC_G_ACK);
+	writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
 
 	/* Data Fetch Engine configuration */
 
 	/* Reset all DFE threads */
 	writel(EIP197_DxE_THR_CTRL_RESET_PE,
-	       priv->base + EIP197_HIA_DFE_THR_CTRL);
+	       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
 
-	/* Reset HIA input interface arbiter */
-	writel(EIP197_HIA_RA_PE_CTRL_RESET,
-	       priv->base + EIP197_HIA_RA_PE_CTRL);
+	if (priv->version == EIP197) {
+		/* Reset HIA input interface arbiter */
+		writel(EIP197_HIA_RA_PE_CTRL_RESET,
+		       EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL);
+	}
 
 	/* DMA transfer size to use */
 	val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
@@ -299,29 +299,32 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 	val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
 	val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
 	val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
-	writel(val, priv->base + EIP197_HIA_DFE_CFG);
+	writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG);
 
 	/* Leave the DFE threads reset state */
-	writel(0, priv->base + EIP197_HIA_DFE_THR_CTRL);
+	writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
 
 	/* Configure the procesing engine thresholds */
 	writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9),
-	      priv->base + EIP197_PE_IN_DBUF_THRES);
+	       EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES);
 	writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7),
-	      priv->base + EIP197_PE_IN_TBUF_THRES);
+	       EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES);
 
-	/* enable HIA input interface arbiter and rings */
-	writel(EIP197_HIA_RA_PE_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
-	       priv->base + EIP197_HIA_RA_PE_CTRL);
+	if (priv->version == EIP197) {
+		/* enable HIA input interface arbiter and rings */
+		writel(EIP197_HIA_RA_PE_CTRL_EN |
+		       GENMASK(priv->config.rings - 1, 0),
+		       EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL);
+	}
 
 	/* Data Store Engine configuration */
 
 	/* Reset all DSE threads */
 	writel(EIP197_DxE_THR_CTRL_RESET_PE,
-	       priv->base + EIP197_HIA_DSE_THR_CTRL);
+	       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
 
 	/* Wait for all DSE threads to complete */
-	while ((readl(priv->base + EIP197_HIA_DSE_THR_STAT) &
+	while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT) &
 		GENMASK(15, 12)) != GENMASK(15, 12))
 		;
 
@@ -330,15 +333,19 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 	val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
 	val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
 	val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE;
-	val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
-	writel(val, priv->base + EIP197_HIA_DSE_CFG);
+	/* FIXME: instability issues can occur for EIP97 but disabling it impact
+	 * performances.
+	 */
+	if (priv->version == EIP197)
+		val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
+	writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG);
 
 	/* Leave the DSE threads reset state */
-	writel(0, priv->base + EIP197_HIA_DSE_THR_CTRL);
+	writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
 
 	/* Configure the procesing engine thresholds */
 	writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8),
-	       priv->base + EIP197_PE_OUT_DBUF_THRES);
+	       EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES);
 
 	/* Processing Engine configuration */
 
@@ -348,73 +355,75 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 	val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
 	val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
 	val |= EIP197_ALG_SHA2;
-	writel(val, priv->base + EIP197_PE_EIP96_FUNCTION_EN);
+	writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN);
 
 	/* Command Descriptor Rings prepare */
 	for (i = 0; i < priv->config.rings; i++) {
 		/* Clear interrupts for this ring */
 		writel(GENMASK(31, 0),
-		       priv->base + EIP197_HIA_AIC_R_ENABLE_CLR(i));
+		       EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
 
 		/* Disable external triggering */
-		writel(0, priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
+		writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
 
 		/* Clear the pending prepared counter */
 		writel(EIP197_xDR_PREP_CLR_COUNT,
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
 
 		/* Clear the pending processed counter */
 		writel(EIP197_xDR_PROC_CLR_COUNT,
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_COUNT);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
 
 		writel(0,
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_PNTR);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
 		writel(0,
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_PNTR);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
 
 		writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
-		       priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_SIZE);
+		       EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
 	}
 
 	/* Result Descriptor Ring prepare */
 	for (i = 0; i < priv->config.rings; i++) {
 		/* Disable external triggering*/
-		writel(0, priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
+		writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
 
 		/* Clear the pending prepared counter */
 		writel(EIP197_xDR_PREP_CLR_COUNT,
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
 
 		/* Clear the pending processed counter */
 		writel(EIP197_xDR_PROC_CLR_COUNT,
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_COUNT);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
 
 		writel(0,
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_PNTR);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
 		writel(0,
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_PNTR);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
 
 		/* Ring size */
 		writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
-		       priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_SIZE);
+		       EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
 	}
 
 	/* Enable command descriptor rings */
 	writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
-	       priv->base + EIP197_HIA_DFE_THR_CTRL);
+	       EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL);
 
 	/* Enable result descriptor rings */
 	writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
-	       priv->base + EIP197_HIA_DSE_THR_CTRL);
+	       EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL);
 
 	/* Clear any HIA interrupt */
-	writel(GENMASK(30, 20), priv->base + EIP197_HIA_AIC_G_ACK);
+	writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
 
-	eip197_trc_cache_init(priv);
+	if (priv->version == EIP197) {
+		eip197_trc_cache_init(priv);
 
-	ret = eip197_load_firmwares(priv);
-	if (ret)
-		return ret;
+		ret = eip197_load_firmwares(priv);
+		if (ret)
+			return ret;
+	}
 
 	safexcel_hw_setup_cdesc_rings(priv);
 	safexcel_hw_setup_rdesc_rings(priv);
@@ -422,6 +431,23 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
 	return 0;
 }
 
+/* Called with ring's lock taken */
+static int safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
+				      int ring, int reqs)
+{
+	int coal = min_t(int, reqs, EIP197_MAX_BATCH_SZ);
+
+	if (!coal)
+		return 0;
+
+	/* Configure when we want an interrupt */
+	writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
+	       EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
+	       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
+
+	return coal;
+}
+
 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
 {
 	struct crypto_async_request *req, *backlog;
@@ -429,34 +455,36 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
 	struct safexcel_request *request;
 	int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
 
-	priv->ring[ring].need_dequeue = false;
+	/* If a request wasn't properly dequeued because of a lack of resources,
+	 * proceeded it first,
+	 */
+	req = priv->ring[ring].req;
+	backlog = priv->ring[ring].backlog;
+	if (req)
+		goto handle_req;
 
-	do {
+	while (true) {
 		spin_lock_bh(&priv->ring[ring].queue_lock);
 		backlog = crypto_get_backlog(&priv->ring[ring].queue);
 		req = crypto_dequeue_request(&priv->ring[ring].queue);
 		spin_unlock_bh(&priv->ring[ring].queue_lock);
 
-		if (!req)
+		if (!req) {
+			priv->ring[ring].req = NULL;
+			priv->ring[ring].backlog = NULL;
 			goto finalize;
+		}
 
+handle_req:
 		request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req));
-		if (!request) {
-			spin_lock_bh(&priv->ring[ring].queue_lock);
-			crypto_enqueue_request(&priv->ring[ring].queue, req);
-			spin_unlock_bh(&priv->ring[ring].queue_lock);
-
-			priv->ring[ring].need_dequeue = true;
-			goto finalize;
-		}
+		if (!request)
+			goto request_failed;
 
 		ctx = crypto_tfm_ctx(req->tfm);
 		ret = ctx->send(req, ring, request, &commands, &results);
 		if (ret) {
 			kfree(request);
-			req->complete(req, ret);
-			priv->ring[ring].need_dequeue = true;
-			goto finalize;
+			goto request_failed;
 		}
 
 		if (backlog)
@@ -468,30 +496,39 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
 
 		cdesc += commands;
 		rdesc += results;
-	} while (nreq++ < EIP197_MAX_BATCH_SZ);
+		nreq++;
+	}
+
+request_failed:
+	/* Not enough resources to handle all the requests. Bail out and save
+	 * the request and the backlog for the next dequeue call (per-ring).
+	 */
+	priv->ring[ring].req = req;
+	priv->ring[ring].backlog = backlog;
 
 finalize:
-	if (nreq == EIP197_MAX_BATCH_SZ)
-		priv->ring[ring].need_dequeue = true;
-	else if (!nreq)
+	if (!nreq)
 		return;
 
-	spin_lock_bh(&priv->ring[ring].lock);
+	spin_lock_bh(&priv->ring[ring].egress_lock);
 
-	/* Configure when we want an interrupt */
-	writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
-	       EIP197_HIA_RDR_THRESH_PROC_PKT(nreq),
-	       priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH);
+	if (!priv->ring[ring].busy) {
+		nreq -= safexcel_try_push_requests(priv, ring, nreq);
+		if (nreq)
+			priv->ring[ring].busy = true;
+	}
+
+	priv->ring[ring].requests_left += nreq;
+
+	spin_unlock_bh(&priv->ring[ring].egress_lock);
 
 	/* let the RDR know we have pending descriptors */
 	writel((rdesc * priv->config.rd_offset) << 2,
-	       priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
+	       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
 
 	/* let the CDR know we have pending descriptors */
 	writel((cdesc * priv->config.cd_offset) << 2,
-	       priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
-
-	spin_unlock_bh(&priv->ring[ring].lock);
+	       EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
 }
 
 void safexcel_free_context(struct safexcel_crypto_priv *priv,
@@ -540,7 +577,6 @@ void safexcel_inv_complete(struct crypto_async_request *req, int error)
 }
 
 int safexcel_invalidate_cache(struct crypto_async_request *async,
-			      struct safexcel_context *ctx,
 			      struct safexcel_crypto_priv *priv,
 			      dma_addr_t ctxr_dma, int ring,
 			      struct safexcel_request *request)
@@ -587,14 +623,17 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv
 {
 	struct safexcel_request *sreq;
 	struct safexcel_context *ctx;
-	int ret, i, nreq, ndesc = 0;
+	int ret, i, nreq, ndesc, tot_descs, done;
 	bool should_complete;
 
-	nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
-	nreq >>= 24;
-	nreq &= GENMASK(6, 0);
+handle_results:
+	tot_descs = 0;
+
+	nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
+	nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
+	nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
 	if (!nreq)
-		return;
+		goto requests_left;
 
 	for (i = 0; i < nreq; i++) {
 		spin_lock_bh(&priv->ring[ring].egress_lock);
@@ -609,13 +648,9 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv
 		if (ndesc < 0) {
 			kfree(sreq);
 			dev_err(priv->dev, "failed to handle result (%d)", ndesc);
-			return;
+			goto acknowledge;
 		}
 
-		writel(EIP197_xDR_PROC_xD_PKT(1) |
-		       EIP197_xDR_PROC_xD_COUNT(ndesc * priv->config.rd_offset),
-		       priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
-
 		if (should_complete) {
 			local_bh_disable();
 			sreq->req->complete(sreq->req, ret);
@@ -623,19 +658,41 @@ static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv
 		}
 
 		kfree(sreq);
+		tot_descs += ndesc;
 	}
+
+acknowledge:
+	if (i) {
+		writel(EIP197_xDR_PROC_xD_PKT(i) |
+		       EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
+		       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
+	}
+
+	/* If the number of requests overflowed the counter, try to proceed more
+	 * requests.
+	 */
+	if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
+		goto handle_results;
+
+requests_left:
+	spin_lock_bh(&priv->ring[ring].egress_lock);
+
+	done = safexcel_try_push_requests(priv, ring,
+					  priv->ring[ring].requests_left);
+
+	priv->ring[ring].requests_left -= done;
+	if (!done && !priv->ring[ring].requests_left)
+		priv->ring[ring].busy = false;
+
+	spin_unlock_bh(&priv->ring[ring].egress_lock);
 }
 
-static void safexcel_handle_result_work(struct work_struct *work)
+static void safexcel_dequeue_work(struct work_struct *work)
 {
 	struct safexcel_work_data *data =
 			container_of(work, struct safexcel_work_data, work);
-	struct safexcel_crypto_priv *priv = data->priv;
-
-	safexcel_handle_result_descriptor(priv, data->ring);
 
-	if (priv->ring[data->ring].need_dequeue)
-		safexcel_dequeue(data->priv, data->ring);
+	safexcel_dequeue(data->priv, data->ring);
 }
 
 struct safexcel_ring_irq_data {
@@ -647,16 +704,16 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data)
 {
 	struct safexcel_ring_irq_data *irq_data = data;
 	struct safexcel_crypto_priv *priv = irq_data->priv;
-	int ring = irq_data->ring;
+	int ring = irq_data->ring, rc = IRQ_NONE;
 	u32 status, stat;
 
-	status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
+	status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
 	if (!status)
-		return IRQ_NONE;
+		return rc;
 
 	/* RDR interrupts */
 	if (status & EIP197_RDR_IRQ(ring)) {
-		stat = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
+		stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
 
 		if (unlikely(stat & EIP197_xDR_ERR)) {
 			/*
@@ -666,22 +723,37 @@ static irqreturn_t safexcel_irq_ring(int irq, void *data)
 			 */
 			dev_err(priv->dev, "RDR: fatal error.");
 		} else if (likely(stat & EIP197_xDR_THRESH)) {
-			queue_work(priv->ring[ring].workqueue, &priv->ring[ring].work_data.work);
+			rc = IRQ_WAKE_THREAD;
 		}
 
 		/* ACK the interrupts */
 		writel(stat & 0xff,
-		       priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
+		       EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
 	}
 
 	/* ACK the interrupts */
-	writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring));
+	writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
+
+	return rc;
+}
+
+static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
+{
+	struct safexcel_ring_irq_data *irq_data = data;
+	struct safexcel_crypto_priv *priv = irq_data->priv;
+	int ring = irq_data->ring;
+
+	safexcel_handle_result_descriptor(priv, ring);
+
+	queue_work(priv->ring[ring].workqueue,
+		   &priv->ring[ring].work_data.work);
 
 	return IRQ_HANDLED;
 }
 
 static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name,
 				     irq_handler_t handler,
+				     irq_handler_t threaded_handler,
 				     struct safexcel_ring_irq_data *ring_irq_priv)
 {
 	int ret, irq = platform_get_irq_byname(pdev, name);
@@ -691,8 +763,9 @@ static int safexcel_request_ring_irq(struct platform_device *pdev, const char *n
 		return irq;
 	}
 
-	ret = devm_request_irq(&pdev->dev, irq, handler, 0,
-			       dev_name(&pdev->dev), ring_irq_priv);
+	ret = devm_request_threaded_irq(&pdev->dev, irq, handler,
+					threaded_handler, IRQF_ONESHOT,
+					dev_name(&pdev->dev), ring_irq_priv);
 	if (ret) {
 		dev_err(&pdev->dev, "unable to request IRQ %d\n", irq);
 		return ret;
@@ -755,11 +828,11 @@ static void safexcel_configure(struct safexcel_crypto_priv *priv)
 {
 	u32 val, mask;
 
-	val = readl(priv->base + EIP197_HIA_OPTIONS);
+	val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
 	val = (val & GENMASK(27, 25)) >> 25;
 	mask = BIT(val) - 1;
 
-	val = readl(priv->base + EIP197_HIA_OPTIONS);
+	val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
 	priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
 
 	priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
@@ -769,6 +842,35 @@ static void safexcel_configure(struct safexcel_crypto_priv *priv)
 	priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
 }
 
+static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
+{
+	struct safexcel_register_offsets *offsets = &priv->offsets;
+
+	if (priv->version == EIP197) {
+		offsets->hia_aic	= EIP197_HIA_AIC_BASE;
+		offsets->hia_aic_g	= EIP197_HIA_AIC_G_BASE;
+		offsets->hia_aic_r	= EIP197_HIA_AIC_R_BASE;
+		offsets->hia_aic_xdr	= EIP197_HIA_AIC_xDR_BASE;
+		offsets->hia_dfe	= EIP197_HIA_DFE_BASE;
+		offsets->hia_dfe_thr	= EIP197_HIA_DFE_THR_BASE;
+		offsets->hia_dse	= EIP197_HIA_DSE_BASE;
+		offsets->hia_dse_thr	= EIP197_HIA_DSE_THR_BASE;
+		offsets->hia_gen_cfg	= EIP197_HIA_GEN_CFG_BASE;
+		offsets->pe		= EIP197_PE_BASE;
+	} else {
+		offsets->hia_aic	= EIP97_HIA_AIC_BASE;
+		offsets->hia_aic_g	= EIP97_HIA_AIC_G_BASE;
+		offsets->hia_aic_r	= EIP97_HIA_AIC_R_BASE;
+		offsets->hia_aic_xdr	= EIP97_HIA_AIC_xDR_BASE;
+		offsets->hia_dfe	= EIP97_HIA_DFE_BASE;
+		offsets->hia_dfe_thr	= EIP97_HIA_DFE_THR_BASE;
+		offsets->hia_dse	= EIP97_HIA_DSE_BASE;
+		offsets->hia_dse_thr	= EIP97_HIA_DSE_THR_BASE;
+		offsets->hia_gen_cfg	= EIP97_HIA_GEN_CFG_BASE;
+		offsets->pe		= EIP97_PE_BASE;
+	}
+}
+
 static int safexcel_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -781,6 +883,9 @@ static int safexcel_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	priv->dev = dev;
+	priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
+
+	safexcel_init_register_offsets(priv);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	priv->base = devm_ioremap_resource(dev, res);
@@ -839,6 +944,7 @@ static int safexcel_probe(struct platform_device *pdev)
 
 		snprintf(irq_name, 6, "ring%d", i);
 		irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
+						safexcel_irq_ring_thread,
 						ring_irq);
 		if (irq < 0) {
 			ret = irq;
@@ -847,7 +953,7 @@ static int safexcel_probe(struct platform_device *pdev)
 
 		priv->ring[i].work_data.priv = priv;
 		priv->ring[i].work_data.ring = i;
-		INIT_WORK(&priv->ring[i].work_data.work, safexcel_handle_result_work);
+		INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work);
 
 		snprintf(wq_name, 9, "wq_ring%d", i);
 		priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
@@ -856,6 +962,9 @@ static int safexcel_probe(struct platform_device *pdev)
 			goto err_clk;
 		}
 
+		priv->ring[i].requests_left = 0;
+		priv->ring[i].busy = false;
+
 		crypto_init_queue(&priv->ring[i].queue,
 				  EIP197_DEFAULT_RING_SIZE);
 
@@ -903,7 +1012,14 @@ static int safexcel_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id safexcel_of_match_table[] = {
-	{ .compatible = "inside-secure,safexcel-eip197" },
+	{
+		.compatible = "inside-secure,safexcel-eip97",
+		.data = (void *)EIP97,
+	},
+	{
+		.compatible = "inside-secure,safexcel-eip197",
+		.data = (void *)EIP197,
+	},
 	{},
 };
 
diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h
index 304c5838c11a..4e219c21608b 100644
--- a/drivers/crypto/inside-secure/safexcel.h
+++ b/drivers/crypto/inside-secure/safexcel.h
@@ -19,64 +19,103 @@
 #define EIP197_HIA_VERSION_BE			0x35ca
 
 /* Static configuration */
-#define EIP197_DEFAULT_RING_SIZE		64
+#define EIP197_DEFAULT_RING_SIZE		400
 #define EIP197_MAX_TOKENS			5
 #define EIP197_MAX_RINGS			4
 #define EIP197_FETCH_COUNT			1
-#define EIP197_MAX_BATCH_SZ			EIP197_DEFAULT_RING_SIZE
+#define EIP197_MAX_BATCH_SZ			64
 
 #define EIP197_GFP_FLAGS(base)	((base).flags & CRYPTO_TFM_REQ_MAY_SLEEP ? \
 				 GFP_KERNEL : GFP_ATOMIC)
 
+/* Register base offsets */
+#define EIP197_HIA_AIC(priv)		((priv)->base + (priv)->offsets.hia_aic)
+#define EIP197_HIA_AIC_G(priv)		((priv)->base + (priv)->offsets.hia_aic_g)
+#define EIP197_HIA_AIC_R(priv)		((priv)->base + (priv)->offsets.hia_aic_r)
+#define EIP197_HIA_AIC_xDR(priv)	((priv)->base + (priv)->offsets.hia_aic_xdr)
+#define EIP197_HIA_DFE(priv)		((priv)->base + (priv)->offsets.hia_dfe)
+#define EIP197_HIA_DFE_THR(priv)	((priv)->base + (priv)->offsets.hia_dfe_thr)
+#define EIP197_HIA_DSE(priv)		((priv)->base + (priv)->offsets.hia_dse)
+#define EIP197_HIA_DSE_THR(priv)	((priv)->base + (priv)->offsets.hia_dse_thr)
+#define EIP197_HIA_GEN_CFG(priv)	((priv)->base + (priv)->offsets.hia_gen_cfg)
+#define EIP197_PE(priv)			((priv)->base + (priv)->offsets.pe)
+
+/* EIP197 base offsets */
+#define EIP197_HIA_AIC_BASE		0x90000
+#define EIP197_HIA_AIC_G_BASE		0x90000
+#define EIP197_HIA_AIC_R_BASE		0x90800
+#define EIP197_HIA_AIC_xDR_BASE		0x80000
+#define EIP197_HIA_DFE_BASE		0x8c000
+#define EIP197_HIA_DFE_THR_BASE		0x8c040
+#define EIP197_HIA_DSE_BASE		0x8d000
+#define EIP197_HIA_DSE_THR_BASE		0x8d040
+#define EIP197_HIA_GEN_CFG_BASE		0xf0000
+#define EIP197_PE_BASE			0xa0000
+
+/* EIP97 base offsets */
+#define EIP97_HIA_AIC_BASE		0x0
+#define EIP97_HIA_AIC_G_BASE		0x0
+#define EIP97_HIA_AIC_R_BASE		0x0
+#define EIP97_HIA_AIC_xDR_BASE		0x0
+#define EIP97_HIA_DFE_BASE		0xf000
+#define EIP97_HIA_DFE_THR_BASE		0xf200
+#define EIP97_HIA_DSE_BASE		0xf400
+#define EIP97_HIA_DSE_THR_BASE		0xf600
+#define EIP97_HIA_GEN_CFG_BASE		0x10000
+#define EIP97_PE_BASE			0x10000
+
 /* CDR/RDR register offsets */
-#define EIP197_HIA_xDR_OFF(r)			(0x80000 + (r) * 0x1000)
-#define EIP197_HIA_CDR(r)			(EIP197_HIA_xDR_OFF(r))
-#define EIP197_HIA_RDR(r)			(EIP197_HIA_xDR_OFF(r) + 0x800)
-#define EIP197_HIA_xDR_RING_BASE_ADDR_LO	0x0
-#define EIP197_HIA_xDR_RING_BASE_ADDR_HI	0x4
-#define EIP197_HIA_xDR_RING_SIZE		0x18
-#define EIP197_HIA_xDR_DESC_SIZE		0x1c
-#define EIP197_HIA_xDR_CFG			0x20
-#define EIP197_HIA_xDR_DMA_CFG			0x24
-#define EIP197_HIA_xDR_THRESH			0x28
-#define EIP197_HIA_xDR_PREP_COUNT		0x2c
-#define EIP197_HIA_xDR_PROC_COUNT		0x30
-#define EIP197_HIA_xDR_PREP_PNTR		0x34
-#define EIP197_HIA_xDR_PROC_PNTR		0x38
-#define EIP197_HIA_xDR_STAT			0x3c
+#define EIP197_HIA_xDR_OFF(priv, r)		(EIP197_HIA_AIC_xDR(priv) + (r) * 0x1000)
+#define EIP197_HIA_CDR(priv, r)			(EIP197_HIA_xDR_OFF(priv, r))
+#define EIP197_HIA_RDR(priv, r)			(EIP197_HIA_xDR_OFF(priv, r) + 0x800)
+#define EIP197_HIA_xDR_RING_BASE_ADDR_LO	0x0000
+#define EIP197_HIA_xDR_RING_BASE_ADDR_HI	0x0004
+#define EIP197_HIA_xDR_RING_SIZE		0x0018
+#define EIP197_HIA_xDR_DESC_SIZE		0x001c
+#define EIP197_HIA_xDR_CFG			0x0020
+#define EIP197_HIA_xDR_DMA_CFG			0x0024
+#define EIP197_HIA_xDR_THRESH			0x0028
+#define EIP197_HIA_xDR_PREP_COUNT		0x002c
+#define EIP197_HIA_xDR_PROC_COUNT		0x0030
+#define EIP197_HIA_xDR_PREP_PNTR		0x0034
+#define EIP197_HIA_xDR_PROC_PNTR		0x0038
+#define EIP197_HIA_xDR_STAT			0x003c
 
 /* register offsets */
-#define EIP197_HIA_DFE_CFG			0x8c000
-#define EIP197_HIA_DFE_THR_CTRL			0x8c040
-#define EIP197_HIA_DFE_THR_STAT			0x8c044
-#define EIP197_HIA_DSE_CFG			0x8d000
-#define EIP197_HIA_DSE_THR_CTRL			0x8d040
-#define EIP197_HIA_DSE_THR_STAT			0x8d044
-#define EIP197_HIA_RA_PE_CTRL			0x90010
-#define EIP197_HIA_RA_PE_STAT			0x90014
+#define EIP197_HIA_DFE_CFG			0x0000
+#define EIP197_HIA_DFE_THR_CTRL			0x0000
+#define EIP197_HIA_DFE_THR_STAT			0x0004
+#define EIP197_HIA_DSE_CFG			0x0000
+#define EIP197_HIA_DSE_THR_CTRL			0x0000
+#define EIP197_HIA_DSE_THR_STAT			0x0004
+#define EIP197_HIA_RA_PE_CTRL			0x0010
+#define EIP197_HIA_RA_PE_STAT			0x0014
 #define EIP197_HIA_AIC_R_OFF(r)			((r) * 0x1000)
-#define EIP197_HIA_AIC_R_ENABLE_CTRL(r)		(0x9e808 - EIP197_HIA_AIC_R_OFF(r))
-#define EIP197_HIA_AIC_R_ENABLED_STAT(r)	(0x9e810 - EIP197_HIA_AIC_R_OFF(r))
-#define EIP197_HIA_AIC_R_ACK(r)			(0x9e810 - EIP197_HIA_AIC_R_OFF(r))
-#define EIP197_HIA_AIC_R_ENABLE_CLR(r)		(0x9e814 - EIP197_HIA_AIC_R_OFF(r))
-#define EIP197_HIA_AIC_G_ENABLE_CTRL		0x9f808
-#define EIP197_HIA_AIC_G_ENABLED_STAT		0x9f810
-#define EIP197_HIA_AIC_G_ACK			0x9f810
-#define EIP197_HIA_MST_CTRL			0x9fff4
-#define EIP197_HIA_OPTIONS			0x9fff8
-#define EIP197_HIA_VERSION			0x9fffc
-#define EIP197_PE_IN_DBUF_THRES			0xa0000
-#define EIP197_PE_IN_TBUF_THRES			0xa0100
-#define EIP197_PE_ICE_SCRATCH_RAM		0xa0800
-#define EIP197_PE_ICE_PUE_CTRL			0xa0c80
-#define EIP197_PE_ICE_SCRATCH_CTRL		0xa0d04
-#define EIP197_PE_ICE_FPP_CTRL			0xa0d80
-#define EIP197_PE_ICE_RAM_CTRL			0xa0ff0
-#define EIP197_PE_EIP96_FUNCTION_EN		0xa1004
-#define EIP197_PE_EIP96_CONTEXT_CTRL		0xa1008
-#define EIP197_PE_EIP96_CONTEXT_STAT		0xa100c
-#define EIP197_PE_OUT_DBUF_THRES		0xa1c00
-#define EIP197_PE_OUT_TBUF_THRES		0xa1d00
+#define EIP197_HIA_AIC_R_ENABLE_CTRL(r)		(0xe008 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_R_ENABLED_STAT(r)	(0xe010 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_R_ACK(r)			(0xe010 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_R_ENABLE_CLR(r)		(0xe014 - EIP197_HIA_AIC_R_OFF(r))
+#define EIP197_HIA_AIC_G_ENABLE_CTRL		0xf808
+#define EIP197_HIA_AIC_G_ENABLED_STAT		0xf810
+#define EIP197_HIA_AIC_G_ACK			0xf810
+#define EIP197_HIA_MST_CTRL			0xfff4
+#define EIP197_HIA_OPTIONS			0xfff8
+#define EIP197_HIA_VERSION			0xfffc
+#define EIP197_PE_IN_DBUF_THRES			0x0000
+#define EIP197_PE_IN_TBUF_THRES			0x0100
+#define EIP197_PE_ICE_SCRATCH_RAM		0x0800
+#define EIP197_PE_ICE_PUE_CTRL			0x0c80
+#define EIP197_PE_ICE_SCRATCH_CTRL		0x0d04
+#define EIP197_PE_ICE_FPP_CTRL			0x0d80
+#define EIP197_PE_ICE_RAM_CTRL			0x0ff0
+#define EIP197_PE_EIP96_FUNCTION_EN		0x1004
+#define EIP197_PE_EIP96_CONTEXT_CTRL		0x1008
+#define EIP197_PE_EIP96_CONTEXT_STAT		0x100c
+#define EIP197_PE_OUT_DBUF_THRES		0x1c00
+#define EIP197_PE_OUT_TBUF_THRES		0x1d00
+#define EIP197_MST_CTRL				0xfff4
+
+/* EIP197-specific registers, no indirection */
 #define EIP197_CLASSIFICATION_RAMS		0xe0000
 #define EIP197_TRC_CTRL				0xf0800
 #define EIP197_TRC_LASTRES			0xf0804
@@ -90,7 +129,6 @@
 #define EIP197_TRC_ECCDATASTAT			0xf083c
 #define EIP197_TRC_ECCDATA			0xf0840
 #define EIP197_CS_RAM_CTRL			0xf7ff0
-#define EIP197_MST_CTRL				0xffff4
 
 /* EIP197_HIA_xDR_DESC_SIZE */
 #define EIP197_xDR_DESC_MODE_64BIT		BIT(31)
@@ -117,6 +155,8 @@
 #define EIP197_xDR_PREP_CLR_COUNT		BIT(31)
 
 /* EIP197_HIA_xDR_PROC_COUNT */
+#define EIP197_xDR_PROC_xD_PKT_OFFSET		24
+#define EIP197_xDR_PROC_xD_PKT_MASK		GENMASK(6, 0)
 #define EIP197_xDR_PROC_xD_COUNT(n)		((n) << 2)
 #define EIP197_xDR_PROC_xD_PKT(n)		((n) << 24)
 #define EIP197_xDR_PROC_CLR_COUNT		BIT(31)
@@ -463,12 +503,33 @@ struct safexcel_work_data {
 	int ring;
 };
 
+enum safexcel_eip_version {
+	EIP97,
+	EIP197,
+};
+
+struct safexcel_register_offsets {
+	u32 hia_aic;
+	u32 hia_aic_g;
+	u32 hia_aic_r;
+	u32 hia_aic_xdr;
+	u32 hia_dfe;
+	u32 hia_dfe_thr;
+	u32 hia_dse;
+	u32 hia_dse_thr;
+	u32 hia_gen_cfg;
+	u32 pe;
+};
+
 struct safexcel_crypto_priv {
 	void __iomem *base;
 	struct device *dev;
 	struct clk *clk;
 	struct safexcel_config config;
 
+	enum safexcel_eip_version version;
+	struct safexcel_register_offsets offsets;
+
 	/* context DMA pool */
 	struct dma_pool *context_pool;
 
@@ -489,7 +550,20 @@ struct safexcel_crypto_priv {
 		/* queue */
 		struct crypto_queue queue;
 		spinlock_t queue_lock;
-		bool need_dequeue;
+
+		/* Number of requests in the engine that needs the threshold
+		 * interrupt to be set up.
+		 */
+		int requests_left;
+
+		/* The ring is currently handling at least one request */
+		bool busy;
+
+		/* Store for current requests when bailing out of the dequeueing
+		 * function when no enough resources are available.
+		 */
+		struct crypto_async_request *req;
+		struct crypto_async_request *backlog;
 	} ring[EIP197_MAX_RINGS];
 };
 
@@ -539,7 +613,6 @@ void safexcel_free_context(struct safexcel_crypto_priv *priv,
 				  struct crypto_async_request *req,
 				  int result_sz);
 int safexcel_invalidate_cache(struct crypto_async_request *async,
-			      struct safexcel_context *ctx,
 			      struct safexcel_crypto_priv *priv,
 			      dma_addr_t ctxr_dma, int ring,
 			      struct safexcel_request *request);
diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c
index fcc0a606d748..63a8768ed2ae 100644
--- a/drivers/crypto/inside-secure/safexcel_cipher.c
+++ b/drivers/crypto/inside-secure/safexcel_cipher.c
@@ -27,7 +27,6 @@ struct safexcel_cipher_ctx {
 	struct safexcel_context base;
 	struct safexcel_crypto_priv *priv;
 
-	enum safexcel_cipher_direction direction;
 	u32 mode;
 
 	__le32 key[8];
@@ -35,6 +34,7 @@ struct safexcel_cipher_ctx {
 };
 
 struct safexcel_cipher_req {
+	enum safexcel_cipher_direction direction;
 	bool needs_inv;
 };
 
@@ -69,6 +69,7 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key,
 {
 	struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct safexcel_crypto_priv *priv = ctx->priv;
 	struct crypto_aes_ctx aes;
 	int ret, i;
 
@@ -78,10 +79,12 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key,
 		return ret;
 	}
 
-	for (i = 0; i < len / sizeof(u32); i++) {
-		if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
-			ctx->base.needs_inv = true;
-			break;
+	if (priv->version == EIP197 && ctx->base.ctxr_dma) {
+		for (i = 0; i < len / sizeof(u32); i++) {
+			if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
+				ctx->base.needs_inv = true;
+				break;
+			}
 		}
 	}
 
@@ -95,12 +98,15 @@ static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key,
 }
 
 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
+				    struct crypto_async_request *async,
 				    struct safexcel_command_desc *cdesc)
 {
 	struct safexcel_crypto_priv *priv = ctx->priv;
+	struct skcipher_request *req = skcipher_request_cast(async);
+	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 	int ctrl_size;
 
-	if (ctx->direction == SAFEXCEL_ENCRYPT)
+	if (sreq->direction == SAFEXCEL_ENCRYPT)
 		cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT;
 	else
 		cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN;
@@ -243,7 +249,7 @@ static int safexcel_aes_send(struct crypto_async_request *async,
 		n_cdesc++;
 
 		if (n_cdesc == 1) {
-			safexcel_context_control(ctx, cdesc);
+			safexcel_context_control(ctx, async, cdesc);
 			safexcel_cipher_token(ctx, async, cdesc, req->cryptlen);
 		}
 
@@ -353,8 +359,8 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 	if (enq_ret != -EINPROGRESS)
 		*ret = enq_ret;
 
-	if (!priv->ring[ring].need_dequeue)
-		safexcel_dequeue(priv, ring);
+	queue_work(priv->ring[ring].workqueue,
+		   &priv->ring[ring].work_data.work);
 
 	*should_complete = false;
 
@@ -390,7 +396,7 @@ static int safexcel_cipher_send_inv(struct crypto_async_request *async,
 	struct safexcel_crypto_priv *priv = ctx->priv;
 	int ret;
 
-	ret = safexcel_invalidate_cache(async, &ctx->base, priv,
+	ret = safexcel_invalidate_cache(async, priv,
 					ctx->base.ctxr_dma, ring, request);
 	if (unlikely(ret))
 		return ret;
@@ -406,9 +412,13 @@ static int safexcel_send(struct crypto_async_request *async,
 			 int *commands, int *results)
 {
 	struct skcipher_request *req = skcipher_request_cast(async);
+	struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 	struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
+	struct safexcel_crypto_priv *priv = ctx->priv;
 	int ret;
 
+	BUG_ON(priv->version == EIP97 && sreq->needs_inv);
+
 	if (sreq->needs_inv)
 		ret = safexcel_cipher_send_inv(async, ring, request,
 					       commands, results);
@@ -443,8 +453,8 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm)
 	crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
 	spin_unlock_bh(&priv->ring[ring].queue_lock);
 
-	if (!priv->ring[ring].need_dequeue)
-		safexcel_dequeue(priv, ring);
+	queue_work(priv->ring[ring].workqueue,
+		   &priv->ring[ring].work_data.work);
 
 	wait_for_completion_interruptible(&result.completion);
 
@@ -467,11 +477,11 @@ static int safexcel_aes(struct skcipher_request *req,
 	int ret, ring;
 
 	sreq->needs_inv = false;
-	ctx->direction = dir;
+	sreq->direction = dir;
 	ctx->mode = mode;
 
 	if (ctx->base.ctxr) {
-		if (ctx->base.needs_inv) {
+		if (priv->version == EIP197 && ctx->base.needs_inv) {
 			sreq->needs_inv = true;
 			ctx->base.needs_inv = false;
 		}
@@ -490,8 +500,8 @@ static int safexcel_aes(struct skcipher_request *req,
 	ret = crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
 	spin_unlock_bh(&priv->ring[ring].queue_lock);
 
-	if (!priv->ring[ring].need_dequeue)
-		safexcel_dequeue(priv, ring);
+	queue_work(priv->ring[ring].workqueue,
+		   &priv->ring[ring].work_data.work);
 
 	return ret;
 }
@@ -539,9 +549,14 @@ static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
 
 	memzero_explicit(ctx->base.ctxr->data, 8 * sizeof(u32));
 
-	ret = safexcel_cipher_exit_inv(tfm);
-	if (ret)
-		dev_warn(priv->dev, "cipher: invalidation error %d\n", ret);
+	if (priv->version == EIP197) {
+		ret = safexcel_cipher_exit_inv(tfm);
+		if (ret)
+			dev_warn(priv->dev, "cipher: invalidation error %d\n", ret);
+	} else {
+		dma_pool_free(priv->context_pool, ctx->base.ctxr,
+			      ctx->base.ctxr_dma);
+	}
 }
 
 struct safexcel_alg_template safexcel_alg_ecb_aes = {
diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c
index 0c5a5820b06e..122a2a58e98f 100644
--- a/drivers/crypto/inside-secure/safexcel_hash.c
+++ b/drivers/crypto/inside-secure/safexcel_hash.c
@@ -14,7 +14,6 @@
 #include <linux/dma-mapping.h>
 #include <linux/dmapool.h>
 
-
 #include "safexcel.h"
 
 struct safexcel_ahash_ctx {
@@ -34,6 +33,8 @@ struct safexcel_ahash_req {
 	bool hmac;
 	bool needs_inv;
 
+	int nents;
+
 	u8 state_sz;    /* expected sate size, only set once */
 	u32 state[SHA256_DIGEST_SIZE / sizeof(u32)] __aligned(sizeof(u32));
 
@@ -152,8 +153,10 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin
 		memcpy(areq->result, sreq->state,
 		       crypto_ahash_digestsize(ahash));
 
-	dma_unmap_sg(priv->dev, areq->src,
-		     sg_nents_for_len(areq->src, areq->nbytes), DMA_TO_DEVICE);
+	if (sreq->nents) {
+		dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
+		sreq->nents = 0;
+	}
 
 	safexcel_free_context(priv, async, sreq->state_sz);
 
@@ -178,7 +181,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
 	struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
 	struct safexcel_result_desc *rdesc;
 	struct scatterlist *sg;
-	int i, nents, queued, len, cache_len, extra, n_cdesc = 0, ret = 0;
+	int i, queued, len, cache_len, extra, n_cdesc = 0, ret = 0;
 
 	queued = len = req->len - req->processed;
 	if (queued < crypto_ahash_blocksize(ahash))
@@ -186,17 +189,31 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
 	else
 		cache_len = queued - areq->nbytes;
 
-	/*
-	 * If this is not the last request and the queued data does not fit
-	 * into full blocks, cache it for the next send() call.
-	 */
-	extra = queued & (crypto_ahash_blocksize(ahash) - 1);
-	if (!req->last_req && extra) {
-		sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
-				   req->cache_next, extra, areq->nbytes - extra);
-
-		queued -= extra;
-		len -= extra;
+	if (!req->last_req) {
+		/* If this is not the last request and the queued data does not
+		 * fit into full blocks, cache it for the next send() call.
+		 */
+		extra = queued & (crypto_ahash_blocksize(ahash) - 1);
+		if (!extra)
+			/* If this is not the last request and the queued data
+			 * is a multiple of a block, cache the last one for now.
+			 */
+			extra = queued - crypto_ahash_blocksize(ahash);
+
+		if (extra) {
+			sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
+					   req->cache_next, extra,
+					   areq->nbytes - extra);
+
+			queued -= extra;
+			len -= extra;
+
+			if (!queued) {
+				*commands = 0;
+				*results = 0;
+				return 0;
+			}
+		}
 	}
 
 	spin_lock_bh(&priv->ring[ring].egress_lock);
@@ -234,15 +251,15 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
 	}
 
 	/* Now handle the current ahash request buffer(s) */
-	nents = dma_map_sg(priv->dev, areq->src,
-		       sg_nents_for_len(areq->src, areq->nbytes),
-		       DMA_TO_DEVICE);
-	if (!nents) {
+	req->nents = dma_map_sg(priv->dev, areq->src,
+				sg_nents_for_len(areq->src, areq->nbytes),
+				DMA_TO_DEVICE);
+	if (!req->nents) {
 		ret = -ENOMEM;
 		goto cdesc_rollback;
 	}
 
-	for_each_sg(areq->src, sg, nents, i) {
+	for_each_sg(areq->src, sg, req->nents, i) {
 		int sglen = sg_dma_len(sg);
 
 		/* Do not overflow the request */
@@ -382,8 +399,8 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 	if (enq_ret != -EINPROGRESS)
 		*ret = enq_ret;
 
-	if (!priv->ring[ring].need_dequeue)
-		safexcel_dequeue(priv, ring);
+	queue_work(priv->ring[ring].workqueue,
+		   &priv->ring[ring].work_data.work);
 
 	*should_complete = false;
 
@@ -398,6 +415,8 @@ static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 	int err;
 
+	BUG_ON(priv->version == EIP97 && req->needs_inv);
+
 	if (req->needs_inv) {
 		req->needs_inv = false;
 		err = safexcel_handle_inv_result(priv, ring, async,
@@ -418,7 +437,7 @@ static int safexcel_ahash_send_inv(struct crypto_async_request *async,
 	struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 	int ret;
 
-	ret = safexcel_invalidate_cache(async, &ctx->base, ctx->priv,
+	ret = safexcel_invalidate_cache(async, ctx->priv,
 					ctx->base.ctxr_dma, ring, request);
 	if (unlikely(ret))
 		return ret;
@@ -471,8 +490,8 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 	crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
 	spin_unlock_bh(&priv->ring[ring].queue_lock);
 
-	if (!priv->ring[ring].need_dequeue)
-		safexcel_dequeue(priv, ring);
+	queue_work(priv->ring[ring].workqueue,
+		   &priv->ring[ring].work_data.work);
 
 	wait_for_completion_interruptible(&result.completion);
 
@@ -485,13 +504,23 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
 	return 0;
 }
 
+/* safexcel_ahash_cache: cache data until at least one request can be sent to
+ * the engine, aka. when there is at least 1 block size in the pipe.
+ */
 static int safexcel_ahash_cache(struct ahash_request *areq)
 {
 	struct safexcel_ahash_req *req = ahash_request_ctx(areq);
 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
 	int queued, cache_len;
 
+	/* cache_len: everyting accepted by the driver but not sent yet,
+	 * tot sz handled by update() - last req sz - tot sz handled by send()
+	 */
 	cache_len = req->len - areq->nbytes - req->processed;
+	/* queued: everything accepted by the driver which will be handled by
+	 * the next send() calls.
+	 * tot sz handled by update() - tot sz handled by send()
+	 */
 	queued = req->len - req->processed;
 
 	/*
@@ -505,7 +534,7 @@ static int safexcel_ahash_cache(struct ahash_request *areq)
 		return areq->nbytes;
 	}
 
-	/* We could'nt cache all the data */
+	/* We couldn't cache all the data */
 	return -E2BIG;
 }
 
@@ -518,10 +547,17 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
 
 	req->needs_inv = false;
 
-	if (req->processed && ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
-		ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq);
-
 	if (ctx->base.ctxr) {
+		if (priv->version == EIP197 &&
+		    !ctx->base.needs_inv && req->processed &&
+		    ctx->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED)
+			/* We're still setting needs_inv here, even though it is
+			 * cleared right away, because the needs_inv flag can be
+			 * set in other functions and we want to keep the same
+			 * logic.
+			 */
+			ctx->base.needs_inv = safexcel_ahash_needs_inv_get(areq);
+
 		if (ctx->base.needs_inv) {
 			ctx->base.needs_inv = false;
 			req->needs_inv = true;
@@ -541,8 +577,8 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq)
 	ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
 	spin_unlock_bh(&priv->ring[ring].queue_lock);
 
-	if (!priv->ring[ring].need_dequeue)
-		safexcel_dequeue(priv, ring);
+	queue_work(priv->ring[ring].workqueue,
+		   &priv->ring[ring].work_data.work);
 
 	return ret;
 }
@@ -625,7 +661,6 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out)
 	export->processed = req->processed;
 
 	memcpy(export->state, req->state, req->state_sz);
-	memset(export->cache, 0, crypto_ahash_blocksize(ahash));
 	memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash));
 
 	return 0;
@@ -707,9 +742,14 @@ static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
 	if (!ctx->base.ctxr)
 		return;
 
-	ret = safexcel_ahash_exit_inv(tfm);
-	if (ret)
-		dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
+	if (priv->version == EIP197) {
+		ret = safexcel_ahash_exit_inv(tfm);
+		if (ret)
+			dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
+	} else {
+		dma_pool_free(priv->context_pool, ctx->base.ctxr,
+			      ctx->base.ctxr_dma);
+	}
 }
 
 struct safexcel_alg_template safexcel_alg_sha1 = {
@@ -848,7 +888,7 @@ static int safexcel_hmac_init_iv(struct ahash_request *areq,
 	req->last_req = true;
 
 	ret = crypto_ahash_update(areq);
-	if (ret && ret != -EINPROGRESS)
+	if (ret && ret != -EINPROGRESS && ret != -EBUSY)
 		return ret;
 
 	wait_for_completion_interruptible(&result.completion);
@@ -913,6 +953,7 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
 				     unsigned int keylen)
 {
 	struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+	struct safexcel_crypto_priv *priv = ctx->priv;
 	struct safexcel_ahash_export_state istate, ostate;
 	int ret, i;
 
@@ -920,11 +961,13 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
 	if (ret)
 		return ret;
 
-	for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
-		if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
-		    ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
-			ctx->base.needs_inv = true;
-			break;
+	if (priv->version == EIP197 && ctx->base.ctxr) {
+		for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
+			if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) ||
+			    ctx->opad[i] != le32_to_cpu(ostate.state[i])) {
+				ctx->base.needs_inv = true;
+				break;
+			}
 		}
 	}
 
diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c
index 8705b28eb02c..717a26607bdb 100644
--- a/drivers/crypto/ixp4xx_crypto.c
+++ b/drivers/crypto/ixp4xx_crypto.c
@@ -260,12 +260,11 @@ static int setup_crypt_desc(void)
 {
 	struct device *dev = &pdev->dev;
 	BUILD_BUG_ON(sizeof(struct crypt_ctl) != 64);
-	crypt_virt = dma_alloc_coherent(dev,
-			NPE_QLEN * sizeof(struct crypt_ctl),
-			&crypt_phys, GFP_ATOMIC);
+	crypt_virt = dma_zalloc_coherent(dev,
+					 NPE_QLEN * sizeof(struct crypt_ctl),
+					 &crypt_phys, GFP_ATOMIC);
 	if (!crypt_virt)
 		return -ENOMEM;
-	memset(crypt_virt, 0, NPE_QLEN * sizeof(struct crypt_ctl));
 	return 0;
 }
 
diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c
index 293832488cc9..aca2373fa1de 100644
--- a/drivers/crypto/marvell/cesa.c
+++ b/drivers/crypto/marvell/cesa.c
@@ -15,6 +15,7 @@
  */
 
 #include <linux/delay.h>
+#include <linux/dma-mapping.h>
 #include <linux/genalloc.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -24,6 +25,7 @@
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <linux/module.h>
+#include <linux/dma-direct.h> /* XXX: drivers shall never use this directly! */
 #include <linux/clk.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
@@ -409,8 +411,11 @@ static int mv_cesa_get_sram(struct platform_device *pdev, int idx)
 	if (IS_ERR(engine->sram))
 		return PTR_ERR(engine->sram);
 
-	engine->sram_dma = phys_to_dma(cesa->dev,
-				       (phys_addr_t)res->start);
+	engine->sram_dma = dma_map_resource(cesa->dev, res->start,
+					    cesa->sram_size,
+					    DMA_BIDIRECTIONAL, 0);
+	if (dma_mapping_error(cesa->dev, engine->sram_dma))
+		return -ENOMEM;
 
 	return 0;
 }
@@ -420,11 +425,12 @@ static void mv_cesa_put_sram(struct platform_device *pdev, int idx)
 	struct mv_cesa_dev *cesa = platform_get_drvdata(pdev);
 	struct mv_cesa_engine *engine = &cesa->engines[idx];
 
-	if (!engine->pool)
-		return;
-
-	gen_pool_free(engine->pool, (unsigned long)engine->sram,
-		      cesa->sram_size);
+	if (engine->pool)
+		gen_pool_free(engine->pool, (unsigned long)engine->sram,
+			      cesa->sram_size);
+	else
+		dma_unmap_resource(cesa->dev, engine->sram_dma,
+				   cesa->sram_size, DMA_BIDIRECTIONAL, 0);
 }
 
 static int mv_cesa_probe(struct platform_device *pdev)
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index f2246a5abcf6..1e87637c412d 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -743,8 +743,8 @@ static int nx842_open_percpu_txwins(void)
 		}
 
 		if (!per_cpu(cpu_txwin, i)) {
-			/* shoudn't happen, Each chip will have NX engine */
-			pr_err("NX engine is not availavle for CPU %d\n", i);
+			/* shouldn't happen, Each chip will have NX engine */
+			pr_err("NX engine is not available for CPU %d\n", i);
 			return -EINVAL;
 		}
 	}
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index 4b6642a25df5..1c6cbda56afe 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -512,7 +512,7 @@ static int __init padlock_init(void)
 
 	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
 
-	if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
+	if (c->x86 == 6 && c->x86_model == 15 && c->x86_stepping == 2) {
 		ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
 		cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
 		printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c
index 5a6dc53b2b9d..4ef52c9d72fc 100644
--- a/drivers/crypto/picoxcell_crypto.c
+++ b/drivers/crypto/picoxcell_crypto.c
@@ -1618,7 +1618,7 @@ MODULE_DEVICE_TABLE(of, spacc_of_id_table);
 
 static int spacc_probe(struct platform_device *pdev)
 {
-	int i, err, ret = -EINVAL;
+	int i, err, ret;
 	struct resource *mem, *irq;
 	struct device_node *np = pdev->dev.of_node;
 	struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),
@@ -1679,22 +1679,18 @@ static int spacc_probe(struct platform_device *pdev)
 	engine->clk = clk_get(&pdev->dev, "ref");
 	if (IS_ERR(engine->clk)) {
 		dev_info(&pdev->dev, "clk unavailable\n");
-		device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
 		return PTR_ERR(engine->clk);
 	}
 
 	if (clk_prepare_enable(engine->clk)) {
 		dev_info(&pdev->dev, "unable to prepare/enable clk\n");
-		clk_put(engine->clk);
-		return -EIO;
+		ret = -EIO;
+		goto err_clk_put;
 	}
 
-	err = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
-	if (err) {
-		clk_disable_unprepare(engine->clk);
-		clk_put(engine->clk);
-		return err;
-	}
+	ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
+	if (ret)
+		goto err_clk_disable;
 
 
 	/*
@@ -1725,6 +1721,7 @@ static int spacc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, engine);
 
+	ret = -EINVAL;
 	INIT_LIST_HEAD(&engine->registered_algs);
 	for (i = 0; i < engine->num_algs; ++i) {
 		engine->algs[i].engine = engine;
@@ -1759,6 +1756,16 @@ static int spacc_probe(struct platform_device *pdev)
 				engine->aeads[i].alg.base.cra_name);
 	}
 
+	if (!ret)
+		return 0;
+
+	del_timer_sync(&engine->packet_timeout);
+	device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
+err_clk_disable:
+	clk_disable_unprepare(engine->clk);
+err_clk_put:
+	clk_put(engine->clk);
+
 	return ret;
 }
 
diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c
index 8c4fd255a601..ff149e176f64 100644
--- a/drivers/crypto/qat/qat_common/qat_hal.c
+++ b/drivers/crypto/qat/qat_common/qat_hal.c
@@ -117,19 +117,19 @@ void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
 
 #define CSR_RETRY_TIMES 500
 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
-			     unsigned char ae, unsigned int csr,
-			     unsigned int *value)
+			     unsigned char ae, unsigned int csr)
 {
 	unsigned int iterations = CSR_RETRY_TIMES;
+	int value;
 
 	do {
-		*value = GET_AE_CSR(handle, ae, csr);
+		value = GET_AE_CSR(handle, ae, csr);
 		if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
-			return 0;
+			return value;
 	} while (iterations--);
 
 	pr_err("QAT: Read CSR timeout\n");
-	return -EFAULT;
+	return 0;
 }
 
 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
@@ -154,9 +154,9 @@ static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int cur_ctx;
 
-	qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
+	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
-	qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events);
+	*events = qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT);
 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 }
 
@@ -169,13 +169,13 @@ static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
 	int times = MAX_RETRY_TIMES;
 	int elapsed_cycles = 0;
 
-	qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt);
+	base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 	base_cnt &= 0xffff;
 	while ((int)cycles > elapsed_cycles && times--) {
 		if (chk_inactive)
-			qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr);
+			csr = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 
-		qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt);
+		cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 		cur_cnt &= 0xffff;
 		elapsed_cycles = cur_cnt - base_cnt;
 
@@ -207,7 +207,7 @@ int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
 	}
 
 	/* Sets the accelaration engine context mode to either four or eight */
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
+	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	csr = IGNORE_W1C_MASK & csr;
 	new_csr = (mode == 4) ?
 		SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
@@ -221,7 +221,7 @@ int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int csr, new_csr;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
+	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	csr &= IGNORE_W1C_MASK;
 
 	new_csr = (mode) ?
@@ -240,7 +240,7 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int csr, new_csr;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
+	csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	csr &= IGNORE_W1C_MASK;
 	switch (lm_type) {
 	case ICP_LMEM0:
@@ -328,7 +328,7 @@ static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int ctx, cur_ctx;
 
-	qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
+	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 
 	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 		if (!(ctx_mask & (1 << ctx)))
@@ -340,16 +340,18 @@ static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 }
 
-static void qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
+static unsigned int qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
 				unsigned char ae, unsigned char ctx,
-				unsigned int ae_csr, unsigned int *csr_val)
+				unsigned int ae_csr)
 {
-	unsigned int cur_ctx;
+	unsigned int cur_ctx, csr_val;
 
-	qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
+	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
-	qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val);
+	csr_val = qat_hal_rd_ae_csr(handle, ae, ae_csr);
 	qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
+
+	return csr_val;
 }
 
 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
@@ -358,7 +360,7 @@ static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int ctx, cur_ctx;
 
-	qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
+	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 		if (!(ctx_mask & (1 << ctx)))
 			continue;
@@ -374,7 +376,7 @@ static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int ctx, cur_ctx;
 
-	qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
+	cur_ctx = qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER);
 	for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 		if (!(ctx_mask & (1 << ctx)))
 			continue;
@@ -392,13 +394,11 @@ static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 	int times = MAX_RETRY_TIMES;
 
 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
-		qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
-				  (unsigned int *)&base_cnt);
+		base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 		base_cnt &= 0xffff;
 
 		do {
-			qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
-					  (unsigned int *)&cur_cnt);
+			cur_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT);
 			cur_cnt &= 0xffff;
 		} while (times-- && (cur_cnt == base_cnt));
 
@@ -416,8 +416,8 @@ int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int enable = 0, active = 0;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &enable);
-	qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &active);
+	enable = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
+	active = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 	if ((enable & (0xff << CE_ENABLE_BITPOS)) ||
 	    (active & (1 << ACS_ABO_BITPOS)))
 		return 1;
@@ -540,7 +540,7 @@ static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int ctx;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
+	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	ctx &= IGNORE_W1C_MASK &
 		(~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
@@ -583,7 +583,7 @@ void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
 	unsigned int ustore_addr;
 	unsigned int i;
 
-	qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
+	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 	uaddr |= UA_ECS;
 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 	for (i = 0; i < words_num; i++) {
@@ -604,7 +604,7 @@ static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int ctx;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
+	ctx = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	ctx &= IGNORE_W1C_MASK;
 	ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
 	ctx |= (ctx_mask << CE_ENABLE_BITPOS);
@@ -636,10 +636,10 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 	int ret = 0;
 
 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
-		qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
+		csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 		csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
 		qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
-		qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val);
+		csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 		csr_val &= IGNORE_W1C_MASK;
 		csr_val |= CE_NN_MODE;
 		qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
@@ -648,7 +648,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 		qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 				    handle->hal_handle->upc_mask &
 				    INIT_PC_VALUE);
-		qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
+		savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
 		qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
 		qat_hal_wr_indr_csr(handle, ae, ctx_mask,
@@ -760,7 +760,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev)
 	for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 		unsigned int csr_val = 0;
 
-		qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val);
+		csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE);
 		csr_val |= 0x1;
 		qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
 	}
@@ -826,16 +826,16 @@ static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
 	unsigned int i, uwrd_lo, uwrd_hi;
 	unsigned int ustore_addr, misc_control;
 
-	qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control);
+	misc_control = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
 			  misc_control & 0xfffffffb);
-	qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
+	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 	uaddr |= UA_ECS;
 	for (i = 0; i < words_num; i++) {
 		qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 		uaddr++;
-		qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo);
-		qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi);
+		uwrd_lo = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER);
+		uwrd_hi = qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER);
 		uword[i] = uwrd_hi;
 		uword[i] = (uword[i] << 0x20) | uwrd_lo;
 	}
@@ -849,7 +849,7 @@ void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
 {
 	unsigned int i, ustore_addr;
 
-	qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
+	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 	uaddr |= UA_ECS;
 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 	for (i = 0; i < words_num; i++) {
@@ -890,26 +890,27 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 		return -EINVAL;
 	}
 	/* save current context */
-	qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0);
-	qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1);
-	qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
-			    &ind_lm_addr_byte0);
-	qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
-			    &ind_lm_addr_byte1);
+	ind_lm_addr0 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT);
+	ind_lm_addr1 = qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT);
+	ind_lm_addr_byte0 = qat_hal_rd_indr_csr(handle, ae, ctx,
+						INDIRECT_LM_ADDR_0_BYTE_INDEX);
+	ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx,
+						INDIRECT_LM_ADDR_1_BYTE_INDEX);
 	if (inst_num <= MAX_EXEC_INST)
 		qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
 	qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
-	qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc);
+	savpc = qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT);
 	savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
+	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	ctx_enables &= IGNORE_W1C_MASK;
-	qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc);
-	qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
-	qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl);
-	qat_hal_rd_indr_csr(handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
-			    &ind_cnt_sig);
-	qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig);
-	qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig);
+	savcc = qat_hal_rd_ae_csr(handle, ae, CC_ENABLE);
+	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
+	ctxarb_ctl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
+	ind_cnt_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
+					  FUTURE_COUNT_SIGNAL_INDIRECT);
+	ind_sig = qat_hal_rd_indr_csr(handle, ae, ctx,
+				      CTX_SIG_EVENTS_INDIRECT);
+	act_sig = qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE);
 	/* execute micro codes */
 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 	qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
@@ -927,8 +928,8 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 	if (endpc) {
 		unsigned int ctx_status;
 
-		qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT,
-				    &ctx_status);
+		ctx_status = qat_hal_rd_indr_csr(handle, ae, ctx,
+						 CTX_STS_INDIRECT);
 		*endpc = ctx_status & handle->hal_handle->upc_mask;
 	}
 	/* retore to saved context */
@@ -938,7 +939,7 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 	qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
 	qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
 			    handle->hal_handle->upc_mask & savpc);
-	qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
+	csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL);
 	newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
 	qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
 	qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
@@ -986,16 +987,16 @@ static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
 		insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
 		break;
 	}
-	qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
-	qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl);
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
+	savctx = qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS);
+	ctxarb_cntl = qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL);
+	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	ctx_enables &= IGNORE_W1C_MASK;
 	if (ctx != (savctx & ACS_ACNO))
 		qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 				  ctx & ACS_ACNO);
 	qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
-	qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
+	ustore_addr = qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS);
 	uaddr = UA_ECS;
 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 	insts = qat_hal_set_uword_ecc(insts);
@@ -1011,7 +1012,7 @@ static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
 	 * the instruction should have been executed
 	 * prior to clearing the ECS in putUwords
 	 */
-	qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data);
+	*data = qat_hal_rd_ae_csr(handle, ae, ALU_OUT);
 	qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 	qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
 	if (ctx != (savctx & ACS_ACNO))
@@ -1188,7 +1189,7 @@ static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
 	unsigned short mask;
 	unsigned short dr_offset = 0x10;
 
-	status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
+	status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	if (CE_INUSE_CONTEXTS & ctx_enables) {
 		if (ctx & 0x1) {
 			pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
@@ -1238,7 +1239,7 @@ static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
 	const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
 	const unsigned short gprnum = 0, dly = num_inst * 0x5;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
+	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	if (CE_INUSE_CONTEXTS & ctx_enables) {
 		if (ctx & 0x1) {
 			pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
@@ -1282,7 +1283,7 @@ static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
 	unsigned int ctx_enables;
 	int stat = 0;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
+	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	ctx_enables &= IGNORE_W1C_MASK;
 	qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
 
@@ -1299,7 +1300,7 @@ static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
 {
 	unsigned int ctx_enables;
 
-	qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
+	ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES);
 	if (ctx_enables & CE_INUSE_CONTEXTS) {
 		/* 4-ctx mode */
 		*relreg = absreg_num & 0x1F;
diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c
index 142c6020cec7..5d64c08b7f47 100644
--- a/drivers/crypto/s5p-sss.c
+++ b/drivers/crypto/s5p-sss.c
@@ -1,17 +1,13 @@
-/*
- * Cryptographic API.
- *
- * Support for Samsung S5PV210 and Exynos HW acceleration.
- *
- * Copyright (C) 2011 NetUP Inc. All rights reserved.
- * Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- *
- * Hash part based on omap-sham.c driver.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Cryptographic API.
+//
+// Support for Samsung S5PV210 and Exynos HW acceleration.
+//
+// Copyright (C) 2011 NetUP Inc. All rights reserved.
+// Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved.
+//
+// Hash part based on omap-sham.c driver.
 
 #include <linux/clk.h>
 #include <linux/crypto.h>
@@ -1461,7 +1457,7 @@ static void s5p_hash_tasklet_cb(unsigned long data)
 				       &dd->hash_flags)) {
 			/* hash or semi-hash ready */
 			clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags);
-				goto finish;
+			goto finish;
 		}
 	}
 
@@ -1926,15 +1922,21 @@ static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
 	uint32_t aes_control;
 	unsigned long flags;
 	int err;
+	u8 *iv;
 
 	aes_control = SSS_AES_KEY_CHANGE_MODE;
 	if (mode & FLAGS_AES_DECRYPT)
 		aes_control |= SSS_AES_MODE_DECRYPT;
 
-	if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC)
+	if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) {
 		aes_control |= SSS_AES_CHAIN_MODE_CBC;
-	else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR)
+		iv = req->info;
+	} else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) {
 		aes_control |= SSS_AES_CHAIN_MODE_CTR;
+		iv = req->info;
+	} else {
+		iv = NULL; /* AES_ECB */
+	}
 
 	if (dev->ctx->keylen == AES_KEYSIZE_192)
 		aes_control |= SSS_AES_KEY_SIZE_192;
@@ -1965,7 +1967,7 @@ static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
 		goto outdata_error;
 
 	SSS_AES_WRITE(dev, AES_CONTROL, aes_control);
-	s5p_set_aes(dev, dev->ctx->aes_key, req->info, dev->ctx->keylen);
+	s5p_set_aes(dev, dev->ctx->aes_key, iv, dev->ctx->keylen);
 
 	s5p_set_dma_indata(dev,  dev->sg_src);
 	s5p_set_dma_outdata(dev, dev->sg_dst);
diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig
index 602332e02729..63aa78c0b12b 100644
--- a/drivers/crypto/stm32/Kconfig
+++ b/drivers/crypto/stm32/Kconfig
@@ -1,4 +1,4 @@
-config CRC_DEV_STM32
+config CRYPTO_DEV_STM32_CRC
 	tristate "Support for STM32 crc accelerators"
 	depends on ARCH_STM32
 	select CRYPTO_HASH
@@ -6,7 +6,7 @@ config CRC_DEV_STM32
           This enables support for the CRC32 hw accelerator which can be found
 	  on STMicroelectronics STM32 SOC.
 
-config HASH_DEV_STM32
+config CRYPTO_DEV_STM32_HASH
 	tristate "Support for STM32 hash accelerators"
 	depends on ARCH_STM32
 	depends on HAS_DMA
@@ -18,3 +18,12 @@ config HASH_DEV_STM32
 	help
           This enables support for the HASH hw accelerator which can be found
 	  on STMicroelectronics STM32 SOC.
+
+config CRYPTO_DEV_STM32_CRYP
+	tristate "Support for STM32 cryp accelerators"
+	depends on ARCH_STM32
+	select CRYPTO_HASH
+	select CRYPTO_ENGINE
+	help
+          This enables support for the CRYP (AES/DES/TDES) hw accelerator which
+	  can be found on STMicroelectronics STM32 SOC.
diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile
index 73cd56cad0cc..53d1bb94b221 100644
--- a/drivers/crypto/stm32/Makefile
+++ b/drivers/crypto/stm32/Makefile
@@ -1,2 +1,3 @@
-obj-$(CONFIG_CRC_DEV_STM32) += stm32_crc32.o
-obj-$(CONFIG_HASH_DEV_STM32) += stm32-hash.o
\ No newline at end of file
+obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32_crc32.o
+obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o
+obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o
diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c
new file mode 100644
index 000000000000..4a06a7a665ee
--- /dev/null
+++ b/drivers/crypto/stm32/stm32-cryp.c
@@ -0,0 +1,1170 @@
+/*
+ * Copyright (C) STMicroelectronics SA 2017
+ * Author: Fabien Dessenne <fabien.dessenne@st.com>
+ * License terms:  GNU General Public License (GPL), version 2
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/engine.h>
+#include <crypto/scatterwalk.h>
+
+#define DRIVER_NAME             "stm32-cryp"
+
+/* Bit [0] encrypt / decrypt */
+#define FLG_ENCRYPT             BIT(0)
+/* Bit [8..1] algo & operation mode */
+#define FLG_AES                 BIT(1)
+#define FLG_DES                 BIT(2)
+#define FLG_TDES                BIT(3)
+#define FLG_ECB                 BIT(4)
+#define FLG_CBC                 BIT(5)
+#define FLG_CTR                 BIT(6)
+/* Mode mask = bits [15..0] */
+#define FLG_MODE_MASK           GENMASK(15, 0)
+
+/* Registers */
+#define CRYP_CR                 0x00000000
+#define CRYP_SR                 0x00000004
+#define CRYP_DIN                0x00000008
+#define CRYP_DOUT               0x0000000C
+#define CRYP_DMACR              0x00000010
+#define CRYP_IMSCR              0x00000014
+#define CRYP_RISR               0x00000018
+#define CRYP_MISR               0x0000001C
+#define CRYP_K0LR               0x00000020
+#define CRYP_K0RR               0x00000024
+#define CRYP_K1LR               0x00000028
+#define CRYP_K1RR               0x0000002C
+#define CRYP_K2LR               0x00000030
+#define CRYP_K2RR               0x00000034
+#define CRYP_K3LR               0x00000038
+#define CRYP_K3RR               0x0000003C
+#define CRYP_IV0LR              0x00000040
+#define CRYP_IV0RR              0x00000044
+#define CRYP_IV1LR              0x00000048
+#define CRYP_IV1RR              0x0000004C
+
+/* Registers values */
+#define CR_DEC_NOT_ENC          0x00000004
+#define CR_TDES_ECB             0x00000000
+#define CR_TDES_CBC             0x00000008
+#define CR_DES_ECB              0x00000010
+#define CR_DES_CBC              0x00000018
+#define CR_AES_ECB              0x00000020
+#define CR_AES_CBC              0x00000028
+#define CR_AES_CTR              0x00000030
+#define CR_AES_KP               0x00000038
+#define CR_AES_UNKNOWN          0xFFFFFFFF
+#define CR_ALGO_MASK            0x00080038
+#define CR_DATA32               0x00000000
+#define CR_DATA16               0x00000040
+#define CR_DATA8                0x00000080
+#define CR_DATA1                0x000000C0
+#define CR_KEY128               0x00000000
+#define CR_KEY192               0x00000100
+#define CR_KEY256               0x00000200
+#define CR_FFLUSH               0x00004000
+#define CR_CRYPEN               0x00008000
+
+#define SR_BUSY                 0x00000010
+#define SR_OFNE                 0x00000004
+
+#define IMSCR_IN                BIT(0)
+#define IMSCR_OUT               BIT(1)
+
+#define MISR_IN                 BIT(0)
+#define MISR_OUT                BIT(1)
+
+/* Misc */
+#define AES_BLOCK_32            (AES_BLOCK_SIZE / sizeof(u32))
+#define _walked_in              (cryp->in_walk.offset - cryp->in_sg->offset)
+#define _walked_out             (cryp->out_walk.offset - cryp->out_sg->offset)
+
+struct stm32_cryp_ctx {
+	struct stm32_cryp       *cryp;
+	int                     keylen;
+	u32                     key[AES_KEYSIZE_256 / sizeof(u32)];
+	unsigned long           flags;
+};
+
+struct stm32_cryp_reqctx {
+	unsigned long mode;
+};
+
+struct stm32_cryp {
+	struct list_head        list;
+	struct device           *dev;
+	void __iomem            *regs;
+	struct clk              *clk;
+	unsigned long           flags;
+	u32                     irq_status;
+	struct stm32_cryp_ctx   *ctx;
+
+	struct crypto_engine    *engine;
+
+	struct mutex            lock; /* protects req */
+	struct ablkcipher_request *req;
+
+	size_t                  hw_blocksize;
+
+	size_t                  total_in;
+	size_t                  total_in_save;
+	size_t                  total_out;
+	size_t                  total_out_save;
+
+	struct scatterlist      *in_sg;
+	struct scatterlist      *out_sg;
+	struct scatterlist      *out_sg_save;
+
+	struct scatterlist      in_sgl;
+	struct scatterlist      out_sgl;
+	bool                    sgs_copied;
+
+	int                     in_sg_len;
+	int                     out_sg_len;
+
+	struct scatter_walk     in_walk;
+	struct scatter_walk     out_walk;
+
+	u32                     last_ctr[4];
+};
+
+struct stm32_cryp_list {
+	struct list_head        dev_list;
+	spinlock_t              lock; /* protect dev_list */
+};
+
+static struct stm32_cryp_list cryp_list = {
+	.dev_list = LIST_HEAD_INIT(cryp_list.dev_list),
+	.lock     = __SPIN_LOCK_UNLOCKED(cryp_list.lock),
+};
+
+static inline bool is_aes(struct stm32_cryp *cryp)
+{
+	return cryp->flags & FLG_AES;
+}
+
+static inline bool is_des(struct stm32_cryp *cryp)
+{
+	return cryp->flags & FLG_DES;
+}
+
+static inline bool is_tdes(struct stm32_cryp *cryp)
+{
+	return cryp->flags & FLG_TDES;
+}
+
+static inline bool is_ecb(struct stm32_cryp *cryp)
+{
+	return cryp->flags & FLG_ECB;
+}
+
+static inline bool is_cbc(struct stm32_cryp *cryp)
+{
+	return cryp->flags & FLG_CBC;
+}
+
+static inline bool is_ctr(struct stm32_cryp *cryp)
+{
+	return cryp->flags & FLG_CTR;
+}
+
+static inline bool is_encrypt(struct stm32_cryp *cryp)
+{
+	return cryp->flags & FLG_ENCRYPT;
+}
+
+static inline bool is_decrypt(struct stm32_cryp *cryp)
+{
+	return !is_encrypt(cryp);
+}
+
+static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst)
+{
+	return readl_relaxed(cryp->regs + ofst);
+}
+
+static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val)
+{
+	writel_relaxed(val, cryp->regs + ofst);
+}
+
+static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp)
+{
+	u32 status;
+
+	return readl_relaxed_poll_timeout(cryp->regs + CRYP_SR, status,
+			!(status & SR_BUSY), 10, 100000);
+}
+
+static struct stm32_cryp *stm32_cryp_find_dev(struct stm32_cryp_ctx *ctx)
+{
+	struct stm32_cryp *tmp, *cryp = NULL;
+
+	spin_lock_bh(&cryp_list.lock);
+	if (!ctx->cryp) {
+		list_for_each_entry(tmp, &cryp_list.dev_list, list) {
+			cryp = tmp;
+			break;
+		}
+		ctx->cryp = cryp;
+	} else {
+		cryp = ctx->cryp;
+	}
+
+	spin_unlock_bh(&cryp_list.lock);
+
+	return cryp;
+}
+
+static int stm32_cryp_check_aligned(struct scatterlist *sg, size_t total,
+				    size_t align)
+{
+	int len = 0;
+
+	if (!total)
+		return 0;
+
+	if (!IS_ALIGNED(total, align))
+		return -EINVAL;
+
+	while (sg) {
+		if (!IS_ALIGNED(sg->offset, sizeof(u32)))
+			return -EINVAL;
+
+		if (!IS_ALIGNED(sg->length, align))
+			return -EINVAL;
+
+		len += sg->length;
+		sg = sg_next(sg);
+	}
+
+	if (len != total)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int stm32_cryp_check_io_aligned(struct stm32_cryp *cryp)
+{
+	int ret;
+
+	ret = stm32_cryp_check_aligned(cryp->in_sg, cryp->total_in,
+				       cryp->hw_blocksize);
+	if (ret)
+		return ret;
+
+	ret = stm32_cryp_check_aligned(cryp->out_sg, cryp->total_out,
+				       cryp->hw_blocksize);
+
+	return ret;
+}
+
+static void sg_copy_buf(void *buf, struct scatterlist *sg,
+			unsigned int start, unsigned int nbytes, int out)
+{
+	struct scatter_walk walk;
+
+	if (!nbytes)
+		return;
+
+	scatterwalk_start(&walk, sg);
+	scatterwalk_advance(&walk, start);
+	scatterwalk_copychunks(buf, &walk, nbytes, out);
+	scatterwalk_done(&walk, out, 0);
+}
+
+static int stm32_cryp_copy_sgs(struct stm32_cryp *cryp)
+{
+	void *buf_in, *buf_out;
+	int pages, total_in, total_out;
+
+	if (!stm32_cryp_check_io_aligned(cryp)) {
+		cryp->sgs_copied = 0;
+		return 0;
+	}
+
+	total_in = ALIGN(cryp->total_in, cryp->hw_blocksize);
+	pages = total_in ? get_order(total_in) : 1;
+	buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages);
+
+	total_out = ALIGN(cryp->total_out, cryp->hw_blocksize);
+	pages = total_out ? get_order(total_out) : 1;
+	buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages);
+
+	if (!buf_in || !buf_out) {
+		dev_err(cryp->dev, "Can't allocate pages when unaligned\n");
+		cryp->sgs_copied = 0;
+		return -EFAULT;
+	}
+
+	sg_copy_buf(buf_in, cryp->in_sg, 0, cryp->total_in, 0);
+
+	sg_init_one(&cryp->in_sgl, buf_in, total_in);
+	cryp->in_sg = &cryp->in_sgl;
+	cryp->in_sg_len = 1;
+
+	sg_init_one(&cryp->out_sgl, buf_out, total_out);
+	cryp->out_sg_save = cryp->out_sg;
+	cryp->out_sg = &cryp->out_sgl;
+	cryp->out_sg_len = 1;
+
+	cryp->sgs_copied = 1;
+
+	return 0;
+}
+
+static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, u32 *iv)
+{
+	if (!iv)
+		return;
+
+	stm32_cryp_write(cryp, CRYP_IV0LR, cpu_to_be32(*iv++));
+	stm32_cryp_write(cryp, CRYP_IV0RR, cpu_to_be32(*iv++));
+
+	if (is_aes(cryp)) {
+		stm32_cryp_write(cryp, CRYP_IV1LR, cpu_to_be32(*iv++));
+		stm32_cryp_write(cryp, CRYP_IV1RR, cpu_to_be32(*iv++));
+	}
+}
+
+static void stm32_cryp_hw_write_key(struct stm32_cryp *c)
+{
+	unsigned int i;
+	int r_id;
+
+	if (is_des(c)) {
+		stm32_cryp_write(c, CRYP_K1LR, cpu_to_be32(c->ctx->key[0]));
+		stm32_cryp_write(c, CRYP_K1RR, cpu_to_be32(c->ctx->key[1]));
+	} else {
+		r_id = CRYP_K3RR;
+		for (i = c->ctx->keylen / sizeof(u32); i > 0; i--, r_id -= 4)
+			stm32_cryp_write(c, r_id,
+					 cpu_to_be32(c->ctx->key[i - 1]));
+	}
+}
+
+static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp)
+{
+	if (is_aes(cryp) && is_ecb(cryp))
+		return CR_AES_ECB;
+
+	if (is_aes(cryp) && is_cbc(cryp))
+		return CR_AES_CBC;
+
+	if (is_aes(cryp) && is_ctr(cryp))
+		return CR_AES_CTR;
+
+	if (is_des(cryp) && is_ecb(cryp))
+		return CR_DES_ECB;
+
+	if (is_des(cryp) && is_cbc(cryp))
+		return CR_DES_CBC;
+
+	if (is_tdes(cryp) && is_ecb(cryp))
+		return CR_TDES_ECB;
+
+	if (is_tdes(cryp) && is_cbc(cryp))
+		return CR_TDES_CBC;
+
+	dev_err(cryp->dev, "Unknown mode\n");
+	return CR_AES_UNKNOWN;
+}
+
+static int stm32_cryp_hw_init(struct stm32_cryp *cryp)
+{
+	int ret;
+	u32 cfg, hw_mode;
+
+	/* Disable interrupt */
+	stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+
+	/* Set key */
+	stm32_cryp_hw_write_key(cryp);
+
+	/* Set configuration */
+	cfg = CR_DATA8 | CR_FFLUSH;
+
+	switch (cryp->ctx->keylen) {
+	case AES_KEYSIZE_128:
+		cfg |= CR_KEY128;
+		break;
+
+	case AES_KEYSIZE_192:
+		cfg |= CR_KEY192;
+		break;
+
+	default:
+	case AES_KEYSIZE_256:
+		cfg |= CR_KEY256;
+		break;
+	}
+
+	hw_mode = stm32_cryp_get_hw_mode(cryp);
+	if (hw_mode == CR_AES_UNKNOWN)
+		return -EINVAL;
+
+	/* AES ECB/CBC decrypt: run key preparation first */
+	if (is_decrypt(cryp) &&
+	    ((hw_mode == CR_AES_ECB) || (hw_mode == CR_AES_CBC))) {
+		stm32_cryp_write(cryp, CRYP_CR, cfg | CR_AES_KP | CR_CRYPEN);
+
+		/* Wait for end of processing */
+		ret = stm32_cryp_wait_busy(cryp);
+		if (ret) {
+			dev_err(cryp->dev, "Timeout (key preparation)\n");
+			return ret;
+		}
+	}
+
+	cfg |= hw_mode;
+
+	if (is_decrypt(cryp))
+		cfg |= CR_DEC_NOT_ENC;
+
+	/* Apply config and flush (valid when CRYPEN = 0) */
+	stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+	switch (hw_mode) {
+	case CR_DES_CBC:
+	case CR_TDES_CBC:
+	case CR_AES_CBC:
+	case CR_AES_CTR:
+		stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->req->info);
+		break;
+
+	default:
+		break;
+	}
+
+	/* Enable now */
+	cfg |= CR_CRYPEN;
+
+	stm32_cryp_write(cryp, CRYP_CR, cfg);
+
+	return 0;
+}
+
+static void stm32_cryp_finish_req(struct stm32_cryp *cryp)
+{
+	int err = 0;
+
+	if (cryp->sgs_copied) {
+		void *buf_in, *buf_out;
+		int pages, len;
+
+		buf_in = sg_virt(&cryp->in_sgl);
+		buf_out = sg_virt(&cryp->out_sgl);
+
+		sg_copy_buf(buf_out, cryp->out_sg_save, 0,
+			    cryp->total_out_save, 1);
+
+		len = ALIGN(cryp->total_in_save, cryp->hw_blocksize);
+		pages = len ? get_order(len) : 1;
+		free_pages((unsigned long)buf_in, pages);
+
+		len = ALIGN(cryp->total_out_save, cryp->hw_blocksize);
+		pages = len ? get_order(len) : 1;
+		free_pages((unsigned long)buf_out, pages);
+	}
+
+	crypto_finalize_cipher_request(cryp->engine, cryp->req, err);
+	cryp->req = NULL;
+
+	memset(cryp->ctx->key, 0, cryp->ctx->keylen);
+
+	mutex_unlock(&cryp->lock);
+}
+
+static int stm32_cryp_cpu_start(struct stm32_cryp *cryp)
+{
+	/* Enable interrupt and let the IRQ handler do everything */
+	stm32_cryp_write(cryp, CRYP_IMSCR, IMSCR_IN | IMSCR_OUT);
+
+	return 0;
+}
+
+static int stm32_cryp_cra_init(struct crypto_tfm *tfm)
+{
+	tfm->crt_ablkcipher.reqsize = sizeof(struct stm32_cryp_reqctx);
+
+	return 0;
+}
+
+static int stm32_cryp_crypt(struct ablkcipher_request *req, unsigned long mode)
+{
+	struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
+			crypto_ablkcipher_reqtfm(req));
+	struct stm32_cryp_reqctx *rctx = ablkcipher_request_ctx(req);
+	struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx);
+
+	if (!cryp)
+		return -ENODEV;
+
+	rctx->mode = mode;
+
+	return crypto_transfer_cipher_request_to_engine(cryp->engine, req);
+}
+
+static int stm32_cryp_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+			     unsigned int keylen)
+{
+	struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+	memcpy(ctx->key, key, keylen);
+	ctx->keylen = keylen;
+
+	return 0;
+}
+
+static int stm32_cryp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+				 unsigned int keylen)
+{
+	if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+	    keylen != AES_KEYSIZE_256)
+		return -EINVAL;
+	else
+		return stm32_cryp_setkey(tfm, key, keylen);
+}
+
+static int stm32_cryp_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+				 unsigned int keylen)
+{
+	if (keylen != DES_KEY_SIZE)
+		return -EINVAL;
+	else
+		return stm32_cryp_setkey(tfm, key, keylen);
+}
+
+static int stm32_cryp_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+				  unsigned int keylen)
+{
+	if (keylen != (3 * DES_KEY_SIZE))
+		return -EINVAL;
+	else
+		return stm32_cryp_setkey(tfm, key, keylen);
+}
+
+static int stm32_cryp_aes_ecb_encrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_AES | FLG_ECB | FLG_ENCRYPT);
+}
+
+static int stm32_cryp_aes_ecb_decrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_AES | FLG_ECB);
+}
+
+static int stm32_cryp_aes_cbc_encrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_AES | FLG_CBC | FLG_ENCRYPT);
+}
+
+static int stm32_cryp_aes_cbc_decrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_AES | FLG_CBC);
+}
+
+static int stm32_cryp_aes_ctr_encrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_AES | FLG_CTR | FLG_ENCRYPT);
+}
+
+static int stm32_cryp_aes_ctr_decrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_AES | FLG_CTR);
+}
+
+static int stm32_cryp_des_ecb_encrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_DES | FLG_ECB | FLG_ENCRYPT);
+}
+
+static int stm32_cryp_des_ecb_decrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_DES | FLG_ECB);
+}
+
+static int stm32_cryp_des_cbc_encrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_DES | FLG_CBC | FLG_ENCRYPT);
+}
+
+static int stm32_cryp_des_cbc_decrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_DES | FLG_CBC);
+}
+
+static int stm32_cryp_tdes_ecb_encrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB | FLG_ENCRYPT);
+}
+
+static int stm32_cryp_tdes_ecb_decrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_TDES | FLG_ECB);
+}
+
+static int stm32_cryp_tdes_cbc_encrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC | FLG_ENCRYPT);
+}
+
+static int stm32_cryp_tdes_cbc_decrypt(struct ablkcipher_request *req)
+{
+	return stm32_cryp_crypt(req, FLG_TDES | FLG_CBC);
+}
+
+static int stm32_cryp_prepare_req(struct crypto_engine *engine,
+				  struct ablkcipher_request *req)
+{
+	struct stm32_cryp_ctx *ctx;
+	struct stm32_cryp *cryp;
+	struct stm32_cryp_reqctx *rctx;
+	int ret;
+
+	if (!req)
+		return -EINVAL;
+
+	ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
+
+	cryp = ctx->cryp;
+
+	if (!cryp)
+		return -ENODEV;
+
+	mutex_lock(&cryp->lock);
+
+	rctx = ablkcipher_request_ctx(req);
+	rctx->mode &= FLG_MODE_MASK;
+
+	ctx->cryp = cryp;
+
+	cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode;
+	cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE;
+	cryp->ctx = ctx;
+
+	cryp->req = req;
+	cryp->total_in = req->nbytes;
+	cryp->total_out = cryp->total_in;
+
+	cryp->total_in_save = cryp->total_in;
+	cryp->total_out_save = cryp->total_out;
+
+	cryp->in_sg = req->src;
+	cryp->out_sg = req->dst;
+	cryp->out_sg_save = cryp->out_sg;
+
+	cryp->in_sg_len = sg_nents_for_len(cryp->in_sg, cryp->total_in);
+	if (cryp->in_sg_len < 0) {
+		dev_err(cryp->dev, "Cannot get in_sg_len\n");
+		ret = cryp->in_sg_len;
+		goto out;
+	}
+
+	cryp->out_sg_len = sg_nents_for_len(cryp->out_sg, cryp->total_out);
+	if (cryp->out_sg_len < 0) {
+		dev_err(cryp->dev, "Cannot get out_sg_len\n");
+		ret = cryp->out_sg_len;
+		goto out;
+	}
+
+	ret = stm32_cryp_copy_sgs(cryp);
+	if (ret)
+		goto out;
+
+	scatterwalk_start(&cryp->in_walk, cryp->in_sg);
+	scatterwalk_start(&cryp->out_walk, cryp->out_sg);
+
+	ret = stm32_cryp_hw_init(cryp);
+out:
+	if (ret)
+		mutex_unlock(&cryp->lock);
+
+	return ret;
+}
+
+static int stm32_cryp_prepare_cipher_req(struct crypto_engine *engine,
+					 struct ablkcipher_request *req)
+{
+	return stm32_cryp_prepare_req(engine, req);
+}
+
+static int stm32_cryp_cipher_one_req(struct crypto_engine *engine,
+				     struct ablkcipher_request *req)
+{
+	struct stm32_cryp_ctx *ctx = crypto_ablkcipher_ctx(
+			crypto_ablkcipher_reqtfm(req));
+	struct stm32_cryp *cryp = ctx->cryp;
+
+	if (!cryp)
+		return -ENODEV;
+
+	return stm32_cryp_cpu_start(cryp);
+}
+
+static u32 *stm32_cryp_next_out(struct stm32_cryp *cryp, u32 *dst,
+				unsigned int n)
+{
+	scatterwalk_advance(&cryp->out_walk, n);
+
+	if (unlikely(cryp->out_sg->length == _walked_out)) {
+		cryp->out_sg = sg_next(cryp->out_sg);
+		if (cryp->out_sg) {
+			scatterwalk_start(&cryp->out_walk, cryp->out_sg);
+			return (sg_virt(cryp->out_sg) + _walked_out);
+		}
+	}
+
+	return (u32 *)((u8 *)dst + n);
+}
+
+static u32 *stm32_cryp_next_in(struct stm32_cryp *cryp, u32 *src,
+			       unsigned int n)
+{
+	scatterwalk_advance(&cryp->in_walk, n);
+
+	if (unlikely(cryp->in_sg->length == _walked_in)) {
+		cryp->in_sg = sg_next(cryp->in_sg);
+		if (cryp->in_sg) {
+			scatterwalk_start(&cryp->in_walk, cryp->in_sg);
+			return (sg_virt(cryp->in_sg) + _walked_in);
+		}
+	}
+
+	return (u32 *)((u8 *)src + n);
+}
+
+static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp)
+{
+	u32 cr;
+
+	if (unlikely(cryp->last_ctr[3] == 0xFFFFFFFF)) {
+		cryp->last_ctr[3] = 0;
+		cryp->last_ctr[2]++;
+		if (!cryp->last_ctr[2]) {
+			cryp->last_ctr[1]++;
+			if (!cryp->last_ctr[1])
+				cryp->last_ctr[0]++;
+		}
+
+		cr = stm32_cryp_read(cryp, CRYP_CR);
+		stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN);
+
+		stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr);
+
+		stm32_cryp_write(cryp, CRYP_CR, cr);
+	}
+
+	cryp->last_ctr[0] = stm32_cryp_read(cryp, CRYP_IV0LR);
+	cryp->last_ctr[1] = stm32_cryp_read(cryp, CRYP_IV0RR);
+	cryp->last_ctr[2] = stm32_cryp_read(cryp, CRYP_IV1LR);
+	cryp->last_ctr[3] = stm32_cryp_read(cryp, CRYP_IV1RR);
+}
+
+static bool stm32_cryp_irq_read_data(struct stm32_cryp *cryp)
+{
+	unsigned int i, j;
+	u32 d32, *dst;
+	u8 *d8;
+
+	dst = sg_virt(cryp->out_sg) + _walked_out;
+
+	for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
+		if (likely(cryp->total_out >= sizeof(u32))) {
+			/* Read a full u32 */
+			*dst = stm32_cryp_read(cryp, CRYP_DOUT);
+
+			dst = stm32_cryp_next_out(cryp, dst, sizeof(u32));
+			cryp->total_out -= sizeof(u32);
+		} else if (!cryp->total_out) {
+			/* Empty fifo out (data from input padding) */
+			d32 = stm32_cryp_read(cryp, CRYP_DOUT);
+		} else {
+			/* Read less than an u32 */
+			d32 = stm32_cryp_read(cryp, CRYP_DOUT);
+			d8 = (u8 *)&d32;
+
+			for (j = 0; j < cryp->total_out; j++) {
+				*((u8 *)dst) = *(d8++);
+				dst = stm32_cryp_next_out(cryp, dst, 1);
+			}
+			cryp->total_out = 0;
+		}
+	}
+
+	return !cryp->total_out || !cryp->total_in;
+}
+
+static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp)
+{
+	unsigned int i, j;
+	u32 *src;
+	u8 d8[4];
+
+	src = sg_virt(cryp->in_sg) + _walked_in;
+
+	for (i = 0; i < cryp->hw_blocksize / sizeof(u32); i++) {
+		if (likely(cryp->total_in >= sizeof(u32))) {
+			/* Write a full u32 */
+			stm32_cryp_write(cryp, CRYP_DIN, *src);
+
+			src = stm32_cryp_next_in(cryp, src, sizeof(u32));
+			cryp->total_in -= sizeof(u32);
+		} else if (!cryp->total_in) {
+			/* Write padding data */
+			stm32_cryp_write(cryp, CRYP_DIN, 0);
+		} else {
+			/* Write less than an u32 */
+			memset(d8, 0, sizeof(u32));
+			for (j = 0; j < cryp->total_in; j++) {
+				d8[j] = *((u8 *)src);
+				src = stm32_cryp_next_in(cryp, src, 1);
+			}
+
+			stm32_cryp_write(cryp, CRYP_DIN, *(u32 *)d8);
+			cryp->total_in = 0;
+		}
+	}
+}
+
+static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp)
+{
+	if (unlikely(!cryp->total_in)) {
+		dev_warn(cryp->dev, "No more data to process\n");
+		return;
+	}
+
+	if (is_aes(cryp) && is_ctr(cryp))
+		stm32_cryp_check_ctr_counter(cryp);
+
+	stm32_cryp_irq_write_block(cryp);
+}
+
+static irqreturn_t stm32_cryp_irq_thread(int irq, void *arg)
+{
+	struct stm32_cryp *cryp = arg;
+
+	if (cryp->irq_status & MISR_OUT)
+		/* Output FIFO IRQ: read data */
+		if (unlikely(stm32_cryp_irq_read_data(cryp))) {
+			/* All bytes processed, finish */
+			stm32_cryp_write(cryp, CRYP_IMSCR, 0);
+			stm32_cryp_finish_req(cryp);
+			return IRQ_HANDLED;
+		}
+
+	if (cryp->irq_status & MISR_IN) {
+		/* Input FIFO IRQ: write data */
+		stm32_cryp_irq_write_data(cryp);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t stm32_cryp_irq(int irq, void *arg)
+{
+	struct stm32_cryp *cryp = arg;
+
+	cryp->irq_status = stm32_cryp_read(cryp, CRYP_MISR);
+
+	return IRQ_WAKE_THREAD;
+}
+
+static struct crypto_alg crypto_algs[] = {
+{
+	.cra_name		= "ecb(aes)",
+	.cra_driver_name	= "stm32-ecb-aes",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= stm32_cryp_cra_init,
+	.cra_ablkcipher = {
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.setkey		= stm32_cryp_aes_setkey,
+		.encrypt	= stm32_cryp_aes_ecb_encrypt,
+		.decrypt	= stm32_cryp_aes_ecb_decrypt,
+	}
+},
+{
+	.cra_name		= "cbc(aes)",
+	.cra_driver_name	= "stm32-cbc-aes",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= AES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= stm32_cryp_cra_init,
+	.cra_ablkcipher = {
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.ivsize		= AES_BLOCK_SIZE,
+		.setkey		= stm32_cryp_aes_setkey,
+		.encrypt	= stm32_cryp_aes_cbc_encrypt,
+		.decrypt	= stm32_cryp_aes_cbc_decrypt,
+	}
+},
+{
+	.cra_name		= "ctr(aes)",
+	.cra_driver_name	= "stm32-ctr-aes",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= 1,
+	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= stm32_cryp_cra_init,
+	.cra_ablkcipher = {
+		.min_keysize	= AES_MIN_KEY_SIZE,
+		.max_keysize	= AES_MAX_KEY_SIZE,
+		.ivsize		= AES_BLOCK_SIZE,
+		.setkey		= stm32_cryp_aes_setkey,
+		.encrypt	= stm32_cryp_aes_ctr_encrypt,
+		.decrypt	= stm32_cryp_aes_ctr_decrypt,
+	}
+},
+{
+	.cra_name		= "ecb(des)",
+	.cra_driver_name	= "stm32-ecb-des",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= DES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= stm32_cryp_cra_init,
+	.cra_ablkcipher = {
+		.min_keysize	= DES_BLOCK_SIZE,
+		.max_keysize	= DES_BLOCK_SIZE,
+		.setkey		= stm32_cryp_des_setkey,
+		.encrypt	= stm32_cryp_des_ecb_encrypt,
+		.decrypt	= stm32_cryp_des_ecb_decrypt,
+	}
+},
+{
+	.cra_name		= "cbc(des)",
+	.cra_driver_name	= "stm32-cbc-des",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= DES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= stm32_cryp_cra_init,
+	.cra_ablkcipher = {
+		.min_keysize	= DES_BLOCK_SIZE,
+		.max_keysize	= DES_BLOCK_SIZE,
+		.ivsize		= DES_BLOCK_SIZE,
+		.setkey		= stm32_cryp_des_setkey,
+		.encrypt	= stm32_cryp_des_cbc_encrypt,
+		.decrypt	= stm32_cryp_des_cbc_decrypt,
+	}
+},
+{
+	.cra_name		= "ecb(des3_ede)",
+	.cra_driver_name	= "stm32-ecb-des3",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= DES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= stm32_cryp_cra_init,
+	.cra_ablkcipher = {
+		.min_keysize	= 3 * DES_BLOCK_SIZE,
+		.max_keysize	= 3 * DES_BLOCK_SIZE,
+		.setkey		= stm32_cryp_tdes_setkey,
+		.encrypt	= stm32_cryp_tdes_ecb_encrypt,
+		.decrypt	= stm32_cryp_tdes_ecb_decrypt,
+	}
+},
+{
+	.cra_name		= "cbc(des3_ede)",
+	.cra_driver_name	= "stm32-cbc-des3",
+	.cra_priority		= 200,
+	.cra_flags		= CRYPTO_ALG_TYPE_ABLKCIPHER |
+				  CRYPTO_ALG_ASYNC,
+	.cra_blocksize		= DES_BLOCK_SIZE,
+	.cra_ctxsize		= sizeof(struct stm32_cryp_ctx),
+	.cra_alignmask		= 0xf,
+	.cra_type		= &crypto_ablkcipher_type,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= stm32_cryp_cra_init,
+	.cra_ablkcipher = {
+		.min_keysize	= 3 * DES_BLOCK_SIZE,
+		.max_keysize	= 3 * DES_BLOCK_SIZE,
+		.ivsize		= DES_BLOCK_SIZE,
+		.setkey		= stm32_cryp_tdes_setkey,
+		.encrypt	= stm32_cryp_tdes_cbc_encrypt,
+		.decrypt	= stm32_cryp_tdes_cbc_decrypt,
+	}
+},
+};
+
+static const struct of_device_id stm32_dt_ids[] = {
+	{ .compatible = "st,stm32f756-cryp", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, stm32_dt_ids);
+
+static int stm32_cryp_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct stm32_cryp *cryp;
+	struct resource *res;
+	struct reset_control *rst;
+	int irq, ret;
+
+	cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL);
+	if (!cryp)
+		return -ENOMEM;
+
+	cryp->dev = dev;
+
+	mutex_init(&cryp->lock);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	cryp->regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(cryp->regs))
+		return PTR_ERR(cryp->regs);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "Cannot get IRQ resource\n");
+		return irq;
+	}
+
+	ret = devm_request_threaded_irq(dev, irq, stm32_cryp_irq,
+					stm32_cryp_irq_thread, IRQF_ONESHOT,
+					dev_name(dev), cryp);
+	if (ret) {
+		dev_err(dev, "Cannot grab IRQ\n");
+		return ret;
+	}
+
+	cryp->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(cryp->clk)) {
+		dev_err(dev, "Could not get clock\n");
+		return PTR_ERR(cryp->clk);
+	}
+
+	ret = clk_prepare_enable(cryp->clk);
+	if (ret) {
+		dev_err(cryp->dev, "Failed to enable clock\n");
+		return ret;
+	}
+
+	rst = devm_reset_control_get(dev, NULL);
+	if (!IS_ERR(rst)) {
+		reset_control_assert(rst);
+		udelay(2);
+		reset_control_deassert(rst);
+	}
+
+	platform_set_drvdata(pdev, cryp);
+
+	spin_lock(&cryp_list.lock);
+	list_add(&cryp->list, &cryp_list.dev_list);
+	spin_unlock(&cryp_list.lock);
+
+	/* Initialize crypto engine */
+	cryp->engine = crypto_engine_alloc_init(dev, 1);
+	if (!cryp->engine) {
+		dev_err(dev, "Could not init crypto engine\n");
+		ret = -ENOMEM;
+		goto err_engine1;
+	}
+
+	cryp->engine->prepare_cipher_request = stm32_cryp_prepare_cipher_req;
+	cryp->engine->cipher_one_request = stm32_cryp_cipher_one_req;
+
+	ret = crypto_engine_start(cryp->engine);
+	if (ret) {
+		dev_err(dev, "Could not start crypto engine\n");
+		goto err_engine2;
+	}
+
+	ret = crypto_register_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
+	if (ret) {
+		dev_err(dev, "Could not register algs\n");
+		goto err_algs;
+	}
+
+	dev_info(dev, "Initialized\n");
+
+	return 0;
+
+err_algs:
+err_engine2:
+	crypto_engine_exit(cryp->engine);
+err_engine1:
+	spin_lock(&cryp_list.lock);
+	list_del(&cryp->list);
+	spin_unlock(&cryp_list.lock);
+
+	clk_disable_unprepare(cryp->clk);
+
+	return ret;
+}
+
+static int stm32_cryp_remove(struct platform_device *pdev)
+{
+	struct stm32_cryp *cryp = platform_get_drvdata(pdev);
+
+	if (!cryp)
+		return -ENODEV;
+
+	crypto_unregister_algs(crypto_algs, ARRAY_SIZE(crypto_algs));
+
+	crypto_engine_exit(cryp->engine);
+
+	spin_lock(&cryp_list.lock);
+	list_del(&cryp->list);
+	spin_unlock(&cryp_list.lock);
+
+	clk_disable_unprepare(cryp->clk);
+
+	return 0;
+}
+
+static struct platform_driver stm32_cryp_driver = {
+	.probe  = stm32_cryp_probe,
+	.remove = stm32_cryp_remove,
+	.driver = {
+		.name           = DRIVER_NAME,
+		.of_match_table = stm32_dt_ids,
+	},
+};
+
+module_platform_driver(stm32_cryp_driver);
+
+MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>");
+MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c
index 090582baecfe..8f09b8430893 100644
--- a/drivers/crypto/stm32/stm32_crc32.c
+++ b/drivers/crypto/stm32/stm32_crc32.c
@@ -208,6 +208,7 @@ static struct shash_alg algs[] = {
 			.cra_name               = "crc32",
 			.cra_driver_name        = DRIVER_NAME,
 			.cra_priority           = 200,
+			.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 			.cra_blocksize          = CHKSUM_BLOCK_SIZE,
 			.cra_alignmask          = 3,
 			.cra_ctxsize            = sizeof(struct stm32_crc_ctx),
@@ -229,6 +230,7 @@ static struct shash_alg algs[] = {
 			.cra_name               = "crc32c",
 			.cra_driver_name        = DRIVER_NAME,
 			.cra_priority           = 200,
+			.cra_flags		= CRYPTO_ALG_OPTIONAL_KEY,
 			.cra_blocksize          = CHKSUM_BLOCK_SIZE,
 			.cra_alignmask          = 3,
 			.cra_ctxsize            = sizeof(struct stm32_crc_ctx),
diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-prng.c b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c
index 0d01d1624252..63d636424161 100644
--- a/drivers/crypto/sunxi-ss/sun4i-ss-prng.c
+++ b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c
@@ -28,7 +28,7 @@ int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
 	algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng);
 	ss = algt->ss;
 
-	spin_lock(&ss->slock);
+	spin_lock_bh(&ss->slock);
 
 	writel(mode, ss->base + SS_CTL);
 
@@ -51,6 +51,6 @@ int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src,
 	}
 
 	writel(0, ss->base + SS_CTL);
-	spin_unlock(&ss->slock);
-	return dlen;
+	spin_unlock_bh(&ss->slock);
+	return 0;
 }
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index 9c80e0cb1664..6882fa2f8bad 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -1138,6 +1138,10 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src,
 	struct talitos_private *priv = dev_get_drvdata(dev);
 	bool is_sec1 = has_ftr_sec1(priv);
 
+	if (!src) {
+		to_talitos_ptr(ptr, 0, 0, is_sec1);
+		return 1;
+	}
 	if (sg_count == 1) {
 		to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
 		return sg_count;
diff --git a/drivers/dax/device.c b/drivers/dax/device.c
index 7b0bf825c4e7..2137dbc29877 100644
--- a/drivers/dax/device.c
+++ b/drivers/dax/device.c
@@ -133,7 +133,7 @@ struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 	dax_region->base = addr;
 	if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
 		kfree(dax_region);
-		return NULL;;
+		return NULL;
 	}
 
 	kref_get(&dax_region->kref);
diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c
index 8d8c852ba8f2..31b6ecce4c64 100644
--- a/drivers/dax/pmem.c
+++ b/drivers/dax/pmem.c
@@ -21,6 +21,7 @@
 struct dax_pmem {
 	struct device *dev;
 	struct percpu_ref ref;
+	struct dev_pagemap pgmap;
 	struct completion cmp;
 };
 
@@ -69,20 +70,23 @@ static int dax_pmem_probe(struct device *dev)
 	struct nd_namespace_common *ndns;
 	struct nd_dax *nd_dax = to_nd_dax(dev);
 	struct nd_pfn *nd_pfn = &nd_dax->nd_pfn;
-	struct vmem_altmap __altmap, *altmap = NULL;
 
 	ndns = nvdimm_namespace_common_probe(dev);
 	if (IS_ERR(ndns))
 		return PTR_ERR(ndns);
 	nsio = to_nd_namespace_io(&ndns->dev);
 
+	dax_pmem = devm_kzalloc(dev, sizeof(*dax_pmem), GFP_KERNEL);
+	if (!dax_pmem)
+		return -ENOMEM;
+
 	/* parse the 'pfn' info block via ->rw_bytes */
 	rc = devm_nsio_enable(dev, nsio);
 	if (rc)
 		return rc;
-	altmap = nvdimm_setup_pfn(nd_pfn, &res, &__altmap);
-	if (IS_ERR(altmap))
-		return PTR_ERR(altmap);
+	rc = nvdimm_setup_pfn(nd_pfn, &dax_pmem->pgmap);
+	if (rc)
+		return rc;
 	devm_nsio_disable(dev, nsio);
 
 	pfn_sb = nd_pfn->pfn_sb;
@@ -94,10 +98,6 @@ static int dax_pmem_probe(struct device *dev)
 		return -EBUSY;
 	}
 
-	dax_pmem = devm_kzalloc(dev, sizeof(*dax_pmem), GFP_KERNEL);
-	if (!dax_pmem)
-		return -ENOMEM;
-
 	dax_pmem->dev = dev;
 	init_completion(&dax_pmem->cmp);
 	rc = percpu_ref_init(&dax_pmem->ref, dax_pmem_percpu_release, 0,
@@ -110,7 +110,8 @@ static int dax_pmem_probe(struct device *dev)
 	if (rc)
 		return rc;
 
-	addr = devm_memremap_pages(dev, &res, &dax_pmem->ref, altmap);
+	dax_pmem->pgmap.ref = &dax_pmem->ref;
+	addr = devm_memremap_pages(dev, &dax_pmem->pgmap);
 	if (IS_ERR(addr))
 		return PTR_ERR(addr);
 
@@ -120,6 +121,7 @@ static int dax_pmem_probe(struct device *dev)
 		return rc;
 
 	/* adjust the dax_region resource to the start of data */
+	memcpy(&res, &dax_pmem->pgmap.res, sizeof(res));
 	res.start += le64_to_cpu(pfn_sb->dataoff);
 
 	rc = sscanf(dev_name(&ndns->dev), "namespace%d.%d", &region_id, &id);
diff --git a/drivers/dax/super.c b/drivers/dax/super.c
index 3ec804672601..ecdc292aa4e4 100644
--- a/drivers/dax/super.c
+++ b/drivers/dax/super.c
@@ -15,6 +15,7 @@
 #include <linux/mount.h>
 #include <linux/magic.h>
 #include <linux/genhd.h>
+#include <linux/pfn_t.h>
 #include <linux/cdev.h>
 #include <linux/hash.h>
 #include <linux/slab.h>
@@ -123,6 +124,15 @@ int __bdev_dax_supported(struct super_block *sb, int blocksize)
 		return len < 0 ? len : -EIO;
 	}
 
+	if ((IS_ENABLED(CONFIG_FS_DAX_LIMITED) && pfn_t_special(pfn))
+			|| pfn_t_devmap(pfn))
+		/* pass */;
+	else {
+		pr_debug("VFS (%s): error: dax support not enabled\n",
+				sb->s_id);
+		return -EOPNOTSUPP;
+	}
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(__bdev_dax_supported);
@@ -236,12 +246,6 @@ long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
 {
 	long avail;
 
-	/*
-	 * The device driver is allowed to sleep, in order to make the
-	 * memory directly accessible.
-	 */
-	might_sleep();
-
 	if (!dax_dev)
 		return -EOPNOTSUPP;
 
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index 78fb496ecb4e..fe2af6aa88fc 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -737,7 +737,7 @@ struct devfreq *devm_devfreq_add_device(struct device *dev,
 	devfreq = devfreq_add_device(dev, profile, governor_name, data);
 	if (IS_ERR(devfreq)) {
 		devres_free(ptr);
-		return ERR_PTR(-ENOMEM);
+		return devfreq;
 	}
 
 	*ptr = devfreq;
@@ -996,7 +996,8 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 	if (df->governor == governor) {
 		ret = 0;
 		goto out;
-	} else if (df->governor->immutable || governor->immutable) {
+	} else if ((df->governor && df->governor->immutable) ||
+					governor->immutable) {
 		ret = -EINVAL;
 		goto out;
 	}
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index bc1cb284111c..d78d5fc173dc 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -135,10 +135,10 @@ static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence)
  * Userspace can query the state of these implicitly tracked fences using poll()
  * and related system calls:
  *
- * - Checking for POLLIN, i.e. read access, can be use to query the state of the
+ * - Checking for EPOLLIN, i.e. read access, can be use to query the state of the
  *   most recent write or exclusive fence.
  *
- * - Checking for POLLOUT, i.e. write access, can be used to query the state of
+ * - Checking for EPOLLOUT, i.e. write access, can be used to query the state of
  *   all attached fences, shared and exclusive ones.
  *
  * Note that this only signals the completion of the respective fences, i.e. the
@@ -157,24 +157,24 @@ static void dma_buf_poll_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
 	spin_unlock_irqrestore(&dcb->poll->lock, flags);
 }
 
-static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
+static __poll_t dma_buf_poll(struct file *file, poll_table *poll)
 {
 	struct dma_buf *dmabuf;
 	struct reservation_object *resv;
 	struct reservation_object_list *fobj;
 	struct dma_fence *fence_excl;
-	unsigned long events;
+	__poll_t events;
 	unsigned shared_count, seq;
 
 	dmabuf = file->private_data;
 	if (!dmabuf || !dmabuf->resv)
-		return POLLERR;
+		return EPOLLERR;
 
 	resv = dmabuf->resv;
 
 	poll_wait(file, &dmabuf->poll, poll);
 
-	events = poll_requested_events(poll) & (POLLIN | POLLOUT);
+	events = poll_requested_events(poll) & (EPOLLIN | EPOLLOUT);
 	if (!events)
 		return 0;
 
@@ -193,12 +193,12 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
 		goto retry;
 	}
 
-	if (fence_excl && (!(events & POLLOUT) || shared_count == 0)) {
+	if (fence_excl && (!(events & EPOLLOUT) || shared_count == 0)) {
 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_excl;
-		unsigned long pevents = POLLIN;
+		__poll_t pevents = EPOLLIN;
 
 		if (shared_count == 0)
-			pevents |= POLLOUT;
+			pevents |= EPOLLOUT;
 
 		spin_lock_irq(&dmabuf->poll.lock);
 		if (dcb->active) {
@@ -228,19 +228,19 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
 		}
 	}
 
-	if ((events & POLLOUT) && shared_count > 0) {
+	if ((events & EPOLLOUT) && shared_count > 0) {
 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_shared;
 		int i;
 
 		/* Only queue a new callback if no event has fired yet */
 		spin_lock_irq(&dmabuf->poll.lock);
 		if (dcb->active)
-			events &= ~POLLOUT;
+			events &= ~EPOLLOUT;
 		else
-			dcb->active = POLLOUT;
+			dcb->active = EPOLLOUT;
 		spin_unlock_irq(&dmabuf->poll.lock);
 
-		if (!(events & POLLOUT))
+		if (!(events & EPOLLOUT))
 			goto out;
 
 		for (i = 0; i < shared_count; ++i) {
@@ -253,14 +253,14 @@ static unsigned int dma_buf_poll(struct file *file, poll_table *poll)
 				 *
 				 * call dma_buf_poll_cb and force a recheck!
 				 */
-				events &= ~POLLOUT;
+				events &= ~EPOLLOUT;
 				dma_buf_poll_cb(NULL, &dcb->cb);
 				break;
 			}
 			if (!dma_fence_add_callback(fence, &dcb->cb,
 						    dma_buf_poll_cb)) {
 				dma_fence_put(fence);
-				events &= ~POLLOUT;
+				events &= ~EPOLLOUT;
 				break;
 			}
 			dma_fence_put(fence);
@@ -351,13 +351,13 @@ static inline int is_dma_buf_file(struct file *file)
  *
  * 2. Userspace passes this file-descriptors to all drivers it wants this buffer
  *    to share with: First the filedescriptor is converted to a &dma_buf using
- *    dma_buf_get(). The the buffer is attached to the device using
+ *    dma_buf_get(). Then the buffer is attached to the device using
  *    dma_buf_attach().
  *
  *    Up to this stage the exporter is still free to migrate or reallocate the
  *    backing storage.
  *
- * 3. Once the buffer is attached to all devices userspace can inniate DMA
+ * 3. Once the buffer is attached to all devices userspace can initiate DMA
  *    access to the shared buffer. In the kernel this is done by calling
  *    dma_buf_map_attachment() and dma_buf_unmap_attachment().
  *
@@ -617,7 +617,7 @@ EXPORT_SYMBOL_GPL(dma_buf_detach);
  * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
  * on error. May return -EINTR if it is interrupted by a signal.
  *
- * A mapping must be unmapped again using dma_buf_map_attachment(). Note that
+ * A mapping must be unmapped by using dma_buf_unmap_attachment(). Note that
  * the underlying backing storage is pinned for as long as a mapping exists,
  * therefore users/importers should not hold onto a mapping for undue amounts of
  * time.
@@ -1179,8 +1179,7 @@ static int dma_buf_init_debugfs(void)
 
 static void dma_buf_uninit_debugfs(void)
 {
-	if (dma_buf_debugfs_dir)
-		debugfs_remove_recursive(dma_buf_debugfs_dir);
+	debugfs_remove_recursive(dma_buf_debugfs_dir);
 }
 #else
 static inline int dma_buf_init_debugfs(void)
diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c
index 0350829ba62e..dd1edfb27b61 100644
--- a/drivers/dma-buf/dma-fence-array.c
+++ b/drivers/dma-buf/dma-fence-array.c
@@ -31,6 +31,14 @@ static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence)
 	return "unbound";
 }
 
+static void irq_dma_fence_array_work(struct irq_work *wrk)
+{
+	struct dma_fence_array *array = container_of(wrk, typeof(*array), work);
+
+	dma_fence_signal(&array->base);
+	dma_fence_put(&array->base);
+}
+
 static void dma_fence_array_cb_func(struct dma_fence *f,
 				    struct dma_fence_cb *cb)
 {
@@ -39,8 +47,9 @@ static void dma_fence_array_cb_func(struct dma_fence *f,
 	struct dma_fence_array *array = array_cb->array;
 
 	if (atomic_dec_and_test(&array->num_pending))
-		dma_fence_signal(&array->base);
-	dma_fence_put(&array->base);
+		irq_work_queue(&array->work);
+	else
+		dma_fence_put(&array->base);
 }
 
 static bool dma_fence_array_enable_signaling(struct dma_fence *fence)
@@ -136,6 +145,7 @@ struct dma_fence_array *dma_fence_array_create(int num_fences,
 	spin_lock_init(&array->lock);
 	dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,
 		       context, seqno);
+	init_irq_work(&array->work, irq_dma_fence_array_work);
 
 	array->num_fences = num_fences;
 	atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index b44d9d7db347..04ebe2204c12 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -104,7 +104,8 @@ reservation_object_add_shared_inplace(struct reservation_object *obj,
 				      struct reservation_object_list *fobj,
 				      struct dma_fence *fence)
 {
-	u32 i;
+	struct dma_fence *signaled = NULL;
+	u32 i, signaled_idx;
 
 	dma_fence_get(fence);
 
@@ -126,17 +127,28 @@ reservation_object_add_shared_inplace(struct reservation_object *obj,
 			dma_fence_put(old_fence);
 			return;
 		}
+
+		if (!signaled && dma_fence_is_signaled(old_fence)) {
+			signaled = old_fence;
+			signaled_idx = i;
+		}
 	}
 
 	/*
 	 * memory barrier is added by write_seqcount_begin,
 	 * fobj->shared_count is protected by this lock too
 	 */
-	RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
-	fobj->shared_count++;
+	if (signaled) {
+		RCU_INIT_POINTER(fobj->shared[signaled_idx], fence);
+	} else {
+		RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
+		fobj->shared_count++;
+	}
 
 	write_seqcount_end(&obj->seq);
 	preempt_enable();
+
+	dma_fence_put(signaled);
 }
 
 static void
@@ -145,8 +157,7 @@ reservation_object_add_shared_replace(struct reservation_object *obj,
 				      struct reservation_object_list *fobj,
 				      struct dma_fence *fence)
 {
-	unsigned i;
-	struct dma_fence *old_fence = NULL;
+	unsigned i, j, k;
 
 	dma_fence_get(fence);
 
@@ -162,24 +173,21 @@ reservation_object_add_shared_replace(struct reservation_object *obj,
 	 * references from the old struct are carried over to
 	 * the new.
 	 */
-	fobj->shared_count = old->shared_count;
-
-	for (i = 0; i < old->shared_count; ++i) {
+	for (i = 0, j = 0, k = fobj->shared_max; i < old->shared_count; ++i) {
 		struct dma_fence *check;
 
 		check = rcu_dereference_protected(old->shared[i],
 						reservation_object_held(obj));
 
-		if (!old_fence && check->context == fence->context) {
-			old_fence = check;
-			RCU_INIT_POINTER(fobj->shared[i], fence);
-		} else
-			RCU_INIT_POINTER(fobj->shared[i], check);
-	}
-	if (!old_fence) {
-		RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
-		fobj->shared_count++;
+		if (check->context == fence->context ||
+		    dma_fence_is_signaled(check))
+			RCU_INIT_POINTER(fobj->shared[--k], check);
+		else
+			RCU_INIT_POINTER(fobj->shared[j++], check);
 	}
+	fobj->shared_count = j;
+	RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence);
+	fobj->shared_count++;
 
 done:
 	preempt_disable();
@@ -192,10 +200,18 @@ reservation_object_add_shared_replace(struct reservation_object *obj,
 	write_seqcount_end(&obj->seq);
 	preempt_enable();
 
-	if (old)
-		kfree_rcu(old, rcu);
+	if (!old)
+		return;
 
-	dma_fence_put(old_fence);
+	/* Drop the references to the signaled fences */
+	for (i = k; i < fobj->shared_max; ++i) {
+		struct dma_fence *f;
+
+		f = rcu_dereference_protected(fobj->shared[i],
+					      reservation_object_held(obj));
+		dma_fence_put(f);
+	}
+	kfree_rcu(old, rcu);
 }
 
 /**
@@ -318,7 +334,7 @@ int reservation_object_copy_fences(struct reservation_object *dst,
 				continue;
 			}
 
-			dst_list->shared[dst_list->shared_count++] = fence;
+			rcu_assign_pointer(dst_list->shared[dst_list->shared_count++], fence);
 		}
 	} else {
 		dst_list = NULL;
@@ -455,13 +471,15 @@ long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
 					 unsigned long timeout)
 {
 	struct dma_fence *fence;
-	unsigned seq, shared_count, i = 0;
+	unsigned seq, shared_count;
 	long ret = timeout ? timeout : 1;
+	int i;
 
 retry:
 	shared_count = 0;
 	seq = read_seqcount_begin(&obj->seq);
 	rcu_read_lock();
+	i = -1;
 
 	fence = rcu_dereference(obj->fence_excl);
 	if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
@@ -477,14 +495,14 @@ long reservation_object_wait_timeout_rcu(struct reservation_object *obj,
 		fence = NULL;
 	}
 
-	if (!fence && wait_all) {
+	if (wait_all) {
 		struct reservation_object_list *fobj =
 						rcu_dereference(obj->fence);
 
 		if (fobj)
 			shared_count = fobj->shared_count;
 
-		for (i = 0; i < shared_count; ++i) {
+		for (i = 0; !fence && i < shared_count; ++i) {
 			struct dma_fence *lfence = rcu_dereference(fobj->shared[i]);
 
 			if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index 03830634e141..35dd06479867 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -312,7 +312,7 @@ static int sync_file_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int sync_file_poll(struct file *file, poll_table *wait)
+static __poll_t sync_file_poll(struct file *file, poll_table *wait)
 {
 	struct sync_file *sync_file = file->private_data;
 
@@ -325,7 +325,7 @@ static unsigned int sync_file_poll(struct file *file, poll_table *wait)
 			wake_up_all(&sync_file->wq);
 	}
 
-	return dma_fence_is_signaled(sync_file->fence) ? POLLIN : 0;
+	return dma_fence_is_signaled(sync_file->fence) ? EPOLLIN : 0;
 }
 
 static long sync_file_ioctl_merge(struct sync_file *sync_file,
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index b52b0d55247e..97483df1f82e 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -2182,7 +2182,7 @@ static int pl08x_terminate_all(struct dma_chan *chan)
 	}
 	/* Dequeue jobs and free LLIs */
 	if (plchan->at) {
-		pl08x_desc_free(&plchan->at->vd);
+		vchan_terminate_vdesc(&plchan->at->vd);
 		plchan->at = NULL;
 	}
 	/* Dequeue jobs not yet fired as well */
@@ -2193,6 +2193,13 @@ static int pl08x_terminate_all(struct dma_chan *chan)
 	return 0;
 }
 
+static void pl08x_synchronize(struct dma_chan *chan)
+{
+	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
+
+	vchan_synchronize(&plchan->vc);
+}
+
 static int pl08x_pause(struct dma_chan *chan)
 {
 	struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
@@ -2773,6 +2780,7 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
 	pl08x->memcpy.device_pause = pl08x_pause;
 	pl08x->memcpy.device_resume = pl08x_resume;
 	pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
+	pl08x->memcpy.device_synchronize = pl08x_synchronize;
 	pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;
 	pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
 	pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);
@@ -2802,6 +2810,7 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
 		pl08x->slave.device_pause = pl08x_pause;
 		pl08x->slave.device_resume = pl08x_resume;
 		pl08x->slave.device_terminate_all = pl08x_terminate_all;
+		pl08x->slave.device_synchronize = pl08x_synchronize;
 		pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;
 		pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;
 		pl08x->slave.directions =
diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c
index 6204cc32d09c..847f84a41a69 100644
--- a/drivers/dma/bcm2835-dma.c
+++ b/drivers/dma/bcm2835-dma.c
@@ -812,7 +812,7 @@ static int bcm2835_dma_terminate_all(struct dma_chan *chan)
 	 * c->desc is NULL and exit.)
 	 */
 	if (c->desc) {
-		bcm2835_dma_desc_free(&c->desc->vd);
+		vchan_terminate_vdesc(&c->desc->vd);
 		c->desc = NULL;
 		bcm2835_dma_abort(c->chan_base);
 
@@ -836,6 +836,13 @@ static int bcm2835_dma_terminate_all(struct dma_chan *chan)
 	return 0;
 }
 
+static void bcm2835_dma_synchronize(struct dma_chan *chan)
+{
+	struct bcm2835_chan *c = to_bcm2835_dma_chan(chan);
+
+	vchan_synchronize(&c->vc);
+}
+
 static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id,
 				 int irq, unsigned int irq_flags)
 {
@@ -942,6 +949,7 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
 	od->ddev.device_prep_dma_memcpy = bcm2835_dma_prep_dma_memcpy;
 	od->ddev.device_config = bcm2835_dma_slave_config;
 	od->ddev.device_terminate_all = bcm2835_dma_terminate_all;
+	od->ddev.device_synchronize = bcm2835_dma_synchronize;
 	od->ddev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 	od->ddev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 	od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c
index f7e965f63274..d9bee65a18a4 100644
--- a/drivers/dma/cppi41.c
+++ b/drivers/dma/cppi41.c
@@ -934,7 +934,7 @@ static bool cpp41_dma_filter_fn(struct dma_chan *chan, void *param)
 
 	BUILD_BUG_ON(ARRAY_SIZE(am335x_usb_queues_rx) !=
 		     ARRAY_SIZE(am335x_usb_queues_tx));
-	if (WARN_ON(cchan->port_num > ARRAY_SIZE(am335x_usb_queues_rx)))
+	if (WARN_ON(cchan->port_num >= ARRAY_SIZE(am335x_usb_queues_rx)))
 		return false;
 
 	cchan->q_num = queues[cchan->port_num].submit;
diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c
index 7373b7a555ec..85820a2d69d4 100644
--- a/drivers/dma/dma-jz4780.c
+++ b/drivers/dma/dma-jz4780.c
@@ -511,7 +511,7 @@ static int jz4780_dma_terminate_all(struct dma_chan *chan)
 	/* Clear the DMA status and stop the transfer. */
 	jz4780_dma_writel(jzdma, JZ_DMA_REG_DCS(jzchan->id), 0);
 	if (jzchan->desc) {
-		jz4780_dma_desc_free(&jzchan->desc->vdesc);
+		vchan_terminate_vdesc(&jzchan->desc->vdesc);
 		jzchan->desc = NULL;
 	}
 
@@ -523,6 +523,13 @@ static int jz4780_dma_terminate_all(struct dma_chan *chan)
 	return 0;
 }
 
+static void jz4780_dma_synchronize(struct dma_chan *chan)
+{
+	struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan);
+
+	vchan_synchronize(&jzchan->vchan);
+}
+
 static int jz4780_dma_config(struct dma_chan *chan,
 	struct dma_slave_config *config)
 {
@@ -813,6 +820,7 @@ static int jz4780_dma_probe(struct platform_device *pdev)
 	dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy;
 	dd->device_config = jz4780_dma_config;
 	dd->device_terminate_all = jz4780_dma_terminate_all;
+	dd->device_synchronize = jz4780_dma_synchronize;
 	dd->device_tx_status = jz4780_dma_tx_status;
 	dd->device_issue_pending = jz4780_dma_issue_pending;
 	dd->src_addr_widths = JZ_DMA_BUSWIDTHS;
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index ec5f9d2bc820..80cc2be6483c 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -355,7 +355,7 @@ static void dmatest_callback(void *arg)
 {
 	struct dmatest_done *done = arg;
 	struct dmatest_thread *thread =
-		container_of(arg, struct dmatest_thread, done_wait);
+		container_of(done, struct dmatest_thread, test_done);
 	if (!thread->done) {
 		done->done = true;
 		wake_up_all(done->wait);
diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c
index 9364a3ed345a..948df1ab5f1a 100644
--- a/drivers/dma/edma.c
+++ b/drivers/dma/edma.c
@@ -860,11 +860,8 @@ static int edma_terminate_all(struct dma_chan *chan)
 		/* Move the cyclic channel back to default queue */
 		if (!echan->tc && echan->edesc->cyclic)
 			edma_assign_channel_eventq(echan, EVENTQ_DEFAULT);
-		/*
-		 * free the running request descriptor
-		 * since it is not in any of the vdesc lists
-		 */
-		edma_desc_free(&echan->edesc->vdesc);
+
+		vchan_terminate_vdesc(&echan->edesc->vdesc);
 		echan->edesc = NULL;
 	}
 
diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c
index 0391f930aecc..25cec9c243e1 100644
--- a/drivers/dma/img-mdc-dma.c
+++ b/drivers/dma/img-mdc-dma.c
@@ -694,7 +694,6 @@ static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
 static int mdc_terminate_all(struct dma_chan *chan)
 {
 	struct mdc_chan *mchan = to_mdc_chan(chan);
-	struct mdc_tx_desc *mdesc;
 	unsigned long flags;
 	LIST_HEAD(head);
 
@@ -703,21 +702,28 @@ static int mdc_terminate_all(struct dma_chan *chan)
 	mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
 			MDC_CONTROL_AND_STATUS);
 
-	mdesc = mchan->desc;
-	mchan->desc = NULL;
+	if (mchan->desc) {
+		vchan_terminate_vdesc(&mchan->desc->vd);
+		mchan->desc = NULL;
+	}
 	vchan_get_all_descriptors(&mchan->vc, &head);
 
 	mdc_get_new_events(mchan);
 
 	spin_unlock_irqrestore(&mchan->vc.lock, flags);
 
-	if (mdesc)
-		mdc_desc_free(&mdesc->vd);
 	vchan_dma_desc_free_list(&mchan->vc, &head);
 
 	return 0;
 }
 
+static void mdc_synchronize(struct dma_chan *chan)
+{
+	struct mdc_chan *mchan = to_mdc_chan(chan);
+
+	vchan_synchronize(&mchan->vc);
+}
+
 static int mdc_slave_config(struct dma_chan *chan,
 			    struct dma_slave_config *config)
 {
@@ -952,6 +958,7 @@ static int mdc_dma_probe(struct platform_device *pdev)
 	mdma->dma_dev.device_tx_status = mdc_tx_status;
 	mdma->dma_dev.device_issue_pending = mdc_issue_pending;
 	mdma->dma_dev.device_terminate_all = mdc_terminate_all;
+	mdma->dma_dev.device_synchronize = mdc_synchronize;
 	mdma->dma_dev.device_config = mdc_slave_config;
 
 	mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c
index 331f863c605e..715b39ae5a46 100644
--- a/drivers/dma/imx-dma.c
+++ b/drivers/dma/imx-dma.c
@@ -765,7 +765,7 @@ static int imxdma_alloc_chan_resources(struct dma_chan *chan)
 		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 		if (!desc)
 			break;
-		__memzero(&desc->desc, sizeof(struct dma_async_tx_descriptor));
+		memset(&desc->desc, 0, sizeof(struct dma_async_tx_descriptor));
 		dma_async_tx_descriptor_init(&desc->desc, chan);
 		desc->desc.tx_submit = imxdma_tx_submit;
 		/* txd.flags will be overwritten in prep funcs */
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 2184881afe76..e7db24c67030 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -1939,4 +1939,10 @@ module_platform_driver(sdma_driver);
 
 MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>");
 MODULE_DESCRIPTION("i.MX SDMA driver");
+#if IS_ENABLED(CONFIG_SOC_IMX6Q)
+MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin");
+#endif
+#if IS_ENABLED(CONFIG_SOC_IMX7D)
+MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin");
+#endif
 MODULE_LICENSE("GPL");
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
index 58d4ccd33672..8b5b23a8ace9 100644
--- a/drivers/dma/ioat/dma.c
+++ b/drivers/dma/ioat/dma.c
@@ -597,7 +597,6 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete)
 	for (i = 0; i < active && !seen_current; i++) {
 		struct dma_async_tx_descriptor *tx;
 
-		smp_read_barrier_depends();
 		prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
 		desc = ioat_get_ring_ent(ioat_chan, idx + i);
 		dump_desc_dbg(ioat_chan, desc);
@@ -715,7 +714,6 @@ static void ioat_abort_descs(struct ioatdma_chan *ioat_chan)
 	for (i = 1; i < active; i++) {
 		struct dma_async_tx_descriptor *tx;
 
-		smp_read_barrier_depends();
 		prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1));
 		desc = ioat_get_ring_ent(ioat_chan, idx + i);
 
diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c
index 01d2a750a621..26b67455208f 100644
--- a/drivers/dma/k3dma.c
+++ b/drivers/dma/k3dma.c
@@ -719,7 +719,7 @@ static int k3_dma_terminate_all(struct dma_chan *chan)
 		c->phy = NULL;
 		p->vchan = NULL;
 		if (p->ds_run) {
-			k3_dma_free_desc(&p->ds_run->vd);
+			vchan_terminate_vdesc(&p->ds_run->vd);
 			p->ds_run = NULL;
 		}
 		p->ds_done = NULL;
@@ -730,6 +730,13 @@ static int k3_dma_terminate_all(struct dma_chan *chan)
 	return 0;
 }
 
+static void k3_dma_synchronize(struct dma_chan *chan)
+{
+	struct k3_dma_chan *c = to_k3_chan(chan);
+
+	vchan_synchronize(&c->vc);
+}
+
 static int k3_dma_transfer_pause(struct dma_chan *chan)
 {
 	struct k3_dma_chan *c = to_k3_chan(chan);
@@ -868,6 +875,7 @@ static int k3_dma_probe(struct platform_device *op)
 	d->slave.device_pause = k3_dma_transfer_pause;
 	d->slave.device_resume = k3_dma_transfer_resume;
 	d->slave.device_terminate_all = k3_dma_terminate_all;
+	d->slave.device_synchronize = k3_dma_synchronize;
 	d->slave.copy_align = DMAENGINE_ALIGN_8_BYTES;
 
 	/* init virtual channel */
diff --git a/drivers/dma/mic_x100_dma.c b/drivers/dma/mic_x100_dma.c
index 5ba5714d0b7c..94d7bd7d2880 100644
--- a/drivers/dma/mic_x100_dma.c
+++ b/drivers/dma/mic_x100_dma.c
@@ -480,9 +480,7 @@ static int mic_dma_setup_irq(struct mic_dma_chan *ch)
 		to_mbus_hw_ops(ch)->request_threaded_irq(to_mbus_device(ch),
 			mic_dma_intr_handler, mic_dma_thread_fn,
 			"mic dma_channel", ch, ch->ch_num);
-	if (IS_ERR(ch->cookie))
-		return PTR_ERR(ch->cookie);
-	return 0;
+	return PTR_ERR_OR_ZERO(ch->cookie);
 }
 
 static inline void mic_dma_free_irq(struct mic_dma_chan *ch)
diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c
index f652a0e0f5a2..3548caa9e933 100644
--- a/drivers/dma/mv_xor_v2.c
+++ b/drivers/dma/mv_xor_v2.c
@@ -163,6 +163,7 @@ struct mv_xor_v2_device {
 	void __iomem *dma_base;
 	void __iomem *glob_base;
 	struct clk *clk;
+	struct clk *reg_clk;
 	struct tasklet_struct irq_tasklet;
 	struct list_head free_sw_desc;
 	struct dma_device dmadev;
@@ -749,13 +750,26 @@ static int mv_xor_v2_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	xor_dev->reg_clk = devm_clk_get(&pdev->dev, "reg");
+	if (PTR_ERR(xor_dev->reg_clk) != -ENOENT) {
+		if (!IS_ERR(xor_dev->reg_clk)) {
+			ret = clk_prepare_enable(xor_dev->reg_clk);
+			if (ret)
+				return ret;
+		} else {
+			return PTR_ERR(xor_dev->reg_clk);
+		}
+	}
+
 	xor_dev->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER)
-		return -EPROBE_DEFER;
+	if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) {
+		ret = EPROBE_DEFER;
+		goto disable_reg_clk;
+	}
 	if (!IS_ERR(xor_dev->clk)) {
 		ret = clk_prepare_enable(xor_dev->clk);
 		if (ret)
-			return ret;
+			goto disable_reg_clk;
 	}
 
 	ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1,
@@ -866,8 +880,9 @@ static int mv_xor_v2_probe(struct platform_device *pdev)
 free_msi_irqs:
 	platform_msi_domain_free_irqs(&pdev->dev);
 disable_clk:
-	if (!IS_ERR(xor_dev->clk))
-		clk_disable_unprepare(xor_dev->clk);
+	clk_disable_unprepare(xor_dev->clk);
+disable_reg_clk:
+	clk_disable_unprepare(xor_dev->reg_clk);
 	return ret;
 }
 
diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c
index f6dd849159d8..d21c19822feb 100644
--- a/drivers/dma/omap-dma.c
+++ b/drivers/dma/omap-dma.c
@@ -1311,7 +1311,7 @@ static int omap_dma_terminate_all(struct dma_chan *chan)
 	 * c->desc is NULL and exit.)
 	 */
 	if (c->desc) {
-		omap_dma_desc_free(&c->desc->vd);
+		vchan_terminate_vdesc(&c->desc->vd);
 		c->desc = NULL;
 		/* Avoid stopping the dma twice */
 		if (!c->paused)
diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c
index e3669850aef4..963cc5228d05 100644
--- a/drivers/dma/qcom/hidma.c
+++ b/drivers/dma/qcom/hidma.c
@@ -50,6 +50,7 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/of_dma.h>
+#include <linux/of_device.h>
 #include <linux/property.h>
 #include <linux/delay.h>
 #include <linux/acpi.h>
@@ -104,6 +105,10 @@ static unsigned int nr_desc_prm;
 module_param(nr_desc_prm, uint, 0644);
 MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)");
 
+enum hidma_cap {
+	HIDMA_MSI_CAP = 1,
+	HIDMA_IDENTITY_CAP,
+};
 
 /* process completed descriptors */
 static void hidma_process_completed(struct hidma_chan *mchan)
@@ -736,25 +741,12 @@ static int hidma_request_msi(struct hidma_dev *dmadev,
 #endif
 }
 
-static bool hidma_msi_capable(struct device *dev)
+static bool hidma_test_capability(struct device *dev, enum hidma_cap test_cap)
 {
-	struct acpi_device *adev = ACPI_COMPANION(dev);
-	const char *of_compat;
-	int ret = -EINVAL;
-
-	if (!adev || acpi_disabled) {
-		ret = device_property_read_string(dev, "compatible",
-						  &of_compat);
-		if (ret)
-			return false;
+	enum hidma_cap cap;
 
-		ret = strcmp(of_compat, "qcom,hidma-1.1");
-	} else {
-#ifdef CONFIG_ACPI
-		ret = strcmp(acpi_device_hid(adev), "QCOM8062");
-#endif
-	}
-	return ret == 0;
+	cap = (enum hidma_cap) device_get_match_data(dev);
+	return cap ? ((cap & test_cap) > 0) : 0;
 }
 
 static int hidma_probe(struct platform_device *pdev)
@@ -834,8 +826,7 @@ static int hidma_probe(struct platform_device *pdev)
 	 * Determine the MSI capability of the platform. Old HW doesn't
 	 * support MSI.
 	 */
-	msi = hidma_msi_capable(&pdev->dev);
-
+	msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP);
 	device_property_read_u32(&pdev->dev, "desc-count",
 				 &dmadev->nr_descriptors);
 
@@ -848,7 +839,10 @@ static int hidma_probe(struct platform_device *pdev)
 	if (!dmadev->nr_descriptors)
 		dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC;
 
-	dmadev->chidx = readl(dmadev->dev_trca + 0x28);
+	if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP))
+		dmadev->chidx = readl(dmadev->dev_trca + 0x40);
+	else
+		dmadev->chidx = readl(dmadev->dev_trca + 0x28);
 
 	/* Set DMA mask to 64 bits. */
 	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
@@ -953,7 +947,8 @@ static int hidma_remove(struct platform_device *pdev)
 #if IS_ENABLED(CONFIG_ACPI)
 static const struct acpi_device_id hidma_acpi_ids[] = {
 	{"QCOM8061"},
-	{"QCOM8062"},
+	{"QCOM8062", HIDMA_MSI_CAP},
+	{"QCOM8063", (HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP)},
 	{},
 };
 MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids);
@@ -961,7 +956,9 @@ MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids);
 
 static const struct of_device_id hidma_match[] = {
 	{.compatible = "qcom,hidma-1.0",},
-	{.compatible = "qcom,hidma-1.1",},
+	{.compatible = "qcom,hidma-1.1", .data = (void *)(HIDMA_MSI_CAP),},
+	{.compatible = "qcom,hidma-1.2",
+	 .data = (void *)(HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP),},
 	{},
 };
 MODULE_DEVICE_TABLE(of, hidma_match);
diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c
index 4999e266b2de..7c6e2ff212a2 100644
--- a/drivers/dma/qcom/hidma_ll.c
+++ b/drivers/dma/qcom/hidma_ll.c
@@ -393,6 +393,8 @@ static int hidma_ll_reset(struct hidma_lldev *lldev)
  */
 static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
 {
+	unsigned long irqflags;
+
 	if (cause & HIDMA_ERR_INT_MASK) {
 		dev_err(lldev->dev, "error 0x%x, disabling...\n",
 				cause);
@@ -410,6 +412,10 @@ static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
 		return;
 	}
 
+	spin_lock_irqsave(&lldev->lock, irqflags);
+	writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
+	spin_unlock_irqrestore(&lldev->lock, irqflags);
+
 	/*
 	 * Fine tuned for this HW...
 	 *
@@ -421,9 +427,6 @@ static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
 	 * Try to consume as many EVREs as possible.
 	 */
 	hidma_handle_tre_completion(lldev);
-
-	/* We consumed TREs or there are pending TREs or EVREs. */
-	writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
 }
 
 irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c
index 7335e2eb9b72..000c7019ca7d 100644
--- a/drivers/dma/qcom/hidma_mgmt.c
+++ b/drivers/dma/qcom/hidma_mgmt.c
@@ -17,6 +17,7 @@
 #include <linux/acpi.h>
 #include <linux/of.h>
 #include <linux/property.h>
+#include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
 #include <linux/module.h>
@@ -356,67 +357,37 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np)
 {
 	struct platform_device *pdev_parent = of_find_device_by_node(np);
 	struct platform_device_info pdevinfo;
-	struct of_phandle_args out_irq;
 	struct device_node *child;
-	struct resource *res = NULL;
-	const __be32 *cell;
-	int ret = 0, size, i, num;
-	u64 addr, addr_size;
+	struct resource *res;
+	int ret = 0;
+
+	/* allocate a resource array */
+	res = kcalloc(3, sizeof(*res), GFP_KERNEL);
+	if (!res)
+		return -ENOMEM;
 
 	for_each_available_child_of_node(np, child) {
-		struct resource *res_iter;
 		struct platform_device *new_pdev;
 
-		cell = of_get_property(child, "reg", &size);
-		if (!cell) {
-			ret = -EINVAL;
+		ret = of_address_to_resource(child, 0, &res[0]);
+		if (!ret)
 			goto out;
-		}
-
-		size /= sizeof(*cell);
-		num = size /
-			(of_n_addr_cells(child) + of_n_size_cells(child)) + 1;
 
-		/* allocate a resource array */
-		res = kcalloc(num, sizeof(*res), GFP_KERNEL);
-		if (!res) {
-			ret = -ENOMEM;
+		ret = of_address_to_resource(child, 1, &res[1]);
+		if (!ret)
 			goto out;
-		}
-
-		/* read each reg value */
-		i = 0;
-		res_iter = res;
-		while (i < size) {
-			addr = of_read_number(&cell[i],
-					      of_n_addr_cells(child));
-			i += of_n_addr_cells(child);
-
-			addr_size = of_read_number(&cell[i],
-						   of_n_size_cells(child));
-			i += of_n_size_cells(child);
-
-			res_iter->start = addr;
-			res_iter->end = res_iter->start + addr_size - 1;
-			res_iter->flags = IORESOURCE_MEM;
-			res_iter++;
-		}
 
-		ret = of_irq_parse_one(child, 0, &out_irq);
-		if (ret)
+		ret = of_irq_to_resource(child, 0, &res[2]);
+		if (ret <= 0)
 			goto out;
 
-		res_iter->start = irq_create_of_mapping(&out_irq);
-		res_iter->name = "hidma event irq";
-		res_iter->flags = IORESOURCE_IRQ;
-
 		memset(&pdevinfo, 0, sizeof(pdevinfo));
 		pdevinfo.fwnode = &child->fwnode;
 		pdevinfo.parent = pdev_parent ? &pdev_parent->dev : NULL;
 		pdevinfo.name = child->name;
 		pdevinfo.id = object_counter++;
 		pdevinfo.res = res;
-		pdevinfo.num_res = num;
+		pdevinfo.num_res = 3;
 		pdevinfo.data = NULL;
 		pdevinfo.size_data = 0;
 		pdevinfo.dma_mask = DMA_BIT_MASK(64);
@@ -434,8 +405,6 @@ static int __init hidma_mgmt_of_populate_channels(struct device_node *np)
 		 */
 		of_msi_configure(&new_pdev->dev, child);
 		of_node_put(child);
-		kfree(res);
-		res = NULL;
 	}
 out:
 	kfree(res);
diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c
index f04c4702d98b..cd92d696bcf9 100644
--- a/drivers/dma/s3c24xx-dma.c
+++ b/drivers/dma/s3c24xx-dma.c
@@ -732,7 +732,7 @@ static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
 
 	/* Dequeue current job */
 	if (s3cchan->at) {
-		s3c24xx_dma_desc_free(&s3cchan->at->vd);
+		vchan_terminate_vdesc(&s3cchan->at->vd);
 		s3cchan->at = NULL;
 	}
 
@@ -744,6 +744,13 @@ static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
 	return ret;
 }
 
+static void s3c24xx_dma_synchronize(struct dma_chan *chan)
+{
+	struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
+
+	vchan_synchronize(&s3cchan->vc);
+}
+
 static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
 {
 	/* Ensure all queued descriptors are freed */
@@ -1282,6 +1289,7 @@ static int s3c24xx_dma_probe(struct platform_device *pdev)
 	s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
 	s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
 	s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
+	s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize;
 
 	/* Initialize slave engine for SoC internal dedicated peripherals */
 	dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
@@ -1296,6 +1304,7 @@ static int s3c24xx_dma_probe(struct platform_device *pdev)
 	s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
 	s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
 	s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
+	s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize;
 	s3cdma->slave.filter.map = pdata->slave_map;
 	s3cdma->slave.filter.mapcnt = pdata->slavecnt;
 	s3cdma->slave.filter.fn = s3c24xx_dma_filter;
diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
index 2b2c7db3e480..d0cacdb0713e 100644
--- a/drivers/dma/sh/rcar-dmac.c
+++ b/drivers/dma/sh/rcar-dmac.c
@@ -10,6 +10,7 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmaengine.h>
 #include <linux/interrupt.h>
@@ -741,6 +742,41 @@ static int rcar_dmac_fill_hwdesc(struct rcar_dmac_chan *chan,
 /* -----------------------------------------------------------------------------
  * Stop and reset
  */
+static void rcar_dmac_chcr_de_barrier(struct rcar_dmac_chan *chan)
+{
+	u32 chcr;
+	unsigned int i;
+
+	/*
+	 * Ensure that the setting of the DE bit is actually 0 after
+	 * clearing it.
+	 */
+	for (i = 0; i < 1024; i++) {
+		chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
+		if (!(chcr & RCAR_DMACHCR_DE))
+			return;
+		udelay(1);
+	}
+
+	dev_err(chan->chan.device->dev, "CHCR DE check error\n");
+}
+
+static void rcar_dmac_sync_tcr(struct rcar_dmac_chan *chan)
+{
+	u32 chcr = rcar_dmac_chan_read(chan, RCAR_DMACHCR);
+
+	if (!(chcr & RCAR_DMACHCR_DE))
+		return;
+
+	/* set DE=0 and flush remaining data */
+	rcar_dmac_chan_write(chan, RCAR_DMACHCR, (chcr & ~RCAR_DMACHCR_DE));
+
+	/* make sure all remaining data was flushed */
+	rcar_dmac_chcr_de_barrier(chan);
+
+	/* back DE */
+	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);
+}
 
 static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
 {
@@ -749,6 +785,7 @@ static void rcar_dmac_chan_halt(struct rcar_dmac_chan *chan)
 	chcr &= ~(RCAR_DMACHCR_DSE | RCAR_DMACHCR_DSIE | RCAR_DMACHCR_IE |
 		  RCAR_DMACHCR_TE | RCAR_DMACHCR_DE);
 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr);
+	rcar_dmac_chcr_de_barrier(chan);
 }
 
 static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan)
@@ -880,7 +917,7 @@ rcar_dmac_chan_prep_sg(struct rcar_dmac_chan *chan, struct scatterlist *sgl,
 
 	rcar_dmac_chan_configure_desc(chan, desc);
 
-	max_chunk_size = (RCAR_DMATCR_MASK + 1) << desc->xfer_shift;
+	max_chunk_size = RCAR_DMATCR_MASK << desc->xfer_shift;
 
 	/*
 	 * Allocate and fill the transfer chunk descriptors. We own the only
@@ -1309,8 +1346,11 @@ static unsigned int rcar_dmac_chan_get_residue(struct rcar_dmac_chan *chan,
 		residue += chunk->size;
 	}
 
+	if (desc->direction == DMA_DEV_TO_MEM)
+		rcar_dmac_sync_tcr(chan);
+
 	/* Add the residue for the current chunk. */
-	residue += rcar_dmac_chan_read(chan, RCAR_DMATCR) << desc->xfer_shift;
+	residue += rcar_dmac_chan_read(chan, RCAR_DMATCRB) << desc->xfer_shift;
 
 	return residue;
 }
@@ -1481,6 +1521,8 @@ static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev)
 	if (chcr & RCAR_DMACHCR_TE)
 		mask |= RCAR_DMACHCR_DE;
 	rcar_dmac_chan_write(chan, RCAR_DMACHCR, chcr & ~mask);
+	if (mask & RCAR_DMACHCR_DE)
+		rcar_dmac_chcr_de_barrier(chan);
 
 	if (chcr & RCAR_DMACHCR_DSE)
 		ret |= rcar_dmac_isr_desc_stage_end(chan);
@@ -1615,22 +1657,6 @@ static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec,
  * Power management
  */
 
-#ifdef CONFIG_PM_SLEEP
-static int rcar_dmac_sleep_suspend(struct device *dev)
-{
-	/*
-	 * TODO: Wait for the current transfer to complete and stop the device.
-	 */
-	return 0;
-}
-
-static int rcar_dmac_sleep_resume(struct device *dev)
-{
-	/* TODO: Resume transfers, if any. */
-	return 0;
-}
-#endif
-
 #ifdef CONFIG_PM
 static int rcar_dmac_runtime_suspend(struct device *dev)
 {
@@ -1646,7 +1672,13 @@ static int rcar_dmac_runtime_resume(struct device *dev)
 #endif
 
 static const struct dev_pm_ops rcar_dmac_pm = {
-	SET_SYSTEM_SLEEP_PM_OPS(rcar_dmac_sleep_suspend, rcar_dmac_sleep_resume)
+	/*
+	 * TODO for system sleep/resume:
+	 *   - Wait for the current transfer to complete and stop the device,
+	 *   - Resume transfers, if any.
+	 */
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				     pm_runtime_force_resume)
 	SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume,
 			   NULL)
 };
diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c
index b652071a2096..b106e8a60af6 100644
--- a/drivers/dma/sprd-dma.c
+++ b/drivers/dma/sprd-dma.c
@@ -710,7 +710,7 @@ static int sprd_dma_config(struct dma_chan *chan, struct sprd_dma_desc *sdesc,
 	return 0;
 }
 
-struct dma_async_tx_descriptor *
+static struct dma_async_tx_descriptor *
 sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 			 size_t len, unsigned long flags)
 {
diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c
index d5db0f6e1ff8..b922db90939a 100644
--- a/drivers/dma/stm32-dmamux.c
+++ b/drivers/dma/stm32-dmamux.c
@@ -118,14 +118,15 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 	spin_lock_irqsave(&dmamux->lock, flags);
 	mux->chan_id = find_first_zero_bit(dmamux->dma_inuse,
 					   dmamux->dma_requests);
-	set_bit(mux->chan_id, dmamux->dma_inuse);
-	spin_unlock_irqrestore(&dmamux->lock, flags);
 
 	if (mux->chan_id == dmamux->dma_requests) {
+		spin_unlock_irqrestore(&dmamux->lock, flags);
 		dev_err(&pdev->dev, "Run out of free DMA requests\n");
 		ret = -ENOMEM;
-		goto error;
+		goto error_chan_id;
 	}
+	set_bit(mux->chan_id, dmamux->dma_inuse);
+	spin_unlock_irqrestore(&dmamux->lock, flags);
 
 	/* Look for DMA Master */
 	for (i = 1, min = 0, max = dmamux->dma_reqs[i];
@@ -173,6 +174,8 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec,
 
 error:
 	clear_bit(mux->chan_id, dmamux->dma_inuse);
+
+error_chan_id:
 	kfree(mux);
 	return ERR_PTR(ret);
 }
@@ -253,9 +256,6 @@ static int stm32_dmamux_probe(struct platform_device *pdev)
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENODEV;
-
 	iomem = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(iomem))
 		return PTR_ERR(iomem);
diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c
index b9d75a54c896..9a558e30c461 100644
--- a/drivers/dma/tegra20-apb-dma.c
+++ b/drivers/dma/tegra20-apb-dma.c
@@ -353,7 +353,8 @@ static int tegra_dma_slave_config(struct dma_chan *dc,
 	}
 
 	memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
-	if (tdc->slave_id == TEGRA_APBDMA_SLAVE_ID_INVALID) {
+	if (tdc->slave_id == TEGRA_APBDMA_SLAVE_ID_INVALID &&
+	    sconfig->device_fc) {
 		if (sconfig->slave_id > TEGRA_APBDMA_CSR_REQ_SEL_MASK)
 			return -EINVAL;
 		tdc->slave_id = sconfig->slave_id;
@@ -970,8 +971,13 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg(
 					TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
 	ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
 
-	csr |= TEGRA_APBDMA_CSR_ONCE | TEGRA_APBDMA_CSR_FLOW;
-	csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
+	csr |= TEGRA_APBDMA_CSR_ONCE;
+
+	if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) {
+		csr |= TEGRA_APBDMA_CSR_FLOW;
+		csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
+	}
+
 	if (flags & DMA_PREP_INTERRUPT)
 		csr |= TEGRA_APBDMA_CSR_IE_EOC;
 
@@ -1110,10 +1116,13 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic(
 					TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT;
 	ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32;
 
-	csr |= TEGRA_APBDMA_CSR_FLOW;
+	if (tdc->slave_id != TEGRA_APBDMA_SLAVE_ID_INVALID) {
+		csr |= TEGRA_APBDMA_CSR_FLOW;
+		csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
+	}
+
 	if (flags & DMA_PREP_INTERRUPT)
 		csr |= TEGRA_APBDMA_CSR_IE_EOC;
-	csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT;
 
 	apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1;
 
diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c
index 7df910e7c348..9272b173c746 100644
--- a/drivers/dma/ti-dma-crossbar.c
+++ b/drivers/dma/ti-dma-crossbar.c
@@ -54,7 +54,15 @@ struct ti_am335x_xbar_map {
 
 static inline void ti_am335x_xbar_write(void __iomem *iomem, int event, u8 val)
 {
-	writeb_relaxed(val, iomem + event);
+	/*
+	 * TPCC_EVT_MUX_60_63 register layout is different than the
+	 * rest, in the sense, that event 63 is mapped to lowest byte
+	 * and event 60 is mapped to highest, handle it separately.
+	 */
+	if (event >= 60 && event <= 63)
+		writeb_relaxed(val, iomem + (63 - event % 4));
+	else
+		writeb_relaxed(val, iomem + event);
 }
 
 static void ti_am335x_xbar_free(struct device *dev, void *route_data)
diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c
index 896bafb7a532..395c698edb4d 100644
--- a/drivers/dma/timb_dma.c
+++ b/drivers/dma/timb_dma.c
@@ -422,7 +422,7 @@ static int td_alloc_chan_resources(struct dma_chan *chan)
 				break;
 			else {
 				dev_err(chan2dev(chan),
-					"Couldnt allocate any descriptors\n");
+					"Couldn't allocate any descriptors\n");
 				return -ENOMEM;
 			}
 		}
diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c
index 545e97279083..88ad8ed2a8d6 100644
--- a/drivers/dma/virt-dma.c
+++ b/drivers/dma/virt-dma.c
@@ -107,10 +107,7 @@ static void vchan_complete(unsigned long arg)
 		dmaengine_desc_get_callback(&vd->tx, &cb);
 
 		list_del(&vd->node);
-		if (dmaengine_desc_test_reuse(&vd->tx))
-			list_add(&vd->node, &vc->desc_allocated);
-		else
-			vc->desc_free(vd);
+		vchan_vdesc_fini(vd);
 
 		dmaengine_desc_callback_invoke(&cb, NULL);
 	}
diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h
index 3f776a46a29c..b09b75ab0751 100644
--- a/drivers/dma/virt-dma.h
+++ b/drivers/dma/virt-dma.h
@@ -35,6 +35,7 @@ struct virt_dma_chan {
 	struct list_head desc_completed;
 
 	struct virt_dma_desc *cyclic;
+	struct virt_dma_desc *vd_terminated;
 };
 
 static inline struct virt_dma_chan *to_virt_chan(struct dma_chan *chan)
@@ -103,6 +104,20 @@ static inline void vchan_cookie_complete(struct virt_dma_desc *vd)
 	tasklet_schedule(&vc->task);
 }
 
+/**
+ * vchan_vdesc_fini - Free or reuse a descriptor
+ * @vd: virtual descriptor to free/reuse
+ */
+static inline void vchan_vdesc_fini(struct virt_dma_desc *vd)
+{
+	struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
+
+	if (dmaengine_desc_test_reuse(&vd->tx))
+		list_add(&vd->node, &vc->desc_allocated);
+	else
+		vc->desc_free(vd);
+}
+
 /**
  * vchan_cyclic_callback - report the completion of a period
  * @vd: virtual descriptor
@@ -115,6 +130,25 @@ static inline void vchan_cyclic_callback(struct virt_dma_desc *vd)
 	tasklet_schedule(&vc->task);
 }
 
+/**
+ * vchan_terminate_vdesc - Disable pending cyclic callback
+ * @vd: virtual descriptor to be terminated
+ *
+ * vc.lock must be held by caller
+ */
+static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd)
+{
+	struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan);
+
+	/* free up stuck descriptor */
+	if (vc->vd_terminated)
+		vchan_vdesc_fini(vc->vd_terminated);
+
+	vc->vd_terminated = vd;
+	if (vc->cyclic == vd)
+		vc->cyclic = NULL;
+}
+
 /**
  * vchan_next_desc - peek at the next descriptor to be processed
  * @vc: virtual channel to obtain descriptor from
@@ -168,10 +202,20 @@ static inline void vchan_free_chan_resources(struct virt_dma_chan *vc)
  * Makes sure that all scheduled or active callbacks have finished running. For
  * proper operation the caller has to ensure that no new callbacks are scheduled
  * after the invocation of this function started.
+ * Free up the terminated cyclic descriptor to prevent memory leakage.
  */
 static inline void vchan_synchronize(struct virt_dma_chan *vc)
 {
+	unsigned long flags;
+
 	tasklet_kill(&vc->task);
+
+	spin_lock_irqsave(&vc->lock, flags);
+	if (vc->vd_terminated) {
+		vchan_vdesc_fini(vc->vd_terminated);
+		vc->vd_terminated = NULL;
+	}
+	spin_unlock_irqrestore(&vc->lock, flags);
 }
 
 #endif
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index 5eef13380ca8..27b523530c4a 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -99,7 +99,9 @@
 #define XILINX_DMA_REG_FRMPTR_STS		0x0024
 #define XILINX_DMA_REG_PARK_PTR		0x0028
 #define XILINX_DMA_PARK_PTR_WR_REF_SHIFT	8
+#define XILINX_DMA_PARK_PTR_WR_REF_MASK		GENMASK(12, 8)
 #define XILINX_DMA_PARK_PTR_RD_REF_SHIFT	0
+#define XILINX_DMA_PARK_PTR_RD_REF_MASK		GENMASK(4, 0)
 #define XILINX_DMA_REG_VDMA_VERSION		0x002c
 
 /* Register Direct Mode Registers */
@@ -163,6 +165,7 @@
 #define XILINX_DMA_BD_SOP		BIT(27)
 #define XILINX_DMA_BD_EOP		BIT(26)
 #define XILINX_DMA_COALESCE_MAX		255
+#define XILINX_DMA_NUM_DESCS		255
 #define XILINX_DMA_NUM_APP_WORDS	5
 
 /* Multi-Channel DMA Descriptor offsets*/
@@ -211,8 +214,8 @@ struct xilinx_vdma_desc_hw {
  * @next_desc_msb: MSB of Next Descriptor Pointer @0x04
  * @buf_addr: Buffer address @0x08
  * @buf_addr_msb: MSB of Buffer address @0x0C
- * @pad1: Reserved @0x10
- * @pad2: Reserved @0x14
+ * @mcdma_control: Control field for mcdma @0x10
+ * @vsize_stride: Vsize and Stride field for mcdma @0x14
  * @control: Control field @0x18
  * @status: Status field @0x1C
  * @app: APP Fields @0x20 - 0x30
@@ -232,11 +235,11 @@ struct xilinx_axidma_desc_hw {
 /**
  * struct xilinx_cdma_desc_hw - Hardware Descriptor
  * @next_desc: Next Descriptor Pointer @0x00
- * @next_descmsb: Next Descriptor Pointer MSB @0x04
+ * @next_desc_msb: Next Descriptor Pointer MSB @0x04
  * @src_addr: Source address @0x08
- * @src_addrmsb: Source address MSB @0x0C
+ * @src_addr_msb: Source address MSB @0x0C
  * @dest_addr: Destination address @0x10
- * @dest_addrmsb: Destination address MSB @0x14
+ * @dest_addr_msb: Destination address MSB @0x14
  * @control: Control field @0x18
  * @status: Status field @0x1C
  */
@@ -310,6 +313,7 @@ struct xilinx_dma_tx_descriptor {
  * @pending_list: Descriptors waiting
  * @active_list: Descriptors ready to submit
  * @done_list: Complete descriptors
+ * @free_seg_list: Free descriptors
  * @common: DMA common channel
  * @desc_pool: Descriptors pool
  * @dev: The dma device
@@ -321,6 +325,7 @@ struct xilinx_dma_tx_descriptor {
  * @cyclic: Check for cyclic transfers.
  * @genlock: Support genlock mode
  * @err: Channel has errors
+ * @idle: Check for channel idle
  * @tasklet: Cleanup work after irq
  * @config: Device configuration info
  * @flush_on_fsync: Flush on Frame sync
@@ -329,9 +334,12 @@ struct xilinx_dma_tx_descriptor {
  * @desc_submitcount: Descriptor h/w submitted count
  * @residue: Residue for AXI DMA
  * @seg_v: Statically allocated segments base
+ * @seg_p: Physical allocated segments base
  * @cyclic_seg_v: Statically allocated segment base for cyclic transfers
+ * @cyclic_seg_p: Physical allocated segments base for cyclic dma
  * @start_transfer: Differentiate b/w DMA IP's transfer
  * @stop_transfer: Differentiate b/w DMA IP's quiesce
+ * @tdest: TDEST value for mcdma
  */
 struct xilinx_dma_chan {
 	struct xilinx_dma_device *xdev;
@@ -341,6 +349,7 @@ struct xilinx_dma_chan {
 	struct list_head pending_list;
 	struct list_head active_list;
 	struct list_head done_list;
+	struct list_head free_seg_list;
 	struct dma_chan common;
 	struct dma_pool *desc_pool;
 	struct device *dev;
@@ -352,6 +361,7 @@ struct xilinx_dma_chan {
 	bool cyclic;
 	bool genlock;
 	bool err;
+	bool idle;
 	struct tasklet_struct tasklet;
 	struct xilinx_vdma_config config;
 	bool flush_on_fsync;
@@ -360,18 +370,20 @@ struct xilinx_dma_chan {
 	u32 desc_submitcount;
 	u32 residue;
 	struct xilinx_axidma_tx_segment *seg_v;
+	dma_addr_t seg_p;
 	struct xilinx_axidma_tx_segment *cyclic_seg_v;
+	dma_addr_t cyclic_seg_p;
 	void (*start_transfer)(struct xilinx_dma_chan *chan);
 	int (*stop_transfer)(struct xilinx_dma_chan *chan);
 	u16 tdest;
 };
 
 /**
- * enum xdma_ip_type: DMA IP type.
+ * enum xdma_ip_type - DMA IP type.
  *
- * XDMA_TYPE_AXIDMA: Axi dma ip.
- * XDMA_TYPE_CDMA: Axi cdma ip.
- * XDMA_TYPE_VDMA: Axi vdma ip.
+ * @XDMA_TYPE_AXIDMA: Axi dma ip.
+ * @XDMA_TYPE_CDMA: Axi cdma ip.
+ * @XDMA_TYPE_VDMA: Axi vdma ip.
  *
  */
 enum xdma_ip_type {
@@ -580,18 +592,32 @@ xilinx_cdma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 static struct xilinx_axidma_tx_segment *
 xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 {
-	struct xilinx_axidma_tx_segment *segment;
-	dma_addr_t phys;
-
-	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);
-	if (!segment)
-		return NULL;
+	struct xilinx_axidma_tx_segment *segment = NULL;
+	unsigned long flags;
 
-	segment->phys = phys;
+	spin_lock_irqsave(&chan->lock, flags);
+	if (!list_empty(&chan->free_seg_list)) {
+		segment = list_first_entry(&chan->free_seg_list,
+					   struct xilinx_axidma_tx_segment,
+					   node);
+		list_del(&segment->node);
+	}
+	spin_unlock_irqrestore(&chan->lock, flags);
 
 	return segment;
 }
 
+static void xilinx_dma_clean_hw_desc(struct xilinx_axidma_desc_hw *hw)
+{
+	u32 next_desc = hw->next_desc;
+	u32 next_desc_msb = hw->next_desc_msb;
+
+	memset(hw, 0, sizeof(struct xilinx_axidma_desc_hw));
+
+	hw->next_desc = next_desc;
+	hw->next_desc_msb = next_desc_msb;
+}
+
 /**
  * xilinx_dma_free_tx_segment - Free transaction segment
  * @chan: Driver specific DMA channel
@@ -600,7 +626,9 @@ xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan,
 				struct xilinx_axidma_tx_segment *segment)
 {
-	dma_pool_free(chan->desc_pool, segment, segment->phys);
+	xilinx_dma_clean_hw_desc(&segment->hw);
+
+	list_add_tail(&segment->node, &chan->free_seg_list);
 }
 
 /**
@@ -725,16 +753,31 @@ static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan)
 static void xilinx_dma_free_chan_resources(struct dma_chan *dchan)
 {
 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
+	unsigned long flags;
 
 	dev_dbg(chan->dev, "Free all channel resources.\n");
 
 	xilinx_dma_free_descriptors(chan);
+
 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
-		xilinx_dma_free_tx_segment(chan, chan->cyclic_seg_v);
-		xilinx_dma_free_tx_segment(chan, chan->seg_v);
+		spin_lock_irqsave(&chan->lock, flags);
+		INIT_LIST_HEAD(&chan->free_seg_list);
+		spin_unlock_irqrestore(&chan->lock, flags);
+
+		/* Free memory that is allocated for BD */
+		dma_free_coherent(chan->dev, sizeof(*chan->seg_v) *
+				  XILINX_DMA_NUM_DESCS, chan->seg_v,
+				  chan->seg_p);
+
+		/* Free Memory that is allocated for cyclic DMA Mode */
+		dma_free_coherent(chan->dev, sizeof(*chan->cyclic_seg_v),
+				  chan->cyclic_seg_v, chan->cyclic_seg_p);
+	}
+
+	if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) {
+		dma_pool_destroy(chan->desc_pool);
+		chan->desc_pool = NULL;
 	}
-	dma_pool_destroy(chan->desc_pool);
-	chan->desc_pool = NULL;
 }
 
 /**
@@ -817,6 +860,7 @@ static void xilinx_dma_do_tasklet(unsigned long data)
 static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan)
 {
 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
+	int i;
 
 	/* Has this channel already been allocated? */
 	if (chan->desc_pool)
@@ -827,11 +871,30 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan)
 	 * for meeting Xilinx VDMA specification requirement.
 	 */
 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
-		chan->desc_pool = dma_pool_create("xilinx_dma_desc_pool",
-				   chan->dev,
-				   sizeof(struct xilinx_axidma_tx_segment),
-				   __alignof__(struct xilinx_axidma_tx_segment),
-				   0);
+		/* Allocate the buffer descriptors. */
+		chan->seg_v = dma_zalloc_coherent(chan->dev,
+						  sizeof(*chan->seg_v) *
+						  XILINX_DMA_NUM_DESCS,
+						  &chan->seg_p, GFP_KERNEL);
+		if (!chan->seg_v) {
+			dev_err(chan->dev,
+				"unable to allocate channel %d descriptors\n",
+				chan->id);
+			return -ENOMEM;
+		}
+
+		for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) {
+			chan->seg_v[i].hw.next_desc =
+			lower_32_bits(chan->seg_p + sizeof(*chan->seg_v) *
+				((i + 1) % XILINX_DMA_NUM_DESCS));
+			chan->seg_v[i].hw.next_desc_msb =
+			upper_32_bits(chan->seg_p + sizeof(*chan->seg_v) *
+				((i + 1) % XILINX_DMA_NUM_DESCS));
+			chan->seg_v[i].phys = chan->seg_p +
+				sizeof(*chan->seg_v) * i;
+			list_add_tail(&chan->seg_v[i].node,
+				      &chan->free_seg_list);
+		}
 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
 		chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool",
 				   chan->dev,
@@ -846,7 +909,8 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan)
 				     0);
 	}
 
-	if (!chan->desc_pool) {
+	if (!chan->desc_pool &&
+	    (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA)) {
 		dev_err(chan->dev,
 			"unable to allocate channel %d descriptor pool\n",
 			chan->id);
@@ -854,23 +918,21 @@ static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan)
 	}
 
 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
-		/*
-		 * For AXI DMA case after submitting a pending_list, keep
-		 * an extra segment allocated so that the "next descriptor"
-		 * pointer on the tail descriptor always points to a
-		 * valid descriptor, even when paused after reaching taildesc.
-		 * This way, it is possible to issue additional
-		 * transfers without halting and restarting the channel.
-		 */
-		chan->seg_v = xilinx_axidma_alloc_tx_segment(chan);
-
 		/*
 		 * For cyclic DMA mode we need to program the tail Descriptor
 		 * register with a value which is not a part of the BD chain
 		 * so allocating a desc segment during channel allocation for
 		 * programming tail descriptor.
 		 */
-		chan->cyclic_seg_v = xilinx_axidma_alloc_tx_segment(chan);
+		chan->cyclic_seg_v = dma_zalloc_coherent(chan->dev,
+					sizeof(*chan->cyclic_seg_v),
+					&chan->cyclic_seg_p, GFP_KERNEL);
+		if (!chan->cyclic_seg_v) {
+			dev_err(chan->dev,
+				"unable to allocate desc segment for cyclic DMA\n");
+			return -ENOMEM;
+		}
+		chan->cyclic_seg_v->phys = chan->cyclic_seg_p;
 	}
 
 	dma_cookie_init(dchan);
@@ -935,35 +997,11 @@ static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan,
 	return ret;
 }
 
-/**
- * xilinx_dma_is_running - Check if DMA channel is running
- * @chan: Driver specific DMA channel
- *
- * Return: '1' if running, '0' if not.
- */
-static bool xilinx_dma_is_running(struct xilinx_dma_chan *chan)
-{
-	return !(dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) &
-		 XILINX_DMA_DMASR_HALTED) &&
-		(dma_ctrl_read(chan, XILINX_DMA_REG_DMACR) &
-		 XILINX_DMA_DMACR_RUNSTOP);
-}
-
-/**
- * xilinx_dma_is_idle - Check if DMA channel is idle
- * @chan: Driver specific DMA channel
- *
- * Return: '1' if idle, '0' if not.
- */
-static bool xilinx_dma_is_idle(struct xilinx_dma_chan *chan)
-{
-	return dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) &
-		XILINX_DMA_DMASR_IDLE;
-}
-
 /**
  * xilinx_dma_stop_transfer - Halt DMA channel
  * @chan: Driver specific DMA channel
+ *
+ * Return: '0' on success and failure value on error
  */
 static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan)
 {
@@ -980,6 +1018,8 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan)
 /**
  * xilinx_cdma_stop_transfer - Wait for the current transfer to complete
  * @chan: Driver specific DMA channel
+ *
+ * Return: '0' on success and failure value on error
  */
 static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan)
 {
@@ -1022,13 +1062,16 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
 {
 	struct xilinx_vdma_config *config = &chan->config;
 	struct xilinx_dma_tx_descriptor *desc, *tail_desc;
-	u32 reg;
+	u32 reg, j;
 	struct xilinx_vdma_tx_segment *tail_segment;
 
 	/* This function was invoked with lock held */
 	if (chan->err)
 		return;
 
+	if (!chan->idle)
+		return;
+
 	if (list_empty(&chan->pending_list))
 		return;
 
@@ -1040,13 +1083,6 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
 	tail_segment = list_last_entry(&tail_desc->segments,
 				       struct xilinx_vdma_tx_segment, node);
 
-	/* If it is SG mode and hardware is busy, cannot submit */
-	if (chan->has_sg && xilinx_dma_is_running(chan) &&
-	    !xilinx_dma_is_idle(chan)) {
-		dev_dbg(chan->dev, "DMA controller still busy\n");
-		return;
-	}
-
 	/*
 	 * If hardware is idle, then all descriptors on the running lists are
 	 * done, start new transfers
@@ -1063,10 +1099,6 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
 	else
 		reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN;
 
-	/* Configure channel to allow number frame buffers */
-	dma_ctrl_write(chan, XILINX_DMA_REG_FRMSTORE,
-			chan->desc_pendingcount);
-
 	/*
 	 * With SG, start with circular mode, so that BDs can be fetched.
 	 * In direct register mode, if not parking, enable circular mode
@@ -1079,17 +1111,16 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
 
 	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
 
-	if (config->park && (config->park_frm >= 0) &&
-			(config->park_frm < chan->num_frms)) {
-		if (chan->direction == DMA_MEM_TO_DEV)
-			dma_write(chan, XILINX_DMA_REG_PARK_PTR,
-				config->park_frm <<
-					XILINX_DMA_PARK_PTR_RD_REF_SHIFT);
-		else
-			dma_write(chan, XILINX_DMA_REG_PARK_PTR,
-				config->park_frm <<
-					XILINX_DMA_PARK_PTR_WR_REF_SHIFT);
+	j = chan->desc_submitcount;
+	reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR);
+	if (chan->direction == DMA_MEM_TO_DEV) {
+		reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK;
+		reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT;
+	} else {
+		reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK;
+		reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT;
 	}
+	dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg);
 
 	/* Start the hardware */
 	xilinx_dma_start(chan);
@@ -1101,6 +1132,8 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
 	if (chan->has_sg) {
 		dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC,
 				tail_segment->phys);
+		list_splice_tail_init(&chan->pending_list, &chan->active_list);
+		chan->desc_pendingcount = 0;
 	} else {
 		struct xilinx_vdma_tx_segment *segment, *last = NULL;
 		int i = 0;
@@ -1130,19 +1163,16 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
 		vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE,
 				last->hw.stride);
 		vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize);
-	}
 
-	if (!chan->has_sg) {
-		list_del(&desc->node);
-		list_add_tail(&desc->node, &chan->active_list);
 		chan->desc_submitcount++;
 		chan->desc_pendingcount--;
+		list_del(&desc->node);
+		list_add_tail(&desc->node, &chan->active_list);
 		if (chan->desc_submitcount == chan->num_frms)
 			chan->desc_submitcount = 0;
-	} else {
-		list_splice_tail_init(&chan->pending_list, &chan->active_list);
-		chan->desc_pendingcount = 0;
 	}
+
+	chan->idle = false;
 }
 
 /**
@@ -1158,6 +1188,9 @@ static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan)
 	if (chan->err)
 		return;
 
+	if (!chan->idle)
+		return;
+
 	if (list_empty(&chan->pending_list))
 		return;
 
@@ -1176,6 +1209,12 @@ static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan)
 	}
 
 	if (chan->has_sg) {
+		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,
+			     XILINX_CDMA_CR_SGMODE);
+
+		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,
+			     XILINX_CDMA_CR_SGMODE);
+
 		xilinx_write(chan, XILINX_DMA_REG_CURDESC,
 			     head_desc->async_tx.phys);
 
@@ -1203,6 +1242,7 @@ static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan)
 
 	list_splice_tail_init(&chan->pending_list, &chan->active_list);
 	chan->desc_pendingcount = 0;
+	chan->idle = false;
 }
 
 /**
@@ -1212,7 +1252,7 @@ static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan)
 static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 {
 	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;
-	struct xilinx_axidma_tx_segment *tail_segment, *old_head, *new_head;
+	struct xilinx_axidma_tx_segment *tail_segment;
 	u32 reg;
 
 	if (chan->err)
@@ -1221,12 +1261,8 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 	if (list_empty(&chan->pending_list))
 		return;
 
-	/* If it is SG mode and hardware is busy, cannot submit */
-	if (chan->has_sg && xilinx_dma_is_running(chan) &&
-	    !xilinx_dma_is_idle(chan)) {
-		dev_dbg(chan->dev, "DMA controller still busy\n");
+	if (!chan->idle)
 		return;
-	}
 
 	head_desc = list_first_entry(&chan->pending_list,
 				     struct xilinx_dma_tx_descriptor, node);
@@ -1235,21 +1271,6 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 	tail_segment = list_last_entry(&tail_desc->segments,
 				       struct xilinx_axidma_tx_segment, node);
 
-	if (chan->has_sg && !chan->xdev->mcdma) {
-		old_head = list_first_entry(&head_desc->segments,
-					struct xilinx_axidma_tx_segment, node);
-		new_head = chan->seg_v;
-		/* Copy Buffer Descriptor fields. */
-		new_head->hw = old_head->hw;
-
-		/* Swap and save new reserve */
-		list_replace_init(&old_head->node, &new_head->node);
-		chan->seg_v = old_head;
-
-		tail_segment->hw.next_desc = chan->seg_v->phys;
-		head_desc->async_tx.phys = new_head->phys;
-	}
-
 	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
 
 	if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) {
@@ -1324,6 +1345,7 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
 
 	list_splice_tail_init(&chan->pending_list, &chan->active_list);
 	chan->desc_pendingcount = 0;
+	chan->idle = false;
 }
 
 /**
@@ -1388,6 +1410,8 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan)
 	}
 
 	chan->err = false;
+	chan->idle = true;
+	chan->desc_submitcount = 0;
 
 	return err;
 }
@@ -1469,6 +1493,7 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data)
 	if (status & XILINX_DMA_DMASR_FRM_CNT_IRQ) {
 		spin_lock(&chan->lock);
 		xilinx_dma_complete_descriptor(chan);
+		chan->idle = true;
 		chan->start_transfer(chan);
 		spin_unlock(&chan->lock);
 	}
@@ -1591,7 +1616,7 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan,
 {
 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
 	struct xilinx_dma_tx_descriptor *desc;
-	struct xilinx_vdma_tx_segment *segment, *prev = NULL;
+	struct xilinx_vdma_tx_segment *segment;
 	struct xilinx_vdma_desc_hw *hw;
 
 	if (!is_slave_direction(xt->dir))
@@ -1645,8 +1670,6 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan,
 	/* Insert the segment into the descriptor segments list. */
 	list_add_tail(&segment->node, &desc->segments);
 
-	prev = segment;
-
 	/* Link the last hardware descriptor with the first. */
 	segment = list_first_entry(&desc->segments,
 				   struct xilinx_vdma_tx_segment, node);
@@ -1733,7 +1756,7 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg(
 {
 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
 	struct xilinx_dma_tx_descriptor *desc;
-	struct xilinx_axidma_tx_segment *segment = NULL, *prev = NULL;
+	struct xilinx_axidma_tx_segment *segment = NULL;
 	u32 *app_w = (u32 *)context;
 	struct scatterlist *sg;
 	size_t copy;
@@ -1784,10 +1807,6 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg(
 					       XILINX_DMA_NUM_APP_WORDS);
 			}
 
-			if (prev)
-				prev->hw.next_desc = segment->phys;
-
-			prev = segment;
 			sg_used += copy;
 
 			/*
@@ -1801,7 +1820,6 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg(
 	segment = list_first_entry(&desc->segments,
 				   struct xilinx_axidma_tx_segment, node);
 	desc->async_tx.phys = segment->phys;
-	prev->hw.next_desc = segment->phys;
 
 	/* For the last DMA_MEM_TO_DEV transfer, set EOP */
 	if (chan->direction == DMA_MEM_TO_DEV) {
@@ -1821,11 +1839,14 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg(
 
 /**
  * xilinx_dma_prep_dma_cyclic - prepare descriptors for a DMA_SLAVE transaction
- * @chan: DMA channel
- * @sgl: scatterlist to transfer to/from
- * @sg_len: number of entries in @scatterlist
+ * @dchan: DMA channel
+ * @buf_addr: Physical address of the buffer
+ * @buf_len: Total length of the cyclic buffers
+ * @period_len: length of individual cyclic buffer
  * @direction: DMA direction
  * @flags: transfer ack flags
+ *
+ * Return: Async transaction descriptor on success and NULL on failure
  */
 static struct dma_async_tx_descriptor *xilinx_dma_prep_dma_cyclic(
 	struct dma_chan *dchan, dma_addr_t buf_addr, size_t buf_len,
@@ -2009,7 +2030,9 @@ xilinx_dma_prep_interleaved(struct dma_chan *dchan,
 
 /**
  * xilinx_dma_terminate_all - Halt the channel and free descriptors
- * @chan: Driver specific DMA Channel pointer
+ * @dchan: Driver specific DMA Channel pointer
+ *
+ * Return: '0' always.
  */
 static int xilinx_dma_terminate_all(struct dma_chan *dchan)
 {
@@ -2029,6 +2052,7 @@ static int xilinx_dma_terminate_all(struct dma_chan *dchan)
 
 	/* Remove and free all of the descriptors in the lists */
 	xilinx_dma_free_descriptors(chan);
+	chan->idle = true;
 
 	if (chan->cyclic) {
 		reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
@@ -2037,6 +2061,10 @@ static int xilinx_dma_terminate_all(struct dma_chan *dchan)
 		chan->cyclic = false;
 	}
 
+	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)
+		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,
+			     XILINX_CDMA_CR_SGMODE);
+
 	return 0;
 }
 
@@ -2323,6 +2351,7 @@ static void xdma_disable_allclks(struct xilinx_dma_device *xdev)
  *
  * @xdev: Driver specific device structure
  * @node: Device node
+ * @chan_id: DMA Channel id
  *
  * Return: '0' on success and failure value on error
  */
@@ -2344,11 +2373,18 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
 	chan->has_sg = xdev->has_sg;
 	chan->desc_pendingcount = 0x0;
 	chan->ext_addr = xdev->ext_addr;
+	/* This variable ensures that descriptors are not
+	 * Submitted when dma engine is in progress. This variable is
+	 * Added to avoid polling for a bit in the status register to
+	 * Know dma state in the driver hot path.
+	 */
+	chan->idle = true;
 
 	spin_lock_init(&chan->lock);
 	INIT_LIST_HEAD(&chan->pending_list);
 	INIT_LIST_HEAD(&chan->done_list);
 	INIT_LIST_HEAD(&chan->active_list);
+	INIT_LIST_HEAD(&chan->free_seg_list);
 
 	/* Retrieve the channel properties from the device tree */
 	has_dre = of_property_read_bool(node, "xlnx,include-dre");
@@ -2379,6 +2415,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
 		chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET;
 		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
 			chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET;
+			chan->config.park = 1;
 
 			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||
 			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S)
@@ -2395,6 +2432,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
 		chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET;
 		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
 			chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET;
+			chan->config.park = 1;
 
 			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||
 			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM)
@@ -2459,7 +2497,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
  * Return: 0 always.
  */
 static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev,
-				    struct device_node *node) {
+				    struct device_node *node)
+{
 	int ret, i, nr_channels = 1;
 
 	ret = of_property_read_u32(node, "dma-channels", &nr_channels);
@@ -2654,7 +2693,12 @@ static int xilinx_dma_probe(struct platform_device *pdev)
 		goto error;
 	}
 
-	dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n");
+	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA)
+		dev_info(&pdev->dev, "Xilinx AXI DMA Engine Driver Probed!!\n");
+	else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA)
+		dev_info(&pdev->dev, "Xilinx AXI CDMA Engine Driver Probed!!\n");
+	else
+		dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n");
 
 	return 0;
 
diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c
index 1ee1241ca797..f14645817ed8 100644
--- a/drivers/dma/xilinx/zynqmp_dma.c
+++ b/drivers/dma/xilinx/zynqmp_dma.c
@@ -23,6 +23,7 @@
 #include <linux/slab.h>
 #include <linux/clk.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/pm_runtime.h>
 
 #include "../dmaengine.h"
 
@@ -47,6 +48,7 @@
 #define ZYNQMP_DMA_SRC_START_MSB	0x15C
 #define ZYNQMP_DMA_DST_START_LSB	0x160
 #define ZYNQMP_DMA_DST_START_MSB	0x164
+#define ZYNQMP_DMA_TOTAL_BYTE		0x188
 #define ZYNQMP_DMA_RATE_CTRL		0x18C
 #define ZYNQMP_DMA_IRQ_SRC_ACCT		0x190
 #define ZYNQMP_DMA_IRQ_DST_ACCT		0x194
@@ -138,6 +140,8 @@
 #define ZYNQMP_DMA_BUS_WIDTH_64		64
 #define ZYNQMP_DMA_BUS_WIDTH_128	128
 
+#define ZDMA_PM_TIMEOUT			100
+
 #define ZYNQMP_DMA_DESC_SIZE(chan)	(chan->desc_size)
 
 #define to_chan(chan)		container_of(chan, struct zynqmp_dma_chan, \
@@ -211,8 +215,6 @@ struct zynqmp_dma_desc_sw {
  * @bus_width: Bus width
  * @src_burst_len: Source burst length
  * @dst_burst_len: Dest burst length
- * @clk_main: Pointer to main clock
- * @clk_apb: Pointer to apb clock
  */
 struct zynqmp_dma_chan {
 	struct zynqmp_dma_device *zdev;
@@ -237,8 +239,6 @@ struct zynqmp_dma_chan {
 	u32 bus_width;
 	u32 src_burst_len;
 	u32 dst_burst_len;
-	struct clk *clk_main;
-	struct clk *clk_apb;
 };
 
 /**
@@ -246,11 +246,15 @@ struct zynqmp_dma_chan {
  * @dev: Device Structure
  * @common: DMA device structure
  * @chan: Driver specific DMA channel
+ * @clk_main: Pointer to main clock
+ * @clk_apb: Pointer to apb clock
  */
 struct zynqmp_dma_device {
 	struct device *dev;
 	struct dma_device common;
 	struct zynqmp_dma_chan *chan;
+	struct clk *clk_main;
+	struct clk *clk_apb;
 };
 
 static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg,
@@ -461,7 +465,11 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan)
 {
 	struct zynqmp_dma_chan *chan = to_chan(dchan);
 	struct zynqmp_dma_desc_sw *desc;
-	int i;
+	int i, ret;
+
+	ret = pm_runtime_get_sync(chan->dev);
+	if (ret < 0)
+		return ret;
 
 	chan->sw_desc_pool = kzalloc(sizeof(*desc) * ZYNQMP_DMA_NUM_DESCS,
 				     GFP_KERNEL);
@@ -506,6 +514,7 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan)
 static void zynqmp_dma_start(struct zynqmp_dma_chan *chan)
 {
 	writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER);
+	writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);
 	chan->idle = false;
 	writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2);
 }
@@ -517,12 +526,12 @@ static void zynqmp_dma_start(struct zynqmp_dma_chan *chan)
  */
 static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status)
 {
-	u32 val;
-
+	if (status & ZYNQMP_DMA_BYTE_CNT_OVRFL)
+		writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);
 	if (status & ZYNQMP_DMA_IRQ_DST_ACCT_ERR)
-		val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
+		readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);
 	if (status & ZYNQMP_DMA_IRQ_SRC_ACCT_ERR)
-		val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);
+		readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);
 }
 
 static void zynqmp_dma_config(struct zynqmp_dma_chan *chan)
@@ -545,6 +554,8 @@ static void zynqmp_dma_config(struct zynqmp_dma_chan *chan)
  * zynqmp_dma_device_config - Zynqmp dma device configuration
  * @dchan: DMA channel
  * @config: DMA device config
+ *
+ * Return: 0 always
  */
 static int zynqmp_dma_device_config(struct dma_chan *dchan,
 				    struct dma_slave_config *config)
@@ -640,7 +651,7 @@ static void zynqmp_dma_issue_pending(struct dma_chan *dchan)
 
 /**
  * zynqmp_dma_free_descriptors - Free channel descriptors
- * @dchan: DMA channel pointer
+ * @chan: ZynqMP DMA channel pointer
  */
 static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan)
 {
@@ -664,6 +675,8 @@ static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan)
 		(2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS),
 		chan->desc_pool_v, chan->desc_pool_p);
 	kfree(chan->sw_desc_pool);
+	pm_runtime_mark_last_busy(chan->dev);
+	pm_runtime_put_autosuspend(chan->dev);
 }
 
 /**
@@ -715,7 +728,7 @@ static irqreturn_t zynqmp_dma_irq_handler(int irq, void *data)
 
 	if (status & ZYNQMP_DMA_INT_OVRFL) {
 		zynqmp_dma_handle_ovfl_int(chan, status);
-		dev_info(chan->dev, "Channel %p overflow interrupt\n", chan);
+		dev_dbg(chan->dev, "Channel %p overflow interrupt\n", chan);
 		ret = IRQ_HANDLED;
 	}
 
@@ -838,11 +851,10 @@ static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan)
 	if (!chan)
 		return;
 
-	devm_free_irq(chan->zdev->dev, chan->irq, chan);
+	if (chan->irq)
+		devm_free_irq(chan->zdev->dev, chan->irq, chan);
 	tasklet_kill(&chan->tasklet);
 	list_del(&chan->common.device_node);
-	clk_disable_unprepare(chan->clk_apb);
-	clk_disable_unprepare(chan->clk_main);
 }
 
 /**
@@ -907,30 +919,6 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev,
 			       "zynqmp-dma", chan);
 	if (err)
 		return err;
-	chan->clk_main = devm_clk_get(&pdev->dev, "clk_main");
-	if (IS_ERR(chan->clk_main)) {
-		dev_err(&pdev->dev, "main clock not found.\n");
-		return PTR_ERR(chan->clk_main);
-	}
-
-	chan->clk_apb = devm_clk_get(&pdev->dev, "clk_apb");
-	if (IS_ERR(chan->clk_apb)) {
-		dev_err(&pdev->dev, "apb clock not found.\n");
-		return PTR_ERR(chan->clk_apb);
-	}
-
-	err = clk_prepare_enable(chan->clk_main);
-	if (err) {
-		dev_err(&pdev->dev, "Unable to enable main clock.\n");
-		return err;
-	}
-
-	err = clk_prepare_enable(chan->clk_apb);
-	if (err) {
-		clk_disable_unprepare(chan->clk_main);
-		dev_err(&pdev->dev, "Unable to enable apb clock.\n");
-		return err;
-	}
 
 	chan->desc_size = sizeof(struct zynqmp_dma_desc_ll);
 	chan->idle = true;
@@ -952,6 +940,87 @@ static struct dma_chan *of_zynqmp_dma_xlate(struct of_phandle_args *dma_spec,
 	return dma_get_slave_channel(&zdev->chan->common);
 }
 
+/**
+ * zynqmp_dma_suspend - Suspend method for the driver
+ * @dev:	Address of the device structure
+ *
+ * Put the driver into low power mode.
+ * Return: 0 on success and failure value on error
+ */
+static int __maybe_unused zynqmp_dma_suspend(struct device *dev)
+{
+	if (!device_may_wakeup(dev))
+		return pm_runtime_force_suspend(dev);
+
+	return 0;
+}
+
+/**
+ * zynqmp_dma_resume - Resume from suspend
+ * @dev:	Address of the device structure
+ *
+ * Resume operation after suspend.
+ * Return: 0 on success and failure value on error
+ */
+static int __maybe_unused zynqmp_dma_resume(struct device *dev)
+{
+	if (!device_may_wakeup(dev))
+		return pm_runtime_force_resume(dev);
+
+	return 0;
+}
+
+/**
+ * zynqmp_dma_runtime_suspend - Runtime suspend method for the driver
+ * @dev:	Address of the device structure
+ *
+ * Put the driver into low power mode.
+ * Return: 0 always
+ */
+static int __maybe_unused zynqmp_dma_runtime_suspend(struct device *dev)
+{
+	struct zynqmp_dma_device *zdev = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(zdev->clk_main);
+	clk_disable_unprepare(zdev->clk_apb);
+
+	return 0;
+}
+
+/**
+ * zynqmp_dma_runtime_resume - Runtime suspend method for the driver
+ * @dev:	Address of the device structure
+ *
+ * Put the driver into low power mode.
+ * Return: 0 always
+ */
+static int __maybe_unused zynqmp_dma_runtime_resume(struct device *dev)
+{
+	struct zynqmp_dma_device *zdev = dev_get_drvdata(dev);
+	int err;
+
+	err = clk_prepare_enable(zdev->clk_main);
+	if (err) {
+		dev_err(dev, "Unable to enable main clock.\n");
+		return err;
+	}
+
+	err = clk_prepare_enable(zdev->clk_apb);
+	if (err) {
+		dev_err(dev, "Unable to enable apb clock.\n");
+		clk_disable_unprepare(zdev->clk_main);
+		return err;
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops zynqmp_dma_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(zynqmp_dma_suspend, zynqmp_dma_resume)
+	SET_RUNTIME_PM_OPS(zynqmp_dma_runtime_suspend,
+			   zynqmp_dma_runtime_resume, NULL)
+};
+
 /**
  * zynqmp_dma_probe - Driver probe function
  * @pdev: Pointer to the platform_device structure
@@ -984,12 +1053,33 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
 	p->device_config = zynqmp_dma_device_config;
 	p->dev = &pdev->dev;
 
+	zdev->clk_main = devm_clk_get(&pdev->dev, "clk_main");
+	if (IS_ERR(zdev->clk_main)) {
+		dev_err(&pdev->dev, "main clock not found.\n");
+		return PTR_ERR(zdev->clk_main);
+	}
+
+	zdev->clk_apb = devm_clk_get(&pdev->dev, "clk_apb");
+	if (IS_ERR(zdev->clk_apb)) {
+		dev_err(&pdev->dev, "apb clock not found.\n");
+		return PTR_ERR(zdev->clk_apb);
+	}
+
 	platform_set_drvdata(pdev, zdev);
+	pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT);
+	pm_runtime_use_autosuspend(zdev->dev);
+	pm_runtime_enable(zdev->dev);
+	pm_runtime_get_sync(zdev->dev);
+	if (!pm_runtime_enabled(zdev->dev)) {
+		ret = zynqmp_dma_runtime_resume(zdev->dev);
+		if (ret)
+			return ret;
+	}
 
 	ret = zynqmp_dma_chan_probe(zdev, pdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Probing channel failed\n");
-		goto free_chan_resources;
+		goto err_disable_pm;
 	}
 
 	p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);
@@ -1005,12 +1095,19 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
 		goto free_chan_resources;
 	}
 
+	pm_runtime_mark_last_busy(zdev->dev);
+	pm_runtime_put_sync_autosuspend(zdev->dev);
+
 	dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n");
 
 	return 0;
 
 free_chan_resources:
 	zynqmp_dma_chan_remove(zdev->chan);
+err_disable_pm:
+	if (!pm_runtime_enabled(zdev->dev))
+		zynqmp_dma_runtime_suspend(zdev->dev);
+	pm_runtime_disable(zdev->dev);
 	return ret;
 }
 
@@ -1028,6 +1125,9 @@ static int zynqmp_dma_remove(struct platform_device *pdev)
 	dma_async_device_unregister(&zdev->common);
 
 	zynqmp_dma_chan_remove(zdev->chan);
+	pm_runtime_disable(zdev->dev);
+	if (!pm_runtime_enabled(zdev->dev))
+		zynqmp_dma_runtime_suspend(zdev->dev);
 
 	return 0;
 }
@@ -1042,6 +1142,7 @@ static struct platform_driver zynqmp_dma_driver = {
 	.driver = {
 		.name = "xilinx-zynqmp-dma",
 		.of_match_table = zynqmp_dma_of_match,
+		.pm = &zynqmp_dma_dev_pm_ops,
 	},
 	.probe = zynqmp_dma_probe,
 	.remove = zynqmp_dma_remove,
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index 96afb2aeed18..3c4017007647 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -457,4 +457,11 @@ config EDAC_XGENE
 	  Support for error detection and correction on the
 	  APM X-Gene family of SOCs.
 
+config EDAC_TI
+	tristate "Texas Instruments DDR3 ECC Controller"
+	depends on ARCH_KEYSTONE || SOC_DRA7XX
+	help
+	  Support for error detection and correction on the
+          TI SoCs.
+
 endif # EDAC
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index 0fd9ffa63299..b54912eb39af 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -78,3 +78,4 @@ obj-$(CONFIG_EDAC_THUNDERX)		+= thunderx_edac.o
 obj-$(CONFIG_EDAC_ALTERA)		+= altera_edac.o
 obj-$(CONFIG_EDAC_SYNOPSYS)		+= synopsys_edac.o
 obj-$(CONFIG_EDAC_XGENE)		+= xgene_edac.o
+obj-$(CONFIG_EDAC_TI)			+= ti_edac.o
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c
index 8b16ec595fa7..329cb96f886f 100644
--- a/drivers/edac/amd64_edac.c
+++ b/drivers/edac/amd64_edac.c
@@ -3147,7 +3147,7 @@ static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
 	struct amd64_family_type *fam_type = NULL;
 
 	pvt->ext_model  = boot_cpu_data.x86_model >> 4;
-	pvt->stepping	= boot_cpu_data.x86_mask;
+	pvt->stepping	= boot_cpu_data.x86_stepping;
 	pvt->model	= boot_cpu_data.x86_model;
 	pvt->fam	= boot_cpu_data.x86;
 
diff --git a/drivers/edac/mv64x60_edac.c b/drivers/edac/mv64x60_edac.c
index ec5d695bbb72..3c68bb525d5d 100644
--- a/drivers/edac/mv64x60_edac.c
+++ b/drivers/edac/mv64x60_edac.c
@@ -758,7 +758,7 @@ static int mv64x60_mc_err_probe(struct platform_device *pdev)
 		/* Non-ECC RAM? */
 		printk(KERN_WARNING "%s: No ECC DIMMs discovered\n", __func__);
 		res = -ENODEV;
-		goto err2;
+		goto err;
 	}
 
 	edac_dbg(3, "init mci\n");
diff --git a/drivers/edac/octeon_edac-lmc.c b/drivers/edac/octeon_edac-lmc.c
index 9c1ffe3e912b..aeb222ca3ed1 100644
--- a/drivers/edac/octeon_edac-lmc.c
+++ b/drivers/edac/octeon_edac-lmc.c
@@ -78,6 +78,7 @@ static void octeon_lmc_edac_poll_o2(struct mem_ctl_info *mci)
 	if (!pvt->inject)
 		int_reg.u64 = cvmx_read_csr(CVMX_LMCX_INT(mci->mc_idx));
 	else {
+		int_reg.u64 = 0;
 		if (pvt->error_type == 1)
 			int_reg.s.sec_err = 1;
 		if (pvt->error_type == 2)
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index f34430f99fd8..872100215ca0 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -279,7 +279,7 @@ static const u32 correrrthrsld[] = {
  * sbridge structs
  */
 
-#define NUM_CHANNELS		4	/* Max channels per MC */
+#define NUM_CHANNELS		6	/* Max channels per MC */
 #define MAX_DIMMS		3	/* Max DIMMS per channel */
 #define KNL_MAX_CHAS		38	/* KNL max num. of Cache Home Agents */
 #define KNL_MAX_CHANNELS	6	/* KNL max num. of PCI channels */
diff --git a/drivers/edac/ti_edac.c b/drivers/edac/ti_edac.c
new file mode 100644
index 000000000000..6ac26d1b929f
--- /dev/null
+++ b/drivers/edac/ti_edac.c
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * Texas Instruments DDR3 ECC error correction and detection driver
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/init.h>
+#include <linux/edac.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/module.h>
+
+#include "edac_module.h"
+
+/* EMIF controller registers */
+#define EMIF_SDRAM_CONFIG		0x008
+#define EMIF_IRQ_STATUS			0x0ac
+#define EMIF_IRQ_ENABLE_SET		0x0b4
+#define EMIF_ECC_CTRL			0x110
+#define EMIF_1B_ECC_ERR_CNT		0x130
+#define EMIF_1B_ECC_ERR_THRSH		0x134
+#define EMIF_1B_ECC_ERR_ADDR_LOG	0x13c
+#define EMIF_2B_ECC_ERR_ADDR_LOG	0x140
+
+/* Bit definitions for EMIF_SDRAM_CONFIG */
+#define SDRAM_TYPE_SHIFT		29
+#define SDRAM_TYPE_MASK			GENMASK(31, 29)
+#define SDRAM_TYPE_DDR3			(3 << SDRAM_TYPE_SHIFT)
+#define SDRAM_TYPE_DDR2			(2 << SDRAM_TYPE_SHIFT)
+#define SDRAM_NARROW_MODE_MASK		GENMASK(15, 14)
+#define SDRAM_K2_NARROW_MODE_SHIFT	12
+#define SDRAM_K2_NARROW_MODE_MASK	GENMASK(13, 12)
+#define SDRAM_ROWSIZE_SHIFT		7
+#define SDRAM_ROWSIZE_MASK		GENMASK(9, 7)
+#define SDRAM_IBANK_SHIFT		4
+#define SDRAM_IBANK_MASK		GENMASK(6, 4)
+#define SDRAM_K2_IBANK_SHIFT		5
+#define SDRAM_K2_IBANK_MASK		GENMASK(6, 5)
+#define SDRAM_K2_EBANK_SHIFT		3
+#define SDRAM_K2_EBANK_MASK		BIT(SDRAM_K2_EBANK_SHIFT)
+#define SDRAM_PAGESIZE_SHIFT		0
+#define SDRAM_PAGESIZE_MASK		GENMASK(2, 0)
+#define SDRAM_K2_PAGESIZE_SHIFT		0
+#define SDRAM_K2_PAGESIZE_MASK		GENMASK(1, 0)
+
+#define EMIF_1B_ECC_ERR_THRSH_SHIFT	24
+
+/* IRQ bit definitions */
+#define EMIF_1B_ECC_ERR			BIT(5)
+#define EMIF_2B_ECC_ERR			BIT(4)
+#define EMIF_WR_ECC_ERR			BIT(3)
+#define EMIF_SYS_ERR			BIT(0)
+/* Bit 31 enables ECC and 28 enables RMW */
+#define ECC_ENABLED			(BIT(31) | BIT(28))
+
+#define EDAC_MOD_NAME			"ti-emif-edac"
+
+enum {
+	EMIF_TYPE_DRA7,
+	EMIF_TYPE_K2
+};
+
+struct ti_edac {
+	void __iomem *reg;
+};
+
+static u32 ti_edac_readl(struct ti_edac *edac, u16 offset)
+{
+	return readl_relaxed(edac->reg + offset);
+}
+
+static void ti_edac_writel(struct ti_edac *edac, u32 val, u16 offset)
+{
+	writel_relaxed(val, edac->reg + offset);
+}
+
+static irqreturn_t ti_edac_isr(int irq, void *data)
+{
+	struct mem_ctl_info *mci = data;
+	struct ti_edac *edac = mci->pvt_info;
+	u32 irq_status;
+	u32 err_addr;
+	int err_count;
+
+	irq_status = ti_edac_readl(edac, EMIF_IRQ_STATUS);
+
+	if (irq_status & EMIF_1B_ECC_ERR) {
+		err_addr = ti_edac_readl(edac, EMIF_1B_ECC_ERR_ADDR_LOG);
+		err_count = ti_edac_readl(edac, EMIF_1B_ECC_ERR_CNT);
+		ti_edac_writel(edac, err_count, EMIF_1B_ECC_ERR_CNT);
+		edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, err_count,
+				     err_addr >> PAGE_SHIFT,
+				     err_addr & ~PAGE_MASK, -1, 0, 0, 0,
+				     mci->ctl_name, "1B");
+	}
+
+	if (irq_status & EMIF_2B_ECC_ERR) {
+		err_addr = ti_edac_readl(edac, EMIF_2B_ECC_ERR_ADDR_LOG);
+		edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
+				     err_addr >> PAGE_SHIFT,
+				     err_addr & ~PAGE_MASK, -1, 0, 0, 0,
+				     mci->ctl_name, "2B");
+	}
+
+	if (irq_status & EMIF_WR_ECC_ERR)
+		edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
+				     0, 0, -1, 0, 0, 0,
+				     mci->ctl_name, "WR");
+
+	ti_edac_writel(edac, irq_status, EMIF_IRQ_STATUS);
+
+	return IRQ_HANDLED;
+}
+
+static void ti_edac_setup_dimm(struct mem_ctl_info *mci, u32 type)
+{
+	struct dimm_info *dimm;
+	struct ti_edac *edac = mci->pvt_info;
+	int bits;
+	u32 val;
+	u32 memsize;
+
+	dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, mci->n_layers, 0, 0, 0);
+
+	val = ti_edac_readl(edac, EMIF_SDRAM_CONFIG);
+
+	if (type == EMIF_TYPE_DRA7) {
+		bits = ((val & SDRAM_PAGESIZE_MASK) >> SDRAM_PAGESIZE_SHIFT) + 8;
+		bits += ((val & SDRAM_ROWSIZE_MASK) >> SDRAM_ROWSIZE_SHIFT) + 9;
+		bits += (val & SDRAM_IBANK_MASK) >> SDRAM_IBANK_SHIFT;
+
+		if (val & SDRAM_NARROW_MODE_MASK) {
+			bits++;
+			dimm->dtype = DEV_X16;
+		} else {
+			bits += 2;
+			dimm->dtype = DEV_X32;
+		}
+	} else {
+		bits = 16;
+		bits += ((val & SDRAM_K2_PAGESIZE_MASK) >>
+			SDRAM_K2_PAGESIZE_SHIFT) + 8;
+		bits += (val & SDRAM_K2_IBANK_MASK) >> SDRAM_K2_IBANK_SHIFT;
+		bits += (val & SDRAM_K2_EBANK_MASK) >> SDRAM_K2_EBANK_SHIFT;
+
+		val = (val & SDRAM_K2_NARROW_MODE_MASK) >>
+			SDRAM_K2_NARROW_MODE_SHIFT;
+		switch (val) {
+		case 0:
+			bits += 3;
+			dimm->dtype = DEV_X64;
+			break;
+		case 1:
+			bits += 2;
+			dimm->dtype = DEV_X32;
+			break;
+		case 2:
+			bits++;
+			dimm->dtype = DEV_X16;
+			break;
+		}
+	}
+
+	memsize = 1 << bits;
+
+	dimm->nr_pages = memsize >> PAGE_SHIFT;
+	dimm->grain = 4;
+	if ((val & SDRAM_TYPE_MASK) == SDRAM_TYPE_DDR2)
+		dimm->mtype = MEM_DDR2;
+	else
+		dimm->mtype = MEM_DDR3;
+
+	val = ti_edac_readl(edac, EMIF_ECC_CTRL);
+	if (val & ECC_ENABLED)
+		dimm->edac_mode = EDAC_SECDED;
+	else
+		dimm->edac_mode = EDAC_NONE;
+}
+
+static const struct of_device_id ti_edac_of_match[] = {
+	{ .compatible = "ti,emif-keystone", .data = (void *)EMIF_TYPE_K2 },
+	{ .compatible = "ti,emif-dra7xx", .data = (void *)EMIF_TYPE_DRA7 },
+	{},
+};
+
+static int _emif_get_id(struct device_node *node)
+{
+	struct device_node *np;
+	const __be32 *addrp;
+	u32 addr, my_addr;
+	int my_id = 0;
+
+	addrp = of_get_address(node, 0, NULL, NULL);
+	my_addr = (u32)of_translate_address(node, addrp);
+
+	for_each_matching_node(np, ti_edac_of_match) {
+		if (np == node)
+			continue;
+
+		addrp = of_get_address(np, 0, NULL, NULL);
+		addr = (u32)of_translate_address(np, addrp);
+
+		edac_printk(KERN_INFO, EDAC_MOD_NAME,
+			    "addr=%x, my_addr=%x\n",
+			    addr, my_addr);
+
+		if (addr < my_addr)
+			my_id++;
+	}
+
+	return my_id;
+}
+
+static int ti_edac_probe(struct platform_device *pdev)
+{
+	int error_irq = 0, ret = -ENODEV;
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	void __iomem *reg;
+	struct mem_ctl_info *mci;
+	struct edac_mc_layer layers[1];
+	const struct of_device_id *id;
+	struct ti_edac *edac;
+	int emif_id;
+
+	id = of_match_device(ti_edac_of_match, &pdev->dev);
+	if (!id)
+		return -ENODEV;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	reg = devm_ioremap_resource(dev, res);
+	if (IS_ERR(reg)) {
+		edac_printk(KERN_ERR, EDAC_MOD_NAME,
+			    "EMIF controller regs not defined\n");
+		return PTR_ERR(reg);
+	}
+
+	layers[0].type = EDAC_MC_LAYER_ALL_MEM;
+	layers[0].size = 1;
+
+	/* Allocate ID number for our EMIF controller */
+	emif_id = _emif_get_id(pdev->dev.of_node);
+	if (emif_id < 0)
+		return -EINVAL;
+
+	mci = edac_mc_alloc(emif_id, 1, layers, sizeof(*edac));
+	if (!mci)
+		return -ENOMEM;
+
+	mci->pdev = &pdev->dev;
+	edac = mci->pvt_info;
+	edac->reg = reg;
+	platform_set_drvdata(pdev, mci);
+
+	mci->mtype_cap = MEM_FLAG_DDR3 | MEM_FLAG_DDR2;
+	mci->edac_ctl_cap = EDAC_FLAG_SECDED | EDAC_FLAG_NONE;
+	mci->mod_name = EDAC_MOD_NAME;
+	mci->ctl_name = id->compatible;
+	mci->dev_name = dev_name(&pdev->dev);
+
+	/* Setup memory layout */
+	ti_edac_setup_dimm(mci, (u32)(id->data));
+
+	/* add EMIF ECC error handler */
+	error_irq = platform_get_irq(pdev, 0);
+	if (!error_irq) {
+		edac_printk(KERN_ERR, EDAC_MOD_NAME,
+			    "EMIF irq number not defined.\n");
+		goto err;
+	}
+
+	ret = devm_request_irq(dev, error_irq, ti_edac_isr, 0,
+			       "emif-edac-irq", mci);
+	if (ret) {
+		edac_printk(KERN_ERR, EDAC_MOD_NAME,
+			    "request_irq fail for EMIF EDAC irq\n");
+		goto err;
+	}
+
+	ret = edac_mc_add_mc(mci);
+	if (ret) {
+		edac_printk(KERN_ERR, EDAC_MOD_NAME,
+			    "Failed to register mci: %d.\n", ret);
+		goto err;
+	}
+
+	/* Generate an interrupt with each 1b error */
+	ti_edac_writel(edac, 1 << EMIF_1B_ECC_ERR_THRSH_SHIFT,
+		       EMIF_1B_ECC_ERR_THRSH);
+
+	/* Enable interrupts */
+	ti_edac_writel(edac,
+		       EMIF_1B_ECC_ERR | EMIF_2B_ECC_ERR | EMIF_WR_ECC_ERR,
+		       EMIF_IRQ_ENABLE_SET);
+
+	return 0;
+
+err:
+	edac_mc_free(mci);
+	return ret;
+}
+
+static int ti_edac_remove(struct platform_device *pdev)
+{
+	struct mem_ctl_info *mci = platform_get_drvdata(pdev);
+
+	edac_mc_del_mc(&pdev->dev);
+	edac_mc_free(mci);
+
+	return 0;
+}
+
+static struct platform_driver ti_edac_driver = {
+	.probe = ti_edac_probe,
+	.remove = ti_edac_remove,
+	.driver = {
+		   .name = EDAC_MOD_NAME,
+		   .of_match_table = ti_edac_of_match,
+	},
+};
+
+module_platform_driver(ti_edac_driver);
+
+MODULE_AUTHOR("Texas Instruments Inc.");
+MODULE_DESCRIPTION("EDAC Driver for Texas Instruments DDR3 MC");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/eisa/eisa-bus.c b/drivers/eisa/eisa-bus.c
index 612afeaec3cb..1e8062f6dbfc 100644
--- a/drivers/eisa/eisa-bus.c
+++ b/drivers/eisa/eisa-bus.c
@@ -75,9 +75,9 @@ static void __init eisa_name_device(struct eisa_device *edev)
 
 static char __init *decode_eisa_sig(unsigned long addr)
 {
-        static char sig_str[EISA_SIG_LEN];
+	static char sig_str[EISA_SIG_LEN];
 	u8 sig[4];
-        u16 rev;
+	u16 rev;
 	int i;
 
 	for (i = 0; i < 4; i++) {
@@ -96,14 +96,14 @@ static char __init *decode_eisa_sig(unsigned long addr)
 		if (!i && (sig[0] & 0x80))
 			return NULL;
 	}
-	
-        sig_str[0] = ((sig[0] >> 2) & 0x1f) + ('A' - 1);
-        sig_str[1] = (((sig[0] & 3) << 3) | (sig[1] >> 5)) + ('A' - 1);
-        sig_str[2] = (sig[1] & 0x1f) + ('A' - 1);
-        rev = (sig[2] << 8) | sig[3];
-        sprintf(sig_str + 3, "%04X", rev);
-
-        return sig_str;
+
+	sig_str[0] = ((sig[0] >> 2) & 0x1f) + ('A' - 1);
+	sig_str[1] = (((sig[0] & 3) << 3) | (sig[1] >> 5)) + ('A' - 1);
+	sig_str[2] = (sig[1] & 0x1f) + ('A' - 1);
+	rev = (sig[2] << 8) | sig[3];
+	sprintf(sig_str + 3, "%04X", rev);
+
+	return sig_str;
 }
 
 static int eisa_bus_match(struct device *dev, struct device_driver *drv)
@@ -198,7 +198,7 @@ static int __init eisa_init_device(struct eisa_root_device *root,
 	sig = decode_eisa_sig(sig_addr);
 	if (!sig)
 		return -1;	/* No EISA device here */
-	
+
 	memcpy(edev->id.sig, sig, EISA_SIG_LEN);
 	edev->slot = slot;
 	edev->state = inb(SLOT_ADDRESS(root, slot) + EISA_CONFIG_OFFSET)
@@ -222,7 +222,7 @@ static int __init eisa_init_device(struct eisa_root_device *root,
 
 	if (is_forced_dev(enable_dev, enable_dev_count, root, edev))
 		edev->state = EISA_CONFIG_ENABLED | EISA_CONFIG_FORCED;
-	
+
 	if (is_forced_dev(disable_dev, disable_dev_count, root, edev))
 		edev->state = EISA_CONFIG_FORCED;
 
@@ -275,7 +275,7 @@ static int __init eisa_request_resources(struct eisa_root_device *root,
 			edev->res[i].start = edev->res[i].end = 0;
 			continue;
 		}
-		
+
 		if (slot) {
 			edev->res[i].name  = NULL;
 			edev->res[i].start = SLOT_ADDRESS(root, slot)
@@ -295,7 +295,7 @@ static int __init eisa_request_resources(struct eisa_root_device *root,
 	}
 
 	return 0;
-	
+
  failed:
 	while (--i >= 0)
 		release_resource(&edev->res[i]);
@@ -314,7 +314,7 @@ static void __init eisa_release_resources(struct eisa_device *edev)
 
 static int __init eisa_probe(struct eisa_root_device *root)
 {
-        int i, c;
+	int i, c;
 	struct eisa_device *edev;
 	char *enabled_str;
 
@@ -322,16 +322,14 @@ static int __init eisa_probe(struct eisa_root_device *root)
 
 	/* First try to get hold of slot 0. If there is no device
 	 * here, simply fail, unless root->force_probe is set. */
-	
+
 	edev = kzalloc(sizeof(*edev), GFP_KERNEL);
-	if (!edev) {
-		dev_err(root->dev, "EISA: Couldn't allocate mainboard slot\n");
+	if (!edev)
 		return -ENOMEM;
-	}
-		
+
 	if (eisa_request_resources(root, edev, 0)) {
 		dev_warn(root->dev,
-		         "EISA: Cannot allocate resource for mainboard\n");
+			 "EISA: Cannot allocate resource for mainboard\n");
 		kfree(edev);
 		if (!root->force_probe)
 			return -EBUSY;
@@ -350,14 +348,14 @@ static int __init eisa_probe(struct eisa_root_device *root)
 
 	if (eisa_register_device(edev)) {
 		dev_err(&edev->dev, "EISA: Failed to register %s\n",
-		        edev->id.sig);
+			edev->id.sig);
 		eisa_release_resources(edev);
 		kfree(edev);
 	}
-	
+
  force_probe:
-	
-        for (c = 0, i = 1; i <= root->slots; i++) {
+
+	for (c = 0, i = 1; i <= root->slots; i++) {
 		edev = kzalloc(sizeof(*edev), GFP_KERNEL);
 		if (!edev) {
 			dev_err(root->dev, "EISA: Out of memory for slot %d\n",
@@ -367,8 +365,8 @@ static int __init eisa_probe(struct eisa_root_device *root)
 
 		if (eisa_request_resources(root, edev, i)) {
 			dev_warn(root->dev,
-			         "Cannot allocate resource for EISA slot %d\n",
-			         i);
+				 "Cannot allocate resource for EISA slot %d\n",
+				 i);
 			kfree(edev);
 			continue;
 		}
@@ -395,11 +393,11 @@ static int __init eisa_probe(struct eisa_root_device *root)
 
 		if (eisa_register_device(edev)) {
 			dev_err(&edev->dev, "EISA: Failed to register %s\n",
-			        edev->id.sig);
+				edev->id.sig);
 			eisa_release_resources(edev);
 			kfree(edev);
 		}
-        }
+	}
 
 	dev_info(root->dev, "EISA: Detected %d card%s\n", c, c == 1 ? "" : "s");
 	return 0;
@@ -422,7 +420,7 @@ int __init eisa_root_register(struct eisa_root_device *root)
 	 * been already registered. This prevents the virtual root
 	 * device from registering after the real one has, for
 	 * example... */
-	
+
 	root->eisa_root_res.name  = eisa_root_res.name;
 	root->eisa_root_res.start = root->res->start;
 	root->eisa_root_res.end   = root->res->end;
@@ -431,7 +429,7 @@ int __init eisa_root_register(struct eisa_root_device *root)
 	err = request_resource(&eisa_root_res, &root->eisa_root_res);
 	if (err)
 		return err;
-	
+
 	root->bus_nr = eisa_bus_count++;
 
 	err = eisa_probe(root);
@@ -444,7 +442,7 @@ int __init eisa_root_register(struct eisa_root_device *root)
 static int __init eisa_init(void)
 {
 	int r;
-	
+
 	r = bus_register(&eisa_bus_type);
 	if (r)
 		return r;
diff --git a/drivers/eisa/pci_eisa.c b/drivers/eisa/pci_eisa.c
index a333bf3517de..b5f367b44413 100644
--- a/drivers/eisa/pci_eisa.c
+++ b/drivers/eisa/pci_eisa.c
@@ -50,11 +50,11 @@ static int __init pci_eisa_init(struct pci_dev *pdev)
 		return -1;
 	}
 
-	pci_eisa_root.dev              = &pdev->dev;
-	pci_eisa_root.res	       = bus_res;
-	pci_eisa_root.bus_base_addr    = bus_res->start;
-	pci_eisa_root.slots	       = EISA_MAX_SLOTS;
-	pci_eisa_root.dma_mask         = pdev->dma_mask;
+	pci_eisa_root.dev		= &pdev->dev;
+	pci_eisa_root.res		= bus_res;
+	pci_eisa_root.bus_base_addr	= bus_res->start;
+	pci_eisa_root.slots		= EISA_MAX_SLOTS;
+	pci_eisa_root.dma_mask		= pdev->dma_mask;
 	dev_set_drvdata(pci_eisa_root.dev, &pci_eisa_root);
 
 	if (eisa_root_register (&pci_eisa_root)) {
diff --git a/drivers/eisa/virtual_root.c b/drivers/eisa/virtual_root.c
index 535e4f9c83f4..f1221c1d6319 100644
--- a/drivers/eisa/virtual_root.c
+++ b/drivers/eisa/virtual_root.c
@@ -35,11 +35,11 @@ static struct platform_device eisa_root_dev = {
 };
 
 static struct eisa_root_device eisa_bus_root = {
-	.dev           = &eisa_root_dev.dev,
-	.bus_base_addr = 0,
-	.res	       = &ioport_resource,
-	.slots	       = EISA_MAX_SLOTS,
-	.dma_mask      = 0xffffffff,
+	.dev		= &eisa_root_dev.dev,
+	.bus_base_addr	= 0,
+	.res		= &ioport_resource,
+	.slots		= EISA_MAX_SLOTS,
+	.dma_mask	= 0xffffffff,
 };
 
 static void virtual_eisa_release (struct device *dev)
@@ -50,13 +50,12 @@ static void virtual_eisa_release (struct device *dev)
 static int __init virtual_eisa_root_init (void)
 {
 	int r;
-	
-        if ((r = platform_device_register (&eisa_root_dev))) {
-                return r;
-        }
+
+	if ((r = platform_device_register (&eisa_root_dev)))
+		return r;
 
 	eisa_bus_root.force_probe = force_probe;
-	
+
 	dev_set_drvdata(&eisa_root_dev.dev, &eisa_bus_root);
 
 	if (eisa_root_register (&eisa_bus_root)) {
diff --git a/drivers/extcon/extcon-adc-jack.c b/drivers/extcon/extcon-adc-jack.c
index 3877d86c746a..18026354c332 100644
--- a/drivers/extcon/extcon-adc-jack.c
+++ b/drivers/extcon/extcon-adc-jack.c
@@ -144,7 +144,7 @@ static int adc_jack_probe(struct platform_device *pdev)
 		return err;
 
 	data->irq = platform_get_irq(pdev, 0);
-	if (!data->irq) {
+	if (data->irq < 0) {
 		dev_err(&pdev->dev, "platform_get_irq failed\n");
 		return -ENODEV;
 	}
diff --git a/drivers/extcon/extcon-axp288.c b/drivers/extcon/extcon-axp288.c
index 981fba56bc18..3ec4c715e240 100644
--- a/drivers/extcon/extcon-axp288.c
+++ b/drivers/extcon/extcon-axp288.c
@@ -24,8 +24,6 @@
 #include <linux/notifier.h>
 #include <linux/extcon-provider.h>
 #include <linux/regmap.h>
-#include <linux/gpio.h>
-#include <linux/gpio/consumer.h>
 #include <linux/mfd/axp20x.h>
 
 /* Power source status register */
@@ -79,11 +77,6 @@ enum axp288_extcon_reg {
 	AXP288_BC_DET_STAT_REG		= 0x2f,
 };
 
-enum axp288_mux_select {
-	EXTCON_GPIO_MUX_SEL_PMIC = 0,
-	EXTCON_GPIO_MUX_SEL_SOC,
-};
-
 enum axp288_extcon_irq {
 	VBUS_FALLING_IRQ = 0,
 	VBUS_RISING_IRQ,
@@ -104,15 +97,13 @@ struct axp288_extcon_info {
 	struct device *dev;
 	struct regmap *regmap;
 	struct regmap_irq_chip_data *regmap_irqc;
-	struct gpio_desc *gpio_mux_cntl;
 	int irq[EXTCON_IRQ_END];
 	struct extcon_dev *edev;
-	struct notifier_block extcon_nb;
 	unsigned int previous_cable;
 };
 
 /* Power up/down reason string array */
-static char *axp288_pwr_up_down_info[] = {
+static const char * const axp288_pwr_up_down_info[] = {
 	"Last wake caused by user pressing the power button",
 	"Last wake caused by a charger insertion",
 	"Last wake caused by a battery insertion",
@@ -130,7 +121,7 @@ static char *axp288_pwr_up_down_info[] = {
  */
 static void axp288_extcon_log_rsi(struct axp288_extcon_info *info)
 {
-	char **rsi;
+	const char * const *rsi;
 	unsigned int val, i, clear_mask = 0;
 	int ret;
 
@@ -192,20 +183,11 @@ static int axp288_handle_chrg_det_event(struct axp288_extcon_info *info)
 		cable = EXTCON_CHG_USB_DCP;
 		break;
 	default:
-		dev_warn(info->dev,
-			"disconnect or unknown or ID event\n");
+		dev_warn(info->dev, "unknown (reserved) bc detect result\n");
+		cable = EXTCON_CHG_USB_SDP;
 	}
 
 no_vbus:
-	/*
-	 * If VBUS is absent Connect D+/D- lines to PMIC for BC
-	 * detection. Else connect them to SOC for USB communication.
-	 */
-	if (info->gpio_mux_cntl)
-		gpiod_set_value(info->gpio_mux_cntl,
-			vbus_attach ? EXTCON_GPIO_MUX_SEL_SOC
-					: EXTCON_GPIO_MUX_SEL_PMIC);
-
 	extcon_set_state_sync(info->edev, info->previous_cable, false);
 	if (info->previous_cable == EXTCON_CHG_USB_SDP)
 		extcon_set_state_sync(info->edev, EXTCON_USB, false);
@@ -253,8 +235,7 @@ static int axp288_extcon_probe(struct platform_device *pdev)
 {
 	struct axp288_extcon_info *info;
 	struct axp20x_dev *axp20x = dev_get_drvdata(pdev->dev.parent);
-	struct axp288_extcon_pdata *pdata = pdev->dev.platform_data;
-	int ret, i, pirq, gpio;
+	int ret, i, pirq;
 
 	info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL);
 	if (!info)
@@ -264,8 +245,6 @@ static int axp288_extcon_probe(struct platform_device *pdev)
 	info->regmap = axp20x->regmap;
 	info->regmap_irqc = axp20x->regmap_irqc;
 	info->previous_cable = EXTCON_NONE;
-	if (pdata)
-		info->gpio_mux_cntl = pdata->gpio_mux_cntl;
 
 	platform_set_drvdata(pdev, info);
 
@@ -286,21 +265,11 @@ static int axp288_extcon_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	/* Set up gpio control for USB Mux */
-	if (info->gpio_mux_cntl) {
-		gpio = desc_to_gpio(info->gpio_mux_cntl);
-		ret = devm_gpio_request(&pdev->dev, gpio, "USB_MUX");
-		if (ret < 0) {
-			dev_err(&pdev->dev,
-				"failed to request the gpio=%d\n", gpio);
-			return ret;
-		}
-		gpiod_direction_output(info->gpio_mux_cntl,
-						EXTCON_GPIO_MUX_SEL_PMIC);
-	}
-
 	for (i = 0; i < EXTCON_IRQ_END; i++) {
 		pirq = platform_get_irq(pdev, i);
+		if (pirq < 0)
+			return pirq;
+
 		info->irq[i] = regmap_irq_get_virq(info->regmap_irqc, pirq);
 		if (info->irq[i] < 0) {
 			dev_err(&pdev->dev,
diff --git a/drivers/extcon/extcon-intel-int3496.c b/drivers/extcon/extcon-intel-int3496.c
index c8691b5a9cb0..191e99f06a9a 100644
--- a/drivers/extcon/extcon-intel-int3496.c
+++ b/drivers/extcon/extcon-intel-int3496.c
@@ -153,8 +153,9 @@ static int int3496_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	/* queue initial processing of id-pin */
+	/* process id-pin so that we start with the right status */
 	queue_delayed_work(system_wq, &data->work, 0);
+	flush_delayed_work(&data->work);
 
 	platform_set_drvdata(pdev, data);
 
diff --git a/drivers/extcon/extcon-max77693.c b/drivers/extcon/extcon-max77693.c
index 643411066ad9..227651ff9666 100644
--- a/drivers/extcon/extcon-max77693.c
+++ b/drivers/extcon/extcon-max77693.c
@@ -266,7 +266,7 @@ static int max77693_muic_set_debounce_time(struct max77693_muic_info *info,
 static int max77693_muic_set_path(struct max77693_muic_info *info,
 		u8 val, bool attached)
 {
-	int ret = 0;
+	int ret;
 	unsigned int ctrl1, ctrl2 = 0;
 
 	if (attached)
diff --git a/drivers/extcon/extcon-max8997.c b/drivers/extcon/extcon-max8997.c
index 8152790d72e1..9f30f4929b72 100644
--- a/drivers/extcon/extcon-max8997.c
+++ b/drivers/extcon/extcon-max8997.c
@@ -204,7 +204,7 @@ static int max8997_muic_set_debounce_time(struct max8997_muic_info *info,
 static int max8997_muic_set_path(struct max8997_muic_info *info,
 		u8 val, bool attached)
 {
-	int ret = 0;
+	int ret;
 	u8 ctrl1, ctrl2 = 0;
 
 	if (attached)
diff --git a/drivers/extcon/extcon-usbc-cros-ec.c b/drivers/extcon/extcon-usbc-cros-ec.c
index 6187f731b29d..6721ab01fe7d 100644
--- a/drivers/extcon/extcon-usbc-cros-ec.c
+++ b/drivers/extcon/extcon-usbc-cros-ec.c
@@ -34,16 +34,26 @@ struct cros_ec_extcon_info {
 
 	struct notifier_block notifier;
 
+	unsigned int dr; /* data role */
+	bool pr; /* power role (true if VBUS enabled) */
 	bool dp; /* DisplayPort enabled */
 	bool mux; /* SuperSpeed (usb3) enabled */
 	unsigned int power_type;
 };
 
 static const unsigned int usb_type_c_cable[] = {
+	EXTCON_USB,
+	EXTCON_USB_HOST,
 	EXTCON_DISP_DP,
 	EXTCON_NONE,
 };
 
+enum usb_data_roles {
+	DR_NONE,
+	DR_HOST,
+	DR_DEVICE,
+};
+
 /**
  * cros_ec_pd_command() - Send a command to the EC.
  * @info: pointer to struct cros_ec_extcon_info
@@ -150,6 +160,7 @@ static int cros_ec_usb_get_role(struct cros_ec_extcon_info *info,
 	pd_control.port = info->port_id;
 	pd_control.role = USB_PD_CTRL_ROLE_NO_CHANGE;
 	pd_control.mux = USB_PD_CTRL_MUX_NO_CHANGE;
+	pd_control.swap = USB_PD_CTRL_SWAP_NONE;
 	ret = cros_ec_pd_command(info, EC_CMD_USB_PD_CONTROL, 1,
 				 &pd_control, sizeof(pd_control),
 				 &resp, sizeof(resp));
@@ -183,11 +194,72 @@ static int cros_ec_pd_get_num_ports(struct cros_ec_extcon_info *info)
 	return resp.num_ports;
 }
 
+static const char *cros_ec_usb_role_string(unsigned int role)
+{
+	return role == DR_NONE ? "DISCONNECTED" :
+		(role == DR_HOST ? "DFP" : "UFP");
+}
+
+static const char *cros_ec_usb_power_type_string(unsigned int type)
+{
+	switch (type) {
+	case USB_CHG_TYPE_NONE:
+		return "USB_CHG_TYPE_NONE";
+	case USB_CHG_TYPE_PD:
+		return "USB_CHG_TYPE_PD";
+	case USB_CHG_TYPE_PROPRIETARY:
+		return "USB_CHG_TYPE_PROPRIETARY";
+	case USB_CHG_TYPE_C:
+		return "USB_CHG_TYPE_C";
+	case USB_CHG_TYPE_BC12_DCP:
+		return "USB_CHG_TYPE_BC12_DCP";
+	case USB_CHG_TYPE_BC12_CDP:
+		return "USB_CHG_TYPE_BC12_CDP";
+	case USB_CHG_TYPE_BC12_SDP:
+		return "USB_CHG_TYPE_BC12_SDP";
+	case USB_CHG_TYPE_OTHER:
+		return "USB_CHG_TYPE_OTHER";
+	case USB_CHG_TYPE_VBUS:
+		return "USB_CHG_TYPE_VBUS";
+	case USB_CHG_TYPE_UNKNOWN:
+		return "USB_CHG_TYPE_UNKNOWN";
+	default:
+		return "USB_CHG_TYPE_UNKNOWN";
+	}
+}
+
+static bool cros_ec_usb_power_type_is_wall_wart(unsigned int type,
+						unsigned int role)
+{
+	switch (type) {
+	/* FIXME : Guppy, Donnettes, and other chargers will be miscategorized
+	 * because they identify with USB_CHG_TYPE_C, but we can't return true
+	 * here from that code because that breaks Suzy-Q and other kinds of
+	 * USB Type-C cables and peripherals.
+	 */
+	case USB_CHG_TYPE_PROPRIETARY:
+	case USB_CHG_TYPE_BC12_DCP:
+		return true;
+	case USB_CHG_TYPE_PD:
+	case USB_CHG_TYPE_C:
+	case USB_CHG_TYPE_BC12_CDP:
+	case USB_CHG_TYPE_BC12_SDP:
+	case USB_CHG_TYPE_OTHER:
+	case USB_CHG_TYPE_VBUS:
+	case USB_CHG_TYPE_UNKNOWN:
+	case USB_CHG_TYPE_NONE:
+	default:
+		return false;
+	}
+}
+
 static int extcon_cros_ec_detect_cable(struct cros_ec_extcon_info *info,
 				       bool force)
 {
 	struct device *dev = info->dev;
 	int role, power_type;
+	unsigned int dr = DR_NONE;
+	bool pr = false;
 	bool polarity = false;
 	bool dp = false;
 	bool mux = false;
@@ -206,9 +278,12 @@ static int extcon_cros_ec_detect_cable(struct cros_ec_extcon_info *info,
 			dev_err(dev, "failed getting role err = %d\n", role);
 			return role;
 		}
+		dev_dbg(dev, "disconnected\n");
 	} else {
 		int pd_mux_state;
 
+		dr = (role & PD_CTRL_RESP_ROLE_DATA) ? DR_HOST : DR_DEVICE;
+		pr = (role & PD_CTRL_RESP_ROLE_POWER);
 		pd_mux_state = cros_ec_usb_get_pd_mux_state(info);
 		if (pd_mux_state < 0)
 			pd_mux_state = USB_PD_MUX_USB_ENABLED;
@@ -216,20 +291,62 @@ static int extcon_cros_ec_detect_cable(struct cros_ec_extcon_info *info,
 		dp = pd_mux_state & USB_PD_MUX_DP_ENABLED;
 		mux = pd_mux_state & USB_PD_MUX_USB_ENABLED;
 		hpd = pd_mux_state & USB_PD_MUX_HPD_IRQ;
-	}
 
-	if (force || info->dp != dp || info->mux != mux ||
-		info->power_type != power_type) {
+		dev_dbg(dev,
+			"connected role 0x%x pwr type %d dr %d pr %d pol %d mux %d dp %d hpd %d\n",
+			role, power_type, dr, pr, polarity, mux, dp, hpd);
+	}
 
+	/*
+	 * When there is no USB host (e.g. USB PD charger),
+	 * we are not really a UFP for the AP.
+	 */
+	if (dr == DR_DEVICE &&
+	    cros_ec_usb_power_type_is_wall_wart(power_type, role))
+		dr = DR_NONE;
+
+	if (force || info->dr != dr || info->pr != pr || info->dp != dp ||
+	    info->mux != mux || info->power_type != power_type) {
+		bool host_connected = false, device_connected = false;
+
+		dev_dbg(dev, "Type/Role switch! type = %s role = %s\n",
+			cros_ec_usb_power_type_string(power_type),
+			cros_ec_usb_role_string(dr));
+		info->dr = dr;
+		info->pr = pr;
 		info->dp = dp;
 		info->mux = mux;
 		info->power_type = power_type;
 
-		extcon_set_state(info->edev, EXTCON_DISP_DP, dp);
+		if (dr == DR_DEVICE)
+			device_connected = true;
+		else if (dr == DR_HOST)
+			host_connected = true;
 
+		extcon_set_state(info->edev, EXTCON_USB, device_connected);
+		extcon_set_state(info->edev, EXTCON_USB_HOST, host_connected);
+		extcon_set_state(info->edev, EXTCON_DISP_DP, dp);
+		extcon_set_property(info->edev, EXTCON_USB,
+				    EXTCON_PROP_USB_VBUS,
+				    (union extcon_property_value)(int)pr);
+		extcon_set_property(info->edev, EXTCON_USB_HOST,
+				    EXTCON_PROP_USB_VBUS,
+				    (union extcon_property_value)(int)pr);
+		extcon_set_property(info->edev, EXTCON_USB,
+				    EXTCON_PROP_USB_TYPEC_POLARITY,
+				    (union extcon_property_value)(int)polarity);
+		extcon_set_property(info->edev, EXTCON_USB_HOST,
+				    EXTCON_PROP_USB_TYPEC_POLARITY,
+				    (union extcon_property_value)(int)polarity);
 		extcon_set_property(info->edev, EXTCON_DISP_DP,
 				    EXTCON_PROP_USB_TYPEC_POLARITY,
 				    (union extcon_property_value)(int)polarity);
+		extcon_set_property(info->edev, EXTCON_USB,
+				    EXTCON_PROP_USB_SS,
+				    (union extcon_property_value)(int)mux);
+		extcon_set_property(info->edev, EXTCON_USB_HOST,
+				    EXTCON_PROP_USB_SS,
+				    (union extcon_property_value)(int)mux);
 		extcon_set_property(info->edev, EXTCON_DISP_DP,
 				    EXTCON_PROP_USB_SS,
 				    (union extcon_property_value)(int)mux);
@@ -237,6 +354,8 @@ static int extcon_cros_ec_detect_cable(struct cros_ec_extcon_info *info,
 				    EXTCON_PROP_DISP_HPD,
 				    (union extcon_property_value)(int)hpd);
 
+		extcon_sync(info->edev, EXTCON_USB);
+		extcon_sync(info->edev, EXTCON_USB_HOST);
 		extcon_sync(info->edev, EXTCON_DISP_DP);
 
 	} else if (hpd) {
@@ -322,13 +441,28 @@ static int extcon_cros_ec_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	extcon_set_property_capability(info->edev, EXTCON_USB,
+				       EXTCON_PROP_USB_VBUS);
+	extcon_set_property_capability(info->edev, EXTCON_USB_HOST,
+				       EXTCON_PROP_USB_VBUS);
+	extcon_set_property_capability(info->edev, EXTCON_USB,
+				       EXTCON_PROP_USB_TYPEC_POLARITY);
+	extcon_set_property_capability(info->edev, EXTCON_USB_HOST,
+				       EXTCON_PROP_USB_TYPEC_POLARITY);
 	extcon_set_property_capability(info->edev, EXTCON_DISP_DP,
 				       EXTCON_PROP_USB_TYPEC_POLARITY);
+	extcon_set_property_capability(info->edev, EXTCON_USB,
+				       EXTCON_PROP_USB_SS);
+	extcon_set_property_capability(info->edev, EXTCON_USB_HOST,
+				       EXTCON_PROP_USB_SS);
 	extcon_set_property_capability(info->edev, EXTCON_DISP_DP,
 				       EXTCON_PROP_USB_SS);
 	extcon_set_property_capability(info->edev, EXTCON_DISP_DP,
 				       EXTCON_PROP_DISP_HPD);
 
+	info->dr = DR_NONE;
+	info->pr = false;
+
 	platform_set_drvdata(pdev, info);
 
 	/* Get PD events from the EC */
diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c
index a301fcf46e88..f0587273940e 100644
--- a/drivers/firewire/core-cdev.c
+++ b/drivers/firewire/core-cdev.c
@@ -1784,17 +1784,17 @@ static int fw_device_op_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int fw_device_op_poll(struct file *file, poll_table * pt)
+static __poll_t fw_device_op_poll(struct file *file, poll_table * pt)
 {
 	struct client *client = file->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &client->wait, pt);
 
 	if (fw_device_is_shutdown(client->device))
-		mask |= POLLHUP | POLLERR;
+		mask |= EPOLLHUP | EPOLLERR;
 	if (!list_empty(&client->event_list))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c
index 242359c2d1f1..60e75e6d9104 100644
--- a/drivers/firewire/net.c
+++ b/drivers/firewire/net.c
@@ -1480,9 +1480,14 @@ static int fwnet_probe(struct fw_unit *unit,
 		goto out;
 	dev->local_fifo = dev->handler.offset;
 
+	/*
+	 * default MTU: RFC 2734 cl. 4, RFC 3146 cl. 4
+	 * maximum MTU: RFC 2734 cl. 4.2, fragment encapsulation header's
+	 *              maximum possible datagram_size + 1 = 0xfff + 1
+	 */
 	net->mtu = 1500U;
 	net->min_mtu = ETH_MIN_MTU;
-	net->max_mtu = 0xfff;
+	net->max_mtu = 4096U;
 
 	/* Set our hardware address while we're at it */
 	ha = (union fwnet_hwaddr *)net->dev_addr;
diff --git a/drivers/firewire/nosy.c b/drivers/firewire/nosy.c
index 180f0a96528c..a128dd1126ae 100644
--- a/drivers/firewire/nosy.c
+++ b/drivers/firewire/nosy.c
@@ -328,19 +328,19 @@ nosy_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int
+static __poll_t
 nosy_poll(struct file *file, poll_table *pt)
 {
 	struct client *client = file->private_data;
-	unsigned int ret = 0;
+	__poll_t ret = 0;
 
 	poll_wait(file, &client->buffer.wait, pt);
 
 	if (atomic_read(&client->buffer.size) > 0)
-		ret = POLLIN | POLLRDNORM;
+		ret = EPOLLIN | EPOLLRDNORM;
 
 	if (list_empty(&client->lynx->link))
-		ret |= POLLHUP;
+		ret |= EPOLLHUP;
 
 	return ret;
 }
diff --git a/drivers/firewire/ohci.c b/drivers/firewire/ohci.c
index ccf52368a073..45c048751f3b 100644
--- a/drivers/firewire/ohci.c
+++ b/drivers/firewire/ohci.c
@@ -1128,7 +1128,13 @@ static int context_add_buffer(struct context *ctx)
 		return -ENOMEM;
 
 	offset = (void *)&desc->buffer - (void *)desc;
-	desc->buffer_size = PAGE_SIZE - offset;
+	/*
+	 * Some controllers, like JMicron ones, always issue 0x20-byte DMA reads
+	 * for descriptors, even 0x10-byte ones. This can cause page faults when
+	 * an IOMMU is in use and the oversized read crosses a page boundary.
+	 * Work around this by always leaving at least 0x10 bytes of padding.
+	 */
+	desc->buffer_size = PAGE_SIZE - offset - 0x10;
 	desc->buffer_bus = bus_addr + offset;
 	desc->used = 0;
 
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index fa87a055905e..b7c748248e53 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -10,7 +10,7 @@ config ARM_PSCI_FW
 
 config ARM_PSCI_CHECKER
 	bool "ARM PSCI checker"
-	depends on ARM_PSCI_FW && HOTPLUG_CPU && !TORTURE_TEST
+	depends on ARM_PSCI_FW && HOTPLUG_CPU && CPU_IDLE && !TORTURE_TEST
 	help
 	  Run the PSCI checker during startup. This checks that hotplug and
 	  suspend operations work correctly when using PSCI.
@@ -48,6 +48,14 @@ config ARM_SCPI_POWER_DOMAIN
 	  This enables support for the SCPI power domains which can be
 	  enabled or disabled via the SCP firmware
 
+config ARM_SDE_INTERFACE
+	bool "ARM Software Delegated Exception Interface (SDEI)"
+	depends on ARM64
+	help
+	  The Software Delegated Exception Interface (SDEI) is an ARM
+	  standard for registering callbacks from the platform firmware
+	  into the OS. This is typically used to implement RAS notifications.
+
 config EDD
 	tristate "BIOS Enhanced Disk Drive calls determine boot disk"
 	depends on X86
diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile
index feaa890197f3..b248238ddc6a 100644
--- a/drivers/firmware/Makefile
+++ b/drivers/firmware/Makefile
@@ -6,6 +6,7 @@ obj-$(CONFIG_ARM_PSCI_FW)	+= psci.o
 obj-$(CONFIG_ARM_PSCI_CHECKER)	+= psci_checker.o
 obj-$(CONFIG_ARM_SCPI_PROTOCOL)	+= arm_scpi.o
 obj-$(CONFIG_ARM_SCPI_POWER_DOMAIN) += scpi_pm_domain.o
+obj-$(CONFIG_ARM_SDE_INTERFACE)	+= arm_sdei.o
 obj-$(CONFIG_DMI)		+= dmi_scan.o
 obj-$(CONFIG_DMI_SYSFS)		+= dmi-sysfs.o
 obj-$(CONFIG_EDD)		+= edd.o
diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c
new file mode 100644
index 000000000000..1ea71640fdc2
--- /dev/null
+++ b/drivers/firmware/arm_sdei.c
@@ -0,0 +1,1092 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2017 Arm Ltd.
+#define pr_fmt(fmt) "sdei: " fmt
+
+#include <linux/acpi.h>
+#include <linux/arm_sdei.h>
+#include <linux/arm-smccc.h>
+#include <linux/atomic.h>
+#include <linux/bitops.h>
+#include <linux/compiler.h>
+#include <linux/cpuhotplug.h>
+#include <linux/cpu.h>
+#include <linux/cpu_pm.h>
+#include <linux/errno.h>
+#include <linux/hardirq.h>
+#include <linux/kernel.h>
+#include <linux/kprobes.h>
+#include <linux/kvm_host.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/notifier.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/percpu.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/ptrace.h>
+#include <linux/preempt.h>
+#include <linux/reboot.h>
+#include <linux/slab.h>
+#include <linux/smp.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+
+/*
+ * The call to use to reach the firmware.
+ */
+static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
+		      unsigned long arg0, unsigned long arg1,
+		      unsigned long arg2, unsigned long arg3,
+		      unsigned long arg4, struct arm_smccc_res *res);
+
+/* entry point from firmware to arch asm code */
+static unsigned long sdei_entry_point;
+
+struct sdei_event {
+	/* These three are protected by the sdei_list_lock */
+	struct list_head	list;
+	bool			reregister;
+	bool			reenable;
+
+	u32			event_num;
+	u8			type;
+	u8			priority;
+
+	/* This pointer is handed to firmware as the event argument. */
+	union {
+		/* Shared events */
+		struct sdei_registered_event *registered;
+
+		/* CPU private events */
+		struct sdei_registered_event __percpu *private_registered;
+	};
+};
+
+/* Take the mutex for any API call or modification. Take the mutex first. */
+static DEFINE_MUTEX(sdei_events_lock);
+
+/* and then hold this when modifying the list */
+static DEFINE_SPINLOCK(sdei_list_lock);
+static LIST_HEAD(sdei_list);
+
+/* Private events are registered/enabled via IPI passing one of these */
+struct sdei_crosscall_args {
+	struct sdei_event *event;
+	atomic_t errors;
+	int first_error;
+};
+
+#define CROSSCALL_INIT(arg, event)	(arg.event = event, \
+					 arg.first_error = 0, \
+					 atomic_set(&arg.errors, 0))
+
+static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
+{
+	struct sdei_crosscall_args arg;
+
+	CROSSCALL_INIT(arg, event);
+	on_each_cpu(fn, &arg, true);
+
+	return arg.first_error;
+}
+
+static inline void
+sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
+{
+	if (err && (atomic_inc_return(&arg->errors) == 1))
+		arg->first_error = err;
+}
+
+static int sdei_to_linux_errno(unsigned long sdei_err)
+{
+	switch (sdei_err) {
+	case SDEI_NOT_SUPPORTED:
+		return -EOPNOTSUPP;
+	case SDEI_INVALID_PARAMETERS:
+		return -EINVAL;
+	case SDEI_DENIED:
+		return -EPERM;
+	case SDEI_PENDING:
+		return -EINPROGRESS;
+	case SDEI_OUT_OF_RESOURCE:
+		return -ENOMEM;
+	}
+
+	/* Not an error value ... */
+	return sdei_err;
+}
+
+/*
+ * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
+ * to translate.
+ */
+static int sdei_is_err(struct arm_smccc_res *res)
+{
+	switch (res->a0) {
+	case SDEI_NOT_SUPPORTED:
+	case SDEI_INVALID_PARAMETERS:
+	case SDEI_DENIED:
+	case SDEI_PENDING:
+	case SDEI_OUT_OF_RESOURCE:
+		return true;
+	}
+
+	return false;
+}
+
+static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
+			  unsigned long arg1, unsigned long arg2,
+			  unsigned long arg3, unsigned long arg4,
+			  u64 *result)
+{
+	int err = 0;
+	struct arm_smccc_res res;
+
+	if (sdei_firmware_call) {
+		sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
+				   &res);
+		if (sdei_is_err(&res))
+			err = sdei_to_linux_errno(res.a0);
+	} else {
+		/*
+		 * !sdei_firmware_call means we failed to probe or called
+		 * sdei_mark_interface_broken(). -EIO is not an error returned
+		 * by sdei_to_linux_errno() and is used to suppress messages
+		 * from this driver.
+		 */
+		err = -EIO;
+		res.a0 = SDEI_NOT_SUPPORTED;
+	}
+
+	if (result)
+		*result = res.a0;
+
+	return err;
+}
+
+static struct sdei_event *sdei_event_find(u32 event_num)
+{
+	struct sdei_event *e, *found = NULL;
+
+	lockdep_assert_held(&sdei_events_lock);
+
+	spin_lock(&sdei_list_lock);
+	list_for_each_entry(e, &sdei_list, list) {
+		if (e->event_num == event_num) {
+			found = e;
+			break;
+		}
+	}
+	spin_unlock(&sdei_list_lock);
+
+	return found;
+}
+
+int sdei_api_event_context(u32 query, u64 *result)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
+			      result);
+}
+NOKPROBE_SYMBOL(sdei_api_event_context);
+
+static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
+			      0, 0, result);
+}
+
+static struct sdei_event *sdei_event_create(u32 event_num,
+					    sdei_event_callback *cb,
+					    void *cb_arg)
+{
+	int err;
+	u64 result;
+	struct sdei_event *event;
+	struct sdei_registered_event *reg;
+
+	lockdep_assert_held(&sdei_events_lock);
+
+	event = kzalloc(sizeof(*event), GFP_KERNEL);
+	if (!event)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&event->list);
+	event->event_num = event_num;
+
+	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
+				      &result);
+	if (err) {
+		kfree(event);
+		return ERR_PTR(err);
+	}
+	event->priority = result;
+
+	err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
+				      &result);
+	if (err) {
+		kfree(event);
+		return ERR_PTR(err);
+	}
+	event->type = result;
+
+	if (event->type == SDEI_EVENT_TYPE_SHARED) {
+		reg = kzalloc(sizeof(*reg), GFP_KERNEL);
+		if (!reg) {
+			kfree(event);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		reg->event_num = event_num;
+		reg->priority = event->priority;
+
+		reg->callback = cb;
+		reg->callback_arg = cb_arg;
+		event->registered = reg;
+	} else {
+		int cpu;
+		struct sdei_registered_event __percpu *regs;
+
+		regs = alloc_percpu(struct sdei_registered_event);
+		if (!regs) {
+			kfree(event);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		for_each_possible_cpu(cpu) {
+			reg = per_cpu_ptr(regs, cpu);
+
+			reg->event_num = event->event_num;
+			reg->priority = event->priority;
+			reg->callback = cb;
+			reg->callback_arg = cb_arg;
+		}
+
+		event->private_registered = regs;
+	}
+
+	if (sdei_event_find(event_num)) {
+		kfree(event->registered);
+		kfree(event);
+		event = ERR_PTR(-EBUSY);
+	} else {
+		spin_lock(&sdei_list_lock);
+		list_add(&event->list, &sdei_list);
+		spin_unlock(&sdei_list_lock);
+	}
+
+	return event;
+}
+
+static void sdei_event_destroy(struct sdei_event *event)
+{
+	lockdep_assert_held(&sdei_events_lock);
+
+	spin_lock(&sdei_list_lock);
+	list_del(&event->list);
+	spin_unlock(&sdei_list_lock);
+
+	if (event->type == SDEI_EVENT_TYPE_SHARED)
+		kfree(event->registered);
+	else
+		free_percpu(event->private_registered);
+
+	kfree(event);
+}
+
+static int sdei_api_get_version(u64 *version)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
+}
+
+int sdei_mask_local_cpu(void)
+{
+	int err;
+
+	WARN_ON_ONCE(preemptible());
+
+	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
+	if (err && err != -EIO) {
+		pr_warn_once("failed to mask CPU[%u]: %d\n",
+			      smp_processor_id(), err);
+		return err;
+	}
+
+	return 0;
+}
+
+static void _ipi_mask_cpu(void *ignored)
+{
+	sdei_mask_local_cpu();
+}
+
+int sdei_unmask_local_cpu(void)
+{
+	int err;
+
+	WARN_ON_ONCE(preemptible());
+
+	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
+	if (err && err != -EIO) {
+		pr_warn_once("failed to unmask CPU[%u]: %d\n",
+			     smp_processor_id(), err);
+		return err;
+	}
+
+	return 0;
+}
+
+static void _ipi_unmask_cpu(void *ignored)
+{
+	sdei_unmask_local_cpu();
+}
+
+static void _ipi_private_reset(void *ignored)
+{
+	int err;
+
+	err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
+			     NULL);
+	if (err && err != -EIO)
+		pr_warn_once("failed to reset CPU[%u]: %d\n",
+			     smp_processor_id(), err);
+}
+
+static int sdei_api_shared_reset(void)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
+			      NULL);
+}
+
+static void sdei_mark_interface_broken(void)
+{
+	pr_err("disabling SDEI firmware interface\n");
+	on_each_cpu(&_ipi_mask_cpu, NULL, true);
+	sdei_firmware_call = NULL;
+}
+
+static int sdei_platform_reset(void)
+{
+	int err;
+
+	on_each_cpu(&_ipi_private_reset, NULL, true);
+	err = sdei_api_shared_reset();
+	if (err) {
+		pr_err("Failed to reset platform: %d\n", err);
+		sdei_mark_interface_broken();
+	}
+
+	return err;
+}
+
+static int sdei_api_event_enable(u32 event_num)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
+			      0, NULL);
+}
+
+/* Called directly by the hotplug callbacks */
+static void _local_event_enable(void *data)
+{
+	int err;
+	struct sdei_crosscall_args *arg = data;
+
+	WARN_ON_ONCE(preemptible());
+
+	err = sdei_api_event_enable(arg->event->event_num);
+
+	sdei_cross_call_return(arg, err);
+}
+
+int sdei_event_enable(u32 event_num)
+{
+	int err = -EINVAL;
+	struct sdei_event *event;
+
+	mutex_lock(&sdei_events_lock);
+	event = sdei_event_find(event_num);
+	if (!event) {
+		mutex_unlock(&sdei_events_lock);
+		return -ENOENT;
+	}
+
+	spin_lock(&sdei_list_lock);
+	event->reenable = true;
+	spin_unlock(&sdei_list_lock);
+
+	if (event->type == SDEI_EVENT_TYPE_SHARED)
+		err = sdei_api_event_enable(event->event_num);
+	else
+		err = sdei_do_cross_call(_local_event_enable, event);
+	mutex_unlock(&sdei_events_lock);
+
+	return err;
+}
+EXPORT_SYMBOL(sdei_event_enable);
+
+static int sdei_api_event_disable(u32 event_num)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
+			      0, 0, NULL);
+}
+
+static void _ipi_event_disable(void *data)
+{
+	int err;
+	struct sdei_crosscall_args *arg = data;
+
+	err = sdei_api_event_disable(arg->event->event_num);
+
+	sdei_cross_call_return(arg, err);
+}
+
+int sdei_event_disable(u32 event_num)
+{
+	int err = -EINVAL;
+	struct sdei_event *event;
+
+	mutex_lock(&sdei_events_lock);
+	event = sdei_event_find(event_num);
+	if (!event) {
+		mutex_unlock(&sdei_events_lock);
+		return -ENOENT;
+	}
+
+	spin_lock(&sdei_list_lock);
+	event->reenable = false;
+	spin_unlock(&sdei_list_lock);
+
+	if (event->type == SDEI_EVENT_TYPE_SHARED)
+		err = sdei_api_event_disable(event->event_num);
+	else
+		err = sdei_do_cross_call(_ipi_event_disable, event);
+	mutex_unlock(&sdei_events_lock);
+
+	return err;
+}
+EXPORT_SYMBOL(sdei_event_disable);
+
+static int sdei_api_event_unregister(u32 event_num)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
+			      0, 0, 0, NULL);
+}
+
+/* Called directly by the hotplug callbacks */
+static void _local_event_unregister(void *data)
+{
+	int err;
+	struct sdei_crosscall_args *arg = data;
+
+	WARN_ON_ONCE(preemptible());
+
+	err = sdei_api_event_unregister(arg->event->event_num);
+
+	sdei_cross_call_return(arg, err);
+}
+
+static int _sdei_event_unregister(struct sdei_event *event)
+{
+	lockdep_assert_held(&sdei_events_lock);
+
+	spin_lock(&sdei_list_lock);
+	event->reregister = false;
+	event->reenable = false;
+	spin_unlock(&sdei_list_lock);
+
+	if (event->type == SDEI_EVENT_TYPE_SHARED)
+		return sdei_api_event_unregister(event->event_num);
+
+	return sdei_do_cross_call(_local_event_unregister, event);
+}
+
+int sdei_event_unregister(u32 event_num)
+{
+	int err;
+	struct sdei_event *event;
+
+	WARN_ON(in_nmi());
+
+	mutex_lock(&sdei_events_lock);
+	event = sdei_event_find(event_num);
+	do {
+		if (!event) {
+			pr_warn("Event %u not registered\n", event_num);
+			err = -ENOENT;
+			break;
+		}
+
+		err = _sdei_event_unregister(event);
+		if (err)
+			break;
+
+		sdei_event_destroy(event);
+	} while (0);
+	mutex_unlock(&sdei_events_lock);
+
+	return err;
+}
+EXPORT_SYMBOL(sdei_event_unregister);
+
+/*
+ * unregister events, but don't destroy them as they are re-registered by
+ * sdei_reregister_shared().
+ */
+static int sdei_unregister_shared(void)
+{
+	int err = 0;
+	struct sdei_event *event;
+
+	mutex_lock(&sdei_events_lock);
+	spin_lock(&sdei_list_lock);
+	list_for_each_entry(event, &sdei_list, list) {
+		if (event->type != SDEI_EVENT_TYPE_SHARED)
+			continue;
+
+		err = _sdei_event_unregister(event);
+		if (err)
+			break;
+	}
+	spin_unlock(&sdei_list_lock);
+	mutex_unlock(&sdei_events_lock);
+
+	return err;
+}
+
+static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
+				   void *arg, u64 flags, u64 affinity)
+{
+	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
+			      (unsigned long)entry_point, (unsigned long)arg,
+			      flags, affinity, NULL);
+}
+
+/* Called directly by the hotplug callbacks */
+static void _local_event_register(void *data)
+{
+	int err;
+	struct sdei_registered_event *reg;
+	struct sdei_crosscall_args *arg = data;
+
+	WARN_ON(preemptible());
+
+	reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
+	err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
+				      reg, 0, 0);
+
+	sdei_cross_call_return(arg, err);
+}
+
+static int _sdei_event_register(struct sdei_event *event)
+{
+	int err;
+
+	lockdep_assert_held(&sdei_events_lock);
+
+	spin_lock(&sdei_list_lock);
+	event->reregister = true;
+	spin_unlock(&sdei_list_lock);
+
+	if (event->type == SDEI_EVENT_TYPE_SHARED)
+		return sdei_api_event_register(event->event_num,
+					       sdei_entry_point,
+					       event->registered,
+					       SDEI_EVENT_REGISTER_RM_ANY, 0);
+
+
+	err = sdei_do_cross_call(_local_event_register, event);
+	if (err) {
+		spin_lock(&sdei_list_lock);
+		event->reregister = false;
+		event->reenable = false;
+		spin_unlock(&sdei_list_lock);
+
+		sdei_do_cross_call(_local_event_unregister, event);
+	}
+
+	return err;
+}
+
+int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
+{
+	int err;
+	struct sdei_event *event;
+
+	WARN_ON(in_nmi());
+
+	mutex_lock(&sdei_events_lock);
+	do {
+		if (sdei_event_find(event_num)) {
+			pr_warn("Event %u already registered\n", event_num);
+			err = -EBUSY;
+			break;
+		}
+
+		event = sdei_event_create(event_num, cb, arg);
+		if (IS_ERR(event)) {
+			err = PTR_ERR(event);
+			pr_warn("Failed to create event %u: %d\n", event_num,
+				err);
+			break;
+		}
+
+		err = _sdei_event_register(event);
+		if (err) {
+			sdei_event_destroy(event);
+			pr_warn("Failed to register event %u: %d\n", event_num,
+				err);
+		}
+	} while (0);
+	mutex_unlock(&sdei_events_lock);
+
+	return err;
+}
+EXPORT_SYMBOL(sdei_event_register);
+
+static int sdei_reregister_event(struct sdei_event *event)
+{
+	int err;
+
+	lockdep_assert_held(&sdei_events_lock);
+
+	err = _sdei_event_register(event);
+	if (err) {
+		pr_err("Failed to re-register event %u\n", event->event_num);
+		sdei_event_destroy(event);
+		return err;
+	}
+
+	if (event->reenable) {
+		if (event->type == SDEI_EVENT_TYPE_SHARED)
+			err = sdei_api_event_enable(event->event_num);
+		else
+			err = sdei_do_cross_call(_local_event_enable, event);
+	}
+
+	if (err)
+		pr_err("Failed to re-enable event %u\n", event->event_num);
+
+	return err;
+}
+
+static int sdei_reregister_shared(void)
+{
+	int err = 0;
+	struct sdei_event *event;
+
+	mutex_lock(&sdei_events_lock);
+	spin_lock(&sdei_list_lock);
+	list_for_each_entry(event, &sdei_list, list) {
+		if (event->type != SDEI_EVENT_TYPE_SHARED)
+			continue;
+
+		if (event->reregister) {
+			err = sdei_reregister_event(event);
+			if (err)
+				break;
+		}
+	}
+	spin_unlock(&sdei_list_lock);
+	mutex_unlock(&sdei_events_lock);
+
+	return err;
+}
+
+static int sdei_cpuhp_down(unsigned int cpu)
+{
+	struct sdei_event *event;
+	struct sdei_crosscall_args arg;
+
+	/* un-register private events */
+	spin_lock(&sdei_list_lock);
+	list_for_each_entry(event, &sdei_list, list) {
+		if (event->type == SDEI_EVENT_TYPE_SHARED)
+			continue;
+
+		CROSSCALL_INIT(arg, event);
+		/* call the cross-call function locally... */
+		_local_event_unregister(&arg);
+		if (arg.first_error)
+			pr_err("Failed to unregister event %u: %d\n",
+			       event->event_num, arg.first_error);
+	}
+	spin_unlock(&sdei_list_lock);
+
+	return sdei_mask_local_cpu();
+}
+
+static int sdei_cpuhp_up(unsigned int cpu)
+{
+	struct sdei_event *event;
+	struct sdei_crosscall_args arg;
+
+	/* re-register/enable private events */
+	spin_lock(&sdei_list_lock);
+	list_for_each_entry(event, &sdei_list, list) {
+		if (event->type == SDEI_EVENT_TYPE_SHARED)
+			continue;
+
+		if (event->reregister) {
+			CROSSCALL_INIT(arg, event);
+			/* call the cross-call function locally... */
+			_local_event_register(&arg);
+			if (arg.first_error)
+				pr_err("Failed to re-register event %u: %d\n",
+				       event->event_num, arg.first_error);
+		}
+
+		if (event->reenable) {
+			CROSSCALL_INIT(arg, event);
+			_local_event_enable(&arg);
+			if (arg.first_error)
+				pr_err("Failed to re-enable event %u: %d\n",
+				       event->event_num, arg.first_error);
+		}
+	}
+	spin_unlock(&sdei_list_lock);
+
+	return sdei_unmask_local_cpu();
+}
+
+/* When entering idle, mask/unmask events for this cpu */
+static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
+			    void *data)
+{
+	int rv;
+
+	switch (action) {
+	case CPU_PM_ENTER:
+		rv = sdei_mask_local_cpu();
+		break;
+	case CPU_PM_EXIT:
+	case CPU_PM_ENTER_FAILED:
+		rv = sdei_unmask_local_cpu();
+		break;
+	default:
+		return NOTIFY_DONE;
+	}
+
+	if (rv)
+		return notifier_from_errno(rv);
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block sdei_pm_nb = {
+	.notifier_call = sdei_pm_notifier,
+};
+
+static int sdei_device_suspend(struct device *dev)
+{
+	on_each_cpu(_ipi_mask_cpu, NULL, true);
+
+	return 0;
+}
+
+static int sdei_device_resume(struct device *dev)
+{
+	on_each_cpu(_ipi_unmask_cpu, NULL, true);
+
+	return 0;
+}
+
+/*
+ * We need all events to be reregistered when we resume from hibernate.
+ *
+ * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
+ * events during freeze, then re-register and re-enable them during thaw
+ * and restore.
+ */
+static int sdei_device_freeze(struct device *dev)
+{
+	int err;
+
+	/* unregister private events */
+	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
+
+	err = sdei_unregister_shared();
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int sdei_device_thaw(struct device *dev)
+{
+	int err;
+
+	/* re-register shared events */
+	err = sdei_reregister_shared();
+	if (err) {
+		pr_warn("Failed to re-register shared events...\n");
+		sdei_mark_interface_broken();
+		return err;
+	}
+
+	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
+				&sdei_cpuhp_up, &sdei_cpuhp_down);
+	if (err)
+		pr_warn("Failed to re-register CPU hotplug notifier...\n");
+
+	return err;
+}
+
+static int sdei_device_restore(struct device *dev)
+{
+	int err;
+
+	err = sdei_platform_reset();
+	if (err)
+		return err;
+
+	return sdei_device_thaw(dev);
+}
+
+static const struct dev_pm_ops sdei_pm_ops = {
+	.suspend = sdei_device_suspend,
+	.resume = sdei_device_resume,
+	.freeze = sdei_device_freeze,
+	.thaw = sdei_device_thaw,
+	.restore = sdei_device_restore,
+};
+
+/*
+ * Mask all CPUs and unregister all events on panic, reboot or kexec.
+ */
+static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
+				void *data)
+{
+	/*
+	 * We are going to reset the interface, after this there is no point
+	 * doing work when we take CPUs offline.
+	 */
+	cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
+
+	sdei_platform_reset();
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block sdei_reboot_nb = {
+	.notifier_call = sdei_reboot_notifier,
+};
+
+static void sdei_smccc_smc(unsigned long function_id,
+			   unsigned long arg0, unsigned long arg1,
+			   unsigned long arg2, unsigned long arg3,
+			   unsigned long arg4, struct arm_smccc_res *res)
+{
+	arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
+}
+
+static void sdei_smccc_hvc(unsigned long function_id,
+			   unsigned long arg0, unsigned long arg1,
+			   unsigned long arg2, unsigned long arg3,
+			   unsigned long arg4, struct arm_smccc_res *res)
+{
+	arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
+}
+
+static int sdei_get_conduit(struct platform_device *pdev)
+{
+	const char *method;
+	struct device_node *np = pdev->dev.of_node;
+
+	sdei_firmware_call = NULL;
+	if (np) {
+		if (of_property_read_string(np, "method", &method)) {
+			pr_warn("missing \"method\" property\n");
+			return CONDUIT_INVALID;
+		}
+
+		if (!strcmp("hvc", method)) {
+			sdei_firmware_call = &sdei_smccc_hvc;
+			return CONDUIT_HVC;
+		} else if (!strcmp("smc", method)) {
+			sdei_firmware_call = &sdei_smccc_smc;
+			return CONDUIT_SMC;
+		}
+
+		pr_warn("invalid \"method\" property: %s\n", method);
+	} else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
+		if (acpi_psci_use_hvc()) {
+			sdei_firmware_call = &sdei_smccc_hvc;
+			return CONDUIT_HVC;
+		} else {
+			sdei_firmware_call = &sdei_smccc_smc;
+			return CONDUIT_SMC;
+		}
+	}
+
+	return CONDUIT_INVALID;
+}
+
+static int sdei_probe(struct platform_device *pdev)
+{
+	int err;
+	u64 ver = 0;
+	int conduit;
+
+	conduit = sdei_get_conduit(pdev);
+	if (!sdei_firmware_call)
+		return 0;
+
+	err = sdei_api_get_version(&ver);
+	if (err == -EOPNOTSUPP)
+		pr_err("advertised but not implemented in platform firmware\n");
+	if (err) {
+		pr_err("Failed to get SDEI version: %d\n", err);
+		sdei_mark_interface_broken();
+		return err;
+	}
+
+	pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
+		(int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
+		(int)SDEI_VERSION_VENDOR(ver));
+
+	if (SDEI_VERSION_MAJOR(ver) != 1) {
+		pr_warn("Conflicting SDEI version detected.\n");
+		sdei_mark_interface_broken();
+		return -EINVAL;
+	}
+
+	err = sdei_platform_reset();
+	if (err)
+		return err;
+
+	sdei_entry_point = sdei_arch_get_entry_point(conduit);
+	if (!sdei_entry_point) {
+		/* Not supported due to hardware or boot configuration */
+		sdei_mark_interface_broken();
+		return 0;
+	}
+
+	err = cpu_pm_register_notifier(&sdei_pm_nb);
+	if (err) {
+		pr_warn("Failed to register CPU PM notifier...\n");
+		goto error;
+	}
+
+	err = register_reboot_notifier(&sdei_reboot_nb);
+	if (err) {
+		pr_warn("Failed to register reboot notifier...\n");
+		goto remove_cpupm;
+	}
+
+	err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
+				&sdei_cpuhp_up, &sdei_cpuhp_down);
+	if (err) {
+		pr_warn("Failed to register CPU hotplug notifier...\n");
+		goto remove_reboot;
+	}
+
+	return 0;
+
+remove_reboot:
+	unregister_reboot_notifier(&sdei_reboot_nb);
+
+remove_cpupm:
+	cpu_pm_unregister_notifier(&sdei_pm_nb);
+
+error:
+	sdei_mark_interface_broken();
+	return err;
+}
+
+static const struct of_device_id sdei_of_match[] = {
+	{ .compatible = "arm,sdei-1.0" },
+	{}
+};
+
+static struct platform_driver sdei_driver = {
+	.driver		= {
+		.name			= "sdei",
+		.pm			= &sdei_pm_ops,
+		.of_match_table		= sdei_of_match,
+	},
+	.probe		= sdei_probe,
+};
+
+static bool __init sdei_present_dt(void)
+{
+	struct platform_device *pdev;
+	struct device_node *np, *fw_np;
+
+	fw_np = of_find_node_by_name(NULL, "firmware");
+	if (!fw_np)
+		return false;
+
+	np = of_find_matching_node(fw_np, sdei_of_match);
+	of_node_put(fw_np);
+	if (!np)
+		return false;
+
+	pdev = of_platform_device_create(np, sdei_driver.driver.name, NULL);
+	of_node_put(np);
+	if (!pdev)
+		return false;
+
+	return true;
+}
+
+static bool __init sdei_present_acpi(void)
+{
+	acpi_status status;
+	struct platform_device *pdev;
+	struct acpi_table_header *sdei_table_header;
+
+	if (acpi_disabled)
+		return false;
+
+	status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
+	if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
+		const char *msg = acpi_format_exception(status);
+
+		pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
+	}
+	if (ACPI_FAILURE(status))
+		return false;
+
+	pdev = platform_device_register_simple(sdei_driver.driver.name, 0, NULL,
+					       0);
+	if (IS_ERR(pdev))
+		return false;
+
+	return true;
+}
+
+static int __init sdei_init(void)
+{
+	if (sdei_present_dt() || sdei_present_acpi())
+		platform_driver_register(&sdei_driver);
+
+	return 0;
+}
+
+/*
+ * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
+ * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
+ * by device_initcall(). We want to be called in the middle.
+ */
+subsys_initcall_sync(sdei_init);
+
+int sdei_event_handler(struct pt_regs *regs,
+		       struct sdei_registered_event *arg)
+{
+	int err;
+	mm_segment_t orig_addr_limit;
+	u32 event_num = arg->event_num;
+
+	orig_addr_limit = get_fs();
+	set_fs(USER_DS);
+
+	err = arg->callback(event_num, regs, arg->callback_arg);
+	if (err)
+		pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
+				   event_num, smp_processor_id(), err);
+
+	set_fs(orig_addr_limit);
+
+	return err;
+}
+NOKPROBE_SYMBOL(sdei_event_handler);
diff --git a/drivers/firmware/dcdbas.c b/drivers/firmware/dcdbas.c
index c16600f30611..0bdea60c65dd 100644
--- a/drivers/firmware/dcdbas.c
+++ b/drivers/firmware/dcdbas.c
@@ -639,7 +639,7 @@ static void __exit dcdbas_exit(void)
 	platform_driver_unregister(&dcdbas_driver);
 }
 
-module_init(dcdbas_init);
+subsys_initcall_sync(dcdbas_init);
 module_exit(dcdbas_exit);
 
 MODULE_DESCRIPTION(DRIVER_DESCRIPTION " (version " DRIVER_VERSION ")");
diff --git a/drivers/firmware/dmi-sysfs.c b/drivers/firmware/dmi-sysfs.c
index d5de6ee8466d..ecf2eeb5f6f9 100644
--- a/drivers/firmware/dmi-sysfs.c
+++ b/drivers/firmware/dmi-sysfs.c
@@ -652,7 +652,7 @@ static int __init dmi_sysfs_init(void)
 	int val;
 
 	if (!dmi_kobj) {
-		pr_err("dmi-sysfs: dmi entry is absent.\n");
+		pr_debug("dmi-sysfs: dmi entry is absent.\n");
 		error = -ENODATA;
 		goto err;
 	}
diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c
index 783041964439..e763e1484331 100644
--- a/drivers/firmware/dmi_scan.c
+++ b/drivers/firmware/dmi_scan.c
@@ -18,7 +18,7 @@ EXPORT_SYMBOL_GPL(dmi_kobj);
  * of and an antecedent to, SMBIOS, which stands for System
  * Management BIOS.  See further: http://www.dmtf.org/standards
  */
-static const char dmi_empty_string[] = "        ";
+static const char dmi_empty_string[] = "";
 
 static u32 dmi_ver __initdata;
 static u32 dmi_len;
@@ -26,11 +26,6 @@ static u16 dmi_num;
 static u8 smbios_entry_point[32];
 static int smbios_entry_point_size;
 
-/*
- * Catch too early calls to dmi_check_system():
- */
-static int dmi_initialized;
-
 /* DMI system identification string used during boot */
 static char dmi_ids_string[128] __initdata;
 
@@ -44,25 +39,21 @@ static int dmi_memdev_nr;
 static const char * __init dmi_string_nosave(const struct dmi_header *dm, u8 s)
 {
 	const u8 *bp = ((u8 *) dm) + dm->length;
+	const u8 *nsp;
 
 	if (s) {
-		s--;
-		while (s > 0 && *bp) {
+		while (--s > 0 && *bp)
 			bp += strlen(bp) + 1;
-			s--;
-		}
 
-		if (*bp != 0) {
-			size_t len = strlen(bp)+1;
-			size_t cmp_len = len > 8 ? 8 : len;
-
-			if (!memcmp(bp, dmi_empty_string, cmp_len))
-				return dmi_empty_string;
+		/* Strings containing only spaces are considered empty */
+		nsp = bp;
+		while (*nsp == ' ')
+			nsp++;
+		if (*nsp != '\0')
 			return bp;
-		}
 	}
 
-	return "";
+	return dmi_empty_string;
 }
 
 static const char * __init dmi_string(const struct dmi_header *dm, u8 s)
@@ -633,7 +624,7 @@ void __init dmi_scan_machine(void)
 
 			if (!dmi_smbios3_present(buf)) {
 				dmi_available = 1;
-				goto out;
+				return;
 			}
 		}
 		if (efi.smbios == EFI_INVALID_TABLE_ADDR)
@@ -651,7 +642,7 @@ void __init dmi_scan_machine(void)
 
 		if (!dmi_present(buf)) {
 			dmi_available = 1;
-			goto out;
+			return;
 		}
 	} else if (IS_ENABLED(CONFIG_DMI_SCAN_MACHINE_NON_EFI_FALLBACK)) {
 		p = dmi_early_remap(0xF0000, 0x10000);
@@ -668,7 +659,7 @@ void __init dmi_scan_machine(void)
 			if (!dmi_smbios3_present(buf)) {
 				dmi_available = 1;
 				dmi_early_unmap(p, 0x10000);
-				goto out;
+				return;
 			}
 			memcpy(buf, buf + 16, 16);
 		}
@@ -686,7 +677,7 @@ void __init dmi_scan_machine(void)
 			if (!dmi_present(buf)) {
 				dmi_available = 1;
 				dmi_early_unmap(p, 0x10000);
-				goto out;
+				return;
 			}
 			memcpy(buf, buf + 16, 16);
 		}
@@ -694,8 +685,6 @@ void __init dmi_scan_machine(void)
 	}
  error:
 	pr_info("DMI not present or invalid.\n");
- out:
-	dmi_initialized = 1;
 }
 
 static ssize_t raw_table_read(struct file *file, struct kobject *kobj,
@@ -715,10 +704,8 @@ static int __init dmi_init(void)
 	u8 *dmi_table;
 	int ret = -ENOMEM;
 
-	if (!dmi_available) {
-		ret = -ENODATA;
-		goto err;
-	}
+	if (!dmi_available)
+		return 0;
 
 	/*
 	 * Set up dmi directory at /sys/firmware/dmi. This entry should stay
@@ -784,19 +771,20 @@ static bool dmi_matches(const struct dmi_system_id *dmi)
 {
 	int i;
 
-	WARN(!dmi_initialized, KERN_ERR "dmi check: not initialized yet.\n");
-
 	for (i = 0; i < ARRAY_SIZE(dmi->matches); i++) {
 		int s = dmi->matches[i].slot;
 		if (s == DMI_NONE)
 			break;
 		if (dmi_ident[s]) {
-			if (!dmi->matches[i].exact_match &&
-			    strstr(dmi_ident[s], dmi->matches[i].substr))
-				continue;
-			else if (dmi->matches[i].exact_match &&
-				 !strcmp(dmi_ident[s], dmi->matches[i].substr))
-				continue;
+			if (dmi->matches[i].exact_match) {
+				if (!strcmp(dmi_ident[s],
+					    dmi->matches[i].substr))
+					continue;
+			} else {
+				if (strstr(dmi_ident[s],
+					   dmi->matches[i].substr))
+					continue;
+			}
 		}
 
 		/* No match */
@@ -826,6 +814,8 @@ static bool dmi_is_end_of_table(const struct dmi_system_id *dmi)
  *	Walk the blacklist table running matching functions until someone
  *	returns non zero or we hit the end. Callback function is called for
  *	each successful match. Returns the number of matches.
+ *
+ *	dmi_scan_machine must be called before this function is called.
  */
 int dmi_check_system(const struct dmi_system_id *list)
 {
@@ -854,6 +844,8 @@ EXPORT_SYMBOL(dmi_check_system);
  *
  *	Walk the blacklist table until the first match is found.  Return the
  *	pointer to the matching entry or NULL if there's no match.
+ *
+ *	dmi_scan_machine must be called before this function is called.
  */
 const struct dmi_system_id *dmi_first_match(const struct dmi_system_id *list)
 {
diff --git a/drivers/firmware/edd.c b/drivers/firmware/edd.c
index e22957665808..60a8f1363a10 100644
--- a/drivers/firmware/edd.c
+++ b/drivers/firmware/edd.c
@@ -669,10 +669,10 @@ edd_get_pci_dev(struct edd_device *edev)
 	struct edd_info *info = edd_dev_get_info(edev);
 
 	if (edd_dev_is_type(edev, "PCI") || edd_dev_is_type(edev, "XPRS")) {
-		return pci_get_bus_and_slot(info->params.interface_path.pci.bus,
-				     PCI_DEVFN(info->params.interface_path.pci.slot,
-					       info->params.interface_path.pci.
-					       function));
+		return pci_get_domain_bus_and_slot(0,
+				info->params.interface_path.pci.bus,
+				PCI_DEVFN(info->params.interface_path.pci.slot,
+				info->params.interface_path.pci.function));
 	}
 	return NULL;
 }
diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig
index 2b4c39fdfa91..3098410abad8 100644
--- a/drivers/firmware/efi/Kconfig
+++ b/drivers/firmware/efi/Kconfig
@@ -113,7 +113,7 @@ config EFI_CAPSULE_LOADER
 	  Most users should say N.
 
 config EFI_CAPSULE_QUIRK_QUARK_CSH
-	boolean "Add support for Quark capsules with non-standard headers"
+	bool "Add support for Quark capsules with non-standard headers"
 	depends on X86 && !64BIT
 	select EFI_CAPSULE_LOADER
 	default y
@@ -159,13 +159,21 @@ config RESET_ATTACK_MITIGATION
 	  using the TCG Platform Reset Attack Mitigation specification. This
 	  protects against an attacker forcibly rebooting the system while it
 	  still contains secrets in RAM, booting another OS and extracting the
-	  secrets.
+	  secrets. This should only be enabled when userland is configured to
+	  clear the MemoryOverwriteRequest flag on clean shutdown after secrets
+	  have been evicted, since otherwise it will trigger even on clean
+	  reboots.
 
 endmenu
 
 config UEFI_CPER
 	bool
 
+config UEFI_CPER_ARM
+	bool
+	depends on UEFI_CPER && ( ARM || ARM64 )
+	default y
+
 config EFI_DEV_PATH_PARSER
 	bool
 	depends on ACPI
diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile
index 269501dfba53..cb805374f4bc 100644
--- a/drivers/firmware/efi/Makefile
+++ b/drivers/firmware/efi/Makefile
@@ -11,7 +11,7 @@
 KASAN_SANITIZE_runtime-wrappers.o	:= n
 
 obj-$(CONFIG_ACPI_BGRT) 		+= efi-bgrt.o
-obj-$(CONFIG_EFI)			+= efi.o vars.o reboot.o memattr.o
+obj-$(CONFIG_EFI)			+= efi.o vars.o reboot.o memattr.o tpm.o
 obj-$(CONFIG_EFI)			+= capsule.o memmap.o
 obj-$(CONFIG_EFI_VARS)			+= efivars.o
 obj-$(CONFIG_EFI_ESRT)			+= esrt.o
@@ -30,3 +30,4 @@ arm-obj-$(CONFIG_EFI)			:= arm-init.o arm-runtime.o
 obj-$(CONFIG_ARM)			+= $(arm-obj-y)
 obj-$(CONFIG_ARM64)			+= $(arm-obj-y)
 obj-$(CONFIG_EFI_CAPSULE_LOADER)	+= capsule-loader.o
+obj-$(CONFIG_UEFI_CPER_ARM)		+= cper-arm.o
diff --git a/drivers/firmware/efi/capsule-loader.c b/drivers/firmware/efi/capsule-loader.c
index 055e2e8f985a..e456f4602df1 100644
--- a/drivers/firmware/efi/capsule-loader.c
+++ b/drivers/firmware/efi/capsule-loader.c
@@ -45,7 +45,7 @@ int __efi_capsule_setup_info(struct capsule_info *cap_info)
 	pages_needed = ALIGN(cap_info->total_size, PAGE_SIZE) / PAGE_SIZE;
 
 	if (pages_needed == 0) {
-		pr_err("invalid capsule size");
+		pr_err("invalid capsule size\n");
 		return -EINVAL;
 	}
 
diff --git a/drivers/firmware/efi/cper-arm.c b/drivers/firmware/efi/cper-arm.c
new file mode 100644
index 000000000000..698e5c8e0c8d
--- /dev/null
+++ b/drivers/firmware/efi/cper-arm.c
@@ -0,0 +1,356 @@
+/*
+ * UEFI Common Platform Error Record (CPER) support
+ *
+ * Copyright (C) 2017, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/time.h>
+#include <linux/cper.h>
+#include <linux/dmi.h>
+#include <linux/acpi.h>
+#include <linux/pci.h>
+#include <linux/aer.h>
+#include <linux/printk.h>
+#include <linux/bcd.h>
+#include <acpi/ghes.h>
+#include <ras/ras_event.h>
+
+#define INDENT_SP	" "
+
+static const char * const arm_reg_ctx_strs[] = {
+	"AArch32 general purpose registers",
+	"AArch32 EL1 context registers",
+	"AArch32 EL2 context registers",
+	"AArch32 secure context registers",
+	"AArch64 general purpose registers",
+	"AArch64 EL1 context registers",
+	"AArch64 EL2 context registers",
+	"AArch64 EL3 context registers",
+	"Misc. system register structure",
+};
+
+static const char * const arm_err_trans_type_strs[] = {
+	"Instruction",
+	"Data Access",
+	"Generic",
+};
+
+static const char * const arm_bus_err_op_strs[] = {
+	"Generic error (type cannot be determined)",
+	"Generic read (type of instruction or data request cannot be determined)",
+	"Generic write (type of instruction of data request cannot be determined)",
+	"Data read",
+	"Data write",
+	"Instruction fetch",
+	"Prefetch",
+};
+
+static const char * const arm_cache_err_op_strs[] = {
+	"Generic error (type cannot be determined)",
+	"Generic read (type of instruction or data request cannot be determined)",
+	"Generic write (type of instruction of data request cannot be determined)",
+	"Data read",
+	"Data write",
+	"Instruction fetch",
+	"Prefetch",
+	"Eviction",
+	"Snooping (processor initiated a cache snoop that resulted in an error)",
+	"Snooped (processor raised a cache error caused by another processor or device snooping its cache)",
+	"Management",
+};
+
+static const char * const arm_tlb_err_op_strs[] = {
+	"Generic error (type cannot be determined)",
+	"Generic read (type of instruction or data request cannot be determined)",
+	"Generic write (type of instruction of data request cannot be determined)",
+	"Data read",
+	"Data write",
+	"Instruction fetch",
+	"Prefetch",
+	"Local management operation (processor initiated a TLB management operation that resulted in an error)",
+	"External management operation (processor raised a TLB error caused by another processor or device broadcasting TLB operations)",
+};
+
+static const char * const arm_bus_err_part_type_strs[] = {
+	"Local processor originated request",
+	"Local processor responded to request",
+	"Local processor observed",
+	"Generic",
+};
+
+static const char * const arm_bus_err_addr_space_strs[] = {
+	"External Memory Access",
+	"Internal Memory Access",
+	"Unknown",
+	"Device Memory Access",
+};
+
+static void cper_print_arm_err_info(const char *pfx, u32 type,
+				    u64 error_info)
+{
+	u8 trans_type, op_type, level, participation_type, address_space;
+	u16 mem_attributes;
+	bool proc_context_corrupt, corrected, precise_pc, restartable_pc;
+	bool time_out, access_mode;
+
+	/* If the type is unknown, bail. */
+	if (type > CPER_ARM_MAX_TYPE)
+		return;
+
+	/*
+	 * Vendor type errors have error information values that are vendor
+	 * specific.
+	 */
+	if (type == CPER_ARM_VENDOR_ERROR)
+		return;
+
+	if (error_info & CPER_ARM_ERR_VALID_TRANSACTION_TYPE) {
+		trans_type = ((error_info >> CPER_ARM_ERR_TRANSACTION_SHIFT)
+			      & CPER_ARM_ERR_TRANSACTION_MASK);
+		if (trans_type < ARRAY_SIZE(arm_err_trans_type_strs)) {
+			printk("%stransaction type: %s\n", pfx,
+			       arm_err_trans_type_strs[trans_type]);
+		}
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_OPERATION_TYPE) {
+		op_type = ((error_info >> CPER_ARM_ERR_OPERATION_SHIFT)
+			   & CPER_ARM_ERR_OPERATION_MASK);
+		switch (type) {
+		case CPER_ARM_CACHE_ERROR:
+			if (op_type < ARRAY_SIZE(arm_cache_err_op_strs)) {
+				printk("%soperation type: %s\n", pfx,
+				       arm_cache_err_op_strs[op_type]);
+			}
+			break;
+		case CPER_ARM_TLB_ERROR:
+			if (op_type < ARRAY_SIZE(arm_tlb_err_op_strs)) {
+				printk("%soperation type: %s\n", pfx,
+				       arm_tlb_err_op_strs[op_type]);
+			}
+			break;
+		case CPER_ARM_BUS_ERROR:
+			if (op_type < ARRAY_SIZE(arm_bus_err_op_strs)) {
+				printk("%soperation type: %s\n", pfx,
+				       arm_bus_err_op_strs[op_type]);
+			}
+			break;
+		}
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_LEVEL) {
+		level = ((error_info >> CPER_ARM_ERR_LEVEL_SHIFT)
+			 & CPER_ARM_ERR_LEVEL_MASK);
+		switch (type) {
+		case CPER_ARM_CACHE_ERROR:
+			printk("%scache level: %d\n", pfx, level);
+			break;
+		case CPER_ARM_TLB_ERROR:
+			printk("%sTLB level: %d\n", pfx, level);
+			break;
+		case CPER_ARM_BUS_ERROR:
+			printk("%saffinity level at which the bus error occurred: %d\n",
+			       pfx, level);
+			break;
+		}
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_PROC_CONTEXT_CORRUPT) {
+		proc_context_corrupt = ((error_info >> CPER_ARM_ERR_PC_CORRUPT_SHIFT)
+					& CPER_ARM_ERR_PC_CORRUPT_MASK);
+		if (proc_context_corrupt)
+			printk("%sprocessor context corrupted\n", pfx);
+		else
+			printk("%sprocessor context not corrupted\n", pfx);
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_CORRECTED) {
+		corrected = ((error_info >> CPER_ARM_ERR_CORRECTED_SHIFT)
+			     & CPER_ARM_ERR_CORRECTED_MASK);
+		if (corrected)
+			printk("%sthe error has been corrected\n", pfx);
+		else
+			printk("%sthe error has not been corrected\n", pfx);
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_PRECISE_PC) {
+		precise_pc = ((error_info >> CPER_ARM_ERR_PRECISE_PC_SHIFT)
+			      & CPER_ARM_ERR_PRECISE_PC_MASK);
+		if (precise_pc)
+			printk("%sPC is precise\n", pfx);
+		else
+			printk("%sPC is imprecise\n", pfx);
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_RESTARTABLE_PC) {
+		restartable_pc = ((error_info >> CPER_ARM_ERR_RESTARTABLE_PC_SHIFT)
+				  & CPER_ARM_ERR_RESTARTABLE_PC_MASK);
+		if (restartable_pc)
+			printk("%sProgram execution can be restarted reliably at the PC associated with the error.\n", pfx);
+	}
+
+	/* The rest of the fields are specific to bus errors */
+	if (type != CPER_ARM_BUS_ERROR)
+		return;
+
+	if (error_info & CPER_ARM_ERR_VALID_PARTICIPATION_TYPE) {
+		participation_type = ((error_info >> CPER_ARM_ERR_PARTICIPATION_TYPE_SHIFT)
+				      & CPER_ARM_ERR_PARTICIPATION_TYPE_MASK);
+		if (participation_type < ARRAY_SIZE(arm_bus_err_part_type_strs)) {
+			printk("%sparticipation type: %s\n", pfx,
+			       arm_bus_err_part_type_strs[participation_type]);
+		}
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_TIME_OUT) {
+		time_out = ((error_info >> CPER_ARM_ERR_TIME_OUT_SHIFT)
+			    & CPER_ARM_ERR_TIME_OUT_MASK);
+		if (time_out)
+			printk("%srequest timed out\n", pfx);
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_ADDRESS_SPACE) {
+		address_space = ((error_info >> CPER_ARM_ERR_ADDRESS_SPACE_SHIFT)
+				 & CPER_ARM_ERR_ADDRESS_SPACE_MASK);
+		if (address_space < ARRAY_SIZE(arm_bus_err_addr_space_strs)) {
+			printk("%saddress space: %s\n", pfx,
+			       arm_bus_err_addr_space_strs[address_space]);
+		}
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_MEM_ATTRIBUTES) {
+		mem_attributes = ((error_info >> CPER_ARM_ERR_MEM_ATTRIBUTES_SHIFT)
+				  & CPER_ARM_ERR_MEM_ATTRIBUTES_MASK);
+		printk("%smemory access attributes:0x%x\n", pfx, mem_attributes);
+	}
+
+	if (error_info & CPER_ARM_ERR_VALID_ACCESS_MODE) {
+		access_mode = ((error_info >> CPER_ARM_ERR_ACCESS_MODE_SHIFT)
+			       & CPER_ARM_ERR_ACCESS_MODE_MASK);
+		if (access_mode)
+			printk("%saccess mode: normal\n", pfx);
+		else
+			printk("%saccess mode: secure\n", pfx);
+	}
+}
+
+void cper_print_proc_arm(const char *pfx,
+			 const struct cper_sec_proc_arm *proc)
+{
+	int i, len, max_ctx_type;
+	struct cper_arm_err_info *err_info;
+	struct cper_arm_ctx_info *ctx_info;
+	char newpfx[64], infopfx[64];
+
+	printk("%sMIDR: 0x%016llx\n", pfx, proc->midr);
+
+	len = proc->section_length - (sizeof(*proc) +
+		proc->err_info_num * (sizeof(*err_info)));
+	if (len < 0) {
+		printk("%ssection length: %d\n", pfx, proc->section_length);
+		printk("%ssection length is too small\n", pfx);
+		printk("%sfirmware-generated error record is incorrect\n", pfx);
+		printk("%sERR_INFO_NUM is %d\n", pfx, proc->err_info_num);
+		return;
+	}
+
+	if (proc->validation_bits & CPER_ARM_VALID_MPIDR)
+		printk("%sMultiprocessor Affinity Register (MPIDR): 0x%016llx\n",
+			pfx, proc->mpidr);
+
+	if (proc->validation_bits & CPER_ARM_VALID_AFFINITY_LEVEL)
+		printk("%serror affinity level: %d\n", pfx,
+			proc->affinity_level);
+
+	if (proc->validation_bits & CPER_ARM_VALID_RUNNING_STATE) {
+		printk("%srunning state: 0x%x\n", pfx, proc->running_state);
+		printk("%sPower State Coordination Interface state: %d\n",
+			pfx, proc->psci_state);
+	}
+
+	snprintf(newpfx, sizeof(newpfx), "%s%s", pfx, INDENT_SP);
+
+	err_info = (struct cper_arm_err_info *)(proc + 1);
+	for (i = 0; i < proc->err_info_num; i++) {
+		printk("%sError info structure %d:\n", pfx, i);
+
+		printk("%snum errors: %d\n", pfx, err_info->multiple_error + 1);
+
+		if (err_info->validation_bits & CPER_ARM_INFO_VALID_FLAGS) {
+			if (err_info->flags & CPER_ARM_INFO_FLAGS_FIRST)
+				printk("%sfirst error captured\n", newpfx);
+			if (err_info->flags & CPER_ARM_INFO_FLAGS_LAST)
+				printk("%slast error captured\n", newpfx);
+			if (err_info->flags & CPER_ARM_INFO_FLAGS_PROPAGATED)
+				printk("%spropagated error captured\n",
+				       newpfx);
+			if (err_info->flags & CPER_ARM_INFO_FLAGS_OVERFLOW)
+				printk("%soverflow occurred, error info is incomplete\n",
+				       newpfx);
+		}
+
+		printk("%serror_type: %d, %s\n", newpfx, err_info->type,
+			err_info->type < ARRAY_SIZE(cper_proc_error_type_strs) ?
+			cper_proc_error_type_strs[err_info->type] : "unknown");
+		if (err_info->validation_bits & CPER_ARM_INFO_VALID_ERR_INFO) {
+			printk("%serror_info: 0x%016llx\n", newpfx,
+			       err_info->error_info);
+			snprintf(infopfx, sizeof(infopfx), "%s%s", newpfx, INDENT_SP);
+			cper_print_arm_err_info(infopfx, err_info->type,
+						err_info->error_info);
+		}
+		if (err_info->validation_bits & CPER_ARM_INFO_VALID_VIRT_ADDR)
+			printk("%svirtual fault address: 0x%016llx\n",
+				newpfx, err_info->virt_fault_addr);
+		if (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR)
+			printk("%sphysical fault address: 0x%016llx\n",
+				newpfx, err_info->physical_fault_addr);
+		err_info += 1;
+	}
+
+	ctx_info = (struct cper_arm_ctx_info *)err_info;
+	max_ctx_type = ARRAY_SIZE(arm_reg_ctx_strs) - 1;
+	for (i = 0; i < proc->context_info_num; i++) {
+		int size = sizeof(*ctx_info) + ctx_info->size;
+
+		printk("%sContext info structure %d:\n", pfx, i);
+		if (len < size) {
+			printk("%ssection length is too small\n", newpfx);
+			printk("%sfirmware-generated error record is incorrect\n", pfx);
+			return;
+		}
+		if (ctx_info->type > max_ctx_type) {
+			printk("%sInvalid context type: %d (max: %d)\n",
+				newpfx, ctx_info->type, max_ctx_type);
+			return;
+		}
+		printk("%sregister context type: %s\n", newpfx,
+			arm_reg_ctx_strs[ctx_info->type]);
+		print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, 4,
+				(ctx_info + 1), ctx_info->size, 0);
+		len -= size;
+		ctx_info = (struct cper_arm_ctx_info *)((long)ctx_info + size);
+	}
+
+	if (len > 0) {
+		printk("%sVendor specific error info has %u bytes:\n", pfx,
+		       len);
+		print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, 4, ctx_info,
+				len, true);
+	}
+}
diff --git a/drivers/firmware/efi/cper.c b/drivers/firmware/efi/cper.c
index d2fcafcea07e..c165933ebf38 100644
--- a/drivers/firmware/efi/cper.c
+++ b/drivers/firmware/efi/cper.c
@@ -122,7 +122,7 @@ static const char * const proc_isa_strs[] = {
 	"ARM A64",
 };
 
-static const char * const proc_error_type_strs[] = {
+const char * const cper_proc_error_type_strs[] = {
 	"cache error",
 	"TLB error",
 	"bus error",
@@ -157,8 +157,8 @@ static void cper_print_proc_generic(const char *pfx,
 	if (proc->validation_bits & CPER_PROC_VALID_ERROR_TYPE) {
 		printk("%s""error_type: 0x%02x\n", pfx, proc->proc_error_type);
 		cper_print_bits(pfx, proc->proc_error_type,
-				proc_error_type_strs,
-				ARRAY_SIZE(proc_error_type_strs));
+				cper_proc_error_type_strs,
+				ARRAY_SIZE(cper_proc_error_type_strs));
 	}
 	if (proc->validation_bits & CPER_PROC_VALID_OPERATION)
 		printk("%s""operation: %d, %s\n", pfx, proc->operation,
@@ -188,122 +188,6 @@ static void cper_print_proc_generic(const char *pfx,
 		printk("%s""IP: 0x%016llx\n", pfx, proc->ip);
 }
 
-#if defined(CONFIG_ARM64) || defined(CONFIG_ARM)
-static const char * const arm_reg_ctx_strs[] = {
-	"AArch32 general purpose registers",
-	"AArch32 EL1 context registers",
-	"AArch32 EL2 context registers",
-	"AArch32 secure context registers",
-	"AArch64 general purpose registers",
-	"AArch64 EL1 context registers",
-	"AArch64 EL2 context registers",
-	"AArch64 EL3 context registers",
-	"Misc. system register structure",
-};
-
-static void cper_print_proc_arm(const char *pfx,
-				const struct cper_sec_proc_arm *proc)
-{
-	int i, len, max_ctx_type;
-	struct cper_arm_err_info *err_info;
-	struct cper_arm_ctx_info *ctx_info;
-	char newpfx[64];
-
-	printk("%sMIDR: 0x%016llx\n", pfx, proc->midr);
-
-	len = proc->section_length - (sizeof(*proc) +
-		proc->err_info_num * (sizeof(*err_info)));
-	if (len < 0) {
-		printk("%ssection length: %d\n", pfx, proc->section_length);
-		printk("%ssection length is too small\n", pfx);
-		printk("%sfirmware-generated error record is incorrect\n", pfx);
-		printk("%sERR_INFO_NUM is %d\n", pfx, proc->err_info_num);
-		return;
-	}
-
-	if (proc->validation_bits & CPER_ARM_VALID_MPIDR)
-		printk("%sMultiprocessor Affinity Register (MPIDR): 0x%016llx\n",
-			pfx, proc->mpidr);
-
-	if (proc->validation_bits & CPER_ARM_VALID_AFFINITY_LEVEL)
-		printk("%serror affinity level: %d\n", pfx,
-			proc->affinity_level);
-
-	if (proc->validation_bits & CPER_ARM_VALID_RUNNING_STATE) {
-		printk("%srunning state: 0x%x\n", pfx, proc->running_state);
-		printk("%sPower State Coordination Interface state: %d\n",
-			pfx, proc->psci_state);
-	}
-
-	snprintf(newpfx, sizeof(newpfx), "%s%s", pfx, INDENT_SP);
-
-	err_info = (struct cper_arm_err_info *)(proc + 1);
-	for (i = 0; i < proc->err_info_num; i++) {
-		printk("%sError info structure %d:\n", pfx, i);
-
-		printk("%snum errors: %d\n", pfx, err_info->multiple_error + 1);
-
-		if (err_info->validation_bits & CPER_ARM_INFO_VALID_FLAGS) {
-			if (err_info->flags & CPER_ARM_INFO_FLAGS_FIRST)
-				printk("%sfirst error captured\n", newpfx);
-			if (err_info->flags & CPER_ARM_INFO_FLAGS_LAST)
-				printk("%slast error captured\n", newpfx);
-			if (err_info->flags & CPER_ARM_INFO_FLAGS_PROPAGATED)
-				printk("%spropagated error captured\n",
-				       newpfx);
-			if (err_info->flags & CPER_ARM_INFO_FLAGS_OVERFLOW)
-				printk("%soverflow occurred, error info is incomplete\n",
-				       newpfx);
-		}
-
-		printk("%serror_type: %d, %s\n", newpfx, err_info->type,
-			err_info->type < ARRAY_SIZE(proc_error_type_strs) ?
-			proc_error_type_strs[err_info->type] : "unknown");
-		if (err_info->validation_bits & CPER_ARM_INFO_VALID_ERR_INFO)
-			printk("%serror_info: 0x%016llx\n", newpfx,
-			       err_info->error_info);
-		if (err_info->validation_bits & CPER_ARM_INFO_VALID_VIRT_ADDR)
-			printk("%svirtual fault address: 0x%016llx\n",
-				newpfx, err_info->virt_fault_addr);
-		if (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR)
-			printk("%sphysical fault address: 0x%016llx\n",
-				newpfx, err_info->physical_fault_addr);
-		err_info += 1;
-	}
-
-	ctx_info = (struct cper_arm_ctx_info *)err_info;
-	max_ctx_type = ARRAY_SIZE(arm_reg_ctx_strs) - 1;
-	for (i = 0; i < proc->context_info_num; i++) {
-		int size = sizeof(*ctx_info) + ctx_info->size;
-
-		printk("%sContext info structure %d:\n", pfx, i);
-		if (len < size) {
-			printk("%ssection length is too small\n", newpfx);
-			printk("%sfirmware-generated error record is incorrect\n", pfx);
-			return;
-		}
-		if (ctx_info->type > max_ctx_type) {
-			printk("%sInvalid context type: %d (max: %d)\n",
-				newpfx, ctx_info->type, max_ctx_type);
-			return;
-		}
-		printk("%sregister context type: %s\n", newpfx,
-			arm_reg_ctx_strs[ctx_info->type]);
-		print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, 4,
-				(ctx_info + 1), ctx_info->size, 0);
-		len -= size;
-		ctx_info = (struct cper_arm_ctx_info *)((long)ctx_info + size);
-	}
-
-	if (len > 0) {
-		printk("%sVendor specific error info has %u bytes:\n", pfx,
-		       len);
-		print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, 4, ctx_info,
-				len, true);
-	}
-}
-#endif
-
 static const char * const mem_err_type_strs[] = {
 	"unknown",
 	"no error",
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index 557a47829d03..cd42f66a7c85 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -52,6 +52,7 @@ struct efi __read_mostly efi = {
 	.properties_table	= EFI_INVALID_TABLE_ADDR,
 	.mem_attr_table		= EFI_INVALID_TABLE_ADDR,
 	.rng_seed		= EFI_INVALID_TABLE_ADDR,
+	.tpm_log		= EFI_INVALID_TABLE_ADDR
 };
 EXPORT_SYMBOL(efi);
 
@@ -464,6 +465,7 @@ static __initdata efi_config_table_type_t common_tables[] = {
 	{EFI_PROPERTIES_TABLE_GUID, "PROP", &efi.properties_table},
 	{EFI_MEMORY_ATTRIBUTES_TABLE_GUID, "MEMATTR", &efi.mem_attr_table},
 	{LINUX_EFI_RANDOM_SEED_TABLE_GUID, "RNG", &efi.rng_seed},
+	{LINUX_EFI_TPM_EVENT_LOG_GUID, "TPMEventLog", &efi.tpm_log},
 	{NULL_GUID, NULL, NULL},
 };
 
@@ -552,6 +554,8 @@ int __init efi_config_parse_tables(void *config_tables, int count, int sz,
 	if (efi_enabled(EFI_MEMMAP))
 		efi_memattr_init();
 
+	efi_tpm_eventlog_init();
+
 	/* Parse the EFI Properties table if it exists */
 	if (efi.properties_table != EFI_INVALID_TABLE_ADDR) {
 		efi_properties_table_t *tbl;
@@ -608,7 +612,7 @@ static int __init efi_load_efivars(void)
 		return 0;
 
 	pdev = platform_device_register_simple("efivars", 0, NULL, 0);
-	return IS_ERR(pdev) ? PTR_ERR(pdev) : 0;
+	return PTR_ERR_OR_ZERO(pdev);
 }
 device_initcall(efi_load_efivars);
 #endif
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index adaa4a964f0c..7b3ba40f0745 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -30,8 +30,7 @@ OBJECT_FILES_NON_STANDARD	:= y
 # Prevents link failures: __sanitizer_cov_trace_pc() is not linked in.
 KCOV_INSTRUMENT			:= n
 
-lib-y				:= efi-stub-helper.o gop.o secureboot.o
-lib-$(CONFIG_RESET_ATTACK_MITIGATION) += tpm.o
+lib-y				:= efi-stub-helper.o gop.o secureboot.o tpm.o
 
 # include the stub's generic dependencies from lib/ when building for ARM/arm64
 arm-deps-y := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c fdt_empty_tree.c fdt_sw.c
diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c
index 6224cdbc9669..13c1edd37e96 100644
--- a/drivers/firmware/efi/libstub/tpm.c
+++ b/drivers/firmware/efi/libstub/tpm.c
@@ -4,15 +4,18 @@
  * Copyright (C) 2016 CoreOS, Inc
  * Copyright (C) 2017 Google, Inc.
  *     Matthew Garrett <mjg59@google.com>
+ *     Thiebaud Weksteen <tweek@google.com>
  *
  * This file is part of the Linux kernel, and is made available under the
  * terms of the GNU General Public License version 2.
  */
 #include <linux/efi.h>
+#include <linux/tpm_eventlog.h>
 #include <asm/efi.h>
 
 #include "efistub.h"
 
+#ifdef CONFIG_RESET_ATTACK_MITIGATION
 static const efi_char16_t efi_MemoryOverWriteRequest_name[] = {
 	'M', 'e', 'm', 'o', 'r', 'y', 'O', 'v', 'e', 'r', 'w', 'r', 'i', 't',
 	'e', 'R', 'e', 'q', 'u', 'e', 's', 't', 'C', 'o', 'n', 't', 'r', 'o',
@@ -56,3 +59,81 @@ void efi_enable_reset_attack_mitigation(efi_system_table_t *sys_table_arg)
 		    EFI_VARIABLE_BOOTSERVICE_ACCESS |
 		    EFI_VARIABLE_RUNTIME_ACCESS, sizeof(val), &val);
 }
+
+#endif
+
+void efi_retrieve_tpm2_eventlog_1_2(efi_system_table_t *sys_table_arg)
+{
+	efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID;
+	efi_guid_t linux_eventlog_guid = LINUX_EFI_TPM_EVENT_LOG_GUID;
+	efi_status_t status;
+	efi_physical_addr_t log_location, log_last_entry;
+	struct linux_efi_tpm_eventlog *log_tbl = NULL;
+	unsigned long first_entry_addr, last_entry_addr;
+	size_t log_size, last_entry_size;
+	efi_bool_t truncated;
+	void *tcg2_protocol = NULL;
+
+	status = efi_call_early(locate_protocol, &tcg2_guid, NULL,
+				&tcg2_protocol);
+	if (status != EFI_SUCCESS)
+		return;
+
+	status = efi_call_proto(efi_tcg2_protocol, get_event_log, tcg2_protocol,
+				EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2,
+				&log_location, &log_last_entry, &truncated);
+	if (status != EFI_SUCCESS)
+		return;
+
+	if (!log_location)
+		return;
+	first_entry_addr = (unsigned long) log_location;
+
+	/*
+	 * We populate the EFI table even if the logs are empty.
+	 */
+	if (!log_last_entry) {
+		log_size = 0;
+	} else {
+		last_entry_addr = (unsigned long) log_last_entry;
+		/*
+		 * get_event_log only returns the address of the last entry.
+		 * We need to calculate its size to deduce the full size of
+		 * the logs.
+		 */
+		last_entry_size = sizeof(struct tcpa_event) +
+			((struct tcpa_event *) last_entry_addr)->event_size;
+		log_size = log_last_entry - log_location + last_entry_size;
+	}
+
+	/* Allocate space for the logs and copy them. */
+	status = efi_call_early(allocate_pool, EFI_LOADER_DATA,
+				sizeof(*log_tbl) + log_size,
+				(void **) &log_tbl);
+
+	if (status != EFI_SUCCESS) {
+		efi_printk(sys_table_arg,
+			   "Unable to allocate memory for event log\n");
+		return;
+	}
+
+	memset(log_tbl, 0, sizeof(*log_tbl) + log_size);
+	log_tbl->size = log_size;
+	log_tbl->version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2;
+	memcpy(log_tbl->log, (void *) first_entry_addr, log_size);
+
+	status = efi_call_early(install_configuration_table,
+				&linux_eventlog_guid, log_tbl);
+	if (status != EFI_SUCCESS)
+		goto err_free;
+	return;
+
+err_free:
+	efi_call_early(free_pool, log_tbl);
+}
+
+void efi_retrieve_tpm2_eventlog(efi_system_table_t *sys_table_arg)
+{
+	/* Only try to retrieve the logs in 1.2 format. */
+	efi_retrieve_tpm2_eventlog_1_2(sys_table_arg);
+}
diff --git a/drivers/firmware/efi/tpm.c b/drivers/firmware/efi/tpm.c
new file mode 100644
index 000000000000..0cbeb3d46b18
--- /dev/null
+++ b/drivers/firmware/efi/tpm.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2017 Google, Inc.
+ *     Thiebaud Weksteen <tweek@google.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/efi.h>
+#include <linux/init.h>
+#include <linux/memblock.h>
+
+#include <asm/early_ioremap.h>
+
+/*
+ * Reserve the memory associated with the TPM Event Log configuration table.
+ */
+int __init efi_tpm_eventlog_init(void)
+{
+	struct linux_efi_tpm_eventlog *log_tbl;
+	unsigned int tbl_size;
+
+	if (efi.tpm_log == EFI_INVALID_TABLE_ADDR)
+		return 0;
+
+	log_tbl = early_memremap(efi.tpm_log, sizeof(*log_tbl));
+	if (!log_tbl) {
+		pr_err("Failed to map TPM Event Log table @ 0x%lx\n",
+			efi.tpm_log);
+		efi.tpm_log = EFI_INVALID_TABLE_ADDR;
+		return -ENOMEM;
+	}
+
+	tbl_size = sizeof(*log_tbl) + log_tbl->size;
+	memblock_reserve(efi.tpm_log, tbl_size);
+	early_memunmap(log_tbl, sizeof(*log_tbl));
+	return 0;
+}
+
diff --git a/drivers/firmware/iscsi_ibft.c b/drivers/firmware/iscsi_ibft.c
index 14042a64bdd5..6bc8e6640d71 100644
--- a/drivers/firmware/iscsi_ibft.c
+++ b/drivers/firmware/iscsi_ibft.c
@@ -719,8 +719,9 @@ static int __init ibft_create_kobject(struct acpi_table_ibft *header,
 		* executes only devices which are in domain 0. Furthermore, the
 		* iBFT spec doesn't have a domain id field :-(
 		*/
-		pci_dev = pci_get_bus_and_slot((nic->pci_bdf & 0xff00) >> 8,
-					       (nic->pci_bdf & 0xff));
+		pci_dev = pci_get_domain_bus_and_slot(0,
+						(nic->pci_bdf & 0xff00) >> 8,
+						(nic->pci_bdf & 0xff));
 		if (pci_dev) {
 			rc = sysfs_create_link(&boot_kobj->kobj,
 					       &pci_dev->dev.kobj, "device");
diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c
index d687ca3d5049..c80ec1d03274 100644
--- a/drivers/firmware/psci.c
+++ b/drivers/firmware/psci.c
@@ -59,7 +59,10 @@ bool psci_tos_resident_on(int cpu)
 	return cpu == resident_cpu;
 }
 
-struct psci_operations psci_ops;
+struct psci_operations psci_ops = {
+	.conduit = PSCI_CONDUIT_NONE,
+	.smccc_version = SMCCC_VERSION_1_0,
+};
 
 typedef unsigned long (psci_fn)(unsigned long, unsigned long,
 				unsigned long, unsigned long);
@@ -210,6 +213,22 @@ static unsigned long psci_migrate_info_up_cpu(void)
 			      0, 0, 0);
 }
 
+static void set_conduit(enum psci_conduit conduit)
+{
+	switch (conduit) {
+	case PSCI_CONDUIT_HVC:
+		invoke_psci_fn = __invoke_psci_fn_hvc;
+		break;
+	case PSCI_CONDUIT_SMC:
+		invoke_psci_fn = __invoke_psci_fn_smc;
+		break;
+	default:
+		WARN(1, "Unexpected PSCI conduit %d\n", conduit);
+	}
+
+	psci_ops.conduit = conduit;
+}
+
 static int get_set_conduit_method(struct device_node *np)
 {
 	const char *method;
@@ -222,9 +241,9 @@ static int get_set_conduit_method(struct device_node *np)
 	}
 
 	if (!strcmp("hvc", method)) {
-		invoke_psci_fn = __invoke_psci_fn_hvc;
+		set_conduit(PSCI_CONDUIT_HVC);
 	} else if (!strcmp("smc", method)) {
-		invoke_psci_fn = __invoke_psci_fn_smc;
+		set_conduit(PSCI_CONDUIT_SMC);
 	} else {
 		pr_warn("invalid \"method\" property: %s\n", method);
 		return -EINVAL;
@@ -493,9 +512,36 @@ static void __init psci_init_migrate(void)
 	pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid);
 }
 
+static void __init psci_init_smccc(void)
+{
+	u32 ver = ARM_SMCCC_VERSION_1_0;
+	int feature;
+
+	feature = psci_features(ARM_SMCCC_VERSION_FUNC_ID);
+
+	if (feature != PSCI_RET_NOT_SUPPORTED) {
+		u32 ret;
+		ret = invoke_psci_fn(ARM_SMCCC_VERSION_FUNC_ID, 0, 0, 0);
+		if (ret == ARM_SMCCC_VERSION_1_1) {
+			psci_ops.smccc_version = SMCCC_VERSION_1_1;
+			ver = ret;
+		}
+	}
+
+	/*
+	 * Conveniently, the SMCCC and PSCI versions are encoded the
+	 * same way. No, this isn't accidental.
+	 */
+	pr_info("SMC Calling Convention v%d.%d\n",
+		PSCI_VERSION_MAJOR(ver), PSCI_VERSION_MINOR(ver));
+
+}
+
 static void __init psci_0_2_set_functions(void)
 {
 	pr_info("Using standard PSCI v0.2 function IDs\n");
+	psci_ops.get_version = psci_get_version;
+
 	psci_function_id[PSCI_FN_CPU_SUSPEND] =
 					PSCI_FN_NATIVE(0_2, CPU_SUSPEND);
 	psci_ops.cpu_suspend = psci_cpu_suspend;
@@ -539,6 +585,7 @@ static int __init psci_probe(void)
 	psci_init_migrate();
 
 	if (PSCI_VERSION_MAJOR(ver) >= 1) {
+		psci_init_smccc();
 		psci_init_cpu_suspend();
 		psci_init_system_suspend();
 	}
@@ -652,9 +699,9 @@ int __init psci_acpi_init(void)
 	pr_info("probing for conduit method from ACPI.\n");
 
 	if (acpi_psci_use_hvc())
-		invoke_psci_fn = __invoke_psci_fn_hvc;
+		set_conduit(PSCI_CONDUIT_HVC);
 	else
-		invoke_psci_fn = __invoke_psci_fn_smc;
+		set_conduit(PSCI_CONDUIT_SMC);
 
 	return psci_probe();
 }
diff --git a/drivers/firmware/psci_checker.c b/drivers/firmware/psci_checker.c
index f3f4f810e5df..bb1c068bff19 100644
--- a/drivers/firmware/psci_checker.c
+++ b/drivers/firmware/psci_checker.c
@@ -77,8 +77,8 @@ static int psci_ops_check(void)
 	return 0;
 }
 
-static int find_clusters(const struct cpumask *cpus,
-			 const struct cpumask **clusters)
+static int find_cpu_groups(const struct cpumask *cpus,
+			   const struct cpumask **cpu_groups)
 {
 	unsigned int nb = 0;
 	cpumask_var_t tmp;
@@ -88,11 +88,11 @@ static int find_clusters(const struct cpumask *cpus,
 	cpumask_copy(tmp, cpus);
 
 	while (!cpumask_empty(tmp)) {
-		const struct cpumask *cluster =
+		const struct cpumask *cpu_group =
 			topology_core_cpumask(cpumask_any(tmp));
 
-		clusters[nb++] = cluster;
-		cpumask_andnot(tmp, tmp, cluster);
+		cpu_groups[nb++] = cpu_group;
+		cpumask_andnot(tmp, tmp, cpu_group);
 	}
 
 	free_cpumask_var(tmp);
@@ -170,24 +170,24 @@ static int hotplug_tests(void)
 {
 	int err;
 	cpumask_var_t offlined_cpus;
-	int i, nb_cluster;
-	const struct cpumask **clusters;
+	int i, nb_cpu_group;
+	const struct cpumask **cpu_groups;
 	char *page_buf;
 
 	err = -ENOMEM;
 	if (!alloc_cpumask_var(&offlined_cpus, GFP_KERNEL))
 		return err;
-	/* We may have up to nb_available_cpus clusters. */
-	clusters = kmalloc_array(nb_available_cpus, sizeof(*clusters),
-				 GFP_KERNEL);
-	if (!clusters)
+	/* We may have up to nb_available_cpus cpu_groups. */
+	cpu_groups = kmalloc_array(nb_available_cpus, sizeof(*cpu_groups),
+				   GFP_KERNEL);
+	if (!cpu_groups)
 		goto out_free_cpus;
 	page_buf = (char *)__get_free_page(GFP_KERNEL);
 	if (!page_buf)
-		goto out_free_clusters;
+		goto out_free_cpu_groups;
 
 	err = 0;
-	nb_cluster = find_clusters(cpu_online_mask, clusters);
+	nb_cpu_group = find_cpu_groups(cpu_online_mask, cpu_groups);
 
 	/*
 	 * Of course the last CPU cannot be powered down and cpu_down() should
@@ -197,24 +197,22 @@ static int hotplug_tests(void)
 	err += down_and_up_cpus(cpu_online_mask, offlined_cpus);
 
 	/*
-	 * Take down CPUs by cluster this time. When the last CPU is turned
-	 * off, the cluster itself should shut down.
+	 * Take down CPUs by cpu group this time. When the last CPU is turned
+	 * off, the cpu group itself should shut down.
 	 */
-	for (i = 0; i < nb_cluster; ++i) {
-		int cluster_id =
-			topology_physical_package_id(cpumask_any(clusters[i]));
+	for (i = 0; i < nb_cpu_group; ++i) {
 		ssize_t len = cpumap_print_to_pagebuf(true, page_buf,
-						      clusters[i]);
+						      cpu_groups[i]);
 		/* Remove trailing newline. */
 		page_buf[len - 1] = '\0';
-		pr_info("Trying to turn off and on again cluster %d "
-			"(CPUs %s)\n", cluster_id, page_buf);
-		err += down_and_up_cpus(clusters[i], offlined_cpus);
+		pr_info("Trying to turn off and on again group %d (CPUs %s)\n",
+			i, page_buf);
+		err += down_and_up_cpus(cpu_groups[i], offlined_cpus);
 	}
 
 	free_page((unsigned long)page_buf);
-out_free_clusters:
-	kfree(clusters);
+out_free_cpu_groups:
+	kfree(cpu_groups);
 out_free_cpus:
 	free_cpumask_var(offlined_cpus);
 	return err;
diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c
index af4c75217ea6..5a7d693009ef 100644
--- a/drivers/firmware/qcom_scm.c
+++ b/drivers/firmware/qcom_scm.c
@@ -622,30 +622,6 @@ static struct platform_driver qcom_scm_driver = {
 
 static int __init qcom_scm_init(void)
 {
-	struct device_node *np, *fw_np;
-	int ret;
-
-	fw_np = of_find_node_by_name(NULL, "firmware");
-
-	if (!fw_np)
-		return -ENODEV;
-
-	np = of_find_matching_node(fw_np, qcom_scm_dt_match);
-
-	if (!np) {
-		of_node_put(fw_np);
-		return -ENODEV;
-	}
-
-	of_node_put(np);
-
-	ret = of_platform_populate(fw_np, qcom_scm_dt_match, NULL, NULL);
-
-	of_node_put(fw_np);
-
-	if (ret)
-		return ret;
-
 	return platform_driver_register(&qcom_scm_driver);
 }
 subsys_initcall(qcom_scm_init);
diff --git a/drivers/firmware/qemu_fw_cfg.c b/drivers/firmware/qemu_fw_cfg.c
index deb483064f53..a41b572eeeb1 100644
--- a/drivers/firmware/qemu_fw_cfg.c
+++ b/drivers/firmware/qemu_fw_cfg.c
@@ -694,10 +694,8 @@ static int fw_cfg_cmdline_set(const char *arg, const struct kernel_param *kp)
 	 */
 	fw_cfg_cmdline_dev = platform_device_register_simple("fw_cfg",
 					PLATFORM_DEVID_NONE, res, processed);
-	if (IS_ERR(fw_cfg_cmdline_dev))
-		return PTR_ERR(fw_cfg_cmdline_dev);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(fw_cfg_cmdline_dev);
 }
 
 static int fw_cfg_cmdline_get(char *buf, const struct kernel_param *kp)
diff --git a/drivers/firmware/raspberrypi.c b/drivers/firmware/raspberrypi.c
index dd506cd3a5b8..6692888f04cf 100644
--- a/drivers/firmware/raspberrypi.c
+++ b/drivers/firmware/raspberrypi.c
@@ -174,7 +174,7 @@ rpi_firmware_print_firmware_revision(struct rpi_firmware *fw)
 	if (ret == 0) {
 		struct tm tm;
 
-		time_to_tm(packet, 0, &tm);
+		time64_to_tm(packet, 0, &tm);
 
 		dev_info(fw->cl.dev,
 			 "Attached to firmware from %04ld-%02d-%02d %02d:%02d\n",
diff --git a/drivers/firmware/ti_sci.c b/drivers/firmware/ti_sci.c
index 23b12d99ddfe..5229036dcfbf 100644
--- a/drivers/firmware/ti_sci.c
+++ b/drivers/firmware/ti_sci.c
@@ -287,13 +287,13 @@ static void ti_sci_rx_callback(struct mbox_client *cl, void *m)
 
 	/* Is the message of valid length? */
 	if (mbox_msg->len > info->desc->max_msg_size) {
-		dev_err(dev, "Unable to handle %d xfer(max %d)\n",
+		dev_err(dev, "Unable to handle %zu xfer(max %d)\n",
 			mbox_msg->len, info->desc->max_msg_size);
 		ti_sci_dump_header_dbg(dev, hdr);
 		return;
 	}
 	if (mbox_msg->len < xfer->rx_len) {
-		dev_err(dev, "Recv xfer %d < expected %d length\n",
+		dev_err(dev, "Recv xfer %zu < expected %d length\n",
 			mbox_msg->len, xfer->rx_len);
 		ti_sci_dump_header_dbg(dev, hdr);
 		return;
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index ad5448f718b3..f47ef848bcd0 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -11,25 +11,30 @@ menuconfig FPGA
 
 if FPGA
 
-config FPGA_REGION
-	tristate "FPGA Region"
-	depends on OF && FPGA_BRIDGE
+config FPGA_MGR_SOCFPGA
+	tristate "Altera SOCFPGA FPGA Manager"
+	depends on ARCH_SOCFPGA || COMPILE_TEST
 	help
-	  FPGA Regions allow loading FPGA images under control of
-	  the Device Tree.
+	  FPGA manager driver support for Altera SOCFPGA.
 
-config FPGA_MGR_ICE40_SPI
-	tristate "Lattice iCE40 SPI"
-	depends on OF && SPI
+config FPGA_MGR_SOCFPGA_A10
+	tristate "Altera SoCFPGA Arria10"
+	depends on ARCH_SOCFPGA || COMPILE_TEST
+	select REGMAP_MMIO
 	help
-	  FPGA manager driver support for Lattice iCE40 FPGAs over SPI.
+	  FPGA manager driver support for Altera Arria10 SoCFPGA.
 
-config FPGA_MGR_ALTERA_CVP
-	tristate "Altera Arria-V/Cyclone-V/Stratix-V CvP FPGA Manager"
-	depends on PCI
+config ALTERA_PR_IP_CORE
+        tristate "Altera Partial Reconfiguration IP Core"
+        help
+          Core driver support for Altera Partial Reconfiguration IP component
+
+config ALTERA_PR_IP_CORE_PLAT
+	tristate "Platform support of Altera Partial Reconfiguration IP Core"
+	depends on ALTERA_PR_IP_CORE && OF && HAS_IOMEM
 	help
-	  FPGA manager driver support for Arria-V, Cyclone-V, Stratix-V
-	  and Arria 10 Altera FPGAs using the CvP interface over PCIe.
+	  Platform driver support for Altera Partial Reconfiguration IP
+	  component
 
 config FPGA_MGR_ALTERA_PS_SPI
 	tristate "Altera FPGA Passive Serial over SPI"
@@ -38,25 +43,19 @@ config FPGA_MGR_ALTERA_PS_SPI
 	  FPGA manager driver support for Altera Arria/Cyclone/Stratix
 	  using the passive serial interface over SPI.
 
-config FPGA_MGR_SOCFPGA
-	tristate "Altera SOCFPGA FPGA Manager"
-	depends on ARCH_SOCFPGA || COMPILE_TEST
-	help
-	  FPGA manager driver support for Altera SOCFPGA.
-
-config FPGA_MGR_SOCFPGA_A10
-	tristate "Altera SoCFPGA Arria10"
-	depends on ARCH_SOCFPGA || COMPILE_TEST
-	select REGMAP_MMIO
+config FPGA_MGR_ALTERA_CVP
+	tristate "Altera Arria-V/Cyclone-V/Stratix-V CvP FPGA Manager"
+	depends on PCI
 	help
-	  FPGA manager driver support for Altera Arria10 SoCFPGA.
+	  FPGA manager driver support for Arria-V, Cyclone-V, Stratix-V
+	  and Arria 10 Altera FPGAs using the CvP interface over PCIe.
 
-config FPGA_MGR_TS73XX
-	tristate "Technologic Systems TS-73xx SBC FPGA Manager"
-	depends on ARCH_EP93XX && MACH_TS72XX
+config FPGA_MGR_ZYNQ_FPGA
+	tristate "Xilinx Zynq FPGA"
+	depends on ARCH_ZYNQ || COMPILE_TEST
+	depends on HAS_DMA
 	help
-	  FPGA manager driver support for the Altera Cyclone II FPGA
-	  present on the TS-73xx SBC boards.
+	  FPGA manager driver support for Xilinx Zynq FPGAs.
 
 config FPGA_MGR_XILINX_SPI
 	tristate "Xilinx Configuration over Slave Serial (SPI)"
@@ -65,16 +64,21 @@ config FPGA_MGR_XILINX_SPI
 	  FPGA manager driver support for Xilinx FPGA configuration
 	  over slave serial interface.
 
-config FPGA_MGR_ZYNQ_FPGA
-	tristate "Xilinx Zynq FPGA"
-	depends on ARCH_ZYNQ || COMPILE_TEST
-	depends on HAS_DMA
+config FPGA_MGR_ICE40_SPI
+	tristate "Lattice iCE40 SPI"
+	depends on OF && SPI
 	help
-	  FPGA manager driver support for Xilinx Zynq FPGAs.
+	  FPGA manager driver support for Lattice iCE40 FPGAs over SPI.
+
+config FPGA_MGR_TS73XX
+	tristate "Technologic Systems TS-73xx SBC FPGA Manager"
+	depends on ARCH_EP93XX && MACH_TS72XX
+	help
+	  FPGA manager driver support for the Altera Cyclone II FPGA
+	  present on the TS-73xx SBC boards.
 
 config FPGA_BRIDGE
 	tristate "FPGA Bridge Framework"
-	depends on OF
 	help
 	  Say Y here if you want to support bridges connected between host
 	  processors and FPGAs or between FPGAs.
@@ -95,18 +99,6 @@ config ALTERA_FREEZE_BRIDGE
 	  isolate one region of the FPGA from the busses while that
 	  region is being reprogrammed.
 
-config ALTERA_PR_IP_CORE
-        tristate "Altera Partial Reconfiguration IP Core"
-        help
-          Core driver support for Altera Partial Reconfiguration IP component
-
-config ALTERA_PR_IP_CORE_PLAT
-	tristate "Platform support of Altera Partial Reconfiguration IP Core"
-	depends on ALTERA_PR_IP_CORE && OF && HAS_IOMEM
-	help
-	  Platform driver support for Altera Partial Reconfiguration IP
-	  component
-
 config XILINX_PR_DECOUPLER
 	tristate "Xilinx LogiCORE PR Decoupler"
 	depends on FPGA_BRIDGE
@@ -117,4 +109,19 @@ config XILINX_PR_DECOUPLER
 	  region of the FPGA from the busses while that region is
 	  being reprogrammed during partial reconfig.
 
+config FPGA_REGION
+	tristate "FPGA Region"
+	depends on FPGA_BRIDGE
+	help
+	  FPGA Region common code.  A FPGA Region controls a FPGA Manager
+	  and the FPGA Bridges associated with either a reconfigurable
+	  region of an FPGA or a whole FPGA.
+
+config OF_FPGA_REGION
+	tristate "FPGA Region Device Tree Overlay Support"
+	depends on OF && FPGA_REGION
+	help
+	  Support for loading FPGA images by applying a Device Tree
+	  overlay.
+
 endif # FPGA
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index f98dcf1d89e1..3cb276a0f88d 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -26,3 +26,4 @@ obj-$(CONFIG_XILINX_PR_DECOUPLER)	+= xilinx-pr-decoupler.o
 
 # High Level Interfaces
 obj-$(CONFIG_FPGA_REGION)		+= fpga-region.o
+obj-$(CONFIG_OF_FPGA_REGION)		+= of-fpga-region.o
diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c
index 9651aa56244a..31bd2c59c305 100644
--- a/drivers/fpga/fpga-bridge.c
+++ b/drivers/fpga/fpga-bridge.c
@@ -2,6 +2,7 @@
  * FPGA Bridge Framework Driver
  *
  *  Copyright (C) 2013-2016 Altera Corporation, All Rights Reserved.
+ *  Copyright (C) 2017 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -70,32 +71,13 @@ int fpga_bridge_disable(struct fpga_bridge *bridge)
 }
 EXPORT_SYMBOL_GPL(fpga_bridge_disable);
 
-/**
- * of_fpga_bridge_get - get an exclusive reference to a fpga bridge
- *
- * @np: node pointer of a FPGA bridge
- * @info: fpga image specific information
- *
- * Return fpga_bridge struct if successful.
- * Return -EBUSY if someone already has a reference to the bridge.
- * Return -ENODEV if @np is not a FPGA Bridge.
- */
-struct fpga_bridge *of_fpga_bridge_get(struct device_node *np,
-				       struct fpga_image_info *info)
-
+static struct fpga_bridge *__fpga_bridge_get(struct device *dev,
+					     struct fpga_image_info *info)
 {
-	struct device *dev;
 	struct fpga_bridge *bridge;
 	int ret = -ENODEV;
 
-	dev = class_find_device(fpga_bridge_class, NULL, np,
-				fpga_bridge_of_node_match);
-	if (!dev)
-		goto err_dev;
-
 	bridge = to_fpga_bridge(dev);
-	if (!bridge)
-		goto err_dev;
 
 	bridge->info = info;
 
@@ -117,8 +99,58 @@ struct fpga_bridge *of_fpga_bridge_get(struct device_node *np,
 	put_device(dev);
 	return ERR_PTR(ret);
 }
+
+/**
+ * of_fpga_bridge_get - get an exclusive reference to a fpga bridge
+ *
+ * @np: node pointer of a FPGA bridge
+ * @info: fpga image specific information
+ *
+ * Return fpga_bridge struct if successful.
+ * Return -EBUSY if someone already has a reference to the bridge.
+ * Return -ENODEV if @np is not a FPGA Bridge.
+ */
+struct fpga_bridge *of_fpga_bridge_get(struct device_node *np,
+				       struct fpga_image_info *info)
+{
+	struct device *dev;
+
+	dev = class_find_device(fpga_bridge_class, NULL, np,
+				fpga_bridge_of_node_match);
+	if (!dev)
+		return ERR_PTR(-ENODEV);
+
+	return __fpga_bridge_get(dev, info);
+}
 EXPORT_SYMBOL_GPL(of_fpga_bridge_get);
 
+static int fpga_bridge_dev_match(struct device *dev, const void *data)
+{
+	return dev->parent == data;
+}
+
+/**
+ * fpga_bridge_get - get an exclusive reference to a fpga bridge
+ * @dev:	parent device that fpga bridge was registered with
+ *
+ * Given a device, get an exclusive reference to a fpga bridge.
+ *
+ * Return: fpga manager struct or IS_ERR() condition containing error code.
+ */
+struct fpga_bridge *fpga_bridge_get(struct device *dev,
+				    struct fpga_image_info *info)
+{
+	struct device *bridge_dev;
+
+	bridge_dev = class_find_device(fpga_bridge_class, NULL, dev,
+				       fpga_bridge_dev_match);
+	if (!bridge_dev)
+		return ERR_PTR(-ENODEV);
+
+	return __fpga_bridge_get(bridge_dev, info);
+}
+EXPORT_SYMBOL_GPL(fpga_bridge_get);
+
 /**
  * fpga_bridge_put - release a reference to a bridge
  *
@@ -206,7 +238,7 @@ void fpga_bridges_put(struct list_head *bridge_list)
 EXPORT_SYMBOL_GPL(fpga_bridges_put);
 
 /**
- * fpga_bridges_get_to_list - get a bridge, add it to a list
+ * of_fpga_bridge_get_to_list - get a bridge, add it to a list
  *
  * @np: node pointer of a FPGA bridge
  * @info: fpga image specific information
@@ -216,14 +248,44 @@ EXPORT_SYMBOL_GPL(fpga_bridges_put);
  *
  * Return 0 for success, error code from of_fpga_bridge_get() othewise.
  */
-int fpga_bridge_get_to_list(struct device_node *np,
+int of_fpga_bridge_get_to_list(struct device_node *np,
+			       struct fpga_image_info *info,
+			       struct list_head *bridge_list)
+{
+	struct fpga_bridge *bridge;
+	unsigned long flags;
+
+	bridge = of_fpga_bridge_get(np, info);
+	if (IS_ERR(bridge))
+		return PTR_ERR(bridge);
+
+	spin_lock_irqsave(&bridge_list_lock, flags);
+	list_add(&bridge->node, bridge_list);
+	spin_unlock_irqrestore(&bridge_list_lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(of_fpga_bridge_get_to_list);
+
+/**
+ * fpga_bridge_get_to_list - given device, get a bridge, add it to a list
+ *
+ * @dev: FPGA bridge device
+ * @info: fpga image specific information
+ * @bridge_list: list of FPGA bridges
+ *
+ * Get an exclusive reference to the bridge and and it to the list.
+ *
+ * Return 0 for success, error code from fpga_bridge_get() othewise.
+ */
+int fpga_bridge_get_to_list(struct device *dev,
 			    struct fpga_image_info *info,
 			    struct list_head *bridge_list)
 {
 	struct fpga_bridge *bridge;
 	unsigned long flags;
 
-	bridge = of_fpga_bridge_get(np, info);
+	bridge = fpga_bridge_get(dev, info);
 	if (IS_ERR(bridge))
 		return PTR_ERR(bridge);
 
@@ -303,6 +365,7 @@ int fpga_bridge_register(struct device *dev, const char *name,
 	bridge->priv = priv;
 
 	device_initialize(&bridge->dev);
+	bridge->dev.groups = br_ops->groups;
 	bridge->dev.class = fpga_bridge_class;
 	bridge->dev.parent = dev;
 	bridge->dev.of_node = dev->of_node;
@@ -381,7 +444,7 @@ static void __exit fpga_bridge_dev_exit(void)
 }
 
 MODULE_DESCRIPTION("FPGA Bridge Driver");
-MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_AUTHOR("Alan Tull <atull@kernel.org>");
 MODULE_LICENSE("GPL v2");
 
 subsys_initcall(fpga_bridge_dev_init);
diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c
index 188ffefa3cc3..9939d2cbc9a6 100644
--- a/drivers/fpga/fpga-mgr.c
+++ b/drivers/fpga/fpga-mgr.c
@@ -2,6 +2,7 @@
  * FPGA Manager Core
  *
  *  Copyright (C) 2013-2015 Altera Corporation
+ *  Copyright (C) 2017 Intel Corporation
  *
  * With code from the mailing list:
  * Copyright (C) 2013 Xilinx, Inc.
@@ -31,6 +32,40 @@
 static DEFINE_IDA(fpga_mgr_ida);
 static struct class *fpga_mgr_class;
 
+struct fpga_image_info *fpga_image_info_alloc(struct device *dev)
+{
+	struct fpga_image_info *info;
+
+	get_device(dev);
+
+	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);
+	if (!info) {
+		put_device(dev);
+		return NULL;
+	}
+
+	info->dev = dev;
+
+	return info;
+}
+EXPORT_SYMBOL_GPL(fpga_image_info_alloc);
+
+void fpga_image_info_free(struct fpga_image_info *info)
+{
+	struct device *dev;
+
+	if (!info)
+		return;
+
+	dev = info->dev;
+	if (info->firmware_name)
+		devm_kfree(dev, info->firmware_name);
+
+	devm_kfree(dev, info);
+	put_device(dev);
+}
+EXPORT_SYMBOL_GPL(fpga_image_info_free);
+
 /*
  * Call the low level driver's write_init function.  This will do the
  * device-specific things to get the FPGA into the state where it is ready to
@@ -137,8 +172,9 @@ static int fpga_mgr_write_complete(struct fpga_manager *mgr,
  *
  * Return: 0 on success, negative error code otherwise.
  */
-int fpga_mgr_buf_load_sg(struct fpga_manager *mgr, struct fpga_image_info *info,
-			 struct sg_table *sgt)
+static int fpga_mgr_buf_load_sg(struct fpga_manager *mgr,
+				struct fpga_image_info *info,
+				struct sg_table *sgt)
 {
 	int ret;
 
@@ -170,7 +206,6 @@ int fpga_mgr_buf_load_sg(struct fpga_manager *mgr, struct fpga_image_info *info,
 
 	return fpga_mgr_write_complete(mgr, info);
 }
-EXPORT_SYMBOL_GPL(fpga_mgr_buf_load_sg);
 
 static int fpga_mgr_buf_load_mapped(struct fpga_manager *mgr,
 				    struct fpga_image_info *info,
@@ -210,8 +245,9 @@ static int fpga_mgr_buf_load_mapped(struct fpga_manager *mgr,
  *
  * Return: 0 on success, negative error code otherwise.
  */
-int fpga_mgr_buf_load(struct fpga_manager *mgr, struct fpga_image_info *info,
-		      const char *buf, size_t count)
+static int fpga_mgr_buf_load(struct fpga_manager *mgr,
+			     struct fpga_image_info *info,
+			     const char *buf, size_t count)
 {
 	struct page **pages;
 	struct sg_table sgt;
@@ -266,7 +302,6 @@ int fpga_mgr_buf_load(struct fpga_manager *mgr, struct fpga_image_info *info,
 
 	return rc;
 }
-EXPORT_SYMBOL_GPL(fpga_mgr_buf_load);
 
 /**
  * fpga_mgr_firmware_load - request firmware and load to fpga
@@ -282,9 +317,9 @@ EXPORT_SYMBOL_GPL(fpga_mgr_buf_load);
  *
  * Return: 0 on success, negative error code otherwise.
  */
-int fpga_mgr_firmware_load(struct fpga_manager *mgr,
-			   struct fpga_image_info *info,
-			   const char *image_name)
+static int fpga_mgr_firmware_load(struct fpga_manager *mgr,
+				  struct fpga_image_info *info,
+				  const char *image_name)
 {
 	struct device *dev = &mgr->dev;
 	const struct firmware *fw;
@@ -307,7 +342,18 @@ int fpga_mgr_firmware_load(struct fpga_manager *mgr,
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(fpga_mgr_firmware_load);
+
+int fpga_mgr_load(struct fpga_manager *mgr, struct fpga_image_info *info)
+{
+	if (info->sgt)
+		return fpga_mgr_buf_load_sg(mgr, info, info->sgt);
+	if (info->buf && info->count)
+		return fpga_mgr_buf_load(mgr, info, info->buf, info->count);
+	if (info->firmware_name)
+		return fpga_mgr_firmware_load(mgr, info, info->firmware_name);
+	return -EINVAL;
+}
+EXPORT_SYMBOL_GPL(fpga_mgr_load);
 
 static const char * const state_str[] = {
 	[FPGA_MGR_STATE_UNKNOWN] =		"unknown",
@@ -364,28 +410,17 @@ ATTRIBUTE_GROUPS(fpga_mgr);
 static struct fpga_manager *__fpga_mgr_get(struct device *dev)
 {
 	struct fpga_manager *mgr;
-	int ret = -ENODEV;
 
 	mgr = to_fpga_manager(dev);
-	if (!mgr)
-		goto err_dev;
-
-	/* Get exclusive use of fpga manager */
-	if (!mutex_trylock(&mgr->ref_mutex)) {
-		ret = -EBUSY;
-		goto err_dev;
-	}
 
 	if (!try_module_get(dev->parent->driver->owner))
-		goto err_ll_mod;
+		goto err_dev;
 
 	return mgr;
 
-err_ll_mod:
-	mutex_unlock(&mgr->ref_mutex);
 err_dev:
 	put_device(dev);
-	return ERR_PTR(ret);
+	return ERR_PTR(-ENODEV);
 }
 
 static int fpga_mgr_dev_match(struct device *dev, const void *data)
@@ -394,10 +429,10 @@ static int fpga_mgr_dev_match(struct device *dev, const void *data)
 }
 
 /**
- * fpga_mgr_get - get an exclusive reference to a fpga mgr
+ * fpga_mgr_get - get a reference to a fpga mgr
  * @dev:	parent device that fpga mgr was registered with
  *
- * Given a device, get an exclusive reference to a fpga mgr.
+ * Given a device, get a reference to a fpga mgr.
  *
  * Return: fpga manager struct or IS_ERR() condition containing error code.
  */
@@ -418,10 +453,10 @@ static int fpga_mgr_of_node_match(struct device *dev, const void *data)
 }
 
 /**
- * of_fpga_mgr_get - get an exclusive reference to a fpga mgr
+ * of_fpga_mgr_get - get a reference to a fpga mgr
  * @node:	device node
  *
- * Given a device node, get an exclusive reference to a fpga mgr.
+ * Given a device node, get a reference to a fpga mgr.
  *
  * Return: fpga manager struct or IS_ERR() condition containing error code.
  */
@@ -445,11 +480,40 @@ EXPORT_SYMBOL_GPL(of_fpga_mgr_get);
 void fpga_mgr_put(struct fpga_manager *mgr)
 {
 	module_put(mgr->dev.parent->driver->owner);
-	mutex_unlock(&mgr->ref_mutex);
 	put_device(&mgr->dev);
 }
 EXPORT_SYMBOL_GPL(fpga_mgr_put);
 
+/**
+ * fpga_mgr_lock - Lock FPGA manager for exclusive use
+ * @mgr:	fpga manager
+ *
+ * Given a pointer to FPGA Manager (from fpga_mgr_get() or
+ * of_fpga_mgr_put()) attempt to get the mutex.
+ *
+ * Return: 0 for success or -EBUSY
+ */
+int fpga_mgr_lock(struct fpga_manager *mgr)
+{
+	if (!mutex_trylock(&mgr->ref_mutex)) {
+		dev_err(&mgr->dev, "FPGA manager is in use.\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fpga_mgr_lock);
+
+/**
+ * fpga_mgr_unlock - Unlock FPGA manager
+ * @mgr:	fpga manager
+ */
+void fpga_mgr_unlock(struct fpga_manager *mgr)
+{
+	mutex_unlock(&mgr->ref_mutex);
+}
+EXPORT_SYMBOL_GPL(fpga_mgr_unlock);
+
 /**
  * fpga_mgr_register - register a low level fpga manager driver
  * @dev:	fpga manager device from pdev
@@ -503,6 +567,7 @@ int fpga_mgr_register(struct device *dev, const char *name,
 
 	device_initialize(&mgr->dev);
 	mgr->dev.class = fpga_mgr_class;
+	mgr->dev.groups = mops->groups;
 	mgr->dev.parent = dev;
 	mgr->dev.of_node = dev->of_node;
 	mgr->dev.id = id;
@@ -578,7 +643,7 @@ static void __exit fpga_mgr_class_exit(void)
 	ida_destroy(&fpga_mgr_ida);
 }
 
-MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_AUTHOR("Alan Tull <atull@kernel.org>");
 MODULE_DESCRIPTION("FPGA manager framework");
 MODULE_LICENSE("GPL v2");
 
diff --git a/drivers/fpga/fpga-region.c b/drivers/fpga/fpga-region.c
index d9ab7c75b14f..edab2a2e03ef 100644
--- a/drivers/fpga/fpga-region.c
+++ b/drivers/fpga/fpga-region.c
@@ -2,6 +2,7 @@
  * FPGA Region - Device Tree support for FPGA programming under Linux
  *
  *  Copyright (C) 2013-2016 Altera Corporation
+ *  Copyright (C) 2017 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -18,61 +19,30 @@
 
 #include <linux/fpga/fpga-bridge.h>
 #include <linux/fpga/fpga-mgr.h>
+#include <linux/fpga/fpga-region.h>
 #include <linux/idr.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
 #include <linux/module.h>
-#include <linux/of_platform.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
-/**
- * struct fpga_region - FPGA Region structure
- * @dev: FPGA Region device
- * @mutex: enforces exclusive reference to region
- * @bridge_list: list of FPGA bridges specified in region
- * @info: fpga image specific information
- */
-struct fpga_region {
-	struct device dev;
-	struct mutex mutex; /* for exclusive reference to region */
-	struct list_head bridge_list;
-	struct fpga_image_info *info;
-};
-
-#define to_fpga_region(d) container_of(d, struct fpga_region, dev)
-
 static DEFINE_IDA(fpga_region_ida);
 static struct class *fpga_region_class;
 
-static const struct of_device_id fpga_region_of_match[] = {
-	{ .compatible = "fpga-region", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, fpga_region_of_match);
-
-static int fpga_region_of_node_match(struct device *dev, const void *data)
-{
-	return dev->of_node == data;
-}
-
-/**
- * fpga_region_find - find FPGA region
- * @np: device node of FPGA Region
- * Caller will need to put_device(&region->dev) when done.
- * Returns FPGA Region struct or NULL
- */
-static struct fpga_region *fpga_region_find(struct device_node *np)
+struct fpga_region *fpga_region_class_find(
+	struct device *start, const void *data,
+	int (*match)(struct device *, const void *))
 {
 	struct device *dev;
 
-	dev = class_find_device(fpga_region_class, NULL, np,
-				fpga_region_of_node_match);
+	dev = class_find_device(fpga_region_class, start, data, match);
 	if (!dev)
 		return NULL;
 
 	return to_fpga_region(dev);
 }
+EXPORT_SYMBOL_GPL(fpga_region_class_find);
 
 /**
  * fpga_region_get - get an exclusive reference to a fpga region
@@ -94,15 +64,13 @@ static struct fpga_region *fpga_region_get(struct fpga_region *region)
 	}
 
 	get_device(dev);
-	of_node_get(dev->of_node);
 	if (!try_module_get(dev->parent->driver->owner)) {
-		of_node_put(dev->of_node);
 		put_device(dev);
 		mutex_unlock(&region->mutex);
 		return ERR_PTR(-ENODEV);
 	}
 
-	dev_dbg(&region->dev, "get\n");
+	dev_dbg(dev, "get\n");
 
 	return region;
 }
@@ -116,403 +84,99 @@ static void fpga_region_put(struct fpga_region *region)
 {
 	struct device *dev = &region->dev;
 
-	dev_dbg(&region->dev, "put\n");
+	dev_dbg(dev, "put\n");
 
 	module_put(dev->parent->driver->owner);
-	of_node_put(dev->of_node);
 	put_device(dev);
 	mutex_unlock(&region->mutex);
 }
 
-/**
- * fpga_region_get_manager - get exclusive reference for FPGA manager
- * @region: FPGA region
- *
- * Get FPGA Manager from "fpga-mgr" property or from ancestor region.
- *
- * Caller should call fpga_mgr_put() when done with manager.
- *
- * Return: fpga manager struct or IS_ERR() condition containing error code.
- */
-static struct fpga_manager *fpga_region_get_manager(struct fpga_region *region)
-{
-	struct device *dev = &region->dev;
-	struct device_node *np = dev->of_node;
-	struct device_node  *mgr_node;
-	struct fpga_manager *mgr;
-
-	of_node_get(np);
-	while (np) {
-		if (of_device_is_compatible(np, "fpga-region")) {
-			mgr_node = of_parse_phandle(np, "fpga-mgr", 0);
-			if (mgr_node) {
-				mgr = of_fpga_mgr_get(mgr_node);
-				of_node_put(np);
-				return mgr;
-			}
-		}
-		np = of_get_next_parent(np);
-	}
-	of_node_put(np);
-
-	return ERR_PTR(-EINVAL);
-}
-
-/**
- * fpga_region_get_bridges - create a list of bridges
- * @region: FPGA region
- * @overlay: device node of the overlay
- *
- * Create a list of bridges including the parent bridge and the bridges
- * specified by "fpga-bridges" property.  Note that the
- * fpga_bridges_enable/disable/put functions are all fine with an empty list
- * if that happens.
- *
- * Caller should call fpga_bridges_put(&region->bridge_list) when
- * done with the bridges.
- *
- * Return 0 for success (even if there are no bridges specified)
- * or -EBUSY if any of the bridges are in use.
- */
-static int fpga_region_get_bridges(struct fpga_region *region,
-				   struct device_node *overlay)
-{
-	struct device *dev = &region->dev;
-	struct device_node *region_np = dev->of_node;
-	struct device_node *br, *np, *parent_br = NULL;
-	int i, ret;
-
-	/* If parent is a bridge, add to list */
-	ret = fpga_bridge_get_to_list(region_np->parent, region->info,
-				      &region->bridge_list);
-	if (ret == -EBUSY)
-		return ret;
-
-	if (!ret)
-		parent_br = region_np->parent;
-
-	/* If overlay has a list of bridges, use it. */
-	if (of_parse_phandle(overlay, "fpga-bridges", 0))
-		np = overlay;
-	else
-		np = region_np;
-
-	for (i = 0; ; i++) {
-		br = of_parse_phandle(np, "fpga-bridges", i);
-		if (!br)
-			break;
-
-		/* If parent bridge is in list, skip it. */
-		if (br == parent_br)
-			continue;
-
-		/* If node is a bridge, get it and add to list */
-		ret = fpga_bridge_get_to_list(br, region->info,
-					      &region->bridge_list);
-
-		/* If any of the bridges are in use, give up */
-		if (ret == -EBUSY) {
-			fpga_bridges_put(&region->bridge_list);
-			return -EBUSY;
-		}
-	}
-
-	return 0;
-}
-
 /**
  * fpga_region_program_fpga - program FPGA
  * @region: FPGA region
- * @firmware_name: name of FPGA image firmware file
- * @overlay: device node of the overlay
- * Program an FPGA using information in the device tree.
- * Function assumes that there is a firmware-name property.
+ * Program an FPGA using fpga image info (region->info).
  * Return 0 for success or negative error code.
  */
-static int fpga_region_program_fpga(struct fpga_region *region,
-				    const char *firmware_name,
-				    struct device_node *overlay)
+int fpga_region_program_fpga(struct fpga_region *region)
 {
-	struct fpga_manager *mgr;
+	struct device *dev = &region->dev;
+	struct fpga_image_info *info = region->info;
 	int ret;
 
 	region = fpga_region_get(region);
 	if (IS_ERR(region)) {
-		pr_err("failed to get fpga region\n");
+		dev_err(dev, "failed to get FPGA region\n");
 		return PTR_ERR(region);
 	}
 
-	mgr = fpga_region_get_manager(region);
-	if (IS_ERR(mgr)) {
-		pr_err("failed to get fpga region manager\n");
-		ret = PTR_ERR(mgr);
+	ret = fpga_mgr_lock(region->mgr);
+	if (ret) {
+		dev_err(dev, "FPGA manager is busy\n");
 		goto err_put_region;
 	}
 
-	ret = fpga_region_get_bridges(region, overlay);
-	if (ret) {
-		pr_err("failed to get fpga region bridges\n");
-		goto err_put_mgr;
+	/*
+	 * In some cases, we already have a list of bridges in the
+	 * fpga region struct.  Or we don't have any bridges.
+	 */
+	if (region->get_bridges) {
+		ret = region->get_bridges(region);
+		if (ret) {
+			dev_err(dev, "failed to get fpga region bridges\n");
+			goto err_unlock_mgr;
+		}
 	}
 
 	ret = fpga_bridges_disable(&region->bridge_list);
 	if (ret) {
-		pr_err("failed to disable region bridges\n");
+		dev_err(dev, "failed to disable bridges\n");
 		goto err_put_br;
 	}
 
-	ret = fpga_mgr_firmware_load(mgr, region->info, firmware_name);
+	ret = fpga_mgr_load(region->mgr, info);
 	if (ret) {
-		pr_err("failed to load fpga image\n");
+		dev_err(dev, "failed to load FPGA image\n");
 		goto err_put_br;
 	}
 
 	ret = fpga_bridges_enable(&region->bridge_list);
 	if (ret) {
-		pr_err("failed to enable region bridges\n");
+		dev_err(dev, "failed to enable region bridges\n");
 		goto err_put_br;
 	}
 
-	fpga_mgr_put(mgr);
+	fpga_mgr_unlock(region->mgr);
 	fpga_region_put(region);
 
 	return 0;
 
 err_put_br:
-	fpga_bridges_put(&region->bridge_list);
-err_put_mgr:
-	fpga_mgr_put(mgr);
+	if (region->get_bridges)
+		fpga_bridges_put(&region->bridge_list);
+err_unlock_mgr:
+	fpga_mgr_unlock(region->mgr);
 err_put_region:
 	fpga_region_put(region);
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(fpga_region_program_fpga);
 
-/**
- * child_regions_with_firmware
- * @overlay: device node of the overlay
- *
- * If the overlay adds child FPGA regions, they are not allowed to have
- * firmware-name property.
- *
- * Return 0 for OK or -EINVAL if child FPGA region adds firmware-name.
- */
-static int child_regions_with_firmware(struct device_node *overlay)
-{
-	struct device_node *child_region;
-	const char *child_firmware_name;
-	int ret = 0;
-
-	of_node_get(overlay);
-
-	child_region = of_find_matching_node(overlay, fpga_region_of_match);
-	while (child_region) {
-		if (!of_property_read_string(child_region, "firmware-name",
-					     &child_firmware_name)) {
-			ret = -EINVAL;
-			break;
-		}
-		child_region = of_find_matching_node(child_region,
-						     fpga_region_of_match);
-	}
-
-	of_node_put(child_region);
-
-	if (ret)
-		pr_err("firmware-name not allowed in child FPGA region: %pOF",
-		       child_region);
-
-	return ret;
-}
-
-/**
- * fpga_region_notify_pre_apply - pre-apply overlay notification
- *
- * @region: FPGA region that the overlay was applied to
- * @nd: overlay notification data
- *
- * Called after when an overlay targeted to a FPGA Region is about to be
- * applied.  Function will check the properties that will be added to the FPGA
- * region.  If the checks pass, it will program the FPGA.
- *
- * The checks are:
- * The overlay must add either firmware-name or external-fpga-config property
- * to the FPGA Region.
- *
- *   firmware-name         : program the FPGA
- *   external-fpga-config  : FPGA is already programmed
- *   encrypted-fpga-config : FPGA bitstream is encrypted
- *
- * The overlay can add other FPGA regions, but child FPGA regions cannot have a
- * firmware-name property since those regions don't exist yet.
- *
- * If the overlay that breaks the rules, notifier returns an error and the
- * overlay is rejected before it goes into the main tree.
- *
- * Returns 0 for success or negative error code for failure.
- */
-static int fpga_region_notify_pre_apply(struct fpga_region *region,
-					struct of_overlay_notify_data *nd)
+int fpga_region_register(struct device *dev, struct fpga_region *region)
 {
-	const char *firmware_name = NULL;
-	struct fpga_image_info *info;
-	int ret;
-
-	info = devm_kzalloc(&region->dev, sizeof(*info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	region->info = info;
-
-	/* Reject overlay if child FPGA Regions have firmware-name property */
-	ret = child_regions_with_firmware(nd->overlay);
-	if (ret)
-		return ret;
-
-	/* Read FPGA region properties from the overlay */
-	if (of_property_read_bool(nd->overlay, "partial-fpga-config"))
-		info->flags |= FPGA_MGR_PARTIAL_RECONFIG;
-
-	if (of_property_read_bool(nd->overlay, "external-fpga-config"))
-		info->flags |= FPGA_MGR_EXTERNAL_CONFIG;
-
-	if (of_property_read_bool(nd->overlay, "encrypted-fpga-config"))
-		info->flags |= FPGA_MGR_ENCRYPTED_BITSTREAM;
-
-	of_property_read_string(nd->overlay, "firmware-name", &firmware_name);
-
-	of_property_read_u32(nd->overlay, "region-unfreeze-timeout-us",
-			     &info->enable_timeout_us);
-
-	of_property_read_u32(nd->overlay, "region-freeze-timeout-us",
-			     &info->disable_timeout_us);
-
-	of_property_read_u32(nd->overlay, "config-complete-timeout-us",
-			     &info->config_complete_timeout_us);
-
-	/* If FPGA was externally programmed, don't specify firmware */
-	if ((info->flags & FPGA_MGR_EXTERNAL_CONFIG) && firmware_name) {
-		pr_err("error: specified firmware and external-fpga-config");
-		return -EINVAL;
-	}
-
-	/* FPGA is already configured externally.  We're done. */
-	if (info->flags & FPGA_MGR_EXTERNAL_CONFIG)
-		return 0;
-
-	/* If we got this far, we should be programming the FPGA */
-	if (!firmware_name) {
-		pr_err("should specify firmware-name or external-fpga-config\n");
-		return -EINVAL;
-	}
-
-	return fpga_region_program_fpga(region, firmware_name, nd->overlay);
-}
-
-/**
- * fpga_region_notify_post_remove - post-remove overlay notification
- *
- * @region: FPGA region that was targeted by the overlay that was removed
- * @nd: overlay notification data
- *
- * Called after an overlay has been removed if the overlay's target was a
- * FPGA region.
- */
-static void fpga_region_notify_post_remove(struct fpga_region *region,
-					   struct of_overlay_notify_data *nd)
-{
-	fpga_bridges_disable(&region->bridge_list);
-	fpga_bridges_put(&region->bridge_list);
-	devm_kfree(&region->dev, region->info);
-	region->info = NULL;
-}
-
-/**
- * of_fpga_region_notify - reconfig notifier for dynamic DT changes
- * @nb:		notifier block
- * @action:	notifier action
- * @arg:	reconfig data
- *
- * This notifier handles programming a FPGA when a "firmware-name" property is
- * added to a fpga-region.
- *
- * Returns NOTIFY_OK or error if FPGA programming fails.
- */
-static int of_fpga_region_notify(struct notifier_block *nb,
-				 unsigned long action, void *arg)
-{
-	struct of_overlay_notify_data *nd = arg;
-	struct fpga_region *region;
-	int ret;
-
-	switch (action) {
-	case OF_OVERLAY_PRE_APPLY:
-		pr_debug("%s OF_OVERLAY_PRE_APPLY\n", __func__);
-		break;
-	case OF_OVERLAY_POST_APPLY:
-		pr_debug("%s OF_OVERLAY_POST_APPLY\n", __func__);
-		return NOTIFY_OK;       /* not for us */
-	case OF_OVERLAY_PRE_REMOVE:
-		pr_debug("%s OF_OVERLAY_PRE_REMOVE\n", __func__);
-		return NOTIFY_OK;       /* not for us */
-	case OF_OVERLAY_POST_REMOVE:
-		pr_debug("%s OF_OVERLAY_POST_REMOVE\n", __func__);
-		break;
-	default:			/* should not happen */
-		return NOTIFY_OK;
-	}
-
-	region = fpga_region_find(nd->target);
-	if (!region)
-		return NOTIFY_OK;
-
-	ret = 0;
-	switch (action) {
-	case OF_OVERLAY_PRE_APPLY:
-		ret = fpga_region_notify_pre_apply(region, nd);
-		break;
-
-	case OF_OVERLAY_POST_REMOVE:
-		fpga_region_notify_post_remove(region, nd);
-		break;
-	}
-
-	put_device(&region->dev);
-
-	if (ret)
-		return notifier_from_errno(ret);
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block fpga_region_of_nb = {
-	.notifier_call = of_fpga_region_notify,
-};
-
-static int fpga_region_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct device_node *np = dev->of_node;
-	struct fpga_region *region;
 	int id, ret = 0;
 
-	region = kzalloc(sizeof(*region), GFP_KERNEL);
-	if (!region)
-		return -ENOMEM;
-
 	id = ida_simple_get(&fpga_region_ida, 0, 0, GFP_KERNEL);
-	if (id < 0) {
-		ret = id;
-		goto err_kfree;
-	}
+	if (id < 0)
+		return id;
 
 	mutex_init(&region->mutex);
 	INIT_LIST_HEAD(&region->bridge_list);
-
 	device_initialize(&region->dev);
+	region->dev.groups = region->groups;
 	region->dev.class = fpga_region_class;
 	region->dev.parent = dev;
-	region->dev.of_node = np;
+	region->dev.of_node = dev->of_node;
 	region->dev.id = id;
 	dev_set_drvdata(dev, region);
 
@@ -524,44 +188,27 @@ static int fpga_region_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_remove;
 
-	of_platform_populate(np, fpga_region_of_match, NULL, &region->dev);
-
-	dev_info(dev, "FPGA Region probed\n");
-
 	return 0;
 
 err_remove:
 	ida_simple_remove(&fpga_region_ida, id);
-err_kfree:
-	kfree(region);
-
 	return ret;
 }
+EXPORT_SYMBOL_GPL(fpga_region_register);
 
-static int fpga_region_remove(struct platform_device *pdev)
+int fpga_region_unregister(struct fpga_region *region)
 {
-	struct fpga_region *region = platform_get_drvdata(pdev);
-
 	device_unregister(&region->dev);
 
 	return 0;
 }
-
-static struct platform_driver fpga_region_driver = {
-	.probe = fpga_region_probe,
-	.remove = fpga_region_remove,
-	.driver = {
-		.name	= "fpga-region",
-		.of_match_table = of_match_ptr(fpga_region_of_match),
-	},
-};
+EXPORT_SYMBOL_GPL(fpga_region_unregister);
 
 static void fpga_region_dev_release(struct device *dev)
 {
 	struct fpga_region *region = to_fpga_region(dev);
 
 	ida_simple_remove(&fpga_region_ida, region->dev.id);
-	kfree(region);
 }
 
 /**
@@ -570,36 +217,17 @@ static void fpga_region_dev_release(struct device *dev)
  */
 static int __init fpga_region_init(void)
 {
-	int ret;
-
 	fpga_region_class = class_create(THIS_MODULE, "fpga_region");
 	if (IS_ERR(fpga_region_class))
 		return PTR_ERR(fpga_region_class);
 
 	fpga_region_class->dev_release = fpga_region_dev_release;
 
-	ret = of_overlay_notifier_register(&fpga_region_of_nb);
-	if (ret)
-		goto err_class;
-
-	ret = platform_driver_register(&fpga_region_driver);
-	if (ret)
-		goto err_plat;
-
 	return 0;
-
-err_plat:
-	of_overlay_notifier_unregister(&fpga_region_of_nb);
-err_class:
-	class_destroy(fpga_region_class);
-	ida_destroy(&fpga_region_ida);
-	return ret;
 }
 
 static void __exit fpga_region_exit(void)
 {
-	platform_driver_unregister(&fpga_region_driver);
-	of_overlay_notifier_unregister(&fpga_region_of_nb);
 	class_destroy(fpga_region_class);
 	ida_destroy(&fpga_region_ida);
 }
@@ -608,5 +236,5 @@ subsys_initcall(fpga_region_init);
 module_exit(fpga_region_exit);
 
 MODULE_DESCRIPTION("FPGA Region");
-MODULE_AUTHOR("Alan Tull <atull@opensource.altera.com>");
+MODULE_AUTHOR("Alan Tull <atull@kernel.org>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c
new file mode 100644
index 000000000000..119ff75522f1
--- /dev/null
+++ b/drivers/fpga/of-fpga-region.c
@@ -0,0 +1,504 @@
+/*
+ * FPGA Region - Device Tree support for FPGA programming under Linux
+ *
+ *  Copyright (C) 2013-2016 Altera Corporation
+ *  Copyright (C) 2017 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/fpga/fpga-bridge.h>
+#include <linux/fpga/fpga-mgr.h>
+#include <linux/fpga/fpga-region.h>
+#include <linux/idr.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+static const struct of_device_id fpga_region_of_match[] = {
+	{ .compatible = "fpga-region", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, fpga_region_of_match);
+
+static int fpga_region_of_node_match(struct device *dev, const void *data)
+{
+	return dev->of_node == data;
+}
+
+/**
+ * of_fpga_region_find - find FPGA region
+ * @np: device node of FPGA Region
+ *
+ * Caller will need to put_device(&region->dev) when done.
+ *
+ * Returns FPGA Region struct or NULL
+ */
+static struct fpga_region *of_fpga_region_find(struct device_node *np)
+{
+	return fpga_region_class_find(NULL, np, fpga_region_of_node_match);
+}
+
+/**
+ * of_fpga_region_get_mgr - get reference for FPGA manager
+ * @np: device node of FPGA region
+ *
+ * Get FPGA Manager from "fpga-mgr" property or from ancestor region.
+ *
+ * Caller should call fpga_mgr_put() when done with manager.
+ *
+ * Return: fpga manager struct or IS_ERR() condition containing error code.
+ */
+static struct fpga_manager *of_fpga_region_get_mgr(struct device_node *np)
+{
+	struct device_node  *mgr_node;
+	struct fpga_manager *mgr;
+
+	of_node_get(np);
+	while (np) {
+		if (of_device_is_compatible(np, "fpga-region")) {
+			mgr_node = of_parse_phandle(np, "fpga-mgr", 0);
+			if (mgr_node) {
+				mgr = of_fpga_mgr_get(mgr_node);
+				of_node_put(mgr_node);
+				of_node_put(np);
+				return mgr;
+			}
+		}
+		np = of_get_next_parent(np);
+	}
+	of_node_put(np);
+
+	return ERR_PTR(-EINVAL);
+}
+
+/**
+ * of_fpga_region_get_bridges - create a list of bridges
+ * @region: FPGA region
+ *
+ * Create a list of bridges including the parent bridge and the bridges
+ * specified by "fpga-bridges" property.  Note that the
+ * fpga_bridges_enable/disable/put functions are all fine with an empty list
+ * if that happens.
+ *
+ * Caller should call fpga_bridges_put(&region->bridge_list) when
+ * done with the bridges.
+ *
+ * Return 0 for success (even if there are no bridges specified)
+ * or -EBUSY if any of the bridges are in use.
+ */
+static int of_fpga_region_get_bridges(struct fpga_region *region)
+{
+	struct device *dev = &region->dev;
+	struct device_node *region_np = dev->of_node;
+	struct fpga_image_info *info = region->info;
+	struct device_node *br, *np, *parent_br = NULL;
+	int i, ret;
+
+	/* If parent is a bridge, add to list */
+	ret = of_fpga_bridge_get_to_list(region_np->parent, info,
+					 &region->bridge_list);
+
+	/* -EBUSY means parent is a bridge that is under use. Give up. */
+	if (ret == -EBUSY)
+		return ret;
+
+	/* Zero return code means parent was a bridge and was added to list. */
+	if (!ret)
+		parent_br = region_np->parent;
+
+	/* If overlay has a list of bridges, use it. */
+	br = of_parse_phandle(info->overlay, "fpga-bridges", 0);
+	if (br) {
+		of_node_put(br);
+		np = info->overlay;
+	} else {
+		np = region_np;
+	}
+
+	for (i = 0; ; i++) {
+		br = of_parse_phandle(np, "fpga-bridges", i);
+		if (!br)
+			break;
+
+		/* If parent bridge is in list, skip it. */
+		if (br == parent_br) {
+			of_node_put(br);
+			continue;
+		}
+
+		/* If node is a bridge, get it and add to list */
+		ret = of_fpga_bridge_get_to_list(br, info,
+						 &region->bridge_list);
+		of_node_put(br);
+
+		/* If any of the bridges are in use, give up */
+		if (ret == -EBUSY) {
+			fpga_bridges_put(&region->bridge_list);
+			return -EBUSY;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * child_regions_with_firmware
+ * @overlay: device node of the overlay
+ *
+ * If the overlay adds child FPGA regions, they are not allowed to have
+ * firmware-name property.
+ *
+ * Return 0 for OK or -EINVAL if child FPGA region adds firmware-name.
+ */
+static int child_regions_with_firmware(struct device_node *overlay)
+{
+	struct device_node *child_region;
+	const char *child_firmware_name;
+	int ret = 0;
+
+	of_node_get(overlay);
+
+	child_region = of_find_matching_node(overlay, fpga_region_of_match);
+	while (child_region) {
+		if (!of_property_read_string(child_region, "firmware-name",
+					     &child_firmware_name)) {
+			ret = -EINVAL;
+			break;
+		}
+		child_region = of_find_matching_node(child_region,
+						     fpga_region_of_match);
+	}
+
+	of_node_put(child_region);
+
+	if (ret)
+		pr_err("firmware-name not allowed in child FPGA region: %pOF",
+		       child_region);
+
+	return ret;
+}
+
+/**
+ * of_fpga_region_parse_ov - parse and check overlay applied to region
+ *
+ * @region: FPGA region
+ * @overlay: overlay applied to the FPGA region
+ *
+ * Given an overlay applied to a FPGA region, parse the FPGA image specific
+ * info in the overlay and do some checking.
+ *
+ * Returns:
+ *   NULL if overlay doesn't direct us to program the FPGA.
+ *   fpga_image_info struct if there is an image to program.
+ *   error code for invalid overlay.
+ */
+static struct fpga_image_info *of_fpga_region_parse_ov(
+						struct fpga_region *region,
+						struct device_node *overlay)
+{
+	struct device *dev = &region->dev;
+	struct fpga_image_info *info;
+	const char *firmware_name;
+	int ret;
+
+	if (region->info) {
+		dev_err(dev, "Region already has overlay applied.\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	/*
+	 * Reject overlay if child FPGA Regions added in the overlay have
+	 * firmware-name property (would mean that an FPGA region that has
+	 * not been added to the live tree yet is doing FPGA programming).
+	 */
+	ret = child_regions_with_firmware(overlay);
+	if (ret)
+		return ERR_PTR(ret);
+
+	info = fpga_image_info_alloc(dev);
+	if (!info)
+		return ERR_PTR(-ENOMEM);
+
+	info->overlay = overlay;
+
+	/* Read FPGA region properties from the overlay */
+	if (of_property_read_bool(overlay, "partial-fpga-config"))
+		info->flags |= FPGA_MGR_PARTIAL_RECONFIG;
+
+	if (of_property_read_bool(overlay, "external-fpga-config"))
+		info->flags |= FPGA_MGR_EXTERNAL_CONFIG;
+
+	if (of_property_read_bool(overlay, "encrypted-fpga-config"))
+		info->flags |= FPGA_MGR_ENCRYPTED_BITSTREAM;
+
+	if (!of_property_read_string(overlay, "firmware-name",
+				     &firmware_name)) {
+		info->firmware_name = devm_kstrdup(dev, firmware_name,
+						   GFP_KERNEL);
+		if (!info->firmware_name)
+			return ERR_PTR(-ENOMEM);
+	}
+
+	of_property_read_u32(overlay, "region-unfreeze-timeout-us",
+			     &info->enable_timeout_us);
+
+	of_property_read_u32(overlay, "region-freeze-timeout-us",
+			     &info->disable_timeout_us);
+
+	of_property_read_u32(overlay, "config-complete-timeout-us",
+			     &info->config_complete_timeout_us);
+
+	/* If overlay is not programming the FPGA, don't need FPGA image info */
+	if (!info->firmware_name) {
+		ret = 0;
+		goto ret_no_info;
+	}
+
+	/*
+	 * If overlay informs us FPGA was externally programmed, specifying
+	 * firmware here would be ambiguous.
+	 */
+	if (info->flags & FPGA_MGR_EXTERNAL_CONFIG) {
+		dev_err(dev, "error: specified firmware and external-fpga-config");
+		ret = -EINVAL;
+		goto ret_no_info;
+	}
+
+	return info;
+ret_no_info:
+	fpga_image_info_free(info);
+	return ERR_PTR(ret);
+}
+
+/**
+ * of_fpga_region_notify_pre_apply - pre-apply overlay notification
+ *
+ * @region: FPGA region that the overlay was applied to
+ * @nd: overlay notification data
+ *
+ * Called when an overlay targeted to a FPGA Region is about to be applied.
+ * Parses the overlay for properties that influence how the FPGA will be
+ * programmed and does some checking. If the checks pass, programs the FPGA.
+ * If the checks fail, overlay is rejected and does not get added to the
+ * live tree.
+ *
+ * Returns 0 for success or negative error code for failure.
+ */
+static int of_fpga_region_notify_pre_apply(struct fpga_region *region,
+					   struct of_overlay_notify_data *nd)
+{
+	struct device *dev = &region->dev;
+	struct fpga_image_info *info;
+	int ret;
+
+	info = of_fpga_region_parse_ov(region, nd->overlay);
+	if (IS_ERR(info))
+		return PTR_ERR(info);
+
+	/* If overlay doesn't program the FPGA, accept it anyway. */
+	if (!info)
+		return 0;
+
+	if (region->info) {
+		dev_err(dev, "Region already has overlay applied.\n");
+		return -EINVAL;
+	}
+
+	region->info = info;
+	ret = fpga_region_program_fpga(region);
+	if (ret) {
+		/* error; reject overlay */
+		fpga_image_info_free(info);
+		region->info = NULL;
+	}
+
+	return ret;
+}
+
+/**
+ * of_fpga_region_notify_post_remove - post-remove overlay notification
+ *
+ * @region: FPGA region that was targeted by the overlay that was removed
+ * @nd: overlay notification data
+ *
+ * Called after an overlay has been removed if the overlay's target was a
+ * FPGA region.
+ */
+static void of_fpga_region_notify_post_remove(struct fpga_region *region,
+					      struct of_overlay_notify_data *nd)
+{
+	fpga_bridges_disable(&region->bridge_list);
+	fpga_bridges_put(&region->bridge_list);
+	fpga_image_info_free(region->info);
+	region->info = NULL;
+}
+
+/**
+ * of_fpga_region_notify - reconfig notifier for dynamic DT changes
+ * @nb:		notifier block
+ * @action:	notifier action
+ * @arg:	reconfig data
+ *
+ * This notifier handles programming a FPGA when a "firmware-name" property is
+ * added to a fpga-region.
+ *
+ * Returns NOTIFY_OK or error if FPGA programming fails.
+ */
+static int of_fpga_region_notify(struct notifier_block *nb,
+				 unsigned long action, void *arg)
+{
+	struct of_overlay_notify_data *nd = arg;
+	struct fpga_region *region;
+	int ret;
+
+	switch (action) {
+	case OF_OVERLAY_PRE_APPLY:
+		pr_debug("%s OF_OVERLAY_PRE_APPLY\n", __func__);
+		break;
+	case OF_OVERLAY_POST_APPLY:
+		pr_debug("%s OF_OVERLAY_POST_APPLY\n", __func__);
+		return NOTIFY_OK;       /* not for us */
+	case OF_OVERLAY_PRE_REMOVE:
+		pr_debug("%s OF_OVERLAY_PRE_REMOVE\n", __func__);
+		return NOTIFY_OK;       /* not for us */
+	case OF_OVERLAY_POST_REMOVE:
+		pr_debug("%s OF_OVERLAY_POST_REMOVE\n", __func__);
+		break;
+	default:			/* should not happen */
+		return NOTIFY_OK;
+	}
+
+	region = of_fpga_region_find(nd->target);
+	if (!region)
+		return NOTIFY_OK;
+
+	ret = 0;
+	switch (action) {
+	case OF_OVERLAY_PRE_APPLY:
+		ret = of_fpga_region_notify_pre_apply(region, nd);
+		break;
+
+	case OF_OVERLAY_POST_REMOVE:
+		of_fpga_region_notify_post_remove(region, nd);
+		break;
+	}
+
+	put_device(&region->dev);
+
+	if (ret)
+		return notifier_from_errno(ret);
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block fpga_region_of_nb = {
+	.notifier_call = of_fpga_region_notify,
+};
+
+static int of_fpga_region_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct fpga_region *region;
+	struct fpga_manager *mgr;
+	int ret;
+
+	/* Find the FPGA mgr specified by region or parent region. */
+	mgr = of_fpga_region_get_mgr(np);
+	if (IS_ERR(mgr))
+		return -EPROBE_DEFER;
+
+	region = devm_kzalloc(dev, sizeof(*region), GFP_KERNEL);
+	if (!region) {
+		ret = -ENOMEM;
+		goto eprobe_mgr_put;
+	}
+
+	region->mgr = mgr;
+
+	/* Specify how to get bridges for this type of region. */
+	region->get_bridges = of_fpga_region_get_bridges;
+
+	ret = fpga_region_register(dev, region);
+	if (ret)
+		goto eprobe_mgr_put;
+
+	of_platform_populate(np, fpga_region_of_match, NULL, &region->dev);
+
+	dev_info(dev, "FPGA Region probed\n");
+
+	return 0;
+
+eprobe_mgr_put:
+	fpga_mgr_put(mgr);
+	return ret;
+}
+
+static int of_fpga_region_remove(struct platform_device *pdev)
+{
+	struct fpga_region *region = platform_get_drvdata(pdev);
+
+	fpga_region_unregister(region);
+	fpga_mgr_put(region->mgr);
+
+	return 0;
+}
+
+static struct platform_driver of_fpga_region_driver = {
+	.probe = of_fpga_region_probe,
+	.remove = of_fpga_region_remove,
+	.driver = {
+		.name	= "of-fpga-region",
+		.of_match_table = of_match_ptr(fpga_region_of_match),
+	},
+};
+
+/**
+ * fpga_region_init - init function for fpga_region class
+ * Creates the fpga_region class and registers a reconfig notifier.
+ */
+static int __init of_fpga_region_init(void)
+{
+	int ret;
+
+	ret = of_overlay_notifier_register(&fpga_region_of_nb);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&of_fpga_region_driver);
+	if (ret)
+		goto err_plat;
+
+	return 0;
+
+err_plat:
+	of_overlay_notifier_unregister(&fpga_region_of_nb);
+	return ret;
+}
+
+static void __exit of_fpga_region_exit(void)
+{
+	platform_driver_unregister(&of_fpga_region_driver);
+	of_overlay_notifier_unregister(&fpga_region_of_nb);
+}
+
+subsys_initcall(of_fpga_region_init);
+module_exit(of_fpga_region_exit);
+
+MODULE_DESCRIPTION("FPGA Region");
+MODULE_AUTHOR("Alan Tull <atull@kernel.org>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/fpga/socfpga-a10.c b/drivers/fpga/socfpga-a10.c
index f8770af0f6b5..a46e343a5b72 100644
--- a/drivers/fpga/socfpga-a10.c
+++ b/drivers/fpga/socfpga-a10.c
@@ -519,8 +519,14 @@ static int socfpga_a10_fpga_probe(struct platform_device *pdev)
 		return -EBUSY;
 	}
 
-	return fpga_mgr_register(dev, "SoCFPGA Arria10 FPGA Manager",
+	ret = fpga_mgr_register(dev, "SoCFPGA Arria10 FPGA Manager",
 				 &socfpga_a10_fpga_mgr_ops, priv);
+	if (ret) {
+		clk_disable_unprepare(priv->clk);
+		return ret;
+	}
+
+	return 0;
 }
 
 static int socfpga_a10_fpga_remove(struct platform_device *pdev)
diff --git a/drivers/fsi/Kconfig b/drivers/fsi/Kconfig
index 6821ed0cd5e8..513e35173aaa 100644
--- a/drivers/fsi/Kconfig
+++ b/drivers/fsi/Kconfig
@@ -2,9 +2,7 @@
 # FSI subsystem
 #
 
-menu "FSI support"
-
-config FSI
+menuconfig FSI
 	tristate "FSI support"
 	select CRC4
 	---help---
@@ -34,5 +32,3 @@ config FSI_SCOM
 	This option enables an FSI based SCOM device driver.
 
 endif
-
-endmenu
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index d6a8e851ad13..8dbb2280538d 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -122,12 +122,6 @@ config GPIO_ATH79
 	  Select this option to enable GPIO driver for
 	  Atheros AR71XX/AR724X/AR913X SoC devices.
 
-config GPIO_AXP209
-	tristate "X-Powers AXP209 PMIC GPIO Support"
-	depends on MFD_AXP20X
-	help
-	  Say yes to enable GPIO support for the AXP209 PMIC
-
 config GPIO_BCM_KONA
 	bool "Broadcom Kona GPIO"
 	depends on OF_GPIO && (ARCH_BCM_MOBILE || COMPILE_TEST)
@@ -704,6 +698,22 @@ config GPIO_TS5500
 	  blocks of the TS-5500: DIO1, DIO2 and the LCD port, and the TS-5600
 	  LCD port.
 
+config GPIO_WINBOND
+	tristate "Winbond Super I/O GPIO support"
+	depends on ISA_BUS_API
+	help
+	  This option enables support for GPIOs found on Winbond Super I/O
+	  chips.
+	  Currently, only W83627UHG (also known as Nuvoton NCT6627UD) is
+	  supported.
+
+	  You will need to provide a module parameter "gpios", or a
+	  boot-time parameter "gpio_winbond.gpios" with a bitmask of GPIO
+	  ports to enable (bit 0 is GPIO1, bit 1 is GPIO2, etc.).
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called gpio-winbond.
+
 config GPIO_WS16C48
 	tristate "WinSystems WS16C48 GPIO support"
 	depends on ISA_BUS_API
@@ -1234,6 +1244,16 @@ config GPIO_PCI_IDIO_16
 	  low). Input filter control is not supported by this driver, and the
 	  input filters are deactivated by this driver.
 
+config GPIO_PCIE_IDIO_24
+	tristate "ACCES PCIe-IDIO-24 GPIO support"
+	select GPIOLIB_IRQCHIP
+	help
+	  Enables GPIO support for the ACCES PCIe-IDIO-24 family (PCIe-IDIO-24,
+	  PCIe-IDI-24, PCIe-IDO-24, PCIe-IDIO-12). An interrupt is generated
+	  when any of the inputs change state (low to high or high to low).
+	  Input filter control is not supported by this driver, and the input
+	  filters are deactivated by this driver.
+
 config GPIO_RDC321X
 	tristate "RDC R-321x GPIO support"
 	select MFD_CORE
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index 4bc24febb889..cccb0d40846c 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -32,7 +32,6 @@ obj-$(CONFIG_GPIO_AMDPT)	+= gpio-amdpt.o
 obj-$(CONFIG_GPIO_ARIZONA)	+= gpio-arizona.o
 obj-$(CONFIG_GPIO_ATH79)	+= gpio-ath79.o
 obj-$(CONFIG_GPIO_ASPEED)	+= gpio-aspeed.o
-obj-$(CONFIG_GPIO_AXP209)	+= gpio-axp209.o
 obj-$(CONFIG_GPIO_BCM_KONA)	+= gpio-bcm-kona.o
 obj-$(CONFIG_GPIO_BD9571MWV)	+= gpio-bd9571mwv.o
 obj-$(CONFIG_GPIO_BRCMSTB)	+= gpio-brcmstb.o
@@ -96,6 +95,7 @@ obj-$(CONFIG_GPIO_PCA953X)	+= gpio-pca953x.o
 obj-$(CONFIG_GPIO_PCF857X)	+= gpio-pcf857x.o
 obj-$(CONFIG_GPIO_PCH)		+= gpio-pch.o
 obj-$(CONFIG_GPIO_PCI_IDIO_16)	+= gpio-pci-idio-16.o
+obj-$(CONFIG_GPIO_PCIE_IDIO_24)	+= gpio-pcie-idio-24.o
 obj-$(CONFIG_GPIO_PISOSR)	+= gpio-pisosr.o
 obj-$(CONFIG_GPIO_PL061)	+= gpio-pl061.o
 obj-$(CONFIG_GPIO_PXA)		+= gpio-pxa.o
@@ -140,6 +140,7 @@ obj-$(CONFIG_GPIO_VIPERBOARD)	+= gpio-viperboard.o
 obj-$(CONFIG_GPIO_VR41XX)	+= gpio-vr41xx.o
 obj-$(CONFIG_GPIO_VX855)	+= gpio-vx855.o
 obj-$(CONFIG_GPIO_WHISKEY_COVE)	+= gpio-wcove.o
+obj-$(CONFIG_GPIO_WINBOND)	+= gpio-winbond.o
 obj-$(CONFIG_GPIO_WM831X)	+= gpio-wm831x.o
 obj-$(CONFIG_GPIO_WM8350)	+= gpio-wm8350.o
 obj-$(CONFIG_GPIO_WM8994)	+= gpio-wm8994.o
diff --git a/drivers/gpio/devres.c b/drivers/gpio/devres.c
index afbff155a0ba..e82cc763633c 100644
--- a/drivers/gpio/devres.c
+++ b/drivers/gpio/devres.c
@@ -124,6 +124,48 @@ struct gpio_desc *__must_check devm_gpiod_get_index(struct device *dev,
 }
 EXPORT_SYMBOL(devm_gpiod_get_index);
 
+/**
+ * devm_gpiod_get_from_of_node() - obtain a GPIO from an OF node
+ * @dev:	device for lifecycle management
+ * @node:	handle of the OF node
+ * @propname:	name of the DT property representing the GPIO
+ * @index:	index of the GPIO to obtain for the consumer
+ * @dflags:	GPIO initialization flags
+ * @label:	label to attach to the requested GPIO
+ *
+ * Returns:
+ * On successful request the GPIO pin is configured in accordance with
+ * provided @dflags.
+ *
+ * In case of error an ERR_PTR() is returned.
+ */
+struct gpio_desc *devm_gpiod_get_from_of_node(struct device *dev,
+					      struct device_node *node,
+					      const char *propname, int index,
+					      enum gpiod_flags dflags,
+					      const char *label)
+{
+	struct gpio_desc **dr;
+	struct gpio_desc *desc;
+
+	dr = devres_alloc(devm_gpiod_release, sizeof(struct gpio_desc *),
+			  GFP_KERNEL);
+	if (!dr)
+		return ERR_PTR(-ENOMEM);
+
+	desc = gpiod_get_from_of_node(node, propname, index, dflags, label);
+	if (IS_ERR(desc)) {
+		devres_free(dr);
+		return desc;
+	}
+
+	*dr = desc;
+	devres_add(dev, dr);
+
+	return desc;
+}
+EXPORT_SYMBOL(devm_gpiod_get_from_of_node);
+
 /**
  * devm_fwnode_get_index_gpiod_from_child - get a GPIO descriptor from a
  *					    device's child node
diff --git a/drivers/gpio/gpio-74x164.c b/drivers/gpio/gpio-74x164.c
index 15a1f4b348c4..fb7b620763a2 100644
--- a/drivers/gpio/gpio-74x164.c
+++ b/drivers/gpio/gpio-74x164.c
@@ -9,12 +9,11 @@
  *  published by the Free Software Foundation.
  */
 
-#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/mutex.h>
 #include <linux/spi/spi.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/driver.h>
+#include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/module.h>
 
diff --git a/drivers/gpio/gpio-adp5520.c b/drivers/gpio/gpio-adp5520.c
index abf199609546..21452622d954 100644
--- a/drivers/gpio/gpio-adp5520.c
+++ b/drivers/gpio/gpio-adp5520.c
@@ -12,8 +12,7 @@
 #include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/adp5520.h>
-
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 
 struct adp5520_gpio {
 	struct device *master;
diff --git a/drivers/gpio/gpio-adp5588.c b/drivers/gpio/gpio-adp5588.c
index e717f8dc3966..3530ccd17e04 100644
--- a/drivers/gpio/gpio-adp5588.c
+++ b/drivers/gpio/gpio-adp5588.c
@@ -12,7 +12,7 @@
 #include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/i2c.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index 8e76d390e653..8c3ff6e2366f 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -18,7 +18,8 @@
 
 #include <linux/io.h>
 #include <linux/module.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/driver.h>
+#include <linux/of_gpio.h> /* For of_mm_gpio_chip */
 #include <linux/platform_device.h>
 
 #define ALTERA_GPIO_MAX_NGPIO		32
diff --git a/drivers/gpio/gpio-amd8111.c b/drivers/gpio/gpio-amd8111.c
index 30ad7d7c1678..fdcebe59510d 100644
--- a/drivers/gpio/gpio-amd8111.c
+++ b/drivers/gpio/gpio-amd8111.c
@@ -28,7 +28,7 @@
 #include <linux/ioport.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/pci.h>
 #include <linux/spinlock.h>
 
diff --git a/drivers/gpio/gpio-arizona.c b/drivers/gpio/gpio-arizona.c
index d4e6ba0301bc..ba51ea15f379 100644
--- a/drivers/gpio/gpio-arizona.c
+++ b/drivers/gpio/gpio-arizona.c
@@ -15,7 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/module.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <linux/seq_file.h>
diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c
index 6b3ca6601af2..77e485557498 100644
--- a/drivers/gpio/gpio-aspeed.c
+++ b/drivers/gpio/gpio-aspeed.c
@@ -60,6 +60,7 @@ struct aspeed_gpio_bank {
 	uint16_t	val_regs;
 	uint16_t	irq_regs;
 	uint16_t	debounce_regs;
+	uint16_t	tolerance_regs;
 	const char	names[4][3];
 };
 
@@ -70,48 +71,56 @@ static const struct aspeed_gpio_bank aspeed_gpio_banks[] = {
 		.val_regs = 0x0000,
 		.irq_regs = 0x0008,
 		.debounce_regs = 0x0040,
+		.tolerance_regs = 0x001c,
 		.names = { "A", "B", "C", "D" },
 	},
 	{
 		.val_regs = 0x0020,
 		.irq_regs = 0x0028,
 		.debounce_regs = 0x0048,
+		.tolerance_regs = 0x003c,
 		.names = { "E", "F", "G", "H" },
 	},
 	{
 		.val_regs = 0x0070,
 		.irq_regs = 0x0098,
 		.debounce_regs = 0x00b0,
+		.tolerance_regs = 0x00ac,
 		.names = { "I", "J", "K", "L" },
 	},
 	{
 		.val_regs = 0x0078,
 		.irq_regs = 0x00e8,
 		.debounce_regs = 0x0100,
+		.tolerance_regs = 0x00fc,
 		.names = { "M", "N", "O", "P" },
 	},
 	{
 		.val_regs = 0x0080,
 		.irq_regs = 0x0118,
 		.debounce_regs = 0x0130,
+		.tolerance_regs = 0x012c,
 		.names = { "Q", "R", "S", "T" },
 	},
 	{
 		.val_regs = 0x0088,
 		.irq_regs = 0x0148,
 		.debounce_regs = 0x0160,
+		.tolerance_regs = 0x015c,
 		.names = { "U", "V", "W", "X" },
 	},
 	{
 		.val_regs = 0x01E0,
 		.irq_regs = 0x0178,
 		.debounce_regs = 0x0190,
+		.tolerance_regs = 0x018c,
 		.names = { "Y", "Z", "AA", "AB" },
 	},
 	{
-		.val_regs = 0x01E8,
-		.irq_regs = 0x01A8,
+		.val_regs = 0x01e8,
+		.irq_regs = 0x01a8,
 		.debounce_regs = 0x01c0,
+		.tolerance_regs = 0x01bc,
 		.names = { "AC", "", "", "" },
 	},
 };
@@ -140,7 +149,7 @@ static const struct aspeed_gpio_bank *to_bank(unsigned int offset)
 {
 	unsigned int bank = GPIO_BANK(offset);
 
-	WARN_ON(bank > ARRAY_SIZE(aspeed_gpio_banks));
+	WARN_ON(bank >= ARRAY_SIZE(aspeed_gpio_banks));
 	return &aspeed_gpio_banks[bank];
 }
 
@@ -534,6 +543,30 @@ static int aspeed_gpio_setup_irqs(struct aspeed_gpio *gpio,
 	return 0;
 }
 
+static int aspeed_gpio_reset_tolerance(struct gpio_chip *chip,
+					unsigned int offset, bool enable)
+{
+	struct aspeed_gpio *gpio = gpiochip_get_data(chip);
+	const struct aspeed_gpio_bank *bank;
+	unsigned long flags;
+	u32 val;
+
+	bank = to_bank(offset);
+
+	spin_lock_irqsave(&gpio->lock, flags);
+	val = readl(gpio->base + bank->tolerance_regs);
+
+	if (enable)
+		val |= GPIO_BIT(offset);
+	else
+		val &= ~GPIO_BIT(offset);
+
+	writel(val, gpio->base + bank->tolerance_regs);
+	spin_unlock_irqrestore(&gpio->lock, flags);
+
+	return 0;
+}
+
 static int aspeed_gpio_request(struct gpio_chip *chip, unsigned int offset)
 {
 	if (!have_gpio(gpiochip_get_data(chip), offset))
@@ -771,6 +804,8 @@ static int aspeed_gpio_set_config(struct gpio_chip *chip, unsigned int offset,
 			param == PIN_CONFIG_DRIVE_OPEN_SOURCE)
 		/* Return -ENOTSUPP to trigger emulation, as per datasheet */
 		return -ENOTSUPP;
+	else if (param == PIN_CONFIG_PERSIST_STATE)
+		return aspeed_gpio_reset_tolerance(chip, offset, arg);
 
 	return -ENOTSUPP;
 }
diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c
index 5fad89dfab7e..3ae7c1876bf4 100644
--- a/drivers/gpio/gpio-ath79.c
+++ b/drivers/gpio/gpio-ath79.c
@@ -324,3 +324,6 @@ static struct platform_driver ath79_gpio_driver = {
 };
 
 module_platform_driver(ath79_gpio_driver);
+
+MODULE_DESCRIPTION("Atheros AR71XX/AR724X/AR913X GPIO API support");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-axp209.c b/drivers/gpio/gpio-axp209.c
deleted file mode 100644
index 4a346b7b4172..000000000000
--- a/drivers/gpio/gpio-axp209.c
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * AXP20x GPIO driver
- *
- * Copyright (C) 2016 Maxime Ripard <maxime.ripard@free-electrons.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under  the terms of the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the License, or (at your
- * option) any later version.
- */
-
-#include <linux/bitops.h>
-#include <linux/device.h>
-#include <linux/gpio/driver.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/mfd/axp20x.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/regmap.h>
-#include <linux/slab.h>
-
-#define AXP20X_GPIO_FUNCTIONS		0x7
-#define AXP20X_GPIO_FUNCTION_OUT_LOW	0
-#define AXP20X_GPIO_FUNCTION_OUT_HIGH	1
-#define AXP20X_GPIO_FUNCTION_INPUT	2
-
-struct axp20x_gpio {
-	struct gpio_chip	chip;
-	struct regmap		*regmap;
-};
-
-static int axp20x_gpio_get_reg(unsigned offset)
-{
-	switch (offset) {
-	case 0:
-		return AXP20X_GPIO0_CTRL;
-	case 1:
-		return AXP20X_GPIO1_CTRL;
-	case 2:
-		return AXP20X_GPIO2_CTRL;
-	}
-
-	return -EINVAL;
-}
-
-static int axp20x_gpio_input(struct gpio_chip *chip, unsigned offset)
-{
-	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
-	int reg;
-
-	reg = axp20x_gpio_get_reg(offset);
-	if (reg < 0)
-		return reg;
-
-	return regmap_update_bits(gpio->regmap, reg,
-				  AXP20X_GPIO_FUNCTIONS,
-				  AXP20X_GPIO_FUNCTION_INPUT);
-}
-
-static int axp20x_gpio_get(struct gpio_chip *chip, unsigned offset)
-{
-	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
-	unsigned int val;
-	int ret;
-
-	ret = regmap_read(gpio->regmap, AXP20X_GPIO20_SS, &val);
-	if (ret)
-		return ret;
-
-	return !!(val & BIT(offset + 4));
-}
-
-static int axp20x_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
-{
-	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
-	unsigned int val;
-	int reg, ret;
-
-	reg = axp20x_gpio_get_reg(offset);
-	if (reg < 0)
-		return reg;
-
-	ret = regmap_read(gpio->regmap, reg, &val);
-	if (ret)
-		return ret;
-
-	/*
-	 * This shouldn't really happen if the pin is in use already,
-	 * or if it's not in use yet, it doesn't matter since we're
-	 * going to change the value soon anyway. Default to output.
-	 */
-	if ((val & AXP20X_GPIO_FUNCTIONS) > 2)
-		return 0;
-
-	/*
-	 * The GPIO directions are the three lowest values.
-	 * 2 is input, 0 and 1 are output
-	 */
-	return val & 2;
-}
-
-static int axp20x_gpio_output(struct gpio_chip *chip, unsigned offset,
-			      int value)
-{
-	struct axp20x_gpio *gpio = gpiochip_get_data(chip);
-	int reg;
-
-	reg = axp20x_gpio_get_reg(offset);
-	if (reg < 0)
-		return reg;
-
-	return regmap_update_bits(gpio->regmap, reg,
-				  AXP20X_GPIO_FUNCTIONS,
-				  value ? AXP20X_GPIO_FUNCTION_OUT_HIGH
-				  : AXP20X_GPIO_FUNCTION_OUT_LOW);
-}
-
-static void axp20x_gpio_set(struct gpio_chip *chip, unsigned offset,
-			    int value)
-{
-	axp20x_gpio_output(chip, offset, value);
-}
-
-static int axp20x_gpio_probe(struct platform_device *pdev)
-{
-	struct axp20x_dev *axp20x = dev_get_drvdata(pdev->dev.parent);
-	struct axp20x_gpio *gpio;
-	int ret;
-
-	if (!of_device_is_available(pdev->dev.of_node))
-		return -ENODEV;
-
-	if (!axp20x) {
-		dev_err(&pdev->dev, "Parent drvdata not set\n");
-		return -EINVAL;
-	}
-
-	gpio = devm_kzalloc(&pdev->dev, sizeof(*gpio), GFP_KERNEL);
-	if (!gpio)
-		return -ENOMEM;
-
-	gpio->chip.base			= -1;
-	gpio->chip.can_sleep		= true;
-	gpio->chip.parent		= &pdev->dev;
-	gpio->chip.label		= dev_name(&pdev->dev);
-	gpio->chip.owner		= THIS_MODULE;
-	gpio->chip.get			= axp20x_gpio_get;
-	gpio->chip.get_direction	= axp20x_gpio_get_direction;
-	gpio->chip.set			= axp20x_gpio_set;
-	gpio->chip.direction_input	= axp20x_gpio_input;
-	gpio->chip.direction_output	= axp20x_gpio_output;
-	gpio->chip.ngpio		= 3;
-
-	gpio->regmap = axp20x->regmap;
-
-	ret = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio);
-	if (ret) {
-		dev_err(&pdev->dev, "Failed to register GPIO chip\n");
-		return ret;
-	}
-
-	dev_info(&pdev->dev, "AXP209 GPIO driver loaded\n");
-
-	return 0;
-}
-
-static const struct of_device_id axp20x_gpio_match[] = {
-	{ .compatible = "x-powers,axp209-gpio" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, axp20x_gpio_match);
-
-static struct platform_driver axp20x_gpio_driver = {
-	.probe		= axp20x_gpio_probe,
-	.driver = {
-		.name		= "axp20x-gpio",
-		.of_match_table	= axp20x_gpio_match,
-	},
-};
-
-module_platform_driver(axp20x_gpio_driver);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
-MODULE_DESCRIPTION("AXP20x PMIC GPIO driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-bcm-kona.c b/drivers/gpio/gpio-bcm-kona.c
index 76861a00bb92..eb8369b21e90 100644
--- a/drivers/gpio/gpio-bcm-kona.c
+++ b/drivers/gpio/gpio-bcm-kona.c
@@ -17,7 +17,7 @@
 #include <linux/bitops.h>
 #include <linux/err.h>
 #include <linux/io.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/init.h>
@@ -127,7 +127,7 @@ static int bcm_kona_gpio_get_dir(struct gpio_chip *chip, unsigned gpio)
 	u32 val;
 
 	val = readl(reg_base + GPIO_CONTROL(gpio)) & GPIO_GPCTR0_IOTR_MASK;
-	return val ? GPIOF_DIR_IN : GPIOF_DIR_OUT;
+	return !!val;
 }
 
 static void bcm_kona_gpio_set(struct gpio_chip *chip, unsigned gpio, int value)
@@ -144,7 +144,7 @@ static void bcm_kona_gpio_set(struct gpio_chip *chip, unsigned gpio, int value)
 	raw_spin_lock_irqsave(&kona_gpio->lock, flags);
 
 	/* this function only applies to output pin */
-	if (bcm_kona_gpio_get_dir(chip, gpio) == GPIOF_DIR_IN)
+	if (bcm_kona_gpio_get_dir(chip, gpio) == 1)
 		goto out;
 
 	reg_offset = value ? GPIO_OUT_SET(bank_id) : GPIO_OUT_CLEAR(bank_id);
@@ -170,7 +170,7 @@ static int bcm_kona_gpio_get(struct gpio_chip *chip, unsigned gpio)
 	reg_base = kona_gpio->reg_base;
 	raw_spin_lock_irqsave(&kona_gpio->lock, flags);
 
-	if (bcm_kona_gpio_get_dir(chip, gpio) == GPIOF_DIR_IN)
+	if (bcm_kona_gpio_get_dir(chip, gpio) == 1)
 		reg_offset = GPIO_IN_STATUS(bank_id);
 	else
 		reg_offset = GPIO_OUT_STATUS(bank_id);
diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c
index bb4f8cf18bd9..16c7f9f49416 100644
--- a/drivers/gpio/gpio-brcmstb.c
+++ b/drivers/gpio/gpio-brcmstb.c
@@ -19,7 +19,6 @@
 #include <linux/irqdomain.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/interrupt.h>
-#include <linux/bitops.h>
 
 enum gio_reg_index {
 	GIO_REG_ODEN = 0,
diff --git a/drivers/gpio/gpio-bt8xx.c b/drivers/gpio/gpio-bt8xx.c
index acefb25e8eca..b8ec75cbd4b5 100644
--- a/drivers/gpio/gpio-bt8xx.c
+++ b/drivers/gpio/gpio-bt8xx.c
@@ -46,7 +46,7 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/spinlock.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/slab.h>
 
 /* Steal the hardware definitions from the bttv driver. */
diff --git a/drivers/gpio/gpio-crystalcove.c b/drivers/gpio/gpio-crystalcove.c
index b6f0f729656c..58531d8b8c6e 100644
--- a/drivers/gpio/gpio-crystalcove.c
+++ b/drivers/gpio/gpio-crystalcove.c
@@ -18,7 +18,7 @@
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/seq_file.h>
 #include <linux/bitops.h>
 #include <linux/regmap.h>
diff --git a/drivers/gpio/gpio-cs5535.c b/drivers/gpio/gpio-cs5535.c
index 90278b19aa0e..8814c8f47e57 100644
--- a/drivers/gpio/gpio-cs5535.c
+++ b/drivers/gpio/gpio-cs5535.c
@@ -12,7 +12,7 @@
 #include <linux/spinlock.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/io.h>
 #include <linux/cs5535.h>
 #include <asm/msr.h>
diff --git a/drivers/gpio/gpio-da9052.c b/drivers/gpio/gpio-da9052.c
index dd8977cf3e85..b6d3e997eb26 100644
--- a/drivers/gpio/gpio-da9052.c
+++ b/drivers/gpio/gpio-da9052.c
@@ -15,7 +15,7 @@
 #include <linux/fs.h>
 #include <linux/uaccess.h>
 #include <linux/platform_device.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/syscalls.h>
 #include <linux/seq_file.h>
 
diff --git a/drivers/gpio/gpio-da9055.c b/drivers/gpio/gpio-da9055.c
index 82053b52cba0..2f1b5d23b10c 100644
--- a/drivers/gpio/gpio-da9055.c
+++ b/drivers/gpio/gpio-da9055.c
@@ -13,7 +13,7 @@
  */
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 
 #include <linux/mfd/da9055/core.h>
 #include <linux/mfd/da9055/reg.h>
diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c
index e4b3d7db68c9..0b951ca78ec4 100644
--- a/drivers/gpio/gpio-davinci.c
+++ b/drivers/gpio/gpio-davinci.c
@@ -9,7 +9,7 @@
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  */
-#include <linux/gpio.h>
+#include <linux/gpio/driver.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/clk.h>
diff --git a/drivers/gpio/gpio-ftgpio010.c b/drivers/gpio/gpio-ftgpio010.c
index 7b3394fdc624..b7a3a2db699b 100644
--- a/drivers/gpio/gpio-ftgpio010.c
+++ b/drivers/gpio/gpio-ftgpio010.c
@@ -176,8 +176,8 @@ static int ftgpio_gpio_probe(struct platform_device *pdev)
 		return PTR_ERR(g->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (!irq)
-		return -EINVAL;
+	if (irq <= 0)
+		return irq ? irq : -EINVAL;
 
 	ret = bgpio_init(&g->gc, dev, 4,
 			 g->base + GPIO_DATA_IN,
diff --git a/drivers/gpio/gpio-iop.c b/drivers/gpio/gpio-iop.c
index 98c7ff2a76e7..8d62db447ec1 100644
--- a/drivers/gpio/gpio-iop.c
+++ b/drivers/gpio/gpio-iop.c
@@ -58,3 +58,7 @@ static int __init iop3xx_gpio_init(void)
 	return platform_driver_register(&iop3xx_gpio_driver);
 }
 arch_initcall(iop3xx_gpio_init);
+
+MODULE_DESCRIPTION("GPIO handling for Intel IOP3xx processors");
+MODULE_AUTHOR("Lennert Buytenhek <buytenh@wantstofly.org>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-it87.c b/drivers/gpio/gpio-it87.c
index d43d0a2cc4c5..efb46edff81f 100644
--- a/drivers/gpio/gpio-it87.c
+++ b/drivers/gpio/gpio-it87.c
@@ -414,6 +414,6 @@ static void __exit it87_gpio_exit(void)
 module_init(it87_gpio_init);
 module_exit(it87_gpio_exit);
 
-MODULE_AUTHOR("Diego Elio PettenÃ² <flameeyes@flameeyes.eu>");
+MODULE_AUTHOR("Diego Elio Pettenò <flameeyes@flameeyes.eu>");
 MODULE_DESCRIPTION("GPIO interface for IT87xx Super I/O chips");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c
index c04fae1ba32a..9d8bcc69f245 100644
--- a/drivers/gpio/gpio-max732x.c
+++ b/drivers/gpio/gpio-max732x.c
@@ -709,8 +709,7 @@ static int max732x_probe(struct i2c_client *client,
 	return 0;
 
 out_failed:
-	if (chip->client_dummy)
-		i2c_unregister_device(chip->client_dummy);
+	i2c_unregister_device(chip->client_dummy);
 	return ret;
 }
 
@@ -734,8 +733,7 @@ static int max732x_remove(struct i2c_client *client)
 	gpiochip_remove(&chip->gpio_chip);
 
 	/* unregister any dummy i2c_client */
-	if (chip->client_dummy)
-		i2c_unregister_device(chip->client_dummy);
+	i2c_unregister_device(chip->client_dummy);
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c
index dd67a31ac337..c38624ea0251 100644
--- a/drivers/gpio/gpio-merrifield.c
+++ b/drivers/gpio/gpio-merrifield.c
@@ -9,6 +9,7 @@
  * published by the Free Software Foundation.
  */
 
+#include <linux/acpi.h>
 #include <linux/bitops.h>
 #include <linux/gpio/driver.h>
 #include <linux/init.h>
@@ -380,9 +381,16 @@ static void mrfld_irq_init_hw(struct mrfld_gpio *priv)
 	}
 }
 
+static const char *mrfld_gpio_get_pinctrl_dev_name(void)
+{
+	const char *dev_name = acpi_dev_get_first_match_name("INTC1002", NULL, -1);
+	return dev_name ? dev_name : "pinctrl-merrifield";
+}
+
 static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	const struct mrfld_gpio_pinrange *range;
+	const char *pinctrl_dev_name;
 	struct mrfld_gpio *priv;
 	u32 gpio_base, irq_base;
 	void __iomem *base;
@@ -439,10 +447,11 @@ static int mrfld_gpio_probe(struct pci_dev *pdev, const struct pci_device_id *id
 		return retval;
 	}
 
+	pinctrl_dev_name = mrfld_gpio_get_pinctrl_dev_name();
 	for (i = 0; i < ARRAY_SIZE(mrfld_gpio_ranges); i++) {
 		range = &mrfld_gpio_ranges[i];
 		retval = gpiochip_add_pin_range(&priv->chip,
-						"pinctrl-merrifield",
+						pinctrl_dev_name,
 						range->gpio_base,
 						range->pin_base,
 						range->npins);
diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c
index 9532d86a82f7..3a545ad17817 100644
--- a/drivers/gpio/gpio-mockup.c
+++ b/drivers/gpio/gpio-mockup.c
@@ -27,13 +27,15 @@
 #include "gpiolib.h"
 
 #define GPIO_MOCKUP_NAME	"gpio-mockup"
-#define	GPIO_MOCKUP_MAX_GC	10
+#define GPIO_MOCKUP_MAX_GC	10
 /*
  * We're storing two values per chip: the GPIO base and the number
  * of GPIO lines.
  */
 #define GPIO_MOCKUP_MAX_RANGES	(GPIO_MOCKUP_MAX_GC * 2)
 
+#define gpio_mockup_err(...)	pr_err(GPIO_MOCKUP_NAME ": " __VA_ARGS__)
+
 enum {
 	GPIO_MOCKUP_DIR_OUT = 0,
 	GPIO_MOCKUP_DIR_IN = 1,
@@ -46,7 +48,7 @@ enum {
  */
 struct gpio_mockup_line_status {
 	int dir;
-	bool value;
+	int value;
 };
 
 struct gpio_mockup_chip {
@@ -62,17 +64,33 @@ struct gpio_mockup_dbgfs_private {
 	int offset;
 };
 
+struct gpio_mockup_platform_data {
+	int base;
+	int ngpio;
+	int index;
+	bool named_lines;
+};
+
 static int gpio_mockup_ranges[GPIO_MOCKUP_MAX_RANGES];
-static int gpio_mockup_params_nr;
-module_param_array(gpio_mockup_ranges, int, &gpio_mockup_params_nr, 0400);
+static int gpio_mockup_num_ranges;
+module_param_array(gpio_mockup_ranges, int, &gpio_mockup_num_ranges, 0400);
 
 static bool gpio_mockup_named_lines;
 module_param_named(gpio_mockup_named_lines,
 		   gpio_mockup_named_lines, bool, 0400);
 
-static const char gpio_mockup_name_start = 'A';
 static struct dentry *gpio_mockup_dbg_dir;
 
+static int gpio_mockup_range_base(unsigned int index)
+{
+	return gpio_mockup_ranges[index * 2];
+}
+
+static int gpio_mockup_range_ngpio(unsigned int index)
+{
+	return gpio_mockup_ranges[index * 2 + 1];
+}
+
 static int gpio_mockup_get(struct gpio_chip *gc, unsigned int offset)
 {
 	struct gpio_mockup_chip *chip = gpiochip_get_data(gc);
@@ -80,16 +98,26 @@ static int gpio_mockup_get(struct gpio_chip *gc, unsigned int offset)
 	return chip->lines[offset].value;
 }
 
-static void gpio_mockup_set(struct gpio_chip *gc, unsigned int offset,
-			    int value)
+static void gpio_mockup_set(struct gpio_chip *gc,
+			    unsigned int offset, int value)
 {
 	struct gpio_mockup_chip *chip = gpiochip_get_data(gc);
 
 	chip->lines[offset].value = !!value;
 }
 
-static int gpio_mockup_dirout(struct gpio_chip *gc, unsigned int offset,
-			      int value)
+static void gpio_mockup_set_multiple(struct gpio_chip *gc,
+				     unsigned long *mask, unsigned long *bits)
+{
+	unsigned int bit;
+
+	for_each_set_bit(bit, mask, gc->ngpio)
+		gpio_mockup_set(gc, bit, test_bit(bit, bits));
+
+}
+
+static int gpio_mockup_dirout(struct gpio_chip *gc,
+			      unsigned int offset, int value)
 {
 	struct gpio_mockup_chip *chip = gpiochip_get_data(gc);
 
@@ -115,29 +143,6 @@ static int gpio_mockup_get_direction(struct gpio_chip *gc, unsigned int offset)
 	return chip->lines[offset].dir;
 }
 
-static int gpio_mockup_name_lines(struct device *dev,
-				  struct gpio_mockup_chip *chip)
-{
-	struct gpio_chip *gc = &chip->gc;
-	char **names;
-	int i;
-
-	names = devm_kcalloc(dev, gc->ngpio, sizeof(char *), GFP_KERNEL);
-	if (!names)
-		return -ENOMEM;
-
-	for (i = 0; i < gc->ngpio; i++) {
-		names[i] = devm_kasprintf(dev, GFP_KERNEL,
-					  "%s-%d", gc->label, i);
-		if (!names[i])
-			return -ENOMEM;
-	}
-
-	gc->names = (const char *const *)names;
-
-	return 0;
-}
-
 static int gpio_mockup_to_irq(struct gpio_chip *gc, unsigned int offset)
 {
 	struct gpio_mockup_chip *chip = gpiochip_get_data(gc);
@@ -188,15 +193,21 @@ static void gpio_mockup_debugfs_setup(struct device *dev,
 				      struct gpio_mockup_chip *chip)
 {
 	struct gpio_mockup_dbgfs_private *priv;
-	struct dentry *evfile;
+	struct dentry *evfile, *link;
 	struct gpio_chip *gc;
+	const char *devname;
 	char *name;
 	int i;
 
 	gc = &chip->gc;
+	devname = dev_name(&gc->gpiodev->dev);
 
-	chip->dbg_dir = debugfs_create_dir(gc->label, gpio_mockup_dbg_dir);
-	if (!chip->dbg_dir)
+	chip->dbg_dir = debugfs_create_dir(devname, gpio_mockup_dbg_dir);
+	if (IS_ERR_OR_NULL(chip->dbg_dir))
+		goto err;
+
+	link = debugfs_create_symlink(gc->label, gpio_mockup_dbg_dir, devname);
+	if (IS_ERR_OR_NULL(link))
 		goto err;
 
 	for (i = 0; i < gc->ngpio; i++) {
@@ -214,23 +225,63 @@ static void gpio_mockup_debugfs_setup(struct device *dev,
 
 		evfile = debugfs_create_file(name, 0200, chip->dbg_dir, priv,
 					     &gpio_mockup_event_ops);
-		if (!evfile)
+		if (IS_ERR_OR_NULL(evfile))
 			goto err;
 	}
 
 	return;
 
 err:
-	dev_err(dev, "error creating debugfs directory\n");
+	dev_err(dev, "error creating debugfs event files\n");
 }
 
-static int gpio_mockup_add(struct device *dev,
-			   struct gpio_mockup_chip *chip,
-			   const char *name, int base, int ngpio)
+static int gpio_mockup_name_lines(struct device *dev,
+				  struct gpio_mockup_chip *chip)
 {
 	struct gpio_chip *gc = &chip->gc;
-	int ret;
+	char **names;
+	int i;
+
+	names = devm_kcalloc(dev, gc->ngpio, sizeof(char *), GFP_KERNEL);
+	if (!names)
+		return -ENOMEM;
+
+	for (i = 0; i < gc->ngpio; i++) {
+		names[i] = devm_kasprintf(dev, GFP_KERNEL,
+					  "%s-%d", gc->label, i);
+		if (!names[i])
+			return -ENOMEM;
+	}
+
+	gc->names = (const char *const *)names;
+
+	return 0;
+}
+
+static int gpio_mockup_probe(struct platform_device *pdev)
+{
+	struct gpio_mockup_platform_data *pdata;
+	struct gpio_mockup_chip *chip;
+	struct gpio_chip *gc;
+	int rv, base, ngpio;
+	struct device *dev;
+	char *name;
+
+	dev = &pdev->dev;
+	pdata = dev_get_platdata(dev);
+	base = pdata->base;
+	ngpio = pdata->ngpio;
 
+	chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL);
+	if (!chip)
+		return -ENOMEM;
+
+	name = devm_kasprintf(dev, GFP_KERNEL, "%s-%c",
+			      pdev->name, pdata->index);
+	if (!name)
+		return -ENOMEM;
+
+	gc = &chip->gc;
 	gc->base = base;
 	gc->ngpio = ngpio;
 	gc->label = name;
@@ -238,6 +289,7 @@ static int gpio_mockup_add(struct device *dev,
 	gc->parent = dev;
 	gc->get = gpio_mockup_get;
 	gc->set = gpio_mockup_set;
+	gc->set_multiple = gpio_mockup_set_multiple;
 	gc->direction_output = gpio_mockup_dirout;
 	gc->direction_input = gpio_mockup_dirin;
 	gc->get_direction = gpio_mockup_get_direction;
@@ -248,19 +300,19 @@ static int gpio_mockup_add(struct device *dev,
 	if (!chip->lines)
 		return -ENOMEM;
 
-	if (gpio_mockup_named_lines) {
-		ret = gpio_mockup_name_lines(dev, chip);
-		if (ret)
-			return ret;
+	if (pdata->named_lines) {
+		rv = gpio_mockup_name_lines(dev, chip);
+		if (rv)
+			return rv;
 	}
 
-	ret = devm_irq_sim_init(dev, &chip->irqsim, gc->ngpio);
-	if (ret)
-		return ret;
+	rv = devm_irq_sim_init(dev, &chip->irqsim, gc->ngpio);
+	if (rv < 0)
+		return rv;
 
-	ret = devm_gpiochip_add_data(dev, &chip->gc, chip);
-	if (ret)
-		return ret;
+	rv = devm_gpiochip_add_data(dev, &chip->gc, chip);
+	if (rv)
+		return rv;
 
 	if (gpio_mockup_dbg_dir)
 		gpio_mockup_debugfs_setup(dev, chip);
@@ -268,58 +320,6 @@ static int gpio_mockup_add(struct device *dev,
 	return 0;
 }
 
-static int gpio_mockup_probe(struct platform_device *pdev)
-{
-	int ret, i, base, ngpio, num_chips;
-	struct device *dev = &pdev->dev;
-	struct gpio_mockup_chip *chips;
-	char *chip_name;
-
-	if (gpio_mockup_params_nr < 2 || (gpio_mockup_params_nr % 2))
-		return -EINVAL;
-
-	/* Each chip is described by two values. */
-	num_chips = gpio_mockup_params_nr / 2;
-
-	chips = devm_kcalloc(dev, num_chips, sizeof(*chips), GFP_KERNEL);
-	if (!chips)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, chips);
-
-	for (i = 0; i < num_chips; i++) {
-		base = gpio_mockup_ranges[i * 2];
-
-		if (base == -1)
-			ngpio = gpio_mockup_ranges[i * 2 + 1];
-		else
-			ngpio = gpio_mockup_ranges[i * 2 + 1] - base;
-
-		if (ngpio >= 0) {
-			chip_name = devm_kasprintf(dev, GFP_KERNEL,
-						   "%s-%c", GPIO_MOCKUP_NAME,
-						   gpio_mockup_name_start + i);
-			if (!chip_name)
-				return -ENOMEM;
-
-			ret = gpio_mockup_add(dev, &chips[i],
-					      chip_name, base, ngpio);
-		} else {
-			ret = -EINVAL;
-		}
-
-		if (ret) {
-			dev_err(dev,
-				"adding gpiochip failed: %d (base: %d, ngpio: %d)\n",
-				ret, base, base < 0 ? ngpio : base + ngpio);
-
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
 static struct platform_driver gpio_mockup_driver = {
 	.driver = {
 		.name = GPIO_MOCKUP_NAME,
@@ -327,44 +327,88 @@ static struct platform_driver gpio_mockup_driver = {
 	.probe = gpio_mockup_probe,
 };
 
-static struct platform_device *pdev;
-static int __init mock_device_init(void)
+static struct platform_device *gpio_mockup_pdevs[GPIO_MOCKUP_MAX_GC];
+
+static void gpio_mockup_unregister_pdevs(void)
 {
-	int err;
+	struct platform_device *pdev;
+	int i;
 
-	gpio_mockup_dbg_dir = debugfs_create_dir("gpio-mockup-event", NULL);
-	if (!gpio_mockup_dbg_dir)
-		pr_err("%s: error creating debugfs directory\n",
-		       GPIO_MOCKUP_NAME);
+	for (i = 0; i < GPIO_MOCKUP_MAX_GC; i++) {
+		pdev = gpio_mockup_pdevs[i];
 
-	pdev = platform_device_alloc(GPIO_MOCKUP_NAME, -1);
-	if (!pdev)
-		return -ENOMEM;
+		if (pdev)
+			platform_device_unregister(pdev);
+	}
+}
 
-	err = platform_device_add(pdev);
-	if (err) {
-		platform_device_put(pdev);
-		return err;
+static int __init gpio_mockup_init(void)
+{
+	int i, num_chips, err = 0, index = 'A';
+	struct gpio_mockup_platform_data pdata;
+	struct platform_device *pdev;
+
+	if ((gpio_mockup_num_ranges < 2) ||
+	    (gpio_mockup_num_ranges % 2) ||
+	    (gpio_mockup_num_ranges > GPIO_MOCKUP_MAX_RANGES))
+		return -EINVAL;
+
+	/* Each chip is described by two values. */
+	num_chips = gpio_mockup_num_ranges / 2;
+
+	/*
+	 * The second value in the <base GPIO - number of GPIOS> pair must
+	 * always be greater than 0.
+	 */
+	for (i = 0; i < num_chips; i++) {
+		if (gpio_mockup_range_ngpio(i) < 0)
+			return -EINVAL;
 	}
 
+	gpio_mockup_dbg_dir = debugfs_create_dir("gpio-mockup-event", NULL);
+	if (IS_ERR_OR_NULL(gpio_mockup_dbg_dir))
+		gpio_mockup_err("error creating debugfs directory\n");
+
 	err = platform_driver_register(&gpio_mockup_driver);
 	if (err) {
-		platform_device_unregister(pdev);
+		gpio_mockup_err("error registering platform driver\n");
 		return err;
 	}
 
+	for (i = 0; i < num_chips; i++) {
+		pdata.index = index++;
+		pdata.base = gpio_mockup_range_base(i);
+		pdata.ngpio = pdata.base < 0
+				? gpio_mockup_range_ngpio(i)
+				: gpio_mockup_range_ngpio(i) - pdata.base;
+		pdata.named_lines = gpio_mockup_named_lines;
+
+		pdev = platform_device_register_resndata(NULL,
+							 GPIO_MOCKUP_NAME,
+							 i, NULL, 0, &pdata,
+							 sizeof(pdata));
+		if (IS_ERR(pdev)) {
+			gpio_mockup_err("error registering device");
+			platform_driver_unregister(&gpio_mockup_driver);
+			gpio_mockup_unregister_pdevs();
+			return PTR_ERR(pdev);
+		}
+
+		gpio_mockup_pdevs[i] = pdev;
+	}
+
 	return 0;
 }
 
-static void __exit mock_device_exit(void)
+static void __exit gpio_mockup_exit(void)
 {
 	debugfs_remove_recursive(gpio_mockup_dbg_dir);
 	platform_driver_unregister(&gpio_mockup_driver);
-	platform_device_unregister(pdev);
+	gpio_mockup_unregister_pdevs();
 }
 
-module_init(mock_device_init);
-module_exit(mock_device_exit);
+module_init(gpio_mockup_init);
+module_exit(gpio_mockup_exit);
 
 MODULE_AUTHOR("Kamlakant Patel <kamlakant.patel@broadcom.com>");
 MODULE_AUTHOR("Bamvor Jian Zhang <bamvor.zhangjian@linaro.org>");
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index e136d666f1e5..ab5035b96886 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -1058,7 +1058,9 @@ static void omap_gpio_mod_init(struct gpio_bank *bank)
 
 static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
 {
+	struct gpio_irq_chip *irq;
 	static int gpio;
+	const char *label;
 	int irq_base = 0;
 	int ret;
 
@@ -1080,21 +1082,15 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
 			bank->chip.parent = &omap_mpuio_device.dev;
 		bank->chip.base = OMAP_MPUIO(0);
 	} else {
-		bank->chip.label = "gpio";
+		label = devm_kasprintf(bank->chip.parent, GFP_KERNEL, "gpio-%d-%d",
+				       gpio, gpio + bank->width - 1);
+		if (!label)
+			return -ENOMEM;
+		bank->chip.label = label;
 		bank->chip.base = gpio;
 	}
 	bank->chip.ngpio = bank->width;
 
-	ret = gpiochip_add_data(&bank->chip, bank);
-	if (ret) {
-		dev_err(bank->chip.parent,
-			"Could not register gpio chip %d\n", ret);
-		return ret;
-	}
-
-	if (!bank->is_mpuio)
-		gpio += bank->width;
-
 #ifdef CONFIG_ARCH_OMAP1
 	/*
 	 * REVISIT: Once we have OMAP1 supporting SPARSE_IRQ, we can drop
@@ -1115,25 +1111,30 @@ static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc)
 			irqc->irq_set_wake = NULL;
 	}
 
-	ret = gpiochip_irqchip_add(&bank->chip, irqc,
-				   irq_base, handle_bad_irq,
-				   IRQ_TYPE_NONE);
+	irq = &bank->chip.irq;
+	irq->chip = irqc;
+	irq->handler = handle_bad_irq;
+	irq->default_type = IRQ_TYPE_NONE;
+	irq->num_parents = 1;
+	irq->parents = &bank->irq;
+	irq->first = irq_base;
 
+	ret = gpiochip_add_data(&bank->chip, bank);
 	if (ret) {
 		dev_err(bank->chip.parent,
-			"Couldn't add irqchip to gpiochip %d\n", ret);
-		gpiochip_remove(&bank->chip);
-		return -ENODEV;
+			"Could not register gpio chip %d\n", ret);
+		return ret;
 	}
 
-	gpiochip_set_chained_irqchip(&bank->chip, irqc, bank->irq, NULL);
-
 	ret = devm_request_irq(bank->chip.parent, bank->irq,
 			       omap_gpio_irq_handler,
 			       0, dev_name(bank->chip.parent), bank);
 	if (ret)
 		gpiochip_remove(&bank->chip);
 
+	if (!bank->is_mpuio)
+		gpio += bank->width;
+
 	return ret;
 }
 
diff --git a/drivers/gpio/gpio-pcie-idio-24.c b/drivers/gpio/gpio-pcie-idio-24.c
new file mode 100644
index 000000000000..f666e2e69074
--- /dev/null
+++ b/drivers/gpio/gpio-pcie-idio-24.c
@@ -0,0 +1,447 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * GPIO driver for the ACCES PCIe-IDIO-24 family
+ * Copyright (C) 2018 William Breathitt Gray
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * This driver supports the following ACCES devices: PCIe-IDIO-24,
+ * PCIe-IDI-24, PCIe-IDO-24, and PCIe-IDIO-12.
+ */
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/gpio/driver.h>
+#include <linux/interrupt.h>
+#include <linux/irqdesc.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+/**
+ * struct idio_24_gpio_reg - GPIO device registers structure
+ * @out0_7:	Read: FET Outputs 0-7
+ *		Write: FET Outputs 0-7
+ * @out8_15:	Read: FET Outputs 8-15
+ *		Write: FET Outputs 8-15
+ * @out16_23:	Read: FET Outputs 16-23
+ *		Write: FET Outputs 16-23
+ * @ttl_out0_7:	Read: TTL/CMOS Outputs 0-7
+ *		Write: TTL/CMOS Outputs 0-7
+ * @in0_7:	Read: Isolated Inputs 0-7
+ *		Write: Reserved
+ * @in8_15:	Read: Isolated Inputs 8-15
+ *		Write: Reserved
+ * @in16_23:	Read: Isolated Inputs 16-23
+ *		Write: Reserved
+ * @ttl_in0_7:	Read: TTL/CMOS Inputs 0-7
+ *		Write: Reserved
+ * @cos0_7:	Read: COS Status Inputs 0-7
+ *		Write: COS Clear Inputs 0-7
+ * @cos8_15:	Read: COS Status Inputs 8-15
+ *		Write: COS Clear Inputs 8-15
+ * @cos16_23:	Read: COS Status Inputs 16-23
+ *		Write: COS Clear Inputs 16-23
+ * @cos_ttl0_7:	Read: COS Status TTL/CMOS 0-7
+ *		Write: COS Clear TTL/CMOS 0-7
+ * @ctl:	Read: Control Register
+ *		Write: Control Register
+ * @reserved:	Read: Reserved
+ *		Write: Reserved
+ * @cos_enable:	Read: COS Enable
+ *		Write: COS Enable
+ * @soft_reset:	Read: IRQ Output Pin Status
+ *		Write: Software Board Reset
+ */
+struct idio_24_gpio_reg {
+	u8 out0_7;
+	u8 out8_15;
+	u8 out16_23;
+	u8 ttl_out0_7;
+	u8 in0_7;
+	u8 in8_15;
+	u8 in16_23;
+	u8 ttl_in0_7;
+	u8 cos0_7;
+	u8 cos8_15;
+	u8 cos16_23;
+	u8 cos_ttl0_7;
+	u8 ctl;
+	u8 reserved;
+	u8 cos_enable;
+	u8 soft_reset;
+};
+
+/**
+ * struct idio_24_gpio - GPIO device private data structure
+ * @chip:	instance of the gpio_chip
+ * @lock:	synchronization lock to prevent I/O race conditions
+ * @reg:	I/O address offset for the GPIO device registers
+ * @irq_mask:	I/O bits affected by interrupts
+ */
+struct idio_24_gpio {
+	struct gpio_chip chip;
+	raw_spinlock_t lock;
+	struct idio_24_gpio_reg __iomem *reg;
+	unsigned long irq_mask;
+};
+
+static int idio_24_gpio_get_direction(struct gpio_chip *chip,
+	unsigned int offset)
+{
+	struct idio_24_gpio *const idio24gpio = gpiochip_get_data(chip);
+	const unsigned long out_mode_mask = BIT(1);
+
+	/* FET Outputs */
+	if (offset < 24)
+		return 0;
+
+	/* Isolated Inputs */
+	if (offset < 48)
+		return 1;
+
+	/* TTL/CMOS I/O */
+	/* OUT MODE = 1 when TTL/CMOS Output Mode is set */
+	return !(ioread8(&idio24gpio->reg->ctl) & out_mode_mask);
+}
+
+static int idio_24_gpio_direction_input(struct gpio_chip *chip,
+	unsigned int offset)
+{
+	struct idio_24_gpio *const idio24gpio = gpiochip_get_data(chip);
+	unsigned long flags;
+	unsigned int ctl_state;
+	const unsigned long out_mode_mask = BIT(1);
+
+	/* TTL/CMOS I/O */
+	if (offset > 47) {
+		raw_spin_lock_irqsave(&idio24gpio->lock, flags);
+
+		/* Clear TTL/CMOS Output Mode */
+		ctl_state = ioread8(&idio24gpio->reg->ctl) & ~out_mode_mask;
+		iowrite8(ctl_state, &idio24gpio->reg->ctl);
+
+		raw_spin_unlock_irqrestore(&idio24gpio->lock, flags);
+	}
+
+	return 0;
+}
+
+static int idio_24_gpio_direction_output(struct gpio_chip *chip,
+	unsigned int offset, int value)
+{
+	struct idio_24_gpio *const idio24gpio = gpiochip_get_data(chip);
+	unsigned long flags;
+	unsigned int ctl_state;
+	const unsigned long out_mode_mask = BIT(1);
+
+	/* TTL/CMOS I/O */
+	if (offset > 47) {
+		raw_spin_lock_irqsave(&idio24gpio->lock, flags);
+
+		/* Set TTL/CMOS Output Mode */
+		ctl_state = ioread8(&idio24gpio->reg->ctl) | out_mode_mask;
+		iowrite8(ctl_state, &idio24gpio->reg->ctl);
+
+		raw_spin_unlock_irqrestore(&idio24gpio->lock, flags);
+	}
+
+	chip->set(chip, offset, value);
+	return 0;
+}
+
+static int idio_24_gpio_get(struct gpio_chip *chip, unsigned int offset)
+{
+	struct idio_24_gpio *const idio24gpio = gpiochip_get_data(chip);
+	const unsigned long offset_mask = BIT(offset % 8);
+	const unsigned long out_mode_mask = BIT(1);
+
+	/* FET Outputs */
+	if (offset < 8)
+		return !!(ioread8(&idio24gpio->reg->out0_7) & offset_mask);
+
+	if (offset < 16)
+		return !!(ioread8(&idio24gpio->reg->out8_15) & offset_mask);
+
+	if (offset < 24)
+		return !!(ioread8(&idio24gpio->reg->out16_23) & offset_mask);
+
+	/* Isolated Inputs */
+	if (offset < 32)
+		return !!(ioread8(&idio24gpio->reg->in0_7) & offset_mask);
+
+	if (offset < 40)
+		return !!(ioread8(&idio24gpio->reg->in8_15) & offset_mask);
+
+	if (offset < 48)
+		return !!(ioread8(&idio24gpio->reg->in16_23) & offset_mask);
+
+	/* TTL/CMOS Outputs */
+	if (ioread8(&idio24gpio->reg->ctl) & out_mode_mask)
+		return !!(ioread8(&idio24gpio->reg->ttl_out0_7) & offset_mask);
+
+	/* TTL/CMOS Inputs */
+	return !!(ioread8(&idio24gpio->reg->ttl_in0_7) & offset_mask);
+}
+
+static void idio_24_gpio_set(struct gpio_chip *chip, unsigned int offset,
+	int value)
+{
+	struct idio_24_gpio *const idio24gpio = gpiochip_get_data(chip);
+	const unsigned long out_mode_mask = BIT(1);
+	void __iomem *base;
+	const unsigned int mask = BIT(offset % 8);
+	unsigned long flags;
+	unsigned int out_state;
+
+	/* Isolated Inputs */
+	if (offset > 23 && offset < 48)
+		return;
+
+	/* TTL/CMOS Inputs */
+	if (offset > 47 && !(ioread8(&idio24gpio->reg->ctl) & out_mode_mask))
+		return;
+
+	/* TTL/CMOS Outputs */
+	if (offset > 47)
+		base = &idio24gpio->reg->ttl_out0_7;
+	/* FET Outputs */
+	else if (offset > 15)
+		base = &idio24gpio->reg->out16_23;
+	else if (offset > 7)
+		base = &idio24gpio->reg->out8_15;
+	else
+		base = &idio24gpio->reg->out0_7;
+
+	raw_spin_lock_irqsave(&idio24gpio->lock, flags);
+
+	if (value)
+		out_state = ioread8(base) | mask;
+	else
+		out_state = ioread8(base) & ~mask;
+
+	iowrite8(out_state, base);
+
+	raw_spin_unlock_irqrestore(&idio24gpio->lock, flags);
+}
+
+static void idio_24_irq_ack(struct irq_data *data)
+{
+}
+
+static void idio_24_irq_mask(struct irq_data *data)
+{
+	struct gpio_chip *const chip = irq_data_get_irq_chip_data(data);
+	struct idio_24_gpio *const idio24gpio = gpiochip_get_data(chip);
+	unsigned long flags;
+	const unsigned long bit_offset = irqd_to_hwirq(data) - 24;
+	unsigned char new_irq_mask;
+	const unsigned long bank_offset = bit_offset/8 * 8;
+	unsigned char cos_enable_state;
+
+	raw_spin_lock_irqsave(&idio24gpio->lock, flags);
+
+	idio24gpio->irq_mask &= BIT(bit_offset);
+	new_irq_mask = idio24gpio->irq_mask >> bank_offset;
+
+	if (!new_irq_mask) {
+		cos_enable_state = ioread8(&idio24gpio->reg->cos_enable);
+
+		/* Disable Rising Edge detection */
+		cos_enable_state &= ~BIT(bank_offset);
+		/* Disable Falling Edge detection */
+		cos_enable_state &= ~BIT(bank_offset + 4);
+
+		iowrite8(cos_enable_state, &idio24gpio->reg->cos_enable);
+	}
+
+	raw_spin_unlock_irqrestore(&idio24gpio->lock, flags);
+}
+
+static void idio_24_irq_unmask(struct irq_data *data)
+{
+	struct gpio_chip *const chip = irq_data_get_irq_chip_data(data);
+	struct idio_24_gpio *const idio24gpio = gpiochip_get_data(chip);
+	unsigned long flags;
+	unsigned char prev_irq_mask;
+	const unsigned long bit_offset = irqd_to_hwirq(data) - 24;
+	const unsigned long bank_offset = bit_offset/8 * 8;
+	unsigned char cos_enable_state;
+
+	raw_spin_lock_irqsave(&idio24gpio->lock, flags);
+
+	prev_irq_mask = idio24gpio->irq_mask >> bank_offset;
+	idio24gpio->irq_mask |= BIT(bit_offset);
+
+	if (!prev_irq_mask) {
+		cos_enable_state = ioread8(&idio24gpio->reg->cos_enable);
+
+		/* Enable Rising Edge detection */
+		cos_enable_state |= BIT(bank_offset);
+		/* Enable Falling Edge detection */
+		cos_enable_state |= BIT(bank_offset + 4);
+
+		iowrite8(cos_enable_state, &idio24gpio->reg->cos_enable);
+	}
+
+	raw_spin_unlock_irqrestore(&idio24gpio->lock, flags);
+}
+
+static int idio_24_irq_set_type(struct irq_data *data, unsigned int flow_type)
+{
+	/* The only valid irq types are none and both-edges */
+	if (flow_type != IRQ_TYPE_NONE &&
+		(flow_type & IRQ_TYPE_EDGE_BOTH) != IRQ_TYPE_EDGE_BOTH)
+		return -EINVAL;
+
+	return 0;
+}
+
+static struct irq_chip idio_24_irqchip = {
+	.name = "pcie-idio-24",
+	.irq_ack = idio_24_irq_ack,
+	.irq_mask = idio_24_irq_mask,
+	.irq_unmask = idio_24_irq_unmask,
+	.irq_set_type = idio_24_irq_set_type
+};
+
+static irqreturn_t idio_24_irq_handler(int irq, void *dev_id)
+{
+	struct idio_24_gpio *const idio24gpio = dev_id;
+	unsigned long irq_status;
+	struct gpio_chip *const chip = &idio24gpio->chip;
+	unsigned long irq_mask;
+	int gpio;
+
+	raw_spin_lock(&idio24gpio->lock);
+
+	/* Read Change-Of-State status */
+	irq_status = ioread32(&idio24gpio->reg->cos0_7);
+
+	raw_spin_unlock(&idio24gpio->lock);
+
+	/* Make sure our device generated IRQ */
+	if (!irq_status)
+		return IRQ_NONE;
+
+	/* Handle only unmasked IRQ */
+	irq_mask = idio24gpio->irq_mask & irq_status;
+
+	for_each_set_bit(gpio, &irq_mask, chip->ngpio - 24)
+		generic_handle_irq(irq_find_mapping(chip->irq.domain,
+			gpio + 24));
+
+	raw_spin_lock(&idio24gpio->lock);
+
+	/* Clear Change-Of-State status */
+	iowrite32(irq_status, &idio24gpio->reg->cos0_7);
+
+	raw_spin_unlock(&idio24gpio->lock);
+
+	return IRQ_HANDLED;
+}
+
+#define IDIO_24_NGPIO 56
+static const char *idio_24_names[IDIO_24_NGPIO] = {
+	"OUT0", "OUT1", "OUT2", "OUT3", "OUT4", "OUT5", "OUT6", "OUT7",
+	"OUT8", "OUT9", "OUT10", "OUT11", "OUT12", "OUT13", "OUT14", "OUT15",
+	"OUT16", "OUT17", "OUT18", "OUT19", "OUT20", "OUT21", "OUT22", "OUT23",
+	"IIN0", "IIN1", "IIN2", "IIN3", "IIN4", "IIN5", "IIN6", "IIN7",
+	"IIN8", "IIN9", "IIN10", "IIN11", "IIN12", "IIN13", "IIN14", "IIN15",
+	"IIN16", "IIN17", "IIN18", "IIN19", "IIN20", "IIN21", "IIN22", "IIN23",
+	"TTL0", "TTL1", "TTL2", "TTL3", "TTL4", "TTL5", "TTL6", "TTL7"
+};
+
+static int idio_24_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct device *const dev = &pdev->dev;
+	struct idio_24_gpio *idio24gpio;
+	int err;
+	const size_t pci_bar_index = 2;
+	const char *const name = pci_name(pdev);
+
+	idio24gpio = devm_kzalloc(dev, sizeof(*idio24gpio), GFP_KERNEL);
+	if (!idio24gpio)
+		return -ENOMEM;
+
+	err = pcim_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Failed to enable PCI device (%d)\n", err);
+		return err;
+	}
+
+	err = pcim_iomap_regions(pdev, BIT(pci_bar_index), name);
+	if (err) {
+		dev_err(dev, "Unable to map PCI I/O addresses (%d)\n", err);
+		return err;
+	}
+
+	idio24gpio->reg = pcim_iomap_table(pdev)[pci_bar_index];
+
+	idio24gpio->chip.label = name;
+	idio24gpio->chip.parent = dev;
+	idio24gpio->chip.owner = THIS_MODULE;
+	idio24gpio->chip.base = -1;
+	idio24gpio->chip.ngpio = IDIO_24_NGPIO;
+	idio24gpio->chip.names = idio_24_names;
+	idio24gpio->chip.get_direction = idio_24_gpio_get_direction;
+	idio24gpio->chip.direction_input = idio_24_gpio_direction_input;
+	idio24gpio->chip.direction_output = idio_24_gpio_direction_output;
+	idio24gpio->chip.get = idio_24_gpio_get;
+	idio24gpio->chip.set = idio_24_gpio_set;
+
+	raw_spin_lock_init(&idio24gpio->lock);
+
+	/* Software board reset */
+	iowrite8(0, &idio24gpio->reg->soft_reset);
+
+	err = devm_gpiochip_add_data(dev, &idio24gpio->chip, idio24gpio);
+	if (err) {
+		dev_err(dev, "GPIO registering failed (%d)\n", err);
+		return err;
+	}
+
+	err = gpiochip_irqchip_add(&idio24gpio->chip, &idio_24_irqchip, 0,
+		handle_edge_irq, IRQ_TYPE_NONE);
+	if (err) {
+		dev_err(dev, "Could not add irqchip (%d)\n", err);
+		return err;
+	}
+
+	err = devm_request_irq(dev, pdev->irq, idio_24_irq_handler, IRQF_SHARED,
+		name, idio24gpio);
+	if (err) {
+		dev_err(dev, "IRQ handler registering failed (%d)\n", err);
+		return err;
+	}
+
+	return 0;
+}
+
+static const struct pci_device_id idio_24_pci_dev_id[] = {
+	{ PCI_DEVICE(0x494F, 0x0FD0) }, { PCI_DEVICE(0x494F, 0x0BD0) },
+	{ PCI_DEVICE(0x494F, 0x07D0) }, { PCI_DEVICE(0x494F, 0x0FC0) },
+	{ 0 }
+};
+MODULE_DEVICE_TABLE(pci, idio_24_pci_dev_id);
+
+static struct pci_driver idio_24_driver = {
+	.name = "pcie-idio-24",
+	.id_table = idio_24_pci_dev_id,
+	.probe = idio_24_probe
+};
+
+module_pci_driver(idio_24_driver);
+
+MODULE_AUTHOR("William Breathitt Gray <vilhelm.gray@gmail.com>");
+MODULE_DESCRIPTION("ACCES PCIe-IDIO-24 GPIO driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c
index e76de57dd617..ebaea8b1594b 100644
--- a/drivers/gpio/gpio-rcar.c
+++ b/drivers/gpio/gpio-rcar.c
@@ -14,7 +14,6 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/clk.h>
 #include <linux/err.h>
 #include <linux/gpio.h>
 #include <linux/init.h>
@@ -37,10 +36,9 @@ struct gpio_rcar_priv {
 	struct platform_device *pdev;
 	struct gpio_chip gpio_chip;
 	struct irq_chip irq_chip;
-	struct clk *clk;
 	unsigned int irq_parent;
+	atomic_t wakeup_path;
 	bool has_both_edge_trigger;
-	bool needs_clk;
 };
 
 #define IOINTSEL 0x00	/* General IO/Interrupt Switching Register */
@@ -186,13 +184,10 @@ static int gpio_rcar_irq_set_wake(struct irq_data *d, unsigned int on)
 		}
 	}
 
-	if (!p->clk)
-		return 0;
-
 	if (on)
-		clk_enable(p->clk);
+		atomic_inc(&p->wakeup_path);
 	else
-		clk_disable(p->clk);
+		atomic_dec(&p->wakeup_path);
 
 	return 0;
 }
@@ -330,17 +325,14 @@ static int gpio_rcar_direction_output(struct gpio_chip *chip, unsigned offset,
 
 struct gpio_rcar_info {
 	bool has_both_edge_trigger;
-	bool needs_clk;
 };
 
 static const struct gpio_rcar_info gpio_rcar_info_gen1 = {
 	.has_both_edge_trigger = false,
-	.needs_clk = false,
 };
 
 static const struct gpio_rcar_info gpio_rcar_info_gen2 = {
 	.has_both_edge_trigger = true,
-	.needs_clk = true,
 };
 
 static const struct of_device_id gpio_rcar_of_table[] = {
@@ -403,7 +395,6 @@ static int gpio_rcar_parse_dt(struct gpio_rcar_priv *p, unsigned int *npins)
 	ret = of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, 0, &args);
 	*npins = ret == 0 ? args.args[2] : RCAR_MAX_GPIO_PER_BANK;
 	p->has_both_edge_trigger = info->has_both_edge_trigger;
-	p->needs_clk = info->needs_clk;
 
 	if (*npins == 0 || *npins > RCAR_MAX_GPIO_PER_BANK) {
 		dev_warn(&p->pdev->dev,
@@ -440,16 +431,6 @@ static int gpio_rcar_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, p);
 
-	p->clk = devm_clk_get(dev, NULL);
-	if (IS_ERR(p->clk)) {
-		if (p->needs_clk) {
-			dev_err(dev, "unable to get clock\n");
-			ret = PTR_ERR(p->clk);
-			goto err0;
-		}
-		p->clk = NULL;
-	}
-
 	pm_runtime_enable(dev);
 
 	irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
@@ -531,11 +512,24 @@ static int gpio_rcar_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int __maybe_unused gpio_rcar_suspend(struct device *dev)
+{
+	struct gpio_rcar_priv *p = dev_get_drvdata(dev);
+
+	if (atomic_read(&p->wakeup_path))
+		device_set_wakeup_path(dev);
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(gpio_rcar_pm_ops, gpio_rcar_suspend, NULL);
+
 static struct platform_driver gpio_rcar_device_driver = {
 	.probe		= gpio_rcar_probe,
 	.remove		= gpio_rcar_remove,
 	.driver		= {
 		.name	= "gpio_rcar",
+		.pm     = &gpio_rcar_pm_ops,
 		.of_match_table = of_match_ptr(gpio_rcar_of_table),
 	}
 };
diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c
index e6e5cca624a7..f8d7d1cd8488 100644
--- a/drivers/gpio/gpio-stmpe.c
+++ b/drivers/gpio/gpio-stmpe.c
@@ -190,6 +190,16 @@ static void stmpe_gpio_irq_sync_unlock(struct irq_data *d)
 	};
 	int i, j;
 
+	/*
+	 * STMPE1600: to be able to get IRQ from pins,
+	 * a read must be done on GPMR register, or a write in
+	 * GPSR or GPCR registers
+	 */
+	if (stmpe->partnum == STMPE1600) {
+		stmpe_reg_read(stmpe, stmpe->regs[STMPE_IDX_GPMR_LSB]);
+		stmpe_reg_read(stmpe, stmpe->regs[STMPE_IDX_GPMR_CSB]);
+	}
+
 	for (i = 0; i < CACHE_NR_REGS; i++) {
 		/* STMPE801 and STMPE1600 don't have RE and FE registers */
 		if ((stmpe->partnum == STMPE801 ||
@@ -227,21 +237,11 @@ static void stmpe_gpio_irq_unmask(struct irq_data *d)
 {
 	struct gpio_chip *gc = irq_data_get_irq_chip_data(d);
 	struct stmpe_gpio *stmpe_gpio = gpiochip_get_data(gc);
-	struct stmpe *stmpe = stmpe_gpio->stmpe;
 	int offset = d->hwirq;
 	int regoffset = offset / 8;
 	int mask = BIT(offset % 8);
 
 	stmpe_gpio->regs[REG_IE][regoffset] |= mask;
-
-	/*
-	 * STMPE1600 workaround: to be able to get IRQ from pins,
-	 * a read must be done on GPMR register, or a write in
-	 * GPSR or GPCR registers
-	 */
-	if (stmpe->partnum == STMPE1600)
-		stmpe_reg_read(stmpe,
-			       stmpe->regs[STMPE_IDX_GPMR_LSB + regoffset]);
 }
 
 static void stmpe_dbg_show_one(struct seq_file *s,
@@ -273,15 +273,21 @@ static void stmpe_dbg_show_one(struct seq_file *s,
 		u8 fall_reg;
 		u8 irqen_reg;
 
-		char *edge_det_values[] = {"edge-inactive",
-					   "edge-asserted",
-					   "not-supported"};
-		char *rise_values[] = {"no-rising-edge-detection",
-				       "rising-edge-detection",
-				       "not-supported"};
-		char *fall_values[] = {"no-falling-edge-detection",
-				       "falling-edge-detection",
-				       "not-supported"};
+		static const char * const edge_det_values[] = {
+			"edge-inactive",
+			"edge-asserted",
+			"not-supported"
+		};
+		static const char * const rise_values[] = {
+			"no-rising-edge-detection",
+			"rising-edge-detection",
+			"not-supported"
+		};
+		static const char * const fall_values[] = {
+			"no-falling-edge-detection",
+			"falling-edge-detection",
+			"not-supported"
+		};
 		#define NOT_SUPPORTED_IDX 2
 		u8 edge_det = NOT_SUPPORTED_IDX;
 		u8 rise = NOT_SUPPORTED_IDX;
@@ -344,7 +350,7 @@ static void stmpe_dbg_show(struct seq_file *s, struct gpio_chip *gc)
 
 	for (i = 0; i < gc->ngpio; i++, gpio++) {
 		stmpe_dbg_show_one(s, gc, i, gpio);
-		seq_printf(s, "\n");
+		seq_putc(s, '\n');
 	}
 }
 
@@ -426,12 +432,9 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 	struct stmpe *stmpe = dev_get_drvdata(pdev->dev.parent);
 	struct device_node *np = pdev->dev.of_node;
 	struct stmpe_gpio *stmpe_gpio;
-	int ret;
-	int irq = 0;
-
-	irq = platform_get_irq(pdev, 0);
+	int ret, irq;
 
-	stmpe_gpio = kzalloc(sizeof(struct stmpe_gpio), GFP_KERNEL);
+	stmpe_gpio = kzalloc(sizeof(*stmpe_gpio), GFP_KERNEL);
 	if (!stmpe_gpio)
 		return -ENOMEM;
 
@@ -453,6 +456,7 @@ static int stmpe_gpio_probe(struct platform_device *pdev)
 	if (stmpe_gpio->norequest_mask)
 		stmpe_gpio->chip.irq.need_valid_mask = true;
 
+	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		dev_info(&pdev->dev,
 			"device configured in no-irq mode: "
diff --git a/drivers/gpio/gpio-thunderx.c b/drivers/gpio/gpio-thunderx.c
index b5adb79a631a..d16e9d4a129b 100644
--- a/drivers/gpio/gpio-thunderx.c
+++ b/drivers/gpio/gpio-thunderx.c
@@ -553,8 +553,10 @@ static int thunderx_gpio_probe(struct pci_dev *pdev,
 	txgpio->irqd = irq_domain_create_hierarchy(irq_get_irq_data(txgpio->msix_entries[0].vector)->domain,
 						   0, 0, of_node_to_fwnode(dev->of_node),
 						   &thunderx_gpio_irqd_ops, txgpio);
-	if (!txgpio->irqd)
+	if (!txgpio->irqd) {
+		err = -ENOMEM;
 		goto out;
+	}
 
 	/* Push on irq_data and the domain for each line. */
 	for (i = 0; i < ngpio; i++) {
diff --git a/drivers/gpio/gpio-uniphier.c b/drivers/gpio/gpio-uniphier.c
index 016d7427ebfa..761d8279abca 100644
--- a/drivers/gpio/gpio-uniphier.c
+++ b/drivers/gpio/gpio-uniphier.c
@@ -505,4 +505,4 @@ module_platform_driver(uniphier_gpio_driver);
 
 MODULE_AUTHOR("Masahiro Yamada <yamada.masahiro@socionext.com>");
 MODULE_DESCRIPTION("UniPhier GPIO driver");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpio/gpio-winbond.c b/drivers/gpio/gpio-winbond.c
new file mode 100644
index 000000000000..7f8f5b02e31d
--- /dev/null
+++ b/drivers/gpio/gpio-winbond.c
@@ -0,0 +1,732 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * GPIO interface for Winbond Super I/O chips
+ * Currently, only W83627UHG (Nuvoton NCT6627UD) is supported.
+ *
+ * Author: Maciej S. Szmigiero <mail@maciej.szmigiero.name>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/gpio/driver.h>
+#include <linux/ioport.h>
+#include <linux/isa.h>
+#include <linux/module.h>
+
+#define WB_GPIO_DRIVER_NAME		KBUILD_MODNAME
+
+#define WB_SIO_BASE			0x2e
+#define WB_SIO_BASE_HIGH		0x4e
+
+#define WB_SIO_EXT_ENTER_KEY		0x87
+#define WB_SIO_EXT_EXIT_KEY		0xaa
+
+/* global chip registers */
+
+#define WB_SIO_REG_LOGICAL		0x07
+
+#define WB_SIO_REG_CHIP_MSB		0x20
+#define WB_SIO_REG_CHIP_LSB		0x21
+
+#define WB_SIO_CHIP_ID_W83627UHG	0xa230
+#define WB_SIO_CHIP_ID_W83627UHG_MASK	GENMASK(15, 4)
+
+#define WB_SIO_REG_DPD			0x22
+#define WB_SIO_REG_DPD_UARTA		4
+#define WB_SIO_REG_DPD_UARTB		5
+
+#define WB_SIO_REG_IDPD		0x23
+#define WB_SIO_REG_IDPD_UARTC		4
+#define WB_SIO_REG_IDPD_UARTD		5
+#define WB_SIO_REG_IDPD_UARTE		6
+#define WB_SIO_REG_IDPD_UARTF		7
+
+#define WB_SIO_REG_GLOBAL_OPT		0x24
+#define WB_SIO_REG_GO_ENFDC		1
+
+#define WB_SIO_REG_OVTGPIO3456		0x29
+#define WB_SIO_REG_OG3456_G3PP		3
+#define WB_SIO_REG_OG3456_G4PP		4
+#define WB_SIO_REG_OG3456_G5PP		5
+#define WB_SIO_REG_OG3456_G6PP		7
+
+#define WB_SIO_REG_I2C_PS		0x2a
+#define WB_SIO_REG_I2CPS_I2CFS		1
+
+#define WB_SIO_REG_GPIO1_MF		0x2c
+#define WB_SIO_REG_G1MF_G1PP		6
+#define WB_SIO_REG_G1MF_G2PP		7
+#define WB_SIO_REG_G1MF_FS_MASK	GENMASK(1, 0)
+#define WB_SIO_REG_G1MF_FS_IR_OFF	0
+#define WB_SIO_REG_G1MF_FS_IR		1
+#define WB_SIO_REG_G1MF_FS_GPIO1	2
+#define WB_SIO_REG_G1MF_FS_UARTB	3
+
+/* not an actual device number, just a value meaning 'no device' */
+#define WB_SIO_DEV_NONE		0xff
+
+/* registers with offsets >= 0x30 are specific for a particular device */
+
+/* UART B logical device */
+#define WB_SIO_DEV_UARTB		0x03
+#define WB_SIO_UARTB_REG_ENABLE	0x30
+#define WB_SIO_UARTB_ENABLE_ON		0
+
+/* UART C logical device */
+#define WB_SIO_DEV_UARTC		0x06
+#define WB_SIO_UARTC_REG_ENABLE	0x30
+#define WB_SIO_UARTC_ENABLE_ON		0
+
+/* GPIO3, GPIO4 logical device */
+#define WB_SIO_DEV_GPIO34		0x07
+#define WB_SIO_GPIO34_REG_ENABLE	0x30
+#define WB_SIO_GPIO34_ENABLE_3		0
+#define WB_SIO_GPIO34_ENABLE_4		1
+#define WB_SIO_GPIO34_REG_IO3		0xe0
+#define WB_SIO_GPIO34_REG_DATA3	0xe1
+#define WB_SIO_GPIO34_REG_INV3		0xe2
+#define WB_SIO_GPIO34_REG_IO4		0xe4
+#define WB_SIO_GPIO34_REG_DATA4	0xe5
+#define WB_SIO_GPIO34_REG_INV4		0xe6
+
+/* WDTO, PLED, GPIO5, GPIO6 logical device */
+#define WB_SIO_DEV_WDGPIO56		0x08
+#define WB_SIO_WDGPIO56_REG_ENABLE	0x30
+#define WB_SIO_WDGPIO56_ENABLE_5	1
+#define WB_SIO_WDGPIO56_ENABLE_6	2
+#define WB_SIO_WDGPIO56_REG_IO5	0xe0
+#define WB_SIO_WDGPIO56_REG_DATA5	0xe1
+#define WB_SIO_WDGPIO56_REG_INV5	0xe2
+#define WB_SIO_WDGPIO56_REG_IO6	0xe4
+#define WB_SIO_WDGPIO56_REG_DATA6	0xe5
+#define WB_SIO_WDGPIO56_REG_INV6	0xe6
+
+/* GPIO1, GPIO2, SUSLED logical device */
+#define WB_SIO_DEV_GPIO12		0x09
+#define WB_SIO_GPIO12_REG_ENABLE	0x30
+#define WB_SIO_GPIO12_ENABLE_1		0
+#define WB_SIO_GPIO12_ENABLE_2		1
+#define WB_SIO_GPIO12_REG_IO1		0xe0
+#define WB_SIO_GPIO12_REG_DATA1	0xe1
+#define WB_SIO_GPIO12_REG_INV1		0xe2
+#define WB_SIO_GPIO12_REG_IO2		0xe4
+#define WB_SIO_GPIO12_REG_DATA2	0xe5
+#define WB_SIO_GPIO12_REG_INV2		0xe6
+
+/* UART D logical device */
+#define WB_SIO_DEV_UARTD		0x0d
+#define WB_SIO_UARTD_REG_ENABLE	0x30
+#define WB_SIO_UARTD_ENABLE_ON		0
+
+/* UART E logical device */
+#define WB_SIO_DEV_UARTE		0x0e
+#define WB_SIO_UARTE_REG_ENABLE	0x30
+#define WB_SIO_UARTE_ENABLE_ON		0
+
+/*
+ * for a description what a particular field of this struct means please see
+ * a description of the relevant module parameter at the bottom of this file
+ */
+struct winbond_gpio_params {
+	unsigned long base;
+	unsigned long gpios;
+	unsigned long ppgpios;
+	unsigned long odgpios;
+	bool pledgpio;
+	bool beepgpio;
+	bool i2cgpio;
+};
+
+static struct winbond_gpio_params params;
+
+static int winbond_sio_enter(unsigned long base)
+{
+	if (!request_muxed_region(base, 2, WB_GPIO_DRIVER_NAME))
+		return -EBUSY;
+
+	/*
+	 * datasheet says two successive writes of the "key" value are needed
+	 * in order for chip to enter the "Extended Function Mode"
+	 */
+	outb(WB_SIO_EXT_ENTER_KEY, base);
+	outb(WB_SIO_EXT_ENTER_KEY, base);
+
+	return 0;
+}
+
+static void winbond_sio_select_logical(unsigned long base, u8 dev)
+{
+	outb(WB_SIO_REG_LOGICAL, base);
+	outb(dev, base + 1);
+}
+
+static void winbond_sio_leave(unsigned long base)
+{
+	outb(WB_SIO_EXT_EXIT_KEY, base);
+
+	release_region(base, 2);
+}
+
+static void winbond_sio_reg_write(unsigned long base, u8 reg, u8 data)
+{
+	outb(reg, base);
+	outb(data, base + 1);
+}
+
+static u8 winbond_sio_reg_read(unsigned long base, u8 reg)
+{
+	outb(reg, base);
+	return inb(base + 1);
+}
+
+static void winbond_sio_reg_bset(unsigned long base, u8 reg, u8 bit)
+{
+	u8 val;
+
+	val = winbond_sio_reg_read(base, reg);
+	val |= BIT(bit);
+	winbond_sio_reg_write(base, reg, val);
+}
+
+static void winbond_sio_reg_bclear(unsigned long base, u8 reg, u8 bit)
+{
+	u8 val;
+
+	val = winbond_sio_reg_read(base, reg);
+	val &= ~BIT(bit);
+	winbond_sio_reg_write(base, reg, val);
+}
+
+static bool winbond_sio_reg_btest(unsigned long base, u8 reg, u8 bit)
+{
+	return winbond_sio_reg_read(base, reg) & BIT(bit);
+}
+
+/**
+ * struct winbond_gpio_port_conflict - possibly conflicting device information
+ * @name:	device name (NULL means no conflicting device defined)
+ * @dev:	Super I/O logical device number where the testreg register
+ *		is located (or WB_SIO_DEV_NONE - don't select any
+ *		logical device)
+ * @testreg:	register number where the testbit bit is located
+ * @testbit:	index of a bit to check whether an actual conflict exists
+ * @warnonly:	if set then a conflict isn't fatal (just warn about it),
+ *		otherwise disable the particular GPIO port if a conflict
+ *		is detected
+ */
+struct winbond_gpio_port_conflict {
+	const char *name;
+	u8 dev;
+	u8 testreg;
+	u8 testbit;
+	bool warnonly;
+};
+
+/**
+ * struct winbond_gpio_info - information about a particular GPIO port (device)
+ * @dev:		Super I/O logical device number of the registers
+ *			specified below
+ * @enablereg:		port enable bit register number
+ * @enablebit:		index of a port enable bit
+ * @outputreg:		output driver mode bit register number
+ * @outputppbit:	index of a push-pull output driver mode bit
+ * @ioreg:		data direction register number
+ * @invreg:		pin data inversion register number
+ * @datareg:		pin data register number
+ * @conflict:		description of a device that possibly conflicts with
+ *			this port
+ */
+struct winbond_gpio_info {
+	u8 dev;
+	u8 enablereg;
+	u8 enablebit;
+	u8 outputreg;
+	u8 outputppbit;
+	u8 ioreg;
+	u8 invreg;
+	u8 datareg;
+	struct winbond_gpio_port_conflict conflict;
+};
+
+static const struct winbond_gpio_info winbond_gpio_infos[6] = {
+	{ /* 0 */
+		.dev = WB_SIO_DEV_GPIO12,
+		.enablereg = WB_SIO_GPIO12_REG_ENABLE,
+		.enablebit = WB_SIO_GPIO12_ENABLE_1,
+		.outputreg = WB_SIO_REG_GPIO1_MF,
+		.outputppbit = WB_SIO_REG_G1MF_G1PP,
+		.ioreg = WB_SIO_GPIO12_REG_IO1,
+		.invreg = WB_SIO_GPIO12_REG_INV1,
+		.datareg = WB_SIO_GPIO12_REG_DATA1,
+		.conflict = {
+			.name = "UARTB",
+			.dev = WB_SIO_DEV_UARTB,
+			.testreg = WB_SIO_UARTB_REG_ENABLE,
+			.testbit = WB_SIO_UARTB_ENABLE_ON,
+			.warnonly = true
+		}
+	},
+	{ /* 1 */
+		.dev = WB_SIO_DEV_GPIO12,
+		.enablereg = WB_SIO_GPIO12_REG_ENABLE,
+		.enablebit = WB_SIO_GPIO12_ENABLE_2,
+		.outputreg = WB_SIO_REG_GPIO1_MF,
+		.outputppbit = WB_SIO_REG_G1MF_G2PP,
+		.ioreg = WB_SIO_GPIO12_REG_IO2,
+		.invreg = WB_SIO_GPIO12_REG_INV2,
+		.datareg = WB_SIO_GPIO12_REG_DATA2
+		/* special conflict handling so doesn't use conflict data */
+	},
+	{ /* 2 */
+		.dev = WB_SIO_DEV_GPIO34,
+		.enablereg = WB_SIO_GPIO34_REG_ENABLE,
+		.enablebit = WB_SIO_GPIO34_ENABLE_3,
+		.outputreg = WB_SIO_REG_OVTGPIO3456,
+		.outputppbit = WB_SIO_REG_OG3456_G3PP,
+		.ioreg = WB_SIO_GPIO34_REG_IO3,
+		.invreg = WB_SIO_GPIO34_REG_INV3,
+		.datareg = WB_SIO_GPIO34_REG_DATA3,
+		.conflict = {
+			.name = "UARTC",
+			.dev = WB_SIO_DEV_UARTC,
+			.testreg = WB_SIO_UARTC_REG_ENABLE,
+			.testbit = WB_SIO_UARTC_ENABLE_ON,
+			.warnonly = true
+		}
+	},
+	{ /* 3 */
+		.dev = WB_SIO_DEV_GPIO34,
+		.enablereg = WB_SIO_GPIO34_REG_ENABLE,
+		.enablebit = WB_SIO_GPIO34_ENABLE_4,
+		.outputreg = WB_SIO_REG_OVTGPIO3456,
+		.outputppbit = WB_SIO_REG_OG3456_G4PP,
+		.ioreg = WB_SIO_GPIO34_REG_IO4,
+		.invreg = WB_SIO_GPIO34_REG_INV4,
+		.datareg = WB_SIO_GPIO34_REG_DATA4,
+		.conflict = {
+			.name = "UARTD",
+			.dev = WB_SIO_DEV_UARTD,
+			.testreg = WB_SIO_UARTD_REG_ENABLE,
+			.testbit = WB_SIO_UARTD_ENABLE_ON,
+			.warnonly = true
+		}
+	},
+	{ /* 4 */
+		.dev = WB_SIO_DEV_WDGPIO56,
+		.enablereg = WB_SIO_WDGPIO56_REG_ENABLE,
+		.enablebit = WB_SIO_WDGPIO56_ENABLE_5,
+		.outputreg = WB_SIO_REG_OVTGPIO3456,
+		.outputppbit = WB_SIO_REG_OG3456_G5PP,
+		.ioreg = WB_SIO_WDGPIO56_REG_IO5,
+		.invreg = WB_SIO_WDGPIO56_REG_INV5,
+		.datareg = WB_SIO_WDGPIO56_REG_DATA5,
+		.conflict = {
+			.name = "UARTE",
+			.dev = WB_SIO_DEV_UARTE,
+			.testreg = WB_SIO_UARTE_REG_ENABLE,
+			.testbit = WB_SIO_UARTE_ENABLE_ON,
+			.warnonly = true
+		}
+	},
+	{ /* 5 */
+		.dev = WB_SIO_DEV_WDGPIO56,
+		.enablereg = WB_SIO_WDGPIO56_REG_ENABLE,
+		.enablebit = WB_SIO_WDGPIO56_ENABLE_6,
+		.outputreg = WB_SIO_REG_OVTGPIO3456,
+		.outputppbit = WB_SIO_REG_OG3456_G6PP,
+		.ioreg = WB_SIO_WDGPIO56_REG_IO6,
+		.invreg = WB_SIO_WDGPIO56_REG_INV6,
+		.datareg = WB_SIO_WDGPIO56_REG_DATA6,
+		.conflict = {
+			.name = "FDC",
+			.dev = WB_SIO_DEV_NONE,
+			.testreg = WB_SIO_REG_GLOBAL_OPT,
+			.testbit = WB_SIO_REG_GO_ENFDC,
+			.warnonly = false
+		}
+	}
+};
+
+/* returns whether changing a pin is allowed */
+static bool winbond_gpio_get_info(unsigned int *gpio_num,
+				  const struct winbond_gpio_info **info)
+{
+	bool allow_changing = true;
+	unsigned long i;
+
+	for_each_set_bit(i, &params.gpios, BITS_PER_LONG) {
+		if (*gpio_num < 8)
+			break;
+
+		*gpio_num -= 8;
+	}
+
+	*info = &winbond_gpio_infos[i];
+
+	/*
+	 * GPIO2 (the second port) shares some pins with a basic PC
+	 * functionality, which is very likely controlled by the firmware.
+	 * Don't allow changing these pins by default.
+	 */
+	if (i == 1) {
+		if (*gpio_num == 0 && !params.pledgpio)
+			allow_changing = false;
+		else if (*gpio_num == 1 && !params.beepgpio)
+			allow_changing = false;
+		else if ((*gpio_num == 5 || *gpio_num == 6) && !params.i2cgpio)
+			allow_changing = false;
+	}
+
+	return allow_changing;
+}
+
+static int winbond_gpio_get(struct gpio_chip *gc, unsigned int offset)
+{
+	unsigned long *base = gpiochip_get_data(gc);
+	const struct winbond_gpio_info *info;
+	bool val;
+
+	winbond_gpio_get_info(&offset, &info);
+
+	val = winbond_sio_enter(*base);
+	if (val)
+		return val;
+
+	winbond_sio_select_logical(*base, info->dev);
+
+	val = winbond_sio_reg_btest(*base, info->datareg, offset);
+	if (winbond_sio_reg_btest(*base, info->invreg, offset))
+		val = !val;
+
+	winbond_sio_leave(*base);
+
+	return val;
+}
+
+static int winbond_gpio_direction_in(struct gpio_chip *gc, unsigned int offset)
+{
+	unsigned long *base = gpiochip_get_data(gc);
+	const struct winbond_gpio_info *info;
+	int ret;
+
+	if (!winbond_gpio_get_info(&offset, &info))
+		return -EACCES;
+
+	ret = winbond_sio_enter(*base);
+	if (ret)
+		return ret;
+
+	winbond_sio_select_logical(*base, info->dev);
+
+	winbond_sio_reg_bset(*base, info->ioreg, offset);
+
+	winbond_sio_leave(*base);
+
+	return 0;
+}
+
+static int winbond_gpio_direction_out(struct gpio_chip *gc,
+				      unsigned int offset,
+				      int val)
+{
+	unsigned long *base = gpiochip_get_data(gc);
+	const struct winbond_gpio_info *info;
+	int ret;
+
+	if (!winbond_gpio_get_info(&offset, &info))
+		return -EACCES;
+
+	ret = winbond_sio_enter(*base);
+	if (ret)
+		return ret;
+
+	winbond_sio_select_logical(*base, info->dev);
+
+	winbond_sio_reg_bclear(*base, info->ioreg, offset);
+
+	if (winbond_sio_reg_btest(*base, info->invreg, offset))
+		val = !val;
+
+	if (val)
+		winbond_sio_reg_bset(*base, info->datareg, offset);
+	else
+		winbond_sio_reg_bclear(*base, info->datareg, offset);
+
+	winbond_sio_leave(*base);
+
+	return 0;
+}
+
+static void winbond_gpio_set(struct gpio_chip *gc, unsigned int offset,
+			     int val)
+{
+	unsigned long *base = gpiochip_get_data(gc);
+	const struct winbond_gpio_info *info;
+
+	if (!winbond_gpio_get_info(&offset, &info))
+		return;
+
+	if (winbond_sio_enter(*base) != 0)
+		return;
+
+	winbond_sio_select_logical(*base, info->dev);
+
+	if (winbond_sio_reg_btest(*base, info->invreg, offset))
+		val = !val;
+
+	if (val)
+		winbond_sio_reg_bset(*base, info->datareg, offset);
+	else
+		winbond_sio_reg_bclear(*base, info->datareg, offset);
+
+	winbond_sio_leave(*base);
+}
+
+static struct gpio_chip winbond_gpio_chip = {
+	.base			= -1,
+	.label			= WB_GPIO_DRIVER_NAME,
+	.owner			= THIS_MODULE,
+	.can_sleep		= true,
+	.get			= winbond_gpio_get,
+	.direction_input	= winbond_gpio_direction_in,
+	.set			= winbond_gpio_set,
+	.direction_output	= winbond_gpio_direction_out,
+};
+
+static void winbond_gpio_configure_port0_pins(unsigned long base)
+{
+	unsigned int val;
+
+	val = winbond_sio_reg_read(base, WB_SIO_REG_GPIO1_MF);
+	if ((val & WB_SIO_REG_G1MF_FS_MASK) == WB_SIO_REG_G1MF_FS_GPIO1)
+		return;
+
+	pr_warn("GPIO1 pins were connected to something else (%.2x), fixing\n",
+		val);
+
+	val &= ~WB_SIO_REG_G1MF_FS_MASK;
+	val |= WB_SIO_REG_G1MF_FS_GPIO1;
+
+	winbond_sio_reg_write(base, WB_SIO_REG_GPIO1_MF, val);
+}
+
+static void winbond_gpio_configure_port1_check_i2c(unsigned long base)
+{
+	params.i2cgpio = !winbond_sio_reg_btest(base, WB_SIO_REG_I2C_PS,
+						WB_SIO_REG_I2CPS_I2CFS);
+	if (!params.i2cgpio)
+		pr_warn("disabling GPIO2.5 and GPIO2.6 as I2C is enabled\n");
+}
+
+static bool winbond_gpio_configure_port(unsigned long base, unsigned int idx)
+{
+	const struct winbond_gpio_info *info = &winbond_gpio_infos[idx];
+	const struct winbond_gpio_port_conflict *conflict = &info->conflict;
+
+	/* is there a possible conflicting device defined? */
+	if (conflict->name != NULL) {
+		if (conflict->dev != WB_SIO_DEV_NONE)
+			winbond_sio_select_logical(base, conflict->dev);
+
+		if (winbond_sio_reg_btest(base, conflict->testreg,
+					  conflict->testbit)) {
+			if (conflict->warnonly)
+				pr_warn("enabled GPIO%u share pins with active %s\n",
+					idx + 1, conflict->name);
+			else {
+				pr_warn("disabling GPIO%u as %s is enabled\n",
+					idx + 1, conflict->name);
+				return false;
+			}
+		}
+	}
+
+	/* GPIO1 and GPIO2 need some (additional) special handling */
+	if (idx == 0)
+		winbond_gpio_configure_port0_pins(base);
+	else if (idx == 1)
+		winbond_gpio_configure_port1_check_i2c(base);
+
+	winbond_sio_select_logical(base, info->dev);
+
+	winbond_sio_reg_bset(base, info->enablereg, info->enablebit);
+
+	if (params.ppgpios & BIT(idx))
+		winbond_sio_reg_bset(base, info->outputreg,
+				     info->outputppbit);
+	else if (params.odgpios & BIT(idx))
+		winbond_sio_reg_bclear(base, info->outputreg,
+				       info->outputppbit);
+	else
+		pr_notice("GPIO%u pins are %s\n", idx + 1,
+			  winbond_sio_reg_btest(base, info->outputreg,
+						info->outputppbit) ?
+			  "push-pull" :
+			  "open drain");
+
+	return true;
+}
+
+static int winbond_gpio_configure(unsigned long base)
+{
+	unsigned long i;
+
+	for_each_set_bit(i, &params.gpios, BITS_PER_LONG)
+		if (!winbond_gpio_configure_port(base, i))
+			__clear_bit(i, &params.gpios);
+
+	if (!params.gpios) {
+		pr_err("please use 'gpios' module parameter to select some active GPIO ports to enable\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int winbond_gpio_check_chip(unsigned long base)
+{
+	int ret;
+	unsigned int chip;
+
+	ret = winbond_sio_enter(base);
+	if (ret)
+		return ret;
+
+	chip = winbond_sio_reg_read(base, WB_SIO_REG_CHIP_MSB) << 8;
+	chip |= winbond_sio_reg_read(base, WB_SIO_REG_CHIP_LSB);
+
+	pr_notice("chip ID at %lx is %.4x\n", base, chip);
+
+	if ((chip & WB_SIO_CHIP_ID_W83627UHG_MASK) !=
+	    WB_SIO_CHIP_ID_W83627UHG) {
+		pr_err("not an our chip\n");
+		ret = -ENODEV;
+	}
+
+	winbond_sio_leave(base);
+
+	return ret;
+}
+
+static int winbond_gpio_imatch(struct device *dev, unsigned int id)
+{
+	unsigned long gpios_rem;
+	int ret;
+
+	gpios_rem = params.gpios & ~GENMASK(ARRAY_SIZE(winbond_gpio_infos) - 1,
+					    0);
+	if (gpios_rem) {
+		pr_warn("unknown ports (%lx) enabled in GPIO ports bitmask\n",
+			gpios_rem);
+		params.gpios &= ~gpios_rem;
+	}
+
+	if (params.ppgpios & params.odgpios) {
+		pr_err("some GPIO ports are set both to push-pull and open drain mode at the same time\n");
+		return 0;
+	}
+
+	if (params.base != 0)
+		return winbond_gpio_check_chip(params.base) == 0;
+
+	/*
+	 * if the 'base' module parameter is unset probe two chip default
+	 * I/O port bases
+	 */
+	params.base = WB_SIO_BASE;
+	ret = winbond_gpio_check_chip(params.base);
+	if (ret == 0)
+		return 1;
+	if (ret != -ENODEV && ret != -EBUSY)
+		return 0;
+
+	params.base = WB_SIO_BASE_HIGH;
+	return winbond_gpio_check_chip(params.base) == 0;
+}
+
+static int winbond_gpio_iprobe(struct device *dev, unsigned int id)
+{
+	int ret;
+
+	if (params.base == 0)
+		return -EINVAL;
+
+	ret = winbond_sio_enter(params.base);
+	if (ret)
+		return ret;
+
+	ret = winbond_gpio_configure(params.base);
+
+	winbond_sio_leave(params.base);
+
+	if (ret)
+		return ret;
+
+	/*
+	 * Add 8 gpios for every GPIO port that was enabled in gpios
+	 * module parameter (that wasn't disabled earlier in
+	 * winbond_gpio_configure() & co. due to, for example, a pin conflict).
+	 */
+	winbond_gpio_chip.ngpio = hweight_long(params.gpios) * 8;
+
+	/*
+	 * GPIO6 port has only 5 pins, so if it is enabled we have to adjust
+	 * the total count appropriately
+	 */
+	if (params.gpios & BIT(5))
+		winbond_gpio_chip.ngpio -= (8 - 5);
+
+	winbond_gpio_chip.parent = dev;
+
+	return devm_gpiochip_add_data(dev, &winbond_gpio_chip, &params.base);
+}
+
+static struct isa_driver winbond_gpio_idriver = {
+	.driver = {
+		.name	= WB_GPIO_DRIVER_NAME,
+	},
+	.match	= winbond_gpio_imatch,
+	.probe	= winbond_gpio_iprobe,
+};
+
+module_isa_driver(winbond_gpio_idriver, 1);
+
+module_param_named(base, params.base, ulong, 0444);
+MODULE_PARM_DESC(base,
+		 "I/O port base (when unset - probe chip default ones)");
+
+/* This parameter sets which GPIO devices (ports) we enable */
+module_param_named(gpios, params.gpios, ulong, 0444);
+MODULE_PARM_DESC(gpios,
+		 "bitmask of GPIO ports to enable (bit 0 - GPIO1, bit 1 - GPIO2, etc.");
+
+/*
+ * These two parameters below set how we configure GPIO ports output drivers.
+ * It can't be a one bitmask since we need three values per port: push-pull,
+ * open-drain and keep as-is (this is the default).
+ */
+module_param_named(ppgpios, params.ppgpios, ulong, 0444);
+MODULE_PARM_DESC(ppgpios,
+		 "bitmask of GPIO ports to set to push-pull mode (bit 0 - GPIO1, bit 1 - GPIO2, etc.");
+
+module_param_named(odgpios, params.odgpios, ulong, 0444);
+MODULE_PARM_DESC(odgpios,
+		 "bitmask of GPIO ports to set to open drain mode (bit 0 - GPIO1, bit 1 - GPIO2, etc.");
+
+/*
+ * GPIO2.0 and GPIO2.1 control a basic PC functionality that we
+ * don't allow tinkering with by default (it is very likely that the
+ * firmware owns these pins).
+ * These two parameters below allow overriding these prohibitions.
+ */
+module_param_named(pledgpio, params.pledgpio, bool, 0644);
+MODULE_PARM_DESC(pledgpio,
+		 "enable changing value of GPIO2.0 bit (Power LED), default no.");
+
+module_param_named(beepgpio, params.beepgpio, bool, 0644);
+MODULE_PARM_DESC(beepgpio,
+		 "enable changing value of GPIO2.1 bit (BEEP), default no.");
+
+MODULE_AUTHOR("Maciej S. Szmigiero <mail@maciej.szmigiero.name>");
+MODULE_DESCRIPTION("GPIO interface for Winbond Super I/O chips");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c
index d6f3d9ee1350..e2232cbcec8b 100644
--- a/drivers/gpio/gpiolib-acpi.c
+++ b/drivers/gpio/gpiolib-acpi.c
@@ -58,58 +58,6 @@ static int acpi_gpiochip_find(struct gpio_chip *gc, void *data)
 	return ACPI_HANDLE(gc->parent) == data;
 }
 
-#ifdef CONFIG_PINCTRL
-/**
- * acpi_gpiochip_pin_to_gpio_offset() - translates ACPI GPIO to Linux GPIO
- * @gdev: GPIO device
- * @pin: ACPI GPIO pin number from GpioIo/GpioInt resource
- *
- * Function takes ACPI GpioIo/GpioInt pin number as a parameter and
- * translates it to a corresponding offset suitable to be passed to a
- * GPIO controller driver.
- *
- * Typically the returned offset is same as @pin, but if the GPIO
- * controller uses pin controller and the mapping is not contiguous the
- * offset might be different.
- */
-static int acpi_gpiochip_pin_to_gpio_offset(struct gpio_device *gdev, int pin)
-{
-	struct gpio_pin_range *pin_range;
-
-	/* If there are no ranges in this chip, use 1:1 mapping */
-	if (list_empty(&gdev->pin_ranges))
-		return pin;
-
-	list_for_each_entry(pin_range, &gdev->pin_ranges, node) {
-		const struct pinctrl_gpio_range *range = &pin_range->range;
-		int i;
-
-		if (range->pins) {
-			for (i = 0; i < range->npins; i++) {
-				if (range->pins[i] == pin)
-					return range->base + i - gdev->base;
-			}
-		} else {
-			if (pin >= range->pin_base &&
-			    pin < range->pin_base + range->npins) {
-				unsigned gpio_base;
-
-				gpio_base = range->base - gdev->base;
-				return gpio_base + pin - range->pin_base;
-			}
-		}
-	}
-
-	return -EINVAL;
-}
-#else
-static inline int acpi_gpiochip_pin_to_gpio_offset(struct gpio_device *gdev,
-						   int pin)
-{
-	return pin;
-}
-#endif
-
 /**
  * acpi_get_gpiod() - Translate ACPI GPIO pin to GPIO descriptor usable with GPIO API
  * @path:	ACPI GPIO controller full path name, (e.g. "\\_SB.GPO1")
@@ -125,7 +73,6 @@ static struct gpio_desc *acpi_get_gpiod(char *path, int pin)
 	struct gpio_chip *chip;
 	acpi_handle handle;
 	acpi_status status;
-	int offset;
 
 	status = acpi_get_handle(NULL, path, &handle);
 	if (ACPI_FAILURE(status))
@@ -135,11 +82,7 @@ static struct gpio_desc *acpi_get_gpiod(char *path, int pin)
 	if (!chip)
 		return ERR_PTR(-EPROBE_DEFER);
 
-	offset = acpi_gpiochip_pin_to_gpio_offset(chip->gpiodev, pin);
-	if (offset < 0)
-		return ERR_PTR(offset);
-
-	return gpiochip_get_desc(chip, offset);
+	return gpiochip_get_desc(chip, pin);
 }
 
 static irqreturn_t acpi_gpio_irq_handler(int irq, void *data)
@@ -216,10 +159,6 @@ static acpi_status acpi_gpiochip_request_interrupt(struct acpi_resource *ares,
 	if (!handler)
 		return AE_OK;
 
-	pin = acpi_gpiochip_pin_to_gpio_offset(chip->gpiodev, pin);
-	if (pin < 0)
-		return AE_BAD_PARAMETER;
-
 	desc = gpiochip_request_own_desc(chip, pin, "ACPI:Event");
 	if (IS_ERR(desc)) {
 		dev_err(chip->parent, "Failed to request GPIO\n");
@@ -414,7 +353,8 @@ EXPORT_SYMBOL_GPL(devm_acpi_dev_remove_driver_gpios);
 
 static bool acpi_get_driver_gpio_data(struct acpi_device *adev,
 				      const char *name, int index,
-				      struct acpi_reference_args *args)
+				      struct acpi_reference_args *args,
+				      unsigned int *quirks)
 {
 	const struct acpi_gpio_mapping *gm;
 
@@ -430,6 +370,8 @@ static bool acpi_get_driver_gpio_data(struct acpi_device *adev,
 			args->args[1] = par->line_index;
 			args->args[2] = par->active_low;
 			args->nargs = 3;
+
+			*quirks = gm->quirks;
 			return true;
 		}
 
@@ -461,8 +403,8 @@ acpi_gpio_to_gpiod_flags(const struct acpi_resource_gpio *agpio)
 	}
 }
 
-int
-acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, enum gpiod_flags update)
+static int
+__acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, enum gpiod_flags update)
 {
 	int ret = 0;
 
@@ -489,12 +431,31 @@ acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, enum gpiod_flags update)
 	return ret;
 }
 
+int
+acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, struct acpi_gpio_info *info)
+{
+	struct device *dev = &info->adev->dev;
+	enum gpiod_flags old = *flags;
+	int ret;
+
+	ret = __acpi_gpio_update_gpiod_flags(&old, info->flags);
+	if (info->quirks & ACPI_GPIO_QUIRK_NO_IO_RESTRICTION) {
+		if (ret)
+			dev_warn(dev, FW_BUG "GPIO not in correct mode, fixing\n");
+	} else {
+		if (ret)
+			dev_dbg(dev, "Override GPIO initialization flags\n");
+		*flags = old;
+	}
+
+	return ret;
+}
+
 struct acpi_gpio_lookup {
 	struct acpi_gpio_info info;
 	int index;
 	int pin_index;
 	bool active_low;
-	struct acpi_device *adev;
 	struct gpio_desc *desc;
 	int n;
 };
@@ -531,8 +492,8 @@ static int acpi_populate_gpio_lookup(struct acpi_resource *ares, void *data)
 			lookup->info.triggering = agpio->triggering;
 		} else {
 			lookup->info.flags = acpi_gpio_to_gpiod_flags(agpio);
+			lookup->info.polarity = lookup->active_low;
 		}
-
 	}
 
 	return 1;
@@ -541,12 +502,13 @@ static int acpi_populate_gpio_lookup(struct acpi_resource *ares, void *data)
 static int acpi_gpio_resource_lookup(struct acpi_gpio_lookup *lookup,
 				     struct acpi_gpio_info *info)
 {
+	struct acpi_device *adev = lookup->info.adev;
 	struct list_head res_list;
 	int ret;
 
 	INIT_LIST_HEAD(&res_list);
 
-	ret = acpi_dev_get_resources(lookup->adev, &res_list,
+	ret = acpi_dev_get_resources(adev, &res_list,
 				     acpi_populate_gpio_lookup,
 				     lookup);
 	if (ret < 0)
@@ -557,11 +519,8 @@ static int acpi_gpio_resource_lookup(struct acpi_gpio_lookup *lookup,
 	if (!lookup->desc)
 		return -ENOENT;
 
-	if (info) {
+	if (info)
 		*info = lookup->info;
-		if (lookup->active_low)
-			info->polarity = lookup->active_low;
-	}
 	return 0;
 }
 
@@ -570,6 +529,7 @@ static int acpi_gpio_property_lookup(struct fwnode_handle *fwnode,
 				     struct acpi_gpio_lookup *lookup)
 {
 	struct acpi_reference_args args;
+	unsigned int quirks = 0;
 	int ret;
 
 	memset(&args, 0, sizeof(args));
@@ -581,14 +541,14 @@ static int acpi_gpio_property_lookup(struct fwnode_handle *fwnode,
 		if (!adev)
 			return ret;
 
-		if (!acpi_get_driver_gpio_data(adev, propname, index, &args))
+		if (!acpi_get_driver_gpio_data(adev, propname, index, &args,
+					       &quirks))
 			return ret;
 	}
 	/*
 	 * The property was found and resolved, so need to lookup the GPIO based
 	 * on returned args.
 	 */
-	lookup->adev = args.adev;
 	if (args.nargs != 3)
 		return -EPROTO;
 
@@ -596,6 +556,8 @@ static int acpi_gpio_property_lookup(struct fwnode_handle *fwnode,
 	lookup->pin_index = args.args[1];
 	lookup->active_low = !!args.args[2];
 
+	lookup->info.adev = args.adev;
+	lookup->info.quirks = quirks;
 	return 0;
 }
 
@@ -643,11 +605,11 @@ static struct gpio_desc *acpi_get_gpiod_by_index(struct acpi_device *adev,
 			return ERR_PTR(ret);
 
 		dev_dbg(&adev->dev, "GPIO: _DSD returned %s %d %d %u\n",
-			dev_name(&lookup.adev->dev), lookup.index,
+			dev_name(&lookup.info.adev->dev), lookup.index,
 			lookup.pin_index, lookup.active_low);
 	} else {
 		dev_dbg(&adev->dev, "GPIO: looking up %d in _CRS\n", index);
-		lookup.adev = adev;
+		lookup.info.adev = adev;
 	}
 
 	ret = acpi_gpio_resource_lookup(&lookup, info);
@@ -664,7 +626,6 @@ struct gpio_desc *acpi_find_gpio(struct device *dev,
 	struct acpi_gpio_info info;
 	struct gpio_desc *desc;
 	char propname[32];
-	int err;
 	int i;
 
 	/* Try first from _DSD */
@@ -703,10 +664,7 @@ struct gpio_desc *acpi_find_gpio(struct device *dev,
 	if (info.polarity == GPIO_ACTIVE_LOW)
 		*lookupflags |= GPIO_ACTIVE_LOW;
 
-	err = acpi_gpio_update_gpiod_flags(dflags, info.flags);
-	if (err)
-		dev_dbg(dev, "Override GPIO initialization flags\n");
-
+	acpi_gpio_update_gpiod_flags(dflags, &info);
 	return desc;
 }
 
@@ -852,12 +810,6 @@ acpi_gpio_adr_space_handler(u32 function, acpi_physical_address address,
 		struct gpio_desc *desc;
 		bool found;
 
-		pin = acpi_gpiochip_pin_to_gpio_offset(chip->gpiodev, pin);
-		if (pin < 0) {
-			status = AE_BAD_PARAMETER;
-			goto out;
-		}
-
 		mutex_lock(&achip->conn_lock);
 
 		found = false;
@@ -990,11 +942,7 @@ static struct gpio_desc *acpi_gpiochip_parse_own_gpio(
 	if (ret < 0)
 		return ERR_PTR(ret);
 
-	ret = acpi_gpiochip_pin_to_gpio_offset(chip->gpiodev, gpios[0]);
-	if (ret < 0)
-		return ERR_PTR(ret);
-
-	desc = gpiochip_get_desc(chip, ret);
+	desc = gpiochip_get_desc(chip, gpios[0]);
 	if (IS_ERR(desc))
 		return desc;
 
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 72a0695d2ac3..84e5a9df2344 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -56,6 +56,42 @@ static struct gpio_desc *of_xlate_and_get_gpiod_flags(struct gpio_chip *chip,
 	return gpiochip_get_desc(chip, ret);
 }
 
+static void of_gpio_flags_quirks(struct device_node *np,
+				 enum of_gpio_flags *flags)
+{
+	/*
+	 * Some GPIO fixed regulator quirks.
+	 * Note that active low is the default.
+	 */
+	if (IS_ENABLED(CONFIG_REGULATOR) &&
+	    (of_device_is_compatible(np, "reg-fixed-voltage") ||
+	     of_device_is_compatible(np, "regulator-gpio"))) {
+		/*
+		 * The regulator GPIO handles are specified such that the
+		 * presence or absence of "enable-active-high" solely controls
+		 * the polarity of the GPIO line. Any phandle flags must
+		 * be actively ignored.
+		 */
+		if (*flags & OF_GPIO_ACTIVE_LOW) {
+			pr_warn("%s GPIO handle specifies active low - ignored\n",
+				of_node_full_name(np));
+			*flags &= ~OF_GPIO_ACTIVE_LOW;
+		}
+		if (!of_property_read_bool(np, "enable-active-high"))
+			*flags |= OF_GPIO_ACTIVE_LOW;
+	}
+	/*
+	 * Legacy open drain handling for fixed voltage regulators.
+	 */
+	if (IS_ENABLED(CONFIG_REGULATOR) &&
+	    of_device_is_compatible(np, "reg-fixed-voltage") &&
+	    of_property_read_bool(np, "gpio-open-drain")) {
+		*flags |= (OF_GPIO_SINGLE_ENDED | OF_GPIO_OPEN_DRAIN);
+		pr_info("%s uses legacy open drain flag - update the DTS if you can\n",
+			of_node_full_name(np));
+	}
+}
+
 /**
  * of_get_named_gpiod_flags() - Get a GPIO descriptor and flags for GPIO API
  * @np:		device node to get GPIO from
@@ -93,6 +129,9 @@ struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np,
 	if (IS_ERR(desc))
 		goto out;
 
+	if (flags)
+		of_gpio_flags_quirks(np, flags);
+
 	pr_debug("%s: parsed '%s' property of node '%pOF[%d]' - status (%d)\n",
 		 __func__, propname, np, index,
 		 PTR_ERR_OR_ZERO(desc));
@@ -117,6 +156,71 @@ int of_get_named_gpio_flags(struct device_node *np, const char *list_name,
 }
 EXPORT_SYMBOL(of_get_named_gpio_flags);
 
+/*
+ * The SPI GPIO bindings happened before we managed to establish that GPIO
+ * properties should be named "foo-gpios" so we have this special kludge for
+ * them.
+ */
+static struct gpio_desc *of_find_spi_gpio(struct device *dev, const char *con_id,
+					  enum of_gpio_flags *of_flags)
+{
+	char prop_name[32]; /* 32 is max size of property name */
+	struct device_node *np = dev->of_node;
+	struct gpio_desc *desc;
+
+	/*
+	 * Hopefully the compiler stubs the rest of the function if this
+	 * is false.
+	 */
+	if (!IS_ENABLED(CONFIG_SPI_MASTER))
+		return ERR_PTR(-ENOENT);
+
+	/* Allow this specifically for "spi-gpio" devices */
+	if (!of_device_is_compatible(np, "spi-gpio") || !con_id)
+		return ERR_PTR(-ENOENT);
+
+	/* Will be "gpio-sck", "gpio-mosi" or "gpio-miso" */
+	snprintf(prop_name, sizeof(prop_name), "%s-%s", "gpio", con_id);
+
+	desc = of_get_named_gpiod_flags(np, prop_name, 0, of_flags);
+	return desc;
+}
+
+/*
+ * Some regulator bindings happened before we managed to establish that GPIO
+ * properties should be named "foo-gpios" so we have this special kludge for
+ * them.
+ */
+static struct gpio_desc *of_find_regulator_gpio(struct device *dev, const char *con_id,
+						enum of_gpio_flags *of_flags)
+{
+	/* These are the connection IDs we accept as legacy GPIO phandles */
+	const char *whitelist[] = {
+		"wlf,ldoena", /* Arizona */
+		"wlf,ldo1ena", /* WM8994 */
+		"wlf,ldo2ena", /* WM8994 */
+	};
+	struct device_node *np = dev->of_node;
+	struct gpio_desc *desc;
+	int i;
+
+	if (!IS_ENABLED(CONFIG_REGULATOR))
+		return ERR_PTR(-ENOENT);
+
+	if (!con_id)
+		return ERR_PTR(-ENOENT);
+
+	for (i = 0; i < ARRAY_SIZE(whitelist); i++)
+		if (!strcmp(con_id, whitelist[i]))
+			break;
+
+	if (i == ARRAY_SIZE(whitelist))
+		return ERR_PTR(-ENOENT);
+
+	desc = of_get_named_gpiod_flags(np, con_id, 0, of_flags);
+	return desc;
+}
+
 struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
 			       unsigned int idx,
 			       enum gpio_lookup_flags *flags)
@@ -126,6 +230,7 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
 	struct gpio_desc *desc;
 	unsigned int i;
 
+	/* Try GPIO property "foo-gpios" and "foo-gpio" */
 	for (i = 0; i < ARRAY_SIZE(gpio_suffixes); i++) {
 		if (con_id)
 			snprintf(prop_name, sizeof(prop_name), "%s-%s", con_id,
@@ -136,10 +241,31 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
 
 		desc = of_get_named_gpiod_flags(dev->of_node, prop_name, idx,
 						&of_flags);
+		/*
+		 * -EPROBE_DEFER in our case means that we found a
+		 * valid GPIO property, but no controller has been
+		 * registered so far.
+		 *
+		 * This means we don't need to look any further for
+		 * alternate name conventions, and we should really
+		 * preserve the return code for our user to be able to
+		 * retry probing later.
+		 */
+		if (IS_ERR(desc) && PTR_ERR(desc) == -EPROBE_DEFER)
+			return desc;
+
 		if (!IS_ERR(desc) || (PTR_ERR(desc) != -ENOENT))
 			break;
 	}
 
+	/* Special handling for SPI GPIOs if used */
+	if (IS_ERR(desc))
+		desc = of_find_spi_gpio(dev, con_id, &of_flags);
+
+	/* Special handling for regulator GPIOs if used */
+	if (IS_ERR(desc) && PTR_ERR(desc) != -EPROBE_DEFER)
+		desc = of_find_regulator_gpio(dev, con_id, &of_flags);
+
 	if (IS_ERR(desc))
 		return desc;
 
@@ -153,8 +279,8 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id,
 			*flags |= GPIO_OPEN_SOURCE;
 	}
 
-	if (of_flags & OF_GPIO_SLEEP_MAY_LOSE_VALUE)
-		*flags |= GPIO_SLEEP_MAY_LOSE_VALUE;
+	if (of_flags & OF_GPIO_TRANSITORY)
+		*flags |= GPIO_TRANSITORY;
 
 	return desc;
 }
@@ -214,6 +340,8 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np,
 
 	if (xlate_flags & OF_GPIO_ACTIVE_LOW)
 		*lflags |= GPIO_ACTIVE_LOW;
+	if (xlate_flags & OF_GPIO_TRANSITORY)
+		*lflags |= GPIO_TRANSITORY;
 
 	if (of_property_read_bool(np, "input"))
 		*dflags |= GPIOD_IN;
diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c
index 3f454eaf2101..3dbaf489a8a5 100644
--- a/drivers/gpio/gpiolib-sysfs.c
+++ b/drivers/gpio/gpiolib-sysfs.c
@@ -8,6 +8,7 @@
 #include <linux/interrupt.h>
 #include <linux/kdev_t.h>
 #include <linux/slab.h>
+#include <linux/ctype.h>
 
 #include "gpiolib.h"
 
@@ -106,8 +107,14 @@ static ssize_t value_show(struct device *dev,
 
 	mutex_lock(&data->mutex);
 
-	status = sprintf(buf, "%d\n", gpiod_get_value_cansleep(desc));
+	status = gpiod_get_value_cansleep(desc);
+	if (status < 0)
+		goto err;
 
+	buf[0] = '0' + status;
+	buf[1] = '\n';
+	status = 2;
+err:
 	mutex_unlock(&data->mutex);
 
 	return status;
@@ -118,7 +125,7 @@ static ssize_t value_store(struct device *dev,
 {
 	struct gpiod_data *data = dev_get_drvdata(dev);
 	struct gpio_desc *desc = data->desc;
-	ssize_t			status;
+	ssize_t status = 0;
 
 	mutex_lock(&data->mutex);
 
@@ -127,7 +134,11 @@ static ssize_t value_store(struct device *dev,
 	} else {
 		long		value;
 
-		status = kstrtol(buf, 0, &value);
+		if (size <= 2 && isdigit(buf[0]) &&
+		    (size == 1 || buf[1] == '\n'))
+			value = buf[0] - '0';
+		else
+			status = kstrtol(buf, 0, &value);
 		if (status == 0) {
 			gpiod_set_value_cansleep(desc, value);
 			status = size;
@@ -138,7 +149,7 @@ static ssize_t value_store(struct device *dev,
 
 	return status;
 }
-static DEVICE_ATTR_RW(value);
+static DEVICE_ATTR_PREALLOC(value, S_IWUSR | S_IRUGO, value_show, value_store);
 
 static irqreturn_t gpio_sysfs_irq(int irq, void *priv)
 {
@@ -474,11 +485,15 @@ static ssize_t export_store(struct class *class,
 			status = -ENODEV;
 		goto done;
 	}
-	status = gpiod_export(desc, true);
-	if (status < 0)
-		gpiod_free(desc);
-	else
-		set_bit(FLAG_SYSFS, &desc->flags);
+
+	status = gpiod_set_transitory(desc, false);
+	if (!status) {
+		status = gpiod_export(desc, true);
+		if (status < 0)
+			gpiod_free(desc);
+		else
+			set_bit(FLAG_SYSFS, &desc->flags);
+	}
 
 done:
 	if (status)
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 14532d9576e4..d66de67ef307 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -162,7 +162,7 @@ EXPORT_SYMBOL_GPL(desc_to_gpio);
  */
 struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc)
 {
-	if (!desc || !desc->gdev || !desc->gdev->chip)
+	if (!desc || !desc->gdev)
 		return NULL;
 	return desc->gdev->chip;
 }
@@ -196,7 +196,7 @@ static int gpiochip_find_base(int ngpio)
  * gpiod_get_direction - return the current direction of a GPIO
  * @desc:	GPIO to get the direction of
  *
- * Return GPIOF_DIR_IN or GPIOF_DIR_OUT, or an error code in case of error.
+ * Returns 0 for output, 1 for input, or an error code in case of error.
  *
  * This function may sleep if gpiod_cansleep() is true.
  */
@@ -460,6 +460,15 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip)
 	if (lflags & ~GPIOHANDLE_REQUEST_VALID_FLAGS)
 		return -EINVAL;
 
+	/*
+	 * Do not allow OPEN_SOURCE & OPEN_DRAIN flags in a single request. If
+	 * the hardware actually supports enabling both at the same time the
+	 * electrical result would be disastrous.
+	 */
+	if ((lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) &&
+	    (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE))
+		return -EINVAL;
+
 	/* OPEN_DRAIN and OPEN_SOURCE flags only make sense for output mode. */
 	if (!(lflags & GPIOHANDLE_REQUEST_OUTPUT) &&
 	    ((lflags & GPIOHANDLE_REQUEST_OPEN_DRAIN) ||
@@ -506,6 +515,10 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip)
 		if (lflags & GPIOHANDLE_REQUEST_OPEN_SOURCE)
 			set_bit(FLAG_OPEN_SOURCE, &desc->flags);
 
+		ret = gpiod_set_transitory(desc, false);
+		if (ret < 0)
+			goto out_free_descs;
+
 		/*
 		 * Lines have to be requested explicitly for input
 		 * or output, else the line will be treated "as is".
@@ -588,6 +601,9 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip)
  * @events: KFIFO for the GPIO events
  * @read_lock: mutex lock to protect reads from colliding with adding
  * new events to the FIFO
+ * @timestamp: cache for the timestamp storing it between hardirq
+ * and IRQ thread, used to bring the timestamp close to the actual
+ * event
  */
 struct lineevent_state {
 	struct gpio_device *gdev;
@@ -598,22 +614,23 @@ struct lineevent_state {
 	wait_queue_head_t wait;
 	DECLARE_KFIFO(events, struct gpioevent_data, 16);
 	struct mutex read_lock;
+	u64 timestamp;
 };
 
 #define GPIOEVENT_REQUEST_VALID_FLAGS \
 	(GPIOEVENT_REQUEST_RISING_EDGE | \
 	GPIOEVENT_REQUEST_FALLING_EDGE)
 
-static unsigned int lineevent_poll(struct file *filep,
+static __poll_t lineevent_poll(struct file *filep,
 				   struct poll_table_struct *wait)
 {
 	struct lineevent_state *le = filep->private_data;
-	unsigned int events = 0;
+	__poll_t events = 0;
 
 	poll_wait(filep, &le->wait, wait);
 
 	if (!kfifo_is_empty(&le->events))
-		events = POLLIN | POLLRDNORM;
+		events = EPOLLIN | EPOLLRDNORM;
 
 	return events;
 }
@@ -732,7 +749,10 @@ static irqreturn_t lineevent_irq_thread(int irq, void *p)
 	struct gpioevent_data ge;
 	int ret, level;
 
-	ge.timestamp = ktime_get_real_ns();
+	/* Do not leak kernel stack to userspace */
+	memset(&ge, 0, sizeof(ge));
+
+	ge.timestamp = le->timestamp;
 	level = gpiod_get_value_cansleep(le->desc);
 
 	if (le->eflags & GPIOEVENT_REQUEST_RISING_EDGE
@@ -755,11 +775,24 @@ static irqreturn_t lineevent_irq_thread(int irq, void *p)
 
 	ret = kfifo_put(&le->events, ge);
 	if (ret != 0)
-		wake_up_poll(&le->wait, POLLIN);
+		wake_up_poll(&le->wait, EPOLLIN);
 
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t lineevent_irq_handler(int irq, void *p)
+{
+	struct lineevent_state *le = p;
+
+	/*
+	 * Just store the timestamp in hardirq context so we get it as
+	 * close in time as possible to the actual event.
+	 */
+	le->timestamp = ktime_get_real_ns();
+
+	return IRQ_WAKE_THREAD;
+}
+
 static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 {
 	struct gpioevent_request eventreq;
@@ -852,7 +885,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip)
 
 	/* Request a thread to read the events */
 	ret = request_threaded_irq(le->irq,
-			NULL,
+			lineevent_irq_handler,
 			lineevent_irq_thread,
 			irqflags,
 			le->label,
@@ -1050,7 +1083,7 @@ static void gpiodevice_release(struct device *dev)
 
 	list_del(&gdev->list);
 	ida_simple_remove(&gpio_ida, gdev->id);
-	kfree(gdev->label);
+	kfree_const(gdev->label);
 	kfree(gdev->descs);
 	kfree(gdev);
 }
@@ -1159,10 +1192,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
 		goto err_free_descs;
 	}
 
-	if (chip->label)
-		gdev->label = kstrdup(chip->label, GFP_KERNEL);
-	else
-		gdev->label = kstrdup("unknown", GFP_KERNEL);
+	gdev->label = kstrdup_const(chip->label ?: "unknown", GFP_KERNEL);
 	if (!gdev->label) {
 		status = -ENOMEM;
 		goto err_free_descs;
@@ -1209,31 +1239,14 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
 		struct gpio_desc *desc = &gdev->descs[i];
 
 		desc->gdev = gdev;
-		/*
-		 * REVISIT: most hardware initializes GPIOs as inputs
-		 * (often with pullups enabled) so power usage is
-		 * minimized. Linux code should set the gpio direction
-		 * first thing; but until it does, and in case
-		 * chip->get_direction is not set, we may expose the
-		 * wrong direction in sysfs.
-		 */
 
-		if (chip->get_direction) {
-			/*
-			 * If we have .get_direction, set up the initial
-			 * direction flag from the hardware.
-			 */
-			int dir = chip->get_direction(chip, i);
-
-			if (!dir)
-				set_bit(FLAG_IS_OUT, &desc->flags);
-		} else if (!chip->direction_input) {
-			/*
-			 * If the chip lacks the .direction_input callback
-			 * we logically assume all lines are outputs.
-			 */
-			set_bit(FLAG_IS_OUT, &desc->flags);
-		}
+		/* REVISIT: most hardware initializes GPIOs as inputs (often
+		 * with pullups enabled) so power usage is minimized. Linux
+		 * code should set the gpio direction first thing; but until
+		 * it does, and in case chip->get_direction is not set, we may
+		 * expose the wrong direction in sysfs.
+		 */
+		desc->flags = !chip->direction_input ? (1 << FLAG_IS_OUT) : 0;
 	}
 
 #ifdef CONFIG_PINCTRL
@@ -1283,7 +1296,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *chip, void *data,
 	list_del(&gdev->list);
 	spin_unlock_irqrestore(&gpio_lock, flags);
 err_free_label:
-	kfree(gdev->label);
+	kfree_const(gdev->label);
 err_free_descs:
 	kfree(gdev->descs);
 err_free_gdev:
@@ -1383,7 +1396,7 @@ static int devm_gpio_chip_match(struct device *dev, void *res, void *data)
 }
 
 /**
- * devm_gpiochip_add_data() - Resource manager piochip_add_data()
+ * devm_gpiochip_add_data() - Resource manager gpiochip_add_data()
  * @dev: the device pointer on which irq_chip belongs to.
  * @chip: the chip to register, with chip->base initialized
  * @data: driver-private data associated with this chip
@@ -1510,14 +1523,15 @@ static void gpiochip_irqchip_free_valid_mask(struct gpio_chip *gpiochip)
 	gpiochip->irq.valid_mask = NULL;
 }
 
-static bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gpiochip,
-				       unsigned int offset)
+bool gpiochip_irqchip_irq_valid(const struct gpio_chip *gpiochip,
+				unsigned int offset)
 {
 	/* No mask means all valid */
 	if (likely(!gpiochip->irq.valid_mask))
 		return true;
 	return test_bit(offset, gpiochip->irq.valid_mask);
 }
+EXPORT_SYMBOL_GPL(gpiochip_irqchip_irq_valid);
 
 /**
  * gpiochip_set_cascaded_irqchip() - connects a cascaded irqchip to a gpiochip
@@ -2174,40 +2188,37 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label)
  * macro to avoid endless duplication. If the desc is NULL it is an
  * optional GPIO and calls should just bail out.
  */
+static int validate_desc(const struct gpio_desc *desc, const char *func)
+{
+	if (!desc)
+		return 0;
+	if (IS_ERR(desc)) {
+		pr_warn("%s: invalid GPIO (errorpointer)\n", func);
+		return PTR_ERR(desc);
+	}
+	if (!desc->gdev) {
+		pr_warn("%s: invalid GPIO (no device)\n", func);
+		return -EINVAL;
+	}
+	if (!desc->gdev->chip) {
+		dev_warn(&desc->gdev->dev,
+			 "%s: backing chip is gone\n", func);
+		return 0;
+	}
+	return 1;
+}
+
 #define VALIDATE_DESC(desc) do { \
-	if (!desc) \
-		return 0; \
-	if (IS_ERR(desc)) {						\
-		pr_warn("%s: invalid GPIO (errorpointer)\n", __func__); \
-		return PTR_ERR(desc); \
-	} \
-	if (!desc->gdev) { \
-		pr_warn("%s: invalid GPIO (no device)\n", __func__); \
-		return -EINVAL; \
-	} \
-	if ( !desc->gdev->chip ) { \
-		dev_warn(&desc->gdev->dev, \
-			 "%s: backing chip is gone\n", __func__); \
-		return 0; \
-	} } while (0)
+	int __valid = validate_desc(desc, __func__); \
+	if (__valid <= 0) \
+		return __valid; \
+	} while (0)
 
 #define VALIDATE_DESC_VOID(desc) do { \
-	if (!desc) \
-		return; \
-	if (IS_ERR(desc)) {						\
-		pr_warn("%s: invalid GPIO (errorpointer)\n", __func__); \
-		return; \
-	} \
-	if (!desc->gdev) { \
-		pr_warn("%s: invalid GPIO (no device)\n", __func__); \
-		return; \
-	} \
-	if (!desc->gdev->chip) { \
-		dev_warn(&desc->gdev->dev, \
-			 "%s: backing chip is gone\n", __func__); \
+	int __valid = validate_desc(desc, __func__); \
+	if (__valid <= 0) \
 		return; \
-	} } while (0)
-
+	} while (0)
 
 int gpiod_request(struct gpio_desc *desc, const char *label)
 {
@@ -2456,7 +2467,7 @@ EXPORT_SYMBOL_GPL(gpiod_direction_output_raw);
  */
 int gpiod_direction_output(struct gpio_desc *desc, int value)
 {
-	struct gpio_chip *gc = desc->gdev->chip;
+	struct gpio_chip *gc;
 	int ret;
 
 	VALIDATE_DESC(desc);
@@ -2473,6 +2484,7 @@ int gpiod_direction_output(struct gpio_desc *desc, int value)
 		return -EIO;
 	}
 
+	gc = desc->gdev->chip;
 	if (test_bit(FLAG_OPEN_DRAIN, &desc->flags)) {
 		/* First see if we can enable open drain in hardware */
 		ret = gpio_set_drive_single_ended(gc, gpio_chip_hwgpio(desc),
@@ -2529,6 +2541,50 @@ int gpiod_set_debounce(struct gpio_desc *desc, unsigned debounce)
 }
 EXPORT_SYMBOL_GPL(gpiod_set_debounce);
 
+/**
+ * gpiod_set_transitory - Lose or retain GPIO state on suspend or reset
+ * @desc: descriptor of the GPIO for which to configure persistence
+ * @transitory: True to lose state on suspend or reset, false for persistence
+ *
+ * Returns:
+ * 0 on success, otherwise a negative error code.
+ */
+int gpiod_set_transitory(struct gpio_desc *desc, bool transitory)
+{
+	struct gpio_chip *chip;
+	unsigned long packed;
+	int gpio;
+	int rc;
+
+	VALIDATE_DESC(desc);
+	/*
+	 * Handle FLAG_TRANSITORY first, enabling queries to gpiolib for
+	 * persistence state.
+	 */
+	if (transitory)
+		set_bit(FLAG_TRANSITORY, &desc->flags);
+	else
+		clear_bit(FLAG_TRANSITORY, &desc->flags);
+
+	/* If the driver supports it, set the persistence state now */
+	chip = desc->gdev->chip;
+	if (!chip->set_config)
+		return 0;
+
+	packed = pinconf_to_config_packed(PIN_CONFIG_PERSIST_STATE,
+					  !transitory);
+	gpio = gpio_chip_hwgpio(desc);
+	rc = chip->set_config(chip, gpio, packed);
+	if (rc == -ENOTSUPP) {
+		dev_dbg(&desc->gdev->dev, "Persistence not supported for GPIO %d\n",
+				gpio);
+		return 0;
+	}
+
+	return rc;
+}
+EXPORT_SYMBOL_GPL(gpiod_set_transitory);
+
 /**
  * gpiod_is_active_low - test whether a GPIO is active-low or not
  * @desc: the gpio descriptor to test
@@ -3129,8 +3185,7 @@ bool gpiochip_line_is_persistent(struct gpio_chip *chip, unsigned int offset)
 	if (offset >= chip->ngpio)
 		return false;
 
-	return !test_bit(FLAG_SLEEP_MAY_LOSE_VALUE,
-			 &chip->gpiodev->descs[offset].flags);
+	return !test_bit(FLAG_TRANSITORY, &chip->gpiodev->descs[offset].flags);
 }
 EXPORT_SYMBOL_GPL(gpiochip_line_is_persistent);
 
@@ -3565,8 +3620,10 @@ int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id,
 
 	if (lflags & GPIO_OPEN_SOURCE)
 		set_bit(FLAG_OPEN_SOURCE, &desc->flags);
-	if (lflags & GPIO_SLEEP_MAY_LOSE_VALUE)
-		set_bit(FLAG_SLEEP_MAY_LOSE_VALUE, &desc->flags);
+
+	status = gpiod_set_transitory(desc, (lflags & GPIO_TRANSITORY));
+	if (status < 0)
+		return status;
 
 	/* No particular flag request, return here... */
 	if (!(dflags & GPIOD_FLAGS_BIT_DIR_SET)) {
@@ -3606,6 +3663,8 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev,
 	struct gpio_desc *desc = NULL;
 	int status;
 	enum gpio_lookup_flags lookupflags = 0;
+	/* Maybe we have a device name, maybe not */
+	const char *devname = dev ? dev_name(dev) : "?";
 
 	dev_dbg(dev, "GPIO lookup for consumer %s\n", con_id);
 
@@ -3634,7 +3693,11 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev,
 		return desc;
 	}
 
-	status = gpiod_request(desc, con_id);
+	/*
+	 * If a connection label was passed use that, else attempt to use
+	 * the device name as label
+	 */
+	status = gpiod_request(desc, con_id ? con_id : devname);
 	if (status < 0)
 		return ERR_PTR(status);
 
@@ -3649,18 +3712,89 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(gpiod_get_index);
 
+/**
+ * gpiod_get_from_of_node() - obtain a GPIO from an OF node
+ * @node:	handle of the OF node
+ * @propname:	name of the DT property representing the GPIO
+ * @index:	index of the GPIO to obtain for the consumer
+ * @dflags:	GPIO initialization flags
+ * @label:	label to attach to the requested GPIO
+ *
+ * Returns:
+ * On successful request the GPIO pin is configured in accordance with
+ * provided @dflags. If the node does not have the requested GPIO
+ * property, NULL is returned.
+ *
+ * In case of error an ERR_PTR() is returned.
+ */
+struct gpio_desc *gpiod_get_from_of_node(struct device_node *node,
+					 const char *propname, int index,
+					 enum gpiod_flags dflags,
+					 const char *label)
+{
+	struct gpio_desc *desc;
+	unsigned long lflags = 0;
+	enum of_gpio_flags flags;
+	bool active_low = false;
+	bool single_ended = false;
+	bool open_drain = false;
+	bool transitory = false;
+	int ret;
+
+	desc = of_get_named_gpiod_flags(node, propname,
+					index, &flags);
+
+	if (!desc || IS_ERR(desc)) {
+		/* If it is not there, just return NULL */
+		if (PTR_ERR(desc) == -ENOENT)
+			return NULL;
+		return desc;
+	}
+
+	active_low = flags & OF_GPIO_ACTIVE_LOW;
+	single_ended = flags & OF_GPIO_SINGLE_ENDED;
+	open_drain = flags & OF_GPIO_OPEN_DRAIN;
+	transitory = flags & OF_GPIO_TRANSITORY;
+
+	ret = gpiod_request(desc, label);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (active_low)
+		lflags |= GPIO_ACTIVE_LOW;
+
+	if (single_ended) {
+		if (open_drain)
+			lflags |= GPIO_OPEN_DRAIN;
+		else
+			lflags |= GPIO_OPEN_SOURCE;
+	}
+
+	if (transitory)
+		lflags |= GPIO_TRANSITORY;
+
+	ret = gpiod_configure_flags(desc, propname, lflags, dflags);
+	if (ret < 0) {
+		gpiod_put(desc);
+		return ERR_PTR(ret);
+	}
+
+	return desc;
+}
+EXPORT_SYMBOL(gpiod_get_from_of_node);
+
 /**
  * fwnode_get_named_gpiod - obtain a GPIO from firmware node
  * @fwnode:	handle of the firmware node
  * @propname:	name of the firmware property representing the GPIO
- * @index:	index of the GPIO to obtain in the consumer
+ * @index:	index of the GPIO to obtain for the consumer
  * @dflags:	GPIO initialization flags
  * @label:	label to attach to the requested GPIO
  *
  * This function can be used for drivers that get their configuration
- * from firmware.
+ * from opaque firmware.
  *
- * Function properly finds the corresponding GPIO using whatever is the
+ * The function properly finds the corresponding GPIO using whatever is the
  * underlying firmware interface and then makes sure that the GPIO
  * descriptor is requested before it is returned to the caller.
  *
@@ -3677,53 +3811,35 @@ struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode,
 {
 	struct gpio_desc *desc = ERR_PTR(-ENODEV);
 	unsigned long lflags = 0;
-	bool active_low = false;
-	bool single_ended = false;
-	bool open_drain = false;
 	int ret;
 
 	if (!fwnode)
 		return ERR_PTR(-EINVAL);
 
 	if (is_of_node(fwnode)) {
-		enum of_gpio_flags flags;
-
-		desc = of_get_named_gpiod_flags(to_of_node(fwnode), propname,
-						index, &flags);
-		if (!IS_ERR(desc)) {
-			active_low = flags & OF_GPIO_ACTIVE_LOW;
-			single_ended = flags & OF_GPIO_SINGLE_ENDED;
-			open_drain = flags & OF_GPIO_OPEN_DRAIN;
-		}
+		desc = gpiod_get_from_of_node(to_of_node(fwnode),
+					      propname, index,
+					      dflags,
+					      label);
+		return desc;
 	} else if (is_acpi_node(fwnode)) {
 		struct acpi_gpio_info info;
 
 		desc = acpi_node_get_gpiod(fwnode, propname, index, &info);
-		if (!IS_ERR(desc)) {
-			active_low = info.polarity == GPIO_ACTIVE_LOW;
-			ret = acpi_gpio_update_gpiod_flags(&dflags, info.flags);
-			if (ret)
-				pr_debug("Override GPIO initialization flags\n");
-		}
-	}
+		if (IS_ERR(desc))
+			return desc;
 
-	if (IS_ERR(desc))
-		return desc;
+		acpi_gpio_update_gpiod_flags(&dflags, &info);
 
+		if (info.polarity == GPIO_ACTIVE_LOW)
+			lflags |= GPIO_ACTIVE_LOW;
+	}
+
+	/* Currently only ACPI takes this path */
 	ret = gpiod_request(desc, label);
 	if (ret)
 		return ERR_PTR(ret);
 
-	if (active_low)
-		lflags |= GPIO_ACTIVE_LOW;
-
-	if (single_ended) {
-		if (open_drain)
-			lflags |= GPIO_OPEN_DRAIN;
-		else
-			lflags |= GPIO_OPEN_SOURCE;
-	}
-
 	ret = gpiod_configure_flags(desc, propname, lflags, dflags);
 	if (ret < 0) {
 		gpiod_put(desc);
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index 6c44d1652139..b17ec6795c81 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -58,7 +58,7 @@ struct gpio_device {
 	struct gpio_desc	*descs;
 	int			base;
 	u16			ngpio;
-	char			*label;
+	const char		*label;
 	void			*data;
 	struct list_head        list;
 
@@ -75,16 +75,20 @@ struct gpio_device {
 
 /**
  * struct acpi_gpio_info - ACPI GPIO specific information
+ * @adev: reference to ACPI device which consumes GPIO resource
  * @flags: GPIO initialization flags
  * @gpioint: if %true this GPIO is of type GpioInt otherwise type is GpioIo
  * @polarity: interrupt polarity as provided by ACPI
  * @triggering: triggering type as provided by ACPI
+ * @quirks: Linux specific quirks as provided by struct acpi_gpio_mapping
  */
 struct acpi_gpio_info {
+	struct acpi_device *adev;
 	enum gpiod_flags flags;
 	bool gpioint;
 	int polarity;
 	int triggering;
+	unsigned int quirks;
 };
 
 /* gpio suffixes used for ACPI and device tree lookup */
@@ -124,7 +128,7 @@ void acpi_gpiochip_request_interrupts(struct gpio_chip *chip);
 void acpi_gpiochip_free_interrupts(struct gpio_chip *chip);
 
 int acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags,
-				 enum gpiod_flags update);
+				 struct acpi_gpio_info *info);
 
 struct gpio_desc *acpi_find_gpio(struct device *dev,
 				 const char *con_id,
@@ -149,7 +153,7 @@ static inline void
 acpi_gpiochip_free_interrupts(struct gpio_chip *chip) { }
 
 static inline int
-acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, enum gpiod_flags update)
+acpi_gpio_update_gpiod_flags(enum gpiod_flags *flags, struct acpi_gpio_info *info)
 {
 	return 0;
 }
@@ -189,6 +193,12 @@ void gpiod_set_array_value_complex(bool raw, bool can_sleep,
 				   struct gpio_desc **desc_array,
 				   int *value_array);
 
+/* This is just passed between gpiolib and devres */
+struct gpio_desc *gpiod_get_from_of_node(struct device_node *node,
+					 const char *propname, int index,
+					 enum gpiod_flags dflags,
+					 const char *label);
+
 extern struct spinlock gpio_lock;
 extern struct list_head gpio_devices;
 
@@ -205,7 +215,7 @@ struct gpio_desc {
 #define FLAG_OPEN_SOURCE 8	/* Gpio is open source type */
 #define FLAG_USED_AS_IRQ 9	/* GPIO is connected to an IRQ */
 #define FLAG_IS_HOGGED	11	/* GPIO is hogged */
-#define FLAG_SLEEP_MAY_LOSE_VALUE 12	/* GPIO may lose value in sleep */
+#define FLAG_TRANSITORY 12	/* GPIO may lose value in sleep or reset */
 
 	/* Connection label */
 	const char		*label;
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 4d9f21831741..deeefa7a1773 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -7,6 +7,7 @@
 menuconfig DRM
 	tristate "Direct Rendering Manager (XFree86 4.1.0 and higher DRI support)"
 	depends on (AGP || AGP=n) && !EMULATED_CMPXCHG && HAS_DMA
+	select DRM_PANEL_ORIENTATION_QUIRKS
 	select HDMI
 	select FB_CMDLINE
 	select I2C
@@ -149,6 +150,10 @@ config DRM_VM
 	bool
 	depends on DRM && MMU
 
+config DRM_SCHED
+	tristate
+	depends on DRM
+
 source "drivers/gpu/drm/i2c/Kconfig"
 
 source "drivers/gpu/drm/arm/Kconfig"
@@ -178,6 +183,7 @@ config DRM_AMDGPU
 	depends on DRM && PCI && MMU
 	select FW_LOADER
         select DRM_KMS_HELPER
+	select DRM_SCHED
         select DRM_TTM
 	select POWER_SUPPLY
 	select HWMON
@@ -362,6 +368,10 @@ config DRM_SAVAGE
 
 endif # DRM_LEGACY
 
+# Separate option because drm_panel_orientation_quirks.c is shared with fbdev
+config DRM_PANEL_ORIENTATION_QUIRKS
+	tristate
+
 config DRM_LIB_RANDOM
 	bool
 	default n
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index e9500844333e..50093ff4479b 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -47,8 +47,10 @@ obj-$(CONFIG_DRM_DEBUG_MM_SELFTEST) += selftests/
 
 obj-$(CONFIG_DRM)	+= drm.o
 obj-$(CONFIG_DRM_MIPI_DSI) += drm_mipi_dsi.o
+obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
 obj-$(CONFIG_DRM_ARM)	+= arm/
 obj-$(CONFIG_DRM_TTM)	+= ttm/
+obj-$(CONFIG_DRM_SCHED)	+= scheduler/
 obj-$(CONFIG_DRM_TDFX)	+= tdfx/
 obj-$(CONFIG_DRM_R128)	+= r128/
 obj-y			+= amd/lib/
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 90202cf4cd1e..d6e5b7273853 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -52,7 +52,8 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_prime.o amdgpu_vm.o amdgpu_ib.o amdgpu_pll.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
 	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
-	amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sched.o
+	amdgpu_queue_mgr.o amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o \
+	amdgpu_ids.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
@@ -62,7 +63,7 @@ amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
 amdgpu-$(CONFIG_DRM_AMDGPU_SI)+= si.o gmc_v6_0.o gfx_v6_0.o si_ih.o si_dma.o dce_v6_0.o si_dpm.o si_smc.o
 
 amdgpu-y += \
-	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o mxgpu_ai.o nbio_v7_0.o
+	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o
 
 # add GMC block
 amdgpu-y += \
@@ -135,10 +136,7 @@ amdgpu-y += \
 amdgpu-y += amdgpu_cgs.o
 
 # GPU scheduler
-amdgpu-y += \
-	../scheduler/gpu_scheduler.o \
-	../scheduler/sched_fence.o \
-	amdgpu_job.o
+amdgpu-y += amdgpu_job.o
 
 # ACP componet
 ifneq ($(CONFIG_DRM_AMD_ACP),)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 0b14b5373783..74edba18b159 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -45,8 +45,11 @@
 #include <drm/drmP.h>
 #include <drm/drm_gem.h>
 #include <drm/amdgpu_drm.h>
+#include <drm/gpu_scheduler.h>
 
 #include <kgd_kfd_interface.h>
+#include "dm_pp_interface.h"
+#include "kgd_pp_interface.h"
 
 #include "amd_shared.h"
 #include "amdgpu_mode.h"
@@ -59,7 +62,6 @@
 #include "amdgpu_sync.h"
 #include "amdgpu_ring.h"
 #include "amdgpu_vm.h"
-#include "amd_powerplay.h"
 #include "amdgpu_dpm.h"
 #include "amdgpu_acp.h"
 #include "amdgpu_uvd.h"
@@ -67,10 +69,9 @@
 #include "amdgpu_vcn.h"
 #include "amdgpu_mn.h"
 #include "amdgpu_dm.h"
-
-#include "gpu_scheduler.h"
 #include "amdgpu_virt.h"
 #include "amdgpu_gart.h"
+#include "amdgpu_debugfs.h"
 
 /*
  * Modules parameters.
@@ -125,6 +126,7 @@ extern int amdgpu_param_buf_per_se;
 extern int amdgpu_job_hang_limit;
 extern int amdgpu_lbpw;
 extern int amdgpu_compute_multipipe;
+extern int amdgpu_gpu_recovery;
 
 #ifdef CONFIG_DRM_AMDGPU_SI
 extern int amdgpu_si_support;
@@ -177,6 +179,10 @@ extern int amdgpu_cik_support;
 #define CIK_CURSOR_WIDTH 128
 #define CIK_CURSOR_HEIGHT 128
 
+/* GPU RESET flags */
+#define AMDGPU_RESET_INFO_VRAM_LOST  (1 << 0)
+#define AMDGPU_RESET_INFO_FULLRESET  (1 << 1)
+
 struct amdgpu_device;
 struct amdgpu_ib;
 struct amdgpu_cs_parser;
@@ -218,17 +224,18 @@ enum amdgpu_kiq_irq {
 	AMDGPU_CP_KIQ_IRQ_LAST
 };
 
-int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
-				  enum amd_ip_block_type block_type,
-				  enum amd_clockgating_state state);
-int amdgpu_set_powergating_state(struct amdgpu_device *adev,
-				  enum amd_ip_block_type block_type,
-				  enum amd_powergating_state state);
-void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags);
-int amdgpu_wait_for_idle(struct amdgpu_device *adev,
-			 enum amd_ip_block_type block_type);
-bool amdgpu_is_idle(struct amdgpu_device *adev,
-		    enum amd_ip_block_type block_type);
+int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_clockgating_state state);
+int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_powergating_state state);
+void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
+					    u32 *flags);
+int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
+				   enum amd_ip_block_type block_type);
+bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
+			      enum amd_ip_block_type block_type);
 
 #define AMDGPU_MAX_IP_NUM 16
 
@@ -253,15 +260,16 @@ struct amdgpu_ip_block {
 	const struct amdgpu_ip_block_version *version;
 };
 
-int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
-				enum amd_ip_block_type type,
-				u32 major, u32 minor);
+int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
+				       enum amd_ip_block_type type,
+				       u32 major, u32 minor);
 
-struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
-					     enum amd_ip_block_type type);
+struct amdgpu_ip_block *
+amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
+			      enum amd_ip_block_type type);
 
-int amdgpu_ip_block_add(struct amdgpu_device *adev,
-			const struct amdgpu_ip_block_version *ip_block_version);
+int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
+			       const struct amdgpu_ip_block_version *ip_block_version);
 
 /* provided by hw blocks that can move/clear data.  e.g., gfx or sdma */
 struct amdgpu_buffer_funcs {
@@ -341,8 +349,9 @@ struct amdgpu_gart_funcs {
 	uint64_t (*get_vm_pte_flags)(struct amdgpu_device *adev,
 				     uint32_t flags);
 	/* get the pde for a given mc addr */
-	u64 (*get_vm_pde)(struct amdgpu_device *adev, u64 addr);
-	uint32_t (*get_invalidate_req)(unsigned int vm_id);
+	void (*get_vm_pde)(struct amdgpu_device *adev, int level,
+			   u64 *dst, u64 *flags);
+	uint32_t (*get_invalidate_req)(unsigned int vmid);
 };
 
 /* provided by the ih block */
@@ -368,9 +377,6 @@ struct amdgpu_dummy_page {
 	struct page	*page;
 	dma_addr_t	addr;
 };
-int amdgpu_dummy_page_init(struct amdgpu_device *adev);
-void amdgpu_dummy_page_fini(struct amdgpu_device *adev);
-
 
 /*
  * Clocks
@@ -418,7 +424,6 @@ struct reservation_object *amdgpu_gem_prime_res_obj(struct drm_gem_object *);
 void *amdgpu_gem_prime_vmap(struct drm_gem_object *obj);
 void amdgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 int amdgpu_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
-int amdgpu_gem_debugfs_init(struct amdgpu_device *adev);
 
 /* sub-allocation manager, it has to be protected by another lock.
  * By conception this is an helper for other part of the driver
@@ -535,6 +540,7 @@ struct amdgpu_mc {
 	u64					private_aperture_end;
 	/* protects concurrent invalidation */
 	spinlock_t		invalidate_lock;
+	bool			translate_further;
 };
 
 /*
@@ -645,12 +651,6 @@ typedef enum _AMDGPU_DOORBELL64_ASSIGNMENT
 	AMDGPU_DOORBELL64_INVALID                 = 0xFFFF
 } AMDGPU_DOORBELL64_ASSIGNMENT;
 
-
-void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
-				phys_addr_t *aperture_base,
-				size_t *aperture_size,
-				size_t *start_offset);
-
 /*
  * IRQS.
  */
@@ -684,7 +684,7 @@ struct amdgpu_ib {
 	uint32_t			flags;
 };
 
-extern const struct amd_sched_backend_ops amdgpu_sched_ops;
+extern const struct drm_sched_backend_ops amdgpu_sched_ops;
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 		     struct amdgpu_job **job, struct amdgpu_vm *vm);
@@ -694,7 +694,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 void amdgpu_job_free_resources(struct amdgpu_job *job);
 void amdgpu_job_free(struct amdgpu_job *job);
 int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
-		      struct amd_sched_entity *entity, void *owner,
+		      struct drm_sched_entity *entity, void *owner,
 		      struct dma_fence **f);
 
 /*
@@ -727,7 +727,7 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 struct amdgpu_ctx_ring {
 	uint64_t		sequence;
 	struct dma_fence	**fences;
-	struct amd_sched_entity	entity;
+	struct drm_sched_entity	entity;
 };
 
 struct amdgpu_ctx {
@@ -735,14 +735,16 @@ struct amdgpu_ctx {
 	struct amdgpu_device    *adev;
 	struct amdgpu_queue_mgr queue_mgr;
 	unsigned		reset_counter;
+	unsigned        reset_counter_query;
 	uint32_t		vram_lost_counter;
 	spinlock_t		ring_lock;
 	struct dma_fence	**fences;
 	struct amdgpu_ctx_ring	rings[AMDGPU_MAX_RINGS];
 	bool			preamble_presented;
-	enum amd_sched_priority init_priority;
-	enum amd_sched_priority override_priority;
+	enum drm_sched_priority init_priority;
+	enum drm_sched_priority override_priority;
 	struct mutex            lock;
+	atomic_t	guilty;
 };
 
 struct amdgpu_ctx_mgr {
@@ -760,7 +762,7 @@ int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
 struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 				   struct amdgpu_ring *ring, uint64_t seq);
 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
-				  enum amd_sched_priority priority);
+				  enum drm_sched_priority priority);
 
 int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 		     struct drm_file *filp);
@@ -957,6 +959,7 @@ struct amdgpu_gfx_config {
 };
 
 struct amdgpu_cu_info {
+	uint32_t simd_per_cu;
 	uint32_t max_waves_per_simd;
 	uint32_t wave_front_size;
 	uint32_t max_scratch_slots_per_cu;
@@ -1109,12 +1112,11 @@ struct amdgpu_cs_parser {
 #define AMDGPU_HAVE_CTX_SWITCH              (1 << 2) /* bit set means context switch occured */
 
 struct amdgpu_job {
-	struct amd_sched_job    base;
+	struct drm_sched_job    base;
 	struct amdgpu_device	*adev;
 	struct amdgpu_vm	*vm;
 	struct amdgpu_ring	*ring;
 	struct amdgpu_sync	sync;
-	struct amdgpu_sync	dep_sync;
 	struct amdgpu_sync	sched_sync;
 	struct amdgpu_ib	*ibs;
 	struct dma_fence	*fence; /* the hw fence */
@@ -1123,7 +1125,7 @@ struct amdgpu_job {
 	void			*owner;
 	uint64_t		fence_ctx; /* the fence_context this job uses */
 	bool                    vm_needs_flush;
-	unsigned		vm_id;
+	unsigned		vmid;
 	uint64_t		vm_pd_addr;
 	uint32_t		gds_base, gds_size;
 	uint32_t		gws_base, gws_size;
@@ -1154,7 +1156,7 @@ static inline void amdgpu_set_ib_value(struct amdgpu_cs_parser *p,
 /*
  * Writeback
  */
-#define AMDGPU_MAX_WB 512	/* Reserve at most 512 WB slots for amdgpu-owned rings. */
+#define AMDGPU_MAX_WB 128	/* Reserve at most 128 WB slots for amdgpu-owned rings. */
 
 struct amdgpu_wb {
 	struct amdgpu_bo	*wb_obj;
@@ -1164,10 +1166,10 @@ struct amdgpu_wb {
 	unsigned long		used[DIV_ROUND_UP(AMDGPU_MAX_WB, BITS_PER_LONG)];
 };
 
-int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb);
-void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb);
+int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb);
+void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb);
 
-void amdgpu_get_pcie_info(struct amdgpu_device *adev);
+void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 
 /*
  * SDMA
@@ -1232,24 +1234,6 @@ void amdgpu_benchmark(struct amdgpu_device *adev, int test_number);
  */
 void amdgpu_test_moves(struct amdgpu_device *adev);
 
-/*
- * Debugfs
- */
-struct amdgpu_debugfs {
-	const struct drm_info_list	*files;
-	unsigned		num_files;
-};
-
-int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
-			     const struct drm_info_list *files,
-			     unsigned nfiles);
-int amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
-
-#if defined(CONFIG_DEBUG_FS)
-int amdgpu_debugfs_init(struct drm_minor *minor);
-#endif
-
-int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
 
 /*
  * amdgpu smumgr functions
@@ -1404,8 +1388,6 @@ struct amdgpu_fw_vram_usage {
 	void *va;
 };
 
-int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev);
-
 /*
  * CGS
  */
@@ -1421,6 +1403,87 @@ typedef void (*amdgpu_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
 typedef uint32_t (*amdgpu_block_rreg_t)(struct amdgpu_device*, uint32_t, uint32_t);
 typedef void (*amdgpu_block_wreg_t)(struct amdgpu_device*, uint32_t, uint32_t, uint32_t);
 
+
+/*
+ * amdgpu nbio functions
+ *
+ */
+struct nbio_hdp_flush_reg {
+	u32 ref_and_mask_cp0;
+	u32 ref_and_mask_cp1;
+	u32 ref_and_mask_cp2;
+	u32 ref_and_mask_cp3;
+	u32 ref_and_mask_cp4;
+	u32 ref_and_mask_cp5;
+	u32 ref_and_mask_cp6;
+	u32 ref_and_mask_cp7;
+	u32 ref_and_mask_cp8;
+	u32 ref_and_mask_cp9;
+	u32 ref_and_mask_sdma0;
+	u32 ref_and_mask_sdma1;
+};
+
+struct amdgpu_nbio_funcs {
+	const struct nbio_hdp_flush_reg *hdp_flush_reg;
+	u32 (*get_hdp_flush_req_offset)(struct amdgpu_device *adev);
+	u32 (*get_hdp_flush_done_offset)(struct amdgpu_device *adev);
+	u32 (*get_pcie_index_offset)(struct amdgpu_device *adev);
+	u32 (*get_pcie_data_offset)(struct amdgpu_device *adev);
+	u32 (*get_rev_id)(struct amdgpu_device *adev);
+	void (*mc_access_enable)(struct amdgpu_device *adev, bool enable);
+	void (*hdp_flush)(struct amdgpu_device *adev);
+	u32 (*get_memsize)(struct amdgpu_device *adev);
+	void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance,
+				    bool use_doorbell, int doorbell_index);
+	void (*enable_doorbell_aperture)(struct amdgpu_device *adev,
+					 bool enable);
+	void (*enable_doorbell_selfring_aperture)(struct amdgpu_device *adev,
+						  bool enable);
+	void (*ih_doorbell_range)(struct amdgpu_device *adev,
+				  bool use_doorbell, int doorbell_index);
+	void (*update_medium_grain_clock_gating)(struct amdgpu_device *adev,
+						 bool enable);
+	void (*update_medium_grain_light_sleep)(struct amdgpu_device *adev,
+						bool enable);
+	void (*get_clockgating_state)(struct amdgpu_device *adev,
+				      u32 *flags);
+	void (*ih_control)(struct amdgpu_device *adev);
+	void (*init_registers)(struct amdgpu_device *adev);
+	void (*detect_hw_virt)(struct amdgpu_device *adev);
+};
+
+
+/* Define the HW IP blocks will be used in driver , add more if necessary */
+enum amd_hw_ip_block_type {
+	GC_HWIP = 1,
+	HDP_HWIP,
+	SDMA0_HWIP,
+	SDMA1_HWIP,
+	MMHUB_HWIP,
+	ATHUB_HWIP,
+	NBIO_HWIP,
+	MP0_HWIP,
+	UVD_HWIP,
+	VCN_HWIP = UVD_HWIP,
+	VCE_HWIP,
+	DF_HWIP,
+	DCE_HWIP,
+	OSSSYS_HWIP,
+	SMUIO_HWIP,
+	PWR_HWIP,
+	NBIF_HWIP,
+	MAX_HWIP
+};
+
+#define HWIP_MAX_INSTANCE	6
+
+struct amd_powerplay {
+	struct cgs_device *cgs_device;
+	void *pp_handle;
+	const struct amd_ip_funcs *ip_funcs;
+	const struct amd_pm_funcs *pp_funcs;
+};
+
 #define AMDGPU_RESET_MAGIC_NUM 64
 struct amdgpu_device {
 	struct device			*dev;
@@ -1606,6 +1669,11 @@ struct amdgpu_device {
 	/* amdkfd interface */
 	struct kfd_dev          *kfd;
 
+	/* soc15 register offset based on ip, instance and  segment */
+	uint32_t 		*reg_offset[MAX_HWIP][HWIP_MAX_INSTANCE];
+
+	const struct amdgpu_nbio_funcs	*nbio_funcs;
+
 	/* delayed work_func for deferring clockgating during resume */
 	struct delayed_work     late_init_work;
 
@@ -1616,9 +1684,6 @@ struct amdgpu_device {
 	/* link all shadow bo */
 	struct list_head                shadow_list;
 	struct mutex                    shadow_list_lock;
-	/* link all gtt */
-	spinlock_t			gtt_list_lock;
-	struct list_head                gtt_list;
 	/* keep an lru list of rings by HW IP */
 	struct list_head		ring_lru_list;
 	spinlock_t			ring_lru_list_lock;
@@ -1629,7 +1694,8 @@ struct amdgpu_device {
 
 	/* record last mm index being written through WREG32*/
 	unsigned long last_mm_index;
-	bool                            in_sriov_reset;
+	bool                            in_gpu_reset;
+	struct mutex  lock_reset;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
@@ -1773,7 +1839,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev))
 #define amdgpu_gart_flush_gpu_tlb(adev, vmid) (adev)->gart.gart_funcs->flush_gpu_tlb((adev), (vmid))
 #define amdgpu_gart_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gart.gart_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
-#define amdgpu_gart_get_vm_pde(adev, addr) (adev)->gart.gart_funcs->get_vm_pde((adev), (addr))
+#define amdgpu_gart_get_vm_pde(adev, level, dst, flags) (adev)->gart.gart_funcs->get_vm_pde((adev), (level), (dst), (flags))
 #define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
 #define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
@@ -1784,7 +1850,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r))
 #define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r))
 #define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
-#define amdgpu_ring_emit_ib(r, ib, vm_id, c) (r)->funcs->emit_ib((r), (ib), (vm_id), (c))
+#define amdgpu_ring_emit_ib(r, ib, vmid, c) (r)->funcs->emit_ib((r), (ib), (vmid), (c))
 #define amdgpu_ring_emit_pipeline_sync(r) (r)->funcs->emit_pipeline_sync((r))
 #define amdgpu_ring_emit_vm_flush(r, vmid, addr) (r)->funcs->emit_vm_flush((r), (vmid), (addr))
 #define amdgpu_ring_emit_fence(r, addr, seq, flags) (r)->funcs->emit_fence((r), (addr), (seq), (flags))
@@ -1823,22 +1889,25 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i))
 
 /* Common functions */
-int amdgpu_gpu_reset(struct amdgpu_device *adev);
-bool amdgpu_need_backup(struct amdgpu_device *adev);
-void amdgpu_pci_config_reset(struct amdgpu_device *adev);
-bool amdgpu_need_post(struct amdgpu_device *adev);
+int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
+			      struct amdgpu_job* job, bool force);
+void amdgpu_device_pci_config_reset(struct amdgpu_device *adev);
+bool amdgpu_device_need_post(struct amdgpu_device *adev);
 void amdgpu_update_display_priority(struct amdgpu_device *adev);
 
 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
 				  u64 num_vis_bytes);
 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain);
 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo);
-void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base);
-void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc);
+void amdgpu_device_vram_location(struct amdgpu_device *adev,
+				 struct amdgpu_mc *mc, u64 base);
+void amdgpu_device_gart_location(struct amdgpu_device *adev,
+				 struct amdgpu_mc *mc);
+int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev);
 void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size);
 int amdgpu_ttm_init(struct amdgpu_device *adev);
 void amdgpu_ttm_fini(struct amdgpu_device *adev);
-void amdgpu_program_register_sequence(struct amdgpu_device *adev,
+void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 					     const u32 *registers,
 					     const u32 array_size);
 
@@ -1872,7 +1941,7 @@ void amdgpu_driver_lastclose_kms(struct drm_device *dev);
 int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv);
 void amdgpu_driver_postclose_kms(struct drm_device *dev,
 				 struct drm_file *file_priv);
-int amdgpu_suspend(struct amdgpu_device *adev);
+int amdgpu_device_ip_suspend(struct amdgpu_device *adev);
 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon);
 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon);
 u32 amdgpu_get_vblank_counter_kms(struct drm_device *dev, unsigned int pipe);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index c04f44a90392..a29362f9ef41 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -277,7 +277,7 @@ static int acp_hw_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	const struct amdgpu_ip_block *ip_block =
-		amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_ACP);
+		amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_ACP);
 
 	if (!ip_block)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 57afad79f55d..8fa850a070e0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -540,6 +540,9 @@ int amdgpu_acpi_pcie_performance_request(struct amdgpu_device *adev,
 	size_t size;
 	u32 retry = 3;
 
+	if (amdgpu_acpi_pcie_notify_device_ready(adev))
+		return -EINVAL;
+
 	/* Get the device handle */
 	handle = ACPI_HANDLE(&adev->pdev->dev);
 	if (!handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 5432af39a674..1d605e1c1d66 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -85,7 +85,7 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
 		kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions();
 		break;
 	default:
-		dev_info(adev->dev, "kfd not supported on this ASIC\n");
+		dev_dbg(adev->dev, "kfd not supported on this ASIC\n");
 		return;
 	}
 
@@ -93,6 +93,39 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev)
 				   adev->pdev, kfd2kgd);
 }
 
+/**
+ * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
+ *                                setup amdkfd
+ *
+ * @adev: amdgpu_device pointer
+ * @aperture_base: output returning doorbell aperture base physical address
+ * @aperture_size: output returning doorbell aperture size in bytes
+ * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
+ *
+ * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
+ * takes doorbells required for its own rings and reports the setup to amdkfd.
+ * amdgpu reserved doorbells are at the start of the doorbell aperture.
+ */
+static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
+					 phys_addr_t *aperture_base,
+					 size_t *aperture_size,
+					 size_t *start_offset)
+{
+	/*
+	 * The first num_doorbells are used by amdgpu.
+	 * amdkfd takes whatever's left in the aperture.
+	 */
+	if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
+		*aperture_base = adev->doorbell.base;
+		*aperture_size = adev->doorbell.size;
+		*start_offset = adev->doorbell.num_doorbells * sizeof(u32);
+	} else {
+		*aperture_base = 0;
+		*aperture_size = 0;
+		*start_offset = 0;
+	}
+}
+
 void amdgpu_amdkfd_device_init(struct amdgpu_device *adev)
 {
 	int i;
@@ -242,14 +275,34 @@ void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
 	kfree(mem);
 }
 
-uint64_t get_vmem_size(struct kgd_dev *kgd)
+void get_local_mem_info(struct kgd_dev *kgd,
+			struct kfd_local_mem_info *mem_info)
 {
-	struct amdgpu_device *adev =
-		(struct amdgpu_device *)kgd;
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+	uint64_t address_mask = adev->dev->dma_mask ? ~*adev->dev->dma_mask :
+					     ~((1ULL << 32) - 1);
+	resource_size_t aper_limit = adev->mc.aper_base + adev->mc.aper_size;
+
+	memset(mem_info, 0, sizeof(*mem_info));
+	if (!(adev->mc.aper_base & address_mask || aper_limit & address_mask)) {
+		mem_info->local_mem_size_public = adev->mc.visible_vram_size;
+		mem_info->local_mem_size_private = adev->mc.real_vram_size -
+				adev->mc.visible_vram_size;
+	} else {
+		mem_info->local_mem_size_public = 0;
+		mem_info->local_mem_size_private = adev->mc.real_vram_size;
+	}
+	mem_info->vram_width = adev->mc.vram_width;
 
-	BUG_ON(kgd == NULL);
+	pr_debug("Address base: %pap limit %pap public 0x%llx private 0x%llx\n",
+			&adev->mc.aper_base, &aper_limit,
+			mem_info->local_mem_size_public,
+			mem_info->local_mem_size_private);
 
-	return adev->mc.real_vram_size;
+	if (amdgpu_sriov_vf(adev))
+		mem_info->mem_clk_max = adev->clock.default_mclk / 100;
+	else
+		mem_info->mem_clk_max = amdgpu_dpm_get_mclk(adev, false) / 100;
 }
 
 uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
@@ -265,6 +318,39 @@ uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
 
-	/* The sclk is in quantas of 10kHz */
-	return adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100;
+	/* the sclk is in quantas of 10kHz */
+	if (amdgpu_sriov_vf(adev))
+		return adev->clock.default_sclk / 100;
+
+	return amdgpu_dpm_get_sclk(adev, false) / 100;
+}
+
+void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+	struct amdgpu_cu_info acu_info = adev->gfx.cu_info;
+
+	memset(cu_info, 0, sizeof(*cu_info));
+	if (sizeof(cu_info->cu_bitmap) != sizeof(acu_info.bitmap))
+		return;
+
+	cu_info->cu_active_number = acu_info.number;
+	cu_info->cu_ao_mask = acu_info.ao_cu_mask;
+	memcpy(&cu_info->cu_bitmap[0], &acu_info.bitmap[0],
+	       sizeof(acu_info.bitmap));
+	cu_info->num_shader_engines = adev->gfx.config.max_shader_engines;
+	cu_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se;
+	cu_info->num_cu_per_sh = adev->gfx.config.max_cu_per_sh;
+	cu_info->simd_per_cu = acu_info.simd_per_cu;
+	cu_info->max_waves_per_simd = acu_info.max_waves_per_simd;
+	cu_info->wave_front_size = acu_info.wave_front_size;
+	cu_info->max_scratch_slots_per_cu = acu_info.max_scratch_slots_per_cu;
+	cu_info->lds_size = acu_info.lds_size;
+}
+
+uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+	return amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 8d689ab7e429..2a519f9062ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -56,10 +56,13 @@ int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 			void **mem_obj, uint64_t *gpu_addr,
 			void **cpu_ptr);
 void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
-uint64_t get_vmem_size(struct kgd_dev *kgd);
+void get_local_mem_info(struct kgd_dev *kgd,
+			struct kfd_local_mem_info *mem_info);
 uint64_t get_gpu_clock_counter(struct kgd_dev *kgd);
 
 uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd);
+void get_cu_info(struct kgd_dev *kgd, struct kfd_cu_info *cu_info);
+uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd);
 
 #define read_user_wptr(mmptr, wptr, dst)				\
 	({								\
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index 1e3e9be7d77e..a9e6aea0e5f8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -105,7 +105,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 			uint32_t queue_id, uint32_t __user *wptr,
 			uint32_t wptr_shift, uint32_t wptr_mask,
 			struct mm_struct *mm);
-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+			uint32_t pipe_id, uint32_t queue_id,
+			uint32_t (**dump)[2], uint32_t *n_regs);
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+			     uint32_t __user *wptr, struct mm_struct *mm);
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+			     uint32_t engine_id, uint32_t queue_id,
+			     uint32_t (**dump)[2], uint32_t *n_regs);
 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 				uint32_t pipe_id, uint32_t queue_id);
 
@@ -166,17 +173,19 @@ static int get_tile_config(struct kgd_dev *kgd,
 static const struct kfd2kgd_calls kfd2kgd = {
 	.init_gtt_mem_allocation = alloc_gtt_mem,
 	.free_gtt_mem = free_gtt_mem,
-	.get_vmem_size = get_vmem_size,
+	.get_local_mem_info = get_local_mem_info,
 	.get_gpu_clock_counter = get_gpu_clock_counter,
 	.get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
-	.alloc_pasid = amdgpu_vm_alloc_pasid,
-	.free_pasid = amdgpu_vm_free_pasid,
+	.alloc_pasid = amdgpu_pasid_alloc,
+	.free_pasid = amdgpu_pasid_free,
 	.program_sh_mem_settings = kgd_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
 	.init_pipeline = kgd_init_pipeline,
 	.init_interrupts = kgd_init_interrupts,
 	.hqd_load = kgd_hqd_load,
 	.hqd_sdma_load = kgd_hqd_sdma_load,
+	.hqd_dump = kgd_hqd_dump,
+	.hqd_sdma_dump = kgd_hqd_sdma_dump,
 	.hqd_is_occupied = kgd_hqd_is_occupied,
 	.hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
 	.hqd_destroy = kgd_hqd_destroy,
@@ -191,6 +200,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.get_fw_version = get_fw_version,
 	.set_scratch_backing_va = set_scratch_backing_va,
 	.get_tile_config = get_tile_config,
+	.get_cu_info = get_cu_info,
+	.get_vram_usage = amdgpu_amdkfd_get_vram_usage
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
@@ -375,7 +386,44 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 	return 0;
 }
 
-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+			uint32_t pipe_id, uint32_t queue_id,
+			uint32_t (**dump)[2], uint32_t *n_regs)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t i = 0, reg;
+#define HQD_N_REGS (35+4)
+#define DUMP_REG(addr) do {				\
+		if (WARN_ON_ONCE(i >= HQD_N_REGS))	\
+			break;				\
+		(*dump)[i][0] = (addr) << 2;		\
+		(*dump)[i++][1] = RREG32(addr);		\
+	} while (0)
+
+	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+	if (*dump == NULL)
+		return -ENOMEM;
+
+	acquire_queue(kgd, pipe_id, queue_id);
+
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0);
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1);
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2);
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3);
+
+	for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_MQD_CONTROL; reg++)
+		DUMP_REG(reg);
+
+	release_queue(kgd);
+
+	WARN_ON_ONCE(i != HQD_N_REGS);
+	*n_regs = i;
+
+	return 0;
+}
+
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+			     uint32_t __user *wptr, struct mm_struct *mm)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
 	struct cik_sdma_rlc_registers *m;
@@ -410,10 +458,17 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
 		WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data);
 	}
 
-	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL,
-				m->sdma_rlc_doorbell);
-	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0);
-	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0);
+	data = REG_SET_FIELD(m->sdma_rlc_doorbell, SDMA0_RLC0_DOORBELL,
+			     ENABLE, 1);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdma_rlc_rb_rptr);
+
+	if (read_user_wptr(mm, wptr, data))
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, data);
+	else
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+		       m->sdma_rlc_rb_rptr);
+
 	WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR,
 				m->sdma_rlc_virtual_addr);
 	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdma_rlc_rb_base);
@@ -423,8 +478,37 @@ static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
 			m->sdma_rlc_rb_rptr_addr_lo);
 	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
 			m->sdma_rlc_rb_rptr_addr_hi);
-	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
-			m->sdma_rlc_rb_cntl);
+
+	data = REG_SET_FIELD(m->sdma_rlc_rb_cntl, SDMA0_RLC0_RB_CNTL,
+			     RB_ENABLE, 1);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
+
+	return 0;
+}
+
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+			     uint32_t engine_id, uint32_t queue_id,
+			     uint32_t (**dump)[2], uint32_t *n_regs)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET +
+		queue_id * KFD_CIK_SDMA_QUEUE_OFFSET;
+	uint32_t i = 0, reg;
+#undef HQD_N_REGS
+#define HQD_N_REGS (19+4)
+
+	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+	if (*dump == NULL)
+		return -ENOMEM;
+
+	for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
+		DUMP_REG(sdma_offset + reg);
+	for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK;
+	     reg++)
+		DUMP_REG(sdma_offset + reg);
+
+	WARN_ON_ONCE(i != HQD_N_REGS);
+	*n_regs = i;
 
 	return 0;
 }
@@ -575,7 +659,7 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 	struct cik_sdma_rlc_registers *m;
 	uint32_t sdma_base_addr;
 	uint32_t temp;
-	int timeout = utimeout;
+	unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
 
 	m = get_sdma_mqd(mqd);
 	sdma_base_addr = get_sdma_base_addr(m);
@@ -588,10 +672,9 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 		temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
 		if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
 			break;
-		if (timeout <= 0)
+		if (time_after(jiffies, end_jiffies))
 			return -ETIME;
-		msleep(20);
-		timeout -= 20;
+		usleep_range(500, 1000);
 	}
 
 	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
@@ -599,6 +682,8 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 		RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
 		SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
 
+	m->sdma_rlc_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 056929b8ccd0..b127259d7d85 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -45,7 +45,7 @@ enum hqd_dequeue_request_type {
 	RESET_WAVES
 };
 
-struct cik_sdma_rlc_registers;
+struct vi_sdma_mqd;
 
 /*
  * Register access functions
@@ -64,7 +64,14 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 			uint32_t queue_id, uint32_t __user *wptr,
 			uint32_t wptr_shift, uint32_t wptr_mask,
 			struct mm_struct *mm);
-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd);
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+			uint32_t pipe_id, uint32_t queue_id,
+			uint32_t (**dump)[2], uint32_t *n_regs);
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+			     uint32_t __user *wptr, struct mm_struct *mm);
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+			     uint32_t engine_id, uint32_t queue_id,
+			     uint32_t (**dump)[2], uint32_t *n_regs);
 static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 		uint32_t pipe_id, uint32_t queue_id);
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd);
@@ -125,17 +132,19 @@ static int get_tile_config(struct kgd_dev *kgd,
 static const struct kfd2kgd_calls kfd2kgd = {
 	.init_gtt_mem_allocation = alloc_gtt_mem,
 	.free_gtt_mem = free_gtt_mem,
-	.get_vmem_size = get_vmem_size,
+	.get_local_mem_info = get_local_mem_info,
 	.get_gpu_clock_counter = get_gpu_clock_counter,
 	.get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
-	.alloc_pasid = amdgpu_vm_alloc_pasid,
-	.free_pasid = amdgpu_vm_free_pasid,
+	.alloc_pasid = amdgpu_pasid_alloc,
+	.free_pasid = amdgpu_pasid_free,
 	.program_sh_mem_settings = kgd_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
 	.init_pipeline = kgd_init_pipeline,
 	.init_interrupts = kgd_init_interrupts,
 	.hqd_load = kgd_hqd_load,
 	.hqd_sdma_load = kgd_hqd_sdma_load,
+	.hqd_dump = kgd_hqd_dump,
+	.hqd_sdma_dump = kgd_hqd_sdma_dump,
 	.hqd_is_occupied = kgd_hqd_is_occupied,
 	.hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied,
 	.hqd_destroy = kgd_hqd_destroy,
@@ -152,6 +161,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.get_fw_version = get_fw_version,
 	.set_scratch_backing_va = set_scratch_backing_va,
 	.get_tile_config = get_tile_config,
+	.get_cu_info = get_cu_info,
+	.get_vram_usage = amdgpu_amdkfd_get_vram_usage
 };
 
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
@@ -268,9 +279,15 @@ static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id)
 	return 0;
 }
 
-static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m)
+static inline uint32_t get_sdma_base_addr(struct vi_sdma_mqd *m)
 {
-	return 0;
+	uint32_t retval;
+
+	retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET +
+		m->sdma_queue_id * KFD_VI_SDMA_QUEUE_OFFSET;
+	pr_debug("kfd: sdma base address: 0x%x\n", retval);
+
+	return retval;
 }
 
 static inline struct vi_mqd *get_mqd(void *mqd)
@@ -278,9 +295,9 @@ static inline struct vi_mqd *get_mqd(void *mqd)
 	return (struct vi_mqd *)mqd;
 }
 
-static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
 {
-	return (struct cik_sdma_rlc_registers *)mqd;
+	return (struct vi_sdma_mqd *)mqd;
 }
 
 static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
@@ -358,8 +375,138 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id,
 	return 0;
 }
 
-static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd)
+static int kgd_hqd_dump(struct kgd_dev *kgd,
+			uint32_t pipe_id, uint32_t queue_id,
+			uint32_t (**dump)[2], uint32_t *n_regs)
 {
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t i = 0, reg;
+#define HQD_N_REGS (54+4)
+#define DUMP_REG(addr) do {				\
+		if (WARN_ON_ONCE(i >= HQD_N_REGS))	\
+			break;				\
+		(*dump)[i][0] = (addr) << 2;		\
+		(*dump)[i++][1] = RREG32(addr);		\
+	} while (0)
+
+	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+	if (*dump == NULL)
+		return -ENOMEM;
+
+	acquire_queue(kgd, pipe_id, queue_id);
+
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE0);
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE1);
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE2);
+	DUMP_REG(mmCOMPUTE_STATIC_THREAD_MGMT_SE3);
+
+	for (reg = mmCP_MQD_BASE_ADDR; reg <= mmCP_HQD_EOP_DONES; reg++)
+		DUMP_REG(reg);
+
+	release_queue(kgd);
+
+	WARN_ON_ONCE(i != HQD_N_REGS);
+	*n_regs = i;
+
+	return 0;
+}
+
+static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd,
+			     uint32_t __user *wptr, struct mm_struct *mm)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	struct vi_sdma_mqd *m;
+	unsigned long end_jiffies;
+	uint32_t sdma_base_addr;
+	uint32_t data;
+
+	m = get_sdma_mqd(mqd);
+	sdma_base_addr = get_sdma_base_addr(m);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+		m->sdmax_rlcx_rb_cntl & (~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK));
+
+	end_jiffies = msecs_to_jiffies(2000) + jiffies;
+	while (true) {
+		data = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
+		if (data & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
+			break;
+		if (time_after(jiffies, end_jiffies))
+			return -ETIME;
+		usleep_range(500, 1000);
+	}
+	if (m->sdma_engine_id) {
+		data = RREG32(mmSDMA1_GFX_CONTEXT_CNTL);
+		data = REG_SET_FIELD(data, SDMA1_GFX_CONTEXT_CNTL,
+				RESUME_CTX, 0);
+		WREG32(mmSDMA1_GFX_CONTEXT_CNTL, data);
+	} else {
+		data = RREG32(mmSDMA0_GFX_CONTEXT_CNTL);
+		data = REG_SET_FIELD(data, SDMA0_GFX_CONTEXT_CNTL,
+				RESUME_CTX, 0);
+		WREG32(mmSDMA0_GFX_CONTEXT_CNTL, data);
+	}
+
+	data = REG_SET_FIELD(m->sdmax_rlcx_doorbell, SDMA0_RLC0_DOORBELL,
+			     ENABLE, 1);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, data);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, m->sdmax_rlcx_rb_rptr);
+
+	if (read_user_wptr(mm, wptr, data))
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, data);
+	else
+		WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR,
+		       m->sdmax_rlcx_rb_rptr);
+
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR,
+				m->sdmax_rlcx_virtual_addr);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, m->sdmax_rlcx_rb_base);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI,
+			m->sdmax_rlcx_rb_base_hi);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO,
+			m->sdmax_rlcx_rb_rptr_addr_lo);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI,
+			m->sdmax_rlcx_rb_rptr_addr_hi);
+
+	data = REG_SET_FIELD(m->sdmax_rlcx_rb_cntl, SDMA0_RLC0_RB_CNTL,
+			     RB_ENABLE, 1);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, data);
+
+	return 0;
+}
+
+static int kgd_hqd_sdma_dump(struct kgd_dev *kgd,
+			     uint32_t engine_id, uint32_t queue_id,
+			     uint32_t (**dump)[2], uint32_t *n_regs)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t sdma_offset = engine_id * SDMA1_REGISTER_OFFSET +
+		queue_id * KFD_VI_SDMA_QUEUE_OFFSET;
+	uint32_t i = 0, reg;
+#undef HQD_N_REGS
+#define HQD_N_REGS (19+4+2+3+7)
+
+	*dump = kmalloc(HQD_N_REGS*2*sizeof(uint32_t), GFP_KERNEL);
+	if (*dump == NULL)
+		return -ENOMEM;
+
+	for (reg = mmSDMA0_RLC0_RB_CNTL; reg <= mmSDMA0_RLC0_DOORBELL; reg++)
+		DUMP_REG(sdma_offset + reg);
+	for (reg = mmSDMA0_RLC0_VIRTUAL_ADDR; reg <= mmSDMA0_RLC0_WATERMARK;
+	     reg++)
+		DUMP_REG(sdma_offset + reg);
+	for (reg = mmSDMA0_RLC0_CSA_ADDR_LO; reg <= mmSDMA0_RLC0_CSA_ADDR_HI;
+	     reg++)
+		DUMP_REG(sdma_offset + reg);
+	for (reg = mmSDMA0_RLC0_IB_SUB_REMAIN; reg <= mmSDMA0_RLC0_DUMMY_REG;
+	     reg++)
+		DUMP_REG(sdma_offset + reg);
+	for (reg = mmSDMA0_RLC0_MIDCMD_DATA0; reg <= mmSDMA0_RLC0_MIDCMD_CNTL;
+	     reg++)
+		DUMP_REG(sdma_offset + reg);
+
+	WARN_ON_ONCE(i != HQD_N_REGS);
+	*n_regs = i;
+
 	return 0;
 }
 
@@ -388,7 +535,7 @@ static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address,
 static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	struct cik_sdma_rlc_registers *m;
+	struct vi_sdma_mqd *m;
 	uint32_t sdma_base_addr;
 	uint32_t sdma_rlc_rb_cntl;
 
@@ -509,10 +656,10 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 				unsigned int utimeout)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	struct cik_sdma_rlc_registers *m;
+	struct vi_sdma_mqd *m;
 	uint32_t sdma_base_addr;
 	uint32_t temp;
-	int timeout = utimeout;
+	unsigned long end_jiffies = (utimeout * HZ / 1000) + jiffies;
 
 	m = get_sdma_mqd(mqd);
 	sdma_base_addr = get_sdma_base_addr(m);
@@ -523,18 +670,19 @@ static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd,
 
 	while (true) {
 		temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS);
-		if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT)
+		if (temp & SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK)
 			break;
-		if (timeout <= 0)
+		if (time_after(jiffies, end_jiffies))
 			return -ETIME;
-		msleep(20);
-		timeout -= 20;
+		usleep_range(500, 1000);
 	}
 
 	WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0);
-	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0);
-	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0);
-	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0);
+	WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL,
+		RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL) |
+		SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK);
+
+	m->sdmax_rlcx_rb_rptr = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index f450b69323fa..bf872f694f50 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -27,6 +27,7 @@
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
 #include "amdgpu_atombios.h"
+#include "amdgpu_atomfirmware.h"
 #include "amdgpu_i2c.h"
 
 #include "atom.h"
@@ -690,12 +691,12 @@ int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev)
 			le32_to_cpu(firmware_info->info_21.ulDefaultDispEngineClkFreq);
 		/* set a reasonable default for DP */
 		if (adev->clock.default_dispclk < 53900) {
-			DRM_INFO("Changing default dispclk from %dMhz to 600Mhz\n",
-				 adev->clock.default_dispclk / 100);
+			DRM_DEBUG("Changing default dispclk from %dMhz to 600Mhz\n",
+				  adev->clock.default_dispclk / 100);
 			adev->clock.default_dispclk = 60000;
 		} else if (adev->clock.default_dispclk <= 60000) {
-			DRM_INFO("Changing default dispclk from %dMhz to 625Mhz\n",
-				 adev->clock.default_dispclk / 100);
+			DRM_DEBUG("Changing default dispclk from %dMhz to 625Mhz\n",
+				  adev->clock.default_dispclk / 100);
 			adev->clock.default_dispclk = 62500;
 		}
 		adev->clock.dp_extclk =
@@ -1699,7 +1700,7 @@ void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock)
 	WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch);
 }
 
-void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev)
+static void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev)
 {
 	uint32_t bios_2_scratch, bios_6_scratch;
 
@@ -1721,28 +1722,6 @@ void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev)
 	WREG32(adev->bios_scratch_reg_offset + 6, bios_6_scratch);
 }
 
-void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev)
-{
-	int i;
-
-	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
-		adev->bios_scratch[i] = RREG32(adev->bios_scratch_reg_offset + i);
-}
-
-void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev)
-{
-	int i;
-
-	/*
-	 * VBIOS will check ASIC_INIT_COMPLETE bit to decide if
-	 * execute ASIC_Init posting via driver
-	 */
-	adev->bios_scratch[7] &= ~ATOM_S7_ASIC_INIT_COMPLETE_MASK;
-
-	for (i = 0; i < AMDGPU_BIOS_NUM_SCRATCH; i++)
-		WREG32(adev->bios_scratch_reg_offset + i, adev->bios_scratch[i]);
-}
-
 void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
 					      bool hung)
 {
@@ -1798,7 +1777,7 @@ void amdgpu_atombios_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le)
 #endif
 }
 
-int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
+static int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
 {
 	struct atom_context *ctx = adev->mode_info.atom_context;
 	int index = GetIndexIntoMasterTable(DATA, VRAM_UsageByFirmware);
@@ -1841,3 +1820,234 @@ int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev)
 	ctx->scratch_size_bytes = usage_bytes;
 	return 0;
 }
+
+/* ATOM accessor methods */
+/*
+ * ATOM is an interpreted byte code stored in tables in the vbios.  The
+ * driver registers callbacks to access registers and the interpreter
+ * in the driver parses the tables and executes then to program specific
+ * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
+ * atombios.h, and atom.c
+ */
+
+/**
+ * cail_pll_read - read PLL register
+ *
+ * @info: atom card_info pointer
+ * @reg: PLL register offset
+ *
+ * Provides a PLL register accessor for the atom interpreter (r4xx+).
+ * Returns the value of the PLL register.
+ */
+static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
+{
+	return 0;
+}
+
+/**
+ * cail_pll_write - write PLL register
+ *
+ * @info: atom card_info pointer
+ * @reg: PLL register offset
+ * @val: value to write to the pll register
+ *
+ * Provides a PLL register accessor for the atom interpreter (r4xx+).
+ */
+static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
+{
+
+}
+
+/**
+ * cail_mc_read - read MC (Memory Controller) register
+ *
+ * @info: atom card_info pointer
+ * @reg: MC register offset
+ *
+ * Provides an MC register accessor for the atom interpreter (r4xx+).
+ * Returns the value of the MC register.
+ */
+static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
+{
+	return 0;
+}
+
+/**
+ * cail_mc_write - write MC (Memory Controller) register
+ *
+ * @info: atom card_info pointer
+ * @reg: MC register offset
+ * @val: value to write to the pll register
+ *
+ * Provides a MC register accessor for the atom interpreter (r4xx+).
+ */
+static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
+{
+
+}
+
+/**
+ * cail_reg_write - write MMIO register
+ *
+ * @info: atom card_info pointer
+ * @reg: MMIO register offset
+ * @val: value to write to the pll register
+ *
+ * Provides a MMIO register accessor for the atom interpreter (r4xx+).
+ */
+static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
+{
+	struct amdgpu_device *adev = info->dev->dev_private;
+
+	WREG32(reg, val);
+}
+
+/**
+ * cail_reg_read - read MMIO register
+ *
+ * @info: atom card_info pointer
+ * @reg: MMIO register offset
+ *
+ * Provides an MMIO register accessor for the atom interpreter (r4xx+).
+ * Returns the value of the MMIO register.
+ */
+static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
+{
+	struct amdgpu_device *adev = info->dev->dev_private;
+	uint32_t r;
+
+	r = RREG32(reg);
+	return r;
+}
+
+/**
+ * cail_ioreg_write - write IO register
+ *
+ * @info: atom card_info pointer
+ * @reg: IO register offset
+ * @val: value to write to the pll register
+ *
+ * Provides a IO register accessor for the atom interpreter (r4xx+).
+ */
+static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
+{
+	struct amdgpu_device *adev = info->dev->dev_private;
+
+	WREG32_IO(reg, val);
+}
+
+/**
+ * cail_ioreg_read - read IO register
+ *
+ * @info: atom card_info pointer
+ * @reg: IO register offset
+ *
+ * Provides an IO register accessor for the atom interpreter (r4xx+).
+ * Returns the value of the IO register.
+ */
+static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
+{
+	struct amdgpu_device *adev = info->dev->dev_private;
+	uint32_t r;
+
+	r = RREG32_IO(reg);
+	return r;
+}
+
+static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev,
+						 struct device_attribute *attr,
+						 char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+	struct atom_context *ctx = adev->mode_info.atom_context;
+
+	return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version);
+}
+
+static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version,
+		   NULL);
+
+/**
+ * amdgpu_atombios_fini - free the driver info and callbacks for atombios
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Frees the driver info and register access callbacks for the ATOM
+ * interpreter (r4xx+).
+ * Called at driver shutdown.
+ */
+void amdgpu_atombios_fini(struct amdgpu_device *adev)
+{
+	if (adev->mode_info.atom_context) {
+		kfree(adev->mode_info.atom_context->scratch);
+		kfree(adev->mode_info.atom_context->iio);
+	}
+	kfree(adev->mode_info.atom_context);
+	adev->mode_info.atom_context = NULL;
+	kfree(adev->mode_info.atom_card_info);
+	adev->mode_info.atom_card_info = NULL;
+	device_remove_file(adev->dev, &dev_attr_vbios_version);
+}
+
+/**
+ * amdgpu_atombios_init - init the driver info and callbacks for atombios
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Initializes the driver info and register access callbacks for the
+ * ATOM interpreter (r4xx+).
+ * Returns 0 on sucess, -ENOMEM on failure.
+ * Called at driver startup.
+ */
+int amdgpu_atombios_init(struct amdgpu_device *adev)
+{
+	struct card_info *atom_card_info =
+	    kzalloc(sizeof(struct card_info), GFP_KERNEL);
+	int ret;
+
+	if (!atom_card_info)
+		return -ENOMEM;
+
+	adev->mode_info.atom_card_info = atom_card_info;
+	atom_card_info->dev = adev->ddev;
+	atom_card_info->reg_read = cail_reg_read;
+	atom_card_info->reg_write = cail_reg_write;
+	/* needed for iio ops */
+	if (adev->rio_mem) {
+		atom_card_info->ioreg_read = cail_ioreg_read;
+		atom_card_info->ioreg_write = cail_ioreg_write;
+	} else {
+		DRM_DEBUG("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
+		atom_card_info->ioreg_read = cail_reg_read;
+		atom_card_info->ioreg_write = cail_reg_write;
+	}
+	atom_card_info->mc_read = cail_mc_read;
+	atom_card_info->mc_write = cail_mc_write;
+	atom_card_info->pll_read = cail_pll_read;
+	atom_card_info->pll_write = cail_pll_write;
+
+	adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
+	if (!adev->mode_info.atom_context) {
+		amdgpu_atombios_fini(adev);
+		return -ENOMEM;
+	}
+
+	mutex_init(&adev->mode_info.atom_context->mutex);
+	if (adev->is_atom_fw) {
+		amdgpu_atomfirmware_scratch_regs_init(adev);
+		amdgpu_atomfirmware_allocate_fb_scratch(adev);
+	} else {
+		amdgpu_atombios_scratch_regs_init(adev);
+		amdgpu_atombios_allocate_fb_scratch(adev);
+	}
+
+	ret = device_create_file(adev->dev, &dev_attr_vbios_version);
+	if (ret) {
+		DRM_ERROR("Failed to create device file for VBIOS version\n");
+		return ret;
+	}
+
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
index b0d5d1d7fdba..fd8f18074f7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.h
@@ -195,9 +195,6 @@ int amdgpu_atombios_init_mc_reg_table(struct amdgpu_device *adev,
 bool amdgpu_atombios_has_gpu_virtualization_table(struct amdgpu_device *adev);
 
 void amdgpu_atombios_scratch_regs_lock(struct amdgpu_device *adev, bool lock);
-void amdgpu_atombios_scratch_regs_init(struct amdgpu_device *adev);
-void amdgpu_atombios_scratch_regs_save(struct amdgpu_device *adev);
-void amdgpu_atombios_scratch_regs_restore(struct amdgpu_device *adev);
 void amdgpu_atombios_scratch_regs_engine_hung(struct amdgpu_device *adev,
 					      bool hung);
 bool amdgpu_atombios_scratch_need_asic_init(struct amdgpu_device *adev);
@@ -219,6 +216,7 @@ int amdgpu_atombios_get_svi2_info(struct amdgpu_device *adev,
 			      u8 voltage_type,
 			      u8 *svd_gpio_id, u8 *svc_gpio_id);
 
-int amdgpu_atombios_allocate_fb_scratch(struct amdgpu_device *adev);
+void amdgpu_atombios_fini(struct amdgpu_device *adev);
+int amdgpu_atombios_init(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index c13c51af0b68..c53095b3b0fb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -14,6 +14,16 @@
 
 #include "amd_acpi.h"
 
+#define AMDGPU_PX_QUIRK_FORCE_ATPX  (1 << 0)
+
+struct amdgpu_px_quirk {
+	u32 chip_vendor;
+	u32 chip_device;
+	u32 subsys_vendor;
+	u32 subsys_device;
+	u32 px_quirk_flags;
+};
+
 struct amdgpu_atpx_functions {
 	bool px_params;
 	bool power_cntl;
@@ -35,6 +45,7 @@ struct amdgpu_atpx {
 static struct amdgpu_atpx_priv {
 	bool atpx_detected;
 	bool bridge_pm_usable;
+	unsigned int quirks;
 	/* handle for device - and atpx */
 	acpi_handle dhandle;
 	acpi_handle other_handle;
@@ -205,13 +216,19 @@ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx)
 
 	atpx->is_hybrid = false;
 	if (valid_bits & ATPX_MS_HYBRID_GFX_SUPPORTED) {
-		printk("ATPX Hybrid Graphics\n");
-		/*
-		 * Disable legacy PM methods only when pcie port PM is usable,
-		 * otherwise the device might fail to power off or power on.
-		 */
-		atpx->functions.power_cntl = !amdgpu_atpx_priv.bridge_pm_usable;
-		atpx->is_hybrid = true;
+		if (amdgpu_atpx_priv.quirks & AMDGPU_PX_QUIRK_FORCE_ATPX) {
+			printk("ATPX Hybrid Graphics, forcing to ATPX\n");
+			atpx->functions.power_cntl = true;
+			atpx->is_hybrid = false;
+		} else {
+			printk("ATPX Hybrid Graphics\n");
+			/*
+			 * Disable legacy PM methods only when pcie port PM is usable,
+			 * otherwise the device might fail to power off or power on.
+			 */
+			atpx->functions.power_cntl = !amdgpu_atpx_priv.bridge_pm_usable;
+			atpx->is_hybrid = true;
+		}
 	}
 
 	atpx->dgpu_req_power_for_displays = false;
@@ -547,6 +564,31 @@ static const struct vga_switcheroo_handler amdgpu_atpx_handler = {
 	.get_client_id = amdgpu_atpx_get_client_id,
 };
 
+static const struct amdgpu_px_quirk amdgpu_px_quirk_list[] = {
+	/* HG _PR3 doesn't seem to work on this A+A weston board */
+	{ 0x1002, 0x6900, 0x1002, 0x0124, AMDGPU_PX_QUIRK_FORCE_ATPX },
+	{ 0x1002, 0x6900, 0x1028, 0x0812, AMDGPU_PX_QUIRK_FORCE_ATPX },
+	{ 0x1002, 0x6900, 0x1028, 0x0813, AMDGPU_PX_QUIRK_FORCE_ATPX },
+	{ 0, 0, 0, 0, 0 },
+};
+
+static void amdgpu_atpx_get_quirks(struct pci_dev *pdev)
+{
+	const struct amdgpu_px_quirk *p = amdgpu_px_quirk_list;
+
+	/* Apply PX quirks */
+	while (p && p->chip_device != 0) {
+		if (pdev->vendor == p->chip_vendor &&
+		    pdev->device == p->chip_device &&
+		    pdev->subsystem_vendor == p->subsys_vendor &&
+		    pdev->subsystem_device == p->subsys_device) {
+			amdgpu_atpx_priv.quirks |= p->px_quirk_flags;
+			break;
+		}
+		++p;
+	}
+}
+
 /**
  * amdgpu_atpx_detect - detect whether we have PX
  *
@@ -570,6 +612,7 @@ static bool amdgpu_atpx_detect(void)
 
 		parent_pdev = pci_upstream_bridge(pdev);
 		d3_supported |= parent_pdev && parent_pdev->bridge_d3;
+		amdgpu_atpx_get_quirks(pdev);
 	}
 
 	while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_OTHER << 8, pdev)) != NULL) {
@@ -579,6 +622,7 @@ static bool amdgpu_atpx_detect(void)
 
 		parent_pdev = pci_upstream_bridge(pdev);
 		d3_supported |= parent_pdev && parent_pdev->bridge_d3;
+		amdgpu_atpx_get_quirks(pdev);
 	}
 
 	if (has_atpx && vga_count == 2) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index 057e1ecd83ce..a5df80d50d44 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -93,7 +93,7 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev)
 	resource_size_t size = 256 * 1024; /* ??? */
 
 	if (!(adev->flags & AMD_IS_APU))
-		if (amdgpu_need_post(adev))
+		if (amdgpu_device_need_post(adev))
 			return false;
 
 	adev->bios = NULL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index f2b72c7c6857..4466f3535e2d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -801,6 +801,11 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 				else
 					strcpy(fw_name, "amdgpu/vega10_smc.bin");
 				break;
+			case CHIP_CARRIZO:
+			case CHIP_STONEY:
+			case CHIP_RAVEN:
+				adev->pm.fw_version = info->version;
+				return 0;
 			default:
 				DRM_ERROR("SMC firmware not supported\n");
 				return -EINVAL;
@@ -948,7 +953,6 @@ static int amdgpu_cgs_get_active_displays_info(struct cgs_device *cgs_device,
 								(amdgpu_crtc->v_border * 2);
 					mode_info->vblank_time_us = vblank_lines * line_time_us;
 					mode_info->refresh_rate = drm_mode_vrefresh(&amdgpu_crtc->hw_mode);
-					mode_info->ref_clock = adev->clock.spll.reference_freq;
 					mode_info = NULL;
 				}
 			}
@@ -958,7 +962,6 @@ static int amdgpu_cgs_get_active_displays_info(struct cgs_device *cgs_device,
 		if (mode_info != NULL) {
 			mode_info->vblank_time_us = adev->pm.pm_display_cfg.min_vblank_time;
 			mode_info->refresh_rate = adev->pm.pm_display_cfg.vrefresh;
-			mode_info->ref_clock = adev->clock.spll.reference_freq;
 		}
 	}
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index df9cbc78e168..7a073ac5f9c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -69,25 +69,18 @@ void amdgpu_connector_hotplug(struct drm_connector *connector)
 		/* don't do anything if sink is not display port, i.e.,
 		 * passive dp->(dvi|hdmi) adaptor
 		 */
-		if (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) {
-			int saved_dpms = connector->dpms;
-			/* Only turn off the display if it's physically disconnected */
-			if (!amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) {
-				drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
-			} else if (amdgpu_atombios_dp_needs_link_train(amdgpu_connector)) {
-				/* Don't try to start link training before we
-				 * have the dpcd */
-				if (amdgpu_atombios_dp_get_dpcd(amdgpu_connector))
-					return;
-
-				/* set it to OFF so that drm_helper_connector_dpms()
-				 * won't return immediately since the current state
-				 * is ON at this point.
-				 */
-				connector->dpms = DRM_MODE_DPMS_OFF;
-				drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
-			}
-			connector->dpms = saved_dpms;
+		if (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT &&
+		    amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd) &&
+		    amdgpu_atombios_dp_needs_link_train(amdgpu_connector)) {
+			/* Don't start link training before we have the DPCD */
+			if (amdgpu_atombios_dp_get_dpcd(amdgpu_connector))
+				return;
+
+			/* Turn the connector off and back on immediately, which
+			 * will trigger link training
+			 */
+			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
 		}
 	}
 }
@@ -358,7 +351,6 @@ static int amdgpu_connector_ddc_get_modes(struct drm_connector *connector)
 	if (amdgpu_connector->edid) {
 		drm_mode_connector_update_edid_property(connector, amdgpu_connector->edid);
 		ret = drm_add_edid_modes(connector, amdgpu_connector->edid);
-		drm_edid_to_eld(connector, amdgpu_connector->edid);
 		return ret;
 	}
 	drm_mode_connector_update_edid_property(connector, NULL);
@@ -737,9 +729,11 @@ amdgpu_connector_lvds_detect(struct drm_connector *connector, bool force)
 	enum drm_connector_status ret = connector_status_disconnected;
 	int r;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	if (encoder) {
 		struct amdgpu_encoder *amdgpu_encoder = to_amdgpu_encoder(encoder);
@@ -758,8 +752,12 @@ amdgpu_connector_lvds_detect(struct drm_connector *connector, bool force)
 	/* check acpi lid status ??? */
 
 	amdgpu_connector_update_scratch_regs(connector, ret);
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
+
 	return ret;
 }
 
@@ -869,9 +867,11 @@ amdgpu_connector_vga_detect(struct drm_connector *connector, bool force)
 	enum drm_connector_status ret = connector_status_disconnected;
 	int r;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	encoder = amdgpu_connector_best_single_encoder(connector);
 	if (!encoder)
@@ -925,8 +925,10 @@ amdgpu_connector_vga_detect(struct drm_connector *connector, bool force)
 	amdgpu_connector_update_scratch_regs(connector, ret);
 
 out:
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
 
 	return ret;
 }
@@ -989,9 +991,11 @@ amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force)
 	enum drm_connector_status ret = connector_status_disconnected;
 	bool dret = false, broken_edid = false;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	if (!force && amdgpu_connector_check_hpd_status_unchanged(connector)) {
 		ret = connector->status;
@@ -1116,8 +1120,10 @@ amdgpu_connector_dvi_detect(struct drm_connector *connector, bool force)
 	amdgpu_connector_update_scratch_regs(connector, ret);
 
 exit:
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
 
 	return ret;
 }
@@ -1360,9 +1366,11 @@ amdgpu_connector_dp_detect(struct drm_connector *connector, bool force)
 	struct drm_encoder *encoder = amdgpu_connector_best_single_encoder(connector);
 	int r;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	if (!force && amdgpu_connector_check_hpd_status_unchanged(connector)) {
 		ret = connector->status;
@@ -1430,8 +1438,10 @@ amdgpu_connector_dp_detect(struct drm_connector *connector, bool force)
 
 	amdgpu_connector_update_scratch_regs(connector, ret);
 out:
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 57abf7abd7a9..e80fc38141b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -90,6 +90,12 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
 		goto free_chunk;
 	}
 
+	/* skip guilty context job */
+	if (atomic_read(&p->ctx->guilty) == 1) {
+		ret = -ECANCELED;
+		goto free_chunk;
+	}
+
 	mutex_lock(&p->ctx->lock);
 
 	/* get chunks */
@@ -337,7 +343,12 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 				 struct amdgpu_bo *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-	u64 initial_bytes_moved, bytes_moved;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false,
+		.allow_reserved_eviction = false,
+		.resv = bo->tbo.resv
+	};
 	uint32_t domain;
 	int r;
 
@@ -367,15 +378,13 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
 
 retry:
 	amdgpu_ttm_placement_from_domain(bo, domain);
-	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
-	bytes_moved = atomic64_read(&adev->num_bytes_moved) -
-		      initial_bytes_moved;
-	p->bytes_moved += bytes_moved;
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+
+	p->bytes_moved += ctx.bytes_moved;
 	if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
 	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&
 	    bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
-		p->bytes_moved_vis += bytes_moved;
+		p->bytes_moved_vis += ctx.bytes_moved;
 
 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
 		domain = bo->allowed_domains;
@@ -390,6 +399,7 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 				struct amdgpu_bo *validated)
 {
 	uint32_t domain = validated->allowed_domains;
+	struct ttm_operation_ctx ctx = { true, false };
 	int r;
 
 	if (!p->evictable)
@@ -431,7 +441,7 @@ static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
 			bo->tbo.mem.mem_type == TTM_PL_VRAM &&
 			bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT;
 		initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
-		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		bytes_moved = atomic64_read(&adev->num_bytes_moved) -
 			initial_bytes_moved;
 		p->bytes_moved += bytes_moved;
@@ -470,6 +480,7 @@ static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo)
 static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
 			    struct list_head *validated)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct amdgpu_bo_list_entry *lobj;
 	int r;
 
@@ -487,8 +498,7 @@ static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
 		    lobj->user_pages) {
 			amdgpu_ttm_placement_from_domain(bo,
 							 AMDGPU_GEM_DOMAIN_CPU);
-			r = ttm_bo_validate(&bo->tbo, &bo->placement, true,
-					    false);
+			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 			if (r)
 				return r;
 			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
@@ -678,7 +688,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 	if (!r && p->uf_entry.robj) {
 		struct amdgpu_bo *uf = p->uf_entry.robj;
 
-		r = amdgpu_ttm_bind(&uf->tbo, &uf->tbo.mem);
+		r = amdgpu_ttm_alloc_gart(&uf->tbo);
 		p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
 	}
 
@@ -768,10 +778,6 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 	struct amdgpu_bo *bo;
 	int i, r;
 
-	r = amdgpu_vm_update_directories(adev, vm);
-	if (r)
-		return r;
-
 	r = amdgpu_vm_clear_freed(adev, vm, NULL);
 	if (r)
 		return r;
@@ -781,7 +787,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 		return r;
 
 	r = amdgpu_sync_fence(adev, &p->job->sync,
-			      fpriv->prt_va->last_pt_update);
+			      fpriv->prt_va->last_pt_update, false);
 	if (r)
 		return r;
 
@@ -795,7 +801,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 			return r;
 
 		f = bo_va->last_pt_update;
-		r = amdgpu_sync_fence(adev, &p->job->sync, f);
+		r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
 		if (r)
 			return r;
 	}
@@ -818,7 +824,7 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 				return r;
 
 			f = bo_va->last_pt_update;
-			r = amdgpu_sync_fence(adev, &p->job->sync, f);
+			r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
 			if (r)
 				return r;
 		}
@@ -829,7 +835,11 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
 	if (r)
 		return r;
 
-	r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update);
+	r = amdgpu_vm_update_directories(adev, vm);
+	if (r)
+		return r;
+
+	r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update, false);
 	if (r)
 		return r;
 
@@ -865,8 +875,8 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 			struct amdgpu_bo_va_mapping *m;
 			struct amdgpu_bo *aobj = NULL;
 			struct amdgpu_cs_chunk *chunk;
+			uint64_t offset, va_start;
 			struct amdgpu_ib *ib;
-			uint64_t offset;
 			uint8_t *kptr;
 
 			chunk = &p->chunks[i];
@@ -876,14 +886,14 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 			if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
 				continue;
 
-			r = amdgpu_cs_find_mapping(p, chunk_ib->va_start,
-						   &aobj, &m);
+			va_start = chunk_ib->va_start & AMDGPU_VA_HOLE_MASK;
+			r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
 			if (r) {
 				DRM_ERROR("IB va_start is invalid\n");
 				return r;
 			}
 
-			if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
+			if ((va_start + chunk_ib->ib_bytes) >
 			    (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
 				DRM_ERROR("IB va_start+ib_bytes is invalid\n");
 				return -EINVAL;
@@ -896,7 +906,7 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 			}
 
 			offset = m->start * AMDGPU_GPU_PAGE_SIZE;
-			kptr += chunk_ib->va_start - offset;
+			kptr += va_start - offset;
 
 			memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
 			amdgpu_bo_kunmap(aobj);
@@ -1033,8 +1043,8 @@ static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
 			amdgpu_ctx_put(ctx);
 			return r;
 		} else if (fence) {
-			r = amdgpu_sync_fence(p->adev, &p->job->sync,
-					      fence);
+			r = amdgpu_sync_fence(p->adev, &p->job->sync, fence,
+					true);
 			dma_fence_put(fence);
 			amdgpu_ctx_put(ctx);
 			if (r)
@@ -1053,7 +1063,7 @@ static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
 	if (r)
 		return r;
 
-	r = amdgpu_sync_fence(p->adev, &p->job->sync, fence);
+	r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true);
 	dma_fence_put(fence);
 
 	return r;
@@ -1145,7 +1155,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 			    union drm_amdgpu_cs *cs)
 {
 	struct amdgpu_ring *ring = p->job->ring;
-	struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
+	struct drm_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
 	struct amdgpu_job *job;
 	unsigned i;
 	uint64_t seq;
@@ -1168,7 +1178,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	job = p->job;
 	p->job = NULL;
 
-	r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
+	r = drm_sched_job_init(&job->base, &ring->sched, entity, p->filp);
 	if (r) {
 		amdgpu_job_free(job);
 		amdgpu_mn_unlock(p->mn);
@@ -1194,11 +1204,10 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	job->uf_sequence = seq;
 
 	amdgpu_job_free_resources(job);
-	amdgpu_ring_priority_get(job->ring,
-				 amd_sched_get_job_priority(&job->base));
+	amdgpu_ring_priority_get(job->ring, job->base.s_priority);
 
 	trace_amdgpu_cs_ioctl(job);
-	amd_sched_entity_push_job(&job->base);
+	drm_sched_entity_push_job(&job->base, entity);
 
 	ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
 	amdgpu_mn_unlock(p->mn);
@@ -1570,6 +1579,7 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 			   struct amdgpu_bo_va_mapping **map)
 {
 	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
+	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_va_mapping *mapping;
 	int r;
@@ -1590,11 +1600,10 @@ int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
 	if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
 		(*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 		amdgpu_ttm_placement_from_domain(*bo, (*bo)->allowed_domains);
-		r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, false,
-				    false);
+		r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
 		if (r)
 			return r;
 	}
 
-	return amdgpu_ttm_bind(&(*bo)->tbo, &(*bo)->tbo.mem);
+	return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index c184468e2b2b..09d35051fdd6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -28,10 +28,10 @@
 #include "amdgpu_sched.h"
 
 static int amdgpu_ctx_priority_permit(struct drm_file *filp,
-				      enum amd_sched_priority priority)
+				      enum drm_sched_priority priority)
 {
 	/* NORMAL and below are accessible by everyone */
-	if (priority <= AMD_SCHED_PRIORITY_NORMAL)
+	if (priority <= DRM_SCHED_PRIORITY_NORMAL)
 		return 0;
 
 	if (capable(CAP_SYS_NICE))
@@ -44,14 +44,14 @@ static int amdgpu_ctx_priority_permit(struct drm_file *filp,
 }
 
 static int amdgpu_ctx_init(struct amdgpu_device *adev,
-			   enum amd_sched_priority priority,
+			   enum drm_sched_priority priority,
 			   struct drm_file *filp,
 			   struct amdgpu_ctx *ctx)
 {
 	unsigned i, j;
 	int r;
 
-	if (priority < 0 || priority >= AMD_SCHED_PRIORITY_MAX)
+	if (priority < 0 || priority >= DRM_SCHED_PRIORITY_MAX)
 		return -EINVAL;
 
 	r = amdgpu_ctx_priority_permit(filp, priority);
@@ -75,22 +75,23 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 	}
 
 	ctx->reset_counter = atomic_read(&adev->gpu_reset_counter);
+	ctx->reset_counter_query = ctx->reset_counter;
 	ctx->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
 	ctx->init_priority = priority;
-	ctx->override_priority = AMD_SCHED_PRIORITY_UNSET;
+	ctx->override_priority = DRM_SCHED_PRIORITY_UNSET;
 
 	/* create context entity for each ring */
 	for (i = 0; i < adev->num_rings; i++) {
 		struct amdgpu_ring *ring = adev->rings[i];
-		struct amd_sched_rq *rq;
+		struct drm_sched_rq *rq;
 
 		rq = &ring->sched.sched_rq[priority];
 
 		if (ring == &adev->gfx.kiq.ring)
 			continue;
 
-		r = amd_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
-					  rq, amdgpu_sched_jobs);
+		r = drm_sched_entity_init(&ring->sched, &ctx->rings[i].entity,
+					  rq, amdgpu_sched_jobs, &ctx->guilty);
 		if (r)
 			goto failed;
 	}
@@ -103,7 +104,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 
 failed:
 	for (j = 0; j < i; j++)
-		amd_sched_entity_fini(&adev->rings[j]->sched,
+		drm_sched_entity_fini(&adev->rings[j]->sched,
 				      &ctx->rings[j].entity);
 	kfree(ctx->fences);
 	ctx->fences = NULL;
@@ -125,7 +126,7 @@ static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
 	ctx->fences = NULL;
 
 	for (i = 0; i < adev->num_rings; i++)
-		amd_sched_entity_fini(&adev->rings[i]->sched,
+		drm_sched_entity_fini(&adev->rings[i]->sched,
 				      &ctx->rings[i].entity);
 
 	amdgpu_queue_mgr_fini(adev, &ctx->queue_mgr);
@@ -136,7 +137,7 @@ static void amdgpu_ctx_fini(struct amdgpu_ctx *ctx)
 static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 			    struct amdgpu_fpriv *fpriv,
 			    struct drm_file *filp,
-			    enum amd_sched_priority priority,
+			    enum drm_sched_priority priority,
 			    uint32_t *id)
 {
 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
@@ -216,11 +217,45 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev,
 	/* determine if a GPU reset has occured since the last call */
 	reset_counter = atomic_read(&adev->gpu_reset_counter);
 	/* TODO: this should ideally return NO, GUILTY, or INNOCENT. */
-	if (ctx->reset_counter == reset_counter)
+	if (ctx->reset_counter_query == reset_counter)
 		out->state.reset_status = AMDGPU_CTX_NO_RESET;
 	else
 		out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET;
-	ctx->reset_counter = reset_counter;
+	ctx->reset_counter_query = reset_counter;
+
+	mutex_unlock(&mgr->lock);
+	return 0;
+}
+
+static int amdgpu_ctx_query2(struct amdgpu_device *adev,
+	struct amdgpu_fpriv *fpriv, uint32_t id,
+	union drm_amdgpu_ctx_out *out)
+{
+	struct amdgpu_ctx *ctx;
+	struct amdgpu_ctx_mgr *mgr;
+
+	if (!fpriv)
+		return -EINVAL;
+
+	mgr = &fpriv->ctx_mgr;
+	mutex_lock(&mgr->lock);
+	ctx = idr_find(&mgr->ctx_handles, id);
+	if (!ctx) {
+		mutex_unlock(&mgr->lock);
+		return -EINVAL;
+	}
+
+	out->state.flags = 0x0;
+	out->state.hangs = 0x0;
+
+	if (ctx->reset_counter != atomic_read(&adev->gpu_reset_counter))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RESET;
+
+	if (ctx->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST;
+
+	if (atomic_read(&ctx->guilty))
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
 
 	mutex_unlock(&mgr->lock);
 	return 0;
@@ -231,7 +266,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 {
 	int r;
 	uint32_t id;
-	enum amd_sched_priority priority;
+	enum drm_sched_priority priority;
 
 	union drm_amdgpu_ctx *args = data;
 	struct amdgpu_device *adev = dev->dev_private;
@@ -243,8 +278,8 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 
 	/* For backwards compatibility reasons, we need to accept
 	 * ioctls with garbage in the priority field */
-	if (priority == AMD_SCHED_PRIORITY_INVALID)
-		priority = AMD_SCHED_PRIORITY_NORMAL;
+	if (priority == DRM_SCHED_PRIORITY_INVALID)
+		priority = DRM_SCHED_PRIORITY_NORMAL;
 
 	switch (args->in.op) {
 	case AMDGPU_CTX_OP_ALLOC_CTX:
@@ -257,6 +292,9 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 	case AMDGPU_CTX_OP_QUERY_STATE:
 		r = amdgpu_ctx_query(adev, fpriv, id, &args->out);
 		break;
+	case AMDGPU_CTX_OP_QUERY_STATE2:
+		r = amdgpu_ctx_query2(adev, fpriv, id, &args->out);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -347,18 +385,18 @@ struct dma_fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx,
 }
 
 void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
-				  enum amd_sched_priority priority)
+				  enum drm_sched_priority priority)
 {
 	int i;
 	struct amdgpu_device *adev = ctx->adev;
-	struct amd_sched_rq *rq;
-	struct amd_sched_entity *entity;
+	struct drm_sched_rq *rq;
+	struct drm_sched_entity *entity;
 	struct amdgpu_ring *ring;
-	enum amd_sched_priority ctx_prio;
+	enum drm_sched_priority ctx_prio;
 
 	ctx->override_priority = priority;
 
-	ctx_prio = (ctx->override_priority == AMD_SCHED_PRIORITY_UNSET) ?
+	ctx_prio = (ctx->override_priority == DRM_SCHED_PRIORITY_UNSET) ?
 			ctx->init_priority : ctx->override_priority;
 
 	for (i = 0; i < adev->num_rings; i++) {
@@ -369,7 +407,7 @@ void amdgpu_ctx_priority_override(struct amdgpu_ctx *ctx,
 		if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 			continue;
 
-		amd_sched_entity_set_rq(entity, rq);
+		drm_sched_entity_set_rq(entity, rq);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
new file mode 100644
index 000000000000..ee76b468774a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -0,0 +1,792 @@
+/*
+ * Copyright 2008 Advanced Micro Devices, Inc.
+ * Copyright 2008 Red Hat Inc.
+ * Copyright 2009 Jerome Glisse.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/kthread.h>
+#include <drm/drmP.h>
+#include <linux/debugfs.h>
+#include "amdgpu.h"
+
+/*
+ * Debugfs
+ */
+int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
+			     const struct drm_info_list *files,
+			     unsigned nfiles)
+{
+	unsigned i;
+
+	for (i = 0; i < adev->debugfs_count; i++) {
+		if (adev->debugfs[i].files == files) {
+			/* Already registered */
+			return 0;
+		}
+	}
+
+	i = adev->debugfs_count + 1;
+	if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
+		DRM_ERROR("Reached maximum number of debugfs components.\n");
+		DRM_ERROR("Report so we increase "
+			  "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
+		return -EINVAL;
+	}
+	adev->debugfs[adev->debugfs_count].files = files;
+	adev->debugfs[adev->debugfs_count].num_files = nfiles;
+	adev->debugfs_count = i;
+#if defined(CONFIG_DEBUG_FS)
+	drm_debugfs_create_files(files, nfiles,
+				 adev->ddev->primary->debugfs_root,
+				 adev->ddev->primary);
+#endif
+	return 0;
+}
+
+#if defined(CONFIG_DEBUG_FS)
+
+static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+	bool pm_pg_lock, use_bank;
+	unsigned instance_bank, sh_bank, se_bank;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	/* are we reading registers for which a PG lock is necessary? */
+	pm_pg_lock = (*pos >> 23) & 1;
+
+	if (*pos & (1ULL << 62)) {
+		se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
+		sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
+		instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
+
+		if (se_bank == 0x3FF)
+			se_bank = 0xFFFFFFFF;
+		if (sh_bank == 0x3FF)
+			sh_bank = 0xFFFFFFFF;
+		if (instance_bank == 0x3FF)
+			instance_bank = 0xFFFFFFFF;
+		use_bank = 1;
+	} else {
+		use_bank = 0;
+	}
+
+	*pos &= (1UL << 22) - 1;
+
+	if (use_bank) {
+		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
+		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
+			return -EINVAL;
+		mutex_lock(&adev->grbm_idx_mutex);
+		amdgpu_gfx_select_se_sh(adev, se_bank,
+					sh_bank, instance_bank);
+	}
+
+	if (pm_pg_lock)
+		mutex_lock(&adev->pm.mutex);
+
+	while (size) {
+		uint32_t value;
+
+		if (*pos > adev->rmmio_size)
+			goto end;
+
+		value = RREG32(*pos >> 2);
+		r = put_user(value, (uint32_t *)buf);
+		if (r) {
+			result = r;
+			goto end;
+		}
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+end:
+	if (use_bank) {
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+		mutex_unlock(&adev->grbm_idx_mutex);
+	}
+
+	if (pm_pg_lock)
+		mutex_unlock(&adev->pm.mutex);
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
+					 size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+	bool pm_pg_lock, use_bank;
+	unsigned instance_bank, sh_bank, se_bank;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	/* are we reading registers for which a PG lock is necessary? */
+	pm_pg_lock = (*pos >> 23) & 1;
+
+	if (*pos & (1ULL << 62)) {
+		se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
+		sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
+		instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
+
+		if (se_bank == 0x3FF)
+			se_bank = 0xFFFFFFFF;
+		if (sh_bank == 0x3FF)
+			sh_bank = 0xFFFFFFFF;
+		if (instance_bank == 0x3FF)
+			instance_bank = 0xFFFFFFFF;
+		use_bank = 1;
+	} else {
+		use_bank = 0;
+	}
+
+	*pos &= (1UL << 22) - 1;
+
+	if (use_bank) {
+		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
+		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
+			return -EINVAL;
+		mutex_lock(&adev->grbm_idx_mutex);
+		amdgpu_gfx_select_se_sh(adev, se_bank,
+					sh_bank, instance_bank);
+	}
+
+	if (pm_pg_lock)
+		mutex_lock(&adev->pm.mutex);
+
+	while (size) {
+		uint32_t value;
+
+		if (*pos > adev->rmmio_size)
+			return result;
+
+		r = get_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		WREG32(*pos >> 2, value);
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	if (use_bank) {
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+		mutex_unlock(&adev->grbm_idx_mutex);
+	}
+
+	if (pm_pg_lock)
+		mutex_unlock(&adev->pm.mutex);
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	while (size) {
+		uint32_t value;
+
+		value = RREG32_PCIE(*pos >> 2);
+		r = put_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
+					 size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	while (size) {
+		uint32_t value;
+
+		r = get_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		WREG32_PCIE(*pos >> 2, value);
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	while (size) {
+		uint32_t value;
+
+		value = RREG32_DIDT(*pos >> 2);
+		r = put_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
+					 size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	while (size) {
+		uint32_t value;
+
+		r = get_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		WREG32_DIDT(*pos >> 2, value);
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	while (size) {
+		uint32_t value;
+
+		value = RREG32_SMC(*pos);
+		r = put_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
+					 size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	while (size) {
+		uint32_t value;
+
+		r = get_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		WREG32_SMC(*pos, value);
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+	uint32_t *config, no_regs = 0;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
+	if (!config)
+		return -ENOMEM;
+
+	/* version, increment each time something is added */
+	config[no_regs++] = 3;
+	config[no_regs++] = adev->gfx.config.max_shader_engines;
+	config[no_regs++] = adev->gfx.config.max_tile_pipes;
+	config[no_regs++] = adev->gfx.config.max_cu_per_sh;
+	config[no_regs++] = adev->gfx.config.max_sh_per_se;
+	config[no_regs++] = adev->gfx.config.max_backends_per_se;
+	config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
+	config[no_regs++] = adev->gfx.config.max_gprs;
+	config[no_regs++] = adev->gfx.config.max_gs_threads;
+	config[no_regs++] = adev->gfx.config.max_hw_contexts;
+	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
+	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
+	config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
+	config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
+	config[no_regs++] = adev->gfx.config.num_tile_pipes;
+	config[no_regs++] = adev->gfx.config.backend_enable_mask;
+	config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
+	config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
+	config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
+	config[no_regs++] = adev->gfx.config.num_gpus;
+	config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
+	config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
+	config[no_regs++] = adev->gfx.config.gb_addr_config;
+	config[no_regs++] = adev->gfx.config.num_rbs;
+
+	/* rev==1 */
+	config[no_regs++] = adev->rev_id;
+	config[no_regs++] = adev->pg_flags;
+	config[no_regs++] = adev->cg_flags;
+
+	/* rev==2 */
+	config[no_regs++] = adev->family;
+	config[no_regs++] = adev->external_rev_id;
+
+	/* rev==3 */
+	config[no_regs++] = adev->pdev->device;
+	config[no_regs++] = adev->pdev->revision;
+	config[no_regs++] = adev->pdev->subsystem_device;
+	config[no_regs++] = adev->pdev->subsystem_vendor;
+
+	while (size && (*pos < no_regs * 4)) {
+		uint32_t value;
+
+		value = config[*pos >> 2];
+		r = put_user(value, (uint32_t *)buf);
+		if (r) {
+			kfree(config);
+			return r;
+		}
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	kfree(config);
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	int idx, x, outsize, r, valuesize;
+	uint32_t values[16];
+
+	if (size & 3 || *pos & 0x3)
+		return -EINVAL;
+
+	if (amdgpu_dpm == 0)
+		return -EINVAL;
+
+	/* convert offset to sensor number */
+	idx = *pos >> 2;
+
+	valuesize = sizeof(values);
+	if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
+		r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
+	else
+		return -EINVAL;
+
+	if (size > valuesize)
+		return -EINVAL;
+
+	outsize = 0;
+	x = 0;
+	if (!r) {
+		while (size) {
+			r = put_user(values[x++], (int32_t *)buf);
+			buf += 4;
+			size -= 4;
+			outsize += 4;
+		}
+	}
+
+	return !r ? outsize : r;
+}
+
+static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = f->f_inode->i_private;
+	int r, x;
+	ssize_t result=0;
+	uint32_t offset, se, sh, cu, wave, simd, data[32];
+
+	if (size & 3 || *pos & 3)
+		return -EINVAL;
+
+	/* decode offset */
+	offset = (*pos & GENMASK_ULL(6, 0));
+	se = (*pos & GENMASK_ULL(14, 7)) >> 7;
+	sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
+	cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
+	wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
+	simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
+
+	/* switch to the specific se/sh/cu */
+	mutex_lock(&adev->grbm_idx_mutex);
+	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
+
+	x = 0;
+	if (adev->gfx.funcs->read_wave_data)
+		adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
+
+	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	if (!x)
+		return -EINVAL;
+
+	while (size && (offset < x * 4)) {
+		uint32_t value;
+
+		value = data[offset >> 2];
+		r = put_user(value, (uint32_t *)buf);
+		if (r)
+			return r;
+
+		result += 4;
+		buf += 4;
+		offset += 4;
+		size -= 4;
+	}
+
+	return result;
+}
+
+static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = f->f_inode->i_private;
+	int r;
+	ssize_t result = 0;
+	uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
+
+	if (size & 3 || *pos & 3)
+		return -EINVAL;
+
+	/* decode offset */
+	offset = *pos & GENMASK_ULL(11, 0);
+	se = (*pos & GENMASK_ULL(19, 12)) >> 12;
+	sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
+	cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
+	wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
+	simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
+	thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
+	bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
+
+	data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	/* switch to the specific se/sh/cu */
+	mutex_lock(&adev->grbm_idx_mutex);
+	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
+
+	if (bank == 0) {
+		if (adev->gfx.funcs->read_wave_vgprs)
+			adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
+	} else {
+		if (adev->gfx.funcs->read_wave_sgprs)
+			adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
+	}
+
+	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
+	mutex_unlock(&adev->grbm_idx_mutex);
+
+	while (size) {
+		uint32_t value;
+
+		value = data[offset++];
+		r = put_user(value, (uint32_t *)buf);
+		if (r) {
+			result = r;
+			goto err;
+		}
+
+		result += 4;
+		buf += 4;
+		size -= 4;
+	}
+
+err:
+	kfree(data);
+	return result;
+}
+
+static const struct file_operations amdgpu_debugfs_regs_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_regs_read,
+	.write = amdgpu_debugfs_regs_write,
+	.llseek = default_llseek
+};
+static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_regs_didt_read,
+	.write = amdgpu_debugfs_regs_didt_write,
+	.llseek = default_llseek
+};
+static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_regs_pcie_read,
+	.write = amdgpu_debugfs_regs_pcie_write,
+	.llseek = default_llseek
+};
+static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_regs_smc_read,
+	.write = amdgpu_debugfs_regs_smc_write,
+	.llseek = default_llseek
+};
+
+static const struct file_operations amdgpu_debugfs_gca_config_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_gca_config_read,
+	.llseek = default_llseek
+};
+
+static const struct file_operations amdgpu_debugfs_sensors_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_sensor_read,
+	.llseek = default_llseek
+};
+
+static const struct file_operations amdgpu_debugfs_wave_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_wave_read,
+	.llseek = default_llseek
+};
+static const struct file_operations amdgpu_debugfs_gpr_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_gpr_read,
+	.llseek = default_llseek
+};
+
+static const struct file_operations *debugfs_regs[] = {
+	&amdgpu_debugfs_regs_fops,
+	&amdgpu_debugfs_regs_didt_fops,
+	&amdgpu_debugfs_regs_pcie_fops,
+	&amdgpu_debugfs_regs_smc_fops,
+	&amdgpu_debugfs_gca_config_fops,
+	&amdgpu_debugfs_sensors_fops,
+	&amdgpu_debugfs_wave_fops,
+	&amdgpu_debugfs_gpr_fops,
+};
+
+static const char *debugfs_regs_names[] = {
+	"amdgpu_regs",
+	"amdgpu_regs_didt",
+	"amdgpu_regs_pcie",
+	"amdgpu_regs_smc",
+	"amdgpu_gca_config",
+	"amdgpu_sensors",
+	"amdgpu_wave",
+	"amdgpu_gpr",
+};
+
+int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
+{
+	struct drm_minor *minor = adev->ddev->primary;
+	struct dentry *ent, *root = minor->debugfs_root;
+	unsigned i, j;
+
+	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
+		ent = debugfs_create_file(debugfs_regs_names[i],
+					  S_IFREG | S_IRUGO, root,
+					  adev, debugfs_regs[i]);
+		if (IS_ERR(ent)) {
+			for (j = 0; j < i; j++) {
+				debugfs_remove(adev->debugfs_regs[i]);
+				adev->debugfs_regs[i] = NULL;
+			}
+			return PTR_ERR(ent);
+		}
+
+		if (!i)
+			i_size_write(ent->d_inode, adev->rmmio_size);
+		adev->debugfs_regs[i] = ent;
+	}
+
+	return 0;
+}
+
+void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
+{
+	unsigned i;
+
+	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
+		if (adev->debugfs_regs[i]) {
+			debugfs_remove(adev->debugfs_regs[i]);
+			adev->debugfs_regs[i] = NULL;
+		}
+	}
+}
+
+static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+	int r = 0, i;
+
+	/* hold on the scheduler */
+	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+		kthread_park(ring->sched.thread);
+	}
+
+	seq_printf(m, "run ib test:\n");
+	r = amdgpu_ib_ring_tests(adev);
+	if (r)
+		seq_printf(m, "ib ring tests failed (%d).\n", r);
+	else
+		seq_printf(m, "ib ring tests passed.\n");
+
+	/* go on the scheduler */
+	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
+		struct amdgpu_ring *ring = adev->rings[i];
+
+		if (!ring || !ring->sched.thread)
+			continue;
+		kthread_unpark(ring->sched.thread);
+	}
+
+	return 0;
+}
+
+static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+
+	seq_write(m, adev->bios, adev->bios_size);
+	return 0;
+}
+
+static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct amdgpu_device *adev = dev->dev_private;
+
+	seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
+	return 0;
+}
+
+static const struct drm_info_list amdgpu_debugfs_list[] = {
+	{"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
+	{"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
+	{"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram}
+};
+
+int amdgpu_debugfs_init(struct amdgpu_device *adev)
+{
+	return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
+					ARRAY_SIZE(amdgpu_debugfs_list));
+}
+
+#else
+int amdgpu_debugfs_init(struct amdgpu_device *adev)
+{
+	return 0;
+}
+int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
+{
+	return 0;
+}
+void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
new file mode 100644
index 000000000000..8260d8073c26
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2008 Advanced Micro Devices, Inc.
+ * Copyright 2008 Red Hat Inc.
+ * Copyright 2009 Jerome Glisse.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+/*
+ * Debugfs
+ */
+struct amdgpu_debugfs {
+	const struct drm_info_list	*files;
+	unsigned		num_files;
+};
+
+int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
+void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
+int amdgpu_debugfs_init(struct amdgpu_device *adev);
+int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
+			     const struct drm_info_list *files,
+			     unsigned nfiles);
+int amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
+int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
+int amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 3573ecdb06ee..66cb10cdc7c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -28,7 +28,6 @@
 #include <linux/kthread.h>
 #include <linux/console.h>
 #include <linux/slab.h>
-#include <linux/debugfs.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_atomic_helper.h>
@@ -64,11 +63,6 @@ MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
 
 #define AMDGPU_RESUME_MS		2000
 
-static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
-static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
-static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev);
-static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev);
-
 static const char *amdgpu_asic_name[] = {
 	"TAHITI",
 	"PITCAIRN",
@@ -333,7 +327,7 @@ static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 	BUG();
 }
 
-static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
+static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 {
 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 				       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
@@ -342,13 +336,13 @@ static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 				       (void **)&adev->vram_scratch.ptr);
 }
 
-static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
+static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 {
 	amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 }
 
 /**
- * amdgpu_program_register_sequence - program an array of registers.
+ * amdgpu_device_program_register_sequence - program an array of registers.
  *
  * @adev: amdgpu_device pointer
  * @registers: pointer to the register array
@@ -357,9 +351,9 @@ static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
  * Programs an array or registers with and and or masks.
  * This is a helper for setting golden registers.
  */
-void amdgpu_program_register_sequence(struct amdgpu_device *adev,
-				      const u32 *registers,
-				      const u32 array_size)
+void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
+					     const u32 *registers,
+					     const u32 array_size)
 {
 	u32 tmp, reg, and_mask, or_mask;
 	int i;
@@ -383,7 +377,7 @@ void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 	}
 }
 
-void amdgpu_pci_config_reset(struct amdgpu_device *adev)
+void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
 {
 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 }
@@ -392,14 +386,14 @@ void amdgpu_pci_config_reset(struct amdgpu_device *adev)
  * GPU doorbell aperture helpers function.
  */
 /**
- * amdgpu_doorbell_init - Init doorbell driver information.
+ * amdgpu_device_doorbell_init - Init doorbell driver information.
  *
  * @adev: amdgpu_device pointer
  *
  * Init doorbell driver information (CIK)
  * Returns 0 on success, error on failure.
  */
-static int amdgpu_doorbell_init(struct amdgpu_device *adev)
+static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 {
 	/* No doorbell on SI hardware generation */
 	if (adev->asic_type < CHIP_BONAIRE) {
@@ -410,6 +404,9 @@ static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 		return 0;
 	}
 
+	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
+		return -EINVAL;
+
 	/* doorbell bar mapping */
 	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 	adev->doorbell.size = pci_resource_len(adev->pdev, 2);
@@ -429,66 +426,35 @@ static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 }
 
 /**
- * amdgpu_doorbell_fini - Tear down doorbell driver information.
+ * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
  *
  * @adev: amdgpu_device pointer
  *
  * Tear down doorbell driver information (CIK)
  */
-static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
+static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
 {
 	iounmap(adev->doorbell.ptr);
 	adev->doorbell.ptr = NULL;
 }
 
-/**
- * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
- *                                setup amdkfd
- *
- * @adev: amdgpu_device pointer
- * @aperture_base: output returning doorbell aperture base physical address
- * @aperture_size: output returning doorbell aperture size in bytes
- * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
- *
- * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
- * takes doorbells required for its own rings and reports the setup to amdkfd.
- * amdgpu reserved doorbells are at the start of the doorbell aperture.
- */
-void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
-				phys_addr_t *aperture_base,
-				size_t *aperture_size,
-				size_t *start_offset)
-{
-	/*
-	 * The first num_doorbells are used by amdgpu.
-	 * amdkfd takes whatever's left in the aperture.
-	 */
-	if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
-		*aperture_base = adev->doorbell.base;
-		*aperture_size = adev->doorbell.size;
-		*start_offset = adev->doorbell.num_doorbells * sizeof(u32);
-	} else {
-		*aperture_base = 0;
-		*aperture_size = 0;
-		*start_offset = 0;
-	}
-}
+
 
 /*
- * amdgpu_wb_*()
+ * amdgpu_device_wb_*()
  * Writeback is the method by which the GPU updates special pages in memory
  * with the status of certain GPU events (fences, ring pointers,etc.).
  */
 
 /**
- * amdgpu_wb_fini - Disable Writeback and free memory
+ * amdgpu_device_wb_fini - Disable Writeback and free memory
  *
  * @adev: amdgpu_device pointer
  *
  * Disables Writeback and frees the Writeback memory (all asics).
  * Used at driver shutdown.
  */
-static void amdgpu_wb_fini(struct amdgpu_device *adev)
+static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
 {
 	if (adev->wb.wb_obj) {
 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
@@ -499,7 +465,7 @@ static void amdgpu_wb_fini(struct amdgpu_device *adev)
 }
 
 /**
- * amdgpu_wb_init- Init Writeback driver info and allocate memory
+ * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
  *
  * @adev: amdgpu_device pointer
  *
@@ -507,7 +473,7 @@ static void amdgpu_wb_fini(struct amdgpu_device *adev)
  * Used at driver startup.
  * Returns 0 on success or an -error on failure.
  */
-static int amdgpu_wb_init(struct amdgpu_device *adev)
+static int amdgpu_device_wb_init(struct amdgpu_device *adev)
 {
 	int r;
 
@@ -526,14 +492,14 @@ static int amdgpu_wb_init(struct amdgpu_device *adev)
 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 
 		/* clear wb memory */
-		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
+		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
 	}
 
 	return 0;
 }
 
 /**
- * amdgpu_wb_get - Allocate a wb entry
+ * amdgpu_device_wb_get - Allocate a wb entry
  *
  * @adev: amdgpu_device pointer
  * @wb: wb index
@@ -541,7 +507,7 @@ static int amdgpu_wb_init(struct amdgpu_device *adev)
  * Allocate a wb slot for use by the driver (all asics).
  * Returns 0 on success or -EINVAL on failure.
  */
-int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
+int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
 {
 	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 
@@ -555,61 +521,35 @@ int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 }
 
 /**
- * amdgpu_wb_free - Free a wb entry
+ * amdgpu_device_wb_free - Free a wb entry
  *
  * @adev: amdgpu_device pointer
  * @wb: wb index
  *
  * Free a wb slot allocated for use by the driver (all asics)
  */
-void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
+void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
 {
+	wb >>= 3;
 	if (wb < adev->wb.num_wb)
-		__clear_bit(wb >> 3, adev->wb.used);
+		__clear_bit(wb, adev->wb.used);
 }
 
 /**
- * amdgpu_vram_location - try to find VRAM location
+ * amdgpu_device_vram_location - try to find VRAM location
  * @adev: amdgpu device structure holding all necessary informations
  * @mc: memory controller structure holding memory informations
  * @base: base address at which to put VRAM
  *
  * Function will try to place VRAM at base address provided
- * as parameter (which is so far either PCI aperture address or
- * for IGP TOM base address).
- *
- * If there is not enough space to fit the unvisible VRAM in the 32bits
- * address space then we limit the VRAM size to the aperture.
- *
- * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
- * this shouldn't be a problem as we are using the PCI aperture as a reference.
- * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
- * not IGP.
- *
- * Note: we use mc_vram_size as on some board we need to program the mc to
- * cover the whole aperture even if VRAM size is inferior to aperture size
- * Novell bug 204882 + along with lots of ubuntu ones
- *
- * Note: when limiting vram it's safe to overwritte real_vram_size because
- * we are not in case where real_vram_size is inferior to mc_vram_size (ie
- * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
- * ones)
- *
- * Note: IGP TOM addr should be the same as the aperture addr, we don't
- * explicitly check for that though.
- *
- * FIXME: when reducing VRAM size align new size on power of 2.
+ * as parameter.
  */
-void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
+void amdgpu_device_vram_location(struct amdgpu_device *adev,
+				 struct amdgpu_mc *mc, u64 base)
 {
 	uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 
 	mc->vram_start = base;
-	if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
-		dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
-		mc->real_vram_size = mc->aper_size;
-		mc->mc_vram_size = mc->aper_size;
-	}
 	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 	if (limit && limit < mc->real_vram_size)
 		mc->real_vram_size = limit;
@@ -619,7 +559,7 @@ void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64
 }
 
 /**
- * amdgpu_gart_location - try to find GTT location
+ * amdgpu_device_gart_location - try to find GTT location
  * @adev: amdgpu device structure holding all necessary informations
  * @mc: memory controller structure holding memory informations
  *
@@ -630,7 +570,8 @@ void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64
  *
  * FIXME: when reducing GTT size align new size on power of 2.
  */
-void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
+void amdgpu_device_gart_location(struct amdgpu_device *adev,
+				 struct amdgpu_mc *mc)
 {
 	u64 size_af, size_bf;
 
@@ -647,93 +588,91 @@ void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 			dev_warn(adev->dev, "limiting GTT\n");
 			mc->gart_size = size_af;
 		}
-		mc->gart_start = mc->vram_end + 1;
+		/* VCE doesn't like it when BOs cross a 4GB segment, so align
+		 * the GART base on a 4GB boundary as well.
+		 */
+		mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
 	}
 	mc->gart_end = mc->gart_start + mc->gart_size - 1;
 	dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 			mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 }
 
-/*
- * Firmware Reservation functions
- */
 /**
- * amdgpu_fw_reserve_vram_fini - free fw reserved vram
+ * amdgpu_device_resize_fb_bar - try to resize FB BAR
  *
  * @adev: amdgpu_device pointer
  *
- * free fw reserved vram if it has been reserved.
+ * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
+ * to fail, but if any of the BARs is not accessible after the size we abort
+ * driver loading by returning -ENODEV.
  */
-void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
+int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
 {
-	amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
-		NULL, &adev->fw_vram_usage.va);
-}
+	u64 space_needed = roundup_pow_of_two(adev->mc.real_vram_size);
+	u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
+	struct pci_bus *root;
+	struct resource *res;
+	unsigned i;
+	u16 cmd;
+	int r;
 
-/**
- * amdgpu_fw_reserve_vram_init - create bo vram reservation from fw
- *
- * @adev: amdgpu_device pointer
- *
- * create bo vram reservation from fw.
- */
-int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
-{
-	int r = 0;
-	u64 gpu_addr;
-	u64 vram_size = adev->mc.visible_vram_size;
+	/* Bypass for VF */
+	if (amdgpu_sriov_vf(adev))
+		return 0;
 
-	adev->fw_vram_usage.va = NULL;
-	adev->fw_vram_usage.reserved_bo = NULL;
+	/* Check if the root BUS has 64bit memory resources */
+	root = adev->pdev->bus;
+	while (root->parent)
+		root = root->parent;
 
-	if (adev->fw_vram_usage.size > 0 &&
-		adev->fw_vram_usage.size <= vram_size) {
+	pci_bus_for_each_resource(root, res, i) {
+		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
+		    res->start > 0x100000000ull)
+			break;
+	}
 
-		r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
-			PAGE_SIZE, true, 0,
-			AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
-			AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
-			&adev->fw_vram_usage.reserved_bo);
-		if (r)
-			goto error_create;
+	/* Trying to resize is pointless without a root hub window above 4GB */
+	if (!res)
+		return 0;
 
-		r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
-		if (r)
-			goto error_reserve;
-		r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
-			AMDGPU_GEM_DOMAIN_VRAM,
-			adev->fw_vram_usage.start_offset,
-			(adev->fw_vram_usage.start_offset +
-			adev->fw_vram_usage.size), &gpu_addr);
-		if (r)
-			goto error_pin;
-		r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
-			&adev->fw_vram_usage.va);
-		if (r)
-			goto error_kmap;
+	/* Disable memory decoding while we change the BAR addresses and size */
+	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
+	pci_write_config_word(adev->pdev, PCI_COMMAND,
+			      cmd & ~PCI_COMMAND_MEMORY);
 
-		amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
-	}
-	return r;
+	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
+	amdgpu_device_doorbell_fini(adev);
+	if (adev->asic_type >= CHIP_BONAIRE)
+		pci_release_resource(adev->pdev, 2);
 
-error_kmap:
-	amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
-error_pin:
-	amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
-error_reserve:
-	amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
-error_create:
-	adev->fw_vram_usage.va = NULL;
-	adev->fw_vram_usage.reserved_bo = NULL;
-	return r;
-}
+	pci_release_resource(adev->pdev, 0);
+
+	r = pci_resize_resource(adev->pdev, 0, rbar_size);
+	if (r == -ENOSPC)
+		DRM_INFO("Not enough PCI address space for a large BAR.");
+	else if (r && r != -ENOTSUPP)
+		DRM_ERROR("Problem resizing BAR0 (%d).", r);
+
+	pci_assign_unassigned_bus_resources(adev->pdev->bus);
+
+	/* When the doorbell or fb BAR isn't available we have no chance of
+	 * using the device.
+	 */
+	r = amdgpu_device_doorbell_init(adev);
+	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
+		return -ENODEV;
 
+	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
+
+	return 0;
+}
 
 /*
  * GPU helpers function.
  */
 /**
- * amdgpu_need_post - check if the hw need post or not
+ * amdgpu_device_need_post - check if the hw need post or not
  *
  * @adev: amdgpu_device pointer
  *
@@ -741,7 +680,7 @@ int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
  * or post is needed if  hw reset is performed.
  * Returns true if need or false if not.
  */
-bool amdgpu_need_post(struct amdgpu_device *adev)
+bool amdgpu_device_need_post(struct amdgpu_device *adev)
 {
 	uint32_t reg;
 
@@ -786,285 +725,9 @@ bool amdgpu_need_post(struct amdgpu_device *adev)
 	return true;
 }
 
-/**
- * amdgpu_dummy_page_init - init dummy page used by the driver
- *
- * @adev: amdgpu_device pointer
- *
- * Allocate the dummy page used by the driver (all asics).
- * This dummy page is used by the driver as a filler for gart entries
- * when pages are taken out of the GART
- * Returns 0 on sucess, -ENOMEM on failure.
- */
-int amdgpu_dummy_page_init(struct amdgpu_device *adev)
-{
-	if (adev->dummy_page.page)
-		return 0;
-	adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
-	if (adev->dummy_page.page == NULL)
-		return -ENOMEM;
-	adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
-					0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
-		dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
-		__free_page(adev->dummy_page.page);
-		adev->dummy_page.page = NULL;
-		return -ENOMEM;
-	}
-	return 0;
-}
-
-/**
- * amdgpu_dummy_page_fini - free dummy page used by the driver
- *
- * @adev: amdgpu_device pointer
- *
- * Frees the dummy page used by the driver (all asics).
- */
-void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
-{
-	if (adev->dummy_page.page == NULL)
-		return;
-	pci_unmap_page(adev->pdev, adev->dummy_page.addr,
-			PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	__free_page(adev->dummy_page.page);
-	adev->dummy_page.page = NULL;
-}
-
-
-/* ATOM accessor methods */
-/*
- * ATOM is an interpreted byte code stored in tables in the vbios.  The
- * driver registers callbacks to access registers and the interpreter
- * in the driver parses the tables and executes then to program specific
- * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
- * atombios.h, and atom.c
- */
-
-/**
- * cail_pll_read - read PLL register
- *
- * @info: atom card_info pointer
- * @reg: PLL register offset
- *
- * Provides a PLL register accessor for the atom interpreter (r4xx+).
- * Returns the value of the PLL register.
- */
-static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
-{
-	return 0;
-}
-
-/**
- * cail_pll_write - write PLL register
- *
- * @info: atom card_info pointer
- * @reg: PLL register offset
- * @val: value to write to the pll register
- *
- * Provides a PLL register accessor for the atom interpreter (r4xx+).
- */
-static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
-{
-
-}
-
-/**
- * cail_mc_read - read MC (Memory Controller) register
- *
- * @info: atom card_info pointer
- * @reg: MC register offset
- *
- * Provides an MC register accessor for the atom interpreter (r4xx+).
- * Returns the value of the MC register.
- */
-static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
-{
-	return 0;
-}
-
-/**
- * cail_mc_write - write MC (Memory Controller) register
- *
- * @info: atom card_info pointer
- * @reg: MC register offset
- * @val: value to write to the pll register
- *
- * Provides a MC register accessor for the atom interpreter (r4xx+).
- */
-static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
-{
-
-}
-
-/**
- * cail_reg_write - write MMIO register
- *
- * @info: atom card_info pointer
- * @reg: MMIO register offset
- * @val: value to write to the pll register
- *
- * Provides a MMIO register accessor for the atom interpreter (r4xx+).
- */
-static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
-{
-	struct amdgpu_device *adev = info->dev->dev_private;
-
-	WREG32(reg, val);
-}
-
-/**
- * cail_reg_read - read MMIO register
- *
- * @info: atom card_info pointer
- * @reg: MMIO register offset
- *
- * Provides an MMIO register accessor for the atom interpreter (r4xx+).
- * Returns the value of the MMIO register.
- */
-static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
-{
-	struct amdgpu_device *adev = info->dev->dev_private;
-	uint32_t r;
-
-	r = RREG32(reg);
-	return r;
-}
-
-/**
- * cail_ioreg_write - write IO register
- *
- * @info: atom card_info pointer
- * @reg: IO register offset
- * @val: value to write to the pll register
- *
- * Provides a IO register accessor for the atom interpreter (r4xx+).
- */
-static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
-{
-	struct amdgpu_device *adev = info->dev->dev_private;
-
-	WREG32_IO(reg, val);
-}
-
-/**
- * cail_ioreg_read - read IO register
- *
- * @info: atom card_info pointer
- * @reg: IO register offset
- *
- * Provides an IO register accessor for the atom interpreter (r4xx+).
- * Returns the value of the IO register.
- */
-static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
-{
-	struct amdgpu_device *adev = info->dev->dev_private;
-	uint32_t r;
-
-	r = RREG32_IO(reg);
-	return r;
-}
-
-static ssize_t amdgpu_atombios_get_vbios_version(struct device *dev,
-						 struct device_attribute *attr,
-						 char *buf)
-{
-	struct drm_device *ddev = dev_get_drvdata(dev);
-	struct amdgpu_device *adev = ddev->dev_private;
-	struct atom_context *ctx = adev->mode_info.atom_context;
-
-	return snprintf(buf, PAGE_SIZE, "%s\n", ctx->vbios_version);
-}
-
-static DEVICE_ATTR(vbios_version, 0444, amdgpu_atombios_get_vbios_version,
-		   NULL);
-
-/**
- * amdgpu_atombios_fini - free the driver info and callbacks for atombios
- *
- * @adev: amdgpu_device pointer
- *
- * Frees the driver info and register access callbacks for the ATOM
- * interpreter (r4xx+).
- * Called at driver shutdown.
- */
-static void amdgpu_atombios_fini(struct amdgpu_device *adev)
-{
-	if (adev->mode_info.atom_context) {
-		kfree(adev->mode_info.atom_context->scratch);
-		kfree(adev->mode_info.atom_context->iio);
-	}
-	kfree(adev->mode_info.atom_context);
-	adev->mode_info.atom_context = NULL;
-	kfree(adev->mode_info.atom_card_info);
-	adev->mode_info.atom_card_info = NULL;
-	device_remove_file(adev->dev, &dev_attr_vbios_version);
-}
-
-/**
- * amdgpu_atombios_init - init the driver info and callbacks for atombios
- *
- * @adev: amdgpu_device pointer
- *
- * Initializes the driver info and register access callbacks for the
- * ATOM interpreter (r4xx+).
- * Returns 0 on sucess, -ENOMEM on failure.
- * Called at driver startup.
- */
-static int amdgpu_atombios_init(struct amdgpu_device *adev)
-{
-	struct card_info *atom_card_info =
-	    kzalloc(sizeof(struct card_info), GFP_KERNEL);
-	int ret;
-
-	if (!atom_card_info)
-		return -ENOMEM;
-
-	adev->mode_info.atom_card_info = atom_card_info;
-	atom_card_info->dev = adev->ddev;
-	atom_card_info->reg_read = cail_reg_read;
-	atom_card_info->reg_write = cail_reg_write;
-	/* needed for iio ops */
-	if (adev->rio_mem) {
-		atom_card_info->ioreg_read = cail_ioreg_read;
-		atom_card_info->ioreg_write = cail_ioreg_write;
-	} else {
-		DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
-		atom_card_info->ioreg_read = cail_reg_read;
-		atom_card_info->ioreg_write = cail_reg_write;
-	}
-	atom_card_info->mc_read = cail_mc_read;
-	atom_card_info->mc_write = cail_mc_write;
-	atom_card_info->pll_read = cail_pll_read;
-	atom_card_info->pll_write = cail_pll_write;
-
-	adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
-	if (!adev->mode_info.atom_context) {
-		amdgpu_atombios_fini(adev);
-		return -ENOMEM;
-	}
-
-	mutex_init(&adev->mode_info.atom_context->mutex);
-	if (adev->is_atom_fw) {
-		amdgpu_atomfirmware_scratch_regs_init(adev);
-		amdgpu_atomfirmware_allocate_fb_scratch(adev);
-	} else {
-		amdgpu_atombios_scratch_regs_init(adev);
-		amdgpu_atombios_allocate_fb_scratch(adev);
-	}
-
-	ret = device_create_file(adev->dev, &dev_attr_vbios_version);
-	if (ret) {
-		DRM_ERROR("Failed to create device file for VBIOS version\n");
-		return ret;
-	}
-
-	return 0;
-}
-
 /* if we get transitioned to only one device, take VGA back */
 /**
- * amdgpu_vga_set_decode - enable/disable vga decode
+ * amdgpu_device_vga_set_decode - enable/disable vga decode
  *
  * @cookie: amdgpu_device pointer
  * @state: enable/disable vga decode
@@ -1072,7 +735,7 @@ static int amdgpu_atombios_init(struct amdgpu_device *adev)
  * Enable/disable vga decode (all asics).
  * Returns VGA resource flags.
  */
-static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
+static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
 {
 	struct amdgpu_device *adev = cookie;
 	amdgpu_asic_set_vga_state(adev, state);
@@ -1083,7 +746,7 @@ static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 }
 
-static void amdgpu_check_block_size(struct amdgpu_device *adev)
+static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
 {
 	/* defines number of bits in page table versus page directory,
 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
@@ -1094,64 +757,32 @@ static void amdgpu_check_block_size(struct amdgpu_device *adev)
 	if (amdgpu_vm_block_size < 9) {
 		dev_warn(adev->dev, "VM page table size (%d) too small\n",
 			 amdgpu_vm_block_size);
-		goto def_value;
+		amdgpu_vm_block_size = -1;
 	}
-
-	if (amdgpu_vm_block_size > 24 ||
-	    (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
-		dev_warn(adev->dev, "VM page table size (%d) too large\n",
-			 amdgpu_vm_block_size);
-		goto def_value;
-	}
-
-	return;
-
-def_value:
-	amdgpu_vm_block_size = -1;
 }
 
-static void amdgpu_check_vm_size(struct amdgpu_device *adev)
+static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
 {
 	/* no need to check the default value */
 	if (amdgpu_vm_size == -1)
 		return;
 
-	if (!is_power_of_2(amdgpu_vm_size)) {
-		dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
-			 amdgpu_vm_size);
-		goto def_value;
-	}
-
 	if (amdgpu_vm_size < 1) {
 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 			 amdgpu_vm_size);
-		goto def_value;
-	}
-
-	/*
-	 * Max GPUVM size for Cayman, SI, CI VI are 40 bits.
-	 */
-	if (amdgpu_vm_size > 1024) {
-		dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
-			 amdgpu_vm_size);
-		goto def_value;
+		amdgpu_vm_size = -1;
 	}
-
-	return;
-
-def_value:
-	amdgpu_vm_size = -1;
 }
 
 /**
- * amdgpu_check_arguments - validate module params
+ * amdgpu_device_check_arguments - validate module params
  *
  * @adev: amdgpu_device pointer
  *
  * Validates certain module parameters and updates
  * the associated values used by the driver (all asics).
  */
-static void amdgpu_check_arguments(struct amdgpu_device *adev)
+static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
 {
 	if (amdgpu_sched_jobs < 4) {
 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
@@ -1184,9 +815,9 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 		amdgpu_vm_fragment_size = -1;
 	}
 
-	amdgpu_check_vm_size(adev);
+	amdgpu_device_check_vm_size(adev);
 
-	amdgpu_check_block_size(adev);
+	amdgpu_device_check_block_size(adev);
 
 	if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
 	    !is_power_of_2(amdgpu_vram_page_split))) {
@@ -1194,6 +825,11 @@ static void amdgpu_check_arguments(struct amdgpu_device *adev)
 			 amdgpu_vram_page_split);
 		amdgpu_vram_page_split = 1024;
 	}
+
+	if (amdgpu_lockup_timeout == 0) {
+		dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
+		amdgpu_lockup_timeout = 10000;
+	}
 }
 
 /**
@@ -1257,9 +893,9 @@ static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
 	.can_switch = amdgpu_switcheroo_can_switch,
 };
 
-int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
-				  enum amd_ip_block_type block_type,
-				  enum amd_clockgating_state state)
+int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_clockgating_state state)
 {
 	int i, r = 0;
 
@@ -1279,9 +915,9 @@ int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
 	return r;
 }
 
-int amdgpu_set_powergating_state(struct amdgpu_device *adev,
-				  enum amd_ip_block_type block_type,
-				  enum amd_powergating_state state)
+int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
+					   enum amd_ip_block_type block_type,
+					   enum amd_powergating_state state)
 {
 	int i, r = 0;
 
@@ -1301,7 +937,8 @@ int amdgpu_set_powergating_state(struct amdgpu_device *adev,
 	return r;
 }
 
-void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
+void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
+					    u32 *flags)
 {
 	int i;
 
@@ -1313,8 +950,8 @@ void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
 	}
 }
 
-int amdgpu_wait_for_idle(struct amdgpu_device *adev,
-			 enum amd_ip_block_type block_type)
+int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
+				   enum amd_ip_block_type block_type)
 {
 	int i, r;
 
@@ -1332,8 +969,8 @@ int amdgpu_wait_for_idle(struct amdgpu_device *adev,
 
 }
 
-bool amdgpu_is_idle(struct amdgpu_device *adev,
-		    enum amd_ip_block_type block_type)
+bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
+			      enum amd_ip_block_type block_type)
 {
 	int i;
 
@@ -1347,8 +984,9 @@ bool amdgpu_is_idle(struct amdgpu_device *adev,
 
 }
 
-struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
-					     enum amd_ip_block_type type)
+struct amdgpu_ip_block *
+amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
+			      enum amd_ip_block_type type)
 {
 	int i;
 
@@ -1360,7 +998,7 @@ struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_ip_block_version_cmp
+ * amdgpu_device_ip_block_version_cmp
  *
  * @adev: amdgpu_device pointer
  * @type: enum amd_ip_block_type
@@ -1370,11 +1008,11 @@ struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
  * return 0 if equal or greater
  * return 1 if smaller or the ip_block doesn't exist
  */
-int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
-				enum amd_ip_block_type type,
-				u32 major, u32 minor)
+int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
+				       enum amd_ip_block_type type,
+				       u32 major, u32 minor)
 {
-	struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
+	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
 
 	if (ip_block && ((ip_block->version->major > major) ||
 			((ip_block->version->major == major) &&
@@ -1385,7 +1023,7 @@ int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_ip_block_add
+ * amdgpu_device_ip_block_add
  *
  * @adev: amdgpu_device pointer
  * @ip_block_version: pointer to the IP to add
@@ -1393,8 +1031,8 @@ int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
  * Adds the IP block driver information to the collection of IPs
  * on the asic.
  */
-int amdgpu_ip_block_add(struct amdgpu_device *adev,
-			const struct amdgpu_ip_block_version *ip_block_version)
+int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
+			       const struct amdgpu_ip_block_version *ip_block_version)
 {
 	if (!ip_block_version)
 		return -EINVAL;
@@ -1550,7 +1188,7 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
 	return err;
 }
 
-static int amdgpu_early_init(struct amdgpu_device *adev)
+static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 {
 	int i, r;
 
@@ -1622,10 +1260,12 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 	if (r)
 		return r;
 
+	amdgpu_amdkfd_device_probe(adev);
+
 	if (amdgpu_sriov_vf(adev)) {
 		r = amdgpu_virt_request_full_gpu(adev, true);
 		if (r)
-			return r;
+			return -EAGAIN;
 	}
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
@@ -1657,7 +1297,7 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_init(struct amdgpu_device *adev)
+static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 {
 	int i, r;
 
@@ -1673,7 +1313,7 @@ static int amdgpu_init(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.sw = true;
 		/* need to do gmc hw init early so we can allocate gpu mem */
 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
-			r = amdgpu_vram_scratch_init(adev);
+			r = amdgpu_device_vram_scratch_init(adev);
 			if (r) {
 				DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
 				return r;
@@ -1683,9 +1323,9 @@ static int amdgpu_init(struct amdgpu_device *adev)
 				DRM_ERROR("hw_init %d failed %d\n", i, r);
 				return r;
 			}
-			r = amdgpu_wb_init(adev);
+			r = amdgpu_device_wb_init(adev);
 			if (r) {
-				DRM_ERROR("amdgpu_wb_init failed %d\n", r);
+				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
 				return r;
 			}
 			adev->ip_blocks[i].status.hw = true;
@@ -1716,21 +1356,26 @@ static int amdgpu_init(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.hw = true;
 	}
 
+	amdgpu_amdkfd_device_init(adev);
+
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, true);
+
 	return 0;
 }
 
-static void amdgpu_fill_reset_magic(struct amdgpu_device *adev)
+static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
 {
 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
 }
 
-static bool amdgpu_check_vram_lost(struct amdgpu_device *adev)
+static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
 {
 	return !!memcmp(adev->gart.ptr, adev->reset_magic,
 			AMDGPU_RESET_MAGIC_NUM);
 }
 
-static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
+static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
 {
 	int i = 0, r;
 
@@ -1753,7 +1398,7 @@ static int amdgpu_late_set_cg_state(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_late_init(struct amdgpu_device *adev)
+static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
 {
 	int i = 0, r;
 
@@ -1774,15 +1419,16 @@ static int amdgpu_late_init(struct amdgpu_device *adev)
 	mod_delayed_work(system_wq, &adev->late_init_work,
 			msecs_to_jiffies(AMDGPU_RESUME_MS));
 
-	amdgpu_fill_reset_magic(adev);
+	amdgpu_device_fill_reset_magic(adev);
 
 	return 0;
 }
 
-static int amdgpu_fini(struct amdgpu_device *adev)
+static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 {
 	int i, r;
 
+	amdgpu_amdkfd_device_fini(adev);
 	/* need to disable SMC first */
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if (!adev->ip_blocks[i].status.hw)
@@ -1810,10 +1456,6 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
 		if (!adev->ip_blocks[i].status.hw)
 			continue;
-		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
-			amdgpu_wb_fini(adev);
-			amdgpu_vram_scratch_fini(adev);
-		}
 
 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
 			adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
@@ -1840,6 +1482,13 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
 		if (!adev->ip_blocks[i].status.sw)
 			continue;
+
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
+			amdgpu_free_static_csa(adev);
+			amdgpu_device_wb_fini(adev);
+			amdgpu_device_vram_scratch_fini(adev);
+		}
+
 		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
 		/* XXX handle errors */
 		if (r) {
@@ -1859,19 +1508,20 @@ static int amdgpu_fini(struct amdgpu_device *adev)
 	}
 
 	if (amdgpu_sriov_vf(adev))
-		amdgpu_virt_release_full_gpu(adev, false);
+		if (amdgpu_virt_release_full_gpu(adev, false))
+			DRM_ERROR("failed to release exclusive mode on fini\n");
 
 	return 0;
 }
 
-static void amdgpu_late_init_func_handler(struct work_struct *work)
+static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
 {
 	struct amdgpu_device *adev =
 		container_of(work, struct amdgpu_device, late_init_work.work);
-	amdgpu_late_set_cg_state(adev);
+	amdgpu_device_ip_late_set_cg_state(adev);
 }
 
-int amdgpu_suspend(struct amdgpu_device *adev)
+int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
 {
 	int i, r;
 
@@ -1879,10 +1529,10 @@ int amdgpu_suspend(struct amdgpu_device *adev)
 		amdgpu_virt_request_full_gpu(adev, false);
 
 	/* ungate SMC block first */
-	r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
-					 AMD_CG_STATE_UNGATE);
+	r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
+						   AMD_CG_STATE_UNGATE);
 	if (r) {
-		DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
+		DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r);
 	}
 
 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
@@ -1912,7 +1562,7 @@ int amdgpu_suspend(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
+static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
 {
 	int i, r;
 
@@ -1941,7 +1591,7 @@ static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
+static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
 {
 	int i, r;
 
@@ -1974,7 +1624,7 @@ static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_resume_phase1(struct amdgpu_device *adev)
+static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
 {
 	int i, r;
 
@@ -1997,7 +1647,7 @@ static int amdgpu_resume_phase1(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_resume_phase2(struct amdgpu_device *adev)
+static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
 {
 	int i, r;
 
@@ -2019,14 +1669,14 @@ static int amdgpu_resume_phase2(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_resume(struct amdgpu_device *adev)
+static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
 {
 	int r;
 
-	r = amdgpu_resume_phase1(adev);
+	r = amdgpu_device_ip_resume_phase1(adev);
 	if (r)
 		return r;
-	r = amdgpu_resume_phase2(adev);
+	r = amdgpu_device_ip_resume_phase2(adev);
 
 	return r;
 }
@@ -2163,8 +1813,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	mutex_init(&adev->mn_lock);
 	mutex_init(&adev->virt.vf_errors.lock);
 	hash_init(adev->mn_hash);
+	mutex_init(&adev->lock_reset);
 
-	amdgpu_check_arguments(adev);
+	amdgpu_device_check_arguments(adev);
 
 	spin_lock_init(&adev->mmio_idx_lock);
 	spin_lock_init(&adev->smc_idx_lock);
@@ -2179,13 +1830,11 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	INIT_LIST_HEAD(&adev->shadow_list);
 	mutex_init(&adev->shadow_list_lock);
 
-	INIT_LIST_HEAD(&adev->gtt_list);
-	spin_lock_init(&adev->gtt_list_lock);
-
 	INIT_LIST_HEAD(&adev->ring_lru_list);
 	spin_lock_init(&adev->ring_lru_list_lock);
 
-	INIT_DELAYED_WORK(&adev->late_init_work, amdgpu_late_init_func_handler);
+	INIT_DELAYED_WORK(&adev->late_init_work,
+			  amdgpu_device_ip_late_init_func_handler);
 
 	/* Registers mapping */
 	/* TODO: block userspace mapping of io register */
@@ -2205,7 +1854,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
 
 	/* doorbell bar mapping */
-	amdgpu_doorbell_init(adev);
+	amdgpu_device_doorbell_init(adev);
 
 	/* io port mapping */
 	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
@@ -2219,17 +1868,15 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		DRM_INFO("PCI I/O BAR is not found.\n");
 
 	/* early init functions */
-	r = amdgpu_early_init(adev);
+	r = amdgpu_device_ip_early_init(adev);
 	if (r)
 		return r;
 
 	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
 	/* this will fail for cards that aren't VGA class devices, just
 	 * ignore it */
-	vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
+	vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
 
-	if (amdgpu_runtime_pm == 1)
-		runtime = true;
 	if (amdgpu_device_is_px(ddev))
 		runtime = true;
 	if (!pci_is_thunderbolt_attached(adev->pdev))
@@ -2255,7 +1902,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	amdgpu_device_detect_sriov_bios(adev);
 
 	/* Post card if necessary */
-	if (amdgpu_need_post(adev)) {
+	if (amdgpu_device_need_post(adev)) {
 		if (!adev->bios) {
 			dev_err(adev->dev, "no vBIOS found\n");
 			r = -EINVAL;
@@ -2267,8 +1914,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 			dev_err(adev->dev, "gpu post error!\n");
 			goto failed;
 		}
-	} else {
-		DRM_INFO("GPU post is not needed\n");
 	}
 
 	if (adev->is_atom_fw) {
@@ -2303,11 +1948,23 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	/* init the mode config */
 	drm_mode_config_init(adev->ddev);
 
-	r = amdgpu_init(adev);
+	r = amdgpu_device_ip_init(adev);
 	if (r) {
-		dev_err(adev->dev, "amdgpu_init failed\n");
+		/* failed in exclusive mode due to timeout */
+		if (amdgpu_sriov_vf(adev) &&
+		    !amdgpu_sriov_runtime(adev) &&
+		    amdgpu_virt_mmio_blocked(adev) &&
+		    !amdgpu_virt_wait_reset(adev)) {
+			dev_err(adev->dev, "VF exclusive mode timeout\n");
+			/* Don't send request since VF is inactive. */
+			adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
+			adev->virt.ops = NULL;
+			r = -EAGAIN;
+			goto failed;
+		}
+		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
-		amdgpu_fini(adev);
+		amdgpu_device_ip_fini(adev);
 		goto failed;
 	}
 
@@ -2343,7 +2000,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	if (r)
 		DRM_ERROR("registering pm debugfs failed (%d).\n", r);
 
-	r = amdgpu_gem_debugfs_init(adev);
+	r = amdgpu_debugfs_gem_init(adev);
 	if (r)
 		DRM_ERROR("registering gem debugfs failed (%d).\n", r);
 
@@ -2351,17 +2008,13 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	if (r)
 		DRM_ERROR("registering register debugfs failed (%d).\n", r);
 
-	r = amdgpu_debugfs_test_ib_ring_init(adev);
-	if (r)
-		DRM_ERROR("registering register test ib ring debugfs failed (%d).\n", r);
-
 	r = amdgpu_debugfs_firmware_init(adev);
 	if (r)
 		DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
 
-	r = amdgpu_debugfs_vbios_dump_init(adev);
+	r = amdgpu_debugfs_init(adev);
 	if (r)
-		DRM_ERROR("Creating vbios dump debugfs failed (%d).\n", r);
+		DRM_ERROR("Creating debugfs files failed (%d).\n", r);
 
 	if ((amdgpu_testing & 1)) {
 		if (adev->accel_working)
@@ -2379,9 +2032,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	/* enable clockgating, etc. after ib tests, etc. since some blocks require
 	 * explicit gating rather than handling it automatically.
 	 */
-	r = amdgpu_late_init(adev);
+	r = amdgpu_device_ip_late_init(adev);
 	if (r) {
-		dev_err(adev->dev, "amdgpu_late_init failed\n");
+		dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
 		goto failed;
 	}
@@ -2392,6 +2045,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	amdgpu_vf_error_trans_all(adev);
 	if (runtime)
 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
+
 	return r;
 }
 
@@ -2409,15 +2063,16 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 
 	DRM_INFO("amdgpu: finishing device.\n");
 	adev->shutdown = true;
-	if (adev->mode_info.mode_config_initialized)
-		drm_crtc_force_disable_all(adev->ddev);
-	/* evict vram memory */
-	amdgpu_bo_evict_vram(adev);
+	if (adev->mode_info.mode_config_initialized){
+		if (!amdgpu_device_has_dc_support(adev))
+			drm_crtc_force_disable_all(adev->ddev);
+		else
+			drm_atomic_helper_shutdown(adev->ddev);
+	}
 	amdgpu_ib_pool_fini(adev);
-	amdgpu_fw_reserve_vram_fini(adev);
 	amdgpu_fence_driver_fini(adev);
 	amdgpu_fbdev_fini(adev);
-	r = amdgpu_fini(adev);
+	r = amdgpu_device_ip_fini(adev);
 	if (adev->firmware.gpu_info_fw) {
 		release_firmware(adev->firmware.gpu_info_fw);
 		adev->firmware.gpu_info_fw = NULL;
@@ -2440,7 +2095,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 	adev->rio_mem = NULL;
 	iounmap(adev->rmmio);
 	adev->rmmio = NULL;
-	amdgpu_doorbell_fini(adev);
+	amdgpu_device_doorbell_fini(adev);
 	amdgpu_pm_sysfs_fini(adev);
 	amdgpu_debugfs_regs_cleanup(adev);
 }
@@ -2521,7 +2176,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 
 	amdgpu_fence_driver_suspend(adev);
 
-	r = amdgpu_suspend(adev);
+	r = amdgpu_device_ip_suspend(adev);
 
 	/* evict remaining vram memory
 	 * This second call to evict vram is to evict the gart page table
@@ -2529,7 +2184,6 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
 	 */
 	amdgpu_bo_evict_vram(adev);
 
-	amdgpu_atombios_scratch_regs_save(adev);
 	pci_save_state(dev->pdev);
 	if (suspend) {
 		/* Shut down the device */
@@ -2578,18 +2232,17 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 		if (r)
 			goto unlock;
 	}
-	amdgpu_atombios_scratch_regs_restore(adev);
 
 	/* post card */
-	if (amdgpu_need_post(adev)) {
+	if (amdgpu_device_need_post(adev)) {
 		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
 		if (r)
 			DRM_ERROR("amdgpu asic init failed\n");
 	}
 
-	r = amdgpu_resume(adev);
+	r = amdgpu_device_ip_resume(adev);
 	if (r) {
-		DRM_ERROR("amdgpu_resume failed (%d).\n", r);
+		DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
 		goto unlock;
 	}
 	amdgpu_fence_driver_resume(adev);
@@ -2600,7 +2253,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 			DRM_ERROR("ib ring test failed (%d).\n", r);
 	}
 
-	r = amdgpu_late_init(adev);
+	r = amdgpu_device_ip_late_init(adev);
 	if (r)
 		goto unlock;
 
@@ -2637,14 +2290,6 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 				drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
 			}
 			drm_modeset_unlock_all(dev);
-		} else {
-			/*
-			 * There is no equivalent atomic helper to turn on
-			 * display, so we defined our own function for this,
-			 * once suspend resume is supported by the atomic
-			 * framework this will be reworked
-			 */
-			amdgpu_dm_display_resume(adev);
 		}
 	}
 
@@ -2680,7 +2325,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
 	return r;
 }
 
-static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
+static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
 {
 	int i;
 	bool asic_hang = false;
@@ -2702,7 +2347,7 @@ static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
 	return asic_hang;
 }
 
-static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
+static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
 {
 	int i, r = 0;
 
@@ -2720,7 +2365,7 @@ static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
 	return 0;
 }
 
-static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
+static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
 {
 	int i;
 
@@ -2741,7 +2386,7 @@ static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
 	return false;
 }
 
-static int amdgpu_soft_reset(struct amdgpu_device *adev)
+static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
 {
 	int i, r = 0;
 
@@ -2759,7 +2404,7 @@ static int amdgpu_soft_reset(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
+static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
 {
 	int i, r = 0;
 
@@ -2776,18 +2421,10 @@ static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
 	return 0;
 }
 
-bool amdgpu_need_backup(struct amdgpu_device *adev)
-{
-	if (adev->flags & AMD_IS_APU)
-		return false;
-
-	return amdgpu_lockup_timeout > 0 ? true : false;
-}
-
-static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
-					   struct amdgpu_ring *ring,
-					   struct amdgpu_bo *bo,
-					   struct dma_fence **fence)
+static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev,
+						  struct amdgpu_ring *ring,
+						  struct amdgpu_bo *bo,
+						  struct dma_fence **fence)
 {
 	uint32_t domain;
 	int r;
@@ -2819,163 +2456,181 @@ static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
 	return r;
 }
 
-/**
- * amdgpu_sriov_gpu_reset - reset the asic
+/*
+ * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough
  *
  * @adev: amdgpu device pointer
- * @job: which job trigger hang
+ * @reset_flags: output param tells caller the reset result
  *
- * Attempt the reset the GPU if it has hung (all asics).
- * for SRIOV case.
- * Returns 0 for success or an error on failure.
- */
-int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
+ * attempt to do soft-reset or full-reset and reinitialize Asic
+ * return 0 means successed otherwise failed
+*/
+static int amdgpu_device_reset(struct amdgpu_device *adev,
+			       uint64_t* reset_flags)
 {
-	int i, j, r = 0;
-	int resched;
-	struct amdgpu_bo *bo, *tmp;
-	struct amdgpu_ring *ring;
-	struct dma_fence *fence = NULL, *next = NULL;
-
-	mutex_lock(&adev->virt.lock_reset);
-	atomic_inc(&adev->gpu_reset_counter);
-	adev->in_sriov_reset = true;
+	bool need_full_reset, vram_lost = 0;
+	int r;
 
-	/* block TTM */
-	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
+	need_full_reset = amdgpu_device_ip_need_full_reset(adev);
 
-	/* we start from the ring trigger GPU hang */
-	j = job ? job->ring->idx : 0;
+	if (!need_full_reset) {
+		amdgpu_device_ip_pre_soft_reset(adev);
+		r = amdgpu_device_ip_soft_reset(adev);
+		amdgpu_device_ip_post_soft_reset(adev);
+		if (r || amdgpu_device_ip_check_soft_reset(adev)) {
+			DRM_INFO("soft reset failed, will fallback to full reset!\n");
+			need_full_reset = true;
+		}
 
-	/* block scheduler */
-	for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
-		ring = adev->rings[i % AMDGPU_MAX_RINGS];
-		if (!ring || !ring->sched.thread)
-			continue;
+	}
 
-		kthread_park(ring->sched.thread);
+	if (need_full_reset) {
+		r = amdgpu_device_ip_suspend(adev);
 
-		if (job && j != i)
-			continue;
+retry:
+		r = amdgpu_asic_reset(adev);
+		/* post card */
+		amdgpu_atom_asic_init(adev->mode_info.atom_context);
 
-		/* here give the last chance to check if job removed from mirror-list
-		 * since we already pay some time on kthread_park */
-		if (job && list_empty(&job->base.node)) {
-			kthread_unpark(ring->sched.thread);
-			goto give_up_reset;
+		if (!r) {
+			dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
+			r = amdgpu_device_ip_resume_phase1(adev);
+			if (r)
+				goto out;
+
+			vram_lost = amdgpu_device_check_vram_lost(adev);
+			if (vram_lost) {
+				DRM_ERROR("VRAM is lost!\n");
+				atomic_inc(&adev->vram_lost_counter);
+			}
+
+			r = amdgpu_gtt_mgr_recover(
+				&adev->mman.bdev.man[TTM_PL_TT]);
+			if (r)
+				goto out;
+
+			r = amdgpu_device_ip_resume_phase2(adev);
+			if (r)
+				goto out;
+
+			if (vram_lost)
+				amdgpu_device_fill_reset_magic(adev);
 		}
+	}
 
-		if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
-			amd_sched_job_kickout(&job->base);
+out:
+	if (!r) {
+		amdgpu_irq_gpu_reset_resume_helper(adev);
+		r = amdgpu_ib_ring_tests(adev);
+		if (r) {
+			dev_err(adev->dev, "ib ring test failed (%d).\n", r);
+			r = amdgpu_device_ip_suspend(adev);
+			need_full_reset = true;
+			goto retry;
+		}
+	}
 
-		/* only do job_reset on the hang ring if @job not NULL */
-		amd_sched_hw_job_reset(&ring->sched);
+	if (reset_flags) {
+		if (vram_lost)
+			(*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
 
-		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
-		amdgpu_fence_driver_force_completion_ring(ring);
+		if (need_full_reset)
+			(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
 	}
 
-	/* request to take full control of GPU before re-initialization  */
-	if (job)
-		amdgpu_virt_reset_gpu(adev);
-	else
-		amdgpu_virt_request_full_gpu(adev, true);
+	return r;
+}
+
+/*
+ * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
+ *
+ * @adev: amdgpu device pointer
+ * @reset_flags: output param tells caller the reset result
+ *
+ * do VF FLR and reinitialize Asic
+ * return 0 means successed otherwise failed
+*/
+static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
+				     uint64_t *reset_flags,
+				     bool from_hypervisor)
+{
+	int r;
 
+	if (from_hypervisor)
+		r = amdgpu_virt_request_full_gpu(adev, true);
+	else
+		r = amdgpu_virt_reset_gpu(adev);
+	if (r)
+		return r;
 
 	/* Resume IP prior to SMC */
-	amdgpu_sriov_reinit_early(adev);
+	r = amdgpu_device_ip_reinit_early_sriov(adev);
+	if (r)
+		goto error;
 
 	/* we need recover gart prior to run SMC/CP/SDMA resume */
-	amdgpu_ttm_recover_gart(adev);
+	amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
 
 	/* now we are okay to resume SMC/CP/SDMA */
-	amdgpu_sriov_reinit_late(adev);
+	r = amdgpu_device_ip_reinit_late_sriov(adev);
+	if (r)
+		goto error;
 
 	amdgpu_irq_gpu_reset_resume_helper(adev);
-
-	if (amdgpu_ib_ring_tests(adev))
+	r = amdgpu_ib_ring_tests(adev);
+	if (r)
 		dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
 
+error:
 	/* release full control of GPU after ib test */
 	amdgpu_virt_release_full_gpu(adev, true);
 
-	DRM_INFO("recover vram bo from shadow\n");
-
-	ring = adev->mman.buffer_funcs_ring;
-	mutex_lock(&adev->shadow_list_lock);
-	list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
-		next = NULL;
-		amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
-		if (fence) {
-			r = dma_fence_wait(fence, false);
-			if (r) {
-				WARN(r, "recovery from shadow isn't completed\n");
-				break;
-			}
-		}
-
-		dma_fence_put(fence);
-		fence = next;
-	}
-	mutex_unlock(&adev->shadow_list_lock);
-
-	if (fence) {
-		r = dma_fence_wait(fence, false);
-		if (r)
-			WARN(r, "recovery from shadow isn't completed\n");
-	}
-	dma_fence_put(fence);
-
-	for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
-		ring = adev->rings[i % AMDGPU_MAX_RINGS];
-		if (!ring || !ring->sched.thread)
-			continue;
-
-		if (job && j != i) {
-			kthread_unpark(ring->sched.thread);
-			continue;
+	if (reset_flags) {
+		if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
+			(*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
+			atomic_inc(&adev->vram_lost_counter);
 		}
 
-		amd_sched_job_recovery(&ring->sched);
-		kthread_unpark(ring->sched.thread);
+		/* VF FLR or hotlink reset is always full-reset */
+		(*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
 	}
 
-	drm_helper_resume_force_mode(adev->ddev);
-give_up_reset:
-	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
-	if (r) {
-		/* bad news, how to tell it to userspace ? */
-		dev_info(adev->dev, "GPU reset failed\n");
-	} else {
-		dev_info(adev->dev, "GPU reset successed!\n");
-	}
-
-	adev->in_sriov_reset = false;
-	mutex_unlock(&adev->virt.lock_reset);
 	return r;
 }
 
 /**
- * amdgpu_gpu_reset - reset the asic
+ * amdgpu_device_gpu_recover - reset the asic and recover scheduler
  *
  * @adev: amdgpu device pointer
+ * @job: which job trigger hang
+ * @force forces reset regardless of amdgpu_gpu_recovery
  *
- * Attempt the reset the GPU if it has hung (all asics).
+ * Attempt to reset the GPU if it has hung (all asics).
  * Returns 0 for success or an error on failure.
  */
-int amdgpu_gpu_reset(struct amdgpu_device *adev)
+int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
+			      struct amdgpu_job *job, bool force)
 {
 	struct drm_atomic_state *state = NULL;
-	int i, r;
-	int resched;
-	bool need_full_reset, vram_lost = false;
+	uint64_t reset_flags = 0;
+	int i, r, resched;
 
-	if (!amdgpu_check_soft_reset(adev)) {
+	if (!force && !amdgpu_device_ip_check_soft_reset(adev)) {
 		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
 		return 0;
 	}
 
+	if (!force && (amdgpu_gpu_recovery == 0 ||
+			(amdgpu_gpu_recovery == -1  && !amdgpu_sriov_vf(adev)))) {
+		DRM_INFO("GPU recovery disabled.\n");
+		return 0;
+	}
+
+	dev_info(adev->dev, "GPU reset begin!\n");
+
+	mutex_lock(&adev->lock_reset);
 	atomic_inc(&adev->gpu_reset_counter);
+	adev->in_gpu_reset = 1;
 
 	/* block TTM */
 	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
@@ -2989,69 +2644,26 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 
 		if (!ring || !ring->sched.thread)
 			continue;
-		kthread_park(ring->sched.thread);
-		amd_sched_hw_job_reset(&ring->sched);
-	}
-	/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
-	amdgpu_fence_driver_force_completion(adev);
 
-	need_full_reset = amdgpu_need_full_reset(adev);
+		/* only focus on the ring hit timeout if &job not NULL */
+		if (job && job->ring->idx != i)
+			continue;
 
-	if (!need_full_reset) {
-		amdgpu_pre_soft_reset(adev);
-		r = amdgpu_soft_reset(adev);
-		amdgpu_post_soft_reset(adev);
-		if (r || amdgpu_check_soft_reset(adev)) {
-			DRM_INFO("soft reset failed, will fallback to full reset!\n");
-			need_full_reset = true;
-		}
-	}
+		kthread_park(ring->sched.thread);
+		drm_sched_hw_job_reset(&ring->sched, &job->base);
 
-	if (need_full_reset) {
-		r = amdgpu_suspend(adev);
+		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
+		amdgpu_fence_driver_force_completion(ring);
+	}
 
-retry:
-		amdgpu_atombios_scratch_regs_save(adev);
-		r = amdgpu_asic_reset(adev);
-		amdgpu_atombios_scratch_regs_restore(adev);
-		/* post card */
-		amdgpu_atom_asic_init(adev->mode_info.atom_context);
+	if (amdgpu_sriov_vf(adev))
+		r = amdgpu_device_reset_sriov(adev, &reset_flags, job ? false : true);
+	else
+		r = amdgpu_device_reset(adev, &reset_flags);
 
-		if (!r) {
-			dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
-			r = amdgpu_resume_phase1(adev);
-			if (r)
-				goto out;
-			vram_lost = amdgpu_check_vram_lost(adev);
-			if (vram_lost) {
-				DRM_ERROR("VRAM is lost!\n");
-				atomic_inc(&adev->vram_lost_counter);
-			}
-			r = amdgpu_ttm_recover_gart(adev);
-			if (r)
-				goto out;
-			r = amdgpu_resume_phase2(adev);
-			if (r)
-				goto out;
-			if (vram_lost)
-				amdgpu_fill_reset_magic(adev);
-		}
-	}
-out:
 	if (!r) {
-		amdgpu_irq_gpu_reset_resume_helper(adev);
-		r = amdgpu_ib_ring_tests(adev);
-		if (r) {
-			dev_err(adev->dev, "ib ring test failed (%d).\n", r);
-			r = amdgpu_suspend(adev);
-			need_full_reset = true;
-			goto retry;
-		}
-		/**
-		 * recovery vm page tables, since we cannot depend on VRAM is
-		 * consistent after gpu full reset.
-		 */
-		if (need_full_reset && amdgpu_need_backup(adev)) {
+		if (((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) ||
+			(reset_flags & AMDGPU_RESET_INFO_VRAM_LOST)) {
 			struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
 			struct amdgpu_bo *bo, *tmp;
 			struct dma_fence *fence = NULL, *next = NULL;
@@ -3060,7 +2672,7 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 			mutex_lock(&adev->shadow_list_lock);
 			list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
 				next = NULL;
-				amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
+				amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next);
 				if (fence) {
 					r = dma_fence_wait(fence, false);
 					if (r) {
@@ -3080,44 +2692,59 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
 			}
 			dma_fence_put(fence);
 		}
+
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 			struct amdgpu_ring *ring = adev->rings[i];
 
 			if (!ring || !ring->sched.thread)
 				continue;
 
-			amd_sched_job_recovery(&ring->sched);
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			drm_sched_job_recovery(&ring->sched);
 			kthread_unpark(ring->sched.thread);
 		}
 	} else {
-		dev_err(adev->dev, "asic resume failed (%d).\n", r);
 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
-			if (adev->rings[i] && adev->rings[i]->sched.thread) {
-				kthread_unpark(adev->rings[i]->sched.thread);
-			}
+			struct amdgpu_ring *ring = adev->rings[i];
+
+			if (!ring || !ring->sched.thread)
+				continue;
+
+			/* only focus on the ring hit timeout if &job not NULL */
+			if (job && job->ring->idx != i)
+				continue;
+
+			kthread_unpark(adev->rings[i]->sched.thread);
 		}
 	}
 
 	if (amdgpu_device_has_dc_support(adev)) {
-		r = drm_atomic_helper_resume(adev->ddev, state);
-		amdgpu_dm_display_resume(adev);
-	} else
+		if (drm_atomic_helper_resume(adev->ddev, state))
+			dev_info(adev->dev, "drm resume failed:%d\n", r);
+	} else {
 		drm_helper_resume_force_mode(adev->ddev);
+	}
 
 	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
+
 	if (r) {
 		/* bad news, how to tell it to userspace ? */
-		dev_info(adev->dev, "GPU reset failed\n");
-	}
-	else {
-		dev_info(adev->dev, "GPU reset successed!\n");
+		dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
+		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
+	} else {
+		dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
 	}
 
 	amdgpu_vf_error_trans_all(adev);
+	adev->in_gpu_reset = 0;
+	mutex_unlock(&adev->lock_reset);
 	return r;
 }
 
-void amdgpu_get_pcie_info(struct amdgpu_device *adev)
+void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
 {
 	u32 mask;
 	int ret;
@@ -3209,773 +2836,3 @@ void amdgpu_get_pcie_info(struct amdgpu_device *adev)
 	}
 }
 
-/*
- * Debugfs
- */
-int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
-			     const struct drm_info_list *files,
-			     unsigned nfiles)
-{
-	unsigned i;
-
-	for (i = 0; i < adev->debugfs_count; i++) {
-		if (adev->debugfs[i].files == files) {
-			/* Already registered */
-			return 0;
-		}
-	}
-
-	i = adev->debugfs_count + 1;
-	if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
-		DRM_ERROR("Reached maximum number of debugfs components.\n");
-		DRM_ERROR("Report so we increase "
-			  "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
-		return -EINVAL;
-	}
-	adev->debugfs[adev->debugfs_count].files = files;
-	adev->debugfs[adev->debugfs_count].num_files = nfiles;
-	adev->debugfs_count = i;
-#if defined(CONFIG_DEBUG_FS)
-	drm_debugfs_create_files(files, nfiles,
-				 adev->ddev->primary->debugfs_root,
-				 adev->ddev->primary);
-#endif
-	return 0;
-}
-
-#if defined(CONFIG_DEBUG_FS)
-
-static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-	bool pm_pg_lock, use_bank;
-	unsigned instance_bank, sh_bank, se_bank;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	/* are we reading registers for which a PG lock is necessary? */
-	pm_pg_lock = (*pos >> 23) & 1;
-
-	if (*pos & (1ULL << 62)) {
-		se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
-		sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
-		instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
-
-		if (se_bank == 0x3FF)
-			se_bank = 0xFFFFFFFF;
-		if (sh_bank == 0x3FF)
-			sh_bank = 0xFFFFFFFF;
-		if (instance_bank == 0x3FF)
-			instance_bank = 0xFFFFFFFF;
-		use_bank = 1;
-	} else {
-		use_bank = 0;
-	}
-
-	*pos &= (1UL << 22) - 1;
-
-	if (use_bank) {
-		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
-		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
-			return -EINVAL;
-		mutex_lock(&adev->grbm_idx_mutex);
-		amdgpu_gfx_select_se_sh(adev, se_bank,
-					sh_bank, instance_bank);
-	}
-
-	if (pm_pg_lock)
-		mutex_lock(&adev->pm.mutex);
-
-	while (size) {
-		uint32_t value;
-
-		if (*pos > adev->rmmio_size)
-			goto end;
-
-		value = RREG32(*pos >> 2);
-		r = put_user(value, (uint32_t *)buf);
-		if (r) {
-			result = r;
-			goto end;
-		}
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-end:
-	if (use_bank) {
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
-		mutex_unlock(&adev->grbm_idx_mutex);
-	}
-
-	if (pm_pg_lock)
-		mutex_unlock(&adev->pm.mutex);
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
-					 size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-	bool pm_pg_lock, use_bank;
-	unsigned instance_bank, sh_bank, se_bank;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	/* are we reading registers for which a PG lock is necessary? */
-	pm_pg_lock = (*pos >> 23) & 1;
-
-	if (*pos & (1ULL << 62)) {
-		se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
-		sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
-		instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
-
-		if (se_bank == 0x3FF)
-			se_bank = 0xFFFFFFFF;
-		if (sh_bank == 0x3FF)
-			sh_bank = 0xFFFFFFFF;
-		if (instance_bank == 0x3FF)
-			instance_bank = 0xFFFFFFFF;
-		use_bank = 1;
-	} else {
-		use_bank = 0;
-	}
-
-	*pos &= (1UL << 22) - 1;
-
-	if (use_bank) {
-		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
-		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
-			return -EINVAL;
-		mutex_lock(&adev->grbm_idx_mutex);
-		amdgpu_gfx_select_se_sh(adev, se_bank,
-					sh_bank, instance_bank);
-	}
-
-	if (pm_pg_lock)
-		mutex_lock(&adev->pm.mutex);
-
-	while (size) {
-		uint32_t value;
-
-		if (*pos > adev->rmmio_size)
-			return result;
-
-		r = get_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		WREG32(*pos >> 2, value);
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	if (use_bank) {
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
-		mutex_unlock(&adev->grbm_idx_mutex);
-	}
-
-	if (pm_pg_lock)
-		mutex_unlock(&adev->pm.mutex);
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	while (size) {
-		uint32_t value;
-
-		value = RREG32_PCIE(*pos >> 2);
-		r = put_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
-					 size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	while (size) {
-		uint32_t value;
-
-		r = get_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		WREG32_PCIE(*pos >> 2, value);
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	while (size) {
-		uint32_t value;
-
-		value = RREG32_DIDT(*pos >> 2);
-		r = put_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
-					 size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	while (size) {
-		uint32_t value;
-
-		r = get_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		WREG32_DIDT(*pos >> 2, value);
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	while (size) {
-		uint32_t value;
-
-		value = RREG32_SMC(*pos);
-		r = put_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
-					 size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	while (size) {
-		uint32_t value;
-
-		r = get_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		WREG32_SMC(*pos, value);
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	ssize_t result = 0;
-	int r;
-	uint32_t *config, no_regs = 0;
-
-	if (size & 0x3 || *pos & 0x3)
-		return -EINVAL;
-
-	config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
-	if (!config)
-		return -ENOMEM;
-
-	/* version, increment each time something is added */
-	config[no_regs++] = 3;
-	config[no_regs++] = adev->gfx.config.max_shader_engines;
-	config[no_regs++] = adev->gfx.config.max_tile_pipes;
-	config[no_regs++] = adev->gfx.config.max_cu_per_sh;
-	config[no_regs++] = adev->gfx.config.max_sh_per_se;
-	config[no_regs++] = adev->gfx.config.max_backends_per_se;
-	config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
-	config[no_regs++] = adev->gfx.config.max_gprs;
-	config[no_regs++] = adev->gfx.config.max_gs_threads;
-	config[no_regs++] = adev->gfx.config.max_hw_contexts;
-	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
-	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
-	config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
-	config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
-	config[no_regs++] = adev->gfx.config.num_tile_pipes;
-	config[no_regs++] = adev->gfx.config.backend_enable_mask;
-	config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
-	config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
-	config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
-	config[no_regs++] = adev->gfx.config.num_gpus;
-	config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
-	config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
-	config[no_regs++] = adev->gfx.config.gb_addr_config;
-	config[no_regs++] = adev->gfx.config.num_rbs;
-
-	/* rev==1 */
-	config[no_regs++] = adev->rev_id;
-	config[no_regs++] = adev->pg_flags;
-	config[no_regs++] = adev->cg_flags;
-
-	/* rev==2 */
-	config[no_regs++] = adev->family;
-	config[no_regs++] = adev->external_rev_id;
-
-	/* rev==3 */
-	config[no_regs++] = adev->pdev->device;
-	config[no_regs++] = adev->pdev->revision;
-	config[no_regs++] = adev->pdev->subsystem_device;
-	config[no_regs++] = adev->pdev->subsystem_vendor;
-
-	while (size && (*pos < no_regs * 4)) {
-		uint32_t value;
-
-		value = config[*pos >> 2];
-		r = put_user(value, (uint32_t *)buf);
-		if (r) {
-			kfree(config);
-			return r;
-		}
-
-		result += 4;
-		buf += 4;
-		*pos += 4;
-		size -= 4;
-	}
-
-	kfree(config);
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = file_inode(f)->i_private;
-	int idx, x, outsize, r, valuesize;
-	uint32_t values[16];
-
-	if (size & 3 || *pos & 0x3)
-		return -EINVAL;
-
-	if (amdgpu_dpm == 0)
-		return -EINVAL;
-
-	/* convert offset to sensor number */
-	idx = *pos >> 2;
-
-	valuesize = sizeof(values);
-	if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
-		r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
-	else
-		return -EINVAL;
-
-	if (size > valuesize)
-		return -EINVAL;
-
-	outsize = 0;
-	x = 0;
-	if (!r) {
-		while (size) {
-			r = put_user(values[x++], (int32_t *)buf);
-			buf += 4;
-			size -= 4;
-			outsize += 4;
-		}
-	}
-
-	return !r ? outsize : r;
-}
-
-static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = f->f_inode->i_private;
-	int r, x;
-	ssize_t result=0;
-	uint32_t offset, se, sh, cu, wave, simd, data[32];
-
-	if (size & 3 || *pos & 3)
-		return -EINVAL;
-
-	/* decode offset */
-	offset = (*pos & GENMASK_ULL(6, 0));
-	se = (*pos & GENMASK_ULL(14, 7)) >> 7;
-	sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
-	cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
-	wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
-	simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
-
-	/* switch to the specific se/sh/cu */
-	mutex_lock(&adev->grbm_idx_mutex);
-	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
-
-	x = 0;
-	if (adev->gfx.funcs->read_wave_data)
-		adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
-
-	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
-	mutex_unlock(&adev->grbm_idx_mutex);
-
-	if (!x)
-		return -EINVAL;
-
-	while (size && (offset < x * 4)) {
-		uint32_t value;
-
-		value = data[offset >> 2];
-		r = put_user(value, (uint32_t *)buf);
-		if (r)
-			return r;
-
-		result += 4;
-		buf += 4;
-		offset += 4;
-		size -= 4;
-	}
-
-	return result;
-}
-
-static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
-					size_t size, loff_t *pos)
-{
-	struct amdgpu_device *adev = f->f_inode->i_private;
-	int r;
-	ssize_t result = 0;
-	uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
-
-	if (size & 3 || *pos & 3)
-		return -EINVAL;
-
-	/* decode offset */
-	offset = *pos & GENMASK_ULL(11, 0);
-	se = (*pos & GENMASK_ULL(19, 12)) >> 12;
-	sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
-	cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
-	wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
-	simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
-	thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
-	bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
-
-	data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
-	if (!data)
-		return -ENOMEM;
-
-	/* switch to the specific se/sh/cu */
-	mutex_lock(&adev->grbm_idx_mutex);
-	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
-
-	if (bank == 0) {
-		if (adev->gfx.funcs->read_wave_vgprs)
-			adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
-	} else {
-		if (adev->gfx.funcs->read_wave_sgprs)
-			adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
-	}
-
-	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
-	mutex_unlock(&adev->grbm_idx_mutex);
-
-	while (size) {
-		uint32_t value;
-
-		value = data[offset++];
-		r = put_user(value, (uint32_t *)buf);
-		if (r) {
-			result = r;
-			goto err;
-		}
-
-		result += 4;
-		buf += 4;
-		size -= 4;
-	}
-
-err:
-	kfree(data);
-	return result;
-}
-
-static const struct file_operations amdgpu_debugfs_regs_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_regs_read,
-	.write = amdgpu_debugfs_regs_write,
-	.llseek = default_llseek
-};
-static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_regs_didt_read,
-	.write = amdgpu_debugfs_regs_didt_write,
-	.llseek = default_llseek
-};
-static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_regs_pcie_read,
-	.write = amdgpu_debugfs_regs_pcie_write,
-	.llseek = default_llseek
-};
-static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_regs_smc_read,
-	.write = amdgpu_debugfs_regs_smc_write,
-	.llseek = default_llseek
-};
-
-static const struct file_operations amdgpu_debugfs_gca_config_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_gca_config_read,
-	.llseek = default_llseek
-};
-
-static const struct file_operations amdgpu_debugfs_sensors_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_sensor_read,
-	.llseek = default_llseek
-};
-
-static const struct file_operations amdgpu_debugfs_wave_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_wave_read,
-	.llseek = default_llseek
-};
-static const struct file_operations amdgpu_debugfs_gpr_fops = {
-	.owner = THIS_MODULE,
-	.read = amdgpu_debugfs_gpr_read,
-	.llseek = default_llseek
-};
-
-static const struct file_operations *debugfs_regs[] = {
-	&amdgpu_debugfs_regs_fops,
-	&amdgpu_debugfs_regs_didt_fops,
-	&amdgpu_debugfs_regs_pcie_fops,
-	&amdgpu_debugfs_regs_smc_fops,
-	&amdgpu_debugfs_gca_config_fops,
-	&amdgpu_debugfs_sensors_fops,
-	&amdgpu_debugfs_wave_fops,
-	&amdgpu_debugfs_gpr_fops,
-};
-
-static const char *debugfs_regs_names[] = {
-	"amdgpu_regs",
-	"amdgpu_regs_didt",
-	"amdgpu_regs_pcie",
-	"amdgpu_regs_smc",
-	"amdgpu_gca_config",
-	"amdgpu_sensors",
-	"amdgpu_wave",
-	"amdgpu_gpr",
-};
-
-static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
-{
-	struct drm_minor *minor = adev->ddev->primary;
-	struct dentry *ent, *root = minor->debugfs_root;
-	unsigned i, j;
-
-	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
-		ent = debugfs_create_file(debugfs_regs_names[i],
-					  S_IFREG | S_IRUGO, root,
-					  adev, debugfs_regs[i]);
-		if (IS_ERR(ent)) {
-			for (j = 0; j < i; j++) {
-				debugfs_remove(adev->debugfs_regs[i]);
-				adev->debugfs_regs[i] = NULL;
-			}
-			return PTR_ERR(ent);
-		}
-
-		if (!i)
-			i_size_write(ent->d_inode, adev->rmmio_size);
-		adev->debugfs_regs[i] = ent;
-	}
-
-	return 0;
-}
-
-static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
-{
-	unsigned i;
-
-	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
-		if (adev->debugfs_regs[i]) {
-			debugfs_remove(adev->debugfs_regs[i]);
-			adev->debugfs_regs[i] = NULL;
-		}
-	}
-}
-
-static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct amdgpu_device *adev = dev->dev_private;
-	int r = 0, i;
-
-	/* hold on the scheduler */
-	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
-		struct amdgpu_ring *ring = adev->rings[i];
-
-		if (!ring || !ring->sched.thread)
-			continue;
-		kthread_park(ring->sched.thread);
-	}
-
-	seq_printf(m, "run ib test:\n");
-	r = amdgpu_ib_ring_tests(adev);
-	if (r)
-		seq_printf(m, "ib ring tests failed (%d).\n", r);
-	else
-		seq_printf(m, "ib ring tests passed.\n");
-
-	/* go on the scheduler */
-	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
-		struct amdgpu_ring *ring = adev->rings[i];
-
-		if (!ring || !ring->sched.thread)
-			continue;
-		kthread_unpark(ring->sched.thread);
-	}
-
-	return 0;
-}
-
-static const struct drm_info_list amdgpu_debugfs_test_ib_ring_list[] = {
-	{"amdgpu_test_ib", &amdgpu_debugfs_test_ib}
-};
-
-static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
-{
-	return amdgpu_debugfs_add_files(adev,
-					amdgpu_debugfs_test_ib_ring_list, 1);
-}
-
-int amdgpu_debugfs_init(struct drm_minor *minor)
-{
-	return 0;
-}
-
-static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct amdgpu_device *adev = dev->dev_private;
-
-	seq_write(m, adev->bios, adev->bios_size);
-	return 0;
-}
-
-static const struct drm_info_list amdgpu_vbios_dump_list[] = {
-		{"amdgpu_vbios",
-		 amdgpu_debugfs_get_vbios_dump,
-		 0, NULL},
-};
-
-static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
-{
-	return amdgpu_debugfs_add_files(adev,
-					amdgpu_vbios_dump_list, 1);
-}
-#else
-static int amdgpu_debugfs_test_ib_ring_init(struct amdgpu_device *adev)
-{
-	return 0;
-}
-static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
-{
-	return 0;
-}
-static int amdgpu_debugfs_vbios_dump_init(struct amdgpu_device *adev)
-{
-	return 0;
-}
-static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
-#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index 138beb550a58..38d47559f098 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -34,6 +34,7 @@
 #include <linux/pm_runtime.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_fb_helper.h>
 
 static void amdgpu_flip_callback(struct dma_fence *f, struct dma_fence_cb *cb)
 {
@@ -556,15 +557,9 @@ amdgpu_user_framebuffer_create(struct drm_device *dev,
 	return &amdgpu_fb->base;
 }
 
-void amdgpu_output_poll_changed(struct drm_device *dev)
-{
-	struct amdgpu_device *adev = dev->dev_private;
-	amdgpu_fb_output_poll_changed(adev);
-}
-
 const struct drm_mode_config_funcs amdgpu_mode_funcs = {
 	.fb_create = amdgpu_user_framebuffer_create,
-	.output_poll_changed = amdgpu_output_poll_changed
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 };
 
 static const struct drm_prop_enum_list amdgpu_underscan_enum_list[] =
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
index 3cc0ef0c055e..0bcb6c6e0ca9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.h
@@ -25,9 +25,7 @@
 
 struct drm_framebuffer *
 amdgpu_user_framebuffer_create(struct drm_device *dev,
-						       struct drm_file *file_priv,
-							   const struct drm_mode_fb_cmd2 *mode_cmd);
-
-void amdgpu_output_poll_changed(struct drm_device *dev);
+			       struct drm_file *file_priv,
+			       const struct drm_mode_fb_cmd2 *mode_cmd);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index 56caaeee6fea..a8437a3296a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -360,6 +360,12 @@ enum amdgpu_pcie_gen {
 		((adev)->powerplay.pp_funcs->set_clockgating_by_smu(\
 			(adev)->powerplay.pp_handle, msg_id))
 
+#define amdgpu_dpm_notify_smu_memory_info(adev, virtual_addr_low, \
+			virtual_addr_hi, mc_addr_low, mc_addr_hi, size) \
+		((adev)->powerplay.pp_funcs->notify_smu_memory_info)( \
+			(adev)->powerplay.pp_handle, virtual_addr_low, \
+			virtual_addr_hi, mc_addr_low, mc_addr_hi, size)
+
 struct amdgpu_dpm {
 	struct amdgpu_ps        *ps;
 	/* number of valid power states */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index c2f414ffb2cc..50afcf65181a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -90,7 +90,7 @@ int amdgpu_disp_priority = 0;
 int amdgpu_hw_i2c = 0;
 int amdgpu_pcie_gen2 = -1;
 int amdgpu_msi = -1;
-int amdgpu_lockup_timeout = 0;
+int amdgpu_lockup_timeout = 10000;
 int amdgpu_dpm = -1;
 int amdgpu_fw_load_type = -1;
 int amdgpu_aspm = -1;
@@ -128,6 +128,7 @@ int amdgpu_param_buf_per_se = 0;
 int amdgpu_job_hang_limit = 0;
 int amdgpu_lbpw = -1;
 int amdgpu_compute_multipipe = -1;
+int amdgpu_gpu_recovery = -1; /* auto */
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -165,7 +166,7 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444);
 MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(msi, amdgpu_msi, int, 0444);
 
-MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default 0 = disable)");
+MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms > 0 (default 10000)");
 module_param_named(lockup_timeout, amdgpu_lockup_timeout, int, 0444);
 
 MODULE_PARM_DESC(dpm, "DPM support (1 = enable, 0 = disable, -1 = auto)");
@@ -216,7 +217,7 @@ module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444);
 MODULE_PARM_DESC(dc, "Display Core driver (1 = enable, 0 = disable, -1 = auto (default))");
 module_param_named(dc, amdgpu_dc, int, 0444);
 
-MODULE_PARM_DESC(dc, "Display Core Log Level (0 = minimal (default), 1 = chatty");
+MODULE_PARM_DESC(dc_log, "Display Core Log Level (0 = minimal (default), 1 = chatty");
 module_param_named(dc_log, amdgpu_dc_log, int, 0444);
 
 MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 32)");
@@ -280,6 +281,9 @@ module_param_named(lbpw, amdgpu_lbpw, int, 0444);
 MODULE_PARM_DESC(compute_multipipe, "Force compute queues to be spread across pipes (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444);
 
+MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto");
+module_param_named(gpu_recovery, amdgpu_gpu_recovery, int, 0444);
+
 #ifdef CONFIG_DRM_AMDGPU_SI
 
 #if defined(CONFIG_DRM_RADEON) || defined(CONFIG_DRM_RADEON_MODULE)
@@ -306,7 +310,6 @@ MODULE_PARM_DESC(cik_support, "CIK support (1 = enabled (default), 0 = disabled)
 module_param_named(cik_support, amdgpu_cik_support, int, 0444);
 #endif
 
-
 static const struct pci_device_id pciidlist[] = {
 #ifdef  CONFIG_DRM_AMDGPU_SI
 	{0x1002, 0x6780, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TAHITI},
@@ -566,12 +569,13 @@ static int amdgpu_kick_out_firmware_fb(struct pci_dev *pdev)
 	return 0;
 }
 
+
 static int amdgpu_pci_probe(struct pci_dev *pdev,
 			    const struct pci_device_id *ent)
 {
 	struct drm_device *dev;
 	unsigned long flags = ent->driver_data;
-	int ret;
+	int ret, retry = 0;
 
 	if ((flags & AMD_EXP_HW_SUPPORT) && !amdgpu_exp_hw_support) {
 		DRM_INFO("This hardware requires experimental hardware support.\n"
@@ -604,8 +608,14 @@ static int amdgpu_pci_probe(struct pci_dev *pdev,
 
 	pci_set_drvdata(pdev, dev);
 
+retry_init:
 	ret = drm_dev_register(dev, ent->driver_data);
-	if (ret)
+	if (ret == -EAGAIN && ++retry <= 3) {
+		DRM_INFO("retry init %d\n", retry);
+		/* Don't request EX mode too frequently which is attacking */
+		msleep(5000);
+		goto retry_init;
+	} else if (ret)
 		goto err_pci;
 
 	return 0;
@@ -639,7 +649,7 @@ amdgpu_pci_shutdown(struct pci_dev *pdev)
 	 * unfortunately we can't detect certain
 	 * hypervisors so just do this all the time.
 	 */
-	amdgpu_suspend(adev);
+	amdgpu_device_ip_suspend(adev);
 }
 
 static int amdgpu_pmops_suspend(struct device *dev)
@@ -844,9 +854,6 @@ static struct drm_driver kms_driver = {
 	.disable_vblank = amdgpu_disable_vblank_kms,
 	.get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos,
 	.get_scanout_position = amdgpu_get_crtc_scanout_position,
-#if defined(CONFIG_DEBUG_FS)
-	.debugfs_init = amdgpu_debugfs_init,
-#endif
 	.irq_preinstall = amdgpu_irq_preinstall,
 	.irq_postinstall = amdgpu_irq_postinstall,
 	.irq_uninstall = amdgpu_irq_uninstall,
@@ -906,10 +913,6 @@ static int __init amdgpu_init(void)
 	if (r)
 		goto error_fence;
 
-	r = amd_sched_fence_slab_init();
-	if (r)
-		goto error_sched;
-
 	if (vgacon_text_force()) {
 		DRM_ERROR("VGACON disables amdgpu kernel modesetting.\n");
 		return -EINVAL;
@@ -922,9 +925,6 @@ static int __init amdgpu_init(void)
 	/* let modprobe override vga console setting */
 	return pci_register_driver(pdriver);
 
-error_sched:
-	amdgpu_fence_slab_fini();
-
 error_fence:
 	amdgpu_sync_fini();
 
@@ -938,7 +938,6 @@ static void __exit amdgpu_exit(void)
 	pci_unregister_driver(pdriver);
 	amdgpu_unregister_atpx_handler();
 	amdgpu_sync_fini();
-	amd_sched_fence_slab_fini();
 	amdgpu_fence_slab_fini();
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 90fa8e8bc6fb..ff3e9beb7d19 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -283,12 +283,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 	return ret;
 }
 
-void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev)
-{
-	if (adev->mode_info.rfbdev)
-		drm_fb_helper_hotplug_event(&adev->mode_info.rfbdev->helper);
-}
-
 static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfbdev)
 {
 	struct amdgpu_framebuffer *rfb = &rfbdev->rfb;
@@ -393,24 +387,3 @@ bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj)
 		return true;
 	return false;
 }
-
-void amdgpu_fbdev_restore_mode(struct amdgpu_device *adev)
-{
-	struct amdgpu_fbdev *afbdev;
-	struct drm_fb_helper *fb_helper;
-	int ret;
-
-	if (!adev)
-		return;
-
-	afbdev = adev->mode_info.rfbdev;
-
-	if (!afbdev)
-		return;
-
-	fb_helper = &afbdev->helper;
-
-	ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper);
-	if (ret)
-		DRM_DEBUG("failed to restore crtc mode\n");
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 2fa95aef74d5..008e1984b7e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -187,7 +187,7 @@ int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s)
 
 	seq = ++ring->fence_drv.sync_seq;
 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
-			       seq, AMDGPU_FENCE_FLAG_INT);
+			       seq, 0);
 
 	*s = seq;
 
@@ -391,9 +391,9 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
 	ring->fence_drv.irq_type = irq_type;
 	ring->fence_drv.initialized = true;
 
-	dev_info(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
-		 "cpu addr 0x%p\n", ring->idx,
-		 ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
+	dev_dbg(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, "
+		"cpu addr 0x%p\n", ring->idx,
+		ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr);
 	return 0;
 }
 
@@ -410,7 +410,6 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
 				  unsigned num_hw_submission)
 {
-	long timeout;
 	int r;
 
 	/* Check that num_hw_submission is a power of two */
@@ -434,20 +433,9 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
 
 	/* No need to setup the GPU scheduler for KIQ ring */
 	if (ring->funcs->type != AMDGPU_RING_TYPE_KIQ) {
-		timeout = msecs_to_jiffies(amdgpu_lockup_timeout);
-		if (timeout == 0) {
-			/*
-			 * FIXME:
-			 * Delayed workqueue cannot use it directly,
-			 * so the scheduler will not use delayed workqueue if
-			 * MAX_SCHEDULE_TIMEOUT is set.
-			 * Currently keep it simple and silly.
-			 */
-			timeout = MAX_SCHEDULE_TIMEOUT;
-		}
-		r = amd_sched_init(&ring->sched, &amdgpu_sched_ops,
-				   num_hw_submission,
-				   timeout, ring->name);
+		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
+				   num_hw_submission, amdgpu_job_hang_limit,
+				   msecs_to_jiffies(amdgpu_lockup_timeout), ring->name);
 		if (r) {
 			DRM_ERROR("Failed to create scheduler on ring %s.\n",
 				  ring->name);
@@ -499,11 +487,11 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev)
 		r = amdgpu_fence_wait_empty(ring);
 		if (r) {
 			/* no need to trigger GPU reset as we are unloading */
-			amdgpu_fence_driver_force_completion(adev);
+			amdgpu_fence_driver_force_completion(ring);
 		}
 		amdgpu_irq_put(adev, ring->fence_drv.irq_src,
 			       ring->fence_drv.irq_type);
-		amd_sched_fini(&ring->sched);
+		drm_sched_fini(&ring->sched);
 		del_timer_sync(&ring->fence_drv.fallback_timer);
 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
 			dma_fence_put(ring->fence_drv.fences[j]);
@@ -534,7 +522,7 @@ void amdgpu_fence_driver_suspend(struct amdgpu_device *adev)
 		r = amdgpu_fence_wait_empty(ring);
 		if (r) {
 			/* delay GPU reset to resume */
-			amdgpu_fence_driver_force_completion(adev);
+			amdgpu_fence_driver_force_completion(ring);
 		}
 
 		/* disable the interrupt */
@@ -571,30 +559,15 @@ void amdgpu_fence_driver_resume(struct amdgpu_device *adev)
 }
 
 /**
- * amdgpu_fence_driver_force_completion - force all fence waiter to complete
+ * amdgpu_fence_driver_force_completion - force signal latest fence of ring
  *
- * @adev: amdgpu device pointer
+ * @ring: fence of the ring to signal
  *
- * In case of GPU reset failure make sure no process keep waiting on fence
- * that will never complete.
  */
-void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev)
+void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
 {
-	int i;
-
-	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
-		struct amdgpu_ring *ring = adev->rings[i];
-		if (!ring || !ring->fence_drv.initialized)
-			continue;
-
-		amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
-	}
-}
-
-void amdgpu_fence_driver_force_completion_ring(struct amdgpu_ring *ring)
-{
-	if (ring)
-		amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
+	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
+	amdgpu_fence_process(ring);
 }
 
 /*
@@ -709,25 +682,25 @@ static int amdgpu_debugfs_fence_info(struct seq_file *m, void *data)
 }
 
 /**
- * amdgpu_debugfs_gpu_reset - manually trigger a gpu reset
+ * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
  *
  * Manually trigger a gpu reset at the next fence wait.
  */
-static int amdgpu_debugfs_gpu_reset(struct seq_file *m, void *data)
+static int amdgpu_debugfs_gpu_recover(struct seq_file *m, void *data)
 {
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct amdgpu_device *adev = dev->dev_private;
 
-	seq_printf(m, "gpu reset\n");
-	amdgpu_gpu_reset(adev);
+	seq_printf(m, "gpu recover\n");
+	amdgpu_device_gpu_recover(adev, NULL, true);
 
 	return 0;
 }
 
 static const struct drm_info_list amdgpu_debugfs_fence_list[] = {
 	{"amdgpu_fence_info", &amdgpu_debugfs_fence_info, 0, NULL},
-	{"amdgpu_gpu_reset", &amdgpu_debugfs_gpu_reset, 0, NULL}
+	{"amdgpu_gpu_recover", &amdgpu_debugfs_gpu_recover, 0, NULL}
 };
 
 static const struct drm_info_list amdgpu_debugfs_fence_list_sriov[] = {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index fe818501c520..0a4f34afaaaa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -57,60 +57,48 @@
  */
 
 /**
- * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table
+ * amdgpu_dummy_page_init - init dummy page used by the driver
  *
  * @adev: amdgpu_device pointer
  *
- * Allocate system memory for GART page table
- * (r1xx-r3xx, non-pcie r4xx, rs400).  These asics require the
- * gart table to be in system memory.
- * Returns 0 for success, -ENOMEM for failure.
+ * Allocate the dummy page used by the driver (all asics).
+ * This dummy page is used by the driver as a filler for gart entries
+ * when pages are taken out of the GART
+ * Returns 0 on sucess, -ENOMEM on failure.
  */
-int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev)
+static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev)
 {
-	void *ptr;
-
-	ptr = pci_alloc_consistent(adev->pdev, adev->gart.table_size,
-				   &adev->gart.table_addr);
-	if (ptr == NULL) {
+	if (adev->dummy_page.page)
+		return 0;
+	adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
+	if (adev->dummy_page.page == NULL)
+		return -ENOMEM;
+	adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
+					0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+	if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
+		dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
+		__free_page(adev->dummy_page.page);
+		adev->dummy_page.page = NULL;
 		return -ENOMEM;
 	}
-#ifdef CONFIG_X86
-	if (0) {
-		set_memory_uc((unsigned long)ptr,
-			      adev->gart.table_size >> PAGE_SHIFT);
-	}
-#endif
-	adev->gart.ptr = ptr;
-	memset((void *)adev->gart.ptr, 0, adev->gart.table_size);
 	return 0;
 }
 
 /**
- * amdgpu_gart_table_ram_free - free system ram for gart page table
+ * amdgpu_dummy_page_fini - free dummy page used by the driver
  *
  * @adev: amdgpu_device pointer
  *
- * Free system memory for GART page table
- * (r1xx-r3xx, non-pcie r4xx, rs400).  These asics require the
- * gart table to be in system memory.
+ * Frees the dummy page used by the driver (all asics).
  */
-void amdgpu_gart_table_ram_free(struct amdgpu_device *adev)
+static void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev)
 {
-	if (adev->gart.ptr == NULL) {
+	if (adev->dummy_page.page == NULL)
 		return;
-	}
-#ifdef CONFIG_X86
-	if (0) {
-		set_memory_wb((unsigned long)adev->gart.ptr,
-			      adev->gart.table_size >> PAGE_SHIFT);
-	}
-#endif
-	pci_free_consistent(adev->pdev, adev->gart.table_size,
-			    (void *)adev->gart.ptr,
-			    adev->gart.table_addr);
-	adev->gart.ptr = NULL;
-	adev->gart.table_addr = 0;
+	pci_unmap_page(adev->pdev, adev->dummy_page.addr,
+			PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+	__free_page(adev->dummy_page.page);
+	adev->dummy_page.page = NULL;
 }
 
 /**
@@ -365,7 +353,7 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
 		DRM_ERROR("Page size is smaller than GPU page size!\n");
 		return -EINVAL;
 	}
-	r = amdgpu_dummy_page_init(adev);
+	r = amdgpu_gart_dummy_page_init(adev);
 	if (r)
 		return r;
 	/* Compute table size */
@@ -377,10 +365,8 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 	/* Allocate pages table */
 	adev->gart.pages = vzalloc(sizeof(void *) * adev->gart.num_cpu_pages);
-	if (adev->gart.pages == NULL) {
-		amdgpu_gart_fini(adev);
+	if (adev->gart.pages == NULL)
 		return -ENOMEM;
-	}
 #endif
 
 	return 0;
@@ -395,14 +381,9 @@ int amdgpu_gart_init(struct amdgpu_device *adev)
  */
 void amdgpu_gart_fini(struct amdgpu_device *adev)
 {
-	if (adev->gart.ready) {
-		/* unbind pages */
-		amdgpu_gart_unbind(adev, 0, adev->gart.num_cpu_pages);
-	}
-	adev->gart.ready = false;
 #ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
 	vfree(adev->gart.pages);
 	adev->gart.pages = NULL;
 #endif
-	amdgpu_dummy_page_fini(adev);
+	amdgpu_gart_dummy_page_fini(adev);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
index afbe803b1a13..d4a43302c2be 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.h
@@ -39,7 +39,7 @@ struct amdgpu_gart_funcs;
 #define AMDGPU_GPU_PAGE_ALIGN(a) (((a) + AMDGPU_GPU_PAGE_MASK) & ~AMDGPU_GPU_PAGE_MASK)
 
 struct amdgpu_gart {
-	dma_addr_t			table_addr;
+	u64				table_addr;
 	struct amdgpu_bo		*robj;
 	void				*ptr;
 	unsigned			num_gpu_pages;
@@ -56,8 +56,6 @@ struct amdgpu_gart {
 	const struct amdgpu_gart_funcs *gart_funcs;
 };
 
-int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev);
-void amdgpu_gart_table_ram_free(struct amdgpu_device *adev);
 int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev);
 void amdgpu_gart_table_vram_free(struct amdgpu_device *adev);
 int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index e87eedcc0da9..ca6c931dabfa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -36,8 +36,6 @@ void amdgpu_gem_object_free(struct drm_gem_object *gobj)
 	struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj);
 
 	if (robj) {
-		if (robj->gem_base.import_attach)
-			drm_prime_gem_destroy(&robj->gem_base, robj->tbo.sg);
 		amdgpu_mn_unregister(robj);
 		amdgpu_bo_unref(&robj);
 	}
@@ -72,7 +70,7 @@ int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
 				initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
 				goto retry;
 			}
-			DRM_ERROR("Failed to allocate GEM object (%ld, %d, %u, %d)\n",
+			DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n",
 				  size, initial_domain, alignment, r);
 		}
 		return r;
@@ -282,6 +280,7 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *filp)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct amdgpu_device *adev = dev->dev_private;
 	struct drm_amdgpu_gem_userptr *args = data;
 	struct drm_gem_object *gobj;
@@ -335,7 +334,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 			goto free_pages;
 
 		amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
-		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		amdgpu_bo_unreserve(bo);
 		if (r)
 			goto free_pages;
@@ -517,10 +516,6 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 	if (!amdgpu_vm_ready(vm))
 		return;
 
-	r = amdgpu_vm_update_directories(adev, vm);
-	if (r)
-		goto error;
-
 	r = amdgpu_vm_clear_freed(adev, vm, NULL);
 	if (r)
 		goto error;
@@ -529,6 +524,10 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
 	    operation == AMDGPU_VA_OP_REPLACE)
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 
+	r = amdgpu_vm_update_directories(adev, vm);
+	if (r)
+		goto error;
+
 error:
 	if (r && r != -ERESTARTSYS)
 		DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
@@ -557,14 +556,25 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	int r = 0;
 
 	if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
-		dev_err(&dev->pdev->dev,
+		dev_dbg(&dev->pdev->dev,
 			"va_address 0x%LX is in reserved area 0x%LX\n",
 			args->va_address, AMDGPU_VA_RESERVED_SIZE);
 		return -EINVAL;
 	}
 
+	if (args->va_address >= AMDGPU_VA_HOLE_START &&
+	    args->va_address < AMDGPU_VA_HOLE_END) {
+		dev_dbg(&dev->pdev->dev,
+			"va_address 0x%LX is in VA hole 0x%LX-0x%LX\n",
+			args->va_address, AMDGPU_VA_HOLE_START,
+			AMDGPU_VA_HOLE_END);
+		return -EINVAL;
+	}
+
+	args->va_address &= AMDGPU_VA_HOLE_MASK;
+
 	if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
-		dev_err(&dev->pdev->dev, "invalid flags combination 0x%08X\n",
+		dev_dbg(&dev->pdev->dev, "invalid flags combination 0x%08X\n",
 			args->flags);
 		return -EINVAL;
 	}
@@ -576,7 +586,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	case AMDGPU_VA_OP_REPLACE:
 		break;
 	default:
-		dev_err(&dev->pdev->dev, "unsupported operation %d\n",
+		dev_dbg(&dev->pdev->dev, "unsupported operation %d\n",
 			args->operation);
 		return -EINVAL;
 	}
@@ -839,7 +849,7 @@ static const struct drm_info_list amdgpu_debugfs_gem_list[] = {
 };
 #endif
 
-int amdgpu_gem_debugfs_init(struct amdgpu_device *adev)
+int amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
 {
 #if defined(CONFIG_DEBUG_FS)
 	return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list, 1);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index ef043361009f..239bf2a4b3c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -179,8 +179,12 @@ static int amdgpu_gfx_kiq_acquire(struct amdgpu_device *adev,
 
 		amdgpu_gfx_bit_to_queue(adev, queue_bit, &mec, &pipe, &queue);
 
-		/* Using pipes 2/3 from MEC 2 seems cause problems */
-		if (mec == 1 && pipe > 1)
+		/*
+		 * 1. Using pipes 2/3 from MEC 2 seems cause problems.
+		 * 2. It must use queue id 0, because CGPG_IDLE/SAVE/LOAD/RUN
+		 * only can be issued on queue 0.
+		 */
+		if ((mec == 1 && pipe > 1) || queue != 0)
 			continue;
 
 		ring->me = mec + 1;
@@ -203,7 +207,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,
 
 	spin_lock_init(&kiq->ring_lock);
 
-	r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
+	r = amdgpu_device_wb_get(adev, &adev->virt.reg_val_offs);
 	if (r)
 		return r;
 
@@ -229,7 +233,7 @@ int amdgpu_gfx_kiq_init_ring(struct amdgpu_device *adev,
 void amdgpu_gfx_kiq_free_ring(struct amdgpu_ring *ring,
 			      struct amdgpu_irq_src *irq)
 {
-	amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
+	amdgpu_device_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
 	amdgpu_ring_fini(ring);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 00e0ce10862f..7c2be32c5aea 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -31,6 +31,11 @@ struct amdgpu_gtt_mgr {
 	atomic64_t available;
 };
 
+struct amdgpu_gtt_node {
+	struct drm_mm_node node;
+	struct ttm_buffer_object *tbo;
+};
+
 /**
  * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
  *
@@ -70,7 +75,7 @@ static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
 static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
 {
 	struct amdgpu_gtt_mgr *mgr = man->priv;
-
+	spin_lock(&mgr->lock);
 	drm_mm_takedown(&mgr->mm);
 	spin_unlock(&mgr->lock);
 	kfree(mgr);
@@ -79,17 +84,17 @@ static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
 }
 
 /**
- * amdgpu_gtt_mgr_is_allocated - Check if mem has address space
+ * amdgpu_gtt_mgr_has_gart_addr - Check if mem has address space
  *
  * @mem: the mem object to check
  *
  * Check if a mem object has already address space allocated.
  */
-bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem)
+bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_mem_reg *mem)
 {
-	struct drm_mm_node *node = mem->mm_node;
+	struct amdgpu_gtt_node *node = mem->mm_node;
 
-	return (node->start != AMDGPU_BO_INVALID_OFFSET);
+	return (node->node.start != AMDGPU_BO_INVALID_OFFSET);
 }
 
 /**
@@ -109,12 +114,12 @@ static int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr = man->priv;
-	struct drm_mm_node *node = mem->mm_node;
+	struct amdgpu_gtt_node *node = mem->mm_node;
 	enum drm_mm_insert_mode mode;
 	unsigned long fpfn, lpfn;
 	int r;
 
-	if (amdgpu_gtt_mgr_is_allocated(mem))
+	if (amdgpu_gtt_mgr_has_gart_addr(mem))
 		return 0;
 
 	if (place)
@@ -132,13 +137,13 @@ static int amdgpu_gtt_mgr_alloc(struct ttm_mem_type_manager *man,
 		mode = DRM_MM_INSERT_HIGH;
 
 	spin_lock(&mgr->lock);
-	r = drm_mm_insert_node_in_range(&mgr->mm, node,
-					mem->num_pages, mem->page_alignment, 0,
-					fpfn, lpfn, mode);
+	r = drm_mm_insert_node_in_range(&mgr->mm, &node->node, mem->num_pages,
+					mem->page_alignment, 0, fpfn, lpfn,
+					mode);
 	spin_unlock(&mgr->lock);
 
 	if (!r)
-		mem->start = node->start;
+		mem->start = node->node.start;
 
 	return r;
 }
@@ -159,7 +164,7 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 			      struct ttm_mem_reg *mem)
 {
 	struct amdgpu_gtt_mgr *mgr = man->priv;
-	struct drm_mm_node *node;
+	struct amdgpu_gtt_node *node;
 	int r;
 
 	spin_lock(&mgr->lock);
@@ -177,8 +182,9 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 		goto err_out;
 	}
 
-	node->start = AMDGPU_BO_INVALID_OFFSET;
-	node->size = mem->num_pages;
+	node->node.start = AMDGPU_BO_INVALID_OFFSET;
+	node->node.size = mem->num_pages;
+	node->tbo = tbo;
 	mem->mm_node = node;
 
 	if (place->fpfn || place->lpfn || place->flags & TTM_PL_FLAG_TOPDOWN) {
@@ -190,7 +196,7 @@ static int amdgpu_gtt_mgr_new(struct ttm_mem_type_manager *man,
 			goto err_out;
 		}
 	} else {
-		mem->start = node->start;
+		mem->start = node->node.start;
 	}
 
 	return 0;
@@ -214,14 +220,14 @@ static void amdgpu_gtt_mgr_del(struct ttm_mem_type_manager *man,
 			       struct ttm_mem_reg *mem)
 {
 	struct amdgpu_gtt_mgr *mgr = man->priv;
-	struct drm_mm_node *node = mem->mm_node;
+	struct amdgpu_gtt_node *node = mem->mm_node;
 
 	if (!node)
 		return;
 
 	spin_lock(&mgr->lock);
-	if (node->start != AMDGPU_BO_INVALID_OFFSET)
-		drm_mm_remove_node(node);
+	if (node->node.start != AMDGPU_BO_INVALID_OFFSET)
+		drm_mm_remove_node(&node->node);
 	spin_unlock(&mgr->lock);
 	atomic64_add(mem->num_pages, &mgr->available);
 
@@ -244,6 +250,25 @@ uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man)
 	return (result > 0 ? result : 0) * PAGE_SIZE;
 }
 
+int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man)
+{
+	struct amdgpu_gtt_mgr *mgr = man->priv;
+	struct amdgpu_gtt_node *node;
+	struct drm_mm_node *mm_node;
+	int r = 0;
+
+	spin_lock(&mgr->lock);
+	drm_mm_for_each_node(mm_node, &mgr->mm) {
+		node = container_of(mm_node, struct amdgpu_gtt_node, node);
+		r = amdgpu_ttm_recover_gart(node->tbo);
+		if (r)
+			break;
+	}
+	spin_unlock(&mgr->lock);
+
+	return r;
+}
+
 /**
  * amdgpu_gtt_mgr_debug - dump VRAM table
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 659997bfff30..a162d87ca0c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -149,7 +149,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 		return -EINVAL;
 	}
 
-	if (vm && !job->vm_id) {
+	if (vm && !job->vmid) {
 		dev_err(adev->dev, "VM IB without ID\n");
 		return -EINVAL;
 	}
@@ -164,7 +164,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	}
 
 	if (ring->funcs->emit_pipeline_sync && job &&
-	    ((tmp = amdgpu_sync_get_fence(&job->sched_sync)) ||
+	    ((tmp = amdgpu_sync_get_fence(&job->sched_sync, NULL)) ||
 	     amdgpu_vm_need_pipeline_sync(ring, job))) {
 		need_pipe_sync = true;
 		dma_fence_put(tmp);
@@ -211,7 +211,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 			!amdgpu_sriov_vf(adev)) /* for SRIOV preemption, Preamble CE ib must be inserted anyway */
 			continue;
 
-		amdgpu_ring_emit_ib(ring, ib, job ? job->vm_id : 0,
+		amdgpu_ring_emit_ib(ring, ib, job ? job->vmid : 0,
 				    need_ctx_switch);
 		need_ctx_switch = false;
 	}
@@ -229,9 +229,8 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 	r = amdgpu_fence_emit(ring, f);
 	if (r) {
 		dev_err(adev->dev, "failed to emit fence (%d)\n", r);
-		if (job && job->vm_id)
-			amdgpu_vm_reset_id(adev, ring->funcs->vmhub,
-					   job->vm_id);
+		if (job && job->vmid)
+			amdgpu_vmid_reset(adev, ring->funcs->vmhub, job->vmid);
 		amdgpu_ring_undo(ring);
 		return r;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
new file mode 100644
index 000000000000..16884a0b677b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -0,0 +1,459 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu_ids.h"
+
+#include <linux/idr.h>
+#include <linux/dma-fence-array.h>
+#include <drm/drmP.h>
+
+#include "amdgpu.h"
+#include "amdgpu_trace.h"
+
+/*
+ * PASID manager
+ *
+ * PASIDs are global address space identifiers that can be shared
+ * between the GPU, an IOMMU and the driver. VMs on different devices
+ * may use the same PASID if they share the same address
+ * space. Therefore PASIDs are allocated using a global IDA. VMs are
+ * looked up from the PASID per amdgpu_device.
+ */
+static DEFINE_IDA(amdgpu_pasid_ida);
+
+/**
+ * amdgpu_pasid_alloc - Allocate a PASID
+ * @bits: Maximum width of the PASID in bits, must be at least 1
+ *
+ * Allocates a PASID of the given width while keeping smaller PASIDs
+ * available if possible.
+ *
+ * Returns a positive integer on success. Returns %-EINVAL if bits==0.
+ * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
+ * memory allocation failure.
+ */
+int amdgpu_pasid_alloc(unsigned int bits)
+{
+	int pasid = -EINVAL;
+
+	for (bits = min(bits, 31U); bits > 0; bits--) {
+		pasid = ida_simple_get(&amdgpu_pasid_ida,
+				       1U << (bits - 1), 1U << bits,
+				       GFP_KERNEL);
+		if (pasid != -ENOSPC)
+			break;
+	}
+
+	return pasid;
+}
+
+/**
+ * amdgpu_pasid_free - Free a PASID
+ * @pasid: PASID to free
+ */
+void amdgpu_pasid_free(unsigned int pasid)
+{
+	ida_simple_remove(&amdgpu_pasid_ida, pasid);
+}
+
+/*
+ * VMID manager
+ *
+ * VMIDs are a per VMHUB identifier for page tables handling.
+ */
+
+/**
+ * amdgpu_vmid_had_gpu_reset - check if reset occured since last use
+ *
+ * @adev: amdgpu_device pointer
+ * @id: VMID structure
+ *
+ * Check if GPU reset occured since last use of the VMID.
+ */
+bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev,
+			       struct amdgpu_vmid *id)
+{
+	return id->current_gpu_reset_count !=
+		atomic_read(&adev->gpu_reset_counter);
+}
+
+/* idr_mgr->lock must be held */
+static int amdgpu_vmid_grab_reserved_locked(struct amdgpu_vm *vm,
+					    struct amdgpu_ring *ring,
+					    struct amdgpu_sync *sync,
+					    struct dma_fence *fence,
+					    struct amdgpu_job *job)
+{
+	struct amdgpu_device *adev = ring->adev;
+	unsigned vmhub = ring->funcs->vmhub;
+	uint64_t fence_context = adev->fence_context + ring->idx;
+	struct amdgpu_vmid *id = vm->reserved_vmid[vmhub];
+	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
+	struct dma_fence *updates = sync->last_vm_update;
+	int r = 0;
+	struct dma_fence *flushed, *tmp;
+	bool needs_flush = vm->use_cpu_for_update;
+
+	flushed  = id->flushed_updates;
+	if ((amdgpu_vmid_had_gpu_reset(adev, id)) ||
+	    (atomic64_read(&id->owner) != vm->entity.fence_context) ||
+	    (job->vm_pd_addr != id->pd_gpu_addr) ||
+	    (updates && (!flushed || updates->context != flushed->context ||
+			dma_fence_is_later(updates, flushed))) ||
+	    (!id->last_flush || (id->last_flush->context != fence_context &&
+				 !dma_fence_is_signaled(id->last_flush)))) {
+		needs_flush = true;
+		/* to prevent one context starved by another context */
+		id->pd_gpu_addr = 0;
+		tmp = amdgpu_sync_peek_fence(&id->active, ring);
+		if (tmp) {
+			r = amdgpu_sync_fence(adev, sync, tmp, false);
+			return r;
+		}
+	}
+
+	/* Good we can use this VMID. Remember this submission as
+	* user of the VMID.
+	*/
+	r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
+	if (r)
+		goto out;
+
+	if (updates && (!flushed || updates->context != flushed->context ||
+			dma_fence_is_later(updates, flushed))) {
+		dma_fence_put(id->flushed_updates);
+		id->flushed_updates = dma_fence_get(updates);
+	}
+	id->pd_gpu_addr = job->vm_pd_addr;
+	atomic64_set(&id->owner, vm->entity.fence_context);
+	job->vm_needs_flush = needs_flush;
+	if (needs_flush) {
+		dma_fence_put(id->last_flush);
+		id->last_flush = NULL;
+	}
+	job->vmid = id - id_mgr->ids;
+	trace_amdgpu_vm_grab_id(vm, ring, job);
+out:
+	return r;
+}
+
+/**
+ * amdgpu_vm_grab_id - allocate the next free VMID
+ *
+ * @vm: vm to allocate id for
+ * @ring: ring we want to submit job to
+ * @sync: sync object where we add dependencies
+ * @fence: fence protecting ID from reuse
+ *
+ * Allocate an id for the vm, adding fences to the sync obj as necessary.
+ */
+int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
+		     struct amdgpu_sync *sync, struct dma_fence *fence,
+		     struct amdgpu_job *job)
+{
+	struct amdgpu_device *adev = ring->adev;
+	unsigned vmhub = ring->funcs->vmhub;
+	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
+	uint64_t fence_context = adev->fence_context + ring->idx;
+	struct dma_fence *updates = sync->last_vm_update;
+	struct amdgpu_vmid *id, *idle;
+	struct dma_fence **fences;
+	unsigned i;
+	int r = 0;
+
+	mutex_lock(&id_mgr->lock);
+	if (vm->reserved_vmid[vmhub]) {
+		r = amdgpu_vmid_grab_reserved_locked(vm, ring, sync, fence, job);
+		mutex_unlock(&id_mgr->lock);
+		return r;
+	}
+	fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
+	if (!fences) {
+		mutex_unlock(&id_mgr->lock);
+		return -ENOMEM;
+	}
+	/* Check if we have an idle VMID */
+	i = 0;
+	list_for_each_entry(idle, &id_mgr->ids_lru, list) {
+		fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
+		if (!fences[i])
+			break;
+		++i;
+	}
+
+	/* If we can't find a idle VMID to use, wait till one becomes available */
+	if (&idle->list == &id_mgr->ids_lru) {
+		u64 fence_context = adev->vm_manager.fence_context + ring->idx;
+		unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
+		struct dma_fence_array *array;
+		unsigned j;
+
+		for (j = 0; j < i; ++j)
+			dma_fence_get(fences[j]);
+
+		array = dma_fence_array_create(i, fences, fence_context,
+					   seqno, true);
+		if (!array) {
+			for (j = 0; j < i; ++j)
+				dma_fence_put(fences[j]);
+			kfree(fences);
+			r = -ENOMEM;
+			goto error;
+		}
+
+
+		r = amdgpu_sync_fence(ring->adev, sync, &array->base, false);
+		dma_fence_put(&array->base);
+		if (r)
+			goto error;
+
+		mutex_unlock(&id_mgr->lock);
+		return 0;
+
+	}
+	kfree(fences);
+
+	job->vm_needs_flush = vm->use_cpu_for_update;
+	/* Check if we can use a VMID already assigned to this VM */
+	list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
+		struct dma_fence *flushed;
+		bool needs_flush = vm->use_cpu_for_update;
+
+		/* Check all the prerequisites to using this VMID */
+		if (amdgpu_vmid_had_gpu_reset(adev, id))
+			continue;
+
+		if (atomic64_read(&id->owner) != vm->entity.fence_context)
+			continue;
+
+		if (job->vm_pd_addr != id->pd_gpu_addr)
+			continue;
+
+		if (!id->last_flush ||
+		    (id->last_flush->context != fence_context &&
+		     !dma_fence_is_signaled(id->last_flush)))
+			needs_flush = true;
+
+		flushed  = id->flushed_updates;
+		if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
+			needs_flush = true;
+
+		/* Concurrent flushes are only possible starting with Vega10 */
+		if (adev->asic_type < CHIP_VEGA10 && needs_flush)
+			continue;
+
+		/* Good we can use this VMID. Remember this submission as
+		 * user of the VMID.
+		 */
+		r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
+		if (r)
+			goto error;
+
+		if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
+			dma_fence_put(id->flushed_updates);
+			id->flushed_updates = dma_fence_get(updates);
+		}
+
+		if (needs_flush)
+			goto needs_flush;
+		else
+			goto no_flush_needed;
+
+	};
+
+	/* Still no ID to use? Then use the idle one found earlier */
+	id = idle;
+
+	/* Remember this submission as user of the VMID */
+	r = amdgpu_sync_fence(ring->adev, &id->active, fence, false);
+	if (r)
+		goto error;
+
+	id->pd_gpu_addr = job->vm_pd_addr;
+	dma_fence_put(id->flushed_updates);
+	id->flushed_updates = dma_fence_get(updates);
+	atomic64_set(&id->owner, vm->entity.fence_context);
+
+needs_flush:
+	job->vm_needs_flush = true;
+	dma_fence_put(id->last_flush);
+	id->last_flush = NULL;
+
+no_flush_needed:
+	list_move_tail(&id->list, &id_mgr->ids_lru);
+
+	job->vmid = id - id_mgr->ids;
+	trace_amdgpu_vm_grab_id(vm, ring, job);
+
+error:
+	mutex_unlock(&id_mgr->lock);
+	return r;
+}
+
+int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev,
+			       struct amdgpu_vm *vm,
+			       unsigned vmhub)
+{
+	struct amdgpu_vmid_mgr *id_mgr;
+	struct amdgpu_vmid *idle;
+	int r = 0;
+
+	id_mgr = &adev->vm_manager.id_mgr[vmhub];
+	mutex_lock(&id_mgr->lock);
+	if (vm->reserved_vmid[vmhub])
+		goto unlock;
+	if (atomic_inc_return(&id_mgr->reserved_vmid_num) >
+	    AMDGPU_VM_MAX_RESERVED_VMID) {
+		DRM_ERROR("Over limitation of reserved vmid\n");
+		atomic_dec(&id_mgr->reserved_vmid_num);
+		r = -EINVAL;
+		goto unlock;
+	}
+	/* Select the first entry VMID */
+	idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vmid, list);
+	list_del_init(&idle->list);
+	vm->reserved_vmid[vmhub] = idle;
+	mutex_unlock(&id_mgr->lock);
+
+	return 0;
+unlock:
+	mutex_unlock(&id_mgr->lock);
+	return r;
+}
+
+void amdgpu_vmid_free_reserved(struct amdgpu_device *adev,
+			       struct amdgpu_vm *vm,
+			       unsigned vmhub)
+{
+	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
+
+	mutex_lock(&id_mgr->lock);
+	if (vm->reserved_vmid[vmhub]) {
+		list_add(&vm->reserved_vmid[vmhub]->list,
+			&id_mgr->ids_lru);
+		vm->reserved_vmid[vmhub] = NULL;
+		atomic_dec(&id_mgr->reserved_vmid_num);
+	}
+	mutex_unlock(&id_mgr->lock);
+}
+
+/**
+ * amdgpu_vmid_reset - reset VMID to zero
+ *
+ * @adev: amdgpu device structure
+ * @vmid: vmid number to use
+ *
+ * Reset saved GDW, GWS and OA to force switch on next flush.
+ */
+void amdgpu_vmid_reset(struct amdgpu_device *adev, unsigned vmhub,
+		       unsigned vmid)
+{
+	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
+	struct amdgpu_vmid *id = &id_mgr->ids[vmid];
+
+	atomic64_set(&id->owner, 0);
+	id->gds_base = 0;
+	id->gds_size = 0;
+	id->gws_base = 0;
+	id->gws_size = 0;
+	id->oa_base = 0;
+	id->oa_size = 0;
+}
+
+/**
+ * amdgpu_vmid_reset_all - reset VMID to zero
+ *
+ * @adev: amdgpu device structure
+ *
+ * Reset VMID to force flush on next use
+ */
+void amdgpu_vmid_reset_all(struct amdgpu_device *adev)
+{
+	unsigned i, j;
+
+	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
+		struct amdgpu_vmid_mgr *id_mgr =
+			&adev->vm_manager.id_mgr[i];
+
+		for (j = 1; j < id_mgr->num_ids; ++j)
+			amdgpu_vmid_reset(adev, i, j);
+	}
+}
+
+/**
+ * amdgpu_vmid_mgr_init - init the VMID manager
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Initialize the VM manager structures
+ */
+void amdgpu_vmid_mgr_init(struct amdgpu_device *adev)
+{
+	unsigned i, j;
+
+	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
+		struct amdgpu_vmid_mgr *id_mgr =
+			&adev->vm_manager.id_mgr[i];
+
+		mutex_init(&id_mgr->lock);
+		INIT_LIST_HEAD(&id_mgr->ids_lru);
+		atomic_set(&id_mgr->reserved_vmid_num, 0);
+
+		/* skip over VMID 0, since it is the system VM */
+		for (j = 1; j < id_mgr->num_ids; ++j) {
+			amdgpu_vmid_reset(adev, i, j);
+			amdgpu_sync_create(&id_mgr->ids[i].active);
+			list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
+		}
+	}
+
+	adev->vm_manager.fence_context =
+		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
+	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
+		adev->vm_manager.seqno[i] = 0;
+}
+
+/**
+ * amdgpu_vmid_mgr_fini - cleanup VM manager
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Cleanup the VM manager and free resources.
+ */
+void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev)
+{
+	unsigned i, j;
+
+	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
+		struct amdgpu_vmid_mgr *id_mgr =
+			&adev->vm_manager.id_mgr[i];
+
+		mutex_destroy(&id_mgr->lock);
+		for (j = 0; j < AMDGPU_NUM_VMID; ++j) {
+			struct amdgpu_vmid *id = &id_mgr->ids[j];
+
+			amdgpu_sync_free(&id->active);
+			dma_fence_put(id->flushed_updates);
+			dma_fence_put(id->last_flush);
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
new file mode 100644
index 000000000000..ad931fa570b3
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.h
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __AMDGPU_IDS_H__
+#define __AMDGPU_IDS_H__
+
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/dma-fence.h>
+
+#include "amdgpu_sync.h"
+
+/* maximum number of VMIDs */
+#define AMDGPU_NUM_VMID	16
+
+struct amdgpu_device;
+struct amdgpu_vm;
+struct amdgpu_ring;
+struct amdgpu_sync;
+struct amdgpu_job;
+
+struct amdgpu_vmid {
+	struct list_head	list;
+	struct amdgpu_sync	active;
+	struct dma_fence	*last_flush;
+	atomic64_t		owner;
+
+	uint64_t		pd_gpu_addr;
+	/* last flushed PD/PT update */
+	struct dma_fence	*flushed_updates;
+
+	uint32_t                current_gpu_reset_count;
+
+	uint32_t		gds_base;
+	uint32_t		gds_size;
+	uint32_t		gws_base;
+	uint32_t		gws_size;
+	uint32_t		oa_base;
+	uint32_t		oa_size;
+};
+
+struct amdgpu_vmid_mgr {
+	struct mutex		lock;
+	unsigned		num_ids;
+	struct list_head	ids_lru;
+	struct amdgpu_vmid	ids[AMDGPU_NUM_VMID];
+	atomic_t		reserved_vmid_num;
+};
+
+int amdgpu_pasid_alloc(unsigned int bits);
+void amdgpu_pasid_free(unsigned int pasid);
+
+bool amdgpu_vmid_had_gpu_reset(struct amdgpu_device *adev,
+			       struct amdgpu_vmid *id);
+int amdgpu_vmid_alloc_reserved(struct amdgpu_device *adev,
+			       struct amdgpu_vm *vm,
+			       unsigned vmhub);
+void amdgpu_vmid_free_reserved(struct amdgpu_device *adev,
+			       struct amdgpu_vm *vm,
+			       unsigned vmhub);
+int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
+		     struct amdgpu_sync *sync, struct dma_fence *fence,
+		     struct amdgpu_job *job);
+void amdgpu_vmid_reset(struct amdgpu_device *adev, unsigned vmhub,
+		       unsigned vmid);
+void amdgpu_vmid_reset_all(struct amdgpu_device *adev);
+
+void amdgpu_vmid_mgr_init(struct amdgpu_device *adev);
+void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index f5f27e4f0f7f..06373d44b3da 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -92,15 +92,15 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size,
 		}
 		return 0;
 	} else {
-		r = amdgpu_wb_get(adev, &adev->irq.ih.wptr_offs);
+		r = amdgpu_device_wb_get(adev, &adev->irq.ih.wptr_offs);
 		if (r) {
 			dev_err(adev->dev, "(%d) ih wptr_offs wb alloc failed\n", r);
 			return r;
 		}
 
-		r = amdgpu_wb_get(adev, &adev->irq.ih.rptr_offs);
+		r = amdgpu_device_wb_get(adev, &adev->irq.ih.rptr_offs);
 		if (r) {
-			amdgpu_wb_free(adev, adev->irq.ih.wptr_offs);
+			amdgpu_device_wb_free(adev, adev->irq.ih.wptr_offs);
 			dev_err(adev->dev, "(%d) ih rptr_offs wb alloc failed\n", r);
 			return r;
 		}
@@ -133,8 +133,8 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev)
 		amdgpu_bo_free_kernel(&adev->irq.ih.ring_obj,
 				      &adev->irq.ih.gpu_addr,
 				      (void **)&adev->irq.ih.ring);
-		amdgpu_wb_free(adev, adev->irq.ih.wptr_offs);
-		amdgpu_wb_free(adev, adev->irq.ih.rptr_offs);
+		amdgpu_device_wb_free(adev, adev->irq.ih.wptr_offs);
+		amdgpu_device_wb_free(adev, adev->irq.ih.rptr_offs);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index ada89358e220..29cf10927a92 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -105,8 +105,8 @@ struct amdgpu_iv_entry {
 	unsigned client_id;
 	unsigned src_id;
 	unsigned ring_id;
-	unsigned vm_id;
-	unsigned vm_id_src;
+	unsigned vmid;
+	unsigned vmid_src;
 	uint64_t timestamp;
 	unsigned timestamp_src;
 	unsigned pas_id;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 47c5ce9807db..36483e0d3c97 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -88,7 +88,7 @@ static void amdgpu_irq_reset_work_func(struct work_struct *work)
 						  reset_work);
 
 	if (!amdgpu_sriov_vf(adev))
-		amdgpu_gpu_reset(adev);
+		amdgpu_device_gpu_recover(adev, NULL, false);
 }
 
 /* Disable *all* interrupts */
@@ -232,7 +232,7 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
 		int ret = pci_enable_msi(adev->pdev);
 		if (!ret) {
 			adev->irq.msi_enabled = true;
-			dev_info(adev->dev, "amdgpu: using MSI.\n");
+			dev_dbg(adev->dev, "amdgpu: using MSI.\n");
 		}
 	}
 
@@ -257,12 +257,13 @@ int amdgpu_irq_init(struct amdgpu_device *adev)
 	r = drm_irq_install(adev->ddev, adev->ddev->pdev->irq);
 	if (r) {
 		adev->irq.installed = false;
-		flush_work(&adev->hotplug_work);
+		if (!amdgpu_device_has_dc_support(adev))
+			flush_work(&adev->hotplug_work);
 		cancel_work_sync(&adev->reset_work);
 		return r;
 	}
 
-	DRM_INFO("amdgpu: irq initialized.\n");
+	DRM_DEBUG("amdgpu: irq initialized.\n");
 	return 0;
 }
 
@@ -282,7 +283,8 @@ void amdgpu_irq_fini(struct amdgpu_device *adev)
 		adev->irq.installed = false;
 		if (adev->irq.msi_enabled)
 			pci_disable_msi(adev->pdev);
-		flush_work(&adev->hotplug_work);
+		if (!amdgpu_device_has_dc_support(adev))
+			flush_work(&adev->hotplug_work);
 		cancel_work_sync(&adev->reset_work);
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 0cfc68db575b..2bd56760c744 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -28,7 +28,7 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
-static void amdgpu_job_timedout(struct amd_sched_job *s_job)
+static void amdgpu_job_timedout(struct drm_sched_job *s_job)
 {
 	struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
 
@@ -37,10 +37,7 @@ static void amdgpu_job_timedout(struct amd_sched_job *s_job)
 		  atomic_read(&job->ring->fence_drv.last_seq),
 		  job->ring->fence_drv.sync_seq);
 
-	if (amdgpu_sriov_vf(job->adev))
-		amdgpu_sriov_gpu_reset(job->adev, job);
-	else
-		amdgpu_gpu_reset(job->adev);
+	amdgpu_device_gpu_recover(job->adev, job, false);
 }
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
@@ -63,7 +60,6 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 	(*job)->num_ibs = num_ibs;
 
 	amdgpu_sync_create(&(*job)->sync);
-	amdgpu_sync_create(&(*job)->dep_sync);
 	amdgpu_sync_create(&(*job)->sched_sync);
 	(*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter);
 
@@ -100,14 +96,13 @@ void amdgpu_job_free_resources(struct amdgpu_job *job)
 		amdgpu_ib_free(job->adev, &job->ibs[i], f);
 }
 
-static void amdgpu_job_free_cb(struct amd_sched_job *s_job)
+static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
 {
 	struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
 
-	amdgpu_ring_priority_put(job->ring, amd_sched_get_job_priority(s_job));
+	amdgpu_ring_priority_put(job->ring, s_job->s_priority);
 	dma_fence_put(job->fence);
 	amdgpu_sync_free(&job->sync);
-	amdgpu_sync_free(&job->dep_sync);
 	amdgpu_sync_free(&job->sched_sync);
 	kfree(job);
 }
@@ -118,13 +113,12 @@ void amdgpu_job_free(struct amdgpu_job *job)
 
 	dma_fence_put(job->fence);
 	amdgpu_sync_free(&job->sync);
-	amdgpu_sync_free(&job->dep_sync);
 	amdgpu_sync_free(&job->sched_sync);
 	kfree(job);
 }
 
 int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
-		      struct amd_sched_entity *entity, void *owner,
+		      struct drm_sched_entity *entity, void *owner,
 		      struct dma_fence **f)
 {
 	int r;
@@ -133,7 +127,7 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 	if (!f)
 		return -EINVAL;
 
-	r = amd_sched_job_init(&job->base, &ring->sched, entity, owner);
+	r = drm_sched_job_init(&job->base, &ring->sched, entity, owner);
 	if (r)
 		return r;
 
@@ -141,46 +135,47 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
 	job->fence_ctx = entity->fence_context;
 	*f = dma_fence_get(&job->base.s_fence->finished);
 	amdgpu_job_free_resources(job);
-	amdgpu_ring_priority_get(job->ring,
-				 amd_sched_get_job_priority(&job->base));
-	amd_sched_entity_push_job(&job->base);
+	amdgpu_ring_priority_get(job->ring, job->base.s_priority);
+	drm_sched_entity_push_job(&job->base, entity);
 
 	return 0;
 }
 
-static struct dma_fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
+static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job,
+					       struct drm_sched_entity *s_entity)
 {
 	struct amdgpu_job *job = to_amdgpu_job(sched_job);
 	struct amdgpu_vm *vm = job->vm;
-
-	struct dma_fence *fence = amdgpu_sync_get_fence(&job->dep_sync);
+	bool explicit = false;
 	int r;
-
-	if (amd_sched_dependency_optimized(fence, sched_job->s_entity)) {
-		r = amdgpu_sync_fence(job->adev, &job->sched_sync, fence);
-		if (r)
-			DRM_ERROR("Error adding fence to sync (%d)\n", r);
+	struct dma_fence *fence = amdgpu_sync_get_fence(&job->sync, &explicit);
+
+	if (fence && explicit) {
+		if (drm_sched_dependency_optimized(fence, s_entity)) {
+			r = amdgpu_sync_fence(job->adev, &job->sched_sync, fence, false);
+			if (r)
+				DRM_ERROR("Error adding fence to sync (%d)\n", r);
+		}
 	}
-	if (!fence)
-		fence = amdgpu_sync_get_fence(&job->sync);
-	while (fence == NULL && vm && !job->vm_id) {
+
+	while (fence == NULL && vm && !job->vmid) {
 		struct amdgpu_ring *ring = job->ring;
 
-		r = amdgpu_vm_grab_id(vm, ring, &job->sync,
-				      &job->base.s_fence->finished,
-				      job);
+		r = amdgpu_vmid_grab(vm, ring, &job->sync,
+				     &job->base.s_fence->finished,
+				     job);
 		if (r)
 			DRM_ERROR("Error getting VM ID (%d)\n", r);
 
-		fence = amdgpu_sync_get_fence(&job->sync);
+		fence = amdgpu_sync_get_fence(&job->sync, NULL);
 	}
 
 	return fence;
 }
 
-static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
+static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 {
-	struct dma_fence *fence = NULL;
+	struct dma_fence *fence = NULL, *finished;
 	struct amdgpu_device *adev;
 	struct amdgpu_job *job;
 	int r;
@@ -190,15 +185,18 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
 		return NULL;
 	}
 	job = to_amdgpu_job(sched_job);
+	finished = &job->base.s_fence->finished;
 	adev = job->adev;
 
 	BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
 	trace_amdgpu_sched_run_job(job);
-	/* skip ib schedule when vram is lost */
-	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) {
-		dma_fence_set_error(&job->base.s_fence->finished, -ECANCELED);
-		DRM_ERROR("Skip scheduling IBs!\n");
+
+	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter))
+		dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
+
+	if (finished->error < 0) {
+		DRM_INFO("Skip scheduling IBs!\n");
 	} else {
 		r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs, job,
 				       &fence);
@@ -213,7 +211,7 @@ static struct dma_fence *amdgpu_job_run(struct amd_sched_job *sched_job)
 	return fence;
 }
 
-const struct amd_sched_backend_ops amdgpu_sched_ops = {
+const struct drm_sched_backend_ops amdgpu_sched_ops = {
 	.dependency = amdgpu_job_dependency,
 	.run_job = amdgpu_job_run,
 	.timedout_job = amdgpu_job_timedout,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 720139e182a3..bd6e9a40f421 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -63,8 +63,6 @@ void amdgpu_driver_unload_kms(struct drm_device *dev)
 		pm_runtime_forbid(dev->dev);
 	}
 
-	amdgpu_amdkfd_device_fini(adev);
-
 	amdgpu_acpi_fini(adev);
 
 	amdgpu_device_fini(adev);
@@ -159,9 +157,6 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
 				"Error during ACPI methods call\n");
 	}
 
-	amdgpu_amdkfd_device_probe(adev);
-	amdgpu_amdkfd_device_init(adev);
-
 	if (amdgpu_device_is_px(dev)) {
 		pm_runtime_use_autosuspend(dev->dev);
 		pm_runtime_set_autosuspend_delay(dev->dev, 5000);
@@ -171,9 +166,6 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
 		pm_runtime_put_autosuspend(dev->dev);
 	}
 
-	if (amdgpu_sriov_vf(adev))
-		amdgpu_virt_release_full_gpu(adev, true);
-
 out:
 	if (r) {
 		/* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */
@@ -558,6 +550,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 	}
 	case AMDGPU_INFO_DEV_INFO: {
 		struct drm_amdgpu_info_device dev_info = {};
+		uint64_t vm_size;
 
 		dev_info.device_id = dev->pdev->device;
 		dev_info.chip_rev = adev->rev_id;
@@ -585,8 +578,17 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION;
 		if (amdgpu_sriov_vf(adev))
 			dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION;
+
+		vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
 		dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE;
-		dev_info.virtual_address_max = (uint64_t)adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
+		dev_info.virtual_address_max =
+			min(vm_size, AMDGPU_VA_HOLE_START);
+
+		vm_size -= AMDGPU_VA_RESERVED_SIZE;
+		if (vm_size > AMDGPU_VA_HOLE_START) {
+			dev_info.high_va_offset = AMDGPU_VA_HOLE_END;
+			dev_info.high_va_max = AMDGPU_VA_HOLE_END | vm_size;
+		}
 		dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE);
 		dev_info.pte_fragment_size = (1 << adev->vm_manager.fragment_size) * AMDGPU_GPU_PAGE_SIZE;
 		dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE;
@@ -786,9 +788,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
  */
 void amdgpu_driver_lastclose_kms(struct drm_device *dev)
 {
-	struct amdgpu_device *adev = dev->dev_private;
-
-	amdgpu_fbdev_restore_mode(adev);
+	drm_fb_helper_lastclose(dev);
 	vga_switcheroo_process_delayed_switch();
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index ffde1e9666e8..2264c5c97009 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -89,7 +89,6 @@ enum amdgpu_hpd_id {
 	AMDGPU_HPD_4,
 	AMDGPU_HPD_5,
 	AMDGPU_HPD_6,
-	AMDGPU_HPD_LAST,
 	AMDGPU_HPD_NONE = 0xff,
 };
 
@@ -106,7 +105,6 @@ enum amdgpu_crtc_irq {
 	AMDGPU_CRTC_IRQ_VLINE4,
 	AMDGPU_CRTC_IRQ_VLINE5,
 	AMDGPU_CRTC_IRQ_VLINE6,
-	AMDGPU_CRTC_IRQ_LAST,
 	AMDGPU_CRTC_IRQ_NONE = 0xff
 };
 
@@ -117,7 +115,6 @@ enum amdgpu_pageflip_irq {
 	AMDGPU_PAGEFLIP_IRQ_D4,
 	AMDGPU_PAGEFLIP_IRQ_D5,
 	AMDGPU_PAGEFLIP_IRQ_D6,
-	AMDGPU_PAGEFLIP_IRQ_LAST,
 	AMDGPU_PAGEFLIP_IRQ_NONE = 0xff
 };
 
@@ -355,6 +352,7 @@ struct amdgpu_mode_info {
 	u16 firmware_flags;
 	/* pointer to backlight encoder */
 	struct amdgpu_encoder *bl_encoder;
+	u8 bl_level; /* saved backlight level */
 	struct amdgpu_audio	audio; /* audio stuff */
 	int			num_crtc; /* number of crtcs */
 	int			num_hpd; /* number of hpd pins */
@@ -661,10 +659,6 @@ void amdgpu_fbdev_fini(struct amdgpu_device *adev);
 void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state);
 int amdgpu_fbdev_total_size(struct amdgpu_device *adev);
 bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj);
-void amdgpu_fbdev_restore_mode(struct amdgpu_device *adev);
-
-void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev);
-
 
 int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tiled);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index ea25164e7f4b..1220322c1680 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -37,6 +37,18 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
+static bool amdgpu_need_backup(struct amdgpu_device *adev)
+{
+	if (adev->flags & AMD_IS_APU)
+		return false;
+
+	if (amdgpu_gpu_recovery == 0 ||
+	    (amdgpu_gpu_recovery == -1  && !amdgpu_sriov_vf(adev)))
+		return false;
+
+	return true;
+}
+
 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
@@ -44,6 +56,8 @@ static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 
 	amdgpu_bo_kunmap(bo);
 
+	if (bo->gem_base.import_attach)
+		drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
 	drm_gem_object_release(&bo->gem_base);
 	amdgpu_bo_unref(&bo->parent);
 	if (!list_empty(&bo->shadow_list)) {
@@ -281,6 +295,44 @@ void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
 		*cpu_addr = NULL;
 }
 
+/* Validate bo size is bit bigger then the request domain */
+static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
+					  unsigned long size, u32 domain)
+{
+	struct ttm_mem_type_manager *man = NULL;
+
+	/*
+	 * If GTT is part of requested domains the check must succeed to
+	 * allow fall back to GTT
+	 */
+	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
+		man = &adev->mman.bdev.man[TTM_PL_TT];
+
+		if (size < (man->size << PAGE_SHIFT))
+			return true;
+		else
+			goto fail;
+	}
+
+	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
+		man = &adev->mman.bdev.man[TTM_PL_VRAM];
+
+		if (size < (man->size << PAGE_SHIFT))
+			return true;
+		else
+			goto fail;
+	}
+
+
+	/* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
+	return true;
+
+fail:
+	DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
+		  man->size << PAGE_SHIFT);
+	return false;
+}
+
 static int amdgpu_bo_do_create(struct amdgpu_device *adev,
 			       unsigned long size, int byte_align,
 			       bool kernel, u32 domain, u64 flags,
@@ -289,16 +341,24 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
 			       uint64_t init_value,
 			       struct amdgpu_bo **bo_ptr)
 {
+	struct ttm_operation_ctx ctx = {
+		.interruptible = !kernel,
+		.no_wait_gpu = false,
+		.allow_reserved_eviction = true,
+		.resv = resv
+	};
 	struct amdgpu_bo *bo;
 	enum ttm_bo_type type;
 	unsigned long page_align;
-	u64 initial_bytes_moved, bytes_moved;
 	size_t acc_size;
 	int r;
 
 	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
 	size = ALIGN(size, PAGE_SIZE);
 
+	if (!amdgpu_bo_validate_size(adev, size, domain))
+		return -ENOMEM;
+
 	if (kernel) {
 		type = ttm_bo_type_kernel;
 	} else if (sg) {
@@ -364,22 +424,19 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev,
 	bo->tbo.bdev = &adev->mman.bdev;
 	amdgpu_ttm_placement_from_domain(bo, domain);
 
-	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
-	/* Kernel allocation are uninterruptible */
 	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
-				 &bo->placement, page_align, !kernel, NULL,
+				 &bo->placement, page_align, &ctx, NULL,
 				 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
 	if (unlikely(r != 0))
 		return r;
 
-	bytes_moved = atomic64_read(&adev->num_bytes_moved) -
-		      initial_bytes_moved;
 	if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
 	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&
 	    bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
-		amdgpu_cs_report_moved_bytes(adev, bytes_moved, bytes_moved);
+		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
+					     ctx.bytes_moved);
 	else
-		amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0);
+		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
 
 	if (kernel)
 		bo->tbo.priority = 1;
@@ -511,6 +568,7 @@ int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
 
 int amdgpu_bo_validate(struct amdgpu_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	uint32_t domain;
 	int r;
 
@@ -521,7 +579,7 @@ int amdgpu_bo_validate(struct amdgpu_bo *bo)
 
 retry:
 	amdgpu_ttm_placement_from_domain(bo, domain);
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
 		domain = bo->allowed_domains;
 		goto retry;
@@ -632,6 +690,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 			     u64 *gpu_addr)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	struct ttm_operation_ctx ctx = { false, false };
 	int r, i;
 
 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
@@ -647,7 +706,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 	if (bo->pin_count) {
 		uint32_t mem_type = bo->tbo.mem.mem_type;
 
-		if (domain != amdgpu_mem_type_to_domain(mem_type))
+		if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
 			return -EINVAL;
 
 		bo->pin_count++;
@@ -682,21 +741,23 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 	}
 
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (unlikely(r)) {
 		dev_err(adev->dev, "%p pin failed\n", bo);
 		goto error;
 	}
 
+	r = amdgpu_ttm_alloc_gart(&bo->tbo);
+	if (unlikely(r)) {
+		dev_err(adev->dev, "%p bind failed\n", bo);
+		goto error;
+	}
+
 	bo->pin_count = 1;
-	if (gpu_addr != NULL) {
-		r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
-		if (unlikely(r)) {
-			dev_err(adev->dev, "%p bind failed\n", bo);
-			goto error;
-		}
+	if (gpu_addr != NULL)
 		*gpu_addr = amdgpu_bo_gpu_offset(bo);
-	}
+
+	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
 	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
 		adev->vram_pin_size += amdgpu_bo_size(bo);
 		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
@@ -717,6 +778,7 @@ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
 int amdgpu_bo_unpin(struct amdgpu_bo *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	struct ttm_operation_ctx ctx = { false, false };
 	int r, i;
 
 	if (!bo->pin_count) {
@@ -730,7 +792,7 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo)
 		bo->placements[i].lpfn = 0;
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
 	}
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (unlikely(r)) {
 		dev_err(adev->dev, "%p validate failed for unpin\n", bo);
 		goto error;
@@ -779,8 +841,8 @@ int amdgpu_bo_init(struct amdgpu_device *adev)
 	adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
 					      adev->mc.aper_size);
 	DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
-		adev->mc.mc_vram_size >> 20,
-		(unsigned long long)adev->mc.aper_size >> 20);
+		 adev->mc.mc_vram_size >> 20,
+		 (unsigned long long)adev->mc.aper_size >> 20);
 	DRM_INFO("RAM width %dbits %s\n",
 		 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
 	return amdgpu_ttm_init(adev);
@@ -902,6 +964,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
+	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_bo *abo;
 	unsigned long offset, size;
 	int r;
@@ -935,7 +998,7 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 	abo->placement.num_busy_placement = 1;
 	abo->placement.busy_placement = &abo->placements[1];
 
-	r = ttm_bo_validate(bo, &abo->placement, false, false);
+	r = ttm_bo_validate(bo, &abo->placement, &ctx);
 	if (unlikely(r != 0))
 		return r;
 
@@ -980,7 +1043,7 @@ u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
 {
 	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
 	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
-		     !amdgpu_ttm_is_bound(bo->tbo.ttm));
+		     !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem));
 	WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
 		     !bo->pin_count);
 	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 428aae048f4b..33615e2ea2e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -187,7 +187,7 @@ static inline u64 amdgpu_bo_mmap_offset(struct amdgpu_bo *bo)
 static inline bool amdgpu_bo_gpu_accessible(struct amdgpu_bo *bo)
 {
 	switch (bo->tbo.mem.mem_type) {
-	case TTM_PL_TT: return amdgpu_ttm_is_bound(bo->tbo.ttm);
+	case TTM_PL_TT: return amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem);
 	case TTM_PL_VRAM: return true;
 	default: return false;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index f8edf5483f11..01a996c6b802 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -32,7 +32,6 @@
 #include <linux/hwmon.h>
 #include <linux/hwmon-sysfs.h>
 
-#include "amd_powerplay.h"
 
 static int amdgpu_debugfs_pm_init(struct amdgpu_device *adev);
 
@@ -1279,16 +1278,16 @@ void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 			/* XXX select vce level based on ring/task */
 			adev->pm.dpm.vce_level = AMD_VCE_LEVEL_AC_ALL;
 			mutex_unlock(&adev->pm.mutex);
-			amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							AMD_CG_STATE_UNGATE);
-			amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							AMD_PG_STATE_UNGATE);
+			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_CG_STATE_UNGATE);
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_PG_STATE_UNGATE);
 			amdgpu_pm_compute_clocks(adev);
 		} else {
-			amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							AMD_PG_STATE_GATE);
-			amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							AMD_CG_STATE_GATE);
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_PG_STATE_GATE);
+			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_CG_STATE_GATE);
 			mutex_lock(&adev->pm.mutex);
 			adev->pm.dpm.vce_active = false;
 			mutex_unlock(&adev->pm.mutex);
@@ -1585,7 +1584,7 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
 	struct drm_device *ddev = adev->ddev;
 	u32 flags = 0;
 
-	amdgpu_get_clockgating_state(adev, &flags);
+	amdgpu_device_ip_get_clockgating_state(adev, &flags);
 	seq_printf(m, "Clock Gating Flags Mask: 0x%x\n", flags);
 	amdgpu_parse_cg_state(m, flags);
 	seq_printf(m, "\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 447d446b5015..2157d4509e84 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -264,7 +264,7 @@ static int psp_hw_start(struct psp_context *psp)
 	struct amdgpu_device *adev = psp->adev;
 	int ret;
 
-	if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
+	if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 		ret = psp_bootloader_load_sysdrv(psp);
 		if (ret)
 			return ret;
@@ -334,23 +334,26 @@ static int psp_load_fw(struct amdgpu_device *adev)
 	int ret;
 	struct psp_context *psp = &adev->psp;
 
+	if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset != 0)
+		goto skip_memalloc;
+
 	psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 	if (!psp->cmd)
 		return -ENOMEM;
 
 	ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
-				      AMDGPU_GEM_DOMAIN_GTT,
-				      &psp->fw_pri_bo,
-				      &psp->fw_pri_mc_addr,
-				      &psp->fw_pri_buf);
+					AMDGPU_GEM_DOMAIN_GTT,
+					&psp->fw_pri_bo,
+					&psp->fw_pri_mc_addr,
+					&psp->fw_pri_buf);
 	if (ret)
 		goto failed;
 
 	ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
-				      AMDGPU_GEM_DOMAIN_VRAM,
-				      &psp->fence_buf_bo,
-				      &psp->fence_buf_mc_addr,
-				      &psp->fence_buf);
+					AMDGPU_GEM_DOMAIN_VRAM,
+					&psp->fence_buf_bo,
+					&psp->fence_buf_mc_addr,
+					&psp->fence_buf);
 	if (ret)
 		goto failed_mem2;
 
@@ -375,6 +378,7 @@ static int psp_load_fw(struct amdgpu_device *adev)
 	if (ret)
 		goto failed_mem;
 
+skip_memalloc:
 	ret = psp_hw_start(psp);
 	if (ret)
 		goto failed_mem;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
index 93d86619e802..262c1267249e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_queue_mgr.c
@@ -225,7 +225,7 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 
 	/* Right now all IPs have only one instance - multiple rings. */
 	if (instance != 0) {
-		DRM_ERROR("invalid ip instance: %d\n", instance);
+		DRM_DEBUG("invalid ip instance: %d\n", instance);
 		return -EINVAL;
 	}
 
@@ -255,13 +255,13 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 		ip_num_rings = adev->vcn.num_enc_rings;
 		break;
 	default:
-		DRM_ERROR("unknown ip type: %d\n", hw_ip);
+		DRM_DEBUG("unknown ip type: %d\n", hw_ip);
 		return -EINVAL;
 	}
 
 	if (ring >= ip_num_rings) {
-		DRM_ERROR("Ring index:%d exceeds maximum:%d for ip:%d\n",
-				ring, ip_num_rings, hw_ip);
+		DRM_DEBUG("Ring index:%d exceeds maximum:%d for ip:%d\n",
+			  ring, ip_num_rings, hw_ip);
 		return -EINVAL;
 	}
 
@@ -292,7 +292,7 @@ int amdgpu_queue_mgr_map(struct amdgpu_device *adev,
 	default:
 		*out_ring = NULL;
 		r = -EINVAL;
-		DRM_ERROR("unknown HW IP type: %d\n", mapper->hw_ip);
+		DRM_DEBUG("unknown HW IP type: %d\n", mapper->hw_ip);
 	}
 
 out_unlock:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index a98fbbb4739f..561d3312af32 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -164,7 +164,7 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring)
  * Release a request for executing at @priority
  */
 void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
-			      enum amd_sched_priority priority)
+			      enum drm_sched_priority priority)
 {
 	int i;
 
@@ -175,7 +175,7 @@ void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
 		return;
 
 	/* no need to restore if the job is already at the lowest priority */
-	if (priority == AMD_SCHED_PRIORITY_NORMAL)
+	if (priority == DRM_SCHED_PRIORITY_NORMAL)
 		return;
 
 	mutex_lock(&ring->priority_mutex);
@@ -184,8 +184,8 @@ void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
 		goto out_unlock;
 
 	/* decay priority to the next level with a job available */
-	for (i = priority; i >= AMD_SCHED_PRIORITY_MIN; i--) {
-		if (i == AMD_SCHED_PRIORITY_NORMAL
+	for (i = priority; i >= DRM_SCHED_PRIORITY_MIN; i--) {
+		if (i == DRM_SCHED_PRIORITY_NORMAL
 				|| atomic_read(&ring->num_jobs[i])) {
 			ring->priority = i;
 			ring->funcs->set_priority(ring, i);
@@ -206,7 +206,7 @@ void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
  * Request a ring's priority to be raised to @priority (refcounted).
  */
 void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
-			      enum amd_sched_priority priority)
+			      enum drm_sched_priority priority)
 {
 	if (!ring->funcs->set_priority)
 		return;
@@ -263,25 +263,25 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 			return r;
 	}
 
-	r = amdgpu_wb_get(adev, &ring->rptr_offs);
+	r = amdgpu_device_wb_get(adev, &ring->rptr_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r);
 		return r;
 	}
 
-	r = amdgpu_wb_get(adev, &ring->wptr_offs);
+	r = amdgpu_device_wb_get(adev, &ring->wptr_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring wptr_offs wb alloc failed\n", r);
 		return r;
 	}
 
-	r = amdgpu_wb_get(adev, &ring->fence_offs);
+	r = amdgpu_device_wb_get(adev, &ring->fence_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring fence_offs wb alloc failed\n", r);
 		return r;
 	}
 
-	r = amdgpu_wb_get(adev, &ring->cond_exe_offs);
+	r = amdgpu_device_wb_get(adev, &ring->cond_exe_offs);
 	if (r) {
 		dev_err(adev->dev, "(%d) ring cond_exec_polling wb alloc failed\n", r);
 		return r;
@@ -317,12 +317,12 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 	}
 
 	ring->max_dw = max_dw;
-	ring->priority = AMD_SCHED_PRIORITY_NORMAL;
+	ring->priority = DRM_SCHED_PRIORITY_NORMAL;
 	mutex_init(&ring->priority_mutex);
 	INIT_LIST_HEAD(&ring->lru_list);
 	amdgpu_ring_lru_touch(adev, ring);
 
-	for (i = 0; i < AMD_SCHED_PRIORITY_MAX; ++i)
+	for (i = 0; i < DRM_SCHED_PRIORITY_MAX; ++i)
 		atomic_set(&ring->num_jobs[i], 0);
 
 	if (amdgpu_debugfs_ring_init(adev, ring)) {
@@ -348,11 +348,11 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
 	if (!(ring->adev) || !(ring->adev->rings[ring->idx]))
 		return;
 
-	amdgpu_wb_free(ring->adev, ring->rptr_offs);
-	amdgpu_wb_free(ring->adev, ring->wptr_offs);
+	amdgpu_device_wb_free(ring->adev, ring->rptr_offs);
+	amdgpu_device_wb_free(ring->adev, ring->wptr_offs);
 
-	amdgpu_wb_free(ring->adev, ring->cond_exe_offs);
-	amdgpu_wb_free(ring->adev, ring->fence_offs);
+	amdgpu_device_wb_free(ring->adev, ring->cond_exe_offs);
+	amdgpu_device_wb_free(ring->adev, ring->fence_offs);
 
 	amdgpu_bo_free_kernel(&ring->ring_obj,
 			      &ring->gpu_addr,
@@ -481,7 +481,7 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
 	result = 0;
 
 	if (*pos < 12) {
-		early[0] = amdgpu_ring_get_rptr(ring);
+		early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask;
 		early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask;
 		early[2] = ring->wptr & ring->buf_mask;
 		for (i = *pos / 4; i < 3 && size; i++) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index b18c2b96691f..102dad3edf6a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -25,7 +25,7 @@
 #define __AMDGPU_RING_H__
 
 #include <drm/amdgpu_drm.h>
-#include "gpu_scheduler.h"
+#include <drm/gpu_scheduler.h>
 
 /* max number of rings */
 #define AMDGPU_MAX_RINGS		18
@@ -79,8 +79,7 @@ struct amdgpu_fence_driver {
 
 int amdgpu_fence_driver_init(struct amdgpu_device *adev);
 void amdgpu_fence_driver_fini(struct amdgpu_device *adev);
-void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev);
-void amdgpu_fence_driver_force_completion_ring(struct amdgpu_ring *ring);
+void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring);
 
 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
 				  unsigned num_hw_submission);
@@ -122,11 +121,11 @@ struct amdgpu_ring_funcs {
 	/* command emit functions */
 	void (*emit_ib)(struct amdgpu_ring *ring,
 			struct amdgpu_ib *ib,
-			unsigned vm_id, bool ctx_switch);
+			unsigned vmid, bool ctx_switch);
 	void (*emit_fence)(struct amdgpu_ring *ring, uint64_t addr,
 			   uint64_t seq, unsigned flags);
 	void (*emit_pipeline_sync)(struct amdgpu_ring *ring);
-	void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vm_id,
+	void (*emit_vm_flush)(struct amdgpu_ring *ring, unsigned vmid,
 			      uint64_t pd_addr);
 	void (*emit_hdp_flush)(struct amdgpu_ring *ring);
 	void (*emit_hdp_invalidate)(struct amdgpu_ring *ring);
@@ -155,14 +154,14 @@ struct amdgpu_ring_funcs {
 	void (*emit_tmz)(struct amdgpu_ring *ring, bool start);
 	/* priority functions */
 	void (*set_priority) (struct amdgpu_ring *ring,
-			      enum amd_sched_priority priority);
+			      enum drm_sched_priority priority);
 };
 
 struct amdgpu_ring {
 	struct amdgpu_device		*adev;
 	const struct amdgpu_ring_funcs	*funcs;
 	struct amdgpu_fence_driver	fence_drv;
-	struct amd_gpu_scheduler	sched;
+	struct drm_gpu_scheduler	sched;
 	struct list_head		lru_list;
 
 	struct amdgpu_bo	*ring_obj;
@@ -187,6 +186,7 @@ struct amdgpu_ring {
 	uint64_t                eop_gpu_addr;
 	u32			doorbell_index;
 	bool			use_doorbell;
+	bool			use_pollmem;
 	unsigned		wptr_offs;
 	unsigned		fence_offs;
 	uint64_t		current_ctx;
@@ -197,7 +197,7 @@ struct amdgpu_ring {
 	unsigned		vm_inv_eng;
 	bool			has_compute_vm_bug;
 
-	atomic_t		num_jobs[AMD_SCHED_PRIORITY_MAX];
+	atomic_t		num_jobs[DRM_SCHED_PRIORITY_MAX];
 	struct mutex		priority_mutex;
 	/* protected by priority_mutex */
 	int			priority;
@@ -213,9 +213,9 @@ void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
 void amdgpu_ring_commit(struct amdgpu_ring *ring);
 void amdgpu_ring_undo(struct amdgpu_ring *ring);
 void amdgpu_ring_priority_get(struct amdgpu_ring *ring,
-			      enum amd_sched_priority priority);
+			      enum drm_sched_priority priority);
 void amdgpu_ring_priority_put(struct amdgpu_ring *ring,
-			      enum amd_sched_priority priority);
+			      enum drm_sched_priority priority);
 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 		     unsigned ring_size, struct amdgpu_irq_src *irq_src,
 		     unsigned irq_type);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
index 290cc3f9c433..86a0715d9431 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c
@@ -29,29 +29,29 @@
 
 #include "amdgpu_vm.h"
 
-enum amd_sched_priority amdgpu_to_sched_priority(int amdgpu_priority)
+enum drm_sched_priority amdgpu_to_sched_priority(int amdgpu_priority)
 {
 	switch (amdgpu_priority) {
 	case AMDGPU_CTX_PRIORITY_VERY_HIGH:
-		return AMD_SCHED_PRIORITY_HIGH_HW;
+		return DRM_SCHED_PRIORITY_HIGH_HW;
 	case AMDGPU_CTX_PRIORITY_HIGH:
-		return AMD_SCHED_PRIORITY_HIGH_SW;
+		return DRM_SCHED_PRIORITY_HIGH_SW;
 	case AMDGPU_CTX_PRIORITY_NORMAL:
-		return AMD_SCHED_PRIORITY_NORMAL;
+		return DRM_SCHED_PRIORITY_NORMAL;
 	case AMDGPU_CTX_PRIORITY_LOW:
 	case AMDGPU_CTX_PRIORITY_VERY_LOW:
-		return AMD_SCHED_PRIORITY_LOW;
+		return DRM_SCHED_PRIORITY_LOW;
 	case AMDGPU_CTX_PRIORITY_UNSET:
-		return AMD_SCHED_PRIORITY_UNSET;
+		return DRM_SCHED_PRIORITY_UNSET;
 	default:
 		WARN(1, "Invalid context priority %d\n", amdgpu_priority);
-		return AMD_SCHED_PRIORITY_INVALID;
+		return DRM_SCHED_PRIORITY_INVALID;
 	}
 }
 
 static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev,
 						  int fd,
-						  enum amd_sched_priority priority)
+						  enum drm_sched_priority priority)
 {
 	struct file *filp = fcheck(fd);
 	struct drm_file *file;
@@ -86,11 +86,11 @@ int amdgpu_sched_ioctl(struct drm_device *dev, void *data,
 {
 	union drm_amdgpu_sched *args = data;
 	struct amdgpu_device *adev = dev->dev_private;
-	enum amd_sched_priority priority;
+	enum drm_sched_priority priority;
 	int r;
 
 	priority = amdgpu_to_sched_priority(args->in.priority);
-	if (args->in.flags || priority == AMD_SCHED_PRIORITY_INVALID)
+	if (args->in.flags || priority == DRM_SCHED_PRIORITY_INVALID)
 		return -EINVAL;
 
 	switch (args->in.op) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
index b28c067d3822..2a1a0c734bdd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.h
@@ -27,7 +27,7 @@
 
 #include <drm/drmP.h>
 
-enum amd_sched_priority amdgpu_to_sched_priority(int amdgpu_priority);
+enum drm_sched_priority amdgpu_to_sched_priority(int amdgpu_priority);
 int amdgpu_sched_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *filp);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index a4bf21f8f1c1..df65c66dc956 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -35,6 +35,7 @@
 struct amdgpu_sync_entry {
 	struct hlist_node	node;
 	struct dma_fence	*fence;
+	bool	explicit;
 };
 
 static struct kmem_cache *amdgpu_sync_slab;
@@ -63,7 +64,7 @@ void amdgpu_sync_create(struct amdgpu_sync *sync)
 static bool amdgpu_sync_same_dev(struct amdgpu_device *adev,
 				 struct dma_fence *f)
 {
-	struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+	struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
 
 	if (s_fence) {
 		struct amdgpu_ring *ring;
@@ -84,7 +85,7 @@ static bool amdgpu_sync_same_dev(struct amdgpu_device *adev,
  */
 static void *amdgpu_sync_get_owner(struct dma_fence *f)
 {
-	struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+	struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
 
 	if (s_fence)
 		return s_fence->owner;
@@ -119,7 +120,7 @@ static void amdgpu_sync_keep_later(struct dma_fence **keep,
  * Tries to add the fence to an existing hash entry. Returns true when an entry
  * was found, false otherwise.
  */
-static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
+static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f, bool explicit)
 {
 	struct amdgpu_sync_entry *e;
 
@@ -128,6 +129,10 @@ static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
 			continue;
 
 		amdgpu_sync_keep_later(&e->fence, f);
+
+		/* Preserve eplicit flag to not loose pipe line sync */
+		e->explicit |= explicit;
+
 		return true;
 	}
 	return false;
@@ -141,24 +146,25 @@ static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
  *
  */
 int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
-		      struct dma_fence *f)
+		      struct dma_fence *f, bool explicit)
 {
 	struct amdgpu_sync_entry *e;
 
 	if (!f)
 		return 0;
-
 	if (amdgpu_sync_same_dev(adev, f) &&
 	    amdgpu_sync_get_owner(f) == AMDGPU_FENCE_OWNER_VM)
 		amdgpu_sync_keep_later(&sync->last_vm_update, f);
 
-	if (amdgpu_sync_add_later(sync, f))
+	if (amdgpu_sync_add_later(sync, f, explicit))
 		return 0;
 
 	e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
 	if (!e)
 		return -ENOMEM;
 
+	e->explicit = explicit;
+
 	hash_add(sync->fences, &e->node, f->context);
 	e->fence = dma_fence_get(f);
 	return 0;
@@ -189,10 +195,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
 
 	/* always sync to the exclusive fence */
 	f = reservation_object_get_excl(resv);
-	r = amdgpu_sync_fence(adev, sync, f);
-
-	if (explicit_sync)
-		return r;
+	r = amdgpu_sync_fence(adev, sync, f, false);
 
 	flist = reservation_object_get_list(resv);
 	if (!flist || r)
@@ -212,15 +215,15 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
 			     (fence_owner == AMDGPU_FENCE_OWNER_VM)))
 				continue;
 
-			/* Ignore fence from the same owner as
+			/* Ignore fence from the same owner and explicit one as
 			 * long as it isn't undefined.
 			 */
 			if (owner != AMDGPU_FENCE_OWNER_UNDEFINED &&
-			    fence_owner == owner)
+			    (fence_owner == owner || explicit_sync))
 				continue;
 		}
 
-		r = amdgpu_sync_fence(adev, sync, f);
+		r = amdgpu_sync_fence(adev, sync, f, false);
 		if (r)
 			break;
 	}
@@ -245,7 +248,7 @@ struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
 
 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
 		struct dma_fence *f = e->fence;
-		struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+		struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
 
 		if (dma_fence_is_signaled(f)) {
 			hash_del(&e->node);
@@ -275,19 +278,21 @@ struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
  * amdgpu_sync_get_fence - get the next fence from the sync object
  *
  * @sync: sync object to use
+ * @explicit: true if the next fence is explicit
  *
  * Get and removes the next fence from the sync object not signaled yet.
  */
-struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
+struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync, bool *explicit)
 {
 	struct amdgpu_sync_entry *e;
 	struct hlist_node *tmp;
 	struct dma_fence *f;
 	int i;
-
 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
 
 		f = e->fence;
+		if (explicit)
+			*explicit = e->explicit;
 
 		hash_del(&e->node);
 		kmem_cache_free(amdgpu_sync_slab, e);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
index 70d7e3a279a0..7aba38d5c9df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
@@ -41,7 +41,7 @@ struct amdgpu_sync {
 
 void amdgpu_sync_create(struct amdgpu_sync *sync);
 int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
-		      struct dma_fence *f);
+		      struct dma_fence *f, bool explicit);
 int amdgpu_sync_resv(struct amdgpu_device *adev,
 		     struct amdgpu_sync *sync,
 		     struct reservation_object *resv,
@@ -49,7 +49,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
 		     bool explicit_sync);
 struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
 				     struct amdgpu_ring *ring);
-struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
+struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync, bool *explicit);
 int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr);
 void amdgpu_sync_free(struct amdgpu_sync *sync);
 int amdgpu_sync_init(void);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 06525f2c36c3..cace7a93fc94 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -82,8 +82,8 @@ TRACE_EVENT(amdgpu_iv,
 			     __field(unsigned, client_id)
 			     __field(unsigned, src_id)
 			     __field(unsigned, ring_id)
-			     __field(unsigned, vm_id)
-			     __field(unsigned, vm_id_src)
+			     __field(unsigned, vmid)
+			     __field(unsigned, vmid_src)
 			     __field(uint64_t, timestamp)
 			     __field(unsigned, timestamp_src)
 			     __field(unsigned, pas_id)
@@ -93,8 +93,8 @@ TRACE_EVENT(amdgpu_iv,
 			   __entry->client_id = iv->client_id;
 			   __entry->src_id = iv->src_id;
 			   __entry->ring_id = iv->ring_id;
-			   __entry->vm_id = iv->vm_id;
-			   __entry->vm_id_src = iv->vm_id_src;
+			   __entry->vmid = iv->vmid;
+			   __entry->vmid_src = iv->vmid_src;
 			   __entry->timestamp = iv->timestamp;
 			   __entry->timestamp_src = iv->timestamp_src;
 			   __entry->pas_id = iv->pas_id;
@@ -103,9 +103,9 @@ TRACE_EVENT(amdgpu_iv,
 			   __entry->src_data[2] = iv->src_data[2];
 			   __entry->src_data[3] = iv->src_data[3];
 			   ),
-	    TP_printk("client_id:%u src_id:%u ring:%u vm_id:%u timestamp: %llu pas_id:%u src_data: %08x %08x %08x %08x\n",
+	    TP_printk("client_id:%u src_id:%u ring:%u vmid:%u timestamp: %llu pas_id:%u src_data: %08x %08x %08x %08x\n",
 		      __entry->client_id, __entry->src_id,
-		      __entry->ring_id, __entry->vm_id,
+		      __entry->ring_id, __entry->vmid,
 		      __entry->timestamp, __entry->pas_id,
 		      __entry->src_data[0], __entry->src_data[1],
 		      __entry->src_data[2], __entry->src_data[3])
@@ -219,7 +219,7 @@ TRACE_EVENT(amdgpu_vm_grab_id,
 	    TP_STRUCT__entry(
 			     __field(struct amdgpu_vm *, vm)
 			     __field(u32, ring)
-			     __field(u32, vm_id)
+			     __field(u32, vmid)
 			     __field(u32, vm_hub)
 			     __field(u64, pd_addr)
 			     __field(u32, needs_flush)
@@ -228,13 +228,13 @@ TRACE_EVENT(amdgpu_vm_grab_id,
 	    TP_fast_assign(
 			   __entry->vm = vm;
 			   __entry->ring = ring->idx;
-			   __entry->vm_id = job->vm_id;
+			   __entry->vmid = job->vmid;
 			   __entry->vm_hub = ring->funcs->vmhub,
 			   __entry->pd_addr = job->vm_pd_addr;
 			   __entry->needs_flush = job->vm_needs_flush;
 			   ),
 	    TP_printk("vm=%p, ring=%u, id=%u, hub=%u, pd_addr=%010Lx needs_flush=%u",
-		      __entry->vm, __entry->ring, __entry->vm_id,
+		      __entry->vm, __entry->ring, __entry->vmid,
 		      __entry->vm_hub, __entry->pd_addr, __entry->needs_flush)
 );
 
@@ -357,24 +357,24 @@ TRACE_EVENT(amdgpu_vm_copy_ptes,
 );
 
 TRACE_EVENT(amdgpu_vm_flush,
-	    TP_PROTO(struct amdgpu_ring *ring, unsigned vm_id,
+	    TP_PROTO(struct amdgpu_ring *ring, unsigned vmid,
 		     uint64_t pd_addr),
-	    TP_ARGS(ring, vm_id, pd_addr),
+	    TP_ARGS(ring, vmid, pd_addr),
 	    TP_STRUCT__entry(
 			     __field(u32, ring)
-			     __field(u32, vm_id)
+			     __field(u32, vmid)
 			     __field(u32, vm_hub)
 			     __field(u64, pd_addr)
 			     ),
 
 	    TP_fast_assign(
 			   __entry->ring = ring->idx;
-			   __entry->vm_id = vm_id;
+			   __entry->vmid = vmid;
 			   __entry->vm_hub = ring->funcs->vmhub;
 			   __entry->pd_addr = pd_addr;
 			   ),
 	    TP_printk("ring=%u, id=%u, hub=%u, pd_addr=%010Lx",
-		      __entry->ring, __entry->vm_id,
+		      __entry->ring, __entry->vmid,
 		      __entry->vm_hub,__entry->pd_addr)
 );
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index ad5bf86ee8a3..e4bb435e614b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -76,7 +76,7 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 {
 	struct drm_global_reference *global_ref;
 	struct amdgpu_ring *ring;
-	struct amd_sched_rq *rq;
+	struct drm_sched_rq *rq;
 	int r;
 
 	adev->mman.mem_global_referenced = false;
@@ -108,9 +108,9 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 	mutex_init(&adev->mman.gtt_window_lock);
 
 	ring = adev->mman.buffer_funcs_ring;
-	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
-	r = amd_sched_entity_init(&ring->sched, &adev->mman.entity,
-				  rq, amdgpu_sched_jobs);
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
+	r = drm_sched_entity_init(&ring->sched, &adev->mman.entity,
+				  rq, amdgpu_sched_jobs, NULL);
 	if (r) {
 		DRM_ERROR("Failed setting up TTM BO move run queue.\n");
 		goto error_entity;
@@ -131,7 +131,7 @@ static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
 static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
 {
 	if (adev->mman.mem_global_referenced) {
-		amd_sched_entity_fini(adev->mman.entity.sched,
+		drm_sched_entity_fini(adev->mman.entity.sched,
 				      &adev->mman.entity);
 		mutex_destroy(&adev->mman.gtt_window_lock);
 		drm_global_item_unref(&adev->mman.bo_global_ref.ref);
@@ -282,8 +282,7 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
 {
 	uint64_t addr = 0;
 
-	if (mem->mem_type != TTM_PL_TT ||
-	    amdgpu_gtt_mgr_is_allocated(mem)) {
+	if (mem->mem_type != TTM_PL_TT || amdgpu_gtt_mgr_has_gart_addr(mem)) {
 		addr = mm_node->start << PAGE_SHIFT;
 		addr += bo->bdev->man[mem->mem_type].gpu_offset;
 	}
@@ -369,7 +368,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 		 * dst to window 1
 		 */
 		if (src->mem->mem_type == TTM_PL_TT &&
-		    !amdgpu_gtt_mgr_is_allocated(src->mem)) {
+		    !amdgpu_gtt_mgr_has_gart_addr(src->mem)) {
 			r = amdgpu_map_buffer(src->bo, src->mem,
 					PFN_UP(cur_size + src_page_offset),
 					src_node_start, 0, ring,
@@ -383,7 +382,7 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
 		}
 
 		if (dst->mem->mem_type == TTM_PL_TT &&
-		    !amdgpu_gtt_mgr_is_allocated(dst->mem)) {
+		    !amdgpu_gtt_mgr_has_gart_addr(dst->mem)) {
 			r = amdgpu_map_buffer(dst->bo, dst->mem,
 					PFN_UP(cur_size + dst_page_offset),
 					dst_node_start, 1, ring,
@@ -467,9 +466,8 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 	return r;
 }
 
-static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
-				bool evict, bool interruptible,
-				bool no_wait_gpu,
+static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
+				struct ttm_operation_ctx *ctx,
 				struct ttm_mem_reg *new_mem)
 {
 	struct amdgpu_device *adev;
@@ -489,8 +487,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 	placements.fpfn = 0;
 	placements.lpfn = 0;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
-	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
-			     interruptible, no_wait_gpu);
+	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
 	if (unlikely(r)) {
 		return r;
 	}
@@ -500,23 +497,22 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
 		goto out_cleanup;
 	}
 
-	r = ttm_tt_bind(bo->ttm, &tmp_mem);
+	r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
-	r = amdgpu_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem);
+	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, &tmp_mem, old_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
-	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem);
+	r = ttm_bo_move_ttm(bo, ctx, new_mem);
 out_cleanup:
 	ttm_bo_mem_put(bo, &tmp_mem);
 	return r;
 }
 
-static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
-				bool evict, bool interruptible,
-				bool no_wait_gpu,
+static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
+				struct ttm_operation_ctx *ctx,
 				struct ttm_mem_reg *new_mem)
 {
 	struct amdgpu_device *adev;
@@ -536,16 +532,15 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	placements.fpfn = 0;
 	placements.lpfn = 0;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
-	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
-			     interruptible, no_wait_gpu);
+	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
 	if (unlikely(r)) {
 		return r;
 	}
-	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem);
+	r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
-	r = amdgpu_move_blit(bo, true, no_wait_gpu, new_mem, old_mem);
+	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, new_mem, old_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
@@ -554,10 +549,9 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
 	return r;
 }
 
-static int amdgpu_bo_move(struct ttm_buffer_object *bo,
-			bool evict, bool interruptible,
-			bool no_wait_gpu,
-			struct ttm_mem_reg *new_mem)
+static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
+			  struct ttm_operation_ctx *ctx,
+			  struct ttm_mem_reg *new_mem)
 {
 	struct amdgpu_device *adev;
 	struct amdgpu_bo *abo;
@@ -592,19 +586,18 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
 
 	if (old_mem->mem_type == TTM_PL_VRAM &&
 	    new_mem->mem_type == TTM_PL_SYSTEM) {
-		r = amdgpu_move_vram_ram(bo, evict, interruptible,
-					no_wait_gpu, new_mem);
+		r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
 	} else if (old_mem->mem_type == TTM_PL_SYSTEM &&
 		   new_mem->mem_type == TTM_PL_VRAM) {
-		r = amdgpu_move_ram_vram(bo, evict, interruptible,
-					    no_wait_gpu, new_mem);
+		r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
 	} else {
-		r = amdgpu_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem);
+		r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
+				     new_mem, old_mem);
 	}
 
 	if (r) {
 memcpy:
-		r = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, new_mem);
+		r = ttm_bo_move_memcpy(bo, ctx, new_mem);
 		if (r) {
 			return r;
 		}
@@ -690,7 +683,6 @@ struct amdgpu_ttm_tt {
 	struct list_head        guptasks;
 	atomic_t		mmu_invalidations;
 	uint32_t		last_set_pages;
-	struct list_head        list;
 };
 
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
@@ -861,44 +853,35 @@ static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
 	    bo_mem->mem_type == AMDGPU_PL_OA)
 		return -EINVAL;
 
-	if (!amdgpu_gtt_mgr_is_allocated(bo_mem))
+	if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
+		gtt->offset = AMDGPU_BO_INVALID_OFFSET;
 		return 0;
+	}
 
-	spin_lock(&gtt->adev->gtt_list_lock);
 	flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
 	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
 	r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
 		ttm->pages, gtt->ttm.dma_address, flags);
 
-	if (r) {
+	if (r)
 		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
 			  ttm->num_pages, gtt->offset);
-		goto error_gart_bind;
-	}
-
-	list_add_tail(&gtt->list, &gtt->adev->gtt_list);
-error_gart_bind:
-	spin_unlock(&gtt->adev->gtt_list_lock);
 	return r;
 }
 
-bool amdgpu_ttm_is_bound(struct ttm_tt *ttm)
-{
-	struct amdgpu_ttm_tt *gtt = (void *)ttm;
-
-	return gtt && !list_empty(&gtt->list);
-}
-
-int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
+int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
-	struct ttm_tt *ttm = bo->ttm;
+	struct ttm_operation_ctx ctx = { false, false };
+	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
 	struct ttm_mem_reg tmp;
 	struct ttm_placement placement;
 	struct ttm_place placements;
+	uint64_t flags;
 	int r;
 
-	if (!ttm || amdgpu_ttm_is_bound(ttm))
+	if (bo->mem.mem_type != TTM_PL_TT ||
+	    amdgpu_gtt_mgr_has_gart_addr(&bo->mem))
 		return 0;
 
 	tmp = bo->mem;
@@ -912,43 +895,44 @@ int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
 	placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
 		TTM_PL_FLAG_TT;
 
-	r = ttm_bo_mem_space(bo, &placement, &tmp, true, false);
+	r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
 	if (unlikely(r))
 		return r;
 
-	r = ttm_bo_move_ttm(bo, true, false, &tmp);
-	if (unlikely(r))
+	flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
+	gtt->offset = (u64)tmp.start << PAGE_SHIFT;
+	r = amdgpu_gart_bind(adev, gtt->offset, bo->ttm->num_pages,
+			     bo->ttm->pages, gtt->ttm.dma_address, flags);
+	if (unlikely(r)) {
 		ttm_bo_mem_put(bo, &tmp);
-	else
-		bo->offset = (bo->mem.start << PAGE_SHIFT) +
-			bo->bdev->man[bo->mem.mem_type].gpu_offset;
+		return r;
+	}
 
-	return r;
+	ttm_bo_mem_put(bo, &bo->mem);
+	bo->mem = tmp;
+	bo->offset = (bo->mem.start << PAGE_SHIFT) +
+		bo->bdev->man[bo->mem.mem_type].gpu_offset;
+
+	return 0;
 }
 
-int amdgpu_ttm_recover_gart(struct amdgpu_device *adev)
+int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
 {
-	struct amdgpu_ttm_tt *gtt, *tmp;
-	struct ttm_mem_reg bo_mem;
+	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
+	struct amdgpu_ttm_tt *gtt = (void *)tbo->ttm;
 	uint64_t flags;
 	int r;
 
-	bo_mem.mem_type = TTM_PL_TT;
-	spin_lock(&adev->gtt_list_lock);
-	list_for_each_entry_safe(gtt, tmp, &adev->gtt_list, list) {
-		flags = amdgpu_ttm_tt_pte_flags(gtt->adev, &gtt->ttm.ttm, &bo_mem);
-		r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
-				     gtt->ttm.ttm.pages, gtt->ttm.dma_address,
-				     flags);
-		if (r) {
-			spin_unlock(&adev->gtt_list_lock);
-			DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
-				  gtt->ttm.ttm.num_pages, gtt->offset);
-			return r;
-		}
-	}
-	spin_unlock(&adev->gtt_list_lock);
-	return 0;
+	if (!gtt)
+		return 0;
+
+	flags = amdgpu_ttm_tt_pte_flags(adev, &gtt->ttm.ttm, &tbo->mem);
+	r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
+			     gtt->ttm.ttm.pages, gtt->ttm.dma_address, flags);
+	if (r)
+		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
+			  gtt->ttm.ttm.num_pages, gtt->offset);
+	return r;
 }
 
 static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
@@ -959,20 +943,14 @@ static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
 	if (gtt->userptr)
 		amdgpu_ttm_tt_unpin_userptr(ttm);
 
-	if (!amdgpu_ttm_is_bound(ttm))
+	if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
 		return 0;
 
 	/* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
-	spin_lock(&gtt->adev->gtt_list_lock);
 	r = amdgpu_gart_unbind(gtt->adev, gtt->offset, ttm->num_pages);
-	if (r) {
+	if (r)
 		DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
 			  gtt->ttm.ttm.num_pages, gtt->offset);
-		goto error_unbind;
-	}
-	list_del_init(&gtt->list);
-error_unbind:
-	spin_unlock(&gtt->adev->gtt_list_lock);
 	return r;
 }
 
@@ -1009,11 +987,11 @@ static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
 		kfree(gtt);
 		return NULL;
 	}
-	INIT_LIST_HEAD(&gtt->list);
 	return &gtt->ttm.ttm;
 }
 
-static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
+static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
+			struct ttm_operation_ctx *ctx)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -1041,11 +1019,11 @@ static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
 
 #ifdef CONFIG_SWIOTLB
 	if (swiotlb_nr_tbl()) {
-		return ttm_dma_populate(&gtt->ttm, adev->dev);
+		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);
 	}
 #endif
 
-	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm);
+	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
 }
 
 static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -1292,6 +1270,101 @@ static struct ttm_bo_driver amdgpu_bo_driver = {
 	.access_memory = &amdgpu_ttm_access_memory
 };
 
+/*
+ * Firmware Reservation functions
+ */
+/**
+ * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * free fw reserved vram if it has been reserved.
+ */
+static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
+{
+	amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
+		NULL, &adev->fw_vram_usage.va);
+}
+
+/**
+ * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * create bo vram reservation from fw.
+ */
+static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
+{
+	struct ttm_operation_ctx ctx = { false, false };
+	int r = 0;
+	int i;
+	u64 vram_size = adev->mc.visible_vram_size;
+	u64 offset = adev->fw_vram_usage.start_offset;
+	u64 size = adev->fw_vram_usage.size;
+	struct amdgpu_bo *bo;
+
+	adev->fw_vram_usage.va = NULL;
+	adev->fw_vram_usage.reserved_bo = NULL;
+
+	if (adev->fw_vram_usage.size > 0 &&
+		adev->fw_vram_usage.size <= vram_size) {
+
+		r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
+			PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
+			AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
+			AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
+			&adev->fw_vram_usage.reserved_bo);
+		if (r)
+			goto error_create;
+
+		r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
+		if (r)
+			goto error_reserve;
+
+		/* remove the original mem node and create a new one at the
+		 * request position
+		 */
+		bo = adev->fw_vram_usage.reserved_bo;
+		offset = ALIGN(offset, PAGE_SIZE);
+		for (i = 0; i < bo->placement.num_placement; ++i) {
+			bo->placements[i].fpfn = offset >> PAGE_SHIFT;
+			bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
+		}
+
+		ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
+		r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
+				     &bo->tbo.mem, &ctx);
+		if (r)
+			goto error_pin;
+
+		r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
+			AMDGPU_GEM_DOMAIN_VRAM,
+			adev->fw_vram_usage.start_offset,
+			(adev->fw_vram_usage.start_offset +
+			adev->fw_vram_usage.size), NULL);
+		if (r)
+			goto error_pin;
+		r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
+			&adev->fw_vram_usage.va);
+		if (r)
+			goto error_kmap;
+
+		amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
+	}
+	return r;
+
+error_kmap:
+	amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
+error_pin:
+	amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
+error_reserve:
+	amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
+error_create:
+	adev->fw_vram_usage.va = NULL;
+	adev->fw_vram_usage.reserved_bo = NULL;
+	return r;
+}
+
 int amdgpu_ttm_init(struct amdgpu_device *adev)
 {
 	uint64_t gtt_size;
@@ -1334,7 +1407,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	 *The reserved vram for firmware must be pinned to the specified
 	 *place on the VRAM, so reserve it early.
 	 */
-	r = amdgpu_fw_reserve_vram_init(adev);
+	r = amdgpu_ttm_fw_reserve_vram_init(adev);
 	if (r) {
 		return r;
 	}
@@ -1348,9 +1421,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
 		 (unsigned) (adev->mc.real_vram_size / (1024 * 1024)));
 
-	if (amdgpu_gtt_size == -1)
-		gtt_size = max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
-			       adev->mc.mc_vram_size);
+	if (amdgpu_gtt_size == -1) {
+		struct sysinfo si;
+
+		si_meminfo(&si);
+		gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
+			       adev->mc.mc_vram_size),
+			       ((uint64_t)si.totalram * si.mem_unit * 3/4));
+	}
 	else
 		gtt_size = (uint64_t)amdgpu_gtt_size << 20;
 	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
@@ -1410,19 +1488,13 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 
 void amdgpu_ttm_fini(struct amdgpu_device *adev)
 {
-	int r;
-
 	if (!adev->mman.initialized)
 		return;
+
 	amdgpu_ttm_debugfs_fini(adev);
-	if (adev->stolen_vga_memory) {
-		r = amdgpu_bo_reserve(adev->stolen_vga_memory, true);
-		if (r == 0) {
-			amdgpu_bo_unpin(adev->stolen_vga_memory);
-			amdgpu_bo_unreserve(adev->stolen_vga_memory);
-		}
-		amdgpu_bo_unref(&adev->stolen_vga_memory);
-	}
+	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
+	amdgpu_ttm_fw_reserve_vram_fini(adev);
+
 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
 	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
 	if (adev->gds.mem.total_size)
@@ -1432,7 +1504,6 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
 	if (adev->gds.oa.total_size)
 		ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
 	ttm_bo_device_release(&adev->mman.bdev);
-	amdgpu_gart_fini(adev);
 	amdgpu_ttm_global_fini(adev);
 	adev->mman.initialized = false;
 	DRM_INFO("amdgpu: ttm finalized\n");
@@ -1628,7 +1699,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 	}
 
 	if (bo->tbo.mem.mem_type == TTM_PL_TT) {
-		r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
+		r = amdgpu_ttm_alloc_gart(&bo->tbo);
 		if (r)
 			return r;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index abd4084982a3..167856f6080f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -25,7 +25,7 @@
 #define __AMDGPU_TTM_H__
 
 #include "amdgpu.h"
-#include "gpu_scheduler.h"
+#include <drm/gpu_scheduler.h>
 
 #define AMDGPU_PL_GDS		(TTM_PL_PRIV + 0)
 #define AMDGPU_PL_GWS		(TTM_PL_PRIV + 1)
@@ -55,7 +55,7 @@ struct amdgpu_mman {
 
 	struct mutex				gtt_window_lock;
 	/* Scheduler entity for buffer moves */
-	struct amd_sched_entity			entity;
+	struct drm_sched_entity			entity;
 };
 
 struct amdgpu_copy_mem {
@@ -67,8 +67,9 @@ struct amdgpu_copy_mem {
 extern const struct ttm_mem_type_manager_func amdgpu_gtt_mgr_func;
 extern const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func;
 
-bool amdgpu_gtt_mgr_is_allocated(struct ttm_mem_reg *mem);
+bool amdgpu_gtt_mgr_has_gart_addr(struct ttm_mem_reg *mem);
 uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man);
+int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man);
 
 uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
@@ -90,9 +91,8 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo,
 			struct dma_fence **fence);
 
 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma);
-bool amdgpu_ttm_is_bound(struct ttm_tt *ttm);
-int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem);
-int amdgpu_ttm_recover_gart(struct amdgpu_device *adev);
+int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
+int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
 
 int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages);
 void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 65649026b836..474f88fbafce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -359,7 +359,6 @@ static int amdgpu_ucode_patch_jt(struct amdgpu_firmware_info *ucode,
 
 int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 {
-	struct amdgpu_bo **bo = &adev->firmware.fw_buf;
 	uint64_t fw_offset = 0;
 	int i, err;
 	struct amdgpu_firmware_info *ucode = NULL;
@@ -370,36 +369,16 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 		return 0;
 	}
 
-	if (!amdgpu_sriov_vf(adev) || !adev->in_sriov_reset) {
-		err = amdgpu_bo_create(adev, adev->firmware.fw_size, PAGE_SIZE, true,
+	if (!adev->in_gpu_reset) {
+		err = amdgpu_bo_create_kernel(adev, adev->firmware.fw_size, PAGE_SIZE,
 					amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
-					AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
-					NULL, NULL, 0, bo);
+					&adev->firmware.fw_buf,
+					&adev->firmware.fw_buf_mc,
+					&adev->firmware.fw_buf_ptr);
 		if (err) {
-			dev_err(adev->dev, "(%d) Firmware buffer allocate failed\n", err);
+			dev_err(adev->dev, "failed to create kernel buffer for firmware.fw_buf\n");
 			goto failed;
 		}
-
-		err = amdgpu_bo_reserve(*bo, false);
-		if (err) {
-			dev_err(adev->dev, "(%d) Firmware buffer reserve failed\n", err);
-			goto failed_reserve;
-		}
-
-		err = amdgpu_bo_pin(*bo, amdgpu_sriov_vf(adev) ? AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
-					&adev->firmware.fw_buf_mc);
-		if (err) {
-			dev_err(adev->dev, "(%d) Firmware buffer pin failed\n", err);
-			goto failed_pin;
-		}
-
-		err = amdgpu_bo_kmap(*bo, &adev->firmware.fw_buf_ptr);
-		if (err) {
-			dev_err(adev->dev, "(%d) Firmware buffer kmap failed\n", err);
-			goto failed_kmap;
-		}
-
-		amdgpu_bo_unreserve(*bo);
 	}
 
 	memset(adev->firmware.fw_buf_ptr, 0, adev->firmware.fw_size);
@@ -436,12 +415,6 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
 	}
 	return 0;
 
-failed_kmap:
-	amdgpu_bo_unpin(*bo);
-failed_pin:
-	amdgpu_bo_unreserve(*bo);
-failed_reserve:
-	amdgpu_bo_unref(bo);
 failed:
 	if (err)
 		adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
@@ -464,8 +437,10 @@ int amdgpu_ucode_fini_bo(struct amdgpu_device *adev)
 			ucode->kaddr = NULL;
 		}
 	}
-	amdgpu_bo_unref(&adev->firmware.fw_buf);
-	adev->firmware.fw_buf = NULL;
+
+	amdgpu_bo_free_kernel(&adev->firmware.fw_buf,
+				&adev->firmware.fw_buf_mc,
+				&adev->firmware.fw_buf_ptr);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index e8bd50cf9785..5c26a8e806b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -116,7 +116,7 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
 int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_ring *ring;
-	struct amd_sched_rq *rq;
+	struct drm_sched_rq *rq;
 	unsigned long bo_size;
 	const char *fw_name;
 	const struct common_firmware_header *hdr;
@@ -230,9 +230,9 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 	}
 
 	ring = &adev->uvd.ring;
-	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
-	r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity,
-				  rq, amdgpu_sched_jobs);
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+	r = drm_sched_entity_init(&ring->sched, &adev->uvd.entity,
+				  rq, amdgpu_sched_jobs, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up UVD run queue.\n");
 		return r;
@@ -244,7 +244,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
 	}
 
 	/* from uvd v5.0 HW addressing capacity increased to 64 bits */
-	if (!amdgpu_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0))
+	if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0))
 		adev->uvd.address_64_bit = true;
 
 	switch (adev->asic_type) {
@@ -272,7 +272,7 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev)
 	int i;
 	kfree(adev->uvd.saved_bo);
 
-	amd_sched_entity_fini(&adev->uvd.ring.sched, &adev->uvd.entity);
+	drm_sched_entity_fini(&adev->uvd.ring.sched, &adev->uvd.entity);
 
 	amdgpu_bo_free_kernel(&adev->uvd.vcpu_bo,
 			      &adev->uvd.gpu_addr,
@@ -297,14 +297,17 @@ int amdgpu_uvd_suspend(struct amdgpu_device *adev)
 	if (adev->uvd.vcpu_bo == NULL)
 		return 0;
 
-	for (i = 0; i < adev->uvd.max_handles; ++i)
-		if (atomic_read(&adev->uvd.handles[i]))
-			break;
+	cancel_delayed_work_sync(&adev->uvd.idle_work);
 
-	if (i == AMDGPU_MAX_UVD_HANDLES)
-		return 0;
+	/* only valid for physical mode */
+	if (adev->asic_type < CHIP_POLARIS10) {
+		for (i = 0; i < adev->uvd.max_handles; ++i)
+			if (atomic_read(&adev->uvd.handles[i]))
+				break;
 
-	cancel_delayed_work_sync(&adev->uvd.idle_work);
+		if (i == adev->uvd.max_handles)
+			return 0;
+	}
 
 	size = amdgpu_bo_size(adev->uvd.vcpu_bo);
 	ptr = adev->uvd.cpu_addr;
@@ -346,6 +349,8 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 			ptr += le32_to_cpu(hdr->ucode_size_bytes);
 		}
 		memset_io(ptr, 0, size);
+		/* to restore uvd fence seq */
+		amdgpu_fence_driver_force_completion(&adev->uvd.ring);
 	}
 
 	return 0;
@@ -408,6 +413,7 @@ static u64 amdgpu_uvd_get_addr_from_ctx(struct amdgpu_uvd_cs_ctx *ctx)
  */
 static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
 {
+	struct ttm_operation_ctx tctx = { false, false };
 	struct amdgpu_bo_va_mapping *mapping;
 	struct amdgpu_bo *bo;
 	uint32_t cmd;
@@ -430,7 +436,7 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx)
 		}
 		amdgpu_uvd_force_into_uvd_segment(bo);
 
-		r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+		r = ttm_bo_validate(&bo->tbo, &bo->placement, &tctx);
 	}
 
 	return r;
@@ -949,6 +955,7 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
 static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 			       bool direct, struct dma_fence **fence)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct ttm_validate_buffer tv;
 	struct ww_acquire_ctx ticket;
 	struct list_head head;
@@ -975,7 +982,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 		amdgpu_uvd_force_into_uvd_segment(bo);
 	}
 
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (r)
 		goto err;
 
@@ -1151,10 +1158,10 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work)
 		} else {
 			amdgpu_asic_set_uvd_clocks(adev, 0, 0);
 			/* shutdown the UVD block */
-			amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							    AMD_PG_STATE_GATE);
-			amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							    AMD_CG_STATE_GATE);
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+							       AMD_PG_STATE_GATE);
+			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+							       AMD_CG_STATE_GATE);
 		}
 	} else {
 		schedule_delayed_work(&adev->uvd.idle_work, UVD_IDLE_TIMEOUT);
@@ -1174,10 +1181,10 @@ void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring)
 			amdgpu_dpm_enable_uvd(adev, true);
 		} else {
 			amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
-			amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							    AMD_CG_STATE_UNGATE);
-			amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							    AMD_PG_STATE_UNGATE);
+			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+							       AMD_CG_STATE_UNGATE);
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+							       AMD_PG_STATE_UNGATE);
 		}
 	}
 }
@@ -1218,7 +1225,7 @@ int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	} else if (r < 0) {
 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 	} else {
-		DRM_INFO("ib test on ring %d succeeded\n",  ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n",  ring->idx);
 		r = 0;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index 3553b92bf69a..32ea20b99e53 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -31,6 +31,10 @@
 #define AMDGPU_UVD_SESSION_SIZE		(50*1024)
 #define AMDGPU_UVD_FIRMWARE_OFFSET	256
 
+#define AMDGPU_UVD_FIRMWARE_SIZE(adev)    \
+	(AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(((const struct common_firmware_header *)(adev)->uvd.fw->data)->ucode_size_bytes) + \
+			       8) - AMDGPU_UVD_FIRMWARE_OFFSET)
+
 struct amdgpu_uvd {
 	struct amdgpu_bo	*vcpu_bo;
 	void			*cpu_addr;
@@ -47,8 +51,8 @@ struct amdgpu_uvd {
 	struct amdgpu_irq_src	irq;
 	bool			address_64_bit;
 	bool			use_ctx_buf;
-	struct amd_sched_entity entity;
-	struct amd_sched_entity entity_enc;
+	struct drm_sched_entity entity;
+	struct drm_sched_entity entity_enc;
 	uint32_t                srbm_soft_reset;
 	unsigned		num_enc_rings;
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 2918de2f39ec..d274ae535530 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -85,7 +85,7 @@ static void amdgpu_vce_idle_work_handler(struct work_struct *work);
 int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 {
 	struct amdgpu_ring *ring;
-	struct amd_sched_rq *rq;
+	struct drm_sched_rq *rq;
 	const char *fw_name;
 	const struct common_firmware_header *hdr;
 	unsigned ucode_version, version_major, version_minor, binary_id;
@@ -174,9 +174,9 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
 	}
 
 	ring = &adev->vce.ring[0];
-	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
-	r = amd_sched_entity_init(&ring->sched, &adev->vce.entity,
-				  rq, amdgpu_sched_jobs);
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+	r = drm_sched_entity_init(&ring->sched, &adev->vce.entity,
+				  rq, amdgpu_sched_jobs, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up VCE run queue.\n");
 		return r;
@@ -207,7 +207,7 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
 	if (adev->vce.vcpu_bo == NULL)
 		return 0;
 
-	amd_sched_entity_fini(&adev->vce.ring[0].sched, &adev->vce.entity);
+	drm_sched_entity_fini(&adev->vce.ring[0].sched, &adev->vce.entity);
 
 	amdgpu_bo_free_kernel(&adev->vce.vcpu_bo, &adev->vce.gpu_addr,
 		(void **)&adev->vce.cpu_addr);
@@ -311,10 +311,10 @@ static void amdgpu_vce_idle_work_handler(struct work_struct *work)
 			amdgpu_dpm_enable_vce(adev, false);
 		} else {
 			amdgpu_asic_set_vce_clocks(adev, 0, 0);
-			amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							    AMD_PG_STATE_GATE);
-			amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							    AMD_CG_STATE_GATE);
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_PG_STATE_GATE);
+			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_CG_STATE_GATE);
 		}
 	} else {
 		schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT);
@@ -343,10 +343,10 @@ void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring)
 			amdgpu_dpm_enable_vce(adev, true);
 		} else {
 			amdgpu_asic_set_vce_clocks(adev, 53300, 40000);
-			amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							    AMD_CG_STATE_UNGATE);
-			amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
-							    AMD_PG_STATE_UNGATE);
+			amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_CG_STATE_UNGATE);
+			amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_VCE,
+							       AMD_PG_STATE_UNGATE);
 
 		}
 	}
@@ -543,6 +543,55 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 	return r;
 }
 
+/**
+ * amdgpu_vce_cs_validate_bo - make sure not to cross 4GB boundary
+ *
+ * @p: parser context
+ * @lo: address of lower dword
+ * @hi: address of higher dword
+ * @size: minimum size
+ * @index: bs/fb index
+ *
+ * Make sure that no BO cross a 4GB boundary.
+ */
+static int amdgpu_vce_validate_bo(struct amdgpu_cs_parser *p, uint32_t ib_idx,
+				  int lo, int hi, unsigned size, int32_t index)
+{
+	int64_t offset = ((uint64_t)size) * ((int64_t)index);
+	struct ttm_operation_ctx ctx = { false, false };
+	struct amdgpu_bo_va_mapping *mapping;
+	unsigned i, fpfn, lpfn;
+	struct amdgpu_bo *bo;
+	uint64_t addr;
+	int r;
+
+	addr = ((uint64_t)amdgpu_get_ib_value(p, ib_idx, lo)) |
+	       ((uint64_t)amdgpu_get_ib_value(p, ib_idx, hi)) << 32;
+	if (index >= 0) {
+		addr += offset;
+		fpfn = PAGE_ALIGN(offset) >> PAGE_SHIFT;
+		lpfn = 0x100000000ULL >> PAGE_SHIFT;
+	} else {
+		fpfn = 0;
+		lpfn = (0x100000000ULL - PAGE_ALIGN(offset)) >> PAGE_SHIFT;
+	}
+
+	r = amdgpu_cs_find_mapping(p, addr, &bo, &mapping);
+	if (r) {
+		DRM_ERROR("Can't find BO for addr 0x%010Lx %d %d %d %d\n",
+			  addr, lo, hi, size, index);
+		return r;
+	}
+
+	for (i = 0; i < bo->placement.num_placement; ++i) {
+		bo->placements[i].fpfn = max(bo->placements[i].fpfn, fpfn);
+		bo->placements[i].lpfn = bo->placements[i].lpfn ?
+			min(bo->placements[i].lpfn, lpfn) : lpfn;
+	}
+	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+}
+
+
 /**
  * amdgpu_vce_cs_reloc - command submission relocation
  *
@@ -648,12 +697,13 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 	uint32_t allocated = 0;
 	uint32_t tmp, handle = 0;
 	uint32_t *size = &tmp;
-	int i, r = 0, idx = 0;
+	unsigned idx;
+	int i, r = 0;
 
 	p->job->vm = NULL;
 	ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
 
-	while (idx < ib->length_dw) {
+	for (idx = 0; idx < ib->length_dw;) {
 		uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
 		uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
 
@@ -663,6 +713,54 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
 			goto out;
 		}
 
+		switch (cmd) {
+		case 0x00000002: /* task info */
+			fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
+			bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
+			break;
+
+		case 0x03000001: /* encode */
+			r = amdgpu_vce_validate_bo(p, ib_idx, idx + 10,
+						   idx + 9, 0, 0);
+			if (r)
+				goto out;
+
+			r = amdgpu_vce_validate_bo(p, ib_idx, idx + 12,
+						   idx + 11, 0, 0);
+			if (r)
+				goto out;
+			break;
+
+		case 0x05000001: /* context buffer */
+			r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3,
+						   idx + 2, 0, 0);
+			if (r)
+				goto out;
+			break;
+
+		case 0x05000004: /* video bitstream buffer */
+			tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
+			r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
+						   tmp, bs_idx);
+			if (r)
+				goto out;
+			break;
+
+		case 0x05000005: /* feedback buffer */
+			r = amdgpu_vce_validate_bo(p, ib_idx, idx + 3, idx + 2,
+						   4096, fb_idx);
+			if (r)
+				goto out;
+			break;
+		}
+
+		idx += len / 4;
+	}
+
+	for (idx = 0; idx < ib->length_dw;) {
+		uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
+		uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
+
 		switch (cmd) {
 		case 0x00000001: /* session */
 			handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
@@ -893,7 +991,7 @@ int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx)
  *
  */
 void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib,
-			     unsigned vm_id, bool ctx_switch)
+			     unsigned vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, VCE_CMD_IB);
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
@@ -954,7 +1052,7 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -999,7 +1097,7 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	} else if (r < 0) {
 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 	} else {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	}
 error:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
index 5ce54cde472d..0fd378ae92c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
@@ -46,7 +46,7 @@ struct amdgpu_vce {
 	struct amdgpu_ring	ring[AMDGPU_MAX_VCE_RINGS];
 	struct amdgpu_irq_src	irq;
 	unsigned		harvest_config;
-	struct amd_sched_entity	entity;
+	struct drm_sched_entity	entity;
 	uint32_t                srbm_soft_reset;
 	unsigned		num_rings;
 };
@@ -63,7 +63,7 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp);
 int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx);
 int amdgpu_vce_ring_parse_cs_vm(struct amdgpu_cs_parser *p, uint32_t ib_idx);
 void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib,
-			     unsigned vm_id, bool ctx_switch);
+			     unsigned vmid, bool ctx_switch);
 void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 				unsigned flags);
 int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 041e0121590c..837962118dbc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -35,8 +35,7 @@
 #include "soc15d.h"
 #include "soc15_common.h"
 
-#include "vega10/soc15ip.h"
-#include "raven1/VCN/vcn_1_0_offset.h"
+#include "vcn/vcn_1_0_offset.h"
 
 /* 1 second timeout */
 #define VCN_IDLE_TIMEOUT	msecs_to_jiffies(1000)
@@ -51,7 +50,7 @@ static void amdgpu_vcn_idle_work_handler(struct work_struct *work);
 int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_ring *ring;
-	struct amd_sched_rq *rq;
+	struct drm_sched_rq *rq;
 	unsigned long bo_size;
 	const char *fw_name;
 	const struct common_firmware_header *hdr;
@@ -104,18 +103,18 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
 	}
 
 	ring = &adev->vcn.ring_dec;
-	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
-	r = amd_sched_entity_init(&ring->sched, &adev->vcn.entity_dec,
-				  rq, amdgpu_sched_jobs);
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+	r = drm_sched_entity_init(&ring->sched, &adev->vcn.entity_dec,
+				  rq, amdgpu_sched_jobs, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up VCN dec run queue.\n");
 		return r;
 	}
 
 	ring = &adev->vcn.ring_enc[0];
-	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
-	r = amd_sched_entity_init(&ring->sched, &adev->vcn.entity_enc,
-				  rq, amdgpu_sched_jobs);
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+	r = drm_sched_entity_init(&ring->sched, &adev->vcn.entity_enc,
+				  rq, amdgpu_sched_jobs, NULL);
 	if (r != 0) {
 		DRM_ERROR("Failed setting up VCN enc run queue.\n");
 		return r;
@@ -130,9 +129,9 @@ int amdgpu_vcn_sw_fini(struct amdgpu_device *adev)
 
 	kfree(adev->vcn.saved_bo);
 
-	amd_sched_entity_fini(&adev->vcn.ring_dec.sched, &adev->vcn.entity_dec);
+	drm_sched_entity_fini(&adev->vcn.ring_dec.sched, &adev->vcn.entity_dec);
 
-	amd_sched_entity_fini(&adev->vcn.ring_enc[0].sched, &adev->vcn.entity_enc);
+	drm_sched_entity_fini(&adev->vcn.ring_enc[0].sched, &adev->vcn.entity_enc);
 
 	amdgpu_bo_free_kernel(&adev->vcn.vcpu_bo,
 			      &adev->vcn.gpu_addr,
@@ -261,7 +260,7 @@ int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
@@ -274,6 +273,7 @@ int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring)
 static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 			       bool direct, struct dma_fence **fence)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct ttm_validate_buffer tv;
 	struct ww_acquire_ctx ticket;
 	struct list_head head;
@@ -294,7 +294,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *b
 	if (r)
 		return r;
 
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (r)
 		goto err;
 
@@ -467,7 +467,7 @@ int amdgpu_vcn_dec_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	} else if (r < 0) {
 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 	} else {
-		DRM_INFO("ib test on ring %d succeeded\n",  ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n",  ring->idx);
 		r = 0;
 	}
 
@@ -500,7 +500,7 @@ int amdgpu_vcn_enc_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -643,7 +643,7 @@ int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	} else if (r < 0) {
 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 	} else {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	}
 error:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
index d50ba0657854..2fd7db891689 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
@@ -56,8 +56,8 @@ struct amdgpu_vcn {
 	struct amdgpu_ring	ring_dec;
 	struct amdgpu_ring	ring_enc[AMDGPU_VCN_MAX_ENC_RINGS];
 	struct amdgpu_irq_src	irq;
-	struct amd_sched_entity entity_dec;
-	struct amd_sched_entity entity_enc;
+	struct drm_sched_entity entity_dec;
+	struct drm_sched_entity entity_enc;
 	unsigned		num_enc_rings;
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 6738df836a70..e7dfb7b44b4b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -24,6 +24,14 @@
 #include "amdgpu.h"
 #define MAX_KIQ_REG_WAIT	100000000 /* in usecs */
 
+bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev)
+{
+	/* By now all MMIO pages except mailbox are blocked */
+	/* if blocking is enabled in hypervisor. Choose the */
+	/* SCRATCH_REG0 to test. */
+	return RREG32_NO_KIQ(0xc040) == 0xffffffff;
+}
+
 int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
 {
 	int r;
@@ -39,6 +47,12 @@ int amdgpu_allocate_static_csa(struct amdgpu_device *adev)
 	return 0;
 }
 
+void amdgpu_free_static_csa(struct amdgpu_device *adev) {
+	amdgpu_bo_free_kernel(&adev->virt.csa_obj,
+						&adev->virt.csa_vmid0_addr,
+						NULL);
+}
+
 /*
  * amdgpu_map_static_csa should be called during amdgpu_vm_init
  * it maps virtual address "AMDGPU_VA_RESERVED_SIZE - AMDGPU_CSA_SIZE"
@@ -107,8 +121,6 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
 	adev->enable_virtual_display = true;
 	adev->cg_flags = 0;
 	adev->pg_flags = 0;
-
-	mutex_init(&adev->virt.lock_reset);
 }
 
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg)
@@ -227,6 +239,22 @@ int amdgpu_virt_reset_gpu(struct amdgpu_device *adev)
 	return 0;
 }
 
+/**
+ * amdgpu_virt_wait_reset() - wait for reset gpu completed
+ * @amdgpu:	amdgpu device.
+ * Wait for GPU reset completed.
+ * Return: Zero if reset success, otherwise will return error.
+ */
+int amdgpu_virt_wait_reset(struct amdgpu_device *adev)
+{
+	struct amdgpu_virt *virt = &adev->virt;
+
+	if (!virt->ops || !virt->ops->wait_reset)
+		return -EINVAL;
+
+	return virt->ops->wait_reset(adev);
+}
+
 /**
  * amdgpu_virt_alloc_mm_table() - alloc memory for mm table
  * @amdgpu:	amdgpu device.
@@ -296,7 +324,6 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj,
 
 void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
 {
-	uint32_t pf2vf_ver = 0;
 	uint32_t pf2vf_size = 0;
 	uint32_t checksum = 0;
 	uint32_t checkval;
@@ -309,9 +336,9 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
 		adev->virt.fw_reserve.p_pf2vf =
 			(struct amdgim_pf2vf_info_header *)(
 			adev->fw_vram_usage.va + AMDGIM_DATAEXCHANGE_OFFSET);
-		pf2vf_ver = adev->virt.fw_reserve.p_pf2vf->version;
 		AMDGPU_FW_VRAM_PF2VF_READ(adev, header.size, &pf2vf_size);
 		AMDGPU_FW_VRAM_PF2VF_READ(adev, checksum, &checksum);
+		AMDGPU_FW_VRAM_PF2VF_READ(adev, feature_flags, &adev->virt.gim_feature);
 
 		/* pf2vf message must be in 4K */
 		if (pf2vf_size > 0 && pf2vf_size < 4096) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index b89d37fc406f..6a83425aa9ed 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -55,6 +55,7 @@ struct amdgpu_virt_ops {
 	int (*req_full_gpu)(struct amdgpu_device *adev, bool init);
 	int (*rel_full_gpu)(struct amdgpu_device *adev, bool init);
 	int (*reset_gpu)(struct amdgpu_device *adev);
+	int (*wait_reset)(struct amdgpu_device *adev);
 	void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3);
 };
 
@@ -80,6 +81,8 @@ enum AMDGIM_FEATURE_FLAG {
 	AMDGIM_FEATURE_ERROR_LOG_COLLECT = 0x1,
 	/* GIM supports feature of loading uCodes */
 	AMDGIM_FEATURE_GIM_LOAD_UCODES   = 0x2,
+	/* VRAM LOST by GIM */
+	AMDGIM_FEATURE_GIM_FLR_VRAMLOST = 0x4,
 };
 
 struct amdgim_pf2vf_info_header {
@@ -238,7 +241,6 @@ struct amdgpu_virt {
 	uint64_t			csa_vmid0_addr;
 	bool chained_ib_support;
 	uint32_t			reg_val_offs;
-	struct mutex                    lock_reset;
 	struct amdgpu_irq_src		ack_irq;
 	struct amdgpu_irq_src		rcv_irq;
 	struct work_struct		flr_work;
@@ -246,6 +248,7 @@ struct amdgpu_virt {
 	const struct amdgpu_virt_ops	*ops;
 	struct amdgpu_vf_error_buffer   vf_errors;
 	struct amdgpu_virt_fw_reserve	fw_reserve;
+	uint32_t gim_feature;
 };
 
 #define AMDGPU_CSA_SIZE    (8 * 1024)
@@ -276,16 +279,18 @@ static inline bool is_virtual_machine(void)
 }
 
 struct amdgpu_vm;
+bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev);
 int amdgpu_allocate_static_csa(struct amdgpu_device *adev);
 int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			  struct amdgpu_bo_va **bo_va);
+void amdgpu_free_static_csa(struct amdgpu_device *adev);
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_virt_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
 int amdgpu_virt_request_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_release_full_gpu(struct amdgpu_device *adev, bool init);
 int amdgpu_virt_reset_gpu(struct amdgpu_device *adev);
-int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job);
+int amdgpu_virt_wait_reset(struct amdgpu_device *adev);
 int amdgpu_virt_alloc_mm_table(struct amdgpu_device *adev);
 void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);
 int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index c8c26f21993c..5afbc5e714d0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -33,52 +33,6 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
-/*
- * PASID manager
- *
- * PASIDs are global address space identifiers that can be shared
- * between the GPU, an IOMMU and the driver. VMs on different devices
- * may use the same PASID if they share the same address
- * space. Therefore PASIDs are allocated using a global IDA. VMs are
- * looked up from the PASID per amdgpu_device.
- */
-static DEFINE_IDA(amdgpu_vm_pasid_ida);
-
-/**
- * amdgpu_vm_alloc_pasid - Allocate a PASID
- * @bits: Maximum width of the PASID in bits, must be at least 1
- *
- * Allocates a PASID of the given width while keeping smaller PASIDs
- * available if possible.
- *
- * Returns a positive integer on success. Returns %-EINVAL if bits==0.
- * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
- * memory allocation failure.
- */
-int amdgpu_vm_alloc_pasid(unsigned int bits)
-{
-	int pasid = -EINVAL;
-
-	for (bits = min(bits, 31U); bits > 0; bits--) {
-		pasid = ida_simple_get(&amdgpu_vm_pasid_ida,
-				       1U << (bits - 1), 1U << bits,
-				       GFP_KERNEL);
-		if (pasid != -ENOSPC)
-			break;
-	}
-
-	return pasid;
-}
-
-/**
- * amdgpu_vm_free_pasid - Free a PASID
- * @pasid: PASID to free
- */
-void amdgpu_vm_free_pasid(unsigned int pasid)
-{
-	ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);
-}
-
 /*
  * GPUVM
  * GPUVM is similar to the legacy gart on older asics, however
@@ -138,6 +92,35 @@ struct amdgpu_prt_cb {
 	struct dma_fence_cb cb;
 };
 
+/**
+ * amdgpu_vm_level_shift - return the addr shift for each level
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns the number of bits the pfn needs to be right shifted for a level.
+ */
+static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
+				      unsigned level)
+{
+	unsigned shift = 0xff;
+
+	switch (level) {
+	case AMDGPU_VM_PDB2:
+	case AMDGPU_VM_PDB1:
+	case AMDGPU_VM_PDB0:
+		shift = 9 * (AMDGPU_VM_PDB0 - level) +
+			adev->vm_manager.block_size;
+		break;
+	case AMDGPU_VM_PTB:
+		shift = 0;
+		break;
+	default:
+		dev_err(adev->dev, "the level%d isn't supported.\n", level);
+	}
+
+	return shift;
+}
+
 /**
  * amdgpu_vm_num_entries - return the number of entries in a PD/PT
  *
@@ -148,17 +131,18 @@ struct amdgpu_prt_cb {
 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
 				      unsigned level)
 {
-	if (level == 0)
+	unsigned shift = amdgpu_vm_level_shift(adev,
+					       adev->vm_manager.root_level);
+
+	if (level == adev->vm_manager.root_level)
 		/* For the root directory */
-		return adev->vm_manager.max_pfn >>
-			(adev->vm_manager.block_size *
-			 adev->vm_manager.num_level);
-	else if (level == adev->vm_manager.num_level)
+		return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
+	else if (level != AMDGPU_VM_PTB)
+		/* Everything in between */
+		return 512;
+	else
 		/* For the page tables on the leaves */
 		return AMDGPU_VM_PTE_COUNT(adev);
-	else
-		/* Everything in between */
-		return 1 << adev->vm_manager.block_size;
 }
 
 /**
@@ -288,8 +272,7 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 				  uint64_t saddr, uint64_t eaddr,
 				  unsigned level)
 {
-	unsigned shift = (adev->vm_manager.num_level - level) *
-		adev->vm_manager.block_size;
+	unsigned shift = amdgpu_vm_level_shift(adev, level);
 	unsigned pt_idx, from, to;
 	int r;
 	u64 flags;
@@ -312,9 +295,6 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 	    to >= amdgpu_vm_num_entries(adev, level))
 		return -EINVAL;
 
-	if (to > parent->last_entry_used)
-		parent->last_entry_used = to;
-
 	++level;
 	saddr = saddr & ((1 << shift) - 1);
 	eaddr = eaddr & ((1 << shift) - 1);
@@ -329,7 +309,7 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 
 	if (vm->pte_support_ats) {
 		init_value = AMDGPU_PTE_DEFAULT_ATC;
-		if (level != adev->vm_manager.num_level - 1)
+		if (level != AMDGPU_VM_PTB)
 			init_value |= AMDGPU_PDE_PTE;
 
 	}
@@ -369,10 +349,9 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
 			spin_lock(&vm->status_lock);
 			list_add(&entry->base.vm_status, &vm->relocated);
 			spin_unlock(&vm->status_lock);
-			entry->addr = 0;
 		}
 
-		if (level < adev->vm_manager.num_level) {
+		if (level < AMDGPU_VM_PTB) {
 			uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
 			uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
 				((1 << shift) - 1);
@@ -418,287 +397,8 @@ int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 	saddr /= AMDGPU_GPU_PAGE_SIZE;
 	eaddr /= AMDGPU_GPU_PAGE_SIZE;
 
-	return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr, 0);
-}
-
-/**
- * amdgpu_vm_had_gpu_reset - check if reset occured since last use
- *
- * @adev: amdgpu_device pointer
- * @id: VMID structure
- *
- * Check if GPU reset occured since last use of the VMID.
- */
-static bool amdgpu_vm_had_gpu_reset(struct amdgpu_device *adev,
-				    struct amdgpu_vm_id *id)
-{
-	return id->current_gpu_reset_count !=
-		atomic_read(&adev->gpu_reset_counter);
-}
-
-static bool amdgpu_vm_reserved_vmid_ready(struct amdgpu_vm *vm, unsigned vmhub)
-{
-	return !!vm->reserved_vmid[vmhub];
-}
-
-/* idr_mgr->lock must be held */
-static int amdgpu_vm_grab_reserved_vmid_locked(struct amdgpu_vm *vm,
-					       struct amdgpu_ring *ring,
-					       struct amdgpu_sync *sync,
-					       struct dma_fence *fence,
-					       struct amdgpu_job *job)
-{
-	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
-	uint64_t fence_context = adev->fence_context + ring->idx;
-	struct amdgpu_vm_id *id = vm->reserved_vmid[vmhub];
-	struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
-	struct dma_fence *updates = sync->last_vm_update;
-	int r = 0;
-	struct dma_fence *flushed, *tmp;
-	bool needs_flush = vm->use_cpu_for_update;
-
-	flushed  = id->flushed_updates;
-	if ((amdgpu_vm_had_gpu_reset(adev, id)) ||
-	    (atomic64_read(&id->owner) != vm->client_id) ||
-	    (job->vm_pd_addr != id->pd_gpu_addr) ||
-	    (updates && (!flushed || updates->context != flushed->context ||
-			dma_fence_is_later(updates, flushed))) ||
-	    (!id->last_flush || (id->last_flush->context != fence_context &&
-				 !dma_fence_is_signaled(id->last_flush)))) {
-		needs_flush = true;
-		/* to prevent one context starved by another context */
-		id->pd_gpu_addr = 0;
-		tmp = amdgpu_sync_peek_fence(&id->active, ring);
-		if (tmp) {
-			r = amdgpu_sync_fence(adev, sync, tmp);
-			return r;
-		}
-	}
-
-	/* Good we can use this VMID. Remember this submission as
-	* user of the VMID.
-	*/
-	r = amdgpu_sync_fence(ring->adev, &id->active, fence);
-	if (r)
-		goto out;
-
-	if (updates && (!flushed || updates->context != flushed->context ||
-			dma_fence_is_later(updates, flushed))) {
-		dma_fence_put(id->flushed_updates);
-		id->flushed_updates = dma_fence_get(updates);
-	}
-	id->pd_gpu_addr = job->vm_pd_addr;
-	atomic64_set(&id->owner, vm->client_id);
-	job->vm_needs_flush = needs_flush;
-	if (needs_flush) {
-		dma_fence_put(id->last_flush);
-		id->last_flush = NULL;
-	}
-	job->vm_id = id - id_mgr->ids;
-	trace_amdgpu_vm_grab_id(vm, ring, job);
-out:
-	return r;
-}
-
-/**
- * amdgpu_vm_grab_id - allocate the next free VMID
- *
- * @vm: vm to allocate id for
- * @ring: ring we want to submit job to
- * @sync: sync object where we add dependencies
- * @fence: fence protecting ID from reuse
- *
- * Allocate an id for the vm, adding fences to the sync obj as necessary.
- */
-int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
-		      struct amdgpu_sync *sync, struct dma_fence *fence,
-		      struct amdgpu_job *job)
-{
-	struct amdgpu_device *adev = ring->adev;
-	unsigned vmhub = ring->funcs->vmhub;
-	struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
-	uint64_t fence_context = adev->fence_context + ring->idx;
-	struct dma_fence *updates = sync->last_vm_update;
-	struct amdgpu_vm_id *id, *idle;
-	struct dma_fence **fences;
-	unsigned i;
-	int r = 0;
-
-	mutex_lock(&id_mgr->lock);
-	if (amdgpu_vm_reserved_vmid_ready(vm, vmhub)) {
-		r = amdgpu_vm_grab_reserved_vmid_locked(vm, ring, sync, fence, job);
-		mutex_unlock(&id_mgr->lock);
-		return r;
-	}
-	fences = kmalloc_array(sizeof(void *), id_mgr->num_ids, GFP_KERNEL);
-	if (!fences) {
-		mutex_unlock(&id_mgr->lock);
-		return -ENOMEM;
-	}
-	/* Check if we have an idle VMID */
-	i = 0;
-	list_for_each_entry(idle, &id_mgr->ids_lru, list) {
-		fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
-		if (!fences[i])
-			break;
-		++i;
-	}
-
-	/* If we can't find a idle VMID to use, wait till one becomes available */
-	if (&idle->list == &id_mgr->ids_lru) {
-		u64 fence_context = adev->vm_manager.fence_context + ring->idx;
-		unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
-		struct dma_fence_array *array;
-		unsigned j;
-
-		for (j = 0; j < i; ++j)
-			dma_fence_get(fences[j]);
-
-		array = dma_fence_array_create(i, fences, fence_context,
-					   seqno, true);
-		if (!array) {
-			for (j = 0; j < i; ++j)
-				dma_fence_put(fences[j]);
-			kfree(fences);
-			r = -ENOMEM;
-			goto error;
-		}
-
-
-		r = amdgpu_sync_fence(ring->adev, sync, &array->base);
-		dma_fence_put(&array->base);
-		if (r)
-			goto error;
-
-		mutex_unlock(&id_mgr->lock);
-		return 0;
-
-	}
-	kfree(fences);
-
-	job->vm_needs_flush = vm->use_cpu_for_update;
-	/* Check if we can use a VMID already assigned to this VM */
-	list_for_each_entry_reverse(id, &id_mgr->ids_lru, list) {
-		struct dma_fence *flushed;
-		bool needs_flush = vm->use_cpu_for_update;
-
-		/* Check all the prerequisites to using this VMID */
-		if (amdgpu_vm_had_gpu_reset(adev, id))
-			continue;
-
-		if (atomic64_read(&id->owner) != vm->client_id)
-			continue;
-
-		if (job->vm_pd_addr != id->pd_gpu_addr)
-			continue;
-
-		if (!id->last_flush ||
-		    (id->last_flush->context != fence_context &&
-		     !dma_fence_is_signaled(id->last_flush)))
-			needs_flush = true;
-
-		flushed  = id->flushed_updates;
-		if (updates && (!flushed || dma_fence_is_later(updates, flushed)))
-			needs_flush = true;
-
-		/* Concurrent flushes are only possible starting with Vega10 */
-		if (adev->asic_type < CHIP_VEGA10 && needs_flush)
-			continue;
-
-		/* Good we can use this VMID. Remember this submission as
-		 * user of the VMID.
-		 */
-		r = amdgpu_sync_fence(ring->adev, &id->active, fence);
-		if (r)
-			goto error;
-
-		if (updates && (!flushed || dma_fence_is_later(updates, flushed))) {
-			dma_fence_put(id->flushed_updates);
-			id->flushed_updates = dma_fence_get(updates);
-		}
-
-		if (needs_flush)
-			goto needs_flush;
-		else
-			goto no_flush_needed;
-
-	};
-
-	/* Still no ID to use? Then use the idle one found earlier */
-	id = idle;
-
-	/* Remember this submission as user of the VMID */
-	r = amdgpu_sync_fence(ring->adev, &id->active, fence);
-	if (r)
-		goto error;
-
-	id->pd_gpu_addr = job->vm_pd_addr;
-	dma_fence_put(id->flushed_updates);
-	id->flushed_updates = dma_fence_get(updates);
-	atomic64_set(&id->owner, vm->client_id);
-
-needs_flush:
-	job->vm_needs_flush = true;
-	dma_fence_put(id->last_flush);
-	id->last_flush = NULL;
-
-no_flush_needed:
-	list_move_tail(&id->list, &id_mgr->ids_lru);
-
-	job->vm_id = id - id_mgr->ids;
-	trace_amdgpu_vm_grab_id(vm, ring, job);
-
-error:
-	mutex_unlock(&id_mgr->lock);
-	return r;
-}
-
-static void amdgpu_vm_free_reserved_vmid(struct amdgpu_device *adev,
-					  struct amdgpu_vm *vm,
-					  unsigned vmhub)
-{
-	struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
-
-	mutex_lock(&id_mgr->lock);
-	if (vm->reserved_vmid[vmhub]) {
-		list_add(&vm->reserved_vmid[vmhub]->list,
-			&id_mgr->ids_lru);
-		vm->reserved_vmid[vmhub] = NULL;
-		atomic_dec(&id_mgr->reserved_vmid_num);
-	}
-	mutex_unlock(&id_mgr->lock);
-}
-
-static int amdgpu_vm_alloc_reserved_vmid(struct amdgpu_device *adev,
-					 struct amdgpu_vm *vm,
-					 unsigned vmhub)
-{
-	struct amdgpu_vm_id_manager *id_mgr;
-	struct amdgpu_vm_id *idle;
-	int r = 0;
-
-	id_mgr = &adev->vm_manager.id_mgr[vmhub];
-	mutex_lock(&id_mgr->lock);
-	if (vm->reserved_vmid[vmhub])
-		goto unlock;
-	if (atomic_inc_return(&id_mgr->reserved_vmid_num) >
-	    AMDGPU_VM_MAX_RESERVED_VMID) {
-		DRM_ERROR("Over limitation of reserved vmid\n");
-		atomic_dec(&id_mgr->reserved_vmid_num);
-		r = -EINVAL;
-		goto unlock;
-	}
-	/* Select the first entry VMID */
-	idle = list_first_entry(&id_mgr->ids_lru, struct amdgpu_vm_id, list);
-	list_del_init(&idle->list);
-	vm->reserved_vmid[vmhub] = idle;
-	mutex_unlock(&id_mgr->lock);
-
-	return 0;
-unlock:
-	mutex_unlock(&id_mgr->lock);
-	return r;
+	return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr,
+				      adev->vm_manager.root_level);
 }
 
 /**
@@ -715,7 +415,7 @@ void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
 
 	has_compute_vm_bug = false;
 
-	ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
+	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 	if (ip_block) {
 		/* Compute has a VM bug for GFX version < 7.
 		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
@@ -741,14 +441,14 @@ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 {
 	struct amdgpu_device *adev = ring->adev;
 	unsigned vmhub = ring->funcs->vmhub;
-	struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
-	struct amdgpu_vm_id *id;
+	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
+	struct amdgpu_vmid *id;
 	bool gds_switch_needed;
 	bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
 
-	if (job->vm_id == 0)
+	if (job->vmid == 0)
 		return false;
-	id = &id_mgr->ids[job->vm_id];
+	id = &id_mgr->ids[job->vmid];
 	gds_switch_needed = ring->funcs->emit_gds_switch && (
 		id->gds_base != job->gds_base ||
 		id->gds_size != job->gds_size ||
@@ -757,7 +457,7 @@ bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 		id->oa_base != job->oa_base ||
 		id->oa_size != job->oa_size);
 
-	if (amdgpu_vm_had_gpu_reset(adev, id))
+	if (amdgpu_vmid_had_gpu_reset(adev, id))
 		return true;
 
 	return vm_flush_needed || gds_switch_needed;
@@ -772,7 +472,7 @@ static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
  * amdgpu_vm_flush - hardware flush the vm
  *
  * @ring: ring to use for flush
- * @vm_id: vmid number to use
+ * @vmid: vmid number to use
  * @pd_addr: address of the page directory
  *
  * Emit a VM flush when it is necessary.
@@ -781,8 +481,8 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_
 {
 	struct amdgpu_device *adev = ring->adev;
 	unsigned vmhub = ring->funcs->vmhub;
-	struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
-	struct amdgpu_vm_id *id = &id_mgr->ids[job->vm_id];
+	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
+	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
 	bool gds_switch_needed = ring->funcs->emit_gds_switch && (
 		id->gds_base != job->gds_base ||
 		id->gds_size != job->gds_size ||
@@ -794,7 +494,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_
 	unsigned patch_offset = 0;
 	int r;
 
-	if (amdgpu_vm_had_gpu_reset(adev, id)) {
+	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
 		gds_switch_needed = true;
 		vm_flush_needed = true;
 	}
@@ -811,8 +511,8 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_
 	if (ring->funcs->emit_vm_flush && vm_flush_needed) {
 		struct dma_fence *fence;
 
-		trace_amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr);
-		amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
+		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
+		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
 
 		r = amdgpu_fence_emit(ring, &fence);
 		if (r)
@@ -832,7 +532,7 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_
 		id->gws_size = job->gws_size;
 		id->oa_base = job->oa_base;
 		id->oa_size = job->oa_size;
-		amdgpu_ring_emit_gds_switch(ring, job->vm_id, job->gds_base,
+		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
 					    job->gds_size, job->gws_base,
 					    job->gws_size, job->oa_base,
 					    job->oa_size);
@@ -849,49 +549,6 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_
 	return 0;
 }
 
-/**
- * amdgpu_vm_reset_id - reset VMID to zero
- *
- * @adev: amdgpu device structure
- * @vm_id: vmid number to use
- *
- * Reset saved GDW, GWS and OA to force switch on next flush.
- */
-void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
-			unsigned vmid)
-{
-	struct amdgpu_vm_id_manager *id_mgr = &adev->vm_manager.id_mgr[vmhub];
-	struct amdgpu_vm_id *id = &id_mgr->ids[vmid];
-
-	atomic64_set(&id->owner, 0);
-	id->gds_base = 0;
-	id->gds_size = 0;
-	id->gws_base = 0;
-	id->gws_size = 0;
-	id->oa_base = 0;
-	id->oa_size = 0;
-}
-
-/**
- * amdgpu_vm_reset_all_id - reset VMID to zero
- *
- * @adev: amdgpu device structure
- *
- * Reset VMID to force flush on next use
- */
-void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev)
-{
-	unsigned i, j;
-
-	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
-		struct amdgpu_vm_id_manager *id_mgr =
-			&adev->vm_manager.id_mgr[i];
-
-		for (j = 1; j < id_mgr->num_ids; ++j)
-			amdgpu_vm_reset_id(adev, i, j);
-	}
-}
-
 /**
  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
  *
@@ -1043,162 +700,52 @@ static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 }
 
 /*
- * amdgpu_vm_update_level - update a single level in the hierarchy
+ * amdgpu_vm_update_pde - update a single level in the hierarchy
  *
- * @adev: amdgpu_device pointer
+ * @param: parameters for the update
  * @vm: requested vm
  * @parent: parent directory
+ * @entry: entry to update
  *
- * Makes sure all entries in @parent are up to date.
- * Returns 0 for success, error for failure.
+ * Makes sure the requested entry in parent is up to date.
  */
-static int amdgpu_vm_update_level(struct amdgpu_device *adev,
-				  struct amdgpu_vm *vm,
-				  struct amdgpu_vm_pt *parent)
+static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params,
+				 struct amdgpu_vm *vm,
+				 struct amdgpu_vm_pt *parent,
+				 struct amdgpu_vm_pt *entry)
 {
-	struct amdgpu_bo *shadow;
-	struct amdgpu_ring *ring = NULL;
+	struct amdgpu_bo *bo = entry->base.bo, *shadow = NULL, *pbo;
 	uint64_t pd_addr, shadow_addr = 0;
-	uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
-	unsigned count = 0, pt_idx, ndw = 0;
-	struct amdgpu_job *job;
-	struct amdgpu_pte_update_params params;
-	struct dma_fence *fence = NULL;
-	uint32_t incr;
-
-	int r;
-
-	if (!parent->entries)
-		return 0;
+	uint64_t pde, pt, flags;
+	unsigned level;
 
-	memset(&params, 0, sizeof(params));
-	params.adev = adev;
-	shadow = parent->base.bo->shadow;
+	/* Don't update huge pages here */
+	if (entry->huge)
+		return;
 
 	if (vm->use_cpu_for_update) {
 		pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
-		r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
-		if (unlikely(r))
-			return r;
-
-		params.func = amdgpu_vm_cpu_set_ptes;
 	} else {
-		ring = container_of(vm->entity.sched, struct amdgpu_ring,
-				    sched);
-
-		/* padding, etc. */
-		ndw = 64;
-
-		/* assume the worst case */
-		ndw += parent->last_entry_used * 6;
-
 		pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
-
-		if (shadow) {
+		shadow = parent->base.bo->shadow;
+		if (shadow)
 			shadow_addr = amdgpu_bo_gpu_offset(shadow);
-			ndw *= 2;
-		} else {
-			shadow_addr = 0;
-		}
-
-		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-		if (r)
-			return r;
-
-		params.ib = &job->ibs[0];
-		params.func = amdgpu_vm_do_set_ptes;
-	}
-
-
-	/* walk over the address space and update the directory */
-	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
-		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
-		struct amdgpu_bo *bo = entry->base.bo;
-		uint64_t pde, pt;
-
-		if (bo == NULL)
-			continue;
-
-		spin_lock(&vm->status_lock);
-		list_del_init(&entry->base.vm_status);
-		spin_unlock(&vm->status_lock);
-
-		pt = amdgpu_bo_gpu_offset(bo);
-		pt = amdgpu_gart_get_vm_pde(adev, pt);
-		/* Don't update huge pages here */
-		if ((parent->entries[pt_idx].addr & AMDGPU_PDE_PTE) ||
-		    parent->entries[pt_idx].addr == (pt | AMDGPU_PTE_VALID))
-			continue;
-
-		parent->entries[pt_idx].addr = pt | AMDGPU_PTE_VALID;
-
-		pde = pd_addr + pt_idx * 8;
-		incr = amdgpu_bo_size(bo);
-		if (((last_pde + 8 * count) != pde) ||
-		    ((last_pt + incr * count) != pt) ||
-		    (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
-
-			if (count) {
-				if (shadow)
-					params.func(&params,
-						    last_shadow,
-						    last_pt, count,
-						    incr,
-						    AMDGPU_PTE_VALID);
-
-				params.func(&params, last_pde,
-					    last_pt, count, incr,
-					    AMDGPU_PTE_VALID);
-			}
-
-			count = 1;
-			last_pde = pde;
-			last_shadow = shadow_addr + pt_idx * 8;
-			last_pt = pt;
-		} else {
-			++count;
-		}
 	}
 
-	if (count) {
-		if (vm->root.base.bo->shadow)
-			params.func(&params, last_shadow, last_pt,
-				    count, incr, AMDGPU_PTE_VALID);
+	for (level = 0, pbo = parent->base.bo->parent; pbo; ++level)
+		pbo = pbo->parent;
 
-		params.func(&params, last_pde, last_pt,
-			    count, incr, AMDGPU_PTE_VALID);
+	level += params->adev->vm_manager.root_level;
+	pt = amdgpu_bo_gpu_offset(bo);
+	flags = AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(params->adev, level, &pt, &flags);
+	if (shadow) {
+		pde = shadow_addr + (entry - parent->entries) * 8;
+		params->func(params, pde, pt, 1, 0, flags);
 	}
 
-	if (!vm->use_cpu_for_update) {
-		if (params.ib->length_dw == 0) {
-			amdgpu_job_free(job);
-		} else {
-			amdgpu_ring_pad_ib(ring, params.ib);
-			amdgpu_sync_resv(adev, &job->sync,
-					 parent->base.bo->tbo.resv,
-					 AMDGPU_FENCE_OWNER_VM, false);
-			if (shadow)
-				amdgpu_sync_resv(adev, &job->sync,
-						 shadow->tbo.resv,
-						 AMDGPU_FENCE_OWNER_VM, false);
-
-			WARN_ON(params.ib->length_dw > ndw);
-			r = amdgpu_job_submit(job, ring, &vm->entity,
-					AMDGPU_FENCE_OWNER_VM, &fence);
-			if (r)
-				goto error_free;
-
-			amdgpu_bo_fence(parent->base.bo, fence, true);
-			dma_fence_put(vm->last_update);
-			vm->last_update = fence;
-		}
-	}
-
-	return 0;
-
-error_free:
-	amdgpu_job_free(job);
-	return r;
+	pde = pd_addr + (entry - parent->entries) * 8;
+	params->func(params, pde, pt, 1, 0, flags);
 }
 
 /*
@@ -1208,27 +755,29 @@ static int amdgpu_vm_update_level(struct amdgpu_device *adev,
  *
  * Mark all PD level as invalid after an error.
  */
-static void amdgpu_vm_invalidate_level(struct amdgpu_vm *vm,
-				       struct amdgpu_vm_pt *parent)
+static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
+				       struct amdgpu_vm *vm,
+				       struct amdgpu_vm_pt *parent,
+				       unsigned level)
 {
-	unsigned pt_idx;
+	unsigned pt_idx, num_entries;
 
 	/*
 	 * Recurse into the subdirectories. This recursion is harmless because
 	 * we only have a maximum of 5 layers.
 	 */
-	for (pt_idx = 0; pt_idx <= parent->last_entry_used; ++pt_idx) {
+	num_entries = amdgpu_vm_num_entries(adev, level);
+	for (pt_idx = 0; pt_idx < num_entries; ++pt_idx) {
 		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
 
 		if (!entry->base.bo)
 			continue;
 
-		entry->addr = ~0ULL;
 		spin_lock(&vm->status_lock);
 		if (list_empty(&entry->base.vm_status))
 			list_add(&entry->base.vm_status, &vm->relocated);
 		spin_unlock(&vm->status_lock);
-		amdgpu_vm_invalidate_level(vm, entry);
+		amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
 	}
 }
 
@@ -1244,38 +793,63 @@ static void amdgpu_vm_invalidate_level(struct amdgpu_vm *vm,
 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 				 struct amdgpu_vm *vm)
 {
+	struct amdgpu_pte_update_params params;
+	struct amdgpu_job *job;
+	unsigned ndw = 0;
 	int r = 0;
 
+	if (list_empty(&vm->relocated))
+		return 0;
+
+restart:
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+
+	if (vm->use_cpu_for_update) {
+		r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
+		if (unlikely(r))
+			return r;
+
+		params.func = amdgpu_vm_cpu_set_ptes;
+	} else {
+		ndw = 512 * 8;
+		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+		if (r)
+			return r;
+
+		params.ib = &job->ibs[0];
+		params.func = amdgpu_vm_do_set_ptes;
+	}
+
 	spin_lock(&vm->status_lock);
 	while (!list_empty(&vm->relocated)) {
-		struct amdgpu_vm_bo_base *bo_base;
+		struct amdgpu_vm_bo_base *bo_base, *parent;
+		struct amdgpu_vm_pt *pt, *entry;
 		struct amdgpu_bo *bo;
 
 		bo_base = list_first_entry(&vm->relocated,
 					   struct amdgpu_vm_bo_base,
 					   vm_status);
+		list_del_init(&bo_base->vm_status);
 		spin_unlock(&vm->status_lock);
 
 		bo = bo_base->bo->parent;
-		if (bo) {
-			struct amdgpu_vm_bo_base *parent;
-			struct amdgpu_vm_pt *pt;
-
-			parent = list_first_entry(&bo->va,
-						  struct amdgpu_vm_bo_base,
-						  bo_list);
-			pt = container_of(parent, struct amdgpu_vm_pt, base);
-
-			r = amdgpu_vm_update_level(adev, vm, pt);
-			if (r) {
-				amdgpu_vm_invalidate_level(vm, &vm->root);
-				return r;
-			}
-			spin_lock(&vm->status_lock);
-		} else {
+		if (!bo) {
 			spin_lock(&vm->status_lock);
-			list_del_init(&bo_base->vm_status);
+			continue;
 		}
+
+		parent = list_first_entry(&bo->va, struct amdgpu_vm_bo_base,
+					  bo_list);
+		pt = container_of(parent, struct amdgpu_vm_pt, base);
+		entry = container_of(bo_base, struct amdgpu_vm_pt, base);
+
+		amdgpu_vm_update_pde(&params, vm, pt, entry);
+
+		spin_lock(&vm->status_lock);
+		if (!vm->use_cpu_for_update &&
+		    (ndw - params.ib->length_dw) < 32)
+			break;
 	}
 	spin_unlock(&vm->status_lock);
 
@@ -1283,8 +857,44 @@ int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 		/* Flush HDP */
 		mb();
 		amdgpu_gart_flush_gpu_tlb(adev, 0);
+	} else if (params.ib->length_dw == 0) {
+		amdgpu_job_free(job);
+	} else {
+		struct amdgpu_bo *root = vm->root.base.bo;
+		struct amdgpu_ring *ring;
+		struct dma_fence *fence;
+
+		ring = container_of(vm->entity.sched, struct amdgpu_ring,
+				    sched);
+
+		amdgpu_ring_pad_ib(ring, params.ib);
+		amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
+				 AMDGPU_FENCE_OWNER_VM, false);
+		if (root->shadow)
+			amdgpu_sync_resv(adev, &job->sync,
+					 root->shadow->tbo.resv,
+					 AMDGPU_FENCE_OWNER_VM, false);
+
+		WARN_ON(params.ib->length_dw > ndw);
+		r = amdgpu_job_submit(job, ring, &vm->entity,
+				      AMDGPU_FENCE_OWNER_VM, &fence);
+		if (r)
+			goto error;
+
+		amdgpu_bo_fence(root, fence, true);
+		dma_fence_put(vm->last_update);
+		vm->last_update = fence;
 	}
 
+	if (!list_empty(&vm->relocated))
+		goto restart;
+
+	return 0;
+
+error:
+	amdgpu_vm_invalidate_level(adev, vm, &vm->root,
+				   adev->vm_manager.root_level);
+	amdgpu_job_free(job);
 	return r;
 }
 
@@ -1302,18 +912,19 @@ void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
 			 struct amdgpu_vm_pt **entry,
 			 struct amdgpu_vm_pt **parent)
 {
-	unsigned idx, level = p->adev->vm_manager.num_level;
+	unsigned level = p->adev->vm_manager.root_level;
 
 	*parent = NULL;
 	*entry = &p->vm->root;
 	while ((*entry)->entries) {
-		idx = addr >> (p->adev->vm_manager.block_size * level--);
-		idx %= amdgpu_bo_size((*entry)->base.bo) / 8;
+		unsigned shift = amdgpu_vm_level_shift(p->adev, level++);
+
 		*parent = *entry;
-		*entry = &(*entry)->entries[idx];
+		*entry = &(*entry)->entries[addr >> shift];
+		addr &= (1ULL << shift) - 1;
 	}
 
-	if (level)
+	if (level != AMDGPU_VM_PTB)
 		*entry = NULL;
 }
 
@@ -1335,56 +946,42 @@ static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
 					unsigned nptes, uint64_t dst,
 					uint64_t flags)
 {
-	bool use_cpu_update = (p->func == amdgpu_vm_cpu_set_ptes);
 	uint64_t pd_addr, pde;
 
 	/* In the case of a mixed PT the PDE must point to it*/
-	if (p->adev->asic_type < CHIP_VEGA10 ||
-	    nptes != AMDGPU_VM_PTE_COUNT(p->adev) ||
-	    p->src ||
-	    !(flags & AMDGPU_PTE_VALID)) {
-
-		dst = amdgpu_bo_gpu_offset(entry->base.bo);
-		dst = amdgpu_gart_get_vm_pde(p->adev, dst);
-		flags = AMDGPU_PTE_VALID;
-	} else {
+	if (p->adev->asic_type >= CHIP_VEGA10 && !p->src &&
+	    nptes == AMDGPU_VM_PTE_COUNT(p->adev)) {
 		/* Set the huge page flag to stop scanning at this PDE */
 		flags |= AMDGPU_PDE_PTE;
 	}
 
-	if (entry->addr == (dst | flags))
+	if (!(flags & AMDGPU_PDE_PTE)) {
+		if (entry->huge) {
+			/* Add the entry to the relocated list to update it. */
+			entry->huge = false;
+			spin_lock(&p->vm->status_lock);
+			list_move(&entry->base.vm_status, &p->vm->relocated);
+			spin_unlock(&p->vm->status_lock);
+		}
 		return;
+	}
 
-	entry->addr = (dst | flags);
-
-	if (use_cpu_update) {
-		/* In case a huge page is replaced with a system
-		 * memory mapping, p->pages_addr != NULL and
-		 * amdgpu_vm_cpu_set_ptes would try to translate dst
-		 * through amdgpu_vm_map_gart. But dst is already a
-		 * GPU address (of the page table). Disable
-		 * amdgpu_vm_map_gart temporarily.
-		 */
-		dma_addr_t *tmp;
-
-		tmp = p->pages_addr;
-		p->pages_addr = NULL;
+	entry->huge = true;
+	amdgpu_gart_get_vm_pde(p->adev, AMDGPU_VM_PDB0,
+			       &dst, &flags);
 
+	if (p->func == amdgpu_vm_cpu_set_ptes) {
 		pd_addr = (unsigned long)amdgpu_bo_kptr(parent->base.bo);
-		pde = pd_addr + (entry - parent->entries) * 8;
-		amdgpu_vm_cpu_set_ptes(p, pde, dst, 1, 0, flags);
-
-		p->pages_addr = tmp;
 	} else {
 		if (parent->base.bo->shadow) {
 			pd_addr = amdgpu_bo_gpu_offset(parent->base.bo->shadow);
 			pde = pd_addr + (entry - parent->entries) * 8;
-			amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
+			p->func(p, pde, dst, 1, 0, flags);
 		}
 		pd_addr = amdgpu_bo_gpu_offset(parent->base.bo);
-		pde = pd_addr + (entry - parent->entries) * 8;
-		amdgpu_vm_do_set_ptes(p, pde, dst, 1, 0, flags);
 	}
+	pde = pd_addr + (entry - parent->entries) * 8;
+	p->func(p, pde, dst, 1, 0, flags);
 }
 
 /**
@@ -1429,7 +1026,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 		amdgpu_vm_handle_huge_pages(params, entry, parent,
 					    nptes, dst, flags);
 		/* We don't need to update PTEs for huge pages */
-		if (entry->addr & AMDGPU_PDE_PTE)
+		if (entry->huge)
 			continue;
 
 		pt = entry->base.bo;
@@ -1588,14 +1185,14 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
          *
          * The second command is for the shadow pagetables.
 	 */
-	ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
+	if (vm->root.base.bo->shadow)
+		ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
+	else
+		ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1);
 
 	/* padding, etc. */
 	ndw = 64;
 
-	/* one PDE write for each huge page */
-	ndw += ((nptes >> adev->vm_manager.block_size) + 1) * 6;
-
 	if (pages_addr) {
 		/* copy commands needed */
 		ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
@@ -1639,7 +1236,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 		addr = 0;
 	}
 
-	r = amdgpu_sync_fence(adev, &job->sync, exclusive);
+	r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
 	if (r)
 		goto error_free;
 
@@ -1670,7 +1267,6 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
 error_free:
 	amdgpu_job_free(job);
-	amdgpu_vm_invalidate_level(vm, &vm->root);
 	return r;
 }
 
@@ -2081,18 +1677,31 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 	spin_lock(&vm->status_lock);
 	while (!list_empty(&vm->moved)) {
 		struct amdgpu_bo_va *bo_va;
+		struct reservation_object *resv;
 
 		bo_va = list_first_entry(&vm->moved,
 			struct amdgpu_bo_va, base.vm_status);
 		spin_unlock(&vm->status_lock);
 
+		resv = bo_va->base.bo->tbo.resv;
+
 		/* Per VM BOs never need to bo cleared in the page tables */
-		clear = bo_va->base.bo->tbo.resv != vm->root.base.bo->tbo.resv;
+		if (resv == vm->root.base.bo->tbo.resv)
+			clear = false;
+		/* Try to reserve the BO to avoid clearing its ptes */
+		else if (!amdgpu_vm_debug && reservation_object_trylock(resv))
+			clear = false;
+		/* Somebody else is using the BO right now */
+		else
+			clear = true;
 
 		r = amdgpu_vm_bo_update(adev, bo_va, clear);
 		if (r)
 			return r;
 
+		if (!clear && resv != vm->root.base.bo->tbo.resv)
+			reservation_object_unlock(resv);
+
 		spin_lock(&vm->status_lock);
 	}
 	spin_unlock(&vm->status_lock);
@@ -2132,8 +1741,26 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
 	INIT_LIST_HEAD(&bo_va->valids);
 	INIT_LIST_HEAD(&bo_va->invalids);
 
-	if (bo)
-		list_add_tail(&bo_va->base.bo_list, &bo->va);
+	if (!bo)
+		return bo_va;
+
+	list_add_tail(&bo_va->base.bo_list, &bo->va);
+
+	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
+		return bo_va;
+
+	if (bo->preferred_domains &
+	    amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
+		return bo_va;
+
+	/*
+	 * We checked all the prerequisites, but it looks like this per VM BO
+	 * is currently evicted. add the BO to the evicted list to make sure it
+	 * is validated on next VM use to avoid fault.
+	 * */
+	spin_lock(&vm->status_lock);
+	list_move_tail(&bo_va->base.vm_status, &vm->evicted);
+	spin_unlock(&vm->status_lock);
 
 	return bo_va;
 }
@@ -2555,48 +2182,70 @@ static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
 		return ((bits + 3) / 2);
 }
 
-/**
- * amdgpu_vm_set_fragment_size - adjust fragment size in PTE
- *
- * @adev: amdgpu_device pointer
- * @fragment_size_default: the default fragment size if it's set auto
- */
-void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
-				 uint32_t fragment_size_default)
-{
-	if (amdgpu_vm_fragment_size == -1)
-		adev->vm_manager.fragment_size = fragment_size_default;
-	else
-		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
-}
-
 /**
  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
  *
  * @adev: amdgpu_device pointer
  * @vm_size: the default vm size if it's set auto
  */
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size,
-			   uint32_t fragment_size_default)
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
+			   uint32_t fragment_size_default, unsigned max_level,
+			   unsigned max_bits)
 {
-	/* adjust vm size firstly */
-	if (amdgpu_vm_size == -1)
-		adev->vm_manager.vm_size = vm_size;
-	else
-		adev->vm_manager.vm_size = amdgpu_vm_size;
+	uint64_t tmp;
 
-	/* block size depends on vm size */
-	if (amdgpu_vm_block_size == -1)
+	/* adjust vm size first */
+	if (amdgpu_vm_size != -1) {
+		unsigned max_size = 1 << (max_bits - 30);
+
+		vm_size = amdgpu_vm_size;
+		if (vm_size > max_size) {
+			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
+				 amdgpu_vm_size, max_size);
+			vm_size = max_size;
+		}
+	}
+
+	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
+
+	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
+	if (amdgpu_vm_block_size != -1)
+		tmp >>= amdgpu_vm_block_size - 9;
+	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
+	adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
+	switch (adev->vm_manager.num_level) {
+	case 3:
+		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
+		break;
+	case 2:
+		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
+		break;
+	case 1:
+		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
+		break;
+	default:
+		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
+	}
+	/* block size depends on vm size and hw setup*/
+	if (amdgpu_vm_block_size != -1)
 		adev->vm_manager.block_size =
-			amdgpu_vm_get_block_size(adev->vm_manager.vm_size);
+			min((unsigned)amdgpu_vm_block_size, max_bits
+			    - AMDGPU_GPU_PAGE_SHIFT
+			    - 9 * adev->vm_manager.num_level);
+	else if (adev->vm_manager.num_level > 1)
+		adev->vm_manager.block_size = 9;
 	else
-		adev->vm_manager.block_size = amdgpu_vm_block_size;
+		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
 
-	amdgpu_vm_set_fragment_size(adev, fragment_size_default);
+	if (amdgpu_vm_fragment_size == -1)
+		adev->vm_manager.fragment_size = fragment_size_default;
+	else
+		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
 
-	DRM_INFO("vm size is %llu GB, block size is %u-bit, fragment size is %u-bit\n",
-		adev->vm_manager.vm_size, adev->vm_manager.block_size,
-		adev->vm_manager.fragment_size);
+	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
+		 vm_size, adev->vm_manager.num_level + 1,
+		 adev->vm_manager.block_size,
+		 adev->vm_manager.fragment_size);
 }
 
 /**
@@ -2613,15 +2262,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 {
 	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
 		AMDGPU_VM_PTE_COUNT(adev) * 8);
+	uint64_t init_pde_value = 0, flags;
 	unsigned ring_instance;
 	struct amdgpu_ring *ring;
-	struct amd_sched_rq *rq;
+	struct drm_sched_rq *rq;
+	unsigned long size;
 	int r, i;
-	u64 flags;
-	uint64_t init_pde_value = 0;
 
 	vm->va = RB_ROOT_CACHED;
-	vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
 		vm->reserved_vmid[i] = NULL;
 	spin_lock_init(&vm->status_lock);
@@ -2635,9 +2283,9 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
 	ring_instance %= adev->vm_manager.vm_pte_num_rings;
 	ring = adev->vm_manager.vm_pte_rings[ring_instance];
-	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
-	r = amd_sched_entity_init(&ring->sched, &vm->entity,
-				  rq, amdgpu_sched_jobs);
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
+	r = drm_sched_entity_init(&ring->sched, &vm->entity,
+				  rq, amdgpu_sched_jobs, NULL);
 	if (r)
 		return r;
 
@@ -2670,27 +2318,21 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
 				AMDGPU_GEM_CREATE_SHADOW);
 
-	r = amdgpu_bo_create(adev, amdgpu_vm_bo_size(adev, 0), align, true,
-			     AMDGPU_GEM_DOMAIN_VRAM,
-			     flags,
-			     NULL, NULL, init_pde_value, &vm->root.base.bo);
+	size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
+	r = amdgpu_bo_create(adev, size, align, true, AMDGPU_GEM_DOMAIN_VRAM,
+			     flags, NULL, NULL, init_pde_value,
+			     &vm->root.base.bo);
 	if (r)
 		goto error_free_sched_entity;
 
+	r = amdgpu_bo_reserve(vm->root.base.bo, true);
+	if (r)
+		goto error_free_root;
+
 	vm->root.base.vm = vm;
 	list_add_tail(&vm->root.base.bo_list, &vm->root.base.bo->va);
-	INIT_LIST_HEAD(&vm->root.base.vm_status);
-
-	if (vm->use_cpu_for_update) {
-		r = amdgpu_bo_reserve(vm->root.base.bo, false);
-		if (r)
-			goto error_free_root;
-
-		r = amdgpu_bo_kmap(vm->root.base.bo, NULL);
-		amdgpu_bo_unreserve(vm->root.base.bo);
-		if (r)
-			goto error_free_root;
-	}
+	list_add_tail(&vm->root.base.vm_status, &vm->evicted);
+	amdgpu_bo_unreserve(vm->root.base.bo);
 
 	if (pasid) {
 		unsigned long flags;
@@ -2716,7 +2358,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	vm->root.base.bo = NULL;
 
 error_free_sched_entity:
-	amd_sched_entity_fini(&ring->sched, &vm->entity);
+	drm_sched_entity_fini(&ring->sched, &vm->entity);
 
 	return r;
 }
@@ -2724,26 +2366,31 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 /**
  * amdgpu_vm_free_levels - free PD/PT levels
  *
- * @level: PD/PT starting level to free
+ * @adev: amdgpu device structure
+ * @parent: PD/PT starting level to free
+ * @level: level of parent structure
  *
  * Free the page directory or page table level and all sub levels.
  */
-static void amdgpu_vm_free_levels(struct amdgpu_vm_pt *level)
+static void amdgpu_vm_free_levels(struct amdgpu_device *adev,
+				  struct amdgpu_vm_pt *parent,
+				  unsigned level)
 {
-	unsigned i;
+	unsigned i, num_entries = amdgpu_vm_num_entries(adev, level);
 
-	if (level->base.bo) {
-		list_del(&level->base.bo_list);
-		list_del(&level->base.vm_status);
-		amdgpu_bo_unref(&level->base.bo->shadow);
-		amdgpu_bo_unref(&level->base.bo);
+	if (parent->base.bo) {
+		list_del(&parent->base.bo_list);
+		list_del(&parent->base.vm_status);
+		amdgpu_bo_unref(&parent->base.bo->shadow);
+		amdgpu_bo_unref(&parent->base.bo);
 	}
 
-	if (level->entries)
-		for (i = 0; i <= level->last_entry_used; i++)
-			amdgpu_vm_free_levels(&level->entries[i]);
+	if (parent->entries)
+		for (i = 0; i < num_entries; i++)
+			amdgpu_vm_free_levels(adev, &parent->entries[i],
+					      level + 1);
 
-	kvfree(level->entries);
+	kvfree(parent->entries);
 }
 
 /**
@@ -2775,7 +2422,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
 	}
 
-	amd_sched_entity_fini(vm->entity.sched, &vm->entity);
+	drm_sched_entity_fini(vm->entity.sched, &vm->entity);
 
 	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
 		dev_err(adev->dev, "still active bo inside vm\n");
@@ -2801,13 +2448,14 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	if (r) {
 		dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
 	} else {
-		amdgpu_vm_free_levels(&vm->root);
+		amdgpu_vm_free_levels(adev, &vm->root,
+				      adev->vm_manager.root_level);
 		amdgpu_bo_unreserve(root);
 	}
 	amdgpu_bo_unref(&root);
 	dma_fence_put(vm->last_update);
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
-		amdgpu_vm_free_reserved_vmid(adev, vm, i);
+		amdgpu_vmid_free_reserved(adev, vm, i);
 }
 
 /**
@@ -2826,17 +2474,21 @@ bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
 
 	spin_lock(&adev->vm_manager.pasid_lock);
 	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
-	spin_unlock(&adev->vm_manager.pasid_lock);
-	if (!vm)
+	if (!vm) {
 		/* VM not found, can't track fault credit */
+		spin_unlock(&adev->vm_manager.pasid_lock);
 		return true;
+	}
 
 	/* No lock needed. only accessed by IRQ handler */
-	if (!vm->fault_credit)
+	if (!vm->fault_credit) {
 		/* Too many faults in this VM */
+		spin_unlock(&adev->vm_manager.pasid_lock);
 		return false;
+	}
 
 	vm->fault_credit--;
+	spin_unlock(&adev->vm_manager.pasid_lock);
 	return true;
 }
 
@@ -2849,23 +2501,9 @@ bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
  */
 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
 {
-	unsigned i, j;
-
-	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
-		struct amdgpu_vm_id_manager *id_mgr =
-			&adev->vm_manager.id_mgr[i];
-
-		mutex_init(&id_mgr->lock);
-		INIT_LIST_HEAD(&id_mgr->ids_lru);
-		atomic_set(&id_mgr->reserved_vmid_num, 0);
+	unsigned i;
 
-		/* skip over VMID 0, since it is the system VM */
-		for (j = 1; j < id_mgr->num_ids; ++j) {
-			amdgpu_vm_reset_id(adev, i, j);
-			amdgpu_sync_create(&id_mgr->ids[i].active);
-			list_add_tail(&id_mgr->ids[j].list, &id_mgr->ids_lru);
-		}
-	}
+	amdgpu_vmid_mgr_init(adev);
 
 	adev->vm_manager.fence_context =
 		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
@@ -2873,7 +2511,6 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
 		adev->vm_manager.seqno[i] = 0;
 
 	atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
-	atomic64_set(&adev->vm_manager.client_counter, 0);
 	spin_lock_init(&adev->vm_manager.prt_lock);
 	atomic_set(&adev->vm_manager.num_prt_users, 0);
 
@@ -2906,24 +2543,10 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
  */
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 {
-	unsigned i, j;
-
 	WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
 	idr_destroy(&adev->vm_manager.pasid_idr);
 
-	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
-		struct amdgpu_vm_id_manager *id_mgr =
-			&adev->vm_manager.id_mgr[i];
-
-		mutex_destroy(&id_mgr->lock);
-		for (j = 0; j < AMDGPU_NUM_VM; ++j) {
-			struct amdgpu_vm_id *id = &id_mgr->ids[j];
-
-			amdgpu_sync_free(&id->active);
-			dma_fence_put(id->flushed_updates);
-			dma_fence_put(id->last_flush);
-		}
-	}
+	amdgpu_vmid_mgr_fini(adev);
 }
 
 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
@@ -2936,13 +2559,12 @@ int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 	switch (args->in.op) {
 	case AMDGPU_VM_OP_RESERVE_VMID:
 		/* current, we only have requirement to reserve vmid from gfxhub */
-		r = amdgpu_vm_alloc_reserved_vmid(adev, &fpriv->vm,
-						  AMDGPU_GFXHUB);
+		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
 		if (r)
 			return r;
 		break;
 	case AMDGPU_VM_OP_UNRESERVE_VMID:
-		amdgpu_vm_free_reserved_vmid(adev, &fpriv->vm, AMDGPU_GFXHUB);
+		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
 		break;
 	default:
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index bae77353447b..21a80f1bb2b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -24,12 +24,14 @@
 #ifndef __AMDGPU_VM_H__
 #define __AMDGPU_VM_H__
 
-#include <linux/rbtree.h>
 #include <linux/idr.h>
+#include <linux/kfifo.h>
+#include <linux/rbtree.h>
+#include <drm/gpu_scheduler.h>
 
-#include "gpu_scheduler.h"
 #include "amdgpu_sync.h"
 #include "amdgpu_ring.h"
+#include "amdgpu_ids.h"
 
 struct amdgpu_bo_va;
 struct amdgpu_job;
@@ -39,9 +41,6 @@ struct amdgpu_bo_list_entry;
  * GPUVM handling
  */
 
-/* maximum number of VMIDs */
-#define AMDGPU_NUM_VM	16
-
 /* Maximum number of PTEs the hardware can write with one command */
 #define AMDGPU_VM_MAX_UPDATE_SIZE	0x3FFFF
 
@@ -69,6 +68,12 @@ struct amdgpu_bo_list_entry;
 /* PDE is handled as PTE for VEGA10 */
 #define AMDGPU_PDE_PTE		(1ULL << 54)
 
+/* PTE is handled as PDE for VEGA10 (Translate Further) */
+#define AMDGPU_PTE_TF		(1ULL << 56)
+
+/* PDE Block Fragment Size for VEGA10 */
+#define AMDGPU_PDE_BFS(a)	((uint64_t)a << 59)
+
 /* VEGA10 only */
 #define AMDGPU_PTE_MTYPE(a)    ((uint64_t)a << 57)
 #define AMDGPU_PTE_MTYPE_MASK	AMDGPU_PTE_MTYPE(3ULL)
@@ -96,6 +101,19 @@ struct amdgpu_bo_list_entry;
 /* hardcode that limit for now */
 #define AMDGPU_VA_RESERVED_SIZE			(8ULL << 20)
 
+/* VA hole for 48bit addresses on Vega10 */
+#define AMDGPU_VA_HOLE_START			0x0000800000000000ULL
+#define AMDGPU_VA_HOLE_END			0xffff800000000000ULL
+
+/*
+ * Hardware is programmed as if the hole doesn't exists with start and end
+ * address values.
+ *
+ * This mask is used to remove the upper 16bits of the VA and so come up with
+ * the linear addr value.
+ */
+#define AMDGPU_VA_HOLE_MASK			0x0000ffffffffffffULL
+
 /* max vmids dedicated for process */
 #define AMDGPU_VM_MAX_RESERVED_VMID	1
 
@@ -106,6 +124,16 @@ struct amdgpu_bo_list_entry;
 #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
 #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
 
+/* VMPT level enumerate, and the hiberachy is:
+ * PDB2->PDB1->PDB0->PTB
+ */
+enum amdgpu_vm_level {
+	AMDGPU_VM_PDB2,
+	AMDGPU_VM_PDB1,
+	AMDGPU_VM_PDB0,
+	AMDGPU_VM_PTB
+};
+
 /* base structure for tracking BO usage in a VM */
 struct amdgpu_vm_bo_base {
 	/* constant after initialization */
@@ -124,11 +152,10 @@ struct amdgpu_vm_bo_base {
 
 struct amdgpu_vm_pt {
 	struct amdgpu_vm_bo_base	base;
-	uint64_t			addr;
+	bool				huge;
 
 	/* array of page tables, one for each directory entry */
 	struct amdgpu_vm_pt		*entries;
-	unsigned			last_entry_used;
 };
 
 #define AMDGPU_VM_FAULT(pasid, addr) (((u64)(pasid) << 48) | (addr))
@@ -162,13 +189,11 @@ struct amdgpu_vm {
 	spinlock_t		freed_lock;
 
 	/* Scheduler entity for page table updates */
-	struct amd_sched_entity	entity;
+	struct drm_sched_entity	entity;
 
-	/* client id and PASID (TODO: replace client_id with PASID) */
-	u64                     client_id;
 	unsigned int		pasid;
 	/* dedicated to vm */
-	struct amdgpu_vm_id	*reserved_vmid[AMDGPU_MAX_VMHUBS];
+	struct amdgpu_vmid	*reserved_vmid[AMDGPU_MAX_VMHUBS];
 
 	/* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */
 	bool                    use_cpu_for_update;
@@ -183,37 +208,9 @@ struct amdgpu_vm {
 	unsigned int		fault_credit;
 };
 
-struct amdgpu_vm_id {
-	struct list_head	list;
-	struct amdgpu_sync	active;
-	struct dma_fence		*last_flush;
-	atomic64_t		owner;
-
-	uint64_t		pd_gpu_addr;
-	/* last flushed PD/PT update */
-	struct dma_fence		*flushed_updates;
-
-	uint32_t                current_gpu_reset_count;
-
-	uint32_t		gds_base;
-	uint32_t		gds_size;
-	uint32_t		gws_base;
-	uint32_t		gws_size;
-	uint32_t		oa_base;
-	uint32_t		oa_size;
-};
-
-struct amdgpu_vm_id_manager {
-	struct mutex		lock;
-	unsigned		num_ids;
-	struct list_head	ids_lru;
-	struct amdgpu_vm_id	ids[AMDGPU_NUM_VM];
-	atomic_t		reserved_vmid_num;
-};
-
 struct amdgpu_vm_manager {
 	/* Handling of VMIDs */
-	struct amdgpu_vm_id_manager		id_mgr[AMDGPU_MAX_VMHUBS];
+	struct amdgpu_vmid_mgr			id_mgr[AMDGPU_MAX_VMHUBS];
 
 	/* Handling of VM fences */
 	u64					fence_context;
@@ -221,9 +218,9 @@ struct amdgpu_vm_manager {
 
 	uint64_t				max_pfn;
 	uint32_t				num_level;
-	uint64_t				vm_size;
 	uint32_t				block_size;
 	uint32_t				fragment_size;
+	enum amdgpu_vm_level			root_level;
 	/* vram base address for page table entry  */
 	u64					vram_base_offset;
 	/* vm pte handling */
@@ -231,8 +228,6 @@ struct amdgpu_vm_manager {
 	struct amdgpu_ring                      *vm_pte_rings[AMDGPU_MAX_RINGS];
 	unsigned				vm_pte_num_rings;
 	atomic_t				vm_pte_next_ring;
-	/* client id counter */
-	atomic64_t				client_counter;
 
 	/* partial resident texture handling */
 	spinlock_t				prt_lock;
@@ -251,8 +246,6 @@ struct amdgpu_vm_manager {
 	spinlock_t				pasid_lock;
 };
 
-int amdgpu_vm_alloc_pasid(unsigned int bits);
-void amdgpu_vm_free_pasid(unsigned int pasid);
 void amdgpu_vm_manager_init(struct amdgpu_device *adev);
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
@@ -270,13 +263,7 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 			struct amdgpu_vm *vm,
 			uint64_t saddr, uint64_t size);
-int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
-		      struct amdgpu_sync *sync, struct dma_fence *fence,
-		      struct amdgpu_job *job);
 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
-void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vmhub,
-			unsigned vmid);
-void amdgpu_vm_reset_all_ids(struct amdgpu_device *adev);
 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 				 struct amdgpu_vm *vm);
 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
@@ -312,10 +299,9 @@ struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
 							 uint64_t addr);
 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 		      struct amdgpu_bo_va *bo_va);
-void amdgpu_vm_set_fragment_size(struct amdgpu_device *adev,
-				uint32_t fragment_size_default);
-void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size,
-				uint32_t fragment_size_default);
+void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
+			   uint32_t fragment_size_default, unsigned max_level,
+			   unsigned max_bits);
 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 				  struct amdgpu_job *job);
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
index 2af26d2da127..d702fb8e3427 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
@@ -34,7 +34,7 @@
 #include <linux/backlight.h>
 #include "bif/bif_4_1_d.h"
 
-static u8
+u8
 amdgpu_atombios_encoder_get_backlight_level_from_reg(struct amdgpu_device *adev)
 {
 	u8 backlight_level;
@@ -48,7 +48,7 @@ amdgpu_atombios_encoder_get_backlight_level_from_reg(struct amdgpu_device *adev)
 	return backlight_level;
 }
 
-static void
+void
 amdgpu_atombios_encoder_set_backlight_level_to_reg(struct amdgpu_device *adev,
 					    u8 backlight_level)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.h b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.h
index 2bdec40515ce..f77cbdef679e 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.h
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.h
@@ -24,6 +24,11 @@
 #ifndef __ATOMBIOS_ENCODER_H__
 #define __ATOMBIOS_ENCODER_H__
 
+u8
+amdgpu_atombios_encoder_get_backlight_level_from_reg(struct amdgpu_device *adev);
+void
+amdgpu_atombios_encoder_set_backlight_level_to_reg(struct amdgpu_device *adev,
+						   u8 backlight_level);
 u8
 amdgpu_atombios_encoder_get_backlight_level(struct amdgpu_encoder *amdgpu_encoder);
 void
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
index b374653bd6cf..f9b2ce9a98f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_i2c.c
@@ -65,8 +65,15 @@ static int amdgpu_atombios_i2c_process_i2c_ch(struct amdgpu_i2c_chan *chan,
 			args.ucRegIndex = buf[0];
 		if (num)
 			num--;
-		if (num)
-			memcpy(&out, &buf[1], num);
+		if (num) {
+			if (buf) {
+				memcpy(&out, &buf[1], num);
+			} else {
+				DRM_ERROR("hw i2c: missing buf with num > 1\n");
+				r = -EINVAL;
+				goto done;
+			}
+		}
 		args.lpI2CDataOut = cpu_to_le16(out);
 	} else {
 		if (num > ATOM_MAX_HW_I2C_READ) {
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 68b505c768ad..a0943aa8d1d3 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -891,12 +891,12 @@ static void ci_dpm_powergate_uvd(void *handle, bool gate)
 
 	if (gate) {
 		/* stop the UVD block */
-		amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							AMD_PG_STATE_GATE);
+		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+						       AMD_PG_STATE_GATE);
 		ci_update_uvd_dpm(adev, gate);
 	} else {
-		amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							AMD_PG_STATE_UNGATE);
+		amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+						       AMD_PG_STATE_UNGATE);
 		ci_update_uvd_dpm(adev, gate);
 	}
 }
@@ -4540,9 +4540,9 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
 					((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
 			}
 			j++;
+
 			if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
 				return -EINVAL;
-
 			temp_reg = RREG32(mmMC_PMG_CMD_MRS);
 			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
 			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
@@ -4553,10 +4553,10 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
 					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
 			}
 			j++;
-			if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
-				return -EINVAL;
 
 			if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) {
+				if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
+					return -EINVAL;
 				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
 				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
 				for (k = 0; k < table->num_entries; k++) {
@@ -4564,8 +4564,6 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
 						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
 				}
 				j++;
-				if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
-					return -EINVAL;
 			}
 			break;
 		case mmMC_SEQ_RESERVE_M:
@@ -4577,8 +4575,6 @@ static int ci_set_mc_special_registers(struct amdgpu_device *adev,
 					(temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
 			}
 			j++;
-			if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
-				return -EINVAL;
 			break;
 		default:
 			break;
@@ -6625,9 +6621,9 @@ static int ci_dpm_print_clock_levels(void *handle,
 
 		for (i = 0; i < pcie_table->count; i++)
 			size += sprintf(buf + size, "%d: %s %s\n", i,
-					(pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x1" :
-					(pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" :
-					(pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "",
+					(pcie_table->dpm_levels[i].value == 0) ? "2.5GT/s, x1" :
+					(pcie_table->dpm_levels[i].value == 1) ? "5.0GT/s, x16" :
+					(pcie_table->dpm_levels[i].value == 2) ? "8.0GT/s, x16" : "",
 					(i == now) ? "*" : "");
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index a296f7bbe57c..8e59e65efd44 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -755,74 +755,74 @@ static void cik_init_golden_registers(struct amdgpu_device *adev)
 
 	switch (adev->asic_type) {
 	case CHIP_BONAIRE:
-		amdgpu_program_register_sequence(adev,
-						 bonaire_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 bonaire_golden_registers,
-						 (const u32)ARRAY_SIZE(bonaire_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 bonaire_golden_common_registers,
-						 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
-		amdgpu_program_register_sequence(adev,
-						 bonaire_golden_spm_registers,
-						 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
+		amdgpu_device_program_register_sequence(adev,
+							bonaire_mgcg_cgcg_init,
+							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							bonaire_golden_registers,
+							ARRAY_SIZE(bonaire_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							bonaire_golden_common_registers,
+							ARRAY_SIZE(bonaire_golden_common_registers));
+		amdgpu_device_program_register_sequence(adev,
+							bonaire_golden_spm_registers,
+							ARRAY_SIZE(bonaire_golden_spm_registers));
 		break;
 	case CHIP_KABINI:
-		amdgpu_program_register_sequence(adev,
-						 kalindi_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 kalindi_golden_registers,
-						 (const u32)ARRAY_SIZE(kalindi_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 kalindi_golden_common_registers,
-						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
-		amdgpu_program_register_sequence(adev,
-						 kalindi_golden_spm_registers,
-						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
+		amdgpu_device_program_register_sequence(adev,
+							kalindi_mgcg_cgcg_init,
+							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							kalindi_golden_registers,
+							ARRAY_SIZE(kalindi_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							kalindi_golden_common_registers,
+							ARRAY_SIZE(kalindi_golden_common_registers));
+		amdgpu_device_program_register_sequence(adev,
+							kalindi_golden_spm_registers,
+							ARRAY_SIZE(kalindi_golden_spm_registers));
 		break;
 	case CHIP_MULLINS:
-		amdgpu_program_register_sequence(adev,
-						 kalindi_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 godavari_golden_registers,
-						 (const u32)ARRAY_SIZE(godavari_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 kalindi_golden_common_registers,
-						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
-		amdgpu_program_register_sequence(adev,
-						 kalindi_golden_spm_registers,
-						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
+		amdgpu_device_program_register_sequence(adev,
+							kalindi_mgcg_cgcg_init,
+							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							godavari_golden_registers,
+							ARRAY_SIZE(godavari_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							kalindi_golden_common_registers,
+							ARRAY_SIZE(kalindi_golden_common_registers));
+		amdgpu_device_program_register_sequence(adev,
+							kalindi_golden_spm_registers,
+							ARRAY_SIZE(kalindi_golden_spm_registers));
 		break;
 	case CHIP_KAVERI:
-		amdgpu_program_register_sequence(adev,
-						 spectre_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 spectre_golden_registers,
-						 (const u32)ARRAY_SIZE(spectre_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 spectre_golden_common_registers,
-						 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
-		amdgpu_program_register_sequence(adev,
-						 spectre_golden_spm_registers,
-						 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
+		amdgpu_device_program_register_sequence(adev,
+							spectre_mgcg_cgcg_init,
+							ARRAY_SIZE(spectre_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							spectre_golden_registers,
+							ARRAY_SIZE(spectre_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							spectre_golden_common_registers,
+							ARRAY_SIZE(spectre_golden_common_registers));
+		amdgpu_device_program_register_sequence(adev,
+							spectre_golden_spm_registers,
+							ARRAY_SIZE(spectre_golden_spm_registers));
 		break;
 	case CHIP_HAWAII:
-		amdgpu_program_register_sequence(adev,
-						 hawaii_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 hawaii_golden_registers,
-						 (const u32)ARRAY_SIZE(hawaii_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 hawaii_golden_common_registers,
-						 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
-		amdgpu_program_register_sequence(adev,
-						 hawaii_golden_spm_registers,
-						 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
+		amdgpu_device_program_register_sequence(adev,
+							hawaii_mgcg_cgcg_init,
+							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							hawaii_golden_registers,
+							ARRAY_SIZE(hawaii_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							hawaii_golden_common_registers,
+							ARRAY_SIZE(hawaii_golden_common_registers));
+		amdgpu_device_program_register_sequence(adev,
+							hawaii_golden_spm_registers,
+							ARRAY_SIZE(hawaii_golden_spm_registers));
 		break;
 	default:
 		break;
@@ -1246,7 +1246,7 @@ static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
 	/* disable BM */
 	pci_clear_master(adev->pdev);
 	/* reset */
-	amdgpu_pci_config_reset(adev);
+	amdgpu_device_pci_config_reset(adev);
 
 	udelay(100);
 
@@ -1866,7 +1866,7 @@ static int cik_common_early_init(void *handle)
 
 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
 
-	amdgpu_get_pcie_info(adev);
+	amdgpu_device_get_pcie_info(adev);
 
 	return 0;
 }
@@ -1974,77 +1974,77 @@ int cik_set_ip_blocks(struct amdgpu_device *adev)
 
 	switch (adev->asic_type) {
 	case CHIP_BONAIRE:
-		amdgpu_ip_block_add(adev, &cik_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
-		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v8_2_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
-		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 		break;
 	case CHIP_HAWAII:
-		amdgpu_ip_block_add(adev, &cik_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
-		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v8_5_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v7_3_ip_block);
-		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 		break;
 	case CHIP_KAVERI:
-		amdgpu_ip_block_add(adev, &cik_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
-		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v8_1_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v7_1_ip_block);
-		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 		break;
 	case CHIP_KABINI:
 	case CHIP_MULLINS:
-		amdgpu_ip_block_add(adev, &cik_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v7_0_ip_block);
-		amdgpu_ip_block_add(adev, &cik_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v8_3_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v7_2_ip_block);
-		amdgpu_ip_block_add(adev, &cik_sdma_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v4_2_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v2_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
 		break;
 	default:
 		/* FIXME: not supported yet */
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_ih.c b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
index a870b354e3f7..d5a05c19708f 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_ih.c
@@ -280,7 +280,7 @@ static void cik_ih_decode_iv(struct amdgpu_device *adev,
 	entry->src_id = dw[0] & 0xff;
 	entry->src_data[0] = dw[1] & 0xfffffff;
 	entry->ring_id = dw[2] & 0xff;
-	entry->vm_id = (dw[2] >> 8) & 0xff;
+	entry->vmid = (dw[2] >> 8) & 0xff;
 	entry->pas_id = (dw[2] >> 16) & 0xffff;
 
 	/* wptr/rptr are in bytes! */
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 60cecd117705..6e8278e689b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -221,9 +221,9 @@ static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
  */
 static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring,
 				  struct amdgpu_ib *ib,
-				  unsigned vm_id, bool ctx_switch)
+				  unsigned vmid, bool ctx_switch)
 {
-	u32 extra_bits = vm_id & 0xf;
+	u32 extra_bits = vmid & 0xf;
 
 	/* IB packet must end on a 8 DW boundary */
 	cik_sdma_ring_insert_nop(ring, (12 - (lower_32_bits(ring->wptr) & 7)) % 8);
@@ -626,7 +626,7 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -639,7 +639,7 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_wb_free(adev, index);
+		amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
@@ -657,13 +657,13 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
 			  ring->idx, tmp);
 		r = -EINVAL;
 	}
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 
 	return r;
 }
@@ -686,7 +686,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -724,7 +724,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = le32_to_cpu(adev->wb.wb[index]);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
@@ -735,7 +735,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(adev, &ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 	return r;
 }
 
@@ -880,23 +880,23 @@ static void cik_sdma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
  * using sDMA (CIK).
  */
 static void cik_sdma_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					unsigned vm_id, uint64_t pd_addr)
+					unsigned vmid, uint64_t pd_addr)
 {
 	u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(0) |
 			  SDMA_POLL_REG_MEM_EXTRA_FUNC(0)); /* always */
 
 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
-	if (vm_id < 8) {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
+	if (vmid < 8) {
+		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid));
 	} else {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));
+		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8));
 	}
 	amdgpu_ring_write(ring, pd_addr >> 12);
 
 	/* flush TLB */
 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 
 	amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
diff --git a/drivers/gpu/drm/amd/amdgpu/cikd.h b/drivers/gpu/drm/amd/amdgpu/cikd.h
index 6a9e38a3d2a0..cee6e8a3ad9c 100644
--- a/drivers/gpu/drm/amd/amdgpu/cikd.h
+++ b/drivers/gpu/drm/amd/amdgpu/cikd.h
@@ -562,7 +562,7 @@
 #define	PRIVATE_BASE(x)	((x) << 0) /* scratch */
 #define	SHARED_BASE(x)	((x) << 16) /* LDS */
 
-#define KFD_CIK_SDMA_QUEUE_OFFSET	0x200
+#define KFD_CIK_SDMA_QUEUE_OFFSET (mmSDMA0_RLC1_RB_CNTL - mmSDMA0_RLC0_RB_CNTL)
 
 /* valid for both DEFAULT_MTYPE and APE1_MTYPE */
 enum {
diff --git a/drivers/gpu/drm/amd/amdgpu/clearstate_gfx9.h b/drivers/gpu/drm/amd/amdgpu/clearstate_gfx9.h
index 003a131bad47..567a904804bc 100644
--- a/drivers/gpu/drm/amd/amdgpu/clearstate_gfx9.h
+++ b/drivers/gpu/drm/amd/amdgpu/clearstate_gfx9.h
@@ -48,7 +48,7 @@ static const unsigned int gfx9_SECT_CONTEXT_def_1[] =
     0x00000000, // DB_STENCIL_WRITE_BASE
     0x00000000, // DB_STENCIL_WRITE_BASE_HI
     0x00000000, // DB_DFSM_CONTROL
-    0x00000000, // DB_RENDER_FILTER
+    0, // HOLE
     0x00000000, // DB_Z_INFO2
     0x00000000, // DB_STENCIL_INFO2
     0, // HOLE
@@ -259,8 +259,8 @@ static const unsigned int gfx9_SECT_CONTEXT_def_2[] =
     0x00000000, // PA_SC_RIGHT_VERT_GRID
     0x00000000, // PA_SC_LEFT_VERT_GRID
     0x00000000, // PA_SC_HORIZ_GRID
-    0x00000000, // PA_SC_FOV_WINDOW_LR
-    0x00000000, // PA_SC_FOV_WINDOW_TB
+    0, // HOLE
+    0, // HOLE
     0, // HOLE
     0, // HOLE
     0, // HOLE
@@ -701,7 +701,7 @@ static const unsigned int gfx9_SECT_CONTEXT_def_7[] =
 {
     0x00000000, // VGT_GS_MAX_PRIMS_PER_SUBGROUP
     0x00000000, // VGT_DRAW_PAYLOAD_CNTL
-    0x00000000, // VGT_INDEX_PAYLOAD_CNTL
+    0, // HOLE
     0x00000000, // VGT_INSTANCE_STEP_RATE_0
     0x00000000, // VGT_INSTANCE_STEP_RATE_1
     0, // HOLE
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
index fa61d649bb44..f576e9cbbc61 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c
@@ -259,7 +259,7 @@ static void cz_ih_decode_iv(struct amdgpu_device *adev,
 	entry->src_id = dw[0] & 0xff;
 	entry->src_data[0] = dw[1] & 0xfffffff;
 	entry->ring_id = dw[2] & 0xff;
-	entry->vm_id = (dw[2] >> 8) & 0xff;
+	entry->vmid = (dw[2] >> 8) & 0xff;
 	entry->pas_id = (dw[2] >> 16) & 0xffff;
 
 	/* wptr/rptr are in bytes! */
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 4e519dc42916..022f303463fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -145,20 +145,20 @@ static void dce_v10_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_FIJI:
-		amdgpu_program_register_sequence(adev,
-						 fiji_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_fiji_a10,
-						 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
+		amdgpu_device_program_register_sequence(adev,
+							fiji_mgcg_cgcg_init,
+							ARRAY_SIZE(fiji_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_fiji_a10,
+							ARRAY_SIZE(golden_settings_fiji_a10));
 		break;
 	case CHIP_TONGA:
-		amdgpu_program_register_sequence(adev,
-						 tonga_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_tonga_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
+		amdgpu_device_program_register_sequence(adev,
+							tonga_mgcg_cgcg_init,
+							ARRAY_SIZE(tonga_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_tonga_a11,
+							ARRAY_SIZE(golden_settings_tonga_a11));
 		break;
 	default:
 		break;
@@ -2773,7 +2773,6 @@ static int dce_v10_0_early_init(void *handle)
 	adev->audio_endpt_wreg = &dce_v10_0_audio_endpt_wreg;
 
 	dce_v10_0_set_display_funcs(adev);
-	dce_v10_0_set_irq_funcs(adev);
 
 	adev->mode_info.num_crtc = dce_v10_0_get_num_crtc(adev);
 
@@ -2788,6 +2787,8 @@ static int dce_v10_0_early_init(void *handle)
 		return -EINVAL;
 	}
 
+	dce_v10_0_set_irq_funcs(adev);
+
 	return 0;
 }
 
@@ -2920,6 +2921,11 @@ static int dce_v10_0_hw_fini(void *handle)
 
 static int dce_v10_0_suspend(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->mode_info.bl_level =
+		amdgpu_atombios_encoder_get_backlight_level_from_reg(adev);
+
 	return dce_v10_0_hw_fini(handle);
 }
 
@@ -2928,6 +2934,9 @@ static int dce_v10_0_resume(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int ret;
 
+	amdgpu_atombios_encoder_set_backlight_level_to_reg(adev,
+							   adev->mode_info.bl_level);
+
 	ret = dce_v10_0_hw_init(handle);
 
 	/* turn on the BL */
@@ -3635,13 +3644,16 @@ static const struct amdgpu_irq_src_funcs dce_v10_0_hpd_irq_funcs = {
 
 static void dce_v10_0_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+	if (adev->mode_info.num_crtc > 0)
+		adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
+	else
+		adev->crtc_irq.num_types = 0;
 	adev->crtc_irq.funcs = &dce_v10_0_crtc_irq_funcs;
 
-	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
+	adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
 	adev->pageflip_irq.funcs = &dce_v10_0_pageflip_irq_funcs;
 
-	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
+	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
 	adev->hpd_irq.funcs = &dce_v10_0_hpd_irq_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index 11edc75edaa9..800a9f36ab4f 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -154,28 +154,28 @@ static void dce_v11_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_CARRIZO:
-		amdgpu_program_register_sequence(adev,
-						 cz_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 cz_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(cz_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							cz_mgcg_cgcg_init,
+							ARRAY_SIZE(cz_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							cz_golden_settings_a11,
+							ARRAY_SIZE(cz_golden_settings_a11));
 		break;
 	case CHIP_STONEY:
-		amdgpu_program_register_sequence(adev,
-						 stoney_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(stoney_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_golden_settings_a11,
+							ARRAY_SIZE(stoney_golden_settings_a11));
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
-		amdgpu_program_register_sequence(adev,
-						 polaris11_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(polaris11_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							polaris11_golden_settings_a11,
+							ARRAY_SIZE(polaris11_golden_settings_a11));
 		break;
 	case CHIP_POLARIS10:
-		amdgpu_program_register_sequence(adev,
-						 polaris10_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(polaris10_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							polaris10_golden_settings_a11,
+							ARRAY_SIZE(polaris10_golden_settings_a11));
 		break;
 	default:
 		break;
@@ -2876,7 +2876,6 @@ static int dce_v11_0_early_init(void *handle)
 	adev->audio_endpt_wreg = &dce_v11_0_audio_endpt_wreg;
 
 	dce_v11_0_set_display_funcs(adev);
-	dce_v11_0_set_irq_funcs(adev);
 
 	adev->mode_info.num_crtc = dce_v11_0_get_num_crtc(adev);
 
@@ -2903,6 +2902,8 @@ static int dce_v11_0_early_init(void *handle)
 		return -EINVAL;
 	}
 
+	dce_v11_0_set_irq_funcs(adev);
+
 	return 0;
 }
 
@@ -3046,6 +3047,11 @@ static int dce_v11_0_hw_fini(void *handle)
 
 static int dce_v11_0_suspend(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->mode_info.bl_level =
+		amdgpu_atombios_encoder_get_backlight_level_from_reg(adev);
+
 	return dce_v11_0_hw_fini(handle);
 }
 
@@ -3054,6 +3060,9 @@ static int dce_v11_0_resume(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int ret;
 
+	amdgpu_atombios_encoder_set_backlight_level_to_reg(adev,
+							   adev->mode_info.bl_level);
+
 	ret = dce_v11_0_hw_init(handle);
 
 	/* turn on the BL */
@@ -3759,13 +3768,16 @@ static const struct amdgpu_irq_src_funcs dce_v11_0_hpd_irq_funcs = {
 
 static void dce_v11_0_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+	if (adev->mode_info.num_crtc > 0)
+		adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
+	else
+		adev->crtc_irq.num_types = 0;
 	adev->crtc_irq.funcs = &dce_v11_0_crtc_irq_funcs;
 
-	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
+	adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
 	adev->pageflip_irq.funcs = &dce_v11_0_pageflip_irq_funcs;
 
-	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
+	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
 	adev->hpd_irq.funcs = &dce_v11_0_hpd_irq_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
index a51e35f824a1..b8368f69ce1f 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c
@@ -2639,7 +2639,6 @@ static int dce_v6_0_early_init(void *handle)
 	adev->audio_endpt_wreg = &dce_v6_0_audio_endpt_wreg;
 
 	dce_v6_0_set_display_funcs(adev);
-	dce_v6_0_set_irq_funcs(adev);
 
 	adev->mode_info.num_crtc = dce_v6_0_get_num_crtc(adev);
 
@@ -2658,6 +2657,8 @@ static int dce_v6_0_early_init(void *handle)
 		return -EINVAL;
 	}
 
+	dce_v6_0_set_irq_funcs(adev);
+
 	return 0;
 }
 
@@ -2786,6 +2787,11 @@ static int dce_v6_0_hw_fini(void *handle)
 
 static int dce_v6_0_suspend(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->mode_info.bl_level =
+		amdgpu_atombios_encoder_get_backlight_level_from_reg(adev);
+
 	return dce_v6_0_hw_fini(handle);
 }
 
@@ -2794,6 +2800,9 @@ static int dce_v6_0_resume(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int ret;
 
+	amdgpu_atombios_encoder_set_backlight_level_to_reg(adev,
+							   adev->mode_info.bl_level);
+
 	ret = dce_v6_0_hw_init(handle);
 
 	/* turn on the BL */
@@ -3092,7 +3101,7 @@ static int dce_v6_0_hpd_irq(struct amdgpu_device *adev,
 		tmp |= DC_HPD1_INT_CONTROL__DC_HPD1_INT_ACK_MASK;
 		WREG32(mmDC_HPD1_INT_CONTROL + hpd_offsets[hpd], tmp);
 		schedule_work(&adev->hotplug_work);
-		DRM_INFO("IH: HPD%d\n", hpd + 1);
+		DRM_DEBUG("IH: HPD%d\n", hpd + 1);
 	}
 
 	return 0;
@@ -3441,13 +3450,16 @@ static const struct amdgpu_irq_src_funcs dce_v6_0_hpd_irq_funcs = {
 
 static void dce_v6_0_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+	if (adev->mode_info.num_crtc > 0)
+		adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
+	else
+		adev->crtc_irq.num_types = 0;
 	adev->crtc_irq.funcs = &dce_v6_0_crtc_irq_funcs;
 
-	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
+	adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
 	adev->pageflip_irq.funcs = &dce_v6_0_pageflip_irq_funcs;
 
-	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
+	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
 	adev->hpd_irq.funcs = &dce_v6_0_hpd_irq_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 9cf14b8b2db9..012e0a9ae0ff 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -2664,7 +2664,6 @@ static int dce_v8_0_early_init(void *handle)
 	adev->audio_endpt_wreg = &dce_v8_0_audio_endpt_wreg;
 
 	dce_v8_0_set_display_funcs(adev);
-	dce_v8_0_set_irq_funcs(adev);
 
 	adev->mode_info.num_crtc = dce_v8_0_get_num_crtc(adev);
 
@@ -2688,6 +2687,8 @@ static int dce_v8_0_early_init(void *handle)
 		return -EINVAL;
 	}
 
+	dce_v8_0_set_irq_funcs(adev);
+
 	return 0;
 }
 
@@ -2818,6 +2819,11 @@ static int dce_v8_0_hw_fini(void *handle)
 
 static int dce_v8_0_suspend(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+	adev->mode_info.bl_level =
+		amdgpu_atombios_encoder_get_backlight_level_from_reg(adev);
+
 	return dce_v8_0_hw_fini(handle);
 }
 
@@ -2826,6 +2832,9 @@ static int dce_v8_0_resume(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int ret;
 
+	amdgpu_atombios_encoder_set_backlight_level_to_reg(adev,
+							   adev->mode_info.bl_level);
+
 	ret = dce_v8_0_hw_init(handle);
 
 	/* turn on the BL */
@@ -3525,13 +3534,16 @@ static const struct amdgpu_irq_src_funcs dce_v8_0_hpd_irq_funcs = {
 
 static void dce_v8_0_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+	if (adev->mode_info.num_crtc > 0)
+		adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_VLINE1 + adev->mode_info.num_crtc;
+	else
+		adev->crtc_irq.num_types = 0;
 	adev->crtc_irq.funcs = &dce_v8_0_crtc_irq_funcs;
 
-	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
+	adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
 	adev->pageflip_irq.funcs = &dce_v8_0_pageflip_irq_funcs;
 
-	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
+	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
 	adev->hpd_irq.funcs = &dce_v8_0_hpd_irq_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
index a8829af120c1..120dd3b26fc2 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c
@@ -44,6 +44,9 @@ static void dce_virtual_set_display_funcs(struct amdgpu_device *adev);
 static void dce_virtual_set_irq_funcs(struct amdgpu_device *adev);
 static int dce_virtual_connector_encoder_init(struct amdgpu_device *adev,
 					      int index);
+static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev,
+							int crtc,
+							enum amdgpu_interrupt_state state);
 
 /**
  * dce_virtual_vblank_wait - vblank wait asic callback.
@@ -437,6 +440,8 @@ static int dce_virtual_sw_fini(void *handle)
 	drm_kms_helper_poll_fini(adev->ddev);
 
 	drm_mode_config_cleanup(adev->ddev);
+	/* clear crtcs pointer to avoid dce irq finish routine access freed data */
+	memset(adev->mode_info.crtcs, 0, sizeof(adev->mode_info.crtcs[0]) * AMDGPU_MAX_CRTCS);
 	adev->mode_info.mode_config_initialized = false;
 	return 0;
 }
@@ -489,6 +494,13 @@ static int dce_virtual_hw_init(void *handle)
 
 static int dce_virtual_hw_fini(void *handle)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int i = 0;
+
+	for (i = 0; i<adev->mode_info.num_crtc; i++)
+		if (adev->mode_info.crtcs[i])
+			dce_virtual_set_crtc_vblank_interrupt_state(adev, i, AMDGPU_IRQ_STATE_DISABLE);
+
 	return 0;
 }
 
@@ -723,7 +735,7 @@ static void dce_virtual_set_crtc_vblank_interrupt_state(struct amdgpu_device *ad
 							int crtc,
 							enum amdgpu_interrupt_state state)
 {
-	if (crtc >= adev->mode_info.num_crtc) {
+	if (crtc >= adev->mode_info.num_crtc || !adev->mode_info.crtcs[crtc]) {
 		DRM_DEBUG("invalid crtc %d\n", crtc);
 		return;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index dbbe986f90f2..9870d83b68c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -1798,7 +1798,7 @@ static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 		DRM_UDELAY(1);
 	}
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
 			  ring->idx, scratch, tmp);
@@ -1874,7 +1874,7 @@ static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 
 static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
 				  struct amdgpu_ib *ib,
-				  unsigned vm_id, bool ctx_switch)
+				  unsigned vmid, bool ctx_switch)
 {
 	u32 header, control = 0;
 
@@ -1889,7 +1889,7 @@ static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
 	else
 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
 
-	control |= ib->length_dw | (vm_id << 24);
+	control |= ib->length_dw | (vmid << 24);
 
 	amdgpu_ring_write(ring, header);
 	amdgpu_ring_write(ring,
@@ -1951,7 +1951,7 @@ static int gfx_v6_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = RREG32(scratch);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -2354,7 +2354,7 @@ static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 }
 
 static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					unsigned vm_id, uint64_t pd_addr)
+					unsigned vmid, uint64_t pd_addr)
 {
 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 
@@ -2362,10 +2362,10 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
 				 WRITE_DATA_DST_SEL(0)));
-	if (vm_id < 8) {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id ));
+	if (vmid < 8) {
+		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid ));
 	} else {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vm_id - 8)));
+		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vmid - 8)));
 	}
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, pd_addr >> 12);
@@ -2376,7 +2376,7 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 				 WRITE_DATA_DST_SEL(0)));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
 	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 
 	/* wait for the invalidate to complete */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
@@ -2962,25 +2962,7 @@ static void gfx_v6_0_get_csb_buffer(struct amdgpu_device *adev,
 
 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
 	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
-
-	switch (adev->asic_type) {
-	case CHIP_TAHITI:
-	case CHIP_PITCAIRN:
-		buffer[count++] = cpu_to_le32(0x2a00126a);
-		break;
-	case CHIP_VERDE:
-		buffer[count++] = cpu_to_le32(0x0000124a);
-		break;
-	case CHIP_OLAND:
-		buffer[count++] = cpu_to_le32(0x00000082);
-		break;
-	case CHIP_HAINAN:
-		buffer[count++] = cpu_to_le32(0x00000000);
-		break;
-	default:
-		buffer[count++] = cpu_to_le32(0x00000000);
-		break;
-	}
+	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
 
 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 419ba0ce7ee5..a4309698e76c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -48,6 +48,8 @@
 #include "oss/oss_2_0_d.h"
 #include "oss/oss_2_0_sh_mask.h"
 
+#define NUM_SIMD_PER_CU 0x4 /* missing from the gfx_7 IP headers */
+
 #define GFX7_NUM_GFX_RINGS     1
 #define GFX7_MEC_HPD_SIZE      2048
 
@@ -2085,7 +2087,7 @@ static int gfx_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 		DRM_UDELAY(1);
 	}
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
 			  ring->idx, scratch, tmp);
@@ -2252,7 +2254,7 @@ static void gfx_v7_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
  */
 static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 				      struct amdgpu_ib *ib,
-				      unsigned vm_id, bool ctx_switch)
+				      unsigned vmid, bool ctx_switch)
 {
 	u32 header, control = 0;
 
@@ -2267,7 +2269,7 @@ static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 	else
 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
 
-	control |= ib->length_dw | (vm_id << 24);
+	control |= ib->length_dw | (vmid << 24);
 
 	amdgpu_ring_write(ring, header);
 	amdgpu_ring_write(ring,
@@ -2281,9 +2283,9 @@ static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 
 static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
 					  struct amdgpu_ib *ib,
-					  unsigned vm_id, bool ctx_switch)
+					  unsigned vmid, bool ctx_switch)
 {
-	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
+	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
 	amdgpu_ring_write(ring,
@@ -2365,7 +2367,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = RREG32(scratch);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -2551,29 +2553,8 @@ static int gfx_v7_0_cp_gfx_start(struct amdgpu_device *adev)
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
 	amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
-	switch (adev->asic_type) {
-	case CHIP_BONAIRE:
-		amdgpu_ring_write(ring, 0x16000012);
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	case CHIP_KAVERI:
-		amdgpu_ring_write(ring, 0x00000000); /* XXX */
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	case CHIP_KABINI:
-	case CHIP_MULLINS:
-		amdgpu_ring_write(ring, 0x00000000); /* XXX */
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	case CHIP_HAWAII:
-		amdgpu_ring_write(ring, 0x3a00161a);
-		amdgpu_ring_write(ring, 0x0000002e);
-		break;
-	default:
-		amdgpu_ring_write(ring, 0x00000000);
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	}
+	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
+	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
@@ -3258,19 +3239,19 @@ static void gfx_v7_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
  * using the CP (CIK).
  */
 static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					unsigned vm_id, uint64_t pd_addr)
+					unsigned vmid, uint64_t pd_addr)
 {
 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
 				 WRITE_DATA_DST_SEL(0)));
-	if (vm_id < 8) {
+	if (vmid < 8) {
 		amdgpu_ring_write(ring,
-				  (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
+				  (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid));
 	} else {
 		amdgpu_ring_write(ring,
-				  (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));
+				  (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8));
 	}
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, pd_addr >> 12);
@@ -3281,7 +3262,7 @@ static void gfx_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 				 WRITE_DATA_DST_SEL(0)));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
 	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 
 	/* wait for the invalidate to complete */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
@@ -4403,34 +4384,8 @@ static void gfx_v7_0_gpu_early_init(struct amdgpu_device *adev)
 	case CHIP_KAVERI:
 		adev->gfx.config.max_shader_engines = 1;
 		adev->gfx.config.max_tile_pipes = 4;
-		if ((adev->pdev->device == 0x1304) ||
-		    (adev->pdev->device == 0x1305) ||
-		    (adev->pdev->device == 0x130C) ||
-		    (adev->pdev->device == 0x130F) ||
-		    (adev->pdev->device == 0x1310) ||
-		    (adev->pdev->device == 0x1311) ||
-		    (adev->pdev->device == 0x131C)) {
-			adev->gfx.config.max_cu_per_sh = 8;
-			adev->gfx.config.max_backends_per_se = 2;
-		} else if ((adev->pdev->device == 0x1309) ||
-			   (adev->pdev->device == 0x130A) ||
-			   (adev->pdev->device == 0x130D) ||
-			   (adev->pdev->device == 0x1313) ||
-			   (adev->pdev->device == 0x131D)) {
-			adev->gfx.config.max_cu_per_sh = 6;
-			adev->gfx.config.max_backends_per_se = 2;
-		} else if ((adev->pdev->device == 0x1306) ||
-			   (adev->pdev->device == 0x1307) ||
-			   (adev->pdev->device == 0x130B) ||
-			   (adev->pdev->device == 0x130E) ||
-			   (adev->pdev->device == 0x1315) ||
-			   (adev->pdev->device == 0x131B)) {
-			adev->gfx.config.max_cu_per_sh = 4;
-			adev->gfx.config.max_backends_per_se = 1;
-		} else {
-			adev->gfx.config.max_cu_per_sh = 3;
-			adev->gfx.config.max_backends_per_se = 1;
-		}
+		adev->gfx.config.max_cu_per_sh = 8;
+		adev->gfx.config.max_backends_per_se = 2;
 		adev->gfx.config.max_sh_per_se = 1;
 		adev->gfx.config.max_texture_channel_caches = 4;
 		adev->gfx.config.max_gprs = 256;
@@ -5298,6 +5253,11 @@ static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev)
 
 	cu_info->number = active_cu_number;
 	cu_info->ao_cu_mask = ao_cu_mask;
+	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
+	cu_info->max_waves_per_simd = 10;
+	cu_info->max_scratch_slots_per_cu = 32;
+	cu_info->wave_front_size = 64;
+	cu_info->lds_size = 64;
 }
 
 const struct amdgpu_ip_block_version gfx_v7_0_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 9ecdf621a74a..4e694ae9f308 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -679,55 +679,55 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
-		amdgpu_program_register_sequence(adev,
-						 iceland_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_iceland_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
-		amdgpu_program_register_sequence(adev,
-						 iceland_golden_common_all,
-						 (const u32)ARRAY_SIZE(iceland_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							iceland_mgcg_cgcg_init,
+							ARRAY_SIZE(iceland_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_iceland_a11,
+							ARRAY_SIZE(golden_settings_iceland_a11));
+		amdgpu_device_program_register_sequence(adev,
+							iceland_golden_common_all,
+							ARRAY_SIZE(iceland_golden_common_all));
 		break;
 	case CHIP_FIJI:
-		amdgpu_program_register_sequence(adev,
-						 fiji_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_fiji_a10,
-						 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
-		amdgpu_program_register_sequence(adev,
-						 fiji_golden_common_all,
-						 (const u32)ARRAY_SIZE(fiji_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							fiji_mgcg_cgcg_init,
+							ARRAY_SIZE(fiji_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_fiji_a10,
+							ARRAY_SIZE(golden_settings_fiji_a10));
+		amdgpu_device_program_register_sequence(adev,
+							fiji_golden_common_all,
+							ARRAY_SIZE(fiji_golden_common_all));
 		break;
 
 	case CHIP_TONGA:
-		amdgpu_program_register_sequence(adev,
-						 tonga_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_tonga_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
-		amdgpu_program_register_sequence(adev,
-						 tonga_golden_common_all,
-						 (const u32)ARRAY_SIZE(tonga_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							tonga_mgcg_cgcg_init,
+							ARRAY_SIZE(tonga_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_tonga_a11,
+							ARRAY_SIZE(golden_settings_tonga_a11));
+		amdgpu_device_program_register_sequence(adev,
+							tonga_golden_common_all,
+							ARRAY_SIZE(tonga_golden_common_all));
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_polaris11_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11));
-		amdgpu_program_register_sequence(adev,
-						 polaris11_golden_common_all,
-						 (const u32)ARRAY_SIZE(polaris11_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_polaris11_a11,
+							ARRAY_SIZE(golden_settings_polaris11_a11));
+		amdgpu_device_program_register_sequence(adev,
+							polaris11_golden_common_all,
+							ARRAY_SIZE(polaris11_golden_common_all));
 		break;
 	case CHIP_POLARIS10:
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_polaris10_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11));
-		amdgpu_program_register_sequence(adev,
-						 polaris10_golden_common_all,
-						 (const u32)ARRAY_SIZE(polaris10_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_polaris10_a11,
+							ARRAY_SIZE(golden_settings_polaris10_a11));
+		amdgpu_device_program_register_sequence(adev,
+							polaris10_golden_common_all,
+							ARRAY_SIZE(polaris10_golden_common_all));
 		WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C);
 		if (adev->pdev->revision == 0xc7 &&
 		    ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
@@ -738,26 +738,26 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 		}
 		break;
 	case CHIP_CARRIZO:
-		amdgpu_program_register_sequence(adev,
-						 cz_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 cz_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(cz_golden_settings_a11));
-		amdgpu_program_register_sequence(adev,
-						 cz_golden_common_all,
-						 (const u32)ARRAY_SIZE(cz_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							cz_mgcg_cgcg_init,
+							ARRAY_SIZE(cz_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							cz_golden_settings_a11,
+							ARRAY_SIZE(cz_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							cz_golden_common_all,
+							ARRAY_SIZE(cz_golden_common_all));
 		break;
 	case CHIP_STONEY:
-		amdgpu_program_register_sequence(adev,
-						 stoney_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 stoney_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(stoney_golden_settings_a11));
-		amdgpu_program_register_sequence(adev,
-						 stoney_golden_common_all,
-						 (const u32)ARRAY_SIZE(stoney_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_mgcg_cgcg_init,
+							ARRAY_SIZE(stoney_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_golden_settings_a11,
+							ARRAY_SIZE(stoney_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_golden_common_all,
+							ARRAY_SIZE(stoney_golden_common_all));
 		break;
 	default:
 		break;
@@ -804,7 +804,7 @@ static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
 		DRM_UDELAY(1);
 	}
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
@@ -856,7 +856,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = RREG32(scratch);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -2114,7 +2114,6 @@ static int gfx_v8_0_sw_fini(void *handle)
 	amdgpu_gfx_compute_mqd_sw_fini(adev);
 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
 	amdgpu_gfx_kiq_fini(adev);
-	amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
 
 	gfx_v8_0_mec_fini(adev);
 	gfx_v8_0_rlc_fini(adev);
@@ -3851,6 +3850,14 @@ static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
 					break;
 				udelay(1);
 			}
+			if (k == adev->usec_timeout) {
+				gfx_v8_0_select_se_sh(adev, 0xffffffff,
+						      0xffffffff, 0xffffffff);
+				mutex_unlock(&adev->grbm_idx_mutex);
+				DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
+					 i, j);
+				return;
+			}
 		}
 	}
 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
@@ -4305,37 +4312,8 @@ static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
 	amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
-	switch (adev->asic_type) {
-	case CHIP_TONGA:
-	case CHIP_POLARIS10:
-		amdgpu_ring_write(ring, 0x16000012);
-		amdgpu_ring_write(ring, 0x0000002A);
-		break;
-	case CHIP_POLARIS11:
-	case CHIP_POLARIS12:
-		amdgpu_ring_write(ring, 0x16000012);
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	case CHIP_FIJI:
-		amdgpu_ring_write(ring, 0x3a00161a);
-		amdgpu_ring_write(ring, 0x0000002e);
-		break;
-	case CHIP_CARRIZO:
-		amdgpu_ring_write(ring, 0x00000002);
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	case CHIP_TOPAZ:
-		amdgpu_ring_write(ring, adev->gfx.config.num_rbs == 1 ?
-				0x00000000 : 0x00000002);
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	case CHIP_STONEY:
-		amdgpu_ring_write(ring, 0x00000000);
-		amdgpu_ring_write(ring, 0x00000000);
-		break;
-	default:
-		BUG();
-	}
+	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
+	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
@@ -4816,7 +4794,7 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
 
 	gfx_v8_0_kiq_setting(ring);
 
-	if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
@@ -4853,7 +4831,7 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
 	struct vi_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-	if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
+	if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
 		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
 		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -4865,13 +4843,10 @@ static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
 
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
-	} else if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	} else if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
-		/* reset ring buffer */
-		ring->wptr = 0;
-		amdgpu_ring_clear_ring(ring);
 	} else {
 		amdgpu_ring_clear_ring(ring);
 	}
@@ -4946,6 +4921,13 @@ static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
 	/* Test KCQs */
 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
 		ring = &adev->gfx.compute_ring[i];
+		if (adev->in_gpu_reset) {
+			/* move reset ring buffer to here to workaround
+			 * compute ring test failed
+			 */
+			ring->wptr = 0;
+			amdgpu_ring_clear_ring(ring);
+		}
 		ring->ready = true;
 		r = amdgpu_ring_test_ring(ring);
 		if (r)
@@ -5080,8 +5062,9 @@ static int gfx_v8_0_hw_fini(void *handle)
 	gfx_v8_0_cp_enable(adev, false);
 	gfx_v8_0_rlc_stop(adev);
 
-	amdgpu_set_powergating_state(adev,
-			AMD_IP_BLOCK_TYPE_GFX, AMD_PG_STATE_UNGATE);
+	amdgpu_device_ip_set_powergating_state(adev,
+					       AMD_IP_BLOCK_TYPE_GFX,
+					       AMD_PG_STATE_UNGATE);
 
 	return 0;
 }
@@ -5498,8 +5481,9 @@ static int gfx_v8_0_late_init(void *handle)
 	if (r)
 		return r;
 
-	amdgpu_set_powergating_state(adev,
-			AMD_IP_BLOCK_TYPE_GFX, AMD_PG_STATE_GATE);
+	amdgpu_device_ip_set_powergating_state(adev,
+					       AMD_IP_BLOCK_TYPE_GFX,
+					       AMD_PG_STATE_GATE);
 
 	return 0;
 }
@@ -5510,10 +5494,10 @@ static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *ade
 	if ((adev->asic_type == CHIP_POLARIS11) ||
 	    (adev->asic_type == CHIP_POLARIS12))
 		/* Send msg to SMU via Powerplay */
-		amdgpu_set_powergating_state(adev,
-					     AMD_IP_BLOCK_TYPE_SMC,
-					     enable ?
-					     AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE);
+		amdgpu_device_ip_set_powergating_state(adev,
+						       AMD_IP_BLOCK_TYPE_SMC,
+						       enable ?
+						       AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE);
 
 	WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
 }
@@ -6261,7 +6245,7 @@ static void gfx_v8_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 
 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 				      struct amdgpu_ib *ib,
-				      unsigned vm_id, bool ctx_switch)
+				      unsigned vmid, bool ctx_switch)
 {
 	u32 header, control = 0;
 
@@ -6270,7 +6254,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 	else
 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
 
-	control |= ib->length_dw | (vm_id << 24);
+	control |= ib->length_dw | (vmid << 24);
 
 	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
@@ -6291,9 +6275,9 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 
 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
 					  struct amdgpu_ib *ib,
-					  unsigned vm_id, bool ctx_switch)
+					  unsigned vmid, bool ctx_switch)
 {
-	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
+	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
 	amdgpu_ring_write(ring,
@@ -6344,7 +6328,7 @@ static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 }
 
 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					unsigned vm_id, uint64_t pd_addr)
+					unsigned vmid, uint64_t pd_addr)
 {
 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
 
@@ -6352,12 +6336,12 @@ static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
 				 WRITE_DATA_DST_SEL(0)) |
 				 WR_CONFIRM);
-	if (vm_id < 8) {
+	if (vmid < 8) {
 		amdgpu_ring_write(ring,
-				  (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
+				  (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid));
 	} else {
 		amdgpu_ring_write(ring,
-				  (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));
+				  (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8));
 	}
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, pd_addr >> 12);
@@ -6369,7 +6353,7 @@ static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 				 WRITE_DATA_DST_SEL(0)));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
 	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 
 	/* wait for the invalidate to complete */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
@@ -6490,10 +6474,10 @@ static void gfx_v8_0_hqd_set_priority(struct amdgpu_device *adev,
 	mutex_unlock(&adev->srbm_mutex);
 }
 static void gfx_v8_0_ring_set_priority_compute(struct amdgpu_ring *ring,
-					       enum amd_sched_priority priority)
+					       enum drm_sched_priority priority)
 {
 	struct amdgpu_device *adev = ring->adev;
-	bool acquire = priority == AMD_SCHED_PRIORITY_HIGH_HW;
+	bool acquire = priority == DRM_SCHED_PRIORITY_HIGH_HW;
 
 	if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
 		return;
@@ -7132,6 +7116,11 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
 
 	cu_info->number = active_cu_number;
 	cu_info->ao_cu_mask = ao_cu_mask;
+	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
+	cu_info->max_waves_per_simd = 10;
+	cu_info->max_scratch_slots_per_cu = 32;
+	cu_info->wave_front_size = 64;
+	cu_info->lds_size = 64;
 }
 
 const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 5aeb5f8816f3..c06479615e8a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -28,11 +28,10 @@
 #include "soc15.h"
 #include "soc15d.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/GC/gc_9_0_offset.h"
-#include "vega10/GC/gc_9_0_sh_mask.h"
-#include "vega10/vega10_enum.h"
-#include "vega10/HDP/hdp_4_0_offset.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "vega10_enum.h"
+#include "hdp/hdp_4_0_offset.h"
 
 #include "soc15_common.h"
 #include "clearstate_gfx9.h"
@@ -65,152 +64,84 @@ MODULE_FIRMWARE("amdgpu/raven_mec.bin");
 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
 
-static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
-{
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID1_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID1),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID1) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID2_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID2),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID2) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID3_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID3),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID3) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID4_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID4),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID4) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID5_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID5),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID5) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID6_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID6),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID6) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID7_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID7),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID7) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID8_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID8),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID8) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID9_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID9),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID9) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID10_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID10),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID10) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID11_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID11),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID11) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID12_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID12),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID12)},
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID13_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID13),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID13) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID14_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID14),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID14) },
-	{ SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_BASE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_VMID15_SIZE),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID15),
-	  SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID15) }
+static const struct soc15_reg_golden golden_settings_gc_9_0[] =
+{
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 };
 
-static const u32 golden_settings_gc_9_0[] =
-{
-	SOC15_REG_OFFSET(GC, 0, mmCPC_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmCPF_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmCPG_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2), 0xf00fffff, 0x00000420,
-	SOC15_REG_OFFSET(GC, 0, mmGB_GPU_ID), 0x0000000f, 0x00000000,
-	SOC15_REG_OFFSET(GC, 0, mmIA_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3), 0x00000003, 0x82400024,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE), 0x3fffffff, 0x00000001,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_LINE_STIPPLE_STATE), 0x0000ff0f, 0x00000000,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UTCL1_CNTL_0), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UTCL1_CNTL_1), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UTCL1_CNTL_2), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG), 0x00001000, 0x00001000,
-	SOC15_REG_OFFSET(GC, 0, mmSPI_CONFIG_CNTL_1), 0x0000000f, 0x01000107,
-	SOC15_REG_OFFSET(GC, 0, mmSQC_CONFIG), 0x03000000, 0x020a2000,
-	SOC15_REG_OFFSET(GC, 0, mmTA_CNTL_AUX), 0xfffffeef, 0x010b0000,
-	SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_HI), 0xffffffff, 0x4a2c0e68,
-	SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_LO), 0xffffffff, 0xb5d3f197,
-	SOC15_REG_OFFSET(GC, 0, mmVGT_CACHE_INVALIDATION), 0x3fff3af3, 0x19200000,
-	SOC15_REG_OFFSET(GC, 0, mmVGT_GS_MAX_WAVE_ID), 0x00000fff, 0x000003ff,
-	SOC15_REG_OFFSET(GC, 0, mmWD_UTCL1_CNTL), 0x08000000, 0x08000080
-};
-
-static const u32 golden_settings_gc_9_0_vg10[] =
+static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
 {
-	SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL), 0x0000f000, 0x00012107,
-	SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL_3), 0x30000000, 0x10000000,
-	SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG), 0xffff77ff, 0x2a114042,
-	SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG_READ), 0xffff77ff, 0x2a114042,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE_1), 0x00008000, 0x00048000,
-	SOC15_REG_OFFSET(GC, 0, mmRMI_UTCL1_CNTL2), 0x00030000, 0x00020000,
-	SOC15_REG_OFFSET(GC, 0, mmTD_CNTL), 0x00001800, 0x00000800
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800)
 };
 
-static const u32 golden_settings_gc_9_1[] =
-{
-	SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL), 0xfffdf3cf, 0x00014104,
-	SOC15_REG_OFFSET(GC, 0, mmCPC_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmCPF_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmCPG_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2), 0xf00fffff, 0x00000420,
-	SOC15_REG_OFFSET(GC, 0, mmGB_GPU_ID), 0x0000000f, 0x00000000,
-	SOC15_REG_OFFSET(GC, 0, mmIA_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3), 0x00000003, 0x82400024,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE), 0x3fffffff, 0x00000001,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_LINE_STIPPLE_STATE), 0x0000ff0f, 0x00000000,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UTCL1_CNTL_0), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UTCL1_CNTL_1), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_UTCL1_CNTL_2), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_UTCL1_CNTL), 0x08000000, 0x08000080,
-	SOC15_REG_OFFSET(GC, 0, mmTA_CNTL_AUX), 0xfffffeef, 0x010b0000,
-	SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_HI), 0xffffffff, 0x00000000,
-	SOC15_REG_OFFSET(GC, 0, mmTCP_CHAN_STEER_LO), 0xffffffff, 0x00003120,
-	SOC15_REG_OFFSET(GC, 0, mmVGT_CACHE_INVALIDATION), 0x3fff3af3, 0x19200000,
-	SOC15_REG_OFFSET(GC, 0, mmVGT_GS_MAX_WAVE_ID), 0x00000fff, 0x000000ff,
-	SOC15_REG_OFFSET(GC, 0, mmWD_UTCL1_CNTL), 0x08000000, 0x08000080
+static const struct soc15_reg_golden golden_settings_gc_9_1[] =
+{
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
 };
 
-static const u32 golden_settings_gc_9_1_rv1[] =
+static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
 {
-	SOC15_REG_OFFSET(GC, 0, mmCB_HW_CONTROL_3), 0x30000000, 0x10000000,
-	SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG), 0xffff77ff, 0x24000042,
-	SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG_READ), 0xffff77ff, 0x24000042,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE_1), 0xffffffff, 0x04048000,
-	SOC15_REG_OFFSET(GC, 0, mmPA_SC_MODE_CNTL_1), 0x06000000, 0x06000000,
-	SOC15_REG_OFFSET(GC, 0, mmRMI_UTCL1_CNTL2), 0x00030000, 0x00020000,
-	SOC15_REG_OFFSET(GC, 0, mmTD_CNTL), 0x01bd9f33, 0x00000800
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
 };
 
-static const u32 golden_settings_gc_9_x_common[] =
+static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
 {
-	SOC15_REG_OFFSET(GC, 0, mmGRBM_CAM_INDEX), 0xffffffff, 0x00000000,
-	SOC15_REG_OFFSET(GC, 0, mmGRBM_CAM_DATA), 0xffffffff, 0x2544c382
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
+	SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
 };
 
 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
@@ -230,26 +161,26 @@ static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
-		amdgpu_program_register_sequence(adev,
+		soc15_program_register_sequence(adev,
 						 golden_settings_gc_9_0,
-						 (const u32)ARRAY_SIZE(golden_settings_gc_9_0));
-		amdgpu_program_register_sequence(adev,
+						 ARRAY_SIZE(golden_settings_gc_9_0));
+		soc15_program_register_sequence(adev,
 						 golden_settings_gc_9_0_vg10,
-						 (const u32)ARRAY_SIZE(golden_settings_gc_9_0_vg10));
+						 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
 		break;
 	case CHIP_RAVEN:
-		amdgpu_program_register_sequence(adev,
+		soc15_program_register_sequence(adev,
 						 golden_settings_gc_9_1,
-						 (const u32)ARRAY_SIZE(golden_settings_gc_9_1));
-		amdgpu_program_register_sequence(adev,
+						 ARRAY_SIZE(golden_settings_gc_9_1));
+		soc15_program_register_sequence(adev,
 						 golden_settings_gc_9_1_rv1,
-						 (const u32)ARRAY_SIZE(golden_settings_gc_9_1_rv1));
+						 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
 		break;
 	default:
 		break;
 	}
 
-	amdgpu_program_register_sequence(adev, golden_settings_gc_9_x_common,
+	soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
 					(const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
 }
 
@@ -327,7 +258,7 @@ static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
 		DRM_UDELAY(1);
 	}
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
@@ -379,7 +310,7 @@ static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
         }
         tmp = RREG32(scratch);
         if (tmp == 0xDEADBEEF) {
-                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
                 r = 0;
         } else {
                 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
@@ -1137,8 +1068,8 @@ static int gfx_v9_0_ngg_init(struct amdgpu_device *adev)
 	adev->gfx.ngg.gds_reserve_size = ALIGN(5 * 4, 0x40);
 	adev->gds.mem.total_size -= adev->gfx.ngg.gds_reserve_size;
 	adev->gds.mem.gfx_partition_size -= adev->gfx.ngg.gds_reserve_size;
-	adev->gfx.ngg.gds_reserve_addr = amdgpu_gds_reg_offset[0].mem_base;
-	adev->gfx.ngg.gds_reserve_addr += adev->gds.mem.gfx_partition_size;
+	adev->gfx.ngg.gds_reserve_addr = RREG32_SOC15(GC, 0, mmGDS_VMID0_BASE);
+	adev->gfx.ngg.gds_reserve_addr += RREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE);
 
 	/* Primitive Buffer */
 	r = gfx_v9_0_ngg_create_buf(adev, &adev->gfx.ngg.buf[NGG_PRIM],
@@ -1243,23 +1174,24 @@ static int gfx_v9_0_ngg_en(struct amdgpu_device *adev)
 	}
 
 	gfx_v9_0_write_data_to_reg(ring, 0, false,
-				   amdgpu_gds_reg_offset[0].mem_size,
+				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE),
 			           (adev->gds.mem.total_size +
 				    adev->gfx.ngg.gds_reserve_size) >>
 				   AMDGPU_GDS_SHIFT);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
 	amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
+				PACKET3_DMA_DATA_DST_SEL(1) |
 				PACKET3_DMA_DATA_SRC_SEL(2)));
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, adev->gfx.ngg.gds_reserve_addr);
 	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, adev->gfx.ngg.gds_reserve_size);
-
+	amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
+				adev->gfx.ngg.gds_reserve_size);
 
 	gfx_v9_0_write_data_to_reg(ring, 0, false,
-				   amdgpu_gds_reg_offset[0].mem_size, 0);
+				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE), 0);
 
 	amdgpu_ring_commit(ring);
 
@@ -1464,7 +1396,6 @@ static int gfx_v9_0_sw_fini(void *handle)
 	amdgpu_gfx_compute_mqd_sw_fini(adev);
 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
 	amdgpu_gfx_kiq_fini(adev);
-	amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
 
 	gfx_v9_0_mec_fini(adev);
 	gfx_v9_0_ngg_fini(adev);
@@ -1596,14 +1527,21 @@ static void gfx_v9_0_gpu_init(struct amdgpu_device *adev)
 	/* XXX SH_MEM regs */
 	/* where to put LDS, scratch, GPUVM in FSA64 space */
 	mutex_lock(&adev->srbm_mutex);
-	for (i = 0; i < 16; i++) {
+	for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids; i++) {
 		soc15_grbm_select(adev, 0, 0, 0, i);
 		/* CP and shaders */
-		tmp = 0;
-		tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
-				    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
-		WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
-		WREG32_SOC15(GC, 0, mmSH_MEM_BASES, 0);
+		if (i == 0) {
+			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
+					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
+			WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
+			WREG32_SOC15(GC, 0, mmSH_MEM_BASES, 0);
+		} else {
+			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
+					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
+			WREG32_SOC15(GC, 0, mmSH_MEM_CONFIG, tmp);
+			tmp = adev->mc.shared_aperture_start >> 48;
+			WREG32_SOC15(GC, 0, mmSH_MEM_BASES, tmp);
+		}
 	}
 	soc15_grbm_select(adev, 0, 0, 0, 0);
 
@@ -1645,6 +1583,14 @@ static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
 					break;
 				udelay(1);
 			}
+			if (k == adev->usec_timeout) {
+				gfx_v9_0_select_se_sh(adev, 0xffffffff,
+						      0xffffffff, 0xffffffff);
+				mutex_unlock(&adev->grbm_idx_mutex);
+				DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
+					 i, j);
+				return;
+			}
 		}
 	}
 	gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
@@ -2749,7 +2695,7 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
 
 	gfx_v9_0_kiq_setting(ring);
 
-	if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
@@ -2787,7 +2733,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
 	struct v9_mqd *mqd = ring->mqd_ptr;
 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
 
-	if (!adev->in_sriov_reset && !adev->gfx.in_suspend) {
+	if (!adev->in_gpu_reset && !adev->gfx.in_suspend) {
 		memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
 		((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
 		((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
@@ -2799,7 +2745,7 @@ static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
 
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
-	} else if (adev->in_sriov_reset) { /* for GPU_RESET case */
+	} else if (adev->in_gpu_reset) { /* for GPU_RESET case */
 		/* reset MQD to a clean status */
 		if (adev->gfx.mec.mqd_backup[mqd_idx])
 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
@@ -3139,6 +3085,8 @@ static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
 					  uint32_t gws_base, uint32_t gws_size,
 					  uint32_t oa_base, uint32_t oa_size)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	gds_base = gds_base >> AMDGPU_GDS_SHIFT;
 	gds_size = gds_size >> AMDGPU_GDS_SHIFT;
 
@@ -3150,22 +3098,22 @@ static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
 
 	/* GDS Base */
 	gfx_v9_0_write_data_to_reg(ring, 0, false,
-				   amdgpu_gds_reg_offset[vmid].mem_base,
+				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
 				   gds_base);
 
 	/* GDS Size */
 	gfx_v9_0_write_data_to_reg(ring, 0, false,
-				   amdgpu_gds_reg_offset[vmid].mem_size,
+				   SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
 				   gds_size);
 
 	/* GWS */
 	gfx_v9_0_write_data_to_reg(ring, 0, false,
-				   amdgpu_gds_reg_offset[vmid].gws,
+				   SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
 				   gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
 
 	/* OA */
 	gfx_v9_0_write_data_to_reg(ring, 0, false,
-				   amdgpu_gds_reg_offset[vmid].oa,
+				   SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
 				   (1 << (oa_size + oa_base)) - (1 << oa_base));
 }
 
@@ -3610,13 +3558,9 @@ static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
 
 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
 	u32 ref_and_mask, reg_mem_engine;
-	const struct nbio_hdp_flush_reg *nbio_hf_reg;
-
-	if (ring->adev->flags & AMD_IS_APU)
-		nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
-	else
-		nbio_hf_reg = &nbio_v6_1_hdp_flush_reg;
+	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio_funcs->hdp_flush_reg;
 
 	if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
 		switch (ring->me) {
@@ -3636,20 +3580,22 @@ static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	}
 
 	gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
-			      nbio_hf_reg->hdp_flush_req_offset,
-			      nbio_hf_reg->hdp_flush_done_offset,
+			      adev->nbio_funcs->get_hdp_flush_req_offset(adev),
+			      adev->nbio_funcs->get_hdp_flush_done_offset(adev),
 			      ref_and_mask, ref_and_mask, 0x20);
 }
 
 static void gfx_v9_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	gfx_v9_0_write_data_to_reg(ring, 0, true,
 				   SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
 }
 
 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
                                       struct amdgpu_ib *ib,
-                                      unsigned vm_id, bool ctx_switch)
+                                      unsigned vmid, bool ctx_switch)
 {
 	u32 header, control = 0;
 
@@ -3658,7 +3604,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 	else
 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
 
-	control |= ib->length_dw | (vm_id << 24);
+	control |= ib->length_dw | (vmid << 24);
 
 	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
@@ -3680,9 +3626,9 @@ BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
 
 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
                                           struct amdgpu_ib *ib,
-                                          unsigned vm_id, bool ctx_switch)
+                                          unsigned vmid, bool ctx_switch)
 {
-        u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
+        u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
 
         amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
 	BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
@@ -3738,22 +3684,23 @@ static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
 }
 
 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					unsigned vm_id, uint64_t pd_addr)
+					unsigned vmid, uint64_t pd_addr)
 {
 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
-	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
+	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
+	uint64_t flags = AMDGPU_PTE_VALID;
 	unsigned eng = ring->vm_inv_eng;
 
-	pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
-	pd_addr |= AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
+	pd_addr |= flags;
 
 	gfx_v9_0_write_data_to_reg(ring, usepfp, true,
-				   hub->ctx0_ptb_addr_lo32 + (2 * vm_id),
+				   hub->ctx0_ptb_addr_lo32 + (2 * vmid),
 				   lower_32_bits(pd_addr));
 
 	gfx_v9_0_write_data_to_reg(ring, usepfp, true,
-				   hub->ctx0_ptb_addr_hi32 + (2 * vm_id),
+				   hub->ctx0_ptb_addr_hi32 + (2 * vmid),
 				   upper_32_bits(pd_addr));
 
 	gfx_v9_0_write_data_to_reg(ring, usepfp, true,
@@ -3761,7 +3708,7 @@ static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 
 	/* wait for the invalidate to complete */
 	gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, hub->vm_inv_eng0_ack +
-			      eng, 0, 1 << vm_id, 1 << vm_id, 0x20);
+			      eng, 0, 1 << vmid, 1 << vmid, 0x20);
 
 	/* compute doesn't have PFP */
 	if (usepfp) {
@@ -3804,6 +3751,8 @@ static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
 					 u64 seq, unsigned int flags)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	/* we only allocate 32bit for each seq wb address */
 	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index c17996e18086..56f5fe4e2fee 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -23,11 +23,10 @@
 #include "amdgpu.h"
 #include "gfxhub_v1_0.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/GC/gc_9_0_offset.h"
-#include "vega10/GC/gc_9_0_sh_mask.h"
-#include "vega10/GC/gc_9_0_default.h"
-#include "vega10/vega10_enum.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "gc/gc_9_0_default.h"
+#include "vega10_enum.h"
 
 #include "soc15_common.h"
 
@@ -144,8 +143,15 @@ static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL2, tmp);
 
 	tmp = mmVM_L2_CNTL3_DEFAULT;
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+	if (adev->mc.translate_further) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+	} else {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+	}
 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL3, tmp);
 
 	tmp = mmVM_L2_CNTL4_DEFAULT;
@@ -183,31 +189,40 @@ static void gfxhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
 
 static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 {
-	int i;
+	unsigned num_level, block_size;
 	uint32_t tmp;
+	int i;
+
+	num_level = adev->vm_manager.num_level;
+	block_size = adev->vm_manager.block_size;
+	if (adev->mc.translate_further)
+		num_level -= 1;
+	else
+		block_size -= 9;
 
 	for (i = 0; i <= 14; i++) {
 		tmp = RREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
-				    adev->vm_manager.num_level);
+				    num_level);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
+				    1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				PAGE_TABLE_BLOCK_SIZE,
-				adev->vm_manager.block_size - 9);
+				    PAGE_TABLE_BLOCK_SIZE,
+				    block_size);
 		/* Send no-retry XNACK on fault to suppress VM fault storm. */
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index f4603a7c8ef3..8e28270d1ea9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -222,13 +222,8 @@ static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
 	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
 	base <<= 24;
 
-	if (mc->mc_vram_size > 0xFFC0000000ULL) {
-		dev_warn(adev->dev, "limiting VRAM\n");
-		mc->real_vram_size = 0xFFC0000000ULL;
-		mc->mc_vram_size = 0xFFC0000000ULL;
-	}
-	amdgpu_vram_location(adev, &adev->mc, base);
-	amdgpu_gart_location(adev, mc);
+	amdgpu_device_vram_location(adev, &adev->mc, base);
+	amdgpu_device_gart_location(adev, mc);
 }
 
 static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
@@ -283,6 +278,7 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 
 	u32 tmp;
 	int chansize, numchan;
+	int r;
 
 	tmp = RREG32(mmMC_ARB_RAMCFG);
 	if (tmp & (1 << 11)) {
@@ -324,12 +320,17 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 		break;
 	}
 	adev->mc.vram_width = numchan * chansize;
-	/* Could aper size report 0 ? */
-	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
-	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
 	/* size in MB on si */
 	adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
+
+	if (!(adev->flags & AMD_IS_APU)) {
+		r = amdgpu_device_resize_fb_bar(adev);
+		if (r)
+			return r;
+	}
+	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
+	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
 	adev->mc.visible_vram_size = adev->mc.aper_size;
 
 	/* set the gart size */
@@ -394,10 +395,10 @@ static uint64_t gmc_v6_0_get_vm_pte_flags(struct amdgpu_device *adev,
 	return pte_flag;
 }
 
-static uint64_t gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
+static void gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, int level,
+				uint64_t *addr, uint64_t *flags)
 {
-	BUG_ON(addr & 0xFFFFFF0000000FFFULL);
-	return addr;
+	BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
 }
 
 static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,
@@ -831,8 +832,7 @@ static int gmc_v6_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	amdgpu_vm_adjust_size(adev, 64, 9);
-	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
+	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
 
 	adev->mc.mc_mask = 0xffffffffffULL;
 
@@ -877,7 +877,6 @@ static int gmc_v6_0_sw_init(void *handle)
 	 * amdkfd will use VMIDs 8-15
 	 */
 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
-	adev->vm_manager.num_level = 1;
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
@@ -897,9 +896,9 @@ static int gmc_v6_0_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
 	gmc_v6_0_gart_fini(adev);
-	amdgpu_gem_force_release(adev);
 	amdgpu_bo_fini(adev);
 	release_firmware(adev->mc.fw);
 	adev->mc.fw = NULL;
@@ -957,7 +956,7 @@ static int gmc_v6_0_resume(void *handle)
 	if (r)
 		return r;
 
-	amdgpu_vm_reset_all_ids(adev);
+	amdgpu_vmid_reset_all(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index b0528ca9207b..86e9d682c59e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -67,12 +67,12 @@ static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
-		amdgpu_program_register_sequence(adev,
-						 iceland_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_iceland_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
+		amdgpu_device_program_register_sequence(adev,
+							iceland_mgcg_cgcg_init,
+							ARRAY_SIZE(iceland_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_iceland_a11,
+							ARRAY_SIZE(golden_settings_iceland_a11));
 		break;
 	default:
 		break;
@@ -240,14 +240,8 @@ static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
 	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
 	base <<= 24;
 
-	if (mc->mc_vram_size > 0xFFC0000000ULL) {
-		/* leave room for at least 1024M GTT */
-		dev_warn(adev->dev, "limiting VRAM\n");
-		mc->real_vram_size = 0xFFC0000000ULL;
-		mc->mc_vram_size = 0xFFC0000000ULL;
-	}
-	amdgpu_vram_location(adev, &adev->mc, base);
-	amdgpu_gart_location(adev, mc);
+	amdgpu_device_vram_location(adev, &adev->mc, base);
+	amdgpu_device_gart_location(adev, mc);
 }
 
 /**
@@ -322,6 +316,8 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
  */
 static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 {
+	int r;
+
 	adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
 	if (!adev->mc.vram_width) {
 		u32 tmp;
@@ -367,13 +363,18 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 		}
 		adev->mc.vram_width = numchan * chansize;
 	}
-	/* Could aper size report 0 ? */
-	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
-	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
 	/* size in MB on si */
 	adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 
+	if (!(adev->flags & AMD_IS_APU)) {
+		r = amdgpu_device_resize_fb_bar(adev);
+		if (r)
+			return r;
+	}
+	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
+	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
+
 #ifdef CONFIG_X86_64
 	if (adev->flags & AMD_IS_APU) {
 		adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
@@ -479,10 +480,10 @@ static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,
 	return pte_flag;
 }
 
-static uint64_t gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
+static void gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, int level,
+				uint64_t *addr, uint64_t *flags)
 {
-	BUG_ON(addr & 0xFFFFFF0000000FFFULL);
-	return addr;
+	BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
 }
 
 /**
@@ -970,8 +971,7 @@ static int gmc_v7_0_sw_init(void *handle)
 	 * Currently set to 4GB ((1 << 20) 4k pages).
 	 * Max GPUVM size for cayman and SI is 40 bits.
 	 */
-	amdgpu_vm_adjust_size(adev, 64, 9);
-	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
+	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
 
 	/* Set the internal MC address mask
 	 * This is the max address of the GPU's
@@ -1026,7 +1026,6 @@ static int gmc_v7_0_sw_init(void *handle)
 	 * amdkfd will use VMIDs 8-15
 	 */
 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
-	adev->vm_manager.num_level = 1;
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
@@ -1046,9 +1045,9 @@ static int gmc_v7_0_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
 	gmc_v7_0_gart_fini(adev);
-	amdgpu_gem_force_release(adev);
 	amdgpu_bo_fini(adev);
 	release_firmware(adev->mc.fw);
 	adev->mc.fw = NULL;
@@ -1108,7 +1107,7 @@ static int gmc_v7_0_resume(void *handle)
 	if (r)
 		return r;
 
-	amdgpu_vm_reset_all_ids(adev);
+	amdgpu_vmid_reset_all(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index f368cfe2f585..9a813d834f1a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -120,44 +120,44 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_FIJI:
-		amdgpu_program_register_sequence(adev,
-						 fiji_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_fiji_a10,
-						 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
+		amdgpu_device_program_register_sequence(adev,
+							fiji_mgcg_cgcg_init,
+							ARRAY_SIZE(fiji_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_fiji_a10,
+							ARRAY_SIZE(golden_settings_fiji_a10));
 		break;
 	case CHIP_TONGA:
-		amdgpu_program_register_sequence(adev,
-						 tonga_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_tonga_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
+		amdgpu_device_program_register_sequence(adev,
+							tonga_mgcg_cgcg_init,
+							ARRAY_SIZE(tonga_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_tonga_a11,
+							ARRAY_SIZE(golden_settings_tonga_a11));
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_polaris11_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_polaris11_a11,
+							ARRAY_SIZE(golden_settings_polaris11_a11));
 		break;
 	case CHIP_POLARIS10:
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_polaris10_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_polaris10_a11,
+							ARRAY_SIZE(golden_settings_polaris10_a11));
 		break;
 	case CHIP_CARRIZO:
-		amdgpu_program_register_sequence(adev,
-						 cz_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							cz_mgcg_cgcg_init,
+							ARRAY_SIZE(cz_mgcg_cgcg_init));
 		break;
 	case CHIP_STONEY:
-		amdgpu_program_register_sequence(adev,
-						 stoney_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_stoney_common,
-						 (const u32)ARRAY_SIZE(golden_settings_stoney_common));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_mgcg_cgcg_init,
+							ARRAY_SIZE(stoney_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_stoney_common,
+							ARRAY_SIZE(golden_settings_stoney_common));
 		break;
 	default:
 		break;
@@ -405,14 +405,8 @@ static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
 		base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
 	base <<= 24;
 
-	if (mc->mc_vram_size > 0xFFC0000000ULL) {
-		/* leave room for at least 1024M GTT */
-		dev_warn(adev->dev, "limiting VRAM\n");
-		mc->real_vram_size = 0xFFC0000000ULL;
-		mc->mc_vram_size = 0xFFC0000000ULL;
-	}
-	amdgpu_vram_location(adev, &adev->mc, base);
-	amdgpu_gart_location(adev, mc);
+	amdgpu_device_vram_location(adev, &adev->mc, base);
+	amdgpu_device_gart_location(adev, mc);
 }
 
 /**
@@ -498,6 +492,8 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
  */
 static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 {
+	int r;
+
 	adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
 	if (!adev->mc.vram_width) {
 		u32 tmp;
@@ -543,13 +539,18 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 		}
 		adev->mc.vram_width = numchan * chansize;
 	}
-	/* Could aper size report 0 ? */
-	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
-	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
 	/* size in MB on si */
 	adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
 
+	if (!(adev->flags & AMD_IS_APU)) {
+		r = amdgpu_device_resize_fb_bar(adev);
+		if (r)
+			return r;
+	}
+	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
+	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
+
 #ifdef CONFIG_X86_64
 	if (adev->flags & AMD_IS_APU) {
 		adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
@@ -676,10 +677,10 @@ static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device *adev,
 	return pte_flag;
 }
 
-static uint64_t gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
+static void gmc_v8_0_get_vm_pde(struct amdgpu_device *adev, int level,
+				uint64_t *addr, uint64_t *flags)
 {
-	BUG_ON(addr & 0xFFFFFF0000000FFFULL);
-	return addr;
+	BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
 }
 
 /**
@@ -1067,8 +1068,7 @@ static int gmc_v8_0_sw_init(void *handle)
 	 * Currently set to 4GB ((1 << 20) 4k pages).
 	 * Max GPUVM size for cayman and SI is 40 bits.
 	 */
-	amdgpu_vm_adjust_size(adev, 64, 9);
-	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
+	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
 
 	/* Set the internal MC address mask
 	 * This is the max address of the GPU's
@@ -1123,7 +1123,6 @@ static int gmc_v8_0_sw_init(void *handle)
 	 * amdkfd will use VMIDs 8-15
 	 */
 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
-	adev->vm_manager.num_level = 1;
 	amdgpu_vm_manager_init(adev);
 
 	/* base offset of vram pages */
@@ -1143,9 +1142,9 @@ static int gmc_v8_0_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
 	gmc_v8_0_gart_fini(adev);
-	amdgpu_gem_force_release(adev);
 	amdgpu_bo_fini(adev);
 	release_firmware(adev->mc.fw);
 	adev->mc.fw = NULL;
@@ -1213,7 +1212,7 @@ static int gmc_v8_0_resume(void *handle)
 	if (r)
 		return r;
 
-	amdgpu_vm_reset_all_ids(adev);
+	amdgpu_vmid_reset_all(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index c8f1aebeac7a..3b7e7af09ead 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -25,20 +25,19 @@
 #include "gmc_v9_0.h"
 #include "amdgpu_atomfirmware.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/HDP/hdp_4_0_offset.h"
-#include "vega10/HDP/hdp_4_0_sh_mask.h"
-#include "vega10/GC/gc_9_0_sh_mask.h"
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/vega10_enum.h"
-#include "vega10/MMHUB/mmhub_1_0_offset.h"
-#include "vega10/ATHUB/athub_1_0_offset.h"
-
+#include "hdp/hdp_4_0_offset.h"
+#include "hdp/hdp_4_0_sh_mask.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "vega10_enum.h"
+#include "mmhub/mmhub_1_0_offset.h"
+#include "athub/athub_1_0_offset.h"
+
+#include "soc15.h"
 #include "soc15_common.h"
+#include "umc/umc_6_0_sh_mask.h"
 
-#include "nbio_v6_1.h"
-#include "nbio_v7_0.h"
 #include "gfxhub_v1_0.h"
 #include "mmhub_v1_0.h"
 
@@ -73,16 +72,131 @@ static const u32 golden_settings_vega10_hdp[] =
 	0xf6e, 0x0fffffff, 0x00000000,
 };
 
-static const u32 golden_settings_mmhub_1_0_0[] =
+static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
 {
-	SOC15_REG_OFFSET(MMHUB, 0, mmDAGB1_WRCLI2), 0x00000007, 0xfe5fe0fa,
-	SOC15_REG_OFFSET(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0), 0x00000030, 0x55555565
+	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
+	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
 };
 
-static const u32 golden_settings_athub_1_0_0[] =
+static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
 {
-	SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL), 0x0000ff00, 0x00000800,
-	SOC15_REG_OFFSET(ATHUB, 0, mmRPB_ARB_CNTL2), 0x00ff00ff, 0x00080008
+	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
+	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
+};
+
+/* Ecc related register addresses, (BASE + reg offset) */
+/* Universal Memory Controller caps (may be fused). */
+/* UMCCH:UmcLocalCap */
+#define UMCLOCALCAPS_ADDR0	(0x00014306 + 0x00000000)
+#define UMCLOCALCAPS_ADDR1	(0x00014306 + 0x00000800)
+#define UMCLOCALCAPS_ADDR2	(0x00014306 + 0x00001000)
+#define UMCLOCALCAPS_ADDR3	(0x00014306 + 0x00001800)
+#define UMCLOCALCAPS_ADDR4	(0x00054306 + 0x00000000)
+#define UMCLOCALCAPS_ADDR5	(0x00054306 + 0x00000800)
+#define UMCLOCALCAPS_ADDR6	(0x00054306 + 0x00001000)
+#define UMCLOCALCAPS_ADDR7	(0x00054306 + 0x00001800)
+#define UMCLOCALCAPS_ADDR8	(0x00094306 + 0x00000000)
+#define UMCLOCALCAPS_ADDR9	(0x00094306 + 0x00000800)
+#define UMCLOCALCAPS_ADDR10	(0x00094306 + 0x00001000)
+#define UMCLOCALCAPS_ADDR11	(0x00094306 + 0x00001800)
+#define UMCLOCALCAPS_ADDR12	(0x000d4306 + 0x00000000)
+#define UMCLOCALCAPS_ADDR13	(0x000d4306 + 0x00000800)
+#define UMCLOCALCAPS_ADDR14	(0x000d4306 + 0x00001000)
+#define UMCLOCALCAPS_ADDR15	(0x000d4306 + 0x00001800)
+
+/* Universal Memory Controller Channel config. */
+/* UMCCH:UMC_CONFIG */
+#define UMCCH_UMC_CONFIG_ADDR0	(0x00014040 + 0x00000000)
+#define UMCCH_UMC_CONFIG_ADDR1	(0x00014040 + 0x00000800)
+#define UMCCH_UMC_CONFIG_ADDR2	(0x00014040 + 0x00001000)
+#define UMCCH_UMC_CONFIG_ADDR3	(0x00014040 + 0x00001800)
+#define UMCCH_UMC_CONFIG_ADDR4	(0x00054040 + 0x00000000)
+#define UMCCH_UMC_CONFIG_ADDR5	(0x00054040 + 0x00000800)
+#define UMCCH_UMC_CONFIG_ADDR6	(0x00054040 + 0x00001000)
+#define UMCCH_UMC_CONFIG_ADDR7	(0x00054040 + 0x00001800)
+#define UMCCH_UMC_CONFIG_ADDR8	(0x00094040 + 0x00000000)
+#define UMCCH_UMC_CONFIG_ADDR9	(0x00094040 + 0x00000800)
+#define UMCCH_UMC_CONFIG_ADDR10	(0x00094040 + 0x00001000)
+#define UMCCH_UMC_CONFIG_ADDR11	(0x00094040 + 0x00001800)
+#define UMCCH_UMC_CONFIG_ADDR12	(0x000d4040 + 0x00000000)
+#define UMCCH_UMC_CONFIG_ADDR13	(0x000d4040 + 0x00000800)
+#define UMCCH_UMC_CONFIG_ADDR14	(0x000d4040 + 0x00001000)
+#define UMCCH_UMC_CONFIG_ADDR15	(0x000d4040 + 0x00001800)
+
+/* Universal Memory Controller Channel Ecc config. */
+/* UMCCH:EccCtrl */
+#define UMCCH_ECCCTRL_ADDR0	(0x00014053 + 0x00000000)
+#define UMCCH_ECCCTRL_ADDR1	(0x00014053 + 0x00000800)
+#define UMCCH_ECCCTRL_ADDR2	(0x00014053 + 0x00001000)
+#define UMCCH_ECCCTRL_ADDR3	(0x00014053 + 0x00001800)
+#define UMCCH_ECCCTRL_ADDR4	(0x00054053 + 0x00000000)
+#define UMCCH_ECCCTRL_ADDR5	(0x00054053 + 0x00000800)
+#define UMCCH_ECCCTRL_ADDR6	(0x00054053 + 0x00001000)
+#define UMCCH_ECCCTRL_ADDR7	(0x00054053 + 0x00001800)
+#define UMCCH_ECCCTRL_ADDR8	(0x00094053 + 0x00000000)
+#define UMCCH_ECCCTRL_ADDR9	(0x00094053 + 0x00000800)
+#define UMCCH_ECCCTRL_ADDR10	(0x00094053 + 0x00001000)
+#define UMCCH_ECCCTRL_ADDR11	(0x00094053 + 0x00001800)
+#define UMCCH_ECCCTRL_ADDR12	(0x000d4053 + 0x00000000)
+#define UMCCH_ECCCTRL_ADDR13	(0x000d4053 + 0x00000800)
+#define UMCCH_ECCCTRL_ADDR14	(0x000d4053 + 0x00001000)
+#define UMCCH_ECCCTRL_ADDR15	(0x000d4053 + 0x00001800)
+
+static const uint32_t ecc_umclocalcap_addrs[] = {
+	UMCLOCALCAPS_ADDR0,
+	UMCLOCALCAPS_ADDR1,
+	UMCLOCALCAPS_ADDR2,
+	UMCLOCALCAPS_ADDR3,
+	UMCLOCALCAPS_ADDR4,
+	UMCLOCALCAPS_ADDR5,
+	UMCLOCALCAPS_ADDR6,
+	UMCLOCALCAPS_ADDR7,
+	UMCLOCALCAPS_ADDR8,
+	UMCLOCALCAPS_ADDR9,
+	UMCLOCALCAPS_ADDR10,
+	UMCLOCALCAPS_ADDR11,
+	UMCLOCALCAPS_ADDR12,
+	UMCLOCALCAPS_ADDR13,
+	UMCLOCALCAPS_ADDR14,
+	UMCLOCALCAPS_ADDR15,
+};
+
+static const uint32_t ecc_umcch_umc_config_addrs[] = {
+	UMCCH_UMC_CONFIG_ADDR0,
+	UMCCH_UMC_CONFIG_ADDR1,
+	UMCCH_UMC_CONFIG_ADDR2,
+	UMCCH_UMC_CONFIG_ADDR3,
+	UMCCH_UMC_CONFIG_ADDR4,
+	UMCCH_UMC_CONFIG_ADDR5,
+	UMCCH_UMC_CONFIG_ADDR6,
+	UMCCH_UMC_CONFIG_ADDR7,
+	UMCCH_UMC_CONFIG_ADDR8,
+	UMCCH_UMC_CONFIG_ADDR9,
+	UMCCH_UMC_CONFIG_ADDR10,
+	UMCCH_UMC_CONFIG_ADDR11,
+	UMCCH_UMC_CONFIG_ADDR12,
+	UMCCH_UMC_CONFIG_ADDR13,
+	UMCCH_UMC_CONFIG_ADDR14,
+	UMCCH_UMC_CONFIG_ADDR15,
+};
+
+static const uint32_t ecc_umcch_eccctrl_addrs[] = {
+	UMCCH_ECCCTRL_ADDR0,
+	UMCCH_ECCCTRL_ADDR1,
+	UMCCH_ECCCTRL_ADDR2,
+	UMCCH_ECCCTRL_ADDR3,
+	UMCCH_ECCCTRL_ADDR4,
+	UMCCH_ECCCTRL_ADDR5,
+	UMCCH_ECCCTRL_ADDR6,
+	UMCCH_ECCCTRL_ADDR7,
+	UMCCH_ECCCTRL_ADDR8,
+	UMCCH_ECCCTRL_ADDR9,
+	UMCCH_ECCCTRL_ADDR10,
+	UMCCH_ECCCTRL_ADDR11,
+	UMCCH_ECCCTRL_ADDR12,
+	UMCCH_ECCCTRL_ADDR13,
+	UMCCH_ECCCTRL_ADDR14,
+	UMCCH_ECCCTRL_ADDR15,
 };
 
 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
@@ -134,7 +248,7 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 				struct amdgpu_irq_src *source,
 				struct amdgpu_iv_entry *entry)
 {
-	struct amdgpu_vmhub *hub = &adev->vmhub[entry->vm_id_src];
+	struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
 	uint32_t status = 0;
 	u64 addr;
 
@@ -148,9 +262,9 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 
 	if (printk_ratelimit()) {
 		dev_err(adev->dev,
-			"[%s] VMC page fault (src_id:%u ring:%u vm_id:%u pas_id:%u)\n",
-			entry->vm_id_src ? "mmhub" : "gfxhub",
-			entry->src_id, entry->ring_id, entry->vm_id,
+			"[%s] VMC page fault (src_id:%u ring:%u vmid:%u pas_id:%u)\n",
+			entry->vmid_src ? "mmhub" : "gfxhub",
+			entry->src_id, entry->ring_id, entry->vmid,
 			entry->pas_id);
 		dev_err(adev->dev, "  at page 0x%016llx from %d\n",
 			addr, entry->client_id);
@@ -174,13 +288,13 @@ static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
 	adev->mc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
 }
 
-static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vm_id)
+static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
 {
 	u32 req = 0;
 
-	/* invalidate using legacy mode on vm_id*/
+	/* invalidate using legacy mode on vmid*/
 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
-			    PER_VMID_INVALIDATE_REQ, 1 << vm_id);
+			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
@@ -216,10 +330,7 @@ static void gmc_v9_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
 	unsigned i, j;
 
 	/* flush hdp cache */
-	if (adev->flags & AMD_IS_APU)
-		nbio_v7_0_hdp_flush(adev);
-	else
-		nbio_v6_1_hdp_flush(adev);
+	adev->nbio_funcs->hdp_flush(adev);
 
 	spin_lock(&adev->mc.invalidate_lock);
 
@@ -358,11 +469,28 @@ static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,
 	return pte_flag;
 }
 
-static u64 gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, u64 addr)
+static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
+				uint64_t *addr, uint64_t *flags)
 {
-	addr = adev->vm_manager.vram_base_offset + addr - adev->mc.vram_start;
-	BUG_ON(addr & 0xFFFF00000000003FULL);
-	return addr;
+	if (!(*flags & AMDGPU_PDE_PTE))
+		*addr = adev->vm_manager.vram_base_offset + *addr -
+			adev->mc.vram_start;
+	BUG_ON(*addr & 0xFFFF00000000003FULL);
+
+	if (!adev->mc.translate_further)
+		return;
+
+	if (level == AMDGPU_VM_PDB1) {
+		/* Set the block fragment size */
+		if (!(*flags & AMDGPU_PDE_PTE))
+			*flags |= AMDGPU_PDE_BFS(0x9);
+
+	} else if (level == AMDGPU_VM_PDB0) {
+		if (*flags & AMDGPU_PDE_PTE)
+			*flags &= ~AMDGPU_PDE_PTE;
+		else
+			*flags |= AMDGPU_PTE_TF;
+	}
 }
 
 static const struct amdgpu_gart_funcs gmc_v9_0_gart_funcs = {
@@ -386,9 +514,96 @@ static int gmc_v9_0_early_init(void *handle)
 	gmc_v9_0_set_gart_funcs(adev);
 	gmc_v9_0_set_irq_funcs(adev);
 
+	adev->mc.shared_aperture_start = 0x2000000000000000ULL;
+	adev->mc.shared_aperture_end =
+		adev->mc.shared_aperture_start + (4ULL << 30) - 1;
+	adev->mc.private_aperture_start =
+		adev->mc.shared_aperture_end + 1;
+	adev->mc.private_aperture_end =
+		adev->mc.private_aperture_start + (4ULL << 30) - 1;
+
 	return 0;
 }
 
+static int gmc_v9_0_ecc_available(struct amdgpu_device *adev)
+{
+	uint32_t reg_val;
+	uint32_t reg_addr;
+	uint32_t field_val;
+	size_t i;
+	uint32_t fv2;
+	size_t lost_sheep;
+
+	DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n");
+
+	lost_sheep = 0;
+	for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) {
+		reg_addr = ecc_umclocalcap_addrs[i];
+		DRM_DEBUG("ecc: "
+			  "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n",
+			  i, reg_addr);
+		reg_val = RREG32(reg_addr);
+		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap,
+					  EccDis);
+		DRM_DEBUG("ecc: "
+			  "reg_val: 0x%08x, "
+			  "EccDis: 0x%08x, ",
+			  reg_val, field_val);
+		if (field_val) {
+			DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n");
+			++lost_sheep;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) {
+		reg_addr = ecc_umcch_umc_config_addrs[i];
+		DRM_DEBUG("ecc: "
+			  "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x",
+			  i, reg_addr);
+		reg_val = RREG32(reg_addr);
+		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG,
+					  DramReady);
+		DRM_DEBUG("ecc: "
+			  "reg_val: 0x%08x, "
+			  "DramReady: 0x%08x\n",
+			  reg_val, field_val);
+
+		if (!field_val) {
+			DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n");
+			++lost_sheep;
+		}
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) {
+		reg_addr = ecc_umcch_eccctrl_addrs[i];
+		DRM_DEBUG("ecc: "
+			  "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ",
+			  i, reg_addr);
+		reg_val = RREG32(reg_addr);
+		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
+					  WrEccEn);
+		fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
+				    RdEccEn);
+		DRM_DEBUG("ecc: "
+			  "reg_val: 0x%08x, "
+			  "WrEccEn: 0x%08x, "
+			  "RdEccEn: 0x%08x\n",
+			  reg_val, field_val, fv2);
+
+		if (!field_val) {
+			DRM_DEBUG("ecc: WrEccEn is not set\n");
+			++lost_sheep;
+		}
+		if (!fv2) {
+			DRM_DEBUG("ecc: RdEccEn is not set\n");
+			++lost_sheep;
+		}
+	}
+
+	DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep);
+	return lost_sheep == 0;
+}
+
 static int gmc_v9_0_late_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -403,6 +618,7 @@ static int gmc_v9_0_late_init(void *handle)
 	 */
 	unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
 	unsigned i;
+	int r;
 
 	for(i = 0; i < adev->num_rings; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
@@ -418,6 +634,18 @@ static int gmc_v9_0_late_init(void *handle)
 	for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
 		BUG_ON(vm_inv_eng[i] > 16);
 
+	if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) {
+		r = gmc_v9_0_ecc_available(adev);
+		if (r == 1) {
+			DRM_INFO("ECC is active.\n");
+		} else if (r == 0) {
+			DRM_INFO("ECC is not present.\n");
+		} else {
+			DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r);
+			return r;
+		}
+	}
+
 	return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
 }
 
@@ -427,8 +655,8 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
 	u64 base = 0;
 	if (!amdgpu_sriov_vf(adev))
 		base = mmhub_v1_0_get_fb_location(adev);
-	amdgpu_vram_location(adev, &adev->mc, base);
-	amdgpu_gart_location(adev, mc);
+	amdgpu_device_vram_location(adev, &adev->mc, base);
+	amdgpu_device_gart_location(adev, mc);
 	/* base offset of vram pages */
 	if (adev->flags & AMD_IS_APU)
 		adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
@@ -449,11 +677,15 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 {
 	u32 tmp;
 	int chansize, numchan;
+	int r;
 
 	adev->mc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
 	if (!adev->mc.vram_width) {
 		/* hbm memory channel size */
-		chansize = 128;
+		if (adev->flags & AMD_IS_APU)
+			chansize = 64;
+		else
+			chansize = 128;
 
 		tmp = RREG32_SOC15(DF, 0, mmDF_CS_AON0_DramBaseAddress0);
 		tmp &= DF_CS_AON0_DramBaseAddress0__IntLvNumChan_MASK;
@@ -491,17 +723,21 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 		adev->mc.vram_width = numchan * chansize;
 	}
 
-	/* Could aper size report 0 ? */
-	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
-	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
 	/* size in MB on si */
 	adev->mc.mc_vram_size =
-		((adev->flags & AMD_IS_APU) ? nbio_v7_0_get_memsize(adev) :
-		 nbio_v6_1_get_memsize(adev)) * 1024ULL * 1024ULL;
+		adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL;
 	adev->mc.real_vram_size = adev->mc.mc_vram_size;
-	adev->mc.visible_vram_size = adev->mc.aper_size;
+
+	if (!(adev->flags & AMD_IS_APU)) {
+		r = amdgpu_device_resize_fb_bar(adev);
+		if (r)
+			return r;
+	}
+	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
+	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
 
 	/* In case the PCI BAR is larger than the actual amount of vram */
+	adev->mc.visible_vram_size = adev->mc.aper_size;
 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
 
@@ -558,14 +794,12 @@ static int gmc_v9_0_sw_init(void *handle)
 	case CHIP_RAVEN:
 		adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
-			adev->vm_manager.vm_size = 1U << 18;
-			adev->vm_manager.block_size = 9;
-			adev->vm_manager.num_level = 3;
-			amdgpu_vm_set_fragment_size(adev, 9);
+			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
 		} else {
-			/* vm_size is 64GB for legacy 2-level page support */
-			amdgpu_vm_adjust_size(adev, 64, 9);
-			adev->vm_manager.num_level = 1;
+			/* vm_size is 128TB + 512GB for legacy 3-level page support */
+			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
+			adev->mc.translate_further =
+				adev->vm_manager.num_level > 1;
 		}
 		break;
 	case CHIP_VEGA10:
@@ -576,20 +810,12 @@ static int gmc_v9_0_sw_init(void *handle)
 		 * vm size is 256TB (48bit), maximum size of Vega10,
 		 * block size 512 (9bit)
 		 */
-		adev->vm_manager.vm_size = 1U << 18;
-		adev->vm_manager.block_size = 9;
-		adev->vm_manager.num_level = 3;
-		amdgpu_vm_set_fragment_size(adev, 9);
+		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
 		break;
 	default:
 		break;
 	}
 
-	DRM_INFO("vm size is %llu GB, block size is %u-bit,fragment size is %u-bit\n",
-			adev->vm_manager.vm_size,
-			adev->vm_manager.block_size,
-			adev->vm_manager.fragment_size);
-
 	/* This interrupt is VMC page fault.*/
 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_VMC, 0,
 				&adev->mc.vm_fault);
@@ -599,8 +825,6 @@ static int gmc_v9_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
-
 	/* Set the internal MC address mask
 	 * This is the max address of the GPU's
 	 * internal address space.
@@ -660,7 +884,7 @@ static int gmc_v9_0_sw_init(void *handle)
 }
 
 /**
- * gmc_v8_0_gart_fini - vm fini callback
+ * gmc_v9_0_gart_fini - vm fini callback
  *
  * @adev: amdgpu_device pointer
  *
@@ -676,9 +900,9 @@ static int gmc_v9_0_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
 	gmc_v9_0_gart_fini(adev);
-	amdgpu_gem_force_release(adev);
 	amdgpu_bo_fini(adev);
 
 	return 0;
@@ -686,19 +910,20 @@ static int gmc_v9_0_sw_fini(void *handle)
 
 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
 {
+
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
-		amdgpu_program_register_sequence(adev,
+		soc15_program_register_sequence(adev,
 						golden_settings_mmhub_1_0_0,
-						(const u32)ARRAY_SIZE(golden_settings_mmhub_1_0_0));
-		amdgpu_program_register_sequence(adev,
+						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
+		soc15_program_register_sequence(adev,
 						golden_settings_athub_1_0_0,
-						(const u32)ARRAY_SIZE(golden_settings_athub_1_0_0));
+						ARRAY_SIZE(golden_settings_athub_1_0_0));
 		break;
 	case CHIP_RAVEN:
-		amdgpu_program_register_sequence(adev,
+		soc15_program_register_sequence(adev,
 						golden_settings_athub_1_0_0,
-						(const u32)ARRAY_SIZE(golden_settings_athub_1_0_0));
+						ARRAY_SIZE(golden_settings_athub_1_0_0));
 		break;
 	default:
 		break;
@@ -716,9 +941,9 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
 	bool value;
 	u32 tmp;
 
-	amdgpu_program_register_sequence(adev,
-		golden_settings_vega10_hdp,
-		(const u32)ARRAY_SIZE(golden_settings_vega10_hdp));
+	amdgpu_device_program_register_sequence(adev,
+						golden_settings_vega10_hdp,
+						ARRAY_SIZE(golden_settings_vega10_hdp));
 
 	if (adev->gart.robj == NULL) {
 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
@@ -751,10 +976,7 @@ static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
 	WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
 
 	/* After HDP is initialized, flush HDP.*/
-	if (adev->flags & AMD_IS_APU)
-		nbio_v7_0_hdp_flush(adev);
-	else
-		nbio_v6_1_hdp_flush(adev);
+	adev->nbio_funcs->hdp_flush(adev);
 
 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
 		value = false;
@@ -839,7 +1061,7 @@ static int gmc_v9_0_resume(void *handle)
 	if (r)
 		return r;
 
-	amdgpu_vm_reset_all_ids(adev);
+	amdgpu_vmid_reset_all(adev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
index bd592cb39f37..c4e4be3dd31d 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c
@@ -259,7 +259,7 @@ static void iceland_ih_decode_iv(struct amdgpu_device *adev,
 	entry->src_id = dw[0] & 0xff;
 	entry->src_data[0] = dw[1] & 0xfffffff;
 	entry->ring_id = dw[2] & 0xff;
-	entry->vm_id = (dw[2] >> 8) & 0xff;
+	entry->vmid = (dw[2] >> 8) & 0xff;
 	entry->pas_id = (dw[2] >> 16) & 0xffff;
 
 	/* wptr/rptr are in bytes! */
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index f33d1ffdb20b..d9e9e52a0def 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -1682,8 +1682,8 @@ static void kv_dpm_powergate_uvd(void *handle, bool gate)
 
 	if (gate) {
 		/* stop the UVD block */
-		ret = amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							AMD_PG_STATE_GATE);
+		ret = amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+							     AMD_PG_STATE_GATE);
 		kv_update_uvd_dpm(adev, gate);
 		if (pi->caps_uvd_pg)
 			/* power off the UVD block */
@@ -1695,8 +1695,8 @@ static void kv_dpm_powergate_uvd(void *handle, bool gate)
 			/* re-init the UVD block */
 		kv_update_uvd_dpm(adev, gate);
 
-		ret = amdgpu_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
-							AMD_PG_STATE_UNGATE);
+		ret = amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_UVD,
+							     AMD_PG_STATE_UNGATE);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index cc21c4bdec27..ffd5b7ee49c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -23,14 +23,12 @@
 #include "amdgpu.h"
 #include "mmhub_v1_0.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/MMHUB/mmhub_1_0_offset.h"
-#include "vega10/MMHUB/mmhub_1_0_sh_mask.h"
-#include "vega10/MMHUB/mmhub_1_0_default.h"
-#include "vega10/ATHUB/athub_1_0_offset.h"
-#include "vega10/ATHUB/athub_1_0_sh_mask.h"
-#include "vega10/ATHUB/athub_1_0_default.h"
-#include "vega10/vega10_enum.h"
+#include "mmhub/mmhub_1_0_offset.h"
+#include "mmhub/mmhub_1_0_sh_mask.h"
+#include "mmhub/mmhub_1_0_default.h"
+#include "athub/athub_1_0_offset.h"
+#include "athub/athub_1_0_sh_mask.h"
+#include "vega10_enum.h"
 
 #include "soc15_common.h"
 
@@ -157,10 +155,15 @@ static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
 	WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2, tmp);
 
-	tmp = mmVM_L2_CNTL3_DEFAULT;
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
-	WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, tmp);
+	if (adev->mc.translate_further) {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+	} else {
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
+		tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+	}
 
 	tmp = mmVM_L2_CNTL4_DEFAULT;
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
@@ -198,32 +201,40 @@ static void mmhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
 
 static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 {
-	int i;
+	unsigned num_level, block_size;
 	uint32_t tmp;
+	int i;
+
+	num_level = adev->vm_manager.num_level;
+	block_size = adev->vm_manager.block_size;
+	if (adev->mc.translate_further)
+		num_level -= 1;
+	else
+		block_size -= 9;
 
 	for (i = 0; i <= 14; i++) {
 		tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
+		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
+				    num_level);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				ENABLE_CONTEXT, 1);
-		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				PAGE_TABLE_DEPTH, adev->vm_manager.num_level);
-		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
+				    1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+				    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				PAGE_TABLE_BLOCK_SIZE,
-				adev->vm_manager.block_size - 9);
+				    PAGE_TABLE_BLOCK_SIZE,
+				    block_size);
 		/* Send no-retry XNACK on fault to suppress VM fault storm. */
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index b4906d2f30d3..271452d3999a 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -22,11 +22,10 @@
  */
 
 #include "amdgpu.h"
-#include "vega10/soc15ip.h"
-#include "vega10/NBIO/nbio_6_1_offset.h"
-#include "vega10/NBIO/nbio_6_1_sh_mask.h"
-#include "vega10/GC/gc_9_0_offset.h"
-#include "vega10/GC/gc_9_0_sh_mask.h"
+#include "nbio/nbio_6_1_offset.h"
+#include "nbio/nbio_6_1_sh_mask.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
 #include "soc15.h"
 #include "vega10_ih.h"
 #include "soc15_common.h"
@@ -254,7 +253,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
 	}
 
 	/* Trigger recovery due to world switch failure */
-	amdgpu_sriov_gpu_reset(adev, NULL);
+	amdgpu_device_gpu_recover(adev, NULL, false);
 }
 
 static int xgpu_ai_set_mailbox_rcv_irq(struct amdgpu_device *adev,
@@ -278,13 +277,21 @@ static int xgpu_ai_mailbox_rcv_irq(struct amdgpu_device *adev,
 	int r;
 
 	/* trigger gpu-reset by hypervisor only if TDR disbaled */
-	if (amdgpu_lockup_timeout == 0) {
+	if (!amdgpu_gpu_recovery) {
 		/* see what event we get */
 		r = xgpu_ai_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
 
-		/* only handle FLR_NOTIFY now */
-		if (!r)
-			schedule_work(&adev->virt.flr_work);
+		/* sometimes the interrupt is delayed to inject to VM, so under such case
+		 * the IDH_FLR_NOTIFICATION is overwritten by VF FLR from GIM side, thus
+		 * above recieve message could be failed, we should schedule the flr_work
+		 * anyway
+		 */
+		if (r) {
+			DRM_ERROR("FLR_NOTIFICATION is missed\n");
+			xgpu_ai_mailbox_send_ack(adev);
+		}
+
+		schedule_work(&adev->virt.flr_work);
 	}
 
 	return 0;
@@ -353,5 +360,6 @@ const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
 	.req_full_gpu	= xgpu_ai_request_full_gpu_access,
 	.rel_full_gpu	= xgpu_ai_release_full_gpu_access,
 	.reset_gpu = xgpu_ai_request_reset,
+	.wait_reset = NULL,
 	.trans_msg = xgpu_ai_mailbox_trans_msg,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
index c25a831f94ec..9fc1c37344ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -279,32 +279,32 @@ void xgpu_vi_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_FIJI:
-		amdgpu_program_register_sequence(adev,
-						 xgpu_fiji_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(
-						 xgpu_fiji_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 xgpu_fiji_golden_settings_a10,
-						 (const u32)ARRAY_SIZE(
-						 xgpu_fiji_golden_settings_a10));
-		amdgpu_program_register_sequence(adev,
-						 xgpu_fiji_golden_common_all,
-						 (const u32)ARRAY_SIZE(
-						 xgpu_fiji_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							xgpu_fiji_mgcg_cgcg_init,
+							ARRAY_SIZE(
+								xgpu_fiji_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							xgpu_fiji_golden_settings_a10,
+							ARRAY_SIZE(
+								xgpu_fiji_golden_settings_a10));
+		amdgpu_device_program_register_sequence(adev,
+							xgpu_fiji_golden_common_all,
+							ARRAY_SIZE(
+								xgpu_fiji_golden_common_all));
 		break;
 	case CHIP_TONGA:
-		amdgpu_program_register_sequence(adev,
-						 xgpu_tonga_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(
-						 xgpu_tonga_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 xgpu_tonga_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(
-						 xgpu_tonga_golden_settings_a11));
-		amdgpu_program_register_sequence(adev,
-						 xgpu_tonga_golden_common_all,
-						 (const u32)ARRAY_SIZE(
-						 xgpu_tonga_golden_common_all));
+		amdgpu_device_program_register_sequence(adev,
+							xgpu_tonga_mgcg_cgcg_init,
+							ARRAY_SIZE(
+								xgpu_tonga_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							xgpu_tonga_golden_settings_a11,
+							ARRAY_SIZE(
+								xgpu_tonga_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							xgpu_tonga_golden_common_all,
+							ARRAY_SIZE(
+								xgpu_tonga_golden_common_all));
 		break;
 	default:
 		BUG_ON("Doesn't support chip type.\n");
@@ -446,8 +446,10 @@ static int xgpu_vi_send_access_requests(struct amdgpu_device *adev,
 		request == IDH_REQ_GPU_FINI_ACCESS ||
 		request == IDH_REQ_GPU_RESET_ACCESS) {
 		r = xgpu_vi_poll_msg(adev, IDH_READY_TO_ACCESS_GPU);
-		if (r)
-			pr_err("Doesn't get ack from pf, continue\n");
+		if (r) {
+			pr_err("Doesn't get ack from pf, give up\n");
+			return r;
+		}
 	}
 
 	return 0;
@@ -458,6 +460,11 @@ static int xgpu_vi_request_reset(struct amdgpu_device *adev)
 	return xgpu_vi_send_access_requests(adev, IDH_REQ_GPU_RESET_ACCESS);
 }
 
+static int xgpu_vi_wait_reset_cmpl(struct amdgpu_device *adev)
+{
+	return xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL);
+}
+
 static int xgpu_vi_request_full_gpu_access(struct amdgpu_device *adev,
 					   bool init)
 {
@@ -514,7 +521,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
 	}
 
 	/* Trigger recovery due to world switch failure */
-	amdgpu_sriov_gpu_reset(adev, NULL);
+	amdgpu_device_gpu_recover(adev, NULL, false);
 }
 
 static int xgpu_vi_set_mailbox_rcv_irq(struct amdgpu_device *adev,
@@ -538,7 +545,7 @@ static int xgpu_vi_mailbox_rcv_irq(struct amdgpu_device *adev,
 	int r;
 
 	/* trigger gpu-reset by hypervisor only if TDR disbaled */
-	if (amdgpu_lockup_timeout == 0) {
+	if (!amdgpu_gpu_recovery) {
 		/* see what event we get */
 		r = xgpu_vi_mailbox_rcv_msg(adev, IDH_FLR_NOTIFICATION);
 
@@ -613,5 +620,6 @@ const struct amdgpu_virt_ops xgpu_vi_virt_ops = {
 	.req_full_gpu		= xgpu_vi_request_full_gpu_access,
 	.rel_full_gpu		= xgpu_vi_release_full_gpu_access,
 	.reset_gpu		= xgpu_vi_request_reset,
+	.wait_reset             = xgpu_vi_wait_reset_cmpl,
 	.trans_msg		= NULL, /* Does not need to trans VF errors to host. */
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
index 904a1bab9b9f..d4da663d5eb0 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
@@ -24,17 +24,16 @@
 #include "amdgpu_atombios.h"
 #include "nbio_v6_1.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/NBIO/nbio_6_1_default.h"
-#include "vega10/NBIO/nbio_6_1_offset.h"
-#include "vega10/NBIO/nbio_6_1_sh_mask.h"
-#include "vega10/vega10_enum.h"
+#include "nbio/nbio_6_1_default.h"
+#include "nbio/nbio_6_1_offset.h"
+#include "nbio/nbio_6_1_sh_mask.h"
+#include "vega10_enum.h"
 
 #define smnCPM_CONTROL                                                                                  0x11180460
 #define smnPCIE_CNTL2                                                                                   0x11180070
 #define smnPCIE_CONFIG_CNTL                                                                             0x11180044
 
-u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev)
+static u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev)
 {
         u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
 
@@ -44,19 +43,7 @@ u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev)
 	return tmp;
 }
 
-u32 nbio_v6_1_get_atombios_scratch_regs(struct amdgpu_device *adev,
-					uint32_t idx)
-{
-	return RREG32_SOC15_OFFSET(NBIO, 0, mmBIOS_SCRATCH_0, idx);
-}
-
-void nbio_v6_1_set_atombios_scratch_regs(struct amdgpu_device *adev,
-					 uint32_t idx, uint32_t val)
-{
-	WREG32_SOC15_OFFSET(NBIO, 0, mmBIOS_SCRATCH_0, idx, val);
-}
-
-void nbio_v6_1_mc_access_enable(struct amdgpu_device *adev, bool enable)
+static void nbio_v6_1_mc_access_enable(struct amdgpu_device *adev, bool enable)
 {
 	if (enable)
 		WREG32_SOC15(NBIO, 0, mmBIF_FB_EN,
@@ -66,26 +53,23 @@ void nbio_v6_1_mc_access_enable(struct amdgpu_device *adev, bool enable)
 		WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0);
 }
 
-void nbio_v6_1_hdp_flush(struct amdgpu_device *adev)
+static void nbio_v6_1_hdp_flush(struct amdgpu_device *adev)
 {
 	WREG32_SOC15_NO_KIQ(NBIO, 0, mmBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
 }
 
-u32 nbio_v6_1_get_memsize(struct amdgpu_device *adev)
+static u32 nbio_v6_1_get_memsize(struct amdgpu_device *adev)
 {
 	return RREG32_SOC15(NBIO, 0, mmRCC_PF_0_0_RCC_CONFIG_MEMSIZE);
 }
 
-static const u32 nbio_sdma_doorbell_range_reg[] =
-{
-	SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE),
-	SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE)
-};
-
-void nbio_v6_1_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
+static void nbio_v6_1_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
 				  bool use_doorbell, int doorbell_index)
 {
-	u32 doorbell_range = RREG32(nbio_sdma_doorbell_range_reg[instance]);
+	u32 reg = instance == 0 ? SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE) :
+			SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE);
+
+	u32 doorbell_range = RREG32(reg);
 
 	if (use_doorbell) {
 		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, OFFSET, doorbell_index);
@@ -93,17 +77,18 @@ void nbio_v6_1_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
 	} else
 		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 0);
 
-	WREG32(nbio_sdma_doorbell_range_reg[instance], doorbell_range);
+	WREG32(reg, doorbell_range);
+
 }
 
-void nbio_v6_1_enable_doorbell_aperture(struct amdgpu_device *adev,
-					bool enable)
+static void nbio_v6_1_enable_doorbell_aperture(struct amdgpu_device *adev,
+					       bool enable)
 {
 	WREG32_FIELD15(NBIO, 0, RCC_PF_0_0_RCC_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN, enable ? 1 : 0);
 }
 
-void nbio_v6_1_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
-					bool enable)
+static void nbio_v6_1_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
+							bool enable)
 {
 	u32 tmp = 0;
 
@@ -122,8 +107,8 @@ void nbio_v6_1_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
 }
 
 
-void nbio_v6_1_ih_doorbell_range(struct amdgpu_device *adev,
-				bool use_doorbell, int doorbell_index)
+static void nbio_v6_1_ih_doorbell_range(struct amdgpu_device *adev,
+					bool use_doorbell, int doorbell_index)
 {
 	u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0 , mmBIF_IH_DOORBELL_RANGE);
 
@@ -136,7 +121,7 @@ void nbio_v6_1_ih_doorbell_range(struct amdgpu_device *adev,
 	WREG32_SOC15(NBIO, 0, mmBIF_IH_DOORBELL_RANGE, ih_doorbell_range);
 }
 
-void nbio_v6_1_ih_control(struct amdgpu_device *adev)
+static void nbio_v6_1_ih_control(struct amdgpu_device *adev)
 {
 	u32 interrupt_cntl;
 
@@ -152,8 +137,8 @@ void nbio_v6_1_ih_control(struct amdgpu_device *adev)
 	WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl);
 }
 
-void nbio_v6_1_update_medium_grain_clock_gating(struct amdgpu_device *adev,
-						bool enable)
+static void nbio_v6_1_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+						       bool enable)
 {
 	uint32_t def, data;
 
@@ -180,8 +165,8 @@ void nbio_v6_1_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 		WREG32_PCIE(smnCPM_CONTROL, data);
 }
 
-void nbio_v6_1_update_medium_grain_light_sleep(struct amdgpu_device *adev,
-					       bool enable)
+static void nbio_v6_1_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+						      bool enable)
 {
 	uint32_t def, data;
 
@@ -200,7 +185,8 @@ void nbio_v6_1_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 		WREG32_PCIE(smnPCIE_CNTL2, data);
 }
 
-void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
+static void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev,
+					    u32 *flags)
 {
 	int data;
 
@@ -215,9 +201,27 @@ void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
 		*flags |= AMD_CG_SUPPORT_BIF_LS;
 }
 
-const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg = {
-	.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ),
-	.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE),
+static u32 nbio_v6_1_get_hdp_flush_req_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_REQ);
+}
+
+static u32 nbio_v6_1_get_hdp_flush_done_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmBIF_BX_PF0_GPU_HDP_FLUSH_DONE);
+}
+
+static u32 nbio_v6_1_get_pcie_index_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX);
+}
+
+static u32 nbio_v6_1_get_pcie_data_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA);
+}
+
+static const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg = {
 	.ref_and_mask_cp0 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK,
 	.ref_and_mask_cp1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK,
 	.ref_and_mask_cp2 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK,
@@ -232,12 +236,7 @@ const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg = {
 	.ref_and_mask_sdma1 = BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK
 };
 
-const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data = {
-	.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX),
-	.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA),
-};
-
-void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev)
+static void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev)
 {
 	uint32_t reg;
 
@@ -254,7 +253,7 @@ void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev)
 	}
 }
 
-void nbio_v6_1_init_registers(struct amdgpu_device *adev)
+static void nbio_v6_1_init_registers(struct amdgpu_device *adev)
 {
 	uint32_t def, data;
 
@@ -265,3 +264,25 @@ void nbio_v6_1_init_registers(struct amdgpu_device *adev)
 	if (def != data)
 		WREG32_PCIE(smnPCIE_CONFIG_CNTL, data);
 }
+
+const struct amdgpu_nbio_funcs nbio_v6_1_funcs = {
+	.hdp_flush_reg = &nbio_v6_1_hdp_flush_reg,
+	.get_hdp_flush_req_offset = nbio_v6_1_get_hdp_flush_req_offset,
+	.get_hdp_flush_done_offset = nbio_v6_1_get_hdp_flush_done_offset,
+	.get_pcie_index_offset = nbio_v6_1_get_pcie_index_offset,
+	.get_pcie_data_offset = nbio_v6_1_get_pcie_data_offset,
+	.get_rev_id = nbio_v6_1_get_rev_id,
+	.mc_access_enable = nbio_v6_1_mc_access_enable,
+	.hdp_flush = nbio_v6_1_hdp_flush,
+	.get_memsize = nbio_v6_1_get_memsize,
+	.sdma_doorbell_range = nbio_v6_1_sdma_doorbell_range,
+	.enable_doorbell_aperture = nbio_v6_1_enable_doorbell_aperture,
+	.enable_doorbell_selfring_aperture = nbio_v6_1_enable_doorbell_selfring_aperture,
+	.ih_doorbell_range = nbio_v6_1_ih_doorbell_range,
+	.update_medium_grain_clock_gating = nbio_v6_1_update_medium_grain_clock_gating,
+	.update_medium_grain_light_sleep = nbio_v6_1_update_medium_grain_light_sleep,
+	.get_clockgating_state = nbio_v6_1_get_clockgating_state,
+	.ih_control = nbio_v6_1_ih_control,
+	.init_registers = nbio_v6_1_init_registers,
+	.detect_hw_virt = nbio_v6_1_detect_hw_virt,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
index 14ca8d45a46c..0743a6f016f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.h
@@ -26,30 +26,6 @@
 
 #include "soc15_common.h"
 
-extern const struct nbio_hdp_flush_reg nbio_v6_1_hdp_flush_reg;
-extern const struct nbio_pcie_index_data nbio_v6_1_pcie_index_data;
-int nbio_v6_1_init(struct amdgpu_device *adev);
-u32 nbio_v6_1_get_atombios_scratch_regs(struct amdgpu_device *adev,
-                                        uint32_t idx);
-void nbio_v6_1_set_atombios_scratch_regs(struct amdgpu_device *adev,
-                                         uint32_t idx, uint32_t val);
-void nbio_v6_1_mc_access_enable(struct amdgpu_device *adev, bool enable);
-void nbio_v6_1_hdp_flush(struct amdgpu_device *adev);
-u32 nbio_v6_1_get_memsize(struct amdgpu_device *adev);
-void nbio_v6_1_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
-				  bool use_doorbell, int doorbell_index);
-void nbio_v6_1_enable_doorbell_aperture(struct amdgpu_device *adev,
-					bool enable);
-void nbio_v6_1_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
-					bool enable);
-void nbio_v6_1_ih_doorbell_range(struct amdgpu_device *adev,
-				bool use_doorbell, int doorbell_index);
-void nbio_v6_1_ih_control(struct amdgpu_device *adev);
-u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev);
-void nbio_v6_1_update_medium_grain_clock_gating(struct amdgpu_device *adev, bool enable);
-void nbio_v6_1_update_medium_grain_light_sleep(struct amdgpu_device *adev, bool enable);
-void nbio_v6_1_get_clockgating_state(struct amdgpu_device *adev, u32 *flags);
-void nbio_v6_1_detect_hw_virt(struct amdgpu_device *adev);
-void nbio_v6_1_init_registers(struct amdgpu_device *adev);
+extern const struct amdgpu_nbio_funcs nbio_v6_1_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
index f802b973410a..17a9131a4598 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c
@@ -24,15 +24,17 @@
 #include "amdgpu_atombios.h"
 #include "nbio_v7_0.h"
 
-#include "vega10/soc15ip.h"
-#include "raven1/NBIO/nbio_7_0_default.h"
-#include "raven1/NBIO/nbio_7_0_offset.h"
-#include "raven1/NBIO/nbio_7_0_sh_mask.h"
-#include "vega10/vega10_enum.h"
+#include "nbio/nbio_7_0_default.h"
+#include "nbio/nbio_7_0_offset.h"
+#include "nbio/nbio_7_0_sh_mask.h"
+#include "vega10_enum.h"
 
 #define smnNBIF_MGCG_CTRL_LCLK	0x1013a05c
 
-u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev)
+#define smnCPM_CONTROL                                                                                  0x11180460
+#define smnPCIE_CNTL2                                                                                   0x11180070
+
+static u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev)
 {
         u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0);
 
@@ -42,19 +44,7 @@ u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev)
 	return tmp;
 }
 
-u32 nbio_v7_0_get_atombios_scratch_regs(struct amdgpu_device *adev,
-					uint32_t idx)
-{
-	return RREG32_SOC15_OFFSET(NBIO, 0, mmBIOS_SCRATCH_0, idx);
-}
-
-void nbio_v7_0_set_atombios_scratch_regs(struct amdgpu_device *adev,
-					 uint32_t idx, uint32_t val)
-{
-	WREG32_SOC15_OFFSET(NBIO, 0, mmBIOS_SCRATCH_0, idx, val);
-}
-
-void nbio_v7_0_mc_access_enable(struct amdgpu_device *adev, bool enable)
+static void nbio_v7_0_mc_access_enable(struct amdgpu_device *adev, bool enable)
 {
 	if (enable)
 		WREG32_SOC15(NBIO, 0, mmBIF_FB_EN,
@@ -63,26 +53,23 @@ void nbio_v7_0_mc_access_enable(struct amdgpu_device *adev, bool enable)
 		WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0);
 }
 
-void nbio_v7_0_hdp_flush(struct amdgpu_device *adev)
+static void nbio_v7_0_hdp_flush(struct amdgpu_device *adev)
 {
 	WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
 }
 
-u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev)
+static u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev)
 {
 	return RREG32_SOC15(NBIO, 0, mmRCC_CONFIG_MEMSIZE);
 }
 
-static const u32 nbio_sdma_doorbell_range_reg[] =
+static void nbio_v7_0_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
+					  bool use_doorbell, int doorbell_index)
 {
-	SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE),
-	SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE)
-};
+	u32 reg = instance == 0 ? SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA0_DOORBELL_RANGE) :
+			SOC15_REG_OFFSET(NBIO, 0, mmBIF_SDMA1_DOORBELL_RANGE);
 
-void nbio_v7_0_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
-				  bool use_doorbell, int doorbell_index)
-{
-	u32 doorbell_range = RREG32(nbio_sdma_doorbell_range_reg[instance]);
+	u32 doorbell_range = RREG32(reg);
 
 	if (use_doorbell) {
 		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, OFFSET, doorbell_index);
@@ -90,17 +77,23 @@ void nbio_v7_0_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
 	} else
 		doorbell_range = REG_SET_FIELD(doorbell_range, BIF_SDMA0_DOORBELL_RANGE, SIZE, 0);
 
-	WREG32(nbio_sdma_doorbell_range_reg[instance], doorbell_range);
+	WREG32(reg, doorbell_range);
 }
 
-void nbio_v7_0_enable_doorbell_aperture(struct amdgpu_device *adev,
-					bool enable)
+static void nbio_v7_0_enable_doorbell_aperture(struct amdgpu_device *adev,
+					       bool enable)
 {
 	WREG32_FIELD15(NBIO, 0, RCC_DOORBELL_APER_EN, BIF_DOORBELL_APER_EN, enable ? 1 : 0);
 }
 
-void nbio_v7_0_ih_doorbell_range(struct amdgpu_device *adev,
-				bool use_doorbell, int doorbell_index)
+static void nbio_v7_0_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
+							bool enable)
+{
+
+}
+
+static void nbio_v7_0_ih_doorbell_range(struct amdgpu_device *adev,
+					bool use_doorbell, int doorbell_index)
 {
 	u32 ih_doorbell_range = RREG32_SOC15(NBIO, 0 , mmBIF_IH_DOORBELL_RANGE);
 
@@ -130,8 +123,8 @@ static void nbio_7_0_write_syshub_ind_mmr(struct amdgpu_device *adev, uint32_t o
 	WREG32_SOC15(NBIO, 0, mmSYSHUB_DATA, data);
 }
 
-void nbio_v7_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
-						bool enable)
+static void nbio_v7_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
+						       bool enable)
 {
 	uint32_t def, data;
 
@@ -169,7 +162,43 @@ void nbio_v7_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 		nbio_7_0_write_syshub_ind_mmr(adev, ixSYSHUB_MMREG_IND_SYSHUB_MGCG_CTRL_SHUBCLK, data);
 }
 
-void nbio_v7_0_ih_control(struct amdgpu_device *adev)
+static void nbio_v7_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
+						      bool enable)
+{
+	uint32_t def, data;
+
+	def = data = RREG32_PCIE(smnPCIE_CNTL2);
+	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
+		data |= (PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
+			 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
+			 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
+	} else {
+		data &= ~(PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
+			  PCIE_CNTL2__MST_MEM_LS_EN_MASK |
+			  PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK);
+	}
+
+	if (def != data)
+		WREG32_PCIE(smnPCIE_CNTL2, data);
+}
+
+static void nbio_v7_0_get_clockgating_state(struct amdgpu_device *adev,
+					    u32 *flags)
+{
+	int data;
+
+	/* AMD_CG_SUPPORT_BIF_MGCG */
+	data = RREG32_PCIE(smnCPM_CONTROL);
+	if (data & CPM_CONTROL__LCLK_DYN_GATE_ENABLE_MASK)
+		*flags |= AMD_CG_SUPPORT_BIF_MGCG;
+
+	/* AMD_CG_SUPPORT_BIF_LS */
+	data = RREG32_PCIE(smnPCIE_CNTL2);
+	if (data & PCIE_CNTL2__SLV_MEM_LS_EN_MASK)
+		*flags |= AMD_CG_SUPPORT_BIF_LS;
+}
+
+static void nbio_v7_0_ih_control(struct amdgpu_device *adev)
 {
 	u32 interrupt_cntl;
 
@@ -185,9 +214,27 @@ void nbio_v7_0_ih_control(struct amdgpu_device *adev)
 	WREG32_SOC15(NBIO, 0, mmINTERRUPT_CNTL, interrupt_cntl);
 }
 
+static u32 nbio_v7_0_get_hdp_flush_req_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ);
+}
+
+static u32 nbio_v7_0_get_hdp_flush_done_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE);
+}
+
+static u32 nbio_v7_0_get_pcie_index_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2);
+}
+
+static u32 nbio_v7_0_get_pcie_data_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2);
+}
+
 const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg = {
-	.hdp_flush_req_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_REQ),
-	.hdp_flush_done_offset = SOC15_REG_OFFSET(NBIO, 0, mmGPU_HDP_FLUSH_DONE),
 	.ref_and_mask_cp0 = GPU_HDP_FLUSH_DONE__CP0_MASK,
 	.ref_and_mask_cp1 = GPU_HDP_FLUSH_DONE__CP1_MASK,
 	.ref_and_mask_cp2 = GPU_HDP_FLUSH_DONE__CP2_MASK,
@@ -202,7 +249,35 @@ const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg = {
 	.ref_and_mask_sdma1 = GPU_HDP_FLUSH_DONE__SDMA1_MASK,
 };
 
-const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data = {
-	.index_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_INDEX2),
-	.data_offset = SOC15_REG_OFFSET(NBIO, 0, mmPCIE_DATA2)
+static void nbio_v7_0_detect_hw_virt(struct amdgpu_device *adev)
+{
+	if (is_virtual_machine())	/* passthrough mode exclus sriov mod */
+		adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
+}
+
+static void nbio_v7_0_init_registers(struct amdgpu_device *adev)
+{
+
+}
+
+const struct amdgpu_nbio_funcs nbio_v7_0_funcs = {
+	.hdp_flush_reg = &nbio_v7_0_hdp_flush_reg,
+	.get_hdp_flush_req_offset = nbio_v7_0_get_hdp_flush_req_offset,
+	.get_hdp_flush_done_offset = nbio_v7_0_get_hdp_flush_done_offset,
+	.get_pcie_index_offset = nbio_v7_0_get_pcie_index_offset,
+	.get_pcie_data_offset = nbio_v7_0_get_pcie_data_offset,
+	.get_rev_id = nbio_v7_0_get_rev_id,
+	.mc_access_enable = nbio_v7_0_mc_access_enable,
+	.hdp_flush = nbio_v7_0_hdp_flush,
+	.get_memsize = nbio_v7_0_get_memsize,
+	.sdma_doorbell_range = nbio_v7_0_sdma_doorbell_range,
+	.enable_doorbell_aperture = nbio_v7_0_enable_doorbell_aperture,
+	.enable_doorbell_selfring_aperture = nbio_v7_0_enable_doorbell_selfring_aperture,
+	.ih_doorbell_range = nbio_v7_0_ih_doorbell_range,
+	.update_medium_grain_clock_gating = nbio_v7_0_update_medium_grain_clock_gating,
+	.update_medium_grain_light_sleep = nbio_v7_0_update_medium_grain_light_sleep,
+	.get_clockgating_state = nbio_v7_0_get_clockgating_state,
+	.ih_control = nbio_v7_0_ih_control,
+	.init_registers = nbio_v7_0_init_registers,
+	.detect_hw_virt = nbio_v7_0_detect_hw_virt,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
index df8fa90f40d7..508d549c5029 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.h
@@ -26,24 +26,6 @@
 
 #include "soc15_common.h"
 
-extern const struct nbio_hdp_flush_reg nbio_v7_0_hdp_flush_reg;
-extern const struct nbio_pcie_index_data nbio_v7_0_pcie_index_data;
-int nbio_v7_0_init(struct amdgpu_device *adev);
-u32 nbio_v7_0_get_atombios_scratch_regs(struct amdgpu_device *adev,
-                                        uint32_t idx);
-void nbio_v7_0_set_atombios_scratch_regs(struct amdgpu_device *adev,
-                                         uint32_t idx, uint32_t val);
-void nbio_v7_0_mc_access_enable(struct amdgpu_device *adev, bool enable);
-void nbio_v7_0_hdp_flush(struct amdgpu_device *adev);
-u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev);
-void nbio_v7_0_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
-				  bool use_doorbell, int doorbell_index);
-void nbio_v7_0_enable_doorbell_aperture(struct amdgpu_device *adev,
-					bool enable);
-void nbio_v7_0_ih_doorbell_range(struct amdgpu_device *adev,
-				bool use_doorbell, int doorbell_index);
-void nbio_v7_0_ih_control(struct amdgpu_device *adev);
-u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev);
-void nbio_v7_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
-						bool enable);
+extern const struct amdgpu_nbio_funcs nbio_v7_0_funcs;
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
index 4e20d91d5d50..5a9fe24697f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c
@@ -30,10 +30,9 @@
 #include "soc15_common.h"
 #include "psp_v10_0.h"
 
-#include "vega10/soc15ip.h"
-#include "raven1/MP/mp_10_0_offset.h"
-#include "raven1/GC/gc_9_1_offset.h"
-#include "raven1/SDMA0/sdma0_4_1_offset.h"
+#include "mp/mp_10_0_offset.h"
+#include "gc/gc_9_1_offset.h"
+#include "sdma0/sdma0_4_1_offset.h"
 
 MODULE_FIRMWARE("amdgpu/raven_asd.bin");
 
@@ -298,9 +297,10 @@ int psp_v10_0_cmd_submit(struct psp_context *psp,
 }
 
 static int
-psp_v10_0_sram_map(unsigned int *sram_offset, unsigned int *sram_addr_reg_offset,
-		  unsigned int *sram_data_reg_offset,
-		  enum AMDGPU_UCODE_ID ucode_id)
+psp_v10_0_sram_map(struct amdgpu_device *adev,
+		unsigned int *sram_offset, unsigned int *sram_addr_reg_offset,
+		unsigned int *sram_data_reg_offset,
+		enum AMDGPU_UCODE_ID ucode_id)
 {
 	int ret = 0;
 
@@ -395,7 +395,7 @@ bool psp_v10_0_compare_sram_data(struct psp_context *psp,
 	uint32_t *ucode_mem = NULL;
 	struct amdgpu_device *adev = psp->adev;
 
-	err = psp_v10_0_sram_map(&fw_sram_reg_val, &fw_sram_addr_reg_offset,
+	err = psp_v10_0_sram_map(adev, &fw_sram_reg_val, &fw_sram_addr_reg_offset,
 				&fw_sram_data_reg_offset, ucode_type);
 	if (err)
 		return false;
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index c7bcfe8e286c..19bd1934e63d 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -31,12 +31,11 @@
 #include "soc15_common.h"
 #include "psp_v3_1.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/MP/mp_9_0_offset.h"
-#include "vega10/MP/mp_9_0_sh_mask.h"
-#include "vega10/GC/gc_9_0_offset.h"
-#include "vega10/SDMA0/sdma0_4_0_offset.h"
-#include "vega10/NBIO/nbio_6_1_offset.h"
+#include "mp/mp_9_0_offset.h"
+#include "mp/mp_9_0_sh_mask.h"
+#include "gc/gc_9_0_offset.h"
+#include "sdma0/sdma0_4_0_offset.h"
+#include "nbio/nbio_6_1_offset.h"
 
 MODULE_FIRMWARE("amdgpu/vega10_sos.bin");
 MODULE_FIRMWARE("amdgpu/vega10_asd.bin");
@@ -410,9 +409,10 @@ int psp_v3_1_cmd_submit(struct psp_context *psp,
 }
 
 static int
-psp_v3_1_sram_map(unsigned int *sram_offset, unsigned int *sram_addr_reg_offset,
-		  unsigned int *sram_data_reg_offset,
-		  enum AMDGPU_UCODE_ID ucode_id)
+psp_v3_1_sram_map(struct amdgpu_device *adev,
+		unsigned int *sram_offset, unsigned int *sram_addr_reg_offset,
+		unsigned int *sram_data_reg_offset,
+		enum AMDGPU_UCODE_ID ucode_id)
 {
 	int ret = 0;
 
@@ -507,7 +507,7 @@ bool psp_v3_1_compare_sram_data(struct psp_context *psp,
 	uint32_t *ucode_mem = NULL;
 	struct amdgpu_device *adev = psp->adev;
 
-	err = psp_v3_1_sram_map(&fw_sram_reg_val, &fw_sram_addr_reg_offset,
+	err = psp_v3_1_sram_map(adev, &fw_sram_reg_val, &fw_sram_addr_reg_offset,
 				&fw_sram_data_reg_offset, ucode_type);
 	if (err)
 		return false;
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 67f375bfe452..d4787ad4d346 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -93,12 +93,12 @@ static void sdma_v2_4_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
-		amdgpu_program_register_sequence(adev,
-						 iceland_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_iceland_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
+		amdgpu_device_program_register_sequence(adev,
+							iceland_mgcg_cgcg_init,
+							ARRAY_SIZE(iceland_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_iceland_a11,
+							ARRAY_SIZE(golden_settings_iceland_a11));
 		break;
 	default:
 		break;
@@ -246,15 +246,13 @@ static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
  */
 static void sdma_v2_4_ring_emit_ib(struct amdgpu_ring *ring,
 				   struct amdgpu_ib *ib,
-				   unsigned vm_id, bool ctx_switch)
+				   unsigned vmid, bool ctx_switch)
 {
-	u32 vmid = vm_id & 0xf;
-
 	/* IB packet must end on a 8 DW boundary */
 	sdma_v2_4_ring_insert_nop(ring, (10 - (lower_32_bits(ring->wptr) & 7)) % 8);
 
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
-			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid));
+			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 	/* base must be 32 byte aligned */
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
@@ -600,7 +598,7 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -613,7 +611,7 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_wb_free(adev, index);
+		amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 
@@ -633,13 +631,13 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
 			  ring->idx, tmp);
 		r = -EINVAL;
 	}
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 
 	return r;
 }
@@ -662,7 +660,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -704,7 +702,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = le32_to_cpu(adev->wb.wb[index]);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
@@ -715,7 +713,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(adev, &ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 	return r;
 }
 
@@ -861,14 +859,14 @@ static void sdma_v2_4_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
  * using sDMA (VI).
  */
 static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					 unsigned vm_id, uint64_t pd_addr)
+					 unsigned vmid, uint64_t pd_addr)
 {
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
-	if (vm_id < 8) {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
+	if (vmid < 8) {
+		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid));
 	} else {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));
+		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8));
 	}
 	amdgpu_ring_write(ring, pd_addr >> 12);
 
@@ -876,7 +874,7 @@ static void sdma_v2_4_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 
 	/* wait for flush */
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 6d06f8eb659f..521978c40537 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -192,47 +192,47 @@ static void sdma_v3_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_FIJI:
-		amdgpu_program_register_sequence(adev,
-						 fiji_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_fiji_a10,
-						 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
+		amdgpu_device_program_register_sequence(adev,
+							fiji_mgcg_cgcg_init,
+							ARRAY_SIZE(fiji_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_fiji_a10,
+							ARRAY_SIZE(golden_settings_fiji_a10));
 		break;
 	case CHIP_TONGA:
-		amdgpu_program_register_sequence(adev,
-						 tonga_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_tonga_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
+		amdgpu_device_program_register_sequence(adev,
+							tonga_mgcg_cgcg_init,
+							ARRAY_SIZE(tonga_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_tonga_a11,
+							ARRAY_SIZE(golden_settings_tonga_a11));
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_polaris11_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_polaris11_a11,
+							ARRAY_SIZE(golden_settings_polaris11_a11));
 		break;
 	case CHIP_POLARIS10:
-		amdgpu_program_register_sequence(adev,
-						 golden_settings_polaris10_a11,
-						 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11));
+		amdgpu_device_program_register_sequence(adev,
+							golden_settings_polaris10_a11,
+							ARRAY_SIZE(golden_settings_polaris10_a11));
 		break;
 	case CHIP_CARRIZO:
-		amdgpu_program_register_sequence(adev,
-						 cz_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 cz_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(cz_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							cz_mgcg_cgcg_init,
+							ARRAY_SIZE(cz_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							cz_golden_settings_a11,
+							ARRAY_SIZE(cz_golden_settings_a11));
 		break;
 	case CHIP_STONEY:
-		amdgpu_program_register_sequence(adev,
-						 stoney_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 stoney_golden_settings_a11,
-						 (const u32)ARRAY_SIZE(stoney_golden_settings_a11));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_mgcg_cgcg_init,
+							ARRAY_SIZE(stoney_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_golden_settings_a11,
+							ARRAY_SIZE(stoney_golden_settings_a11));
 		break;
 	default:
 		break;
@@ -355,7 +355,7 @@ static uint64_t sdma_v3_0_ring_get_wptr(struct amdgpu_ring *ring)
 	struct amdgpu_device *adev = ring->adev;
 	u32 wptr;
 
-	if (ring->use_doorbell) {
+	if (ring->use_doorbell || ring->use_pollmem) {
 		/* XXX check if swapping is necessary on BE */
 		wptr = ring->adev->wb.wb[ring->wptr_offs] >> 2;
 	} else {
@@ -380,10 +380,13 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring)
 
 	if (ring->use_doorbell) {
 		u32 *wb = (u32 *)&adev->wb.wb[ring->wptr_offs];
-
 		/* XXX check if swapping is necessary on BE */
 		WRITE_ONCE(*wb, (lower_32_bits(ring->wptr) << 2));
 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr) << 2);
+	} else if (ring->use_pollmem) {
+		u32 *wb = (u32 *)&adev->wb.wb[ring->wptr_offs];
+
+		WRITE_ONCE(*wb, (lower_32_bits(ring->wptr) << 2));
 	} else {
 		int me = (ring == &ring->adev->sdma.instance[0].ring) ? 0 : 1;
 
@@ -414,15 +417,13 @@ static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
  */
 static void sdma_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
 				   struct amdgpu_ib *ib,
-				   unsigned vm_id, bool ctx_switch)
+				   unsigned vmid, bool ctx_switch)
 {
-	u32 vmid = vm_id & 0xf;
-
 	/* IB packet must end on a 8 DW boundary */
 	sdma_v3_0_ring_insert_nop(ring, (10 - (lower_32_bits(ring->wptr) & 7)) % 8);
 
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
-			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid));
+			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 	/* base must be 32 byte aligned */
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
@@ -718,10 +719,14 @@ static int sdma_v3_0_gfx_resume(struct amdgpu_device *adev)
 		WREG32(mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI + sdma_offsets[i],
 		       upper_32_bits(wptr_gpu_addr));
 		wptr_poll_cntl = RREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i]);
-		if (amdgpu_sriov_vf(adev))
-			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 1);
+		if (ring->use_pollmem)
+			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
+						       SDMA0_GFX_RB_WPTR_POLL_CNTL,
+						       ENABLE, 1);
 		else
-			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 0);
+			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
+						       SDMA0_GFX_RB_WPTR_POLL_CNTL,
+						       ENABLE, 0);
 		WREG32(mmSDMA0_GFX_RB_WPTR_POLL_CNTL + sdma_offsets[i], wptr_poll_cntl);
 
 		/* enable DMA RB */
@@ -860,7 +865,7 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -873,7 +878,7 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_wb_free(adev, index);
+		amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 
@@ -893,13 +898,13 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
 			  ring->idx, tmp);
 		r = -EINVAL;
 	}
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 
 	return r;
 }
@@ -922,7 +927,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -964,7 +969,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = le32_to_cpu(adev->wb.wb[index]);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
@@ -974,7 +979,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(adev, &ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 	return r;
 }
 
@@ -1120,14 +1125,14 @@ static void sdma_v3_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
  * using sDMA (VI).
  */
 static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					 unsigned vm_id, uint64_t pd_addr)
+					 unsigned vmid, uint64_t pd_addr)
 {
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
-	if (vm_id < 8) {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
+	if (vmid < 8) {
+		amdgpu_ring_write(ring, (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid));
 	} else {
-		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));
+		amdgpu_ring_write(ring, (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8));
 	}
 	amdgpu_ring_write(ring, pd_addr >> 12);
 
@@ -1135,7 +1140,7 @@ static void sdma_v3_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 
 	/* wait for flush */
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
@@ -1203,9 +1208,13 @@ static int sdma_v3_0_sw_init(void *handle)
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
 		ring->ring_obj = NULL;
-		ring->use_doorbell = true;
-		ring->doorbell_index = (i == 0) ?
-			AMDGPU_DOORBELL_sDMA_ENGINE0 : AMDGPU_DOORBELL_sDMA_ENGINE1;
+		if (!amdgpu_sriov_vf(adev)) {
+			ring->use_doorbell = true;
+			ring->doorbell_index = (i == 0) ?
+				AMDGPU_DOORBELL_sDMA_ENGINE0 : AMDGPU_DOORBELL_sDMA_ENGINE1;
+		} else {
+			ring->use_pollmem = true;
+		}
 
 		sprintf(ring->name, "sdma%d", i);
 		r = amdgpu_ring_init(adev, ring, 1024,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 46009db3d195..91cf95a8c39c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -27,15 +27,14 @@
 #include "amdgpu_ucode.h"
 #include "amdgpu_trace.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/SDMA0/sdma0_4_0_offset.h"
-#include "vega10/SDMA0/sdma0_4_0_sh_mask.h"
-#include "vega10/SDMA1/sdma1_4_0_offset.h"
-#include "vega10/SDMA1/sdma1_4_0_sh_mask.h"
-#include "vega10/MMHUB/mmhub_1_0_offset.h"
-#include "vega10/MMHUB/mmhub_1_0_sh_mask.h"
-#include "vega10/HDP/hdp_4_0_offset.h"
-#include "raven1/SDMA0/sdma0_4_1_default.h"
+#include "sdma0/sdma0_4_0_offset.h"
+#include "sdma0/sdma0_4_0_sh_mask.h"
+#include "sdma1/sdma1_4_0_offset.h"
+#include "sdma1/sdma1_4_0_sh_mask.h"
+#include "mmhub/mmhub_1_0_offset.h"
+#include "mmhub/mmhub_1_0_sh_mask.h"
+#include "hdp/hdp_4_0_offset.h"
+#include "sdma0/sdma0_4_1_default.h"
 
 #include "soc15_common.h"
 #include "soc15.h"
@@ -53,97 +52,85 @@ static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev);
 static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev);
 static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev);
 
-static const u32 golden_settings_sdma_4[] = {
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831d07,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xff000ff0, 0x3f000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_PAGE_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL), 0x0000fff0, 0x00403000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), 0x003ff006, 0x0003c000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL), 0x0000fff0, 0x00403000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL), 0x0000fff0, 0x00403000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_UTCL1_PAGE), 0x000003ff, 0x000003c0,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_CHICKEN_BITS), 0xfe931f07, 0x02831f07,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_CLK_CTRL), 0xffffffff, 0x3f000100,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL), 0x0000fff0, 0x00403000,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_PAGE_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL), 0x0000fff0, 0x00403000,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_POWER_CNTL), 0x003ff000, 0x0003c000,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_RLC0_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL), 0x0000fff0, 0x00403000,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_RLC1_IB_CNTL), 0x800f0100, 0x00000100,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL), 0x0000fff0, 0x00403000,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_UTCL1_PAGE), 0x000003ff, 0x000003c0
+static const struct soc15_reg_golden golden_settings_sdma_4[] = {
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xff000ff0, 0x3f000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0x003ff006, 0x0003c000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CHICKEN_BITS, 0xfe931f07, 0x02831f07),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_CLK_CTRL, 0xffffffff, 0x3f000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GFX_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_PAGE_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_POWER_CNTL, 0x003ff000, 0x0003c000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC0_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_IB_CNTL, 0x800f0100, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC1_RB_WPTR_POLL_CNTL, 0x0000fff0, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0)
 };
 
-static const u32 golden_settings_sdma_vg10[] = {
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG), 0x0018773f, 0x00104002,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ), 0x0018773f, 0x00104002,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG), 0x0018773f, 0x00104002,
-	SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ), 0x0018773f, 0x00104002
+static const struct soc15_reg_golden golden_settings_sdma_vg10[] = {
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG, 0x0018773f, 0x00104002),
+	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00104002)
 };
 
-static const u32 golden_settings_sdma_4_1[] =
+static const struct soc15_reg_golden golden_settings_sdma_4_1[] =
 {
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CHICKEN_BITS), 0xfe931f07, 0x02831d07,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_CLK_CTRL), 0xffffffff, 0x3f000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_IB_CNTL), 0x800f0111, 0x00000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_POWER_CNTL), 0xfc3fffff, 0x40000051,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL), 0x800f0111, 0x00000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL), 0x800f0111, 0x00000100,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL), 0xfffffff7, 0x00403000,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_UTCL1_PAGE), 0x000003ff, 0x000003c0
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CHICKEN_BITS, 0xfe931f07, 0x02831d07),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_CLK_CTRL, 0xffffffff, 0x3f000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_POWER_CNTL, 0xfc3fffff, 0x40000051),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC0_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC1_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0)
 };
 
-static const u32 golden_settings_sdma_rv1[] =
+static const struct soc15_reg_golden golden_settings_sdma_rv1[] =
 {
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG), 0x0018773f, 0x00000002,
-	SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ), 0x0018773f, 0x00000002
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG, 0x0018773f, 0x00000002),
+	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_GB_ADDR_CONFIG_READ, 0x0018773f, 0x00000002)
 };
 
-static u32 sdma_v4_0_get_reg_offset(u32 instance, u32 internal_offset)
+static u32 sdma_v4_0_get_reg_offset(struct amdgpu_device *adev,
+		u32 instance, u32 offset)
 {
-	u32 base = 0;
-
-	switch (instance) {
-	case 0:
-		base = SDMA0_BASE.instance[0].segment[0];
-		break;
-	case 1:
-		base = SDMA1_BASE.instance[0].segment[0];
-		break;
-	default:
-		BUG();
-		break;
-	}
-
-	return base + internal_offset;
+	return ( 0 == instance ? (adev->reg_offset[SDMA0_HWIP][0][0] + offset) :
+			(adev->reg_offset[SDMA1_HWIP][0][0] + offset));
 }
 
 static void sdma_v4_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
-		amdgpu_program_register_sequence(adev,
+		soc15_program_register_sequence(adev,
 						 golden_settings_sdma_4,
-						 (const u32)ARRAY_SIZE(golden_settings_sdma_4));
-		amdgpu_program_register_sequence(adev,
+						 ARRAY_SIZE(golden_settings_sdma_4));
+		soc15_program_register_sequence(adev,
 						 golden_settings_sdma_vg10,
-						 (const u32)ARRAY_SIZE(golden_settings_sdma_vg10));
+						 ARRAY_SIZE(golden_settings_sdma_vg10));
 		break;
 	case CHIP_RAVEN:
-		amdgpu_program_register_sequence(adev,
+		soc15_program_register_sequence(adev,
 						 golden_settings_sdma_4_1,
-						 (const u32)ARRAY_SIZE(golden_settings_sdma_4_1));
-		amdgpu_program_register_sequence(adev,
+						 ARRAY_SIZE(golden_settings_sdma_4_1));
+		soc15_program_register_sequence(adev,
 						 golden_settings_sdma_rv1,
-						 (const u32)ARRAY_SIZE(golden_settings_sdma_rv1));
+						 ARRAY_SIZE(golden_settings_sdma_rv1));
 		break;
 	default:
 		break;
@@ -251,31 +238,27 @@ static uint64_t sdma_v4_0_ring_get_rptr(struct amdgpu_ring *ring)
 static uint64_t sdma_v4_0_ring_get_wptr(struct amdgpu_ring *ring)
 {
 	struct amdgpu_device *adev = ring->adev;
-	u64 *wptr = NULL;
-	uint64_t local_wptr = 0;
+	u64 wptr;
 
 	if (ring->use_doorbell) {
 		/* XXX check if swapping is necessary on BE */
-		wptr = ((u64 *)&adev->wb.wb[ring->wptr_offs]);
-		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", *wptr);
-		*wptr = (*wptr) >> 2;
-		DRM_DEBUG("wptr/doorbell after shift == 0x%016llx\n", *wptr);
+		wptr = READ_ONCE(*((u64 *)&adev->wb.wb[ring->wptr_offs]));
+		DRM_DEBUG("wptr/doorbell before shift == 0x%016llx\n", wptr);
 	} else {
 		u32 lowbit, highbit;
 		int me = (ring == &adev->sdma.instance[0].ring) ? 0 : 1;
 
-		wptr = &local_wptr;
-		lowbit = RREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR)) >> 2;
-		highbit = RREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR_HI)) >> 2;
+		lowbit = RREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR)) >> 2;
+		highbit = RREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR_HI)) >> 2;
 
 		DRM_DEBUG("wptr [%i]high== 0x%08x low==0x%08x\n",
 				me, highbit, lowbit);
-		*wptr = highbit;
-		*wptr = (*wptr) << 32;
-		*wptr |= lowbit;
+		wptr = highbit;
+		wptr = wptr << 32;
+		wptr |= lowbit;
 	}
 
-	return *wptr;
+	return wptr >> 2;
 }
 
 /**
@@ -315,8 +298,8 @@ static void sdma_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
 				lower_32_bits(ring->wptr << 2),
 				me,
 				upper_32_bits(ring->wptr << 2));
-		WREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
-		WREG32(sdma_v4_0_get_reg_offset(me, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
+		WREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr << 2));
+		WREG32(sdma_v4_0_get_reg_offset(adev, me, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr << 2));
 	}
 }
 
@@ -343,15 +326,13 @@ static void sdma_v4_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
  */
 static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
 					struct amdgpu_ib *ib,
-					unsigned vm_id, bool ctx_switch)
+					unsigned vmid, bool ctx_switch)
 {
-	u32 vmid = vm_id & 0xf;
-
 	/* IB packet must end on a 8 DW boundary */
 	sdma_v4_0_ring_insert_nop(ring, (10 - (lower_32_bits(ring->wptr) & 7)) % 8);
 
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) |
-			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid));
+			  SDMA_PKT_INDIRECT_HEADER_VMID(vmid & 0xf));
 	/* base must be 32 byte aligned */
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr) & 0xffffffe0);
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
@@ -370,13 +351,9 @@ static void sdma_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
  */
 static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
 	u32 ref_and_mask = 0;
-	const struct nbio_hdp_flush_reg *nbio_hf_reg;
-
-	if (ring->adev->flags & AMD_IS_APU)
-		nbio_hf_reg = &nbio_v7_0_hdp_flush_reg;
-	else
-		nbio_hf_reg = &nbio_v6_1_hdp_flush_reg;
+	const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio_funcs->hdp_flush_reg;
 
 	if (ring == &ring->adev->sdma.instance[0].ring)
 		ref_and_mask = nbio_hf_reg->ref_and_mask_sdma0;
@@ -386,8 +363,8 @@ static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_POLL_REGMEM) |
 			  SDMA_PKT_POLL_REGMEM_HEADER_HDP_FLUSH(1) |
 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* == */
-	amdgpu_ring_write(ring, nbio_hf_reg->hdp_flush_done_offset << 2);
-	amdgpu_ring_write(ring, nbio_hf_reg->hdp_flush_req_offset << 2);
+	amdgpu_ring_write(ring, (adev->nbio_funcs->get_hdp_flush_done_offset(adev)) << 2);
+	amdgpu_ring_write(ring, (adev->nbio_funcs->get_hdp_flush_req_offset(adev)) << 2);
 	amdgpu_ring_write(ring, ref_and_mask); /* reference */
 	amdgpu_ring_write(ring, ref_and_mask); /* mask */
 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
@@ -396,6 +373,8 @@ static void sdma_v4_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 
 static void sdma_v4_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
 	amdgpu_ring_write(ring, SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE));
@@ -460,12 +439,12 @@ static void sdma_v4_0_gfx_stop(struct amdgpu_device *adev)
 		amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
-		rb_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_CNTL));
+		rb_cntl = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
-		ib_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_IB_CNTL));
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
+		ib_cntl = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 0);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
 	}
 
 	sdma0->ready = false;
@@ -522,18 +501,18 @@ static void sdma_v4_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable)
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
-		f32_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_CNTL));
+		f32_cntl = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_CNTL));
 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_CNTL,
 				AUTO_CTXSW_ENABLE, enable ? 1 : 0);
 		if (enable && amdgpu_sdma_phase_quantum) {
-			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_PHASE0_QUANTUM),
+			WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_PHASE0_QUANTUM),
 			       phase_quantum);
-			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_PHASE1_QUANTUM),
+			WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_PHASE1_QUANTUM),
 			       phase_quantum);
-			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_PHASE2_QUANTUM),
+			WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_PHASE2_QUANTUM),
 			       phase_quantum);
 		}
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_CNTL), f32_cntl);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_CNTL), f32_cntl);
 	}
 
 }
@@ -557,9 +536,9 @@ static void sdma_v4_0_enable(struct amdgpu_device *adev, bool enable)
 	}
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
-		f32_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_F32_CNTL));
+		f32_cntl = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
 		f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_F32_CNTL), f32_cntl);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), f32_cntl);
 	}
 }
 
@@ -587,48 +566,48 @@ static int sdma_v4_0_gfx_resume(struct amdgpu_device *adev)
 		ring = &adev->sdma.instance[i].ring;
 		wb_offset = (ring->rptr_offs * 4);
 
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
 
 		/* Set ring buffer size in dwords */
 		rb_bufsz = order_base_2(ring->ring_size / 4);
-		rb_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_CNTL));
+		rb_cntl = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL));
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SIZE, rb_bufsz);
 #ifdef __BIG_ENDIAN
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_SWAP_ENABLE, 1);
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL,
 					RPTR_WRITEBACK_SWAP_ENABLE, 1);
 #endif
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 
 		/* Initialize the ring buffer's read and write pointers */
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_RPTR), 0);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_RPTR_HI), 0);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR), 0);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_HI), 0);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR), 0);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_HI), 0);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), 0);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), 0);
 
 		/* set the wb address whether it's enabled or not */
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_RPTR_ADDR_HI),
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_HI),
 		       upper_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFF);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_RPTR_ADDR_LO),
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_RPTR_ADDR_LO),
 		       lower_32_bits(adev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC);
 
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
 
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE), ring->gpu_addr >> 8);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_BASE_HI), ring->gpu_addr >> 40);
 
 		ring->wptr = 0;
 
 		/* before programing wptr to a less value, need set minor_ptr_update first */
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 1);
 
 		if (!amdgpu_sriov_vf(adev)) { /* only bare-metal use register write for wptr */
-			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr) << 2);
-			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
+			WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR), lower_32_bits(ring->wptr) << 2);
+			WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
 		}
 
-		doorbell = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL));
-		doorbell_offset = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL_OFFSET));
+		doorbell = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL));
+		doorbell_offset = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET));
 
 		if (ring->use_doorbell) {
 			doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 1);
@@ -637,55 +616,53 @@ static int sdma_v4_0_gfx_resume(struct amdgpu_device *adev)
 		} else {
 			doorbell = REG_SET_FIELD(doorbell, SDMA0_GFX_DOORBELL, ENABLE, 0);
 		}
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL), doorbell);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset);
-		if (adev->flags & AMD_IS_APU)
-			nbio_v7_0_sdma_doorbell_range(adev, i, ring->use_doorbell, ring->doorbell_index);
-		else
-			nbio_v6_1_sdma_doorbell_range(adev, i, ring->use_doorbell, ring->doorbell_index);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL), doorbell);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_DOORBELL_OFFSET), doorbell_offset);
+		adev->nbio_funcs->sdma_doorbell_range(adev, i, ring->use_doorbell,
+						      ring->doorbell_index);
 
 		if (amdgpu_sriov_vf(adev))
 			sdma_v4_0_ring_set_wptr(ring);
 
 		/* set minor_ptr_update to 0 after wptr programed */
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_MINOR_PTR_UPDATE), 0);
 
 		/* set utc l1 enable flag always to 1 */
-		temp = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_CNTL));
+		temp = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_CNTL));
 		temp = REG_SET_FIELD(temp, SDMA0_CNTL, UTC_L1_ENABLE, 1);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_CNTL), temp);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_CNTL), temp);
 
 		if (!amdgpu_sriov_vf(adev)) {
 			/* unhalt engine */
-			temp = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_F32_CNTL));
+			temp = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL));
 			temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0);
-			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_F32_CNTL), temp);
+			WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_F32_CNTL), temp);
 		}
 
 		/* setup the wptr shadow polling */
 		wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO),
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO),
 		       lower_32_bits(wptr_gpu_addr));
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI),
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI),
 		       upper_32_bits(wptr_gpu_addr));
-		wptr_poll_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
+		wptr_poll_cntl = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL));
 		if (amdgpu_sriov_vf(adev))
 			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 1);
 		else
 			wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl, SDMA0_GFX_RB_WPTR_POLL_CNTL, F32_POLL_ENABLE, 0);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), wptr_poll_cntl);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL), wptr_poll_cntl);
 
 		/* enable DMA RB */
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 1);
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL), rb_cntl);
 
-		ib_cntl = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_IB_CNTL));
+		ib_cntl = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL));
 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_ENABLE, 1);
 #ifdef __BIG_ENDIAN
 		ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_GFX_IB_CNTL, IB_SWAP_ENABLE, 1);
 #endif
 		/* enable DMA IBs */
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_GFX_IB_CNTL), ib_cntl);
 
 		ring->ready = true;
 
@@ -816,12 +793,12 @@ static int sdma_v4_0_load_microcode(struct amdgpu_device *adev)
 			(adev->sdma.instance[i].fw->data +
 				le32_to_cpu(hdr->header.ucode_array_offset_bytes));
 
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_ADDR), 0);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), 0);
 
 		for (j = 0; j < fw_size; j++)
-			WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
+			WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_UCODE_DATA), le32_to_cpup(fw_data++));
 
-		WREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version);
+		WREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_UCODE_ADDR), adev->sdma.instance[i].fw_version);
 	}
 
 	return 0;
@@ -886,7 +863,7 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -899,7 +876,7 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 5);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_wb_free(adev, index);
+		amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 
@@ -919,13 +896,13 @@ static int sdma_v4_0_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
 			  ring->idx, tmp);
 		r = -EINVAL;
 	}
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 
 	return r;
 }
@@ -948,7 +925,7 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u32 tmp = 0;
 	u64 gpu_addr;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -990,7 +967,7 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = le32_to_cpu(adev->wb.wb[index]);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
@@ -1000,7 +977,7 @@ static int sdma_v4_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(adev, &ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 	return r;
 }
 
@@ -1152,23 +1129,24 @@ static void sdma_v4_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
  * using sDMA (VEGA10).
  */
 static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					 unsigned vm_id, uint64_t pd_addr)
+					 unsigned vmid, uint64_t pd_addr)
 {
 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
-	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
+	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
+	uint64_t flags = AMDGPU_PTE_VALID;
 	unsigned eng = ring->vm_inv_eng;
 
-	pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
-	pd_addr |= AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
+	pd_addr |= flags;
 
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
-	amdgpu_ring_write(ring, hub->ctx0_ptb_addr_lo32 + vm_id * 2);
+	amdgpu_ring_write(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2);
 	amdgpu_ring_write(ring, lower_32_bits(pd_addr));
 
 	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_SRBM_WRITE) |
 			  SDMA_PKT_SRBM_WRITE_HEADER_BYTE_EN(0xf));
-	amdgpu_ring_write(ring, hub->ctx0_ptb_addr_hi32 + vm_id * 2);
+	amdgpu_ring_write(ring, hub->ctx0_ptb_addr_hi32 + vmid * 2);
 	amdgpu_ring_write(ring, upper_32_bits(pd_addr));
 
 	/* flush TLB */
@@ -1183,8 +1161,8 @@ static void sdma_v4_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 			  SDMA_PKT_POLL_REGMEM_HEADER_FUNC(3)); /* equal */
 	amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
 	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, 1 << vm_id); /* reference */
-	amdgpu_ring_write(ring, 1 << vm_id); /* mask */
+	amdgpu_ring_write(ring, 1 << vmid); /* reference */
+	amdgpu_ring_write(ring, 1 << vmid); /* mask */
 	amdgpu_ring_write(ring, SDMA_PKT_POLL_REGMEM_DW5_RETRY_COUNT(0xfff) |
 			  SDMA_PKT_POLL_REGMEM_DW5_INTERVAL(10));
 }
@@ -1317,7 +1295,7 @@ static bool sdma_v4_0_is_idle(void *handle)
 	u32 i;
 
 	for (i = 0; i < adev->sdma.num_instances; i++) {
-		u32 tmp = RREG32(sdma_v4_0_get_reg_offset(i, mmSDMA0_STATUS_REG));
+		u32 tmp = RREG32(sdma_v4_0_get_reg_offset(adev, i, mmSDMA0_STATUS_REG));
 
 		if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
 			return false;
@@ -1333,8 +1311,8 @@ static int sdma_v4_0_wait_for_idle(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	for (i = 0; i < adev->usec_timeout; i++) {
-		sdma0 = RREG32(sdma_v4_0_get_reg_offset(0, mmSDMA0_STATUS_REG));
-		sdma1 = RREG32(sdma_v4_0_get_reg_offset(1, mmSDMA0_STATUS_REG));
+		sdma0 = RREG32(sdma_v4_0_get_reg_offset(adev, 0, mmSDMA0_STATUS_REG));
+		sdma1 = RREG32(sdma_v4_0_get_reg_offset(adev, 1, mmSDMA0_STATUS_REG));
 
 		if (sdma0 & sdma1 & SDMA0_STATUS_REG__IDLE_MASK)
 			return 0;
@@ -1358,8 +1336,8 @@ static int sdma_v4_0_set_trap_irq_state(struct amdgpu_device *adev,
 	u32 sdma_cntl;
 
 	u32 reg_offset = (type == AMDGPU_SDMA_IRQ_TRAP0) ?
-		sdma_v4_0_get_reg_offset(0, mmSDMA0_CNTL) :
-		sdma_v4_0_get_reg_offset(1, mmSDMA0_CNTL);
+		sdma_v4_0_get_reg_offset(adev, 0, mmSDMA0_CNTL) :
+		sdma_v4_0_get_reg_offset(adev, 1, mmSDMA0_CNTL);
 
 	sdma_cntl = RREG32(reg_offset);
 	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c
index 8284d5dbfc30..2095173aaabf 100644
--- a/drivers/gpu/drm/amd/amdgpu/si.c
+++ b/drivers/gpu/drm/amd/amdgpu/si.c
@@ -31,6 +31,7 @@
 #include "amdgpu_uvd.h"
 #include "amdgpu_vce.h"
 #include "atom.h"
+#include "amd_pcie.h"
 #include "amdgpu_powerplay.h"
 #include "sid.h"
 #include "si_ih.h"
@@ -1390,65 +1391,65 @@ static void si_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->asic_type) {
 	case CHIP_TAHITI:
-		amdgpu_program_register_sequence(adev,
-						 tahiti_golden_registers,
-						 (const u32)ARRAY_SIZE(tahiti_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 tahiti_golden_rlc_registers,
-						 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
-		amdgpu_program_register_sequence(adev,
-						 tahiti_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 tahiti_golden_registers2,
-						 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
+		amdgpu_device_program_register_sequence(adev,
+							tahiti_golden_registers,
+							ARRAY_SIZE(tahiti_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							tahiti_golden_rlc_registers,
+							ARRAY_SIZE(tahiti_golden_rlc_registers));
+		amdgpu_device_program_register_sequence(adev,
+							tahiti_mgcg_cgcg_init,
+							ARRAY_SIZE(tahiti_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							tahiti_golden_registers2,
+							ARRAY_SIZE(tahiti_golden_registers2));
 		break;
 	case CHIP_PITCAIRN:
-		amdgpu_program_register_sequence(adev,
-						 pitcairn_golden_registers,
-						 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 pitcairn_golden_rlc_registers,
-						 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
-		amdgpu_program_register_sequence(adev,
-						 pitcairn_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							pitcairn_golden_registers,
+							ARRAY_SIZE(pitcairn_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							pitcairn_golden_rlc_registers,
+							ARRAY_SIZE(pitcairn_golden_rlc_registers));
+		amdgpu_device_program_register_sequence(adev,
+							pitcairn_mgcg_cgcg_init,
+							ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
 		break;
 	case CHIP_VERDE:
-		amdgpu_program_register_sequence(adev,
-						 verde_golden_registers,
-						 (const u32)ARRAY_SIZE(verde_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 verde_golden_rlc_registers,
-						 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
-		amdgpu_program_register_sequence(adev,
-						 verde_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
-		amdgpu_program_register_sequence(adev,
-						 verde_pg_init,
-						 (const u32)ARRAY_SIZE(verde_pg_init));
+		amdgpu_device_program_register_sequence(adev,
+							verde_golden_registers,
+							ARRAY_SIZE(verde_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							verde_golden_rlc_registers,
+							ARRAY_SIZE(verde_golden_rlc_registers));
+		amdgpu_device_program_register_sequence(adev,
+							verde_mgcg_cgcg_init,
+							ARRAY_SIZE(verde_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							verde_pg_init,
+							ARRAY_SIZE(verde_pg_init));
 		break;
 	case CHIP_OLAND:
-		amdgpu_program_register_sequence(adev,
-						 oland_golden_registers,
-						 (const u32)ARRAY_SIZE(oland_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 oland_golden_rlc_registers,
-						 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
-		amdgpu_program_register_sequence(adev,
-						 oland_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							oland_golden_registers,
+							ARRAY_SIZE(oland_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							oland_golden_rlc_registers,
+							ARRAY_SIZE(oland_golden_rlc_registers));
+		amdgpu_device_program_register_sequence(adev,
+							oland_mgcg_cgcg_init,
+							ARRAY_SIZE(oland_mgcg_cgcg_init));
 		break;
 	case CHIP_HAINAN:
-		amdgpu_program_register_sequence(adev,
-						 hainan_golden_registers,
-						 (const u32)ARRAY_SIZE(hainan_golden_registers));
-		amdgpu_program_register_sequence(adev,
-						 hainan_golden_registers2,
-						 (const u32)ARRAY_SIZE(hainan_golden_registers2));
-		amdgpu_program_register_sequence(adev,
-						 hainan_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							hainan_golden_registers,
+							ARRAY_SIZE(hainan_golden_registers));
+		amdgpu_device_program_register_sequence(adev,
+							hainan_golden_registers2,
+							ARRAY_SIZE(hainan_golden_registers2));
+		amdgpu_device_program_register_sequence(adev,
+							hainan_mgcg_cgcg_init,
+							ARRAY_SIZE(hainan_mgcg_cgcg_init));
 		break;
 
 
@@ -1461,8 +1462,8 @@ static void si_pcie_gen3_enable(struct amdgpu_device *adev)
 {
 	struct pci_dev *root = adev->pdev->bus->self;
 	int bridge_pos, gpu_pos;
-	u32 speed_cntl, mask, current_data_rate;
-	int ret, i;
+	u32 speed_cntl, current_data_rate;
+	int i;
 	u16 tmp16;
 
 	if (pci_is_root_bus(adev->pdev->bus))
@@ -1474,23 +1475,20 @@ static void si_pcie_gen3_enable(struct amdgpu_device *adev)
 	if (adev->flags & AMD_IS_APU)
 		return;
 
-	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
-	if (ret != 0)
-		return;
-
-	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
+	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
+					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
 		return;
 
 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
 	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
 		LC_CURRENT_DATA_RATE_SHIFT;
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
 		if (current_data_rate == 2) {
 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
 			return;
 		}
 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
-	} else if (mask & DRM_PCIE_SPEED_50) {
+	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
 		if (current_data_rate == 1) {
 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
 			return;
@@ -1506,7 +1504,7 @@ static void si_pcie_gen3_enable(struct amdgpu_device *adev)
 	if (!gpu_pos)
 		return;
 
-	if (mask & DRM_PCIE_SPEED_80) {
+	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
 		if (current_data_rate != 2) {
 			u16 bridge_cfg, gpu_cfg;
 			u16 bridge_cfg2, gpu_cfg2;
@@ -1589,9 +1587,9 @@ static void si_pcie_gen3_enable(struct amdgpu_device *adev)
 
 	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
 	tmp16 &= ~0xf;
-	if (mask & DRM_PCIE_SPEED_80)
+	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
 		tmp16 |= 3;
-	else if (mask & DRM_PCIE_SPEED_50)
+	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
 		tmp16 |= 2;
 	else
 		tmp16 |= 1;
@@ -1959,42 +1957,42 @@ int si_set_ip_blocks(struct amdgpu_device *adev)
 	case CHIP_VERDE:
 	case CHIP_TAHITI:
 	case CHIP_PITCAIRN:
-		amdgpu_ip_block_add(adev, &si_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &si_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 		else
-			amdgpu_ip_block_add(adev, &dce_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &si_dma_ip_block);
-		/* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
-		/* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
+			amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
+		/* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
+		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
 		break;
 	case CHIP_OLAND:
-		amdgpu_ip_block_add(adev, &si_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &si_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 		else
-			amdgpu_ip_block_add(adev, &dce_v6_4_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &si_dma_ip_block);
-		/* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
-		/* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
+			amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
+		/* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
+		/* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
 		break;
 	case CHIP_HAINAN:
-		amdgpu_ip_block_add(adev, &si_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &si_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &si_dma_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
 		break;
 	default:
 		BUG();
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 3fa2fbf8c9a1..9a29c1399091 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -61,14 +61,14 @@ static void si_dma_ring_set_wptr(struct amdgpu_ring *ring)
 
 static void si_dma_ring_emit_ib(struct amdgpu_ring *ring,
 				struct amdgpu_ib *ib,
-				unsigned vm_id, bool ctx_switch)
+				unsigned vmid, bool ctx_switch)
 {
 	/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
 	 * Pad as necessary with NOPs.
 	 */
 	while ((lower_32_bits(ring->wptr) & 7) != 5)
 		amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
-	amdgpu_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vm_id, 0));
+	amdgpu_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, vmid, 0));
 	amdgpu_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
 	amdgpu_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
 
@@ -221,7 +221,7 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
 	u32 tmp;
 	u64 gpu_addr;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
 		return r;
@@ -234,7 +234,7 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
 	r = amdgpu_ring_alloc(ring, 4);
 	if (r) {
 		DRM_ERROR("amdgpu: dma failed to lock ring %d (%d).\n", ring->idx, r);
-		amdgpu_wb_free(adev, index);
+		amdgpu_device_wb_free(adev, index);
 		return r;
 	}
 
@@ -252,13 +252,13 @@ static int si_dma_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n", ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
 			  ring->idx, tmp);
 		r = -EINVAL;
 	}
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 
 	return r;
 }
@@ -281,7 +281,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	u64 gpu_addr;
 	long r;
 
-	r = amdgpu_wb_get(adev, &index);
+	r = amdgpu_device_wb_get(adev, &index);
 	if (r) {
 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
 		return r;
@@ -317,7 +317,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 	tmp = le32_to_cpu(adev->wb.wb[index]);
 	if (tmp == 0xDEADBEEF) {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	} else {
 		DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
@@ -328,7 +328,7 @@ static int si_dma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	amdgpu_ib_free(adev, &ib, NULL);
 	dma_fence_put(f);
 err0:
-	amdgpu_wb_free(adev, index);
+	amdgpu_device_wb_free(adev, index);
 	return r;
 }
 
@@ -473,25 +473,25 @@ static void si_dma_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
  * using sDMA (VI).
  */
 static void si_dma_ring_emit_vm_flush(struct amdgpu_ring *ring,
-				      unsigned vm_id, uint64_t pd_addr)
+				      unsigned vmid, uint64_t pd_addr)
 {
 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
-	if (vm_id < 8)
-		amdgpu_ring_write(ring, (0xf << 16) | (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
+	if (vmid < 8)
+		amdgpu_ring_write(ring, (0xf << 16) | (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid));
 	else
-		amdgpu_ring_write(ring, (0xf << 16) | (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vm_id - 8)));
+		amdgpu_ring_write(ring, (0xf << 16) | (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vmid - 8)));
 	amdgpu_ring_write(ring, pd_addr >> 12);
 
 	/* bits 0-7 are the VM contexts0-7 */
 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
 	amdgpu_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST));
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 
 	/* wait for invalidate to complete */
 	amdgpu_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
 	amdgpu_ring_write(ring, VM_INVALIDATE_REQUEST);
 	amdgpu_ring_write(ring, 0xff << 16); /* retry */
-	amdgpu_ring_write(ring, 1 << vm_id); /* mask */
+	amdgpu_ring_write(ring, 1 << vmid); /* mask */
 	amdgpu_ring_write(ring, 0); /* value */
 	amdgpu_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 51fd0c9a20a5..22f0b7ff3ac9 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -26,6 +26,7 @@
 #include "amdgpu_pm.h"
 #include "amdgpu_dpm.h"
 #include "amdgpu_atombios.h"
+#include "amd_pcie.h"
 #include "sid.h"
 #include "r600_dpm.h"
 #include "si_dpm.h"
@@ -3331,29 +3332,6 @@ static void btc_apply_voltage_delta_rules(struct amdgpu_device *adev,
 	}
 }
 
-static enum amdgpu_pcie_gen r600_get_pcie_gen_support(struct amdgpu_device *adev,
-					       u32 sys_mask,
-					       enum amdgpu_pcie_gen asic_gen,
-					       enum amdgpu_pcie_gen default_gen)
-{
-	switch (asic_gen) {
-	case AMDGPU_PCIE_GEN1:
-		return AMDGPU_PCIE_GEN1;
-	case AMDGPU_PCIE_GEN2:
-		return AMDGPU_PCIE_GEN2;
-	case AMDGPU_PCIE_GEN3:
-		return AMDGPU_PCIE_GEN3;
-	default:
-		if ((sys_mask & DRM_PCIE_SPEED_80) && (default_gen == AMDGPU_PCIE_GEN3))
-			return AMDGPU_PCIE_GEN3;
-		else if ((sys_mask & DRM_PCIE_SPEED_50) && (default_gen == AMDGPU_PCIE_GEN2))
-			return AMDGPU_PCIE_GEN2;
-		else
-			return AMDGPU_PCIE_GEN1;
-	}
-	return AMDGPU_PCIE_GEN1;
-}
-
 static void r600_calculate_u_and_p(u32 i, u32 r_c, u32 p_b,
 			    u32 *p, u32 *u)
 {
@@ -3464,6 +3442,11 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev,
 		    (adev->pdev->device == 0x6667)) {
 			max_sclk = 75000;
 		}
+		if ((adev->pdev->revision == 0xC3) ||
+		    (adev->pdev->device == 0x6665)) {
+			max_sclk = 60000;
+			max_mclk = 80000;
+		}
 	} else if (adev->asic_type == CHIP_OLAND) {
 		if ((adev->pdev->revision == 0xC7) ||
 		    (adev->pdev->revision == 0x80) ||
@@ -5023,10 +5006,11 @@ static int si_populate_smc_acpi_state(struct amdgpu_device *adev,
 							      table->ACPIState.levels[0].vddc.index,
 							      &table->ACPIState.levels[0].std_vddc);
 		}
-		table->ACPIState.levels[0].gen2PCIE = (u8)r600_get_pcie_gen_support(adev,
-										    si_pi->sys_pcie_mask,
-										    si_pi->boot_pcie_gen,
-										    AMDGPU_PCIE_GEN1);
+		table->ACPIState.levels[0].gen2PCIE =
+			(u8)amdgpu_get_pcie_gen_support(adev,
+							si_pi->sys_pcie_mask,
+							si_pi->boot_pcie_gen,
+							AMDGPU_PCIE_GEN1);
 
 		if (si_pi->vddc_phase_shed_control)
 			si_populate_phase_shedding_value(adev,
@@ -5845,9 +5829,9 @@ static int si_set_mc_special_registers(struct amdgpu_device *adev,
 					((temp_reg & 0xffff0000)) |
 					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
 			j++;
+
 			if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
 				return -EINVAL;
-
 			temp_reg = RREG32(MC_PMG_CMD_MRS);
 			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS;
 			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP;
@@ -5859,18 +5843,16 @@ static int si_set_mc_special_registers(struct amdgpu_device *adev,
 					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
 			}
 			j++;
-			if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
-				return -EINVAL;
 
 			if (adev->mc.vram_type != AMDGPU_VRAM_TYPE_GDDR5) {
+				if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
+					return -EINVAL;
 				table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD;
 				table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD;
 				for (k = 0; k < table->num_entries; k++)
 					table->mc_reg_table_entry[k].mc_data[j] =
 						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
 				j++;
-				if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
-					return -EINVAL;
 			}
 			break;
 		case MC_SEQ_RESERVE_M:
@@ -5882,8 +5864,6 @@ static int si_set_mc_special_registers(struct amdgpu_device *adev,
 					(temp_reg & 0xffff0000) |
 					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
 			j++;
-			if (j >= SMC_SISLANDS_MC_REGISTER_ARRAY_SIZE)
-				return -EINVAL;
 			break;
 		default:
 			break;
@@ -7167,10 +7147,10 @@ static void si_parse_pplib_clock_info(struct amdgpu_device *adev,
 	pl->vddc = le16_to_cpu(clock_info->si.usVDDC);
 	pl->vddci = le16_to_cpu(clock_info->si.usVDDCI);
 	pl->flags = le32_to_cpu(clock_info->si.ulFlags);
-	pl->pcie_gen = r600_get_pcie_gen_support(adev,
-						 si_pi->sys_pcie_mask,
-						 si_pi->boot_pcie_gen,
-						 clock_info->si.ucPCIEGen);
+	pl->pcie_gen = amdgpu_get_pcie_gen_support(adev,
+						   si_pi->sys_pcie_mask,
+						   si_pi->boot_pcie_gen,
+						   clock_info->si.ucPCIEGen);
 
 	/* patch up vddc if necessary */
 	ret = si_get_leakage_voltage_from_leakage_index(adev, pl->vddc,
@@ -7325,7 +7305,6 @@ static int si_dpm_init(struct amdgpu_device *adev)
 	struct si_power_info *si_pi;
 	struct atom_clock_dividers dividers;
 	int ret;
-	u32 mask;
 
 	si_pi = kzalloc(sizeof(struct si_power_info), GFP_KERNEL);
 	if (si_pi == NULL)
@@ -7335,11 +7314,9 @@ static int si_dpm_init(struct amdgpu_device *adev)
 	eg_pi = &ni_pi->eg;
 	pi = &eg_pi->rv7xx;
 
-	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
-	if (ret)
-		si_pi->sys_pcie_mask = 0;
-	else
-		si_pi->sys_pcie_mask = mask;
+	si_pi->sys_pcie_mask =
+		(adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_MASK) >>
+		CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
 	si_pi->force_pcie_gen = AMDGPU_PCIE_GEN_INVALID;
 	si_pi->boot_pcie_gen = si_get_current_pcie_speed(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/si_ih.c b/drivers/gpu/drm/amd/amdgpu/si_ih.c
index d2c6b80309c8..60dad63098a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_ih.c
@@ -146,7 +146,7 @@ static void si_ih_decode_iv(struct amdgpu_device *adev,
 	entry->src_id = dw[0] & 0xff;
 	entry->src_data[0] = dw[1] & 0xfffffff;
 	entry->ring_id = dw[2] & 0xff;
-	entry->vm_id = (dw[2] >> 8) & 0xff;
+	entry->vmid = (dw[2] >> 8) & 0xff;
 
 	adev->irq.ih.rptr += 16;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index 4e67fe1e7955..a04a033f57de 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -34,18 +34,17 @@
 #include "atom.h"
 #include "amd_pcie.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/UVD/uvd_7_0_offset.h"
-#include "vega10/GC/gc_9_0_offset.h"
-#include "vega10/GC/gc_9_0_sh_mask.h"
-#include "vega10/SDMA0/sdma0_4_0_offset.h"
-#include "vega10/SDMA1/sdma1_4_0_offset.h"
-#include "vega10/HDP/hdp_4_0_offset.h"
-#include "vega10/HDP/hdp_4_0_sh_mask.h"
-#include "vega10/MP/mp_9_0_offset.h"
-#include "vega10/MP/mp_9_0_sh_mask.h"
-#include "vega10/SMUIO/smuio_9_0_offset.h"
-#include "vega10/SMUIO/smuio_9_0_sh_mask.h"
+#include "uvd/uvd_7_0_offset.h"
+#include "gc/gc_9_0_offset.h"
+#include "gc/gc_9_0_sh_mask.h"
+#include "sdma0/sdma0_4_0_offset.h"
+#include "sdma1/sdma1_4_0_offset.h"
+#include "hdp/hdp_4_0_offset.h"
+#include "hdp/hdp_4_0_sh_mask.h"
+#include "mp/mp_9_0_offset.h"
+#include "mp/mp_9_0_sh_mask.h"
+#include "smuio/smuio_9_0_offset.h"
+#include "smuio/smuio_9_0_sh_mask.h"
 
 #include "soc15.h"
 #include "soc15_common.h"
@@ -101,15 +100,8 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 {
 	unsigned long flags, address, data;
 	u32 r;
-	const struct nbio_pcie_index_data *nbio_pcie_id;
-
-	if (adev->flags & AMD_IS_APU)
-		nbio_pcie_id = &nbio_v7_0_pcie_index_data;
-	else
-		nbio_pcie_id = &nbio_v6_1_pcie_index_data;
-
-	address = nbio_pcie_id->index_offset;
-	data = nbio_pcie_id->data_offset;
+	address = adev->nbio_funcs->get_pcie_index_offset(adev);
+	data = adev->nbio_funcs->get_pcie_data_offset(adev);
 
 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 	WREG32(address, reg);
@@ -122,15 +114,9 @@ static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg)
 static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 {
 	unsigned long flags, address, data;
-	const struct nbio_pcie_index_data *nbio_pcie_id;
 
-	if (adev->flags & AMD_IS_APU)
-		nbio_pcie_id = &nbio_v7_0_pcie_index_data;
-	else
-		nbio_pcie_id = &nbio_v6_1_pcie_index_data;
-
-	address = nbio_pcie_id->index_offset;
-	data = nbio_pcie_id->data_offset;
+	address = adev->nbio_funcs->get_pcie_index_offset(adev);
+	data = adev->nbio_funcs->get_pcie_data_offset(adev);
 
 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 	WREG32(address, reg);
@@ -242,41 +228,9 @@ static void soc15_se_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 
 static u32 soc15_get_config_memsize(struct amdgpu_device *adev)
 {
-	if (adev->flags & AMD_IS_APU)
-		return nbio_v7_0_get_memsize(adev);
-	else
-		return nbio_v6_1_get_memsize(adev);
+	return adev->nbio_funcs->get_memsize(adev);
 }
 
-static const u32 vega10_golden_init[] =
-{
-};
-
-static const u32 raven_golden_init[] =
-{
-};
-
-static void soc15_init_golden_registers(struct amdgpu_device *adev)
-{
-	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
-	mutex_lock(&adev->grbm_idx_mutex);
-
-	switch (adev->asic_type) {
-	case CHIP_VEGA10:
-		amdgpu_program_register_sequence(adev,
-						 vega10_golden_init,
-						 (const u32)ARRAY_SIZE(vega10_golden_init));
-		break;
-	case CHIP_RAVEN:
-		amdgpu_program_register_sequence(adev,
-						 raven_golden_init,
-						 (const u32)ARRAY_SIZE(raven_golden_init));
-		break;
-	default:
-		break;
-	}
-	mutex_unlock(&adev->grbm_idx_mutex);
-}
 static u32 soc15_get_xclk(struct amdgpu_device *adev)
 {
 	return adev->clock.spll.reference_freq;
@@ -332,25 +286,34 @@ static bool soc15_read_bios_from_rom(struct amdgpu_device *adev,
 	return true;
 }
 
-static struct amdgpu_allowed_register_entry soc15_allowed_read_registers[] = {
-	{ SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS)},
-	{ SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS2)},
-	{ SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE0)},
-	{ SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE1)},
-	{ SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE2)},
-	{ SOC15_REG_OFFSET(GC, 0, mmGRBM_STATUS_SE3)},
-	{ SOC15_REG_OFFSET(SDMA0, 0, mmSDMA0_STATUS_REG)},
-	{ SOC15_REG_OFFSET(SDMA1, 0, mmSDMA1_STATUS_REG)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_STAT)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT1)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT2)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_STALLED_STAT3)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_CPF_BUSY_STAT)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_CPF_STALLED_STAT1)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_CPF_STATUS)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_CPC_STALLED_STAT1)},
-	{ SOC15_REG_OFFSET(GC, 0, mmCP_CPC_STATUS)},
-	{ SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG)},
+struct soc15_allowed_register_entry {
+	uint32_t hwip;
+	uint32_t inst;
+	uint32_t seg;
+	uint32_t reg_offset;
+	bool grbm_indexed;
+};
+
+
+static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = {
+	{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS)},
+	{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS2)},
+	{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE0)},
+	{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE1)},
+	{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE2)},
+	{ SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE3)},
+	{ SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_STATUS_REG)},
+	{ SOC15_REG_ENTRY(SDMA1, 0, mmSDMA1_STATUS_REG)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_STAT)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT1)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT2)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT3)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_CPF_BUSY_STAT)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STALLED_STAT1)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STATUS)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STALLED_STAT1)},
+	{ SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STATUS)},
+	{ SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)},
 };
 
 static uint32_t soc15_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
@@ -377,12 +340,9 @@ static uint32_t soc15_get_register_value(struct amdgpu_device *adev,
 	if (indexed) {
 		return soc15_read_indexed_register(adev, se_num, sh_num, reg_offset);
 	} else {
-		switch (reg_offset) {
-		case SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG):
+		if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
 			return adev->gfx.config.gb_addr_config;
-		default:
-			return RREG32(reg_offset);
-		}
+		return RREG32(reg_offset);
 	}
 }
 
@@ -390,10 +350,13 @@ static int soc15_read_register(struct amdgpu_device *adev, u32 se_num,
 			    u32 sh_num, u32 reg_offset, u32 *value)
 {
 	uint32_t i;
+	struct soc15_allowed_register_entry  *en;
 
 	*value = 0;
 	for (i = 0; i < ARRAY_SIZE(soc15_allowed_read_registers); i++) {
-		if (reg_offset != soc15_allowed_read_registers[i].reg_offset)
+		en = &soc15_allowed_read_registers[i];
+		if (reg_offset != (adev->reg_offset[en->hwip][en->inst][en->seg]
+					+ en->reg_offset))
 			continue;
 
 		*value = soc15_get_register_value(adev,
@@ -404,6 +367,43 @@ static int soc15_read_register(struct amdgpu_device *adev, u32 se_num,
 	return -EINVAL;
 }
 
+
+/**
+ * soc15_program_register_sequence - program an array of registers.
+ *
+ * @adev: amdgpu_device pointer
+ * @regs: pointer to the register array
+ * @array_size: size of the register array
+ *
+ * Programs an array or registers with and and or masks.
+ * This is a helper for setting golden registers.
+ */
+
+void soc15_program_register_sequence(struct amdgpu_device *adev,
+					     const struct soc15_reg_golden *regs,
+					     const u32 array_size)
+{
+	const struct soc15_reg_golden *entry;
+	u32 tmp, reg;
+	int i;
+
+	for (i = 0; i < array_size; ++i) {
+		entry = &regs[i];
+		reg =  adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
+
+		if (entry->and_mask == 0xffffffff) {
+			tmp = entry->or_mask;
+		} else {
+			tmp = RREG32(reg);
+			tmp &= ~(entry->and_mask);
+			tmp |= entry->or_mask;
+		}
+		WREG32(reg, tmp);
+	}
+
+}
+
+
 static int soc15_asic_reset(struct amdgpu_device *adev)
 {
 	u32 i;
@@ -428,9 +428,8 @@ static int soc15_asic_reset(struct amdgpu_device *adev)
 
 	/* wait for asic to come out of reset */
 	for (i = 0; i < adev->usec_timeout; i++) {
-		u32 memsize = (adev->flags & AMD_IS_APU) ?
-			nbio_v7_0_get_memsize(adev) :
-			nbio_v6_1_get_memsize(adev);
+		u32 memsize = adev->nbio_funcs->get_memsize(adev);
+
 		if (memsize != 0xffffffff)
 			break;
 		udelay(1);
@@ -495,14 +494,10 @@ static void soc15_program_aspm(struct amdgpu_device *adev)
 }
 
 static void soc15_enable_doorbell_aperture(struct amdgpu_device *adev,
-					bool enable)
+					   bool enable)
 {
-	if (adev->flags & AMD_IS_APU) {
-		nbio_v7_0_enable_doorbell_aperture(adev, enable);
-	} else {
-		nbio_v6_1_enable_doorbell_aperture(adev, enable);
-		nbio_v6_1_enable_doorbell_selfring_aperture(adev, enable);
-	}
+	adev->nbio_funcs->enable_doorbell_aperture(adev, enable);
+	adev->nbio_funcs->enable_doorbell_selfring_aperture(adev, enable);
 }
 
 static const struct amdgpu_ip_block_version vega10_common_ip_block =
@@ -516,50 +511,65 @@ static const struct amdgpu_ip_block_version vega10_common_ip_block =
 
 int soc15_set_ip_blocks(struct amdgpu_device *adev)
 {
-	nbio_v6_1_detect_hw_virt(adev);
+	/* Set IP register base before any HW register access */
+	switch (adev->asic_type) {
+	case CHIP_VEGA10:
+	case CHIP_RAVEN:
+		vega10_reg_base_init(adev);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (adev->flags & AMD_IS_APU)
+		adev->nbio_funcs = &nbio_v7_0_funcs;
+	else
+		adev->nbio_funcs = &nbio_v6_1_funcs;
+
+	adev->nbio_funcs->detect_hw_virt(adev);
 
 	if (amdgpu_sriov_vf(adev))
 		adev->virt.ops = &xgpu_ai_virt_ops;
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
-		amdgpu_ip_block_add(adev, &vega10_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v9_0_ip_block);
-		amdgpu_ip_block_add(adev, &vega10_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
 		if (amdgpu_fw_load_type == 2 || amdgpu_fw_load_type == -1)
-			amdgpu_ip_block_add(adev, &psp_v3_1_ip_block);
+			amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block);
 		if (!amdgpu_sriov_vf(adev))
-			amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+			amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #else
 #	warning "Enable CONFIG_DRM_AMD_DC for display support on SOC15."
 #endif
-		amdgpu_ip_block_add(adev, &gfx_v9_0_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v4_0_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v7_0_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v4_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v7_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v4_0_ip_block);
 		break;
 	case CHIP_RAVEN:
-		amdgpu_ip_block_add(adev, &vega10_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v9_0_ip_block);
-		amdgpu_ip_block_add(adev, &vega10_ih_ip_block);
-		amdgpu_ip_block_add(adev, &psp_v10_0_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &vega10_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &psp_v10_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #else
 #	warning "Enable CONFIG_DRM_AMD_DC for display support on SOC15."
 #endif
-		amdgpu_ip_block_add(adev, &gfx_v9_0_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v4_0_ip_block);
-		amdgpu_ip_block_add(adev, &vcn_v1_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &vcn_v1_0_ip_block);
 		break;
 	default:
 		return -EINVAL;
@@ -570,10 +580,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
 
 static uint32_t soc15_get_rev_id(struct amdgpu_device *adev)
 {
-	if (adev->flags & AMD_IS_APU)
-		return nbio_v7_0_get_rev_id(adev);
-	else
-		return nbio_v6_1_get_rev_id(adev);
+	return adev->nbio_funcs->get_rev_id(adev);
 }
 
 static const struct amdgpu_asic_funcs soc15_asic_funcs =
@@ -609,8 +616,8 @@ static int soc15_common_early_init(void *handle)
 
 	adev->asic_funcs = &soc15_asic_funcs;
 
-	if (amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP) &&
-		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_PSP)))
+	if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP) &&
+	    (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_PSP)))
 		psp_enabled = true;
 
 	adev->rev_id = soc15_get_rev_id(adev);
@@ -659,8 +666,8 @@ static int soc15_common_early_init(void *handle)
 			AMD_CG_SUPPORT_MC_LS |
 			AMD_CG_SUPPORT_SDMA_MGCG |
 			AMD_CG_SUPPORT_SDMA_LS;
-		adev->pg_flags = AMD_PG_SUPPORT_SDMA |
-				 AMD_PG_SUPPORT_MMHUB;
+		adev->pg_flags = AMD_PG_SUPPORT_SDMA;
+
 		adev->external_rev_id = 0x1;
 		break;
 	default:
@@ -675,7 +682,7 @@ static int soc15_common_early_init(void *handle)
 
 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
 
-	amdgpu_get_pcie_info(adev);
+	amdgpu_device_get_pcie_info(adev);
 
 	return 0;
 }
@@ -709,15 +716,12 @@ static int soc15_common_hw_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	/* move the golden regs per IP block */
-	soc15_init_golden_registers(adev);
 	/* enable pcie gen2/3 link */
 	soc15_pcie_gen3_enable(adev);
 	/* enable aspm */
 	soc15_program_aspm(adev);
 	/* setup nbio registers */
-	if (!(adev->flags & AMD_IS_APU))
-		nbio_v6_1_init_registers(adev);
+	adev->nbio_funcs->init_registers(adev);
 	/* enable the doorbell aperture */
 	soc15_enable_doorbell_aperture(adev, true);
 
@@ -878,9 +882,9 @@ static int soc15_common_set_clockgating_state(void *handle,
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
-		nbio_v6_1_update_medium_grain_clock_gating(adev,
+		adev->nbio_funcs->update_medium_grain_clock_gating(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
-		nbio_v6_1_update_medium_grain_light_sleep(adev,
+		adev->nbio_funcs->update_medium_grain_light_sleep(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
 		soc15_update_hdp_light_sleep(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
@@ -894,9 +898,9 @@ static int soc15_common_set_clockgating_state(void *handle,
 				state == AMD_CG_STATE_GATE ? true : false);
 		break;
 	case CHIP_RAVEN:
-		nbio_v7_0_update_medium_grain_clock_gating(adev,
+		adev->nbio_funcs->update_medium_grain_clock_gating(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
-		nbio_v6_1_update_medium_grain_light_sleep(adev,
+		adev->nbio_funcs->update_medium_grain_light_sleep(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
 		soc15_update_hdp_light_sleep(adev,
 				state == AMD_CG_STATE_GATE ? true : false);
@@ -921,7 +925,7 @@ static void soc15_common_get_clockgating_state(void *handle, u32 *flags)
 	if (amdgpu_sriov_vf(adev))
 		*flags = 0;
 
-	nbio_v6_1_get_clockgating_state(adev, flags);
+	adev->nbio_funcs->get_clockgating_state(adev, flags);
 
 	/* AMD_CG_SUPPORT_HDP_LS */
 	data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS));
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.h b/drivers/gpu/drm/amd/amdgpu/soc15.h
index acb3cdb119f2..26b3feac5d06 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.h
@@ -29,8 +29,28 @@
 
 extern const struct amd_ip_funcs soc15_common_ip_funcs;
 
+struct soc15_reg_golden {
+	u32	hwip;
+	u32	instance;
+	u32	segment;
+	u32	reg;
+	u32	and_mask;
+	u32	or_mask;
+};
+
+#define SOC15_REG_ENTRY(ip, inst, reg)	ip##_HWIP, inst, reg##_BASE_IDX, reg
+
+#define SOC15_REG_GOLDEN_VALUE(ip, inst, reg, and_mask, or_mask) \
+	{ ip##_HWIP, inst, reg##_BASE_IDX, reg, and_mask, or_mask }
+
 void soc15_grbm_select(struct amdgpu_device *adev,
 		    u32 me, u32 pipe, u32 queue, u32 vmid);
 int soc15_set_ip_blocks(struct amdgpu_device *adev);
 
+void soc15_program_register_sequence(struct amdgpu_device *adev,
+					     const struct soc15_reg_golden *registers,
+					     const u32 array_size);
+
+int vega10_reg_base_init(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15_common.h b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
index 7a8e4e28abb2..def865067edd 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15_common.h
+++ b/drivers/gpu/drm/amd/amdgpu/soc15_common.h
@@ -24,72 +24,28 @@
 #ifndef __SOC15_COMMON_H__
 #define __SOC15_COMMON_H__
 
-struct nbio_hdp_flush_reg {
-	u32 hdp_flush_req_offset;
-	u32 hdp_flush_done_offset;
-	u32 ref_and_mask_cp0;
-	u32 ref_and_mask_cp1;
-	u32 ref_and_mask_cp2;
-	u32 ref_and_mask_cp3;
-	u32 ref_and_mask_cp4;
-	u32 ref_and_mask_cp5;
-	u32 ref_and_mask_cp6;
-	u32 ref_and_mask_cp7;
-	u32 ref_and_mask_cp8;
-	u32 ref_and_mask_cp9;
-	u32 ref_and_mask_sdma0;
-	u32 ref_and_mask_sdma1;
-};
-
-struct nbio_pcie_index_data {
-	u32 index_offset;
-	u32 data_offset;
-};
-
 /* Register Access Macros */
-#define SOC15_REG_OFFSET(ip, inst, reg)       (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
-                                                (1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
-                                                    (2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
-                                                        (3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
-                                                            (ip##_BASE__INST##inst##_SEG4 + reg)))))
+#define SOC15_REG_OFFSET(ip, inst, reg)	(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg)
 
 #define WREG32_FIELD15(ip, idx, reg, field, val)	\
-	WREG32(SOC15_REG_OFFSET(ip, idx, mm##reg), (RREG32(SOC15_REG_OFFSET(ip, idx, mm##reg)) & ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
+	WREG32(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg,	\
+	(RREG32(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg)	\
+	& ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field))
 
 #define RREG32_SOC15(ip, inst, reg) \
-	RREG32( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
-		(1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
-		(2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
-		(3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
-		(ip##_BASE__INST##inst##_SEG4 + reg))))))
+	RREG32(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg)
 
 #define RREG32_SOC15_OFFSET(ip, inst, reg, offset) \
-	RREG32( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
-		(1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
-		(2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
-		(3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
-		(ip##_BASE__INST##inst##_SEG4 + reg))))) + offset)
+	RREG32((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset)
 
 #define WREG32_SOC15(ip, inst, reg, value) \
-	WREG32( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
-		(1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
-		(2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
-		(3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
-		(ip##_BASE__INST##inst##_SEG4 + reg))))), value)
+	WREG32((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg), value)
 
 #define WREG32_SOC15_NO_KIQ(ip, inst, reg, value) \
-	WREG32_NO_KIQ( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
-		(1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
-		(2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
-		(3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
-		(ip##_BASE__INST##inst##_SEG4 + reg))))), value)
+	WREG32_NO_KIQ((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg), value)
 
 #define WREG32_SOC15_OFFSET(ip, inst, reg, offset, value) \
-	WREG32( (0 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG0 + reg : \
-		(1 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG1 + reg : \
-		(2 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG2 + reg : \
-		(3 == reg##_BASE_IDX ? ip##_BASE__INST##inst##_SEG3 + reg : \
-		(ip##_BASE__INST##inst##_SEG4 + reg))))) + offset, value)
+	WREG32((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset, value)
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
index aa4e320e31f8..5995ffc183de 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c
@@ -270,7 +270,7 @@ static void tonga_ih_decode_iv(struct amdgpu_device *adev,
 	entry->src_id = dw[0] & 0xff;
 	entry->src_data[0] = dw[1] & 0xfffffff;
 	entry->ring_id = dw[2] & 0xff;
-	entry->vm_id = (dw[2] >> 8) & 0xff;
+	entry->vmid = (dw[2] >> 8) & 0xff;
 	entry->pas_id = (dw[2] >> 16) & 0xffff;
 
 	/* wptr/rptr are in bytes! */
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index 8ab0f78794a5..8ab10c220910 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -521,7 +521,7 @@ static int uvd_v4_2_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
@@ -541,7 +541,7 @@ static int uvd_v4_2_ring_test_ring(struct amdgpu_ring *ring)
  */
 static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring,
 				  struct amdgpu_ib *ib,
-				  unsigned vm_id, bool ctx_switch)
+				  unsigned vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, PACKET0(mmUVD_RBC_IB_BASE, 0));
 	amdgpu_ring_write(ring, ib->gpu_addr);
@@ -563,7 +563,7 @@ static void uvd_v4_2_mc_resume(struct amdgpu_device *adev)
 
 	/* programm the VCPU memory controller bits 0-27 */
 	addr = (adev->uvd.gpu_addr + AMDGPU_UVD_FIRMWARE_OFFSET) >> 3;
-	size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4) >> 3;
+	size = AMDGPU_UVD_FIRMWARE_SIZE(adev) >> 3;
 	WREG32(mmUVD_VCPU_CACHE_OFFSET0, addr);
 	WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index bb6d46e168a3..c1fe30cdba32 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -258,7 +258,7 @@ static void uvd_v5_0_mc_resume(struct amdgpu_device *adev)
 			upper_32_bits(adev->uvd.gpu_addr));
 
 	offset = AMDGPU_UVD_FIRMWARE_OFFSET;
-	size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
+	size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
 	WREG32(mmUVD_VCPU_CACHE_OFFSET0, offset >> 3);
 	WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
 
@@ -536,7 +536,7 @@ static int uvd_v5_0_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
@@ -556,7 +556,7 @@ static int uvd_v5_0_ring_test_ring(struct amdgpu_ring *ring)
  */
 static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring,
 				  struct amdgpu_ib *ib,
-				  unsigned vm_id, bool ctx_switch)
+				  unsigned vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_LOW, 0));
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index 920910ac8663..9bab4842cd44 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -37,6 +37,9 @@
 #include "gmc/gmc_8_1_d.h"
 #include "vi.h"
 
+/* Polaris10/11/12 firmware version */
+#define FW_1_130_16 ((1 << 24) | (130 << 16) | (16 << 8))
+
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev);
 static void uvd_v6_0_set_enc_ring_funcs(struct amdgpu_device *adev);
 
@@ -58,7 +61,9 @@ static void uvd_v6_0_enable_mgcg(struct amdgpu_device *adev,
 */
 static inline bool uvd_v6_0_enc_support(struct amdgpu_device *adev)
 {
-	return ((adev->asic_type >= CHIP_POLARIS10) && (adev->asic_type <= CHIP_POLARIS12));
+	return ((adev->asic_type >= CHIP_POLARIS10) &&
+			(adev->asic_type <= CHIP_POLARIS12) &&
+			(!adev->uvd.fw_version || adev->uvd.fw_version >= FW_1_130_16));
 }
 
 /**
@@ -184,7 +189,7 @@ static int uvd_v6_0_enc_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -360,7 +365,7 @@ static int uvd_v6_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	} else if (r < 0) {
 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 	} else {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	}
 error:
@@ -411,12 +416,20 @@ static int uvd_v6_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	if (uvd_v6_0_enc_support(adev)) {
-		struct amd_sched_rq *rq;
+	if (!uvd_v6_0_enc_support(adev)) {
+		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
+			adev->uvd.ring_enc[i].funcs = NULL;
+
+		adev->uvd.irq.num_types = 1;
+		adev->uvd.num_enc_rings = 0;
+
+		DRM_INFO("UVD ENC is disabled\n");
+	} else {
+		struct drm_sched_rq *rq;
 		ring = &adev->uvd.ring_enc[0];
-		rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
-		r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
-					  rq, amdgpu_sched_jobs);
+		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+		r = drm_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
+					  rq, amdgpu_sched_jobs, NULL);
 		if (r) {
 			DRM_ERROR("Failed setting up UVD ENC run queue.\n");
 			return r;
@@ -456,7 +469,7 @@ static int uvd_v6_0_sw_fini(void *handle)
 		return r;
 
 	if (uvd_v6_0_enc_support(adev)) {
-		amd_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
+		drm_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
 
 		for (i = 0; i < adev->uvd.num_enc_rings; ++i)
 			amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
@@ -603,7 +616,7 @@ static void uvd_v6_0_mc_resume(struct amdgpu_device *adev)
 			upper_32_bits(adev->uvd.gpu_addr));
 
 	offset = AMDGPU_UVD_FIRMWARE_OFFSET;
-	size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
+	size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
 	WREG32(mmUVD_VCPU_CACHE_OFFSET0, offset >> 3);
 	WREG32(mmUVD_VCPU_CACHE_SIZE0, size);
 
@@ -1008,7 +1021,7 @@ static int uvd_v6_0_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
@@ -1028,10 +1041,10 @@ static int uvd_v6_0_ring_test_ring(struct amdgpu_ring *ring)
  */
 static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
 				  struct amdgpu_ib *ib,
-				  unsigned vm_id, bool ctx_switch)
+				  unsigned vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_VMID, 0));
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 
 	amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_LOW, 0));
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
@@ -1050,24 +1063,24 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
  * Write enc ring commands to execute the indirect buffer
  */
 static void uvd_v6_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
-		struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+		struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
 static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					 unsigned vm_id, uint64_t pd_addr)
+					 unsigned vmid, uint64_t pd_addr)
 {
 	uint32_t reg;
 
-	if (vm_id < 8)
-		reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id;
+	if (vmid < 8)
+		reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid;
 	else
-		reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8;
+		reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8;
 
 	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
 	amdgpu_ring_write(ring, reg << 2);
@@ -1079,7 +1092,7 @@ static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
 	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
 	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
 	amdgpu_ring_write(ring, 0x8);
 
@@ -1088,7 +1101,7 @@ static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
 	amdgpu_ring_write(ring, 0);
 	amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8, 0));
-	amdgpu_ring_write(ring, 1 << vm_id); /* mask */
+	amdgpu_ring_write(ring, 1 << vmid); /* mask */
 	amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
 	amdgpu_ring_write(ring, 0xC);
 }
@@ -1127,14 +1140,14 @@ static void uvd_v6_0_enc_ring_insert_end(struct amdgpu_ring *ring)
 }
 
 static void uvd_v6_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
-        unsigned int vm_id, uint64_t pd_addr)
+        unsigned int vmid, uint64_t pd_addr)
 {
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_UPDATE_PTB);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, pd_addr >> 12);
 
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_FLUSH_TLB);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 }
 
 static bool uvd_v6_0_is_idle(void *handle)
@@ -1605,7 +1618,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_enc_ring_vm_funcs = {
 	.set_wptr = uvd_v6_0_enc_ring_set_wptr,
 	.emit_frame_size =
 		4 + /* uvd_v6_0_enc_ring_emit_pipeline_sync */
-		6 + /* uvd_v6_0_enc_ring_emit_vm_flush */
+		5 + /* uvd_v6_0_enc_ring_emit_vm_flush */
 		5 + 5 + /* uvd_v6_0_enc_ring_emit_fence x2 vm fence */
 		1, /* uvd_v6_0_enc_ring_insert_end */
 	.emit_ib_size = 5, /* uvd_v6_0_enc_ring_emit_ib */
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
index 6634545060fd..6b95f4f344b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c
@@ -29,16 +29,15 @@
 #include "soc15_common.h"
 #include "mmsch_v1_0.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/UVD/uvd_7_0_offset.h"
-#include "vega10/UVD/uvd_7_0_sh_mask.h"
-#include "vega10/VCE/vce_4_0_offset.h"
-#include "vega10/VCE/vce_4_0_default.h"
-#include "vega10/VCE/vce_4_0_sh_mask.h"
-#include "vega10/NBIF/nbif_6_1_offset.h"
-#include "vega10/HDP/hdp_4_0_offset.h"
-#include "vega10/MMHUB/mmhub_1_0_offset.h"
-#include "vega10/MMHUB/mmhub_1_0_sh_mask.h"
+#include "uvd/uvd_7_0_offset.h"
+#include "uvd/uvd_7_0_sh_mask.h"
+#include "vce/vce_4_0_offset.h"
+#include "vce/vce_4_0_default.h"
+#include "vce/vce_4_0_sh_mask.h"
+#include "nbif/nbif_6_1_offset.h"
+#include "hdp/hdp_4_0_offset.h"
+#include "mmhub/mmhub_1_0_offset.h"
+#include "mmhub/mmhub_1_0_sh_mask.h"
 
 static void uvd_v7_0_set_ring_funcs(struct amdgpu_device *adev);
 static void uvd_v7_0_set_enc_ring_funcs(struct amdgpu_device *adev);
@@ -184,7 +183,7 @@ static int uvd_v7_0_enc_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed\n",
@@ -359,7 +358,7 @@ static int uvd_v7_0_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	} else if (r < 0) {
 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
 	} else {
-		DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
 		r = 0;
 	}
 error:
@@ -385,7 +384,7 @@ static int uvd_v7_0_early_init(void *handle)
 static int uvd_v7_0_sw_init(void *handle)
 {
 	struct amdgpu_ring *ring;
-	struct amd_sched_rq *rq;
+	struct drm_sched_rq *rq;
 	int i, r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
@@ -416,9 +415,9 @@ static int uvd_v7_0_sw_init(void *handle)
 	}
 
 	ring = &adev->uvd.ring_enc[0];
-	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_NORMAL];
-	r = amd_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
-				  rq, amdgpu_sched_jobs);
+	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+	r = drm_sched_entity_init(&ring->sched, &adev->uvd.entity_enc,
+				  rq, amdgpu_sched_jobs, NULL);
 	if (r) {
 		DRM_ERROR("Failed setting up UVD ENC run queue.\n");
 		return r;
@@ -472,7 +471,7 @@ static int uvd_v7_0_sw_fini(void *handle)
 	if (r)
 		return r;
 
-	amd_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
+	drm_sched_entity_fini(&adev->uvd.ring_enc[0].sched, &adev->uvd.entity_enc);
 
 	for (i = 0; i < adev->uvd.num_enc_rings; ++i)
 		amdgpu_ring_fini(&adev->uvd.ring_enc[i]);
@@ -616,7 +615,7 @@ static int uvd_v7_0_resume(void *handle)
  */
 static void uvd_v7_0_mc_resume(struct amdgpu_device *adev)
 {
-	uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->uvd.fw->size + 4);
+	uint32_t size = AMDGPU_UVD_FIRMWARE_SIZE(adev);
 	uint32_t offset;
 
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
@@ -1086,6 +1085,8 @@ static void uvd_v7_0_stop(struct amdgpu_device *adev)
 static void uvd_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 				     unsigned flags)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 
 	amdgpu_ring_write(ring,
@@ -1123,6 +1124,7 @@ static void uvd_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
 static void uvd_v7_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 			u64 seq, unsigned flags)
 {
+
 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_FENCE);
@@ -1141,6 +1143,8 @@ static void uvd_v7_0_enc_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
  */
 static void uvd_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(NBIF, 0,
 		mmHDP_MEM_COHERENCY_FLUSH_CNTL), 0));
 	amdgpu_ring_write(ring, 0);
@@ -1155,6 +1159,8 @@ static void uvd_v7_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
  */
 static void uvd_v7_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
 	amdgpu_ring_write(ring, 1);
 }
@@ -1192,7 +1198,7 @@ static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)
 	}
 
 	if (i < adev->usec_timeout) {
-		DRM_INFO("ring test on %d succeeded in %d usecs\n",
+		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
 			 ring->idx, i);
 	} else {
 		DRM_ERROR("amdgpu: ring %d test failed (0x%08X)\n",
@@ -1212,11 +1218,13 @@ static int uvd_v7_0_ring_test_ring(struct amdgpu_ring *ring)
  */
 static void uvd_v7_0_ring_emit_ib(struct amdgpu_ring *ring,
 				  struct amdgpu_ib *ib,
-				  unsigned vm_id, bool ctx_switch)
+				  unsigned vmid, bool ctx_switch)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
@@ -1238,10 +1246,10 @@ static void uvd_v7_0_ring_emit_ib(struct amdgpu_ring *ring,
  * Write enc ring commands to execute the indirect buffer
  */
 static void uvd_v7_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
-		struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+		struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_IB_VM);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, ib->length_dw);
@@ -1250,6 +1258,8 @@ static void uvd_v7_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
 static void uvd_v7_0_vm_reg_write(struct amdgpu_ring *ring,
 				uint32_t data0, uint32_t data1)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, data0);
@@ -1264,6 +1274,8 @@ static void uvd_v7_0_vm_reg_write(struct amdgpu_ring *ring,
 static void uvd_v7_0_vm_reg_wait(struct amdgpu_ring *ring,
 				uint32_t data0, uint32_t data1, uint32_t mask)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, data0);
@@ -1279,25 +1291,26 @@ static void uvd_v7_0_vm_reg_wait(struct amdgpu_ring *ring,
 }
 
 static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					unsigned vm_id, uint64_t pd_addr)
+					unsigned vmid, uint64_t pd_addr)
 {
 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
-	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
-	uint32_t data0, data1, mask;
+	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
+	uint64_t flags = AMDGPU_PTE_VALID;
 	unsigned eng = ring->vm_inv_eng;
+	uint32_t data0, data1, mask;
 
-	pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
-	pd_addr |= AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
+	pd_addr |= flags;
 
-	data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2;
+	data0 = (hub->ctx0_ptb_addr_hi32 + vmid * 2) << 2;
 	data1 = upper_32_bits(pd_addr);
 	uvd_v7_0_vm_reg_write(ring, data0, data1);
 
-	data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
+	data0 = (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2;
 	data1 = lower_32_bits(pd_addr);
 	uvd_v7_0_vm_reg_write(ring, data0, data1);
 
-	data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
+	data0 = (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2;
 	data1 = lower_32_bits(pd_addr);
 	mask = 0xffffffff;
 	uvd_v7_0_vm_reg_wait(ring, data0, data1, mask);
@@ -1309,36 +1322,47 @@ static void uvd_v7_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
 
 	/* wait for flush */
 	data0 = (hub->vm_inv_eng0_ack + eng) << 2;
-	data1 = 1 << vm_id;
-	mask =  1 << vm_id;
+	data1 = 1 << vmid;
+	mask =  1 << vmid;
 	uvd_v7_0_vm_reg_wait(ring, data0, data1, mask);
 }
 
+static void uvd_v7_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+	int i;
+	struct amdgpu_device *adev = ring->adev;
+
+	for (i = 0; i < count; i++)
+		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
+
+}
+
 static void uvd_v7_0_enc_ring_insert_end(struct amdgpu_ring *ring)
 {
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_END);
 }
 
 static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
-			 unsigned int vm_id, uint64_t pd_addr)
+			 unsigned int vmid, uint64_t pd_addr)
 {
 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
-	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
+	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
+	uint64_t flags = AMDGPU_PTE_VALID;
 	unsigned eng = ring->vm_inv_eng;
 
-	pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
-	pd_addr |= AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
+	pd_addr |= flags;
 
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
-	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2);
+	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_hi32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, upper_32_bits(pd_addr));
 
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WRITE);
-	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
+	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, lower_32_bits(pd_addr));
 
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);
-	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
+	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, 0xffffffff);
 	amdgpu_ring_write(ring, lower_32_bits(pd_addr));
 
@@ -1350,8 +1374,8 @@ static void uvd_v7_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	/* wait for flush */
 	amdgpu_ring_write(ring, HEVC_ENC_CMD_REG_WAIT);
 	amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
-	amdgpu_ring_write(ring, 1 << vm_id);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
+	amdgpu_ring_write(ring, 1 << vmid);
 }
 
 #if 0
@@ -1681,7 +1705,7 @@ const struct amd_ip_funcs uvd_v7_0_ip_funcs = {
 static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_UVD,
 	.align_mask = 0xf,
-	.nop = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0),
+	.nop = PACKET0(0x81ff, 0),
 	.support_64bit_ptrs = false,
 	.vmhub = AMDGPU_MMHUB,
 	.get_rptr = uvd_v7_0_ring_get_rptr,
@@ -1700,7 +1724,7 @@ static const struct amdgpu_ring_funcs uvd_v7_0_ring_vm_funcs = {
 	.emit_hdp_invalidate = uvd_v7_0_ring_emit_hdp_invalidate,
 	.test_ring = uvd_v7_0_ring_test_ring,
 	.test_ib = amdgpu_uvd_ring_test_ib,
-	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_nop = uvd_v7_0_ring_insert_nop,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.begin_use = amdgpu_uvd_ring_begin_use,
 	.end_use = amdgpu_uvd_ring_end_use,
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index cf81065e3c5a..a5355eb689f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -834,24 +834,24 @@ static void vce_v3_0_get_clockgating_state(void *handle, u32 *flags)
 }
 
 static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
-		struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+		struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, VCE_CMD_IB_VM);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
 static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring,
-			 unsigned int vm_id, uint64_t pd_addr)
+			 unsigned int vmid, uint64_t pd_addr)
 {
 	amdgpu_ring_write(ring, VCE_CMD_UPDATE_PTB);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, pd_addr >> 12);
 
 	amdgpu_ring_write(ring, VCE_CMD_FLUSH_TLB);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, VCE_CMD_END);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
old mode 100644
new mode 100755
index 75745544600a..7cf2eef68cf2
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -32,12 +32,11 @@
 #include "soc15_common.h"
 #include "mmsch_v1_0.h"
 
-#include "vega10/soc15ip.h"
-#include "vega10/VCE/vce_4_0_offset.h"
-#include "vega10/VCE/vce_4_0_default.h"
-#include "vega10/VCE/vce_4_0_sh_mask.h"
-#include "vega10/MMHUB/mmhub_1_0_offset.h"
-#include "vega10/MMHUB/mmhub_1_0_sh_mask.h"
+#include "vce/vce_4_0_offset.h"
+#include "vce/vce_4_0_default.h"
+#include "vce/vce_4_0_sh_mask.h"
+#include "mmhub/mmhub_1_0_offset.h"
+#include "mmhub/mmhub_1_0_sh_mask.h"
 
 #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
 
@@ -243,37 +242,49 @@ static int vce_v4_0_sriov_start(struct amdgpu_device *adev)
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);
 
 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		    MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
-						adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
-		    MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
-						adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
-		    MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
 						adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
+						(adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 40) & 0xff);
 		} else {
-		    MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
 						adev->vce.gpu_addr >> 8);
-		    MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
+			MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
+						(adev->vce.gpu_addr >> 40) & 0xff);
+		}
+		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
 						adev->vce.gpu_addr >> 8);
-		    MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
+		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_64BIT_BAR1),
+						(adev->vce.gpu_addr >> 40) & 0xff);
+		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
 						adev->vce.gpu_addr >> 8);
-		}
+		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
+						mmVCE_LMI_VCPU_CACHE_64BIT_BAR2),
+						(adev->vce.gpu_addr >> 40) & 0xff);
 
 		offset = AMDGPU_VCE_FIRMWARE_OFFSET;
 		size = VCE_V4_0_FW_SIZE;
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0),
-					    offset & 0x7FFFFFFF);
+					offset & ~0x0f000000);
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);
 
-		offset += size;
+		offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;
 		size = VCE_V4_0_STACK_SIZE;
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1),
-					    offset & 0x7FFFFFFF);
+					(offset & ~0x0f000000) | (1 << 24));
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);
 
 		offset += size;
 		size = VCE_V4_0_DATA_SIZE;
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2),
-					    offset & 0x7FFFFFFF);
+					(offset & ~0x0f000000) | (2 << 24));
 		MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);
 
 		MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0);
@@ -412,7 +423,7 @@ static int vce_v4_0_sw_init(void *handle)
 	if (r)
 		return r;
 
-	size  = (VCE_V4_0_STACK_SIZE + VCE_V4_0_DATA_SIZE) * 2;
+	size  = VCE_V4_0_STACK_SIZE + VCE_V4_0_DATA_SIZE;
 	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
 		size += VCE_V4_0_FW_SIZE;
 
@@ -927,10 +938,10 @@ static int vce_v4_0_set_powergating_state(void *handle,
 #endif
 
 static void vce_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
-		struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+		struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, VCE_CMD_IB_VM);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, ib->length_dw);
@@ -954,25 +965,26 @@ static void vce_v4_0_ring_insert_end(struct amdgpu_ring *ring)
 }
 
 static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
-			 unsigned int vm_id, uint64_t pd_addr)
+			 unsigned int vmid, uint64_t pd_addr)
 {
 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
-	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
+	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
+	uint64_t flags = AMDGPU_PTE_VALID;
 	unsigned eng = ring->vm_inv_eng;
 
-	pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
-	pd_addr |= AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
+	pd_addr |= flags;
 
 	amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
-	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2);
+	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_hi32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, upper_32_bits(pd_addr));
 
 	amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
-	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
+	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, lower_32_bits(pd_addr));
 
 	amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);
-	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
+	amdgpu_ring_write(ring,	(hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, 0xffffffff);
 	amdgpu_ring_write(ring, lower_32_bits(pd_addr));
 
@@ -984,8 +996,8 @@ static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
 	/* wait for flush */
 	amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);
 	amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
-	amdgpu_ring_write(ring, 1 << vm_id);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
+	amdgpu_ring_write(ring, 1 << vmid);
 }
 
 static int vce_v4_0_set_interrupt_state(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
index 0450ac5ba6b6..b99e15c43e45 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c
@@ -28,12 +28,11 @@
 #include "soc15d.h"
 #include "soc15_common.h"
 
-#include "vega10/soc15ip.h"
-#include "raven1/VCN/vcn_1_0_offset.h"
-#include "raven1/VCN/vcn_1_0_sh_mask.h"
-#include "vega10/HDP/hdp_4_0_offset.h"
-#include "raven1/MMHUB/mmhub_9_1_offset.h"
-#include "raven1/MMHUB/mmhub_9_1_sh_mask.h"
+#include "vcn/vcn_1_0_offset.h"
+#include "vcn/vcn_1_0_sh_mask.h"
+#include "hdp/hdp_4_0_offset.h"
+#include "mmhub/mmhub_9_1_offset.h"
+#include "mmhub/mmhub_9_1_sh_mask.h"
 
 static int vcn_v1_0_start(struct amdgpu_device *adev);
 static int vcn_v1_0_stop(struct amdgpu_device *adev);
@@ -744,6 +743,8 @@ static void vcn_v1_0_dec_ring_set_wptr(struct amdgpu_ring *ring)
  */
 static void vcn_v1_0_dec_ring_insert_start(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, 0);
@@ -761,6 +762,8 @@ static void vcn_v1_0_dec_ring_insert_start(struct amdgpu_ring *ring)
  */
 static void vcn_v1_0_dec_ring_insert_end(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_CMD), 0));
 	amdgpu_ring_write(ring, VCN_DEC_CMD_PACKET_END << 1);
@@ -777,6 +780,8 @@ static void vcn_v1_0_dec_ring_insert_end(struct amdgpu_ring *ring)
 static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
 				     unsigned flags)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
 
 	amdgpu_ring_write(ring,
@@ -812,6 +817,8 @@ static void vcn_v1_0_dec_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64
  */
 static void vcn_v1_0_dec_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 0));
 	amdgpu_ring_write(ring, 1);
 }
@@ -826,11 +833,13 @@ static void vcn_v1_0_dec_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
  */
 static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
 				  struct amdgpu_ib *ib,
-				  unsigned vm_id, bool ctx_switch)
+				  unsigned vmid, bool ctx_switch)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_VMID), 0));
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_LMI_RBC_IB_64BIT_BAR_LOW), 0));
@@ -846,6 +855,8 @@ static void vcn_v1_0_dec_ring_emit_ib(struct amdgpu_ring *ring,
 static void vcn_v1_0_dec_vm_reg_write(struct amdgpu_ring *ring,
 				uint32_t data0, uint32_t data1)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, data0);
@@ -860,6 +871,8 @@ static void vcn_v1_0_dec_vm_reg_write(struct amdgpu_ring *ring,
 static void vcn_v1_0_dec_vm_reg_wait(struct amdgpu_ring *ring,
 				uint32_t data0, uint32_t data1, uint32_t mask)
 {
+	struct amdgpu_device *adev = ring->adev;
+
 	amdgpu_ring_write(ring,
 		PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_GPCOM_VCPU_DATA0), 0));
 	amdgpu_ring_write(ring, data0);
@@ -875,25 +888,26 @@ static void vcn_v1_0_dec_vm_reg_wait(struct amdgpu_ring *ring,
 }
 
 static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
-					unsigned vm_id, uint64_t pd_addr)
+					unsigned vmid, uint64_t pd_addr)
 {
 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
-	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
-	uint32_t data0, data1, mask;
+	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
+	uint64_t flags = AMDGPU_PTE_VALID;
 	unsigned eng = ring->vm_inv_eng;
+	uint32_t data0, data1, mask;
 
-	pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
-	pd_addr |= AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
+	pd_addr |= flags;
 
-	data0 = (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2;
+	data0 = (hub->ctx0_ptb_addr_hi32 + vmid * 2) << 2;
 	data1 = upper_32_bits(pd_addr);
 	vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 
-	data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
+	data0 = (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2;
 	data1 = lower_32_bits(pd_addr);
 	vcn_v1_0_dec_vm_reg_write(ring, data0, data1);
 
-	data0 = (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2;
+	data0 = (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2;
 	data1 = lower_32_bits(pd_addr);
 	mask = 0xffffffff;
 	vcn_v1_0_dec_vm_reg_wait(ring, data0, data1, mask);
@@ -905,8 +919,8 @@ static void vcn_v1_0_dec_ring_emit_vm_flush(struct amdgpu_ring *ring,
 
 	/* wait for flush */
 	data0 = (hub->vm_inv_eng0_ack + eng) << 2;
-	data1 = 1 << vm_id;
-	mask =  1 << vm_id;
+	data1 = 1 << vmid;
+	mask =  1 << vmid;
 	vcn_v1_0_dec_vm_reg_wait(ring, data0, data1, mask);
 }
 
@@ -997,38 +1011,39 @@ static void vcn_v1_0_enc_ring_insert_end(struct amdgpu_ring *ring)
  * Write enc ring commands to execute the indirect buffer
  */
 static void vcn_v1_0_enc_ring_emit_ib(struct amdgpu_ring *ring,
-		struct amdgpu_ib *ib, unsigned int vm_id, bool ctx_switch)
+		struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
 {
 	amdgpu_ring_write(ring, VCN_ENC_CMD_IB);
-	amdgpu_ring_write(ring, vm_id);
+	amdgpu_ring_write(ring, vmid);
 	amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
 	amdgpu_ring_write(ring, ib->length_dw);
 }
 
 static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
-			 unsigned int vm_id, uint64_t pd_addr)
+			 unsigned int vmid, uint64_t pd_addr)
 {
 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
-	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vm_id);
+	uint32_t req = ring->adev->gart.gart_funcs->get_invalidate_req(vmid);
+	uint64_t flags = AMDGPU_PTE_VALID;
 	unsigned eng = ring->vm_inv_eng;
 
-	pd_addr = amdgpu_gart_get_vm_pde(ring->adev, pd_addr);
-	pd_addr |= AMDGPU_PTE_VALID;
+	amdgpu_gart_get_vm_pde(ring->adev, -1, &pd_addr, &flags);
+	pd_addr |= flags;
 
 	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
 	amdgpu_ring_write(ring,
-			  (hub->ctx0_ptb_addr_hi32 + vm_id * 2) << 2);
+			  (hub->ctx0_ptb_addr_hi32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, upper_32_bits(pd_addr));
 
 	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE);
 	amdgpu_ring_write(ring,
-			  (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
+			  (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, lower_32_bits(pd_addr));
 
 	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
 	amdgpu_ring_write(ring,
-			  (hub->ctx0_ptb_addr_lo32 + vm_id * 2) << 2);
+			  (hub->ctx0_ptb_addr_lo32 + vmid * 2) << 2);
 	amdgpu_ring_write(ring, 0xffffffff);
 	amdgpu_ring_write(ring, lower_32_bits(pd_addr));
 
@@ -1040,8 +1055,8 @@ static void vcn_v1_0_enc_ring_emit_vm_flush(struct amdgpu_ring *ring,
 	/* wait for flush */
 	amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT);
 	amdgpu_ring_write(ring, (hub->vm_inv_eng0_ack + eng) << 2);
-	amdgpu_ring_write(ring, 1 << vm_id);
-	amdgpu_ring_write(ring, 1 << vm_id);
+	amdgpu_ring_write(ring, 1 << vmid);
+	amdgpu_ring_write(ring, 1 << vmid);
 }
 
 static int vcn_v1_0_set_interrupt_state(struct amdgpu_device *adev,
@@ -1077,6 +1092,17 @@ static int vcn_v1_0_process_interrupt(struct amdgpu_device *adev,
 	return 0;
 }
 
+static void vcn_v1_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
+{
+	int i;
+	struct amdgpu_device *adev = ring->adev;
+
+	for (i = 0; i < count; i++)
+		amdgpu_ring_write(ring, PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0));
+
+}
+
+
 static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
 	.name = "vcn_v1_0",
 	.early_init = vcn_v1_0_early_init,
@@ -1100,7 +1126,7 @@ static const struct amd_ip_funcs vcn_v1_0_ip_funcs = {
 static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
 	.type = AMDGPU_RING_TYPE_VCN_DEC,
 	.align_mask = 0xf,
-	.nop = PACKET0(SOC15_REG_OFFSET(UVD, 0, mmUVD_NO_OP), 0),
+	.nop = PACKET0(0x81ff, 0),
 	.support_64bit_ptrs = false,
 	.vmhub = AMDGPU_MMHUB,
 	.get_rptr = vcn_v1_0_dec_ring_get_rptr,
@@ -1118,7 +1144,7 @@ static const struct amdgpu_ring_funcs vcn_v1_0_dec_ring_vm_funcs = {
 	.emit_hdp_invalidate = vcn_v1_0_dec_ring_emit_hdp_invalidate,
 	.test_ring = amdgpu_vcn_dec_ring_test_ring,
 	.test_ib = amdgpu_vcn_dec_ring_test_ib,
-	.insert_nop = amdgpu_ring_insert_nop,
+	.insert_nop = vcn_v1_0_ring_insert_nop,
 	.insert_start = vcn_v1_0_dec_ring_insert_start,
 	.insert_end = vcn_v1_0_dec_ring_insert_end,
 	.pad_ib = amdgpu_ring_generic_pad_ib,
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index 697325737ba8..ee14d78be2a9 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -25,10 +25,8 @@
 #include "amdgpu_ih.h"
 #include "soc15.h"
 
-
-#include "vega10/soc15ip.h"
-#include "vega10/OSSSYS/osssys_4_0_offset.h"
-#include "vega10/OSSSYS/osssys_4_0_sh_mask.h"
+#include "oss/osssys_4_0_offset.h"
+#include "oss/osssys_4_0_sh_mask.h"
 
 #include "soc15_common.h"
 #include "vega10_ih.h"
@@ -46,11 +44,11 @@ static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
  */
 static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
 {
-	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
+	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 
 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 	adev->irq.ih.enabled = true;
 }
 
@@ -63,14 +61,14 @@ static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
  */
 static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
 {
-	u32 ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
+	u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 
 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 	/* set rptr, wptr to 0 */
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
 	adev->irq.ih.enabled = false;
 	adev->irq.ih.rptr = 0;
 }
@@ -97,20 +95,17 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
 	/* disable irqs */
 	vega10_ih_disable_interrupts(adev);
 
-	if (adev->flags & AMD_IS_APU)
-		nbio_v7_0_ih_control(adev);
-	else
-		nbio_v6_1_ih_control(adev);
+	adev->nbio_funcs->ih_control(adev);
 
-	ih_rb_cntl = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
+	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
 	if (adev->irq.ih.use_bus_addr) {
-		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.rb_dma_addr >> 8);
-		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
 		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
 	} else {
-		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE), adev->irq.ih.gpu_addr >> 8);
-		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI), (adev->irq.ih.gpu_addr >> 40) & 0xff);
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
 		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
 	}
 	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
@@ -126,21 +121,21 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
 	if (adev->irq.msi_enabled)
 		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
 
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), ih_rb_cntl);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 
 	/* set the writeback address whether it's enabled or not */
 	if (adev->irq.ih.use_bus_addr)
 		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
 	else
 		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO), lower_32_bits(wptr_off));
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI), upper_32_bits(wptr_off) & 0xFF);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF);
 
 	/* set rptr, wptr to 0 */
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), 0);
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR), 0);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
 
-	ih_doorbell_rtpr = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR));
+	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
 	if (adev->irq.ih.use_doorbell) {
 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
 						 OFFSET, adev->irq.ih.doorbell_index);
@@ -150,20 +145,18 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
 		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
 						 ENABLE, 0);
 	}
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR), ih_doorbell_rtpr);
-	if (adev->flags & AMD_IS_APU)
-		nbio_v7_0_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
-	else
-		nbio_v6_1_ih_doorbell_range(adev, adev->irq.ih.use_doorbell, adev->irq.ih.doorbell_index);
+	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
+	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
+					    adev->irq.ih.doorbell_index);
 
-	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL));
+	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
 	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
 			    CLIENT18_IS_STORM_CLIENT, 1);
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL), tmp);
+	WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
 
-	tmp = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL));
+	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
 	tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
-	WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_INT_FLOOD_CNTL), tmp);
+	WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
 
 	pci_set_master(adev->pdev);
 
@@ -285,9 +278,9 @@ static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
 	/* Track retry faults in per-VM fault FIFO. */
 	spin_lock(&adev->vm_manager.pasid_lock);
 	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
-	spin_unlock(&adev->vm_manager.pasid_lock);
-	if (WARN_ON_ONCE(!vm)) {
+	if (!vm) {
 		/* VM not found, process it normally */
+		spin_unlock(&adev->vm_manager.pasid_lock);
 		amdgpu_ih_clear_fault(adev, key);
 		return true;
 	}
@@ -295,9 +288,11 @@ static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
 	r = kfifo_put(&vm->faults, key);
 	if (!r) {
 		/* FIFO is full. Ignore it until there is space */
+		spin_unlock(&adev->vm_manager.pasid_lock);
 		amdgpu_ih_clear_fault(adev, key);
 		goto ignore_iv;
 	}
+	spin_unlock(&adev->vm_manager.pasid_lock);
 
 	/* It's the first fault for this address, process it normally */
 	return true;
@@ -334,8 +329,8 @@ static void vega10_ih_decode_iv(struct amdgpu_device *adev,
 	entry->client_id = dw[0] & 0xff;
 	entry->src_id = (dw[0] >> 8) & 0xff;
 	entry->ring_id = (dw[0] >> 16) & 0xff;
-	entry->vm_id = (dw[0] >> 24) & 0xf;
-	entry->vm_id_src = (dw[0] >> 31);
+	entry->vmid = (dw[0] >> 24) & 0xf;
+	entry->vmid_src = (dw[0] >> 31);
 	entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
 	entry->timestamp_src = dw[2] >> 31;
 	entry->pas_id = dw[3] & 0xffff;
@@ -367,7 +362,7 @@ static void vega10_ih_set_rptr(struct amdgpu_device *adev)
 			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
 		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
 	} else {
-		WREG32(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR), adev->irq.ih.rptr);
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c
new file mode 100644
index 000000000000..b7bdd04793d6
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_reg_init.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "soc15.h"
+
+#include "soc15_common.h"
+#include "soc15ip.h"
+
+int vega10_reg_base_init(struct amdgpu_device *adev)
+{
+	/* HW has more IP blocks,  only initialized the blocke beend by our driver  */
+	uint32_t i;
+	for (i = 0 ; i < MAX_INSTANCE ; ++i) {
+		adev->reg_offset[GC_HWIP][i] = (uint32_t *)(&(GC_BASE.instance[i]));
+		adev->reg_offset[HDP_HWIP][i] = (uint32_t *)(&(HDP_BASE.instance[i]));
+		adev->reg_offset[MMHUB_HWIP][i] = (uint32_t *)(&(MMHUB_BASE.instance[i]));
+		adev->reg_offset[ATHUB_HWIP][i] = (uint32_t *)(&(ATHUB_BASE.instance[i]));
+		adev->reg_offset[NBIO_HWIP][i] = (uint32_t *)(&(NBIO_BASE.instance[i]));
+		adev->reg_offset[MP0_HWIP][i] = (uint32_t *)(&(MP0_BASE.instance[i]));
+		adev->reg_offset[UVD_HWIP][i] = (uint32_t *)(&(UVD_BASE.instance[i]));
+		adev->reg_offset[VCE_HWIP][i] = (uint32_t *)(&(VCE_BASE.instance[i]));
+		adev->reg_offset[VCN_HWIP][i] = (uint32_t *)(&(VCN_BASE.instance[i]));
+		adev->reg_offset[DF_HWIP][i] = (uint32_t *)(&(DF_BASE.instance[i]));
+		adev->reg_offset[DCE_HWIP][i] = (uint32_t *)(&(DCE_BASE.instance[i]));
+		adev->reg_offset[OSSSYS_HWIP][i] = (uint32_t *)(&(OSSSYS_BASE.instance[i]));
+		adev->reg_offset[SDMA0_HWIP][i] = (uint32_t *)(&(SDMA0_BASE.instance[i]));
+		adev->reg_offset[SDMA1_HWIP][i] = (uint32_t *)(&(SDMA1_BASE.instance[i]));
+		adev->reg_offset[SMUIO_HWIP][i] = (uint32_t *)(&(SMUIO_BASE.instance[i]));
+		adev->reg_offset[PWR_HWIP][i] = (uint32_t *)(&(PWR_BASE.instance[i]));
+		adev->reg_offset[NBIF_HWIP][i] = (uint32_t *)(&(NBIF_BASE.instance[i]));
+
+	}
+	return 0;
+}
+
+
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index 3a4c2fa7e36d..1e3e05a11f7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -282,29 +282,29 @@ static void vi_init_golden_registers(struct amdgpu_device *adev)
 
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
-		amdgpu_program_register_sequence(adev,
-						 iceland_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							iceland_mgcg_cgcg_init,
+							ARRAY_SIZE(iceland_mgcg_cgcg_init));
 		break;
 	case CHIP_FIJI:
-		amdgpu_program_register_sequence(adev,
-						 fiji_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							fiji_mgcg_cgcg_init,
+							ARRAY_SIZE(fiji_mgcg_cgcg_init));
 		break;
 	case CHIP_TONGA:
-		amdgpu_program_register_sequence(adev,
-						 tonga_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							tonga_mgcg_cgcg_init,
+							ARRAY_SIZE(tonga_mgcg_cgcg_init));
 		break;
 	case CHIP_CARRIZO:
-		amdgpu_program_register_sequence(adev,
-						 cz_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							cz_mgcg_cgcg_init,
+							ARRAY_SIZE(cz_mgcg_cgcg_init));
 		break;
 	case CHIP_STONEY:
-		amdgpu_program_register_sequence(adev,
-						 stoney_mgcg_cgcg_init,
-						 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
+		amdgpu_device_program_register_sequence(adev,
+							stoney_mgcg_cgcg_init,
+							ARRAY_SIZE(stoney_mgcg_cgcg_init));
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
@@ -449,14 +449,18 @@ static bool vi_read_bios_from_rom(struct amdgpu_device *adev,
 
 static void vi_detect_hw_virtualization(struct amdgpu_device *adev)
 {
-	uint32_t reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER);
-	/* bit0: 0 means pf and 1 means vf */
-	/* bit31: 0 means disable IOV and 1 means enable */
-	if (reg & 1)
-		adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF;
-
-	if (reg & 0x80000000)
-		adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
+	uint32_t reg = 0;
+
+	if (adev->asic_type == CHIP_TONGA ||
+	    adev->asic_type == CHIP_FIJI) {
+	       reg = RREG32(mmBIF_IOV_FUNC_IDENTIFIER);
+	       /* bit0: 0 means pf and 1 means vf */
+	       if (REG_GET_FIELD(reg, BIF_IOV_FUNC_IDENTIFIER, FUNC_IDENTIFIER))
+		       adev->virt.caps |= AMDGPU_SRIOV_CAPS_IS_VF;
+	       /* bit31: 0 means disable IOV and 1 means enable */
+	       if (REG_GET_FIELD(reg, BIF_IOV_FUNC_IDENTIFIER, IOV_ENABLE))
+		       adev->virt.caps |= AMDGPU_SRIOV_CAPS_ENABLE_IOV;
+	}
 
 	if (reg == 0) {
 		if (is_virtual_machine()) /* passthrough mode exclus sr-iov mode */
@@ -667,7 +671,7 @@ static int vi_gpu_pci_config_reset(struct amdgpu_device *adev)
 	/* disable BM */
 	pci_clear_master(adev->pdev);
 	/* reset */
-	amdgpu_pci_config_reset(adev);
+	amdgpu_device_pci_config_reset(adev);
 
 	udelay(100);
 
@@ -891,8 +895,8 @@ static int vi_common_early_init(void *handle)
 
 	adev->asic_funcs = &vi_asic_funcs;
 
-	if (amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_SMC) &&
-		(amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
+	if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_SMC) &&
+	    (amdgpu_ip_block_mask & (1 << AMD_IP_BLOCK_TYPE_SMC)))
 		smc_enabled = true;
 
 	adev->rev_id = vi_get_rev_id(adev);
@@ -1045,7 +1049,6 @@ static int vi_common_early_init(void *handle)
 			AMD_CG_SUPPORT_GFX_CP_LS |
 			AMD_CG_SUPPORT_GFX_CGTS |
 			AMD_CG_SUPPORT_GFX_CGTS_LS |
-			AMD_CG_SUPPORT_GFX_CGCG |
 			AMD_CG_SUPPORT_GFX_CGLS |
 			AMD_CG_SUPPORT_BIF_LS |
 			AMD_CG_SUPPORT_HDP_MGCG |
@@ -1074,7 +1077,7 @@ static int vi_common_early_init(void *handle)
 	/* vi use smc load by default */
 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
 
-	amdgpu_get_pcie_info(adev);
+	amdgpu_device_get_pcie_info(adev);
 
 	return 0;
 }
@@ -1487,115 +1490,115 @@ int vi_set_ip_blocks(struct amdgpu_device *adev)
 	switch (adev->asic_type) {
 	case CHIP_TOPAZ:
 		/* topaz has no DCE, UVD, VCE */
-		amdgpu_ip_block_add(adev, &vi_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v7_4_ip_block);
-		amdgpu_ip_block_add(adev, &iceland_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v7_4_ip_block);
+		amdgpu_device_ip_block_add(adev, &iceland_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v2_4_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v2_4_ip_block);
 		break;
 	case CHIP_FIJI:
-		amdgpu_ip_block_add(adev, &vi_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v8_5_ip_block);
-		amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v8_5_ip_block);
+		amdgpu_device_ip_block_add(adev, &tonga_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v10_1_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v10_1_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
 		if (!amdgpu_sriov_vf(adev)) {
-			amdgpu_ip_block_add(adev, &uvd_v6_0_ip_block);
-			amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &uvd_v6_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &vce_v3_0_ip_block);
 		}
 		break;
 	case CHIP_TONGA:
-		amdgpu_ip_block_add(adev, &vi_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &tonga_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v10_0_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v10_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
 		if (!amdgpu_sriov_vf(adev)) {
-			amdgpu_ip_block_add(adev, &uvd_v5_0_ip_block);
-			amdgpu_ip_block_add(adev, &vce_v3_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &uvd_v5_0_ip_block);
+			amdgpu_device_ip_block_add(adev, &vce_v3_0_ip_block);
 		}
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS10:
 	case CHIP_POLARIS12:
-		amdgpu_ip_block_add(adev, &vi_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v8_1_ip_block);
-		amdgpu_ip_block_add(adev, &tonga_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v8_1_ip_block);
+		amdgpu_device_ip_block_add(adev, &tonga_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v11_2_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v3_1_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v6_3_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v3_4_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v11_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v3_1_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v6_3_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v3_4_ip_block);
 		break;
 	case CHIP_CARRIZO:
-		amdgpu_ip_block_add(adev, &vi_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &cz_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &cz_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v11_0_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v6_0_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v3_1_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v11_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v6_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v3_1_ip_block);
 #if defined(CONFIG_DRM_AMD_ACP)
-		amdgpu_ip_block_add(adev, &acp_ip_block);
+		amdgpu_device_ip_block_add(adev, &acp_ip_block);
 #endif
 		break;
 	case CHIP_STONEY:
-		amdgpu_ip_block_add(adev, &vi_common_ip_block);
-		amdgpu_ip_block_add(adev, &gmc_v8_0_ip_block);
-		amdgpu_ip_block_add(adev, &cz_ih_ip_block);
-		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
+		amdgpu_device_ip_block_add(adev, &vi_common_ip_block);
+		amdgpu_device_ip_block_add(adev, &gmc_v8_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &cz_ih_ip_block);
+		amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
 		if (adev->enable_virtual_display)
-			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
 		else if (amdgpu_device_has_dc_support(adev))
-			amdgpu_ip_block_add(adev, &dm_ip_block);
+			amdgpu_device_ip_block_add(adev, &dm_ip_block);
 #endif
 		else
-			amdgpu_ip_block_add(adev, &dce_v11_0_ip_block);
-		amdgpu_ip_block_add(adev, &gfx_v8_1_ip_block);
-		amdgpu_ip_block_add(adev, &sdma_v3_0_ip_block);
-		amdgpu_ip_block_add(adev, &uvd_v6_2_ip_block);
-		amdgpu_ip_block_add(adev, &vce_v3_4_ip_block);
+			amdgpu_device_ip_block_add(adev, &dce_v11_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &gfx_v8_1_ip_block);
+		amdgpu_device_ip_block_add(adev, &sdma_v3_0_ip_block);
+		amdgpu_device_ip_block_add(adev, &uvd_v6_2_ip_block);
+		amdgpu_device_ip_block_add(adev, &vce_v3_4_ip_block);
 #if defined(CONFIG_DRM_AMD_ACP)
-		amdgpu_ip_block_add(adev, &acp_ip_block);
+		amdgpu_device_ip_block_add(adev, &acp_ip_block);
 #endif
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h
index dbf3703cbd1b..19ddd2312e00 100644
--- a/drivers/gpu/drm/amd/amdgpu/vid.h
+++ b/drivers/gpu/drm/amd/amdgpu/vid.h
@@ -27,6 +27,8 @@
 #define SDMA1_REGISTER_OFFSET                             0x200 /* not a register */
 #define SDMA_MAX_INSTANCE 2
 
+#define KFD_VI_SDMA_QUEUE_OFFSET                      0x80 /* not a register */
+
 /* crtc instance offsets */
 #define CRTC0_REGISTER_OFFSET                 (0x1b9c - 0x1b9c)
 #define CRTC1_REGISTER_OFFSET                 (0x1d9c - 0x1b9c)
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index 342c2d937b17..a317e76ffb5e 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -35,6 +35,8 @@ amdkfd-y	:= kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
 		kfd_process_queue_manager.o kfd_device_queue_manager.o \
 		kfd_device_queue_manager_cik.o kfd_device_queue_manager_vi.o \
 		kfd_interrupt.o kfd_events.o cik_event_interrupt.o \
-		kfd_dbgdev.o kfd_dbgmgr.o
+		kfd_dbgdev.o kfd_dbgmgr.o kfd_crat.o
+
+amdkfd-$(CONFIG_DEBUG_FS) += kfd_debugfs.o
 
 obj-$(CONFIG_HSA_AMD)	+= amdkfd.o
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
new file mode 100644
index 000000000000..997a383dcb8b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm
@@ -0,0 +1,1384 @@
+/*
+ * Copyright 2015-2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#if 0
+HW (VI) source code for CWSR trap handler
+#Version 18 + multiple trap handler
+
+// this performance-optimal version was originally from Seven Xu at SRDC
+
+// Revison #18   --...
+/* Rev History
+** #1. Branch from gc dv.   //gfxip/gfx8/main/src/test/suites/block/cs/sr/cs_trap_handler.sp3#1,#50, #51, #52-53(Skip, Already Fixed by PV), #54-56(merged),#57-58(mergerd, skiped-already fixed by PV)
+** #4. SR Memory Layout:
+**             1. VGPR-SGPR-HWREG-{LDS}
+**             2. tba_hi.bits.26 - reconfigured as the first wave in tg bits, for defer Save LDS for a threadgroup.. performance concern..
+** #5. Update: 1. Accurate g8sr_ts_save_d timestamp
+** #6. Update: 1. Fix s_barrier usage; 2. VGPR s/r using swizzle buffer?(NoNeed, already matched the swizzle pattern, more investigation)
+** #7. Update: 1. don't barrier if noLDS
+** #8. Branch: 1. Branch to ver#0, which is very similar to gc dv version
+**             2. Fix SQ issue by s_sleep 2
+** #9. Update: 1. Fix scc restore failed issue, restore wave_status at last
+**             2. optimize s_buffer save by burst 16sgprs...
+** #10. Update 1. Optimize restore sgpr by busrt 16 sgprs.
+** #11. Update 1. Add 2 more timestamp for debug version
+** #12. Update 1. Add VGPR SR using DWx4, some case improve and some case drop performance
+** #13. Integ  1. Always use MUBUF for PV trap shader...
+** #14. Update 1. s_buffer_store soft clause...
+** #15. Update 1. PERF - sclar write with glc:0/mtype0 to allow L2 combine. perf improvement a lot.
+** #16. Update 1. PRRF - UNROLL LDS_DMA got 2500cycle save in IP tree
+** #17. Update 1. FUNC - LDS_DMA has issues while ATC, replace with ds_read/buffer_store for save part[TODO restore part]
+**             2. PERF - Save LDS before save VGPR to cover LDS save long latency...
+** #18. Update 1. FUNC - Implicitly estore STATUS.VCCZ, which is not writable by s_setreg_b32
+**             2. FUNC - Handle non-CWSR traps
+*/
+
+var G8SR_WDMEM_HWREG_OFFSET = 0
+var G8SR_WDMEM_SGPR_OFFSET  = 128  // in bytes
+
+// Keep definition same as the app shader, These 2 time stamps are part of the app shader... Should before any Save and after restore.
+
+var G8SR_DEBUG_TIMESTAMP = 0
+var G8SR_DEBUG_TS_SAVE_D_OFFSET = 40*4  // ts_save_d timestamp offset relative to SGPR_SR_memory_offset
+var s_g8sr_ts_save_s    = s[34:35]   // save start
+var s_g8sr_ts_sq_save_msg  = s[36:37]   // The save shader send SAVEWAVE msg to spi
+var s_g8sr_ts_spi_wrexec   = s[38:39]   // the SPI write the sr address to SQ
+var s_g8sr_ts_save_d    = s[40:41]   // save end
+var s_g8sr_ts_restore_s = s[42:43]   // restore start
+var s_g8sr_ts_restore_d = s[44:45]   // restore end
+
+var G8SR_VGPR_SR_IN_DWX4 = 0
+var G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4 = 0x00100000    // DWx4 stride is 4*4Bytes
+var G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4  = G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4
+
+
+/*************************************************************************/
+/*                  control on how to run the shader                     */
+/*************************************************************************/
+//any hack that needs to be made to run this code in EMU (either because various EMU code are not ready or no compute save & restore in EMU run)
+var EMU_RUN_HACK                    =   0
+var EMU_RUN_HACK_RESTORE_NORMAL     =   0
+var EMU_RUN_HACK_SAVE_NORMAL_EXIT   =   0
+var EMU_RUN_HACK_SAVE_SINGLE_WAVE   =   0
+var EMU_RUN_HACK_SAVE_FIRST_TIME    =   0                   //for interrupted restore in which the first save is through EMU_RUN_HACK
+var EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_LO =   0                   //for interrupted restore in which the first save is through EMU_RUN_HACK
+var EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_HI =   0                   //for interrupted restore in which the first save is through EMU_RUN_HACK
+var SAVE_LDS                        =   1
+var WG_BASE_ADDR_LO                 =   0x9000a000
+var WG_BASE_ADDR_HI                 =   0x0
+var WAVE_SPACE                      =   0x5000              //memory size that each wave occupies in workgroup state mem
+var CTX_SAVE_CONTROL                =   0x0
+var CTX_RESTORE_CONTROL             =   CTX_SAVE_CONTROL
+var SIM_RUN_HACK                    =   0                   //any hack that needs to be made to run this code in SIM (either because various RTL code are not ready or no compute save & restore in RTL run)
+var SGPR_SAVE_USE_SQC               =   1                   //use SQC D$ to do the write
+var USE_MTBUF_INSTEAD_OF_MUBUF      =   0                   //because TC EMU currently asserts on 0 of // overload DFMT field to carry 4 more bits of stride for MUBUF opcodes
+var SWIZZLE_EN                      =   0                   //whether we use swizzled buffer addressing
+
+/**************************************************************************/
+/*                      variables                                         */
+/**************************************************************************/
+var SQ_WAVE_STATUS_INST_ATC_SHIFT  = 23
+var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
+var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
+
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT    = 12
+var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE     = 9
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT   = 8
+var SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE    = 6
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT   = 24
+var SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE    = 3                     //FIXME  sq.blk still has 4 bits at this time while SQ programming guide has 3 bits
+
+var SQ_WAVE_TRAPSTS_SAVECTX_MASK    =   0x400
+var SQ_WAVE_TRAPSTS_EXCE_MASK       =   0x1FF                   // Exception mask
+var SQ_WAVE_TRAPSTS_SAVECTX_SHIFT   =   10
+var SQ_WAVE_TRAPSTS_MEM_VIOL_MASK   =   0x100
+var SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT  =   8
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK    =   0x3FF
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT   =   0x0
+var SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE    =   10
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK   =   0xFFFFF800
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT  =   11
+var SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE   =   21
+
+var SQ_WAVE_IB_STS_RCNT_SHIFT           =   16                  //FIXME
+var SQ_WAVE_IB_STS_RCNT_SIZE            =   4                   //FIXME
+var SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT   =   15                  //FIXME
+var SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE    =   1                   //FIXME
+var SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG   = 0x00007FFF    //FIXME
+
+var SQ_BUF_RSRC_WORD1_ATC_SHIFT     =   24
+var SQ_BUF_RSRC_WORD3_MTYPE_SHIFT   =   27
+
+
+/*      Save        */
+var S_SAVE_BUF_RSRC_WORD1_STRIDE        =   0x00040000          //stride is 4 bytes
+var S_SAVE_BUF_RSRC_WORD3_MISC          =   0x00807FAC          //SQ_SEL_X/Y/Z/W, BUF_NUM_FORMAT_FLOAT, (0 for MUBUF stride[17:14] when ADD_TID_ENABLE and BUF_DATA_FORMAT_32 for MTBUF), ADD_TID_ENABLE
+
+var S_SAVE_SPI_INIT_ATC_MASK            =   0x08000000          //bit[27]: ATC bit
+var S_SAVE_SPI_INIT_ATC_SHIFT           =   27
+var S_SAVE_SPI_INIT_MTYPE_MASK          =   0x70000000          //bit[30:28]: Mtype
+var S_SAVE_SPI_INIT_MTYPE_SHIFT         =   28
+var S_SAVE_SPI_INIT_FIRST_WAVE_MASK     =   0x04000000          //bit[26]: FirstWaveInTG
+var S_SAVE_SPI_INIT_FIRST_WAVE_SHIFT    =   26
+
+var S_SAVE_PC_HI_RCNT_SHIFT             =   28                  //FIXME  check with Brian to ensure all fields other than PC[47:0] can be used
+var S_SAVE_PC_HI_RCNT_MASK              =   0xF0000000          //FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_SHIFT     =   27                  //FIXME
+var S_SAVE_PC_HI_FIRST_REPLAY_MASK      =   0x08000000          //FIXME
+
+var s_save_spi_init_lo              =   exec_lo
+var s_save_spi_init_hi              =   exec_hi
+
+                                                //tba_lo and tba_hi need to be saved/restored
+var s_save_pc_lo            =   ttmp0           //{TTMP1, TTMP0} = {3??h0,pc_rewind[3:0], HT[0],trapID[7:0], PC[47:0]}
+var s_save_pc_hi            =   ttmp1
+var s_save_exec_lo          =   ttmp2
+var s_save_exec_hi          =   ttmp3
+var s_save_status           =   ttmp4
+var s_save_trapsts          =   ttmp5           //not really used until the end of the SAVE routine
+var s_save_xnack_mask_lo    =   ttmp6
+var s_save_xnack_mask_hi    =   ttmp7
+var s_save_buf_rsrc0        =   ttmp8
+var s_save_buf_rsrc1        =   ttmp9
+var s_save_buf_rsrc2        =   ttmp10
+var s_save_buf_rsrc3        =   ttmp11
+
+var s_save_mem_offset       =   tma_lo
+var s_save_alloc_size       =   s_save_trapsts          //conflict
+var s_save_tmp              =   s_save_buf_rsrc2        //shared with s_save_buf_rsrc2  (conflict: should not use mem access with s_save_tmp at the same time)
+var s_save_m0               =   tma_hi
+
+/*      Restore     */
+var S_RESTORE_BUF_RSRC_WORD1_STRIDE         =   S_SAVE_BUF_RSRC_WORD1_STRIDE
+var S_RESTORE_BUF_RSRC_WORD3_MISC           =   S_SAVE_BUF_RSRC_WORD3_MISC
+
+var S_RESTORE_SPI_INIT_ATC_MASK             =   0x08000000          //bit[27]: ATC bit
+var S_RESTORE_SPI_INIT_ATC_SHIFT            =   27
+var S_RESTORE_SPI_INIT_MTYPE_MASK           =   0x70000000          //bit[30:28]: Mtype
+var S_RESTORE_SPI_INIT_MTYPE_SHIFT          =   28
+var S_RESTORE_SPI_INIT_FIRST_WAVE_MASK      =   0x04000000          //bit[26]: FirstWaveInTG
+var S_RESTORE_SPI_INIT_FIRST_WAVE_SHIFT     =   26
+
+var S_RESTORE_PC_HI_RCNT_SHIFT              =   S_SAVE_PC_HI_RCNT_SHIFT
+var S_RESTORE_PC_HI_RCNT_MASK               =   S_SAVE_PC_HI_RCNT_MASK
+var S_RESTORE_PC_HI_FIRST_REPLAY_SHIFT      =   S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+var S_RESTORE_PC_HI_FIRST_REPLAY_MASK       =   S_SAVE_PC_HI_FIRST_REPLAY_MASK
+
+var s_restore_spi_init_lo                   =   exec_lo
+var s_restore_spi_init_hi                   =   exec_hi
+
+var s_restore_mem_offset        =   ttmp2
+var s_restore_alloc_size        =   ttmp3
+var s_restore_tmp               =   ttmp6               //tba_lo/hi need to be restored
+var s_restore_mem_offset_save   =   s_restore_tmp       //no conflict
+
+var s_restore_m0            =   s_restore_alloc_size    //no conflict
+
+var s_restore_mode          =   ttmp7
+
+var s_restore_pc_lo         =   ttmp0
+var s_restore_pc_hi         =   ttmp1
+var s_restore_exec_lo       =   tma_lo                  //no conflict
+var s_restore_exec_hi       =   tma_hi                  //no conflict
+var s_restore_status        =   ttmp4
+var s_restore_trapsts       =   ttmp5
+var s_restore_xnack_mask_lo =   xnack_mask_lo
+var s_restore_xnack_mask_hi =   xnack_mask_hi
+var s_restore_buf_rsrc0     =   ttmp8
+var s_restore_buf_rsrc1     =   ttmp9
+var s_restore_buf_rsrc2     =   ttmp10
+var s_restore_buf_rsrc3     =   ttmp11
+
+/**************************************************************************/
+/*                      trap handler entry points                         */
+/**************************************************************************/
+/* Shader Main*/
+
+shader main
+  asic(VI)
+  type(CS)
+
+
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))                   //hack to use trap_id for determining save/restore
+        //FIXME VCCZ un-init assertion s_getreg_b32     s_save_status, hwreg(HW_REG_STATUS)         //save STATUS since we will change SCC
+        s_and_b32 s_save_tmp, s_save_pc_hi, 0xffff0000              //change SCC
+        s_cmp_eq_u32 s_save_tmp, 0x007e0000                         //Save: trap_id = 0x7e. Restore: trap_id = 0x7f.
+        s_cbranch_scc0 L_JUMP_TO_RESTORE                            //do not need to recover STATUS here  since we are going to RESTORE
+        //FIXME  s_setreg_b32   hwreg(HW_REG_STATUS),   s_save_status       //need to recover STATUS since we are going to SAVE
+        s_branch L_SKIP_RESTORE                                     //NOT restore, SAVE actually
+    else
+        s_branch L_SKIP_RESTORE                                     //NOT restore. might be a regular trap or save
+    end
+
+L_JUMP_TO_RESTORE:
+    s_branch L_RESTORE                                              //restore
+
+L_SKIP_RESTORE:
+
+    s_getreg_b32    s_save_status, hwreg(HW_REG_STATUS)                             //save STATUS since we will change SCC
+    s_andn2_b32     s_save_status, s_save_status, SQ_WAVE_STATUS_SPI_PRIO_MASK      //check whether this is for save
+    s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+    s_and_b32       s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_SAVECTX_MASK    //check whether this is for save
+    s_cbranch_scc1  L_SAVE                                      //this is the operation for save
+
+    // *********    Handle non-CWSR traps       *******************
+if (!EMU_RUN_HACK)
+    /* read tba and tma for next level trap handler, ttmp4 is used as s_save_status */
+    s_load_dwordx4  [ttmp8,ttmp9,ttmp10, ttmp11], [tma_lo,tma_hi], 0
+    s_waitcnt lgkmcnt(0)
+    s_or_b32        ttmp7, ttmp8, ttmp9
+    s_cbranch_scc0  L_NO_NEXT_TRAP //next level trap handler not been set
+    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+    s_setpc_b64     [ttmp8,ttmp9] //jump to next level trap handler
+
+L_NO_NEXT_TRAP:
+    s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+    s_and_b32       s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_EXCE_MASK // Check whether it is an exception
+    s_cbranch_scc1  L_EXCP_CASE   // Exception, jump back to the shader program directly.
+    s_add_u32       ttmp0, ttmp0, 4   // S_TRAP case, add 4 to ttmp0
+    s_addc_u32  ttmp1, ttmp1, 0
+L_EXCP_CASE:
+    s_and_b32   ttmp1, ttmp1, 0xFFFF
+    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+    s_rfe_b64       [ttmp0, ttmp1]
+end
+    // *********        End handling of non-CWSR traps   *******************
+
+/**************************************************************************/
+/*                      save routine                                      */
+/**************************************************************************/
+
+L_SAVE:
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime   s_g8sr_ts_save_s
+        s_waitcnt lgkmcnt(0)         //FIXME, will cause xnack??
+end
+
+    //check whether there is mem_viol
+    s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+    s_and_b32   s_save_trapsts, s_save_trapsts, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK
+    s_cbranch_scc0  L_NO_PC_REWIND
+
+    //if so, need rewind PC assuming GDS operation gets NACKed
+    s_mov_b32       s_save_tmp, 0                                                           //clear mem_viol bit
+    s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT, 1), s_save_tmp    //clear mem_viol bit
+    s_and_b32       s_save_pc_hi, s_save_pc_hi, 0x0000ffff    //pc[47:32]
+    s_sub_u32       s_save_pc_lo, s_save_pc_lo, 8             //pc[31:0]-8
+    s_subb_u32      s_save_pc_hi, s_save_pc_hi, 0x0           // -scc
+
+L_NO_PC_REWIND:
+    s_mov_b32       s_save_tmp, 0                                                           //clear saveCtx bit
+    s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_SAVECTX_SHIFT, 1), s_save_tmp     //clear saveCtx bit
+
+    s_mov_b32       s_save_xnack_mask_lo,   xnack_mask_lo                                   //save XNACK_MASK
+    s_mov_b32       s_save_xnack_mask_hi,   xnack_mask_hi    //save XNACK must before any memory operation
+    s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_RCNT_SHIFT, SQ_WAVE_IB_STS_RCNT_SIZE)                   //save RCNT
+    s_lshl_b32      s_save_tmp, s_save_tmp, S_SAVE_PC_HI_RCNT_SHIFT
+    s_or_b32        s_save_pc_hi, s_save_pc_hi, s_save_tmp
+    s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, SQ_WAVE_IB_STS_FIRST_REPLAY_SIZE)   //save FIRST_REPLAY
+    s_lshl_b32      s_save_tmp, s_save_tmp, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+    s_or_b32        s_save_pc_hi, s_save_pc_hi, s_save_tmp
+    s_getreg_b32    s_save_tmp, hwreg(HW_REG_IB_STS)                                        //clear RCNT and FIRST_REPLAY in IB_STS
+    s_and_b32       s_save_tmp, s_save_tmp, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK_NEG
+
+    s_setreg_b32    hwreg(HW_REG_IB_STS), s_save_tmp
+
+    /*      inform SPI the readiness and wait for SPI's go signal */
+    s_mov_b32       s_save_exec_lo, exec_lo                                                 //save EXEC and use EXEC for the go signal from SPI
+    s_mov_b32       s_save_exec_hi, exec_hi
+    s_mov_b64       exec,   0x0                                                             //clear EXEC to get ready to receive
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime  s_g8sr_ts_sq_save_msg
+        s_waitcnt lgkmcnt(0)
+end
+
+    if (EMU_RUN_HACK)
+
+    else
+        s_sendmsg   sendmsg(MSG_SAVEWAVE)  //send SPI a message and wait for SPI's write to EXEC
+    end
+
+  L_SLEEP:
+    s_sleep 0x2                // sleep 1 (64clk) is not enough for 8 waves per SIMD, which will cause SQ hang, since the 7,8th wave could not get arbit to exec inst, while other waves are stuck into the sleep-loop and waiting for wrexec!=0
+
+    if (EMU_RUN_HACK)
+
+    else
+        s_cbranch_execz L_SLEEP
+    end
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime  s_g8sr_ts_spi_wrexec
+        s_waitcnt lgkmcnt(0)
+end
+
+    /*      setup Resource Contants    */
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+        //calculate wd_addr using absolute thread id
+        v_readlane_b32 s_save_tmp, v9, 0
+        s_lshr_b32 s_save_tmp, s_save_tmp, 6
+        s_mul_i32 s_save_tmp, s_save_tmp, WAVE_SPACE
+        s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+        s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+        s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+    else
+    end
+    if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_SINGLE_WAVE))
+        s_add_i32 s_save_spi_init_lo, s_save_tmp, WG_BASE_ADDR_LO
+        s_mov_b32 s_save_spi_init_hi, WG_BASE_ADDR_HI
+        s_and_b32 s_save_spi_init_hi, s_save_spi_init_hi, CTX_SAVE_CONTROL
+    else
+    end
+
+
+    s_mov_b32       s_save_buf_rsrc0,   s_save_spi_init_lo                                                      //base_addr_lo
+    s_and_b32       s_save_buf_rsrc1,   s_save_spi_init_hi, 0x0000FFFF                                          //base_addr_hi
+    s_or_b32        s_save_buf_rsrc1,   s_save_buf_rsrc1,  S_SAVE_BUF_RSRC_WORD1_STRIDE
+    s_mov_b32       s_save_buf_rsrc2,   0                                                                       //NUM_RECORDS initial value = 0 (in bytes) although not neccessarily inited
+    s_mov_b32       s_save_buf_rsrc3,   S_SAVE_BUF_RSRC_WORD3_MISC
+    s_and_b32       s_save_tmp,         s_save_spi_init_hi, S_SAVE_SPI_INIT_ATC_MASK
+    s_lshr_b32      s_save_tmp,         s_save_tmp, (S_SAVE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)         //get ATC bit into position
+    s_or_b32        s_save_buf_rsrc3,   s_save_buf_rsrc3,  s_save_tmp                                           //or ATC
+    s_and_b32       s_save_tmp,         s_save_spi_init_hi, S_SAVE_SPI_INIT_MTYPE_MASK
+    s_lshr_b32      s_save_tmp,         s_save_tmp, (S_SAVE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)     //get MTYPE bits into position
+    s_or_b32        s_save_buf_rsrc3,   s_save_buf_rsrc3,  s_save_tmp                                           //or MTYPE
+
+    //FIXME  right now s_save_m0/s_save_mem_offset use tma_lo/tma_hi  (might need to save them before using them?)
+    s_mov_b32       s_save_m0,          m0                                                                  //save M0
+
+    /*      global mem offset           */
+    s_mov_b32       s_save_mem_offset,  0x0                                                                     //mem offset initial value = 0
+
+
+
+
+    /*      save HW registers   */
+    //////////////////////////////
+
+  L_SAVE_HWREG:
+        // HWREG SR memory offset : size(VGPR)+size(SGPR)
+       get_vgpr_size_bytes(s_save_mem_offset)
+       get_sgpr_size_bytes(s_save_tmp)
+       s_add_u32 s_save_mem_offset, s_save_mem_offset, s_save_tmp
+
+
+    s_mov_b32       s_save_buf_rsrc2, 0x4                               //NUM_RECORDS   in bytes
+    if (SWIZZLE_EN)
+        s_add_u32       s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0                     //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
+    end
+
+
+    write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)                  //M0
+
+    if ((EMU_RUN_HACK) && (EMU_RUN_HACK_SAVE_FIRST_TIME))
+        s_add_u32 s_save_pc_lo, s_save_pc_lo, 4             //pc[31:0]+4
+        s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0          //carry bit over
+        s_mov_b32   tba_lo, EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_LO
+        s_mov_b32   tba_hi, EMU_RUN_HACK_SAVE_FIRST_TIME_TBA_HI
+    end
+
+    write_hwreg_to_mem(s_save_pc_lo, s_save_buf_rsrc0, s_save_mem_offset)                   //PC
+    write_hwreg_to_mem(s_save_pc_hi, s_save_buf_rsrc0, s_save_mem_offset)
+    write_hwreg_to_mem(s_save_exec_lo, s_save_buf_rsrc0, s_save_mem_offset)             //EXEC
+    write_hwreg_to_mem(s_save_exec_hi, s_save_buf_rsrc0, s_save_mem_offset)
+    write_hwreg_to_mem(s_save_status, s_save_buf_rsrc0, s_save_mem_offset)              //STATUS
+
+    //s_save_trapsts conflicts with s_save_alloc_size
+    s_getreg_b32    s_save_trapsts, hwreg(HW_REG_TRAPSTS)
+    write_hwreg_to_mem(s_save_trapsts, s_save_buf_rsrc0, s_save_mem_offset)             //TRAPSTS
+
+    write_hwreg_to_mem(s_save_xnack_mask_lo, s_save_buf_rsrc0, s_save_mem_offset)           //XNACK_MASK_LO
+    write_hwreg_to_mem(s_save_xnack_mask_hi, s_save_buf_rsrc0, s_save_mem_offset)           //XNACK_MASK_HI
+
+    //use s_save_tmp would introduce conflict here between s_save_tmp and s_save_buf_rsrc2
+    s_getreg_b32    s_save_m0, hwreg(HW_REG_MODE)                                                   //MODE
+    write_hwreg_to_mem(s_save_m0, s_save_buf_rsrc0, s_save_mem_offset)
+    write_hwreg_to_mem(tba_lo, s_save_buf_rsrc0, s_save_mem_offset)                     //TBA_LO
+    write_hwreg_to_mem(tba_hi, s_save_buf_rsrc0, s_save_mem_offset)                     //TBA_HI
+
+
+
+    /*      the first wave in the threadgroup    */
+        // save fist_wave bits in tba_hi unused bit.26
+    s_and_b32       s_save_tmp, s_save_spi_init_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK     // extract fisrt wave bit
+    //s_or_b32        tba_hi, s_save_tmp, tba_hi                                        // save first wave bit to tba_hi.bits[26]
+    s_mov_b32        s_save_exec_hi, 0x0
+    s_or_b32         s_save_exec_hi, s_save_tmp, s_save_exec_hi                          // save first wave bit to s_save_exec_hi.bits[26]
+
+
+    /*          save SGPRs      */
+        // Save SGPR before LDS save, then the s0 to s4 can be used during LDS save...
+    //////////////////////////////
+
+    // SGPR SR memory offset : size(VGPR)
+    get_vgpr_size_bytes(s_save_mem_offset)
+    // TODO, change RSRC word to rearrange memory layout for SGPRS
+
+    s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)               //spgr_size
+    s_add_u32       s_save_alloc_size, s_save_alloc_size, 1
+    s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 4                         //Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value)
+
+    if (SGPR_SAVE_USE_SQC)
+        s_lshl_b32      s_save_buf_rsrc2,   s_save_alloc_size, 2                    //NUM_RECORDS in bytes
+    else
+        s_lshl_b32      s_save_buf_rsrc2,   s_save_alloc_size, 8                    //NUM_RECORDS in bytes (64 threads)
+    end
+
+    if (SWIZZLE_EN)
+        s_add_u32       s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0                     //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
+    end
+
+
+    // backup s_save_buf_rsrc0,1 to s_save_pc_lo/hi, since write_16sgpr_to_mem function will change the rsrc0
+    //s_mov_b64 s_save_pc_lo, s_save_buf_rsrc0
+    s_mov_b64 s_save_xnack_mask_lo, s_save_buf_rsrc0
+    s_add_u32 s_save_buf_rsrc0, s_save_buf_rsrc0, s_save_mem_offset
+    s_addc_u32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0
+
+    s_mov_b32       m0, 0x0                         //SGPR initial index value =0
+  L_SAVE_SGPR_LOOP:
+    // SGPR is allocated in 16 SGPR granularity
+    s_movrels_b64   s0, s0     //s0 = s[0+m0], s1 = s[1+m0]
+    s_movrels_b64   s2, s2     //s2 = s[2+m0], s3 = s[3+m0]
+    s_movrels_b64   s4, s4     //s4 = s[4+m0], s5 = s[5+m0]
+    s_movrels_b64   s6, s6     //s6 = s[6+m0], s7 = s[7+m0]
+    s_movrels_b64   s8, s8     //s8 = s[8+m0], s9 = s[9+m0]
+    s_movrels_b64   s10, s10   //s10 = s[10+m0], s11 = s[11+m0]
+    s_movrels_b64   s12, s12   //s12 = s[12+m0], s13 = s[13+m0]
+    s_movrels_b64   s14, s14   //s14 = s[14+m0], s15 = s[15+m0]
+
+    write_16sgpr_to_mem(s0, s_save_buf_rsrc0, s_save_mem_offset) //PV: the best performance should be using s_buffer_store_dwordx4
+    s_add_u32       m0, m0, 16                                                      //next sgpr index
+    s_cmp_lt_u32    m0, s_save_alloc_size                                           //scc = (m0 < s_save_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_SAVE_SGPR_LOOP                                    //SGPR save is complete?
+    // restore s_save_buf_rsrc0,1
+    //s_mov_b64 s_save_buf_rsrc0, s_save_pc_lo
+    s_mov_b64 s_save_buf_rsrc0, s_save_xnack_mask_lo
+
+
+
+
+    /*          save first 4 VGPR, then LDS save could use   */
+        // each wave will alloc 4 vgprs at least...
+    /////////////////////////////////////////////////////////////////////////////////////
+
+    s_mov_b32       s_save_mem_offset, 0
+    s_mov_b32       exec_lo, 0xFFFFFFFF                                             //need every thread from now on
+    s_mov_b32       exec_hi, 0xFFFFFFFF
+
+    if (SWIZZLE_EN)
+        s_add_u32       s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0                     //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
+    end
+
+
+    // VGPR Allocated in 4-GPR granularity
+
+if G8SR_VGPR_SR_IN_DWX4
+        // the const stride for DWx4 is 4*4 bytes
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+
+        buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE  // reset const stride to 4 bytes
+else
+        buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+        buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
+        buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
+        buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
+end
+
+
+
+    /*          save LDS        */
+    //////////////////////////////
+
+  L_SAVE_LDS:
+
+        // Change EXEC to all threads...
+    s_mov_b32       exec_lo, 0xFFFFFFFF   //need every thread from now on
+    s_mov_b32       exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)             //lds_size
+    s_and_b32       s_save_alloc_size, s_save_alloc_size, 0xFFFFFFFF                //lds_size is zero?
+    s_cbranch_scc0  L_SAVE_LDS_DONE                                                                            //no lds used? jump to L_SAVE_DONE
+
+    s_barrier               //LDS is used? wait for other waves in the same TG
+    //s_and_b32     s_save_tmp, tba_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK                //exec is still used here
+    s_and_b32       s_save_tmp, s_save_exec_hi, S_SAVE_SPI_INIT_FIRST_WAVE_MASK                //exec is still used here
+    s_cbranch_scc0  L_SAVE_LDS_DONE
+
+        // first wave do LDS save;
+
+    s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 6                         //LDS size in dwords = lds_size * 64dw
+    s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 2                         //LDS size in bytes
+    s_mov_b32       s_save_buf_rsrc2,  s_save_alloc_size                            //NUM_RECORDS in bytes
+
+    // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+    //
+    get_vgpr_size_bytes(s_save_mem_offset)
+    get_sgpr_size_bytes(s_save_tmp)
+    s_add_u32  s_save_mem_offset, s_save_mem_offset, s_save_tmp
+    s_add_u32 s_save_mem_offset, s_save_mem_offset, get_hwreg_size_bytes()
+
+
+    if (SWIZZLE_EN)
+        s_add_u32       s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0       //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_save_buf_rsrc2,  0x1000000                  //NUM_RECORDS in bytes
+    end
+
+    s_mov_b32       m0, 0x0                                               //lds_offset initial value = 0
+
+
+var LDS_DMA_ENABLE = 0
+var UNROLL = 0
+if UNROLL==0 && LDS_DMA_ENABLE==1
+        s_mov_b32  s3, 256*2
+        s_nop 0
+        s_nop 0
+        s_nop 0
+  L_SAVE_LDS_LOOP:
+        //TODO: looks the 2 buffer_store/load clause for s/r will hurt performance.???
+    if (SAVE_LDS)     //SPI always alloc LDS space in 128DW granularity
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1            // first 64DW
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256 // second 64DW
+    end
+
+    s_add_u32       m0, m0, s3                                          //every buffer_store_lds does 256 bytes
+    s_add_u32       s_save_mem_offset, s_save_mem_offset, s3                            //mem offset increased by 256 bytes
+    s_cmp_lt_u32    m0, s_save_alloc_size                                               //scc=(m0 < s_save_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_SAVE_LDS_LOOP                                                     //LDS save is complete?
+
+elsif LDS_DMA_ENABLE==1 && UNROLL==1 // UNROOL  , has ichace miss
+      // store from higest LDS address to lowest
+      s_mov_b32  s3, 256*2
+      s_sub_u32  m0, s_save_alloc_size, s3
+      s_add_u32 s_save_mem_offset, s_save_mem_offset, m0
+      s_lshr_b32 s_save_alloc_size, s_save_alloc_size, 9   // how many 128 trunks...
+      s_sub_u32 s_save_alloc_size, 128, s_save_alloc_size   // store from higheset addr to lowest
+      s_mul_i32 s_save_alloc_size, s_save_alloc_size, 6*4   // PC offset increment,  each LDS save block cost 6*4 Bytes instruction
+      s_add_u32 s_save_alloc_size, s_save_alloc_size, 3*4   //2is the below 2 inst...//s_addc and s_setpc
+      s_nop 0
+      s_nop 0
+      s_nop 0   //pad 3 dw to let LDS_DMA align with 64Bytes
+      s_getpc_b64 s[0:1]                              // reuse s[0:1], since s[0:1] already saved
+      s_add_u32   s0, s0,s_save_alloc_size
+      s_addc_u32  s1, s1, 0
+      s_setpc_b64 s[0:1]
+
+
+       for var i =0; i< 128; i++
+            // be careful to make here a 64Byte aligned address, which could improve performance...
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:0           // first 64DW
+            buffer_store_lds_dword s_save_buf_rsrc0, s_save_mem_offset lds:1 offset:256           // second 64DW
+
+        if i!=127
+        s_sub_u32  m0, m0, s3      // use a sgpr to shrink 2DW-inst to 1DW inst to improve performance , i.e.  pack more LDS_DMA inst to one Cacheline
+            s_sub_u32  s_save_mem_offset, s_save_mem_offset,  s3
+            end
+       end
+
+else   // BUFFER_STORE
+      v_mbcnt_lo_u32_b32 v2, 0xffffffff, 0x0
+      v_mbcnt_hi_u32_b32 v3, 0xffffffff, v2     // tid
+      v_mul_i32_i24 v2, v3, 8   // tid*8
+      v_mov_b32 v3, 256*2
+      s_mov_b32 m0, 0x10000
+      s_mov_b32 s0, s_save_buf_rsrc3
+      s_and_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0xFF7FFFFF    // disable add_tid
+      s_or_b32 s_save_buf_rsrc3, s_save_buf_rsrc3, 0x58000   //DFMT
+
+L_SAVE_LDS_LOOP_VECTOR:
+      ds_read_b64 v[0:1], v2    //x =LDS[a], byte address
+      s_waitcnt lgkmcnt(0)
+      buffer_store_dwordx2  v[0:1], v2, s_save_buf_rsrc0, s_save_mem_offset offen:1  glc:1  slc:1
+//      s_waitcnt vmcnt(0)
+      v_add_u32 v2, vcc[0:1], v2, v3
+      v_cmp_lt_u32 vcc[0:1], v2, s_save_alloc_size
+      s_cbranch_vccnz L_SAVE_LDS_LOOP_VECTOR
+
+      // restore rsrc3
+      s_mov_b32 s_save_buf_rsrc3, s0
+
+end
+
+L_SAVE_LDS_DONE:
+
+
+    /*          save VGPRs  - set the Rest VGPRs        */
+    //////////////////////////////////////////////////////////////////////////////////////
+  L_SAVE_VGPR:
+    // VGPR SR memory offset: 0
+    // TODO rearrange the RSRC words to use swizzle for VGPR save...
+
+    s_mov_b32       s_save_mem_offset, (0+256*4)                                    // for the rest VGPRs
+    s_mov_b32       exec_lo, 0xFFFFFFFF                                             //need every thread from now on
+    s_mov_b32       exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_save_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)                   //vpgr_size
+    s_add_u32       s_save_alloc_size, s_save_alloc_size, 1
+    s_lshl_b32      s_save_alloc_size, s_save_alloc_size, 2                         //Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)   //FIXME for GFX, zero is possible
+    s_lshl_b32      s_save_buf_rsrc2,  s_save_alloc_size, 8                         //NUM_RECORDS in bytes (64 threads*4)
+    if (SWIZZLE_EN)
+        s_add_u32       s_save_buf_rsrc2, s_save_buf_rsrc2, 0x0                     //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_save_buf_rsrc2,  0x1000000                                //NUM_RECORDS in bytes
+    end
+
+
+    // VGPR Allocated in 4-GPR granularity
+
+if G8SR_VGPR_SR_IN_DWX4
+        // the const stride for DWx4 is 4*4 bytes
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, G8SR_SAVE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+
+        s_mov_b32         m0, 4     // skip first 4 VGPRs
+        s_cmp_lt_u32      m0, s_save_alloc_size
+        s_cbranch_scc0    L_SAVE_VGPR_LOOP_END      // no more vgprs
+
+        s_set_gpr_idx_on  m0, 0x1   // This will change M0
+        s_add_u32         s_save_alloc_size, s_save_alloc_size, 0x1000  // because above inst change m0
+L_SAVE_VGPR_LOOP:
+        v_mov_b32         v0, v0   // v0 = v[0+m0]
+        v_mov_b32         v1, v1
+        v_mov_b32         v2, v2
+        v_mov_b32         v3, v3
+
+
+        buffer_store_dwordx4 v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+        s_add_u32         m0, m0, 4
+        s_add_u32         s_save_mem_offset, s_save_mem_offset, 256*4
+        s_cmp_lt_u32      m0, s_save_alloc_size
+    s_cbranch_scc1  L_SAVE_VGPR_LOOP                                                //VGPR save is complete?
+    s_set_gpr_idx_off
+L_SAVE_VGPR_LOOP_END:
+
+        s_and_b32 s_save_buf_rsrc1, s_save_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+        s_or_b32  s_save_buf_rsrc1, s_save_buf_rsrc1, S_SAVE_BUF_RSRC_WORD1_STRIDE  // reset const stride to 4 bytes
+else
+    // VGPR store using dw burst
+    s_mov_b32         m0, 0x4   //VGPR initial index value =0
+    s_cmp_lt_u32      m0, s_save_alloc_size
+    s_cbranch_scc0    L_SAVE_VGPR_END
+
+
+    s_set_gpr_idx_on    m0, 0x1 //M0[7:0] = M0[7:0] and M0[15:12] = 0x1
+    s_add_u32       s_save_alloc_size, s_save_alloc_size, 0x1000                    //add 0x1000 since we compare m0 against it later
+
+  L_SAVE_VGPR_LOOP:
+    v_mov_b32       v0, v0              //v0 = v[0+m0]
+    v_mov_b32       v1, v1              //v0 = v[0+m0]
+    v_mov_b32       v2, v2              //v0 = v[0+m0]
+    v_mov_b32       v3, v3              //v0 = v[0+m0]
+
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)
+        tbuffer_store_format_x v0, v0, s_save_buf_rsrc0, s_save_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+        buffer_store_dword v0, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1
+        buffer_store_dword v1, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256
+        buffer_store_dword v2, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*2
+        buffer_store_dword v3, v0, s_save_buf_rsrc0, s_save_mem_offset slc:1 glc:1  offset:256*3
+    end
+
+    s_add_u32       m0, m0, 4                                                       //next vgpr index
+    s_add_u32       s_save_mem_offset, s_save_mem_offset, 256*4                     //every buffer_store_dword does 256 bytes
+    s_cmp_lt_u32    m0, s_save_alloc_size                                           //scc = (m0 < s_save_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_SAVE_VGPR_LOOP                                                //VGPR save is complete?
+    s_set_gpr_idx_off
+end
+
+L_SAVE_VGPR_END:
+
+
+
+
+
+
+    /*     S_PGM_END_SAVED  */                              //FIXME  graphics ONLY
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_SAVE_NORMAL_EXIT))
+        s_and_b32 s_save_pc_hi, s_save_pc_hi, 0x0000ffff    //pc[47:32]
+        s_add_u32 s_save_pc_lo, s_save_pc_lo, 4             //pc[31:0]+4
+        s_addc_u32 s_save_pc_hi, s_save_pc_hi, 0x0          //carry bit over
+        s_rfe_b64 s_save_pc_lo                              //Return to the main shader program
+    else
+    end
+
+// Save Done timestamp
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime   s_g8sr_ts_save_d
+        // SGPR SR memory offset : size(VGPR)
+        get_vgpr_size_bytes(s_save_mem_offset)
+        s_add_u32 s_save_mem_offset, s_save_mem_offset, G8SR_DEBUG_TS_SAVE_D_OFFSET
+        s_waitcnt lgkmcnt(0)         //FIXME, will cause xnack??
+        // Need reset rsrc2??
+        s_mov_b32 m0, s_save_mem_offset
+        s_mov_b32 s_save_buf_rsrc2,  0x1000000                                  //NUM_RECORDS in bytes
+        s_buffer_store_dwordx2 s_g8sr_ts_save_d, s_save_buf_rsrc0, m0       glc:1
+end
+
+
+    s_branch    L_END_PGM
+
+
+
+/**************************************************************************/
+/*                      restore routine                                   */
+/**************************************************************************/
+
+L_RESTORE:
+    /*      Setup Resource Contants    */
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+        //calculate wd_addr using absolute thread id
+        v_readlane_b32 s_restore_tmp, v9, 0
+        s_lshr_b32 s_restore_tmp, s_restore_tmp, 6
+        s_mul_i32 s_restore_tmp, s_restore_tmp, WAVE_SPACE
+        s_add_i32 s_restore_spi_init_lo, s_restore_tmp, WG_BASE_ADDR_LO
+        s_mov_b32 s_restore_spi_init_hi, WG_BASE_ADDR_HI
+        s_and_b32 s_restore_spi_init_hi, s_restore_spi_init_hi, CTX_RESTORE_CONTROL
+    else
+    end
+
+if G8SR_DEBUG_TIMESTAMP
+        s_memrealtime   s_g8sr_ts_restore_s
+        s_waitcnt lgkmcnt(0)         //FIXME, will cause xnack??
+        // tma_lo/hi are sgpr 110, 111, which will not used for 112 SGPR allocated case...
+        s_mov_b32 s_restore_pc_lo, s_g8sr_ts_restore_s[0]
+        s_mov_b32 s_restore_pc_hi, s_g8sr_ts_restore_s[1]   //backup ts to ttmp0/1, sicne exec will be finally restored..
+end
+
+
+
+    s_mov_b32       s_restore_buf_rsrc0,    s_restore_spi_init_lo                                                           //base_addr_lo
+    s_and_b32       s_restore_buf_rsrc1,    s_restore_spi_init_hi, 0x0000FFFF                                               //base_addr_hi
+    s_or_b32        s_restore_buf_rsrc1,    s_restore_buf_rsrc1,  S_RESTORE_BUF_RSRC_WORD1_STRIDE
+    s_mov_b32       s_restore_buf_rsrc2,    0                                                                               //NUM_RECORDS initial value = 0 (in bytes)
+    s_mov_b32       s_restore_buf_rsrc3,    S_RESTORE_BUF_RSRC_WORD3_MISC
+    s_and_b32       s_restore_tmp,          s_restore_spi_init_hi, S_RESTORE_SPI_INIT_ATC_MASK
+    s_lshr_b32      s_restore_tmp,          s_restore_tmp, (S_RESTORE_SPI_INIT_ATC_SHIFT-SQ_BUF_RSRC_WORD1_ATC_SHIFT)       //get ATC bit into position
+    s_or_b32        s_restore_buf_rsrc3,    s_restore_buf_rsrc3,  s_restore_tmp                                             //or ATC
+    s_and_b32       s_restore_tmp,          s_restore_spi_init_hi, S_RESTORE_SPI_INIT_MTYPE_MASK
+    s_lshr_b32      s_restore_tmp,          s_restore_tmp, (S_RESTORE_SPI_INIT_MTYPE_SHIFT-SQ_BUF_RSRC_WORD3_MTYPE_SHIFT)   //get MTYPE bits into position
+    s_or_b32        s_restore_buf_rsrc3,    s_restore_buf_rsrc3,  s_restore_tmp                                             //or MTYPE
+
+    /*      global mem offset           */
+//  s_mov_b32       s_restore_mem_offset, 0x0                               //mem offset initial value = 0
+
+    /*      the first wave in the threadgroup    */
+    s_and_b32       s_restore_tmp, s_restore_spi_init_hi, S_RESTORE_SPI_INIT_FIRST_WAVE_MASK
+    s_cbranch_scc0  L_RESTORE_VGPR
+
+    /*          restore LDS     */
+    //////////////////////////////
+  L_RESTORE_LDS:
+
+    s_mov_b32       exec_lo, 0xFFFFFFFF                                                     //need every thread from now on   //be consistent with SAVE although can be moved ahead
+    s_mov_b32       exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_LDS_ALLOC,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT,SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)              //lds_size
+    s_and_b32       s_restore_alloc_size, s_restore_alloc_size, 0xFFFFFFFF                  //lds_size is zero?
+    s_cbranch_scc0  L_RESTORE_VGPR                                                          //no lds used? jump to L_RESTORE_VGPR
+    s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 6                           //LDS size in dwords = lds_size * 64dw
+    s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 2                           //LDS size in bytes
+    s_mov_b32       s_restore_buf_rsrc2,    s_restore_alloc_size                            //NUM_RECORDS in bytes
+
+    // LDS at offset: size(VGPR)+SIZE(SGPR)+SIZE(HWREG)
+    //
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32  s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+    s_add_u32  s_restore_mem_offset, s_restore_mem_offset, get_hwreg_size_bytes()            //FIXME, Check if offset overflow???
+
+
+    if (SWIZZLE_EN)
+        s_add_u32       s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0                       //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
+    end
+    s_mov_b32       m0, 0x0                                                                 //lds_offset initial value = 0
+
+  L_RESTORE_LDS_LOOP:
+    if (SAVE_LDS)
+        buffer_load_dword   v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1                    // first 64DW
+        buffer_load_dword   v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset lds:1 offset:256         // second 64DW
+    end
+    s_add_u32       m0, m0, 256*2                                               // 128 DW
+    s_add_u32       s_restore_mem_offset, s_restore_mem_offset, 256*2           //mem offset increased by 128DW
+    s_cmp_lt_u32    m0, s_restore_alloc_size                                    //scc=(m0 < s_restore_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_RESTORE_LDS_LOOP                                                      //LDS restore is complete?
+
+
+    /*          restore VGPRs       */
+    //////////////////////////////
+  L_RESTORE_VGPR:
+        // VGPR SR memory offset : 0
+    s_mov_b32       s_restore_mem_offset, 0x0
+    s_mov_b32       exec_lo, 0xFFFFFFFF                                                     //need every thread from now on   //be consistent with SAVE although can be moved ahead
+    s_mov_b32       exec_hi, 0xFFFFFFFF
+
+    s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)    //vpgr_size
+    s_add_u32       s_restore_alloc_size, s_restore_alloc_size, 1
+    s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 2                           //Number of VGPRs = (vgpr_size + 1) * 4    (non-zero value)
+    s_lshl_b32      s_restore_buf_rsrc2,  s_restore_alloc_size, 8                           //NUM_RECORDS in bytes (64 threads*4)
+    if (SWIZZLE_EN)
+        s_add_u32       s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0                       //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
+    end
+
+if G8SR_VGPR_SR_IN_DWX4
+     get_vgpr_size_bytes(s_restore_mem_offset)
+     s_sub_u32         s_restore_mem_offset, s_restore_mem_offset, 256*4
+
+     // the const stride for DWx4 is 4*4 bytes
+     s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+     s_or_b32  s_restore_buf_rsrc1, s_restore_buf_rsrc1, G8SR_RESTORE_BUF_RSRC_WORD1_STRIDE_DWx4  // const stride to 4*4 bytes
+
+     s_mov_b32         m0, s_restore_alloc_size
+     s_set_gpr_idx_on  m0, 0x8    // Note.. This will change m0
+
+L_RESTORE_VGPR_LOOP:
+     buffer_load_dwordx4 v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+     s_waitcnt vmcnt(0)
+     s_sub_u32         m0, m0, 4
+     v_mov_b32         v0, v0   // v[0+m0] = v0
+     v_mov_b32         v1, v1
+     v_mov_b32         v2, v2
+     v_mov_b32         v3, v3
+     s_sub_u32         s_restore_mem_offset, s_restore_mem_offset, 256*4
+     s_cmp_eq_u32      m0, 0x8000
+     s_cbranch_scc0    L_RESTORE_VGPR_LOOP
+     s_set_gpr_idx_off
+
+     s_and_b32 s_restore_buf_rsrc1, s_restore_buf_rsrc1, 0x0000FFFF   // reset const stride to 0
+     s_or_b32  s_restore_buf_rsrc1, s_restore_buf_rsrc1, S_RESTORE_BUF_RSRC_WORD1_STRIDE  // const stride to 4*4 bytes
+
+else
+    // VGPR load using dw burst
+    s_mov_b32       s_restore_mem_offset_save, s_restore_mem_offset     // restore start with v1, v0 will be the last
+    s_add_u32       s_restore_mem_offset, s_restore_mem_offset, 256*4
+    s_mov_b32       m0, 4                               //VGPR initial index value = 1
+    s_set_gpr_idx_on  m0, 0x8                       //M0[7:0] = M0[7:0] and M0[15:12] = 0x8
+    s_add_u32       s_restore_alloc_size, s_restore_alloc_size, 0x8000                      //add 0x8000 since we compare m0 against it later
+
+  L_RESTORE_VGPR_LOOP:
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)
+        tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+        buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1
+        buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256
+        buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*2
+        buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset slc:1 glc:1 offset:256*3
+    end
+    s_waitcnt       vmcnt(0)                                                                //ensure data ready
+    v_mov_b32       v0, v0                                                                  //v[0+m0] = v0
+    v_mov_b32       v1, v1
+    v_mov_b32       v2, v2
+    v_mov_b32       v3, v3
+    s_add_u32       m0, m0, 4                                                               //next vgpr index
+    s_add_u32       s_restore_mem_offset, s_restore_mem_offset, 256*4                           //every buffer_load_dword does 256 bytes
+    s_cmp_lt_u32    m0, s_restore_alloc_size                                                //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+    s_cbranch_scc1  L_RESTORE_VGPR_LOOP                                                     //VGPR restore (except v0) is complete?
+    s_set_gpr_idx_off
+                                                                                            /* VGPR restore on v0 */
+    if(USE_MTBUF_INSTEAD_OF_MUBUF)
+        tbuffer_load_format_x v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save format:BUF_NUM_FORMAT_FLOAT format: BUF_DATA_FORMAT_32 slc:1 glc:1
+    else
+        buffer_load_dword v0, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1
+        buffer_load_dword v1, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256
+        buffer_load_dword v2, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256*2
+        buffer_load_dword v3, v0, s_restore_buf_rsrc0, s_restore_mem_offset_save    slc:1 glc:1 offset:256*3
+    end
+
+end
+
+    /*          restore SGPRs       */
+    //////////////////////////////
+
+    // SGPR SR memory offset : size(VGPR)
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+    s_sub_u32 s_restore_mem_offset, s_restore_mem_offset, 16*4     // restore SGPR from S[n] to S[0], by 16 sgprs group
+    // TODO, change RSRC word to rearrange memory layout for SGPRS
+
+    s_getreg_b32    s_restore_alloc_size, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)                //spgr_size
+    s_add_u32       s_restore_alloc_size, s_restore_alloc_size, 1
+    s_lshl_b32      s_restore_alloc_size, s_restore_alloc_size, 4                           //Number of SGPRs = (sgpr_size + 1) * 16   (non-zero value)
+
+    if (SGPR_SAVE_USE_SQC)
+        s_lshl_b32      s_restore_buf_rsrc2,    s_restore_alloc_size, 2                     //NUM_RECORDS in bytes
+    else
+        s_lshl_b32      s_restore_buf_rsrc2,    s_restore_alloc_size, 8                     //NUM_RECORDS in bytes (64 threads)
+    end
+    if (SWIZZLE_EN)
+        s_add_u32       s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0                       //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
+    end
+
+    /* If 112 SGPRs ar allocated, 4 sgprs are not used TBA(108,109),TMA(110,111),
+       However, we are safe to restore these 4 SGPRs anyway, since TBA,TMA will later be restored by HWREG
+    */
+    s_mov_b32 m0, s_restore_alloc_size
+
+ L_RESTORE_SGPR_LOOP:
+    read_16sgpr_from_mem(s0, s_restore_buf_rsrc0, s_restore_mem_offset)  //PV: further performance improvement can be made
+    s_waitcnt       lgkmcnt(0)                                                              //ensure data ready
+
+    s_sub_u32 m0, m0, 16    // Restore from S[n] to S[0]
+
+    s_movreld_b64   s0, s0      //s[0+m0] = s0
+    s_movreld_b64   s2, s2
+    s_movreld_b64   s4, s4
+    s_movreld_b64   s6, s6
+    s_movreld_b64   s8, s8
+    s_movreld_b64   s10, s10
+    s_movreld_b64   s12, s12
+    s_movreld_b64   s14, s14
+
+    s_cmp_eq_u32    m0, 0               //scc = (m0 < s_restore_alloc_size) ? 1 : 0
+    s_cbranch_scc0  L_RESTORE_SGPR_LOOP             //SGPR restore (except s0) is complete?
+
+    /*      restore HW registers    */
+    //////////////////////////////
+  L_RESTORE_HWREG:
+
+
+if G8SR_DEBUG_TIMESTAMP
+      s_mov_b32 s_g8sr_ts_restore_s[0], s_restore_pc_lo
+      s_mov_b32 s_g8sr_ts_restore_s[1], s_restore_pc_hi
+end
+
+    // HWREG SR memory offset : size(VGPR)+size(SGPR)
+    get_vgpr_size_bytes(s_restore_mem_offset)
+    get_sgpr_size_bytes(s_restore_tmp)
+    s_add_u32 s_restore_mem_offset, s_restore_mem_offset, s_restore_tmp
+
+
+    s_mov_b32       s_restore_buf_rsrc2, 0x4                                                //NUM_RECORDS   in bytes
+    if (SWIZZLE_EN)
+        s_add_u32       s_restore_buf_rsrc2, s_restore_buf_rsrc2, 0x0                       //FIXME need to use swizzle to enable bounds checking?
+    else
+        s_mov_b32       s_restore_buf_rsrc2,  0x1000000                                     //NUM_RECORDS in bytes
+    end
+
+    read_hwreg_from_mem(s_restore_m0, s_restore_buf_rsrc0, s_restore_mem_offset)                    //M0
+    read_hwreg_from_mem(s_restore_pc_lo, s_restore_buf_rsrc0, s_restore_mem_offset)             //PC
+    read_hwreg_from_mem(s_restore_pc_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+    read_hwreg_from_mem(s_restore_exec_lo, s_restore_buf_rsrc0, s_restore_mem_offset)               //EXEC
+    read_hwreg_from_mem(s_restore_exec_hi, s_restore_buf_rsrc0, s_restore_mem_offset)
+    read_hwreg_from_mem(s_restore_status, s_restore_buf_rsrc0, s_restore_mem_offset)                //STATUS
+    read_hwreg_from_mem(s_restore_trapsts, s_restore_buf_rsrc0, s_restore_mem_offset)               //TRAPSTS
+    read_hwreg_from_mem(xnack_mask_lo, s_restore_buf_rsrc0, s_restore_mem_offset)                   //XNACK_MASK_LO
+    read_hwreg_from_mem(xnack_mask_hi, s_restore_buf_rsrc0, s_restore_mem_offset)                   //XNACK_MASK_HI
+    read_hwreg_from_mem(s_restore_mode, s_restore_buf_rsrc0, s_restore_mem_offset)              //MODE
+    read_hwreg_from_mem(tba_lo, s_restore_buf_rsrc0, s_restore_mem_offset)                      //TBA_LO
+    read_hwreg_from_mem(tba_hi, s_restore_buf_rsrc0, s_restore_mem_offset)                      //TBA_HI
+
+    s_waitcnt       lgkmcnt(0)                                                                                      //from now on, it is safe to restore STATUS and IB_STS
+
+    s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff      //pc[47:32]        //Do it here in order not to affect STATUS
+
+    //for normal save & restore, the saved PC points to the next inst to execute, no adjustment needs to be made, otherwise:
+    if ((EMU_RUN_HACK) && (!EMU_RUN_HACK_RESTORE_NORMAL))
+        s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 8            //pc[31:0]+8     //two back-to-back s_trap are used (first for save and second for restore)
+        s_addc_u32  s_restore_pc_hi, s_restore_pc_hi, 0x0        //carry bit over
+    end
+    if ((EMU_RUN_HACK) && (EMU_RUN_HACK_RESTORE_NORMAL))
+        s_add_u32 s_restore_pc_lo, s_restore_pc_lo, 4            //pc[31:0]+4     // save is hack through s_trap but restore is normal
+        s_addc_u32  s_restore_pc_hi, s_restore_pc_hi, 0x0        //carry bit over
+    end
+
+    s_mov_b32       m0,         s_restore_m0
+    s_mov_b32       exec_lo,    s_restore_exec_lo
+    s_mov_b32       exec_hi,    s_restore_exec_hi
+
+    s_and_b32       s_restore_m0, SQ_WAVE_TRAPSTS_PRE_SAVECTX_MASK, s_restore_trapsts
+    s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_PRE_SAVECTX_SIZE), s_restore_m0
+    s_and_b32       s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_MASK, s_restore_trapsts
+    s_lshr_b32      s_restore_m0, s_restore_m0, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT
+    s_setreg_b32    hwreg(HW_REG_TRAPSTS, SQ_WAVE_TRAPSTS_POST_SAVECTX_SHIFT, SQ_WAVE_TRAPSTS_POST_SAVECTX_SIZE), s_restore_m0
+    //s_setreg_b32  hwreg(HW_REG_TRAPSTS),  s_restore_trapsts      //don't overwrite SAVECTX bit as it may be set through external SAVECTX during restore
+    s_setreg_b32    hwreg(HW_REG_MODE),     s_restore_mode
+    //reuse s_restore_m0 as a temp register
+    s_and_b32       s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_RCNT_MASK
+    s_lshr_b32      s_restore_m0, s_restore_m0, S_SAVE_PC_HI_RCNT_SHIFT
+    s_lshl_b32      s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_RCNT_SHIFT
+    s_mov_b32       s_restore_tmp, 0x0                                                                              //IB_STS is zero
+    s_or_b32        s_restore_tmp, s_restore_tmp, s_restore_m0
+    s_and_b32       s_restore_m0, s_restore_pc_hi, S_SAVE_PC_HI_FIRST_REPLAY_MASK
+    s_lshr_b32      s_restore_m0, s_restore_m0, S_SAVE_PC_HI_FIRST_REPLAY_SHIFT
+    s_lshl_b32      s_restore_m0, s_restore_m0, SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT
+    s_or_b32        s_restore_tmp, s_restore_tmp, s_restore_m0
+    s_and_b32       s_restore_m0, s_restore_status, SQ_WAVE_STATUS_INST_ATC_MASK
+    s_lshr_b32      s_restore_m0, s_restore_m0, SQ_WAVE_STATUS_INST_ATC_SHIFT
+    s_setreg_b32    hwreg(HW_REG_IB_STS),   s_restore_tmp
+
+    s_and_b64    exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
+    s_and_b64    vcc, vcc, vcc  // Restore STATUS.VCCZ, not writable by s_setreg_b32
+    s_setreg_b32    hwreg(HW_REG_STATUS),   s_restore_status     // SCC is included, which is changed by previous salu
+
+    s_barrier                                                   //barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
+
+if G8SR_DEBUG_TIMESTAMP
+    s_memrealtime s_g8sr_ts_restore_d
+    s_waitcnt lgkmcnt(0)
+end
+
+//  s_rfe_b64 s_restore_pc_lo                                   //Return to the main shader program and resume execution
+    s_rfe_restore_b64  s_restore_pc_lo, s_restore_m0            // s_restore_m0[0] is used to set STATUS.inst_atc
+
+
+/**************************************************************************/
+/*                      the END                                           */
+/**************************************************************************/
+L_END_PGM:
+    s_endpgm
+
+end
+
+
+/**************************************************************************/
+/*                      the helper functions                              */
+/**************************************************************************/
+
+//Only for save hwreg to mem
+function write_hwreg_to_mem(s, s_rsrc, s_mem_offset)
+        s_mov_b32 exec_lo, m0                   //assuming exec_lo is not needed anymore from this point on
+        s_mov_b32 m0, s_mem_offset
+        s_buffer_store_dword s, s_rsrc, m0      glc:1
+        s_add_u32       s_mem_offset, s_mem_offset, 4
+        s_mov_b32   m0, exec_lo
+end
+
+
+// HWREG are saved before SGPRs, so all HWREG could be use.
+function write_16sgpr_to_mem(s, s_rsrc, s_mem_offset)
+
+        s_buffer_store_dwordx4 s[0], s_rsrc, 0  glc:1
+        s_buffer_store_dwordx4 s[4], s_rsrc, 16  glc:1
+        s_buffer_store_dwordx4 s[8], s_rsrc, 32  glc:1
+        s_buffer_store_dwordx4 s[12], s_rsrc, 48 glc:1
+        s_add_u32       s_rsrc[0], s_rsrc[0], 4*16
+        s_addc_u32      s_rsrc[1], s_rsrc[1], 0x0             // +scc
+end
+
+
+function read_hwreg_from_mem(s, s_rsrc, s_mem_offset)
+    s_buffer_load_dword s, s_rsrc, s_mem_offset     glc:1
+    s_add_u32       s_mem_offset, s_mem_offset, 4
+end
+
+function read_16sgpr_from_mem(s, s_rsrc, s_mem_offset)
+    s_buffer_load_dwordx16 s, s_rsrc, s_mem_offset      glc:1
+    s_sub_u32       s_mem_offset, s_mem_offset, 4*16
+end
+
+
+
+function get_lds_size_bytes(s_lds_size_byte)
+    // SQ LDS granularity is 64DW, while PGM_RSRC2.lds_size is in granularity 128DW
+    s_getreg_b32   s_lds_size_byte, hwreg(HW_REG_LDS_ALLOC, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT, SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE)          // lds_size
+    s_lshl_b32     s_lds_size_byte, s_lds_size_byte, 8                      //LDS size in dwords = lds_size * 64 *4Bytes    // granularity 64DW
+end
+
+function get_vgpr_size_bytes(s_vgpr_size_byte)
+    s_getreg_b32   s_vgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_VGPR_SIZE_SIZE)  //vpgr_size
+    s_add_u32      s_vgpr_size_byte, s_vgpr_size_byte, 1
+    s_lshl_b32     s_vgpr_size_byte, s_vgpr_size_byte, (2+8) //Number of VGPRs = (vgpr_size + 1) * 4 * 64 * 4   (non-zero value)   //FIXME for GFX, zero is possible
+end
+
+function get_sgpr_size_bytes(s_sgpr_size_byte)
+    s_getreg_b32   s_sgpr_size_byte, hwreg(HW_REG_GPR_ALLOC,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SHIFT,SQ_WAVE_GPR_ALLOC_SGPR_SIZE_SIZE)  //spgr_size
+    s_add_u32      s_sgpr_size_byte, s_sgpr_size_byte, 1
+    s_lshl_b32     s_sgpr_size_byte, s_sgpr_size_byte, 6 //Number of SGPRs = (sgpr_size + 1) * 16 *4   (non-zero value)
+end
+
+function get_hwreg_size_bytes
+    return 128 //HWREG size 128 bytes
+end
+
+
+#endif
+
+static const uint32_t cwsr_trap_gfx8_hex[] = {
+	0xbf820001, 0xbf820123,
+	0xb8f4f802, 0x89748674,
+	0xb8f5f803, 0x8675ff75,
+	0x00000400, 0xbf850011,
+	0xc00a1e37, 0x00000000,
+	0xbf8c007f, 0x87777978,
+	0xbf840002, 0xb974f802,
+	0xbe801d78, 0xb8f5f803,
+	0x8675ff75, 0x000001ff,
+	0xbf850002, 0x80708470,
+	0x82718071, 0x8671ff71,
+	0x0000ffff, 0xb974f802,
+	0xbe801f70, 0xb8f5f803,
+	0x8675ff75, 0x00000100,
+	0xbf840006, 0xbefa0080,
+	0xb97a0203, 0x8671ff71,
+	0x0000ffff, 0x80f08870,
+	0x82f18071, 0xbefa0080,
+	0xb97a0283, 0xbef60068,
+	0xbef70069, 0xb8fa1c07,
+	0x8e7a9c7a, 0x87717a71,
+	0xb8fa03c7, 0x8e7a9b7a,
+	0x87717a71, 0xb8faf807,
+	0x867aff7a, 0x00007fff,
+	0xb97af807, 0xbef2007e,
+	0xbef3007f, 0xbefe0180,
+	0xbf900004, 0xbf8e0002,
+	0xbf88fffe, 0xbef8007e,
+	0x8679ff7f, 0x0000ffff,
+	0x8779ff79, 0x00040000,
+	0xbefa0080, 0xbefb00ff,
+	0x00807fac, 0x867aff7f,
+	0x08000000, 0x8f7a837a,
+	0x877b7a7b, 0x867aff7f,
+	0x70000000, 0x8f7a817a,
+	0x877b7a7b, 0xbeef007c,
+	0xbeee0080, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8fa1605, 0x807a817a,
+	0x8e7a867a, 0x806e7a6e,
+	0xbefa0084, 0xbefa00ff,
+	0x01000000, 0xbefe007c,
+	0xbefc006e, 0xc0611bfc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611c3c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611c7c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611cbc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611cfc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611d3c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xb8f5f803,
+	0xbefe007c, 0xbefc006e,
+	0xc0611d7c, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0611dbc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xbefe007c, 0xbefc006e,
+	0xc0611dfc, 0x0000007c,
+	0x806e846e, 0xbefc007e,
+	0xb8eff801, 0xbefe007c,
+	0xbefc006e, 0xc0611bfc,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611b3c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0xbefe007c,
+	0xbefc006e, 0xc0611b7c,
+	0x0000007c, 0x806e846e,
+	0xbefc007e, 0x867aff7f,
+	0x04000000, 0xbef30080,
+	0x8773737a, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8f51605, 0x80758175,
+	0x8e758475, 0x8e7a8275,
+	0xbefa00ff, 0x01000000,
+	0xbef60178, 0x80786e78,
+	0x82798079, 0xbefc0080,
+	0xbe802b00, 0xbe822b02,
+	0xbe842b04, 0xbe862b06,
+	0xbe882b08, 0xbe8a2b0a,
+	0xbe8c2b0c, 0xbe8e2b0e,
+	0xc06b003c, 0x00000000,
+	0xc06b013c, 0x00000010,
+	0xc06b023c, 0x00000020,
+	0xc06b033c, 0x00000030,
+	0x8078c078, 0x82798079,
+	0x807c907c, 0xbf0a757c,
+	0xbf85ffeb, 0xbef80176,
+	0xbeee0080, 0xbefe00c1,
+	0xbeff00c1, 0xbefa00ff,
+	0x01000000, 0xe0724000,
+	0x6e1e0000, 0xe0724100,
+	0x6e1e0100, 0xe0724200,
+	0x6e1e0200, 0xe0724300,
+	0x6e1e0300, 0xbefe00c1,
+	0xbeff00c1, 0xb8f54306,
+	0x8675c175, 0xbf84002c,
+	0xbf8a0000, 0x867aff73,
+	0x04000000, 0xbf840028,
+	0x8e758675, 0x8e758275,
+	0xbefa0075, 0xb8ee2a05,
+	0x806e816e, 0x8e6e8a6e,
+	0xb8fa1605, 0x807a817a,
+	0x8e7a867a, 0x806e7a6e,
+	0x806eff6e, 0x00000080,
+	0xbefa00ff, 0x01000000,
+	0xbefc0080, 0xd28c0002,
+	0x000100c1, 0xd28d0003,
+	0x000204c1, 0xd1060002,
+	0x00011103, 0x7e0602ff,
+	0x00000200, 0xbefc00ff,
+	0x00010000, 0xbe80007b,
+	0x867bff7b, 0xff7fffff,
+	0x877bff7b, 0x00058000,
+	0xd8ec0000, 0x00000002,
+	0xbf8c007f, 0xe0765000,
+	0x6e1e0002, 0x32040702,
+	0xd0c9006a, 0x0000eb02,
+	0xbf87fff7, 0xbefb0000,
+	0xbeee00ff, 0x00000400,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8f52a05, 0x80758175,
+	0x8e758275, 0x8e7a8875,
+	0xbefa00ff, 0x01000000,
+	0xbefc0084, 0xbf0a757c,
+	0xbf840015, 0xbf11017c,
+	0x8075ff75, 0x00001000,
+	0x7e000300, 0x7e020301,
+	0x7e040302, 0x7e060303,
+	0xe0724000, 0x6e1e0000,
+	0xe0724100, 0x6e1e0100,
+	0xe0724200, 0x6e1e0200,
+	0xe0724300, 0x6e1e0300,
+	0x807c847c, 0x806eff6e,
+	0x00000400, 0xbf0a757c,
+	0xbf85ffef, 0xbf9c0000,
+	0xbf8200ca, 0xbef8007e,
+	0x8679ff7f, 0x0000ffff,
+	0x8779ff79, 0x00040000,
+	0xbefa0080, 0xbefb00ff,
+	0x00807fac, 0x8676ff7f,
+	0x08000000, 0x8f768376,
+	0x877b767b, 0x8676ff7f,
+	0x70000000, 0x8f768176,
+	0x877b767b, 0x8676ff7f,
+	0x04000000, 0xbf84001e,
+	0xbefe00c1, 0xbeff00c1,
+	0xb8f34306, 0x8673c173,
+	0xbf840019, 0x8e738673,
+	0x8e738273, 0xbefa0073,
+	0xb8f22a05, 0x80728172,
+	0x8e728a72, 0xb8f61605,
+	0x80768176, 0x8e768676,
+	0x80727672, 0x8072ff72,
+	0x00000080, 0xbefa00ff,
+	0x01000000, 0xbefc0080,
+	0xe0510000, 0x721e0000,
+	0xe0510100, 0x721e0000,
+	0x807cff7c, 0x00000200,
+	0x8072ff72, 0x00000200,
+	0xbf0a737c, 0xbf85fff6,
+	0xbef20080, 0xbefe00c1,
+	0xbeff00c1, 0xb8f32a05,
+	0x80738173, 0x8e738273,
+	0x8e7a8873, 0xbefa00ff,
+	0x01000000, 0xbef60072,
+	0x8072ff72, 0x00000400,
+	0xbefc0084, 0xbf11087c,
+	0x8073ff73, 0x00008000,
+	0xe0524000, 0x721e0000,
+	0xe0524100, 0x721e0100,
+	0xe0524200, 0x721e0200,
+	0xe0524300, 0x721e0300,
+	0xbf8c0f70, 0x7e000300,
+	0x7e020301, 0x7e040302,
+	0x7e060303, 0x807c847c,
+	0x8072ff72, 0x00000400,
+	0xbf0a737c, 0xbf85ffee,
+	0xbf9c0000, 0xe0524000,
+	0x761e0000, 0xe0524100,
+	0x761e0100, 0xe0524200,
+	0x761e0200, 0xe0524300,
+	0x761e0300, 0xb8f22a05,
+	0x80728172, 0x8e728a72,
+	0xb8f61605, 0x80768176,
+	0x8e768676, 0x80727672,
+	0x80f2c072, 0xb8f31605,
+	0x80738173, 0x8e738473,
+	0x8e7a8273, 0xbefa00ff,
+	0x01000000, 0xbefc0073,
+	0xc031003c, 0x00000072,
+	0x80f2c072, 0xbf8c007f,
+	0x80fc907c, 0xbe802d00,
+	0xbe822d02, 0xbe842d04,
+	0xbe862d06, 0xbe882d08,
+	0xbe8a2d0a, 0xbe8c2d0c,
+	0xbe8e2d0e, 0xbf06807c,
+	0xbf84fff1, 0xb8f22a05,
+	0x80728172, 0x8e728a72,
+	0xb8f61605, 0x80768176,
+	0x8e768676, 0x80727672,
+	0xbefa0084, 0xbefa00ff,
+	0x01000000, 0xc0211cfc,
+	0x00000072, 0x80728472,
+	0xc0211c3c, 0x00000072,
+	0x80728472, 0xc0211c7c,
+	0x00000072, 0x80728472,
+	0xc0211bbc, 0x00000072,
+	0x80728472, 0xc0211bfc,
+	0x00000072, 0x80728472,
+	0xc0211d3c, 0x00000072,
+	0x80728472, 0xc0211d7c,
+	0x00000072, 0x80728472,
+	0xc0211a3c, 0x00000072,
+	0x80728472, 0xc0211a7c,
+	0x00000072, 0x80728472,
+	0xc0211dfc, 0x00000072,
+	0x80728472, 0xc0211b3c,
+	0x00000072, 0x80728472,
+	0xc0211b7c, 0x00000072,
+	0x80728472, 0xbf8c007f,
+	0x8671ff71, 0x0000ffff,
+	0xbefc0073, 0xbefe006e,
+	0xbeff006f, 0x867375ff,
+	0x000003ff, 0xb9734803,
+	0x867375ff, 0xfffff800,
+	0x8f738b73, 0xb973a2c3,
+	0xb977f801, 0x8673ff71,
+	0xf0000000, 0x8f739c73,
+	0x8e739073, 0xbef60080,
+	0x87767376, 0x8673ff71,
+	0x08000000, 0x8f739b73,
+	0x8e738f73, 0x87767376,
+	0x8673ff74, 0x00800000,
+	0x8f739773, 0xb976f807,
+	0x86fe7e7e, 0x86ea6a6a,
+	0xb974f802, 0xbf8a0000,
+	0x95807370, 0xbf810000,
+};
+
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index 505d39156acd..62c3d9cd6ef1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -117,7 +117,7 @@ static int kfd_open(struct inode *inode, struct file *filep)
 		return -EPERM;
 	}
 
-	process = kfd_create_process(current);
+	process = kfd_create_process(filep);
 	if (IS_ERR(process))
 		return PTR_ERR(process);
 
@@ -206,6 +206,7 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties,
 	q_properties->ctx_save_restore_area_address =
 			args->ctx_save_restore_address;
 	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
+	q_properties->ctl_stack_size = args->ctl_stack_size;
 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
 		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
 		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
@@ -431,6 +432,38 @@ static int kfd_ioctl_set_memory_policy(struct file *filep,
 	return err;
 }
 
+static int kfd_ioctl_set_trap_handler(struct file *filep,
+					struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_set_trap_handler_args *args = data;
+	struct kfd_dev *dev;
+	int err = 0;
+	struct kfd_process_device *pdd;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (dev == NULL)
+		return -EINVAL;
+
+	mutex_lock(&p->mutex);
+
+	pdd = kfd_bind_process_to_device(dev, p);
+	if (IS_ERR(pdd)) {
+		err = -ESRCH;
+		goto out;
+	}
+
+	if (dev->dqm->ops.set_trap_handler(dev->dqm,
+					&pdd->qpd,
+					args->tba_addr,
+					args->tma_addr))
+		err = -EINVAL;
+
+out:
+	mutex_unlock(&p->mutex);
+
+	return err;
+}
+
 static int kfd_ioctl_dbg_register(struct file *filep,
 				struct kfd_process *p, void *data)
 {
@@ -493,7 +526,7 @@ static int kfd_ioctl_dbg_unregister(struct file *filep,
 	long status;
 
 	dev = kfd_device_by_id(args->gpu_id);
-	if (!dev)
+	if (!dev || !dev->dbgmgr)
 		return -EINVAL;
 
 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
@@ -979,7 +1012,10 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 			kfd_ioctl_set_scratch_backing_va, 0),
 
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
-			kfd_ioctl_get_tile_config, 0)
+			kfd_ioctl_get_tile_config, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
+			kfd_ioctl_set_trap_handler, 0),
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
@@ -1088,6 +1124,10 @@ static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
 			KFD_MMAP_EVENTS_MASK) {
 		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK;
 		return kfd_event_mmap(process, vma);
+	} else if ((vma->vm_pgoff & KFD_MMAP_RESERVED_MEM_MASK) ==
+			KFD_MMAP_RESERVED_MEM_MASK) {
+		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_RESERVED_MEM_MASK;
+		return kfd_reserved_mem_mmap(process, vma);
 	}
 
 	return -EFAULT;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
new file mode 100644
index 000000000000..2bc2816767a7
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -0,0 +1,1267 @@
+/*
+ * Copyright 2015-2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/pci.h>
+#include <linux/acpi.h>
+#include <linux/amd-iommu.h>
+#include "kfd_crat.h"
+#include "kfd_priv.h"
+#include "kfd_topology.h"
+
+/* GPU Processor ID base for dGPUs for which VCRAT needs to be created.
+ * GPU processor ID are expressed with Bit[31]=1.
+ * The base is set to 0x8000_0000 + 0x1000 to avoid collision with GPU IDs
+ * used in the CRAT.
+ */
+static uint32_t gpu_processor_id_low = 0x80001000;
+
+/* Return the next available gpu_processor_id and increment it for next GPU
+ *	@total_cu_count - Total CUs present in the GPU including ones
+ *			  masked off
+ */
+static inline unsigned int get_and_inc_gpu_processor_id(
+				unsigned int total_cu_count)
+{
+	int current_id = gpu_processor_id_low;
+
+	gpu_processor_id_low += total_cu_count;
+	return current_id;
+}
+
+/* Static table to describe GPU Cache information */
+struct kfd_gpu_cache_info {
+	uint32_t	cache_size;
+	uint32_t	cache_level;
+	uint32_t	flags;
+	/* Indicates how many Compute Units share this cache
+	 * Value = 1 indicates the cache is not shared
+	 */
+	uint32_t	num_cu_shared;
+};
+
+static struct kfd_gpu_cache_info kaveri_cache_info[] = {
+	{
+		/* TCP L1 Cache per CU */
+		.cache_size = 16,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 1,
+
+	},
+	{
+		/* Scalar L1 Instruction Cache (in SQC module) per bank */
+		.cache_size = 16,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_INST_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 2,
+	},
+	{
+		/* Scalar L1 Data Cache (in SQC module) per bank */
+		.cache_size = 8,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 2,
+	},
+
+	/* TODO: Add L2 Cache information */
+};
+
+
+static struct kfd_gpu_cache_info carrizo_cache_info[] = {
+	{
+		/* TCP L1 Cache per CU */
+		.cache_size = 16,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 1,
+	},
+	{
+		/* Scalar L1 Instruction Cache (in SQC module) per bank */
+		.cache_size = 8,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_INST_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 4,
+	},
+	{
+		/* Scalar L1 Data Cache (in SQC module) per bank. */
+		.cache_size = 4,
+		.cache_level = 1,
+		.flags = (CRAT_CACHE_FLAGS_ENABLED |
+				CRAT_CACHE_FLAGS_DATA_CACHE |
+				CRAT_CACHE_FLAGS_SIMD_CACHE),
+		.num_cu_shared = 4,
+	},
+
+	/* TODO: Add L2 Cache information */
+};
+
+/* NOTE: In future if more information is added to struct kfd_gpu_cache_info
+ * the following ASICs may need a separate table.
+ */
+#define hawaii_cache_info kaveri_cache_info
+#define tonga_cache_info carrizo_cache_info
+#define fiji_cache_info  carrizo_cache_info
+#define polaris10_cache_info carrizo_cache_info
+#define polaris11_cache_info carrizo_cache_info
+
+static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
+		struct crat_subtype_computeunit *cu)
+{
+	dev->node_props.cpu_cores_count = cu->num_cpu_cores;
+	dev->node_props.cpu_core_id_base = cu->processor_id_low;
+	if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
+		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
+
+	pr_debug("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
+			cu->processor_id_low);
+}
+
+static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
+		struct crat_subtype_computeunit *cu)
+{
+	dev->node_props.simd_id_base = cu->processor_id_low;
+	dev->node_props.simd_count = cu->num_simd_cores;
+	dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
+	dev->node_props.max_waves_per_simd = cu->max_waves_simd;
+	dev->node_props.wave_front_size = cu->wave_front_size;
+	dev->node_props.array_count = cu->array_count;
+	dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
+	dev->node_props.simd_per_cu = cu->num_simd_per_cu;
+	dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
+	if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
+		dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
+	pr_debug("CU GPU: id_base=%d\n", cu->processor_id_low);
+}
+
+/* kfd_parse_subtype_cu - parse compute unit subtypes and attach it to correct
+ * topology device present in the device_list
+ */
+static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu,
+				struct list_head *device_list)
+{
+	struct kfd_topology_device *dev;
+
+	pr_debug("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
+			cu->proximity_domain, cu->hsa_capability);
+	list_for_each_entry(dev, device_list, list) {
+		if (cu->proximity_domain == dev->proximity_domain) {
+			if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
+				kfd_populated_cu_info_cpu(dev, cu);
+
+			if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
+				kfd_populated_cu_info_gpu(dev, cu);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
+ * topology device present in the device_list
+ */
+static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem,
+				struct list_head *device_list)
+{
+	struct kfd_mem_properties *props;
+	struct kfd_topology_device *dev;
+
+	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
+			mem->proximity_domain);
+	list_for_each_entry(dev, device_list, list) {
+		if (mem->proximity_domain == dev->proximity_domain) {
+			props = kfd_alloc_struct(props);
+			if (!props)
+				return -ENOMEM;
+
+			/* We're on GPU node */
+			if (dev->node_props.cpu_cores_count == 0) {
+				/* APU */
+				if (mem->visibility_type == 0)
+					props->heap_type =
+						HSA_MEM_HEAP_TYPE_FB_PRIVATE;
+				/* dGPU */
+				else
+					props->heap_type = mem->visibility_type;
+			} else
+				props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
+
+			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
+				props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
+			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
+				props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
+
+			props->size_in_bytes =
+				((uint64_t)mem->length_high << 32) +
+							mem->length_low;
+			props->width = mem->width;
+
+			dev->node_props.mem_banks_count++;
+			list_add_tail(&props->list, &dev->mem_props);
+
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype_cache - parse cache subtypes and attach it to correct
+ * topology device present in the device_list
+ */
+static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache,
+			struct list_head *device_list)
+{
+	struct kfd_cache_properties *props;
+	struct kfd_topology_device *dev;
+	uint32_t id;
+	uint32_t total_num_of_cu;
+
+	id = cache->processor_id_low;
+
+	pr_debug("Found cache entry in CRAT table with processor_id=%d\n", id);
+	list_for_each_entry(dev, device_list, list) {
+		total_num_of_cu = (dev->node_props.array_count *
+					dev->node_props.cu_per_simd_array);
+
+		/* Cache infomration in CRAT doesn't have proximity_domain
+		 * information as it is associated with a CPU core or GPU
+		 * Compute Unit. So map the cache using CPU core Id or SIMD
+		 * (GPU) ID.
+		 * TODO: This works because currently we can safely assume that
+		 *  Compute Units are parsed before caches are parsed. In
+		 *  future, remove this dependency
+		 */
+		if ((id >= dev->node_props.cpu_core_id_base &&
+			id <= dev->node_props.cpu_core_id_base +
+				dev->node_props.cpu_cores_count) ||
+			(id >= dev->node_props.simd_id_base &&
+			id < dev->node_props.simd_id_base +
+				total_num_of_cu)) {
+			props = kfd_alloc_struct(props);
+			if (!props)
+				return -ENOMEM;
+
+			props->processor_id_low = id;
+			props->cache_level = cache->cache_level;
+			props->cache_size = cache->cache_size;
+			props->cacheline_size = cache->cache_line_size;
+			props->cachelines_per_tag = cache->lines_per_tag;
+			props->cache_assoc = cache->associativity;
+			props->cache_latency = cache->cache_latency;
+			memcpy(props->sibling_map, cache->sibling_map,
+					sizeof(props->sibling_map));
+
+			if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_DATA;
+			if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
+			if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_CPU;
+			if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
+				props->cache_type |= HSA_CACHE_TYPE_HSACU;
+
+			dev->cache_count++;
+			dev->node_props.caches_count++;
+			list_add_tail(&props->list, &dev->cache_props);
+
+			break;
+		}
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype_iolink - parse iolink subtypes and attach it to correct
+ * topology device present in the device_list
+ */
+static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
+					struct list_head *device_list)
+{
+	struct kfd_iolink_properties *props = NULL, *props2;
+	struct kfd_topology_device *dev, *cpu_dev;
+	uint32_t id_from;
+	uint32_t id_to;
+
+	id_from = iolink->proximity_domain_from;
+	id_to = iolink->proximity_domain_to;
+
+	pr_debug("Found IO link entry in CRAT table with id_from=%d\n",
+			id_from);
+	list_for_each_entry(dev, device_list, list) {
+		if (id_from == dev->proximity_domain) {
+			props = kfd_alloc_struct(props);
+			if (!props)
+				return -ENOMEM;
+
+			props->node_from = id_from;
+			props->node_to = id_to;
+			props->ver_maj = iolink->version_major;
+			props->ver_min = iolink->version_minor;
+			props->iolink_type = iolink->io_interface_type;
+
+			if (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS)
+				props->weight = 20;
+			else
+				props->weight = node_distance(id_from, id_to);
+
+			props->min_latency = iolink->minimum_latency;
+			props->max_latency = iolink->maximum_latency;
+			props->min_bandwidth = iolink->minimum_bandwidth_mbs;
+			props->max_bandwidth = iolink->maximum_bandwidth_mbs;
+			props->rec_transfer_size =
+					iolink->recommended_transfer_size;
+
+			dev->io_link_count++;
+			dev->node_props.io_links_count++;
+			list_add_tail(&props->list, &dev->io_link_props);
+			break;
+		}
+	}
+
+	/* CPU topology is created before GPUs are detected, so CPU->GPU
+	 * links are not built at that time. If a PCIe type is discovered, it
+	 * means a GPU is detected and we are adding GPU->CPU to the topology.
+	 * At this time, also add the corresponded CPU->GPU link.
+	 */
+	if (props && props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS) {
+		cpu_dev = kfd_topology_device_by_proximity_domain(id_to);
+		if (!cpu_dev)
+			return -ENODEV;
+		/* same everything but the other direction */
+		props2 = kmemdup(props, sizeof(*props2), GFP_KERNEL);
+		props2->node_from = id_to;
+		props2->node_to = id_from;
+		props2->kobj = NULL;
+		cpu_dev->io_link_count++;
+		cpu_dev->node_props.io_links_count++;
+		list_add_tail(&props2->list, &cpu_dev->io_link_props);
+	}
+
+	return 0;
+}
+
+/* kfd_parse_subtype - parse subtypes and attach it to correct topology device
+ * present in the device_list
+ *	@sub_type_hdr - subtype section of crat_image
+ *	@device_list - list of topology devices present in this crat_image
+ */
+static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr,
+				struct list_head *device_list)
+{
+	struct crat_subtype_computeunit *cu;
+	struct crat_subtype_memory *mem;
+	struct crat_subtype_cache *cache;
+	struct crat_subtype_iolink *iolink;
+	int ret = 0;
+
+	switch (sub_type_hdr->type) {
+	case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
+		cu = (struct crat_subtype_computeunit *)sub_type_hdr;
+		ret = kfd_parse_subtype_cu(cu, device_list);
+		break;
+	case CRAT_SUBTYPE_MEMORY_AFFINITY:
+		mem = (struct crat_subtype_memory *)sub_type_hdr;
+		ret = kfd_parse_subtype_mem(mem, device_list);
+		break;
+	case CRAT_SUBTYPE_CACHE_AFFINITY:
+		cache = (struct crat_subtype_cache *)sub_type_hdr;
+		ret = kfd_parse_subtype_cache(cache, device_list);
+		break;
+	case CRAT_SUBTYPE_TLB_AFFINITY:
+		/*
+		 * For now, nothing to do here
+		 */
+		pr_debug("Found TLB entry in CRAT table (not processing)\n");
+		break;
+	case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
+		/*
+		 * For now, nothing to do here
+		 */
+		pr_debug("Found CCOMPUTE entry in CRAT table (not processing)\n");
+		break;
+	case CRAT_SUBTYPE_IOLINK_AFFINITY:
+		iolink = (struct crat_subtype_iolink *)sub_type_hdr;
+		ret = kfd_parse_subtype_iolink(iolink, device_list);
+		break;
+	default:
+		pr_warn("Unknown subtype %d in CRAT\n",
+				sub_type_hdr->type);
+	}
+
+	return ret;
+}
+
+/* kfd_parse_crat_table - parse CRAT table. For each node present in CRAT
+ * create a kfd_topology_device and add in to device_list. Also parse
+ * CRAT subtypes and attach it to appropriate kfd_topology_device
+ *	@crat_image - input image containing CRAT
+ *	@device_list - [OUT] list of kfd_topology_device generated after
+ *		       parsing crat_image
+ *	@proximity_domain - Proximity domain of the first device in the table
+ *
+ *	Return - 0 if successful else -ve value
+ */
+int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
+			 uint32_t proximity_domain)
+{
+	struct kfd_topology_device *top_dev = NULL;
+	struct crat_subtype_generic *sub_type_hdr;
+	uint16_t node_id;
+	int ret = 0;
+	struct crat_header *crat_table = (struct crat_header *)crat_image;
+	uint16_t num_nodes;
+	uint32_t image_len;
+
+	if (!crat_image)
+		return -EINVAL;
+
+	if (!list_empty(device_list)) {
+		pr_warn("Error device list should be empty\n");
+		return -EINVAL;
+	}
+
+	num_nodes = crat_table->num_domains;
+	image_len = crat_table->length;
+
+	pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
+
+	for (node_id = 0; node_id < num_nodes; node_id++) {
+		top_dev = kfd_create_topology_device(device_list);
+		if (!top_dev)
+			break;
+		top_dev->proximity_domain = proximity_domain++;
+	}
+
+	if (!top_dev) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	memcpy(top_dev->oem_id, crat_table->oem_id, CRAT_OEMID_LENGTH);
+	memcpy(top_dev->oem_table_id, crat_table->oem_table_id,
+			CRAT_OEMTABLEID_LENGTH);
+	top_dev->oem_revision = crat_table->oem_revision;
+
+	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
+	while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
+			((char *)crat_image) + image_len) {
+		if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
+			ret = kfd_parse_subtype(sub_type_hdr, device_list);
+			if (ret)
+				break;
+		}
+
+		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+				sub_type_hdr->length);
+	}
+
+err:
+	if (ret)
+		kfd_release_topology_device_list(device_list);
+
+	return ret;
+}
+
+/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
+static int fill_in_pcache(struct crat_subtype_cache *pcache,
+				struct kfd_gpu_cache_info *pcache_info,
+				struct kfd_cu_info *cu_info,
+				int mem_available,
+				int cu_bitmask,
+				int cache_type, unsigned int cu_processor_id,
+				int cu_block)
+{
+	unsigned int cu_sibling_map_mask;
+	int first_active_cu;
+
+	/* First check if enough memory is available */
+	if (sizeof(struct crat_subtype_cache) > mem_available)
+		return -ENOMEM;
+
+	cu_sibling_map_mask = cu_bitmask;
+	cu_sibling_map_mask >>= cu_block;
+	cu_sibling_map_mask &=
+		((1 << pcache_info[cache_type].num_cu_shared) - 1);
+	first_active_cu = ffs(cu_sibling_map_mask);
+
+	/* CU could be inactive. In case of shared cache find the first active
+	 * CU. and incase of non-shared cache check if the CU is inactive. If
+	 * inactive active skip it
+	 */
+	if (first_active_cu) {
+		memset(pcache, 0, sizeof(struct crat_subtype_cache));
+		pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
+		pcache->length = sizeof(struct crat_subtype_cache);
+		pcache->flags = pcache_info[cache_type].flags;
+		pcache->processor_id_low = cu_processor_id
+					 + (first_active_cu - 1);
+		pcache->cache_level = pcache_info[cache_type].cache_level;
+		pcache->cache_size = pcache_info[cache_type].cache_size;
+
+		/* Sibling map is w.r.t processor_id_low, so shift out
+		 * inactive CU
+		 */
+		cu_sibling_map_mask =
+			cu_sibling_map_mask >> (first_active_cu - 1);
+
+		pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF);
+		pcache->sibling_map[1] =
+				(uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
+		pcache->sibling_map[2] =
+				(uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
+		pcache->sibling_map[3] =
+				(uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
+		return 0;
+	}
+	return 1;
+}
+
+/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
+ * tables
+ *
+ *	@kdev - [IN] GPU device
+ *	@gpu_processor_id - [IN] GPU processor ID to which these caches
+ *			    associate
+ *	@available_size - [IN] Amount of memory available in pcache
+ *	@cu_info - [IN] Compute Unit info obtained from KGD
+ *	@pcache - [OUT] memory into which cache data is to be filled in.
+ *	@size_filled - [OUT] amount of data used up in pcache.
+ *	@num_of_entries - [OUT] number of caches added
+ */
+static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
+			int gpu_processor_id,
+			int available_size,
+			struct kfd_cu_info *cu_info,
+			struct crat_subtype_cache *pcache,
+			int *size_filled,
+			int *num_of_entries)
+{
+	struct kfd_gpu_cache_info *pcache_info;
+	int num_of_cache_types = 0;
+	int i, j, k;
+	int ct = 0;
+	int mem_available = available_size;
+	unsigned int cu_processor_id;
+	int ret;
+
+	switch (kdev->device_info->asic_family) {
+	case CHIP_KAVERI:
+		pcache_info = kaveri_cache_info;
+		num_of_cache_types = ARRAY_SIZE(kaveri_cache_info);
+		break;
+	case CHIP_HAWAII:
+		pcache_info = hawaii_cache_info;
+		num_of_cache_types = ARRAY_SIZE(hawaii_cache_info);
+		break;
+	case CHIP_CARRIZO:
+		pcache_info = carrizo_cache_info;
+		num_of_cache_types = ARRAY_SIZE(carrizo_cache_info);
+		break;
+	case CHIP_TONGA:
+		pcache_info = tonga_cache_info;
+		num_of_cache_types = ARRAY_SIZE(tonga_cache_info);
+		break;
+	case CHIP_FIJI:
+		pcache_info = fiji_cache_info;
+		num_of_cache_types = ARRAY_SIZE(fiji_cache_info);
+		break;
+	case CHIP_POLARIS10:
+		pcache_info = polaris10_cache_info;
+		num_of_cache_types = ARRAY_SIZE(polaris10_cache_info);
+		break;
+	case CHIP_POLARIS11:
+		pcache_info = polaris11_cache_info;
+		num_of_cache_types = ARRAY_SIZE(polaris11_cache_info);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	*size_filled = 0;
+	*num_of_entries = 0;
+
+	/* For each type of cache listed in the kfd_gpu_cache_info table,
+	 * go through all available Compute Units.
+	 * The [i,j,k] loop will
+	 *		if kfd_gpu_cache_info.num_cu_shared = 1
+	 *			will parse through all available CU
+	 *		If (kfd_gpu_cache_info.num_cu_shared != 1)
+	 *			then it will consider only one CU from
+	 *			the shared unit
+	 */
+
+	for (ct = 0; ct < num_of_cache_types; ct++) {
+		cu_processor_id = gpu_processor_id;
+		for (i = 0; i < cu_info->num_shader_engines; i++) {
+			for (j = 0; j < cu_info->num_shader_arrays_per_engine;
+				j++) {
+				for (k = 0; k < cu_info->num_cu_per_sh;
+					k += pcache_info[ct].num_cu_shared) {
+
+					ret = fill_in_pcache(pcache,
+						pcache_info,
+						cu_info,
+						mem_available,
+						cu_info->cu_bitmap[i][j],
+						ct,
+						cu_processor_id,
+						k);
+
+					if (ret < 0)
+						break;
+
+					if (!ret) {
+						pcache++;
+						(*num_of_entries)++;
+						mem_available -=
+							sizeof(*pcache);
+						(*size_filled) +=
+							sizeof(*pcache);
+					}
+
+					/* Move to next CU block */
+					cu_processor_id +=
+						pcache_info[ct].num_cu_shared;
+				}
+			}
+		}
+	}
+
+	pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
+
+	return 0;
+}
+
+/*
+ * kfd_create_crat_image_acpi - Allocates memory for CRAT image and
+ * copies CRAT from ACPI (if available).
+ * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
+ *
+ *	@crat_image: CRAT read from ACPI. If no CRAT in ACPI then
+ *		     crat_image will be NULL
+ *	@size: [OUT] size of crat_image
+ *
+ *	Return 0 if successful else return error code
+ */
+int kfd_create_crat_image_acpi(void **crat_image, size_t *size)
+{
+	struct acpi_table_header *crat_table;
+	acpi_status status;
+	void *pcrat_image;
+
+	if (!crat_image)
+		return -EINVAL;
+
+	*crat_image = NULL;
+
+	/* Fetch the CRAT table from ACPI */
+	status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
+	if (status == AE_NOT_FOUND) {
+		pr_warn("CRAT table not found\n");
+		return -ENODATA;
+	} else if (ACPI_FAILURE(status)) {
+		const char *err = acpi_format_exception(status);
+
+		pr_err("CRAT table error: %s\n", err);
+		return -EINVAL;
+	}
+
+	if (ignore_crat) {
+		pr_info("CRAT table disabled by module option\n");
+		return -ENODATA;
+	}
+
+	pcrat_image = kmalloc(crat_table->length, GFP_KERNEL);
+	if (!pcrat_image)
+		return -ENOMEM;
+
+	memcpy(pcrat_image, crat_table, crat_table->length);
+
+	*crat_image = pcrat_image;
+	*size = crat_table->length;
+
+	return 0;
+}
+
+/* Memory required to create Virtual CRAT.
+ * Since there is no easy way to predict the amount of memory required, the
+ * following amount are allocated for CPU and GPU Virtual CRAT. This is
+ * expected to cover all known conditions. But to be safe additional check
+ * is put in the code to ensure we don't overwrite.
+ */
+#define VCRAT_SIZE_FOR_CPU	(2 * PAGE_SIZE)
+#define VCRAT_SIZE_FOR_GPU	(3 * PAGE_SIZE)
+
+/* kfd_fill_cu_for_cpu - Fill in Compute info for the given CPU NUMA node
+ *
+ *	@numa_node_id: CPU NUMA node id
+ *	@avail_size: Available size in the memory
+ *	@sub_type_hdr: Memory into which compute info will be filled in
+ *
+ *	Return 0 if successful else return -ve value
+ */
+static int kfd_fill_cu_for_cpu(int numa_node_id, int *avail_size,
+				int proximity_domain,
+				struct crat_subtype_computeunit *sub_type_hdr)
+{
+	const struct cpumask *cpumask;
+
+	*avail_size -= sizeof(struct crat_subtype_computeunit);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
+
+	/* Fill in subtype header data */
+	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
+	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	cpumask = cpumask_of_node(numa_node_id);
+
+	/* Fill in CU data */
+	sub_type_hdr->flags |= CRAT_CU_FLAGS_CPU_PRESENT;
+	sub_type_hdr->proximity_domain = proximity_domain;
+	sub_type_hdr->processor_id_low = kfd_numa_node_to_apic_id(numa_node_id);
+	if (sub_type_hdr->processor_id_low == -1)
+		return -EINVAL;
+
+	sub_type_hdr->num_cpu_cores = cpumask_weight(cpumask);
+
+	return 0;
+}
+
+/* kfd_fill_mem_info_for_cpu - Fill in Memory info for the given CPU NUMA node
+ *
+ *	@numa_node_id: CPU NUMA node id
+ *	@avail_size: Available size in the memory
+ *	@sub_type_hdr: Memory into which compute info will be filled in
+ *
+ *	Return 0 if successful else return -ve value
+ */
+static int kfd_fill_mem_info_for_cpu(int numa_node_id, int *avail_size,
+			int proximity_domain,
+			struct crat_subtype_memory *sub_type_hdr)
+{
+	uint64_t mem_in_bytes = 0;
+	pg_data_t *pgdat;
+	int zone_type;
+
+	*avail_size -= sizeof(struct crat_subtype_memory);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
+
+	/* Fill in subtype header data */
+	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
+	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	/* Fill in Memory Subunit data */
+
+	/* Unlike si_meminfo, si_meminfo_node is not exported. So
+	 * the following lines are duplicated from si_meminfo_node
+	 * function
+	 */
+	pgdat = NODE_DATA(numa_node_id);
+	for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
+		mem_in_bytes += pgdat->node_zones[zone_type].managed_pages;
+	mem_in_bytes <<= PAGE_SHIFT;
+
+	sub_type_hdr->length_low = lower_32_bits(mem_in_bytes);
+	sub_type_hdr->length_high = upper_32_bits(mem_in_bytes);
+	sub_type_hdr->proximity_domain = proximity_domain;
+
+	return 0;
+}
+
+static int kfd_fill_iolink_info_for_cpu(int numa_node_id, int *avail_size,
+				uint32_t *num_entries,
+				struct crat_subtype_iolink *sub_type_hdr)
+{
+	int nid;
+	struct cpuinfo_x86 *c = &cpu_data(0);
+	uint8_t link_type;
+
+	if (c->x86_vendor == X86_VENDOR_AMD)
+		link_type = CRAT_IOLINK_TYPE_HYPERTRANSPORT;
+	else
+		link_type = CRAT_IOLINK_TYPE_QPI_1_1;
+
+	*num_entries = 0;
+
+	/* Create IO links from this node to other CPU nodes */
+	for_each_online_node(nid) {
+		if (nid == numa_node_id) /* node itself */
+			continue;
+
+		*avail_size -= sizeof(struct crat_subtype_iolink);
+		if (*avail_size < 0)
+			return -ENOMEM;
+
+		memset(sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
+
+		/* Fill in subtype header data */
+		sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
+		sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
+		sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+
+		/* Fill in IO link data */
+		sub_type_hdr->proximity_domain_from = numa_node_id;
+		sub_type_hdr->proximity_domain_to = nid;
+		sub_type_hdr->io_interface_type = link_type;
+
+		(*num_entries)++;
+		sub_type_hdr++;
+	}
+
+	return 0;
+}
+
+/* kfd_create_vcrat_image_cpu - Create Virtual CRAT for CPU
+ *
+ *	@pcrat_image: Fill in VCRAT for CPU
+ *	@size:	[IN] allocated size of crat_image.
+ *		[OUT] actual size of data filled in crat_image
+ */
+static int kfd_create_vcrat_image_cpu(void *pcrat_image, size_t *size)
+{
+	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
+	struct acpi_table_header *acpi_table;
+	acpi_status status;
+	struct crat_subtype_generic *sub_type_hdr;
+	int avail_size = *size;
+	int numa_node_id;
+	uint32_t entries = 0;
+	int ret = 0;
+
+	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_CPU)
+		return -EINVAL;
+
+	/* Fill in CRAT Header.
+	 * Modify length and total_entries as subunits are added.
+	 */
+	avail_size -= sizeof(struct crat_header);
+	if (avail_size < 0)
+		return -ENOMEM;
+
+	memset(crat_table, 0, sizeof(struct crat_header));
+	memcpy(&crat_table->signature, CRAT_SIGNATURE,
+			sizeof(crat_table->signature));
+	crat_table->length = sizeof(struct crat_header);
+
+	status = acpi_get_table("DSDT", 0, &acpi_table);
+	if (status == AE_NOT_FOUND)
+		pr_warn("DSDT table not found for OEM information\n");
+	else {
+		crat_table->oem_revision = acpi_table->revision;
+		memcpy(crat_table->oem_id, acpi_table->oem_id,
+				CRAT_OEMID_LENGTH);
+		memcpy(crat_table->oem_table_id, acpi_table->oem_table_id,
+				CRAT_OEMTABLEID_LENGTH);
+	}
+	crat_table->total_entries = 0;
+	crat_table->num_domains = 0;
+
+	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
+
+	for_each_online_node(numa_node_id) {
+		if (kfd_numa_node_to_apic_id(numa_node_id) == -1)
+			continue;
+
+		/* Fill in Subtype: Compute Unit */
+		ret = kfd_fill_cu_for_cpu(numa_node_id, &avail_size,
+			crat_table->num_domains,
+			(struct crat_subtype_computeunit *)sub_type_hdr);
+		if (ret < 0)
+			return ret;
+		crat_table->length += sub_type_hdr->length;
+		crat_table->total_entries++;
+
+		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+			sub_type_hdr->length);
+
+		/* Fill in Subtype: Memory */
+		ret = kfd_fill_mem_info_for_cpu(numa_node_id, &avail_size,
+			crat_table->num_domains,
+			(struct crat_subtype_memory *)sub_type_hdr);
+		if (ret < 0)
+			return ret;
+		crat_table->length += sub_type_hdr->length;
+		crat_table->total_entries++;
+
+		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+			sub_type_hdr->length);
+
+		/* Fill in Subtype: IO Link */
+		ret = kfd_fill_iolink_info_for_cpu(numa_node_id, &avail_size,
+				&entries,
+				(struct crat_subtype_iolink *)sub_type_hdr);
+		if (ret < 0)
+			return ret;
+		crat_table->length += (sub_type_hdr->length * entries);
+		crat_table->total_entries += entries;
+
+		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+				sub_type_hdr->length * entries);
+
+		crat_table->num_domains++;
+	}
+
+	/* TODO: Add cache Subtype for CPU.
+	 * Currently, CPU cache information is available in function
+	 * detect_cache_attributes(cpu) defined in the file
+	 * ./arch/x86/kernel/cpu/intel_cacheinfo.c. This function is not
+	 * exported and to get the same information the code needs to be
+	 * duplicated.
+	 */
+
+	*size = crat_table->length;
+	pr_info("Virtual CRAT table created for CPU\n");
+
+	return 0;
+}
+
+static int kfd_fill_gpu_memory_affinity(int *avail_size,
+		struct kfd_dev *kdev, uint8_t type, uint64_t size,
+		struct crat_subtype_memory *sub_type_hdr,
+		uint32_t proximity_domain,
+		const struct kfd_local_mem_info *local_mem_info)
+{
+	*avail_size -= sizeof(struct crat_subtype_memory);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_memory));
+	sub_type_hdr->type = CRAT_SUBTYPE_MEMORY_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_memory);
+	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	sub_type_hdr->proximity_domain = proximity_domain;
+
+	pr_debug("Fill gpu memory affinity - type 0x%x size 0x%llx\n",
+			type, size);
+
+	sub_type_hdr->length_low = lower_32_bits(size);
+	sub_type_hdr->length_high = upper_32_bits(size);
+
+	sub_type_hdr->width = local_mem_info->vram_width;
+	sub_type_hdr->visibility_type = type;
+
+	return 0;
+}
+
+/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
+ * to its NUMA node
+ *	@avail_size: Available size in the memory
+ *	@kdev - [IN] GPU device
+ *	@sub_type_hdr: Memory into which io link info will be filled in
+ *	@proximity_domain - proximity domain of the GPU node
+ *
+ *	Return 0 if successful else return -ve value
+ */
+static int kfd_fill_gpu_direct_io_link(int *avail_size,
+			struct kfd_dev *kdev,
+			struct crat_subtype_iolink *sub_type_hdr,
+			uint32_t proximity_domain)
+{
+	*avail_size -= sizeof(struct crat_subtype_iolink);
+	if (*avail_size < 0)
+		return -ENOMEM;
+
+	memset((void *)sub_type_hdr, 0, sizeof(struct crat_subtype_iolink));
+
+	/* Fill in subtype header data */
+	sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
+	sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	/* Fill in IOLINK subtype.
+	 * TODO: Fill-in other fields of iolink subtype
+	 */
+	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
+	sub_type_hdr->proximity_domain_from = proximity_domain;
+#ifdef CONFIG_NUMA
+	if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
+		sub_type_hdr->proximity_domain_to = 0;
+	else
+		sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node;
+#else
+	sub_type_hdr->proximity_domain_to = 0;
+#endif
+	return 0;
+}
+
+/* kfd_create_vcrat_image_gpu - Create Virtual CRAT for CPU
+ *
+ *	@pcrat_image: Fill in VCRAT for GPU
+ *	@size:	[IN] allocated size of crat_image.
+ *		[OUT] actual size of data filled in crat_image
+ */
+static int kfd_create_vcrat_image_gpu(void *pcrat_image,
+				      size_t *size, struct kfd_dev *kdev,
+				      uint32_t proximity_domain)
+{
+	struct crat_header *crat_table = (struct crat_header *)pcrat_image;
+	struct crat_subtype_generic *sub_type_hdr;
+	struct crat_subtype_computeunit *cu;
+	struct kfd_cu_info cu_info;
+	struct amd_iommu_device_info iommu_info;
+	int avail_size = *size;
+	uint32_t total_num_of_cu;
+	int num_of_cache_entries = 0;
+	int cache_mem_filled = 0;
+	int ret = 0;
+	const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
+					 AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
+					 AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
+	struct kfd_local_mem_info local_mem_info;
+
+	if (!pcrat_image || avail_size < VCRAT_SIZE_FOR_GPU)
+		return -EINVAL;
+
+	/* Fill the CRAT Header.
+	 * Modify length and total_entries as subunits are added.
+	 */
+	avail_size -= sizeof(struct crat_header);
+	if (avail_size < 0)
+		return -ENOMEM;
+
+	memset(crat_table, 0, sizeof(struct crat_header));
+
+	memcpy(&crat_table->signature, CRAT_SIGNATURE,
+			sizeof(crat_table->signature));
+	/* Change length as we add more subtypes*/
+	crat_table->length = sizeof(struct crat_header);
+	crat_table->num_domains = 1;
+	crat_table->total_entries = 0;
+
+	/* Fill in Subtype: Compute Unit
+	 * First fill in the sub type header and then sub type data
+	 */
+	avail_size -= sizeof(struct crat_subtype_computeunit);
+	if (avail_size < 0)
+		return -ENOMEM;
+
+	sub_type_hdr = (struct crat_subtype_generic *)(crat_table + 1);
+	memset(sub_type_hdr, 0, sizeof(struct crat_subtype_computeunit));
+
+	sub_type_hdr->type = CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY;
+	sub_type_hdr->length = sizeof(struct crat_subtype_computeunit);
+	sub_type_hdr->flags = CRAT_SUBTYPE_FLAGS_ENABLED;
+
+	/* Fill CU subtype data */
+	cu = (struct crat_subtype_computeunit *)sub_type_hdr;
+	cu->flags |= CRAT_CU_FLAGS_GPU_PRESENT;
+	cu->proximity_domain = proximity_domain;
+
+	kdev->kfd2kgd->get_cu_info(kdev->kgd, &cu_info);
+	cu->num_simd_per_cu = cu_info.simd_per_cu;
+	cu->num_simd_cores = cu_info.simd_per_cu * cu_info.cu_active_number;
+	cu->max_waves_simd = cu_info.max_waves_per_simd;
+
+	cu->wave_front_size = cu_info.wave_front_size;
+	cu->array_count = cu_info.num_shader_arrays_per_engine *
+		cu_info.num_shader_engines;
+	total_num_of_cu = (cu->array_count * cu_info.num_cu_per_sh);
+	cu->processor_id_low = get_and_inc_gpu_processor_id(total_num_of_cu);
+	cu->num_cu_per_array = cu_info.num_cu_per_sh;
+	cu->max_slots_scatch_cu = cu_info.max_scratch_slots_per_cu;
+	cu->num_banks = cu_info.num_shader_engines;
+	cu->lds_size_in_kb = cu_info.lds_size;
+
+	cu->hsa_capability = 0;
+
+	/* Check if this node supports IOMMU. During parsing this flag will
+	 * translate to HSA_CAP_ATS_PRESENT
+	 */
+	iommu_info.flags = 0;
+	if (amd_iommu_device_info(kdev->pdev, &iommu_info) == 0) {
+		if ((iommu_info.flags & required_iommu_flags) ==
+				required_iommu_flags)
+			cu->hsa_capability |= CRAT_CU_FLAGS_IOMMU_PRESENT;
+	}
+
+	crat_table->length += sub_type_hdr->length;
+	crat_table->total_entries++;
+
+	/* Fill in Subtype: Memory. Only on systems with large BAR (no
+	 * private FB), report memory as public. On other systems
+	 * report the total FB size (public+private) as a single
+	 * private heap.
+	 */
+	kdev->kfd2kgd->get_local_mem_info(kdev->kgd, &local_mem_info);
+	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+			sub_type_hdr->length);
+
+	if (local_mem_info.local_mem_size_private == 0)
+		ret = kfd_fill_gpu_memory_affinity(&avail_size,
+				kdev, HSA_MEM_HEAP_TYPE_FB_PUBLIC,
+				local_mem_info.local_mem_size_public,
+				(struct crat_subtype_memory *)sub_type_hdr,
+				proximity_domain,
+				&local_mem_info);
+	else
+		ret = kfd_fill_gpu_memory_affinity(&avail_size,
+				kdev, HSA_MEM_HEAP_TYPE_FB_PRIVATE,
+				local_mem_info.local_mem_size_public +
+				local_mem_info.local_mem_size_private,
+				(struct crat_subtype_memory *)sub_type_hdr,
+				proximity_domain,
+				&local_mem_info);
+	if (ret < 0)
+		return ret;
+
+	crat_table->length += sizeof(struct crat_subtype_memory);
+	crat_table->total_entries++;
+
+	/* TODO: Fill in cache information. This information is NOT readily
+	 * available in KGD
+	 */
+	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+		sub_type_hdr->length);
+	ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low,
+				avail_size,
+				&cu_info,
+				(struct crat_subtype_cache *)sub_type_hdr,
+				&cache_mem_filled,
+				&num_of_cache_entries);
+
+	if (ret < 0)
+		return ret;
+
+	crat_table->length += cache_mem_filled;
+	crat_table->total_entries += num_of_cache_entries;
+	avail_size -= cache_mem_filled;
+
+	/* Fill in Subtype: IO_LINKS
+	 *  Only direct links are added here which is Link from GPU to
+	 *  to its NUMA node. Indirect links are added by userspace.
+	 */
+	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
+		cache_mem_filled);
+	ret = kfd_fill_gpu_direct_io_link(&avail_size, kdev,
+		(struct crat_subtype_iolink *)sub_type_hdr, proximity_domain);
+
+	if (ret < 0)
+		return ret;
+
+	crat_table->length += sub_type_hdr->length;
+	crat_table->total_entries++;
+
+	*size = crat_table->length;
+	pr_info("Virtual CRAT table created for GPU\n");
+
+	return ret;
+}
+
+/* kfd_create_crat_image_virtual - Allocates memory for CRAT image and
+ *		creates a Virtual CRAT (VCRAT) image
+ *
+ * NOTE: Call kfd_destroy_crat_image to free CRAT image memory
+ *
+ *	@crat_image: VCRAT image created because ACPI does not have a
+ *		     CRAT for this device
+ *	@size: [OUT] size of virtual crat_image
+ *	@flags:	COMPUTE_UNIT_CPU - Create VCRAT for CPU device
+ *		COMPUTE_UNIT_GPU - Create VCRAT for GPU
+ *		(COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU) - Create VCRAT for APU
+ *			-- this option is not currently implemented.
+ *			The assumption is that all AMD APUs will have CRAT
+ *	@kdev: Valid kfd_device required if flags contain COMPUTE_UNIT_GPU
+ *
+ *	Return 0 if successful else return -ve value
+ */
+int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
+				  int flags, struct kfd_dev *kdev,
+				  uint32_t proximity_domain)
+{
+	void *pcrat_image = NULL;
+	int ret = 0;
+
+	if (!crat_image)
+		return -EINVAL;
+
+	*crat_image = NULL;
+
+	/* Allocate one VCRAT_SIZE_FOR_CPU for CPU virtual CRAT image and
+	 * VCRAT_SIZE_FOR_GPU for GPU virtual CRAT image. This should cover
+	 * all the current conditions. A check is put not to overwrite beyond
+	 * allocated size
+	 */
+	switch (flags) {
+	case COMPUTE_UNIT_CPU:
+		pcrat_image = kmalloc(VCRAT_SIZE_FOR_CPU, GFP_KERNEL);
+		if (!pcrat_image)
+			return -ENOMEM;
+		*size = VCRAT_SIZE_FOR_CPU;
+		ret = kfd_create_vcrat_image_cpu(pcrat_image, size);
+		break;
+	case COMPUTE_UNIT_GPU:
+		if (!kdev)
+			return -EINVAL;
+		pcrat_image = kmalloc(VCRAT_SIZE_FOR_GPU, GFP_KERNEL);
+		if (!pcrat_image)
+			return -ENOMEM;
+		*size = VCRAT_SIZE_FOR_GPU;
+		ret = kfd_create_vcrat_image_gpu(pcrat_image, size, kdev,
+						 proximity_domain);
+		break;
+	case (COMPUTE_UNIT_CPU | COMPUTE_UNIT_GPU):
+		/* TODO: */
+		ret = -EINVAL;
+		pr_err("VCRAT not implemented for APU\n");
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	if (!ret)
+		*crat_image = pcrat_image;
+	else
+		kfree(pcrat_image);
+
+	return ret;
+}
+
+
+/* kfd_destroy_crat_image
+ *
+ *	@crat_image: [IN] - crat_image from kfd_create_crat_image_xxx(..)
+ *
+ */
+void kfd_destroy_crat_image(void *crat_image)
+{
+	kfree(crat_image);
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
index a374fa3d3ee6..b5cd182b9edd 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
@@ -44,6 +44,10 @@
 
 #define CRAT_OEMID_64BIT_MASK ((1ULL << (CRAT_OEMID_LENGTH * 8)) - 1)
 
+/* Compute Unit flags */
+#define COMPUTE_UNIT_CPU	(1 << 0)  /* Create Virtual CRAT for CPU */
+#define COMPUTE_UNIT_GPU	(1 << 1)  /* Create Virtual CRAT for GPU */
+
 struct crat_header {
 	uint32_t	signature;
 	uint32_t	length;
@@ -105,7 +109,7 @@ struct crat_subtype_computeunit {
 	uint8_t		wave_front_size;
 	uint8_t		num_banks;
 	uint16_t	micro_engine_id;
-	uint8_t		num_arrays;
+	uint8_t		array_count;
 	uint8_t		num_cu_per_array;
 	uint8_t		num_simd_per_cu;
 	uint8_t		max_slots_scatch_cu;
@@ -127,13 +131,14 @@ struct crat_subtype_memory {
 	uint8_t		length;
 	uint16_t	reserved;
 	uint32_t	flags;
-	uint32_t	promixity_domain;
+	uint32_t	proximity_domain;
 	uint32_t	base_addr_low;
 	uint32_t	base_addr_high;
 	uint32_t	length_low;
 	uint32_t	length_high;
 	uint32_t	width;
-	uint8_t		reserved2[CRAT_MEMORY_RESERVED_LENGTH];
+	uint8_t		visibility_type; /* for virtual (dGPU) CRAT */
+	uint8_t		reserved2[CRAT_MEMORY_RESERVED_LENGTH - 1];
 };
 
 /*
@@ -222,9 +227,12 @@ struct crat_subtype_ccompute {
 /*
  * HSA IO Link Affinity structure and definitions
  */
-#define CRAT_IOLINK_FLAGS_ENABLED	0x00000001
-#define CRAT_IOLINK_FLAGS_COHERENCY	0x00000002
-#define CRAT_IOLINK_FLAGS_RESERVED	0xfffffffc
+#define CRAT_IOLINK_FLAGS_ENABLED		(1 << 0)
+#define CRAT_IOLINK_FLAGS_NON_COHERENT		(1 << 1)
+#define CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT	(1 << 2)
+#define CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT	(1 << 3)
+#define CRAT_IOLINK_FLAGS_NO_PEER_TO_PEER_DMA	(1 << 4)
+#define CRAT_IOLINK_FLAGS_RESERVED_MASK		0xffffffe0
 
 /*
  * IO interface types
@@ -232,10 +240,18 @@ struct crat_subtype_ccompute {
 #define CRAT_IOLINK_TYPE_UNDEFINED	0
 #define CRAT_IOLINK_TYPE_HYPERTRANSPORT	1
 #define CRAT_IOLINK_TYPE_PCIEXPRESS	2
-#define CRAT_IOLINK_TYPE_OTHER		3
+#define CRAT_IOLINK_TYPE_AMBA		3
+#define CRAT_IOLINK_TYPE_MIPI		4
+#define CRAT_IOLINK_TYPE_QPI_1_1	5
+#define CRAT_IOLINK_TYPE_RESERVED1	6
+#define CRAT_IOLINK_TYPE_RESERVED2	7
+#define CRAT_IOLINK_TYPE_RAPID_IO	8
+#define CRAT_IOLINK_TYPE_INFINIBAND	9
+#define CRAT_IOLINK_TYPE_RESERVED3	10
+#define CRAT_IOLINK_TYPE_OTHER		11
 #define CRAT_IOLINK_TYPE_MAX		255
 
-#define CRAT_IOLINK_RESERVED_LENGTH 24
+#define CRAT_IOLINK_RESERVED_LENGTH	24
 
 struct crat_subtype_iolink {
 	uint8_t		type;
@@ -291,4 +307,14 @@ struct cdit_header {
 
 #pragma pack()
 
+struct kfd_dev;
+
+int kfd_create_crat_image_acpi(void **crat_image, size_t *size);
+void kfd_destroy_crat_image(void *crat_image);
+int kfd_parse_crat_table(void *crat_image, struct list_head *device_list,
+			 uint32_t proximity_domain);
+int kfd_create_crat_image_virtual(void **crat_image, size_t *size,
+				  int flags, struct kfd_dev *kdev,
+				  uint32_t proximity_domain);
+
 #endif /* KFD_CRAT_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
index c407f6bd9956..afb26f205d29 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c
@@ -95,7 +95,7 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
 	ib_packet->bitfields3.ib_base_hi = largep->u.high_part;
 
 	ib_packet->control = (1 << 23) | (1 << 31) |
-			((size_in_bytes / sizeof(uint32_t)) & 0xfffff);
+			((size_in_bytes / 4) & 0xfffff);
 
 	ib_packet->bitfields5.pasid = pasid;
 
@@ -126,8 +126,7 @@ static int dbgdev_diq_submit_ib(struct kfd_dbgdev *dbgdev,
 
 	rm_packet->header.opcode = IT_RELEASE_MEM;
 	rm_packet->header.type = PM4_TYPE_3;
-	rm_packet->header.count = sizeof(struct pm4__release_mem) /
-					sizeof(unsigned int) - 2;
+	rm_packet->header.count = sizeof(struct pm4__release_mem) / 4 - 2;
 
 	rm_packet->bitfields2.event_type = CACHE_FLUSH_AND_INV_TS_EVENT;
 	rm_packet->bitfields2.event_index =
@@ -652,8 +651,7 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 	packets_vec[0].header.opcode = IT_SET_UCONFIG_REG;
 	packets_vec[0].header.type = PM4_TYPE_3;
 	packets_vec[0].bitfields2.reg_offset =
-			GRBM_GFX_INDEX / (sizeof(uint32_t)) -
-				USERCONFIG_REG_BASE;
+			GRBM_GFX_INDEX / 4 - USERCONFIG_REG_BASE;
 
 	packets_vec[0].bitfields2.insert_vmid = 0;
 	packets_vec[0].reg_data[0] = reg_gfx_index.u32All;
@@ -661,8 +659,7 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 	packets_vec[1].header.count = 1;
 	packets_vec[1].header.opcode = IT_SET_CONFIG_REG;
 	packets_vec[1].header.type = PM4_TYPE_3;
-	packets_vec[1].bitfields2.reg_offset = SQ_CMD / (sizeof(uint32_t)) -
-						AMD_CONFIG_REG_BASE;
+	packets_vec[1].bitfields2.reg_offset = SQ_CMD / 4 - AMD_CONFIG_REG_BASE;
 
 	packets_vec[1].bitfields2.vmid_shift = SQ_CMD_VMID_OFFSET;
 	packets_vec[1].bitfields2.insert_vmid = 1;
@@ -678,8 +675,7 @@ static int dbgdev_wave_control_diq(struct kfd_dbgdev *dbgdev,
 
 	packets_vec[2].ordinal1 = packets_vec[0].ordinal1;
 	packets_vec[2].bitfields2.reg_offset =
-				GRBM_GFX_INDEX / (sizeof(uint32_t)) -
-					USERCONFIG_REG_BASE;
+				GRBM_GFX_INDEX / 4 - USERCONFIG_REG_BASE;
 
 	packets_vec[2].bitfields2.insert_vmid = 0;
 	packets_vec[2].reg_data[0] = reg_gfx_index.u32All;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
new file mode 100644
index 000000000000..4bd6ebfaf425
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2016-2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/debugfs.h>
+#include "kfd_priv.h"
+
+static struct dentry *debugfs_root;
+
+static int kfd_debugfs_open(struct inode *inode, struct file *file)
+{
+	int (*show)(struct seq_file *, void *) = inode->i_private;
+
+	return single_open(file, show, NULL);
+}
+
+static const struct file_operations kfd_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.open = kfd_debugfs_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+void kfd_debugfs_init(void)
+{
+	struct dentry *ent;
+
+	debugfs_root = debugfs_create_dir("kfd", NULL);
+	if (!debugfs_root || debugfs_root == ERR_PTR(-ENODEV)) {
+		pr_warn("Failed to create kfd debugfs dir\n");
+		return;
+	}
+
+	ent = debugfs_create_file("mqds", S_IFREG | 0444, debugfs_root,
+				  kfd_debugfs_mqds_by_process,
+				  &kfd_debugfs_fops);
+	if (!ent)
+		pr_warn("Failed to create mqds in kfd debugfs\n");
+
+	ent = debugfs_create_file("hqds", S_IFREG | 0444, debugfs_root,
+				  kfd_debugfs_hqds_by_device,
+				  &kfd_debugfs_fops);
+	if (!ent)
+		pr_warn("Failed to create hqds in kfd debugfs\n");
+
+	ent = debugfs_create_file("rls", S_IFREG | 0444, debugfs_root,
+				  kfd_debugfs_rls_by_device,
+				  &kfd_debugfs_fops);
+	if (!ent)
+		pr_warn("Failed to create rls in kfd debugfs\n");
+}
+
+void kfd_debugfs_fini(void)
+{
+	debugfs_remove_recursive(debugfs_root);
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 621a3b53a038..a8fa33a08de3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -27,6 +27,7 @@
 #include "kfd_priv.h"
 #include "kfd_device_queue_manager.h"
 #include "kfd_pm4_headers_vi.h"
+#include "cwsr_trap_handler_gfx8.asm"
 
 #define MQD_SIZE_ALIGNED 768
 
@@ -38,7 +39,8 @@ static const struct kfd_device_info kaveri_device_info = {
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
-	.mqd_size_aligned = MQD_SIZE_ALIGNED
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.supports_cwsr = false,
 };
 
 static const struct kfd_device_info carrizo_device_info = {
@@ -49,7 +51,8 @@ static const struct kfd_device_info carrizo_device_info = {
 	.ih_ring_entry_size = 4 * sizeof(uint32_t),
 	.event_interrupt_class = &event_interrupt_class_cik,
 	.num_of_watch_points = 4,
-	.mqd_size_aligned = MQD_SIZE_ALIGNED
+	.mqd_size_aligned = MQD_SIZE_ALIGNED,
+	.supports_cwsr = true,
 };
 
 struct kfd_deviceid {
@@ -212,6 +215,17 @@ static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
 	return AMD_IOMMU_INV_PRI_RSP_INVALID;
 }
 
+static void kfd_cwsr_init(struct kfd_dev *kfd)
+{
+	if (cwsr_enable && kfd->device_info->supports_cwsr) {
+		BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
+
+		kfd->cwsr_isa = cwsr_trap_gfx8_hex;
+		kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
+		kfd->cwsr_enabled = true;
+	}
+}
+
 bool kgd2kfd_device_init(struct kfd_dev *kfd,
 			 const struct kgd2kfd_shared_resources *gpu_resources)
 {
@@ -224,6 +238,17 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 	kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd
 			- kfd->vm_info.first_vmid_kfd + 1;
 
+	/* Verify module parameters regarding mapped process number*/
+	if ((hws_max_conc_proc < 0)
+			|| (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) {
+		dev_err(kfd_device,
+			"hws_max_conc_proc %d must be between 0 and %d, use %d instead\n",
+			hws_max_conc_proc, kfd->vm_info.vmid_num_kfd,
+			kfd->vm_info.vmid_num_kfd);
+		kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd;
+	} else
+		kfd->max_proc_per_quantum = hws_max_conc_proc;
+
 	/* calculate max size of mqds needed for queues */
 	size = max_num_of_queues_per_device *
 			kfd->device_info->mqd_size_aligned;
@@ -286,6 +311,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 		goto device_iommu_pasid_error;
 	}
 
+	kfd_cwsr_init(kfd);
+
 	if (kfd_resume(kfd))
 		goto kfd_resume_error;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index e202921c150e..1bd5f26b3f00 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -149,8 +149,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm,
 
 static int create_queue_nocpsch(struct device_queue_manager *dqm,
 				struct queue *q,
-				struct qcm_process_device *qpd,
-				int *allocated_vmid)
+				struct qcm_process_device *qpd)
 {
 	int retval;
 
@@ -170,9 +169,11 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 		if (retval)
 			goto out_unlock;
 	}
-	*allocated_vmid = qpd->vmid;
 	q->properties.vmid = qpd->vmid;
 
+	q->properties.tba_addr = qpd->tba_addr;
+	q->properties.tma_addr = qpd->tma_addr;
+
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 		retval = create_compute_queue_nocpsch(dqm, q, qpd);
 	else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
@@ -181,10 +182,8 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm,
 		retval = -EINVAL;
 
 	if (retval) {
-		if (list_empty(&qpd->queues_list)) {
+		if (list_empty(&qpd->queues_list))
 			deallocate_vmid(dqm, qpd, q);
-			*allocated_vmid = 0;
-		}
 		goto out_unlock;
 	}
 
@@ -809,29 +808,26 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 }
 
 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
-			struct qcm_process_device *qpd, int *allocate_vmid)
+			struct qcm_process_device *qpd)
 {
 	int retval;
 	struct mqd_manager *mqd;
 
 	retval = 0;
 
-	if (allocate_vmid)
-		*allocate_vmid = 0;
-
 	mutex_lock(&dqm->lock);
 
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
 				dqm->total_queue_count);
 		retval = -EPERM;
-		goto out;
+		goto out_unlock;
 	}
 
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 		retval = allocate_sdma_queue(dqm, &q->sdma_id);
 		if (retval)
-			goto out;
+			goto out_unlock;
 		q->properties.sdma_queue_id =
 			q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
 		q->properties.sdma_engine_id =
@@ -842,14 +838,17 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 
 	if (!mqd) {
 		retval = -ENOMEM;
-		goto out;
+		goto out_deallocate_sdma_queue;
 	}
 
 	dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
+
+	q->properties.tba_addr = qpd->tba_addr;
+	q->properties.tma_addr = qpd->tma_addr;
 	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
-		goto out;
+		goto out_deallocate_sdma_queue;
 
 	list_add(&q->list, &qpd->queues_list);
 	qpd->queue_count++;
@@ -870,7 +869,13 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
 
-out:
+	mutex_unlock(&dqm->lock);
+	return retval;
+
+out_deallocate_sdma_queue:
+	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+		deallocate_sdma_queue(dqm, q->sdma_id);
+out_unlock:
 	mutex_unlock(&dqm->lock);
 	return retval;
 }
@@ -1014,13 +1019,13 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 
 	list_del(&q->list);
 	qpd->queue_count--;
-	if (q->properties.is_active)
+	if (q->properties.is_active) {
 		dqm->queue_count--;
-
-	retval = execute_queues_cpsch(dqm,
+		retval = execute_queues_cpsch(dqm,
 				KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
-	if (retval == -ETIME)
-		qpd->reset_wavefronts = true;
+		if (retval == -ETIME)
+			qpd->reset_wavefronts = true;
+	}
 
 	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 
@@ -1034,7 +1039,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 
 	mutex_unlock(&dqm->lock);
 
-	return 0;
+	return retval;
 
 failed:
 failed_try_destroy_debugged_queue:
@@ -1110,6 +1115,26 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 	return retval;
 }
 
+static int set_trap_handler(struct device_queue_manager *dqm,
+				struct qcm_process_device *qpd,
+				uint64_t tba_addr,
+				uint64_t tma_addr)
+{
+	uint64_t *tma;
+
+	if (dqm->dev->cwsr_enabled) {
+		/* Jump from CWSR trap handler to user trap */
+		tma = (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
+		tma[0] = tba_addr;
+		tma[1] = tma_addr;
+	} else {
+		qpd->tba_addr = tba_addr;
+		qpd->tma_addr = tma_addr;
+	}
+
+	return 0;
+}
+
 static int process_termination_nocpsch(struct device_queue_manager *dqm,
 		struct qcm_process_device *qpd)
 {
@@ -1169,8 +1194,10 @@ static int process_termination_cpsch(struct device_queue_manager *dqm,
 
 	/* Clear all user mode queues */
 	list_for_each_entry(q, &qpd->queues_list, list) {
-		if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
+		if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 			dqm->sdma_queue_count--;
+			deallocate_sdma_queue(dqm, q->sdma_id);
+		}
 
 		if (q->properties.is_active)
 			dqm->queue_count--;
@@ -1241,6 +1268,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 		dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
 		dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
 		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+		dqm->ops.set_trap_handler = set_trap_handler;
 		dqm->ops.process_termination = process_termination_cpsch;
 		break;
 	case KFD_SCHED_POLICY_NO_HWS:
@@ -1256,6 +1284,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 		dqm->ops.initialize = initialize_nocpsch;
 		dqm->ops.uninitialize = uninitialize;
 		dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
+		dqm->ops.set_trap_handler = set_trap_handler;
 		dqm->ops.process_termination = process_termination_nocpsch;
 		break;
 	default:
@@ -1290,3 +1319,74 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm)
 	dqm->ops.uninitialize(dqm);
 	kfree(dqm);
 }
+
+#if defined(CONFIG_DEBUG_FS)
+
+static void seq_reg_dump(struct seq_file *m,
+			 uint32_t (*dump)[2], uint32_t n_regs)
+{
+	uint32_t i, count;
+
+	for (i = 0, count = 0; i < n_regs; i++) {
+		if (count == 0 ||
+		    dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
+			seq_printf(m, "%s    %08x: %08x",
+				   i ? "\n" : "",
+				   dump[i][0], dump[i][1]);
+			count = 7;
+		} else {
+			seq_printf(m, " %08x", dump[i][1]);
+			count--;
+		}
+	}
+
+	seq_puts(m, "\n");
+}
+
+int dqm_debugfs_hqds(struct seq_file *m, void *data)
+{
+	struct device_queue_manager *dqm = data;
+	uint32_t (*dump)[2], n_regs;
+	int pipe, queue;
+	int r = 0;
+
+	for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
+		int pipe_offset = pipe * get_queues_per_pipe(dqm);
+
+		for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
+			if (!test_bit(pipe_offset + queue,
+				      dqm->dev->shared_resources.queue_bitmap))
+				continue;
+
+			r = dqm->dev->kfd2kgd->hqd_dump(
+				dqm->dev->kgd, pipe, queue, &dump, &n_regs);
+			if (r)
+				break;
+
+			seq_printf(m, "  CP Pipe %d, Queue %d\n",
+				  pipe, queue);
+			seq_reg_dump(m, dump, n_regs);
+
+			kfree(dump);
+		}
+	}
+
+	for (pipe = 0; pipe < CIK_SDMA_ENGINE_NUM; pipe++) {
+		for (queue = 0; queue < CIK_SDMA_QUEUES_PER_ENGINE; queue++) {
+			r = dqm->dev->kfd2kgd->hqd_sdma_dump(
+				dqm->dev->kgd, pipe, queue, &dump, &n_regs);
+			if (r)
+				break;
+
+			seq_printf(m, "  SDMA Engine %d, RLC %d\n",
+				  pipe, queue);
+			seq_reg_dump(m, dump, n_regs);
+
+			kfree(dump);
+		}
+	}
+
+	return r;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 5b77cb69f732..c61b693bfa8c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -84,8 +84,7 @@ struct device_process_node {
 struct device_queue_manager_ops {
 	int	(*create_queue)(struct device_queue_manager *dqm,
 				struct queue *q,
-				struct qcm_process_device *qpd,
-				int *allocate_vmid);
+				struct qcm_process_device *qpd);
 
 	int	(*destroy_queue)(struct device_queue_manager *dqm,
 				struct qcm_process_device *qpd,
@@ -123,6 +122,11 @@ struct device_queue_manager_ops {
 					   void __user *alternate_aperture_base,
 					   uint64_t alternate_aperture_size);
 
+	int	(*set_trap_handler)(struct device_queue_manager *dqm,
+				    struct qcm_process_device *qpd,
+				    uint64_t tba_addr,
+				    uint64_t tma_addr);
+
 	int (*process_termination)(struct device_queue_manager *dqm,
 			struct qcm_process_device *qpd);
 };
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index feb76c235b1a..ebb4da14e3df 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -116,8 +116,7 @@ int kfd_doorbell_init(struct kfd_dev *kfd)
 	pr_debug("doorbell aperture size  == 0x%08lX\n",
 			kfd->shared_resources.doorbell_aperture_size);
 
-	pr_debug("doorbell kernel address == 0x%08lX\n",
-			(uintptr_t)kfd->doorbell_kernel_ptr);
+	pr_debug("doorbell kernel address == %p\n", kfd->doorbell_kernel_ptr);
 
 	return 0;
 }
@@ -194,8 +193,8 @@ u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
 
 	pr_debug("Get kernel queue doorbell\n"
 			 "     doorbell offset   == 0x%08X\n"
-			 "     kernel address    == 0x%08lX\n",
-		*doorbell_off, (uintptr_t)(kfd->doorbell_kernel_ptr + inx));
+			 "     kernel address    == %p\n",
+		*doorbell_off, (kfd->doorbell_kernel_ptr + inx));
 
 	return kfd->doorbell_kernel_ptr + inx;
 }
@@ -215,7 +214,7 @@ inline void write_kernel_doorbell(u32 __iomem *db, u32 value)
 {
 	if (db) {
 		writel(value, db);
-		pr_debug("Writing %d to doorbell address 0x%p\n", value, db);
+		pr_debug("Writing %d to doorbell address %p\n", value, db);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index cb92d4b72400..93aae5c1e78b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -441,7 +441,7 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
 	/*
 	 * Because we are called from arbitrary context (workqueue) as opposed
 	 * to process context, kfd_process could attempt to exit while we are
-	 * running so the lookup function returns a locked process.
+	 * running so the lookup function increments the process ref count.
 	 */
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 
@@ -493,7 +493,7 @@ void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
 	}
 
 	mutex_unlock(&p->event_mutex);
-	mutex_unlock(&p->mutex);
+	kfd_unref_process(p);
 }
 
 static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
@@ -847,7 +847,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 	/*
 	 * Because we are called from arbitrary context (workqueue) as opposed
 	 * to process context, kfd_process could attempt to exit while we are
-	 * running so the lookup function returns a locked process.
+	 * running so the lookup function increments the process ref count.
 	 */
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 	struct mm_struct *mm;
@@ -860,7 +860,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 	 */
 	mm = get_task_mm(p->lead_thread);
 	if (!mm) {
-		mutex_unlock(&p->mutex);
+		kfd_unref_process(p);
 		return; /* Process is exiting */
 	}
 
@@ -903,7 +903,7 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid,
 			&memory_exception_data);
 
 	mutex_unlock(&p->event_mutex);
-	mutex_unlock(&p->mutex);
+	kfd_unref_process(p);
 }
 
 void kfd_signal_hw_exception_event(unsigned int pasid)
@@ -911,7 +911,7 @@ void kfd_signal_hw_exception_event(unsigned int pasid)
 	/*
 	 * Because we are called from arbitrary context (workqueue) as opposed
 	 * to process context, kfd_process could attempt to exit while we are
-	 * running so the lookup function returns a locked process.
+	 * running so the lookup function increments the process ref count.
 	 */
 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
 
@@ -924,5 +924,5 @@ void kfd_signal_hw_exception_event(unsigned int pasid)
 	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);
 
 	mutex_unlock(&p->event_mutex);
-	mutex_unlock(&p->mutex);
+	kfd_unref_process(p);
 }
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
index c59384bbbc5f..7377513050e6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c
@@ -300,9 +300,14 @@ int kfd_init_apertures(struct kfd_process *process)
 	struct kfd_process_device *pdd;
 
 	/*Iterating over all devices*/
-	while ((dev = kfd_topology_enum_kfd_devices(id)) != NULL &&
+	while (kfd_topology_enum_kfd_devices(id, &dev) == 0 &&
 		id < NUM_OF_SUPPORTED_GPUS) {
 
+		if (!dev) {
+			id++; /* Skip non GPU devices */
+			continue;
+		}
+
 		pdd = kfd_create_process_device_data(dev, process);
 		if (!pdd) {
 			pr_err("Failed to create process device data\n");
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
index 8b0c0645d7c0..5dc6567d4a13 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c
@@ -218,7 +218,7 @@ static int acquire_packet_buffer(struct kernel_queue *kq,
 	rptr = *kq->rptr_kernel;
 	wptr = *kq->wptr_kernel;
 	queue_address = (unsigned int *)kq->pq_kernel_addr;
-	queue_size_dwords = kq->queue->properties.queue_size / sizeof(uint32_t);
+	queue_size_dwords = kq->queue->properties.queue_size / 4;
 
 	pr_debug("rptr: %d\n", rptr);
 	pr_debug("wptr: %d\n", wptr);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index f744caeaee04..3ac72bed4f31 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -50,6 +50,15 @@ module_param(sched_policy, int, 0444);
 MODULE_PARM_DESC(sched_policy,
 	"Scheduling policy (0 = HWS (Default), 1 = HWS without over-subscription, 2 = Non-HWS (Used for debugging only)");
 
+int hws_max_conc_proc = 8;
+module_param(hws_max_conc_proc, int, 0444);
+MODULE_PARM_DESC(hws_max_conc_proc,
+	"Max # processes HWS can execute concurrently when sched_policy=0 (0 = no concurrency, #VMIDs for KFD = Maximum(default))");
+
+int cwsr_enable = 1;
+module_param(cwsr_enable, int, 0444);
+MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
+
 int max_num_of_queues_per_device = KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT;
 module_param(max_num_of_queues_per_device, int, 0444);
 MODULE_PARM_DESC(max_num_of_queues_per_device,
@@ -60,6 +69,11 @@ module_param(send_sigterm, int, 0444);
 MODULE_PARM_DESC(send_sigterm,
 	"Send sigterm to HSA process on unhandled exception (0 = disable, 1 = enable)");
 
+int ignore_crat;
+module_param(ignore_crat, int, 0444);
+MODULE_PARM_DESC(ignore_crat,
+	"Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)");
+
 static int amdkfd_init_completed;
 
 int kgd2kfd_init(unsigned int interface_version,
@@ -114,6 +128,8 @@ static int __init kfd_module_init(void)
 
 	kfd_process_create_wq();
 
+	kfd_debugfs_init();
+
 	amdkfd_init_completed = 1;
 
 	dev_info(kfd_device, "Initialized module\n");
@@ -130,6 +146,7 @@ static void __exit kfd_module_exit(void)
 {
 	amdkfd_init_completed = 0;
 
+	kfd_debugfs_fini();
 	kfd_process_destroy_wq();
 	kfd_topology_shutdown();
 	kfd_chardev_exit();
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
index 1f3a6ba7eed2..8972bcfbf701 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h
@@ -85,6 +85,10 @@ struct mqd_manager {
 				uint64_t queue_address,	uint32_t pipe_id,
 				uint32_t queue_id);
 
+#if defined(CONFIG_DEBUG_FS)
+	int	(*debugfs_show_mqd)(struct seq_file *m, void *data);
+#endif
+
 	struct mutex	mqd_mutex;
 	struct kfd_dev	*dev;
 };
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
index 4728fad3fd74..f8ef4a051e08 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c
@@ -36,6 +36,11 @@ static inline struct cik_mqd *get_mqd(void *mqd)
 	return (struct cik_mqd *)mqd;
 }
 
+static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
+{
+	return (struct cik_sdma_rlc_registers *)mqd;
+}
+
 static int init_mqd(struct mqd_manager *mm, void **mqd,
 		struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 		struct queue_properties *q)
@@ -149,7 +154,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id,
 {
 	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
 	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
-	uint32_t wptr_mask = (uint32_t)((p->queue_size / sizeof(uint32_t)) - 1);
+	uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
 
 	return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
 					  (uint32_t __user *)p->write_ptr,
@@ -160,7 +165,9 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
 			 uint32_t pipe_id, uint32_t queue_id,
 			 struct queue_properties *p, struct mm_struct *mms)
 {
-	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd);
+	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+					       (uint32_t __user *)p->write_ptr,
+					       mms);
 }
 
 static int update_mqd(struct mqd_manager *mm, void *mqd,
@@ -176,8 +183,7 @@ static int update_mqd(struct mqd_manager *mm, void *mqd,
 	 * Calculating queue size which is log base 2 of actual queue size -1
 	 * dwords and another -1 for ffs
 	 */
-	m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
-								- 1 - 1;
+	m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
 	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
 	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
@@ -202,7 +208,7 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
 	struct cik_sdma_rlc_registers *m;
 
 	m = get_sdma_mqd(mqd);
-	m->sdma_rlc_rb_cntl = (ffs(q->queue_size / sizeof(unsigned int)) - 1)
+	m->sdma_rlc_rb_cntl = order_base_2(q->queue_size / 4)
 			<< SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
 			q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
 			1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
@@ -343,8 +349,7 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
 	 * Calculating queue size which is log base 2 of actual queue
 	 * size -1 dwords
 	 */
-	m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int))
-								- 1 - 1;
+	m->cp_hqd_pq_control |= order_base_2(q->queue_size / 4) - 1;
 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
 	m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8);
 	m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
@@ -360,15 +365,25 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
 	return 0;
 }
 
-struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd)
-{
-	struct cik_sdma_rlc_registers *m;
+#if defined(CONFIG_DEBUG_FS)
 
-	m = (struct cik_sdma_rlc_registers *)mqd;
+static int debugfs_show_mqd(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct cik_mqd), false);
+	return 0;
+}
 
-	return m;
+static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct cik_sdma_rlc_registers), false);
+	return 0;
 }
 
+#endif
+
+
 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev)
 {
@@ -392,6 +407,9 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 		mqd->update_mqd = update_mqd;
 		mqd->destroy_mqd = destroy_mqd;
 		mqd->is_occupied = is_occupied;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
 		break;
 	case KFD_MQD_TYPE_HIQ:
 		mqd->init_mqd = init_mqd_hiq;
@@ -400,6 +418,9 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 		mqd->update_mqd = update_mqd_hiq;
 		mqd->destroy_mqd = destroy_mqd;
 		mqd->is_occupied = is_occupied;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
 		break;
 	case KFD_MQD_TYPE_SDMA:
 		mqd->init_mqd = init_mqd_sdma;
@@ -408,6 +429,9 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
 		mqd->update_mqd = update_mqd_sdma;
 		mqd->destroy_mqd = destroy_mqd_sdma;
 		mqd->is_occupied = is_occupied_sdma;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+#endif
 		break;
 	default:
 		kfree(mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 4ea854f9007b..971aec0637dc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -30,7 +30,7 @@
 #include "vi_structs.h"
 #include "gca/gfx_8_0_sh_mask.h"
 #include "gca/gfx_8_0_enum.h"
-
+#include "oss/oss_3_0_sh_mask.h"
 #define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8
 
 static inline struct vi_mqd *get_mqd(void *mqd)
@@ -38,6 +38,11 @@ static inline struct vi_mqd *get_mqd(void *mqd)
 	return (struct vi_mqd *)mqd;
 }
 
+static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd)
+{
+	return (struct vi_sdma_mqd *)mqd;
+}
+
 static int init_mqd(struct mqd_manager *mm, void **mqd,
 			struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
 			struct queue_properties *q)
@@ -84,6 +89,28 @@ static int init_mqd(struct mqd_manager *mm, void **mqd,
 	if (q->format == KFD_QUEUE_FORMAT_AQL)
 		m->cp_hqd_iq_rptr = 1;
 
+	if (q->tba_addr) {
+		m->compute_tba_lo = lower_32_bits(q->tba_addr >> 8);
+		m->compute_tba_hi = upper_32_bits(q->tba_addr >> 8);
+		m->compute_tma_lo = lower_32_bits(q->tma_addr >> 8);
+		m->compute_tma_hi = upper_32_bits(q->tma_addr >> 8);
+		m->compute_pgm_rsrc2 |=
+			(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
+	}
+
+	if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address) {
+		m->cp_hqd_persistent_state |=
+			(1 << CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT);
+		m->cp_hqd_ctx_save_base_addr_lo =
+			lower_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_base_addr_hi =
+			upper_32_bits(q->ctx_save_restore_area_address);
+		m->cp_hqd_ctx_save_size = q->ctx_save_restore_area_size;
+		m->cp_hqd_cntl_stack_size = q->ctl_stack_size;
+		m->cp_hqd_cntl_stack_offset = q->ctl_stack_size;
+		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
+	}
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
@@ -98,7 +125,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
 {
 	/* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
 	uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
-	uint32_t wptr_mask = (uint32_t)((p->queue_size / sizeof(uint32_t)) - 1);
+	uint32_t wptr_mask = (uint32_t)((p->queue_size / 4) - 1);
 
 	return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
 					  (uint32_t __user *)p->write_ptr,
@@ -116,8 +143,7 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 	m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT |
 			atc_bit << CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT |
 			mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT;
-	m->cp_hqd_pq_control |=
-			ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1;
+	m->cp_hqd_pq_control |=	order_base_2(q->queue_size / 4) - 1;
 	pr_debug("cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control);
 
 	m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8);
@@ -147,7 +173,7 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 	 * is safe, giving a maximum field value of 0xA.
 	 */
 	m->cp_hqd_eop_control |= min(0xA,
-		ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1);
+		order_base_2(q->eop_ring_buffer_size / 4) - 1);
 	m->cp_hqd_eop_base_addr_lo =
 			lower_32_bits(q->eop_ring_buffer_address >> 8);
 	m->cp_hqd_eop_base_addr_hi =
@@ -163,6 +189,11 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd,
 				2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT;
 	}
 
+	if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
+		m->cp_hqd_ctx_save_control =
+			atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
+			mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
+
 	q->is_active = (q->queue_size > 0 &&
 			q->queue_address != 0 &&
 			q->queue_percent > 0);
@@ -234,6 +265,117 @@ static int update_mqd_hiq(struct mqd_manager *mm, void *mqd,
 	return retval;
 }
 
+static int init_mqd_sdma(struct mqd_manager *mm, void **mqd,
+		struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
+		struct queue_properties *q)
+{
+	int retval;
+	struct vi_sdma_mqd *m;
+
+
+	retval = kfd_gtt_sa_allocate(mm->dev,
+			sizeof(struct vi_sdma_mqd),
+			mqd_mem_obj);
+
+	if (retval != 0)
+		return -ENOMEM;
+
+	m = (struct vi_sdma_mqd *) (*mqd_mem_obj)->cpu_ptr;
+
+	memset(m, 0, sizeof(struct vi_sdma_mqd));
+
+	*mqd = m;
+	if (gart_addr != NULL)
+		*gart_addr = (*mqd_mem_obj)->gpu_addr;
+
+	retval = mm->update_mqd(mm, m, q);
+
+	return retval;
+}
+
+static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		struct kfd_mem_obj *mqd_mem_obj)
+{
+	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
+}
+
+static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		uint32_t pipe_id, uint32_t queue_id,
+		struct queue_properties *p, struct mm_struct *mms)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+					       (uint32_t __user *)p->write_ptr,
+					       mms);
+}
+
+static int update_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		struct queue_properties *q)
+{
+	struct vi_sdma_mqd *m;
+
+	m = get_sdma_mqd(mqd);
+	m->sdmax_rlcx_rb_cntl = order_base_2(q->queue_size / 4)
+		<< SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT |
+		q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT |
+		1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT |
+		6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT;
+
+	m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8);
+	m->sdmax_rlcx_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr);
+	m->sdmax_rlcx_doorbell =
+		q->doorbell_off << SDMA0_RLC0_DOORBELL__OFFSET__SHIFT;
+
+	m->sdmax_rlcx_virtual_addr = q->sdma_vm_addr;
+
+	m->sdma_engine_id = q->sdma_engine_id;
+	m->sdma_queue_id = q->sdma_queue_id;
+
+	q->is_active = (q->queue_size > 0 &&
+			q->queue_address != 0 &&
+			q->queue_percent > 0);
+
+	return 0;
+}
+
+/*
+ *  * preempt type here is ignored because there is only one way
+ *  * to preempt sdma queue
+ */
+static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
+		enum kfd_preempt_type type,
+		unsigned int timeout, uint32_t pipe_id,
+		uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
+}
+
+static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
+		uint64_t queue_address, uint32_t pipe_id,
+		uint32_t queue_id)
+{
+	return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
+}
+
+#if defined(CONFIG_DEBUG_FS)
+
+static int debugfs_show_mqd(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct vi_mqd), false);
+	return 0;
+}
+
+static int debugfs_show_mqd_sdma(struct seq_file *m, void *data)
+{
+	seq_hex_dump(m, "    ", DUMP_PREFIX_OFFSET, 32, 4,
+		     data, sizeof(struct vi_sdma_mqd), false);
+	return 0;
+}
+
+#endif
+
 struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		struct kfd_dev *dev)
 {
@@ -257,6 +399,9 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		mqd->update_mqd = update_mqd;
 		mqd->destroy_mqd = destroy_mqd;
 		mqd->is_occupied = is_occupied;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
 		break;
 	case KFD_MQD_TYPE_HIQ:
 		mqd->init_mqd = init_mqd_hiq;
@@ -265,8 +410,20 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
 		mqd->update_mqd = update_mqd_hiq;
 		mqd->destroy_mqd = destroy_mqd;
 		mqd->is_occupied = is_occupied;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd;
+#endif
 		break;
 	case KFD_MQD_TYPE_SDMA:
+		mqd->init_mqd = init_mqd_sdma;
+		mqd->uninit_mqd = uninit_mqd_sdma;
+		mqd->load_mqd = load_mqd_sdma;
+		mqd->update_mqd = update_mqd_sdma;
+		mqd->destroy_mqd = destroy_mqd_sdma;
+		mqd->is_occupied = is_occupied_sdma;
+#if defined(CONFIG_DEBUG_FS)
+		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
+#endif
 		break;
 	default:
 		kfree(mqd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index 16da8ad02d8b..0c3bc00978f7 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -45,7 +45,7 @@ static unsigned int build_pm4_header(unsigned int opcode, size_t packet_size)
 
 	header.u32All = 0;
 	header.opcode = opcode;
-	header.count = packet_size/sizeof(uint32_t) - 2;
+	header.count = packet_size / 4 - 2;
 	header.type = PM4_TYPE_3;
 
 	return header.u32All;
@@ -55,15 +55,27 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 				unsigned int *rlib_size,
 				bool *over_subscription)
 {
-	unsigned int process_count, queue_count;
+	unsigned int process_count, queue_count, compute_queue_count;
 	unsigned int map_queue_size;
+	unsigned int max_proc_per_quantum = 1;
+	struct kfd_dev *dev = pm->dqm->dev;
 
 	process_count = pm->dqm->processes_count;
 	queue_count = pm->dqm->queue_count;
+	compute_queue_count = queue_count - pm->dqm->sdma_queue_count;
 
-	/* check if there is over subscription*/
+	/* check if there is over subscription
+	 * Note: the arbitration between the number of VMIDs and
+	 * hws_max_conc_proc has been done in
+	 * kgd2kfd_device_init().
+	 */
 	*over_subscription = false;
-	if ((process_count > 1) || queue_count > get_queues_num(pm->dqm)) {
+
+	if (dev->max_proc_per_quantum > 1)
+		max_proc_per_quantum = dev->max_proc_per_quantum;
+
+	if ((process_count > max_proc_per_quantum) ||
+	    compute_queue_count > get_queues_num(pm->dqm)) {
 		*over_subscription = true;
 		pr_debug("Over subscribed runlist\n");
 	}
@@ -116,10 +128,24 @@ static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
 			uint64_t ib, size_t ib_size_in_dwords, bool chain)
 {
 	struct pm4_mes_runlist *packet;
+	int concurrent_proc_cnt = 0;
+	struct kfd_dev *kfd = pm->dqm->dev;
 
 	if (WARN_ON(!ib))
 		return -EFAULT;
 
+	/* Determine the number of processes to map together to HW:
+	 * it can not exceed the number of VMIDs available to the
+	 * scheduler, and it is determined by the smaller of the number
+	 * of processes in the runlist and kfd module parameter
+	 * hws_max_conc_proc.
+	 * Note: the arbitration between the number of VMIDs and
+	 * hws_max_conc_proc has been done in
+	 * kgd2kfd_device_init().
+	 */
+	concurrent_proc_cnt = min(pm->dqm->processes_count,
+			kfd->max_proc_per_quantum);
+
 	packet = (struct pm4_mes_runlist *)buffer;
 
 	memset(buffer, 0, sizeof(struct pm4_mes_runlist));
@@ -130,6 +156,7 @@ static int pm_create_runlist(struct packet_manager *pm, uint32_t *buffer,
 	packet->bitfields4.chain = chain ? 1 : 0;
 	packet->bitfields4.offload_polling = 0;
 	packet->bitfields4.valid = 1;
+	packet->bitfields4.process_cnt = concurrent_proc_cnt;
 	packet->ordinal2 = lower_32_bits(ib);
 	packet->bitfields3.ib_base_hi = upper_32_bits(ib);
 
@@ -161,8 +188,7 @@ static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer,
 	packet->sh_mem_ape1_base = qpd->sh_mem_ape1_base;
 	packet->sh_mem_ape1_limit = qpd->sh_mem_ape1_limit;
 
-	/* TODO: scratch support */
-	packet->sh_hidden_private_base_vmid = 0;
+	packet->sh_hidden_private_base_vmid = qpd->sh_hidden_private_base;
 
 	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
 	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
@@ -251,6 +277,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 		return retval;
 
 	*rl_size_bytes = alloc_size_bytes;
+	pm->ib_size_bytes = alloc_size_bytes;
 
 	pr_debug("Building runlist ib process count: %d queues count %d\n",
 		pm->dqm->processes_count, pm->dqm->queue_count);
@@ -564,3 +591,26 @@ void pm_release_ib(struct packet_manager *pm)
 	}
 	mutex_unlock(&pm->lock);
 }
+
+#if defined(CONFIG_DEBUG_FS)
+
+int pm_debugfs_runlist(struct seq_file *m, void *data)
+{
+	struct packet_manager *pm = data;
+
+	mutex_lock(&pm->lock);
+
+	if (!pm->allocated) {
+		seq_puts(m, "  No active runlist\n");
+		goto out;
+	}
+
+	seq_hex_dump(m, "  ", DUMP_PREFIX_OFFSET, 32, 4,
+		     pm->ib_buffer_obj->cpu_ptr, pm->ib_size_bytes, false);
+
+out:
+	mutex_unlock(&pm->lock);
+	return 0;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
index d6a796144269..15fff4420e53 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_pasid.c
@@ -59,7 +59,7 @@ unsigned int kfd_pasid_alloc(void)
 		struct kfd_dev *dev = NULL;
 		unsigned int i = 0;
 
-		while ((dev = kfd_topology_enum_kfd_devices(i)) != NULL) {
+		while ((kfd_topology_enum_kfd_devices(i, &dev)) == 0) {
 			if (dev && dev->kfd2kgd) {
 				kfd2kgd = dev->kfd2kgd;
 				break;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 9e4134c5b481..0bedcf9cc08c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -33,14 +33,17 @@
 #include <linux/kfd_ioctl.h>
 #include <linux/idr.h>
 #include <linux/kfifo.h>
+#include <linux/seq_file.h>
+#include <linux/kref.h>
 #include <kgd_kfd_interface.h>
 
 #include "amd_shared.h"
 
 #define KFD_SYSFS_FILE_MODE 0444
 
-#define KFD_MMAP_DOORBELL_MASK 0x8000000000000
-#define KFD_MMAP_EVENTS_MASK 0x4000000000000
+#define KFD_MMAP_DOORBELL_MASK 0x8000000000000ull
+#define KFD_MMAP_EVENTS_MASK 0x4000000000000ull
+#define KFD_MMAP_RESERVED_MEM_MASK 0x2000000000000ull
 
 /*
  * When working with cp scheduler we should assign the HIQ manually or via
@@ -62,6 +65,15 @@
 #define KFD_MAX_NUM_OF_PROCESSES 512
 #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
 
+/*
+ * Size of the per-process TBA+TMA buffer: 2 pages
+ *
+ * The first page is the TBA used for the CWSR ISA code. The second
+ * page is used as TMA for daisy changing a user-mode trap handler.
+ */
+#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
+#define KFD_CWSR_TMA_OFFSET PAGE_SIZE
+
 /*
  * Kernel module parameter to specify maximum number of supported queues per
  * device
@@ -78,12 +90,26 @@ extern int max_num_of_queues_per_device;
 /* Kernel module parameter to specify the scheduling policy */
 extern int sched_policy;
 
+/*
+ * Kernel module parameter to specify the maximum process
+ * number per HW scheduler
+ */
+extern int hws_max_conc_proc;
+
+extern int cwsr_enable;
+
 /*
  * Kernel module parameter to specify whether to send sigterm to HSA process on
  * unhandled exception
  */
 extern int send_sigterm;
 
+/*
+ * Ignore CRAT table during KFD initialization, can be used to work around
+ * broken CRAT tables on some AMD systems
+ */
+extern int ignore_crat;
+
 /**
  * enum kfd_sched_policy
  *
@@ -131,6 +157,7 @@ struct kfd_device_info {
 	size_t ih_ring_entry_size;
 	uint8_t num_of_watch_points;
 	uint16_t mqd_size_aligned;
+	bool supports_cwsr;
 };
 
 struct kfd_mem_obj {
@@ -200,6 +227,14 @@ struct kfd_dev {
 
 	/* Debug manager */
 	struct kfd_dbgmgr           *dbgmgr;
+
+	/* Maximum process number mapped to HW scheduler */
+	unsigned int max_proc_per_quantum;
+
+	/* CWSR */
+	bool cwsr_enabled;
+	const void *cwsr_isa;
+	unsigned int cwsr_isa_size;
 };
 
 /* KGD2KFD callbacks */
@@ -332,6 +367,9 @@ struct queue_properties {
 	uint32_t eop_ring_buffer_size;
 	uint64_t ctx_save_restore_area_address;
 	uint32_t ctx_save_restore_area_size;
+	uint32_t ctl_stack_size;
+	uint64_t tba_addr;
+	uint64_t tma_addr;
 };
 
 /**
@@ -439,6 +477,11 @@ struct qcm_process_device {
 	uint32_t num_gws;
 	uint32_t num_oac;
 	uint32_t sh_hidden_private_base;
+
+	/* CWSR memory */
+	void *cwsr_kaddr;
+	uint64_t tba_addr;
+	uint64_t tma_addr;
 };
 
 
@@ -501,6 +544,9 @@ struct kfd_process {
 	 */
 	void *mm;
 
+	struct kref ref;
+	struct work_struct release_work;
+
 	struct mutex mutex;
 
 	/*
@@ -563,9 +609,10 @@ struct amdkfd_ioctl_desc {
 
 void kfd_process_create_wq(void);
 void kfd_process_destroy_wq(void);
-struct kfd_process *kfd_create_process(const struct task_struct *);
+struct kfd_process *kfd_create_process(struct file *filep);
 struct kfd_process *kfd_get_process(const struct task_struct *);
 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
+void kfd_unref_process(struct kfd_process *p);
 
 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
 						struct kfd_process *p);
@@ -577,6 +624,9 @@ struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 							struct kfd_process *p);
 
+int kfd_reserved_mem_mmap(struct kfd_process *process,
+			  struct vm_area_struct *vma);
+
 /* Process device data iterator */
 struct kfd_process_device *kfd_get_first_process_device_data(
 							struct kfd_process *p);
@@ -624,9 +674,12 @@ int kfd_topology_init(void);
 void kfd_topology_shutdown(void);
 int kfd_topology_add_device(struct kfd_dev *gpu);
 int kfd_topology_remove_device(struct kfd_dev *gpu);
+struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
+						uint32_t proximity_domain);
 struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
 struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
-struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
+int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
+int kfd_numa_node_to_apic_id(int numa_node_id);
 
 /* Interrupts */
 int kfd_interrupt_init(struct kfd_dev *dev);
@@ -643,8 +696,6 @@ int kgd2kfd_resume(struct kfd_dev *kfd);
 int kfd_init_apertures(struct kfd_process *process);
 
 /* Queue Context Management */
-struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd);
-
 int init_queue(struct queue **q, const struct queue_properties *properties);
 void uninit_queue(struct queue *q);
 void print_queue_properties(struct queue_properties *q);
@@ -699,6 +750,7 @@ struct packet_manager {
 	struct mutex lock;
 	bool allocated;
 	struct kfd_mem_obj *ib_buffer_obj;
+	unsigned int ib_size_bytes;
 };
 
 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
@@ -745,4 +797,23 @@ int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
 
 int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
 
+/* Debugfs */
+#if defined(CONFIG_DEBUG_FS)
+
+void kfd_debugfs_init(void);
+void kfd_debugfs_fini(void);
+int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
+int pqm_debugfs_mqds(struct seq_file *m, void *data);
+int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
+int dqm_debugfs_hqds(struct seq_file *m, void *data);
+int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
+int pm_debugfs_runlist(struct seq_file *m, void *data);
+
+#else
+
+static inline void kfd_debugfs_init(void) {}
+static inline void kfd_debugfs_fini(void) {}
+
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 1f5ccd28bd41..4ff5f0fe6db8 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -24,10 +24,12 @@
 #include <linux/log2.h>
 #include <linux/sched.h>
 #include <linux/sched/mm.h>
+#include <linux/sched/task.h>
 #include <linux/slab.h>
 #include <linux/amd-iommu.h>
 #include <linux/notifier.h>
 #include <linux/compat.h>
+#include <linux/mman.h>
 
 struct mm_struct;
 
@@ -46,13 +48,12 @@ DEFINE_STATIC_SRCU(kfd_processes_srcu);
 
 static struct workqueue_struct *kfd_process_wq;
 
-struct kfd_process_release_work {
-	struct work_struct kfd_work;
-	struct kfd_process *p;
-};
-
 static struct kfd_process *find_process(const struct task_struct *thread);
-static struct kfd_process *create_process(const struct task_struct *thread);
+static void kfd_process_ref_release(struct kref *ref);
+static struct kfd_process *create_process(const struct task_struct *thread,
+					struct file *filep);
+static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep);
+
 
 void kfd_process_create_wq(void)
 {
@@ -68,9 +69,10 @@ void kfd_process_destroy_wq(void)
 	}
 }
 
-struct kfd_process *kfd_create_process(const struct task_struct *thread)
+struct kfd_process *kfd_create_process(struct file *filep)
 {
 	struct kfd_process *process;
+	struct task_struct *thread = current;
 
 	if (!thread->mm)
 		return ERR_PTR(-EINVAL);
@@ -79,9 +81,6 @@ struct kfd_process *kfd_create_process(const struct task_struct *thread)
 	if (thread->group_leader->mm != thread->mm)
 		return ERR_PTR(-EINVAL);
 
-	/* Take mmap_sem because we call __mmu_notifier_register inside */
-	down_write(&thread->mm->mmap_sem);
-
 	/*
 	 * take kfd processes mutex before starting of process creation
 	 * so there won't be a case where two threads of the same process
@@ -93,14 +92,11 @@ struct kfd_process *kfd_create_process(const struct task_struct *thread)
 	process = find_process(thread);
 	if (process)
 		pr_debug("Process already found\n");
-
-	if (!process)
-		process = create_process(thread);
+	else
+		process = create_process(thread, filep);
 
 	mutex_unlock(&kfd_processes_mutex);
 
-	up_write(&thread->mm->mmap_sem);
-
 	return process;
 }
 
@@ -144,63 +140,75 @@ static struct kfd_process *find_process(const struct task_struct *thread)
 	return p;
 }
 
-static void kfd_process_wq_release(struct work_struct *work)
+void kfd_unref_process(struct kfd_process *p)
+{
+	kref_put(&p->ref, kfd_process_ref_release);
+}
+
+static void kfd_process_destroy_pdds(struct kfd_process *p)
 {
-	struct kfd_process_release_work *my_work;
 	struct kfd_process_device *pdd, *temp;
-	struct kfd_process *p;
 
-	my_work = (struct kfd_process_release_work *) work;
+	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
+				 per_device_list) {
+		pr_debug("Releasing pdd (topology id %d) for process (pasid %d)\n",
+				pdd->dev->id, p->pasid);
 
-	p = my_work->p;
+		list_del(&pdd->per_device_list);
 
-	pr_debug("Releasing process (pasid %d) in workqueue\n",
-			p->pasid);
+		if (pdd->qpd.cwsr_kaddr)
+			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
+				get_order(KFD_CWSR_TBA_TMA_SIZE));
 
-	mutex_lock(&p->mutex);
+		kfree(pdd);
+	}
+}
 
-	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
-							per_device_list) {
-		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
-				pdd->dev->id, p->pasid);
+/* No process locking is needed in this function, because the process
+ * is not findable any more. We must assume that no other thread is
+ * using it any more, otherwise we couldn't safely free the process
+ * structure in the end.
+ */
+static void kfd_process_wq_release(struct work_struct *work)
+{
+	struct kfd_process *p = container_of(work, struct kfd_process,
+					     release_work);
+	struct kfd_process_device *pdd;
 
+	pr_debug("Releasing process (pasid %d) in workqueue\n", p->pasid);
+
+	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 		if (pdd->bound == PDD_BOUND)
 			amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
-
-		list_del(&pdd->per_device_list);
-		kfree(pdd);
 	}
 
+	kfd_process_destroy_pdds(p);
+
 	kfd_event_free_process(p);
 
 	kfd_pasid_free(p->pasid);
 	kfd_free_process_doorbells(p);
 
-	mutex_unlock(&p->mutex);
-
 	mutex_destroy(&p->mutex);
 
-	kfree(p);
+	put_task_struct(p->lead_thread);
 
-	kfree(work);
+	kfree(p);
 }
 
-static void kfd_process_destroy_delayed(struct rcu_head *rcu)
+static void kfd_process_ref_release(struct kref *ref)
 {
-	struct kfd_process_release_work *work;
-	struct kfd_process *p;
-
-	p = container_of(rcu, struct kfd_process, rcu);
+	struct kfd_process *p = container_of(ref, struct kfd_process, ref);
 
-	mmdrop(p->mm);
+	INIT_WORK(&p->release_work, kfd_process_wq_release);
+	queue_work(kfd_process_wq, &p->release_work);
+}
 
-	work = kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
+static void kfd_process_destroy_delayed(struct rcu_head *rcu)
+{
+	struct kfd_process *p = container_of(rcu, struct kfd_process, rcu);
 
-	if (work) {
-		INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
-		work->p = p;
-		queue_work(kfd_process_wq, (struct work_struct *) work);
-	}
+	kfd_unref_process(p);
 }
 
 static void kfd_process_notifier_release(struct mmu_notifier *mn,
@@ -244,15 +252,12 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn,
 	kfd_process_dequeue_from_all_devices(p);
 	pqm_uninit(&p->pqm);
 
+	/* Indicate to other users that MM is no longer valid */
+	p->mm = NULL;
+
 	mutex_unlock(&p->mutex);
 
-	/*
-	 * Because we drop mm_count inside kfd_process_destroy_delayed
-	 * and because the mmu_notifier_unregister function also drop
-	 * mm_count we need to take an extra count here.
-	 */
-	mmgrab(p->mm);
-	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
+	mmu_notifier_unregister_no_release(&p->mmu_notifier, mm);
 	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
 }
 
@@ -260,7 +265,44 @@ static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
 	.release = kfd_process_notifier_release,
 };
 
-static struct kfd_process *create_process(const struct task_struct *thread)
+static int kfd_process_init_cwsr(struct kfd_process *p, struct file *filep)
+{
+	unsigned long  offset;
+	struct kfd_process_device *pdd = NULL;
+	struct kfd_dev *dev = NULL;
+	struct qcm_process_device *qpd = NULL;
+
+	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
+		dev = pdd->dev;
+		qpd = &pdd->qpd;
+		if (!dev->cwsr_enabled || qpd->cwsr_kaddr)
+			continue;
+		offset = (dev->id | KFD_MMAP_RESERVED_MEM_MASK) << PAGE_SHIFT;
+		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
+			KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
+			MAP_SHARED, offset);
+
+		if (IS_ERR_VALUE(qpd->tba_addr)) {
+			int err = qpd->tba_addr;
+
+			pr_err("Failure to set tba address. error %d.\n", err);
+			qpd->tba_addr = 0;
+			qpd->cwsr_kaddr = NULL;
+			return err;
+		}
+
+		memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
+
+		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
+		pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
+			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
+	}
+
+	return 0;
+}
+
+static struct kfd_process *create_process(const struct task_struct *thread,
+					struct file *filep)
 {
 	struct kfd_process *process;
 	int err = -ENOMEM;
@@ -277,13 +319,15 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 	if (kfd_alloc_process_doorbells(process) < 0)
 		goto err_alloc_doorbells;
 
+	kref_init(&process->ref);
+
 	mutex_init(&process->mutex);
 
 	process->mm = thread->mm;
 
 	/* register notifier */
 	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
-	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
+	err = mmu_notifier_register(&process->mmu_notifier, process->mm);
 	if (err)
 		goto err_mmu_notifier;
 
@@ -291,6 +335,7 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 			(uintptr_t)process->mm);
 
 	process->lead_thread = thread->group_leader;
+	get_task_struct(process->lead_thread);
 
 	INIT_LIST_HEAD(&process->per_device_data);
 
@@ -306,8 +351,14 @@ static struct kfd_process *create_process(const struct task_struct *thread)
 	if (err != 0)
 		goto err_init_apertures;
 
+	err = kfd_process_init_cwsr(process, filep);
+	if (err)
+		goto err_init_cwsr;
+
 	return process;
 
+err_init_cwsr:
+	kfd_process_destroy_pdds(process);
 err_init_apertures:
 	pqm_uninit(&process->pqm);
 err_process_pqm_init:
@@ -343,16 +394,18 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 	struct kfd_process_device *pdd = NULL;
 
 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
-	if (pdd != NULL) {
-		pdd->dev = dev;
-		INIT_LIST_HEAD(&pdd->qpd.queues_list);
-		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
-		pdd->qpd.dqm = dev->dqm;
-		pdd->process = p;
-		pdd->bound = PDD_UNBOUND;
-		pdd->already_dequeued = false;
-		list_add(&pdd->per_device_list, &p->per_device_data);
-	}
+	if (!pdd)
+		return NULL;
+
+	pdd->dev = dev;
+	INIT_LIST_HEAD(&pdd->qpd.queues_list);
+	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
+	pdd->qpd.dqm = dev->dqm;
+	pdd->qpd.pqm = &p->pqm;
+	pdd->process = p;
+	pdd->bound = PDD_UNBOUND;
+	pdd->already_dequeued = false;
+	list_add(&pdd->per_device_list, &p->per_device_data);
 
 	return pdd;
 }
@@ -408,7 +461,8 @@ int kfd_bind_processes_to_device(struct kfd_dev *dev)
 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
 		mutex_lock(&p->mutex);
 		pdd = kfd_get_process_device_data(dev, p);
-		if (pdd->bound != PDD_BOUND_SUSPENDED) {
+
+		if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
 			mutex_unlock(&p->mutex);
 			continue;
 		}
@@ -448,6 +502,11 @@ void kfd_unbind_processes_from_device(struct kfd_dev *dev)
 		mutex_lock(&p->mutex);
 		pdd = kfd_get_process_device_data(dev, p);
 
+		if (WARN_ON(!pdd)) {
+			mutex_unlock(&p->mutex);
+			continue;
+		}
+
 		if (pdd->bound == PDD_BOUND)
 			pdd->bound = PDD_BOUND_SUSPENDED;
 		mutex_unlock(&p->mutex);
@@ -483,6 +542,8 @@ void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid)
 
 	mutex_unlock(kfd_get_dbgmgr_mutex());
 
+	mutex_lock(&p->mutex);
+
 	pdd = kfd_get_process_device_data(dev, p);
 	if (pdd)
 		/* For GPU relying on IOMMU, we need to dequeue here
@@ -491,6 +552,8 @@ void kfd_process_iommu_unbind_callback(struct kfd_dev *dev, unsigned int pasid)
 		kfd_process_dequeue_from_device(pdd);
 
 	mutex_unlock(&p->mutex);
+
+	kfd_unref_process(p);
 }
 
 struct kfd_process_device *kfd_get_first_process_device_data(
@@ -515,22 +578,86 @@ bool kfd_has_process_device_data(struct kfd_process *p)
 	return !(list_empty(&p->per_device_data));
 }
 
-/* This returns with process->mutex locked. */
+/* This increments the process->ref counter. */
 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
 {
-	struct kfd_process *p;
+	struct kfd_process *p, *ret_p = NULL;
 	unsigned int temp;
 
 	int idx = srcu_read_lock(&kfd_processes_srcu);
 
 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
 		if (p->pasid == pasid) {
-			mutex_lock(&p->mutex);
+			kref_get(&p->ref);
+			ret_p = p;
 			break;
 		}
 	}
 
 	srcu_read_unlock(&kfd_processes_srcu, idx);
 
-	return p;
+	return ret_p;
+}
+
+int kfd_reserved_mem_mmap(struct kfd_process *process,
+			  struct vm_area_struct *vma)
+{
+	struct kfd_dev *dev = kfd_device_by_id(vma->vm_pgoff);
+	struct kfd_process_device *pdd;
+	struct qcm_process_device *qpd;
+
+	if (!dev)
+		return -EINVAL;
+	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
+		pr_err("Incorrect CWSR mapping size.\n");
+		return -EINVAL;
+	}
+
+	pdd = kfd_get_process_device_data(dev, process);
+	if (!pdd)
+		return -EINVAL;
+	qpd = &pdd->qpd;
+
+	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
+					get_order(KFD_CWSR_TBA_TMA_SIZE));
+	if (!qpd->cwsr_kaddr) {
+		pr_err("Error allocating per process CWSR buffer.\n");
+		return -ENOMEM;
+	}
+
+	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
+		| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
+	/* Mapping pages to user process */
+	return remap_pfn_range(vma, vma->vm_start,
+			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),
+			       KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
+}
+
+#if defined(CONFIG_DEBUG_FS)
+
+int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
+{
+	struct kfd_process *p;
+	unsigned int temp;
+	int r = 0;
+
+	int idx = srcu_read_lock(&kfd_processes_srcu);
+
+	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
+		seq_printf(m, "Process %d PASID %d:\n",
+			   p->lead_thread->tgid, p->pasid);
+
+		mutex_lock(&p->mutex);
+		r = pqm_debugfs_mqds(m, &p->pqm);
+		mutex_unlock(&p->mutex);
+
+		if (r)
+			break;
+	}
+
+	srcu_read_unlock(&kfd_processes_srcu, idx);
+
+	return r;
 }
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index a3f1e62c60ba..876380632668 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -178,10 +178,8 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 		return retval;
 
 	if (list_empty(&pdd->qpd.queues_list) &&
-	    list_empty(&pdd->qpd.priv_queue_list)) {
-		pdd->qpd.pqm = pqm;
+	    list_empty(&pdd->qpd.priv_queue_list))
 		dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
-	}
 
 	pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
 	if (!pqn) {
@@ -203,8 +201,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 			goto err_create_queue;
 		pqn->q = q;
 		pqn->kq = NULL;
-		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd,
-						&q->properties.vmid);
+		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
 		pr_debug("DQM returned %d for create_queue\n", retval);
 		print_queue(q);
 		break;
@@ -224,8 +221,7 @@ int pqm_create_queue(struct process_queue_manager *pqm,
 			goto err_create_queue;
 		pqn->q = q;
 		pqn->kq = NULL;
-		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd,
-						&q->properties.vmid);
+		retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
 		pr_debug("DQM returned %d for create_queue\n", retval);
 		print_queue(q);
 		break;
@@ -315,6 +311,10 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 	if (pqn->q) {
 		dqm = pqn->q->device->dqm;
 		retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
+		if (retval) {
+			pr_debug("Destroy queue failed, returned %d\n", retval);
+			goto err_destroy_queue;
+		}
 		uninit_queue(pqn->q);
 	}
 
@@ -326,6 +326,7 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
 	    list_empty(&pdd->qpd.priv_queue_list))
 		dqm->ops.unregister_process(dqm, &pdd->qpd);
 
+err_destroy_queue:
 	return retval;
 }
 
@@ -367,4 +368,67 @@ struct kernel_queue *pqm_get_kernel_queue(
 	return NULL;
 }
 
+#if defined(CONFIG_DEBUG_FS)
+
+int pqm_debugfs_mqds(struct seq_file *m, void *data)
+{
+	struct process_queue_manager *pqm = data;
+	struct process_queue_node *pqn;
+	struct queue *q;
+	enum KFD_MQD_TYPE mqd_type;
+	struct mqd_manager *mqd_manager;
+	int r = 0;
+
+	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
+		if (pqn->q) {
+			q = pqn->q;
+			switch (q->properties.type) {
+			case KFD_QUEUE_TYPE_SDMA:
+				seq_printf(m, "  SDMA queue on device %x\n",
+					   q->device->id);
+				mqd_type = KFD_MQD_TYPE_SDMA;
+				break;
+			case KFD_QUEUE_TYPE_COMPUTE:
+				seq_printf(m, "  Compute queue on device %x\n",
+					   q->device->id);
+				mqd_type = KFD_MQD_TYPE_CP;
+				break;
+			default:
+				seq_printf(m,
+				"  Bad user queue type %d on device %x\n",
+					   q->properties.type, q->device->id);
+				continue;
+			}
+			mqd_manager = q->device->dqm->ops.get_mqd_manager(
+				q->device->dqm, mqd_type);
+		} else if (pqn->kq) {
+			q = pqn->kq->queue;
+			mqd_manager = pqn->kq->mqd;
+			switch (q->properties.type) {
+			case KFD_QUEUE_TYPE_DIQ:
+				seq_printf(m, "  DIQ on device %x\n",
+					   pqn->kq->dev->id);
+				mqd_type = KFD_MQD_TYPE_HIQ;
+				break;
+			default:
+				seq_printf(m,
+				"  Bad kernel queue type %d on device %x\n",
+					   q->properties.type,
+					   pqn->kq->dev->id);
+				continue;
+			}
+		} else {
+			seq_printf(m,
+		"  Weird: Queue node with neither kernel nor user queue\n");
+			continue;
+		}
+
+		r = mqd_manager->debugfs_show_mqd(m, q->mqd);
+		if (r != 0)
+			break;
+	}
+
+	return r;
+}
 
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 19ce59028d6b..c6a76090a725 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -28,27 +28,32 @@
 #include <linux/hash.h>
 #include <linux/cpufreq.h>
 #include <linux/log2.h>
+#include <linux/dmi.h>
+#include <linux/atomic.h>
 
 #include "kfd_priv.h"
 #include "kfd_crat.h"
 #include "kfd_topology.h"
+#include "kfd_device_queue_manager.h"
 
+/* topology_device_list - Master list of all topology devices */
 static struct list_head topology_device_list;
-static int topology_crat_parsed;
 static struct kfd_system_properties sys_props;
 
 static DECLARE_RWSEM(topology_lock);
+static atomic_t topology_crat_proximity_domain;
 
-struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
+struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
+						uint32_t proximity_domain)
 {
 	struct kfd_topology_device *top_dev;
-	struct kfd_dev *device = NULL;
+	struct kfd_topology_device *device = NULL;
 
 	down_read(&topology_lock);
 
 	list_for_each_entry(top_dev, &topology_device_list, list)
-		if (top_dev->gpu_id == gpu_id) {
-			device = top_dev->gpu;
+		if (top_dev->proximity_domain == proximity_domain) {
+			device = top_dev;
 			break;
 		}
 
@@ -57,7 +62,7 @@ struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
 	return device;
 }
 
-struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
+struct kfd_dev *kfd_device_by_id(uint32_t gpu_id)
 {
 	struct kfd_topology_device *top_dev;
 	struct kfd_dev *device = NULL;
@@ -65,7 +70,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
 	down_read(&topology_lock);
 
 	list_for_each_entry(top_dev, &topology_device_list, list)
-		if (top_dev->gpu->pdev == pdev) {
+		if (top_dev->gpu_id == gpu_id) {
 			device = top_dev->gpu;
 			break;
 		}
@@ -75,282 +80,31 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
 	return device;
 }
 
-static int kfd_topology_get_crat_acpi(void *crat_image, size_t *size)
-{
-	struct acpi_table_header *crat_table;
-	acpi_status status;
-
-	if (!size)
-		return -EINVAL;
-
-	/*
-	 * Fetch the CRAT table from ACPI
-	 */
-	status = acpi_get_table(CRAT_SIGNATURE, 0, &crat_table);
-	if (status == AE_NOT_FOUND) {
-		pr_warn("CRAT table not found\n");
-		return -ENODATA;
-	} else if (ACPI_FAILURE(status)) {
-		const char *err = acpi_format_exception(status);
-
-		pr_err("CRAT table error: %s\n", err);
-		return -EINVAL;
-	}
-
-	if (*size >= crat_table->length && crat_image != NULL)
-		memcpy(crat_image, crat_table, crat_table->length);
-
-	*size = crat_table->length;
-
-	return 0;
-}
-
-static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
-		struct crat_subtype_computeunit *cu)
-{
-	dev->node_props.cpu_cores_count = cu->num_cpu_cores;
-	dev->node_props.cpu_core_id_base = cu->processor_id_low;
-	if (cu->hsa_capability & CRAT_CU_FLAGS_IOMMU_PRESENT)
-		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
-
-	pr_info("CU CPU: cores=%d id_base=%d\n", cu->num_cpu_cores,
-			cu->processor_id_low);
-}
-
-static void kfd_populated_cu_info_gpu(struct kfd_topology_device *dev,
-		struct crat_subtype_computeunit *cu)
-{
-	dev->node_props.simd_id_base = cu->processor_id_low;
-	dev->node_props.simd_count = cu->num_simd_cores;
-	dev->node_props.lds_size_in_kb = cu->lds_size_in_kb;
-	dev->node_props.max_waves_per_simd = cu->max_waves_simd;
-	dev->node_props.wave_front_size = cu->wave_front_size;
-	dev->node_props.mem_banks_count = cu->num_banks;
-	dev->node_props.array_count = cu->num_arrays;
-	dev->node_props.cu_per_simd_array = cu->num_cu_per_array;
-	dev->node_props.simd_per_cu = cu->num_simd_per_cu;
-	dev->node_props.max_slots_scratch_cu = cu->max_slots_scatch_cu;
-	if (cu->hsa_capability & CRAT_CU_FLAGS_HOT_PLUGGABLE)
-		dev->node_props.capability |= HSA_CAP_HOT_PLUGGABLE;
-	pr_info("CU GPU: simds=%d id_base=%d\n", cu->num_simd_cores,
-				cu->processor_id_low);
-}
-
-/* kfd_parse_subtype_cu is called when the topology mutex is already acquired */
-static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu)
-{
-	struct kfd_topology_device *dev;
-	int i = 0;
-
-	pr_info("Found CU entry in CRAT table with proximity_domain=%d caps=%x\n",
-			cu->proximity_domain, cu->hsa_capability);
-	list_for_each_entry(dev, &topology_device_list, list) {
-		if (cu->proximity_domain == i) {
-			if (cu->flags & CRAT_CU_FLAGS_CPU_PRESENT)
-				kfd_populated_cu_info_cpu(dev, cu);
-
-			if (cu->flags & CRAT_CU_FLAGS_GPU_PRESENT)
-				kfd_populated_cu_info_gpu(dev, cu);
-			break;
-		}
-		i++;
-	}
-
-	return 0;
-}
-
-/*
- * kfd_parse_subtype_mem is called when the topology mutex is
- * already acquired
- */
-static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem)
-{
-	struct kfd_mem_properties *props;
-	struct kfd_topology_device *dev;
-	int i = 0;
-
-	pr_info("Found memory entry in CRAT table with proximity_domain=%d\n",
-			mem->promixity_domain);
-	list_for_each_entry(dev, &topology_device_list, list) {
-		if (mem->promixity_domain == i) {
-			props = kfd_alloc_struct(props);
-			if (props == NULL)
-				return -ENOMEM;
-
-			if (dev->node_props.cpu_cores_count == 0)
-				props->heap_type = HSA_MEM_HEAP_TYPE_FB_PRIVATE;
-			else
-				props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
-
-			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
-				props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
-			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
-				props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
-
-			props->size_in_bytes =
-				((uint64_t)mem->length_high << 32) +
-							mem->length_low;
-			props->width = mem->width;
-
-			dev->mem_bank_count++;
-			list_add_tail(&props->list, &dev->mem_props);
-
-			break;
-		}
-		i++;
-	}
-
-	return 0;
-}
-
-/*
- * kfd_parse_subtype_cache is called when the topology mutex
- * is already acquired
- */
-static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache)
-{
-	struct kfd_cache_properties *props;
-	struct kfd_topology_device *dev;
-	uint32_t id;
-
-	id = cache->processor_id_low;
-
-	pr_info("Found cache entry in CRAT table with processor_id=%d\n", id);
-	list_for_each_entry(dev, &topology_device_list, list)
-		if (id == dev->node_props.cpu_core_id_base ||
-		    id == dev->node_props.simd_id_base) {
-			props = kfd_alloc_struct(props);
-			if (props == NULL)
-				return -ENOMEM;
-
-			props->processor_id_low = id;
-			props->cache_level = cache->cache_level;
-			props->cache_size = cache->cache_size;
-			props->cacheline_size = cache->cache_line_size;
-			props->cachelines_per_tag = cache->lines_per_tag;
-			props->cache_assoc = cache->associativity;
-			props->cache_latency = cache->cache_latency;
-
-			if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_DATA;
-			if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_INSTRUCTION;
-			if (cache->flags & CRAT_CACHE_FLAGS_CPU_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_CPU;
-			if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE)
-				props->cache_type |= HSA_CACHE_TYPE_HSACU;
-
-			dev->cache_count++;
-			dev->node_props.caches_count++;
-			list_add_tail(&props->list, &dev->cache_props);
-
-			break;
-		}
-
-	return 0;
-}
-
-/*
- * kfd_parse_subtype_iolink is called when the topology mutex
- * is already acquired
- */
-static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink)
+struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev)
 {
-	struct kfd_iolink_properties *props;
-	struct kfd_topology_device *dev;
-	uint32_t i = 0;
-	uint32_t id_from;
-	uint32_t id_to;
-
-	id_from = iolink->proximity_domain_from;
-	id_to = iolink->proximity_domain_to;
+	struct kfd_topology_device *top_dev;
+	struct kfd_dev *device = NULL;
 
-	pr_info("Found IO link entry in CRAT table with id_from=%d\n", id_from);
-	list_for_each_entry(dev, &topology_device_list, list) {
-		if (id_from == i) {
-			props = kfd_alloc_struct(props);
-			if (props == NULL)
-				return -ENOMEM;
-
-			props->node_from = id_from;
-			props->node_to = id_to;
-			props->ver_maj = iolink->version_major;
-			props->ver_min = iolink->version_minor;
-
-			/*
-			 * weight factor (derived from CDIR), currently always 1
-			 */
-			props->weight = 1;
-
-			props->min_latency = iolink->minimum_latency;
-			props->max_latency = iolink->maximum_latency;
-			props->min_bandwidth = iolink->minimum_bandwidth_mbs;
-			props->max_bandwidth = iolink->maximum_bandwidth_mbs;
-			props->rec_transfer_size =
-					iolink->recommended_transfer_size;
-
-			dev->io_link_count++;
-			dev->node_props.io_links_count++;
-			list_add_tail(&props->list, &dev->io_link_props);
+	down_read(&topology_lock);
 
+	list_for_each_entry(top_dev, &topology_device_list, list)
+		if (top_dev->gpu->pdev == pdev) {
+			device = top_dev->gpu;
 			break;
 		}
-		i++;
-	}
 
-	return 0;
-}
-
-static int kfd_parse_subtype(struct crat_subtype_generic *sub_type_hdr)
-{
-	struct crat_subtype_computeunit *cu;
-	struct crat_subtype_memory *mem;
-	struct crat_subtype_cache *cache;
-	struct crat_subtype_iolink *iolink;
-	int ret = 0;
-
-	switch (sub_type_hdr->type) {
-	case CRAT_SUBTYPE_COMPUTEUNIT_AFFINITY:
-		cu = (struct crat_subtype_computeunit *)sub_type_hdr;
-		ret = kfd_parse_subtype_cu(cu);
-		break;
-	case CRAT_SUBTYPE_MEMORY_AFFINITY:
-		mem = (struct crat_subtype_memory *)sub_type_hdr;
-		ret = kfd_parse_subtype_mem(mem);
-		break;
-	case CRAT_SUBTYPE_CACHE_AFFINITY:
-		cache = (struct crat_subtype_cache *)sub_type_hdr;
-		ret = kfd_parse_subtype_cache(cache);
-		break;
-	case CRAT_SUBTYPE_TLB_AFFINITY:
-		/*
-		 * For now, nothing to do here
-		 */
-		pr_info("Found TLB entry in CRAT table (not processing)\n");
-		break;
-	case CRAT_SUBTYPE_CCOMPUTE_AFFINITY:
-		/*
-		 * For now, nothing to do here
-		 */
-		pr_info("Found CCOMPUTE entry in CRAT table (not processing)\n");
-		break;
-	case CRAT_SUBTYPE_IOLINK_AFFINITY:
-		iolink = (struct crat_subtype_iolink *)sub_type_hdr;
-		ret = kfd_parse_subtype_iolink(iolink);
-		break;
-	default:
-		pr_warn("Unknown subtype (%d) in CRAT\n",
-				sub_type_hdr->type);
-	}
+	up_read(&topology_lock);
 
-	return ret;
+	return device;
 }
 
+/* Called with write topology_lock acquired */
 static void kfd_release_topology_device(struct kfd_topology_device *dev)
 {
 	struct kfd_mem_properties *mem;
 	struct kfd_cache_properties *cache;
 	struct kfd_iolink_properties *iolink;
+	struct kfd_perf_properties *perf;
 
 	list_del(&dev->list);
 
@@ -375,25 +129,35 @@ static void kfd_release_topology_device(struct kfd_topology_device *dev)
 		kfree(iolink);
 	}
 
-	kfree(dev);
+	while (dev->perf_props.next != &dev->perf_props) {
+		perf = container_of(dev->perf_props.next,
+				struct kfd_perf_properties, list);
+		list_del(&perf->list);
+		kfree(perf);
+	}
 
-	sys_props.num_devices--;
+	kfree(dev);
 }
 
-static void kfd_release_live_view(void)
+void kfd_release_topology_device_list(struct list_head *device_list)
 {
 	struct kfd_topology_device *dev;
 
-	while (topology_device_list.next != &topology_device_list) {
-		dev = container_of(topology_device_list.next,
-				 struct kfd_topology_device, list);
+	while (!list_empty(device_list)) {
+		dev = list_first_entry(device_list,
+				       struct kfd_topology_device, list);
 		kfd_release_topology_device(dev);
+	}
 }
 
+static void kfd_release_live_view(void)
+{
+	kfd_release_topology_device_list(&topology_device_list);
 	memset(&sys_props, 0, sizeof(sys_props));
 }
 
-static struct kfd_topology_device *kfd_create_topology_device(void)
+struct kfd_topology_device *kfd_create_topology_device(
+				struct list_head *device_list)
 {
 	struct kfd_topology_device *dev;
 
@@ -406,65 +170,13 @@ static struct kfd_topology_device *kfd_create_topology_device(void)
 	INIT_LIST_HEAD(&dev->mem_props);
 	INIT_LIST_HEAD(&dev->cache_props);
 	INIT_LIST_HEAD(&dev->io_link_props);
+	INIT_LIST_HEAD(&dev->perf_props);
 
-	list_add_tail(&dev->list, &topology_device_list);
-	sys_props.num_devices++;
+	list_add_tail(&dev->list, device_list);
 
 	return dev;
 }
 
-static int kfd_parse_crat_table(void *crat_image)
-{
-	struct kfd_topology_device *top_dev;
-	struct crat_subtype_generic *sub_type_hdr;
-	uint16_t node_id;
-	int ret;
-	struct crat_header *crat_table = (struct crat_header *)crat_image;
-	uint16_t num_nodes;
-	uint32_t image_len;
-
-	if (!crat_image)
-		return -EINVAL;
-
-	num_nodes = crat_table->num_domains;
-	image_len = crat_table->length;
-
-	pr_info("Parsing CRAT table with %d nodes\n", num_nodes);
-
-	for (node_id = 0; node_id < num_nodes; node_id++) {
-		top_dev = kfd_create_topology_device();
-		if (!top_dev) {
-			kfd_release_live_view();
-			return -ENOMEM;
-		}
-	}
-
-	sys_props.platform_id =
-		(*((uint64_t *)crat_table->oem_id)) & CRAT_OEMID_64BIT_MASK;
-	sys_props.platform_oem = *((uint64_t *)crat_table->oem_table_id);
-	sys_props.platform_rev = crat_table->revision;
-
-	sub_type_hdr = (struct crat_subtype_generic *)(crat_table+1);
-	while ((char *)sub_type_hdr + sizeof(struct crat_subtype_generic) <
-			((char *)crat_image) + image_len) {
-		if (sub_type_hdr->flags & CRAT_SUBTYPE_FLAGS_ENABLED) {
-			ret = kfd_parse_subtype(sub_type_hdr);
-			if (ret != 0) {
-				kfd_release_live_view();
-				return ret;
-			}
-		}
-
-		sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
-				sub_type_hdr->length);
-	}
-
-	sys_props.generation_count++;
-	topology_crat_parsed = 1;
-
-	return 0;
-}
-
 
 #define sysfs_show_gen_prop(buffer, fmt, ...) \
 		snprintf(buffer, PAGE_SIZE, "%s"fmt, buffer, __VA_ARGS__)
@@ -501,11 +213,17 @@ static ssize_t sysprops_show(struct kobject *kobj, struct attribute *attr,
 	return ret;
 }
 
+static void kfd_topology_kobj_release(struct kobject *kobj)
+{
+	kfree(kobj);
+}
+
 static const struct sysfs_ops sysprops_ops = {
 	.show = sysprops_show,
 };
 
 static struct kobj_type sysprops_type = {
+	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &sysprops_ops,
 };
 
@@ -541,6 +259,7 @@ static const struct sysfs_ops iolink_ops = {
 };
 
 static struct kobj_type iolink_type = {
+	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &iolink_ops,
 };
 
@@ -568,6 +287,7 @@ static const struct sysfs_ops mem_ops = {
 };
 
 static struct kobj_type mem_type = {
+	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &mem_ops,
 };
 
@@ -575,7 +295,7 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 		char *buffer)
 {
 	ssize_t ret;
-	uint32_t i;
+	uint32_t i, j;
 	struct kfd_cache_properties *cache;
 
 	/* Making sure that the buffer is an empty string */
@@ -593,12 +313,18 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr,
 	sysfs_show_32bit_prop(buffer, "latency", cache->cache_latency);
 	sysfs_show_32bit_prop(buffer, "type", cache->cache_type);
 	snprintf(buffer, PAGE_SIZE, "%ssibling_map ", buffer);
-	for (i = 0; i < KFD_TOPOLOGY_CPU_SIBLINGS; i++)
-		ret = snprintf(buffer, PAGE_SIZE, "%s%d%s",
-				buffer, cache->sibling_map[i],
-				(i == KFD_TOPOLOGY_CPU_SIBLINGS-1) ?
-						"\n" : ",");
-
+	for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++)
+		for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) {
+			/* Check each bit */
+			if (cache->sibling_map[i] & (1 << j))
+				ret = snprintf(buffer, PAGE_SIZE,
+					 "%s%d%s", buffer, 1, ",");
+			else
+				ret = snprintf(buffer, PAGE_SIZE,
+					 "%s%d%s", buffer, 0, ",");
+		}
+	/* Replace the last "," with end of line */
+	*(buffer + strlen(buffer) - 1) = 0xA;
 	return ret;
 }
 
@@ -607,9 +333,43 @@ static const struct sysfs_ops cache_ops = {
 };
 
 static struct kobj_type cache_type = {
+	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &cache_ops,
 };
 
+/****** Sysfs of Performance Counters ******/
+
+struct kfd_perf_attr {
+	struct kobj_attribute attr;
+	uint32_t data;
+};
+
+static ssize_t perf_show(struct kobject *kobj, struct kobj_attribute *attrs,
+			char *buf)
+{
+	struct kfd_perf_attr *attr;
+
+	buf[0] = 0;
+	attr = container_of(attrs, struct kfd_perf_attr, attr);
+	if (!attr->data) /* invalid data for PMC */
+		return 0;
+	else
+		return sysfs_show_32bit_val(buf, attr->data);
+}
+
+#define KFD_PERF_DESC(_name, _data)			\
+{							\
+	.attr  = __ATTR(_name, 0444, perf_show, NULL),	\
+	.data = _data,					\
+}
+
+static struct kfd_perf_attr perf_attr_iommu[] = {
+	KFD_PERF_DESC(max_concurrent, 0),
+	KFD_PERF_DESC(num_counters, 0),
+	KFD_PERF_DESC(counter_ids, 0),
+};
+/****************************************/
+
 static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 		char *buffer)
 {
@@ -646,18 +406,8 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 			dev->node_props.cpu_cores_count);
 	sysfs_show_32bit_prop(buffer, "simd_count",
 			dev->node_props.simd_count);
-
-	if (dev->mem_bank_count < dev->node_props.mem_banks_count) {
-		pr_info_once("mem_banks_count truncated from %d to %d\n",
-				dev->node_props.mem_banks_count,
-				dev->mem_bank_count);
-		sysfs_show_32bit_prop(buffer, "mem_banks_count",
-				dev->mem_bank_count);
-	} else {
-		sysfs_show_32bit_prop(buffer, "mem_banks_count",
-				dev->node_props.mem_banks_count);
-	}
-
+	sysfs_show_32bit_prop(buffer, "mem_banks_count",
+			dev->node_props.mem_banks_count);
 	sysfs_show_32bit_prop(buffer, "caches_count",
 			dev->node_props.caches_count);
 	sysfs_show_32bit_prop(buffer, "io_links_count",
@@ -705,9 +455,12 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 				HSA_CAP_WATCH_POINTS_TOTALBITS_MASK);
 		}
 
+		if (dev->gpu->device_info->asic_family == CHIP_TONGA)
+			dev->node_props.capability |=
+					HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
+
 		sysfs_show_32bit_prop(buffer, "max_engine_clk_fcompute",
-			dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(
-					dev->gpu->kgd));
+			dev->node_props.max_engine_clk_fcompute);
 
 		sysfs_show_64bit_prop(buffer, "local_mem_size",
 				(unsigned long long int) 0);
@@ -729,6 +482,7 @@ static const struct sysfs_ops node_ops = {
 };
 
 static struct kobj_type node_type = {
+	.release = kfd_topology_kobj_release,
 	.sysfs_ops = &node_ops,
 };
 
@@ -744,6 +498,7 @@ static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
 	struct kfd_iolink_properties *iolink;
 	struct kfd_cache_properties *cache;
 	struct kfd_mem_properties *mem;
+	struct kfd_perf_properties *perf;
 
 	if (dev->kobj_iolink) {
 		list_for_each_entry(iolink, &dev->io_link_props, list)
@@ -780,6 +535,16 @@ static void kfd_remove_sysfs_node_entry(struct kfd_topology_device *dev)
 		dev->kobj_mem = NULL;
 	}
 
+	if (dev->kobj_perf) {
+		list_for_each_entry(perf, &dev->perf_props, list) {
+			kfree(perf->attr_group);
+			perf->attr_group = NULL;
+		}
+		kobject_del(dev->kobj_perf);
+		kobject_put(dev->kobj_perf);
+		dev->kobj_perf = NULL;
+	}
+
 	if (dev->kobj_node) {
 		sysfs_remove_file(dev->kobj_node, &dev->attr_gpuid);
 		sysfs_remove_file(dev->kobj_node, &dev->attr_name);
@@ -796,8 +561,10 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 	struct kfd_iolink_properties *iolink;
 	struct kfd_cache_properties *cache;
 	struct kfd_mem_properties *mem;
+	struct kfd_perf_properties *perf;
 	int ret;
-	uint32_t i;
+	uint32_t i, num_attrs;
+	struct attribute **attrs;
 
 	if (WARN_ON(dev->kobj_node))
 		return -EEXIST;
@@ -826,6 +593,10 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 	if (!dev->kobj_iolink)
 		return -ENOMEM;
 
+	dev->kobj_perf = kobject_create_and_add("perf", dev->kobj_node);
+	if (!dev->kobj_perf)
+		return -ENOMEM;
+
 	/*
 	 * Creating sysfs files for node properties
 	 */
@@ -903,11 +674,38 @@ static int kfd_build_sysfs_node_entry(struct kfd_topology_device *dev,
 		if (ret < 0)
 			return ret;
 		i++;
-}
+	}
+
+	/* All hardware blocks have the same number of attributes. */
+	num_attrs = sizeof(perf_attr_iommu)/sizeof(struct kfd_perf_attr);
+	list_for_each_entry(perf, &dev->perf_props, list) {
+		perf->attr_group = kzalloc(sizeof(struct kfd_perf_attr)
+			* num_attrs + sizeof(struct attribute_group),
+			GFP_KERNEL);
+		if (!perf->attr_group)
+			return -ENOMEM;
+
+		attrs = (struct attribute **)(perf->attr_group + 1);
+		if (!strcmp(perf->block_name, "iommu")) {
+		/* Information of IOMMU's num_counters and counter_ids is shown
+		 * under /sys/bus/event_source/devices/amd_iommu. We don't
+		 * duplicate here.
+		 */
+			perf_attr_iommu[0].data = perf->max_concurrent;
+			for (i = 0; i < num_attrs; i++)
+				attrs[i] = &perf_attr_iommu[i].attr.attr;
+		}
+		perf->attr_group->name = perf->block_name;
+		perf->attr_group->attrs = attrs;
+		ret = sysfs_create_group(dev->kobj_perf, perf->attr_group);
+		if (ret < 0)
+			return ret;
+	}
 
 	return 0;
 }
 
+/* Called with write topology lock acquired */
 static int kfd_build_sysfs_node_tree(void)
 {
 	struct kfd_topology_device *dev;
@@ -924,6 +722,7 @@ static int kfd_build_sysfs_node_tree(void)
 	return 0;
 }
 
+/* Called with write topology lock acquired */
 static void kfd_remove_sysfs_node_tree(void)
 {
 	struct kfd_topology_device *dev;
@@ -995,75 +794,246 @@ static void kfd_topology_release_sysfs(void)
 	}
 }
 
+/* Called with write topology_lock acquired */
+static void kfd_topology_update_device_list(struct list_head *temp_list,
+					struct list_head *master_list)
+{
+	while (!list_empty(temp_list)) {
+		list_move_tail(temp_list->next, master_list);
+		sys_props.num_devices++;
+	}
+}
+
+static void kfd_debug_print_topology(void)
+{
+	struct kfd_topology_device *dev;
+
+	down_read(&topology_lock);
+
+	dev = list_last_entry(&topology_device_list,
+			struct kfd_topology_device, list);
+	if (dev) {
+		if (dev->node_props.cpu_cores_count &&
+				dev->node_props.simd_count) {
+			pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
+				dev->node_props.device_id,
+				dev->node_props.vendor_id);
+		} else if (dev->node_props.cpu_cores_count)
+			pr_info("Topology: Add CPU node\n");
+		else if (dev->node_props.simd_count)
+			pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
+				dev->node_props.device_id,
+				dev->node_props.vendor_id);
+	}
+	up_read(&topology_lock);
+}
+
+/* Helper function for intializing platform_xx members of
+ * kfd_system_properties. Uses OEM info from the last CPU/APU node.
+ */
+static void kfd_update_system_properties(void)
+{
+	struct kfd_topology_device *dev;
+
+	down_read(&topology_lock);
+	dev = list_last_entry(&topology_device_list,
+			struct kfd_topology_device, list);
+	if (dev) {
+		sys_props.platform_id =
+			(*((uint64_t *)dev->oem_id)) & CRAT_OEMID_64BIT_MASK;
+		sys_props.platform_oem = *((uint64_t *)dev->oem_table_id);
+		sys_props.platform_rev = dev->oem_revision;
+	}
+	up_read(&topology_lock);
+}
+
+static void find_system_memory(const struct dmi_header *dm,
+	void *private)
+{
+	struct kfd_mem_properties *mem;
+	u16 mem_width, mem_clock;
+	struct kfd_topology_device *kdev =
+		(struct kfd_topology_device *)private;
+	const u8 *dmi_data = (const u8 *)(dm + 1);
+
+	if (dm->type == DMI_ENTRY_MEM_DEVICE && dm->length >= 0x15) {
+		mem_width = (u16)(*(const u16 *)(dmi_data + 0x6));
+		mem_clock = (u16)(*(const u16 *)(dmi_data + 0x11));
+		list_for_each_entry(mem, &kdev->mem_props, list) {
+			if (mem_width != 0xFFFF && mem_width != 0)
+				mem->width = mem_width;
+			if (mem_clock != 0)
+				mem->mem_clk_max = mem_clock;
+		}
+	}
+}
+
+/*
+ * Performance counters information is not part of CRAT but we would like to
+ * put them in the sysfs under topology directory for Thunk to get the data.
+ * This function is called before updating the sysfs.
+ */
+static int kfd_add_perf_to_topology(struct kfd_topology_device *kdev)
+{
+	struct kfd_perf_properties *props;
+
+	if (amd_iommu_pc_supported()) {
+		props = kfd_alloc_struct(props);
+		if (!props)
+			return -ENOMEM;
+		strcpy(props->block_name, "iommu");
+		props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
+			amd_iommu_pc_get_max_counters(0); /* assume one iommu */
+		list_add_tail(&props->list, &kdev->perf_props);
+	}
+
+	return 0;
+}
+
+/* kfd_add_non_crat_information - Add information that is not currently
+ *	defined in CRAT but is necessary for KFD topology
+ * @dev - topology device to which addition info is added
+ */
+static void kfd_add_non_crat_information(struct kfd_topology_device *kdev)
+{
+	/* Check if CPU only node. */
+	if (!kdev->gpu) {
+		/* Add system memory information */
+		dmi_walk(find_system_memory, kdev);
+	}
+	/* TODO: For GPU node, rearrange code from kfd_topology_add_device */
+}
+
+/* kfd_is_acpi_crat_invalid - CRAT from ACPI is valid only for AMD APU devices.
+ *	Ignore CRAT for all other devices. AMD APU is identified if both CPU
+ *	and GPU cores are present.
+ * @device_list - topology device list created by parsing ACPI CRAT table.
+ * @return - TRUE if invalid, FALSE is valid.
+ */
+static bool kfd_is_acpi_crat_invalid(struct list_head *device_list)
+{
+	struct kfd_topology_device *dev;
+
+	list_for_each_entry(dev, device_list, list) {
+		if (dev->node_props.cpu_cores_count &&
+			dev->node_props.simd_count)
+			return false;
+	}
+	pr_info("Ignoring ACPI CRAT on non-APU system\n");
+	return true;
+}
+
 int kfd_topology_init(void)
 {
 	void *crat_image = NULL;
 	size_t image_size = 0;
 	int ret;
-
-	/*
-	 * Initialize the head for the topology device list
+	struct list_head temp_topology_device_list;
+	int cpu_only_node = 0;
+	struct kfd_topology_device *kdev;
+	int proximity_domain;
+
+	/* topology_device_list - Master list of all topology devices
+	 * temp_topology_device_list - temporary list created while parsing CRAT
+	 * or VCRAT. Once parsing is complete the contents of list is moved to
+	 * topology_device_list
 	 */
+
+	/* Initialize the head for the both the lists */
 	INIT_LIST_HEAD(&topology_device_list);
+	INIT_LIST_HEAD(&temp_topology_device_list);
 	init_rwsem(&topology_lock);
-	topology_crat_parsed = 0;
 
 	memset(&sys_props, 0, sizeof(sys_props));
 
+	/* Proximity domains in ACPI CRAT tables start counting at
+	 * 0. The same should be true for virtual CRAT tables created
+	 * at this stage. GPUs added later in kfd_topology_add_device
+	 * use a counter.
+	 */
+	proximity_domain = 0;
+
 	/*
-	 * Get the CRAT image from the ACPI
+	 * Get the CRAT image from the ACPI. If ACPI doesn't have one
+	 * or if ACPI CRAT is invalid create a virtual CRAT.
+	 * NOTE: The current implementation expects all AMD APUs to have
+	 *	CRAT. If no CRAT is available, it is assumed to be a CPU
 	 */
-	ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
-	if (ret == 0 && image_size > 0) {
-		pr_info("Found CRAT image with size=%zd\n", image_size);
-		crat_image = kmalloc(image_size, GFP_KERNEL);
-		if (!crat_image) {
-			ret = -ENOMEM;
-			pr_err("No memory for allocating CRAT image\n");
-			goto err;
+	ret = kfd_create_crat_image_acpi(&crat_image, &image_size);
+	if (!ret) {
+		ret = kfd_parse_crat_table(crat_image,
+					   &temp_topology_device_list,
+					   proximity_domain);
+		if (ret ||
+		    kfd_is_acpi_crat_invalid(&temp_topology_device_list)) {
+			kfd_release_topology_device_list(
+				&temp_topology_device_list);
+			kfd_destroy_crat_image(crat_image);
+			crat_image = NULL;
 		}
-		ret = kfd_topology_get_crat_acpi(crat_image, &image_size);
-
-		if (ret == 0) {
-			down_write(&topology_lock);
-			ret = kfd_parse_crat_table(crat_image);
-			if (ret == 0)
-				ret = kfd_topology_update_sysfs();
-			up_write(&topology_lock);
-		} else {
-			pr_err("Couldn't get CRAT table size from ACPI\n");
+	}
+
+	if (!crat_image) {
+		ret = kfd_create_crat_image_virtual(&crat_image, &image_size,
+						    COMPUTE_UNIT_CPU, NULL,
+						    proximity_domain);
+		cpu_only_node = 1;
+		if (ret) {
+			pr_err("Error creating VCRAT table for CPU\n");
+			return ret;
 		}
-		kfree(crat_image);
-	} else if (ret == -ENODATA) {
-		ret = 0;
-	} else {
-		pr_err("Couldn't get CRAT table size from ACPI\n");
+
+		ret = kfd_parse_crat_table(crat_image,
+					   &temp_topology_device_list,
+					   proximity_domain);
+		if (ret) {
+			pr_err("Error parsing VCRAT table for CPU\n");
+			goto err;
+		}
+	}
+
+	kdev = list_first_entry(&temp_topology_device_list,
+				struct kfd_topology_device, list);
+	kfd_add_perf_to_topology(kdev);
+
+	down_write(&topology_lock);
+	kfd_topology_update_device_list(&temp_topology_device_list,
+					&topology_device_list);
+	atomic_set(&topology_crat_proximity_domain, sys_props.num_devices-1);
+	ret = kfd_topology_update_sysfs();
+	up_write(&topology_lock);
+
+	if (!ret) {
+		sys_props.generation_count++;
+		kfd_update_system_properties();
+		kfd_debug_print_topology();
+		pr_info("Finished initializing topology\n");
+	} else
+		pr_err("Failed to update topology in sysfs ret=%d\n", ret);
+
+	/* For nodes with GPU, this information gets added
+	 * when GPU is detected (kfd_topology_add_device).
+	 */
+	if (cpu_only_node) {
+		/* Add additional information to CPU only node created above */
+		down_write(&topology_lock);
+		kdev = list_first_entry(&topology_device_list,
+				struct kfd_topology_device, list);
+		up_write(&topology_lock);
+		kfd_add_non_crat_information(kdev);
 	}
 
 err:
-	pr_info("Finished initializing topology ret=%d\n", ret);
+	kfd_destroy_crat_image(crat_image);
 	return ret;
 }
 
 void kfd_topology_shutdown(void)
 {
+	down_write(&topology_lock);
 	kfd_topology_release_sysfs();
 	kfd_release_live_view();
-}
-
-static void kfd_debug_print_topology(void)
-{
-	struct kfd_topology_device *dev;
-	uint32_t i = 0;
-
-	pr_info("DEBUG PRINT OF TOPOLOGY:");
-	list_for_each_entry(dev, &topology_device_list, list) {
-		pr_info("Node: %d\n", i);
-		pr_info("\tGPU assigned: %s\n", (dev->gpu ? "yes" : "no"));
-		pr_info("\tCPU count: %d\n", dev->node_props.cpu_cores_count);
-		pr_info("\tSIMD count: %d", dev->node_props.simd_count);
-		i++;
-	}
+	up_write(&topology_lock);
 }
 
 static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
@@ -1072,11 +1042,15 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
 	uint32_t buf[7];
 	uint64_t local_mem_size;
 	int i;
+	struct kfd_local_mem_info local_mem_info;
 
 	if (!gpu)
 		return 0;
 
-	local_mem_size = gpu->kfd2kgd->get_vmem_size(gpu->kgd);
+	gpu->kfd2kgd->get_local_mem_info(gpu->kgd, &local_mem_info);
+
+	local_mem_size = local_mem_info.local_mem_size_private +
+			local_mem_info.local_mem_size_public;
 
 	buf[0] = gpu->pdev->devfn;
 	buf[1] = gpu->pdev->subsystem_vendor;
@@ -1091,19 +1065,26 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu)
 
 	return hashout;
 }
-
+/* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
+ *		the GPU device is not already present in the topology device
+ *		list then return NULL. This means a new topology device has to
+ *		be created for this GPU.
+ * TODO: Rather than assiging @gpu to first topology device withtout
+ *		gpu attached, it will better to have more stringent check.
+ */
 static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu)
 {
 	struct kfd_topology_device *dev;
 	struct kfd_topology_device *out_dev = NULL;
 
+	down_write(&topology_lock);
 	list_for_each_entry(dev, &topology_device_list, list)
 		if (!dev->gpu && (dev->node_props.simd_count > 0)) {
 			dev->gpu = gpu;
 			out_dev = dev;
 			break;
 		}
-
+	up_write(&topology_lock);
 	return out_dev;
 }
 
@@ -1115,84 +1096,196 @@ static void kfd_notify_gpu_change(uint32_t gpu_id, int arrival)
 	 */
 }
 
+/* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
+ *		patch this after CRAT parsing.
+ */
+static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
+{
+	struct kfd_mem_properties *mem;
+	struct kfd_local_mem_info local_mem_info;
+
+	if (!dev)
+		return;
+
+	/* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
+	 * single bank of VRAM local memory.
+	 * for dGPUs - VCRAT reports only one bank of Local Memory
+	 * for APUs - If CRAT from ACPI reports more than one bank, then
+	 *	all the banks will report the same mem_clk_max information
+	 */
+	dev->gpu->kfd2kgd->get_local_mem_info(dev->gpu->kgd,
+		&local_mem_info);
+
+	list_for_each_entry(mem, &dev->mem_props, list)
+		mem->mem_clk_max = local_mem_info.mem_clk_max;
+}
+
+static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
+{
+	struct kfd_iolink_properties *link;
+
+	if (!dev || !dev->gpu)
+		return;
+
+	/* GPU only creates direck links so apply flags setting to all */
+	if (dev->gpu->device_info->asic_family == CHIP_HAWAII)
+		list_for_each_entry(link, &dev->io_link_props, list)
+			link->flags = CRAT_IOLINK_FLAGS_ENABLED |
+				CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
+				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
+}
+
 int kfd_topology_add_device(struct kfd_dev *gpu)
 {
 	uint32_t gpu_id;
 	struct kfd_topology_device *dev;
-	int res;
+	struct kfd_cu_info cu_info;
+	int res = 0;
+	struct list_head temp_topology_device_list;
+	void *crat_image = NULL;
+	size_t image_size = 0;
+	int proximity_domain;
+
+	INIT_LIST_HEAD(&temp_topology_device_list);
 
 	gpu_id = kfd_generate_gpu_id(gpu);
 
 	pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id);
 
-	down_write(&topology_lock);
-	/*
-	 * Try to assign the GPU to existing topology device (generated from
-	 * CRAT table
+	proximity_domain = atomic_inc_return(&topology_crat_proximity_domain);
+
+	/* Check to see if this gpu device exists in the topology_device_list.
+	 * If so, assign the gpu to that device,
+	 * else create a Virtual CRAT for this gpu device and then parse that
+	 * CRAT to create a new topology device. Once created assign the gpu to
+	 * that topology device
 	 */
 	dev = kfd_assign_gpu(gpu);
 	if (!dev) {
-		pr_info("GPU was not found in the current topology. Extending.\n");
-		kfd_debug_print_topology();
-		dev = kfd_create_topology_device();
-		if (!dev) {
-			res = -ENOMEM;
+		res = kfd_create_crat_image_virtual(&crat_image, &image_size,
+						    COMPUTE_UNIT_GPU, gpu,
+						    proximity_domain);
+		if (res) {
+			pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
+			       gpu_id);
+			return res;
+		}
+		res = kfd_parse_crat_table(crat_image,
+					   &temp_topology_device_list,
+					   proximity_domain);
+		if (res) {
+			pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
+			       gpu_id);
 			goto err;
 		}
-		dev->gpu = gpu;
 
-		/*
-		 * TODO: Make a call to retrieve topology information from the
-		 * GPU vBIOS
-		 */
+		down_write(&topology_lock);
+		kfd_topology_update_device_list(&temp_topology_device_list,
+			&topology_device_list);
 
 		/* Update the SYSFS tree, since we added another topology
 		 * device
 		 */
-		if (kfd_topology_update_sysfs() < 0)
-			kfd_topology_release_sysfs();
-
+		res = kfd_topology_update_sysfs();
+		up_write(&topology_lock);
+
+		if (!res)
+			sys_props.generation_count++;
+		else
+			pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
+						gpu_id, res);
+		dev = kfd_assign_gpu(gpu);
+		if (WARN_ON(!dev)) {
+			res = -ENODEV;
+			goto err;
+		}
 	}
 
 	dev->gpu_id = gpu_id;
 	gpu->id = gpu_id;
+
+	/* TODO: Move the following lines to function
+	 *	kfd_add_non_crat_information
+	 */
+
+	/* Fill-in additional information that is not available in CRAT but
+	 * needed for the topology
+	 */
+
+	dev->gpu->kfd2kgd->get_cu_info(dev->gpu->kgd, &cu_info);
+	dev->node_props.simd_arrays_per_engine =
+		cu_info.num_shader_arrays_per_engine;
+
 	dev->node_props.vendor_id = gpu->pdev->vendor;
 	dev->node_props.device_id = gpu->pdev->device;
-	dev->node_props.location_id = (gpu->pdev->bus->number << 24) +
-			(gpu->pdev->devfn & 0xffffff);
-	/*
-	 * TODO: Retrieve max engine clock values from KGD
-	 */
+	dev->node_props.location_id = PCI_DEVID(gpu->pdev->bus->number,
+		gpu->pdev->devfn);
+	dev->node_props.max_engine_clk_fcompute =
+		dev->gpu->kfd2kgd->get_max_engine_clock_in_mhz(dev->gpu->kgd);
+	dev->node_props.max_engine_clk_ccompute =
+		cpufreq_quick_get_max(0) / 1000;
+
+	kfd_fill_mem_clk_max_info(dev);
+	kfd_fill_iolink_non_crat_info(dev);
+
+	switch (dev->gpu->device_info->asic_family) {
+	case CHIP_KAVERI:
+	case CHIP_HAWAII:
+	case CHIP_TONGA:
+		dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0 <<
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+		break;
+	case CHIP_CARRIZO:
+	case CHIP_FIJI:
+	case CHIP_POLARIS10:
+	case CHIP_POLARIS11:
+		pr_debug("Adding doorbell packet type capability\n");
+		dev->node_props.capability |= ((HSA_CAP_DOORBELL_TYPE_1_0 <<
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT) &
+			HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK);
+		break;
+	default:
+		WARN(1, "Unexpected ASIC family %u",
+		     dev->gpu->device_info->asic_family);
+	}
 
+	/* Fix errors in CZ CRAT.
+	 * simd_count: Carrizo CRAT reports wrong simd_count, probably
+	 *		because it doesn't consider masked out CUs
+	 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
+	 * capability flag: Carrizo CRAT doesn't report IOMMU flags
+	 */
 	if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) {
-		dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE;
-		pr_info("Adding doorbell packet type capability\n");
+		dev->node_props.simd_count =
+			cu_info.simd_per_cu * cu_info.cu_active_number;
+		dev->node_props.max_waves_per_simd = 10;
+		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
 	}
 
-	res = 0;
-
-err:
-	up_write(&topology_lock);
+	kfd_debug_print_topology();
 
-	if (res == 0)
+	if (!res)
 		kfd_notify_gpu_change(gpu_id, 1);
-
+err:
+	kfd_destroy_crat_image(crat_image);
 	return res;
 }
 
 int kfd_topology_remove_device(struct kfd_dev *gpu)
 {
-	struct kfd_topology_device *dev;
+	struct kfd_topology_device *dev, *tmp;
 	uint32_t gpu_id;
 	int res = -ENODEV;
 
 	down_write(&topology_lock);
 
-	list_for_each_entry(dev, &topology_device_list, list)
+	list_for_each_entry_safe(dev, tmp, &topology_device_list, list)
 		if (dev->gpu == gpu) {
 			gpu_id = dev->gpu_id;
 			kfd_remove_sysfs_node_entry(dev);
 			kfd_release_topology_device(dev);
+			sys_props.num_devices--;
 			res = 0;
 			if (kfd_topology_update_sysfs() < 0)
 				kfd_topology_release_sysfs();
@@ -1201,28 +1294,32 @@ int kfd_topology_remove_device(struct kfd_dev *gpu)
 
 	up_write(&topology_lock);
 
-	if (res == 0)
+	if (!res)
 		kfd_notify_gpu_change(gpu_id, 0);
 
 	return res;
 }
 
-/*
- * When idx is out of bounds, the function will return NULL
+/* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
+ *	topology. If GPU device is found @idx, then valid kfd_dev pointer is
+ *	returned through @kdev
+ * Return -	0: On success (@kdev will be NULL for non GPU nodes)
+ *		-1: If end of list
  */
-struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
+int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev)
 {
 
 	struct kfd_topology_device *top_dev;
-	struct kfd_dev *device = NULL;
 	uint8_t device_idx = 0;
 
+	*kdev = NULL;
 	down_read(&topology_lock);
 
 	list_for_each_entry(top_dev, &topology_device_list, list) {
 		if (device_idx == idx) {
-			device = top_dev->gpu;
-			break;
+			*kdev = top_dev->gpu;
+			up_read(&topology_lock);
+			return 0;
 		}
 
 		device_idx++;
@@ -1230,6 +1327,88 @@ struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx)
 
 	up_read(&topology_lock);
 
-	return device;
+	return -1;
+
+}
+
+static int kfd_cpumask_to_apic_id(const struct cpumask *cpumask)
+{
+	const struct cpuinfo_x86 *cpuinfo;
+	int first_cpu_of_numa_node;
+
+	if (!cpumask || cpumask == cpu_none_mask)
+		return -1;
+	first_cpu_of_numa_node = cpumask_first(cpumask);
+	if (first_cpu_of_numa_node >= nr_cpu_ids)
+		return -1;
+	cpuinfo = &cpu_data(first_cpu_of_numa_node);
 
+	return cpuinfo->apicid;
 }
+
+/* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
+ *	of the given NUMA node (numa_node_id)
+ * Return -1 on failure
+ */
+int kfd_numa_node_to_apic_id(int numa_node_id)
+{
+	if (numa_node_id == -1) {
+		pr_warn("Invalid NUMA Node. Use online CPU mask\n");
+		return kfd_cpumask_to_apic_id(cpu_online_mask);
+	}
+	return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id));
+}
+
+#if defined(CONFIG_DEBUG_FS)
+
+int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data)
+{
+	struct kfd_topology_device *dev;
+	unsigned int i = 0;
+	int r = 0;
+
+	down_read(&topology_lock);
+
+	list_for_each_entry(dev, &topology_device_list, list) {
+		if (!dev->gpu) {
+			i++;
+			continue;
+		}
+
+		seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
+		r = dqm_debugfs_hqds(m, dev->gpu->dqm);
+		if (r)
+			break;
+	}
+
+	up_read(&topology_lock);
+
+	return r;
+}
+
+int kfd_debugfs_rls_by_device(struct seq_file *m, void *data)
+{
+	struct kfd_topology_device *dev;
+	unsigned int i = 0;
+	int r = 0;
+
+	down_read(&topology_lock);
+
+	list_for_each_entry(dev, &topology_device_list, list) {
+		if (!dev->gpu) {
+			i++;
+			continue;
+		}
+
+		seq_printf(m, "Node %u, gpu_id %x:\n", i++, dev->gpu->id);
+		r = pm_debugfs_runlist(m, &dev->gpu->dqm->packets);
+		if (r)
+			break;
+	}
+
+	up_read(&topology_lock);
+
+	return r;
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index c3ddb9b95ff8..53fca1f45401 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -39,8 +39,13 @@
 #define HSA_CAP_WATCH_POINTS_SUPPORTED		0x00000080
 #define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK	0x00000f00
 #define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT	8
-#define HSA_CAP_RESERVED			0xfffff000
-#define HSA_CAP_DOORBELL_PACKET_TYPE		0x00001000
+#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK	0x00003000
+#define HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT	12
+#define HSA_CAP_RESERVED			0xffffc000
+
+#define HSA_CAP_DOORBELL_TYPE_PRE_1_0		0x0
+#define HSA_CAP_DOORBELL_TYPE_1_0		0x1
+#define HSA_CAP_AQL_QUEUE_DOUBLE_MAP		0x00004000
 
 struct kfd_node_properties {
 	uint32_t cpu_cores_count;
@@ -91,8 +96,6 @@ struct kfd_mem_properties {
 	struct attribute	attr;
 };
 
-#define KFD_TOPOLOGY_CPU_SIBLINGS 256
-
 #define HSA_CACHE_TYPE_DATA		0x00000001
 #define HSA_CACHE_TYPE_INSTRUCTION	0x00000002
 #define HSA_CACHE_TYPE_CPU		0x00000004
@@ -109,7 +112,7 @@ struct kfd_cache_properties {
 	uint32_t		cache_assoc;
 	uint32_t		cache_latency;
 	uint32_t		cache_type;
-	uint8_t			sibling_map[KFD_TOPOLOGY_CPU_SIBLINGS];
+	uint8_t			sibling_map[CRAT_SIBLINGMAP_SIZE];
 	struct kobject		*kobj;
 	struct attribute	attr;
 };
@@ -132,24 +135,36 @@ struct kfd_iolink_properties {
 	struct attribute	attr;
 };
 
+struct kfd_perf_properties {
+	struct list_head	list;
+	char			block_name[16];
+	uint32_t		max_concurrent;
+	struct attribute_group	*attr_group;
+};
+
 struct kfd_topology_device {
 	struct list_head		list;
 	uint32_t			gpu_id;
+	uint32_t			proximity_domain;
 	struct kfd_node_properties	node_props;
-	uint32_t			mem_bank_count;
 	struct list_head		mem_props;
 	uint32_t			cache_count;
 	struct list_head		cache_props;
 	uint32_t			io_link_count;
 	struct list_head		io_link_props;
+	struct list_head		perf_props;
 	struct kfd_dev			*gpu;
 	struct kobject			*kobj_node;
 	struct kobject			*kobj_mem;
 	struct kobject			*kobj_cache;
 	struct kobject			*kobj_iolink;
+	struct kobject			*kobj_perf;
 	struct attribute		attr_gpuid;
 	struct attribute		attr_name;
 	struct attribute		attr_props;
+	uint8_t				oem_id[CRAT_OEMID_LENGTH];
+	uint8_t				oem_table_id[CRAT_OEMTABLEID_LENGTH];
+	uint32_t			oem_revision;
 };
 
 struct kfd_system_properties {
@@ -164,6 +179,12 @@ struct kfd_system_properties {
 	struct attribute	attr_props;
 };
 
+struct kfd_topology_device *kfd_create_topology_device(
+		struct list_head *device_list);
+void kfd_release_topology_device_list(struct list_head *device_list);
 
+extern bool amd_iommu_pc_supported(void);
+extern u8 amd_iommu_pc_get_max_banks(u16 devid);
+extern u8 amd_iommu_pc_get_max_counters(u16 devid);
 
 #endif /* __KFD_TOPOLOGY_H__ */
diff --git a/drivers/gpu/drm/amd/display/TODO b/drivers/gpu/drm/amd/display/TODO
index 46464678f2b3..357d59648401 100644
--- a/drivers/gpu/drm/amd/display/TODO
+++ b/drivers/gpu/drm/amd/display/TODO
@@ -105,3 +105,6 @@ useless with filtering output. dynamic debug printing might be an option.
 20. Use kernel i2c device to program HDMI retimer. Some boards have an HDMI
 retimer that we need to program to pass PHY compliance. Currently that's
 bypassing the i2c device and goes directly to HW. This should be changed.
+
+21. Remove vector.c from dc/basics. It's used in DDC code which can probably
+be simplified enough to no longer need a vector implementation.
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index bb5fa895fb64..63c67346d316 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -59,9 +59,9 @@
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 #include "ivsrcid/irqsrcs_dcn_1_0.h"
 
-#include "raven1/DCN/dcn_1_0_offset.h"
-#include "raven1/DCN/dcn_1_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
+#include "soc15ip.h"
 
 #include "soc15_common.h"
 #endif
@@ -431,9 +431,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 	adev->dm.dc = dc_create(&init_data);
 
 	if (adev->dm.dc) {
-		DRM_INFO("Display Core initialized!\n");
+		DRM_INFO("Display Core initialized with v%s!\n", DC_VER);
 	} else {
-		DRM_INFO("Display Core failed to initialize!\n");
+		DRM_INFO("Display Core failed to initialize with v%s!\n", DC_VER);
 		goto error;
 	}
 
@@ -629,11 +629,13 @@ static int dm_resume(void *handle)
 {
 	struct amdgpu_device *adev = handle;
 	struct amdgpu_display_manager *dm = &adev->dm;
+	int ret = 0;
 
 	/* power on hardware */
 	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
 
-	return 0;
+	ret = amdgpu_dm_display_resume(adev);
+	return ret;
 }
 
 int amdgpu_dm_display_resume(struct amdgpu_device *adev)
@@ -792,7 +794,7 @@ dm_atomic_state_alloc_free(struct drm_atomic_state *state)
 
 static const struct drm_mode_config_funcs amdgpu_dm_mode_funcs = {
 	.fb_create = amdgpu_user_framebuffer_create,
-	.output_poll_changed = amdgpu_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = amdgpu_dm_atomic_check,
 	.atomic_commit = amdgpu_dm_atomic_commit,
 	.atomic_state_alloc = dm_atomic_state_alloc,
@@ -1035,6 +1037,10 @@ static void handle_hpd_rx_irq(void *param)
 			!is_mst_root_connector) {
 		/* Downstream Port status changed. */
 		if (dc_link_detect(dc_link, DETECT_REASON_HPDRX)) {
+
+			if (aconnector->fake_enable)
+				aconnector->fake_enable = false;
+
 			amdgpu_dm_update_connector_after_detect(aconnector);
 
 
@@ -1590,7 +1596,6 @@ static int dm_early_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	adev->ddev->driver->driver_features |= DRIVER_ATOMIC;
-	amdgpu_dm_set_irq_funcs(adev);
 
 	switch (adev->asic_type) {
 	case CHIP_BONAIRE:
@@ -1664,6 +1669,8 @@ static int dm_early_init(void *handle)
 		return -EINVAL;
 	}
 
+	amdgpu_dm_set_irq_funcs(adev);
+
 	if (adev->mode_info.funcs == NULL)
 		adev->mode_info.funcs = &dm_display_funcs;
 
@@ -1679,18 +1686,6 @@ static int dm_early_init(void *handle)
 	return 0;
 }
 
-struct dm_connector_state {
-	struct drm_connector_state base;
-
-	enum amdgpu_rmx_type scaling;
-	uint8_t underscan_vborder;
-	uint8_t underscan_hborder;
-	bool underscan_enable;
-};
-
-#define to_dm_connector_state(x)\
-	container_of((x), struct dm_connector_state, base)
-
 static bool modeset_required(struct drm_crtc_state *crtc_state,
 			     struct dc_stream_state *new_stream,
 			     struct dc_stream_state *old_stream)
@@ -1773,8 +1768,7 @@ static bool fill_rects_from_plane_state(const struct drm_plane_state *state,
 	return true;
 }
 static int get_fb_info(const struct amdgpu_framebuffer *amdgpu_fb,
-		       uint64_t *tiling_flags,
-		       uint64_t *fb_location)
+		       uint64_t *tiling_flags)
 {
 	struct amdgpu_bo *rbo = gem_to_amdgpu_bo(amdgpu_fb->obj);
 	int r = amdgpu_bo_reserve(rbo, false);
@@ -1786,9 +1780,6 @@ static int get_fb_info(const struct amdgpu_framebuffer *amdgpu_fb,
 		return r;
 	}
 
-	if (fb_location)
-		*fb_location = amdgpu_bo_gpu_offset(rbo);
-
 	if (tiling_flags)
 		amdgpu_bo_get_tiling_flags(rbo, tiling_flags);
 
@@ -1799,12 +1790,9 @@ static int get_fb_info(const struct amdgpu_framebuffer *amdgpu_fb,
 
 static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
 					 struct dc_plane_state *plane_state,
-					 const struct amdgpu_framebuffer *amdgpu_fb,
-					 bool addReq)
+					 const struct amdgpu_framebuffer *amdgpu_fb)
 {
 	uint64_t tiling_flags;
-	uint64_t fb_location = 0;
-	uint64_t chroma_addr = 0;
 	unsigned int awidth;
 	const struct drm_framebuffer *fb = &amdgpu_fb->base;
 	int ret = 0;
@@ -1812,8 +1800,7 @@ static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
 
 	ret = get_fb_info(
 		amdgpu_fb,
-		&tiling_flags,
-		addReq == true ? &fb_location:NULL);
+		&tiling_flags);
 
 	if (ret)
 		return ret;
@@ -1851,8 +1838,6 @@ static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
 
 	if (plane_state->format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
 		plane_state->address.type = PLN_ADDR_TYPE_GRAPHICS;
-		plane_state->address.grph.addr.low_part = lower_32_bits(fb_location);
-		plane_state->address.grph.addr.high_part = upper_32_bits(fb_location);
 		plane_state->plane_size.grph.surface_size.x = 0;
 		plane_state->plane_size.grph.surface_size.y = 0;
 		plane_state->plane_size.grph.surface_size.width = fb->width;
@@ -1865,15 +1850,6 @@ static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
 	} else {
 		awidth = ALIGN(fb->width, 64);
 		plane_state->address.type = PLN_ADDR_TYPE_VIDEO_PROGRESSIVE;
-		plane_state->address.video_progressive.luma_addr.low_part
-						= lower_32_bits(fb_location);
-		plane_state->address.video_progressive.luma_addr.high_part
-						= upper_32_bits(fb_location);
-		chroma_addr = fb_location + (u64)(awidth * fb->height);
-		plane_state->address.video_progressive.chroma_addr.low_part
-						= lower_32_bits(chroma_addr);
-		plane_state->address.video_progressive.chroma_addr.high_part
-						= upper_32_bits(chroma_addr);
 		plane_state->plane_size.video.luma_size.x = 0;
 		plane_state->plane_size.video.luma_size.y = 0;
 		plane_state->plane_size.video.luma_size.width = awidth;
@@ -1983,8 +1959,7 @@ static void fill_gamma_from_crtc_state(const struct drm_crtc_state *crtc_state,
 static int fill_plane_attributes(struct amdgpu_device *adev,
 				 struct dc_plane_state *dc_plane_state,
 				 struct drm_plane_state *plane_state,
-				 struct drm_crtc_state *crtc_state,
-				 bool addrReq)
+				 struct drm_crtc_state *crtc_state)
 {
 	const struct amdgpu_framebuffer *amdgpu_fb =
 		to_amdgpu_framebuffer(plane_state->fb);
@@ -1998,8 +1973,7 @@ static int fill_plane_attributes(struct amdgpu_device *adev,
 	ret = fill_plane_attributes_from_fb(
 		crtc->dev->dev_private,
 		dc_plane_state,
-		amdgpu_fb,
-		addrReq);
+		amdgpu_fb);
 
 	if (ret)
 		return ret;
@@ -2042,30 +2016,32 @@ static void update_stream_scaling_settings(const struct drm_display_mode *mode,
 	dst.width = stream->timing.h_addressable;
 	dst.height = stream->timing.v_addressable;
 
-	rmx_type = dm_state->scaling;
-	if (rmx_type == RMX_ASPECT || rmx_type == RMX_OFF) {
-		if (src.width * dst.height <
-				src.height * dst.width) {
-			/* height needs less upscaling/more downscaling */
-			dst.width = src.width *
-					dst.height / src.height;
-		} else {
-			/* width needs less upscaling/more downscaling */
-			dst.height = src.height *
-					dst.width / src.width;
+	if (dm_state) {
+		rmx_type = dm_state->scaling;
+		if (rmx_type == RMX_ASPECT || rmx_type == RMX_OFF) {
+			if (src.width * dst.height <
+					src.height * dst.width) {
+				/* height needs less upscaling/more downscaling */
+				dst.width = src.width *
+						dst.height / src.height;
+			} else {
+				/* width needs less upscaling/more downscaling */
+				dst.height = src.height *
+						dst.width / src.width;
+			}
+		} else if (rmx_type == RMX_CENTER) {
+			dst = src;
 		}
-	} else if (rmx_type == RMX_CENTER) {
-		dst = src;
-	}
 
-	dst.x = (stream->timing.h_addressable - dst.width) / 2;
-	dst.y = (stream->timing.v_addressable - dst.height) / 2;
+		dst.x = (stream->timing.h_addressable - dst.width) / 2;
+		dst.y = (stream->timing.v_addressable - dst.height) / 2;
 
-	if (dm_state->underscan_enable) {
-		dst.x += dm_state->underscan_hborder / 2;
-		dst.y += dm_state->underscan_vborder / 2;
-		dst.width -= dm_state->underscan_hborder;
-		dst.height -= dm_state->underscan_vborder;
+		if (dm_state->underscan_enable) {
+			dst.x += dm_state->underscan_hborder / 2;
+			dst.y += dm_state->underscan_vborder / 2;
+			dst.width -= dm_state->underscan_hborder;
+			dst.height -= dm_state->underscan_vborder;
+		}
 	}
 
 	stream->src = src;
@@ -2174,6 +2150,7 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
 					     const struct drm_connector *connector)
 {
 	struct dc_crtc_timing *timing_out = &stream->timing;
+	struct dc_transfer_func *tf = dc_create_transfer_func();
 
 	memset(timing_out, 0, sizeof(struct dc_crtc_timing));
 
@@ -2217,13 +2194,9 @@ fill_stream_properties_from_drm_display_mode(struct dc_stream_state *stream,
 
 	stream->output_color_space = get_output_color_space(timing_out);
 
-	{
-		struct dc_transfer_func *tf = dc_create_transfer_func();
-
-		tf->type = TF_TYPE_PREDEFINED;
-		tf->tf = TRANSFER_FUNCTION_SRGB;
-		stream->out_transfer_func = tf;
-	}
+	tf->type = TF_TYPE_PREDEFINED;
+	tf->tf = TRANSFER_FUNCTION_SRGB;
+	stream->out_transfer_func = tf;
 }
 
 static void fill_audio_info(struct audio_info *audio_info,
@@ -2330,6 +2303,56 @@ static int create_fake_sink(struct amdgpu_dm_connector *aconnector)
 	return 0;
 }
 
+static void set_multisync_trigger_params(
+		struct dc_stream_state *stream)
+{
+	if (stream->triggered_crtc_reset.enabled) {
+		stream->triggered_crtc_reset.event = CRTC_EVENT_VSYNC_RISING;
+		stream->triggered_crtc_reset.delay = TRIGGER_DELAY_NEXT_LINE;
+	}
+}
+
+static void set_master_stream(struct dc_stream_state *stream_set[],
+			      int stream_count)
+{
+	int j, highest_rfr = 0, master_stream = 0;
+
+	for (j = 0;  j < stream_count; j++) {
+		if (stream_set[j] && stream_set[j]->triggered_crtc_reset.enabled) {
+			int refresh_rate = 0;
+
+			refresh_rate = (stream_set[j]->timing.pix_clk_khz*1000)/
+				(stream_set[j]->timing.h_total*stream_set[j]->timing.v_total);
+			if (refresh_rate > highest_rfr) {
+				highest_rfr = refresh_rate;
+				master_stream = j;
+			}
+		}
+	}
+	for (j = 0;  j < stream_count; j++) {
+		if (stream_set[j] && j != master_stream)
+			stream_set[j]->triggered_crtc_reset.event_source = stream_set[master_stream];
+	}
+}
+
+static void dm_enable_per_frame_crtc_master_sync(struct dc_state *context)
+{
+	int i = 0;
+
+	if (context->stream_count < 2)
+		return;
+	for (i = 0; i < context->stream_count ; i++) {
+		if (!context->streams[i])
+			continue;
+		/* TODO: add a function to read AMD VSDB bits and will set
+		 * crtc_sync_master.multi_sync_enabled flag
+		 * For now its set to false
+		 */
+		set_multisync_trigger_params(context->streams[i]);
+	}
+	set_master_stream(context->streams, context->stream_count);
+}
+
 static struct dc_stream_state *
 create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 		       const struct drm_display_mode *drm_mode,
@@ -2343,12 +2366,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 
 	if (aconnector == NULL) {
 		DRM_ERROR("aconnector is NULL!\n");
-		goto drm_connector_null;
-	}
-
-	if (dm_state == NULL) {
-		DRM_ERROR("dm_state is NULL!\n");
-		goto dm_state_null;
+		return stream;
 	}
 
 	drm_connector = &aconnector->base;
@@ -2360,18 +2378,18 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 		 */
 		if (aconnector->mst_port) {
 			dm_dp_mst_dc_sink_create(drm_connector);
-			goto mst_dc_sink_create_done;
+			return stream;
 		}
 
 		if (create_fake_sink(aconnector))
-			goto stream_create_fail;
+			return stream;
 	}
 
 	stream = dc_create_stream_for_sink(aconnector->dc_sink);
 
 	if (stream == NULL) {
 		DRM_ERROR("Failed to create stream for sink!\n");
-		goto stream_create_fail;
+		return stream;
 	}
 
 	list_for_each_entry(preferred_mode, &aconnector->base.modes, head) {
@@ -2397,9 +2415,12 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 	} else {
 		decide_crtc_timing_for_drm_display_mode(
 				&mode, preferred_mode,
-				dm_state->scaling != RMX_OFF);
+				dm_state ? (dm_state->scaling != RMX_OFF) : false);
 	}
 
+	if (!dm_state)
+		drm_mode_set_crtcinfo(&mode, 0);
+
 	fill_stream_properties_from_drm_display_mode(stream,
 			&mode, &aconnector->base);
 	update_stream_scaling_settings(&mode, dm_state, stream);
@@ -2409,10 +2430,8 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector,
 		drm_connector,
 		aconnector->dc_sink);
 
-stream_create_fail:
-dm_state_null:
-drm_connector_null:
-mst_dc_sink_create_done:
+	update_stream_signal(stream);
+
 	return stream;
 }
 
@@ -2480,6 +2499,27 @@ dm_crtc_duplicate_state(struct drm_crtc *crtc)
 	return &state->base;
 }
 
+
+static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable)
+{
+	enum dc_irq_source irq_source;
+	struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
+
+	irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst;
+	return dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY;
+}
+
+static int dm_enable_vblank(struct drm_crtc *crtc)
+{
+	return dm_set_vblank(crtc, true);
+}
+
+static void dm_disable_vblank(struct drm_crtc *crtc)
+{
+	dm_set_vblank(crtc, false);
+}
+
 /* Implemented only the options currently availible for the driver */
 static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = {
 	.reset = dm_crtc_reset_state,
@@ -2489,6 +2529,8 @@ static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = {
 	.page_flip = drm_atomic_helper_page_flip,
 	.atomic_duplicate_state = dm_crtc_duplicate_state,
 	.atomic_destroy_state = dm_crtc_destroy_state,
+	.enable_vblank = dm_enable_vblank,
+	.disable_vblank = dm_disable_vblank,
 };
 
 static enum drm_connector_status
@@ -2713,8 +2755,7 @@ static void create_eml_sink(struct amdgpu_dm_connector *aconnector)
 	};
 	struct edid *edid;
 
-	if (!aconnector->base.edid_blob_ptr ||
-		!aconnector->base.edid_blob_ptr->data) {
+	if (!aconnector->base.edid_blob_ptr) {
 		DRM_ERROR("No EDID firmware found on connector: %s ,forcing to OFF!\n",
 				aconnector->base.name);
 
@@ -2784,7 +2825,7 @@ int amdgpu_dm_connector_mode_valid(struct drm_connector *connector,
 		goto fail;
 	}
 
-	stream = dc_create_stream_for_sink(dc_sink);
+	stream = create_stream_for_sink(aconnector, mode, NULL);
 	if (stream == NULL) {
 		DRM_ERROR("Failed to create stream for sink!\n");
 		goto fail;
@@ -2989,7 +3030,7 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
 							= lower_32_bits(afb->address);
 			plane_state->address.video_progressive.luma_addr.high_part
 							= upper_32_bits(afb->address);
-			chroma_addr = afb->address + (u64)(awidth * new_state->fb->height);
+			chroma_addr = afb->address + (u64)awidth * new_state->fb->height;
 			plane_state->address.video_progressive.chroma_addr.low_part
 							= lower_32_bits(chroma_addr);
 			plane_state->address.video_progressive.chroma_addr.high_part
@@ -3044,6 +3085,9 @@ static int dm_plane_atomic_check(struct drm_plane *plane,
 	if (!dm_plane_state->dc_state)
 		return 0;
 
+	if (!fill_rects_from_plane_state(state, dm_plane_state->dc_state))
+		return -EINVAL;
+
 	if (dc_validate_plane(dc, dm_plane_state->dc_state) == DC_OK)
 		return 0;
 
@@ -3090,8 +3134,6 @@ static int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
 
 	switch (aplane->base.type) {
 	case DRM_PLANE_TYPE_PRIMARY:
-		aplane->base.format_default = true;
-
 		res = drm_universal_plane_init(
 				dm->adev->ddev,
 				&aplane->base,
@@ -3336,8 +3378,6 @@ static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector,
 		amdgpu_dm_connector->num_modes =
 				drm_add_edid_modes(connector, edid);
 
-		drm_edid_to_eld(connector, edid);
-
 		amdgpu_dm_get_native_mode(connector);
 	} else {
 		amdgpu_dm_connector->num_modes = 0;
@@ -3999,6 +4039,19 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 	}
 }
 
+/**
+ * amdgpu_dm_crtc_copy_transient_flags - copy mirrored flags from DRM to DC
+ * @crtc_state: the DRM CRTC state
+ * @stream_state: the DC stream state.
+ *
+ * Copy the mirrored transient state flags from DRM, to DC. It is used to bring
+ * a dc_stream_state's flags in sync with a drm_crtc_state's flags.
+ */
+static void amdgpu_dm_crtc_copy_transient_flags(struct drm_crtc_state *crtc_state,
+						struct dc_stream_state *stream_state)
+{
+	stream_state->mode_changed = crtc_state->mode_changed;
+}
 
 static int amdgpu_dm_atomic_commit(struct drm_device *dev,
 				   struct drm_atomic_state *state,
@@ -4038,11 +4091,8 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 	struct amdgpu_display_manager *dm = &adev->dm;
 	struct dm_atomic_state *dm_state;
 	uint32_t i, j;
-	uint32_t new_crtcs_count = 0;
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
-	struct amdgpu_crtc *new_crtcs[MAX_STREAMS];
-	struct dc_stream_state *new_stream = NULL;
 	unsigned long flags;
 	bool wait_for_vblank = true;
 	struct drm_connector *connector;
@@ -4072,6 +4122,12 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 			new_crtc_state->active_changed,
 			new_crtc_state->connectors_changed);
 
+		/* Copy all transient state flags into dc state */
+		if (dm_new_crtc_state->stream) {
+			amdgpu_dm_crtc_copy_transient_flags(&dm_new_crtc_state->base,
+							    dm_new_crtc_state->stream);
+		}
+
 		/* handles headless hotplug case, updating new_state and
 		 * aconnector as needed
 		 */
@@ -4101,25 +4157,9 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 				continue;
 			}
 
-
 			if (dm_old_crtc_state->stream)
 				remove_stream(adev, acrtc, dm_old_crtc_state->stream);
 
-
-			/*
-			 * this loop saves set mode crtcs
-			 * we needed to enable vblanks once all
-			 * resources acquired in dc after dc_commit_streams
-			 */
-
-			/*TODO move all this into dm_crtc_state, get rid of
-			 * new_crtcs array and use old and new atomic states
-			 * instead
-			 */
-			new_crtcs[new_crtcs_count] = acrtc;
-			new_crtcs_count++;
-
-			new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 			acrtc->enabled = true;
 			acrtc->hw_mode = new_crtc_state->mode;
 			crtc->hwmode = new_crtc_state->mode;
@@ -4137,31 +4177,61 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 	 * are removed from freesync module
 	 */
 	if (adev->dm.freesync_module) {
-		for (i = 0; i < new_crtcs_count; i++) {
+		for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
+					      new_crtc_state, i) {
 			struct amdgpu_dm_connector *aconnector = NULL;
+			struct dm_connector_state *dm_new_con_state = NULL;
+			struct amdgpu_crtc *acrtc = NULL;
+			bool modeset_needed;
 
-			new_crtc_state = drm_atomic_get_new_crtc_state(state,
-					&new_crtcs[i]->base);
 			dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
+			dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
+			modeset_needed = modeset_required(
+					new_crtc_state,
+					dm_new_crtc_state->stream,
+					dm_old_crtc_state->stream);
+			/* We add stream to freesync if:
+			 * 1. Said stream is not null, and
+			 * 2. A modeset is requested. This means that the
+			 *    stream was removed previously, and needs to be
+			 *    replaced.
+			 */
+			if (dm_new_crtc_state->stream == NULL ||
+					!modeset_needed)
+				continue;
+
+			acrtc = to_amdgpu_crtc(crtc);
 
-			new_stream = dm_new_crtc_state->stream;
-			aconnector = amdgpu_dm_find_first_crtc_matching_connector(
-					state,
-					&new_crtcs[i]->base);
+			aconnector =
+				amdgpu_dm_find_first_crtc_matching_connector(
+					state, crtc);
 			if (!aconnector) {
-				DRM_DEBUG_DRIVER("Atomic commit: Failed to find connector for acrtc id:%d "
-					 "skipping freesync init\n",
-					 new_crtcs[i]->crtc_id);
+				DRM_DEBUG_DRIVER("Atomic commit: Failed to "
+						 "find connector for acrtc "
+						 "id:%d skipping freesync "
+						 "init\n",
+						 acrtc->crtc_id);
 				continue;
 			}
 
 			mod_freesync_add_stream(adev->dm.freesync_module,
-						new_stream, &aconnector->caps);
+						dm_new_crtc_state->stream,
+						&aconnector->caps);
+			new_con_state = drm_atomic_get_new_connector_state(
+					state, &aconnector->base);
+			dm_new_con_state = to_dm_connector_state(new_con_state);
+
+			mod_freesync_set_user_enable(adev->dm.freesync_module,
+						     &dm_new_crtc_state->stream,
+						     1,
+						     &dm_new_con_state->user_enable);
 		}
 	}
 
-	if (dm_state->context)
+	if (dm_state->context) {
+		dm_enable_per_frame_crtc_master_sync(dm_state->context);
 		WARN_ON(!dc_commit_state(dm->dc, dm_state->context));
+	}
 
 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
 		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
@@ -4219,18 +4289,28 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 			dm_error("%s: Failed to update stream scaling!\n", __func__);
 	}
 
-	for (i = 0; i < new_crtcs_count; i++) {
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
+			new_crtc_state, i) {
 		/*
 		 * loop to enable interrupts on newly arrived crtc
 		 */
-		struct amdgpu_crtc *acrtc = new_crtcs[i];
+		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
+		bool modeset_needed;
 
-		new_crtc_state = drm_atomic_get_new_crtc_state(state, &acrtc->base);
 		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
+		dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
+		modeset_needed = modeset_required(
+				new_crtc_state,
+				dm_new_crtc_state->stream,
+				dm_old_crtc_state->stream);
+
+		if (dm_new_crtc_state->stream == NULL || !modeset_needed)
+			continue;
 
 		if (adev->dm.freesync_module)
 			mod_freesync_notify_mode_change(
-				adev->dm.freesync_module, &dm_new_crtc_state->stream, 1);
+				adev->dm.freesync_module,
+				&dm_new_crtc_state->stream, 1);
 
 		manage_dm_interrupts(adev, acrtc, true);
 	}
@@ -4464,18 +4544,15 @@ static int dm_update_crtcs_state(struct dc *dc,
 						__func__, acrtc->base.base.id);
 				break;
 			}
-		}
-
-		if (enable && dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) &&
-				dc_is_stream_scaling_unchanged(new_stream, dm_old_crtc_state->stream)) {
 
-			new_crtc_state->mode_changed = false;
-
-			DRM_DEBUG_DRIVER("Mode change not required, setting mode_changed to %d",
-				         new_crtc_state->mode_changed);
+			if (dc_is_stream_unchanged(new_stream, dm_old_crtc_state->stream) &&
+			    dc_is_stream_scaling_unchanged(new_stream, dm_old_crtc_state->stream)) {
+				new_crtc_state->mode_changed = false;
+				DRM_DEBUG_DRIVER("Mode change not required, setting mode_changed to %d",
+						 new_crtc_state->mode_changed);
+			}
 		}
 
-
 		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
 			goto next_crtc;
 
@@ -4532,6 +4609,7 @@ static int dm_update_crtcs_state(struct dc *dc,
 				WARN_ON(dm_new_crtc_state->stream);
 
 				dm_new_crtc_state->stream = new_stream;
+
 				dc_stream_retain(new_stream);
 
 				DRM_DEBUG_DRIVER("Enabling DRM crtc: %d\n",
@@ -4580,8 +4658,6 @@ static int dm_update_planes_state(struct dc *dc,
 	bool pflip_needed  = !state->allow_modeset;
 	int ret = 0;
 
-	if (pflip_needed)
-		return ret;
 
 	/* Add new planes */
 	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
@@ -4596,6 +4672,8 @@ static int dm_update_planes_state(struct dc *dc,
 
 		/* Remove any changed/removed planes */
 		if (!enable) {
+			if (pflip_needed)
+				continue;
 
 			if (!old_plane_crtc)
 				continue;
@@ -4627,6 +4705,7 @@ static int dm_update_planes_state(struct dc *dc,
 			*lock_and_validation_needed = true;
 
 		} else { /* Add new planes */
+			struct dc_plane_state *dc_new_plane_state;
 
 			if (drm_atomic_plane_disabling(plane->state, new_plane_state))
 				continue;
@@ -4640,39 +4719,55 @@ static int dm_update_planes_state(struct dc *dc,
 			if (!dm_new_crtc_state->stream)
 				continue;
 
+			if (pflip_needed)
+				continue;
 
 			WARN_ON(dm_new_plane_state->dc_state);
 
-			dm_new_plane_state->dc_state = dc_create_plane_state(dc);
-
-			DRM_DEBUG_DRIVER("Enabling DRM plane: %d on DRM crtc %d\n",
-					plane->base.id, new_plane_crtc->base.id);
-
-			if (!dm_new_plane_state->dc_state) {
+			dc_new_plane_state = dc_create_plane_state(dc);
+			if (!dc_new_plane_state) {
 				ret = -EINVAL;
 				return ret;
 			}
 
+			DRM_DEBUG_DRIVER("Enabling DRM plane: %d on DRM crtc %d\n",
+					plane->base.id, new_plane_crtc->base.id);
+
 			ret = fill_plane_attributes(
 				new_plane_crtc->dev->dev_private,
-				dm_new_plane_state->dc_state,
+				dc_new_plane_state,
 				new_plane_state,
-				new_crtc_state,
-				false);
-			if (ret)
+				new_crtc_state);
+			if (ret) {
+				dc_plane_state_release(dc_new_plane_state);
 				return ret;
+			}
 
-
+			/*
+			 * Any atomic check errors that occur after this will
+			 * not need a release. The plane state will be attached
+			 * to the stream, and therefore part of the atomic
+			 * state. It'll be released when the atomic state is
+			 * cleaned.
+			 */
 			if (!dc_add_plane_to_context(
 					dc,
 					dm_new_crtc_state->stream,
-					dm_new_plane_state->dc_state,
+					dc_new_plane_state,
 					dm_state->context)) {
 
+				dc_plane_state_release(dc_new_plane_state);
 				ret = -EINVAL;
 				return ret;
 			}
 
+			dm_new_plane_state->dc_state = dc_new_plane_state;
+
+			/* Tell DC to do a full surface update every time there
+			 * is a plane change. Inefficient, but works for now.
+			 */
+			dm_new_plane_state->dc_state->update_flags.bits.full_update = 1;
+
 			*lock_and_validation_needed = true;
 		}
 	}
@@ -4681,11 +4776,36 @@ static int dm_update_planes_state(struct dc *dc,
 	return ret;
 }
 
+static int dm_atomic_check_plane_state_fb(struct drm_atomic_state *state,
+					  struct drm_crtc *crtc)
+{
+	struct drm_plane *plane;
+	struct drm_crtc_state *crtc_state;
+
+	WARN_ON(!drm_atomic_get_new_crtc_state(state, crtc));
+
+	drm_for_each_plane_mask(plane, state->dev, crtc->state->plane_mask) {
+		struct drm_plane_state *plane_state =
+			drm_atomic_get_plane_state(state, plane);
+
+		if (IS_ERR(plane_state))
+			return -EDEADLK;
+
+		crtc_state = drm_atomic_get_crtc_state(plane_state->state, crtc);
+		if (IS_ERR(crtc_state))
+			return PTR_ERR(crtc_state);
+
+		if (crtc->primary == plane && crtc_state->active) {
+			if (!plane_state->fb)
+				return -EINVAL;
+		}
+	}
+	return 0;
+}
+
 static int amdgpu_dm_atomic_check(struct drm_device *dev,
 				  struct drm_atomic_state *state)
 {
-	int i;
-	int ret;
 	struct amdgpu_device *adev = dev->dev_private;
 	struct dc *dc = adev->dm.dc;
 	struct dm_atomic_state *dm_state = to_dm_atomic_state(state);
@@ -4693,6 +4813,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	struct drm_connector_state *old_con_state, *new_con_state;
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	int ret, i;
 
 	/*
 	 * This bool will be set for true for any modeset/reset
@@ -4704,37 +4825,25 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
-	/*
-	 * legacy_cursor_update should be made false for SoC's having
-	 * a dedicated hardware plane for cursor in amdgpu_dm_atomic_commit(),
-	 * otherwise for software cursor plane,
-	 * we should not add it to list of affected planes.
-	 */
-	if (state->legacy_cursor_update) {
-		for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
-			if (new_crtc_state->color_mgmt_changed) {
-				ret = drm_atomic_add_affected_planes(state, crtc);
-				if (ret)
-					goto fail;
-			}
-		}
-	} else {
-		for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
-			if (!drm_atomic_crtc_needs_modeset(new_crtc_state) &&
-					!new_crtc_state->color_mgmt_changed)
-				continue;
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		ret = dm_atomic_check_plane_state_fb(state, crtc);
+		if (ret)
+			goto fail;
 
-			if (!new_crtc_state->enable)
-				continue;
+		if (!drm_atomic_crtc_needs_modeset(new_crtc_state) &&
+		    !new_crtc_state->color_mgmt_changed)
+			continue;
 
-			ret = drm_atomic_add_affected_connectors(state, crtc);
-			if (ret)
-				return ret;
+		if (!new_crtc_state->enable)
+			continue;
 
-			ret = drm_atomic_add_affected_planes(state, crtc);
-			if (ret)
-				goto fail;
-		}
+		ret = drm_atomic_add_affected_connectors(state, crtc);
+		if (ret)
+			return ret;
+
+		ret = drm_atomic_add_affected_planes(state, crtc);
+		if (ret)
+			goto fail;
 	}
 
 	dm_state->context = dc_create_state();
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index 0230250a1164..2faa77a7eeda 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -222,6 +222,18 @@ struct dm_atomic_state {
 
 #define to_dm_atomic_state(x) container_of(x, struct dm_atomic_state, base)
 
+struct dm_connector_state {
+	struct drm_connector_state base;
+
+	enum amdgpu_rmx_type scaling;
+	uint8_t underscan_vborder;
+	uint8_t underscan_hborder;
+	bool underscan_enable;
+	struct mod_freesync_user_enable user_enable;
+};
+
+#define to_dm_connector_state(x)\
+	container_of((x), struct dm_connector_state, base)
 
 void amdgpu_dm_connector_funcs_reset(struct drm_connector *connector);
 struct drm_connector_state *
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index 9bd142f65f9b..e1acc10e35a2 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -109,7 +109,7 @@ enum dc_edid_status dm_helpers_parse_edid_caps(
 		struct cea_sad *sad = &sads[i];
 
 		edid_caps->audio_modes[i].format_code = sad->format;
-		edid_caps->audio_modes[i].channel_count = sad->channels;
+		edid_caps->audio_modes[i].channel_count = sad->channels + 1;
 		edid_caps->audio_modes[i].sample_rate = sad->freq;
 		edid_caps->audio_modes[i].sample_size = sad->byte2;
 	}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
index ca5d0d1581dc..422055080df4 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
@@ -683,13 +683,14 @@ static const struct amdgpu_irq_src_funcs dm_hpd_irq_funcs = {
 
 void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev)
 {
-	adev->crtc_irq.num_types = AMDGPU_CRTC_IRQ_LAST;
+
+	adev->crtc_irq.num_types = adev->mode_info.num_crtc;
 	adev->crtc_irq.funcs = &dm_crtc_irq_funcs;
 
-	adev->pageflip_irq.num_types = AMDGPU_PAGEFLIP_IRQ_LAST;
+	adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
 	adev->pageflip_irq.funcs = &dm_pageflip_irq_funcs;
 
-	adev->hpd_irq.num_types = AMDGPU_HPD_LAST;
+	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
 	adev->hpd_irq.funcs = &dm_hpd_irq_funcs;
 }
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index 638c2c2b5cd7..93421dad21bd 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -177,12 +177,7 @@ static const struct drm_connector_funcs dm_dp_mst_connector_funcs = {
 static int dm_connector_update_modes(struct drm_connector *connector,
 				struct edid *edid)
 {
-	int ret;
-
-	ret = drm_add_edid_modes(connector, edid);
-	drm_edid_to_eld(connector, edid);
-
-	return ret;
+	return drm_add_edid_modes(connector, edid);
 }
 
 void dm_dp_mst_dc_sink_create(struct drm_connector *connector)
@@ -194,6 +189,12 @@ void dm_dp_mst_dc_sink_create(struct drm_connector *connector)
 			.link = aconnector->dc_link,
 			.sink_signal = SIGNAL_TYPE_DISPLAY_PORT_MST };
 
+	/*
+	 * TODO: Need to further figure out why ddc.algo is NULL while MST port exists
+	 */
+	if (!aconnector->port || !aconnector->port->aux.ddc.algo)
+		return;
+
 	edid = drm_dp_mst_get_edid(connector, &aconnector->mst_port->mst_mgr, aconnector->port);
 
 	if (!edid) {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index 5df8fd5b537c..56e549249134 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -41,6 +41,10 @@ unsigned long long dm_get_timestamp(struct dc_context *ctx)
 	return 0;
 }
 
+void dm_perf_trace_timestamp(const char *func_name, unsigned int line)
+{
+}
+
 bool dm_write_persistent_data(struct dc_context *ctx,
 		const struct dc_sink *sink,
 		const char *module_name,
@@ -131,11 +135,12 @@ bool dm_pp_apply_display_requirements(
 		adev->pm.pm_display_cfg.min_bus_bandwidth = 0;
 
 		/* TODO: complete implementation of
-		 * amd_powerplay_display_configuration_change().
+		 * pp_display_configuration_change().
 		 * Follow example of:
 		 * PHM_StoreDALConfigurationData - powerplay\hwmgr\hardwaremanager.c
 		 * PP_IRI_DisplayConfigurationChange - powerplay\eventmgr\iri.c */
-		amd_powerplay_display_configuration_change(
+		if (adev->powerplay.pp_funcs->display_configuration_change)
+			adev->powerplay.pp_funcs->display_configuration_change(
 				adev->powerplay.pp_handle,
 				&adev->pm.pm_display_cfg);
 
@@ -264,22 +269,26 @@ bool dm_pp_get_clock_levels_by_type(
 	struct amd_pp_simple_clock_info validation_clks = { 0 };
 	uint32_t i;
 
-	if (amd_powerplay_get_clock_by_type(pp_handle,
+	if (adev->powerplay.pp_funcs->get_clock_by_type) {
+		if (adev->powerplay.pp_funcs->get_clock_by_type(pp_handle,
 			dc_to_pp_clock_type(clk_type), &pp_clks)) {
 		/* Error in pplib. Provide default values. */
-		get_default_clock_levels(clk_type, dc_clks);
-		return true;
+			get_default_clock_levels(clk_type, dc_clks);
+			return true;
+		}
 	}
 
 	pp_to_dc_clock_levels(&pp_clks, dc_clks, clk_type);
 
-	if (amd_powerplay_get_display_mode_validation_clocks(pp_handle,
-			&validation_clks)) {
-		/* Error in pplib. Provide default values. */
-		DRM_INFO("DM_PPLIB: Warning: using default validation clocks!\n");
-		validation_clks.engine_max_clock = 72000;
-		validation_clks.memory_max_clock = 80000;
-		validation_clks.level = 0;
+	if (adev->powerplay.pp_funcs->get_display_mode_validation_clocks) {
+		if (adev->powerplay.pp_funcs->get_display_mode_validation_clocks(
+						pp_handle, &validation_clks)) {
+			/* Error in pplib. Provide default values. */
+			DRM_INFO("DM_PPLIB: Warning: using default validation clocks!\n");
+			validation_clks.engine_max_clock = 72000;
+			validation_clks.memory_max_clock = 80000;
+			validation_clks.level = 0;
+		}
 	}
 
 	DRM_INFO("DM_PPLIB: Validation clocks:\n");
diff --git a/drivers/gpu/drm/amd/display/dc/basics/Makefile b/drivers/gpu/drm/amd/display/dc/basics/Makefile
index 6af8c8a9ad80..bca33bd9a0d2 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/basics/Makefile
@@ -24,7 +24,7 @@
 # It provides the general basic services required by other DAL
 # subcomponents.
 
-BASICS = conversion.o fixpt31_32.o fixpt32_32.o grph_object_id.o \
+BASICS = conversion.o fixpt31_32.o fixpt32_32.o \
 	logger.o log_helpers.o vector.o
 
 AMD_DAL_BASICS = $(addprefix $(AMDDALPATH)/dc/basics/,$(BASICS))
diff --git a/drivers/gpu/drm/amd/display/dc/basics/conversion.c b/drivers/gpu/drm/amd/display/dc/basics/conversion.c
index 23c9a0ec0181..310964915a83 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/conversion.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/conversion.c
@@ -46,7 +46,7 @@ uint16_t fixed_point_to_int_frac(
 			arg));
 
 	if (d <= (uint16_t)(1 << integer_bits) - (1 / (uint16_t)divisor))
-		numerator = (uint16_t)dal_fixed31_32_floor(
+		numerator = (uint16_t)dal_fixed31_32_round(
 			dal_fixed31_32_mul_int(
 				arg,
 				divisor));
diff --git a/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c b/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
index 26936892c6f5..011a97f82fb6 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
@@ -554,6 +554,22 @@ static inline uint32_t ux_dy(
 	return result | fractional_part;
 }
 
+static inline uint32_t clamp_ux_dy(
+	int64_t value,
+	uint32_t integer_bits,
+	uint32_t fractional_bits,
+	uint32_t min_clamp)
+{
+	uint32_t truncated_val = ux_dy(value, integer_bits, fractional_bits);
+
+	if (value >= (1LL << (integer_bits + FIXED31_32_BITS_PER_FRACTIONAL_PART)))
+		return (1 << (integer_bits + fractional_bits)) - 1;
+	else if (truncated_val > min_clamp)
+		return truncated_val;
+	else
+		return min_clamp;
+}
+
 uint32_t dal_fixed31_32_u2d19(
 	struct fixed31_32 arg)
 {
@@ -565,3 +581,15 @@ uint32_t dal_fixed31_32_u0d19(
 {
 	return ux_dy(arg.value, 0, 19);
 }
+
+uint32_t dal_fixed31_32_clamp_u0d14(
+	struct fixed31_32 arg)
+{
+	return clamp_ux_dy(arg.value, 0, 14, 1);
+}
+
+uint32_t dal_fixed31_32_clamp_u0d10(
+	struct fixed31_32 arg)
+{
+	return clamp_ux_dy(arg.value, 0, 10, 1);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c b/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
index 6e43168fbdd6..854678a0c54b 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/log_helpers.c
@@ -83,15 +83,11 @@ void dc_conn_log(struct dc_context *ctx,
 			link->link_index);
 
 	va_start(args, msg);
-	entry.buf_offset += dm_log_to_buffer(
-		&entry.buf[entry.buf_offset],
-		LOG_MAX_LINE_SIZE - entry.buf_offset,
-		msg, args);
+	dm_logger_append_va(&entry, msg, args);
 
-	if (entry.buf[strlen(entry.buf) - 1] == '\n') {
-		entry.buf[strlen(entry.buf) - 1] = '\0';
+	if (entry.buf_offset > 0 &&
+	    entry.buf[entry.buf_offset - 1] == '\n')
 		entry.buf_offset--;
-	}
 
 	if (hex_data)
 		for (i = 0; i < hex_data_count; i++)
diff --git a/drivers/gpu/drm/amd/display/dc/basics/logger.c b/drivers/gpu/drm/amd/display/dc/basics/logger.c
index e04e8ecd4874..180a9d69d351 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/logger.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/logger.c
@@ -70,9 +70,8 @@ static bool construct(struct dc_context *ctx, struct dal_logger *logger,
 {
 	/* malloc buffer and init offsets */
 	logger->log_buffer_size = DAL_LOGGER_BUFFER_MAX_SIZE;
-	logger->log_buffer = (char *)kzalloc(logger->log_buffer_size * sizeof(char),
-					     GFP_KERNEL);
-
+	logger->log_buffer = kcalloc(logger->log_buffer_size, sizeof(char),
+				     GFP_KERNEL);
 	if (!logger->log_buffer)
 		return false;
 
@@ -312,6 +311,18 @@ void dm_logger_append(
 	struct log_entry *entry,
 	const char *msg,
 	...)
+{
+	va_list args;
+
+	va_start(args, msg);
+	dm_logger_append_va(entry, msg, args);
+	va_end(args);
+}
+
+void dm_logger_append_va(
+	struct log_entry *entry,
+	const char *msg,
+	va_list args)
 {
 	struct dal_logger *logger;
 
@@ -326,11 +337,8 @@ void dm_logger_append(
 		dal_logger_should_log(logger, entry->type)) {
 
 		uint32_t size;
-		va_list args;
 		char buffer[LOG_MAX_LINE_SIZE];
 
-		va_start(args, msg);
-
 		size = dm_log_to_buffer(
 			buffer, LOG_MAX_LINE_SIZE, msg, args);
 
@@ -339,8 +347,6 @@ void dm_logger_append(
 		} else {
 			append_entry(entry, "LOG_ERROR, line too long\n", 27);
 		}
-
-		va_end(args);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
index 86e6438c5cf3..c00e405b63e8 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
@@ -190,6 +190,7 @@ static struct graphics_object_id bios_parser_get_connector_id(
 	struct bios_parser *bp = BP_FROM_DCB(dcb);
 	struct graphics_object_id object_id = dal_graphics_object_id_init(
 		0, ENUM_ID_UNKNOWN, OBJECT_TYPE_UNKNOWN);
+	uint16_t id;
 
 	uint32_t connector_table_offset = bp->object_info_tbl_offset
 		+ le16_to_cpu(bp->object_info_tbl.v1_1->usConnectorObjectTableOffset);
@@ -197,12 +198,19 @@ static struct graphics_object_id bios_parser_get_connector_id(
 	ATOM_OBJECT_TABLE *tbl =
 		GET_IMAGE(ATOM_OBJECT_TABLE, connector_table_offset);
 
-	if (tbl && tbl->ucNumberOfObjects > i) {
-		const uint16_t id = le16_to_cpu(tbl->asObjects[i].usObjectID);
+	if (!tbl) {
+		dm_error("Can't get connector table from atom bios.\n");
+		return object_id;
+	}
 
-		object_id = object_id_from_bios_object_id(id);
+	if (tbl->ucNumberOfObjects <= i) {
+		dm_error("Can't find connector id %d in connector table of size %d.\n",
+			 i, tbl->ucNumberOfObjects);
+		return object_id;
 	}
 
+	id = le16_to_cpu(tbl->asObjects[i].usObjectID);
+	object_id = object_id_from_bios_object_id(id);
 	return object_id;
 }
 
@@ -2254,6 +2262,52 @@ static enum bp_result get_gpio_i2c_info(struct bios_parser *bp,
 	return BP_RESULT_OK;
 }
 
+static bool dal_graphics_object_id_is_valid(struct graphics_object_id id)
+{
+	bool rc = true;
+
+	switch (id.type) {
+	case OBJECT_TYPE_UNKNOWN:
+		rc = false;
+		break;
+	case OBJECT_TYPE_GPU:
+	case OBJECT_TYPE_ENGINE:
+		/* do NOT check for id.id == 0 */
+		if (id.enum_id == ENUM_ID_UNKNOWN)
+			rc = false;
+		break;
+	default:
+		if (id.id == 0 || id.enum_id == ENUM_ID_UNKNOWN)
+			rc = false;
+		break;
+	}
+
+	return rc;
+}
+
+static bool dal_graphics_object_id_is_equal(
+	struct graphics_object_id id1,
+	struct graphics_object_id id2)
+{
+	if (false == dal_graphics_object_id_is_valid(id1)) {
+		dm_output_to_console(
+		"%s: Warning: comparing invalid object 'id1'!\n", __func__);
+		return false;
+	}
+
+	if (false == dal_graphics_object_id_is_valid(id2)) {
+		dm_output_to_console(
+		"%s: Warning: comparing invalid object 'id2'!\n", __func__);
+		return false;
+	}
+
+	if (id1.id == id2.id && id1.enum_id == id2.enum_id
+		&& id1.type == id2.type)
+		return true;
+
+	return false;
+}
+
 static ATOM_OBJECT *get_bios_object(struct bios_parser *bp,
 	struct graphics_object_id id)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
index 3f7b2dabc2b0..4b5fdd577848 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
@@ -387,6 +387,7 @@ static void init_transmitter_control(struct bios_parser *bp)
 		bp->cmd_tbl.transmitter_control = transmitter_control_v1_6;
 		break;
 	default:
+		dm_output_to_console("Don't have transmitter_control for v%d\n", crev);
 		bp->cmd_tbl.transmitter_control = NULL;
 		break;
 	}
@@ -910,6 +911,8 @@ static void init_set_pixel_clock(struct bios_parser *bp)
 		bp->cmd_tbl.set_pixel_clock = set_pixel_clock_v7;
 		break;
 	default:
+		dm_output_to_console("Don't have set_pixel_clock for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(SetPixelClock));
 		bp->cmd_tbl.set_pixel_clock = NULL;
 		break;
 	}
@@ -1227,6 +1230,8 @@ static void init_enable_spread_spectrum_on_ppll(struct bios_parser *bp)
 				enable_spread_spectrum_on_ppll_v3;
 		break;
 	default:
+		dm_output_to_console("Don't have enable_spread_spectrum_on_ppll for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(EnableSpreadSpectrumOnPPLL));
 		bp->cmd_tbl.enable_spread_spectrum_on_ppll = NULL;
 		break;
 	}
@@ -1422,6 +1427,8 @@ static void init_adjust_display_pll(struct bios_parser *bp)
 		bp->cmd_tbl.adjust_display_pll = adjust_display_pll_v3;
 		break;
 	default:
+		dm_output_to_console("Don't have adjust_display_pll for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(AdjustDisplayPll));
 		bp->cmd_tbl.adjust_display_pll = NULL;
 		break;
 	}
@@ -1695,6 +1702,8 @@ static void init_set_crtc_timing(struct bios_parser *bp)
 					set_crtc_using_dtd_timing_v3;
 			break;
 		default:
+			dm_output_to_console("Don't have set_crtc_timing for dtd v%d\n",
+				 dtd_version);
 			bp->cmd_tbl.set_crtc_timing = NULL;
 			break;
 		}
@@ -1704,6 +1713,8 @@ static void init_set_crtc_timing(struct bios_parser *bp)
 			bp->cmd_tbl.set_crtc_timing = set_crtc_timing_v1;
 			break;
 		default:
+			dm_output_to_console("Don't have set_crtc_timing for v%d\n",
+				 BIOS_CMD_TABLE_PARA_REVISION(SetCRTC_Timing));
 			bp->cmd_tbl.set_crtc_timing = NULL;
 			break;
 		}
@@ -1890,6 +1901,8 @@ static void init_select_crtc_source(struct bios_parser *bp)
 		bp->cmd_tbl.select_crtc_source = select_crtc_source_v3;
 		break;
 	default:
+		dm_output_to_console("Don't select_crtc_source enable_crtc for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(SelectCRTC_Source));
 		bp->cmd_tbl.select_crtc_source = NULL;
 		break;
 	}
@@ -1997,6 +2010,8 @@ static void init_enable_crtc(struct bios_parser *bp)
 		bp->cmd_tbl.enable_crtc = enable_crtc_v1;
 		break;
 	default:
+		dm_output_to_console("Don't have enable_crtc for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(EnableCRTC));
 		bp->cmd_tbl.enable_crtc = NULL;
 		break;
 	}
@@ -2103,6 +2118,8 @@ static void init_program_clock(struct bios_parser *bp)
 		bp->cmd_tbl.program_clock = program_clock_v6;
 		break;
 	default:
+		dm_output_to_console("Don't have program_clock for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(SetPixelClock));
 		bp->cmd_tbl.program_clock = NULL;
 		break;
 	}
@@ -2324,6 +2341,8 @@ static void init_enable_disp_power_gating(
 				enable_disp_power_gating_v2_1;
 		break;
 	default:
+		dm_output_to_console("Don't enable_disp_power_gating enable_crtc for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(EnableDispPowerGating));
 		bp->cmd_tbl.enable_disp_power_gating = NULL;
 		break;
 	}
@@ -2371,6 +2390,8 @@ static void init_set_dce_clock(struct bios_parser *bp)
 		bp->cmd_tbl.set_dce_clock = set_dce_clock_v2_1;
 		break;
 	default:
+		dm_output_to_console("Don't have set_dce_clock for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(SetDCEClock));
 		bp->cmd_tbl.set_dce_clock = NULL;
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
index ba68693758a7..fea5e83736fd 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
@@ -118,6 +118,7 @@ static void init_dig_encoder_control(struct bios_parser *bp)
 		bp->cmd_tbl.dig_encoder_control = encoder_control_digx_v1_5;
 		break;
 	default:
+		dm_output_to_console("Don't have dig_encoder_control for v%d\n", version);
 		bp->cmd_tbl.dig_encoder_control = NULL;
 		break;
 	}
@@ -205,6 +206,7 @@ static void init_transmitter_control(struct bios_parser *bp)
 		bp->cmd_tbl.transmitter_control = transmitter_control_v1_6;
 		break;
 	default:
+		dm_output_to_console("Don't have transmitter_control for v%d\n", crev);
 		bp->cmd_tbl.transmitter_control = NULL;
 		break;
 	}
@@ -268,6 +270,8 @@ static void init_set_pixel_clock(struct bios_parser *bp)
 		bp->cmd_tbl.set_pixel_clock = set_pixel_clock_v7;
 		break;
 	default:
+		dm_output_to_console("Don't have set_pixel_clock for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(setpixelclock));
 		bp->cmd_tbl.set_pixel_clock = NULL;
 		break;
 	}
@@ -379,6 +383,7 @@ static void init_set_crtc_timing(struct bios_parser *bp)
 			set_crtc_using_dtd_timing_v3;
 		break;
 	default:
+		dm_output_to_console("Don't have set_crtc_timing for v%d\n", dtd_version);
 		bp->cmd_tbl.set_crtc_timing = NULL;
 		break;
 	}
@@ -498,6 +503,8 @@ static void init_select_crtc_source(struct bios_parser *bp)
 		bp->cmd_tbl.select_crtc_source = select_crtc_source_v3;
 		break;
 	default:
+		dm_output_to_console("Don't select_crtc_source enable_crtc for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(selectcrtc_source));
 		bp->cmd_tbl.select_crtc_source = NULL;
 		break;
 	}
@@ -565,6 +572,8 @@ static void init_enable_crtc(struct bios_parser *bp)
 		bp->cmd_tbl.enable_crtc = enable_crtc_v1;
 		break;
 	default:
+		dm_output_to_console("Don't have enable_crtc for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(enablecrtc));
 		bp->cmd_tbl.enable_crtc = NULL;
 		break;
 	}
@@ -661,6 +670,8 @@ static void init_enable_disp_power_gating(
 				enable_disp_power_gating_v2_1;
 		break;
 	default:
+		dm_output_to_console("Don't enable_disp_power_gating enable_crtc for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(enabledisppowergating));
 		bp->cmd_tbl.enable_disp_power_gating = NULL;
 		break;
 	}
@@ -710,6 +721,8 @@ static void init_set_dce_clock(struct bios_parser *bp)
 		bp->cmd_tbl.set_dce_clock = set_dce_clock_v2_1;
 		break;
 	default:
+		dm_output_to_console("Don't have set_dce_clock for v%d\n",
+			 BIOS_CMD_TABLE_PARA_REVISION(setdceclock));
 		bp->cmd_tbl.set_dce_clock = NULL;
 		break;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
index 6347712db834..2e11fac2a63d 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c
@@ -29,6 +29,15 @@
 #include "core_types.h"
 #include "dal_asic_id.h"
 
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
 /*******************************************************************************
  * Private Functions
  ******************************************************************************/
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.c
index 626f9cf8aad2..5e2ea12fbb73 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_auto.c
@@ -27,6 +27,15 @@
 #include "dcn_calc_auto.h"
 #include "dcn_calc_math.h"
 
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
 /*REVISION#250*/
 void scaler_settings_calculation(struct dcn_bw_internal_vars *v)
 {
@@ -773,11 +782,11 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 					v->dst_y_after_scaler = 0.0;
 				}
 				v->time_calc = 24.0 / v->projected_dcfclk_deep_sleep;
-				v->v_update_offset[k] =dcn_bw_ceil2(v->htotal[k] / 4.0, 1.0);
+				v->v_update_offset[k][j] = dcn_bw_ceil2(v->htotal[k] / 4.0, 1.0);
 				v->total_repeater_delay = v->max_inter_dcn_tile_repeaters * (2.0 / (v->required_dispclk[i][j] / (j + 1)) + 3.0 / v->required_dispclk[i][j]);
-				v->v_update_width[k] = (14.0 / v->projected_dcfclk_deep_sleep + 12.0 / (v->required_dispclk[i][j] / (j + 1)) + v->total_repeater_delay) * v->pixel_clock[k];
-				v->v_ready_offset[k] =dcn_bw_max2(150.0 / (v->required_dispclk[i][j] / (j + 1)), v->total_repeater_delay + 20.0 / v->projected_dcfclk_deep_sleep + 10.0 / (v->required_dispclk[i][j] / (j + 1))) * v->pixel_clock[k];
-				v->time_setup = (v->v_update_offset[k] + v->v_update_width[k] + v->v_ready_offset[k]) / v->pixel_clock[k];
+				v->v_update_width[k][j] = (14.0 / v->projected_dcfclk_deep_sleep + 12.0 / (v->required_dispclk[i][j] / (j + 1)) + v->total_repeater_delay) * v->pixel_clock[k];
+				v->v_ready_offset[k][j] = dcn_bw_max2(150.0 / (v->required_dispclk[i][j] / (j + 1)), v->total_repeater_delay + 20.0 / v->projected_dcfclk_deep_sleep + 10.0 / (v->required_dispclk[i][j] / (j + 1))) * v->pixel_clock[k];
+				v->time_setup = (v->v_update_offset[k][j] + v->v_update_width[k][j] + v->v_ready_offset[k][j]) / v->pixel_clock[k];
 				v->extra_latency = v->urgent_round_trip_and_out_of_order_latency_per_state[i] + (v->total_number_of_active_dpp[i][j] * v->pixel_chunk_size_in_kbyte + v->total_number_of_dcc_active_dpp[i][j] * v->meta_chunk_size) * 1024.0 / v->return_bw_per_state[i];
 				if (v->pte_enable == dcn_bw_yes) {
 					v->extra_latency = v->extra_latency + v->total_number_of_active_dpp[i][j] * v->pte_chunk_size * 1024.0 / v->return_bw_per_state[i];
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c
index b6abe0f3bb15..7600a4a4abc7 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calc_math.c
@@ -25,37 +25,44 @@
 
 #include "dcn_calc_math.h"
 
+#define isNaN(number) ((number) != (number))
+
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
 float dcn_bw_mod(const float arg1, const float arg2)
 {
-	if (arg1 != arg1)
+	if (isNaN(arg1))
 		return arg2;
-	if (arg2 != arg2)
+	if (isNaN(arg2))
 		return arg1;
 	return arg1 - arg1 * ((int) (arg1 / arg2));
 }
 
 float dcn_bw_min2(const float arg1, const float arg2)
 {
-	if (arg1 != arg1)
+	if (isNaN(arg1))
 		return arg2;
-	if (arg2 != arg2)
+	if (isNaN(arg2))
 		return arg1;
 	return arg1 < arg2 ? arg1 : arg2;
 }
 
 unsigned int dcn_bw_max(const unsigned int arg1, const unsigned int arg2)
 {
-	if (arg1 != arg1)
-		return arg2;
-	if (arg2 != arg2)
-		return arg1;
 	return arg1 > arg2 ? arg1 : arg2;
 }
 float dcn_bw_max2(const float arg1, const float arg2)
 {
-	if (arg1 != arg1)
+	if (isNaN(arg1))
 		return arg2;
-	if (arg2 != arg2)
+	if (isNaN(arg2))
 		return arg1;
 	return arg1 > arg2 ? arg1 : arg2;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
index b142629a1058..331891c2c71a 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
@@ -33,6 +33,15 @@
 #include "dcn10/dcn10_resource.h"
 #include "dcn_calc_math.h"
 
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
 /* Defaults from spreadsheet rev#247 */
 const struct dcn_soc_bounding_box dcn10_soc_defaults = {
 		/* latencies */
@@ -432,25 +441,13 @@ static void dcn_bw_calc_rq_dlg_ttu(
 	input.clks_cfg.dcfclk_mhz = v->dcfclk;
 	input.clks_cfg.dispclk_mhz = v->dispclk;
 	input.clks_cfg.dppclk_mhz = v->dppclk;
-	input.clks_cfg.refclk_mhz = dc->res_pool->ref_clock_inKhz/1000;
+	input.clks_cfg.refclk_mhz = dc->res_pool->ref_clock_inKhz / 1000.0;
 	input.clks_cfg.socclk_mhz = v->socclk;
 	input.clks_cfg.voltage = v->voltage_level;
 //	dc->dml.logger = pool->base.logger;
 	input.dout.output_format = (v->output_format[in_idx] == dcn_bw_420) ? dm_420 : dm_444;
 	input.dout.output_type  = (v->output[in_idx] == dcn_bw_hdmi) ? dm_hdmi : dm_dp;
 	//input[in_idx].dout.output_standard;
-	switch (v->output_deep_color[in_idx]) {
-	case dcn_bw_encoder_12bpc:
-		input.dout.output_bpc = dm_out_12;
-	break;
-	case dcn_bw_encoder_10bpc:
-		input.dout.output_bpc = dm_out_10;
-	break;
-	case dcn_bw_encoder_8bpc:
-	default:
-		input.dout.output_bpc = dm_out_8;
-	break;
-	}
 
 	/*todo: soc->sr_enter_plus_exit_time??*/
 	dlg_sys_param.t_srx_delay_us = dc->dcn_ip->dcfclk_cstate_latency / v->dcf_clk_deep_sleep;
@@ -890,6 +887,17 @@ bool dcn_validate_bandwidth(
 						+ pipe->bottom_pipe->plane_res.scl_data.recout.width;
 			}
 
+			if (pipe->plane_state->rotation % 2 == 0) {
+				ASSERT(pipe->plane_res.scl_data.ratios.horz.value != dal_fixed31_32_one.value
+					|| v->scaler_rec_out_width[input_idx] == v->viewport_width[input_idx]);
+				ASSERT(pipe->plane_res.scl_data.ratios.vert.value != dal_fixed31_32_one.value
+					|| v->scaler_recout_height[input_idx] == v->viewport_height[input_idx]);
+			} else {
+				ASSERT(pipe->plane_res.scl_data.ratios.horz.value != dal_fixed31_32_one.value
+					|| v->scaler_recout_height[input_idx] == v->viewport_width[input_idx]);
+				ASSERT(pipe->plane_res.scl_data.ratios.vert.value != dal_fixed31_32_one.value
+					|| v->scaler_rec_out_width[input_idx] == v->viewport_height[input_idx]);
+			}
 			v->dcc_enable[input_idx] = pipe->plane_state->dcc.enable ? dcn_bw_yes : dcn_bw_no;
 			v->source_pixel_format[input_idx] = tl_pixel_format_to_bw_defs(
 					pipe->plane_state->format);
@@ -1006,9 +1014,9 @@ bool dcn_validate_bandwidth(
 			if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state)
 				continue;
 
-			pipe->pipe_dlg_param.vupdate_width = v->v_update_width[input_idx];
-			pipe->pipe_dlg_param.vupdate_offset = v->v_update_offset[input_idx];
-			pipe->pipe_dlg_param.vready_offset = v->v_ready_offset[input_idx];
+			pipe->pipe_dlg_param.vupdate_width = v->v_update_width[input_idx][v->dpp_per_plane[input_idx] == 2 ? 1 : 0];
+			pipe->pipe_dlg_param.vupdate_offset = v->v_update_offset[input_idx][v->dpp_per_plane[input_idx] == 2 ? 1 : 0];
+			pipe->pipe_dlg_param.vready_offset = v->v_ready_offset[input_idx][v->dpp_per_plane[input_idx] == 2 ? 1 : 0];
 			pipe->pipe_dlg_param.vstartup_start = v->v_startup[input_idx];
 
 			pipe->pipe_dlg_param.htotal = pipe->stream->timing.h_total;
@@ -1034,6 +1042,8 @@ bool dcn_validate_bandwidth(
 			if (pipe->plane_state) {
 				struct pipe_ctx *hsplit_pipe = pipe->bottom_pipe;
 
+				pipe->plane_state->update_flags.bits.full_update = 1;
+
 				if (v->dpp_per_plane[input_idx] == 2 ||
 					((pipe->stream->view_format ==
 					  VIEW_3D_FORMAT_SIDE_BY_SIDE ||
@@ -1045,9 +1055,9 @@ bool dcn_validate_bandwidth(
 					 TIMING_3D_FORMAT_SIDE_BY_SIDE))) {
 					if (hsplit_pipe && hsplit_pipe->plane_state == pipe->plane_state) {
 						/* update previously split pipe */
-						hsplit_pipe->pipe_dlg_param.vupdate_width = v->v_update_width[input_idx];
-						hsplit_pipe->pipe_dlg_param.vupdate_offset = v->v_update_offset[input_idx];
-						hsplit_pipe->pipe_dlg_param.vready_offset = v->v_ready_offset[input_idx];
+						hsplit_pipe->pipe_dlg_param.vupdate_width = v->v_update_width[input_idx][v->dpp_per_plane[input_idx] == 2 ? 1 : 0];
+						hsplit_pipe->pipe_dlg_param.vupdate_offset = v->v_update_offset[input_idx][v->dpp_per_plane[input_idx] == 2 ? 1 : 0];
+						hsplit_pipe->pipe_dlg_param.vready_offset = v->v_ready_offset[input_idx][v->dpp_per_plane[input_idx] == 2 ? 1 : 0];
 						hsplit_pipe->pipe_dlg_param.vstartup_start = v->v_startup[input_idx];
 
 						hsplit_pipe->pipe_dlg_param.htotal = pipe->stream->timing.h_total;
@@ -1073,6 +1083,9 @@ bool dcn_validate_bandwidth(
 					hsplit_pipe->stream = NULL;
 					hsplit_pipe->top_pipe = NULL;
 					hsplit_pipe->bottom_pipe = NULL;
+					/* Clear plane_res and stream_res */
+					memset(&hsplit_pipe->plane_res, 0, sizeof(hsplit_pipe->plane_res));
+					memset(&hsplit_pipe->stream_res, 0, sizeof(hsplit_pipe->stream_res));
 					resource_build_scaling_params(pipe);
 				}
 				/* for now important to do this after pipe split for building e2e params */
@@ -1240,40 +1253,62 @@ unsigned int dcn_find_dcfclk_suits_all(
 	return dcf_clk;
 }
 
+static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks)
+{
+	int i;
+
+	if (clks->num_levels == 0)
+		return false;
+
+	for (i = 0; i < clks->num_levels; i++)
+		/* Ensure that the result is sane */
+		if (clks->data[i].clocks_in_khz == 0)
+			return false;
+
+	return true;
+}
+
 void dcn_bw_update_from_pplib(struct dc *dc)
 {
 	struct dc_context *ctx = dc->ctx;
-	struct dm_pp_clock_levels_with_voltage clks = {0};
+	struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0};
+	bool res;
 
 	kernel_fpu_begin();
 
 	/* TODO: This is not the proper way to obtain fabric_and_dram_bandwidth, should be min(fclk, memclk) */
-
-	if (dm_pp_get_clock_levels_by_type_with_voltage(
-			ctx, DM_PP_CLOCK_TYPE_FCLK, &clks) &&
-			clks.num_levels != 0) {
-		ASSERT(clks.num_levels >= 3);
-		dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = 32 * (clks.data[0].clocks_in_khz / 1000.0) / 1000.0;
-		if (clks.num_levels > 2) {
-			dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = dc->dcn_soc->number_of_channels *
-					(clks.data[clks.num_levels - 3].clocks_in_khz / 1000.0) * ddr4_dram_factor_single_Channel / 1000.0;
-		} else {
-			dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = dc->dcn_soc->number_of_channels *
-					(clks.data[clks.num_levels - 2].clocks_in_khz / 1000.0) * ddr4_dram_factor_single_Channel / 1000.0;
-		}
+	res = dm_pp_get_clock_levels_by_type_with_voltage(
+			ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks);
+
+	if (res)
+		res = verify_clock_values(&fclks);
+
+	if (res) {
+		ASSERT(fclks.num_levels >= 3);
+		dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = 32 * (fclks.data[0].clocks_in_khz / 1000.0) / 1000.0;
+		dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = dc->dcn_soc->number_of_channels *
+				(fclks.data[fclks.num_levels - (fclks.num_levels > 2 ? 3 : 2)].clocks_in_khz / 1000.0)
+				* ddr4_dram_factor_single_Channel / 1000.0;
 		dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = dc->dcn_soc->number_of_channels *
-				(clks.data[clks.num_levels - 2].clocks_in_khz / 1000.0) * ddr4_dram_factor_single_Channel / 1000.0;
+				(fclks.data[fclks.num_levels - 2].clocks_in_khz / 1000.0)
+				* ddr4_dram_factor_single_Channel / 1000.0;
 		dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = dc->dcn_soc->number_of_channels *
-				(clks.data[clks.num_levels - 1].clocks_in_khz / 1000.0) * ddr4_dram_factor_single_Channel / 1000.0;
+				(fclks.data[fclks.num_levels - 1].clocks_in_khz / 1000.0)
+				* ddr4_dram_factor_single_Channel / 1000.0;
 	} else
 		BREAK_TO_DEBUGGER();
-	if (dm_pp_get_clock_levels_by_type_with_voltage(
-				ctx, DM_PP_CLOCK_TYPE_DCFCLK, &clks) &&
-				clks.num_levels >= 3) {
-		dc->dcn_soc->dcfclkv_min0p65 = clks.data[0].clocks_in_khz / 1000.0;
-		dc->dcn_soc->dcfclkv_mid0p72 = clks.data[clks.num_levels - 3].clocks_in_khz / 1000.0;
-		dc->dcn_soc->dcfclkv_nom0p8 = clks.data[clks.num_levels - 2].clocks_in_khz / 1000.0;
-		dc->dcn_soc->dcfclkv_max0p9 = clks.data[clks.num_levels - 1].clocks_in_khz / 1000.0;
+
+	res = dm_pp_get_clock_levels_by_type_with_voltage(
+			ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks);
+
+	if (res)
+		res = verify_clock_values(&dcfclks);
+
+	if (res && dcfclks.num_levels >= 3) {
+		dc->dcn_soc->dcfclkv_min0p65 = dcfclks.data[0].clocks_in_khz / 1000.0;
+		dc->dcn_soc->dcfclkv_mid0p72 = dcfclks.data[dcfclks.num_levels - 3].clocks_in_khz / 1000.0;
+		dc->dcn_soc->dcfclkv_nom0p8 = dcfclks.data[dcfclks.num_levels - 2].clocks_in_khz / 1000.0;
+		dc->dcn_soc->dcfclkv_max0p9 = dcfclks.data[dcfclks.num_levels - 1].clocks_in_khz / 1000.0;
 	} else
 		BREAK_TO_DEBUGGER();
 
@@ -1550,35 +1585,6 @@ void dcn_bw_sync_calcs_and_dml(struct dc *dc)
 			dc->dcn_ip->can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one,
 			dc->dcn_ip->bug_forcing_luma_and_chroma_request_to_same_size_fixed,
 			dc->dcn_ip->dcfclk_cstate_latency);
-	dc->dml.soc.vmin.socclk_mhz = dc->dcn_soc->socclk;
-	dc->dml.soc.vmid.socclk_mhz = dc->dcn_soc->socclk;
-	dc->dml.soc.vnom.socclk_mhz = dc->dcn_soc->socclk;
-	dc->dml.soc.vmax.socclk_mhz = dc->dcn_soc->socclk;
-
-	dc->dml.soc.vmin.dcfclk_mhz = dc->dcn_soc->dcfclkv_min0p65;
-	dc->dml.soc.vmid.dcfclk_mhz = dc->dcn_soc->dcfclkv_mid0p72;
-	dc->dml.soc.vnom.dcfclk_mhz = dc->dcn_soc->dcfclkv_nom0p8;
-	dc->dml.soc.vmax.dcfclk_mhz = dc->dcn_soc->dcfclkv_max0p9;
-
-	dc->dml.soc.vmin.dispclk_mhz = dc->dcn_soc->max_dispclk_vmin0p65;
-	dc->dml.soc.vmid.dispclk_mhz = dc->dcn_soc->max_dispclk_vmid0p72;
-	dc->dml.soc.vnom.dispclk_mhz = dc->dcn_soc->max_dispclk_vnom0p8;
-	dc->dml.soc.vmax.dispclk_mhz = dc->dcn_soc->max_dispclk_vmax0p9;
-
-	dc->dml.soc.vmin.dppclk_mhz = dc->dcn_soc->max_dppclk_vmin0p65;
-	dc->dml.soc.vmid.dppclk_mhz = dc->dcn_soc->max_dppclk_vmid0p72;
-	dc->dml.soc.vnom.dppclk_mhz = dc->dcn_soc->max_dppclk_vnom0p8;
-	dc->dml.soc.vmax.dppclk_mhz = dc->dcn_soc->max_dppclk_vmax0p9;
-
-	dc->dml.soc.vmin.phyclk_mhz = dc->dcn_soc->phyclkv_min0p65;
-	dc->dml.soc.vmid.phyclk_mhz = dc->dcn_soc->phyclkv_mid0p72;
-	dc->dml.soc.vnom.phyclk_mhz = dc->dcn_soc->phyclkv_nom0p8;
-	dc->dml.soc.vmax.phyclk_mhz = dc->dcn_soc->phyclkv_max0p9;
-
-	dc->dml.soc.vmin.dram_bw_per_chan_gbps = dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65;
-	dc->dml.soc.vmid.dram_bw_per_chan_gbps = dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72;
-	dc->dml.soc.vnom.dram_bw_per_chan_gbps = dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8;
-	dc->dml.soc.vmax.dram_bw_per_chan_gbps = dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9;
 
 	dc->dml.soc.sr_exit_time_us = dc->dcn_soc->sr_exit_time;
 	dc->dml.soc.sr_enter_plus_exit_time_us = dc->dcn_soc->sr_enter_plus_exit_time;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 7240db2e6f09..12868c769606 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -54,6 +54,13 @@
 /*******************************************************************************
  * Private functions
  ******************************************************************************/
+
+static inline void elevate_update_type(enum surface_update_type *original, enum surface_update_type new)
+{
+	if (new > *original)
+		*original = new;
+}
+
 static void destroy_links(struct dc *dc)
 {
 	uint32_t i;
@@ -157,7 +164,7 @@ static bool create_links(
 	return false;
 }
 
-static bool stream_adjust_vmin_vmax(struct dc *dc,
+bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 		struct dc_stream_state **streams, int num_streams,
 		int vmin, int vmax)
 {
@@ -182,7 +189,7 @@ static bool stream_adjust_vmin_vmax(struct dc *dc,
 	return ret;
 }
 
-static bool stream_get_crtc_position(struct dc *dc,
+bool dc_stream_get_crtc_position(struct dc *dc,
 		struct dc_stream_state **streams, int num_streams,
 		unsigned int *v_pos, unsigned int *nom_v_pos)
 {
@@ -207,45 +214,7 @@ static bool stream_get_crtc_position(struct dc *dc,
 	return ret;
 }
 
-static bool set_gamut_remap(struct dc *dc, const struct dc_stream_state *stream)
-{
-	int i = 0;
-	bool ret = false;
-	struct pipe_ctx *pipes;
-
-	for (i = 0; i < MAX_PIPES; i++) {
-		if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream) {
-			pipes = &dc->current_state->res_ctx.pipe_ctx[i];
-			dc->hwss.program_gamut_remap(pipes);
-			ret = true;
-		}
-	}
-
-	return ret;
-}
-
-static bool program_csc_matrix(struct dc *dc, struct dc_stream_state *stream)
-{
-	int i = 0;
-	bool ret = false;
-	struct pipe_ctx *pipes;
-
-	for (i = 0; i < MAX_PIPES; i++) {
-		if (dc->current_state->res_ctx.pipe_ctx[i].stream
-				== stream) {
-
-			pipes = &dc->current_state->res_ctx.pipe_ctx[i];
-			dc->hwss.program_csc_matrix(pipes,
-			stream->output_color_space,
-			stream->csc_color_matrix.matrix);
-			ret = true;
-		}
-	}
-
-	return ret;
-}
-
-static void set_static_screen_events(struct dc *dc,
+void dc_stream_set_static_screen_events(struct dc *dc,
 		struct dc_stream_state **streams,
 		int num_streams,
 		const struct dc_static_screen_events *events)
@@ -270,177 +239,6 @@ static void set_static_screen_events(struct dc *dc,
 	dc->hwss.set_static_screen_control(pipes_affected, num_pipes_affected, events);
 }
 
-static void set_drive_settings(struct dc *dc,
-		struct link_training_settings *lt_settings,
-		const struct dc_link *link)
-{
-
-	int i;
-
-	for (i = 0; i < dc->link_count; i++) {
-		if (dc->links[i] == link)
-			break;
-	}
-
-	if (i >= dc->link_count)
-		ASSERT_CRITICAL(false);
-
-	dc_link_dp_set_drive_settings(dc->links[i], lt_settings);
-}
-
-static void perform_link_training(struct dc *dc,
-		struct dc_link_settings *link_setting,
-		bool skip_video_pattern)
-{
-	int i;
-
-	for (i = 0; i < dc->link_count; i++)
-		dc_link_dp_perform_link_training(
-			dc->links[i],
-			link_setting,
-			skip_video_pattern);
-}
-
-static void set_preferred_link_settings(struct dc *dc,
-		struct dc_link_settings *link_setting,
-		struct dc_link *link)
-{
-	link->preferred_link_setting = *link_setting;
-	dp_retrain_link_dp_test(link, link_setting, false);
-}
-
-static void enable_hpd(const struct dc_link *link)
-{
-	dc_link_dp_enable_hpd(link);
-}
-
-static void disable_hpd(const struct dc_link *link)
-{
-	dc_link_dp_disable_hpd(link);
-}
-
-
-static void set_test_pattern(
-		struct dc_link *link,
-		enum dp_test_pattern test_pattern,
-		const struct link_training_settings *p_link_settings,
-		const unsigned char *p_custom_pattern,
-		unsigned int cust_pattern_size)
-{
-	if (link != NULL)
-		dc_link_dp_set_test_pattern(
-			link,
-			test_pattern,
-			p_link_settings,
-			p_custom_pattern,
-			cust_pattern_size);
-}
-
-static void set_dither_option(struct dc_stream_state *stream,
-		enum dc_dither_option option)
-{
-	struct bit_depth_reduction_params params;
-	struct dc_link *link = stream->status.link;
-	struct pipe_ctx *pipes = NULL;
-	int i;
-
-	for (i = 0; i < MAX_PIPES; i++) {
-		if (link->dc->current_state->res_ctx.pipe_ctx[i].stream ==
-				stream) {
-			pipes = &link->dc->current_state->res_ctx.pipe_ctx[i];
-			break;
-		}
-	}
-
-	memset(&params, 0, sizeof(params));
-	if (!pipes)
-		return;
-	if (option > DITHER_OPTION_MAX)
-		return;
-
-	stream->dither_option = option;
-
-	resource_build_bit_depth_reduction_params(stream,
-				&params);
-	stream->bit_depth_params = params;
-	pipes->stream_res.opp->funcs->
-		opp_program_bit_depth_reduction(pipes->stream_res.opp, &params);
-}
-
-void set_dpms(
-	struct dc *dc,
-	struct dc_stream_state *stream,
-	bool dpms_off)
-{
-	struct pipe_ctx *pipe_ctx = NULL;
-	int i;
-
-	for (i = 0; i < MAX_PIPES; i++) {
-		if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream) {
-			pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
-			break;
-		}
-	}
-
-	if (!pipe_ctx) {
-		ASSERT(0);
-		return;
-	}
-
-	if (stream->dpms_off != dpms_off) {
-		stream->dpms_off = dpms_off;
-		if (dpms_off)
-			core_link_disable_stream(pipe_ctx,
-					KEEP_ACQUIRED_RESOURCE);
-		else
-			core_link_enable_stream(dc->current_state, pipe_ctx);
-	}
-}
-
-static void allocate_dc_stream_funcs(struct dc  *dc)
-{
-	if (dc->hwss.set_drr != NULL) {
-		dc->stream_funcs.adjust_vmin_vmax =
-				stream_adjust_vmin_vmax;
-	}
-
-	dc->stream_funcs.set_static_screen_events =
-			set_static_screen_events;
-
-	dc->stream_funcs.get_crtc_position =
-			stream_get_crtc_position;
-
-	dc->stream_funcs.set_gamut_remap =
-			set_gamut_remap;
-
-	dc->stream_funcs.program_csc_matrix =
-			program_csc_matrix;
-
-	dc->stream_funcs.set_dither_option =
-			set_dither_option;
-
-	dc->stream_funcs.set_dpms =
-			set_dpms;
-
-	dc->link_funcs.set_drive_settings =
-			set_drive_settings;
-
-	dc->link_funcs.perform_link_training =
-			perform_link_training;
-
-	dc->link_funcs.set_preferred_link_settings =
-			set_preferred_link_settings;
-
-	dc->link_funcs.enable_hpd =
-			enable_hpd;
-
-	dc->link_funcs.disable_hpd =
-			disable_hpd;
-
-	dc->link_funcs.set_test_pattern =
-			set_test_pattern;
-}
-
 static void destruct(struct dc *dc)
 {
 	dc_release_state(dc->current_state);
@@ -485,19 +283,17 @@ static bool construct(struct dc *dc,
 		const struct dc_init_data *init_params)
 {
 	struct dal_logger *logger;
-	struct dc_context *dc_ctx = kzalloc(sizeof(*dc_ctx), GFP_KERNEL);
-	struct bw_calcs_dceip *dc_dceip = kzalloc(sizeof(*dc_dceip),
-						  GFP_KERNEL);
-	struct bw_calcs_vbios *dc_vbios = kzalloc(sizeof(*dc_vbios),
-						  GFP_KERNEL);
+	struct dc_context *dc_ctx;
+	struct bw_calcs_dceip *dc_dceip;
+	struct bw_calcs_vbios *dc_vbios;
 #ifdef CONFIG_DRM_AMD_DC_DCN1_0
-	struct dcn_soc_bounding_box *dcn_soc = kzalloc(sizeof(*dcn_soc),
-						       GFP_KERNEL);
-	struct dcn_ip_params *dcn_ip = kzalloc(sizeof(*dcn_ip), GFP_KERNEL);
+	struct dcn_soc_bounding_box *dcn_soc;
+	struct dcn_ip_params *dcn_ip;
 #endif
 
 	enum dce_version dc_version = DCE_VERSION_UNKNOWN;
 
+	dc_dceip = kzalloc(sizeof(*dc_dceip), GFP_KERNEL);
 	if (!dc_dceip) {
 		dm_error("%s: failed to create dceip\n", __func__);
 		goto fail;
@@ -505,6 +301,7 @@ static bool construct(struct dc *dc,
 
 	dc->bw_dceip = dc_dceip;
 
+	dc_vbios = kzalloc(sizeof(*dc_vbios), GFP_KERNEL);
 	if (!dc_vbios) {
 		dm_error("%s: failed to create vbios\n", __func__);
 		goto fail;
@@ -512,6 +309,7 @@ static bool construct(struct dc *dc,
 
 	dc->bw_vbios = dc_vbios;
 #ifdef CONFIG_DRM_AMD_DC_DCN1_0
+	dcn_soc = kzalloc(sizeof(*dcn_soc), GFP_KERNEL);
 	if (!dcn_soc) {
 		dm_error("%s: failed to create dcn_soc\n", __func__);
 		goto fail;
@@ -519,6 +317,7 @@ static bool construct(struct dc *dc,
 
 	dc->dcn_soc = dcn_soc;
 
+	dcn_ip = kzalloc(sizeof(*dcn_ip), GFP_KERNEL);
 	if (!dcn_ip) {
 		dm_error("%s: failed to create dcn_ip\n", __func__);
 		goto fail;
@@ -527,11 +326,18 @@ static bool construct(struct dc *dc,
 	dc->dcn_ip = dcn_ip;
 #endif
 
+	dc_ctx = kzalloc(sizeof(*dc_ctx), GFP_KERNEL);
 	if (!dc_ctx) {
 		dm_error("%s: failed to create ctx\n", __func__);
 		goto fail;
 	}
 
+	dc_ctx->cgs_device = init_params->cgs_device;
+	dc_ctx->driver_context = init_params->driver;
+	dc_ctx->dc = dc;
+	dc_ctx->asic_id = init_params->asic_id;
+	dc->ctx = dc_ctx;
+
 	dc->current_state = dc_create_state();
 
 	if (!dc->current_state) {
@@ -539,11 +345,6 @@ static bool construct(struct dc *dc,
 		goto fail;
 	}
 
-	dc_ctx->cgs_device = init_params->cgs_device;
-	dc_ctx->driver_context = init_params->driver;
-	dc_ctx->dc = dc;
-	dc_ctx->asic_id = init_params->asic_id;
-
 	/* Create logger */
 	logger = dal_logger_create(dc_ctx, init_params->log_mask);
 
@@ -553,11 +354,11 @@ static bool construct(struct dc *dc,
 		goto fail;
 	}
 	dc_ctx->logger = logger;
-	dc->ctx = dc_ctx;
-	dc->ctx->dce_environment = init_params->dce_environment;
+	dc_ctx->dce_environment = init_params->dce_environment;
 
 	dc_version = resource_parse_asic_id(init_params->asic_id);
-	dc->ctx->dce_version = dc_version;
+	dc_ctx->dce_version = dc_version;
+
 #if defined(CONFIG_DRM_AMD_DC_FBC)
 	dc->ctx->fbc_gpu_addr = init_params->fbc_gpu_addr;
 #endif
@@ -616,8 +417,6 @@ static bool construct(struct dc *dc,
 	if (!create_links(dc, init_params->num_virtual_links))
 		goto fail;
 
-	allocate_dc_stream_funcs(dc);
-
 	return true;
 
 fail:
@@ -686,6 +485,7 @@ struct dc *dc_create(const struct dc_init_data *init_params)
 
 	dc->caps.max_links = dc->link_count;
 	dc->caps.max_audios = dc->res_pool->audio_count;
+	dc->caps.linear_pitch_alignment = 64;
 
 	dc->config = init_params->flags;
 
@@ -712,6 +512,28 @@ void dc_destroy(struct dc **dc)
 	*dc = NULL;
 }
 
+static void enable_timing_multisync(
+		struct dc *dc,
+		struct dc_state *ctx)
+{
+	int i = 0, multisync_count = 0;
+	int pipe_count = dc->res_pool->pipe_count;
+	struct pipe_ctx *multisync_pipes[MAX_PIPES] = { NULL };
+
+	for (i = 0; i < pipe_count; i++) {
+		if (!ctx->res_ctx.pipe_ctx[i].stream ||
+				!ctx->res_ctx.pipe_ctx[i].stream->triggered_crtc_reset.enabled)
+			continue;
+		multisync_pipes[multisync_count] = &ctx->res_ctx.pipe_ctx[i];
+		multisync_count++;
+	}
+
+	if (multisync_count > 1) {
+		dc->hwss.enable_per_frame_crtc_position_reset(
+			dc, multisync_count, multisync_pipes);
+	}
+}
+
 static void program_timing_sync(
 		struct dc *dc,
 		struct dc_state *ctx)
@@ -758,7 +580,7 @@ static void program_timing_sync(
 		for (j = 0; j < group_size; j++) {
 			struct pipe_ctx *temp;
 
-			if (!pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg)) {
+			if (pipe_set[j]->stream_res.tg->funcs->is_blanked && !pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg)) {
 				if (j == 0)
 					break;
 
@@ -771,7 +593,7 @@ static void program_timing_sync(
 
 		/* remove any other unblanked pipes as they have already been synced */
 		for (j = j + 1; j < group_size; j++) {
-			if (!pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg)) {
+			if (pipe_set[j]->stream_res.tg->funcs->is_blanked && !pipe_set[j]->stream_res.tg->funcs->is_blanked(pipe_set[j]->stream_res.tg)) {
 				group_size--;
 				pipe_set[j] = pipe_set[group_size];
 				j--;
@@ -838,7 +660,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	enum dc_status result = DC_ERROR_UNEXPECTED;
 	struct pipe_ctx *pipe;
-	int i, j, k, l;
+	int i, k, l;
 	struct dc_stream_state *dc_streams[MAX_STREAMS] = {0};
 
 	disable_dangling_plane(dc, context);
@@ -849,9 +671,44 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	if (!dcb->funcs->is_accelerated_mode(dcb))
 		dc->hwss.enable_accelerated_mode(dc);
 
+	/* re-program planes for existing stream, in case we need to
+	 * free up plane resource for later use
+	 */
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->streams[i]->mode_changed)
+			continue;
+
+		dc->hwss.apply_ctx_for_surface(
+			dc, context->streams[i],
+			context->stream_status[i].plane_count,
+			context); /* use new pipe config in new context */
+	}
+
+	/* Program hardware */
+	dc->hwss.ready_shared_resources(dc, context);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe = &context->res_ctx.pipe_ctx[i];
+		dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe);
+	}
+
+	result = dc->hwss.apply_ctx_to_hw(dc, context);
+
+	if (result != DC_OK)
+		return result;
+
+	if (context->stream_count > 1) {
+		enable_timing_multisync(dc, context);
+		program_timing_sync(dc, context);
+	}
+
+	/* Program all planes within new context*/
 	for (i = 0; i < context->stream_count; i++) {
 		const struct dc_sink *sink = context->streams[i]->sink;
 
+		if (!context->streams[i]->mode_changed)
+			continue;
+
 		dc->hwss.apply_ctx_for_surface(
 				dc, context->streams[i],
 				context->stream_status[i].plane_count,
@@ -880,27 +737,8 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 				context->streams[i]->timing.pix_clk_khz);
 	}
 
-	dc->hwss.ready_shared_resources(dc, context);
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		pipe = &context->res_ctx.pipe_ctx[i];
-		dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe);
-	}
-	result = dc->hwss.apply_ctx_to_hw(dc, context);
-
-	program_timing_sync(dc, context);
-
 	dc_enable_stereo(dc, context, dc_streams, context->stream_count);
 
-	for (i = 0; i < context->stream_count; i++) {
-		for (j = 0; j < MAX_PIPES; j++) {
-			pipe = &context->res_ctx.pipe_ctx[j];
-
-			if (!pipe->top_pipe && pipe->stream == context->streams[i])
-				dc->hwss.pipe_control_lock(dc, pipe, false);
-		}
-	}
-
 	dc_release_state(dc->current_state);
 
 	dc->current_state = context;
@@ -936,7 +774,6 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context)
 	return (result == DC_OK);
 }
 
-
 bool dc_post_update_surfaces_to_stream(struct dc *dc)
 {
 	int i;
@@ -945,9 +782,13 @@ bool dc_post_update_surfaces_to_stream(struct dc *dc)
 	post_surface_trace(dc);
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++)
-		if (context->res_ctx.pipe_ctx[i].stream == NULL
-				|| context->res_ctx.pipe_ctx[i].plane_state == NULL)
-			dc->hwss.power_down_front_end(dc, i);
+		if (context->res_ctx.pipe_ctx[i].stream == NULL ||
+		    context->res_ctx.pipe_ctx[i].plane_state == NULL) {
+			context->res_ctx.pipe_ctx[i].pipe_idx = i;
+			dc->hwss.disable_plane(dc, &context->res_ctx.pipe_ctx[i]);
+		}
+
+	dc->optimized_required = false;
 
 	/* 3rd param should be true, temp w/a for RV*/
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
@@ -1014,6 +855,7 @@ bool dc_commit_planes_to_stream(
 		flip_addr[i].address = plane_states[i]->address;
 		flip_addr[i].flip_immediate = plane_states[i]->flip_immediate;
 		plane_info[i].color_space = plane_states[i]->color_space;
+		plane_info[i].input_tf = plane_states[i]->input_tf;
 		plane_info[i].format = plane_states[i]->format;
 		plane_info[i].plane_size = plane_states[i]->plane_size;
 		plane_info[i].rotation = plane_states[i]->rotation;
@@ -1118,79 +960,96 @@ static unsigned int pixel_format_to_bpp(enum surface_pixel_format format)
 	}
 }
 
-static enum surface_update_type get_plane_info_update_type(
-		const struct dc_surface_update *u,
-		int surface_index)
+static enum surface_update_type get_plane_info_update_type(const struct dc_surface_update *u)
 {
-	struct dc_plane_info temp_plane_info;
-	memset(&temp_plane_info, 0, sizeof(temp_plane_info));
+	union surface_update_flags *update_flags = &u->surface->update_flags;
 
 	if (!u->plane_info)
 		return UPDATE_TYPE_FAST;
 
-	temp_plane_info = *u->plane_info;
+	if (u->plane_info->color_space != u->surface->color_space)
+		update_flags->bits.color_space_change = 1;
 
-	/* Copy all parameters that will cause a full update
-	 * from current surface, the rest of the parameters
-	 * from provided plane configuration.
-	 * Perform memory compare and special validation
-	 * for those that can cause fast/medium updates
-	 */
+	if (u->plane_info->input_tf != u->surface->input_tf)
+		update_flags->bits.input_tf_change = 1;
 
-	/* Full update parameters */
-	temp_plane_info.color_space = u->surface->color_space;
-	temp_plane_info.dcc = u->surface->dcc;
-	temp_plane_info.horizontal_mirror = u->surface->horizontal_mirror;
-	temp_plane_info.plane_size = u->surface->plane_size;
-	temp_plane_info.rotation = u->surface->rotation;
-	temp_plane_info.stereo_format = u->surface->stereo_format;
-
-	if (surface_index == 0)
-		temp_plane_info.visible = u->plane_info->visible;
-	else
-		temp_plane_info.visible = u->surface->visible;
-
-	if (memcmp(u->plane_info, &temp_plane_info,
-			sizeof(struct dc_plane_info)) != 0)
-		return UPDATE_TYPE_FULL;
+	if (u->plane_info->horizontal_mirror != u->surface->horizontal_mirror)
+		update_flags->bits.horizontal_mirror_change = 1;
+
+	if (u->plane_info->rotation != u->surface->rotation)
+		update_flags->bits.rotation_change = 1;
+
+	if (u->plane_info->stereo_format != u->surface->stereo_format)
+		update_flags->bits.stereo_format_change = 1;
+
+	if (u->plane_info->per_pixel_alpha != u->surface->per_pixel_alpha)
+		update_flags->bits.per_pixel_alpha_change = 1;
+
+	if (u->plane_info->dcc.enable != u->surface->dcc.enable
+			|| u->plane_info->dcc.grph.independent_64b_blks != u->surface->dcc.grph.independent_64b_blks
+			|| u->plane_info->dcc.grph.meta_pitch != u->surface->dcc.grph.meta_pitch)
+		update_flags->bits.dcc_change = 1;
 
 	if (pixel_format_to_bpp(u->plane_info->format) !=
-			pixel_format_to_bpp(u->surface->format)) {
+			pixel_format_to_bpp(u->surface->format))
 		/* different bytes per element will require full bandwidth
 		 * and DML calculation
 		 */
-		return UPDATE_TYPE_FULL;
-	}
+		update_flags->bits.bpp_change = 1;
 
 	if (memcmp(&u->plane_info->tiling_info, &u->surface->tiling_info,
 			sizeof(union dc_tiling_info)) != 0) {
+		update_flags->bits.swizzle_change = 1;
 		/* todo: below are HW dependent, we should add a hook to
 		 * DCE/N resource and validated there.
 		 */
-		if (u->plane_info->tiling_info.gfx9.swizzle != DC_SW_LINEAR) {
+		if (u->plane_info->tiling_info.gfx9.swizzle != DC_SW_LINEAR)
 			/* swizzled mode requires RQ to be setup properly,
 			 * thus need to run DML to calculate RQ settings
 			 */
-			return UPDATE_TYPE_FULL;
-		}
+			update_flags->bits.bandwidth_change = 1;
 	}
 
+	if (update_flags->bits.rotation_change
+			|| update_flags->bits.stereo_format_change
+			|| update_flags->bits.bpp_change
+			|| update_flags->bits.bandwidth_change)
+		return UPDATE_TYPE_FULL;
+
 	return UPDATE_TYPE_MED;
 }
 
-static enum surface_update_type  get_scaling_info_update_type(
+static enum surface_update_type get_scaling_info_update_type(
 		const struct dc_surface_update *u)
 {
+	union surface_update_flags *update_flags = &u->surface->update_flags;
+
 	if (!u->scaling_info)
 		return UPDATE_TYPE_FAST;
 
-	if (u->scaling_info->src_rect.width != u->surface->src_rect.width
-			|| u->scaling_info->src_rect.height != u->surface->src_rect.height
-			|| u->scaling_info->clip_rect.width != u->surface->clip_rect.width
+	if (u->scaling_info->clip_rect.width != u->surface->clip_rect.width
 			|| u->scaling_info->clip_rect.height != u->surface->clip_rect.height
 			|| u->scaling_info->dst_rect.width != u->surface->dst_rect.width
-			|| u->scaling_info->dst_rect.height != u->surface->dst_rect.height)
-		return UPDATE_TYPE_FULL;
+			|| u->scaling_info->dst_rect.height != u->surface->dst_rect.height) {
+		update_flags->bits.scaling_change = 1;
+
+		if ((u->scaling_info->dst_rect.width < u->surface->dst_rect.width
+			|| u->scaling_info->dst_rect.height < u->surface->dst_rect.height)
+				&& (u->scaling_info->dst_rect.width < u->surface->src_rect.width
+					|| u->scaling_info->dst_rect.height < u->surface->src_rect.height))
+			/* Making dst rect smaller requires a bandwidth change */
+			update_flags->bits.bandwidth_change = 1;
+	}
+
+	if (u->scaling_info->src_rect.width != u->surface->src_rect.width
+		|| u->scaling_info->src_rect.height != u->surface->src_rect.height) {
+
+		update_flags->bits.scaling_change = 1;
+		if (u->scaling_info->src_rect.width > u->surface->src_rect.width
+				&& u->scaling_info->src_rect.height > u->surface->src_rect.height)
+			/* Making src rect bigger requires a bandwidth change */
+			update_flags->bits.clock_change = 1;
+	}
 
 	if (u->scaling_info->src_rect.x != u->surface->src_rect.x
 			|| u->scaling_info->src_rect.y != u->surface->src_rect.y
@@ -1198,41 +1057,56 @@ static enum surface_update_type  get_scaling_info_update_type(
 			|| u->scaling_info->clip_rect.y != u->surface->clip_rect.y
 			|| u->scaling_info->dst_rect.x != u->surface->dst_rect.x
 			|| u->scaling_info->dst_rect.y != u->surface->dst_rect.y)
+		update_flags->bits.position_change = 1;
+
+	if (update_flags->bits.clock_change
+			|| update_flags->bits.bandwidth_change)
+		return UPDATE_TYPE_FULL;
+
+	if (update_flags->bits.scaling_change
+			|| update_flags->bits.position_change)
 		return UPDATE_TYPE_MED;
 
 	return UPDATE_TYPE_FAST;
 }
 
-static enum surface_update_type det_surface_update(
-		const struct dc *dc,
-		const struct dc_surface_update *u,
-		int surface_index)
+static enum surface_update_type det_surface_update(const struct dc *dc,
+		const struct dc_surface_update *u)
 {
 	const struct dc_state *context = dc->current_state;
-	enum surface_update_type type = UPDATE_TYPE_FAST;
+	enum surface_update_type type;
 	enum surface_update_type overall_type = UPDATE_TYPE_FAST;
+	union surface_update_flags *update_flags = &u->surface->update_flags;
+
+	update_flags->raw = 0; // Reset all flags
 
-	if (!is_surface_in_context(context, u->surface))
+	if (!is_surface_in_context(context, u->surface)) {
+		update_flags->bits.new_plane = 1;
 		return UPDATE_TYPE_FULL;
+	}
 
-	type = get_plane_info_update_type(u, surface_index);
-	if (overall_type < type)
-		overall_type = type;
+	type = get_plane_info_update_type(u);
+	elevate_update_type(&overall_type, type);
 
 	type = get_scaling_info_update_type(u);
-	if (overall_type < type)
-		overall_type = type;
+	elevate_update_type(&overall_type, type);
+
+	if (u->in_transfer_func)
+		update_flags->bits.in_transfer_func = 1;
 
-	if (u->in_transfer_func ||
-		u->hdr_static_metadata) {
-		if (overall_type < UPDATE_TYPE_MED)
-			overall_type = UPDATE_TYPE_MED;
+	if (u->input_csc_color_matrix)
+		update_flags->bits.input_csc_change = 1;
+
+	if (update_flags->bits.in_transfer_func
+			|| update_flags->bits.input_csc_change) {
+		type = UPDATE_TYPE_MED;
+		elevate_update_type(&overall_type, type);
 	}
 
 	return overall_type;
 }
 
-enum surface_update_type dc_check_update_surfaces_for_stream(
+static enum surface_update_type check_update_surfaces_for_stream(
 		struct dc *dc,
 		struct dc_surface_update *updates,
 		int surface_count,
@@ -1250,18 +1124,38 @@ enum surface_update_type dc_check_update_surfaces_for_stream(
 
 	for (i = 0 ; i < surface_count; i++) {
 		enum surface_update_type type =
-				det_surface_update(dc, &updates[i], i);
+				det_surface_update(dc, &updates[i]);
 
 		if (type == UPDATE_TYPE_FULL)
 			return type;
 
-		if (overall_type < type)
-			overall_type = type;
+		elevate_update_type(&overall_type, type);
 	}
 
 	return overall_type;
 }
 
+enum surface_update_type dc_check_update_surfaces_for_stream(
+		struct dc *dc,
+		struct dc_surface_update *updates,
+		int surface_count,
+		struct dc_stream_update *stream_update,
+		const struct dc_stream_status *stream_status)
+{
+	int i;
+	enum surface_update_type type;
+
+	for (i = 0; i < surface_count; i++)
+		updates[i].surface->update_flags.raw = 0;
+
+	type = check_update_surfaces_for_stream(dc, updates, surface_count, stream_update, stream_status);
+	if (type == UPDATE_TYPE_FULL)
+		for (i = 0; i < surface_count; i++)
+			updates[i].surface->update_flags.bits.full_update = 1;
+
+	return type;
+}
+
 static struct dc_stream_status *stream_get_status(
 	struct dc_state *ctx,
 	struct dc_stream_state *stream)
@@ -1295,14 +1189,6 @@ static void commit_planes_for_stream(struct dc *dc,
 		context_clock_trace(dc, context);
 	}
 
-	if (update_type > UPDATE_TYPE_FAST) {
-		for (j = 0; j < dc->res_pool->pipe_count; j++) {
-			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
-
-			dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe_ctx);
-		}
-	}
-
 	if (surface_count == 0) {
 		/*
 		 * In case of turning off screen, no need to program front end a second time.
@@ -1312,103 +1198,58 @@ static void commit_planes_for_stream(struct dc *dc,
 		return;
 	}
 
-	/* Lock pipes for provided surfaces, or all active if full update*/
-	for (i = 0; i < surface_count; i++) {
-		struct dc_plane_state *plane_state = srf_updates[i].surface;
-
-		for (j = 0; j < dc->res_pool->pipe_count; j++) {
-			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
-
-			if (update_type != UPDATE_TYPE_FULL && pipe_ctx->plane_state != plane_state)
-				continue;
-			if (!pipe_ctx->plane_state || pipe_ctx->top_pipe)
-				continue;
-
-			dc->hwss.pipe_control_lock(
-					dc,
-					pipe_ctx,
-					true);
-		}
-		if (update_type == UPDATE_TYPE_FULL)
-			break;
-	}
-
 	/* Full fe update*/
 	for (j = 0; j < dc->res_pool->pipe_count; j++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
 
-		if (update_type != UPDATE_TYPE_FULL || !pipe_ctx->plane_state)
+		if (update_type == UPDATE_TYPE_FAST || !pipe_ctx->plane_state)
 			continue;
 
-		if (!pipe_ctx->top_pipe && pipe_ctx->stream) {
-			struct dc_stream_status *stream_status = stream_get_status(context, pipe_ctx->stream);
+		if (!pipe_ctx->top_pipe &&
+		    pipe_ctx->stream &&
+		    pipe_ctx->stream == stream) {
+			struct dc_stream_status *stream_status =
+					stream_get_status(context, pipe_ctx->stream);
 
 			dc->hwss.apply_ctx_for_surface(
 					dc, pipe_ctx->stream, stream_status->plane_count, context);
 		}
 	}
 
-	if (update_type > UPDATE_TYPE_FAST)
+	if (update_type == UPDATE_TYPE_FULL)
 		context_timing_trace(dc, &context->res_ctx);
 
 	/* Perform requested Updates */
 	for (i = 0; i < surface_count; i++) {
 		struct dc_plane_state *plane_state = srf_updates[i].surface;
 
-		if (update_type == UPDATE_TYPE_MED)
-			dc->hwss.apply_ctx_for_surface(
-					dc, stream, surface_count, context);
-
 		for (j = 0; j < dc->res_pool->pipe_count; j++) {
 			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
 
-			if (pipe_ctx->plane_state != plane_state)
+			if (pipe_ctx->stream != stream)
 				continue;
 
-			if (srf_updates[i].flip_addr)
-				dc->hwss.update_plane_addr(dc, pipe_ctx);
-
-			if (update_type == UPDATE_TYPE_FAST)
+			if (pipe_ctx->plane_state != plane_state)
 				continue;
 
-			/* work around to program degamma regs for split pipe after set mode. */
-			if (srf_updates[i].in_transfer_func || (pipe_ctx->top_pipe &&
-					pipe_ctx->top_pipe->plane_state == pipe_ctx->plane_state))
-				dc->hwss.set_input_transfer_func(
-						pipe_ctx, pipe_ctx->plane_state);
-
-			if (stream_update != NULL &&
-					stream_update->out_transfer_func != NULL) {
-				dc->hwss.set_output_transfer_func(
-						pipe_ctx, pipe_ctx->stream);
-			}
-
-			if (srf_updates[i].hdr_static_metadata) {
-				resource_build_info_frame(pipe_ctx);
-				dc->hwss.update_info_frame(pipe_ctx);
-			}
+			if (update_type == UPDATE_TYPE_FAST && srf_updates[i].flip_addr)
+					dc->hwss.update_plane_addr(dc, pipe_ctx);
 		}
 	}
 
-	/* Unlock pipes */
-	for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
-		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+	if (stream && stream_update && update_type > UPDATE_TYPE_FAST)
+		for (j = 0; j < dc->res_pool->pipe_count; j++) {
+			struct pipe_ctx *pipe_ctx =
+					&context->res_ctx.pipe_ctx[j];
 
-		for (j = 0; j < surface_count; j++) {
-			if (update_type != UPDATE_TYPE_FULL &&
-			    srf_updates[j].surface != pipe_ctx->plane_state)
-				continue;
-			if (!pipe_ctx->plane_state || pipe_ctx->top_pipe)
+			if (pipe_ctx->stream != stream)
 				continue;
 
-			dc->hwss.pipe_control_lock(
-					dc,
-					pipe_ctx,
-					false);
-
-			break;
+			if (stream_update->hdr_static_metadata) {
+				resource_build_info_frame(pipe_ctx);
+				dc->hwss.update_info_frame(pipe_ctx);
+			}
 		}
-	}
 }
 
 void dc_commit_updates_for_stream(struct dc *dc,
@@ -1480,10 +1321,7 @@ void dc_commit_updates_for_stream(struct dc *dc,
 				stream_update,
 				update_type,
 				context);
-
-	if (update_type >= UPDATE_TYPE_FULL)
-		dc_post_update_surfaces_to_stream(dc);
-
+	/*update current_State*/
 	if (dc->current_state != context) {
 
 		struct dc_state *old = dc->current_state;
@@ -1492,6 +1330,9 @@ void dc_commit_updates_for_stream(struct dc *dc,
 		dc_release_state(old);
 
 	}
+	/*let's use current_state to update watermark etc*/
+	if (update_type >= UPDATE_TYPE_FULL)
+		dc_post_update_surfaces_to_stream(dc);
 
 	return;
 
@@ -1517,13 +1358,13 @@ enum dc_irq_source dc_interrupt_to_irq_source(
 	return dal_irq_service_to_irq_source(dc->res_pool->irqs, src_id, ext_id);
 }
 
-void dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable)
+bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable)
 {
 
 	if (dc == NULL)
-		return;
+		return false;
 
-	dal_irq_service_set(dc->res_pool->irqs, src, enable);
+	return dal_irq_service_set(dc->res_pool->irqs, src, enable);
 }
 
 void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src)
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
index 43c7a7fddb83..1babac07bcc9 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
@@ -159,6 +159,7 @@ void pre_surface_trace(
 				"plane_state->tiling_info.gfx8.pipe_config = %d;\n"
 				"plane_state->tiling_info.gfx8.array_mode = %d;\n"
 				"plane_state->color_space = %d;\n"
+				"plane_state->input_tf = %d;\n"
 				"plane_state->dcc.enable = %d;\n"
 				"plane_state->format = %d;\n"
 				"plane_state->rotation = %d;\n"
@@ -166,6 +167,7 @@ void pre_surface_trace(
 				plane_state->tiling_info.gfx8.pipe_config,
 				plane_state->tiling_info.gfx8.array_mode,
 				plane_state->color_space,
+				plane_state->input_tf,
 				plane_state->dcc.enable,
 				plane_state->format,
 				plane_state->rotation,
@@ -206,6 +208,7 @@ void update_surface_trace(
 		if (update->plane_info) {
 			SURFACE_TRACE(
 					"plane_info->color_space = %d;\n"
+					"plane_info->input_tf = %d;\n"
 					"plane_info->format = %d;\n"
 					"plane_info->plane_size.grph.surface_pitch = %d;\n"
 					"plane_info->plane_size.grph.surface_size.height = %d;\n"
@@ -214,6 +217,7 @@ void update_surface_trace(
 					"plane_info->plane_size.grph.surface_size.y = %d;\n"
 					"plane_info->rotation = %d;\n",
 					update->plane_info->color_space,
+					update->plane_info->input_tf,
 					update->plane_info->format,
 					update->plane_info->plane_size.grph.surface_pitch,
 					update->plane_info->plane_size.grph.surface_size.height,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 71993d5983bf..ebc96b720083 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -28,6 +28,8 @@
 #include "timing_generator.h"
 #include "hw_sequencer.h"
 
+#define NUM_ELEMENTS(a) (sizeof(a) / sizeof((a)[0]))
+
 /* used as index in array of black_color_format */
 enum black_color_format {
 	BLACK_COLOR_FORMAT_RGB_FULLRANGE = 0,
@@ -38,6 +40,15 @@ enum black_color_format {
 	BLACK_COLOR_FORMAT_DEBUG,
 };
 
+enum dc_color_space_type {
+	COLOR_SPACE_RGB_TYPE,
+	COLOR_SPACE_RGB_LIMITED_TYPE,
+	COLOR_SPACE_YCBCR601_TYPE,
+	COLOR_SPACE_YCBCR709_TYPE,
+	COLOR_SPACE_YCBCR601_LIMITED_TYPE,
+	COLOR_SPACE_YCBCR709_LIMITED_TYPE
+};
+
 static const struct tg_color black_color_format[] = {
 	/* BlackColorFormat_RGB_FullRange */
 	{0, 0, 0},
@@ -53,6 +64,140 @@ static const struct tg_color black_color_format[] = {
 	{0xff, 0xff, 0},
 };
 
+struct out_csc_color_matrix_type {
+	enum dc_color_space_type color_space_type;
+	uint16_t regval[12];
+};
+
+static const struct out_csc_color_matrix_type output_csc_matrix[] = {
+	{ COLOR_SPACE_RGB_TYPE,
+		{ 0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} },
+	{ COLOR_SPACE_RGB_LIMITED_TYPE,
+		{ 0x1B67, 0, 0, 0x201, 0, 0x1B67, 0, 0x201, 0, 0, 0x1B67, 0x201} },
+	{ COLOR_SPACE_YCBCR601_TYPE,
+		{ 0xE04, 0xF444, 0xFDB9, 0x1004, 0x831, 0x1016, 0x320, 0x201, 0xFB45,
+				0xF6B7, 0xE04, 0x1004} },
+	{ COLOR_SPACE_YCBCR709_TYPE,
+		{ 0xE04, 0xF345, 0xFEB7, 0x1004, 0x5D3, 0x1399, 0x1FA,
+				0x201, 0xFCCA, 0xF533, 0xE04, 0x1004} },
+
+	/* TODO: correct values below */
+	{ COLOR_SPACE_YCBCR601_LIMITED_TYPE,
+		{ 0xE00, 0xF447, 0xFDB9, 0x1000, 0x991,
+				0x12C9, 0x3A6, 0x200, 0xFB47, 0xF6B9, 0xE00, 0x1000} },
+	{ COLOR_SPACE_YCBCR709_LIMITED_TYPE,
+		{ 0xE00, 0xF349, 0xFEB7, 0x1000, 0x6CE, 0x16E3,
+				0x24F, 0x200, 0xFCCB, 0xF535, 0xE00, 0x1000} },
+};
+
+static bool is_rgb_type(
+		enum dc_color_space color_space)
+{
+	bool ret = false;
+
+	if (color_space == COLOR_SPACE_SRGB			||
+		color_space == COLOR_SPACE_XR_RGB		||
+		color_space == COLOR_SPACE_MSREF_SCRGB		||
+		color_space == COLOR_SPACE_2020_RGB_FULLRANGE	||
+		color_space == COLOR_SPACE_ADOBERGB		||
+		color_space == COLOR_SPACE_DCIP3	||
+		color_space == COLOR_SPACE_DOLBYVISION)
+		ret = true;
+	return ret;
+}
+
+static bool is_rgb_limited_type(
+		enum dc_color_space color_space)
+{
+	bool ret = false;
+
+	if (color_space == COLOR_SPACE_SRGB_LIMITED		||
+		color_space == COLOR_SPACE_2020_RGB_LIMITEDRANGE)
+		ret = true;
+	return ret;
+}
+
+static bool is_ycbcr601_type(
+		enum dc_color_space color_space)
+{
+	bool ret = false;
+
+	if (color_space == COLOR_SPACE_YCBCR601	||
+		color_space == COLOR_SPACE_XV_YCC_601)
+		ret = true;
+	return ret;
+}
+
+static bool is_ycbcr601_limited_type(
+		enum dc_color_space color_space)
+{
+	bool ret = false;
+
+	if (color_space == COLOR_SPACE_YCBCR601_LIMITED)
+		ret = true;
+	return ret;
+}
+
+static bool is_ycbcr709_type(
+		enum dc_color_space color_space)
+{
+	bool ret = false;
+
+	if (color_space == COLOR_SPACE_YCBCR709	||
+		color_space == COLOR_SPACE_XV_YCC_709)
+		ret = true;
+	return ret;
+}
+
+static bool is_ycbcr709_limited_type(
+		enum dc_color_space color_space)
+{
+	bool ret = false;
+
+	if (color_space == COLOR_SPACE_YCBCR709_LIMITED)
+		ret = true;
+	return ret;
+}
+enum dc_color_space_type get_color_space_type(enum dc_color_space color_space)
+{
+	enum dc_color_space_type type = COLOR_SPACE_RGB_TYPE;
+
+	if (is_rgb_type(color_space))
+		type = COLOR_SPACE_RGB_TYPE;
+	else if (is_rgb_limited_type(color_space))
+		type = COLOR_SPACE_RGB_LIMITED_TYPE;
+	else if (is_ycbcr601_type(color_space))
+		type = COLOR_SPACE_YCBCR601_TYPE;
+	else if (is_ycbcr709_type(color_space))
+		type = COLOR_SPACE_YCBCR709_TYPE;
+	else if (is_ycbcr601_limited_type(color_space))
+		type = COLOR_SPACE_YCBCR601_LIMITED_TYPE;
+	else if (is_ycbcr709_limited_type(color_space))
+		type = COLOR_SPACE_YCBCR709_LIMITED_TYPE;
+
+	return type;
+}
+
+const uint16_t *find_color_matrix(enum dc_color_space color_space,
+							uint32_t *array_size)
+{
+	int i;
+	enum dc_color_space_type type;
+	const uint16_t *val = NULL;
+	int arr_size = NUM_ELEMENTS(output_csc_matrix);
+
+	type = get_color_space_type(color_space);
+	for (i = 0; i < arr_size; i++)
+		if (output_csc_matrix[i].color_space_type == type) {
+			val = output_csc_matrix[i].regval;
+			*array_size = 12;
+			break;
+		}
+
+	return val;
+}
+
+
 void color_space_to_black_color(
 	const struct dc *dc,
 	enum dc_color_space colorspace,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
index 42a111b9505d..be5546181fa8 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
@@ -938,8 +938,9 @@ static bool construct(
 	link->link_id = bios->funcs->get_connector_id(bios, init_params->connector_index);
 
 	if (link->link_id.type != OBJECT_TYPE_CONNECTOR) {
-		dm_error("%s: Invalid Connector ObjectID from Adapter Service for connector index:%d!\n",
-				__func__, init_params->connector_index);
+		dm_error("%s: Invalid Connector ObjectID from Adapter Service for connector index:%d! type %d expected %d\n",
+			 __func__, init_params->connector_index,
+			 link->link_id.type, OBJECT_TYPE_CONNECTOR);
 		goto create_fail;
 	}
 
@@ -1271,6 +1272,24 @@ static enum dc_status enable_link_dp(
 	return status;
 }
 
+static enum dc_status enable_link_edp(
+		struct dc_state *state,
+		struct pipe_ctx *pipe_ctx)
+{
+	enum dc_status status;
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	struct dc_link *link = stream->sink->link;
+
+	link->dc->hwss.edp_power_control(link, true);
+	link->dc->hwss.edp_wait_for_hpd_ready(link, true);
+
+	status = enable_link_dp(state, pipe_ctx);
+
+	link->dc->hwss.edp_backlight_control(link, true);
+
+	return status;
+}
+
 static enum dc_status enable_link_dp_mst(
 		struct dc_state *state,
 		struct pipe_ctx *pipe_ctx)
@@ -1730,8 +1749,7 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx)
 			link->link_enc,
 			pipe_ctx->clock_source->id,
 			display_color_depth,
-			pipe_ctx->stream->signal == SIGNAL_TYPE_HDMI_TYPE_A,
-			pipe_ctx->stream->signal == SIGNAL_TYPE_DVI_DUAL_LINK,
+			pipe_ctx->stream->signal,
 			stream->phy_pix_clk);
 
 	if (pipe_ctx->stream->signal == SIGNAL_TYPE_HDMI_TYPE_A)
@@ -1746,9 +1764,11 @@ static enum dc_status enable_link(
 	enum dc_status status = DC_ERROR_UNEXPECTED;
 	switch (pipe_ctx->stream->signal) {
 	case SIGNAL_TYPE_DISPLAY_PORT:
-	case SIGNAL_TYPE_EDP:
 		status = enable_link_dp(state, pipe_ctx);
 		break;
+	case SIGNAL_TYPE_EDP:
+		status = enable_link_edp(state, pipe_ctx);
+		break;
 	case SIGNAL_TYPE_DISPLAY_PORT_MST:
 		status = enable_link_dp_mst(state, pipe_ctx);
 		msleep(200);
@@ -1798,7 +1818,7 @@ static void disable_link(struct dc_link *link, enum signal_type signal)
 		else
 			dp_disable_link_phy_mst(link, signal);
 	} else
-		link->link_enc->funcs->disable_output(link->link_enc, signal, link);
+		link->link_enc->funcs->disable_output(link->link_enc, signal);
 }
 
 static bool dp_active_dongle_validate_timing(
@@ -1871,7 +1891,7 @@ enum dc_status dc_link_validate_mode_timing(
 		const struct dc_crtc_timing *timing)
 {
 	uint32_t max_pix_clk = stream->sink->dongle_max_pix_clk;
-	struct dc_dongle_caps *dongle_caps = &link->link_status.dpcd_caps->dongle_caps;
+	struct dc_dongle_caps *dongle_caps = &link->dpcd_caps.dongle_caps;
 
 	/* A hack to avoid failing any modes for EDID override feature on
 	 * topology change such as lower quality cable for DP or different dongle
@@ -1909,12 +1929,18 @@ bool dc_link_set_backlight_level(const struct dc_link *link, uint32_t level,
 {
 	struct dc  *core_dc = link->ctx->dc;
 	struct abm *abm = core_dc->res_pool->abm;
+	struct dmcu *dmcu = core_dc->res_pool->dmcu;
 	unsigned int controller_id = 0;
+	bool use_smooth_brightness = true;
 	int i;
 
-	if ((abm == NULL) || (abm->funcs->set_backlight_level == NULL))
+	if ((dmcu == NULL) ||
+		(abm == NULL) ||
+		(abm->funcs->set_backlight_level == NULL))
 		return false;
 
+	use_smooth_brightness = dmcu->funcs->is_dmcu_initialized(dmcu);
+
 	dm_logger_write(link->ctx->logger, LOG_BACKLIGHT,
 			"New Backlight level: %d (0x%X)\n", level, level);
 
@@ -1937,7 +1963,8 @@ bool dc_link_set_backlight_level(const struct dc_link *link, uint32_t level,
 				abm,
 				level,
 				frame_ramp,
-				controller_id);
+				controller_id,
+				use_smooth_brightness);
 	}
 
 	return true;
@@ -1954,144 +1981,6 @@ bool dc_link_set_psr_enable(const struct dc_link *link, bool enable, bool wait)
 	return true;
 }
 
-bool dc_link_get_psr_state(const struct dc_link *link, uint32_t *psr_state)
-{
-	struct dc  *core_dc = link->ctx->dc;
-	struct dmcu *dmcu = core_dc->res_pool->dmcu;
-
-	if (dmcu != NULL && link->psr_enabled)
-		dmcu->funcs->get_psr_state(dmcu, psr_state);
-
-	return true;
-}
-
-bool dc_link_setup_psr(struct dc_link *link,
-		const struct dc_stream_state *stream, struct psr_config *psr_config,
-		struct psr_context *psr_context)
-{
-	struct dc  *core_dc = link->ctx->dc;
-	struct dmcu *dmcu = core_dc->res_pool->dmcu;
-	int i;
-
-	psr_context->controllerId = CONTROLLER_ID_UNDEFINED;
-
-	if (link != NULL &&
-		dmcu != NULL) {
-		/* updateSinkPsrDpcdConfig*/
-		union dpcd_psr_configuration psr_configuration;
-
-		memset(&psr_configuration, 0, sizeof(psr_configuration));
-
-		psr_configuration.bits.ENABLE                    = 1;
-		psr_configuration.bits.CRC_VERIFICATION          = 1;
-		psr_configuration.bits.FRAME_CAPTURE_INDICATION  =
-				psr_config->psr_frame_capture_indication_req;
-
-		/* Check for PSR v2*/
-		if (psr_config->psr_version == 0x2) {
-			/* For PSR v2 selective update.
-			 * Indicates whether sink should start capturing
-			 * immediately following active scan line,
-			 * or starting with the 2nd active scan line.
-			 */
-			psr_configuration.bits.LINE_CAPTURE_INDICATION = 0;
-			/*For PSR v2, determines whether Sink should generate
-			 * IRQ_HPD when CRC mismatch is detected.
-			 */
-			psr_configuration.bits.IRQ_HPD_WITH_CRC_ERROR    = 1;
-		}
-
-		dm_helpers_dp_write_dpcd(
-			link->ctx,
-			link,
-			368,
-			&psr_configuration.raw,
-			sizeof(psr_configuration.raw));
-
-		psr_context->channel = link->ddc->ddc_pin->hw_info.ddc_channel;
-		psr_context->transmitterId = link->link_enc->transmitter;
-		psr_context->engineId = link->link_enc->preferred_engine;
-
-		for (i = 0; i < MAX_PIPES; i++) {
-			if (core_dc->current_state->res_ctx.pipe_ctx[i].stream
-					== stream) {
-				/* dmcu -1 for all controller id values,
-				 * therefore +1 here
-				 */
-				psr_context->controllerId =
-					core_dc->current_state->res_ctx.
-					pipe_ctx[i].stream_res.tg->inst + 1;
-				break;
-			}
-		}
-
-		/* Hardcoded for now.  Can be Pcie or Uniphy (or Unknown)*/
-		psr_context->phyType = PHY_TYPE_UNIPHY;
-		/*PhyId is associated with the transmitter id*/
-		psr_context->smuPhyId = link->link_enc->transmitter;
-
-		psr_context->crtcTimingVerticalTotal = stream->timing.v_total;
-		psr_context->vsyncRateHz = div64_u64(div64_u64((stream->
-						timing.pix_clk_khz * 1000),
-						stream->timing.v_total),
-						stream->timing.h_total);
-
-		psr_context->psrSupportedDisplayConfig = true;
-		psr_context->psrExitLinkTrainingRequired =
-			psr_config->psr_exit_link_training_required;
-		psr_context->sdpTransmitLineNumDeadline =
-			psr_config->psr_sdp_transmit_line_num_deadline;
-		psr_context->psrFrameCaptureIndicationReq =
-			psr_config->psr_frame_capture_indication_req;
-
-		psr_context->skipPsrWaitForPllLock = 0; /* only = 1 in KV */
-
-		psr_context->numberOfControllers =
-				link->dc->res_pool->res_cap->num_timing_generator;
-
-		psr_context->rfb_update_auto_en = true;
-
-		/* 2 frames before enter PSR. */
-		psr_context->timehyst_frames = 2;
-		/* half a frame
-		 * (units in 100 lines, i.e. a value of 1 represents 100 lines)
-		 */
-		psr_context->hyst_lines = stream->timing.v_total / 2 / 100;
-		psr_context->aux_repeats = 10;
-
-		psr_context->psr_level.u32all = 0;
-
-#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-		/*skip power down the single pipe since it blocks the cstate*/
-		if (ASIC_REV_IS_RAVEN(link->ctx->asic_id.hw_internal_rev))
-			psr_context->psr_level.bits.SKIP_CRTC_DISABLE = true;
-#endif
-
-		/* SMU will perform additional powerdown sequence.
-		 * For unsupported ASICs, set psr_level flag to skip PSR
-		 *  static screen notification to SMU.
-		 *  (Always set for DAL2, did not check ASIC)
-		 */
-		psr_context->psr_level.bits.SKIP_SMU_NOTIFICATION = 1;
-
-		/* Complete PSR entry before aborting to prevent intermittent
-		 * freezes on certain eDPs
-		 */
-		psr_context->psr_level.bits.DISABLE_PSR_ENTRY_ABORT = 1;
-
-		/* Controls additional delay after remote frame capture before
-		 * continuing power down, default = 0
-		 */
-		psr_context->frame_delay = 0;
-
-		link->psr_enabled = true;
-		dmcu->funcs->setup_psr(dmcu, link, psr_context);
-		return true;
-	} else
-		return false;
-
-}
-
 const struct dc_link_status *dc_link_get_status(const struct dc_link *link)
 {
 	return &link->link_status;
@@ -2420,6 +2309,9 @@ void core_link_disable_stream(struct pipe_ctx *pipe_ctx, int option)
 	if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST)
 		deallocate_mst_payload(pipe_ctx);
 
+	if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP)
+		core_dc->hwss.edp_backlight_control(pipe_ctx->stream->sink->link, false);
+
 	core_dc->hwss.disable_stream(pipe_ctx, option);
 
 	disable_link(pipe_ctx->stream->sink->link, pipe_ctx->stream->signal);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
index e6bf05d76a94..639421a00ab6 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
@@ -220,8 +220,7 @@ static void dpcd_set_lt_pattern_and_lane_settings(
 		size_in_bytes);
 
 	dm_logger_write(link->ctx->logger, LOG_HW_LINK_TRAINING,
-		"%s:\n %x VS set = %x  PE set = %x \
-		max VS Reached = %x  max PE Reached = %x\n",
+		"%s:\n %x VS set = %x  PE set = %x max VS Reached = %x  max PE Reached = %x\n",
 		__func__,
 		DP_TRAINING_LANE0_SET,
 		dpcd_lane[0].bits.VOLTAGE_SWING_SET,
@@ -558,8 +557,7 @@ static void dpcd_set_lane_settings(
 	*/
 
 	dm_logger_write(link->ctx->logger, LOG_HW_LINK_TRAINING,
-		"%s\n %x VS set = %x  PE set = %x \
-		max VS Reached = %x  max PE Reached = %x\n",
+		"%s\n %x VS set = %x  PE set = %x max VS Reached = %x  max PE Reached = %x\n",
 		__func__,
 		DP_TRAINING_LANE0_SET,
 		dpcd_lane[0].bits.VOLTAGE_SWING_SET,
@@ -720,7 +718,7 @@ static enum link_training_result perform_channel_equalization_sequence(
 	uint32_t retries_ch_eq;
 	enum dc_lane_count lane_count = lt_settings->link_settings.lane_count;
 	union lane_align_status_updated dpcd_lane_status_updated = {{0}};
-	union lane_status dpcd_lane_status[LANE_COUNT_DP_MAX] = {{{0}}};;
+	union lane_status dpcd_lane_status[LANE_COUNT_DP_MAX] = {{{0}}};
 
 	hw_tr_pattern = get_supported_tp(link);
 
@@ -872,9 +870,8 @@ static bool perform_clock_recovery_sequence(
 	if (retry_count >= LINK_TRAINING_MAX_CR_RETRY) {
 		ASSERT(0);
 		dm_logger_write(link->ctx->logger, LOG_ERROR,
-			"%s: Link Training Error, could not \
-			 get CR after %d tries. \
-			Possibly voltage swing issue", __func__,
+			"%s: Link Training Error, could not get CR after %d tries. Possibly voltage swing issue",
+			__func__,
 			LINK_TRAINING_MAX_CR_RETRY);
 
 	}
@@ -1468,7 +1465,13 @@ void decide_link_settings(struct dc_stream_state *stream,
 	/* MST doesn't perform link training for now
 	 * TODO: add MST specific link training routine
 	 */
-	if (is_mst_supported(link)) {
+	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
+		*link_setting = link->verified_link_cap;
+		return;
+	}
+
+	/* EDP use the link cap setting */
+	if (stream->sink->sink_signal == SIGNAL_TYPE_EDP) {
 		*link_setting = link->verified_link_cap;
 		return;
 	}
@@ -2127,7 +2130,7 @@ static void get_active_converter_info(
 
 				union dwnstream_port_caps_byte3_hdmi
 					hdmi_caps = {.raw = det_caps[3] };
-				union dwnstream_port_caps_byte1
+				union dwnstream_port_caps_byte2
 					hdmi_color_caps = {.raw = det_caps[2] };
 				link->dpcd_caps.dongle_caps.dp_hdmi_max_pixel_clk =
 					det_caps[1] * 25000;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
index 9a33b471270a..2096f2a179f2 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
@@ -88,15 +88,7 @@ void dp_enable_link_phy(
 	}
 
 	if (dc_is_dp_sst_signal(signal)) {
-		if (signal == SIGNAL_TYPE_EDP) {
-			link->dc->hwss.edp_power_control(link->link_enc, true);
-			link_enc->funcs->enable_dp_output(
-						link_enc,
-						link_settings,
-						clock_source);
-			link->dc->hwss.edp_backlight_control(link, true);
-		} else
-			link_enc->funcs->enable_dp_output(
+		link_enc->funcs->enable_dp_output(
 						link_enc,
 						link_settings,
 						clock_source);
@@ -138,12 +130,11 @@ void dp_disable_link_phy(struct dc_link *link, enum signal_type signal)
 		dp_receiver_power_ctrl(link, false);
 
 	if (signal == SIGNAL_TYPE_EDP) {
-		link->dc->hwss.edp_backlight_control(link, false);
 		edp_receiver_ready_T9(link);
-		link->link_enc->funcs->disable_output(link->link_enc, signal, link);
-		link->dc->hwss.edp_power_control(link->link_enc, false);
+		link->link_enc->funcs->disable_output(link->link_enc, signal);
+		link->dc->hwss.edp_power_control(link, false);
 	} else
-		link->link_enc->funcs->disable_output(link->link_enc, signal, link);
+		link->link_enc->funcs->disable_output(link->link_enc, signal);
 
 	/* Clear current link setting.*/
 	memset(&link->cur_link_settings, 0,
@@ -286,8 +277,7 @@ void dp_retrain_link_dp_test(struct dc_link *link,
 
 			link->link_enc->funcs->disable_output(
 					link->link_enc,
-					SIGNAL_TYPE_DISPLAY_PORT,
-					link);
+					SIGNAL_TYPE_DISPLAY_PORT);
 
 			/* Clear current link setting. */
 			memset(&link->cur_link_settings, 0,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index 928895809867..4d07ffebfd31 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -426,15 +426,8 @@ static enum pixel_format convert_pixel_format_to_dalsurface(
 
 static void rect_swap_helper(struct rect *rect)
 {
-	uint32_t temp = 0;
-
-	temp = rect->height;
-	rect->height = rect->width;
-	rect->width = temp;
-
-	temp = rect->x;
-	rect->x = rect->y;
-	rect->y = temp;
+	swap(rect->height, rect->width);
+	swap(rect->x, rect->y);
 }
 
 static void calculate_viewport(struct pipe_ctx *pipe_ctx)
@@ -505,26 +498,15 @@ static void calculate_viewport(struct pipe_ctx *pipe_ctx)
 	data->viewport_c.height = (data->viewport.height + vpc_div - 1) / vpc_div;
 
 	/* Handle hsplit */
-	if (pri_split || sec_split) {
-		/* HMirror XOR Secondary_pipe XOR Rotation_180 */
-		bool right_view = (sec_split != plane_state->horizontal_mirror) !=
-					(plane_state->rotation == ROTATION_ANGLE_180);
-
-		if (plane_state->rotation == ROTATION_ANGLE_90
-				|| plane_state->rotation == ROTATION_ANGLE_270)
-			/* Secondary_pipe XOR Rotation_270 */
-			right_view = (plane_state->rotation == ROTATION_ANGLE_270) != sec_split;
-
-		if (right_view) {
-			data->viewport.x +=  data->viewport.width / 2;
-			data->viewport_c.x +=  data->viewport_c.width / 2;
-			/* Ceil offset pipe */
-			data->viewport.width = (data->viewport.width + 1) / 2;
-			data->viewport_c.width = (data->viewport_c.width + 1) / 2;
-		} else {
-			data->viewport.width /= 2;
-			data->viewport_c.width /= 2;
-		}
+	if (sec_split) {
+		data->viewport.x +=  data->viewport.width / 2;
+		data->viewport_c.x +=  data->viewport_c.width / 2;
+		/* Ceil offset pipe */
+		data->viewport.width = (data->viewport.width + 1) / 2;
+		data->viewport_c.width = (data->viewport_c.width + 1) / 2;
+	} else if (pri_split) {
+		data->viewport.width /= 2;
+		data->viewport_c.width /= 2;
 	}
 
 	if (plane_state->rotation == ROTATION_ANGLE_90 ||
@@ -541,6 +523,11 @@ static void calculate_recout(struct pipe_ctx *pipe_ctx, struct view *recout_skip
 	struct rect surf_src = plane_state->src_rect;
 	struct rect surf_clip = plane_state->clip_rect;
 	int recout_full_x, recout_full_y;
+	bool pri_split = pipe_ctx->bottom_pipe &&
+			pipe_ctx->bottom_pipe->plane_state == pipe_ctx->plane_state;
+	bool sec_split = pipe_ctx->top_pipe &&
+			pipe_ctx->top_pipe->plane_state == pipe_ctx->plane_state;
+	bool top_bottom_split = stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM;
 
 	if (pipe_ctx->plane_state->rotation == ROTATION_ANGLE_90 ||
 			pipe_ctx->plane_state->rotation == ROTATION_ANGLE_270)
@@ -575,33 +562,43 @@ static void calculate_recout(struct pipe_ctx *pipe_ctx, struct view *recout_skip
 						- pipe_ctx->plane_res.scl_data.recout.y;
 
 	/* Handle h & vsplit */
-	if (pipe_ctx->top_pipe && pipe_ctx->top_pipe->plane_state ==
-		pipe_ctx->plane_state) {
-		if (stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM) {
-			pipe_ctx->plane_res.scl_data.recout.y += pipe_ctx->plane_res.scl_data.recout.height / 2;
-			/* Floor primary pipe, ceil 2ndary pipe */
-			pipe_ctx->plane_res.scl_data.recout.height = (pipe_ctx->plane_res.scl_data.recout.height + 1) / 2;
+	if (sec_split && top_bottom_split) {
+		pipe_ctx->plane_res.scl_data.recout.y +=
+				pipe_ctx->plane_res.scl_data.recout.height / 2;
+		/* Floor primary pipe, ceil 2ndary pipe */
+		pipe_ctx->plane_res.scl_data.recout.height =
+				(pipe_ctx->plane_res.scl_data.recout.height + 1) / 2;
+	} else if (pri_split && top_bottom_split)
+		pipe_ctx->plane_res.scl_data.recout.height /= 2;
+	else if (pri_split || sec_split) {
+		/* HMirror XOR Secondary_pipe XOR Rotation_180 */
+		bool right_view = (sec_split != plane_state->horizontal_mirror) !=
+					(plane_state->rotation == ROTATION_ANGLE_180);
+
+		if (plane_state->rotation == ROTATION_ANGLE_90
+				|| plane_state->rotation == ROTATION_ANGLE_270)
+			/* Secondary_pipe XOR Rotation_270 */
+			right_view = (plane_state->rotation == ROTATION_ANGLE_270) != sec_split;
+
+		if (right_view) {
+			pipe_ctx->plane_res.scl_data.recout.x +=
+					pipe_ctx->plane_res.scl_data.recout.width / 2;
+			/* Ceil offset pipe */
+			pipe_ctx->plane_res.scl_data.recout.width =
+					(pipe_ctx->plane_res.scl_data.recout.width + 1) / 2;
 		} else {
-			pipe_ctx->plane_res.scl_data.recout.x += pipe_ctx->plane_res.scl_data.recout.width / 2;
-			pipe_ctx->plane_res.scl_data.recout.width = (pipe_ctx->plane_res.scl_data.recout.width + 1) / 2;
-		}
-	} else if (pipe_ctx->bottom_pipe &&
-			pipe_ctx->bottom_pipe->plane_state == pipe_ctx->plane_state) {
-		if (stream->view_format == VIEW_3D_FORMAT_TOP_AND_BOTTOM)
-			pipe_ctx->plane_res.scl_data.recout.height /= 2;
-		else
 			pipe_ctx->plane_res.scl_data.recout.width /= 2;
+		}
 	}
-
 	/* Unclipped recout offset = stream dst offset + ((surf dst offset - stream surf_src offset)
 	 * 				* 1/ stream scaling ratio) - (surf surf_src offset * 1/ full scl
 	 * 				ratio)
 	 */
-	recout_full_x = stream->dst.x + (plane_state->dst_rect.x -  stream->src.x)
+	recout_full_x = stream->dst.x + (plane_state->dst_rect.x - stream->src.x)
 					* stream->dst.width / stream->src.width -
 			surf_src.x * plane_state->dst_rect.width / surf_src.width
 					* stream->dst.width / stream->src.width;
-	recout_full_y = stream->dst.y + (plane_state->dst_rect.y -  stream->src.y)
+	recout_full_y = stream->dst.y + (plane_state->dst_rect.y - stream->src.y)
 					* stream->dst.height / stream->src.height -
 			surf_src.y * plane_state->dst_rect.height / surf_src.height
 					* stream->dst.height / stream->src.height;
@@ -657,7 +654,20 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 	struct rect src = pipe_ctx->plane_state->src_rect;
 	int vpc_div = (data->format == PIXEL_FORMAT_420BPP8
 			|| data->format == PIXEL_FORMAT_420BPP10) ? 2 : 1;
+	bool flip_vert_scan_dir = false, flip_horz_scan_dir = false;
 
+	/*
+	 * Need to calculate the scan direction for viewport to make adjustments
+	 */
+	if (pipe_ctx->plane_state->rotation == ROTATION_ANGLE_180) {
+		flip_vert_scan_dir = true;
+		flip_horz_scan_dir = true;
+	} else if (pipe_ctx->plane_state->rotation == ROTATION_ANGLE_90)
+		flip_vert_scan_dir = true;
+	else if (pipe_ctx->plane_state->rotation == ROTATION_ANGLE_270)
+		flip_horz_scan_dir = true;
+	if (pipe_ctx->plane_state->horizontal_mirror)
+		flip_horz_scan_dir = !flip_horz_scan_dir;
 
 	if (pipe_ctx->plane_state->rotation == ROTATION_ANGLE_90 ||
 			pipe_ctx->plane_state->rotation == ROTATION_ANGLE_270) {
@@ -722,7 +732,7 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 	}
 
 	/* Adjust for non-0 viewport offset */
-	if (data->viewport.x) {
+	if (data->viewport.x && !flip_horz_scan_dir) {
 		int int_part;
 
 		data->inits.h = dal_fixed31_32_add(data->inits.h, dal_fixed31_32_mul_int(
@@ -743,7 +753,7 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 		data->inits.h = dal_fixed31_32_add_int(data->inits.h, int_part);
 	}
 
-	if (data->viewport_c.x) {
+	if (data->viewport_c.x && !flip_horz_scan_dir) {
 		int int_part;
 
 		data->inits.h_c = dal_fixed31_32_add(data->inits.h_c, dal_fixed31_32_mul_int(
@@ -764,7 +774,7 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 		data->inits.h_c = dal_fixed31_32_add_int(data->inits.h_c, int_part);
 	}
 
-	if (data->viewport.y) {
+	if (data->viewport.y && !flip_vert_scan_dir) {
 		int int_part;
 
 		data->inits.v = dal_fixed31_32_add(data->inits.v, dal_fixed31_32_mul_int(
@@ -785,7 +795,7 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx, struct view *r
 		data->inits.v = dal_fixed31_32_add_int(data->inits.v, int_part);
 	}
 
-	if (data->viewport_c.y) {
+	if (data->viewport_c.y && !flip_vert_scan_dir) {
 		int int_part;
 
 		data->inits.v_c = dal_fixed31_32_add(data->inits.v_c, dal_fixed31_32_mul_int(
@@ -1350,9 +1360,6 @@ bool dc_is_stream_scaling_unchanged(
 	return true;
 }
 
-/* Maximum TMDS single link pixel clock 165MHz */
-#define TMDS_MAX_PIXEL_CLOCK_IN_KHZ 165000
-
 static void update_stream_engine_usage(
 		struct resource_context *res_ctx,
 		const struct resource_pool *pool,
@@ -2319,20 +2326,13 @@ static void set_spd_info_packet(
 
 static void set_hdr_static_info_packet(
 		struct encoder_info_packet *info_packet,
-		struct dc_plane_state *plane_state,
 		struct dc_stream_state *stream)
 {
 	uint16_t i = 0;
 	enum signal_type signal = stream->signal;
-	struct dc_hdr_static_metadata hdr_metadata;
 	uint32_t data;
 
-	if (!plane_state)
-		return;
-
-	hdr_metadata = plane_state->hdr_static_ctx;
-
-	if (!hdr_metadata.hdr_supported)
+	if (!stream->hdr_static_metadata.hdr_supported)
 		return;
 
 	if (dc_is_hdmi_signal(signal)) {
@@ -2352,55 +2352,55 @@ static void set_hdr_static_info_packet(
 		i = 2;
 	}
 
-	data = hdr_metadata.is_hdr;
+	data = stream->hdr_static_metadata.is_hdr;
 	info_packet->sb[i++] = data ? 0x02 : 0x00;
 	info_packet->sb[i++] = 0x00;
 
-	data = hdr_metadata.chromaticity_green_x / 2;
+	data = stream->hdr_static_metadata.chromaticity_green_x / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.chromaticity_green_y / 2;
+	data = stream->hdr_static_metadata.chromaticity_green_y / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.chromaticity_blue_x / 2;
+	data = stream->hdr_static_metadata.chromaticity_blue_x / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.chromaticity_blue_y / 2;
+	data = stream->hdr_static_metadata.chromaticity_blue_y / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.chromaticity_red_x / 2;
+	data = stream->hdr_static_metadata.chromaticity_red_x / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.chromaticity_red_y / 2;
+	data = stream->hdr_static_metadata.chromaticity_red_y / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.chromaticity_white_point_x / 2;
+	data = stream->hdr_static_metadata.chromaticity_white_point_x / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.chromaticity_white_point_y / 2;
+	data = stream->hdr_static_metadata.chromaticity_white_point_y / 2;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.max_luminance;
+	data = stream->hdr_static_metadata.max_luminance;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.min_luminance;
+	data = stream->hdr_static_metadata.min_luminance;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.maximum_content_light_level;
+	data = stream->hdr_static_metadata.maximum_content_light_level;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
-	data = hdr_metadata.maximum_frame_average_light_level;
+	data = stream->hdr_static_metadata.maximum_frame_average_light_level;
 	info_packet->sb[i++] = data & 0xFF;
 	info_packet->sb[i++] = (data & 0xFF00) >> 8;
 
@@ -2551,16 +2551,14 @@ void resource_build_info_frame(struct pipe_ctx *pipe_ctx)
 
 		set_spd_info_packet(&info->spd, pipe_ctx->stream);
 
-		set_hdr_static_info_packet(&info->hdrsmd,
-				pipe_ctx->plane_state, pipe_ctx->stream);
+		set_hdr_static_info_packet(&info->hdrsmd, pipe_ctx->stream);
 
 	} else if (dc_is_dp_signal(signal)) {
 		set_vsc_info_packet(&info->vsc, pipe_ctx->stream);
 
 		set_spd_info_packet(&info->spd, pipe_ctx->stream);
 
-		set_hdr_static_info_packet(&info->hdrsmd,
-				pipe_ctx->plane_state, pipe_ctx->stream);
+		set_hdr_static_info_packet(&info->hdrsmd, pipe_ctx->stream);
 	}
 
 	patch_gamut_packet_checksum(&info->gamut);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index e230cc44a0a7..cd5819789d76 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -33,23 +33,20 @@
 /*******************************************************************************
  * Private functions
  ******************************************************************************/
-#define TMDS_MAX_PIXEL_CLOCK_IN_KHZ_UPMOST 297000
-static void update_stream_signal(struct dc_stream_state *stream)
+void update_stream_signal(struct dc_stream_state *stream)
 {
-	if (stream->output_signal == SIGNAL_TYPE_NONE) {
-		struct dc_sink *dc_sink = stream->sink;
 
-		if (dc_sink->sink_signal == SIGNAL_TYPE_NONE)
-			stream->signal = stream->sink->link->connector_signal;
-		else
-			stream->signal = dc_sink->sink_signal;
-	} else {
-		stream->signal = stream->output_signal;
-	}
+	struct dc_sink *dc_sink = stream->sink;
+
+	if (dc_sink->sink_signal == SIGNAL_TYPE_NONE)
+		stream->signal = stream->sink->link->connector_signal;
+	else
+		stream->signal = dc_sink->sink_signal;
 
 	if (dc_is_dvi_signal(stream->signal)) {
-		if (stream->timing.pix_clk_khz > TMDS_MAX_PIXEL_CLOCK_IN_KHZ_UPMOST &&
-			stream->sink->sink_signal != SIGNAL_TYPE_DVI_SINGLE_LINK)
+		if (stream->ctx->dc->caps.dual_link_dvi &&
+		    stream->timing.pix_clk_khz > TMDS_MAX_PIXEL_CLOCK &&
+		    stream->sink->sink_signal != SIGNAL_TYPE_DVI_SINGLE_LINK)
 			stream->signal = SIGNAL_TYPE_DVI_DUAL_LINK;
 		else
 			stream->signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
@@ -196,44 +193,20 @@ bool dc_stream_set_cursor_attributes(
 
 	core_dc = stream->ctx->dc;
 	res_ctx = &core_dc->current_state->res_ctx;
+	stream->cursor_attributes = *attributes;
 
 	for (i = 0; i < MAX_PIPES; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 
-		if (pipe_ctx->stream != stream || (!pipe_ctx->plane_res.xfm && !pipe_ctx->plane_res.dpp))
+		if (pipe_ctx->stream != stream || (!pipe_ctx->plane_res.xfm &&
+		    !pipe_ctx->plane_res.dpp) || !pipe_ctx->plane_res.ipp)
 			continue;
 		if (pipe_ctx->top_pipe && pipe_ctx->plane_state != pipe_ctx->top_pipe->plane_state)
 			continue;
 
 
-		if (pipe_ctx->plane_res.ipp->funcs->ipp_cursor_set_attributes != NULL)
-			pipe_ctx->plane_res.ipp->funcs->ipp_cursor_set_attributes(
-						pipe_ctx->plane_res.ipp, attributes);
-
-		if (pipe_ctx->plane_res.hubp != NULL &&
-				pipe_ctx->plane_res.hubp->funcs->set_cursor_attributes != NULL)
-			pipe_ctx->plane_res.hubp->funcs->set_cursor_attributes(
-					pipe_ctx->plane_res.hubp, attributes);
-
-		if (pipe_ctx->plane_res.mi != NULL &&
-				pipe_ctx->plane_res.mi->funcs->set_cursor_attributes != NULL)
-			pipe_ctx->plane_res.mi->funcs->set_cursor_attributes(
-					pipe_ctx->plane_res.mi, attributes);
-
-
-		if (pipe_ctx->plane_res.xfm != NULL &&
-				pipe_ctx->plane_res.xfm->funcs->set_cursor_attributes != NULL)
-			pipe_ctx->plane_res.xfm->funcs->set_cursor_attributes(
-				pipe_ctx->plane_res.xfm, attributes);
-
-		if (pipe_ctx->plane_res.dpp != NULL &&
-				pipe_ctx->plane_res.dpp->funcs->set_cursor_attributes != NULL)
-			pipe_ctx->plane_res.dpp->funcs->set_cursor_attributes(
-				pipe_ctx->plane_res.dpp, attributes);
+		core_dc->hwss.set_cursor_attribute(pipe_ctx);
 	}
-
-	stream->cursor_attributes = *attributes;
-
 	return true;
 }
 
@@ -257,51 +230,19 @@ bool dc_stream_set_cursor_position(
 
 	core_dc = stream->ctx->dc;
 	res_ctx = &core_dc->current_state->res_ctx;
+	stream->cursor_position = *position;
 
 	for (i = 0; i < MAX_PIPES; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
-		struct input_pixel_processor *ipp = pipe_ctx->plane_res.ipp;
-		struct mem_input *mi = pipe_ctx->plane_res.mi;
-		struct hubp *hubp = pipe_ctx->plane_res.hubp;
-		struct dpp *dpp = pipe_ctx->plane_res.dpp;
-		struct dc_cursor_position pos_cpy = *position;
-		struct dc_cursor_mi_param param = {
-			.pixel_clk_khz = stream->timing.pix_clk_khz,
-			.ref_clk_khz = core_dc->res_pool->ref_clock_inKhz,
-			.viewport_x_start = pipe_ctx->plane_res.scl_data.viewport.x,
-			.viewport_width = pipe_ctx->plane_res.scl_data.viewport.width,
-			.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz
-		};
 
 		if (pipe_ctx->stream != stream ||
 				(!pipe_ctx->plane_res.mi  && !pipe_ctx->plane_res.hubp) ||
 				!pipe_ctx->plane_state ||
-				(!pipe_ctx->plane_res.xfm && !pipe_ctx->plane_res.dpp))
+				(!pipe_ctx->plane_res.xfm && !pipe_ctx->plane_res.dpp) ||
+				!pipe_ctx->plane_res.ipp)
 			continue;
 
-		if (pipe_ctx->plane_state->address.type
-				== PLN_ADDR_TYPE_VIDEO_PROGRESSIVE)
-			pos_cpy.enable = false;
-
-		if (pipe_ctx->top_pipe && pipe_ctx->plane_state != pipe_ctx->top_pipe->plane_state)
-			pos_cpy.enable = false;
-
-
-		if (ipp != NULL && ipp->funcs->ipp_cursor_set_position != NULL)
-			ipp->funcs->ipp_cursor_set_position(ipp, &pos_cpy, &param);
-
-		if (mi != NULL && mi->funcs->set_cursor_position != NULL)
-			mi->funcs->set_cursor_position(mi, &pos_cpy, &param);
-
-		if (!hubp)
-			continue;
-
-		if (hubp->funcs->set_cursor_position != NULL)
-			hubp->funcs->set_cursor_position(hubp, &pos_cpy, &param);
-
-		if (dpp != NULL && dpp->funcs->set_cursor_position != NULL)
-			dpp->funcs->set_cursor_position(dpp, &pos_cpy, &param, hubp->curs_attr.width);
-
+		core_dc->hwss.set_cursor_position(pipe_ctx);
 	}
 
 	return true;
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 9d8f4a55c74e..d6d56611604e 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -38,7 +38,7 @@
 #include "inc/compressor.h"
 #include "dml/display_mode_lib.h"
 
-#define DC_VER "3.1.07"
+#define DC_VER "3.1.27"
 
 #define MAX_SURFACES 3
 #define MAX_STREAMS 6
@@ -58,8 +58,11 @@ struct dc_caps {
 	uint32_t i2c_speed_in_khz;
 	unsigned int max_cursor_size;
 	unsigned int max_video_width;
+	int linear_pitch_alignment;
 	bool dcc_const_color;
 	bool dynamic_audio;
+	bool is_apu;
+	bool dual_link_dvi;
 };
 
 struct dc_dcc_surface_param {
@@ -97,69 +100,53 @@ struct dc_static_screen_events {
 	bool overlay_update;
 };
 
+
+/* Surface update type is used by dc_update_surfaces_and_stream
+ * The update type is determined at the very beginning of the function based
+ * on parameters passed in and decides how much programming (or updating) is
+ * going to be done during the call.
+ *
+ * UPDATE_TYPE_FAST is used for really fast updates that do not require much
+ * logical calculations or hardware register programming. This update MUST be
+ * ISR safe on windows. Currently fast update will only be used to flip surface
+ * address.
+ *
+ * UPDATE_TYPE_MED is used for slower updates which require significant hw
+ * re-programming however do not affect bandwidth consumption or clock
+ * requirements. At present, this is the level at which front end updates
+ * that do not require us to run bw_calcs happen. These are in/out transfer func
+ * updates, viewport offset changes, recout size changes and pixel depth changes.
+ * This update can be done at ISR, but we want to minimize how often this happens.
+ *
+ * UPDATE_TYPE_FULL is slow. Really slow. This requires us to recalculate our
+ * bandwidth and clocks, possibly rearrange some pipes and reprogram anything front
+ * end related. Any time viewport dimensions, recout dimensions, scaling ratios or
+ * gamma need to be adjusted or pipe needs to be turned on (or disconnected) we do
+ * a full update. This cannot be done at ISR level and should be a rare event.
+ * Unless someone is stress testing mpo enter/exit, playing with colour or adjusting
+ * underscan we don't expect to see this call at all.
+ */
+
+enum surface_update_type {
+	UPDATE_TYPE_FAST, /* super fast, safe to execute in isr */
+	UPDATE_TYPE_MED,  /* ISR safe, most of programming needed, no bw/clk change*/
+	UPDATE_TYPE_FULL, /* may need to shuffle resources */
+};
+
 /* Forward declaration*/
 struct dc;
 struct dc_plane_state;
 struct dc_state;
 
+
 struct dc_cap_funcs {
 	bool (*get_dcc_compression_cap)(const struct dc *dc,
 			const struct dc_dcc_surface_param *input,
 			struct dc_surface_dcc_cap *output);
 };
 
-struct dc_stream_state_funcs {
-	bool (*adjust_vmin_vmax)(struct dc *dc,
-			struct dc_stream_state **stream,
-			int num_streams,
-			int vmin,
-			int vmax);
-	bool (*get_crtc_position)(struct dc *dc,
-			struct dc_stream_state **stream,
-			int num_streams,
-			unsigned int *v_pos,
-			unsigned int *nom_v_pos);
-
-	bool (*set_gamut_remap)(struct dc *dc,
-			const struct dc_stream_state *stream);
-
-	bool (*program_csc_matrix)(struct dc *dc,
-			struct dc_stream_state *stream);
-
-	void (*set_static_screen_events)(struct dc *dc,
-			struct dc_stream_state **stream,
-			int num_streams,
-			const struct dc_static_screen_events *events);
-
-	void (*set_dither_option)(struct dc_stream_state *stream,
-			enum dc_dither_option option);
-
-	void (*set_dpms)(struct dc *dc,
-			struct dc_stream_state *stream,
-			bool dpms_off);
-};
-
 struct link_training_settings;
 
-struct dc_link_funcs {
-	void (*set_drive_settings)(struct dc *dc,
-			struct link_training_settings *lt_settings,
-			const struct dc_link *link);
-	void (*perform_link_training)(struct dc *dc,
-			struct dc_link_settings *link_setting,
-			bool skip_video_pattern);
-	void (*set_preferred_link_settings)(struct dc *dc,
-			struct dc_link_settings *link_setting,
-			struct dc_link *link);
-	void (*enable_hpd)(const struct dc_link *link);
-	void (*disable_hpd)(const struct dc_link *link);
-	void (*set_test_pattern)(
-			struct dc_link *link,
-			enum dp_test_pattern test_pattern,
-			const struct link_training_settings *p_link_settings,
-			const unsigned char *p_custom_pattern,
-			unsigned int cust_pattern_size);
-};
 
 /* Structure to hold configuration flags set by dm at dc creation. */
 struct dc_config {
@@ -232,8 +219,6 @@ struct dce_hwseq;
 struct dc {
 	struct dc_caps caps;
 	struct dc_cap_funcs cap_funcs;
-	struct dc_stream_state_funcs stream_funcs;
-	struct dc_link_funcs link_funcs;
 	struct dc_config config;
 	struct dc_debug debug;
 
@@ -266,6 +251,8 @@ struct dc {
 	 */
 	struct dm_pp_display_configuration prev_display_config;
 
+	bool optimized_required;
+
 	/* FBC compressor */
 #if defined(CONFIG_DRM_AMD_DC_FBC)
 	struct compressor *fbc_compressor;
@@ -333,24 +320,6 @@ enum color_transfer_func {
 	transfer_func_gamma_26
 };
 
-enum color_color_space {
-	color_space_unsupported,
-	color_space_srgb,
-	color_space_bt601,
-	color_space_bt709,
-	color_space_xv_ycc_bt601,
-	color_space_xv_ycc_bt709,
-	color_space_xr_rgb,
-	color_space_bt2020,
-	color_space_adobe,
-	color_space_dci_p3,
-	color_space_sc_rgb_ms_ref,
-	color_space_display_native,
-	color_space_app_ctrl,
-	color_space_dolby_vision,
-	color_space_custom_coordinates
-};
-
 struct dc_hdr_static_metadata {
 	/* display chromaticities and white point in units of 0.00001 */
 	unsigned int chromaticity_green_x;
@@ -374,7 +343,7 @@ struct dc_hdr_static_metadata {
 enum dc_transfer_func_type {
 	TF_TYPE_PREDEFINED,
 	TF_TYPE_DISTRIBUTED_POINTS,
-	TF_TYPE_BYPASS
+	TF_TYPE_BYPASS,
 };
 
 struct dc_transfer_func_distributed_points {
@@ -393,6 +362,7 @@ enum dc_transfer_func_predefined {
 	TRANSFER_FUNCTION_BT709,
 	TRANSFER_FUNCTION_PQ,
 	TRANSFER_FUNCTION_LINEAR,
+	TRANSFER_FUNCTION_UNITY,
 };
 
 struct dc_transfer_func {
@@ -415,6 +385,34 @@ struct dc_plane_status {
 	bool is_right_eye;
 };
 
+union surface_update_flags {
+
+	struct {
+		/* Medium updates */
+		uint32_t dcc_change:1;
+		uint32_t color_space_change:1;
+		uint32_t input_tf_change:1;
+		uint32_t horizontal_mirror_change:1;
+		uint32_t per_pixel_alpha_change:1;
+		uint32_t rotation_change:1;
+		uint32_t swizzle_change:1;
+		uint32_t scaling_change:1;
+		uint32_t position_change:1;
+		uint32_t in_transfer_func:1;
+		uint32_t input_csc_change:1;
+
+		/* Full updates */
+		uint32_t new_plane:1;
+		uint32_t bpp_change:1;
+		uint32_t bandwidth_change:1;
+		uint32_t clock_change:1;
+		uint32_t stereo_format_change:1;
+		uint32_t full_update:1;
+	} bits;
+
+	uint32_t raw;
+};
+
 struct dc_plane_state {
 	struct dc_plane_address address;
 	struct scaling_taps scaling_quality;
@@ -426,27 +424,30 @@ struct dc_plane_state {
 	union dc_tiling_info tiling_info;
 
 	struct dc_plane_dcc_param dcc;
-	struct dc_hdr_static_metadata hdr_static_ctx;
 
 	struct dc_gamma *gamma_correction;
 	struct dc_transfer_func *in_transfer_func;
+	struct dc_bias_and_scale *bias_and_scale;
+	struct csc_transform input_csc_color_matrix;
+	struct fixed31_32 coeff_reduction_factor;
 
-	// sourceContentAttribute cache
-	bool is_source_input_valid;
-	struct dc_hdr_static_metadata source_input_mastering_info;
-	enum color_color_space source_input_color_space;
-	enum color_transfer_func source_input_tf;
+	// TODO: No longer used, remove
+	struct dc_hdr_static_metadata hdr_static_ctx;
 
 	enum dc_color_space color_space;
+	enum color_transfer_func input_tf;
+
 	enum surface_pixel_format format;
 	enum dc_rotation_angle rotation;
 	enum plane_stereo_format stereo_format;
 
+	bool is_tiling_rotated;
 	bool per_pixel_alpha;
 	bool visible;
 	bool flip_immediate;
 	bool horizontal_mirror;
 
+	union surface_update_flags update_flags;
 	/* private to DC core */
 	struct dc_plane_status status;
 	struct dc_context *ctx;
@@ -463,10 +464,12 @@ struct dc_plane_info {
 	enum surface_pixel_format format;
 	enum dc_rotation_angle rotation;
 	enum plane_stereo_format stereo_format;
-	enum dc_color_space color_space; /*todo: wrong place, fits in scaling info*/
+	enum dc_color_space color_space;
+	enum color_transfer_func input_tf;
 	bool horizontal_mirror;
 	bool visible;
 	bool per_pixel_alpha;
+	bool input_csc_enabled;
 };
 
 struct dc_scaling_info {
@@ -483,13 +486,18 @@ struct dc_surface_update {
 	struct dc_flip_addrs *flip_addr;
 	struct dc_plane_info *plane_info;
 	struct dc_scaling_info *scaling_info;
+
 	/* following updates require alloc/sleep/spin that is not isr safe,
 	 * null means no updates
 	 */
 	/* gamma TO BE REMOVED */
 	struct dc_gamma *gamma;
+	enum color_transfer_func color_input_tf;
+	enum color_transfer_func color_output_tf;
 	struct dc_transfer_func *in_transfer_func;
-	struct dc_hdr_static_metadata *hdr_static_metadata;
+
+	struct csc_transform *input_csc_color_matrix;
+	struct fixed31_32 *coeff_reduction_factor;
 };
 
 /*
@@ -524,197 +532,7 @@ struct dc_flip_addrs {
 bool dc_post_update_surfaces_to_stream(
 		struct dc *dc);
 
-/* Surface update type is used by dc_update_surfaces_and_stream
- * The update type is determined at the very beginning of the function based
- * on parameters passed in and decides how much programming (or updating) is
- * going to be done during the call.
- *
- * UPDATE_TYPE_FAST is used for really fast updates that do not require much
- * logical calculations or hardware register programming. This update MUST be
- * ISR safe on windows. Currently fast update will only be used to flip surface
- * address.
- *
- * UPDATE_TYPE_MED is used for slower updates which require significant hw
- * re-programming however do not affect bandwidth consumption or clock
- * requirements. At present, this is the level at which front end updates
- * that do not require us to run bw_calcs happen. These are in/out transfer func
- * updates, viewport offset changes, recout size changes and pixel depth changes.
- * This update can be done at ISR, but we want to minimize how often this happens.
- *
- * UPDATE_TYPE_FULL is slow. Really slow. This requires us to recalculate our
- * bandwidth and clocks, possibly rearrange some pipes and reprogram anything front
- * end related. Any time viewport dimensions, recout dimensions, scaling ratios or
- * gamma need to be adjusted or pipe needs to be turned on (or disconnected) we do
- * a full update. This cannot be done at ISR level and should be a rare event.
- * Unless someone is stress testing mpo enter/exit, playing with colour or adjusting
- * underscan we don't expect to see this call at all.
- */
-
-enum surface_update_type {
-	UPDATE_TYPE_FAST, /* super fast, safe to execute in isr */
-	UPDATE_TYPE_MED,  /* ISR safe, most of programming needed, no bw/clk change*/
-	UPDATE_TYPE_FULL, /* may need to shuffle resources */
-};
-
-/*******************************************************************************
- * Stream Interfaces
- ******************************************************************************/
-
-struct dc_stream_status {
-	int primary_otg_inst;
-	int stream_enc_inst;
-	int plane_count;
-	struct dc_plane_state *plane_states[MAX_SURFACE_NUM];
-
-	/*
-	 * link this stream passes through
-	 */
-	struct dc_link *link;
-};
-
-struct dc_stream_state {
-	struct dc_sink *sink;
-	struct dc_crtc_timing timing;
-
-	struct rect src; /* composition area */
-	struct rect dst; /* stream addressable area */
-
-	struct audio_info audio_info;
-
-	struct freesync_context freesync_ctx;
-
-	struct dc_transfer_func *out_transfer_func;
-	struct colorspace_transform gamut_remap_matrix;
-	struct csc_transform csc_color_matrix;
-
-	enum signal_type output_signal;
-
-	enum dc_color_space output_color_space;
-	enum dc_dither_option dither_option;
-
-	enum view_3d_format view_format;
-
-	bool ignore_msa_timing_param;
-	/* TODO: custom INFO packets */
-	/* TODO: ABM info (DMCU) */
-	/* TODO: PSR info */
-	/* TODO: CEA VIC */
-
-	/* from core_stream struct */
-	struct dc_context *ctx;
-
-	/* used by DCP and FMT */
-	struct bit_depth_reduction_params bit_depth_params;
-	struct clamping_and_pixel_encoding_params clamping;
-
-	int phy_pix_clk;
-	enum signal_type signal;
-	bool dpms_off;
-
-	struct dc_stream_status status;
-
-	struct dc_cursor_attributes cursor_attributes;
-
-	/* from stream struct */
-	struct kref refcount;
-};
-
-struct dc_stream_update {
-	struct rect src;
-	struct rect dst;
-	struct dc_transfer_func *out_transfer_func;
-};
-
-bool dc_is_stream_unchanged(
-	struct dc_stream_state *old_stream, struct dc_stream_state *stream);
-bool dc_is_stream_scaling_unchanged(
-	struct dc_stream_state *old_stream, struct dc_stream_state *stream);
-
-/*
- * Set up surface attributes and associate to a stream
- * The surfaces parameter is an absolute set of all surface active for the stream.
- * If no surfaces are provided, the stream will be blanked; no memory read.
- * Any flip related attribute changes must be done through this interface.
- *
- * After this call:
- *   Surfaces attributes are programmed and configured to be composed into stream.
- *   This does not trigger a flip.  No surface address is programmed.
- */
-
-bool dc_commit_planes_to_stream(
-		struct dc *dc,
-		struct dc_plane_state **plane_states,
-		uint8_t new_plane_count,
-		struct dc_stream_state *dc_stream,
-		struct dc_state *state);
-
-void dc_commit_updates_for_stream(struct dc *dc,
-		struct dc_surface_update *srf_updates,
-		int surface_count,
-		struct dc_stream_state *stream,
-		struct dc_stream_update *stream_update,
-		struct dc_plane_state **plane_states,
-		struct dc_state *state);
-/*
- * Log the current stream state.
- */
-void dc_stream_log(
-	const struct dc_stream_state *stream,
-	struct dal_logger *dc_logger,
-	enum dc_log_type log_type);
-
-uint8_t dc_get_current_stream_count(struct dc *dc);
-struct dc_stream_state *dc_get_stream_at_index(struct dc *dc, uint8_t i);
-
-/*
- * Return the current frame counter.
- */
-uint32_t dc_stream_get_vblank_counter(const struct dc_stream_state *stream);
-
-/* TODO: Return parsed values rather than direct register read
- * This has a dependency on the caller (amdgpu_get_crtc_scanoutpos)
- * being refactored properly to be dce-specific
- */
-bool dc_stream_get_scanoutpos(const struct dc_stream_state *stream,
-				  uint32_t *v_blank_start,
-				  uint32_t *v_blank_end,
-				  uint32_t *h_position,
-				  uint32_t *v_position);
-
-enum dc_status dc_add_stream_to_ctx(
-			struct dc *dc,
-		struct dc_state *new_ctx,
-		struct dc_stream_state *stream);
-
-enum dc_status dc_remove_stream_from_ctx(
-		struct dc *dc,
-			struct dc_state *new_ctx,
-			struct dc_stream_state *stream);
-
-
-bool dc_add_plane_to_context(
-		const struct dc *dc,
-		struct dc_stream_state *stream,
-		struct dc_plane_state *plane_state,
-		struct dc_state *context);
-
-bool dc_remove_plane_from_context(
-		const struct dc *dc,
-		struct dc_stream_state *stream,
-		struct dc_plane_state *plane_state,
-		struct dc_state *context);
-
-bool dc_rem_all_planes_for_stream(
-		const struct dc *dc,
-		struct dc_stream_state *stream,
-		struct dc_state *context);
-
-bool dc_add_all_planes_for_stream(
-		const struct dc *dc,
-		struct dc_stream_state *stream,
-		struct dc_plane_state * const *plane_states,
-		int plane_count,
-		struct dc_state *context);
+#include "dc_stream.h"
 
 /*
  * Structure to store surface/stream associations for validation
@@ -725,22 +543,12 @@ struct dc_validation_set {
 	uint8_t plane_count;
 };
 
-enum dc_status dc_validate_stream(struct dc *dc, struct dc_stream_state *stream);
-
 enum dc_status dc_validate_plane(struct dc *dc, const struct dc_plane_state *plane_state);
 
 enum dc_status dc_validate_global_state(
 		struct dc *dc,
 		struct dc_state *new_ctx);
 
-/*
- * This function takes a stream and checks if it is guaranteed to be supported.
- * Guaranteed means that MAX_COFUNC similar streams are supported.
- *
- * After this call:
- *   No hardware is programmed for call.  Only validation is done.
- */
-
 
 void dc_resource_state_construct(
 		const struct dc *dc,
@@ -767,42 +575,6 @@ void dc_resource_state_destruct(struct dc_state *context);
  */
 bool dc_commit_state(struct dc *dc, struct dc_state *context);
 
-/*
- * Set up streams and links associated to drive sinks
- * The streams parameter is an absolute set of all active streams.
- *
- * After this call:
- *   Phy, Encoder, Timing Generator are programmed and enabled.
- *   New streams are enabled with blank stream; no memory read.
- */
-/*
- * Enable stereo when commit_streams is not required,
- * for example, frame alternate.
- */
-bool dc_enable_stereo(
-	struct dc *dc,
-	struct dc_state *context,
-	struct dc_stream_state *streams[],
-	uint8_t stream_count);
-
-/**
- * Create a new default stream for the requested sink
- */
-struct dc_stream_state *dc_create_stream_for_sink(struct dc_sink *dc_sink);
-
-void dc_stream_retain(struct dc_stream_state *dc_stream);
-void dc_stream_release(struct dc_stream_state *dc_stream);
-
-struct dc_stream_status *dc_stream_get_status(
-	struct dc_stream_state *dc_stream);
-
-enum surface_update_type dc_check_update_surfaces_for_stream(
-		struct dc *dc,
-		struct dc_surface_update *updates,
-		int surface_count,
-		struct dc_stream_update *stream_update,
-		const struct dc_stream_status *stream_status);
-
 
 struct dc_state *dc_create_state(void);
 void dc_retain_state(struct dc_state *context);
@@ -835,171 +607,7 @@ struct dpcd_caps {
 	bool dpcd_display_control_capable;
 };
 
-struct dc_link_status {
-	struct dpcd_caps *dpcd_caps;
-};
-
-/* DP MST stream allocation (payload bandwidth number) */
-struct link_mst_stream_allocation {
-	/* DIG front */
-	const struct stream_encoder *stream_enc;
-	/* associate DRM payload table with DC stream encoder */
-	uint8_t vcp_id;
-	/* number of slots required for the DP stream in transport packet */
-	uint8_t slot_count;
-};
-
-/* DP MST stream allocation table */
-struct link_mst_stream_allocation_table {
-	/* number of DP video streams */
-	int stream_count;
-	/* array of stream allocations */
-	struct link_mst_stream_allocation stream_allocations[MAX_CONTROLLER_NUM];
-};
-
-/*
- * A link contains one or more sinks and their connected status.
- * The currently active signal type (HDMI, DP-SST, DP-MST) is also reported.
- */
-struct dc_link {
-	struct dc_sink *remote_sinks[MAX_SINKS_PER_LINK];
-	unsigned int sink_count;
-	struct dc_sink *local_sink;
-	unsigned int link_index;
-	enum dc_connection_type type;
-	enum signal_type connector_signal;
-	enum dc_irq_source irq_source_hpd;
-	enum dc_irq_source irq_source_hpd_rx;/* aka DP Short Pulse  */
-	/* caps is the same as reported_link_cap. link_traing use
-	 * reported_link_cap. Will clean up.  TODO
-	 */
-	struct dc_link_settings reported_link_cap;
-	struct dc_link_settings verified_link_cap;
-	struct dc_link_settings cur_link_settings;
-	struct dc_lane_settings cur_lane_setting;
-	struct dc_link_settings preferred_link_setting;
-
-	uint8_t ddc_hw_inst;
-
-	uint8_t hpd_src;
-
-	uint8_t link_enc_hw_inst;
-
-	bool test_pattern_enabled;
-	union compliance_test_state compliance_test_state;
-
-	void *priv;
-
-	struct ddc_service *ddc;
-
-	bool aux_mode;
-
-	/* Private to DC core */
-
-	const struct dc *dc;
-
-	struct dc_context *ctx;
-
-	struct link_encoder *link_enc;
-	struct graphics_object_id link_id;
-	union ddi_channel_mapping ddi_channel_mapping;
-	struct connector_device_tag_info device_tag;
-	struct dpcd_caps dpcd_caps;
-	unsigned short chip_caps;
-	unsigned int dpcd_sink_count;
-	enum edp_revision edp_revision;
-	bool psr_enabled;
-
-	/* MST record stream using this link */
-	struct link_flags {
-		bool dp_keep_receiver_powered;
-	} wa_flags;
-	struct link_mst_stream_allocation_table mst_stream_alloc_table;
-
-	struct dc_link_status link_status;
-
-};
-
-const struct dc_link_status *dc_link_get_status(const struct dc_link *dc_link);
-
-/*
- * Return an enumerated dc_link.  dc_link order is constant and determined at
- * boot time.  They cannot be created or destroyed.
- * Use dc_get_caps() to get number of links.
- */
-static inline struct dc_link *dc_get_link_at_index(struct dc *dc, uint32_t link_index)
-{
-	return dc->links[link_index];
-}
-
-/* Set backlight level of an embedded panel (eDP, LVDS). */
-bool dc_link_set_backlight_level(const struct dc_link *dc_link, uint32_t level,
-		uint32_t frame_ramp, const struct dc_stream_state *stream);
-
-bool dc_link_set_psr_enable(const struct dc_link *dc_link, bool enable, bool wait);
-
-bool dc_link_get_psr_state(const struct dc_link *dc_link, uint32_t *psr_state);
-
-bool dc_link_setup_psr(struct dc_link *dc_link,
-		const struct dc_stream_state *stream, struct psr_config *psr_config,
-		struct psr_context *psr_context);
-
-/* Request DC to detect if there is a Panel connected.
- * boot - If this call is during initial boot.
- * Return false for any type of detection failure or MST detection
- * true otherwise. True meaning further action is required (status update
- * and OS notification).
- */
-enum dc_detect_reason {
-	DETECT_REASON_BOOT,
-	DETECT_REASON_HPD,
-	DETECT_REASON_HPDRX,
-};
-
-bool dc_link_detect(struct dc_link *dc_link, enum dc_detect_reason reason);
-
-/* Notify DC about DP RX Interrupt (aka Short Pulse Interrupt).
- * Return:
- * true - Downstream port status changed. DM should call DC to do the
- * detection.
- * false - no change in Downstream port status. No further action required
- * from DM. */
-bool dc_link_handle_hpd_rx_irq(struct dc_link *dc_link,
-		union hpd_irq_data *hpd_irq_dpcd_data);
-
-struct dc_sink_init_data;
-
-struct dc_sink *dc_link_add_remote_sink(
-		struct dc_link *dc_link,
-		const uint8_t *edid,
-		int len,
-		struct dc_sink_init_data *init_data);
-
-void dc_link_remove_remote_sink(
-	struct dc_link *link,
-	struct dc_sink *sink);
-
-/* Used by diagnostics for virtual link at the moment */
-
-void dc_link_dp_set_drive_settings(
-	struct dc_link *link,
-	struct link_training_settings *lt_settings);
-
-enum link_training_result dc_link_dp_perform_link_training(
-	struct dc_link *link,
-	const struct dc_link_settings *link_setting,
-	bool skip_video_pattern);
-
-void dc_link_dp_enable_hpd(const struct dc_link *link);
-
-void dc_link_dp_disable_hpd(const struct dc_link *link);
-
-bool dc_link_dp_set_test_pattern(
-	struct dc_link *link,
-	enum dp_test_pattern test_pattern,
-	const struct link_training_settings *p_link_settings,
-	const unsigned char *p_custom_pattern,
-	unsigned int cust_pattern_size);
+#include "dc_link.h"
 
 /*******************************************************************************
  * Sink Interfaces - A sink corresponds to a display output device
@@ -1037,6 +645,7 @@ struct dc_sink {
 
 	/* private to dc_sink.c */
 	struct kref refcount;
+
 };
 
 void dc_sink_retain(struct dc_sink *sink);
@@ -1051,18 +660,6 @@ struct dc_sink_init_data {
 
 struct dc_sink *dc_sink_create(const struct dc_sink_init_data *init_params);
 
-/*******************************************************************************
- * Cursor interfaces - To manages the cursor within a stream
- ******************************************************************************/
-/* TODO: Deprecated once we switch to dc_set_cursor_position */
-bool dc_stream_set_cursor_attributes(
-	struct dc_stream_state *stream,
-	const struct dc_cursor_attributes *attributes);
-
-bool dc_stream_set_cursor_position(
-	struct dc_stream_state *stream,
-	const struct dc_cursor_position *position);
-
 /* Newer interfaces  */
 struct dc_cursor {
 	struct dc_plane_address address;
@@ -1076,7 +673,7 @@ enum dc_irq_source dc_interrupt_to_irq_source(
 		struct dc *dc,
 		uint32_t src_id,
 		uint32_t ext_id);
-void dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable);
+bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable);
 void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src);
 enum dc_irq_source dc_get_hpd_irq_source_at_index(
 		struct dc *dc, uint32_t link_index);
@@ -1090,14 +687,4 @@ void dc_set_power_state(
 		enum dc_acpi_cm_power_state power_state);
 void dc_resume(struct dc *dc);
 
-/*
- * DPCD access interfaces
- */
-
-bool dc_submit_i2c(
-		struct dc *dc,
-		uint32_t link_index,
-		struct i2c_command *cmd);
-
-
 #endif /* DC_INTERFACE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index 77e2de69cca3..2726b02e006b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -255,7 +255,7 @@ enum dpcd_downstream_port_detailed_type {
 	DOWN_STREAM_DETAILED_DP_PLUS_PLUS
 };
 
-union dwnstream_port_caps_byte1 {
+union dwnstream_port_caps_byte2 {
 	struct {
 		uint8_t MAX_BITS_PER_COLOR_COMPONENT:2;
 		uint8_t RESERVED:6;
@@ -298,6 +298,32 @@ union dwnstream_port_caps_byte3_hdmi {
 
 /*4-byte structure for detailed capabilities of a down-stream port
 (DP-to-TMDS converter).*/
+union dwnstream_portxcaps {
+	struct {
+		union dwnstream_port_caps_byte0 byte0;
+		unsigned char max_TMDS_clock;   //byte1
+		union dwnstream_port_caps_byte2 byte2;
+
+		union {
+			union dwnstream_port_caps_byte3_dvi byteDVI;
+			union dwnstream_port_caps_byte3_hdmi byteHDMI;
+		} byte3;
+	} bytes;
+
+	unsigned char raw[4];
+};
+
+union downstream_port {
+	struct {
+		unsigned char   present:1;
+		unsigned char   type:2;
+		unsigned char   format_conv:1;
+		unsigned char   detailed_caps:1;
+		unsigned char   reserved:3;
+	} bits;
+	unsigned char raw;
+};
+
 
 union sink_status {
 	struct {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index 90e81f7ba919..48e1fcf53d43 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -178,8 +178,13 @@ uint32_t generic_reg_wait(const struct dc_context *ctx,
 
 		field_value = get_reg_field_value_ex(reg_val, mask, shift);
 
-		if (field_value == condition_value)
+		if (field_value == condition_value) {
+			if (i * delay_between_poll_us > 1000)
+				dm_output_to_console("REG_WAIT taking a while: %dms in %s line:%d\n",
+						delay_between_poll_us * i / 1000,
+						func_name, line);
 			return reg_val;
+		}
 	}
 
 	dm_error("REG_WAIT timeout %dus * %d tries - %s line:%d\n",
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index 1a9f57fb0838..03029f72dc3f 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -492,15 +492,24 @@ struct dc_cursor_attributes {
 enum dc_color_space {
 	COLOR_SPACE_UNKNOWN,
 	COLOR_SPACE_SRGB,
+	COLOR_SPACE_XR_RGB,
 	COLOR_SPACE_SRGB_LIMITED,
+	COLOR_SPACE_MSREF_SCRGB,
 	COLOR_SPACE_YCBCR601,
 	COLOR_SPACE_YCBCR709,
+	COLOR_SPACE_XV_YCC_709,
+	COLOR_SPACE_XV_YCC_601,
 	COLOR_SPACE_YCBCR601_LIMITED,
 	COLOR_SPACE_YCBCR709_LIMITED,
 	COLOR_SPACE_2020_RGB_FULLRANGE,
 	COLOR_SPACE_2020_RGB_LIMITEDRANGE,
 	COLOR_SPACE_2020_YCBCR,
 	COLOR_SPACE_ADOBERGB,
+	COLOR_SPACE_DCIP3,
+	COLOR_SPACE_DISPLAYNATIVE,
+	COLOR_SPACE_DOLBYVISION,
+	COLOR_SPACE_APPCTRL,
+	COLOR_SPACE_CUSTOMPOINTS,
 };
 
 enum dc_dither_option {
@@ -570,8 +579,6 @@ enum dc_timing_standard {
 	TIMING_STANDARD_MAX
 };
 
-
-
 enum dc_color_depth {
 	COLOR_DEPTH_UNDEFINED,
 	COLOR_DEPTH_666,
@@ -664,6 +671,22 @@ enum dc_timing_3d_format {
 	TIMING_3D_FORMAT_MAX,
 };
 
+enum trigger_delay {
+	TRIGGER_DELAY_NEXT_PIXEL = 0,
+	TRIGGER_DELAY_NEXT_LINE,
+};
+
+enum crtc_event {
+	CRTC_EVENT_VSYNC_RISING = 0,
+	CRTC_EVENT_VSYNC_FALLING
+};
+
+struct crtc_trigger_info {
+	bool enabled;
+	struct dc_stream_state *event_source;
+	enum crtc_event event;
+	enum trigger_delay delay;
+};
 
 struct dc_crtc_timing {
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h
new file mode 100644
index 000000000000..f11a734da1db
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dc_link.h
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2012-14 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DC_LINK_H_
+#define DC_LINK_H_
+
+#include "dc_types.h"
+#include "grph_object_defs.h"
+
+struct dc_link_status {
+	struct dpcd_caps *dpcd_caps;
+};
+
+/* DP MST stream allocation (payload bandwidth number) */
+struct link_mst_stream_allocation {
+	/* DIG front */
+	const struct stream_encoder *stream_enc;
+	/* associate DRM payload table with DC stream encoder */
+	uint8_t vcp_id;
+	/* number of slots required for the DP stream in transport packet */
+	uint8_t slot_count;
+};
+
+/* DP MST stream allocation table */
+struct link_mst_stream_allocation_table {
+	/* number of DP video streams */
+	int stream_count;
+	/* array of stream allocations */
+	struct link_mst_stream_allocation stream_allocations[MAX_CONTROLLER_NUM];
+};
+
+/*
+ * A link contains one or more sinks and their connected status.
+ * The currently active signal type (HDMI, DP-SST, DP-MST) is also reported.
+ */
+struct dc_link {
+	struct dc_sink *remote_sinks[MAX_SINKS_PER_LINK];
+	unsigned int sink_count;
+	struct dc_sink *local_sink;
+	unsigned int link_index;
+	enum dc_connection_type type;
+	enum signal_type connector_signal;
+	enum dc_irq_source irq_source_hpd;
+	enum dc_irq_source irq_source_hpd_rx;/* aka DP Short Pulse  */
+	/* caps is the same as reported_link_cap. link_traing use
+	 * reported_link_cap. Will clean up.  TODO
+	 */
+	struct dc_link_settings reported_link_cap;
+	struct dc_link_settings verified_link_cap;
+	struct dc_link_settings cur_link_settings;
+	struct dc_lane_settings cur_lane_setting;
+	struct dc_link_settings preferred_link_setting;
+
+	uint8_t ddc_hw_inst;
+
+	uint8_t hpd_src;
+
+	uint8_t link_enc_hw_inst;
+
+	bool test_pattern_enabled;
+	union compliance_test_state compliance_test_state;
+
+	void *priv;
+
+	struct ddc_service *ddc;
+
+	bool aux_mode;
+
+	/* Private to DC core */
+
+	const struct dc *dc;
+
+	struct dc_context *ctx;
+
+	struct link_encoder *link_enc;
+	struct graphics_object_id link_id;
+	union ddi_channel_mapping ddi_channel_mapping;
+	struct connector_device_tag_info device_tag;
+	struct dpcd_caps dpcd_caps;
+	unsigned short chip_caps;
+	unsigned int dpcd_sink_count;
+	enum edp_revision edp_revision;
+	bool psr_enabled;
+
+	/* MST record stream using this link */
+	struct link_flags {
+		bool dp_keep_receiver_powered;
+	} wa_flags;
+	struct link_mst_stream_allocation_table mst_stream_alloc_table;
+
+	struct dc_link_status link_status;
+
+};
+
+const struct dc_link_status *dc_link_get_status(const struct dc_link *dc_link);
+
+/*
+ * Return an enumerated dc_link.  dc_link order is constant and determined at
+ * boot time.  They cannot be created or destroyed.
+ * Use dc_get_caps() to get number of links.
+ */
+static inline struct dc_link *dc_get_link_at_index(struct dc *dc, uint32_t link_index)
+{
+	return dc->links[link_index];
+}
+
+/* Set backlight level of an embedded panel (eDP, LVDS). */
+bool dc_link_set_backlight_level(const struct dc_link *dc_link, uint32_t level,
+		uint32_t frame_ramp, const struct dc_stream_state *stream);
+
+bool dc_link_set_psr_enable(const struct dc_link *dc_link, bool enable, bool wait);
+
+bool dc_link_get_psr_state(const struct dc_link *dc_link, uint32_t *psr_state);
+
+bool dc_link_setup_psr(struct dc_link *dc_link,
+		const struct dc_stream_state *stream, struct psr_config *psr_config,
+		struct psr_context *psr_context);
+
+/* Request DC to detect if there is a Panel connected.
+ * boot - If this call is during initial boot.
+ * Return false for any type of detection failure or MST detection
+ * true otherwise. True meaning further action is required (status update
+ * and OS notification).
+ */
+enum dc_detect_reason {
+	DETECT_REASON_BOOT,
+	DETECT_REASON_HPD,
+	DETECT_REASON_HPDRX,
+};
+
+bool dc_link_detect(struct dc_link *dc_link, enum dc_detect_reason reason);
+
+/* Notify DC about DP RX Interrupt (aka Short Pulse Interrupt).
+ * Return:
+ * true - Downstream port status changed. DM should call DC to do the
+ * detection.
+ * false - no change in Downstream port status. No further action required
+ * from DM. */
+bool dc_link_handle_hpd_rx_irq(struct dc_link *dc_link,
+		union hpd_irq_data *hpd_irq_dpcd_data);
+
+struct dc_sink_init_data;
+
+struct dc_sink *dc_link_add_remote_sink(
+		struct dc_link *dc_link,
+		const uint8_t *edid,
+		int len,
+		struct dc_sink_init_data *init_data);
+
+void dc_link_remove_remote_sink(
+	struct dc_link *link,
+	struct dc_sink *sink);
+
+/* Used by diagnostics for virtual link at the moment */
+
+void dc_link_dp_set_drive_settings(
+	struct dc_link *link,
+	struct link_training_settings *lt_settings);
+
+enum link_training_result dc_link_dp_perform_link_training(
+	struct dc_link *link,
+	const struct dc_link_settings *link_setting,
+	bool skip_video_pattern);
+
+void dc_link_dp_enable_hpd(const struct dc_link *link);
+
+void dc_link_dp_disable_hpd(const struct dc_link *link);
+
+bool dc_link_dp_set_test_pattern(
+	struct dc_link *link,
+	enum dp_test_pattern test_pattern,
+	const struct link_training_settings *p_link_settings,
+	const unsigned char *p_custom_pattern,
+	unsigned int cust_pattern_size);
+
+/*
+ * DPCD access interfaces
+ */
+
+bool dc_submit_i2c(
+		struct dc *dc,
+		uint32_t link_index,
+		struct i2c_command *cmd);
+
+#endif /* DC_LINK_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
new file mode 100644
index 000000000000..456e4d29eadd
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -0,0 +1,295 @@
+/*
+ * Copyright 2012-14 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef DC_STREAM_H_
+#define DC_STREAM_H_
+
+#include "dc_types.h"
+#include "grph_object_defs.h"
+
+/*******************************************************************************
+ * Stream Interfaces
+ ******************************************************************************/
+
+struct dc_stream_status {
+	int primary_otg_inst;
+	int stream_enc_inst;
+	int plane_count;
+	struct dc_plane_state *plane_states[MAX_SURFACE_NUM];
+
+	/*
+	 * link this stream passes through
+	 */
+	struct dc_link *link;
+};
+
+struct dc_stream_state {
+	struct dc_sink *sink;
+	struct dc_crtc_timing timing;
+
+	struct rect src; /* composition area */
+	struct rect dst; /* stream addressable area */
+
+	struct audio_info audio_info;
+
+	struct freesync_context freesync_ctx;
+
+	struct dc_hdr_static_metadata hdr_static_metadata;
+	struct dc_transfer_func *out_transfer_func;
+	struct colorspace_transform gamut_remap_matrix;
+	struct csc_transform csc_color_matrix;
+
+	enum dc_color_space output_color_space;
+	enum dc_dither_option dither_option;
+
+	enum view_3d_format view_format;
+
+	bool ignore_msa_timing_param;
+	/* TODO: custom INFO packets */
+	/* TODO: ABM info (DMCU) */
+	/* TODO: PSR info */
+	/* TODO: CEA VIC */
+
+	/* from core_stream struct */
+	struct dc_context *ctx;
+
+	/* used by DCP and FMT */
+	struct bit_depth_reduction_params bit_depth_params;
+	struct clamping_and_pixel_encoding_params clamping;
+
+	int phy_pix_clk;
+	enum signal_type signal;
+	bool dpms_off;
+
+	struct dc_stream_status status;
+
+	struct dc_cursor_attributes cursor_attributes;
+	struct dc_cursor_position cursor_position;
+
+	/* from stream struct */
+	struct kref refcount;
+
+	struct crtc_trigger_info triggered_crtc_reset;
+
+	/* Computed state bits */
+	bool mode_changed : 1;
+
+};
+
+struct dc_stream_update {
+	struct rect src;
+	struct rect dst;
+	struct dc_transfer_func *out_transfer_func;
+	struct dc_hdr_static_metadata *hdr_static_metadata;
+};
+
+bool dc_is_stream_unchanged(
+	struct dc_stream_state *old_stream, struct dc_stream_state *stream);
+bool dc_is_stream_scaling_unchanged(
+	struct dc_stream_state *old_stream, struct dc_stream_state *stream);
+
+/*
+ * Set up surface attributes and associate to a stream
+ * The surfaces parameter is an absolute set of all surface active for the stream.
+ * If no surfaces are provided, the stream will be blanked; no memory read.
+ * Any flip related attribute changes must be done through this interface.
+ *
+ * After this call:
+ *   Surfaces attributes are programmed and configured to be composed into stream.
+ *   This does not trigger a flip.  No surface address is programmed.
+ */
+
+bool dc_commit_planes_to_stream(
+		struct dc *dc,
+		struct dc_plane_state **plane_states,
+		uint8_t new_plane_count,
+		struct dc_stream_state *dc_stream,
+		struct dc_state *state);
+
+void dc_commit_updates_for_stream(struct dc *dc,
+		struct dc_surface_update *srf_updates,
+		int surface_count,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		struct dc_plane_state **plane_states,
+		struct dc_state *state);
+/*
+ * Log the current stream state.
+ */
+void dc_stream_log(
+	const struct dc_stream_state *stream,
+	struct dal_logger *dc_logger,
+	enum dc_log_type log_type);
+
+uint8_t dc_get_current_stream_count(struct dc *dc);
+struct dc_stream_state *dc_get_stream_at_index(struct dc *dc, uint8_t i);
+
+/*
+ * Return the current frame counter.
+ */
+uint32_t dc_stream_get_vblank_counter(const struct dc_stream_state *stream);
+
+/* TODO: Return parsed values rather than direct register read
+ * This has a dependency on the caller (amdgpu_get_crtc_scanoutpos)
+ * being refactored properly to be dce-specific
+ */
+bool dc_stream_get_scanoutpos(const struct dc_stream_state *stream,
+				  uint32_t *v_blank_start,
+				  uint32_t *v_blank_end,
+				  uint32_t *h_position,
+				  uint32_t *v_position);
+
+enum dc_status dc_add_stream_to_ctx(
+			struct dc *dc,
+		struct dc_state *new_ctx,
+		struct dc_stream_state *stream);
+
+enum dc_status dc_remove_stream_from_ctx(
+		struct dc *dc,
+			struct dc_state *new_ctx,
+			struct dc_stream_state *stream);
+
+
+bool dc_add_plane_to_context(
+		const struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_plane_state *plane_state,
+		struct dc_state *context);
+
+bool dc_remove_plane_from_context(
+		const struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_plane_state *plane_state,
+		struct dc_state *context);
+
+bool dc_rem_all_planes_for_stream(
+		const struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_state *context);
+
+bool dc_add_all_planes_for_stream(
+		const struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_plane_state * const *plane_states,
+		int plane_count,
+		struct dc_state *context);
+
+enum dc_status dc_validate_stream(struct dc *dc, struct dc_stream_state *stream);
+
+/*
+ * This function takes a stream and checks if it is guaranteed to be supported.
+ * Guaranteed means that MAX_COFUNC similar streams are supported.
+ *
+ * After this call:
+ *   No hardware is programmed for call.  Only validation is done.
+ */
+
+/*
+ * Set up streams and links associated to drive sinks
+ * The streams parameter is an absolute set of all active streams.
+ *
+ * After this call:
+ *   Phy, Encoder, Timing Generator are programmed and enabled.
+ *   New streams are enabled with blank stream; no memory read.
+ */
+/*
+ * Enable stereo when commit_streams is not required,
+ * for example, frame alternate.
+ */
+bool dc_enable_stereo(
+	struct dc *dc,
+	struct dc_state *context,
+	struct dc_stream_state *streams[],
+	uint8_t stream_count);
+
+
+enum surface_update_type dc_check_update_surfaces_for_stream(
+		struct dc *dc,
+		struct dc_surface_update *updates,
+		int surface_count,
+		struct dc_stream_update *stream_update,
+		const struct dc_stream_status *stream_status);
+
+/**
+ * Create a new default stream for the requested sink
+ */
+struct dc_stream_state *dc_create_stream_for_sink(struct dc_sink *dc_sink);
+
+void update_stream_signal(struct dc_stream_state *stream);
+
+void dc_stream_retain(struct dc_stream_state *dc_stream);
+void dc_stream_release(struct dc_stream_state *dc_stream);
+
+struct dc_stream_status *dc_stream_get_status(
+	struct dc_stream_state *dc_stream);
+
+/*******************************************************************************
+ * Cursor interfaces - To manages the cursor within a stream
+ ******************************************************************************/
+/* TODO: Deprecated once we switch to dc_set_cursor_position */
+bool dc_stream_set_cursor_attributes(
+	struct dc_stream_state *stream,
+	const struct dc_cursor_attributes *attributes);
+
+bool dc_stream_set_cursor_position(
+	struct dc_stream_state *stream,
+	const struct dc_cursor_position *position);
+
+bool dc_stream_adjust_vmin_vmax(struct dc *dc,
+				struct dc_stream_state **stream,
+				int num_streams,
+				int vmin,
+				int vmax);
+
+bool dc_stream_get_crtc_position(struct dc *dc,
+				 struct dc_stream_state **stream,
+				 int num_streams,
+				 unsigned int *v_pos,
+				 unsigned int *nom_v_pos);
+
+void dc_stream_set_static_screen_events(struct dc *dc,
+					struct dc_stream_state **stream,
+					int num_streams,
+					const struct dc_static_screen_events *events);
+
+
+bool dc_stream_adjust_vmin_vmax(struct dc *dc,
+				struct dc_stream_state **stream,
+				int num_streams,
+				int vmin,
+				int vmax);
+
+bool dc_stream_get_crtc_position(struct dc *dc,
+				 struct dc_stream_state **stream,
+				 int num_streams,
+				 unsigned int *v_pos,
+				 unsigned int *nom_v_pos);
+
+void dc_stream_set_static_screen_events(struct dc *dc,
+					struct dc_stream_state **stream,
+					int num_streams,
+					const struct dc_static_screen_events *events);
+
+#endif /* DC_STREAM_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index a8698e399111..9faddfae241d 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -218,6 +218,7 @@ struct dc_edid_caps {
 	bool lte_340mcsc_scramble;
 
 	bool edid_hdmi;
+	bool hdr_supported;
 };
 
 struct view {
@@ -638,11 +639,6 @@ struct colorspace_transform {
 	bool enable_remap;
 };
 
-struct csc_transform {
-	uint16_t matrix[12];
-	bool enable_adjustment;
-};
-
 enum i2c_mot_mode {
 	I2C_MOT_UNDEF,
 	I2C_MOT_TRUE,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
index 0e0336c5af4e..b48190f54907 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.c
@@ -51,16 +51,6 @@
 
 #define MCP_DISABLE_ABM_IMMEDIATELY 255
 
-struct abm_backlight_registers {
-	unsigned int BL_PWM_CNTL;
-	unsigned int BL_PWM_CNTL2;
-	unsigned int BL_PWM_PERIOD_CNTL;
-	unsigned int LVTMA_PWRSEQ_REF_DIV_BL_PWM_REF_DIV;
-};
-
-/* registers setting needs to be save and restored used at InitBacklight */
-static struct abm_backlight_registers stored_backlight_registers = {0};
-
 
 static unsigned int get_current_backlight_16_bit(struct dce_abm *abm_dce)
 {
@@ -347,16 +337,16 @@ static bool dce_abm_init_backlight(struct abm *abm)
 	 */
 	REG_GET(BL_PWM_CNTL, BL_ACTIVE_INT_FRAC_CNT, &value);
 	if (value == 0 || value == 1) {
-		if (stored_backlight_registers.BL_PWM_CNTL != 0) {
+		if (abm->stored_backlight_registers.BL_PWM_CNTL != 0) {
 			REG_WRITE(BL_PWM_CNTL,
-				stored_backlight_registers.BL_PWM_CNTL);
+				abm->stored_backlight_registers.BL_PWM_CNTL);
 			REG_WRITE(BL_PWM_CNTL2,
-				stored_backlight_registers.BL_PWM_CNTL2);
+				abm->stored_backlight_registers.BL_PWM_CNTL2);
 			REG_WRITE(BL_PWM_PERIOD_CNTL,
-				stored_backlight_registers.BL_PWM_PERIOD_CNTL);
+				abm->stored_backlight_registers.BL_PWM_PERIOD_CNTL);
 			REG_UPDATE(LVTMA_PWRSEQ_REF_DIV,
 				BL_PWM_REF_DIV,
-				stored_backlight_registers.
+				abm->stored_backlight_registers.
 				LVTMA_PWRSEQ_REF_DIV_BL_PWM_REF_DIV);
 		} else {
 			/* TODO: Note: This should not really happen since VBIOS
@@ -366,15 +356,15 @@ static bool dce_abm_init_backlight(struct abm *abm)
 			REG_WRITE(BL_PWM_PERIOD_CNTL, 0x000C0FA0);
 		}
 	} else {
-		stored_backlight_registers.BL_PWM_CNTL =
+		abm->stored_backlight_registers.BL_PWM_CNTL =
 				REG_READ(BL_PWM_CNTL);
-		stored_backlight_registers.BL_PWM_CNTL2 =
+		abm->stored_backlight_registers.BL_PWM_CNTL2 =
 				REG_READ(BL_PWM_CNTL2);
-		stored_backlight_registers.BL_PWM_PERIOD_CNTL =
+		abm->stored_backlight_registers.BL_PWM_PERIOD_CNTL =
 				REG_READ(BL_PWM_PERIOD_CNTL);
 
 		REG_GET(LVTMA_PWRSEQ_REF_DIV, BL_PWM_REF_DIV,
-				&stored_backlight_registers.
+				&abm->stored_backlight_registers.
 				LVTMA_PWRSEQ_REF_DIV_BL_PWM_REF_DIV);
 	}
 
@@ -395,21 +385,12 @@ static bool dce_abm_init_backlight(struct abm *abm)
 	return true;
 }
 
-static bool is_dmcu_initialized(struct abm *abm)
-{
-	struct dce_abm *abm_dce = TO_DCE_ABM(abm);
-	unsigned int dmcu_uc_reset;
-
-	REG_GET(DMCU_STATUS, UC_IN_RESET, &dmcu_uc_reset);
-
-	return !dmcu_uc_reset;
-}
-
 static bool dce_abm_set_backlight_level(
 		struct abm *abm,
 		unsigned int backlight_level,
 		unsigned int frame_ramp,
-		unsigned int controller_id)
+		unsigned int controller_id,
+		bool use_smooth_brightness)
 {
 	struct dce_abm *abm_dce = TO_DCE_ABM(abm);
 
@@ -418,7 +399,7 @@ static bool dce_abm_set_backlight_level(
 			backlight_level, backlight_level);
 
 	/* If DMCU is in reset state, DMCU is uninitialized */
-	if (is_dmcu_initialized(abm))
+	if (use_smooth_brightness)
 		dmcu_set_backlight_level(abm_dce,
 				backlight_level,
 				frame_ramp,
@@ -435,8 +416,7 @@ static const struct abm_funcs dce_funcs = {
 	.init_backlight = dce_abm_init_backlight,
 	.set_backlight_level = dce_abm_set_backlight_level,
 	.get_current_backlight_8_bit = dce_abm_get_current_backlight_8_bit,
-	.set_abm_immediate_disable = dce_abm_immediate_disable,
-	.is_dmcu_initialized = is_dmcu_initialized
+	.set_abm_immediate_disable = dce_abm_immediate_disable
 };
 
 static void dce_abm_construct(
@@ -450,6 +430,10 @@ static void dce_abm_construct(
 
 	base->ctx = ctx;
 	base->funcs = &dce_funcs;
+	base->stored_backlight_registers.BL_PWM_CNTL = 0;
+	base->stored_backlight_registers.BL_PWM_CNTL2 = 0;
+	base->stored_backlight_registers.BL_PWM_PERIOD_CNTL = 0;
+	base->stored_backlight_registers.LVTMA_PWRSEQ_REF_DIV_BL_PWM_REF_DIV = 0;
 
 	abm_dce->regs = regs;
 	abm_dce->abm_shift = abm_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
index 59e909ec88f2..ff9436966041 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_abm.h
@@ -37,8 +37,7 @@
 	SR(LVTMA_PWRSEQ_REF_DIV), \
 	SR(MASTER_COMM_CNTL_REG), \
 	SR(MASTER_COMM_CMD_REG), \
-	SR(MASTER_COMM_DATA_REG1), \
-	SR(DMCU_STATUS)
+	SR(MASTER_COMM_DATA_REG1)
 
 #define ABM_DCE110_COMMON_REG_LIST() \
 	ABM_COMMON_REG_LIST_DCE_BASE(), \
@@ -84,8 +83,7 @@
 	ABM_SF(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, mask_sh), \
 	ABM_SF(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0, mask_sh), \
 	ABM_SF(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE1, mask_sh), \
-	ABM_SF(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE2, mask_sh), \
-	ABM_SF(DMCU_STATUS, UC_IN_RESET, mask_sh)
+	ABM_SF(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE2, mask_sh)
 
 #define ABM_MASK_SH_LIST_DCE110(mask_sh) \
 	ABM_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(mask_sh), \
@@ -174,7 +172,6 @@
 	type MASTER_COMM_CMD_REG_BYTE2; \
 	type BL_PWM_REF_DIV; \
 	type BL_PWM_EN; \
-	type UC_IN_RESET; \
 	type BL_PWM_GRP1_IGNORE_MASTER_LOCK_EN; \
 	type BL_PWM_GRP1_REG_LOCK; \
 	type BL_PWM_GRP1_REG_UPDATE_PENDING
@@ -206,7 +203,6 @@ struct dce_abm_registers {
 	uint32_t MASTER_COMM_CMD_REG;
 	uint32_t MASTER_COMM_DATA_REG1;
 	uint32_t BIOS_SCRATCH_2;
-	uint32_t DMCU_STATUS;
 	uint32_t BL_PWM_GRP1_REG_LOCK;
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
index 9031d22285ea..9e98a5f39a6d 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clocks.c
@@ -29,7 +29,6 @@
 #include "fixed32_32.h"
 #include "bios_parser_interface.h"
 #include "dc.h"
-#include "dce_abm.h"
 #include "dmcu.h"
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 #include "dcn_calcs.h"
@@ -384,7 +383,6 @@ static int dce112_set_clock(
 	struct bp_set_dce_clock_parameters dce_clk_params;
 	struct dc_bios *bp = clk->ctx->dc_bios;
 	struct dc *core_dc = clk->ctx->dc;
-	struct abm *abm =  core_dc->res_pool->abm;
 	struct dmcu *dmcu = core_dc->res_pool->dmcu;
 	int actual_clock = requested_clk_khz;
 	/* Prepare to program display clock*/
@@ -417,7 +415,7 @@ static int dce112_set_clock(
 
 	bp->funcs->set_dce_clock(bp, &dce_clk_params);
 
-	if (abm->funcs->is_dmcu_initialized(abm) && clk_dce->dfs_bypass_disp_clk != actual_clock)
+	if (clk_dce->dfs_bypass_disp_clk != actual_clock)
 		dmcu->funcs->set_psr_wait_loop(dmcu,
 				actual_clock / 1000 / 7);
 	clk_dce->dfs_bypass_disp_clk = actual_clock;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
index fd77df573b61..f663adb33584 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
@@ -49,8 +49,16 @@
 #define PSR_EXIT 0x21
 #define PSR_SET 0x23
 #define PSR_SET_WAITLOOP 0x31
+#define MCP_INIT_DMCU 0x88
+#define MCP_INIT_IRAM 0x89
+#define MCP_DMCU_VERSION 0x90
 #define MASTER_COMM_CNTL_REG__MASTER_COMM_INTERRUPT_MASK   0x00000001L
-unsigned int cached_wait_loop_number = 0;
+
+static bool dce_dmcu_init(struct dmcu *dmcu)
+{
+	// Do nothing
+	return true;
+}
 
 bool dce_dmcu_load_iram(struct dmcu *dmcu,
 		unsigned int start_offset,
@@ -84,7 +92,7 @@ static void dce_get_dmcu_psr_state(struct dmcu *dmcu, uint32_t *psr_state)
 {
 	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
 
-	uint32_t psrStateOffset = 0xf0;
+	uint32_t psr_state_offset = 0xf0;
 
 	/* Enable write access to IRAM */
 	REG_UPDATE(DMCU_RAM_ACCESS_CTRL, IRAM_HOST_ACCESS_EN, 1);
@@ -92,7 +100,7 @@ static void dce_get_dmcu_psr_state(struct dmcu *dmcu, uint32_t *psr_state)
 	REG_WAIT(DCI_MEM_PWR_STATUS, DMCU_IRAM_MEM_PWR_STATE, 0, 2, 10);
 
 	/* Write address to IRAM_RD_ADDR in DMCU_IRAM_RD_CTRL */
-	REG_WRITE(DMCU_IRAM_RD_CTRL, psrStateOffset);
+	REG_WRITE(DMCU_IRAM_RD_CTRL, psr_state_offset);
 
 	/* Read data from IRAM_RD_DATA in DMCU_IRAM_RD_DATA*/
 	*psr_state = REG_READ(DMCU_IRAM_RD_DATA);
@@ -255,13 +263,33 @@ static void dce_dmcu_setup_psr(struct dmcu *dmcu,
 	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
 }
 
+static bool dce_is_dmcu_initialized(struct dmcu *dmcu)
+{
+	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+	unsigned int dmcu_uc_reset;
+
+	/* microcontroller is not running */
+	REG_GET(DMCU_STATUS, UC_IN_RESET, &dmcu_uc_reset);
+
+	/* DMCU is not running */
+	if (dmcu_uc_reset)
+		return false;
+
+	return true;
+}
+
 static void dce_psr_wait_loop(
 	struct dmcu *dmcu,
 	unsigned int wait_loop_number)
 {
 	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
 	union dce_dmcu_psr_config_data_wait_loop_reg1 masterCmdData1;
-	if (cached_wait_loop_number == wait_loop_number)
+
+	if (dmcu->cached_wait_loop_number == wait_loop_number)
+		return;
+
+	/* DMCU is not running */
+	if (!dce_is_dmcu_initialized(dmcu))
 		return;
 
 	/* waitDMCUReadyForCmd */
@@ -269,7 +297,7 @@ static void dce_psr_wait_loop(
 
 	masterCmdData1.u32 = 0;
 	masterCmdData1.bits.wait_loop = wait_loop_number;
-	cached_wait_loop_number = wait_loop_number;
+	dmcu->cached_wait_loop_number = wait_loop_number;
 	dm_write_reg(dmcu->ctx, REG(MASTER_COMM_DATA_REG1), masterCmdData1.u32);
 
 	/* setDMCUParam_Cmd */
@@ -279,14 +307,136 @@ static void dce_psr_wait_loop(
 	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
 }
 
-static void dce_get_psr_wait_loop(unsigned int *psr_wait_loop_number)
+static void dce_get_psr_wait_loop(
+		struct dmcu *dmcu, unsigned int *psr_wait_loop_number)
 {
-	*psr_wait_loop_number = cached_wait_loop_number;
+	*psr_wait_loop_number = dmcu->cached_wait_loop_number;
 	return;
 }
 
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-bool dcn10_dmcu_load_iram(struct dmcu *dmcu,
+static void dcn10_get_dmcu_state(struct dmcu *dmcu)
+{
+	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+	uint32_t dmcu_state_offset = 0xf6;
+
+	/* Enable write access to IRAM */
+	REG_UPDATE_2(DMCU_RAM_ACCESS_CTRL,
+			IRAM_HOST_ACCESS_EN, 1,
+			IRAM_RD_ADDR_AUTO_INC, 1);
+
+	REG_WAIT(DMU_MEM_PWR_CNTL, DMCU_IRAM_MEM_PWR_STATE, 0, 2, 10);
+
+	/* Write address to IRAM_RD_ADDR in DMCU_IRAM_RD_CTRL */
+	REG_WRITE(DMCU_IRAM_RD_CTRL, dmcu_state_offset);
+
+	/* Read data from IRAM_RD_DATA in DMCU_IRAM_RD_DATA*/
+	dmcu->dmcu_state = REG_READ(DMCU_IRAM_RD_DATA);
+
+	/* Disable write access to IRAM to allow dynamic sleep state */
+	REG_UPDATE_2(DMCU_RAM_ACCESS_CTRL,
+			IRAM_HOST_ACCESS_EN, 0,
+			IRAM_RD_ADDR_AUTO_INC, 0);
+}
+
+static void dcn10_get_dmcu_version(struct dmcu *dmcu)
+{
+	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+	uint32_t dmcu_version_offset = 0xf1;
+
+	/* Clear scratch */
+	REG_WRITE(DC_DMCU_SCRATCH, 0);
+
+	/* Enable write access to IRAM */
+	REG_UPDATE_2(DMCU_RAM_ACCESS_CTRL,
+			IRAM_HOST_ACCESS_EN, 1,
+			IRAM_RD_ADDR_AUTO_INC, 1);
+
+	REG_WAIT(DMU_MEM_PWR_CNTL, DMCU_IRAM_MEM_PWR_STATE, 0, 2, 10);
+
+	/* Write address to IRAM_RD_ADDR and read from DATA register */
+	REG_WRITE(DMCU_IRAM_RD_CTRL, dmcu_version_offset);
+	dmcu->dmcu_version.interface_version = REG_READ(DMCU_IRAM_RD_DATA);
+	dmcu->dmcu_version.year = ((REG_READ(DMCU_IRAM_RD_DATA) << 8) |
+						REG_READ(DMCU_IRAM_RD_DATA));
+	dmcu->dmcu_version.month = REG_READ(DMCU_IRAM_RD_DATA);
+	dmcu->dmcu_version.day = REG_READ(DMCU_IRAM_RD_DATA);
+
+	/* Disable write access to IRAM to allow dynamic sleep state */
+	REG_UPDATE_2(DMCU_RAM_ACCESS_CTRL,
+			IRAM_HOST_ACCESS_EN, 0,
+			IRAM_RD_ADDR_AUTO_INC, 0);
+
+	/* Send MCP command message to DMCU to get version reply from FW.
+	 * We expect this version should match the one in IRAM, otherwise
+	 * something is wrong with DMCU and we should fail and disable UC.
+	 */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+
+	/* Set command to get DMCU version from microcontroller */
+	REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0,
+			MCP_DMCU_VERSION);
+
+	/* Notify microcontroller of new command */
+	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+	/* Ensure command has been executed before continuing */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+
+	/* Somehow version does not match, so fail and return version 0 */
+	if (dmcu->dmcu_version.interface_version != REG_READ(DC_DMCU_SCRATCH))
+		dmcu->dmcu_version.interface_version = 0;
+}
+
+static bool dcn10_dmcu_init(struct dmcu *dmcu)
+{
+	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+
+	/* DMCU FW should populate the scratch register if running */
+	if (REG_READ(DC_DMCU_SCRATCH) == 0)
+		return false;
+
+	/* Check state is uninitialized */
+	dcn10_get_dmcu_state(dmcu);
+
+	/* If microcontroller is already initialized, do nothing */
+	if (dmcu->dmcu_state == DMCU_RUNNING)
+		return true;
+
+	/* Retrieve and cache the DMCU firmware version. */
+	dcn10_get_dmcu_version(dmcu);
+
+	/* Check interface version to confirm firmware is loaded and running */
+	if (dmcu->dmcu_version.interface_version == 0)
+		return false;
+
+	/* Wait until microcontroller is ready to process interrupt */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+
+	/* Set initialized ramping boundary value */
+	REG_WRITE(MASTER_COMM_DATA_REG1, 0xFFFF);
+
+	/* Set command to initialize microcontroller */
+	REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0,
+			MCP_INIT_DMCU);
+
+	/* Notify microcontroller of new command */
+	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+	/* Ensure command has been executed before continuing */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+
+	// Check state is initialized
+	dcn10_get_dmcu_state(dmcu);
+
+	// If microcontroller is not in running state, fail
+	if (dmcu->dmcu_state != DMCU_RUNNING)
+		return false;
+
+	return true;
+}
+
+static bool dcn10_dmcu_load_iram(struct dmcu *dmcu,
 		unsigned int start_offset,
 		const char *src,
 		unsigned int bytes)
@@ -294,7 +444,9 @@ bool dcn10_dmcu_load_iram(struct dmcu *dmcu,
 	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
 	unsigned int count = 0;
 
-	REG_UPDATE(DMCU_CTRL, DMCU_ENABLE, 1);
+	/* If microcontroller is not running, do nothing */
+	if (dmcu->dmcu_state != DMCU_RUNNING)
+		return false;
 
 	/* Enable write access to IRAM */
 	REG_UPDATE_2(DMCU_RAM_ACCESS_CTRL,
@@ -313,6 +465,19 @@ bool dcn10_dmcu_load_iram(struct dmcu *dmcu,
 			IRAM_HOST_ACCESS_EN, 0,
 			IRAM_WR_ADDR_AUTO_INC, 0);
 
+	/* Wait until microcontroller is ready to process interrupt */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+
+	/* Set command to signal IRAM is loaded and to initialize IRAM */
+	REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0,
+			MCP_INIT_IRAM);
+
+	/* Notify microcontroller of new command */
+	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+	/* Ensure command has been executed before continuing */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+
 	return true;
 }
 
@@ -320,7 +485,11 @@ static void dcn10_get_dmcu_psr_state(struct dmcu *dmcu, uint32_t *psr_state)
 {
 	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
 
-	uint32_t psrStateOffset = 0xf0;
+	uint32_t psr_state_offset = 0xf0;
+
+	/* If microcontroller is not running, do nothing */
+	if (dmcu->dmcu_state != DMCU_RUNNING)
+		return;
 
 	/* Enable write access to IRAM */
 	REG_UPDATE(DMCU_RAM_ACCESS_CTRL, IRAM_HOST_ACCESS_EN, 1);
@@ -328,7 +497,7 @@ static void dcn10_get_dmcu_psr_state(struct dmcu *dmcu, uint32_t *psr_state)
 	REG_WAIT(DMU_MEM_PWR_CNTL, DMCU_IRAM_MEM_PWR_STATE, 0, 2, 10);
 
 	/* Write address to IRAM_RD_ADDR in DMCU_IRAM_RD_CTRL */
-	REG_WRITE(DMCU_IRAM_RD_CTRL, psrStateOffset);
+	REG_WRITE(DMCU_IRAM_RD_CTRL, psr_state_offset);
 
 	/* Read data from IRAM_RD_DATA in DMCU_IRAM_RD_DATA*/
 	*psr_state = REG_READ(DMCU_IRAM_RD_DATA);
@@ -348,6 +517,10 @@ static void dcn10_dmcu_set_psr_enable(struct dmcu *dmcu, bool enable, bool wait)
 	unsigned int retryCount;
 	uint32_t psr_state = 0;
 
+	/* If microcontroller is not running, do nothing */
+	if (dmcu->dmcu_state != DMCU_RUNNING)
+		return;
+
 	/* waitDMCUReadyForCmd */
 	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0,
 				dmcu_wait_reg_ready_interval,
@@ -399,6 +572,10 @@ static void dcn10_dmcu_setup_psr(struct dmcu *dmcu,
 	union dce_dmcu_psr_config_data_reg2 masterCmdData2;
 	union dce_dmcu_psr_config_data_reg3 masterCmdData3;
 
+	/* If microcontroller is not running, do nothing */
+	if (dmcu->dmcu_state != DMCU_RUNNING)
+		return;
+
 	link->link_enc->funcs->psr_program_dp_dphy_fast_training(link->link_enc,
 			psr_context->psrExitLinkTrainingRequired);
 
@@ -505,13 +682,18 @@ static void dcn10_psr_wait_loop(
 {
 	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
 	union dce_dmcu_psr_config_data_wait_loop_reg1 masterCmdData1;
+
+	/* If microcontroller is not running, do nothing */
+	if (dmcu->dmcu_state != DMCU_RUNNING)
+		return;
+
 	if (wait_loop_number != 0) {
 	/* waitDMCUReadyForCmd */
 	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000);
 
 	masterCmdData1.u32 = 0;
 	masterCmdData1.bits.wait_loop = wait_loop_number;
-	cached_wait_loop_number = wait_loop_number;
+	dmcu->cached_wait_loop_number = wait_loop_number;
 	dm_write_reg(dmcu->ctx, REG(MASTER_COMM_DATA_REG1), masterCmdData1.u32);
 
 	/* setDMCUParam_Cmd */
@@ -522,31 +704,44 @@ static void dcn10_psr_wait_loop(
 	}
 }
 
-static void dcn10_get_psr_wait_loop(unsigned int *psr_wait_loop_number)
+static void dcn10_get_psr_wait_loop(
+		struct dmcu *dmcu, unsigned int *psr_wait_loop_number)
 {
-	*psr_wait_loop_number = cached_wait_loop_number;
+	*psr_wait_loop_number = dmcu->cached_wait_loop_number;
 	return;
 }
 
+static bool dcn10_is_dmcu_initialized(struct dmcu *dmcu)
+{
+	/* microcontroller is not running */
+	if (dmcu->dmcu_state != DMCU_RUNNING)
+		return false;
+	return true;
+}
+
 #endif
 
 static const struct dmcu_funcs dce_funcs = {
+	.dmcu_init = dce_dmcu_init,
 	.load_iram = dce_dmcu_load_iram,
 	.set_psr_enable = dce_dmcu_set_psr_enable,
 	.setup_psr = dce_dmcu_setup_psr,
 	.get_psr_state = dce_get_dmcu_psr_state,
 	.set_psr_wait_loop = dce_psr_wait_loop,
-	.get_psr_wait_loop = dce_get_psr_wait_loop
+	.get_psr_wait_loop = dce_get_psr_wait_loop,
+	.is_dmcu_initialized = dce_is_dmcu_initialized
 };
 
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 static const struct dmcu_funcs dcn10_funcs = {
+	.dmcu_init = dcn10_dmcu_init,
 	.load_iram = dcn10_dmcu_load_iram,
 	.set_psr_enable = dcn10_dmcu_set_psr_enable,
 	.setup_psr = dcn10_dmcu_setup_psr,
 	.get_psr_state = dcn10_get_dmcu_psr_state,
 	.set_psr_wait_loop = dcn10_psr_wait_loop,
-	.get_psr_wait_loop = dcn10_get_psr_wait_loop
+	.get_psr_wait_loop = dcn10_get_psr_wait_loop,
+	.is_dmcu_initialized = dcn10_is_dmcu_initialized
 };
 #endif
 
@@ -561,6 +756,7 @@ static void dce_dmcu_construct(
 
 	base->ctx = ctx;
 	base->funcs = &dce_funcs;
+	base->cached_wait_loop_number = 0;
 
 	dmcu_dce->regs = regs;
 	dmcu_dce->dmcu_shift = dmcu_shift;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
index b85f53c2f6f8..1d4546f23135 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
@@ -31,6 +31,7 @@
 
 #define DMCU_COMMON_REG_LIST_DCE_BASE() \
 	SR(DMCU_CTRL), \
+	SR(DMCU_STATUS), \
 	SR(DMCU_RAM_ACCESS_CTRL), \
 	SR(DMCU_IRAM_WR_CTRL), \
 	SR(DMCU_IRAM_WR_DATA), \
@@ -42,7 +43,8 @@
 	SR(DMCU_IRAM_RD_CTRL), \
 	SR(DMCU_IRAM_RD_DATA), \
 	SR(DMCU_INTERRUPT_TO_UC_EN_MASK), \
-	SR(SMU_INTERRUPT_CONTROL)
+	SR(SMU_INTERRUPT_CONTROL), \
+	SR(DC_DMCU_SCRATCH)
 
 #define DMCU_DCE110_COMMON_REG_LIST() \
 	DMCU_COMMON_REG_LIST_DCE_BASE(), \
@@ -58,10 +60,16 @@
 #define DMCU_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(mask_sh) \
 	DMCU_SF(DMCU_CTRL, \
 			DMCU_ENABLE, mask_sh), \
+	DMCU_SF(DMCU_STATUS, \
+			UC_IN_STOP_MODE, mask_sh), \
+	DMCU_SF(DMCU_STATUS, \
+			UC_IN_RESET, mask_sh), \
 	DMCU_SF(DMCU_RAM_ACCESS_CTRL, \
 			IRAM_HOST_ACCESS_EN, mask_sh), \
 	DMCU_SF(DMCU_RAM_ACCESS_CTRL, \
 			IRAM_WR_ADDR_AUTO_INC, mask_sh), \
+	DMCU_SF(DMCU_RAM_ACCESS_CTRL, \
+			IRAM_RD_ADDR_AUTO_INC, mask_sh), \
 	DMCU_SF(MASTER_COMM_CMD_REG, \
 			MASTER_COMM_CMD_REG_BYTE0, mask_sh), \
 	DMCU_SF(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, mask_sh), \
@@ -89,7 +97,10 @@
 	type DMCU_IRAM_MEM_PWR_STATE; \
 	type IRAM_HOST_ACCESS_EN; \
 	type IRAM_WR_ADDR_AUTO_INC; \
+	type IRAM_RD_ADDR_AUTO_INC; \
 	type DMCU_ENABLE; \
+	type UC_IN_STOP_MODE; \
+	type UC_IN_RESET; \
 	type MASTER_COMM_CMD_REG_BYTE0; \
 	type MASTER_COMM_INTERRUPT; \
 	type DPHY_RX_FAST_TRAINING_CAPABLE; \
@@ -112,6 +123,7 @@ struct dce_dmcu_mask {
 
 struct dce_dmcu_registers {
 	uint32_t DMCU_CTRL;
+	uint32_t DMCU_STATUS;
 	uint32_t DMCU_RAM_ACCESS_CTRL;
 	uint32_t DCI_MEM_PWR_STATUS;
 	uint32_t DMU_MEM_PWR_CNTL;
@@ -127,6 +139,7 @@ struct dce_dmcu_registers {
 	uint32_t DMCU_IRAM_RD_DATA;
 	uint32_t DMCU_INTERRUPT_TO_UC_EN_MASK;
 	uint32_t SMU_INTERRUPT_CONTROL;
+	uint32_t DC_DMCU_SCRATCH;
 };
 
 struct dce_dmcu {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
index 52506155e361..f11f17fe08f9 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_hwseq.h
@@ -140,10 +140,6 @@
 	BL_REG_LIST()
 
 #define HWSEQ_DCN_REG_LIST()\
-	SRII(OTG_GLOBAL_SYNC_STATUS, OTG, 0), \
-	SRII(OTG_GLOBAL_SYNC_STATUS, OTG, 1), \
-	SRII(OTG_GLOBAL_SYNC_STATUS, OTG, 2), \
-	SRII(OTG_GLOBAL_SYNC_STATUS, OTG, 3), \
 	SRII(DCHUBP_CNTL, HUBP, 0), \
 	SRII(DCHUBP_CNTL, HUBP, 1), \
 	SRII(DCHUBP_CNTL, HUBP, 2), \
@@ -240,6 +236,7 @@
 	SR(D2VGA_CONTROL), \
 	SR(D3VGA_CONTROL), \
 	SR(D4VGA_CONTROL), \
+	SR(VGA_TEST_CONTROL), \
 	SR(DC_IP_REQUEST_CNTL), \
 	BL_REG_LIST()
 
@@ -264,7 +261,6 @@ struct dce_hwseq_registers {
 	uint32_t DCHUB_AGP_BOT;
 	uint32_t DCHUB_AGP_TOP;
 
-	uint32_t OTG_GLOBAL_SYNC_STATUS[4];
 	uint32_t DCHUBP_CNTL[4];
 	uint32_t HUBP_CLK_CNTL[4];
 	uint32_t DPP_CONTROL[4];
@@ -342,6 +338,7 @@ struct dce_hwseq_registers {
 	uint32_t D2VGA_CONTROL;
 	uint32_t D3VGA_CONTROL;
 	uint32_t D4VGA_CONTROL;
+	uint32_t VGA_TEST_CONTROL;
 	/* MMHUB registers. read only. temporary hack */
 	uint32_t VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32;
 	uint32_t VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
@@ -438,8 +435,6 @@ struct dce_hwseq_registers {
 #define HWSEQ_DCN_MASK_SH_LIST(mask_sh)\
 	HWSEQ_PIXEL_RATE_MASK_SH_LIST(mask_sh, OTG0_),\
 	HWS_SF1(OTG0_, PHYPLL_PIXEL_RATE_CNTL, PHYPLL_PIXEL_RATE_SOURCE, mask_sh), \
-	HWS_SF(OTG0_, OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_EVENT_CLEAR, mask_sh), \
-	HWS_SF(OTG0_, OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_EVENT_OCCURRED, mask_sh), \
 	HWS_SF(HUBP0_, DCHUBP_CNTL, HUBP_VTG_SEL, mask_sh), \
 	HWS_SF(HUBP0_, HUBP_CLK_CNTL, HUBP_CLOCK_ENABLE, mask_sh), \
 	HWS_SF(DPP_TOP0_, DPP_CONTROL, DPP_CLOCK_ENABLE, mask_sh), \
@@ -500,6 +495,12 @@ struct dce_hwseq_registers {
 	HWS_SF(, DOMAIN6_PG_STATUS, DOMAIN6_PGFSM_PWR_STATUS, mask_sh), \
 	HWS_SF(, DOMAIN7_PG_STATUS, DOMAIN7_PGFSM_PWR_STATUS, mask_sh), \
 	HWS_SF(, DC_IP_REQUEST_CNTL, IP_REQUEST_EN, mask_sh), \
+	HWS_SF(, D1VGA_CONTROL, D1VGA_MODE_ENABLE, mask_sh),\
+	HWS_SF(, D2VGA_CONTROL, D2VGA_MODE_ENABLE, mask_sh),\
+	HWS_SF(, D3VGA_CONTROL, D3VGA_MODE_ENABLE, mask_sh),\
+	HWS_SF(, D4VGA_CONTROL, D4VGA_MODE_ENABLE, mask_sh),\
+	HWS_SF(, VGA_TEST_CONTROL, VGA_TEST_ENABLE, mask_sh),\
+	HWS_SF(, VGA_TEST_CONTROL, VGA_TEST_RENDER_START, mask_sh),\
 	HWS_SF(, LVTMA_PWRSEQ_CNTL, LVTMA_BLON, mask_sh), \
 	HWS_SF(, LVTMA_PWRSEQ_STATE, LVTMA_PWRSEQ_TARGET_STATE_R, mask_sh)
 
@@ -536,8 +537,6 @@ struct dce_hwseq_registers {
 	type LVTMA_PWRSEQ_TARGET_STATE_R;
 
 #define HWSEQ_DCN_REG_FIELD_LIST(type) \
-	type VUPDATE_NO_LOCK_EVENT_CLEAR; \
-	type VUPDATE_NO_LOCK_EVENT_OCCURRED; \
 	type HUBP_VTG_SEL; \
 	type HUBP_CLOCK_ENABLE; \
 	type DPP_CLOCK_ENABLE; \
@@ -591,7 +590,14 @@ struct dce_hwseq_registers {
 	type DOMAIN7_PGFSM_PWR_STATUS; \
 	type DCFCLK_GATE_DIS; \
 	type DCHUBBUB_GLOBAL_TIMER_REFDIV; \
-	type DENTIST_DPPCLK_WDIVIDER;
+	type DENTIST_DPPCLK_WDIVIDER; \
+	type DENTIST_DISPCLK_WDIVIDER; \
+	type VGA_TEST_ENABLE; \
+	type VGA_TEST_RENDER_START; \
+	type D1VGA_MODE_ENABLE; \
+	type D2VGA_MODE_ENABLE; \
+	type D3VGA_MODE_ENABLE; \
+	type D4VGA_MODE_ENABLE;
 
 struct dce_hwseq_shift {
 	HWSEQ_REG_FIELD_LIST(uint8_t)
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
index d618fdd0cc82..d737e911971b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
@@ -135,36 +135,34 @@ static void dce_ipp_cursor_set_attributes(
 }
 
 
-static void dce_ipp_program_prescale(
-	struct input_pixel_processor *ipp,
-	struct ipp_prescale_params *params)
+static void dce_ipp_program_prescale(struct input_pixel_processor *ipp,
+				     struct ipp_prescale_params *params)
 {
 	struct dce_ipp *ipp_dce = TO_DCE_IPP(ipp);
 
 	/* set to bypass mode first before change */
 	REG_UPDATE(PRESCALE_GRPH_CONTROL,
-		GRPH_PRESCALE_BYPASS,
-		1);
+		   GRPH_PRESCALE_BYPASS, 1);
 
 	REG_SET_2(PRESCALE_VALUES_GRPH_R, 0,
-		GRPH_PRESCALE_SCALE_R, params->scale,
-		GRPH_PRESCALE_BIAS_R, params->bias);
+		  GRPH_PRESCALE_SCALE_R, params->scale,
+		  GRPH_PRESCALE_BIAS_R, params->bias);
 
 	REG_SET_2(PRESCALE_VALUES_GRPH_G, 0,
-		GRPH_PRESCALE_SCALE_G, params->scale,
-		GRPH_PRESCALE_BIAS_G, params->bias);
+		  GRPH_PRESCALE_SCALE_G, params->scale,
+		  GRPH_PRESCALE_BIAS_G, params->bias);
 
 	REG_SET_2(PRESCALE_VALUES_GRPH_B, 0,
-		GRPH_PRESCALE_SCALE_B, params->scale,
-		GRPH_PRESCALE_BIAS_B, params->bias);
+		  GRPH_PRESCALE_SCALE_B, params->scale,
+		  GRPH_PRESCALE_BIAS_B, params->bias);
 
 	if (params->mode != IPP_PRESCALE_MODE_BYPASS) {
 		REG_UPDATE(PRESCALE_GRPH_CONTROL,
-				GRPH_PRESCALE_BYPASS, 0);
+			   GRPH_PRESCALE_BYPASS, 0);
 
 		/* If prescale is in use, then legacy lut should be bypassed */
 		REG_UPDATE(INPUT_GAMMA_CONTROL,
-				GRPH_INPUT_GAMMA_MODE, 1);
+			   GRPH_INPUT_GAMMA_MODE, 1);
 	}
 }
 
@@ -223,13 +221,12 @@ static void dce_ipp_set_degamma(
 	struct dce_ipp *ipp_dce = TO_DCE_IPP(ipp);
 	uint32_t degamma_type = (mode == IPP_DEGAMMA_MODE_HW_sRGB) ? 1 : 0;
 
-	ASSERT(mode == IPP_DEGAMMA_MODE_BYPASS ||
-			mode == IPP_DEGAMMA_MODE_HW_sRGB);
+	ASSERT(mode == IPP_DEGAMMA_MODE_BYPASS || mode == IPP_DEGAMMA_MODE_HW_sRGB);
 
 	REG_SET_3(DEGAMMA_CONTROL, 0,
-		GRPH_DEGAMMA_MODE, degamma_type,
-		CURSOR_DEGAMMA_MODE, degamma_type,
-		CURSOR2_DEGAMMA_MODE, degamma_type);
+		  GRPH_DEGAMMA_MODE, degamma_type,
+		  CURSOR_DEGAMMA_MODE, degamma_type,
+		  CURSOR2_DEGAMMA_MODE, degamma_type);
 }
 
 static const struct ipp_funcs dce_ipp_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
index fe88852b4774..e4741f1a2b01 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
@@ -82,13 +82,6 @@
 #define DCE110_DIG_FE_SOURCE_SELECT_DIGF 0x20
 #define DCE110_DIG_FE_SOURCE_SELECT_DIGG 0x40
 
-/* Minimum pixel clock, in KHz. For TMDS signal is 25.00 MHz */
-#define TMDS_MIN_PIXEL_CLOCK 25000
-/* Maximum pixel clock, in KHz. For TMDS signal is 165.00 MHz */
-#define TMDS_MAX_PIXEL_CLOCK 165000
-/* For current ASICs pixel clock - 600MHz */
-#define MAX_ENCODER_CLOCK 600000
-
 enum {
 	DP_MST_UPDATE_MAX_RETRY = 50
 };
@@ -683,6 +676,7 @@ void dce110_link_encoder_construct(
 {
 	struct bp_encoder_cap_info bp_cap_info = {0};
 	const struct dc_vbios_funcs *bp_funcs = init_data->ctx->dc_bios->funcs;
+	enum bp_result result = BP_RESULT_OK;
 
 	enc110->base.funcs = &dce110_lnk_enc_funcs;
 	enc110->base.ctx = init_data->ctx;
@@ -757,15 +751,24 @@ void dce110_link_encoder_construct(
 		enc110->base.preferred_engine = ENGINE_ID_UNKNOWN;
 	}
 
+	/* default to one to mirror Windows behavior */
+	enc110->base.features.flags.bits.HDMI_6GB_EN = 1;
+
+	result = bp_funcs->get_encoder_cap_info(enc110->base.ctx->dc_bios,
+						enc110->base.id, &bp_cap_info);
+
 	/* Override features with DCE-specific values */
-	if (BP_RESULT_OK == bp_funcs->get_encoder_cap_info(
-			enc110->base.ctx->dc_bios, enc110->base.id,
-			&bp_cap_info)) {
+	if (BP_RESULT_OK == result) {
 		enc110->base.features.flags.bits.IS_HBR2_CAPABLE =
 				bp_cap_info.DP_HBR2_EN;
 		enc110->base.features.flags.bits.IS_HBR3_CAPABLE =
 				bp_cap_info.DP_HBR3_EN;
 		enc110->base.features.flags.bits.HDMI_6GB_EN = bp_cap_info.HDMI_6GB_EN;
+	} else {
+		dm_logger_write(enc110->base.ctx->logger, LOG_WARNING,
+				"%s: Failed to get encoder_cap_info from VBIOS with error code %d!\n",
+				__func__,
+				result);
 	}
 }
 
@@ -845,8 +848,6 @@ void dce110_link_encoder_hw_init(
 
 		ASSERT(result == BP_RESULT_OK);
 
-	} else if (enc110->base.connector.id == CONNECTOR_ID_EDP) {
-		ctx->dc->hwss.edp_power_control(enc, true);
 	}
 	aux_initialize(enc110);
 
@@ -906,8 +907,7 @@ void dce110_link_encoder_enable_tmds_output(
 	struct link_encoder *enc,
 	enum clock_source_id clock_source,
 	enum dc_color_depth color_depth,
-	bool hdmi,
-	bool dual_link,
+	enum signal_type signal,
 	uint32_t pixel_clock)
 {
 	struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc);
@@ -921,16 +921,12 @@ void dce110_link_encoder_enable_tmds_output(
 	cntl.engine_id = enc->preferred_engine;
 	cntl.transmitter = enc110->base.transmitter;
 	cntl.pll_id = clock_source;
-	if (hdmi) {
-		cntl.signal = SIGNAL_TYPE_HDMI_TYPE_A;
-		cntl.lanes_number = 4;
-	} else if (dual_link) {
-		cntl.signal = SIGNAL_TYPE_DVI_DUAL_LINK;
+	cntl.signal = signal;
+	if (cntl.signal == SIGNAL_TYPE_DVI_DUAL_LINK)
 		cntl.lanes_number = 8;
-	} else {
-		cntl.signal = SIGNAL_TYPE_DVI_SINGLE_LINK;
+	else
 		cntl.lanes_number = 4;
-	}
+
 	cntl.hpd_sel = enc110->base.hpd_source;
 
 	cntl.pixel_clock = pixel_clock;
@@ -1033,8 +1029,7 @@ void dce110_link_encoder_enable_dp_mst_output(
  */
 void dce110_link_encoder_disable_output(
 	struct link_encoder *enc,
-	enum signal_type signal,
-	struct dc_link *link)
+	enum signal_type signal)
 {
 	struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc);
 	struct dc_context *ctx = enc110->base.ctx;
@@ -1045,8 +1040,6 @@ void dce110_link_encoder_disable_output(
 		/* OF_SKIP_POWER_DOWN_INACTIVE_ENCODER */
 		return;
 	}
-	if (enc110->base.connector.id == CONNECTOR_ID_EDP)
-		ctx->dc->hwss.edp_backlight_control(link, false);
 	/* Power-down RX and disable GPU PHY should be paired.
 	 * Disabling PHY without powering down RX may cause
 	 * symbol lock loss, on which we will get DP Sink interrupt. */
@@ -1077,20 +1070,6 @@ void dce110_link_encoder_disable_output(
 	/* disable encoder */
 	if (dc_is_dp_signal(signal))
 		link_encoder_disable(enc110);
-
-	if (enc110->base.connector.id == CONNECTOR_ID_EDP) {
-		/* power down eDP panel */
-		/* TODO: Power control cause regression, we should implement
-		 * it properly, for now just comment it.
-		 *
-		 * link_encoder_edp_wait_for_hpd_ready(
-			link_enc,
-			link_enc->connector,
-			false);
-
-		 * link_encoder_edp_power_control(
-				link_enc, false); */
-	}
 }
 
 void dce110_link_encoder_dp_set_lane_settings(
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
index 494067dedd03..0ec3433d34b6 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.h
@@ -210,8 +210,7 @@ void dce110_link_encoder_enable_tmds_output(
 	struct link_encoder *enc,
 	enum clock_source_id clock_source,
 	enum dc_color_depth color_depth,
-	bool hdmi,
-	bool dual_link,
+	enum signal_type signal,
 	uint32_t pixel_clock);
 
 /* enables DP PHY output */
@@ -228,9 +227,8 @@ void dce110_link_encoder_enable_dp_mst_output(
 
 /* disable PHY output */
 void dce110_link_encoder_disable_output(
-	struct link_encoder *link_enc,
-	enum signal_type signal,
-	struct dc_link *link);
+	struct link_encoder *enc,
+	enum signal_type signal);
 
 /* set DP lane settings */
 void dce110_link_encoder_dp_set_lane_settings(
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c
index 3931412ab6d3..87093894ea9e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c
@@ -128,23 +128,22 @@ static void set_truncation(
 		return;
 	}
 	/* on other format-to do */
-	if (params->flags.TRUNCATE_ENABLED == 0 ||
-			params->flags.TRUNCATE_DEPTH == 2)
+	if (params->flags.TRUNCATE_ENABLED == 0)
 		return;
 	/*Set truncation depth and Enable truncation*/
 	REG_UPDATE_3(FMT_BIT_DEPTH_CONTROL,
 				FMT_TRUNCATE_EN, 1,
 				FMT_TRUNCATE_DEPTH,
-				params->flags.TRUNCATE_MODE,
+				params->flags.TRUNCATE_DEPTH,
 				FMT_TRUNCATE_MODE,
-				params->flags.TRUNCATE_DEPTH);
+				params->flags.TRUNCATE_MODE);
 }
 
 
 /**
  *	set_spatial_dither
  *	1) set spatial dithering mode: pattern of seed
- *	2) set spatical dithering depth: 0 for 18bpp or 1 for 24bpp
+ *	2) set spatial dithering depth: 0 for 18bpp or 1 for 24bpp
  *	3) set random seed
  *	4) set random mode
  *		lfsr is reset every frame or not reset
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
index e42b6eb1c1f0..83bae207371d 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
@@ -300,6 +300,8 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 	uint32_t h_back_porch;
 	uint8_t synchronous_clock = 0; /* asynchronous mode */
 	uint8_t colorimetry_bpc;
+	uint8_t dynamic_range_rgb = 0; /*full range*/
+	uint8_t dynamic_range_ycbcr = 1; /*bt709*/
 #endif
 
 	struct dce110_stream_encoder *enc110 = DCE110STRENC_FROM_STRENC(enc);
@@ -380,11 +382,7 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 	}
 
 	/* set dynamic range and YCbCr range */
-	if (enc110->se_mask->DP_DYN_RANGE && enc110->se_mask->DP_YCBCR_RANGE)
-		REG_UPDATE_2(
-			DP_PIXEL_FORMAT,
-			DP_DYN_RANGE, 0,
-			DP_YCBCR_RANGE, 0);
+
 
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 	switch (crtc_timing->display_color_depth) {
@@ -413,37 +411,57 @@ static void dce110_stream_encoder_dp_set_stream_attribute(
 		case COLOR_SPACE_SRGB:
 			misc0 = misc0 | 0x0;
 			misc1 = misc1 & ~0x80; /* bit7 = 0*/
+			dynamic_range_rgb = 0; /*full range*/
 			break;
 		case COLOR_SPACE_SRGB_LIMITED:
 			misc0 = misc0 | 0x8; /* bit3=1 */
 			misc1 = misc1 & ~0x80; /* bit7 = 0*/
+			dynamic_range_rgb = 1; /*limited range*/
 			break;
 		case COLOR_SPACE_YCBCR601:
+		case COLOR_SPACE_YCBCR601_LIMITED:
 			misc0 = misc0 | 0x8; /* bit3=1, bit4=0 */
 			misc1 = misc1 & ~0x80; /* bit7 = 0*/
+			dynamic_range_ycbcr = 0; /*bt601*/
 			if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
 				misc0 = misc0 | 0x2; /* bit2=0, bit1=1 */
 			else if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR444)
 				misc0 = misc0 | 0x4; /* bit2=1, bit1=0 */
 			break;
 		case COLOR_SPACE_YCBCR709:
+		case COLOR_SPACE_YCBCR709_LIMITED:
 			misc0 = misc0 | 0x18; /* bit3=1, bit4=1 */
 			misc1 = misc1 & ~0x80; /* bit7 = 0*/
+			dynamic_range_ycbcr = 1; /*bt709*/
 			if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
 				misc0 = misc0 | 0x2; /* bit2=0, bit1=1 */
 			else if (crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR444)
 				misc0 = misc0 | 0x4; /* bit2=1, bit1=0 */
 			break;
-		case COLOR_SPACE_2020_RGB_FULLRANGE:
 		case COLOR_SPACE_2020_RGB_LIMITEDRANGE:
+			dynamic_range_rgb = 1; /*limited range*/
+			break;
+		case COLOR_SPACE_2020_RGB_FULLRANGE:
 		case COLOR_SPACE_2020_YCBCR:
+		case COLOR_SPACE_XR_RGB:
+		case COLOR_SPACE_MSREF_SCRGB:
 		case COLOR_SPACE_ADOBERGB:
+		case COLOR_SPACE_DCIP3:
+		case COLOR_SPACE_XV_YCC_709:
+		case COLOR_SPACE_XV_YCC_601:
+		case COLOR_SPACE_DISPLAYNATIVE:
+		case COLOR_SPACE_DOLBYVISION:
+		case COLOR_SPACE_APPCTRL:
+		case COLOR_SPACE_CUSTOMPOINTS:
 		case COLOR_SPACE_UNKNOWN:
-		case COLOR_SPACE_YCBCR601_LIMITED:
-		case COLOR_SPACE_YCBCR709_LIMITED:
 			/* do nothing */
 			break;
 		}
+		if (enc110->se_mask->DP_DYN_RANGE && enc110->se_mask->DP_YCBCR_RANGE)
+			REG_UPDATE_2(
+				DP_PIXEL_FORMAT,
+				DP_DYN_RANGE, dynamic_range_rgb,
+				DP_YCBCR_RANGE, dynamic_range_ycbcr);
 
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 		if (REG(DP_MSA_COLORIMETRY))
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
index ae32af31eff1..0f662e6ee9bd 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
@@ -1177,207 +1177,160 @@ void dce110_opp_set_csc_default(
 		default_adjust->out_color_space);
 }
 
-static void program_pwl(
-	struct dce_transform *xfm_dce,
-	const struct pwl_params *params)
+static void program_pwl(struct dce_transform *xfm_dce,
+			const struct pwl_params *params)
 {
-	uint32_t value;
 	int retval;
+	uint8_t max_tries = 10;
+	uint8_t counter = 0;
+	uint32_t i = 0;
+	const struct pwl_result_data *rgb = params->rgb_resulted;
 
-	{
-		uint8_t max_tries = 10;
-		uint8_t counter = 0;
+	/* Power on LUT memory */
+	if (REG(DCFE_MEM_PWR_CTRL))
+		REG_UPDATE(DCFE_MEM_PWR_CTRL,
+			   DCP_REGAMMA_MEM_PWR_DIS, 1);
+	else
+		REG_UPDATE(DCFE_MEM_LIGHT_SLEEP_CNTL,
+			   REGAMMA_LUT_LIGHT_SLEEP_DIS, 1);
 
-		/* Power on LUT memory */
-		if (REG(DCFE_MEM_PWR_CTRL))
-			REG_UPDATE(DCFE_MEM_PWR_CTRL,
-				DCP_REGAMMA_MEM_PWR_DIS, 1);
-		else
-			REG_UPDATE(DCFE_MEM_LIGHT_SLEEP_CNTL,
-				REGAMMA_LUT_LIGHT_SLEEP_DIS, 1);
-
-		while (counter < max_tries) {
-			if (REG(DCFE_MEM_PWR_STATUS)) {
-				value = REG_READ(DCFE_MEM_PWR_STATUS);
-				REG_GET(DCFE_MEM_PWR_STATUS,
-						DCP_REGAMMA_MEM_PWR_STATE,
-						&retval);
-
-				if (retval == 0)
-						break;
-				++counter;
-			} else {
-				value = REG_READ(DCFE_MEM_LIGHT_SLEEP_CNTL);
-				REG_GET(DCFE_MEM_LIGHT_SLEEP_CNTL,
-						REGAMMA_LUT_MEM_PWR_STATE,
-						&retval);
-
-				if (retval == 0)
-						break;
-				++counter;
-			}
+	while (counter < max_tries) {
+		if (REG(DCFE_MEM_PWR_STATUS)) {
+			REG_GET(DCFE_MEM_PWR_STATUS,
+				DCP_REGAMMA_MEM_PWR_STATE,
+				&retval);
+
+			if (retval == 0)
+				break;
+			++counter;
+		} else {
+			REG_GET(DCFE_MEM_LIGHT_SLEEP_CNTL,
+				REGAMMA_LUT_MEM_PWR_STATE,
+				&retval);
+
+			if (retval == 0)
+				break;
+			++counter;
 		}
+	}
 
-		if (counter == max_tries) {
-			dm_logger_write(xfm_dce->base.ctx->logger, LOG_WARNING,
+	if (counter == max_tries) {
+		dm_logger_write(xfm_dce->base.ctx->logger, LOG_WARNING,
 				"%s: regamma lut was not powered on "
 				"in a timely manner,"
 				" programming still proceeds\n",
 				__func__);
-		}
 	}
 
 	REG_UPDATE(REGAMMA_LUT_WRITE_EN_MASK,
-			REGAMMA_LUT_WRITE_EN_MASK, 7);
+		   REGAMMA_LUT_WRITE_EN_MASK, 7);
 
 	REG_WRITE(REGAMMA_LUT_INDEX, 0);
 
 	/* Program REGAMMA_LUT_DATA */
-	{
-		uint32_t i = 0;
-		const struct pwl_result_data *rgb = params->rgb_resulted;
-
-		while (i != params->hw_points_num) {
+	while (i != params->hw_points_num) {
 
-			REG_WRITE(REGAMMA_LUT_DATA, rgb->red_reg);
-			REG_WRITE(REGAMMA_LUT_DATA, rgb->green_reg);
-			REG_WRITE(REGAMMA_LUT_DATA, rgb->blue_reg);
-			REG_WRITE(REGAMMA_LUT_DATA, rgb->delta_red_reg);
-			REG_WRITE(REGAMMA_LUT_DATA, rgb->delta_green_reg);
-			REG_WRITE(REGAMMA_LUT_DATA, rgb->delta_blue_reg);
+		REG_WRITE(REGAMMA_LUT_DATA, rgb->red_reg);
+		REG_WRITE(REGAMMA_LUT_DATA, rgb->green_reg);
+		REG_WRITE(REGAMMA_LUT_DATA, rgb->blue_reg);
+		REG_WRITE(REGAMMA_LUT_DATA, rgb->delta_red_reg);
+		REG_WRITE(REGAMMA_LUT_DATA, rgb->delta_green_reg);
+		REG_WRITE(REGAMMA_LUT_DATA, rgb->delta_blue_reg);
 
-			++rgb;
-			++i;
-		}
+		++rgb;
+		++i;
 	}
 
 	/*  we are done with DCP LUT memory; re-enable low power mode */
 	if (REG(DCFE_MEM_PWR_CTRL))
 		REG_UPDATE(DCFE_MEM_PWR_CTRL,
-			DCP_REGAMMA_MEM_PWR_DIS, 0);
+			   DCP_REGAMMA_MEM_PWR_DIS, 0);
 	else
 		REG_UPDATE(DCFE_MEM_LIGHT_SLEEP_CNTL,
-			REGAMMA_LUT_LIGHT_SLEEP_DIS, 0);
+			   REGAMMA_LUT_LIGHT_SLEEP_DIS, 0);
 }
 
-static void regamma_config_regions_and_segments(
-	struct dce_transform *xfm_dce,
-	const struct pwl_params *params)
+static void regamma_config_regions_and_segments(struct dce_transform *xfm_dce,
+						const struct pwl_params *params)
 {
 	const struct gamma_curve *curve;
 
-	{
-		REG_SET_2(REGAMMA_CNTLA_START_CNTL, 0,
-			REGAMMA_CNTLA_EXP_REGION_START, params->arr_points[0].custom_float_x,
-			REGAMMA_CNTLA_EXP_REGION_START_SEGMENT, 0);
-	}
-	{
-		REG_SET(REGAMMA_CNTLA_SLOPE_CNTL, 0,
-			REGAMMA_CNTLA_EXP_REGION_LINEAR_SLOPE, params->arr_points[0].custom_float_slope);
+	REG_SET_2(REGAMMA_CNTLA_START_CNTL, 0,
+		  REGAMMA_CNTLA_EXP_REGION_START, params->arr_points[0].custom_float_x,
+		  REGAMMA_CNTLA_EXP_REGION_START_SEGMENT, 0);
 
-	}
-	{
-		REG_SET(REGAMMA_CNTLA_END_CNTL1, 0,
-			REGAMMA_CNTLA_EXP_REGION_END, params->arr_points[1].custom_float_x);
-	}
-	{
-		REG_SET_2(REGAMMA_CNTLA_END_CNTL2, 0,
-			REGAMMA_CNTLA_EXP_REGION_END_BASE, params->arr_points[1].custom_float_y,
-			REGAMMA_CNTLA_EXP_REGION_END_SLOPE, params->arr_points[2].custom_float_slope);
-	}
+	REG_SET(REGAMMA_CNTLA_SLOPE_CNTL, 0,
+		REGAMMA_CNTLA_EXP_REGION_LINEAR_SLOPE, params->arr_points[0].custom_float_slope);
 
-	curve = params->arr_curve_points;
+	REG_SET(REGAMMA_CNTLA_END_CNTL1, 0,
+		REGAMMA_CNTLA_EXP_REGION_END, params->arr_points[1].custom_float_x);
 
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_0_1, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-	}
+	REG_SET_2(REGAMMA_CNTLA_END_CNTL2, 0,
+		  REGAMMA_CNTLA_EXP_REGION_END_BASE, params->arr_points[1].custom_float_y,
+		  REGAMMA_CNTLA_EXP_REGION_END_SLOPE, params->arr_points[1].custom_float_slope);
 
-	curve += 2;
-
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_2_3, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-	}
+	curve = params->arr_curve_points;
 
+	REG_SET_4(REGAMMA_CNTLA_REGION_0_1, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
 	curve += 2;
 
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_4_5, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-	}
-
+	REG_SET_4(REGAMMA_CNTLA_REGION_2_3, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
 	curve += 2;
 
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_6_7, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-	}
-
+	REG_SET_4(REGAMMA_CNTLA_REGION_4_5, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
 	curve += 2;
 
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_8_9, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-	}
-
+	REG_SET_4(REGAMMA_CNTLA_REGION_6_7, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
 	curve += 2;
 
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_10_11, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-	}
-
+	REG_SET_4(REGAMMA_CNTLA_REGION_8_9, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
 	curve += 2;
 
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_12_13, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-
-	}
+	REG_SET_4(REGAMMA_CNTLA_REGION_10_11, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
+	curve += 2;
 
+	REG_SET_4(REGAMMA_CNTLA_REGION_12_13, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
 	curve += 2;
 
-	{
-		REG_SET_4(REGAMMA_CNTLA_REGION_14_15, 0,
-			REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
-			REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
-			REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
-			REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
-	}
+	REG_SET_4(REGAMMA_CNTLA_REGION_14_15, 0,
+		  REGAMMA_CNTLA_EXP_REGION0_LUT_OFFSET, curve[0].offset,
+		  REGAMMA_CNTLA_EXP_REGION0_NUM_SEGMENTS, curve[0].segments_num,
+		  REGAMMA_CNTLA_EXP_REGION1_LUT_OFFSET, curve[1].offset,
+		  REGAMMA_CNTLA_EXP_REGION1_NUM_SEGMENTS, curve[1].segments_num);
 }
 
 
 
-void dce110_opp_program_regamma_pwl(
-	struct transform *xfm,
-	const struct pwl_params *params)
+void dce110_opp_program_regamma_pwl(struct transform *xfm,
+				    const struct pwl_params *params)
 {
 	struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm);
 
@@ -1388,47 +1341,42 @@ void dce110_opp_program_regamma_pwl(
 	program_pwl(xfm_dce, params);
 }
 
-void dce110_opp_power_on_regamma_lut(
-	struct transform *xfm,
-	bool power_on)
+void dce110_opp_power_on_regamma_lut(struct transform *xfm,
+				     bool power_on)
 {
 	struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm);
 
 	if (REG(DCFE_MEM_PWR_CTRL))
 		REG_UPDATE_2(DCFE_MEM_PWR_CTRL,
-			DCP_REGAMMA_MEM_PWR_DIS, power_on,
-			DCP_LUT_MEM_PWR_DIS, power_on);
+			     DCP_REGAMMA_MEM_PWR_DIS, power_on,
+			     DCP_LUT_MEM_PWR_DIS, power_on);
 	else
 		REG_UPDATE_2(DCFE_MEM_LIGHT_SLEEP_CNTL,
-			REGAMMA_LUT_LIGHT_SLEEP_DIS, power_on,
-			DCP_LUT_LIGHT_SLEEP_DIS, power_on);
+			    REGAMMA_LUT_LIGHT_SLEEP_DIS, power_on,
+			    DCP_LUT_LIGHT_SLEEP_DIS, power_on);
 
 }
 
 void dce110_opp_set_regamma_mode(struct transform *xfm,
-		enum opp_regamma mode)
+				 enum opp_regamma mode)
 {
 	struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm);
 
 	REG_SET(REGAMMA_CONTROL, 0,
-			GRPH_REGAMMA_MODE, mode);
+		GRPH_REGAMMA_MODE, mode);
 }
 
 static const struct transform_funcs dce_transform_funcs = {
 	.transform_reset = dce_transform_reset,
-	.transform_set_scaler =
-		dce_transform_set_scaler,
-	.transform_set_gamut_remap =
-		dce_transform_set_gamut_remap,
+	.transform_set_scaler = dce_transform_set_scaler,
+	.transform_set_gamut_remap = dce_transform_set_gamut_remap,
 	.opp_set_csc_adjustment = dce110_opp_set_csc_adjustment,
 	.opp_set_csc_default = dce110_opp_set_csc_default,
 	.opp_power_on_regamma_lut = dce110_opp_power_on_regamma_lut,
 	.opp_program_regamma_pwl = dce110_opp_program_regamma_pwl,
 	.opp_set_regamma_mode = dce110_opp_set_regamma_mode,
-	.transform_set_pixel_storage_depth =
-		dce_transform_set_pixel_storage_depth,
-	.transform_get_optimal_number_of_taps =
-		dce_transform_get_optimal_number_of_taps
+	.transform_set_pixel_storage_depth = dce_transform_set_pixel_storage_depth,
+	.transform_get_optimal_number_of_taps = dce_transform_get_optimal_number_of_taps
 };
 
 /*****************************************/
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
index e7a694835e3e..469af0587604 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_hw_sequencer.c
@@ -148,5 +148,7 @@ void dce100_hw_sequencer_construct(struct dc *dc)
 
 	dc->hwss.enable_display_power_gating = dce100_enable_display_power_gating;
 	dc->hwss.set_bandwidth = dce100_set_bandwidth;
+	dc->hwss.pplib_apply_display_requirements =
+			dce100_pplib_apply_display_requirements;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
index 3ea43e2a9450..442dd2d93618 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
@@ -852,6 +852,7 @@ static bool construct(
 	dc->caps.max_downscale_ratio = 200;
 	dc->caps.i2c_speed_in_khz = 40;
 	dc->caps.max_cursor_size = 128;
+	dc->caps.dual_link_dvi = true;
 
 	for (i = 0; i < pool->base.pipe_count; i++) {
 		pool->base.timing_generators[i] =
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
index d844fadcd56f..6f382a3ac90f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
@@ -257,9 +257,9 @@ static void build_prescale_params(struct ipp_prescale_params *prescale_params,
 	}
 }
 
-static bool dce110_set_input_transfer_func(
-	struct pipe_ctx *pipe_ctx,
-	const struct dc_plane_state *plane_state)
+static bool
+dce110_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
+			       const struct dc_plane_state *plane_state)
 {
 	struct input_pixel_processor *ipp = pipe_ctx->plane_res.ipp;
 	const struct dc_transfer_func *tf = NULL;
@@ -280,25 +280,19 @@ static bool dce110_set_input_transfer_func(
 
 	if (tf == NULL) {
 		/* Default case if no input transfer function specified */
-		ipp->funcs->ipp_set_degamma(ipp,
-				IPP_DEGAMMA_MODE_HW_sRGB);
+		ipp->funcs->ipp_set_degamma(ipp, IPP_DEGAMMA_MODE_HW_sRGB);
 	} else if (tf->type == TF_TYPE_PREDEFINED) {
 		switch (tf->tf) {
 		case TRANSFER_FUNCTION_SRGB:
-			ipp->funcs->ipp_set_degamma(ipp,
-					IPP_DEGAMMA_MODE_HW_sRGB);
+			ipp->funcs->ipp_set_degamma(ipp, IPP_DEGAMMA_MODE_HW_sRGB);
 			break;
 		case TRANSFER_FUNCTION_BT709:
-			ipp->funcs->ipp_set_degamma(ipp,
-					IPP_DEGAMMA_MODE_HW_xvYCC);
+			ipp->funcs->ipp_set_degamma(ipp, IPP_DEGAMMA_MODE_HW_xvYCC);
 			break;
 		case TRANSFER_FUNCTION_LINEAR:
-			ipp->funcs->ipp_set_degamma(ipp,
-					IPP_DEGAMMA_MODE_BYPASS);
+			ipp->funcs->ipp_set_degamma(ipp, IPP_DEGAMMA_MODE_BYPASS);
 			break;
 		case TRANSFER_FUNCTION_PQ:
-			result = false;
-			break;
 		default:
 			result = false;
 			break;
@@ -313,10 +307,9 @@ static bool dce110_set_input_transfer_func(
 	return result;
 }
 
-static bool convert_to_custom_float(
-		struct pwl_result_data *rgb_resulted,
-		struct curve_points *arr_points,
-		uint32_t hw_points_num)
+static bool convert_to_custom_float(struct pwl_result_data *rgb_resulted,
+				    struct curve_points *arr_points,
+				    uint32_t hw_points_num)
 {
 	struct custom_float_format fmt;
 
@@ -328,26 +321,20 @@ static bool convert_to_custom_float(
 	fmt.mantissa_bits = 12;
 	fmt.sign = true;
 
-	if (!convert_to_custom_float_format(
-		arr_points[0].x,
-		&fmt,
-		&arr_points[0].custom_float_x)) {
+	if (!convert_to_custom_float_format(arr_points[0].x, &fmt,
+					    &arr_points[0].custom_float_x)) {
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
 
-	if (!convert_to_custom_float_format(
-		arr_points[0].offset,
-		&fmt,
-		&arr_points[0].custom_float_offset)) {
+	if (!convert_to_custom_float_format(arr_points[0].offset, &fmt,
+					    &arr_points[0].custom_float_offset)) {
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
 
-	if (!convert_to_custom_float_format(
-		arr_points[0].slope,
-		&fmt,
-		&arr_points[0].custom_float_slope)) {
+	if (!convert_to_custom_float_format(arr_points[0].slope, &fmt,
+					    &arr_points[0].custom_float_slope)) {
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
@@ -355,26 +342,20 @@ static bool convert_to_custom_float(
 	fmt.mantissa_bits = 10;
 	fmt.sign = false;
 
-	if (!convert_to_custom_float_format(
-		arr_points[1].x,
-		&fmt,
-		&arr_points[1].custom_float_x)) {
+	if (!convert_to_custom_float_format(arr_points[1].x, &fmt,
+					    &arr_points[1].custom_float_x)) {
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
 
-	if (!convert_to_custom_float_format(
-		arr_points[1].y,
-		&fmt,
-		&arr_points[1].custom_float_y)) {
+	if (!convert_to_custom_float_format(arr_points[1].y, &fmt,
+					    &arr_points[1].custom_float_y)) {
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
 
-	if (!convert_to_custom_float_format(
-		arr_points[2].slope,
-		&fmt,
-		&arr_points[2].custom_float_slope)) {
+	if (!convert_to_custom_float_format(arr_points[1].slope, &fmt,
+					    &arr_points[1].custom_float_slope)) {
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
@@ -383,50 +364,38 @@ static bool convert_to_custom_float(
 	fmt.sign = true;
 
 	while (i != hw_points_num) {
-		if (!convert_to_custom_float_format(
-			rgb->red,
-			&fmt,
-			&rgb->red_reg)) {
+		if (!convert_to_custom_float_format(rgb->red, &fmt,
+						    &rgb->red_reg)) {
 			BREAK_TO_DEBUGGER();
 			return false;
 		}
 
-		if (!convert_to_custom_float_format(
-			rgb->green,
-			&fmt,
-			&rgb->green_reg)) {
+		if (!convert_to_custom_float_format(rgb->green, &fmt,
+						    &rgb->green_reg)) {
 			BREAK_TO_DEBUGGER();
 			return false;
 		}
 
-		if (!convert_to_custom_float_format(
-			rgb->blue,
-			&fmt,
-			&rgb->blue_reg)) {
+		if (!convert_to_custom_float_format(rgb->blue, &fmt,
+						    &rgb->blue_reg)) {
 			BREAK_TO_DEBUGGER();
 			return false;
 		}
 
-		if (!convert_to_custom_float_format(
-			rgb->delta_red,
-			&fmt,
-			&rgb->delta_red_reg)) {
+		if (!convert_to_custom_float_format(rgb->delta_red, &fmt,
+						    &rgb->delta_red_reg)) {
 			BREAK_TO_DEBUGGER();
 			return false;
 		}
 
-		if (!convert_to_custom_float_format(
-			rgb->delta_green,
-			&fmt,
-			&rgb->delta_green_reg)) {
+		if (!convert_to_custom_float_format(rgb->delta_green, &fmt,
+						    &rgb->delta_green_reg)) {
 			BREAK_TO_DEBUGGER();
 			return false;
 		}
 
-		if (!convert_to_custom_float_format(
-			rgb->delta_blue,
-			&fmt,
-			&rgb->delta_blue_reg)) {
+		if (!convert_to_custom_float_format(rgb->delta_blue, &fmt,
+						    &rgb->delta_blue_reg)) {
 			BREAK_TO_DEBUGGER();
 			return false;
 		}
@@ -438,8 +407,9 @@ static bool convert_to_custom_float(
 	return true;
 }
 
-static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
-		*output_tf, struct pwl_params *regamma_params)
+static bool
+dce110_translate_regamma_to_hw_format(const struct dc_transfer_func *output_tf,
+				      struct pwl_params *regamma_params)
 {
 	struct curve_points *arr_points;
 	struct pwl_result_data *rgb_resulted;
@@ -454,8 +424,7 @@ static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
 	int32_t segment_start, segment_end;
 	uint32_t i, j, k, seg_distr[16], increment, start_index, hw_points;
 
-	if (output_tf == NULL || regamma_params == NULL ||
-			output_tf->type == TF_TYPE_BYPASS)
+	if (output_tf == NULL || regamma_params == NULL || output_tf->type == TF_TYPE_BYPASS)
 		return false;
 
 	arr_points = regamma_params->arr_points;
@@ -534,19 +503,14 @@ static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
 
 	/* last point */
 	start_index = (segment_end + 25) * 32;
-	rgb_resulted[hw_points - 1].red =
-			output_tf->tf_pts.red[start_index];
-	rgb_resulted[hw_points - 1].green =
-			output_tf->tf_pts.green[start_index];
-	rgb_resulted[hw_points - 1].blue =
-			output_tf->tf_pts.blue[start_index];
+	rgb_resulted[hw_points - 1].red = output_tf->tf_pts.red[start_index];
+	rgb_resulted[hw_points - 1].green = output_tf->tf_pts.green[start_index];
+	rgb_resulted[hw_points - 1].blue = output_tf->tf_pts.blue[start_index];
 
 	arr_points[0].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-			dal_fixed31_32_from_int(segment_start));
+					     dal_fixed31_32_from_int(segment_start));
 	arr_points[1].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-			dal_fixed31_32_from_int(segment_end));
-	arr_points[2].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-			dal_fixed31_32_from_int(segment_end));
+					     dal_fixed31_32_from_int(segment_end));
 
 	y_r = rgb_resulted[0].red;
 	y_g = rgb_resulted[0].green;
@@ -555,9 +519,8 @@ static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
 	y1_min = dal_fixed31_32_min(y_r, dal_fixed31_32_min(y_g, y_b));
 
 	arr_points[0].y = y1_min;
-	arr_points[0].slope = dal_fixed31_32_div(
-					arr_points[0].y,
-					arr_points[0].x);
+	arr_points[0].slope = dal_fixed31_32_div(arr_points[0].y,
+						 arr_points[0].x);
 
 	y_r = rgb_resulted[hw_points - 1].red;
 	y_g = rgb_resulted[hw_points - 1].green;
@@ -569,24 +532,18 @@ static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
 	y3_max = dal_fixed31_32_max(y_r, dal_fixed31_32_max(y_g, y_b));
 
 	arr_points[1].y = y3_max;
-	arr_points[2].y = y3_max;
 
 	arr_points[1].slope = dal_fixed31_32_zero;
-	arr_points[2].slope = dal_fixed31_32_zero;
 
 	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
 		/* for PQ, we want to have a straight line from last HW X point,
 		 * and the slope to be such that we hit 1.0 at 10000 nits.
 		 */
-		const struct fixed31_32 end_value =
-				dal_fixed31_32_from_int(125);
+		const struct fixed31_32 end_value = dal_fixed31_32_from_int(125);
 
 		arr_points[1].slope = dal_fixed31_32_div(
-			dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
-			dal_fixed31_32_sub(end_value, arr_points[1].x));
-		arr_points[2].slope = dal_fixed31_32_div(
-			dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
-			dal_fixed31_32_sub(end_value, arr_points[1].x));
+				dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
+				dal_fixed31_32_sub(end_value, arr_points[1].x));
 	}
 
 	regamma_params->hw_points_num = hw_points;
@@ -594,18 +551,15 @@ static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
 	i = 1;
 	for (k = 0; k < 16 && i < 16; k++) {
 		if (seg_distr[k] != -1) {
-			regamma_params->arr_curve_points[k].segments_num =
-					seg_distr[k];
+			regamma_params->arr_curve_points[k].segments_num = seg_distr[k];
 			regamma_params->arr_curve_points[i].offset =
-					regamma_params->arr_curve_points[k].
-					offset + (1 << seg_distr[k]);
+					regamma_params->arr_curve_points[k].offset + (1 << seg_distr[k]);
 		}
 		i++;
 	}
 
 	if (seg_distr[k] != -1)
-		regamma_params->arr_curve_points[k].segments_num =
-				seg_distr[k];
+		regamma_params->arr_curve_points[k].segments_num = seg_distr[k];
 
 	rgb = rgb_resulted;
 	rgb_plus_1 = rgb_resulted + 1;
@@ -620,15 +574,9 @@ static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
 		if (dal_fixed31_32_lt(rgb_plus_1->blue, rgb->blue))
 			rgb_plus_1->blue = rgb->blue;
 
-		rgb->delta_red = dal_fixed31_32_sub(
-			rgb_plus_1->red,
-			rgb->red);
-		rgb->delta_green = dal_fixed31_32_sub(
-			rgb_plus_1->green,
-			rgb->green);
-		rgb->delta_blue = dal_fixed31_32_sub(
-			rgb_plus_1->blue,
-			rgb->blue);
+		rgb->delta_red = dal_fixed31_32_sub(rgb_plus_1->red, rgb->red);
+		rgb->delta_green = dal_fixed31_32_sub(rgb_plus_1->green, rgb->green);
+		rgb->delta_blue = dal_fixed31_32_sub(rgb_plus_1->blue, rgb->blue);
 
 		++rgb_plus_1;
 		++rgb;
@@ -640,9 +588,9 @@ static bool dce110_translate_regamma_to_hw_format(const struct dc_transfer_func
 	return true;
 }
 
-static bool dce110_set_output_transfer_func(
-	struct pipe_ctx *pipe_ctx,
-	const struct dc_stream_state *stream)
+static bool
+dce110_set_output_transfer_func(struct pipe_ctx *pipe_ctx,
+				const struct dc_stream_state *stream)
 {
 	struct transform *xfm = pipe_ctx->plane_res.xfm;
 
@@ -650,13 +598,11 @@ static bool dce110_set_output_transfer_func(
 	xfm->regamma_params.hw_points_num = GAMMA_HW_POINTS_NUM;
 
 	if (stream->out_transfer_func &&
-		stream->out_transfer_func->type ==
-			TF_TYPE_PREDEFINED &&
-		stream->out_transfer_func->tf ==
-			TRANSFER_FUNCTION_SRGB) {
+	    stream->out_transfer_func->type == TF_TYPE_PREDEFINED &&
+	    stream->out_transfer_func->tf == TRANSFER_FUNCTION_SRGB) {
 		xfm->funcs->opp_set_regamma_mode(xfm, OPP_REGAMMA_SRGB);
-	} else if (dce110_translate_regamma_to_hw_format(
-				stream->out_transfer_func, &xfm->regamma_params)) {
+	} else if (dce110_translate_regamma_to_hw_format(stream->out_transfer_func,
+							 &xfm->regamma_params)) {
 		xfm->funcs->opp_program_regamma_pwl(xfm, &xfm->regamma_params);
 		xfm->funcs->opp_set_regamma_mode(xfm, OPP_REGAMMA_USER);
 	} else {
@@ -742,15 +688,22 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
 	struct dc_crtc_timing *timing = &pipe_ctx->stream->timing;
 	struct dc_link *link = pipe_ctx->stream->sink->link;
 
-	/* 1. update AVI info frame (HDMI, DP)
-	 * we always need to update info frame
-	*/
+
 	uint32_t active_total_with_borders;
 	uint32_t early_control = 0;
 	struct timing_generator *tg = pipe_ctx->stream_res.tg;
 
-	/* TODOFPGA may change to hwss.update_info_frame */
+	/* For MST, there are multiply stream go to only one link.
+	 * connect DIG back_end to front_end while enable_stream and
+	 * disconnect them during disable_stream
+	 * BY this, it is logic clean to separate stream and link */
+	link->link_enc->funcs->connect_dig_be_to_fe(link->link_enc,
+						    pipe_ctx->stream_res.stream_enc->id, true);
+
+	/* update AVI info frame (HDMI, DP)*/
+	/* TODO: FPGA may change to hwss.update_info_frame */
 	dce110_update_info_frame(pipe_ctx);
+
 	/* enable early control to avoid corruption on DP monitor*/
 	active_total_with_borders =
 			timing->h_addressable
@@ -771,12 +724,8 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->stream_res.stream_enc->funcs->dp_audio_enable(pipe_ctx->stream_res.stream_enc);
 	}
 
-	/* For MST, there are multiply stream go to only one link.
-	 * connect DIG back_end to front_end while enable_stream and
-	 * disconnect them during disable_stream
-	 * BY this, it is logic clean to separate stream and link */
-	link->link_enc->funcs->connect_dig_be_to_fe(link->link_enc,
-						    pipe_ctx->stream_res.stream_enc->id, true);
+
+
 
 }
 
@@ -814,11 +763,11 @@ static enum bp_result link_transmitter_control(
  * eDP only.
  */
 void hwss_edp_wait_for_hpd_ready(
-	struct link_encoder *enc,
-	bool power_up)
+		struct dc_link *link,
+		bool power_up)
 {
-	struct dc_context *ctx = enc->ctx;
-	struct graphics_object_id connector = enc->connector;
+	struct dc_context *ctx = link->ctx;
+	struct graphics_object_id connector = link->link_enc->connector;
 	struct gpio *hpd;
 	bool edp_hpd_high = false;
 	uint32_t time_elapsed = 0;
@@ -882,16 +831,16 @@ void hwss_edp_wait_for_hpd_ready(
 }
 
 void hwss_edp_power_control(
-	struct link_encoder *enc,
-	bool power_up)
+		struct dc_link *link,
+		bool power_up)
 {
-	struct dc_context *ctx = enc->ctx;
+	struct dc_context *ctx = link->ctx;
 	struct dce_hwseq *hwseq = ctx->dc->hwseq;
 	struct bp_transmitter_control cntl = { 0 };
 	enum bp_result bp_result;
 
 
-	if (dal_graphics_object_id_get_connector_id(enc->connector)
+	if (dal_graphics_object_id_get_connector_id(link->link_enc->connector)
 			!= CONNECTOR_ID_EDP) {
 		BREAK_TO_DEBUGGER();
 		return;
@@ -907,11 +856,11 @@ void hwss_edp_power_control(
 		cntl.action = power_up ?
 			TRANSMITTER_CONTROL_POWER_ON :
 			TRANSMITTER_CONTROL_POWER_OFF;
-		cntl.transmitter = enc->transmitter;
-		cntl.connector_obj_id = enc->connector;
+		cntl.transmitter = link->link_enc->transmitter;
+		cntl.connector_obj_id = link->link_enc->connector;
 		cntl.coherent = false;
 		cntl.lanes_number = LANE_COUNT_FOUR;
-		cntl.hpd_sel = enc->hpd_source;
+		cntl.hpd_sel = link->link_enc->hpd_source;
 
 		bp_result = link_transmitter_control(ctx->dc_bios, &cntl);
 
@@ -924,8 +873,6 @@ void hwss_edp_power_control(
 				"%s: Skipping Panel Power action: %s\n",
 				__func__, (power_up ? "On":"Off"));
 	}
-
-	hwss_edp_wait_for_hpd_ready(enc, true);
 }
 
 /*todo: cloned in stream enc, fix*/
@@ -934,14 +881,14 @@ void hwss_edp_power_control(
  * eDP only. Control the backlight of the eDP panel
  */
 void hwss_edp_backlight_control(
-	struct dc_link *link,
-	bool enable)
+		struct dc_link *link,
+		bool enable)
 {
-	struct dce_hwseq *hws = link->dc->hwseq;
-	struct dc_context *ctx = link->dc->ctx;
+	struct dc_context *ctx = link->ctx;
+	struct dce_hwseq *hws = ctx->dc->hwseq;
 	struct bp_transmitter_control cntl = { 0 };
 
-	if (dal_graphics_object_id_get_connector_id(link->link_id)
+	if (dal_graphics_object_id_get_connector_id(link->link_enc->connector)
 		!= CONNECTOR_ID_EDP) {
 		BREAK_TO_DEBUGGER();
 		return;
@@ -982,7 +929,7 @@ void hwss_edp_backlight_control(
 	 * Enable it in the future if necessary.
 	 */
 	/* dc_service_sleep_in_milliseconds(50); */
-	link_transmitter_control(link->dc->ctx->dc_bios, &cntl);
+	link_transmitter_control(ctx->dc_bios, &cntl);
 }
 
 void dce110_disable_stream(struct pipe_ctx *pipe_ctx, int option)
@@ -1026,11 +973,9 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx, int option)
 	}
 
 	/* blank at encoder level */
-	if (dc_is_dp_signal(pipe_ctx->stream->signal)) {
-		if (pipe_ctx->stream->sink->link->connector_signal == SIGNAL_TYPE_EDP)
-			hwss_edp_backlight_control(link, false);
+	if (dc_is_dp_signal(pipe_ctx->stream->signal))
 		pipe_ctx->stream_res.stream_enc->funcs->dp_blank(pipe_ctx->stream_res.stream_enc);
-	}
+
 	link->link_enc->funcs->connect_dig_be_to_fe(
 			link->link_enc,
 			pipe_ctx->stream_res.stream_enc->id,
@@ -1042,15 +987,12 @@ void dce110_unblank_stream(struct pipe_ctx *pipe_ctx,
 		struct dc_link_settings *link_settings)
 {
 	struct encoder_unblank_param params = { { 0 } };
-	struct dc_link *link = pipe_ctx->stream->sink->link;
 
 	/* only 3 items below are used by unblank */
 	params.pixel_clk_khz =
 		pipe_ctx->stream->timing.pix_clk_khz;
 	params.link_settings.link_rate = link_settings->link_rate;
 	pipe_ctx->stream_res.stream_enc->funcs->dp_unblank(pipe_ctx->stream_res.stream_enc, &params);
-	if (link->connector_signal == SIGNAL_TYPE_EDP)
-		hwss_edp_backlight_control(link, true);
 }
 
 
@@ -1401,7 +1343,7 @@ static void power_down_encoders(struct dc *dc)
 		}
 
 		dc->links[i]->link_enc->funcs->disable_output(
-				dc->links[i]->link_enc, signal, dc->links[i]);
+				dc->links[i]->link_enc, signal);
 	}
 }
 
@@ -1462,7 +1404,9 @@ static void disable_vga_and_power_gate_all_controllers(
 		enable_display_pipe_clock_gating(ctx,
 				true);
 
-		dc->hwss.power_down_front_end(dc, i);
+		dc->current_state->res_ctx.pipe_ctx[i].pipe_idx = i;
+		dc->hwss.disable_plane(dc,
+			&dc->current_state->res_ctx.pipe_ctx[i]);
 	}
 }
 
@@ -1748,60 +1692,69 @@ static void apply_min_clocks(
 /*
  *  Check if FBC can be enabled
  */
-static enum dc_status validate_fbc(struct dc *dc,
-		struct dc_state *context)
+static bool should_enable_fbc(struct dc *dc,
+			      struct dc_state *context,
+			      uint32_t *pipe_idx)
 {
-	struct pipe_ctx *pipe_ctx =
-			      &context->res_ctx.pipe_ctx[0];
+	uint32_t i;
+	struct pipe_ctx *pipe_ctx = NULL;
+	struct resource_context *res_ctx = &context->res_ctx;
+
 
 	ASSERT(dc->fbc_compressor);
 
 	/* FBC memory should be allocated */
 	if (!dc->ctx->fbc_gpu_addr)
-		return DC_ERROR_UNEXPECTED;
+		return false;
 
 	/* Only supports single display */
 	if (context->stream_count != 1)
-		return DC_ERROR_UNEXPECTED;
+		return false;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (res_ctx->pipe_ctx[i].stream) {
+			pipe_ctx = &res_ctx->pipe_ctx[i];
+			*pipe_idx = i;
+			break;
+		}
+	}
 
 	/* Only supports eDP */
 	if (pipe_ctx->stream->sink->link->connector_signal != SIGNAL_TYPE_EDP)
-		return DC_ERROR_UNEXPECTED;
+		return false;
 
 	/* PSR should not be enabled */
 	if (pipe_ctx->stream->sink->link->psr_enabled)
-		return DC_ERROR_UNEXPECTED;
+		return false;
 
 	/* Nothing to compress */
 	if (!pipe_ctx->plane_state)
-		return DC_ERROR_UNEXPECTED;
+		return false;
 
 	/* Only for non-linear tiling */
 	if (pipe_ctx->plane_state->tiling_info.gfx8.array_mode == DC_ARRAY_LINEAR_GENERAL)
-		return DC_ERROR_UNEXPECTED;
+		return false;
 
-	return DC_OK;
+	return true;
 }
 
 /*
  *  Enable FBC
  */
-static enum dc_status enable_fbc(struct dc *dc,
-		struct dc_state *context)
+static void enable_fbc(struct dc *dc,
+		       struct dc_state *context)
 {
-	enum dc_status status = validate_fbc(dc, context);
+	uint32_t pipe_idx = 0;
 
-	if (status == DC_OK) {
+	if (should_enable_fbc(dc, context, &pipe_idx)) {
 		/* Program GRPH COMPRESSED ADDRESS and PITCH */
 		struct compr_addr_and_pitch_params params = {0, 0, 0};
 		struct compressor *compr = dc->fbc_compressor;
-		struct pipe_ctx *pipe_ctx =
-				      &context->res_ctx.pipe_ctx[0];
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[pipe_idx];
+
 
-		params.source_view_width =
-				pipe_ctx->stream->timing.h_addressable;
-		params.source_view_height =
-				pipe_ctx->stream->timing.v_addressable;
+		params.source_view_width = pipe_ctx->stream->timing.h_addressable;
+		params.source_view_height = pipe_ctx->stream->timing.v_addressable;
 
 		compr->compr_surface_address.quad_part = dc->ctx->fbc_gpu_addr;
 
@@ -1810,7 +1763,6 @@ static enum dc_status enable_fbc(struct dc *dc,
 
 		compr->funcs->enable_fbc(compr, &params);
 	}
-	return status;
 }
 #endif
 
@@ -1888,7 +1840,7 @@ static void dce110_reset_hw_ctx_wrap(
 			if (old_clk)
 				old_clk->funcs->cs_power_down(old_clk);
 
-			dc->hwss.power_down_front_end(dc, pipe_ctx_old->pipe_idx);
+			dc->hwss.disable_plane(dc, pipe_ctx_old);
 
 			pipe_ctx_old->stream = NULL;
 		}
@@ -2076,8 +2028,7 @@ enum dc_status dce110_apply_ctx_to_hw(
 		if (pipe_ctx->stream == pipe_ctx_old->stream)
 			continue;
 
-		if (pipe_ctx->stream && pipe_ctx_old->stream
-				&& !pipe_need_reprogram(pipe_ctx_old, pipe_ctx))
+		if (pipe_ctx_old->stream && !pipe_need_reprogram(pipe_ctx_old, pipe_ctx))
 			continue;
 
 		if (pipe_ctx->top_pipe)
@@ -2113,9 +2064,6 @@ enum dc_status dce110_apply_ctx_to_hw(
 				context,
 				dc);
 
-		if (dc->hwss.power_on_front_end)
-			dc->hwss.power_on_front_end(dc, pipe_ctx, context);
-
 		if (DC_OK != status)
 			return status;
 	}
@@ -2145,16 +2093,8 @@ static void set_default_colors(struct pipe_ctx *pipe_ctx)
 	struct default_adjustment default_adjust = { 0 };
 
 	default_adjust.force_hw_default = false;
-	if (pipe_ctx->plane_state == NULL)
-		default_adjust.in_color_space = COLOR_SPACE_SRGB;
-	else
-		default_adjust.in_color_space =
-				pipe_ctx->plane_state->color_space;
-	if (pipe_ctx->stream == NULL)
-		default_adjust.out_color_space = COLOR_SPACE_SRGB;
-	else
-		default_adjust.out_color_space =
-				pipe_ctx->stream->output_color_space;
+	default_adjust.in_color_space = pipe_ctx->plane_state->color_space;
+	default_adjust.out_color_space = pipe_ctx->stream->output_color_space;
 	default_adjust.csc_adjust_type = GRAPHICS_CSC_ADJUST_TYPE_SW;
 	default_adjust.surface_pixel_format = pipe_ctx->plane_res.scl_data.format;
 
@@ -2279,8 +2219,7 @@ static void set_plane_config(
 	dce_enable_fe_clock(dc->hwseq, pipe_ctx->pipe_idx, true);
 
 	set_default_colors(pipe_ctx);
-	if (pipe_ctx->stream->csc_color_matrix.enable_adjustment
-			== true) {
+	if (pipe_ctx->stream->csc_color_matrix.enable_adjustment == true) {
 		tbl_entry.color_space =
 			pipe_ctx->stream->output_color_space;
 
@@ -2458,20 +2397,16 @@ static void dce110_enable_timing_synchronization(
 
 	for (i = 1 /* skip the master */; i < group_size; i++)
 		grouped_pipes[i]->stream_res.tg->funcs->enable_reset_trigger(
-					grouped_pipes[i]->stream_res.tg, gsl_params.gsl_group);
-
-
+				grouped_pipes[i]->stream_res.tg,
+				gsl_params.gsl_group);
 
 	for (i = 1 /* skip the master */; i < group_size; i++) {
 		DC_SYNC_INFO("GSL: waiting for reset to occur.\n");
 		wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[i]->stream_res.tg);
-		/* Regardless of success of the wait above, remove the reset or
-		 * the driver will start timing out on Display requests. */
-		DC_SYNC_INFO("GSL: disabling trigger-reset.\n");
-		grouped_pipes[i]->stream_res.tg->funcs->disable_reset_trigger(grouped_pipes[i]->stream_res.tg);
+		grouped_pipes[i]->stream_res.tg->funcs->disable_reset_trigger(
+				grouped_pipes[i]->stream_res.tg);
 	}
 
-
 	/* GSL Vblank synchronization is a one time sync mechanism, assumption
 	 * is that the sync'ed displays will not drift out of sync over time*/
 	DC_SYNC_INFO("GSL: Restoring register states.\n");
@@ -2481,6 +2416,39 @@ static void dce110_enable_timing_synchronization(
 	DC_SYNC_INFO("GSL: Set-up complete.\n");
 }
 
+static void dce110_enable_per_frame_crtc_position_reset(
+		struct dc *dc,
+		int group_size,
+		struct pipe_ctx *grouped_pipes[])
+{
+	struct dc_context *dc_ctx = dc->ctx;
+	struct dcp_gsl_params gsl_params = { 0 };
+	int i;
+
+	gsl_params.gsl_group = 0;
+	gsl_params.gsl_master = grouped_pipes[0]->stream->triggered_crtc_reset.event_source->status.primary_otg_inst;
+
+	for (i = 0; i < group_size; i++)
+		grouped_pipes[i]->stream_res.tg->funcs->setup_global_swap_lock(
+					grouped_pipes[i]->stream_res.tg, &gsl_params);
+
+	DC_SYNC_INFO("GSL: enabling trigger-reset\n");
+
+	for (i = 1; i < group_size; i++)
+		grouped_pipes[i]->stream_res.tg->funcs->enable_crtc_reset(
+				grouped_pipes[i]->stream_res.tg,
+				gsl_params.gsl_master,
+				&grouped_pipes[i]->stream->triggered_crtc_reset);
+
+	DC_SYNC_INFO("GSL: waiting for reset to occur.\n");
+	for (i = 1; i < group_size; i++)
+		wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[i]->stream_res.tg);
+
+	for (i = 0; i < group_size; i++)
+		grouped_pipes[i]->stream_res.tg->funcs->tear_down_global_swap_lock(grouped_pipes[i]->stream_res.tg);
+
+}
+
 static void init_hw(struct dc *dc)
 {
 	int i;
@@ -2513,6 +2481,10 @@ static void init_hw(struct dc *dc)
 		 * required signal (which may be different from the
 		 * default signal on connector). */
 		struct dc_link *link = dc->links[i];
+
+		if (link->link_enc->connector.id == CONNECTOR_ID_EDP)
+			dc->hwss.edp_power_control(link, true);
+
 		link->link_enc->funcs->hw_init(link->link_enc);
 	}
 
@@ -2567,6 +2539,10 @@ void dce110_fill_display_configs(
 
 		ASSERT(pipe_ctx != NULL);
 
+		/* only notify active stream */
+		if (stream->dpms_off)
+			continue;
+
 		num_cfgs++;
 		cfg->signal = pipe_ctx->stream->signal;
 		cfg->pipe_idx = pipe_ctx->pipe_idx;
@@ -2722,8 +2698,7 @@ static void dce110_program_front_end_for_pipe(
 	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 	struct xfm_grph_csc_adjustment adjust;
 	struct out_csc_color_matrix tbl_entry;
-	struct pipe_ctx *cur_pipe_ctx =
-					&dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx];
+	struct pipe_ctx *cur_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx];
 	unsigned int i;
 
 	memset(&tbl_entry, 0, sizeof(tbl_entry));
@@ -2816,10 +2791,8 @@ static void dce110_program_front_end_for_pipe(
 
 	/* Moved programming gamma from dc to hwss */
 	if (cur_pipe_ctx->plane_state != pipe_ctx->plane_state) {
-		dc->hwss.set_input_transfer_func(
-				pipe_ctx, pipe_ctx->plane_state);
-		dc->hwss.set_output_transfer_func(
-				pipe_ctx, pipe_ctx->stream);
+		dc->hwss.set_input_transfer_func(pipe_ctx, pipe_ctx->plane_state);
+		dc->hwss.set_output_transfer_func(pipe_ctx, pipe_ctx->stream);
 	}
 
 	dm_logger_write(dc->ctx->logger, LOG_SURFACE,
@@ -2889,13 +2862,12 @@ static void dce110_apply_ctx_for_surface(
 			continue;
 
 		/* Need to allocate mem before program front end for Fiji */
-		if (pipe_ctx->plane_res.mi != NULL)
-			pipe_ctx->plane_res.mi->funcs->allocate_mem_input(
-					pipe_ctx->plane_res.mi,
-					pipe_ctx->stream->timing.h_total,
-					pipe_ctx->stream->timing.v_total,
-					pipe_ctx->stream->timing.pix_clk_khz,
-					context->stream_count);
+		pipe_ctx->plane_res.mi->funcs->allocate_mem_input(
+				pipe_ctx->plane_res.mi,
+				pipe_ctx->stream->timing.h_total,
+				pipe_ctx->stream->timing.v_total,
+				pipe_ctx->stream->timing.pix_clk_khz,
+				context->stream_count);
 
 		dce110_program_front_end_for_pipe(dc, pipe_ctx);
 
@@ -2916,8 +2888,10 @@ static void dce110_apply_ctx_for_surface(
 	}
 }
 
-static void dce110_power_down_fe(struct dc *dc, int fe_idx)
+static void dce110_power_down_fe(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
+	int fe_idx = pipe_ctx->pipe_idx;
+
 	/* Do not power down fe when stream is active on dce*/
 	if (dc->current_state->res_ctx.pipe_ctx[fe_idx].stream)
 		return;
@@ -2959,6 +2933,49 @@ static void program_csc_matrix(struct pipe_ctx *pipe_ctx,
 	}
 }
 
+void dce110_set_cursor_position(struct pipe_ctx *pipe_ctx)
+{
+	struct dc_cursor_position pos_cpy = pipe_ctx->stream->cursor_position;
+	struct input_pixel_processor *ipp = pipe_ctx->plane_res.ipp;
+	struct mem_input *mi = pipe_ctx->plane_res.mi;
+	struct dc_cursor_mi_param param = {
+		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_khz,
+		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clock_inKhz,
+		.viewport_x_start = pipe_ctx->plane_res.scl_data.viewport.x,
+		.viewport_width = pipe_ctx->plane_res.scl_data.viewport.width,
+		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz
+	};
+
+	if (pipe_ctx->plane_state->address.type
+			== PLN_ADDR_TYPE_VIDEO_PROGRESSIVE)
+		pos_cpy.enable = false;
+
+	if (pipe_ctx->top_pipe && pipe_ctx->plane_state != pipe_ctx->top_pipe->plane_state)
+		pos_cpy.enable = false;
+
+	if (ipp->funcs->ipp_cursor_set_position)
+		ipp->funcs->ipp_cursor_set_position(ipp, &pos_cpy, &param);
+	if (mi->funcs->set_cursor_position)
+		mi->funcs->set_cursor_position(mi, &pos_cpy, &param);
+}
+
+void dce110_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
+{
+	struct dc_cursor_attributes *attributes = &pipe_ctx->stream->cursor_attributes;
+
+	if (pipe_ctx->plane_res.ipp->funcs->ipp_cursor_set_attributes)
+		pipe_ctx->plane_res.ipp->funcs->ipp_cursor_set_attributes(
+				pipe_ctx->plane_res.ipp, attributes);
+
+	if (pipe_ctx->plane_res.mi->funcs->set_cursor_attributes)
+		pipe_ctx->plane_res.mi->funcs->set_cursor_attributes(
+				pipe_ctx->plane_res.mi, attributes);
+
+	if (pipe_ctx->plane_res.xfm->funcs->set_cursor_attributes)
+		pipe_ctx->plane_res.xfm->funcs->set_cursor_attributes(
+				pipe_ctx->plane_res.xfm, attributes);
+}
+
 static void ready_shared_resources(struct dc *dc, struct dc_state *context) {}
 
 static void optimize_shared_resources(struct dc *dc) {}
@@ -2977,13 +2994,14 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.power_down = dce110_power_down,
 	.enable_accelerated_mode = dce110_enable_accelerated_mode,
 	.enable_timing_synchronization = dce110_enable_timing_synchronization,
+	.enable_per_frame_crtc_position_reset = dce110_enable_per_frame_crtc_position_reset,
 	.update_info_frame = dce110_update_info_frame,
 	.enable_stream = dce110_enable_stream,
 	.disable_stream = dce110_disable_stream,
 	.unblank_stream = dce110_unblank_stream,
 	.enable_display_pipe_clock_gating = enable_display_pipe_clock_gating,
 	.enable_display_power_gating = dce110_enable_display_power_gating,
-	.power_down_front_end = dce110_power_down_fe,
+	.disable_plane = dce110_power_down_fe,
 	.pipe_control_lock = dce_pipe_control_lock,
 	.set_bandwidth = dce110_set_bandwidth,
 	.set_drr = set_drr,
@@ -2996,8 +3014,12 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.wait_for_mpcc_disconnect = dce110_wait_for_mpcc_disconnect,
 	.ready_shared_resources = ready_shared_resources,
 	.optimize_shared_resources = optimize_shared_resources,
+	.pplib_apply_display_requirements = pplib_apply_display_requirements,
 	.edp_backlight_control = hwss_edp_backlight_control,
 	.edp_power_control = hwss_edp_power_control,
+	.edp_wait_for_hpd_ready = hwss_edp_wait_for_hpd_ready,
+	.set_cursor_position = dce110_set_cursor_position,
+	.set_cursor_attribute = dce110_set_cursor_attribute
 };
 
 void dce110_hw_sequencer_construct(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
index 4d72bb99be93..fc637647f643 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
@@ -70,12 +70,16 @@ uint32_t dce110_get_min_vblank_time_us(const struct dc_state *context);
 void dp_receiver_power_ctrl(struct dc_link *link, bool on);
 
 void hwss_edp_power_control(
-	struct link_encoder *enc,
-	bool power_up);
+		struct dc_link *link,
+		bool power_up);
 
 void hwss_edp_backlight_control(
 	struct dc_link *link,
 	bool enable);
 
+void hwss_edp_wait_for_hpd_ready(
+		struct dc_link *link,
+		bool power_up);
+
 #endif /* __DC_HWSS_DCE110_H__ */
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
index a06c6024deb4..7bab8c6d2a73 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c
@@ -237,26 +237,14 @@ static void program_size_and_rotation(
 	if (rotation == ROTATION_ANGLE_90 ||
 		rotation == ROTATION_ANGLE_270) {
 
-		uint32_t swap;
-		swap = local_size.video.luma_size.x;
-		local_size.video.luma_size.x =
-			local_size.video.luma_size.y;
-		local_size.video.luma_size.y  = swap;
-
-		swap = local_size.video.luma_size.width;
-		local_size.video.luma_size.width =
-			local_size.video.luma_size.height;
-		local_size.video.luma_size.height = swap;
-
-		swap = local_size.video.chroma_size.x;
-		local_size.video.chroma_size.x =
-			local_size.video.chroma_size.y;
-		local_size.video.chroma_size.y  = swap;
-
-		swap = local_size.video.chroma_size.width;
-		local_size.video.chroma_size.width =
-			local_size.video.chroma_size.height;
-		local_size.video.chroma_size.height = swap;
+		swap(local_size.video.luma_size.x,
+		     local_size.video.luma_size.y);
+		swap(local_size.video.luma_size.width,
+		     local_size.video.luma_size.height);
+		swap(local_size.video.chroma_size.x,
+		     local_size.video.chroma_size.y);
+		swap(local_size.video.chroma_size.width,
+		     local_size.video.chroma_size.height);
 	}
 
 	value = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c
index e98ed3058ea2..9b65b77e8823 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_regamma_v.c
@@ -175,7 +175,7 @@ static void regamma_config_regions_and_segments(
 		value = 0;
 		set_reg_field_value(
 			value,
-			params->arr_points[2].custom_float_slope,
+			params->arr_points[1].custom_float_slope,
 			GAMMA_CORR_CNTLA_END_CNTL2,
 			GAMMA_CORR_CNTLA_EXP_REGION_END_BASE);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
index 42df17f9aa8d..00f18c485e1e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
@@ -846,6 +846,16 @@ static bool dce110_validate_bandwidth(
 	return result;
 }
 
+enum dc_status dce110_validate_plane(const struct dc_plane_state *plane_state,
+				     struct dc_caps *caps)
+{
+	if (((plane_state->dst_rect.width * 2) < plane_state->src_rect.width) ||
+	    ((plane_state->dst_rect.height * 2) < plane_state->src_rect.height))
+		return DC_FAIL_SURFACE_VALIDATE;
+
+	return DC_OK;
+}
+
 static bool dce110_validate_surface_sets(
 		struct dc_state *context)
 {
@@ -869,6 +879,13 @@ static bool dce110_validate_surface_sets(
 					plane->src_rect.height > 1080))
 					return false;
 
+				/* we don't have the logic to support underlay
+				 * only yet so block the use case where we get
+				 * NV12 plane as top layer
+				 */
+				if (j == 0)
+					return false;
+
 				/* irrespective of plane format,
 				 * stream should be RGB encoded
 				 */
@@ -1021,6 +1038,7 @@ static const struct resource_funcs dce110_res_pool_funcs = {
 	.link_enc_create = dce110_link_encoder_create,
 	.validate_guaranteed = dce110_validate_guaranteed,
 	.validate_bandwidth = dce110_validate_bandwidth,
+	.validate_plane = dce110_validate_plane,
 	.acquire_idle_pipe_for_layer = dce110_acquire_underlay,
 	.add_stream_to_ctx = dce110_add_stream_to_ctx,
 	.validate_global = dce110_validate_global
@@ -1156,6 +1174,7 @@ static bool construct(
 	dc->caps.max_downscale_ratio = 150;
 	dc->caps.i2c_speed_in_khz = 100;
 	dc->caps.max_cursor_size = 128;
+	dc->caps.is_apu = true;
 
 	/*************************************************
 	 *  Create resources                             *
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
index 4befce6cd87a..25ca72139e5f 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c
@@ -1224,26 +1224,46 @@ void dce110_timing_generator_setup_global_swap_lock(
 
 	/* This pipe will belong to GSL Group zero. */
 	set_reg_field_value(value,
-			1,
-			DCP_GSL_CONTROL,
-			DCP_GSL0_EN);
+			    1,
+			    DCP_GSL_CONTROL,
+			    DCP_GSL0_EN);
 
 	set_reg_field_value(value,
-			gsl_params->gsl_master == tg->inst,
-			DCP_GSL_CONTROL,
-			DCP_GSL_MASTER_EN);
+			    gsl_params->gsl_master == tg->inst,
+			    DCP_GSL_CONTROL,
+			    DCP_GSL_MASTER_EN);
 
 	set_reg_field_value(value,
-			HFLIP_READY_DELAY,
-			DCP_GSL_CONTROL,
-			DCP_GSL_HSYNC_FLIP_FORCE_DELAY);
+			    HFLIP_READY_DELAY,
+			    DCP_GSL_CONTROL,
+			    DCP_GSL_HSYNC_FLIP_FORCE_DELAY);
 
 	/* Keep signal low (pending high) during 6 lines.
 	 * Also defines minimum interval before re-checking signal. */
 	set_reg_field_value(value,
-			HFLIP_CHECK_DELAY,
-			DCP_GSL_CONTROL,
-			DCP_GSL_HSYNC_FLIP_CHECK_DELAY);
+			    HFLIP_CHECK_DELAY,
+			    DCP_GSL_CONTROL,
+			    DCP_GSL_HSYNC_FLIP_CHECK_DELAY);
+
+	dm_write_reg(tg->ctx, CRTC_REG(mmDCP_GSL_CONTROL), value);
+	value = 0;
+
+	set_reg_field_value(value,
+			    gsl_params->gsl_master,
+			    DCIO_GSL0_CNTL,
+			    DCIO_GSL0_VSYNC_SEL);
+
+	set_reg_field_value(value,
+			    0,
+			    DCIO_GSL0_CNTL,
+			    DCIO_GSL0_TIMING_SYNC_SEL);
+
+	set_reg_field_value(value,
+			    0,
+			    DCIO_GSL0_CNTL,
+			    DCIO_GSL0_GLOBAL_UNLOCK_SEL);
+
+	dm_write_reg(tg->ctx, CRTC_REG(mmDCIO_GSL0_CNTL), value);
 
 
 	{
@@ -1253,38 +1273,38 @@ void dce110_timing_generator_setup_global_swap_lock(
 				CRTC_REG(mmCRTC_V_TOTAL));
 
 		set_reg_field_value(value,
-				0,/* DCP_GSL_PURPOSE_SURFACE_FLIP */
-				DCP_GSL_CONTROL,
-				DCP_GSL_SYNC_SOURCE);
+				    0,/* DCP_GSL_PURPOSE_SURFACE_FLIP */
+				    DCP_GSL_CONTROL,
+				    DCP_GSL_SYNC_SOURCE);
 
 		/* Checkpoint relative to end of frame */
 		check_point = get_reg_field_value(value_crtc_vtotal,
-				CRTC_V_TOTAL,
-				CRTC_V_TOTAL);
+						  CRTC_V_TOTAL,
+						  CRTC_V_TOTAL);
 
 		dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_GSL_WINDOW), 0);
 	}
 
 	set_reg_field_value(value,
-			1,
-			DCP_GSL_CONTROL,
-			DCP_GSL_DELAY_SURFACE_UPDATE_PENDING);
+			    1,
+			    DCP_GSL_CONTROL,
+			    DCP_GSL_DELAY_SURFACE_UPDATE_PENDING);
 
 	dm_write_reg(tg->ctx, address, value);
 
 	/********************************************************************/
 	address = CRTC_REG(mmCRTC_GSL_CONTROL);
 
-	value = 0;
+	value = dm_read_reg(tg->ctx, address);
 	set_reg_field_value(value,
-			check_point - FLIP_READY_BACK_LOOKUP,
-			CRTC_GSL_CONTROL,
-			CRTC_GSL_CHECK_LINE_NUM);
+			    check_point - FLIP_READY_BACK_LOOKUP,
+			    CRTC_GSL_CONTROL,
+			    CRTC_GSL_CHECK_LINE_NUM);
 
 	set_reg_field_value(value,
-			VFLIP_READY_DELAY,
-			CRTC_GSL_CONTROL,
-			CRTC_GSL_FORCE_DELAY);
+			    VFLIP_READY_DELAY,
+			    CRTC_GSL_CONTROL,
+			    CRTC_GSL_FORCE_DELAY);
 
 	dm_write_reg(tg->ctx, address, value);
 }
@@ -1555,6 +1575,138 @@ void dce110_timing_generator_enable_reset_trigger(
 	dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL), value);
 }
 
+void dce110_timing_generator_enable_crtc_reset(
+		struct timing_generator *tg,
+		int source_tg_inst,
+		struct crtc_trigger_info *crtc_tp)
+{
+	uint32_t value = 0;
+	uint32_t rising_edge = 0;
+	uint32_t falling_edge = 0;
+	struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg);
+
+	/* Setup trigger edge */
+	switch (crtc_tp->event) {
+	case CRTC_EVENT_VSYNC_RISING:
+			rising_edge = 1;
+			break;
+
+	case CRTC_EVENT_VSYNC_FALLING:
+		falling_edge = 1;
+		break;
+	}
+
+	value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_TRIGB_CNTL));
+
+	set_reg_field_value(value,
+			    source_tg_inst,
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_SOURCE_SELECT);
+
+	set_reg_field_value(value,
+			    TRIGGER_POLARITY_SELECT_LOGIC_ZERO,
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_POLARITY_SELECT);
+
+	set_reg_field_value(value,
+			    rising_edge,
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_RISING_EDGE_DETECT_CNTL);
+
+	set_reg_field_value(value,
+			    falling_edge,
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_FALLING_EDGE_DETECT_CNTL);
+
+	set_reg_field_value(value,
+			    1, /* clear trigger status */
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_CLEAR);
+
+	dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_TRIGB_CNTL), value);
+
+	/**************************************************************/
+
+	switch (crtc_tp->delay) {
+	case TRIGGER_DELAY_NEXT_LINE:
+		value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL));
+
+		set_reg_field_value(value,
+				    0, /* force H count to H_TOTAL and V count to V_TOTAL */
+				    CRTC_FORCE_COUNT_NOW_CNTL,
+				    CRTC_FORCE_COUNT_NOW_MODE);
+
+		set_reg_field_value(value,
+				    0, /* TriggerB - we never use TriggerA */
+				    CRTC_FORCE_COUNT_NOW_CNTL,
+				    CRTC_FORCE_COUNT_NOW_TRIG_SEL);
+
+		set_reg_field_value(value,
+				    1, /* clear trigger status */
+				    CRTC_FORCE_COUNT_NOW_CNTL,
+				    CRTC_FORCE_COUNT_NOW_CLEAR);
+
+		dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL), value);
+
+		value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_VERT_SYNC_CONTROL));
+
+		set_reg_field_value(value,
+				    1,
+				    CRTC_VERT_SYNC_CONTROL,
+				    CRTC_FORCE_VSYNC_NEXT_LINE_CLEAR);
+
+		set_reg_field_value(value,
+				    2,
+				    CRTC_VERT_SYNC_CONTROL,
+				    CRTC_AUTO_FORCE_VSYNC_MODE);
+
+		break;
+
+	case TRIGGER_DELAY_NEXT_PIXEL:
+		value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_VERT_SYNC_CONTROL));
+
+		set_reg_field_value(value,
+				    1,
+				    CRTC_VERT_SYNC_CONTROL,
+				    CRTC_FORCE_VSYNC_NEXT_LINE_CLEAR);
+
+		set_reg_field_value(value,
+				    0,
+				    CRTC_VERT_SYNC_CONTROL,
+				    CRTC_AUTO_FORCE_VSYNC_MODE);
+
+		dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_VERT_SYNC_CONTROL), value);
+
+		value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL));
+
+		set_reg_field_value(value,
+				    2, /* force H count to H_TOTAL and V count to V_TOTAL */
+				    CRTC_FORCE_COUNT_NOW_CNTL,
+				    CRTC_FORCE_COUNT_NOW_MODE);
+
+		set_reg_field_value(value,
+				    1, /* TriggerB - we never use TriggerA */
+				    CRTC_FORCE_COUNT_NOW_CNTL,
+				    CRTC_FORCE_COUNT_NOW_TRIG_SEL);
+
+		set_reg_field_value(value,
+				    1, /* clear trigger status */
+				    CRTC_FORCE_COUNT_NOW_CNTL,
+				    CRTC_FORCE_COUNT_NOW_CLEAR);
+
+		dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL), value);
+		break;
+	}
+
+	value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_MASTER_UPDATE_MODE));
+
+	set_reg_field_value(value,
+			    2,
+			    CRTC_MASTER_UPDATE_MODE,
+			    MASTER_UPDATE_MODE);
+
+	dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_MASTER_UPDATE_MODE), value);
+}
 void dce110_timing_generator_disable_reset_trigger(
 	struct timing_generator *tg)
 {
@@ -1564,34 +1716,48 @@ void dce110_timing_generator_disable_reset_trigger(
 	value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL));
 
 	set_reg_field_value(value,
-			0, /* force counter now mode is disabled */
-			CRTC_FORCE_COUNT_NOW_CNTL,
-			CRTC_FORCE_COUNT_NOW_MODE);
+			    0, /* force counter now mode is disabled */
+			    CRTC_FORCE_COUNT_NOW_CNTL,
+			    CRTC_FORCE_COUNT_NOW_MODE);
 
 	set_reg_field_value(value,
-			1, /* clear trigger status */
-			CRTC_FORCE_COUNT_NOW_CNTL,
-			CRTC_FORCE_COUNT_NOW_CLEAR);
+			    1, /* clear trigger status */
+			    CRTC_FORCE_COUNT_NOW_CNTL,
+			    CRTC_FORCE_COUNT_NOW_CLEAR);
 
 	dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL), value);
 
+	value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_VERT_SYNC_CONTROL));
+
+	set_reg_field_value(value,
+			    1,
+			    CRTC_VERT_SYNC_CONTROL,
+			    CRTC_FORCE_VSYNC_NEXT_LINE_CLEAR);
+
+	set_reg_field_value(value,
+			    0,
+			    CRTC_VERT_SYNC_CONTROL,
+			    CRTC_AUTO_FORCE_VSYNC_MODE);
+
+	dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_VERT_SYNC_CONTROL), value);
+
 	/********************************************************************/
 	value = dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_TRIGB_CNTL));
 
 	set_reg_field_value(value,
-			TRIGGER_SOURCE_SELECT_LOGIC_ZERO,
-			CRTC_TRIGB_CNTL,
-			CRTC_TRIGB_SOURCE_SELECT);
+			    TRIGGER_SOURCE_SELECT_LOGIC_ZERO,
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_SOURCE_SELECT);
 
 	set_reg_field_value(value,
-			TRIGGER_POLARITY_SELECT_LOGIC_ZERO,
-			CRTC_TRIGB_CNTL,
-			CRTC_TRIGB_POLARITY_SELECT);
+			    TRIGGER_POLARITY_SELECT_LOGIC_ZERO,
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_POLARITY_SELECT);
 
 	set_reg_field_value(value,
-			1, /* clear trigger status */
-			CRTC_TRIGB_CNTL,
-			CRTC_TRIGB_CLEAR);
+			    1, /* clear trigger status */
+			    CRTC_TRIGB_CNTL,
+			    CRTC_TRIGB_CLEAR);
 
 	dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_TRIGB_CNTL), value);
 }
@@ -1611,10 +1777,16 @@ bool dce110_timing_generator_did_triggered_reset_occur(
 	struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg);
 	uint32_t value = dm_read_reg(tg->ctx,
 			CRTC_REG(mmCRTC_FORCE_COUNT_NOW_CNTL));
-
-	return get_reg_field_value(value,
-			CRTC_FORCE_COUNT_NOW_CNTL,
-			CRTC_FORCE_COUNT_NOW_OCCURRED) != 0;
+	uint32_t value1 = dm_read_reg(tg->ctx,
+			CRTC_REG(mmCRTC_VERT_SYNC_CONTROL));
+	bool force = get_reg_field_value(value,
+					 CRTC_FORCE_COUNT_NOW_CNTL,
+					 CRTC_FORCE_COUNT_NOW_OCCURRED) != 0;
+	bool vert_sync = get_reg_field_value(value1,
+					     CRTC_VERT_SYNC_CONTROL,
+					     CRTC_FORCE_VSYNC_NEXT_LINE_OCCURRED) != 0;
+
+	return (force || vert_sync);
 }
 
 /**
@@ -1928,6 +2100,7 @@ static const struct timing_generator_funcs dce110_tg_funcs = {
 		.setup_global_swap_lock =
 				dce110_timing_generator_setup_global_swap_lock,
 		.enable_reset_trigger = dce110_timing_generator_enable_reset_trigger,
+		.enable_crtc_reset = dce110_timing_generator_enable_crtc_reset,
 		.disable_reset_trigger = dce110_timing_generator_disable_reset_trigger,
 		.tear_down_global_swap_lock =
 				dce110_timing_generator_tear_down_global_swap_lock,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h
index 82737dea6984..232747c7c60b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.h
@@ -174,6 +174,12 @@ void dce110_timing_generator_setup_global_swap_lock(
 void dce110_timing_generator_tear_down_global_swap_lock(
 	struct timing_generator *tg);
 
+/* Reset crtc position on master VSync */
+void dce110_timing_generator_enable_crtc_reset(
+	struct timing_generator *tg,
+	int source,
+	struct crtc_trigger_info *crtc_tp);
+
 /* Reset slave controllers on master VSync */
 void dce110_timing_generator_enable_reset_trigger(
 	struct timing_generator *tg,
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
index 663e0a047a4b..98d9cd0109e1 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
@@ -1103,6 +1103,8 @@ static bool construct(
 	dc->caps.max_downscale_ratio = 200;
 	dc->caps.i2c_speed_in_khz = 100;
 	dc->caps.max_cursor_size = 128;
+	dc->caps.dual_link_dvi = true;
+
 
 	/*************************************************
 	 *  Create resources                             *
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c
index 1a0b54d6034e..75d029742f96 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c
@@ -31,9 +31,9 @@
 
 #include "dce110/dce110_hw_sequencer.h"
 
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15ip.h"
 #include "reg_helper.h"
 
 #define CTX \
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
index 5c48c22d9d98..5aab01db28ee 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
@@ -54,10 +54,10 @@
 #include "dce/dce_abm.h"
 #include "dce/dce_dmcu.h"
 
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/soc15ip.h"
-#include "vega10/NBIO/nbio_6_1_offset.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15ip.h"
+#include "nbio/nbio_6_1_offset.h"
 #include "reg_helper.h"
 
 #include "dce100/dce100_resource.h"
@@ -835,6 +835,8 @@ static bool construct(
 	dc->caps.max_downscale_ratio = 200;
 	dc->caps.i2c_speed_in_khz = 100;
 	dc->caps.max_cursor_size = 128;
+	dc->caps.dual_link_dvi = true;
+
 	dc->debug = debug_defaults;
 
 	/*************************************************
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
index 2502182d5e82..0aa60e5727e0 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c
@@ -25,9 +25,9 @@
 
 #include "dm_services.h"
 
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15ip.h"
 
 #include "dc_types.h"
 #include "dc_bios_types.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
index 9c18efd3446f..25d7eb1567ae 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
@@ -793,6 +793,7 @@ static bool dce80_construct(
 	dc->caps.max_downscale_ratio = 200;
 	dc->caps.i2c_speed_in_khz = 40;
 	dc->caps.max_cursor_size = 128;
+	dc->caps.dual_link_dvi = true;
 
 	/*************************************************
 	 *  Create resources                             *
@@ -957,6 +958,7 @@ static bool dce81_construct(
 	dc->caps.max_downscale_ratio = 200;
 	dc->caps.i2c_speed_in_khz = 40;
 	dc->caps.max_cursor_size = 128;
+	dc->caps.is_apu = true;
 
 	/*************************************************
 	 *  Create resources                             *
@@ -1121,6 +1123,7 @@ static bool dce83_construct(
 	dc->caps.max_downscale_ratio = 200;
 	dc->caps.i2c_speed_in_khz = 40;
 	dc->caps.max_cursor_size = 128;
+	dc->caps.is_apu = true;
 
 	/*************************************************
 	 *  Create resources                             *
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
index f565a6042970..5469bdfe19f3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/Makefile
@@ -23,9 +23,10 @@
 # Makefile for DCN.
 
 DCN10 = dcn10_resource.o dcn10_ipp.o dcn10_hw_sequencer.o \
-		dcn10_dpp.o dcn10_opp.o dcn10_timing_generator.o \
+		dcn10_dpp.o dcn10_opp.o dcn10_optc.o \
 		dcn10_hubp.o dcn10_mpc.o \
-		dcn10_dpp_dscl.o dcn10_dpp_cm.o dcn10_cm_common.o
+		dcn10_dpp_dscl.o dcn10_dpp_cm.o dcn10_cm_common.o \
+		dcn10_hubbub.o
 
 AMD_DAL_DCN10 = $(addprefix $(AMDDALPATH)/dc/dcn10/,$(DCN10))
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
index 7f579cb19f4b..53ba3600ee6a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
@@ -22,11 +22,12 @@
  * Authors: AMD
  *
  */
-
+#include "dc.h"
 #include "reg_helper.h"
 #include "dcn10_dpp.h"
 
 #include "dcn10_cm_common.h"
+#include "custom_float.h"
 
 #define REG(reg) reg
 
@@ -121,3 +122,294 @@ void cm_helper_program_xfer_func(
 	}
 
 }
+
+
+
+bool cm_helper_convert_to_custom_float(
+		struct pwl_result_data *rgb_resulted,
+		struct curve_points *arr_points,
+		uint32_t hw_points_num,
+		bool fixpoint)
+{
+	struct custom_float_format fmt;
+
+	struct pwl_result_data *rgb = rgb_resulted;
+
+	uint32_t i = 0;
+
+	fmt.exponenta_bits = 6;
+	fmt.mantissa_bits = 12;
+	fmt.sign = false;
+
+	if (!convert_to_custom_float_format(arr_points[0].x, &fmt,
+					    &arr_points[0].custom_float_x)) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	if (!convert_to_custom_float_format(arr_points[0].offset, &fmt,
+					    &arr_points[0].custom_float_offset)) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	if (!convert_to_custom_float_format(arr_points[0].slope, &fmt,
+					    &arr_points[0].custom_float_slope)) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	fmt.mantissa_bits = 10;
+	fmt.sign = false;
+
+	if (!convert_to_custom_float_format(arr_points[1].x, &fmt,
+					    &arr_points[1].custom_float_x)) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	if (fixpoint == true)
+		arr_points[1].custom_float_y = dal_fixed31_32_clamp_u0d14(arr_points[1].y);
+	else if (!convert_to_custom_float_format(arr_points[1].y, &fmt,
+		&arr_points[1].custom_float_y)) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	if (!convert_to_custom_float_format(arr_points[1].slope, &fmt,
+					    &arr_points[1].custom_float_slope)) {
+		BREAK_TO_DEBUGGER();
+		return false;
+	}
+
+	if (hw_points_num == 0 || rgb_resulted == NULL || fixpoint == true)
+		return true;
+
+	fmt.mantissa_bits = 12;
+	fmt.sign = true;
+
+	while (i != hw_points_num) {
+		if (!convert_to_custom_float_format(rgb->red, &fmt,
+						    &rgb->red_reg)) {
+			BREAK_TO_DEBUGGER();
+			return false;
+		}
+
+		if (!convert_to_custom_float_format(rgb->green, &fmt,
+						    &rgb->green_reg)) {
+			BREAK_TO_DEBUGGER();
+			return false;
+		}
+
+		if (!convert_to_custom_float_format(rgb->blue, &fmt,
+						    &rgb->blue_reg)) {
+			BREAK_TO_DEBUGGER();
+			return false;
+		}
+
+		if (!convert_to_custom_float_format(rgb->delta_red, &fmt,
+						    &rgb->delta_red_reg)) {
+			BREAK_TO_DEBUGGER();
+			return false;
+		}
+
+		if (!convert_to_custom_float_format(rgb->delta_green, &fmt,
+						    &rgb->delta_green_reg)) {
+			BREAK_TO_DEBUGGER();
+			return false;
+		}
+
+		if (!convert_to_custom_float_format(rgb->delta_blue, &fmt,
+						    &rgb->delta_blue_reg)) {
+			BREAK_TO_DEBUGGER();
+			return false;
+		}
+
+		++rgb;
+		++i;
+	}
+
+	return true;
+}
+
+
+#define MAX_REGIONS_NUMBER 34
+#define MAX_LOW_POINT      25
+#define NUMBER_SEGMENTS    32
+
+bool cm_helper_translate_curve_to_hw_format(
+				const struct dc_transfer_func *output_tf,
+				struct pwl_params *lut_params, bool fixpoint)
+{
+	struct curve_points *arr_points;
+	struct pwl_result_data *rgb_resulted;
+	struct pwl_result_data *rgb;
+	struct pwl_result_data *rgb_plus_1;
+	struct fixed31_32 y_r;
+	struct fixed31_32 y_g;
+	struct fixed31_32 y_b;
+	struct fixed31_32 y1_min;
+	struct fixed31_32 y3_max;
+
+	int32_t segment_start, segment_end;
+	int32_t i;
+	uint32_t j, k, seg_distr[MAX_REGIONS_NUMBER], increment, start_index, hw_points;
+
+	if (output_tf == NULL || lut_params == NULL || output_tf->type == TF_TYPE_BYPASS)
+		return false;
+
+	PERF_TRACE();
+
+	arr_points = lut_params->arr_points;
+	rgb_resulted = lut_params->rgb_resulted;
+	hw_points = 0;
+
+	memset(lut_params, 0, sizeof(struct pwl_params));
+	memset(seg_distr, 0, sizeof(seg_distr));
+
+	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
+		/* 32 segments
+		 * segments are from 2^-25 to 2^7
+		 */
+		for (i = 0; i < 32 ; i++)
+			seg_distr[i] = 3;
+
+		segment_start = -25;
+		segment_end   = 7;
+	} else {
+		/* 10 segments
+		 * segment is from 2^-10 to 2^0
+		 * There are less than 256 points, for optimization
+		 */
+		seg_distr[0] = 3;
+		seg_distr[1] = 4;
+		seg_distr[2] = 4;
+		seg_distr[3] = 4;
+		seg_distr[4] = 4;
+		seg_distr[5] = 4;
+		seg_distr[6] = 4;
+		seg_distr[7] = 4;
+		seg_distr[8] = 5;
+		seg_distr[9] = 5;
+
+		segment_start = -10;
+		segment_end = 0;
+	}
+
+	for (i = segment_end - segment_start; i < MAX_REGIONS_NUMBER ; i++)
+		seg_distr[i] = -1;
+
+	for (k = 0; k < MAX_REGIONS_NUMBER; k++) {
+		if (seg_distr[k] != -1)
+			hw_points += (1 << seg_distr[k]);
+	}
+
+	j = 0;
+	for (k = 0; k < (segment_end - segment_start); k++) {
+		increment = NUMBER_SEGMENTS / (1 << seg_distr[k]);
+		start_index = (segment_start + k + MAX_LOW_POINT) * NUMBER_SEGMENTS;
+		for (i = start_index; i < start_index + NUMBER_SEGMENTS; i += increment) {
+			if (j == hw_points - 1)
+				break;
+			rgb_resulted[j].red = output_tf->tf_pts.red[i];
+			rgb_resulted[j].green = output_tf->tf_pts.green[i];
+			rgb_resulted[j].blue = output_tf->tf_pts.blue[i];
+			j++;
+		}
+	}
+
+	/* last point */
+	start_index = (segment_end + MAX_LOW_POINT) * NUMBER_SEGMENTS;
+	rgb_resulted[hw_points - 1].red = output_tf->tf_pts.red[start_index];
+	rgb_resulted[hw_points - 1].green = output_tf->tf_pts.green[start_index];
+	rgb_resulted[hw_points - 1].blue = output_tf->tf_pts.blue[start_index];
+
+	arr_points[0].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
+					     dal_fixed31_32_from_int(segment_start));
+	arr_points[1].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
+					     dal_fixed31_32_from_int(segment_end));
+
+	y_r = rgb_resulted[0].red;
+	y_g = rgb_resulted[0].green;
+	y_b = rgb_resulted[0].blue;
+
+	y1_min = dal_fixed31_32_min(y_r, dal_fixed31_32_min(y_g, y_b));
+
+	arr_points[0].y = y1_min;
+	arr_points[0].slope = dal_fixed31_32_div(arr_points[0].y, arr_points[0].x);
+	y_r = rgb_resulted[hw_points - 1].red;
+	y_g = rgb_resulted[hw_points - 1].green;
+	y_b = rgb_resulted[hw_points - 1].blue;
+
+	/* see comment above, m_arrPoints[1].y should be the Y value for the
+	 * region end (m_numOfHwPoints), not last HW point(m_numOfHwPoints - 1)
+	 */
+	y3_max = dal_fixed31_32_max(y_r, dal_fixed31_32_max(y_g, y_b));
+
+	arr_points[1].y = y3_max;
+
+	arr_points[1].slope = dal_fixed31_32_zero;
+
+	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
+		/* for PQ, we want to have a straight line from last HW X point,
+		 * and the slope to be such that we hit 1.0 at 10000 nits.
+		 */
+		const struct fixed31_32 end_value =
+				dal_fixed31_32_from_int(125);
+
+		arr_points[1].slope = dal_fixed31_32_div(
+			dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
+			dal_fixed31_32_sub(end_value, arr_points[1].x));
+	}
+
+	lut_params->hw_points_num = hw_points;
+
+	i = 1;
+	for (k = 0; k < MAX_REGIONS_NUMBER && i < MAX_REGIONS_NUMBER; k++) {
+		if (seg_distr[k] != -1) {
+			lut_params->arr_curve_points[k].segments_num =
+					seg_distr[k];
+			lut_params->arr_curve_points[i].offset =
+					lut_params->arr_curve_points[k].offset + (1 << seg_distr[k]);
+		}
+		i++;
+	}
+
+	if (seg_distr[k] != -1)
+		lut_params->arr_curve_points[k].segments_num = seg_distr[k];
+
+	rgb = rgb_resulted;
+	rgb_plus_1 = rgb_resulted + 1;
+
+	i = 1;
+	while (i != hw_points + 1) {
+		if (dal_fixed31_32_lt(rgb_plus_1->red, rgb->red))
+			rgb_plus_1->red = rgb->red;
+		if (dal_fixed31_32_lt(rgb_plus_1->green, rgb->green))
+			rgb_plus_1->green = rgb->green;
+		if (dal_fixed31_32_lt(rgb_plus_1->blue, rgb->blue))
+			rgb_plus_1->blue = rgb->blue;
+
+		rgb->delta_red   = dal_fixed31_32_sub(rgb_plus_1->red,   rgb->red);
+		rgb->delta_green = dal_fixed31_32_sub(rgb_plus_1->green, rgb->green);
+		rgb->delta_blue  = dal_fixed31_32_sub(rgb_plus_1->blue,  rgb->blue);
+
+		if (fixpoint == true) {
+			rgb->delta_red_reg   = dal_fixed31_32_clamp_u0d10(rgb->delta_red);
+			rgb->delta_green_reg = dal_fixed31_32_clamp_u0d10(rgb->delta_green);
+			rgb->delta_blue_reg  = dal_fixed31_32_clamp_u0d10(rgb->delta_blue);
+			rgb->red_reg         = dal_fixed31_32_clamp_u0d14(rgb->red);
+			rgb->green_reg       = dal_fixed31_32_clamp_u0d14(rgb->green);
+			rgb->blue_reg        = dal_fixed31_32_clamp_u0d14(rgb->blue);
+		}
+
+		++rgb_plus_1;
+		++rgb;
+		++i;
+	}
+	cm_helper_convert_to_custom_float(rgb_resulted,
+						lut_params->arr_points,
+						hw_points, fixpoint);
+
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h
index 64836dcf21f2..64e476b83bcb 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.h
@@ -96,4 +96,14 @@ void cm_helper_program_xfer_func(
 		const struct pwl_params *params,
 		const struct xfer_func_reg *reg);
 
+bool cm_helper_convert_to_custom_float(
+		struct pwl_result_data *rgb_resulted,
+		struct curve_points *arr_points,
+		uint32_t hw_points_num,
+		bool fixpoint);
+
+bool cm_helper_translate_curve_to_hw_format(
+		const struct dc_transfer_func *output_tf,
+		struct pwl_params *lut_params, bool fixpoint);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
index a9d55d0dd69e..f2a08b156cf0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
@@ -177,37 +177,17 @@ void dpp_reset(struct dpp *dpp_base)
 	dpp->filter_h = NULL;
 	dpp->filter_v = NULL;
 
-	/* set boundary mode to 0 */
-	REG_SET(DSCL_CONTROL, 0, SCL_BOUNDARY_MODE, 0);
+	memset(&dpp->scl_data, 0, sizeof(dpp->scl_data));
+	memset(&dpp->pwl_data, 0, sizeof(dpp->pwl_data));
 }
 
 
 
 static void dpp1_cm_set_regamma_pwl(
-	struct dpp *dpp_base, const struct pwl_params *params)
-{
-	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
-
-	dpp1_cm_power_on_regamma_lut(dpp_base, true);
-	dpp1_cm_configure_regamma_lut(dpp_base, dpp->is_write_to_ram_a_safe);
-
-	if (dpp->is_write_to_ram_a_safe)
-		dpp1_cm_program_regamma_luta_settings(dpp_base, params);
-	else
-		dpp1_cm_program_regamma_lutb_settings(dpp_base, params);
-
-	dpp1_cm_program_regamma_lut(
-			dpp_base, params->rgb_resulted, params->hw_points_num);
-}
-
-static void dpp1_cm_set_regamma_mode(
-	struct dpp *dpp_base,
-	enum opp_regamma mode)
+	struct dpp *dpp_base, const struct pwl_params *params, enum opp_regamma mode)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 	uint32_t re_mode = 0;
-	uint32_t obuf_bypass = 0; /* need for pipe split */
-	uint32_t obuf_hupscale = 0;
 
 	switch (mode) {
 	case OPP_REGAMMA_BYPASS:
@@ -220,17 +200,29 @@ static void dpp1_cm_set_regamma_mode(
 		re_mode = 2;
 		break;
 	case OPP_REGAMMA_USER:
+		re_mode = dpp->is_write_to_ram_a_safe ? 4 : 3;
+		if (memcmp(&dpp->pwl_data, params, sizeof(*params)) == 0)
+			break;
+
+		dpp1_cm_power_on_regamma_lut(dpp_base, true);
+		dpp1_cm_configure_regamma_lut(dpp_base, dpp->is_write_to_ram_a_safe);
+
+		if (dpp->is_write_to_ram_a_safe)
+			dpp1_cm_program_regamma_luta_settings(dpp_base, params);
+		else
+			dpp1_cm_program_regamma_lutb_settings(dpp_base, params);
+
+		dpp1_cm_program_regamma_lut(dpp_base, params->rgb_resulted,
+					    params->hw_points_num);
+		dpp->pwl_data = *params;
+
 		re_mode = dpp->is_write_to_ram_a_safe ? 3 : 4;
 		dpp->is_write_to_ram_a_safe = !dpp->is_write_to_ram_a_safe;
 		break;
 	default:
 		break;
 	}
-
 	REG_SET(CM_RGAM_CONTROL, 0, CM_RGAM_LUT_MODE, re_mode);
-	REG_UPDATE_2(OBUF_CONTROL,
-			OBUF_BYPASS, obuf_bypass,
-			OBUF_H_2X_UPSCALE_EN, obuf_hupscale);
 }
 
 static void dpp1_setup_format_flags(enum surface_pixel_format input_format,\
@@ -263,8 +255,10 @@ static void dpp1_set_degamma_format_float(
 
 void dpp1_cnv_setup (
 		struct dpp *dpp_base,
-		enum surface_pixel_format input_format,
-		enum expansion_mode mode)
+		enum surface_pixel_format format,
+		enum expansion_mode mode,
+		struct csc_transform input_csc_color_matrix,
+		enum dc_color_space input_color_space)
 {
 	uint32_t pixel_format;
 	uint32_t alpha_en;
@@ -274,8 +268,10 @@ void dpp1_cnv_setup (
 	bool is_float;
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 	bool force_disable_cursor = false;
+	struct out_csc_color_matrix tbl_entry;
+	int i = 0;
 
-	dpp1_setup_format_flags(input_format, &fmt);
+	dpp1_setup_format_flags(format, &fmt);
 	alpha_en = 1;
 	pixel_format = 0;
 	color_space = COLOR_SPACE_SRGB;
@@ -305,7 +301,7 @@ void dpp1_cnv_setup (
 
 	dpp1_set_degamma_format_float(dpp_base, is_float);
 
-	switch (input_format) {
+	switch (format) {
 	case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
 		pixel_format = 1;
 		break;
@@ -361,7 +357,23 @@ void dpp1_cnv_setup (
 			CNVC_SURFACE_PIXEL_FORMAT, pixel_format);
 	REG_UPDATE(FORMAT_CONTROL, FORMAT_CONTROL__ALPHA_EN, alpha_en);
 
-	dpp1_program_input_csc(dpp_base, color_space, select);
+	// if input adjustments exist, program icsc with those values
+
+	if (input_csc_color_matrix.enable_adjustment
+				== true) {
+		for (i = 0; i < 12; i++)
+			tbl_entry.regval[i] = input_csc_color_matrix.matrix[i];
+
+		tbl_entry.color_space = input_color_space;
+
+		if (color_space >= COLOR_SPACE_YCBCR601)
+			select = INPUT_CSC_SELECT_ICSC;
+		else
+			select = INPUT_CSC_SELECT_BYPASS;
+
+		dpp1_program_input_csc(dpp_base, color_space, select, &tbl_entry);
+	} else
+		dpp1_program_input_csc(dpp_base, color_space, select, NULL);
 
 	if (force_disable_cursor) {
 		REG_UPDATE(CURSOR_CONTROL,
@@ -373,10 +385,9 @@ void dpp1_cnv_setup (
 
 void dpp1_set_cursor_attributes(
 		struct dpp *dpp_base,
-		const struct dc_cursor_attributes *attr)
+		enum dc_cursor_color_format color_format)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
-	enum dc_cursor_color_format color_format = attr->color_format;
 
 	REG_UPDATE_2(CURSOR0_CONTROL,
 			CUR0_MODE, color_format,
@@ -389,13 +400,6 @@ void dpp1_set_cursor_attributes(
 		REG_UPDATE(CURSOR0_COLOR1,
 				CUR0_COLOR1, 0xFFFFFFFF);
 	}
-
-	/* TODO: Fixed vs float */
-
-	REG_UPDATE_3(FORMAT_CONTROL,
-				CNVC_BYPASS, 0,
-				FORMAT_CONTROL__ALPHA_EN, 1,
-				FORMAT_EXPANSION_MODE, 0);
 }
 
 
@@ -425,20 +429,20 @@ static const struct dpp_funcs dcn10_dpp_funcs = {
 		.dpp_set_scaler = dpp1_dscl_set_scaler_manual_scale,
 		.dpp_get_optimal_number_of_taps = dpp_get_optimal_number_of_taps,
 		.dpp_set_gamut_remap = dpp1_cm_set_gamut_remap,
-		.opp_set_csc_adjustment = dpp1_cm_set_output_csc_adjustment,
-		.opp_set_csc_default = dpp1_cm_set_output_csc_default,
-		.opp_power_on_regamma_lut = dpp1_cm_power_on_regamma_lut,
-		.opp_program_regamma_lut = dpp1_cm_program_regamma_lut,
-		.opp_configure_regamma_lut = dpp1_cm_configure_regamma_lut,
-		.opp_program_regamma_lutb_settings = dpp1_cm_program_regamma_lutb_settings,
-		.opp_program_regamma_luta_settings = dpp1_cm_program_regamma_luta_settings,
-		.opp_program_regamma_pwl = dpp1_cm_set_regamma_pwl,
-		.opp_set_regamma_mode = dpp1_cm_set_regamma_mode,
-		.ipp_set_degamma = dpp1_set_degamma,
-		.ipp_program_input_lut		= dpp1_program_input_lut,
-		.ipp_program_degamma_pwl	= dpp1_set_degamma_pwl,
-		.ipp_setup			= dpp1_cnv_setup,
-		.ipp_full_bypass		= dpp1_full_bypass,
+		.dpp_set_csc_adjustment = dpp1_cm_set_output_csc_adjustment,
+		.dpp_set_csc_default = dpp1_cm_set_output_csc_default,
+		.dpp_power_on_regamma_lut = dpp1_cm_power_on_regamma_lut,
+		.dpp_program_regamma_lut = dpp1_cm_program_regamma_lut,
+		.dpp_configure_regamma_lut = dpp1_cm_configure_regamma_lut,
+		.dpp_program_regamma_lutb_settings = dpp1_cm_program_regamma_lutb_settings,
+		.dpp_program_regamma_luta_settings = dpp1_cm_program_regamma_luta_settings,
+		.dpp_program_regamma_pwl = dpp1_cm_set_regamma_pwl,
+		.dpp_program_bias_and_scale = dpp1_program_bias_and_scale,
+		.dpp_set_degamma = dpp1_set_degamma,
+		.dpp_program_input_lut		= dpp1_program_input_lut,
+		.dpp_program_degamma_pwl	= dpp1_set_degamma_pwl,
+		.dpp_setup			= dpp1_cnv_setup,
+		.dpp_full_bypass		= dpp1_full_bypass,
 		.set_cursor_attributes = dpp1_set_cursor_attributes,
 		.set_cursor_position = dpp1_set_cursor_position,
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
index 34daf895f848..f56ee4d08d89 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.h
@@ -54,7 +54,6 @@
 	SRI(LB_MEMORY_CTRL, DSCL, id), \
 	SRI(DSCL_AUTOCAL, DSCL, id), \
 	SRI(SCL_BLACK_OFFSET, DSCL, id), \
-	SRI(DSCL_CONTROL, DSCL, id), \
 	SRI(SCL_TAP_CONTROL, DSCL, id), \
 	SRI(SCL_COEF_RAM_TAP_SELECT, DSCL, id), \
 	SRI(SCL_COEF_RAM_TAP_DATA, DSCL, id), \
@@ -72,7 +71,6 @@
 	SRI(SCL_VERT_FILTER_INIT_BOT_C, DSCL, id), \
 	SRI(RECOUT_START, DSCL, id), \
 	SRI(RECOUT_SIZE, DSCL, id), \
-	SRI(OBUF_CONTROL, DSCL, id), \
 	SRI(CM_ICSC_CONTROL, CM, id), \
 	SRI(CM_ICSC_C11_C12, CM, id), \
 	SRI(CM_ICSC_C33_C34, CM, id), \
@@ -127,6 +125,9 @@
 	SRI(CM_OCSC_CONTROL, CM, id), \
 	SRI(CM_OCSC_C11_C12, CM, id), \
 	SRI(CM_OCSC_C33_C34, CM, id), \
+	SRI(CM_BNS_VALUES_R, CM, id), \
+	SRI(CM_BNS_VALUES_G, CM, id), \
+	SRI(CM_BNS_VALUES_B, CM, id), \
 	SRI(CM_MEM_PWR_CTRL, CM, id), \
 	SRI(CM_RGAM_LUT_DATA, CM, id), \
 	SRI(CM_RGAM_LUT_WRITE_EN_MASK, CM, id),\
@@ -191,7 +192,6 @@
 	TF_SF(DSCL0_DSCL_AUTOCAL, AUTOCAL_PIPE_ID, mask_sh),\
 	TF_SF(DSCL0_SCL_BLACK_OFFSET, SCL_BLACK_OFFSET_RGB_Y, mask_sh),\
 	TF_SF(DSCL0_SCL_BLACK_OFFSET, SCL_BLACK_OFFSET_CBCR, mask_sh),\
-	TF_SF(DSCL0_DSCL_CONTROL, SCL_BOUNDARY_MODE, mask_sh),\
 	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_V_NUM_TAPS, mask_sh),\
 	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_H_NUM_TAPS, mask_sh),\
 	TF_SF(DSCL0_SCL_TAP_CONTROL, SCL_V_NUM_TAPS_C, mask_sh),\
@@ -235,7 +235,6 @@
 	TF_SF(DSCL0_SCL_VERT_FILTER_INIT_BOT_C, SCL_V_INIT_INT_BOT_C, mask_sh),\
 	TF_SF(DSCL0_SCL_MODE, SCL_CHROMA_COEF_MODE, mask_sh),\
 	TF_SF(DSCL0_SCL_MODE, SCL_COEF_RAM_SELECT_CURRENT, mask_sh), \
-	TF_SF(DSCL0_OBUF_CONTROL, OBUF_BYPASS, mask_sh), \
 	TF_SF(CM0_CM_ICSC_CONTROL, CM_ICSC_MODE, mask_sh), \
 	TF_SF(CM0_CM_ICSC_C11_C12, CM_ICSC_C11, mask_sh), \
 	TF_SF(CM0_CM_ICSC_C11_C12, CM_ICSC_C12, mask_sh), \
@@ -329,6 +328,12 @@
 	TF_SF(CM0_CM_OCSC_C11_C12, CM_OCSC_C12, mask_sh), \
 	TF_SF(CM0_CM_OCSC_C33_C34, CM_OCSC_C33, mask_sh), \
 	TF_SF(CM0_CM_OCSC_C33_C34, CM_OCSC_C34, mask_sh), \
+	TF_SF(CM0_CM_BNS_VALUES_R, CM_BNS_BIAS_R, mask_sh), \
+	TF_SF(CM0_CM_BNS_VALUES_G, CM_BNS_BIAS_G, mask_sh), \
+	TF_SF(CM0_CM_BNS_VALUES_B, CM_BNS_BIAS_B, mask_sh), \
+	TF_SF(CM0_CM_BNS_VALUES_R, CM_BNS_SCALE_R, mask_sh), \
+	TF_SF(CM0_CM_BNS_VALUES_G, CM_BNS_SCALE_G, mask_sh), \
+	TF_SF(CM0_CM_BNS_VALUES_B, CM_BNS_SCALE_B, mask_sh), \
 	TF_SF(CM0_CM_MEM_PWR_CTRL, RGAM_MEM_PWR_FORCE, mask_sh), \
 	TF_SF(CM0_CM_RGAM_LUT_DATA, CM_RGAM_LUT_DATA, mask_sh), \
 	TF_SF(CM0_CM_RGAM_LUT_WRITE_EN_MASK, CM_RGAM_LUT_WRITE_EN_MASK, mask_sh), \
@@ -387,7 +392,6 @@
 	TF_SF(CM0_CM_RGAM_RAMA_REGION_32_33, CM_RGAM_RAMA_EXP_REGION33_LUT_OFFSET, mask_sh), \
 	TF_SF(CM0_CM_RGAM_RAMA_REGION_32_33, CM_RGAM_RAMA_EXP_REGION33_NUM_SEGMENTS, mask_sh), \
 	TF_SF(CM0_CM_RGAM_CONTROL, CM_RGAM_LUT_MODE, mask_sh), \
-	TF_SF(DSCL0_OBUF_CONTROL, OBUF_H_2X_UPSCALE_EN, mask_sh), \
 	TF_SF(CM0_CM_IGAM_CONTROL, CM_IGAM_LUT_MODE, mask_sh), \
 	TF_SF(CM0_CM_IGAM_CONTROL, CM_IGAM_LUT_FORMAT_R, mask_sh), \
 	TF_SF(CM0_CM_IGAM_CONTROL, CM_IGAM_LUT_FORMAT_G, mask_sh), \
@@ -431,7 +435,6 @@
 	type AUTOCAL_PIPE_ID; \
 	type SCL_BLACK_OFFSET_RGB_Y; \
 	type SCL_BLACK_OFFSET_CBCR; \
-	type SCL_BOUNDARY_MODE; \
 	type SCL_V_NUM_TAPS; \
 	type SCL_H_NUM_TAPS; \
 	type SCL_V_NUM_TAPS_C; \
@@ -552,8 +555,6 @@
 	type CM_RGAM_RAMA_EXP_REGION33_NUM_SEGMENTS; \
 	type CM_RGAM_LUT_MODE; \
 	type CM_CMOUT_ROUND_TRUNC_MODE; \
-	type OBUF_BYPASS; \
-	type OBUF_H_2X_UPSCALE_EN; \
 	type CM_BLNDGAM_LUT_MODE; \
 	type CM_BLNDGAM_RAMB_EXP_REGION_START_B; \
 	type CM_BLNDGAM_RAMB_EXP_REGION_START_SEGMENT_B; \
@@ -729,8 +730,9 @@
 	type CM_BLNDGAM_RAMA_EXP_REGION33_NUM_SEGMENTS; \
 	type CM_BLNDGAM_LUT_WRITE_EN_MASK; \
 	type CM_BLNDGAM_LUT_WRITE_SEL; \
+	type CM_BLNDGAM_CONFIG_STATUS; \
 	type CM_BLNDGAM_LUT_INDEX; \
-	type CM_BLNDGAM_LUT_DATA; \
+	type BLNDGAM_MEM_PWR_FORCE; \
 	type CM_3DLUT_MODE; \
 	type CM_3DLUT_SIZE; \
 	type CM_3DLUT_INDEX; \
@@ -904,6 +906,7 @@
 	type CM_SHAPER_RAMA_EXP_REGION33_LUT_OFFSET; \
 	type CM_SHAPER_RAMA_EXP_REGION33_NUM_SEGMENTS; \
 	type CM_SHAPER_LUT_WRITE_EN_MASK; \
+	type CM_SHAPER_CONFIG_STATUS; \
 	type CM_SHAPER_LUT_WRITE_SEL; \
 	type CM_SHAPER_LUT_INDEX; \
 	type CM_SHAPER_LUT_DATA; \
@@ -913,6 +916,12 @@
 	type CM_ICSC_C12; \
 	type CM_ICSC_C33; \
 	type CM_ICSC_C34; \
+	type CM_BNS_BIAS_R; \
+	type CM_BNS_BIAS_G; \
+	type CM_BNS_BIAS_B; \
+	type CM_BNS_SCALE_R; \
+	type CM_BNS_SCALE_G; \
+	type CM_BNS_SCALE_B; \
 	type CM_DGAM_RAMB_EXP_REGION_START_B; \
 	type CM_DGAM_RAMB_EXP_REGION_START_SEGMENT_B; \
 	type CM_DGAM_RAMB_EXP_REGION_START_G; \
@@ -998,257 +1007,255 @@
 	type CM_BYPASS; \
 	type FORMAT_CONTROL__ALPHA_EN; \
 	type CUR0_COLOR0; \
-	type CUR0_COLOR1
-
-
+	type CUR0_COLOR1;
 
 struct dcn_dpp_shift {
-	TF_REG_FIELD_LIST(uint8_t);
+	TF_REG_FIELD_LIST(uint8_t)
 };
 
 struct dcn_dpp_mask {
-	TF_REG_FIELD_LIST(uint32_t);
+	TF_REG_FIELD_LIST(uint32_t)
 };
 
-
-
+#define DPP_COMMON_REG_VARIABLE_LIST \
+	uint32_t DSCL_EXT_OVERSCAN_LEFT_RIGHT; \
+	uint32_t DSCL_EXT_OVERSCAN_TOP_BOTTOM; \
+	uint32_t OTG_H_BLANK; \
+	uint32_t OTG_V_BLANK; \
+	uint32_t SCL_MODE; \
+	uint32_t LB_DATA_FORMAT; \
+	uint32_t LB_MEMORY_CTRL; \
+	uint32_t DSCL_AUTOCAL; \
+	uint32_t SCL_BLACK_OFFSET; \
+	uint32_t SCL_TAP_CONTROL; \
+	uint32_t SCL_COEF_RAM_TAP_SELECT; \
+	uint32_t SCL_COEF_RAM_TAP_DATA; \
+	uint32_t DSCL_2TAP_CONTROL; \
+	uint32_t MPC_SIZE; \
+	uint32_t SCL_HORZ_FILTER_SCALE_RATIO; \
+	uint32_t SCL_VERT_FILTER_SCALE_RATIO; \
+	uint32_t SCL_HORZ_FILTER_SCALE_RATIO_C; \
+	uint32_t SCL_VERT_FILTER_SCALE_RATIO_C; \
+	uint32_t SCL_HORZ_FILTER_INIT; \
+	uint32_t SCL_HORZ_FILTER_INIT_C; \
+	uint32_t SCL_VERT_FILTER_INIT; \
+	uint32_t SCL_VERT_FILTER_INIT_BOT; \
+	uint32_t SCL_VERT_FILTER_INIT_C; \
+	uint32_t SCL_VERT_FILTER_INIT_BOT_C; \
+	uint32_t RECOUT_START; \
+	uint32_t RECOUT_SIZE; \
+	uint32_t CM_GAMUT_REMAP_CONTROL; \
+	uint32_t CM_GAMUT_REMAP_C11_C12; \
+	uint32_t CM_GAMUT_REMAP_C33_C34; \
+	uint32_t CM_COMA_C11_C12; \
+	uint32_t CM_COMA_C33_C34; \
+	uint32_t CM_COMB_C11_C12; \
+	uint32_t CM_COMB_C33_C34; \
+	uint32_t CM_OCSC_CONTROL; \
+	uint32_t CM_OCSC_C11_C12; \
+	uint32_t CM_OCSC_C33_C34; \
+	uint32_t CM_MEM_PWR_CTRL; \
+	uint32_t CM_RGAM_LUT_DATA; \
+	uint32_t CM_RGAM_LUT_WRITE_EN_MASK; \
+	uint32_t CM_RGAM_LUT_INDEX; \
+	uint32_t CM_RGAM_RAMB_START_CNTL_B; \
+	uint32_t CM_RGAM_RAMB_START_CNTL_G; \
+	uint32_t CM_RGAM_RAMB_START_CNTL_R; \
+	uint32_t CM_RGAM_RAMB_SLOPE_CNTL_B; \
+	uint32_t CM_RGAM_RAMB_SLOPE_CNTL_G; \
+	uint32_t CM_RGAM_RAMB_SLOPE_CNTL_R; \
+	uint32_t CM_RGAM_RAMB_END_CNTL1_B; \
+	uint32_t CM_RGAM_RAMB_END_CNTL2_B; \
+	uint32_t CM_RGAM_RAMB_END_CNTL1_G; \
+	uint32_t CM_RGAM_RAMB_END_CNTL2_G; \
+	uint32_t CM_RGAM_RAMB_END_CNTL1_R; \
+	uint32_t CM_RGAM_RAMB_END_CNTL2_R; \
+	uint32_t CM_RGAM_RAMB_REGION_0_1; \
+	uint32_t CM_RGAM_RAMB_REGION_32_33; \
+	uint32_t CM_RGAM_RAMA_START_CNTL_B; \
+	uint32_t CM_RGAM_RAMA_START_CNTL_G; \
+	uint32_t CM_RGAM_RAMA_START_CNTL_R; \
+	uint32_t CM_RGAM_RAMA_SLOPE_CNTL_B; \
+	uint32_t CM_RGAM_RAMA_SLOPE_CNTL_G; \
+	uint32_t CM_RGAM_RAMA_SLOPE_CNTL_R; \
+	uint32_t CM_RGAM_RAMA_END_CNTL1_B; \
+	uint32_t CM_RGAM_RAMA_END_CNTL2_B; \
+	uint32_t CM_RGAM_RAMA_END_CNTL1_G; \
+	uint32_t CM_RGAM_RAMA_END_CNTL2_G; \
+	uint32_t CM_RGAM_RAMA_END_CNTL1_R; \
+	uint32_t CM_RGAM_RAMA_END_CNTL2_R; \
+	uint32_t CM_RGAM_RAMA_REGION_0_1; \
+	uint32_t CM_RGAM_RAMA_REGION_32_33; \
+	uint32_t CM_RGAM_CONTROL; \
+	uint32_t CM_CMOUT_CONTROL; \
+	uint32_t CM_BLNDGAM_LUT_WRITE_EN_MASK; \
+	uint32_t CM_BLNDGAM_CONTROL; \
+	uint32_t CM_BLNDGAM_RAMB_START_CNTL_B; \
+	uint32_t CM_BLNDGAM_RAMB_START_CNTL_G; \
+	uint32_t CM_BLNDGAM_RAMB_START_CNTL_R; \
+	uint32_t CM_BLNDGAM_RAMB_SLOPE_CNTL_B; \
+	uint32_t CM_BLNDGAM_RAMB_SLOPE_CNTL_G; \
+	uint32_t CM_BLNDGAM_RAMB_SLOPE_CNTL_R; \
+	uint32_t CM_BLNDGAM_RAMB_END_CNTL1_B; \
+	uint32_t CM_BLNDGAM_RAMB_END_CNTL2_B; \
+	uint32_t CM_BLNDGAM_RAMB_END_CNTL1_G; \
+	uint32_t CM_BLNDGAM_RAMB_END_CNTL2_G; \
+	uint32_t CM_BLNDGAM_RAMB_END_CNTL1_R; \
+	uint32_t CM_BLNDGAM_RAMB_END_CNTL2_R; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_0_1; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_2_3; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_4_5; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_6_7; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_8_9; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_10_11; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_12_13; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_14_15; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_16_17; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_18_19; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_20_21; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_22_23; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_24_25; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_26_27; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_28_29; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_30_31; \
+	uint32_t CM_BLNDGAM_RAMB_REGION_32_33; \
+	uint32_t CM_BLNDGAM_RAMA_START_CNTL_B; \
+	uint32_t CM_BLNDGAM_RAMA_START_CNTL_G; \
+	uint32_t CM_BLNDGAM_RAMA_START_CNTL_R; \
+	uint32_t CM_BLNDGAM_RAMA_SLOPE_CNTL_B; \
+	uint32_t CM_BLNDGAM_RAMA_SLOPE_CNTL_G; \
+	uint32_t CM_BLNDGAM_RAMA_SLOPE_CNTL_R; \
+	uint32_t CM_BLNDGAM_RAMA_END_CNTL1_B; \
+	uint32_t CM_BLNDGAM_RAMA_END_CNTL2_B; \
+	uint32_t CM_BLNDGAM_RAMA_END_CNTL1_G; \
+	uint32_t CM_BLNDGAM_RAMA_END_CNTL2_G; \
+	uint32_t CM_BLNDGAM_RAMA_END_CNTL1_R; \
+	uint32_t CM_BLNDGAM_RAMA_END_CNTL2_R; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_0_1; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_2_3; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_4_5; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_6_7; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_8_9; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_10_11; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_12_13; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_14_15; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_16_17; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_18_19; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_20_21; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_22_23; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_24_25; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_26_27; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_28_29; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_30_31; \
+	uint32_t CM_BLNDGAM_RAMA_REGION_32_33; \
+	uint32_t CM_BLNDGAM_LUT_INDEX; \
+	uint32_t CM_3DLUT_MODE; \
+	uint32_t CM_3DLUT_INDEX; \
+	uint32_t CM_3DLUT_DATA; \
+	uint32_t CM_3DLUT_DATA_30BIT; \
+	uint32_t CM_3DLUT_READ_WRITE_CONTROL; \
+	uint32_t CM_SHAPER_LUT_WRITE_EN_MASK; \
+	uint32_t CM_SHAPER_CONTROL; \
+	uint32_t CM_SHAPER_RAMB_START_CNTL_B; \
+	uint32_t CM_SHAPER_RAMB_START_CNTL_G; \
+	uint32_t CM_SHAPER_RAMB_START_CNTL_R; \
+	uint32_t CM_SHAPER_RAMB_END_CNTL_B; \
+	uint32_t CM_SHAPER_RAMB_END_CNTL_G; \
+	uint32_t CM_SHAPER_RAMB_END_CNTL_R; \
+	uint32_t CM_SHAPER_RAMB_REGION_0_1; \
+	uint32_t CM_SHAPER_RAMB_REGION_2_3; \
+	uint32_t CM_SHAPER_RAMB_REGION_4_5; \
+	uint32_t CM_SHAPER_RAMB_REGION_6_7; \
+	uint32_t CM_SHAPER_RAMB_REGION_8_9; \
+	uint32_t CM_SHAPER_RAMB_REGION_10_11; \
+	uint32_t CM_SHAPER_RAMB_REGION_12_13; \
+	uint32_t CM_SHAPER_RAMB_REGION_14_15; \
+	uint32_t CM_SHAPER_RAMB_REGION_16_17; \
+	uint32_t CM_SHAPER_RAMB_REGION_18_19; \
+	uint32_t CM_SHAPER_RAMB_REGION_20_21; \
+	uint32_t CM_SHAPER_RAMB_REGION_22_23; \
+	uint32_t CM_SHAPER_RAMB_REGION_24_25; \
+	uint32_t CM_SHAPER_RAMB_REGION_26_27; \
+	uint32_t CM_SHAPER_RAMB_REGION_28_29; \
+	uint32_t CM_SHAPER_RAMB_REGION_30_31; \
+	uint32_t CM_SHAPER_RAMB_REGION_32_33; \
+	uint32_t CM_SHAPER_RAMA_START_CNTL_B; \
+	uint32_t CM_SHAPER_RAMA_START_CNTL_G; \
+	uint32_t CM_SHAPER_RAMA_START_CNTL_R; \
+	uint32_t CM_SHAPER_RAMA_END_CNTL_B; \
+	uint32_t CM_SHAPER_RAMA_END_CNTL_G; \
+	uint32_t CM_SHAPER_RAMA_END_CNTL_R; \
+	uint32_t CM_SHAPER_RAMA_REGION_0_1; \
+	uint32_t CM_SHAPER_RAMA_REGION_2_3; \
+	uint32_t CM_SHAPER_RAMA_REGION_4_5; \
+	uint32_t CM_SHAPER_RAMA_REGION_6_7; \
+	uint32_t CM_SHAPER_RAMA_REGION_8_9; \
+	uint32_t CM_SHAPER_RAMA_REGION_10_11; \
+	uint32_t CM_SHAPER_RAMA_REGION_12_13; \
+	uint32_t CM_SHAPER_RAMA_REGION_14_15; \
+	uint32_t CM_SHAPER_RAMA_REGION_16_17; \
+	uint32_t CM_SHAPER_RAMA_REGION_18_19; \
+	uint32_t CM_SHAPER_RAMA_REGION_20_21; \
+	uint32_t CM_SHAPER_RAMA_REGION_22_23; \
+	uint32_t CM_SHAPER_RAMA_REGION_24_25; \
+	uint32_t CM_SHAPER_RAMA_REGION_26_27; \
+	uint32_t CM_SHAPER_RAMA_REGION_28_29; \
+	uint32_t CM_SHAPER_RAMA_REGION_30_31; \
+	uint32_t CM_SHAPER_RAMA_REGION_32_33; \
+	uint32_t CM_SHAPER_LUT_INDEX; \
+	uint32_t CM_SHAPER_LUT_DATA; \
+	uint32_t CM_ICSC_CONTROL; \
+	uint32_t CM_ICSC_C11_C12; \
+	uint32_t CM_ICSC_C33_C34; \
+	uint32_t CM_BNS_VALUES_R; \
+	uint32_t CM_BNS_VALUES_G; \
+	uint32_t CM_BNS_VALUES_B; \
+	uint32_t CM_DGAM_RAMB_START_CNTL_B; \
+	uint32_t CM_DGAM_RAMB_START_CNTL_G; \
+	uint32_t CM_DGAM_RAMB_START_CNTL_R; \
+	uint32_t CM_DGAM_RAMB_SLOPE_CNTL_B; \
+	uint32_t CM_DGAM_RAMB_SLOPE_CNTL_G; \
+	uint32_t CM_DGAM_RAMB_SLOPE_CNTL_R; \
+	uint32_t CM_DGAM_RAMB_END_CNTL1_B; \
+	uint32_t CM_DGAM_RAMB_END_CNTL2_B; \
+	uint32_t CM_DGAM_RAMB_END_CNTL1_G; \
+	uint32_t CM_DGAM_RAMB_END_CNTL2_G; \
+	uint32_t CM_DGAM_RAMB_END_CNTL1_R; \
+	uint32_t CM_DGAM_RAMB_END_CNTL2_R; \
+	uint32_t CM_DGAM_RAMB_REGION_0_1; \
+	uint32_t CM_DGAM_RAMB_REGION_14_15; \
+	uint32_t CM_DGAM_RAMA_START_CNTL_B; \
+	uint32_t CM_DGAM_RAMA_START_CNTL_G; \
+	uint32_t CM_DGAM_RAMA_START_CNTL_R; \
+	uint32_t CM_DGAM_RAMA_SLOPE_CNTL_B; \
+	uint32_t CM_DGAM_RAMA_SLOPE_CNTL_G; \
+	uint32_t CM_DGAM_RAMA_SLOPE_CNTL_R; \
+	uint32_t CM_DGAM_RAMA_END_CNTL1_B; \
+	uint32_t CM_DGAM_RAMA_END_CNTL2_B; \
+	uint32_t CM_DGAM_RAMA_END_CNTL1_G; \
+	uint32_t CM_DGAM_RAMA_END_CNTL2_G; \
+	uint32_t CM_DGAM_RAMA_END_CNTL1_R; \
+	uint32_t CM_DGAM_RAMA_END_CNTL2_R; \
+	uint32_t CM_DGAM_RAMA_REGION_0_1; \
+	uint32_t CM_DGAM_RAMA_REGION_14_15; \
+	uint32_t CM_DGAM_LUT_WRITE_EN_MASK; \
+	uint32_t CM_DGAM_LUT_INDEX; \
+	uint32_t CM_DGAM_LUT_DATA; \
+	uint32_t CM_CONTROL; \
+	uint32_t CM_DGAM_CONTROL; \
+	uint32_t CM_IGAM_CONTROL; \
+	uint32_t CM_IGAM_LUT_RW_CONTROL; \
+	uint32_t CM_IGAM_LUT_RW_INDEX; \
+	uint32_t CM_IGAM_LUT_SEQ_COLOR; \
+	uint32_t FORMAT_CONTROL; \
+	uint32_t CNVC_SURFACE_PIXEL_FORMAT; \
+	uint32_t CURSOR_CONTROL; \
+	uint32_t CURSOR0_CONTROL; \
+	uint32_t CURSOR0_COLOR0; \
+	uint32_t CURSOR0_COLOR1;
 
 struct dcn_dpp_registers {
-	uint32_t DSCL_EXT_OVERSCAN_LEFT_RIGHT;
-	uint32_t DSCL_EXT_OVERSCAN_TOP_BOTTOM;
-	uint32_t OTG_H_BLANK;
-	uint32_t OTG_V_BLANK;
-	uint32_t SCL_MODE;
-	uint32_t LB_DATA_FORMAT;
-	uint32_t LB_MEMORY_CTRL;
-	uint32_t DSCL_AUTOCAL;
-	uint32_t SCL_BLACK_OFFSET;
-	uint32_t DSCL_CONTROL;
-	uint32_t SCL_TAP_CONTROL;
-	uint32_t SCL_COEF_RAM_TAP_SELECT;
-	uint32_t SCL_COEF_RAM_TAP_DATA;
-	uint32_t DSCL_2TAP_CONTROL;
-	uint32_t MPC_SIZE;
-	uint32_t SCL_HORZ_FILTER_SCALE_RATIO;
-	uint32_t SCL_VERT_FILTER_SCALE_RATIO;
-	uint32_t SCL_HORZ_FILTER_SCALE_RATIO_C;
-	uint32_t SCL_VERT_FILTER_SCALE_RATIO_C;
-	uint32_t SCL_HORZ_FILTER_INIT;
-	uint32_t SCL_HORZ_FILTER_INIT_C;
-	uint32_t SCL_VERT_FILTER_INIT;
-	uint32_t SCL_VERT_FILTER_INIT_BOT;
-	uint32_t SCL_VERT_FILTER_INIT_C;
-	uint32_t SCL_VERT_FILTER_INIT_BOT_C;
-	uint32_t RECOUT_START;
-	uint32_t RECOUT_SIZE;
-	uint32_t CM_GAMUT_REMAP_CONTROL;
-	uint32_t CM_GAMUT_REMAP_C11_C12;
-	uint32_t CM_GAMUT_REMAP_C33_C34;
-	uint32_t CM_COMA_C11_C12;
-	uint32_t CM_COMA_C33_C34;
-	uint32_t CM_COMB_C11_C12;
-	uint32_t CM_COMB_C33_C34;
-	uint32_t CM_OCSC_CONTROL;
-	uint32_t CM_OCSC_C11_C12;
-	uint32_t CM_OCSC_C33_C34;
-	uint32_t CM_MEM_PWR_CTRL;
-	uint32_t CM_RGAM_LUT_DATA;
-	uint32_t CM_RGAM_LUT_WRITE_EN_MASK;
-	uint32_t CM_RGAM_LUT_INDEX;
-	uint32_t CM_RGAM_RAMB_START_CNTL_B;
-	uint32_t CM_RGAM_RAMB_START_CNTL_G;
-	uint32_t CM_RGAM_RAMB_START_CNTL_R;
-	uint32_t CM_RGAM_RAMB_SLOPE_CNTL_B;
-	uint32_t CM_RGAM_RAMB_SLOPE_CNTL_G;
-	uint32_t CM_RGAM_RAMB_SLOPE_CNTL_R;
-	uint32_t CM_RGAM_RAMB_END_CNTL1_B;
-	uint32_t CM_RGAM_RAMB_END_CNTL2_B;
-	uint32_t CM_RGAM_RAMB_END_CNTL1_G;
-	uint32_t CM_RGAM_RAMB_END_CNTL2_G;
-	uint32_t CM_RGAM_RAMB_END_CNTL1_R;
-	uint32_t CM_RGAM_RAMB_END_CNTL2_R;
-	uint32_t CM_RGAM_RAMB_REGION_0_1;
-	uint32_t CM_RGAM_RAMB_REGION_32_33;
-	uint32_t CM_RGAM_RAMA_START_CNTL_B;
-	uint32_t CM_RGAM_RAMA_START_CNTL_G;
-	uint32_t CM_RGAM_RAMA_START_CNTL_R;
-	uint32_t CM_RGAM_RAMA_SLOPE_CNTL_B;
-	uint32_t CM_RGAM_RAMA_SLOPE_CNTL_G;
-	uint32_t CM_RGAM_RAMA_SLOPE_CNTL_R;
-	uint32_t CM_RGAM_RAMA_END_CNTL1_B;
-	uint32_t CM_RGAM_RAMA_END_CNTL2_B;
-	uint32_t CM_RGAM_RAMA_END_CNTL1_G;
-	uint32_t CM_RGAM_RAMA_END_CNTL2_G;
-	uint32_t CM_RGAM_RAMA_END_CNTL1_R;
-	uint32_t CM_RGAM_RAMA_END_CNTL2_R;
-	uint32_t CM_RGAM_RAMA_REGION_0_1;
-	uint32_t CM_RGAM_RAMA_REGION_32_33;
-	uint32_t CM_RGAM_CONTROL;
-	uint32_t CM_CMOUT_CONTROL;
-	uint32_t OBUF_CONTROL;
-	uint32_t CM_BLNDGAM_LUT_WRITE_EN_MASK;
-	uint32_t CM_BLNDGAM_CONTROL;
-	uint32_t CM_BLNDGAM_RAMB_START_CNTL_B;
-	uint32_t CM_BLNDGAM_RAMB_START_CNTL_G;
-	uint32_t CM_BLNDGAM_RAMB_START_CNTL_R;
-	uint32_t CM_BLNDGAM_RAMB_SLOPE_CNTL_B;
-	uint32_t CM_BLNDGAM_RAMB_SLOPE_CNTL_G;
-	uint32_t CM_BLNDGAM_RAMB_SLOPE_CNTL_R;
-	uint32_t CM_BLNDGAM_RAMB_END_CNTL1_B;
-	uint32_t CM_BLNDGAM_RAMB_END_CNTL2_B;
-	uint32_t CM_BLNDGAM_RAMB_END_CNTL1_G;
-	uint32_t CM_BLNDGAM_RAMB_END_CNTL2_G;
-	uint32_t CM_BLNDGAM_RAMB_END_CNTL1_R;
-	uint32_t CM_BLNDGAM_RAMB_END_CNTL2_R;
-	uint32_t CM_BLNDGAM_RAMB_REGION_0_1;
-	uint32_t CM_BLNDGAM_RAMB_REGION_2_3;
-	uint32_t CM_BLNDGAM_RAMB_REGION_4_5;
-	uint32_t CM_BLNDGAM_RAMB_REGION_6_7;
-	uint32_t CM_BLNDGAM_RAMB_REGION_8_9;
-	uint32_t CM_BLNDGAM_RAMB_REGION_10_11;
-	uint32_t CM_BLNDGAM_RAMB_REGION_12_13;
-	uint32_t CM_BLNDGAM_RAMB_REGION_14_15;
-	uint32_t CM_BLNDGAM_RAMB_REGION_16_17;
-	uint32_t CM_BLNDGAM_RAMB_REGION_18_19;
-	uint32_t CM_BLNDGAM_RAMB_REGION_20_21;
-	uint32_t CM_BLNDGAM_RAMB_REGION_22_23;
-	uint32_t CM_BLNDGAM_RAMB_REGION_24_25;
-	uint32_t CM_BLNDGAM_RAMB_REGION_26_27;
-	uint32_t CM_BLNDGAM_RAMB_REGION_28_29;
-	uint32_t CM_BLNDGAM_RAMB_REGION_30_31;
-	uint32_t CM_BLNDGAM_RAMB_REGION_32_33;
-	uint32_t CM_BLNDGAM_RAMA_START_CNTL_B;
-	uint32_t CM_BLNDGAM_RAMA_START_CNTL_G;
-	uint32_t CM_BLNDGAM_RAMA_START_CNTL_R;
-	uint32_t CM_BLNDGAM_RAMA_SLOPE_CNTL_B;
-	uint32_t CM_BLNDGAM_RAMA_SLOPE_CNTL_G;
-	uint32_t CM_BLNDGAM_RAMA_SLOPE_CNTL_R;
-	uint32_t CM_BLNDGAM_RAMA_END_CNTL1_B;
-	uint32_t CM_BLNDGAM_RAMA_END_CNTL2_B;
-	uint32_t CM_BLNDGAM_RAMA_END_CNTL1_G;
-	uint32_t CM_BLNDGAM_RAMA_END_CNTL2_G;
-	uint32_t CM_BLNDGAM_RAMA_END_CNTL1_R;
-	uint32_t CM_BLNDGAM_RAMA_END_CNTL2_R;
-	uint32_t CM_BLNDGAM_RAMA_REGION_0_1;
-	uint32_t CM_BLNDGAM_RAMA_REGION_2_3;
-	uint32_t CM_BLNDGAM_RAMA_REGION_4_5;
-	uint32_t CM_BLNDGAM_RAMA_REGION_6_7;
-	uint32_t CM_BLNDGAM_RAMA_REGION_8_9;
-	uint32_t CM_BLNDGAM_RAMA_REGION_10_11;
-	uint32_t CM_BLNDGAM_RAMA_REGION_12_13;
-	uint32_t CM_BLNDGAM_RAMA_REGION_14_15;
-	uint32_t CM_BLNDGAM_RAMA_REGION_16_17;
-	uint32_t CM_BLNDGAM_RAMA_REGION_18_19;
-	uint32_t CM_BLNDGAM_RAMA_REGION_20_21;
-	uint32_t CM_BLNDGAM_RAMA_REGION_22_23;
-	uint32_t CM_BLNDGAM_RAMA_REGION_24_25;
-	uint32_t CM_BLNDGAM_RAMA_REGION_26_27;
-	uint32_t CM_BLNDGAM_RAMA_REGION_28_29;
-	uint32_t CM_BLNDGAM_RAMA_REGION_30_31;
-	uint32_t CM_BLNDGAM_RAMA_REGION_32_33;
-	uint32_t CM_BLNDGAM_LUT_INDEX;
-	uint32_t CM_BLNDGAM_LUT_DATA;
-	uint32_t CM_3DLUT_MODE;
-	uint32_t CM_3DLUT_INDEX;
-	uint32_t CM_3DLUT_DATA;
-	uint32_t CM_3DLUT_DATA_30BIT;
-	uint32_t CM_3DLUT_READ_WRITE_CONTROL;
-	uint32_t CM_SHAPER_LUT_WRITE_EN_MASK;
-	uint32_t CM_SHAPER_CONTROL;
-	uint32_t CM_SHAPER_RAMB_START_CNTL_B;
-	uint32_t CM_SHAPER_RAMB_START_CNTL_G;
-	uint32_t CM_SHAPER_RAMB_START_CNTL_R;
-	uint32_t CM_SHAPER_RAMB_END_CNTL_B;
-	uint32_t CM_SHAPER_RAMB_END_CNTL_G;
-	uint32_t CM_SHAPER_RAMB_END_CNTL_R;
-	uint32_t CM_SHAPER_RAMB_REGION_0_1;
-	uint32_t CM_SHAPER_RAMB_REGION_2_3;
-	uint32_t CM_SHAPER_RAMB_REGION_4_5;
-	uint32_t CM_SHAPER_RAMB_REGION_6_7;
-	uint32_t CM_SHAPER_RAMB_REGION_8_9;
-	uint32_t CM_SHAPER_RAMB_REGION_10_11;
-	uint32_t CM_SHAPER_RAMB_REGION_12_13;
-	uint32_t CM_SHAPER_RAMB_REGION_14_15;
-	uint32_t CM_SHAPER_RAMB_REGION_16_17;
-	uint32_t CM_SHAPER_RAMB_REGION_18_19;
-	uint32_t CM_SHAPER_RAMB_REGION_20_21;
-	uint32_t CM_SHAPER_RAMB_REGION_22_23;
-	uint32_t CM_SHAPER_RAMB_REGION_24_25;
-	uint32_t CM_SHAPER_RAMB_REGION_26_27;
-	uint32_t CM_SHAPER_RAMB_REGION_28_29;
-	uint32_t CM_SHAPER_RAMB_REGION_30_31;
-	uint32_t CM_SHAPER_RAMB_REGION_32_33;
-	uint32_t CM_SHAPER_RAMA_START_CNTL_B;
-	uint32_t CM_SHAPER_RAMA_START_CNTL_G;
-	uint32_t CM_SHAPER_RAMA_START_CNTL_R;
-	uint32_t CM_SHAPER_RAMA_END_CNTL_B;
-	uint32_t CM_SHAPER_RAMA_END_CNTL_G;
-	uint32_t CM_SHAPER_RAMA_END_CNTL_R;
-	uint32_t CM_SHAPER_RAMA_REGION_0_1;
-	uint32_t CM_SHAPER_RAMA_REGION_2_3;
-	uint32_t CM_SHAPER_RAMA_REGION_4_5;
-	uint32_t CM_SHAPER_RAMA_REGION_6_7;
-	uint32_t CM_SHAPER_RAMA_REGION_8_9;
-	uint32_t CM_SHAPER_RAMA_REGION_10_11;
-	uint32_t CM_SHAPER_RAMA_REGION_12_13;
-	uint32_t CM_SHAPER_RAMA_REGION_14_15;
-	uint32_t CM_SHAPER_RAMA_REGION_16_17;
-	uint32_t CM_SHAPER_RAMA_REGION_18_19;
-	uint32_t CM_SHAPER_RAMA_REGION_20_21;
-	uint32_t CM_SHAPER_RAMA_REGION_22_23;
-	uint32_t CM_SHAPER_RAMA_REGION_24_25;
-	uint32_t CM_SHAPER_RAMA_REGION_26_27;
-	uint32_t CM_SHAPER_RAMA_REGION_28_29;
-	uint32_t CM_SHAPER_RAMA_REGION_30_31;
-	uint32_t CM_SHAPER_RAMA_REGION_32_33;
-	uint32_t CM_SHAPER_LUT_INDEX;
-	uint32_t CM_SHAPER_LUT_DATA;
-	uint32_t CM_ICSC_CONTROL;
-	uint32_t CM_ICSC_C11_C12;
-	uint32_t CM_ICSC_C33_C34;
-	uint32_t CM_DGAM_RAMB_START_CNTL_B;
-	uint32_t CM_DGAM_RAMB_START_CNTL_G;
-	uint32_t CM_DGAM_RAMB_START_CNTL_R;
-	uint32_t CM_DGAM_RAMB_SLOPE_CNTL_B;
-	uint32_t CM_DGAM_RAMB_SLOPE_CNTL_G;
-	uint32_t CM_DGAM_RAMB_SLOPE_CNTL_R;
-	uint32_t CM_DGAM_RAMB_END_CNTL1_B;
-	uint32_t CM_DGAM_RAMB_END_CNTL2_B;
-	uint32_t CM_DGAM_RAMB_END_CNTL1_G;
-	uint32_t CM_DGAM_RAMB_END_CNTL2_G;
-	uint32_t CM_DGAM_RAMB_END_CNTL1_R;
-	uint32_t CM_DGAM_RAMB_END_CNTL2_R;
-	uint32_t CM_DGAM_RAMB_REGION_0_1;
-	uint32_t CM_DGAM_RAMB_REGION_14_15;
-	uint32_t CM_DGAM_RAMA_START_CNTL_B;
-	uint32_t CM_DGAM_RAMA_START_CNTL_G;
-	uint32_t CM_DGAM_RAMA_START_CNTL_R;
-	uint32_t CM_DGAM_RAMA_SLOPE_CNTL_B;
-	uint32_t CM_DGAM_RAMA_SLOPE_CNTL_G;
-	uint32_t CM_DGAM_RAMA_SLOPE_CNTL_R;
-	uint32_t CM_DGAM_RAMA_END_CNTL1_B;
-	uint32_t CM_DGAM_RAMA_END_CNTL2_B;
-	uint32_t CM_DGAM_RAMA_END_CNTL1_G;
-	uint32_t CM_DGAM_RAMA_END_CNTL2_G;
-	uint32_t CM_DGAM_RAMA_END_CNTL1_R;
-	uint32_t CM_DGAM_RAMA_END_CNTL2_R;
-	uint32_t CM_DGAM_RAMA_REGION_0_1;
-	uint32_t CM_DGAM_RAMA_REGION_14_15;
-	uint32_t CM_DGAM_LUT_WRITE_EN_MASK;
-	uint32_t CM_DGAM_LUT_INDEX;
-	uint32_t CM_DGAM_LUT_DATA;
-	uint32_t CM_CONTROL;
-	uint32_t CM_DGAM_CONTROL;
-	uint32_t CM_IGAM_CONTROL;
-	uint32_t CM_IGAM_LUT_RW_CONTROL;
-	uint32_t CM_IGAM_LUT_RW_INDEX;
-	uint32_t CM_IGAM_LUT_SEQ_COLOR;
-	uint32_t FORMAT_CONTROL;
-	uint32_t CNVC_SURFACE_PIXEL_FORMAT;
-	uint32_t CURSOR_CONTROL;
-	uint32_t CURSOR0_CONTROL;
-	uint32_t CURSOR0_COLOR0;
-	uint32_t CURSOR0_COLOR1;
+	DPP_COMMON_REG_VARIABLE_LIST
 };
 
 struct dcn10_dpp {
@@ -1266,6 +1273,8 @@ struct dcn10_dpp {
 	int lb_memory_size;
 	int lb_bits_per_entry;
 	bool is_write_to_ram_a_safe;
+	struct scaler_data scl_data;
+	struct pwl_params pwl_data;
 };
 
 enum dcn10_input_csc_select {
@@ -1274,6 +1283,10 @@ enum dcn10_input_csc_select {
 	INPUT_CSC_SELECT_COMA
 };
 
+void dpp1_set_cursor_attributes(
+		struct dpp *dpp_base,
+		enum dc_cursor_color_format color_format);
+
 bool dpp1_dscl_is_lb_conf_valid(
 		int ceil_vratio,
 		int num_partitions,
@@ -1310,7 +1323,12 @@ void dpp1_power_on_degamma_lut(
 void dpp1_program_input_csc(
 		struct dpp *dpp_base,
 		enum dc_color_space color_space,
-		enum dcn10_input_csc_select select);
+		enum dcn10_input_csc_select select,
+		const struct out_csc_color_matrix *tbl_entry);
+
+void dpp1_program_bias_and_scale(
+		struct dpp *dpp_base,
+		struct dc_bias_and_scale *params);
 
 void dpp1_program_input_lut(
 		struct dpp *dpp_base,
@@ -1356,7 +1374,7 @@ void dpp1_cm_program_regamma_lutb_settings(
 		const struct pwl_params *params);
 void dpp1_cm_set_output_csc_adjustment(
 		struct dpp *dpp_base,
-		const struct out_csc_color_matrix *tbl_entry);
+		const uint16_t *regval);
 
 void dpp1_cm_set_output_csc_default(
 		struct dpp *dpp_base,
@@ -1372,8 +1390,10 @@ void dpp1_dscl_set_scaler_manual_scale(
 
 void dpp1_cnv_setup (
 		struct dpp *dpp_base,
-		enum surface_pixel_format input_format,
-		enum expansion_mode mode);
+		enum surface_pixel_format format,
+		enum expansion_mode mode,
+		struct csc_transform input_csc_color_matrix,
+		enum dc_color_space input_color_space);
 
 void dpp1_full_bypass(struct dpp *dpp_base);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
index ed1216b53465..a5b099023652 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
@@ -49,6 +49,8 @@
 #define FN(reg_name, field_name) \
 	dpp->tf_shift->field_name, dpp->tf_mask->field_name
 
+#define NUM_ELEMENTS(a) (sizeof(a) / sizeof((a)[0]))
+
 struct dcn10_input_csc_matrix {
 	enum dc_color_space color_space;
 	uint16_t regval[12];
@@ -117,8 +119,6 @@ static const struct dcn10_input_csc_matrix dcn10_input_csc_matrix[] = {
 						0x2568, 0x43ee, 0xdbb2} }
 };
 
-
-
 static void program_gamut_remap(
 		struct dcn10_dpp *dpp,
 		const uint16_t *regval,
@@ -223,82 +223,20 @@ void dpp1_cm_set_gamut_remap(
 	}
 }
 
-void dpp1_cm_set_output_csc_default(
-		struct dpp *dpp_base,
-		enum dc_color_space colorspace)
-{
-
-	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
-	uint32_t ocsc_mode = 0;
-
-	switch (colorspace) {
-		case COLOR_SPACE_SRGB:
-		case COLOR_SPACE_2020_RGB_FULLRANGE:
-			ocsc_mode = 0;
-			break;
-		case COLOR_SPACE_SRGB_LIMITED:
-		case COLOR_SPACE_2020_RGB_LIMITEDRANGE:
-			ocsc_mode = 1;
-			break;
-		case COLOR_SPACE_YCBCR601:
-		case COLOR_SPACE_YCBCR601_LIMITED:
-			ocsc_mode = 2;
-			break;
-		case COLOR_SPACE_YCBCR709:
-		case COLOR_SPACE_YCBCR709_LIMITED:
-		case COLOR_SPACE_2020_YCBCR:
-			ocsc_mode = 3;
-			break;
-		case COLOR_SPACE_UNKNOWN:
-		default:
-			break;
-	}
-
-	REG_SET(CM_OCSC_CONTROL, 0, CM_OCSC_MODE, ocsc_mode);
-
-}
-
-static void dpp1_cm_get_reg_field(
-		struct dcn10_dpp *dpp,
-		struct xfer_func_reg *reg)
-{
-	reg->shifts.exp_region0_lut_offset = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION0_LUT_OFFSET;
-	reg->masks.exp_region0_lut_offset = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION0_LUT_OFFSET;
-	reg->shifts.exp_region0_num_segments = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
-	reg->masks.exp_region0_num_segments = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
-	reg->shifts.exp_region1_lut_offset = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION1_LUT_OFFSET;
-	reg->masks.exp_region1_lut_offset = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION1_LUT_OFFSET;
-	reg->shifts.exp_region1_num_segments = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
-	reg->masks.exp_region1_num_segments = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
-
-	reg->shifts.field_region_end = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_END_B;
-	reg->masks.field_region_end = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_END_B;
-	reg->shifts.field_region_end_slope = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_END_SLOPE_B;
-	reg->masks.field_region_end_slope = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_END_SLOPE_B;
-	reg->shifts.field_region_end_base = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_END_BASE_B;
-	reg->masks.field_region_end_base = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_END_BASE_B;
-	reg->shifts.field_region_linear_slope = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_LINEAR_SLOPE_B;
-	reg->masks.field_region_linear_slope = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_LINEAR_SLOPE_B;
-	reg->shifts.exp_region_start = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_START_B;
-	reg->masks.exp_region_start = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_START_B;
-	reg->shifts.exp_resion_start_segment = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_START_SEGMENT_B;
-	reg->masks.exp_resion_start_segment = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_START_SEGMENT_B;
-}
-
 static void dpp1_cm_program_color_matrix(
 		struct dcn10_dpp *dpp,
-		const struct out_csc_color_matrix *tbl_entry)
+		const uint16_t *regval)
 {
 	uint32_t mode;
 	struct color_matrices_reg gam_regs;
 
 	REG_GET(CM_OCSC_CONTROL, CM_OCSC_MODE, &mode);
 
-	if (tbl_entry == NULL) {
+	if (regval == NULL) {
 		BREAK_TO_DEBUGGER();
 		return;
 	}
-
+	mode = 4;
 	gam_regs.shifts.csc_c11 = dpp->tf_shift->CM_OCSC_C11;
 	gam_regs.masks.csc_c11  = dpp->tf_mask->CM_OCSC_C11;
 	gam_regs.shifts.csc_c12 = dpp->tf_shift->CM_OCSC_C12;
@@ -311,7 +249,7 @@ static void dpp1_cm_program_color_matrix(
 
 		cm_helper_program_color_matrices(
 				dpp->base.ctx,
-				tbl_entry->regval,
+				regval,
 				&gam_regs);
 
 	} else {
@@ -321,78 +259,91 @@ static void dpp1_cm_program_color_matrix(
 
 		cm_helper_program_color_matrices(
 				dpp->base.ctx,
-				tbl_entry->regval,
+				regval,
 				&gam_regs);
 	}
 }
 
-void dpp1_cm_set_output_csc_adjustment(
+void dpp1_cm_set_output_csc_default(
 		struct dpp *dpp_base,
-		const struct out_csc_color_matrix *tbl_entry)
+		enum dc_color_space colorspace)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
-	//enum csc_color_mode config = CSC_COLOR_MODE_GRAPHICS_OUTPUT_CSC;
+	const uint16_t *regval = NULL;
+	int arr_size;
 	uint32_t ocsc_mode = 4;
 
-	/**
-	*if (tbl_entry != NULL) {
-	*	switch (tbl_entry->color_space) {
-	*	case COLOR_SPACE_SRGB:
-	*	case COLOR_SPACE_2020_RGB_FULLRANGE:
-	*		ocsc_mode = 0;
-	*		break;
-	*	case COLOR_SPACE_SRGB_LIMITED:
-	*	case COLOR_SPACE_2020_RGB_LIMITEDRANGE:
-	*		ocsc_mode = 1;
-	*		break;
-	*	case COLOR_SPACE_YCBCR601:
-	*	case COLOR_SPACE_YCBCR601_LIMITED:
-	*		ocsc_mode = 2;
-	*		break;
-	*	case COLOR_SPACE_YCBCR709:
-	*	case COLOR_SPACE_YCBCR709_LIMITED:
-	*	case COLOR_SPACE_2020_YCBCR:
-	*		ocsc_mode = 3;
-	*		break;
-	*	case COLOR_SPACE_UNKNOWN:
-	*	default:
-	*		break;
-	*	}
-	*}
-	*/
+	regval = find_color_matrix(colorspace, &arr_size);
+	if (regval == NULL) {
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+	dpp1_cm_program_color_matrix(dpp, regval);
+	REG_SET(CM_OCSC_CONTROL, 0, CM_OCSC_MODE, ocsc_mode);
+}
+
+static void dpp1_cm_get_reg_field(
+		struct dcn10_dpp *dpp,
+		struct xfer_func_reg *reg)
+{
+	reg->shifts.exp_region0_lut_offset = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION0_LUT_OFFSET;
+	reg->masks.exp_region0_lut_offset = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION0_LUT_OFFSET;
+	reg->shifts.exp_region0_num_segments = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
+	reg->masks.exp_region0_num_segments = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION0_NUM_SEGMENTS;
+	reg->shifts.exp_region1_lut_offset = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION1_LUT_OFFSET;
+	reg->masks.exp_region1_lut_offset = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION1_LUT_OFFSET;
+	reg->shifts.exp_region1_num_segments = dpp->tf_shift->CM_RGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
+	reg->masks.exp_region1_num_segments = dpp->tf_mask->CM_RGAM_RAMA_EXP_REGION1_NUM_SEGMENTS;
+
+	reg->shifts.field_region_end = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_END_B;
+	reg->masks.field_region_end = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_END_B;
+	reg->shifts.field_region_end_slope = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_END_SLOPE_B;
+	reg->masks.field_region_end_slope = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_END_SLOPE_B;
+	reg->shifts.field_region_end_base = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_END_BASE_B;
+	reg->masks.field_region_end_base = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_END_BASE_B;
+	reg->shifts.field_region_linear_slope = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_LINEAR_SLOPE_B;
+	reg->masks.field_region_linear_slope = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_LINEAR_SLOPE_B;
+	reg->shifts.exp_region_start = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_START_B;
+	reg->masks.exp_region_start = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_START_B;
+	reg->shifts.exp_resion_start_segment = dpp->tf_shift->CM_RGAM_RAMB_EXP_REGION_START_SEGMENT_B;
+	reg->masks.exp_resion_start_segment = dpp->tf_mask->CM_RGAM_RAMB_EXP_REGION_START_SEGMENT_B;
+}
 
+void dpp1_cm_set_output_csc_adjustment(
+		struct dpp *dpp_base,
+		const uint16_t *regval)
+{
+	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
+	uint32_t ocsc_mode = 4;
+	dpp1_cm_program_color_matrix(dpp, regval);
 	REG_SET(CM_OCSC_CONTROL, 0, CM_OCSC_MODE, ocsc_mode);
-	dpp1_cm_program_color_matrix(dpp, tbl_entry);
 }
 
-void dpp1_cm_power_on_regamma_lut(
-	struct dpp *dpp_base,
-	bool power_on)
+void dpp1_cm_power_on_regamma_lut(struct dpp *dpp_base,
+				  bool power_on)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
+
 	REG_SET(CM_MEM_PWR_CTRL, 0,
-			RGAM_MEM_PWR_FORCE, power_on == true ? 0:1);
+		RGAM_MEM_PWR_FORCE, power_on == true ? 0:1);
 
 }
 
-void dpp1_cm_program_regamma_lut(
-		struct dpp *dpp_base,
-		const struct pwl_result_data *rgb,
-		uint32_t num)
+void dpp1_cm_program_regamma_lut(struct dpp *dpp_base,
+				 const struct pwl_result_data *rgb,
+				 uint32_t num)
 {
 	uint32_t i;
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
+
 	for (i = 0 ; i < num; i++) {
 		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].red_reg);
 		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].green_reg);
 		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].blue_reg);
 
-		REG_SET(CM_RGAM_LUT_DATA, 0,
-				CM_RGAM_LUT_DATA, rgb[i].delta_red_reg);
-		REG_SET(CM_RGAM_LUT_DATA, 0,
-				CM_RGAM_LUT_DATA, rgb[i].delta_green_reg);
-		REG_SET(CM_RGAM_LUT_DATA, 0,
-				CM_RGAM_LUT_DATA, rgb[i].delta_blue_reg);
+		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].delta_red_reg);
+		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].delta_green_reg);
+		REG_SET(CM_RGAM_LUT_DATA, 0, CM_RGAM_LUT_DATA, rgb[i].delta_blue_reg);
 
 	}
 
@@ -471,7 +422,8 @@ void dpp1_cm_program_regamma_lutb_settings(
 void dpp1_program_input_csc(
 		struct dpp *dpp_base,
 		enum dc_color_space color_space,
-		enum dcn10_input_csc_select select)
+		enum dcn10_input_csc_select select,
+		const struct out_csc_color_matrix *tbl_entry)
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 	int i;
@@ -485,15 +437,19 @@ void dpp1_program_input_csc(
 		return;
 	}
 
-	for (i = 0; i < arr_size; i++)
-		if (dcn10_input_csc_matrix[i].color_space == color_space) {
-			regval = dcn10_input_csc_matrix[i].regval;
-			break;
+	if (tbl_entry == NULL) {
+		for (i = 0; i < arr_size; i++)
+			if (dcn10_input_csc_matrix[i].color_space == color_space) {
+				regval = dcn10_input_csc_matrix[i].regval;
+				break;
+			}
+
+		if (regval == NULL) {
+			BREAK_TO_DEBUGGER();
+			return;
 		}
-
-	if (regval == NULL) {
-		BREAK_TO_DEBUGGER();
-		return;
+	} else {
+		regval = tbl_entry->regval;
 	}
 
 	if (select == INPUT_CSC_SELECT_COMA)
@@ -528,6 +484,27 @@ void dpp1_program_input_csc(
 	}
 }
 
+//keep here for now, decide multi dce support later
+void dpp1_program_bias_and_scale(
+	struct dpp *dpp_base,
+	struct dc_bias_and_scale *params)
+{
+	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
+
+	REG_SET_2(CM_BNS_VALUES_R, 0,
+		CM_BNS_SCALE_R, params->scale_red,
+		CM_BNS_BIAS_R, params->bias_red);
+
+	REG_SET_2(CM_BNS_VALUES_G, 0,
+		CM_BNS_SCALE_G, params->scale_green,
+		CM_BNS_BIAS_G, params->bias_green);
+
+	REG_SET_2(CM_BNS_VALUES_B, 0,
+		CM_BNS_SCALE_B, params->scale_blue,
+		CM_BNS_BIAS_B, params->bias_blue);
+
+}
+
 /*program de gamma RAM B*/
 void dpp1_program_degamma_lutb_settings(
 		struct dpp *dpp_base,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
index cbad36410b32..3eb824debf43 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
@@ -648,6 +648,13 @@ void dpp1_dscl_set_scaler_manual_scale(
 	bool ycbcr = scl_data->format >= PIXEL_FORMAT_VIDEO_BEGIN
 				&& scl_data->format <= PIXEL_FORMAT_VIDEO_END;
 
+	if (memcmp(&dpp->scl_data, scl_data, sizeof(*scl_data)) == 0)
+		return;
+
+	PERF_TRACE();
+
+	dpp->scl_data = *scl_data;
+
 	/* Recout */
 	dpp1_dscl_set_recout(dpp, &scl_data->recout);
 
@@ -699,4 +706,5 @@ void dpp1_dscl_set_scaler_manual_scale(
 		SCL_H_NUM_TAPS_C, scl_data->taps.h_taps_c - 1);
 
 	dpp1_dscl_set_scl_filter(dpp, scl_data, ycbcr);
+	PERF_TRACE();
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
new file mode 100644
index 000000000000..eb8317187f30
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
@@ -0,0 +1,516 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dcn10_hubp.h"
+#include "dcn10_hubbub.h"
+#include "reg_helper.h"
+
+#define CTX \
+	hubbub->ctx
+#define REG(reg)\
+	hubbub->regs->reg
+
+#undef FN
+#define FN(reg_name, field_name) \
+	hubbub->shifts->field_name, hubbub->masks->field_name
+
+void hubbub1_wm_read_state(struct hubbub *hubbub,
+		struct dcn_hubbub_wm *wm)
+{
+	struct dcn_hubbub_wm_set *s;
+
+	memset(wm, 0, sizeof(struct dcn_hubbub_wm));
+
+	s = &wm->sets[0];
+	s->wm_set = 0;
+	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A);
+	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A);
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A)) {
+		s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A);
+		s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A);
+	}
+	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A);
+
+	s = &wm->sets[1];
+	s->wm_set = 1;
+	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B);
+	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B);
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B)) {
+		s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B);
+		s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B);
+	}
+	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B);
+
+	s = &wm->sets[2];
+	s->wm_set = 2;
+	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C);
+	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C);
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C)) {
+		s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C);
+		s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C);
+	}
+	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C);
+
+	s = &wm->sets[3];
+	s->wm_set = 3;
+	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D);
+	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D);
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D)) {
+		s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D);
+		s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D);
+	}
+	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D);
+}
+
+bool hubbub1_verify_allow_pstate_change_high(
+	struct hubbub *hubbub)
+{
+	/* pstate latency is ~20us so if we wait over 40us and pstate allow
+	 * still not asserted, we are probably stuck and going to hang
+	 *
+	 * TODO: Figure out why it takes ~100us on linux
+	 * pstate takes around ~100us on linux. Unknown currently as to
+	 * why it takes that long on linux
+	 */
+	static unsigned int pstate_wait_timeout_us = 200;
+	static unsigned int pstate_wait_expected_timeout_us = 40;
+	static unsigned int max_sampled_pstate_wait_us; /* data collection */
+	static bool forced_pstate_allow; /* help with revert wa */
+
+	unsigned int debug_index = 0x7;
+	unsigned int debug_data;
+	unsigned int i;
+
+	if (forced_pstate_allow) {
+		/* we hacked to force pstate allow to prevent hang last time
+		 * we verify_allow_pstate_change_high.  so disable force
+		 * here so we can check status
+		 */
+		REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
+			     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, 0,
+			     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, 0);
+		forced_pstate_allow = false;
+	}
+
+	/* description "3-0:   Pipe0 cursor0 QOS
+	 * 7-4:   Pipe1 cursor0 QOS
+	 * 11-8:  Pipe2 cursor0 QOS
+	 * 15-12: Pipe3 cursor0 QOS
+	 * 16:    Pipe0 Plane0 Allow Pstate Change
+	 * 17:    Pipe1 Plane0 Allow Pstate Change
+	 * 18:    Pipe2 Plane0 Allow Pstate Change
+	 * 19:    Pipe3 Plane0 Allow Pstate Change
+	 * 20:    Pipe0 Plane1 Allow Pstate Change
+	 * 21:    Pipe1 Plane1 Allow Pstate Change
+	 * 22:    Pipe2 Plane1 Allow Pstate Change
+	 * 23:    Pipe3 Plane1 Allow Pstate Change
+	 * 24:    Pipe0 cursor0 Allow Pstate Change
+	 * 25:    Pipe1 cursor0 Allow Pstate Change
+	 * 26:    Pipe2 cursor0 Allow Pstate Change
+	 * 27:    Pipe3 cursor0 Allow Pstate Change
+	 * 28:    WB0 Allow Pstate Change
+	 * 29:    WB1 Allow Pstate Change
+	 * 30:    Arbiter's allow_pstate_change
+	 * 31:    SOC pstate change request
+	 */
+
+	REG_WRITE(DCHUBBUB_TEST_DEBUG_INDEX, debug_index);
+
+	for (i = 0; i < pstate_wait_timeout_us; i++) {
+		debug_data = REG_READ(DCHUBBUB_TEST_DEBUG_DATA);
+
+		if (debug_data & (1 << 30)) {
+
+			if (i > pstate_wait_expected_timeout_us)
+				dm_logger_write(hubbub->ctx->logger, LOG_WARNING,
+						"pstate took longer than expected ~%dus\n",
+						i);
+
+			return true;
+		}
+		if (max_sampled_pstate_wait_us < i)
+			max_sampled_pstate_wait_us = i;
+
+		udelay(1);
+	}
+
+	/* force pstate allow to prevent system hang
+	 * and break to debugger to investigate
+	 */
+	REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
+		     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, 1,
+		     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, 1);
+	forced_pstate_allow = true;
+
+	dm_logger_write(hubbub->ctx->logger, LOG_WARNING,
+			"pstate TEST_DEBUG_DATA: 0x%X\n",
+			debug_data);
+
+	return false;
+}
+
+static uint32_t convert_and_clamp(
+	uint32_t wm_ns,
+	uint32_t refclk_mhz,
+	uint32_t clamp_value)
+{
+	uint32_t ret_val = 0;
+	ret_val = wm_ns * refclk_mhz;
+	ret_val /= 1000;
+
+	if (ret_val > clamp_value)
+		ret_val = clamp_value;
+
+	return ret_val;
+}
+
+
+void hubbub1_program_watermarks(
+		struct hubbub *hubbub,
+		struct dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz)
+{
+	uint32_t force_en = hubbub->ctx->dc->debug.disable_stutter ? 1 : 0;
+	/*
+	 * Need to clamp to max of the register values (i.e. no wrap)
+	 * for dcn1, all wm registers are 21-bit wide
+	 */
+	uint32_t prog_wm_value;
+
+	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 0);
+
+	/* Repeat for water mark set A, B, C and D. */
+	/* clock state A */
+	prog_wm_value = convert_and_clamp(watermarks->a.urgent_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, prog_wm_value);
+
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"URGENCY_WATERMARK_A calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->a.urgent_ns, prog_wm_value);
+
+	prog_wm_value = convert_and_clamp(watermarks->a.pte_meta_urgent_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"PTE_META_URGENCY_WATERMARK_A calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->a.pte_meta_urgent_ns, prog_wm_value);
+
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A)) {
+		prog_wm_value = convert_and_clamp(
+				watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+
+
+		prog_wm_value = convert_and_clamp(
+				watermarks->a.cstate_pstate.cstate_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_EXIT_WATERMARK_A calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->a.cstate_pstate.cstate_exit_ns, prog_wm_value);
+	}
+
+	prog_wm_value = convert_and_clamp(
+			watermarks->a.cstate_pstate.pstate_change_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
+		"HW register value = 0x%x\n\n",
+		watermarks->a.cstate_pstate.pstate_change_ns, prog_wm_value);
+
+
+	/* clock state B */
+	prog_wm_value = convert_and_clamp(
+			watermarks->b.urgent_ns, refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"URGENCY_WATERMARK_B calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->b.urgent_ns, prog_wm_value);
+
+
+	prog_wm_value = convert_and_clamp(
+			watermarks->b.pte_meta_urgent_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"PTE_META_URGENCY_WATERMARK_B calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->b.pte_meta_urgent_ns, prog_wm_value);
+
+
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B)) {
+		prog_wm_value = convert_and_clamp(
+				watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_ENTER_WATERMARK_B calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+
+
+		prog_wm_value = convert_and_clamp(
+				watermarks->b.cstate_pstate.cstate_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_EXIT_WATERMARK_B calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->b.cstate_pstate.cstate_exit_ns, prog_wm_value);
+	}
+
+	prog_wm_value = convert_and_clamp(
+			watermarks->b.cstate_pstate.pstate_change_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n\n"
+		"HW register value = 0x%x\n",
+		watermarks->b.cstate_pstate.pstate_change_ns, prog_wm_value);
+
+	/* clock state C */
+	prog_wm_value = convert_and_clamp(
+			watermarks->c.urgent_ns, refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"URGENCY_WATERMARK_C calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->c.urgent_ns, prog_wm_value);
+
+
+	prog_wm_value = convert_and_clamp(
+			watermarks->c.pte_meta_urgent_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"PTE_META_URGENCY_WATERMARK_C calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->c.pte_meta_urgent_ns, prog_wm_value);
+
+
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C)) {
+		prog_wm_value = convert_and_clamp(
+				watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_ENTER_WATERMARK_C calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+
+
+		prog_wm_value = convert_and_clamp(
+				watermarks->c.cstate_pstate.cstate_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_EXIT_WATERMARK_C calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->c.cstate_pstate.cstate_exit_ns, prog_wm_value);
+	}
+
+	prog_wm_value = convert_and_clamp(
+			watermarks->c.cstate_pstate.pstate_change_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n\n"
+		"HW register value = 0x%x\n",
+		watermarks->c.cstate_pstate.pstate_change_ns, prog_wm_value);
+
+	/* clock state D */
+	prog_wm_value = convert_and_clamp(
+			watermarks->d.urgent_ns, refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"URGENCY_WATERMARK_D calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->d.urgent_ns, prog_wm_value);
+
+	prog_wm_value = convert_and_clamp(
+			watermarks->d.pte_meta_urgent_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"PTE_META_URGENCY_WATERMARK_D calculated =%d\n"
+		"HW register value = 0x%x\n",
+		watermarks->d.pte_meta_urgent_ns, prog_wm_value);
+
+
+	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D)) {
+		prog_wm_value = convert_and_clamp(
+				watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_ENTER_WATERMARK_D calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
+
+
+		prog_wm_value = convert_and_clamp(
+				watermarks->d.cstate_pstate.cstate_exit_ns,
+				refclk_mhz, 0x1fffff);
+		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
+		dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+			"SR_EXIT_WATERMARK_D calculated =%d\n"
+			"HW register value = 0x%x\n",
+			watermarks->d.cstate_pstate.cstate_exit_ns, prog_wm_value);
+	}
+
+
+	prog_wm_value = convert_and_clamp(
+			watermarks->d.cstate_pstate.pstate_change_ns,
+			refclk_mhz, 0x1fffff);
+	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, prog_wm_value);
+	dm_logger_write(hubbub->ctx->logger, LOG_BANDWIDTH_CALCS,
+		"DRAM_CLK_CHANGE_WATERMARK_D calculated =%d\n"
+		"HW register value = 0x%x\n\n",
+		watermarks->d.cstate_pstate.pstate_change_ns, prog_wm_value);
+
+	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 1);
+
+	REG_UPDATE(DCHUBBUB_ARB_SAT_LEVEL,
+			DCHUBBUB_ARB_SAT_LEVEL, 60 * refclk_mhz);
+	REG_UPDATE(DCHUBBUB_ARB_DF_REQ_OUTSTAND,
+			DCHUBBUB_ARB_MIN_REQ_OUTSTAND, 68);
+
+	REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
+			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE, 0,
+			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE, force_en);
+
+#if 0
+	REG_UPDATE_2(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE, 1,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 1);
+#endif
+}
+
+void hubbub1_update_dchub(
+	struct hubbub *hubbub,
+	struct dchub_init_data *dh_data)
+{
+	/* TODO: port code from dal2 */
+	switch (dh_data->fb_mode) {
+	case FRAME_BUFFER_MODE_ZFB_ONLY:
+		/*For ZFB case need to put DCHUB FB BASE and TOP upside down to indicate ZFB mode*/
+		REG_UPDATE(DCHUBBUB_SDPIF_FB_TOP,
+				SDPIF_FB_TOP, 0);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_FB_BASE,
+				SDPIF_FB_BASE, 0x0FFFF);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
+				SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
+				SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
+				SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
+						dh_data->zfb_size_in_byte - 1) >> 22);
+		break;
+	case FRAME_BUFFER_MODE_MIXED_ZFB_AND_LOCAL:
+		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
+				SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
+				SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
+				SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
+						dh_data->zfb_size_in_byte - 1) >> 22);
+		break;
+	case FRAME_BUFFER_MODE_LOCAL_ONLY:
+		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
+				SDPIF_AGP_BASE, 0);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
+				SDPIF_AGP_BOT, 0X03FFFF);
+
+		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
+				SDPIF_AGP_TOP, 0);
+		break;
+	default:
+		break;
+	}
+
+	dh_data->dchub_initialzied = true;
+	dh_data->dchub_info_valid = false;
+}
+
+void hubbub1_toggle_watermark_change_req(struct hubbub *hubbub)
+{
+	uint32_t watermark_change_req;
+
+	REG_GET(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, &watermark_change_req);
+
+	if (watermark_change_req)
+		watermark_change_req = 0;
+	else
+		watermark_change_req = 1;
+
+	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, watermark_change_req);
+}
+
+static const struct hubbub_funcs hubbub1_funcs = {
+	.update_dchub = hubbub1_update_dchub
+};
+
+void hubbub1_construct(struct hubbub *hubbub,
+	struct dc_context *ctx,
+	const struct dcn_hubbub_registers *hubbub_regs,
+	const struct dcn_hubbub_shift *hubbub_shift,
+	const struct dcn_hubbub_mask *hubbub_mask)
+{
+	hubbub->ctx = ctx;
+
+	hubbub->funcs = &hubbub1_funcs;
+
+	hubbub->regs = hubbub_regs;
+	hubbub->shifts = hubbub_shift;
+	hubbub->masks = hubbub_mask;
+
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
new file mode 100644
index 000000000000..d5c97844312f
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
@@ -0,0 +1,214 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef __DC_HUBBUB_DCN10_H__
+#define __DC_HUBBUB_DCN10_H__
+
+#include "core_types.h"
+
+#define HUBHUB_REG_LIST_DCN()\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),\
+	SR(DCHUBBUB_ARB_DRAM_STATE_CNTL),\
+	SR(DCHUBBUB_ARB_SAT_LEVEL),\
+	SR(DCHUBBUB_ARB_DF_REQ_OUTSTAND),\
+	SR(DCHUBBUB_GLOBAL_TIMER_CNTL), \
+	SR(DCHUBBUB_TEST_DEBUG_INDEX), \
+	SR(DCHUBBUB_TEST_DEBUG_DATA)
+
+#define HUBBUB_SR_WATERMARK_REG_LIST()\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D),\
+	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D)
+
+#define HUBBUB_REG_LIST_DCN10(id)\
+	HUBHUB_REG_LIST_DCN(), \
+	HUBBUB_SR_WATERMARK_REG_LIST(), \
+	SR(DCHUBBUB_SDPIF_FB_TOP),\
+	SR(DCHUBBUB_SDPIF_FB_BASE),\
+	SR(DCHUBBUB_SDPIF_FB_OFFSET),\
+	SR(DCHUBBUB_SDPIF_AGP_BASE),\
+	SR(DCHUBBUB_SDPIF_AGP_BOT),\
+	SR(DCHUBBUB_SDPIF_AGP_TOP)
+
+struct dcn_hubbub_registers {
+	uint32_t DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A;
+	uint32_t DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A;
+	uint32_t DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A;
+	uint32_t DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B;
+	uint32_t DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B;
+	uint32_t DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B;
+	uint32_t DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C;
+	uint32_t DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C;
+	uint32_t DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C;
+	uint32_t DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D;
+	uint32_t DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D;
+	uint32_t DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D;
+	uint32_t DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D;
+	uint32_t DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL;
+	uint32_t DCHUBBUB_ARB_SAT_LEVEL;
+	uint32_t DCHUBBUB_ARB_DF_REQ_OUTSTAND;
+	uint32_t DCHUBBUB_GLOBAL_TIMER_CNTL;
+	uint32_t DCHUBBUB_ARB_DRAM_STATE_CNTL;
+	uint32_t DCHUBBUB_TEST_DEBUG_INDEX;
+	uint32_t DCHUBBUB_TEST_DEBUG_DATA;
+	uint32_t DCHUBBUB_SDPIF_FB_TOP;
+	uint32_t DCHUBBUB_SDPIF_FB_BASE;
+	uint32_t DCHUBBUB_SDPIF_FB_OFFSET;
+	uint32_t DCHUBBUB_SDPIF_AGP_BASE;
+	uint32_t DCHUBBUB_SDPIF_AGP_BOT;
+	uint32_t DCHUBBUB_SDPIF_AGP_TOP;
+	uint32_t DCHUBBUB_CRC_CTRL;
+};
+
+/* set field name */
+#define HUBBUB_SF(reg_name, field_name, post_fix)\
+	.field_name = reg_name ## __ ## field_name ## post_fix
+
+
+#define HUBBUB_MASK_SH_LIST_DCN(mask_sh)\
+		HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_SAT_LEVEL, DCHUBBUB_ARB_SAT_LEVEL, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MIN_REQ_OUTSTAND, mask_sh)
+
+#define HUBBUB_MASK_SH_LIST_DCN10(mask_sh)\
+		HUBBUB_MASK_SH_LIST_DCN(mask_sh), \
+		HUBBUB_SF(DCHUBBUB_SDPIF_FB_TOP, SDPIF_FB_TOP, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_SDPIF_FB_BASE, SDPIF_FB_BASE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_SDPIF_FB_OFFSET, SDPIF_FB_OFFSET, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_SDPIF_AGP_BASE, SDPIF_AGP_BASE, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_SDPIF_AGP_BOT, SDPIF_AGP_BOT, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_SDPIF_AGP_TOP, SDPIF_AGP_TOP, mask_sh)
+
+#define DCN_HUBBUB_REG_FIELD_LIST(type) \
+		type DCHUBBUB_GLOBAL_TIMER_ENABLE; \
+		type DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST;\
+		type DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE;\
+		type DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE;\
+		type DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE;\
+		type DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE;\
+		type DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE;\
+		type DCHUBBUB_ARB_SAT_LEVEL;\
+		type DCHUBBUB_ARB_MIN_REQ_OUTSTAND;\
+		type DCHUBBUB_GLOBAL_TIMER_REFDIV;\
+		type SDPIF_FB_TOP;\
+		type SDPIF_FB_BASE;\
+		type SDPIF_FB_OFFSET;\
+		type SDPIF_AGP_BASE;\
+		type SDPIF_AGP_BOT;\
+		type SDPIF_AGP_TOP
+
+
+struct dcn_hubbub_shift {
+	DCN_HUBBUB_REG_FIELD_LIST(uint8_t);
+};
+
+struct dcn_hubbub_mask {
+	DCN_HUBBUB_REG_FIELD_LIST(uint32_t);
+};
+
+struct dc;
+
+struct dcn_hubbub_wm_set {
+	uint32_t wm_set;
+	uint32_t data_urgent;
+	uint32_t pte_meta_urgent;
+	uint32_t sr_enter;
+	uint32_t sr_exit;
+	uint32_t dram_clk_chanage;
+};
+
+struct dcn_hubbub_wm {
+	struct dcn_hubbub_wm_set sets[4];
+};
+
+struct hubbub_funcs {
+	void (*update_dchub)(
+			struct hubbub *hubbub,
+			struct dchub_init_data *dh_data);
+};
+
+struct hubbub {
+	const struct hubbub_funcs *funcs;
+	struct dc_context *ctx;
+	const struct dcn_hubbub_registers *regs;
+	const struct dcn_hubbub_shift *shifts;
+	const struct dcn_hubbub_mask *masks;
+};
+
+void hubbub1_update_dchub(
+	struct hubbub *hubbub,
+	struct dchub_init_data *dh_data);
+
+bool hubbub1_verify_allow_pstate_change_high(
+	struct hubbub *hubbub);
+
+void hubbub1_program_watermarks(
+		struct hubbub *hubbub,
+		struct dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz);
+
+void hubbub1_toggle_watermark_change_req(
+		struct hubbub *hubbub);
+
+void hubbub1_wm_read_state(struct hubbub *hubbub,
+		struct dcn_hubbub_wm *wm);
+
+void hubbub1_construct(struct hubbub *hubbub,
+	struct dc_context *ctx,
+	const struct dcn_hubbub_registers *hubbub_regs,
+	const struct dcn_hubbub_shift *hubbub_shift,
+	const struct dcn_hubbub_mask *hubbub_mask);
+
+#endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
index b13dee64e0ce..585b33384002 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
@@ -29,14 +29,14 @@
 #include "dcn10_hubp.h"
 
 #define REG(reg)\
-	hubp1->mi_regs->reg
+	hubp1->hubp_regs->reg
 
 #define CTX \
 	hubp1->base.ctx
 
 #undef FN
 #define FN(reg_name, field_name) \
-	hubp1->mi_shift->field_name, hubp1->mi_mask->field_name
+	hubp1->hubp_shift->field_name, hubp1->hubp_mask->field_name
 
 void hubp1_set_blank(struct hubp *hubp, bool blank)
 {
@@ -48,14 +48,33 @@ void hubp1_set_blank(struct hubp *hubp, bool blank)
 			HUBP_TTU_DISABLE, blank_en);
 
 	if (blank) {
-		REG_WAIT(DCHUBP_CNTL,
-				HUBP_NO_OUTSTANDING_REQ, 1,
-				1, 200);
+		uint32_t reg_val = REG_READ(DCHUBP_CNTL);
+
+		if (reg_val) {
+			/* init sequence workaround: in case HUBP is
+			 * power gated, this wait would timeout.
+			 *
+			 * we just wrote reg_val to non-0, if it stay 0
+			 * it means HUBP is gated
+			 */
+			REG_WAIT(DCHUBP_CNTL,
+					HUBP_NO_OUTSTANDING_REQ, 1,
+					1, 200);
+		}
+
 		hubp->mpcc_id = 0xf;
 		hubp->opp_id = 0xf;
 	}
 }
 
+static void hubp1_disconnect(struct hubp *hubp)
+{
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
+
+	REG_UPDATE(DCHUBP_CNTL,
+			HUBP_TTU_DISABLE, 1);
+}
+
 static void hubp1_set_hubp_blank_en(struct hubp *hubp, bool blank)
 {
 	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
@@ -88,10 +107,12 @@ static void hubp1_vready_workaround(struct hubp *hubp,
 }
 
 void hubp1_program_tiling(
-	struct dcn10_hubp *hubp1,
+	struct hubp *hubp,
 	const union dc_tiling_info *info,
 	const enum surface_pixel_format pixel_format)
 {
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
+
 	REG_UPDATE_6(DCSURF_ADDR_CONFIG,
 			NUM_PIPES, log_2(info->gfx9.num_pipes),
 			NUM_BANKS, log_2(info->gfx9.num_banks),
@@ -108,13 +129,14 @@ void hubp1_program_tiling(
 }
 
 void hubp1_program_size_and_rotation(
-	struct dcn10_hubp *hubp1,
+	struct hubp *hubp,
 	enum dc_rotation_angle rotation,
 	enum surface_pixel_format format,
 	const union plane_size *plane_size,
 	struct dc_plane_dcc_param *dcc,
 	bool horizontal_mirror)
 {
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
 	uint32_t pitch, meta_pitch, pitch_c, meta_pitch_c, mirror;
 
 	/* Program data and meta surface pitch (calculation from addrlib)
@@ -170,9 +192,10 @@ void hubp1_program_size_and_rotation(
 }
 
 void hubp1_program_pixel_format(
-	struct dcn10_hubp *hubp1,
+	struct hubp *hubp,
 	enum surface_pixel_format format)
 {
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
 	uint32_t red_bar = 3;
 	uint32_t blue_bar = 2;
 
@@ -416,13 +439,11 @@ void hubp1_program_surface_config(
 	struct dc_plane_dcc_param *dcc,
 	bool horizontal_mirror)
 {
-	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
-
 	hubp1_dcc_control(hubp, dcc->enable, dcc->grph.independent_64b_blks);
-	hubp1_program_tiling(hubp1, tiling_info, format);
+	hubp1_program_tiling(hubp, tiling_info, format);
 	hubp1_program_size_and_rotation(
-			hubp1, rotation, format, plane_size, dcc, horizontal_mirror);
-	hubp1_program_pixel_format(hubp1, format);
+			hubp, rotation, format, plane_size, dcc, horizontal_mirror);
+	hubp1_program_pixel_format(hubp, format);
 }
 
 void hubp1_program_requestor(
@@ -757,42 +778,7 @@ void hubp1_read_state(struct dcn10_hubp *hubp1,
 			QoS_LEVEL_HIGH_WM, &s->qos_level_high_wm);
 }
 
-enum cursor_pitch {
-	CURSOR_PITCH_64_PIXELS = 0,
-	CURSOR_PITCH_128_PIXELS,
-	CURSOR_PITCH_256_PIXELS
-};
-
-enum cursor_lines_per_chunk {
-	CURSOR_LINE_PER_CHUNK_2 = 1,
-	CURSOR_LINE_PER_CHUNK_4,
-	CURSOR_LINE_PER_CHUNK_8,
-	CURSOR_LINE_PER_CHUNK_16
-};
-
-static bool ippn10_cursor_program_control(
-		struct dcn10_hubp *hubp1,
-		bool pixel_data_invert,
-		enum dc_cursor_color_format color_format)
-{
-	if (REG(CURSOR_SETTINS))
-		REG_SET_2(CURSOR_SETTINS, 0,
-				/* no shift of the cursor HDL schedule */
-				CURSOR0_DST_Y_OFFSET, 0,
-				 /* used to shift the cursor chunk request deadline */
-				CURSOR0_CHUNK_HDL_ADJUST, 3);
-	else
-		REG_SET_2(CURSOR_SETTINGS, 0,
-				/* no shift of the cursor HDL schedule */
-				CURSOR0_DST_Y_OFFSET, 0,
-				 /* used to shift the cursor chunk request deadline */
-				CURSOR0_CHUNK_HDL_ADJUST, 3);
-
-	return true;
-}
-
-static enum cursor_pitch ippn10_get_cursor_pitch(
-		unsigned int pitch)
+enum cursor_pitch hubp1_get_cursor_pitch(unsigned int pitch)
 {
 	enum cursor_pitch hw_pitch;
 
@@ -815,7 +801,7 @@ static enum cursor_pitch ippn10_get_cursor_pitch(
 	return hw_pitch;
 }
 
-static enum cursor_lines_per_chunk ippn10_get_lines_per_chunk(
+static enum cursor_lines_per_chunk hubp1_get_lines_per_chunk(
 		unsigned int cur_width,
 		enum dc_cursor_color_format format)
 {
@@ -841,8 +827,8 @@ void hubp1_cursor_set_attributes(
 		const struct dc_cursor_attributes *attr)
 {
 	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
-	enum cursor_pitch hw_pitch = ippn10_get_cursor_pitch(attr->pitch);
-	enum cursor_lines_per_chunk lpc = ippn10_get_lines_per_chunk(
+	enum cursor_pitch hw_pitch = hubp1_get_cursor_pitch(attr->pitch);
+	enum cursor_lines_per_chunk lpc = hubp1_get_lines_per_chunk(
 			attr->width, attr->color_format);
 
 	hubp->curs_attr = *attr;
@@ -855,13 +841,17 @@ void hubp1_cursor_set_attributes(
 	REG_UPDATE_2(CURSOR_SIZE,
 			CURSOR_WIDTH, attr->width,
 			CURSOR_HEIGHT, attr->height);
+
 	REG_UPDATE_3(CURSOR_CONTROL,
 			CURSOR_MODE, attr->color_format,
 			CURSOR_PITCH, hw_pitch,
 			CURSOR_LINES_PER_CHUNK, lpc);
-	ippn10_cursor_program_control(hubp1,
-			attr->attribute_flags.bits.INVERT_PIXEL_DATA,
-			attr->color_format);
+
+	REG_SET_2(CURSOR_SETTINS, 0,
+			/* no shift of the cursor HDL schedule */
+			CURSOR0_DST_Y_OFFSET, 0,
+			 /* used to shift the cursor chunk request deadline */
+			CURSOR0_CHUNK_HDL_ADJUST, 3);
 }
 
 void hubp1_cursor_set_position(
@@ -901,7 +891,8 @@ void hubp1_cursor_set_position(
 		cur_en = 0;  /* not visible beyond left edge*/
 
 	if (cur_en && REG_READ(CURSOR_SURFACE_ADDRESS) == 0)
-		hubp1_cursor_set_attributes(hubp, &hubp->curs_attr);
+		hubp->funcs->set_cursor_attributes(hubp, &hubp->curs_attr);
+
 	REG_UPDATE(CURSOR_CONTROL,
 			CURSOR_ENABLE, cur_en);
 
@@ -933,6 +924,7 @@ static struct hubp_funcs dcn10_hubp_funcs = {
 	.set_hubp_blank_en = hubp1_set_hubp_blank_en,
 	.set_cursor_attributes	= hubp1_cursor_set_attributes,
 	.set_cursor_position	= hubp1_cursor_set_position,
+	.hubp_disconnect = hubp1_disconnect,
 };
 
 /*****************************************/
@@ -943,15 +935,15 @@ void dcn10_hubp_construct(
 	struct dcn10_hubp *hubp1,
 	struct dc_context *ctx,
 	uint32_t inst,
-	const struct dcn_mi_registers *mi_regs,
-	const struct dcn_mi_shift *mi_shift,
-	const struct dcn_mi_mask *mi_mask)
+	const struct dcn_mi_registers *hubp_regs,
+	const struct dcn_mi_shift *hubp_shift,
+	const struct dcn_mi_mask *hubp_mask)
 {
 	hubp1->base.funcs = &dcn10_hubp_funcs;
 	hubp1->base.ctx = ctx;
-	hubp1->mi_regs = mi_regs;
-	hubp1->mi_shift = mi_shift;
-	hubp1->mi_mask = mi_mask;
+	hubp1->hubp_regs = hubp_regs;
+	hubp1->hubp_shift = hubp_shift;
+	hubp1->hubp_mask = hubp_mask;
 	hubp1->base.inst = inst;
 	hubp1->base.opp_id = 0xf;
 	hubp1->base.mpcc_id = 0xf;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
index 66db453c801b..33e91d9c010f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
@@ -30,7 +30,7 @@
 #define TO_DCN10_HUBP(hubp)\
 	container_of(hubp, struct dcn10_hubp, base)
 
-#define MI_REG_LIST_DCN(id)\
+#define HUBP_REG_LIST_DCN(id)\
 	SRI(DCHUBP_CNTL, HUBP, id),\
 	SRI(HUBPREQ_DEBUG_DB, HUBP, id),\
 	SRI(DCSURF_ADDR_CONFIG, HUBP, id),\
@@ -98,8 +98,8 @@
 	SRI(DCN_SURF1_TTU_CNTL1, HUBPREQ, id),\
 	SRI(DCN_VM_MX_L1_TLB_CNTL, HUBPREQ, id)
 
-#define MI_REG_LIST_DCN10(id)\
-	MI_REG_LIST_DCN(id),\
+#define HUBP_REG_LIST_DCN10(id)\
+	HUBP_REG_LIST_DCN(id),\
 	SRI(PREFETCH_SETTINS, HUBPREQ, id),\
 	SRI(PREFETCH_SETTINS_C, HUBPREQ, id),\
 	SRI(DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB, HUBPREQ, id),\
@@ -127,277 +127,274 @@
 	SRI(CURSOR_HOT_SPOT, CURSOR, id), \
 	SRI(CURSOR_DST_OFFSET, CURSOR, id)
 
-
-
-struct dcn_mi_registers {
-	uint32_t DCHUBP_CNTL;
-	uint32_t HUBPREQ_DEBUG_DB;
-	uint32_t DCSURF_ADDR_CONFIG;
-	uint32_t DCSURF_TILING_CONFIG;
-	uint32_t DCSURF_SURFACE_PITCH;
-	uint32_t DCSURF_SURFACE_PITCH_C;
-	uint32_t DCSURF_SURFACE_CONFIG;
-	uint32_t DCSURF_FLIP_CONTROL;
-	uint32_t DCSURF_PRI_VIEWPORT_DIMENSION;
-	uint32_t DCSURF_PRI_VIEWPORT_START;
-	uint32_t DCSURF_SEC_VIEWPORT_DIMENSION;
-	uint32_t DCSURF_SEC_VIEWPORT_START;
-	uint32_t DCSURF_PRI_VIEWPORT_DIMENSION_C;
-	uint32_t DCSURF_PRI_VIEWPORT_START_C;
-	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH;
-	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS;
-	uint32_t DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH;
-	uint32_t DCSURF_SECONDARY_SURFACE_ADDRESS;
-	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH;
-	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS;
-	uint32_t DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH;
-	uint32_t DCSURF_SECONDARY_META_SURFACE_ADDRESS;
-	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C;
-	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS_C;
-	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C;
-	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS_C;
-	uint32_t DCSURF_SURFACE_INUSE;
-	uint32_t DCSURF_SURFACE_INUSE_HIGH;
-	uint32_t DCSURF_SURFACE_INUSE_C;
-	uint32_t DCSURF_SURFACE_INUSE_HIGH_C;
-	uint32_t DCSURF_SURFACE_EARLIEST_INUSE;
-	uint32_t DCSURF_SURFACE_EARLIEST_INUSE_HIGH;
-	uint32_t DCSURF_SURFACE_EARLIEST_INUSE_C;
-	uint32_t DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C;
-	uint32_t DCSURF_SURFACE_CONTROL;
-	uint32_t HUBPRET_CONTROL;
-	uint32_t DCN_EXPANSION_MODE;
-	uint32_t DCHUBP_REQ_SIZE_CONFIG;
-	uint32_t DCHUBP_REQ_SIZE_CONFIG_C;
-	uint32_t BLANK_OFFSET_0;
-	uint32_t BLANK_OFFSET_1;
-	uint32_t DST_DIMENSIONS;
-	uint32_t DST_AFTER_SCALER;
-	uint32_t PREFETCH_SETTINS;
-	uint32_t PREFETCH_SETTINGS;
-	uint32_t VBLANK_PARAMETERS_0;
-	uint32_t REF_FREQ_TO_PIX_FREQ;
-	uint32_t VBLANK_PARAMETERS_1;
-	uint32_t VBLANK_PARAMETERS_3;
-	uint32_t NOM_PARAMETERS_0;
-	uint32_t NOM_PARAMETERS_1;
-	uint32_t NOM_PARAMETERS_4;
-	uint32_t NOM_PARAMETERS_5;
-	uint32_t PER_LINE_DELIVERY_PRE;
-	uint32_t PER_LINE_DELIVERY;
-	uint32_t PREFETCH_SETTINS_C;
-	uint32_t PREFETCH_SETTINGS_C;
-	uint32_t VBLANK_PARAMETERS_2;
-	uint32_t VBLANK_PARAMETERS_4;
-	uint32_t NOM_PARAMETERS_2;
-	uint32_t NOM_PARAMETERS_3;
-	uint32_t NOM_PARAMETERS_6;
-	uint32_t NOM_PARAMETERS_7;
-	uint32_t DCN_TTU_QOS_WM;
-	uint32_t DCN_GLOBAL_TTU_CNTL;
-	uint32_t DCN_SURF0_TTU_CNTL0;
-	uint32_t DCN_SURF0_TTU_CNTL1;
-	uint32_t DCN_SURF1_TTU_CNTL0;
-	uint32_t DCN_SURF1_TTU_CNTL1;
-	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB;
-	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB;
-	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB;
-	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB;
-	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB;
-	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB;
-	uint32_t DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB;
-	uint32_t DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB;
-	uint32_t DCN_VM_MX_L1_TLB_CNTL;
-	uint32_t DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB;
-	uint32_t DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB;
-	uint32_t DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB;
-	uint32_t DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB;
-	uint32_t DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB;
-	uint32_t DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB;
-	uint32_t DCN_VM_SYSTEM_APERTURE_LOW_ADDR;
-	uint32_t DCN_VM_SYSTEM_APERTURE_HIGH_ADDR;
-	uint32_t DCHUBBUB_SDPIF_FB_BASE;
-	uint32_t DCHUBBUB_SDPIF_FB_OFFSET;
-	uint32_t DCN_VM_FB_LOCATION_TOP;
-	uint32_t DCN_VM_FB_LOCATION_BASE;
-	uint32_t DCN_VM_FB_OFFSET;
-	uint32_t DCN_VM_AGP_BASE;
-	uint32_t DCN_VM_AGP_BOT;
-	uint32_t DCN_VM_AGP_TOP;
-	uint32_t CURSOR_SETTINS;
-	uint32_t CURSOR_SETTINGS;
-	uint32_t CURSOR_SURFACE_ADDRESS_HIGH;
-	uint32_t CURSOR_SURFACE_ADDRESS;
-	uint32_t CURSOR_SIZE;
-	uint32_t CURSOR_CONTROL;
-	uint32_t CURSOR_POSITION;
-	uint32_t CURSOR_HOT_SPOT;
-	uint32_t CURSOR_DST_OFFSET;
-};
-
-#define MI_SF(reg_name, field_name, post_fix)\
+#define HUBP_COMMON_REG_VARIABLE_LIST \
+	uint32_t DCHUBP_CNTL; \
+	uint32_t HUBPREQ_DEBUG_DB; \
+	uint32_t DCSURF_ADDR_CONFIG; \
+	uint32_t DCSURF_TILING_CONFIG; \
+	uint32_t DCSURF_SURFACE_PITCH; \
+	uint32_t DCSURF_SURFACE_PITCH_C; \
+	uint32_t DCSURF_SURFACE_CONFIG; \
+	uint32_t DCSURF_FLIP_CONTROL; \
+	uint32_t DCSURF_PRI_VIEWPORT_DIMENSION; \
+	uint32_t DCSURF_PRI_VIEWPORT_START; \
+	uint32_t DCSURF_SEC_VIEWPORT_DIMENSION; \
+	uint32_t DCSURF_SEC_VIEWPORT_START; \
+	uint32_t DCSURF_PRI_VIEWPORT_DIMENSION_C; \
+	uint32_t DCSURF_PRI_VIEWPORT_START_C; \
+	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH; \
+	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS; \
+	uint32_t DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH; \
+	uint32_t DCSURF_SECONDARY_SURFACE_ADDRESS; \
+	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH; \
+	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS; \
+	uint32_t DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH; \
+	uint32_t DCSURF_SECONDARY_META_SURFACE_ADDRESS; \
+	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C; \
+	uint32_t DCSURF_PRIMARY_SURFACE_ADDRESS_C; \
+	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C; \
+	uint32_t DCSURF_PRIMARY_META_SURFACE_ADDRESS_C; \
+	uint32_t DCSURF_SURFACE_INUSE; \
+	uint32_t DCSURF_SURFACE_INUSE_HIGH; \
+	uint32_t DCSURF_SURFACE_INUSE_C; \
+	uint32_t DCSURF_SURFACE_INUSE_HIGH_C; \
+	uint32_t DCSURF_SURFACE_EARLIEST_INUSE; \
+	uint32_t DCSURF_SURFACE_EARLIEST_INUSE_HIGH; \
+	uint32_t DCSURF_SURFACE_EARLIEST_INUSE_C; \
+	uint32_t DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C; \
+	uint32_t DCSURF_SURFACE_CONTROL; \
+	uint32_t HUBPRET_CONTROL; \
+	uint32_t DCN_EXPANSION_MODE; \
+	uint32_t DCHUBP_REQ_SIZE_CONFIG; \
+	uint32_t DCHUBP_REQ_SIZE_CONFIG_C; \
+	uint32_t BLANK_OFFSET_0; \
+	uint32_t BLANK_OFFSET_1; \
+	uint32_t DST_DIMENSIONS; \
+	uint32_t DST_AFTER_SCALER; \
+	uint32_t PREFETCH_SETTINS; \
+	uint32_t PREFETCH_SETTINGS; \
+	uint32_t VBLANK_PARAMETERS_0; \
+	uint32_t REF_FREQ_TO_PIX_FREQ; \
+	uint32_t VBLANK_PARAMETERS_1; \
+	uint32_t VBLANK_PARAMETERS_3; \
+	uint32_t NOM_PARAMETERS_0; \
+	uint32_t NOM_PARAMETERS_1; \
+	uint32_t NOM_PARAMETERS_4; \
+	uint32_t NOM_PARAMETERS_5; \
+	uint32_t PER_LINE_DELIVERY_PRE; \
+	uint32_t PER_LINE_DELIVERY; \
+	uint32_t PREFETCH_SETTINS_C; \
+	uint32_t PREFETCH_SETTINGS_C; \
+	uint32_t VBLANK_PARAMETERS_2; \
+	uint32_t VBLANK_PARAMETERS_4; \
+	uint32_t NOM_PARAMETERS_2; \
+	uint32_t NOM_PARAMETERS_3; \
+	uint32_t NOM_PARAMETERS_6; \
+	uint32_t NOM_PARAMETERS_7; \
+	uint32_t DCN_TTU_QOS_WM; \
+	uint32_t DCN_GLOBAL_TTU_CNTL; \
+	uint32_t DCN_SURF0_TTU_CNTL0; \
+	uint32_t DCN_SURF0_TTU_CNTL1; \
+	uint32_t DCN_SURF1_TTU_CNTL0; \
+	uint32_t DCN_SURF1_TTU_CNTL1; \
+	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB; \
+	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB; \
+	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB; \
+	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB; \
+	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB; \
+	uint32_t DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB; \
+	uint32_t DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB; \
+	uint32_t DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB; \
+	uint32_t DCN_VM_MX_L1_TLB_CNTL; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_LOW_ADDR; \
+	uint32_t DCN_VM_SYSTEM_APERTURE_HIGH_ADDR; \
+	uint32_t DCHUBBUB_SDPIF_FB_BASE; \
+	uint32_t DCHUBBUB_SDPIF_FB_OFFSET; \
+	uint32_t DCN_VM_FB_LOCATION_TOP; \
+	uint32_t DCN_VM_FB_LOCATION_BASE; \
+	uint32_t DCN_VM_FB_OFFSET; \
+	uint32_t DCN_VM_AGP_BASE; \
+	uint32_t DCN_VM_AGP_BOT; \
+	uint32_t DCN_VM_AGP_TOP; \
+	uint32_t CURSOR_SETTINS; \
+	uint32_t CURSOR_SETTINGS; \
+	uint32_t CURSOR_SURFACE_ADDRESS_HIGH; \
+	uint32_t CURSOR_SURFACE_ADDRESS; \
+	uint32_t CURSOR_SIZE; \
+	uint32_t CURSOR_CONTROL; \
+	uint32_t CURSOR_POSITION; \
+	uint32_t CURSOR_HOT_SPOT; \
+	uint32_t CURSOR_DST_OFFSET
+
+#define HUBP_SF(reg_name, field_name, post_fix)\
 	.field_name = reg_name ## __ ## field_name ## post_fix
 
-#define MI_MASK_SH_LIST_DCN(mask_sh)\
-	MI_SF(HUBP0_DCHUBP_CNTL, HUBP_BLANK_EN, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_CNTL, HUBP_TTU_DISABLE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_CNTL, HUBP_UNDERFLOW_STATUS, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_CNTL, HUBP_NO_OUTSTANDING_REQ, mask_sh),\
-	MI_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_PIPES, mask_sh),\
-	MI_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_BANKS, mask_sh),\
-	MI_SF(HUBP0_DCSURF_ADDR_CONFIG, PIPE_INTERLEAVE, mask_sh),\
-	MI_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_SE, mask_sh),\
-	MI_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_RB_PER_SE, mask_sh),\
-	MI_SF(HUBP0_DCSURF_ADDR_CONFIG, MAX_COMPRESSED_FRAGS, mask_sh),\
-	MI_SF(HUBP0_DCSURF_TILING_CONFIG, SW_MODE, mask_sh),\
-	MI_SF(HUBP0_DCSURF_TILING_CONFIG, META_LINEAR, mask_sh),\
-	MI_SF(HUBP0_DCSURF_TILING_CONFIG, RB_ALIGNED, mask_sh),\
-	MI_SF(HUBP0_DCSURF_TILING_CONFIG, PIPE_ALIGNED, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_PITCH, META_PITCH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_PITCH_C, PITCH_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_PITCH_C, META_PITCH_C, mask_sh),\
-	MI_SF(HUBP0_DCSURF_SURFACE_CONFIG, ROTATION_ANGLE, mask_sh),\
-	MI_SF(HUBP0_DCSURF_SURFACE_CONFIG, H_MIRROR_EN, mask_sh),\
-	MI_SF(HUBP0_DCSURF_SURFACE_CONFIG, SURFACE_PIXEL_FORMAT, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_FLIP_TYPE, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_FLIP_PENDING, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_UPDATE_LOCK, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_START, PRI_VIEWPORT_X_START, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_START, PRI_VIEWPORT_Y_START, mask_sh),\
-	MI_SF(HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION, SEC_VIEWPORT_WIDTH, mask_sh),\
-	MI_SF(HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION, SEC_VIEWPORT_HEIGHT, mask_sh),\
-	MI_SF(HUBP0_DCSURF_SEC_VIEWPORT_START, SEC_VIEWPORT_X_START, mask_sh),\
-	MI_SF(HUBP0_DCSURF_SEC_VIEWPORT_START, SEC_VIEWPORT_Y_START, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C, PRI_VIEWPORT_WIDTH_C, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C, PRI_VIEWPORT_HEIGHT_C, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_START_C, PRI_VIEWPORT_X_START_C, mask_sh),\
-	MI_SF(HUBP0_DCSURF_PRI_VIEWPORT_START_C, PRI_VIEWPORT_Y_START_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, PRIMARY_SURFACE_ADDRESS_HIGH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS, PRIMARY_SURFACE_ADDRESS, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, SECONDARY_SURFACE_ADDRESS_HIGH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS, SECONDARY_SURFACE_ADDRESS, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, PRIMARY_META_SURFACE_ADDRESS_HIGH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS, PRIMARY_META_SURFACE_ADDRESS, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, SECONDARY_META_SURFACE_ADDRESS_HIGH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS, SECONDARY_META_SURFACE_ADDRESS, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, PRIMARY_SURFACE_ADDRESS_HIGH_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C, PRIMARY_SURFACE_ADDRESS_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, PRIMARY_META_SURFACE_ADDRESS_HIGH_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, PRIMARY_META_SURFACE_ADDRESS_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_INUSE, SURFACE_INUSE_ADDRESS, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH, SURFACE_INUSE_ADDRESS_HIGH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_INUSE_C, SURFACE_INUSE_ADDRESS_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C, SURFACE_INUSE_ADDRESS_HIGH_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE, SURFACE_EARLIEST_INUSE_ADDRESS, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH, SURFACE_EARLIEST_INUSE_ADDRESS_HIGH, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C, SURFACE_EARLIEST_INUSE_ADDRESS_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C, SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_CONTROL, PRIMARY_SURFACE_TMZ, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_CONTROL, PRIMARY_SURFACE_DCC_EN, mask_sh),\
-	MI_SF(HUBPREQ0_DCSURF_SURFACE_CONTROL, PRIMARY_SURFACE_DCC_IND_64B_BLK, mask_sh),\
-	MI_SF(HUBPRET0_HUBPRET_CONTROL, DET_BUF_PLANE1_BASE_ADDRESS, mask_sh),\
-	MI_SF(HUBPRET0_HUBPRET_CONTROL, CROSSBAR_SRC_CB_B, mask_sh),\
-	MI_SF(HUBPRET0_HUBPRET_CONTROL, CROSSBAR_SRC_CR_R, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_EXPANSION_MODE, DRQ_EXPANSION_MODE, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_EXPANSION_MODE, PRQ_EXPANSION_MODE, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_EXPANSION_MODE, MRQ_EXPANSION_MODE, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_EXPANSION_MODE, CRQ_EXPANSION_MODE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, CHUNK_SIZE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MIN_CHUNK_SIZE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, META_CHUNK_SIZE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MIN_META_CHUNK_SIZE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, DPTE_GROUP_SIZE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MPTE_GROUP_SIZE, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, SWATH_HEIGHT, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, PTE_ROW_HEIGHT_LINEAR, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, CHUNK_SIZE_C, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MIN_CHUNK_SIZE_C, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, META_CHUNK_SIZE_C, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MIN_META_CHUNK_SIZE_C, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, DPTE_GROUP_SIZE_C, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MPTE_GROUP_SIZE_C, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, SWATH_HEIGHT_C, mask_sh),\
-	MI_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, PTE_ROW_HEIGHT_LINEAR_C, mask_sh),\
-	MI_SF(HUBPREQ0_BLANK_OFFSET_0, REFCYC_H_BLANK_END, mask_sh),\
-	MI_SF(HUBPREQ0_BLANK_OFFSET_0, DLG_V_BLANK_END, mask_sh),\
-	MI_SF(HUBPREQ0_BLANK_OFFSET_1, MIN_DST_Y_NEXT_START, mask_sh),\
-	MI_SF(HUBPREQ0_DST_DIMENSIONS, REFCYC_PER_HTOTAL, mask_sh),\
-	MI_SF(HUBPREQ0_DST_AFTER_SCALER, REFCYC_X_AFTER_SCALER, mask_sh),\
-	MI_SF(HUBPREQ0_DST_AFTER_SCALER, DST_Y_AFTER_SCALER, mask_sh),\
-	MI_SF(HUBPREQ0_VBLANK_PARAMETERS_0, DST_Y_PER_VM_VBLANK, mask_sh),\
-	MI_SF(HUBPREQ0_VBLANK_PARAMETERS_0, DST_Y_PER_ROW_VBLANK, mask_sh),\
-	MI_SF(HUBPREQ0_REF_FREQ_TO_PIX_FREQ, REF_FREQ_TO_PIX_FREQ, mask_sh),\
-	MI_SF(HUBPREQ0_VBLANK_PARAMETERS_1, REFCYC_PER_PTE_GROUP_VBLANK_L, mask_sh),\
-	MI_SF(HUBPREQ0_VBLANK_PARAMETERS_3, REFCYC_PER_META_CHUNK_VBLANK_L, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_0, DST_Y_PER_PTE_ROW_NOM_L, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_1, REFCYC_PER_PTE_GROUP_NOM_L, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_4, DST_Y_PER_META_ROW_NOM_L, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_5, REFCYC_PER_META_CHUNK_NOM_L, mask_sh),\
-	MI_SF(HUBPREQ0_PER_LINE_DELIVERY_PRE, REFCYC_PER_LINE_DELIVERY_PRE_L, mask_sh),\
-	MI_SF(HUBPREQ0_PER_LINE_DELIVERY_PRE, REFCYC_PER_LINE_DELIVERY_PRE_C, mask_sh),\
-	MI_SF(HUBPREQ0_PER_LINE_DELIVERY, REFCYC_PER_LINE_DELIVERY_L, mask_sh),\
-	MI_SF(HUBPREQ0_PER_LINE_DELIVERY, REFCYC_PER_LINE_DELIVERY_C, mask_sh),\
-	MI_SF(HUBPREQ0_VBLANK_PARAMETERS_2, REFCYC_PER_PTE_GROUP_VBLANK_C, mask_sh),\
-	MI_SF(HUBPREQ0_VBLANK_PARAMETERS_4, REFCYC_PER_META_CHUNK_VBLANK_C, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_2, DST_Y_PER_PTE_ROW_NOM_C, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_3, REFCYC_PER_PTE_GROUP_NOM_C, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_6, DST_Y_PER_META_ROW_NOM_C, mask_sh),\
-	MI_SF(HUBPREQ0_NOM_PARAMETERS_7, REFCYC_PER_META_CHUNK_NOM_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_TTU_QOS_WM, QoS_LEVEL_LOW_WM, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_TTU_QOS_WM, QoS_LEVEL_HIGH_WM, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_GLOBAL_TTU_CNTL, MIN_TTU_VBLANK, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_GLOBAL_TTU_CNTL, QoS_LEVEL_FLIP, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, REFCYC_PER_REQ_DELIVERY, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, QoS_LEVEL_FIXED, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, QoS_RAMP_DISABLE, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL1, REFCYC_PER_REQ_DELIVERY_PRE, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, mask_sh)
-
-#define MI_MASK_SH_LIST_DCN10(mask_sh)\
-	MI_MASK_SH_LIST_DCN(mask_sh),\
-	MI_SF(HUBPREQ0_PREFETCH_SETTINS, DST_Y_PREFETCH, mask_sh),\
-	MI_SF(HUBPREQ0_PREFETCH_SETTINS, VRATIO_PREFETCH, mask_sh),\
-	MI_SF(HUBPREQ0_PREFETCH_SETTINS_C, VRATIO_PREFETCH_C, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB, VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB, VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB, VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB, VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB, VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB, VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB, VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB, VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_SYSTEM, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB, VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB, MC_VM_SYSTEM_APERTURE_LOW_ADDR_MSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB, MC_VM_SYSTEM_APERTURE_LOW_ADDR_LSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB, MC_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB, MC_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB, mask_sh),\
-	MI_SF(DCHUBBUB_SDPIF_FB_BASE, SDPIF_FB_BASE, mask_sh),\
-	MI_SF(DCHUBBUB_SDPIF_FB_OFFSET, SDPIF_FB_OFFSET, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, MC_VM_SYSTEM_APERTURE_DEFAULT_SYSTEM, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, mask_sh),\
-	MI_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, mask_sh),\
-	MI_SF(HUBPREQ0_CURSOR_SETTINS, CURSOR0_DST_Y_OFFSET, mask_sh), \
-	MI_SF(HUBPREQ0_CURSOR_SETTINS, CURSOR0_CHUNK_HDL_ADJUST, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_SURFACE_ADDRESS_HIGH, CURSOR_SURFACE_ADDRESS_HIGH, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_SURFACE_ADDRESS, CURSOR_SURFACE_ADDRESS, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_SIZE, CURSOR_WIDTH, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_SIZE, CURSOR_HEIGHT, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_CONTROL, CURSOR_MODE, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_CONTROL, CURSOR_2X_MAGNIFY, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_CONTROL, CURSOR_PITCH, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_CONTROL, CURSOR_LINES_PER_CHUNK, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_CONTROL, CURSOR_ENABLE, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_POSITION, CURSOR_X_POSITION, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_POSITION, CURSOR_Y_POSITION, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_X, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_Y, mask_sh), \
-	MI_SF(CURSOR0_CURSOR_DST_OFFSET, CURSOR_DST_X_OFFSET, mask_sh)
-
-#define DCN_MI_REG_FIELD_LIST(type) \
+#define HUBP_MASK_SH_LIST_DCN(mask_sh)\
+	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_BLANK_EN, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_TTU_DISABLE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_UNDERFLOW_STATUS, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_NO_OUTSTANDING_REQ, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_PIPES, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_BANKS, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, PIPE_INTERLEAVE, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_SE, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_RB_PER_SE, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, MAX_COMPRESSED_FRAGS, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_TILING_CONFIG, SW_MODE, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_TILING_CONFIG, META_LINEAR, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_TILING_CONFIG, RB_ALIGNED, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_TILING_CONFIG, PIPE_ALIGNED, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_PITCH, META_PITCH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_PITCH_C, PITCH_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_PITCH_C, META_PITCH_C, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, ROTATION_ANGLE, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, H_MIRROR_EN, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SURFACE_CONFIG, SURFACE_PIXEL_FORMAT, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_FLIP_TYPE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_FLIP_PENDING, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_FLIP_CONTROL, SURFACE_UPDATE_LOCK, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_START, PRI_VIEWPORT_X_START, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_START, PRI_VIEWPORT_Y_START, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION, SEC_VIEWPORT_WIDTH, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SEC_VIEWPORT_DIMENSION, SEC_VIEWPORT_HEIGHT, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SEC_VIEWPORT_START, SEC_VIEWPORT_X_START, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_SEC_VIEWPORT_START, SEC_VIEWPORT_Y_START, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C, PRI_VIEWPORT_WIDTH_C, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C, PRI_VIEWPORT_HEIGHT_C, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_START_C, PRI_VIEWPORT_X_START_C, mask_sh),\
+	HUBP_SF(HUBP0_DCSURF_PRI_VIEWPORT_START_C, PRI_VIEWPORT_Y_START_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, PRIMARY_SURFACE_ADDRESS_HIGH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS, PRIMARY_SURFACE_ADDRESS, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, SECONDARY_SURFACE_ADDRESS_HIGH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS, SECONDARY_SURFACE_ADDRESS, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, PRIMARY_META_SURFACE_ADDRESS_HIGH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS, PRIMARY_META_SURFACE_ADDRESS, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, SECONDARY_META_SURFACE_ADDRESS_HIGH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS, SECONDARY_META_SURFACE_ADDRESS, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, PRIMARY_SURFACE_ADDRESS_HIGH_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C, PRIMARY_SURFACE_ADDRESS_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, PRIMARY_META_SURFACE_ADDRESS_HIGH_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, PRIMARY_META_SURFACE_ADDRESS_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_INUSE, SURFACE_INUSE_ADDRESS, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH, SURFACE_INUSE_ADDRESS_HIGH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_INUSE_C, SURFACE_INUSE_ADDRESS_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C, SURFACE_INUSE_ADDRESS_HIGH_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE, SURFACE_EARLIEST_INUSE_ADDRESS, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH, SURFACE_EARLIEST_INUSE_ADDRESS_HIGH, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C, SURFACE_EARLIEST_INUSE_ADDRESS_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C, SURFACE_EARLIEST_INUSE_ADDRESS_HIGH_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_CONTROL, PRIMARY_SURFACE_TMZ, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_CONTROL, PRIMARY_SURFACE_DCC_EN, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCSURF_SURFACE_CONTROL, PRIMARY_SURFACE_DCC_IND_64B_BLK, mask_sh),\
+	HUBP_SF(HUBPRET0_HUBPRET_CONTROL, DET_BUF_PLANE1_BASE_ADDRESS, mask_sh),\
+	HUBP_SF(HUBPRET0_HUBPRET_CONTROL, CROSSBAR_SRC_CB_B, mask_sh),\
+	HUBP_SF(HUBPRET0_HUBPRET_CONTROL, CROSSBAR_SRC_CR_R, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_EXPANSION_MODE, DRQ_EXPANSION_MODE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_EXPANSION_MODE, PRQ_EXPANSION_MODE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_EXPANSION_MODE, MRQ_EXPANSION_MODE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_EXPANSION_MODE, CRQ_EXPANSION_MODE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, CHUNK_SIZE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MIN_CHUNK_SIZE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, META_CHUNK_SIZE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MIN_META_CHUNK_SIZE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, DPTE_GROUP_SIZE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MPTE_GROUP_SIZE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, SWATH_HEIGHT, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, PTE_ROW_HEIGHT_LINEAR, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, CHUNK_SIZE_C, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MIN_CHUNK_SIZE_C, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, META_CHUNK_SIZE_C, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MIN_META_CHUNK_SIZE_C, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, DPTE_GROUP_SIZE_C, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MPTE_GROUP_SIZE_C, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, SWATH_HEIGHT_C, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, PTE_ROW_HEIGHT_LINEAR_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_BLANK_OFFSET_0, REFCYC_H_BLANK_END, mask_sh),\
+	HUBP_SF(HUBPREQ0_BLANK_OFFSET_0, DLG_V_BLANK_END, mask_sh),\
+	HUBP_SF(HUBPREQ0_BLANK_OFFSET_1, MIN_DST_Y_NEXT_START, mask_sh),\
+	HUBP_SF(HUBPREQ0_DST_DIMENSIONS, REFCYC_PER_HTOTAL, mask_sh),\
+	HUBP_SF(HUBPREQ0_DST_AFTER_SCALER, REFCYC_X_AFTER_SCALER, mask_sh),\
+	HUBP_SF(HUBPREQ0_DST_AFTER_SCALER, DST_Y_AFTER_SCALER, mask_sh),\
+	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_0, DST_Y_PER_VM_VBLANK, mask_sh),\
+	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_0, DST_Y_PER_ROW_VBLANK, mask_sh),\
+	HUBP_SF(HUBPREQ0_REF_FREQ_TO_PIX_FREQ, REF_FREQ_TO_PIX_FREQ, mask_sh),\
+	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_1, REFCYC_PER_PTE_GROUP_VBLANK_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_3, REFCYC_PER_META_CHUNK_VBLANK_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_0, DST_Y_PER_PTE_ROW_NOM_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_1, REFCYC_PER_PTE_GROUP_NOM_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_4, DST_Y_PER_META_ROW_NOM_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_5, REFCYC_PER_META_CHUNK_NOM_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_PER_LINE_DELIVERY_PRE, REFCYC_PER_LINE_DELIVERY_PRE_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_PER_LINE_DELIVERY_PRE, REFCYC_PER_LINE_DELIVERY_PRE_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_PER_LINE_DELIVERY, REFCYC_PER_LINE_DELIVERY_L, mask_sh),\
+	HUBP_SF(HUBPREQ0_PER_LINE_DELIVERY, REFCYC_PER_LINE_DELIVERY_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_2, REFCYC_PER_PTE_GROUP_VBLANK_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_VBLANK_PARAMETERS_4, REFCYC_PER_META_CHUNK_VBLANK_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_2, DST_Y_PER_PTE_ROW_NOM_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_3, REFCYC_PER_PTE_GROUP_NOM_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_6, DST_Y_PER_META_ROW_NOM_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_7, REFCYC_PER_META_CHUNK_NOM_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_TTU_QOS_WM, QoS_LEVEL_LOW_WM, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_TTU_QOS_WM, QoS_LEVEL_HIGH_WM, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_GLOBAL_TTU_CNTL, MIN_TTU_VBLANK, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_GLOBAL_TTU_CNTL, QoS_LEVEL_FLIP, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, REFCYC_PER_REQ_DELIVERY, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, QoS_LEVEL_FIXED, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL0, QoS_RAMP_DISABLE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL1, REFCYC_PER_REQ_DELIVERY_PRE, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, mask_sh)
+
+#define HUBP_MASK_SH_LIST_DCN10(mask_sh)\
+	HUBP_MASK_SH_LIST_DCN(mask_sh),\
+	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS, DST_Y_PREFETCH, mask_sh),\
+	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS, VRATIO_PREFETCH, mask_sh),\
+	HUBP_SF(HUBPREQ0_PREFETCH_SETTINS_C, VRATIO_PREFETCH_C, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB, VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB, VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB, VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB, VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB, VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB, VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB, VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB, VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_SYSTEM, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB, VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB, MC_VM_SYSTEM_APERTURE_LOW_ADDR_MSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB, MC_VM_SYSTEM_APERTURE_LOW_ADDR_LSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB, MC_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB, MC_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB, mask_sh),\
+	HUBP_SF(DCHUBBUB_SDPIF_FB_BASE, SDPIF_FB_BASE, mask_sh),\
+	HUBP_SF(DCHUBBUB_SDPIF_FB_OFFSET, SDPIF_FB_OFFSET, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, MC_VM_SYSTEM_APERTURE_DEFAULT_SYSTEM, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, mask_sh),\
+	HUBP_SF(HUBPREQ0_CURSOR_SETTINS, CURSOR0_DST_Y_OFFSET, mask_sh), \
+	HUBP_SF(HUBPREQ0_CURSOR_SETTINS, CURSOR0_CHUNK_HDL_ADJUST, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_SURFACE_ADDRESS_HIGH, CURSOR_SURFACE_ADDRESS_HIGH, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_SURFACE_ADDRESS, CURSOR_SURFACE_ADDRESS, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_SIZE, CURSOR_WIDTH, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_SIZE, CURSOR_HEIGHT, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_CONTROL, CURSOR_MODE, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_CONTROL, CURSOR_2X_MAGNIFY, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_CONTROL, CURSOR_PITCH, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_CONTROL, CURSOR_LINES_PER_CHUNK, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_CONTROL, CURSOR_ENABLE, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_POSITION, CURSOR_X_POSITION, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_POSITION, CURSOR_Y_POSITION, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_X, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_HOT_SPOT, CURSOR_HOT_SPOT_Y, mask_sh), \
+	HUBP_SF(CURSOR0_CURSOR_DST_OFFSET, CURSOR_DST_X_OFFSET, mask_sh)
+
+#define DCN_HUBP_REG_FIELD_LIST(type) \
 	type HUBP_BLANK_EN;\
 	type HUBP_TTU_DISABLE;\
 	type HUBP_NO_OUTSTANDING_REQ;\
@@ -576,19 +573,23 @@ struct dcn_mi_registers {
 	type CURSOR_DST_X_OFFSET; \
 	type OUTPUT_FP
 
+struct dcn_mi_registers {
+	HUBP_COMMON_REG_VARIABLE_LIST;
+};
+
 struct dcn_mi_shift {
-	DCN_MI_REG_FIELD_LIST(uint8_t);
+	DCN_HUBP_REG_FIELD_LIST(uint8_t);
 };
 
 struct dcn_mi_mask {
-	DCN_MI_REG_FIELD_LIST(uint32_t);
+	DCN_HUBP_REG_FIELD_LIST(uint32_t);
 };
 
 struct dcn10_hubp {
 	struct hubp base;
-	const struct dcn_mi_registers *mi_regs;
-	const struct dcn_mi_shift *mi_shift;
-	const struct dcn_mi_mask *mi_mask;
+	const struct dcn_mi_registers *hubp_regs;
+	const struct dcn_mi_shift *hubp_shift;
+	const struct dcn_mi_mask *hubp_mask;
 };
 
 void hubp1_program_surface_config(
@@ -610,11 +611,11 @@ void hubp1_program_requestor(
 		struct _vcs_dpi_display_rq_regs_st *rq_regs);
 
 void hubp1_program_pixel_format(
-	struct dcn10_hubp *hubp,
+	struct hubp *hubp,
 	enum surface_pixel_format format);
 
 void hubp1_program_size_and_rotation(
-	struct dcn10_hubp *hubp,
+	struct hubp *hubp,
 	enum dc_rotation_angle rotation,
 	enum surface_pixel_format format,
 	const union plane_size *plane_size,
@@ -622,7 +623,7 @@ void hubp1_program_size_and_rotation(
 	bool horizontal_mirror);
 
 void hubp1_program_tiling(
-	struct dcn10_hubp *hubp,
+	struct hubp *hubp,
 	const union dc_tiling_info *info,
 	const enum surface_pixel_format pixel_format);
 
@@ -656,9 +657,9 @@ void dcn10_hubp_construct(
 	struct dcn10_hubp *hubp1,
 	struct dc_context *ctx,
 	uint32_t inst,
-	const struct dcn_mi_registers *mi_regs,
-	const struct dcn_mi_shift *mi_shift,
-	const struct dcn_mi_mask *mi_mask);
+	const struct dcn_mi_registers *hubp_regs,
+	const struct dcn_mi_shift *hubp_shift,
+	const struct dcn_mi_mask *hubp_mask);
 
 
 struct dcn_hubp_state {
@@ -680,4 +681,6 @@ struct dcn_hubp_state {
 void hubp1_read_state(struct dcn10_hubp *hubp1,
 		struct dcn_hubp_state *s);
 
+enum cursor_pitch hubp1_get_cursor_pitch(unsigned int pitch);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
index 05dc01e54531..dc1e010725c1 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
@@ -31,7 +31,8 @@
 #include "dce110/dce110_hw_sequencer.h"
 #include "dce/dce_hwseq.h"
 #include "abm.h"
-#include "dcn10/dcn10_timing_generator.h"
+#include "dmcu.h"
+#include "dcn10_optc.h"
 #include "dcn10/dcn10_dpp.h"
 #include "dcn10/dcn10_mpc.h"
 #include "timing_generator.h"
@@ -41,6 +42,8 @@
 #include "reg_helper.h"
 #include "custom_float.h"
 #include "dcn10_hubp.h"
+#include "dcn10_hubbub.h"
+#include "dcn10_cm_common.h"
 
 #define CTX \
 	hws->ctx
@@ -51,18 +54,8 @@
 #define FN(reg_name, field_name) \
 	hws->shifts->field_name, hws->masks->field_name
 
-static void log_mpc_crc(struct dc *dc)
-{
-	struct dc_context *dc_ctx = dc->ctx;
-	struct dce_hwseq *hws = dc->hwseq;
-
-	if (REG(MPC_CRC_RESULT_GB))
-		DTN_INFO("MPC_CRC_RESULT_GB:%d MPC_CRC_RESULT_C:%d MPC_CRC_RESULT_AR:%d\n",
-		REG_READ(MPC_CRC_RESULT_GB), REG_READ(MPC_CRC_RESULT_C), REG_READ(MPC_CRC_RESULT_AR));
-	if (REG(DPP_TOP0_DPP_CRC_VAL_B_A))
-		DTN_INFO("DPP_TOP0_DPP_CRC_VAL_B_A:%d DPP_TOP0_DPP_CRC_VAL_R_G:%d\n",
-		REG_READ(DPP_TOP0_DPP_CRC_VAL_B_A), REG_READ(DPP_TOP0_DPP_CRC_VAL_R_G));
-}
+#define DTN_INFO_MICRO_SEC(ref_cycle) \
+	print_microsec(dc_ctx, ref_cycle)
 
 void print_microsec(struct dc_context *dc_ctx, uint32_t ref_cycle)
 {
@@ -75,67 +68,27 @@ void print_microsec(struct dc_context *dc_ctx, uint32_t ref_cycle)
 			us_x10 % frac);
 }
 
-#define DTN_INFO_MICRO_SEC(ref_cycle) \
-	print_microsec(dc_ctx, ref_cycle)
 
-struct dcn_hubbub_wm_set {
-	uint32_t wm_set;
-	uint32_t data_urgent;
-	uint32_t pte_meta_urgent;
-	uint32_t sr_enter;
-	uint32_t sr_exit;
-	uint32_t dram_clk_chanage;
-};
+static void log_mpc_crc(struct dc *dc)
+{
+	struct dc_context *dc_ctx = dc->ctx;
+	struct dce_hwseq *hws = dc->hwseq;
 
-struct dcn_hubbub_wm {
-	struct dcn_hubbub_wm_set sets[4];
-};
+	if (REG(MPC_CRC_RESULT_GB))
+		DTN_INFO("MPC_CRC_RESULT_GB:%d MPC_CRC_RESULT_C:%d MPC_CRC_RESULT_AR:%d\n",
+		REG_READ(MPC_CRC_RESULT_GB), REG_READ(MPC_CRC_RESULT_C), REG_READ(MPC_CRC_RESULT_AR));
+	if (REG(DPP_TOP0_DPP_CRC_VAL_B_A))
+		DTN_INFO("DPP_TOP0_DPP_CRC_VAL_B_A:%d DPP_TOP0_DPP_CRC_VAL_R_G:%d\n",
+		REG_READ(DPP_TOP0_DPP_CRC_VAL_B_A), REG_READ(DPP_TOP0_DPP_CRC_VAL_R_G));
+}
 
-static void dcn10_hubbub_wm_read_state(struct dce_hwseq *hws,
-		struct dcn_hubbub_wm *wm)
-{
-	struct dcn_hubbub_wm_set *s;
-
-	s = &wm->sets[0];
-	s->wm_set = 0;
-	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A);
-	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A);
-	s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A);
-	s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A);
-	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A);
-
-	s = &wm->sets[1];
-	s->wm_set = 1;
-	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B);
-	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B);
-	s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B);
-	s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B);
-	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B);
-
-	s = &wm->sets[2];
-	s->wm_set = 2;
-	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C);
-	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C);
-	s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C);
-	s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C);
-	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C);
-
-	s = &wm->sets[3];
-	s->wm_set = 3;
-	s->data_urgent = REG_READ(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D);
-	s->pte_meta_urgent = REG_READ(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D);
-	s->sr_enter = REG_READ(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D);
-	s->sr_exit = REG_READ(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D);
-	s->dram_clk_chanage = REG_READ(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D);
-}
-
-static void dcn10_log_hubbub_state(struct dc *dc)
+void dcn10_log_hubbub_state(struct dc *dc)
 {
 	struct dc_context *dc_ctx = dc->ctx;
 	struct dcn_hubbub_wm wm;
 	int i;
 
-	dcn10_hubbub_wm_read_state(dc->hwseq, &wm);
+	hubbub1_wm_read_state(dc->res_pool->hubbub, &wm);
 
 	DTN_INFO("HUBBUB WM: \t data_urgent \t pte_meta_urgent \t "
 			"sr_enter \t sr_exit \t dram_clk_change \n");
@@ -156,7 +109,7 @@ static void dcn10_log_hubbub_state(struct dc *dc)
 	DTN_INFO("\n");
 }
 
-static void dcn10_log_hw_state(struct dc *dc)
+void dcn10_log_hw_state(struct dc *dc)
 {
 	struct dc_context *dc_ctx = dc->ctx;
 	struct resource_pool *pool = dc->res_pool;
@@ -206,7 +159,7 @@ static void dcn10_log_hw_state(struct dc *dc)
 		struct timing_generator *tg = pool->timing_generators[i];
 		struct dcn_otg_state s = {0};
 
-		tgn10_read_otg_state(DCN10TG_FROM_TG(tg), &s);
+		optc1_read_otg_state(DCN10TG_FROM_TG(tg), &s);
 
 		//only print if OTG master is enabled
 		if ((s.otg_enabled & 1) == 0)
@@ -240,97 +193,6 @@ static void dcn10_log_hw_state(struct dc *dc)
 	DTN_INFO_END();
 }
 
-static void verify_allow_pstate_change_high(
-	struct dce_hwseq *hws)
-{
-	/* pstate latency is ~20us so if we wait over 40us and pstate allow
-	 * still not asserted, we are probably stuck and going to hang
-	 *
-	 * TODO: Figure out why it takes ~100us on linux
-	 * pstate takes around ~100us on linux. Unknown currently as to
-	 * why it takes that long on linux
-	 */
-	static unsigned int pstate_wait_timeout_us = 200;
-	static unsigned int pstate_wait_expected_timeout_us = 40;
-	static unsigned int max_sampled_pstate_wait_us; /* data collection */
-	static bool forced_pstate_allow; /* help with revert wa */
-	static bool should_log_hw_state; /* prevent hw state log by default */
-
-	unsigned int debug_index = 0x7;
-	unsigned int debug_data;
-	unsigned int i;
-
-	if (forced_pstate_allow) {
-		/* we hacked to force pstate allow to prevent hang last time
-		 * we verify_allow_pstate_change_high.  so disable force
-		 * here so we can check status
-		 */
-		REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
-			     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, 0,
-			     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, 0);
-		forced_pstate_allow = false;
-	}
-
-	/* description "3-0:   Pipe0 cursor0 QOS
-	 * 7-4:   Pipe1 cursor0 QOS
-	 * 11-8:  Pipe2 cursor0 QOS
-	 * 15-12: Pipe3 cursor0 QOS
-	 * 16:    Pipe0 Plane0 Allow Pstate Change
-	 * 17:    Pipe1 Plane0 Allow Pstate Change
-	 * 18:    Pipe2 Plane0 Allow Pstate Change
-	 * 19:    Pipe3 Plane0 Allow Pstate Change
-	 * 20:    Pipe0 Plane1 Allow Pstate Change
-	 * 21:    Pipe1 Plane1 Allow Pstate Change
-	 * 22:    Pipe2 Plane1 Allow Pstate Change
-	 * 23:    Pipe3 Plane1 Allow Pstate Change
-	 * 24:    Pipe0 cursor0 Allow Pstate Change
-	 * 25:    Pipe1 cursor0 Allow Pstate Change
-	 * 26:    Pipe2 cursor0 Allow Pstate Change
-	 * 27:    Pipe3 cursor0 Allow Pstate Change
-	 * 28:    WB0 Allow Pstate Change
-	 * 29:    WB1 Allow Pstate Change
-	 * 30:    Arbiter's allow_pstate_change
-	 * 31:    SOC pstate change request
-	 */
-
-	REG_WRITE(DCHUBBUB_TEST_DEBUG_INDEX, debug_index);
-
-	for (i = 0; i < pstate_wait_timeout_us; i++) {
-		debug_data = REG_READ(DCHUBBUB_TEST_DEBUG_DATA);
-
-		if (debug_data & (1 << 30)) {
-
-			if (i > pstate_wait_expected_timeout_us)
-				dm_logger_write(hws->ctx->logger, LOG_WARNING,
-						"pstate took longer than expected ~%dus\n",
-						i);
-
-			return;
-		}
-		if (max_sampled_pstate_wait_us < i)
-			max_sampled_pstate_wait_us = i;
-
-		udelay(1);
-	}
-
-	/* force pstate allow to prevent system hang
-	 * and break to debugger to investigate
-	 */
-	REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
-		     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, 1,
-		     DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, 1);
-	forced_pstate_allow = true;
-
-	if (should_log_hw_state) {
-		dcn10_log_hw_state(hws->ctx->dc);
-	}
-
-	dm_logger_write(hws->ctx->logger, LOG_WARNING,
-			"pstate TEST_DEBUG_DATA: 0x%X\n",
-			debug_data);
-	BREAK_TO_DEBUGGER();
-}
-
 static void enable_dppclk(
 	struct dce_hwseq *hws,
 	uint8_t plane_id,
@@ -376,10 +238,34 @@ static void enable_power_gating_plane(
 static void disable_vga(
 	struct dce_hwseq *hws)
 {
+	unsigned int in_vga1_mode = 0;
+	unsigned int in_vga2_mode = 0;
+	unsigned int in_vga3_mode = 0;
+	unsigned int in_vga4_mode = 0;
+
+	REG_GET(D1VGA_CONTROL, D1VGA_MODE_ENABLE, &in_vga1_mode);
+	REG_GET(D2VGA_CONTROL, D2VGA_MODE_ENABLE, &in_vga2_mode);
+	REG_GET(D3VGA_CONTROL, D3VGA_MODE_ENABLE, &in_vga3_mode);
+	REG_GET(D4VGA_CONTROL, D4VGA_MODE_ENABLE, &in_vga4_mode);
+
+	if (in_vga1_mode == 0 && in_vga2_mode == 0 &&
+			in_vga3_mode == 0 && in_vga4_mode == 0)
+		return;
+
 	REG_WRITE(D1VGA_CONTROL, 0);
 	REG_WRITE(D2VGA_CONTROL, 0);
 	REG_WRITE(D3VGA_CONTROL, 0);
 	REG_WRITE(D4VGA_CONTROL, 0);
+
+	/* HW Engineer's Notes:
+	 *  During switch from vga->extended, if we set the VGA_TEST_ENABLE and
+	 *  then hit the VGA_TEST_RENDER_START, then the DCHUBP timing gets updated correctly.
+	 *
+	 *  Then vBIOS will have it poll for the VGA_TEST_RENDER_DONE and unset
+	 *  VGA_TEST_ENABLE, to leave it in the same state as before.
+	 */
+	REG_UPDATE(VGA_TEST_CONTROL, VGA_TEST_ENABLE, 1);
+	REG_UPDATE(VGA_TEST_CONTROL, VGA_TEST_RENDER_START, 1);
 }
 
 static void dpp_pg_control(
@@ -432,312 +318,6 @@ static void dpp_pg_control(
 	}
 }
 
-static uint32_t convert_and_clamp(
-	uint32_t wm_ns,
-	uint32_t refclk_mhz,
-	uint32_t clamp_value)
-{
-	uint32_t ret_val = 0;
-	ret_val = wm_ns * refclk_mhz;
-	ret_val /= 1000;
-
-	if (ret_val > clamp_value)
-		ret_val = clamp_value;
-
-	return ret_val;
-}
-
-static void program_watermarks(
-		struct dce_hwseq *hws,
-		struct dcn_watermark_set *watermarks,
-		unsigned int refclk_mhz)
-{
-	uint32_t force_en = hws->ctx->dc->debug.disable_stutter ? 1 : 0;
-	/*
-	 * Need to clamp to max of the register values (i.e. no wrap)
-	 * for dcn1, all wm registers are 21-bit wide
-	 */
-	uint32_t prog_wm_value;
-
-	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 0);
-
-	/* Repeat for water mark set A, B, C and D. */
-	/* clock state A */
-	prog_wm_value = convert_and_clamp(watermarks->a.urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, prog_wm_value);
-
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"URGENCY_WATERMARK_A calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->a.urgent_ns, prog_wm_value);
-
-	prog_wm_value = convert_and_clamp(watermarks->a.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"PTE_META_URGENCY_WATERMARK_A calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->a.pte_meta_urgent_ns, prog_wm_value);
-
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
-
-
-		prog_wm_value = convert_and_clamp(
-				watermarks->a.cstate_pstate.cstate_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_EXIT_WATERMARK_A calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->a.cstate_pstate.cstate_exit_ns, prog_wm_value);
-	}
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->a.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
-		"HW register value = 0x%x\n\n",
-		watermarks->a.cstate_pstate.pstate_change_ns, prog_wm_value);
-
-
-	/* clock state B */
-	prog_wm_value = convert_and_clamp(
-			watermarks->b.urgent_ns, refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"URGENCY_WATERMARK_B calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->b.urgent_ns, prog_wm_value);
-
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->b.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"PTE_META_URGENCY_WATERMARK_B calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->b.pte_meta_urgent_ns, prog_wm_value);
-
-
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_ENTER_WATERMARK_B calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
-
-
-		prog_wm_value = convert_and_clamp(
-				watermarks->b.cstate_pstate.cstate_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_EXIT_WATERMARK_B calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->b.cstate_pstate.cstate_exit_ns, prog_wm_value);
-	}
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->b.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n\n"
-		"HW register value = 0x%x\n",
-		watermarks->b.cstate_pstate.pstate_change_ns, prog_wm_value);
-
-	/* clock state C */
-	prog_wm_value = convert_and_clamp(
-			watermarks->c.urgent_ns, refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"URGENCY_WATERMARK_C calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->c.urgent_ns, prog_wm_value);
-
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->c.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"PTE_META_URGENCY_WATERMARK_C calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->c.pte_meta_urgent_ns, prog_wm_value);
-
-
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_ENTER_WATERMARK_C calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
-
-
-		prog_wm_value = convert_and_clamp(
-				watermarks->c.cstate_pstate.cstate_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_EXIT_WATERMARK_C calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->c.cstate_pstate.cstate_exit_ns, prog_wm_value);
-	}
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->c.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n\n"
-		"HW register value = 0x%x\n",
-		watermarks->c.cstate_pstate.pstate_change_ns, prog_wm_value);
-
-	/* clock state D */
-	prog_wm_value = convert_and_clamp(
-			watermarks->d.urgent_ns, refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"URGENCY_WATERMARK_D calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->d.urgent_ns, prog_wm_value);
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->d.pte_meta_urgent_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"PTE_META_URGENCY_WATERMARK_D calculated =%d\n"
-		"HW register value = 0x%x\n",
-		watermarks->d.pte_meta_urgent_ns, prog_wm_value);
-
-
-	if (REG(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D)) {
-		prog_wm_value = convert_and_clamp(
-				watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_ENTER_WATERMARK_D calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
-
-
-		prog_wm_value = convert_and_clamp(
-				watermarks->d.cstate_pstate.cstate_exit_ns,
-				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
-		dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-			"SR_EXIT_WATERMARK_D calculated =%d\n"
-			"HW register value = 0x%x\n",
-			watermarks->d.cstate_pstate.cstate_exit_ns, prog_wm_value);
-	}
-
-
-	prog_wm_value = convert_and_clamp(
-			watermarks->d.cstate_pstate.pstate_change_ns,
-			refclk_mhz, 0x1fffff);
-	REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, prog_wm_value);
-	dm_logger_write(hws->ctx->logger, LOG_BANDWIDTH_CALCS,
-		"DRAM_CLK_CHANGE_WATERMARK_D calculated =%d\n"
-		"HW register value = 0x%x\n\n",
-		watermarks->d.cstate_pstate.pstate_change_ns, prog_wm_value);
-
-	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 1);
-
-	REG_UPDATE(DCHUBBUB_ARB_SAT_LEVEL,
-			DCHUBBUB_ARB_SAT_LEVEL, 60 * refclk_mhz);
-	REG_UPDATE(DCHUBBUB_ARB_DF_REQ_OUTSTAND,
-			DCHUBBUB_ARB_MIN_REQ_OUTSTAND, 68);
-
-	REG_UPDATE_2(DCHUBBUB_ARB_DRAM_STATE_CNTL,
-			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_VALUE, 0,
-			DCHUBBUB_ARB_ALLOW_SELF_REFRESH_FORCE_ENABLE, force_en);
-
-#if 0
-	REG_UPDATE_2(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_DONE_INTERRUPT_DISABLE, 1,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 1);
-#endif
-}
-
-
-static void dcn10_update_dchub(
-	struct dce_hwseq *hws,
-	struct dchub_init_data *dh_data)
-{
-	/* TODO: port code from dal2 */
-	switch (dh_data->fb_mode) {
-	case FRAME_BUFFER_MODE_ZFB_ONLY:
-		/*For ZFB case need to put DCHUB FB BASE and TOP upside down to indicate ZFB mode*/
-		REG_UPDATE(DCHUBBUB_SDPIF_FB_TOP,
-				SDPIF_FB_TOP, 0);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_FB_BASE,
-				SDPIF_FB_BASE, 0x0FFFF);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
-				SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
-				SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
-				SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
-						dh_data->zfb_size_in_byte - 1) >> 22);
-		break;
-	case FRAME_BUFFER_MODE_MIXED_ZFB_AND_LOCAL:
-		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
-				SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
-				SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
-				SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
-						dh_data->zfb_size_in_byte - 1) >> 22);
-		break;
-	case FRAME_BUFFER_MODE_LOCAL_ONLY:
-		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
-				SDPIF_AGP_BASE, 0);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
-				SDPIF_AGP_BOT, 0X03FFFF);
-
-		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
-				SDPIF_AGP_TOP, 0);
-		break;
-	default:
-		break;
-	}
-
-	dh_data->dchub_initialzied = true;
-	dh_data->dchub_info_valid = false;
-}
-
 static void hubp_pg_control(
 		struct dce_hwseq *hws,
 		unsigned int hubp_inst,
@@ -808,11 +388,8 @@ static void undo_DEGVIDCN10_253_wa(struct dc *dc)
 {
 	struct dce_hwseq *hws = dc->hwseq;
 	struct hubp *hubp = dc->res_pool->hubps[0];
-	int pwr_status = 0;
 
-	REG_GET(DOMAIN0_PG_STATUS, DOMAIN0_PGFSM_PWR_STATUS, &pwr_status);
-	/* Don't need to blank if hubp is power gated*/
-	if (pwr_status == 2)
+	if (!hws->wa_state.DEGVIDCN10_253_applied)
 		return;
 
 	hubp->funcs->set_blank(hubp, true);
@@ -823,16 +400,29 @@ static void undo_DEGVIDCN10_253_wa(struct dc *dc)
 	hubp_pg_control(hws, 0, false);
 	REG_SET(DC_IP_REQUEST_CNTL, 0,
 			IP_REQUEST_EN, 0);
+
+	hws->wa_state.DEGVIDCN10_253_applied = false;
 }
 
 static void apply_DEGVIDCN10_253_wa(struct dc *dc)
 {
 	struct dce_hwseq *hws = dc->hwseq;
 	struct hubp *hubp = dc->res_pool->hubps[0];
+	int i;
 
 	if (dc->debug.disable_stutter)
 		return;
 
+	if (!hws->wa.DEGVIDCN10_253)
+		return;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		if (!dc->res_pool->hubps[i]->power_gated)
+			return;
+	}
+
+	/* all pipe power gated, apply work around to enable stutter. */
+
 	REG_SET(DC_IP_REQUEST_CNTL, 0,
 			IP_REQUEST_EN, 1);
 
@@ -841,6 +431,7 @@ static void apply_DEGVIDCN10_253_wa(struct dc *dc)
 			IP_REQUEST_EN, 0);
 
 	hubp->funcs->set_hubp_blank_en(hubp, false);
+	hws->wa_state.DEGVIDCN10_253_applied = true;
 }
 
 static void bios_golden_init(struct dc *dc)
@@ -859,85 +450,32 @@ static void bios_golden_init(struct dc *dc)
 	}
 }
 
-static void dcn10_init_hw(struct dc *dc)
+static void false_optc_underflow_wa(
+		struct dc *dc,
+		const struct dc_stream_state *stream,
+		struct timing_generator *tg)
 {
 	int i;
-	struct abm *abm = dc->res_pool->abm;
-	struct dce_hwseq *hws = dc->hwseq;
-
-	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
-		REG_WRITE(REFCLK_CNTL, 0);
-		REG_UPDATE(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, 1);
-		REG_WRITE(DIO_MEM_PWR_CTRL, 0);
-
-		if (!dc->debug.disable_clock_gate) {
-			/* enable all DCN clock gating */
-			REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
-
-			REG_WRITE(DCCG_GATE_DISABLE_CNTL2, 0);
+	bool underflow;
 
-			REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
-		}
-
-		enable_power_gating_plane(dc->hwseq, true);
+	if (!dc->hwseq->wa.false_optc_underflow)
 		return;
-	}
-	/* end of FPGA. Below if real ASIC */
-
-	bios_golden_init(dc);
-
-	disable_vga(dc->hwseq);
 
-	for (i = 0; i < dc->link_count; i++) {
-		/* Power up AND update implementation according to the
-		 * required signal (which may be different from the
-		 * default signal on connector).
-		 */
-		struct dc_link *link = dc->links[i];
-
-		link->link_enc->funcs->hw_init(link->link_enc);
-	}
+	underflow = tg->funcs->is_optc_underflow_occurred(tg);
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct dpp *dpp = dc->res_pool->dpps[i];
-		struct timing_generator *tg = dc->res_pool->timing_generators[i];
-
-		dpp->funcs->dpp_reset(dpp);
-		dc->res_pool->mpc->funcs->remove(
-				dc->res_pool->mpc, &(dc->res_pool->opps[i]->mpc_tree),
-				dc->res_pool->opps[i]->inst, i);
-
-		/* Blank controller using driver code instead of
-		 * command table.
-		 */
-		tg->funcs->set_blank(tg, true);
-		hwss_wait_for_blank_complete(tg);
-	}
-
-	for (i = 0; i < dc->res_pool->audio_count; i++) {
-		struct audio *audio = dc->res_pool->audios[i];
+		struct pipe_ctx *old_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
 
-		audio->funcs->hw_init(audio);
-	}
+		if (old_pipe_ctx->stream != stream)
+			continue;
 
-	if (abm != NULL) {
-		abm->funcs->init_backlight(abm);
-		abm->funcs->abm_init(abm);
+		dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, old_pipe_ctx);
 	}
 
-	/* power AFMT HDMI memory TODO: may move to dis/en output save power*/
-	REG_WRITE(DIO_MEM_PWR_CTRL, 0);
-
-	if (!dc->debug.disable_clock_gate) {
-		/* enable all DCN clock gating */
-		REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
-
-		REG_WRITE(DCCG_GATE_DISABLE_CNTL2, 0);
-
-		REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
-	}
+	tg->funcs->set_blank_data_double_buffer(tg, true);
 
-	enable_power_gating_plane(dc->hwseq, true);
+	if (tg->funcs->is_optc_underflow_occurred(tg) && !underflow)
+		tg->funcs->clear_optc_underflow(tg);
 }
 
 static enum dc_status dcn10_prog_pixclk_crtc_otg(
@@ -948,10 +486,6 @@ static enum dc_status dcn10_prog_pixclk_crtc_otg(
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	enum dc_color_space color_space;
 	struct tg_color black_color = {0};
-	bool enableStereo    = stream->timing.timing_3d_format == TIMING_3D_FORMAT_NONE ?
-			false:true;
-	bool rightEyePolarity = stream->timing.flags.RIGHT_EYE_3D_POLARITY;
-
 
 	/* by upper caller loop, pipe0 is parent pipe and be called first.
 	 * back end is set up by for pipe0. Other children pipe share back end
@@ -986,11 +520,6 @@ static enum dc_status dcn10_prog_pixclk_crtc_otg(
 			&stream->timing,
 			true);
 
-	pipe_ctx->stream_res.opp->funcs->opp_set_stereo_polarity(
-				pipe_ctx->stream_res.opp,
-				enableStereo,
-				rightEyePolarity);
-
 #if 0 /* move to after enable_crtc */
 	/* TODO: OPP FMT, ABM. etc. should be done here. */
 	/* or FPGA now. instance 0 only. TODO: move to opp.c */
@@ -1005,12 +534,18 @@ static enum dc_status dcn10_prog_pixclk_crtc_otg(
 	/* program otg blank color */
 	color_space = stream->output_color_space;
 	color_space_to_black_color(dc, color_space, &black_color);
-	pipe_ctx->stream_res.tg->funcs->set_blank_color(
-			pipe_ctx->stream_res.tg,
-			&black_color);
 
-	pipe_ctx->stream_res.tg->funcs->set_blank(pipe_ctx->stream_res.tg, true);
-	hwss_wait_for_blank_complete(pipe_ctx->stream_res.tg);
+	if (pipe_ctx->stream_res.tg->funcs->set_blank_color)
+		pipe_ctx->stream_res.tg->funcs->set_blank_color(
+				pipe_ctx->stream_res.tg,
+				&black_color);
+
+	if (pipe_ctx->stream_res.tg->funcs->is_blanked &&
+			!pipe_ctx->stream_res.tg->funcs->is_blanked(pipe_ctx->stream_res.tg)) {
+		pipe_ctx->stream_res.tg->funcs->set_blank(pipe_ctx->stream_res.tg, true);
+		hwss_wait_for_blank_complete(pipe_ctx->stream_res.tg);
+		false_optc_underflow_wa(dc, pipe_ctx->stream, pipe_ctx->stream_res.tg);
+	}
 
 	/* VTG is  within DCHUB command block. DCFCLK is always on */
 	if (false == pipe_ctx->stream_res.tg->funcs->enable_crtc(pipe_ctx->stream_res.tg)) {
@@ -1070,83 +605,55 @@ static void reset_back_end_for_pipe(
 					pipe_ctx->pipe_idx, pipe_ctx->stream_res.tg->inst);
 }
 
+static void dcn10_verify_allow_pstate_change_high(struct dc *dc)
+{
+	static bool should_log_hw_state; /* prevent hw state log by default */
+
+	if (!hubbub1_verify_allow_pstate_change_high(dc->res_pool->hubbub)) {
+		if (should_log_hw_state) {
+			dcn10_log_hw_state(dc);
+		}
+
+		BREAK_TO_DEBUGGER();
+	}
+}
+
 /* trigger HW to start disconnect plane from stream on the next vsync */
-static void plane_atomic_disconnect(struct dc *dc,
-		int fe_idx)
+static void plane_atomic_disconnect(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
+	int fe_idx = pipe_ctx->pipe_idx;
 	struct hubp *hubp = dc->res_pool->hubps[fe_idx];
 	struct mpc *mpc = dc->res_pool->mpc;
-	int opp_id, z_idx;
-	int mpcc_id = -1;
+	int opp_id;
+	struct mpc_tree *mpc_tree_params;
+	struct mpcc *mpcc_to_remove = NULL;
 
 	/* look at tree rather than mi here to know if we already reset */
 	for (opp_id = 0; opp_id < dc->res_pool->pipe_count; opp_id++) {
 		struct output_pixel_processor *opp = dc->res_pool->opps[opp_id];
 
-		for (z_idx = 0; z_idx < opp->mpc_tree.num_pipes; z_idx++) {
-			if (opp->mpc_tree.dpp[z_idx] == fe_idx) {
-				mpcc_id = opp->mpc_tree.mpcc[z_idx];
-				break;
-			}
-		}
-		if (mpcc_id != -1)
+		mpc_tree_params = &(opp->mpc_tree_params);
+		mpcc_to_remove = mpc->funcs->get_mpcc_for_dpp(mpc_tree_params, fe_idx);
+		if (mpcc_to_remove != NULL)
 			break;
 	}
+
 	/*Already reset*/
 	if (opp_id == dc->res_pool->pipe_count)
 		return;
 
-	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
-	hubp->funcs->dcc_control(hubp, false, false);
-	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
-
-	mpc->funcs->remove(mpc, &(dc->res_pool->opps[opp_id]->mpc_tree),
-			dc->res_pool->opps[opp_id]->inst, fe_idx);
-}
-
-/* disable HW used by plane.
- * note:  cannot disable until disconnect is complete */
-static void plane_atomic_disable(struct dc *dc,
-		int fe_idx)
-{
-	struct dce_hwseq *hws = dc->hwseq;
-	struct hubp *hubp = dc->res_pool->hubps[fe_idx];
-	struct mpc *mpc = dc->res_pool->mpc;
-	int opp_id = hubp->opp_id;
-
-	if (opp_id == 0xf)
-		return;
-
-	mpc->funcs->wait_for_idle(mpc, hubp->mpcc_id);
-	dc->res_pool->opps[hubp->opp_id]->mpcc_disconnect_pending[hubp->mpcc_id] = false;
-	/*dm_logger_write(dc->ctx->logger, LOG_ERROR,
-			"[debug_mpo: atomic disable finished on mpcc %d]\n",
-			fe_idx);*/
-
-	hubp->funcs->set_blank(hubp, true);
-
-	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
+	mpc->funcs->remove_mpcc(mpc, mpc_tree_params, mpcc_to_remove);
+	dc->res_pool->opps[opp_id]->mpcc_disconnect_pending[fe_idx] = true;
 
-	REG_UPDATE(HUBP_CLK_CNTL[fe_idx],
-			HUBP_CLOCK_ENABLE, 0);
-	REG_UPDATE(DPP_CONTROL[fe_idx],
-			DPP_CLOCK_ENABLE, 0);
+	dc->optimized_required = true;
 
-	if (dc->res_pool->opps[opp_id]->mpc_tree.num_pipes == 0)
-		REG_UPDATE(OPP_PIPE_CONTROL[opp_id],
-				OPP_PIPE_CLOCK_EN, 0);
+	if (hubp->funcs->hubp_disconnect)
+		hubp->funcs->hubp_disconnect(hubp);
 
 	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 }
 
-/*
- * kill power to plane hw
- * note: cannot power down until plane is disable
- */
 static void plane_atomic_power_down(struct dc *dc, int fe_idx)
 {
 	struct dce_hwseq *hws = dc->hwseq;
@@ -1162,125 +669,200 @@ static void plane_atomic_power_down(struct dc *dc, int fe_idx)
 				IP_REQUEST_EN, 0);
 		dm_logger_write(dc->ctx->logger, LOG_DEBUG,
 				"Power gated front end %d\n", fe_idx);
-
-		if (dc->debug.sanity_checks)
-			verify_allow_pstate_change_high(dc->hwseq);
 	}
 }
 
-
-static void reset_front_end(
-		struct dc *dc,
-		int fe_idx)
+/* disable HW used by plane.
+ * note:  cannot disable until disconnect is complete
+ */
+static void plane_atomic_disable(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
+	int fe_idx = pipe_ctx->pipe_idx;
 	struct dce_hwseq *hws = dc->hwseq;
-	struct timing_generator *tg;
-	int opp_id = dc->res_pool->hubps[fe_idx]->opp_id;
-
-	/*Already reset*/
-	if (opp_id == 0xf)
-		return;
-
-	tg = dc->res_pool->timing_generators[opp_id];
-	tg->funcs->lock(tg);
+	struct hubp *hubp = dc->res_pool->hubps[fe_idx];
+	int opp_id = hubp->opp_id;
 
-	plane_atomic_disconnect(dc, fe_idx);
+	dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe_ctx);
 
-	REG_UPDATE(OTG_GLOBAL_SYNC_STATUS[tg->inst], VUPDATE_NO_LOCK_EVENT_CLEAR, 1);
-	tg->funcs->unlock(tg);
+	REG_UPDATE(HUBP_CLK_CNTL[fe_idx],
+			HUBP_CLOCK_ENABLE, 0);
+	REG_UPDATE(DPP_CONTROL[fe_idx],
+			DPP_CLOCK_ENABLE, 0);
 
-	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(hws);
+	if (opp_id != 0xf && dc->res_pool->opps[opp_id]->mpc_tree_params.opp_list == NULL)
+		REG_UPDATE(OPP_PIPE_CONTROL[opp_id],
+				OPP_PIPE_CLOCK_EN, 0);
 
-	if (tg->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
-		REG_WAIT(OTG_GLOBAL_SYNC_STATUS[tg->inst],
-				VUPDATE_NO_LOCK_EVENT_OCCURRED, 1,
-				1, 100000);
+	hubp->power_gated = true;
+	dc->optimized_required = false; /* We're powering off, no need to optimize */
 
-	plane_atomic_disable(dc, fe_idx);
+	plane_atomic_power_down(dc, fe_idx);
 
-	dm_logger_write(dc->ctx->logger, LOG_DC,
-					"Reset front end %d\n",
-					fe_idx);
+	pipe_ctx->stream = NULL;
+	memset(&pipe_ctx->stream_res, 0, sizeof(pipe_ctx->stream_res));
+	memset(&pipe_ctx->plane_res, 0, sizeof(pipe_ctx->plane_res));
+	pipe_ctx->top_pipe = NULL;
+	pipe_ctx->bottom_pipe = NULL;
+	pipe_ctx->plane_state = NULL;
 }
 
-static void dcn10_power_down_fe(struct dc *dc, int fe_idx)
+static void dcn10_disable_plane(struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
-	struct dce_hwseq *hws = dc->hwseq;
-	struct dpp *dpp = dc->res_pool->dpps[fe_idx];
+	if (dc->res_pool->hubps[pipe_ctx->pipe_idx]->power_gated)
+		return;
 
-	reset_front_end(dc, fe_idx);
+	plane_atomic_disable(dc, pipe_ctx);
 
-	REG_SET(DC_IP_REQUEST_CNTL, 0,
-			IP_REQUEST_EN, 1);
-	dpp_pg_control(hws, fe_idx, false);
-	hubp_pg_control(hws, fe_idx, false);
-	dpp->funcs->dpp_reset(dpp);
-	REG_SET(DC_IP_REQUEST_CNTL, 0,
-			IP_REQUEST_EN, 0);
-	dm_logger_write(dc->ctx->logger, LOG_DEBUG,
-			"Power gated front end %d\n", fe_idx);
+	apply_DEGVIDCN10_253_wa(dc);
 
-	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
+	dm_logger_write(dc->ctx->logger, LOG_DC,
+					"Power down front end %d\n",
+					pipe_ctx->pipe_idx);
 }
 
-static void reset_hw_ctx_wrap(
-		struct dc *dc,
-		struct dc_state *context)
+static void dcn10_init_hw(struct dc *dc)
 {
 	int i;
+	struct abm *abm = dc->res_pool->abm;
+	struct dmcu *dmcu = dc->res_pool->dmcu;
+	struct dce_hwseq *hws = dc->hwseq;
+	struct dc_bios *dcb = dc->ctx->dc_bios;
+	struct dc_state  *context = dc->current_state;
 
-	/* Reset Front End*/
-	/* Lock*/
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *cur_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
-		struct timing_generator *tg = cur_pipe_ctx->stream_res.tg;
+	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
+		REG_WRITE(REFCLK_CNTL, 0);
+		REG_UPDATE(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, 1);
+		REG_WRITE(DIO_MEM_PWR_CTRL, 0);
 
-		if (cur_pipe_ctx->stream)
-			tg->funcs->lock(tg);
-	}
-	/* Disconnect*/
-	for (i = dc->res_pool->pipe_count - 1; i >= 0 ; i--) {
-		struct pipe_ctx *pipe_ctx_old =
-			&dc->current_state->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+		if (!dc->debug.disable_clock_gate) {
+			/* enable all DCN clock gating */
+			REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
 
-		if (!pipe_ctx->stream ||
-				!pipe_ctx->plane_state ||
-				pipe_need_reprogram(pipe_ctx_old, pipe_ctx)) {
+			REG_WRITE(DCCG_GATE_DISABLE_CNTL2, 0);
 
-			plane_atomic_disconnect(dc, i);
+			REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
 		}
+
+		enable_power_gating_plane(dc->hwseq, true);
+		return;
+	}
+	/* end of FPGA. Below if real ASIC */
+
+	if (!dcb->funcs->is_accelerated_mode(dcb)) {
+		bios_golden_init(dc);
+		disable_vga(dc->hwseq);
+	}
+
+	for (i = 0; i < dc->link_count; i++) {
+		/* Power up AND update implementation according to the
+		 * required signal (which may be different from the
+		 * default signal on connector).
+		 */
+		struct dc_link *link = dc->links[i];
+
+		if (link->link_enc->connector.id == CONNECTOR_ID_EDP)
+			dc->hwss.edp_power_control(link, true);
+
+		link->link_enc->funcs->hw_init(link->link_enc);
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+		if (tg->funcs->is_tg_enabled(tg))
+			tg->funcs->lock(tg);
 	}
-	/* Unlock*/
-	for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
-		struct pipe_ctx *cur_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
-		struct timing_generator *tg = cur_pipe_ctx->stream_res.tg;
 
-		if (cur_pipe_ctx->stream)
+	/* Blank controller using driver code instead of
+	 * command table.
+	 */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+		if (tg->funcs->is_tg_enabled(tg)) {
+			tg->funcs->set_blank(tg, true);
+			hwss_wait_for_blank_complete(tg);
+		}
+	}
+
+	/* Reset all MPCC muxes */
+	dc->res_pool->mpc->funcs->mpc_init(dc->res_pool->mpc);
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[i];
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+		struct hubp *hubp = dc->res_pool->hubps[i];
+
+		pipe_ctx->stream_res.tg = tg;
+		pipe_ctx->pipe_idx = i;
+
+		pipe_ctx->plane_res.hubp = hubp;
+		hubp->mpcc_id = i;
+		hubp->opp_id = 0xf;
+		hubp->power_gated = false;
+
+		dc->res_pool->opps[i]->mpc_tree_params.opp_id = dc->res_pool->opps[i]->inst;
+		dc->res_pool->opps[i]->mpc_tree_params.opp_list = NULL;
+		dc->res_pool->opps[i]->mpcc_disconnect_pending[i] = true;
+		pipe_ctx->stream_res.opp = dc->res_pool->opps[i];
+
+		plane_atomic_disconnect(dc, pipe_ctx);
+	}
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[i];
+
+		if (tg->funcs->is_tg_enabled(tg))
 			tg->funcs->unlock(tg);
 	}
 
-	/* Disable and Powerdown*/
-	for (i = dc->res_pool->pipe_count - 1; i >= 0 ; i--) {
-		struct pipe_ctx *pipe_ctx_old =
-			&dc->current_state->res_ctx.pipe_ctx[i];
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[i];
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
-		/*if (!pipe_ctx_old->stream)
-			continue;*/
+		dcn10_disable_plane(dc, pipe_ctx);
 
-		if (pipe_ctx->stream && pipe_ctx->plane_state
-				&& !pipe_need_reprogram(pipe_ctx_old, pipe_ctx))
-			continue;
+		pipe_ctx->stream_res.tg = NULL;
+		pipe_ctx->plane_res.hubp = NULL;
 
-		plane_atomic_disable(dc, i);
+		tg->funcs->tg_init(tg);
+	}
+
+	for (i = 0; i < dc->res_pool->audio_count; i++) {
+		struct audio *audio = dc->res_pool->audios[i];
 
-		if (!pipe_ctx->stream || !pipe_ctx->plane_state)
-			plane_atomic_power_down(dc, i);
+		audio->funcs->hw_init(audio);
 	}
 
+	if (abm != NULL) {
+		abm->funcs->init_backlight(abm);
+		abm->funcs->abm_init(abm);
+	}
+
+	if (dmcu != NULL)
+		dmcu->funcs->dmcu_init(dmcu);
+
+	/* power AFMT HDMI memory TODO: may move to dis/en output save power*/
+	REG_WRITE(DIO_MEM_PWR_CTRL, 0);
+
+	if (!dc->debug.disable_clock_gate) {
+		/* enable all DCN clock gating */
+		REG_WRITE(DCCG_GATE_DISABLE_CNTL, 0);
+
+		REG_WRITE(DCCG_GATE_DISABLE_CNTL2, 0);
+
+		REG_UPDATE(DCFCLK_CNTL, DCFCLK_GATE_DIS, 0);
+	}
+
+	enable_power_gating_plane(dc->hwseq, true);
+}
+
+static void reset_hw_ctx_wrap(
+		struct dc *dc,
+		struct dc_state *context)
+{
+	int i;
+
 	/* Reset Back End*/
 	for (i = dc->res_pool->pipe_count - 1; i >= 0 ; i--) {
 		struct pipe_ctx *pipe_ctx_old =
@@ -1298,7 +880,6 @@ static void reset_hw_ctx_wrap(
 			struct clock_source *old_clk = pipe_ctx_old->clock_source;
 
 			reset_back_end_for_pipe(dc, pipe_ctx_old, dc->current_state);
-
 			if (old_clk)
 				old_clk->funcs->cs_power_down(old_clk);
 		}
@@ -1332,21 +913,7 @@ static bool patch_address_for_sbs_tb_stereo(
 	return false;
 }
 
-static void toggle_watermark_change_req(struct dce_hwseq *hws)
-{
-	uint32_t watermark_change_req;
-
-	REG_GET(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, &watermark_change_req);
-
-	if (watermark_change_req)
-		watermark_change_req = 0;
-	else
-		watermark_change_req = 1;
 
-	REG_UPDATE(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, watermark_change_req);
-}
 
 static void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 {
@@ -1366,8 +933,8 @@ static void dcn10_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_c
 		pipe_ctx->plane_state->address.grph_stereo.left_addr = addr;
 }
 
-static bool dcn10_set_input_transfer_func(
-	struct pipe_ctx *pipe_ctx, const struct dc_plane_state *plane_state)
+static bool dcn10_set_input_transfer_func(struct pipe_ctx *pipe_ctx,
+					  const struct dc_plane_state *plane_state)
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	const struct dc_transfer_func *tf = NULL;
@@ -1380,34 +947,28 @@ static bool dcn10_set_input_transfer_func(
 		tf = plane_state->in_transfer_func;
 
 	if (plane_state->gamma_correction && dce_use_lut(plane_state))
-		dpp_base->funcs->ipp_program_input_lut(dpp_base,
-				plane_state->gamma_correction);
+		dpp_base->funcs->dpp_program_input_lut(dpp_base, plane_state->gamma_correction);
 
 	if (tf == NULL)
-		dpp_base->funcs->ipp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
+		dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
 	else if (tf->type == TF_TYPE_PREDEFINED) {
 		switch (tf->tf) {
 		case TRANSFER_FUNCTION_SRGB:
-			dpp_base->funcs->ipp_set_degamma(dpp_base,
-					IPP_DEGAMMA_MODE_HW_sRGB);
+			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_HW_sRGB);
 			break;
 		case TRANSFER_FUNCTION_BT709:
-			dpp_base->funcs->ipp_set_degamma(dpp_base,
-					IPP_DEGAMMA_MODE_HW_xvYCC);
+			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_HW_xvYCC);
 			break;
 		case TRANSFER_FUNCTION_LINEAR:
-			dpp_base->funcs->ipp_set_degamma(dpp_base,
-					IPP_DEGAMMA_MODE_BYPASS);
+			dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
 			break;
 		case TRANSFER_FUNCTION_PQ:
-			result = false;
-			break;
 		default:
 			result = false;
 			break;
 		}
 	} else if (tf->type == TF_TYPE_BYPASS) {
-		dpp_base->funcs->ipp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
+		dpp_base->funcs->dpp_set_degamma(dpp_base, IPP_DEGAMMA_MODE_BYPASS);
 	} else {
 		/*TF_TYPE_DISTRIBUTED_POINTS*/
 		result = false;
@@ -1415,324 +976,14 @@ static bool dcn10_set_input_transfer_func(
 
 	return result;
 }
-/*modify the method to handle rgb for arr_points*/
-static bool convert_to_custom_float(
-		struct pwl_result_data *rgb_resulted,
-		struct curve_points *arr_points,
-		uint32_t hw_points_num)
-{
-	struct custom_float_format fmt;
 
-	struct pwl_result_data *rgb = rgb_resulted;
 
-	uint32_t i = 0;
 
-	fmt.exponenta_bits = 6;
-	fmt.mantissa_bits = 12;
-	fmt.sign = false;
 
-	if (!convert_to_custom_float_format(
-		arr_points[0].x,
-		&fmt,
-		&arr_points[0].custom_float_x)) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	if (!convert_to_custom_float_format(
-		arr_points[0].offset,
-		&fmt,
-		&arr_points[0].custom_float_offset)) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	if (!convert_to_custom_float_format(
-		arr_points[0].slope,
-		&fmt,
-		&arr_points[0].custom_float_slope)) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	fmt.mantissa_bits = 10;
-	fmt.sign = false;
-
-	if (!convert_to_custom_float_format(
-		arr_points[1].x,
-		&fmt,
-		&arr_points[1].custom_float_x)) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	if (!convert_to_custom_float_format(
-		arr_points[1].y,
-		&fmt,
-		&arr_points[1].custom_float_y)) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	if (!convert_to_custom_float_format(
-		arr_points[1].slope,
-		&fmt,
-		&arr_points[1].custom_float_slope)) {
-		BREAK_TO_DEBUGGER();
-		return false;
-	}
-
-	fmt.mantissa_bits = 12;
-	fmt.sign = true;
-
-	while (i != hw_points_num) {
-		if (!convert_to_custom_float_format(
-			rgb->red,
-			&fmt,
-			&rgb->red_reg)) {
-			BREAK_TO_DEBUGGER();
-			return false;
-		}
-
-		if (!convert_to_custom_float_format(
-			rgb->green,
-			&fmt,
-			&rgb->green_reg)) {
-			BREAK_TO_DEBUGGER();
-			return false;
-		}
-
-		if (!convert_to_custom_float_format(
-			rgb->blue,
-			&fmt,
-			&rgb->blue_reg)) {
-			BREAK_TO_DEBUGGER();
-			return false;
-		}
-
-		if (!convert_to_custom_float_format(
-			rgb->delta_red,
-			&fmt,
-			&rgb->delta_red_reg)) {
-			BREAK_TO_DEBUGGER();
-			return false;
-		}
 
-		if (!convert_to_custom_float_format(
-			rgb->delta_green,
-			&fmt,
-			&rgb->delta_green_reg)) {
-			BREAK_TO_DEBUGGER();
-			return false;
-		}
-
-		if (!convert_to_custom_float_format(
-			rgb->delta_blue,
-			&fmt,
-			&rgb->delta_blue_reg)) {
-			BREAK_TO_DEBUGGER();
-			return false;
-		}
-
-		++rgb;
-		++i;
-	}
-
-	return true;
-}
-#define MAX_REGIONS_NUMBER 34
-#define MAX_LOW_POINT      25
-#define NUMBER_SEGMENTS    32
-
-static bool dcn10_translate_regamma_to_hw_format(const struct dc_transfer_func
-		*output_tf, struct pwl_params *regamma_params)
-{
-	struct curve_points *arr_points;
-	struct pwl_result_data *rgb_resulted;
-	struct pwl_result_data *rgb;
-	struct pwl_result_data *rgb_plus_1;
-	struct fixed31_32 y_r;
-	struct fixed31_32 y_g;
-	struct fixed31_32 y_b;
-	struct fixed31_32 y1_min;
-	struct fixed31_32 y3_max;
-
-	int32_t segment_start, segment_end;
-	int32_t i;
-	uint32_t j, k, seg_distr[MAX_REGIONS_NUMBER], increment, start_index, hw_points;
-
-	if (output_tf == NULL || regamma_params == NULL ||
-			output_tf->type == TF_TYPE_BYPASS)
-		return false;
-
-	arr_points = regamma_params->arr_points;
-	rgb_resulted = regamma_params->rgb_resulted;
-	hw_points = 0;
-
-	memset(regamma_params, 0, sizeof(struct pwl_params));
-	memset(seg_distr, 0, sizeof(seg_distr));
-
-	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
-		/* 32 segments
-		 * segments are from 2^-25 to 2^7
-		 */
-		for (i = 0; i < 32 ; i++)
-			seg_distr[i] = 3;
-
-		segment_start = -25;
-		segment_end   = 7;
-	} else {
-		/* 10 segments
-		 * segment is from 2^-10 to 2^0
-		 * There are less than 256 points, for optimization
-		 */
-		seg_distr[0] = 3;
-		seg_distr[1] = 4;
-		seg_distr[2] = 4;
-		seg_distr[3] = 4;
-		seg_distr[4] = 4;
-		seg_distr[5] = 4;
-		seg_distr[6] = 4;
-		seg_distr[7] = 4;
-		seg_distr[8] = 5;
-		seg_distr[9] = 5;
-
-		segment_start = -10;
-		segment_end = 0;
-	}
-
-	for (i = segment_end - segment_start; i < MAX_REGIONS_NUMBER ; i++)
-		seg_distr[i] = -1;
-
-	for (k = 0; k < MAX_REGIONS_NUMBER; k++) {
-		if (seg_distr[k] != -1)
-			hw_points += (1 << seg_distr[k]);
-	}
-
-	j = 0;
-	for (k = 0; k < (segment_end - segment_start); k++) {
-		increment = NUMBER_SEGMENTS / (1 << seg_distr[k]);
-		start_index = (segment_start + k + MAX_LOW_POINT) * NUMBER_SEGMENTS;
-		for (i = start_index; i < start_index + NUMBER_SEGMENTS; i += increment) {
-			if (j == hw_points - 1)
-				break;
-			rgb_resulted[j].red = output_tf->tf_pts.red[i];
-			rgb_resulted[j].green = output_tf->tf_pts.green[i];
-			rgb_resulted[j].blue = output_tf->tf_pts.blue[i];
-			j++;
-		}
-	}
-
-	/* last point */
-	start_index = (segment_end + MAX_LOW_POINT) * NUMBER_SEGMENTS;
-	rgb_resulted[hw_points - 1].red =
-			output_tf->tf_pts.red[start_index];
-	rgb_resulted[hw_points - 1].green =
-			output_tf->tf_pts.green[start_index];
-	rgb_resulted[hw_points - 1].blue =
-			output_tf->tf_pts.blue[start_index];
-
-	arr_points[0].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-			dal_fixed31_32_from_int(segment_start));
-	arr_points[1].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-			dal_fixed31_32_from_int(segment_end));
-	arr_points[2].x = dal_fixed31_32_pow(dal_fixed31_32_from_int(2),
-			dal_fixed31_32_from_int(segment_end));
-
-	y_r = rgb_resulted[0].red;
-	y_g = rgb_resulted[0].green;
-	y_b = rgb_resulted[0].blue;
-
-	y1_min = dal_fixed31_32_min(y_r, dal_fixed31_32_min(y_g, y_b));
-
-	arr_points[0].y = y1_min;
-	arr_points[0].slope = dal_fixed31_32_div(
-					arr_points[0].y,
-					arr_points[0].x);
-	y_r = rgb_resulted[hw_points - 1].red;
-	y_g = rgb_resulted[hw_points - 1].green;
-	y_b = rgb_resulted[hw_points - 1].blue;
-
-	/* see comment above, m_arrPoints[1].y should be the Y value for the
-	 * region end (m_numOfHwPoints), not last HW point(m_numOfHwPoints - 1)
-	 */
-	y3_max = dal_fixed31_32_max(y_r, dal_fixed31_32_max(y_g, y_b));
-
-	arr_points[1].y = y3_max;
-	arr_points[2].y = y3_max;
-
-	arr_points[1].slope = dal_fixed31_32_zero;
-	arr_points[2].slope = dal_fixed31_32_zero;
-
-	if (output_tf->tf == TRANSFER_FUNCTION_PQ) {
-		/* for PQ, we want to have a straight line from last HW X point,
-		 * and the slope to be such that we hit 1.0 at 10000 nits.
-		 */
-		const struct fixed31_32 end_value =
-				dal_fixed31_32_from_int(125);
-
-		arr_points[1].slope = dal_fixed31_32_div(
-			dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
-			dal_fixed31_32_sub(end_value, arr_points[1].x));
-		arr_points[2].slope = dal_fixed31_32_div(
-			dal_fixed31_32_sub(dal_fixed31_32_one, arr_points[1].y),
-			dal_fixed31_32_sub(end_value, arr_points[1].x));
-	}
-
-	regamma_params->hw_points_num = hw_points;
-
-	i = 1;
-	for (k = 0; k < MAX_REGIONS_NUMBER && i < MAX_REGIONS_NUMBER; k++) {
-		if (seg_distr[k] != -1) {
-			regamma_params->arr_curve_points[k].segments_num =
-					seg_distr[k];
-			regamma_params->arr_curve_points[i].offset =
-					regamma_params->arr_curve_points[k].
-					offset + (1 << seg_distr[k]);
-		}
-		i++;
-	}
-
-	if (seg_distr[k] != -1)
-		regamma_params->arr_curve_points[k].segments_num =
-				seg_distr[k];
-
-	rgb = rgb_resulted;
-	rgb_plus_1 = rgb_resulted + 1;
-
-	i = 1;
-
-	while (i != hw_points + 1) {
-		if (dal_fixed31_32_lt(rgb_plus_1->red, rgb->red))
-			rgb_plus_1->red = rgb->red;
-		if (dal_fixed31_32_lt(rgb_plus_1->green, rgb->green))
-			rgb_plus_1->green = rgb->green;
-		if (dal_fixed31_32_lt(rgb_plus_1->blue, rgb->blue))
-			rgb_plus_1->blue = rgb->blue;
-
-		rgb->delta_red = dal_fixed31_32_sub(
-			rgb_plus_1->red,
-			rgb->red);
-		rgb->delta_green = dal_fixed31_32_sub(
-			rgb_plus_1->green,
-			rgb->green);
-		rgb->delta_blue = dal_fixed31_32_sub(
-			rgb_plus_1->blue,
-			rgb->blue);
-
-		++rgb_plus_1;
-		++rgb;
-		++i;
-	}
-
-	convert_to_custom_float(rgb_resulted, arr_points, hw_points);
-
-	return true;
-}
-
-static bool dcn10_set_output_transfer_func(
-	struct pipe_ctx *pipe_ctx,
-	const struct dc_stream_state *stream)
+static bool
+dcn10_set_output_transfer_func(struct pipe_ctx *pipe_ctx,
+			       const struct dc_stream_state *stream)
 {
 	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 
@@ -1742,18 +993,21 @@ static bool dcn10_set_output_transfer_func(
 	dpp->regamma_params.hw_points_num = GAMMA_HW_POINTS_NUM;
 
 	if (stream->out_transfer_func &&
-		stream->out_transfer_func->type ==
-			TF_TYPE_PREDEFINED &&
-		stream->out_transfer_func->tf ==
-			TRANSFER_FUNCTION_SRGB) {
-		dpp->funcs->opp_set_regamma_mode(dpp, OPP_REGAMMA_SRGB);
-	} else if (dcn10_translate_regamma_to_hw_format(
-				stream->out_transfer_func, &dpp->regamma_params)) {
-			dpp->funcs->opp_program_regamma_pwl(dpp, &dpp->regamma_params);
-			dpp->funcs->opp_set_regamma_mode(dpp, OPP_REGAMMA_USER);
-	} else {
-		dpp->funcs->opp_set_regamma_mode(dpp, OPP_REGAMMA_BYPASS);
-	}
+	    stream->out_transfer_func->type == TF_TYPE_PREDEFINED &&
+	    stream->out_transfer_func->tf == TRANSFER_FUNCTION_SRGB)
+		dpp->funcs->dpp_program_regamma_pwl(dpp, NULL, OPP_REGAMMA_SRGB);
+
+	/* dcn10_translate_regamma_to_hw_format takes 750us, only do it when full
+	 * update.
+	 */
+	else if (cm_helper_translate_curve_to_hw_format(
+			stream->out_transfer_func,
+			&dpp->regamma_params, false)) {
+		dpp->funcs->dpp_program_regamma_pwl(
+				dpp,
+				&dpp->regamma_params, OPP_REGAMMA_USER);
+	} else
+		dpp->funcs->dpp_program_regamma_pwl(dpp, NULL, OPP_REGAMMA_BYPASS);
 
 	return true;
 }
@@ -1772,7 +1026,7 @@ static void dcn10_pipe_control_lock(
 		return;
 
 	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 
 	if (lock)
 		pipe->stream_res.tg->funcs->lock(pipe->stream_res.tg);
@@ -1780,7 +1034,7 @@ static void dcn10_pipe_control_lock(
 		pipe->stream_res.tg->funcs->unlock(pipe->stream_res.tg);
 
 	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 }
 
 static bool wait_for_reset_trigger_to_occur(
@@ -1833,14 +1087,15 @@ static void dcn10_enable_timing_synchronization(
 
 	for (i = 1; i < group_size; i++)
 		grouped_pipes[i]->stream_res.tg->funcs->enable_reset_trigger(
-				grouped_pipes[i]->stream_res.tg, grouped_pipes[0]->stream_res.tg->inst);
-
+				grouped_pipes[i]->stream_res.tg,
+				grouped_pipes[0]->stream_res.tg->inst);
 
 	DC_SYNC_INFO("Waiting for trigger\n");
 
 	/* Need to get only check 1 pipe for having reset as all the others are
 	 * synchronized. Look at last pipe programmed to reset.
 	 */
+
 	wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[1]->stream_res.tg);
 	for (i = 1; i < group_size; i++)
 		grouped_pipes[i]->stream_res.tg->funcs->disable_reset_trigger(
@@ -1849,7 +1104,30 @@ static void dcn10_enable_timing_synchronization(
 	DC_SYNC_INFO("Sync complete\n");
 }
 
-static void print_rq_dlg_ttu(
+static void dcn10_enable_per_frame_crtc_position_reset(
+	struct dc *dc,
+	int group_size,
+	struct pipe_ctx *grouped_pipes[])
+{
+	struct dc_context *dc_ctx = dc->ctx;
+	int i;
+
+	DC_SYNC_INFO("Setting up\n");
+	for (i = 0; i < group_size; i++)
+		grouped_pipes[i]->stream_res.tg->funcs->enable_crtc_reset(
+				grouped_pipes[i]->stream_res.tg,
+				grouped_pipes[i]->stream->triggered_crtc_reset.event_source->status.primary_otg_inst,
+				&grouped_pipes[i]->stream->triggered_crtc_reset);
+
+	DC_SYNC_INFO("Waiting for trigger\n");
+
+	for (i = 1; i < group_size; i++)
+		wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[i]->stream_res.tg);
+
+	DC_SYNC_INFO("Multi-display sync is complete\n");
+}
+
+/*static void print_rq_dlg_ttu(
 		struct dc *core_dc,
 		struct pipe_ctx *pipe_ctx)
 {
@@ -1970,19 +1248,104 @@ static void print_rq_dlg_ttu(
 			pipe_ctx->rq_regs.rq_regs_l.pte_row_height_linear
 			);
 }
+*/
+
+static void mmhub_read_vm_system_aperture_settings(struct dcn10_hubp *hubp1,
+		struct vm_system_aperture_param *apt,
+		struct dce_hwseq *hws)
+{
+	PHYSICAL_ADDRESS_LOC physical_page_number;
+	uint32_t logical_addr_low;
+	uint32_t logical_addr_high;
+
+	REG_GET(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+			PHYSICAL_PAGE_NUMBER_MSB, &physical_page_number.high_part);
+	REG_GET(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+			PHYSICAL_PAGE_NUMBER_LSB, &physical_page_number.low_part);
+
+	REG_GET(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
+			LOGICAL_ADDR, &logical_addr_low);
+
+	REG_GET(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+			LOGICAL_ADDR, &logical_addr_high);
+
+	apt->sys_default.quad_part =  physical_page_number.quad_part << 12;
+	apt->sys_low.quad_part =  (int64_t)logical_addr_low << 18;
+	apt->sys_high.quad_part =  (int64_t)logical_addr_high << 18;
+}
+
+/* Temporary read settings, future will get values from kmd directly */
+static void mmhub_read_vm_context0_settings(struct dcn10_hubp *hubp1,
+		struct vm_context0_param *vm0,
+		struct dce_hwseq *hws)
+{
+	PHYSICAL_ADDRESS_LOC fb_base;
+	PHYSICAL_ADDRESS_LOC fb_offset;
+	uint32_t fb_base_value;
+	uint32_t fb_offset_value;
+
+	REG_GET(DCHUBBUB_SDPIF_FB_BASE, SDPIF_FB_BASE, &fb_base_value);
+	REG_GET(DCHUBBUB_SDPIF_FB_OFFSET, SDPIF_FB_OFFSET, &fb_offset_value);
+
+	REG_GET(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+			PAGE_DIRECTORY_ENTRY_HI32, &vm0->pte_base.high_part);
+	REG_GET(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+			PAGE_DIRECTORY_ENTRY_LO32, &vm0->pte_base.low_part);
+
+	REG_GET(VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+			LOGICAL_PAGE_NUMBER_HI4, &vm0->pte_start.high_part);
+	REG_GET(VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+			LOGICAL_PAGE_NUMBER_LO32, &vm0->pte_start.low_part);
+
+	REG_GET(VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+			LOGICAL_PAGE_NUMBER_HI4, &vm0->pte_end.high_part);
+	REG_GET(VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+			LOGICAL_PAGE_NUMBER_LO32, &vm0->pte_end.low_part);
+
+	REG_GET(VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+			PHYSICAL_PAGE_ADDR_HI4, &vm0->fault_default.high_part);
+	REG_GET(VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+			PHYSICAL_PAGE_ADDR_LO32, &vm0->fault_default.low_part);
+
+	/*
+	 * The values in VM_CONTEXT0_PAGE_TABLE_BASE_ADDR is in UMA space.
+	 * Therefore we need to do
+	 * DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
+	 * - DCHUBBUB_SDPIF_FB_OFFSET + DCHUBBUB_SDPIF_FB_BASE
+	 */
+	fb_base.quad_part = (uint64_t)fb_base_value << 24;
+	fb_offset.quad_part = (uint64_t)fb_offset_value << 24;
+	vm0->pte_base.quad_part += fb_base.quad_part;
+	vm0->pte_base.quad_part -= fb_offset.quad_part;
+}
+
+
+static void dcn10_program_pte_vm(struct dce_hwseq *hws, struct hubp *hubp)
+{
+	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
+	struct vm_system_aperture_param apt = { {{ 0 } } };
+	struct vm_context0_param vm0 = { { { 0 } } };
+
+	mmhub_read_vm_system_aperture_settings(hubp1, &apt, hws);
+	mmhub_read_vm_context0_settings(hubp1, &vm0, hws);
+
+	hubp->funcs->hubp_set_vm_system_aperture_settings(hubp, &apt);
+	hubp->funcs->hubp_set_vm_context0_settings(hubp, &vm0);
+}
 
-static void dcn10_power_on_fe(
+static void dcn10_enable_plane(
 	struct dc *dc,
 	struct pipe_ctx *pipe_ctx,
 	struct dc_state *context)
 {
-	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 	struct dce_hwseq *hws = dc->hwseq;
 
 	if (dc->debug.sanity_checks) {
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 	}
 
+	undo_DEGVIDCN10_253_wa(dc);
+
 	power_on_plane(dc->hwseq,
 		pipe_ctx->pipe_idx);
 
@@ -1993,8 +1356,8 @@ static void dcn10_power_on_fe(
 	/* make sure OPP_PIPE_CLOCK_EN = 1 */
 	REG_UPDATE(OPP_PIPE_CONTROL[pipe_ctx->stream_res.tg->inst],
 			OPP_PIPE_CLOCK_EN, 1);
-	/*TODO: REG_UPDATE(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_WDIVIDER, 0x1f);*/
 
+/* TODO: enable/disable in dm as per update type.
 	if (plane_state) {
 		dm_logger_write(dc->ctx->logger, LOG_DC,
 				"Pipe:%d 0x%x: addr hi:0x%x, "
@@ -2030,9 +1393,12 @@ static void dcn10_power_on_fe(
 				pipe_ctx->plane_res.scl_data.recout.y);
 		print_rq_dlg_ttu(dc, pipe_ctx);
 	}
+*/
+	if (dc->config.gpu_vm_support)
+		dcn10_program_pte_vm(hws, pipe_ctx->plane_res.hubp);
 
 	if (dc->debug.sanity_checks) {
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 	}
 }
 
@@ -2082,25 +1448,27 @@ static void program_csc_matrix(struct pipe_ctx *pipe_ctx,
 		enum dc_color_space colorspace,
 		uint16_t *matrix)
 {
-	int i;
-	struct out_csc_color_matrix tbl_entry;
-
-	if (pipe_ctx->stream->csc_color_matrix.enable_adjustment
-				== true) {
-			enum dc_color_space color_space =
-				pipe_ctx->stream->output_color_space;
-
-			//uint16_t matrix[12];
-			for (i = 0; i < 12; i++)
-				tbl_entry.regval[i] = pipe_ctx->stream->csc_color_matrix.matrix[i];
-
-			tbl_entry.color_space = color_space;
-			//tbl_entry.regval = matrix;
-			pipe_ctx->plane_res.dpp->funcs->opp_set_csc_adjustment(pipe_ctx->plane_res.dpp, &tbl_entry);
+	if (pipe_ctx->stream->csc_color_matrix.enable_adjustment == true) {
+			if (pipe_ctx->plane_res.dpp->funcs->dpp_set_csc_adjustment != NULL)
+				pipe_ctx->plane_res.dpp->funcs->dpp_set_csc_adjustment(pipe_ctx->plane_res.dpp, matrix);
 	} else {
-		pipe_ctx->plane_res.dpp->funcs->opp_set_csc_default(pipe_ctx->plane_res.dpp, colorspace);
+		if (pipe_ctx->plane_res.dpp->funcs->dpp_set_csc_default != NULL)
+			pipe_ctx->plane_res.dpp->funcs->dpp_set_csc_default(pipe_ctx->plane_res.dpp, colorspace);
 	}
 }
+
+static void program_output_csc(struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		enum dc_color_space colorspace,
+		uint16_t *matrix,
+		int opp_id)
+{
+	if (pipe_ctx->plane_res.dpp->funcs->dpp_set_csc_adjustment != NULL)
+		program_csc_matrix(pipe_ctx,
+				colorspace,
+				matrix);
+}
+
 static bool is_lower_pipe_tree_visible(struct pipe_ctx *pipe_ctx)
 {
 	if (pipe_ctx->plane_state->visible)
@@ -2188,91 +1556,169 @@ static void dcn10_get_surface_visual_confirm_color(
 	}
 }
 
-static void mmhub_read_vm_system_aperture_settings(struct dcn10_hubp *hubp1,
-		struct vm_system_aperture_param *apt,
-		struct dce_hwseq *hws)
+static uint16_t fixed_point_to_int_frac(
+	struct fixed31_32 arg,
+	uint8_t integer_bits,
+	uint8_t fractional_bits)
 {
-	PHYSICAL_ADDRESS_LOC physical_page_number;
-	uint32_t logical_addr_low;
-	uint32_t logical_addr_high;
+	int32_t numerator;
+	int32_t divisor = 1 << fractional_bits;
 
-	REG_GET(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
-			PHYSICAL_PAGE_NUMBER_MSB, &physical_page_number.high_part);
-	REG_GET(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
-			PHYSICAL_PAGE_NUMBER_LSB, &physical_page_number.low_part);
+	uint16_t result;
 
-	REG_GET(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
-			LOGICAL_ADDR, &logical_addr_low);
+	uint16_t d = (uint16_t)dal_fixed31_32_floor(
+		dal_fixed31_32_abs(
+			arg));
 
-	REG_GET(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
-			LOGICAL_ADDR, &logical_addr_high);
+	if (d <= (uint16_t)(1 << integer_bits) - (1 / (uint16_t)divisor))
+		numerator = (uint16_t)dal_fixed31_32_floor(
+			dal_fixed31_32_mul_int(
+				arg,
+				divisor));
+	else {
+		numerator = dal_fixed31_32_floor(
+			dal_fixed31_32_sub(
+				dal_fixed31_32_from_int(
+					1LL << integer_bits),
+				dal_fixed31_32_recip(
+					dal_fixed31_32_from_int(
+						divisor))));
+	}
 
-	apt->sys_default.quad_part =  physical_page_number.quad_part << 12;
-	apt->sys_low.quad_part =  (int64_t)logical_addr_low << 18;
-	apt->sys_high.quad_part =  (int64_t)logical_addr_high << 18;
+	if (numerator >= 0)
+		result = (uint16_t)numerator;
+	else
+		result = (uint16_t)(
+		(1 << (integer_bits + fractional_bits + 1)) + numerator);
+
+	if ((result != 0) && dal_fixed31_32_lt(
+		arg, dal_fixed31_32_zero))
+		result |= 1 << (integer_bits + fractional_bits);
+
+	return result;
 }
 
-/* Temporary read settings, future will get values from kmd directly */
-static void mmhub_read_vm_context0_settings(struct dcn10_hubp *hubp1,
-		struct vm_context0_param *vm0,
-		struct dce_hwseq *hws)
+void build_prescale_params(struct  dc_bias_and_scale *bias_and_scale,
+		const struct dc_plane_state *plane_state)
+{
+	if (plane_state->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN
+			&& plane_state->format != SURFACE_PIXEL_FORMAT_INVALID
+			&& plane_state->input_csc_color_matrix.enable_adjustment
+			&& plane_state->coeff_reduction_factor.value != 0) {
+		bias_and_scale->scale_blue = fixed_point_to_int_frac(
+			dal_fixed31_32_mul(plane_state->coeff_reduction_factor,
+					dal_fixed31_32_from_fraction(256, 255)),
+				2,
+				13);
+		bias_and_scale->scale_red = bias_and_scale->scale_blue;
+		bias_and_scale->scale_green = bias_and_scale->scale_blue;
+	} else {
+		bias_and_scale->scale_blue = 0x2000;
+		bias_and_scale->scale_red = 0x2000;
+		bias_and_scale->scale_green = 0x2000;
+	}
+}
+
+static void update_dpp(struct dpp *dpp, struct dc_plane_state *plane_state)
 {
-	PHYSICAL_ADDRESS_LOC fb_base;
-	PHYSICAL_ADDRESS_LOC fb_offset;
-	uint32_t fb_base_value;
-	uint32_t fb_offset_value;
+	struct dc_bias_and_scale bns_params = {0};
 
-	REG_GET(DCHUBBUB_SDPIF_FB_BASE, SDPIF_FB_BASE, &fb_base_value);
-	REG_GET(DCHUBBUB_SDPIF_FB_OFFSET, SDPIF_FB_OFFSET, &fb_offset_value);
+	// program the input csc
+	dpp->funcs->dpp_setup(dpp,
+			plane_state->format,
+			EXPANSION_MODE_ZERO,
+			plane_state->input_csc_color_matrix,
+			COLOR_SPACE_YCBCR601_LIMITED);
 
-	REG_GET(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
-			PAGE_DIRECTORY_ENTRY_HI32, &vm0->pte_base.high_part);
-	REG_GET(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
-			PAGE_DIRECTORY_ENTRY_LO32, &vm0->pte_base.low_part);
+	//set scale and bias registers
+	build_prescale_params(&bns_params, plane_state);
+	if (dpp->funcs->dpp_program_bias_and_scale)
+		dpp->funcs->dpp_program_bias_and_scale(dpp, &bns_params);
+}
 
-	REG_GET(VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
-			LOGICAL_PAGE_NUMBER_HI4, &vm0->pte_start.high_part);
-	REG_GET(VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
-			LOGICAL_PAGE_NUMBER_LO32, &vm0->pte_start.low_part);
+static void update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+	struct mpcc_blnd_cfg blnd_cfg;
+	bool per_pixel_alpha = pipe_ctx->plane_state->per_pixel_alpha && pipe_ctx->bottom_pipe;
+	int mpcc_id;
+	struct mpcc *new_mpcc;
+	struct mpc *mpc = dc->res_pool->mpc;
+	struct mpc_tree *mpc_tree_params = &(pipe_ctx->stream_res.opp->mpc_tree_params);
 
-	REG_GET(VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
-			LOGICAL_PAGE_NUMBER_HI4, &vm0->pte_end.high_part);
-	REG_GET(VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
-			LOGICAL_PAGE_NUMBER_LO32, &vm0->pte_end.low_part);
+	/* TODO: proper fix once fpga works */
 
-	REG_GET(VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
-			PHYSICAL_PAGE_ADDR_HI4, &vm0->fault_default.high_part);
-	REG_GET(VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
-			PHYSICAL_PAGE_ADDR_LO32, &vm0->fault_default.low_part);
+	if (dc->debug.surface_visual_confirm)
+		dcn10_get_surface_visual_confirm_color(
+				pipe_ctx, &blnd_cfg.black_color);
+	else
+		color_space_to_black_color(
+			dc, pipe_ctx->stream->output_color_space,
+			&blnd_cfg.black_color);
+
+	if (per_pixel_alpha)
+		blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA;
+	else
+		blnd_cfg.alpha_mode = MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA;
+
+	blnd_cfg.overlap_only = false;
+	blnd_cfg.global_alpha = 0xff;
+	blnd_cfg.global_gain = 0xff;
+
+	/* DCN1.0 has output CM before MPC which seems to screw with
+	 * pre-multiplied alpha.
+	 */
+	blnd_cfg.pre_multiplied_alpha = is_rgb_cspace(
+			pipe_ctx->stream->output_color_space)
+					&& per_pixel_alpha;
 
 	/*
-	 * The values in VM_CONTEXT0_PAGE_TABLE_BASE_ADDR is in UMA space.
-	 * Therefore we need to do
-	 * DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR
-	 * - DCHUBBUB_SDPIF_FB_OFFSET + DCHUBBUB_SDPIF_FB_BASE
+	 * TODO: remove hack
+	 * Note: currently there is a bug in init_hw such that
+	 * on resume from hibernate, BIOS sets up MPCC0, and
+	 * we do mpcc_remove but the mpcc cannot go to idle
+	 * after remove. This cause us to pick mpcc1 here,
+	 * which causes a pstate hang for yet unknown reason.
 	 */
-	fb_base.quad_part = (uint64_t)fb_base_value << 24;
-	fb_offset.quad_part = (uint64_t)fb_offset_value << 24;
-	vm0->pte_base.quad_part += fb_base.quad_part;
-	vm0->pte_base.quad_part -= fb_offset.quad_part;
+	mpcc_id = hubp->inst;
+
+	/* check if this MPCC is already being used */
+	new_mpcc = mpc->funcs->get_mpcc_for_dpp(mpc_tree_params, mpcc_id);
+	/* remove MPCC if being used */
+	if (new_mpcc != NULL)
+		mpc->funcs->remove_mpcc(mpc, mpc_tree_params, new_mpcc);
+	else
+		if (dc->debug.sanity_checks)
+			mpc->funcs->assert_mpcc_idle_before_connect(
+					dc->res_pool->mpc, mpcc_id);
+
+	/* Call MPC to insert new plane */
+	new_mpcc = mpc->funcs->insert_plane(dc->res_pool->mpc,
+			mpc_tree_params,
+			&blnd_cfg,
+			NULL,
+			NULL,
+			hubp->inst,
+			mpcc_id);
+
+	ASSERT(new_mpcc != NULL);
+
+	hubp->opp_id = pipe_ctx->stream_res.opp->inst;
+	hubp->mpcc_id = mpcc_id;
 }
 
-static void dcn10_program_pte_vm(struct hubp *hubp,
-		enum surface_pixel_format format,
-		union dc_tiling_info *tiling_info,
-		enum dc_rotation_angle rotation,
-		struct dce_hwseq *hws)
+static void update_scaler(struct pipe_ctx *pipe_ctx)
 {
-	struct dcn10_hubp *hubp1 = TO_DCN10_HUBP(hubp);
-	struct vm_system_aperture_param apt = { {{ 0 } } };
-	struct vm_context0_param vm0 = { { { 0 } } };
+	bool per_pixel_alpha =
+			pipe_ctx->plane_state->per_pixel_alpha && pipe_ctx->bottom_pipe;
 
+	/* TODO: proper fix once fpga works */
 
-	mmhub_read_vm_system_aperture_settings(hubp1, &apt, hws);
-	mmhub_read_vm_context0_settings(hubp1, &vm0, hws);
-
-	hubp->funcs->hubp_set_vm_system_aperture_settings(hubp, &apt);
-	hubp->funcs->hubp_set_vm_context0_settings(hubp, &vm0);
+	pipe_ctx->plane_res.scl_data.lb_params.alpha_en = per_pixel_alpha;
+	pipe_ctx->plane_res.scl_data.lb_params.depth = LB_PIXEL_DEPTH_30BPP;
+	/* scaler configuration */
+	pipe_ctx->plane_res.dpp->funcs->dpp_set_scaler(
+			pipe_ctx->plane_res.dpp, &pipe_ctx->plane_res.scl_data);
 }
 
 static void update_dchubp_dpp(
@@ -2285,95 +1731,94 @@ static void update_dchubp_dpp(
 	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 	union plane_size size = plane_state->plane_size;
-	struct mpcc_cfg mpcc_cfg = {0};
-	struct pipe_ctx *top_pipe;
-	bool per_pixel_alpha = plane_state->per_pixel_alpha && pipe_ctx->bottom_pipe;
 
-	/* TODO: proper fix once fpga works */
 	/* depends on DML calculation, DPP clock value may change dynamically */
-	enable_dppclk(
-		dc->hwseq,
-		pipe_ctx->pipe_idx,
-		pipe_ctx->stream_res.pix_clk_params.requested_pix_clk,
-		context->bw.dcn.calc_clk.dppclk_div);
-	dc->current_state->bw.dcn.cur_clk.dppclk_div =
-			context->bw.dcn.calc_clk.dppclk_div;
-	context->bw.dcn.cur_clk.dppclk_div = context->bw.dcn.calc_clk.dppclk_div;
+	if (plane_state->update_flags.bits.full_update) {
+		enable_dppclk(
+			dc->hwseq,
+			pipe_ctx->pipe_idx,
+			pipe_ctx->stream_res.pix_clk_params.requested_pix_clk,
+			context->bw.dcn.calc_clk.dppclk_div);
+		dc->current_state->bw.dcn.cur_clk.dppclk_div =
+				context->bw.dcn.calc_clk.dppclk_div;
+		context->bw.dcn.cur_clk.dppclk_div = context->bw.dcn.calc_clk.dppclk_div;
+	}
 
 	/* TODO: Need input parameter to tell current DCHUB pipe tie to which OTG
 	 * VTG is within DCHUBBUB which is commond block share by each pipe HUBP.
 	 * VTG is 1:1 mapping with OTG. Each pipe HUBP will select which VTG
 	 */
-	REG_UPDATE(DCHUBP_CNTL[pipe_ctx->pipe_idx], HUBP_VTG_SEL, pipe_ctx->stream_res.tg->inst);
+	if (plane_state->update_flags.bits.full_update) {
+		REG_UPDATE(DCHUBP_CNTL[pipe_ctx->pipe_idx], HUBP_VTG_SEL, pipe_ctx->stream_res.tg->inst);
 
-	hubp->funcs->hubp_setup(
-		hubp,
-		&pipe_ctx->dlg_regs,
-		&pipe_ctx->ttu_regs,
-		&pipe_ctx->rq_regs,
-		&pipe_ctx->pipe_dlg_param);
+		hubp->funcs->hubp_setup(
+			hubp,
+			&pipe_ctx->dlg_regs,
+			&pipe_ctx->ttu_regs,
+			&pipe_ctx->rq_regs,
+			&pipe_ctx->pipe_dlg_param);
+	}
 
 	size.grph.surface_size = pipe_ctx->plane_res.scl_data.viewport;
 
-	if (dc->config.gpu_vm_support)
-		dcn10_program_pte_vm(
-				pipe_ctx->plane_res.hubp,
-				plane_state->format,
-				&plane_state->tiling_info,
-				plane_state->rotation,
-				hws
-				);
+	if (plane_state->update_flags.bits.full_update ||
+		plane_state->update_flags.bits.bpp_change)
+		update_dpp(dpp, plane_state);
 
-	dpp->funcs->ipp_setup(dpp,
-			plane_state->format,
-			EXPANSION_MODE_ZERO);
+	if (plane_state->update_flags.bits.full_update ||
+		plane_state->update_flags.bits.per_pixel_alpha_change)
+		update_mpcc(dc, pipe_ctx);
 
-	mpcc_cfg.dpp_id = hubp->inst;
-	mpcc_cfg.opp_id = pipe_ctx->stream_res.opp->inst;
-	mpcc_cfg.tree_cfg = &(pipe_ctx->stream_res.opp->mpc_tree);
-	for (top_pipe = pipe_ctx->top_pipe; top_pipe; top_pipe = top_pipe->top_pipe)
-		mpcc_cfg.z_index++;
-	if (dc->debug.surface_visual_confirm)
-		dcn10_get_surface_visual_confirm_color(
-				pipe_ctx, &mpcc_cfg.black_color);
-	else
-		color_space_to_black_color(
-			dc, pipe_ctx->stream->output_color_space,
-			&mpcc_cfg.black_color);
-	mpcc_cfg.per_pixel_alpha = per_pixel_alpha;
-	/* DCN1.0 has output CM before MPC which seems to screw with
-	 * pre-multiplied alpha.
-	 */
-	mpcc_cfg.pre_multiplied_alpha = is_rgb_cspace(
-			pipe_ctx->stream->output_color_space)
-					&& per_pixel_alpha;
-	hubp->mpcc_id = dc->res_pool->mpc->funcs->add(dc->res_pool->mpc, &mpcc_cfg);
-	hubp->opp_id = mpcc_cfg.opp_id;
+	if (plane_state->update_flags.bits.full_update ||
+		plane_state->update_flags.bits.per_pixel_alpha_change ||
+		plane_state->update_flags.bits.scaling_change ||
+		plane_state->update_flags.bits.position_change) {
+		update_scaler(pipe_ctx);
+	}
 
-	pipe_ctx->plane_res.scl_data.lb_params.alpha_en = per_pixel_alpha;
-	pipe_ctx->plane_res.scl_data.lb_params.depth = LB_PIXEL_DEPTH_30BPP;
-	/* scaler configuration */
-	pipe_ctx->plane_res.dpp->funcs->dpp_set_scaler(
-			pipe_ctx->plane_res.dpp, &pipe_ctx->plane_res.scl_data);
+	if (plane_state->update_flags.bits.full_update ||
+		plane_state->update_flags.bits.scaling_change ||
+		plane_state->update_flags.bits.position_change) {
+		hubp->funcs->mem_program_viewport(
+			hubp,
+			&pipe_ctx->plane_res.scl_data.viewport,
+			&pipe_ctx->plane_res.scl_data.viewport_c);
+	}
+
+	if (pipe_ctx->stream->cursor_attributes.address.quad_part != 0) {
+		dc->hwss.set_cursor_position(pipe_ctx);
+		dc->hwss.set_cursor_attribute(pipe_ctx);
+	}
 
-	hubp->funcs->mem_program_viewport(hubp,
-			&pipe_ctx->plane_res.scl_data.viewport, &pipe_ctx->plane_res.scl_data.viewport_c);
+	if (plane_state->update_flags.bits.full_update) {
+		/*gamut remap*/
+		program_gamut_remap(pipe_ctx);
 
-	/*gamut remap*/
-	program_gamut_remap(pipe_ctx);
+		program_output_csc(dc,
+				pipe_ctx,
+				pipe_ctx->stream->output_color_space,
+				pipe_ctx->stream->csc_color_matrix.matrix,
+				hubp->opp_id);
+	}
 
-	program_csc_matrix(pipe_ctx,
-			pipe_ctx->stream->output_color_space,
-			pipe_ctx->stream->csc_color_matrix.matrix);
+	if (plane_state->update_flags.bits.full_update ||
+		plane_state->update_flags.bits.horizontal_mirror_change ||
+		plane_state->update_flags.bits.rotation_change ||
+		plane_state->update_flags.bits.swizzle_change ||
+		plane_state->update_flags.bits.dcc_change ||
+		plane_state->update_flags.bits.bpp_change ||
+		plane_state->update_flags.bits.scaling_change) {
+		hubp->funcs->hubp_program_surface_config(
+			hubp,
+			plane_state->format,
+			&plane_state->tiling_info,
+			&size,
+			plane_state->rotation,
+			&plane_state->dcc,
+			plane_state->horizontal_mirror);
+	}
 
-	hubp->funcs->hubp_program_surface_config(
-		hubp,
-		plane_state->format,
-		&plane_state->tiling_info,
-		&size,
-		plane_state->rotation,
-		&plane_state->dcc,
-		plane_state->horizontal_mirror);
+	hubp->power_gated = false;
 
 	dc->hwss.update_plane_addr(dc, pipe_ctx);
 
@@ -2387,23 +1832,9 @@ static void program_all_pipe_in_tree(
 		struct pipe_ctx *pipe_ctx,
 		struct dc_state *context)
 {
-	unsigned int ref_clk_mhz = dc->res_pool->ref_clock_inKhz/1000;
 
 	if (pipe_ctx->top_pipe == NULL) {
 
-		/* lock otg_master_update to process all pipes associated with
-		 * this OTG. this is done only one time.
-		 */
-		/* watermark is for all pipes */
-		program_watermarks(dc->hwseq, &context->bw.dcn.watermarks, ref_clk_mhz);
-
-		if (dc->debug.sanity_checks) {
-			/* pstate stuck check after watermark update */
-			verify_allow_pstate_change_high(dc->hwseq);
-		}
-
-		pipe_ctx->stream_res.tg->funcs->lock(pipe_ctx->stream_res.tg);
-
 		pipe_ctx->stream_res.tg->dlg_otg_param.vready_offset = pipe_ctx->pipe_dlg_param.vready_offset;
 		pipe_ctx->stream_res.tg->dlg_otg_param.vstartup_start = pipe_ctx->pipe_dlg_param.vstartup_start;
 		pipe_ctx->stream_res.tg->dlg_otg_param.vupdate_offset = pipe_ctx->pipe_dlg_param.vupdate_offset;
@@ -2412,46 +1843,33 @@ static void program_all_pipe_in_tree(
 
 		pipe_ctx->stream_res.tg->funcs->program_global_sync(
 				pipe_ctx->stream_res.tg);
-		pipe_ctx->stream_res.tg->funcs->set_blank(pipe_ctx->stream_res.tg, !is_pipe_tree_visible(pipe_ctx));
+
+		if (pipe_ctx->stream_res.tg->funcs->set_blank)
+			pipe_ctx->stream_res.tg->funcs->set_blank(
+					pipe_ctx->stream_res.tg,
+					!is_pipe_tree_visible(pipe_ctx));
 	}
 
 	if (pipe_ctx->plane_state != NULL) {
-		struct dc_cursor_position position = { 0 };
 		struct pipe_ctx *cur_pipe_ctx =
 				&dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx];
 
-		dcn10_power_on_fe(dc, pipe_ctx, context);
-
-		/* temporary dcn1 wa:
-		 *   watermark update requires toggle after a/b/c/d sets are programmed
-		 *   if hubp is pg then wm value doesn't get properaged to hubp
-		 *   need to toggle after ungate to ensure wm gets to hubp.
-		 *
-		 * final solution:  we need to get SMU to do the toggle as
-		 * DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST is owned by SMU we should have
-		 * both driver and fw accessing same register
-		 */
-		toggle_watermark_change_req(dc->hwseq);
+		if (pipe_ctx->plane_state->update_flags.bits.full_update)
+			dcn10_enable_plane(dc, pipe_ctx, context);
 
 		update_dchubp_dpp(dc, pipe_ctx, context);
 
-		/* TODO: this is a hack w/a for switching from mpo to pipe split */
-		dc_stream_set_cursor_position(pipe_ctx->stream, &position);
-
-		dc_stream_set_cursor_attributes(pipe_ctx->stream,
-				&pipe_ctx->stream->cursor_attributes);
-
-		if (cur_pipe_ctx->plane_state != pipe_ctx->plane_state) {
-			dc->hwss.set_input_transfer_func(
-					pipe_ctx, pipe_ctx->plane_state);
-			dc->hwss.set_output_transfer_func(
-					pipe_ctx, pipe_ctx->stream);
-		}
-	}
+		if (cur_pipe_ctx->plane_state != pipe_ctx->plane_state)
+			dc->hwss.set_input_transfer_func(pipe_ctx, pipe_ctx->plane_state);
 
-	if (dc->debug.sanity_checks) {
-		/* pstate stuck check after each pipe is programmed */
-		verify_allow_pstate_change_high(dc->hwseq);
+		/* dcn10_translate_regamma_to_hw_format takes 750us to finish
+		 * only do gamma programming for full update.
+		 * TODO: This can be further optimized/cleaned up
+		 * Always call this for now since it does memcmp inside before
+		 * doing heavy calculation and programming
+		 */
+		if (pipe_ctx->plane_state->update_flags.bits.full_update)
+			dc->hwss.set_output_transfer_func(pipe_ctx, pipe_ctx->stream);
 	}
 
 	if (pipe_ctx->bottom_pipe != NULL && pipe_ctx->bottom_pipe != pipe_ctx)
@@ -2488,7 +1906,6 @@ static void dcn10_pplib_apply_display_requirements(
 static void optimize_shared_resources(struct dc *dc)
 {
 	if (dc->current_state->stream_count == 0) {
-		apply_DEGVIDCN10_253_wa(dc);
 		/* S0i2 message */
 		dcn10_pplib_apply_display_requirements(dc, dc->current_state);
 	}
@@ -2499,67 +1916,84 @@ static void optimize_shared_resources(struct dc *dc)
 
 static void ready_shared_resources(struct dc *dc, struct dc_state *context)
 {
-	if (dc->current_state->stream_count == 0 &&
-			!dc->debug.disable_stutter)
-		undo_DEGVIDCN10_253_wa(dc);
-
 	/* S0i2 message */
 	if (dc->current_state->stream_count == 0 &&
 			context->stream_count != 0)
 		dcn10_pplib_apply_display_requirements(dc, context);
 }
 
+static struct pipe_ctx *find_top_pipe_for_stream(
+		struct dc *dc,
+		struct dc_state *context,
+		const struct dc_stream_state *stream)
+{
+	int i;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *old_pipe_ctx =
+				&dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (!pipe_ctx->plane_state && !old_pipe_ctx->plane_state)
+			continue;
+
+		if (pipe_ctx->stream != stream)
+			continue;
+
+		if (!pipe_ctx->top_pipe)
+			return pipe_ctx;
+	}
+	return NULL;
+}
+
 static void dcn10_apply_ctx_for_surface(
 		struct dc *dc,
 		const struct dc_stream_state *stream,
 		int num_planes,
 		struct dc_state *context)
 {
-	int i, be_idx;
+	int i;
+	struct timing_generator *tg;
+	struct output_pixel_processor *opp;
+	bool removed_pipe[4] = { false };
+	unsigned int ref_clk_mhz = dc->res_pool->ref_clock_inKhz/1000;
+	bool program_water_mark = false;
 
-	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
+	struct pipe_ctx *top_pipe_to_program =
+			find_top_pipe_for_stream(dc, context, stream);
 
-	be_idx = -1;
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		if (stream == context->res_ctx.pipe_ctx[i].stream) {
-			be_idx = context->res_ctx.pipe_ctx[i].stream_res.tg->inst;
-			break;
-		}
-	}
+	if (!top_pipe_to_program)
+		return;
+
+	opp = top_pipe_to_program->stream_res.opp;
+
+	tg = top_pipe_to_program->stream_res.tg;
 
-	ASSERT(be_idx != -1);
+	tg->funcs->lock(tg);
 
 	if (num_planes == 0) {
-		for (i = dc->res_pool->pipe_count - 1; i >= 0 ; i--) {
-			struct pipe_ctx *old_pipe_ctx =
-							&dc->current_state->res_ctx.pipe_ctx[i];
 
-			if (old_pipe_ctx->stream_res.tg && old_pipe_ctx->stream_res.tg->inst == be_idx) {
-				old_pipe_ctx->stream_res.tg->funcs->set_blank(old_pipe_ctx->stream_res.tg, true);
-				dcn10_power_down_fe(dc, old_pipe_ctx->pipe_idx);
-			}
-		}
-		return;
+		/* OTG blank before remove all front end */
+		if (tg->funcs->set_blank)
+			tg->funcs->set_blank(tg, true);
 	}
 
-	/* reset unused mpcc */
+	/* Disconnect unused mpcc */
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 		struct pipe_ctx *old_pipe_ctx =
 				&dc->current_state->res_ctx.pipe_ctx[i];
-
-		if (!pipe_ctx->plane_state && !old_pipe_ctx->plane_state)
-			continue;
-
 		/*
 		 * Powergate reused pipes that are not powergated
 		 * fairly hacky right now, using opp_id as indicator
+		 * TODO: After move dc_post to dc_update, this will
+		 * be removed.
 		 */
-
 		if (pipe_ctx->plane_state && !old_pipe_ctx->plane_state) {
-			if (pipe_ctx->plane_res.hubp->opp_id != 0xf && pipe_ctx->stream_res.tg->inst == be_idx) {
-				dcn10_power_down_fe(dc, pipe_ctx->pipe_idx);
+			if (old_pipe_ctx->stream_res.tg == tg &&
+				old_pipe_ctx->plane_res.hubp &&
+				old_pipe_ctx->plane_res.hubp->opp_id != 0xf) {
+				dcn10_disable_plane(dc, pipe_ctx);
 				/*
 				 * power down fe will unlock when calling reset, need
 				 * to lock it back here. Messy, need rework.
@@ -2568,36 +2002,12 @@ static void dcn10_apply_ctx_for_surface(
 			}
 		}
 
+		if (!pipe_ctx->plane_state &&
+			old_pipe_ctx->plane_state &&
+			old_pipe_ctx->stream_res.tg == tg) {
 
-		if ((!pipe_ctx->plane_state && old_pipe_ctx->plane_state)
-				|| (!pipe_ctx->stream && old_pipe_ctx->stream)) {
-			if (old_pipe_ctx->stream_res.tg->inst != be_idx)
-				continue;
-
-			if (!old_pipe_ctx->top_pipe) {
-				ASSERT(0);
-				continue;
-			}
-
-			/* reset mpc */
-			dc->res_pool->mpc->funcs->remove(
-					dc->res_pool->mpc,
-					&(old_pipe_ctx->stream_res.opp->mpc_tree),
-					old_pipe_ctx->stream_res.opp->inst,
-					old_pipe_ctx->pipe_idx);
-			old_pipe_ctx->stream_res.opp->mpcc_disconnect_pending[old_pipe_ctx->plane_res.hubp->mpcc_id] = true;
-
-			/*dm_logger_write(dc->ctx->logger, LOG_ERROR,
-					"[debug_mpo: apply_ctx disconnect pending on mpcc %d]\n",
-					old_pipe_ctx->mpcc->inst);*/
-
-			if (dc->debug.sanity_checks)
-				verify_allow_pstate_change_high(dc->hwseq);
-
-			old_pipe_ctx->top_pipe = NULL;
-			old_pipe_ctx->bottom_pipe = NULL;
-			old_pipe_ctx->plane_state = NULL;
-			old_pipe_ctx->stream = NULL;
+			plane_atomic_disconnect(dc, old_pipe_ctx);
+			removed_pipe[i] = true;
 
 			dm_logger_write(dc->ctx->logger, LOG_DC,
 					"Reset mpcc for pipe %d\n",
@@ -2605,18 +2015,44 @@ static void dcn10_apply_ctx_for_surface(
 		}
 	}
 
+	if (num_planes > 0)
+		program_all_pipe_in_tree(dc, top_pipe_to_program, context);
+
+	tg->funcs->unlock(tg);
+
+	if (num_planes == 0)
+		false_optc_underflow_wa(dc, stream, tg);
+
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *old_pipe_ctx =
+				&dc->current_state->res_ctx.pipe_ctx[i];
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
-		if (pipe_ctx->stream != stream)
-			continue;
+		if (pipe_ctx->stream == stream &&
+				pipe_ctx->plane_state &&
+			pipe_ctx->plane_state->update_flags.bits.full_update)
+			program_water_mark = true;
 
-		/* looking for top pipe to program */
-		if (!pipe_ctx->top_pipe)
-			program_all_pipe_in_tree(dc, pipe_ctx, context);
+		if (removed_pipe[i])
+			dcn10_disable_plane(dc, old_pipe_ctx);
 	}
 
-	dm_logger_write(dc->ctx->logger, LOG_BANDWIDTH_CALCS,
+	if (program_water_mark) {
+		if (dc->debug.sanity_checks) {
+			/* pstate stuck check after watermark update */
+			dcn10_verify_allow_pstate_change_high(dc);
+		}
+
+		/* watermark is for all pipes */
+		hubbub1_program_watermarks(dc->res_pool->hubbub,
+				&context->bw.dcn.watermarks, ref_clk_mhz);
+
+		if (dc->debug.sanity_checks) {
+			/* pstate stuck check after watermark update */
+			dcn10_verify_allow_pstate_change_high(dc);
+		}
+	}
+/*	dm_logger_write(dc->ctx->logger, LOG_BANDWIDTH_CALCS,
 			"\n============== Watermark parameters ==============\n"
 			"a.urgent_ns: %d \n"
 			"a.cstate_enter_plus_exit: %d \n"
@@ -2662,9 +2098,7 @@ static void dcn10_apply_ctx_for_surface(
 			context->bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns,
 			context->bw.dcn.watermarks.d.pte_meta_urgent_ns
 			);
-
-	if (dc->debug.sanity_checks)
-		verify_allow_pstate_change_high(dc->hwseq);
+*/
 }
 
 static void dcn10_set_bandwidth(
@@ -2678,7 +2112,7 @@ static void dcn10_set_bandwidth(
 	struct pp_smu_funcs_rv *pp_smu = dc->res_pool->pp_smu;
 
 	if (dc->debug.sanity_checks) {
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 	}
 
 	if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
@@ -2734,7 +2168,7 @@ static void dcn10_set_bandwidth(
 	dcn10_pplib_apply_display_requirements(dc, context);
 
 	if (dc->debug.sanity_checks) {
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 	}
 
 	/* need to fix this function.  not doing the right thing here */
@@ -2838,10 +2272,10 @@ static void dcn10_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc)
 
 	dcn10_config_stereo_parameters(stream, &flags);
 
-	pipe_ctx->stream_res.opp->funcs->opp_set_stereo_polarity(
+	pipe_ctx->stream_res.opp->funcs->opp_program_stereo(
 		pipe_ctx->stream_res.opp,
 		flags.PROGRAM_STEREO == 1 ? true:false,
-		stream->timing.flags.RIGHT_EYE_3D_POLARITY == 1 ? true:false);
+		&stream->timing);
 
 	pipe_ctx->stream_res.tg->funcs->program_stereo(
 		pipe_ctx->stream_res.tg,
@@ -2859,7 +2293,7 @@ static void dcn10_wait_for_mpcc_disconnect(
 	int i;
 
 	if (dc->debug.sanity_checks) {
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 	}
 
 	if (!pipe_ctx->stream_res.opp)
@@ -2877,7 +2311,7 @@ static void dcn10_wait_for_mpcc_disconnect(
 	}
 
 	if (dc->debug.sanity_checks) {
-		verify_allow_pstate_change_high(dc->hwseq);
+		dcn10_verify_allow_pstate_change_high(dc);
 	}
 
 }
@@ -2891,7 +2325,7 @@ static bool dcn10_dummy_display_power_gating(
 	return true;
 }
 
-void dcn10_update_pending_status(struct pipe_ctx *pipe_ctx)
+static void dcn10_update_pending_status(struct pipe_ctx *pipe_ctx)
 {
 	struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 	struct timing_generator *tg = pipe_ctx->stream_res.tg;
@@ -2911,7 +2345,45 @@ void dcn10_update_pending_status(struct pipe_ctx *pipe_ctx)
 	}
 }
 
+static void dcn10_update_dchub(struct dce_hwseq *hws, struct dchub_init_data *dh_data)
+{
+	if (hws->ctx->dc->res_pool->hubbub != NULL)
+		hubbub1_update_dchub(hws->ctx->dc->res_pool->hubbub, dh_data);
+}
+
+static void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
+{
+	struct dc_cursor_position pos_cpy = pipe_ctx->stream->cursor_position;
+	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+	struct dpp *dpp = pipe_ctx->plane_res.dpp;
+	struct dc_cursor_mi_param param = {
+		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_khz,
+		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clock_inKhz,
+		.viewport_x_start = pipe_ctx->plane_res.scl_data.viewport.x,
+		.viewport_width = pipe_ctx->plane_res.scl_data.viewport.width,
+		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz
+	};
+
+	if (pipe_ctx->plane_state->address.type
+			== PLN_ADDR_TYPE_VIDEO_PROGRESSIVE)
+		pos_cpy.enable = false;
+
+	if (pipe_ctx->top_pipe && pipe_ctx->plane_state != pipe_ctx->top_pipe->plane_state)
+		pos_cpy.enable = false;
+
+	hubp->funcs->set_cursor_position(hubp, &pos_cpy, &param);
+	dpp->funcs->set_cursor_position(dpp, &pos_cpy, &param, hubp->curs_attr.width);
+}
+
+static void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx)
+{
+	struct dc_cursor_attributes *attributes = &pipe_ctx->stream->cursor_attributes;
 
+	pipe_ctx->plane_res.hubp->funcs->set_cursor_attributes(
+			pipe_ctx->plane_res.hubp, attributes);
+	pipe_ctx->plane_res.dpp->funcs->set_cursor_attributes(
+		pipe_ctx->plane_res.dpp, attributes->color_format);
+}
 
 static const struct hw_sequencer_funcs dcn10_funcs = {
 	.program_gamut_remap = program_gamut_remap,
@@ -2928,13 +2400,13 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.power_down = dce110_power_down,
 	.enable_accelerated_mode = dce110_enable_accelerated_mode,
 	.enable_timing_synchronization = dcn10_enable_timing_synchronization,
+	.enable_per_frame_crtc_position_reset = dcn10_enable_per_frame_crtc_position_reset,
 	.update_info_frame = dce110_update_info_frame,
 	.enable_stream = dce110_enable_stream,
 	.disable_stream = dce110_disable_stream,
 	.unblank_stream = dce110_unblank_stream,
 	.enable_display_power_gating = dcn10_dummy_display_power_gating,
-	.power_down_front_end = dcn10_power_down_fe,
-	.power_on_front_end = dcn10_power_on_fe,
+	.disable_plane = dcn10_disable_plane,
 	.pipe_control_lock = dcn10_pipe_control_lock,
 	.set_bandwidth = dcn10_set_bandwidth,
 	.reset_hw_ctx_wrap = reset_hw_ctx_wrap,
@@ -2948,8 +2420,13 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.wait_for_mpcc_disconnect = dcn10_wait_for_mpcc_disconnect,
 	.ready_shared_resources = ready_shared_resources,
 	.optimize_shared_resources = optimize_shared_resources,
+	.pplib_apply_display_requirements =
+			dcn10_pplib_apply_display_requirements,
 	.edp_backlight_control = hwss_edp_backlight_control,
-	.edp_power_control = hwss_edp_power_control
+	.edp_power_control = hwss_edp_power_control,
+	.edp_wait_for_hpd_ready = hwss_edp_wait_for_hpd_ready,
+	.set_cursor_position = dcn10_set_cursor_position,
+	.set_cursor_attribute = dcn10_set_cursor_attribute
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
index ca53dc1cc19b..b9d326082717 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
@@ -35,4 +35,5 @@ extern void fill_display_configs(
 	const struct dc_state *context,
 	struct dm_pp_display_configuration *pp_display_cfg);
 
+
 #endif /* __DC_HWSS_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
index 76573e1f5b01..179890b1a8c4 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.c
@@ -25,8 +25,6 @@
 
 #include "reg_helper.h"
 #include "dcn10_mpc.h"
-#include "dc.h"
-#include "mem_input.h"
 
 #define REG(reg)\
 	mpc10->mpc_regs->reg
@@ -38,17 +36,13 @@
 #define FN(reg_name, field_name) \
 	mpc10->mpc_shift->field_name, mpc10->mpc_mask->field_name
 
-#define MODE_TOP_ONLY 1
-#define MODE_BLEND 3
-#define BLND_PP_ALPHA 0
-#define BLND_GLOBAL_ALPHA 2
 
-
-static void mpc10_set_bg_color(
-		struct dcn10_mpc *mpc10,
+void mpc1_set_bg_color(struct mpc *mpc,
 		struct tg_color *bg_color,
-		int id)
+		int mpcc_id)
 {
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
+
 	/* mpc color is 12 bit.  tg_color is 10 bit */
 	/* todo: might want to use 16 bit to represent color and have each
 	 * hw block translate to correct color depth.
@@ -57,15 +51,47 @@ static void mpc10_set_bg_color(
 	uint32_t bg_g_y = bg_color->color_g_y << 2;
 	uint32_t bg_b_cb = bg_color->color_b_cb << 2;
 
-	REG_SET(MPCC_BG_R_CR[id], 0,
+	REG_SET(MPCC_BG_R_CR[mpcc_id], 0,
 			MPCC_BG_R_CR, bg_r_cr);
-	REG_SET(MPCC_BG_G_Y[id], 0,
+	REG_SET(MPCC_BG_G_Y[mpcc_id], 0,
 			MPCC_BG_G_Y, bg_g_y);
-	REG_SET(MPCC_BG_B_CB[id], 0,
+	REG_SET(MPCC_BG_B_CB[mpcc_id], 0,
 			MPCC_BG_B_CB, bg_b_cb);
 }
 
-void mpc10_assert_idle_mpcc(struct mpc *mpc, int id)
+static void mpc1_update_blending(
+	struct mpc *mpc,
+	struct mpcc_blnd_cfg *blnd_cfg,
+	int mpcc_id)
+{
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
+
+	REG_UPDATE_5(MPCC_CONTROL[mpcc_id],
+			MPCC_ALPHA_BLND_MODE,		blnd_cfg->alpha_mode,
+			MPCC_ALPHA_MULTIPLIED_MODE,	blnd_cfg->pre_multiplied_alpha,
+			MPCC_BLND_ACTIVE_OVERLAP_ONLY,	blnd_cfg->overlap_only,
+			MPCC_GLOBAL_ALPHA,		blnd_cfg->global_alpha,
+			MPCC_GLOBAL_GAIN,		blnd_cfg->global_gain);
+
+	mpc1_set_bg_color(mpc, &blnd_cfg->black_color, mpcc_id);
+}
+
+void mpc1_update_stereo_mix(
+	struct mpc *mpc,
+	struct mpcc_sm_cfg *sm_cfg,
+	int mpcc_id)
+{
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
+
+	REG_UPDATE_6(MPCC_SM_CONTROL[mpcc_id],
+			MPCC_SM_EN,			sm_cfg->enable,
+			MPCC_SM_MODE,			sm_cfg->sm_mode,
+			MPCC_SM_FRAME_ALT,		sm_cfg->frame_alt,
+			MPCC_SM_FIELD_ALT,		sm_cfg->field_alt,
+			MPCC_SM_FORCE_NEXT_FRAME_POL,	sm_cfg->force_next_frame_porlarity,
+			MPCC_SM_FORCE_NEXT_TOP_POL,	sm_cfg->force_next_field_polarity);
+}
+void mpc1_assert_idle_mpcc(struct mpc *mpc, int id)
 {
 	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
 
@@ -75,39 +101,52 @@ void mpc10_assert_idle_mpcc(struct mpc *mpc, int id)
 			1, 100000);
 }
 
-static int mpc10_get_idle_mpcc_id(struct dcn10_mpc *mpc10)
+struct mpcc *mpc1_get_mpcc(struct mpc *mpc, int mpcc_id)
 {
-	int i;
-	int last_free_mpcc_id = -1;
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
 
-	for (i = 0; i < mpc10->num_mpcc; i++) {
-		uint32_t is_idle = 0;
+	ASSERT(mpcc_id < mpc10->num_mpcc);
+	return &(mpc->mpcc_array[mpcc_id]);
+}
 
-		if (mpc10->mpcc_in_use_mask & 1 << i)
-			continue;
+struct mpcc *mpc1_get_mpcc_for_dpp(struct mpc_tree *tree, int dpp_id)
+{
+	struct mpcc *tmp_mpcc = tree->opp_list;
 
-		last_free_mpcc_id = i;
-		REG_GET(MPCC_STATUS[i], MPCC_IDLE, &is_idle);
-		if (is_idle)
-			return i;
+	while (tmp_mpcc != NULL) {
+		if (tmp_mpcc->dpp_id == dpp_id)
+			return tmp_mpcc;
+		tmp_mpcc = tmp_mpcc->mpcc_bot;
 	}
+	return NULL;
+}
 
-	/* This assert should never trigger, we have mpcc leak if it does */
-	ASSERT(last_free_mpcc_id != -1);
-
-	mpc10_assert_idle_mpcc(&mpc10->base, last_free_mpcc_id);
-	return last_free_mpcc_id;
+bool mpc1_is_mpcc_idle(struct mpc *mpc, int mpcc_id)
+{
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
+	unsigned int top_sel;
+	unsigned int opp_id;
+	unsigned int idle;
+
+	REG_GET(MPCC_TOP_SEL[mpcc_id], MPCC_TOP_SEL, &top_sel);
+	REG_GET(MPCC_OPP_ID[mpcc_id],  MPCC_OPP_ID, &opp_id);
+	REG_GET(MPCC_STATUS[mpcc_id],  MPCC_IDLE,   &idle);
+	if (top_sel == 0xf && opp_id == 0xf && idle)
+		return true;
+	else
+		return false;
 }
 
-static void mpc10_assert_mpcc_idle_before_connect(struct dcn10_mpc *mpc10, int id)
+void mpc1_assert_mpcc_idle_before_connect(struct mpc *mpc, int mpcc_id)
 {
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
 	unsigned int top_sel, mpc_busy, mpc_idle;
 
-	REG_GET(MPCC_TOP_SEL[id],
+	REG_GET(MPCC_TOP_SEL[mpcc_id],
 			MPCC_TOP_SEL, &top_sel);
 
 	if (top_sel == 0xf) {
-		REG_GET_2(MPCC_STATUS[id],
+		REG_GET_2(MPCC_STATUS[mpcc_id],
 				MPCC_BUSY, &mpc_busy,
 				MPCC_IDLE, &mpc_idle);
 
@@ -116,230 +155,269 @@ static void mpc10_assert_mpcc_idle_before_connect(struct dcn10_mpc *mpc10, int i
 	}
 }
 
-void mpc10_mpcc_remove(
-		struct mpc *mpc,
-		struct mpc_tree_cfg *tree_cfg,
-		int opp_id,
-		int dpp_id)
-{
-	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
-	int mpcc_id, z_idx;
-
-	/* find z_idx for the dpp to be removed */
-	for (z_idx = 0; z_idx < tree_cfg->num_pipes; z_idx++)
-		if (tree_cfg->dpp[z_idx] == dpp_id)
-			break;
-
-	if (z_idx == tree_cfg->num_pipes) {
-		/* In case of resume from S3/S4, remove mpcc from bios left over */
-		REG_SET(MPCC_OPP_ID[dpp_id], 0,
-				MPCC_OPP_ID, 0xf);
-		REG_SET(MPCC_TOP_SEL[dpp_id], 0,
-				MPCC_TOP_SEL, 0xf);
-		REG_SET(MPCC_BOT_SEL[dpp_id], 0,
-				MPCC_BOT_SEL, 0xf);
-		return;
-	}
-
-	mpcc_id = tree_cfg->mpcc[z_idx];
-
-	REG_SET(MPCC_OPP_ID[mpcc_id], 0,
-			MPCC_OPP_ID, 0xf);
-	REG_SET(MPCC_TOP_SEL[mpcc_id], 0,
-			MPCC_TOP_SEL, 0xf);
-	REG_SET(MPCC_BOT_SEL[mpcc_id], 0,
-			MPCC_BOT_SEL, 0xf);
-
-	if (z_idx > 0) {
-		int top_mpcc_id = tree_cfg->mpcc[z_idx - 1];
-
-		if (z_idx + 1 < tree_cfg->num_pipes)
-			/* mpcc to be removed is in the middle of the tree */
-			REG_SET(MPCC_BOT_SEL[top_mpcc_id], 0,
-					MPCC_BOT_SEL, tree_cfg->mpcc[z_idx + 1]);
-		else {
-			/* mpcc to be removed is at the bottom of the tree */
-			REG_SET(MPCC_BOT_SEL[top_mpcc_id], 0,
-					MPCC_BOT_SEL, 0xf);
-			REG_UPDATE(MPCC_CONTROL[top_mpcc_id],
-					MPCC_MODE, MODE_TOP_ONLY);
-		}
-	} else if (tree_cfg->num_pipes > 1)
-		/* mpcc to be removed is at the top of the tree */
-		REG_SET(MUX[opp_id], 0,
-				MPC_OUT_MUX, tree_cfg->mpcc[z_idx + 1]);
-	else
-		/* mpcc to be removed is the only one in the tree */
-		REG_SET(MUX[opp_id], 0, MPC_OUT_MUX, 0xf);
-
-	/* mark this mpcc as not in use */
-	mpc10->mpcc_in_use_mask &= ~(1 << mpcc_id);
-	tree_cfg->num_pipes--;
-	for (; z_idx < tree_cfg->num_pipes; z_idx++) {
-		tree_cfg->dpp[z_idx] = tree_cfg->dpp[z_idx + 1];
-		tree_cfg->mpcc[z_idx] = tree_cfg->mpcc[z_idx + 1];
-	}
-	tree_cfg->dpp[tree_cfg->num_pipes] = 0xdeadbeef;
-	tree_cfg->mpcc[tree_cfg->num_pipes] = 0xdeadbeef;
-}
-
-static void mpc10_add_to_tree_cfg(
+/*
+ * Insert DPP into MPC tree based on specified blending position.
+ * Only used for planes that are part of blending chain for OPP output
+ *
+ * Parameters:
+ * [in/out] mpc		- MPC context.
+ * [in/out] tree	- MPC tree structure that plane will be added to.
+ * [in]	blnd_cfg	- MPCC blending configuration for the new blending layer.
+ * [in]	sm_cfg		- MPCC stereo mix configuration for the new blending layer.
+ *			  stereo mix must disable for the very bottom layer of the tree config.
+ * [in]	insert_above_mpcc - Insert new plane above this MPCC.  If NULL, insert as bottom plane.
+ * [in]	dpp_id		- DPP instance for the plane to be added.
+ * [in]	mpcc_id		- The MPCC physical instance to use for blending.
+ *
+ * Return:  struct mpcc* - MPCC that was added.
+ */
+struct mpcc *mpc1_insert_plane(
 	struct mpc *mpc,
-	struct mpcc_cfg *cfg,
+	struct mpc_tree *tree,
+	struct mpcc_blnd_cfg *blnd_cfg,
+	struct mpcc_sm_cfg *sm_cfg,
+	struct mpcc *insert_above_mpcc,
+	int dpp_id,
 	int mpcc_id)
 {
 	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
-	int mpcc_mode = MODE_TOP_ONLY;
-	int position = cfg->z_index;
-	struct mpc_tree_cfg *tree_cfg = cfg->tree_cfg;
-	int alpha_blnd_mode = cfg->per_pixel_alpha ?
-			BLND_PP_ALPHA : BLND_GLOBAL_ALPHA;
-	int z_idx;
+	struct mpcc *new_mpcc = NULL;
 
-	REG_SET(MPCC_OPP_ID[mpcc_id], 0,
-			MPCC_OPP_ID, cfg->opp_id);
+	/* sanity check parameters */
+	ASSERT(mpcc_id < mpc10->num_mpcc);
+	ASSERT(!(mpc10->mpcc_in_use_mask & 1 << mpcc_id));
 
-	REG_SET(MPCC_TOP_SEL[mpcc_id], 0,
-			MPCC_TOP_SEL, cfg->dpp_id);
+	if (insert_above_mpcc) {
+		/* check insert_above_mpcc exist in tree->opp_list */
+		struct mpcc *temp_mpcc = tree->opp_list;
 
-	if (position == 0) {
-		/* idle dpp/mpcc is added to the top layer of tree */
+		while (temp_mpcc && temp_mpcc->mpcc_bot != insert_above_mpcc)
+			temp_mpcc = temp_mpcc->mpcc_bot;
+		if (temp_mpcc == NULL)
+			return NULL;
+	}
 
-		if (tree_cfg->num_pipes > 0) {
-			/* get instance of previous top mpcc */
-			int prev_top_mpcc_id = tree_cfg->mpcc[0];
+	/* Get and update MPCC struct parameters */
+	new_mpcc = mpc1_get_mpcc(mpc, mpcc_id);
+	new_mpcc->dpp_id = dpp_id;
 
-			REG_SET(MPCC_BOT_SEL[mpcc_id], 0,
-					MPCC_BOT_SEL, prev_top_mpcc_id);
-			mpcc_mode = MODE_BLEND;
+	/* program mux and MPCC_MODE */
+	if (insert_above_mpcc) {
+		new_mpcc->mpcc_bot = insert_above_mpcc;
+		REG_SET(MPCC_BOT_SEL[mpcc_id], 0, MPCC_BOT_SEL, insert_above_mpcc->mpcc_id);
+		REG_UPDATE(MPCC_CONTROL[mpcc_id], MPCC_MODE, MPCC_BLEND_MODE_TOP_BOT_BLENDING);
+	} else {
+		new_mpcc->mpcc_bot = NULL;
+		REG_SET(MPCC_BOT_SEL[mpcc_id], 0, MPCC_BOT_SEL, 0xf);
+		REG_UPDATE(MPCC_CONTROL[mpcc_id], MPCC_MODE, MPCC_BLEND_MODE_TOP_LAYER_PASSTHROUGH);
+	}
+	REG_SET(MPCC_TOP_SEL[mpcc_id], 0, MPCC_TOP_SEL, dpp_id);
+	REG_SET(MPCC_OPP_ID[mpcc_id], 0, MPCC_OPP_ID, tree->opp_id);
+
+	/* update mpc tree mux setting */
+	if (tree->opp_list == insert_above_mpcc) {
+		/* insert the toppest mpcc */
+		tree->opp_list = new_mpcc;
+		REG_UPDATE(MUX[tree->opp_id], MPC_OUT_MUX, mpcc_id);
+	} else {
+		/* find insert position */
+		struct mpcc *temp_mpcc = tree->opp_list;
+
+		while (temp_mpcc && temp_mpcc->mpcc_bot != insert_above_mpcc)
+			temp_mpcc = temp_mpcc->mpcc_bot;
+		if (temp_mpcc && temp_mpcc->mpcc_bot == insert_above_mpcc) {
+			REG_SET(MPCC_BOT_SEL[temp_mpcc->mpcc_id], 0, MPCC_BOT_SEL, mpcc_id);
+			temp_mpcc->mpcc_bot = new_mpcc;
+			if (!insert_above_mpcc)
+				REG_UPDATE(MPCC_CONTROL[temp_mpcc->mpcc_id],
+						MPCC_MODE, MPCC_BLEND_MODE_TOP_BOT_BLENDING);
 		}
+	}
 
-		/* opp will get new output. from new added mpcc */
-		REG_SET(MUX[cfg->opp_id], 0, MPC_OUT_MUX, mpcc_id);
-
-	} else if (position == tree_cfg->num_pipes) {
-		/* idle dpp/mpcc is added to the bottom layer of tree */
-
-		/* get instance of previous bottom mpcc, set to middle layer */
-		int prev_bot_mpcc_id = tree_cfg->mpcc[tree_cfg->num_pipes - 1];
-
-		REG_SET(MPCC_BOT_SEL[prev_bot_mpcc_id], 0,
-				MPCC_BOT_SEL, mpcc_id);
-		REG_UPDATE(MPCC_CONTROL[prev_bot_mpcc_id],
-				MPCC_MODE, MODE_BLEND);
-
-		/* mpcc_id become new bottom mpcc*/
-		REG_SET(MPCC_BOT_SEL[mpcc_id], 0,
-				MPCC_BOT_SEL, 0xf);
+	/* update the blending configuration */
+	new_mpcc->blnd_cfg = *blnd_cfg;
+	mpc->funcs->update_blending(mpc, &new_mpcc->blnd_cfg, mpcc_id);
 
-	} else {
-		/* idle dpp/mpcc is added to middle of tree */
-		int above_mpcc_id = tree_cfg->mpcc[position - 1];
-		int below_mpcc_id = tree_cfg->mpcc[position];
-
-		/* mpcc above new mpcc_id has new bottom mux*/
-		REG_SET(MPCC_BOT_SEL[above_mpcc_id], 0,
-				MPCC_BOT_SEL, mpcc_id);
-		REG_UPDATE(MPCC_CONTROL[above_mpcc_id],
-				MPCC_MODE, MODE_BLEND);
-
-		/* mpcc_id bottom mux is from below mpcc*/
-		REG_SET(MPCC_BOT_SEL[mpcc_id], 0,
-				MPCC_BOT_SEL, below_mpcc_id);
-		mpcc_mode = MODE_BLEND;
+	/* update the stereo mix settings, if provided */
+	if (sm_cfg != NULL) {
+		new_mpcc->sm_cfg = *sm_cfg;
+		mpc1_update_stereo_mix(mpc, sm_cfg, mpcc_id);
 	}
 
-	REG_SET_4(MPCC_CONTROL[mpcc_id], 0xffffffff,
-		MPCC_MODE, mpcc_mode,
-		MPCC_ALPHA_BLND_MODE, alpha_blnd_mode,
-		MPCC_ALPHA_MULTIPLIED_MODE, cfg->pre_multiplied_alpha,
-		MPCC_BLND_ACTIVE_OVERLAP_ONLY, false);
+	/* mark this mpcc as in use */
+	mpc10->mpcc_in_use_mask |= 1 << mpcc_id;
 
-	/* update mpc_tree_cfg with new mpcc */
-	for (z_idx = tree_cfg->num_pipes; z_idx > position; z_idx--) {
-		tree_cfg->dpp[z_idx] = tree_cfg->dpp[z_idx - 1];
-		tree_cfg->mpcc[z_idx] = tree_cfg->mpcc[z_idx - 1];
-	}
-	tree_cfg->dpp[position] = cfg->dpp_id;
-	tree_cfg->mpcc[position] = mpcc_id;
-	tree_cfg->num_pipes++;
+	return new_mpcc;
 }
 
-int mpc10_mpcc_add(struct mpc *mpc, struct mpcc_cfg *cfg)
+/*
+ * Remove a specified MPCC from the MPC tree.
+ *
+ * Parameters:
+ * [in/out] mpc		- MPC context.
+ * [in/out] tree	- MPC tree structure that plane will be removed from.
+ * [in/out] mpcc	- MPCC to be removed from tree.
+ *
+ * Return:  void
+ */
+void mpc1_remove_mpcc(
+	struct mpc *mpc,
+	struct mpc_tree *tree,
+	struct mpcc *mpcc_to_remove)
 {
 	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
-	int mpcc_id, z_idx;
-
-	ASSERT(cfg->z_index < mpc10->num_mpcc);
-
-	/* check in dpp already exists in mpc tree */
-	for (z_idx = 0; z_idx < cfg->tree_cfg->num_pipes; z_idx++)
-		if (cfg->tree_cfg->dpp[z_idx] == cfg->dpp_id)
-			break;
-	if (z_idx == cfg->tree_cfg->num_pipes) {
-		ASSERT(cfg->z_index <= cfg->tree_cfg->num_pipes);
-		mpcc_id = mpc10_get_idle_mpcc_id(mpc10);
-
-		/*
-		 * TODO: remove hack
-		 * Note: currently there is a bug in init_hw such that
-		 * on resume from hibernate, BIOS sets up MPCC0, and
-		 * we do mpcc_remove but the mpcc cannot go to idle
-		 * after remove. This cause us to pick mpcc1 here,
-		 * which causes a pstate hang for yet unknown reason.
-		 */
-		mpcc_id = cfg->dpp_id;
-		/* end hack*/
-
-		ASSERT(!(mpc10->mpcc_in_use_mask & 1 << mpcc_id));
-
-		if (mpc->ctx->dc->debug.sanity_checks)
-			mpc10_assert_mpcc_idle_before_connect(mpc10, mpcc_id);
+	bool found = false;
+	int mpcc_id = mpcc_to_remove->mpcc_id;
+
+	if (tree->opp_list == mpcc_to_remove) {
+		found = true;
+		/* remove MPCC from top of tree */
+		if (mpcc_to_remove->mpcc_bot) {
+			/* set the next MPCC in list to be the top MPCC */
+			tree->opp_list = mpcc_to_remove->mpcc_bot;
+			REG_UPDATE(MUX[tree->opp_id], MPC_OUT_MUX, tree->opp_list->mpcc_id);
+		} else {
+			/* there are no other MPCC is list */
+			tree->opp_list = NULL;
+			REG_UPDATE(MUX[tree->opp_id], MPC_OUT_MUX, 0xf);
+		}
 	} else {
-		ASSERT(cfg->z_index < cfg->tree_cfg->num_pipes);
-		mpcc_id = cfg->tree_cfg->mpcc[z_idx];
-		mpc10_mpcc_remove(mpc, cfg->tree_cfg, cfg->opp_id, cfg->dpp_id);
+		/* find mpcc to remove MPCC list */
+		struct mpcc *temp_mpcc = tree->opp_list;
+
+		while (temp_mpcc && temp_mpcc->mpcc_bot != mpcc_to_remove)
+			temp_mpcc = temp_mpcc->mpcc_bot;
+
+		if (temp_mpcc && temp_mpcc->mpcc_bot == mpcc_to_remove) {
+			found = true;
+			temp_mpcc->mpcc_bot = mpcc_to_remove->mpcc_bot;
+			if (mpcc_to_remove->mpcc_bot) {
+				/* remove MPCC in middle of list */
+				REG_SET(MPCC_BOT_SEL[temp_mpcc->mpcc_id], 0,
+						MPCC_BOT_SEL, mpcc_to_remove->mpcc_bot->mpcc_id);
+			} else {
+				/* remove MPCC from bottom of list */
+				REG_SET(MPCC_BOT_SEL[temp_mpcc->mpcc_id], 0,
+						MPCC_BOT_SEL, 0xf);
+				REG_UPDATE(MPCC_CONTROL[temp_mpcc->mpcc_id],
+						MPCC_MODE, MPCC_BLEND_MODE_TOP_LAYER_PASSTHROUGH);
+			}
+		}
 	}
 
-	/* add dpp/mpcc pair to mpc_tree_cfg and update mpcc registers */
-	mpc10_add_to_tree_cfg(mpc, cfg, mpcc_id);
-
-	/* set background color */
-	mpc10_set_bg_color(mpc10, &cfg->black_color, mpcc_id);
+	if (found) {
+		/* turn off MPCC mux registers */
+		REG_SET(MPCC_TOP_SEL[mpcc_id], 0, MPCC_TOP_SEL, 0xf);
+		REG_SET(MPCC_BOT_SEL[mpcc_id], 0, MPCC_BOT_SEL, 0xf);
+		REG_SET(MPCC_OPP_ID[mpcc_id],  0, MPCC_OPP_ID,  0xf);
 
-	/* mark this mpcc as in use */
-	mpc10->mpcc_in_use_mask |= 1 << mpcc_id;
+		/* mark this mpcc as not in use */
+		mpc10->mpcc_in_use_mask &= ~(1 << mpcc_id);
+		mpcc_to_remove->dpp_id = 0xf;
+		mpcc_to_remove->mpcc_bot = NULL;
+	} else {
+		/* In case of resume from S3/S4, remove mpcc from bios left over */
+		REG_SET(MPCC_TOP_SEL[mpcc_id], 0, MPCC_TOP_SEL, 0xf);
+		REG_SET(MPCC_BOT_SEL[mpcc_id], 0, MPCC_BOT_SEL, 0xf);
+		REG_SET(MPCC_OPP_ID[mpcc_id],  0, MPCC_OPP_ID,  0xf);
+	}
+}
 
-	return mpcc_id;
+static void mpc1_init_mpcc(struct mpcc *mpcc, int mpcc_inst)
+{
+	mpcc->mpcc_id = mpcc_inst;
+	mpcc->dpp_id = 0xf;
+	mpcc->mpcc_bot = NULL;
+	mpcc->blnd_cfg.overlap_only = false;
+	mpcc->blnd_cfg.global_alpha = 0xff;
+	mpcc->blnd_cfg.global_gain = 0xff;
+	mpcc->sm_cfg.enable = false;
 }
 
-void mpc10_update_blend_mode(
-		struct mpc *mpc,
-		struct mpcc_cfg *cfg)
+/*
+ * Reset the MPCC HW status by disconnecting all muxes.
+ *
+ * Parameters:
+ * [in/out] mpc		- MPC context.
+ *
+ * Return:  void
+ */
+void mpc1_mpc_init(struct mpc *mpc)
 {
 	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
-	int mpcc_id, z_idx;
-	int alpha_blnd_mode = cfg->per_pixel_alpha ?
-			BLND_PP_ALPHA : BLND_GLOBAL_ALPHA;
+	int mpcc_id;
+	int opp_id;
+
+	mpc10->mpcc_in_use_mask = 0;
+	for (mpcc_id = 0; mpcc_id < mpc10->num_mpcc; mpcc_id++) {
+		REG_SET(MPCC_TOP_SEL[mpcc_id], 0, MPCC_TOP_SEL, 0xf);
+		REG_SET(MPCC_BOT_SEL[mpcc_id], 0, MPCC_BOT_SEL, 0xf);
+		REG_SET(MPCC_OPP_ID[mpcc_id],  0, MPCC_OPP_ID,  0xf);
 
-	/* find z_idx for the dpp that requires blending mode update*/
-	for (z_idx = 0; z_idx < cfg->tree_cfg->num_pipes; z_idx++)
-		if (cfg->tree_cfg->dpp[z_idx] == cfg->dpp_id)
-			break;
+		mpc1_init_mpcc(&(mpc->mpcc_array[mpcc_id]), mpcc_id);
+	}
 
-	ASSERT(z_idx < cfg->tree_cfg->num_pipes);
-	mpcc_id = cfg->tree_cfg->mpcc[z_idx];
+	for (opp_id = 0; opp_id < MAX_OPP; opp_id++) {
+		if (REG(MUX[opp_id]))
+			REG_UPDATE(MUX[opp_id], MPC_OUT_MUX, 0xf);
+	}
+}
 
-	REG_UPDATE_2(MPCC_CONTROL[mpcc_id],
-			MPCC_ALPHA_BLND_MODE, alpha_blnd_mode,
-			MPCC_ALPHA_MULTIPLIED_MODE, cfg->pre_multiplied_alpha);
+void mpc1_init_mpcc_list_from_hw(
+	struct mpc *mpc,
+	struct mpc_tree *tree)
+{
+	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
+	unsigned int opp_id;
+	unsigned int top_sel;
+	unsigned int bot_sel;
+	unsigned int out_mux;
+	struct mpcc *mpcc;
+	int mpcc_id;
+	int bot_mpcc_id;
+
+	REG_GET(MUX[tree->opp_id], MPC_OUT_MUX, &out_mux);
+
+	if (out_mux != 0xf) {
+		for (mpcc_id = 0; mpcc_id < mpc10->num_mpcc; mpcc_id++) {
+			REG_GET(MPCC_OPP_ID[mpcc_id],  MPCC_OPP_ID,  &opp_id);
+			REG_GET(MPCC_TOP_SEL[mpcc_id], MPCC_TOP_SEL, &top_sel);
+			REG_GET(MPCC_BOT_SEL[mpcc_id],  MPCC_BOT_SEL, &bot_sel);
+
+			if (bot_sel == mpcc_id)
+				bot_sel = 0xf;
+
+			if ((opp_id == tree->opp_id) && (top_sel != 0xf)) {
+				mpcc = mpc1_get_mpcc(mpc, mpcc_id);
+				mpcc->dpp_id = top_sel;
+				mpc10->mpcc_in_use_mask |= 1 << mpcc_id;
+
+				if (out_mux == mpcc_id)
+					tree->opp_list = mpcc;
+				if (bot_sel != 0xf && bot_sel < mpc10->num_mpcc) {
+					bot_mpcc_id = bot_sel;
+					REG_GET(MPCC_OPP_ID[bot_mpcc_id],  MPCC_OPP_ID,  &opp_id);
+					REG_GET(MPCC_TOP_SEL[bot_mpcc_id], MPCC_TOP_SEL, &top_sel);
+					if ((opp_id == tree->opp_id) && (top_sel != 0xf)) {
+						struct mpcc *mpcc_bottom = mpc1_get_mpcc(mpc, bot_mpcc_id);
+
+						mpcc->mpcc_bot = mpcc_bottom;
+					}
+				}
+			}
+		}
+	}
 }
 
 const struct mpc_funcs dcn10_mpc_funcs = {
-		.add = mpc10_mpcc_add,
-		.remove = mpc10_mpcc_remove,
-		.wait_for_idle = mpc10_assert_idle_mpcc,
-		.update_blend_mode = mpc10_update_blend_mode,
+	.insert_plane = mpc1_insert_plane,
+	.remove_mpcc = mpc1_remove_mpcc,
+	.mpc_init = mpc1_mpc_init,
+	.get_mpcc_for_dpp = mpc1_get_mpcc_for_dpp,
+	.wait_for_idle = mpc1_assert_idle_mpcc,
+	.assert_mpcc_idle_before_connect = mpc1_assert_mpcc_idle_before_connect,
+	.init_mpcc_list_from_hw = mpc1_init_mpcc_list_from_hw,
+	.update_blending = mpc1_update_blending,
 };
 
 void dcn10_mpc_construct(struct dcn10_mpc *mpc10,
@@ -349,6 +427,8 @@ void dcn10_mpc_construct(struct dcn10_mpc *mpc10,
 	const struct dcn_mpc_mask *mpc_mask,
 	int num_mpcc)
 {
+	int i;
+
 	mpc10->base.ctx = ctx;
 
 	mpc10->base.funcs = &dcn10_mpc_funcs;
@@ -359,5 +439,8 @@ void dcn10_mpc_construct(struct dcn10_mpc *mpc10,
 
 	mpc10->mpcc_in_use_mask = 0;
 	mpc10->num_mpcc = num_mpcc;
+
+	for (i = 0; i < MAX_MPCC; i++)
+		mpc1_init_mpcc(&mpc10->base.mpcc_array[i], i);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h
index 683ce4aaa76e..267a2995ef6e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_mpc.h
@@ -30,9 +30,6 @@
 #define TO_DCN10_MPC(mpc_base) \
 	container_of(mpc_base, struct dcn10_mpc, base)
 
-#define MAX_MPCC 6
-#define MAX_OPP 6
-
 #define MPC_COMMON_REG_LIST_DCN1_0(inst) \
 	SRII(MPCC_TOP_SEL, MPCC, inst),\
 	SRII(MPCC_BOT_SEL, MPCC, inst),\
@@ -42,7 +39,8 @@
 	SRII(MPCC_BG_G_Y, MPCC, inst),\
 	SRII(MPCC_BG_R_CR, MPCC, inst),\
 	SRII(MPCC_BG_B_CB, MPCC, inst),\
-	SRII(MPCC_BG_B_CB, MPCC, inst)
+	SRII(MPCC_BG_B_CB, MPCC, inst),\
+	SRII(MPCC_SM_CONTROL, MPCC, inst)
 
 #define MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0(inst) \
 	SRII(MUX, MPC_OUT, inst)
@@ -56,6 +54,7 @@
 	uint32_t MPCC_BG_G_Y[MAX_MPCC]; \
 	uint32_t MPCC_BG_R_CR[MAX_MPCC]; \
 	uint32_t MPCC_BG_B_CB[MAX_MPCC]; \
+	uint32_t MPCC_SM_CONTROL[MAX_MPCC]; \
 	uint32_t MUX[MAX_OPP];
 
 #define MPC_COMMON_MASK_SH_LIST_DCN1_0(mask_sh)\
@@ -65,12 +64,20 @@
 	SF(MPCC0_MPCC_CONTROL, MPCC_ALPHA_BLND_MODE, mask_sh),\
 	SF(MPCC0_MPCC_CONTROL, MPCC_ALPHA_MULTIPLIED_MODE, mask_sh),\
 	SF(MPCC0_MPCC_CONTROL, MPCC_BLND_ACTIVE_OVERLAP_ONLY, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_GLOBAL_ALPHA, mask_sh),\
+	SF(MPCC0_MPCC_CONTROL, MPCC_GLOBAL_GAIN, mask_sh),\
 	SF(MPCC0_MPCC_STATUS, MPCC_IDLE, mask_sh),\
 	SF(MPCC0_MPCC_STATUS, MPCC_BUSY, mask_sh),\
 	SF(MPCC0_MPCC_OPP_ID, MPCC_OPP_ID, mask_sh),\
 	SF(MPCC0_MPCC_BG_G_Y, MPCC_BG_G_Y, mask_sh),\
 	SF(MPCC0_MPCC_BG_R_CR, MPCC_BG_R_CR, mask_sh),\
 	SF(MPCC0_MPCC_BG_B_CB, MPCC_BG_B_CB, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_EN, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_MODE, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FRAME_ALT, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FIELD_ALT, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FORCE_NEXT_FRAME_POL, mask_sh),\
+	SF(MPCC0_MPCC_SM_CONTROL, MPCC_SM_FORCE_NEXT_TOP_POL, mask_sh),\
 	SF(MPC_OUT0_MUX, MPC_OUT_MUX, mask_sh)
 
 #define MPC_REG_FIELD_LIST(type) \
@@ -80,12 +87,20 @@
 	type MPCC_ALPHA_BLND_MODE;\
 	type MPCC_ALPHA_MULTIPLIED_MODE;\
 	type MPCC_BLND_ACTIVE_OVERLAP_ONLY;\
+	type MPCC_GLOBAL_ALPHA;\
+	type MPCC_GLOBAL_GAIN;\
 	type MPCC_IDLE;\
 	type MPCC_BUSY;\
 	type MPCC_OPP_ID;\
 	type MPCC_BG_G_Y;\
 	type MPCC_BG_R_CR;\
 	type MPCC_BG_B_CB;\
+	type MPCC_SM_EN;\
+	type MPCC_SM_MODE;\
+	type MPCC_SM_FRAME_ALT;\
+	type MPCC_SM_FIELD_ALT;\
+	type MPCC_SM_FORCE_NEXT_FRAME_POL;\
+	type MPCC_SM_FORCE_NEXT_TOP_POL;\
 	type MPC_OUT_MUX;
 
 struct dcn_mpc_registers {
@@ -117,22 +132,55 @@ void dcn10_mpc_construct(struct dcn10_mpc *mpcc10,
 	const struct dcn_mpc_mask *mpc_mask,
 	int num_mpcc);
 
-int mpc10_mpcc_add(
-		struct mpc *mpc,
-		struct mpcc_cfg *cfg);
-
-void mpc10_mpcc_remove(
-		struct mpc *mpc,
-		struct mpc_tree_cfg *tree_cfg,
-		int opp_id,
-		int dpp_id);
-
-void mpc10_assert_idle_mpcc(
-		struct mpc *mpc,
-		int id);
-
-void mpc10_update_blend_mode(
-		struct mpc *mpc,
-		struct mpcc_cfg *cfg);
+struct mpcc *mpc1_insert_plane(
+	struct mpc *mpc,
+	struct mpc_tree *tree,
+	struct mpcc_blnd_cfg *blnd_cfg,
+	struct mpcc_sm_cfg *sm_cfg,
+	struct mpcc *insert_above_mpcc,
+	int dpp_id,
+	int mpcc_id);
+
+void mpc1_remove_mpcc(
+	struct mpc *mpc,
+	struct mpc_tree *tree,
+	struct mpcc *mpcc);
+
+void mpc1_mpc_init(
+	struct mpc *mpc);
+
+void mpc1_assert_idle_mpcc(
+	struct mpc *mpc,
+	int id);
+
+void mpc1_set_bg_color(
+	struct mpc *mpc,
+	struct tg_color *bg_color,
+	int id);
+
+void mpc1_update_stereo_mix(
+	struct mpc *mpc,
+	struct mpcc_sm_cfg *sm_cfg,
+	int mpcc_id);
+
+bool mpc1_is_mpcc_idle(
+	struct mpc *mpc,
+	int mpcc_id);
+
+void mpc1_assert_mpcc_idle_before_connect(
+	struct mpc *mpc,
+	int mpcc_id);
+
+void mpc1_init_mpcc_list_from_hw(
+	struct mpc *mpc,
+	struct mpc_tree *tree);
+
+struct mpcc *mpc1_get_mpcc(
+	struct mpc *mpc,
+	int mpcc_id);
+
+struct mpcc *mpc1_get_mpcc_for_dpp(
+	struct mpc_tree *tree,
+	int dpp_id);
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
index a136f70b7a3c..f6ba0eef4489 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.c
@@ -38,7 +38,6 @@
 	oppn10->base.ctx
 
 
-
 /************* FORMATTER ************/
 
 /**
@@ -47,7 +46,7 @@
  *	2) enable truncation
  *	3) HW remove 12bit FMT support for DCE11 power saving reason.
  */
-static void set_truncation(
+static void opp1_set_truncation(
 		struct dcn10_opp *oppn10,
 		const struct bit_depth_reduction_params *params)
 {
@@ -57,7 +56,7 @@ static void set_truncation(
 		FMT_TRUNCATE_MODE, params->flags.TRUNCATE_MODE);
 }
 
-static void set_spatial_dither(
+static void opp1_set_spatial_dither(
 	struct dcn10_opp *oppn10,
 	const struct bit_depth_reduction_params *params)
 {
@@ -136,14 +135,14 @@ static void set_spatial_dither(
 			FMT_RGB_RANDOM_ENABLE, params->flags.RGB_RANDOM);
 }
 
-static void oppn10_program_bit_depth_reduction(
+void opp1_program_bit_depth_reduction(
 	struct output_pixel_processor *opp,
 	const struct bit_depth_reduction_params *params)
 {
 	struct dcn10_opp *oppn10 = TO_DCN10_OPP(opp);
 
-	set_truncation(oppn10, params);
-	set_spatial_dither(oppn10, params);
+	opp1_set_truncation(oppn10, params);
+	opp1_set_spatial_dither(oppn10, params);
 	/* TODO
 	 * set_temporal_dither(oppn10, params);
 	 */
@@ -156,7 +155,7 @@ static void oppn10_program_bit_depth_reduction(
  *		0: RGB 4:4:4 or YCbCr 4:4:4 or YOnly
  *		1: YCbCr 4:2:2
  */
-static void set_pixel_encoding(
+static void opp1_set_pixel_encoding(
 	struct dcn10_opp *oppn10,
 	const struct clamping_and_pixel_encoding_params *params)
 {
@@ -186,7 +185,7 @@ static void set_pixel_encoding(
  *		7 for programable
  *	2) Enable clamp if Limited range requested
  */
-static void opp_set_clamping(
+static void opp1_set_clamping(
 	struct dcn10_opp *oppn10,
 	const struct clamping_and_pixel_encoding_params *params)
 {
@@ -224,7 +223,7 @@ static void opp_set_clamping(
 
 }
 
-static void oppn10_set_dyn_expansion(
+void opp1_set_dyn_expansion(
 	struct output_pixel_processor *opp,
 	enum dc_color_space color_sp,
 	enum dc_color_depth color_dpth,
@@ -264,17 +263,17 @@ static void oppn10_set_dyn_expansion(
 	}
 }
 
-static void opp_program_clamping_and_pixel_encoding(
+static void opp1_program_clamping_and_pixel_encoding(
 	struct output_pixel_processor *opp,
 	const struct clamping_and_pixel_encoding_params *params)
 {
 	struct dcn10_opp *oppn10 = TO_DCN10_OPP(opp);
 
-	opp_set_clamping(oppn10, params);
-	set_pixel_encoding(oppn10, params);
+	opp1_set_clamping(oppn10, params);
+	opp1_set_pixel_encoding(oppn10, params);
 }
 
-static void oppn10_program_fmt(
+void opp1_program_fmt(
 	struct output_pixel_processor *opp,
 	struct bit_depth_reduction_params *fmt_bit_depth,
 	struct clamping_and_pixel_encoding_params *clamping)
@@ -286,44 +285,104 @@ static void oppn10_program_fmt(
 
 	/* dithering is affected by <CrtcSourceSelect>, hence should be
 	 * programmed afterwards */
-	oppn10_program_bit_depth_reduction(
+	opp1_program_bit_depth_reduction(
 		opp,
 		fmt_bit_depth);
 
-	opp_program_clamping_and_pixel_encoding(
+	opp1_program_clamping_and_pixel_encoding(
 		opp,
 		clamping);
 
 	return;
 }
 
+void opp1_program_stereo(
+	struct output_pixel_processor *opp,
+	bool enable,
+	const struct dc_crtc_timing *timing)
+{
+	struct dcn10_opp *oppn10 = TO_DCN10_OPP(opp);
+
+	uint32_t active_width = timing->h_addressable - timing->h_border_right - timing->h_border_right;
+	uint32_t space1_size = timing->v_total - timing->v_addressable;
+	/* TODO: confirm computation of space2_size */
+	uint32_t space2_size = timing->v_total - timing->v_addressable;
 
+	if (!enable) {
+		active_width = 0;
+		space1_size = 0;
+		space2_size = 0;
+	}
+
+	/* TODO: for which cases should FMT_STEREOSYNC_OVERRIDE be set? */
+	REG_UPDATE(FMT_CONTROL, FMT_STEREOSYNC_OVERRIDE, 0);
+
+	REG_UPDATE(OPPBUF_CONTROL, OPPBUF_ACTIVE_WIDTH, active_width);
+
+	/* Program OPPBUF_3D_VACT_SPACE1_SIZE and OPPBUF_VACT_SPACE2_SIZE registers
+	 * In 3D progressive frames, Vactive space happens only in between the 2 frames,
+	 * so only need to program OPPBUF_3D_VACT_SPACE1_SIZE
+	 * In 3D alternative frames, left and right frames, top and bottom field.
+	 */
+	if (timing->timing_3d_format == TIMING_3D_FORMAT_FRAME_ALTERNATE)
+		REG_UPDATE(OPPBUF_3D_PARAMETERS_0, OPPBUF_3D_VACT_SPACE2_SIZE, space2_size);
+	else
+		REG_UPDATE(OPPBUF_3D_PARAMETERS_0, OPPBUF_3D_VACT_SPACE1_SIZE, space1_size);
+
+	/* TODO: Is programming of OPPBUF_DUMMY_DATA_R/G/B needed? */
+	/*
+	REG_UPDATE(OPPBUF_3D_PARAMETERS_0,
+			OPPBUF_DUMMY_DATA_R, data_r);
+	REG_UPDATE(OPPBUF_3D_PARAMETERS_1,
+			OPPBUF_DUMMY_DATA_G, data_g);
+	REG_UPDATE(OPPBUF_3D_PARAMETERS_1,
+			OPPBUF_DUMMY_DATA_B, _data_b);
+	*/
+}
 
-static void oppn10_set_stereo_polarity(
-		struct output_pixel_processor *opp,
-		bool enable, bool rightEyePolarity)
+void opp1_program_oppbuf(
+	struct output_pixel_processor *opp,
+	struct oppbuf_params *oppbuf)
 {
 	struct dcn10_opp *oppn10 = TO_DCN10_OPP(opp);
 
-	REG_UPDATE(FMT_CONTROL, FMT_STEREOSYNC_OVERRIDE, enable);
+	/* Program the oppbuf active width to be the frame width from mpc */
+	REG_UPDATE(OPPBUF_CONTROL, OPPBUF_ACTIVE_WIDTH, oppbuf->active_width);
+
+	/* Specifies the number of segments in multi-segment mode (DP-MSO operation)
+	 * description  "In 1/2/4 segment mode, specifies the horizontal active width in pixels of the display panel.
+	 * In 4 segment split left/right mode, specifies the horizontal 1/2 active width in pixels of the display panel.
+	 * Used to determine segment boundaries in multi-segment mode. Used to determine the width of the vertical active space in 3D frame packed modes.
+	 * OPPBUF_ACTIVE_WIDTH must be integer divisible by the total number of segments."
+	 */
+	REG_UPDATE(OPPBUF_CONTROL, OPPBUF_DISPLAY_SEGMENTATION, oppbuf->mso_segmentation);
+
+	/* description  "Specifies the number of overlap pixels (1-8 overlapping pixels supported), used in multi-segment mode (DP-MSO operation)" */
+	REG_UPDATE(OPPBUF_CONTROL, OPPBUF_OVERLAP_PIXEL_NUM, oppbuf->mso_overlap_pixel_num);
+
+	/* description  "Specifies the number of times a pixel is replicated (0-15 pixel replications supported).
+	 * A value of 0 disables replication. The total number of times a pixel is output is OPPBUF_PIXEL_REPETITION + 1."
+	 */
+	REG_UPDATE(OPPBUF_CONTROL, OPPBUF_PIXEL_REPETITION, oppbuf->pixel_repetition);
+
 }
 
 /*****************************************/
 /* Constructor, Destructor               */
 /*****************************************/
 
-static void dcn10_opp_destroy(struct output_pixel_processor **opp)
+void opp1_destroy(struct output_pixel_processor **opp)
 {
 	kfree(TO_DCN10_OPP(*opp));
 	*opp = NULL;
 }
 
 static struct opp_funcs dcn10_opp_funcs = {
-		.opp_set_dyn_expansion = oppn10_set_dyn_expansion,
-		.opp_program_fmt = oppn10_program_fmt,
-		.opp_program_bit_depth_reduction = oppn10_program_bit_depth_reduction,
-		.opp_set_stereo_polarity = oppn10_set_stereo_polarity,
-		.opp_destroy = dcn10_opp_destroy
+		.opp_set_dyn_expansion = opp1_set_dyn_expansion,
+		.opp_program_fmt = opp1_program_fmt,
+		.opp_program_bit_depth_reduction = opp1_program_bit_depth_reduction,
+		.opp_program_stereo = opp1_program_stereo,
+		.opp_destroy = opp1_destroy
 };
 
 void dcn10_opp_construct(struct dcn10_opp *oppn10,
@@ -333,19 +392,12 @@ void dcn10_opp_construct(struct dcn10_opp *oppn10,
 	const struct dcn10_opp_shift *opp_shift,
 	const struct dcn10_opp_mask *opp_mask)
 {
-	int i;
+
 	oppn10->base.ctx = ctx;
 	oppn10->base.inst = inst;
 	oppn10->base.funcs = &dcn10_opp_funcs;
 
-	oppn10->base.mpc_tree.dpp[0] = inst;
-	oppn10->base.mpc_tree.mpcc[0] = inst;
-	oppn10->base.mpc_tree.num_pipes = 1;
-	for (i = 0; i < MAX_PIPES; i++)
-		oppn10->base.mpcc_disconnect_pending[i] = false;
-
 	oppn10->regs = regs;
 	oppn10->opp_shift = opp_shift;
 	oppn10->opp_mask = opp_mask;
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h
index 790ce6014832..bc5058af6266 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_opp.h
@@ -41,11 +41,28 @@
 	SRI(FMT_DITHER_RAND_B_SEED, FMT, id), \
 	SRI(FMT_CLAMP_CNTL, FMT, id), \
 	SRI(FMT_DYNAMIC_EXP_CNTL, FMT, id), \
-	SRI(FMT_MAP420_MEMORY_CONTROL, FMT, id)
+	SRI(FMT_MAP420_MEMORY_CONTROL, FMT, id), \
+	SRI(OPPBUF_CONTROL, OPPBUF, id),\
+	SRI(OPPBUF_3D_PARAMETERS_0, OPPBUF, id), \
+	SRI(OPPBUF_3D_PARAMETERS_1, OPPBUF, id)
 
 #define OPP_REG_LIST_DCN10(id) \
 	OPP_REG_LIST_DCN(id)
 
+#define OPP_COMMON_REG_VARIABLE_LIST \
+	uint32_t FMT_BIT_DEPTH_CONTROL; \
+	uint32_t FMT_CONTROL; \
+	uint32_t FMT_DITHER_RAND_R_SEED; \
+	uint32_t FMT_DITHER_RAND_G_SEED; \
+	uint32_t FMT_DITHER_RAND_B_SEED; \
+	uint32_t FMT_CLAMP_CNTL; \
+	uint32_t FMT_DYNAMIC_EXP_CNTL; \
+	uint32_t FMT_MAP420_MEMORY_CONTROL; \
+	uint32_t OPPBUF_CONTROL; \
+	uint32_t OPPBUF_CONTROL1; \
+	uint32_t OPPBUF_3D_PARAMETERS_0; \
+	uint32_t OPPBUF_3D_PARAMETERS_1
+
 #define OPP_MASK_SH_LIST_DCN(mask_sh) \
 	OPP_SF(FMT0_FMT_BIT_DEPTH_CONTROL, FMT_TRUNCATE_EN, mask_sh), \
 	OPP_SF(FMT0_FMT_BIT_DEPTH_CONTROL, FMT_TRUNCATE_DEPTH, mask_sh), \
@@ -68,46 +85,18 @@
 	OPP_SF(FMT0_FMT_CLAMP_CNTL, FMT_CLAMP_COLOR_FORMAT, mask_sh), \
 	OPP_SF(FMT0_FMT_DYNAMIC_EXP_CNTL, FMT_DYNAMIC_EXP_EN, mask_sh), \
 	OPP_SF(FMT0_FMT_DYNAMIC_EXP_CNTL, FMT_DYNAMIC_EXP_MODE, mask_sh), \
-	OPP_SF(FMT0_FMT_MAP420_MEMORY_CONTROL, FMT_MAP420MEM_PWR_FORCE, mask_sh)
+	OPP_SF(FMT0_FMT_MAP420_MEMORY_CONTROL, FMT_MAP420MEM_PWR_FORCE, mask_sh), \
+	OPP_SF(OPPBUF0_OPPBUF_CONTROL, OPPBUF_ACTIVE_WIDTH, mask_sh),\
+	OPP_SF(OPPBUF0_OPPBUF_CONTROL, OPPBUF_PIXEL_REPETITION, mask_sh),\
+	OPP_SF(OPPBUF0_OPPBUF_3D_PARAMETERS_0, OPPBUF_3D_VACT_SPACE1_SIZE, mask_sh), \
+	OPP_SF(OPPBUF0_OPPBUF_3D_PARAMETERS_0, OPPBUF_3D_VACT_SPACE2_SIZE, mask_sh)
 
 #define OPP_MASK_SH_LIST_DCN10(mask_sh) \
-	OPP_MASK_SH_LIST_DCN(mask_sh)
+	OPP_MASK_SH_LIST_DCN(mask_sh), \
+	OPP_SF(OPPBUF0_OPPBUF_CONTROL, OPPBUF_DISPLAY_SEGMENTATION, mask_sh),\
+	OPP_SF(OPPBUF0_OPPBUF_CONTROL, OPPBUF_OVERLAP_PIXEL_NUM, mask_sh)
 
 #define OPP_DCN10_REG_FIELD_LIST(type) \
-	type DPG_EN; \
-	type DPG_MODE; \
-	type DPG_VRES; \
-	type DPG_HRES; \
-	type DPG_COLOUR0_R_CR; \
-	type DPG_COLOUR1_R_CR; \
-	type DPG_COLOUR0_B_CB; \
-	type DPG_COLOUR1_B_CB; \
-	type DPG_COLOUR0_G_Y; \
-	type DPG_COLOUR1_G_Y; \
-	type CM_OCSC_C11; \
-	type CM_OCSC_C12; \
-	type CM_OCSC_C13; \
-	type CM_OCSC_C14; \
-	type CM_OCSC_C21; \
-	type CM_OCSC_C22; \
-	type CM_OCSC_C23; \
-	type CM_OCSC_C24; \
-	type CM_OCSC_C31; \
-	type CM_OCSC_C32; \
-	type CM_OCSC_C33; \
-	type CM_OCSC_C34; \
-	type CM_COMB_C11; \
-	type CM_COMB_C12; \
-	type CM_COMB_C13; \
-	type CM_COMB_C14; \
-	type CM_COMB_C21; \
-	type CM_COMB_C22; \
-	type CM_COMB_C23; \
-	type CM_COMB_C24; \
-	type CM_COMB_C31; \
-	type CM_COMB_C32; \
-	type CM_COMB_C33; \
-	type CM_COMB_C34; \
 	type FMT_TRUNCATE_EN; \
 	type FMT_TRUNCATE_DEPTH; \
 	type FMT_TRUNCATE_MODE; \
@@ -129,7 +118,18 @@
 	type FMT_DYNAMIC_EXP_EN; \
 	type FMT_DYNAMIC_EXP_MODE; \
 	type FMT_MAP420MEM_PWR_FORCE; \
-	type FMT_STEREOSYNC_OVERRIDE
+	type FMT_STEREOSYNC_OVERRIDE; \
+	type OPPBUF_ACTIVE_WIDTH;\
+	type OPPBUF_PIXEL_REPETITION;\
+	type OPPBUF_DISPLAY_SEGMENTATION;\
+	type OPPBUF_OVERLAP_PIXEL_NUM;\
+	type OPPBUF_NUM_SEGMENT_PADDED_PIXELS; \
+	type OPPBUF_3D_VACT_SPACE1_SIZE; \
+	type OPPBUF_3D_VACT_SPACE2_SIZE
+
+struct dcn10_opp_registers {
+	OPP_COMMON_REG_VARIABLE_LIST;
+};
 
 struct dcn10_opp_shift {
 	OPP_DCN10_REG_FIELD_LIST(uint8_t);
@@ -139,33 +139,6 @@ struct dcn10_opp_mask {
 	OPP_DCN10_REG_FIELD_LIST(uint32_t);
 };
 
-struct dcn10_opp_registers {
-	uint32_t DPG_CONTROL;
-	uint32_t DPG_COLOUR_B_CB;
-	uint32_t DPG_COLOUR_G_Y;
-	uint32_t DPG_COLOUR_R_CR;
-	uint32_t CM_OCSC_C11_C12;
-	uint32_t CM_OCSC_C13_C14;
-	uint32_t CM_OCSC_C21_C22;
-	uint32_t CM_OCSC_C23_C24;
-	uint32_t CM_OCSC_C31_C32;
-	uint32_t CM_OCSC_C33_C34;
-	uint32_t CM_COMB_C11_C12;
-	uint32_t CM_COMB_C13_C14;
-	uint32_t CM_COMB_C21_C22;
-	uint32_t CM_COMB_C23_C24;
-	uint32_t CM_COMB_C31_C32;
-	uint32_t CM_COMB_C33_C34;
-	uint32_t FMT_BIT_DEPTH_CONTROL;
-	uint32_t FMT_CONTROL;
-	uint32_t FMT_DITHER_RAND_R_SEED;
-	uint32_t FMT_DITHER_RAND_G_SEED;
-	uint32_t FMT_DITHER_RAND_B_SEED;
-	uint32_t FMT_CLAMP_CNTL;
-	uint32_t FMT_DYNAMIC_EXP_CNTL;
-	uint32_t FMT_MAP420_MEMORY_CONTROL;
-};
-
 struct dcn10_opp {
 	struct output_pixel_processor base;
 
@@ -183,4 +156,26 @@ void dcn10_opp_construct(struct dcn10_opp *oppn10,
 	const struct dcn10_opp_shift *opp_shift,
 	const struct dcn10_opp_mask *opp_mask);
 
+void opp1_set_dyn_expansion(
+	struct output_pixel_processor *opp,
+	enum dc_color_space color_sp,
+	enum dc_color_depth color_dpth,
+	enum signal_type signal);
+
+void opp1_program_fmt(
+	struct output_pixel_processor *opp,
+	struct bit_depth_reduction_params *fmt_bit_depth,
+	struct clamping_and_pixel_encoding_params *clamping);
+
+void opp1_program_bit_depth_reduction(
+	struct output_pixel_processor *opp,
+	const struct bit_depth_reduction_params *params);
+
+void opp1_program_stereo(
+	struct output_pixel_processor *opp,
+	bool enable,
+	const struct dc_crtc_timing *timing);
+
+void opp1_destroy(struct output_pixel_processor **opp);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
similarity index 72%
rename from drivers/gpu/drm/amd/display/dc/dcn10/dcn10_timing_generator.c
rename to drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
index fced178c8c79..4bf64d1b2c60 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
@@ -23,19 +23,20 @@
  *
  */
 
+
 #include "reg_helper.h"
-#include "dcn10_timing_generator.h"
+#include "dcn10_optc.h"
 #include "dc.h"
 
 #define REG(reg)\
-	tgn10->tg_regs->reg
+	optc1->tg_regs->reg
 
 #define CTX \
-	tgn10->base.ctx
+	optc1->base.ctx
 
 #undef FN
 #define FN(reg_name, field_name) \
-	tgn10->tg_shift->field_name, tgn10->tg_mask->field_name
+	optc1->tg_shift->field_name, optc1->tg_mask->field_name
 
 #define STATIC_SCREEN_EVENT_MASK_RANGETIMING_DOUBLE_BUFFER_UPDATE_EN 0x100
 
@@ -45,8 +46,8 @@
 * This is a workaround for a bug that has existed since R5xx and has not been
 * fixed keep Front porch at minimum 2 for Interlaced mode or 1 for progressive.
 */
-static void tgn10_apply_front_porch_workaround(
-	struct timing_generator *tg,
+static void optc1_apply_front_porch_workaround(
+	struct timing_generator *optc,
 	struct dc_crtc_timing *timing)
 {
 	if (timing->flags.INTERLACE == 1) {
@@ -58,30 +59,30 @@ static void tgn10_apply_front_porch_workaround(
 	}
 }
 
-static void tgn10_program_global_sync(
-		struct timing_generator *tg)
+void optc1_program_global_sync(
+		struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
-	if (tg->dlg_otg_param.vstartup_start == 0) {
+	if (optc->dlg_otg_param.vstartup_start == 0) {
 		BREAK_TO_DEBUGGER();
 		return;
 	}
 
 	REG_SET(OTG_VSTARTUP_PARAM, 0,
-		VSTARTUP_START, tg->dlg_otg_param.vstartup_start);
+		VSTARTUP_START, optc->dlg_otg_param.vstartup_start);
 
 	REG_SET_2(OTG_VUPDATE_PARAM, 0,
-			VUPDATE_OFFSET, tg->dlg_otg_param.vupdate_offset,
-			VUPDATE_WIDTH, tg->dlg_otg_param.vupdate_width);
+			VUPDATE_OFFSET, optc->dlg_otg_param.vupdate_offset,
+			VUPDATE_WIDTH, optc->dlg_otg_param.vupdate_width);
 
 	REG_SET(OTG_VREADY_PARAM, 0,
-			VREADY_OFFSET, tg->dlg_otg_param.vready_offset);
+			VREADY_OFFSET, optc->dlg_otg_param.vready_offset);
 }
 
-static void tgn10_disable_stereo(struct timing_generator *tg)
+static void optc1_disable_stereo(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_SET(OTG_STEREO_CONTROL, 0,
 		OTG_STEREO_EN, 0);
@@ -90,11 +91,6 @@ static void tgn10_disable_stereo(struct timing_generator *tg)
 		OTG_3D_STRUCTURE_EN, 0,
 		OTG_3D_STRUCTURE_V_UPDATE_MODE, 0,
 		OTG_3D_STRUCTURE_STEREO_SEL_OVR, 0);
-
-	REG_UPDATE(OPPBUF_CONTROL,
-		OPPBUF_ACTIVE_WIDTH, 0);
-	REG_UPDATE(OPPBUF_3D_PARAMETERS_0,
-		OPPBUF_3D_VACT_SPACE1_SIZE, 0);
 }
 
 /**
@@ -102,8 +98,8 @@ static void tgn10_disable_stereo(struct timing_generator *tg)
  * Program CRTC Timing Registers - OTG_H_*, OTG_V_*, Pixel repetition.
  * Including SYNC. Call BIOS command table to program Timings.
  */
-static void tgn10_program_timing(
-	struct timing_generator *tg,
+void optc1_program_timing(
+	struct timing_generator *optc,
 	const struct dc_crtc_timing *dc_crtc_timing,
 	bool use_vbios)
 {
@@ -121,10 +117,10 @@ static void tgn10_program_timing(
 	uint32_t h_div_2;
 	int32_t vertical_line_start;
 
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	patched_crtc_timing = *dc_crtc_timing;
-	tgn10_apply_front_porch_workaround(tg, &patched_crtc_timing);
+	optc1_apply_front_porch_workaround(optc, &patched_crtc_timing);
 
 	/* Load horizontal timing */
 
@@ -217,7 +213,7 @@ static void tgn10_program_timing(
 	/* Use OTG_VERTICAL_INTERRUPT2 replace VUPDATE interrupt,
 	 * program the reg for interrupt postition.
 	 */
-	vertical_line_start = asic_blank_end - tg->dlg_otg_param.vstartup_start + 1;
+	vertical_line_start = asic_blank_end - optc->dlg_otg_param.vstartup_start + 1;
 	if (vertical_line_start < 0) {
 		ASSERT(0);
 		vertical_line_start = 0;
@@ -233,26 +229,25 @@ static void tgn10_program_timing(
 			OTG_V_SYNC_A_POL, v_sync_polarity);
 
 	v_init = asic_blank_start;
-	if (tg->dlg_otg_param.signal == SIGNAL_TYPE_DISPLAY_PORT ||
-		tg->dlg_otg_param.signal == SIGNAL_TYPE_DISPLAY_PORT_MST ||
-		tg->dlg_otg_param.signal == SIGNAL_TYPE_EDP) {
+	if (optc->dlg_otg_param.signal == SIGNAL_TYPE_DISPLAY_PORT ||
+		optc->dlg_otg_param.signal == SIGNAL_TYPE_DISPLAY_PORT_MST ||
+		optc->dlg_otg_param.signal == SIGNAL_TYPE_EDP) {
 		start_point = 1;
 		if (patched_crtc_timing.flags.INTERLACE == 1)
 			field_num = 1;
 	}
 	v_fp2 = 0;
-	if (tg->dlg_otg_param.vstartup_start > asic_blank_end)
-		v_fp2 = tg->dlg_otg_param.vstartup_start > asic_blank_end;
+	if (optc->dlg_otg_param.vstartup_start > asic_blank_end)
+		v_fp2 = optc->dlg_otg_param.vstartup_start > asic_blank_end;
 
 	/* Interlace */
 	if (patched_crtc_timing.flags.INTERLACE == 1) {
 		REG_UPDATE(OTG_INTERLACE_CONTROL,
 				OTG_INTERLACE_ENABLE, 1);
 		v_init = v_init / 2;
-		if ((tg->dlg_otg_param.vstartup_start/2)*2 > asic_blank_end)
+		if ((optc->dlg_otg_param.vstartup_start/2)*2 > asic_blank_end)
 			v_fp2 = v_fp2 / 2;
-	}
-	else
+	} else
 		REG_UPDATE(OTG_INTERLACE_CONTROL,
 				OTG_INTERLACE_ENABLE, 0);
 
@@ -270,13 +265,13 @@ static void tgn10_program_timing(
 			OTG_START_POINT_CNTL, start_point,
 			OTG_FIELD_NUMBER_CNTL, field_num);
 
-	tgn10_program_global_sync(tg);
+	optc1_program_global_sync(optc);
 
 	/* TODO
 	 * patched_crtc_timing.flags.HORZ_COUNT_BY_TWO == 1
 	 * program_horz_count_by_2
 	 * for DVI 30bpp mode, 0 otherwise
-	 * program_horz_count_by_2(tg, &patched_crtc_timing);
+	 * program_horz_count_by_2(optc, &patched_crtc_timing);
 	 */
 
 	/* Enable stereo - only when we need to pack 3D frame. Other types
@@ -290,13 +285,23 @@ static void tgn10_program_timing(
 
 }
 
+static void optc1_set_blank_data_double_buffer(struct timing_generator *optc, bool enable)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	uint32_t blank_data_double_buffer_enable = enable ? 1 : 0;
+
+	REG_UPDATE(OTG_DOUBLE_BUFFER_CONTROL,
+			OTG_BLANK_DATA_DOUBLE_BUFFER_EN, blank_data_double_buffer_enable);
+}
+
 /**
  * unblank_crtc
  * Call ASIC Control Object to UnBlank CRTC.
  */
-static void tgn10_unblank_crtc(struct timing_generator *tg)
+static void optc1_unblank_crtc(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	uint32_t vertical_interrupt_enable = 0;
 
 	REG_GET(OTG_VERTICAL_INTERRUPT2_CONTROL,
@@ -306,8 +311,7 @@ static void tgn10_unblank_crtc(struct timing_generator *tg)
 	 * this check will be removed.
 	 */
 	if (vertical_interrupt_enable)
-		REG_UPDATE(OTG_DOUBLE_BUFFER_CONTROL,
-				OTG_BLANK_DATA_DOUBLE_BUFFER_EN, 1);
+		optc1_set_blank_data_double_buffer(optc, true);
 
 	REG_UPDATE_2(OTG_BLANK_CONTROL,
 			OTG_BLANK_DATA_EN, 0,
@@ -319,37 +323,29 @@ static void tgn10_unblank_crtc(struct timing_generator *tg)
  * Call ASIC Control Object to Blank CRTC.
  */
 
-static void tgn10_blank_crtc(struct timing_generator *tg)
+static void optc1_blank_crtc(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_UPDATE_2(OTG_BLANK_CONTROL,
 			OTG_BLANK_DATA_EN, 1,
 			OTG_BLANK_DE_MODE, 0);
 
-	/* todo: why are we waiting for BLANK_DATA_EN?  shouldn't we be waiting
-	 * for status?
-	 */
-	REG_WAIT(OTG_BLANK_CONTROL,
-			OTG_BLANK_DATA_EN, 1,
-			1, 100000);
-
-	REG_UPDATE(OTG_DOUBLE_BUFFER_CONTROL,
-			OTG_BLANK_DATA_DOUBLE_BUFFER_EN, 0);
+	optc1_set_blank_data_double_buffer(optc, false);
 }
 
-static void tgn10_set_blank(struct timing_generator *tg,
+void optc1_set_blank(struct timing_generator *optc,
 		bool enable_blanking)
 {
 	if (enable_blanking)
-		tgn10_blank_crtc(tg);
+		optc1_blank_crtc(optc);
 	else
-		tgn10_unblank_crtc(tg);
+		optc1_unblank_crtc(optc);
 }
 
-static bool tgn10_is_blanked(struct timing_generator *tg)
+bool optc1_is_blanked(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	uint32_t blank_en;
 	uint32_t blank_state;
 
@@ -360,9 +356,9 @@ static bool tgn10_is_blanked(struct timing_generator *tg)
 	return blank_en && blank_state;
 }
 
-static void tgn10_enable_optc_clock(struct timing_generator *tg, bool enable)
+void optc1_enable_optc_clock(struct timing_generator *optc, bool enable)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	if (enable) {
 		REG_UPDATE_2(OPTC_INPUT_CLOCK_CONTROL,
@@ -385,19 +381,9 @@ static void tgn10_enable_optc_clock(struct timing_generator *tg, bool enable)
 				OTG_CLOCK_GATE_DIS, 0,
 				OTG_CLOCK_EN, 0);
 
-		if (tg->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
-			REG_WAIT(OTG_CLOCK_CONTROL,
-					OTG_CLOCK_ON, 0,
-					1, 1000);
-
 		REG_UPDATE_2(OPTC_INPUT_CLOCK_CONTROL,
 				OPTC_INPUT_CLK_GATE_DIS, 0,
 				OPTC_INPUT_CLK_EN, 0);
-
-		if (tg->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
-			REG_WAIT(OPTC_INPUT_CLOCK_CONTROL,
-					OPTC_INPUT_CLK_ON, 0,
-					1, 1000);
 	}
 }
 
@@ -405,19 +391,19 @@ static void tgn10_enable_optc_clock(struct timing_generator *tg, bool enable)
  * Enable CRTC
  * Enable CRTC - call ASIC Control Object to enable Timing generator.
  */
-static bool tgn10_enable_crtc(struct timing_generator *tg)
+static bool optc1_enable_crtc(struct timing_generator *optc)
 {
 	/* TODO FPGA wait for answer
 	 * OTG_MASTER_UPDATE_MODE != CRTC_MASTER_UPDATE_MODE
 	 * OTG_MASTER_UPDATE_LOCK != CRTC_MASTER_UPDATE_LOCK
 	 */
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	/* opp instance for OTG. For DCN1.0, ODM is remoed.
 	 * OPP and OPTC should 1:1 mapping
 	 */
 	REG_UPDATE(OPTC_DATA_SOURCE_SELECT,
-			OPTC_SRC_SEL, tg->inst);
+			OPTC_SRC_SEL, optc->inst);
 
 	/* VTG enable first is for HW workaround */
 	REG_UPDATE(CONTROL,
@@ -432,9 +418,9 @@ static bool tgn10_enable_crtc(struct timing_generator *tg)
 }
 
 /* disable_crtc - call ASIC Control Object to disable Timing generator. */
-static bool tgn10_disable_crtc(struct timing_generator *tg)
+bool optc1_disable_crtc(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	/* disable otg request until end of the first line
 	 * in the vertical blank region
@@ -455,11 +441,11 @@ static bool tgn10_disable_crtc(struct timing_generator *tg)
 }
 
 
-static void tgn10_program_blank_color(
-		struct timing_generator *tg,
+void optc1_program_blank_color(
+		struct timing_generator *optc,
 		const struct tg_color *black_color)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_SET_3(OTG_BLACK_COLOR, 0,
 			OTG_BLACK_COLOR_B_CB, black_color->color_b_cb,
@@ -467,15 +453,15 @@ static void tgn10_program_blank_color(
 			OTG_BLACK_COLOR_R_CR, black_color->color_r_cr);
 }
 
-static bool tgn10_validate_timing(
-	struct timing_generator *tg,
+bool optc1_validate_timing(
+	struct timing_generator *optc,
 	const struct dc_crtc_timing *timing)
 {
 	uint32_t interlace_factor;
 	uint32_t v_blank;
 	uint32_t h_blank;
 	uint32_t min_v_blank;
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	ASSERT(timing != NULL);
 
@@ -505,19 +491,19 @@ static bool tgn10_validate_timing(
 	 * needs more than 8192 horizontal and
 	 * more than 8192 vertical total pixels)
 	 */
-	if (timing->h_total > tgn10->max_h_total ||
-		timing->v_total > tgn10->max_v_total)
+	if (timing->h_total > optc1->max_h_total ||
+		timing->v_total > optc1->max_v_total)
 		return false;
 
 
-	if (h_blank < tgn10->min_h_blank)
+	if (h_blank < optc1->min_h_blank)
 		return false;
 
-	if (timing->h_sync_width  < tgn10->min_h_sync_width ||
-		 timing->v_sync_width  < tgn10->min_v_sync_width)
+	if (timing->h_sync_width  < optc1->min_h_sync_width ||
+		 timing->v_sync_width  < optc1->min_v_sync_width)
 		return false;
 
-	min_v_blank = timing->flags.INTERLACE?tgn10->min_v_blank_interlace:tgn10->min_v_blank;
+	min_v_blank = timing->flags.INTERLACE?optc1->min_v_blank_interlace:optc1->min_v_blank;
 
 	if (v_blank < min_v_blank)
 		return false;
@@ -534,15 +520,15 @@ static bool tgn10_validate_timing(
  * holds the counter of frames.
  *
  * @param
- * struct timing_generator *tg - [in] timing generator which controls the
+ * struct timing_generator *optc - [in] timing generator which controls the
  * desired CRTC
  *
  * @return
  * Counter of frames, which should equal to number of vblanks.
  */
-static uint32_t tgn10_get_vblank_counter(struct timing_generator *tg)
+uint32_t optc1_get_vblank_counter(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	uint32_t frame_count;
 
 	REG_GET(OTG_STATUS_FRAME_COUNT,
@@ -551,38 +537,34 @@ static uint32_t tgn10_get_vblank_counter(struct timing_generator *tg)
 	return frame_count;
 }
 
-static void tgn10_lock(struct timing_generator *tg)
+void optc1_lock(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_SET(OTG_GLOBAL_CONTROL0, 0,
-			OTG_MASTER_UPDATE_LOCK_SEL, tg->inst);
+			OTG_MASTER_UPDATE_LOCK_SEL, optc->inst);
 	REG_SET(OTG_MASTER_UPDATE_LOCK, 0,
 			OTG_MASTER_UPDATE_LOCK, 1);
 
-	if (tg->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
+	/* Should be fast, status does not update on maximus */
+	if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS)
 		REG_WAIT(OTG_MASTER_UPDATE_LOCK,
 				UPDATE_LOCK_STATUS, 1,
-				1, 100);
+				1, 10);
 }
 
-static void tgn10_unlock(struct timing_generator *tg)
+void optc1_unlock(struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_SET(OTG_MASTER_UPDATE_LOCK, 0,
 			OTG_MASTER_UPDATE_LOCK, 0);
-
-	/* why are we waiting here? */
-	REG_WAIT(OTG_DOUBLE_BUFFER_CONTROL,
-			OTG_UPDATE_PENDING, 0,
-			1, 100000);
 }
 
-static void tgn10_get_position(struct timing_generator *tg,
+void optc1_get_position(struct timing_generator *optc,
 		struct crtc_position *position)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_GET_2(OTG_STATUS_POSITION,
 			OTG_HORZ_COUNT, &position->horizontal_count,
@@ -592,12 +574,12 @@ static void tgn10_get_position(struct timing_generator *tg,
 			OTG_VERT_COUNT_NOM, &position->nominal_vcount);
 }
 
-static bool tgn10_is_counter_moving(struct timing_generator *tg)
+bool optc1_is_counter_moving(struct timing_generator *optc)
 {
 	struct crtc_position position1, position2;
 
-	tg->funcs->get_position(tg, &position1);
-	tg->funcs->get_position(tg, &position2);
+	optc->funcs->get_position(optc, &position1);
+	optc->funcs->get_position(optc, &position2);
 
 	if (position1.horizontal_count == position2.horizontal_count &&
 		position1.vertical_count == position2.vertical_count)
@@ -606,21 +588,37 @@ static bool tgn10_is_counter_moving(struct timing_generator *tg)
 		return true;
 }
 
-static bool tgn10_did_triggered_reset_occur(
-	struct timing_generator *tg)
+bool optc1_did_triggered_reset_occur(
+	struct timing_generator *optc)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
-	uint32_t occurred;
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t occurred_force, occurred_vsync;
 
 	REG_GET(OTG_FORCE_COUNT_NOW_CNTL,
-		OTG_FORCE_COUNT_NOW_OCCURRED, &occurred);
+		OTG_FORCE_COUNT_NOW_OCCURRED, &occurred_force);
+
+	REG_GET(OTG_VERT_SYNC_CONTROL,
+		OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED, &occurred_vsync);
+
+	return occurred_vsync != 0 || occurred_force != 0;
+}
+
+void optc1_disable_reset_trigger(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	REG_WRITE(OTG_TRIGA_CNTL, 0);
+
+	REG_SET(OTG_FORCE_COUNT_NOW_CNTL, 0,
+		OTG_FORCE_COUNT_NOW_CLEAR, 1);
 
-	return occurred != 0;
+	REG_SET(OTG_VERT_SYNC_CONTROL, 0,
+		OTG_FORCE_VSYNC_NEXT_LINE_CLEAR, 1);
 }
 
-static void tgn10_enable_reset_trigger(struct timing_generator *tg, int source_tg_inst)
+void optc1_enable_reset_trigger(struct timing_generator *optc, int source_tg_inst)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	uint32_t falling_edge;
 
 	REG_GET(OTG_V_SYNC_A_CNTL,
@@ -652,20 +650,55 @@ static void tgn10_enable_reset_trigger(struct timing_generator *tg, int source_t
 			OTG_FORCE_COUNT_NOW_MODE, 2);
 }
 
-static void tgn10_disable_reset_trigger(struct timing_generator *tg)
+void optc1_enable_crtc_reset(
+		struct timing_generator *optc,
+		int source_tg_inst,
+		struct crtc_trigger_info *crtc_tp)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t falling_edge = 0;
+	uint32_t rising_edge = 0;
 
-	REG_WRITE(OTG_TRIGA_CNTL, 0);
+	switch (crtc_tp->event) {
 
-	REG_SET(OTG_FORCE_COUNT_NOW_CNTL, 0,
-			OTG_FORCE_COUNT_NOW_CLEAR, 1);
+	case CRTC_EVENT_VSYNC_RISING:
+		rising_edge = 1;
+		break;
+
+	case CRTC_EVENT_VSYNC_FALLING:
+		falling_edge = 1;
+		break;
+	}
+
+	REG_SET_4(OTG_TRIGA_CNTL, 0,
+		 /* vsync signal from selected OTG pipe based
+		  * on OTG_TRIG_SOURCE_PIPE_SELECT setting
+		  */
+		  OTG_TRIGA_SOURCE_SELECT, 20,
+		  OTG_TRIGA_SOURCE_PIPE_SELECT, source_tg_inst,
+		  /* always detect falling edge */
+		  OTG_TRIGA_RISING_EDGE_DETECT_CNTL, rising_edge,
+		  OTG_TRIGA_FALLING_EDGE_DETECT_CNTL, falling_edge);
+
+	switch (crtc_tp->delay) {
+	case TRIGGER_DELAY_NEXT_LINE:
+		REG_SET(OTG_VERT_SYNC_CONTROL, 0,
+				OTG_AUTO_FORCE_VSYNC_MODE, 1);
+		break;
+	case TRIGGER_DELAY_NEXT_PIXEL:
+		REG_SET(OTG_FORCE_COUNT_NOW_CNTL, 0,
+			/* force H count to H_TOTAL and V count to V_TOTAL in
+			 * progressive mode and V_TOTAL-1 in interlaced mode
+			 */
+			OTG_FORCE_COUNT_NOW_MODE, 2);
+		break;
+	}
 }
 
-static void tgn10_wait_for_state(struct timing_generator *tg,
+void optc1_wait_for_state(struct timing_generator *optc,
 		enum crtc_state state)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	switch (state) {
 	case CRTC_STATE_VBLANK:
@@ -685,8 +718,8 @@ static void tgn10_wait_for_state(struct timing_generator *tg,
 	}
 }
 
-static void tgn10_set_early_control(
-	struct timing_generator *tg,
+void optc1_set_early_control(
+	struct timing_generator *optc,
 	uint32_t early_cntl)
 {
 	/* asic design change, do not need this control
@@ -695,11 +728,11 @@ static void tgn10_set_early_control(
 }
 
 
-static void tgn10_set_static_screen_control(
-	struct timing_generator *tg,
+void optc1_set_static_screen_control(
+	struct timing_generator *optc,
 	uint32_t value)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	/* Bit 8 is no longer applicable in RV for PSR case,
 	 * set bit 8 to 0 if given
@@ -724,11 +757,11 @@ static void tgn10_set_static_screen_control(
  *
  *****************************************************************************
  */
-static void tgn10_set_drr(
-	struct timing_generator *tg,
+void optc1_set_drr(
+	struct timing_generator *optc,
 	const struct drr_params *params)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	if (params != NULL &&
 		params->vertical_total_max > 0 &&
@@ -761,15 +794,15 @@ static void tgn10_set_drr(
 	}
 }
 
-static void tgn10_set_test_pattern(
-	struct timing_generator *tg,
+static void optc1_set_test_pattern(
+	struct timing_generator *optc,
 	/* TODO: replace 'controller_dp_test_pattern' by 'test_pattern_mode'
 	 * because this is not DP-specific (which is probably somewhere in DP
 	 * encoder) */
 	enum controller_dp_test_pattern test_pattern,
 	enum dc_color_depth color_depth)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	enum test_pattern_color_format bit_depth;
 	enum test_pattern_dyn_range dyn_range;
 	enum test_pattern_mode mode;
@@ -1020,35 +1053,30 @@ static void tgn10_set_test_pattern(
 	}
 }
 
-static void tgn10_get_crtc_scanoutpos(
-	struct timing_generator *tg,
+void optc1_get_crtc_scanoutpos(
+	struct timing_generator *optc,
 	uint32_t *v_blank_start,
 	uint32_t *v_blank_end,
 	uint32_t *h_position,
 	uint32_t *v_position)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	struct crtc_position position;
 
 	REG_GET_2(OTG_V_BLANK_START_END,
 			OTG_V_BLANK_START, v_blank_start,
 			OTG_V_BLANK_END, v_blank_end);
 
-	tgn10_get_position(tg, &position);
+	optc1_get_position(optc, &position);
 
 	*h_position = position.horizontal_count;
 	*v_position = position.vertical_count;
 }
 
-
-
-static void tgn10_enable_stereo(struct timing_generator *tg,
+static void optc1_enable_stereo(struct timing_generator *optc,
 	const struct dc_crtc_timing *timing, struct crtc_stereo_flags *flags)
 {
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
-
-	uint32_t active_width = timing->h_addressable;
-	uint32_t space1_size = timing->v_total - timing->v_addressable;
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	if (flags) {
 		uint32_t stereo_en;
@@ -1076,29 +1104,23 @@ static void tgn10_enable_stereo(struct timing_generator *tg,
 				OTG_3D_STRUCTURE_STEREO_SEL_OVR, flags->FRAME_PACKED);
 
 	}
-
-	REG_UPDATE(OPPBUF_CONTROL,
-		OPPBUF_ACTIVE_WIDTH, active_width);
-
-	REG_UPDATE(OPPBUF_3D_PARAMETERS_0,
-		OPPBUF_3D_VACT_SPACE1_SIZE, space1_size);
 }
 
-static void tgn10_program_stereo(struct timing_generator *tg,
+void optc1_program_stereo(struct timing_generator *optc,
 	const struct dc_crtc_timing *timing, struct crtc_stereo_flags *flags)
 {
 	if (flags->PROGRAM_STEREO)
-		tgn10_enable_stereo(tg, timing, flags);
+		optc1_enable_stereo(optc, timing, flags);
 	else
-		tgn10_disable_stereo(tg);
+		optc1_disable_stereo(optc);
 }
 
 
-static bool tgn10_is_stereo_left_eye(struct timing_generator *tg)
+bool optc1_is_stereo_left_eye(struct timing_generator *optc)
 {
 	bool ret = false;
 	uint32_t left_eye = 0;
-	struct dcn10_timing_generator *tgn10 = DCN10TG_FROM_TG(tg);
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
 	REG_GET(OTG_STEREO_STATUS,
 		OTG_STEREO_CURRENT_EYE, &left_eye);
@@ -1110,7 +1132,7 @@ static bool tgn10_is_stereo_left_eye(struct timing_generator *tg)
 	return ret;
 }
 
-void tgn10_read_otg_state(struct dcn10_timing_generator *tgn10,
+void optc1_read_otg_state(struct optc *optc1,
 		struct dcn_otg_state *s)
 {
 	REG_GET(OTG_CONTROL,
@@ -1154,47 +1176,88 @@ void tgn10_read_otg_state(struct dcn10_timing_generator *tgn10,
 			OPTC_UNDERFLOW_OCCURRED_STATUS, &s->underflow_occurred_status);
 }
 
+static void optc1_clear_optc_underflow(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+
+	REG_UPDATE(OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_CLEAR, 1);
+}
+
+static void optc1_tg_init(struct timing_generator *optc)
+{
+	optc1_set_blank_data_double_buffer(optc, true);
+	optc1_clear_optc_underflow(optc);
+}
+
+static bool optc1_is_tg_enabled(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t otg_enabled = 0;
+
+	REG_GET(OTG_CONTROL, OTG_MASTER_EN, &otg_enabled);
+
+	return (otg_enabled != 0);
+
+}
+
+static bool optc1_is_optc_underflow_occurred(struct timing_generator *optc)
+{
+	struct optc *optc1 = DCN10TG_FROM_TG(optc);
+	uint32_t underflow_occurred = 0;
+
+	REG_GET(OPTC_INPUT_GLOBAL_CONTROL,
+			OPTC_UNDERFLOW_OCCURRED_STATUS,
+			&underflow_occurred);
+
+	return (underflow_occurred == 1);
+}
 
 static const struct timing_generator_funcs dcn10_tg_funcs = {
-		.validate_timing = tgn10_validate_timing,
-		.program_timing = tgn10_program_timing,
-		.program_global_sync = tgn10_program_global_sync,
-		.enable_crtc = tgn10_enable_crtc,
-		.disable_crtc = tgn10_disable_crtc,
+		.validate_timing = optc1_validate_timing,
+		.program_timing = optc1_program_timing,
+		.program_global_sync = optc1_program_global_sync,
+		.enable_crtc = optc1_enable_crtc,
+		.disable_crtc = optc1_disable_crtc,
 		/* used by enable_timing_synchronization. Not need for FPGA */
-		.is_counter_moving = tgn10_is_counter_moving,
-		.get_position = tgn10_get_position,
-		.get_frame_count = tgn10_get_vblank_counter,
-		.get_scanoutpos = tgn10_get_crtc_scanoutpos,
-		.set_early_control = tgn10_set_early_control,
+		.is_counter_moving = optc1_is_counter_moving,
+		.get_position = optc1_get_position,
+		.get_frame_count = optc1_get_vblank_counter,
+		.get_scanoutpos = optc1_get_crtc_scanoutpos,
+		.set_early_control = optc1_set_early_control,
 		/* used by enable_timing_synchronization. Not need for FPGA */
-		.wait_for_state = tgn10_wait_for_state,
-		.set_blank = tgn10_set_blank,
-		.is_blanked = tgn10_is_blanked,
-		.set_blank_color = tgn10_program_blank_color,
-		.did_triggered_reset_occur = tgn10_did_triggered_reset_occur,
-		.enable_reset_trigger = tgn10_enable_reset_trigger,
-		.disable_reset_trigger = tgn10_disable_reset_trigger,
-		.lock = tgn10_lock,
-		.unlock = tgn10_unlock,
-		.enable_optc_clock = tgn10_enable_optc_clock,
-		.set_drr = tgn10_set_drr,
-		.set_static_screen_control = tgn10_set_static_screen_control,
-		.set_test_pattern = tgn10_set_test_pattern,
-		.program_stereo = tgn10_program_stereo,
-		.is_stereo_left_eye = tgn10_is_stereo_left_eye
+		.wait_for_state = optc1_wait_for_state,
+		.set_blank = optc1_set_blank,
+		.is_blanked = optc1_is_blanked,
+		.set_blank_color = optc1_program_blank_color,
+		.did_triggered_reset_occur = optc1_did_triggered_reset_occur,
+		.enable_reset_trigger = optc1_enable_reset_trigger,
+		.enable_crtc_reset = optc1_enable_crtc_reset,
+		.disable_reset_trigger = optc1_disable_reset_trigger,
+		.lock = optc1_lock,
+		.unlock = optc1_unlock,
+		.enable_optc_clock = optc1_enable_optc_clock,
+		.set_drr = optc1_set_drr,
+		.set_static_screen_control = optc1_set_static_screen_control,
+		.set_test_pattern = optc1_set_test_pattern,
+		.program_stereo = optc1_program_stereo,
+		.is_stereo_left_eye = optc1_is_stereo_left_eye,
+		.set_blank_data_double_buffer = optc1_set_blank_data_double_buffer,
+		.tg_init = optc1_tg_init,
+		.is_tg_enabled = optc1_is_tg_enabled,
+		.is_optc_underflow_occurred = optc1_is_optc_underflow_occurred,
+		.clear_optc_underflow = optc1_clear_optc_underflow,
 };
 
-void dcn10_timing_generator_init(struct dcn10_timing_generator *tgn10)
+void dcn10_timing_generator_init(struct optc *optc1)
 {
-	tgn10->base.funcs = &dcn10_tg_funcs;
+	optc1->base.funcs = &dcn10_tg_funcs;
 
-	tgn10->max_h_total = tgn10->tg_mask->OTG_H_TOTAL + 1;
-	tgn10->max_v_total = tgn10->tg_mask->OTG_V_TOTAL + 1;
+	optc1->max_h_total = optc1->tg_mask->OTG_H_TOTAL + 1;
+	optc1->max_v_total = optc1->tg_mask->OTG_V_TOTAL + 1;
 
-	tgn10->min_h_blank = 32;
-	tgn10->min_v_blank = 3;
-	tgn10->min_v_blank_interlace = 5;
-	tgn10->min_h_sync_width = 8;
-	tgn10->min_v_sync_width = 1;
+	optc1->min_h_blank = 32;
+	optc1->min_v_blank = 3;
+	optc1->min_v_blank_interlace = 5;
+	optc1->min_h_sync_width = 8;
+	optc1->min_v_sync_width = 1;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_timing_generator.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
similarity index 76%
rename from drivers/gpu/drm/amd/display/dc/dcn10/dcn10_timing_generator.h
rename to drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
index 7d4818d7aa31..a3c7c2012f05 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
@@ -29,7 +29,7 @@
 #include "timing_generator.h"
 
 #define DCN10TG_FROM_TG(tg)\
-	container_of(tg, struct dcn10_timing_generator, base)
+	container_of(tg, struct optc, base)
 
 #define TG_COMMON_REG_LIST_DCN(inst) \
 	SRI(OTG_VSTARTUP_PARAM, OTG, inst),\
@@ -70,9 +70,10 @@
 	SRI(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),\
 	SRI(OPTC_DATA_SOURCE_SELECT, ODM, inst),\
 	SRI(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),\
-	SRI(OPPBUF_CONTROL, OPPBUF, inst),\
-	SRI(OPPBUF_3D_PARAMETERS_0, OPPBUF, inst),\
-	SRI(CONTROL, VTG, inst)
+	SRI(CONTROL, VTG, inst),\
+	SRI(OTG_VERT_SYNC_CONTROL, OTG, inst),\
+	SRI(OTG_MASTER_UPDATE_MODE, OTG, inst),\
+	SRI(OTG_GSL_CONTROL, OTG, inst)
 
 #define TG_COMMON_REG_LIST_DCN1_0(inst) \
 	TG_COMMON_REG_LIST_DCN(inst),\
@@ -81,7 +82,10 @@
 	SRI(OTG_TEST_PATTERN_COLOR, OTG, inst)
 
 
-struct dcn_tg_registers {
+struct dcn_optc_registers {
+	uint32_t OTG_VERT_SYNC_CONTROL;
+	uint32_t OTG_MASTER_UPDATE_MODE;
+	uint32_t OTG_GSL_CONTROL;
 	uint32_t OTG_VSTARTUP_PARAM;
 	uint32_t OTG_VUPDATE_PARAM;
 	uint32_t OTG_VREADY_PARAM;
@@ -123,9 +127,11 @@ struct dcn_tg_registers {
 	uint32_t OPTC_INPUT_CLOCK_CONTROL;
 	uint32_t OPTC_DATA_SOURCE_SELECT;
 	uint32_t OPTC_INPUT_GLOBAL_CONTROL;
-	uint32_t OPPBUF_CONTROL;
-	uint32_t OPPBUF_3D_PARAMETERS_0;
 	uint32_t CONTROL;
+	uint32_t OTG_GSL_WINDOW_X;
+	uint32_t OTG_GSL_WINDOW_Y;
+	uint32_t OTG_VUPDATE_KEEPOUT;
+	uint32_t OTG_DSC_START_POSITION;
 };
 
 #define TG_COMMON_MASK_SH_LIST_DCN(mask_sh)\
@@ -204,11 +210,21 @@ struct dcn_tg_registers {
 	SF(ODM0_OPTC_INPUT_CLOCK_CONTROL, OPTC_INPUT_CLK_ON, mask_sh),\
 	SF(ODM0_OPTC_INPUT_CLOCK_CONTROL, OPTC_INPUT_CLK_GATE_DIS, mask_sh),\
 	SF(ODM0_OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_OCCURRED_STATUS, mask_sh),\
-	SF(OPPBUF0_OPPBUF_CONTROL, OPPBUF_ACTIVE_WIDTH, mask_sh),\
-	SF(OPPBUF0_OPPBUF_3D_PARAMETERS_0, OPPBUF_3D_VACT_SPACE1_SIZE, mask_sh),\
+	SF(ODM0_OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_CLEAR, mask_sh),\
 	SF(VTG0_CONTROL, VTG0_ENABLE, mask_sh),\
 	SF(VTG0_CONTROL, VTG0_FP2, mask_sh),\
-	SF(VTG0_CONTROL, VTG0_VCOUNT_INIT, mask_sh)
+	SF(VTG0_CONTROL, VTG0_VCOUNT_INIT, mask_sh),\
+	SF(OTG0_OTG_VERT_SYNC_CONTROL, OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED, mask_sh),\
+	SF(OTG0_OTG_VERT_SYNC_CONTROL, OTG_FORCE_VSYNC_NEXT_LINE_CLEAR, mask_sh),\
+	SF(OTG0_OTG_VERT_SYNC_CONTROL, OTG_AUTO_FORCE_VSYNC_MODE, mask_sh),\
+	SF(OTG0_OTG_MASTER_UPDATE_MODE, MASTER_UPDATE_INTERLACED_MODE, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL0_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL1_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL2_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_MASTER_EN, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_FORCE_DELAY, mask_sh),\
+	SF(OTG0_OTG_GSL_CONTROL, OTG_GSL_CHECK_ALL_FIELDS, mask_sh)
+
 
 #define TG_COMMON_MASK_SH_LIST_DCN1_0(mask_sh)\
 	TG_COMMON_MASK_SH_LIST_DCN(mask_sh),\
@@ -313,26 +329,48 @@ struct dcn_tg_registers {
 	type OPTC_SRC_SEL;\
 	type OPTC_SEG0_SRC_SEL;\
 	type OPTC_UNDERFLOW_OCCURRED_STATUS;\
-	type OPPBUF_ACTIVE_WIDTH;\
-	type OPPBUF_3D_VACT_SPACE1_SIZE;\
+	type OPTC_UNDERFLOW_CLEAR;\
 	type VTG0_ENABLE;\
 	type VTG0_FP2;\
-	type VTG0_VCOUNT_INIT;
+	type VTG0_VCOUNT_INIT;\
+	type OTG_FORCE_VSYNC_NEXT_LINE_OCCURRED;\
+	type OTG_FORCE_VSYNC_NEXT_LINE_CLEAR;\
+	type OTG_AUTO_FORCE_VSYNC_MODE;\
+	type MASTER_UPDATE_INTERLACED_MODE;\
+	type OTG_GSL0_EN;\
+	type OTG_GSL1_EN;\
+	type OTG_GSL2_EN;\
+	type OTG_GSL_MASTER_EN;\
+	type OTG_GSL_FORCE_DELAY;\
+	type OTG_GSL_CHECK_ALL_FIELDS;\
+	type OTG_GSL_WINDOW_START_X;\
+	type OTG_GSL_WINDOW_END_X;\
+	type OTG_GSL_WINDOW_START_Y;\
+	type OTG_GSL_WINDOW_END_Y;\
+	type OTG_RANGE_TIMING_DBUF_UPDATE_MODE;\
+	type OTG_GSL_MASTER_MODE;\
+	type OTG_MASTER_UPDATE_LOCK_GSL_EN;\
+	type MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET;\
+	type MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET;\
+	type OTG_DSC_START_POSITION_X;\
+	type OTG_DSC_START_POSITION_LINE_NUM;\
+	type OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN;
+
 
-struct dcn_tg_shift {
+struct dcn_optc_shift {
 	TG_REG_FIELD_LIST(uint8_t)
 };
 
-struct dcn_tg_mask {
+struct dcn_optc_mask {
 	TG_REG_FIELD_LIST(uint32_t)
 };
 
-struct dcn10_timing_generator {
+struct optc {
 	struct timing_generator base;
 
-	const struct dcn_tg_registers *tg_regs;
-	const struct dcn_tg_shift *tg_shift;
-	const struct dcn_tg_mask *tg_mask;
+	const struct dcn_optc_registers *tg_regs;
+	const struct dcn_optc_shift *tg_shift;
+	const struct dcn_optc_mask *tg_mask;
 
 	enum controller_id controller_id;
 
@@ -347,7 +385,7 @@ struct dcn10_timing_generator {
 	uint32_t min_v_blank_interlace;
 };
 
-void dcn10_timing_generator_init(struct dcn10_timing_generator *tg);
+void dcn10_timing_generator_init(struct optc *optc);
 
 struct dcn_otg_state {
 	uint32_t v_blank_start;
@@ -368,7 +406,77 @@ struct dcn_otg_state {
 	uint32_t otg_enabled;
 };
 
-void tgn10_read_otg_state(struct dcn10_timing_generator *tgn10,
+void optc1_read_otg_state(struct optc *optc1,
 		struct dcn_otg_state *s);
 
+bool optc1_validate_timing(
+	struct timing_generator *optc,
+	const struct dc_crtc_timing *timing);
+
+void optc1_program_timing(
+	struct timing_generator *optc,
+	const struct dc_crtc_timing *dc_crtc_timing,
+	bool use_vbios);
+
+void optc1_program_global_sync(
+		struct timing_generator *optc);
+
+bool optc1_disable_crtc(struct timing_generator *optc);
+
+bool optc1_is_counter_moving(struct timing_generator *optc);
+
+void optc1_get_position(struct timing_generator *optc,
+		struct crtc_position *position);
+
+uint32_t optc1_get_vblank_counter(struct timing_generator *optc);
+
+void optc1_get_crtc_scanoutpos(
+	struct timing_generator *optc,
+	uint32_t *v_blank_start,
+	uint32_t *v_blank_end,
+	uint32_t *h_position,
+	uint32_t *v_position);
+
+void optc1_set_early_control(
+	struct timing_generator *optc,
+	uint32_t early_cntl);
+
+void optc1_wait_for_state(struct timing_generator *optc,
+		enum crtc_state state);
+
+void optc1_set_blank(struct timing_generator *optc,
+		bool enable_blanking);
+
+bool optc1_is_blanked(struct timing_generator *optc);
+
+void optc1_program_blank_color(
+		struct timing_generator *optc,
+		const struct tg_color *black_color);
+
+bool optc1_did_triggered_reset_occur(
+	struct timing_generator *optc);
+
+void optc1_enable_reset_trigger(struct timing_generator *optc, int source_tg_inst);
+
+void optc1_disable_reset_trigger(struct timing_generator *optc);
+
+void optc1_lock(struct timing_generator *optc);
+
+void optc1_unlock(struct timing_generator *optc);
+
+void optc1_enable_optc_clock(struct timing_generator *optc, bool enable);
+
+void optc1_set_drr(
+	struct timing_generator *optc,
+	const struct drr_params *params);
+
+void optc1_set_static_screen_control(
+	struct timing_generator *optc,
+	uint32_t value);
+
+void optc1_program_stereo(struct timing_generator *optc,
+	const struct dc_crtc_timing *timing, struct crtc_stereo_flags *flags);
+
+bool optc1_is_stereo_left_eye(struct timing_generator *optc);
+
 #endif /* __DC_TIMING_GENERATOR_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
index 9fc8f827f2a1..44825e2c9ebb 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
@@ -34,7 +34,7 @@
 #include "dcn10/dcn10_mpc.h"
 #include "irq/dcn10/irq_service_dcn10.h"
 #include "dcn10/dcn10_dpp.h"
-#include "dcn10/dcn10_timing_generator.h"
+#include "dcn10_optc.h"
 #include "dcn10/dcn10_hw_sequencer.h"
 #include "dce110/dce110_hw_sequencer.h"
 #include "dcn10/dcn10_opp.h"
@@ -48,16 +48,17 @@
 #include "dce110/dce110_resource.h"
 #include "dce112/dce112_resource.h"
 #include "dcn10_hubp.h"
+#include "dcn10_hubbub.h"
 
-#include "vega10/soc15ip.h"
+#include "soc15ip.h"
 
-#include "raven1/DCN/dcn_1_0_offset.h"
-#include "raven1/DCN/dcn_1_0_sh_mask.h"
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
 
-#include "raven1/NBIO/nbio_7_0_offset.h"
+#include "nbio/nbio_7_0_offset.h"
 
-#include "raven1/MMHUB/mmhub_9_1_offset.h"
-#include "raven1/MMHUB/mmhub_9_1_sh_mask.h"
+#include "mmhub/mmhub_9_1_offset.h"
+#include "mmhub/mmhub_9_1_sh_mask.h"
 
 #include "reg_helper.h"
 #include "dce/dce_abm.h"
@@ -347,18 +348,18 @@ static const struct dcn_mpc_mask mpc_mask = {
 #define tg_regs(id)\
 [id] = {TG_COMMON_REG_LIST_DCN1_0(id)}
 
-static const struct dcn_tg_registers tg_regs[] = {
+static const struct dcn_optc_registers tg_regs[] = {
 	tg_regs(0),
 	tg_regs(1),
 	tg_regs(2),
 	tg_regs(3),
 };
 
-static const struct dcn_tg_shift tg_shift = {
+static const struct dcn_optc_shift tg_shift = {
 	TG_COMMON_MASK_SH_LIST_DCN1_0(__SHIFT)
 };
 
-static const struct dcn_tg_mask tg_mask = {
+static const struct dcn_optc_mask tg_mask = {
 	TG_COMMON_MASK_SH_LIST_DCN1_0(_MASK)
 };
 
@@ -367,25 +368,38 @@ static const struct bios_registers bios_regs = {
 		NBIO_SR(BIOS_SCRATCH_6)
 };
 
-#define mi_regs(id)\
+#define hubp_regs(id)\
 [id] = {\
-	MI_REG_LIST_DCN10(id)\
+	HUBP_REG_LIST_DCN10(id)\
 }
 
 
-static const struct dcn_mi_registers mi_regs[] = {
-	mi_regs(0),
-	mi_regs(1),
-	mi_regs(2),
-	mi_regs(3),
+static const struct dcn_mi_registers hubp_regs[] = {
+	hubp_regs(0),
+	hubp_regs(1),
+	hubp_regs(2),
+	hubp_regs(3),
 };
 
-static const struct dcn_mi_shift mi_shift = {
-		MI_MASK_SH_LIST_DCN10(__SHIFT)
+static const struct dcn_mi_shift hubp_shift = {
+		HUBP_MASK_SH_LIST_DCN10(__SHIFT)
 };
 
-static const struct dcn_mi_mask mi_mask = {
-		MI_MASK_SH_LIST_DCN10(_MASK)
+static const struct dcn_mi_mask hubp_mask = {
+		HUBP_MASK_SH_LIST_DCN10(_MASK)
+};
+
+
+static const struct dcn_hubbub_registers hubbub_reg = {
+		HUBBUB_REG_LIST_DCN10(0)
+};
+
+static const struct dcn_hubbub_shift hubbub_shift = {
+		HUBBUB_MASK_SH_LIST_DCN10(__SHIFT)
+};
+
+static const struct dcn_hubbub_mask hubbub_mask = {
+		HUBBUB_MASK_SH_LIST_DCN10(_MASK)
 };
 
 #define clk_src_regs(index, pllid)\
@@ -519,12 +533,28 @@ static struct mpc *dcn10_mpc_create(struct dc_context *ctx)
 	return &mpc10->base;
 }
 
+static struct hubbub *dcn10_hubbub_create(struct dc_context *ctx)
+{
+	struct hubbub *hubbub = kzalloc(sizeof(struct hubbub),
+					  GFP_KERNEL);
+
+	if (!hubbub)
+		return NULL;
+
+	hubbub1_construct(hubbub, ctx,
+			&hubbub_reg,
+			&hubbub_shift,
+			&hubbub_mask);
+
+	return hubbub;
+}
+
 static struct timing_generator *dcn10_timing_generator_create(
 		struct dc_context *ctx,
 		uint32_t instance)
 {
-	struct dcn10_timing_generator *tgn10 =
-		kzalloc(sizeof(struct dcn10_timing_generator), GFP_KERNEL);
+	struct optc *tgn10 =
+		kzalloc(sizeof(struct optc), GFP_KERNEL);
 
 	if (!tgn10)
 		return NULL;
@@ -647,6 +677,8 @@ static struct dce_hwseq *dcn10_hwseq_create(
 		hws->regs = &hwseq_reg;
 		hws->shifts = &hwseq_shift;
 		hws->masks = &hwseq_mask;
+		hws->wa.DEGVIDCN10_253 = true;
+		hws->wa.false_optc_underflow = true;
 	}
 	return hws;
 }
@@ -700,6 +732,12 @@ static void destruct(struct dcn10_resource_pool *pool)
 		kfree(TO_DCN10_MPC(pool->base.mpc));
 		pool->base.mpc = NULL;
 	}
+
+	if (pool->base.hubbub != NULL) {
+		kfree(pool->base.hubbub);
+		pool->base.hubbub = NULL;
+	}
+
 	for (i = 0; i < pool->base.pipe_count; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
@@ -768,7 +806,7 @@ static struct hubp *dcn10_hubp_create(
 		return NULL;
 
 	dcn10_hubp_construct(hubp1, ctx, inst,
-			     &mi_regs[inst], &mi_shift, &mi_mask);
+			     &hubp_regs[inst], &hubp_shift, &hubp_mask);
 	return &hubp1->base;
 }
 
@@ -1233,8 +1271,8 @@ static bool construct(
 	dc->caps.max_downscale_ratio = 200;
 	dc->caps.i2c_speed_in_khz = 100;
 	dc->caps.max_cursor_size = 256;
-
 	dc->caps.max_slave_planes = 1;
+	dc->caps.is_apu = true;
 
 	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
 		dc->debug = debug_defaults_drv;
@@ -1274,7 +1312,7 @@ static bool construct(
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
-			goto clock_source_create_fail;
+			goto fail;
 		}
 	}
 
@@ -1283,7 +1321,7 @@ static bool construct(
 		if (pool->base.display_clock == NULL) {
 			dm_error("DC: failed to create display clock!\n");
 			BREAK_TO_DEBUGGER();
-			goto disp_clk_create_fail;
+			goto fail;
 		}
 	}
 
@@ -1294,7 +1332,7 @@ static bool construct(
 	if (pool->base.dmcu == NULL) {
 		dm_error("DC: failed to create dmcu!\n");
 		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
+		goto fail;
 	}
 
 	pool->base.abm = dce_abm_create(ctx,
@@ -1304,7 +1342,7 @@ static bool construct(
 	if (pool->base.abm == NULL) {
 		dm_error("DC: failed to create abm!\n");
 		BREAK_TO_DEBUGGER();
-		goto res_create_fail;
+		goto fail;
 	}
 
 	dml_init_instance(&dc->dml, DML_PROJECT_RAVEN1);
@@ -1344,13 +1382,11 @@ static bool construct(
 	}
 
 	{
-	#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 		struct irq_service_init_data init_data;
 		init_data.ctx = dc->ctx;
 		pool->base.irqs = dal_irq_service_dcn10_create(&init_data);
 		if (!pool->base.irqs)
-			goto irqs_create_fail;
-	#endif
+			goto fail;
 	}
 
 	/* index to valid pipe resource  */
@@ -1368,7 +1404,7 @@ static bool construct(
 			BREAK_TO_DEBUGGER();
 			dm_error(
 				"DC: failed to create memory input!\n");
-			goto mi_create_fail;
+			goto fail;
 		}
 
 		pool->base.ipps[j] = dcn10_ipp_create(ctx, i);
@@ -1376,7 +1412,7 @@ static bool construct(
 			BREAK_TO_DEBUGGER();
 			dm_error(
 				"DC: failed to create input pixel processor!\n");
-			goto ipp_create_fail;
+			goto fail;
 		}
 
 		pool->base.dpps[j] = dcn10_dpp_create(ctx, i);
@@ -1384,7 +1420,7 @@ static bool construct(
 			BREAK_TO_DEBUGGER();
 			dm_error(
 				"DC: failed to create dpp!\n");
-			goto dpp_create_fail;
+			goto fail;
 		}
 
 		pool->base.opps[j] = dcn10_opp_create(ctx, i);
@@ -1392,7 +1428,7 @@ static bool construct(
 			BREAK_TO_DEBUGGER();
 			dm_error(
 				"DC: failed to create output pixel processor!\n");
-			goto opp_create_fail;
+			goto fail;
 		}
 
 		pool->base.timing_generators[j] = dcn10_timing_generator_create(
@@ -1400,8 +1436,9 @@ static bool construct(
 		if (pool->base.timing_generators[j] == NULL) {
 			BREAK_TO_DEBUGGER();
 			dm_error("DC: failed to create tg!\n");
-			goto otg_create_fail;
+			goto fail;
 		}
+
 		/* check next valid pipe */
 		j++;
 	}
@@ -1419,13 +1456,20 @@ static bool construct(
 	if (pool->base.mpc == NULL) {
 		BREAK_TO_DEBUGGER();
 		dm_error("DC: failed to create mpc!\n");
-		goto mpc_create_fail;
+		goto fail;
+	}
+
+	pool->base.hubbub = dcn10_hubbub_create(ctx);
+	if (pool->base.hubbub == NULL) {
+		BREAK_TO_DEBUGGER();
+		dm_error("DC: failed to create hubbub!\n");
+		goto fail;
 	}
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
 			(!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment) ?
 			&res_create_funcs : &res_create_maximus_funcs)))
-			goto res_create_fail;
+			goto fail;
 
 	dcn10_hw_sequencer_construct(dc);
 	dc->caps.max_planes =  pool->base.pipe_count;
@@ -1434,16 +1478,7 @@ static bool construct(
 
 	return true;
 
-disp_clk_create_fail:
-mpc_create_fail:
-otg_create_fail:
-opp_create_fail:
-dpp_create_fail:
-ipp_create_fail:
-mi_create_fail:
-irqs_create_fail:
-res_create_fail:
-clock_source_create_fail:
+fail:
 
 	destruct(pool);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dm_services.h b/drivers/gpu/drm/amd/display/dc/dm_services.h
index d4917037ac42..225b7bfb09a9 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_services.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_services.h
@@ -372,6 +372,13 @@ bool dm_dmcu_set_pipe(struct dc_context *ctx, unsigned int controller_id);
 
 unsigned long long dm_get_timestamp(struct dc_context *ctx);
 
+/*
+ * performance tracing
+ */
+void dm_perf_trace_timestamp(const char *func_name, unsigned int line);
+#define PERF_TRACE()	dm_perf_trace_timestamp(__func__, __LINE__)
+
+
 /*
  * Debug and verification hooks
  */
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
index 4c31fa54af39..c109b2c34c8f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
@@ -35,35 +35,6 @@ static void set_soc_bounding_box(struct _vcs_dpi_soc_bounding_box_st *soc, enum
 		soc->writeback_latency_us = 12.0;
 		soc->ideal_dram_bw_after_urgent_percent = 80.0;
 		soc->max_request_size_bytes = 256;
-
-		soc->vmin.dcfclk_mhz = 300.0;
-		soc->vmin.dispclk_mhz = 608.0;
-		soc->vmin.dppclk_mhz = 435.0;
-		soc->vmin.dram_bw_per_chan_gbps = 12.8;
-		soc->vmin.phyclk_mhz = 540.0;
-		soc->vmin.socclk_mhz = 208.0;
-
-		soc->vmid.dcfclk_mhz = 600.0;
-		soc->vmid.dispclk_mhz = 661.0;
-		soc->vmid.dppclk_mhz = 661.0;
-		soc->vmid.dram_bw_per_chan_gbps = 12.8;
-		soc->vmid.phyclk_mhz = 540.0;
-		soc->vmid.socclk_mhz = 208.0;
-
-		soc->vnom.dcfclk_mhz = 600.0;
-		soc->vnom.dispclk_mhz = 661.0;
-		soc->vnom.dppclk_mhz = 661.0;
-		soc->vnom.dram_bw_per_chan_gbps = 38.4;
-		soc->vnom.phyclk_mhz = 810;
-		soc->vnom.socclk_mhz = 208.0;
-
-		soc->vmax.dcfclk_mhz = 600.0;
-		soc->vmax.dispclk_mhz = 1086.0;
-		soc->vmax.dppclk_mhz = 661.0;
-		soc->vmax.dram_bw_per_chan_gbps = 38.4;
-		soc->vmax.phyclk_mhz = 810.0;
-		soc->vmax.socclk_mhz = 208.0;
-
 		soc->downspread_percent = 0.5;
 		soc->dram_page_open_time_ns = 50.0;
 		soc->dram_rw_turnaround_time_ns = 17.5;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
index baf182177736..aeebd8bee628 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
@@ -79,10 +79,6 @@ struct	_vcs_dpi_soc_bounding_box_st	{
 	double	writeback_latency_us;
 	double	ideal_dram_bw_after_urgent_percent;
 	unsigned int	max_request_size_bytes;
-	struct _vcs_dpi_voltage_scaling_st	vmin;
-	struct _vcs_dpi_voltage_scaling_st	vmid;
-	struct _vcs_dpi_voltage_scaling_st	vnom;
-	struct _vcs_dpi_voltage_scaling_st	vmax;
 	double	downspread_percent;
 	double	dram_page_open_time_ns;
 	double	dram_rw_turnaround_time_ns;
@@ -229,7 +225,7 @@ struct	_vcs_dpi_display_output_params_st	{
 	int	output_bpp;
 	int	dsc_enable;
 	int	wb_enable;
-	int	output_bpc;
+	int	opp_input_bpc;
 	int	output_type;
 	int	output_format;
 	int	output_standard;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
index ea661ee44674..260e113fcc02 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
@@ -28,6 +28,17 @@
 
 #include "dml_inline_defs.h"
 
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
+#define BPP_INVALID 0
+#define BPP_BLENDED_PIPE 0xffffffff
 static const unsigned int NumberOfStates = DC__VOLTAGE_STATES;
 
 static void fetch_socbb_params(struct display_mode_lib *mode_lib);
@@ -587,7 +598,7 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib)
 		mode_lib->vba.NumberOfDSCSlices[mode_lib->vba.NumberOfActivePlanes] =
 				dout->dsc_slices;
 		mode_lib->vba.DSCInputBitPerComponent[mode_lib->vba.NumberOfActivePlanes] =
-				dout->output_bpc == 0 ? 12 : dout->output_bpc;
+				dout->opp_input_bpc == 0 ? 12 : dout->opp_input_bpc;
 		mode_lib->vba.WritebackEnable[mode_lib->vba.NumberOfActivePlanes] = dout->wb_enable;
 		mode_lib->vba.WritebackSourceHeight[mode_lib->vba.NumberOfActivePlanes] =
 				dout->wb.wb_src_height;
@@ -3928,7 +3939,7 @@ static unsigned int TruncToValidBPP(
 			else if (DecimalBPP >= 12)
 				return 12;
 			else
-				return 0;
+				return BPP_INVALID;
 		} else if (Format == dm_444) {
 			if (DecimalBPP >= 36)
 				return 36;
@@ -3937,7 +3948,7 @@ static unsigned int TruncToValidBPP(
 			else if (DecimalBPP >= 24)
 				return 24;
 			else
-				return 0;
+				return BPP_INVALID;
 		} else {
 			if (DecimalBPP / 1.5 >= 24)
 				return 24;
@@ -3946,27 +3957,27 @@ static unsigned int TruncToValidBPP(
 			else if (DecimalBPP / 1.5 >= 16)
 				return 16;
 			else
-				return 0;
+				return BPP_INVALID;
 		}
 	} else {
 		if (DSCEnabled) {
 			if (Format == dm_420) {
 				if (DecimalBPP < 6)
-					return 0;
+					return BPP_INVALID;
 				else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1 / 16)
 					return 1.5 * DSCInputBitPerComponent - 1 / 16;
 				else
 					return dml_floor(16 * DecimalBPP, 1) / 16;
 			} else if (Format == dm_n422) {
 				if (DecimalBPP < 7)
-					return 0;
+					return BPP_INVALID;
 				else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1 / 16)
 					return 2 * DSCInputBitPerComponent - 1 / 16;
 				else
 					return dml_floor(16 * DecimalBPP, 1) / 16;
 			} else {
 				if (DecimalBPP < 8)
-					return 0;
+					return BPP_INVALID;
 				else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1 / 16)
 					return 3 * DSCInputBitPerComponent - 1 / 16;
 				else
@@ -3980,7 +3991,7 @@ static unsigned int TruncToValidBPP(
 			else if (DecimalBPP >= 12)
 				return 12;
 			else
-				return 0;
+				return BPP_INVALID;
 		} else if (Format == dm_s422 || Format == dm_n422) {
 			if (DecimalBPP >= 24)
 				return 24;
@@ -3989,7 +4000,7 @@ static unsigned int TruncToValidBPP(
 			else if (DecimalBPP >= 16)
 				return 16;
 			else
-				return 0;
+				return BPP_INVALID;
 		} else {
 			if (DecimalBPP >= 36)
 				return 36;
@@ -3998,7 +4009,7 @@ static unsigned int TruncToValidBPP(
 			else if (DecimalBPP >= 24)
 				return 24;
 			else
-				return 0;
+				return BPP_INVALID;
 		}
 	}
 }
@@ -4922,11 +4933,7 @@ static void ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 		mode_lib->vba.ViewportSizeSupport[i] = true;
 		for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
 			if (mode_lib->vba.ODMCombineEnablePerState[i][k] == true) {
-				if (dml_min(
-						mode_lib->vba.SwathWidthYSingleDPP[k],
-						dml_round(
-								mode_lib->vba.HActive[k] / 2.0
-										* mode_lib->vba.HRatio[k]))
+				if (dml_min(mode_lib->vba.SwathWidthYSingleDPP[k], dml_round(mode_lib->vba.HActive[k] / 2.0 * mode_lib->vba.HRatio[k]))
 						> mode_lib->vba.MaximumSwathWidth[k]) {
 					mode_lib->vba.ViewportSizeSupport[i] = false;
 				}
@@ -4980,12 +4987,8 @@ static void ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 					mode_lib->vba.RequiresDSC[i][k] = 0;
 					mode_lib->vba.RequiresFEC[i][k] = 0;
 					mode_lib->vba.OutputBppPerState[i][k] =
-							TruncToValidBPP(
-									dml_min(
-											600.0,
-											mode_lib->vba.PHYCLKPerState[i])
-											/ mode_lib->vba.PixelClockBackEnd[k]
-											* 24,
+							TruncToValidBPP(dml_min(600.0, mode_lib->vba.PHYCLKPerState[i])
+										/ mode_lib->vba.PixelClockBackEnd[k] * 24,
 									false,
 									mode_lib->vba.Output[k],
 									mode_lib->vba.OutputFormat[k],
@@ -5000,30 +5003,16 @@ static void ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 					}
 					if (mode_lib->vba.PHYCLKPerState[i] >= 270.0) {
 						mode_lib->vba.Outbpp =
-								TruncToValidBPP(
-										(1.0
-												- mode_lib->vba.Downspreading
-														/ 100.0)
-												* 270.0
-												* mode_lib->vba.OutputLinkDPLanes[k]
-												/ mode_lib->vba.PixelClockBackEnd[k]
-												* 8.0,
+								TruncToValidBPP((1.0 - mode_lib->vba.Downspreading / 100.0) * 270.0
+											* mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
 										false,
 										mode_lib->vba.Output[k],
 										mode_lib->vba.OutputFormat[k],
 										mode_lib->vba.DSCInputBitPerComponent[k]);
 						mode_lib->vba.OutbppDSC =
-								TruncToValidBPP(
-										(1.0
-												- mode_lib->vba.Downspreading
-														/ 100.0)
-												* (1.0
-														- mode_lib->vba.EffectiveFECOverhead
-																/ 100.0)
-												* 270.0
-												* mode_lib->vba.OutputLinkDPLanes[k]
-												/ mode_lib->vba.PixelClockBackEnd[k]
-												* 8.0,
+								TruncToValidBPP((1.0 - mode_lib->vba.Downspreading / 100.0)
+											* (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 270.0
+											* mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
 										true,
 										mode_lib->vba.Output[k],
 										mode_lib->vba.OutputFormat[k],
@@ -5046,32 +5035,18 @@ static void ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 						mode_lib->vba.OutputBppPerState[i][k] =
 								mode_lib->vba.Outbpp;
 					}
-					if (mode_lib->vba.Outbpp == 0) {
+					if (mode_lib->vba.Outbpp == BPP_INVALID) {
 						mode_lib->vba.Outbpp =
-								TruncToValidBPP(
-										(1.0
-												- mode_lib->vba.Downspreading
-														/ 100.0)
-												* 540.0
-												* mode_lib->vba.OutputLinkDPLanes[k]
-												/ mode_lib->vba.PixelClockBackEnd[k]
-												* 8.0,
+								TruncToValidBPP((1.0 - mode_lib->vba.Downspreading / 100.0) * 540.0
+											* mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
 										false,
 										mode_lib->vba.Output[k],
 										mode_lib->vba.OutputFormat[k],
 										mode_lib->vba.DSCInputBitPerComponent[k]);
 						mode_lib->vba.OutbppDSC =
-								TruncToValidBPP(
-										(1.0
-												- mode_lib->vba.Downspreading
-														/ 100.0)
-												* (1.0
-														- mode_lib->vba.EffectiveFECOverhead
-																/ 100.0)
-												* 540.0
-												* mode_lib->vba.OutputLinkDPLanes[k]
-												/ mode_lib->vba.PixelClockBackEnd[k]
-												* 8.0,
+								TruncToValidBPP((1.0 - mode_lib->vba.Downspreading / 100.0)
+											* (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 540.0
+											* mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
 										true,
 										mode_lib->vba.Output[k],
 										mode_lib->vba.OutputFormat[k],
@@ -5094,40 +5069,26 @@ static void ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 						mode_lib->vba.OutputBppPerState[i][k] =
 								mode_lib->vba.Outbpp;
 					}
-					if (mode_lib->vba.Outbpp == 0
+					if (mode_lib->vba.Outbpp == BPP_INVALID
 							&& mode_lib->vba.PHYCLKPerState[i]
 									>= 810.0) {
 						mode_lib->vba.Outbpp =
-								TruncToValidBPP(
-										(1.0
-												- mode_lib->vba.Downspreading
-														/ 100.0)
-												* 810.0
-												* mode_lib->vba.OutputLinkDPLanes[k]
-												/ mode_lib->vba.PixelClockBackEnd[k]
-												* 8.0,
+								TruncToValidBPP((1.0 - mode_lib->vba.Downspreading / 100.0) * 810.0
+											* mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
 										false,
 										mode_lib->vba.Output[k],
 										mode_lib->vba.OutputFormat[k],
 										mode_lib->vba.DSCInputBitPerComponent[k]);
 						mode_lib->vba.OutbppDSC =
-								TruncToValidBPP(
-										(1.0
-												- mode_lib->vba.Downspreading
-														/ 100.0)
-												* (1.0
-														- mode_lib->vba.EffectiveFECOverhead
-																/ 100.0)
-												* 810.0
-												* mode_lib->vba.OutputLinkDPLanes[k]
-												/ mode_lib->vba.PixelClockBackEnd[k]
-												* 8.0,
+								TruncToValidBPP((1.0 - mode_lib->vba.Downspreading / 100.0)
+											* (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 810.0
+											* mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0,
 										true,
 										mode_lib->vba.Output[k],
 										mode_lib->vba.OutputFormat[k],
 										mode_lib->vba.DSCInputBitPerComponent[k]);
 						if (mode_lib->vba.DSCEnabled[k] == true
-								|| mode_lib->vba.Outbpp == 0) {
+								|| mode_lib->vba.Outbpp == BPP_INVALID) {
 							mode_lib->vba.RequiresDSC[i][k] = true;
 							if (mode_lib->vba.Output[k] == dm_dp) {
 								mode_lib->vba.RequiresFEC[i][k] =
@@ -5147,14 +5108,14 @@ static void ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 					}
 				}
 			} else {
-				mode_lib->vba.OutputBppPerState[i][k] = 0;
+				mode_lib->vba.OutputBppPerState[i][k] = BPP_BLENDED_PIPE;
 			}
 		}
 	}
 	for (i = 0; i <= DC__VOLTAGE_STATES; i++) {
 		mode_lib->vba.DIOSupport[i] = true;
 		for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
-			if (mode_lib->vba.OutputBppPerState[i][k] == 0
+			if (mode_lib->vba.OutputBppPerState[i][k] == BPP_INVALID
 					|| (mode_lib->vba.OutputFormat[k] == dm_420
 							&& mode_lib->vba.ProgressiveToInterlaceUnitInOPP
 									== true)) {
@@ -5243,8 +5204,8 @@ static void ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 			} else {
 				mode_lib->vba.slices = 1.0;
 			}
-			if (mode_lib->vba.OutputBppPerState[i][k] == 0
-					|| mode_lib->vba.OutputBppPerState[i][k] == 0) {
+			if (mode_lib->vba.OutputBppPerState[i][k] == BPP_BLENDED_PIPE
+					|| mode_lib->vba.OutputBppPerState[i][k] == BPP_INVALID) {
 				mode_lib->vba.bpp = 0.0;
 			} else {
 				mode_lib->vba.bpp = mode_lib->vba.OutputBppPerState[i][k];
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_calc.c b/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_calc.c
index 8ba962df42e6..325dd2b757d6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_calc.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_calc.c
@@ -27,6 +27,15 @@
 #include "display_mode_vba.h"
 #include "display_rq_dlg_calc.h"
 
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
 static void calculate_ttu_cursor(struct display_mode_lib *mode_lib,
 		double *refcyc_per_req_delivery_pre_cur,
 		double *refcyc_per_req_delivery_cur,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c
index 1e4b1e383401..c2037daa8e66 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c
@@ -28,6 +28,15 @@
 
 #include "dml_inline_defs.h"
 
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
 static unsigned int get_bytes_per_element(enum source_format_class source_format, bool is_chroma)
 {
 	unsigned int ret_val = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.c b/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.c
index bc7d8c707221..324239c77958 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/soc_bounding_box.c
@@ -27,6 +27,16 @@
 #include "dc_features.h"
 
 #include "dml_inline_defs.h"
+
+/*
+ * NOTE:
+ *   This file is gcc-parseable HW gospel, coming straight from HW engineers.
+ *
+ * It doesn't adhere to Linux kernel style and sometimes will do things in odd
+ * ways. Unless there is something clearly wrong with it the code should
+ * remain as-is as it provides us with a guarantee from HW that it is correct.
+ */
+
 void dml_socbb_set_latencies(soc_bounding_box_st *to_box, soc_bounding_box_st *from_box)
 {
 	to_box->dram_clock_change_latency_us = from_box->dram_clock_change_latency_us;
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_factory_dce120.c b/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_factory_dce120.c
index 4ced9a7d63dd..0c2314efb47e 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_factory_dce120.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_factory_dce120.c
@@ -34,9 +34,9 @@
 
 #include "hw_factory_dce120.h"
 
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15ip.h"
 
 #define block HPD
 #define reg_num 0
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_translate_dce120.c b/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_translate_dce120.c
index af3843a69652..a225b02cc779 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_translate_dce120.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dce120/hw_translate_dce120.c
@@ -33,9 +33,9 @@
 #include "include/gpio_types.h"
 #include "../hw_translate.h"
 
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15ip.h"
 
 /* begin *********************
  * macros to expend register list macro defined in HW object header file */
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_factory_dcn10.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_factory_dcn10.c
index 409763c70ce5..5235f69f0602 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_factory_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_factory_dcn10.c
@@ -34,9 +34,9 @@
 
 #include "hw_factory_dcn10.h"
 
-#include "raven1/DCN/dcn_1_0_offset.h"
-#include "raven1/DCN/dcn_1_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
+#include "soc15ip.h"
 
 #define block HPD
 #define reg_num 0
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c
index 64a6915b846b..347864810d01 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn10/hw_translate_dcn10.c
@@ -33,9 +33,9 @@
 #include "include/gpio_types.h"
 #include "../hw_translate.h"
 
-#include "raven1/DCN/dcn_1_0_offset.h"
-#include "raven1/DCN/dcn_1_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
+#include "soc15ip.h"
 
 /* begin *********************
  * macros to expend register list macro defined in HW object header file */
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c b/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c
index 668981a4c285..a401636bf3f8 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dce120/i2caux_dce120.c
@@ -36,9 +36,9 @@
 #include "../dce110/aux_engine_dce110.h"
 #include "../dce110/i2caux_dce110.h"
 
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15ip.h"
 
 /* begin *********************
  * macros to expend register list macro defined in HW object header file */
diff --git a/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c b/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c
index 13b807d8aff8..bed7cc3e77de 100644
--- a/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/i2caux/dcn10/i2caux_dcn10.c
@@ -36,9 +36,9 @@
 #include "../dce110/i2c_hw_engine_dce110.h"
 #include "../dce110/i2caux_dce110.h"
 
-#include "raven1/DCN/dcn_1_0_offset.h"
-#include "raven1/DCN/dcn_1_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
+#include "soc15ip.h"
 
 /* begin *********************
  * macros to expend register list macro defined in HW object header file */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index b69f321e2ab6..d6971054ec07 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -139,6 +139,7 @@ struct resource_pool {
 	struct timing_generator *timing_generators[MAX_PIPES];
 	struct stream_encoder *stream_enc[MAX_PIPES * 2];
 
+	struct hubbub *hubbub;
 	struct mpc *mpc;
 	struct pp_smu_funcs_rv *pp_smu;
 	struct pp_smu_display_requirement_rv pp_smu_req;
@@ -211,7 +212,6 @@ struct pipe_ctx {
 	struct _vcs_dpi_display_rq_regs_st rq_regs;
 	struct _vcs_dpi_display_pipe_dest_params_st pipe_dlg_param;
 #endif
-	struct dwbc *dwbc;
 };
 
 struct resource_context {
@@ -240,6 +240,7 @@ struct dce_bw_output {
 
 struct dcn_bw_clocks {
 	int dispclk_khz;
+	int dppclk_khz;
 	bool dppclk_div;
 	int dcfclk_khz;
 	int dcfclk_deep_sleep_khz;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
index 1e231f6de732..132d18d4b293 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
@@ -349,10 +349,10 @@ struct dcn_bw_internal_vars {
 	float dst_x_after_scaler;
 	float dst_y_after_scaler;
 	float time_calc;
-	float v_update_offset[number_of_planes_minus_one + 1];
+	float v_update_offset[number_of_planes_minus_one + 1][2];
 	float total_repeater_delay;
-	float v_update_width[number_of_planes_minus_one + 1];
-	float v_ready_offset[number_of_planes_minus_one + 1];
+	float v_update_width[number_of_planes_minus_one + 1][2];
+	float v_ready_offset[number_of_planes_minus_one + 1][2];
 	float time_setup;
 	float extra_latency;
 	float maximum_vstartup;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
index c93b9b9a817c..a83a48494613 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/abm.h
@@ -27,9 +27,19 @@
 
 #include "dm_services_types.h"
 
+struct abm_backlight_registers {
+	unsigned int BL_PWM_CNTL;
+	unsigned int BL_PWM_CNTL2;
+	unsigned int BL_PWM_PERIOD_CNTL;
+	unsigned int LVTMA_PWRSEQ_REF_DIV_BL_PWM_REF_DIV;
+};
+
 struct abm {
 	struct dc_context *ctx;
 	const struct abm_funcs *funcs;
+
+	/* registers setting needs to be saved and restored at InitBacklight */
+	struct abm_backlight_registers stored_backlight_registers;
 };
 
 struct abm_funcs {
@@ -40,9 +50,9 @@ struct abm_funcs {
 	bool (*set_backlight_level)(struct abm *abm,
 			unsigned int backlight_level,
 			unsigned int frame_ramp,
-			unsigned int controller_id);
+			unsigned int controller_id,
+			bool use_smooth_brightness);
 	unsigned int (*get_current_backlight_8_bit)(struct abm *abm);
-	bool (*is_dmcu_initialized)(struct abm *abm);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
index 0574c29cc4a8..ce206355461b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
@@ -27,12 +27,29 @@
 
 #include "dm_services_types.h"
 
+enum dmcu_state {
+	DMCU_NOT_INITIALIZED = 0,
+	DMCU_RUNNING = 1
+};
+
+struct dmcu_version {
+	unsigned int day;
+	unsigned int month;
+	unsigned int year;
+	unsigned int interface_version;
+};
+
 struct dmcu {
 	struct dc_context *ctx;
 	const struct dmcu_funcs *funcs;
+
+	enum dmcu_state dmcu_state;
+	struct dmcu_version dmcu_version;
+	unsigned int cached_wait_loop_number;
 };
 
 struct dmcu_funcs {
+	bool (*dmcu_init)(struct dmcu *dmcu);
 	bool (*load_iram)(struct dmcu *dmcu,
 			unsigned int start_offset,
 			const char *src,
@@ -44,7 +61,9 @@ struct dmcu_funcs {
 	void (*get_psr_state)(struct dmcu *dmcu, uint32_t *psr_state);
 	void (*set_psr_wait_loop)(struct dmcu *dmcu,
 			unsigned int wait_loop_number);
-	void (*get_psr_wait_loop)(unsigned int *psr_wait_loop_number);
+	void (*get_psr_wait_loop)(struct dmcu *dmcu,
+			unsigned int *psr_wait_loop_number);
+	bool (*is_dmcu_initialized)(struct dmcu *dmcu);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
index 9420dfb94d39..25edbde6163e 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h
@@ -62,63 +62,67 @@ struct dpp_funcs {
 			struct dpp *dpp,
 			const struct dpp_grph_csc_adjustment *adjust);
 
-	void (*opp_set_csc_default)(
+	void (*dpp_set_csc_default)(
 		struct dpp *dpp,
 		enum dc_color_space colorspace);
 
-	void (*opp_set_csc_adjustment)(
+	void (*dpp_set_csc_adjustment)(
 		struct dpp *dpp,
-		const struct out_csc_color_matrix *tbl_entry);
+		const uint16_t *regval);
 
-	void (*opp_power_on_regamma_lut)(
+	void (*dpp_power_on_regamma_lut)(
 		struct dpp *dpp,
 		bool power_on);
 
-	void (*opp_program_regamma_lut)(
+	void (*dpp_program_regamma_lut)(
 			struct dpp *dpp,
 			const struct pwl_result_data *rgb,
 			uint32_t num);
 
-	void (*opp_configure_regamma_lut)(
+	void (*dpp_configure_regamma_lut)(
 			struct dpp *dpp,
 			bool is_ram_a);
 
-	void (*opp_program_regamma_lutb_settings)(
+	void (*dpp_program_regamma_lutb_settings)(
 			struct dpp *dpp,
 			const struct pwl_params *params);
 
-	void (*opp_program_regamma_luta_settings)(
+	void (*dpp_program_regamma_luta_settings)(
 			struct dpp *dpp,
 			const struct pwl_params *params);
 
-	void (*opp_program_regamma_pwl)(
-		struct dpp *dpp, const struct pwl_params *params);
+	void (*dpp_program_regamma_pwl)(
+		struct dpp *dpp,
+		const struct pwl_params *params,
+		enum opp_regamma mode);
 
-	void (*opp_set_regamma_mode)(
-			struct dpp *dpp_base,
-			enum opp_regamma mode);
+	void (*dpp_program_bias_and_scale)(
+			struct dpp *dpp,
+			struct dc_bias_and_scale *params);
 
-	void (*ipp_set_degamma)(
+	void (*dpp_set_degamma)(
 			struct dpp *dpp_base,
 			enum ipp_degamma_mode mode);
 
-	void (*ipp_program_input_lut)(
+	void (*dpp_program_input_lut)(
 			struct dpp *dpp_base,
 			const struct dc_gamma *gamma);
 
-	void (*ipp_program_degamma_pwl)(struct dpp *dpp_base,
+	void (*dpp_program_degamma_pwl)(struct dpp *dpp_base,
 									 const struct pwl_params *params);
 
-	void (*ipp_setup)(
+	void (*dpp_setup)(
 			struct dpp *dpp_base,
-			enum surface_pixel_format input_format,
-			enum expansion_mode mode);
+			enum surface_pixel_format format,
+			enum expansion_mode mode,
+			struct csc_transform input_csc_color_matrix,
+			enum dc_color_space input_color_space);
 
-	void (*ipp_full_bypass)(struct dpp *dpp_base);
+	void (*dpp_full_bypass)(struct dpp *dpp_base);
 
 	void (*set_cursor_attributes)(
 			struct dpp *dpp_base,
-			const struct dc_cursor_attributes *attr);
+			enum dc_cursor_color_format color_format);
 
 	void (*set_cursor_position)(
 			struct dpp *dpp_base,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 0d186be24cf4..b7c7e70022e4 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -28,15 +28,32 @@
 
 #include "mem_input.h"
 
+
+enum cursor_pitch {
+	CURSOR_PITCH_64_PIXELS = 0,
+	CURSOR_PITCH_128_PIXELS,
+	CURSOR_PITCH_256_PIXELS
+};
+
+enum cursor_lines_per_chunk {
+	CURSOR_LINE_PER_CHUNK_2 = 1,
+	CURSOR_LINE_PER_CHUNK_4,
+	CURSOR_LINE_PER_CHUNK_8,
+	CURSOR_LINE_PER_CHUNK_16
+};
+
 struct hubp {
 	struct hubp_funcs *funcs;
 	struct dc_context *ctx;
 	struct dc_plane_address request_address;
 	struct dc_plane_address current_address;
 	int inst;
+
+	/* run time states */
 	int opp_id;
 	int mpcc_id;
 	struct dc_cursor_attributes curs_attr;
+	bool power_gated;
 };
 
 
@@ -100,6 +117,8 @@ struct hubp_funcs {
 			const struct dc_cursor_position *pos,
 			const struct dc_cursor_mi_param *param);
 
+	void (*hubp_disconnect)(struct hubp *hubp);
+
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index 9602f261b614..e3f0b4056318 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -73,7 +73,7 @@ struct pwl_result_data {
 
 struct pwl_params {
 	struct gamma_curve arr_curve_points[34];
-	struct curve_points arr_points[3];
+	struct curve_points arr_points[2];
 	struct pwl_result_data rgb_resulted[256 + 3];
 	uint32_t hw_points_num;
 };
@@ -126,11 +126,13 @@ struct default_adjustment {
 	bool force_hw_default;
 };
 
+
 struct out_csc_color_matrix {
 	enum dc_color_space color_space;
 	uint16_t regval[12];
 };
 
+
 enum opp_regamma {
 	OPP_REGAMMA_BYPASS = 0,
 	OPP_REGAMMA_SRGB,
@@ -138,4 +140,55 @@ enum opp_regamma {
 	OPP_REGAMMA_USER
 };
 
+struct csc_transform {
+	uint16_t matrix[12];
+	bool enable_adjustment;
+};
+
+struct dc_bias_and_scale {
+	uint16_t scale_red;
+	uint16_t bias_red;
+	uint16_t scale_green;
+	uint16_t bias_green;
+	uint16_t scale_blue;
+	uint16_t bias_blue;
+};
+
+enum test_pattern_dyn_range {
+	TEST_PATTERN_DYN_RANGE_VESA = 0,
+	TEST_PATTERN_DYN_RANGE_CEA
+};
+
+enum test_pattern_mode {
+	TEST_PATTERN_MODE_COLORSQUARES_RGB = 0,
+	TEST_PATTERN_MODE_COLORSQUARES_YCBCR601,
+	TEST_PATTERN_MODE_COLORSQUARES_YCBCR709,
+	TEST_PATTERN_MODE_VERTICALBARS,
+	TEST_PATTERN_MODE_HORIZONTALBARS,
+	TEST_PATTERN_MODE_SINGLERAMP_RGB,
+	TEST_PATTERN_MODE_DUALRAMP_RGB
+};
+
+enum test_pattern_color_format {
+	TEST_PATTERN_COLOR_FORMAT_BPC_6 = 0,
+	TEST_PATTERN_COLOR_FORMAT_BPC_8,
+	TEST_PATTERN_COLOR_FORMAT_BPC_10,
+	TEST_PATTERN_COLOR_FORMAT_BPC_12
+};
+
+enum controller_dp_test_pattern {
+	CONTROLLER_DP_TEST_PATTERN_D102 = 0,
+	CONTROLLER_DP_TEST_PATTERN_SYMBOLERROR,
+	CONTROLLER_DP_TEST_PATTERN_PRBS7,
+	CONTROLLER_DP_TEST_PATTERN_COLORSQUARES,
+	CONTROLLER_DP_TEST_PATTERN_VERTICALBARS,
+	CONTROLLER_DP_TEST_PATTERN_HORIZONTALBARS,
+	CONTROLLER_DP_TEST_PATTERN_COLORRAMP,
+	CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
+	CONTROLLER_DP_TEST_PATTERN_RESERVED_8,
+	CONTROLLER_DP_TEST_PATTERN_RESERVED_9,
+	CONTROLLER_DP_TEST_PATTERN_RESERVED_A,
+	CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA
+};
+
 #endif /* __DAL_HW_SHARED_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h
index f11aa484f46e..2109eac20a3d 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/ipp.h
@@ -85,8 +85,10 @@ struct ipp_funcs {
 	/* setup ipp to expand/convert input to pixel processor internal format */
 	void (*ipp_setup)(
 		struct input_pixel_processor *ipp,
-		enum surface_pixel_format input_format,
-		enum expansion_mode mode);
+		enum surface_pixel_format format,
+		enum expansion_mode mode,
+		struct csc_transform input_csc_color_matrix,
+		enum dc_color_space input_color_space);
 
 	/* DCE function to setup IPP.  TODO: see if we can consolidate to setup */
 	void (*ipp_program_prescale)(
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
index 498b7f05c5ca..54d8a1386142 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/link_encoder.h
@@ -123,8 +123,7 @@ struct link_encoder_funcs {
 	void (*enable_tmds_output)(struct link_encoder *enc,
 		enum clock_source_id clock_source,
 		enum dc_color_depth color_depth,
-		bool hdmi,
-		bool dual_link,
+		enum signal_type signal,
 		uint32_t pixel_clock);
 	void (*enable_dp_output)(struct link_encoder *enc,
 		const struct dc_link_settings *link_settings,
@@ -133,7 +132,7 @@ struct link_encoder_funcs {
 		const struct dc_link_settings *link_settings,
 		enum clock_source_id clock_source);
 	void (*disable_output)(struct link_encoder *link_enc,
-		enum signal_type signal, struct dc_link *link);
+		enum signal_type signal);
 	void (*dp_set_lane_settings)(struct link_encoder *enc,
 		const struct link_training_settings *link_settings);
 	void (*dp_set_phy_pattern)(struct link_encoder *enc,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
index d4188b2c0626..23a8d5e53a89 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
@@ -26,35 +26,162 @@
 #define __DC_MPCC_H__
 
 #include "dc_hw_types.h"
-#include "opp.h"
+#include "hw_shared.h"
 
-struct mpcc_cfg {
-	int dpp_id;
-	int opp_id;
-	struct mpc_tree_cfg *tree_cfg;
-	unsigned int z_index;
+#define MAX_MPCC 6
+#define MAX_OPP 6
 
-	struct tg_color black_color;
-	bool per_pixel_alpha;
-	bool pre_multiplied_alpha;
+enum mpc_output_csc_mode {
+	MPC_OUTPUT_CSC_DISABLE = 0,
+	MPC_OUTPUT_CSC_COEF_A,
+	MPC_OUTPUT_CSC_COEF_B
+};
+
+
+enum mpcc_blend_mode {
+	MPCC_BLEND_MODE_BYPASS,
+	MPCC_BLEND_MODE_TOP_LAYER_PASSTHROUGH,
+	MPCC_BLEND_MODE_TOP_LAYER_ONLY,
+	MPCC_BLEND_MODE_TOP_BOT_BLENDING
+};
+
+enum mpcc_alpha_blend_mode {
+	MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA,
+	MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA_COMBINED_GLOBAL_GAIN,
+	MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA
+};
+
+/*
+ * MPCC blending configuration
+ */
+struct mpcc_blnd_cfg {
+	struct tg_color black_color;	/* background color */
+	enum mpcc_alpha_blend_mode alpha_mode;	/* alpha blend mode */
+	bool pre_multiplied_alpha;	/* alpha pre-multiplied mode flag */
+	int global_gain;
+	int global_alpha;
+	bool overlap_only;
+
+};
+
+struct mpcc_sm_cfg {
+	bool enable;
+	/* 0-single plane,2-row subsampling,4-column subsampling,6-checkboard subsampling */
+	int sm_mode;
+	/* 0- disable frame alternate, 1- enable frame alternate */
+	bool frame_alt;
+	/* 0- disable field alternate, 1- enable field alternate */
+	bool field_alt;
+	/* 0-no force,2-force frame polarity from top,3-force frame polarity from bottom */
+	int force_next_frame_porlarity;
+	/* 0-no force,2-force field polarity from top,3-force field polarity from bottom */
+	int force_next_field_polarity;
+};
+
+/*
+ * MPCC connection and blending configuration for a single MPCC instance.
+ * This struct is used as a node in an MPC tree.
+ */
+struct mpcc {
+	int mpcc_id;			/* MPCC physical instance */
+	int dpp_id;			/* DPP input to this MPCC */
+	struct mpcc *mpcc_bot;		/* pointer to bottom layer MPCC.  NULL when not connected */
+	struct mpcc_blnd_cfg blnd_cfg;	/* The blending configuration for this MPCC */
+	struct mpcc_sm_cfg sm_cfg;	/* stereo mix setting for this MPCC */
+};
+
+/*
+ * MPC tree represents all MPCC connections for a pipe.
+ */
+struct mpc_tree {
+	int opp_id;			/* The OPP instance that owns this MPC tree */
+	struct mpcc *opp_list;		/* The top MPCC layer of the MPC tree that outputs to OPP endpoint */
 };
 
 struct mpc {
 	const struct mpc_funcs *funcs;
 	struct dc_context *ctx;
+
+	struct mpcc mpcc_array[MAX_MPCC];
 };
 
 struct mpc_funcs {
-	int (*add)(struct mpc *mpc, struct mpcc_cfg *cfg);
+	/*
+	 * Insert DPP into MPC tree based on specified blending position.
+	 * Only used for planes that are part of blending chain for OPP output
+	 *
+	 * Parameters:
+	 * [in/out] mpc		- MPC context.
+	 * [in/out] tree	- MPC tree structure that plane will be added to.
+	 * [in]	blnd_cfg	- MPCC blending configuration for the new blending layer.
+	 * [in]	sm_cfg		- MPCC stereo mix configuration for the new blending layer.
+	 *			  stereo mix must disable for the very bottom layer of the tree config.
+	 * [in]	insert_above_mpcc - Insert new plane above this MPCC.  If NULL, insert as bottom plane.
+	 * [in]	dpp_id		 - DPP instance for the plane to be added.
+	 * [in]	mpcc_id		 - The MPCC physical instance to use for blending.
+	 *
+	 * Return:  struct mpcc* - MPCC that was added.
+	 */
+	struct mpcc* (*insert_plane)(
+			struct mpc *mpc,
+			struct mpc_tree *tree,
+			struct mpcc_blnd_cfg *blnd_cfg,
+			struct mpcc_sm_cfg *sm_cfg,
+			struct mpcc *insert_above_mpcc,
+			int dpp_id,
+			int mpcc_id);
+
+	/*
+	 * Remove a specified MPCC from the MPC tree.
+	 *
+	 * Parameters:
+	 * [in/out] mpc		- MPC context.
+	 * [in/out] tree	- MPC tree structure that plane will be removed from.
+	 * [in/out] mpcc	- MPCC to be removed from tree.
+	 *
+	 * Return:  void
+	 */
+	void (*remove_mpcc)(
+			struct mpc *mpc,
+			struct mpc_tree *tree,
+			struct mpcc *mpcc);
+
+	/*
+	 * Reset the MPCC HW status by disconnecting all muxes.
+	 *
+	 * Parameters:
+	 * [in/out] mpc		- MPC context.
+	 *
+	 * Return:  void
+	 */
+	void (*mpc_init)(struct mpc *mpc);
 
-	void (*remove)(struct mpc *mpc,
-			struct mpc_tree_cfg *tree_cfg,
-			int opp_id,
-			int mpcc_inst);
+	/*
+	 * Update the blending configuration for a specified MPCC.
+	 *
+	 * Parameters:
+	 * [in/out] mpc		- MPC context.
+	 * [in]     blnd_cfg	- MPCC blending configuration.
+	 * [in]     mpcc_id	- The MPCC physical instance.
+	 *
+	 * Return:  void
+	 */
+	void (*update_blending)(
+		struct mpc *mpc,
+		struct mpcc_blnd_cfg *blnd_cfg,
+		int mpcc_id);
+
+	struct mpcc* (*get_mpcc_for_dpp)(
+			struct mpc_tree *tree,
+			int dpp_id);
 
 	void (*wait_for_idle)(struct mpc *mpc, int id);
 
-	void (*update_blend_mode)(struct mpc *mpc, struct mpcc_cfg *cfg);
+	void (*assert_mpcc_idle_before_connect)(struct mpc *mpc, int mpcc_id);
+
+	void (*init_mpcc_list_from_hw)(
+		struct mpc *mpc,
+		struct mpc_tree *tree);
 
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
index 75adb8fec551..ab8fb77f1ae5 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
@@ -29,6 +29,7 @@
 #include "hw_shared.h"
 #include "dc_hw_types.h"
 #include "transform.h"
+#include "mpc.h"
 
 struct fixed31_32;
 
@@ -204,7 +205,7 @@ struct output_pixel_processor {
 	struct dc_context *ctx;
 	uint32_t inst;
 	struct pwl_params regamma_params;
-	struct mpc_tree_cfg mpc_tree;
+	struct mpc_tree mpc_tree_params;
 	bool mpcc_disconnect_pending[MAX_PIPES];
 	const struct opp_funcs *funcs;
 };
@@ -248,6 +249,21 @@ enum ovl_csc_adjust_item {
 	OVERLAY_COLOR_TEMPERATURE
 };
 
+enum oppbuf_display_segmentation {
+	OPPBUF_DISPLAY_SEGMENTATION_1_SEGMENT = 0,
+	OPPBUF_DISPLAY_SEGMENTATION_2_SEGMENT = 1,
+	OPPBUF_DISPLAY_SEGMENTATION_4_SEGMENT = 2,
+	OPPBUF_DISPLAY_SEGMENTATION_4_SEGMENT_SPLIT_LEFT = 3,
+	OPPBUF_DISPLAY_SEGMENTATION_4_SEGMENT_SPLIT_RIGHT = 4
+};
+
+struct oppbuf_params {
+	uint32_t active_width;
+	enum oppbuf_display_segmentation mso_segmentation;
+	uint32_t mso_overlap_pixel_num;
+	uint32_t pixel_repetition;
+};
+
 struct opp_funcs {
 
 
@@ -276,14 +292,11 @@ struct opp_funcs {
 
 	void (*opp_destroy)(struct output_pixel_processor **opp);
 
-	void (*opp_set_stereo_polarity)(
-			struct output_pixel_processor *opp,
-			bool enable,
-			bool rightEyePolarity);
+	void (*opp_program_stereo)(
+		struct output_pixel_processor *opp,
+		bool enable,
+		const struct dc_crtc_timing *timing);
 
-	void (*opp_set_test_pattern)(
-			struct output_pixel_processor *opp,
-			bool enable);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index c6ab38c5b2be..ec312f1a3e55 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -26,6 +26,8 @@
 #ifndef __DAL_TIMING_GENERATOR_TYPES_H__
 #define __DAL_TIMING_GENERATOR_TYPES_H__
 
+#include "hw_shared.h"
+
 struct dc_bios;
 
 /* Contains CRTC vertical/horizontal pixel counters */
@@ -40,6 +42,19 @@ struct dcp_gsl_params {
 	int gsl_master;
 };
 
+struct gsl_params {
+	int gsl0_en;
+	int gsl1_en;
+	int gsl2_en;
+	int gsl_master_en;
+	int gsl_master_mode;
+	int master_update_lock_gsl_en;
+	int gsl_window_start_x;
+	int gsl_window_end_x;
+	int gsl_window_start_y;
+	int gsl_window_end_y;
+};
+
 /* define the structure of Dynamic Refresh Mode */
 struct drr_params {
 	uint32_t vertical_total_min;
@@ -50,43 +65,6 @@ struct drr_params {
 #define LEFT_EYE_3D_PRIMARY_SURFACE 1
 #define RIGHT_EYE_3D_PRIMARY_SURFACE 0
 
-enum test_pattern_dyn_range {
-	TEST_PATTERN_DYN_RANGE_VESA = 0,
-	TEST_PATTERN_DYN_RANGE_CEA
-};
-
-enum test_pattern_mode {
-	TEST_PATTERN_MODE_COLORSQUARES_RGB = 0,
-	TEST_PATTERN_MODE_COLORSQUARES_YCBCR601,
-	TEST_PATTERN_MODE_COLORSQUARES_YCBCR709,
-	TEST_PATTERN_MODE_VERTICALBARS,
-	TEST_PATTERN_MODE_HORIZONTALBARS,
-	TEST_PATTERN_MODE_SINGLERAMP_RGB,
-	TEST_PATTERN_MODE_DUALRAMP_RGB
-};
-
-enum test_pattern_color_format {
-	TEST_PATTERN_COLOR_FORMAT_BPC_6 = 0,
-	TEST_PATTERN_COLOR_FORMAT_BPC_8,
-	TEST_PATTERN_COLOR_FORMAT_BPC_10,
-	TEST_PATTERN_COLOR_FORMAT_BPC_12
-};
-
-enum controller_dp_test_pattern {
-	CONTROLLER_DP_TEST_PATTERN_D102 = 0,
-	CONTROLLER_DP_TEST_PATTERN_SYMBOLERROR,
-	CONTROLLER_DP_TEST_PATTERN_PRBS7,
-	CONTROLLER_DP_TEST_PATTERN_COLORSQUARES,
-	CONTROLLER_DP_TEST_PATTERN_VERTICALBARS,
-	CONTROLLER_DP_TEST_PATTERN_HORIZONTALBARS,
-	CONTROLLER_DP_TEST_PATTERN_COLORRAMP,
-	CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
-	CONTROLLER_DP_TEST_PATTERN_RESERVED_8,
-	CONTROLLER_DP_TEST_PATTERN_RESERVED_9,
-	CONTROLLER_DP_TEST_PATTERN_RESERVED_A,
-	CONTROLLER_DP_TEST_PATTERN_COLORSQUARES_CEA
-};
-
 enum crtc_state {
 	CRTC_STATE_VBLANK = 0,
 	CRTC_STATE_VACTIVE
@@ -100,6 +78,12 @@ struct _dlg_otg_param {
 	enum signal_type signal;
 };
 
+struct vupdate_keepout_params {
+	int start_offset;
+	int end_offset;
+	int enable;
+};
+
 struct crtc_stereo_flags {
 	uint8_t PROGRAM_STEREO         : 1;
 	uint8_t PROGRAM_POLARITY       : 1;
@@ -158,7 +142,11 @@ struct timing_generator_funcs {
 							const struct dcp_gsl_params *gsl_params);
 	void (*unlock)(struct timing_generator *tg);
 	void (*lock)(struct timing_generator *tg);
-	void (*enable_reset_trigger)(struct timing_generator *tg, int source_tg_inst);
+	void (*enable_reset_trigger)(struct timing_generator *tg,
+				     int source_tg_inst);
+	void (*enable_crtc_reset)(struct timing_generator *tg,
+				  int source_tg_inst,
+				  struct crtc_trigger_info *crtc_tp);
 	void (*disable_reset_trigger)(struct timing_generator *tg);
 	void (*tear_down_global_swap_lock)(struct timing_generator *tg);
 	void (*enable_advanced_request)(struct timing_generator *tg,
@@ -178,6 +166,13 @@ struct timing_generator_funcs {
 	void (*program_stereo)(struct timing_generator *tg,
 		const struct dc_crtc_timing *timing, struct crtc_stereo_flags *flags);
 	bool (*is_stereo_left_eye)(struct timing_generator *tg);
+
+	void (*set_blank_data_double_buffer)(struct timing_generator *tg, bool enable);
+
+	void (*tg_init)(struct timing_generator *tg);
+	bool (*is_tg_enabled)(struct timing_generator *tg);
+	bool (*is_optc_underflow_occurred)(struct timing_generator *tg);
+	void (*clear_optc_underflow)(struct timing_generator *tg);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h b/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h
index ea88997e1bbd..6f6c02b89f90 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/transform.h
@@ -250,8 +250,10 @@ struct transform_funcs {
 
 	void (*ipp_setup)(
 			struct transform *xfm_base,
-			enum surface_pixel_format input_format,
-			enum expansion_mode mode);
+			enum surface_pixel_format format,
+			enum expansion_mode mode,
+			struct csc_transform input_csc_color_matrix,
+			enum dc_color_space input_color_space);
 
 	void (*ipp_full_bypass)(struct transform *xfm_base);
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
index 8734689a9245..379c6ecd271a 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
@@ -28,6 +28,7 @@
 #include "dc_types.h"
 #include "clock_source.h"
 #include "inc/hw/timing_generator.h"
+#include "inc/hw/opp.h"
 #include "inc/hw/link_encoder.h"
 #include "core_status.h"
 
@@ -39,6 +40,12 @@ enum pipe_gating_control {
 
 struct dce_hwseq_wa {
 	bool blnd_crtc_trigger;
+	bool DEGVIDCN10_253;
+	bool false_optc_underflow;
+};
+
+struct hwseq_wa_state {
+	bool DEGVIDCN10_253_applied;
 };
 
 struct dce_hwseq {
@@ -47,6 +54,7 @@ struct dce_hwseq {
 	const struct dce_hwseq_shift *shifts;
 	const struct dce_hwseq_mask *masks;
 	struct dce_hwseq_wa wa;
+	struct hwseq_wa_state wa_state;
 };
 
 struct pipe_ctx;
@@ -114,6 +122,11 @@ struct hw_sequencer_funcs {
 			int group_size,
 			struct pipe_ctx *grouped_pipes[]);
 
+	void (*enable_per_frame_crtc_position_reset)(
+			struct dc *dc,
+			int group_size,
+			struct pipe_ctx *grouped_pipes[]);
+
 	void (*enable_display_pipe_clock_gating)(
 					struct dc_context *ctx,
 					bool clock_gating);
@@ -124,11 +137,7 @@ struct hw_sequencer_funcs {
 					struct dc_bios *dcb,
 					enum pipe_gating_control power_gating);
 
-	void (*power_down_front_end)(struct dc *dc, int fe_idx);
-
-	void (*power_on_front_end)(struct dc *dc,
-			struct pipe_ctx *pipe,
-			struct dc_state *context);
+	void (*disable_plane)(struct dc *dc, struct pipe_ctx *pipe_ctx);
 
 	void (*update_info_frame)(struct pipe_ctx *pipe_ctx);
 
@@ -178,12 +187,20 @@ struct hw_sequencer_funcs {
 
 	void (*ready_shared_resources)(struct dc *dc, struct dc_state *context);
 	void (*optimize_shared_resources)(struct dc *dc);
+	void (*pplib_apply_display_requirements)(
+			struct dc *dc,
+			struct dc_state *context);
 	void (*edp_power_control)(
-			struct link_encoder *enc,
+			struct dc_link *link,
 			bool enable);
 	void (*edp_backlight_control)(
 			struct dc_link *link,
 			bool enable);
+	void (*edp_wait_for_hpd_ready)(struct dc_link *link, bool power_up);
+
+	void (*set_cursor_position)(struct pipe_ctx *pipe);
+	void (*set_cursor_attribute)(struct pipe_ctx *pipe);
+
 };
 
 void color_space_to_black_color(
@@ -194,4 +211,8 @@ void color_space_to_black_color(
 bool hwss_wait_for_blank_complete(
 		struct timing_generator *tg);
 
+const uint16_t *find_color_matrix(
+		enum dc_color_space color_space,
+		uint32_t *array_size);
+
 #endif /* __DC_HW_SEQUENCER_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
index f7e40b292dfb..d3e1923b01a8 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
@@ -217,7 +217,7 @@ bool dce110_vblank_set(
 			core_dc->current_state->res_ctx.pipe_ctx[pipe_offset].stream_res.tg;
 
 	if (enable) {
-		if (!tg->funcs->arm_vert_intr(tg, 2)) {
+		if (!tg || !tg->funcs->arm_vert_intr(tg, 2)) {
 			DC_ERROR("Failed to get VBLANK!\n");
 			return false;
 		}
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c b/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
index 2ad56b1a4099..66d52580e29f 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
@@ -30,9 +30,9 @@
 #include "irq_service_dce120.h"
 #include "../dce110/irq_service_dce110.h"
 
-#include "vega10/DC/dce_12_0_offset.h"
-#include "vega10/DC/dce_12_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dce/dce_12_0_offset.h"
+#include "dce/dce_12_0_sh_mask.h"
+#include "soc15ip.h"
 
 #include "ivsrcid/ivsrcid_vislands30.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
index 74ad24714f6b..7f7db66c48b0 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
@@ -29,9 +29,9 @@
 
 #include "../dce110/irq_service_dce110.h"
 
-#include "raven1/DCN/dcn_1_0_offset.h"
-#include "raven1/DCN/dcn_1_0_sh_mask.h"
-#include "vega10/soc15ip.h"
+#include "dcn/dcn_1_0_offset.h"
+#include "dcn/dcn_1_0_sh_mask.h"
+#include "soc15ip.h"
 
 #include "irq_service_dcn10.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/os_types.h b/drivers/gpu/drm/amd/display/dc/os_types.h
index a87c0329541f..1fcbc99e63b5 100644
--- a/drivers/gpu/drm/amd/display/dc/os_types.h
+++ b/drivers/gpu/drm/amd/display/dc/os_types.h
@@ -26,8 +26,6 @@
 #ifndef _OS_TYPES_H_
 #define _OS_TYPES_H_
 
-#if defined __KERNEL__
-
 #include <asm/byteorder.h>
 #include <linux/types.h>
 #include <drm/drmP.h>
@@ -46,14 +44,12 @@
 #undef WRITE
 #undef FRAME_SIZE
 
-#define dm_output_to_console(fmt, ...) DRM_INFO(fmt, ##__VA_ARGS__)
+#define dm_output_to_console(fmt, ...) DRM_DEBUG_KMS(fmt, ##__VA_ARGS__)
 
 #define dm_error(fmt, ...) DRM_ERROR(fmt, ##__VA_ARGS__)
 
-#define dm_debug(fmt, ...) DRM_DEBUG_KMS(fmt, ##__VA_ARGS__)
-
-#define dm_vlog(fmt, args) vprintk(fmt, args)
-
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+#include <asm/fpu/api.h>
 #endif
 
 /*
@@ -89,8 +85,4 @@
 	BREAK_TO_DEBUGGER(); \
 } while (0)
 
-#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
-#include <asm/fpu/api.h>
-#endif
-
 #endif /* _OS_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c
index 88c2bde3f039..1c079ba37c30 100644
--- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_encoder.c
@@ -42,8 +42,7 @@ static void virtual_link_encoder_enable_tmds_output(
 	struct link_encoder *enc,
 	enum clock_source_id clock_source,
 	enum dc_color_depth color_depth,
-	bool hdmi,
-	bool dual_link,
+	enum signal_type signal,
 	uint32_t pixel_clock) {}
 
 static void virtual_link_encoder_enable_dp_output(
@@ -58,8 +57,7 @@ static void virtual_link_encoder_enable_dp_mst_output(
 
 static void virtual_link_encoder_disable_output(
 	struct link_encoder *link_enc,
-	enum signal_type signal,
-	struct dc_link *link) {}
+	enum signal_type signal) {}
 
 static void virtual_link_encoder_dp_set_lane_settings(
 	struct link_encoder *enc,
diff --git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
index 0ff2a899b8f7..019e7a095ea1 100644
--- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
@@ -27,12 +27,8 @@
 
 #define DP_BRANCH_DEVICE_ID_1 0x0010FA
 #define DP_BRANCH_DEVICE_ID_2 0x0022B9
-#define DP_SINK_DEVICE_ID_1 0x4CE000
 #define DP_BRANCH_DEVICE_ID_3 0x00001A
 #define DP_BRANCH_DEVICE_ID_4 0x0080e1
-#define DP_BRANCH_DEVICE_ID_5 0x006037
-#define DP_SINK_DEVICE_ID_2 0x001CF8
-
 
 enum ddc_result {
 	DDC_RESULT_UNKNOWN = 0,
@@ -115,40 +111,11 @@ struct av_sync_data {
 	uint8_t aud_del_ins3;/* DPCD 0002Dh */
 };
 
-/*DP to VGA converter*/
-static const uint8_t DP_VGA_CONVERTER_ID_1[] = "mVGAa";
-/*DP to Dual link DVI converter*/
-static const uint8_t DP_DVI_CONVERTER_ID_1[] = "m2DVIa";
 /*Travis*/
 static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
 /*Nutmeg*/
 static const uint8_t DP_VGA_LVDS_CONVERTER_ID_3[] = "dnomlA";
-/*DP to VGA converter*/
-static const uint8_t DP_VGA_CONVERTER_ID_4[] = "DpVga";
 /*DP to Dual link DVI converter*/
 static const uint8_t DP_DVI_CONVERTER_ID_4[] = "m2DVIa";
-/*DP to Dual link DVI converter 2*/
-static const uint8_t DP_DVI_CONVERTER_ID_42[] = "v2DVIa";
-
-static const uint8_t DP_SINK_DEV_STRING_ID2_REV0[] = "\0\0\0\0\0\0";
-
-/* Identifies second generation PSR TCON from Parade: Device ID string:
- * yy-xx-**-**-**-**
- */
-/* xx - Hw ID high byte */
-static const uint32_t DP_SINK_DEV_STRING_ID2_REV1_HW_ID_HIGH_BYTE =
-	0x06;
-
-/* yy - HW ID low byte, the same silicon has several package/feature flavors */
-static const uint32_t DP_SINK_DEV_STRING_ID2_REV1_HW_ID_LOW_BYTE1 =
-	0x61;
-static const uint32_t DP_SINK_DEV_STRING_ID2_REV1_HW_ID_LOW_BYTE2 =
-	0x62;
-static const uint32_t DP_SINK_DEV_STRING_ID2_REV1_HW_ID_LOW_BYTE3 =
-	0x63;
-static const uint32_t DP_SINK_DEV_STRING_ID2_REV1_HW_ID_LOW_BYTE4 =
-	0x72;
-static const uint32_t DP_SINK_DEV_STRING_ID2_REV1_HW_ID_LOW_BYTE5 =
-	0x73;
 
 #endif /* __DAL_DDC_SERVICE_TYPES_H__ */
diff --git a/drivers/gpu/drm/amd/display/include/fixed31_32.h b/drivers/gpu/drm/amd/display/include/fixed31_32.h
index 3248f699daf2..4badaedbaadd 100644
--- a/drivers/gpu/drm/amd/display/include/fixed31_32.h
+++ b/drivers/gpu/drm/amd/display/include/fixed31_32.h
@@ -463,4 +463,11 @@ uint32_t dal_fixed31_32_u2d19(
 uint32_t dal_fixed31_32_u0d19(
 	struct fixed31_32 arg);
 
+
+uint32_t dal_fixed31_32_clamp_u0d14(
+	struct fixed31_32 arg);
+
+uint32_t dal_fixed31_32_clamp_u0d10(
+	struct fixed31_32 arg);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
index 7a9b43f84a31..36bbad594267 100644
--- a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
+++ b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h
@@ -419,11 +419,6 @@ struct bios_event_info {
 	bool backlight_changed;
 };
 
-enum {
-	HDMI_PIXEL_CLOCK_IN_KHZ_297 = 297000,
-	TMDS_PIXEL_CLOCK_IN_KHZ_165 = 165000
-};
-
 /*
  * DFS-bypass flag
  */
diff --git a/drivers/gpu/drm/amd/display/include/grph_object_id.h b/drivers/gpu/drm/amd/display/include/grph_object_id.h
index 5eb2b4dc7b9c..c4197432eb7c 100644
--- a/drivers/gpu/drm/amd/display/include/grph_object_id.h
+++ b/drivers/gpu/drm/amd/display/include/grph_object_id.h
@@ -233,10 +233,6 @@ static inline struct graphics_object_id dal_graphics_object_id_init(
 	return result;
 }
 
-bool dal_graphics_object_id_is_equal(
-	struct graphics_object_id id1,
-	struct graphics_object_id id2);
-
 /* Based on internal data members memory layout */
 static inline uint32_t dal_graphics_object_id_to_uint(
 	struct graphics_object_id id)
@@ -248,7 +244,7 @@ static inline enum controller_id dal_graphics_object_id_get_controller_id(
 	struct graphics_object_id id)
 {
 	if (id.type == OBJECT_TYPE_CONTROLLER)
-		return id.id;
+		return (enum controller_id) id.id;
 	return CONTROLLER_ID_UNDEFINED;
 }
 
@@ -256,7 +252,7 @@ static inline enum clock_source_id dal_graphics_object_id_get_clock_source_id(
 	struct graphics_object_id id)
 {
 	if (id.type == OBJECT_TYPE_CLOCK_SOURCE)
-		return id.id;
+		return (enum clock_source_id) id.id;
 	return CLOCK_SOURCE_ID_UNDEFINED;
 }
 
@@ -264,7 +260,7 @@ static inline enum encoder_id dal_graphics_object_id_get_encoder_id(
 	struct graphics_object_id id)
 {
 	if (id.type == OBJECT_TYPE_ENCODER)
-		return id.id;
+		return (enum encoder_id) id.id;
 	return ENCODER_ID_UNKNOWN;
 }
 
@@ -272,7 +268,7 @@ static inline enum connector_id dal_graphics_object_id_get_connector_id(
 	struct graphics_object_id id)
 {
 	if (id.type == OBJECT_TYPE_CONNECTOR)
-		return id.id;
+		return (enum connector_id) id.id;
 	return CONNECTOR_ID_UNKNOWN;
 }
 
@@ -280,7 +276,7 @@ static inline enum audio_id dal_graphics_object_id_get_audio_id(
 	struct graphics_object_id id)
 {
 	if (id.type == OBJECT_TYPE_AUDIO)
-		return id.id;
+		return (enum audio_id) id.id;
 	return AUDIO_ID_UNKNOWN;
 }
 
@@ -288,7 +284,7 @@ static inline enum engine_id dal_graphics_object_id_get_engine_id(
 	struct graphics_object_id id)
 {
 	if (id.type == OBJECT_TYPE_ENGINE)
-		return id.id;
+		return (enum engine_id) id.id;
 	return ENGINE_ID_UNKNOWN;
 }
 #endif
diff --git a/drivers/gpu/drm/amd/display/include/logger_interface.h b/drivers/gpu/drm/amd/display/include/logger_interface.h
index 8e1fe70097be..28dee960d509 100644
--- a/drivers/gpu/drm/amd/display/include/logger_interface.h
+++ b/drivers/gpu/drm/amd/display/include/logger_interface.h
@@ -57,6 +57,11 @@ void dm_logger_append(
 		const char *msg,
 		...);
 
+void dm_logger_append_va(
+		struct log_entry *entry,
+		const char *msg,
+		va_list args);
+
 void dm_logger_open(
 		struct dal_logger *logger,
 		struct log_entry *entry,
diff --git a/drivers/gpu/drm/amd/display/include/signal_types.h b/drivers/gpu/drm/amd/display/include/signal_types.h
index b5ebde642207..199c5db67cbc 100644
--- a/drivers/gpu/drm/amd/display/include/signal_types.h
+++ b/drivers/gpu/drm/amd/display/include/signal_types.h
@@ -26,6 +26,11 @@
 #ifndef __DC_SIGNAL_TYPES_H__
 #define __DC_SIGNAL_TYPES_H__
 
+/* Minimum pixel clock, in KHz. For TMDS signal is 25.00 MHz */
+#define TMDS_MIN_PIXEL_CLOCK 25000
+/* Maximum pixel clock, in KHz. For TMDS signal is 165.00 MHz */
+#define TMDS_MAX_PIXEL_CLOCK 165000
+
 enum signal_type {
 	SIGNAL_TYPE_NONE		= 0L,		/* no signal */
 	SIGNAL_TYPE_DVI_SINGLE_LINK	= (1 << 0),
diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
index 4d7db4aa28e0..b4723af368a5 100644
--- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
+++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
@@ -132,14 +132,6 @@ struct core_freesync {
 #define MOD_FREESYNC_TO_CORE(mod_freesync)\
 		container_of(mod_freesync, struct core_freesync, public)
 
-static bool check_dc_support(const struct dc *dc)
-{
-	if (dc->stream_funcs.adjust_vmin_vmax == NULL)
-		return false;
-
-	return true;
-}
-
 struct mod_freesync *mod_freesync_create(struct dc *dc)
 {
 	struct core_freesync *core_freesync =
@@ -169,9 +161,6 @@ struct mod_freesync *mod_freesync_create(struct dc *dc)
 
 	core_freesync->dc = dc;
 
-	if (!check_dc_support(dc))
-		goto fail_construct;
-
 	/* Create initial module folder in registry for freesync enable data */
 	flag.save_per_edid = true;
 	flag.save_per_link = false;
@@ -599,10 +588,9 @@ static bool set_freesync_on_streams(struct core_freesync *core_freesync,
 				update_stream_freesync_context(core_freesync,
 						streams[stream_idx]);
 
-				core_freesync->dc->stream_funcs.
-				adjust_vmin_vmax(core_freesync->dc, streams,
-						num_streams, v_total_min,
-						v_total_max);
+				dc_stream_adjust_vmin_vmax(core_freesync->dc, streams,
+							   num_streams, v_total_min,
+							   v_total_max);
 
 				return true;
 
@@ -625,8 +613,7 @@ static bool set_freesync_on_streams(struct core_freesync *core_freesync,
 						core_freesync,
 						streams[stream_idx]);
 
-					core_freesync->dc->stream_funcs.
-					adjust_vmin_vmax(
+					dc_stream_adjust_vmin_vmax(
 						core_freesync->dc, streams,
 						num_streams, v_total_nominal,
 						v_total_nominal);
@@ -645,11 +632,9 @@ static bool set_freesync_on_streams(struct core_freesync *core_freesync,
 					core_freesync,
 					streams[stream_idx]);
 
-				core_freesync->dc->stream_funcs.
-						adjust_vmin_vmax(
-						core_freesync->dc, streams,
-						num_streams, v_total_nominal,
-						v_total_nominal);
+				dc_stream_adjust_vmin_vmax(core_freesync->dc, streams,
+							   num_streams, v_total_nominal,
+							   v_total_nominal);
 
 				/* Reset the cached variables */
 				reset_freesync_state_variables(state);
@@ -665,11 +650,9 @@ static bool set_freesync_on_streams(struct core_freesync *core_freesync,
 			 * not support freesync because a former stream has
 			 * be programmed
 			 */
-			core_freesync->dc->stream_funcs.
-					adjust_vmin_vmax(
-					core_freesync->dc, streams,
-					num_streams, v_total_nominal,
-					v_total_nominal);
+			dc_stream_adjust_vmin_vmax(core_freesync->dc, streams,
+						   num_streams, v_total_nominal,
+						   v_total_nominal);
 			/* Reset the cached variables */
 			reset_freesync_state_variables(state);
 		}
@@ -786,9 +769,8 @@ void mod_freesync_handle_v_update(struct mod_freesync *mod_freesync,
 			vmin = inserted_frame_v_total;
 
 			/* Program V_TOTAL */
-			core_freesync->dc->stream_funcs.adjust_vmin_vmax(
-				core_freesync->dc, streams,
-				num_streams, vmin, vmax);
+			dc_stream_adjust_vmin_vmax(core_freesync->dc, streams,
+						   num_streams, vmin, vmax);
 		}
 
 		if (state->btr.frame_counter > 0)
@@ -822,17 +804,15 @@ void mod_freesync_handle_v_update(struct mod_freesync *mod_freesync,
 		update_stream_freesync_context(core_freesync, streams[0]);
 
 		/* Program static screen ramp values */
-		core_freesync->dc->stream_funcs.adjust_vmin_vmax(
-					core_freesync->dc, streams,
-					num_streams, v_total,
-					v_total);
+		dc_stream_adjust_vmin_vmax(core_freesync->dc, streams,
+					   num_streams, v_total,
+					   v_total);
 
 		triggers.overlay_update = true;
 		triggers.surface_update = true;
 
-		core_freesync->dc->stream_funcs.set_static_screen_events(
-					core_freesync->dc, streams,	num_streams,
-					&triggers);
+		dc_stream_set_static_screen_events(core_freesync->dc, streams,
+						   num_streams, &triggers);
 	}
 }
 
@@ -916,9 +896,8 @@ void mod_freesync_update_state(struct mod_freesync *mod_freesync,
 	triggers.overlay_update = true;
 	triggers.surface_update = true;
 
-	core_freesync->dc->stream_funcs.set_static_screen_events(
-		core_freesync->dc, streams, num_streams,
-		&triggers);
+	dc_stream_set_static_screen_events(core_freesync->dc, streams,
+					   num_streams, &triggers);
 
 	if (freesync_program_required)
 		/* Program freesync according to current state*/
@@ -1084,10 +1063,9 @@ bool mod_freesync_override_min_max(struct mod_freesync *mod_freesync,
 				max_refresh);
 
 		/* Program vtotal min/max */
-		core_freesync->dc->stream_funcs.adjust_vmin_vmax(
-			core_freesync->dc, &streams, 1,
-			state->freesync_range.vmin,
-			state->freesync_range.vmax);
+		dc_stream_adjust_vmin_vmax(core_freesync->dc, &streams, 1,
+					   state->freesync_range.vmin,
+					   state->freesync_range.vmax);
 	}
 
 	if (min_refresh != 0 &&
@@ -1163,9 +1141,9 @@ bool mod_freesync_get_v_position(struct mod_freesync *mod_freesync,
 	core_freesync = MOD_FREESYNC_TO_CORE(mod_freesync);
 	index = map_index_from_stream(core_freesync, stream);
 
-	if (core_freesync->dc->stream_funcs.get_crtc_position(
-			core_freesync->dc, &stream, 1,
-			&position.vertical_count, &position.nominal_vcount)) {
+	if (dc_stream_get_crtc_position(core_freesync->dc, &stream, 1,
+					&position.vertical_count,
+					&position.nominal_vcount)) {
 
 		*nom_v_pos = position.nominal_vcount;
 		*v_pos = position.vertical_count;
@@ -1223,9 +1201,9 @@ void mod_freesync_notify_mode_change(struct mod_freesync *mod_freesync,
 			triggers.overlay_update = true;
 			triggers.surface_update = true;
 
-			core_freesync->dc->stream_funcs.set_static_screen_events(
-				core_freesync->dc, streams, num_streams,
-				&triggers);
+			dc_stream_set_static_screen_events(core_freesync->dc,
+							   streams, num_streams,
+							   &triggers);
 		}
 	}
 
@@ -1424,10 +1402,8 @@ static void apply_fixed_refresh(struct core_freesync *core_freesync,
 
 		vmax = vmin;
 
-		core_freesync->dc->stream_funcs.adjust_vmin_vmax(
-				core_freesync->dc, &stream,
-				1, vmin,
-				vmax);
+		dc_stream_adjust_vmin_vmax(core_freesync->dc, &stream,
+					   1, vmin, vmax);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index b72f8a43d86b..9fa3aaef3f33 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -25,7 +25,6 @@
 
 #include <drm/amd_asic_type.h>
 
-struct seq_file;
 
 #define AMD_MAX_USEC_TIMEOUT		200000  /* 200 ms */
 
@@ -61,71 +60,12 @@ enum amd_clockgating_state {
 	AMD_CG_STATE_UNGATE,
 };
 
-enum amd_dpm_forced_level {
-	AMD_DPM_FORCED_LEVEL_AUTO = 0x1,
-	AMD_DPM_FORCED_LEVEL_MANUAL = 0x2,
-	AMD_DPM_FORCED_LEVEL_LOW = 0x4,
-	AMD_DPM_FORCED_LEVEL_HIGH = 0x8,
-	AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD = 0x10,
-	AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK = 0x20,
-	AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK = 0x40,
-	AMD_DPM_FORCED_LEVEL_PROFILE_PEAK = 0x80,
-	AMD_DPM_FORCED_LEVEL_PROFILE_EXIT = 0x100,
-};
 
 enum amd_powergating_state {
 	AMD_PG_STATE_GATE = 0,
 	AMD_PG_STATE_UNGATE,
 };
 
-struct amd_vce_state {
-	/* vce clocks */
-	u32 evclk;
-	u32 ecclk;
-	/* gpu clocks */
-	u32 sclk;
-	u32 mclk;
-	u8 clk_idx;
-	u8 pstate;
-};
-
-
-#define AMD_MAX_VCE_LEVELS 6
-
-enum amd_vce_level {
-	AMD_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
-	AMD_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
-	AMD_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
-	AMD_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
-	AMD_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
-	AMD_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
-};
-
-enum amd_pp_profile_type {
-	AMD_PP_GFX_PROFILE,
-	AMD_PP_COMPUTE_PROFILE,
-};
-
-struct amd_pp_profile {
-	enum amd_pp_profile_type type;
-	uint32_t min_sclk;
-	uint32_t min_mclk;
-	uint16_t activity_threshold;
-	uint8_t up_hyst;
-	uint8_t down_hyst;
-};
-
-enum amd_fan_ctrl_mode {
-	AMD_FAN_CTRL_NONE = 0,
-	AMD_FAN_CTRL_MANUAL = 1,
-	AMD_FAN_CTRL_AUTO = 2,
-};
-
-enum pp_clock_type {
-	PP_SCLK,
-	PP_MCLK,
-	PP_PCIE,
-};
 
 /* CG flags */
 #define AMD_CG_SUPPORT_GFX_MGCG			(1 << 0)
@@ -169,27 +109,6 @@ enum pp_clock_type {
 #define AMD_PG_SUPPORT_GFX_PIPELINE		(1 << 12)
 #define AMD_PG_SUPPORT_MMHUB			(1 << 13)
 
-enum amd_pm_state_type {
-	/* not used for dpm */
-	POWER_STATE_TYPE_DEFAULT,
-	POWER_STATE_TYPE_POWERSAVE,
-	/* user selectable states */
-	POWER_STATE_TYPE_BATTERY,
-	POWER_STATE_TYPE_BALANCED,
-	POWER_STATE_TYPE_PERFORMANCE,
-	/* internal states */
-	POWER_STATE_TYPE_INTERNAL_UVD,
-	POWER_STATE_TYPE_INTERNAL_UVD_SD,
-	POWER_STATE_TYPE_INTERNAL_UVD_HD,
-	POWER_STATE_TYPE_INTERNAL_UVD_HD2,
-	POWER_STATE_TYPE_INTERNAL_UVD_MVC,
-	POWER_STATE_TYPE_INTERNAL_BOOT,
-	POWER_STATE_TYPE_INTERNAL_THERMAL,
-	POWER_STATE_TYPE_INTERNAL_ACPI,
-	POWER_STATE_TYPE_INTERNAL_ULV,
-	POWER_STATE_TYPE_INTERNAL_3DPERF,
-};
-
 struct amd_ip_funcs {
 	/* Name of IP block */
 	char *name;
@@ -233,95 +152,4 @@ struct amd_ip_funcs {
 };
 
 
-enum amd_pp_task;
-enum amd_pp_clock_type;
-struct pp_states_info;
-struct amd_pp_simple_clock_info;
-struct amd_pp_display_configuration;
-struct amd_pp_clock_info;
-struct pp_display_clock_request;
-struct pp_wm_sets_with_clock_ranges_soc15;
-struct pp_clock_levels_with_voltage;
-struct pp_clock_levels_with_latency;
-struct amd_pp_clocks;
-
-struct amd_pm_funcs {
-/* export for dpm on ci and si */
-	int (*pre_set_power_state)(void *handle);
-	int (*set_power_state)(void *handle);
-	void (*post_set_power_state)(void *handle);
-	void (*display_configuration_changed)(void *handle);
-	void (*print_power_state)(void *handle, void *ps);
-	bool (*vblank_too_short)(void *handle);
-	void (*enable_bapm)(void *handle, bool enable);
-	int (*check_state_equal)(void *handle,
-				void  *cps,
-				void  *rps,
-				bool  *equal);
-/* export for sysfs */
-	int (*get_temperature)(void *handle);
-	void (*set_fan_control_mode)(void *handle, u32 mode);
-	u32 (*get_fan_control_mode)(void *handle);
-	int (*set_fan_speed_percent)(void *handle, u32 speed);
-	int (*get_fan_speed_percent)(void *handle, u32 *speed);
-	int (*force_clock_level)(void *handle, enum pp_clock_type type, uint32_t mask);
-	int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf);
-	int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level);
-	int (*get_sclk_od)(void *handle);
-	int (*set_sclk_od)(void *handle, uint32_t value);
-	int (*get_mclk_od)(void *handle);
-	int (*set_mclk_od)(void *handle, uint32_t value);
-	int (*read_sensor)(void *handle, int idx, void *value, int *size);
-	enum amd_dpm_forced_level (*get_performance_level)(void *handle);
-	enum amd_pm_state_type (*get_current_power_state)(void *handle);
-	int (*get_fan_speed_rpm)(void *handle, uint32_t *rpm);
-	int (*get_pp_num_states)(void *handle, struct pp_states_info *data);
-	int (*get_pp_table)(void *handle, char **table);
-	int (*set_pp_table)(void *handle, const char *buf, size_t size);
-	void (*debugfs_print_current_performance_level)(void *handle, struct seq_file *m);
-
-	int (*reset_power_profile_state)(void *handle,
-			struct amd_pp_profile *request);
-	int (*get_power_profile_state)(void *handle,
-			struct amd_pp_profile *query);
-	int (*set_power_profile_state)(void *handle,
-			struct amd_pp_profile *request);
-	int (*switch_power_profile)(void *handle,
-			enum amd_pp_profile_type type);
-/* export to amdgpu */
-	void (*powergate_uvd)(void *handle, bool gate);
-	void (*powergate_vce)(void *handle, bool gate);
-	struct amd_vce_state* (*get_vce_clock_state)(void *handle, u32 idx);
-	int (*dispatch_tasks)(void *handle, enum amd_pp_task task_id,
-				   void *input, void *output);
-	int (*load_firmware)(void *handle);
-	int (*wait_for_fw_loading_complete)(void *handle);
-	int (*set_clockgating_by_smu)(void *handle, uint32_t msg_id);
-/* export to DC */
-	u32 (*get_sclk)(void *handle, bool low);
-	u32 (*get_mclk)(void *handle, bool low);
-	int (*display_configuration_change)(void *handle,
-		const struct amd_pp_display_configuration *input);
-	int (*get_display_power_level)(void *handle,
-		struct amd_pp_simple_clock_info *output);
-	int (*get_current_clocks)(void *handle,
-		struct amd_pp_clock_info *clocks);
-	int (*get_clock_by_type)(void *handle,
-		enum amd_pp_clock_type type,
-		struct amd_pp_clocks *clocks);
-	int (*get_clock_by_type_with_latency)(void *handle,
-		enum amd_pp_clock_type type,
-		struct pp_clock_levels_with_latency *clocks);
-	int (*get_clock_by_type_with_voltage)(void *handle,
-		enum amd_pp_clock_type type,
-		struct pp_clock_levels_with_voltage *clocks);
-	int (*set_watermarks_for_clocks_ranges)(void *handle,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
-	int (*display_clock_voltage_request)(void *handle,
-		struct pp_display_clock_request *clock);
-	int (*get_display_mode_validation_clocks)(void *handle,
-		struct amd_pp_simple_clock_info *clocks);
-};
-
-
 #endif /* __AMD_SHARED_H__ */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_offset.h
new file mode 100644
index 000000000000..b1e878ecf9bf
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_offset.h
@@ -0,0 +1,453 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _athub_1_0_OFFSET_HEADER
+#define _athub_1_0_OFFSET_HEADER
+
+
+
+// addressBlock: athub_atsdec
+// base address:	0x3080
+#define mmATC_ATS_CNTL	0x0000
+#define mmATC_ATS_CNTL_BASE_IDX	0
+#define mmATC_ATS_STATUS	0x0003
+#define mmATC_ATS_STATUS_BASE_IDX	0
+#define mmATC_ATS_FAULT_CNTL	0x0004
+#define mmATC_ATS_FAULT_CNTL_BASE_IDX	0
+#define mmATC_ATS_FAULT_STATUS_INFO	0x0005
+#define mmATC_ATS_FAULT_STATUS_INFO_BASE_IDX	0
+#define mmATC_ATS_FAULT_STATUS_ADDR	0x0006
+#define mmATC_ATS_FAULT_STATUS_ADDR_BASE_IDX	0
+#define mmATC_ATS_DEFAULT_PAGE_LOW	0x0007
+#define mmATC_ATS_DEFAULT_PAGE_LOW_BASE_IDX	0
+#define mmATC_TRANS_FAULT_RSPCNTRL	0x0008
+#define mmATC_TRANS_FAULT_RSPCNTRL_BASE_IDX	0
+#define mmATC_ATS_FAULT_STATUS_INFO2	0x0009
+#define mmATC_ATS_FAULT_STATUS_INFO2_BASE_IDX	0
+#define mmATHUB_MISC_CNTL	0x000a
+#define mmATHUB_MISC_CNTL_BASE_IDX	0
+#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS	0x000b
+#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS_BASE_IDX	0
+#define mmATC_VMID0_PASID_MAPPING	0x000c
+#define mmATC_VMID0_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID1_PASID_MAPPING	0x000d
+#define mmATC_VMID1_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID2_PASID_MAPPING	0x000e
+#define mmATC_VMID2_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID3_PASID_MAPPING	0x000f
+#define mmATC_VMID3_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID4_PASID_MAPPING	0x0010
+#define mmATC_VMID4_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID5_PASID_MAPPING	0x0011
+#define mmATC_VMID5_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID6_PASID_MAPPING	0x0012
+#define mmATC_VMID6_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID7_PASID_MAPPING	0x0013
+#define mmATC_VMID7_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID8_PASID_MAPPING	0x0014
+#define mmATC_VMID8_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID9_PASID_MAPPING	0x0015
+#define mmATC_VMID9_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID10_PASID_MAPPING	0x0016
+#define mmATC_VMID10_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID11_PASID_MAPPING	0x0017
+#define mmATC_VMID11_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID12_PASID_MAPPING	0x0018
+#define mmATC_VMID12_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID13_PASID_MAPPING	0x0019
+#define mmATC_VMID13_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID14_PASID_MAPPING	0x001a
+#define mmATC_VMID14_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID15_PASID_MAPPING	0x001b
+#define mmATC_VMID15_PASID_MAPPING_BASE_IDX	0
+#define mmATC_ATS_VMID_STATUS	0x001c
+#define mmATC_ATS_VMID_STATUS_BASE_IDX	0
+#define mmATC_ATS_GFX_ATCL2_STATUS	0x001d
+#define mmATC_ATS_GFX_ATCL2_STATUS_BASE_IDX	0
+#define mmATC_PERFCOUNTER0_CFG	0x001e
+#define mmATC_PERFCOUNTER0_CFG_BASE_IDX	0
+#define mmATC_PERFCOUNTER1_CFG	0x001f
+#define mmATC_PERFCOUNTER1_CFG_BASE_IDX	0
+#define mmATC_PERFCOUNTER2_CFG	0x0020
+#define mmATC_PERFCOUNTER2_CFG_BASE_IDX	0
+#define mmATC_PERFCOUNTER3_CFG	0x0021
+#define mmATC_PERFCOUNTER3_CFG_BASE_IDX	0
+#define mmATC_PERFCOUNTER_RSLT_CNTL	0x0022
+#define mmATC_PERFCOUNTER_RSLT_CNTL_BASE_IDX	0
+#define mmATC_PERFCOUNTER_LO	0x0023
+#define mmATC_PERFCOUNTER_LO_BASE_IDX	0
+#define mmATC_PERFCOUNTER_HI	0x0024
+#define mmATC_PERFCOUNTER_HI_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL	0x0025
+#define mmATHUB_PCIE_ATS_CNTL_BASE_IDX	0
+#define mmATHUB_PCIE_PASID_CNTL	0x0026
+#define mmATHUB_PCIE_PASID_CNTL_BASE_IDX	0
+#define mmATHUB_PCIE_PAGE_REQ_CNTL	0x0027
+#define mmATHUB_PCIE_PAGE_REQ_CNTL_BASE_IDX	0
+#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC	0x0028
+#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC_BASE_IDX	0
+#define mmATHUB_COMMAND	0x0029
+#define mmATHUB_COMMAND_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_0	0x002a
+#define mmATHUB_PCIE_ATS_CNTL_VF_0_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_1	0x002b
+#define mmATHUB_PCIE_ATS_CNTL_VF_1_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_2	0x002c
+#define mmATHUB_PCIE_ATS_CNTL_VF_2_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_3	0x002d
+#define mmATHUB_PCIE_ATS_CNTL_VF_3_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_4	0x002e
+#define mmATHUB_PCIE_ATS_CNTL_VF_4_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_5	0x002f
+#define mmATHUB_PCIE_ATS_CNTL_VF_5_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_6	0x0030
+#define mmATHUB_PCIE_ATS_CNTL_VF_6_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_7	0x0031
+#define mmATHUB_PCIE_ATS_CNTL_VF_7_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_8	0x0032
+#define mmATHUB_PCIE_ATS_CNTL_VF_8_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_9	0x0033
+#define mmATHUB_PCIE_ATS_CNTL_VF_9_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_10	0x0034
+#define mmATHUB_PCIE_ATS_CNTL_VF_10_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_11	0x0035
+#define mmATHUB_PCIE_ATS_CNTL_VF_11_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_12	0x0036
+#define mmATHUB_PCIE_ATS_CNTL_VF_12_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_13	0x0037
+#define mmATHUB_PCIE_ATS_CNTL_VF_13_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_14	0x0038
+#define mmATHUB_PCIE_ATS_CNTL_VF_14_BASE_IDX	0
+#define mmATHUB_PCIE_ATS_CNTL_VF_15	0x0039
+#define mmATHUB_PCIE_ATS_CNTL_VF_15_BASE_IDX	0
+#define mmATHUB_MEM_POWER_LS	0x003a
+#define mmATHUB_MEM_POWER_LS_BASE_IDX	0
+#define mmATS_IH_CREDIT	0x003b
+#define mmATS_IH_CREDIT_BASE_IDX	0
+#define mmATHUB_IH_CREDIT	0x003c
+#define mmATHUB_IH_CREDIT_BASE_IDX	0
+#define mmATC_VMID16_PASID_MAPPING	0x003d
+#define mmATC_VMID16_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID17_PASID_MAPPING	0x003e
+#define mmATC_VMID17_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID18_PASID_MAPPING	0x003f
+#define mmATC_VMID18_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID19_PASID_MAPPING	0x0040
+#define mmATC_VMID19_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID20_PASID_MAPPING	0x0041
+#define mmATC_VMID20_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID21_PASID_MAPPING	0x0042
+#define mmATC_VMID21_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID22_PASID_MAPPING	0x0043
+#define mmATC_VMID22_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID23_PASID_MAPPING	0x0044
+#define mmATC_VMID23_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID24_PASID_MAPPING	0x0045
+#define mmATC_VMID24_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID25_PASID_MAPPING	0x0046
+#define mmATC_VMID25_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID26_PASID_MAPPING	0x0047
+#define mmATC_VMID26_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID27_PASID_MAPPING	0x0048
+#define mmATC_VMID27_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID28_PASID_MAPPING	0x0049
+#define mmATC_VMID28_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID29_PASID_MAPPING	0x004a
+#define mmATC_VMID29_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID30_PASID_MAPPING	0x004b
+#define mmATC_VMID30_PASID_MAPPING_BASE_IDX	0
+#define mmATC_VMID31_PASID_MAPPING	0x004c
+#define mmATC_VMID31_PASID_MAPPING_BASE_IDX	0
+#define mmATC_ATS_MMHUB_ATCL2_STATUS	0x004d
+#define mmATC_ATS_MMHUB_ATCL2_STATUS_BASE_IDX	0
+#define mmATHUB_SHARED_VIRT_RESET_REQ	0x004e
+#define mmATHUB_SHARED_VIRT_RESET_REQ_BASE_IDX	0
+#define mmATHUB_SHARED_ACTIVE_FCN_ID	0x004f
+#define mmATHUB_SHARED_ACTIVE_FCN_ID_BASE_IDX	0
+#define mmATC_ATS_SDPPORT_CNTL	0x0050
+#define mmATC_ATS_SDPPORT_CNTL_BASE_IDX	0
+#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT	0x0052
+#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT_BASE_IDX	0
+#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT	0x0053
+#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT_BASE_IDX	0
+
+
+// addressBlock: athub_xpbdec
+// base address:	0x31f0
+#define mmXPB_RTR_SRC_APRTR0	0x005c
+#define mmXPB_RTR_SRC_APRTR0_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR1	0x005d
+#define mmXPB_RTR_SRC_APRTR1_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR2	0x005e
+#define mmXPB_RTR_SRC_APRTR2_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR3	0x005f
+#define mmXPB_RTR_SRC_APRTR3_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR4	0x0060
+#define mmXPB_RTR_SRC_APRTR4_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR5	0x0061
+#define mmXPB_RTR_SRC_APRTR5_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR6	0x0062
+#define mmXPB_RTR_SRC_APRTR6_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR7	0x0063
+#define mmXPB_RTR_SRC_APRTR7_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR8	0x0064
+#define mmXPB_RTR_SRC_APRTR8_BASE_IDX	0
+#define mmXPB_RTR_SRC_APRTR9	0x0065
+#define mmXPB_RTR_SRC_APRTR9_BASE_IDX	0
+#define mmXPB_XDMA_RTR_SRC_APRTR0	0x0066
+#define mmXPB_XDMA_RTR_SRC_APRTR0_BASE_IDX	0
+#define mmXPB_XDMA_RTR_SRC_APRTR1	0x0067
+#define mmXPB_XDMA_RTR_SRC_APRTR1_BASE_IDX	0
+#define mmXPB_XDMA_RTR_SRC_APRTR2	0x0068
+#define mmXPB_XDMA_RTR_SRC_APRTR2_BASE_IDX	0
+#define mmXPB_XDMA_RTR_SRC_APRTR3	0x0069
+#define mmXPB_XDMA_RTR_SRC_APRTR3_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP0	0x006a
+#define mmXPB_RTR_DEST_MAP0_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP1	0x006b
+#define mmXPB_RTR_DEST_MAP1_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP2	0x006c
+#define mmXPB_RTR_DEST_MAP2_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP3	0x006d
+#define mmXPB_RTR_DEST_MAP3_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP4	0x006e
+#define mmXPB_RTR_DEST_MAP4_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP5	0x006f
+#define mmXPB_RTR_DEST_MAP5_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP6	0x0070
+#define mmXPB_RTR_DEST_MAP6_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP7	0x0071
+#define mmXPB_RTR_DEST_MAP7_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP8	0x0072
+#define mmXPB_RTR_DEST_MAP8_BASE_IDX	0
+#define mmXPB_RTR_DEST_MAP9	0x0073
+#define mmXPB_RTR_DEST_MAP9_BASE_IDX	0
+#define mmXPB_XDMA_RTR_DEST_MAP0	0x0074
+#define mmXPB_XDMA_RTR_DEST_MAP0_BASE_IDX	0
+#define mmXPB_XDMA_RTR_DEST_MAP1	0x0075
+#define mmXPB_XDMA_RTR_DEST_MAP1_BASE_IDX	0
+#define mmXPB_XDMA_RTR_DEST_MAP2	0x0076
+#define mmXPB_XDMA_RTR_DEST_MAP2_BASE_IDX	0
+#define mmXPB_XDMA_RTR_DEST_MAP3	0x0077
+#define mmXPB_XDMA_RTR_DEST_MAP3_BASE_IDX	0
+#define mmXPB_CLG_CFG0	0x0078
+#define mmXPB_CLG_CFG0_BASE_IDX	0
+#define mmXPB_CLG_CFG1	0x0079
+#define mmXPB_CLG_CFG1_BASE_IDX	0
+#define mmXPB_CLG_CFG2	0x007a
+#define mmXPB_CLG_CFG2_BASE_IDX	0
+#define mmXPB_CLG_CFG3	0x007b
+#define mmXPB_CLG_CFG3_BASE_IDX	0
+#define mmXPB_CLG_CFG4	0x007c
+#define mmXPB_CLG_CFG4_BASE_IDX	0
+#define mmXPB_CLG_CFG5	0x007d
+#define mmXPB_CLG_CFG5_BASE_IDX	0
+#define mmXPB_CLG_CFG6	0x007e
+#define mmXPB_CLG_CFG6_BASE_IDX	0
+#define mmXPB_CLG_CFG7	0x007f
+#define mmXPB_CLG_CFG7_BASE_IDX	0
+#define mmXPB_CLG_EXTRA	0x0080
+#define mmXPB_CLG_EXTRA_BASE_IDX	0
+#define mmXPB_CLG_EXTRA_MSK	0x0081
+#define mmXPB_CLG_EXTRA_MSK_BASE_IDX	0
+#define mmXPB_LB_ADDR	0x0082
+#define mmXPB_LB_ADDR_BASE_IDX	0
+#define mmXPB_WCB_STS	0x0083
+#define mmXPB_WCB_STS_BASE_IDX	0
+#define mmXPB_HST_CFG	0x0084
+#define mmXPB_HST_CFG_BASE_IDX	0
+#define mmXPB_P2P_BAR_CFG	0x0085
+#define mmXPB_P2P_BAR_CFG_BASE_IDX	0
+#define mmXPB_P2P_BAR0	0x0086
+#define mmXPB_P2P_BAR0_BASE_IDX	0
+#define mmXPB_P2P_BAR1	0x0087
+#define mmXPB_P2P_BAR1_BASE_IDX	0
+#define mmXPB_P2P_BAR2	0x0088
+#define mmXPB_P2P_BAR2_BASE_IDX	0
+#define mmXPB_P2P_BAR3	0x0089
+#define mmXPB_P2P_BAR3_BASE_IDX	0
+#define mmXPB_P2P_BAR4	0x008a
+#define mmXPB_P2P_BAR4_BASE_IDX	0
+#define mmXPB_P2P_BAR5	0x008b
+#define mmXPB_P2P_BAR5_BASE_IDX	0
+#define mmXPB_P2P_BAR6	0x008c
+#define mmXPB_P2P_BAR6_BASE_IDX	0
+#define mmXPB_P2P_BAR7	0x008d
+#define mmXPB_P2P_BAR7_BASE_IDX	0
+#define mmXPB_P2P_BAR_SETUP	0x008e
+#define mmXPB_P2P_BAR_SETUP_BASE_IDX	0
+#define mmXPB_P2P_BAR_DELTA_ABOVE	0x0090
+#define mmXPB_P2P_BAR_DELTA_ABOVE_BASE_IDX	0
+#define mmXPB_P2P_BAR_DELTA_BELOW	0x0091
+#define mmXPB_P2P_BAR_DELTA_BELOW_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR0	0x0092
+#define mmXPB_PEER_SYS_BAR0_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR1	0x0093
+#define mmXPB_PEER_SYS_BAR1_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR2	0x0094
+#define mmXPB_PEER_SYS_BAR2_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR3	0x0095
+#define mmXPB_PEER_SYS_BAR3_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR4	0x0096
+#define mmXPB_PEER_SYS_BAR4_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR5	0x0097
+#define mmXPB_PEER_SYS_BAR5_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR6	0x0098
+#define mmXPB_PEER_SYS_BAR6_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR7	0x0099
+#define mmXPB_PEER_SYS_BAR7_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR8	0x009a
+#define mmXPB_PEER_SYS_BAR8_BASE_IDX	0
+#define mmXPB_PEER_SYS_BAR9	0x009b
+#define mmXPB_PEER_SYS_BAR9_BASE_IDX	0
+#define mmXPB_XDMA_PEER_SYS_BAR0	0x009c
+#define mmXPB_XDMA_PEER_SYS_BAR0_BASE_IDX	0
+#define mmXPB_XDMA_PEER_SYS_BAR1	0x009d
+#define mmXPB_XDMA_PEER_SYS_BAR1_BASE_IDX	0
+#define mmXPB_XDMA_PEER_SYS_BAR2	0x009e
+#define mmXPB_XDMA_PEER_SYS_BAR2_BASE_IDX	0
+#define mmXPB_XDMA_PEER_SYS_BAR3	0x009f
+#define mmXPB_XDMA_PEER_SYS_BAR3_BASE_IDX	0
+#define mmXPB_CLK_GAT	0x00a0
+#define mmXPB_CLK_GAT_BASE_IDX	0
+#define mmXPB_INTF_CFG	0x00a1
+#define mmXPB_INTF_CFG_BASE_IDX	0
+#define mmXPB_INTF_STS	0x00a2
+#define mmXPB_INTF_STS_BASE_IDX	0
+#define mmXPB_PIPE_STS	0x00a3
+#define mmXPB_PIPE_STS_BASE_IDX	0
+#define mmXPB_SUB_CTRL	0x00a4
+#define mmXPB_SUB_CTRL_BASE_IDX	0
+#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB	0x00a5
+#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB_BASE_IDX	0
+#define mmXPB_PERF_KNOBS	0x00a6
+#define mmXPB_PERF_KNOBS_BASE_IDX	0
+#define mmXPB_STICKY	0x00a7
+#define mmXPB_STICKY_BASE_IDX	0
+#define mmXPB_STICKY_W1C	0x00a8
+#define mmXPB_STICKY_W1C_BASE_IDX	0
+#define mmXPB_MISC_CFG	0x00a9
+#define mmXPB_MISC_CFG_BASE_IDX	0
+#define mmXPB_INTF_CFG2	0x00aa
+#define mmXPB_INTF_CFG2_BASE_IDX	0
+#define mmXPB_CLG_EXTRA_RD	0x00ab
+#define mmXPB_CLG_EXTRA_RD_BASE_IDX	0
+#define mmXPB_CLG_EXTRA_MSK_RD	0x00ac
+#define mmXPB_CLG_EXTRA_MSK_RD_BASE_IDX	0
+#define mmXPB_CLG_GFX_MATCH	0x00ad
+#define mmXPB_CLG_GFX_MATCH_BASE_IDX	0
+#define mmXPB_CLG_GFX_MATCH_MSK	0x00ae
+#define mmXPB_CLG_GFX_MATCH_MSK_BASE_IDX	0
+#define mmXPB_CLG_MM_MATCH	0x00af
+#define mmXPB_CLG_MM_MATCH_BASE_IDX	0
+#define mmXPB_CLG_MM_MATCH_MSK	0x00b0
+#define mmXPB_CLG_MM_MATCH_MSK_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING0	0x00b1
+#define mmXPB_CLG_GFX_UNITID_MAPPING0_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING1	0x00b2
+#define mmXPB_CLG_GFX_UNITID_MAPPING1_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING2	0x00b3
+#define mmXPB_CLG_GFX_UNITID_MAPPING2_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING3	0x00b4
+#define mmXPB_CLG_GFX_UNITID_MAPPING3_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING4	0x00b5
+#define mmXPB_CLG_GFX_UNITID_MAPPING4_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING5	0x00b6
+#define mmXPB_CLG_GFX_UNITID_MAPPING5_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING6	0x00b7
+#define mmXPB_CLG_GFX_UNITID_MAPPING6_BASE_IDX	0
+#define mmXPB_CLG_GFX_UNITID_MAPPING7	0x00b8
+#define mmXPB_CLG_GFX_UNITID_MAPPING7_BASE_IDX	0
+#define mmXPB_CLG_MM_UNITID_MAPPING0	0x00b9
+#define mmXPB_CLG_MM_UNITID_MAPPING0_BASE_IDX	0
+#define mmXPB_CLG_MM_UNITID_MAPPING1	0x00ba
+#define mmXPB_CLG_MM_UNITID_MAPPING1_BASE_IDX	0
+#define mmXPB_CLG_MM_UNITID_MAPPING2	0x00bb
+#define mmXPB_CLG_MM_UNITID_MAPPING2_BASE_IDX	0
+#define mmXPB_CLG_MM_UNITID_MAPPING3	0x00bc
+#define mmXPB_CLG_MM_UNITID_MAPPING3_BASE_IDX	0
+
+
+// addressBlock: athub_rpbdec
+// base address:	0x33b0
+#define mmRPB_PASSPW_CONF	0x00cc
+#define mmRPB_PASSPW_CONF_BASE_IDX	0
+#define mmRPB_BLOCKLEVEL_CONF	0x00cd
+#define mmRPB_BLOCKLEVEL_CONF_BASE_IDX	0
+#define mmRPB_TAG_CONF	0x00cf
+#define mmRPB_TAG_CONF_BASE_IDX	0
+#define mmRPB_EFF_CNTL	0x00d1
+#define mmRPB_EFF_CNTL_BASE_IDX	0
+#define mmRPB_ARB_CNTL	0x00d2
+#define mmRPB_ARB_CNTL_BASE_IDX	0
+#define mmRPB_ARB_CNTL2	0x00d3
+#define mmRPB_ARB_CNTL2_BASE_IDX	0
+#define mmRPB_BIF_CNTL	0x00d4
+#define mmRPB_BIF_CNTL_BASE_IDX	0
+#define mmRPB_WR_SWITCH_CNTL	0x00d5
+#define mmRPB_WR_SWITCH_CNTL_BASE_IDX	0
+#define mmRPB_RD_SWITCH_CNTL	0x00d7
+#define mmRPB_RD_SWITCH_CNTL_BASE_IDX	0
+#define mmRPB_CID_QUEUE_WR	0x00d8
+#define mmRPB_CID_QUEUE_WR_BASE_IDX	0
+#define mmRPB_CID_QUEUE_RD	0x00d9
+#define mmRPB_CID_QUEUE_RD_BASE_IDX	0
+#define mmRPB_CID_QUEUE_EX	0x00dc
+#define mmRPB_CID_QUEUE_EX_BASE_IDX	0
+#define mmRPB_CID_QUEUE_EX_DATA	0x00dd
+#define mmRPB_CID_QUEUE_EX_DATA_BASE_IDX	0
+#define mmRPB_SWITCH_CNTL2	0x00de
+#define mmRPB_SWITCH_CNTL2_BASE_IDX	0
+#define mmRPB_DEINTRLV_COMBINE_CNTL	0x00df
+#define mmRPB_DEINTRLV_COMBINE_CNTL_BASE_IDX	0
+#define mmRPB_VC_SWITCH_RDWR	0x00e0
+#define mmRPB_VC_SWITCH_RDWR_BASE_IDX	0
+#define mmRPB_PERFCOUNTER_LO	0x00e1
+#define mmRPB_PERFCOUNTER_LO_BASE_IDX	0
+#define mmRPB_PERFCOUNTER_HI	0x00e2
+#define mmRPB_PERFCOUNTER_HI_BASE_IDX	0
+#define mmRPB_PERFCOUNTER0_CFG	0x00e3
+#define mmRPB_PERFCOUNTER0_CFG_BASE_IDX	0
+#define mmRPB_PERFCOUNTER1_CFG	0x00e4
+#define mmRPB_PERFCOUNTER1_CFG_BASE_IDX	0
+#define mmRPB_PERFCOUNTER2_CFG	0x00e5
+#define mmRPB_PERFCOUNTER2_CFG_BASE_IDX	0
+#define mmRPB_PERFCOUNTER3_CFG	0x00e6
+#define mmRPB_PERFCOUNTER3_CFG_BASE_IDX	0
+#define mmRPB_PERFCOUNTER_RSLT_CNTL	0x00e7
+#define mmRPB_PERFCOUNTER_RSLT_CNTL_BASE_IDX	0
+#define mmRPB_RD_QUEUE_CNTL	0x00e9
+#define mmRPB_RD_QUEUE_CNTL_BASE_IDX	0
+#define mmRPB_RD_QUEUE_CNTL2	0x00ea
+#define mmRPB_RD_QUEUE_CNTL2_BASE_IDX	0
+#define mmRPB_WR_QUEUE_CNTL	0x00eb
+#define mmRPB_WR_QUEUE_CNTL_BASE_IDX	0
+#define mmRPB_WR_QUEUE_CNTL2	0x00ec
+#define mmRPB_WR_QUEUE_CNTL2_BASE_IDX	0
+#define mmRPB_EA_QUEUE_WR	0x00ed
+#define mmRPB_EA_QUEUE_WR_BASE_IDX	0
+#define mmRPB_ATS_CNTL	0x00ee
+#define mmRPB_ATS_CNTL_BASE_IDX	0
+#define mmRPB_ATS_CNTL2	0x00ef
+#define mmRPB_ATS_CNTL2_BASE_IDX	0
+#define mmRPB_SDPPORT_CNTL	0x00f0
+#define mmRPB_SDPPORT_CNTL_BASE_IDX	0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_sh_mask.h
new file mode 100644
index 000000000000..2968c6e2f7b9
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/athub/athub_1_0_sh_mask.h
@@ -0,0 +1,2045 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _athub_1_0_SH_MASK_HEADER
+#define _athub_1_0_SH_MASK_HEADER
+
+
+// addressBlock: athub_atsdec
+//ATC_ATS_CNTL
+#define ATC_ATS_CNTL__DISABLE_ATC__SHIFT	0x0
+#define ATC_ATS_CNTL__DISABLE_PRI__SHIFT	0x1
+#define ATC_ATS_CNTL__DISABLE_PASID__SHIFT	0x2
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB__SHIFT	0x8
+#define ATC_ATS_CNTL__INVALIDATION_LOG_KEEP_ORDER__SHIFT	0x14
+#define ATC_ATS_CNTL__TRANS_LOG_KEEP_ORDER__SHIFT	0x15
+#define ATC_ATS_CNTL__TRANS_EXE_RETURN__SHIFT	0x16
+#define ATC_ATS_CNTL__DISABLE_ATC_MASK	0x00000001L
+#define ATC_ATS_CNTL__DISABLE_PRI_MASK	0x00000002L
+#define ATC_ATS_CNTL__DISABLE_PASID_MASK	0x00000004L
+#define ATC_ATS_CNTL__CREDITS_ATS_RPB_MASK	0x00003F00L
+#define ATC_ATS_CNTL__INVALIDATION_LOG_KEEP_ORDER_MASK	0x00100000L
+#define ATC_ATS_CNTL__TRANS_LOG_KEEP_ORDER_MASK	0x00200000L
+#define ATC_ATS_CNTL__TRANS_EXE_RETURN_MASK	0x00C00000L
+//ATC_ATS_STATUS
+#define ATC_ATS_STATUS__BUSY__SHIFT	0x0
+#define ATC_ATS_STATUS__CRASHED__SHIFT	0x1
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION__SHIFT	0x2
+#define ATC_ATS_STATUS__FLUSH_INVALIDATION_OUTSTANDING__SHIFT	0x3
+#define ATC_ATS_STATUS__NONFLUSH_INVALIDATION_OUTSTANDING__SHIFT	0x6
+#define ATC_ATS_STATUS__BUSY_MASK	0x00000001L
+#define ATC_ATS_STATUS__CRASHED_MASK	0x00000002L
+#define ATC_ATS_STATUS__DEADLOCK_DETECTION_MASK	0x00000004L
+#define ATC_ATS_STATUS__FLUSH_INVALIDATION_OUTSTANDING_MASK	0x00000038L
+#define ATC_ATS_STATUS__NONFLUSH_INVALIDATION_OUTSTANDING_MASK	0x000001C0L
+//ATC_ATS_FAULT_CNTL
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG__SHIFT	0x0
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE__SHIFT	0xa
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE__SHIFT	0x14
+#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG_MASK	0x000001FFL
+#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE_MASK	0x0007FC00L
+#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE_MASK	0x1FF00000L
+//ATC_ATS_FAULT_STATUS_INFO
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE__SHIFT	0x0
+#define ATC_ATS_FAULT_STATUS_INFO__VMID__SHIFT	0xa
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO__SHIFT	0xf
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2__SHIFT	0x10
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION__SHIFT	0x11
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST__SHIFT	0x12
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS__SHIFT	0x13
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH__SHIFT	0x18
+#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE_MASK	0x000001FFL
+#define ATC_ATS_FAULT_STATUS_INFO__VMID_MASK	0x00007C00L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO_MASK	0x00008000L
+#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2_MASK	0x00010000L
+#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION_MASK	0x00020000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST_MASK	0x00040000L
+#define ATC_ATS_FAULT_STATUS_INFO__STATUS_MASK	0x00F80000L
+#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH_MASK	0x0F000000L
+//ATC_ATS_FAULT_STATUS_ADDR
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR__SHIFT	0x0
+#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR_MASK	0xFFFFFFFFL
+//ATC_ATS_DEFAULT_PAGE_LOW
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE__SHIFT	0x0
+#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE_MASK	0xFFFFFFFFL
+//ATC_TRANS_FAULT_RSPCNTRL
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID0__SHIFT	0x0
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID1__SHIFT	0x1
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID2__SHIFT	0x2
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID3__SHIFT	0x3
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID4__SHIFT	0x4
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID5__SHIFT	0x5
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID6__SHIFT	0x6
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID7__SHIFT	0x7
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID8__SHIFT	0x8
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID9__SHIFT	0x9
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID10__SHIFT	0xa
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID11__SHIFT	0xb
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID12__SHIFT	0xc
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID13__SHIFT	0xd
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID14__SHIFT	0xe
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID15__SHIFT	0xf
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID16__SHIFT	0x10
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID17__SHIFT	0x11
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID18__SHIFT	0x12
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID19__SHIFT	0x13
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID20__SHIFT	0x14
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID21__SHIFT	0x15
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID22__SHIFT	0x16
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID23__SHIFT	0x17
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID24__SHIFT	0x18
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID25__SHIFT	0x19
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID26__SHIFT	0x1a
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID27__SHIFT	0x1b
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID28__SHIFT	0x1c
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID29__SHIFT	0x1d
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID30__SHIFT	0x1e
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID31__SHIFT	0x1f
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID0_MASK	0x00000001L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID1_MASK	0x00000002L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID2_MASK	0x00000004L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID3_MASK	0x00000008L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID4_MASK	0x00000010L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID5_MASK	0x00000020L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID6_MASK	0x00000040L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID7_MASK	0x00000080L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID8_MASK	0x00000100L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID9_MASK	0x00000200L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID10_MASK	0x00000400L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID11_MASK	0x00000800L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID12_MASK	0x00001000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID13_MASK	0x00002000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID14_MASK	0x00004000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID15_MASK	0x00008000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID16_MASK	0x00010000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID17_MASK	0x00020000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID18_MASK	0x00040000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID19_MASK	0x00080000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID20_MASK	0x00100000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID21_MASK	0x00200000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID22_MASK	0x00400000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID23_MASK	0x00800000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID24_MASK	0x01000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID25_MASK	0x02000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID26_MASK	0x04000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID27_MASK	0x08000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID28_MASK	0x10000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID29_MASK	0x20000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID30_MASK	0x40000000L
+#define ATC_TRANS_FAULT_RSPCNTRL__VMID31_MASK	0x80000000L
+//ATC_ATS_FAULT_STATUS_INFO2
+#define ATC_ATS_FAULT_STATUS_INFO2__VF__SHIFT	0x0
+#define ATC_ATS_FAULT_STATUS_INFO2__VFID__SHIFT	0x1
+#define ATC_ATS_FAULT_STATUS_INFO2__MMHUB_INV_VMID__SHIFT	0x9
+#define ATC_ATS_FAULT_STATUS_INFO2__VF_MASK	0x00000001L
+#define ATC_ATS_FAULT_STATUS_INFO2__VFID_MASK	0x0000001EL
+#define ATC_ATS_FAULT_STATUS_INFO2__MMHUB_INV_VMID_MASK	0x00003E00L
+//ATHUB_MISC_CNTL
+#define ATHUB_MISC_CNTL__CG_OFFDLY__SHIFT	0x6
+#define ATHUB_MISC_CNTL__CG_ENABLE__SHIFT	0x12
+#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE__SHIFT	0x13
+#define ATHUB_MISC_CNTL__PG_ENABLE__SHIFT	0x14
+#define ATHUB_MISC_CNTL__PG_OFFDLY__SHIFT	0x15
+#define ATHUB_MISC_CNTL__CG_STATUS__SHIFT	0x1b
+#define ATHUB_MISC_CNTL__PG_STATUS__SHIFT	0x1c
+#define ATHUB_MISC_CNTL__CG_OFFDLY_MASK	0x00000FC0L
+#define ATHUB_MISC_CNTL__CG_ENABLE_MASK	0x00040000L
+#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK	0x00080000L
+#define ATHUB_MISC_CNTL__PG_ENABLE_MASK	0x00100000L
+#define ATHUB_MISC_CNTL__PG_OFFDLY_MASK	0x07E00000L
+#define ATHUB_MISC_CNTL__CG_STATUS_MASK	0x08000000L
+#define ATHUB_MISC_CNTL__PG_STATUS_MASK	0x10000000L
+//ATC_VMID_PASID_MAPPING_UPDATE_STATUS
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED__SHIFT	0x0
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED__SHIFT	0x1
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED__SHIFT	0x2
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED__SHIFT	0x3
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED__SHIFT	0x4
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED__SHIFT	0x5
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED__SHIFT	0x6
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED__SHIFT	0x7
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED__SHIFT	0x8
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED__SHIFT	0x9
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED__SHIFT	0xa
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED__SHIFT	0xb
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED__SHIFT	0xc
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED__SHIFT	0xd
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED__SHIFT	0xe
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED__SHIFT	0xf
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID16_REMAPPING_FINISHED__SHIFT	0x10
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID17_REMAPPING_FINISHED__SHIFT	0x11
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID18_REMAPPING_FINISHED__SHIFT	0x12
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID19_REMAPPING_FINISHED__SHIFT	0x13
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID20_REMAPPING_FINISHED__SHIFT	0x14
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID21_REMAPPING_FINISHED__SHIFT	0x15
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID22_REMAPPING_FINISHED__SHIFT	0x16
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID23_REMAPPING_FINISHED__SHIFT	0x17
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID24_REMAPPING_FINISHED__SHIFT	0x18
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID25_REMAPPING_FINISHED__SHIFT	0x19
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID26_REMAPPING_FINISHED__SHIFT	0x1a
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID27_REMAPPING_FINISHED__SHIFT	0x1b
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID28_REMAPPING_FINISHED__SHIFT	0x1c
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID29_REMAPPING_FINISHED__SHIFT	0x1d
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID30_REMAPPING_FINISHED__SHIFT	0x1e
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID31_REMAPPING_FINISHED__SHIFT	0x1f
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED_MASK	0x00000001L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED_MASK	0x00000002L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED_MASK	0x00000004L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED_MASK	0x00000008L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED_MASK	0x00000010L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED_MASK	0x00000020L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED_MASK	0x00000040L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED_MASK	0x00000080L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED_MASK	0x00000100L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED_MASK	0x00000200L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED_MASK	0x00000400L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED_MASK	0x00000800L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED_MASK	0x00001000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED_MASK	0x00002000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED_MASK	0x00004000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED_MASK	0x00008000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID16_REMAPPING_FINISHED_MASK	0x00010000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID17_REMAPPING_FINISHED_MASK	0x00020000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID18_REMAPPING_FINISHED_MASK	0x00040000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID19_REMAPPING_FINISHED_MASK	0x00080000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID20_REMAPPING_FINISHED_MASK	0x00100000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID21_REMAPPING_FINISHED_MASK	0x00200000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID22_REMAPPING_FINISHED_MASK	0x00400000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID23_REMAPPING_FINISHED_MASK	0x00800000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID24_REMAPPING_FINISHED_MASK	0x01000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID25_REMAPPING_FINISHED_MASK	0x02000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID26_REMAPPING_FINISHED_MASK	0x04000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID27_REMAPPING_FINISHED_MASK	0x08000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID28_REMAPPING_FINISHED_MASK	0x10000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID29_REMAPPING_FINISHED_MASK	0x20000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID30_REMAPPING_FINISHED_MASK	0x40000000L
+#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID31_REMAPPING_FINISHED_MASK	0x80000000L
+//ATC_VMID0_PASID_MAPPING
+#define ATC_VMID0_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID0_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID0_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID0_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID1_PASID_MAPPING
+#define ATC_VMID1_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID1_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID1_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID1_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID2_PASID_MAPPING
+#define ATC_VMID2_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID2_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID2_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID2_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID3_PASID_MAPPING
+#define ATC_VMID3_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID3_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID3_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID3_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID4_PASID_MAPPING
+#define ATC_VMID4_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID4_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID4_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID4_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID5_PASID_MAPPING
+#define ATC_VMID5_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID5_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID5_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID5_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID6_PASID_MAPPING
+#define ATC_VMID6_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID6_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID6_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID6_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID7_PASID_MAPPING
+#define ATC_VMID7_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID7_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID7_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID7_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID8_PASID_MAPPING
+#define ATC_VMID8_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID8_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID8_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID8_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID9_PASID_MAPPING
+#define ATC_VMID9_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID9_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID9_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID9_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID10_PASID_MAPPING
+#define ATC_VMID10_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID10_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID10_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID10_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID11_PASID_MAPPING
+#define ATC_VMID11_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID11_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID11_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID11_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID12_PASID_MAPPING
+#define ATC_VMID12_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID12_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID12_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID12_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID13_PASID_MAPPING
+#define ATC_VMID13_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID13_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID13_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID13_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID14_PASID_MAPPING
+#define ATC_VMID14_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID14_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID14_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID14_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID15_PASID_MAPPING
+#define ATC_VMID15_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID15_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID15_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID15_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_ATS_VMID_STATUS
+#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING__SHIFT	0x0
+#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING__SHIFT	0x1
+#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING__SHIFT	0x2
+#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING__SHIFT	0x3
+#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING__SHIFT	0x4
+#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING__SHIFT	0x5
+#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING__SHIFT	0x6
+#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING__SHIFT	0x7
+#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING__SHIFT	0x8
+#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING__SHIFT	0x9
+#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING__SHIFT	0xa
+#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING__SHIFT	0xb
+#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING__SHIFT	0xc
+#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING__SHIFT	0xd
+#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING__SHIFT	0xe
+#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING__SHIFT	0xf
+#define ATC_ATS_VMID_STATUS__VMID16_OUTSTANDING__SHIFT	0x10
+#define ATC_ATS_VMID_STATUS__VMID17_OUTSTANDING__SHIFT	0x11
+#define ATC_ATS_VMID_STATUS__VMID18_OUTSTANDING__SHIFT	0x12
+#define ATC_ATS_VMID_STATUS__VMID19_OUTSTANDING__SHIFT	0x13
+#define ATC_ATS_VMID_STATUS__VMID20_OUTSTANDING__SHIFT	0x14
+#define ATC_ATS_VMID_STATUS__VMID21_OUTSTANDING__SHIFT	0x15
+#define ATC_ATS_VMID_STATUS__VMID22_OUTSTANDING__SHIFT	0x16
+#define ATC_ATS_VMID_STATUS__VMID23_OUTSTANDING__SHIFT	0x17
+#define ATC_ATS_VMID_STATUS__VMID24_OUTSTANDING__SHIFT	0x18
+#define ATC_ATS_VMID_STATUS__VMID25_OUTSTANDING__SHIFT	0x19
+#define ATC_ATS_VMID_STATUS__VMID26_OUTSTANDING__SHIFT	0x1a
+#define ATC_ATS_VMID_STATUS__VMID27_OUTSTANDING__SHIFT	0x1b
+#define ATC_ATS_VMID_STATUS__VMID28_OUTSTANDING__SHIFT	0x1c
+#define ATC_ATS_VMID_STATUS__VMID29_OUTSTANDING__SHIFT	0x1d
+#define ATC_ATS_VMID_STATUS__VMID30_OUTSTANDING__SHIFT	0x1e
+#define ATC_ATS_VMID_STATUS__VMID31_OUTSTANDING__SHIFT	0x1f
+#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING_MASK	0x00000001L
+#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING_MASK	0x00000002L
+#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING_MASK	0x00000004L
+#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING_MASK	0x00000008L
+#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING_MASK	0x00000010L
+#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING_MASK	0x00000020L
+#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING_MASK	0x00000040L
+#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING_MASK	0x00000080L
+#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING_MASK	0x00000100L
+#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING_MASK	0x00000200L
+#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING_MASK	0x00000400L
+#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING_MASK	0x00000800L
+#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING_MASK	0x00001000L
+#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING_MASK	0x00002000L
+#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING_MASK	0x00004000L
+#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING_MASK	0x00008000L
+#define ATC_ATS_VMID_STATUS__VMID16_OUTSTANDING_MASK	0x00010000L
+#define ATC_ATS_VMID_STATUS__VMID17_OUTSTANDING_MASK	0x00020000L
+#define ATC_ATS_VMID_STATUS__VMID18_OUTSTANDING_MASK	0x00040000L
+#define ATC_ATS_VMID_STATUS__VMID19_OUTSTANDING_MASK	0x00080000L
+#define ATC_ATS_VMID_STATUS__VMID20_OUTSTANDING_MASK	0x00100000L
+#define ATC_ATS_VMID_STATUS__VMID21_OUTSTANDING_MASK	0x00200000L
+#define ATC_ATS_VMID_STATUS__VMID22_OUTSTANDING_MASK	0x00400000L
+#define ATC_ATS_VMID_STATUS__VMID23_OUTSTANDING_MASK	0x00800000L
+#define ATC_ATS_VMID_STATUS__VMID24_OUTSTANDING_MASK	0x01000000L
+#define ATC_ATS_VMID_STATUS__VMID25_OUTSTANDING_MASK	0x02000000L
+#define ATC_ATS_VMID_STATUS__VMID26_OUTSTANDING_MASK	0x04000000L
+#define ATC_ATS_VMID_STATUS__VMID27_OUTSTANDING_MASK	0x08000000L
+#define ATC_ATS_VMID_STATUS__VMID28_OUTSTANDING_MASK	0x10000000L
+#define ATC_ATS_VMID_STATUS__VMID29_OUTSTANDING_MASK	0x20000000L
+#define ATC_ATS_VMID_STATUS__VMID30_OUTSTANDING_MASK	0x40000000L
+#define ATC_ATS_VMID_STATUS__VMID31_OUTSTANDING_MASK	0x80000000L
+//ATC_ATS_GFX_ATCL2_STATUS
+#define ATC_ATS_GFX_ATCL2_STATUS__POWERED_DOWN__SHIFT	0x0
+#define ATC_ATS_GFX_ATCL2_STATUS__POWERED_DOWN_MASK	0x00000001L
+//ATC_PERFCOUNTER0_CFG
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL__SHIFT	0x0
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT	0x8
+#define ATC_PERFCOUNTER0_CFG__PERF_MODE__SHIFT	0x18
+#define ATC_PERFCOUNTER0_CFG__ENABLE__SHIFT	0x1c
+#define ATC_PERFCOUNTER0_CFG__CLEAR__SHIFT	0x1d
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_MASK	0x000000FFL
+#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define ATC_PERFCOUNTER0_CFG__PERF_MODE_MASK	0x0F000000L
+#define ATC_PERFCOUNTER0_CFG__ENABLE_MASK	0x10000000L
+#define ATC_PERFCOUNTER0_CFG__CLEAR_MASK	0x20000000L
+//ATC_PERFCOUNTER1_CFG
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL__SHIFT	0x0
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT	0x8
+#define ATC_PERFCOUNTER1_CFG__PERF_MODE__SHIFT	0x18
+#define ATC_PERFCOUNTER1_CFG__ENABLE__SHIFT	0x1c
+#define ATC_PERFCOUNTER1_CFG__CLEAR__SHIFT	0x1d
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_MASK	0x000000FFL
+#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define ATC_PERFCOUNTER1_CFG__PERF_MODE_MASK	0x0F000000L
+#define ATC_PERFCOUNTER1_CFG__ENABLE_MASK	0x10000000L
+#define ATC_PERFCOUNTER1_CFG__CLEAR_MASK	0x20000000L
+//ATC_PERFCOUNTER2_CFG
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL__SHIFT	0x0
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT	0x8
+#define ATC_PERFCOUNTER2_CFG__PERF_MODE__SHIFT	0x18
+#define ATC_PERFCOUNTER2_CFG__ENABLE__SHIFT	0x1c
+#define ATC_PERFCOUNTER2_CFG__CLEAR__SHIFT	0x1d
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_MASK	0x000000FFL
+#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define ATC_PERFCOUNTER2_CFG__PERF_MODE_MASK	0x0F000000L
+#define ATC_PERFCOUNTER2_CFG__ENABLE_MASK	0x10000000L
+#define ATC_PERFCOUNTER2_CFG__CLEAR_MASK	0x20000000L
+//ATC_PERFCOUNTER3_CFG
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL__SHIFT	0x0
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT	0x8
+#define ATC_PERFCOUNTER3_CFG__PERF_MODE__SHIFT	0x18
+#define ATC_PERFCOUNTER3_CFG__ENABLE__SHIFT	0x1c
+#define ATC_PERFCOUNTER3_CFG__CLEAR__SHIFT	0x1d
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_MASK	0x000000FFL
+#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define ATC_PERFCOUNTER3_CFG__PERF_MODE_MASK	0x0F000000L
+#define ATC_PERFCOUNTER3_CFG__ENABLE_MASK	0x10000000L
+#define ATC_PERFCOUNTER3_CFG__CLEAR_MASK	0x20000000L
+//ATC_PERFCOUNTER_RSLT_CNTL
+#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT	0x0
+#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT	0x8
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT	0x10
+#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT	0x18
+#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT	0x19
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT	0x1a
+#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK	0x0000000FL
+#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK	0x0000FF00L
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK	0x00FF0000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK	0x01000000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK	0x02000000L
+#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK	0x04000000L
+//ATC_PERFCOUNTER_LO
+#define ATC_PERFCOUNTER_LO__COUNTER_LO__SHIFT	0x0
+#define ATC_PERFCOUNTER_LO__COUNTER_LO_MASK	0xFFFFFFFFL
+//ATC_PERFCOUNTER_HI
+#define ATC_PERFCOUNTER_HI__COUNTER_HI__SHIFT	0x0
+#define ATC_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT	0x10
+#define ATC_PERFCOUNTER_HI__COUNTER_HI_MASK	0x0000FFFFL
+#define ATC_PERFCOUNTER_HI__COMPARE_VALUE_MASK	0xFFFF0000L
+//ATHUB_PCIE_ATS_CNTL
+#define ATHUB_PCIE_ATS_CNTL__STU__SHIFT	0x10
+#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL__STU_MASK	0x001F0000L
+#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_PASID_CNTL
+#define ATHUB_PCIE_PASID_CNTL__PASID_EN__SHIFT	0x10
+#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT	0x11
+#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT	0x12
+#define ATHUB_PCIE_PASID_CNTL__PASID_EN_MASK	0x00010000L
+#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK	0x00020000L
+#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK	0x00040000L
+//ATHUB_PCIE_PAGE_REQ_CNTL
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT	0x0
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT	0x1
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK	0x00000001L
+#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK	0x00000002L
+//ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT	0x0
+#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK	0xFFFFFFFFL
+//ATHUB_COMMAND
+#define ATHUB_COMMAND__BUS_MASTER_EN__SHIFT	0x2
+#define ATHUB_COMMAND__BUS_MASTER_EN_MASK	0x00000004L
+//ATHUB_PCIE_ATS_CNTL_VF_0
+#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_1
+#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_2
+#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_3
+#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_4
+#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_5
+#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_6
+#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_7
+#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_8
+#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_9
+#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_10
+#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_11
+#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_12
+#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_13
+#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_14
+#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_PCIE_ATS_CNTL_VF_15
+#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE__SHIFT	0x1f
+#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE_MASK	0x80000000L
+//ATHUB_MEM_POWER_LS
+#define ATHUB_MEM_POWER_LS__LS_SETUP__SHIFT	0x0
+#define ATHUB_MEM_POWER_LS__LS_HOLD__SHIFT	0x6
+#define ATHUB_MEM_POWER_LS__LS_SETUP_MASK	0x0000003FL
+#define ATHUB_MEM_POWER_LS__LS_HOLD_MASK	0x00000FC0L
+//ATS_IH_CREDIT
+#define ATS_IH_CREDIT__CREDIT_VALUE__SHIFT	0x0
+#define ATS_IH_CREDIT__IH_CLIENT_ID__SHIFT	0x10
+#define ATS_IH_CREDIT__CREDIT_VALUE_MASK	0x00000003L
+#define ATS_IH_CREDIT__IH_CLIENT_ID_MASK	0x00FF0000L
+//ATHUB_IH_CREDIT
+#define ATHUB_IH_CREDIT__CREDIT_VALUE__SHIFT	0x0
+#define ATHUB_IH_CREDIT__IH_CLIENT_ID__SHIFT	0x10
+#define ATHUB_IH_CREDIT__CREDIT_VALUE_MASK	0x00000003L
+#define ATHUB_IH_CREDIT__IH_CLIENT_ID_MASK	0x00FF0000L
+//ATC_VMID16_PASID_MAPPING
+#define ATC_VMID16_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID16_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID16_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID16_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID16_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID16_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID17_PASID_MAPPING
+#define ATC_VMID17_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID17_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID17_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID17_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID17_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID17_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID18_PASID_MAPPING
+#define ATC_VMID18_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID18_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID18_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID18_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID18_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID18_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID19_PASID_MAPPING
+#define ATC_VMID19_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID19_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID19_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID19_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID19_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID19_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID20_PASID_MAPPING
+#define ATC_VMID20_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID20_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID20_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID20_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID20_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID20_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID21_PASID_MAPPING
+#define ATC_VMID21_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID21_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID21_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID21_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID21_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID21_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID22_PASID_MAPPING
+#define ATC_VMID22_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID22_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID22_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID22_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID22_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID22_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID23_PASID_MAPPING
+#define ATC_VMID23_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID23_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID23_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID23_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID23_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID23_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID24_PASID_MAPPING
+#define ATC_VMID24_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID24_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID24_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID24_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID24_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID24_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID25_PASID_MAPPING
+#define ATC_VMID25_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID25_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID25_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID25_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID25_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID25_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID26_PASID_MAPPING
+#define ATC_VMID26_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID26_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID26_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID26_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID26_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID26_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID27_PASID_MAPPING
+#define ATC_VMID27_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID27_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID27_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID27_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID27_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID27_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID28_PASID_MAPPING
+#define ATC_VMID28_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID28_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID28_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID28_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID28_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID28_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID29_PASID_MAPPING
+#define ATC_VMID29_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID29_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID29_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID29_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID29_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID29_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID30_PASID_MAPPING
+#define ATC_VMID30_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID30_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID30_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID30_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID30_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID30_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_VMID31_PASID_MAPPING
+#define ATC_VMID31_PASID_MAPPING__PASID__SHIFT	0x0
+#define ATC_VMID31_PASID_MAPPING__NO_INVALIDATION__SHIFT	0x1e
+#define ATC_VMID31_PASID_MAPPING__VALID__SHIFT	0x1f
+#define ATC_VMID31_PASID_MAPPING__PASID_MASK	0x0000FFFFL
+#define ATC_VMID31_PASID_MAPPING__NO_INVALIDATION_MASK	0x40000000L
+#define ATC_VMID31_PASID_MAPPING__VALID_MASK	0x80000000L
+//ATC_ATS_MMHUB_ATCL2_STATUS
+#define ATC_ATS_MMHUB_ATCL2_STATUS__POWERED_DOWN__SHIFT	0x0
+#define ATC_ATS_MMHUB_ATCL2_STATUS__POWERED_DOWN_MASK	0x00000001L
+//ATHUB_SHARED_VIRT_RESET_REQ
+#define ATHUB_SHARED_VIRT_RESET_REQ__VF__SHIFT	0x0
+#define ATHUB_SHARED_VIRT_RESET_REQ__PF__SHIFT	0x1f
+#define ATHUB_SHARED_VIRT_RESET_REQ__VF_MASK	0x0000FFFFL
+#define ATHUB_SHARED_VIRT_RESET_REQ__PF_MASK	0x80000000L
+//ATHUB_SHARED_ACTIVE_FCN_ID
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID__SHIFT	0x0
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VF__SHIFT	0x1f
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID_MASK	0x0000000FL
+#define ATHUB_SHARED_ACTIVE_FCN_ID__VF_MASK	0x80000000L
+//ATC_ATS_SDPPORT_CNTL
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE__SHIFT	0x0
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE__SHIFT	0x1
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD__SHIFT	0x3
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE__SHIFT	0x7
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK__SHIFT	0x8
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD__SHIFT	0x9
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE__SHIFT	0xd
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_RDY_MODE__SHIFT	0xe
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_MMHUB_RDY_MODE__SHIFT	0xf
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN__SHIFT	0x10
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV__SHIFT	0x11
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN__SHIFT	0x12
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV__SHIFT	0x13
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN__SHIFT	0x14
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV__SHIFT	0x15
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN__SHIFT	0x16
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV__SHIFT	0x17
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN__SHIFT	0x18
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV__SHIFT	0x19
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE_MASK	0x00000001L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE_MASK	0x00000006L
+#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD_MASK	0x00000078L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE_MASK	0x00000080L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK_MASK	0x00000100L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD_MASK	0x00001E00L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE_MASK	0x00002000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_RDY_MODE_MASK	0x00004000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_MMHUB_RDY_MODE_MASK	0x00008000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN_MASK	0x00010000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV_MASK	0x00020000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN_MASK	0x00040000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV_MASK	0x00080000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN_MASK	0x00100000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV_MASK	0x00200000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN_MASK	0x00400000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV_MASK	0x00800000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN_MASK	0x01000000L
+#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV_MASK	0x02000000L
+//ATC_ATS_VMID_SNAPSHOT_GFX_STAT
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID0__SHIFT	0x0
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID1__SHIFT	0x1
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID2__SHIFT	0x2
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID3__SHIFT	0x3
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID4__SHIFT	0x4
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID5__SHIFT	0x5
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID6__SHIFT	0x6
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID7__SHIFT	0x7
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID8__SHIFT	0x8
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID9__SHIFT	0x9
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID10__SHIFT	0xa
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID11__SHIFT	0xb
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID12__SHIFT	0xc
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID13__SHIFT	0xd
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID14__SHIFT	0xe
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID15__SHIFT	0xf
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID0_MASK	0x00000001L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID1_MASK	0x00000002L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID2_MASK	0x00000004L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID3_MASK	0x00000008L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID4_MASK	0x00000010L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID5_MASK	0x00000020L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID6_MASK	0x00000040L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID7_MASK	0x00000080L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID8_MASK	0x00000100L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID9_MASK	0x00000200L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID10_MASK	0x00000400L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID11_MASK	0x00000800L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID12_MASK	0x00001000L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID13_MASK	0x00002000L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID14_MASK	0x00004000L
+#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID15_MASK	0x00008000L
+//ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID0__SHIFT	0x0
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID1__SHIFT	0x1
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID2__SHIFT	0x2
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID3__SHIFT	0x3
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID4__SHIFT	0x4
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID5__SHIFT	0x5
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID6__SHIFT	0x6
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID7__SHIFT	0x7
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID8__SHIFT	0x8
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID9__SHIFT	0x9
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID10__SHIFT	0xa
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID11__SHIFT	0xb
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID12__SHIFT	0xc
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID13__SHIFT	0xd
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID14__SHIFT	0xe
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID15__SHIFT	0xf
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID0_MASK	0x00000001L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID1_MASK	0x00000002L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID2_MASK	0x00000004L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID3_MASK	0x00000008L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID4_MASK	0x00000010L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID5_MASK	0x00000020L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID6_MASK	0x00000040L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID7_MASK	0x00000080L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID8_MASK	0x00000100L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID9_MASK	0x00000200L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID10_MASK	0x00000400L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID11_MASK	0x00000800L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID12_MASK	0x00001000L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID13_MASK	0x00002000L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID14_MASK	0x00004000L
+#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID15_MASK	0x00008000L
+
+
+// addressBlock: athub_xpbdec
+//XPB_RTR_SRC_APRTR0
+#define XPB_RTR_SRC_APRTR0__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR0__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR1
+#define XPB_RTR_SRC_APRTR1__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR1__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR2
+#define XPB_RTR_SRC_APRTR2__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR2__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR3
+#define XPB_RTR_SRC_APRTR3__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR3__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR4
+#define XPB_RTR_SRC_APRTR4__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR4__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR5
+#define XPB_RTR_SRC_APRTR5__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR5__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR6
+#define XPB_RTR_SRC_APRTR6__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR6__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR7
+#define XPB_RTR_SRC_APRTR7__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR7__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR8
+#define XPB_RTR_SRC_APRTR8__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR8__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_SRC_APRTR9
+#define XPB_RTR_SRC_APRTR9__BASE_ADDR__SHIFT	0x0
+#define XPB_RTR_SRC_APRTR9__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR0
+#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR__SHIFT	0x0
+#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR1
+#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR__SHIFT	0x0
+#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR2
+#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR__SHIFT	0x0
+#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_XDMA_RTR_SRC_APRTR3
+#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR__SHIFT	0x0
+#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR_MASK	0x7FFFFFFFL
+//XPB_RTR_DEST_MAP0
+#define XPB_RTR_DEST_MAP0__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP0__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP0__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP0__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP0__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP0__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP0__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP0__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP1
+#define XPB_RTR_DEST_MAP1__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP1__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP1__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP1__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP1__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP1__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP1__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP1__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP2
+#define XPB_RTR_DEST_MAP2__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP2__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP2__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP2__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP2__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP2__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP2__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP2__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP3
+#define XPB_RTR_DEST_MAP3__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP3__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP3__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP3__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP3__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP3__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP3__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP3__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP4
+#define XPB_RTR_DEST_MAP4__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP4__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP4__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP4__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP4__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP4__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP4__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP4__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP5
+#define XPB_RTR_DEST_MAP5__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP5__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP5__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP5__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP5__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP5__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP5__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP5__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP6
+#define XPB_RTR_DEST_MAP6__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP6__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP6__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP6__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP6__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP6__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP6__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP6__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP7
+#define XPB_RTR_DEST_MAP7__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP7__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP7__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP7__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP7__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP7__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP7__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP7__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP8
+#define XPB_RTR_DEST_MAP8__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP8__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP8__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP8__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP8__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP8__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP8__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP8__APRTR_SIZE_MASK	0x7C000000L
+//XPB_RTR_DEST_MAP9
+#define XPB_RTR_DEST_MAP9__NMR__SHIFT	0x0
+#define XPB_RTR_DEST_MAP9__DEST_OFFSET__SHIFT	0x1
+#define XPB_RTR_DEST_MAP9__DEST_SEL__SHIFT	0x14
+#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_RTR_DEST_MAP9__APRTR_SIZE__SHIFT	0x1a
+#define XPB_RTR_DEST_MAP9__NMR_MASK	0x00000001L
+#define XPB_RTR_DEST_MAP9__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_RTR_DEST_MAP9__DEST_SEL_MASK	0x00F00000L
+#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_RTR_DEST_MAP9__APRTR_SIZE_MASK	0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP0
+#define XPB_XDMA_RTR_DEST_MAP0__NMR__SHIFT	0x0
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET__SHIFT	0x1
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL__SHIFT	0x14
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE__SHIFT	0x1a
+#define XPB_XDMA_RTR_DEST_MAP0__NMR_MASK	0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_MASK	0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE_MASK	0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP1
+#define XPB_XDMA_RTR_DEST_MAP1__NMR__SHIFT	0x0
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET__SHIFT	0x1
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL__SHIFT	0x14
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE__SHIFT	0x1a
+#define XPB_XDMA_RTR_DEST_MAP1__NMR_MASK	0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_MASK	0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE_MASK	0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP2
+#define XPB_XDMA_RTR_DEST_MAP2__NMR__SHIFT	0x0
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET__SHIFT	0x1
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL__SHIFT	0x14
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE__SHIFT	0x1a
+#define XPB_XDMA_RTR_DEST_MAP2__NMR_MASK	0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_MASK	0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE_MASK	0x7C000000L
+//XPB_XDMA_RTR_DEST_MAP3
+#define XPB_XDMA_RTR_DEST_MAP3__NMR__SHIFT	0x0
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET__SHIFT	0x1
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL__SHIFT	0x14
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT	0x18
+#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE__SHIFT	0x1a
+#define XPB_XDMA_RTR_DEST_MAP3__NMR_MASK	0x00000001L
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET_MASK	0x000FFFFEL
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_MASK	0x00F00000L
+#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB_MASK	0x01000000L
+#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE_MASK	0x7C000000L
+//XPB_CLG_CFG0
+#define XPB_CLG_CFG0__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG0__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG0__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG0__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG0__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG0__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_CFG1
+#define XPB_CLG_CFG1__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG1__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG1__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG1__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG1__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG1__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_CFG2
+#define XPB_CLG_CFG2__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG2__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG2__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG2__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG2__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG2__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_CFG3
+#define XPB_CLG_CFG3__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG3__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG3__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG3__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG3__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG3__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_CFG4
+#define XPB_CLG_CFG4__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG4__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG4__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG4__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG4__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG4__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_CFG5
+#define XPB_CLG_CFG5__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG5__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG5__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG5__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG5__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG5__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_CFG6
+#define XPB_CLG_CFG6__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG6__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG6__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG6__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG6__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG6__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_CFG7
+#define XPB_CLG_CFG7__WCB_NUM__SHIFT	0x0
+#define XPB_CLG_CFG7__P2P_BAR__SHIFT	0x7
+#define XPB_CLG_CFG7__HOST_FLUSH__SHIFT	0xa
+#define XPB_CLG_CFG7__WCB_NUM_MASK	0x0000000FL
+#define XPB_CLG_CFG7__P2P_BAR_MASK	0x00000380L
+#define XPB_CLG_CFG7__HOST_FLUSH_MASK	0x00003C00L
+//XPB_CLG_EXTRA
+#define XPB_CLG_EXTRA__CMP0_HIGH__SHIFT	0x0
+#define XPB_CLG_EXTRA__CMP0_LOW__SHIFT	0x6
+#define XPB_CLG_EXTRA__VLD0__SHIFT	0xb
+#define XPB_CLG_EXTRA__CLG0_NUM__SHIFT	0xc
+#define XPB_CLG_EXTRA__CMP1_HIGH__SHIFT	0xf
+#define XPB_CLG_EXTRA__CMP1_LOW__SHIFT	0x15
+#define XPB_CLG_EXTRA__VLD1__SHIFT	0x1a
+#define XPB_CLG_EXTRA__CLG1_NUM__SHIFT	0x1b
+#define XPB_CLG_EXTRA__CMP0_HIGH_MASK	0x0000003FL
+#define XPB_CLG_EXTRA__CMP0_LOW_MASK	0x000007C0L
+#define XPB_CLG_EXTRA__VLD0_MASK	0x00000800L
+#define XPB_CLG_EXTRA__CLG0_NUM_MASK	0x00007000L
+#define XPB_CLG_EXTRA__CMP1_HIGH_MASK	0x001F8000L
+#define XPB_CLG_EXTRA__CMP1_LOW_MASK	0x03E00000L
+#define XPB_CLG_EXTRA__VLD1_MASK	0x04000000L
+#define XPB_CLG_EXTRA__CLG1_NUM_MASK	0x38000000L
+//XPB_CLG_EXTRA_MSK
+#define XPB_CLG_EXTRA_MSK__MSK0_HIGH__SHIFT	0x0
+#define XPB_CLG_EXTRA_MSK__MSK0_LOW__SHIFT	0x6
+#define XPB_CLG_EXTRA_MSK__MSK1_HIGH__SHIFT	0xb
+#define XPB_CLG_EXTRA_MSK__MSK1_LOW__SHIFT	0x11
+#define XPB_CLG_EXTRA_MSK__MSK0_HIGH_MASK	0x0000003FL
+#define XPB_CLG_EXTRA_MSK__MSK0_LOW_MASK	0x000007C0L
+#define XPB_CLG_EXTRA_MSK__MSK1_HIGH_MASK	0x0001F800L
+#define XPB_CLG_EXTRA_MSK__MSK1_LOW_MASK	0x003E0000L
+//XPB_LB_ADDR
+#define XPB_LB_ADDR__CMP0__SHIFT	0x0
+#define XPB_LB_ADDR__MASK0__SHIFT	0xa
+#define XPB_LB_ADDR__CMP1__SHIFT	0x14
+#define XPB_LB_ADDR__MASK1__SHIFT	0x1a
+#define XPB_LB_ADDR__CMP0_MASK	0x000003FFL
+#define XPB_LB_ADDR__MASK0_MASK	0x000FFC00L
+#define XPB_LB_ADDR__CMP1_MASK	0x03F00000L
+#define XPB_LB_ADDR__MASK1_MASK	0xFC000000L
+//XPB_WCB_STS
+#define XPB_WCB_STS__PBUF_VLD__SHIFT	0x0
+#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT__SHIFT	0x10
+#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT__SHIFT	0x17
+#define XPB_WCB_STS__PBUF_VLD_MASK	0x0000FFFFL
+#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT_MASK	0x007F0000L
+#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT_MASK	0x3F800000L
+//XPB_HST_CFG
+#define XPB_HST_CFG__BAR_UP_WR_CMD__SHIFT	0x0
+#define XPB_HST_CFG__BAR_UP_WR_CMD_MASK	0x00000001L
+//XPB_P2P_BAR_CFG
+#define XPB_P2P_BAR_CFG__ADDR_SIZE__SHIFT	0x0
+#define XPB_P2P_BAR_CFG__SEND_BAR__SHIFT	0x4
+#define XPB_P2P_BAR_CFG__SNOOP__SHIFT	0x6
+#define XPB_P2P_BAR_CFG__SEND_DIS__SHIFT	0x7
+#define XPB_P2P_BAR_CFG__COMPRESS_DIS__SHIFT	0x8
+#define XPB_P2P_BAR_CFG__UPDATE_DIS__SHIFT	0x9
+#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR__SHIFT	0xa
+#define XPB_P2P_BAR_CFG__RD_EN__SHIFT	0xb
+#define XPB_P2P_BAR_CFG__ATC_TRANSLATED__SHIFT	0xc
+#define XPB_P2P_BAR_CFG__ADDR_SIZE_MASK	0x0000000FL
+#define XPB_P2P_BAR_CFG__SEND_BAR_MASK	0x00000030L
+#define XPB_P2P_BAR_CFG__SNOOP_MASK	0x00000040L
+#define XPB_P2P_BAR_CFG__SEND_DIS_MASK	0x00000080L
+#define XPB_P2P_BAR_CFG__COMPRESS_DIS_MASK	0x00000100L
+#define XPB_P2P_BAR_CFG__UPDATE_DIS_MASK	0x00000200L
+#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR_MASK	0x00000400L
+#define XPB_P2P_BAR_CFG__RD_EN_MASK	0x00000800L
+#define XPB_P2P_BAR_CFG__ATC_TRANSLATED_MASK	0x00001000L
+//XPB_P2P_BAR0
+#define XPB_P2P_BAR0__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR0__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR0__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR0__VALID__SHIFT	0xc
+#define XPB_P2P_BAR0__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR0__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR0__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR0__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR0__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR0__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR0__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR0__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR0__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR0__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR0__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR0__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR1
+#define XPB_P2P_BAR1__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR1__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR1__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR1__VALID__SHIFT	0xc
+#define XPB_P2P_BAR1__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR1__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR1__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR1__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR1__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR1__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR1__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR1__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR1__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR1__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR1__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR1__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR2
+#define XPB_P2P_BAR2__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR2__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR2__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR2__VALID__SHIFT	0xc
+#define XPB_P2P_BAR2__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR2__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR2__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR2__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR2__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR2__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR2__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR2__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR2__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR2__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR2__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR2__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR3
+#define XPB_P2P_BAR3__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR3__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR3__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR3__VALID__SHIFT	0xc
+#define XPB_P2P_BAR3__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR3__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR3__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR3__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR3__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR3__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR3__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR3__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR3__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR3__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR3__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR3__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR4
+#define XPB_P2P_BAR4__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR4__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR4__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR4__VALID__SHIFT	0xc
+#define XPB_P2P_BAR4__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR4__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR4__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR4__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR4__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR4__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR4__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR4__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR4__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR4__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR4__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR4__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR5
+#define XPB_P2P_BAR5__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR5__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR5__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR5__VALID__SHIFT	0xc
+#define XPB_P2P_BAR5__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR5__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR5__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR5__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR5__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR5__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR5__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR5__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR5__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR5__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR5__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR5__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR6
+#define XPB_P2P_BAR6__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR6__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR6__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR6__VALID__SHIFT	0xc
+#define XPB_P2P_BAR6__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR6__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR6__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR6__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR6__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR6__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR6__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR6__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR6__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR6__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR6__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR6__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR7
+#define XPB_P2P_BAR7__HOST_FLUSH__SHIFT	0x0
+#define XPB_P2P_BAR7__REG_SYS_BAR__SHIFT	0x4
+#define XPB_P2P_BAR7__MEM_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR7__VALID__SHIFT	0xc
+#define XPB_P2P_BAR7__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR7__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR7__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR7__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR7__HOST_FLUSH_MASK	0x0000000FL
+#define XPB_P2P_BAR7__REG_SYS_BAR_MASK	0x000000F0L
+#define XPB_P2P_BAR7__MEM_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR7__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR7__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR7__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR7__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR7__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR_SETUP
+#define XPB_P2P_BAR_SETUP__SEL__SHIFT	0x0
+#define XPB_P2P_BAR_SETUP__REG_SYS_BAR__SHIFT	0x8
+#define XPB_P2P_BAR_SETUP__VALID__SHIFT	0xc
+#define XPB_P2P_BAR_SETUP__SEND_DIS__SHIFT	0xd
+#define XPB_P2P_BAR_SETUP__COMPRESS_DIS__SHIFT	0xe
+#define XPB_P2P_BAR_SETUP__RESERVED__SHIFT	0xf
+#define XPB_P2P_BAR_SETUP__ADDRESS__SHIFT	0x10
+#define XPB_P2P_BAR_SETUP__SEL_MASK	0x000000FFL
+#define XPB_P2P_BAR_SETUP__REG_SYS_BAR_MASK	0x00000F00L
+#define XPB_P2P_BAR_SETUP__VALID_MASK	0x00001000L
+#define XPB_P2P_BAR_SETUP__SEND_DIS_MASK	0x00002000L
+#define XPB_P2P_BAR_SETUP__COMPRESS_DIS_MASK	0x00004000L
+#define XPB_P2P_BAR_SETUP__RESERVED_MASK	0x00008000L
+#define XPB_P2P_BAR_SETUP__ADDRESS_MASK	0xFFFF0000L
+//XPB_P2P_BAR_DELTA_ABOVE
+#define XPB_P2P_BAR_DELTA_ABOVE__EN__SHIFT	0x0
+#define XPB_P2P_BAR_DELTA_ABOVE__DELTA__SHIFT	0x8
+#define XPB_P2P_BAR_DELTA_ABOVE__EN_MASK	0x000000FFL
+#define XPB_P2P_BAR_DELTA_ABOVE__DELTA_MASK	0x0FFFFF00L
+//XPB_P2P_BAR_DELTA_BELOW
+#define XPB_P2P_BAR_DELTA_BELOW__EN__SHIFT	0x0
+#define XPB_P2P_BAR_DELTA_BELOW__DELTA__SHIFT	0x8
+#define XPB_P2P_BAR_DELTA_BELOW__EN_MASK	0x000000FFL
+#define XPB_P2P_BAR_DELTA_BELOW__DELTA_MASK	0x0FFFFF00L
+//XPB_PEER_SYS_BAR0
+#define XPB_PEER_SYS_BAR0__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR0__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR0__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR0__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR1
+#define XPB_PEER_SYS_BAR1__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR1__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR1__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR1__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR2
+#define XPB_PEER_SYS_BAR2__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR2__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR2__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR2__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR3
+#define XPB_PEER_SYS_BAR3__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR3__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR3__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR3__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR4
+#define XPB_PEER_SYS_BAR4__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR4__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR4__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR4__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR5
+#define XPB_PEER_SYS_BAR5__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR5__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR5__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR5__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR6
+#define XPB_PEER_SYS_BAR6__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR6__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR6__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR6__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR7
+#define XPB_PEER_SYS_BAR7__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR7__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR7__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR7__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR8
+#define XPB_PEER_SYS_BAR8__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR8__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR8__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR8__ADDR_MASK	0xFFFFFFFEL
+//XPB_PEER_SYS_BAR9
+#define XPB_PEER_SYS_BAR9__VALID__SHIFT	0x0
+#define XPB_PEER_SYS_BAR9__ADDR__SHIFT	0x1
+#define XPB_PEER_SYS_BAR9__VALID_MASK	0x00000001L
+#define XPB_PEER_SYS_BAR9__ADDR_MASK	0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR0
+#define XPB_XDMA_PEER_SYS_BAR0__VALID__SHIFT	0x0
+#define XPB_XDMA_PEER_SYS_BAR0__ADDR__SHIFT	0x1
+#define XPB_XDMA_PEER_SYS_BAR0__VALID_MASK	0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR0__ADDR_MASK	0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR1
+#define XPB_XDMA_PEER_SYS_BAR1__VALID__SHIFT	0x0
+#define XPB_XDMA_PEER_SYS_BAR1__ADDR__SHIFT	0x1
+#define XPB_XDMA_PEER_SYS_BAR1__VALID_MASK	0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR1__ADDR_MASK	0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR2
+#define XPB_XDMA_PEER_SYS_BAR2__VALID__SHIFT	0x0
+#define XPB_XDMA_PEER_SYS_BAR2__ADDR__SHIFT	0x1
+#define XPB_XDMA_PEER_SYS_BAR2__VALID_MASK	0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR2__ADDR_MASK	0xFFFFFFFEL
+//XPB_XDMA_PEER_SYS_BAR3
+#define XPB_XDMA_PEER_SYS_BAR3__VALID__SHIFT	0x0
+#define XPB_XDMA_PEER_SYS_BAR3__ADDR__SHIFT	0x1
+#define XPB_XDMA_PEER_SYS_BAR3__VALID_MASK	0x00000001L
+#define XPB_XDMA_PEER_SYS_BAR3__ADDR_MASK	0xFFFFFFFEL
+//XPB_CLK_GAT
+#define XPB_CLK_GAT__ONDLY__SHIFT	0x0
+#define XPB_CLK_GAT__OFFDLY__SHIFT	0x6
+#define XPB_CLK_GAT__RDYDLY__SHIFT	0xc
+#define XPB_CLK_GAT__ENABLE__SHIFT	0x12
+#define XPB_CLK_GAT__MEM_LS_ENABLE__SHIFT	0x13
+#define XPB_CLK_GAT__ONDLY_MASK	0x0000003FL
+#define XPB_CLK_GAT__OFFDLY_MASK	0x00000FC0L
+#define XPB_CLK_GAT__RDYDLY_MASK	0x0003F000L
+#define XPB_CLK_GAT__ENABLE_MASK	0x00040000L
+#define XPB_CLK_GAT__MEM_LS_ENABLE_MASK	0x00080000L
+//XPB_INTF_CFG
+#define XPB_INTF_CFG__RPB_WRREQ_CRD__SHIFT	0x0
+#define XPB_INTF_CFG__MC_WRRET_ASK__SHIFT	0x8
+#define XPB_INTF_CFG__XSP_REQ_CRD__SHIFT	0x10
+#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL__SHIFT	0x17
+#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL__SHIFT	0x18
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL__SHIFT	0x19
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL__SHIFT	0x1a
+#define XPB_INTF_CFG__XSP_SNOOP_SEL__SHIFT	0x1b
+#define XPB_INTF_CFG__XSP_SNOOP_VAL__SHIFT	0x1d
+#define XPB_INTF_CFG__XSP_ORDERING_SEL__SHIFT	0x1e
+#define XPB_INTF_CFG__XSP_ORDERING_VAL__SHIFT	0x1f
+#define XPB_INTF_CFG__RPB_WRREQ_CRD_MASK	0x000000FFL
+#define XPB_INTF_CFG__MC_WRRET_ASK_MASK	0x0000FF00L
+#define XPB_INTF_CFG__XSP_REQ_CRD_MASK	0x007F0000L
+#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL_MASK	0x00800000L
+#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL_MASK	0x01000000L
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL_MASK	0x02000000L
+#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL_MASK	0x04000000L
+#define XPB_INTF_CFG__XSP_SNOOP_SEL_MASK	0x18000000L
+#define XPB_INTF_CFG__XSP_SNOOP_VAL_MASK	0x20000000L
+#define XPB_INTF_CFG__XSP_ORDERING_SEL_MASK	0x40000000L
+#define XPB_INTF_CFG__XSP_ORDERING_VAL_MASK	0x80000000L
+//XPB_INTF_STS
+#define XPB_INTF_STS__RPB_WRREQ_CRD__SHIFT	0x0
+#define XPB_INTF_STS__XSP_REQ_CRD__SHIFT	0x8
+#define XPB_INTF_STS__HOP_DATA_BUF_FULL__SHIFT	0xf
+#define XPB_INTF_STS__HOP_ATTR_BUF_FULL__SHIFT	0x10
+#define XPB_INTF_STS__CNS_BUF_FULL__SHIFT	0x11
+#define XPB_INTF_STS__CNS_BUF_BUSY__SHIFT	0x12
+#define XPB_INTF_STS__RPB_RDREQ_CRD__SHIFT	0x13
+#define XPB_INTF_STS__RPB_WRREQ_CRD_MASK	0x000000FFL
+#define XPB_INTF_STS__XSP_REQ_CRD_MASK	0x00007F00L
+#define XPB_INTF_STS__HOP_DATA_BUF_FULL_MASK	0x00008000L
+#define XPB_INTF_STS__HOP_ATTR_BUF_FULL_MASK	0x00010000L
+#define XPB_INTF_STS__CNS_BUF_FULL_MASK	0x00020000L
+#define XPB_INTF_STS__CNS_BUF_BUSY_MASK	0x00040000L
+#define XPB_INTF_STS__RPB_RDREQ_CRD_MASK	0x07F80000L
+//XPB_PIPE_STS
+#define XPB_PIPE_STS__WCB_ANY_PBUF__SHIFT	0x0
+#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT__SHIFT	0x1
+#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT__SHIFT	0x8
+#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL__SHIFT	0xf
+#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL__SHIFT	0x10
+#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL__SHIFT	0x11
+#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL__SHIFT	0x12
+#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL__SHIFT	0x13
+#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL__SHIFT	0x14
+#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL__SHIFT	0x15
+#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL__SHIFT	0x16
+#define XPB_PIPE_STS__RET_BUF_FULL__SHIFT	0x17
+#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS__SHIFT	0x18
+#define XPB_PIPE_STS__WCB_ANY_PBUF_MASK	0x00000001L
+#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT_MASK	0x000000FEL
+#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT_MASK	0x00007F00L
+#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL_MASK	0x00008000L
+#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL_MASK	0x00010000L
+#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL_MASK	0x00020000L
+#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL_MASK	0x00040000L
+#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL_MASK	0x00080000L
+#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL_MASK	0x00100000L
+#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL_MASK	0x00200000L
+#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL_MASK	0x00400000L
+#define XPB_PIPE_STS__RET_BUF_FULL_MASK	0x00800000L
+#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS_MASK	0xFF000000L
+//XPB_SUB_CTRL
+#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB__SHIFT	0x0
+#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ__SHIFT	0x1
+#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ__SHIFT	0x2
+#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ__SHIFT	0x3
+#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ__SHIFT	0x4
+#define XPB_SUB_CTRL__STALL_WCB_SID_REQ__SHIFT	0x5
+#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND__SHIFT	0x6
+#define XPB_SUB_CTRL__STALL_WCB_HST_REQ__SHIFT	0x7
+#define XPB_SUB_CTRL__STALL_HST_HOP_REQ__SHIFT	0x8
+#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR__SHIFT	0x9
+#define XPB_SUB_CTRL__RESET_CNS__SHIFT	0xa
+#define XPB_SUB_CTRL__RESET_RTR__SHIFT	0xb
+#define XPB_SUB_CTRL__RESET_RET__SHIFT	0xc
+#define XPB_SUB_CTRL__RESET_MAP__SHIFT	0xd
+#define XPB_SUB_CTRL__RESET_WCB__SHIFT	0xe
+#define XPB_SUB_CTRL__RESET_HST__SHIFT	0xf
+#define XPB_SUB_CTRL__RESET_HOP__SHIFT	0x10
+#define XPB_SUB_CTRL__RESET_SID__SHIFT	0x11
+#define XPB_SUB_CTRL__RESET_SRB__SHIFT	0x12
+#define XPB_SUB_CTRL__RESET_CGR__SHIFT	0x13
+#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB_MASK	0x00000001L
+#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ_MASK	0x00000002L
+#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ_MASK	0x00000004L
+#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ_MASK	0x00000008L
+#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ_MASK	0x00000010L
+#define XPB_SUB_CTRL__STALL_WCB_SID_REQ_MASK	0x00000020L
+#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND_MASK	0x00000040L
+#define XPB_SUB_CTRL__STALL_WCB_HST_REQ_MASK	0x00000080L
+#define XPB_SUB_CTRL__STALL_HST_HOP_REQ_MASK	0x00000100L
+#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR_MASK	0x00000200L
+#define XPB_SUB_CTRL__RESET_CNS_MASK	0x00000400L
+#define XPB_SUB_CTRL__RESET_RTR_MASK	0x00000800L
+#define XPB_SUB_CTRL__RESET_RET_MASK	0x00001000L
+#define XPB_SUB_CTRL__RESET_MAP_MASK	0x00002000L
+#define XPB_SUB_CTRL__RESET_WCB_MASK	0x00004000L
+#define XPB_SUB_CTRL__RESET_HST_MASK	0x00008000L
+#define XPB_SUB_CTRL__RESET_HOP_MASK	0x00010000L
+#define XPB_SUB_CTRL__RESET_SID_MASK	0x00020000L
+#define XPB_SUB_CTRL__RESET_SRB_MASK	0x00040000L
+#define XPB_SUB_CTRL__RESET_CGR_MASK	0x00080000L
+//XPB_MAP_INVERT_FLUSH_NUM_LSB
+#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM__SHIFT	0x0
+#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM_MASK	0x0000FFFFL
+//XPB_PERF_KNOBS
+#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH__SHIFT	0x0
+#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH__SHIFT	0x6
+#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH__SHIFT	0xc
+#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH_MASK	0x0000003FL
+#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH_MASK	0x00000FC0L
+#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH_MASK	0x0003F000L
+//XPB_STICKY
+#define XPB_STICKY__BITS__SHIFT	0x0
+#define XPB_STICKY__BITS_MASK	0xFFFFFFFFL
+//XPB_STICKY_W1C
+#define XPB_STICKY_W1C__BITS__SHIFT	0x0
+#define XPB_STICKY_W1C__BITS_MASK	0xFFFFFFFFL
+//XPB_MISC_CFG
+#define XPB_MISC_CFG__FIELDNAME0__SHIFT	0x0
+#define XPB_MISC_CFG__FIELDNAME1__SHIFT	0x8
+#define XPB_MISC_CFG__FIELDNAME2__SHIFT	0x10
+#define XPB_MISC_CFG__FIELDNAME3__SHIFT	0x18
+#define XPB_MISC_CFG__TRIGGERNAME__SHIFT	0x1f
+#define XPB_MISC_CFG__FIELDNAME0_MASK	0x000000FFL
+#define XPB_MISC_CFG__FIELDNAME1_MASK	0x0000FF00L
+#define XPB_MISC_CFG__FIELDNAME2_MASK	0x00FF0000L
+#define XPB_MISC_CFG__FIELDNAME3_MASK	0x7F000000L
+#define XPB_MISC_CFG__TRIGGERNAME_MASK	0x80000000L
+//XPB_INTF_CFG2
+#define XPB_INTF_CFG2__RPB_RDREQ_CRD__SHIFT	0x0
+#define XPB_INTF_CFG2__RPB_RDREQ_CRD_MASK	0x000000FFL
+//XPB_CLG_EXTRA_RD
+#define XPB_CLG_EXTRA_RD__CMP0_HIGH__SHIFT	0x0
+#define XPB_CLG_EXTRA_RD__CMP0_LOW__SHIFT	0x6
+#define XPB_CLG_EXTRA_RD__VLD0__SHIFT	0xb
+#define XPB_CLG_EXTRA_RD__CLG0_NUM__SHIFT	0xc
+#define XPB_CLG_EXTRA_RD__CMP1_HIGH__SHIFT	0xf
+#define XPB_CLG_EXTRA_RD__CMP1_LOW__SHIFT	0x15
+#define XPB_CLG_EXTRA_RD__VLD1__SHIFT	0x1a
+#define XPB_CLG_EXTRA_RD__CLG1_NUM__SHIFT	0x1b
+#define XPB_CLG_EXTRA_RD__CMP0_HIGH_MASK	0x0000003FL
+#define XPB_CLG_EXTRA_RD__CMP0_LOW_MASK	0x000007C0L
+#define XPB_CLG_EXTRA_RD__VLD0_MASK	0x00000800L
+#define XPB_CLG_EXTRA_RD__CLG0_NUM_MASK	0x00007000L
+#define XPB_CLG_EXTRA_RD__CMP1_HIGH_MASK	0x001F8000L
+#define XPB_CLG_EXTRA_RD__CMP1_LOW_MASK	0x03E00000L
+#define XPB_CLG_EXTRA_RD__VLD1_MASK	0x04000000L
+#define XPB_CLG_EXTRA_RD__CLG1_NUM_MASK	0x38000000L
+//XPB_CLG_EXTRA_MSK_RD
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH__SHIFT	0x0
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW__SHIFT	0x6
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH__SHIFT	0xb
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW__SHIFT	0x11
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH_MASK	0x0000003FL
+#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW_MASK	0x000007C0L
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH_MASK	0x0001F800L
+#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW_MASK	0x003E0000L
+//XPB_CLG_GFX_MATCH
+#define XPB_CLG_GFX_MATCH__FARBIRC0_ID__SHIFT	0x0
+#define XPB_CLG_GFX_MATCH__FARBIRC1_ID__SHIFT	0x6
+#define XPB_CLG_GFX_MATCH__FARBIRC2_ID__SHIFT	0xc
+#define XPB_CLG_GFX_MATCH__FARBIRC3_ID__SHIFT	0x12
+#define XPB_CLG_GFX_MATCH__FARBIRC0_VLD__SHIFT	0x18
+#define XPB_CLG_GFX_MATCH__FARBIRC1_VLD__SHIFT	0x19
+#define XPB_CLG_GFX_MATCH__FARBIRC2_VLD__SHIFT	0x1a
+#define XPB_CLG_GFX_MATCH__FARBIRC3_VLD__SHIFT	0x1b
+#define XPB_CLG_GFX_MATCH__FARBIRC0_ID_MASK	0x0000003FL
+#define XPB_CLG_GFX_MATCH__FARBIRC1_ID_MASK	0x00000FC0L
+#define XPB_CLG_GFX_MATCH__FARBIRC2_ID_MASK	0x0003F000L
+#define XPB_CLG_GFX_MATCH__FARBIRC3_ID_MASK	0x00FC0000L
+#define XPB_CLG_GFX_MATCH__FARBIRC0_VLD_MASK	0x01000000L
+#define XPB_CLG_GFX_MATCH__FARBIRC1_VLD_MASK	0x02000000L
+#define XPB_CLG_GFX_MATCH__FARBIRC2_VLD_MASK	0x04000000L
+#define XPB_CLG_GFX_MATCH__FARBIRC3_VLD_MASK	0x08000000L
+//XPB_CLG_GFX_MATCH_MSK
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT	0x0
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT	0x6
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK__SHIFT	0xc
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK__SHIFT	0x12
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK_MASK	0x0000003FL
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK_MASK	0x00000FC0L
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK_MASK	0x0003F000L
+#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK_MASK	0x00FC0000L
+//XPB_CLG_MM_MATCH
+#define XPB_CLG_MM_MATCH__FARBIRC0_ID__SHIFT	0x0
+#define XPB_CLG_MM_MATCH__FARBIRC1_ID__SHIFT	0x6
+#define XPB_CLG_MM_MATCH__FARBIRC2_ID__SHIFT	0xc
+#define XPB_CLG_MM_MATCH__FARBIRC3_ID__SHIFT	0x12
+#define XPB_CLG_MM_MATCH__FARBIRC0_VLD__SHIFT	0x18
+#define XPB_CLG_MM_MATCH__FARBIRC1_VLD__SHIFT	0x19
+#define XPB_CLG_MM_MATCH__FARBIRC2_VLD__SHIFT	0x1a
+#define XPB_CLG_MM_MATCH__FARBIRC3_VLD__SHIFT	0x1b
+#define XPB_CLG_MM_MATCH__FARBIRC0_ID_MASK	0x0000003FL
+#define XPB_CLG_MM_MATCH__FARBIRC1_ID_MASK	0x00000FC0L
+#define XPB_CLG_MM_MATCH__FARBIRC2_ID_MASK	0x0003F000L
+#define XPB_CLG_MM_MATCH__FARBIRC3_ID_MASK	0x00FC0000L
+#define XPB_CLG_MM_MATCH__FARBIRC0_VLD_MASK	0x01000000L
+#define XPB_CLG_MM_MATCH__FARBIRC1_VLD_MASK	0x02000000L
+#define XPB_CLG_MM_MATCH__FARBIRC2_VLD_MASK	0x04000000L
+#define XPB_CLG_MM_MATCH__FARBIRC3_VLD_MASK	0x08000000L
+//XPB_CLG_MM_MATCH_MSK
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT	0x0
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT	0x6
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC2_ID_MSK__SHIFT	0xc
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC3_ID_MSK__SHIFT	0x12
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK_MASK	0x0000003FL
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK_MASK	0x00000FC0L
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC2_ID_MSK_MASK	0x0003F000L
+#define XPB_CLG_MM_MATCH_MSK__FARBIRC3_ID_MSK_MASK	0x00FC0000L
+//XPB_CLG_GFX_UNITID_MAPPING0
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING1
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING2
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING3
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING4
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING5
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING6
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_GFX_UNITID_MAPPING7
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING0
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING1
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING2
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM_MASK	0x000001C0L
+//XPB_CLG_MM_UNITID_MAPPING3
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW__SHIFT	0x0
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD__SHIFT	0x5
+#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT	0x6
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW_MASK	0x0000001FL
+#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD_MASK	0x00000020L
+#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM_MASK	0x000001C0L
+
+
+// addressBlock: athub_rpbdec
+//RPB_PASSPW_CONF
+#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE__SHIFT	0x0
+#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE__SHIFT	0x1
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE__SHIFT	0x2
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE__SHIFT	0x3
+#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE__SHIFT	0x4
+#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE__SHIFT	0x5
+#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE__SHIFT	0x6
+#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE__SHIFT	0x7
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE__SHIFT	0x8
+#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE__SHIFT	0x9
+#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE__SHIFT	0xa
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN__SHIFT	0xb
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN__SHIFT	0xc
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN__SHIFT	0xd
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE__SHIFT	0xe
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN__SHIFT	0xf
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE__SHIFT	0x10
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN__SHIFT	0x11
+#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE_MASK	0x00000001L
+#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE_MASK	0x00000002L
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_MASK	0x00000004L
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_MASK	0x00000008L
+#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE_MASK	0x00000010L
+#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE_MASK	0x00000020L
+#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE_MASK	0x00000040L
+#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE_MASK	0x00000080L
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_MASK	0x00000100L
+#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE_MASK	0x00000200L
+#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE_MASK	0x00000400L
+#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN_MASK	0x00000800L
+#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN_MASK	0x00001000L
+#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN_MASK	0x00002000L
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_MASK	0x00004000L
+#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN_MASK	0x00008000L
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_MASK	0x00010000L
+#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN_MASK	0x00020000L
+//RPB_BLOCKLEVEL_CONF
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE__SHIFT	0x0
+#define RPB_BLOCKLEVEL_CONF__ATC_TR_BLOCKLEVEL__SHIFT	0x2
+#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL__SHIFT	0x4
+#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL__SHIFT	0x6
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE__SHIFT	0x8
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE__SHIFT	0xa
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE__SHIFT	0xc
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN__SHIFT	0xe
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN__SHIFT	0xf
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN__SHIFT	0x10
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN__SHIFT	0x11
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_MASK	0x00000003L
+#define RPB_BLOCKLEVEL_CONF__ATC_TR_BLOCKLEVEL_MASK	0x0000000CL
+#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL_MASK	0x00000030L
+#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL_MASK	0x000000C0L
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_MASK	0x00000300L
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_MASK	0x00000C00L
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_MASK	0x00003000L
+#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN_MASK	0x00004000L
+#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN_MASK	0x00008000L
+#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN_MASK	0x00010000L
+#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN_MASK	0x00020000L
+//RPB_TAG_CONF
+#define RPB_TAG_CONF__RPB_ATS_TR__SHIFT	0x0
+#define RPB_TAG_CONF__RPB_IO_WR__SHIFT	0x8
+#define RPB_TAG_CONF__RPB_ATS_PR__SHIFT	0x10
+#define RPB_TAG_CONF__RPB_ATS_TR_MASK	0x000000FFL
+#define RPB_TAG_CONF__RPB_IO_WR_MASK	0x0000FF00L
+#define RPB_TAG_CONF__RPB_ATS_PR_MASK	0x00FF0000L
+//RPB_EFF_CNTL
+#define RPB_EFF_CNTL__WR_LAZY_TIMER__SHIFT	0x0
+#define RPB_EFF_CNTL__RD_LAZY_TIMER__SHIFT	0x8
+#define RPB_EFF_CNTL__WR_LAZY_TIMER_MASK	0x000000FFL
+#define RPB_EFF_CNTL__RD_LAZY_TIMER_MASK	0x0000FF00L
+//RPB_ARB_CNTL
+#define RPB_ARB_CNTL__RD_SWITCH_NUM__SHIFT	0x0
+#define RPB_ARB_CNTL__WR_SWITCH_NUM__SHIFT	0x8
+#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM__SHIFT	0x10
+#define RPB_ARB_CNTL__ARB_MODE__SHIFT	0x18
+#define RPB_ARB_CNTL__SWITCH_NUM_MODE__SHIFT	0x19
+#define RPB_ARB_CNTL__RD_SWITCH_NUM_MASK	0x000000FFL
+#define RPB_ARB_CNTL__WR_SWITCH_NUM_MASK	0x0000FF00L
+#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM_MASK	0x00FF0000L
+#define RPB_ARB_CNTL__ARB_MODE_MASK	0x01000000L
+#define RPB_ARB_CNTL__SWITCH_NUM_MODE_MASK	0x02000000L
+//RPB_ARB_CNTL2
+#define RPB_ARB_CNTL2__P2P_SWITCH_NUM__SHIFT	0x0
+#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM__SHIFT	0x8
+#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM__SHIFT	0x10
+#define RPB_ARB_CNTL2__P2P_SWITCH_NUM_MASK	0x000000FFL
+#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM_MASK	0x0000FF00L
+#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM_MASK	0x00FF0000L
+//RPB_BIF_CNTL
+#define RPB_BIF_CNTL__VC0_SWITCH_NUM__SHIFT	0x0
+#define RPB_BIF_CNTL__VC1_SWITCH_NUM__SHIFT	0x8
+#define RPB_BIF_CNTL__ARB_MODE__SHIFT	0x10
+#define RPB_BIF_CNTL__DRAIN_VC_NUM__SHIFT	0x11
+#define RPB_BIF_CNTL__SWITCH_ENABLE__SHIFT	0x12
+#define RPB_BIF_CNTL__SWITCH_THRESHOLD__SHIFT	0x13
+#define RPB_BIF_CNTL__PAGE_PRI_EN__SHIFT	0x1b
+#define RPB_BIF_CNTL__TR_PRI_EN__SHIFT	0x1c
+#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE__SHIFT	0x1d
+#define RPB_BIF_CNTL__PARITY_CHECK_EN__SHIFT	0x1e
+#define RPB_BIF_CNTL__VC0_SWITCH_NUM_MASK	0x000000FFL
+#define RPB_BIF_CNTL__VC1_SWITCH_NUM_MASK	0x0000FF00L
+#define RPB_BIF_CNTL__ARB_MODE_MASK	0x00010000L
+#define RPB_BIF_CNTL__DRAIN_VC_NUM_MASK	0x00020000L
+#define RPB_BIF_CNTL__SWITCH_ENABLE_MASK	0x00040000L
+#define RPB_BIF_CNTL__SWITCH_THRESHOLD_MASK	0x07F80000L
+#define RPB_BIF_CNTL__PAGE_PRI_EN_MASK	0x08000000L
+#define RPB_BIF_CNTL__TR_PRI_EN_MASK	0x10000000L
+#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE_MASK	0x20000000L
+#define RPB_BIF_CNTL__PARITY_CHECK_EN_MASK	0x40000000L
+//RPB_WR_SWITCH_CNTL
+#define RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT	0x0
+#define RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT	0x7
+#define RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT	0xe
+#define RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT	0x15
+#define RPB_WR_SWITCH_CNTL__SWITCH_NUM_MODE__SHIFT	0x1c
+#define RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK	0x0000007FL
+#define RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK	0x00003F80L
+#define RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK	0x001FC000L
+#define RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK	0x0FE00000L
+#define RPB_WR_SWITCH_CNTL__SWITCH_NUM_MODE_MASK	0x10000000L
+//RPB_RD_SWITCH_CNTL
+#define RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT	0x0
+#define RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT	0x7
+#define RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT	0xe
+#define RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT	0x15
+#define RPB_RD_SWITCH_CNTL__SWITCH_NUM_MODE__SHIFT	0x1c
+#define RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK	0x0000007FL
+#define RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK	0x00003F80L
+#define RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK	0x001FC000L
+#define RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK	0x0FE00000L
+#define RPB_RD_SWITCH_CNTL__SWITCH_NUM_MODE_MASK	0x10000000L
+//RPB_CID_QUEUE_WR
+#define RPB_CID_QUEUE_WR__CLIENT_ID_LOW__SHIFT	0x0
+#define RPB_CID_QUEUE_WR__CLIENT_ID_HIGH__SHIFT	0x5
+#define RPB_CID_QUEUE_WR__UPDATE_MODE__SHIFT	0xb
+#define RPB_CID_QUEUE_WR__WRITE_QUEUE__SHIFT	0xc
+#define RPB_CID_QUEUE_WR__READ_QUEUE__SHIFT	0xf
+#define RPB_CID_QUEUE_WR__UPDATE__SHIFT	0x12
+#define RPB_CID_QUEUE_WR__CLIENT_ID_LOW_MASK	0x0000001FL
+#define RPB_CID_QUEUE_WR__CLIENT_ID_HIGH_MASK	0x000007E0L
+#define RPB_CID_QUEUE_WR__UPDATE_MODE_MASK	0x00000800L
+#define RPB_CID_QUEUE_WR__WRITE_QUEUE_MASK	0x00007000L
+#define RPB_CID_QUEUE_WR__READ_QUEUE_MASK	0x00038000L
+#define RPB_CID_QUEUE_WR__UPDATE_MASK	0x00040000L
+//RPB_CID_QUEUE_RD
+#define RPB_CID_QUEUE_RD__CLIENT_ID_LOW__SHIFT	0x0
+#define RPB_CID_QUEUE_RD__CLIENT_ID_HIGH__SHIFT	0x5
+#define RPB_CID_QUEUE_RD__WRITE_QUEUE__SHIFT	0xb
+#define RPB_CID_QUEUE_RD__READ_QUEUE__SHIFT	0xe
+#define RPB_CID_QUEUE_RD__CLIENT_ID_LOW_MASK	0x0000001FL
+#define RPB_CID_QUEUE_RD__CLIENT_ID_HIGH_MASK	0x000007E0L
+#define RPB_CID_QUEUE_RD__WRITE_QUEUE_MASK	0x00003800L
+#define RPB_CID_QUEUE_RD__READ_QUEUE_MASK	0x0001C000L
+//RPB_CID_QUEUE_EX
+#define RPB_CID_QUEUE_EX__START__SHIFT	0x0
+#define RPB_CID_QUEUE_EX__OFFSET__SHIFT	0x1
+#define RPB_CID_QUEUE_EX__START_MASK	0x00000001L
+#define RPB_CID_QUEUE_EX__OFFSET_MASK	0x000001FEL
+//RPB_CID_QUEUE_EX_DATA
+#define RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES__SHIFT	0x0
+#define RPB_CID_QUEUE_EX_DATA__READ_ENTRIES__SHIFT	0x10
+#define RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES_MASK	0x0000FFFFL
+#define RPB_CID_QUEUE_EX_DATA__READ_ENTRIES_MASK	0xFFFF0000L
+//RPB_SWITCH_CNTL2
+#define RPB_SWITCH_CNTL2__RD_QUEUE4_SWITCH_NUM__SHIFT	0x0
+#define RPB_SWITCH_CNTL2__RD_QUEUE5_SWITCH_NUM__SHIFT	0x7
+#define RPB_SWITCH_CNTL2__WR_QUEUE4_SWITCH_NUM__SHIFT	0xe
+#define RPB_SWITCH_CNTL2__WR_QUEUE5_SWITCH_NUM__SHIFT	0x15
+#define RPB_SWITCH_CNTL2__RD_QUEUE4_SWITCH_NUM_MASK	0x0000007FL
+#define RPB_SWITCH_CNTL2__RD_QUEUE5_SWITCH_NUM_MASK	0x00003F80L
+#define RPB_SWITCH_CNTL2__WR_QUEUE4_SWITCH_NUM_MASK	0x001FC000L
+#define RPB_SWITCH_CNTL2__WR_QUEUE5_SWITCH_NUM_MASK	0x0FE00000L
+//RPB_DEINTRLV_COMBINE_CNTL
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER__SHIFT	0x0
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN__SHIFT	0x4
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE__SHIFT	0x5
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER_MASK	0x0000000FL
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN_MASK	0x00000010L
+#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE_MASK	0x00000020L
+//RPB_VC_SWITCH_RDWR
+#define RPB_VC_SWITCH_RDWR__MODE__SHIFT	0x0
+#define RPB_VC_SWITCH_RDWR__NUM_RD__SHIFT	0x2
+#define RPB_VC_SWITCH_RDWR__NUM_WR__SHIFT	0xa
+#define RPB_VC_SWITCH_RDWR__MODE_MASK	0x00000003L
+#define RPB_VC_SWITCH_RDWR__NUM_RD_MASK	0x000003FCL
+#define RPB_VC_SWITCH_RDWR__NUM_WR_MASK	0x0003FC00L
+//RPB_PERFCOUNTER_LO
+#define RPB_PERFCOUNTER_LO__COUNTER_LO__SHIFT	0x0
+#define RPB_PERFCOUNTER_LO__COUNTER_LO_MASK	0xFFFFFFFFL
+//RPB_PERFCOUNTER_HI
+#define RPB_PERFCOUNTER_HI__COUNTER_HI__SHIFT	0x0
+#define RPB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT	0x10
+#define RPB_PERFCOUNTER_HI__COUNTER_HI_MASK	0x0000FFFFL
+#define RPB_PERFCOUNTER_HI__COMPARE_VALUE_MASK	0xFFFF0000L
+//RPB_PERFCOUNTER0_CFG
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT	0x0
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT	0x8
+#define RPB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT	0x18
+#define RPB_PERFCOUNTER0_CFG__ENABLE__SHIFT	0x1c
+#define RPB_PERFCOUNTER0_CFG__CLEAR__SHIFT	0x1d
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_MASK	0x000000FFL
+#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define RPB_PERFCOUNTER0_CFG__PERF_MODE_MASK	0x0F000000L
+#define RPB_PERFCOUNTER0_CFG__ENABLE_MASK	0x10000000L
+#define RPB_PERFCOUNTER0_CFG__CLEAR_MASK	0x20000000L
+//RPB_PERFCOUNTER1_CFG
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT	0x0
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT	0x8
+#define RPB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT	0x18
+#define RPB_PERFCOUNTER1_CFG__ENABLE__SHIFT	0x1c
+#define RPB_PERFCOUNTER1_CFG__CLEAR__SHIFT	0x1d
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_MASK	0x000000FFL
+#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define RPB_PERFCOUNTER1_CFG__PERF_MODE_MASK	0x0F000000L
+#define RPB_PERFCOUNTER1_CFG__ENABLE_MASK	0x10000000L
+#define RPB_PERFCOUNTER1_CFG__CLEAR_MASK	0x20000000L
+//RPB_PERFCOUNTER2_CFG
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT	0x0
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT	0x8
+#define RPB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT	0x18
+#define RPB_PERFCOUNTER2_CFG__ENABLE__SHIFT	0x1c
+#define RPB_PERFCOUNTER2_CFG__CLEAR__SHIFT	0x1d
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_MASK	0x000000FFL
+#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define RPB_PERFCOUNTER2_CFG__PERF_MODE_MASK	0x0F000000L
+#define RPB_PERFCOUNTER2_CFG__ENABLE_MASK	0x10000000L
+#define RPB_PERFCOUNTER2_CFG__CLEAR_MASK	0x20000000L
+//RPB_PERFCOUNTER3_CFG
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT	0x0
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT	0x8
+#define RPB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT	0x18
+#define RPB_PERFCOUNTER3_CFG__ENABLE__SHIFT	0x1c
+#define RPB_PERFCOUNTER3_CFG__CLEAR__SHIFT	0x1d
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_MASK	0x000000FFL
+#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK	0x0000FF00L
+#define RPB_PERFCOUNTER3_CFG__PERF_MODE_MASK	0x0F000000L
+#define RPB_PERFCOUNTER3_CFG__ENABLE_MASK	0x10000000L
+#define RPB_PERFCOUNTER3_CFG__CLEAR_MASK	0x20000000L
+//RPB_PERFCOUNTER_RSLT_CNTL
+#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT	0x0
+#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT	0x8
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT	0x10
+#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT	0x18
+#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT	0x19
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT	0x1a
+#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK	0x0000000FL
+#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK	0x0000FF00L
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK	0x00FF0000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK	0x01000000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK	0x02000000L
+#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK	0x04000000L
+//RPB_RD_QUEUE_CNTL
+#define RPB_RD_QUEUE_CNTL__ARB_MODE__SHIFT	0x0
+#define RPB_RD_QUEUE_CNTL__Q4_SHARED__SHIFT	0x1
+#define RPB_RD_QUEUE_CNTL__Q5_SHARED__SHIFT	0x2
+#define RPB_RD_QUEUE_CNTL__Q4_UNITID_EA_MODE__SHIFT	0x3
+#define RPB_RD_QUEUE_CNTL__Q5_UNITID_EA_MODE__SHIFT	0x4
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_LOW__SHIFT	0x5
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_HIGH__SHIFT	0xa
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_LOW__SHIFT	0x10
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_HIGH__SHIFT	0x15
+#define RPB_RD_QUEUE_CNTL__ARB_MODE_MASK	0x00000001L
+#define RPB_RD_QUEUE_CNTL__Q4_SHARED_MASK	0x00000002L
+#define RPB_RD_QUEUE_CNTL__Q5_SHARED_MASK	0x00000004L
+#define RPB_RD_QUEUE_CNTL__Q4_UNITID_EA_MODE_MASK	0x00000008L
+#define RPB_RD_QUEUE_CNTL__Q5_UNITID_EA_MODE_MASK	0x00000010L
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_LOW_MASK	0x000003E0L
+#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_HIGH_MASK	0x0000FC00L
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_LOW_MASK	0x001F0000L
+#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_HIGH_MASK	0x07E00000L
+//RPB_RD_QUEUE_CNTL2
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW__SHIFT	0x0
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH__SHIFT	0x5
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW__SHIFT	0xb
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH__SHIFT	0x10
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW_MASK	0x0000001FL
+#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH_MASK	0x000007E0L
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW_MASK	0x0000F800L
+#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH_MASK	0x003F0000L
+//RPB_WR_QUEUE_CNTL
+#define RPB_WR_QUEUE_CNTL__ARB_MODE__SHIFT	0x0
+#define RPB_WR_QUEUE_CNTL__Q4_SHARED__SHIFT	0x1
+#define RPB_WR_QUEUE_CNTL__Q5_SHARED__SHIFT	0x2
+#define RPB_WR_QUEUE_CNTL__Q4_UNITID_EA_MODE__SHIFT	0x3
+#define RPB_WR_QUEUE_CNTL__Q5_UNITID_EA_MODE__SHIFT	0x4
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_LOW__SHIFT	0x5
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_HIGH__SHIFT	0xa
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_LOW__SHIFT	0x10
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_HIGH__SHIFT	0x15
+#define RPB_WR_QUEUE_CNTL__ARB_MODE_MASK	0x00000001L
+#define RPB_WR_QUEUE_CNTL__Q4_SHARED_MASK	0x00000002L
+#define RPB_WR_QUEUE_CNTL__Q5_SHARED_MASK	0x00000004L
+#define RPB_WR_QUEUE_CNTL__Q4_UNITID_EA_MODE_MASK	0x00000008L
+#define RPB_WR_QUEUE_CNTL__Q5_UNITID_EA_MODE_MASK	0x00000010L
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_LOW_MASK	0x000003E0L
+#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_HIGH_MASK	0x0000FC00L
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_LOW_MASK	0x001F0000L
+#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_HIGH_MASK	0x07E00000L
+//RPB_WR_QUEUE_CNTL2
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW__SHIFT	0x0
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH__SHIFT	0x5
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW__SHIFT	0xb
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH__SHIFT	0x10
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW_MASK	0x0000001FL
+#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH_MASK	0x000007E0L
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW_MASK	0x0000F800L
+#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH_MASK	0x003F0000L
+//RPB_EA_QUEUE_WR
+#define RPB_EA_QUEUE_WR__EA_NUMBER__SHIFT	0x0
+#define RPB_EA_QUEUE_WR__WRITE_QUEUE__SHIFT	0x5
+#define RPB_EA_QUEUE_WR__READ_QUEUE__SHIFT	0x8
+#define RPB_EA_QUEUE_WR__UPDATE__SHIFT	0xb
+#define RPB_EA_QUEUE_WR__EA_NUMBER_MASK	0x0000001FL
+#define RPB_EA_QUEUE_WR__WRITE_QUEUE_MASK	0x000000E0L
+#define RPB_EA_QUEUE_WR__READ_QUEUE_MASK	0x00000700L
+#define RPB_EA_QUEUE_WR__UPDATE_MASK	0x00000800L
+//RPB_ATS_CNTL
+#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE__SHIFT	0x0
+#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE__SHIFT	0x1
+#define RPB_ATS_CNTL__SWITCH_THRESHOLD__SHIFT	0x2
+#define RPB_ATS_CNTL__TIME_SLICE__SHIFT	0x7
+#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM__SHIFT	0xf
+#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM__SHIFT	0x13
+#define RPB_ATS_CNTL__WR_AT__SHIFT	0x17
+#define RPB_ATS_CNTL__INVAL_COM_CMD__SHIFT	0x19
+#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE_MASK	0x00000001L
+#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE_MASK	0x00000002L
+#define RPB_ATS_CNTL__SWITCH_THRESHOLD_MASK	0x0000007CL
+#define RPB_ATS_CNTL__TIME_SLICE_MASK	0x00007F80L
+#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM_MASK	0x00078000L
+#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM_MASK	0x00780000L
+#define RPB_ATS_CNTL__WR_AT_MASK	0x01800000L
+#define RPB_ATS_CNTL__INVAL_COM_CMD_MASK	0x7E000000L
+//RPB_ATS_CNTL2
+#define RPB_ATS_CNTL2__TRANS_CMD__SHIFT	0x0
+#define RPB_ATS_CNTL2__PAGE_REQ_CMD__SHIFT	0x6
+#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE__SHIFT	0xc
+#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE__SHIFT	0xf
+#define RPB_ATS_CNTL2__VENDOR_ID__SHIFT	0x12
+#define RPB_ATS_CNTL2__TRANS_CMD_MASK	0x0000003FL
+#define RPB_ATS_CNTL2__PAGE_REQ_CMD_MASK	0x00000FC0L
+#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE_MASK	0x00007000L
+#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE_MASK	0x00038000L
+#define RPB_ATS_CNTL2__VENDOR_ID_MASK	0x000C0000L
+//RPB_SDPPORT_CNTL
+#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE__SHIFT	0x0
+#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE__SHIFT	0x1
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT__SHIFT	0x3
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER__SHIFT	0x4
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS__SHIFT	0x5
+#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD__SHIFT	0x6
+#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE__SHIFT	0xa
+#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE__SHIFT	0xb
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT__SHIFT	0xd
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER__SHIFT	0xe
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS__SHIFT	0xf
+#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD__SHIFT	0x10
+#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE__SHIFT	0x14
+#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK__SHIFT	0x15
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN__SHIFT	0x16
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV__SHIFT	0x17
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN__SHIFT	0x18
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV__SHIFT	0x19
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN__SHIFT	0x1a
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV__SHIFT	0x1b
+#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE_MASK	0x00000001L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE_MASK	0x00000006L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT_MASK	0x00000008L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER_MASK	0x00000010L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS_MASK	0x00000020L
+#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD_MASK	0x000003C0L
+#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE_MASK	0x00000400L
+#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE_MASK	0x00001800L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT_MASK	0x00002000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER_MASK	0x00004000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS_MASK	0x00008000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD_MASK	0x000F0000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE_MASK	0x00100000L
+#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK_MASK	0x00200000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN_MASK	0x00400000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV_MASK	0x00800000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN_MASK	0x01000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV_MASK	0x02000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN_MASK	0x04000000L
+#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV_MASK	0x08000000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/dce/dce_12_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_default.h
similarity index 99%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_default.h
index 663d3af35baf..5bf84c6d0ec3 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_default.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_default.h
@@ -436,7 +436,6 @@
 #define mmTA_CNTL_DEFAULT                                                        0x8004d850
 #define mmTA_CNTL_AUX_DEFAULT                                                    0x00000000
 #define mmTA_RESERVED_010C_DEFAULT                                               0x00000000
-#define mmTA_GRAD_ADJ_DEFAULT                                                    0x40000040
 #define mmTA_STATUS_DEFAULT                                                      0x00000000
 #define mmTA_SCRATCH_DEFAULT                                                     0x00000000
 
@@ -1700,7 +1699,6 @@
 #define mmDB_STENCIL_WRITE_BASE_DEFAULT                                          0x00000000
 #define mmDB_STENCIL_WRITE_BASE_HI_DEFAULT                                       0x00000000
 #define mmDB_DFSM_CONTROL_DEFAULT                                                0x00000000
-#define mmDB_RENDER_FILTER_DEFAULT                                               0x00000000
 #define mmDB_Z_INFO2_DEFAULT                                                     0x00000000
 #define mmDB_STENCIL_INFO2_DEFAULT                                               0x00000000
 #define mmTA_BC_BASE_ADDR_DEFAULT                                                0x00000000
@@ -1806,8 +1804,6 @@
 #define mmPA_SC_RIGHT_VERT_GRID_DEFAULT                                          0x00000000
 #define mmPA_SC_LEFT_VERT_GRID_DEFAULT                                           0x00000000
 #define mmPA_SC_HORIZ_GRID_DEFAULT                                               0x00000000
-#define mmPA_SC_FOV_WINDOW_LR_DEFAULT                                            0x00000000
-#define mmPA_SC_FOV_WINDOW_TB_DEFAULT                                            0x00000000
 #define mmVGT_MULTI_PRIM_IB_RESET_INDX_DEFAULT                                   0x00000000
 #define mmCB_BLEND_RED_DEFAULT                                                   0x00000000
 #define mmCB_BLEND_GREEN_DEFAULT                                                 0x00000000
@@ -2072,7 +2068,6 @@
 #define mmVGT_EVENT_INITIATOR_DEFAULT                                            0x00000000
 #define mmVGT_GS_MAX_PRIMS_PER_SUBGROUP_DEFAULT                                  0x00000000
 #define mmVGT_DRAW_PAYLOAD_CNTL_DEFAULT                                          0x00000000
-#define mmVGT_INDEX_PAYLOAD_CNTL_DEFAULT                                         0x00000000
 #define mmVGT_INSTANCE_STEP_RATE_0_DEFAULT                                       0x00000000
 #define mmVGT_INSTANCE_STEP_RATE_1_DEFAULT                                       0x00000000
 #define mmVGT_ESGS_RING_ITEMSIZE_DEFAULT                                         0x00000000
@@ -2490,7 +2485,6 @@
 #define mmWD_INDEX_BUF_BASE_DEFAULT                                              0x00000000
 #define mmWD_INDEX_BUF_BASE_HI_DEFAULT                                           0x00000000
 #define mmIA_MULTI_VGT_PARAM_DEFAULT                                             0x006000ff
-#define mmVGT_OBJECT_ID_DEFAULT                                                  0x00000000
 #define mmVGT_INSTANCE_BASE_ID_DEFAULT                                           0x00000000
 #define mmPA_SU_LINE_STIPPLE_VALUE_DEFAULT                                       0x00000000
 #define mmPA_SC_LINE_STIPPLE_STATE_DEFAULT                                       0x00000000
@@ -2534,7 +2528,6 @@
 #define mmSQC_WRITEBACK_DEFAULT                                                  0x00000000
 #define mmTA_CS_BC_BASE_ADDR_DEFAULT                                             0x00000000
 #define mmTA_CS_BC_BASE_ADDR_HI_DEFAULT                                          0x00000000
-#define mmTA_GRAD_ADJ_UCONFIG_DEFAULT                                            0x40000040
 #define mmDB_OCCLUSION_COUNT0_LOW_DEFAULT                                        0x00000000
 #define mmDB_OCCLUSION_COUNT0_HI_DEFAULT                                         0x00000000
 #define mmDB_OCCLUSION_COUNT1_LOW_DEFAULT                                        0x00000000
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_offset.h
similarity index 99%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_offset.h
index e6d6171aa8b9..4ce090db7ef7 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_offset.h
@@ -841,8 +841,6 @@
 #define mmTA_CNTL_AUX_BASE_IDX                                                                         0
 #define mmTA_RESERVED_010C                                                                             0x0543
 #define mmTA_RESERVED_010C_BASE_IDX                                                                    0
-#define mmTA_GRAD_ADJ                                                                                  0x0544
-#define mmTA_GRAD_ADJ_BASE_IDX                                                                         0
 #define mmTA_STATUS                                                                                    0x0548
 #define mmTA_STATUS_BASE_IDX                                                                           0
 #define mmTA_SCRATCH                                                                                   0x0564
@@ -3330,8 +3328,6 @@
 #define mmDB_STENCIL_WRITE_BASE_HI_BASE_IDX                                                            1
 #define mmDB_DFSM_CONTROL                                                                              0x0018
 #define mmDB_DFSM_CONTROL_BASE_IDX                                                                     1
-#define mmDB_RENDER_FILTER                                                                             0x0019
-#define mmDB_RENDER_FILTER_BASE_IDX                                                                    1
 #define mmDB_Z_INFO2                                                                                   0x001a
 #define mmDB_Z_INFO2_BASE_IDX                                                                          1
 #define mmDB_STENCIL_INFO2                                                                             0x001b
@@ -3542,10 +3538,6 @@
 #define mmPA_SC_LEFT_VERT_GRID_BASE_IDX                                                                1
 #define mmPA_SC_HORIZ_GRID                                                                             0x00ea
 #define mmPA_SC_HORIZ_GRID_BASE_IDX                                                                    1
-#define mmPA_SC_FOV_WINDOW_LR                                                                          0x00eb
-#define mmPA_SC_FOV_WINDOW_LR_BASE_IDX                                                                 1
-#define mmPA_SC_FOV_WINDOW_TB                                                                          0x00ec
-#define mmPA_SC_FOV_WINDOW_TB_BASE_IDX                                                                 1
 #define mmVGT_MULTI_PRIM_IB_RESET_INDX                                                                 0x0103
 #define mmVGT_MULTI_PRIM_IB_RESET_INDX_BASE_IDX                                                        1
 #define mmCB_BLEND_RED                                                                                 0x0105
@@ -4074,8 +4066,6 @@
 #define mmVGT_GS_MAX_PRIMS_PER_SUBGROUP_BASE_IDX                                                       1
 #define mmVGT_DRAW_PAYLOAD_CNTL                                                                        0x02a6
 #define mmVGT_DRAW_PAYLOAD_CNTL_BASE_IDX                                                               1
-#define mmVGT_INDEX_PAYLOAD_CNTL                                                                       0x02a7
-#define mmVGT_INDEX_PAYLOAD_CNTL_BASE_IDX                                                              1
 #define mmVGT_INSTANCE_STEP_RATE_0                                                                     0x02a8
 #define mmVGT_INSTANCE_STEP_RATE_0_BASE_IDX                                                            1
 #define mmVGT_INSTANCE_STEP_RATE_1                                                                     0x02a9
@@ -4908,8 +4898,6 @@
 #define mmWD_INDEX_BUF_BASE_HI_BASE_IDX                                                                1
 #define mmIA_MULTI_VGT_PARAM                                                                           0x2258
 #define mmIA_MULTI_VGT_PARAM_BASE_IDX                                                                  1
-#define mmVGT_OBJECT_ID                                                                                0x2259
-#define mmVGT_OBJECT_ID_BASE_IDX                                                                       1
 #define mmVGT_INSTANCE_BASE_ID                                                                         0x225a
 #define mmVGT_INSTANCE_BASE_ID_BASE_IDX                                                                1
 #define mmPA_SU_LINE_STIPPLE_VALUE                                                                     0x2280
@@ -4996,8 +4984,6 @@
 #define mmTA_CS_BC_BASE_ADDR_BASE_IDX                                                                  1
 #define mmTA_CS_BC_BASE_ADDR_HI                                                                        0x2381
 #define mmTA_CS_BC_BASE_ADDR_HI_BASE_IDX                                                               1
-#define mmTA_GRAD_ADJ_UCONFIG                                                                          0x2382
-#define mmTA_GRAD_ADJ_UCONFIG_BASE_IDX                                                                 1
 #define mmDB_OCCLUSION_COUNT0_LOW                                                                      0x23c0
 #define mmDB_OCCLUSION_COUNT0_LOW_BASE_IDX                                                             1
 #define mmDB_OCCLUSION_COUNT0_HI                                                                       0x23c1
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h
similarity index 99%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h
index 5c5e9b445432..2e1214be67a2 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/GC/gc_9_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_0_sh_mask.h
@@ -4576,15 +4576,6 @@
 //TA_RESERVED_010C
 #define TA_RESERVED_010C__Unused__SHIFT                                                                       0x0
 #define TA_RESERVED_010C__Unused_MASK                                                                         0xFFFFFFFFL
-//TA_GRAD_ADJ
-#define TA_GRAD_ADJ__GRAD_ADJ_0__SHIFT                                                                        0x0
-#define TA_GRAD_ADJ__GRAD_ADJ_1__SHIFT                                                                        0x8
-#define TA_GRAD_ADJ__GRAD_ADJ_2__SHIFT                                                                        0x10
-#define TA_GRAD_ADJ__GRAD_ADJ_3__SHIFT                                                                        0x18
-#define TA_GRAD_ADJ__GRAD_ADJ_0_MASK                                                                          0x000000FFL
-#define TA_GRAD_ADJ__GRAD_ADJ_1_MASK                                                                          0x0000FF00L
-#define TA_GRAD_ADJ__GRAD_ADJ_2_MASK                                                                          0x00FF0000L
-#define TA_GRAD_ADJ__GRAD_ADJ_3_MASK                                                                          0xFF000000L
 //TA_STATUS
 #define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT                                                                     0xc
 #define TA_STATUS__FG_LFIFO_EMPTYB__SHIFT                                                                     0xd
@@ -14459,9 +14450,6 @@
 #define DB_DFSM_CONTROL__PUNCHOUT_MODE_MASK                                                                   0x00000003L
 #define DB_DFSM_CONTROL__POPS_DRAIN_PS_ON_OVERLAP_MASK                                                        0x00000004L
 #define DB_DFSM_CONTROL__DISALLOW_OVERFLOW_MASK                                                               0x00000008L
-//DB_RENDER_FILTER
-#define DB_RENDER_FILTER__PS_INVOKE_MASK__SHIFT                                                               0x0
-#define DB_RENDER_FILTER__PS_INVOKE_MASK_MASK                                                                 0x0000FFFFL
 //DB_Z_INFO2
 #define DB_Z_INFO2__EPITCH__SHIFT                                                                             0x0
 #define DB_Z_INFO2__EPITCH_MASK                                                                               0x0000FFFFL
@@ -14959,11 +14947,9 @@
 #define PA_SC_TILE_STEERING_OVERRIDE__ENABLE__SHIFT                                                           0x0
 #define PA_SC_TILE_STEERING_OVERRIDE__NUM_SE__SHIFT                                                           0x1
 #define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SE__SHIFT                                                    0x5
-#define PA_SC_TILE_STEERING_OVERRIDE__DISABLE_SRBSL_DB_OPTIMIZED_PACKING__SHIFT                               0x8
 #define PA_SC_TILE_STEERING_OVERRIDE__ENABLE_MASK                                                             0x00000001L
 #define PA_SC_TILE_STEERING_OVERRIDE__NUM_SE_MASK                                                             0x00000006L
 #define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SE_MASK                                                      0x00000060L
-#define PA_SC_TILE_STEERING_OVERRIDE__DISABLE_SRBSL_DB_OPTIMIZED_PACKING_MASK                                 0x00000100L
 //CP_PERFMON_CNTX_CNTL
 #define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT                                                           0x1f
 #define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK                                                             0x80000000L
@@ -15003,20 +14989,6 @@
 #define PA_SC_HORIZ_GRID__TOP_HALF_MASK                                                                       0x0000FF00L
 #define PA_SC_HORIZ_GRID__BOT_HALF_MASK                                                                       0x00FF0000L
 #define PA_SC_HORIZ_GRID__BOT_QTR_MASK                                                                        0xFF000000L
-//PA_SC_FOV_WINDOW_LR
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_LEFT__SHIFT                                                         0x0
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_RIGHT__SHIFT                                                        0x8
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_LEFT__SHIFT                                                        0x10
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_RIGHT__SHIFT                                                       0x18
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_LEFT_MASK                                                           0x000000FFL
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_RIGHT_MASK                                                          0x0000FF00L
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_LEFT_MASK                                                          0x00FF0000L
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_RIGHT_MASK                                                         0xFF000000L
-//PA_SC_FOV_WINDOW_TB
-#define PA_SC_FOV_WINDOW_TB__FOV_TOP__SHIFT                                                                   0x0
-#define PA_SC_FOV_WINDOW_TB__FOV_BOT__SHIFT                                                                   0x8
-#define PA_SC_FOV_WINDOW_TB__FOV_TOP_MASK                                                                     0x000000FFL
-#define PA_SC_FOV_WINDOW_TB__FOV_BOT_MASK                                                                     0x0000FF00L
 //VGT_MULTI_PRIM_IB_RESET_INDX
 #define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT                                                       0x0
 #define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK                                                         0xFFFFFFFFL
@@ -17010,13 +16982,11 @@
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                              0x2
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                             0x3
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                         0x4
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__SRBSL_ENABLE__SHIFT                                                     0x5
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE_MASK                                           0x00000001L
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                            0x00000002L
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                0x00000004L
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                               0x00000008L
 #define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                           0x00000010L
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__SRBSL_ENABLE_MASK                                                       0x00000020L
 //PA_CL_OBJPRIM_ID_CNTL
 #define PA_CL_OBJPRIM_ID_CNTL__OBJ_ID_SEL__SHIFT                                                              0x0
 #define PA_CL_OBJPRIM_ID_CNTL__ADD_PIPED_PRIM_ID__SHIFT                                                       0x1
@@ -17345,9 +17315,6 @@
 #define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX_MASK                                                           0x00000002L
 #define VGT_DRAW_PAYLOAD_CNTL__EN_PIPELINE_PRIMID_MASK                                                        0x00000004L
 #define VGT_DRAW_PAYLOAD_CNTL__OBJECT_ID_INST_EN_MASK                                                         0x00000008L
-//VGT_INDEX_PAYLOAD_CNTL
-#define VGT_INDEX_PAYLOAD_CNTL__COMPOUND_INDEX_EN__SHIFT                                                      0x0
-#define VGT_INDEX_PAYLOAD_CNTL__COMPOUND_INDEX_EN_MASK                                                        0x00000001L
 //VGT_INSTANCE_STEP_RATE_0
 #define VGT_INSTANCE_STEP_RATE_0__STEP_RATE__SHIFT                                                            0x0
 #define VGT_INSTANCE_STEP_RATE_0__STEP_RATE_MASK                                                              0xFFFFFFFFL
@@ -19849,9 +19816,6 @@
 #define IA_MULTI_VGT_PARAM__EN_INST_OPT_BASIC_MASK                                                            0x00200000L
 #define IA_MULTI_VGT_PARAM__EN_INST_OPT_ADV_MASK                                                              0x00400000L
 #define IA_MULTI_VGT_PARAM__HW_USE_ONLY_MASK                                                                  0x00800000L
-//VGT_OBJECT_ID
-#define VGT_OBJECT_ID__REG_OBJ_ID__SHIFT                                                                      0x0
-#define VGT_OBJECT_ID__REG_OBJ_ID_MASK                                                                        0xFFFFFFFFL
 //VGT_INSTANCE_BASE_ID
 #define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID__SHIFT                                                         0x0
 #define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID_MASK                                                           0xFFFFFFFFL
@@ -20067,15 +20031,6 @@
 //TA_CS_BC_BASE_ADDR_HI
 #define TA_CS_BC_BASE_ADDR_HI__ADDRESS__SHIFT                                                                 0x0
 #define TA_CS_BC_BASE_ADDR_HI__ADDRESS_MASK                                                                   0x000000FFL
-//TA_GRAD_ADJ_UCONFIG
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_0__SHIFT                                                                0x0
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_1__SHIFT                                                                0x8
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_2__SHIFT                                                                0x10
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_3__SHIFT                                                                0x18
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_0_MASK                                                                  0x000000FFL
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_1_MASK                                                                  0x0000FF00L
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_2_MASK                                                                  0x00FF0000L
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_3_MASK                                                                  0xFF000000L
 //DB_OCCLUSION_COUNT0_LOW
 #define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW__SHIFT                                                             0x0
 #define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_1_offset.h
similarity index 99%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_1_offset.h
index db7ef5ede0e5..030e0020902b 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_9_1_offset.h
@@ -815,8 +815,6 @@
 #define mmTA_CNTL_AUX_BASE_IDX                                                                         0
 #define mmTA_RESERVED_010C                                                                             0x0543
 #define mmTA_RESERVED_010C_BASE_IDX                                                                    0
-#define mmTA_GRAD_ADJ                                                                                  0x0544
-#define mmTA_GRAD_ADJ_BASE_IDX                                                                         0
 #define mmTA_STATUS                                                                                    0x0548
 #define mmTA_STATUS_BASE_IDX                                                                           0
 #define mmTA_SCRATCH                                                                                   0x0564
@@ -3617,8 +3615,6 @@
 #define mmDB_STENCIL_WRITE_BASE_HI_BASE_IDX                                                            1
 #define mmDB_DFSM_CONTROL                                                                              0x0018
 #define mmDB_DFSM_CONTROL_BASE_IDX                                                                     1
-#define mmDB_RENDER_FILTER                                                                             0x0019
-#define mmDB_RENDER_FILTER_BASE_IDX                                                                    1
 #define mmDB_Z_INFO2                                                                                   0x001a
 #define mmDB_Z_INFO2_BASE_IDX                                                                          1
 #define mmDB_STENCIL_INFO2                                                                             0x001b
@@ -3829,10 +3825,6 @@
 #define mmPA_SC_LEFT_VERT_GRID_BASE_IDX                                                                1
 #define mmPA_SC_HORIZ_GRID                                                                             0x00ea
 #define mmPA_SC_HORIZ_GRID_BASE_IDX                                                                    1
-#define mmPA_SC_FOV_WINDOW_LR                                                                          0x00eb
-#define mmPA_SC_FOV_WINDOW_LR_BASE_IDX                                                                 1
-#define mmPA_SC_FOV_WINDOW_TB                                                                          0x00ec
-#define mmPA_SC_FOV_WINDOW_TB_BASE_IDX                                                                 1
 #define mmVGT_MULTI_PRIM_IB_RESET_INDX                                                                 0x0103
 #define mmVGT_MULTI_PRIM_IB_RESET_INDX_BASE_IDX                                                        1
 #define mmCB_BLEND_RED                                                                                 0x0105
@@ -4361,8 +4353,6 @@
 #define mmVGT_GS_MAX_PRIMS_PER_SUBGROUP_BASE_IDX                                                       1
 #define mmVGT_DRAW_PAYLOAD_CNTL                                                                        0x02a6
 #define mmVGT_DRAW_PAYLOAD_CNTL_BASE_IDX                                                               1
-#define mmVGT_INDEX_PAYLOAD_CNTL                                                                       0x02a7
-#define mmVGT_INDEX_PAYLOAD_CNTL_BASE_IDX                                                              1
 #define mmVGT_INSTANCE_STEP_RATE_0                                                                     0x02a8
 #define mmVGT_INSTANCE_STEP_RATE_0_BASE_IDX                                                            1
 #define mmVGT_INSTANCE_STEP_RATE_1                                                                     0x02a9
@@ -5195,8 +5185,6 @@
 #define mmWD_INDEX_BUF_BASE_HI_BASE_IDX                                                                1
 #define mmIA_MULTI_VGT_PARAM                                                                           0x2258
 #define mmIA_MULTI_VGT_PARAM_BASE_IDX                                                                  1
-#define mmVGT_OBJECT_ID                                                                                0x2259
-#define mmVGT_OBJECT_ID_BASE_IDX                                                                       1
 #define mmVGT_INSTANCE_BASE_ID                                                                         0x225a
 #define mmVGT_INSTANCE_BASE_ID_BASE_IDX                                                                1
 #define mmPA_SU_LINE_STIPPLE_VALUE                                                                     0x2280
@@ -5283,8 +5271,6 @@
 #define mmTA_CS_BC_BASE_ADDR_BASE_IDX                                                                  1
 #define mmTA_CS_BC_BASE_ADDR_HI                                                                        0x2381
 #define mmTA_CS_BC_BASE_ADDR_HI_BASE_IDX                                                               1
-#define mmTA_GRAD_ADJ_UCONFIG                                                                          0x2382
-#define mmTA_GRAD_ADJ_UCONFIG_BASE_IDX                                                                 1
 #define mmDB_OCCLUSION_COUNT0_LOW                                                                      0x23c0
 #define mmDB_OCCLUSION_COUNT0_LOW_BASE_IDX                                                             1
 #define mmDB_OCCLUSION_COUNT0_HI                                                                       0x23c1
diff --git a/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_offset.h
new file mode 100644
index 000000000000..94325fc3abd5
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_offset.h
@@ -0,0 +1,209 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _hdp_4_0_OFFSET_HEADER
+#define _hdp_4_0_OFFSET_HEADER
+
+
+
+// addressBlock: hdp_hdpdec
+// base address:	0x3c80
+#define mmHDP_MMHUB_TLVL	0x0000
+#define mmHDP_MMHUB_TLVL_BASE_IDX	0
+#define mmHDP_MMHUB_UNITID	0x0001
+#define mmHDP_MMHUB_UNITID_BASE_IDX	0
+#define mmHDP_NONSURFACE_BASE	0x0040
+#define mmHDP_NONSURFACE_BASE_BASE_IDX	0
+#define mmHDP_NONSURFACE_INFO	0x0041
+#define mmHDP_NONSURFACE_INFO_BASE_IDX	0
+#define mmHDP_NONSURFACE_BASE_HI	0x0042
+#define mmHDP_NONSURFACE_BASE_HI_BASE_IDX	0
+#define mmHDP_NONSURF_FLAGS	0x00c8
+#define mmHDP_NONSURF_FLAGS_BASE_IDX	0
+#define mmHDP_NONSURF_FLAGS_CLR	0x00c9
+#define mmHDP_NONSURF_FLAGS_CLR_BASE_IDX	0
+#define mmHDP_HOST_PATH_CNTL	0x00cc
+#define mmHDP_HOST_PATH_CNTL_BASE_IDX	0
+#define mmHDP_SW_SEMAPHORE	0x00cd
+#define mmHDP_SW_SEMAPHORE_BASE_IDX	0
+#define mmHDP_DEBUG0	0x00ce
+#define mmHDP_DEBUG0_BASE_IDX	0
+#define mmHDP_LAST_SURFACE_HIT	0x00d0
+#define mmHDP_LAST_SURFACE_HIT_BASE_IDX	0
+#define mmHDP_READ_CACHE_INVALIDATE	0x00d1
+#define mmHDP_READ_CACHE_INVALIDATE_BASE_IDX	0
+#define mmHDP_OUTSTANDING_REQ	0x00d2
+#define mmHDP_OUTSTANDING_REQ_BASE_IDX	0
+#define mmHDP_MISC_CNTL	0x00d3
+#define mmHDP_MISC_CNTL_BASE_IDX	0
+#define mmHDP_MEM_POWER_LS	0x00d4
+#define mmHDP_MEM_POWER_LS_BASE_IDX	0
+#define mmHDP_MMHUB_CNTL	0x00d5
+#define mmHDP_MMHUB_CNTL_BASE_IDX	0
+#define mmHDP_EDC_CNT	0x00d6
+#define mmHDP_EDC_CNT_BASE_IDX	0
+#define mmHDP_VERSION	0x00d7
+#define mmHDP_VERSION_BASE_IDX	0
+#define mmHDP_CLK_CNTL	0x00d8
+#define mmHDP_CLK_CNTL_BASE_IDX	0
+#define mmHDP_MEMIO_CNTL	0x00f6
+#define mmHDP_MEMIO_CNTL_BASE_IDX	0
+#define mmHDP_MEMIO_ADDR	0x00f7
+#define mmHDP_MEMIO_ADDR_BASE_IDX	0
+#define mmHDP_MEMIO_STATUS	0x00f8
+#define mmHDP_MEMIO_STATUS_BASE_IDX	0
+#define mmHDP_MEMIO_WR_DATA	0x00f9
+#define mmHDP_MEMIO_WR_DATA_BASE_IDX	0
+#define mmHDP_MEMIO_RD_DATA	0x00fa
+#define mmHDP_MEMIO_RD_DATA_BASE_IDX	0
+#define mmHDP_XDP_DIRECT2HDP_FIRST	0x0100
+#define mmHDP_XDP_DIRECT2HDP_FIRST_BASE_IDX	0
+#define mmHDP_XDP_D2H_FLUSH	0x0101
+#define mmHDP_XDP_D2H_FLUSH_BASE_IDX	0
+#define mmHDP_XDP_D2H_BAR_UPDATE	0x0102
+#define mmHDP_XDP_D2H_BAR_UPDATE_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_3	0x0103
+#define mmHDP_XDP_D2H_RSVD_3_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_4	0x0104
+#define mmHDP_XDP_D2H_RSVD_4_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_5	0x0105
+#define mmHDP_XDP_D2H_RSVD_5_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_6	0x0106
+#define mmHDP_XDP_D2H_RSVD_6_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_7	0x0107
+#define mmHDP_XDP_D2H_RSVD_7_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_8	0x0108
+#define mmHDP_XDP_D2H_RSVD_8_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_9	0x0109
+#define mmHDP_XDP_D2H_RSVD_9_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_10	0x010a
+#define mmHDP_XDP_D2H_RSVD_10_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_11	0x010b
+#define mmHDP_XDP_D2H_RSVD_11_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_12	0x010c
+#define mmHDP_XDP_D2H_RSVD_12_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_13	0x010d
+#define mmHDP_XDP_D2H_RSVD_13_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_14	0x010e
+#define mmHDP_XDP_D2H_RSVD_14_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_15	0x010f
+#define mmHDP_XDP_D2H_RSVD_15_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_16	0x0110
+#define mmHDP_XDP_D2H_RSVD_16_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_17	0x0111
+#define mmHDP_XDP_D2H_RSVD_17_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_18	0x0112
+#define mmHDP_XDP_D2H_RSVD_18_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_19	0x0113
+#define mmHDP_XDP_D2H_RSVD_19_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_20	0x0114
+#define mmHDP_XDP_D2H_RSVD_20_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_21	0x0115
+#define mmHDP_XDP_D2H_RSVD_21_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_22	0x0116
+#define mmHDP_XDP_D2H_RSVD_22_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_23	0x0117
+#define mmHDP_XDP_D2H_RSVD_23_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_24	0x0118
+#define mmHDP_XDP_D2H_RSVD_24_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_25	0x0119
+#define mmHDP_XDP_D2H_RSVD_25_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_26	0x011a
+#define mmHDP_XDP_D2H_RSVD_26_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_27	0x011b
+#define mmHDP_XDP_D2H_RSVD_27_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_28	0x011c
+#define mmHDP_XDP_D2H_RSVD_28_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_29	0x011d
+#define mmHDP_XDP_D2H_RSVD_29_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_30	0x011e
+#define mmHDP_XDP_D2H_RSVD_30_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_31	0x011f
+#define mmHDP_XDP_D2H_RSVD_31_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_32	0x0120
+#define mmHDP_XDP_D2H_RSVD_32_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_33	0x0121
+#define mmHDP_XDP_D2H_RSVD_33_BASE_IDX	0
+#define mmHDP_XDP_D2H_RSVD_34	0x0122
+#define mmHDP_XDP_D2H_RSVD_34_BASE_IDX	0
+#define mmHDP_XDP_DIRECT2HDP_LAST	0x0123
+#define mmHDP_XDP_DIRECT2HDP_LAST_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR_CFG	0x0124
+#define mmHDP_XDP_P2P_BAR_CFG_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_OFFSET	0x0125
+#define mmHDP_XDP_P2P_MBX_OFFSET_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_ADDR0	0x0126
+#define mmHDP_XDP_P2P_MBX_ADDR0_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_ADDR1	0x0127
+#define mmHDP_XDP_P2P_MBX_ADDR1_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_ADDR2	0x0128
+#define mmHDP_XDP_P2P_MBX_ADDR2_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_ADDR3	0x0129
+#define mmHDP_XDP_P2P_MBX_ADDR3_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_ADDR4	0x012a
+#define mmHDP_XDP_P2P_MBX_ADDR4_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_ADDR5	0x012b
+#define mmHDP_XDP_P2P_MBX_ADDR5_BASE_IDX	0
+#define mmHDP_XDP_P2P_MBX_ADDR6	0x012c
+#define mmHDP_XDP_P2P_MBX_ADDR6_BASE_IDX	0
+#define mmHDP_XDP_HDP_MBX_MC_CFG	0x012d
+#define mmHDP_XDP_HDP_MBX_MC_CFG_BASE_IDX	0
+#define mmHDP_XDP_HDP_MC_CFG	0x012e
+#define mmHDP_XDP_HDP_MC_CFG_BASE_IDX	0
+#define mmHDP_XDP_HST_CFG	0x012f
+#define mmHDP_XDP_HST_CFG_BASE_IDX	0
+#define mmHDP_XDP_HDP_IPH_CFG	0x0131
+#define mmHDP_XDP_HDP_IPH_CFG_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR0	0x0134
+#define mmHDP_XDP_P2P_BAR0_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR1	0x0135
+#define mmHDP_XDP_P2P_BAR1_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR2	0x0136
+#define mmHDP_XDP_P2P_BAR2_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR3	0x0137
+#define mmHDP_XDP_P2P_BAR3_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR4	0x0138
+#define mmHDP_XDP_P2P_BAR4_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR5	0x0139
+#define mmHDP_XDP_P2P_BAR5_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR6	0x013a
+#define mmHDP_XDP_P2P_BAR6_BASE_IDX	0
+#define mmHDP_XDP_P2P_BAR7	0x013b
+#define mmHDP_XDP_P2P_BAR7_BASE_IDX	0
+#define mmHDP_XDP_FLUSH_ARMED_STS	0x013c
+#define mmHDP_XDP_FLUSH_ARMED_STS_BASE_IDX	0
+#define mmHDP_XDP_FLUSH_CNTR0_STS	0x013d
+#define mmHDP_XDP_FLUSH_CNTR0_STS_BASE_IDX	0
+#define mmHDP_XDP_BUSY_STS	0x013e
+#define mmHDP_XDP_BUSY_STS_BASE_IDX	0
+#define mmHDP_XDP_STICKY	0x013f
+#define mmHDP_XDP_STICKY_BASE_IDX	0
+#define mmHDP_XDP_CHKN	0x0140
+#define mmHDP_XDP_CHKN_BASE_IDX	0
+#define mmHDP_XDP_BARS_ADDR_39_36	0x0144
+#define mmHDP_XDP_BARS_ADDR_39_36_BASE_IDX	0
+#define mmHDP_XDP_MC_VM_FB_LOCATION_BASE	0x0145
+#define mmHDP_XDP_MC_VM_FB_LOCATION_BASE_BASE_IDX	0
+#define mmHDP_XDP_GPU_IOV_VIOLATION_LOG	0x0148
+#define mmHDP_XDP_GPU_IOV_VIOLATION_LOG_BASE_IDX	0
+#define mmHDP_XDP_MMHUB_ERROR	0x0149
+#define mmHDP_XDP_MMHUB_ERROR_BASE_IDX	0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_sh_mask.h
new file mode 100644
index 000000000000..25e28691d62d
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/hdp/hdp_4_0_sh_mask.h
@@ -0,0 +1,601 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _hdp_4_0_SH_MASK_HEADER
+#define _hdp_4_0_SH_MASK_HEADER
+
+
+// addressBlock: hdp_hdpdec
+//HDP_MMHUB_TLVL
+#define HDP_MMHUB_TLVL__HDP_WR_TLVL__SHIFT	0x0
+#define HDP_MMHUB_TLVL__HDP_RD_TLVL__SHIFT	0x4
+#define HDP_MMHUB_TLVL__XDP_WR_TLVL__SHIFT	0x8
+#define HDP_MMHUB_TLVL__XDP_RD_TLVL__SHIFT	0xc
+#define HDP_MMHUB_TLVL__XDP_MBX_WR_TLVL__SHIFT	0x10
+#define HDP_MMHUB_TLVL__HDP_WR_TLVL_MASK	0x00000007L
+#define HDP_MMHUB_TLVL__HDP_RD_TLVL_MASK	0x00000070L
+#define HDP_MMHUB_TLVL__XDP_WR_TLVL_MASK	0x00000700L
+#define HDP_MMHUB_TLVL__XDP_RD_TLVL_MASK	0x00007000L
+#define HDP_MMHUB_TLVL__XDP_MBX_WR_TLVL_MASK	0x00070000L
+//HDP_MMHUB_UNITID
+#define HDP_MMHUB_UNITID__HDP_UNITID__SHIFT	0x0
+#define HDP_MMHUB_UNITID__XDP_UNITID__SHIFT	0x8
+#define HDP_MMHUB_UNITID__XDP_MBX_UNITID__SHIFT	0x10
+#define HDP_MMHUB_UNITID__HDP_UNITID_MASK	0x0000003FL
+#define HDP_MMHUB_UNITID__XDP_UNITID_MASK	0x00003F00L
+#define HDP_MMHUB_UNITID__XDP_MBX_UNITID_MASK	0x003F0000L
+//HDP_NONSURFACE_BASE
+#define HDP_NONSURFACE_BASE__NONSURF_BASE_39_8__SHIFT	0x0
+#define HDP_NONSURFACE_BASE__NONSURF_BASE_39_8_MASK	0xFFFFFFFFL
+//HDP_NONSURFACE_INFO
+#define HDP_NONSURFACE_INFO__NONSURF_SWAP__SHIFT	0x4
+#define HDP_NONSURFACE_INFO__NONSURF_VMID__SHIFT	0x8
+#define HDP_NONSURFACE_INFO__NONSURF_SWAP_MASK	0x00000030L
+#define HDP_NONSURFACE_INFO__NONSURF_VMID_MASK	0x00000F00L
+//HDP_NONSURFACE_BASE_HI
+#define HDP_NONSURFACE_BASE_HI__NONSURF_BASE_47_40__SHIFT	0x0
+#define HDP_NONSURFACE_BASE_HI__NONSURF_BASE_47_40_MASK	0x000000FFL
+//HDP_NONSURF_FLAGS
+#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG__SHIFT	0x0
+#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG__SHIFT	0x1
+#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG_MASK	0x00000001L
+#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG_MASK	0x00000002L
+//HDP_NONSURF_FLAGS_CLR
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR__SHIFT	0x0
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR__SHIFT	0x1
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR_MASK	0x00000001L
+#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR_MASK	0x00000002L
+//HDP_HOST_PATH_CNTL
+#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER__SHIFT	0x9
+#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER__SHIFT	0xb
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_PRELOAD_CFG__SHIFT	0x12
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER__SHIFT	0x13
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN__SHIFT	0x15
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_64B_EN__SHIFT	0x16
+#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS__SHIFT	0x1d
+#define HDP_HOST_PATH_CNTL__WRITE_THROUGH_CACHE_DIS__SHIFT	0x1e
+#define HDP_HOST_PATH_CNTL__LIN_RD_CACHE_DIS__SHIFT	0x1f
+#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER_MASK	0x00000600L
+#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER_MASK	0x00001800L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_PRELOAD_CFG_MASK	0x00040000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_MASK	0x00180000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN_MASK	0x00200000L
+#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_64B_EN_MASK	0x00400000L
+#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS_MASK	0x20000000L
+#define HDP_HOST_PATH_CNTL__WRITE_THROUGH_CACHE_DIS_MASK	0x40000000L
+#define HDP_HOST_PATH_CNTL__LIN_RD_CACHE_DIS_MASK	0x80000000L
+//HDP_SW_SEMAPHORE
+#define HDP_SW_SEMAPHORE__SW_SEMAPHORE__SHIFT	0x0
+#define HDP_SW_SEMAPHORE__SW_SEMAPHORE_MASK	0xFFFFFFFFL
+//HDP_DEBUG0
+#define HDP_DEBUG0__HDP_DEBUG__SHIFT	0x0
+#define HDP_DEBUG0__HDP_DEBUG_MASK	0xFFFFFFFFL
+//HDP_LAST_SURFACE_HIT
+#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT__SHIFT	0x0
+#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT_MASK	0x00000003L
+//HDP_READ_CACHE_INVALIDATE
+#define HDP_READ_CACHE_INVALIDATE__READ_CACHE_INVALIDATE__SHIFT	0x0
+#define HDP_READ_CACHE_INVALIDATE__READ_CACHE_INVALIDATE_MASK	0x00000001L
+//HDP_OUTSTANDING_REQ
+#define HDP_OUTSTANDING_REQ__WRITE_REQ__SHIFT	0x0
+#define HDP_OUTSTANDING_REQ__READ_REQ__SHIFT	0x8
+#define HDP_OUTSTANDING_REQ__WRITE_REQ_MASK	0x000000FFL
+#define HDP_OUTSTANDING_REQ__READ_REQ_MASK	0x0000FF00L
+//HDP_MISC_CNTL
+#define HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE__SHIFT	0x0
+#define HDP_MISC_CNTL__IDLE_HYSTERESIS_CNTL__SHIFT	0x2
+#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024__SHIFT	0x5
+#define HDP_MISC_CNTL__MULTIPLE_READS__SHIFT	0x6
+#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES__SHIFT	0xb
+#define HDP_MISC_CNTL__FED_ENABLE__SHIFT	0x15
+#define HDP_MISC_CNTL__SYSHUB_CHANNEL_PRIORITY__SHIFT	0x17
+#define HDP_MISC_CNTL__MMHUB_WRBURST_ENABLE__SHIFT	0x18
+#define HDP_MISC_CNTL__ALL_FUNCTION_CACHELINE_INVALID__SHIFT	0x19
+#define HDP_MISC_CNTL__HDP_MMHUB_PENDING_WR_TAG_CHECK__SHIFT	0x1a
+#define HDP_MISC_CNTL__XDP_MMHUB_PENDING_WR_TAG_CHECK__SHIFT	0x1b
+#define HDP_MISC_CNTL__VARIABLE_CACHELINE_SIZE__SHIFT	0x1c
+#define HDP_MISC_CNTL__ADAPTIVE_CACHELINE_SIZE__SHIFT	0x1d
+#define HDP_MISC_CNTL__MMHUB_WRBURST_SIZE__SHIFT	0x1e
+#define HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK	0x00000001L
+#define HDP_MISC_CNTL__IDLE_HYSTERESIS_CNTL_MASK	0x0000000CL
+#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024_MASK	0x00000020L
+#define HDP_MISC_CNTL__MULTIPLE_READS_MASK	0x00000040L
+#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES_MASK	0x00000800L
+#define HDP_MISC_CNTL__FED_ENABLE_MASK	0x00200000L
+#define HDP_MISC_CNTL__SYSHUB_CHANNEL_PRIORITY_MASK	0x00800000L
+#define HDP_MISC_CNTL__MMHUB_WRBURST_ENABLE_MASK	0x01000000L
+#define HDP_MISC_CNTL__ALL_FUNCTION_CACHELINE_INVALID_MASK	0x02000000L
+#define HDP_MISC_CNTL__HDP_MMHUB_PENDING_WR_TAG_CHECK_MASK	0x04000000L
+#define HDP_MISC_CNTL__XDP_MMHUB_PENDING_WR_TAG_CHECK_MASK	0x08000000L
+#define HDP_MISC_CNTL__VARIABLE_CACHELINE_SIZE_MASK	0x10000000L
+#define HDP_MISC_CNTL__ADAPTIVE_CACHELINE_SIZE_MASK	0x20000000L
+#define HDP_MISC_CNTL__MMHUB_WRBURST_SIZE_MASK	0x40000000L
+//HDP_MEM_POWER_LS
+#define HDP_MEM_POWER_LS__LS_ENABLE__SHIFT	0x0
+#define HDP_MEM_POWER_LS__LS_HOLD__SHIFT	0x7
+#define HDP_MEM_POWER_LS__LS_ENABLE_MASK	0x00000001L
+#define HDP_MEM_POWER_LS__LS_HOLD_MASK	0x00001F80L
+//HDP_MMHUB_CNTL
+#define HDP_MMHUB_CNTL__HDP_MMHUB_RO__SHIFT	0x0
+#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC__SHIFT	0x1
+#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP__SHIFT	0x2
+#define HDP_MMHUB_CNTL__HDP_MMHUB_RO_MASK	0x00000001L
+#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC_MASK	0x00000002L
+#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP_MASK	0x00000004L
+//HDP_EDC_CNT
+#define HDP_EDC_CNT__MEM0_SED_COUNT__SHIFT	0x0
+#define HDP_EDC_CNT__MEM1_SED_COUNT__SHIFT	0x2
+#define HDP_EDC_CNT__MEM0_SED_COUNT_MASK	0x00000003L
+#define HDP_EDC_CNT__MEM1_SED_COUNT_MASK	0x0000000CL
+//HDP_VERSION
+#define HDP_VERSION__MINVER__SHIFT	0x0
+#define HDP_VERSION__MAJVER__SHIFT	0x8
+#define HDP_VERSION__REV__SHIFT	0x10
+#define HDP_VERSION__MINVER_MASK	0x000000FFL
+#define HDP_VERSION__MAJVER_MASK	0x0000FF00L
+#define HDP_VERSION__REV_MASK	0x00FF0000L
+//HDP_CLK_CNTL
+#define HDP_CLK_CNTL__REG_CLK_ENABLE_COUNT__SHIFT	0x0
+#define HDP_CLK_CNTL__REG_WAKE_DYN_CLK__SHIFT	0x4
+#define HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE__SHIFT	0x1c
+#define HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE__SHIFT	0x1d
+#define HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE__SHIFT	0x1e
+#define HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE__SHIFT	0x1f
+#define HDP_CLK_CNTL__REG_CLK_ENABLE_COUNT_MASK	0x0000000FL
+#define HDP_CLK_CNTL__REG_WAKE_DYN_CLK_MASK	0x00000010L
+#define HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK	0x10000000L
+#define HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK	0x20000000L
+#define HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK	0x40000000L
+#define HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK	0x80000000L
+//HDP_MEMIO_CNTL
+#define HDP_MEMIO_CNTL__MEMIO_SEND__SHIFT	0x0
+#define HDP_MEMIO_CNTL__MEMIO_OP__SHIFT	0x1
+#define HDP_MEMIO_CNTL__MEMIO_BE__SHIFT	0x2
+#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE__SHIFT	0x6
+#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE__SHIFT	0x7
+#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER__SHIFT	0x8
+#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR__SHIFT	0xe
+#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR__SHIFT	0xf
+#define HDP_MEMIO_CNTL__MEMIO_VF__SHIFT	0x10
+#define HDP_MEMIO_CNTL__MEMIO_VFID__SHIFT	0x11
+#define HDP_MEMIO_CNTL__MEMIO_SEND_MASK	0x00000001L
+#define HDP_MEMIO_CNTL__MEMIO_OP_MASK	0x00000002L
+#define HDP_MEMIO_CNTL__MEMIO_BE_MASK	0x0000003CL
+#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE_MASK	0x00000040L
+#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE_MASK	0x00000080L
+#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER_MASK	0x00003F00L
+#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR_MASK	0x00004000L
+#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR_MASK	0x00008000L
+#define HDP_MEMIO_CNTL__MEMIO_VF_MASK	0x00010000L
+#define HDP_MEMIO_CNTL__MEMIO_VFID_MASK	0x003E0000L
+//HDP_MEMIO_ADDR
+#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER__SHIFT	0x0
+#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER_MASK	0xFFFFFFFFL
+//HDP_MEMIO_STATUS
+#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS__SHIFT	0x0
+#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS__SHIFT	0x1
+#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR__SHIFT	0x2
+#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR__SHIFT	0x3
+#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS_MASK	0x00000001L
+#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS_MASK	0x00000002L
+#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR_MASK	0x00000004L
+#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR_MASK	0x00000008L
+//HDP_MEMIO_WR_DATA
+#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA__SHIFT	0x0
+#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA_MASK	0xFFFFFFFFL
+//HDP_MEMIO_RD_DATA
+#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA__SHIFT	0x0
+#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA_MASK	0xFFFFFFFFL
+//HDP_XDP_DIRECT2HDP_FIRST
+#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED__SHIFT	0x0
+#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_FLUSH
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM__SHIFT	0x0
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA__SHIFT	0x4
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL__SHIFT	0x8
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG__SHIFT	0xb
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST__SHIFT	0x10
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM__SHIFT	0x12
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0__SHIFT	0x13
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1__SHIFT	0x14
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM_MASK	0x0000000FL
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA_MASK	0x000000F0L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL_MASK	0x00000700L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG_MASK	0x0000F800L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST_MASK	0x00010000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM_MASK	0x00040000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0_MASK	0x00080000L
+#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1_MASK	0x00100000L
+//HDP_XDP_D2H_BAR_UPDATE
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR__SHIFT	0x0
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM__SHIFT	0x10
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM__SHIFT	0x14
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM_MASK	0x000F0000L
+#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM_MASK	0x00700000L
+//HDP_XDP_D2H_RSVD_3
+#define HDP_XDP_D2H_RSVD_3__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_3__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_4
+#define HDP_XDP_D2H_RSVD_4__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_4__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_5
+#define HDP_XDP_D2H_RSVD_5__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_5__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_6
+#define HDP_XDP_D2H_RSVD_6__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_6__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_7
+#define HDP_XDP_D2H_RSVD_7__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_7__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_8
+#define HDP_XDP_D2H_RSVD_8__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_8__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_9
+#define HDP_XDP_D2H_RSVD_9__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_9__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_10
+#define HDP_XDP_D2H_RSVD_10__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_10__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_11
+#define HDP_XDP_D2H_RSVD_11__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_11__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_12
+#define HDP_XDP_D2H_RSVD_12__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_12__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_13
+#define HDP_XDP_D2H_RSVD_13__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_13__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_14
+#define HDP_XDP_D2H_RSVD_14__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_14__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_15
+#define HDP_XDP_D2H_RSVD_15__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_15__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_16
+#define HDP_XDP_D2H_RSVD_16__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_16__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_17
+#define HDP_XDP_D2H_RSVD_17__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_17__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_18
+#define HDP_XDP_D2H_RSVD_18__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_18__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_19
+#define HDP_XDP_D2H_RSVD_19__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_19__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_20
+#define HDP_XDP_D2H_RSVD_20__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_20__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_21
+#define HDP_XDP_D2H_RSVD_21__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_21__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_22
+#define HDP_XDP_D2H_RSVD_22__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_22__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_23
+#define HDP_XDP_D2H_RSVD_23__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_23__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_24
+#define HDP_XDP_D2H_RSVD_24__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_24__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_25
+#define HDP_XDP_D2H_RSVD_25__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_25__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_26
+#define HDP_XDP_D2H_RSVD_26__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_26__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_27
+#define HDP_XDP_D2H_RSVD_27__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_27__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_28
+#define HDP_XDP_D2H_RSVD_28__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_28__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_29
+#define HDP_XDP_D2H_RSVD_29__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_29__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_30
+#define HDP_XDP_D2H_RSVD_30__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_30__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_31
+#define HDP_XDP_D2H_RSVD_31__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_31__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_32
+#define HDP_XDP_D2H_RSVD_32__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_32__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_33
+#define HDP_XDP_D2H_RSVD_33__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_33__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_D2H_RSVD_34
+#define HDP_XDP_D2H_RSVD_34__RESERVED__SHIFT	0x0
+#define HDP_XDP_D2H_RSVD_34__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_DIRECT2HDP_LAST
+#define HDP_XDP_DIRECT2HDP_LAST__RESERVED__SHIFT	0x0
+#define HDP_XDP_DIRECT2HDP_LAST__RESERVED_MASK	0xFFFFFFFFL
+//HDP_XDP_P2P_BAR_CFG
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE__SHIFT	0x0
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM__SHIFT	0x4
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE_MASK	0x0000000FL
+#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM_MASK	0x00000030L
+//HDP_XDP_P2P_MBX_OFFSET
+#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET_MASK	0x0001FFFFL
+//HDP_XDP_P2P_MBX_ADDR0
+#define HDP_XDP_P2P_MBX_ADDR0__VALID__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_35_19__SHIFT	0x3
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_47_40__SHIFT	0x18
+#define HDP_XDP_P2P_MBX_ADDR0__VALID_MASK	0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_35_19_MASK	0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR0__ADDR_47_40_MASK	0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR1
+#define HDP_XDP_P2P_MBX_ADDR1__VALID__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_35_19__SHIFT	0x3
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_47_40__SHIFT	0x18
+#define HDP_XDP_P2P_MBX_ADDR1__VALID_MASK	0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_35_19_MASK	0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR1__ADDR_47_40_MASK	0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR2
+#define HDP_XDP_P2P_MBX_ADDR2__VALID__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_35_19__SHIFT	0x3
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_47_40__SHIFT	0x18
+#define HDP_XDP_P2P_MBX_ADDR2__VALID_MASK	0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_35_19_MASK	0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR2__ADDR_47_40_MASK	0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR3
+#define HDP_XDP_P2P_MBX_ADDR3__VALID__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_35_19__SHIFT	0x3
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_47_40__SHIFT	0x18
+#define HDP_XDP_P2P_MBX_ADDR3__VALID_MASK	0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_35_19_MASK	0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR3__ADDR_47_40_MASK	0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR4
+#define HDP_XDP_P2P_MBX_ADDR4__VALID__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_35_19__SHIFT	0x3
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_47_40__SHIFT	0x18
+#define HDP_XDP_P2P_MBX_ADDR4__VALID_MASK	0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_35_19_MASK	0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR4__ADDR_47_40_MASK	0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR5
+#define HDP_XDP_P2P_MBX_ADDR5__VALID__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_35_19__SHIFT	0x3
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_47_40__SHIFT	0x18
+#define HDP_XDP_P2P_MBX_ADDR5__VALID_MASK	0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_35_19_MASK	0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR5__ADDR_47_40_MASK	0xFF000000L
+//HDP_XDP_P2P_MBX_ADDR6
+#define HDP_XDP_P2P_MBX_ADDR6__VALID__SHIFT	0x0
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_35_19__SHIFT	0x3
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_47_40__SHIFT	0x18
+#define HDP_XDP_P2P_MBX_ADDR6__VALID_MASK	0x00000001L
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_35_19_MASK	0x000FFFF8L
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_P2P_MBX_ADDR6__ADDR_47_40_MASK	0xFF000000L
+//HDP_XDP_HDP_MBX_MC_CFG
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_QOS__SHIFT	0x0
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP__SHIFT	0x4
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID__SHIFT	0x8
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_RO__SHIFT	0xc
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_GCC__SHIFT	0xd
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SNOOP__SHIFT	0xe
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_QOS_MASK	0x0000000FL
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP_MASK	0x00000030L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID_MASK	0x00000F00L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_RO_MASK	0x00001000L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_GCC_MASK	0x00002000L
+#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SNOOP_MASK	0x00004000L
+//HDP_XDP_HDP_MC_CFG
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP__SHIFT	0x3
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SWAP__SHIFT	0x4
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_VMID__SHIFT	0x8
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO__SHIFT	0xc
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC__SHIFT	0xd
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH__SHIFT	0xe
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP_MASK	0x00000008L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SWAP_MASK	0x00000030L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_VMID_MASK	0x00000F00L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO_MASK	0x00001000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC_MASK	0x00002000L
+#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH_MASK	0x000FC000L
+//HDP_XDP_HST_CFG
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN__SHIFT	0x0
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER__SHIFT	0x1
+#define HDP_XDP_HST_CFG__HST_CFG_WR_BURST_EN__SHIFT	0x3
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_64B_EN__SHIFT	0x4
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_PRELOAD_CFG__SHIFT	0x5
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN_MASK	0x00000001L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_MASK	0x00000006L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_BURST_EN_MASK	0x00000008L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_64B_EN_MASK	0x00000010L
+#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_PRELOAD_CFG_MASK	0x00000020L
+//HDP_XDP_HDP_IPH_CFG
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE__SHIFT	0x0
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE__SHIFT	0x6
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING__SHIFT	0xc
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN__SHIFT	0xd
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE_MASK	0x0000003FL
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE_MASK	0x00000FC0L
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING_MASK	0x00001000L
+#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN_MASK	0x00002000L
+//HDP_XDP_P2P_BAR0
+#define HDP_XDP_P2P_BAR0__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR0__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR0__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR0__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR0__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR0__VALID_MASK	0x00100000L
+//HDP_XDP_P2P_BAR1
+#define HDP_XDP_P2P_BAR1__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR1__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR1__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR1__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR1__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR1__VALID_MASK	0x00100000L
+//HDP_XDP_P2P_BAR2
+#define HDP_XDP_P2P_BAR2__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR2__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR2__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR2__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR2__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR2__VALID_MASK	0x00100000L
+//HDP_XDP_P2P_BAR3
+#define HDP_XDP_P2P_BAR3__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR3__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR3__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR3__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR3__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR3__VALID_MASK	0x00100000L
+//HDP_XDP_P2P_BAR4
+#define HDP_XDP_P2P_BAR4__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR4__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR4__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR4__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR4__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR4__VALID_MASK	0x00100000L
+//HDP_XDP_P2P_BAR5
+#define HDP_XDP_P2P_BAR5__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR5__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR5__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR5__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR5__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR5__VALID_MASK	0x00100000L
+//HDP_XDP_P2P_BAR6
+#define HDP_XDP_P2P_BAR6__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR6__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR6__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR6__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR6__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR6__VALID_MASK	0x00100000L
+//HDP_XDP_P2P_BAR7
+#define HDP_XDP_P2P_BAR7__ADDR__SHIFT	0x0
+#define HDP_XDP_P2P_BAR7__FLUSH__SHIFT	0x10
+#define HDP_XDP_P2P_BAR7__VALID__SHIFT	0x14
+#define HDP_XDP_P2P_BAR7__ADDR_MASK	0x0000FFFFL
+#define HDP_XDP_P2P_BAR7__FLUSH_MASK	0x000F0000L
+#define HDP_XDP_P2P_BAR7__VALID_MASK	0x00100000L
+//HDP_XDP_FLUSH_ARMED_STS
+#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS__SHIFT	0x0
+#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS_MASK	0xFFFFFFFFL
+//HDP_XDP_FLUSH_CNTR0_STS
+#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS__SHIFT	0x0
+#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS_MASK	0x03FFFFFFL
+//HDP_XDP_BUSY_STS
+#define HDP_XDP_BUSY_STS__BUSY_BITS__SHIFT	0x0
+#define HDP_XDP_BUSY_STS__BUSY_BITS_MASK	0x0003FFFFL
+//HDP_XDP_STICKY
+#define HDP_XDP_STICKY__STICKY_STS__SHIFT	0x0
+#define HDP_XDP_STICKY__STICKY_W1C__SHIFT	0x10
+#define HDP_XDP_STICKY__STICKY_STS_MASK	0x0000FFFFL
+#define HDP_XDP_STICKY__STICKY_W1C_MASK	0xFFFF0000L
+//HDP_XDP_CHKN
+#define HDP_XDP_CHKN__CHKN_0_RSVD__SHIFT	0x0
+#define HDP_XDP_CHKN__CHKN_1_RSVD__SHIFT	0x8
+#define HDP_XDP_CHKN__CHKN_2_RSVD__SHIFT	0x10
+#define HDP_XDP_CHKN__CHKN_3_RSVD__SHIFT	0x18
+#define HDP_XDP_CHKN__CHKN_0_RSVD_MASK	0x000000FFL
+#define HDP_XDP_CHKN__CHKN_1_RSVD_MASK	0x0000FF00L
+#define HDP_XDP_CHKN__CHKN_2_RSVD_MASK	0x00FF0000L
+#define HDP_XDP_CHKN__CHKN_3_RSVD_MASK	0xFF000000L
+//HDP_XDP_BARS_ADDR_39_36
+#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36__SHIFT	0x0
+#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36__SHIFT	0x4
+#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36__SHIFT	0x8
+#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36__SHIFT	0xc
+#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36__SHIFT	0x10
+#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36__SHIFT	0x14
+#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36__SHIFT	0x18
+#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36__SHIFT	0x1c
+#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36_MASK	0x0000000FL
+#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36_MASK	0x000000F0L
+#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36_MASK	0x00000F00L
+#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36_MASK	0x0000F000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36_MASK	0x000F0000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36_MASK	0x00F00000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36_MASK	0x0F000000L
+#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36_MASK	0xF0000000L
+//HDP_XDP_MC_VM_FB_LOCATION_BASE
+#define HDP_XDP_MC_VM_FB_LOCATION_BASE__FB_BASE__SHIFT	0x0
+#define HDP_XDP_MC_VM_FB_LOCATION_BASE__FB_BASE_MASK	0x03FFFFFFL
+//HDP_XDP_GPU_IOV_VIOLATION_LOG
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT	0x0
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT	0x1
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT	0x2
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT	0x12
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VF__SHIFT	0x13
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VFID__SHIFT	0x14
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT	0x18
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK	0x00000001L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK	0x00000002L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK	0x0003FFFCL
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__OPCODE_MASK	0x00040000L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VF_MASK	0x00080000L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VFID_MASK	0x00F00000L
+#define HDP_XDP_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK	0xFF000000L
+//HDP_XDP_MMHUB_ERROR
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_01__SHIFT	0x1
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_10__SHIFT	0x2
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_11__SHIFT	0x3
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_01__SHIFT	0x5
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_10__SHIFT	0x6
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_11__SHIFT	0x7
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_01__SHIFT	0x9
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_10__SHIFT	0xa
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_11__SHIFT	0xb
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_01__SHIFT	0xd
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_10__SHIFT	0xe
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_11__SHIFT	0xf
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_01__SHIFT	0x11
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_10__SHIFT	0x12
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_11__SHIFT	0x13
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_01__SHIFT	0x15
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_10__SHIFT	0x16
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_11__SHIFT	0x17
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_01_MASK	0x00000002L
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_10_MASK	0x00000004L
+#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_11_MASK	0x00000008L
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_01_MASK	0x00000020L
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_10_MASK	0x00000040L
+#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_11_MASK	0x00000080L
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_01_MASK	0x00000200L
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_10_MASK	0x00000400L
+#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_11_MASK	0x00000800L
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_01_MASK	0x00002000L
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_10_MASK	0x00004000L
+#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_11_MASK	0x00008000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_01_MASK	0x00020000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_10_MASK	0x00040000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_11_MASK	0x00080000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_01_MASK	0x00200000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_10_MASK	0x00400000L
+#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_11_MASK	0x00800000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/MMHUB/mmhub_1_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_0_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/MMHUB/mmhub_1_0_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_0_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/MMHUB/mmhub_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/MMHUB/mmhub_1_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/MMHUB/mmhub_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/MMHUB/mmhub_1_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_1_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_1_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_1_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_1_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_1_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/MP/mp_10_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_10_0_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/MP/mp_10_0_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/mp/mp_10_0_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/MP/mp_10_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_10_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/MP/mp_10_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/mp/mp_10_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/MP/mp_10_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_10_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/MP/mp_10_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/mp/mp_10_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_offset.h
new file mode 100644
index 000000000000..299e5266a8c0
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_offset.h
@@ -0,0 +1,375 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _mp_9_0_OFFSET_HEADER
+#define _mp_9_0_OFFSET_HEADER
+
+
+
+// addressBlock: mp_SmuMp0_SmnDec
+// base address:	0x0
+#define mmMP0_SMN_C2PMSG_32	0x0060
+#define mmMP0_SMN_C2PMSG_32_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_33	0x0061
+#define mmMP0_SMN_C2PMSG_33_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_34	0x0062
+#define mmMP0_SMN_C2PMSG_34_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_35	0x0063
+#define mmMP0_SMN_C2PMSG_35_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_36	0x0064
+#define mmMP0_SMN_C2PMSG_36_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_37	0x0065
+#define mmMP0_SMN_C2PMSG_37_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_38	0x0066
+#define mmMP0_SMN_C2PMSG_38_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_39	0x0067
+#define mmMP0_SMN_C2PMSG_39_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_40	0x0068
+#define mmMP0_SMN_C2PMSG_40_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_41	0x0069
+#define mmMP0_SMN_C2PMSG_41_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_42	0x006a
+#define mmMP0_SMN_C2PMSG_42_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_43	0x006b
+#define mmMP0_SMN_C2PMSG_43_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_44	0x006c
+#define mmMP0_SMN_C2PMSG_44_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_45	0x006d
+#define mmMP0_SMN_C2PMSG_45_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_46	0x006e
+#define mmMP0_SMN_C2PMSG_46_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_47	0x006f
+#define mmMP0_SMN_C2PMSG_47_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_48	0x0070
+#define mmMP0_SMN_C2PMSG_48_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_49	0x0071
+#define mmMP0_SMN_C2PMSG_49_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_50	0x0072
+#define mmMP0_SMN_C2PMSG_50_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_51	0x0073
+#define mmMP0_SMN_C2PMSG_51_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_52	0x0074
+#define mmMP0_SMN_C2PMSG_52_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_53	0x0075
+#define mmMP0_SMN_C2PMSG_53_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_54	0x0076
+#define mmMP0_SMN_C2PMSG_54_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_55	0x0077
+#define mmMP0_SMN_C2PMSG_55_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_56	0x0078
+#define mmMP0_SMN_C2PMSG_56_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_57	0x0079
+#define mmMP0_SMN_C2PMSG_57_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_58	0x007a
+#define mmMP0_SMN_C2PMSG_58_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_59	0x007b
+#define mmMP0_SMN_C2PMSG_59_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_60	0x007c
+#define mmMP0_SMN_C2PMSG_60_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_61	0x007d
+#define mmMP0_SMN_C2PMSG_61_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_62	0x007e
+#define mmMP0_SMN_C2PMSG_62_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_63	0x007f
+#define mmMP0_SMN_C2PMSG_63_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_64	0x0080
+#define mmMP0_SMN_C2PMSG_64_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_65	0x0081
+#define mmMP0_SMN_C2PMSG_65_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_66	0x0082
+#define mmMP0_SMN_C2PMSG_66_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_67	0x0083
+#define mmMP0_SMN_C2PMSG_67_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_68	0x0084
+#define mmMP0_SMN_C2PMSG_68_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_69	0x0085
+#define mmMP0_SMN_C2PMSG_69_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_70	0x0086
+#define mmMP0_SMN_C2PMSG_70_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_71	0x0087
+#define mmMP0_SMN_C2PMSG_71_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_72	0x0088
+#define mmMP0_SMN_C2PMSG_72_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_73	0x0089
+#define mmMP0_SMN_C2PMSG_73_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_74	0x008a
+#define mmMP0_SMN_C2PMSG_74_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_75	0x008b
+#define mmMP0_SMN_C2PMSG_75_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_76	0x008c
+#define mmMP0_SMN_C2PMSG_76_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_77	0x008d
+#define mmMP0_SMN_C2PMSG_77_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_78	0x008e
+#define mmMP0_SMN_C2PMSG_78_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_79	0x008f
+#define mmMP0_SMN_C2PMSG_79_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_80	0x0090
+#define mmMP0_SMN_C2PMSG_80_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_81	0x0091
+#define mmMP0_SMN_C2PMSG_81_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_82	0x0092
+#define mmMP0_SMN_C2PMSG_82_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_83	0x0093
+#define mmMP0_SMN_C2PMSG_83_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_84	0x0094
+#define mmMP0_SMN_C2PMSG_84_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_85	0x0095
+#define mmMP0_SMN_C2PMSG_85_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_86	0x0096
+#define mmMP0_SMN_C2PMSG_86_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_87	0x0097
+#define mmMP0_SMN_C2PMSG_87_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_88	0x0098
+#define mmMP0_SMN_C2PMSG_88_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_89	0x0099
+#define mmMP0_SMN_C2PMSG_89_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_90	0x009a
+#define mmMP0_SMN_C2PMSG_90_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_91	0x009b
+#define mmMP0_SMN_C2PMSG_91_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_92	0x009c
+#define mmMP0_SMN_C2PMSG_92_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_93	0x009d
+#define mmMP0_SMN_C2PMSG_93_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_94	0x009e
+#define mmMP0_SMN_C2PMSG_94_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_95	0x009f
+#define mmMP0_SMN_C2PMSG_95_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_96	0x00a0
+#define mmMP0_SMN_C2PMSG_96_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_97	0x00a1
+#define mmMP0_SMN_C2PMSG_97_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_98	0x00a2
+#define mmMP0_SMN_C2PMSG_98_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_99	0x00a3
+#define mmMP0_SMN_C2PMSG_99_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_100	0x00a4
+#define mmMP0_SMN_C2PMSG_100_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_101	0x00a5
+#define mmMP0_SMN_C2PMSG_101_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_102	0x00a6
+#define mmMP0_SMN_C2PMSG_102_BASE_IDX	0
+#define mmMP0_SMN_C2PMSG_103	0x00a7
+#define mmMP0_SMN_C2PMSG_103_BASE_IDX	0
+#define mmMP0_SMN_ACTIVE_FCN_ID	0x00c0
+#define mmMP0_SMN_ACTIVE_FCN_ID_BASE_IDX	0
+#define mmMP0_SMN_IH_CREDIT	0x00c1
+#define mmMP0_SMN_IH_CREDIT_BASE_IDX	0
+#define mmMP0_SMN_IH_SW_INT	0x00c2
+#define mmMP0_SMN_IH_SW_INT_BASE_IDX	0
+#define mmMP0_SMN_IH_SW_INT_CTRL	0x00c3
+#define mmMP0_SMN_IH_SW_INT_CTRL_BASE_IDX	0
+
+
+// addressBlock: mp_SmuMp1_SmnDec
+// base address:	0x0
+#define mmMP1_SMN_ACP2MP_RESP	0x0240
+#define mmMP1_SMN_ACP2MP_RESP_BASE_IDX	0
+#define mmMP1_SMN_DC2MP_RESP	0x0241
+#define mmMP1_SMN_DC2MP_RESP_BASE_IDX	0
+#define mmMP1_SMN_UVD2MP_RESP	0x0242
+#define mmMP1_SMN_UVD2MP_RESP_BASE_IDX	0
+#define mmMP1_SMN_VCE2MP_RESP	0x0243
+#define mmMP1_SMN_VCE2MP_RESP_BASE_IDX	0
+#define mmMP1_SMN_RLC2MP_RESP	0x0244
+#define mmMP1_SMN_RLC2MP_RESP_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_32	0x0260
+#define mmMP1_SMN_C2PMSG_32_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_33	0x0261
+#define mmMP1_SMN_C2PMSG_33_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_34	0x0262
+#define mmMP1_SMN_C2PMSG_34_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_35	0x0263
+#define mmMP1_SMN_C2PMSG_35_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_36	0x0264
+#define mmMP1_SMN_C2PMSG_36_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_37	0x0265
+#define mmMP1_SMN_C2PMSG_37_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_38	0x0266
+#define mmMP1_SMN_C2PMSG_38_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_39	0x0267
+#define mmMP1_SMN_C2PMSG_39_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_40	0x0268
+#define mmMP1_SMN_C2PMSG_40_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_41	0x0269
+#define mmMP1_SMN_C2PMSG_41_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_42	0x026a
+#define mmMP1_SMN_C2PMSG_42_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_43	0x026b
+#define mmMP1_SMN_C2PMSG_43_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_44	0x026c
+#define mmMP1_SMN_C2PMSG_44_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_45	0x026d
+#define mmMP1_SMN_C2PMSG_45_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_46	0x026e
+#define mmMP1_SMN_C2PMSG_46_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_47	0x026f
+#define mmMP1_SMN_C2PMSG_47_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_48	0x0270
+#define mmMP1_SMN_C2PMSG_48_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_49	0x0271
+#define mmMP1_SMN_C2PMSG_49_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_50	0x0272
+#define mmMP1_SMN_C2PMSG_50_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_51	0x0273
+#define mmMP1_SMN_C2PMSG_51_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_52	0x0274
+#define mmMP1_SMN_C2PMSG_52_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_53	0x0275
+#define mmMP1_SMN_C2PMSG_53_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_54	0x0276
+#define mmMP1_SMN_C2PMSG_54_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_55	0x0277
+#define mmMP1_SMN_C2PMSG_55_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_56	0x0278
+#define mmMP1_SMN_C2PMSG_56_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_57	0x0279
+#define mmMP1_SMN_C2PMSG_57_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_58	0x027a
+#define mmMP1_SMN_C2PMSG_58_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_59	0x027b
+#define mmMP1_SMN_C2PMSG_59_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_60	0x027c
+#define mmMP1_SMN_C2PMSG_60_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_61	0x027d
+#define mmMP1_SMN_C2PMSG_61_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_62	0x027e
+#define mmMP1_SMN_C2PMSG_62_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_63	0x027f
+#define mmMP1_SMN_C2PMSG_63_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_64	0x0280
+#define mmMP1_SMN_C2PMSG_64_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_65	0x0281
+#define mmMP1_SMN_C2PMSG_65_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_66	0x0282
+#define mmMP1_SMN_C2PMSG_66_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_67	0x0283
+#define mmMP1_SMN_C2PMSG_67_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_68	0x0284
+#define mmMP1_SMN_C2PMSG_68_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_69	0x0285
+#define mmMP1_SMN_C2PMSG_69_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_70	0x0286
+#define mmMP1_SMN_C2PMSG_70_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_71	0x0287
+#define mmMP1_SMN_C2PMSG_71_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_72	0x0288
+#define mmMP1_SMN_C2PMSG_72_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_73	0x0289
+#define mmMP1_SMN_C2PMSG_73_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_74	0x028a
+#define mmMP1_SMN_C2PMSG_74_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_75	0x028b
+#define mmMP1_SMN_C2PMSG_75_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_76	0x028c
+#define mmMP1_SMN_C2PMSG_76_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_77	0x028d
+#define mmMP1_SMN_C2PMSG_77_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_78	0x028e
+#define mmMP1_SMN_C2PMSG_78_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_79	0x028f
+#define mmMP1_SMN_C2PMSG_79_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_80	0x0290
+#define mmMP1_SMN_C2PMSG_80_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_81	0x0291
+#define mmMP1_SMN_C2PMSG_81_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_82	0x0292
+#define mmMP1_SMN_C2PMSG_82_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_83	0x0293
+#define mmMP1_SMN_C2PMSG_83_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_84	0x0294
+#define mmMP1_SMN_C2PMSG_84_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_85	0x0295
+#define mmMP1_SMN_C2PMSG_85_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_86	0x0296
+#define mmMP1_SMN_C2PMSG_86_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_87	0x0297
+#define mmMP1_SMN_C2PMSG_87_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_88	0x0298
+#define mmMP1_SMN_C2PMSG_88_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_89	0x0299
+#define mmMP1_SMN_C2PMSG_89_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_90	0x029a
+#define mmMP1_SMN_C2PMSG_90_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_91	0x029b
+#define mmMP1_SMN_C2PMSG_91_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_92	0x029c
+#define mmMP1_SMN_C2PMSG_92_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_93	0x029d
+#define mmMP1_SMN_C2PMSG_93_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_94	0x029e
+#define mmMP1_SMN_C2PMSG_94_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_95	0x029f
+#define mmMP1_SMN_C2PMSG_95_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_96	0x02a0
+#define mmMP1_SMN_C2PMSG_96_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_97	0x02a1
+#define mmMP1_SMN_C2PMSG_97_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_98	0x02a2
+#define mmMP1_SMN_C2PMSG_98_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_99	0x02a3
+#define mmMP1_SMN_C2PMSG_99_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_100	0x02a4
+#define mmMP1_SMN_C2PMSG_100_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_101	0x02a5
+#define mmMP1_SMN_C2PMSG_101_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_102	0x02a6
+#define mmMP1_SMN_C2PMSG_102_BASE_IDX	0
+#define mmMP1_SMN_C2PMSG_103	0x02a7
+#define mmMP1_SMN_C2PMSG_103_BASE_IDX	0
+#define mmMP1_SMN_ACTIVE_FCN_ID	0x02c0
+#define mmMP1_SMN_ACTIVE_FCN_ID_BASE_IDX	0
+#define mmMP1_SMN_IH_CREDIT	0x02c1
+#define mmMP1_SMN_IH_CREDIT_BASE_IDX	0
+#define mmMP1_SMN_IH_SW_INT	0x02c2
+#define mmMP1_SMN_IH_SW_INT_BASE_IDX	0
+#define mmMP1_SMN_IH_SW_INT_CTRL	0x02c3
+#define mmMP1_SMN_IH_SW_INT_CTRL_BASE_IDX	0
+#define mmMP1_SMN_FPS_CNT	0x02c4
+#define mmMP1_SMN_FPS_CNT_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH0	0x03c0
+#define mmMP1_SMN_EXT_SCRATCH0_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH1	0x03c1
+#define mmMP1_SMN_EXT_SCRATCH1_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH2	0x03c2
+#define mmMP1_SMN_EXT_SCRATCH2_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH3	0x03c3
+#define mmMP1_SMN_EXT_SCRATCH3_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH4	0x03c4
+#define mmMP1_SMN_EXT_SCRATCH4_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH5	0x03c5
+#define mmMP1_SMN_EXT_SCRATCH5_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH6	0x03c6
+#define mmMP1_SMN_EXT_SCRATCH6_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH7	0x03c7
+#define mmMP1_SMN_EXT_SCRATCH7_BASE_IDX	0
+#define mmMP1_SMN_EXT_SCRATCH8	0x03c8
+#define mmMP1_SMN_EXT_SCRATCH8_BASE_IDX	0
+
+
+// addressBlock: mp_SmuMp1Pub_CruDec
+// base address:	0x0
+#define mmMP1_SMN_PUB_CTRL	0x02c5
+#define mmMP1_SMN_PUB_CTRL_BASE_IDX	0
+
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_sh_mask.h
new file mode 100644
index 000000000000..d5a623deca77
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/mp/mp_9_0_sh_mask.h
@@ -0,0 +1,1463 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _mp_9_0_SH_MASK_HEADER
+#define _mp_9_0_SH_MASK_HEADER
+
+
+// addressBlock: mp_SmuMp0_SmnDec
+//MP0_SMN_C2PMSG_32
+#define MP0_SMN_C2PMSG_32__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_32__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_33
+#define MP0_SMN_C2PMSG_33__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_33__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_34
+#define MP0_SMN_C2PMSG_34__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_34__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_35
+#define MP0_SMN_C2PMSG_35__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_35__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_36
+#define MP0_SMN_C2PMSG_36__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_36__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_37
+#define MP0_SMN_C2PMSG_37__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_37__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_38
+#define MP0_SMN_C2PMSG_38__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_38__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_39
+#define MP0_SMN_C2PMSG_39__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_39__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_40
+#define MP0_SMN_C2PMSG_40__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_40__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_41
+#define MP0_SMN_C2PMSG_41__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_41__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_42
+#define MP0_SMN_C2PMSG_42__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_42__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_43
+#define MP0_SMN_C2PMSG_43__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_43__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_44
+#define MP0_SMN_C2PMSG_44__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_44__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_45
+#define MP0_SMN_C2PMSG_45__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_45__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_46
+#define MP0_SMN_C2PMSG_46__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_46__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_47
+#define MP0_SMN_C2PMSG_47__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_47__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_48
+#define MP0_SMN_C2PMSG_48__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_48__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_49
+#define MP0_SMN_C2PMSG_49__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_49__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_50
+#define MP0_SMN_C2PMSG_50__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_50__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_51
+#define MP0_SMN_C2PMSG_51__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_51__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_52
+#define MP0_SMN_C2PMSG_52__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_52__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_53
+#define MP0_SMN_C2PMSG_53__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_53__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_54
+#define MP0_SMN_C2PMSG_54__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_54__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_55
+#define MP0_SMN_C2PMSG_55__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_55__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_56
+#define MP0_SMN_C2PMSG_56__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_56__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_57
+#define MP0_SMN_C2PMSG_57__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_57__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_58
+#define MP0_SMN_C2PMSG_58__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_58__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_59
+#define MP0_SMN_C2PMSG_59__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_59__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_60
+#define MP0_SMN_C2PMSG_60__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_60__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_61
+#define MP0_SMN_C2PMSG_61__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_61__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_62
+#define MP0_SMN_C2PMSG_62__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_62__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_63
+#define MP0_SMN_C2PMSG_63__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_63__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_64
+#define MP0_SMN_C2PMSG_64__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_64__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_65
+#define MP0_SMN_C2PMSG_65__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_65__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_66
+#define MP0_SMN_C2PMSG_66__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_66__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_67
+#define MP0_SMN_C2PMSG_67__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_67__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_68
+#define MP0_SMN_C2PMSG_68__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_68__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_69
+#define MP0_SMN_C2PMSG_69__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_69__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_70
+#define MP0_SMN_C2PMSG_70__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_70__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_71
+#define MP0_SMN_C2PMSG_71__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_71__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_72
+#define MP0_SMN_C2PMSG_72__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_72__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_73
+#define MP0_SMN_C2PMSG_73__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_73__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_74
+#define MP0_SMN_C2PMSG_74__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_74__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_75
+#define MP0_SMN_C2PMSG_75__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_75__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_76
+#define MP0_SMN_C2PMSG_76__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_76__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_77
+#define MP0_SMN_C2PMSG_77__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_77__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_78
+#define MP0_SMN_C2PMSG_78__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_78__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_79
+#define MP0_SMN_C2PMSG_79__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_79__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_80
+#define MP0_SMN_C2PMSG_80__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_80__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_81
+#define MP0_SMN_C2PMSG_81__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_81__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_82
+#define MP0_SMN_C2PMSG_82__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_82__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_83
+#define MP0_SMN_C2PMSG_83__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_83__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_84
+#define MP0_SMN_C2PMSG_84__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_84__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_85
+#define MP0_SMN_C2PMSG_85__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_85__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_86
+#define MP0_SMN_C2PMSG_86__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_86__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_87
+#define MP0_SMN_C2PMSG_87__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_87__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_88
+#define MP0_SMN_C2PMSG_88__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_88__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_89
+#define MP0_SMN_C2PMSG_89__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_89__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_90
+#define MP0_SMN_C2PMSG_90__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_90__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_91
+#define MP0_SMN_C2PMSG_91__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_91__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_92
+#define MP0_SMN_C2PMSG_92__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_92__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_93
+#define MP0_SMN_C2PMSG_93__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_93__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_94
+#define MP0_SMN_C2PMSG_94__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_94__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_95
+#define MP0_SMN_C2PMSG_95__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_95__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_96
+#define MP0_SMN_C2PMSG_96__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_96__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_97
+#define MP0_SMN_C2PMSG_97__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_97__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_98
+#define MP0_SMN_C2PMSG_98__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_98__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_99
+#define MP0_SMN_C2PMSG_99__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_99__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_100
+#define MP0_SMN_C2PMSG_100__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_100__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_101
+#define MP0_SMN_C2PMSG_101__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_101__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_102
+#define MP0_SMN_C2PMSG_102__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_102__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_C2PMSG_103
+#define MP0_SMN_C2PMSG_103__CONTENT__SHIFT	0x0
+#define MP0_SMN_C2PMSG_103__CONTENT_MASK	0xFFFFFFFFL
+//MP0_SMN_ACTIVE_FCN_ID
+#define MP0_SMN_ACTIVE_FCN_ID__VFID__SHIFT	0x0
+#define MP0_SMN_ACTIVE_FCN_ID__VF__SHIFT	0x1f
+#define MP0_SMN_ACTIVE_FCN_ID__VFID_MASK	0x0000000FL
+#define MP0_SMN_ACTIVE_FCN_ID__VF_MASK	0x80000000L
+//MP0_SMN_IH_CREDIT
+#define MP0_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT	0x0
+#define MP0_SMN_IH_CREDIT__CLIENT_ID__SHIFT	0x10
+#define MP0_SMN_IH_CREDIT__CREDIT_VALUE_MASK	0x00000003L
+#define MP0_SMN_IH_CREDIT__CLIENT_ID_MASK	0x00FF0000L
+//MP0_SMN_IH_SW_INT
+#define MP0_SMN_IH_SW_INT__VALID__SHIFT	0x0
+#define MP0_SMN_IH_SW_INT__ID__SHIFT	0x1
+#define MP0_SMN_IH_SW_INT__VALID_MASK	0x00000001L
+#define MP0_SMN_IH_SW_INT__ID_MASK	0x000001FEL
+//MP0_SMN_IH_SW_INT_CTRL
+#define MP0_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK__SHIFT	0x0
+#define MP0_SMN_IH_SW_INT_CTRL__SW_INT_ACK__SHIFT	0x8
+#define MP0_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK_MASK	0x00000001L
+#define MP0_SMN_IH_SW_INT_CTRL__SW_INT_ACK_MASK	0x00000100L
+
+
+// addressBlock: mp_SmuMp1_SmnDec
+//MP1_SMN_ACP2MP_RESP
+#define MP1_SMN_ACP2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_SMN_ACP2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_DC2MP_RESP
+#define MP1_SMN_DC2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_SMN_DC2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_UVD2MP_RESP
+#define MP1_SMN_UVD2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_SMN_UVD2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_VCE2MP_RESP
+#define MP1_SMN_VCE2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_SMN_VCE2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_RLC2MP_RESP
+#define MP1_SMN_RLC2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_SMN_RLC2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_32
+#define MP1_SMN_C2PMSG_32__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_32__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_33
+#define MP1_SMN_C2PMSG_33__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_33__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_34
+#define MP1_SMN_C2PMSG_34__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_34__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_35
+#define MP1_SMN_C2PMSG_35__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_35__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_36
+#define MP1_SMN_C2PMSG_36__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_36__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_37
+#define MP1_SMN_C2PMSG_37__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_37__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_38
+#define MP1_SMN_C2PMSG_38__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_38__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_39
+#define MP1_SMN_C2PMSG_39__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_39__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_40
+#define MP1_SMN_C2PMSG_40__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_40__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_41
+#define MP1_SMN_C2PMSG_41__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_41__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_42
+#define MP1_SMN_C2PMSG_42__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_42__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_43
+#define MP1_SMN_C2PMSG_43__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_43__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_44
+#define MP1_SMN_C2PMSG_44__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_44__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_45
+#define MP1_SMN_C2PMSG_45__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_45__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_46
+#define MP1_SMN_C2PMSG_46__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_46__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_47
+#define MP1_SMN_C2PMSG_47__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_47__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_48
+#define MP1_SMN_C2PMSG_48__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_48__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_49
+#define MP1_SMN_C2PMSG_49__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_49__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_50
+#define MP1_SMN_C2PMSG_50__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_50__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_51
+#define MP1_SMN_C2PMSG_51__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_51__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_52
+#define MP1_SMN_C2PMSG_52__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_52__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_53
+#define MP1_SMN_C2PMSG_53__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_53__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_54
+#define MP1_SMN_C2PMSG_54__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_54__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_55
+#define MP1_SMN_C2PMSG_55__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_55__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_56
+#define MP1_SMN_C2PMSG_56__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_56__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_57
+#define MP1_SMN_C2PMSG_57__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_57__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_58
+#define MP1_SMN_C2PMSG_58__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_58__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_59
+#define MP1_SMN_C2PMSG_59__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_59__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_60
+#define MP1_SMN_C2PMSG_60__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_60__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_61
+#define MP1_SMN_C2PMSG_61__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_61__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_62
+#define MP1_SMN_C2PMSG_62__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_62__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_63
+#define MP1_SMN_C2PMSG_63__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_63__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_64
+#define MP1_SMN_C2PMSG_64__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_64__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_65
+#define MP1_SMN_C2PMSG_65__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_65__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_66
+#define MP1_SMN_C2PMSG_66__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_66__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_67
+#define MP1_SMN_C2PMSG_67__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_67__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_68
+#define MP1_SMN_C2PMSG_68__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_68__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_69
+#define MP1_SMN_C2PMSG_69__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_69__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_70
+#define MP1_SMN_C2PMSG_70__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_70__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_71
+#define MP1_SMN_C2PMSG_71__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_71__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_72
+#define MP1_SMN_C2PMSG_72__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_72__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_73
+#define MP1_SMN_C2PMSG_73__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_73__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_74
+#define MP1_SMN_C2PMSG_74__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_74__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_75
+#define MP1_SMN_C2PMSG_75__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_75__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_76
+#define MP1_SMN_C2PMSG_76__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_76__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_77
+#define MP1_SMN_C2PMSG_77__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_77__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_78
+#define MP1_SMN_C2PMSG_78__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_78__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_79
+#define MP1_SMN_C2PMSG_79__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_79__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_80
+#define MP1_SMN_C2PMSG_80__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_80__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_81
+#define MP1_SMN_C2PMSG_81__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_81__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_82
+#define MP1_SMN_C2PMSG_82__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_82__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_83
+#define MP1_SMN_C2PMSG_83__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_83__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_84
+#define MP1_SMN_C2PMSG_84__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_84__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_85
+#define MP1_SMN_C2PMSG_85__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_85__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_86
+#define MP1_SMN_C2PMSG_86__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_86__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_87
+#define MP1_SMN_C2PMSG_87__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_87__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_88
+#define MP1_SMN_C2PMSG_88__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_88__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_89
+#define MP1_SMN_C2PMSG_89__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_89__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_90
+#define MP1_SMN_C2PMSG_90__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_90__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_91
+#define MP1_SMN_C2PMSG_91__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_91__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_92
+#define MP1_SMN_C2PMSG_92__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_92__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_93
+#define MP1_SMN_C2PMSG_93__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_93__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_94
+#define MP1_SMN_C2PMSG_94__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_94__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_95
+#define MP1_SMN_C2PMSG_95__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_95__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_96
+#define MP1_SMN_C2PMSG_96__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_96__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_97
+#define MP1_SMN_C2PMSG_97__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_97__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_98
+#define MP1_SMN_C2PMSG_98__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_98__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_99
+#define MP1_SMN_C2PMSG_99__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_99__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_100
+#define MP1_SMN_C2PMSG_100__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_100__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_101
+#define MP1_SMN_C2PMSG_101__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_101__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_102
+#define MP1_SMN_C2PMSG_102__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_102__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_C2PMSG_103
+#define MP1_SMN_C2PMSG_103__CONTENT__SHIFT	0x0
+#define MP1_SMN_C2PMSG_103__CONTENT_MASK	0xFFFFFFFFL
+//MP1_SMN_ACTIVE_FCN_ID
+#define MP1_SMN_ACTIVE_FCN_ID__VFID__SHIFT	0x0
+#define MP1_SMN_ACTIVE_FCN_ID__VF__SHIFT	0x1f
+#define MP1_SMN_ACTIVE_FCN_ID__VFID_MASK	0x0000000FL
+#define MP1_SMN_ACTIVE_FCN_ID__VF_MASK	0x80000000L
+//MP1_SMN_IH_CREDIT
+#define MP1_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT	0x0
+#define MP1_SMN_IH_CREDIT__CLIENT_ID__SHIFT	0x10
+#define MP1_SMN_IH_CREDIT__CREDIT_VALUE_MASK	0x00000003L
+#define MP1_SMN_IH_CREDIT__CLIENT_ID_MASK	0x00FF0000L
+//MP1_SMN_IH_SW_INT
+#define MP1_SMN_IH_SW_INT__VALID__SHIFT	0x0
+#define MP1_SMN_IH_SW_INT__ID__SHIFT	0x1
+#define MP1_SMN_IH_SW_INT__VALID_MASK	0x00000001L
+#define MP1_SMN_IH_SW_INT__ID_MASK	0x000001FEL
+//MP1_SMN_IH_SW_INT_CTRL
+#define MP1_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK__SHIFT	0x0
+#define MP1_SMN_IH_SW_INT_CTRL__SW_INT_ACK__SHIFT	0x8
+#define MP1_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK_MASK	0x00000001L
+#define MP1_SMN_IH_SW_INT_CTRL__SW_INT_ACK_MASK	0x00000100L
+//MP1_SMN_FPS_CNT
+#define MP1_SMN_FPS_CNT__COUNT__SHIFT	0x0
+#define MP1_SMN_FPS_CNT__COUNT_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH0
+#define MP1_SMN_EXT_SCRATCH0__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH0__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH1
+#define MP1_SMN_EXT_SCRATCH1__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH1__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH2
+#define MP1_SMN_EXT_SCRATCH2__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH2__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH3
+#define MP1_SMN_EXT_SCRATCH3__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH3__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH4
+#define MP1_SMN_EXT_SCRATCH4__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH4__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH5
+#define MP1_SMN_EXT_SCRATCH5__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH5__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH6
+#define MP1_SMN_EXT_SCRATCH6__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH6__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH7
+#define MP1_SMN_EXT_SCRATCH7__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH7__DATA_MASK	0xFFFFFFFFL
+//MP1_SMN_EXT_SCRATCH8
+#define MP1_SMN_EXT_SCRATCH8__DATA__SHIFT	0x0
+#define MP1_SMN_EXT_SCRATCH8__DATA_MASK	0xFFFFFFFFL
+
+
+
+
+// addressBlock: mp_SmuMp0Pub_CruDec
+//MP0_SOC_INFO
+#define MP0_SOC_INFO__SOC_DIE_ID__SHIFT	0x0
+#define MP0_SOC_INFO__SOC_PKG_TYPE__SHIFT	0x2
+#define MP0_SOC_INFO__SOC_DIE_ID_MASK	0x00000003L
+#define MP0_SOC_INFO__SOC_PKG_TYPE_MASK	0x0000001CL
+//MP0_PUB_SCRATCH0
+#define MP0_PUB_SCRATCH0__DATA__SHIFT	0x0
+#define MP0_PUB_SCRATCH0__DATA_MASK	0xFFFFFFFFL
+//MP0_PUB_SCRATCH1
+#define MP0_PUB_SCRATCH1__DATA__SHIFT	0x0
+#define MP0_PUB_SCRATCH1__DATA_MASK	0xFFFFFFFFL
+//MP0_PUB_SCRATCH2
+#define MP0_PUB_SCRATCH2__DATA__SHIFT	0x0
+#define MP0_PUB_SCRATCH2__DATA_MASK	0xFFFFFFFFL
+//MP0_PUB_SCRATCH3
+#define MP0_PUB_SCRATCH3__DATA__SHIFT	0x0
+#define MP0_PUB_SCRATCH3__DATA_MASK	0xFFFFFFFFL
+//MP0_FW_INTF
+#define MP0_FW_INTF__SS_SECURE__SHIFT	0x13
+#define MP0_FW_INTF__SS_SECURE_MASK	0x00080000L
+//MP0_C2PMSG_0
+#define MP0_C2PMSG_0__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_1
+#define MP0_C2PMSG_1__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_1__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_2
+#define MP0_C2PMSG_2__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_2__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_3
+#define MP0_C2PMSG_3__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_3__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_4
+#define MP0_C2PMSG_4__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_4__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_5
+#define MP0_C2PMSG_5__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_5__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_6
+#define MP0_C2PMSG_6__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_6__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_7
+#define MP0_C2PMSG_7__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_7__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_8
+#define MP0_C2PMSG_8__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_8__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_9
+#define MP0_C2PMSG_9__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_9__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_10
+#define MP0_C2PMSG_10__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_10__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_11
+#define MP0_C2PMSG_11__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_11__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_12
+#define MP0_C2PMSG_12__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_12__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_13
+#define MP0_C2PMSG_13__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_13__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_14
+#define MP0_C2PMSG_14__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_14__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_15
+#define MP0_C2PMSG_15__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_15__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_16
+#define MP0_C2PMSG_16__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_16__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_17
+#define MP0_C2PMSG_17__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_17__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_18
+#define MP0_C2PMSG_18__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_18__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_19
+#define MP0_C2PMSG_19__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_19__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_20
+#define MP0_C2PMSG_20__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_20__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_21
+#define MP0_C2PMSG_21__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_21__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_22
+#define MP0_C2PMSG_22__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_22__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_23
+#define MP0_C2PMSG_23__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_23__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_24
+#define MP0_C2PMSG_24__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_24__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_25
+#define MP0_C2PMSG_25__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_25__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_26
+#define MP0_C2PMSG_26__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_26__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_27
+#define MP0_C2PMSG_27__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_27__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_28
+#define MP0_C2PMSG_28__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_28__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_29
+#define MP0_C2PMSG_29__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_29__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_30
+#define MP0_C2PMSG_30__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_30__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_31
+#define MP0_C2PMSG_31__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_31__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2CMSG_0
+#define MP0_P2CMSG_0__CONTENT__SHIFT	0x0
+#define MP0_P2CMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2CMSG_1
+#define MP0_P2CMSG_1__CONTENT__SHIFT	0x0
+#define MP0_P2CMSG_1__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2CMSG_2
+#define MP0_P2CMSG_2__CONTENT__SHIFT	0x0
+#define MP0_P2CMSG_2__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2CMSG_3
+#define MP0_P2CMSG_3__CONTENT__SHIFT	0x0
+#define MP0_P2CMSG_3__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2CMSG_INTEN
+#define MP0_P2CMSG_INTEN__INTEN__SHIFT	0x0
+#define MP0_P2CMSG_INTEN__INTEN_MASK	0x0000000FL
+//MP0_P2CMSG_INTSTS
+#define MP0_P2CMSG_INTSTS__INTSTS0__SHIFT	0x0
+#define MP0_P2CMSG_INTSTS__INTSTS1__SHIFT	0x1
+#define MP0_P2CMSG_INTSTS__INTSTS2__SHIFT	0x2
+#define MP0_P2CMSG_INTSTS__INTSTS3__SHIFT	0x3
+#define MP0_P2CMSG_INTSTS__INTSTS0_MASK	0x00000001L
+#define MP0_P2CMSG_INTSTS__INTSTS1_MASK	0x00000002L
+#define MP0_P2CMSG_INTSTS__INTSTS2_MASK	0x00000004L
+#define MP0_P2CMSG_INTSTS__INTSTS3_MASK	0x00000008L
+//MP0_C2PMSG_ATTR_0
+#define MP0_C2PMSG_ATTR_0__MSG_ATTR__SHIFT	0x0
+#define MP0_C2PMSG_ATTR_0__MSG_ATTR_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_ATTR_1
+#define MP0_C2PMSG_ATTR_1__MSG_ATTR__SHIFT	0x0
+#define MP0_C2PMSG_ATTR_1__MSG_ATTR_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_ATTR_2
+#define MP0_C2PMSG_ATTR_2__MSG_ATTR__SHIFT	0x0
+#define MP0_C2PMSG_ATTR_2__MSG_ATTR_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_ATTR_3
+#define MP0_C2PMSG_ATTR_3__MSG_ATTR__SHIFT	0x0
+#define MP0_C2PMSG_ATTR_3__MSG_ATTR_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_ATTR_4
+#define MP0_C2PMSG_ATTR_4__MSG_ATTR__SHIFT	0x0
+#define MP0_C2PMSG_ATTR_4__MSG_ATTR_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_ATTR_5
+#define MP0_C2PMSG_ATTR_5__MSG_ATTR__SHIFT	0x0
+#define MP0_C2PMSG_ATTR_5__MSG_ATTR_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_ATTR_6
+#define MP0_C2PMSG_ATTR_6__MSG_ATTR__SHIFT	0x0
+#define MP0_C2PMSG_ATTR_6__MSG_ATTR_MASK	0x0000FFFFL
+//MP0_P2CMSG_ATTR
+#define MP0_P2CMSG_ATTR__MSG_ATTR__SHIFT	0x0
+#define MP0_P2CMSG_ATTR__MSG_ATTR_MASK	0x000000FFL
+//MP0_P2SMSG_0
+#define MP0_P2SMSG_0__CONTENT__SHIFT	0x0
+#define MP0_P2SMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2SMSG_1
+#define MP0_P2SMSG_1__CONTENT__SHIFT	0x0
+#define MP0_P2SMSG_1__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2SMSG_2
+#define MP0_P2SMSG_2__CONTENT__SHIFT	0x0
+#define MP0_P2SMSG_2__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2SMSG_3
+#define MP0_P2SMSG_3__CONTENT__SHIFT	0x0
+#define MP0_P2SMSG_3__CONTENT_MASK	0xFFFFFFFFL
+//MP0_P2SMSG_ATTR
+#define MP0_P2SMSG_ATTR__MSG_ATTR__SHIFT	0x0
+#define MP0_P2SMSG_ATTR__MSG_ATTR_MASK	0x000000FFL
+//MP0_S2PMSG_ATTR
+#define MP0_S2PMSG_ATTR__MSG_ATTR__SHIFT	0x0
+#define MP0_S2PMSG_ATTR__MSG_ATTR_MASK	0x00000003L
+//MP0_P2SMSG_INTSTS
+#define MP0_P2SMSG_INTSTS__INTSTS0__SHIFT	0x0
+#define MP0_P2SMSG_INTSTS__INTSTS1__SHIFT	0x1
+#define MP0_P2SMSG_INTSTS__INTSTS2__SHIFT	0x2
+#define MP0_P2SMSG_INTSTS__INTSTS3__SHIFT	0x3
+#define MP0_P2SMSG_INTSTS__INTSTS0_MASK	0x00000001L
+#define MP0_P2SMSG_INTSTS__INTSTS1_MASK	0x00000002L
+#define MP0_P2SMSG_INTSTS__INTSTS2_MASK	0x00000004L
+#define MP0_P2SMSG_INTSTS__INTSTS3_MASK	0x00000008L
+//MP0_S2PMSG_0
+#define MP0_S2PMSG_0__CONTENT__SHIFT	0x0
+#define MP0_S2PMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_32
+#define MP0_C2PMSG_32__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_32__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_33
+#define MP0_C2PMSG_33__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_33__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_34
+#define MP0_C2PMSG_34__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_34__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_35
+#define MP0_C2PMSG_35__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_35__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_36
+#define MP0_C2PMSG_36__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_36__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_37
+#define MP0_C2PMSG_37__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_37__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_38
+#define MP0_C2PMSG_38__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_38__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_39
+#define MP0_C2PMSG_39__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_39__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_40
+#define MP0_C2PMSG_40__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_40__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_41
+#define MP0_C2PMSG_41__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_41__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_42
+#define MP0_C2PMSG_42__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_42__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_43
+#define MP0_C2PMSG_43__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_43__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_44
+#define MP0_C2PMSG_44__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_44__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_45
+#define MP0_C2PMSG_45__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_45__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_46
+#define MP0_C2PMSG_46__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_46__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_47
+#define MP0_C2PMSG_47__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_47__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_48
+#define MP0_C2PMSG_48__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_48__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_49
+#define MP0_C2PMSG_49__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_49__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_50
+#define MP0_C2PMSG_50__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_50__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_51
+#define MP0_C2PMSG_51__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_51__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_52
+#define MP0_C2PMSG_52__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_52__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_53
+#define MP0_C2PMSG_53__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_53__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_54
+#define MP0_C2PMSG_54__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_54__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_55
+#define MP0_C2PMSG_55__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_55__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_56
+#define MP0_C2PMSG_56__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_56__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_57
+#define MP0_C2PMSG_57__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_57__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_58
+#define MP0_C2PMSG_58__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_58__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_59
+#define MP0_C2PMSG_59__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_59__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_60
+#define MP0_C2PMSG_60__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_60__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_61
+#define MP0_C2PMSG_61__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_61__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_62
+#define MP0_C2PMSG_62__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_62__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_63
+#define MP0_C2PMSG_63__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_63__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_64
+#define MP0_C2PMSG_64__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_64__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_65
+#define MP0_C2PMSG_65__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_65__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_66
+#define MP0_C2PMSG_66__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_66__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_67
+#define MP0_C2PMSG_67__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_67__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_68
+#define MP0_C2PMSG_68__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_68__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_69
+#define MP0_C2PMSG_69__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_69__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_70
+#define MP0_C2PMSG_70__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_70__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_71
+#define MP0_C2PMSG_71__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_71__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_72
+#define MP0_C2PMSG_72__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_72__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_73
+#define MP0_C2PMSG_73__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_73__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_74
+#define MP0_C2PMSG_74__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_74__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_75
+#define MP0_C2PMSG_75__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_75__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_76
+#define MP0_C2PMSG_76__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_76__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_77
+#define MP0_C2PMSG_77__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_77__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_78
+#define MP0_C2PMSG_78__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_78__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_79
+#define MP0_C2PMSG_79__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_79__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_80
+#define MP0_C2PMSG_80__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_80__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_81
+#define MP0_C2PMSG_81__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_81__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_82
+#define MP0_C2PMSG_82__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_82__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_83
+#define MP0_C2PMSG_83__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_83__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_84
+#define MP0_C2PMSG_84__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_84__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_85
+#define MP0_C2PMSG_85__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_85__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_86
+#define MP0_C2PMSG_86__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_86__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_87
+#define MP0_C2PMSG_87__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_87__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_88
+#define MP0_C2PMSG_88__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_88__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_89
+#define MP0_C2PMSG_89__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_89__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_90
+#define MP0_C2PMSG_90__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_90__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_91
+#define MP0_C2PMSG_91__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_91__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_92
+#define MP0_C2PMSG_92__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_92__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_93
+#define MP0_C2PMSG_93__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_93__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_94
+#define MP0_C2PMSG_94__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_94__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_95
+#define MP0_C2PMSG_95__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_95__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_96
+#define MP0_C2PMSG_96__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_96__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_97
+#define MP0_C2PMSG_97__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_97__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_98
+#define MP0_C2PMSG_98__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_98__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_99
+#define MP0_C2PMSG_99__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_99__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_100
+#define MP0_C2PMSG_100__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_100__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_101
+#define MP0_C2PMSG_101__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_101__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_102
+#define MP0_C2PMSG_102__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_102__CONTENT_MASK	0xFFFFFFFFL
+//MP0_C2PMSG_103
+#define MP0_C2PMSG_103__CONTENT__SHIFT	0x0
+#define MP0_C2PMSG_103__CONTENT_MASK	0xFFFFFFFFL
+//MP0_ACTIVE_FCN_ID
+#define MP0_ACTIVE_FCN_ID__VFID__SHIFT	0x0
+#define MP0_ACTIVE_FCN_ID__VF__SHIFT	0x1f
+#define MP0_ACTIVE_FCN_ID__VFID_MASK	0x0000000FL
+#define MP0_ACTIVE_FCN_ID__VF_MASK	0x80000000L
+//MP0_IH_CREDIT
+#define MP0_IH_CREDIT__CREDIT_VALUE__SHIFT	0x0
+#define MP0_IH_CREDIT__CLIENT_ID__SHIFT	0x10
+#define MP0_IH_CREDIT__CREDIT_VALUE_MASK	0x00000003L
+#define MP0_IH_CREDIT__CLIENT_ID_MASK	0x00FF0000L
+//MP0_IH_SW_INT
+#define MP0_IH_SW_INT__ID__SHIFT	0x0
+#define MP0_IH_SW_INT__VALID__SHIFT	0x8
+#define MP0_IH_SW_INT__ID_MASK	0x000000FFL
+#define MP0_IH_SW_INT__VALID_MASK	0x00000100L
+//MP0_IH_SW_INT_CTRL
+#define MP0_IH_SW_INT_CTRL__INT_MASK__SHIFT	0x0
+#define MP0_IH_SW_INT_CTRL__INT_ACK__SHIFT	0x8
+#define MP0_IH_SW_INT_CTRL__INT_MASK_MASK	0x00000001L
+#define MP0_IH_SW_INT_CTRL__INT_ACK_MASK	0x00000100L
+
+
+//CGTT_DRM_CLK_CTRL0
+#define CGTT_DRM_CLK_CTRL0__ON_DELAY__SHIFT	0x0
+#define CGTT_DRM_CLK_CTRL0__OFF_HYSTERESIS__SHIFT	0x4
+#define CGTT_DRM_CLK_CTRL0__DIV_ID__SHIFT	0xc
+#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_0__SHIFT	0x15
+#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_REG__SHIFT	0x16
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT	0x18
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT	0x19
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT	0x1a
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT	0x1b
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT	0x1c
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT	0x1d
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT	0x1e
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT	0x1f
+#define CGTT_DRM_CLK_CTRL0__ON_DELAY_MASK	0x0000000FL
+#define CGTT_DRM_CLK_CTRL0__OFF_HYSTERESIS_MASK	0x00000FF0L
+#define CGTT_DRM_CLK_CTRL0__DIV_ID_MASK	0x00007000L
+#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_0_MASK	0x00200000L
+#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_REG_MASK	0x00400000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE7_MASK	0x01000000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE6_MASK	0x02000000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE5_MASK	0x04000000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE4_MASK	0x08000000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE3_MASK	0x10000000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE2_MASK	0x20000000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE1_MASK	0x40000000L
+#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE0_MASK	0x80000000L
+//DRM_LIGHT_SLEEP_CTRL
+#define DRM_LIGHT_SLEEP_CTRL__MEM_LIGHT_SLEEP_EN__SHIFT	0x0
+#define DRM_LIGHT_SLEEP_CTRL__MEM_LIGHT_SLEEP_EN_MASK	0x00000001L
+
+
+// addressBlock: mp_SmuMp1Pub_CruDec
+//MP1_SMN_PUB_CTRL
+#define MP1_SMN_PUB_CTRL__RESET__SHIFT	0x0
+#define MP1_SMN_PUB_CTRL__RESET_MASK	0x00000001L
+//MP1_FIRMWARE_FLAGS
+#define MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT	0x0
+#define MP1_FIRMWARE_FLAGS__RESERVED__SHIFT	0x1
+#define MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK	0x00000001L
+#define MP1_FIRMWARE_FLAGS__RESERVED_MASK	0xFFFFFFFEL
+//MP1_PUB_SCRATCH0
+#define MP1_PUB_SCRATCH0__DATA__SHIFT	0x0
+#define MP1_PUB_SCRATCH0__DATA_MASK	0xFFFFFFFFL
+//MP1_PUB_SCRATCH1
+#define MP1_PUB_SCRATCH1__DATA__SHIFT	0x0
+#define MP1_PUB_SCRATCH1__DATA_MASK	0xFFFFFFFFL
+//MP1_PUB_SCRATCH2
+#define MP1_PUB_SCRATCH2__DATA__SHIFT	0x0
+#define MP1_PUB_SCRATCH2__DATA_MASK	0xFFFFFFFFL
+//MP1_PUB_SCRATCH3
+#define MP1_PUB_SCRATCH3__DATA__SHIFT	0x0
+#define MP1_PUB_SCRATCH3__DATA_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_0
+#define MP1_C2PMSG_0__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_1
+#define MP1_C2PMSG_1__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_1__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_2
+#define MP1_C2PMSG_2__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_2__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_3
+#define MP1_C2PMSG_3__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_3__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_4
+#define MP1_C2PMSG_4__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_4__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_5
+#define MP1_C2PMSG_5__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_5__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_6
+#define MP1_C2PMSG_6__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_6__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_7
+#define MP1_C2PMSG_7__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_7__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_8
+#define MP1_C2PMSG_8__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_8__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_9
+#define MP1_C2PMSG_9__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_9__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_10
+#define MP1_C2PMSG_10__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_10__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_11
+#define MP1_C2PMSG_11__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_11__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_12
+#define MP1_C2PMSG_12__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_12__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_13
+#define MP1_C2PMSG_13__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_13__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_14
+#define MP1_C2PMSG_14__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_14__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_15
+#define MP1_C2PMSG_15__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_15__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_16
+#define MP1_C2PMSG_16__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_16__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_17
+#define MP1_C2PMSG_17__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_17__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_18
+#define MP1_C2PMSG_18__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_18__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_19
+#define MP1_C2PMSG_19__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_19__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_20
+#define MP1_C2PMSG_20__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_20__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_21
+#define MP1_C2PMSG_21__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_21__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_22
+#define MP1_C2PMSG_22__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_22__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_23
+#define MP1_C2PMSG_23__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_23__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_24
+#define MP1_C2PMSG_24__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_24__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_25
+#define MP1_C2PMSG_25__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_25__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_26
+#define MP1_C2PMSG_26__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_26__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_27
+#define MP1_C2PMSG_27__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_27__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_28
+#define MP1_C2PMSG_28__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_28__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_29
+#define MP1_C2PMSG_29__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_29__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_30
+#define MP1_C2PMSG_30__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_30__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_31
+#define MP1_C2PMSG_31__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_31__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2CMSG_0
+#define MP1_P2CMSG_0__CONTENT__SHIFT	0x0
+#define MP1_P2CMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2CMSG_1
+#define MP1_P2CMSG_1__CONTENT__SHIFT	0x0
+#define MP1_P2CMSG_1__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2CMSG_2
+#define MP1_P2CMSG_2__CONTENT__SHIFT	0x0
+#define MP1_P2CMSG_2__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2CMSG_3
+#define MP1_P2CMSG_3__CONTENT__SHIFT	0x0
+#define MP1_P2CMSG_3__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2CMSG_INTEN
+#define MP1_P2CMSG_INTEN__INTEN__SHIFT	0x0
+#define MP1_P2CMSG_INTEN__INTEN_MASK	0x0000000FL
+//MP1_P2CMSG_INTSTS
+#define MP1_P2CMSG_INTSTS__INTSTS0__SHIFT	0x0
+#define MP1_P2CMSG_INTSTS__INTSTS1__SHIFT	0x1
+#define MP1_P2CMSG_INTSTS__INTSTS2__SHIFT	0x2
+#define MP1_P2CMSG_INTSTS__INTSTS3__SHIFT	0x3
+#define MP1_P2CMSG_INTSTS__INTSTS0_MASK	0x00000001L
+#define MP1_P2CMSG_INTSTS__INTSTS1_MASK	0x00000002L
+#define MP1_P2CMSG_INTSTS__INTSTS2_MASK	0x00000004L
+#define MP1_P2CMSG_INTSTS__INTSTS3_MASK	0x00000008L
+//MP1_P2SMSG_0
+#define MP1_P2SMSG_0__CONTENT__SHIFT	0x0
+#define MP1_P2SMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2SMSG_1
+#define MP1_P2SMSG_1__CONTENT__SHIFT	0x0
+#define MP1_P2SMSG_1__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2SMSG_2
+#define MP1_P2SMSG_2__CONTENT__SHIFT	0x0
+#define MP1_P2SMSG_2__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2SMSG_3
+#define MP1_P2SMSG_3__CONTENT__SHIFT	0x0
+#define MP1_P2SMSG_3__CONTENT_MASK	0xFFFFFFFFL
+//MP1_P2SMSG_INTSTS
+#define MP1_P2SMSG_INTSTS__INTSTS0__SHIFT	0x0
+#define MP1_P2SMSG_INTSTS__INTSTS1__SHIFT	0x1
+#define MP1_P2SMSG_INTSTS__INTSTS2__SHIFT	0x2
+#define MP1_P2SMSG_INTSTS__INTSTS3__SHIFT	0x3
+#define MP1_P2SMSG_INTSTS__INTSTS0_MASK	0x00000001L
+#define MP1_P2SMSG_INTSTS__INTSTS1_MASK	0x00000002L
+#define MP1_P2SMSG_INTSTS__INTSTS2_MASK	0x00000004L
+#define MP1_P2SMSG_INTSTS__INTSTS3_MASK	0x00000008L
+//MP1_S2PMSG_0
+#define MP1_S2PMSG_0__CONTENT__SHIFT	0x0
+#define MP1_S2PMSG_0__CONTENT_MASK	0xFFFFFFFFL
+//MP1_ACP2MP_RESP
+#define MP1_ACP2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_ACP2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_DC2MP_RESP
+#define MP1_DC2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_DC2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_UVD2MP_RESP
+#define MP1_UVD2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_UVD2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_VCE2MP_RESP
+#define MP1_VCE2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_VCE2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_RLC2MP_RESP
+#define MP1_RLC2MP_RESP__CONTENT__SHIFT	0x0
+#define MP1_RLC2MP_RESP__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_32
+#define MP1_C2PMSG_32__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_32__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_33
+#define MP1_C2PMSG_33__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_33__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_34
+#define MP1_C2PMSG_34__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_34__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_35
+#define MP1_C2PMSG_35__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_35__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_36
+#define MP1_C2PMSG_36__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_36__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_37
+#define MP1_C2PMSG_37__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_37__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_38
+#define MP1_C2PMSG_38__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_38__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_39
+#define MP1_C2PMSG_39__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_39__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_40
+#define MP1_C2PMSG_40__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_40__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_41
+#define MP1_C2PMSG_41__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_41__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_42
+#define MP1_C2PMSG_42__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_42__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_43
+#define MP1_C2PMSG_43__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_43__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_44
+#define MP1_C2PMSG_44__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_44__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_45
+#define MP1_C2PMSG_45__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_45__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_46
+#define MP1_C2PMSG_46__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_46__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_47
+#define MP1_C2PMSG_47__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_47__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_48
+#define MP1_C2PMSG_48__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_48__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_49
+#define MP1_C2PMSG_49__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_49__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_50
+#define MP1_C2PMSG_50__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_50__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_51
+#define MP1_C2PMSG_51__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_51__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_52
+#define MP1_C2PMSG_52__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_52__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_53
+#define MP1_C2PMSG_53__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_53__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_54
+#define MP1_C2PMSG_54__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_54__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_55
+#define MP1_C2PMSG_55__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_55__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_56
+#define MP1_C2PMSG_56__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_56__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_57
+#define MP1_C2PMSG_57__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_57__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_58
+#define MP1_C2PMSG_58__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_58__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_59
+#define MP1_C2PMSG_59__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_59__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_60
+#define MP1_C2PMSG_60__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_60__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_61
+#define MP1_C2PMSG_61__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_61__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_62
+#define MP1_C2PMSG_62__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_62__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_63
+#define MP1_C2PMSG_63__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_63__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_64
+#define MP1_C2PMSG_64__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_64__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_65
+#define MP1_C2PMSG_65__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_65__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_66
+#define MP1_C2PMSG_66__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_66__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_67
+#define MP1_C2PMSG_67__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_67__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_68
+#define MP1_C2PMSG_68__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_68__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_69
+#define MP1_C2PMSG_69__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_69__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_70
+#define MP1_C2PMSG_70__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_70__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_71
+#define MP1_C2PMSG_71__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_71__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_72
+#define MP1_C2PMSG_72__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_72__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_73
+#define MP1_C2PMSG_73__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_73__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_74
+#define MP1_C2PMSG_74__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_74__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_75
+#define MP1_C2PMSG_75__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_75__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_76
+#define MP1_C2PMSG_76__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_76__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_77
+#define MP1_C2PMSG_77__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_77__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_78
+#define MP1_C2PMSG_78__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_78__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_79
+#define MP1_C2PMSG_79__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_79__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_80
+#define MP1_C2PMSG_80__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_80__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_81
+#define MP1_C2PMSG_81__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_81__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_82
+#define MP1_C2PMSG_82__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_82__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_83
+#define MP1_C2PMSG_83__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_83__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_84
+#define MP1_C2PMSG_84__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_84__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_85
+#define MP1_C2PMSG_85__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_85__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_86
+#define MP1_C2PMSG_86__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_86__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_87
+#define MP1_C2PMSG_87__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_87__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_88
+#define MP1_C2PMSG_88__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_88__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_89
+#define MP1_C2PMSG_89__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_89__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_90
+#define MP1_C2PMSG_90__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_90__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_91
+#define MP1_C2PMSG_91__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_91__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_92
+#define MP1_C2PMSG_92__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_92__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_93
+#define MP1_C2PMSG_93__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_93__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_94
+#define MP1_C2PMSG_94__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_94__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_95
+#define MP1_C2PMSG_95__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_95__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_96
+#define MP1_C2PMSG_96__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_96__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_97
+#define MP1_C2PMSG_97__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_97__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_98
+#define MP1_C2PMSG_98__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_98__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_99
+#define MP1_C2PMSG_99__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_99__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_100
+#define MP1_C2PMSG_100__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_100__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_101
+#define MP1_C2PMSG_101__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_101__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_102
+#define MP1_C2PMSG_102__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_102__CONTENT_MASK	0xFFFFFFFFL
+//MP1_C2PMSG_103
+#define MP1_C2PMSG_103__CONTENT__SHIFT	0x0
+#define MP1_C2PMSG_103__CONTENT_MASK	0xFFFFFFFFL
+//MP1_ACTIVE_FCN_ID
+#define MP1_ACTIVE_FCN_ID__VFID__SHIFT	0x0
+#define MP1_ACTIVE_FCN_ID__VF__SHIFT	0x1f
+#define MP1_ACTIVE_FCN_ID__VFID_MASK	0x0000000FL
+#define MP1_ACTIVE_FCN_ID__VF_MASK	0x80000000L
+//MP1_IH_CREDIT
+#define MP1_IH_CREDIT__CREDIT_VALUE__SHIFT	0x0
+#define MP1_IH_CREDIT__CLIENT_ID__SHIFT	0x10
+#define MP1_IH_CREDIT__CREDIT_VALUE_MASK	0x00000003L
+#define MP1_IH_CREDIT__CLIENT_ID_MASK	0x00FF0000L
+//MP1_IH_SW_INT
+#define MP1_IH_SW_INT__ID__SHIFT	0x0
+#define MP1_IH_SW_INT__VALID__SHIFT	0x8
+#define MP1_IH_SW_INT__ID_MASK	0x000000FFL
+#define MP1_IH_SW_INT__VALID_MASK	0x00000100L
+//MP1_IH_SW_INT_CTRL
+#define MP1_IH_SW_INT_CTRL__INT_MASK__SHIFT	0x0
+#define MP1_IH_SW_INT_CTRL__INT_ACK__SHIFT	0x8
+#define MP1_IH_SW_INT_CTRL__INT_MASK_MASK	0x00000001L
+#define MP1_IH_SW_INT_CTRL__INT_ACK_MASK	0x00000100L
+//MP1_FPS_CNT
+#define MP1_FPS_CNT__COUNT__SHIFT	0x0
+#define MP1_FPS_CNT__COUNT_MASK	0xFFFFFFFFL
+//MP1_PUB_CTRL
+#define MP1_PUB_CTRL__RESET__SHIFT	0x0
+#define MP1_PUB_CTRL__RESET_MASK	0x00000001L
+//MP1_EXT_SCRATCH0
+#define MP1_EXT_SCRATCH0__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH0__DATA_MASK	0xFFFFFFFFL
+//MP1_EXT_SCRATCH1
+#define MP1_EXT_SCRATCH1__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH1__DATA_MASK	0xFFFFFFFFL
+//MP1_EXT_SCRATCH2
+#define MP1_EXT_SCRATCH2__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH2__DATA_MASK	0xFFFFFFFFL
+//MP1_EXT_SCRATCH3
+#define MP1_EXT_SCRATCH3__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH3__DATA_MASK	0xFFFFFFFFL
+//MP1_EXT_SCRATCH4
+#define MP1_EXT_SCRATCH4__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH4__DATA_MASK	0xFFFFFFFFL
+//MP1_EXT_SCRATCH5
+#define MP1_EXT_SCRATCH5__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH5__DATA_MASK	0xFFFFFFFFL
+//MP1_EXT_SCRATCH6
+#define MP1_EXT_SCRATCH6__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH6__DATA_MASK	0xFFFFFFFFL
+//MP1_EXT_SCRATCH7
+#define MP1_EXT_SCRATCH7__DATA__SHIFT	0x0
+#define MP1_EXT_SCRATCH7__DATA_MASK	0xFFFFFFFFL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_6_1_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_6_1_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_6_1_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbif/nbif_6_1_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIO/nbio_6_1_default.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_1_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/NBIO/nbio_6_1_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_1_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIO/nbio_6_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_1_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/NBIO/nbio_6_1_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_1_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIO/nbio_6_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_1_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/NBIO/nbio_6_1_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_1_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/NBIO/nbio_7_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_0_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/NBIO/nbio_7_0_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_0_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/NBIO/nbio_7_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/NBIO/nbio_7_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/NBIO/nbio_7_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/NBIO/nbio_7_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_7_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_default.h
deleted file mode 100644
index eac125c9e300..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/raven1/DCN/dcn_1_0_default.h
+++ /dev/null
@@ -1,7988 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _dcn_1_0_DEFAULT_HEADER
-#define _dcn_1_0_DEFAULT_HEADER
-
-
-// addressBlock: dce_dc_hda_azcontroller_azdec
-#define smnAZCONTROLLER0_GLOBAL_CAPABILITIES_DEFAULT                              0x00000000
-#define smnAZCONTROLLER0_MINOR_VERSION_DEFAULT                                    0x00000000
-#define smnAZCONTROLLER0_MAJOR_VERSION_DEFAULT                                    0x00000000
-#define smnAZCONTROLLER0_OUTPUT_PAYLOAD_CAPABILITY_DEFAULT                        0x00000000
-#define smnAZCONTROLLER0_INPUT_PAYLOAD_CAPABILITY_DEFAULT                         0x00000000
-#define smnAZCONTROLLER0_GLOBAL_CONTROL_DEFAULT                                   0x00000000
-#define smnAZCONTROLLER0_WAKE_ENABLE_DEFAULT                                      0x00000000
-#define smnAZCONTROLLER0_STATE_CHANGE_STATUS_DEFAULT                              0x00000000
-#define smnAZCONTROLLER0_GLOBAL_STATUS_DEFAULT                                    0x00000000
-#define smnAZCONTROLLER0_OUTPUT_STREAM_PAYLOAD_CAPABILITY_DEFAULT                 0x00000000
-#define smnAZCONTROLLER0_INPUT_STREAM_PAYLOAD_CAPABILITY_DEFAULT                  0x00000000
-#define smnAZCONTROLLER0_INTERRUPT_CONTROL_DEFAULT                                0x00000000
-#define smnAZCONTROLLER0_INTERRUPT_STATUS_DEFAULT                                 0x00000000
-#define smnAZCONTROLLER0_WALL_CLOCK_COUNTER_DEFAULT                               0x00000000
-#define smnAZCONTROLLER0_STREAM_SYNCHRONIZATION_DEFAULT                           0x00000000
-#define smnAZCONTROLLER0_CORB_LOWER_BASE_ADDRESS_DEFAULT                          0x00000000
-#define smnAZCONTROLLER0_CORB_UPPER_BASE_ADDRESS_DEFAULT                          0x00000000
-#define smnAZCONTROLLER0_CORB_WRITE_POINTER_DEFAULT                               0x00000000
-#define smnAZCONTROLLER0_CORB_READ_POINTER_DEFAULT                                0x00000000
-#define smnAZCONTROLLER0_CORB_CONTROL_DEFAULT                                     0x00000000
-#define smnAZCONTROLLER0_CORB_STATUS_DEFAULT                                      0x00000000
-#define smnAZCONTROLLER0_CORB_SIZE_DEFAULT                                        0x00000002
-#define smnAZCONTROLLER0_RIRB_LOWER_BASE_ADDRESS_DEFAULT                          0x00000000
-#define smnAZCONTROLLER0_RIRB_UPPER_BASE_ADDRESS_DEFAULT                          0x00000000
-#define smnAZCONTROLLER0_RIRB_WRITE_POINTER_DEFAULT                               0x00000000
-#define smnAZCONTROLLER0_RESPONSE_INTERRUPT_COUNT_DEFAULT                         0x00000000
-#define smnAZCONTROLLER0_RIRB_CONTROL_DEFAULT                                     0x00000000
-#define smnAZCONTROLLER0_RIRB_STATUS_DEFAULT                                      0x00000000
-#define smnAZCONTROLLER0_RIRB_SIZE_DEFAULT                                        0x00000002
-#define smnAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DEFAULT               0x00000000
-#define smnAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT          0x00000000
-#define smnAZCONTROLLER0_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT         0x00000000
-#define smnAZCONTROLLER0_IMMEDIATE_RESPONSE_INPUT_INTERFACE_DEFAULT               0x00000000
-#define smnAZCONTROLLER0_IMMEDIATE_COMMAND_STATUS_DEFAULT                         0x00000000
-#define smnAZCONTROLLER0_DMA_POSITION_LOWER_BASE_ADDRESS_DEFAULT                  0x00000000
-#define smnAZCONTROLLER0_DMA_POSITION_UPPER_BASE_ADDRESS_DEFAULT                  0x00000000
-#define smnAZCONTROLLER0_WALL_CLOCK_COUNTER_ALIAS_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_hda_azendpoint_azdec
-#define smnAZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT 0x00000000
-#define smnAZENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azinputendpoint_azdec
-#define smnAZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA_DEFAULT 0x00000000
-#define smnAZINPUTENDPOINT0_AZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azroot_azdec
-#define smnAZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT         0x00000000
-#define smnAZROOT0_AZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT        0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream0_azdec
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM0_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream1_azdec
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM1_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream2_azdec
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM2_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream3_azdec
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM3_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream4_azdec
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM4_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream5_azdec
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM5_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream6_azdec
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM6_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream7_azdec
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define smnAZSTREAM7_0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_mmhubbub_vga_dispdec[72..76]
-#define mmVGA_MEM_WRITE_PAGE_ADDR_DEFAULT                                        0x00000000
-#define mmVGA_MEM_READ_PAGE_ADDR_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_mmhubbub_vga_dispdec[948..986]
-#define mmCRTC8_IDX_DEFAULT                                                      0x00000000
-#define mmCRTC8_DATA_DEFAULT                                                     0x00000000
-#define mmGENFC_WT_DEFAULT                                                       0x00000000
-#define mmGENS1_DEFAULT                                                          0x00000000
-#define mmATTRDW_DEFAULT                                                         0x00000000
-#define mmATTRX_DEFAULT                                                          0x00000000
-#define mmATTRDR_DEFAULT                                                         0x00000000
-#define mmGENMO_WT_DEFAULT                                                       0x00000000
-#define mmGENS0_DEFAULT                                                          0x00000000
-#define mmGENENB_DEFAULT                                                         0x00000000
-#define mmSEQ8_IDX_DEFAULT                                                       0x00000000
-#define mmSEQ8_DATA_DEFAULT                                                      0x00000000
-#define mmDAC_MASK_DEFAULT                                                       0x00000000
-#define mmDAC_R_INDEX_DEFAULT                                                    0x00000000
-#define mmDAC_W_INDEX_DEFAULT                                                    0x00000000
-#define mmDAC_DATA_DEFAULT                                                       0x00000000
-#define mmGENFC_RD_DEFAULT                                                       0x00000000
-#define mmGENMO_RD_DEFAULT                                                       0x00000000
-#define mmGRPH8_IDX_DEFAULT                                                      0x00000000
-#define mmGRPH8_DATA_DEFAULT                                                     0x00000000
-#define mmCRTC8_IDX_1_DEFAULT                                                    0x00000000
-#define mmCRTC8_DATA_1_DEFAULT                                                   0x00000000
-#define mmGENFC_WT_1_DEFAULT                                                     0x00000000
-#define mmGENS1_1_DEFAULT                                                        0x00000000
-
-
-// addressBlock: dce_dc_hda_azcontroller_azdec
-#define mmCORB_WRITE_POINTER_DEFAULT                                             0x00000000
-#define mmCORB_READ_POINTER_DEFAULT                                              0x00000000
-#define mmCORB_CONTROL_DEFAULT                                                   0x00000000
-#define mmCORB_STATUS_DEFAULT                                                    0x00000000
-#define mmCORB_SIZE_DEFAULT                                                      0x00000002
-#define mmRIRB_LOWER_BASE_ADDRESS_DEFAULT                                        0x00000000
-#define mmRIRB_UPPER_BASE_ADDRESS_DEFAULT                                        0x00000000
-#define mmRIRB_WRITE_POINTER_DEFAULT                                             0x00000000
-#define mmRESPONSE_INTERRUPT_COUNT_DEFAULT                                       0x00000000
-#define mmRIRB_CONTROL_DEFAULT                                                   0x00000000
-#define mmRIRB_STATUS_DEFAULT                                                    0x00000000
-#define mmRIRB_SIZE_DEFAULT                                                      0x00000002
-#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_DEFAULT                             0x00000000
-#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT                        0x00000000
-#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT                       0x00000000
-#define mmIMMEDIATE_RESPONSE_INPUT_INTERFACE_DEFAULT                             0x00000000
-#define mmIMMEDIATE_COMMAND_STATUS_DEFAULT                                       0x00000000
-#define mmDMA_POSITION_LOWER_BASE_ADDRESS_DEFAULT                                0x00000000
-#define mmDMA_POSITION_UPPER_BASE_ADDRESS_DEFAULT                                0x00000000
-#define mmWALL_CLOCK_COUNTER_ALIAS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_hda_azendpoint_azdec
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT             0x00000000
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT            0x00000000
-
-
-// addressBlock: dce_dc_hda_azinputendpoint_azdec
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA_DEFAULT              0x00000000
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX_DEFAULT             0x00000000
-
-
-// addressBlock: dce_dc_hda_azroot_azdec
-#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT                 0x00000000
-#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT                0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream0_azdec
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM0_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream1_azdec
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM1_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream2_azdec
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM2_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream3_azdec
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM3_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream4_azdec
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM4_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream5_azdec
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM5_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream6_azdec
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM6_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_hda_azstream7_azdec
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT        0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT      0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT          0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                 0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                    0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM7_1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_mmhubbub_vga_dispdec[72..76]
-
-
-// addressBlock: dce_dc_mmhubbub_vga_dispdec
-#define mmVGA_RENDER_CONTROL_DEFAULT                                             0x0000000f
-#define mmVGA_SEQUENCER_RESET_CONTROL_DEFAULT                                    0x00003f3f
-#define mmVGA_MODE_CONTROL_DEFAULT                                               0x00000000
-#define mmVGA_SURFACE_PITCH_SELECT_DEFAULT                                       0x00000002
-#define mmVGA_MEMORY_BASE_ADDRESS_DEFAULT                                        0x00000000
-#define mmVGA_DISPBUF1_SURFACE_ADDR_DEFAULT                                      0x00000000
-#define mmVGA_DISPBUF2_SURFACE_ADDR_DEFAULT                                      0x00000000
-#define mmVGA_MEMORY_BASE_ADDRESS_HIGH_DEFAULT                                   0x00000000
-#define mmVGA_HDP_CONTROL_DEFAULT                                                0x00000000
-#define mmVGA_CACHE_CONTROL_DEFAULT                                              0x00000000
-#define mmD1VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD2VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmVGA_STATUS_DEFAULT                                                     0x00000000
-#define mmVGA_INTERRUPT_CONTROL_DEFAULT                                          0x00000000
-#define mmVGA_STATUS_CLEAR_DEFAULT                                               0x00000000
-#define mmVGA_INTERRUPT_STATUS_DEFAULT                                           0x00000000
-#define mmVGA_MAIN_CONTROL_DEFAULT                                               0x00005018
-#define mmVGA_TEST_CONTROL_DEFAULT                                               0x00000000
-#define mmVGA_QOS_CTRL_DEFAULT                                                   0x00000000
-#define mmD3VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD4VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD5VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD6VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmVGA_SOURCE_SELECT_DEFAULT                                              0x00000100
-
-
-// addressBlock: dce_dc_dccg_dccg_dispdec
-#define mmPHYPLLA_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmPHYPLLB_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmPHYPLLC_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmPHYPLLD_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDP_DTO_DBUF_EN_DEFAULT                                                 0x00000000
-#define mmDPREFCLK_CGTT_BLK_CTRL_REG_DEFAULT                                     0x00000200
-#define mmREFCLK_CNTL_DEFAULT                                                    0x00000000
-#define mmMIPI_CLK_CNTL_DEFAULT                                                  0x00000000
-#define mmREFCLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmPHYPLLE_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDCCG_PERFMON_CNTL2_DEFAULT                                             0x00000000
-#define mmDSICLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmDCCG_CBUS_WRCMD_DELAY_DEFAULT                                          0x00000003
-#define mmDCCG_DS_DTO_INCR_DEFAULT                                               0x00000000
-#define mmDCCG_DS_DTO_MODULO_DEFAULT                                             0x00000000
-#define mmDCCG_DS_CNTL_DEFAULT                                                   0x00000000
-#define mmDCCG_DS_HW_CAL_INTERVAL_DEFAULT                                        0x00989680
-#define mmSYMCLKG_CLOCK_ENABLE_DEFAULT                                           0x00000600
-#define mmDPREFCLK_CNTL_DEFAULT                                                  0x00000000
-#define mmAOMCLK0_CNTL_DEFAULT                                                   0x00000000
-#define mmAOMCLK1_CNTL_DEFAULT                                                   0x00000000
-#define mmAOMCLK2_CNTL_DEFAULT                                                   0x00000000
-#define mmDCCG_AUDIO_DTO2_PHASE_DEFAULT                                          0x00000000
-#define mmDCCG_AUDIO_DTO2_MODULO_DEFAULT                                         0x00000001
-#define mmDCE_VERSION_DEFAULT                                                    0x00000000
-#define mmPHYPLLG_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDCCG_GTC_CNTL_DEFAULT                                                  0x00000000
-#define mmDCCG_GTC_DTO_INCR_DEFAULT                                              0x00000000
-#define mmDCCG_GTC_DTO_MODULO_DEFAULT                                            0x00000000
-#define mmDCCG_GTC_CURRENT_DEFAULT                                               0x00000000
-#define mmMIPI_DTO_CNTL_DEFAULT                                                  0x00000000
-#define mmMIPI_DTO_PHASE_DEFAULT                                                 0x00000000
-#define mmMIPI_DTO_MODULO_DEFAULT                                                0x00000000
-#define mmDAC_CLK_ENABLE_DEFAULT                                                 0x00000000
-#define mmDVO_CLK_ENABLE_DEFAULT                                                 0x00000000
-#define mmAVSYNC_COUNTER_WRITE_DEFAULT                                           0x00000000
-#define mmAVSYNC_COUNTER_CONTROL_DEFAULT                                         0x00000000
-#define mmAVSYNC_COUNTER_READ_DEFAULT                                            0x00000000
-#define mmMILLISECOND_TIME_BASE_DIV_DEFAULT                                      0x001186a0
-#define mmDISPCLK_FREQ_CHANGE_CNTL_DEFAULT                                       0x08010028
-#define mmDC_MEM_GLOBAL_PWR_REQ_CNTL_DEFAULT                                     0x00000001
-#define mmDCCG_PERFMON_CNTL_DEFAULT                                              0xfffff800
-#define mmDCCG_GATE_DISABLE_CNTL_DEFAULT                                         0x74ee02dd
-#define mmDISPCLK_CGTT_BLK_CTRL_REG_DEFAULT                                      0x00000200
-#define mmSOCCLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmDCCG_CAC_STATUS_DEFAULT                                                0x00000000
-#define mmPIXCLK1_RESYNC_CNTL_DEFAULT                                            0x00000000
-#define mmPIXCLK2_RESYNC_CNTL_DEFAULT                                            0x00000000
-#define mmPIXCLK0_RESYNC_CNTL_DEFAULT                                            0x00000000
-#define mmMICROSECOND_TIME_BASE_DIV_DEFAULT                                      0x00120464
-#define mmDCCG_GATE_DISABLE_CNTL2_DEFAULT                                        0x007f007f
-#define mmSYMCLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmPHYPLLF_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDCCG_DISP_CNTL_REG_DEFAULT                                             0x00000000
-#define mmOTG0_PIXEL_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP_DTO0_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO0_MODULO_DEFAULT                                                 0x00000000
-#define mmOTG0_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmOTG1_PIXEL_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP_DTO1_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO1_MODULO_DEFAULT                                                 0x00000000
-#define mmOTG1_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmOTG2_PIXEL_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP_DTO2_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO2_MODULO_DEFAULT                                                 0x00000000
-#define mmOTG2_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmOTG3_PIXEL_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP_DTO3_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO3_MODULO_DEFAULT                                                 0x00000000
-#define mmOTG3_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmOTG4_PIXEL_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP_DTO4_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO4_MODULO_DEFAULT                                                 0x00000000
-#define mmOTG4_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmOTG5_PIXEL_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP_DTO5_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO5_MODULO_DEFAULT                                                 0x00000000
-#define mmOTG5_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmDPPCLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmSYMCLKA_CLOCK_ENABLE_DEFAULT                                           0x00000000
-#define mmSYMCLKB_CLOCK_ENABLE_DEFAULT                                           0x00000100
-#define mmSYMCLKC_CLOCK_ENABLE_DEFAULT                                           0x00000200
-#define mmSYMCLKD_CLOCK_ENABLE_DEFAULT                                           0x00000300
-#define mmSYMCLKE_CLOCK_ENABLE_DEFAULT                                           0x00000400
-#define mmSYMCLKF_CLOCK_ENABLE_DEFAULT                                           0x00000500
-#define mmDCCG_SOFT_RESET_DEFAULT                                                0x00000000
-#define mmDVOACLKD_CNTL_DEFAULT                                                  0x00070000
-#define mmDVOACLKC_MVP_CNTL_DEFAULT                                              0x00030000
-#define mmDVOACLKC_CNTL_DEFAULT                                                  0x00030000
-#define mmDCCG_AUDIO_DTO_SOURCE_DEFAULT                                          0x00000030
-#define mmDCCG_AUDIO_DTO0_PHASE_DEFAULT                                          0x00000000
-#define mmDCCG_AUDIO_DTO0_MODULE_DEFAULT                                         0x00000001
-#define mmDCCG_AUDIO_DTO1_PHASE_DEFAULT                                          0x00000000
-#define mmDCCG_AUDIO_DTO1_MODULE_DEFAULT                                         0x00000001
-#define mmDCCG_VSYNC_OTG0_LATCH_VALUE_DEFAULT                                    0x00000000
-#define mmDCCG_VSYNC_OTG1_LATCH_VALUE_DEFAULT                                    0x00000000
-#define mmDCCG_VSYNC_OTG2_LATCH_VALUE_DEFAULT                                    0x00000000
-#define mmDCCG_VSYNC_OTG3_LATCH_VALUE_DEFAULT                                    0x00000000
-#define mmDCCG_VSYNC_OTG4_LATCH_VALUE_DEFAULT                                    0x00000000
-#define mmDCCG_VSYNC_OTG5_LATCH_VALUE_DEFAULT                                    0x00000000
-#define mmDCCG_VSYNC_CNT_CTRL_DEFAULT                                            0x00000000
-#define mmDCCG_VSYNC_CNT_INT_CTRL_DEFAULT                                        0x00000000
-#define mmDCCG_TEST_CLK_SEL_DEFAULT                                              0x01ff01ff
-
-
-// addressBlock: dce_dc_dccg_dccg_dfs_dispdec
-#define mmDENTIST_DISPCLK_CNTL_DEFAULT                                           0x64010064
-
-
-// addressBlock: dce_dc_dccg_dccg_dcperfmon0_dc_perfmon_dispdec
-#define mmDC_PERFMON0_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON0_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON0_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON0_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON0_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON0_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON0_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON0_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON0_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dccg_dccg_dcperfmon1_dc_perfmon_dispdec
-#define mmDC_PERFMON1_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON1_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON1_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON1_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON1_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON1_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON1_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON1_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON1_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dccg_dccg_pll_dispdec
-#define mmPLL_MACRO_CNTL_RESERVED0_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED1_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED2_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED3_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED4_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED5_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED6_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED7_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED8_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED9_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED10_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED11_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED12_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED13_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED14_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED15_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED16_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED17_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED18_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED19_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED20_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED21_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED22_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED23_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED24_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED25_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED26_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED27_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED28_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED29_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED30_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED31_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED32_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED33_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED34_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED35_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED36_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED37_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED38_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED39_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED40_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED41_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_dmu_rbbmif_dispdec
-#define mmRBBMIF_TIMEOUT_DEFAULT                                                 0x20000a00
-#define mmRBBMIF_STATUS_DEFAULT                                                  0x00000000
-#define mmRBBMIF_INT_STATUS_DEFAULT                                              0x80000000
-#define mmRBBMIF_TIMEOUT_DIS_DEFAULT                                             0x00000000
-#define mmRBBMIF_STATUS_FLAG_DEFAULT                                             0x00000000
-
-
-// addressBlock: dce_dc_dmu_dc_pg_dispdec
-#define mmDOMAIN0_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN0_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN1_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN1_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN2_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN2_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN3_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN3_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN4_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN4_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN5_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN5_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN6_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN6_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN7_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN7_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN8_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN8_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN9_PG_CONFIG_DEFAULT                                              0x00000001
-#define mmDOMAIN9_PG_STATUS_DEFAULT                                              0x00000000
-#define mmDOMAIN10_PG_CONFIG_DEFAULT                                             0x00000001
-#define mmDOMAIN10_PG_STATUS_DEFAULT                                             0x00000000
-#define mmDOMAIN11_PG_CONFIG_DEFAULT                                             0x00000001
-#define mmDOMAIN11_PG_STATUS_DEFAULT                                             0x00000000
-#define mmDOMAIN12_PG_CONFIG_DEFAULT                                             0x00000001
-#define mmDOMAIN12_PG_STATUS_DEFAULT                                             0x00000000
-#define mmDOMAIN13_PG_CONFIG_DEFAULT                                             0x00000001
-#define mmDOMAIN13_PG_STATUS_DEFAULT                                             0x00000000
-#define mmDOMAIN14_PG_CONFIG_DEFAULT                                             0x00000001
-#define mmDOMAIN14_PG_STATUS_DEFAULT                                             0x00000000
-#define mmDOMAIN15_PG_CONFIG_DEFAULT                                             0x00000001
-#define mmDOMAIN15_PG_STATUS_DEFAULT                                             0x00000000
-#define mmDCPG_INTERRUPT_STATUS_DEFAULT                                          0x00000000
-#define mmDCPG_INTERRUPT_CONTROL_1_DEFAULT                                       0x00000000
-#define mmDCPG_INTERRUPT_CONTROL_2_DEFAULT                                       0x00000000
-#define mmDC_IP_REQUEST_CNTL_DEFAULT                                             0x00000000
-#define mmDC_PGCNTL_STATUS_REG_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dmu_dmu_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON2_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON2_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON2_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON2_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON2_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON2_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON2_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON2_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON2_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dmu_dmu_misc_dispdec
-#define mmCC_DC_PIPE_DIS_DEFAULT                                                 0x00000000
-#define mmDMU_CLK_CNTL_DEFAULT                                                   0x00000000
-#define mmDMU_MEM_PWR_CNTL_DEFAULT                                               0x00000000
-#define mmDMCU_SMU_INTERRUPT_CNTL_DEFAULT                                        0x00000000
-#define mmSMU_INTERRUPT_CONTROL_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_dmu_dmcu_dispdec
-#define mmDMCU_CTRL_DEFAULT                                                      0xffff0101
-#define mmDMCU_STATUS_DEFAULT                                                    0x00000001
-#define mmDMCU_PC_START_ADDR_DEFAULT                                             0x00000000
-#define mmDMCU_FW_START_ADDR_DEFAULT                                             0x00000000
-#define mmDMCU_FW_END_ADDR_DEFAULT                                               0x00000000
-#define mmDMCU_FW_ISR_START_ADDR_DEFAULT                                         0x00000004
-#define mmDMCU_FW_CS_HI_DEFAULT                                                  0x00000000
-#define mmDMCU_FW_CS_LO_DEFAULT                                                  0x00000000
-#define mmDMCU_RAM_ACCESS_CTRL_DEFAULT                                           0x00000000
-#define mmDMCU_ERAM_WR_CTRL_DEFAULT                                              0x000f0000
-#define mmDMCU_ERAM_WR_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_ERAM_RD_CTRL_DEFAULT                                              0x000f0000
-#define mmDMCU_ERAM_RD_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_WR_CTRL_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_WR_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_RD_CTRL_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_RD_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_EVENT_TRIGGER_DEFAULT                                             0x00000000
-#define mmDMCU_UC_INTERNAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmDMCU_SS_INTERRUPT_CNTL_STATUS_DEFAULT                                  0x00000000
-#define mmDMCU_INTERRUPT_STATUS_DEFAULT                                          0x00000000
-#define mmDMCU_INTERRUPT_STATUS_1_DEFAULT                                        0x00000000
-#define mmDMCU_INTERRUPT_TO_HOST_EN_MASK_DEFAULT                                 0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_DEFAULT                                   0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_1_DEFAULT                                 0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_DEFAULT                              0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_1_DEFAULT                            0x00000000
-#define mmDC_DMCU_SCRATCH_DEFAULT                                                0x00000000
-#define mmDMCU_INT_CNT_DEFAULT                                                   0x00000000
-#define mmDMCU_FW_CHECKSUM_SMPL_BYTE_POS_DEFAULT                                 0x00000000
-#define mmDMCU_UC_CLK_GATING_CNTL_DEFAULT                                        0x00010102
-#define mmMASTER_COMM_DATA_REG1_DEFAULT                                          0x00000000
-#define mmMASTER_COMM_DATA_REG2_DEFAULT                                          0x00000000
-#define mmMASTER_COMM_DATA_REG3_DEFAULT                                          0x00000000
-#define mmMASTER_COMM_CMD_REG_DEFAULT                                            0x00000000
-#define mmMASTER_COMM_CNTL_REG_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_DATA_REG1_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_DATA_REG2_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_DATA_REG3_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_CMD_REG_DEFAULT                                             0x00000000
-#define mmSLAVE_COMM_CNTL_REG_DEFAULT                                            0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS1_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS2_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS3_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS4_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS5_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK1_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK2_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK3_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK4_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK5_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL2_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL3_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL4_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL5_DEFAULT                     0x00000000
-#define mmDMCU_DPRX_INTERRUPT_STATUS1_DEFAULT                                    0x00000000
-#define mmDMCU_DPRX_INTERRUPT_TO_UC_EN_MASK1_DEFAULT                             0x00000000
-#define mmDMCU_DPRX_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1_DEFAULT                        0x00000000
-#define mmDMCU_INTERRUPT_STATUS_CONTINUE_DEFAULT                                 0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_CONTINUE_DEFAULT                          0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_CONTINUE_DEFAULT                     0x00000000
-#define mmDMCU_INT_CNT_CONTINUE_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_dmu_ihc_dispdec
-#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE_DEFAULT                           0x00000000
-#define mmDC_GPU_TIMER_START_POSITION_VSTARTUP_DEFAULT                           0x00000000
-#define mmDC_GPU_TIMER_READ_DEFAULT                                              0x00000000
-#define mmDC_GPU_TIMER_READ_CNTL_DEFAULT                                         0x00000000
-#define mmDISP_INTERRUPT_STATUS_DEFAULT                                          0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE_DEFAULT                                 0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE2_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE3_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE4_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE5_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE6_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE7_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE8_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE9_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE10_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE11_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE12_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE13_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE14_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE15_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE16_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE17_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE18_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE19_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE20_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE21_DEFAULT                               0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE22_DEFAULT                               0x00000000
-#define mmDC_GPU_TIMER_START_POSITION_VREADY_DEFAULT                             0x00000000
-#define mmDC_GPU_TIMER_START_POSITION_FLIP_DEFAULT                               0x00000000
-#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE_NO_LOCK_DEFAULT                   0x00000000
-#define mmDC_GPU_TIMER_START_POSITION_FLIP_AWAY_DEFAULT                          0x00000000
-
-
-// addressBlock: dce_dc_wb0_dispdec_cnv_dispdec
-#define mmCNV0_WB_ENABLE_DEFAULT                                                 0x00000000
-#define mmCNV0_WB_EC_CONFIG_DEFAULT                                              0x55000000
-#define mmCNV0_CNV_MODE_DEFAULT                                                  0x00000000
-#define mmCNV0_CNV_WINDOW_START_DEFAULT                                          0x00000000
-#define mmCNV0_CNV_WINDOW_SIZE_DEFAULT                                           0x00100010
-#define mmCNV0_CNV_UPDATE_DEFAULT                                                0x00000000
-#define mmCNV0_CNV_SOURCE_SIZE_DEFAULT                                           0x00100010
-#define mmCNV0_CNV_CSC_CONTROL_DEFAULT                                           0x00000000
-#define mmCNV0_CNV_CSC_C11_C12_DEFAULT                                           0x00000000
-#define mmCNV0_CNV_CSC_C13_C14_DEFAULT                                           0x00000000
-#define mmCNV0_CNV_CSC_C21_C22_DEFAULT                                           0x00000000
-#define mmCNV0_CNV_CSC_C23_C24_DEFAULT                                           0x00000000
-#define mmCNV0_CNV_CSC_C31_C32_DEFAULT                                           0x00000000
-#define mmCNV0_CNV_CSC_C33_C34_DEFAULT                                           0x00000000
-#define mmCNV0_CNV_CSC_ROUND_OFFSET_R_DEFAULT                                    0x00000000
-#define mmCNV0_CNV_CSC_ROUND_OFFSET_G_DEFAULT                                    0x00000000
-#define mmCNV0_CNV_CSC_ROUND_OFFSET_B_DEFAULT                                    0x00000000
-#define mmCNV0_CNV_CSC_CLAMP_R_DEFAULT                                           0x00000fff
-#define mmCNV0_CNV_CSC_CLAMP_G_DEFAULT                                           0x00000fff
-#define mmCNV0_CNV_CSC_CLAMP_B_DEFAULT                                           0x00000fff
-#define mmCNV0_CNV_TEST_CNTL_DEFAULT                                             0x00000000
-#define mmCNV0_CNV_TEST_CRC_RED_DEFAULT                                          0x0000fff0
-#define mmCNV0_CNV_TEST_CRC_GREEN_DEFAULT                                        0x0000fff0
-#define mmCNV0_CNV_TEST_CRC_BLUE_DEFAULT                                         0x0000fff0
-#define mmCNV0_CNV_INPUT_SELECT_DEFAULT                                          0x00000001
-#define mmCNV0_WB_SOFT_RESET_DEFAULT                                             0x00000000
-#define mmCNV0_WB_WARM_UP_MODE_CTL1_DEFAULT                                      0x88700100
-#define mmCNV0_WB_WARM_UP_MODE_CTL2_DEFAULT                                      0x00000100
-
-
-// addressBlock: dce_dc_wb0_dispdec_wbscl_dispdec
-#define mmWBSCL0_WBSCL_COEF_RAM_SELECT_DEFAULT                                   0x00000000
-#define mmWBSCL0_WBSCL_COEF_RAM_TAP_DATA_DEFAULT                                 0x00000000
-#define mmWBSCL0_WBSCL_MODE_DEFAULT                                              0x00000000
-#define mmWBSCL0_WBSCL_TAP_CONTROL_DEFAULT                                       0x00001111
-#define mmWBSCL0_WBSCL_DEST_SIZE_DEFAULT                                         0x00010001
-#define mmWBSCL0_WBSCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                           0x00080000
-#define mmWBSCL0_WBSCL_HORZ_FILTER_INIT_Y_RGB_DEFAULT                            0x01000000
-#define mmWBSCL0_WBSCL_HORZ_FILTER_INIT_CBCR_DEFAULT                             0x01000000
-#define mmWBSCL0_WBSCL_VERT_FILTER_SCALE_RATIO_DEFAULT                           0x00080000
-#define mmWBSCL0_WBSCL_VERT_FILTER_INIT_Y_RGB_DEFAULT                            0x01000000
-#define mmWBSCL0_WBSCL_VERT_FILTER_INIT_CBCR_DEFAULT                             0x01000000
-#define mmWBSCL0_WBSCL_ROUND_OFFSET_DEFAULT                                      0x00800010
-#define mmWBSCL0_WBSCL_CLAMP_DEFAULT                                             0x01fe01fe
-#define mmWBSCL0_WBSCL_OVERFLOW_STATUS_DEFAULT                                   0x00000000
-#define mmWBSCL0_WBSCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                          0x00000000
-#define mmWBSCL0_WBSCL_OUTSIDE_PIX_STRATEGY_DEFAULT                              0x80108000
-#define mmWBSCL0_WBSCL_TEST_CNTL_DEFAULT                                         0x00000000
-#define mmWBSCL0_WBSCL_TEST_CRC_RED_DEFAULT                                      0x0000ff00
-#define mmWBSCL0_WBSCL_TEST_CRC_GREEN_DEFAULT                                    0x0000ffff
-#define mmWBSCL0_WBSCL_TEST_CRC_BLUE_DEFAULT                                     0x0000ff00
-#define mmWBSCL0_WBSCL_BACKPRESSURE_CNT_EN_DEFAULT                               0x00000000
-#define mmWBSCL0_WB_MCIF_BACKPRESSURE_CNT_DEFAULT                                0x00000000
-#define mmWBSCL0_WBSCL_RAM_SHUTDOWN_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_wb0_dispdec_wb_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON3_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON3_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON3_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON3_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON3_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON3_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON3_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON3_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON3_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_wb1_dispdec_cnv_dispdec
-#define mmCNV1_WB_ENABLE_DEFAULT                                                 0x00000000
-#define mmCNV1_WB_EC_CONFIG_DEFAULT                                              0x55000000
-#define mmCNV1_CNV_MODE_DEFAULT                                                  0x00000000
-#define mmCNV1_CNV_WINDOW_START_DEFAULT                                          0x00000000
-#define mmCNV1_CNV_WINDOW_SIZE_DEFAULT                                           0x00100010
-#define mmCNV1_CNV_UPDATE_DEFAULT                                                0x00000000
-#define mmCNV1_CNV_SOURCE_SIZE_DEFAULT                                           0x00100010
-#define mmCNV1_CNV_CSC_CONTROL_DEFAULT                                           0x00000000
-#define mmCNV1_CNV_CSC_C11_C12_DEFAULT                                           0x00000000
-#define mmCNV1_CNV_CSC_C13_C14_DEFAULT                                           0x00000000
-#define mmCNV1_CNV_CSC_C21_C22_DEFAULT                                           0x00000000
-#define mmCNV1_CNV_CSC_C23_C24_DEFAULT                                           0x00000000
-#define mmCNV1_CNV_CSC_C31_C32_DEFAULT                                           0x00000000
-#define mmCNV1_CNV_CSC_C33_C34_DEFAULT                                           0x00000000
-#define mmCNV1_CNV_CSC_ROUND_OFFSET_R_DEFAULT                                    0x00000000
-#define mmCNV1_CNV_CSC_ROUND_OFFSET_G_DEFAULT                                    0x00000000
-#define mmCNV1_CNV_CSC_ROUND_OFFSET_B_DEFAULT                                    0x00000000
-#define mmCNV1_CNV_CSC_CLAMP_R_DEFAULT                                           0x00000fff
-#define mmCNV1_CNV_CSC_CLAMP_G_DEFAULT                                           0x00000fff
-#define mmCNV1_CNV_CSC_CLAMP_B_DEFAULT                                           0x00000fff
-#define mmCNV1_CNV_TEST_CNTL_DEFAULT                                             0x00000000
-#define mmCNV1_CNV_TEST_CRC_RED_DEFAULT                                          0x0000fff0
-#define mmCNV1_CNV_TEST_CRC_GREEN_DEFAULT                                        0x0000fff0
-#define mmCNV1_CNV_TEST_CRC_BLUE_DEFAULT                                         0x0000fff0
-#define mmCNV1_CNV_INPUT_SELECT_DEFAULT                                          0x00000001
-#define mmCNV1_WB_SOFT_RESET_DEFAULT                                             0x00000000
-#define mmCNV1_WB_WARM_UP_MODE_CTL1_DEFAULT                                      0x88700100
-#define mmCNV1_WB_WARM_UP_MODE_CTL2_DEFAULT                                      0x00000100
-
-
-// addressBlock: dce_dc_wb1_dispdec_wbscl_dispdec
-#define mmWBSCL1_WBSCL_COEF_RAM_SELECT_DEFAULT                                   0x00000000
-#define mmWBSCL1_WBSCL_COEF_RAM_TAP_DATA_DEFAULT                                 0x00000000
-#define mmWBSCL1_WBSCL_MODE_DEFAULT                                              0x00000000
-#define mmWBSCL1_WBSCL_TAP_CONTROL_DEFAULT                                       0x00001111
-#define mmWBSCL1_WBSCL_DEST_SIZE_DEFAULT                                         0x00010001
-#define mmWBSCL1_WBSCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                           0x00080000
-#define mmWBSCL1_WBSCL_HORZ_FILTER_INIT_Y_RGB_DEFAULT                            0x01000000
-#define mmWBSCL1_WBSCL_HORZ_FILTER_INIT_CBCR_DEFAULT                             0x01000000
-#define mmWBSCL1_WBSCL_VERT_FILTER_SCALE_RATIO_DEFAULT                           0x00080000
-#define mmWBSCL1_WBSCL_VERT_FILTER_INIT_Y_RGB_DEFAULT                            0x01000000
-#define mmWBSCL1_WBSCL_VERT_FILTER_INIT_CBCR_DEFAULT                             0x01000000
-#define mmWBSCL1_WBSCL_ROUND_OFFSET_DEFAULT                                      0x00800010
-#define mmWBSCL1_WBSCL_CLAMP_DEFAULT                                             0x01fe01fe
-#define mmWBSCL1_WBSCL_OVERFLOW_STATUS_DEFAULT                                   0x00000000
-#define mmWBSCL1_WBSCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                          0x00000000
-#define mmWBSCL1_WBSCL_OUTSIDE_PIX_STRATEGY_DEFAULT                              0x80108000
-#define mmWBSCL1_WBSCL_TEST_CNTL_DEFAULT                                         0x00000000
-#define mmWBSCL1_WBSCL_TEST_CRC_RED_DEFAULT                                      0x0000ff00
-#define mmWBSCL1_WBSCL_TEST_CRC_GREEN_DEFAULT                                    0x0000ffff
-#define mmWBSCL1_WBSCL_TEST_CRC_BLUE_DEFAULT                                     0x0000ff00
-#define mmWBSCL1_WBSCL_BACKPRESSURE_CNT_EN_DEFAULT                               0x00000000
-#define mmWBSCL1_WB_MCIF_BACKPRESSURE_CNT_DEFAULT                                0x00000000
-#define mmWBSCL1_WBSCL_RAM_SHUTDOWN_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_wb1_dispdec_wb_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON4_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON4_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON4_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON4_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON4_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON4_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON4_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON4_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON4_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_mmhubbub_mcif_wb0_dispdec
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL_DEFAULT                             0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_CUR_LINE_R_DEFAULT                             0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_STATUS_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_PITCH_DEFAULT                                     0x04000400
-#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_ARBITRATION_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_SCLK_CHANGE_DEFAULT                                   0x00000008
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL_DEFAULT                            0x000f0000
-#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_DEFAULT                   0x00000000
-#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL_DEFAULT                             0x00000040
-#define mmMCIF_WB0_MCIF_WB_WATERMARK_DEFAULT                                     0x00000000
-#define mmMCIF_WB0_MCIF_WB_CLOCK_GATER_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_WARM_UP_CNTL_DEFAULT                                  0x00001000
-#define mmMCIF_WB0_MCIF_WB_SELF_REFRESH_CONTROL_DEFAULT                          0x00000002
-#define mmMCIF_WB0_MULTI_LEVEL_QOS_CTRL_DEFAULT                                  0x00000080
-#define mmMCIF_WB0_MCIF_WB_BUF_LUMA_SIZE_DEFAULT                                 0x000fffff
-#define mmMCIF_WB0_MCIF_WB_BUF_CHROMA_SIZE_DEFAULT                               0x000fffff
-
-
-// addressBlock: dce_dc_mmhubbub_mcif_wb1_dispdec
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_SW_CONTROL_DEFAULT                             0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_CUR_LINE_R_DEFAULT                             0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_STATUS_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_PITCH_DEFAULT                                     0x04000400
-#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_ARBITRATION_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_SCLK_CHANGE_DEFAULT                                   0x00000008
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_VCE_CONTROL_DEFAULT                            0x000f0000
-#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_DEFAULT                   0x00000000
-#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_CONTROL_DEFAULT                             0x00000040
-#define mmMCIF_WB1_MCIF_WB_WATERMARK_DEFAULT                                     0x00000000
-#define mmMCIF_WB1_MCIF_WB_CLOCK_GATER_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_WARM_UP_CNTL_DEFAULT                                  0x00001000
-#define mmMCIF_WB1_MCIF_WB_SELF_REFRESH_CONTROL_DEFAULT                          0x00000002
-#define mmMCIF_WB1_MULTI_LEVEL_QOS_CTRL_DEFAULT                                  0x00000080
-#define mmMCIF_WB1_MCIF_WB_BUF_LUMA_SIZE_DEFAULT                                 0x000fffff
-#define mmMCIF_WB1_MCIF_WB_BUF_CHROMA_SIZE_DEFAULT                               0x000fffff
-
-
-// addressBlock: dce_dc_mmhubbub_mmhubbub_dispdec
-#define mmWBIF0_MISC_CTRL_DEFAULT                                                0x00010001
-#define mmWBIF0_SMU_WM_CONTROL_DEFAULT                                           0x00000000
-#define mmWBIF0_PHASE0_OUTSTANDING_COUNTER_DEFAULT                               0x00000000
-#define mmWBIF0_PHASE1_OUTSTANDING_COUNTER_DEFAULT                               0x00000000
-#define mmWBIF1_MISC_CTRL_DEFAULT                                                0x00010001
-#define mmWBIF1_SMU_WM_CONTROL_DEFAULT                                           0x00000000
-#define mmWBIF1_PHASE0_OUTSTANDING_COUNTER_DEFAULT                               0x00000000
-#define mmWBIF1_PHASE1_OUTSTANDING_COUNTER_DEFAULT                               0x00000000
-#define mmVGA_SRC_SPLIT_CNTL_DEFAULT                                             0x00000000
-#define mmMMHUBBUB_MEM_PWR_STATUS_DEFAULT                                        0x00000000
-#define mmMMHUBBUB_MEM_PWR_CNTL_DEFAULT                                          0x0000c180
-#define mmMMHUBBUB_CLOCK_CNTL_DEFAULT                                            0x00000000
-#define mmMMHUBBUB_SOFT_RESET_DEFAULT                                            0x00000000
-
-
-// addressBlock: dce_dc_mmhubbub_vgaif_dispdec
-#define mmMCIF_CONTROL_DEFAULT                                                   0x00000000
-#define mmMCIF_WRITE_COMBINE_CONTROL_DEFAULT                                     0x00000080
-#define mmMCIF_PHASE0_OUTSTANDING_COUNTER_DEFAULT                                0x00000000
-#define mmMCIF_PHASE1_OUTSTANDING_COUNTER_DEFAULT                                0x00000000
-#define mmMCIF_PHASE2_OUTSTANDING_COUNTER_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_mmhubbub_mmhubbub_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON5_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON5_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON5_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON5_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON5_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON5_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON5_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON5_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON5_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream0_dispdec
-#define mmAZF0STREAM0_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM0_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream1_dispdec
-#define mmAZF0STREAM1_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM1_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream2_dispdec
-#define mmAZF0STREAM2_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM2_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream3_dispdec
-#define mmAZF0STREAM3_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM3_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream4_dispdec
-#define mmAZF0STREAM4_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM4_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream5_dispdec
-#define mmAZF0STREAM5_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM5_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream6_dispdec
-#define mmAZF0STREAM6_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM6_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream7_dispdec
-#define mmAZF0STREAM7_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM7_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_az_misc_dispdec
-#define mmAZ_CLOCK_CNTL_DEFAULT                                                  0x00000000
-
-
-// addressBlock: dce_dc_hda_az_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON6_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON6_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON6_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON6_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON6_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON6_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON6_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON6_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON6_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint0_dispdec
-#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint1_dispdec
-#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint2_dispdec
-#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint3_dispdec
-#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint4_dispdec
-#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint5_dispdec
-#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint6_dispdec
-#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0endpoint7_dispdec
-#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0controller_dispdec
-#define mmAZALIA_CONTROLLER_CLOCK_GATING_DEFAULT                                 0x00000000
-#define mmAZALIA_AUDIO_DTO_DEFAULT                                               0x00300018
-#define mmAZALIA_AUDIO_DTO_CONTROL_DEFAULT                                       0x00000000
-#define mmAZALIA_SOCCLK_CONTROL_DEFAULT                                          0x00000001
-#define mmAZALIA_UNDERFLOW_FILLER_SAMPLE_DEFAULT                                 0x00000000
-#define mmAZALIA_DATA_DMA_CONTROL_DEFAULT                                        0x0000000a
-#define mmAZALIA_BDL_DMA_CONTROL_DEFAULT                                         0x0000000a
-#define mmAZALIA_RIRB_AND_DP_CONTROL_DEFAULT                                     0x00000000
-#define mmAZALIA_CORB_DMA_CONTROL_DEFAULT                                        0x00000000
-#define mmAZALIA_APPLICATION_POSITION_IN_CYCLIC_BUFFER_DEFAULT                   0x00000000
-#define mmAZALIA_CYCLIC_BUFFER_SYNC_DEFAULT                                      0x00000000
-#define mmAZALIA_GLOBAL_CAPABILITIES_DEFAULT                                     0x00000000
-#define mmAZALIA_OUTPUT_PAYLOAD_CAPABILITY_DEFAULT                               0x00000060
-#define mmAZALIA_OUTPUT_STREAM_ARBITER_CONTROL_DEFAULT                           0x00080008
-#define mmAZALIA_INPUT_PAYLOAD_CAPABILITY_DEFAULT                                0x00000080
-#define mmAZALIA_INPUT_CRC0_CONTROL0_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_CONTROL1_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_CONTROL2_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_CONTROL3_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_RESULT_DEFAULT                                       0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL0_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL1_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL2_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL3_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_RESULT_DEFAULT                                       0x00000000
-#define mmAZALIA_CRC0_CONTROL0_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_CONTROL1_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_CONTROL2_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_CONTROL3_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_RESULT_DEFAULT                                             0x00000000
-#define mmAZALIA_CRC1_CONTROL0_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_CONTROL1_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_CONTROL2_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_CONTROL3_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_RESULT_DEFAULT                                             0x00000000
-#define mmAZALIA_MEM_PWR_CTRL_DEFAULT                                            0x00000000
-#define mmAZALIA_MEM_PWR_STATUS_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0root_dispdec
-#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_DEFAULT            0x1002aa01
-#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID_DEFAULT                     0x00100700
-#define mmAZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL_DEFAULT                          0x00000000
-#define mmAZALIA_F0_CODEC_RESYNC_FIFO_CONTROL_DEFAULT                            0x0000000d
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_DEFAULT                  0x00000001
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT        0x00020070
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_DEFAULT              0x00000001
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES_DEFAULT                0xc0000009
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE_DEFAULT                   0x00000200
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESET_DEFAULT                         0x00000000
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_DEFAULT         0x00aa0100
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION_DEFAULT     0x00000000
-#define mmCC_RCU_DC_AUDIO_PORT_CONNECTIVITY_DEFAULT                              0x00000000
-#define mmCC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY_DEFAULT                        0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET0_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET1_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET2_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET3_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET4_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET5_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET6_DEFAULT                                    0x00000000
-#define mmREG_DC_AUDIO_PORT_CONNECTIVITY_DEFAULT                                 0x00000000
-#define mmREG_DC_AUDIO_INPUT_PORT_CONNECTIVITY_DEFAULT                           0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream8_dispdec
-#define mmAZF0STREAM8_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM8_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream9_dispdec
-#define mmAZF0STREAM9_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM9_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream10_dispdec
-#define mmAZF0STREAM10_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM10_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream11_dispdec
-#define mmAZF0STREAM11_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM11_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream12_dispdec
-#define mmAZF0STREAM12_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM12_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream13_dispdec
-#define mmAZF0STREAM13_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM13_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream14_dispdec
-#define mmAZF0STREAM14_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM14_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0stream15_dispdec
-#define mmAZF0STREAM15_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM15_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint0_dispdec
-#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint1_dispdec
-#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint2_dispdec
-#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint3_dispdec
-#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint4_dispdec
-#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint5_dispdec
-#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint6_dispdec
-#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_hda_azf0inputendpoint7_dispdec
-#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_dchubbub_hubbub_sdpif_dispdec
-#define mmDCHUBBUB_SDPIF_CFG0_DEFAULT                                            0x00cd3001
-#define mmDCHUBBUB_SDPIF_CFG1_DEFAULT                                            0x0000005c
-#define mmDCHUBBUB_FORCE_IO_STATUS_0_DEFAULT                                     0x00000002
-#define mmDCHUBBUB_FORCE_IO_STATUS_1_DEFAULT                                     0x00000000
-#define mmDCHUBBUB_SDPIF_FB_BASE_DEFAULT                                         0x00000000
-#define mmDCHUBBUB_SDPIF_FB_TOP_DEFAULT                                          0x00000000
-#define mmDCHUBBUB_SDPIF_FB_OFFSET_DEFAULT                                       0x00000000
-#define mmDCHUBBUB_SDPIF_AGP_BOT_DEFAULT                                         0x00000000
-#define mmDCHUBBUB_SDPIF_AGP_TOP_DEFAULT                                         0x00000000
-#define mmDCHUBBUB_SDPIF_AGP_BASE_DEFAULT                                        0x00000000
-#define mmDCHUBBUB_SDPIF_APER_BASE_DEFAULT                                       0x00000000
-#define mmDCHUBBUB_SDPIF_APER_TOP_DEFAULT                                        0x00000000
-#define mmDCHUBBUB_SDPIF_APER_DEF_0_DEFAULT                                      0x00000000
-#define mmDCHUBBUB_SDPIF_APER_DEF_1_DEFAULT                                      0x00000000
-#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_DEFAULT                                    0x00000000
-#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_1_DEFAULT                                    0x00000000
-#define mmDCHUBBUB_SDPIF_MMIO_CNTRL_W_DEFAULT                                    0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_LO_0_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_HI_0_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_LO_0_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_HI_0_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_LO_0_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_HI_0_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_LO_1_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_HI_1_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_LO_1_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_HI_1_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_LO_1_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_HI_1_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_LO_2_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_HI_2_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_LO_2_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_HI_2_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_LO_2_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_HI_2_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_LO_3_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_BASE_HI_3_DEFAULT                                  0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_LO_3_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_RELOC_HI_3_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_LO_3_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_MARC_LENGTH_HI_3_DEFAULT                                0x00000000
-#define mmDCHUBBUB_SDPIF_PIPE_SEC_LVL_DEFAULT                                    0x00000000
-#define mmDCHUBBUB_SDPIF_MEM_PWR_CTRL_DEFAULT                                    0x00000000
-#define mmDCHUBBUB_SDPIF_MEM_PWR_STATUS_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dchubbub_hubbub_ret_path_dispdec
-#define mmDCHUBBUB_RET_PATH_DCC_CFG_DEFAULT                                      0x00000001
-#define mmDCHUBBUB_RET_PATH_DCC_CFG0_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG0_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG1_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG1_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG2_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG2_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG3_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG3_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG4_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG4_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG5_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG5_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG6_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG6_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG7_0_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_DCC_CFG7_1_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_RET_PATH_MEM_PWR_CTRL_DEFAULT                                 0x00000000
-#define mmDCHUBBUB_RET_PATH_MEM_PWR_STATUS_DEFAULT                               0x00000000
-#define mmDCHUBBUB_CRC_CTRL_DEFAULT                                              0x00000000
-#define mmDCHUBBUB_CRC0_VAL_R_G_DEFAULT                                          0x00000000
-#define mmDCHUBBUB_CRC0_VAL_B_A_DEFAULT                                          0x00000000
-#define mmDCHUBBUB_CRC1_VAL_R_G_DEFAULT                                          0x00000000
-#define mmDCHUBBUB_CRC1_VAL_B_A_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_dchubbub_hubbub_dispdec
-#define mmDCHUBBUB_ARB_DF_REQ_OUTSTAND_DEFAULT                                   0x01000100
-#define mmDCHUBBUB_ARB_SAT_LEVEL_DEFAULT                                         0xffffffff
-#define mmDCHUBBUB_ARB_QOS_FORCE_DEFAULT                                         0x00000000
-#define mmDCHUBBUB_ARB_DRAM_STATE_CNTL_DEFAULT                                   0x00000000
-#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A_DEFAULT                          0x00000000
-#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A_DEFAULT                      0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A_DEFAULT                        0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A_DEFAULT                         0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A_DEFAULT                 0x00000000
-#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B_DEFAULT                          0x00000000
-#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B_DEFAULT                      0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B_DEFAULT                        0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B_DEFAULT                         0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B_DEFAULT                 0x00000000
-#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C_DEFAULT                          0x00000000
-#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C_DEFAULT                      0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C_DEFAULT                        0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C_DEFAULT                         0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C_DEFAULT                 0x00000000
-#define mmDCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D_DEFAULT                          0x00000000
-#define mmDCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D_DEFAULT                      0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D_DEFAULT                        0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D_DEFAULT                         0x00000000
-#define mmDCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D_DEFAULT                 0x00000000
-#define mmDCHUBBUB_ARB_WATERMARK_CHANGE_CNTL_DEFAULT                             0x00000010
-#define mmDCHUBBUB_ARB_TIMEOUT_ENABLE_DEFAULT                                    0x00000000
-#define mmDCHUBBUB_GLOBAL_TIMER_CNTL_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK0_ADDRESS_LSB_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK0_ADDRESS_MSB_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK1_ADDRESS_LSB_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK1_ADDRESS_MSB_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK2_ADDRESS_LSB_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK2_ADDRESS_MSB_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK3_ADDRESS_LSB_DEFAULT                                     0x00000000
-#define mmSURFACE_CHECK3_ADDRESS_MSB_DEFAULT                                     0x00000000
-#define mmVTG0_CONTROL_DEFAULT                                                   0x00000000
-#define mmVTG1_CONTROL_DEFAULT                                                   0x00000000
-#define mmVTG2_CONTROL_DEFAULT                                                   0x00000000
-#define mmVTG3_CONTROL_DEFAULT                                                   0x00000000
-#define mmVTG4_CONTROL_DEFAULT                                                   0x00000000
-#define mmVTG5_CONTROL_DEFAULT                                                   0x00000000
-#define mmDCHUBBUB_SOFT_RESET_DEFAULT                                            0x00000000
-#define mmDCHUBBUB_CLOCK_CNTL_DEFAULT                                            0x00000000
-#define mmDCFCLK_CNTL_DEFAULT                                                    0x80000200
-#define mmDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL_DEFAULT                          0x00000000
-#define mmDCHUBBUB_PERFORMANCE_MEASUREMENT_CNTL2_DEFAULT                         0x00000000
-#define mmDCHUBBUB_VLINE_SNAPSHOT_DEFAULT                                        0x00000000
-#define mmDCHUBBUB_SPARE_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dchubbub_dchubbub_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON7_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON7_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON7_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON7_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON7_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON7_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON7_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON7_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON7_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dispdec
-#define mmHUBP0_DCSURF_SURFACE_CONFIG_DEFAULT                                    0x00000008
-#define mmHUBP0_DCSURF_ADDR_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP0_DCSURF_TILING_CONFIG_DEFAULT                                     0x00000080
-#define mmHUBP0_DCSURF_PRI_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP0_DCSURF_PRI_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP0_DCSURF_SEC_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP0_DCSURF_SEC_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP0_DCSURF_SEC_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP0_DCHUBP_REQ_SIZE_CONFIG_DEFAULT                                   0x00000000
-#define mmHUBP0_DCHUBP_REQ_SIZE_CONFIG_C_DEFAULT                                 0x00000000
-#define mmHUBP0_DCHUBP_CNTL_DEFAULT                                              0x00001001
-#define mmHUBP0_HUBP_CLK_CNTL_DEFAULT                                            0x00000000
-#define mmHUBP0_DCHUBP_VMPG_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP0_HUBPREQ_DEBUG_DB_DEFAULT                                         0x00000000
-#define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK_DEFAULT                             0x00000000
-#define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_dcbubp0_dispdec_hubpreq_dispdec
-#define mmHUBPREQ0_DCSURF_SURFACE_PITCH_DEFAULT                                  0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_PITCH_C_DEFAULT                                0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_C_DEFAULT                      0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                 0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_DEFAULT                      0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                 0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_C_DEFAULT                    0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_DEFAULT               0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_DEFAULT                   0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_DEFAULT              0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_DEFAULT                 0x00000000
-#define mmHUBPREQ0_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT            0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_DEFAULT                 0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_DEFAULT            0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_DEFAULT               0x00000000
-#define mmHUBPREQ0_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT          0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_CONTROL_DEFAULT                                0x00000000
-#define mmHUBPREQ0_DCSURF_FLIP_CONTROL_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_DCSURF_FLIP_CONTROL2_DEFAULT                                  0x00003040
-#define mmHUBPREQ0_DCSURF_FRAME_PACING_CONTROL_DEFAULT                           0x04000000
-#define mmHUBPREQ0_DCSURF_FRAME_PACING_TIME_DEFAULT                              0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_FLIP_INTERRUPT_DEFAULT                         0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_DEFAULT                                  0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_DEFAULT                             0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_C_DEFAULT                                0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_INUSE_HIGH_C_DEFAULT                           0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_DEFAULT                         0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_DEFAULT                    0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_C_DEFAULT                       0x00000000
-#define mmHUBPREQ0_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_DEFAULT                  0x00000000
-#define mmHUBPREQ0_DCN_EXPANSION_MODE_DEFAULT                                    0x00000055
-#define mmHUBPREQ0_DCN_TTU_QOS_WM_DEFAULT                                        0x00000000
-#define mmHUBPREQ0_DCN_GLOBAL_TTU_CNTL_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_DCN_SURF0_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_DCN_SURF0_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_DCN_SURF1_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_DCN_SURF1_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_DCN_CUR0_TTU_CNTL0_DEFAULT                                    0x00000000
-#define mmHUBPREQ0_DCN_CUR0_TTU_CNTL1_DEFAULT                                    0x00000000
-#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB_DEFAULT                   0x00000000
-#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB_DEFAULT                   0x00000000
-#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB_DEFAULT                  0x00000000
-#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB_DEFAULT                  0x00000000
-#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ0_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_DEFAULT     0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_DEFAULT     0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_DEFAULT              0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_DEFAULT              0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_DEFAULT             0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_STATUS_DEFAULT                                0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ0_DCN_VM_CONTEXT0_CNTL_DEFAULT                                  0x00012010
-#define mmHUBPREQ0_DCN_VM_MX_L1_TLB_CNTL_DEFAULT                                 0x00000000
-#define mmHUBPREQ0_BLANK_OFFSET_0_DEFAULT                                        0x00000000
-#define mmHUBPREQ0_BLANK_OFFSET_1_DEFAULT                                        0x00000000
-#define mmHUBPREQ0_DST_DIMENSIONS_DEFAULT                                        0x00000000
-#define mmHUBPREQ0_DST_AFTER_SCALER_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_PREFETCH_SETTINS_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_PREFETCH_SETTINS_C_DEFAULT                                    0x00000000
-#define mmHUBPREQ0_VBLANK_PARAMETERS_0_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_VBLANK_PARAMETERS_1_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_VBLANK_PARAMETERS_2_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_VBLANK_PARAMETERS_3_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_VBLANK_PARAMETERS_4_DEFAULT                                   0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_0_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_1_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_2_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_3_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_4_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_5_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_6_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_NOM_PARAMETERS_7_DEFAULT                                      0x00000000
-#define mmHUBPREQ0_PER_LINE_DELIVERY_PRE_DEFAULT                                 0x00000000
-#define mmHUBPREQ0_PER_LINE_DELIVERY_DEFAULT                                     0x00000000
-#define mmHUBPREQ0_CURSOR_SETTINS_DEFAULT                                        0x00000000
-#define mmHUBPREQ0_REF_FREQ_TO_PIX_FREQ_DEFAULT                                  0x00000000
-#define mmHUBPREQ0_HUBPREQ_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPREQ0_HUBPREQ_MEM_PWR_STATUS_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcbubp0_dispdec_hubpret_dispdec
-#define mmHUBPRET0_HUBPRET_CONTROL_DEFAULT                                       0x00e40000
-#define mmHUBPRET0_HUBPRET_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPRET0_HUBPRET_MEM_PWR_STATUS_DEFAULT                                0x00000000
-#define mmHUBPRET0_HUBPRET_READ_LINE_CTRL0_DEFAULT                               0x00000000
-#define mmHUBPRET0_HUBPRET_READ_LINE_CTRL1_DEFAULT                               0x00000000
-#define mmHUBPRET0_HUBPRET_READ_LINE0_DEFAULT                                    0x00000000
-#define mmHUBPRET0_HUBPRET_READ_LINE1_DEFAULT                                    0x00000000
-#define mmHUBPRET0_HUBPRET_INTERRUPT_DEFAULT                                     0x00000000
-#define mmHUBPRET0_HUBPRET_READ_LINE_VALUE_DEFAULT                               0x00000000
-#define mmHUBPRET0_HUBPRET_READ_LINE_STATUS_DEFAULT                              0x00000421
-
-
-// addressBlock: dce_dc_dcbubp0_dispdec_cursor_dispdec
-#define mmCURSOR0_CURSOR_CONTROL_DEFAULT                                         0x01000000
-#define mmCURSOR0_CURSOR_SURFACE_ADDRESS_DEFAULT                                 0x00000000
-#define mmCURSOR0_CURSOR_SURFACE_ADDRESS_HIGH_DEFAULT                            0x00000000
-#define mmCURSOR0_CURSOR_SIZE_DEFAULT                                            0x00000000
-#define mmCURSOR0_CURSOR_POSITION_DEFAULT                                        0x00000000
-#define mmCURSOR0_CURSOR_HOT_SPOT_DEFAULT                                        0x00000000
-#define mmCURSOR0_CURSOR_STEREO_CONTROL_DEFAULT                                  0x00000000
-#define mmCURSOR0_CURSOR_DST_OFFSET_DEFAULT                                      0x00000000
-#define mmCURSOR0_CURSOR_MEM_PWR_CTRL_DEFAULT                                    0x00000000
-#define mmCURSOR0_CURSOR_MEM_PWR_STATUS_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dcbubp0_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON8_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON8_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON8_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON8_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON8_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON8_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON8_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON8_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON8_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dispdec
-#define mmHUBP1_DCSURF_SURFACE_CONFIG_DEFAULT                                    0x00000008
-#define mmHUBP1_DCSURF_ADDR_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP1_DCSURF_TILING_CONFIG_DEFAULT                                     0x00000080
-#define mmHUBP1_DCSURF_PRI_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP1_DCSURF_PRI_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP1_DCSURF_PRI_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP1_DCSURF_SEC_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP1_DCSURF_SEC_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP1_DCSURF_SEC_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP1_DCHUBP_REQ_SIZE_CONFIG_DEFAULT                                   0x00000000
-#define mmHUBP1_DCHUBP_REQ_SIZE_CONFIG_C_DEFAULT                                 0x00000000
-#define mmHUBP1_DCHUBP_CNTL_DEFAULT                                              0x00001001
-#define mmHUBP1_HUBP_CLK_CNTL_DEFAULT                                            0x00000000
-#define mmHUBP1_DCHUBP_VMPG_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP1_HUBPREQ_DEBUG_DB_DEFAULT                                         0x00000000
-#define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK_DEFAULT                             0x00000000
-#define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_dcbubp1_dispdec_hubpreq_dispdec
-#define mmHUBPREQ1_DCSURF_SURFACE_PITCH_DEFAULT                                  0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_PITCH_C_DEFAULT                                0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_C_DEFAULT                      0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                 0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_DEFAULT                      0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                 0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_C_DEFAULT                    0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_DEFAULT               0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_DEFAULT                   0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_DEFAULT              0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_DEFAULT                 0x00000000
-#define mmHUBPREQ1_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT            0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_DEFAULT                 0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_DEFAULT            0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_DEFAULT               0x00000000
-#define mmHUBPREQ1_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT          0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_CONTROL_DEFAULT                                0x00000000
-#define mmHUBPREQ1_DCSURF_FLIP_CONTROL_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_DCSURF_FLIP_CONTROL2_DEFAULT                                  0x00003040
-#define mmHUBPREQ1_DCSURF_FRAME_PACING_CONTROL_DEFAULT                           0x04000000
-#define mmHUBPREQ1_DCSURF_FRAME_PACING_TIME_DEFAULT                              0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_FLIP_INTERRUPT_DEFAULT                         0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_DEFAULT                                  0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_DEFAULT                             0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_C_DEFAULT                                0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_INUSE_HIGH_C_DEFAULT                           0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_DEFAULT                         0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_DEFAULT                    0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_C_DEFAULT                       0x00000000
-#define mmHUBPREQ1_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_DEFAULT                  0x00000000
-#define mmHUBPREQ1_DCN_EXPANSION_MODE_DEFAULT                                    0x00000055
-#define mmHUBPREQ1_DCN_TTU_QOS_WM_DEFAULT                                        0x00000000
-#define mmHUBPREQ1_DCN_GLOBAL_TTU_CNTL_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_DCN_SURF0_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_DCN_SURF0_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_DCN_SURF1_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_DCN_SURF1_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_DCN_CUR0_TTU_CNTL0_DEFAULT                                    0x00000000
-#define mmHUBPREQ1_DCN_CUR0_TTU_CNTL1_DEFAULT                                    0x00000000
-#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB_DEFAULT                   0x00000000
-#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB_DEFAULT                   0x00000000
-#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB_DEFAULT                  0x00000000
-#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB_DEFAULT                  0x00000000
-#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ1_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_DEFAULT     0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_DEFAULT     0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_DEFAULT              0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_DEFAULT              0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_DEFAULT             0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_STATUS_DEFAULT                                0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ1_DCN_VM_CONTEXT0_CNTL_DEFAULT                                  0x00012010
-#define mmHUBPREQ1_DCN_VM_MX_L1_TLB_CNTL_DEFAULT                                 0x00000000
-#define mmHUBPREQ1_BLANK_OFFSET_0_DEFAULT                                        0x00000000
-#define mmHUBPREQ1_BLANK_OFFSET_1_DEFAULT                                        0x00000000
-#define mmHUBPREQ1_DST_DIMENSIONS_DEFAULT                                        0x00000000
-#define mmHUBPREQ1_DST_AFTER_SCALER_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_PREFETCH_SETTINS_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_PREFETCH_SETTINS_C_DEFAULT                                    0x00000000
-#define mmHUBPREQ1_VBLANK_PARAMETERS_0_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_VBLANK_PARAMETERS_1_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_VBLANK_PARAMETERS_2_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_VBLANK_PARAMETERS_3_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_VBLANK_PARAMETERS_4_DEFAULT                                   0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_0_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_1_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_2_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_3_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_4_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_5_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_6_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_NOM_PARAMETERS_7_DEFAULT                                      0x00000000
-#define mmHUBPREQ1_PER_LINE_DELIVERY_PRE_DEFAULT                                 0x00000000
-#define mmHUBPREQ1_PER_LINE_DELIVERY_DEFAULT                                     0x00000000
-#define mmHUBPREQ1_CURSOR_SETTINS_DEFAULT                                        0x00000000
-#define mmHUBPREQ1_REF_FREQ_TO_PIX_FREQ_DEFAULT                                  0x00000000
-#define mmHUBPREQ1_HUBPREQ_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPREQ1_HUBPREQ_MEM_PWR_STATUS_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcbubp1_dispdec_hubpret_dispdec
-#define mmHUBPRET1_HUBPRET_CONTROL_DEFAULT                                       0x00e40000
-#define mmHUBPRET1_HUBPRET_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPRET1_HUBPRET_MEM_PWR_STATUS_DEFAULT                                0x00000000
-#define mmHUBPRET1_HUBPRET_READ_LINE_CTRL0_DEFAULT                               0x00000000
-#define mmHUBPRET1_HUBPRET_READ_LINE_CTRL1_DEFAULT                               0x00000000
-#define mmHUBPRET1_HUBPRET_READ_LINE0_DEFAULT                                    0x00000000
-#define mmHUBPRET1_HUBPRET_READ_LINE1_DEFAULT                                    0x00000000
-#define mmHUBPRET1_HUBPRET_INTERRUPT_DEFAULT                                     0x00000000
-#define mmHUBPRET1_HUBPRET_READ_LINE_VALUE_DEFAULT                               0x00000000
-#define mmHUBPRET1_HUBPRET_READ_LINE_STATUS_DEFAULT                              0x00000421
-
-
-// addressBlock: dce_dc_dcbubp1_dispdec_cursor_dispdec
-#define mmCURSOR1_CURSOR_CONTROL_DEFAULT                                         0x01000000
-#define mmCURSOR1_CURSOR_SURFACE_ADDRESS_DEFAULT                                 0x00000000
-#define mmCURSOR1_CURSOR_SURFACE_ADDRESS_HIGH_DEFAULT                            0x00000000
-#define mmCURSOR1_CURSOR_SIZE_DEFAULT                                            0x00000000
-#define mmCURSOR1_CURSOR_POSITION_DEFAULT                                        0x00000000
-#define mmCURSOR1_CURSOR_HOT_SPOT_DEFAULT                                        0x00000000
-#define mmCURSOR1_CURSOR_STEREO_CONTROL_DEFAULT                                  0x00000000
-#define mmCURSOR1_CURSOR_DST_OFFSET_DEFAULT                                      0x00000000
-#define mmCURSOR1_CURSOR_MEM_PWR_CTRL_DEFAULT                                    0x00000000
-#define mmCURSOR1_CURSOR_MEM_PWR_STATUS_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dcbubp1_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON9_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON9_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON9_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON9_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON9_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON9_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON9_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON9_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON9_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dispdec
-#define mmHUBP2_DCSURF_SURFACE_CONFIG_DEFAULT                                    0x00000008
-#define mmHUBP2_DCSURF_ADDR_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP2_DCSURF_TILING_CONFIG_DEFAULT                                     0x00000080
-#define mmHUBP2_DCSURF_PRI_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP2_DCSURF_PRI_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP2_DCSURF_PRI_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP2_DCSURF_SEC_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP2_DCSURF_SEC_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP2_DCSURF_SEC_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP2_DCHUBP_REQ_SIZE_CONFIG_DEFAULT                                   0x00000000
-#define mmHUBP2_DCHUBP_REQ_SIZE_CONFIG_C_DEFAULT                                 0x00000000
-#define mmHUBP2_DCHUBP_CNTL_DEFAULT                                              0x00001001
-#define mmHUBP2_HUBP_CLK_CNTL_DEFAULT                                            0x00000000
-#define mmHUBP2_DCHUBP_VMPG_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP2_HUBPREQ_DEBUG_DB_DEFAULT                                         0x00000000
-#define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK_DEFAULT                             0x00000000
-#define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_dcbubp2_dispdec_hubpreq_dispdec
-#define mmHUBPREQ2_DCSURF_SURFACE_PITCH_DEFAULT                                  0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_PITCH_C_DEFAULT                                0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_C_DEFAULT                      0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                 0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_DEFAULT                      0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                 0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_C_DEFAULT                    0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_DEFAULT               0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_DEFAULT                   0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_DEFAULT              0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_DEFAULT                 0x00000000
-#define mmHUBPREQ2_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT            0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_DEFAULT                 0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_DEFAULT            0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_DEFAULT               0x00000000
-#define mmHUBPREQ2_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT          0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_CONTROL_DEFAULT                                0x00000000
-#define mmHUBPREQ2_DCSURF_FLIP_CONTROL_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_DCSURF_FLIP_CONTROL2_DEFAULT                                  0x00003040
-#define mmHUBPREQ2_DCSURF_FRAME_PACING_CONTROL_DEFAULT                           0x04000000
-#define mmHUBPREQ2_DCSURF_FRAME_PACING_TIME_DEFAULT                              0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_FLIP_INTERRUPT_DEFAULT                         0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_DEFAULT                                  0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_DEFAULT                             0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_C_DEFAULT                                0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_INUSE_HIGH_C_DEFAULT                           0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_DEFAULT                         0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_DEFAULT                    0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_C_DEFAULT                       0x00000000
-#define mmHUBPREQ2_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_DEFAULT                  0x00000000
-#define mmHUBPREQ2_DCN_EXPANSION_MODE_DEFAULT                                    0x00000055
-#define mmHUBPREQ2_DCN_TTU_QOS_WM_DEFAULT                                        0x00000000
-#define mmHUBPREQ2_DCN_GLOBAL_TTU_CNTL_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_DCN_SURF0_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_DCN_SURF0_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_DCN_SURF1_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_DCN_SURF1_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_DCN_CUR0_TTU_CNTL0_DEFAULT                                    0x00000000
-#define mmHUBPREQ2_DCN_CUR0_TTU_CNTL1_DEFAULT                                    0x00000000
-#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB_DEFAULT                   0x00000000
-#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB_DEFAULT                   0x00000000
-#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB_DEFAULT                  0x00000000
-#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB_DEFAULT                  0x00000000
-#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ2_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_DEFAULT     0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_DEFAULT     0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_DEFAULT              0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_DEFAULT              0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_DEFAULT             0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_STATUS_DEFAULT                                0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ2_DCN_VM_CONTEXT0_CNTL_DEFAULT                                  0x00012010
-#define mmHUBPREQ2_DCN_VM_MX_L1_TLB_CNTL_DEFAULT                                 0x00000000
-#define mmHUBPREQ2_BLANK_OFFSET_0_DEFAULT                                        0x00000000
-#define mmHUBPREQ2_BLANK_OFFSET_1_DEFAULT                                        0x00000000
-#define mmHUBPREQ2_DST_DIMENSIONS_DEFAULT                                        0x00000000
-#define mmHUBPREQ2_DST_AFTER_SCALER_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_PREFETCH_SETTINS_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_PREFETCH_SETTINS_C_DEFAULT                                    0x00000000
-#define mmHUBPREQ2_VBLANK_PARAMETERS_0_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_VBLANK_PARAMETERS_1_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_VBLANK_PARAMETERS_2_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_VBLANK_PARAMETERS_3_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_VBLANK_PARAMETERS_4_DEFAULT                                   0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_0_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_1_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_2_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_3_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_4_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_5_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_6_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_NOM_PARAMETERS_7_DEFAULT                                      0x00000000
-#define mmHUBPREQ2_PER_LINE_DELIVERY_PRE_DEFAULT                                 0x00000000
-#define mmHUBPREQ2_PER_LINE_DELIVERY_DEFAULT                                     0x00000000
-#define mmHUBPREQ2_CURSOR_SETTINS_DEFAULT                                        0x00000000
-#define mmHUBPREQ2_REF_FREQ_TO_PIX_FREQ_DEFAULT                                  0x00000000
-#define mmHUBPREQ2_HUBPREQ_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPREQ2_HUBPREQ_MEM_PWR_STATUS_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcbubp2_dispdec_hubpret_dispdec
-#define mmHUBPRET2_HUBPRET_CONTROL_DEFAULT                                       0x00e40000
-#define mmHUBPRET2_HUBPRET_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPRET2_HUBPRET_MEM_PWR_STATUS_DEFAULT                                0x00000000
-#define mmHUBPRET2_HUBPRET_READ_LINE_CTRL0_DEFAULT                               0x00000000
-#define mmHUBPRET2_HUBPRET_READ_LINE_CTRL1_DEFAULT                               0x00000000
-#define mmHUBPRET2_HUBPRET_READ_LINE0_DEFAULT                                    0x00000000
-#define mmHUBPRET2_HUBPRET_READ_LINE1_DEFAULT                                    0x00000000
-#define mmHUBPRET2_HUBPRET_INTERRUPT_DEFAULT                                     0x00000000
-#define mmHUBPRET2_HUBPRET_READ_LINE_VALUE_DEFAULT                               0x00000000
-#define mmHUBPRET2_HUBPRET_READ_LINE_STATUS_DEFAULT                              0x00000421
-
-
-// addressBlock: dce_dc_dcbubp2_dispdec_cursor_dispdec
-#define mmCURSOR2_CURSOR_CONTROL_DEFAULT                                         0x01000000
-#define mmCURSOR2_CURSOR_SURFACE_ADDRESS_DEFAULT                                 0x00000000
-#define mmCURSOR2_CURSOR_SURFACE_ADDRESS_HIGH_DEFAULT                            0x00000000
-#define mmCURSOR2_CURSOR_SIZE_DEFAULT                                            0x00000000
-#define mmCURSOR2_CURSOR_POSITION_DEFAULT                                        0x00000000
-#define mmCURSOR2_CURSOR_HOT_SPOT_DEFAULT                                        0x00000000
-#define mmCURSOR2_CURSOR_STEREO_CONTROL_DEFAULT                                  0x00000000
-#define mmCURSOR2_CURSOR_DST_OFFSET_DEFAULT                                      0x00000000
-#define mmCURSOR2_CURSOR_MEM_PWR_CTRL_DEFAULT                                    0x00000000
-#define mmCURSOR2_CURSOR_MEM_PWR_STATUS_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dcbubp2_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON10_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON10_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON10_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON10_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON10_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON10_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON10_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON10_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON10_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dispdec
-#define mmHUBP3_DCSURF_SURFACE_CONFIG_DEFAULT                                    0x00000008
-#define mmHUBP3_DCSURF_ADDR_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP3_DCSURF_TILING_CONFIG_DEFAULT                                     0x00000080
-#define mmHUBP3_DCSURF_PRI_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP3_DCSURF_PRI_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP3_DCSURF_PRI_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP3_DCSURF_SEC_VIEWPORT_START_DEFAULT                                0x00000000
-#define mmHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_DEFAULT                            0x00000000
-#define mmHUBP3_DCSURF_SEC_VIEWPORT_START_C_DEFAULT                              0x00000000
-#define mmHUBP3_DCSURF_SEC_VIEWPORT_DIMENSION_C_DEFAULT                          0x00000000
-#define mmHUBP3_DCHUBP_REQ_SIZE_CONFIG_DEFAULT                                   0x00000000
-#define mmHUBP3_DCHUBP_REQ_SIZE_CONFIG_C_DEFAULT                                 0x00000000
-#define mmHUBP3_DCHUBP_CNTL_DEFAULT                                              0x00001001
-#define mmHUBP3_HUBP_CLK_CNTL_DEFAULT                                            0x00000000
-#define mmHUBP3_DCHUBP_VMPG_CONFIG_DEFAULT                                       0x00000000
-#define mmHUBP3_HUBPREQ_DEBUG_DB_DEFAULT                                         0x00000000
-#define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK_DEFAULT                             0x00000000
-#define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_dcbubp3_dispdec_hubpreq_dispdec
-#define mmHUBPREQ3_DCSURF_SURFACE_PITCH_DEFAULT                                  0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_PITCH_C_DEFAULT                                0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_C_DEFAULT                      0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                 0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_DEFAULT                      0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                 0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_C_DEFAULT                    0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C_DEFAULT               0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_DEFAULT                   0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_DEFAULT              0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_C_DEFAULT                 0x00000000
-#define mmHUBPREQ3_DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT            0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_DEFAULT                 0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_DEFAULT            0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_C_DEFAULT               0x00000000
-#define mmHUBPREQ3_DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C_DEFAULT          0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_CONTROL_DEFAULT                                0x00000000
-#define mmHUBPREQ3_DCSURF_FLIP_CONTROL_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_DCSURF_FLIP_CONTROL2_DEFAULT                                  0x00003040
-#define mmHUBPREQ3_DCSURF_FRAME_PACING_CONTROL_DEFAULT                           0x04000000
-#define mmHUBPREQ3_DCSURF_FRAME_PACING_TIME_DEFAULT                              0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_FLIP_INTERRUPT_DEFAULT                         0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_DEFAULT                                  0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_DEFAULT                             0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_C_DEFAULT                                0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_INUSE_HIGH_C_DEFAULT                           0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_DEFAULT                         0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_DEFAULT                    0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_C_DEFAULT                       0x00000000
-#define mmHUBPREQ3_DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C_DEFAULT                  0x00000000
-#define mmHUBPREQ3_DCN_EXPANSION_MODE_DEFAULT                                    0x00000055
-#define mmHUBPREQ3_DCN_TTU_QOS_WM_DEFAULT                                        0x00000000
-#define mmHUBPREQ3_DCN_GLOBAL_TTU_CNTL_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_DCN_SURF0_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_DCN_SURF0_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_DCN_SURF1_TTU_CNTL0_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_DCN_SURF1_TTU_CNTL1_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_DCN_CUR0_TTU_CNTL0_DEFAULT                                    0x00000000
-#define mmHUBPREQ3_DCN_CUR0_TTU_CNTL1_DEFAULT                                    0x00000000
-#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_LSB_DEFAULT                   0x00000000
-#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_LOW_ADDR_MSB_DEFAULT                   0x00000000
-#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_LSB_DEFAULT                  0x00000000
-#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_HIGH_ADDR_MSB_DEFAULT                  0x00000000
-#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ3_DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_LSB_DEFAULT     0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR_MSB_DEFAULT     0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LSB_DEFAULT              0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_MSB_DEFAULT              0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PAGE_TABLE_START_ADDR_MSB_DEFAULT             0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LSB_DEFAULT               0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PAGE_TABLE_END_ADDR_MSB_DEFAULT               0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_STATUS_DEFAULT                                0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_PROTECTION_FAULT_ADDR_LSB_DEFAULT             0x00000000
-#define mmHUBPREQ3_DCN_VM_CONTEXT0_CNTL_DEFAULT                                  0x00012010
-#define mmHUBPREQ3_DCN_VM_MX_L1_TLB_CNTL_DEFAULT                                 0x00000000
-#define mmHUBPREQ3_BLANK_OFFSET_0_DEFAULT                                        0x00000000
-#define mmHUBPREQ3_BLANK_OFFSET_1_DEFAULT                                        0x00000000
-#define mmHUBPREQ3_DST_DIMENSIONS_DEFAULT                                        0x00000000
-#define mmHUBPREQ3_DST_AFTER_SCALER_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_PREFETCH_SETTINS_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_PREFETCH_SETTINS_C_DEFAULT                                    0x00000000
-#define mmHUBPREQ3_VBLANK_PARAMETERS_0_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_VBLANK_PARAMETERS_1_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_VBLANK_PARAMETERS_2_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_VBLANK_PARAMETERS_3_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_VBLANK_PARAMETERS_4_DEFAULT                                   0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_0_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_1_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_2_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_3_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_4_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_5_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_6_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_NOM_PARAMETERS_7_DEFAULT                                      0x00000000
-#define mmHUBPREQ3_PER_LINE_DELIVERY_PRE_DEFAULT                                 0x00000000
-#define mmHUBPREQ3_PER_LINE_DELIVERY_DEFAULT                                     0x00000000
-#define mmHUBPREQ3_CURSOR_SETTINS_DEFAULT                                        0x00000000
-#define mmHUBPREQ3_REF_FREQ_TO_PIX_FREQ_DEFAULT                                  0x00000000
-#define mmHUBPREQ3_HUBPREQ_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPREQ3_HUBPREQ_MEM_PWR_STATUS_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcbubp3_dispdec_hubpret_dispdec
-#define mmHUBPRET3_HUBPRET_CONTROL_DEFAULT                                       0x00e40000
-#define mmHUBPRET3_HUBPRET_MEM_PWR_CTRL_DEFAULT                                  0x00000000
-#define mmHUBPRET3_HUBPRET_MEM_PWR_STATUS_DEFAULT                                0x00000000
-#define mmHUBPRET3_HUBPRET_READ_LINE_CTRL0_DEFAULT                               0x00000000
-#define mmHUBPRET3_HUBPRET_READ_LINE_CTRL1_DEFAULT                               0x00000000
-#define mmHUBPRET3_HUBPRET_READ_LINE0_DEFAULT                                    0x00000000
-#define mmHUBPRET3_HUBPRET_READ_LINE1_DEFAULT                                    0x00000000
-#define mmHUBPRET3_HUBPRET_INTERRUPT_DEFAULT                                     0x00000000
-#define mmHUBPRET3_HUBPRET_READ_LINE_VALUE_DEFAULT                               0x00000000
-#define mmHUBPRET3_HUBPRET_READ_LINE_STATUS_DEFAULT                              0x00000421
-
-
-// addressBlock: dce_dc_dcbubp3_dispdec_cursor_dispdec
-#define mmCURSOR3_CURSOR_CONTROL_DEFAULT                                         0x01000000
-#define mmCURSOR3_CURSOR_SURFACE_ADDRESS_DEFAULT                                 0x00000000
-#define mmCURSOR3_CURSOR_SURFACE_ADDRESS_HIGH_DEFAULT                            0x00000000
-#define mmCURSOR3_CURSOR_SIZE_DEFAULT                                            0x00000000
-#define mmCURSOR3_CURSOR_POSITION_DEFAULT                                        0x00000000
-#define mmCURSOR3_CURSOR_HOT_SPOT_DEFAULT                                        0x00000000
-#define mmCURSOR3_CURSOR_STEREO_CONTROL_DEFAULT                                  0x00000000
-#define mmCURSOR3_CURSOR_DST_OFFSET_DEFAULT                                      0x00000000
-#define mmCURSOR3_CURSOR_MEM_PWR_CTRL_DEFAULT                                    0x00000000
-#define mmCURSOR3_CURSOR_MEM_PWR_STATUS_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dcbubp3_dispdec_hubp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON11_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON11_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON11_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON11_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON11_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON11_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON11_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON11_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON11_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dpp0_dispdec_dpp_top_dispdec
-#define mmDPP_TOP0_DPP_CONTROL_DEFAULT                                           0x70000000
-#define mmDPP_TOP0_DPP_SOFT_RESET_DEFAULT                                        0x00000000
-#define mmDPP_TOP0_DPP_CRC_VAL_R_G_DEFAULT                                       0x00000000
-#define mmDPP_TOP0_DPP_CRC_VAL_B_A_DEFAULT                                       0x00000000
-#define mmDPP_TOP0_DPP_CRC_CTRL_DEFAULT                                          0x00000000
-#define mmDPP_TOP0_HOST_READ_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp0_dispdec_cnvc_cfg_dispdec
-#define mmCNVC_CFG0_CNVC_SURFACE_PIXEL_FORMAT_DEFAULT                            0x00000008
-#define mmCNVC_CFG0_FORMAT_CONTROL_DEFAULT                                       0x00000000
-#define mmCNVC_CFG0_FCNV_FP_SCALE_BIAS_DEFAULT                                   0x00003c00
-#define mmCNVC_CFG0_DENORM_CONTROL_DEFAULT                                       0x00002000
-#define mmCNVC_CFG0_COLOR_KEYER_CONTROL_DEFAULT                                  0x00000000
-#define mmCNVC_CFG0_COLOR_KEYER_ALPHA_DEFAULT                                    0x00000000
-#define mmCNVC_CFG0_COLOR_KEYER_RED_DEFAULT                                      0x00000000
-#define mmCNVC_CFG0_COLOR_KEYER_GREEN_DEFAULT                                    0x00000000
-#define mmCNVC_CFG0_COLOR_KEYER_BLUE_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp0_dispdec_cnvc_cur_dispdec
-#define mmCNVC_CUR0_CURSOR0_CONTROL_DEFAULT                                      0x0003ff00
-#define mmCNVC_CUR0_CURSOR0_COLOR0_DEFAULT                                       0x00000000
-#define mmCNVC_CUR0_CURSOR0_COLOR1_DEFAULT                                       0x00000000
-#define mmCNVC_CUR0_CURSOR0_FP_SCALE_BIAS_DEFAULT                                0x00003c00
-
-
-// addressBlock: dce_dc_dpp0_dispdec_dscl_dispdec
-#define mmDSCL0_SCL_COEF_RAM_TAP_SELECT_DEFAULT                                  0x00000000
-#define mmDSCL0_SCL_COEF_RAM_TAP_DATA_DEFAULT                                    0x00000000
-#define mmDSCL0_SCL_MODE_DEFAULT                                                 0x00000000
-#define mmDSCL0_SCL_TAP_CONTROL_DEFAULT                                          0x00000000
-#define mmDSCL0_DSCL_CONTROL_DEFAULT                                             0x00000000
-#define mmDSCL0_DSCL_2TAP_CONTROL_DEFAULT                                        0x01000100
-#define mmDSCL0_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                             0x00000000
-#define mmDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL0_SCL_HORZ_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL0_SCL_HORZ_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL0_SCL_HORZ_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL0_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL0_SCL_VERT_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL0_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                 0x01000000
-#define mmDSCL0_SCL_VERT_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL0_SCL_VERT_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL0_SCL_VERT_FILTER_INIT_BOT_C_DEFAULT                               0x01000000
-#define mmDSCL0_SCL_BLACK_OFFSET_DEFAULT                                         0x80000000
-#define mmDSCL0_DSCL_UPDATE_DEFAULT                                              0x00000000
-#define mmDSCL0_DSCL_AUTOCAL_DEFAULT                                             0x00000000
-#define mmDSCL0_DSCL_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                             0x00000000
-#define mmDSCL0_DSCL_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                             0x00000000
-#define mmDSCL0_OTG_H_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL0_OTG_V_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL0_RECOUT_START_DEFAULT                                             0x00000000
-#define mmDSCL0_RECOUT_SIZE_DEFAULT                                              0x00000000
-#define mmDSCL0_MPC_SIZE_DEFAULT                                                 0x00000000
-#define mmDSCL0_LB_DATA_FORMAT_DEFAULT                                           0x00000000
-#define mmDSCL0_LB_MEMORY_CTRL_DEFAULT                                           0x00003f00
-#define mmDSCL0_LB_V_COUNTER_DEFAULT                                             0x00000000
-#define mmDSCL0_DSCL_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDSCL0_DSCL_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDSCL0_OBUF_CONTROL_DEFAULT                                             0xe0000000
-#define mmDSCL0_OBUF_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dpp0_dispdec_cm_dispdec
-#define mmCM0_CM_CONTROL_DEFAULT                                                 0x00000000
-#define mmCM0_CM_COMA_C11_C12_DEFAULT                                            0x00002000
-#define mmCM0_CM_COMA_C13_C14_DEFAULT                                            0x00000000
-#define mmCM0_CM_COMA_C21_C22_DEFAULT                                            0x20000000
-#define mmCM0_CM_COMA_C23_C24_DEFAULT                                            0x00000000
-#define mmCM0_CM_COMA_C31_C32_DEFAULT                                            0x00000000
-#define mmCM0_CM_COMA_C33_C34_DEFAULT                                            0x00002000
-#define mmCM0_CM_COMB_C11_C12_DEFAULT                                            0x00002000
-#define mmCM0_CM_COMB_C13_C14_DEFAULT                                            0x00000000
-#define mmCM0_CM_COMB_C21_C22_DEFAULT                                            0x20000000
-#define mmCM0_CM_COMB_C23_C24_DEFAULT                                            0x00000000
-#define mmCM0_CM_COMB_C31_C32_DEFAULT                                            0x00000000
-#define mmCM0_CM_COMB_C33_C34_DEFAULT                                            0x00002000
-#define mmCM0_CM_IGAM_CONTROL_DEFAULT                                            0x08000002
-#define mmCM0_CM_IGAM_LUT_RW_CONTROL_DEFAULT                                     0x00011070
-#define mmCM0_CM_IGAM_LUT_RW_INDEX_DEFAULT                                       0x00000000
-#define mmCM0_CM_IGAM_LUT_SEQ_COLOR_DEFAULT                                      0x00000000
-#define mmCM0_CM_IGAM_LUT_30_COLOR_DEFAULT                                       0x00000000
-#define mmCM0_CM_IGAM_LUT_PWL_DATA_DEFAULT                                       0x00000000
-#define mmCM0_CM_IGAM_LUT_AUTOFILL_DEFAULT                                       0x00000000
-#define mmCM0_CM_IGAM_LUT_BW_OFFSET_BLUE_DEFAULT                                 0xffff0000
-#define mmCM0_CM_IGAM_LUT_BW_OFFSET_GREEN_DEFAULT                                0xffff0000
-#define mmCM0_CM_IGAM_LUT_BW_OFFSET_RED_DEFAULT                                  0xffff0000
-#define mmCM0_CM_ICSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM0_CM_ICSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM0_CM_ICSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM0_CM_ICSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM0_CM_ICSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM0_CM_ICSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM0_CM_ICSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM0_CM_GAMUT_REMAP_CONTROL_DEFAULT                                     0x00000000
-#define mmCM0_CM_GAMUT_REMAP_C11_C12_DEFAULT                                     0x00002000
-#define mmCM0_CM_GAMUT_REMAP_C13_C14_DEFAULT                                     0x00000000
-#define mmCM0_CM_GAMUT_REMAP_C21_C22_DEFAULT                                     0x20000000
-#define mmCM0_CM_GAMUT_REMAP_C23_C24_DEFAULT                                     0x00000000
-#define mmCM0_CM_GAMUT_REMAP_C31_C32_DEFAULT                                     0x00000000
-#define mmCM0_CM_GAMUT_REMAP_C33_C34_DEFAULT                                     0x00002000
-#define mmCM0_CM_OCSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM0_CM_OCSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM0_CM_OCSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM0_CM_OCSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM0_CM_OCSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM0_CM_OCSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM0_CM_OCSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM0_CM_BNS_VALUES_R_DEFAULT                                            0x20000000
-#define mmCM0_CM_BNS_VALUES_G_DEFAULT                                            0x20000000
-#define mmCM0_CM_BNS_VALUES_B_DEFAULT                                            0x20000000
-#define mmCM0_CM_DGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM0_CM_DGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM0_CM_DGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM0_CM_DGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM0_CM_DGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM0_CM_DGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM0_CM_RGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM0_CM_RGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM0_CM_RGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM0_CM_RGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMA_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM0_CM_RGAM_RAMB_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM0_CM_HDR_MULT_COEF_DEFAULT                                           0x0001f000
-#define mmCM0_CM_RANGE_CLAMP_CONTROL_R_DEFAULT                                   0xfbff7bff
-#define mmCM0_CM_RANGE_CLAMP_CONTROL_G_DEFAULT                                   0xfbff7bff
-#define mmCM0_CM_RANGE_CLAMP_CONTROL_B_DEFAULT                                   0xfbff7bff
-#define mmCM0_CM_DENORM_CONTROL_DEFAULT                                          0x00000000
-#define mmCM0_CM_CMOUT_CONTROL_DEFAULT                                           0x0000000a
-#define mmCM0_CM_CMOUT_RANDOM_SEEDS_DEFAULT                                      0x00000000
-#define mmCM0_CM_MEM_PWR_CTRL_DEFAULT                                            0x00000000
-#define mmCM0_CM_MEM_PWR_STATUS_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_dpp0_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON12_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON12_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON12_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON12_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON12_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON12_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON12_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON12_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON12_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dpp1_dispdec_dpp_top_dispdec
-#define mmDPP_TOP1_DPP_CONTROL_DEFAULT                                           0x70000000
-#define mmDPP_TOP1_DPP_SOFT_RESET_DEFAULT                                        0x00000000
-#define mmDPP_TOP1_DPP_CRC_VAL_R_G_DEFAULT                                       0x00000000
-#define mmDPP_TOP1_DPP_CRC_VAL_B_A_DEFAULT                                       0x00000000
-#define mmDPP_TOP1_DPP_CRC_CTRL_DEFAULT                                          0x00000000
-#define mmDPP_TOP1_HOST_READ_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp1_dispdec_cnvc_cfg_dispdec
-#define mmCNVC_CFG1_CNVC_SURFACE_PIXEL_FORMAT_DEFAULT                            0x00000008
-#define mmCNVC_CFG1_FORMAT_CONTROL_DEFAULT                                       0x00000000
-#define mmCNVC_CFG1_FCNV_FP_SCALE_BIAS_DEFAULT                                   0x00003c00
-#define mmCNVC_CFG1_DENORM_CONTROL_DEFAULT                                       0x00002000
-#define mmCNVC_CFG1_COLOR_KEYER_CONTROL_DEFAULT                                  0x00000000
-#define mmCNVC_CFG1_COLOR_KEYER_ALPHA_DEFAULT                                    0x00000000
-#define mmCNVC_CFG1_COLOR_KEYER_RED_DEFAULT                                      0x00000000
-#define mmCNVC_CFG1_COLOR_KEYER_GREEN_DEFAULT                                    0x00000000
-#define mmCNVC_CFG1_COLOR_KEYER_BLUE_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp1_dispdec_cnvc_cur_dispdec
-#define mmCNVC_CUR1_CURSOR0_CONTROL_DEFAULT                                      0x0003ff00
-#define mmCNVC_CUR1_CURSOR0_COLOR0_DEFAULT                                       0x00000000
-#define mmCNVC_CUR1_CURSOR0_COLOR1_DEFAULT                                       0x00000000
-#define mmCNVC_CUR1_CURSOR0_FP_SCALE_BIAS_DEFAULT                                0x00003c00
-
-
-// addressBlock: dce_dc_dpp1_dispdec_dscl_dispdec
-#define mmDSCL1_SCL_COEF_RAM_TAP_SELECT_DEFAULT                                  0x00000000
-#define mmDSCL1_SCL_COEF_RAM_TAP_DATA_DEFAULT                                    0x00000000
-#define mmDSCL1_SCL_MODE_DEFAULT                                                 0x00000000
-#define mmDSCL1_SCL_TAP_CONTROL_DEFAULT                                          0x00000000
-#define mmDSCL1_DSCL_CONTROL_DEFAULT                                             0x00000000
-#define mmDSCL1_DSCL_2TAP_CONTROL_DEFAULT                                        0x01000100
-#define mmDSCL1_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                             0x00000000
-#define mmDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL1_SCL_HORZ_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL1_SCL_HORZ_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL1_SCL_HORZ_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL1_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL1_SCL_VERT_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL1_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                 0x01000000
-#define mmDSCL1_SCL_VERT_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL1_SCL_VERT_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL1_SCL_VERT_FILTER_INIT_BOT_C_DEFAULT                               0x01000000
-#define mmDSCL1_SCL_BLACK_OFFSET_DEFAULT                                         0x80000000
-#define mmDSCL1_DSCL_UPDATE_DEFAULT                                              0x00000000
-#define mmDSCL1_DSCL_AUTOCAL_DEFAULT                                             0x00000000
-#define mmDSCL1_DSCL_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                             0x00000000
-#define mmDSCL1_DSCL_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                             0x00000000
-#define mmDSCL1_OTG_H_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL1_OTG_V_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL1_RECOUT_START_DEFAULT                                             0x00000000
-#define mmDSCL1_RECOUT_SIZE_DEFAULT                                              0x00000000
-#define mmDSCL1_MPC_SIZE_DEFAULT                                                 0x00000000
-#define mmDSCL1_LB_DATA_FORMAT_DEFAULT                                           0x00000000
-#define mmDSCL1_LB_MEMORY_CTRL_DEFAULT                                           0x00003f00
-#define mmDSCL1_LB_V_COUNTER_DEFAULT                                             0x00000000
-#define mmDSCL1_DSCL_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDSCL1_DSCL_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDSCL1_OBUF_CONTROL_DEFAULT                                             0xe0000000
-#define mmDSCL1_OBUF_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dpp1_dispdec_cm_dispdec
-#define mmCM1_CM_CONTROL_DEFAULT                                                 0x00000000
-#define mmCM1_CM_COMA_C11_C12_DEFAULT                                            0x00002000
-#define mmCM1_CM_COMA_C13_C14_DEFAULT                                            0x00000000
-#define mmCM1_CM_COMA_C21_C22_DEFAULT                                            0x20000000
-#define mmCM1_CM_COMA_C23_C24_DEFAULT                                            0x00000000
-#define mmCM1_CM_COMA_C31_C32_DEFAULT                                            0x00000000
-#define mmCM1_CM_COMA_C33_C34_DEFAULT                                            0x00002000
-#define mmCM1_CM_COMB_C11_C12_DEFAULT                                            0x00002000
-#define mmCM1_CM_COMB_C13_C14_DEFAULT                                            0x00000000
-#define mmCM1_CM_COMB_C21_C22_DEFAULT                                            0x20000000
-#define mmCM1_CM_COMB_C23_C24_DEFAULT                                            0x00000000
-#define mmCM1_CM_COMB_C31_C32_DEFAULT                                            0x00000000
-#define mmCM1_CM_COMB_C33_C34_DEFAULT                                            0x00002000
-#define mmCM1_CM_IGAM_CONTROL_DEFAULT                                            0x08000002
-#define mmCM1_CM_IGAM_LUT_RW_CONTROL_DEFAULT                                     0x00011070
-#define mmCM1_CM_IGAM_LUT_RW_INDEX_DEFAULT                                       0x00000000
-#define mmCM1_CM_IGAM_LUT_SEQ_COLOR_DEFAULT                                      0x00000000
-#define mmCM1_CM_IGAM_LUT_30_COLOR_DEFAULT                                       0x00000000
-#define mmCM1_CM_IGAM_LUT_PWL_DATA_DEFAULT                                       0x00000000
-#define mmCM1_CM_IGAM_LUT_AUTOFILL_DEFAULT                                       0x00000000
-#define mmCM1_CM_IGAM_LUT_BW_OFFSET_BLUE_DEFAULT                                 0xffff0000
-#define mmCM1_CM_IGAM_LUT_BW_OFFSET_GREEN_DEFAULT                                0xffff0000
-#define mmCM1_CM_IGAM_LUT_BW_OFFSET_RED_DEFAULT                                  0xffff0000
-#define mmCM1_CM_ICSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM1_CM_ICSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM1_CM_ICSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM1_CM_ICSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM1_CM_ICSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM1_CM_ICSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM1_CM_ICSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM1_CM_GAMUT_REMAP_CONTROL_DEFAULT                                     0x00000000
-#define mmCM1_CM_GAMUT_REMAP_C11_C12_DEFAULT                                     0x00002000
-#define mmCM1_CM_GAMUT_REMAP_C13_C14_DEFAULT                                     0x00000000
-#define mmCM1_CM_GAMUT_REMAP_C21_C22_DEFAULT                                     0x20000000
-#define mmCM1_CM_GAMUT_REMAP_C23_C24_DEFAULT                                     0x00000000
-#define mmCM1_CM_GAMUT_REMAP_C31_C32_DEFAULT                                     0x00000000
-#define mmCM1_CM_GAMUT_REMAP_C33_C34_DEFAULT                                     0x00002000
-#define mmCM1_CM_OCSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM1_CM_OCSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM1_CM_OCSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM1_CM_OCSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM1_CM_OCSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM1_CM_OCSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM1_CM_OCSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM1_CM_BNS_VALUES_R_DEFAULT                                            0x20000000
-#define mmCM1_CM_BNS_VALUES_G_DEFAULT                                            0x20000000
-#define mmCM1_CM_BNS_VALUES_B_DEFAULT                                            0x20000000
-#define mmCM1_CM_DGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM1_CM_DGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM1_CM_DGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM1_CM_DGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM1_CM_DGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM1_CM_DGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM1_CM_RGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM1_CM_RGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM1_CM_RGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM1_CM_RGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMA_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM1_CM_RGAM_RAMB_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM1_CM_HDR_MULT_COEF_DEFAULT                                           0x0001f000
-#define mmCM1_CM_RANGE_CLAMP_CONTROL_R_DEFAULT                                   0xfbff7bff
-#define mmCM1_CM_RANGE_CLAMP_CONTROL_G_DEFAULT                                   0xfbff7bff
-#define mmCM1_CM_RANGE_CLAMP_CONTROL_B_DEFAULT                                   0xfbff7bff
-#define mmCM1_CM_DENORM_CONTROL_DEFAULT                                          0x00000000
-#define mmCM1_CM_CMOUT_CONTROL_DEFAULT                                           0x0000000a
-#define mmCM1_CM_CMOUT_RANDOM_SEEDS_DEFAULT                                      0x00000000
-#define mmCM1_CM_MEM_PWR_CTRL_DEFAULT                                            0x00000000
-#define mmCM1_CM_MEM_PWR_STATUS_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_dpp1_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON13_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON13_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON13_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON13_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON13_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON13_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON13_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON13_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON13_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dpp2_dispdec_dpp_top_dispdec
-#define mmDPP_TOP2_DPP_CONTROL_DEFAULT                                           0x70000000
-#define mmDPP_TOP2_DPP_SOFT_RESET_DEFAULT                                        0x00000000
-#define mmDPP_TOP2_DPP_CRC_VAL_R_G_DEFAULT                                       0x00000000
-#define mmDPP_TOP2_DPP_CRC_VAL_B_A_DEFAULT                                       0x00000000
-#define mmDPP_TOP2_DPP_CRC_CTRL_DEFAULT                                          0x00000000
-#define mmDPP_TOP2_HOST_READ_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp2_dispdec_cnvc_cfg_dispdec
-#define mmCNVC_CFG2_CNVC_SURFACE_PIXEL_FORMAT_DEFAULT                            0x00000008
-#define mmCNVC_CFG2_FORMAT_CONTROL_DEFAULT                                       0x00000000
-#define mmCNVC_CFG2_FCNV_FP_SCALE_BIAS_DEFAULT                                   0x00003c00
-#define mmCNVC_CFG2_DENORM_CONTROL_DEFAULT                                       0x00002000
-#define mmCNVC_CFG2_COLOR_KEYER_CONTROL_DEFAULT                                  0x00000000
-#define mmCNVC_CFG2_COLOR_KEYER_ALPHA_DEFAULT                                    0x00000000
-#define mmCNVC_CFG2_COLOR_KEYER_RED_DEFAULT                                      0x00000000
-#define mmCNVC_CFG2_COLOR_KEYER_GREEN_DEFAULT                                    0x00000000
-#define mmCNVC_CFG2_COLOR_KEYER_BLUE_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp2_dispdec_cnvc_cur_dispdec
-#define mmCNVC_CUR2_CURSOR0_CONTROL_DEFAULT                                      0x0003ff00
-#define mmCNVC_CUR2_CURSOR0_COLOR0_DEFAULT                                       0x00000000
-#define mmCNVC_CUR2_CURSOR0_COLOR1_DEFAULT                                       0x00000000
-#define mmCNVC_CUR2_CURSOR0_FP_SCALE_BIAS_DEFAULT                                0x00003c00
-
-
-// addressBlock: dce_dc_dpp2_dispdec_dscl_dispdec
-#define mmDSCL2_SCL_COEF_RAM_TAP_SELECT_DEFAULT                                  0x00000000
-#define mmDSCL2_SCL_COEF_RAM_TAP_DATA_DEFAULT                                    0x00000000
-#define mmDSCL2_SCL_MODE_DEFAULT                                                 0x00000000
-#define mmDSCL2_SCL_TAP_CONTROL_DEFAULT                                          0x00000000
-#define mmDSCL2_DSCL_CONTROL_DEFAULT                                             0x00000000
-#define mmDSCL2_DSCL_2TAP_CONTROL_DEFAULT                                        0x01000100
-#define mmDSCL2_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                             0x00000000
-#define mmDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL2_SCL_HORZ_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL2_SCL_HORZ_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL2_SCL_HORZ_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL2_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL2_SCL_VERT_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL2_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                 0x01000000
-#define mmDSCL2_SCL_VERT_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL2_SCL_VERT_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL2_SCL_VERT_FILTER_INIT_BOT_C_DEFAULT                               0x01000000
-#define mmDSCL2_SCL_BLACK_OFFSET_DEFAULT                                         0x80000000
-#define mmDSCL2_DSCL_UPDATE_DEFAULT                                              0x00000000
-#define mmDSCL2_DSCL_AUTOCAL_DEFAULT                                             0x00000000
-#define mmDSCL2_DSCL_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                             0x00000000
-#define mmDSCL2_DSCL_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                             0x00000000
-#define mmDSCL2_OTG_H_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL2_OTG_V_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL2_RECOUT_START_DEFAULT                                             0x00000000
-#define mmDSCL2_RECOUT_SIZE_DEFAULT                                              0x00000000
-#define mmDSCL2_MPC_SIZE_DEFAULT                                                 0x00000000
-#define mmDSCL2_LB_DATA_FORMAT_DEFAULT                                           0x00000000
-#define mmDSCL2_LB_MEMORY_CTRL_DEFAULT                                           0x00003f00
-#define mmDSCL2_LB_V_COUNTER_DEFAULT                                             0x00000000
-#define mmDSCL2_DSCL_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDSCL2_DSCL_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDSCL2_OBUF_CONTROL_DEFAULT                                             0xe0000000
-#define mmDSCL2_OBUF_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dpp2_dispdec_cm_dispdec
-#define mmCM2_CM_CONTROL_DEFAULT                                                 0x00000000
-#define mmCM2_CM_COMA_C11_C12_DEFAULT                                            0x00002000
-#define mmCM2_CM_COMA_C13_C14_DEFAULT                                            0x00000000
-#define mmCM2_CM_COMA_C21_C22_DEFAULT                                            0x20000000
-#define mmCM2_CM_COMA_C23_C24_DEFAULT                                            0x00000000
-#define mmCM2_CM_COMA_C31_C32_DEFAULT                                            0x00000000
-#define mmCM2_CM_COMA_C33_C34_DEFAULT                                            0x00002000
-#define mmCM2_CM_COMB_C11_C12_DEFAULT                                            0x00002000
-#define mmCM2_CM_COMB_C13_C14_DEFAULT                                            0x00000000
-#define mmCM2_CM_COMB_C21_C22_DEFAULT                                            0x20000000
-#define mmCM2_CM_COMB_C23_C24_DEFAULT                                            0x00000000
-#define mmCM2_CM_COMB_C31_C32_DEFAULT                                            0x00000000
-#define mmCM2_CM_COMB_C33_C34_DEFAULT                                            0x00002000
-#define mmCM2_CM_IGAM_CONTROL_DEFAULT                                            0x08000002
-#define mmCM2_CM_IGAM_LUT_RW_CONTROL_DEFAULT                                     0x00011070
-#define mmCM2_CM_IGAM_LUT_RW_INDEX_DEFAULT                                       0x00000000
-#define mmCM2_CM_IGAM_LUT_SEQ_COLOR_DEFAULT                                      0x00000000
-#define mmCM2_CM_IGAM_LUT_30_COLOR_DEFAULT                                       0x00000000
-#define mmCM2_CM_IGAM_LUT_PWL_DATA_DEFAULT                                       0x00000000
-#define mmCM2_CM_IGAM_LUT_AUTOFILL_DEFAULT                                       0x00000000
-#define mmCM2_CM_IGAM_LUT_BW_OFFSET_BLUE_DEFAULT                                 0xffff0000
-#define mmCM2_CM_IGAM_LUT_BW_OFFSET_GREEN_DEFAULT                                0xffff0000
-#define mmCM2_CM_IGAM_LUT_BW_OFFSET_RED_DEFAULT                                  0xffff0000
-#define mmCM2_CM_ICSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM2_CM_ICSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM2_CM_ICSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM2_CM_ICSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM2_CM_ICSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM2_CM_ICSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM2_CM_ICSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM2_CM_GAMUT_REMAP_CONTROL_DEFAULT                                     0x00000000
-#define mmCM2_CM_GAMUT_REMAP_C11_C12_DEFAULT                                     0x00002000
-#define mmCM2_CM_GAMUT_REMAP_C13_C14_DEFAULT                                     0x00000000
-#define mmCM2_CM_GAMUT_REMAP_C21_C22_DEFAULT                                     0x20000000
-#define mmCM2_CM_GAMUT_REMAP_C23_C24_DEFAULT                                     0x00000000
-#define mmCM2_CM_GAMUT_REMAP_C31_C32_DEFAULT                                     0x00000000
-#define mmCM2_CM_GAMUT_REMAP_C33_C34_DEFAULT                                     0x00002000
-#define mmCM2_CM_OCSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM2_CM_OCSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM2_CM_OCSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM2_CM_OCSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM2_CM_OCSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM2_CM_OCSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM2_CM_OCSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM2_CM_BNS_VALUES_R_DEFAULT                                            0x20000000
-#define mmCM2_CM_BNS_VALUES_G_DEFAULT                                            0x20000000
-#define mmCM2_CM_BNS_VALUES_B_DEFAULT                                            0x20000000
-#define mmCM2_CM_DGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM2_CM_DGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM2_CM_DGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM2_CM_DGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM2_CM_DGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM2_CM_DGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM2_CM_RGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM2_CM_RGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM2_CM_RGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM2_CM_RGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMA_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM2_CM_RGAM_RAMB_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM2_CM_HDR_MULT_COEF_DEFAULT                                           0x0001f000
-#define mmCM2_CM_RANGE_CLAMP_CONTROL_R_DEFAULT                                   0xfbff7bff
-#define mmCM2_CM_RANGE_CLAMP_CONTROL_G_DEFAULT                                   0xfbff7bff
-#define mmCM2_CM_RANGE_CLAMP_CONTROL_B_DEFAULT                                   0xfbff7bff
-#define mmCM2_CM_DENORM_CONTROL_DEFAULT                                          0x00000000
-#define mmCM2_CM_CMOUT_CONTROL_DEFAULT                                           0x0000000a
-#define mmCM2_CM_CMOUT_RANDOM_SEEDS_DEFAULT                                      0x00000000
-#define mmCM2_CM_MEM_PWR_CTRL_DEFAULT                                            0x00000000
-#define mmCM2_CM_MEM_PWR_STATUS_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_dpp2_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON14_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON14_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON14_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON14_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON14_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON14_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON14_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON14_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON14_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dpp3_dispdec_dpp_top_dispdec
-#define mmDPP_TOP3_DPP_CONTROL_DEFAULT                                           0x70000000
-#define mmDPP_TOP3_DPP_SOFT_RESET_DEFAULT                                        0x00000000
-#define mmDPP_TOP3_DPP_CRC_VAL_R_G_DEFAULT                                       0x00000000
-#define mmDPP_TOP3_DPP_CRC_VAL_B_A_DEFAULT                                       0x00000000
-#define mmDPP_TOP3_DPP_CRC_CTRL_DEFAULT                                          0x00000000
-#define mmDPP_TOP3_HOST_READ_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp3_dispdec_cnvc_cfg_dispdec
-#define mmCNVC_CFG3_CNVC_SURFACE_PIXEL_FORMAT_DEFAULT                            0x00000008
-#define mmCNVC_CFG3_FORMAT_CONTROL_DEFAULT                                       0x00000000
-#define mmCNVC_CFG3_FCNV_FP_SCALE_BIAS_DEFAULT                                   0x00003c00
-#define mmCNVC_CFG3_DENORM_CONTROL_DEFAULT                                       0x00002000
-#define mmCNVC_CFG3_COLOR_KEYER_CONTROL_DEFAULT                                  0x00000000
-#define mmCNVC_CFG3_COLOR_KEYER_ALPHA_DEFAULT                                    0x00000000
-#define mmCNVC_CFG3_COLOR_KEYER_RED_DEFAULT                                      0x00000000
-#define mmCNVC_CFG3_COLOR_KEYER_GREEN_DEFAULT                                    0x00000000
-#define mmCNVC_CFG3_COLOR_KEYER_BLUE_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_dpp3_dispdec_cnvc_cur_dispdec
-#define mmCNVC_CUR3_CURSOR0_CONTROL_DEFAULT                                      0x0003ff00
-#define mmCNVC_CUR3_CURSOR0_COLOR0_DEFAULT                                       0x00000000
-#define mmCNVC_CUR3_CURSOR0_COLOR1_DEFAULT                                       0x00000000
-#define mmCNVC_CUR3_CURSOR0_FP_SCALE_BIAS_DEFAULT                                0x00003c00
-
-
-// addressBlock: dce_dc_dpp3_dispdec_dscl_dispdec
-#define mmDSCL3_SCL_COEF_RAM_TAP_SELECT_DEFAULT                                  0x00000000
-#define mmDSCL3_SCL_COEF_RAM_TAP_DATA_DEFAULT                                    0x00000000
-#define mmDSCL3_SCL_MODE_DEFAULT                                                 0x00000000
-#define mmDSCL3_SCL_TAP_CONTROL_DEFAULT                                          0x00000000
-#define mmDSCL3_DSCL_CONTROL_DEFAULT                                             0x00000000
-#define mmDSCL3_DSCL_2TAP_CONTROL_DEFAULT                                        0x01000100
-#define mmDSCL3_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                             0x00000000
-#define mmDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL3_SCL_HORZ_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL3_SCL_HORZ_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL3_SCL_HORZ_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL3_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                              0x00000000
-#define mmDSCL3_SCL_VERT_FILTER_INIT_DEFAULT                                     0x01000000
-#define mmDSCL3_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                 0x01000000
-#define mmDSCL3_SCL_VERT_FILTER_SCALE_RATIO_C_DEFAULT                            0x00000000
-#define mmDSCL3_SCL_VERT_FILTER_INIT_C_DEFAULT                                   0x01000000
-#define mmDSCL3_SCL_VERT_FILTER_INIT_BOT_C_DEFAULT                               0x01000000
-#define mmDSCL3_SCL_BLACK_OFFSET_DEFAULT                                         0x80000000
-#define mmDSCL3_DSCL_UPDATE_DEFAULT                                              0x00000000
-#define mmDSCL3_DSCL_AUTOCAL_DEFAULT                                             0x00000000
-#define mmDSCL3_DSCL_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                             0x00000000
-#define mmDSCL3_DSCL_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                             0x00000000
-#define mmDSCL3_OTG_H_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL3_OTG_V_BLANK_DEFAULT                                              0x00000000
-#define mmDSCL3_RECOUT_START_DEFAULT                                             0x00000000
-#define mmDSCL3_RECOUT_SIZE_DEFAULT                                              0x00000000
-#define mmDSCL3_MPC_SIZE_DEFAULT                                                 0x00000000
-#define mmDSCL3_LB_DATA_FORMAT_DEFAULT                                           0x00000000
-#define mmDSCL3_LB_MEMORY_CTRL_DEFAULT                                           0x00003f00
-#define mmDSCL3_LB_V_COUNTER_DEFAULT                                             0x00000000
-#define mmDSCL3_DSCL_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDSCL3_DSCL_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDSCL3_OBUF_CONTROL_DEFAULT                                             0xe0000000
-#define mmDSCL3_OBUF_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dpp3_dispdec_cm_dispdec
-#define mmCM3_CM_CONTROL_DEFAULT                                                 0x00000000
-#define mmCM3_CM_COMA_C11_C12_DEFAULT                                            0x00002000
-#define mmCM3_CM_COMA_C13_C14_DEFAULT                                            0x00000000
-#define mmCM3_CM_COMA_C21_C22_DEFAULT                                            0x20000000
-#define mmCM3_CM_COMA_C23_C24_DEFAULT                                            0x00000000
-#define mmCM3_CM_COMA_C31_C32_DEFAULT                                            0x00000000
-#define mmCM3_CM_COMA_C33_C34_DEFAULT                                            0x00002000
-#define mmCM3_CM_COMB_C11_C12_DEFAULT                                            0x00002000
-#define mmCM3_CM_COMB_C13_C14_DEFAULT                                            0x00000000
-#define mmCM3_CM_COMB_C21_C22_DEFAULT                                            0x20000000
-#define mmCM3_CM_COMB_C23_C24_DEFAULT                                            0x00000000
-#define mmCM3_CM_COMB_C31_C32_DEFAULT                                            0x00000000
-#define mmCM3_CM_COMB_C33_C34_DEFAULT                                            0x00002000
-#define mmCM3_CM_IGAM_CONTROL_DEFAULT                                            0x08000002
-#define mmCM3_CM_IGAM_LUT_RW_CONTROL_DEFAULT                                     0x00011070
-#define mmCM3_CM_IGAM_LUT_RW_INDEX_DEFAULT                                       0x00000000
-#define mmCM3_CM_IGAM_LUT_SEQ_COLOR_DEFAULT                                      0x00000000
-#define mmCM3_CM_IGAM_LUT_30_COLOR_DEFAULT                                       0x00000000
-#define mmCM3_CM_IGAM_LUT_PWL_DATA_DEFAULT                                       0x00000000
-#define mmCM3_CM_IGAM_LUT_AUTOFILL_DEFAULT                                       0x00000000
-#define mmCM3_CM_IGAM_LUT_BW_OFFSET_BLUE_DEFAULT                                 0xffff0000
-#define mmCM3_CM_IGAM_LUT_BW_OFFSET_GREEN_DEFAULT                                0xffff0000
-#define mmCM3_CM_IGAM_LUT_BW_OFFSET_RED_DEFAULT                                  0xffff0000
-#define mmCM3_CM_ICSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM3_CM_ICSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM3_CM_ICSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM3_CM_ICSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM3_CM_ICSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM3_CM_ICSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM3_CM_ICSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM3_CM_GAMUT_REMAP_CONTROL_DEFAULT                                     0x00000000
-#define mmCM3_CM_GAMUT_REMAP_C11_C12_DEFAULT                                     0x00002000
-#define mmCM3_CM_GAMUT_REMAP_C13_C14_DEFAULT                                     0x00000000
-#define mmCM3_CM_GAMUT_REMAP_C21_C22_DEFAULT                                     0x20000000
-#define mmCM3_CM_GAMUT_REMAP_C23_C24_DEFAULT                                     0x00000000
-#define mmCM3_CM_GAMUT_REMAP_C31_C32_DEFAULT                                     0x00000000
-#define mmCM3_CM_GAMUT_REMAP_C33_C34_DEFAULT                                     0x00002000
-#define mmCM3_CM_OCSC_CONTROL_DEFAULT                                            0x00000000
-#define mmCM3_CM_OCSC_C11_C12_DEFAULT                                            0x00002000
-#define mmCM3_CM_OCSC_C13_C14_DEFAULT                                            0x00000000
-#define mmCM3_CM_OCSC_C21_C22_DEFAULT                                            0x20000000
-#define mmCM3_CM_OCSC_C23_C24_DEFAULT                                            0x00000000
-#define mmCM3_CM_OCSC_C31_C32_DEFAULT                                            0x00000000
-#define mmCM3_CM_OCSC_C33_C34_DEFAULT                                            0x00002000
-#define mmCM3_CM_BNS_VALUES_R_DEFAULT                                            0x20000000
-#define mmCM3_CM_BNS_VALUES_G_DEFAULT                                            0x20000000
-#define mmCM3_CM_BNS_VALUES_B_DEFAULT                                            0x20000000
-#define mmCM3_CM_DGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM3_CM_DGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM3_CM_DGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM3_CM_DGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM3_CM_DGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM3_CM_DGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_CONTROL_DEFAULT                                            0x00000000
-#define mmCM3_CM_RGAM_LUT_INDEX_DEFAULT                                          0x00000000
-#define mmCM3_CM_RGAM_LUT_DATA_DEFAULT                                           0x00000000
-#define mmCM3_CM_RGAM_LUT_WRITE_EN_MASK_DEFAULT                                  0x00000007
-#define mmCM3_CM_RGAM_RAMA_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMA_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMA_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMA_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMA_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMA_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMA_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_START_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_START_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_START_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_SLOPE_CNTL_B_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_SLOPE_CNTL_G_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_SLOPE_CNTL_R_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_END_CNTL1_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMB_END_CNTL2_B_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMB_END_CNTL1_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMB_END_CNTL2_G_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMB_END_CNTL1_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMB_END_CNTL2_R_DEFAULT                                   0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_0_1_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_2_3_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_4_5_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_6_7_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_8_9_DEFAULT                                    0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_10_11_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_12_13_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_14_15_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_16_17_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_18_19_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_20_21_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_22_23_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_24_25_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_26_27_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_28_29_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_30_31_DEFAULT                                  0x00000000
-#define mmCM3_CM_RGAM_RAMB_REGION_32_33_DEFAULT                                  0x00000000
-#define mmCM3_CM_HDR_MULT_COEF_DEFAULT                                           0x0001f000
-#define mmCM3_CM_RANGE_CLAMP_CONTROL_R_DEFAULT                                   0xfbff7bff
-#define mmCM3_CM_RANGE_CLAMP_CONTROL_G_DEFAULT                                   0xfbff7bff
-#define mmCM3_CM_RANGE_CLAMP_CONTROL_B_DEFAULT                                   0xfbff7bff
-#define mmCM3_CM_DENORM_CONTROL_DEFAULT                                          0x00000000
-#define mmCM3_CM_CMOUT_CONTROL_DEFAULT                                           0x0000000a
-#define mmCM3_CM_CMOUT_RANDOM_SEEDS_DEFAULT                                      0x00000000
-#define mmCM3_CM_MEM_PWR_CTRL_DEFAULT                                            0x00000000
-#define mmCM3_CM_MEM_PWR_STATUS_DEFAULT                                          0x00000000
-
-
-// addressBlock: dce_dc_dpp3_dispdec_dpp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON15_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON15_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON15_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON15_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON15_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON15_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON15_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON15_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON15_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_mpc_mpcc0_dispdec
-#define mmMPCC0_MPCC_TOP_SEL_DEFAULT                                             0x00000000
-#define mmMPCC0_MPCC_BOT_SEL_DEFAULT                                             0x0000000f
-#define mmMPCC0_MPCC_OPP_ID_DEFAULT                                              0x00000000
-#define mmMPCC0_MPCC_CONTROL_DEFAULT                                             0xffff0061
-#define mmMPCC0_MPCC_SM_CONTROL_DEFAULT                                          0x00000000
-#define mmMPCC0_MPCC_UPDATE_LOCK_SEL_DEFAULT                                     0x0000000f
-#define mmMPCC0_MPCC_TOP_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC0_MPCC_BOT_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC0_MPCC_OFFSET_DEFAULT                                              0x00000000
-#define mmMPCC0_MPCC_BG_R_CR_DEFAULT                                             0x00000000
-#define mmMPCC0_MPCC_BG_G_Y_DEFAULT                                              0x00000000
-#define mmMPCC0_MPCC_BG_B_CB_DEFAULT                                             0x00000000
-#define mmMPCC0_MPCC_STALL_STATUS_DEFAULT                                        0x00000000
-#define mmMPCC0_MPCC_STATUS_DEFAULT                                              0x00000000
-
-
-// addressBlock: dce_dc_mpc_mpcc1_dispdec
-#define mmMPCC1_MPCC_TOP_SEL_DEFAULT                                             0x00000000
-#define mmMPCC1_MPCC_BOT_SEL_DEFAULT                                             0x0000000f
-#define mmMPCC1_MPCC_OPP_ID_DEFAULT                                              0x00000000
-#define mmMPCC1_MPCC_CONTROL_DEFAULT                                             0xffff0061
-#define mmMPCC1_MPCC_SM_CONTROL_DEFAULT                                          0x00000000
-#define mmMPCC1_MPCC_UPDATE_LOCK_SEL_DEFAULT                                     0x0000000f
-#define mmMPCC1_MPCC_TOP_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC1_MPCC_BOT_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC1_MPCC_OFFSET_DEFAULT                                              0x00000000
-#define mmMPCC1_MPCC_BG_R_CR_DEFAULT                                             0x00000000
-#define mmMPCC1_MPCC_BG_G_Y_DEFAULT                                              0x00000000
-#define mmMPCC1_MPCC_BG_B_CB_DEFAULT                                             0x00000000
-#define mmMPCC1_MPCC_STALL_STATUS_DEFAULT                                        0x00000000
-#define mmMPCC1_MPCC_STATUS_DEFAULT                                              0x00000000
-
-
-// addressBlock: dce_dc_mpc_mpcc2_dispdec
-#define mmMPCC2_MPCC_TOP_SEL_DEFAULT                                             0x00000000
-#define mmMPCC2_MPCC_BOT_SEL_DEFAULT                                             0x0000000f
-#define mmMPCC2_MPCC_OPP_ID_DEFAULT                                              0x00000000
-#define mmMPCC2_MPCC_CONTROL_DEFAULT                                             0xffff0061
-#define mmMPCC2_MPCC_SM_CONTROL_DEFAULT                                          0x00000000
-#define mmMPCC2_MPCC_UPDATE_LOCK_SEL_DEFAULT                                     0x0000000f
-#define mmMPCC2_MPCC_TOP_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC2_MPCC_BOT_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC2_MPCC_OFFSET_DEFAULT                                              0x00000000
-#define mmMPCC2_MPCC_BG_R_CR_DEFAULT                                             0x00000000
-#define mmMPCC2_MPCC_BG_G_Y_DEFAULT                                              0x00000000
-#define mmMPCC2_MPCC_BG_B_CB_DEFAULT                                             0x00000000
-#define mmMPCC2_MPCC_STALL_STATUS_DEFAULT                                        0x00000000
-#define mmMPCC2_MPCC_STATUS_DEFAULT                                              0x00000000
-
-
-// addressBlock: dce_dc_mpc_mpcc3_dispdec
-#define mmMPCC3_MPCC_TOP_SEL_DEFAULT                                             0x00000000
-#define mmMPCC3_MPCC_BOT_SEL_DEFAULT                                             0x0000000f
-#define mmMPCC3_MPCC_OPP_ID_DEFAULT                                              0x00000000
-#define mmMPCC3_MPCC_CONTROL_DEFAULT                                             0xffff0061
-#define mmMPCC3_MPCC_SM_CONTROL_DEFAULT                                          0x00000000
-#define mmMPCC3_MPCC_UPDATE_LOCK_SEL_DEFAULT                                     0x0000000f
-#define mmMPCC3_MPCC_TOP_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC3_MPCC_BOT_OFFSET_DEFAULT                                          0x00000000
-#define mmMPCC3_MPCC_OFFSET_DEFAULT                                              0x00000000
-#define mmMPCC3_MPCC_BG_R_CR_DEFAULT                                             0x00000000
-#define mmMPCC3_MPCC_BG_G_Y_DEFAULT                                              0x00000000
-#define mmMPCC3_MPCC_BG_B_CB_DEFAULT                                             0x00000000
-#define mmMPCC3_MPCC_STALL_STATUS_DEFAULT                                        0x00000000
-#define mmMPCC3_MPCC_STATUS_DEFAULT                                              0x00000000
-
-
-// addressBlock: dce_dc_mpc_mpc_cfg_dispdec
-#define mmMPC_CLOCK_CONTROL_DEFAULT                                              0x00000000
-#define mmMPC_SOFT_RESET_DEFAULT                                                 0x00000000
-#define mmMPC_CRC_CTRL_DEFAULT                                                   0x00000000
-#define mmMPC_CRC_SEL_CONTROL_DEFAULT                                            0x00000000
-#define mmMPC_CRC_RESULT_AR_DEFAULT                                              0x00000000
-#define mmMPC_CRC_RESULT_GB_DEFAULT                                              0x00000000
-#define mmMPC_CRC_RESULT_C_DEFAULT                                               0x00000000
-#define mmMPC_PERFMON_EVENT_CTRL_DEFAULT                                         0x00000000
-#define mmMPC_BYPASS_BG_AR_DEFAULT                                               0x00000000
-#define mmMPC_BYPASS_BG_GB_DEFAULT                                               0x00000000
-#define mmMPC_OUT0_MUX_DEFAULT                                                   0x0000000f
-#define mmMPC_OUT1_MUX_DEFAULT                                                   0x0000000f
-#define mmMPC_OUT2_MUX_DEFAULT                                                   0x0000000f
-#define mmMPC_OUT3_MUX_DEFAULT                                                   0x0000000f
-#define mmMPC_STALL_GRACE_WINDOW_DEFAULT                                         0x00000000
-#define mmADR_CFG_VUPDATE_LOCK_SET0_DEFAULT                                      0x00000000
-#define mmADR_VUPDATE_LOCK_SET0_DEFAULT                                          0x00000000
-#define mmCUR0_VUPDATE_LOCK_SET0_DEFAULT                                         0x00000000
-#define mmCUR1_VUPDATE_LOCK_SET0_DEFAULT                                         0x00000000
-#define mmADR_CFG_VUPDATE_LOCK_SET1_DEFAULT                                      0x00000000
-#define mmADR_VUPDATE_LOCK_SET1_DEFAULT                                          0x00000000
-#define mmCUR0_VUPDATE_LOCK_SET1_DEFAULT                                         0x00000000
-#define mmCUR1_VUPDATE_LOCK_SET1_DEFAULT                                         0x00000000
-#define mmADR_CFG_VUPDATE_LOCK_SET2_DEFAULT                                      0x00000000
-#define mmADR_VUPDATE_LOCK_SET2_DEFAULT                                          0x00000000
-#define mmCUR0_VUPDATE_LOCK_SET2_DEFAULT                                         0x00000000
-#define mmCUR1_VUPDATE_LOCK_SET2_DEFAULT                                         0x00000000
-#define mmADR_CFG_VUPDATE_LOCK_SET3_DEFAULT                                      0x00000000
-#define mmADR_VUPDATE_LOCK_SET3_DEFAULT                                          0x00000000
-#define mmCUR0_VUPDATE_LOCK_SET3_DEFAULT                                         0x00000000
-#define mmCUR1_VUPDATE_LOCK_SET3_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_mpc_mpc_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON16_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON16_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON16_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON16_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON16_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON16_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON16_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON16_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON16_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_opp_abm0_dispdec
-#define mmABM0_BL1_PWM_AMBIENT_LIGHT_LEVEL_DEFAULT                               0x00000000
-#define mmABM0_BL1_PWM_USER_LEVEL_DEFAULT                                        0x00000000
-#define mmABM0_BL1_PWM_TARGET_ABM_LEVEL_DEFAULT                                  0x00000000
-#define mmABM0_BL1_PWM_CURRENT_ABM_LEVEL_DEFAULT                                 0x00000000
-#define mmABM0_BL1_PWM_FINAL_DUTY_CYCLE_DEFAULT                                  0x00000000
-#define mmABM0_BL1_PWM_MINIMUM_DUTY_CYCLE_DEFAULT                                0x00000000
-#define mmABM0_BL1_PWM_ABM_CNTL_DEFAULT                                          0x00000000
-#define mmABM0_BL1_PWM_BL_UPDATE_SAMPLE_RATE_DEFAULT                             0x00000000
-#define mmABM0_BL1_PWM_GRP2_REG_LOCK_DEFAULT                                     0x00000000
-#define mmABM0_DC_ABM1_CNTL_DEFAULT                                              0x00000000
-#define mmABM0_DC_ABM1_IPCSC_COEFF_SEL_DEFAULT                                   0x00000000
-#define mmABM0_DC_ABM1_ACE_OFFSET_SLOPE_0_DEFAULT                                0x00000400
-#define mmABM0_DC_ABM1_ACE_OFFSET_SLOPE_1_DEFAULT                                0x00000400
-#define mmABM0_DC_ABM1_ACE_OFFSET_SLOPE_2_DEFAULT                                0x00000400
-#define mmABM0_DC_ABM1_ACE_OFFSET_SLOPE_3_DEFAULT                                0x00000400
-#define mmABM0_DC_ABM1_ACE_OFFSET_SLOPE_4_DEFAULT                                0x00000400
-#define mmABM0_DC_ABM1_ACE_THRES_12_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_ACE_THRES_34_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_ACE_CNTL_MISC_DEFAULT                                     0x00000000
-#define mmABM0_DC_ABM1_HGLS_REG_READ_PROGRESS_DEFAULT                            0x00000000
-#define mmABM0_DC_ABM1_HG_MISC_CTRL_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_LS_SUM_OF_LUMA_DEFAULT                                    0x00000000
-#define mmABM0_DC_ABM1_LS_MIN_MAX_LUMA_DEFAULT                                   0x00000000
-#define mmABM0_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA_DEFAULT                          0x00000000
-#define mmABM0_DC_ABM1_LS_PIXEL_COUNT_DEFAULT                                    0x00000000
-#define mmABM0_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_DEFAULT                      0x00000000
-#define mmABM0_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_DEFAULT                          0x00000000
-#define mmABM0_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_DEFAULT                          0x00000000
-#define mmABM0_DC_ABM1_HG_SAMPLE_RATE_DEFAULT                                    0x00000000
-#define mmABM0_DC_ABM1_LS_SAMPLE_RATE_DEFAULT                                    0x00000000
-#define mmABM0_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG_DEFAULT                            0x00000000
-#define mmABM0_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX_DEFAULT                            0x00000000
-#define mmABM0_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX_DEFAULT                           0x00000000
-#define mmABM0_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX_DEFAULT                          0x00000000
-#define mmABM0_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX_DEFAULT                          0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_1_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_2_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_3_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_4_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_5_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_6_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_7_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_8_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_9_DEFAULT                                       0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_10_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_11_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_12_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_13_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_14_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_15_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_16_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_17_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_18_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_19_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_20_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_21_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_22_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_23_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_HG_RESULT_24_DEFAULT                                      0x00000000
-#define mmABM0_DC_ABM1_BL_MASTER_LOCK_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_opp_abm1_dispdec
-#define mmABM1_BL1_PWM_AMBIENT_LIGHT_LEVEL_DEFAULT                               0x00000000
-#define mmABM1_BL1_PWM_USER_LEVEL_DEFAULT                                        0x00000000
-#define mmABM1_BL1_PWM_TARGET_ABM_LEVEL_DEFAULT                                  0x00000000
-#define mmABM1_BL1_PWM_CURRENT_ABM_LEVEL_DEFAULT                                 0x00000000
-#define mmABM1_BL1_PWM_FINAL_DUTY_CYCLE_DEFAULT                                  0x00000000
-#define mmABM1_BL1_PWM_MINIMUM_DUTY_CYCLE_DEFAULT                                0x00000000
-#define mmABM1_BL1_PWM_ABM_CNTL_DEFAULT                                          0x00000000
-#define mmABM1_BL1_PWM_BL_UPDATE_SAMPLE_RATE_DEFAULT                             0x00000000
-#define mmABM1_BL1_PWM_GRP2_REG_LOCK_DEFAULT                                     0x00000000
-#define mmABM1_DC_ABM1_CNTL_DEFAULT                                              0x00000000
-#define mmABM1_DC_ABM1_IPCSC_COEFF_SEL_DEFAULT                                   0x00000000
-#define mmABM1_DC_ABM1_ACE_OFFSET_SLOPE_0_DEFAULT                                0x00000400
-#define mmABM1_DC_ABM1_ACE_OFFSET_SLOPE_1_DEFAULT                                0x00000400
-#define mmABM1_DC_ABM1_ACE_OFFSET_SLOPE_2_DEFAULT                                0x00000400
-#define mmABM1_DC_ABM1_ACE_OFFSET_SLOPE_3_DEFAULT                                0x00000400
-#define mmABM1_DC_ABM1_ACE_OFFSET_SLOPE_4_DEFAULT                                0x00000400
-#define mmABM1_DC_ABM1_ACE_THRES_12_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_ACE_THRES_34_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_ACE_CNTL_MISC_DEFAULT                                     0x00000000
-#define mmABM1_DC_ABM1_HGLS_REG_READ_PROGRESS_DEFAULT                            0x00000000
-#define mmABM1_DC_ABM1_HG_MISC_CTRL_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_LS_SUM_OF_LUMA_DEFAULT                                    0x00000000
-#define mmABM1_DC_ABM1_LS_MIN_MAX_LUMA_DEFAULT                                   0x00000000
-#define mmABM1_DC_ABM1_LS_FILTERED_MIN_MAX_LUMA_DEFAULT                          0x00000000
-#define mmABM1_DC_ABM1_LS_PIXEL_COUNT_DEFAULT                                    0x00000000
-#define mmABM1_DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_DEFAULT                      0x00000000
-#define mmABM1_DC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_DEFAULT                          0x00000000
-#define mmABM1_DC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_DEFAULT                          0x00000000
-#define mmABM1_DC_ABM1_HG_SAMPLE_RATE_DEFAULT                                    0x00000000
-#define mmABM1_DC_ABM1_LS_SAMPLE_RATE_DEFAULT                                    0x00000000
-#define mmABM1_DC_ABM1_HG_BIN_1_32_SHIFT_FLAG_DEFAULT                            0x00000000
-#define mmABM1_DC_ABM1_HG_BIN_1_8_SHIFT_INDEX_DEFAULT                            0x00000000
-#define mmABM1_DC_ABM1_HG_BIN_9_16_SHIFT_INDEX_DEFAULT                           0x00000000
-#define mmABM1_DC_ABM1_HG_BIN_17_24_SHIFT_INDEX_DEFAULT                          0x00000000
-#define mmABM1_DC_ABM1_HG_BIN_25_32_SHIFT_INDEX_DEFAULT                          0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_1_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_2_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_3_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_4_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_5_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_6_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_7_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_8_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_9_DEFAULT                                       0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_10_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_11_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_12_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_13_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_14_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_15_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_16_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_17_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_18_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_19_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_20_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_21_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_22_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_23_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_HG_RESULT_24_DEFAULT                                      0x00000000
-#define mmABM1_DC_ABM1_BL_MASTER_LOCK_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_opp_fmt0_dispdec
-#define mmFMT0_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT0_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT0_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT0_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT0_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT0_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT0_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT0_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT0_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT0_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT0_FMT_MAP420_MEMORY_CONTROL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_oppbuf0_dispdec
-#define mmOPPBUF0_OPPBUF_CONTROL_DEFAULT                                         0x00000000
-#define mmOPPBUF0_OPPBUF_3D_PARAMETERS_0_DEFAULT                                 0x00000000
-#define mmOPPBUF0_OPPBUF_3D_PARAMETERS_1_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe0_dispdec
-#define mmOPP_PIPE0_OPP_PIPE_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe_crc0_dispdec
-#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_CONTROL_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_MASK_DEFAULT                                0x0000ffff
-#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT0_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT1_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC0_OPP_PIPE_CRC_RESULT2_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_opp_fmt1_dispdec
-#define mmFMT1_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT1_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT1_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT1_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT1_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT1_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT1_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT1_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT1_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT1_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT1_FMT_MAP420_MEMORY_CONTROL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_oppbuf1_dispdec
-#define mmOPPBUF1_OPPBUF_CONTROL_DEFAULT                                         0x00000000
-#define mmOPPBUF1_OPPBUF_3D_PARAMETERS_0_DEFAULT                                 0x00000000
-#define mmOPPBUF1_OPPBUF_3D_PARAMETERS_1_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe1_dispdec
-#define mmOPP_PIPE1_OPP_PIPE_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe_crc1_dispdec
-#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_CONTROL_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_MASK_DEFAULT                                0x0000ffff
-#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT0_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT1_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC1_OPP_PIPE_CRC_RESULT2_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_opp_fmt2_dispdec
-#define mmFMT2_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT2_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT2_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT2_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT2_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT2_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT2_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT2_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT2_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT2_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT2_FMT_MAP420_MEMORY_CONTROL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_oppbuf2_dispdec
-#define mmOPPBUF2_OPPBUF_CONTROL_DEFAULT                                         0x00000000
-#define mmOPPBUF2_OPPBUF_3D_PARAMETERS_0_DEFAULT                                 0x00000000
-#define mmOPPBUF2_OPPBUF_3D_PARAMETERS_1_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe2_dispdec
-#define mmOPP_PIPE2_OPP_PIPE_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe_crc2_dispdec
-#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_CONTROL_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_MASK_DEFAULT                                0x0000ffff
-#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT0_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT1_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC2_OPP_PIPE_CRC_RESULT2_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_opp_fmt3_dispdec
-#define mmFMT3_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT3_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT3_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT3_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT3_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT3_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT3_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT3_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT3_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT3_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT3_FMT_MAP420_MEMORY_CONTROL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_oppbuf3_dispdec
-#define mmOPPBUF3_OPPBUF_CONTROL_DEFAULT                                         0x00000000
-#define mmOPPBUF3_OPPBUF_3D_PARAMETERS_0_DEFAULT                                 0x00000000
-#define mmOPPBUF3_OPPBUF_3D_PARAMETERS_1_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe3_dispdec
-#define mmOPP_PIPE3_OPP_PIPE_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe_crc3_dispdec
-#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_CONTROL_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_MASK_DEFAULT                                0x0000ffff
-#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT0_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT1_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC3_OPP_PIPE_CRC_RESULT2_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_opp_fmt4_dispdec
-#define mmFMT4_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT4_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT4_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT4_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT4_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT4_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT4_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT4_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT4_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT4_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT4_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT4_FMT_MAP420_MEMORY_CONTROL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_oppbuf4_dispdec
-#define mmOPPBUF4_OPPBUF_CONTROL_DEFAULT                                         0x00000000
-#define mmOPPBUF4_OPPBUF_3D_PARAMETERS_0_DEFAULT                                 0x00000000
-#define mmOPPBUF4_OPPBUF_3D_PARAMETERS_1_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe4_dispdec
-#define mmOPP_PIPE4_OPP_PIPE_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe_crc4_dispdec
-#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_CONTROL_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_MASK_DEFAULT                                0x0000ffff
-#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT0_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT1_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC4_OPP_PIPE_CRC_RESULT2_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_opp_fmt5_dispdec
-#define mmFMT5_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT5_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT5_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT5_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT5_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT5_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT5_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT5_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT5_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT5_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT5_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT5_FMT_MAP420_MEMORY_CONTROL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_oppbuf5_dispdec
-#define mmOPPBUF5_OPPBUF_CONTROL_DEFAULT                                         0x00000000
-#define mmOPPBUF5_OPPBUF_3D_PARAMETERS_0_DEFAULT                                 0x00000000
-#define mmOPPBUF5_OPPBUF_3D_PARAMETERS_1_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe5_dispdec
-#define mmOPP_PIPE5_OPP_PIPE_CONTROL_DEFAULT                                     0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_pipe_crc5_dispdec
-#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_CONTROL_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_MASK_DEFAULT                                0x0000ffff
-#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT0_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT1_DEFAULT                             0x00000000
-#define mmOPP_PIPE_CRC5_OPP_PIPE_CRC_RESULT2_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_top_dispdec
-#define mmOPP_TOP_CLK_CONTROL_DEFAULT                                            0x00000000
-
-
-// addressBlock: dce_dc_opp_opp_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON17_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON17_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON17_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON17_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON17_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON17_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON17_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON17_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON17_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_optc_odm0_dispdec
-#define mmODM0_OPTC_INPUT_GLOBAL_CONTROL_DEFAULT                                 0x00000000
-#define mmODM0_OPTC_DATA_SOURCE_SELECT_DEFAULT                                   0x00000000
-#define mmODM0_OPTC_INPUT_CLOCK_CONTROL_DEFAULT                                  0x00000000
-#define mmODM0_OPTC_INPUT_SPARE_REGISTER_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_optc_odm1_dispdec
-#define mmODM1_OPTC_INPUT_GLOBAL_CONTROL_DEFAULT                                 0x00000000
-#define mmODM1_OPTC_DATA_SOURCE_SELECT_DEFAULT                                   0x00000000
-#define mmODM1_OPTC_INPUT_CLOCK_CONTROL_DEFAULT                                  0x00000000
-#define mmODM1_OPTC_INPUT_SPARE_REGISTER_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_optc_odm2_dispdec
-#define mmODM2_OPTC_INPUT_GLOBAL_CONTROL_DEFAULT                                 0x00000000
-#define mmODM2_OPTC_DATA_SOURCE_SELECT_DEFAULT                                   0x00000000
-#define mmODM2_OPTC_INPUT_CLOCK_CONTROL_DEFAULT                                  0x00000000
-#define mmODM2_OPTC_INPUT_SPARE_REGISTER_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_optc_odm3_dispdec
-#define mmODM3_OPTC_INPUT_GLOBAL_CONTROL_DEFAULT                                 0x00000000
-#define mmODM3_OPTC_DATA_SOURCE_SELECT_DEFAULT                                   0x00000000
-#define mmODM3_OPTC_INPUT_CLOCK_CONTROL_DEFAULT                                  0x00000000
-#define mmODM3_OPTC_INPUT_SPARE_REGISTER_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_optc_odm4_dispdec
-#define mmODM4_OPTC_INPUT_GLOBAL_CONTROL_DEFAULT                                 0x00000000
-#define mmODM4_OPTC_DATA_SOURCE_SELECT_DEFAULT                                   0x00000000
-#define mmODM4_OPTC_INPUT_CLOCK_CONTROL_DEFAULT                                  0x00000000
-#define mmODM4_OPTC_INPUT_SPARE_REGISTER_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_optc_odm5_dispdec
-#define mmODM5_OPTC_INPUT_GLOBAL_CONTROL_DEFAULT                                 0x00000000
-#define mmODM5_OPTC_DATA_SOURCE_SELECT_DEFAULT                                   0x00000000
-#define mmODM5_OPTC_INPUT_CLOCK_CONTROL_DEFAULT                                  0x00000000
-#define mmODM5_OPTC_INPUT_SPARE_REGISTER_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_optc_otg0_dispdec
-#define mmOTG0_OTG_H_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG0_OTG_H_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_H_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG0_OTG_H_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_H_TIMING_CNTL_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_V_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG0_OTG_V_TOTAL_MIN_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_V_TOTAL_MAX_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_V_TOTAL_MID_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_V_TOTAL_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_V_TOTAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmOTG0_OTG_VSYNC_NOM_INT_STATUS_DEFAULT                                  0x00000000
-#define mmOTG0_OTG_V_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_V_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG0_OTG_V_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_TRIGA_CNTL_DEFAULT                                            0x00000000
-#define mmOTG0_OTG_TRIGA_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_TRIGB_CNTL_DEFAULT                                            0x00000000
-#define mmOTG0_OTG_TRIGB_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_FORCE_COUNT_NOW_CNTL_DEFAULT                                  0x00000000
-#define mmOTG0_OTG_FLOW_CONTROL_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_STEREO_FORCE_NEXT_EYE_DEFAULT                                 0x00000000
-#define mmOTG0_OTG_AVSYNC_COUNTER_DEFAULT                                        0x00000000
-#define mmOTG0_OTG_CONTROL_DEFAULT                                               0x80000110
-#define mmOTG0_OTG_BLANK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_PIPE_ABORT_CONTROL_DEFAULT                                    0x00000000
-#define mmOTG0_OTG_INTERLACE_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_INTERLACE_STATUS_DEFAULT                                      0x00000000
-#define mmOTG0_OTG_FIELD_INDICATION_CONTROL_DEFAULT                              0x00000000
-#define mmOTG0_OTG_PIXEL_DATA_READBACK0_DEFAULT                                  0x00000000
-#define mmOTG0_OTG_PIXEL_DATA_READBACK1_DEFAULT                                  0x00000000
-#define mmOTG0_OTG_STATUS_DEFAULT                                                0x00000000
-#define mmOTG0_OTG_STATUS_POSITION_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_NOM_VERT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_STATUS_FRAME_COUNT_DEFAULT                                    0x00000000
-#define mmOTG0_OTG_STATUS_VF_COUNT_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_STATUS_HV_COUNT_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_COUNT_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_COUNT_RESET_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                          0x00000000
-#define mmOTG0_OTG_VERT_SYNC_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_STEREO_STATUS_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmOTG0_OTG_SNAPSHOT_STATUS_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_SNAPSHOT_CONTROL_DEFAULT                                      0x00000000
-#define mmOTG0_OTG_SNAPSHOT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_SNAPSHOT_FRAME_DEFAULT                                        0x00000000
-#define mmOTG0_OTG_INTERRUPT_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG0_OTG_UPDATE_LOCK_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_DOUBLE_BUFFER_CONTROL_DEFAULT                                 0x00000000
-#define mmOTG0_OTG_TEST_PATTERN_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG0_OTG_TEST_PATTERN_PARAMETERS_DEFAULT                               0x00000000
-#define mmOTG0_OTG_TEST_PATTERN_COLOR_DEFAULT                                    0x00000000
-#define mmOTG0_OTG_MASTER_EN_DEFAULT                                             0x00000000
-#define mmOTG0_OTG_BLANK_DATA_COLOR_DEFAULT                                      0x00000000
-#define mmOTG0_OTG_BLANK_DATA_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmOTG0_OTG_BLACK_COLOR_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_BLACK_COLOR_EXT_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_VERTICAL_INTERRUPT0_POSITION_DEFAULT                          0x00000000
-#define mmOTG0_OTG_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                           0x00000000
-#define mmOTG0_OTG_VERTICAL_INTERRUPT1_POSITION_DEFAULT                          0x00000000
-#define mmOTG0_OTG_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                           0x00000000
-#define mmOTG0_OTG_VERTICAL_INTERRUPT2_POSITION_DEFAULT                          0x00000000
-#define mmOTG0_OTG_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                           0x00000000
-#define mmOTG0_OTG_CRC_CNTL_DEFAULT                                              0x00000000
-#define mmOTG0_OTG_CRC0_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC0_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC0_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC0_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC0_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_CRC0_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_CRC1_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC1_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC1_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC1_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG0_OTG_CRC1_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_CRC1_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_CRC2_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_CRC2_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_CRC3_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_CRC3_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0xffffffff
-#define mmOTG0_OTG_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0xffffffff
-#define mmOTG0_OTG_STATIC_SCREEN_CONTROL_DEFAULT                                 0x00010000
-#define mmOTG0_OTG_3D_STRUCTURE_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG0_OTG_GSL_VSYNC_GAP_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_MASTER_UPDATE_MODE_DEFAULT                                    0x00000000
-#define mmOTG0_OTG_CLOCK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG0_OTG_VSTARTUP_PARAM_DEFAULT                                        0x00000000
-#define mmOTG0_OTG_VUPDATE_PARAM_DEFAULT                                         0x00010000
-#define mmOTG0_OTG_VREADY_PARAM_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_GLOBAL_SYNC_STATUS_DEFAULT                                    0x00000000
-#define mmOTG0_OTG_MASTER_UPDATE_LOCK_DEFAULT                                    0x00000000
-#define mmOTG0_OTG_GSL_CONTROL_DEFAULT                                           0x00020000
-#define mmOTG0_OTG_GSL_WINDOW_X_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_GSL_WINDOW_Y_DEFAULT                                          0x00000000
-#define mmOTG0_OTG_VUPDATE_KEEPOUT_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_GLOBAL_CONTROL0_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_GLOBAL_CONTROL1_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_GLOBAL_CONTROL2_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_GLOBAL_CONTROL3_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_TRIG_MANUAL_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG0_OTG_MANUAL_FLOW_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG0_OTG_RANGE_TIMING_INT_STATUS_DEFAULT                               0x00000000
-#define mmOTG0_OTG_DRR_CONTROL_DEFAULT                                           0x00000000
-#define mmOTG0_OTG_REQUEST_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG0_OTG_SPARE_REGISTER_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_optc_otg1_dispdec
-#define mmOTG1_OTG_H_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG1_OTG_H_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_H_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG1_OTG_H_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_H_TIMING_CNTL_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_V_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG1_OTG_V_TOTAL_MIN_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_V_TOTAL_MAX_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_V_TOTAL_MID_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_V_TOTAL_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_V_TOTAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmOTG1_OTG_VSYNC_NOM_INT_STATUS_DEFAULT                                  0x00000000
-#define mmOTG1_OTG_V_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_V_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG1_OTG_V_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_TRIGA_CNTL_DEFAULT                                            0x00000000
-#define mmOTG1_OTG_TRIGA_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_TRIGB_CNTL_DEFAULT                                            0x00000000
-#define mmOTG1_OTG_TRIGB_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_FORCE_COUNT_NOW_CNTL_DEFAULT                                  0x00000000
-#define mmOTG1_OTG_FLOW_CONTROL_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_STEREO_FORCE_NEXT_EYE_DEFAULT                                 0x00000000
-#define mmOTG1_OTG_AVSYNC_COUNTER_DEFAULT                                        0x00000000
-#define mmOTG1_OTG_CONTROL_DEFAULT                                               0x80000110
-#define mmOTG1_OTG_BLANK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_PIPE_ABORT_CONTROL_DEFAULT                                    0x00000000
-#define mmOTG1_OTG_INTERLACE_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_INTERLACE_STATUS_DEFAULT                                      0x00000000
-#define mmOTG1_OTG_FIELD_INDICATION_CONTROL_DEFAULT                              0x00000000
-#define mmOTG1_OTG_PIXEL_DATA_READBACK0_DEFAULT                                  0x00000000
-#define mmOTG1_OTG_PIXEL_DATA_READBACK1_DEFAULT                                  0x00000000
-#define mmOTG1_OTG_STATUS_DEFAULT                                                0x00000000
-#define mmOTG1_OTG_STATUS_POSITION_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_NOM_VERT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_STATUS_FRAME_COUNT_DEFAULT                                    0x00000000
-#define mmOTG1_OTG_STATUS_VF_COUNT_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_STATUS_HV_COUNT_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_COUNT_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_COUNT_RESET_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                          0x00000000
-#define mmOTG1_OTG_VERT_SYNC_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_STEREO_STATUS_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmOTG1_OTG_SNAPSHOT_STATUS_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_SNAPSHOT_CONTROL_DEFAULT                                      0x00000000
-#define mmOTG1_OTG_SNAPSHOT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_SNAPSHOT_FRAME_DEFAULT                                        0x00000000
-#define mmOTG1_OTG_INTERRUPT_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG1_OTG_UPDATE_LOCK_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_DOUBLE_BUFFER_CONTROL_DEFAULT                                 0x00000000
-#define mmOTG1_OTG_TEST_PATTERN_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG1_OTG_TEST_PATTERN_PARAMETERS_DEFAULT                               0x00000000
-#define mmOTG1_OTG_TEST_PATTERN_COLOR_DEFAULT                                    0x00000000
-#define mmOTG1_OTG_MASTER_EN_DEFAULT                                             0x00000000
-#define mmOTG1_OTG_BLANK_DATA_COLOR_DEFAULT                                      0x00000000
-#define mmOTG1_OTG_BLANK_DATA_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmOTG1_OTG_BLACK_COLOR_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_BLACK_COLOR_EXT_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_VERTICAL_INTERRUPT0_POSITION_DEFAULT                          0x00000000
-#define mmOTG1_OTG_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                           0x00000000
-#define mmOTG1_OTG_VERTICAL_INTERRUPT1_POSITION_DEFAULT                          0x00000000
-#define mmOTG1_OTG_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                           0x00000000
-#define mmOTG1_OTG_VERTICAL_INTERRUPT2_POSITION_DEFAULT                          0x00000000
-#define mmOTG1_OTG_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                           0x00000000
-#define mmOTG1_OTG_CRC_CNTL_DEFAULT                                              0x00000000
-#define mmOTG1_OTG_CRC0_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC0_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC0_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC0_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC0_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_CRC0_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_CRC1_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC1_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC1_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC1_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG1_OTG_CRC1_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_CRC1_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_CRC2_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_CRC2_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_CRC3_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_CRC3_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0xffffffff
-#define mmOTG1_OTG_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0xffffffff
-#define mmOTG1_OTG_STATIC_SCREEN_CONTROL_DEFAULT                                 0x00010000
-#define mmOTG1_OTG_3D_STRUCTURE_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG1_OTG_GSL_VSYNC_GAP_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_MASTER_UPDATE_MODE_DEFAULT                                    0x00000000
-#define mmOTG1_OTG_CLOCK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG1_OTG_VSTARTUP_PARAM_DEFAULT                                        0x00000000
-#define mmOTG1_OTG_VUPDATE_PARAM_DEFAULT                                         0x00010000
-#define mmOTG1_OTG_VREADY_PARAM_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_GLOBAL_SYNC_STATUS_DEFAULT                                    0x00000000
-#define mmOTG1_OTG_MASTER_UPDATE_LOCK_DEFAULT                                    0x00000000
-#define mmOTG1_OTG_GSL_CONTROL_DEFAULT                                           0x00020000
-#define mmOTG1_OTG_GSL_WINDOW_X_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_GSL_WINDOW_Y_DEFAULT                                          0x00000000
-#define mmOTG1_OTG_VUPDATE_KEEPOUT_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_GLOBAL_CONTROL0_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_GLOBAL_CONTROL1_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_GLOBAL_CONTROL2_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_GLOBAL_CONTROL3_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_TRIG_MANUAL_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG1_OTG_MANUAL_FLOW_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG1_OTG_RANGE_TIMING_INT_STATUS_DEFAULT                               0x00000000
-#define mmOTG1_OTG_DRR_CONTROL_DEFAULT                                           0x00000000
-#define mmOTG1_OTG_REQUEST_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG1_OTG_SPARE_REGISTER_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_optc_otg2_dispdec
-#define mmOTG2_OTG_H_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG2_OTG_H_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_H_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG2_OTG_H_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_H_TIMING_CNTL_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_V_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG2_OTG_V_TOTAL_MIN_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_V_TOTAL_MAX_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_V_TOTAL_MID_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_V_TOTAL_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_V_TOTAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmOTG2_OTG_VSYNC_NOM_INT_STATUS_DEFAULT                                  0x00000000
-#define mmOTG2_OTG_V_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_V_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG2_OTG_V_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_TRIGA_CNTL_DEFAULT                                            0x00000000
-#define mmOTG2_OTG_TRIGA_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_TRIGB_CNTL_DEFAULT                                            0x00000000
-#define mmOTG2_OTG_TRIGB_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_FORCE_COUNT_NOW_CNTL_DEFAULT                                  0x00000000
-#define mmOTG2_OTG_FLOW_CONTROL_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_STEREO_FORCE_NEXT_EYE_DEFAULT                                 0x00000000
-#define mmOTG2_OTG_AVSYNC_COUNTER_DEFAULT                                        0x00000000
-#define mmOTG2_OTG_CONTROL_DEFAULT                                               0x80000110
-#define mmOTG2_OTG_BLANK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_PIPE_ABORT_CONTROL_DEFAULT                                    0x00000000
-#define mmOTG2_OTG_INTERLACE_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_INTERLACE_STATUS_DEFAULT                                      0x00000000
-#define mmOTG2_OTG_FIELD_INDICATION_CONTROL_DEFAULT                              0x00000000
-#define mmOTG2_OTG_PIXEL_DATA_READBACK0_DEFAULT                                  0x00000000
-#define mmOTG2_OTG_PIXEL_DATA_READBACK1_DEFAULT                                  0x00000000
-#define mmOTG2_OTG_STATUS_DEFAULT                                                0x00000000
-#define mmOTG2_OTG_STATUS_POSITION_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_NOM_VERT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_STATUS_FRAME_COUNT_DEFAULT                                    0x00000000
-#define mmOTG2_OTG_STATUS_VF_COUNT_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_STATUS_HV_COUNT_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_COUNT_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_COUNT_RESET_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                          0x00000000
-#define mmOTG2_OTG_VERT_SYNC_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_STEREO_STATUS_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmOTG2_OTG_SNAPSHOT_STATUS_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_SNAPSHOT_CONTROL_DEFAULT                                      0x00000000
-#define mmOTG2_OTG_SNAPSHOT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_SNAPSHOT_FRAME_DEFAULT                                        0x00000000
-#define mmOTG2_OTG_INTERRUPT_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG2_OTG_UPDATE_LOCK_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_DOUBLE_BUFFER_CONTROL_DEFAULT                                 0x00000000
-#define mmOTG2_OTG_TEST_PATTERN_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG2_OTG_TEST_PATTERN_PARAMETERS_DEFAULT                               0x00000000
-#define mmOTG2_OTG_TEST_PATTERN_COLOR_DEFAULT                                    0x00000000
-#define mmOTG2_OTG_MASTER_EN_DEFAULT                                             0x00000000
-#define mmOTG2_OTG_BLANK_DATA_COLOR_DEFAULT                                      0x00000000
-#define mmOTG2_OTG_BLANK_DATA_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmOTG2_OTG_BLACK_COLOR_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_BLACK_COLOR_EXT_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_VERTICAL_INTERRUPT0_POSITION_DEFAULT                          0x00000000
-#define mmOTG2_OTG_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                           0x00000000
-#define mmOTG2_OTG_VERTICAL_INTERRUPT1_POSITION_DEFAULT                          0x00000000
-#define mmOTG2_OTG_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                           0x00000000
-#define mmOTG2_OTG_VERTICAL_INTERRUPT2_POSITION_DEFAULT                          0x00000000
-#define mmOTG2_OTG_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                           0x00000000
-#define mmOTG2_OTG_CRC_CNTL_DEFAULT                                              0x00000000
-#define mmOTG2_OTG_CRC0_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC0_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC0_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC0_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC0_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_CRC0_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_CRC1_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC1_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC1_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC1_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG2_OTG_CRC1_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_CRC1_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_CRC2_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_CRC2_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_CRC3_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_CRC3_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0xffffffff
-#define mmOTG2_OTG_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0xffffffff
-#define mmOTG2_OTG_STATIC_SCREEN_CONTROL_DEFAULT                                 0x00010000
-#define mmOTG2_OTG_3D_STRUCTURE_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG2_OTG_GSL_VSYNC_GAP_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_MASTER_UPDATE_MODE_DEFAULT                                    0x00000000
-#define mmOTG2_OTG_CLOCK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG2_OTG_VSTARTUP_PARAM_DEFAULT                                        0x00000000
-#define mmOTG2_OTG_VUPDATE_PARAM_DEFAULT                                         0x00010000
-#define mmOTG2_OTG_VREADY_PARAM_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_GLOBAL_SYNC_STATUS_DEFAULT                                    0x00000000
-#define mmOTG2_OTG_MASTER_UPDATE_LOCK_DEFAULT                                    0x00000000
-#define mmOTG2_OTG_GSL_CONTROL_DEFAULT                                           0x00020000
-#define mmOTG2_OTG_GSL_WINDOW_X_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_GSL_WINDOW_Y_DEFAULT                                          0x00000000
-#define mmOTG2_OTG_VUPDATE_KEEPOUT_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_GLOBAL_CONTROL0_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_GLOBAL_CONTROL1_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_GLOBAL_CONTROL2_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_GLOBAL_CONTROL3_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_TRIG_MANUAL_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG2_OTG_MANUAL_FLOW_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG2_OTG_RANGE_TIMING_INT_STATUS_DEFAULT                               0x00000000
-#define mmOTG2_OTG_DRR_CONTROL_DEFAULT                                           0x00000000
-#define mmOTG2_OTG_REQUEST_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG2_OTG_SPARE_REGISTER_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_optc_otg3_dispdec
-#define mmOTG3_OTG_H_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG3_OTG_H_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_H_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG3_OTG_H_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_H_TIMING_CNTL_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_V_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG3_OTG_V_TOTAL_MIN_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_V_TOTAL_MAX_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_V_TOTAL_MID_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_V_TOTAL_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_V_TOTAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmOTG3_OTG_VSYNC_NOM_INT_STATUS_DEFAULT                                  0x00000000
-#define mmOTG3_OTG_V_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_V_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG3_OTG_V_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_TRIGA_CNTL_DEFAULT                                            0x00000000
-#define mmOTG3_OTG_TRIGA_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_TRIGB_CNTL_DEFAULT                                            0x00000000
-#define mmOTG3_OTG_TRIGB_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_FORCE_COUNT_NOW_CNTL_DEFAULT                                  0x00000000
-#define mmOTG3_OTG_FLOW_CONTROL_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_STEREO_FORCE_NEXT_EYE_DEFAULT                                 0x00000000
-#define mmOTG3_OTG_AVSYNC_COUNTER_DEFAULT                                        0x00000000
-#define mmOTG3_OTG_CONTROL_DEFAULT                                               0x80000110
-#define mmOTG3_OTG_BLANK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_PIPE_ABORT_CONTROL_DEFAULT                                    0x00000000
-#define mmOTG3_OTG_INTERLACE_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_INTERLACE_STATUS_DEFAULT                                      0x00000000
-#define mmOTG3_OTG_FIELD_INDICATION_CONTROL_DEFAULT                              0x00000000
-#define mmOTG3_OTG_PIXEL_DATA_READBACK0_DEFAULT                                  0x00000000
-#define mmOTG3_OTG_PIXEL_DATA_READBACK1_DEFAULT                                  0x00000000
-#define mmOTG3_OTG_STATUS_DEFAULT                                                0x00000000
-#define mmOTG3_OTG_STATUS_POSITION_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_NOM_VERT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_STATUS_FRAME_COUNT_DEFAULT                                    0x00000000
-#define mmOTG3_OTG_STATUS_VF_COUNT_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_STATUS_HV_COUNT_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_COUNT_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_COUNT_RESET_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                          0x00000000
-#define mmOTG3_OTG_VERT_SYNC_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_STEREO_STATUS_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmOTG3_OTG_SNAPSHOT_STATUS_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_SNAPSHOT_CONTROL_DEFAULT                                      0x00000000
-#define mmOTG3_OTG_SNAPSHOT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_SNAPSHOT_FRAME_DEFAULT                                        0x00000000
-#define mmOTG3_OTG_INTERRUPT_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG3_OTG_UPDATE_LOCK_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_DOUBLE_BUFFER_CONTROL_DEFAULT                                 0x00000000
-#define mmOTG3_OTG_TEST_PATTERN_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG3_OTG_TEST_PATTERN_PARAMETERS_DEFAULT                               0x00000000
-#define mmOTG3_OTG_TEST_PATTERN_COLOR_DEFAULT                                    0x00000000
-#define mmOTG3_OTG_MASTER_EN_DEFAULT                                             0x00000000
-#define mmOTG3_OTG_BLANK_DATA_COLOR_DEFAULT                                      0x00000000
-#define mmOTG3_OTG_BLANK_DATA_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmOTG3_OTG_BLACK_COLOR_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_BLACK_COLOR_EXT_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_VERTICAL_INTERRUPT0_POSITION_DEFAULT                          0x00000000
-#define mmOTG3_OTG_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                           0x00000000
-#define mmOTG3_OTG_VERTICAL_INTERRUPT1_POSITION_DEFAULT                          0x00000000
-#define mmOTG3_OTG_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                           0x00000000
-#define mmOTG3_OTG_VERTICAL_INTERRUPT2_POSITION_DEFAULT                          0x00000000
-#define mmOTG3_OTG_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                           0x00000000
-#define mmOTG3_OTG_CRC_CNTL_DEFAULT                                              0x00000000
-#define mmOTG3_OTG_CRC0_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC0_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC0_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC0_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC0_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_CRC0_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_CRC1_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC1_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC1_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC1_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG3_OTG_CRC1_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_CRC1_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_CRC2_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_CRC2_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_CRC3_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_CRC3_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0xffffffff
-#define mmOTG3_OTG_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0xffffffff
-#define mmOTG3_OTG_STATIC_SCREEN_CONTROL_DEFAULT                                 0x00010000
-#define mmOTG3_OTG_3D_STRUCTURE_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG3_OTG_GSL_VSYNC_GAP_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_MASTER_UPDATE_MODE_DEFAULT                                    0x00000000
-#define mmOTG3_OTG_CLOCK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG3_OTG_VSTARTUP_PARAM_DEFAULT                                        0x00000000
-#define mmOTG3_OTG_VUPDATE_PARAM_DEFAULT                                         0x00010000
-#define mmOTG3_OTG_VREADY_PARAM_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_GLOBAL_SYNC_STATUS_DEFAULT                                    0x00000000
-#define mmOTG3_OTG_MASTER_UPDATE_LOCK_DEFAULT                                    0x00000000
-#define mmOTG3_OTG_GSL_CONTROL_DEFAULT                                           0x00020000
-#define mmOTG3_OTG_GSL_WINDOW_X_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_GSL_WINDOW_Y_DEFAULT                                          0x00000000
-#define mmOTG3_OTG_VUPDATE_KEEPOUT_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_GLOBAL_CONTROL0_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_GLOBAL_CONTROL1_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_GLOBAL_CONTROL2_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_GLOBAL_CONTROL3_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_TRIG_MANUAL_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG3_OTG_MANUAL_FLOW_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG3_OTG_RANGE_TIMING_INT_STATUS_DEFAULT                               0x00000000
-#define mmOTG3_OTG_DRR_CONTROL_DEFAULT                                           0x00000000
-#define mmOTG3_OTG_REQUEST_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG3_OTG_SPARE_REGISTER_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_optc_otg4_dispdec
-#define mmOTG4_OTG_H_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG4_OTG_H_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_H_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG4_OTG_H_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_H_TIMING_CNTL_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_V_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG4_OTG_V_TOTAL_MIN_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_V_TOTAL_MAX_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_V_TOTAL_MID_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_V_TOTAL_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_V_TOTAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmOTG4_OTG_VSYNC_NOM_INT_STATUS_DEFAULT                                  0x00000000
-#define mmOTG4_OTG_V_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_V_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG4_OTG_V_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_TRIGA_CNTL_DEFAULT                                            0x00000000
-#define mmOTG4_OTG_TRIGA_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_TRIGB_CNTL_DEFAULT                                            0x00000000
-#define mmOTG4_OTG_TRIGB_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_FORCE_COUNT_NOW_CNTL_DEFAULT                                  0x00000000
-#define mmOTG4_OTG_FLOW_CONTROL_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_STEREO_FORCE_NEXT_EYE_DEFAULT                                 0x00000000
-#define mmOTG4_OTG_AVSYNC_COUNTER_DEFAULT                                        0x00000000
-#define mmOTG4_OTG_CONTROL_DEFAULT                                               0x80000110
-#define mmOTG4_OTG_BLANK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_PIPE_ABORT_CONTROL_DEFAULT                                    0x00000000
-#define mmOTG4_OTG_INTERLACE_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_INTERLACE_STATUS_DEFAULT                                      0x00000000
-#define mmOTG4_OTG_FIELD_INDICATION_CONTROL_DEFAULT                              0x00000000
-#define mmOTG4_OTG_PIXEL_DATA_READBACK0_DEFAULT                                  0x00000000
-#define mmOTG4_OTG_PIXEL_DATA_READBACK1_DEFAULT                                  0x00000000
-#define mmOTG4_OTG_STATUS_DEFAULT                                                0x00000000
-#define mmOTG4_OTG_STATUS_POSITION_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_NOM_VERT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_STATUS_FRAME_COUNT_DEFAULT                                    0x00000000
-#define mmOTG4_OTG_STATUS_VF_COUNT_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_STATUS_HV_COUNT_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_COUNT_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_COUNT_RESET_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                          0x00000000
-#define mmOTG4_OTG_VERT_SYNC_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_STEREO_STATUS_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmOTG4_OTG_SNAPSHOT_STATUS_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_SNAPSHOT_CONTROL_DEFAULT                                      0x00000000
-#define mmOTG4_OTG_SNAPSHOT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_SNAPSHOT_FRAME_DEFAULT                                        0x00000000
-#define mmOTG4_OTG_INTERRUPT_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG4_OTG_UPDATE_LOCK_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_DOUBLE_BUFFER_CONTROL_DEFAULT                                 0x00000000
-#define mmOTG4_OTG_TEST_PATTERN_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG4_OTG_TEST_PATTERN_PARAMETERS_DEFAULT                               0x00000000
-#define mmOTG4_OTG_TEST_PATTERN_COLOR_DEFAULT                                    0x00000000
-#define mmOTG4_OTG_MASTER_EN_DEFAULT                                             0x00000000
-#define mmOTG4_OTG_BLANK_DATA_COLOR_DEFAULT                                      0x00000000
-#define mmOTG4_OTG_BLANK_DATA_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmOTG4_OTG_BLACK_COLOR_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_BLACK_COLOR_EXT_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_VERTICAL_INTERRUPT0_POSITION_DEFAULT                          0x00000000
-#define mmOTG4_OTG_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                           0x00000000
-#define mmOTG4_OTG_VERTICAL_INTERRUPT1_POSITION_DEFAULT                          0x00000000
-#define mmOTG4_OTG_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                           0x00000000
-#define mmOTG4_OTG_VERTICAL_INTERRUPT2_POSITION_DEFAULT                          0x00000000
-#define mmOTG4_OTG_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                           0x00000000
-#define mmOTG4_OTG_CRC_CNTL_DEFAULT                                              0x00000000
-#define mmOTG4_OTG_CRC0_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC0_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC0_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC0_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC0_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_CRC0_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_CRC1_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC1_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC1_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC1_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG4_OTG_CRC1_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_CRC1_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_CRC2_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_CRC2_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_CRC3_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_CRC3_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0xffffffff
-#define mmOTG4_OTG_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0xffffffff
-#define mmOTG4_OTG_STATIC_SCREEN_CONTROL_DEFAULT                                 0x00010000
-#define mmOTG4_OTG_3D_STRUCTURE_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG4_OTG_GSL_VSYNC_GAP_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_MASTER_UPDATE_MODE_DEFAULT                                    0x00000000
-#define mmOTG4_OTG_CLOCK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG4_OTG_VSTARTUP_PARAM_DEFAULT                                        0x00000000
-#define mmOTG4_OTG_VUPDATE_PARAM_DEFAULT                                         0x00010000
-#define mmOTG4_OTG_VREADY_PARAM_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_GLOBAL_SYNC_STATUS_DEFAULT                                    0x00000000
-#define mmOTG4_OTG_MASTER_UPDATE_LOCK_DEFAULT                                    0x00000000
-#define mmOTG4_OTG_GSL_CONTROL_DEFAULT                                           0x00020000
-#define mmOTG4_OTG_GSL_WINDOW_X_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_GSL_WINDOW_Y_DEFAULT                                          0x00000000
-#define mmOTG4_OTG_VUPDATE_KEEPOUT_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_GLOBAL_CONTROL0_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_GLOBAL_CONTROL1_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_GLOBAL_CONTROL2_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_GLOBAL_CONTROL3_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_TRIG_MANUAL_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG4_OTG_MANUAL_FLOW_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG4_OTG_RANGE_TIMING_INT_STATUS_DEFAULT                               0x00000000
-#define mmOTG4_OTG_DRR_CONTROL_DEFAULT                                           0x00000000
-#define mmOTG4_OTG_REQUEST_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG4_OTG_SPARE_REGISTER_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_optc_otg5_dispdec
-#define mmOTG5_OTG_H_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG5_OTG_H_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_H_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG5_OTG_H_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_H_TIMING_CNTL_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_V_TOTAL_DEFAULT                                               0x00000000
-#define mmOTG5_OTG_V_TOTAL_MIN_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_V_TOTAL_MAX_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_V_TOTAL_MID_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_V_TOTAL_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_V_TOTAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmOTG5_OTG_VSYNC_NOM_INT_STATUS_DEFAULT                                  0x00000000
-#define mmOTG5_OTG_V_BLANK_START_END_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_V_SYNC_A_DEFAULT                                              0x00000000
-#define mmOTG5_OTG_V_SYNC_A_CNTL_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_TRIGA_CNTL_DEFAULT                                            0x00000000
-#define mmOTG5_OTG_TRIGA_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_TRIGB_CNTL_DEFAULT                                            0x00000000
-#define mmOTG5_OTG_TRIGB_MANUAL_TRIG_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_FORCE_COUNT_NOW_CNTL_DEFAULT                                  0x00000000
-#define mmOTG5_OTG_FLOW_CONTROL_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_STEREO_FORCE_NEXT_EYE_DEFAULT                                 0x00000000
-#define mmOTG5_OTG_AVSYNC_COUNTER_DEFAULT                                        0x00000000
-#define mmOTG5_OTG_CONTROL_DEFAULT                                               0x80000110
-#define mmOTG5_OTG_BLANK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_PIPE_ABORT_CONTROL_DEFAULT                                    0x00000000
-#define mmOTG5_OTG_INTERLACE_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_INTERLACE_STATUS_DEFAULT                                      0x00000000
-#define mmOTG5_OTG_FIELD_INDICATION_CONTROL_DEFAULT                              0x00000000
-#define mmOTG5_OTG_PIXEL_DATA_READBACK0_DEFAULT                                  0x00000000
-#define mmOTG5_OTG_PIXEL_DATA_READBACK1_DEFAULT                                  0x00000000
-#define mmOTG5_OTG_STATUS_DEFAULT                                                0x00000000
-#define mmOTG5_OTG_STATUS_POSITION_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_NOM_VERT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_STATUS_FRAME_COUNT_DEFAULT                                    0x00000000
-#define mmOTG5_OTG_STATUS_VF_COUNT_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_STATUS_HV_COUNT_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_COUNT_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_COUNT_RESET_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                          0x00000000
-#define mmOTG5_OTG_VERT_SYNC_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_STEREO_STATUS_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmOTG5_OTG_SNAPSHOT_STATUS_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_SNAPSHOT_CONTROL_DEFAULT                                      0x00000000
-#define mmOTG5_OTG_SNAPSHOT_POSITION_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_SNAPSHOT_FRAME_DEFAULT                                        0x00000000
-#define mmOTG5_OTG_INTERRUPT_CONTROL_DEFAULT                                     0x00000000
-#define mmOTG5_OTG_UPDATE_LOCK_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_DOUBLE_BUFFER_CONTROL_DEFAULT                                 0x00000000
-#define mmOTG5_OTG_TEST_PATTERN_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG5_OTG_TEST_PATTERN_PARAMETERS_DEFAULT                               0x00000000
-#define mmOTG5_OTG_TEST_PATTERN_COLOR_DEFAULT                                    0x00000000
-#define mmOTG5_OTG_MASTER_EN_DEFAULT                                             0x00000000
-#define mmOTG5_OTG_BLANK_DATA_COLOR_DEFAULT                                      0x00000000
-#define mmOTG5_OTG_BLANK_DATA_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmOTG5_OTG_BLACK_COLOR_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_BLACK_COLOR_EXT_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_VERTICAL_INTERRUPT0_POSITION_DEFAULT                          0x00000000
-#define mmOTG5_OTG_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                           0x00000000
-#define mmOTG5_OTG_VERTICAL_INTERRUPT1_POSITION_DEFAULT                          0x00000000
-#define mmOTG5_OTG_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                           0x00000000
-#define mmOTG5_OTG_VERTICAL_INTERRUPT2_POSITION_DEFAULT                          0x00000000
-#define mmOTG5_OTG_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                           0x00000000
-#define mmOTG5_OTG_CRC_CNTL_DEFAULT                                              0x00000000
-#define mmOTG5_OTG_CRC0_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC0_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC0_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC0_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC0_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_CRC0_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_CRC1_WINDOWA_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC1_WINDOWA_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC1_WINDOWB_X_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC1_WINDOWB_Y_CONTROL_DEFAULT                                0x00000000
-#define mmOTG5_OTG_CRC1_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_CRC1_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_CRC2_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_CRC2_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_CRC3_DATA_RG_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_CRC3_DATA_B_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0xffffffff
-#define mmOTG5_OTG_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0xffffffff
-#define mmOTG5_OTG_STATIC_SCREEN_CONTROL_DEFAULT                                 0x00010000
-#define mmOTG5_OTG_3D_STRUCTURE_CONTROL_DEFAULT                                  0x00000000
-#define mmOTG5_OTG_GSL_VSYNC_GAP_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_MASTER_UPDATE_MODE_DEFAULT                                    0x00000000
-#define mmOTG5_OTG_CLOCK_CONTROL_DEFAULT                                         0x00000000
-#define mmOTG5_OTG_VSTARTUP_PARAM_DEFAULT                                        0x00000000
-#define mmOTG5_OTG_VUPDATE_PARAM_DEFAULT                                         0x00010000
-#define mmOTG5_OTG_VREADY_PARAM_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_GLOBAL_SYNC_STATUS_DEFAULT                                    0x00000000
-#define mmOTG5_OTG_MASTER_UPDATE_LOCK_DEFAULT                                    0x00000000
-#define mmOTG5_OTG_GSL_CONTROL_DEFAULT                                           0x00020000
-#define mmOTG5_OTG_GSL_WINDOW_X_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_GSL_WINDOW_Y_DEFAULT                                          0x00000000
-#define mmOTG5_OTG_VUPDATE_KEEPOUT_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_GLOBAL_CONTROL0_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_GLOBAL_CONTROL1_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_GLOBAL_CONTROL2_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_GLOBAL_CONTROL3_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_TRIG_MANUAL_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG5_OTG_MANUAL_FLOW_CONTROL_DEFAULT                                   0x00000000
-#define mmOTG5_OTG_RANGE_TIMING_INT_STATUS_DEFAULT                               0x00000000
-#define mmOTG5_OTG_DRR_CONTROL_DEFAULT                                           0x00000000
-#define mmOTG5_OTG_REQUEST_CONTROL_DEFAULT                                       0x00000000
-#define mmOTG5_OTG_SPARE_REGISTER_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_optc_optc_misc_dispdec
-#define mmDWB_SOURCE_SELECT_DEFAULT                                              0x00000000
-#define mmGSL_SOURCE_SELECT_DEFAULT                                              0x00000000
-#define mmOPTC_CLOCK_CONTROL_DEFAULT                                             0x00000000
-#define mmOPTC_MISC_SPARE_REGISTER_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_optc_optc_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON18_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON18_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON18_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON18_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON18_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON18_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON18_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON18_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON18_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dio_dac_dispdec
-#define mmDAC_ENABLE_DEFAULT                                                     0x00000004
-#define mmDAC_SOURCE_SELECT_DEFAULT                                              0x00000000
-#define mmDAC_CRC_EN_DEFAULT                                                     0x00000000
-#define mmDAC_CRC_CONTROL_DEFAULT                                                0x00000000
-#define mmDAC_CRC_SIG_RGB_MASK_DEFAULT                                           0x3fffffff
-#define mmDAC_CRC_SIG_CONTROL_MASK_DEFAULT                                       0x0000003f
-#define mmDAC_CRC_SIG_RGB_DEFAULT                                                0x3fffffff
-#define mmDAC_CRC_SIG_CONTROL_DEFAULT                                            0x0000003f
-#define mmDAC_SYNC_TRISTATE_CONTROL_DEFAULT                                      0x00000000
-#define mmDAC_STEREOSYNC_SELECT_DEFAULT                                          0x00000000
-#define mmDAC_AUTODETECT_CONTROL_DEFAULT                                         0x00070000
-#define mmDAC_AUTODETECT_CONTROL2_DEFAULT                                        0x0000000b
-#define mmDAC_AUTODETECT_CONTROL3_DEFAULT                                        0x00000519
-#define mmDAC_AUTODETECT_STATUS_DEFAULT                                          0x00000000
-#define mmDAC_AUTODETECT_INT_CONTROL_DEFAULT                                     0x00000000
-#define mmDAC_FORCE_OUTPUT_CNTL_DEFAULT                                          0x00000000
-#define mmDAC_FORCE_DATA_DEFAULT                                                 0x000001e6
-#define mmDAC_POWERDOWN_DEFAULT                                                  0x01010100
-#define mmDAC_CONTROL_DEFAULT                                                    0x00000000
-#define mmDAC_COMPARATOR_ENABLE_DEFAULT                                          0x00000000
-#define mmDAC_COMPARATOR_OUTPUT_DEFAULT                                          0x00000000
-#define mmDAC_PWR_CNTL_DEFAULT                                                   0x00000000
-#define mmDAC_DFT_CONFIG_DEFAULT                                                 0x00000000
-#define mmDAC_FIFO_STATUS_DEFAULT                                                0x00000000
-
-
-// addressBlock: dce_dc_dio_dout_i2c_dispdec
-#define mmDC_I2C_CONTROL_DEFAULT                                                 0x00000000
-#define mmDC_I2C_ARBITRATION_DEFAULT                                             0x00000001
-#define mmDC_I2C_INTERRUPT_CONTROL_DEFAULT                                       0x00000000
-#define mmDC_I2C_SW_STATUS_DEFAULT                                               0x00000000
-#define mmDC_I2C_DDC1_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC2_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC3_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC4_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC5_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC6_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC1_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC1_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC2_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC2_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC3_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC3_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC4_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC4_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC5_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC5_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC6_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC6_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_TRANSACTION0_DEFAULT                                            0x00000000
-#define mmDC_I2C_TRANSACTION1_DEFAULT                                            0x00000000
-#define mmDC_I2C_TRANSACTION2_DEFAULT                                            0x00000000
-#define mmDC_I2C_TRANSACTION3_DEFAULT                                            0x00000000
-#define mmDC_I2C_DATA_DEFAULT                                                    0x00000000
-#define mmDC_I2C_DDCVGA_HW_STATUS_DEFAULT                                        0x00000000
-#define mmDC_I2C_DDCVGA_SPEED_DEFAULT                                            0x00000002
-#define mmDC_I2C_DDCVGA_SETUP_DEFAULT                                            0x00000000
-#define mmDC_I2C_EDID_DETECT_CTRL_DEFAULT                                        0x004001f4
-#define mmDC_I2C_READ_REQUEST_INTERRUPT_DEFAULT                                  0x40000000
-
-
-// addressBlock: dce_dc_dio_generic_i2c_dispdec
-#define mmGENERIC_I2C_CONTROL_DEFAULT                                            0x00000000
-#define mmGENERIC_I2C_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmGENERIC_I2C_STATUS_DEFAULT                                             0x00000000
-#define mmGENERIC_I2C_SPEED_DEFAULT                                              0x00000002
-#define mmGENERIC_I2C_SETUP_DEFAULT                                              0x00000000
-#define mmGENERIC_I2C_TRANSACTION_DEFAULT                                        0x00000000
-#define mmGENERIC_I2C_DATA_DEFAULT                                               0x00000000
-#define mmGENERIC_I2C_PIN_SELECTION_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_dio_dio_misc_dispdec
-#define mmDIO_SCRATCH0_DEFAULT                                                   0x00000000
-#define mmDIO_SCRATCH1_DEFAULT                                                   0x00000000
-#define mmDIO_SCRATCH2_DEFAULT                                                   0x00000000
-#define mmDIO_SCRATCH3_DEFAULT                                                   0x00000000
-#define mmDIO_SCRATCH4_DEFAULT                                                   0x00000000
-#define mmDIO_SCRATCH5_DEFAULT                                                   0x00000000
-#define mmDIO_SCRATCH6_DEFAULT                                                   0x00000000
-#define mmDIO_SCRATCH7_DEFAULT                                                   0x00000000
-#define mmDCE_VCE_CONTROL_DEFAULT                                                0x00000000
-#define mmDIO_MEM_PWR_STATUS_DEFAULT                                             0x00000000
-#define mmDIO_MEM_PWR_CTRL_DEFAULT                                               0x6db6d800
-#define mmDIO_MEM_PWR_CTRL2_DEFAULT                                              0x00000000
-#define mmDIO_CLK_CNTL_DEFAULT                                                   0x00000000
-#define mmDIO_POWER_MANAGEMENT_CNTL_DEFAULT                                      0x00000000
-#define mmDIO_STEREOSYNC_SEL_DEFAULT                                             0x00000000
-#define mmDIO_SOFT_RESET_DEFAULT                                                 0x00000000
-#define mmDIG_SOFT_RESET_DEFAULT                                                 0x00000000
-#define mmDIO_MEM_PWR_STATUS1_DEFAULT                                            0x00000000
-#define mmDIO_CLK_CNTL2_DEFAULT                                                  0x00000000
-#define mmDIO_CLK_CNTL3_DEFAULT                                                  0x00000000
-#define mmDIO_HDMI_RXSTATUS_TIMER_CONTROL_DEFAULT                                0x00000000
-#define mmDIO_PSP_INTERRUPT_STATUS_DEFAULT                                       0x00000000
-#define mmDIO_PSP_INTERRUPT_CLEAR_DEFAULT                                        0x00000000
-#define mmDIO_GENERIC_INTERRUPT_MESSAGE_DEFAULT                                  0x00000000
-#define mmDIO_GENERIC_INTERRUPT_CLEAR_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_hpd0_dispdec
-#define mmHPD0_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD0_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD0_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD0_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD0_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_dio_hpd1_dispdec
-#define mmHPD1_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD1_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD1_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD1_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD1_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_dio_hpd2_dispdec
-#define mmHPD2_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD2_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD2_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD2_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD2_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_dio_hpd3_dispdec
-#define mmHPD3_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD3_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD3_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD3_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD3_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_dio_hpd4_dispdec
-#define mmHPD4_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD4_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD4_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD4_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD4_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_dio_hpd5_dispdec
-#define mmHPD5_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD5_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD5_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD5_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD5_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_dio_dio_dcperfmon_dc_perfmon_dispdec
-#define mmDC_PERFMON19_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON19_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON19_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON19_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON19_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON19_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON19_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON19_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON19_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dio_dp_aux0_dispdec
-#define mmDP_AUX0_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX0_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX0_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX0_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX0_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX0_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX0_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX0_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX0_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX0_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX0_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX0_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX0_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX0_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX0_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX0_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX0_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_dp_aux1_dispdec
-#define mmDP_AUX1_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX1_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX1_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX1_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX1_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX1_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX1_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX1_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX1_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX1_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX1_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX1_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX1_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX1_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX1_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX1_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX1_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_dp_aux2_dispdec
-#define mmDP_AUX2_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX2_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX2_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX2_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX2_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX2_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX2_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX2_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX2_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX2_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX2_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX2_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX2_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX2_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX2_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX2_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX2_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_dp_aux3_dispdec
-#define mmDP_AUX3_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX3_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX3_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX3_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX3_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX3_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX3_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX3_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX3_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX3_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX3_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX3_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX3_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX3_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX3_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX3_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX3_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_dp_aux4_dispdec
-#define mmDP_AUX4_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX4_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX4_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX4_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX4_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX4_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX4_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX4_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX4_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX4_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX4_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX4_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX4_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX4_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX4_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX4_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX4_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_dp_aux5_dispdec
-#define mmDP_AUX5_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX5_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX5_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX5_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX5_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX5_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX5_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX5_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX5_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX5_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX5_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX5_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX5_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX5_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX5_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX5_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX5_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_dp_aux6_dispdec
-#define mmDP_AUX6_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX6_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX6_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX6_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX6_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX6_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX6_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX6_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX6_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX6_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX6_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX6_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX6_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX6_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX6_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX6_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX6_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dio_dig0_dispdec
-#define mmDIG0_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG0_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG0_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG0_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG0_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG0_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG0_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG0_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG0_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG0_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG0_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG0_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG0_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG0_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG0_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG0_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG0_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL2_DEFAULT                              0x00000000
-#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL3_DEFAULT                              0x00000000
-#define mmDIG0_HDMI_DB_CONTROL_DEFAULT                                           0x00000000
-#define mmDIG0_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG0_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG0_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG0_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG0_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG0_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG0_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG0_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG0_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG0_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG0_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG0_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG0_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG0_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG0_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG0_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG0_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG0_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG0_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG0_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG0_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG0_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG0_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG0_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG0_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG0_AFMT_CNTL_DEFAULT                                                 0x00000000
-#define mmDIG0_AFMT_VBI_PACKET_CONTROL1_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dio_dp0_dispdec
-#define mmDP0_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP0_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP0_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP0_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP0_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP0_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP0_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP0_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP0_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP0_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP0_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP0_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP0_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP0_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP0_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP0_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP0_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP0_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP0_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP0_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP0_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP0_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP0_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP0_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP0_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP0_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP0_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP0_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP0_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP0_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP0_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP0_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP0_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP0_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP0_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP0_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP0_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP0_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP0_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP0_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP0_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP0_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP0_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP0_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP0_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP0_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP0_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-#define mmDP0_DP_MSA_TIMING_PARAM1_DEFAULT                                       0x00000000
-#define mmDP0_DP_MSA_TIMING_PARAM2_DEFAULT                                       0x00000000
-#define mmDP0_DP_MSA_TIMING_PARAM3_DEFAULT                                       0x00000000
-#define mmDP0_DP_MSA_TIMING_PARAM4_DEFAULT                                       0x00000000
-#define mmDP0_DP_MSO_CNTL_DEFAULT                                                0xfffffff0
-#define mmDP0_DP_MSO_CNTL1_DEFAULT                                               0xffffffff
-#define mmDP0_DP_DSC_CNTL_DEFAULT                                                0x00000000
-#define mmDP0_DP_SEC_CNTL2_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_CNTL3_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_CNTL4_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_CNTL5_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_CNTL6_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_CNTL7_DEFAULT                                               0x00000000
-#define mmDP0_DP_DB_CNTL_DEFAULT                                                 0x00000000
-#define mmDP0_DP_MSA_VBID_MISC_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dio_dig1_dispdec
-#define mmDIG1_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG1_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG1_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG1_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG1_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG1_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG1_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG1_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG1_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG1_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG1_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG1_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG1_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG1_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG1_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG1_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG1_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL2_DEFAULT                              0x00000000
-#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL3_DEFAULT                              0x00000000
-#define mmDIG1_HDMI_DB_CONTROL_DEFAULT                                           0x00000000
-#define mmDIG1_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG1_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG1_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG1_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG1_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG1_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG1_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG1_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG1_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG1_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG1_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG1_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG1_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG1_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG1_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG1_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG1_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG1_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG1_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG1_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG1_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG1_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG1_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG1_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG1_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG1_AFMT_CNTL_DEFAULT                                                 0x00000000
-#define mmDIG1_AFMT_VBI_PACKET_CONTROL1_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dio_dp1_dispdec
-#define mmDP1_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP1_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP1_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP1_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP1_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP1_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP1_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP1_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP1_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP1_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP1_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP1_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP1_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP1_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP1_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP1_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP1_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP1_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP1_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP1_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP1_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP1_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP1_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP1_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP1_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP1_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP1_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP1_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP1_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP1_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP1_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP1_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP1_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP1_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP1_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP1_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP1_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP1_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP1_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP1_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP1_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP1_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP1_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP1_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP1_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP1_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP1_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-#define mmDP1_DP_MSA_TIMING_PARAM1_DEFAULT                                       0x00000000
-#define mmDP1_DP_MSA_TIMING_PARAM2_DEFAULT                                       0x00000000
-#define mmDP1_DP_MSA_TIMING_PARAM3_DEFAULT                                       0x00000000
-#define mmDP1_DP_MSA_TIMING_PARAM4_DEFAULT                                       0x00000000
-#define mmDP1_DP_MSO_CNTL_DEFAULT                                                0xfffffff0
-#define mmDP1_DP_MSO_CNTL1_DEFAULT                                               0xffffffff
-#define mmDP1_DP_DSC_CNTL_DEFAULT                                                0x00000000
-#define mmDP1_DP_SEC_CNTL2_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_CNTL3_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_CNTL4_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_CNTL5_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_CNTL6_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_CNTL7_DEFAULT                                               0x00000000
-#define mmDP1_DP_DB_CNTL_DEFAULT                                                 0x00000000
-#define mmDP1_DP_MSA_VBID_MISC_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dio_dig2_dispdec
-#define mmDIG2_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG2_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG2_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG2_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG2_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG2_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG2_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG2_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG2_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG2_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG2_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG2_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG2_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG2_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG2_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG2_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG2_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL2_DEFAULT                              0x00000000
-#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL3_DEFAULT                              0x00000000
-#define mmDIG2_HDMI_DB_CONTROL_DEFAULT                                           0x00000000
-#define mmDIG2_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG2_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG2_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG2_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG2_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG2_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG2_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG2_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG2_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG2_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG2_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG2_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG2_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG2_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG2_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG2_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG2_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG2_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG2_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG2_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG2_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG2_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG2_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG2_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG2_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG2_AFMT_CNTL_DEFAULT                                                 0x00000000
-#define mmDIG2_AFMT_VBI_PACKET_CONTROL1_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dio_dp2_dispdec
-#define mmDP2_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP2_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP2_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP2_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP2_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP2_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP2_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP2_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP2_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP2_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP2_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP2_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP2_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP2_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP2_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP2_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP2_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP2_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP2_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP2_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP2_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP2_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP2_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP2_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP2_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP2_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP2_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP2_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP2_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP2_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP2_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP2_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP2_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP2_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP2_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP2_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP2_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP2_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP2_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP2_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP2_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP2_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP2_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP2_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP2_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP2_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP2_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-#define mmDP2_DP_MSA_TIMING_PARAM1_DEFAULT                                       0x00000000
-#define mmDP2_DP_MSA_TIMING_PARAM2_DEFAULT                                       0x00000000
-#define mmDP2_DP_MSA_TIMING_PARAM3_DEFAULT                                       0x00000000
-#define mmDP2_DP_MSA_TIMING_PARAM4_DEFAULT                                       0x00000000
-#define mmDP2_DP_MSO_CNTL_DEFAULT                                                0xfffffff0
-#define mmDP2_DP_MSO_CNTL1_DEFAULT                                               0xffffffff
-#define mmDP2_DP_DSC_CNTL_DEFAULT                                                0x00000000
-#define mmDP2_DP_SEC_CNTL2_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_CNTL3_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_CNTL4_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_CNTL5_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_CNTL6_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_CNTL7_DEFAULT                                               0x00000000
-#define mmDP2_DP_DB_CNTL_DEFAULT                                                 0x00000000
-#define mmDP2_DP_MSA_VBID_MISC_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dio_dig3_dispdec
-#define mmDIG3_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG3_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG3_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG3_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG3_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG3_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG3_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG3_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG3_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG3_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG3_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG3_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG3_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG3_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG3_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG3_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG3_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL2_DEFAULT                              0x00000000
-#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL3_DEFAULT                              0x00000000
-#define mmDIG3_HDMI_DB_CONTROL_DEFAULT                                           0x00000000
-#define mmDIG3_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG3_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG3_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG3_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG3_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG3_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG3_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG3_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG3_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG3_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG3_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG3_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG3_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG3_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG3_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG3_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG3_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG3_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG3_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG3_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG3_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG3_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG3_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG3_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG3_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG3_AFMT_CNTL_DEFAULT                                                 0x00000000
-#define mmDIG3_AFMT_VBI_PACKET_CONTROL1_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dio_dp3_dispdec
-#define mmDP3_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP3_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP3_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP3_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP3_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP3_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP3_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP3_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP3_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP3_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP3_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP3_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP3_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP3_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP3_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP3_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP3_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP3_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP3_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP3_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP3_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP3_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP3_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP3_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP3_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP3_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP3_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP3_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP3_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP3_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP3_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP3_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP3_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP3_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP3_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP3_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP3_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP3_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP3_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP3_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP3_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP3_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP3_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP3_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP3_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP3_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP3_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-#define mmDP3_DP_MSA_TIMING_PARAM1_DEFAULT                                       0x00000000
-#define mmDP3_DP_MSA_TIMING_PARAM2_DEFAULT                                       0x00000000
-#define mmDP3_DP_MSA_TIMING_PARAM3_DEFAULT                                       0x00000000
-#define mmDP3_DP_MSA_TIMING_PARAM4_DEFAULT                                       0x00000000
-#define mmDP3_DP_MSO_CNTL_DEFAULT                                                0xfffffff0
-#define mmDP3_DP_MSO_CNTL1_DEFAULT                                               0xffffffff
-#define mmDP3_DP_DSC_CNTL_DEFAULT                                                0x00000000
-#define mmDP3_DP_SEC_CNTL2_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_CNTL3_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_CNTL4_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_CNTL5_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_CNTL6_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_CNTL7_DEFAULT                                               0x00000000
-#define mmDP3_DP_DB_CNTL_DEFAULT                                                 0x00000000
-#define mmDP3_DP_MSA_VBID_MISC_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dio_dig4_dispdec
-#define mmDIG4_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG4_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG4_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG4_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG4_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG4_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG4_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG4_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG4_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG4_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG4_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG4_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG4_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG4_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG4_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG4_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG4_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL2_DEFAULT                              0x00000000
-#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL3_DEFAULT                              0x00000000
-#define mmDIG4_HDMI_DB_CONTROL_DEFAULT                                           0x00000000
-#define mmDIG4_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG4_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG4_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG4_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG4_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG4_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG4_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG4_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG4_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG4_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG4_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG4_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG4_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG4_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG4_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG4_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG4_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG4_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG4_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG4_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG4_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG4_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG4_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG4_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG4_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG4_AFMT_CNTL_DEFAULT                                                 0x00000000
-#define mmDIG4_AFMT_VBI_PACKET_CONTROL1_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dio_dp4_dispdec
-#define mmDP4_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP4_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP4_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP4_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP4_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP4_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP4_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP4_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP4_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP4_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP4_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP4_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP4_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP4_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP4_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP4_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP4_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP4_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP4_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP4_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP4_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP4_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP4_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP4_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP4_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP4_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP4_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP4_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP4_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP4_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP4_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP4_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP4_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP4_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP4_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP4_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP4_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP4_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP4_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP4_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP4_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP4_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP4_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP4_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP4_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP4_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP4_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-#define mmDP4_DP_MSA_TIMING_PARAM1_DEFAULT                                       0x00000000
-#define mmDP4_DP_MSA_TIMING_PARAM2_DEFAULT                                       0x00000000
-#define mmDP4_DP_MSA_TIMING_PARAM3_DEFAULT                                       0x00000000
-#define mmDP4_DP_MSA_TIMING_PARAM4_DEFAULT                                       0x00000000
-#define mmDP4_DP_MSO_CNTL_DEFAULT                                                0xfffffff0
-#define mmDP4_DP_MSO_CNTL1_DEFAULT                                               0xffffffff
-#define mmDP4_DP_DSC_CNTL_DEFAULT                                                0x00000000
-#define mmDP4_DP_SEC_CNTL2_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_CNTL3_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_CNTL4_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_CNTL5_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_CNTL6_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_CNTL7_DEFAULT                                               0x00000000
-#define mmDP4_DP_DB_CNTL_DEFAULT                                                 0x00000000
-#define mmDP4_DP_MSA_VBID_MISC_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dio_dig5_dispdec
-#define mmDIG5_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG5_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG5_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG5_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG5_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG5_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG5_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG5_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG5_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG5_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG5_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG5_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG5_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG5_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG5_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG5_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG5_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL2_DEFAULT                              0x00000000
-#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL3_DEFAULT                              0x00000000
-#define mmDIG5_HDMI_DB_CONTROL_DEFAULT                                           0x00000000
-#define mmDIG5_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG5_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG5_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG5_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG5_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG5_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG5_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG5_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG5_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG5_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG5_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG5_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG5_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG5_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG5_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG5_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG5_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG5_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG5_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG5_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG5_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG5_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG5_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG5_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG5_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG5_AFMT_CNTL_DEFAULT                                                 0x00000000
-#define mmDIG5_AFMT_VBI_PACKET_CONTROL1_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dio_dp5_dispdec
-#define mmDP5_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP5_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP5_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP5_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP5_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP5_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP5_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP5_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP5_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP5_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP5_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP5_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP5_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP5_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP5_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP5_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP5_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP5_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP5_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP5_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP5_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP5_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP5_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP5_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP5_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP5_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP5_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP5_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP5_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP5_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP5_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP5_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP5_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP5_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP5_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP5_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP5_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP5_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP5_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP5_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP5_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP5_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP5_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP5_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP5_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP5_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP5_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-#define mmDP5_DP_MSA_TIMING_PARAM1_DEFAULT                                       0x00000000
-#define mmDP5_DP_MSA_TIMING_PARAM2_DEFAULT                                       0x00000000
-#define mmDP5_DP_MSA_TIMING_PARAM3_DEFAULT                                       0x00000000
-#define mmDP5_DP_MSA_TIMING_PARAM4_DEFAULT                                       0x00000000
-#define mmDP5_DP_MSO_CNTL_DEFAULT                                                0xfffffff0
-#define mmDP5_DP_MSO_CNTL1_DEFAULT                                               0xffffffff
-#define mmDP5_DP_DSC_CNTL_DEFAULT                                                0x00000000
-#define mmDP5_DP_SEC_CNTL2_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_CNTL3_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_CNTL4_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_CNTL5_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_CNTL6_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_CNTL7_DEFAULT                                               0x00000000
-#define mmDP5_DP_DB_CNTL_DEFAULT                                                 0x00000000
-#define mmDP5_DP_MSA_VBID_MISC_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dio_dig6_dispdec
-#define mmDIG6_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG6_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG6_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG6_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG6_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG6_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG6_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG6_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG6_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG6_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG6_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG6_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG6_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG6_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG6_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG6_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG6_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG6_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG6_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG6_HDMI_GENERIC_PACKET_CONTROL2_DEFAULT                              0x00000000
-#define mmDIG6_HDMI_GENERIC_PACKET_CONTROL3_DEFAULT                              0x00000000
-#define mmDIG6_HDMI_DB_CONTROL_DEFAULT                                           0x00000000
-#define mmDIG6_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG6_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG6_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG6_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG6_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG6_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG6_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG6_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG6_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG6_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG6_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG6_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG6_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG6_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG6_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG6_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG6_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG6_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG6_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG6_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG6_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG6_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG6_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG6_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG6_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG6_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG6_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG6_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG6_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG6_AFMT_CNTL_DEFAULT                                                 0x00000000
-#define mmDIG6_AFMT_VBI_PACKET_CONTROL1_DEFAULT                                  0x00000000
-
-
-// addressBlock: dce_dc_dio_dp6_dispdec
-#define mmDP6_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP6_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP6_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP6_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP6_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP6_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP6_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP6_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP6_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP6_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP6_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP6_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP6_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP6_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP6_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP6_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP6_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP6_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP6_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP6_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP6_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP6_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP6_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP6_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP6_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP6_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP6_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP6_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP6_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP6_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP6_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP6_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP6_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP6_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP6_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP6_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP6_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP6_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP6_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP6_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP6_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP6_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP6_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP6_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP6_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP6_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP6_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-#define mmDP6_DP_MSA_TIMING_PARAM1_DEFAULT                                       0x00000000
-#define mmDP6_DP_MSA_TIMING_PARAM2_DEFAULT                                       0x00000000
-#define mmDP6_DP_MSA_TIMING_PARAM3_DEFAULT                                       0x00000000
-#define mmDP6_DP_MSA_TIMING_PARAM4_DEFAULT                                       0x00000000
-#define mmDP6_DP_MSO_CNTL_DEFAULT                                                0xfffffff0
-#define mmDP6_DP_MSO_CNTL1_DEFAULT                                               0xffffffff
-#define mmDP6_DP_DSC_CNTL_DEFAULT                                                0x00000000
-#define mmDP6_DP_SEC_CNTL2_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_CNTL3_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_CNTL4_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_CNTL5_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_CNTL6_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_CNTL7_DEFAULT                                               0x00000000
-#define mmDP6_DP_DB_CNTL_DEFAULT                                                 0x00000000
-#define mmDP6_DP_MSA_VBID_MISC_DEFAULT                                           0x00000000
-
-
-// addressBlock: dce_dc_dcio_dcio_dispdec
-#define mmDC_GENERICA_DEFAULT                                                    0x00000000
-#define mmDC_GENERICB_DEFAULT                                                    0x00000000
-#define mmDC_REF_CLK_CNTL_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DEBUG_DEFAULT                                                  0x00000101
-#define mmUNIPHYA_LINK_CNTL_DEFAULT                                              0x01000100
-#define mmUNIPHYA_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYB_LINK_CNTL_DEFAULT                                              0x01000100
-#define mmUNIPHYB_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYC_LINK_CNTL_DEFAULT                                              0x01000100
-#define mmUNIPHYC_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYD_LINK_CNTL_DEFAULT                                              0x01000100
-#define mmUNIPHYD_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYE_LINK_CNTL_DEFAULT                                              0x01000100
-#define mmUNIPHYE_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYF_LINK_CNTL_DEFAULT                                              0x01000100
-#define mmUNIPHYF_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYG_LINK_CNTL_DEFAULT                                              0x01000100
-#define mmUNIPHYG_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmDCIO_WRCMD_DELAY_DEFAULT                                               0x00033333
-#define mmDC_DVODATA_CONFIG_DEFAULT                                              0x00000000
-#define mmLVTMA_PWRSEQ_CNTL_DEFAULT                                              0x00000000
-#define mmLVTMA_PWRSEQ_STATE_DEFAULT                                             0x00000000
-#define mmLVTMA_PWRSEQ_REF_DIV_DEFAULT                                           0x00010000
-#define mmLVTMA_PWRSEQ_DELAY1_DEFAULT                                            0x00000000
-#define mmLVTMA_PWRSEQ_DELAY2_DEFAULT                                            0x00000000
-#define mmBL_PWM_CNTL_DEFAULT                                                    0x00000000
-#define mmBL_PWM_CNTL2_DEFAULT                                                   0x00000000
-#define mmBL_PWM_PERIOD_CNTL_DEFAULT                                             0x00000001
-#define mmBL_PWM_GRP1_REG_LOCK_DEFAULT                                           0x00000000
-#define mmDCIO_GSL_GENLK_PAD_CNTL_DEFAULT                                        0x00000000
-#define mmDCIO_GSL_SWAPLOCK_PAD_CNTL_DEFAULT                                     0x00000000
-#define mmDCIO_CLOCK_CNTL_DEFAULT                                                0x00000000
-#define mmDIO_OTG_EXT_VSYNC_CNTL_DEFAULT                                         0x00000000
-#define mmDCIO_SOFT_RESET_DEFAULT                                                0x00000000
-#define mmDCIO_DPHY_SEL_DEFAULT                                                  0x000000e4
-#define mmUNIPHY_IMPCAL_LINKA_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_LINKB_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_PERIOD_DEFAULT                                           0x00000000
-#define mmAUXP_IMPCAL_DEFAULT                                                    0x0a000000
-#define mmAUXN_IMPCAL_DEFAULT                                                    0x04000000
-#define mmDCIO_IMPCAL_CNTL_DEFAULT                                               0x00000000
-#define mmUNIPHY_IMPCAL_PSW_AB_DEFAULT                                           0x00000000
-#define mmUNIPHY_IMPCAL_LINKC_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_LINKD_DEFAULT                                            0x0f000000
-#define mmDCIO_IMPCAL_CNTL_CD_DEFAULT                                            0x00000000
-#define mmUNIPHY_IMPCAL_PSW_CD_DEFAULT                                           0x00000000
-#define mmUNIPHY_IMPCAL_LINKE_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_LINKF_DEFAULT                                            0x0f000000
-#define mmDCIO_IMPCAL_CNTL_EF_DEFAULT                                            0x00000000
-#define mmUNIPHY_IMPCAL_PSW_EF_DEFAULT                                           0x00000000
-#define mmDCIO_DPCS_TX_INTERRUPT_DEFAULT                                         0x00000000
-#define mmDCIO_DPCS_RX_INTERRUPT_DEFAULT                                         0x00000000
-#define mmDCIO_SEMAPHORE0_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE1_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE2_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE3_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE4_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE5_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE6_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE7_DEFAULT                                                0x00000000
-#define mmDCIO_USBC_FLIP_EN_SEL_DEFAULT                                          0x00543210
-
-
-// addressBlock: dce_dc_dcio_dcio_chip_dispdec
-#define mmDC_GPIO_GENERIC_MASK_DEFAULT                                           0x04444444
-#define mmDC_GPIO_GENERIC_A_DEFAULT                                              0x00000000
-#define mmDC_GPIO_GENERIC_EN_DEFAULT                                             0x00000000
-#define mmDC_GPIO_GENERIC_Y_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DVODATA_MASK_DEFAULT                                           0x00000000
-#define mmDC_GPIO_DVODATA_A_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DVODATA_EN_DEFAULT                                             0x00000000
-#define mmDC_GPIO_DVODATA_Y_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DDC1_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC1_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC1_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC1_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC2_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC2_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC2_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC2_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC3_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC3_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC3_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC3_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC4_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC4_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC4_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC4_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC5_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC5_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC5_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC5_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC6_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC6_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC6_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC6_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDCVGA_MASK_DEFAULT                                            0xcf400000
-#define mmDC_GPIO_DDCVGA_A_DEFAULT                                               0x00000000
-#define mmDC_GPIO_DDCVGA_EN_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DDCVGA_Y_DEFAULT                                               0x00000000
-#define mmDC_GPIO_SYNCA_MASK_DEFAULT                                             0x00004040
-#define mmDC_GPIO_SYNCA_A_DEFAULT                                                0x00000000
-#define mmDC_GPIO_SYNCA_EN_DEFAULT                                               0x00000000
-#define mmDC_GPIO_SYNCA_Y_DEFAULT                                                0x00000000
-#define mmDC_GPIO_GENLK_MASK_DEFAULT                                             0x10101a10
-#define mmDC_GPIO_GENLK_A_DEFAULT                                                0x00000000
-#define mmDC_GPIO_GENLK_EN_DEFAULT                                               0x00000000
-#define mmDC_GPIO_GENLK_Y_DEFAULT                                                0x00000000
-#define mmDC_GPIO_HPD_MASK_DEFAULT                                               0x44440440
-#define mmDC_GPIO_HPD_A_DEFAULT                                                  0x00000000
-#define mmDC_GPIO_HPD_EN_DEFAULT                                                 0x22220202
-#define mmDC_GPIO_HPD_Y_DEFAULT                                                  0x00000000
-#define mmDC_GPIO_PWRSEQ_MASK_DEFAULT                                            0x66404040
-#define mmDC_GPIO_PWRSEQ_A_DEFAULT                                               0x00000000
-#define mmDC_GPIO_PWRSEQ_EN_DEFAULT                                              0x00000000
-#define mmDC_GPIO_PWRSEQ_Y_DEFAULT                                               0x00000000
-#define mmDC_GPIO_PAD_STRENGTH_1_DEFAULT                                         0x47fc470f
-#define mmDC_GPIO_PAD_STRENGTH_2_DEFAULT                                         0x00472147
-#define mmPHY_AUX_CNTL_DEFAULT                                                   0x00010001
-#define mmDC_GPIO_I2CPAD_MASK_DEFAULT                                            0x00000000
-#define mmDC_GPIO_I2CPAD_A_DEFAULT                                               0x00000000
-#define mmDC_GPIO_I2CPAD_EN_DEFAULT                                              0x00000000
-#define mmDC_GPIO_I2CPAD_Y_DEFAULT                                               0x00000000
-#define mmDC_GPIO_I2CPAD_STRENGTH_DEFAULT                                        0x0000004c
-#define mmDVO_STRENGTH_CONTROL_DEFAULT                                           0x31116060
-#define mmDVO_VREF_CONTROL_DEFAULT                                               0x00000000
-#define mmDVO_SKEW_ADJUST_DEFAULT                                                0x00000000
-#define mmDC_GPIO_I2S_SPDIF_MASK_DEFAULT                                         0x00000000
-#define mmDC_GPIO_I2S_SPDIF_A_DEFAULT                                            0x00000000
-#define mmDC_GPIO_I2S_SPDIF_EN_DEFAULT                                           0x00008000
-#define mmDC_GPIO_I2S_SPDIF_Y_DEFAULT                                            0x00000000
-#define mmDC_GPIO_I2S_SPDIF_STRENGTH_DEFAULT                                     0x01021202
-#define mmDC_GPIO_TX12_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_AUX_CTRL_0_DEFAULT                                             0x00000000
-#define mmDC_GPIO_AUX_CTRL_1_DEFAULT                                             0x00500000
-#define mmDC_GPIO_AUX_CTRL_2_DEFAULT                                             0x00000000
-#define mmDC_GPIO_RXEN_DEFAULT                                                   0x007fff7f
-#define mmDC_GPIO_PULLUPEN_DEFAULT                                               0x00000000
-
-
-// addressBlock: dce_dc_dcio_dcio_dac_dispdec
-#define mmDAC_MACRO_CNTL_RESERVED0_DEFAULT                                       0x00000000
-#define mmDAC_MACRO_CNTL_RESERVED1_DEFAULT                                       0x00000000
-#define mmDAC_MACRO_CNTL_RESERVED2_DEFAULT                                       0x00000000
-#define mmDAC_MACRO_CNTL_RESERVED3_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dcio_dcio_uniphy0_dispdec
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophycmregs0_dispdec
-#define mmDC_COMBOPHYCMREGS0_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_FUSE3_DEFAULT                                0x1c010000
-#define mmDC_COMBOPHYCMREGS0_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS0_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS0_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS0_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS0_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophytxregs0_dispdec
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophypllregs0_dispdec
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS0_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS0_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS0_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS0_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS0_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS0_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS0_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS0_DFT_OUT_DEFAULT                                    0x00000000
-#define mmDC_COMBOPHYPLLREGS0_PLL_WRAP_CNTRL1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYPLLREGS0_PLL_WRAP_CNTRL_DEFAULT                             0x00010520
-
-
-// addressBlock: dce_dc_dcio_dcio_uniphy1_dispdec
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophycmregs1_dispdec
-#define mmDC_COMBOPHYCMREGS1_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_FUSE3_DEFAULT                                0x1c010000
-#define mmDC_COMBOPHYCMREGS1_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS1_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS1_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS1_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS1_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophytxregs1_dispdec
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophypllregs1_dispdec
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS1_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS1_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS1_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS1_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS1_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS1_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS1_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS1_DFT_OUT_DEFAULT                                    0x00000000
-#define mmDC_COMBOPHYPLLREGS1_PLL_WRAP_CNTRL1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYPLLREGS1_PLL_WRAP_CNTRL_DEFAULT                             0x00010520
-
-
-// addressBlock: dce_dc_dcio_dcio_uniphy2_dispdec
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophycmregs2_dispdec
-#define mmDC_COMBOPHYCMREGS2_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_FUSE3_DEFAULT                                0x1c010000
-#define mmDC_COMBOPHYCMREGS2_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS2_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS2_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS2_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS2_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophytxregs2_dispdec
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophypllregs2_dispdec
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS2_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS2_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS2_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS2_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS2_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS2_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS2_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS2_DFT_OUT_DEFAULT                                    0x00000000
-#define mmDC_COMBOPHYPLLREGS2_PLL_WRAP_CNTRL1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYPLLREGS2_PLL_WRAP_CNTRL_DEFAULT                             0x00010520
-
-
-// addressBlock: dce_dc_dcio_dcio_uniphy3_dispdec
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophycmregs3_dispdec
-#define mmDC_COMBOPHYCMREGS3_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_FUSE3_DEFAULT                                0x1c010000
-#define mmDC_COMBOPHYCMREGS3_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS3_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS3_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS3_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS3_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophytxregs3_dispdec
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_combophy_dc_combophypllregs3_dispdec
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS3_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS3_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS3_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS3_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS3_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS3_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS3_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS3_DFT_OUT_DEFAULT                                    0x00000000
-#define mmDC_COMBOPHYPLLREGS3_PLL_WRAP_CNTRL1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYPLLREGS3_PLL_WRAP_CNTRL_DEFAULT                             0x00010520
-
-
-// addressBlock: dce_dc_dcio_dcio_zcal_dispdec
-#define mmZCAL_MACRO_CNTL_RESERVED0_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED1_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED2_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED3_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED4_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_zcal_dc_zcalregs_dispdec
-#define mmCOMP_EN_CTL_DEFAULT                                                    0x00080000
-#define mmCOMP_EN_DFX_DEFAULT                                                    0x00000000
-#define mmZCAL_FUSES_DEFAULT                                                     0x00000000
-
-
-// addressBlock: vga_vgaseqind
-#define ixSEQ00_DEFAULT                                                          0x00000003
-#define ixSEQ01_DEFAULT                                                          0x00000021
-#define ixSEQ02_DEFAULT                                                          0x00000000
-#define ixSEQ03_DEFAULT                                                          0x00000000
-#define ixSEQ04_DEFAULT                                                          0x00000000
-
-
-// addressBlock: vga_vgacrtind
-#define ixCRT00_DEFAULT                                                          0x00000000
-#define ixCRT01_DEFAULT                                                          0x00000000
-#define ixCRT02_DEFAULT                                                          0x00000000
-#define ixCRT03_DEFAULT                                                          0x00000000
-#define ixCRT04_DEFAULT                                                          0x00000000
-#define ixCRT05_DEFAULT                                                          0x00000000
-#define ixCRT06_DEFAULT                                                          0x00000000
-#define ixCRT07_DEFAULT                                                          0x00000000
-#define ixCRT08_DEFAULT                                                          0x00000000
-#define ixCRT09_DEFAULT                                                          0x00000000
-#define ixCRT0A_DEFAULT                                                          0x00000000
-#define ixCRT0B_DEFAULT                                                          0x00000000
-#define ixCRT0C_DEFAULT                                                          0x00000000
-#define ixCRT0D_DEFAULT                                                          0x00000000
-#define ixCRT0E_DEFAULT                                                          0x00000000
-#define ixCRT0F_DEFAULT                                                          0x00000000
-#define ixCRT10_DEFAULT                                                          0x00000000
-#define ixCRT11_DEFAULT                                                          0x00000000
-#define ixCRT12_DEFAULT                                                          0x00000000
-#define ixCRT13_DEFAULT                                                          0x00000000
-#define ixCRT14_DEFAULT                                                          0x00000000
-#define ixCRT15_DEFAULT                                                          0x00000000
-#define ixCRT16_DEFAULT                                                          0x00000000
-#define ixCRT17_DEFAULT                                                          0x00000000
-#define ixCRT18_DEFAULT                                                          0x00000000
-#define ixCRT1E_DEFAULT                                                          0x00000000
-#define ixCRT1F_DEFAULT                                                          0x00000000
-#define ixCRT22_DEFAULT                                                          0x00000000
-
-
-// addressBlock: vga_vgagrphind
-#define ixGRA00_DEFAULT                                                          0x00000000
-#define ixGRA01_DEFAULT                                                          0x00000000
-#define ixGRA02_DEFAULT                                                          0x00000000
-#define ixGRA03_DEFAULT                                                          0x00000000
-#define ixGRA04_DEFAULT                                                          0x00000000
-#define ixGRA05_DEFAULT                                                          0x00000000
-#define ixGRA06_DEFAULT                                                          0x00000000
-#define ixGRA07_DEFAULT                                                          0x00000000
-#define ixGRA08_DEFAULT                                                          0x00000000
-
-
-// addressBlock: vga_vgaattrind
-#define ixATTR00_DEFAULT                                                         0x00000000
-#define ixATTR01_DEFAULT                                                         0x00000000
-#define ixATTR02_DEFAULT                                                         0x00000000
-#define ixATTR03_DEFAULT                                                         0x00000000
-#define ixATTR04_DEFAULT                                                         0x00000000
-#define ixATTR05_DEFAULT                                                         0x00000000
-#define ixATTR06_DEFAULT                                                         0x00000000
-#define ixATTR07_DEFAULT                                                         0x00000000
-#define ixATTR08_DEFAULT                                                         0x00000000
-#define ixATTR09_DEFAULT                                                         0x00000000
-#define ixATTR0A_DEFAULT                                                         0x00000000
-#define ixATTR0B_DEFAULT                                                         0x00000000
-#define ixATTR0C_DEFAULT                                                         0x00000000
-#define ixATTR0D_DEFAULT                                                         0x00000000
-#define ixATTR0E_DEFAULT                                                         0x00000000
-#define ixATTR0F_DEFAULT                                                         0x00000000
-#define ixATTR10_DEFAULT                                                         0x00000000
-#define ixATTR11_DEFAULT                                                         0x00000000
-#define ixATTR12_DEFAULT                                                         0x00000000
-#define ixATTR13_DEFAULT                                                         0x00000000
-#define ixATTR14_DEFAULT                                                         0x00000000
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-// addressBlock: azendpoint_f2codecind
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT            0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT            0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT                       0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT                    0x000000b4
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT                0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT  0x00000020
-#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY_DEFAULT     0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT                     0x00000040
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT                 0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT     0x00000010
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2_DEFAULT   0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3_DEFAULT   0x00000056
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4_DEFAULT   0x00000018
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT                 0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO_DEFAULT                      0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DEFAULT                   0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC_DEFAULT                            0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_HBR_DEFAULT                                0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT                  0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT                      0x00000000
-#define ixAZALIA_F2_CODEC_PIN_ASSOCIATION_INFO_DEFAULT                           0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_DEFAULT                               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT                0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT                        0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT                     0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT    0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT                   0x00000000
-#define ixAZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT                     0x00000000
-#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH_DEFAULT           0x00000000
-
-
-// addressBlock: azendpoint_descriptorind
-#define ixAUDIO_DESCRIPTOR0_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR1_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR2_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR3_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR4_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR5_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR6_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR7_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR8_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR9_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR10_DEFAULT                                             0x00000000
-#define ixAUDIO_DESCRIPTOR11_DEFAULT                                             0x00000000
-#define ixAUDIO_DESCRIPTOR12_DEFAULT                                             0x00000000
-#define ixAUDIO_DESCRIPTOR13_DEFAULT                                             0x00000000
-
-
-// addressBlock: azendpoint_sinkinfoind
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID_DEFAULT                    0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID_DEFAULT                         0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID0_DEFAULT                            0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID1_DEFAULT                            0x00000000
-#define ixSINK_DESCRIPTION0_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION1_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION2_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION3_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION4_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION5_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION6_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION7_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION8_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION9_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION10_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION11_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION12_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION13_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION14_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION15_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION16_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION17_DEFAULT                                             0x00000000
-
-
-// addressBlock: azf0controller_azinputcrc0resultind
-#define ixAZALIA_INPUT_CRC0_CHANNEL0_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL1_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL2_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL3_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL4_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL5_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL6_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL7_DEFAULT                                     0x00000000
-
-
-// addressBlock: azf0controller_azinputcrc1resultind
-#define ixAZALIA_INPUT_CRC1_CHANNEL0_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL1_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL2_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL3_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL4_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL5_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL6_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL7_DEFAULT                                     0x00000000
-
-
-// addressBlock: azf0controller_azcrc0resultind
-#define ixAZALIA_CRC0_CHANNEL0_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL1_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL2_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL3_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL4_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL5_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL6_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL7_DEFAULT                                           0x00000000
-
-
-// addressBlock: azf0controller_azcrc1resultind
-#define ixAZALIA_CRC1_CHANNEL0_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL1_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL2_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL3_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL4_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL5_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL6_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL7_DEFAULT                                           0x00000000
-
-
-// addressBlock: azinputendpoint_f2codecind
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT      0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT      0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000020
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT               0x00000020
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT           0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x000000f0
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2_DEFAULT 0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3_DEFAULT 0x000000d6
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4_DEFAULT 0x00000018
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT           0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR_DEFAULT                          0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT                         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT         0x00000010
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT                    0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_L_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_H_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT  0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT               0x00000000
-
-
-// addressBlock: azroot_f2codecind
-#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_DEFAULT            0x00000000
-#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID_DEFAULT                     0x00000000
-#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE_DEFAULT                   0x00000003
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2_DEFAULT       0x00000001
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3_DEFAULT       0x000000aa
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION_DEFAULT     0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESET_DEFAULT                         0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT_DEFAULT      0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_DEFAULT                  0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES_DEFAULT                0x00000000
-
-
-// addressBlock: azf0stream0_streamind
-#define ixAZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM0_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM0_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM0_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM0_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream1_streamind
-#define ixAZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM1_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM1_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM1_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM1_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream2_streamind
-#define ixAZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM2_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM2_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM2_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM2_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream3_streamind
-#define ixAZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM3_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM3_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM3_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM3_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream4_streamind
-#define ixAZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM4_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM4_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM4_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM4_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream5_streamind
-#define ixAZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM5_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM5_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM5_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM5_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream6_streamind
-#define ixAZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM6_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM6_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM6_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM6_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream7_streamind
-#define ixAZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM7_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM7_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM7_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM7_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream8_streamind
-#define ixAZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM8_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM8_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM8_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM8_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream9_streamind
-#define ixAZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM9_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM9_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM9_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM9_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream10_streamind
-#define ixAZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM10_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM10_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM10_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM10_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream11_streamind
-#define ixAZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM11_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM11_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM11_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM11_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream12_streamind
-#define ixAZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM12_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM12_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM12_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM12_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream13_streamind
-#define ixAZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM13_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM13_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM13_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM13_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream14_streamind
-#define ixAZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM14_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM14_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM14_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM14_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream15_streamind
-#define ixAZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM15_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM15_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM15_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM15_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0endpoint0_endpointind
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint1_endpointind
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint2_endpointind
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint3_endpointind
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint4_endpointind
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint5_endpointind
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint6_endpointind
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint7_endpointind
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0inputendpoint0_inputendpointind
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint1_inputendpointind
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint2_inputendpointind
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint3_inputendpointind
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint4_inputendpointind
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint5_inputendpointind
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint6_inputendpointind
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint7_inputendpointind
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_default.h b/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_default.h
deleted file mode 100644
index 582f1a66e354..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_default.h
+++ /dev/null
@@ -1,4005 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _gc_9_1_DEFAULT_HEADER
-#define _gc_9_1_DEFAULT_HEADER
-
-
-// addressBlock: gc_grbmdec
-#define mmGRBM_CNTL_DEFAULT                                                      0x00000018
-#define mmGRBM_SKEW_CNTL_DEFAULT                                                 0x00000020
-#define mmGRBM_STATUS2_DEFAULT                                                   0x00000000
-#define mmGRBM_PWR_CNTL_DEFAULT                                                  0x00000000
-#define mmGRBM_STATUS_DEFAULT                                                    0x00000000
-#define mmGRBM_STATUS_SE0_DEFAULT                                                0x00000000
-#define mmGRBM_STATUS_SE1_DEFAULT                                                0x00000000
-#define mmGRBM_SOFT_RESET_DEFAULT                                                0x00000000
-#define mmGRBM_CGTT_CLK_CNTL_DEFAULT                                             0x00000100
-#define mmGRBM_GFX_CLKEN_CNTL_DEFAULT                                            0x00001008
-#define mmGRBM_WAIT_IDLE_CLOCKS_DEFAULT                                          0x00000030
-#define mmGRBM_STATUS_SE2_DEFAULT                                                0x00000000
-#define mmGRBM_STATUS_SE3_DEFAULT                                                0x00000000
-#define mmGRBM_READ_ERROR_DEFAULT                                                0x00000000
-#define mmGRBM_READ_ERROR2_DEFAULT                                               0x00000000
-#define mmGRBM_INT_CNTL_DEFAULT                                                  0x00000000
-#define mmGRBM_TRAP_OP_DEFAULT                                                   0x00000000
-#define mmGRBM_TRAP_ADDR_DEFAULT                                                 0x00000000
-#define mmGRBM_TRAP_ADDR_MSK_DEFAULT                                             0x0003ffff
-#define mmGRBM_TRAP_WD_DEFAULT                                                   0x00000000
-#define mmGRBM_TRAP_WD_MSK_DEFAULT                                               0xffffffff
-#define mmGRBM_DSM_BYPASS_DEFAULT                                                0x00000000
-#define mmGRBM_WRITE_ERROR_DEFAULT                                               0x00000000
-#define mmGRBM_IOV_ERROR_DEFAULT                                                 0x00000000
-#define mmGRBM_CHIP_REVISION_DEFAULT                                             0x00000000
-#define mmGRBM_GFX_CNTL_DEFAULT                                                  0x00000000
-#define mmGRBM_RSMU_CFG_DEFAULT                                                  0x00011000
-#define mmGRBM_IH_CREDIT_DEFAULT                                                 0x00010000
-#define mmGRBM_PWR_CNTL2_DEFAULT                                                 0x00010000
-#define mmGRBM_UTCL2_INVAL_RANGE_START_DEFAULT                                   0x00002891
-#define mmGRBM_UTCL2_INVAL_RANGE_END_DEFAULT                                     0x000028ea
-#define mmGRBM_RSMU_READ_ERROR_DEFAULT                                           0x00000000
-#define mmGRBM_CHICKEN_BITS_DEFAULT                                              0x00000000
-#define mmGRBM_NOWHERE_DEFAULT                                                   0x00000000
-#define mmGRBM_SCRATCH_REG0_DEFAULT                                              0x00000000
-#define mmGRBM_SCRATCH_REG1_DEFAULT                                              0x00000000
-#define mmGRBM_SCRATCH_REG2_DEFAULT                                              0x00000000
-#define mmGRBM_SCRATCH_REG3_DEFAULT                                              0x00000000
-#define mmGRBM_SCRATCH_REG4_DEFAULT                                              0x00000000
-#define mmGRBM_SCRATCH_REG5_DEFAULT                                              0x00000000
-#define mmGRBM_SCRATCH_REG6_DEFAULT                                              0x00000000
-#define mmGRBM_SCRATCH_REG7_DEFAULT                                              0x00000000
-
-
-// addressBlock: gc_cpdec
-#define mmCP_CPC_STATUS_DEFAULT                                                  0x00000000
-#define mmCP_CPC_BUSY_STAT_DEFAULT                                               0x00000000
-#define mmCP_CPC_STALLED_STAT1_DEFAULT                                           0x00000000
-#define mmCP_CPF_STATUS_DEFAULT                                                  0x00000000
-#define mmCP_CPF_BUSY_STAT_DEFAULT                                               0x00000000
-#define mmCP_CPF_STALLED_STAT1_DEFAULT                                           0x00000000
-#define mmCP_CPC_GRBM_FREE_COUNT_DEFAULT                                         0x00000008
-#define mmCP_MEC_CNTL_DEFAULT                                                    0x50000000
-#define mmCP_MEC_ME1_HEADER_DUMP_DEFAULT                                         0x00000000
-#define mmCP_MEC_ME2_HEADER_DUMP_DEFAULT                                         0x00000000
-#define mmCP_CPC_SCRATCH_INDEX_DEFAULT                                           0x00000000
-#define mmCP_CPC_SCRATCH_DATA_DEFAULT                                            0x00000000
-#define mmCP_CPF_GRBM_FREE_COUNT_DEFAULT                                         0x00000004
-#define mmCP_CPC_HALT_HYST_COUNT_DEFAULT                                         0x00000002
-#define mmCP_PRT_LOD_STATS_CNTL0_DEFAULT                                         0x00000000
-#define mmCP_PRT_LOD_STATS_CNTL1_DEFAULT                                         0x00000000
-#define mmCP_PRT_LOD_STATS_CNTL2_DEFAULT                                         0x00000000
-#define mmCP_PRT_LOD_STATS_CNTL3_DEFAULT                                         0x00000000
-#define mmCP_CE_COMPARE_COUNT_DEFAULT                                            0x00000000
-#define mmCP_CE_DE_COUNT_DEFAULT                                                 0x00000000
-#define mmCP_DE_CE_COUNT_DEFAULT                                                 0x00000000
-#define mmCP_DE_LAST_INVAL_COUNT_DEFAULT                                         0x00000000
-#define mmCP_DE_DE_COUNT_DEFAULT                                                 0x00000000
-#define mmCP_STALLED_STAT3_DEFAULT                                               0x00000000
-#define mmCP_STALLED_STAT1_DEFAULT                                               0x00000000
-#define mmCP_STALLED_STAT2_DEFAULT                                               0x00000000
-#define mmCP_BUSY_STAT_DEFAULT                                                   0x00000000
-#define mmCP_STAT_DEFAULT                                                        0x00000000
-#define mmCP_ME_HEADER_DUMP_DEFAULT                                              0x00000000
-#define mmCP_PFP_HEADER_DUMP_DEFAULT                                             0x00000000
-#define mmCP_GRBM_FREE_COUNT_DEFAULT                                             0x00080808
-#define mmCP_CE_HEADER_DUMP_DEFAULT                                              0x00000000
-#define mmCP_PFP_INSTR_PNTR_DEFAULT                                              0x00000000
-#define mmCP_ME_INSTR_PNTR_DEFAULT                                               0x00000000
-#define mmCP_CE_INSTR_PNTR_DEFAULT                                               0x00000000
-#define mmCP_MEC1_INSTR_PNTR_DEFAULT                                             0x00000000
-#define mmCP_MEC2_INSTR_PNTR_DEFAULT                                             0x00000000
-#define mmCP_CSF_STAT_DEFAULT                                                    0x00000000
-#define mmCP_ME_CNTL_DEFAULT                                                     0x15000000
-#define mmCP_CNTX_STAT_DEFAULT                                                   0x00000000
-#define mmCP_ME_PREEMPTION_DEFAULT                                               0x00000000
-#define mmCP_ROQ_THRESHOLDS_DEFAULT                                              0x00003010
-#define mmCP_MEQ_STQ_THRESHOLD_DEFAULT                                           0x00000010
-#define mmCP_RB2_RPTR_DEFAULT                                                    0x00000000
-#define mmCP_RB1_RPTR_DEFAULT                                                    0x00000000
-#define mmCP_RB0_RPTR_DEFAULT                                                    0x00000000
-#define mmCP_RB_RPTR_DEFAULT                                                     0x00000000
-#define mmCP_RB_WPTR_DELAY_DEFAULT                                               0x00000000
-#define mmCP_RB_WPTR_POLL_CNTL_DEFAULT                                           0x00400100
-#define mmCP_ROQ1_THRESHOLDS_DEFAULT                                             0x30101010
-#define mmCP_ROQ2_THRESHOLDS_DEFAULT                                             0x40403030
-#define mmCP_STQ_THRESHOLDS_DEFAULT                                              0x00804000
-#define mmCP_QUEUE_THRESHOLDS_DEFAULT                                            0x00002b16
-#define mmCP_MEQ_THRESHOLDS_DEFAULT                                              0x00008040
-#define mmCP_ROQ_AVAIL_DEFAULT                                                   0x00000000
-#define mmCP_STQ_AVAIL_DEFAULT                                                   0x00000000
-#define mmCP_ROQ2_AVAIL_DEFAULT                                                  0x00000000
-#define mmCP_MEQ_AVAIL_DEFAULT                                                   0x00000000
-#define mmCP_CMD_INDEX_DEFAULT                                                   0x00000000
-#define mmCP_CMD_DATA_DEFAULT                                                    0x00000000
-#define mmCP_ROQ_RB_STAT_DEFAULT                                                 0x00000000
-#define mmCP_ROQ_IB1_STAT_DEFAULT                                                0x00000000
-#define mmCP_ROQ_IB2_STAT_DEFAULT                                                0x00000000
-#define mmCP_STQ_STAT_DEFAULT                                                    0x00000000
-#define mmCP_STQ_WR_STAT_DEFAULT                                                 0x00000000
-#define mmCP_MEQ_STAT_DEFAULT                                                    0x00000000
-#define mmCP_CEQ1_AVAIL_DEFAULT                                                  0x00000000
-#define mmCP_CEQ2_AVAIL_DEFAULT                                                  0x00000000
-#define mmCP_CE_ROQ_RB_STAT_DEFAULT                                              0x00000000
-#define mmCP_CE_ROQ_IB1_STAT_DEFAULT                                             0x00000000
-#define mmCP_CE_ROQ_IB2_STAT_DEFAULT                                             0x00000000
-
-
-// addressBlock: gc_padec
-#define mmVGT_VTX_VECT_EJECT_REG_DEFAULT                                         0x0000007d
-#define mmVGT_DMA_DATA_FIFO_DEPTH_DEFAULT                                        0x00040180
-#define mmVGT_DMA_REQ_FIFO_DEPTH_DEFAULT                                         0x00000020
-#define mmVGT_DRAW_INIT_FIFO_DEPTH_DEFAULT                                       0x00000020
-#define mmVGT_LAST_COPY_STATE_DEFAULT                                            0x00000000
-#define mmVGT_CACHE_INVALIDATION_DEFAULT                                         0x09000000
-#define mmVGT_STRMOUT_DELAY_DEFAULT                                              0x00092410
-#define mmVGT_FIFO_DEPTHS_DEFAULT                                                0x08000040
-#define mmVGT_GS_VERTEX_REUSE_DEFAULT                                            0x00000010
-#define mmVGT_MC_LAT_CNTL_DEFAULT                                                0x000000fe
-#define mmIA_CNTL_STATUS_DEFAULT                                                 0x00000000
-#define mmVGT_CNTL_STATUS_DEFAULT                                                0x00000000
-#define mmWD_CNTL_STATUS_DEFAULT                                                 0x00000000
-#define mmCC_GC_PRIM_CONFIG_DEFAULT                                              0x0e020000
-#define mmGC_USER_PRIM_CONFIG_DEFAULT                                            0x00000000
-#define mmWD_QOS_DEFAULT                                                         0x00000000
-#define mmWD_UTCL1_CNTL_DEFAULT                                                  0x00000080
-#define mmWD_UTCL1_STATUS_DEFAULT                                                0x00000000
-#define mmIA_UTCL1_CNTL_DEFAULT                                                  0x00000080
-#define mmIA_UTCL1_STATUS_DEFAULT                                                0x00000000
-#define mmVGT_SYS_CONFIG_DEFAULT                                                 0x00000011
-#define mmVGT_VS_MAX_WAVE_ID_DEFAULT                                             0x0000007f
-#define mmVGT_GS_MAX_WAVE_ID_DEFAULT                                             0x000000ff
-#define mmGFX_PIPE_CONTROL_DEFAULT                                               0x00000000
-#define mmCC_GC_SHADER_ARRAY_CONFIG_DEFAULT                                      0xf8000000
-#define mmGC_USER_SHADER_ARRAY_CONFIG_DEFAULT                                    0x00000000
-#define mmVGT_DMA_PRIMITIVE_TYPE_DEFAULT                                         0x00000000
-#define mmVGT_DMA_CONTROL_DEFAULT                                                0x000000ff
-#define mmVGT_DMA_LS_HS_CONFIG_DEFAULT                                           0x00000000
-#define mmWD_BUF_RESOURCE_1_DEFAULT                                              0x00000000
-#define mmWD_BUF_RESOURCE_2_DEFAULT                                              0x00000000
-#define mmPA_CL_CNTL_STATUS_DEFAULT                                              0x00000000
-#define mmPA_CL_ENHANCE_DEFAULT                                                  0x00000007
-#define mmPA_SU_CNTL_STATUS_DEFAULT                                              0x00000000
-#define mmPA_SC_FIFO_DEPTH_CNTL_DEFAULT                                          0x00000018
-#define mmPA_SC_P3D_TRAP_SCREEN_HV_LOCK_DEFAULT                                  0x00000000
-#define mmPA_SC_HP3D_TRAP_SCREEN_HV_LOCK_DEFAULT                                 0x00000000
-#define mmPA_SC_TRAP_SCREEN_HV_LOCK_DEFAULT                                      0x00000000
-#define mmPA_SC_FORCE_EOV_MAX_CNTS_DEFAULT                                       0x00ffffff
-#define mmPA_SC_BINNER_EVENT_CNTL_0_DEFAULT                                      0x842a4402
-#define mmPA_SC_BINNER_EVENT_CNTL_1_DEFAULT                                      0x8a000008
-#define mmPA_SC_BINNER_EVENT_CNTL_2_DEFAULT                                      0x9118aaa8
-#define mmPA_SC_BINNER_EVENT_CNTL_3_DEFAULT                                      0x82400025
-#define mmPA_SC_BINNER_TIMEOUT_COUNTER_DEFAULT                                   0x00000000
-#define mmPA_SC_BINNER_PERF_CNTL_0_DEFAULT                                       0x00000000
-#define mmPA_SC_BINNER_PERF_CNTL_1_DEFAULT                                       0x00000000
-#define mmPA_SC_BINNER_PERF_CNTL_2_DEFAULT                                       0x00000000
-#define mmPA_SC_BINNER_PERF_CNTL_3_DEFAULT                                       0x00000000
-#define mmPA_SC_FIFO_SIZE_DEFAULT                                                0x00000000
-#define mmPA_SC_IF_FIFO_SIZE_DEFAULT                                             0x00000000
-#define mmPA_SC_PKR_WAVE_TABLE_CNTL_DEFAULT                                      0x00000000
-#define mmPA_UTCL1_CNTL1_DEFAULT                                                 0x00000600
-#define mmPA_UTCL1_CNTL2_DEFAULT                                                 0x00000000
-#define mmPA_SIDEBAND_REQUEST_DELAYS_DEFAULT                                     0x08000020
-#define mmPA_SC_ENHANCE_DEFAULT                                                  0x00000001
-#define mmPA_SC_ENHANCE_1_DEFAULT                                                0x00040000
-#define mmPA_SC_DSM_CNTL_DEFAULT                                                 0x00000000
-#define mmPA_SC_TILE_STEERING_CREST_OVERRIDE_DEFAULT                             0x00000000
-
-
-// addressBlock: gc_sqdec
-#define mmSQ_CONFIG_DEFAULT                                                      0x01180000
-#define mmSQC_CONFIG_DEFAULT                                                     0x010a2000
-#define mmLDS_CONFIG_DEFAULT                                                     0x00000000
-#define mmSQ_RANDOM_WAVE_PRI_DEFAULT                                             0x0000007f
-#define mmSQ_REG_CREDITS_DEFAULT                                                 0x00000820
-#define mmSQ_FIFO_SIZES_DEFAULT                                                  0x00000f01
-#define mmSQ_DSM_CNTL_DEFAULT                                                    0x00000000
-#define mmSQ_DSM_CNTL2_DEFAULT                                                   0x00000000
-#define mmSQ_RUNTIME_CONFIG_DEFAULT                                              0x00000000
-#define mmSH_MEM_BASES_DEFAULT                                                   0x00000000
-#define mmSH_MEM_CONFIG_DEFAULT                                                  0x00000000
-#define mmCC_GC_SHADER_RATE_CONFIG_DEFAULT                                       0x00000000
-#define mmGC_USER_SHADER_RATE_CONFIG_DEFAULT                                     0x00000000
-#define mmSQ_INTERRUPT_AUTO_MASK_DEFAULT                                         0x00ffffff
-#define mmSQ_INTERRUPT_MSG_CTRL_DEFAULT                                          0x00000000
-#define mmSQ_UTCL1_CNTL1_DEFAULT                                                 0x00000580
-#define mmSQ_UTCL1_CNTL2_DEFAULT                                                 0x00000000
-#define mmSQ_UTCL1_STATUS_DEFAULT                                                0x00000000
-#define mmSQ_SHADER_TBA_LO_DEFAULT                                               0x00000000
-#define mmSQ_SHADER_TBA_HI_DEFAULT                                               0x00000000
-#define mmSQ_SHADER_TMA_LO_DEFAULT                                               0x00000000
-#define mmSQ_SHADER_TMA_HI_DEFAULT                                               0x00000000
-#define mmSQC_DSM_CNTL_DEFAULT                                                   0x00000000
-#define mmSQC_DSM_CNTLA_DEFAULT                                                  0x00000000
-#define mmSQC_DSM_CNTLB_DEFAULT                                                  0x00000000
-#define mmSQC_DSM_CNTL2_DEFAULT                                                  0x00000000
-#define mmSQC_DSM_CNTL2A_DEFAULT                                                 0x00000000
-#define mmSQC_DSM_CNTL2B_DEFAULT                                                 0x00000000
-#define mmSQC_EDC_FUE_CNTL_DEFAULT                                               0x00000000
-#define mmSQC_EDC_CNT2_DEFAULT                                                   0x00000000
-#define mmSQC_EDC_CNT3_DEFAULT                                                   0x00000000
-#define mmSQ_REG_TIMESTAMP_DEFAULT                                               0x00000000
-#define mmSQ_CMD_TIMESTAMP_DEFAULT                                               0x00000000
-#define mmSQ_IND_INDEX_DEFAULT                                                   0x00000000
-#define mmSQ_IND_DATA_DEFAULT                                                    0x00000000
-#define mmSQ_CMD_DEFAULT                                                         0x00000000
-#define mmSQ_TIME_HI_DEFAULT                                                     0x00000000
-#define mmSQ_TIME_LO_DEFAULT                                                     0x00000000
-#define mmSQ_DS_0_DEFAULT                                                        0x00000000
-#define mmSQ_DS_1_DEFAULT                                                        0x00000000
-#define mmSQ_EXP_0_DEFAULT                                                       0x00000000
-#define mmSQ_EXP_1_DEFAULT                                                       0x00000000
-#define mmSQ_FLAT_0_DEFAULT                                                      0x00000000
-#define mmSQ_FLAT_1_DEFAULT                                                      0x00000000
-#define mmSQ_GLBL_0_DEFAULT                                                      0x00000000
-#define mmSQ_GLBL_1_DEFAULT                                                      0x00000000
-#define mmSQ_INST_DEFAULT                                                        0x00000000
-#define mmSQ_MIMG_0_DEFAULT                                                      0x00000000
-#define mmSQ_MIMG_1_DEFAULT                                                      0x00000000
-#define mmSQ_MTBUF_0_DEFAULT                                                     0x00000000
-#define mmSQ_MTBUF_1_DEFAULT                                                     0x00000000
-#define mmSQ_MUBUF_0_DEFAULT                                                     0x00000000
-#define mmSQ_MUBUF_1_DEFAULT                                                     0x00000000
-#define mmSQ_SCRATCH_0_DEFAULT                                                   0x00000000
-#define mmSQ_SCRATCH_1_DEFAULT                                                   0x00000000
-#define mmSQ_SMEM_0_DEFAULT                                                      0x00000000
-#define mmSQ_SMEM_1_DEFAULT                                                      0x00000000
-#define mmSQ_SOP1_DEFAULT                                                        0x00000000
-#define mmSQ_SOP2_DEFAULT                                                        0x00000000
-#define mmSQ_SOPC_DEFAULT                                                        0x00000000
-#define mmSQ_SOPK_DEFAULT                                                        0x00000000
-#define mmSQ_SOPP_DEFAULT                                                        0x00000000
-#define mmSQ_VINTRP_DEFAULT                                                      0x00000000
-#define mmSQ_VOP1_DEFAULT                                                        0x00000000
-#define mmSQ_VOP2_DEFAULT                                                        0x00000000
-#define mmSQ_VOP3P_0_DEFAULT                                                     0x00000000
-#define mmSQ_VOP3P_1_DEFAULT                                                     0x00000000
-#define mmSQ_VOP3_0_DEFAULT                                                      0x00000000
-#define mmSQ_VOP3_0_SDST_ENC_DEFAULT                                             0x00000000
-#define mmSQ_VOP3_1_DEFAULT                                                      0x00000000
-#define mmSQ_VOPC_DEFAULT                                                        0x00000000
-#define mmSQ_VOP_DPP_DEFAULT                                                     0x00000000
-#define mmSQ_VOP_SDWA_DEFAULT                                                    0x00000000
-#define mmSQ_VOP_SDWA_SDST_ENC_DEFAULT                                           0x00000000
-#define mmSQ_LB_CTR_CTRL_DEFAULT                                                 0x00000000
-#define mmSQ_LB_DATA0_DEFAULT                                                    0x00000000
-#define mmSQ_LB_DATA1_DEFAULT                                                    0x00000000
-#define mmSQ_LB_DATA2_DEFAULT                                                    0x00000000
-#define mmSQ_LB_DATA3_DEFAULT                                                    0x00000000
-#define mmSQ_LB_CTR_SEL_DEFAULT                                                  0x00000000
-#define mmSQ_LB_CTR0_CU_DEFAULT                                                  0xffffffff
-#define mmSQ_LB_CTR1_CU_DEFAULT                                                  0xffffffff
-#define mmSQ_LB_CTR2_CU_DEFAULT                                                  0xffffffff
-#define mmSQ_LB_CTR3_CU_DEFAULT                                                  0xffffffff
-#define mmSQC_EDC_CNT_DEFAULT                                                    0x00000000
-#define mmSQ_EDC_SEC_CNT_DEFAULT                                                 0x00000000
-#define mmSQ_EDC_DED_CNT_DEFAULT                                                 0x00000000
-#define mmSQ_EDC_INFO_DEFAULT                                                    0x00000000
-#define mmSQ_EDC_CNT_DEFAULT                                                     0x00000000
-#define mmSQ_EDC_FUE_CNTL_DEFAULT                                                0x00000000
-#define mmSQ_THREAD_TRACE_WORD_CMN_DEFAULT                                       0x00000000
-#define mmSQ_THREAD_TRACE_WORD_EVENT_DEFAULT                                     0x00000000
-#define mmSQ_THREAD_TRACE_WORD_INST_DEFAULT                                      0x00000000
-#define mmSQ_THREAD_TRACE_WORD_INST_PC_1_OF_2_DEFAULT                            0x00000000
-#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2_DEFAULT                      0x00000000
-#define mmSQ_THREAD_TRACE_WORD_ISSUE_DEFAULT                                     0x00000000
-#define mmSQ_THREAD_TRACE_WORD_MISC_DEFAULT                                      0x00000000
-#define mmSQ_THREAD_TRACE_WORD_PERF_1_OF_2_DEFAULT                               0x00000000
-#define mmSQ_THREAD_TRACE_WORD_REG_1_OF_2_DEFAULT                                0x00000000
-#define mmSQ_THREAD_TRACE_WORD_REG_2_OF_2_DEFAULT                                0x00000000
-#define mmSQ_THREAD_TRACE_WORD_REG_CS_1_OF_2_DEFAULT                             0x00000000
-#define mmSQ_THREAD_TRACE_WORD_REG_CS_2_OF_2_DEFAULT                             0x00000000
-#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2_DEFAULT                          0x00000000
-#define mmSQ_THREAD_TRACE_WORD_WAVE_DEFAULT                                      0x00000000
-#define mmSQ_THREAD_TRACE_WORD_WAVE_START_DEFAULT                                0x00000000
-#define mmSQ_THREAD_TRACE_WORD_INST_PC_2_OF_2_DEFAULT                            0x00000000
-#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2_DEFAULT                      0x00000000
-#define mmSQ_THREAD_TRACE_WORD_PERF_2_OF_2_DEFAULT                               0x00000000
-#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2_DEFAULT                          0x00000000
-#define mmSQ_WREXEC_EXEC_HI_DEFAULT                                              0x00000000
-#define mmSQ_WREXEC_EXEC_LO_DEFAULT                                              0x00000000
-#define mmSQ_BUF_RSRC_WORD0_DEFAULT                                              0x00000000
-#define mmSQ_BUF_RSRC_WORD1_DEFAULT                                              0x00000000
-#define mmSQ_BUF_RSRC_WORD2_DEFAULT                                              0x00000000
-#define mmSQ_BUF_RSRC_WORD3_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD0_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD1_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD2_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD3_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD4_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD5_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD6_DEFAULT                                              0x00000000
-#define mmSQ_IMG_RSRC_WORD7_DEFAULT                                              0x00000000
-#define mmSQ_IMG_SAMP_WORD0_DEFAULT                                              0x00000000
-#define mmSQ_IMG_SAMP_WORD1_DEFAULT                                              0x00000000
-#define mmSQ_IMG_SAMP_WORD2_DEFAULT                                              0x00000000
-#define mmSQ_IMG_SAMP_WORD3_DEFAULT                                              0x00000000
-#define mmSQ_FLAT_SCRATCH_WORD0_DEFAULT                                          0x00000000
-#define mmSQ_FLAT_SCRATCH_WORD1_DEFAULT                                          0x00000000
-#define mmSQ_M0_GPR_IDX_WORD_DEFAULT                                             0x00000000
-#define mmSQC_ICACHE_UTCL1_CNTL1_DEFAULT                                         0x00000480
-#define mmSQC_ICACHE_UTCL1_CNTL2_DEFAULT                                         0x00000000
-#define mmSQC_DCACHE_UTCL1_CNTL1_DEFAULT                                         0x00000500
-#define mmSQC_DCACHE_UTCL1_CNTL2_DEFAULT                                         0x00000000
-#define mmSQC_ICACHE_UTCL1_STATUS_DEFAULT                                        0x00000000
-#define mmSQC_DCACHE_UTCL1_STATUS_DEFAULT                                        0x00000000
-
-
-// addressBlock: gc_shsdec
-#define mmSX_DEBUG_1_DEFAULT                                                     0x00000020
-#define mmSPI_PS_MAX_WAVE_ID_DEFAULT                                             0x020000ff
-#define mmSPI_START_PHASE_DEFAULT                                                0x00000000
-#define mmSPI_GFX_CNTL_DEFAULT                                                   0x00000000
-#define mmSPI_DSM_CNTL_DEFAULT                                                   0x00000000
-#define mmSPI_DSM_CNTL2_DEFAULT                                                  0x00000000
-#define mmSPI_EDC_CNT_DEFAULT                                                    0x00000000
-#define mmSPI_CONFIG_PS_CU_EN_DEFAULT                                            0x00000000
-#define mmSPI_WF_LIFETIME_CNTL_DEFAULT                                           0x00000000
-#define mmSPI_WF_LIFETIME_LIMIT_0_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_1_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_2_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_3_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_4_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_5_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_6_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_7_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_8_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_LIMIT_9_DEFAULT                                        0x00000100
-#define mmSPI_WF_LIFETIME_STATUS_0_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_1_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_2_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_3_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_4_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_5_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_6_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_7_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_8_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_9_DEFAULT                                       0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_10_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_11_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_12_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_13_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_14_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_15_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_16_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_17_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_18_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_19_DEFAULT                                      0x00000000
-#define mmSPI_WF_LIFETIME_STATUS_20_DEFAULT                                      0x00000000
-#define mmSPI_LB_CTR_CTRL_DEFAULT                                                0x00000000
-#define mmSPI_LB_CU_MASK_DEFAULT                                                 0x0000ffff
-#define mmSPI_LB_DATA_REG_DEFAULT                                                0x00000000
-#define mmSPI_PG_ENABLE_STATIC_CU_MASK_DEFAULT                                   0x0000ffff
-#define mmSPI_GDS_CREDITS_DEFAULT                                                0x00001080
-#define mmSPI_SX_EXPORT_BUFFER_SIZES_DEFAULT                                     0x08000800
-#define mmSPI_SX_SCOREBOARD_BUFFER_SIZES_DEFAULT                                 0x00200040
-#define mmSPI_CSQ_WF_ACTIVE_STATUS_DEFAULT                                       0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_0_DEFAULT                                      0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_1_DEFAULT                                      0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_2_DEFAULT                                      0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_3_DEFAULT                                      0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_4_DEFAULT                                      0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_5_DEFAULT                                      0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_6_DEFAULT                                      0x00000000
-#define mmSPI_CSQ_WF_ACTIVE_COUNT_7_DEFAULT                                      0x00000000
-#define mmSPI_LB_DATA_WAVES_DEFAULT                                              0x00000000
-#define mmSPI_LB_DATA_PERCU_WAVE_HSGS_DEFAULT                                    0x00000000
-#define mmSPI_LB_DATA_PERCU_WAVE_VSPS_DEFAULT                                    0x00000000
-#define mmSPI_LB_DATA_PERCU_WAVE_CS_DEFAULT                                      0x00000000
-#define mmSPI_P0_TRAP_SCREEN_PSBA_LO_DEFAULT                                     0x00000000
-#define mmSPI_P0_TRAP_SCREEN_PSBA_HI_DEFAULT                                     0x00000000
-#define mmSPI_P0_TRAP_SCREEN_PSMA_LO_DEFAULT                                     0x00000000
-#define mmSPI_P0_TRAP_SCREEN_PSMA_HI_DEFAULT                                     0x00000000
-#define mmSPI_P0_TRAP_SCREEN_GPR_MIN_DEFAULT                                     0x00000000
-#define mmSPI_P1_TRAP_SCREEN_PSBA_LO_DEFAULT                                     0x00000000
-#define mmSPI_P1_TRAP_SCREEN_PSBA_HI_DEFAULT                                     0x00000000
-#define mmSPI_P1_TRAP_SCREEN_PSMA_LO_DEFAULT                                     0x00000000
-#define mmSPI_P1_TRAP_SCREEN_PSMA_HI_DEFAULT                                     0x00000000
-#define mmSPI_P1_TRAP_SCREEN_GPR_MIN_DEFAULT                                     0x00000000
-
-
-// addressBlock: gc_tpdec
-#define mmTD_CNTL_DEFAULT                                                        0x00000000
-#define mmTD_STATUS_DEFAULT                                                      0x00000000
-#define mmTD_DSM_CNTL_DEFAULT                                                    0x00000000
-#define mmTD_DSM_CNTL2_DEFAULT                                                   0x00000000
-#define mmTD_SCRATCH_DEFAULT                                                     0x00000000
-#define mmTA_CNTL_DEFAULT                                                        0x8004d850
-#define mmTA_CNTL_AUX_DEFAULT                                                    0x00000000
-#define mmTA_RESERVED_010C_DEFAULT                                               0x00000000
-#define mmTA_GRAD_ADJ_DEFAULT                                                    0x40000040
-#define mmTA_STATUS_DEFAULT                                                      0x00000000
-#define mmTA_SCRATCH_DEFAULT                                                     0x00000000
-
-
-// addressBlock: gc_gdsdec
-#define mmGDS_CONFIG_DEFAULT                                                     0x00000000
-#define mmGDS_CNTL_STATUS_DEFAULT                                                0x00000000
-#define mmGDS_ENHANCE2_DEFAULT                                                   0x00000000
-#define mmGDS_PROTECTION_FAULT_DEFAULT                                           0x00000000
-#define mmGDS_VM_PROTECTION_FAULT_DEFAULT                                        0x00000000
-#define mmGDS_EDC_CNT_DEFAULT                                                    0x00000000
-#define mmGDS_EDC_GRBM_CNT_DEFAULT                                               0x00000000
-#define mmGDS_EDC_OA_DED_DEFAULT                                                 0x00000000
-#define mmGDS_DSM_CNTL_DEFAULT                                                   0x00000000
-#define mmGDS_EDC_OA_PHY_CNT_DEFAULT                                             0x00000000
-#define mmGDS_EDC_OA_PIPE_CNT_DEFAULT                                            0x00000000
-#define mmGDS_DSM_CNTL2_DEFAULT                                                  0x00000000
-#define mmGDS_WD_GDS_CSB_DEFAULT                                                 0x00000000
-
-
-// addressBlock: gc_rbdec
-#define mmDB_DEBUG_DEFAULT                                                       0x00000000
-#define mmDB_DEBUG2_DEFAULT                                                      0x00000000
-#define mmDB_DEBUG3_DEFAULT                                                      0x00000000
-#define mmDB_DEBUG4_DEFAULT                                                      0x00000000
-#define mmDB_CREDIT_LIMIT_DEFAULT                                                0x00000000
-#define mmDB_WATERMARKS_DEFAULT                                                  0x01020204
-#define mmDB_SUBTILE_CONTROL_DEFAULT                                             0x00000000
-#define mmDB_FREE_CACHELINES_DEFAULT                                             0x00000000
-#define mmDB_FIFO_DEPTH1_DEFAULT                                                 0x00000000
-#define mmDB_FIFO_DEPTH2_DEFAULT                                                 0x00000000
-#define mmDB_EXCEPTION_CONTROL_DEFAULT                                           0x00000000
-#define mmDB_RING_CONTROL_DEFAULT                                                0x00000001
-#define mmDB_MEM_ARB_WATERMARKS_DEFAULT                                          0x04040404
-#define mmDB_RMI_CACHE_POLICY_DEFAULT                                            0x0f0f0f07
-#define mmDB_DFSM_CONFIG_DEFAULT                                                 0x00007f00
-#define mmDB_DFSM_WATERMARK_DEFAULT                                              0x00640064
-#define mmDB_DFSM_TILES_IN_FLIGHT_DEFAULT                                        0x05dc03e8
-#define mmDB_DFSM_PRIMS_IN_FLIGHT_DEFAULT                                        0x00fa00c8
-#define mmDB_DFSM_WATCHDOG_DEFAULT                                               0x000f4240
-#define mmDB_DFSM_FLUSH_ENABLE_DEFAULT                                           0x000003ff
-#define mmDB_DFSM_FLUSH_AUX_EVENT_DEFAULT                                        0x00000000
-#define mmCC_RB_REDUNDANCY_DEFAULT                                               0x00000000
-#define mmCC_RB_BACKEND_DISABLE_DEFAULT                                          0x00000000
-#define mmGB_ADDR_CONFIG_DEFAULT                                                 0x26010011
-#define mmGB_BACKEND_MAP_DEFAULT                                                 0x33221100
-#define mmGB_GPU_ID_DEFAULT                                                      0x00000000
-#define mmCC_RB_DAISY_CHAIN_DEFAULT                                              0x76543210
-#define mmGB_ADDR_CONFIG_READ_DEFAULT                                            0x26010011
-#define mmGB_TILE_MODE0_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE1_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE2_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE3_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE4_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE5_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE6_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE7_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE8_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE9_DEFAULT                                                  0x00000000
-#define mmGB_TILE_MODE10_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE11_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE12_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE13_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE14_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE15_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE16_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE17_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE18_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE19_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE20_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE21_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE22_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE23_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE24_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE25_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE26_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE27_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE28_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE29_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE30_DEFAULT                                                 0x00000000
-#define mmGB_TILE_MODE31_DEFAULT                                                 0x00000000
-#define mmGB_MACROTILE_MODE0_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE1_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE2_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE3_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE4_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE5_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE6_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE7_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE8_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE9_DEFAULT                                             0x00000000
-#define mmGB_MACROTILE_MODE10_DEFAULT                                            0x00000000
-#define mmGB_MACROTILE_MODE11_DEFAULT                                            0x00000000
-#define mmGB_MACROTILE_MODE12_DEFAULT                                            0x00000000
-#define mmGB_MACROTILE_MODE13_DEFAULT                                            0x00000000
-#define mmGB_MACROTILE_MODE14_DEFAULT                                            0x00000000
-#define mmGB_MACROTILE_MODE15_DEFAULT                                            0x00000000
-#define mmCB_HW_CONTROL_DEFAULT                                                  0x00014107
-#define mmCB_HW_CONTROL_1_DEFAULT                                                0x10000000
-#define mmCB_HW_CONTROL_2_DEFAULT                                                0x00000000
-#define mmCB_HW_CONTROL_3_DEFAULT                                                0x00000000
-#define mmCB_HW_MEM_ARBITER_RD_DEFAULT                                           0x00029000
-#define mmCB_HW_MEM_ARBITER_WR_DEFAULT                                           0x00029000
-#define mmCB_DCC_CONFIG_DEFAULT                                                  0x04000000
-#define mmGC_USER_RB_REDUNDANCY_DEFAULT                                          0x00000000
-#define mmGC_USER_RB_BACKEND_DISABLE_DEFAULT                                     0x00000000
-
-
-// addressBlock: gc_ea_gceadec2
-#define mmGCEA_EDC_CNT_DEFAULT                                                   0x00000000
-#define mmGCEA_EDC_CNT2_DEFAULT                                                  0x00000000
-#define mmGCEA_DSM_CNTL_DEFAULT                                                  0x00000000
-#define mmGCEA_DSM_CNTLA_DEFAULT                                                 0x00000000
-#define mmGCEA_DSM_CNTLB_DEFAULT                                                 0x00000000
-#define mmGCEA_DSM_CNTL2_DEFAULT                                                 0x00000000
-#define mmGCEA_DSM_CNTL2A_DEFAULT                                                0x00000000
-#define mmGCEA_DSM_CNTL2B_DEFAULT                                                0x00000000
-#define mmGCEA_TCC_XBR_CREDITS_DEFAULT                                           0x637f637f
-#define mmGCEA_TCC_XBR_MAXBURST_DEFAULT                                          0x00003333
-#define mmGCEA_PROBE_CNTL_DEFAULT                                                0x00000000
-#define mmGCEA_PROBE_MAP_DEFAULT                                                 0x0000aaaa
-#define mmGCEA_ERR_STATUS_DEFAULT                                                0x00000000
-#define mmGCEA_MISC2_DEFAULT                                                     0x00000000
-#define mmGCEA_SDP_BACKDOOR_CMDCREDITS0_DEFAULT                                  0x00000000
-#define mmGCEA_SDP_BACKDOOR_CMDCREDITS1_DEFAULT                                  0x00000000
-#define mmGCEA_SDP_BACKDOOR_DATACREDITS0_DEFAULT                                 0x00000000
-#define mmGCEA_SDP_BACKDOOR_DATACREDITS1_DEFAULT                                 0x00000000
-#define mmGCEA_SDP_BACKDOOR_MISCCREDITS_DEFAULT                                  0x00000000
-#define mmGCEA_SDP_ENABLE_DEFAULT                                                0x00000000
-
-
-// addressBlock: gc_rmi_rmidec
-#define mmRMI_GENERAL_CNTL_DEFAULT                                               0x00000000
-#define mmRMI_GENERAL_CNTL1_DEFAULT                                              0x00001a03
-#define mmRMI_GENERAL_STATUS_DEFAULT                                             0x00000000
-#define mmRMI_SUBBLOCK_STATUS0_DEFAULT                                           0x00000000
-#define mmRMI_SUBBLOCK_STATUS1_DEFAULT                                           0x00000000
-#define mmRMI_SUBBLOCK_STATUS2_DEFAULT                                           0x00000000
-#define mmRMI_SUBBLOCK_STATUS3_DEFAULT                                           0x00000000
-#define mmRMI_XBAR_CONFIG_DEFAULT                                                0x00000f00
-#define mmRMI_PROBE_POP_LOGIC_CNTL_DEFAULT                                       0x000300c0
-#define mmRMI_UTC_XNACK_N_MISC_CNTL_DEFAULT                                      0x00000564
-#define mmRMI_DEMUX_CNTL_DEFAULT                                                 0x02000200
-#define mmRMI_UTCL1_CNTL1_DEFAULT                                                0x00020000
-#define mmRMI_UTCL1_CNTL2_DEFAULT                                                0x00010000
-#define mmRMI_UTC_UNIT_CONFIG_DEFAULT                                            0x00000000
-#define mmRMI_TCIW_FORMATTER0_CNTL_DEFAULT                                       0x4404001e
-#define mmRMI_TCIW_FORMATTER1_CNTL_DEFAULT                                       0x4404001e
-#define mmRMI_SCOREBOARD_CNTL_DEFAULT                                            0x001ffe00
-#define mmRMI_SCOREBOARD_STATUS0_DEFAULT                                         0x00000000
-#define mmRMI_SCOREBOARD_STATUS1_DEFAULT                                         0x00000000
-#define mmRMI_SCOREBOARD_STATUS2_DEFAULT                                         0x00000000
-#define mmRMI_XBAR_ARBITER_CONFIG_DEFAULT                                        0x08000800
-#define mmRMI_XBAR_ARBITER_CONFIG_1_DEFAULT                                      0xffffffff
-#define mmRMI_CLOCK_CNTRL_DEFAULT                                                0x04208822
-#define mmRMI_UTCL1_STATUS_DEFAULT                                               0x00000000
-#define mmRMI_SPARE_DEFAULT                                                      0x00000001
-#define mmRMI_SPARE_1_DEFAULT                                                    0x00000000
-#define mmRMI_SPARE_2_DEFAULT                                                    0x00000000
-
-
-// addressBlock: gc_dbgu_gfx_dbgudec
-#define mmport_a_addr_DEFAULT                                                    0x00000000
-#define mmport_a_data_lo_DEFAULT                                                 0x00000000
-#define mmport_a_data_hi_DEFAULT                                                 0x00000000
-#define mmport_b_addr_DEFAULT                                                    0x00000000
-#define mmport_b_data_lo_DEFAULT                                                 0x00000000
-#define mmport_b_data_hi_DEFAULT                                                 0x00000000
-#define mmport_c_addr_DEFAULT                                                    0x00000000
-#define mmport_c_data_lo_DEFAULT                                                 0x00000000
-#define mmport_c_data_hi_DEFAULT                                                 0x00000000
-#define mmport_d_addr_DEFAULT                                                    0x00000000
-#define mmport_d_data_lo_DEFAULT                                                 0x00000000
-#define mmport_d_data_hi_DEFAULT                                                 0x00000000
-
-
-// addressBlock: gc_utcl2_atcl2dec
-#define mmATC_L2_CNTL_DEFAULT                                                    0x000001c9
-#define mmATC_L2_CNTL2_DEFAULT                                                   0x00000100
-#define mmATC_L2_CACHE_DATA0_DEFAULT                                             0x00000000
-#define mmATC_L2_CACHE_DATA1_DEFAULT                                             0x00000000
-#define mmATC_L2_CACHE_DATA2_DEFAULT                                             0x00000000
-#define mmATC_L2_CNTL3_DEFAULT                                                   0x000001f8
-#define mmATC_L2_STATUS_DEFAULT                                                  0x00000000
-#define mmATC_L2_STATUS2_DEFAULT                                                 0x00000000
-#define mmATC_L2_MISC_CG_DEFAULT                                                 0x00000200
-#define mmATC_L2_MEM_POWER_LS_DEFAULT                                            0x00000208
-#define mmATC_L2_CGTT_CLK_CTRL_DEFAULT                                           0x00000080
-
-
-// addressBlock: gc_utcl2_vml2pfdec
-#define mmVM_L2_CNTL_DEFAULT                                                     0x00080602
-#define mmVM_L2_CNTL2_DEFAULT                                                    0x00000000
-#define mmVM_L2_CNTL3_DEFAULT                                                    0x80100007
-#define mmVM_L2_STATUS_DEFAULT                                                   0x00000000
-#define mmVM_DUMMY_PAGE_FAULT_CNTL_DEFAULT                                       0x00000090
-#define mmVM_DUMMY_PAGE_FAULT_ADDR_LO32_DEFAULT                                  0x00000000
-#define mmVM_DUMMY_PAGE_FAULT_ADDR_HI32_DEFAULT                                  0x00000000
-#define mmVM_L2_PROTECTION_FAULT_CNTL_DEFAULT                                    0x3ffffffc
-#define mmVM_L2_PROTECTION_FAULT_CNTL2_DEFAULT                                   0x000a0000
-#define mmVM_L2_PROTECTION_FAULT_MM_CNTL3_DEFAULT                                0xffffffff
-#define mmVM_L2_PROTECTION_FAULT_MM_CNTL4_DEFAULT                                0xffffffff
-#define mmVM_L2_PROTECTION_FAULT_STATUS_DEFAULT                                  0x00000000
-#define mmVM_L2_PROTECTION_FAULT_ADDR_LO32_DEFAULT                               0x00000000
-#define mmVM_L2_PROTECTION_FAULT_ADDR_HI32_DEFAULT                               0x00000000
-#define mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_DEFAULT                       0x00000000
-#define mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_DEFAULT                       0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_DEFAULT                 0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32_DEFAULT                 0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32_DEFAULT                0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32_DEFAULT                0x00000000
-#define mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32_DEFAULT                    0x00000000
-#define mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32_DEFAULT                    0x00000000
-#define mmVM_L2_CNTL4_DEFAULT                                                    0x000000c1
-#define mmVM_L2_MM_GROUP_RT_CLASSES_DEFAULT                                      0x00000000
-#define mmVM_L2_BANK_SELECT_RESERVED_CID_DEFAULT                                 0x00000000
-#define mmVM_L2_BANK_SELECT_RESERVED_CID2_DEFAULT                                0x00000000
-#define mmVM_L2_CACHE_PARITY_CNTL_DEFAULT                                        0x00000000
-#define mmVM_L2_CGTT_CLK_CTRL_DEFAULT                                            0x00000080
-
-
-// addressBlock: gc_utcl2_vml2vcdec
-#define mmVM_CONTEXT0_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT1_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT2_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT3_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT4_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT5_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT6_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT7_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT8_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT9_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT10_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT11_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT12_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT13_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT14_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT15_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXTS_DISABLE_DEFAULT                                            0x00000000
-#define mmVM_INVALIDATE_ENG0_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG1_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG2_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG3_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG4_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG5_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG6_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG7_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG8_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG9_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG10_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG11_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG12_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG13_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG14_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG15_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG16_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG17_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG0_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG1_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG2_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG3_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG4_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG5_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG6_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG7_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG8_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG9_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG10_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG11_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG12_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG13_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG14_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG15_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG16_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG17_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG0_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG1_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG2_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG3_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG4_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG5_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG6_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG7_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG8_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG9_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG10_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG11_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG12_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG13_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG14_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG15_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG16_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG17_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG1_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG1_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG2_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG2_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG3_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG3_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG4_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG4_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG5_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG5_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG6_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG6_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG7_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG7_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG8_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG8_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG9_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG9_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG10_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG10_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG11_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG11_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG12_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG12_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG13_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG13_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG14_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG14_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG15_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG15_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG16_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG16_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG17_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG17_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-
-
-// addressBlock: gc_utcl2_vmsharedpfdec
-#define mmMC_VM_NB_MMIOBASE_DEFAULT                                              0x00000000
-#define mmMC_VM_NB_MMIOLIMIT_DEFAULT                                             0x00000000
-#define mmMC_VM_NB_PCI_CTRL_DEFAULT                                              0x00000000
-#define mmMC_VM_NB_PCI_ARB_DEFAULT                                               0x00000008
-#define mmMC_VM_NB_TOP_OF_DRAM_SLOT1_DEFAULT                                     0x00000000
-#define mmMC_VM_NB_LOWER_TOP_OF_DRAM2_DEFAULT                                    0x00000000
-#define mmMC_VM_NB_UPPER_TOP_OF_DRAM2_DEFAULT                                    0x00000000
-#define mmMC_VM_FB_OFFSET_DEFAULT                                                0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_DEFAULT                         0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_DEFAULT                         0x00000000
-#define mmMC_VM_STEERING_DEFAULT                                                 0x00000001
-#define mmMC_SHARED_VIRT_RESET_REQ_DEFAULT                                       0x00000000
-#define mmMC_MEM_POWER_LS_DEFAULT                                                0x00000208
-#define mmMC_VM_CACHEABLE_DRAM_ADDRESS_START_DEFAULT                             0x00000000
-#define mmMC_VM_CACHEABLE_DRAM_ADDRESS_END_DEFAULT                               0x00000000
-#define mmMC_VM_APT_CNTL_DEFAULT                                                 0x00000000
-#define mmMC_VM_LOCAL_HBM_ADDRESS_START_DEFAULT                                  0x00000000
-#define mmMC_VM_LOCAL_HBM_ADDRESS_END_DEFAULT                                    0x000fffff
-#define mmMC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL_DEFAULT                              0x00000000
-
-
-// addressBlock: gc_utcl2_vmsharedvcdec
-#define mmMC_VM_FB_LOCATION_BASE_DEFAULT                                         0x00000000
-#define mmMC_VM_FB_LOCATION_TOP_DEFAULT                                          0x00000000
-#define mmMC_VM_AGP_TOP_DEFAULT                                                  0x00000000
-#define mmMC_VM_AGP_BOT_DEFAULT                                                  0x00000000
-#define mmMC_VM_AGP_BASE_DEFAULT                                                 0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_LOW_ADDR_DEFAULT                                 0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR_DEFAULT                                0x00000000
-#define mmMC_VM_MX_L1_TLB_CNTL_DEFAULT                                           0x00002501
-
-
-// addressBlock: gc_ea_gceadec
-#define mmGCEA_DRAM_RD_CLI2GRP_MAP0_DEFAULT                                      0xeaaa9580
-#define mmGCEA_DRAM_RD_CLI2GRP_MAP1_DEFAULT                                      0xeaaa9580
-#define mmGCEA_DRAM_WR_CLI2GRP_MAP0_DEFAULT                                      0xeaaa9580
-#define mmGCEA_DRAM_WR_CLI2GRP_MAP1_DEFAULT                                      0xeaaa9580
-#define mmGCEA_DRAM_RD_GRP2VC_MAP_DEFAULT                                        0x00000924
-#define mmGCEA_DRAM_WR_GRP2VC_MAP_DEFAULT                                        0x00000324
-#define mmGCEA_DRAM_RD_LAZY_DEFAULT                                              0x00000924
-#define mmGCEA_DRAM_WR_LAZY_DEFAULT                                              0x00000924
-#define mmGCEA_DRAM_RD_CAM_CNTL_DEFAULT                                          0x06db3333
-#define mmGCEA_DRAM_WR_CAM_CNTL_DEFAULT                                          0x06db3333
-#define mmGCEA_DRAM_PAGE_BURST_DEFAULT                                           0x20082008
-#define mmGCEA_DRAM_RD_PRI_AGE_DEFAULT                                           0x00db6249
-#define mmGCEA_DRAM_WR_PRI_AGE_DEFAULT                                           0x00db6249
-#define mmGCEA_DRAM_RD_PRI_QUEUING_DEFAULT                                       0x00000db6
-#define mmGCEA_DRAM_WR_PRI_QUEUING_DEFAULT                                       0x00000db6
-#define mmGCEA_DRAM_RD_PRI_FIXED_DEFAULT                                         0x00000924
-#define mmGCEA_DRAM_WR_PRI_FIXED_DEFAULT                                         0x00000924
-#define mmGCEA_DRAM_RD_PRI_URGENCY_DEFAULT                                       0x0000fdb6
-#define mmGCEA_DRAM_WR_PRI_URGENCY_DEFAULT                                       0x0000fdb6
-#define mmGCEA_DRAM_RD_PRI_QUANT_PRI1_DEFAULT                                    0x3f3f3f3f
-#define mmGCEA_DRAM_RD_PRI_QUANT_PRI2_DEFAULT                                    0x7f7f7f7f
-#define mmGCEA_DRAM_RD_PRI_QUANT_PRI3_DEFAULT                                    0xffffffff
-#define mmGCEA_DRAM_WR_PRI_QUANT_PRI1_DEFAULT                                    0x3f3f3f3f
-#define mmGCEA_DRAM_WR_PRI_QUANT_PRI2_DEFAULT                                    0x7f7f7f7f
-#define mmGCEA_DRAM_WR_PRI_QUANT_PRI3_DEFAULT                                    0xffffffff
-#define mmGCEA_ADDRNORM_BASE_ADDR0_DEFAULT                                       0x00000000
-#define mmGCEA_ADDRNORM_LIMIT_ADDR0_DEFAULT                                      0x00000000
-#define mmGCEA_ADDRNORM_BASE_ADDR1_DEFAULT                                       0x00000000
-#define mmGCEA_ADDRNORM_LIMIT_ADDR1_DEFAULT                                      0x00000000
-#define mmGCEA_ADDRNORM_OFFSET_ADDR1_DEFAULT                                     0x00000000
-#define mmGCEA_ADDRNORM_HOLE_CNTL_DEFAULT                                        0x00000000
-#define mmGCEA_ADDRDEC_BANK_CFG_DEFAULT                                          0x000001ef
-#define mmGCEA_ADDRDEC_MISC_CFG_DEFAULT                                          0x3ffff000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_BANK0_DEFAULT                               0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_BANK1_DEFAULT                               0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_BANK2_DEFAULT                               0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_BANK3_DEFAULT                               0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_BANK4_DEFAULT                               0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_PC_DEFAULT                                  0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_PC2_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_CS0_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDECDRAM_ADDR_HASH_CS1_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDECDRAM_HARVEST_ENABLE_DEFAULT                                0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_CS0_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_CS1_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_CS2_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_CS3_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_SECCS0_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_SECCS1_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_SECCS2_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC0_BASE_ADDR_SECCS3_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC0_ADDR_MASK_CS01_DEFAULT                                   0xfffffffe
-#define mmGCEA_ADDRDEC0_ADDR_MASK_CS23_DEFAULT                                   0xfffffffe
-#define mmGCEA_ADDRDEC0_ADDR_MASK_SECCS01_DEFAULT                                0xfffffffe
-#define mmGCEA_ADDRDEC0_ADDR_MASK_SECCS23_DEFAULT                                0xfffffffe
-#define mmGCEA_ADDRDEC0_ADDR_CFG_CS01_DEFAULT                                    0x00050408
-#define mmGCEA_ADDRDEC0_ADDR_CFG_CS23_DEFAULT                                    0x00050408
-#define mmGCEA_ADDRDEC0_ADDR_SEL_CS01_DEFAULT                                    0x04076543
-#define mmGCEA_ADDRDEC0_ADDR_SEL_CS23_DEFAULT                                    0x04076543
-#define mmGCEA_ADDRDEC0_COL_SEL_LO_CS01_DEFAULT                                  0x87654321
-#define mmGCEA_ADDRDEC0_COL_SEL_LO_CS23_DEFAULT                                  0x87654321
-#define mmGCEA_ADDRDEC0_COL_SEL_HI_CS01_DEFAULT                                  0xa9876543
-#define mmGCEA_ADDRDEC0_COL_SEL_HI_CS23_DEFAULT                                  0xa9876543
-#define mmGCEA_ADDRDEC0_RM_SEL_CS01_DEFAULT                                      0x00000000
-#define mmGCEA_ADDRDEC0_RM_SEL_CS23_DEFAULT                                      0x00000000
-#define mmGCEA_ADDRDEC0_RM_SEL_SECCS01_DEFAULT                                   0x00000000
-#define mmGCEA_ADDRDEC0_RM_SEL_SECCS23_DEFAULT                                   0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_CS0_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_CS1_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_CS2_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_CS3_DEFAULT                                    0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_SECCS0_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_SECCS1_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_SECCS2_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC1_BASE_ADDR_SECCS3_DEFAULT                                 0x00000000
-#define mmGCEA_ADDRDEC1_ADDR_MASK_CS01_DEFAULT                                   0xfffffffe
-#define mmGCEA_ADDRDEC1_ADDR_MASK_CS23_DEFAULT                                   0xfffffffe
-#define mmGCEA_ADDRDEC1_ADDR_MASK_SECCS01_DEFAULT                                0xfffffffe
-#define mmGCEA_ADDRDEC1_ADDR_MASK_SECCS23_DEFAULT                                0xfffffffe
-#define mmGCEA_ADDRDEC1_ADDR_CFG_CS01_DEFAULT                                    0x00050408
-#define mmGCEA_ADDRDEC1_ADDR_CFG_CS23_DEFAULT                                    0x00050408
-#define mmGCEA_ADDRDEC1_ADDR_SEL_CS01_DEFAULT                                    0x04076543
-#define mmGCEA_ADDRDEC1_ADDR_SEL_CS23_DEFAULT                                    0x04076543
-#define mmGCEA_ADDRDEC1_COL_SEL_LO_CS01_DEFAULT                                  0x87654321
-#define mmGCEA_ADDRDEC1_COL_SEL_LO_CS23_DEFAULT                                  0x87654321
-#define mmGCEA_ADDRDEC1_COL_SEL_HI_CS01_DEFAULT                                  0xa9876543
-#define mmGCEA_ADDRDEC1_COL_SEL_HI_CS23_DEFAULT                                  0xa9876543
-#define mmGCEA_ADDRDEC1_RM_SEL_CS01_DEFAULT                                      0x00000000
-#define mmGCEA_ADDRDEC1_RM_SEL_CS23_DEFAULT                                      0x00000000
-#define mmGCEA_ADDRDEC1_RM_SEL_SECCS01_DEFAULT                                   0x00000000
-#define mmGCEA_ADDRDEC1_RM_SEL_SECCS23_DEFAULT                                   0x00000000
-#define mmGCEA_IO_RD_CLI2GRP_MAP0_DEFAULT                                        0xeaaa9580
-#define mmGCEA_IO_RD_CLI2GRP_MAP1_DEFAULT                                        0xeaaa9580
-#define mmGCEA_IO_WR_CLI2GRP_MAP0_DEFAULT                                        0xeaaa9580
-#define mmGCEA_IO_WR_CLI2GRP_MAP1_DEFAULT                                        0xeaaa9580
-#define mmGCEA_IO_RD_COMBINE_FLUSH_DEFAULT                                       0x00007777
-#define mmGCEA_IO_WR_COMBINE_FLUSH_DEFAULT                                       0x00007777
-#define mmGCEA_IO_GROUP_BURST_DEFAULT                                            0x1f031f03
-#define mmGCEA_IO_RD_PRI_AGE_DEFAULT                                             0x00db6249
-#define mmGCEA_IO_WR_PRI_AGE_DEFAULT                                             0x00db6249
-#define mmGCEA_IO_RD_PRI_QUEUING_DEFAULT                                         0x00000db6
-#define mmGCEA_IO_WR_PRI_QUEUING_DEFAULT                                         0x00000db6
-#define mmGCEA_IO_RD_PRI_FIXED_DEFAULT                                           0x00000924
-#define mmGCEA_IO_WR_PRI_FIXED_DEFAULT                                           0x00000924
-#define mmGCEA_IO_RD_PRI_URGENCY_DEFAULT                                         0x00000492
-#define mmGCEA_IO_WR_PRI_URGENCY_DEFAULT                                         0x00000492
-#define mmGCEA_IO_RD_PRI_URGENCY_MASK_DEFAULT                                    0xffffffff
-#define mmGCEA_IO_WR_PRI_URGENCY_MASK_DEFAULT                                    0xffffffff
-#define mmGCEA_IO_RD_PRI_QUANT_PRI1_DEFAULT                                      0x3f3f3f3f
-#define mmGCEA_IO_RD_PRI_QUANT_PRI2_DEFAULT                                      0x7f7f7f7f
-#define mmGCEA_IO_RD_PRI_QUANT_PRI3_DEFAULT                                      0xffffffff
-#define mmGCEA_IO_WR_PRI_QUANT_PRI1_DEFAULT                                      0x3f3f3f3f
-#define mmGCEA_IO_WR_PRI_QUANT_PRI2_DEFAULT                                      0x7f7f7f7f
-#define mmGCEA_IO_WR_PRI_QUANT_PRI3_DEFAULT                                      0xffffffff
-#define mmGCEA_SDP_ARB_DRAM_DEFAULT                                              0x00102040
-#define mmGCEA_SDP_ARB_FINAL_DEFAULT                                             0x00007fff
-#define mmGCEA_SDP_DRAM_PRIORITY_DEFAULT                                         0x00000000
-#define mmGCEA_SDP_IO_PRIORITY_DEFAULT                                           0x00000000
-#define mmGCEA_SDP_CREDITS_DEFAULT                                               0x000100bf
-#define mmGCEA_SDP_TAG_RESERVE0_DEFAULT                                          0x00000000
-#define mmGCEA_SDP_TAG_RESERVE1_DEFAULT                                          0x00000000
-#define mmGCEA_SDP_VCC_RESERVE0_DEFAULT                                          0x00000000
-#define mmGCEA_SDP_VCC_RESERVE1_DEFAULT                                          0x00000000
-#define mmGCEA_SDP_VCD_RESERVE0_DEFAULT                                          0x00000000
-#define mmGCEA_SDP_VCD_RESERVE1_DEFAULT                                          0x00000000
-#define mmGCEA_SDP_REQ_CNTL_DEFAULT                                              0x0000000f
-#define mmGCEA_MISC_DEFAULT                                                      0x0de03ff0
-#define mmGCEA_LATENCY_SAMPLING_DEFAULT                                          0x00000000
-#define mmGCEA_PERFCOUNTER_LO_DEFAULT                                            0x00000000
-#define mmGCEA_PERFCOUNTER_HI_DEFAULT                                            0x00000000
-#define mmGCEA_PERFCOUNTER0_CFG_DEFAULT                                          0x00000000
-#define mmGCEA_PERFCOUNTER1_CFG_DEFAULT                                          0x00000000
-#define mmGCEA_PERFCOUNTER_RSLT_CNTL_DEFAULT                                     0x04000000
-
-
-// addressBlock: gc_tcdec
-#define mmTCP_INVALIDATE_DEFAULT                                                 0x00000000
-#define mmTCP_STATUS_DEFAULT                                                     0x00000000
-#define mmTCP_CNTL_DEFAULT                                                       0x2f9c0000
-#define mmTCP_CHAN_STEER_LO_DEFAULT                                              0x76543210
-#define mmTCP_CHAN_STEER_HI_DEFAULT                                              0xfedcba98
-#define mmTCP_ADDR_CONFIG_DEFAULT                                                0x000000f3
-#define mmTCP_CREDIT_DEFAULT                                                     0x804001c0
-#define mmTCP_BUFFER_ADDR_HASH_CNTL_DEFAULT                                      0x00000000
-#define mmTCP_EDC_CNT_DEFAULT                                                    0x00000000
-#define mmTC_CFG_L1_LOAD_POLICY0_DEFAULT                                         0x00000000
-#define mmTC_CFG_L1_LOAD_POLICY1_DEFAULT                                         0x00000000
-#define mmTC_CFG_L1_STORE_POLICY_DEFAULT                                         0x00000000
-#define mmTC_CFG_L2_LOAD_POLICY0_DEFAULT                                         0x00000000
-#define mmTC_CFG_L2_LOAD_POLICY1_DEFAULT                                         0x00000000
-#define mmTC_CFG_L2_STORE_POLICY0_DEFAULT                                        0x00000000
-#define mmTC_CFG_L2_STORE_POLICY1_DEFAULT                                        0x00000000
-#define mmTC_CFG_L2_ATOMIC_POLICY_DEFAULT                                        0x00000000
-#define mmTC_CFG_L1_VOLATILE_DEFAULT                                             0x00000000
-#define mmTC_CFG_L2_VOLATILE_DEFAULT                                             0x00000000
-#define mmTCI_STATUS_DEFAULT                                                     0x00000000
-#define mmTCI_CNTL_1_DEFAULT                                                     0x40080022
-#define mmTCI_CNTL_2_DEFAULT                                                     0x00000041
-#define mmTCC_CTRL_DEFAULT                                                       0xf30fff7f
-#define mmTCC_CTRL2_DEFAULT                                                      0x0000000f
-#define mmTCC_EDC_CNT_DEFAULT                                                    0x00000000
-#define mmTCC_EDC_CNT2_DEFAULT                                                   0x00000000
-#define mmTCC_REDUNDANCY_DEFAULT                                                 0x00000000
-#define mmTCC_EXE_DISABLE_DEFAULT                                                0x00000000
-#define mmTCC_DSM_CNTL_DEFAULT                                                   0x00000000
-#define mmTCC_DSM_CNTLA_DEFAULT                                                  0x00000000
-#define mmTCC_DSM_CNTL2_DEFAULT                                                  0x00000000
-#define mmTCC_DSM_CNTL2A_DEFAULT                                                 0x00000000
-#define mmTCC_DSM_CNTL2B_DEFAULT                                                 0x00000000
-#define mmTCC_WBINVL2_DEFAULT                                                    0x00000010
-#define mmTCC_SOFT_RESET_DEFAULT                                                 0x00000000
-#define mmTCA_CTRL_DEFAULT                                                       0x00000088
-#define mmTCA_BURST_MASK_DEFAULT                                                 0xffffffff
-#define mmTCA_BURST_CTRL_DEFAULT                                                 0x00000007
-#define mmTCA_DSM_CNTL_DEFAULT                                                   0x00000000
-#define mmTCA_DSM_CNTL2_DEFAULT                                                  0x00000000
-#define mmTCA_EDC_CNT_DEFAULT                                                    0x00000000
-
-
-// addressBlock: gc_shdec
-#define mmSPI_SHADER_PGM_RSRC3_PS_DEFAULT                                        0x0000ffff
-#define mmSPI_SHADER_PGM_LO_PS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_HI_PS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_RSRC1_PS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_PGM_RSRC2_PS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_0_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_1_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_2_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_3_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_4_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_5_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_6_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_7_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_8_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_9_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_10_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_11_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_12_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_13_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_14_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_15_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_16_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_17_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_18_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_19_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_20_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_21_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_22_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_23_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_24_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_25_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_26_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_27_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_28_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_29_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_30_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_PS_31_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_PGM_RSRC3_VS_DEFAULT                                        0x0000ffff
-#define mmSPI_SHADER_LATE_ALLOC_VS_DEFAULT                                       0x00000000
-#define mmSPI_SHADER_PGM_LO_VS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_HI_VS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_RSRC1_VS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_PGM_RSRC2_VS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_0_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_1_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_2_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_3_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_4_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_5_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_6_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_7_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_8_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_9_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_10_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_11_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_12_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_13_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_14_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_15_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_16_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_17_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_18_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_19_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_20_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_21_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_22_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_23_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_24_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_25_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_26_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_27_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_28_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_29_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_30_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_VS_31_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_PGM_RSRC2_GS_VS_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_PGM_RSRC4_GS_DEFAULT                                        0x00000800
-#define mmSPI_SHADER_USER_DATA_ADDR_LO_GS_DEFAULT                                0x00000000
-#define mmSPI_SHADER_USER_DATA_ADDR_HI_GS_DEFAULT                                0x00000000
-#define mmSPI_SHADER_PGM_LO_ES_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_HI_ES_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_RSRC3_GS_DEFAULT                                        0x0000fffe
-#define mmSPI_SHADER_PGM_LO_GS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_HI_GS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_RSRC1_GS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_PGM_RSRC2_GS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_0_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_1_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_2_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_3_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_4_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_5_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_6_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_7_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_8_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_9_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_10_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_11_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_12_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_13_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_14_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_15_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_16_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_17_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_18_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_19_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_20_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_21_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_22_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_23_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_24_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_25_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_26_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_27_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_28_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_29_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_30_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_ES_31_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_PGM_RSRC4_HS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_USER_DATA_ADDR_LO_HS_DEFAULT                                0x00000000
-#define mmSPI_SHADER_USER_DATA_ADDR_HI_HS_DEFAULT                                0x00000000
-#define mmSPI_SHADER_PGM_LO_LS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_HI_LS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_RSRC3_HS_DEFAULT                                        0xffff0000
-#define mmSPI_SHADER_PGM_LO_HS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_HI_HS_DEFAULT                                           0x00000000
-#define mmSPI_SHADER_PGM_RSRC1_HS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_PGM_RSRC2_HS_DEFAULT                                        0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_0_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_1_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_2_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_3_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_4_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_5_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_6_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_7_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_8_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_9_DEFAULT                                      0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_10_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_11_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_12_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_13_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_14_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_15_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_16_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_17_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_18_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_19_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_20_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_21_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_22_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_23_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_24_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_25_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_26_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_27_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_28_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_29_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_30_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_LS_31_DEFAULT                                     0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_0_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_1_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_2_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_3_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_4_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_5_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_6_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_7_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_8_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_9_DEFAULT                                  0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_10_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_11_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_12_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_13_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_14_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_15_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_16_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_17_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_18_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_19_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_20_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_21_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_22_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_23_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_24_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_25_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_26_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_27_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_28_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_29_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_30_DEFAULT                                 0x00000000
-#define mmSPI_SHADER_USER_DATA_COMMON_31_DEFAULT                                 0x00000000
-#define mmCOMPUTE_DISPATCH_INITIATOR_DEFAULT                                     0x00000000
-#define mmCOMPUTE_DIM_X_DEFAULT                                                  0x00000000
-#define mmCOMPUTE_DIM_Y_DEFAULT                                                  0x00000000
-#define mmCOMPUTE_DIM_Z_DEFAULT                                                  0x00000000
-#define mmCOMPUTE_START_X_DEFAULT                                                0x00000000
-#define mmCOMPUTE_START_Y_DEFAULT                                                0x00000000
-#define mmCOMPUTE_START_Z_DEFAULT                                                0x00000000
-#define mmCOMPUTE_NUM_THREAD_X_DEFAULT                                           0x00000000
-#define mmCOMPUTE_NUM_THREAD_Y_DEFAULT                                           0x00000000
-#define mmCOMPUTE_NUM_THREAD_Z_DEFAULT                                           0x00000000
-#define mmCOMPUTE_PIPELINESTAT_ENABLE_DEFAULT                                    0x00000001
-#define mmCOMPUTE_PERFCOUNT_ENABLE_DEFAULT                                       0x00000000
-#define mmCOMPUTE_PGM_LO_DEFAULT                                                 0x00000000
-#define mmCOMPUTE_PGM_HI_DEFAULT                                                 0x00000000
-#define mmCOMPUTE_DISPATCH_PKT_ADDR_LO_DEFAULT                                   0x00000000
-#define mmCOMPUTE_DISPATCH_PKT_ADDR_HI_DEFAULT                                   0x00000000
-#define mmCOMPUTE_DISPATCH_SCRATCH_BASE_LO_DEFAULT                               0x00000000
-#define mmCOMPUTE_DISPATCH_SCRATCH_BASE_HI_DEFAULT                               0x00000000
-#define mmCOMPUTE_PGM_RSRC1_DEFAULT                                              0x00000000
-#define mmCOMPUTE_PGM_RSRC2_DEFAULT                                              0x00000000
-#define mmCOMPUTE_VMID_DEFAULT                                                   0x00000000
-#define mmCOMPUTE_RESOURCE_LIMITS_DEFAULT                                        0x00000000
-#define mmCOMPUTE_STATIC_THREAD_MGMT_SE0_DEFAULT                                 0xffffffff
-#define mmCOMPUTE_STATIC_THREAD_MGMT_SE1_DEFAULT                                 0xffffffff
-#define mmCOMPUTE_TMPRING_SIZE_DEFAULT                                           0x00000000
-#define mmCOMPUTE_STATIC_THREAD_MGMT_SE2_DEFAULT                                 0xffffffff
-#define mmCOMPUTE_STATIC_THREAD_MGMT_SE3_DEFAULT                                 0xffffffff
-#define mmCOMPUTE_RESTART_X_DEFAULT                                              0x00000000
-#define mmCOMPUTE_RESTART_Y_DEFAULT                                              0x00000000
-#define mmCOMPUTE_RESTART_Z_DEFAULT                                              0x00000000
-#define mmCOMPUTE_THREAD_TRACE_ENABLE_DEFAULT                                    0x00000000
-#define mmCOMPUTE_MISC_RESERVED_DEFAULT                                          0x00000002
-#define mmCOMPUTE_DISPATCH_ID_DEFAULT                                            0x00000000
-#define mmCOMPUTE_THREADGROUP_ID_DEFAULT                                         0x00000000
-#define mmCOMPUTE_RELAUNCH_DEFAULT                                               0x00000000
-#define mmCOMPUTE_WAVE_RESTORE_ADDR_LO_DEFAULT                                   0x00000000
-#define mmCOMPUTE_WAVE_RESTORE_ADDR_HI_DEFAULT                                   0x00000000
-#define mmCOMPUTE_USER_DATA_0_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_1_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_2_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_3_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_4_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_5_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_6_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_7_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_8_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_9_DEFAULT                                            0x00000000
-#define mmCOMPUTE_USER_DATA_10_DEFAULT                                           0x00000000
-#define mmCOMPUTE_USER_DATA_11_DEFAULT                                           0x00000000
-#define mmCOMPUTE_USER_DATA_12_DEFAULT                                           0x00000000
-#define mmCOMPUTE_USER_DATA_13_DEFAULT                                           0x00000000
-#define mmCOMPUTE_USER_DATA_14_DEFAULT                                           0x00000000
-#define mmCOMPUTE_USER_DATA_15_DEFAULT                                           0x00000000
-#define mmCOMPUTE_NOWHERE_DEFAULT                                                0x00000000
-
-
-// addressBlock: gc_cppdec
-#define mmCP_DFY_CNTL_DEFAULT                                                    0x00000000
-#define mmCP_DFY_STAT_DEFAULT                                                    0x00000000
-#define mmCP_DFY_ADDR_HI_DEFAULT                                                 0x00000000
-#define mmCP_DFY_ADDR_LO_DEFAULT                                                 0x00000000
-#define mmCP_DFY_DATA_0_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_1_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_2_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_3_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_4_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_5_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_6_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_7_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_8_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_9_DEFAULT                                                  0x00000000
-#define mmCP_DFY_DATA_10_DEFAULT                                                 0x00000000
-#define mmCP_DFY_DATA_11_DEFAULT                                                 0x00000000
-#define mmCP_DFY_DATA_12_DEFAULT                                                 0x00000000
-#define mmCP_DFY_DATA_13_DEFAULT                                                 0x00000000
-#define mmCP_DFY_DATA_14_DEFAULT                                                 0x00000000
-#define mmCP_DFY_DATA_15_DEFAULT                                                 0x00000000
-#define mmCP_DFY_CMD_DEFAULT                                                     0x00000000
-#define mmCP_EOPQ_WAIT_TIME_DEFAULT                                              0x0000052c
-#define mmCP_CPC_MGCG_SYNC_CNTL_DEFAULT                                          0x00001020
-#define mmCPC_INT_INFO_DEFAULT                                                   0x00000000
-#define mmCP_VIRT_STATUS_DEFAULT                                                 0x00000000
-#define mmCPC_INT_ADDR_DEFAULT                                                   0x00000000
-#define mmCPC_INT_PASID_DEFAULT                                                  0x00000000
-#define mmCP_GFX_ERROR_DEFAULT                                                   0x00000000
-#define mmCPG_UTCL1_CNTL_DEFAULT                                                 0x00000080
-#define mmCPC_UTCL1_CNTL_DEFAULT                                                 0x00000080
-#define mmCPF_UTCL1_CNTL_DEFAULT                                                 0x00000080
-#define mmCP_AQL_SMM_STATUS_DEFAULT                                              0x00000000
-#define mmCP_RB0_BASE_DEFAULT                                                    0x00000000
-#define mmCP_RB_BASE_DEFAULT                                                     0x00000000
-#define mmCP_RB0_CNTL_DEFAULT                                                    0x00400000
-#define mmCP_RB_CNTL_DEFAULT                                                     0x00400000
-#define mmCP_RB_RPTR_WR_DEFAULT                                                  0x00000000
-#define mmCP_RB0_RPTR_ADDR_DEFAULT                                               0x00000000
-#define mmCP_RB_RPTR_ADDR_DEFAULT                                                0x00000000
-#define mmCP_RB0_RPTR_ADDR_HI_DEFAULT                                            0x00000000
-#define mmCP_RB_RPTR_ADDR_HI_DEFAULT                                             0x00000000
-#define mmCP_RB0_BUFSZ_MASK_DEFAULT                                              0x00000000
-#define mmCP_RB_BUFSZ_MASK_DEFAULT                                               0x00000000
-#define mmCP_RB_WPTR_POLL_ADDR_LO_DEFAULT                                        0x00000000
-#define mmCP_RB_WPTR_POLL_ADDR_HI_DEFAULT                                        0x00000000
-#define mmGC_PRIV_MODE_DEFAULT                                                   0x00000000
-#define mmCP_INT_CNTL_DEFAULT                                                    0x00000000
-#define mmCP_INT_STATUS_DEFAULT                                                  0x00000000
-#define mmCP_DEVICE_ID_DEFAULT                                                   0x00000000
-#define mmCP_ME0_PIPE_PRIORITY_CNTS_DEFAULT                                      0x08081020
-#define mmCP_RING_PRIORITY_CNTS_DEFAULT                                          0x08081020
-#define mmCP_ME0_PIPE0_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_RING0_PRIORITY_DEFAULT                                              0x00000002
-#define mmCP_ME0_PIPE1_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_RING1_PRIORITY_DEFAULT                                              0x00000002
-#define mmCP_ME0_PIPE2_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_RING2_PRIORITY_DEFAULT                                              0x00000002
-#define mmCP_FATAL_ERROR_DEFAULT                                                 0x00000000
-#define mmCP_RB_VMID_DEFAULT                                                     0x00000000
-#define mmCP_ME0_PIPE0_VMID_DEFAULT                                              0x00000000
-#define mmCP_ME0_PIPE1_VMID_DEFAULT                                              0x00000000
-#define mmCP_RB0_WPTR_DEFAULT                                                    0x00000000
-#define mmCP_RB_WPTR_DEFAULT                                                     0x00000000
-#define mmCP_RB0_WPTR_HI_DEFAULT                                                 0x00000000
-#define mmCP_RB_WPTR_HI_DEFAULT                                                  0x00000000
-#define mmCP_RB1_WPTR_DEFAULT                                                    0x00000000
-#define mmCP_RB1_WPTR_HI_DEFAULT                                                 0x00000000
-#define mmCP_RB2_WPTR_DEFAULT                                                    0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_DEFAULT                                         0x00000000
-#define mmCP_RB_DOORBELL_RANGE_LOWER_DEFAULT                                     0x00000000
-#define mmCP_RB_DOORBELL_RANGE_UPPER_DEFAULT                                     0x00000044
-#define mmCP_MEC_DOORBELL_RANGE_LOWER_DEFAULT                                    0x00000048
-#define mmCP_MEC_DOORBELL_RANGE_UPPER_DEFAULT                                    0x0ffffffc
-#define mmCPG_UTCL1_ERROR_DEFAULT                                                0x00000000
-#define mmCPC_UTCL1_ERROR_DEFAULT                                                0x00000000
-#define mmCP_RB1_BASE_DEFAULT                                                    0x00000000
-#define mmCP_RB1_CNTL_DEFAULT                                                    0x00400000
-#define mmCP_RB1_RPTR_ADDR_DEFAULT                                               0x00000000
-#define mmCP_RB1_RPTR_ADDR_HI_DEFAULT                                            0x00000000
-#define mmCP_RB2_BASE_DEFAULT                                                    0x00000000
-#define mmCP_RB2_CNTL_DEFAULT                                                    0x00400000
-#define mmCP_RB2_RPTR_ADDR_DEFAULT                                               0x00000000
-#define mmCP_RB2_RPTR_ADDR_HI_DEFAULT                                            0x00000000
-#define mmCP_RB0_ACTIVE_DEFAULT                                                  0x00000001
-#define mmCP_RB_ACTIVE_DEFAULT                                                   0x00000001
-#define mmCP_INT_CNTL_RING0_DEFAULT                                              0x00000000
-#define mmCP_INT_CNTL_RING1_DEFAULT                                              0x00000000
-#define mmCP_INT_CNTL_RING2_DEFAULT                                              0x00000000
-#define mmCP_INT_STATUS_RING0_DEFAULT                                            0x00000000
-#define mmCP_INT_STATUS_RING1_DEFAULT                                            0x00000000
-#define mmCP_INT_STATUS_RING2_DEFAULT                                            0x00000000
-#define mmCP_PWR_CNTL_DEFAULT                                                    0x00000000
-#define mmCP_MEM_SLP_CNTL_DEFAULT                                                0x00020200
-#define mmCP_ECC_FIRSTOCCURRENCE_DEFAULT                                         0x00000000
-#define mmCP_ECC_FIRSTOCCURRENCE_RING0_DEFAULT                                   0x00000000
-#define mmCP_ECC_FIRSTOCCURRENCE_RING1_DEFAULT                                   0x00000000
-#define mmCP_ECC_FIRSTOCCURRENCE_RING2_DEFAULT                                   0x00000000
-#define mmGB_EDC_MODE_DEFAULT                                                    0x00000000
-#define mmCP_PQ_WPTR_POLL_CNTL_DEFAULT                                           0x00000001
-#define mmCP_PQ_WPTR_POLL_CNTL1_DEFAULT                                          0x00000000
-#define mmCP_ME1_PIPE0_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME1_PIPE1_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME1_PIPE2_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME1_PIPE3_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME2_PIPE0_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME2_PIPE1_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME2_PIPE2_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME2_PIPE3_INT_CNTL_DEFAULT                                          0x00000000
-#define mmCP_ME1_PIPE0_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCP_ME1_PIPE1_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCP_ME1_PIPE2_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCP_ME1_PIPE3_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCP_ME2_PIPE0_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCP_ME2_PIPE1_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCP_ME2_PIPE2_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCP_ME2_PIPE3_INT_STATUS_DEFAULT                                        0x00000000
-#define mmCC_GC_EDC_CONFIG_DEFAULT                                               0x00000000
-#define mmCP_ME1_PIPE_PRIORITY_CNTS_DEFAULT                                      0x08081020
-#define mmCP_ME1_PIPE0_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_ME1_PIPE1_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_ME1_PIPE2_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_ME1_PIPE3_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_ME2_PIPE_PRIORITY_CNTS_DEFAULT                                      0x08081020
-#define mmCP_ME2_PIPE0_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_ME2_PIPE1_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_ME2_PIPE2_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_ME2_PIPE3_PRIORITY_DEFAULT                                          0x00000002
-#define mmCP_CE_PRGRM_CNTR_START_DEFAULT                                         0x00000000
-#define mmCP_PFP_PRGRM_CNTR_START_DEFAULT                                        0x00000000
-#define mmCP_ME_PRGRM_CNTR_START_DEFAULT                                         0x00000000
-#define mmCP_MEC1_PRGRM_CNTR_START_DEFAULT                                       0x00000000
-#define mmCP_MEC2_PRGRM_CNTR_START_DEFAULT                                       0x00000000
-#define mmCP_CE_INTR_ROUTINE_START_DEFAULT                                       0x00000002
-#define mmCP_PFP_INTR_ROUTINE_START_DEFAULT                                      0x00000002
-#define mmCP_ME_INTR_ROUTINE_START_DEFAULT                                       0x00000002
-#define mmCP_MEC1_INTR_ROUTINE_START_DEFAULT                                     0x00000002
-#define mmCP_MEC2_INTR_ROUTINE_START_DEFAULT                                     0x00000002
-#define mmCP_CONTEXT_CNTL_DEFAULT                                                0x00750075
-#define mmCP_MAX_CONTEXT_DEFAULT                                                 0x00000007
-#define mmCP_IQ_WAIT_TIME1_DEFAULT                                               0x40404040
-#define mmCP_IQ_WAIT_TIME2_DEFAULT                                               0x40404040
-#define mmCP_RB0_BASE_HI_DEFAULT                                                 0x00000000
-#define mmCP_RB1_BASE_HI_DEFAULT                                                 0x00000000
-#define mmCP_VMID_RESET_DEFAULT                                                  0x00000000
-#define mmCPC_INT_CNTL_DEFAULT                                                   0x00000000
-#define mmCPC_INT_STATUS_DEFAULT                                                 0x00000000
-#define mmCP_VMID_PREEMPT_DEFAULT                                                0x00000000
-#define mmCPC_INT_CNTX_ID_DEFAULT                                                0x00000000
-#define mmCP_PQ_STATUS_DEFAULT                                                   0x00000000
-#define mmCP_CPC_IC_BASE_LO_DEFAULT                                              0x00000000
-#define mmCP_CPC_IC_BASE_HI_DEFAULT                                              0x00000000
-#define mmCP_CPC_IC_BASE_CNTL_DEFAULT                                            0x00000000
-#define mmCP_CPC_IC_OP_CNTL_DEFAULT                                              0x00000000
-#define mmCP_MEC1_F32_INT_DIS_DEFAULT                                            0x00000000
-#define mmCP_MEC2_F32_INT_DIS_DEFAULT                                            0x00000000
-#define mmCP_VMID_STATUS_DEFAULT                                                 0x00000000
-
-
-// addressBlock: gc_cppdec2
-#define mmCP_RB_DOORBELL_CONTROL_SCH_0_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_SCH_1_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_SCH_2_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_SCH_3_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_SCH_4_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_SCH_5_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_SCH_6_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CONTROL_SCH_7_DEFAULT                                   0x00000000
-#define mmCP_RB_DOORBELL_CLEAR_DEFAULT                                           0x00000000
-#define mmCP_GFX_MQD_CONTROL_DEFAULT                                             0x00000100
-#define mmCP_GFX_MQD_BASE_ADDR_DEFAULT                                           0x00000000
-#define mmCP_GFX_MQD_BASE_ADDR_HI_DEFAULT                                        0x00000000
-#define mmCP_RB_STATUS_DEFAULT                                                   0x00000000
-#define mmCPG_UTCL1_STATUS_DEFAULT                                               0x00000000
-#define mmCPC_UTCL1_STATUS_DEFAULT                                               0x00000000
-#define mmCPF_UTCL1_STATUS_DEFAULT                                               0x00000000
-#define mmCP_SD_CNTL_DEFAULT                                                     0x0000001f
-#define mmCP_SOFT_RESET_CNTL_DEFAULT                                             0x00000000
-#define mmCP_CPC_GFX_CNTL_DEFAULT                                                0x00000000
-
-
-// addressBlock: gc_spipdec
-#define mmSPI_ARB_PRIORITY_DEFAULT                                               0x00000000
-#define mmSPI_ARB_CYCLES_0_DEFAULT                                               0x00000000
-#define mmSPI_ARB_CYCLES_1_DEFAULT                                               0x00000000
-#define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT                                       0x07ffffff
-#define mmSPI_WCL_PIPE_PERCENT_HP3D_DEFAULT                                      0x07c1f07f
-#define mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT                                       0x0000007f
-#define mmSPI_WCL_PIPE_PERCENT_CS1_DEFAULT                                       0x0000007f
-#define mmSPI_WCL_PIPE_PERCENT_CS2_DEFAULT                                       0x0000007f
-#define mmSPI_WCL_PIPE_PERCENT_CS3_DEFAULT                                       0x0000007f
-#define mmSPI_WCL_PIPE_PERCENT_CS4_DEFAULT                                       0x0000007f
-#define mmSPI_WCL_PIPE_PERCENT_CS5_DEFAULT                                       0x0000007f
-#define mmSPI_WCL_PIPE_PERCENT_CS6_DEFAULT                                       0x0000007f
-#define mmSPI_WCL_PIPE_PERCENT_CS7_DEFAULT                                       0x0000007f
-#define mmSPI_COMPUTE_QUEUE_RESET_DEFAULT                                        0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_0_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_1_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_2_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_3_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_4_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_5_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_6_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_7_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_8_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_9_DEFAULT                                      0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_0_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_1_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_2_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_3_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_4_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_5_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_6_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_7_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_8_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_9_DEFAULT                                   0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_10_DEFAULT                                     0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_11_DEFAULT                                     0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_10_DEFAULT                                  0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_11_DEFAULT                                  0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_12_DEFAULT                                     0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_13_DEFAULT                                     0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_14_DEFAULT                                     0x00000000
-#define mmSPI_RESOURCE_RESERVE_CU_15_DEFAULT                                     0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_12_DEFAULT                                  0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_13_DEFAULT                                  0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_14_DEFAULT                                  0x00000000
-#define mmSPI_RESOURCE_RESERVE_EN_CU_15_DEFAULT                                  0x00000000
-#define mmSPI_COMPUTE_WF_CTX_SAVE_DEFAULT                                        0x00000000
-#define mmSPI_ARB_CNTL_0_DEFAULT                                                 0x00000000
-
-
-// addressBlock: gc_cpphqddec
-#define mmCP_HQD_GFX_CONTROL_DEFAULT                                             0x00000000
-#define mmCP_HQD_GFX_STATUS_DEFAULT                                              0x00000000
-#define mmCP_HPD_ROQ_OFFSETS_DEFAULT                                             0x00200604
-#define mmCP_HPD_STATUS0_DEFAULT                                                 0x01000000
-#define mmCP_HPD_UTCL1_CNTL_DEFAULT                                              0x00000000
-#define mmCP_HPD_UTCL1_ERROR_DEFAULT                                             0x00000000
-#define mmCP_HPD_UTCL1_ERROR_ADDR_DEFAULT                                        0x00000000
-#define mmCP_MQD_BASE_ADDR_DEFAULT                                               0x00000000
-#define mmCP_MQD_BASE_ADDR_HI_DEFAULT                                            0x00000000
-#define mmCP_HQD_ACTIVE_DEFAULT                                                  0x00000000
-#define mmCP_HQD_VMID_DEFAULT                                                    0x00000000
-#define mmCP_HQD_PERSISTENT_STATE_DEFAULT                                        0x0be05301
-#define mmCP_HQD_PIPE_PRIORITY_DEFAULT                                           0x00000000
-#define mmCP_HQD_QUEUE_PRIORITY_DEFAULT                                          0x00000000
-#define mmCP_HQD_QUANTUM_DEFAULT                                                 0x00000000
-#define mmCP_HQD_PQ_BASE_DEFAULT                                                 0x00000000
-#define mmCP_HQD_PQ_BASE_HI_DEFAULT                                              0x00000000
-#define mmCP_HQD_PQ_RPTR_DEFAULT                                                 0x00000000
-#define mmCP_HQD_PQ_RPTR_REPORT_ADDR_DEFAULT                                     0x00000000
-#define mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI_DEFAULT                                  0x00000000
-#define mmCP_HQD_PQ_WPTR_POLL_ADDR_DEFAULT                                       0x00000000
-#define mmCP_HQD_PQ_WPTR_POLL_ADDR_HI_DEFAULT                                    0x00000000
-#define mmCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT                                     0x00000000
-#define mmCP_HQD_PQ_CONTROL_DEFAULT                                              0x00308509
-#define mmCP_HQD_IB_BASE_ADDR_DEFAULT                                            0x00000000
-#define mmCP_HQD_IB_BASE_ADDR_HI_DEFAULT                                         0x00000000
-#define mmCP_HQD_IB_RPTR_DEFAULT                                                 0x00000000
-#define mmCP_HQD_IB_CONTROL_DEFAULT                                              0x00300000
-#define mmCP_HQD_IQ_TIMER_DEFAULT                                                0x00000000
-#define mmCP_HQD_IQ_RPTR_DEFAULT                                                 0x00000000
-#define mmCP_HQD_DEQUEUE_REQUEST_DEFAULT                                         0x00000000
-#define mmCP_HQD_DMA_OFFLOAD_DEFAULT                                             0x00000000
-#define mmCP_HQD_OFFLOAD_DEFAULT                                                 0x00000000
-#define mmCP_HQD_SEMA_CMD_DEFAULT                                                0x00000000
-#define mmCP_HQD_MSG_TYPE_DEFAULT                                                0x00000000
-#define mmCP_HQD_ATOMIC0_PREOP_LO_DEFAULT                                        0x00000000
-#define mmCP_HQD_ATOMIC0_PREOP_HI_DEFAULT                                        0x00000000
-#define mmCP_HQD_ATOMIC1_PREOP_LO_DEFAULT                                        0x00000000
-#define mmCP_HQD_ATOMIC1_PREOP_HI_DEFAULT                                        0x00000000
-#define mmCP_HQD_HQ_SCHEDULER0_DEFAULT                                           0x00000000
-#define mmCP_HQD_HQ_STATUS0_DEFAULT                                              0x40000000
-#define mmCP_HQD_HQ_CONTROL0_DEFAULT                                             0x00000000
-#define mmCP_HQD_HQ_SCHEDULER1_DEFAULT                                           0x00000000
-#define mmCP_MQD_CONTROL_DEFAULT                                                 0x00000100
-#define mmCP_HQD_HQ_STATUS1_DEFAULT                                              0x00000000
-#define mmCP_HQD_HQ_CONTROL1_DEFAULT                                             0x00000000
-#define mmCP_HQD_EOP_BASE_ADDR_DEFAULT                                           0x00000000
-#define mmCP_HQD_EOP_BASE_ADDR_HI_DEFAULT                                        0x00000000
-#define mmCP_HQD_EOP_CONTROL_DEFAULT                                             0x00000006
-#define mmCP_HQD_EOP_RPTR_DEFAULT                                                0x40000000
-#define mmCP_HQD_EOP_WPTR_DEFAULT                                                0x007f8000
-#define mmCP_HQD_EOP_EVENTS_DEFAULT                                              0x00000000
-#define mmCP_HQD_CTX_SAVE_BASE_ADDR_LO_DEFAULT                                   0x00000000
-#define mmCP_HQD_CTX_SAVE_BASE_ADDR_HI_DEFAULT                                   0x00000000
-#define mmCP_HQD_CTX_SAVE_CONTROL_DEFAULT                                        0x00000000
-#define mmCP_HQD_CNTL_STACK_OFFSET_DEFAULT                                       0x00000000
-#define mmCP_HQD_CNTL_STACK_SIZE_DEFAULT                                         0x00000000
-#define mmCP_HQD_WG_STATE_OFFSET_DEFAULT                                         0x00000000
-#define mmCP_HQD_CTX_SAVE_SIZE_DEFAULT                                           0x00000000
-#define mmCP_HQD_GDS_RESOURCE_STATE_DEFAULT                                      0x00000000
-#define mmCP_HQD_ERROR_DEFAULT                                                   0x00000000
-#define mmCP_HQD_EOP_WPTR_MEM_DEFAULT                                            0x00000000
-#define mmCP_HQD_AQL_CONTROL_DEFAULT                                             0x00000000
-#define mmCP_HQD_PQ_WPTR_LO_DEFAULT                                              0x00000000
-#define mmCP_HQD_PQ_WPTR_HI_DEFAULT                                              0x00000000
-
-
-// addressBlock: gc_didtdec
-#define mmDIDT_IND_INDEX_DEFAULT                                                 0x00000000
-#define mmDIDT_IND_DATA_DEFAULT                                                  0x00000000
-
-
-// addressBlock: gc_gccacdec
-#define mmGC_CAC_CTRL_1_DEFAULT                                                  0x01000000
-#define mmGC_CAC_CTRL_2_DEFAULT                                                  0x00000000
-#define mmGC_CAC_CGTT_CLK_CTRL_DEFAULT                                           0x00000100
-#define mmGC_CAC_AGGR_LOWER_DEFAULT                                              0x00000000
-#define mmGC_CAC_AGGR_UPPER_DEFAULT                                              0x00000000
-#define mmGC_CAC_PG_AGGR_LOWER_DEFAULT                                           0x00000000
-#define mmGC_CAC_PG_AGGR_UPPER_DEFAULT                                           0x00000000
-#define mmGC_CAC_SOFT_CTRL_DEFAULT                                               0x00000000
-#define mmGC_DIDT_CTRL0_DEFAULT                                                  0x00000000
-#define mmGC_DIDT_CTRL1_DEFAULT                                                  0xffff0000
-#define mmGC_DIDT_CTRL2_DEFAULT                                                  0x1880000f
-#define mmGC_DIDT_WEIGHT_DEFAULT                                                 0x00000000
-#define mmGC_EDC_CTRL_DEFAULT                                                    0x00000000
-#define mmGC_EDC_THRESHOLD_DEFAULT                                               0x00000000
-#define mmGC_EDC_STATUS_DEFAULT                                                  0x00000000
-#define mmGC_EDC_OVERFLOW_DEFAULT                                                0x00000000
-#define mmGC_EDC_ROLLING_POWER_DELTA_DEFAULT                                     0x00000000
-#define mmGC_DIDT_DROOP_CTRL_DEFAULT                                             0x00000000
-#define mmGC_EDC_DROOP_CTRL_DEFAULT                                              0x00100000
-#define mmGC_CAC_IND_INDEX_DEFAULT                                               0x00000000
-#define mmGC_CAC_IND_DATA_DEFAULT                                                0x00000000
-#define mmSE_CAC_CGTT_CLK_CTRL_DEFAULT                                           0x00000100
-#define mmSE_CAC_IND_INDEX_DEFAULT                                               0x00000000
-#define mmSE_CAC_IND_DATA_DEFAULT                                                0x00000000
-
-
-// addressBlock: gc_tcpdec
-#define mmTCP_WATCH0_ADDR_H_DEFAULT                                              0x00000000
-#define mmTCP_WATCH0_ADDR_L_DEFAULT                                              0x00000000
-#define mmTCP_WATCH0_CNTL_DEFAULT                                                0x00000000
-#define mmTCP_WATCH1_ADDR_H_DEFAULT                                              0x00000000
-#define mmTCP_WATCH1_ADDR_L_DEFAULT                                              0x00000000
-#define mmTCP_WATCH1_CNTL_DEFAULT                                                0x00000000
-#define mmTCP_WATCH2_ADDR_H_DEFAULT                                              0x00000000
-#define mmTCP_WATCH2_ADDR_L_DEFAULT                                              0x00000000
-#define mmTCP_WATCH2_CNTL_DEFAULT                                                0x00000000
-#define mmTCP_WATCH3_ADDR_H_DEFAULT                                              0x00000000
-#define mmTCP_WATCH3_ADDR_L_DEFAULT                                              0x00000000
-#define mmTCP_WATCH3_CNTL_DEFAULT                                                0x00000000
-#define mmTCP_GATCL1_CNTL_DEFAULT                                                0x00000000
-#define mmTCP_ATC_EDC_GATCL1_CNT_DEFAULT                                         0x00000000
-#define mmTCP_GATCL1_DSM_CNTL_DEFAULT                                            0x00000000
-#define mmTCP_CNTL2_DEFAULT                                                      0x0000000a
-#define mmTCP_UTCL1_CNTL1_DEFAULT                                                0x00800400
-#define mmTCP_UTCL1_CNTL2_DEFAULT                                                0x00000000
-#define mmTCP_UTCL1_STATUS_DEFAULT                                               0x00000000
-#define mmTCP_PERFCOUNTER_FILTER_DEFAULT                                         0x00000000
-#define mmTCP_PERFCOUNTER_FILTER_EN_DEFAULT                                      0x00000000
-
-
-// addressBlock: gc_gdspdec
-#define mmGDS_VMID0_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID0_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID1_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID1_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID2_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID2_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID3_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID3_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID4_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID4_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID5_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID5_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID6_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID6_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID7_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID7_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID8_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID8_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID9_BASE_DEFAULT                                                 0x00000000
-#define mmGDS_VMID9_SIZE_DEFAULT                                                 0x00010000
-#define mmGDS_VMID10_BASE_DEFAULT                                                0x00000000
-#define mmGDS_VMID10_SIZE_DEFAULT                                                0x00010000
-#define mmGDS_VMID11_BASE_DEFAULT                                                0x00000000
-#define mmGDS_VMID11_SIZE_DEFAULT                                                0x00010000
-#define mmGDS_VMID12_BASE_DEFAULT                                                0x00000000
-#define mmGDS_VMID12_SIZE_DEFAULT                                                0x00010000
-#define mmGDS_VMID13_BASE_DEFAULT                                                0x00000000
-#define mmGDS_VMID13_SIZE_DEFAULT                                                0x00010000
-#define mmGDS_VMID14_BASE_DEFAULT                                                0x00000000
-#define mmGDS_VMID14_SIZE_DEFAULT                                                0x00010000
-#define mmGDS_VMID15_BASE_DEFAULT                                                0x00000000
-#define mmGDS_VMID15_SIZE_DEFAULT                                                0x00010000
-#define mmGDS_GWS_VMID0_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID1_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID2_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID3_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID4_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID5_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID6_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID7_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID8_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID9_DEFAULT                                                  0x00400000
-#define mmGDS_GWS_VMID10_DEFAULT                                                 0x00400000
-#define mmGDS_GWS_VMID11_DEFAULT                                                 0x00400000
-#define mmGDS_GWS_VMID12_DEFAULT                                                 0x00400000
-#define mmGDS_GWS_VMID13_DEFAULT                                                 0x00400000
-#define mmGDS_GWS_VMID14_DEFAULT                                                 0x00400000
-#define mmGDS_GWS_VMID15_DEFAULT                                                 0x00400000
-#define mmGDS_OA_VMID0_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID1_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID2_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID3_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID4_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID5_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID6_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID7_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID8_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID9_DEFAULT                                                   0x00000000
-#define mmGDS_OA_VMID10_DEFAULT                                                  0x00000000
-#define mmGDS_OA_VMID11_DEFAULT                                                  0x00000000
-#define mmGDS_OA_VMID12_DEFAULT                                                  0x00000000
-#define mmGDS_OA_VMID13_DEFAULT                                                  0x00000000
-#define mmGDS_OA_VMID14_DEFAULT                                                  0x00000000
-#define mmGDS_OA_VMID15_DEFAULT                                                  0x00000000
-#define mmGDS_GWS_RESET0_DEFAULT                                                 0x00000000
-#define mmGDS_GWS_RESET1_DEFAULT                                                 0x00000000
-#define mmGDS_GWS_RESOURCE_RESET_DEFAULT                                         0x00000000
-#define mmGDS_COMPUTE_MAX_WAVE_ID_DEFAULT                                        0x0000015f
-#define mmGDS_OA_RESET_MASK_DEFAULT                                              0x00000000
-#define mmGDS_OA_RESET_DEFAULT                                                   0x00000000
-#define mmGDS_ENHANCE_DEFAULT                                                    0x00000000
-#define mmGDS_OA_CGPG_RESTORE_DEFAULT                                            0x00000000
-#define mmGDS_CS_CTXSW_STATUS_DEFAULT                                            0x00000000
-#define mmGDS_CS_CTXSW_CNT0_DEFAULT                                              0x00000000
-#define mmGDS_CS_CTXSW_CNT1_DEFAULT                                              0x00000000
-#define mmGDS_CS_CTXSW_CNT2_DEFAULT                                              0x00000000
-#define mmGDS_CS_CTXSW_CNT3_DEFAULT                                              0x00000000
-#define mmGDS_GFX_CTXSW_STATUS_DEFAULT                                           0x00000000
-#define mmGDS_VS_CTXSW_CNT0_DEFAULT                                              0x00000000
-#define mmGDS_VS_CTXSW_CNT1_DEFAULT                                              0x00000000
-#define mmGDS_VS_CTXSW_CNT2_DEFAULT                                              0x00000000
-#define mmGDS_VS_CTXSW_CNT3_DEFAULT                                              0x00000000
-#define mmGDS_PS0_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS0_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS0_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS0_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_PS1_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS1_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS1_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS1_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_PS2_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS2_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS2_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS2_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_PS3_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS3_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS3_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS3_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_PS4_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS4_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS4_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS4_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_PS5_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS5_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS5_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS5_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_PS6_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS6_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS6_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS6_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_PS7_CTXSW_CNT0_DEFAULT                                             0x00000000
-#define mmGDS_PS7_CTXSW_CNT1_DEFAULT                                             0x00000000
-#define mmGDS_PS7_CTXSW_CNT2_DEFAULT                                             0x00000000
-#define mmGDS_PS7_CTXSW_CNT3_DEFAULT                                             0x00000000
-#define mmGDS_GS_CTXSW_CNT0_DEFAULT                                              0x00000000
-#define mmGDS_GS_CTXSW_CNT1_DEFAULT                                              0x00000000
-#define mmGDS_GS_CTXSW_CNT2_DEFAULT                                              0x00000000
-#define mmGDS_GS_CTXSW_CNT3_DEFAULT                                              0x00000000
-
-
-// addressBlock: gc_rasdec
-#define mmRAS_SIGNATURE_CONTROL_DEFAULT                                          0x00000000
-#define mmRAS_SIGNATURE_MASK_DEFAULT                                             0x00000000
-#define mmRAS_SX_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_SX_SIGNATURE1_DEFAULT                                              0x00000000
-#define mmRAS_SX_SIGNATURE2_DEFAULT                                              0x00000000
-#define mmRAS_SX_SIGNATURE3_DEFAULT                                              0x00000000
-#define mmRAS_DB_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_PA_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_VGT_SIGNATURE0_DEFAULT                                             0x00000000
-#define mmRAS_SQ_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE1_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE2_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE3_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE4_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE5_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE6_DEFAULT                                              0x00000000
-#define mmRAS_SC_SIGNATURE7_DEFAULT                                              0x00000000
-#define mmRAS_IA_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_IA_SIGNATURE1_DEFAULT                                              0x00000000
-#define mmRAS_SPI_SIGNATURE0_DEFAULT                                             0x00000000
-#define mmRAS_SPI_SIGNATURE1_DEFAULT                                             0x00000000
-#define mmRAS_TA_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_TD_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_CB_SIGNATURE0_DEFAULT                                              0x00000000
-#define mmRAS_BCI_SIGNATURE0_DEFAULT                                             0x00000000
-#define mmRAS_BCI_SIGNATURE1_DEFAULT                                             0x00000000
-#define mmRAS_TA_SIGNATURE1_DEFAULT                                              0x00000000
-
-
-// addressBlock: gc_gfxdec0
-#define mmDB_RENDER_CONTROL_DEFAULT                                              0x00000000
-#define mmDB_COUNT_CONTROL_DEFAULT                                               0x00000000
-#define mmDB_DEPTH_VIEW_DEFAULT                                                  0x00000000
-#define mmDB_RENDER_OVERRIDE_DEFAULT                                             0x00000000
-#define mmDB_RENDER_OVERRIDE2_DEFAULT                                            0x00000000
-#define mmDB_HTILE_DATA_BASE_DEFAULT                                             0x00000000
-#define mmDB_HTILE_DATA_BASE_HI_DEFAULT                                          0x00000000
-#define mmDB_DEPTH_SIZE_DEFAULT                                                  0x00000000
-#define mmDB_DEPTH_BOUNDS_MIN_DEFAULT                                            0x00000000
-#define mmDB_DEPTH_BOUNDS_MAX_DEFAULT                                            0x00000000
-#define mmDB_STENCIL_CLEAR_DEFAULT                                               0x00000000
-#define mmDB_DEPTH_CLEAR_DEFAULT                                                 0x00000000
-#define mmPA_SC_SCREEN_SCISSOR_TL_DEFAULT                                        0x00000000
-#define mmPA_SC_SCREEN_SCISSOR_BR_DEFAULT                                        0x00000000
-#define mmDB_Z_INFO_DEFAULT                                                      0x00000000
-#define mmDB_STENCIL_INFO_DEFAULT                                                0x00000000
-#define mmDB_Z_READ_BASE_DEFAULT                                                 0x00000000
-#define mmDB_Z_READ_BASE_HI_DEFAULT                                              0x00000000
-#define mmDB_STENCIL_READ_BASE_DEFAULT                                           0x00000000
-#define mmDB_STENCIL_READ_BASE_HI_DEFAULT                                        0x00000000
-#define mmDB_Z_WRITE_BASE_DEFAULT                                                0x00000000
-#define mmDB_Z_WRITE_BASE_HI_DEFAULT                                             0x00000000
-#define mmDB_STENCIL_WRITE_BASE_DEFAULT                                          0x00000000
-#define mmDB_STENCIL_WRITE_BASE_HI_DEFAULT                                       0x00000000
-#define mmDB_DFSM_CONTROL_DEFAULT                                                0x00000000
-#define mmDB_RENDER_FILTER_DEFAULT                                               0x00000000
-#define mmDB_Z_INFO2_DEFAULT                                                     0x00000000
-#define mmDB_STENCIL_INFO2_DEFAULT                                               0x00000000
-#define mmTA_BC_BASE_ADDR_DEFAULT                                                0x00000000
-#define mmTA_BC_BASE_ADDR_HI_DEFAULT                                             0x00000000
-#define mmCOHER_DEST_BASE_HI_0_DEFAULT                                           0x00000000
-#define mmCOHER_DEST_BASE_HI_1_DEFAULT                                           0x00000000
-#define mmCOHER_DEST_BASE_HI_2_DEFAULT                                           0x00000000
-#define mmCOHER_DEST_BASE_HI_3_DEFAULT                                           0x00000000
-#define mmCOHER_DEST_BASE_2_DEFAULT                                              0x00000000
-#define mmCOHER_DEST_BASE_3_DEFAULT                                              0x00000000
-#define mmPA_SC_WINDOW_OFFSET_DEFAULT                                            0x00000000
-#define mmPA_SC_WINDOW_SCISSOR_TL_DEFAULT                                        0x00000000
-#define mmPA_SC_WINDOW_SCISSOR_BR_DEFAULT                                        0x00000000
-#define mmPA_SC_CLIPRECT_RULE_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_0_TL_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_0_BR_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_1_TL_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_1_BR_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_2_TL_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_2_BR_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_3_TL_DEFAULT                                            0x00000000
-#define mmPA_SC_CLIPRECT_3_BR_DEFAULT                                            0x00000000
-#define mmPA_SC_EDGERULE_DEFAULT                                                 0x00000000
-#define mmPA_SU_HARDWARE_SCREEN_OFFSET_DEFAULT                                   0x00000000
-#define mmCB_TARGET_MASK_DEFAULT                                                 0x00000000
-#define mmCB_SHADER_MASK_DEFAULT                                                 0x00000000
-#define mmPA_SC_GENERIC_SCISSOR_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_GENERIC_SCISSOR_BR_DEFAULT                                       0x00000000
-#define mmCOHER_DEST_BASE_0_DEFAULT                                              0x00000000
-#define mmCOHER_DEST_BASE_1_DEFAULT                                              0x00000000
-#define mmPA_SC_VPORT_SCISSOR_0_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_0_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_1_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_1_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_2_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_2_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_3_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_3_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_4_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_4_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_5_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_5_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_6_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_6_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_7_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_7_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_8_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_8_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_9_TL_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_9_BR_DEFAULT                                       0x00000000
-#define mmPA_SC_VPORT_SCISSOR_10_TL_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_10_BR_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_11_TL_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_11_BR_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_12_TL_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_12_BR_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_13_TL_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_13_BR_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_14_TL_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_14_BR_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_15_TL_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_SCISSOR_15_BR_DEFAULT                                      0x00000000
-#define mmPA_SC_VPORT_ZMIN_0_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_0_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_1_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_1_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_2_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_2_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_3_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_3_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_4_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_4_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_5_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_5_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_6_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_6_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_7_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_7_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_8_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_8_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_9_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMAX_9_DEFAULT                                             0x00000000
-#define mmPA_SC_VPORT_ZMIN_10_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMAX_10_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMIN_11_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMAX_11_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMIN_12_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMAX_12_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMIN_13_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMAX_13_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMIN_14_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMAX_14_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMIN_15_DEFAULT                                            0x00000000
-#define mmPA_SC_VPORT_ZMAX_15_DEFAULT                                            0x00000000
-#define mmPA_SC_RASTER_CONFIG_DEFAULT                                            0x00000000
-#define mmPA_SC_RASTER_CONFIG_1_DEFAULT                                          0x00000000
-#define mmPA_SC_SCREEN_EXTENT_CONTROL_DEFAULT                                    0x00000000
-#define mmPA_SC_TILE_STEERING_OVERRIDE_DEFAULT                                   0x00000000
-#define mmCP_PERFMON_CNTX_CNTL_DEFAULT                                           0x00000000
-#define mmCP_PIPEID_DEFAULT                                                      0x00000000
-#define mmCP_RINGID_DEFAULT                                                      0x00000000
-#define mmCP_VMID_DEFAULT                                                        0x00000000
-#define mmPA_SC_RIGHT_VERT_GRID_DEFAULT                                          0x00000000
-#define mmPA_SC_LEFT_VERT_GRID_DEFAULT                                           0x00000000
-#define mmPA_SC_HORIZ_GRID_DEFAULT                                               0x00000000
-#define mmPA_SC_FOV_WINDOW_LR_DEFAULT                                            0x00000000
-#define mmPA_SC_FOV_WINDOW_TB_DEFAULT                                            0x00000000
-#define mmVGT_MULTI_PRIM_IB_RESET_INDX_DEFAULT                                   0x00000000
-#define mmCB_BLEND_RED_DEFAULT                                                   0x00000000
-#define mmCB_BLEND_GREEN_DEFAULT                                                 0x00000000
-#define mmCB_BLEND_BLUE_DEFAULT                                                  0x00000000
-#define mmCB_BLEND_ALPHA_DEFAULT                                                 0x00000000
-#define mmCB_DCC_CONTROL_DEFAULT                                                 0x00000000
-#define mmDB_STENCIL_CONTROL_DEFAULT                                             0x00000000
-#define mmDB_STENCILREFMASK_DEFAULT                                              0x00000000
-#define mmDB_STENCILREFMASK_BF_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XSCALE_DEFAULT                                             0x00000000
-#define mmPA_CL_VPORT_XOFFSET_DEFAULT                                            0x00000000
-#define mmPA_CL_VPORT_YSCALE_DEFAULT                                             0x00000000
-#define mmPA_CL_VPORT_YOFFSET_DEFAULT                                            0x00000000
-#define mmPA_CL_VPORT_ZSCALE_DEFAULT                                             0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_DEFAULT                                            0x00000000
-#define mmPA_CL_VPORT_XSCALE_1_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_1_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_1_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_1_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_1_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_1_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_2_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_2_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_2_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_2_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_2_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_2_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_3_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_3_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_3_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_3_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_3_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_3_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_4_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_4_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_4_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_4_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_4_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_4_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_5_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_5_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_5_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_5_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_5_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_5_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_6_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_6_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_6_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_6_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_6_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_6_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_7_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_7_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_7_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_7_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_7_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_7_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_8_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_8_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_8_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_8_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_8_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_8_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_9_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_XOFFSET_9_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YSCALE_9_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_YOFFSET_9_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZSCALE_9_DEFAULT                                           0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_9_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XSCALE_10_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XOFFSET_10_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_YSCALE_10_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YOFFSET_10_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_ZSCALE_10_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_10_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_XSCALE_11_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XOFFSET_11_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_YSCALE_11_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YOFFSET_11_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_ZSCALE_11_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_11_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_XSCALE_12_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XOFFSET_12_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_YSCALE_12_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YOFFSET_12_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_ZSCALE_12_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_12_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_XSCALE_13_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XOFFSET_13_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_YSCALE_13_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YOFFSET_13_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_ZSCALE_13_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_13_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_XSCALE_14_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XOFFSET_14_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_YSCALE_14_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YOFFSET_14_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_ZSCALE_14_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_14_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_XSCALE_15_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_XOFFSET_15_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_YSCALE_15_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_YOFFSET_15_DEFAULT                                         0x00000000
-#define mmPA_CL_VPORT_ZSCALE_15_DEFAULT                                          0x00000000
-#define mmPA_CL_VPORT_ZOFFSET_15_DEFAULT                                         0x00000000
-#define mmPA_CL_UCP_0_X_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_0_Y_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_0_Z_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_0_W_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_1_X_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_1_Y_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_1_Z_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_1_W_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_2_X_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_2_Y_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_2_Z_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_2_W_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_3_X_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_3_Y_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_3_Z_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_3_W_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_4_X_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_4_Y_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_4_Z_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_4_W_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_5_X_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_5_Y_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_5_Z_DEFAULT                                                  0x00000000
-#define mmPA_CL_UCP_5_W_DEFAULT                                                  0x00000000
-#define mmSPI_PS_INPUT_CNTL_0_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_1_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_2_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_3_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_4_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_5_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_6_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_7_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_8_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_9_DEFAULT                                            0x00000000
-#define mmSPI_PS_INPUT_CNTL_10_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_11_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_12_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_13_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_14_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_15_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_16_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_17_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_18_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_19_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_20_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_21_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_22_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_23_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_24_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_25_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_26_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_27_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_28_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_29_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_30_DEFAULT                                           0x00000000
-#define mmSPI_PS_INPUT_CNTL_31_DEFAULT                                           0x00000000
-#define mmSPI_VS_OUT_CONFIG_DEFAULT                                              0x00000000
-#define mmSPI_PS_INPUT_ENA_DEFAULT                                               0x00000000
-#define mmSPI_PS_INPUT_ADDR_DEFAULT                                              0x00000000
-#define mmSPI_INTERP_CONTROL_0_DEFAULT                                           0x00000000
-#define mmSPI_PS_IN_CONTROL_DEFAULT                                              0x00000000
-#define mmSPI_BARYC_CNTL_DEFAULT                                                 0x00000000
-#define mmSPI_TMPRING_SIZE_DEFAULT                                               0x00000000
-#define mmSPI_SHADER_POS_FORMAT_DEFAULT                                          0x00000000
-#define mmSPI_SHADER_Z_FORMAT_DEFAULT                                            0x00000000
-#define mmSPI_SHADER_COL_FORMAT_DEFAULT                                          0x00000000
-#define mmSX_PS_DOWNCONVERT_DEFAULT                                              0x00000000
-#define mmSX_BLEND_OPT_EPSILON_DEFAULT                                           0x00000000
-#define mmSX_BLEND_OPT_CONTROL_DEFAULT                                           0x00000000
-#define mmSX_MRT0_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmSX_MRT1_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmSX_MRT2_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmSX_MRT3_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmSX_MRT4_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmSX_MRT5_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmSX_MRT6_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmSX_MRT7_BLEND_OPT_DEFAULT                                              0x00000000
-#define mmCB_BLEND0_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_BLEND1_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_BLEND2_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_BLEND3_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_BLEND4_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_BLEND5_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_BLEND6_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_BLEND7_CONTROL_DEFAULT                                              0x00000000
-#define mmCB_MRT0_EPITCH_DEFAULT                                                 0x00000000
-#define mmCB_MRT1_EPITCH_DEFAULT                                                 0x00000000
-#define mmCB_MRT2_EPITCH_DEFAULT                                                 0x00000000
-#define mmCB_MRT3_EPITCH_DEFAULT                                                 0x00000000
-#define mmCB_MRT4_EPITCH_DEFAULT                                                 0x00000000
-#define mmCB_MRT5_EPITCH_DEFAULT                                                 0x00000000
-#define mmCB_MRT6_EPITCH_DEFAULT                                                 0x00000000
-#define mmCB_MRT7_EPITCH_DEFAULT                                                 0x00000000
-#define mmCS_COPY_STATE_DEFAULT                                                  0x00000000
-#define mmGFX_COPY_STATE_DEFAULT                                                 0x00000000
-#define mmPA_CL_POINT_X_RAD_DEFAULT                                              0x00000000
-#define mmPA_CL_POINT_Y_RAD_DEFAULT                                              0x00000000
-#define mmPA_CL_POINT_SIZE_DEFAULT                                               0x00000000
-#define mmPA_CL_POINT_CULL_RAD_DEFAULT                                           0x00000000
-#define mmVGT_DMA_BASE_HI_DEFAULT                                                0x00000000
-#define mmVGT_DMA_BASE_DEFAULT                                                   0x00000000
-#define mmVGT_DRAW_INITIATOR_DEFAULT                                             0x00000000
-#define mmVGT_IMMED_DATA_DEFAULT                                                 0x00000000
-#define mmVGT_EVENT_ADDRESS_REG_DEFAULT                                          0x00000000
-#define mmDB_DEPTH_CONTROL_DEFAULT                                               0x00000000
-#define mmDB_EQAA_DEFAULT                                                        0x00000000
-#define mmCB_COLOR_CONTROL_DEFAULT                                               0x00000000
-#define mmDB_SHADER_CONTROL_DEFAULT                                              0x00000000
-#define mmPA_CL_CLIP_CNTL_DEFAULT                                                0x00000000
-#define mmPA_SU_SC_MODE_CNTL_DEFAULT                                             0x00000000
-#define mmPA_CL_VTE_CNTL_DEFAULT                                                 0x00000000
-#define mmPA_CL_VS_OUT_CNTL_DEFAULT                                              0x00000000
-#define mmPA_CL_NANINF_CNTL_DEFAULT                                              0x00000000
-#define mmPA_SU_LINE_STIPPLE_CNTL_DEFAULT                                        0x00000000
-#define mmPA_SU_LINE_STIPPLE_SCALE_DEFAULT                                       0x00000000
-#define mmPA_SU_PRIM_FILTER_CNTL_DEFAULT                                         0x00000000
-#define mmPA_SU_SMALL_PRIM_FILTER_CNTL_DEFAULT                                   0x00000000
-#define mmPA_CL_OBJPRIM_ID_CNTL_DEFAULT                                          0x00000000
-#define mmPA_CL_NGG_CNTL_DEFAULT                                                 0x00000000
-#define mmPA_SU_OVER_RASTERIZATION_CNTL_DEFAULT                                  0x00000000
-#define mmPA_SU_POINT_SIZE_DEFAULT                                               0x00000000
-#define mmPA_SU_POINT_MINMAX_DEFAULT                                             0x00000000
-#define mmPA_SU_LINE_CNTL_DEFAULT                                                0x00000000
-#define mmPA_SC_LINE_STIPPLE_DEFAULT                                             0x00000000
-#define mmVGT_OUTPUT_PATH_CNTL_DEFAULT                                           0x00000000
-#define mmVGT_HOS_CNTL_DEFAULT                                                   0x00000000
-#define mmVGT_HOS_MAX_TESS_LEVEL_DEFAULT                                         0x00000000
-#define mmVGT_HOS_MIN_TESS_LEVEL_DEFAULT                                         0x00000000
-#define mmVGT_HOS_REUSE_DEPTH_DEFAULT                                            0x00000000
-#define mmVGT_GROUP_PRIM_TYPE_DEFAULT                                            0x00000000
-#define mmVGT_GROUP_FIRST_DECR_DEFAULT                                           0x00000000
-#define mmVGT_GROUP_DECR_DEFAULT                                                 0x00000000
-#define mmVGT_GROUP_VECT_0_CNTL_DEFAULT                                          0x00000000
-#define mmVGT_GROUP_VECT_1_CNTL_DEFAULT                                          0x00000000
-#define mmVGT_GROUP_VECT_0_FMT_CNTL_DEFAULT                                      0x00000000
-#define mmVGT_GROUP_VECT_1_FMT_CNTL_DEFAULT                                      0x00000000
-#define mmVGT_GS_MODE_DEFAULT                                                    0x00000000
-#define mmVGT_GS_ONCHIP_CNTL_DEFAULT                                             0x00000000
-#define mmPA_SC_MODE_CNTL_0_DEFAULT                                              0x00000000
-#define mmPA_SC_MODE_CNTL_1_DEFAULT                                              0x06000000
-#define mmVGT_ENHANCE_DEFAULT                                                    0x00000000
-#define mmVGT_GS_PER_ES_DEFAULT                                                  0x00000000
-#define mmVGT_ES_PER_GS_DEFAULT                                                  0x00000000
-#define mmVGT_GS_PER_VS_DEFAULT                                                  0x00000000
-#define mmVGT_GSVS_RING_OFFSET_1_DEFAULT                                         0x00000000
-#define mmVGT_GSVS_RING_OFFSET_2_DEFAULT                                         0x00000000
-#define mmVGT_GSVS_RING_OFFSET_3_DEFAULT                                         0x00000000
-#define mmVGT_GS_OUT_PRIM_TYPE_DEFAULT                                           0x00000000
-#define mmIA_ENHANCE_DEFAULT                                                     0x00000000
-#define mmVGT_DMA_SIZE_DEFAULT                                                   0x00000000
-#define mmVGT_DMA_MAX_SIZE_DEFAULT                                               0x00000000
-#define mmVGT_DMA_INDEX_TYPE_DEFAULT                                             0x00000000
-#define mmWD_ENHANCE_DEFAULT                                                     0x00000000
-#define mmVGT_PRIMITIVEID_EN_DEFAULT                                             0x00000000
-#define mmVGT_DMA_NUM_INSTANCES_DEFAULT                                          0x00000000
-#define mmVGT_PRIMITIVEID_RESET_DEFAULT                                          0x00000000
-#define mmVGT_EVENT_INITIATOR_DEFAULT                                            0x00000000
-#define mmVGT_GS_MAX_PRIMS_PER_SUBGROUP_DEFAULT                                  0x00000000
-#define mmVGT_DRAW_PAYLOAD_CNTL_DEFAULT                                          0x00000000
-#define mmVGT_INDEX_PAYLOAD_CNTL_DEFAULT                                         0x00000000
-#define mmVGT_INSTANCE_STEP_RATE_0_DEFAULT                                       0x00000000
-#define mmVGT_INSTANCE_STEP_RATE_1_DEFAULT                                       0x00000000
-#define mmVGT_ESGS_RING_ITEMSIZE_DEFAULT                                         0x00000000
-#define mmVGT_GSVS_RING_ITEMSIZE_DEFAULT                                         0x00000000
-#define mmVGT_REUSE_OFF_DEFAULT                                                  0x00000000
-#define mmVGT_VTX_CNT_EN_DEFAULT                                                 0x00000000
-#define mmDB_HTILE_SURFACE_DEFAULT                                               0x00000000
-#define mmDB_SRESULTS_COMPARE_STATE0_DEFAULT                                     0x00000000
-#define mmDB_SRESULTS_COMPARE_STATE1_DEFAULT                                     0x00000000
-#define mmDB_PRELOAD_CONTROL_DEFAULT                                             0x00000000
-#define mmVGT_STRMOUT_BUFFER_SIZE_0_DEFAULT                                      0x00000000
-#define mmVGT_STRMOUT_VTX_STRIDE_0_DEFAULT                                       0x00000000
-#define mmVGT_STRMOUT_BUFFER_OFFSET_0_DEFAULT                                    0x00000000
-#define mmVGT_STRMOUT_BUFFER_SIZE_1_DEFAULT                                      0x00000000
-#define mmVGT_STRMOUT_VTX_STRIDE_1_DEFAULT                                       0x00000000
-#define mmVGT_STRMOUT_BUFFER_OFFSET_1_DEFAULT                                    0x00000000
-#define mmVGT_STRMOUT_BUFFER_SIZE_2_DEFAULT                                      0x00000000
-#define mmVGT_STRMOUT_VTX_STRIDE_2_DEFAULT                                       0x00000000
-#define mmVGT_STRMOUT_BUFFER_OFFSET_2_DEFAULT                                    0x00000000
-#define mmVGT_STRMOUT_BUFFER_SIZE_3_DEFAULT                                      0x00000000
-#define mmVGT_STRMOUT_VTX_STRIDE_3_DEFAULT                                       0x00000000
-#define mmVGT_STRMOUT_BUFFER_OFFSET_3_DEFAULT                                    0x00000000
-#define mmVGT_STRMOUT_DRAW_OPAQUE_OFFSET_DEFAULT                                 0x00000000
-#define mmVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE_DEFAULT                     0x00000000
-#define mmVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE_DEFAULT                          0x00000000
-#define mmVGT_GS_MAX_VERT_OUT_DEFAULT                                            0x00000000
-#define mmVGT_TESS_DISTRIBUTION_DEFAULT                                          0x00000000
-#define mmVGT_SHADER_STAGES_EN_DEFAULT                                           0x00000000
-#define mmVGT_LS_HS_CONFIG_DEFAULT                                               0x00000000
-#define mmVGT_GS_VERT_ITEMSIZE_DEFAULT                                           0x00000000
-#define mmVGT_GS_VERT_ITEMSIZE_1_DEFAULT                                         0x00000000
-#define mmVGT_GS_VERT_ITEMSIZE_2_DEFAULT                                         0x00000000
-#define mmVGT_GS_VERT_ITEMSIZE_3_DEFAULT                                         0x00000000
-#define mmVGT_TF_PARAM_DEFAULT                                                   0x00000000
-#define mmDB_ALPHA_TO_MASK_DEFAULT                                               0x00000000
-#define mmVGT_DISPATCH_DRAW_INDEX_DEFAULT                                        0x00000000
-#define mmPA_SU_POLY_OFFSET_DB_FMT_CNTL_DEFAULT                                  0x00000000
-#define mmPA_SU_POLY_OFFSET_CLAMP_DEFAULT                                        0x00000000
-#define mmPA_SU_POLY_OFFSET_FRONT_SCALE_DEFAULT                                  0x00000000
-#define mmPA_SU_POLY_OFFSET_FRONT_OFFSET_DEFAULT                                 0x00000000
-#define mmPA_SU_POLY_OFFSET_BACK_SCALE_DEFAULT                                   0x00000000
-#define mmPA_SU_POLY_OFFSET_BACK_OFFSET_DEFAULT                                  0x00000000
-#define mmVGT_GS_INSTANCE_CNT_DEFAULT                                            0x00000000
-#define mmVGT_STRMOUT_CONFIG_DEFAULT                                             0x00000000
-#define mmVGT_STRMOUT_BUFFER_CONFIG_DEFAULT                                      0x00000000
-#define mmVGT_DMA_EVENT_INITIATOR_DEFAULT                                        0x00000000
-#define mmPA_SC_CENTROID_PRIORITY_0_DEFAULT                                      0x00000000
-#define mmPA_SC_CENTROID_PRIORITY_1_DEFAULT                                      0x00000000
-#define mmPA_SC_LINE_CNTL_DEFAULT                                                0x00000000
-#define mmPA_SC_AA_CONFIG_DEFAULT                                                0x00000000
-#define mmPA_SU_VTX_CNTL_DEFAULT                                                 0x00000000
-#define mmPA_CL_GB_VERT_CLIP_ADJ_DEFAULT                                         0x00000000
-#define mmPA_CL_GB_VERT_DISC_ADJ_DEFAULT                                         0x00000000
-#define mmPA_CL_GB_HORZ_CLIP_ADJ_DEFAULT                                         0x00000000
-#define mmPA_CL_GB_HORZ_DISC_ADJ_DEFAULT                                         0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2_DEFAULT                              0x00000000
-#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3_DEFAULT                              0x00000000
-#define mmPA_SC_AA_MASK_X0Y0_X1Y0_DEFAULT                                        0x00000000
-#define mmPA_SC_AA_MASK_X0Y1_X1Y1_DEFAULT                                        0x00000000
-#define mmPA_SC_SHADER_CONTROL_DEFAULT                                           0x00000000
-#define mmPA_SC_BINNER_CNTL_0_DEFAULT                                            0x00000000
-#define mmPA_SC_BINNER_CNTL_1_DEFAULT                                            0x00000000
-#define mmPA_SC_CONSERVATIVE_RASTERIZATION_CNTL_DEFAULT                          0x00000000
-#define mmPA_SC_NGG_MODE_CNTL_DEFAULT                                            0x00000000
-#define mmVGT_VERTEX_REUSE_BLOCK_CNTL_DEFAULT                                    0x00000000
-#define mmVGT_OUT_DEALLOC_CNTL_DEFAULT                                           0x00000000
-#define mmCB_COLOR0_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR0_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR0_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR0_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR0_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR0_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR0_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR0_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR0_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR0_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR0_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR0_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR0_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR0_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR0_DCC_BASE_EXT_DEFAULT                                         0x00000000
-#define mmCB_COLOR1_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR1_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR1_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR1_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR1_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR1_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR1_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR1_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR1_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR1_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR1_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR1_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR1_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR1_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR1_DCC_BASE_EXT_DEFAULT                                         0x00000000
-#define mmCB_COLOR2_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR2_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR2_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR2_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR2_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR2_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR2_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR2_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR2_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR2_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR2_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR2_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR2_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR2_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR2_DCC_BASE_EXT_DEFAULT                                         0x00000000
-#define mmCB_COLOR3_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR3_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR3_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR3_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR3_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR3_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR3_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR3_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR3_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR3_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR3_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR3_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR3_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR3_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR3_DCC_BASE_EXT_DEFAULT                                         0x00000000
-#define mmCB_COLOR4_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR4_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR4_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR4_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR4_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR4_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR4_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR4_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR4_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR4_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR4_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR4_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR4_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR4_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR4_DCC_BASE_EXT_DEFAULT                                         0x00000000
-#define mmCB_COLOR5_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR5_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR5_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR5_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR5_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR5_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR5_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR5_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR5_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR5_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR5_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR5_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR5_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR5_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR5_DCC_BASE_EXT_DEFAULT                                         0x00000000
-#define mmCB_COLOR6_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR6_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR6_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR6_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR6_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR6_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR6_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR6_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR6_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR6_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR6_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR6_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR6_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR6_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR6_DCC_BASE_EXT_DEFAULT                                         0x00000000
-#define mmCB_COLOR7_BASE_DEFAULT                                                 0x00000000
-#define mmCB_COLOR7_BASE_EXT_DEFAULT                                             0x00000000
-#define mmCB_COLOR7_ATTRIB2_DEFAULT                                              0x00000000
-#define mmCB_COLOR7_VIEW_DEFAULT                                                 0x00000000
-#define mmCB_COLOR7_INFO_DEFAULT                                                 0x00000000
-#define mmCB_COLOR7_ATTRIB_DEFAULT                                               0x00000000
-#define mmCB_COLOR7_DCC_CONTROL_DEFAULT                                          0x00000000
-#define mmCB_COLOR7_CMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR7_CMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR7_FMASK_DEFAULT                                                0x00000000
-#define mmCB_COLOR7_FMASK_BASE_EXT_DEFAULT                                       0x00000000
-#define mmCB_COLOR7_CLEAR_WORD0_DEFAULT                                          0x00000000
-#define mmCB_COLOR7_CLEAR_WORD1_DEFAULT                                          0x00000000
-#define mmCB_COLOR7_DCC_BASE_DEFAULT                                             0x00000000
-#define mmCB_COLOR7_DCC_BASE_EXT_DEFAULT                                         0x00000000
-
-
-// addressBlock: gc_gfxudec
-#define mmCP_EOP_DONE_ADDR_LO_DEFAULT                                            0x00000000
-#define mmCP_EOP_DONE_ADDR_HI_DEFAULT                                            0x00000000
-#define mmCP_EOP_DONE_DATA_LO_DEFAULT                                            0x00000000
-#define mmCP_EOP_DONE_DATA_HI_DEFAULT                                            0x00000000
-#define mmCP_EOP_LAST_FENCE_LO_DEFAULT                                           0x00000000
-#define mmCP_EOP_LAST_FENCE_HI_DEFAULT                                           0x00000000
-#define mmCP_STREAM_OUT_ADDR_LO_DEFAULT                                          0x00000000
-#define mmCP_STREAM_OUT_ADDR_HI_DEFAULT                                          0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT0_LO_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT0_HI_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT0_LO_DEFAULT                                   0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT0_HI_DEFAULT                                   0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT1_LO_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT1_HI_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT1_LO_DEFAULT                                   0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT1_HI_DEFAULT                                   0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT2_LO_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT2_HI_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT2_LO_DEFAULT                                   0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT2_HI_DEFAULT                                   0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT3_LO_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_WRITTEN_COUNT3_HI_DEFAULT                                  0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT3_LO_DEFAULT                                   0x00000000
-#define mmCP_NUM_PRIM_NEEDED_COUNT3_HI_DEFAULT                                   0x00000000
-#define mmCP_PIPE_STATS_ADDR_LO_DEFAULT                                          0x00000000
-#define mmCP_PIPE_STATS_ADDR_HI_DEFAULT                                          0x00000000
-#define mmCP_VGT_IAVERT_COUNT_LO_DEFAULT                                         0x00000000
-#define mmCP_VGT_IAVERT_COUNT_HI_DEFAULT                                         0x00000000
-#define mmCP_VGT_IAPRIM_COUNT_LO_DEFAULT                                         0x00000000
-#define mmCP_VGT_IAPRIM_COUNT_HI_DEFAULT                                         0x00000000
-#define mmCP_VGT_GSPRIM_COUNT_LO_DEFAULT                                         0x00000000
-#define mmCP_VGT_GSPRIM_COUNT_HI_DEFAULT                                         0x00000000
-#define mmCP_VGT_VSINVOC_COUNT_LO_DEFAULT                                        0x00000000
-#define mmCP_VGT_VSINVOC_COUNT_HI_DEFAULT                                        0x00000000
-#define mmCP_VGT_GSINVOC_COUNT_LO_DEFAULT                                        0x00000000
-#define mmCP_VGT_GSINVOC_COUNT_HI_DEFAULT                                        0x00000000
-#define mmCP_VGT_HSINVOC_COUNT_LO_DEFAULT                                        0x00000000
-#define mmCP_VGT_HSINVOC_COUNT_HI_DEFAULT                                        0x00000000
-#define mmCP_VGT_DSINVOC_COUNT_LO_DEFAULT                                        0x00000000
-#define mmCP_VGT_DSINVOC_COUNT_HI_DEFAULT                                        0x00000000
-#define mmCP_PA_CINVOC_COUNT_LO_DEFAULT                                          0x00000000
-#define mmCP_PA_CINVOC_COUNT_HI_DEFAULT                                          0x00000000
-#define mmCP_PA_CPRIM_COUNT_LO_DEFAULT                                           0x00000000
-#define mmCP_PA_CPRIM_COUNT_HI_DEFAULT                                           0x00000000
-#define mmCP_SC_PSINVOC_COUNT0_LO_DEFAULT                                        0x00000000
-#define mmCP_SC_PSINVOC_COUNT0_HI_DEFAULT                                        0x00000000
-#define mmCP_SC_PSINVOC_COUNT1_LO_DEFAULT                                        0x00000000
-#define mmCP_SC_PSINVOC_COUNT1_HI_DEFAULT                                        0x00000000
-#define mmCP_VGT_CSINVOC_COUNT_LO_DEFAULT                                        0x00000000
-#define mmCP_VGT_CSINVOC_COUNT_HI_DEFAULT                                        0x00000000
-#define mmCP_PIPE_STATS_CONTROL_DEFAULT                                          0x00000000
-#define mmCP_STREAM_OUT_CONTROL_DEFAULT                                          0x00000000
-#define mmCP_STRMOUT_CNTL_DEFAULT                                                0x00000000
-#define mmSCRATCH_REG0_DEFAULT                                                   0x00000000
-#define mmSCRATCH_REG1_DEFAULT                                                   0x00000000
-#define mmSCRATCH_REG2_DEFAULT                                                   0x00000000
-#define mmSCRATCH_REG3_DEFAULT                                                   0x00000000
-#define mmSCRATCH_REG4_DEFAULT                                                   0x00000000
-#define mmSCRATCH_REG5_DEFAULT                                                   0x00000000
-#define mmSCRATCH_REG6_DEFAULT                                                   0x00000000
-#define mmSCRATCH_REG7_DEFAULT                                                   0x00000000
-#define mmCP_APPEND_DATA_HI_DEFAULT                                              0x00000000
-#define mmCP_APPEND_LAST_CS_FENCE_HI_DEFAULT                                     0x00000000
-#define mmCP_APPEND_LAST_PS_FENCE_HI_DEFAULT                                     0x00000000
-#define mmSCRATCH_UMSK_DEFAULT                                                   0x00000000
-#define mmSCRATCH_ADDR_DEFAULT                                                   0x00000000
-#define mmCP_PFP_ATOMIC_PREOP_LO_DEFAULT                                         0x00000000
-#define mmCP_PFP_ATOMIC_PREOP_HI_DEFAULT                                         0x00000000
-#define mmCP_PFP_GDS_ATOMIC0_PREOP_LO_DEFAULT                                    0x00000000
-#define mmCP_PFP_GDS_ATOMIC0_PREOP_HI_DEFAULT                                    0x00000000
-#define mmCP_PFP_GDS_ATOMIC1_PREOP_LO_DEFAULT                                    0x00000000
-#define mmCP_PFP_GDS_ATOMIC1_PREOP_HI_DEFAULT                                    0x00000000
-#define mmCP_APPEND_ADDR_LO_DEFAULT                                              0x00000000
-#define mmCP_APPEND_ADDR_HI_DEFAULT                                              0x00000000
-#define mmCP_APPEND_DATA_LO_DEFAULT                                              0x00000000
-#define mmCP_APPEND_LAST_CS_FENCE_LO_DEFAULT                                     0x00000000
-#define mmCP_APPEND_LAST_PS_FENCE_LO_DEFAULT                                     0x00000000
-#define mmCP_ATOMIC_PREOP_LO_DEFAULT                                             0x00000000
-#define mmCP_ME_ATOMIC_PREOP_LO_DEFAULT                                          0x00000000
-#define mmCP_ATOMIC_PREOP_HI_DEFAULT                                             0x00000000
-#define mmCP_ME_ATOMIC_PREOP_HI_DEFAULT                                          0x00000000
-#define mmCP_GDS_ATOMIC0_PREOP_LO_DEFAULT                                        0x00000000
-#define mmCP_ME_GDS_ATOMIC0_PREOP_LO_DEFAULT                                     0x00000000
-#define mmCP_GDS_ATOMIC0_PREOP_HI_DEFAULT                                        0x00000000
-#define mmCP_ME_GDS_ATOMIC0_PREOP_HI_DEFAULT                                     0x00000000
-#define mmCP_GDS_ATOMIC1_PREOP_LO_DEFAULT                                        0x00000000
-#define mmCP_ME_GDS_ATOMIC1_PREOP_LO_DEFAULT                                     0x00000000
-#define mmCP_GDS_ATOMIC1_PREOP_HI_DEFAULT                                        0x00000000
-#define mmCP_ME_GDS_ATOMIC1_PREOP_HI_DEFAULT                                     0x00000000
-#define mmCP_ME_MC_WADDR_LO_DEFAULT                                              0x00000000
-#define mmCP_ME_MC_WADDR_HI_DEFAULT                                              0x00000000
-#define mmCP_ME_MC_WDATA_LO_DEFAULT                                              0x00000000
-#define mmCP_ME_MC_WDATA_HI_DEFAULT                                              0x00000000
-#define mmCP_ME_MC_RADDR_LO_DEFAULT                                              0x00000000
-#define mmCP_ME_MC_RADDR_HI_DEFAULT                                              0x00000000
-#define mmCP_SEM_WAIT_TIMER_DEFAULT                                              0x00000000
-#define mmCP_SIG_SEM_ADDR_LO_DEFAULT                                             0x00000000
-#define mmCP_SIG_SEM_ADDR_HI_DEFAULT                                             0x00000000
-#define mmCP_WAIT_REG_MEM_TIMEOUT_DEFAULT                                        0x00000000
-#define mmCP_WAIT_SEM_ADDR_LO_DEFAULT                                            0x00000000
-#define mmCP_WAIT_SEM_ADDR_HI_DEFAULT                                            0x00000000
-#define mmCP_DMA_PFP_CONTROL_DEFAULT                                             0x00000000
-#define mmCP_DMA_ME_CONTROL_DEFAULT                                              0x00000000
-#define mmCP_COHER_BASE_HI_DEFAULT                                               0x00000000
-#define mmCP_COHER_START_DELAY_DEFAULT                                           0x00000020
-#define mmCP_COHER_CNTL_DEFAULT                                                  0x00000000
-#define mmCP_COHER_SIZE_DEFAULT                                                  0x00000000
-#define mmCP_COHER_BASE_DEFAULT                                                  0x00000000
-#define mmCP_COHER_STATUS_DEFAULT                                                0x00000000
-#define mmCP_DMA_ME_SRC_ADDR_DEFAULT                                             0x00000000
-#define mmCP_DMA_ME_SRC_ADDR_HI_DEFAULT                                          0x00000000
-#define mmCP_DMA_ME_DST_ADDR_DEFAULT                                             0x00000000
-#define mmCP_DMA_ME_DST_ADDR_HI_DEFAULT                                          0x00000000
-#define mmCP_DMA_ME_COMMAND_DEFAULT                                              0x00000000
-#define mmCP_DMA_PFP_SRC_ADDR_DEFAULT                                            0x00000000
-#define mmCP_DMA_PFP_SRC_ADDR_HI_DEFAULT                                         0x00000000
-#define mmCP_DMA_PFP_DST_ADDR_DEFAULT                                            0x00000000
-#define mmCP_DMA_PFP_DST_ADDR_HI_DEFAULT                                         0x00000000
-#define mmCP_DMA_PFP_COMMAND_DEFAULT                                             0x00000000
-#define mmCP_DMA_CNTL_DEFAULT                                                    0x00080030
-#define mmCP_DMA_READ_TAGS_DEFAULT                                               0x00000000
-#define mmCP_COHER_SIZE_HI_DEFAULT                                               0x00000000
-#define mmCP_PFP_IB_CONTROL_DEFAULT                                              0x00000000
-#define mmCP_PFP_LOAD_CONTROL_DEFAULT                                            0x00000000
-#define mmCP_SCRATCH_INDEX_DEFAULT                                               0x00000000
-#define mmCP_SCRATCH_DATA_DEFAULT                                                0x00000000
-#define mmCP_RB_OFFSET_DEFAULT                                                   0x00000000
-#define mmCP_IB1_OFFSET_DEFAULT                                                  0x00000000
-#define mmCP_IB2_OFFSET_DEFAULT                                                  0x00000000
-#define mmCP_IB1_PREAMBLE_BEGIN_DEFAULT                                          0x00000000
-#define mmCP_IB1_PREAMBLE_END_DEFAULT                                            0x00000000
-#define mmCP_IB2_PREAMBLE_BEGIN_DEFAULT                                          0x00000000
-#define mmCP_IB2_PREAMBLE_END_DEFAULT                                            0x00000000
-#define mmCP_CE_IB1_OFFSET_DEFAULT                                               0x00000000
-#define mmCP_CE_IB2_OFFSET_DEFAULT                                               0x00000000
-#define mmCP_CE_COUNTER_DEFAULT                                                  0x00000000
-#define mmCP_CE_RB_OFFSET_DEFAULT                                                0x00000000
-#define mmCP_CE_INIT_CMD_BUFSZ_DEFAULT                                           0x00000000
-#define mmCP_CE_IB1_CMD_BUFSZ_DEFAULT                                            0x00000000
-#define mmCP_CE_IB2_CMD_BUFSZ_DEFAULT                                            0x00000000
-#define mmCP_IB1_CMD_BUFSZ_DEFAULT                                               0x00000000
-#define mmCP_IB2_CMD_BUFSZ_DEFAULT                                               0x00000000
-#define mmCP_ST_CMD_BUFSZ_DEFAULT                                                0x00000000
-#define mmCP_CE_INIT_BASE_LO_DEFAULT                                             0x00000000
-#define mmCP_CE_INIT_BASE_HI_DEFAULT                                             0x00000000
-#define mmCP_CE_INIT_BUFSZ_DEFAULT                                               0x00000000
-#define mmCP_CE_IB1_BASE_LO_DEFAULT                                              0x00000000
-#define mmCP_CE_IB1_BASE_HI_DEFAULT                                              0x00000000
-#define mmCP_CE_IB1_BUFSZ_DEFAULT                                                0x00000000
-#define mmCP_CE_IB2_BASE_LO_DEFAULT                                              0x00000000
-#define mmCP_CE_IB2_BASE_HI_DEFAULT                                              0x00000000
-#define mmCP_CE_IB2_BUFSZ_DEFAULT                                                0x00000000
-#define mmCP_IB1_BASE_LO_DEFAULT                                                 0x00000000
-#define mmCP_IB1_BASE_HI_DEFAULT                                                 0x00000000
-#define mmCP_IB1_BUFSZ_DEFAULT                                                   0x00000000
-#define mmCP_IB2_BASE_LO_DEFAULT                                                 0x00000000
-#define mmCP_IB2_BASE_HI_DEFAULT                                                 0x00000000
-#define mmCP_IB2_BUFSZ_DEFAULT                                                   0x00000000
-#define mmCP_ST_BASE_LO_DEFAULT                                                  0x00000000
-#define mmCP_ST_BASE_HI_DEFAULT                                                  0x00000000
-#define mmCP_ST_BUFSZ_DEFAULT                                                    0x00000000
-#define mmCP_EOP_DONE_EVENT_CNTL_DEFAULT                                         0x00000000
-#define mmCP_EOP_DONE_DATA_CNTL_DEFAULT                                          0x00000000
-#define mmCP_EOP_DONE_CNTX_ID_DEFAULT                                            0x00000000
-#define mmCP_PFP_COMPLETION_STATUS_DEFAULT                                       0x00000000
-#define mmCP_CE_COMPLETION_STATUS_DEFAULT                                        0x00000000
-#define mmCP_PRED_NOT_VISIBLE_DEFAULT                                            0x00000000
-#define mmCP_PFP_METADATA_BASE_ADDR_DEFAULT                                      0x00000000
-#define mmCP_PFP_METADATA_BASE_ADDR_HI_DEFAULT                                   0x00000000
-#define mmCP_CE_METADATA_BASE_ADDR_DEFAULT                                       0x00000000
-#define mmCP_CE_METADATA_BASE_ADDR_HI_DEFAULT                                    0x00000000
-#define mmCP_DRAW_INDX_INDR_ADDR_DEFAULT                                         0x00000000
-#define mmCP_DRAW_INDX_INDR_ADDR_HI_DEFAULT                                      0x00000000
-#define mmCP_DISPATCH_INDR_ADDR_DEFAULT                                          0x00000000
-#define mmCP_DISPATCH_INDR_ADDR_HI_DEFAULT                                       0x00000000
-#define mmCP_INDEX_BASE_ADDR_DEFAULT                                             0x00000000
-#define mmCP_INDEX_BASE_ADDR_HI_DEFAULT                                          0x00000000
-#define mmCP_INDEX_TYPE_DEFAULT                                                  0x00000000
-#define mmCP_GDS_BKUP_ADDR_DEFAULT                                               0x00000000
-#define mmCP_GDS_BKUP_ADDR_HI_DEFAULT                                            0x00000000
-#define mmCP_SAMPLE_STATUS_DEFAULT                                               0x00000000
-#define mmCP_ME_COHER_CNTL_DEFAULT                                               0x00000000
-#define mmCP_ME_COHER_SIZE_DEFAULT                                               0x00000000
-#define mmCP_ME_COHER_SIZE_HI_DEFAULT                                            0x00000000
-#define mmCP_ME_COHER_BASE_DEFAULT                                               0x00000000
-#define mmCP_ME_COHER_BASE_HI_DEFAULT                                            0x00000000
-#define mmCP_ME_COHER_STATUS_DEFAULT                                             0x00000000
-#define mmRLC_GPM_PERF_COUNT_0_DEFAULT                                           0x00000000
-#define mmRLC_GPM_PERF_COUNT_1_DEFAULT                                           0x00000000
-#define mmGRBM_GFX_INDEX_DEFAULT                                                 0xe0000000
-#define mmVGT_GSVS_RING_SIZE_DEFAULT                                             0x00000000
-#define mmVGT_PRIMITIVE_TYPE_DEFAULT                                             0x00000000
-#define mmVGT_INDEX_TYPE_DEFAULT                                                 0x00000000
-#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_0_DEFAULT                               0x00000000
-#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_1_DEFAULT                               0x00000000
-#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_2_DEFAULT                               0x00000000
-#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_3_DEFAULT                               0x00000000
-#define mmVGT_MAX_VTX_INDX_DEFAULT                                               0x00000000
-#define mmVGT_MIN_VTX_INDX_DEFAULT                                               0x00000000
-#define mmVGT_INDX_OFFSET_DEFAULT                                                0x00000000
-#define mmVGT_MULTI_PRIM_IB_RESET_EN_DEFAULT                                     0x00000000
-#define mmVGT_NUM_INDICES_DEFAULT                                                0x00000000
-#define mmVGT_NUM_INSTANCES_DEFAULT                                              0x00000000
-#define mmVGT_TF_RING_SIZE_DEFAULT                                               0x00002000
-#define mmVGT_HS_OFFCHIP_PARAM_DEFAULT                                           0x00000000
-#define mmVGT_TF_MEMORY_BASE_DEFAULT                                             0x00000000
-#define mmVGT_TF_MEMORY_BASE_HI_DEFAULT                                          0x00000000
-#define mmWD_POS_BUF_BASE_DEFAULT                                                0x00000000
-#define mmWD_POS_BUF_BASE_HI_DEFAULT                                             0x00000000
-#define mmWD_CNTL_SB_BUF_BASE_DEFAULT                                            0x00000000
-#define mmWD_CNTL_SB_BUF_BASE_HI_DEFAULT                                         0x00000000
-#define mmWD_INDEX_BUF_BASE_DEFAULT                                              0x00000000
-#define mmWD_INDEX_BUF_BASE_HI_DEFAULT                                           0x00000000
-#define mmIA_MULTI_VGT_PARAM_DEFAULT                                             0x006000ff
-#define mmVGT_OBJECT_ID_DEFAULT                                                  0x00000000
-#define mmVGT_INSTANCE_BASE_ID_DEFAULT                                           0x00000000
-#define mmPA_SU_LINE_STIPPLE_VALUE_DEFAULT                                       0x00000000
-#define mmPA_SC_LINE_STIPPLE_STATE_DEFAULT                                       0x00000000
-#define mmPA_SC_SCREEN_EXTENT_MIN_0_DEFAULT                                      0x7fff7fff
-#define mmPA_SC_SCREEN_EXTENT_MAX_0_DEFAULT                                      0x80008000
-#define mmPA_SC_SCREEN_EXTENT_MIN_1_DEFAULT                                      0x7fff7fff
-#define mmPA_SC_SCREEN_EXTENT_MAX_1_DEFAULT                                      0x80008000
-#define mmPA_SC_P3D_TRAP_SCREEN_HV_EN_DEFAULT                                    0x00000000
-#define mmPA_SC_P3D_TRAP_SCREEN_H_DEFAULT                                        0x00000000
-#define mmPA_SC_P3D_TRAP_SCREEN_V_DEFAULT                                        0x00000000
-#define mmPA_SC_P3D_TRAP_SCREEN_OCCURRENCE_DEFAULT                               0x00000000
-#define mmPA_SC_P3D_TRAP_SCREEN_COUNT_DEFAULT                                    0x00000000
-#define mmPA_SC_HP3D_TRAP_SCREEN_HV_EN_DEFAULT                                   0x00000000
-#define mmPA_SC_HP3D_TRAP_SCREEN_H_DEFAULT                                       0x00000000
-#define mmPA_SC_HP3D_TRAP_SCREEN_V_DEFAULT                                       0x00000000
-#define mmPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE_DEFAULT                              0x00000000
-#define mmPA_SC_HP3D_TRAP_SCREEN_COUNT_DEFAULT                                   0x00000000
-#define mmPA_SC_TRAP_SCREEN_HV_EN_DEFAULT                                        0x00000000
-#define mmPA_SC_TRAP_SCREEN_H_DEFAULT                                            0x00000000
-#define mmPA_SC_TRAP_SCREEN_V_DEFAULT                                            0x00000000
-#define mmPA_SC_TRAP_SCREEN_OCCURRENCE_DEFAULT                                   0x00000000
-#define mmPA_SC_TRAP_SCREEN_COUNT_DEFAULT                                        0x00000000
-#define mmSQ_THREAD_TRACE_BASE_DEFAULT                                           0x00000000
-#define mmSQ_THREAD_TRACE_SIZE_DEFAULT                                           0x00000000
-#define mmSQ_THREAD_TRACE_MASK_DEFAULT                                           0x0000cf80
-#define mmSQ_THREAD_TRACE_TOKEN_MASK_DEFAULT                                     0x00ffffff
-#define mmSQ_THREAD_TRACE_PERF_MASK_DEFAULT                                      0xffffffff
-#define mmSQ_THREAD_TRACE_CTRL_DEFAULT                                           0x00000000
-#define mmSQ_THREAD_TRACE_MODE_DEFAULT                                           0x02049249
-#define mmSQ_THREAD_TRACE_BASE2_DEFAULT                                          0x00000000
-#define mmSQ_THREAD_TRACE_TOKEN_MASK2_DEFAULT                                    0xffffffff
-#define mmSQ_THREAD_TRACE_WPTR_DEFAULT                                           0x00000000
-#define mmSQ_THREAD_TRACE_STATUS_DEFAULT                                         0x00000000
-#define mmSQ_THREAD_TRACE_HIWATER_DEFAULT                                        0x00000000
-#define mmSQ_THREAD_TRACE_CNTR_DEFAULT                                           0x00000000
-#define mmSQ_THREAD_TRACE_USERDATA_0_DEFAULT                                     0x00000000
-#define mmSQ_THREAD_TRACE_USERDATA_1_DEFAULT                                     0x00000000
-#define mmSQ_THREAD_TRACE_USERDATA_2_DEFAULT                                     0x00000000
-#define mmSQ_THREAD_TRACE_USERDATA_3_DEFAULT                                     0x00000000
-#define mmSQC_CACHES_DEFAULT                                                     0x00000000
-#define mmSQC_WRITEBACK_DEFAULT                                                  0x00000000
-#define mmTA_CS_BC_BASE_ADDR_DEFAULT                                             0x00000000
-#define mmTA_CS_BC_BASE_ADDR_HI_DEFAULT                                          0x00000000
-#define mmTA_GRAD_ADJ_UCONFIG_DEFAULT                                            0x40000040
-#define mmDB_OCCLUSION_COUNT0_LOW_DEFAULT                                        0x00000000
-#define mmDB_OCCLUSION_COUNT0_HI_DEFAULT                                         0x00000000
-#define mmDB_OCCLUSION_COUNT1_LOW_DEFAULT                                        0x00000000
-#define mmDB_OCCLUSION_COUNT1_HI_DEFAULT                                         0x00000000
-#define mmDB_OCCLUSION_COUNT2_LOW_DEFAULT                                        0x00000000
-#define mmDB_OCCLUSION_COUNT2_HI_DEFAULT                                         0x00000000
-#define mmDB_OCCLUSION_COUNT3_LOW_DEFAULT                                        0x00000000
-#define mmDB_OCCLUSION_COUNT3_HI_DEFAULT                                         0x00000000
-#define mmDB_ZPASS_COUNT_LOW_DEFAULT                                             0x00000000
-#define mmDB_ZPASS_COUNT_HI_DEFAULT                                              0x00000000
-#define mmGDS_RD_ADDR_DEFAULT                                                    0x00000000
-#define mmGDS_RD_DATA_DEFAULT                                                    0x00000000
-#define mmGDS_RD_BURST_ADDR_DEFAULT                                              0x00000000
-#define mmGDS_RD_BURST_COUNT_DEFAULT                                             0x00000000
-#define mmGDS_RD_BURST_DATA_DEFAULT                                              0x00000000
-#define mmGDS_WR_ADDR_DEFAULT                                                    0x00000000
-#define mmGDS_WR_DATA_DEFAULT                                                    0x00000000
-#define mmGDS_WR_BURST_ADDR_DEFAULT                                              0x00000000
-#define mmGDS_WR_BURST_DATA_DEFAULT                                              0x00000000
-#define mmGDS_WRITE_COMPLETE_DEFAULT                                             0x00000000
-#define mmGDS_ATOM_CNTL_DEFAULT                                                  0x00000000
-#define mmGDS_ATOM_COMPLETE_DEFAULT                                              0x00000001
-#define mmGDS_ATOM_BASE_DEFAULT                                                  0x00000000
-#define mmGDS_ATOM_SIZE_DEFAULT                                                  0x00000000
-#define mmGDS_ATOM_OFFSET0_DEFAULT                                               0x00000000
-#define mmGDS_ATOM_OFFSET1_DEFAULT                                               0x00000000
-#define mmGDS_ATOM_DST_DEFAULT                                                   0x00000000
-#define mmGDS_ATOM_OP_DEFAULT                                                    0x00000000
-#define mmGDS_ATOM_SRC0_DEFAULT                                                  0x00000000
-#define mmGDS_ATOM_SRC0_U_DEFAULT                                                0x00000000
-#define mmGDS_ATOM_SRC1_DEFAULT                                                  0x00000000
-#define mmGDS_ATOM_SRC1_U_DEFAULT                                                0x00000000
-#define mmGDS_ATOM_READ0_DEFAULT                                                 0x00000000
-#define mmGDS_ATOM_READ0_U_DEFAULT                                               0x00000000
-#define mmGDS_ATOM_READ1_DEFAULT                                                 0x00000000
-#define mmGDS_ATOM_READ1_U_DEFAULT                                               0x00000000
-#define mmGDS_GWS_RESOURCE_CNTL_DEFAULT                                          0x00000000
-#define mmGDS_GWS_RESOURCE_DEFAULT                                               0x00000000
-#define mmGDS_GWS_RESOURCE_CNT_DEFAULT                                           0x00000000
-#define mmGDS_OA_CNTL_DEFAULT                                                    0x00000000
-#define mmGDS_OA_COUNTER_DEFAULT                                                 0x00000000
-#define mmGDS_OA_ADDRESS_DEFAULT                                                 0x00000000
-#define mmGDS_OA_INCDEC_DEFAULT                                                  0x00000000
-#define mmGDS_OA_RING_SIZE_DEFAULT                                               0x00000000
-#define mmSPI_CONFIG_CNTL_DEFAULT                                                0x0062c688
-#define mmSPI_CONFIG_CNTL_1_DEFAULT                                              0x01000104
-#define mmSPI_CONFIG_CNTL_2_DEFAULT                                              0x00000011
-
-
-// addressBlock: gc_perfddec
-#define mmCPG_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmCPG_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmCPG_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmCPG_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmCPC_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmCPC_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmCPC_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmCPC_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmCPF_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmCPF_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmCPF_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmCPF_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmCPF_LATENCY_STATS_DATA_DEFAULT                                         0x00000000
-#define mmCPG_LATENCY_STATS_DATA_DEFAULT                                         0x00000000
-#define mmCPC_LATENCY_STATS_DATA_DEFAULT                                         0x00000000
-#define mmGRBM_PERFCOUNTER0_LO_DEFAULT                                           0x00000000
-#define mmGRBM_PERFCOUNTER0_HI_DEFAULT                                           0x00000000
-#define mmGRBM_PERFCOUNTER1_LO_DEFAULT                                           0x00000000
-#define mmGRBM_PERFCOUNTER1_HI_DEFAULT                                           0x00000000
-#define mmGRBM_SE0_PERFCOUNTER_LO_DEFAULT                                        0x00000000
-#define mmGRBM_SE0_PERFCOUNTER_HI_DEFAULT                                        0x00000000
-#define mmGRBM_SE1_PERFCOUNTER_LO_DEFAULT                                        0x00000000
-#define mmGRBM_SE1_PERFCOUNTER_HI_DEFAULT                                        0x00000000
-#define mmGRBM_SE2_PERFCOUNTER_LO_DEFAULT                                        0x00000000
-#define mmGRBM_SE2_PERFCOUNTER_HI_DEFAULT                                        0x00000000
-#define mmGRBM_SE3_PERFCOUNTER_LO_DEFAULT                                        0x00000000
-#define mmGRBM_SE3_PERFCOUNTER_HI_DEFAULT                                        0x00000000
-#define mmWD_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmWD_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmWD_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmWD_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmWD_PERFCOUNTER2_LO_DEFAULT                                             0x00000000
-#define mmWD_PERFCOUNTER2_HI_DEFAULT                                             0x00000000
-#define mmWD_PERFCOUNTER3_LO_DEFAULT                                             0x00000000
-#define mmWD_PERFCOUNTER3_HI_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER2_LO_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER2_HI_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER3_LO_DEFAULT                                             0x00000000
-#define mmIA_PERFCOUNTER3_HI_DEFAULT                                             0x00000000
-#define mmVGT_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmVGT_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmVGT_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmVGT_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmVGT_PERFCOUNTER2_LO_DEFAULT                                            0x00000000
-#define mmVGT_PERFCOUNTER2_HI_DEFAULT                                            0x00000000
-#define mmVGT_PERFCOUNTER3_LO_DEFAULT                                            0x00000000
-#define mmVGT_PERFCOUNTER3_HI_DEFAULT                                            0x00000000
-#define mmPA_SU_PERFCOUNTER0_LO_DEFAULT                                          0x00000000
-#define mmPA_SU_PERFCOUNTER0_HI_DEFAULT                                          0x00000000
-#define mmPA_SU_PERFCOUNTER1_LO_DEFAULT                                          0x00000000
-#define mmPA_SU_PERFCOUNTER1_HI_DEFAULT                                          0x00000000
-#define mmPA_SU_PERFCOUNTER2_LO_DEFAULT                                          0x00000000
-#define mmPA_SU_PERFCOUNTER2_HI_DEFAULT                                          0x00000000
-#define mmPA_SU_PERFCOUNTER3_LO_DEFAULT                                          0x00000000
-#define mmPA_SU_PERFCOUNTER3_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER0_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER0_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER1_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER1_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER2_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER2_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER3_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER3_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER4_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER4_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER5_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER5_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER6_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER6_HI_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER7_LO_DEFAULT                                          0x00000000
-#define mmPA_SC_PERFCOUNTER7_HI_DEFAULT                                          0x00000000
-#define mmSPI_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER2_HI_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER2_LO_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER3_HI_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER3_LO_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER4_HI_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER4_LO_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER5_HI_DEFAULT                                            0x00000000
-#define mmSPI_PERFCOUNTER5_LO_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER2_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER2_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER3_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER3_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER4_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER4_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER5_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER5_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER6_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER6_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER7_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER7_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER8_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER8_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER9_LO_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER9_HI_DEFAULT                                             0x00000000
-#define mmSQ_PERFCOUNTER10_LO_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER10_HI_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER11_LO_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER11_HI_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER12_LO_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER12_HI_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER13_LO_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER13_HI_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER14_LO_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER14_HI_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER15_LO_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER15_HI_DEFAULT                                            0x00000000
-#define mmSX_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmSX_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmSX_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmSX_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmSX_PERFCOUNTER2_LO_DEFAULT                                             0x00000000
-#define mmSX_PERFCOUNTER2_HI_DEFAULT                                             0x00000000
-#define mmSX_PERFCOUNTER3_LO_DEFAULT                                             0x00000000
-#define mmSX_PERFCOUNTER3_HI_DEFAULT                                             0x00000000
-#define mmGDS_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmGDS_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmGDS_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmGDS_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmGDS_PERFCOUNTER2_LO_DEFAULT                                            0x00000000
-#define mmGDS_PERFCOUNTER2_HI_DEFAULT                                            0x00000000
-#define mmGDS_PERFCOUNTER3_LO_DEFAULT                                            0x00000000
-#define mmGDS_PERFCOUNTER3_HI_DEFAULT                                            0x00000000
-#define mmTA_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmTA_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmTA_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmTA_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmTD_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmTD_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmTD_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmTD_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmTCP_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmTCP_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmTCP_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmTCP_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmTCP_PERFCOUNTER2_LO_DEFAULT                                            0x00000000
-#define mmTCP_PERFCOUNTER2_HI_DEFAULT                                            0x00000000
-#define mmTCP_PERFCOUNTER3_LO_DEFAULT                                            0x00000000
-#define mmTCP_PERFCOUNTER3_HI_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER2_LO_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER2_HI_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER3_LO_DEFAULT                                            0x00000000
-#define mmTCC_PERFCOUNTER3_HI_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER2_LO_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER2_HI_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER3_LO_DEFAULT                                            0x00000000
-#define mmTCA_PERFCOUNTER3_HI_DEFAULT                                            0x00000000
-#define mmCB_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmCB_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmCB_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmCB_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmCB_PERFCOUNTER2_LO_DEFAULT                                             0x00000000
-#define mmCB_PERFCOUNTER2_HI_DEFAULT                                             0x00000000
-#define mmCB_PERFCOUNTER3_LO_DEFAULT                                             0x00000000
-#define mmCB_PERFCOUNTER3_HI_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER0_LO_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER0_HI_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER1_LO_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER1_HI_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER2_LO_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER2_HI_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER3_LO_DEFAULT                                             0x00000000
-#define mmDB_PERFCOUNTER3_HI_DEFAULT                                             0x00000000
-#define mmRLC_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmRLC_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmRLC_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmRLC_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER0_LO_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER0_HI_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER1_LO_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER1_HI_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER2_LO_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER2_HI_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER3_LO_DEFAULT                                            0x00000000
-#define mmRMI_PERFCOUNTER3_HI_DEFAULT                                            0x00000000
-
-
-// addressBlock: gc_utcl2_atcl2pfcntrdec
-#define mmATC_L2_PERFCOUNTER_LO_DEFAULT                                          0x00000000
-#define mmATC_L2_PERFCOUNTER_HI_DEFAULT                                          0x00000000
-
-
-// addressBlock: gc_utcl2_vml2prdec
-#define mmMC_VM_L2_PERFCOUNTER_LO_DEFAULT                                        0x00000000
-#define mmMC_VM_L2_PERFCOUNTER_HI_DEFAULT                                        0x00000000
-
-
-// addressBlock: gc_perfsdec
-#define mmCPG_PERFCOUNTER1_SELECT_DEFAULT                                        0x11000401
-#define mmCPG_PERFCOUNTER0_SELECT1_DEFAULT                                       0x11000401
-#define mmCPG_PERFCOUNTER0_SELECT_DEFAULT                                        0x11000401
-#define mmCPC_PERFCOUNTER1_SELECT_DEFAULT                                        0x11000401
-#define mmCPC_PERFCOUNTER0_SELECT1_DEFAULT                                       0x11000401
-#define mmCPF_PERFCOUNTER1_SELECT_DEFAULT                                        0x11000401
-#define mmCPF_PERFCOUNTER0_SELECT1_DEFAULT                                       0x11000401
-#define mmCPF_PERFCOUNTER0_SELECT_DEFAULT                                        0x11000401
-#define mmCP_PERFMON_CNTL_DEFAULT                                                0x00000000
-#define mmCPC_PERFCOUNTER0_SELECT_DEFAULT                                        0x11000401
-#define mmCPF_TC_PERF_COUNTER_WINDOW_SELECT_DEFAULT                              0x00000000
-#define mmCPG_TC_PERF_COUNTER_WINDOW_SELECT_DEFAULT                              0x00000000
-#define mmCPF_LATENCY_STATS_SELECT_DEFAULT                                       0x00000000
-#define mmCPG_LATENCY_STATS_SELECT_DEFAULT                                       0x00000000
-#define mmCPC_LATENCY_STATS_SELECT_DEFAULT                                       0x00000000
-#define mmCP_DRAW_OBJECT_DEFAULT                                                 0x00000000
-#define mmCP_DRAW_OBJECT_COUNTER_DEFAULT                                         0x00000000
-#define mmCP_DRAW_WINDOW_MASK_HI_DEFAULT                                         0x00000000
-#define mmCP_DRAW_WINDOW_HI_DEFAULT                                              0x00000000
-#define mmCP_DRAW_WINDOW_LO_DEFAULT                                              0x00000000
-#define mmCP_DRAW_WINDOW_CNTL_DEFAULT                                            0x00000007
-#define mmGRBM_PERFCOUNTER0_SELECT_DEFAULT                                       0x00000000
-#define mmGRBM_PERFCOUNTER1_SELECT_DEFAULT                                       0x00000000
-#define mmGRBM_SE0_PERFCOUNTER_SELECT_DEFAULT                                    0x00000000
-#define mmGRBM_SE1_PERFCOUNTER_SELECT_DEFAULT                                    0x00000000
-#define mmGRBM_SE2_PERFCOUNTER_SELECT_DEFAULT                                    0x00000000
-#define mmGRBM_SE3_PERFCOUNTER_SELECT_DEFAULT                                    0x00000000
-#define mmWD_PERFCOUNTER0_SELECT_DEFAULT                                         0x00000000
-#define mmWD_PERFCOUNTER1_SELECT_DEFAULT                                         0x00000000
-#define mmWD_PERFCOUNTER2_SELECT_DEFAULT                                         0x00000000
-#define mmWD_PERFCOUNTER3_SELECT_DEFAULT                                         0x00000000
-#define mmIA_PERFCOUNTER0_SELECT_DEFAULT                                         0x00000000
-#define mmIA_PERFCOUNTER1_SELECT_DEFAULT                                         0x00000000
-#define mmIA_PERFCOUNTER2_SELECT_DEFAULT                                         0x00000000
-#define mmIA_PERFCOUNTER3_SELECT_DEFAULT                                         0x00000000
-#define mmIA_PERFCOUNTER0_SELECT1_DEFAULT                                        0x00000000
-#define mmVGT_PERFCOUNTER0_SELECT_DEFAULT                                        0x00000000
-#define mmVGT_PERFCOUNTER1_SELECT_DEFAULT                                        0x00000000
-#define mmVGT_PERFCOUNTER2_SELECT_DEFAULT                                        0x00000000
-#define mmVGT_PERFCOUNTER3_SELECT_DEFAULT                                        0x00000000
-#define mmVGT_PERFCOUNTER0_SELECT1_DEFAULT                                       0x00000000
-#define mmVGT_PERFCOUNTER1_SELECT1_DEFAULT                                       0x00000000
-#define mmVGT_PERFCOUNTER_SEID_MASK_DEFAULT                                      0x00000000
-#define mmPA_SU_PERFCOUNTER0_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SU_PERFCOUNTER0_SELECT1_DEFAULT                                     0x00000000
-#define mmPA_SU_PERFCOUNTER1_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SU_PERFCOUNTER1_SELECT1_DEFAULT                                     0x00000000
-#define mmPA_SU_PERFCOUNTER2_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SU_PERFCOUNTER3_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER0_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER0_SELECT1_DEFAULT                                     0x00000000
-#define mmPA_SC_PERFCOUNTER1_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER2_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER3_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER4_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER5_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER6_SELECT_DEFAULT                                      0x00000000
-#define mmPA_SC_PERFCOUNTER7_SELECT_DEFAULT                                      0x00000000
-#define mmSPI_PERFCOUNTER0_SELECT_DEFAULT                                        0x000fffff
-#define mmSPI_PERFCOUNTER1_SELECT_DEFAULT                                        0x000fffff
-#define mmSPI_PERFCOUNTER2_SELECT_DEFAULT                                        0x000fffff
-#define mmSPI_PERFCOUNTER3_SELECT_DEFAULT                                        0x000fffff
-#define mmSPI_PERFCOUNTER0_SELECT1_DEFAULT                                       0x000fffff
-#define mmSPI_PERFCOUNTER1_SELECT1_DEFAULT                                       0x000fffff
-#define mmSPI_PERFCOUNTER2_SELECT1_DEFAULT                                       0x000fffff
-#define mmSPI_PERFCOUNTER3_SELECT1_DEFAULT                                       0x000fffff
-#define mmSPI_PERFCOUNTER4_SELECT_DEFAULT                                        0x000000ff
-#define mmSPI_PERFCOUNTER5_SELECT_DEFAULT                                        0x000000ff
-#define mmSPI_PERFCOUNTER_BINS_DEFAULT                                           0xfcb87430
-#define mmSQ_PERFCOUNTER0_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER1_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER2_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER3_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER4_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER5_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER6_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER7_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER8_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER9_SELECT_DEFAULT                                         0x0f0ff000
-#define mmSQ_PERFCOUNTER10_SELECT_DEFAULT                                        0x0f0ff000
-#define mmSQ_PERFCOUNTER11_SELECT_DEFAULT                                        0x0f0ff000
-#define mmSQ_PERFCOUNTER12_SELECT_DEFAULT                                        0x0f0ff000
-#define mmSQ_PERFCOUNTER13_SELECT_DEFAULT                                        0x0f0ff000
-#define mmSQ_PERFCOUNTER14_SELECT_DEFAULT                                        0x0f0ff000
-#define mmSQ_PERFCOUNTER15_SELECT_DEFAULT                                        0x0f0ff000
-#define mmSQ_PERFCOUNTER_CTRL_DEFAULT                                            0x00000000
-#define mmSQ_PERFCOUNTER_MASK_DEFAULT                                            0xffffffff
-#define mmSQ_PERFCOUNTER_CTRL2_DEFAULT                                           0x00000000
-#define mmSX_PERFCOUNTER0_SELECT_DEFAULT                                         0x00000000
-#define mmSX_PERFCOUNTER1_SELECT_DEFAULT                                         0x00000000
-#define mmSX_PERFCOUNTER2_SELECT_DEFAULT                                         0x00000000
-#define mmSX_PERFCOUNTER3_SELECT_DEFAULT                                         0x00000000
-#define mmSX_PERFCOUNTER0_SELECT1_DEFAULT                                        0x00000000
-#define mmSX_PERFCOUNTER1_SELECT1_DEFAULT                                        0x00000000
-#define mmGDS_PERFCOUNTER0_SELECT_DEFAULT                                        0x00000000
-#define mmGDS_PERFCOUNTER1_SELECT_DEFAULT                                        0x00000000
-#define mmGDS_PERFCOUNTER2_SELECT_DEFAULT                                        0x00000000
-#define mmGDS_PERFCOUNTER3_SELECT_DEFAULT                                        0x00000000
-#define mmGDS_PERFCOUNTER0_SELECT1_DEFAULT                                       0x00000000
-#define mmTA_PERFCOUNTER0_SELECT_DEFAULT                                         0x00000000
-#define mmTA_PERFCOUNTER0_SELECT1_DEFAULT                                        0x00000000
-#define mmTA_PERFCOUNTER1_SELECT_DEFAULT                                         0x00000000
-#define mmTD_PERFCOUNTER0_SELECT_DEFAULT                                         0x00000000
-#define mmTD_PERFCOUNTER0_SELECT1_DEFAULT                                        0x00000000
-#define mmTD_PERFCOUNTER1_SELECT_DEFAULT                                         0x00000000
-#define mmTCP_PERFCOUNTER0_SELECT_DEFAULT                                        0x000fffff
-#define mmTCP_PERFCOUNTER0_SELECT1_DEFAULT                                       0x000fffff
-#define mmTCP_PERFCOUNTER1_SELECT_DEFAULT                                        0x000fffff
-#define mmTCP_PERFCOUNTER1_SELECT1_DEFAULT                                       0x000fffff
-#define mmTCP_PERFCOUNTER2_SELECT_DEFAULT                                        0x000003ff
-#define mmTCP_PERFCOUNTER3_SELECT_DEFAULT                                        0x000003ff
-#define mmTCC_PERFCOUNTER0_SELECT_DEFAULT                                        0x000fffff
-#define mmTCC_PERFCOUNTER0_SELECT1_DEFAULT                                       0x000fffff
-#define mmTCC_PERFCOUNTER1_SELECT_DEFAULT                                        0x000fffff
-#define mmTCC_PERFCOUNTER1_SELECT1_DEFAULT                                       0x000fffff
-#define mmTCC_PERFCOUNTER2_SELECT_DEFAULT                                        0x000003ff
-#define mmTCC_PERFCOUNTER3_SELECT_DEFAULT                                        0x000003ff
-#define mmTCA_PERFCOUNTER0_SELECT_DEFAULT                                        0x000fffff
-#define mmTCA_PERFCOUNTER0_SELECT1_DEFAULT                                       0x000fffff
-#define mmTCA_PERFCOUNTER1_SELECT_DEFAULT                                        0x000fffff
-#define mmTCA_PERFCOUNTER1_SELECT1_DEFAULT                                       0x000fffff
-#define mmTCA_PERFCOUNTER2_SELECT_DEFAULT                                        0x000003ff
-#define mmTCA_PERFCOUNTER3_SELECT_DEFAULT                                        0x000003ff
-#define mmCB_PERFCOUNTER_FILTER_DEFAULT                                          0x00000000
-#define mmCB_PERFCOUNTER0_SELECT_DEFAULT                                         0x00000000
-#define mmCB_PERFCOUNTER0_SELECT1_DEFAULT                                        0x00000000
-#define mmCB_PERFCOUNTER1_SELECT_DEFAULT                                         0x00000000
-#define mmCB_PERFCOUNTER2_SELECT_DEFAULT                                         0x00000000
-#define mmCB_PERFCOUNTER3_SELECT_DEFAULT                                         0x00000000
-#define mmDB_PERFCOUNTER0_SELECT_DEFAULT                                         0x00000000
-#define mmDB_PERFCOUNTER0_SELECT1_DEFAULT                                        0x00000000
-#define mmDB_PERFCOUNTER1_SELECT_DEFAULT                                         0x00000000
-#define mmDB_PERFCOUNTER1_SELECT1_DEFAULT                                        0x00000000
-#define mmDB_PERFCOUNTER2_SELECT_DEFAULT                                         0x00000000
-#define mmDB_PERFCOUNTER3_SELECT_DEFAULT                                         0x00000000
-#define mmRLC_SPM_PERFMON_CNTL_DEFAULT                                           0x00000000
-#define mmRLC_SPM_PERFMON_RING_BASE_LO_DEFAULT                                   0x00000000
-#define mmRLC_SPM_PERFMON_RING_BASE_HI_DEFAULT                                   0x00000000
-#define mmRLC_SPM_PERFMON_RING_SIZE_DEFAULT                                      0x00000000
-#define mmRLC_SPM_PERFMON_SEGMENT_SIZE_DEFAULT                                   0x00000000
-#define mmRLC_SPM_SE_MUXSEL_ADDR_DEFAULT                                         0x00000000
-#define mmRLC_SPM_SE_MUXSEL_DATA_DEFAULT                                         0x00000000
-#define mmRLC_SPM_CPG_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_CPC_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_CPF_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_CB_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_DB_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_PA_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_GDS_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_IA_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_SC_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_TCC_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_TCA_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_TCP_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_TA_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_TD_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_VGT_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_SPI_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_SQG_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_SPM_SX_PERFMON_SAMPLE_DELAY_DEFAULT                                0x00000000
-#define mmRLC_SPM_GLOBAL_MUXSEL_ADDR_DEFAULT                                     0x00000000
-#define mmRLC_SPM_GLOBAL_MUXSEL_DATA_DEFAULT                                     0x00000000
-#define mmRLC_SPM_RING_RDPTR_DEFAULT                                             0x00000000
-#define mmRLC_SPM_SEGMENT_THRESHOLD_DEFAULT                                      0x00000000
-#define mmRLC_SPM_RMI_PERFMON_SAMPLE_DELAY_DEFAULT                               0x00000000
-#define mmRLC_PERFMON_CLK_CNTL_DEFAULT                                           0x00000001
-#define mmRLC_PERFMON_CNTL_DEFAULT                                               0x00000000
-#define mmRLC_PERFCOUNTER0_SELECT_DEFAULT                                        0x00000000
-#define mmRLC_PERFCOUNTER1_SELECT_DEFAULT                                        0x00000000
-#define mmRLC_GPU_IOV_PERF_CNT_CNTL_DEFAULT                                      0x00000000
-#define mmRLC_GPU_IOV_PERF_CNT_WR_ADDR_DEFAULT                                   0x00000000
-#define mmRLC_GPU_IOV_PERF_CNT_WR_DATA_DEFAULT                                   0x00000000
-#define mmRLC_GPU_IOV_PERF_CNT_RD_ADDR_DEFAULT                                   0x00000000
-#define mmRLC_GPU_IOV_PERF_CNT_RD_DATA_DEFAULT                                   0x00000000
-#define mmRMI_PERFCOUNTER0_SELECT_DEFAULT                                        0x00000000
-#define mmRMI_PERFCOUNTER0_SELECT1_DEFAULT                                       0x00000000
-#define mmRMI_PERFCOUNTER1_SELECT_DEFAULT                                        0x00000000
-#define mmRMI_PERFCOUNTER2_SELECT_DEFAULT                                        0x00000000
-#define mmRMI_PERFCOUNTER2_SELECT1_DEFAULT                                       0x00000000
-#define mmRMI_PERFCOUNTER3_SELECT_DEFAULT                                        0x00000000
-#define mmRMI_PERF_COUNTER_CNTL_DEFAULT                                          0x00080240
-
-
-// addressBlock: gc_utcl2_atcl2pfcntldec
-#define mmATC_L2_PERFCOUNTER0_CFG_DEFAULT                                        0x00000000
-#define mmATC_L2_PERFCOUNTER1_CFG_DEFAULT                                        0x00000000
-#define mmATC_L2_PERFCOUNTER_RSLT_CNTL_DEFAULT                                   0x04000000
-
-
-// addressBlock: gc_utcl2_vml2pldec
-#define mmMC_VM_L2_PERFCOUNTER0_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER1_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER2_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER3_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER4_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER5_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER6_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER7_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER_RSLT_CNTL_DEFAULT                                 0x04000000
-
-
-// addressBlock: gc_rlcpdec
-#define mmRLC_CNTL_DEFAULT                                                       0x00000001
-#define mmRLC_STAT_DEFAULT                                                       0x00000000
-#define mmRLC_SAFE_MODE_DEFAULT                                                  0x00000000
-#define mmRLC_MEM_SLP_CNTL_DEFAULT                                               0x00020200
-#define mmSMU_RLC_RESPONSE_DEFAULT                                               0x00000000
-#define mmRLC_RLCV_SAFE_MODE_DEFAULT                                             0x00000000
-#define mmRLC_SMU_SAFE_MODE_DEFAULT                                              0x00000000
-#define mmRLC_RLCV_COMMAND_DEFAULT                                               0x00000000
-#define mmRLC_REFCLOCK_TIMESTAMP_LSB_DEFAULT                                     0x00000000
-#define mmRLC_REFCLOCK_TIMESTAMP_MSB_DEFAULT                                     0x00000000
-#define mmRLC_GPM_TIMER_INT_0_DEFAULT                                            0x00000000
-#define mmRLC_GPM_TIMER_INT_1_DEFAULT                                            0x00000000
-#define mmRLC_GPM_TIMER_INT_2_DEFAULT                                            0x00000000
-#define mmRLC_GPM_TIMER_CTRL_DEFAULT                                             0x00000000
-#define mmRLC_LB_CNTR_MAX_DEFAULT                                                0xffffffff
-#define mmRLC_GPM_TIMER_STAT_DEFAULT                                             0x00000000
-#define mmRLC_GPM_TIMER_INT_3_DEFAULT                                            0x00000000
-#define mmRLC_SERDES_WR_NONCU_MASTER_MASK_1_DEFAULT                              0x00000000
-#define mmRLC_SERDES_NONCU_MASTER_BUSY_1_DEFAULT                                 0x00000000
-#define mmRLC_INT_STAT_DEFAULT                                                   0x00000000
-#define mmRLC_LB_CNTL_DEFAULT                                                    0x00000010
-#define mmRLC_MGCG_CTRL_DEFAULT                                                  0x00018800
-#define mmRLC_LB_CNTR_INIT_DEFAULT                                               0x00000000
-#define mmRLC_LOAD_BALANCE_CNTR_DEFAULT                                          0x00000000
-#define mmRLC_JUMP_TABLE_RESTORE_DEFAULT                                         0x00000000
-#define mmRLC_PG_DELAY_2_DEFAULT                                                 0x00000004
-#define mmRLC_GPU_CLOCK_COUNT_LSB_DEFAULT                                        0x00000000
-#define mmRLC_GPU_CLOCK_COUNT_MSB_DEFAULT                                        0x00000000
-#define mmRLC_CAPTURE_GPU_CLOCK_COUNT_DEFAULT                                    0x00000000
-#define mmRLC_UCODE_CNTL_DEFAULT                                                 0x00000000
-#define mmRLC_GPM_THREAD_RESET_DEFAULT                                           0x0000000f
-#define mmRLC_GPM_CP_DMA_COMPLETE_T0_DEFAULT                                     0x00000000
-#define mmRLC_GPM_CP_DMA_COMPLETE_T1_DEFAULT                                     0x00000000
-#define mmRLC_FIREWALL_VIOLATION_DEFAULT                                         0x00000000
-#define mmRLC_GPM_STAT_DEFAULT                                                   0x00100016
-#define mmRLC_GPU_CLOCK_32_RES_SEL_DEFAULT                                       0x00000000
-#define mmRLC_GPU_CLOCK_32_DEFAULT                                               0x00000000
-#define mmRLC_PG_CNTL_DEFAULT                                                    0x00000000
-#define mmRLC_GPM_THREAD_PRIORITY_DEFAULT                                        0x08080808
-#define mmRLC_GPM_THREAD_ENABLE_DEFAULT                                          0x00000001
-#define mmRLC_CGTT_MGCG_OVERRIDE_DEFAULT                                         0xffffffff
-#define mmRLC_CGCG_CGLS_CTRL_DEFAULT                                             0x0001003c
-#define mmRLC_CGCG_RAMP_CTRL_DEFAULT                                             0x00021711
-#define mmRLC_DYN_PG_STATUS_DEFAULT                                              0xffffffff
-#define mmRLC_DYN_PG_REQUEST_DEFAULT                                             0xffffffff
-#define mmRLC_PG_DELAY_DEFAULT                                                   0x00101010
-#define mmRLC_CU_STATUS_DEFAULT                                                  0x00000000
-#define mmRLC_LB_INIT_CU_MASK_DEFAULT                                            0xffffffff
-#define mmRLC_LB_ALWAYS_ACTIVE_CU_MASK_DEFAULT                                   0x00000001
-#define mmRLC_LB_PARAMS_DEFAULT                                                  0x00601008
-#define mmRLC_THREAD1_DELAY_DEFAULT                                              0x00400401
-#define mmRLC_PG_ALWAYS_ON_CU_MASK_DEFAULT                                       0x00000003
-#define mmRLC_MAX_PG_CU_DEFAULT                                                  0x0000000b
-#define mmRLC_AUTO_PG_CTRL_DEFAULT                                               0x00000000
-#define mmRLC_SMU_GRBM_REG_SAVE_CTRL_DEFAULT                                     0x00000000
-#define mmRLC_SERDES_RD_MASTER_INDEX_DEFAULT                                     0x00000000
-#define mmRLC_SERDES_RD_DATA_0_DEFAULT                                           0x00000000
-#define mmRLC_SERDES_RD_DATA_1_DEFAULT                                           0x00000000
-#define mmRLC_SERDES_RD_DATA_2_DEFAULT                                           0x00000000
-#define mmRLC_SERDES_WR_CU_MASTER_MASK_DEFAULT                                   0x00000000
-#define mmRLC_SERDES_WR_NONCU_MASTER_MASK_DEFAULT                                0x00000000
-#define mmRLC_SERDES_WR_CTRL_DEFAULT                                             0x00000000
-#define mmRLC_SERDES_WR_DATA_DEFAULT                                             0x00000000
-#define mmRLC_SERDES_CU_MASTER_BUSY_DEFAULT                                      0x00000000
-#define mmRLC_SERDES_NONCU_MASTER_BUSY_DEFAULT                                   0x00000000
-#define mmRLC_GPM_GENERAL_0_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_1_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_2_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_3_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_4_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_5_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_6_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_7_DEFAULT                                              0x00000000
-#define mmRLC_GPM_SCRATCH_ADDR_DEFAULT                                           0x00000000
-#define mmRLC_GPM_SCRATCH_DATA_DEFAULT                                           0x00000000
-#define mmRLC_STATIC_PG_STATUS_DEFAULT                                           0xffffffff
-#define mmRLC_SPM_MC_CNTL_DEFAULT                                                0x00000000
-#define mmRLC_SPM_INT_CNTL_DEFAULT                                               0x00000000
-#define mmRLC_SPM_INT_STATUS_DEFAULT                                             0x00000000
-#define mmRLC_SMU_MESSAGE_DEFAULT                                                0x00000000
-#define mmRLC_GPM_LOG_SIZE_DEFAULT                                               0x00000000
-#define mmRLC_PG_DELAY_3_DEFAULT                                                 0x00000000
-#define mmRLC_GPR_REG1_DEFAULT                                                   0x00000000
-#define mmRLC_GPR_REG2_DEFAULT                                                   0x00000000
-#define mmRLC_GPM_LOG_CONT_DEFAULT                                               0x00000000
-#define mmRLC_GPM_INT_DISABLE_TH0_DEFAULT                                        0x00000000
-#define mmRLC_GPM_INT_DISABLE_TH1_DEFAULT                                        0x00000000
-#define mmRLC_GPM_INT_FORCE_TH0_DEFAULT                                          0x00000000
-#define mmRLC_GPM_INT_FORCE_TH1_DEFAULT                                          0x00000000
-#define mmRLC_SRM_CNTL_DEFAULT                                                   0x00000002
-#define mmRLC_SRM_ARAM_ADDR_DEFAULT                                              0x00000000
-#define mmRLC_SRM_ARAM_DATA_DEFAULT                                              0x00000000
-#define mmRLC_SRM_DRAM_ADDR_DEFAULT                                              0x00000000
-#define mmRLC_SRM_DRAM_DATA_DEFAULT                                              0x00000000
-#define mmRLC_SRM_GPM_COMMAND_DEFAULT                                            0x00000000
-#define mmRLC_SRM_GPM_COMMAND_STATUS_DEFAULT                                     0x00000000
-#define mmRLC_SRM_RLCV_COMMAND_DEFAULT                                           0x00000000
-#define mmRLC_SRM_RLCV_COMMAND_STATUS_DEFAULT                                    0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_0_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_1_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_2_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_3_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_4_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_5_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_6_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_ADDR_7_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_0_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_1_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_2_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_3_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_4_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_5_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_6_DEFAULT                                      0x00000000
-#define mmRLC_SRM_INDEX_CNTL_DATA_7_DEFAULT                                      0x00000000
-#define mmRLC_SRM_STAT_DEFAULT                                                   0x00000000
-#define mmRLC_SRM_GPM_ABORT_DEFAULT                                              0x00000000
-#define mmRLC_CSIB_ADDR_LO_DEFAULT                                               0x00000000
-#define mmRLC_CSIB_ADDR_HI_DEFAULT                                               0x00000000
-#define mmRLC_CSIB_LENGTH_DEFAULT                                                0x00000000
-#define mmRLC_SMU_COMMAND_DEFAULT                                                0x00000000
-#define mmRLC_CP_SCHEDULERS_DEFAULT                                              0x58504840
-#define mmRLC_SMU_ARGUMENT_1_DEFAULT                                             0x00000000
-#define mmRLC_SMU_ARGUMENT_2_DEFAULT                                             0x00000000
-#define mmRLC_GPM_GENERAL_8_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_9_DEFAULT                                              0x00000000
-#define mmRLC_GPM_GENERAL_10_DEFAULT                                             0x00000000
-#define mmRLC_GPM_GENERAL_11_DEFAULT                                             0x00000000
-#define mmRLC_GPM_GENERAL_12_DEFAULT                                             0x00000000
-#define mmRLC_GPM_UTCL1_CNTL_0_DEFAULT                                           0x00000080
-#define mmRLC_GPM_UTCL1_CNTL_1_DEFAULT                                           0x00000080
-#define mmRLC_GPM_UTCL1_CNTL_2_DEFAULT                                           0x00000080
-#define mmRLC_SPM_UTCL1_CNTL_DEFAULT                                             0x00000080
-#define mmRLC_UTCL1_STATUS_2_DEFAULT                                             0x00000000
-#define mmRLC_LB_THR_CONFIG_2_DEFAULT                                            0x00000000
-#define mmRLC_LB_THR_CONFIG_3_DEFAULT                                            0x00000000
-#define mmRLC_LB_THR_CONFIG_4_DEFAULT                                            0x00000000
-#define mmRLC_SPM_UTCL1_ERROR_1_DEFAULT                                          0x00000000
-#define mmRLC_SPM_UTCL1_ERROR_2_DEFAULT                                          0x00000000
-#define mmRLC_GPM_UTCL1_TH0_ERROR_1_DEFAULT                                      0x00000000
-#define mmRLC_LB_THR_CONFIG_1_DEFAULT                                            0x00000000
-#define mmRLC_GPM_UTCL1_TH0_ERROR_2_DEFAULT                                      0x00000000
-#define mmRLC_GPM_UTCL1_TH1_ERROR_1_DEFAULT                                      0x00000000
-#define mmRLC_GPM_UTCL1_TH1_ERROR_2_DEFAULT                                      0x00000000
-#define mmRLC_GPM_UTCL1_TH2_ERROR_1_DEFAULT                                      0x00000000
-#define mmRLC_GPM_UTCL1_TH2_ERROR_2_DEFAULT                                      0x00000000
-#define mmRLC_CGCG_CGLS_CTRL_3D_DEFAULT                                          0x0001003c
-#define mmRLC_CGCG_RAMP_CTRL_3D_DEFAULT                                          0x00021711
-#define mmRLC_SEMAPHORE_0_DEFAULT                                                0x00000000
-#define mmRLC_SEMAPHORE_1_DEFAULT                                                0x00000000
-#define mmRLC_CP_EOF_INT_DEFAULT                                                 0x00000000
-#define mmRLC_CP_EOF_INT_CNT_DEFAULT                                             0x00000000
-#define mmRLC_SPARE_INT_DEFAULT                                                  0x00000000
-#define mmRLC_PREWALKER_UTCL1_CNTL_DEFAULT                                       0x00000080
-#define mmRLC_PREWALKER_UTCL1_TRIG_DEFAULT                                       0x00000000
-#define mmRLC_PREWALKER_UTCL1_ADDR_LSB_DEFAULT                                   0x00000000
-#define mmRLC_PREWALKER_UTCL1_ADDR_MSB_DEFAULT                                   0x00000000
-#define mmRLC_PREWALKER_UTCL1_SIZE_LSB_DEFAULT                                   0x00000000
-#define mmRLC_PREWALKER_UTCL1_SIZE_MSB_DEFAULT                                   0x00000000
-#define mmRLC_DSM_TRIG_DEFAULT                                                   0x00000000
-#define mmRLC_UTCL1_STATUS_DEFAULT                                               0x00000000
-#define mmRLC_R2I_CNTL_0_DEFAULT                                                 0x00000000
-#define mmRLC_R2I_CNTL_1_DEFAULT                                                 0x00000000
-#define mmRLC_R2I_CNTL_2_DEFAULT                                                 0x00000000
-#define mmRLC_R2I_CNTL_3_DEFAULT                                                 0x00000000
-#define mmRLC_UTCL2_CNTL_DEFAULT                                                 0x00000000
-#define mmRLC_LBPW_CU_STAT_DEFAULT                                               0x00000000
-#define mmRLC_DS_CNTL_DEFAULT                                                    0x00030003
-#define mmRLC_RLCV_SPARE_INT_DEFAULT                                             0x00000000
-
-
-// addressBlock: gc_pwrdec
-#define mmCGTS_SM_CTRL_REG_DEFAULT                                               0x00600200
-#define mmCGTS_RD_CTRL_REG_DEFAULT                                               0x00000000
-#define mmCGTS_RD_REG_DEFAULT                                                    0x00000000
-#define mmCGTS_TCC_DISABLE_DEFAULT                                               0x00000000
-#define mmCGTS_USER_TCC_DISABLE_DEFAULT                                          0x00000000
-#define mmCGTS_CU0_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU0_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU0_TA_SQC_CTRL_REG_DEFAULT                                       0x00040007
-#define mmCGTS_CU0_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU0_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU1_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU1_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU1_TA_SQC_CTRL_REG_DEFAULT                                       0x00000007
-#define mmCGTS_CU1_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU1_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU2_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU2_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU2_TA_SQC_CTRL_REG_DEFAULT                                       0x00000007
-#define mmCGTS_CU2_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU2_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU3_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU3_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU3_TA_SQC_CTRL_REG_DEFAULT                                       0x00040007
-#define mmCGTS_CU3_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU3_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU4_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU4_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU4_TA_SQC_CTRL_REG_DEFAULT                                       0x00000007
-#define mmCGTS_CU4_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU4_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU5_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU5_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU5_TA_SQC_CTRL_REG_DEFAULT                                       0x00000007
-#define mmCGTS_CU5_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU5_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU6_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU6_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU6_TA_SQC_CTRL_REG_DEFAULT                                       0x00040007
-#define mmCGTS_CU6_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU6_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU7_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU7_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU7_TA_SQC_CTRL_REG_DEFAULT                                       0x00000007
-#define mmCGTS_CU7_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU7_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU8_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU8_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU8_TA_SQC_CTRL_REG_DEFAULT                                       0x00000007
-#define mmCGTS_CU8_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU8_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU9_SP0_CTRL_REG_DEFAULT                                          0x00010000
-#define mmCGTS_CU9_LDS_SQ_CTRL_REG_DEFAULT                                       0x00030002
-#define mmCGTS_CU9_TA_SQC_CTRL_REG_DEFAULT                                       0x00040007
-#define mmCGTS_CU9_SP1_CTRL_REG_DEFAULT                                          0x00060005
-#define mmCGTS_CU9_TD_TCP_CTRL_REG_DEFAULT                                       0x00090008
-#define mmCGTS_CU10_SP0_CTRL_REG_DEFAULT                                         0x00010000
-#define mmCGTS_CU10_LDS_SQ_CTRL_REG_DEFAULT                                      0x00030002
-#define mmCGTS_CU10_TA_SQC_CTRL_REG_DEFAULT                                      0x00000007
-#define mmCGTS_CU10_SP1_CTRL_REG_DEFAULT                                         0x00060005
-#define mmCGTS_CU10_TD_TCP_CTRL_REG_DEFAULT                                      0x00090008
-#define mmCGTS_CU11_SP0_CTRL_REG_DEFAULT                                         0x00010000
-#define mmCGTS_CU11_LDS_SQ_CTRL_REG_DEFAULT                                      0x00030002
-#define mmCGTS_CU11_TA_SQC_CTRL_REG_DEFAULT                                      0x00000007
-#define mmCGTS_CU11_SP1_CTRL_REG_DEFAULT                                         0x00060005
-#define mmCGTS_CU11_TD_TCP_CTRL_REG_DEFAULT                                      0x00090008
-#define mmCGTS_CU12_SP0_CTRL_REG_DEFAULT                                         0x00010000
-#define mmCGTS_CU12_LDS_SQ_CTRL_REG_DEFAULT                                      0x00030002
-#define mmCGTS_CU12_TA_SQC_CTRL_REG_DEFAULT                                      0x00040007
-#define mmCGTS_CU12_SP1_CTRL_REG_DEFAULT                                         0x00060005
-#define mmCGTS_CU12_TD_TCP_CTRL_REG_DEFAULT                                      0x00090008
-#define mmCGTS_CU13_SP0_CTRL_REG_DEFAULT                                         0x00010000
-#define mmCGTS_CU13_LDS_SQ_CTRL_REG_DEFAULT                                      0x00030002
-#define mmCGTS_CU13_TA_SQC_CTRL_REG_DEFAULT                                      0x00000007
-#define mmCGTS_CU13_SP1_CTRL_REG_DEFAULT                                         0x00060005
-#define mmCGTS_CU13_TD_TCP_CTRL_REG_DEFAULT                                      0x00090008
-#define mmCGTS_CU14_SP0_CTRL_REG_DEFAULT                                         0x00010000
-#define mmCGTS_CU14_LDS_SQ_CTRL_REG_DEFAULT                                      0x00030002
-#define mmCGTS_CU14_TA_SQC_CTRL_REG_DEFAULT                                      0x00000007
-#define mmCGTS_CU14_SP1_CTRL_REG_DEFAULT                                         0x00060005
-#define mmCGTS_CU14_TD_TCP_CTRL_REG_DEFAULT                                      0x00090008
-#define mmCGTS_CU15_SP0_CTRL_REG_DEFAULT                                         0x00010000
-#define mmCGTS_CU15_LDS_SQ_CTRL_REG_DEFAULT                                      0x00030002
-#define mmCGTS_CU15_TA_SQC_CTRL_REG_DEFAULT                                      0x00040007
-#define mmCGTS_CU15_SP1_CTRL_REG_DEFAULT                                         0x00060005
-#define mmCGTS_CU15_TD_TCP_CTRL_REG_DEFAULT                                      0x00090008
-#define mmCGTS_CU0_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU1_TCPI_CTRL_REG_DEFAULT                                         0x00000001
-#define mmCGTS_CU2_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU3_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU4_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU5_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU6_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU7_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU8_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU9_TCPI_CTRL_REG_DEFAULT                                         0x0000000a
-#define mmCGTS_CU10_TCPI_CTRL_REG_DEFAULT                                        0x0000000a
-#define mmCGTS_CU11_TCPI_CTRL_REG_DEFAULT                                        0x0000000a
-#define mmCGTS_CU12_TCPI_CTRL_REG_DEFAULT                                        0x0000000a
-#define mmCGTS_CU13_TCPI_CTRL_REG_DEFAULT                                        0x0000000a
-#define mmCGTS_CU14_TCPI_CTRL_REG_DEFAULT                                        0x0000000a
-#define mmCGTS_CU15_TCPI_CTRL_REG_DEFAULT                                        0x0000000a
-#define mmCGTT_SPI_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmCGTT_PC_CLK_CTRL_DEFAULT                                               0x00000100
-#define mmCGTT_BCI_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmCGTT_VGT_CLK_CTRL_DEFAULT                                              0x00018100
-#define mmCGTT_IA_CLK_CTRL_DEFAULT                                               0x06000100
-#define mmCGTT_WD_CLK_CTRL_DEFAULT                                               0x00018100
-#define mmCGTT_PA_CLK_CTRL_DEFAULT                                               0x00000100
-#define mmCGTT_SC_CLK_CTRL0_DEFAULT                                              0x00000100
-#define mmCGTT_SC_CLK_CTRL1_DEFAULT                                              0x00000100
-#define mmCGTT_SQ_CLK_CTRL_DEFAULT                                               0x00000100
-#define mmCGTT_SQG_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmSQ_ALU_CLK_CTRL_DEFAULT                                                0x00000000
-#define mmSQ_TEX_CLK_CTRL_DEFAULT                                                0x00000000
-#define mmSQ_LDS_CLK_CTRL_DEFAULT                                                0x00000000
-#define mmSQ_POWER_THROTTLE_DEFAULT                                              0x3fff3fff
-#define mmSQ_POWER_THROTTLE2_DEFAULT                                             0x18800004
-#define mmCGTT_SX_CLK_CTRL0_DEFAULT                                              0x00000100
-#define mmCGTT_SX_CLK_CTRL1_DEFAULT                                              0x00000100
-#define mmCGTT_SX_CLK_CTRL2_DEFAULT                                              0x00000100
-#define mmCGTT_SX_CLK_CTRL3_DEFAULT                                              0x00000100
-#define mmCGTT_SX_CLK_CTRL4_DEFAULT                                              0x00000100
-#define mmTD_CGTT_CTRL_DEFAULT                                                   0x00000100
-#define mmTA_CGTT_CTRL_DEFAULT                                                   0x00000100
-#define mmCGTT_TCPI_CLK_CTRL_DEFAULT                                             0x00000100
-#define mmCGTT_TCI_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmCGTT_GDS_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmDB_CGTT_CLK_CTRL_0_DEFAULT                                             0x00000100
-#define mmCB_CGTT_SCLK_CTRL_DEFAULT                                              0x00000100
-#define mmTCC_CGTT_SCLK_CTRL_DEFAULT                                             0x00000100
-#define mmTCA_CGTT_SCLK_CTRL_DEFAULT                                             0x00000100
-#define mmCGTT_CP_CLK_CTRL_DEFAULT                                               0x00000100
-#define mmCGTT_CPF_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmCGTT_CPC_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmRLC_PWR_CTRL_DEFAULT                                                   0x00000000
-#define mmCGTT_RLC_CLK_CTRL_DEFAULT                                              0x00000100
-#define mmRLC_GFX_RM_CNTL_DEFAULT                                                0x00000000
-#define mmRMI_CGTT_SCLK_CTRL_DEFAULT                                             0x00000100
-#define mmCGTT_TCPF_CLK_CTRL_DEFAULT                                             0x00000100
-
-
-// addressBlock: gc_ea_pwrdec
-#define mmGCEA_CGTT_CLK_CTRL_DEFAULT                                             0x00000100
-
-
-// addressBlock: gc_utcl2_vmsharedhvdec
-#define mmMC_VM_FB_SIZE_OFFSET_VF0_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF1_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF2_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF3_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF4_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF5_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF6_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF7_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF8_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF9_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF10_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF11_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF12_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF13_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF14_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF15_DEFAULT                                      0x00000000
-#define mmVM_IOMMU_MMIO_CNTRL_1_DEFAULT                                          0x00000100
-#define mmMC_VM_MARC_BASE_LO_0_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_LO_1_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_LO_2_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_LO_3_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_0_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_1_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_2_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_3_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_RELOC_LO_0_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_LO_1_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_LO_2_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_LO_3_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_0_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_1_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_2_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_3_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_LEN_LO_0_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_LO_1_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_LO_2_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_LO_3_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_0_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_1_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_2_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_3_DEFAULT                                            0x00000000
-#define mmVM_IOMMU_CONTROL_REGISTER_DEFAULT                                      0x00000000
-#define mmVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER_DEFAULT             0x00000000
-#define mmVM_PCIE_ATS_CNTL_DEFAULT                                               0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_0_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_1_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_2_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_3_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_4_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_5_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_6_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_7_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_8_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_9_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_10_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_11_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_12_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_13_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_14_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_15_DEFAULT                                         0x00000000
-#define mmUTCL2_CGTT_CLK_CTRL_DEFAULT                                            0x00000080
-
-
-// addressBlock: gc_hypdec
-#define mmCP_HYP_PFP_UCODE_ADDR_DEFAULT                                          0x00000000
-#define mmCP_PFP_UCODE_ADDR_DEFAULT                                              0x00000000
-#define mmCP_HYP_PFP_UCODE_DATA_DEFAULT                                          0x00000000
-#define mmCP_PFP_UCODE_DATA_DEFAULT                                              0x00000000
-#define mmCP_HYP_ME_UCODE_ADDR_DEFAULT                                           0x00000000
-#define mmCP_ME_RAM_RADDR_DEFAULT                                                0x00000000
-#define mmCP_ME_RAM_WADDR_DEFAULT                                                0x00000000
-#define mmCP_HYP_ME_UCODE_DATA_DEFAULT                                           0x00000000
-#define mmCP_ME_RAM_DATA_DEFAULT                                                 0x00000000
-#define mmCP_CE_UCODE_ADDR_DEFAULT                                               0x00000000
-#define mmCP_HYP_CE_UCODE_ADDR_DEFAULT                                           0x00000000
-#define mmCP_CE_UCODE_DATA_DEFAULT                                               0x00000000
-#define mmCP_HYP_CE_UCODE_DATA_DEFAULT                                           0x00000000
-#define mmCP_HYP_MEC1_UCODE_ADDR_DEFAULT                                         0x00000000
-#define mmCP_MEC_ME1_UCODE_ADDR_DEFAULT                                          0x00000000
-#define mmCP_HYP_MEC1_UCODE_DATA_DEFAULT                                         0x00000000
-#define mmCP_MEC_ME1_UCODE_DATA_DEFAULT                                          0x00000000
-#define mmCP_HYP_MEC2_UCODE_ADDR_DEFAULT                                         0x00000000
-#define mmCP_MEC_ME2_UCODE_ADDR_DEFAULT                                          0x00000000
-#define mmCP_HYP_MEC2_UCODE_DATA_DEFAULT                                         0x00000000
-#define mmCP_MEC_ME2_UCODE_DATA_DEFAULT                                          0x00000000
-#define mmRLC_GPM_UCODE_ADDR_DEFAULT                                             0x00000000
-#define mmRLC_GPM_UCODE_DATA_DEFAULT                                             0x00000000
-#define mmGRBM_GFX_INDEX_SR_SELECT_DEFAULT                                       0x00000000
-#define mmGRBM_GFX_INDEX_SR_DATA_DEFAULT                                         0xe0000000
-#define mmGRBM_GFX_CNTL_SR_SELECT_DEFAULT                                        0x00000000
-#define mmGRBM_GFX_CNTL_SR_DATA_DEFAULT                                          0x00000000
-#define mmGRBM_CAM_INDEX_DEFAULT                                                 0x00000000
-#define mmGRBM_HYP_CAM_INDEX_DEFAULT                                             0x00000000
-#define mmGRBM_CAM_DATA_DEFAULT                                                  0x00000000
-#define mmGRBM_HYP_CAM_DATA_DEFAULT                                              0x00000000
-#define mmRLC_GPU_IOV_VF_ENABLE_DEFAULT                                          0x00000000
-#define mmRLC_GFX_RM_CNTL_ADJ_DEFAULT                                            0x00000000
-#define mmRLC_GPU_IOV_CFG_REG6_DEFAULT                                           0x00000000
-#define mmRLC_GPU_IOV_CFG_REG8_DEFAULT                                           0x00000000
-#define mmRLC_RLCV_TIMER_INT_0_DEFAULT                                           0x00000000
-#define mmRLC_RLCV_TIMER_CTRL_DEFAULT                                            0x00000000
-#define mmRLC_RLCV_TIMER_STAT_DEFAULT                                            0x00000000
-#define mmRLC_GPU_IOV_VF_DOORBELL_STATUS_DEFAULT                                 0x0000ffff
-#define mmRLC_GPU_IOV_VF_DOORBELL_STATUS_SET_DEFAULT                             0x00000000
-#define mmRLC_GPU_IOV_VF_DOORBELL_STATUS_CLR_DEFAULT                             0x00000000
-#define mmRLC_GPU_IOV_VF_MASK_DEFAULT                                            0x00010001
-#define mmRLC_HYP_SEMAPHORE_2_DEFAULT                                            0x00000000
-#define mmRLC_HYP_SEMAPHORE_3_DEFAULT                                            0x00000000
-#define mmRLC_CLK_CNTL_DEFAULT                                                   0x00000003
-#define mmRLC_GPU_IOV_SCH_BLOCK_DEFAULT                                          0x00000000
-#define mmRLC_GPU_IOV_CFG_REG1_DEFAULT                                           0x00000000
-#define mmRLC_GPU_IOV_CFG_REG2_DEFAULT                                           0x00000000
-#define mmRLC_GPU_IOV_VM_BUSY_STATUS_DEFAULT                                     0x00000000
-#define mmRLC_GPU_IOV_SCH_0_DEFAULT                                              0x00000000
-#define mmRLC_GPU_IOV_ACTIVE_FCN_ID_DEFAULT                                      0x00000000
-#define mmRLC_GPU_IOV_SCH_3_DEFAULT                                              0x00000000
-#define mmRLC_GPU_IOV_SCH_1_DEFAULT                                              0x00000000
-#define mmRLC_GPU_IOV_SCH_2_DEFAULT                                              0x00000000
-#define mmRLC_GPU_IOV_UCODE_ADDR_DEFAULT                                         0x00000000
-#define mmRLC_GPU_IOV_UCODE_DATA_DEFAULT                                         0x00000000
-#define mmRLC_GPU_IOV_SCRATCH_ADDR_DEFAULT                                       0x00000000
-#define mmRLC_GPU_IOV_SCRATCH_DATA_DEFAULT                                       0x00000000
-#define mmRLC_GPU_IOV_F32_CNTL_DEFAULT                                           0x00000000
-#define mmRLC_GPU_IOV_F32_RESET_DEFAULT                                          0x00000000
-#define mmRLC_GPU_IOV_SDMA0_STATUS_DEFAULT                                       0x00000000
-#define mmRLC_GPU_IOV_SDMA1_STATUS_DEFAULT                                       0x00000000
-#define mmRLC_GPU_IOV_SMU_RESPONSE_DEFAULT                                       0x00000000
-#define mmRLC_GPU_IOV_VIRT_RESET_REQ_DEFAULT                                     0x00000000
-#define mmRLC_GPU_IOV_RLC_RESPONSE_DEFAULT                                       0x00000000
-#define mmRLC_GPU_IOV_INT_DISABLE_DEFAULT                                        0x00000000
-#define mmRLC_GPU_IOV_INT_FORCE_DEFAULT                                          0x00000000
-#define mmRLC_GPU_IOV_SDMA0_BUSY_STATUS_DEFAULT                                  0x00000000
-#define mmRLC_GPU_IOV_SDMA1_BUSY_STATUS_DEFAULT                                  0x00000000
-
-
-// addressBlock: gccacind
-#define ixGC_CAC_CNTL_DEFAULT                                                    0x000001fe
-#define ixGC_CAC_OVR_SEL_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVR_VAL_DEFAULT                                                 0x00000000
-#define ixGC_CAC_WEIGHT_BCI_0_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_CB_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CB_1_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CP_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CP_1_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_DB_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_DB_1_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_GDS_0_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_GDS_1_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_IA_0_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_LDS_0_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_LDS_1_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_PA_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_PC_0_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_SC_0_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_SPI_0_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_SPI_1_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_SPI_2_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_SQ_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_SQ_1_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_SQ_2_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_SQ_3_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_SQ_4_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_SX_0_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_SXRB_0_DEFAULT                                           0x00010001
-#define ixGC_CAC_WEIGHT_TA_0_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_TCC_0_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_TCC_1_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_TCC_2_DEFAULT                                            0x00000001
-#define ixGC_CAC_WEIGHT_TCP_0_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_TCP_1_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_TCP_2_DEFAULT                                            0x00000001
-#define ixGC_CAC_WEIGHT_TD_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_TD_1_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_TD_2_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_VGT_0_DEFAULT                                            0x00010001
-#define ixGC_CAC_WEIGHT_VGT_1_DEFAULT                                            0x00000001
-#define ixGC_CAC_WEIGHT_WD_0_DEFAULT                                             0x00000001
-#define ixGC_CAC_WEIGHT_CU_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CU_1_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CU_2_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CU_3_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CU_4_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_CU_5_DEFAULT                                             0x00010001
-#define ixGC_CAC_ACC_BCI0_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_CB0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CB1_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CB2_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CB3_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CP0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CP1_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CP2_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_DB0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_DB1_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_DB2_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_DB3_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_GDS0_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_GDS1_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_GDS2_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_GDS3_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_IA0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_LDS0_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_LDS1_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_LDS2_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_LDS3_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_PA0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_PA1_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_PC0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_SC0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_SPI0_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_SPI1_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_SPI2_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_SPI3_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_SPI4_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_SPI5_DEFAULT                                                0x00000000
-#define ixGC_CAC_WEIGHT_PG_0_DEFAULT                                             0x00000001
-#define ixGC_CAC_ACC_PG0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_PG_DEFAULT                                                 0x00000000
-#define ixGC_CAC_WEIGHT_UTCL2_ATCL2_0_DEFAULT                                    0x00010001
-#define ixGC_CAC_ACC_EA0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_EA1_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_EA2_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_EA3_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_UTCL2_ATCL20_DEFAULT                                        0x00000000
-#define ixGC_CAC_OVRD_EA_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_UTCL2_ATCL2_DEFAULT                                        0x00000000
-#define ixGC_CAC_WEIGHT_EA_0_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_EA_1_DEFAULT                                             0x00010001
-#define ixGC_CAC_WEIGHT_RMI_0_DEFAULT                                            0x00000001
-#define ixGC_CAC_ACC_RMI0_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_RMI_DEFAULT                                                0x00000000
-#define ixGC_CAC_WEIGHT_UTCL2_ATCL2_1_DEFAULT                                    0x00010001
-#define ixGC_CAC_ACC_UTCL2_ATCL21_DEFAULT                                        0x00000000
-#define ixGC_CAC_ACC_UTCL2_ATCL22_DEFAULT                                        0x00000000
-#define ixGC_CAC_ACC_UTCL2_ATCL23_DEFAULT                                        0x00000000
-#define ixGC_CAC_ACC_EA4_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_EA5_DEFAULT                                                 0x00000000
-#define ixGC_CAC_WEIGHT_EA_2_DEFAULT                                             0x00010001
-#define ixGC_CAC_ACC_SQ0_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ0_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ1_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ1_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ2_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ2_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ3_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ3_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ4_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ4_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ5_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ5_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ6_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ6_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ7_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ7_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ8_LOWER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SQ8_UPPER_DEFAULT                                           0x00000000
-#define ixGC_CAC_ACC_SX0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_SXRB0_DEFAULT                                               0x00000000
-#define ixGC_CAC_ACC_SXRB1_DEFAULT                                               0x00000000
-#define ixGC_CAC_ACC_TA0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_TCC0_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCC1_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCC2_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCC3_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCC4_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCP0_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCP1_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCP2_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCP3_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TCP4_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_TD0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_TD1_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_TD2_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_TD3_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_TD4_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_TD5_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_VGT0_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_VGT1_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_VGT2_DEFAULT                                                0x00000000
-#define ixGC_CAC_ACC_WD0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU0_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU1_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU2_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU3_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU4_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU5_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU6_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU7_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU8_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU9_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_CU10_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_BCI_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_CB_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_CP_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_DB_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_GDS_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_IA_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_LDS_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_PA_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_PC_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_SC_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_SPI_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_CU_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_SQ_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_SX_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_SXRB_DEFAULT                                               0x00000000
-#define ixGC_CAC_OVRD_TA_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_TCC_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_TCP_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_TD_DEFAULT                                                 0x00000000
-#define ixGC_CAC_OVRD_VGT_DEFAULT                                                0x00000000
-#define ixGC_CAC_OVRD_WD_DEFAULT                                                 0x00000000
-#define ixGC_CAC_ACC_BCI1_DEFAULT                                                0x00000000
-#define ixGC_CAC_WEIGHT_UTCL2_ATCL2_2_DEFAULT                                    0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_ROUTER_0_DEFAULT                                   0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_ROUTER_1_DEFAULT                                   0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_ROUTER_2_DEFAULT                                   0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_ROUTER_3_DEFAULT                                   0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_ROUTER_4_DEFAULT                                   0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_VML2_0_DEFAULT                                     0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_VML2_1_DEFAULT                                     0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_VML2_2_DEFAULT                                     0x00010001
-#define ixGC_CAC_ACC_UTCL2_ATCL24_DEFAULT                                        0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER0_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER1_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER2_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER3_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER4_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER5_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER6_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER7_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER8_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_ROUTER9_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_VML20_DEFAULT                                         0x00000000
-#define ixGC_CAC_ACC_UTCL2_VML21_DEFAULT                                         0x00000000
-#define ixGC_CAC_ACC_UTCL2_VML22_DEFAULT                                         0x00000000
-#define ixGC_CAC_ACC_UTCL2_VML23_DEFAULT                                         0x00000000
-#define ixGC_CAC_ACC_UTCL2_VML24_DEFAULT                                         0x00000000
-#define ixGC_CAC_OVRD_UTCL2_ROUTER_DEFAULT                                       0x00000000
-#define ixGC_CAC_OVRD_UTCL2_VML2_DEFAULT                                         0x00000000
-#define ixGC_CAC_WEIGHT_UTCL2_WALKER_0_DEFAULT                                   0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_WALKER_1_DEFAULT                                   0x00010001
-#define ixGC_CAC_WEIGHT_UTCL2_WALKER_2_DEFAULT                                   0x00010001
-#define ixGC_CAC_ACC_UTCL2_WALKER0_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_WALKER1_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_WALKER2_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_WALKER3_DEFAULT                                       0x00000000
-#define ixGC_CAC_ACC_UTCL2_WALKER4_DEFAULT                                       0x00000000
-#define ixGC_CAC_OVRD_UTCL2_WALKER_DEFAULT                                       0x00000000
-
-
-// addressBlock: secacind
-#define ixSE_CAC_CNTL_DEFAULT                                                    0x000001fe
-#define ixSE_CAC_OVR_SEL_DEFAULT                                                 0x00000000
-#define ixSE_CAC_OVR_VAL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: sqind
-#define ixSQ_WAVE_MODE_DEFAULT                                                   0x00000000
-#define ixSQ_WAVE_STATUS_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_TRAPSTS_DEFAULT                                                0x00000000
-#define ixSQ_WAVE_HW_ID_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_GPR_ALLOC_DEFAULT                                              0x00000000
-#define ixSQ_WAVE_LDS_ALLOC_DEFAULT                                              0x00000000
-#define ixSQ_WAVE_IB_STS_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_PC_LO_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_PC_HI_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_INST_DW0_DEFAULT                                               0x00000000
-#define ixSQ_WAVE_INST_DW1_DEFAULT                                               0x00000000
-#define ixSQ_WAVE_IB_DBG0_DEFAULT                                                0x00000000
-#define ixSQ_WAVE_IB_DBG1_DEFAULT                                                0x00000000
-#define ixSQ_WAVE_FLUSH_IB_DEFAULT                                               0x00000000
-#define ixSQ_WAVE_TTMP0_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP1_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP2_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP3_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP4_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP5_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP6_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP7_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP8_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP9_DEFAULT                                                  0x00000000
-#define ixSQ_WAVE_TTMP10_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_TTMP11_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_TTMP12_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_TTMP13_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_TTMP14_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_TTMP15_DEFAULT                                                 0x00000000
-#define ixSQ_WAVE_M0_DEFAULT                                                     0x00000000
-#define ixSQ_WAVE_EXEC_LO_DEFAULT                                                0x00000000
-#define ixSQ_WAVE_EXEC_HI_DEFAULT                                                0x00000000
-#define ixSQ_INTERRUPT_WORD_AUTO_CTXID_DEFAULT                                   0x00000000
-#define ixSQ_INTERRUPT_WORD_AUTO_HI_DEFAULT                                      0x00000000
-#define ixSQ_INTERRUPT_WORD_AUTO_LO_DEFAULT                                      0x00000000
-#define ixSQ_INTERRUPT_WORD_CMN_CTXID_DEFAULT                                    0x00000000
-#define ixSQ_INTERRUPT_WORD_CMN_HI_DEFAULT                                       0x00000000
-#define ixSQ_INTERRUPT_WORD_WAVE_CTXID_DEFAULT                                   0x00000000
-#define ixSQ_INTERRUPT_WORD_WAVE_HI_DEFAULT                                      0x00000000
-#define ixSQ_INTERRUPT_WORD_WAVE_LO_DEFAULT                                      0x00000000
-
-
-
-
-
-
-
-
-// addressBlock: didtind
-#define ixDIDT_SQ_CTRL0_DEFAULT                                                  0x0000ff00
-#define ixDIDT_SQ_CTRL1_DEFAULT                                                  0x00ff00ff
-#define ixDIDT_SQ_CTRL2_DEFAULT                                                  0x18800004
-#define ixDIDT_SQ_STALL_CTRL_DEFAULT                                             0x00fff000
-#define ixDIDT_SQ_TUNING_CTRL_DEFAULT                                            0x00010004
-#define ixDIDT_SQ_STALL_AUTO_RELEASE_CTRL_DEFAULT                                0x00ffffff
-#define ixDIDT_SQ_CTRL3_DEFAULT                                                  0x00038000
-#define ixDIDT_SQ_STALL_PATTERN_1_2_DEFAULT                                      0x01010001
-#define ixDIDT_SQ_STALL_PATTERN_3_4_DEFAULT                                      0x11110421
-#define ixDIDT_SQ_STALL_PATTERN_5_6_DEFAULT                                      0x25291249
-#define ixDIDT_SQ_STALL_PATTERN_7_DEFAULT                                        0x00002aaa
-#define ixDIDT_SQ_WEIGHT0_3_DEFAULT                                              0x00000000
-#define ixDIDT_SQ_WEIGHT4_7_DEFAULT                                              0x00000000
-#define ixDIDT_SQ_WEIGHT8_11_DEFAULT                                             0x00000000
-#define ixDIDT_SQ_EDC_CTRL_DEFAULT                                               0x00001c00
-#define ixDIDT_SQ_EDC_THRESHOLD_DEFAULT                                          0x00000000
-#define ixDIDT_SQ_EDC_STALL_PATTERN_1_2_DEFAULT                                  0x01010001
-#define ixDIDT_SQ_EDC_STALL_PATTERN_3_4_DEFAULT                                  0x11110421
-#define ixDIDT_SQ_EDC_STALL_PATTERN_5_6_DEFAULT                                  0x25291249
-#define ixDIDT_SQ_EDC_STALL_PATTERN_7_DEFAULT                                    0x00002aaa
-#define ixDIDT_SQ_EDC_STATUS_DEFAULT                                             0x00000000
-#define ixDIDT_SQ_EDC_STALL_DELAY_1_DEFAULT                                      0x00000000
-#define ixDIDT_SQ_EDC_STALL_DELAY_2_DEFAULT                                      0x00000000
-#define ixDIDT_SQ_EDC_STALL_DELAY_3_DEFAULT                                      0x00000000
-#define ixDIDT_SQ_EDC_OVERFLOW_DEFAULT                                           0x00000000
-#define ixDIDT_SQ_EDC_ROLLING_POWER_DELTA_DEFAULT                                0x00000000
-#define ixDIDT_DB_CTRL0_DEFAULT                                                  0x0000ff00
-#define ixDIDT_DB_CTRL1_DEFAULT                                                  0x00ff00ff
-#define ixDIDT_DB_CTRL2_DEFAULT                                                  0x18800004
-#define ixDIDT_DB_STALL_CTRL_DEFAULT                                             0x00fff000
-#define ixDIDT_DB_TUNING_CTRL_DEFAULT                                            0x00010004
-#define ixDIDT_DB_STALL_AUTO_RELEASE_CTRL_DEFAULT                                0x00ffffff
-#define ixDIDT_DB_CTRL3_DEFAULT                                                  0x00038000
-#define ixDIDT_DB_STALL_PATTERN_1_2_DEFAULT                                      0x01010001
-#define ixDIDT_DB_STALL_PATTERN_3_4_DEFAULT                                      0x11110421
-#define ixDIDT_DB_STALL_PATTERN_5_6_DEFAULT                                      0x25291249
-#define ixDIDT_DB_STALL_PATTERN_7_DEFAULT                                        0x00002aaa
-#define ixDIDT_DB_WEIGHT0_3_DEFAULT                                              0x00000000
-#define ixDIDT_DB_WEIGHT4_7_DEFAULT                                              0x00000000
-#define ixDIDT_DB_WEIGHT8_11_DEFAULT                                             0x00000000
-#define ixDIDT_DB_EDC_CTRL_DEFAULT                                               0x00001c00
-#define ixDIDT_DB_EDC_THRESHOLD_DEFAULT                                          0x00000000
-#define ixDIDT_DB_EDC_STALL_PATTERN_1_2_DEFAULT                                  0x01010001
-#define ixDIDT_DB_EDC_STALL_PATTERN_3_4_DEFAULT                                  0x11110421
-#define ixDIDT_DB_EDC_STALL_PATTERN_5_6_DEFAULT                                  0x25291249
-#define ixDIDT_DB_EDC_STALL_PATTERN_7_DEFAULT                                    0x00002aaa
-#define ixDIDT_DB_EDC_STATUS_DEFAULT                                             0x00000000
-#define ixDIDT_DB_EDC_STALL_DELAY_1_DEFAULT                                      0x00000000
-#define ixDIDT_DB_EDC_OVERFLOW_DEFAULT                                           0x00000000
-#define ixDIDT_DB_EDC_ROLLING_POWER_DELTA_DEFAULT                                0x00000000
-#define ixDIDT_TD_CTRL0_DEFAULT                                                  0x0000ff00
-#define ixDIDT_TD_CTRL1_DEFAULT                                                  0x00ff00ff
-#define ixDIDT_TD_CTRL2_DEFAULT                                                  0x18800004
-#define ixDIDT_TD_STALL_CTRL_DEFAULT                                             0x00fff000
-#define ixDIDT_TD_TUNING_CTRL_DEFAULT                                            0x00010004
-#define ixDIDT_TD_STALL_AUTO_RELEASE_CTRL_DEFAULT                                0x00ffffff
-#define ixDIDT_TD_CTRL3_DEFAULT                                                  0x00038000
-#define ixDIDT_TD_STALL_PATTERN_1_2_DEFAULT                                      0x01010001
-#define ixDIDT_TD_STALL_PATTERN_3_4_DEFAULT                                      0x11110421
-#define ixDIDT_TD_STALL_PATTERN_5_6_DEFAULT                                      0x25291249
-#define ixDIDT_TD_STALL_PATTERN_7_DEFAULT                                        0x00002aaa
-#define ixDIDT_TD_WEIGHT0_3_DEFAULT                                              0x00000000
-#define ixDIDT_TD_WEIGHT4_7_DEFAULT                                              0x00000000
-#define ixDIDT_TD_WEIGHT8_11_DEFAULT                                             0x00000000
-#define ixDIDT_TD_EDC_CTRL_DEFAULT                                               0x00001c00
-#define ixDIDT_TD_EDC_THRESHOLD_DEFAULT                                          0x00000000
-#define ixDIDT_TD_EDC_STALL_PATTERN_1_2_DEFAULT                                  0x01010001
-#define ixDIDT_TD_EDC_STALL_PATTERN_3_4_DEFAULT                                  0x11110421
-#define ixDIDT_TD_EDC_STALL_PATTERN_5_6_DEFAULT                                  0x25291249
-#define ixDIDT_TD_EDC_STALL_PATTERN_7_DEFAULT                                    0x00002aaa
-#define ixDIDT_TD_EDC_STATUS_DEFAULT                                             0x00000000
-#define ixDIDT_TD_EDC_STALL_DELAY_1_DEFAULT                                      0x00000000
-#define ixDIDT_TD_EDC_STALL_DELAY_2_DEFAULT                                      0x00000000
-#define ixDIDT_TD_EDC_STALL_DELAY_3_DEFAULT                                      0x00000000
-#define ixDIDT_TD_EDC_OVERFLOW_DEFAULT                                           0x00000000
-#define ixDIDT_TD_EDC_ROLLING_POWER_DELTA_DEFAULT                                0x00000000
-#define ixDIDT_TCP_CTRL0_DEFAULT                                                 0x0000ff00
-#define ixDIDT_TCP_CTRL1_DEFAULT                                                 0x00ff00ff
-#define ixDIDT_TCP_CTRL2_DEFAULT                                                 0x18800004
-#define ixDIDT_TCP_STALL_CTRL_DEFAULT                                            0x00fff000
-#define ixDIDT_TCP_TUNING_CTRL_DEFAULT                                           0x00010004
-#define ixDIDT_TCP_STALL_AUTO_RELEASE_CTRL_DEFAULT                               0x00ffffff
-#define ixDIDT_TCP_CTRL3_DEFAULT                                                 0x00038000
-#define ixDIDT_TCP_STALL_PATTERN_1_2_DEFAULT                                     0x01010001
-#define ixDIDT_TCP_STALL_PATTERN_3_4_DEFAULT                                     0x11110421
-#define ixDIDT_TCP_STALL_PATTERN_5_6_DEFAULT                                     0x25291249
-#define ixDIDT_TCP_STALL_PATTERN_7_DEFAULT                                       0x00002aaa
-#define ixDIDT_TCP_WEIGHT0_3_DEFAULT                                             0x00000000
-#define ixDIDT_TCP_WEIGHT4_7_DEFAULT                                             0x00000000
-#define ixDIDT_TCP_WEIGHT8_11_DEFAULT                                            0x00000000
-#define ixDIDT_TCP_EDC_CTRL_DEFAULT                                              0x00001c00
-#define ixDIDT_TCP_EDC_THRESHOLD_DEFAULT                                         0x00000000
-#define ixDIDT_TCP_EDC_STALL_PATTERN_1_2_DEFAULT                                 0x01010001
-#define ixDIDT_TCP_EDC_STALL_PATTERN_3_4_DEFAULT                                 0x11110421
-#define ixDIDT_TCP_EDC_STALL_PATTERN_5_6_DEFAULT                                 0x25291249
-#define ixDIDT_TCP_EDC_STALL_PATTERN_7_DEFAULT                                   0x00002aaa
-#define ixDIDT_TCP_EDC_STATUS_DEFAULT                                            0x00000000
-#define ixDIDT_TCP_EDC_STALL_DELAY_1_DEFAULT                                     0x00000000
-#define ixDIDT_TCP_EDC_STALL_DELAY_2_DEFAULT                                     0x00000000
-#define ixDIDT_TCP_EDC_STALL_DELAY_3_DEFAULT                                     0x00000000
-#define ixDIDT_TCP_EDC_OVERFLOW_DEFAULT                                          0x00000000
-#define ixDIDT_TCP_EDC_ROLLING_POWER_DELTA_DEFAULT                               0x00000000
-#define ixDIDT_DBR_CTRL0_DEFAULT                                                 0x0000ff00
-#define ixDIDT_DBR_CTRL1_DEFAULT                                                 0x00ff00ff
-#define ixDIDT_DBR_CTRL2_DEFAULT                                                 0x18800004
-#define ixDIDT_DBR_STALL_CTRL_DEFAULT                                            0x00fff000
-#define ixDIDT_DBR_TUNING_CTRL_DEFAULT                                           0x00010004
-#define ixDIDT_DBR_STALL_AUTO_RELEASE_CTRL_DEFAULT                               0x00ffffff
-#define ixDIDT_DBR_CTRL3_DEFAULT                                                 0x00038000
-#define ixDIDT_DBR_STALL_PATTERN_1_2_DEFAULT                                     0x01010001
-#define ixDIDT_DBR_STALL_PATTERN_3_4_DEFAULT                                     0x11110421
-#define ixDIDT_DBR_STALL_PATTERN_5_6_DEFAULT                                     0x25291249
-#define ixDIDT_DBR_STALL_PATTERN_7_DEFAULT                                       0x00002aaa
-#define ixDIDT_DBR_WEIGHT0_3_DEFAULT                                             0x00000000
-#define ixDIDT_DBR_WEIGHT4_7_DEFAULT                                             0x00000000
-#define ixDIDT_DBR_WEIGHT8_11_DEFAULT                                            0x00000000
-#define ixDIDT_DBR_EDC_CTRL_DEFAULT                                              0x00001c00
-#define ixDIDT_DBR_EDC_THRESHOLD_DEFAULT                                         0x00000000
-#define ixDIDT_DBR_EDC_STALL_PATTERN_1_2_DEFAULT                                 0x01010001
-#define ixDIDT_DBR_EDC_STALL_PATTERN_3_4_DEFAULT                                 0x11110421
-#define ixDIDT_DBR_EDC_STALL_PATTERN_5_6_DEFAULT                                 0x25291249
-#define ixDIDT_DBR_EDC_STALL_PATTERN_7_DEFAULT                                   0x00002aaa
-#define ixDIDT_DBR_EDC_STATUS_DEFAULT                                            0x00000000
-#define ixDIDT_DBR_EDC_STALL_DELAY_1_DEFAULT                                     0x00000000
-#define ixDIDT_DBR_EDC_OVERFLOW_DEFAULT                                          0x00000000
-#define ixDIDT_DBR_EDC_ROLLING_POWER_DELTA_DEFAULT                               0x00000000
-#define ixDIDT_SQ_STALL_EVENT_COUNTER_DEFAULT                                    0x00000000
-#define ixDIDT_DB_STALL_EVENT_COUNTER_DEFAULT                                    0x00000000
-#define ixDIDT_TD_STALL_EVENT_COUNTER_DEFAULT                                    0x00000000
-#define ixDIDT_TCP_STALL_EVENT_COUNTER_DEFAULT                                   0x00000000
-#define ixDIDT_DBR_STALL_EVENT_COUNTER_DEFAULT                                   0x00000000
-
-
-
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_sh_mask.h
deleted file mode 100644
index ab0a25eba483..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/raven1/GC/gc_9_1_sh_mask.h
+++ /dev/null
@@ -1,31191 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _gc_9_1_SH_MASK_HEADER
-#define _gc_9_1_SH_MASK_HEADER
-
-
-// addressBlock: gc_grbmdec
-//GRBM_CNTL
-#define GRBM_CNTL__READ_TIMEOUT__SHIFT                                                                        0x0
-#define GRBM_CNTL__REPORT_LAST_RDERR__SHIFT                                                                   0x1f
-#define GRBM_CNTL__READ_TIMEOUT_MASK                                                                          0x000000FFL
-#define GRBM_CNTL__REPORT_LAST_RDERR_MASK                                                                     0x80000000L
-//GRBM_SKEW_CNTL
-#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD__SHIFT                                                             0x0
-#define GRBM_SKEW_CNTL__SKEW_COUNT__SHIFT                                                                     0x6
-#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD_MASK                                                               0x0000003FL
-#define GRBM_SKEW_CNTL__SKEW_COUNT_MASK                                                                       0x00000FC0L
-//GRBM_STATUS2
-#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL__SHIFT                                                           0x0
-#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING__SHIFT                                                           0x4
-#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING__SHIFT                                                           0x5
-#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING__SHIFT                                                              0x6
-#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING__SHIFT                                                              0x7
-#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING__SHIFT                                                              0x8
-#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING__SHIFT                                                              0x9
-#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING__SHIFT                                                              0xa
-#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING__SHIFT                                                              0xb
-#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING__SHIFT                                                              0xc
-#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING__SHIFT                                                              0xd
-#define GRBM_STATUS2__RLC_RQ_PENDING__SHIFT                                                                   0xe
-#define GRBM_STATUS2__UTCL2_BUSY__SHIFT                                                                       0xf
-#define GRBM_STATUS2__EA_BUSY__SHIFT                                                                          0x10
-#define GRBM_STATUS2__RMI_BUSY__SHIFT                                                                         0x11
-#define GRBM_STATUS2__UTCL2_RQ_PENDING__SHIFT                                                                 0x12
-#define GRBM_STATUS2__CPF_RQ_PENDING__SHIFT                                                                   0x13
-#define GRBM_STATUS2__EA_LINK_BUSY__SHIFT                                                                     0x14
-#define GRBM_STATUS2__RLC_BUSY__SHIFT                                                                         0x18
-#define GRBM_STATUS2__TC_BUSY__SHIFT                                                                          0x19
-#define GRBM_STATUS2__TCC_CC_RESIDENT__SHIFT                                                                  0x1a
-#define GRBM_STATUS2__CPF_BUSY__SHIFT                                                                         0x1c
-#define GRBM_STATUS2__CPC_BUSY__SHIFT                                                                         0x1d
-#define GRBM_STATUS2__CPG_BUSY__SHIFT                                                                         0x1e
-#define GRBM_STATUS2__CPAXI_BUSY__SHIFT                                                                       0x1f
-#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL_MASK                                                             0x0000000FL
-#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING_MASK                                                             0x00000010L
-#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING_MASK                                                             0x00000020L
-#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING_MASK                                                                0x00000040L
-#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING_MASK                                                                0x00000080L
-#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING_MASK                                                                0x00000100L
-#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING_MASK                                                                0x00000200L
-#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING_MASK                                                                0x00000400L
-#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING_MASK                                                                0x00000800L
-#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING_MASK                                                                0x00001000L
-#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING_MASK                                                                0x00002000L
-#define GRBM_STATUS2__RLC_RQ_PENDING_MASK                                                                     0x00004000L
-#define GRBM_STATUS2__UTCL2_BUSY_MASK                                                                         0x00008000L
-#define GRBM_STATUS2__EA_BUSY_MASK                                                                            0x00010000L
-#define GRBM_STATUS2__RMI_BUSY_MASK                                                                           0x00020000L
-#define GRBM_STATUS2__UTCL2_RQ_PENDING_MASK                                                                   0x00040000L
-#define GRBM_STATUS2__CPF_RQ_PENDING_MASK                                                                     0x00080000L
-#define GRBM_STATUS2__EA_LINK_BUSY_MASK                                                                       0x00100000L
-#define GRBM_STATUS2__RLC_BUSY_MASK                                                                           0x01000000L
-#define GRBM_STATUS2__TC_BUSY_MASK                                                                            0x02000000L
-#define GRBM_STATUS2__TCC_CC_RESIDENT_MASK                                                                    0x04000000L
-#define GRBM_STATUS2__CPF_BUSY_MASK                                                                           0x10000000L
-#define GRBM_STATUS2__CPC_BUSY_MASK                                                                           0x20000000L
-#define GRBM_STATUS2__CPG_BUSY_MASK                                                                           0x40000000L
-#define GRBM_STATUS2__CPAXI_BUSY_MASK                                                                         0x80000000L
-//GRBM_PWR_CNTL
-#define GRBM_PWR_CNTL__ALL_REQ_TYPE__SHIFT                                                                    0x0
-#define GRBM_PWR_CNTL__GFX_REQ_TYPE__SHIFT                                                                    0x2
-#define GRBM_PWR_CNTL__ALL_RSP_TYPE__SHIFT                                                                    0x4
-#define GRBM_PWR_CNTL__GFX_RSP_TYPE__SHIFT                                                                    0x6
-#define GRBM_PWR_CNTL__GFX_REQ_EN__SHIFT                                                                      0xe
-#define GRBM_PWR_CNTL__ALL_REQ_EN__SHIFT                                                                      0xf
-#define GRBM_PWR_CNTL__ALL_REQ_TYPE_MASK                                                                      0x00000003L
-#define GRBM_PWR_CNTL__GFX_REQ_TYPE_MASK                                                                      0x0000000CL
-#define GRBM_PWR_CNTL__ALL_RSP_TYPE_MASK                                                                      0x00000030L
-#define GRBM_PWR_CNTL__GFX_RSP_TYPE_MASK                                                                      0x000000C0L
-#define GRBM_PWR_CNTL__GFX_REQ_EN_MASK                                                                        0x00004000L
-#define GRBM_PWR_CNTL__ALL_REQ_EN_MASK                                                                        0x00008000L
-//GRBM_STATUS
-#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL__SHIFT                                                            0x0
-#define GRBM_STATUS__RSMU_RQ_PENDING__SHIFT                                                                   0x5
-#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING__SHIFT                                                            0x7
-#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING__SHIFT                                                            0x8
-#define GRBM_STATUS__GDS_DMA_RQ_PENDING__SHIFT                                                                0x9
-#define GRBM_STATUS__DB_CLEAN__SHIFT                                                                          0xc
-#define GRBM_STATUS__CB_CLEAN__SHIFT                                                                          0xd
-#define GRBM_STATUS__TA_BUSY__SHIFT                                                                           0xe
-#define GRBM_STATUS__GDS_BUSY__SHIFT                                                                          0xf
-#define GRBM_STATUS__WD_BUSY_NO_DMA__SHIFT                                                                    0x10
-#define GRBM_STATUS__VGT_BUSY__SHIFT                                                                          0x11
-#define GRBM_STATUS__IA_BUSY_NO_DMA__SHIFT                                                                    0x12
-#define GRBM_STATUS__IA_BUSY__SHIFT                                                                           0x13
-#define GRBM_STATUS__SX_BUSY__SHIFT                                                                           0x14
-#define GRBM_STATUS__WD_BUSY__SHIFT                                                                           0x15
-#define GRBM_STATUS__SPI_BUSY__SHIFT                                                                          0x16
-#define GRBM_STATUS__BCI_BUSY__SHIFT                                                                          0x17
-#define GRBM_STATUS__SC_BUSY__SHIFT                                                                           0x18
-#define GRBM_STATUS__PA_BUSY__SHIFT                                                                           0x19
-#define GRBM_STATUS__DB_BUSY__SHIFT                                                                           0x1a
-#define GRBM_STATUS__CP_COHERENCY_BUSY__SHIFT                                                                 0x1c
-#define GRBM_STATUS__CP_BUSY__SHIFT                                                                           0x1d
-#define GRBM_STATUS__CB_BUSY__SHIFT                                                                           0x1e
-#define GRBM_STATUS__GUI_ACTIVE__SHIFT                                                                        0x1f
-#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL_MASK                                                              0x0000000FL
-#define GRBM_STATUS__RSMU_RQ_PENDING_MASK                                                                     0x00000020L
-#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING_MASK                                                              0x00000080L
-#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING_MASK                                                              0x00000100L
-#define GRBM_STATUS__GDS_DMA_RQ_PENDING_MASK                                                                  0x00000200L
-#define GRBM_STATUS__DB_CLEAN_MASK                                                                            0x00001000L
-#define GRBM_STATUS__CB_CLEAN_MASK                                                                            0x00002000L
-#define GRBM_STATUS__TA_BUSY_MASK                                                                             0x00004000L
-#define GRBM_STATUS__GDS_BUSY_MASK                                                                            0x00008000L
-#define GRBM_STATUS__WD_BUSY_NO_DMA_MASK                                                                      0x00010000L
-#define GRBM_STATUS__VGT_BUSY_MASK                                                                            0x00020000L
-#define GRBM_STATUS__IA_BUSY_NO_DMA_MASK                                                                      0x00040000L
-#define GRBM_STATUS__IA_BUSY_MASK                                                                             0x00080000L
-#define GRBM_STATUS__SX_BUSY_MASK                                                                             0x00100000L
-#define GRBM_STATUS__WD_BUSY_MASK                                                                             0x00200000L
-#define GRBM_STATUS__SPI_BUSY_MASK                                                                            0x00400000L
-#define GRBM_STATUS__BCI_BUSY_MASK                                                                            0x00800000L
-#define GRBM_STATUS__SC_BUSY_MASK                                                                             0x01000000L
-#define GRBM_STATUS__PA_BUSY_MASK                                                                             0x02000000L
-#define GRBM_STATUS__DB_BUSY_MASK                                                                             0x04000000L
-#define GRBM_STATUS__CP_COHERENCY_BUSY_MASK                                                                   0x10000000L
-#define GRBM_STATUS__CP_BUSY_MASK                                                                             0x20000000L
-#define GRBM_STATUS__CB_BUSY_MASK                                                                             0x40000000L
-#define GRBM_STATUS__GUI_ACTIVE_MASK                                                                          0x80000000L
-//GRBM_STATUS_SE0
-#define GRBM_STATUS_SE0__DB_CLEAN__SHIFT                                                                      0x1
-#define GRBM_STATUS_SE0__CB_CLEAN__SHIFT                                                                      0x2
-#define GRBM_STATUS_SE0__RMI_BUSY__SHIFT                                                                      0x15
-#define GRBM_STATUS_SE0__BCI_BUSY__SHIFT                                                                      0x16
-#define GRBM_STATUS_SE0__VGT_BUSY__SHIFT                                                                      0x17
-#define GRBM_STATUS_SE0__PA_BUSY__SHIFT                                                                       0x18
-#define GRBM_STATUS_SE0__TA_BUSY__SHIFT                                                                       0x19
-#define GRBM_STATUS_SE0__SX_BUSY__SHIFT                                                                       0x1a
-#define GRBM_STATUS_SE0__SPI_BUSY__SHIFT                                                                      0x1b
-#define GRBM_STATUS_SE0__SC_BUSY__SHIFT                                                                       0x1d
-#define GRBM_STATUS_SE0__DB_BUSY__SHIFT                                                                       0x1e
-#define GRBM_STATUS_SE0__CB_BUSY__SHIFT                                                                       0x1f
-#define GRBM_STATUS_SE0__DB_CLEAN_MASK                                                                        0x00000002L
-#define GRBM_STATUS_SE0__CB_CLEAN_MASK                                                                        0x00000004L
-#define GRBM_STATUS_SE0__RMI_BUSY_MASK                                                                        0x00200000L
-#define GRBM_STATUS_SE0__BCI_BUSY_MASK                                                                        0x00400000L
-#define GRBM_STATUS_SE0__VGT_BUSY_MASK                                                                        0x00800000L
-#define GRBM_STATUS_SE0__PA_BUSY_MASK                                                                         0x01000000L
-#define GRBM_STATUS_SE0__TA_BUSY_MASK                                                                         0x02000000L
-#define GRBM_STATUS_SE0__SX_BUSY_MASK                                                                         0x04000000L
-#define GRBM_STATUS_SE0__SPI_BUSY_MASK                                                                        0x08000000L
-#define GRBM_STATUS_SE0__SC_BUSY_MASK                                                                         0x20000000L
-#define GRBM_STATUS_SE0__DB_BUSY_MASK                                                                         0x40000000L
-#define GRBM_STATUS_SE0__CB_BUSY_MASK                                                                         0x80000000L
-//GRBM_STATUS_SE1
-#define GRBM_STATUS_SE1__DB_CLEAN__SHIFT                                                                      0x1
-#define GRBM_STATUS_SE1__CB_CLEAN__SHIFT                                                                      0x2
-#define GRBM_STATUS_SE1__RMI_BUSY__SHIFT                                                                      0x15
-#define GRBM_STATUS_SE1__BCI_BUSY__SHIFT                                                                      0x16
-#define GRBM_STATUS_SE1__VGT_BUSY__SHIFT                                                                      0x17
-#define GRBM_STATUS_SE1__PA_BUSY__SHIFT                                                                       0x18
-#define GRBM_STATUS_SE1__TA_BUSY__SHIFT                                                                       0x19
-#define GRBM_STATUS_SE1__SX_BUSY__SHIFT                                                                       0x1a
-#define GRBM_STATUS_SE1__SPI_BUSY__SHIFT                                                                      0x1b
-#define GRBM_STATUS_SE1__SC_BUSY__SHIFT                                                                       0x1d
-#define GRBM_STATUS_SE1__DB_BUSY__SHIFT                                                                       0x1e
-#define GRBM_STATUS_SE1__CB_BUSY__SHIFT                                                                       0x1f
-#define GRBM_STATUS_SE1__DB_CLEAN_MASK                                                                        0x00000002L
-#define GRBM_STATUS_SE1__CB_CLEAN_MASK                                                                        0x00000004L
-#define GRBM_STATUS_SE1__RMI_BUSY_MASK                                                                        0x00200000L
-#define GRBM_STATUS_SE1__BCI_BUSY_MASK                                                                        0x00400000L
-#define GRBM_STATUS_SE1__VGT_BUSY_MASK                                                                        0x00800000L
-#define GRBM_STATUS_SE1__PA_BUSY_MASK                                                                         0x01000000L
-#define GRBM_STATUS_SE1__TA_BUSY_MASK                                                                         0x02000000L
-#define GRBM_STATUS_SE1__SX_BUSY_MASK                                                                         0x04000000L
-#define GRBM_STATUS_SE1__SPI_BUSY_MASK                                                                        0x08000000L
-#define GRBM_STATUS_SE1__SC_BUSY_MASK                                                                         0x20000000L
-#define GRBM_STATUS_SE1__DB_BUSY_MASK                                                                         0x40000000L
-#define GRBM_STATUS_SE1__CB_BUSY_MASK                                                                         0x80000000L
-//GRBM_SOFT_RESET
-#define GRBM_SOFT_RESET__SOFT_RESET_CP__SHIFT                                                                 0x0
-#define GRBM_SOFT_RESET__SOFT_RESET_RLC__SHIFT                                                                0x2
-#define GRBM_SOFT_RESET__SOFT_RESET_GFX__SHIFT                                                                0x10
-#define GRBM_SOFT_RESET__SOFT_RESET_CPF__SHIFT                                                                0x11
-#define GRBM_SOFT_RESET__SOFT_RESET_CPC__SHIFT                                                                0x12
-#define GRBM_SOFT_RESET__SOFT_RESET_CPG__SHIFT                                                                0x13
-#define GRBM_SOFT_RESET__SOFT_RESET_CAC__SHIFT                                                                0x14
-#define GRBM_SOFT_RESET__SOFT_RESET_CPAXI__SHIFT                                                              0x15
-#define GRBM_SOFT_RESET__SOFT_RESET_EA__SHIFT                                                                 0x16
-#define GRBM_SOFT_RESET__SOFT_RESET_CP_MASK                                                                   0x00000001L
-#define GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK                                                                  0x00000004L
-#define GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK                                                                  0x00010000L
-#define GRBM_SOFT_RESET__SOFT_RESET_CPF_MASK                                                                  0x00020000L
-#define GRBM_SOFT_RESET__SOFT_RESET_CPC_MASK                                                                  0x00040000L
-#define GRBM_SOFT_RESET__SOFT_RESET_CPG_MASK                                                                  0x00080000L
-#define GRBM_SOFT_RESET__SOFT_RESET_CAC_MASK                                                                  0x00100000L
-#define GRBM_SOFT_RESET__SOFT_RESET_CPAXI_MASK                                                                0x00200000L
-#define GRBM_SOFT_RESET__SOFT_RESET_EA_MASK                                                                   0x00400000L
-//GRBM_CGTT_CLK_CNTL
-#define GRBM_CGTT_CLK_CNTL__ON_DELAY__SHIFT                                                                   0x0
-#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
-#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN__SHIFT                                                          0x1e
-#define GRBM_CGTT_CLK_CNTL__ON_DELAY_MASK                                                                     0x0000000FL
-#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
-#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN_MASK                                                            0x40000000L
-//GRBM_GFX_CLKEN_CNTL
-#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT                                                          0x0
-#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT                                                            0x8
-#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK                                                            0x0000000FL
-#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK                                                              0x00001F00L
-//GRBM_WAIT_IDLE_CLOCKS
-#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS__SHIFT                                                        0x0
-#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS_MASK                                                          0x000000FFL
-//GRBM_STATUS_SE2
-#define GRBM_STATUS_SE2__DB_CLEAN__SHIFT                                                                      0x1
-#define GRBM_STATUS_SE2__CB_CLEAN__SHIFT                                                                      0x2
-#define GRBM_STATUS_SE2__RMI_BUSY__SHIFT                                                                      0x15
-#define GRBM_STATUS_SE2__BCI_BUSY__SHIFT                                                                      0x16
-#define GRBM_STATUS_SE2__VGT_BUSY__SHIFT                                                                      0x17
-#define GRBM_STATUS_SE2__PA_BUSY__SHIFT                                                                       0x18
-#define GRBM_STATUS_SE2__TA_BUSY__SHIFT                                                                       0x19
-#define GRBM_STATUS_SE2__SX_BUSY__SHIFT                                                                       0x1a
-#define GRBM_STATUS_SE2__SPI_BUSY__SHIFT                                                                      0x1b
-#define GRBM_STATUS_SE2__SC_BUSY__SHIFT                                                                       0x1d
-#define GRBM_STATUS_SE2__DB_BUSY__SHIFT                                                                       0x1e
-#define GRBM_STATUS_SE2__CB_BUSY__SHIFT                                                                       0x1f
-#define GRBM_STATUS_SE2__DB_CLEAN_MASK                                                                        0x00000002L
-#define GRBM_STATUS_SE2__CB_CLEAN_MASK                                                                        0x00000004L
-#define GRBM_STATUS_SE2__RMI_BUSY_MASK                                                                        0x00200000L
-#define GRBM_STATUS_SE2__BCI_BUSY_MASK                                                                        0x00400000L
-#define GRBM_STATUS_SE2__VGT_BUSY_MASK                                                                        0x00800000L
-#define GRBM_STATUS_SE2__PA_BUSY_MASK                                                                         0x01000000L
-#define GRBM_STATUS_SE2__TA_BUSY_MASK                                                                         0x02000000L
-#define GRBM_STATUS_SE2__SX_BUSY_MASK                                                                         0x04000000L
-#define GRBM_STATUS_SE2__SPI_BUSY_MASK                                                                        0x08000000L
-#define GRBM_STATUS_SE2__SC_BUSY_MASK                                                                         0x20000000L
-#define GRBM_STATUS_SE2__DB_BUSY_MASK                                                                         0x40000000L
-#define GRBM_STATUS_SE2__CB_BUSY_MASK                                                                         0x80000000L
-//GRBM_STATUS_SE3
-#define GRBM_STATUS_SE3__DB_CLEAN__SHIFT                                                                      0x1
-#define GRBM_STATUS_SE3__CB_CLEAN__SHIFT                                                                      0x2
-#define GRBM_STATUS_SE3__RMI_BUSY__SHIFT                                                                      0x15
-#define GRBM_STATUS_SE3__BCI_BUSY__SHIFT                                                                      0x16
-#define GRBM_STATUS_SE3__VGT_BUSY__SHIFT                                                                      0x17
-#define GRBM_STATUS_SE3__PA_BUSY__SHIFT                                                                       0x18
-#define GRBM_STATUS_SE3__TA_BUSY__SHIFT                                                                       0x19
-#define GRBM_STATUS_SE3__SX_BUSY__SHIFT                                                                       0x1a
-#define GRBM_STATUS_SE3__SPI_BUSY__SHIFT                                                                      0x1b
-#define GRBM_STATUS_SE3__SC_BUSY__SHIFT                                                                       0x1d
-#define GRBM_STATUS_SE3__DB_BUSY__SHIFT                                                                       0x1e
-#define GRBM_STATUS_SE3__CB_BUSY__SHIFT                                                                       0x1f
-#define GRBM_STATUS_SE3__DB_CLEAN_MASK                                                                        0x00000002L
-#define GRBM_STATUS_SE3__CB_CLEAN_MASK                                                                        0x00000004L
-#define GRBM_STATUS_SE3__RMI_BUSY_MASK                                                                        0x00200000L
-#define GRBM_STATUS_SE3__BCI_BUSY_MASK                                                                        0x00400000L
-#define GRBM_STATUS_SE3__VGT_BUSY_MASK                                                                        0x00800000L
-#define GRBM_STATUS_SE3__PA_BUSY_MASK                                                                         0x01000000L
-#define GRBM_STATUS_SE3__TA_BUSY_MASK                                                                         0x02000000L
-#define GRBM_STATUS_SE3__SX_BUSY_MASK                                                                         0x04000000L
-#define GRBM_STATUS_SE3__SPI_BUSY_MASK                                                                        0x08000000L
-#define GRBM_STATUS_SE3__SC_BUSY_MASK                                                                         0x20000000L
-#define GRBM_STATUS_SE3__DB_BUSY_MASK                                                                         0x40000000L
-#define GRBM_STATUS_SE3__CB_BUSY_MASK                                                                         0x80000000L
-//GRBM_READ_ERROR
-#define GRBM_READ_ERROR__READ_ADDRESS__SHIFT                                                                  0x2
-#define GRBM_READ_ERROR__READ_PIPEID__SHIFT                                                                   0x14
-#define GRBM_READ_ERROR__READ_MEID__SHIFT                                                                     0x16
-#define GRBM_READ_ERROR__READ_ERROR__SHIFT                                                                    0x1f
-#define GRBM_READ_ERROR__READ_ADDRESS_MASK                                                                    0x0003FFFCL
-#define GRBM_READ_ERROR__READ_PIPEID_MASK                                                                     0x00300000L
-#define GRBM_READ_ERROR__READ_MEID_MASK                                                                       0x00C00000L
-#define GRBM_READ_ERROR__READ_ERROR_MASK                                                                      0x80000000L
-//GRBM_READ_ERROR2
-#define GRBM_READ_ERROR2__READ_REQUESTER_CPF__SHIFT                                                           0x10
-#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU__SHIFT                                                          0x11
-#define GRBM_READ_ERROR2__READ_REQUESTER_RLC__SHIFT                                                           0x12
-#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA__SHIFT                                                       0x13
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF__SHIFT                                                   0x14
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF__SHIFT                                                   0x15
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF__SHIFT                                                   0x16
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF__SHIFT                                                   0x17
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0__SHIFT                                                      0x18
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1__SHIFT                                                      0x19
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2__SHIFT                                                      0x1a
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3__SHIFT                                                      0x1b
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0__SHIFT                                                      0x1c
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1__SHIFT                                                      0x1d
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2__SHIFT                                                      0x1e
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3__SHIFT                                                      0x1f
-#define GRBM_READ_ERROR2__READ_REQUESTER_CPF_MASK                                                             0x00010000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU_MASK                                                            0x00020000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_RLC_MASK                                                             0x00040000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA_MASK                                                         0x00080000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF_MASK                                                     0x00100000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF_MASK                                                     0x00200000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF_MASK                                                     0x00400000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF_MASK                                                     0x00800000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0_MASK                                                        0x01000000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1_MASK                                                        0x02000000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2_MASK                                                        0x04000000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3_MASK                                                        0x08000000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0_MASK                                                        0x10000000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1_MASK                                                        0x20000000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2_MASK                                                        0x40000000L
-#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3_MASK                                                        0x80000000L
-//GRBM_INT_CNTL
-#define GRBM_INT_CNTL__RDERR_INT_ENABLE__SHIFT                                                                0x0
-#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE__SHIFT                                                             0x13
-#define GRBM_INT_CNTL__RDERR_INT_ENABLE_MASK                                                                  0x00000001L
-#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE_MASK                                                               0x00080000L
-//GRBM_TRAP_OP
-#define GRBM_TRAP_OP__RW__SHIFT                                                                               0x0
-#define GRBM_TRAP_OP__RW_MASK                                                                                 0x00000001L
-//GRBM_TRAP_ADDR
-#define GRBM_TRAP_ADDR__DATA__SHIFT                                                                           0x0
-#define GRBM_TRAP_ADDR__DATA_MASK                                                                             0x0003FFFFL
-//GRBM_TRAP_ADDR_MSK
-#define GRBM_TRAP_ADDR_MSK__DATA__SHIFT                                                                       0x0
-#define GRBM_TRAP_ADDR_MSK__DATA_MASK                                                                         0x0003FFFFL
-//GRBM_TRAP_WD
-#define GRBM_TRAP_WD__DATA__SHIFT                                                                             0x0
-#define GRBM_TRAP_WD__DATA_MASK                                                                               0xFFFFFFFFL
-//GRBM_TRAP_WD_MSK
-#define GRBM_TRAP_WD_MSK__DATA__SHIFT                                                                         0x0
-#define GRBM_TRAP_WD_MSK__DATA_MASK                                                                           0xFFFFFFFFL
-//GRBM_DSM_BYPASS
-#define GRBM_DSM_BYPASS__BYPASS_BITS__SHIFT                                                                   0x0
-#define GRBM_DSM_BYPASS__BYPASS_EN__SHIFT                                                                     0x2
-#define GRBM_DSM_BYPASS__BYPASS_BITS_MASK                                                                     0x00000003L
-#define GRBM_DSM_BYPASS__BYPASS_EN_MASK                                                                       0x00000004L
-//GRBM_WRITE_ERROR
-#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC__SHIFT                                                          0x0
-#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU__SHIFT                                                         0x1
-#define GRBM_WRITE_ERROR__WRITE_SSRCID__SHIFT                                                                 0x2
-#define GRBM_WRITE_ERROR__WRITE_VFID__SHIFT                                                                   0x5
-#define GRBM_WRITE_ERROR__WRITE_VF__SHIFT                                                                     0xc
-#define GRBM_WRITE_ERROR__WRITE_VMID__SHIFT                                                                   0xd
-#define GRBM_WRITE_ERROR__WRITE_PIPEID__SHIFT                                                                 0x14
-#define GRBM_WRITE_ERROR__WRITE_MEID__SHIFT                                                                   0x16
-#define GRBM_WRITE_ERROR__WRITE_ERROR__SHIFT                                                                  0x1f
-#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC_MASK                                                            0x00000001L
-#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU_MASK                                                           0x00000002L
-#define GRBM_WRITE_ERROR__WRITE_SSRCID_MASK                                                                   0x0000001CL
-#define GRBM_WRITE_ERROR__WRITE_VFID_MASK                                                                     0x000001E0L
-#define GRBM_WRITE_ERROR__WRITE_VF_MASK                                                                       0x00001000L
-#define GRBM_WRITE_ERROR__WRITE_VMID_MASK                                                                     0x0001E000L
-#define GRBM_WRITE_ERROR__WRITE_PIPEID_MASK                                                                   0x00300000L
-#define GRBM_WRITE_ERROR__WRITE_MEID_MASK                                                                     0x00C00000L
-#define GRBM_WRITE_ERROR__WRITE_ERROR_MASK                                                                    0x80000000L
-//GRBM_IOV_ERROR
-#define GRBM_IOV_ERROR__IOV_ADDR__SHIFT                                                                       0x2
-#define GRBM_IOV_ERROR__IOV_VFID__SHIFT                                                                       0x14
-#define GRBM_IOV_ERROR__IOV_VF__SHIFT                                                                         0x1a
-#define GRBM_IOV_ERROR__IOV_OP__SHIFT                                                                         0x1b
-#define GRBM_IOV_ERROR__IOV_ERROR__SHIFT                                                                      0x1f
-#define GRBM_IOV_ERROR__IOV_ADDR_MASK                                                                         0x000FFFFCL
-#define GRBM_IOV_ERROR__IOV_VFID_MASK                                                                         0x03F00000L
-#define GRBM_IOV_ERROR__IOV_VF_MASK                                                                           0x04000000L
-#define GRBM_IOV_ERROR__IOV_OP_MASK                                                                           0x08000000L
-#define GRBM_IOV_ERROR__IOV_ERROR_MASK                                                                        0x80000000L
-//GRBM_CHIP_REVISION
-#define GRBM_CHIP_REVISION__CHIP_REVISION__SHIFT                                                              0x0
-#define GRBM_CHIP_REVISION__CHIP_REVISION_MASK                                                                0x000000FFL
-//GRBM_GFX_CNTL
-#define GRBM_GFX_CNTL__PIPEID__SHIFT                                                                          0x0
-#define GRBM_GFX_CNTL__MEID__SHIFT                                                                            0x2
-#define GRBM_GFX_CNTL__VMID__SHIFT                                                                            0x4
-#define GRBM_GFX_CNTL__QUEUEID__SHIFT                                                                         0x8
-#define GRBM_GFX_CNTL__PIPEID_MASK                                                                            0x00000003L
-#define GRBM_GFX_CNTL__MEID_MASK                                                                              0x0000000CL
-#define GRBM_GFX_CNTL__VMID_MASK                                                                              0x000000F0L
-#define GRBM_GFX_CNTL__QUEUEID_MASK                                                                           0x00000700L
-//GRBM_RSMU_CFG
-#define GRBM_RSMU_CFG__APERTURE_ID__SHIFT                                                                     0x0
-#define GRBM_RSMU_CFG__QOS__SHIFT                                                                             0xc
-#define GRBM_RSMU_CFG__POSTED_WR__SHIFT                                                                       0x10
-#define GRBM_RSMU_CFG__APERTURE_ID_MASK                                                                       0x00000FFFL
-#define GRBM_RSMU_CFG__QOS_MASK                                                                               0x0000F000L
-#define GRBM_RSMU_CFG__POSTED_WR_MASK                                                                         0x00010000L
-//GRBM_IH_CREDIT
-#define GRBM_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                   0x0
-#define GRBM_IH_CREDIT__IH_CLIENT_ID__SHIFT                                                                   0x10
-#define GRBM_IH_CREDIT__CREDIT_VALUE_MASK                                                                     0x00000003L
-#define GRBM_IH_CREDIT__IH_CLIENT_ID_MASK                                                                     0x00FF0000L
-//GRBM_PWR_CNTL2
-#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT__SHIFT                                                               0x10
-#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT__SHIFT                                                         0x14
-#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT_MASK                                                                 0x00010000L
-#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT_MASK                                                           0x00100000L
-//GRBM_UTCL2_INVAL_RANGE_START
-#define GRBM_UTCL2_INVAL_RANGE_START__DATA__SHIFT                                                             0x0
-#define GRBM_UTCL2_INVAL_RANGE_START__DATA_MASK                                                               0x0003FFFFL
-//GRBM_UTCL2_INVAL_RANGE_END
-#define GRBM_UTCL2_INVAL_RANGE_END__DATA__SHIFT                                                               0x0
-#define GRBM_UTCL2_INVAL_RANGE_END__DATA_MASK                                                                 0x0003FFFFL
-//GRBM_RSMU_READ_ERROR
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS__SHIFT                                                        0x2
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF__SHIFT                                                             0x14
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID__SHIFT                                                           0x15
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE__SHIFT                                                     0x1b
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR__SHIFT                                                          0x1f
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS_MASK                                                          0x000FFFFCL
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF_MASK                                                               0x00100000L
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID_MASK                                                             0x07E00000L
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE_MASK                                                       0x08000000L
-#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_MASK                                                            0x80000000L
-//GRBM_CHICKEN_BITS
-#define GRBM_CHICKEN_BITS__DISABLE_CP_VMID_RESET_REQ__SHIFT                                                   0x0
-#define GRBM_CHICKEN_BITS__DISABLE_CP_VMID_RESET_REQ_MASK                                                     0x00000001L
-//GRBM_NOWHERE
-#define GRBM_NOWHERE__DATA__SHIFT                                                                             0x0
-#define GRBM_NOWHERE__DATA_MASK                                                                               0xFFFFFFFFL
-//GRBM_SCRATCH_REG0
-#define GRBM_SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG0__SCRATCH_REG0_MASK                                                                  0xFFFFFFFFL
-//GRBM_SCRATCH_REG1
-#define GRBM_SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG1__SCRATCH_REG1_MASK                                                                  0xFFFFFFFFL
-//GRBM_SCRATCH_REG2
-#define GRBM_SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG2__SCRATCH_REG2_MASK                                                                  0xFFFFFFFFL
-//GRBM_SCRATCH_REG3
-#define GRBM_SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG3__SCRATCH_REG3_MASK                                                                  0xFFFFFFFFL
-//GRBM_SCRATCH_REG4
-#define GRBM_SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG4__SCRATCH_REG4_MASK                                                                  0xFFFFFFFFL
-//GRBM_SCRATCH_REG5
-#define GRBM_SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG5__SCRATCH_REG5_MASK                                                                  0xFFFFFFFFL
-//GRBM_SCRATCH_REG6
-#define GRBM_SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG6__SCRATCH_REG6_MASK                                                                  0xFFFFFFFFL
-//GRBM_SCRATCH_REG7
-#define GRBM_SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                0x0
-#define GRBM_SCRATCH_REG7__SCRATCH_REG7_MASK                                                                  0xFFFFFFFFL
-
-
-// addressBlock: gc_cpdec
-//CP_CPC_STATUS
-#define CP_CPC_STATUS__MEC1_BUSY__SHIFT                                                                       0x0
-#define CP_CPC_STATUS__MEC2_BUSY__SHIFT                                                                       0x1
-#define CP_CPC_STATUS__DC0_BUSY__SHIFT                                                                        0x2
-#define CP_CPC_STATUS__DC1_BUSY__SHIFT                                                                        0x3
-#define CP_CPC_STATUS__RCIU1_BUSY__SHIFT                                                                      0x4
-#define CP_CPC_STATUS__RCIU2_BUSY__SHIFT                                                                      0x5
-#define CP_CPC_STATUS__ROQ1_BUSY__SHIFT                                                                       0x6
-#define CP_CPC_STATUS__ROQ2_BUSY__SHIFT                                                                       0x7
-#define CP_CPC_STATUS__TCIU_BUSY__SHIFT                                                                       0xa
-#define CP_CPC_STATUS__SCRATCH_RAM_BUSY__SHIFT                                                                0xb
-#define CP_CPC_STATUS__QU_BUSY__SHIFT                                                                         0xc
-#define CP_CPC_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0xd
-#define CP_CPC_STATUS__SAVE_RESTORE_BUSY__SHIFT                                                               0xe
-#define CP_CPC_STATUS__CPG_CPC_BUSY__SHIFT                                                                    0x1d
-#define CP_CPC_STATUS__CPF_CPC_BUSY__SHIFT                                                                    0x1e
-#define CP_CPC_STATUS__CPC_BUSY__SHIFT                                                                        0x1f
-#define CP_CPC_STATUS__MEC1_BUSY_MASK                                                                         0x00000001L
-#define CP_CPC_STATUS__MEC2_BUSY_MASK                                                                         0x00000002L
-#define CP_CPC_STATUS__DC0_BUSY_MASK                                                                          0x00000004L
-#define CP_CPC_STATUS__DC1_BUSY_MASK                                                                          0x00000008L
-#define CP_CPC_STATUS__RCIU1_BUSY_MASK                                                                        0x00000010L
-#define CP_CPC_STATUS__RCIU2_BUSY_MASK                                                                        0x00000020L
-#define CP_CPC_STATUS__ROQ1_BUSY_MASK                                                                         0x00000040L
-#define CP_CPC_STATUS__ROQ2_BUSY_MASK                                                                         0x00000080L
-#define CP_CPC_STATUS__TCIU_BUSY_MASK                                                                         0x00000400L
-#define CP_CPC_STATUS__SCRATCH_RAM_BUSY_MASK                                                                  0x00000800L
-#define CP_CPC_STATUS__QU_BUSY_MASK                                                                           0x00001000L
-#define CP_CPC_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00002000L
-#define CP_CPC_STATUS__SAVE_RESTORE_BUSY_MASK                                                                 0x00004000L
-#define CP_CPC_STATUS__CPG_CPC_BUSY_MASK                                                                      0x20000000L
-#define CP_CPC_STATUS__CPF_CPC_BUSY_MASK                                                                      0x40000000L
-#define CP_CPC_STATUS__CPC_BUSY_MASK                                                                          0x80000000L
-//CP_CPC_BUSY_STAT
-#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY__SHIFT                                                               0x0
-#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY__SHIFT                                                          0x1
-#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY__SHIFT                                                              0x2
-#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY__SHIFT                                                            0x3
-#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY__SHIFT                                                          0x4
-#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY__SHIFT                                                           0x5
-#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY__SHIFT                                                           0x6
-#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY__SHIFT                                                                 0x7
-#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY__SHIFT                                                                0x8
-#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x9
-#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY__SHIFT                                                              0xa
-#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY__SHIFT                                                              0xb
-#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY__SHIFT                                                              0xc
-#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY__SHIFT                                                              0xd
-#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY__SHIFT                                                               0x10
-#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY__SHIFT                                                          0x11
-#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY__SHIFT                                                              0x12
-#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY__SHIFT                                                            0x13
-#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY__SHIFT                                                          0x14
-#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY__SHIFT                                                           0x15
-#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY__SHIFT                                                           0x16
-#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY__SHIFT                                                                 0x17
-#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY__SHIFT                                                                0x18
-#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x19
-#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY__SHIFT                                                              0x1a
-#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY__SHIFT                                                              0x1b
-#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY__SHIFT                                                              0x1c
-#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY__SHIFT                                                              0x1d
-#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY_MASK                                                                 0x00000001L
-#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY_MASK                                                            0x00000002L
-#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY_MASK                                                                0x00000004L
-#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY_MASK                                                              0x00000008L
-#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY_MASK                                                            0x00000010L
-#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY_MASK                                                             0x00000020L
-#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY_MASK                                                             0x00000040L
-#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY_MASK                                                                   0x00000080L
-#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY_MASK                                                                  0x00000100L
-#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY_MASK                                                        0x00000200L
-#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY_MASK                                                                0x00000400L
-#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY_MASK                                                                0x00000800L
-#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY_MASK                                                                0x00001000L
-#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY_MASK                                                                0x00002000L
-#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY_MASK                                                                 0x00010000L
-#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY_MASK                                                            0x00020000L
-#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY_MASK                                                                0x00040000L
-#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY_MASK                                                              0x00080000L
-#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY_MASK                                                            0x00100000L
-#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY_MASK                                                             0x00200000L
-#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY_MASK                                                             0x00400000L
-#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY_MASK                                                                   0x00800000L
-#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY_MASK                                                                  0x01000000L
-#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY_MASK                                                        0x02000000L
-#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY_MASK                                                                0x04000000L
-#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY_MASK                                                                0x08000000L
-#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY_MASK                                                                0x10000000L
-#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY_MASK                                                                0x20000000L
-//CP_CPC_STALLED_STAT1
-#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL__SHIFT                                                       0x3
-#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION__SHIFT                                                      0x4
-#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL__SHIFT                                                       0x6
-#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET__SHIFT                                                     0x8
-#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU__SHIFT                                                        0x9
-#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ__SHIFT                                                   0xa
-#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA__SHIFT                                                    0xd
-#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET__SHIFT                                                     0x10
-#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU__SHIFT                                                        0x11
-#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ__SHIFT                                                   0x12
-#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA__SHIFT                                                    0x15
-#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x16
-#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x17
-#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS__SHIFT                                                   0x18
-#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL_MASK                                                         0x00000008L
-#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION_MASK                                                        0x00000010L
-#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL_MASK                                                         0x00000040L
-#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET_MASK                                                       0x00000100L
-#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_MASK                                                          0x00000200L
-#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ_MASK                                                     0x00000400L
-#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA_MASK                                                      0x00002000L
-#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET_MASK                                                       0x00010000L
-#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_MASK                                                          0x00020000L
-#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ_MASK                                                     0x00040000L
-#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA_MASK                                                      0x00200000L
-#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00400000L
-#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00800000L
-#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS_MASK                                                     0x01000000L
-//CP_CPF_STATUS
-#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY__SHIFT                                                              0x0
-#define CP_CPF_STATUS__CSF_BUSY__SHIFT                                                                        0x1
-#define CP_CPF_STATUS__ROQ_ALIGN_BUSY__SHIFT                                                                  0x4
-#define CP_CPF_STATUS__ROQ_RING_BUSY__SHIFT                                                                   0x5
-#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY__SHIFT                                                              0x6
-#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY__SHIFT                                                              0x7
-#define CP_CPF_STATUS__ROQ_STATE_BUSY__SHIFT                                                                  0x8
-#define CP_CPF_STATUS__ROQ_CE_RING_BUSY__SHIFT                                                                0x9
-#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                           0xa
-#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                           0xb
-#define CP_CPF_STATUS__SEMAPHORE_BUSY__SHIFT                                                                  0xc
-#define CP_CPF_STATUS__INTERRUPT_BUSY__SHIFT                                                                  0xd
-#define CP_CPF_STATUS__TCIU_BUSY__SHIFT                                                                       0xe
-#define CP_CPF_STATUS__HQD_BUSY__SHIFT                                                                        0xf
-#define CP_CPF_STATUS__PRT_BUSY__SHIFT                                                                        0x10
-#define CP_CPF_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0x11
-#define CP_CPF_STATUS__CPF_GFX_BUSY__SHIFT                                                                    0x1a
-#define CP_CPF_STATUS__CPF_CMP_BUSY__SHIFT                                                                    0x1b
-#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY__SHIFT                                                              0x1c
-#define CP_CPF_STATUS__CPC_CPF_BUSY__SHIFT                                                                    0x1e
-#define CP_CPF_STATUS__CPF_BUSY__SHIFT                                                                        0x1f
-#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY_MASK                                                                0x00000001L
-#define CP_CPF_STATUS__CSF_BUSY_MASK                                                                          0x00000002L
-#define CP_CPF_STATUS__ROQ_ALIGN_BUSY_MASK                                                                    0x00000010L
-#define CP_CPF_STATUS__ROQ_RING_BUSY_MASK                                                                     0x00000020L
-#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY_MASK                                                                0x00000040L
-#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY_MASK                                                                0x00000080L
-#define CP_CPF_STATUS__ROQ_STATE_BUSY_MASK                                                                    0x00000100L
-#define CP_CPF_STATUS__ROQ_CE_RING_BUSY_MASK                                                                  0x00000200L
-#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY_MASK                                                             0x00000400L
-#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY_MASK                                                             0x00000800L
-#define CP_CPF_STATUS__SEMAPHORE_BUSY_MASK                                                                    0x00001000L
-#define CP_CPF_STATUS__INTERRUPT_BUSY_MASK                                                                    0x00002000L
-#define CP_CPF_STATUS__TCIU_BUSY_MASK                                                                         0x00004000L
-#define CP_CPF_STATUS__HQD_BUSY_MASK                                                                          0x00008000L
-#define CP_CPF_STATUS__PRT_BUSY_MASK                                                                          0x00010000L
-#define CP_CPF_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00020000L
-#define CP_CPF_STATUS__CPF_GFX_BUSY_MASK                                                                      0x04000000L
-#define CP_CPF_STATUS__CPF_CMP_BUSY_MASK                                                                      0x08000000L
-#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY_MASK                                                                0x30000000L
-#define CP_CPF_STATUS__CPC_CPF_BUSY_MASK                                                                      0x40000000L
-#define CP_CPF_STATUS__CPF_BUSY_MASK                                                                          0x80000000L
-//CP_CPF_BUSY_STAT
-#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                            0x0
-#define CP_CPF_BUSY_STAT__CSF_RING_BUSY__SHIFT                                                                0x1
-#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY__SHIFT                                                           0x2
-#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY__SHIFT                                                           0x3
-#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY__SHIFT                                                               0x4
-#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY__SHIFT                                                            0x5
-#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY__SHIFT                                                            0x6
-#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY__SHIFT                                                             0x7
-#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY__SHIFT                                                               0x8
-#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS__SHIFT                                                        0x9
-#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY__SHIFT                                                      0xb
-#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY__SHIFT                                                            0xc
-#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY__SHIFT                                                            0xd
-#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY__SHIFT                                                         0xe
-#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY__SHIFT                                                      0xf
-#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY__SHIFT                                                    0x10
-#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY__SHIFT                                                             0x11
-#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY__SHIFT                                                          0x12
-#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY__SHIFT                                                          0x13
-#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY__SHIFT                                                          0x14
-#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY__SHIFT                                                         0x15
-#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY__SHIFT                                                       0x16
-#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY__SHIFT                                                         0x17
-#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY__SHIFT                                                           0x18
-#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY__SHIFT                                                             0x19
-#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY__SHIFT                                                              0x1a
-#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY__SHIFT                                                              0x1b
-#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY__SHIFT                                                              0x1c
-#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY__SHIFT                                                           0x1d
-#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY__SHIFT                                                                  0x1e
-#define CP_CPF_BUSY_STAT__HQD_IB_BUSY__SHIFT                                                                  0x1f
-#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                              0x00000001L
-#define CP_CPF_BUSY_STAT__CSF_RING_BUSY_MASK                                                                  0x00000002L
-#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY_MASK                                                             0x00000004L
-#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY_MASK                                                             0x00000008L
-#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY_MASK                                                                 0x00000010L
-#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY_MASK                                                              0x00000020L
-#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY_MASK                                                              0x00000040L
-#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY_MASK                                                               0x00000080L
-#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY_MASK                                                                 0x00000100L
-#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS_MASK                                                          0x00000200L
-#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY_MASK                                                        0x00000800L
-#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY_MASK                                                              0x00001000L
-#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY_MASK                                                              0x00002000L
-#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY_MASK                                                           0x00004000L
-#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY_MASK                                                        0x00008000L
-#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY_MASK                                                      0x00010000L
-#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY_MASK                                                               0x00020000L
-#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY_MASK                                                            0x00040000L
-#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY_MASK                                                            0x00080000L
-#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY_MASK                                                            0x00100000L
-#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY_MASK                                                           0x00200000L
-#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY_MASK                                                         0x00400000L
-#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY_MASK                                                           0x00800000L
-#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY_MASK                                                             0x01000000L
-#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY_MASK                                                               0x02000000L
-#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY_MASK                                                                0x04000000L
-#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY_MASK                                                                0x08000000L
-#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY_MASK                                                                0x10000000L
-#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY_MASK                                                             0x20000000L
-#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY_MASK                                                                    0x40000000L
-#define CP_CPF_BUSY_STAT__HQD_IB_BUSY_MASK                                                                    0x80000000L
-//CP_CPF_STALLED_STAT1
-#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA__SHIFT                                                       0x0
-#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA__SHIFT                                                      0x1
-#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA__SHIFT                                                      0x2
-#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA__SHIFT                                                      0x3
-#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE__SHIFT                                                     0x5
-#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT                                                     0x6
-#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x7
-#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x8
-#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS__SHIFT                                               0x9
-#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS__SHIFT                                               0xa
-#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE__SHIFT                                                     0xb
-#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA_MASK                                                         0x00000001L
-#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA_MASK                                                        0x00000002L
-#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA_MASK                                                        0x00000004L
-#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA_MASK                                                        0x00000008L
-#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE_MASK                                                       0x00000020L
-#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK                                                       0x00000040L
-#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00000080L
-#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00000100L
-#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000200L
-#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000400L
-#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE_MASK                                                       0x00000800L
-//CP_CPC_GRBM_FREE_COUNT
-#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
-#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x0000003FL
-//CP_MEC_CNTL
-#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                             0x4
-#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET__SHIFT                                                               0x10
-#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET__SHIFT                                                               0x11
-#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET__SHIFT                                                               0x12
-#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET__SHIFT                                                               0x13
-#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET__SHIFT                                                               0x14
-#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET__SHIFT                                                               0x15
-#define CP_MEC_CNTL__MEC_ME2_HALT__SHIFT                                                                      0x1c
-#define CP_MEC_CNTL__MEC_ME2_STEP__SHIFT                                                                      0x1d
-#define CP_MEC_CNTL__MEC_ME1_HALT__SHIFT                                                                      0x1e
-#define CP_MEC_CNTL__MEC_ME1_STEP__SHIFT                                                                      0x1f
-#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                               0x00000010L
-#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK                                                                 0x00010000L
-#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK                                                                 0x00020000L
-#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK                                                                 0x00040000L
-#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK                                                                 0x00080000L
-#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK                                                                 0x00100000L
-#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK                                                                 0x00200000L
-#define CP_MEC_CNTL__MEC_ME2_HALT_MASK                                                                        0x10000000L
-#define CP_MEC_CNTL__MEC_ME2_STEP_MASK                                                                        0x20000000L
-#define CP_MEC_CNTL__MEC_ME1_HALT_MASK                                                                        0x40000000L
-#define CP_MEC_CNTL__MEC_ME1_STEP_MASK                                                                        0x80000000L
-//CP_MEC_ME1_HEADER_DUMP
-#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP__SHIFT                                                            0x0
-#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP_MASK                                                              0xFFFFFFFFL
-//CP_MEC_ME2_HEADER_DUMP
-#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP__SHIFT                                                            0x0
-#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP_MASK                                                              0xFFFFFFFFL
-//CP_CPC_SCRATCH_INDEX
-#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                            0x0
-#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                              0x000001FFL
-//CP_CPC_SCRATCH_DATA
-#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                              0x0
-#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                0xFFFFFFFFL
-//CP_CPF_GRBM_FREE_COUNT
-#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
-#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x00000007L
-//CP_CPC_HALT_HYST_COUNT
-#define CP_CPC_HALT_HYST_COUNT__COUNT__SHIFT                                                                  0x0
-#define CP_CPC_HALT_HYST_COUNT__COUNT_MASK                                                                    0x0000000FL
-//CP_PRT_LOD_STATS_CNTL0
-#define CP_PRT_LOD_STATS_CNTL0__BU_SIZE__SHIFT                                                                0x0
-#define CP_PRT_LOD_STATS_CNTL0__BU_SIZE_MASK                                                                  0xFFFFFFFFL
-//CP_PRT_LOD_STATS_CNTL1
-#define CP_PRT_LOD_STATS_CNTL1__BASE_LO__SHIFT                                                                0x0
-#define CP_PRT_LOD_STATS_CNTL1__BASE_LO_MASK                                                                  0xFFFFFFFFL
-//CP_PRT_LOD_STATS_CNTL2
-#define CP_PRT_LOD_STATS_CNTL2__BASE_HI__SHIFT                                                                0x0
-#define CP_PRT_LOD_STATS_CNTL2__BASE_HI_MASK                                                                  0x000003FFL
-//CP_PRT_LOD_STATS_CNTL3
-#define CP_PRT_LOD_STATS_CNTL3__INTERVAL__SHIFT                                                               0x2
-#define CP_PRT_LOD_STATS_CNTL3__RESET_CNT__SHIFT                                                              0xa
-#define CP_PRT_LOD_STATS_CNTL3__RESET_FORCE__SHIFT                                                            0x12
-#define CP_PRT_LOD_STATS_CNTL3__REPORT_AND_RESET__SHIFT                                                       0x13
-#define CP_PRT_LOD_STATS_CNTL3__MC_VMID__SHIFT                                                                0x17
-#define CP_PRT_LOD_STATS_CNTL3__CACHE_POLICY__SHIFT                                                           0x1c
-#define CP_PRT_LOD_STATS_CNTL3__INTERVAL_MASK                                                                 0x000003FCL
-#define CP_PRT_LOD_STATS_CNTL3__RESET_CNT_MASK                                                                0x0003FC00L
-#define CP_PRT_LOD_STATS_CNTL3__RESET_FORCE_MASK                                                              0x00040000L
-#define CP_PRT_LOD_STATS_CNTL3__REPORT_AND_RESET_MASK                                                         0x00080000L
-#define CP_PRT_LOD_STATS_CNTL3__MC_VMID_MASK                                                                  0x07800000L
-#define CP_PRT_LOD_STATS_CNTL3__CACHE_POLICY_MASK                                                             0x10000000L
-//CP_CE_COMPARE_COUNT
-#define CP_CE_COMPARE_COUNT__COMPARE_COUNT__SHIFT                                                             0x0
-#define CP_CE_COMPARE_COUNT__COMPARE_COUNT_MASK                                                               0xFFFFFFFFL
-//CP_CE_DE_COUNT
-#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT                                                              0x0
-#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT_MASK                                                                0xFFFFFFFFL
-//CP_DE_CE_COUNT
-#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT__SHIFT                                                             0x0
-#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT_MASK                                                               0xFFFFFFFFL
-//CP_DE_LAST_INVAL_COUNT
-#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT__SHIFT                                                       0x0
-#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT_MASK                                                         0xFFFFFFFFL
-//CP_DE_DE_COUNT
-#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT                                                              0x0
-#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT_MASK                                                                0xFFFFFFFFL
-//CP_STALLED_STAT3
-#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                     0x0
-#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV__SHIFT                                        0x1
-#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER__SHIFT                                     0x2
-#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY__SHIFT                                                       0x3
-#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY__SHIFT                                                       0x4
-#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY__SHIFT                                                      0x5
-#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV__SHIFT                                                0x6
-#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV__SHIFT                                                 0x7
-#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA__SHIFT                                                    0xa
-#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG__SHIFT                                                 0xb
-#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER__SHIFT                                                     0xc
-#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW__SHIFT                                           0xd
-#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE__SHIFT                                                         0xe
-#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS__SHIFT                                                         0xf
-#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0x10
-#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0x11
-#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE__SHIFT                                                      0x12
-#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                      0x13
-#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS__SHIFT                                                       0x14
-#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV_MASK                                                       0x00000001L
-#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV_MASK                                          0x00000002L
-#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER_MASK                                       0x00000004L
-#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY_MASK                                                         0x00000008L
-#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY_MASK                                                         0x00000010L
-#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY_MASK                                                        0x00000020L
-#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV_MASK                                                  0x00000040L
-#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV_MASK                                                   0x00000080L
-#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA_MASK                                                      0x00000400L
-#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG_MASK                                                   0x00000800L
-#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_MASK                                                       0x00001000L
-#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW_MASK                                             0x00002000L
-#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE_MASK                                                           0x00004000L
-#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS_MASK                                                           0x00008000L
-#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00010000L
-#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00020000L
-#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE_MASK                                                        0x00040000L
-#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS_MASK                                                        0x00080000L
-#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS_MASK                                                         0x00100000L
-//CP_STALLED_STAT1
-#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV__SHIFT                                                   0x0
-#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV__SHIFT                                                   0x2
-#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV__SHIFT                                                 0x4
-#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG__SHIFT                                                 0xa
-#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG__SHIFT                                                 0xb
-#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0xc
-#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0xd
-#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA__SHIFT                                                   0xe
-#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA__SHIFT                                                  0xf
-#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE__SHIFT                                                     0x17
-#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE__SHIFT                                                    0x18
-#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE__SHIFT                                                     0x19
-#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ__SHIFT                                                      0x1a
-#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ__SHIFT                                                     0x1b
-#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ__SHIFT                                                  0x1c
-#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION__SHIFT                                                 0x1d
-#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV_MASK                                                     0x00000001L
-#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_MASK                                                     0x00000004L
-#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_MASK                                                   0x00000010L
-#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG_MASK                                                   0x00000400L
-#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG_MASK                                                   0x00000800L
-#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00001000L
-#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00002000L
-#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA_MASK                                                     0x00004000L
-#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA_MASK                                                    0x00008000L
-#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE_MASK                                                       0x00800000L
-#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE_MASK                                                      0x01000000L
-#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE_MASK                                                       0x02000000L
-#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ_MASK                                                        0x04000000L
-#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ_MASK                                                       0x08000000L
-#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ_MASK                                                    0x10000000L
-#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION_MASK                                                   0x20000000L
-//CP_STALLED_STAT2
-#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                    0x0
-#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV__SHIFT                                                    0x1
-#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV__SHIFT                                                   0x2
-#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING__SHIFT                                                    0x4
-#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING__SHIFT                                                        0x5
-#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA__SHIFT                                                   0x8
-#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER__SHIFT                                                        0x9
-#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER__SHIFT                                                      0xa
-#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME__SHIFT                                                     0xb
-#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV__SHIFT                                                       0xc
-#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV__SHIFT                                                   0xd
-#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP__SHIFT                                                     0xe
-#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH__SHIFT                                                  0xf
-#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x10
-#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x11
-#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ__SHIFT                                                     0x12
-#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                 0x13
-#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                               0x14
-#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE__SHIFT                                                  0x15
-#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM__SHIFT                                                   0x16
-#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING__SHIFT                                                0x17
-#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING__SHIFT                                                   0x18
-#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE__SHIFT                                                   0x19
-#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE__SHIFT                                                   0x1a
-#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM__SHIFT                                                    0x1b
-#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION__SHIFT                                                      0x1c
-#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE__SHIFT                                              0x1d
-#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS__SHIFT                                                   0x1e
-#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN__SHIFT                                                    0x1f
-#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV_MASK                                                      0x00000001L
-#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV_MASK                                                      0x00000002L
-#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV_MASK                                                     0x00000004L
-#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING_MASK                                                      0x00000010L
-#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING_MASK                                                          0x00000020L
-#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA_MASK                                                     0x00000100L
-#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER_MASK                                                          0x00000200L
-#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER_MASK                                                        0x00000400L
-#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME_MASK                                                       0x00000800L
-#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV_MASK                                                         0x00001000L
-#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV_MASK                                                     0x00002000L
-#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP_MASK                                                       0x00004000L
-#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH_MASK                                                    0x00008000L
-#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00010000L
-#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00020000L
-#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ_MASK                                                       0x00040000L
-#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM_MASK                                                   0x00080000L
-#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                 0x00100000L
-#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE_MASK                                                    0x00200000L
-#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM_MASK                                                     0x00400000L
-#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING_MASK                                                  0x00800000L
-#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING_MASK                                                     0x01000000L
-#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE_MASK                                                     0x02000000L
-#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE_MASK                                                     0x04000000L
-#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM_MASK                                                      0x08000000L
-#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION_MASK                                                        0x10000000L
-#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE_MASK                                                0x20000000L
-#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS_MASK                                                     0x40000000L
-#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN_MASK                                                      0x80000000L
-//CP_BUSY_STAT
-#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                                0x0
-#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO__SHIFT                                                               0x6
-#define CP_BUSY_STAT__PFP_PARSING_PACKETS__SHIFT                                                              0x7
-#define CP_BUSY_STAT__ME_PARSING_PACKETS__SHIFT                                                               0x8
-#define CP_BUSY_STAT__RCIU_PFP_BUSY__SHIFT                                                                    0x9
-#define CP_BUSY_STAT__RCIU_ME_BUSY__SHIFT                                                                     0xa
-#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY__SHIFT                                                            0xc
-#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING__SHIFT                                                           0xd
-#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS__SHIFT                                                             0xe
-#define CP_BUSY_STAT__GFX_CONTEXT_BUSY__SHIFT                                                                 0xf
-#define CP_BUSY_STAT__ME_PARSER_BUSY__SHIFT                                                                   0x11
-#define CP_BUSY_STAT__EOP_DONE_BUSY__SHIFT                                                                    0x12
-#define CP_BUSY_STAT__STRM_OUT_BUSY__SHIFT                                                                    0x13
-#define CP_BUSY_STAT__PIPE_STATS_BUSY__SHIFT                                                                  0x14
-#define CP_BUSY_STAT__RCIU_CE_BUSY__SHIFT                                                                     0x15
-#define CP_BUSY_STAT__CE_PARSING_PACKETS__SHIFT                                                               0x16
-#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                                  0x00000001L
-#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO_MASK                                                                 0x00000040L
-#define CP_BUSY_STAT__PFP_PARSING_PACKETS_MASK                                                                0x00000080L
-#define CP_BUSY_STAT__ME_PARSING_PACKETS_MASK                                                                 0x00000100L
-#define CP_BUSY_STAT__RCIU_PFP_BUSY_MASK                                                                      0x00000200L
-#define CP_BUSY_STAT__RCIU_ME_BUSY_MASK                                                                       0x00000400L
-#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY_MASK                                                              0x00001000L
-#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING_MASK                                                             0x00002000L
-#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS_MASK                                                               0x00004000L
-#define CP_BUSY_STAT__GFX_CONTEXT_BUSY_MASK                                                                   0x00008000L
-#define CP_BUSY_STAT__ME_PARSER_BUSY_MASK                                                                     0x00020000L
-#define CP_BUSY_STAT__EOP_DONE_BUSY_MASK                                                                      0x00040000L
-#define CP_BUSY_STAT__STRM_OUT_BUSY_MASK                                                                      0x00080000L
-#define CP_BUSY_STAT__PIPE_STATS_BUSY_MASK                                                                    0x00100000L
-#define CP_BUSY_STAT__RCIU_CE_BUSY_MASK                                                                       0x00200000L
-#define CP_BUSY_STAT__CE_PARSING_PACKETS_MASK                                                                 0x00400000L
-//CP_STAT
-#define CP_STAT__ROQ_RING_BUSY__SHIFT                                                                         0x9
-#define CP_STAT__ROQ_INDIRECT1_BUSY__SHIFT                                                                    0xa
-#define CP_STAT__ROQ_INDIRECT2_BUSY__SHIFT                                                                    0xb
-#define CP_STAT__ROQ_STATE_BUSY__SHIFT                                                                        0xc
-#define CP_STAT__DC_BUSY__SHIFT                                                                               0xd
-#define CP_STAT__UTCL2IU_BUSY__SHIFT                                                                          0xe
-#define CP_STAT__PFP_BUSY__SHIFT                                                                              0xf
-#define CP_STAT__MEQ_BUSY__SHIFT                                                                              0x10
-#define CP_STAT__ME_BUSY__SHIFT                                                                               0x11
-#define CP_STAT__QUERY_BUSY__SHIFT                                                                            0x12
-#define CP_STAT__SEMAPHORE_BUSY__SHIFT                                                                        0x13
-#define CP_STAT__INTERRUPT_BUSY__SHIFT                                                                        0x14
-#define CP_STAT__SURFACE_SYNC_BUSY__SHIFT                                                                     0x15
-#define CP_STAT__DMA_BUSY__SHIFT                                                                              0x16
-#define CP_STAT__RCIU_BUSY__SHIFT                                                                             0x17
-#define CP_STAT__SCRATCH_RAM_BUSY__SHIFT                                                                      0x18
-#define CP_STAT__CE_BUSY__SHIFT                                                                               0x1a
-#define CP_STAT__TCIU_BUSY__SHIFT                                                                             0x1b
-#define CP_STAT__ROQ_CE_RING_BUSY__SHIFT                                                                      0x1c
-#define CP_STAT__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                                 0x1d
-#define CP_STAT__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                                 0x1e
-#define CP_STAT__CP_BUSY__SHIFT                                                                               0x1f
-#define CP_STAT__ROQ_RING_BUSY_MASK                                                                           0x00000200L
-#define CP_STAT__ROQ_INDIRECT1_BUSY_MASK                                                                      0x00000400L
-#define CP_STAT__ROQ_INDIRECT2_BUSY_MASK                                                                      0x00000800L
-#define CP_STAT__ROQ_STATE_BUSY_MASK                                                                          0x00001000L
-#define CP_STAT__DC_BUSY_MASK                                                                                 0x00002000L
-#define CP_STAT__UTCL2IU_BUSY_MASK                                                                            0x00004000L
-#define CP_STAT__PFP_BUSY_MASK                                                                                0x00008000L
-#define CP_STAT__MEQ_BUSY_MASK                                                                                0x00010000L
-#define CP_STAT__ME_BUSY_MASK                                                                                 0x00020000L
-#define CP_STAT__QUERY_BUSY_MASK                                                                              0x00040000L
-#define CP_STAT__SEMAPHORE_BUSY_MASK                                                                          0x00080000L
-#define CP_STAT__INTERRUPT_BUSY_MASK                                                                          0x00100000L
-#define CP_STAT__SURFACE_SYNC_BUSY_MASK                                                                       0x00200000L
-#define CP_STAT__DMA_BUSY_MASK                                                                                0x00400000L
-#define CP_STAT__RCIU_BUSY_MASK                                                                               0x00800000L
-#define CP_STAT__SCRATCH_RAM_BUSY_MASK                                                                        0x01000000L
-#define CP_STAT__CE_BUSY_MASK                                                                                 0x04000000L
-#define CP_STAT__TCIU_BUSY_MASK                                                                               0x08000000L
-#define CP_STAT__ROQ_CE_RING_BUSY_MASK                                                                        0x10000000L
-#define CP_STAT__ROQ_CE_INDIRECT1_BUSY_MASK                                                                   0x20000000L
-#define CP_STAT__ROQ_CE_INDIRECT2_BUSY_MASK                                                                   0x40000000L
-#define CP_STAT__CP_BUSY_MASK                                                                                 0x80000000L
-//CP_ME_HEADER_DUMP
-#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP__SHIFT                                                              0x0
-#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP_MASK                                                                0xFFFFFFFFL
-//CP_PFP_HEADER_DUMP
-#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP__SHIFT                                                            0x0
-#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP_MASK                                                              0xFFFFFFFFL
-//CP_GRBM_FREE_COUNT
-#define CP_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                                 0x0
-#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS__SHIFT                                                             0x8
-#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP__SHIFT                                                             0x10
-#define CP_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                                   0x0000003FL
-#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS_MASK                                                               0x00003F00L
-#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP_MASK                                                               0x003F0000L
-//CP_CE_HEADER_DUMP
-#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP__SHIFT                                                              0x0
-#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP_MASK                                                                0xFFFFFFFFL
-//CP_PFP_INSTR_PNTR
-#define CP_PFP_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                  0x0
-#define CP_PFP_INSTR_PNTR__INSTR_PNTR_MASK                                                                    0x0000FFFFL
-//CP_ME_INSTR_PNTR
-#define CP_ME_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                   0x0
-#define CP_ME_INSTR_PNTR__INSTR_PNTR_MASK                                                                     0x0000FFFFL
-//CP_CE_INSTR_PNTR
-#define CP_CE_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                   0x0
-#define CP_CE_INSTR_PNTR__INSTR_PNTR_MASK                                                                     0x0000FFFFL
-//CP_MEC1_INSTR_PNTR
-#define CP_MEC1_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                 0x0
-#define CP_MEC1_INSTR_PNTR__INSTR_PNTR_MASK                                                                   0x0000FFFFL
-//CP_MEC2_INSTR_PNTR
-#define CP_MEC2_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                 0x0
-#define CP_MEC2_INSTR_PNTR__INSTR_PNTR_MASK                                                                   0x0000FFFFL
-//CP_CSF_STAT
-#define CP_CSF_STAT__BUFFER_REQUEST_COUNT__SHIFT                                                              0x8
-#define CP_CSF_STAT__BUFFER_REQUEST_COUNT_MASK                                                                0x0001FF00L
-//CP_ME_CNTL
-#define CP_ME_CNTL__CE_INVALIDATE_ICACHE__SHIFT                                                               0x4
-#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE__SHIFT                                                              0x6
-#define CP_ME_CNTL__ME_INVALIDATE_ICACHE__SHIFT                                                               0x8
-#define CP_ME_CNTL__CE_PIPE0_RESET__SHIFT                                                                     0x10
-#define CP_ME_CNTL__CE_PIPE1_RESET__SHIFT                                                                     0x11
-#define CP_ME_CNTL__PFP_PIPE0_RESET__SHIFT                                                                    0x12
-#define CP_ME_CNTL__PFP_PIPE1_RESET__SHIFT                                                                    0x13
-#define CP_ME_CNTL__ME_PIPE0_RESET__SHIFT                                                                     0x14
-#define CP_ME_CNTL__ME_PIPE1_RESET__SHIFT                                                                     0x15
-#define CP_ME_CNTL__CE_HALT__SHIFT                                                                            0x18
-#define CP_ME_CNTL__CE_STEP__SHIFT                                                                            0x19
-#define CP_ME_CNTL__PFP_HALT__SHIFT                                                                           0x1a
-#define CP_ME_CNTL__PFP_STEP__SHIFT                                                                           0x1b
-#define CP_ME_CNTL__ME_HALT__SHIFT                                                                            0x1c
-#define CP_ME_CNTL__ME_STEP__SHIFT                                                                            0x1d
-#define CP_ME_CNTL__CE_INVALIDATE_ICACHE_MASK                                                                 0x00000010L
-#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE_MASK                                                                0x00000040L
-#define CP_ME_CNTL__ME_INVALIDATE_ICACHE_MASK                                                                 0x00000100L
-#define CP_ME_CNTL__CE_PIPE0_RESET_MASK                                                                       0x00010000L
-#define CP_ME_CNTL__CE_PIPE1_RESET_MASK                                                                       0x00020000L
-#define CP_ME_CNTL__PFP_PIPE0_RESET_MASK                                                                      0x00040000L
-#define CP_ME_CNTL__PFP_PIPE1_RESET_MASK                                                                      0x00080000L
-#define CP_ME_CNTL__ME_PIPE0_RESET_MASK                                                                       0x00100000L
-#define CP_ME_CNTL__ME_PIPE1_RESET_MASK                                                                       0x00200000L
-#define CP_ME_CNTL__CE_HALT_MASK                                                                              0x01000000L
-#define CP_ME_CNTL__CE_STEP_MASK                                                                              0x02000000L
-#define CP_ME_CNTL__PFP_HALT_MASK                                                                             0x04000000L
-#define CP_ME_CNTL__PFP_STEP_MASK                                                                             0x08000000L
-#define CP_ME_CNTL__ME_HALT_MASK                                                                              0x10000000L
-#define CP_ME_CNTL__ME_STEP_MASK                                                                              0x20000000L
-//CP_CNTX_STAT
-#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS__SHIFT                                                             0x0
-#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT__SHIFT                                                             0x8
-#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS__SHIFT                                                              0x14
-#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT__SHIFT                                                              0x1c
-#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS_MASK                                                               0x000000FFL
-#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT_MASK                                                               0x00000700L
-#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS_MASK                                                                0x0FF00000L
-#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT_MASK                                                                0x70000000L
-//CP_ME_PREEMPTION
-#define CP_ME_PREEMPTION__OBSOLETE__SHIFT                                                                     0x0
-#define CP_ME_PREEMPTION__OBSOLETE_MASK                                                                       0x00000001L
-//CP_ROQ_THRESHOLDS
-#define CP_ROQ_THRESHOLDS__IB1_START__SHIFT                                                                   0x0
-#define CP_ROQ_THRESHOLDS__IB2_START__SHIFT                                                                   0x8
-#define CP_ROQ_THRESHOLDS__IB1_START_MASK                                                                     0x000000FFL
-#define CP_ROQ_THRESHOLDS__IB2_START_MASK                                                                     0x0000FF00L
-//CP_MEQ_STQ_THRESHOLD
-#define CP_MEQ_STQ_THRESHOLD__STQ_START__SHIFT                                                                0x0
-#define CP_MEQ_STQ_THRESHOLD__STQ_START_MASK                                                                  0x000000FFL
-//CP_RB2_RPTR
-#define CP_RB2_RPTR__RB_RPTR__SHIFT                                                                           0x0
-#define CP_RB2_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
-//CP_RB1_RPTR
-#define CP_RB1_RPTR__RB_RPTR__SHIFT                                                                           0x0
-#define CP_RB1_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
-//CP_RB0_RPTR
-#define CP_RB0_RPTR__RB_RPTR__SHIFT                                                                           0x0
-#define CP_RB0_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
-//CP_RB_RPTR
-#define CP_RB_RPTR__RB_RPTR__SHIFT                                                                            0x0
-#define CP_RB_RPTR__RB_RPTR_MASK                                                                              0x000FFFFFL
-//CP_RB_WPTR_DELAY
-#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER__SHIFT                                                              0x0
-#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT__SHIFT                                                              0x1c
-#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER_MASK                                                                0x0FFFFFFFL
-#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT_MASK                                                                0xF0000000L
-//CP_RB_WPTR_POLL_CNTL
-#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT                                                           0x0
-#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                          0x10
-#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK                                                             0x0000FFFFL
-#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                            0xFFFF0000L
-//CP_ROQ1_THRESHOLDS
-#define CP_ROQ1_THRESHOLDS__RB1_START__SHIFT                                                                  0x0
-#define CP_ROQ1_THRESHOLDS__RB2_START__SHIFT                                                                  0x8
-#define CP_ROQ1_THRESHOLDS__R0_IB1_START__SHIFT                                                               0x10
-#define CP_ROQ1_THRESHOLDS__R1_IB1_START__SHIFT                                                               0x18
-#define CP_ROQ1_THRESHOLDS__RB1_START_MASK                                                                    0x000000FFL
-#define CP_ROQ1_THRESHOLDS__RB2_START_MASK                                                                    0x0000FF00L
-#define CP_ROQ1_THRESHOLDS__R0_IB1_START_MASK                                                                 0x00FF0000L
-#define CP_ROQ1_THRESHOLDS__R1_IB1_START_MASK                                                                 0xFF000000L
-//CP_ROQ2_THRESHOLDS
-#define CP_ROQ2_THRESHOLDS__R2_IB1_START__SHIFT                                                               0x0
-#define CP_ROQ2_THRESHOLDS__R0_IB2_START__SHIFT                                                               0x8
-#define CP_ROQ2_THRESHOLDS__R1_IB2_START__SHIFT                                                               0x10
-#define CP_ROQ2_THRESHOLDS__R2_IB2_START__SHIFT                                                               0x18
-#define CP_ROQ2_THRESHOLDS__R2_IB1_START_MASK                                                                 0x000000FFL
-#define CP_ROQ2_THRESHOLDS__R0_IB2_START_MASK                                                                 0x0000FF00L
-#define CP_ROQ2_THRESHOLDS__R1_IB2_START_MASK                                                                 0x00FF0000L
-#define CP_ROQ2_THRESHOLDS__R2_IB2_START_MASK                                                                 0xFF000000L
-//CP_STQ_THRESHOLDS
-#define CP_STQ_THRESHOLDS__STQ0_START__SHIFT                                                                  0x0
-#define CP_STQ_THRESHOLDS__STQ1_START__SHIFT                                                                  0x8
-#define CP_STQ_THRESHOLDS__STQ2_START__SHIFT                                                                  0x10
-#define CP_STQ_THRESHOLDS__STQ0_START_MASK                                                                    0x000000FFL
-#define CP_STQ_THRESHOLDS__STQ1_START_MASK                                                                    0x0000FF00L
-#define CP_STQ_THRESHOLDS__STQ2_START_MASK                                                                    0x00FF0000L
-//CP_QUEUE_THRESHOLDS
-#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START__SHIFT                                                             0x0
-#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START__SHIFT                                                             0x8
-#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START_MASK                                                               0x0000003FL
-#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START_MASK                                                               0x00003F00L
-//CP_MEQ_THRESHOLDS
-#define CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT                                                                  0x0
-#define CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT                                                                  0x8
-#define CP_MEQ_THRESHOLDS__MEQ1_START_MASK                                                                    0x000000FFL
-#define CP_MEQ_THRESHOLDS__MEQ2_START_MASK                                                                    0x0000FF00L
-//CP_ROQ_AVAIL
-#define CP_ROQ_AVAIL__ROQ_CNT_RING__SHIFT                                                                     0x0
-#define CP_ROQ_AVAIL__ROQ_CNT_IB1__SHIFT                                                                      0x10
-#define CP_ROQ_AVAIL__ROQ_CNT_RING_MASK                                                                       0x000007FFL
-#define CP_ROQ_AVAIL__ROQ_CNT_IB1_MASK                                                                        0x07FF0000L
-//CP_STQ_AVAIL
-#define CP_STQ_AVAIL__STQ_CNT__SHIFT                                                                          0x0
-#define CP_STQ_AVAIL__STQ_CNT_MASK                                                                            0x000001FFL
-//CP_ROQ2_AVAIL
-#define CP_ROQ2_AVAIL__ROQ_CNT_IB2__SHIFT                                                                     0x0
-#define CP_ROQ2_AVAIL__ROQ_CNT_IB2_MASK                                                                       0x000007FFL
-//CP_MEQ_AVAIL
-#define CP_MEQ_AVAIL__MEQ_CNT__SHIFT                                                                          0x0
-#define CP_MEQ_AVAIL__MEQ_CNT_MASK                                                                            0x000003FFL
-//CP_CMD_INDEX
-#define CP_CMD_INDEX__CMD_INDEX__SHIFT                                                                        0x0
-#define CP_CMD_INDEX__CMD_ME_SEL__SHIFT                                                                       0xc
-#define CP_CMD_INDEX__CMD_QUEUE_SEL__SHIFT                                                                    0x10
-#define CP_CMD_INDEX__CMD_INDEX_MASK                                                                          0x000007FFL
-#define CP_CMD_INDEX__CMD_ME_SEL_MASK                                                                         0x00003000L
-#define CP_CMD_INDEX__CMD_QUEUE_SEL_MASK                                                                      0x00070000L
-//CP_CMD_DATA
-#define CP_CMD_DATA__CMD_DATA__SHIFT                                                                          0x0
-#define CP_CMD_DATA__CMD_DATA_MASK                                                                            0xFFFFFFFFL
-//CP_ROQ_RB_STAT
-#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY__SHIFT                                                               0x0
-#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY__SHIFT                                                               0x10
-#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY_MASK                                                                 0x000003FFL
-#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY_MASK                                                                 0x03FF0000L
-//CP_ROQ_IB1_STAT
-#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1__SHIFT                                                            0x0
-#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1__SHIFT                                                            0x10
-#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1_MASK                                                              0x000003FFL
-#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1_MASK                                                              0x03FF0000L
-//CP_ROQ_IB2_STAT
-#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2__SHIFT                                                            0x0
-#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2__SHIFT                                                            0x10
-#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2_MASK                                                              0x000003FFL
-#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2_MASK                                                              0x03FF0000L
-//CP_STQ_STAT
-#define CP_STQ_STAT__STQ_RPTR__SHIFT                                                                          0x0
-#define CP_STQ_STAT__STQ_RPTR_MASK                                                                            0x000003FFL
-//CP_STQ_WR_STAT
-#define CP_STQ_WR_STAT__STQ_WPTR__SHIFT                                                                       0x0
-#define CP_STQ_WR_STAT__STQ_WPTR_MASK                                                                         0x000003FFL
-//CP_MEQ_STAT
-#define CP_MEQ_STAT__MEQ_RPTR__SHIFT                                                                          0x0
-#define CP_MEQ_STAT__MEQ_WPTR__SHIFT                                                                          0x10
-#define CP_MEQ_STAT__MEQ_RPTR_MASK                                                                            0x000003FFL
-#define CP_MEQ_STAT__MEQ_WPTR_MASK                                                                            0x03FF0000L
-//CP_CEQ1_AVAIL
-#define CP_CEQ1_AVAIL__CEQ_CNT_RING__SHIFT                                                                    0x0
-#define CP_CEQ1_AVAIL__CEQ_CNT_IB1__SHIFT                                                                     0x10
-#define CP_CEQ1_AVAIL__CEQ_CNT_RING_MASK                                                                      0x000007FFL
-#define CP_CEQ1_AVAIL__CEQ_CNT_IB1_MASK                                                                       0x07FF0000L
-//CP_CEQ2_AVAIL
-#define CP_CEQ2_AVAIL__CEQ_CNT_IB2__SHIFT                                                                     0x0
-#define CP_CEQ2_AVAIL__CEQ_CNT_IB2_MASK                                                                       0x000007FFL
-//CP_CE_ROQ_RB_STAT
-#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY__SHIFT                                                            0x0
-#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY__SHIFT                                                            0x10
-#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY_MASK                                                              0x000003FFL
-#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY_MASK                                                              0x03FF0000L
-//CP_CE_ROQ_IB1_STAT
-#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1__SHIFT                                                         0x0
-#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1__SHIFT                                                         0x10
-#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1_MASK                                                           0x000003FFL
-#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1_MASK                                                           0x03FF0000L
-//CP_CE_ROQ_IB2_STAT
-#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2__SHIFT                                                         0x0
-#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2__SHIFT                                                         0x10
-#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2_MASK                                                           0x000003FFL
-#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2_MASK                                                           0x03FF0000L
-#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED__SHIFT                                                     0x16
-#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                       0x17
-#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED_MASK                                                       0x00400000L
-#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                         0x00800000L
-
-
-// addressBlock: gc_padec
-//VGT_VTX_VECT_EJECT_REG
-#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT__SHIFT                                                             0x0
-#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT_MASK                                                               0x0000007FL
-//VGT_DMA_DATA_FIFO_DEPTH
-#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH__SHIFT                                                   0x0
-#define VGT_DMA_DATA_FIFO_DEPTH__DMA2DRAW_FIFO_DEPTH__SHIFT                                                   0x9
-#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH_MASK                                                     0x000001FFL
-#define VGT_DMA_DATA_FIFO_DEPTH__DMA2DRAW_FIFO_DEPTH_MASK                                                     0x0007FE00L
-//VGT_DMA_REQ_FIFO_DEPTH
-#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH__SHIFT                                                     0x0
-#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH_MASK                                                       0x0000003FL
-//VGT_DRAW_INIT_FIFO_DEPTH
-#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH__SHIFT                                                 0x0
-#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH_MASK                                                   0x0000003FL
-//VGT_LAST_COPY_STATE
-#define VGT_LAST_COPY_STATE__SRC_STATE_ID__SHIFT                                                              0x0
-#define VGT_LAST_COPY_STATE__DST_STATE_ID__SHIFT                                                              0x10
-#define VGT_LAST_COPY_STATE__SRC_STATE_ID_MASK                                                                0x00000007L
-#define VGT_LAST_COPY_STATE__DST_STATE_ID_MASK                                                                0x00070000L
-//VGT_CACHE_INVALIDATION
-#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT                                                     0x0
-#define VGT_CACHE_INVALIDATION__DIS_INSTANCING_OPT__SHIFT                                                     0x4
-#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER__SHIFT                                                     0x5
-#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT                                                          0x6
-#define VGT_CACHE_INVALIDATION__USE_GS_DONE__SHIFT                                                            0x9
-#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD__SHIFT                                                   0xb
-#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN__SHIFT                                                       0xc
-#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH__SHIFT                                                   0xd
-#define VGT_CACHE_INVALIDATION__ES_LIMIT__SHIFT                                                               0x10
-#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG__SHIFT                                                       0x15
-#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_1__SHIFT                                                        0x16
-#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_2__SHIFT                                                        0x19
-#define VGT_CACHE_INVALIDATION__EN_WAVE_MERGE__SHIFT                                                          0x1c
-#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG_EOI__SHIFT                                                   0x1d
-#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION_MASK                                                       0x00000003L
-#define VGT_CACHE_INVALIDATION__DIS_INSTANCING_OPT_MASK                                                       0x00000010L
-#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER_MASK                                                       0x00000020L
-#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN_MASK                                                            0x000000C0L
-#define VGT_CACHE_INVALIDATION__USE_GS_DONE_MASK                                                              0x00000200L
-#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD_MASK                                                     0x00000800L
-#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN_MASK                                                         0x00001000L
-#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH_MASK                                                     0x00002000L
-#define VGT_CACHE_INVALIDATION__ES_LIMIT_MASK                                                                 0x001F0000L
-#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG_MASK                                                         0x00200000L
-#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_1_MASK                                                          0x01C00000L
-#define VGT_CACHE_INVALIDATION__OPT_FLOW_CNTL_2_MASK                                                          0x0E000000L
-#define VGT_CACHE_INVALIDATION__EN_WAVE_MERGE_MASK                                                            0x10000000L
-#define VGT_CACHE_INVALIDATION__ENABLE_PING_PONG_EOI_MASK                                                     0x20000000L
-//VGT_STRMOUT_DELAY
-#define VGT_STRMOUT_DELAY__SKIP_DELAY__SHIFT                                                                  0x0
-#define VGT_STRMOUT_DELAY__SE0_WD_DELAY__SHIFT                                                                0x8
-#define VGT_STRMOUT_DELAY__SE1_WD_DELAY__SHIFT                                                                0xb
-#define VGT_STRMOUT_DELAY__SE2_WD_DELAY__SHIFT                                                                0xe
-#define VGT_STRMOUT_DELAY__SE3_WD_DELAY__SHIFT                                                                0x11
-#define VGT_STRMOUT_DELAY__SKIP_DELAY_MASK                                                                    0x000000FFL
-#define VGT_STRMOUT_DELAY__SE0_WD_DELAY_MASK                                                                  0x00000700L
-#define VGT_STRMOUT_DELAY__SE1_WD_DELAY_MASK                                                                  0x00003800L
-#define VGT_STRMOUT_DELAY__SE2_WD_DELAY_MASK                                                                  0x0001C000L
-#define VGT_STRMOUT_DELAY__SE3_WD_DELAY_MASK                                                                  0x000E0000L
-//VGT_FIFO_DEPTHS
-#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH__SHIFT                                                          0x0
-#define VGT_FIFO_DEPTHS__RESERVED_0__SHIFT                                                                    0x7
-#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH__SHIFT                                                              0x8
-#define VGT_FIFO_DEPTHS__HSINPUT_FIFO_DEPTH__SHIFT                                                            0x16
-#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH_MASK                                                            0x0000007FL
-#define VGT_FIFO_DEPTHS__RESERVED_0_MASK                                                                      0x00000080L
-#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH_MASK                                                                0x003FFF00L
-#define VGT_FIFO_DEPTHS__HSINPUT_FIFO_DEPTH_MASK                                                              0x0FC00000L
-//VGT_GS_VERTEX_REUSE
-#define VGT_GS_VERTEX_REUSE__VERT_REUSE__SHIFT                                                                0x0
-#define VGT_GS_VERTEX_REUSE__VERT_REUSE_MASK                                                                  0x0000001FL
-//VGT_MC_LAT_CNTL
-#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES__SHIFT                                                             0x0
-#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES_MASK                                                               0x0000000FL
-//IA_CNTL_STATUS
-#define IA_CNTL_STATUS__IA_BUSY__SHIFT                                                                        0x0
-#define IA_CNTL_STATUS__IA_DMA_BUSY__SHIFT                                                                    0x1
-#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY__SHIFT                                                                0x2
-#define IA_CNTL_STATUS__IA_GRP_BUSY__SHIFT                                                                    0x3
-#define IA_CNTL_STATUS__IA_ADC_BUSY__SHIFT                                                                    0x4
-#define IA_CNTL_STATUS__IA_BUSY_MASK                                                                          0x00000001L
-#define IA_CNTL_STATUS__IA_DMA_BUSY_MASK                                                                      0x00000002L
-#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY_MASK                                                                  0x00000004L
-#define IA_CNTL_STATUS__IA_GRP_BUSY_MASK                                                                      0x00000008L
-#define IA_CNTL_STATUS__IA_ADC_BUSY_MASK                                                                      0x00000010L
-//VGT_CNTL_STATUS
-#define VGT_CNTL_STATUS__VGT_BUSY__SHIFT                                                                      0x0
-#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY__SHIFT                                                             0x1
-#define VGT_CNTL_STATUS__VGT_OUT_BUSY__SHIFT                                                                  0x2
-#define VGT_CNTL_STATUS__VGT_PT_BUSY__SHIFT                                                                   0x3
-#define VGT_CNTL_STATUS__VGT_TE_BUSY__SHIFT                                                                   0x4
-#define VGT_CNTL_STATUS__VGT_VR_BUSY__SHIFT                                                                   0x5
-#define VGT_CNTL_STATUS__VGT_PI_BUSY__SHIFT                                                                   0x6
-#define VGT_CNTL_STATUS__VGT_GS_BUSY__SHIFT                                                                   0x7
-#define VGT_CNTL_STATUS__VGT_HS_BUSY__SHIFT                                                                   0x8
-#define VGT_CNTL_STATUS__VGT_TE11_BUSY__SHIFT                                                                 0x9
-#define VGT_CNTL_STATUS__VGT_PRIMGEN_BUSY__SHIFT                                                              0xa
-#define VGT_CNTL_STATUS__VGT_BUSY_MASK                                                                        0x00000001L
-#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY_MASK                                                               0x00000002L
-#define VGT_CNTL_STATUS__VGT_OUT_BUSY_MASK                                                                    0x00000004L
-#define VGT_CNTL_STATUS__VGT_PT_BUSY_MASK                                                                     0x00000008L
-#define VGT_CNTL_STATUS__VGT_TE_BUSY_MASK                                                                     0x00000010L
-#define VGT_CNTL_STATUS__VGT_VR_BUSY_MASK                                                                     0x00000020L
-#define VGT_CNTL_STATUS__VGT_PI_BUSY_MASK                                                                     0x00000040L
-#define VGT_CNTL_STATUS__VGT_GS_BUSY_MASK                                                                     0x00000080L
-#define VGT_CNTL_STATUS__VGT_HS_BUSY_MASK                                                                     0x00000100L
-#define VGT_CNTL_STATUS__VGT_TE11_BUSY_MASK                                                                   0x00000200L
-#define VGT_CNTL_STATUS__VGT_PRIMGEN_BUSY_MASK                                                                0x00000400L
-//WD_CNTL_STATUS
-#define WD_CNTL_STATUS__WD_BUSY__SHIFT                                                                        0x0
-#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY__SHIFT                                                                0x1
-#define WD_CNTL_STATUS__WD_SPL_DI_BUSY__SHIFT                                                                 0x2
-#define WD_CNTL_STATUS__WD_ADC_BUSY__SHIFT                                                                    0x3
-#define WD_CNTL_STATUS__WD_BUSY_MASK                                                                          0x00000001L
-#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY_MASK                                                                  0x00000002L
-#define WD_CNTL_STATUS__WD_SPL_DI_BUSY_MASK                                                                   0x00000004L
-#define WD_CNTL_STATUS__WD_ADC_BUSY_MASK                                                                      0x00000008L
-//CC_GC_PRIM_CONFIG
-#define CC_GC_PRIM_CONFIG__INACTIVE_IA__SHIFT                                                                 0x10
-#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT                                                             0x18
-#define CC_GC_PRIM_CONFIG__INACTIVE_IA_MASK                                                                   0x00030000L
-#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA_MASK                                                               0x0F000000L
-//GC_USER_PRIM_CONFIG
-#define GC_USER_PRIM_CONFIG__INACTIVE_IA__SHIFT                                                               0x10
-#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT                                                           0x18
-#define GC_USER_PRIM_CONFIG__INACTIVE_IA_MASK                                                                 0x00030000L
-#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA_MASK                                                             0x0F000000L
-//WD_QOS
-#define WD_QOS__DRAW_STALL__SHIFT                                                                             0x0
-#define WD_QOS__DRAW_STALL_MASK                                                                               0x00000001L
-//WD_UTCL1_CNTL
-#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
-#define WD_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
-#define WD_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
-#define WD_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
-#define WD_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
-#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
-#define WD_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
-#define WD_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                             0x1d
-#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
-#define WD_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
-#define WD_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
-#define WD_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
-#define WD_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
-#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
-#define WD_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
-#define WD_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                               0x20000000L
-//WD_UTCL1_STATUS
-#define WD_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
-#define WD_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
-#define WD_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
-#define WD_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
-#define WD_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
-#define WD_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
-#define WD_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
-#define WD_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
-#define WD_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
-#define WD_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
-#define WD_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
-#define WD_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
-//IA_UTCL1_CNTL
-#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
-#define IA_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
-#define IA_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
-#define IA_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
-#define IA_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
-#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
-#define IA_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
-#define IA_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                             0x1d
-#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
-#define IA_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
-#define IA_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
-#define IA_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
-#define IA_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
-#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
-#define IA_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
-#define IA_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                               0x20000000L
-//IA_UTCL1_STATUS
-#define IA_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
-#define IA_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
-#define IA_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
-#define IA_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
-#define IA_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
-#define IA_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
-#define IA_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
-#define IA_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
-#define IA_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
-#define IA_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
-#define IA_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
-#define IA_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
-//VGT_SYS_CONFIG
-#define VGT_SYS_CONFIG__DUAL_CORE_EN__SHIFT                                                                   0x0
-#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP__SHIFT                                                               0x1
-#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE__SHIFT                                                       0x7
-#define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK                                                                     0x00000001L
-#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP_MASK                                                                 0x0000007EL
-#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE_MASK                                                         0x00000080L
-//VGT_VS_MAX_WAVE_ID
-#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
-#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
-//VGT_GS_MAX_WAVE_ID
-#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
-#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
-//GFX_PIPE_CONTROL
-#define GFX_PIPE_CONTROL__HYSTERESIS_CNT__SHIFT                                                               0x0
-#define GFX_PIPE_CONTROL__RESERVED__SHIFT                                                                     0xd
-#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN__SHIFT                                                           0x10
-#define GFX_PIPE_CONTROL__HYSTERESIS_CNT_MASK                                                                 0x00001FFFL
-#define GFX_PIPE_CONTROL__RESERVED_MASK                                                                       0x0000E000L
-#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN_MASK                                                             0x00010000L
-//CC_GC_SHADER_ARRAY_CONFIG
-#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT                                                        0x10
-#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK                                                          0xFFFF0000L
-//GC_USER_SHADER_ARRAY_CONFIG
-#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT                                                      0x10
-#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK                                                        0xFFFF0000L
-//VGT_DMA_PRIMITIVE_TYPE
-#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT                                                              0x0
-#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE_MASK                                                                0x0000003FL
-//VGT_DMA_CONTROL
-#define VGT_DMA_CONTROL__PRIMGROUP_SIZE__SHIFT                                                                0x0
-#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP__SHIFT                                                              0x11
-#define VGT_DMA_CONTROL__SWITCH_ON_EOI__SHIFT                                                                 0x13
-#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP__SHIFT                                                              0x14
-#define VGT_DMA_CONTROL__EN_INST_OPT_BASIC__SHIFT                                                             0x15
-#define VGT_DMA_CONTROL__EN_INST_OPT_ADV__SHIFT                                                               0x16
-#define VGT_DMA_CONTROL__HW_USE_ONLY__SHIFT                                                                   0x17
-#define VGT_DMA_CONTROL__PRIMGROUP_SIZE_MASK                                                                  0x0000FFFFL
-#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP_MASK                                                                0x00020000L
-#define VGT_DMA_CONTROL__SWITCH_ON_EOI_MASK                                                                   0x00080000L
-#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP_MASK                                                                0x00100000L
-#define VGT_DMA_CONTROL__EN_INST_OPT_BASIC_MASK                                                               0x00200000L
-#define VGT_DMA_CONTROL__EN_INST_OPT_ADV_MASK                                                                 0x00400000L
-#define VGT_DMA_CONTROL__HW_USE_ONLY_MASK                                                                     0x00800000L
-//VGT_DMA_LS_HS_CONFIG
-#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT                                                          0x8
-#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK                                                            0x00003F00L
-//WD_BUF_RESOURCE_1
-#define WD_BUF_RESOURCE_1__POS_BUF_SIZE__SHIFT                                                                0x0
-#define WD_BUF_RESOURCE_1__INDEX_BUF_SIZE__SHIFT                                                              0x10
-#define WD_BUF_RESOURCE_1__POS_BUF_SIZE_MASK                                                                  0x0000FFFFL
-#define WD_BUF_RESOURCE_1__INDEX_BUF_SIZE_MASK                                                                0xFFFF0000L
-//WD_BUF_RESOURCE_2
-#define WD_BUF_RESOURCE_2__PARAM_BUF_SIZE__SHIFT                                                              0x0
-#define WD_BUF_RESOURCE_2__ADDR_MODE__SHIFT                                                                   0xf
-#define WD_BUF_RESOURCE_2__CNTL_SB_BUF_SIZE__SHIFT                                                            0x10
-#define WD_BUF_RESOURCE_2__PARAM_BUF_SIZE_MASK                                                                0x00001FFFL
-#define WD_BUF_RESOURCE_2__ADDR_MODE_MASK                                                                     0x00008000L
-#define WD_BUF_RESOURCE_2__CNTL_SB_BUF_SIZE_MASK                                                              0xFFFF0000L
-//PA_CL_CNTL_STATUS
-#define PA_CL_CNTL_STATUS__UTC_FAULT_DETECTED__SHIFT                                                          0x0
-#define PA_CL_CNTL_STATUS__UTC_RETRY_DETECTED__SHIFT                                                          0x1
-#define PA_CL_CNTL_STATUS__UTC_PRT_DETECTED__SHIFT                                                            0x2
-#define PA_CL_CNTL_STATUS__UTC_FAULT_DETECTED_MASK                                                            0x00000001L
-#define PA_CL_CNTL_STATUS__UTC_RETRY_DETECTED_MASK                                                            0x00000002L
-#define PA_CL_CNTL_STATUS__UTC_PRT_DETECTED_MASK                                                              0x00000004L
-//PA_CL_ENHANCE
-#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA__SHIFT                                                            0x0
-#define PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT                                                                    0x1
-#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL__SHIFT                                                          0x3
-#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE__SHIFT                                                             0x4
-#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET__SHIFT                                                           0x6
-#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS__SHIFT                                                           0x7
-#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC__SHIFT                                                                0x8
-#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION__SHIFT                                                0x9
-#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER__SHIFT                                                          0xb
-#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH__SHIFT                                                       0xc
-#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH__SHIFT                                                     0xe
-#define PA_CL_ENHANCE__ECO_SPARE3__SHIFT                                                                      0x1c
-#define PA_CL_ENHANCE__ECO_SPARE2__SHIFT                                                                      0x1d
-#define PA_CL_ENHANCE__ECO_SPARE1__SHIFT                                                                      0x1e
-#define PA_CL_ENHANCE__ECO_SPARE0__SHIFT                                                                      0x1f
-#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK                                                              0x00000001L
-#define PA_CL_ENHANCE__NUM_CLIP_SEQ_MASK                                                                      0x00000006L
-#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL_MASK                                                            0x00000008L
-#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE_MASK                                                               0x00000010L
-#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET_MASK                                                             0x00000040L
-#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS_MASK                                                             0x00000080L
-#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC_MASK                                                                  0x00000100L
-#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION_MASK                                                  0x00000600L
-#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER_MASK                                                            0x00000800L
-#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH_MASK                                                         0x00003000L
-#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH_MASK                                                       0x0001C000L
-#define PA_CL_ENHANCE__ECO_SPARE3_MASK                                                                        0x10000000L
-#define PA_CL_ENHANCE__ECO_SPARE2_MASK                                                                        0x20000000L
-#define PA_CL_ENHANCE__ECO_SPARE1_MASK                                                                        0x40000000L
-#define PA_CL_ENHANCE__ECO_SPARE0_MASK                                                                        0x80000000L
-//PA_SU_CNTL_STATUS
-#define PA_SU_CNTL_STATUS__SU_BUSY__SHIFT                                                                     0x1f
-#define PA_SU_CNTL_STATUS__SU_BUSY_MASK                                                                       0x80000000L
-//PA_SC_FIFO_DEPTH_CNTL
-#define PA_SC_FIFO_DEPTH_CNTL__DEPTH__SHIFT                                                                   0x0
-#define PA_SC_FIFO_DEPTH_CNTL__DEPTH_MASK                                                                     0x000003FFL
-//PA_SC_P3D_TRAP_SCREEN_HV_LOCK
-#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                         0x0
-#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                           0x00000001L
-//PA_SC_HP3D_TRAP_SCREEN_HV_LOCK
-#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                        0x0
-#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                          0x00000001L
-//PA_SC_TRAP_SCREEN_HV_LOCK
-#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                             0x0
-#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                               0x00000001L
-//PA_SC_FORCE_EOV_MAX_CNTS
-#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT                                                0x0
-#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT                                                0x10
-#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT_MASK                                                  0x0000FFFFL
-#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT_MASK                                                  0xFFFF0000L
-//PA_SC_BINNER_EVENT_CNTL_0
-#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0__SHIFT                                                          0x0
-#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1__SHIFT                                              0x2
-#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2__SHIFT                                              0x4
-#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3__SHIFT                                              0x6
-#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS__SHIFT                                                      0x8
-#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE__SHIFT                                                        0xa
-#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH__SHIFT                                                         0xc
-#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH__SHIFT                                                    0xe
-#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC__SHIFT                                                  0x10
-#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_9__SHIFT                                                          0x12
-#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET__SHIFT                                                 0x14
-#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE__SHIFT                                                 0x16
-#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END__SHIFT                                                  0x18
-#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT__SHIFT                                                         0x1a
-#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH__SHIFT                                                         0x1c
-#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH__SHIFT                                                    0x1e
-#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0_MASK                                                            0x00000003L
-#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1_MASK                                                0x0000000CL
-#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2_MASK                                                0x00000030L
-#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3_MASK                                                0x000000C0L
-#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS_MASK                                                        0x00000300L
-#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE_MASK                                                          0x00000C00L
-#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_MASK                                                           0x00003000L
-#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH_MASK                                                      0x0000C000L
-#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC_MASK                                                    0x00030000L
-#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_9_MASK                                                            0x000C0000L
-#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET_MASK                                                   0x00300000L
-#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE_MASK                                                   0x00C00000L
-#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END_MASK                                                    0x03000000L
-#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT_MASK                                                           0x0C000000L
-#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH_MASK                                                           0x30000000L
-#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH_MASK                                                      0xC0000000L
-//PA_SC_BINNER_EVENT_CNTL_1
-#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH__SHIFT                                                    0x0
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT__SHIFT                                                     0x2
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM__SHIFT                                                          0x4
-#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT__SHIFT                                                 0x6
-#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT__SHIFT                                        0x8
-#define PA_SC_BINNER_EVENT_CNTL_1__ZPASS_DONE__SHIFT                                                          0xa
-#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT__SHIFT                                           0xc
-#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START__SHIFT                                                   0xe
-#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP__SHIFT                                                    0x10
-#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START__SHIFT                                                  0x12
-#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP__SHIFT                                                   0x14
-#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE__SHIFT                                                  0x16
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT__SHIFT                                                     0x18
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_GS_OUTPUT__SHIFT                                                     0x1a
-#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT__SHIFT                                                 0x1c
-#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH__SHIFT                                               0x1e
-#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH_MASK                                                      0x00000003L
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT_MASK                                                       0x0000000CL
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM_MASK                                                            0x00000030L
-#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT_MASK                                                   0x000000C0L
-#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT_MASK                                          0x00000300L
-#define PA_SC_BINNER_EVENT_CNTL_1__ZPASS_DONE_MASK                                                            0x00000C00L
-#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT_MASK                                             0x00003000L
-#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START_MASK                                                     0x0000C000L
-#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP_MASK                                                      0x00030000L
-#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START_MASK                                                    0x000C0000L
-#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP_MASK                                                     0x00300000L
-#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE_MASK                                                    0x00C00000L
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT_MASK                                                       0x03000000L
-#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_GS_OUTPUT_MASK                                                       0x0C000000L
-#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT_MASK                                                   0x30000000L
-#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH_MASK                                                 0xC0000000L
-//PA_SC_BINNER_EVENT_CNTL_2
-#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS__SHIFT                                               0x0
-#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT__SHIFT                                                       0x2
-#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE__SHIFT                                                  0x4
-#define PA_SC_BINNER_EVENT_CNTL_2__CS_CONTEXT_DONE__SHIFT                                                     0x6
-#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH__SHIFT                                                           0x8
-#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER__SHIFT                                                       0xa
-#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT__SHIFT                                                        0xc
-#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ__SHIFT                                                      0xe
-#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS__SHIFT                                                   0x10
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_SX_TS__SHIFT                                                         0x12
-#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV__SHIFT                                              0x14
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS__SHIFT                                            0x16
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META__SHIFT                                               0x18
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS__SHIFT                                            0x1a
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META__SHIFT                                               0x1c
-#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE__SHIFT                                                             0x1e
-#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS_MASK                                                 0x00000003L
-#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT_MASK                                                         0x0000000CL
-#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE_MASK                                                    0x00000030L
-#define PA_SC_BINNER_EVENT_CNTL_2__CS_CONTEXT_DONE_MASK                                                       0x000000C0L
-#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH_MASK                                                             0x00000300L
-#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER_MASK                                                         0x00000C00L
-#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT_MASK                                                          0x00003000L
-#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ_MASK                                                        0x0000C000L
-#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS_MASK                                                     0x00030000L
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_SX_TS_MASK                                                           0x000C0000L
-#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV_MASK                                                0x00300000L
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS_MASK                                              0x00C00000L
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META_MASK                                                 0x03000000L
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS_MASK                                              0x0C000000L
-#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META_MASK                                                 0x30000000L
-#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE_MASK                                                               0xC0000000L
-//PA_SC_BINNER_EVENT_CNTL_3
-#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE__SHIFT                                                             0x0
-#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA__SHIFT                                         0x2
-#define PA_SC_BINNER_EVENT_CNTL_3__SX_CB_RAT_ACK_REQUEST__SHIFT                                               0x4
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START__SHIFT                                                  0x6
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP__SHIFT                                                   0x8
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER__SHIFT                                                 0xa
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FLUSH__SHIFT                                                  0xc
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH__SHIFT                                                 0xe
-#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL__SHIFT                                             0x10
-#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP__SHIFT                                                0x12
-#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET__SHIFT                                               0x14
-#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND__SHIFT                                                     0x16
-#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC__SHIFT                                                  0x18
-#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE__SHIFT                                                 0x1a
-#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_LEGACY_PIPELINE__SHIFT                                              0x1c
-#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_63__SHIFT                                                         0x1e
-#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE_MASK                                                               0x00000003L
-#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA_MASK                                           0x0000000CL
-#define PA_SC_BINNER_EVENT_CNTL_3__SX_CB_RAT_ACK_REQUEST_MASK                                                 0x00000030L
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START_MASK                                                    0x000000C0L
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP_MASK                                                     0x00000300L
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER_MASK                                                   0x00000C00L
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FLUSH_MASK                                                    0x00003000L
-#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH_MASK                                                   0x0000C000L
-#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL_MASK                                               0x00030000L
-#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP_MASK                                                  0x000C0000L
-#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET_MASK                                                 0x00300000L
-#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND_MASK                                                       0x00C00000L
-#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC_MASK                                                    0x03000000L
-#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE_MASK                                                   0x0C000000L
-#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_LEGACY_PIPELINE_MASK                                                0x30000000L
-#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_63_MASK                                                           0xC0000000L
-//PA_SC_BINNER_TIMEOUT_COUNTER
-#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD__SHIFT                                                        0x0
-#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD_MASK                                                          0xFFFFFFFFL
-//PA_SC_BINNER_PERF_CNTL_0
-#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                         0x0
-#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                       0xa
-#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                       0x14
-#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                     0x17
-#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD_MASK                                           0x000003FFL
-#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD_MASK                                         0x000FFC00L
-#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD_MASK                                         0x00700000L
-#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD_MASK                                       0x03800000L
-//PA_SC_BINNER_PERF_CNTL_1
-#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                              0x0
-#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                            0x5
-#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD__SHIFT                         0xa
-#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                                0x0000001FL
-#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                              0x000003E0L
-#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD_MASK                           0x03FFFC00L
-//PA_SC_BINNER_PERF_CNTL_2
-#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD__SHIFT                               0x0
-#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD__SHIFT                             0xb
-#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD_MASK                                 0x000007FFL
-#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD_MASK                               0x003FF800L
-//PA_SC_BINNER_PERF_CNTL_3
-#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD__SHIFT                              0x0
-#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD_MASK                                0xFFFFFFFFL
-//PA_SC_FIFO_SIZE
-#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT                                                    0x0
-#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT                                                     0x6
-#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT                                                         0xf
-#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT                                                      0x15
-#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE_MASK                                                      0x0000003FL
-#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE_MASK                                                       0x00007FC0L
-#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE_MASK                                                           0x001F8000L
-#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE_MASK                                                        0xFFE00000L
-//PA_SC_IF_FIFO_SIZE
-#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE__SHIFT                                                    0x0
-#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE__SHIFT                                                    0x6
-#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE__SHIFT                                                        0xc
-#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE__SHIFT                                                        0x12
-#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE_MASK                                                      0x0000003FL
-#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE_MASK                                                      0x00000FC0L
-#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE_MASK                                                          0x0003F000L
-#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE_MASK                                                          0x00FC0000L
-//PA_SC_PKR_WAVE_TABLE_CNTL
-#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE__SHIFT                                                                0x0
-#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE_MASK                                                                  0x0000003FL
-//PA_UTCL1_CNTL1
-#define PA_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                               0x0
-#define PA_UTCL1_CNTL1__GPUVM_64K_DEFAULT__SHIFT                                                              0x1
-#define PA_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                                0x2
-#define PA_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                      0x3
-#define PA_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                                0x5
-#define PA_UTCL1_CNTL1__CLIENTID__SHIFT                                                                       0x7
-#define PA_UTCL1_CNTL1__SPARE__SHIFT                                                                          0x10
-#define PA_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                              0x11
-#define PA_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                           0x12
-#define PA_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                   0x13
-#define PA_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                               0x17
-#define PA_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                 0x18
-#define PA_UTCL1_CNTL1__INVALIDATE_ALL_VMID__SHIFT                                                            0x19
-#define PA_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                     0x1a
-#define PA_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                 0x1b
-#define PA_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                         0x1c
-#define PA_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                         0x1e
-#define PA_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                 0x00000001L
-#define PA_UTCL1_CNTL1__GPUVM_64K_DEFAULT_MASK                                                                0x00000002L
-#define PA_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                  0x00000004L
-#define PA_UTCL1_CNTL1__RESP_MODE_MASK                                                                        0x00000018L
-#define PA_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                  0x00000060L
-#define PA_UTCL1_CNTL1__CLIENTID_MASK                                                                         0x0000FF80L
-#define PA_UTCL1_CNTL1__SPARE_MASK                                                                            0x00010000L
-#define PA_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                                0x00020000L
-#define PA_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                             0x00040000L
-#define PA_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                     0x00780000L
-#define PA_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                 0x00800000L
-#define PA_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                   0x01000000L
-#define PA_UTCL1_CNTL1__INVALIDATE_ALL_VMID_MASK                                                              0x02000000L
-#define PA_UTCL1_CNTL1__FORCE_MISS_MASK                                                                       0x04000000L
-#define PA_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                   0x08000000L
-#define PA_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                           0x30000000L
-#define PA_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                           0xC0000000L
-//PA_UTCL1_CNTL2
-#define PA_UTCL1_CNTL2__SPARE1__SHIFT                                                                         0x0
-#define PA_UTCL1_CNTL2__SPARE2__SHIFT                                                                         0x8
-#define PA_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                 0x9
-#define PA_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                     0xa
-#define PA_UTCL1_CNTL2__SPARE3__SHIFT                                                                         0xb
-#define PA_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                 0xc
-#define PA_UTCL1_CNTL2__ENABLE_SHOOTDOWN_OPT__SHIFT                                                           0xd
-#define PA_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                    0xe
-#define PA_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                            0xf
-#define PA_UTCL1_CNTL2__SPARE4__SHIFT                                                                         0x10
-#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                        0x12
-#define PA_UTCL1_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                               0x13
-#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                         0x14
-#define PA_UTCL1_CNTL2__PERF_EVENT_VMID__SHIFT                                                                0x15
-#define PA_UTCL1_CNTL2__SPARE5__SHIFT                                                                         0x19
-#define PA_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                           0x1a
-#define PA_UTCL1_CNTL2__RESERVED__SHIFT                                                                       0x1b
-#define PA_UTCL1_CNTL2__SPARE1_MASK                                                                           0x000000FFL
-#define PA_UTCL1_CNTL2__SPARE2_MASK                                                                           0x00000100L
-#define PA_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                   0x00000200L
-#define PA_UTCL1_CNTL2__LINE_VALID_MASK                                                                       0x00000400L
-#define PA_UTCL1_CNTL2__SPARE3_MASK                                                                           0x00000800L
-#define PA_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                   0x00001000L
-#define PA_UTCL1_CNTL2__ENABLE_SHOOTDOWN_OPT_MASK                                                             0x00002000L
-#define PA_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                      0x00004000L
-#define PA_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                              0x00008000L
-#define PA_UTCL1_CNTL2__SPARE4_MASK                                                                           0x00030000L
-#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                          0x00040000L
-#define PA_UTCL1_CNTL2__PERF_EVENT_RD_WR_MASK                                                                 0x00080000L
-#define PA_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                           0x00100000L
-#define PA_UTCL1_CNTL2__PERF_EVENT_VMID_MASK                                                                  0x01E00000L
-#define PA_UTCL1_CNTL2__SPARE5_MASK                                                                           0x02000000L
-#define PA_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                             0x04000000L
-#define PA_UTCL1_CNTL2__RESERVED_MASK                                                                         0xF8000000L
-//PA_SIDEBAND_REQUEST_DELAYS
-#define PA_SIDEBAND_REQUEST_DELAYS__RETRY_DELAY__SHIFT                                                        0x0
-#define PA_SIDEBAND_REQUEST_DELAYS__INITIAL_DELAY__SHIFT                                                      0x10
-#define PA_SIDEBAND_REQUEST_DELAYS__RETRY_DELAY_MASK                                                          0x0000FFFFL
-#define PA_SIDEBAND_REQUEST_DELAYS__INITIAL_DELAY_MASK                                                        0xFFFF0000L
-//PA_SC_ENHANCE
-#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER__SHIFT                                                       0x0
-#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX__SHIFT                                                          0x1
-#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX__SHIFT                                                        0x2
-#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS__SHIFT                                                  0x3
-#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID__SHIFT                                               0x4
-#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX__SHIFT                                                             0x5
-#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER__SHIFT                                                     0x6
-#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION__SHIFT                                              0x7
-#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM__SHIFT                                                   0x8
-#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE__SHIFT                                              0x9
-#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE__SHIFT                                                   0xa
-#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE__SHIFT                                                          0xb
-#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS__SHIFT                                          0xc
-#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE__SHIFT                                                 0xd
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE__SHIFT                                             0xe
-#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE__SHIFT                                                   0xf
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST__SHIFT                                   0x10
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING__SHIFT                                        0x11
-#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY__SHIFT                               0x12
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING__SHIFT                               0x13
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING__SHIFT                              0x14
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS__SHIFT                                 0x15
-#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID__SHIFT                                   0x16
-#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO__SHIFT                           0x17
-#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT                                          0x18
-#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING__SHIFT                                       0x19
-#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET__SHIFT                                                  0x1a
-#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET__SHIFT                                              0x1b
-#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE__SHIFT                      0x1c
-#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING__SHIFT                              0x1d
-#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK                                                         0x00000001L
-#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX_MASK                                                            0x00000002L
-#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX_MASK                                                          0x00000004L
-#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS_MASK                                                    0x00000008L
-#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID_MASK                                                 0x00000010L
-#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX_MASK                                                               0x00000020L
-#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER_MASK                                                       0x00000040L
-#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION_MASK                                                0x00000080L
-#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM_MASK                                                     0x00000100L
-#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE_MASK                                                0x00000200L
-#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE_MASK                                                     0x00000400L
-#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE_MASK                                                            0x00000800L
-#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS_MASK                                            0x00001000L
-#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE_MASK                                                   0x00002000L
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE_MASK                                               0x00004000L
-#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE_MASK                                                     0x00008000L
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST_MASK                                     0x00010000L
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING_MASK                                          0x00020000L
-#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY_MASK                                 0x00040000L
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING_MASK                                 0x00080000L
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING_MASK                                0x00100000L
-#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS_MASK                                   0x00200000L
-#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID_MASK                                     0x00400000L
-#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO_MASK                             0x00800000L
-#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK                                            0x01000000L
-#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING_MASK                                         0x02000000L
-#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET_MASK                                                    0x04000000L
-#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET_MASK                                                0x08000000L
-#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE_MASK                        0x10000000L
-#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING_MASK                                0x20000000L
-//PA_SC_ENHANCE_1
-#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE__SHIFT                                                0x0
-#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE__SHIFT                                                       0x1
-#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING__SHIFT                                                            0x3
-#define PA_SC_ENHANCE_1__BYPASS_PBB__SHIFT                                                                    0x4
-#define PA_SC_ENHANCE_1__ECO_SPARE0__SHIFT                                                                    0x5
-#define PA_SC_ENHANCE_1__ECO_SPARE1__SHIFT                                                                    0x6
-#define PA_SC_ENHANCE_1__ECO_SPARE2__SHIFT                                                                    0x7
-#define PA_SC_ENHANCE_1__ECO_SPARE3__SHIFT                                                                    0x8
-#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB__SHIFT                                                  0x9
-#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT__SHIFT                                                       0xa
-#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM__SHIFT                                     0xb
-#define PA_SC_ENHANCE_1__DISABLE_PACKER_GRAD_FDCE_ENHANCE__SHIFT                                              0xd
-#define PA_SC_ENHANCE_1__DISABLE_SC_DB_TILE_INTF_FINE_CLOCK_GATE__SHIFT                                       0xe
-#define PA_SC_ENHANCE_1__DISABLE_SC_PIPELINE_RESET_LEGACY_MODE_TRANSITION__SHIFT                              0xf
-#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE__SHIFT                                                    0x10
-#define PA_SC_ENHANCE_1__ALLOW_SCALE_LINE_WIDTH_PAD_WITH_BINNING__SHIFT                                       0x11
-#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION__SHIFT                                                         0x12
-#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS__SHIFT                                                  0x13
-#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION__SHIFT                                                  0x14
-#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION__SHIFT                                          0x15
-#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION__SHIFT                                          0x16
-#define PA_SC_ENHANCE_1__RSVD__SHIFT                                                                          0x17
-#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE_MASK                                                  0x00000001L
-#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_MASK                                                         0x00000006L
-#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING_MASK                                                              0x00000008L
-#define PA_SC_ENHANCE_1__BYPASS_PBB_MASK                                                                      0x00000010L
-#define PA_SC_ENHANCE_1__ECO_SPARE0_MASK                                                                      0x00000020L
-#define PA_SC_ENHANCE_1__ECO_SPARE1_MASK                                                                      0x00000040L
-#define PA_SC_ENHANCE_1__ECO_SPARE2_MASK                                                                      0x00000080L
-#define PA_SC_ENHANCE_1__ECO_SPARE3_MASK                                                                      0x00000100L
-#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB_MASK                                                    0x00000200L
-#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT_MASK                                                         0x00000400L
-#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM_MASK                                       0x00000800L
-#define PA_SC_ENHANCE_1__DISABLE_PACKER_GRAD_FDCE_ENHANCE_MASK                                                0x00002000L
-#define PA_SC_ENHANCE_1__DISABLE_SC_DB_TILE_INTF_FINE_CLOCK_GATE_MASK                                         0x00004000L
-#define PA_SC_ENHANCE_1__DISABLE_SC_PIPELINE_RESET_LEGACY_MODE_TRANSITION_MASK                                0x00008000L
-#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE_MASK                                                      0x00010000L
-#define PA_SC_ENHANCE_1__ALLOW_SCALE_LINE_WIDTH_PAD_WITH_BINNING_MASK                                         0x00020000L
-#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION_MASK                                                           0x00040000L
-#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS_MASK                                                    0x00080000L
-#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION_MASK                                                    0x00100000L
-#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION_MASK                                            0x00200000L
-#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION_MASK                                            0x00400000L
-#define PA_SC_ENHANCE_1__RSVD_MASK                                                                            0xFF800000L
-//PA_SC_DSM_CNTL
-#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0__SHIFT                                                                0x0
-#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1__SHIFT                                                                0x1
-#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0_MASK                                                                  0x00000001L
-#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1_MASK                                                                  0x00000002L
-//PA_SC_TILE_STEERING_CREST_OVERRIDE
-#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE__SHIFT                                         0x0
-#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT__SHIFT                                                  0x1
-#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT__SHIFT                                                  0x5
-#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE_MASK                                           0x00000001L
-#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT_MASK                                                    0x00000006L
-#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT_MASK                                                    0x00000060L
-
-
-// addressBlock: gc_sqdec
-//SQ_CONFIG
-#define SQ_CONFIG__UNUSED__SHIFT                                                                              0x0
-#define SQ_CONFIG__OVERRIDE_ALU_BUSY__SHIFT                                                                   0x7
-#define SQ_CONFIG__OVERRIDE_LDS_IDX_BUSY__SHIFT                                                               0xb
-#define SQ_CONFIG__EARLY_TA_DONE_DISABLE__SHIFT                                                               0xc
-#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE__SHIFT                                                                0xd
-#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE__SHIFT                                                              0xe
-#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE__SHIFT                                                       0xf
-#define SQ_CONFIG__DISABLE_VMEM_SOFT_CLAUSE__SHIFT                                                            0x10
-#define SQ_CONFIG__DISABLE_SMEM_SOFT_CLAUSE__SHIFT                                                            0x11
-#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_VS__SHIFT                                                         0x12
-#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_VS__SHIFT                                                              0x13
-#define SQ_CONFIG__REPLAY_SLEEP_CNT__SHIFT                                                                    0x15
-#define SQ_CONFIG__DISABLE_SP_VGPR_WRITE_SKIP__SHIFT                                                          0x1c
-#define SQ_CONFIG__DISABLE_SP_REDUNDANT_THREAD_GATING__SHIFT                                                  0x1d
-#define SQ_CONFIG__DISABLE_FLAT_SOFT_CLAUSE__SHIFT                                                            0x1e
-#define SQ_CONFIG__DISABLE_MIMG_SOFT_CLAUSE__SHIFT                                                            0x1f
-#define SQ_CONFIG__UNUSED_MASK                                                                                0x0000007FL
-#define SQ_CONFIG__OVERRIDE_ALU_BUSY_MASK                                                                     0x00000080L
-#define SQ_CONFIG__OVERRIDE_LDS_IDX_BUSY_MASK                                                                 0x00000800L
-#define SQ_CONFIG__EARLY_TA_DONE_DISABLE_MASK                                                                 0x00001000L
-#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE_MASK                                                                  0x00002000L
-#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE_MASK                                                                0x00004000L
-#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE_MASK                                                         0x00008000L
-#define SQ_CONFIG__DISABLE_VMEM_SOFT_CLAUSE_MASK                                                              0x00010000L
-#define SQ_CONFIG__DISABLE_SMEM_SOFT_CLAUSE_MASK                                                              0x00020000L
-#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_VS_MASK                                                           0x00040000L
-#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_VS_MASK                                                                0x00180000L
-#define SQ_CONFIG__REPLAY_SLEEP_CNT_MASK                                                                      0x0FE00000L
-#define SQ_CONFIG__DISABLE_SP_VGPR_WRITE_SKIP_MASK                                                            0x10000000L
-#define SQ_CONFIG__DISABLE_SP_REDUNDANT_THREAD_GATING_MASK                                                    0x20000000L
-#define SQ_CONFIG__DISABLE_FLAT_SOFT_CLAUSE_MASK                                                              0x40000000L
-#define SQ_CONFIG__DISABLE_MIMG_SOFT_CLAUSE_MASK                                                              0x80000000L
-//SQC_CONFIG
-#define SQC_CONFIG__INST_CACHE_SIZE__SHIFT                                                                    0x0
-#define SQC_CONFIG__DATA_CACHE_SIZE__SHIFT                                                                    0x2
-#define SQC_CONFIG__MISS_FIFO_DEPTH__SHIFT                                                                    0x4
-#define SQC_CONFIG__HIT_FIFO_DEPTH__SHIFT                                                                     0x6
-#define SQC_CONFIG__FORCE_ALWAYS_MISS__SHIFT                                                                  0x7
-#define SQC_CONFIG__FORCE_IN_ORDER__SHIFT                                                                     0x8
-#define SQC_CONFIG__IDENTITY_HASH_BANK__SHIFT                                                                 0x9
-#define SQC_CONFIG__IDENTITY_HASH_SET__SHIFT                                                                  0xa
-#define SQC_CONFIG__PER_VMID_INV_DISABLE__SHIFT                                                               0xb
-#define SQC_CONFIG__EVICT_LRU__SHIFT                                                                          0xc
-#define SQC_CONFIG__FORCE_2_BANK__SHIFT                                                                       0xe
-#define SQC_CONFIG__FORCE_1_BANK__SHIFT                                                                       0xf
-#define SQC_CONFIG__LS_DISABLE_CLOCKS__SHIFT                                                                  0x10
-#define SQC_CONFIG__INST_PRF_COUNT__SHIFT                                                                     0x18
-#define SQC_CONFIG__INST_PRF_FILTER_DIS__SHIFT                                                                0x1a
-#define SQC_CONFIG__INST_CACHE_SIZE_MASK                                                                      0x00000003L
-#define SQC_CONFIG__DATA_CACHE_SIZE_MASK                                                                      0x0000000CL
-#define SQC_CONFIG__MISS_FIFO_DEPTH_MASK                                                                      0x00000030L
-#define SQC_CONFIG__HIT_FIFO_DEPTH_MASK                                                                       0x00000040L
-#define SQC_CONFIG__FORCE_ALWAYS_MISS_MASK                                                                    0x00000080L
-#define SQC_CONFIG__FORCE_IN_ORDER_MASK                                                                       0x00000100L
-#define SQC_CONFIG__IDENTITY_HASH_BANK_MASK                                                                   0x00000200L
-#define SQC_CONFIG__IDENTITY_HASH_SET_MASK                                                                    0x00000400L
-#define SQC_CONFIG__PER_VMID_INV_DISABLE_MASK                                                                 0x00000800L
-#define SQC_CONFIG__EVICT_LRU_MASK                                                                            0x00003000L
-#define SQC_CONFIG__FORCE_2_BANK_MASK                                                                         0x00004000L
-#define SQC_CONFIG__FORCE_1_BANK_MASK                                                                         0x00008000L
-#define SQC_CONFIG__LS_DISABLE_CLOCKS_MASK                                                                    0x00FF0000L
-#define SQC_CONFIG__INST_PRF_COUNT_MASK                                                                       0x03000000L
-#define SQC_CONFIG__INST_PRF_FILTER_DIS_MASK                                                                  0x04000000L
-//LDS_CONFIG
-#define LDS_CONFIG__ADDR_OUT_OF_RANGE_REPORTING__SHIFT                                                        0x0
-#define LDS_CONFIG__ADDR_OUT_OF_RANGE_REPORTING_MASK                                                          0x00000001L
-//SQ_RANDOM_WAVE_PRI
-#define SQ_RANDOM_WAVE_PRI__RET__SHIFT                                                                        0x0
-#define SQ_RANDOM_WAVE_PRI__RUI__SHIFT                                                                        0x7
-#define SQ_RANDOM_WAVE_PRI__RNG__SHIFT                                                                        0xa
-#define SQ_RANDOM_WAVE_PRI__RET_MASK                                                                          0x0000007FL
-#define SQ_RANDOM_WAVE_PRI__RUI_MASK                                                                          0x00000380L
-#define SQ_RANDOM_WAVE_PRI__RNG_MASK                                                                          0x007FFC00L
-//SQ_REG_CREDITS
-#define SQ_REG_CREDITS__SRBM_CREDITS__SHIFT                                                                   0x0
-#define SQ_REG_CREDITS__CMD_CREDITS__SHIFT                                                                    0x8
-#define SQ_REG_CREDITS__REG_BUSY__SHIFT                                                                       0x1c
-#define SQ_REG_CREDITS__SRBM_OVERFLOW__SHIFT                                                                  0x1d
-#define SQ_REG_CREDITS__IMMED_OVERFLOW__SHIFT                                                                 0x1e
-#define SQ_REG_CREDITS__CMD_OVERFLOW__SHIFT                                                                   0x1f
-#define SQ_REG_CREDITS__SRBM_CREDITS_MASK                                                                     0x0000003FL
-#define SQ_REG_CREDITS__CMD_CREDITS_MASK                                                                      0x00000F00L
-#define SQ_REG_CREDITS__REG_BUSY_MASK                                                                         0x10000000L
-#define SQ_REG_CREDITS__SRBM_OVERFLOW_MASK                                                                    0x20000000L
-#define SQ_REG_CREDITS__IMMED_OVERFLOW_MASK                                                                   0x40000000L
-#define SQ_REG_CREDITS__CMD_OVERFLOW_MASK                                                                     0x80000000L
-//SQ_FIFO_SIZES
-#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE__SHIFT                                                             0x0
-#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE__SHIFT                                                                0x8
-#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE__SHIFT                                                                 0x10
-#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE__SHIFT                                                             0x12
-#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE_MASK                                                               0x0000000FL
-#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE_MASK                                                                  0x00000F00L
-#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE_MASK                                                                   0x00030000L
-#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE_MASK                                                               0x000C0000L
-//SQ_DSM_CNTL
-#define SQ_DSM_CNTL__WAVEFRONT_STALL_0__SHIFT                                                                 0x0
-#define SQ_DSM_CNTL__WAVEFRONT_STALL_1__SHIFT                                                                 0x1
-#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0__SHIFT                                                                0x2
-#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1__SHIFT                                                                0x3
-#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0__SHIFT                                                      0x8
-#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1__SHIFT                                                      0x9
-#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE__SHIFT                                                          0xa
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0__SHIFT                                                       0x10
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1__SHIFT                                                       0x11
-#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01__SHIFT                                                         0x12
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2__SHIFT                                                       0x13
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3__SHIFT                                                       0x14
-#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23__SHIFT                                                         0x15
-#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0__SHIFT                                                        0x18
-#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1__SHIFT                                                        0x19
-#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE__SHIFT                                                            0x1a
-#define SQ_DSM_CNTL__WAVEFRONT_STALL_0_MASK                                                                   0x00000001L
-#define SQ_DSM_CNTL__WAVEFRONT_STALL_1_MASK                                                                   0x00000002L
-#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0_MASK                                                                  0x00000004L
-#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1_MASK                                                                  0x00000008L
-#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0_MASK                                                        0x00000100L
-#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1_MASK                                                        0x00000200L
-#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE_MASK                                                            0x00000400L
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0_MASK                                                         0x00010000L
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1_MASK                                                         0x00020000L
-#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01_MASK                                                           0x00040000L
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2_MASK                                                         0x00080000L
-#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3_MASK                                                         0x00100000L
-#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23_MASK                                                           0x00200000L
-#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0_MASK                                                          0x01000000L
-#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1_MASK                                                          0x02000000L
-#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE_MASK                                                              0x04000000L
-//SQ_DSM_CNTL2
-#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT__SHIFT                                                         0x0
-#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY__SHIFT                                                         0x2
-#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT__SHIFT                                                        0x3
-#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY__SHIFT                                                        0x5
-#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT__SHIFT                                                        0x6
-#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY__SHIFT                                                        0x8
-#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT__SHIFT                                                           0x9
-#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY__SHIFT                                                           0xb
-#define SQ_DSM_CNTL2__LDS_INJECT_DELAY__SHIFT                                                                 0xe
-#define SQ_DSM_CNTL2__SP_INJECT_DELAY__SHIFT                                                                  0x14
-#define SQ_DSM_CNTL2__SQ_INJECT_DELAY__SHIFT                                                                  0x1a
-#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT_MASK                                                           0x00000003L
-#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY_MASK                                                           0x00000004L
-#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT_MASK                                                          0x00000018L
-#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY_MASK                                                          0x00000020L
-#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT_MASK                                                          0x000000C0L
-#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY_MASK                                                          0x00000100L
-#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT_MASK                                                             0x00000600L
-#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY_MASK                                                             0x00000800L
-#define SQ_DSM_CNTL2__LDS_INJECT_DELAY_MASK                                                                   0x000FC000L
-#define SQ_DSM_CNTL2__SP_INJECT_DELAY_MASK                                                                    0x03F00000L
-#define SQ_DSM_CNTL2__SQ_INJECT_DELAY_MASK                                                                    0xFC000000L
-//SQ_RUNTIME_CONFIG
-#define SQ_RUNTIME_CONFIG__ENABLE_TEX_ARB_OLDEST__SHIFT                                                       0x0
-#define SQ_RUNTIME_CONFIG__ENABLE_TEX_ARB_OLDEST_MASK                                                         0x00000001L
-//SH_MEM_BASES
-#define SH_MEM_BASES__PRIVATE_BASE__SHIFT                                                                     0x0
-#define SH_MEM_BASES__SHARED_BASE__SHIFT                                                                      0x10
-#define SH_MEM_BASES__PRIVATE_BASE_MASK                                                                       0x0000FFFFL
-#define SH_MEM_BASES__SHARED_BASE_MASK                                                                        0xFFFF0000L
-//SH_MEM_CONFIG
-#define SH_MEM_CONFIG__ADDRESS_MODE__SHIFT                                                                    0x0
-#define SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT                                                                  0x3
-#define SH_MEM_CONFIG__RETRY_DISABLE__SHIFT                                                                   0xc
-#define SH_MEM_CONFIG__PRIVATE_NV__SHIFT                                                                      0xd
-#define SH_MEM_CONFIG__ADDRESS_MODE_MASK                                                                      0x00000001L
-#define SH_MEM_CONFIG__ALIGNMENT_MODE_MASK                                                                    0x00000018L
-#define SH_MEM_CONFIG__RETRY_DISABLE_MASK                                                                     0x00001000L
-#define SH_MEM_CONFIG__PRIVATE_NV_MASK                                                                        0x00002000L
-//CC_GC_SHADER_RATE_CONFIG
-#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT                                                            0x1
-#define CC_GC_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE__SHIFT                                                  0x3
-#define CC_GC_SHADER_RATE_CONFIG__HALF_LDS__SHIFT                                                             0x4
-#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE_MASK                                                              0x00000006L
-#define CC_GC_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE_MASK                                                    0x00000008L
-#define CC_GC_SHADER_RATE_CONFIG__HALF_LDS_MASK                                                               0x00000010L
-//GC_USER_SHADER_RATE_CONFIG
-#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT                                                          0x1
-#define GC_USER_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE__SHIFT                                                0x3
-#define GC_USER_SHADER_RATE_CONFIG__HALF_LDS__SHIFT                                                           0x4
-#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE_MASK                                                            0x00000006L
-#define GC_USER_SHADER_RATE_CONFIG__SQC_BALANCE_DISABLE_MASK                                                  0x00000008L
-#define GC_USER_SHADER_RATE_CONFIG__HALF_LDS_MASK                                                             0x00000010L
-//SQ_INTERRUPT_AUTO_MASK
-#define SQ_INTERRUPT_AUTO_MASK__MASK__SHIFT                                                                   0x0
-#define SQ_INTERRUPT_AUTO_MASK__MASK_MASK                                                                     0x00FFFFFFL
-//SQ_INTERRUPT_MSG_CTRL
-#define SQ_INTERRUPT_MSG_CTRL__STALL__SHIFT                                                                   0x0
-#define SQ_INTERRUPT_MSG_CTRL__STALL_MASK                                                                     0x00000001L
-//SQ_UTCL1_CNTL1
-#define SQ_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                               0x0
-#define SQ_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                                  0x1
-#define SQ_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                                0x2
-#define SQ_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                      0x3
-#define SQ_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                                0x5
-#define SQ_UTCL1_CNTL1__CLIENTID__SHIFT                                                                       0x7
-#define SQ_UTCL1_CNTL1__USERVM_DIS__SHIFT                                                                     0x10
-#define SQ_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                              0x11
-#define SQ_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                           0x12
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_VMID__SHIFT                                                            0x13
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID__SHIFT                                                        0x17
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE__SHIFT                                                          0x18
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL__SHIFT                                                             0x19
-#define SQ_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                     0x1a
-#define SQ_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                 0x1b
-#define SQ_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                         0x1c
-#define SQ_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                         0x1e
-#define SQ_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                 0x00000001L
-#define SQ_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                                    0x00000002L
-#define SQ_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                  0x00000004L
-#define SQ_UTCL1_CNTL1__RESP_MODE_MASK                                                                        0x00000018L
-#define SQ_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                  0x00000060L
-#define SQ_UTCL1_CNTL1__CLIENTID_MASK                                                                         0x0000FF80L
-#define SQ_UTCL1_CNTL1__USERVM_DIS_MASK                                                                       0x00010000L
-#define SQ_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                                0x00020000L
-#define SQ_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                             0x00040000L
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_VMID_MASK                                                              0x00780000L
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID_MASK                                                          0x00800000L
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE_MASK                                                            0x01000000L
-#define SQ_UTCL1_CNTL1__REG_INVALIDATE_ALL_MASK                                                               0x02000000L
-#define SQ_UTCL1_CNTL1__FORCE_MISS_MASK                                                                       0x04000000L
-#define SQ_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                   0x08000000L
-#define SQ_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                           0x30000000L
-#define SQ_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                           0xC0000000L
-//SQ_UTCL1_CNTL2
-#define SQ_UTCL1_CNTL2__SPARE__SHIFT                                                                          0x0
-#define SQ_UTCL1_CNTL2__LFIFO_SCAN_DISABLE__SHIFT                                                             0x8
-#define SQ_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                 0x9
-#define SQ_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                     0xa
-#define SQ_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                        0xb
-#define SQ_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                 0xc
-#define SQ_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                                  0xd
-#define SQ_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                    0xe
-#define SQ_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                            0xf
-#define SQ_UTCL1_CNTL2__RETRY_TIMER__SHIFT                                                                    0x10
-#define SQ_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                           0x1a
-#define SQ_UTCL1_CNTL2__PREFETCH_PAGE__SHIFT                                                                  0x1c
-#define SQ_UTCL1_CNTL2__SPARE_MASK                                                                            0x000000FFL
-#define SQ_UTCL1_CNTL2__LFIFO_SCAN_DISABLE_MASK                                                               0x00000100L
-#define SQ_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                   0x00000200L
-#define SQ_UTCL1_CNTL2__LINE_VALID_MASK                                                                       0x00000400L
-#define SQ_UTCL1_CNTL2__DIS_EDC_MASK                                                                          0x00000800L
-#define SQ_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                   0x00001000L
-#define SQ_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                                    0x00002000L
-#define SQ_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                      0x00004000L
-#define SQ_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                              0x00008000L
-#define SQ_UTCL1_CNTL2__RETRY_TIMER_MASK                                                                      0x007F0000L
-#define SQ_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                             0x04000000L
-#define SQ_UTCL1_CNTL2__PREFETCH_PAGE_MASK                                                                    0xF0000000L
-//SQ_UTCL1_STATUS
-#define SQ_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
-#define SQ_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
-#define SQ_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
-#define SQ_UTCL1_STATUS__RESERVED__SHIFT                                                                      0x3
-#define SQ_UTCL1_STATUS__UNUSED__SHIFT                                                                        0x10
-#define SQ_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
-#define SQ_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
-#define SQ_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
-#define SQ_UTCL1_STATUS__RESERVED_MASK                                                                        0x0000FFF8L
-#define SQ_UTCL1_STATUS__UNUSED_MASK                                                                          0xFFFF0000L
-//SQ_SHADER_TBA_LO
-#define SQ_SHADER_TBA_LO__ADDR_LO__SHIFT                                                                      0x0
-#define SQ_SHADER_TBA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
-//SQ_SHADER_TBA_HI
-#define SQ_SHADER_TBA_HI__ADDR_HI__SHIFT                                                                      0x0
-#define SQ_SHADER_TBA_HI__ADDR_HI_MASK                                                                        0x000000FFL
-//SQ_SHADER_TMA_LO
-#define SQ_SHADER_TMA_LO__ADDR_LO__SHIFT                                                                      0x0
-#define SQ_SHADER_TMA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
-//SQ_SHADER_TMA_HI
-#define SQ_SHADER_TMA_HI__ADDR_HI__SHIFT                                                                      0x0
-#define SQ_SHADER_TMA_HI__ADDR_HI_MASK                                                                        0x000000FFL
-//SQC_DSM_CNTL
-#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                              0x0
-#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                             0x2
-#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_DSM_IRRITATOR_DATA__SHIFT                                       0x3
-#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE__SHIFT                                      0x5
-#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                          0x6
-#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                         0x8
-#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_DSM_IRRITATOR_DATA__SHIFT                                       0x9
-#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE__SHIFT                                      0xb
-#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                          0xc
-#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                         0xe
-#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_DSM_IRRITATOR_DATA__SHIFT                                       0xf
-#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE__SHIFT                                      0x11
-#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_DSM_IRRITATOR_DATA__SHIFT                                          0x12
-#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_ENABLE_SINGLE_WRITE__SHIFT                                         0x14
-#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                                0x00000003L
-#define SQC_DSM_CNTL__INST_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                               0x00000004L
-#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_DSM_IRRITATOR_DATA_MASK                                         0x00000018L
-#define SQC_DSM_CNTL__DATA_CU0_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE_MASK                                        0x00000020L
-#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                            0x000000C0L
-#define SQC_DSM_CNTL__DATA_CU0_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                           0x00000100L
-#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_DSM_IRRITATOR_DATA_MASK                                         0x00000600L
-#define SQC_DSM_CNTL__DATA_CU1_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE_MASK                                        0x00000800L
-#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                            0x00003000L
-#define SQC_DSM_CNTL__DATA_CU1_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                           0x00004000L
-#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_DSM_IRRITATOR_DATA_MASK                                         0x00018000L
-#define SQC_DSM_CNTL__DATA_CU2_WRITE_DATA_BUF_ENABLE_SINGLE_WRITE_MASK                                        0x00020000L
-#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_DSM_IRRITATOR_DATA_MASK                                            0x000C0000L
-#define SQC_DSM_CNTL__DATA_CU2_UTCL1_LFIFO_ENABLE_SINGLE_WRITE_MASK                                           0x00100000L
-//SQC_DSM_CNTLA
-#define SQC_DSM_CNTLA__INST_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0x0
-#define SQC_DSM_CNTLA__INST_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0x2
-#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                         0x3
-#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                        0x5
-#define SQC_DSM_CNTLA__INST_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x6
-#define SQC_DSM_CNTLA__INST_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
-#define SQC_DSM_CNTLA__INST_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
-#define SQC_DSM_CNTLA__INST_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
-#define SQC_DSM_CNTLA__DATA_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
-#define SQC_DSM_CNTLA__DATA_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
-#define SQC_DSM_CNTLA__DATA_HIT_FIFO_DSM_IRRITATOR_DATA__SHIFT                                                0xf
-#define SQC_DSM_CNTLA__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
-#define SQC_DSM_CNTLA__DATA_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x12
-#define SQC_DSM_CNTLA__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x14
-#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA__SHIFT                                           0x15
-#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE__SHIFT                                          0x17
-#define SQC_DSM_CNTLA__DATA_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x18
-#define SQC_DSM_CNTLA__DATA_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0x1a
-#define SQC_DSM_CNTLA__INST_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00000003L
-#define SQC_DSM_CNTLA__INST_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000004L
-#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                           0x00000018L
-#define SQC_DSM_CNTLA__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                          0x00000020L
-#define SQC_DSM_CNTLA__INST_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000000C0L
-#define SQC_DSM_CNTLA__INST_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
-#define SQC_DSM_CNTLA__INST_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
-#define SQC_DSM_CNTLA__INST_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
-#define SQC_DSM_CNTLA__DATA_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
-#define SQC_DSM_CNTLA__DATA_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
-#define SQC_DSM_CNTLA__DATA_HIT_FIFO_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
-#define SQC_DSM_CNTLA__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
-#define SQC_DSM_CNTLA__DATA_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000C0000L
-#define SQC_DSM_CNTLA__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00100000L
-#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA_MASK                                             0x00600000L
-#define SQC_DSM_CNTLA__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE_MASK                                            0x00800000L
-#define SQC_DSM_CNTLA__DATA_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x03000000L
-#define SQC_DSM_CNTLA__DATA_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x04000000L
-//SQC_DSM_CNTLB
-#define SQC_DSM_CNTLB__INST_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0x0
-#define SQC_DSM_CNTLB__INST_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0x2
-#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                         0x3
-#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                        0x5
-#define SQC_DSM_CNTLB__INST_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x6
-#define SQC_DSM_CNTLB__INST_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
-#define SQC_DSM_CNTLB__INST_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x9
-#define SQC_DSM_CNTLB__INST_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0xb
-#define SQC_DSM_CNTLB__DATA_TAG_RAM_DSM_IRRITATOR_DATA__SHIFT                                                 0xc
-#define SQC_DSM_CNTLB__DATA_TAG_RAM_ENABLE_SINGLE_WRITE__SHIFT                                                0xe
-#define SQC_DSM_CNTLB__DATA_HIT_FIFO_DSM_IRRITATOR_DATA__SHIFT                                                0xf
-#define SQC_DSM_CNTLB__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                               0x11
-#define SQC_DSM_CNTLB__DATA_MISS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                               0x12
-#define SQC_DSM_CNTLB__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                              0x14
-#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA__SHIFT                                           0x15
-#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE__SHIFT                                          0x17
-#define SQC_DSM_CNTLB__DATA_BANK_RAM_DSM_IRRITATOR_DATA__SHIFT                                                0x18
-#define SQC_DSM_CNTLB__DATA_BANK_RAM_ENABLE_SINGLE_WRITE__SHIFT                                               0x1a
-#define SQC_DSM_CNTLB__INST_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00000003L
-#define SQC_DSM_CNTLB__INST_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000004L
-#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                           0x00000018L
-#define SQC_DSM_CNTLB__INST_UTCL1_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                          0x00000020L
-#define SQC_DSM_CNTLB__INST_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000000C0L
-#define SQC_DSM_CNTLB__INST_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
-#define SQC_DSM_CNTLB__INST_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x00000600L
-#define SQC_DSM_CNTLB__INST_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000800L
-#define SQC_DSM_CNTLB__DATA_TAG_RAM_DSM_IRRITATOR_DATA_MASK                                                   0x00003000L
-#define SQC_DSM_CNTLB__DATA_TAG_RAM_ENABLE_SINGLE_WRITE_MASK                                                  0x00004000L
-#define SQC_DSM_CNTLB__DATA_HIT_FIFO_DSM_IRRITATOR_DATA_MASK                                                  0x00018000L
-#define SQC_DSM_CNTLB__DATA_HIT_FIFO_ENABLE_SINGLE_WRITE_MASK                                                 0x00020000L
-#define SQC_DSM_CNTLB__DATA_MISS_FIFO_DSM_IRRITATOR_DATA_MASK                                                 0x000C0000L
-#define SQC_DSM_CNTLB__DATA_MISS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                0x00100000L
-#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_DSM_IRRITATOR_DATA_MASK                                             0x00600000L
-#define SQC_DSM_CNTLB__DATA_DIRTY_BIT_RAM_ENABLE_SINGLE_WRITE_MASK                                            0x00800000L
-#define SQC_DSM_CNTLB__DATA_BANK_RAM_DSM_IRRITATOR_DATA_MASK                                                  0x03000000L
-#define SQC_DSM_CNTLB__DATA_BANK_RAM_ENABLE_SINGLE_WRITE_MASK                                                 0x04000000L
-//SQC_DSM_CNTL2
-#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                            0x0
-#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                            0x2
-#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_ENABLE_ERROR_INJECT__SHIFT                                     0x3
-#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_SELECT_INJECT_DELAY__SHIFT                                     0x5
-#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                        0x6
-#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                        0x8
-#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_ENABLE_ERROR_INJECT__SHIFT                                     0x9
-#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_SELECT_INJECT_DELAY__SHIFT                                     0xb
-#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                        0xc
-#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                        0xe
-#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_ENABLE_ERROR_INJECT__SHIFT                                     0xf
-#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_SELECT_INJECT_DELAY__SHIFT                                     0x11
-#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_ENABLE_ERROR_INJECT__SHIFT                                        0x12
-#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_SELECT_INJECT_DELAY__SHIFT                                        0x14
-#define SQC_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
-#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                              0x00000003L
-#define SQC_DSM_CNTL2__INST_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                              0x00000004L
-#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_ENABLE_ERROR_INJECT_MASK                                       0x00000018L
-#define SQC_DSM_CNTL2__DATA_CU0_WRITE_DATA_BUF_SELECT_INJECT_DELAY_MASK                                       0x00000020L
-#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                          0x000000C0L
-#define SQC_DSM_CNTL2__DATA_CU0_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                          0x00000100L
-#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_ENABLE_ERROR_INJECT_MASK                                       0x00000600L
-#define SQC_DSM_CNTL2__DATA_CU1_WRITE_DATA_BUF_SELECT_INJECT_DELAY_MASK                                       0x00000800L
-#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                          0x00003000L
-#define SQC_DSM_CNTL2__DATA_CU1_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                          0x00004000L
-#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_ENABLE_ERROR_INJECT_MASK                                       0x00018000L
-#define SQC_DSM_CNTL2__DATA_CU2_WRITE_DATA_BUF_SELECT_INJECT_DELAY_MASK                                       0x00020000L
-#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_ENABLE_ERROR_INJECT_MASK                                          0x000C0000L
-#define SQC_DSM_CNTL2__DATA_CU2_UTCL1_LFIFO_SELECT_INJECT_DELAY_MASK                                          0x00100000L
-#define SQC_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
-//SQC_DSM_CNTL2A
-#define SQC_DSM_CNTL2A__INST_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0x0
-#define SQC_DSM_CNTL2A__INST_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0x2
-#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                       0x3
-#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                       0x5
-#define SQC_DSM_CNTL2A__INST_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x6
-#define SQC_DSM_CNTL2A__INST_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x8
-#define SQC_DSM_CNTL2A__INST_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
-#define SQC_DSM_CNTL2A__INST_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0xb
-#define SQC_DSM_CNTL2A__DATA_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
-#define SQC_DSM_CNTL2A__DATA_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0xe
-#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_ENABLE_ERROR_INJECT__SHIFT                                              0xf
-#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_SELECT_INJECT_DELAY__SHIFT                                              0x11
-#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x12
-#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x14
-#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT__SHIFT                                         0x15
-#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY__SHIFT                                         0x17
-#define SQC_DSM_CNTL2A__DATA_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x18
-#define SQC_DSM_CNTL2A__DATA_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0x1a
-#define SQC_DSM_CNTL2A__INST_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
-#define SQC_DSM_CNTL2A__INST_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
-#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                         0x00000018L
-#define SQC_DSM_CNTL2A__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                         0x00000020L
-#define SQC_DSM_CNTL2A__INST_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
-#define SQC_DSM_CNTL2A__INST_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00000100L
-#define SQC_DSM_CNTL2A__INST_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
-#define SQC_DSM_CNTL2A__INST_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
-#define SQC_DSM_CNTL2A__DATA_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
-#define SQC_DSM_CNTL2A__DATA_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
-#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
-#define SQC_DSM_CNTL2A__DATA_HIT_FIFO_SELECT_INJECT_DELAY_MASK                                                0x00020000L
-#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000C0000L
-#define SQC_DSM_CNTL2A__DATA_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00100000L
-#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT_MASK                                           0x00600000L
-#define SQC_DSM_CNTL2A__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY_MASK                                           0x00800000L
-#define SQC_DSM_CNTL2A__DATA_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x03000000L
-#define SQC_DSM_CNTL2A__DATA_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x04000000L
-//SQC_DSM_CNTL2B
-#define SQC_DSM_CNTL2B__INST_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0x0
-#define SQC_DSM_CNTL2B__INST_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0x2
-#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                       0x3
-#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                       0x5
-#define SQC_DSM_CNTL2B__INST_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x6
-#define SQC_DSM_CNTL2B__INST_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x8
-#define SQC_DSM_CNTL2B__INST_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x9
-#define SQC_DSM_CNTL2B__INST_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0xb
-#define SQC_DSM_CNTL2B__DATA_TAG_RAM_ENABLE_ERROR_INJECT__SHIFT                                               0xc
-#define SQC_DSM_CNTL2B__DATA_TAG_RAM_SELECT_INJECT_DELAY__SHIFT                                               0xe
-#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_ENABLE_ERROR_INJECT__SHIFT                                              0xf
-#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_SELECT_INJECT_DELAY__SHIFT                                              0x11
-#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0x12
-#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0x14
-#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT__SHIFT                                         0x15
-#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY__SHIFT                                         0x17
-#define SQC_DSM_CNTL2B__DATA_BANK_RAM_ENABLE_ERROR_INJECT__SHIFT                                              0x18
-#define SQC_DSM_CNTL2B__DATA_BANK_RAM_SELECT_INJECT_DELAY__SHIFT                                              0x1a
-#define SQC_DSM_CNTL2B__INST_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
-#define SQC_DSM_CNTL2B__INST_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
-#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                         0x00000018L
-#define SQC_DSM_CNTL2B__INST_UTCL1_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                         0x00000020L
-#define SQC_DSM_CNTL2B__INST_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
-#define SQC_DSM_CNTL2B__INST_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00000100L
-#define SQC_DSM_CNTL2B__INST_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x00000600L
-#define SQC_DSM_CNTL2B__INST_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x00000800L
-#define SQC_DSM_CNTL2B__DATA_TAG_RAM_ENABLE_ERROR_INJECT_MASK                                                 0x00003000L
-#define SQC_DSM_CNTL2B__DATA_TAG_RAM_SELECT_INJECT_DELAY_MASK                                                 0x00004000L
-#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_ENABLE_ERROR_INJECT_MASK                                                0x00018000L
-#define SQC_DSM_CNTL2B__DATA_HIT_FIFO_SELECT_INJECT_DELAY_MASK                                                0x00020000L
-#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x000C0000L
-#define SQC_DSM_CNTL2B__DATA_MISS_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00100000L
-#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_ENABLE_ERROR_INJECT_MASK                                           0x00600000L
-#define SQC_DSM_CNTL2B__DATA_DIRTY_BIT_RAM_SELECT_INJECT_DELAY_MASK                                           0x00800000L
-#define SQC_DSM_CNTL2B__DATA_BANK_RAM_ENABLE_ERROR_INJECT_MASK                                                0x03000000L
-#define SQC_DSM_CNTL2B__DATA_BANK_RAM_SELECT_INJECT_DELAY_MASK                                                0x04000000L
-//SQC_EDC_FUE_CNTL
-#define SQC_EDC_FUE_CNTL__BLOCK_FUE_FLAGS__SHIFT                                                              0x0
-#define SQC_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES__SHIFT                                                        0x10
-#define SQC_EDC_FUE_CNTL__BLOCK_FUE_FLAGS_MASK                                                                0x0000FFFFL
-#define SQC_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES_MASK                                                          0xFFFF0000L
-//SQC_EDC_CNT2
-#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_SEC_COUNT__SHIFT                                                     0x0
-#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_DED_COUNT__SHIFT                                                     0x2
-#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_SEC_COUNT__SHIFT                                                    0x4
-#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_DED_COUNT__SHIFT                                                    0x6
-#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_SEC_COUNT__SHIFT                                                     0x8
-#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_DED_COUNT__SHIFT                                                     0xa
-#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_SEC_COUNT__SHIFT                                                    0xc
-#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_DED_COUNT__SHIFT                                                    0xe
-#define SQC_EDC_CNT2__INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT__SHIFT                                             0x10
-#define SQC_EDC_CNT2__INST_BANKA_MISS_FIFO_SED_COUNT__SHIFT                                                   0x12
-#define SQC_EDC_CNT2__DATA_BANKA_HIT_FIFO_SED_COUNT__SHIFT                                                    0x14
-#define SQC_EDC_CNT2__DATA_BANKA_MISS_FIFO_SED_COUNT__SHIFT                                                   0x16
-#define SQC_EDC_CNT2__DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT__SHIFT                                               0x18
-#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                       0x1a
-#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_DED_COUNT__SHIFT                                                       0x1c
-#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_SEC_COUNT_MASK                                                       0x00000003L
-#define SQC_EDC_CNT2__INST_BANKA_TAG_RAM_DED_COUNT_MASK                                                       0x0000000CL
-#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_SEC_COUNT_MASK                                                      0x00000030L
-#define SQC_EDC_CNT2__INST_BANKA_BANK_RAM_DED_COUNT_MASK                                                      0x000000C0L
-#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_SEC_COUNT_MASK                                                       0x00000300L
-#define SQC_EDC_CNT2__DATA_BANKA_TAG_RAM_DED_COUNT_MASK                                                       0x00000C00L
-#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_SEC_COUNT_MASK                                                      0x00003000L
-#define SQC_EDC_CNT2__DATA_BANKA_BANK_RAM_DED_COUNT_MASK                                                      0x0000C000L
-#define SQC_EDC_CNT2__INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT_MASK                                               0x00030000L
-#define SQC_EDC_CNT2__INST_BANKA_MISS_FIFO_SED_COUNT_MASK                                                     0x000C0000L
-#define SQC_EDC_CNT2__DATA_BANKA_HIT_FIFO_SED_COUNT_MASK                                                      0x00300000L
-#define SQC_EDC_CNT2__DATA_BANKA_MISS_FIFO_SED_COUNT_MASK                                                     0x00C00000L
-#define SQC_EDC_CNT2__DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT_MASK                                                 0x03000000L
-#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_SEC_COUNT_MASK                                                         0x0C000000L
-#define SQC_EDC_CNT2__INST_UTCL1_LFIFO_DED_COUNT_MASK                                                         0x30000000L
-//SQC_EDC_CNT3
-#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_SEC_COUNT__SHIFT                                                     0x0
-#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_DED_COUNT__SHIFT                                                     0x2
-#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_SEC_COUNT__SHIFT                                                    0x4
-#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_DED_COUNT__SHIFT                                                    0x6
-#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_SEC_COUNT__SHIFT                                                     0x8
-#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_DED_COUNT__SHIFT                                                     0xa
-#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_SEC_COUNT__SHIFT                                                    0xc
-#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_DED_COUNT__SHIFT                                                    0xe
-#define SQC_EDC_CNT3__INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT__SHIFT                                             0x10
-#define SQC_EDC_CNT3__INST_BANKB_MISS_FIFO_SED_COUNT__SHIFT                                                   0x12
-#define SQC_EDC_CNT3__DATA_BANKB_HIT_FIFO_SED_COUNT__SHIFT                                                    0x14
-#define SQC_EDC_CNT3__DATA_BANKB_MISS_FIFO_SED_COUNT__SHIFT                                                   0x16
-#define SQC_EDC_CNT3__DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT__SHIFT                                               0x18
-#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_SEC_COUNT_MASK                                                       0x00000003L
-#define SQC_EDC_CNT3__INST_BANKB_TAG_RAM_DED_COUNT_MASK                                                       0x0000000CL
-#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_SEC_COUNT_MASK                                                      0x00000030L
-#define SQC_EDC_CNT3__INST_BANKB_BANK_RAM_DED_COUNT_MASK                                                      0x000000C0L
-#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_SEC_COUNT_MASK                                                       0x00000300L
-#define SQC_EDC_CNT3__DATA_BANKB_TAG_RAM_DED_COUNT_MASK                                                       0x00000C00L
-#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_SEC_COUNT_MASK                                                      0x00003000L
-#define SQC_EDC_CNT3__DATA_BANKB_BANK_RAM_DED_COUNT_MASK                                                      0x0000C000L
-#define SQC_EDC_CNT3__INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT_MASK                                               0x00030000L
-#define SQC_EDC_CNT3__INST_BANKB_MISS_FIFO_SED_COUNT_MASK                                                     0x000C0000L
-#define SQC_EDC_CNT3__DATA_BANKB_HIT_FIFO_SED_COUNT_MASK                                                      0x00300000L
-#define SQC_EDC_CNT3__DATA_BANKB_MISS_FIFO_SED_COUNT_MASK                                                     0x00C00000L
-#define SQC_EDC_CNT3__DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT_MASK                                                 0x03000000L
-//SQ_REG_TIMESTAMP
-#define SQ_REG_TIMESTAMP__TIMESTAMP__SHIFT                                                                    0x0
-#define SQ_REG_TIMESTAMP__TIMESTAMP_MASK                                                                      0x000000FFL
-//SQ_CMD_TIMESTAMP
-#define SQ_CMD_TIMESTAMP__TIMESTAMP__SHIFT                                                                    0x0
-#define SQ_CMD_TIMESTAMP__TIMESTAMP_MASK                                                                      0x000000FFL
-//SQ_IND_INDEX
-#define SQ_IND_INDEX__WAVE_ID__SHIFT                                                                          0x0
-#define SQ_IND_INDEX__SIMD_ID__SHIFT                                                                          0x4
-#define SQ_IND_INDEX__THREAD_ID__SHIFT                                                                        0x6
-#define SQ_IND_INDEX__AUTO_INCR__SHIFT                                                                        0xc
-#define SQ_IND_INDEX__FORCE_READ__SHIFT                                                                       0xd
-#define SQ_IND_INDEX__READ_TIMEOUT__SHIFT                                                                     0xe
-#define SQ_IND_INDEX__UNINDEXED__SHIFT                                                                        0xf
-#define SQ_IND_INDEX__INDEX__SHIFT                                                                            0x10
-#define SQ_IND_INDEX__WAVE_ID_MASK                                                                            0x0000000FL
-#define SQ_IND_INDEX__SIMD_ID_MASK                                                                            0x00000030L
-#define SQ_IND_INDEX__THREAD_ID_MASK                                                                          0x00000FC0L
-#define SQ_IND_INDEX__AUTO_INCR_MASK                                                                          0x00001000L
-#define SQ_IND_INDEX__FORCE_READ_MASK                                                                         0x00002000L
-#define SQ_IND_INDEX__READ_TIMEOUT_MASK                                                                       0x00004000L
-#define SQ_IND_INDEX__UNINDEXED_MASK                                                                          0x00008000L
-#define SQ_IND_INDEX__INDEX_MASK                                                                              0xFFFF0000L
-//SQ_IND_DATA
-#define SQ_IND_DATA__DATA__SHIFT                                                                              0x0
-#define SQ_IND_DATA__DATA_MASK                                                                                0xFFFFFFFFL
-//SQ_CMD
-#define SQ_CMD__CMD__SHIFT                                                                                    0x0
-#define SQ_CMD__MODE__SHIFT                                                                                   0x4
-#define SQ_CMD__CHECK_VMID__SHIFT                                                                             0x7
-#define SQ_CMD__DATA__SHIFT                                                                                   0x8
-#define SQ_CMD__WAVE_ID__SHIFT                                                                                0x10
-#define SQ_CMD__SIMD_ID__SHIFT                                                                                0x14
-#define SQ_CMD__QUEUE_ID__SHIFT                                                                               0x18
-#define SQ_CMD__VM_ID__SHIFT                                                                                  0x1c
-#define SQ_CMD__CMD_MASK                                                                                      0x00000007L
-#define SQ_CMD__MODE_MASK                                                                                     0x00000070L
-#define SQ_CMD__CHECK_VMID_MASK                                                                               0x00000080L
-#define SQ_CMD__DATA_MASK                                                                                     0x00000F00L
-#define SQ_CMD__WAVE_ID_MASK                                                                                  0x000F0000L
-#define SQ_CMD__SIMD_ID_MASK                                                                                  0x00300000L
-#define SQ_CMD__QUEUE_ID_MASK                                                                                 0x07000000L
-#define SQ_CMD__VM_ID_MASK                                                                                    0xF0000000L
-//SQ_TIME_HI
-#define SQ_TIME_HI__TIME__SHIFT                                                                               0x0
-#define SQ_TIME_HI__TIME_MASK                                                                                 0xFFFFFFFFL
-//SQ_TIME_LO
-#define SQ_TIME_LO__TIME__SHIFT                                                                               0x0
-#define SQ_TIME_LO__TIME_MASK                                                                                 0xFFFFFFFFL
-//SQ_DS_0
-#define SQ_DS_0__OFFSET0__SHIFT                                                                               0x0
-#define SQ_DS_0__OFFSET1__SHIFT                                                                               0x8
-#define SQ_DS_0__GDS__SHIFT                                                                                   0x10
-#define SQ_DS_0__OP__SHIFT                                                                                    0x11
-#define SQ_DS_0__ENCODING__SHIFT                                                                              0x1a
-#define SQ_DS_0__OFFSET0_MASK                                                                                 0x000000FFL
-#define SQ_DS_0__OFFSET1_MASK                                                                                 0x0000FF00L
-#define SQ_DS_0__GDS_MASK                                                                                     0x00010000L
-#define SQ_DS_0__OP_MASK                                                                                      0x01FE0000L
-#define SQ_DS_0__ENCODING_MASK                                                                                0xFC000000L
-//SQ_DS_1
-#define SQ_DS_1__ADDR__SHIFT                                                                                  0x0
-#define SQ_DS_1__DATA0__SHIFT                                                                                 0x8
-#define SQ_DS_1__DATA1__SHIFT                                                                                 0x10
-#define SQ_DS_1__VDST__SHIFT                                                                                  0x18
-#define SQ_DS_1__ADDR_MASK                                                                                    0x000000FFL
-#define SQ_DS_1__DATA0_MASK                                                                                   0x0000FF00L
-#define SQ_DS_1__DATA1_MASK                                                                                   0x00FF0000L
-#define SQ_DS_1__VDST_MASK                                                                                    0xFF000000L
-//SQ_EXP_0
-#define SQ_EXP_0__EN__SHIFT                                                                                   0x0
-#define SQ_EXP_0__TGT__SHIFT                                                                                  0x4
-#define SQ_EXP_0__COMPR__SHIFT                                                                                0xa
-#define SQ_EXP_0__DONE__SHIFT                                                                                 0xb
-#define SQ_EXP_0__VM__SHIFT                                                                                   0xc
-#define SQ_EXP_0__ENCODING__SHIFT                                                                             0x1a
-#define SQ_EXP_0__EN_MASK                                                                                     0x0000000FL
-#define SQ_EXP_0__TGT_MASK                                                                                    0x000003F0L
-#define SQ_EXP_0__COMPR_MASK                                                                                  0x00000400L
-#define SQ_EXP_0__DONE_MASK                                                                                   0x00000800L
-#define SQ_EXP_0__VM_MASK                                                                                     0x00001000L
-#define SQ_EXP_0__ENCODING_MASK                                                                               0xFC000000L
-//SQ_EXP_1
-#define SQ_EXP_1__VSRC0__SHIFT                                                                                0x0
-#define SQ_EXP_1__VSRC1__SHIFT                                                                                0x8
-#define SQ_EXP_1__VSRC2__SHIFT                                                                                0x10
-#define SQ_EXP_1__VSRC3__SHIFT                                                                                0x18
-#define SQ_EXP_1__VSRC0_MASK                                                                                  0x000000FFL
-#define SQ_EXP_1__VSRC1_MASK                                                                                  0x0000FF00L
-#define SQ_EXP_1__VSRC2_MASK                                                                                  0x00FF0000L
-#define SQ_EXP_1__VSRC3_MASK                                                                                  0xFF000000L
-//SQ_FLAT_0
-#define SQ_FLAT_0__OFFSET__SHIFT                                                                              0x0
-#define SQ_FLAT_0__LDS__SHIFT                                                                                 0xd
-#define SQ_FLAT_0__SEG__SHIFT                                                                                 0xe
-#define SQ_FLAT_0__GLC__SHIFT                                                                                 0x10
-#define SQ_FLAT_0__SLC__SHIFT                                                                                 0x11
-#define SQ_FLAT_0__OP__SHIFT                                                                                  0x12
-#define SQ_FLAT_0__ENCODING__SHIFT                                                                            0x1a
-#define SQ_FLAT_0__OFFSET_MASK                                                                                0x00000FFFL
-#define SQ_FLAT_0__LDS_MASK                                                                                   0x00002000L
-#define SQ_FLAT_0__SEG_MASK                                                                                   0x0000C000L
-#define SQ_FLAT_0__GLC_MASK                                                                                   0x00010000L
-#define SQ_FLAT_0__SLC_MASK                                                                                   0x00020000L
-#define SQ_FLAT_0__OP_MASK                                                                                    0x01FC0000L
-#define SQ_FLAT_0__ENCODING_MASK                                                                              0xFC000000L
-//SQ_FLAT_1
-#define SQ_FLAT_1__ADDR__SHIFT                                                                                0x0
-#define SQ_FLAT_1__DATA__SHIFT                                                                                0x8
-#define SQ_FLAT_1__SADDR__SHIFT                                                                               0x10
-#define SQ_FLAT_1__NV__SHIFT                                                                                  0x17
-#define SQ_FLAT_1__VDST__SHIFT                                                                                0x18
-#define SQ_FLAT_1__ADDR_MASK                                                                                  0x000000FFL
-#define SQ_FLAT_1__DATA_MASK                                                                                  0x0000FF00L
-#define SQ_FLAT_1__SADDR_MASK                                                                                 0x007F0000L
-#define SQ_FLAT_1__NV_MASK                                                                                    0x00800000L
-#define SQ_FLAT_1__VDST_MASK                                                                                  0xFF000000L
-//SQ_GLBL_0
-#define SQ_GLBL_0__OFFSET__SHIFT                                                                              0x0
-#define SQ_GLBL_0__LDS__SHIFT                                                                                 0xd
-#define SQ_GLBL_0__SEG__SHIFT                                                                                 0xe
-#define SQ_GLBL_0__GLC__SHIFT                                                                                 0x10
-#define SQ_GLBL_0__SLC__SHIFT                                                                                 0x11
-#define SQ_GLBL_0__OP__SHIFT                                                                                  0x12
-#define SQ_GLBL_0__ENCODING__SHIFT                                                                            0x1a
-#define SQ_GLBL_0__OFFSET_MASK                                                                                0x00001FFFL
-#define SQ_GLBL_0__LDS_MASK                                                                                   0x00002000L
-#define SQ_GLBL_0__SEG_MASK                                                                                   0x0000C000L
-#define SQ_GLBL_0__GLC_MASK                                                                                   0x00010000L
-#define SQ_GLBL_0__SLC_MASK                                                                                   0x00020000L
-#define SQ_GLBL_0__OP_MASK                                                                                    0x01FC0000L
-#define SQ_GLBL_0__ENCODING_MASK                                                                              0xFC000000L
-//SQ_GLBL_1
-#define SQ_GLBL_1__ADDR__SHIFT                                                                                0x0
-#define SQ_GLBL_1__DATA__SHIFT                                                                                0x8
-#define SQ_GLBL_1__SADDR__SHIFT                                                                               0x10
-#define SQ_GLBL_1__NV__SHIFT                                                                                  0x17
-#define SQ_GLBL_1__VDST__SHIFT                                                                                0x18
-#define SQ_GLBL_1__ADDR_MASK                                                                                  0x000000FFL
-#define SQ_GLBL_1__DATA_MASK                                                                                  0x0000FF00L
-#define SQ_GLBL_1__SADDR_MASK                                                                                 0x007F0000L
-#define SQ_GLBL_1__NV_MASK                                                                                    0x00800000L
-#define SQ_GLBL_1__VDST_MASK                                                                                  0xFF000000L
-//SQ_INST
-#define SQ_INST__ENCODING__SHIFT                                                                              0x0
-#define SQ_INST__ENCODING_MASK                                                                                0xFFFFFFFFL
-//SQ_MIMG_0
-#define SQ_MIMG_0__OPM__SHIFT                                                                                 0x0
-#define SQ_MIMG_0__DMASK__SHIFT                                                                               0x8
-#define SQ_MIMG_0__UNORM__SHIFT                                                                               0xc
-#define SQ_MIMG_0__GLC__SHIFT                                                                                 0xd
-#define SQ_MIMG_0__DA__SHIFT                                                                                  0xe
-#define SQ_MIMG_0__A16__SHIFT                                                                                 0xf
-#define SQ_MIMG_0__TFE__SHIFT                                                                                 0x10
-#define SQ_MIMG_0__LWE__SHIFT                                                                                 0x11
-#define SQ_MIMG_0__OP__SHIFT                                                                                  0x12
-#define SQ_MIMG_0__SLC__SHIFT                                                                                 0x19
-#define SQ_MIMG_0__ENCODING__SHIFT                                                                            0x1a
-#define SQ_MIMG_0__OPM_MASK                                                                                   0x00000001L
-#define SQ_MIMG_0__DMASK_MASK                                                                                 0x00000F00L
-#define SQ_MIMG_0__UNORM_MASK                                                                                 0x00001000L
-#define SQ_MIMG_0__GLC_MASK                                                                                   0x00002000L
-#define SQ_MIMG_0__DA_MASK                                                                                    0x00004000L
-#define SQ_MIMG_0__A16_MASK                                                                                   0x00008000L
-#define SQ_MIMG_0__TFE_MASK                                                                                   0x00010000L
-#define SQ_MIMG_0__LWE_MASK                                                                                   0x00020000L
-#define SQ_MIMG_0__OP_MASK                                                                                    0x01FC0000L
-#define SQ_MIMG_0__SLC_MASK                                                                                   0x02000000L
-#define SQ_MIMG_0__ENCODING_MASK                                                                              0xFC000000L
-//SQ_MIMG_1
-#define SQ_MIMG_1__VADDR__SHIFT                                                                               0x0
-#define SQ_MIMG_1__VDATA__SHIFT                                                                               0x8
-#define SQ_MIMG_1__SRSRC__SHIFT                                                                               0x10
-#define SQ_MIMG_1__SSAMP__SHIFT                                                                               0x15
-#define SQ_MIMG_1__D16__SHIFT                                                                                 0x1f
-#define SQ_MIMG_1__VADDR_MASK                                                                                 0x000000FFL
-#define SQ_MIMG_1__VDATA_MASK                                                                                 0x0000FF00L
-#define SQ_MIMG_1__SRSRC_MASK                                                                                 0x001F0000L
-#define SQ_MIMG_1__SSAMP_MASK                                                                                 0x03E00000L
-#define SQ_MIMG_1__D16_MASK                                                                                   0x80000000L
-//SQ_MTBUF_0
-#define SQ_MTBUF_0__OFFSET__SHIFT                                                                             0x0
-#define SQ_MTBUF_0__OFFEN__SHIFT                                                                              0xc
-#define SQ_MTBUF_0__IDXEN__SHIFT                                                                              0xd
-#define SQ_MTBUF_0__GLC__SHIFT                                                                                0xe
-#define SQ_MTBUF_0__OP__SHIFT                                                                                 0xf
-#define SQ_MTBUF_0__DFMT__SHIFT                                                                               0x13
-#define SQ_MTBUF_0__NFMT__SHIFT                                                                               0x17
-#define SQ_MTBUF_0__ENCODING__SHIFT                                                                           0x1a
-#define SQ_MTBUF_0__OFFSET_MASK                                                                               0x00000FFFL
-#define SQ_MTBUF_0__OFFEN_MASK                                                                                0x00001000L
-#define SQ_MTBUF_0__IDXEN_MASK                                                                                0x00002000L
-#define SQ_MTBUF_0__GLC_MASK                                                                                  0x00004000L
-#define SQ_MTBUF_0__OP_MASK                                                                                   0x00078000L
-#define SQ_MTBUF_0__DFMT_MASK                                                                                 0x00780000L
-#define SQ_MTBUF_0__NFMT_MASK                                                                                 0x03800000L
-#define SQ_MTBUF_0__ENCODING_MASK                                                                             0xFC000000L
-//SQ_MTBUF_1
-#define SQ_MTBUF_1__VADDR__SHIFT                                                                              0x0
-#define SQ_MTBUF_1__VDATA__SHIFT                                                                              0x8
-#define SQ_MTBUF_1__SRSRC__SHIFT                                                                              0x10
-#define SQ_MTBUF_1__SLC__SHIFT                                                                                0x16
-#define SQ_MTBUF_1__TFE__SHIFT                                                                                0x17
-#define SQ_MTBUF_1__SOFFSET__SHIFT                                                                            0x18
-#define SQ_MTBUF_1__VADDR_MASK                                                                                0x000000FFL
-#define SQ_MTBUF_1__VDATA_MASK                                                                                0x0000FF00L
-#define SQ_MTBUF_1__SRSRC_MASK                                                                                0x001F0000L
-#define SQ_MTBUF_1__SLC_MASK                                                                                  0x00400000L
-#define SQ_MTBUF_1__TFE_MASK                                                                                  0x00800000L
-#define SQ_MTBUF_1__SOFFSET_MASK                                                                              0xFF000000L
-//SQ_MUBUF_0
-#define SQ_MUBUF_0__OFFSET__SHIFT                                                                             0x0
-#define SQ_MUBUF_0__OFFEN__SHIFT                                                                              0xc
-#define SQ_MUBUF_0__IDXEN__SHIFT                                                                              0xd
-#define SQ_MUBUF_0__GLC__SHIFT                                                                                0xe
-#define SQ_MUBUF_0__LDS__SHIFT                                                                                0x10
-#define SQ_MUBUF_0__SLC__SHIFT                                                                                0x11
-#define SQ_MUBUF_0__OP__SHIFT                                                                                 0x12
-#define SQ_MUBUF_0__ENCODING__SHIFT                                                                           0x1a
-#define SQ_MUBUF_0__OFFSET_MASK                                                                               0x00000FFFL
-#define SQ_MUBUF_0__OFFEN_MASK                                                                                0x00001000L
-#define SQ_MUBUF_0__IDXEN_MASK                                                                                0x00002000L
-#define SQ_MUBUF_0__GLC_MASK                                                                                  0x00004000L
-#define SQ_MUBUF_0__LDS_MASK                                                                                  0x00010000L
-#define SQ_MUBUF_0__SLC_MASK                                                                                  0x00020000L
-#define SQ_MUBUF_0__OP_MASK                                                                                   0x01FC0000L
-#define SQ_MUBUF_0__ENCODING_MASK                                                                             0xFC000000L
-//SQ_MUBUF_1
-#define SQ_MUBUF_1__VADDR__SHIFT                                                                              0x0
-#define SQ_MUBUF_1__VDATA__SHIFT                                                                              0x8
-#define SQ_MUBUF_1__SRSRC__SHIFT                                                                              0x10
-#define SQ_MUBUF_1__TFE__SHIFT                                                                                0x17
-#define SQ_MUBUF_1__SOFFSET__SHIFT                                                                            0x18
-#define SQ_MUBUF_1__VADDR_MASK                                                                                0x000000FFL
-#define SQ_MUBUF_1__VDATA_MASK                                                                                0x0000FF00L
-#define SQ_MUBUF_1__SRSRC_MASK                                                                                0x001F0000L
-#define SQ_MUBUF_1__TFE_MASK                                                                                  0x00800000L
-#define SQ_MUBUF_1__SOFFSET_MASK                                                                              0xFF000000L
-//SQ_SCRATCH_0
-#define SQ_SCRATCH_0__OFFSET__SHIFT                                                                           0x0
-#define SQ_SCRATCH_0__LDS__SHIFT                                                                              0xd
-#define SQ_SCRATCH_0__SEG__SHIFT                                                                              0xe
-#define SQ_SCRATCH_0__GLC__SHIFT                                                                              0x10
-#define SQ_SCRATCH_0__SLC__SHIFT                                                                              0x11
-#define SQ_SCRATCH_0__OP__SHIFT                                                                               0x12
-#define SQ_SCRATCH_0__ENCODING__SHIFT                                                                         0x1a
-#define SQ_SCRATCH_0__OFFSET_MASK                                                                             0x00001FFFL
-#define SQ_SCRATCH_0__LDS_MASK                                                                                0x00002000L
-#define SQ_SCRATCH_0__SEG_MASK                                                                                0x0000C000L
-#define SQ_SCRATCH_0__GLC_MASK                                                                                0x00010000L
-#define SQ_SCRATCH_0__SLC_MASK                                                                                0x00020000L
-#define SQ_SCRATCH_0__OP_MASK                                                                                 0x01FC0000L
-#define SQ_SCRATCH_0__ENCODING_MASK                                                                           0xFC000000L
-//SQ_SCRATCH_1
-#define SQ_SCRATCH_1__ADDR__SHIFT                                                                             0x0
-#define SQ_SCRATCH_1__DATA__SHIFT                                                                             0x8
-#define SQ_SCRATCH_1__SADDR__SHIFT                                                                            0x10
-#define SQ_SCRATCH_1__NV__SHIFT                                                                               0x17
-#define SQ_SCRATCH_1__VDST__SHIFT                                                                             0x18
-#define SQ_SCRATCH_1__ADDR_MASK                                                                               0x000000FFL
-#define SQ_SCRATCH_1__DATA_MASK                                                                               0x0000FF00L
-#define SQ_SCRATCH_1__SADDR_MASK                                                                              0x007F0000L
-#define SQ_SCRATCH_1__NV_MASK                                                                                 0x00800000L
-#define SQ_SCRATCH_1__VDST_MASK                                                                               0xFF000000L
-//SQ_SMEM_0
-#define SQ_SMEM_0__SBASE__SHIFT                                                                               0x0
-#define SQ_SMEM_0__SDATA__SHIFT                                                                               0x6
-#define SQ_SMEM_0__SOFFSET_EN__SHIFT                                                                          0xe
-#define SQ_SMEM_0__NV__SHIFT                                                                                  0xf
-#define SQ_SMEM_0__GLC__SHIFT                                                                                 0x10
-#define SQ_SMEM_0__IMM__SHIFT                                                                                 0x11
-#define SQ_SMEM_0__OP__SHIFT                                                                                  0x12
-#define SQ_SMEM_0__ENCODING__SHIFT                                                                            0x1a
-#define SQ_SMEM_0__SBASE_MASK                                                                                 0x0000003FL
-#define SQ_SMEM_0__SDATA_MASK                                                                                 0x00001FC0L
-#define SQ_SMEM_0__SOFFSET_EN_MASK                                                                            0x00004000L
-#define SQ_SMEM_0__NV_MASK                                                                                    0x00008000L
-#define SQ_SMEM_0__GLC_MASK                                                                                   0x00010000L
-#define SQ_SMEM_0__IMM_MASK                                                                                   0x00020000L
-#define SQ_SMEM_0__OP_MASK                                                                                    0x03FC0000L
-#define SQ_SMEM_0__ENCODING_MASK                                                                              0xFC000000L
-//SQ_SMEM_1
-#define SQ_SMEM_1__OFFSET__SHIFT                                                                              0x0
-#define SQ_SMEM_1__SOFFSET__SHIFT                                                                             0x19
-#define SQ_SMEM_1__OFFSET_MASK                                                                                0x001FFFFFL
-#define SQ_SMEM_1__SOFFSET_MASK                                                                               0xFE000000L
-//SQ_SOP1
-#define SQ_SOP1__SSRC0__SHIFT                                                                                 0x0
-#define SQ_SOP1__OP__SHIFT                                                                                    0x8
-#define SQ_SOP1__SDST__SHIFT                                                                                  0x10
-#define SQ_SOP1__ENCODING__SHIFT                                                                              0x17
-#define SQ_SOP1__SSRC0_MASK                                                                                   0x000000FFL
-#define SQ_SOP1__OP_MASK                                                                                      0x0000FF00L
-#define SQ_SOP1__SDST_MASK                                                                                    0x007F0000L
-#define SQ_SOP1__ENCODING_MASK                                                                                0xFF800000L
-//SQ_SOP2
-#define SQ_SOP2__SSRC0__SHIFT                                                                                 0x0
-#define SQ_SOP2__SSRC1__SHIFT                                                                                 0x8
-#define SQ_SOP2__SDST__SHIFT                                                                                  0x10
-#define SQ_SOP2__OP__SHIFT                                                                                    0x17
-#define SQ_SOP2__ENCODING__SHIFT                                                                              0x1e
-#define SQ_SOP2__SSRC0_MASK                                                                                   0x000000FFL
-#define SQ_SOP2__SSRC1_MASK                                                                                   0x0000FF00L
-#define SQ_SOP2__SDST_MASK                                                                                    0x007F0000L
-#define SQ_SOP2__OP_MASK                                                                                      0x3F800000L
-#define SQ_SOP2__ENCODING_MASK                                                                                0xC0000000L
-//SQ_SOPC
-#define SQ_SOPC__SSRC0__SHIFT                                                                                 0x0
-#define SQ_SOPC__SSRC1__SHIFT                                                                                 0x8
-#define SQ_SOPC__OP__SHIFT                                                                                    0x10
-#define SQ_SOPC__ENCODING__SHIFT                                                                              0x17
-#define SQ_SOPC__SSRC0_MASK                                                                                   0x000000FFL
-#define SQ_SOPC__SSRC1_MASK                                                                                   0x0000FF00L
-#define SQ_SOPC__OP_MASK                                                                                      0x007F0000L
-#define SQ_SOPC__ENCODING_MASK                                                                                0xFF800000L
-//SQ_SOPK
-#define SQ_SOPK__SIMM16__SHIFT                                                                                0x0
-#define SQ_SOPK__SDST__SHIFT                                                                                  0x10
-#define SQ_SOPK__OP__SHIFT                                                                                    0x17
-#define SQ_SOPK__ENCODING__SHIFT                                                                              0x1c
-#define SQ_SOPK__SIMM16_MASK                                                                                  0x0000FFFFL
-#define SQ_SOPK__SDST_MASK                                                                                    0x007F0000L
-#define SQ_SOPK__OP_MASK                                                                                      0x0F800000L
-#define SQ_SOPK__ENCODING_MASK                                                                                0xF0000000L
-//SQ_SOPP
-#define SQ_SOPP__SIMM16__SHIFT                                                                                0x0
-#define SQ_SOPP__OP__SHIFT                                                                                    0x10
-#define SQ_SOPP__ENCODING__SHIFT                                                                              0x17
-#define SQ_SOPP__SIMM16_MASK                                                                                  0x0000FFFFL
-#define SQ_SOPP__OP_MASK                                                                                      0x007F0000L
-#define SQ_SOPP__ENCODING_MASK                                                                                0xFF800000L
-//SQ_VINTRP
-#define SQ_VINTRP__VSRC__SHIFT                                                                                0x0
-#define SQ_VINTRP__ATTRCHAN__SHIFT                                                                            0x8
-#define SQ_VINTRP__ATTR__SHIFT                                                                                0xa
-#define SQ_VINTRP__OP__SHIFT                                                                                  0x10
-#define SQ_VINTRP__VDST__SHIFT                                                                                0x12
-#define SQ_VINTRP__ENCODING__SHIFT                                                                            0x1a
-#define SQ_VINTRP__VSRC_MASK                                                                                  0x000000FFL
-#define SQ_VINTRP__ATTRCHAN_MASK                                                                              0x00000300L
-#define SQ_VINTRP__ATTR_MASK                                                                                  0x0000FC00L
-#define SQ_VINTRP__OP_MASK                                                                                    0x00030000L
-#define SQ_VINTRP__VDST_MASK                                                                                  0x03FC0000L
-#define SQ_VINTRP__ENCODING_MASK                                                                              0xFC000000L
-//SQ_VOP1
-#define SQ_VOP1__SRC0__SHIFT                                                                                  0x0
-#define SQ_VOP1__OP__SHIFT                                                                                    0x9
-#define SQ_VOP1__VDST__SHIFT                                                                                  0x11
-#define SQ_VOP1__ENCODING__SHIFT                                                                              0x19
-#define SQ_VOP1__SRC0_MASK                                                                                    0x000001FFL
-#define SQ_VOP1__OP_MASK                                                                                      0x0001FE00L
-#define SQ_VOP1__VDST_MASK                                                                                    0x01FE0000L
-#define SQ_VOP1__ENCODING_MASK                                                                                0xFE000000L
-//SQ_VOP2
-#define SQ_VOP2__SRC0__SHIFT                                                                                  0x0
-#define SQ_VOP2__VSRC1__SHIFT                                                                                 0x9
-#define SQ_VOP2__VDST__SHIFT                                                                                  0x11
-#define SQ_VOP2__OP__SHIFT                                                                                    0x19
-#define SQ_VOP2__ENCODING__SHIFT                                                                              0x1f
-#define SQ_VOP2__SRC0_MASK                                                                                    0x000001FFL
-#define SQ_VOP2__VSRC1_MASK                                                                                   0x0001FE00L
-#define SQ_VOP2__VDST_MASK                                                                                    0x01FE0000L
-#define SQ_VOP2__OP_MASK                                                                                      0x7E000000L
-#define SQ_VOP2__ENCODING_MASK                                                                                0x80000000L
-//SQ_VOP3P_0
-#define SQ_VOP3P_0__VDST__SHIFT                                                                               0x0
-#define SQ_VOP3P_0__NEG_HI__SHIFT                                                                             0x8
-#define SQ_VOP3P_0__OP_SEL__SHIFT                                                                             0xb
-#define SQ_VOP3P_0__OP_SEL_HI_2__SHIFT                                                                        0xe
-#define SQ_VOP3P_0__CLAMP__SHIFT                                                                              0xf
-#define SQ_VOP3P_0__OP__SHIFT                                                                                 0x10
-#define SQ_VOP3P_0__ENCODING__SHIFT                                                                           0x17
-#define SQ_VOP3P_0__VDST_MASK                                                                                 0x000000FFL
-#define SQ_VOP3P_0__NEG_HI_MASK                                                                               0x00000700L
-#define SQ_VOP3P_0__OP_SEL_MASK                                                                               0x00003800L
-#define SQ_VOP3P_0__OP_SEL_HI_2_MASK                                                                          0x00004000L
-#define SQ_VOP3P_0__CLAMP_MASK                                                                                0x00008000L
-#define SQ_VOP3P_0__OP_MASK                                                                                   0x007F0000L
-#define SQ_VOP3P_0__ENCODING_MASK                                                                             0xFF800000L
-//SQ_VOP3P_1
-#define SQ_VOP3P_1__SRC0__SHIFT                                                                               0x0
-#define SQ_VOP3P_1__SRC1__SHIFT                                                                               0x9
-#define SQ_VOP3P_1__SRC2__SHIFT                                                                               0x12
-#define SQ_VOP3P_1__OP_SEL_HI__SHIFT                                                                          0x1b
-#define SQ_VOP3P_1__NEG__SHIFT                                                                                0x1d
-#define SQ_VOP3P_1__SRC0_MASK                                                                                 0x000001FFL
-#define SQ_VOP3P_1__SRC1_MASK                                                                                 0x0003FE00L
-#define SQ_VOP3P_1__SRC2_MASK                                                                                 0x07FC0000L
-#define SQ_VOP3P_1__OP_SEL_HI_MASK                                                                            0x18000000L
-#define SQ_VOP3P_1__NEG_MASK                                                                                  0xE0000000L
-//SQ_VOP3_0
-#define SQ_VOP3_0__VDST__SHIFT                                                                                0x0
-#define SQ_VOP3_0__ABS__SHIFT                                                                                 0x8
-#define SQ_VOP3_0__OP_SEL__SHIFT                                                                              0xb
-#define SQ_VOP3_0__CLAMP__SHIFT                                                                               0xf
-#define SQ_VOP3_0__OP__SHIFT                                                                                  0x10
-#define SQ_VOP3_0__ENCODING__SHIFT                                                                            0x1a
-#define SQ_VOP3_0__VDST_MASK                                                                                  0x000000FFL
-#define SQ_VOP3_0__ABS_MASK                                                                                   0x00000700L
-#define SQ_VOP3_0__OP_SEL_MASK                                                                                0x00007800L
-#define SQ_VOP3_0__CLAMP_MASK                                                                                 0x00008000L
-#define SQ_VOP3_0__OP_MASK                                                                                    0x03FF0000L
-#define SQ_VOP3_0__ENCODING_MASK                                                                              0xFC000000L
-//SQ_VOP3_0_SDST_ENC
-#define SQ_VOP3_0_SDST_ENC__VDST__SHIFT                                                                       0x0
-#define SQ_VOP3_0_SDST_ENC__SDST__SHIFT                                                                       0x8
-#define SQ_VOP3_0_SDST_ENC__CLAMP__SHIFT                                                                      0xf
-#define SQ_VOP3_0_SDST_ENC__OP__SHIFT                                                                         0x10
-#define SQ_VOP3_0_SDST_ENC__ENCODING__SHIFT                                                                   0x1a
-#define SQ_VOP3_0_SDST_ENC__VDST_MASK                                                                         0x000000FFL
-#define SQ_VOP3_0_SDST_ENC__SDST_MASK                                                                         0x00007F00L
-#define SQ_VOP3_0_SDST_ENC__CLAMP_MASK                                                                        0x00008000L
-#define SQ_VOP3_0_SDST_ENC__OP_MASK                                                                           0x03FF0000L
-#define SQ_VOP3_0_SDST_ENC__ENCODING_MASK                                                                     0xFC000000L
-//SQ_VOP3_1
-#define SQ_VOP3_1__SRC0__SHIFT                                                                                0x0
-#define SQ_VOP3_1__SRC1__SHIFT                                                                                0x9
-#define SQ_VOP3_1__SRC2__SHIFT                                                                                0x12
-#define SQ_VOP3_1__OMOD__SHIFT                                                                                0x1b
-#define SQ_VOP3_1__NEG__SHIFT                                                                                 0x1d
-#define SQ_VOP3_1__SRC0_MASK                                                                                  0x000001FFL
-#define SQ_VOP3_1__SRC1_MASK                                                                                  0x0003FE00L
-#define SQ_VOP3_1__SRC2_MASK                                                                                  0x07FC0000L
-#define SQ_VOP3_1__OMOD_MASK                                                                                  0x18000000L
-#define SQ_VOP3_1__NEG_MASK                                                                                   0xE0000000L
-//SQ_VOPC
-#define SQ_VOPC__SRC0__SHIFT                                                                                  0x0
-#define SQ_VOPC__VSRC1__SHIFT                                                                                 0x9
-#define SQ_VOPC__OP__SHIFT                                                                                    0x11
-#define SQ_VOPC__ENCODING__SHIFT                                                                              0x19
-#define SQ_VOPC__SRC0_MASK                                                                                    0x000001FFL
-#define SQ_VOPC__VSRC1_MASK                                                                                   0x0001FE00L
-#define SQ_VOPC__OP_MASK                                                                                      0x01FE0000L
-#define SQ_VOPC__ENCODING_MASK                                                                                0xFE000000L
-//SQ_VOP_DPP
-#define SQ_VOP_DPP__SRC0__SHIFT                                                                               0x0
-#define SQ_VOP_DPP__DPP_CTRL__SHIFT                                                                           0x8
-#define SQ_VOP_DPP__BOUND_CTRL__SHIFT                                                                         0x13
-#define SQ_VOP_DPP__SRC0_NEG__SHIFT                                                                           0x14
-#define SQ_VOP_DPP__SRC0_ABS__SHIFT                                                                           0x15
-#define SQ_VOP_DPP__SRC1_NEG__SHIFT                                                                           0x16
-#define SQ_VOP_DPP__SRC1_ABS__SHIFT                                                                           0x17
-#define SQ_VOP_DPP__BANK_MASK__SHIFT                                                                          0x18
-#define SQ_VOP_DPP__ROW_MASK__SHIFT                                                                           0x1c
-#define SQ_VOP_DPP__SRC0_MASK                                                                                 0x000000FFL
-#define SQ_VOP_DPP__DPP_CTRL_MASK                                                                             0x0001FF00L
-#define SQ_VOP_DPP__BOUND_CTRL_MASK                                                                           0x00080000L
-#define SQ_VOP_DPP__SRC0_NEG_MASK                                                                             0x00100000L
-#define SQ_VOP_DPP__SRC0_ABS_MASK                                                                             0x00200000L
-#define SQ_VOP_DPP__SRC1_NEG_MASK                                                                             0x00400000L
-#define SQ_VOP_DPP__SRC1_ABS_MASK                                                                             0x00800000L
-#define SQ_VOP_DPP__BANK_MASK_MASK                                                                            0x0F000000L
-#define SQ_VOP_DPP__ROW_MASK_MASK                                                                             0xF0000000L
-//SQ_VOP_SDWA
-#define SQ_VOP_SDWA__SRC0__SHIFT                                                                              0x0
-#define SQ_VOP_SDWA__DST_SEL__SHIFT                                                                           0x8
-#define SQ_VOP_SDWA__DST_UNUSED__SHIFT                                                                        0xb
-#define SQ_VOP_SDWA__CLAMP__SHIFT                                                                             0xd
-#define SQ_VOP_SDWA__OMOD__SHIFT                                                                              0xe
-#define SQ_VOP_SDWA__SRC0_SEL__SHIFT                                                                          0x10
-#define SQ_VOP_SDWA__SRC0_SEXT__SHIFT                                                                         0x13
-#define SQ_VOP_SDWA__SRC0_NEG__SHIFT                                                                          0x14
-#define SQ_VOP_SDWA__SRC0_ABS__SHIFT                                                                          0x15
-#define SQ_VOP_SDWA__S0__SHIFT                                                                                0x17
-#define SQ_VOP_SDWA__SRC1_SEL__SHIFT                                                                          0x18
-#define SQ_VOP_SDWA__SRC1_SEXT__SHIFT                                                                         0x1b
-#define SQ_VOP_SDWA__SRC1_NEG__SHIFT                                                                          0x1c
-#define SQ_VOP_SDWA__SRC1_ABS__SHIFT                                                                          0x1d
-#define SQ_VOP_SDWA__S1__SHIFT                                                                                0x1f
-#define SQ_VOP_SDWA__SRC0_MASK                                                                                0x000000FFL
-#define SQ_VOP_SDWA__DST_SEL_MASK                                                                             0x00000700L
-#define SQ_VOP_SDWA__DST_UNUSED_MASK                                                                          0x00001800L
-#define SQ_VOP_SDWA__CLAMP_MASK                                                                               0x00002000L
-#define SQ_VOP_SDWA__OMOD_MASK                                                                                0x0000C000L
-#define SQ_VOP_SDWA__SRC0_SEL_MASK                                                                            0x00070000L
-#define SQ_VOP_SDWA__SRC0_SEXT_MASK                                                                           0x00080000L
-#define SQ_VOP_SDWA__SRC0_NEG_MASK                                                                            0x00100000L
-#define SQ_VOP_SDWA__SRC0_ABS_MASK                                                                            0x00200000L
-#define SQ_VOP_SDWA__S0_MASK                                                                                  0x00800000L
-#define SQ_VOP_SDWA__SRC1_SEL_MASK                                                                            0x07000000L
-#define SQ_VOP_SDWA__SRC1_SEXT_MASK                                                                           0x08000000L
-#define SQ_VOP_SDWA__SRC1_NEG_MASK                                                                            0x10000000L
-#define SQ_VOP_SDWA__SRC1_ABS_MASK                                                                            0x20000000L
-#define SQ_VOP_SDWA__S1_MASK                                                                                  0x80000000L
-//SQ_VOP_SDWA_SDST_ENC
-#define SQ_VOP_SDWA_SDST_ENC__SRC0__SHIFT                                                                     0x0
-#define SQ_VOP_SDWA_SDST_ENC__SDST__SHIFT                                                                     0x8
-#define SQ_VOP_SDWA_SDST_ENC__SD__SHIFT                                                                       0xf
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEL__SHIFT                                                                 0x10
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEXT__SHIFT                                                                0x13
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_NEG__SHIFT                                                                 0x14
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_ABS__SHIFT                                                                 0x15
-#define SQ_VOP_SDWA_SDST_ENC__S0__SHIFT                                                                       0x17
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEL__SHIFT                                                                 0x18
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEXT__SHIFT                                                                0x1b
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_NEG__SHIFT                                                                 0x1c
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_ABS__SHIFT                                                                 0x1d
-#define SQ_VOP_SDWA_SDST_ENC__S1__SHIFT                                                                       0x1f
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_MASK                                                                       0x000000FFL
-#define SQ_VOP_SDWA_SDST_ENC__SDST_MASK                                                                       0x00007F00L
-#define SQ_VOP_SDWA_SDST_ENC__SD_MASK                                                                         0x00008000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEL_MASK                                                                   0x00070000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_SEXT_MASK                                                                  0x00080000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_NEG_MASK                                                                   0x00100000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC0_ABS_MASK                                                                   0x00200000L
-#define SQ_VOP_SDWA_SDST_ENC__S0_MASK                                                                         0x00800000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEL_MASK                                                                   0x07000000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_SEXT_MASK                                                                  0x08000000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_NEG_MASK                                                                   0x10000000L
-#define SQ_VOP_SDWA_SDST_ENC__SRC1_ABS_MASK                                                                   0x20000000L
-#define SQ_VOP_SDWA_SDST_ENC__S1_MASK                                                                         0x80000000L
-//SQ_LB_CTR_CTRL
-#define SQ_LB_CTR_CTRL__START__SHIFT                                                                          0x0
-#define SQ_LB_CTR_CTRL__LOAD__SHIFT                                                                           0x1
-#define SQ_LB_CTR_CTRL__CLEAR__SHIFT                                                                          0x2
-#define SQ_LB_CTR_CTRL__START_MASK                                                                            0x00000001L
-#define SQ_LB_CTR_CTRL__LOAD_MASK                                                                             0x00000002L
-#define SQ_LB_CTR_CTRL__CLEAR_MASK                                                                            0x00000004L
-//SQ_LB_DATA0
-#define SQ_LB_DATA0__DATA__SHIFT                                                                              0x0
-#define SQ_LB_DATA0__DATA_MASK                                                                                0xFFFFFFFFL
-//SQ_LB_DATA1
-#define SQ_LB_DATA1__DATA__SHIFT                                                                              0x0
-#define SQ_LB_DATA1__DATA_MASK                                                                                0xFFFFFFFFL
-//SQ_LB_DATA2
-#define SQ_LB_DATA2__DATA__SHIFT                                                                              0x0
-#define SQ_LB_DATA2__DATA_MASK                                                                                0xFFFFFFFFL
-//SQ_LB_DATA3
-#define SQ_LB_DATA3__DATA__SHIFT                                                                              0x0
-#define SQ_LB_DATA3__DATA_MASK                                                                                0xFFFFFFFFL
-//SQ_LB_CTR_SEL
-#define SQ_LB_CTR_SEL__SEL0__SHIFT                                                                            0x0
-#define SQ_LB_CTR_SEL__SEL1__SHIFT                                                                            0x4
-#define SQ_LB_CTR_SEL__SEL2__SHIFT                                                                            0x8
-#define SQ_LB_CTR_SEL__SEL3__SHIFT                                                                            0xc
-#define SQ_LB_CTR_SEL__SEL0_MASK                                                                              0x0000000FL
-#define SQ_LB_CTR_SEL__SEL1_MASK                                                                              0x000000F0L
-#define SQ_LB_CTR_SEL__SEL2_MASK                                                                              0x00000F00L
-#define SQ_LB_CTR_SEL__SEL3_MASK                                                                              0x0000F000L
-//SQ_LB_CTR0_CU
-#define SQ_LB_CTR0_CU__SH0_MASK__SHIFT                                                                        0x0
-#define SQ_LB_CTR0_CU__SH1_MASK__SHIFT                                                                        0x10
-#define SQ_LB_CTR0_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
-#define SQ_LB_CTR0_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
-//SQ_LB_CTR1_CU
-#define SQ_LB_CTR1_CU__SH0_MASK__SHIFT                                                                        0x0
-#define SQ_LB_CTR1_CU__SH1_MASK__SHIFT                                                                        0x10
-#define SQ_LB_CTR1_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
-#define SQ_LB_CTR1_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
-//SQ_LB_CTR2_CU
-#define SQ_LB_CTR2_CU__SH0_MASK__SHIFT                                                                        0x0
-#define SQ_LB_CTR2_CU__SH1_MASK__SHIFT                                                                        0x10
-#define SQ_LB_CTR2_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
-#define SQ_LB_CTR2_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
-//SQ_LB_CTR3_CU
-#define SQ_LB_CTR3_CU__SH0_MASK__SHIFT                                                                        0x0
-#define SQ_LB_CTR3_CU__SH1_MASK__SHIFT                                                                        0x10
-#define SQ_LB_CTR3_CU__SH0_MASK_MASK                                                                          0x0000FFFFL
-#define SQ_LB_CTR3_CU__SH1_MASK_MASK                                                                          0xFFFF0000L
-//SQC_EDC_CNT
-#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x0
-#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0x2
-#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0x4
-#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0x6
-#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x8
-#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0xa
-#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0xc
-#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0xe
-#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x10
-#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0x12
-#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0x14
-#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0x16
-#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_SEC_COUNT__SHIFT                                                 0x18
-#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_DED_COUNT__SHIFT                                                 0x1a
-#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_SEC_COUNT__SHIFT                                                    0x1c
-#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_DED_COUNT__SHIFT                                                    0x1e
-#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x00000003L
-#define SQC_EDC_CNT__DATA_CU0_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x0000000CL
-#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x00000030L
-#define SQC_EDC_CNT__DATA_CU0_UTCL1_LFIFO_DED_COUNT_MASK                                                      0x000000C0L
-#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x00000300L
-#define SQC_EDC_CNT__DATA_CU1_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x00000C00L
-#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x00003000L
-#define SQC_EDC_CNT__DATA_CU1_UTCL1_LFIFO_DED_COUNT_MASK                                                      0x0000C000L
-#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x00030000L
-#define SQC_EDC_CNT__DATA_CU2_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x000C0000L
-#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x00300000L
-#define SQC_EDC_CNT__DATA_CU2_UTCL1_LFIFO_DED_COUNT_MASK                                                      0x00C00000L
-#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_SEC_COUNT_MASK                                                   0x03000000L
-#define SQC_EDC_CNT__DATA_CU3_WRITE_DATA_BUF_DED_COUNT_MASK                                                   0x0C000000L
-#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_SEC_COUNT_MASK                                                      0x30000000L
-#define SQC_EDC_CNT__DATA_CU3_UTCL1_LFIFO_DED_COUNT_MASK                                                      0xC0000000L
-//SQ_EDC_SEC_CNT
-#define SQ_EDC_SEC_CNT__LDS_SEC__SHIFT                                                                        0x0
-#define SQ_EDC_SEC_CNT__SGPR_SEC__SHIFT                                                                       0x8
-#define SQ_EDC_SEC_CNT__VGPR_SEC__SHIFT                                                                       0x10
-#define SQ_EDC_SEC_CNT__LDS_SEC_MASK                                                                          0x000000FFL
-#define SQ_EDC_SEC_CNT__SGPR_SEC_MASK                                                                         0x0000FF00L
-#define SQ_EDC_SEC_CNT__VGPR_SEC_MASK                                                                         0x00FF0000L
-//SQ_EDC_DED_CNT
-#define SQ_EDC_DED_CNT__LDS_DED__SHIFT                                                                        0x0
-#define SQ_EDC_DED_CNT__SGPR_DED__SHIFT                                                                       0x8
-#define SQ_EDC_DED_CNT__VGPR_DED__SHIFT                                                                       0x10
-#define SQ_EDC_DED_CNT__LDS_DED_MASK                                                                          0x000000FFL
-#define SQ_EDC_DED_CNT__SGPR_DED_MASK                                                                         0x0000FF00L
-#define SQ_EDC_DED_CNT__VGPR_DED_MASK                                                                         0x00FF0000L
-//SQ_EDC_INFO
-#define SQ_EDC_INFO__WAVE_ID__SHIFT                                                                           0x0
-#define SQ_EDC_INFO__SIMD_ID__SHIFT                                                                           0x4
-#define SQ_EDC_INFO__SOURCE__SHIFT                                                                            0x6
-#define SQ_EDC_INFO__VM_ID__SHIFT                                                                             0x9
-#define SQ_EDC_INFO__WAVE_ID_MASK                                                                             0x0000000FL
-#define SQ_EDC_INFO__SIMD_ID_MASK                                                                             0x00000030L
-#define SQ_EDC_INFO__SOURCE_MASK                                                                              0x000001C0L
-#define SQ_EDC_INFO__VM_ID_MASK                                                                               0x00001E00L
-//SQ_EDC_CNT
-#define SQ_EDC_CNT__LDS_D_SEC_COUNT__SHIFT                                                                    0x0
-#define SQ_EDC_CNT__LDS_D_DED_COUNT__SHIFT                                                                    0x2
-#define SQ_EDC_CNT__LDS_I_SEC_COUNT__SHIFT                                                                    0x4
-#define SQ_EDC_CNT__LDS_I_DED_COUNT__SHIFT                                                                    0x6
-#define SQ_EDC_CNT__SGPR_SEC_COUNT__SHIFT                                                                     0x8
-#define SQ_EDC_CNT__SGPR_DED_COUNT__SHIFT                                                                     0xa
-#define SQ_EDC_CNT__VGPR0_SEC_COUNT__SHIFT                                                                    0xc
-#define SQ_EDC_CNT__VGPR0_DED_COUNT__SHIFT                                                                    0xe
-#define SQ_EDC_CNT__VGPR1_SEC_COUNT__SHIFT                                                                    0x10
-#define SQ_EDC_CNT__VGPR1_DED_COUNT__SHIFT                                                                    0x12
-#define SQ_EDC_CNT__VGPR2_SEC_COUNT__SHIFT                                                                    0x14
-#define SQ_EDC_CNT__VGPR2_DED_COUNT__SHIFT                                                                    0x16
-#define SQ_EDC_CNT__VGPR3_SEC_COUNT__SHIFT                                                                    0x18
-#define SQ_EDC_CNT__VGPR3_DED_COUNT__SHIFT                                                                    0x1a
-#define SQ_EDC_CNT__LDS_D_SEC_COUNT_MASK                                                                      0x00000003L
-#define SQ_EDC_CNT__LDS_D_DED_COUNT_MASK                                                                      0x0000000CL
-#define SQ_EDC_CNT__LDS_I_SEC_COUNT_MASK                                                                      0x00000030L
-#define SQ_EDC_CNT__LDS_I_DED_COUNT_MASK                                                                      0x000000C0L
-#define SQ_EDC_CNT__SGPR_SEC_COUNT_MASK                                                                       0x00000300L
-#define SQ_EDC_CNT__SGPR_DED_COUNT_MASK                                                                       0x00000C00L
-#define SQ_EDC_CNT__VGPR0_SEC_COUNT_MASK                                                                      0x00003000L
-#define SQ_EDC_CNT__VGPR0_DED_COUNT_MASK                                                                      0x0000C000L
-#define SQ_EDC_CNT__VGPR1_SEC_COUNT_MASK                                                                      0x00030000L
-#define SQ_EDC_CNT__VGPR1_DED_COUNT_MASK                                                                      0x000C0000L
-#define SQ_EDC_CNT__VGPR2_SEC_COUNT_MASK                                                                      0x00300000L
-#define SQ_EDC_CNT__VGPR2_DED_COUNT_MASK                                                                      0x00C00000L
-#define SQ_EDC_CNT__VGPR3_SEC_COUNT_MASK                                                                      0x03000000L
-#define SQ_EDC_CNT__VGPR3_DED_COUNT_MASK                                                                      0x0C000000L
-//SQ_EDC_FUE_CNTL
-#define SQ_EDC_FUE_CNTL__BLOCK_FUE_FLAGS__SHIFT                                                               0x0
-#define SQ_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES__SHIFT                                                         0x10
-#define SQ_EDC_FUE_CNTL__BLOCK_FUE_FLAGS_MASK                                                                 0x0000FFFFL
-#define SQ_EDC_FUE_CNTL__FUE_INTERRUPT_ENABLES_MASK                                                           0xFFFF0000L
-//SQ_THREAD_TRACE_WORD_CMN
-#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE__SHIFT                                                           0x0
-#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA__SHIFT                                                           0x4
-#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE_MASK                                                             0x000FL
-#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA_MASK                                                             0x0010L
-//SQ_THREAD_TRACE_WORD_EVENT
-#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE__SHIFT                                                         0x0
-#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA__SHIFT                                                         0x4
-#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID__SHIFT                                                              0x5
-#define SQ_THREAD_TRACE_WORD_EVENT__STAGE__SHIFT                                                              0x6
-#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE__SHIFT                                                         0xa
-#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE_MASK                                                           0x000FL
-#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA_MASK                                                           0x0010L
-#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID_MASK                                                                0x0020L
-#define SQ_THREAD_TRACE_WORD_EVENT__STAGE_MASK                                                                0x01C0L
-#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE_MASK                                                           0xFC00L
-//SQ_THREAD_TRACE_WORD_INST
-#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE__SHIFT                                                          0x0
-#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA__SHIFT                                                          0x4
-#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID__SHIFT                                                             0x5
-#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID__SHIFT                                                             0x9
-#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE__SHIFT                                                           0xb
-#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE_MASK                                                            0x000FL
-#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA_MASK                                                            0x0010L
-#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID_MASK                                                               0x01E0L
-#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID_MASK                                                               0x0600L
-#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE_MASK                                                             0xF800L
-//SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE__SHIFT                                                0x0
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA__SHIFT                                                0x4
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID__SHIFT                                                   0x5
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID__SHIFT                                                   0x9
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TRAP_ERROR__SHIFT                                                0xf
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO__SHIFT                                                     0x10
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE_MASK                                                  0x0000000FL
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA_MASK                                                  0x00000010L
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID_MASK                                                     0x000001E0L
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID_MASK                                                     0x00000600L
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TRAP_ERROR_MASK                                                  0x00008000L
-#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO_MASK                                                       0xFFFF0000L
-//SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE__SHIFT                                          0x0
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA__SHIFT                                          0x4
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID__SHIFT                                               0x5
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID__SHIFT                                               0x6
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID__SHIFT                                             0xa
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID__SHIFT                                             0xe
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO__SHIFT                                             0x10
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE_MASK                                            0x0000000FL
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA_MASK                                            0x00000010L
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID_MASK                                                 0x00000020L
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID_MASK                                                 0x000003C0L
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID_MASK                                               0x00003C00L
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID_MASK                                               0x0000C000L
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO_MASK                                               0xFFFF0000L
-//SQ_THREAD_TRACE_WORD_ISSUE
-#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE__SHIFT                                                         0x0
-#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA__SHIFT                                                         0x4
-#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID__SHIFT                                                            0x5
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST0__SHIFT                                                              0x8
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST1__SHIFT                                                              0xa
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST2__SHIFT                                                              0xc
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST3__SHIFT                                                              0xe
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST4__SHIFT                                                              0x10
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST5__SHIFT                                                              0x12
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST6__SHIFT                                                              0x14
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST7__SHIFT                                                              0x16
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST8__SHIFT                                                              0x18
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST9__SHIFT                                                              0x1a
-#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE_MASK                                                           0x0000000FL
-#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA_MASK                                                           0x00000010L
-#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID_MASK                                                              0x00000060L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST0_MASK                                                                0x00000300L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST1_MASK                                                                0x00000C00L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST2_MASK                                                                0x00003000L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST3_MASK                                                                0x0000C000L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST4_MASK                                                                0x00030000L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST5_MASK                                                                0x000C0000L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST6_MASK                                                                0x00300000L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST7_MASK                                                                0x00C00000L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST8_MASK                                                                0x03000000L
-#define SQ_THREAD_TRACE_WORD_ISSUE__INST9_MASK                                                                0x0C000000L
-//SQ_THREAD_TRACE_WORD_MISC
-#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE__SHIFT                                                          0x0
-#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA__SHIFT                                                          0x4
-#define SQ_THREAD_TRACE_WORD_MISC__SH_ID__SHIFT                                                               0xc
-#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE__SHIFT                                                     0xd
-#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE_MASK                                                            0x000FL
-#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA_MASK                                                            0x0FF0L
-#define SQ_THREAD_TRACE_WORD_MISC__SH_ID_MASK                                                                 0x1000L
-#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE_MASK                                                       0xE000L
-//SQ_THREAD_TRACE_WORD_PERF_1_OF_2
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE__SHIFT                                                   0x0
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA__SHIFT                                                   0x4
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID__SHIFT                                                        0x5
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID__SHIFT                                                        0x6
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK__SHIFT                                                    0xa
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0__SHIFT                                                        0xc
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO__SHIFT                                                     0x19
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE_MASK                                                     0x0000000FL
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA_MASK                                                     0x00000010L
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID_MASK                                                          0x00000020L
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID_MASK                                                          0x000003C0L
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK_MASK                                                      0x00000C00L
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0_MASK                                                          0x01FFF000L
-#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO_MASK                                                       0xFE000000L
-//SQ_THREAD_TRACE_WORD_REG_1_OF_2
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE__SHIFT                                                    0x0
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA__SHIFT                                                    0x4
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID__SHIFT                                                       0x5
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID__SHIFT                                                         0x7
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV__SHIFT                                              0x9
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE__SHIFT                                                      0xa
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV__SHIFT                                                      0xe
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP__SHIFT                                                        0xf
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR__SHIFT                                                      0x10
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE_MASK                                                      0x0000000FL
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA_MASK                                                      0x00000010L
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID_MASK                                                         0x00000060L
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID_MASK                                                           0x00000180L
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV_MASK                                                0x00000200L
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE_MASK                                                        0x00001C00L
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV_MASK                                                        0x00004000L
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP_MASK                                                          0x00008000L
-#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR_MASK                                                        0xFFFF0000L
-//SQ_THREAD_TRACE_WORD_REG_2_OF_2
-#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA__SHIFT                                                          0x0
-#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA_MASK                                                            0xFFFFFFFFL
-//SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE__SHIFT                                                 0x0
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA__SHIFT                                                 0x4
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID__SHIFT                                                    0x5
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID__SHIFT                                                      0x7
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR__SHIFT                                                   0x9
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO__SHIFT                                                    0x10
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE_MASK                                                   0x0000000FL
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA_MASK                                                   0x00000010L
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID_MASK                                                      0x00000060L
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID_MASK                                                        0x00000180L
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR_MASK                                                     0x0000FE00L
-#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO_MASK                                                      0xFFFF0000L
-//SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2
-#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI__SHIFT                                                    0x0
-#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI_MASK                                                      0x0000FFFFL
-//SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2
-#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE__SHIFT                                              0x0
-#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO__SHIFT                                                 0x10
-#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE_MASK                                                0x0000000FL
-#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO_MASK                                                   0xFFFF0000L
-//SQ_THREAD_TRACE_WORD_WAVE
-#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE__SHIFT                                                          0x0
-#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA__SHIFT                                                          0x4
-#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID__SHIFT                                                               0x5
-#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID__SHIFT                                                               0x6
-#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID__SHIFT                                                             0xa
-#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID__SHIFT                                                             0xe
-#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE_MASK                                                            0x000FL
-#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA_MASK                                                            0x0010L
-#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID_MASK                                                                 0x0020L
-#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID_MASK                                                                 0x03C0L
-#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID_MASK                                                               0x3C00L
-#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID_MASK                                                               0xC000L
-//SQ_THREAD_TRACE_WORD_WAVE_START
-#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE__SHIFT                                                    0x0
-#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA__SHIFT                                                    0x4
-#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID__SHIFT                                                         0x5
-#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID__SHIFT                                                         0x6
-#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID__SHIFT                                                       0xa
-#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID__SHIFT                                                       0xe
-#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER__SHIFT                                                    0x10
-#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED__SHIFT                                        0x15
-#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT__SHIFT                                                         0x16
-#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID__SHIFT                                                         0x1d
-#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE_MASK                                                      0x0000000FL
-#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA_MASK                                                      0x00000010L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID_MASK                                                           0x00000020L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID_MASK                                                           0x000003C0L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID_MASK                                                         0x00003C00L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID_MASK                                                         0x0000C000L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER_MASK                                                      0x001F0000L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED_MASK                                          0x00200000L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT_MASK                                                           0x1FC00000L
-#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID_MASK                                                           0xE0000000L
-//SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2
-#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI__SHIFT                                                     0x0
-#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI_MASK                                                       0x00FFFFFFL
-//SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI__SHIFT                                             0x0
-#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI_MASK                                               0xFFFFL
-//SQ_THREAD_TRACE_WORD_PERF_2_OF_2
-#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI__SHIFT                                                     0x0
-#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2__SHIFT                                                        0x6
-#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3__SHIFT                                                        0x13
-#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI_MASK                                                       0x0000003FL
-#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2_MASK                                                          0x0007FFC0L
-#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3_MASK                                                          0xFFF80000L
-//SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2
-#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI__SHIFT                                                 0x0
-#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI_MASK                                                   0xFFFFFFFFL
-//SQ_WREXEC_EXEC_HI
-#define SQ_WREXEC_EXEC_HI__ADDR_HI__SHIFT                                                                     0x0
-#define SQ_WREXEC_EXEC_HI__FIRST_WAVE__SHIFT                                                                  0x1a
-#define SQ_WREXEC_EXEC_HI__ATC__SHIFT                                                                         0x1b
-#define SQ_WREXEC_EXEC_HI__MTYPE__SHIFT                                                                       0x1c
-#define SQ_WREXEC_EXEC_HI__MSB__SHIFT                                                                         0x1f
-#define SQ_WREXEC_EXEC_HI__ADDR_HI_MASK                                                                       0x0000FFFFL
-#define SQ_WREXEC_EXEC_HI__FIRST_WAVE_MASK                                                                    0x04000000L
-#define SQ_WREXEC_EXEC_HI__ATC_MASK                                                                           0x08000000L
-#define SQ_WREXEC_EXEC_HI__MTYPE_MASK                                                                         0x70000000L
-#define SQ_WREXEC_EXEC_HI__MSB_MASK                                                                           0x80000000L
-//SQ_WREXEC_EXEC_LO
-#define SQ_WREXEC_EXEC_LO__ADDR_LO__SHIFT                                                                     0x0
-#define SQ_WREXEC_EXEC_LO__ADDR_LO_MASK                                                                       0xFFFFFFFFL
-//SQ_BUF_RSRC_WORD0
-#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS__SHIFT                                                                0x0
-#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS_MASK                                                                  0xFFFFFFFFL
-//SQ_BUF_RSRC_WORD1
-#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT                                                             0x0
-#define SQ_BUF_RSRC_WORD1__STRIDE__SHIFT                                                                      0x10
-#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE__SHIFT                                                               0x1e
-#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE__SHIFT                                                              0x1f
-#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI_MASK                                                               0x0000FFFFL
-#define SQ_BUF_RSRC_WORD1__STRIDE_MASK                                                                        0x3FFF0000L
-#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE_MASK                                                                 0x40000000L
-#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE_MASK                                                                0x80000000L
-//SQ_BUF_RSRC_WORD2
-#define SQ_BUF_RSRC_WORD2__NUM_RECORDS__SHIFT                                                                 0x0
-#define SQ_BUF_RSRC_WORD2__NUM_RECORDS_MASK                                                                   0xFFFFFFFFL
-//SQ_BUF_RSRC_WORD3
-#define SQ_BUF_RSRC_WORD3__DST_SEL_X__SHIFT                                                                   0x0
-#define SQ_BUF_RSRC_WORD3__DST_SEL_Y__SHIFT                                                                   0x3
-#define SQ_BUF_RSRC_WORD3__DST_SEL_Z__SHIFT                                                                   0x6
-#define SQ_BUF_RSRC_WORD3__DST_SEL_W__SHIFT                                                                   0x9
-#define SQ_BUF_RSRC_WORD3__NUM_FORMAT__SHIFT                                                                  0xc
-#define SQ_BUF_RSRC_WORD3__DATA_FORMAT__SHIFT                                                                 0xf
-#define SQ_BUF_RSRC_WORD3__USER_VM_ENABLE__SHIFT                                                              0x13
-#define SQ_BUF_RSRC_WORD3__USER_VM_MODE__SHIFT                                                                0x14
-#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE__SHIFT                                                                0x15
-#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE__SHIFT                                                              0x17
-#define SQ_BUF_RSRC_WORD3__NV__SHIFT                                                                          0x1b
-#define SQ_BUF_RSRC_WORD3__TYPE__SHIFT                                                                        0x1e
-#define SQ_BUF_RSRC_WORD3__DST_SEL_X_MASK                                                                     0x00000007L
-#define SQ_BUF_RSRC_WORD3__DST_SEL_Y_MASK                                                                     0x00000038L
-#define SQ_BUF_RSRC_WORD3__DST_SEL_Z_MASK                                                                     0x000001C0L
-#define SQ_BUF_RSRC_WORD3__DST_SEL_W_MASK                                                                     0x00000E00L
-#define SQ_BUF_RSRC_WORD3__NUM_FORMAT_MASK                                                                    0x00007000L
-#define SQ_BUF_RSRC_WORD3__DATA_FORMAT_MASK                                                                   0x00078000L
-#define SQ_BUF_RSRC_WORD3__USER_VM_ENABLE_MASK                                                                0x00080000L
-#define SQ_BUF_RSRC_WORD3__USER_VM_MODE_MASK                                                                  0x00100000L
-#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE_MASK                                                                  0x00600000L
-#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE_MASK                                                                0x00800000L
-#define SQ_BUF_RSRC_WORD3__NV_MASK                                                                            0x08000000L
-#define SQ_BUF_RSRC_WORD3__TYPE_MASK                                                                          0xC0000000L
-//SQ_IMG_RSRC_WORD0
-#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS__SHIFT                                                                0x0
-#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS_MASK                                                                  0xFFFFFFFFL
-//SQ_IMG_RSRC_WORD1
-#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT                                                             0x0
-#define SQ_IMG_RSRC_WORD1__MIN_LOD__SHIFT                                                                     0x8
-#define SQ_IMG_RSRC_WORD1__DATA_FORMAT__SHIFT                                                                 0x14
-#define SQ_IMG_RSRC_WORD1__NUM_FORMAT__SHIFT                                                                  0x1a
-#define SQ_IMG_RSRC_WORD1__NV__SHIFT                                                                          0x1e
-#define SQ_IMG_RSRC_WORD1__META_DIRECT__SHIFT                                                                 0x1f
-#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI_MASK                                                               0x000000FFL
-#define SQ_IMG_RSRC_WORD1__MIN_LOD_MASK                                                                       0x000FFF00L
-#define SQ_IMG_RSRC_WORD1__DATA_FORMAT_MASK                                                                   0x03F00000L
-#define SQ_IMG_RSRC_WORD1__NUM_FORMAT_MASK                                                                    0x3C000000L
-#define SQ_IMG_RSRC_WORD1__NV_MASK                                                                            0x40000000L
-#define SQ_IMG_RSRC_WORD1__META_DIRECT_MASK                                                                   0x80000000L
-//SQ_IMG_RSRC_WORD2
-#define SQ_IMG_RSRC_WORD2__WIDTH__SHIFT                                                                       0x0
-#define SQ_IMG_RSRC_WORD2__HEIGHT__SHIFT                                                                      0xe
-#define SQ_IMG_RSRC_WORD2__PERF_MOD__SHIFT                                                                    0x1c
-#define SQ_IMG_RSRC_WORD2__WIDTH_MASK                                                                         0x00003FFFL
-#define SQ_IMG_RSRC_WORD2__HEIGHT_MASK                                                                        0x0FFFC000L
-#define SQ_IMG_RSRC_WORD2__PERF_MOD_MASK                                                                      0x70000000L
-//SQ_IMG_RSRC_WORD3
-#define SQ_IMG_RSRC_WORD3__DST_SEL_X__SHIFT                                                                   0x0
-#define SQ_IMG_RSRC_WORD3__DST_SEL_Y__SHIFT                                                                   0x3
-#define SQ_IMG_RSRC_WORD3__DST_SEL_Z__SHIFT                                                                   0x6
-#define SQ_IMG_RSRC_WORD3__DST_SEL_W__SHIFT                                                                   0x9
-#define SQ_IMG_RSRC_WORD3__BASE_LEVEL__SHIFT                                                                  0xc
-#define SQ_IMG_RSRC_WORD3__LAST_LEVEL__SHIFT                                                                  0x10
-#define SQ_IMG_RSRC_WORD3__SW_MODE__SHIFT                                                                     0x14
-#define SQ_IMG_RSRC_WORD3__TYPE__SHIFT                                                                        0x1c
-#define SQ_IMG_RSRC_WORD3__DST_SEL_X_MASK                                                                     0x00000007L
-#define SQ_IMG_RSRC_WORD3__DST_SEL_Y_MASK                                                                     0x00000038L
-#define SQ_IMG_RSRC_WORD3__DST_SEL_Z_MASK                                                                     0x000001C0L
-#define SQ_IMG_RSRC_WORD3__DST_SEL_W_MASK                                                                     0x00000E00L
-#define SQ_IMG_RSRC_WORD3__BASE_LEVEL_MASK                                                                    0x0000F000L
-#define SQ_IMG_RSRC_WORD3__LAST_LEVEL_MASK                                                                    0x000F0000L
-#define SQ_IMG_RSRC_WORD3__SW_MODE_MASK                                                                       0x01F00000L
-#define SQ_IMG_RSRC_WORD3__TYPE_MASK                                                                          0xF0000000L
-//SQ_IMG_RSRC_WORD4
-#define SQ_IMG_RSRC_WORD4__DEPTH__SHIFT                                                                       0x0
-#define SQ_IMG_RSRC_WORD4__PITCH__SHIFT                                                                       0xd
-#define SQ_IMG_RSRC_WORD4__BC_SWIZZLE__SHIFT                                                                  0x1d
-#define SQ_IMG_RSRC_WORD4__DEPTH_MASK                                                                         0x00001FFFL
-#define SQ_IMG_RSRC_WORD4__PITCH_MASK                                                                         0x1FFFE000L
-#define SQ_IMG_RSRC_WORD4__BC_SWIZZLE_MASK                                                                    0xE0000000L
-//SQ_IMG_RSRC_WORD5
-#define SQ_IMG_RSRC_WORD5__BASE_ARRAY__SHIFT                                                                  0x0
-#define SQ_IMG_RSRC_WORD5__ARRAY_PITCH__SHIFT                                                                 0xd
-#define SQ_IMG_RSRC_WORD5__META_DATA_ADDRESS__SHIFT                                                           0x11
-#define SQ_IMG_RSRC_WORD5__META_LINEAR__SHIFT                                                                 0x19
-#define SQ_IMG_RSRC_WORD5__META_PIPE_ALIGNED__SHIFT                                                           0x1a
-#define SQ_IMG_RSRC_WORD5__META_RB_ALIGNED__SHIFT                                                             0x1b
-#define SQ_IMG_RSRC_WORD5__MAX_MIP__SHIFT                                                                     0x1c
-#define SQ_IMG_RSRC_WORD5__BASE_ARRAY_MASK                                                                    0x00001FFFL
-#define SQ_IMG_RSRC_WORD5__ARRAY_PITCH_MASK                                                                   0x0001E000L
-#define SQ_IMG_RSRC_WORD5__META_DATA_ADDRESS_MASK                                                             0x01FE0000L
-#define SQ_IMG_RSRC_WORD5__META_LINEAR_MASK                                                                   0x02000000L
-#define SQ_IMG_RSRC_WORD5__META_PIPE_ALIGNED_MASK                                                             0x04000000L
-#define SQ_IMG_RSRC_WORD5__META_RB_ALIGNED_MASK                                                               0x08000000L
-#define SQ_IMG_RSRC_WORD5__MAX_MIP_MASK                                                                       0xF0000000L
-//SQ_IMG_RSRC_WORD6
-#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN__SHIFT                                                                0x0
-#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID__SHIFT                                                             0xc
-#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN__SHIFT                                                              0x14
-#define SQ_IMG_RSRC_WORD6__COMPRESSION_EN__SHIFT                                                              0x15
-#define SQ_IMG_RSRC_WORD6__ALPHA_IS_ON_MSB__SHIFT                                                             0x16
-#define SQ_IMG_RSRC_WORD6__COLOR_TRANSFORM__SHIFT                                                             0x17
-#define SQ_IMG_RSRC_WORD6__LOST_ALPHA_BITS__SHIFT                                                             0x18
-#define SQ_IMG_RSRC_WORD6__LOST_COLOR_BITS__SHIFT                                                             0x1c
-#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN_MASK                                                                  0x00000FFFL
-#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID_MASK                                                               0x000FF000L
-#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN_MASK                                                                0x00100000L
-#define SQ_IMG_RSRC_WORD6__COMPRESSION_EN_MASK                                                                0x00200000L
-#define SQ_IMG_RSRC_WORD6__ALPHA_IS_ON_MSB_MASK                                                               0x00400000L
-#define SQ_IMG_RSRC_WORD6__COLOR_TRANSFORM_MASK                                                               0x00800000L
-#define SQ_IMG_RSRC_WORD6__LOST_ALPHA_BITS_MASK                                                               0x0F000000L
-#define SQ_IMG_RSRC_WORD6__LOST_COLOR_BITS_MASK                                                               0xF0000000L
-//SQ_IMG_RSRC_WORD7
-#define SQ_IMG_RSRC_WORD7__META_DATA_ADDRESS__SHIFT                                                           0x0
-#define SQ_IMG_RSRC_WORD7__META_DATA_ADDRESS_MASK                                                             0xFFFFFFFFL
-//SQ_IMG_SAMP_WORD0
-#define SQ_IMG_SAMP_WORD0__CLAMP_X__SHIFT                                                                     0x0
-#define SQ_IMG_SAMP_WORD0__CLAMP_Y__SHIFT                                                                     0x3
-#define SQ_IMG_SAMP_WORD0__CLAMP_Z__SHIFT                                                                     0x6
-#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO__SHIFT                                                             0x9
-#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC__SHIFT                                                          0xc
-#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED__SHIFT                                                          0xf
-#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD__SHIFT                                                             0x10
-#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC__SHIFT                                                              0x13
-#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA__SHIFT                                                               0x14
-#define SQ_IMG_SAMP_WORD0__ANISO_BIAS__SHIFT                                                                  0x15
-#define SQ_IMG_SAMP_WORD0__TRUNC_COORD__SHIFT                                                                 0x1b
-#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP__SHIFT                                                           0x1c
-#define SQ_IMG_SAMP_WORD0__FILTER_MODE__SHIFT                                                                 0x1d
-#define SQ_IMG_SAMP_WORD0__COMPAT_MODE__SHIFT                                                                 0x1f
-#define SQ_IMG_SAMP_WORD0__CLAMP_X_MASK                                                                       0x00000007L
-#define SQ_IMG_SAMP_WORD0__CLAMP_Y_MASK                                                                       0x00000038L
-#define SQ_IMG_SAMP_WORD0__CLAMP_Z_MASK                                                                       0x000001C0L
-#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO_MASK                                                               0x00000E00L
-#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC_MASK                                                            0x00007000L
-#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED_MASK                                                            0x00008000L
-#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD_MASK                                                               0x00070000L
-#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC_MASK                                                                0x00080000L
-#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA_MASK                                                                 0x00100000L
-#define SQ_IMG_SAMP_WORD0__ANISO_BIAS_MASK                                                                    0x07E00000L
-#define SQ_IMG_SAMP_WORD0__TRUNC_COORD_MASK                                                                   0x08000000L
-#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP_MASK                                                             0x10000000L
-#define SQ_IMG_SAMP_WORD0__FILTER_MODE_MASK                                                                   0x60000000L
-#define SQ_IMG_SAMP_WORD0__COMPAT_MODE_MASK                                                                   0x80000000L
-//SQ_IMG_SAMP_WORD1
-#define SQ_IMG_SAMP_WORD1__MIN_LOD__SHIFT                                                                     0x0
-#define SQ_IMG_SAMP_WORD1__MAX_LOD__SHIFT                                                                     0xc
-#define SQ_IMG_SAMP_WORD1__PERF_MIP__SHIFT                                                                    0x18
-#define SQ_IMG_SAMP_WORD1__PERF_Z__SHIFT                                                                      0x1c
-#define SQ_IMG_SAMP_WORD1__MIN_LOD_MASK                                                                       0x00000FFFL
-#define SQ_IMG_SAMP_WORD1__MAX_LOD_MASK                                                                       0x00FFF000L
-#define SQ_IMG_SAMP_WORD1__PERF_MIP_MASK                                                                      0x0F000000L
-#define SQ_IMG_SAMP_WORD1__PERF_Z_MASK                                                                        0xF0000000L
-//SQ_IMG_SAMP_WORD2
-#define SQ_IMG_SAMP_WORD2__LOD_BIAS__SHIFT                                                                    0x0
-#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC__SHIFT                                                                0xe
-#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER__SHIFT                                                               0x14
-#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER__SHIFT                                                               0x16
-#define SQ_IMG_SAMP_WORD2__Z_FILTER__SHIFT                                                                    0x18
-#define SQ_IMG_SAMP_WORD2__MIP_FILTER__SHIFT                                                                  0x1a
-#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP__SHIFT                                                          0x1c
-#define SQ_IMG_SAMP_WORD2__BLEND_ZERO_PRT__SHIFT                                                              0x1d
-#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX__SHIFT                                                             0x1e
-#define SQ_IMG_SAMP_WORD2__ANISO_OVERRIDE__SHIFT                                                              0x1f
-#define SQ_IMG_SAMP_WORD2__LOD_BIAS_MASK                                                                      0x00003FFFL
-#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC_MASK                                                                  0x000FC000L
-#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER_MASK                                                                 0x00300000L
-#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER_MASK                                                                 0x00C00000L
-#define SQ_IMG_SAMP_WORD2__Z_FILTER_MASK                                                                      0x03000000L
-#define SQ_IMG_SAMP_WORD2__MIP_FILTER_MASK                                                                    0x0C000000L
-#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP_MASK                                                            0x10000000L
-#define SQ_IMG_SAMP_WORD2__BLEND_ZERO_PRT_MASK                                                                0x20000000L
-#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX_MASK                                                               0x40000000L
-#define SQ_IMG_SAMP_WORD2__ANISO_OVERRIDE_MASK                                                                0x80000000L
-//SQ_IMG_SAMP_WORD3
-#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR__SHIFT                                                            0x0
-#define SQ_IMG_SAMP_WORD3__SKIP_DEGAMMA__SHIFT                                                                0xc
-#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE__SHIFT                                                           0x1e
-#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR_MASK                                                              0x00000FFFL
-#define SQ_IMG_SAMP_WORD3__SKIP_DEGAMMA_MASK                                                                  0x00001000L
-#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE_MASK                                                             0xC0000000L
-//SQ_FLAT_SCRATCH_WORD0
-#define SQ_FLAT_SCRATCH_WORD0__SIZE__SHIFT                                                                    0x0
-#define SQ_FLAT_SCRATCH_WORD0__SIZE_MASK                                                                      0x0007FFFFL
-//SQ_FLAT_SCRATCH_WORD1
-#define SQ_FLAT_SCRATCH_WORD1__OFFSET__SHIFT                                                                  0x0
-#define SQ_FLAT_SCRATCH_WORD1__OFFSET_MASK                                                                    0x00FFFFFFL
-//SQ_M0_GPR_IDX_WORD
-#define SQ_M0_GPR_IDX_WORD__INDEX__SHIFT                                                                      0x0
-#define SQ_M0_GPR_IDX_WORD__VSRC0_REL__SHIFT                                                                  0xc
-#define SQ_M0_GPR_IDX_WORD__VSRC1_REL__SHIFT                                                                  0xd
-#define SQ_M0_GPR_IDX_WORD__VSRC2_REL__SHIFT                                                                  0xe
-#define SQ_M0_GPR_IDX_WORD__VDST_REL__SHIFT                                                                   0xf
-#define SQ_M0_GPR_IDX_WORD__INDEX_MASK                                                                        0x000000FFL
-#define SQ_M0_GPR_IDX_WORD__VSRC0_REL_MASK                                                                    0x00001000L
-#define SQ_M0_GPR_IDX_WORD__VSRC1_REL_MASK                                                                    0x00002000L
-#define SQ_M0_GPR_IDX_WORD__VSRC2_REL_MASK                                                                    0x00004000L
-#define SQ_M0_GPR_IDX_WORD__VDST_REL_MASK                                                                     0x00008000L
-//SQC_ICACHE_UTCL1_CNTL1
-#define SQC_ICACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                       0x0
-#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                          0x1
-#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                        0x2
-#define SQC_ICACHE_UTCL1_CNTL1__RESP_MODE__SHIFT                                                              0x3
-#define SQC_ICACHE_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                        0x5
-#define SQC_ICACHE_UTCL1_CNTL1__CLIENTID__SHIFT                                                               0x7
-#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                      0x11
-#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                   0x12
-#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID__SHIFT                                                    0x13
-#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID__SHIFT                                                0x17
-#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE__SHIFT                                                  0x18
-#define SQC_ICACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                             0x19
-#define SQC_ICACHE_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                             0x1a
-#define SQC_ICACHE_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                         0x1b
-#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                 0x1c
-#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                 0x1e
-#define SQC_ICACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                         0x00000001L
-#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                            0x00000002L
-#define SQC_ICACHE_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                          0x00000004L
-#define SQC_ICACHE_UTCL1_CNTL1__RESP_MODE_MASK                                                                0x00000018L
-#define SQC_ICACHE_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                          0x00000060L
-#define SQC_ICACHE_UTCL1_CNTL1__CLIENTID_MASK                                                                 0x0000FF80L
-#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                        0x00020000L
-#define SQC_ICACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                     0x00040000L
-#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID_MASK                                                      0x00780000L
-#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID_MASK                                                  0x00800000L
-#define SQC_ICACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE_MASK                                                    0x01000000L
-#define SQC_ICACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                               0x02000000L
-#define SQC_ICACHE_UTCL1_CNTL1__FORCE_MISS_MASK                                                               0x04000000L
-#define SQC_ICACHE_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                           0x08000000L
-#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                   0x30000000L
-#define SQC_ICACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                   0xC0000000L
-//SQC_ICACHE_UTCL1_CNTL2
-#define SQC_ICACHE_UTCL1_CNTL2__SPARE__SHIFT                                                                  0x0
-#define SQC_ICACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE__SHIFT                                                     0x8
-#define SQC_ICACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                         0x9
-#define SQC_ICACHE_UTCL1_CNTL2__LINE_VALID__SHIFT                                                             0xa
-#define SQC_ICACHE_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                0xb
-#define SQC_ICACHE_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                         0xc
-#define SQC_ICACHE_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                          0xd
-#define SQC_ICACHE_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                            0xe
-#define SQC_ICACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                    0xf
-#define SQC_ICACHE_UTCL1_CNTL2__ARB_BURST_MODE__SHIFT                                                         0x10
-#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                0x12
-#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                       0x13
-#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                 0x14
-#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_VMID__SHIFT                                                        0x15
-#define SQC_ICACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                   0x1a
-#define SQC_ICACHE_UTCL1_CNTL2__SPARE_MASK                                                                    0x000000FFL
-#define SQC_ICACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE_MASK                                                       0x00000100L
-#define SQC_ICACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                           0x00000200L
-#define SQC_ICACHE_UTCL1_CNTL2__LINE_VALID_MASK                                                               0x00000400L
-#define SQC_ICACHE_UTCL1_CNTL2__DIS_EDC_MASK                                                                  0x00000800L
-#define SQC_ICACHE_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                           0x00001000L
-#define SQC_ICACHE_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                            0x00002000L
-#define SQC_ICACHE_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                              0x00004000L
-#define SQC_ICACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                      0x00008000L
-#define SQC_ICACHE_UTCL1_CNTL2__ARB_BURST_MODE_MASK                                                           0x00030000L
-#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                  0x00040000L
-#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR_MASK                                                         0x00080000L
-#define SQC_ICACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                   0x00100000L
-#define SQC_ICACHE_UTCL1_CNTL2__PERF_EVENT_VMID_MASK                                                          0x01E00000L
-#define SQC_ICACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                     0x04000000L
-//SQC_DCACHE_UTCL1_CNTL1
-#define SQC_DCACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                       0x0
-#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                          0x1
-#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                        0x2
-#define SQC_DCACHE_UTCL1_CNTL1__RESP_MODE__SHIFT                                                              0x3
-#define SQC_DCACHE_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                        0x5
-#define SQC_DCACHE_UTCL1_CNTL1__CLIENTID__SHIFT                                                               0x7
-#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                      0x11
-#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                   0x12
-#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID__SHIFT                                                    0x13
-#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID__SHIFT                                                0x17
-#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE__SHIFT                                                  0x18
-#define SQC_DCACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                             0x19
-#define SQC_DCACHE_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                             0x1a
-#define SQC_DCACHE_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                         0x1b
-#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                 0x1c
-#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                 0x1e
-#define SQC_DCACHE_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                         0x00000001L
-#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                            0x00000002L
-#define SQC_DCACHE_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                          0x00000004L
-#define SQC_DCACHE_UTCL1_CNTL1__RESP_MODE_MASK                                                                0x00000018L
-#define SQC_DCACHE_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                          0x00000060L
-#define SQC_DCACHE_UTCL1_CNTL1__CLIENTID_MASK                                                                 0x0000FF80L
-#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                        0x00020000L
-#define SQC_DCACHE_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                     0x00040000L
-#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_VMID_MASK                                                      0x00780000L
-#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_ALL_VMID_MASK                                                  0x00800000L
-#define SQC_DCACHE_UTCL1_CNTL1__REG_INVALIDATE_TOGGLE_MASK                                                    0x01000000L
-#define SQC_DCACHE_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                               0x02000000L
-#define SQC_DCACHE_UTCL1_CNTL1__FORCE_MISS_MASK                                                               0x04000000L
-#define SQC_DCACHE_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                           0x08000000L
-#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                   0x30000000L
-#define SQC_DCACHE_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                   0xC0000000L
-//SQC_DCACHE_UTCL1_CNTL2
-#define SQC_DCACHE_UTCL1_CNTL2__SPARE__SHIFT                                                                  0x0
-#define SQC_DCACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE__SHIFT                                                     0x8
-#define SQC_DCACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                         0x9
-#define SQC_DCACHE_UTCL1_CNTL2__LINE_VALID__SHIFT                                                             0xa
-#define SQC_DCACHE_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                0xb
-#define SQC_DCACHE_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                         0xc
-#define SQC_DCACHE_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                          0xd
-#define SQC_DCACHE_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                            0xe
-#define SQC_DCACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                    0xf
-#define SQC_DCACHE_UTCL1_CNTL2__ARB_BURST_MODE__SHIFT                                                         0x10
-#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR__SHIFT                                                0x12
-#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR__SHIFT                                                       0x13
-#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID__SHIFT                                                 0x14
-#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_VMID__SHIFT                                                        0x15
-#define SQC_DCACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                   0x1a
-#define SQC_DCACHE_UTCL1_CNTL2__SPARE_MASK                                                                    0x000000FFL
-#define SQC_DCACHE_UTCL1_CNTL2__LFIFO_SCAN_DISABLE_MASK                                                       0x00000100L
-#define SQC_DCACHE_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                           0x00000200L
-#define SQC_DCACHE_UTCL1_CNTL2__LINE_VALID_MASK                                                               0x00000400L
-#define SQC_DCACHE_UTCL1_CNTL2__DIS_EDC_MASK                                                                  0x00000800L
-#define SQC_DCACHE_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                           0x00001000L
-#define SQC_DCACHE_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                            0x00002000L
-#define SQC_DCACHE_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                              0x00004000L
-#define SQC_DCACHE_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                      0x00008000L
-#define SQC_DCACHE_UTCL1_CNTL2__ARB_BURST_MODE_MASK                                                           0x00030000L
-#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_RD_WR_MASK                                                  0x00040000L
-#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_RD_WR_MASK                                                         0x00080000L
-#define SQC_DCACHE_UTCL1_CNTL2__ENABLE_PERF_EVENT_VMID_MASK                                                   0x00100000L
-#define SQC_DCACHE_UTCL1_CNTL2__PERF_EVENT_VMID_MASK                                                          0x01E00000L
-#define SQC_DCACHE_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                     0x04000000L
-//SQC_ICACHE_UTCL1_STATUS
-#define SQC_ICACHE_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                        0x0
-#define SQC_ICACHE_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                        0x1
-#define SQC_ICACHE_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                          0x2
-#define SQC_ICACHE_UTCL1_STATUS__FAULT_DETECTED_MASK                                                          0x00000001L
-#define SQC_ICACHE_UTCL1_STATUS__RETRY_DETECTED_MASK                                                          0x00000002L
-#define SQC_ICACHE_UTCL1_STATUS__PRT_DETECTED_MASK                                                            0x00000004L
-//SQC_DCACHE_UTCL1_STATUS
-#define SQC_DCACHE_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                        0x0
-#define SQC_DCACHE_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                        0x1
-#define SQC_DCACHE_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                          0x2
-#define SQC_DCACHE_UTCL1_STATUS__FAULT_DETECTED_MASK                                                          0x00000001L
-#define SQC_DCACHE_UTCL1_STATUS__RETRY_DETECTED_MASK                                                          0x00000002L
-#define SQC_DCACHE_UTCL1_STATUS__PRT_DETECTED_MASK                                                            0x00000004L
-
-
-// addressBlock: gc_shsdec
-//SX_DEBUG_1
-#define SX_DEBUG_1__SX_DB_QUAD_CREDIT__SHIFT                                                                  0x0
-#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT                                                      0x8
-#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS__SHIFT                                                           0x9
-#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT                                                    0xa
-#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT__SHIFT                                                              0xb
-#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT__SHIFT                                                            0xc
-#define SX_DEBUG_1__PC_CFG__SHIFT                                                                             0xd
-#define SX_DEBUG_1__DEBUG_DATA__SHIFT                                                                         0xe
-#define SX_DEBUG_1__SX_DB_QUAD_CREDIT_MASK                                                                    0x0000007FL
-#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST_MASK                                                        0x00000100L
-#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS_MASK                                                             0x00000200L
-#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK                                                      0x00000400L
-#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT_MASK                                                                0x00000800L
-#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT_MASK                                                              0x00001000L
-#define SX_DEBUG_1__PC_CFG_MASK                                                                               0x00002000L
-#define SX_DEBUG_1__DEBUG_DATA_MASK                                                                           0xFFFFC000L
-//SPI_PS_MAX_WAVE_ID
-#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
-#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID__SHIFT                                                      0x10
-#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
-#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID_MASK                                                        0x03FF0000L
-//SPI_START_PHASE
-#define SPI_START_PHASE__VGPR_START_PHASE__SHIFT                                                              0x0
-#define SPI_START_PHASE__SGPR_START_PHASE__SHIFT                                                              0x2
-#define SPI_START_PHASE__WAVE_START_PHASE__SHIFT                                                              0x4
-#define SPI_START_PHASE__VGPR_START_PHASE_MASK                                                                0x00000003L
-#define SPI_START_PHASE__SGPR_START_PHASE_MASK                                                                0x0000000CL
-#define SPI_START_PHASE__WAVE_START_PHASE_MASK                                                                0x00000030L
-//SPI_GFX_CNTL
-#define SPI_GFX_CNTL__RESET_COUNTS__SHIFT                                                                     0x0
-#define SPI_GFX_CNTL__RESET_COUNTS_MASK                                                                       0x00000001L
-//SPI_DSM_CNTL
-#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA__SHIFT                                                    0x0
-#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                   0x2
-#define SPI_DSM_CNTL__UNUSED__SHIFT                                                                           0x3
-#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA_MASK                                                      0x00000003L
-#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE_MASK                                                     0x00000004L
-#define SPI_DSM_CNTL__UNUSED_MASK                                                                             0xFFFFFFF8L
-//SPI_DSM_CNTL2
-#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT__SHIFT                                                  0x0
-#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY__SHIFT                                                  0x2
-#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY__SHIFT                                                         0x4
-#define SPI_DSM_CNTL2__UNUSED__SHIFT                                                                          0xa
-#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT_MASK                                                    0x00000003L
-#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY_MASK                                                    0x00000004L
-#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY_MASK                                                           0x000003F0L
-#define SPI_DSM_CNTL2__UNUSED_MASK                                                                            0xFFFFFC00L
-//SPI_EDC_CNT
-#define SPI_EDC_CNT__SPI_SR_MEM_SED_COUNT__SHIFT                                                              0x0
-#define SPI_EDC_CNT__SPI_SR_MEM_SED_COUNT_MASK                                                                0x00000003L
-//SPI_CONFIG_PS_CU_EN
-#define SPI_CONFIG_PS_CU_EN__ENABLE__SHIFT                                                                    0x0
-#define SPI_CONFIG_PS_CU_EN__PKR0_CU_EN__SHIFT                                                                0x1
-#define SPI_CONFIG_PS_CU_EN__PKR1_CU_EN__SHIFT                                                                0x10
-#define SPI_CONFIG_PS_CU_EN__ENABLE_MASK                                                                      0x00000001L
-#define SPI_CONFIG_PS_CU_EN__PKR0_CU_EN_MASK                                                                  0x0000FFFEL
-#define SPI_CONFIG_PS_CU_EN__PKR1_CU_EN_MASK                                                                  0xFFFF0000L
-//SPI_WF_LIFETIME_CNTL
-#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD__SHIFT                                                            0x0
-#define SPI_WF_LIFETIME_CNTL__EN__SHIFT                                                                       0x4
-#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD_MASK                                                              0x0000000FL
-#define SPI_WF_LIFETIME_CNTL__EN_MASK                                                                         0x00000010L
-//SPI_WF_LIFETIME_LIMIT_0
-#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_1
-#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_2
-#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_3
-#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_4
-#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_5
-#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_6
-#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_7
-#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_8
-#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_LIMIT_9
-#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT__SHIFT                                                               0x0
-#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN__SHIFT                                                               0x1f
-#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT_MASK                                                                 0x7FFFFFFFL
-#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN_MASK                                                                 0x80000000L
-//SPI_WF_LIFETIME_STATUS_0
-#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_0__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_0__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_1
-#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_1__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_1__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_2
-#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_2__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_2__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_3
-#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_3__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_3__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_4
-#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_4__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_4__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_5
-#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_5__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_5__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_6
-#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_6__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_6__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_7
-#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_7__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_7__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_8
-#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_8__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_8__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_9
-#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT__SHIFT                                                              0x0
-#define SPI_WF_LIFETIME_STATUS_9__INT_SENT__SHIFT                                                             0x1f
-#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT_MASK                                                                0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_9__INT_SENT_MASK                                                               0x80000000L
-//SPI_WF_LIFETIME_STATUS_10
-#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_10__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_10__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_11
-#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_11__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_11__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_12
-#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_12__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_12__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_13
-#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_13__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_13__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_14
-#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_14__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_14__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_15
-#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_15__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_15__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_16
-#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_16__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_16__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_17
-#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_17__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_17__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_18
-#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_18__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_18__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_19
-#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_19__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_19__INT_SENT_MASK                                                              0x80000000L
-//SPI_WF_LIFETIME_STATUS_20
-#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT__SHIFT                                                             0x0
-#define SPI_WF_LIFETIME_STATUS_20__INT_SENT__SHIFT                                                            0x1f
-#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT_MASK                                                               0x7FFFFFFFL
-#define SPI_WF_LIFETIME_STATUS_20__INT_SENT_MASK                                                              0x80000000L
-//SPI_LB_CTR_CTRL
-#define SPI_LB_CTR_CTRL__LOAD__SHIFT                                                                          0x0
-#define SPI_LB_CTR_CTRL__WAVES_SELECT__SHIFT                                                                  0x1
-#define SPI_LB_CTR_CTRL__CLEAR_ON_READ__SHIFT                                                                 0x3
-#define SPI_LB_CTR_CTRL__RESET_COUNTS__SHIFT                                                                  0x4
-#define SPI_LB_CTR_CTRL__LOAD_MASK                                                                            0x00000001L
-#define SPI_LB_CTR_CTRL__WAVES_SELECT_MASK                                                                    0x00000006L
-#define SPI_LB_CTR_CTRL__CLEAR_ON_READ_MASK                                                                   0x00000008L
-#define SPI_LB_CTR_CTRL__RESET_COUNTS_MASK                                                                    0x00000010L
-//SPI_LB_CU_MASK
-#define SPI_LB_CU_MASK__CU_MASK__SHIFT                                                                        0x0
-#define SPI_LB_CU_MASK__CU_MASK_MASK                                                                          0xFFFFL
-//SPI_LB_DATA_REG
-#define SPI_LB_DATA_REG__CNT_DATA__SHIFT                                                                      0x0
-#define SPI_LB_DATA_REG__CNT_DATA_MASK                                                                        0xFFFFFFFFL
-//SPI_PG_ENABLE_STATIC_CU_MASK
-#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK__SHIFT                                                          0x0
-#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK_MASK                                                            0xFFFFL
-//SPI_GDS_CREDITS
-#define SPI_GDS_CREDITS__DS_DATA_CREDITS__SHIFT                                                               0x0
-#define SPI_GDS_CREDITS__DS_CMD_CREDITS__SHIFT                                                                0x8
-#define SPI_GDS_CREDITS__UNUSED__SHIFT                                                                        0x10
-#define SPI_GDS_CREDITS__DS_DATA_CREDITS_MASK                                                                 0x000000FFL
-#define SPI_GDS_CREDITS__DS_CMD_CREDITS_MASK                                                                  0x0000FF00L
-#define SPI_GDS_CREDITS__UNUSED_MASK                                                                          0xFFFF0000L
-//SPI_SX_EXPORT_BUFFER_SIZES
-#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE__SHIFT                                                  0x0
-#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE__SHIFT                                               0x10
-#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE_MASK                                                    0x0000FFFFL
-#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE_MASK                                                 0xFFFF0000L
-//SPI_SX_SCOREBOARD_BUFFER_SIZES
-#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE__SHIFT                                          0x0
-#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE__SHIFT                                       0x10
-#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE_MASK                                            0x0000FFFFL
-#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE_MASK                                         0xFFFF0000L
-//SPI_CSQ_WF_ACTIVE_STATUS
-#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE_MASK                                                                 0xFFFFFFFFL
-//SPI_CSQ_WF_ACTIVE_COUNT_0
-#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS_MASK                                                                0x07FF0000L
-//SPI_CSQ_WF_ACTIVE_COUNT_1
-#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS_MASK                                                                0x07FF0000L
-//SPI_CSQ_WF_ACTIVE_COUNT_2
-#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS_MASK                                                                0x07FF0000L
-//SPI_CSQ_WF_ACTIVE_COUNT_3
-#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS_MASK                                                                0x07FF0000L
-//SPI_CSQ_WF_ACTIVE_COUNT_4
-#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_4__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_4__EVENTS_MASK                                                                0x07FF0000L
-//SPI_CSQ_WF_ACTIVE_COUNT_5
-#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_5__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_5__EVENTS_MASK                                                                0x07FF0000L
-//SPI_CSQ_WF_ACTIVE_COUNT_6
-#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_6__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_6__EVENTS_MASK                                                                0x07FF0000L
-//SPI_CSQ_WF_ACTIVE_COUNT_7
-#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT__SHIFT                                                               0x0
-#define SPI_CSQ_WF_ACTIVE_COUNT_7__EVENTS__SHIFT                                                              0x10
-#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT_MASK                                                                 0x000007FFL
-#define SPI_CSQ_WF_ACTIVE_COUNT_7__EVENTS_MASK                                                                0x07FF0000L
-//SPI_LB_DATA_WAVES
-#define SPI_LB_DATA_WAVES__COUNT0__SHIFT                                                                      0x0
-#define SPI_LB_DATA_WAVES__COUNT1__SHIFT                                                                      0x10
-#define SPI_LB_DATA_WAVES__COUNT0_MASK                                                                        0x0000FFFFL
-#define SPI_LB_DATA_WAVES__COUNT1_MASK                                                                        0xFFFF0000L
-//SPI_LB_DATA_PERCU_WAVE_HSGS
-#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_HS__SHIFT                                                        0x0
-#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_GS__SHIFT                                                        0x10
-#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_HS_MASK                                                          0x0000FFFFL
-#define SPI_LB_DATA_PERCU_WAVE_HSGS__CU_USED_GS_MASK                                                          0xFFFF0000L
-//SPI_LB_DATA_PERCU_WAVE_VSPS
-#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_VS__SHIFT                                                        0x0
-#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_PS__SHIFT                                                        0x10
-#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_VS_MASK                                                          0x0000FFFFL
-#define SPI_LB_DATA_PERCU_WAVE_VSPS__CU_USED_PS_MASK                                                          0xFFFF0000L
-//SPI_LB_DATA_PERCU_WAVE_CS
-#define SPI_LB_DATA_PERCU_WAVE_CS__ACTIVE__SHIFT                                                              0x0
-#define SPI_LB_DATA_PERCU_WAVE_CS__ACTIVE_MASK                                                                0xFFFFL
-//SPI_P0_TRAP_SCREEN_PSBA_LO
-#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
-//SPI_P0_TRAP_SCREEN_PSBA_HI
-#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0xFFL
-//SPI_P0_TRAP_SCREEN_PSMA_LO
-#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
-//SPI_P0_TRAP_SCREEN_PSMA_HI
-#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0xFFL
-//SPI_P0_TRAP_SCREEN_GPR_MIN
-#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
-#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
-#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x003FL
-#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x03C0L
-//SPI_P1_TRAP_SCREEN_PSBA_LO
-#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
-//SPI_P1_TRAP_SCREEN_PSBA_HI
-#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0xFFL
-//SPI_P1_TRAP_SCREEN_PSMA_LO
-#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
-//SPI_P1_TRAP_SCREEN_PSMA_HI
-#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
-#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0xFFL
-//SPI_P1_TRAP_SCREEN_GPR_MIN
-#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
-#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
-#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x003FL
-#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x03C0L
-
-
-// addressBlock: gc_tpdec
-//TD_CNTL
-#define TD_CNTL__SYNC_PHASE_SH__SHIFT                                                                         0x0
-#define TD_CNTL__SYNC_PHASE_VC_SMX__SHIFT                                                                     0x4
-#define TD_CNTL__PAD_STALL_EN__SHIFT                                                                          0x8
-#define TD_CNTL__EXTEND_LDS_STALL__SHIFT                                                                      0x9
-#define TD_CNTL__LDS_STALL_PHASE_ADJUST__SHIFT                                                                0xb
-#define TD_CNTL__PRECISION_COMPATIBILITY__SHIFT                                                               0xf
-#define TD_CNTL__GATHER4_FLOAT_MODE__SHIFT                                                                    0x10
-#define TD_CNTL__LD_FLOAT_MODE__SHIFT                                                                         0x12
-#define TD_CNTL__GATHER4_DX9_MODE__SHIFT                                                                      0x13
-#define TD_CNTL__DISABLE_POWER_THROTTLE__SHIFT                                                                0x14
-#define TD_CNTL__ENABLE_ROUND_TO_ZERO__SHIFT                                                                  0x15
-#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT__SHIFT                                                            0x17
-#define TD_CNTL__DISABLE_MM_QNAN_COMPARE_RESULT__SHIFT                                                        0x18
-#define TD_CNTL__SYNC_PHASE_SH_MASK                                                                           0x00000003L
-#define TD_CNTL__SYNC_PHASE_VC_SMX_MASK                                                                       0x00000030L
-#define TD_CNTL__PAD_STALL_EN_MASK                                                                            0x00000100L
-#define TD_CNTL__EXTEND_LDS_STALL_MASK                                                                        0x00000600L
-#define TD_CNTL__LDS_STALL_PHASE_ADJUST_MASK                                                                  0x00001800L
-#define TD_CNTL__PRECISION_COMPATIBILITY_MASK                                                                 0x00008000L
-#define TD_CNTL__GATHER4_FLOAT_MODE_MASK                                                                      0x00010000L
-#define TD_CNTL__LD_FLOAT_MODE_MASK                                                                           0x00040000L
-#define TD_CNTL__GATHER4_DX9_MODE_MASK                                                                        0x00080000L
-#define TD_CNTL__DISABLE_POWER_THROTTLE_MASK                                                                  0x00100000L
-#define TD_CNTL__ENABLE_ROUND_TO_ZERO_MASK                                                                    0x00200000L
-#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT_MASK                                                              0x00800000L
-#define TD_CNTL__DISABLE_MM_QNAN_COMPARE_RESULT_MASK                                                          0x01000000L
-//TD_STATUS
-#define TD_STATUS__BUSY__SHIFT                                                                                0x1f
-#define TD_STATUS__BUSY_MASK                                                                                  0x80000000L
-//TD_DSM_CNTL
-#define TD_DSM_CNTL__TD_SS_FIFO_LO_DSM_IRRITATOR_DATA__SHIFT                                                  0x0
-#define TD_DSM_CNTL__TD_SS_FIFO_LO_ENABLE_SINGLE_WRITE__SHIFT                                                 0x2
-#define TD_DSM_CNTL__TD_SS_FIFO_HI_DSM_IRRITATOR_DATA__SHIFT                                                  0x3
-#define TD_DSM_CNTL__TD_SS_FIFO_HI_ENABLE_SINGLE_WRITE__SHIFT                                                 0x5
-#define TD_DSM_CNTL__TD_CS_FIFO_DSM_IRRITATOR_DATA__SHIFT                                                     0x6
-#define TD_DSM_CNTL__TD_CS_FIFO_ENABLE_SINGLE_WRITE__SHIFT                                                    0x8
-#define TD_DSM_CNTL__TD_SS_FIFO_LO_DSM_IRRITATOR_DATA_MASK                                                    0x00000003L
-#define TD_DSM_CNTL__TD_SS_FIFO_LO_ENABLE_SINGLE_WRITE_MASK                                                   0x00000004L
-#define TD_DSM_CNTL__TD_SS_FIFO_HI_DSM_IRRITATOR_DATA_MASK                                                    0x00000018L
-#define TD_DSM_CNTL__TD_SS_FIFO_HI_ENABLE_SINGLE_WRITE_MASK                                                   0x00000020L
-#define TD_DSM_CNTL__TD_CS_FIFO_DSM_IRRITATOR_DATA_MASK                                                       0x000000C0L
-#define TD_DSM_CNTL__TD_CS_FIFO_ENABLE_SINGLE_WRITE_MASK                                                      0x00000100L
-//TD_DSM_CNTL2
-#define TD_DSM_CNTL2__TD_SS_FIFO_LO_ENABLE_ERROR_INJECT__SHIFT                                                0x0
-#define TD_DSM_CNTL2__TD_SS_FIFO_LO_SELECT_INJECT_DELAY__SHIFT                                                0x2
-#define TD_DSM_CNTL2__TD_SS_FIFO_HI_ENABLE_ERROR_INJECT__SHIFT                                                0x3
-#define TD_DSM_CNTL2__TD_SS_FIFO_HI_SELECT_INJECT_DELAY__SHIFT                                                0x5
-#define TD_DSM_CNTL2__TD_CS_FIFO_ENABLE_ERROR_INJECT__SHIFT                                                   0x6
-#define TD_DSM_CNTL2__TD_CS_FIFO_SELECT_INJECT_DELAY__SHIFT                                                   0x8
-#define TD_DSM_CNTL2__TD_INJECT_DELAY__SHIFT                                                                  0x1a
-#define TD_DSM_CNTL2__TD_SS_FIFO_LO_ENABLE_ERROR_INJECT_MASK                                                  0x00000003L
-#define TD_DSM_CNTL2__TD_SS_FIFO_LO_SELECT_INJECT_DELAY_MASK                                                  0x00000004L
-#define TD_DSM_CNTL2__TD_SS_FIFO_HI_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
-#define TD_DSM_CNTL2__TD_SS_FIFO_HI_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
-#define TD_DSM_CNTL2__TD_CS_FIFO_ENABLE_ERROR_INJECT_MASK                                                     0x000000C0L
-#define TD_DSM_CNTL2__TD_CS_FIFO_SELECT_INJECT_DELAY_MASK                                                     0x00000100L
-#define TD_DSM_CNTL2__TD_INJECT_DELAY_MASK                                                                    0xFC000000L
-//TD_SCRATCH
-#define TD_SCRATCH__SCRATCH__SHIFT                                                                            0x0
-#define TD_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
-//TA_CNTL
-#define TA_CNTL__FX_XNACK_CREDIT__SHIFT                                                                       0x0
-#define TA_CNTL__SQ_XNACK_CREDIT__SHIFT                                                                       0x9
-#define TA_CNTL__TC_DATA_CREDIT__SHIFT                                                                        0xd
-#define TA_CNTL__ALIGNER_CREDIT__SHIFT                                                                        0x10
-#define TA_CNTL__TD_FIFO_CREDIT__SHIFT                                                                        0x16
-#define TA_CNTL__FX_XNACK_CREDIT_MASK                                                                         0x0000007FL
-#define TA_CNTL__SQ_XNACK_CREDIT_MASK                                                                         0x00001E00L
-#define TA_CNTL__TC_DATA_CREDIT_MASK                                                                          0x0000E000L
-#define TA_CNTL__ALIGNER_CREDIT_MASK                                                                          0x001F0000L
-#define TA_CNTL__TD_FIFO_CREDIT_MASK                                                                          0xFFC00000L
-//TA_CNTL_AUX
-#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N__SHIFT                                                                  0x0
-#define TA_CNTL_AUX__RESERVED__SHIFT                                                                          0x1
-#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE__SHIFT                                                                0x5
-#define TA_CNTL_AUX__GATHERH_DST_SEL__SHIFT                                                                   0x6
-#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE__SHIFT                                                        0x7
-#define TA_CNTL_AUX__NONIMG_ANISO_BYPASS__SHIFT                                                               0x9
-#define TA_CNTL_AUX__ANISO_HALF_THRESH__SHIFT                                                                 0xa
-#define TA_CNTL_AUX__ANISO_ERROR_FP_VBIAS__SHIFT                                                              0xc
-#define TA_CNTL_AUX__ANISO_STEP_ORDER__SHIFT                                                                  0xd
-#define TA_CNTL_AUX__ANISO_STEP__SHIFT                                                                        0xe
-#define TA_CNTL_AUX__MINMAG_UNNORM__SHIFT                                                                     0xf
-#define TA_CNTL_AUX__ANISO_WEIGHT_MODE__SHIFT                                                                 0x10
-#define TA_CNTL_AUX__ANISO_RATIO_LUT__SHIFT                                                                   0x11
-#define TA_CNTL_AUX__ANISO_TAP__SHIFT                                                                         0x12
-#define TA_CNTL_AUX__ANISO_MIP_ADJ_MODE__SHIFT                                                                0x13
-#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE__SHIFT                                                      0x14
-#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE__SHIFT                                                 0x15
-#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE__SHIFT                                                          0x16
-#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE__SHIFT                                                 0x17
-#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE__SHIFT                                                  0x18
-#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE__SHIFT                                               0x19
-#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE__SHIFT                                                     0x1a
-#define TA_CNTL_AUX__DISABLE_DWORD_X2_COALESCE__SHIFT                                                         0x1b
-#define TA_CNTL_AUX__CUBEMAP_SLICE_CLAMP__SHIFT                                                               0x1c
-#define TA_CNTL_AUX__TRUNC_SMALL_NEG__SHIFT                                                                   0x1d
-#define TA_CNTL_AUX__ARRAY_ROUND_MODE__SHIFT                                                                  0x1e
-#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N_MASK                                                                    0x00000001L
-#define TA_CNTL_AUX__RESERVED_MASK                                                                            0x0000000EL
-#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE_MASK                                                                  0x00000020L
-#define TA_CNTL_AUX__GATHERH_DST_SEL_MASK                                                                     0x00000040L
-#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE_MASK                                                          0x00000080L
-#define TA_CNTL_AUX__NONIMG_ANISO_BYPASS_MASK                                                                 0x00000200L
-#define TA_CNTL_AUX__ANISO_HALF_THRESH_MASK                                                                   0x00000C00L
-#define TA_CNTL_AUX__ANISO_ERROR_FP_VBIAS_MASK                                                                0x00001000L
-#define TA_CNTL_AUX__ANISO_STEP_ORDER_MASK                                                                    0x00002000L
-#define TA_CNTL_AUX__ANISO_STEP_MASK                                                                          0x00004000L
-#define TA_CNTL_AUX__MINMAG_UNNORM_MASK                                                                       0x00008000L
-#define TA_CNTL_AUX__ANISO_WEIGHT_MODE_MASK                                                                   0x00010000L
-#define TA_CNTL_AUX__ANISO_RATIO_LUT_MASK                                                                     0x00020000L
-#define TA_CNTL_AUX__ANISO_TAP_MASK                                                                           0x00040000L
-#define TA_CNTL_AUX__ANISO_MIP_ADJ_MODE_MASK                                                                  0x00080000L
-#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE_MASK                                                        0x00100000L
-#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE_MASK                                                   0x00200000L
-#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE_MASK                                                            0x00400000L
-#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE_MASK                                                   0x00800000L
-#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE_MASK                                                    0x01000000L
-#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE_MASK                                                 0x02000000L
-#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE_MASK                                                       0x04000000L
-#define TA_CNTL_AUX__DISABLE_DWORD_X2_COALESCE_MASK                                                           0x08000000L
-#define TA_CNTL_AUX__CUBEMAP_SLICE_CLAMP_MASK                                                                 0x10000000L
-#define TA_CNTL_AUX__TRUNC_SMALL_NEG_MASK                                                                     0x20000000L
-#define TA_CNTL_AUX__ARRAY_ROUND_MODE_MASK                                                                    0xC0000000L
-//TA_RESERVED_010C
-#define TA_RESERVED_010C__Unused__SHIFT                                                                       0x0
-#define TA_RESERVED_010C__Unused_MASK                                                                         0xFFFFFFFFL
-//TA_GRAD_ADJ
-#define TA_GRAD_ADJ__GRAD_ADJ_0__SHIFT                                                                        0x0
-#define TA_GRAD_ADJ__GRAD_ADJ_1__SHIFT                                                                        0x8
-#define TA_GRAD_ADJ__GRAD_ADJ_2__SHIFT                                                                        0x10
-#define TA_GRAD_ADJ__GRAD_ADJ_3__SHIFT                                                                        0x18
-#define TA_GRAD_ADJ__GRAD_ADJ_0_MASK                                                                          0x000000FFL
-#define TA_GRAD_ADJ__GRAD_ADJ_1_MASK                                                                          0x0000FF00L
-#define TA_GRAD_ADJ__GRAD_ADJ_2_MASK                                                                          0x00FF0000L
-#define TA_GRAD_ADJ__GRAD_ADJ_3_MASK                                                                          0xFF000000L
-//TA_STATUS
-#define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT                                                                     0xc
-#define TA_STATUS__FG_LFIFO_EMPTYB__SHIFT                                                                     0xd
-#define TA_STATUS__FG_SFIFO_EMPTYB__SHIFT                                                                     0xe
-#define TA_STATUS__FL_PFIFO_EMPTYB__SHIFT                                                                     0x10
-#define TA_STATUS__FL_LFIFO_EMPTYB__SHIFT                                                                     0x11
-#define TA_STATUS__FL_SFIFO_EMPTYB__SHIFT                                                                     0x12
-#define TA_STATUS__FA_PFIFO_EMPTYB__SHIFT                                                                     0x14
-#define TA_STATUS__FA_LFIFO_EMPTYB__SHIFT                                                                     0x15
-#define TA_STATUS__FA_SFIFO_EMPTYB__SHIFT                                                                     0x16
-#define TA_STATUS__IN_BUSY__SHIFT                                                                             0x18
-#define TA_STATUS__FG_BUSY__SHIFT                                                                             0x19
-#define TA_STATUS__LA_BUSY__SHIFT                                                                             0x1a
-#define TA_STATUS__FL_BUSY__SHIFT                                                                             0x1b
-#define TA_STATUS__TA_BUSY__SHIFT                                                                             0x1c
-#define TA_STATUS__FA_BUSY__SHIFT                                                                             0x1d
-#define TA_STATUS__AL_BUSY__SHIFT                                                                             0x1e
-#define TA_STATUS__BUSY__SHIFT                                                                                0x1f
-#define TA_STATUS__FG_PFIFO_EMPTYB_MASK                                                                       0x00001000L
-#define TA_STATUS__FG_LFIFO_EMPTYB_MASK                                                                       0x00002000L
-#define TA_STATUS__FG_SFIFO_EMPTYB_MASK                                                                       0x00004000L
-#define TA_STATUS__FL_PFIFO_EMPTYB_MASK                                                                       0x00010000L
-#define TA_STATUS__FL_LFIFO_EMPTYB_MASK                                                                       0x00020000L
-#define TA_STATUS__FL_SFIFO_EMPTYB_MASK                                                                       0x00040000L
-#define TA_STATUS__FA_PFIFO_EMPTYB_MASK                                                                       0x00100000L
-#define TA_STATUS__FA_LFIFO_EMPTYB_MASK                                                                       0x00200000L
-#define TA_STATUS__FA_SFIFO_EMPTYB_MASK                                                                       0x00400000L
-#define TA_STATUS__IN_BUSY_MASK                                                                               0x01000000L
-#define TA_STATUS__FG_BUSY_MASK                                                                               0x02000000L
-#define TA_STATUS__LA_BUSY_MASK                                                                               0x04000000L
-#define TA_STATUS__FL_BUSY_MASK                                                                               0x08000000L
-#define TA_STATUS__TA_BUSY_MASK                                                                               0x10000000L
-#define TA_STATUS__FA_BUSY_MASK                                                                               0x20000000L
-#define TA_STATUS__AL_BUSY_MASK                                                                               0x40000000L
-#define TA_STATUS__BUSY_MASK                                                                                  0x80000000L
-//TA_SCRATCH
-#define TA_SCRATCH__SCRATCH__SHIFT                                                                            0x0
-#define TA_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
-
-
-// addressBlock: gc_gdsdec
-//GDS_CONFIG
-#define GDS_CONFIG__SH0_GPR_PHASE_SEL__SHIFT                                                                  0x1
-#define GDS_CONFIG__SH1_GPR_PHASE_SEL__SHIFT                                                                  0x3
-#define GDS_CONFIG__SH2_GPR_PHASE_SEL__SHIFT                                                                  0x5
-#define GDS_CONFIG__SH3_GPR_PHASE_SEL__SHIFT                                                                  0x7
-#define GDS_CONFIG__SH0_GPR_PHASE_SEL_MASK                                                                    0x00000006L
-#define GDS_CONFIG__SH1_GPR_PHASE_SEL_MASK                                                                    0x00000018L
-#define GDS_CONFIG__SH2_GPR_PHASE_SEL_MASK                                                                    0x00000060L
-#define GDS_CONFIG__SH3_GPR_PHASE_SEL_MASK                                                                    0x00000180L
-//GDS_CNTL_STATUS
-#define GDS_CNTL_STATUS__GDS_BUSY__SHIFT                                                                      0x0
-#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY__SHIFT                                                                0x1
-#define GDS_CNTL_STATUS__ORD_APP_BUSY__SHIFT                                                                  0x2
-#define GDS_CNTL_STATUS__DS_BANK_CONFLICT__SHIFT                                                              0x3
-#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT__SHIFT                                                              0x4
-#define GDS_CNTL_STATUS__DS_WR_CLAMP__SHIFT                                                                   0x5
-#define GDS_CNTL_STATUS__DS_RD_CLAMP__SHIFT                                                                   0x6
-#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY__SHIFT                                                                0x7
-#define GDS_CNTL_STATUS__DS_BUSY__SHIFT                                                                       0x8
-#define GDS_CNTL_STATUS__GWS_BUSY__SHIFT                                                                      0x9
-#define GDS_CNTL_STATUS__ORD_FIFO_BUSY__SHIFT                                                                 0xa
-#define GDS_CNTL_STATUS__CREDIT_BUSY0__SHIFT                                                                  0xb
-#define GDS_CNTL_STATUS__CREDIT_BUSY1__SHIFT                                                                  0xc
-#define GDS_CNTL_STATUS__CREDIT_BUSY2__SHIFT                                                                  0xd
-#define GDS_CNTL_STATUS__CREDIT_BUSY3__SHIFT                                                                  0xe
-#define GDS_CNTL_STATUS__GDS_BUSY_MASK                                                                        0x00000001L
-#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY_MASK                                                                  0x00000002L
-#define GDS_CNTL_STATUS__ORD_APP_BUSY_MASK                                                                    0x00000004L
-#define GDS_CNTL_STATUS__DS_BANK_CONFLICT_MASK                                                                0x00000008L
-#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT_MASK                                                                0x00000010L
-#define GDS_CNTL_STATUS__DS_WR_CLAMP_MASK                                                                     0x00000020L
-#define GDS_CNTL_STATUS__DS_RD_CLAMP_MASK                                                                     0x00000040L
-#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY_MASK                                                                  0x00000080L
-#define GDS_CNTL_STATUS__DS_BUSY_MASK                                                                         0x00000100L
-#define GDS_CNTL_STATUS__GWS_BUSY_MASK                                                                        0x00000200L
-#define GDS_CNTL_STATUS__ORD_FIFO_BUSY_MASK                                                                   0x00000400L
-#define GDS_CNTL_STATUS__CREDIT_BUSY0_MASK                                                                    0x00000800L
-#define GDS_CNTL_STATUS__CREDIT_BUSY1_MASK                                                                    0x00001000L
-#define GDS_CNTL_STATUS__CREDIT_BUSY2_MASK                                                                    0x00002000L
-#define GDS_CNTL_STATUS__CREDIT_BUSY3_MASK                                                                    0x00004000L
-//GDS_ENHANCE2
-#define GDS_ENHANCE2__MISC__SHIFT                                                                             0x0
-#define GDS_ENHANCE2__UNUSED__SHIFT                                                                           0x10
-#define GDS_ENHANCE2__MISC_MASK                                                                               0x0000FFFFL
-#define GDS_ENHANCE2__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_PROTECTION_FAULT
-#define GDS_PROTECTION_FAULT__WRITE_DIS__SHIFT                                                                0x0
-#define GDS_PROTECTION_FAULT__FAULT_DETECTED__SHIFT                                                           0x1
-#define GDS_PROTECTION_FAULT__GRBM__SHIFT                                                                     0x2
-#define GDS_PROTECTION_FAULT__SH_ID__SHIFT                                                                    0x3
-#define GDS_PROTECTION_FAULT__CU_ID__SHIFT                                                                    0x6
-#define GDS_PROTECTION_FAULT__SIMD_ID__SHIFT                                                                  0xa
-#define GDS_PROTECTION_FAULT__WAVE_ID__SHIFT                                                                  0xc
-#define GDS_PROTECTION_FAULT__ADDRESS__SHIFT                                                                  0x10
-#define GDS_PROTECTION_FAULT__WRITE_DIS_MASK                                                                  0x00000001L
-#define GDS_PROTECTION_FAULT__FAULT_DETECTED_MASK                                                             0x00000002L
-#define GDS_PROTECTION_FAULT__GRBM_MASK                                                                       0x00000004L
-#define GDS_PROTECTION_FAULT__SH_ID_MASK                                                                      0x00000038L
-#define GDS_PROTECTION_FAULT__CU_ID_MASK                                                                      0x000003C0L
-#define GDS_PROTECTION_FAULT__SIMD_ID_MASK                                                                    0x00000C00L
-#define GDS_PROTECTION_FAULT__WAVE_ID_MASK                                                                    0x0000F000L
-#define GDS_PROTECTION_FAULT__ADDRESS_MASK                                                                    0xFFFF0000L
-//GDS_VM_PROTECTION_FAULT
-#define GDS_VM_PROTECTION_FAULT__WRITE_DIS__SHIFT                                                             0x0
-#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED__SHIFT                                                        0x1
-#define GDS_VM_PROTECTION_FAULT__GWS__SHIFT                                                                   0x2
-#define GDS_VM_PROTECTION_FAULT__OA__SHIFT                                                                    0x3
-#define GDS_VM_PROTECTION_FAULT__GRBM__SHIFT                                                                  0x4
-#define GDS_VM_PROTECTION_FAULT__TMZ__SHIFT                                                                   0x5
-#define GDS_VM_PROTECTION_FAULT__VMID__SHIFT                                                                  0x8
-#define GDS_VM_PROTECTION_FAULT__ADDRESS__SHIFT                                                               0x10
-#define GDS_VM_PROTECTION_FAULT__WRITE_DIS_MASK                                                               0x00000001L
-#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED_MASK                                                          0x00000002L
-#define GDS_VM_PROTECTION_FAULT__GWS_MASK                                                                     0x00000004L
-#define GDS_VM_PROTECTION_FAULT__OA_MASK                                                                      0x00000008L
-#define GDS_VM_PROTECTION_FAULT__GRBM_MASK                                                                    0x00000010L
-#define GDS_VM_PROTECTION_FAULT__TMZ_MASK                                                                     0x00000020L
-#define GDS_VM_PROTECTION_FAULT__VMID_MASK                                                                    0x00000F00L
-#define GDS_VM_PROTECTION_FAULT__ADDRESS_MASK                                                                 0xFFFF0000L
-//GDS_EDC_CNT
-#define GDS_EDC_CNT__GDS_MEM_DED__SHIFT                                                                       0x0
-#define GDS_EDC_CNT__GDS_INPUT_QUEUE_SED__SHIFT                                                               0x2
-#define GDS_EDC_CNT__GDS_MEM_SEC__SHIFT                                                                       0x4
-#define GDS_EDC_CNT__UNUSED__SHIFT                                                                            0x6
-#define GDS_EDC_CNT__GDS_MEM_DED_MASK                                                                         0x00000003L
-#define GDS_EDC_CNT__GDS_INPUT_QUEUE_SED_MASK                                                                 0x0000000CL
-#define GDS_EDC_CNT__GDS_MEM_SEC_MASK                                                                         0x00000030L
-#define GDS_EDC_CNT__UNUSED_MASK                                                                              0xFFFFFFC0L
-//GDS_EDC_GRBM_CNT
-#define GDS_EDC_GRBM_CNT__DED__SHIFT                                                                          0x0
-#define GDS_EDC_GRBM_CNT__SEC__SHIFT                                                                          0x2
-#define GDS_EDC_GRBM_CNT__UNUSED__SHIFT                                                                       0x4
-#define GDS_EDC_GRBM_CNT__DED_MASK                                                                            0x00000003L
-#define GDS_EDC_GRBM_CNT__SEC_MASK                                                                            0x0000000CL
-#define GDS_EDC_GRBM_CNT__UNUSED_MASK                                                                         0xFFFFFFF0L
-//GDS_EDC_OA_DED
-#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED__SHIFT                                                            0x0
-#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED__SHIFT                                                            0x1
-#define GDS_EDC_OA_DED__ME0_CS_DED__SHIFT                                                                     0x2
-#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED__SHIFT                                                             0x3
-#define GDS_EDC_OA_DED__ME1_PIPE0_DED__SHIFT                                                                  0x4
-#define GDS_EDC_OA_DED__ME1_PIPE1_DED__SHIFT                                                                  0x5
-#define GDS_EDC_OA_DED__ME1_PIPE2_DED__SHIFT                                                                  0x6
-#define GDS_EDC_OA_DED__ME1_PIPE3_DED__SHIFT                                                                  0x7
-#define GDS_EDC_OA_DED__ME2_PIPE0_DED__SHIFT                                                                  0x8
-#define GDS_EDC_OA_DED__ME2_PIPE1_DED__SHIFT                                                                  0x9
-#define GDS_EDC_OA_DED__ME2_PIPE2_DED__SHIFT                                                                  0xa
-#define GDS_EDC_OA_DED__ME2_PIPE3_DED__SHIFT                                                                  0xb
-#define GDS_EDC_OA_DED__UNUSED1__SHIFT                                                                        0xc
-#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED_MASK                                                              0x00000001L
-#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED_MASK                                                              0x00000002L
-#define GDS_EDC_OA_DED__ME0_CS_DED_MASK                                                                       0x00000004L
-#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED_MASK                                                               0x00000008L
-#define GDS_EDC_OA_DED__ME1_PIPE0_DED_MASK                                                                    0x00000010L
-#define GDS_EDC_OA_DED__ME1_PIPE1_DED_MASK                                                                    0x00000020L
-#define GDS_EDC_OA_DED__ME1_PIPE2_DED_MASK                                                                    0x00000040L
-#define GDS_EDC_OA_DED__ME1_PIPE3_DED_MASK                                                                    0x00000080L
-#define GDS_EDC_OA_DED__ME2_PIPE0_DED_MASK                                                                    0x00000100L
-#define GDS_EDC_OA_DED__ME2_PIPE1_DED_MASK                                                                    0x00000200L
-#define GDS_EDC_OA_DED__ME2_PIPE2_DED_MASK                                                                    0x00000400L
-#define GDS_EDC_OA_DED__ME2_PIPE3_DED_MASK                                                                    0x00000800L
-#define GDS_EDC_OA_DED__UNUSED1_MASK                                                                          0xFFFFF000L
-//GDS_DSM_CNTL
-#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_0__SHIFT                                                 0x0
-#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_1__SHIFT                                                 0x1
-#define GDS_DSM_CNTL__GDS_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                      0x2
-#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_0__SHIFT                                         0x3
-#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_1__SHIFT                                         0x4
-#define GDS_DSM_CNTL__GDS_INPUT_QUEUE_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_0__SHIFT                                         0x6
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_1__SHIFT                                         0x7
-#define GDS_DSM_CNTL__GDS_PHY_CMD_RAM_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_0__SHIFT                                        0x9
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_1__SHIFT                                        0xa
-#define GDS_DSM_CNTL__GDS_PHY_DATA_RAM_ENABLE_SINGLE_WRITE__SHIFT                                             0xb
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_0__SHIFT                                            0xc
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_1__SHIFT                                            0xd
-#define GDS_DSM_CNTL__GDS_PIPE_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xe
-#define GDS_DSM_CNTL__UNUSED__SHIFT                                                                           0xf
-#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_0_MASK                                                   0x00000001L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_1_MASK                                                   0x00000002L
-#define GDS_DSM_CNTL__GDS_MEM_ENABLE_SINGLE_WRITE_MASK                                                        0x00000004L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_0_MASK                                           0x00000008L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_1_MASK                                           0x00000010L
-#define GDS_DSM_CNTL__GDS_INPUT_QUEUE_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_0_MASK                                           0x00000040L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_1_MASK                                           0x00000080L
-#define GDS_DSM_CNTL__GDS_PHY_CMD_RAM_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_0_MASK                                          0x00000200L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_1_MASK                                          0x00000400L
-#define GDS_DSM_CNTL__GDS_PHY_DATA_RAM_ENABLE_SINGLE_WRITE_MASK                                               0x00000800L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_0_MASK                                              0x00001000L
-#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_1_MASK                                              0x00002000L
-#define GDS_DSM_CNTL__GDS_PIPE_MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00004000L
-#define GDS_DSM_CNTL__UNUSED_MASK                                                                             0xFFFF8000L
-//GDS_EDC_OA_PHY_CNT
-#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC__SHIFT                                                        0x0
-#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED__SHIFT                                                        0x2
-#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC__SHIFT                                                        0x4
-#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED__SHIFT                                                        0x6
-#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SED__SHIFT                                                       0x8
-#define GDS_EDC_OA_PHY_CNT__UNUSED1__SHIFT                                                                    0xa
-#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC_MASK                                                          0x00000003L
-#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED_MASK                                                          0x0000000CL
-#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC_MASK                                                          0x00000030L
-#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED_MASK                                                          0x000000C0L
-#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SED_MASK                                                         0x00000300L
-#define GDS_EDC_OA_PHY_CNT__UNUSED1_MASK                                                                      0xFFFFFC00L
-//GDS_EDC_OA_PIPE_CNT
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC__SHIFT                                                    0x0
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED__SHIFT                                                    0x2
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC__SHIFT                                                    0x4
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED__SHIFT                                                    0x6
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC__SHIFT                                                    0x8
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED__SHIFT                                                    0xa
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC__SHIFT                                                    0xc
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED__SHIFT                                                    0xe
-#define GDS_EDC_OA_PIPE_CNT__UNUSED__SHIFT                                                                    0x10
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC_MASK                                                      0x00000003L
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED_MASK                                                      0x0000000CL
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC_MASK                                                      0x00000030L
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED_MASK                                                      0x000000C0L
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC_MASK                                                      0x00000300L
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED_MASK                                                      0x00000C00L
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC_MASK                                                      0x00003000L
-#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED_MASK                                                      0x0000C000L
-#define GDS_EDC_OA_PIPE_CNT__UNUSED_MASK                                                                      0xFFFF0000L
-//GDS_DSM_CNTL2
-#define GDS_DSM_CNTL2__GDS_MEM_ENABLE_ERROR_INJECT__SHIFT                                                     0x0
-#define GDS_DSM_CNTL2__GDS_MEM_SELECT_INJECT_DELAY__SHIFT                                                     0x2
-#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_ENABLE_ERROR_INJECT__SHIFT                                             0x3
-#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_SELECT_INJECT_DELAY__SHIFT                                             0x5
-#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_ENABLE_ERROR_INJECT__SHIFT                                             0x6
-#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_SELECT_INJECT_DELAY__SHIFT                                             0x8
-#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_ENABLE_ERROR_INJECT__SHIFT                                            0x9
-#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_SELECT_INJECT_DELAY__SHIFT                                            0xb
-#define GDS_DSM_CNTL2__GDS_PIPE_MEM_ENABLE_ERROR_INJECT__SHIFT                                                0xc
-#define GDS_DSM_CNTL2__GDS_PIPE_MEM_SELECT_INJECT_DELAY__SHIFT                                                0xe
-#define GDS_DSM_CNTL2__UNUSED__SHIFT                                                                          0xf
-#define GDS_DSM_CNTL2__GDS_INJECT_DELAY__SHIFT                                                                0x1a
-#define GDS_DSM_CNTL2__GDS_MEM_ENABLE_ERROR_INJECT_MASK                                                       0x00000003L
-#define GDS_DSM_CNTL2__GDS_MEM_SELECT_INJECT_DELAY_MASK                                                       0x00000004L
-#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
-#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_SELECT_INJECT_DELAY_MASK                                               0x00000020L
-#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
-#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_SELECT_INJECT_DELAY_MASK                                               0x00000100L
-#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_ENABLE_ERROR_INJECT_MASK                                              0x00000600L
-#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_SELECT_INJECT_DELAY_MASK                                              0x00000800L
-#define GDS_DSM_CNTL2__GDS_PIPE_MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00003000L
-#define GDS_DSM_CNTL2__GDS_PIPE_MEM_SELECT_INJECT_DELAY_MASK                                                  0x00004000L
-#define GDS_DSM_CNTL2__UNUSED_MASK                                                                            0x03FF8000L
-#define GDS_DSM_CNTL2__GDS_INJECT_DELAY_MASK                                                                  0xFC000000L
-//GDS_WD_GDS_CSB
-#define GDS_WD_GDS_CSB__COUNTER__SHIFT                                                                        0x0
-#define GDS_WD_GDS_CSB__UNUSED__SHIFT                                                                         0xd
-#define GDS_WD_GDS_CSB__COUNTER_MASK                                                                          0x00001FFFL
-#define GDS_WD_GDS_CSB__UNUSED_MASK                                                                           0xFFFFE000L
-
-
-// addressBlock: gc_rbdec
-//DB_DEBUG
-#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE__SHIFT                                                       0x0
-#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE__SHIFT                                                         0x1
-#define DB_DEBUG__FETCH_FULL_Z_TILE__SHIFT                                                                    0x2
-#define DB_DEBUG__FETCH_FULL_STENCIL_TILE__SHIFT                                                              0x3
-#define DB_DEBUG__FORCE_Z_MODE__SHIFT                                                                         0x4
-#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ__SHIFT                                                               0x6
-#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ__SHIFT                                                             0x7
-#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE__SHIFT                                                               0x8
-#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0__SHIFT                                                              0xa
-#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1__SHIFT                                                              0xc
-#define DB_DEBUG__DEBUG_FAST_Z_DISABLE__SHIFT                                                                 0xe
-#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE__SHIFT                                                           0xf
-#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE__SHIFT                                                              0x10
-#define DB_DEBUG__DISABLE_SUMM_SQUADS__SHIFT                                                                  0x11
-#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS__SHIFT                                                               0x12
-#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE__SHIFT                                                             0x13
-#define DB_DEBUG__NEVER_FREE_Z_ONLY__SHIFT                                                                    0x15
-#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS__SHIFT                                                0x16
-#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT                                                    0x17
-#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES__SHIFT                                                           0x18
-#define DB_DEBUG__ONE_FREE_IN_FLIGHT__SHIFT                                                                   0x1c
-#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT__SHIFT                                                           0x1d
-#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC__SHIFT                                                           0x1e
-#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC__SHIFT                                                           0x1f
-#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE_MASK                                                         0x00000001L
-#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE_MASK                                                           0x00000002L
-#define DB_DEBUG__FETCH_FULL_Z_TILE_MASK                                                                      0x00000004L
-#define DB_DEBUG__FETCH_FULL_STENCIL_TILE_MASK                                                                0x00000008L
-#define DB_DEBUG__FORCE_Z_MODE_MASK                                                                           0x00000030L
-#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ_MASK                                                                 0x00000040L
-#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ_MASK                                                               0x00000080L
-#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE_MASK                                                                 0x00000300L
-#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0_MASK                                                                0x00000C00L
-#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1_MASK                                                                0x00003000L
-#define DB_DEBUG__DEBUG_FAST_Z_DISABLE_MASK                                                                   0x00004000L
-#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE_MASK                                                             0x00008000L
-#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE_MASK                                                                0x00010000L
-#define DB_DEBUG__DISABLE_SUMM_SQUADS_MASK                                                                    0x00020000L
-#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS_MASK                                                                 0x00040000L
-#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE_MASK                                                               0x00180000L
-#define DB_DEBUG__NEVER_FREE_Z_ONLY_MASK                                                                      0x00200000L
-#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS_MASK                                                  0x00400000L
-#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK                                                      0x00800000L
-#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES_MASK                                                             0x0F000000L
-#define DB_DEBUG__ONE_FREE_IN_FLIGHT_MASK                                                                     0x10000000L
-#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT_MASK                                                             0x20000000L
-#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC_MASK                                                             0x40000000L
-#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC_MASK                                                             0x80000000L
-//DB_DEBUG2
-#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING__SHIFT                                                            0x0
-#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE__SHIFT                                                          0x1
-#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE__SHIFT                                                            0x2
-#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB__SHIFT                                                                 0x3
-#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM__SHIFT                                                        0x4
-#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL__SHIFT                                                            0x5
-#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ__SHIFT                                                        0x6
-#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL__SHIFT                                                        0x7
-#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE__SHIFT                                                     0x8
-#define DB_DEBUG2__CLK_OFF_DELAY__SHIFT                                                                       0x9
-#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER__SHIFT                                                    0xe
-#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING__SHIFT                                                             0xf
-#define DB_DEBUG2__RESERVED__SHIFT                                                                            0x10
-#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING__SHIFT                                                         0x11
-#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING__SHIFT                                                         0x12
-#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL__SHIFT                                                        0x13
-#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM__SHIFT                                                             0x1c
-#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL__SHIFT                                                        0x1d
-#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM__SHIFT                                                    0x1e
-#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT__SHIFT                                                0x1f
-#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING_MASK                                                              0x00000001L
-#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE_MASK                                                            0x00000002L
-#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE_MASK                                                              0x00000004L
-#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB_MASK                                                                   0x00000008L
-#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM_MASK                                                          0x00000010L
-#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_MASK                                                              0x00000020L
-#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ_MASK                                                          0x00000040L
-#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL_MASK                                                          0x00000080L
-#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE_MASK                                                       0x00000100L
-#define DB_DEBUG2__CLK_OFF_DELAY_MASK                                                                         0x00003E00L
-#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER_MASK                                                      0x00004000L
-#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING_MASK                                                               0x00008000L
-#define DB_DEBUG2__RESERVED_MASK                                                                              0x00010000L
-#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING_MASK                                                           0x00020000L
-#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING_MASK                                                           0x00040000L
-#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL_MASK                                                          0x00080000L
-#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM_MASK                                                               0x10000000L
-#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL_MASK                                                          0x20000000L
-#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM_MASK                                                      0x40000000L
-#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT_MASK                                                  0x80000000L
-//DB_DEBUG3
-#define DB_DEBUG3__DISABLE_CLEAR_ZRANGE_CORRECTION__SHIFT                                                     0x0
-#define DB_DEBUG3__ROUND_ZRANGE_CORRECTION__SHIFT                                                             0x1
-#define DB_DEBUG3__FORCE_DB_IS_GOOD__SHIFT                                                                    0x2
-#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION__SHIFT                                                     0x3
-#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT                                                          0x4
-#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z__SHIFT                                                             0x5
-#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z__SHIFT                                                              0x6
-#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS__SHIFT                                                              0x7
-#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION__SHIFT                                                      0x8
-#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT__SHIFT                                                 0x9
-#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP__SHIFT                                            0xa
-#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS__SHIFT                                                        0xb
-#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING__SHIFT                                                        0xc
-#define DB_DEBUG3__DISABLE_OP_DF_BYPASS__SHIFT                                                                0xd
-#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE__SHIFT                                                         0xe
-#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK__SHIFT                                                       0xf
-#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION__SHIFT                                                             0x10
-#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE__SHIFT                                                         0x11
-#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING__SHIFT                                                        0x12
-#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE__SHIFT                                                     0x13
-#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE__SHIFT                                                         0x14
-#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT__SHIFT                                                0x15
-#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB__SHIFT                                                        0x16
-#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD__SHIFT                                                  0x17
-#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT__SHIFT                                                           0x18
-#define DB_DEBUG3__DISABLE_DI_DT_STALL__SHIFT                                                                 0x19
-#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET__SHIFT                                                             0x1a
-#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX__SHIFT                                                       0x1b
-#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND__SHIFT                                                         0x1c
-#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND__SHIFT                                                         0x1d
-#define DB_DEBUG3__DISABLE_4XAA_2P_DELAYED_WRITE__SHIFT                                                       0x1e
-#define DB_DEBUG3__DISABLE_4XAA_2P_INTERLEAVED_PMASK__SHIFT                                                   0x1f
-#define DB_DEBUG3__DISABLE_CLEAR_ZRANGE_CORRECTION_MASK                                                       0x00000001L
-#define DB_DEBUG3__ROUND_ZRANGE_CORRECTION_MASK                                                               0x00000002L
-#define DB_DEBUG3__FORCE_DB_IS_GOOD_MASK                                                                      0x00000004L
-#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION_MASK                                                       0x00000008L
-#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP_MASK                                                            0x00000010L
-#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z_MASK                                                               0x00000020L
-#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z_MASK                                                                0x00000040L
-#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS_MASK                                                                0x00000080L
-#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION_MASK                                                        0x00000100L
-#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT_MASK                                                   0x00000200L
-#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP_MASK                                              0x00000400L
-#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS_MASK                                                          0x00000800L
-#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING_MASK                                                          0x00001000L
-#define DB_DEBUG3__DISABLE_OP_DF_BYPASS_MASK                                                                  0x00002000L
-#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE_MASK                                                           0x00004000L
-#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK_MASK                                                         0x00008000L
-#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION_MASK                                                               0x00010000L
-#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE_MASK                                                           0x00020000L
-#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING_MASK                                                          0x00040000L
-#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE_MASK                                                       0x00080000L
-#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE_MASK                                                           0x00100000L
-#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT_MASK                                                  0x00200000L
-#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB_MASK                                                          0x00400000L
-#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD_MASK                                                    0x00800000L
-#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT_MASK                                                             0x01000000L
-#define DB_DEBUG3__DISABLE_DI_DT_STALL_MASK                                                                   0x02000000L
-#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET_MASK                                                               0x04000000L
-#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX_MASK                                                         0x08000000L
-#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND_MASK                                                           0x10000000L
-#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND_MASK                                                           0x20000000L
-#define DB_DEBUG3__DISABLE_4XAA_2P_DELAYED_WRITE_MASK                                                         0x40000000L
-#define DB_DEBUG3__DISABLE_4XAA_2P_INTERLEAVED_PMASK_MASK                                                     0x80000000L
-//DB_DEBUG4
-#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION__SHIFT                                                         0x0
-#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION__SHIFT                                                   0x1
-#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL__SHIFT                                                    0x2
-#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL__SHIFT                                             0x3
-#define DB_DEBUG4__DISABLE_4XAA_2P_ZD_HOLDOFF__SHIFT                                                          0x4
-#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION__SHIFT                                                       0x5
-#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE__SHIFT                                                    0x6
-#define DB_DEBUG4__ALWAYS_ON_RMI_CLK_EN__SHIFT                                                                0x7
-#define DB_DEBUG4__DFSM_CONVERT_PASSTHROUGH_TO_BYPASS__SHIFT                                                  0x8
-#define DB_DEBUG4__DISABLE_UNMAPPED_Z_INDICATOR__SHIFT                                                        0x9
-#define DB_DEBUG4__DISABLE_UNMAPPED_S_INDICATOR__SHIFT                                                        0xa
-#define DB_DEBUG4__DISABLE_UNMAPPED_H_INDICATOR__SHIFT                                                        0xb
-#define DB_DEBUG4__DISABLE_SEPARATE_DFSM_CLK__SHIFT                                                           0xc
-#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP__SHIFT                                                   0xd
-#define DB_DEBUG4__DISABLE_RESCHECK_MEMCOHER_OPTIMIZATION__SHIFT                                              0xe
-#define DB_DEBUG4__DISABLE_TS_WRITE_L0__SHIFT                                                                 0xf
-#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE__SHIFT                                                0x10
-#define DB_DEBUG4__DISABLE_HIZ_Q1_TS_COLLISION_DETECT__SHIFT                                                  0x11
-#define DB_DEBUG4__DISABLE_HIZ_Q2_TS_COLLISION_DETECT__SHIFT                                                  0x12
-#define DB_DEBUG4__DB_EXTRA_DEBUG4__SHIFT                                                                     0x13
-#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION_MASK                                                           0x00000001L
-#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION_MASK                                                     0x00000002L
-#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL_MASK                                                      0x00000004L
-#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL_MASK                                               0x00000008L
-#define DB_DEBUG4__DISABLE_4XAA_2P_ZD_HOLDOFF_MASK                                                            0x00000010L
-#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION_MASK                                                         0x00000020L
-#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE_MASK                                                      0x00000040L
-#define DB_DEBUG4__ALWAYS_ON_RMI_CLK_EN_MASK                                                                  0x00000080L
-#define DB_DEBUG4__DFSM_CONVERT_PASSTHROUGH_TO_BYPASS_MASK                                                    0x00000100L
-#define DB_DEBUG4__DISABLE_UNMAPPED_Z_INDICATOR_MASK                                                          0x00000200L
-#define DB_DEBUG4__DISABLE_UNMAPPED_S_INDICATOR_MASK                                                          0x00000400L
-#define DB_DEBUG4__DISABLE_UNMAPPED_H_INDICATOR_MASK                                                          0x00000800L
-#define DB_DEBUG4__DISABLE_SEPARATE_DFSM_CLK_MASK                                                             0x00001000L
-#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP_MASK                                                     0x00002000L
-#define DB_DEBUG4__DISABLE_RESCHECK_MEMCOHER_OPTIMIZATION_MASK                                                0x00004000L
-#define DB_DEBUG4__DISABLE_TS_WRITE_L0_MASK                                                                   0x00008000L
-#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE_MASK                                                  0x00010000L
-#define DB_DEBUG4__DISABLE_HIZ_Q1_TS_COLLISION_DETECT_MASK                                                    0x00020000L
-#define DB_DEBUG4__DISABLE_HIZ_Q2_TS_COLLISION_DETECT_MASK                                                    0x00040000L
-#define DB_DEBUG4__DB_EXTRA_DEBUG4_MASK                                                                       0xFFF80000L
-//DB_CREDIT_LIMIT
-#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS__SHIFT                                                            0x0
-#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS__SHIFT                                                            0x5
-#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS__SHIFT                                                           0xa
-#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS__SHIFT                                                            0x18
-#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS_MASK                                                              0x0000001FL
-#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS_MASK                                                              0x000003E0L
-#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS_MASK                                                             0x00001C00L
-#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS_MASK                                                              0x7F000000L
-//DB_WATERMARKS
-#define DB_WATERMARKS__DEPTH_FREE__SHIFT                                                                      0x0
-#define DB_WATERMARKS__DEPTH_FLUSH__SHIFT                                                                     0x5
-#define DB_WATERMARKS__FORCE_SUMMARIZE__SHIFT                                                                 0xb
-#define DB_WATERMARKS__DEPTH_PENDING_FREE__SHIFT                                                              0xf
-#define DB_WATERMARKS__DEPTH_CACHELINE_FREE__SHIFT                                                            0x14
-#define DB_WATERMARKS__AUTO_FLUSH_HTILE__SHIFT                                                                0x1e
-#define DB_WATERMARKS__AUTO_FLUSH_QUAD__SHIFT                                                                 0x1f
-#define DB_WATERMARKS__DEPTH_FREE_MASK                                                                        0x0000001FL
-#define DB_WATERMARKS__DEPTH_FLUSH_MASK                                                                       0x000007E0L
-#define DB_WATERMARKS__FORCE_SUMMARIZE_MASK                                                                   0x00007800L
-#define DB_WATERMARKS__DEPTH_PENDING_FREE_MASK                                                                0x000F8000L
-#define DB_WATERMARKS__DEPTH_CACHELINE_FREE_MASK                                                              0x0FF00000L
-#define DB_WATERMARKS__AUTO_FLUSH_HTILE_MASK                                                                  0x40000000L
-#define DB_WATERMARKS__AUTO_FLUSH_QUAD_MASK                                                                   0x80000000L
-//DB_SUBTILE_CONTROL
-#define DB_SUBTILE_CONTROL__MSAA1_X__SHIFT                                                                    0x0
-#define DB_SUBTILE_CONTROL__MSAA1_Y__SHIFT                                                                    0x2
-#define DB_SUBTILE_CONTROL__MSAA2_X__SHIFT                                                                    0x4
-#define DB_SUBTILE_CONTROL__MSAA2_Y__SHIFT                                                                    0x6
-#define DB_SUBTILE_CONTROL__MSAA4_X__SHIFT                                                                    0x8
-#define DB_SUBTILE_CONTROL__MSAA4_Y__SHIFT                                                                    0xa
-#define DB_SUBTILE_CONTROL__MSAA8_X__SHIFT                                                                    0xc
-#define DB_SUBTILE_CONTROL__MSAA8_Y__SHIFT                                                                    0xe
-#define DB_SUBTILE_CONTROL__MSAA16_X__SHIFT                                                                   0x10
-#define DB_SUBTILE_CONTROL__MSAA16_Y__SHIFT                                                                   0x12
-#define DB_SUBTILE_CONTROL__MSAA1_X_MASK                                                                      0x00000003L
-#define DB_SUBTILE_CONTROL__MSAA1_Y_MASK                                                                      0x0000000CL
-#define DB_SUBTILE_CONTROL__MSAA2_X_MASK                                                                      0x00000030L
-#define DB_SUBTILE_CONTROL__MSAA2_Y_MASK                                                                      0x000000C0L
-#define DB_SUBTILE_CONTROL__MSAA4_X_MASK                                                                      0x00000300L
-#define DB_SUBTILE_CONTROL__MSAA4_Y_MASK                                                                      0x00000C00L
-#define DB_SUBTILE_CONTROL__MSAA8_X_MASK                                                                      0x00003000L
-#define DB_SUBTILE_CONTROL__MSAA8_Y_MASK                                                                      0x0000C000L
-#define DB_SUBTILE_CONTROL__MSAA16_X_MASK                                                                     0x00030000L
-#define DB_SUBTILE_CONTROL__MSAA16_Y_MASK                                                                     0x000C0000L
-//DB_FREE_CACHELINES
-#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH__SHIFT                                                           0x0
-#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH__SHIFT                                                           0x7
-#define DB_FREE_CACHELINES__FREE_Z_DEPTH__SHIFT                                                               0xe
-#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH__SHIFT                                                           0x14
-#define DB_FREE_CACHELINES__QUAD_READ_REQS__SHIFT                                                             0x18
-#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH_MASK                                                             0x0000007FL
-#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH_MASK                                                             0x00003F80L
-#define DB_FREE_CACHELINES__FREE_Z_DEPTH_MASK                                                                 0x000FC000L
-#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH_MASK                                                             0x00F00000L
-#define DB_FREE_CACHELINES__QUAD_READ_REQS_MASK                                                               0xFF000000L
-//DB_FIFO_DEPTH1
-#define DB_FIFO_DEPTH1__DB_RMI_RDREQ_CREDITS__SHIFT                                                           0x0
-#define DB_FIFO_DEPTH1__DB_RMI_WRREQ_CREDITS__SHIFT                                                           0x5
-#define DB_FIFO_DEPTH1__MCC_DEPTH__SHIFT                                                                      0xa
-#define DB_FIFO_DEPTH1__QC_DEPTH__SHIFT                                                                       0x10
-#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH__SHIFT                                                         0x15
-#define DB_FIFO_DEPTH1__DB_RMI_RDREQ_CREDITS_MASK                                                             0x0000001FL
-#define DB_FIFO_DEPTH1__DB_RMI_WRREQ_CREDITS_MASK                                                             0x000003E0L
-#define DB_FIFO_DEPTH1__MCC_DEPTH_MASK                                                                        0x0000FC00L
-#define DB_FIFO_DEPTH1__QC_DEPTH_MASK                                                                         0x001F0000L
-#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH_MASK                                                           0x1FE00000L
-//DB_FIFO_DEPTH2
-#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH__SHIFT                                                               0x0
-#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH__SHIFT                                                            0x8
-#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH__SHIFT                                                               0xf
-#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH__SHIFT                                                            0x19
-#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH_MASK                                                                 0x000000FFL
-#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH_MASK                                                              0x00007F00L
-#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH_MASK                                                                 0x01FF8000L
-#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH_MASK                                                              0xFE000000L
-//DB_EXCEPTION_CONTROL
-#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE__SHIFT                                                    0x0
-#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE__SHIFT                                                     0x1
-#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE__SHIFT                                                       0x2
-#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE_MASK                                                      0x00000001L
-#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE_MASK                                                       0x00000002L
-#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE_MASK                                                         0x00000004L
-//DB_RING_CONTROL
-#define DB_RING_CONTROL__COUNTER_CONTROL__SHIFT                                                               0x0
-#define DB_RING_CONTROL__COUNTER_CONTROL_MASK                                                                 0x00000003L
-//DB_MEM_ARB_WATERMARKS
-#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK__SHIFT                                                       0x0
-#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK__SHIFT                                                       0x8
-#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK__SHIFT                                                       0x10
-#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK__SHIFT                                                       0x18
-#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK_MASK                                                         0x00000007L
-#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK_MASK                                                         0x00000700L
-#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK_MASK                                                         0x00070000L
-#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK_MASK                                                         0x07000000L
-//DB_RMI_CACHE_POLICY
-#define DB_RMI_CACHE_POLICY__Z_RD__SHIFT                                                                      0x0
-#define DB_RMI_CACHE_POLICY__S_RD__SHIFT                                                                      0x1
-#define DB_RMI_CACHE_POLICY__HTILE_RD__SHIFT                                                                  0x2
-#define DB_RMI_CACHE_POLICY__Z_WR__SHIFT                                                                      0x8
-#define DB_RMI_CACHE_POLICY__S_WR__SHIFT                                                                      0x9
-#define DB_RMI_CACHE_POLICY__HTILE_WR__SHIFT                                                                  0xa
-#define DB_RMI_CACHE_POLICY__ZPCPSD_WR__SHIFT                                                                 0xb
-#define DB_RMI_CACHE_POLICY__CC_RD__SHIFT                                                                     0x10
-#define DB_RMI_CACHE_POLICY__FMASK_RD__SHIFT                                                                  0x11
-#define DB_RMI_CACHE_POLICY__CMASK_RD__SHIFT                                                                  0x12
-#define DB_RMI_CACHE_POLICY__DCC_RD__SHIFT                                                                    0x13
-#define DB_RMI_CACHE_POLICY__CC_WR__SHIFT                                                                     0x18
-#define DB_RMI_CACHE_POLICY__FMASK_WR__SHIFT                                                                  0x19
-#define DB_RMI_CACHE_POLICY__CMASK_WR__SHIFT                                                                  0x1a
-#define DB_RMI_CACHE_POLICY__DCC_WR__SHIFT                                                                    0x1b
-#define DB_RMI_CACHE_POLICY__Z_RD_MASK                                                                        0x00000001L
-#define DB_RMI_CACHE_POLICY__S_RD_MASK                                                                        0x00000002L
-#define DB_RMI_CACHE_POLICY__HTILE_RD_MASK                                                                    0x00000004L
-#define DB_RMI_CACHE_POLICY__Z_WR_MASK                                                                        0x00000100L
-#define DB_RMI_CACHE_POLICY__S_WR_MASK                                                                        0x00000200L
-#define DB_RMI_CACHE_POLICY__HTILE_WR_MASK                                                                    0x00000400L
-#define DB_RMI_CACHE_POLICY__ZPCPSD_WR_MASK                                                                   0x00000800L
-#define DB_RMI_CACHE_POLICY__CC_RD_MASK                                                                       0x00010000L
-#define DB_RMI_CACHE_POLICY__FMASK_RD_MASK                                                                    0x00020000L
-#define DB_RMI_CACHE_POLICY__CMASK_RD_MASK                                                                    0x00040000L
-#define DB_RMI_CACHE_POLICY__DCC_RD_MASK                                                                      0x00080000L
-#define DB_RMI_CACHE_POLICY__CC_WR_MASK                                                                       0x01000000L
-#define DB_RMI_CACHE_POLICY__FMASK_WR_MASK                                                                    0x02000000L
-#define DB_RMI_CACHE_POLICY__CMASK_WR_MASK                                                                    0x04000000L
-#define DB_RMI_CACHE_POLICY__DCC_WR_MASK                                                                      0x08000000L
-//DB_DFSM_CONFIG
-#define DB_DFSM_CONFIG__BYPASS_DFSM__SHIFT                                                                    0x0
-#define DB_DFSM_CONFIG__DISABLE_PUNCHOUT__SHIFT                                                               0x1
-#define DB_DFSM_CONFIG__DISABLE_POPS__SHIFT                                                                   0x2
-#define DB_DFSM_CONFIG__FORCE_FLUSH__SHIFT                                                                    0x3
-#define DB_DFSM_CONFIG__MIDDLE_PIPE_MAX_DEPTH__SHIFT                                                          0x8
-#define DB_DFSM_CONFIG__BYPASS_DFSM_MASK                                                                      0x00000001L
-#define DB_DFSM_CONFIG__DISABLE_PUNCHOUT_MASK                                                                 0x00000002L
-#define DB_DFSM_CONFIG__DISABLE_POPS_MASK                                                                     0x00000004L
-#define DB_DFSM_CONFIG__FORCE_FLUSH_MASK                                                                      0x00000008L
-#define DB_DFSM_CONFIG__MIDDLE_PIPE_MAX_DEPTH_MASK                                                            0x00007F00L
-//DB_DFSM_WATERMARK
-#define DB_DFSM_WATERMARK__DFSM_HIGH_WATERMARK__SHIFT                                                         0x0
-#define DB_DFSM_WATERMARK__POPS_HIGH_WATERMARK__SHIFT                                                         0x10
-#define DB_DFSM_WATERMARK__DFSM_HIGH_WATERMARK_MASK                                                           0x0000FFFFL
-#define DB_DFSM_WATERMARK__POPS_HIGH_WATERMARK_MASK                                                           0xFFFF0000L
-//DB_DFSM_TILES_IN_FLIGHT
-#define DB_DFSM_TILES_IN_FLIGHT__HIGH_WATERMARK__SHIFT                                                        0x0
-#define DB_DFSM_TILES_IN_FLIGHT__HARD_LIMIT__SHIFT                                                            0x10
-#define DB_DFSM_TILES_IN_FLIGHT__HIGH_WATERMARK_MASK                                                          0x0000FFFFL
-#define DB_DFSM_TILES_IN_FLIGHT__HARD_LIMIT_MASK                                                              0xFFFF0000L
-//DB_DFSM_PRIMS_IN_FLIGHT
-#define DB_DFSM_PRIMS_IN_FLIGHT__HIGH_WATERMARK__SHIFT                                                        0x0
-#define DB_DFSM_PRIMS_IN_FLIGHT__HARD_LIMIT__SHIFT                                                            0x10
-#define DB_DFSM_PRIMS_IN_FLIGHT__HIGH_WATERMARK_MASK                                                          0x0000FFFFL
-#define DB_DFSM_PRIMS_IN_FLIGHT__HARD_LIMIT_MASK                                                              0xFFFF0000L
-//DB_DFSM_WATCHDOG
-#define DB_DFSM_WATCHDOG__TIMER_TARGET__SHIFT                                                                 0x0
-#define DB_DFSM_WATCHDOG__TIMER_TARGET_MASK                                                                   0xFFFFFFFFL
-//DB_DFSM_FLUSH_ENABLE
-#define DB_DFSM_FLUSH_ENABLE__PRIMARY_EVENTS__SHIFT                                                           0x0
-#define DB_DFSM_FLUSH_ENABLE__AUX_FORCE_PASSTHRU__SHIFT                                                       0x18
-#define DB_DFSM_FLUSH_ENABLE__AUX_EVENTS__SHIFT                                                               0x1c
-#define DB_DFSM_FLUSH_ENABLE__PRIMARY_EVENTS_MASK                                                             0x000003FFL
-#define DB_DFSM_FLUSH_ENABLE__AUX_FORCE_PASSTHRU_MASK                                                         0x0F000000L
-#define DB_DFSM_FLUSH_ENABLE__AUX_EVENTS_MASK                                                                 0xF0000000L
-//DB_DFSM_FLUSH_AUX_EVENT
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_A__SHIFT                                                               0x0
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_B__SHIFT                                                               0x8
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_C__SHIFT                                                               0x10
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_D__SHIFT                                                               0x18
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_A_MASK                                                                 0x000000FFL
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_B_MASK                                                                 0x0000FF00L
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_C_MASK                                                                 0x00FF0000L
-#define DB_DFSM_FLUSH_AUX_EVENT__EVENT_D_MASK                                                                 0xFF000000L
-//CC_RB_REDUNDANCY
-#define CC_RB_REDUNDANCY__FAILED_RB0__SHIFT                                                                   0x8
-#define CC_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT                                                               0xc
-#define CC_RB_REDUNDANCY__FAILED_RB1__SHIFT                                                                   0x10
-#define CC_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT                                                               0x14
-#define CC_RB_REDUNDANCY__FAILED_RB0_MASK                                                                     0x00000F00L
-#define CC_RB_REDUNDANCY__EN_REDUNDANCY0_MASK                                                                 0x00001000L
-#define CC_RB_REDUNDANCY__FAILED_RB1_MASK                                                                     0x000F0000L
-#define CC_RB_REDUNDANCY__EN_REDUNDANCY1_MASK                                                                 0x00100000L
-//CC_RB_BACKEND_DISABLE
-#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                         0x10
-#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                           0x00FF0000L
-//GB_ADDR_CONFIG
-#define GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                      0x0
-#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                           0x3
-#define GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                           0x6
-#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT                                                           0x8
-#define GB_ADDR_CONFIG__NUM_BANKS__SHIFT                                                                      0xc
-#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT                                                        0x10
-#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                             0x13
-#define GB_ADDR_CONFIG__NUM_GPUS__SHIFT                                                                       0x15
-#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT                                                            0x18
-#define GB_ADDR_CONFIG__NUM_RB_PER_SE__SHIFT                                                                  0x1a
-#define GB_ADDR_CONFIG__ROW_SIZE__SHIFT                                                                       0x1c
-#define GB_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT                                                                0x1e
-#define GB_ADDR_CONFIG__SE_ENABLE__SHIFT                                                                      0x1f
-#define GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                        0x00000007L
-#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                             0x00000038L
-#define GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                             0x000000C0L
-#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK                                                             0x00000700L
-#define GB_ADDR_CONFIG__NUM_BANKS_MASK                                                                        0x00007000L
-#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK                                                          0x00070000L
-#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                               0x00180000L
-#define GB_ADDR_CONFIG__NUM_GPUS_MASK                                                                         0x00E00000L
-#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK                                                              0x03000000L
-#define GB_ADDR_CONFIG__NUM_RB_PER_SE_MASK                                                                    0x0C000000L
-#define GB_ADDR_CONFIG__ROW_SIZE_MASK                                                                         0x30000000L
-#define GB_ADDR_CONFIG__NUM_LOWER_PIPES_MASK                                                                  0x40000000L
-#define GB_ADDR_CONFIG__SE_ENABLE_MASK                                                                        0x80000000L
-//GB_BACKEND_MAP
-#define GB_BACKEND_MAP__BACKEND_MAP__SHIFT                                                                    0x0
-#define GB_BACKEND_MAP__BACKEND_MAP_MASK                                                                      0xFFFFFFFFL
-//GB_GPU_ID
-#define GB_GPU_ID__GPU_ID__SHIFT                                                                              0x0
-#define GB_GPU_ID__GPU_ID_MASK                                                                                0x0000000FL
-//CC_RB_DAISY_CHAIN
-#define CC_RB_DAISY_CHAIN__RB_0__SHIFT                                                                        0x0
-#define CC_RB_DAISY_CHAIN__RB_1__SHIFT                                                                        0x4
-#define CC_RB_DAISY_CHAIN__RB_2__SHIFT                                                                        0x8
-#define CC_RB_DAISY_CHAIN__RB_3__SHIFT                                                                        0xc
-#define CC_RB_DAISY_CHAIN__RB_4__SHIFT                                                                        0x10
-#define CC_RB_DAISY_CHAIN__RB_5__SHIFT                                                                        0x14
-#define CC_RB_DAISY_CHAIN__RB_6__SHIFT                                                                        0x18
-#define CC_RB_DAISY_CHAIN__RB_7__SHIFT                                                                        0x1c
-#define CC_RB_DAISY_CHAIN__RB_0_MASK                                                                          0x0000000FL
-#define CC_RB_DAISY_CHAIN__RB_1_MASK                                                                          0x000000F0L
-#define CC_RB_DAISY_CHAIN__RB_2_MASK                                                                          0x00000F00L
-#define CC_RB_DAISY_CHAIN__RB_3_MASK                                                                          0x0000F000L
-#define CC_RB_DAISY_CHAIN__RB_4_MASK                                                                          0x000F0000L
-#define CC_RB_DAISY_CHAIN__RB_5_MASK                                                                          0x00F00000L
-#define CC_RB_DAISY_CHAIN__RB_6_MASK                                                                          0x0F000000L
-#define CC_RB_DAISY_CHAIN__RB_7_MASK                                                                          0xF0000000L
-//GB_ADDR_CONFIG_READ
-#define GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                                 0x0
-#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x3
-#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS__SHIFT                                                      0x6
-#define GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT                                                      0x8
-#define GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT                                                                 0xc
-#define GB_ADDR_CONFIG_READ__SHADER_ENGINE_TILE_SIZE__SHIFT                                                   0x10
-#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                        0x13
-#define GB_ADDR_CONFIG_READ__NUM_GPUS__SHIFT                                                                  0x15
-#define GB_ADDR_CONFIG_READ__MULTI_GPU_TILE_SIZE__SHIFT                                                       0x18
-#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE__SHIFT                                                             0x1a
-#define GB_ADDR_CONFIG_READ__ROW_SIZE__SHIFT                                                                  0x1c
-#define GB_ADDR_CONFIG_READ__NUM_LOWER_PIPES__SHIFT                                                           0x1e
-#define GB_ADDR_CONFIG_READ__SE_ENABLE__SHIFT                                                                 0x1f
-#define GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                                   0x00000007L
-#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000038L
-#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS_MASK                                                        0x000000C0L
-#define GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK                                                        0x00000700L
-#define GB_ADDR_CONFIG_READ__NUM_BANKS_MASK                                                                   0x00007000L
-#define GB_ADDR_CONFIG_READ__SHADER_ENGINE_TILE_SIZE_MASK                                                     0x00070000L
-#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                          0x00180000L
-#define GB_ADDR_CONFIG_READ__NUM_GPUS_MASK                                                                    0x00E00000L
-#define GB_ADDR_CONFIG_READ__MULTI_GPU_TILE_SIZE_MASK                                                         0x03000000L
-#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE_MASK                                                               0x0C000000L
-#define GB_ADDR_CONFIG_READ__ROW_SIZE_MASK                                                                    0x30000000L
-#define GB_ADDR_CONFIG_READ__NUM_LOWER_PIPES_MASK                                                             0x40000000L
-#define GB_ADDR_CONFIG_READ__SE_ENABLE_MASK                                                                   0x80000000L
-//GB_TILE_MODE0
-#define GB_TILE_MODE0__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE0__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE0__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE0__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE0__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE0__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE0__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE1
-#define GB_TILE_MODE1__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE1__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE1__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE1__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE1__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE1__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE1__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE1__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE2
-#define GB_TILE_MODE2__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE2__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE2__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE2__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE2__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE2__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE2__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE2__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE3
-#define GB_TILE_MODE3__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE3__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE3__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE3__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE3__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE3__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE3__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE3__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE4
-#define GB_TILE_MODE4__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE4__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE4__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE4__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE4__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE4__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE4__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE4__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE5
-#define GB_TILE_MODE5__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE5__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE5__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE5__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE5__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE5__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE5__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE5__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE6
-#define GB_TILE_MODE6__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE6__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE6__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE6__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE6__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE6__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE6__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE6__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE7
-#define GB_TILE_MODE7__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE7__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE7__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE7__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE7__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE7__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE7__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE7__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE8
-#define GB_TILE_MODE8__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE8__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE8__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE8__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE8__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE8__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE8__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE8__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE9
-#define GB_TILE_MODE9__ARRAY_MODE__SHIFT                                                                      0x2
-#define GB_TILE_MODE9__PIPE_CONFIG__SHIFT                                                                     0x6
-#define GB_TILE_MODE9__TILE_SPLIT__SHIFT                                                                      0xb
-#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW__SHIFT                                                             0x16
-#define GB_TILE_MODE9__SAMPLE_SPLIT__SHIFT                                                                    0x19
-#define GB_TILE_MODE9__ARRAY_MODE_MASK                                                                        0x0000003CL
-#define GB_TILE_MODE9__PIPE_CONFIG_MASK                                                                       0x000007C0L
-#define GB_TILE_MODE9__TILE_SPLIT_MASK                                                                        0x00003800L
-#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW_MASK                                                               0x01C00000L
-#define GB_TILE_MODE9__SAMPLE_SPLIT_MASK                                                                      0x06000000L
-//GB_TILE_MODE10
-#define GB_TILE_MODE10__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE10__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE10__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE10__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE10__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE10__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE10__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE10__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE11
-#define GB_TILE_MODE11__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE11__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE11__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE11__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE11__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE11__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE11__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE11__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE12
-#define GB_TILE_MODE12__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE12__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE12__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE12__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE12__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE12__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE12__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE12__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE13
-#define GB_TILE_MODE13__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE13__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE13__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE13__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE13__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE13__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE13__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE13__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE14
-#define GB_TILE_MODE14__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE14__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE14__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE14__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE14__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE14__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE14__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE14__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE15
-#define GB_TILE_MODE15__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE15__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE15__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE15__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE15__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE15__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE15__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE15__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE16
-#define GB_TILE_MODE16__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE16__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE16__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE16__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE16__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE16__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE16__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE16__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE17
-#define GB_TILE_MODE17__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE17__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE17__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE17__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE17__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE17__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE17__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE17__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE18
-#define GB_TILE_MODE18__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE18__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE18__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE18__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE18__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE18__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE18__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE18__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE19
-#define GB_TILE_MODE19__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE19__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE19__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE19__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE19__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE19__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE19__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE19__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE20
-#define GB_TILE_MODE20__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE20__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE20__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE20__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE20__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE20__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE20__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE20__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE21
-#define GB_TILE_MODE21__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE21__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE21__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE21__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE21__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE21__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE21__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE21__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE22
-#define GB_TILE_MODE22__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE22__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE22__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE22__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE22__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE22__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE22__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE22__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE23
-#define GB_TILE_MODE23__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE23__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE23__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE23__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE23__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE23__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE23__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE23__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE24
-#define GB_TILE_MODE24__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE24__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE24__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE24__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE24__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE24__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE24__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE24__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE25
-#define GB_TILE_MODE25__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE25__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE25__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE25__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE25__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE25__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE25__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE25__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE26
-#define GB_TILE_MODE26__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE26__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE26__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE26__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE26__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE26__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE26__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE26__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE27
-#define GB_TILE_MODE27__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE27__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE27__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE27__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE27__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE27__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE27__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE27__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE28
-#define GB_TILE_MODE28__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE28__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE28__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE28__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE28__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE28__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE28__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE28__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE29
-#define GB_TILE_MODE29__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE29__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE29__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE29__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE29__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE29__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE29__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE29__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE30
-#define GB_TILE_MODE30__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE30__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE30__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE30__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE30__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE30__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE30__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE30__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_TILE_MODE31
-#define GB_TILE_MODE31__ARRAY_MODE__SHIFT                                                                     0x2
-#define GB_TILE_MODE31__PIPE_CONFIG__SHIFT                                                                    0x6
-#define GB_TILE_MODE31__TILE_SPLIT__SHIFT                                                                     0xb
-#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW__SHIFT                                                            0x16
-#define GB_TILE_MODE31__SAMPLE_SPLIT__SHIFT                                                                   0x19
-#define GB_TILE_MODE31__ARRAY_MODE_MASK                                                                       0x0000003CL
-#define GB_TILE_MODE31__PIPE_CONFIG_MASK                                                                      0x000007C0L
-#define GB_TILE_MODE31__TILE_SPLIT_MASK                                                                       0x00003800L
-#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW_MASK                                                              0x01C00000L
-#define GB_TILE_MODE31__SAMPLE_SPLIT_MASK                                                                     0x06000000L
-//GB_MACROTILE_MODE0
-#define GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE0__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE0__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE0__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE0__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE1
-#define GB_MACROTILE_MODE1__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE1__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE1__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE1__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE1__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE1__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE2
-#define GB_MACROTILE_MODE2__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE2__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE2__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE2__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE2__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE2__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE3
-#define GB_MACROTILE_MODE3__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE3__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE3__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE3__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE3__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE3__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE4
-#define GB_MACROTILE_MODE4__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE4__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE4__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE4__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE4__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE4__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE5
-#define GB_MACROTILE_MODE5__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE5__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE5__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE5__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE5__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE5__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE6
-#define GB_MACROTILE_MODE6__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE6__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE6__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE6__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE6__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE6__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE7
-#define GB_MACROTILE_MODE7__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE7__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE7__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE7__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE7__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE7__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE8
-#define GB_MACROTILE_MODE8__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE8__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE8__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE8__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE8__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE8__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE9
-#define GB_MACROTILE_MODE9__BANK_WIDTH__SHIFT                                                                 0x0
-#define GB_MACROTILE_MODE9__BANK_HEIGHT__SHIFT                                                                0x2
-#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT__SHIFT                                                          0x4
-#define GB_MACROTILE_MODE9__NUM_BANKS__SHIFT                                                                  0x6
-#define GB_MACROTILE_MODE9__BANK_WIDTH_MASK                                                                   0x00000003L
-#define GB_MACROTILE_MODE9__BANK_HEIGHT_MASK                                                                  0x0000000CL
-#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT_MASK                                                            0x00000030L
-#define GB_MACROTILE_MODE9__NUM_BANKS_MASK                                                                    0x000000C0L
-//GB_MACROTILE_MODE10
-#define GB_MACROTILE_MODE10__BANK_WIDTH__SHIFT                                                                0x0
-#define GB_MACROTILE_MODE10__BANK_HEIGHT__SHIFT                                                               0x2
-#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT__SHIFT                                                         0x4
-#define GB_MACROTILE_MODE10__NUM_BANKS__SHIFT                                                                 0x6
-#define GB_MACROTILE_MODE10__BANK_WIDTH_MASK                                                                  0x00000003L
-#define GB_MACROTILE_MODE10__BANK_HEIGHT_MASK                                                                 0x0000000CL
-#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
-#define GB_MACROTILE_MODE10__NUM_BANKS_MASK                                                                   0x000000C0L
-//GB_MACROTILE_MODE11
-#define GB_MACROTILE_MODE11__BANK_WIDTH__SHIFT                                                                0x0
-#define GB_MACROTILE_MODE11__BANK_HEIGHT__SHIFT                                                               0x2
-#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT__SHIFT                                                         0x4
-#define GB_MACROTILE_MODE11__NUM_BANKS__SHIFT                                                                 0x6
-#define GB_MACROTILE_MODE11__BANK_WIDTH_MASK                                                                  0x00000003L
-#define GB_MACROTILE_MODE11__BANK_HEIGHT_MASK                                                                 0x0000000CL
-#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
-#define GB_MACROTILE_MODE11__NUM_BANKS_MASK                                                                   0x000000C0L
-//GB_MACROTILE_MODE12
-#define GB_MACROTILE_MODE12__BANK_WIDTH__SHIFT                                                                0x0
-#define GB_MACROTILE_MODE12__BANK_HEIGHT__SHIFT                                                               0x2
-#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT__SHIFT                                                         0x4
-#define GB_MACROTILE_MODE12__NUM_BANKS__SHIFT                                                                 0x6
-#define GB_MACROTILE_MODE12__BANK_WIDTH_MASK                                                                  0x00000003L
-#define GB_MACROTILE_MODE12__BANK_HEIGHT_MASK                                                                 0x0000000CL
-#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
-#define GB_MACROTILE_MODE12__NUM_BANKS_MASK                                                                   0x000000C0L
-//GB_MACROTILE_MODE13
-#define GB_MACROTILE_MODE13__BANK_WIDTH__SHIFT                                                                0x0
-#define GB_MACROTILE_MODE13__BANK_HEIGHT__SHIFT                                                               0x2
-#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT__SHIFT                                                         0x4
-#define GB_MACROTILE_MODE13__NUM_BANKS__SHIFT                                                                 0x6
-#define GB_MACROTILE_MODE13__BANK_WIDTH_MASK                                                                  0x00000003L
-#define GB_MACROTILE_MODE13__BANK_HEIGHT_MASK                                                                 0x0000000CL
-#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
-#define GB_MACROTILE_MODE13__NUM_BANKS_MASK                                                                   0x000000C0L
-//GB_MACROTILE_MODE14
-#define GB_MACROTILE_MODE14__BANK_WIDTH__SHIFT                                                                0x0
-#define GB_MACROTILE_MODE14__BANK_HEIGHT__SHIFT                                                               0x2
-#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT__SHIFT                                                         0x4
-#define GB_MACROTILE_MODE14__NUM_BANKS__SHIFT                                                                 0x6
-#define GB_MACROTILE_MODE14__BANK_WIDTH_MASK                                                                  0x00000003L
-#define GB_MACROTILE_MODE14__BANK_HEIGHT_MASK                                                                 0x0000000CL
-#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
-#define GB_MACROTILE_MODE14__NUM_BANKS_MASK                                                                   0x000000C0L
-//GB_MACROTILE_MODE15
-#define GB_MACROTILE_MODE15__BANK_WIDTH__SHIFT                                                                0x0
-#define GB_MACROTILE_MODE15__BANK_HEIGHT__SHIFT                                                               0x2
-#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT__SHIFT                                                         0x4
-#define GB_MACROTILE_MODE15__NUM_BANKS__SHIFT                                                                 0x6
-#define GB_MACROTILE_MODE15__BANK_WIDTH_MASK                                                                  0x00000003L
-#define GB_MACROTILE_MODE15__BANK_HEIGHT_MASK                                                                 0x0000000CL
-#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT_MASK                                                           0x00000030L
-#define GB_MACROTILE_MODE15__NUM_BANKS_MASK                                                                   0x000000C0L
-//CB_HW_CONTROL
-#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT__SHIFT                                                            0x0
-#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT__SHIFT                                                            0x6
-#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT__SHIFT                                                            0xc
-#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE__SHIFT                                                      0x10
-#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING__SHIFT                                                0x12
-#define CB_HW_CONTROL__FORCE_NEEDS_DST__SHIFT                                                                 0x13
-#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE__SHIFT                                                             0x14
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST__SHIFT                                                0x15
-#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK__SHIFT                                                         0x16
-#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG__SHIFT                                             0x17
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT                                                   0x18
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS__SHIFT                                                        0x19
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT                                                 0x1a
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED__SHIFT                                0x1b
-#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT__SHIFT                                   0x1c
-#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT__SHIFT                                0x1d
-#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT__SHIFT                                              0x1e
-#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE__SHIFT                                    0x1f
-#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT_MASK                                                              0x0000000FL
-#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT_MASK                                                              0x000003C0L
-#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT_MASK                                                              0x0000F000L
-#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE_MASK                                                        0x00010000L
-#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING_MASK                                                  0x00040000L
-#define CB_HW_CONTROL__FORCE_NEEDS_DST_MASK                                                                   0x00080000L
-#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE_MASK                                                               0x00100000L
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST_MASK                                                  0x00200000L
-#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK_MASK                                                           0x00400000L
-#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG_MASK                                               0x00800000L
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST_MASK                                                     0x01000000L
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS_MASK                                                          0x02000000L
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK                                                   0x04000000L
-#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED_MASK                                  0x08000000L
-#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT_MASK                                     0x10000000L
-#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT_MASK                                  0x20000000L
-#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT_MASK                                                0x40000000L
-#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE_MASK                                      0x80000000L
-//CB_HW_CONTROL_1
-#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS__SHIFT                                                             0x0
-#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS__SHIFT                                                             0x5
-#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS__SHIFT                                                             0xb
-#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH__SHIFT                                                            0x11
-#define CB_HW_CONTROL_1__RMI_CREDITS__SHIFT                                                                   0x1a
-#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS_MASK                                                               0x0000001FL
-#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS_MASK                                                               0x000007E0L
-#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS_MASK                                                               0x0001F800L
-#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH_MASK                                                              0x03FE0000L
-#define CB_HW_CONTROL_1__RMI_CREDITS_MASK                                                                     0xFC000000L
-//CB_HW_CONTROL_2
-#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH__SHIFT                                                        0x0
-#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH__SHIFT                                                      0x8
-#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH__SHIFT                                                      0xf
-#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8__SHIFT                                                   0x18
-#define CB_HW_CONTROL_2__CHICKEN_BITS__SHIFT                                                                  0x1c
-#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH_MASK                                                          0x000000FFL
-#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH_MASK                                                        0x00007F00L
-#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH_MASK                                                        0x007F8000L
-#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8_MASK                                                     0x0F000000L
-#define CB_HW_CONTROL_2__CHICKEN_BITS_MASK                                                                    0xF0000000L
-//CB_HW_CONTROL_3
-#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL__SHIFT                                        0x0
-#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED__SHIFT                                              0x1
-#define CB_HW_CONTROL_3__DISABLE_FAST_CLEAR_FETCH_OPT__SHIFT                                                  0x2
-#define CB_HW_CONTROL_3__DISABLE_QUAD_MARKER_DROP_STOP__SHIFT                                                 0x3
-#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_CAM_CLR__SHIFT                                            0x4
-#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM__SHIFT                                            0x5
-#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_KEY_MOD__SHIFT                                                 0x6
-#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING__SHIFT                                                 0x7
-#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_TARGET_MASK_VALIDATION__SHIFT                             0x8
-#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS__SHIFT                                                 0x9
-#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS__SHIFT                                                     0xa
-#define CB_HW_CONTROL_3__DISABLE_CMASK_LAST_QUAD_INSERTION__SHIFT                                             0xb
-#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_511967__SHIFT                                              0xc
-#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_520657__SHIFT                                              0xd
-#define CB_HW_CONTROL_3__DISABLE_OC_FIXES_OF_BUG_522542__SHIFT                                                0xe
-#define CB_HW_CONTROL_3__FORCE_RMI_LAST_HIGH__SHIFT                                                           0xf
-#define CB_HW_CONTROL_3__FORCE_RMI_CLKEN_HIGH__SHIFT                                                          0x10
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CC__SHIFT                                                       0x11
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_FC__SHIFT                                                       0x12
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_DC__SHIFT                                                       0x13
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CM__SHIFT                                                       0x14
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CC__SHIFT                                                    0x15
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_FC__SHIFT                                                    0x16
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_DC__SHIFT                                                    0x17
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CM__SHIFT                                                    0x18
-#define CB_HW_CONTROL_3__DISABLE_NACK_COLOR_RD_WR_OPT__SHIFT                                                  0x19
-#define CB_HW_CONTROL_3__DISABLE_BLENDER_CLOCK_GATING__SHIFT                                                  0x1a
-#define CB_HW_CONTROL_3__DISABLE_DUALSRC_WITH_OBJPRIMID_FIX__SHIFT                                            0x1b
-#define CB_HW_CONTROL_3__COLOR_CACHE_PREFETCH_NUM_CLS__SHIFT                                                  0x1c
-#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL_MASK                                          0x00000001L
-#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED_MASK                                                0x00000002L
-#define CB_HW_CONTROL_3__DISABLE_FAST_CLEAR_FETCH_OPT_MASK                                                    0x00000004L
-#define CB_HW_CONTROL_3__DISABLE_QUAD_MARKER_DROP_STOP_MASK                                                   0x00000008L
-#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_CAM_CLR_MASK                                              0x00000010L
-#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM_MASK                                              0x00000020L
-#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_KEY_MOD_MASK                                                   0x00000040L
-#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING_MASK                                                   0x00000080L
-#define CB_HW_CONTROL_3__DISABLE_OVERWRITE_COMBINER_TARGET_MASK_VALIDATION_MASK                               0x00000100L
-#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS_MASK                                                   0x00000200L
-#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS_MASK                                                       0x00000400L
-#define CB_HW_CONTROL_3__DISABLE_CMASK_LAST_QUAD_INSERTION_MASK                                               0x00000800L
-#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_511967_MASK                                                0x00001000L
-#define CB_HW_CONTROL_3__DISABLE_ROP3_FIXES_OF_BUG_520657_MASK                                                0x00002000L
-#define CB_HW_CONTROL_3__DISABLE_OC_FIXES_OF_BUG_522542_MASK                                                  0x00004000L
-#define CB_HW_CONTROL_3__FORCE_RMI_LAST_HIGH_MASK                                                             0x00008000L
-#define CB_HW_CONTROL_3__FORCE_RMI_CLKEN_HIGH_MASK                                                            0x00010000L
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CC_MASK                                                         0x00020000L
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_FC_MASK                                                         0x00040000L
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_DC_MASK                                                         0x00080000L
-#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CM_MASK                                                         0x00100000L
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CC_MASK                                                      0x00200000L
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_FC_MASK                                                      0x00400000L
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_DC_MASK                                                      0x00800000L
-#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CM_MASK                                                      0x01000000L
-#define CB_HW_CONTROL_3__DISABLE_NACK_COLOR_RD_WR_OPT_MASK                                                    0x02000000L
-#define CB_HW_CONTROL_3__DISABLE_BLENDER_CLOCK_GATING_MASK                                                    0x04000000L
-#define CB_HW_CONTROL_3__DISABLE_DUALSRC_WITH_OBJPRIMID_FIX_MASK                                              0x08000000L
-#define CB_HW_CONTROL_3__COLOR_CACHE_PREFETCH_NUM_CLS_MASK                                                    0x30000000L
-//CB_HW_MEM_ARBITER_RD
-#define CB_HW_MEM_ARBITER_RD__MODE__SHIFT                                                                     0x0
-#define CB_HW_MEM_ARBITER_RD__IGNORE_URGENT_AGE__SHIFT                                                        0x2
-#define CB_HW_MEM_ARBITER_RD__BREAK_GROUP_AGE__SHIFT                                                          0x6
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_CC__SHIFT                                                                0xa
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_FC__SHIFT                                                                0xc
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_CM__SHIFT                                                                0xe
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_DC__SHIFT                                                                0x10
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_REQS__SHIFT                                                        0x12
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_NOREQS__SHIFT                                                      0x14
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_IGNORE_NUM_TIDS__SHIFT                                                   0x16
-#define CB_HW_MEM_ARBITER_RD__SCALE_AGE__SHIFT                                                                0x17
-#define CB_HW_MEM_ARBITER_RD__SCALE_WEIGHT__SHIFT                                                             0x1a
-#define CB_HW_MEM_ARBITER_RD__SEND_LASTS_WITHIN_GROUPS__SHIFT                                                 0x1d
-#define CB_HW_MEM_ARBITER_RD__MODE_MASK                                                                       0x00000003L
-#define CB_HW_MEM_ARBITER_RD__IGNORE_URGENT_AGE_MASK                                                          0x0000003CL
-#define CB_HW_MEM_ARBITER_RD__BREAK_GROUP_AGE_MASK                                                            0x000003C0L
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_CC_MASK                                                                  0x00000C00L
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_FC_MASK                                                                  0x00003000L
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_CM_MASK                                                                  0x0000C000L
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_DC_MASK                                                                  0x00030000L
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_REQS_MASK                                                          0x000C0000L
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_NOREQS_MASK                                                        0x00300000L
-#define CB_HW_MEM_ARBITER_RD__WEIGHT_IGNORE_NUM_TIDS_MASK                                                     0x00400000L
-#define CB_HW_MEM_ARBITER_RD__SCALE_AGE_MASK                                                                  0x03800000L
-#define CB_HW_MEM_ARBITER_RD__SCALE_WEIGHT_MASK                                                               0x1C000000L
-#define CB_HW_MEM_ARBITER_RD__SEND_LASTS_WITHIN_GROUPS_MASK                                                   0x20000000L
-//CB_HW_MEM_ARBITER_WR
-#define CB_HW_MEM_ARBITER_WR__MODE__SHIFT                                                                     0x0
-#define CB_HW_MEM_ARBITER_WR__IGNORE_URGENT_AGE__SHIFT                                                        0x2
-#define CB_HW_MEM_ARBITER_WR__BREAK_GROUP_AGE__SHIFT                                                          0x6
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_CC__SHIFT                                                                0xa
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_FC__SHIFT                                                                0xc
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_CM__SHIFT                                                                0xe
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_DC__SHIFT                                                                0x10
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_REQS__SHIFT                                                        0x12
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_NOREQS__SHIFT                                                      0x14
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_IGNORE_BYTE_MASK__SHIFT                                                  0x16
-#define CB_HW_MEM_ARBITER_WR__SCALE_AGE__SHIFT                                                                0x17
-#define CB_HW_MEM_ARBITER_WR__SCALE_WEIGHT__SHIFT                                                             0x1a
-#define CB_HW_MEM_ARBITER_WR__SEND_LASTS_WITHIN_GROUPS__SHIFT                                                 0x1d
-#define CB_HW_MEM_ARBITER_WR__MODE_MASK                                                                       0x00000003L
-#define CB_HW_MEM_ARBITER_WR__IGNORE_URGENT_AGE_MASK                                                          0x0000003CL
-#define CB_HW_MEM_ARBITER_WR__BREAK_GROUP_AGE_MASK                                                            0x000003C0L
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_CC_MASK                                                                  0x00000C00L
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_FC_MASK                                                                  0x00003000L
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_CM_MASK                                                                  0x0000C000L
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_DC_MASK                                                                  0x00030000L
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_REQS_MASK                                                          0x000C0000L
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_NOREQS_MASK                                                        0x00300000L
-#define CB_HW_MEM_ARBITER_WR__WEIGHT_IGNORE_BYTE_MASK_MASK                                                    0x00400000L
-#define CB_HW_MEM_ARBITER_WR__SCALE_AGE_MASK                                                                  0x03800000L
-#define CB_HW_MEM_ARBITER_WR__SCALE_WEIGHT_MASK                                                               0x1C000000L
-#define CB_HW_MEM_ARBITER_WR__SEND_LASTS_WITHIN_GROUPS_MASK                                                   0x20000000L
-//CB_DCC_CONFIG
-#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DEPTH__SHIFT                                                        0x0
-#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DISABLE__SHIFT                                                      0x5
-#define CB_DCC_CONFIG__OVERWRITE_COMBINER_CC_POP_DISABLE__SHIFT                                               0x6
-#define CB_DCC_CONFIG__FC_RDLAT_KEYID_FIFO_DEPTH__SHIFT                                                       0x8
-#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH__SHIFT                                                     0x10
-#define CB_DCC_CONFIG__DCC_CACHE_EVICT_POINT__SHIFT                                                           0x18
-#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS__SHIFT                                                              0x1c
-#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DEPTH_MASK                                                          0x0000001FL
-#define CB_DCC_CONFIG__OVERWRITE_COMBINER_DISABLE_MASK                                                        0x00000020L
-#define CB_DCC_CONFIG__OVERWRITE_COMBINER_CC_POP_DISABLE_MASK                                                 0x00000040L
-#define CB_DCC_CONFIG__FC_RDLAT_KEYID_FIFO_DEPTH_MASK                                                         0x0000FF00L
-#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH_MASK                                                       0x007F0000L
-#define CB_DCC_CONFIG__DCC_CACHE_EVICT_POINT_MASK                                                             0x0F000000L
-#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS_MASK                                                                0xF0000000L
-//GC_USER_RB_REDUNDANCY
-#define GC_USER_RB_REDUNDANCY__FAILED_RB0__SHIFT                                                              0x8
-#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT                                                          0xc
-#define GC_USER_RB_REDUNDANCY__FAILED_RB1__SHIFT                                                              0x10
-#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT                                                          0x14
-#define GC_USER_RB_REDUNDANCY__FAILED_RB0_MASK                                                                0x00000F00L
-#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0_MASK                                                            0x00001000L
-#define GC_USER_RB_REDUNDANCY__FAILED_RB1_MASK                                                                0x000F0000L
-#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1_MASK                                                            0x00100000L
-//GC_USER_RB_BACKEND_DISABLE
-#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                    0x10
-#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                      0x00FF0000L
-
-
-// addressBlock: gc_ea_gceadec2
-//GCEA_EDC_CNT
-#define GCEA_EDC_CNT__DRAMRD_CMDMEM_SEC_COUNT__SHIFT                                                          0x0
-#define GCEA_EDC_CNT__DRAMRD_CMDMEM_DED_COUNT__SHIFT                                                          0x2
-#define GCEA_EDC_CNT__DRAMWR_CMDMEM_SEC_COUNT__SHIFT                                                          0x4
-#define GCEA_EDC_CNT__DRAMWR_CMDMEM_DED_COUNT__SHIFT                                                          0x6
-#define GCEA_EDC_CNT__DRAMWR_DATAMEM_SEC_COUNT__SHIFT                                                         0x8
-#define GCEA_EDC_CNT__DRAMWR_DATAMEM_DED_COUNT__SHIFT                                                         0xa
-#define GCEA_EDC_CNT__RRET_TAGMEM_SEC_COUNT__SHIFT                                                            0xc
-#define GCEA_EDC_CNT__RRET_TAGMEM_DED_COUNT__SHIFT                                                            0xe
-#define GCEA_EDC_CNT__WRET_TAGMEM_SEC_COUNT__SHIFT                                                            0x10
-#define GCEA_EDC_CNT__WRET_TAGMEM_DED_COUNT__SHIFT                                                            0x12
-#define GCEA_EDC_CNT__DRAMRD_PAGEMEM_SED_COUNT__SHIFT                                                         0x14
-#define GCEA_EDC_CNT__DRAMWR_PAGEMEM_SED_COUNT__SHIFT                                                         0x16
-#define GCEA_EDC_CNT__IORD_CMDMEM_SED_COUNT__SHIFT                                                            0x18
-#define GCEA_EDC_CNT__IOWR_CMDMEM_SED_COUNT__SHIFT                                                            0x1a
-#define GCEA_EDC_CNT__IOWR_DATAMEM_SED_COUNT__SHIFT                                                           0x1c
-#define GCEA_EDC_CNT__DRAMRD_CMDMEM_SEC_COUNT_MASK                                                            0x00000003L
-#define GCEA_EDC_CNT__DRAMRD_CMDMEM_DED_COUNT_MASK                                                            0x0000000CL
-#define GCEA_EDC_CNT__DRAMWR_CMDMEM_SEC_COUNT_MASK                                                            0x00000030L
-#define GCEA_EDC_CNT__DRAMWR_CMDMEM_DED_COUNT_MASK                                                            0x000000C0L
-#define GCEA_EDC_CNT__DRAMWR_DATAMEM_SEC_COUNT_MASK                                                           0x00000300L
-#define GCEA_EDC_CNT__DRAMWR_DATAMEM_DED_COUNT_MASK                                                           0x00000C00L
-#define GCEA_EDC_CNT__RRET_TAGMEM_SEC_COUNT_MASK                                                              0x00003000L
-#define GCEA_EDC_CNT__RRET_TAGMEM_DED_COUNT_MASK                                                              0x0000C000L
-#define GCEA_EDC_CNT__WRET_TAGMEM_SEC_COUNT_MASK                                                              0x00030000L
-#define GCEA_EDC_CNT__WRET_TAGMEM_DED_COUNT_MASK                                                              0x000C0000L
-#define GCEA_EDC_CNT__DRAMRD_PAGEMEM_SED_COUNT_MASK                                                           0x00300000L
-#define GCEA_EDC_CNT__DRAMWR_PAGEMEM_SED_COUNT_MASK                                                           0x00C00000L
-#define GCEA_EDC_CNT__IORD_CMDMEM_SED_COUNT_MASK                                                              0x03000000L
-#define GCEA_EDC_CNT__IOWR_CMDMEM_SED_COUNT_MASK                                                              0x0C000000L
-#define GCEA_EDC_CNT__IOWR_DATAMEM_SED_COUNT_MASK                                                             0x30000000L
-//GCEA_EDC_CNT2
-#define GCEA_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT__SHIFT                                                          0x0
-#define GCEA_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT__SHIFT                                                          0x2
-#define GCEA_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT__SHIFT                                                          0x4
-#define GCEA_EDC_CNT2__GMIWR_CMDMEM_DED_COUNT__SHIFT                                                          0x6
-#define GCEA_EDC_CNT2__GMIWR_DATAMEM_SEC_COUNT__SHIFT                                                         0x8
-#define GCEA_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT                                                         0xa
-#define GCEA_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT                                                         0xc
-#define GCEA_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT                                                         0xe
-#define GCEA_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK                                                            0x00000003L
-#define GCEA_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK                                                            0x0000000CL
-#define GCEA_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK                                                            0x00000030L
-#define GCEA_EDC_CNT2__GMIWR_CMDMEM_DED_COUNT_MASK                                                            0x000000C0L
-#define GCEA_EDC_CNT2__GMIWR_DATAMEM_SEC_COUNT_MASK                                                           0x00000300L
-#define GCEA_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK                                                           0x00000C00L
-#define GCEA_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK                                                           0x00003000L
-#define GCEA_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK                                                           0x0000C000L
-//GCEA_DSM_CNTL
-#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x0
-#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x2
-#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x3
-#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x5
-#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x6
-#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
-#define GCEA_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
-#define GCEA_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
-#define GCEA_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                  0xc
-#define GCEA_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xe
-#define GCEA_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xf
-#define GCEA_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x11
-#define GCEA_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x12
-#define GCEA_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x14
-#define GCEA_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x15
-#define GCEA_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x17
-#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000003L
-#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000004L
-#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000018L
-#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000020L
-#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x000000C0L
-#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
-#define GCEA_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
-#define GCEA_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
-#define GCEA_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                    0x00003000L
-#define GCEA_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00004000L
-#define GCEA_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00018000L
-#define GCEA_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00020000L
-#define GCEA_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x000C0000L
-#define GCEA_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00100000L
-#define GCEA_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00600000L
-#define GCEA_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00800000L
-//GCEA_DSM_CNTLA
-#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x0
-#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x2
-#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x3
-#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x5
-#define GCEA_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x6
-#define GCEA_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x8
-#define GCEA_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
-#define GCEA_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
-#define GCEA_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xc
-#define GCEA_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xe
-#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xf
-#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x11
-#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x12
-#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x14
-#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00000003L
-#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000004L
-#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00000018L
-#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000020L
-#define GCEA_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x000000C0L
-#define GCEA_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000100L
-#define GCEA_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
-#define GCEA_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
-#define GCEA_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00003000L
-#define GCEA_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00004000L
-#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00018000L
-#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00020000L
-#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                 0x000C0000L
-#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00100000L
-//GCEA_DSM_CNTLB
-//GCEA_DSM_CNTL2
-#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x0
-#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x2
-#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x3
-#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x5
-#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x6
-#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x8
-#define GCEA_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
-#define GCEA_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
-#define GCEA_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                                0xc
-#define GCEA_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                                0xe
-#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xf
-#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x11
-#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x12
-#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x14
-#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x15
-#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                              0x17
-#define GCEA_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                   0x1a
-#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000003L
-#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000004L
-#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000018L
-#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000020L
-#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
-#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00000100L
-#define GCEA_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
-#define GCEA_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
-#define GCEA_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                  0x00003000L
-#define GCEA_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                  0x00004000L
-#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00018000L
-#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00020000L
-#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x000C0000L
-#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00100000L
-#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                                0x00600000L
-#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                                0x00800000L
-#define GCEA_DSM_CNTL2__INJECT_DELAY_MASK                                                                     0xFC000000L
-//GCEA_DSM_CNTL2A
-#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x0
-#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x2
-#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x3
-#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x5
-#define GCEA_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x6
-#define GCEA_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x8
-#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
-#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
-#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xc
-#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                              0xe
-#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xf
-#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                             0x11
-#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x12
-#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                             0x14
-#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00000003L
-#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00000004L
-#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00000018L
-#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00000020L
-#define GCEA_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x000000C0L
-#define GCEA_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000100L
-#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
-#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
-#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                                0x00003000L
-#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                                0x00004000L
-#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                               0x00018000L
-#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                               0x00020000L
-#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                               0x000C0000L
-#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                               0x00100000L
-//GCEA_DSM_CNTL2B
-//GCEA_TCC_XBR_CREDITS
-#define GCEA_TCC_XBR_CREDITS__DRAM_RD_LIMIT__SHIFT                                                            0x0
-#define GCEA_TCC_XBR_CREDITS__DRAM_RD_RESERVE__SHIFT                                                          0x6
-#define GCEA_TCC_XBR_CREDITS__IO_RD_LIMIT__SHIFT                                                              0x8
-#define GCEA_TCC_XBR_CREDITS__IO_RD_RESERVE__SHIFT                                                            0xe
-#define GCEA_TCC_XBR_CREDITS__DRAM_WR_LIMIT__SHIFT                                                            0x10
-#define GCEA_TCC_XBR_CREDITS__DRAM_WR_RESERVE__SHIFT                                                          0x16
-#define GCEA_TCC_XBR_CREDITS__IO_WR_LIMIT__SHIFT                                                              0x18
-#define GCEA_TCC_XBR_CREDITS__IO_WR_RESERVE__SHIFT                                                            0x1e
-#define GCEA_TCC_XBR_CREDITS__DRAM_RD_LIMIT_MASK                                                              0x0000003FL
-#define GCEA_TCC_XBR_CREDITS__DRAM_RD_RESERVE_MASK                                                            0x000000C0L
-#define GCEA_TCC_XBR_CREDITS__IO_RD_LIMIT_MASK                                                                0x00003F00L
-#define GCEA_TCC_XBR_CREDITS__IO_RD_RESERVE_MASK                                                              0x0000C000L
-#define GCEA_TCC_XBR_CREDITS__DRAM_WR_LIMIT_MASK                                                              0x003F0000L
-#define GCEA_TCC_XBR_CREDITS__DRAM_WR_RESERVE_MASK                                                            0x00C00000L
-#define GCEA_TCC_XBR_CREDITS__IO_WR_LIMIT_MASK                                                                0x3F000000L
-#define GCEA_TCC_XBR_CREDITS__IO_WR_RESERVE_MASK                                                              0xC0000000L
-//GCEA_TCC_XBR_MAXBURST
-#define GCEA_TCC_XBR_MAXBURST__DRAM_RD__SHIFT                                                                 0x0
-#define GCEA_TCC_XBR_MAXBURST__IO_RD__SHIFT                                                                   0x4
-#define GCEA_TCC_XBR_MAXBURST__DRAM_WR__SHIFT                                                                 0x8
-#define GCEA_TCC_XBR_MAXBURST__IO_WR__SHIFT                                                                   0xc
-#define GCEA_TCC_XBR_MAXBURST__DRAM_RD_MASK                                                                   0x0000000FL
-#define GCEA_TCC_XBR_MAXBURST__IO_RD_MASK                                                                     0x000000F0L
-#define GCEA_TCC_XBR_MAXBURST__DRAM_WR_MASK                                                                   0x00000F00L
-#define GCEA_TCC_XBR_MAXBURST__IO_WR_MASK                                                                     0x0000F000L
-//GCEA_PROBE_CNTL
-#define GCEA_PROBE_CNTL__REQ2RSP_DELAY__SHIFT                                                                 0x0
-#define GCEA_PROBE_CNTL__PRB_FILTER_DISABLE__SHIFT                                                            0x5
-#define GCEA_PROBE_CNTL__REQ2RSP_DELAY_MASK                                                                   0x0000001FL
-#define GCEA_PROBE_CNTL__PRB_FILTER_DISABLE_MASK                                                              0x00000020L
-//GCEA_PROBE_MAP
-#define GCEA_PROBE_MAP__CHADDR0_TO_RIGHTTCC__SHIFT                                                            0x0
-#define GCEA_PROBE_MAP__CHADDR1_TO_RIGHTTCC__SHIFT                                                            0x1
-#define GCEA_PROBE_MAP__CHADDR2_TO_RIGHTTCC__SHIFT                                                            0x2
-#define GCEA_PROBE_MAP__CHADDR3_TO_RIGHTTCC__SHIFT                                                            0x3
-#define GCEA_PROBE_MAP__CHADDR4_TO_RIGHTTCC__SHIFT                                                            0x4
-#define GCEA_PROBE_MAP__CHADDR5_TO_RIGHTTCC__SHIFT                                                            0x5
-#define GCEA_PROBE_MAP__CHADDR6_TO_RIGHTTCC__SHIFT                                                            0x6
-#define GCEA_PROBE_MAP__CHADDR7_TO_RIGHTTCC__SHIFT                                                            0x7
-#define GCEA_PROBE_MAP__CHADDR8_TO_RIGHTTCC__SHIFT                                                            0x8
-#define GCEA_PROBE_MAP__CHADDR9_TO_RIGHTTCC__SHIFT                                                            0x9
-#define GCEA_PROBE_MAP__CHADDR10_TO_RIGHTTCC__SHIFT                                                           0xa
-#define GCEA_PROBE_MAP__CHADDR11_TO_RIGHTTCC__SHIFT                                                           0xb
-#define GCEA_PROBE_MAP__CHADDR12_TO_RIGHTTCC__SHIFT                                                           0xc
-#define GCEA_PROBE_MAP__CHADDR13_TO_RIGHTTCC__SHIFT                                                           0xd
-#define GCEA_PROBE_MAP__CHADDR14_TO_RIGHTTCC__SHIFT                                                           0xe
-#define GCEA_PROBE_MAP__CHADDR15_TO_RIGHTTCC__SHIFT                                                           0xf
-#define GCEA_PROBE_MAP__INTLV_SIZE__SHIFT                                                                     0x10
-#define GCEA_PROBE_MAP__CHADDR0_TO_RIGHTTCC_MASK                                                              0x00000001L
-#define GCEA_PROBE_MAP__CHADDR1_TO_RIGHTTCC_MASK                                                              0x00000002L
-#define GCEA_PROBE_MAP__CHADDR2_TO_RIGHTTCC_MASK                                                              0x00000004L
-#define GCEA_PROBE_MAP__CHADDR3_TO_RIGHTTCC_MASK                                                              0x00000008L
-#define GCEA_PROBE_MAP__CHADDR4_TO_RIGHTTCC_MASK                                                              0x00000010L
-#define GCEA_PROBE_MAP__CHADDR5_TO_RIGHTTCC_MASK                                                              0x00000020L
-#define GCEA_PROBE_MAP__CHADDR6_TO_RIGHTTCC_MASK                                                              0x00000040L
-#define GCEA_PROBE_MAP__CHADDR7_TO_RIGHTTCC_MASK                                                              0x00000080L
-#define GCEA_PROBE_MAP__CHADDR8_TO_RIGHTTCC_MASK                                                              0x00000100L
-#define GCEA_PROBE_MAP__CHADDR9_TO_RIGHTTCC_MASK                                                              0x00000200L
-#define GCEA_PROBE_MAP__CHADDR10_TO_RIGHTTCC_MASK                                                             0x00000400L
-#define GCEA_PROBE_MAP__CHADDR11_TO_RIGHTTCC_MASK                                                             0x00000800L
-#define GCEA_PROBE_MAP__CHADDR12_TO_RIGHTTCC_MASK                                                             0x00001000L
-#define GCEA_PROBE_MAP__CHADDR13_TO_RIGHTTCC_MASK                                                             0x00002000L
-#define GCEA_PROBE_MAP__CHADDR14_TO_RIGHTTCC_MASK                                                             0x00004000L
-#define GCEA_PROBE_MAP__CHADDR15_TO_RIGHTTCC_MASK                                                             0x00008000L
-#define GCEA_PROBE_MAP__INTLV_SIZE_MASK                                                                       0x00030000L
-//GCEA_ERR_STATUS
-#define GCEA_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                              0x0
-#define GCEA_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                              0x4
-#define GCEA_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                    0x8
-#define GCEA_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                            0x9
-#define GCEA_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                 0xa
-#define GCEA_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                                0x0000000FL
-#define GCEA_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                                0x000000F0L
-#define GCEA_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                      0x00000100L
-#define GCEA_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                              0x00000200L
-#define GCEA_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                   0x00000400L
-//GCEA_MISC2
-#define GCEA_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                           0x0
-#define GCEA_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                            0x1
-#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                        0x2
-#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                         0x7
-#define GCEA_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                             0x00000001L
-#define GCEA_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                              0x00000002L
-#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                          0x0000007CL
-#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                           0x00000F80L
-//GCEA_SDP_BACKDOOR_CMDCREDITS0
-#define GCEA_SDP_BACKDOOR_CMDCREDITS0__CREDITS_RECEIVED__SHIFT                                                0x0
-#define GCEA_SDP_BACKDOOR_CMDCREDITS0__CREDITS_RECEIVED_MASK                                                  0xFFFFFFFFL
-//GCEA_SDP_BACKDOOR_CMDCREDITS1
-#define GCEA_SDP_BACKDOOR_CMDCREDITS1__CREDITS_RECEIVED__SHIFT                                                0x0
-#define GCEA_SDP_BACKDOOR_CMDCREDITS1__CREDITS_RECEIVED_MASK                                                  0x7FFFFFFFL
-//GCEA_SDP_BACKDOOR_DATACREDITS0
-#define GCEA_SDP_BACKDOOR_DATACREDITS0__CREDITS_RECEIVED__SHIFT                                               0x0
-#define GCEA_SDP_BACKDOOR_DATACREDITS0__CREDITS_RECEIVED_MASK                                                 0xFFFFFFFFL
-//GCEA_SDP_BACKDOOR_DATACREDITS1
-#define GCEA_SDP_BACKDOOR_DATACREDITS1__CREDITS_RECEIVED__SHIFT                                               0x0
-#define GCEA_SDP_BACKDOOR_DATACREDITS1__CREDITS_RECEIVED_MASK                                                 0x7FFFFFFFL
-//GCEA_SDP_BACKDOOR_MISCCREDITS
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__RDRSP_CREDITS_RELEASED__SHIFT                                          0x0
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__WRRSP_CREDITS_RELEASED__SHIFT                                          0x8
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_REQ_CREDITS_RELEASED__SHIFT                                        0x10
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_RSP_CREDITS_RECEIVED__SHIFT                                        0x17
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__RDRSP_CREDITS_RELEASED_MASK                                            0x000000FFL
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__WRRSP_CREDITS_RELEASED_MASK                                            0x0000FF00L
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_REQ_CREDITS_RELEASED_MASK                                          0x007F0000L
-#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_RSP_CREDITS_RECEIVED_MASK                                          0x3F800000L
-//GCEA_SDP_ENABLE
-#define GCEA_SDP_ENABLE__ENABLE__SHIFT                                                                        0x0
-#define GCEA_SDP_ENABLE__ENABLE_MASK                                                                          0x00000001L
-
-
-// addressBlock: gc_rmi_rmidec
-//RMI_GENERAL_CNTL
-#define RMI_GENERAL_CNTL__BURST_DISABLE__SHIFT                                                                0x0
-#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE__SHIFT                                                           0x1
-#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG__SHIFT                                                              0x11
-#define RMI_GENERAL_CNTL__RB0_HARVEST_EN__SHIFT                                                               0x13
-#define RMI_GENERAL_CNTL__RB1_HARVEST_EN__SHIFT                                                               0x14
-#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE__SHIFT                                                     0x15
-#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG_UPDATE__SHIFT                                                       0x19
-#define RMI_GENERAL_CNTL__SKID_FIFO_0_OVERFLOW_ERROR_MASK__SHIFT                                              0x1a
-#define RMI_GENERAL_CNTL__SKID_FIFO_0_UNDERFLOW_ERROR_MASK__SHIFT                                             0x1b
-#define RMI_GENERAL_CNTL__SKID_FIFO_1_OVERFLOW_ERROR_MASK__SHIFT                                              0x1c
-#define RMI_GENERAL_CNTL__SKID_FIFO_1_UNDERFLOW_ERROR_MASK__SHIFT                                             0x1d
-#define RMI_GENERAL_CNTL__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK__SHIFT                                       0x1e
-#define RMI_GENERAL_CNTL__BURST_DISABLE_MASK                                                                  0x00000001L
-#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE_MASK                                                             0x0001FFFEL
-#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG_MASK                                                                0x00060000L
-#define RMI_GENERAL_CNTL__RB0_HARVEST_EN_MASK                                                                 0x00080000L
-#define RMI_GENERAL_CNTL__RB1_HARVEST_EN_MASK                                                                 0x00100000L
-#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE_MASK                                                       0x01E00000L
-#define RMI_GENERAL_CNTL__XBAR_MUX_CONFIG_UPDATE_MASK                                                         0x02000000L
-#define RMI_GENERAL_CNTL__SKID_FIFO_0_OVERFLOW_ERROR_MASK_MASK                                                0x04000000L
-#define RMI_GENERAL_CNTL__SKID_FIFO_0_UNDERFLOW_ERROR_MASK_MASK                                               0x08000000L
-#define RMI_GENERAL_CNTL__SKID_FIFO_1_OVERFLOW_ERROR_MASK_MASK                                                0x10000000L
-#define RMI_GENERAL_CNTL__SKID_FIFO_1_UNDERFLOW_ERROR_MASK_MASK                                               0x20000000L
-#define RMI_GENERAL_CNTL__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK_MASK                                         0x40000000L
-//RMI_GENERAL_CNTL1
-#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE__SHIFT                                                0x0
-#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE__SHIFT                                                     0x4
-#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE__SHIFT                                                     0x6
-#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK__SHIFT                                            0x8
-#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE__SHIFT                                                       0x9
-#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE__SHIFT                                                             0xa
-#define RMI_GENERAL_CNTL1__UTCL1_PROBE0_RR_ARB_BURST_HINT_EN__SHIFT                                           0xb
-#define RMI_GENERAL_CNTL1__UTCL1_PROBE1_RR_ARB_BURST_HINT_EN__SHIFT                                           0xc
-#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE_MASK                                                  0x0000000FL
-#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE_MASK                                                       0x00000030L
-#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE_MASK                                                       0x000000C0L
-#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK_MASK                                              0x00000100L
-#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE_MASK                                                         0x00000200L
-#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_MASK                                                               0x00000400L
-#define RMI_GENERAL_CNTL1__UTCL1_PROBE0_RR_ARB_BURST_HINT_EN_MASK                                             0x00000800L
-#define RMI_GENERAL_CNTL1__UTCL1_PROBE1_RR_ARB_BURST_HINT_EN_MASK                                             0x00001000L
-//RMI_GENERAL_STATUS
-#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED__SHIFT                                                0x0
-#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR__SHIFT                                                 0x1
-#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR__SHIFT                                                0x2
-#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR__SHIFT                                                 0x3
-#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR__SHIFT                                                0x4
-#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY__SHIFT                                                              0x5
-#define RMI_GENERAL_STATUS__RMI_UTCL1_BUSY__SHIFT                                                             0x6
-#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY__SHIFT                                                        0x7
-#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY__SHIFT                                                        0x8
-#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY__SHIFT                                                           0x9
-#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY__SHIFT                                                       0xa
-#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xb
-#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xc
-#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY__SHIFT                                                        0xd
-#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY__SHIFT                                                           0xe
-#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY__SHIFT                                                       0xf
-#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_1_BUSY__SHIFT                                                 0x10
-#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_1_BUSY__SHIFT                                                 0x11
-#define RMI_GENERAL_STATUS__UTC_PROBE1_BUSY__SHIFT                                                            0x12
-#define RMI_GENERAL_STATUS__UTC_PROBE0_BUSY__SHIFT                                                            0x13
-#define RMI_GENERAL_STATUS__RMI_XNACK_BUSY__SHIFT                                                             0x14
-#define RMI_GENERAL_STATUS__XNACK_FIFO_NUM_USED__SHIFT                                                        0x15
-#define RMI_GENERAL_STATUS__XNACK_FIFO_EMPTY__SHIFT                                                           0x1d
-#define RMI_GENERAL_STATUS__XNACK_FIFO_FULL__SHIFT                                                            0x1e
-#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR__SHIFT                                          0x1f
-#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED_MASK                                                  0x00000001L
-#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR_MASK                                                   0x00000002L
-#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR_MASK                                                  0x00000004L
-#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR_MASK                                                   0x00000008L
-#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR_MASK                                                  0x00000010L
-#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY_MASK                                                                0x00000020L
-#define RMI_GENERAL_STATUS__RMI_UTCL1_BUSY_MASK                                                               0x00000040L
-#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY_MASK                                                          0x00000080L
-#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY_MASK                                                          0x00000100L
-#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY_MASK                                                             0x00000200L
-#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY_MASK                                                         0x00000400L
-#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00000800L
-#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00001000L
-#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY_MASK                                                          0x00002000L
-#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY_MASK                                                             0x00004000L
-#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY_MASK                                                         0x00008000L
-#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_1_BUSY_MASK                                                   0x00010000L
-#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_1_BUSY_MASK                                                   0x00020000L
-#define RMI_GENERAL_STATUS__UTC_PROBE1_BUSY_MASK                                                              0x00040000L
-#define RMI_GENERAL_STATUS__UTC_PROBE0_BUSY_MASK                                                              0x00080000L
-#define RMI_GENERAL_STATUS__RMI_XNACK_BUSY_MASK                                                               0x00100000L
-#define RMI_GENERAL_STATUS__XNACK_FIFO_NUM_USED_MASK                                                          0x1FE00000L
-#define RMI_GENERAL_STATUS__XNACK_FIFO_EMPTY_MASK                                                             0x20000000L
-#define RMI_GENERAL_STATUS__XNACK_FIFO_FULL_MASK                                                              0x40000000L
-#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK                                            0x80000000L
-//RMI_SUBBLOCK_STATUS0
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0__SHIFT                                     0x0
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0__SHIFT                                         0x7
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0__SHIFT                                        0x8
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1__SHIFT                                     0x9
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1__SHIFT                                         0x10
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1__SHIFT                                        0x11
-#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT__SHIFT                                                       0x12
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0_MASK                                       0x0000007FL
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0_MASK                                           0x00000080L
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0_MASK                                          0x00000100L
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1_MASK                                       0x0000FE00L
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1_MASK                                           0x00010000L
-#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1_MASK                                          0x00020000L
-#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT_MASK                                                         0x0FFC0000L
-//RMI_SUBBLOCK_STATUS1
-#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE__SHIFT                                                   0x0
-#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE__SHIFT                                                   0xa
-#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT__SHIFT                                                       0x14
-#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE_MASK                                                     0x000003FFL
-#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE_MASK                                                     0x000FFC00L
-#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT_MASK                                                         0x3FF00000L
-//RMI_SUBBLOCK_STATUS2
-#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED__SHIFT                                                      0x0
-#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED__SHIFT                                                      0x9
-#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED_MASK                                                        0x000001FFL
-#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED_MASK                                                        0x0003FE00L
-//RMI_SUBBLOCK_STATUS3
-#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL__SHIFT                                             0x0
-#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL__SHIFT                                             0xa
-#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL_MASK                                               0x000003FFL
-#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL_MASK                                               0x000FFC00L
-//RMI_XBAR_CONFIG
-#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE__SHIFT                                                      0x0
-#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE__SHIFT                                             0x2
-#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE__SHIFT                                                0x6
-#define RMI_XBAR_CONFIG__ARBITER_DIS__SHIFT                                                                   0x7
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ__SHIFT                                                                0x8
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE__SHIFT                                                       0xc
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0__SHIFT                                                                0xd
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB1__SHIFT                                                                0xe
-#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE_MASK                                                        0x00000003L
-#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE_MASK                                               0x0000003CL
-#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE_MASK                                                  0x00000040L
-#define RMI_XBAR_CONFIG__ARBITER_DIS_MASK                                                                     0x00000080L
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_MASK                                                                  0x00000F00L
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE_MASK                                                         0x00001000L
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0_MASK                                                                  0x00002000L
-#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB1_MASK                                                                  0x00004000L
-//RMI_PROBE_POP_LOGIC_CNTL
-#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH__SHIFT                                             0x0
-#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS__SHIFT                                                    0x7
-#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2__SHIFT                                      0x8
-#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH__SHIFT                                             0xa
-#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS__SHIFT                                                    0x11
-#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH_MASK                                               0x0000007FL
-#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS_MASK                                                      0x00000080L
-#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2_MASK                                        0x00000300L
-#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH_MASK                                               0x0001FC00L
-#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS_MASK                                                      0x00020000L
-//RMI_UTC_XNACK_N_MISC_CNTL
-#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC__SHIFT                                              0x0
-#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE__SHIFT                                         0x8
-#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE__SHIFT                                                     0xc
-#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE__SHIFT                                       0xd
-#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC_MASK                                                0x000000FFL
-#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE_MASK                                           0x00000F00L
-#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE_MASK                                                       0x00001000L
-#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE_MASK                                         0x00002000L
-//RMI_DEMUX_CNTL
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL__SHIFT                                                               0x0
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_BREAK_LOB_ON_IDLEIN__SHIFT                                                 0x1
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_OVERRIDE__SHIFT                                                0x4
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE__SHIFT                                             0x6
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE__SHIFT                                                                0xe
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL__SHIFT                                                               0x10
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_BREAK_LOB_ON_IDLEIN__SHIFT                                                 0x11
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_OVERRIDE__SHIFT                                                0x14
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE__SHIFT                                             0x16
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE__SHIFT                                                                0x1e
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_MASK                                                                 0x00000001L
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_BREAK_LOB_ON_IDLEIN_MASK                                                   0x00000002L
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_OVERRIDE_MASK                                                  0x00000030L
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE_MASK                                               0x00003FC0L
-#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_MASK                                                                  0x0000C000L
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_MASK                                                                 0x00010000L
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_BREAK_LOB_ON_IDLEIN_MASK                                                   0x00020000L
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_OVERRIDE_MASK                                                  0x00300000L
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE_MASK                                               0x3FC00000L
-#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_MASK                                                                  0xC0000000L
-//RMI_UTCL1_CNTL1
-#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
-#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                                 0x1
-#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
-#define RMI_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                     0x3
-#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
-#define RMI_UTCL1_CNTL1__CLIENTID__SHIFT                                                                      0x7
-#define RMI_UTCL1_CNTL1__USERVM_DIS__SHIFT                                                                    0x10
-#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                             0x11
-#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                          0x12
-#define RMI_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
-#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                              0x17
-#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
-#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                                    0x19
-#define RMI_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
-#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                0x1b
-#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
-#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
-#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
-#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                                   0x00000002L
-#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
-#define RMI_UTCL1_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
-#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
-#define RMI_UTCL1_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
-#define RMI_UTCL1_CNTL1__USERVM_DIS_MASK                                                                      0x00010000L
-#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                               0x00020000L
-#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                            0x00040000L
-#define RMI_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
-#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                0x00800000L
-#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
-#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                                      0x02000000L
-#define RMI_UTCL1_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
-#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                  0x08000000L
-#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
-#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
-//RMI_UTCL1_CNTL2
-#define RMI_UTCL1_CNTL2__UTC_SPARE__SHIFT                                                                     0x0
-#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
-#define RMI_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                    0xa
-#define RMI_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                       0xb
-#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                0xc
-#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                                 0xd
-#define RMI_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
-#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                           0xf
-#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE__SHIFT                                                          0x10
-#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR__SHIFT                                                 0x12
-#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR__SHIFT                                                        0x13
-#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID__SHIFT                                                  0x14
-#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID__SHIFT                                                         0x15
-#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ__SHIFT                                                         0x19
-#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K__SHIFT                                                    0x1a
-#define RMI_UTCL1_CNTL2__UTC_SPARE_MASK                                                                       0x000000FFL
-#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
-#define RMI_UTCL1_CNTL2__LINE_VALID_MASK                                                                      0x00000400L
-#define RMI_UTCL1_CNTL2__DIS_EDC_MASK                                                                         0x00000800L
-#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                  0x00001000L
-#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                                   0x00002000L
-#define RMI_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
-#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                             0x00008000L
-#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE_MASK                                                            0x00030000L
-#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR_MASK                                                   0x00040000L
-#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR_MASK                                                          0x00080000L
-#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID_MASK                                                    0x00100000L
-#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID_MASK                                                           0x01E00000L
-#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ_MASK                                                           0x02000000L
-#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K_MASK                                                      0x04000000L
-//RMI_UTC_UNIT_CONFIG
-//RMI_TCIW_FORMATTER0_CNTL
-#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_OVERRIDE__SHIFT                                             0x0
-#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_TIME_OUT_WINDOW__SHIFT                                          0x1
-#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
-#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA__SHIFT                                         0x13
-#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA_UPDATE__SHIFT                                  0x1b
-#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_REQ_SAFE_MODE__SHIFT                                                  0x1c
-#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS__SHIFT                                                  0x1d
-#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_AT_LAST_OF_BURST__SHIFT                                     0x1e
-#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA__SHIFT                                                  0x1f
-#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_OVERRIDE_MASK                                               0x00000001L
-#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_TIME_OUT_WINDOW_MASK                                            0x000001FEL
-#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
-#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA_MASK                                           0x07F80000L
-#define RMI_TCIW_FORMATTER0_CNTL__SKID_FIFO_0_FREE_SPACE_DELTA_UPDATE_MASK                                    0x08000000L
-#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_REQ_SAFE_MODE_MASK                                                    0x10000000L
-#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS_MASK                                                    0x20000000L
-#define RMI_TCIW_FORMATTER0_CNTL__WR_COMBINE0_DIS_AT_LAST_OF_BURST_MASK                                       0x40000000L
-#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA_MASK                                                    0x80000000L
-//RMI_TCIW_FORMATTER1_CNTL
-#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE__SHIFT                                             0x0
-#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW__SHIFT                                          0x1
-#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
-#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA__SHIFT                                         0x13
-#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA_UPDATE__SHIFT                                  0x1b
-#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_REQ_SAFE_MODE__SHIFT                                                  0x1c
-#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS__SHIFT                                                  0x1d
-#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST__SHIFT                                     0x1e
-#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA__SHIFT                                                  0x1f
-#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE_MASK                                               0x00000001L
-#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW_MASK                                            0x000001FEL
-#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
-#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA_MASK                                           0x07F80000L
-#define RMI_TCIW_FORMATTER1_CNTL__SKID_FIFO_1_FREE_SPACE_DELTA_UPDATE_MASK                                    0x08000000L
-#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_REQ_SAFE_MODE_MASK                                                    0x10000000L
-#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS_MASK                                                    0x20000000L
-#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST_MASK                                       0x40000000L
-#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA_MASK                                                    0x80000000L
-//RMI_SCOREBOARD_CNTL
-#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH__SHIFT                                                        0x0
-#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0__SHIFT                                              0x1
-#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH__SHIFT                                                        0x2
-#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1__SHIFT                                              0x3
-#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB1__SHIFT                                                      0x4
-#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN__SHIFT                                         0x5
-#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE__SHIFT                                      0x6
-#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB0__SHIFT                                                      0x7
-#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_EN__SHIFT                                                  0x8
-#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE__SHIFT                                   0x9
-#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH_MASK                                                          0x00000001L
-#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0_MASK                                                0x00000002L
-#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH_MASK                                                          0x00000004L
-#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1_MASK                                                0x00000008L
-#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB1_MASK                                                        0x00000010L
-#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN_MASK                                           0x00000020L
-#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE_MASK                                        0x00000040L
-#define RMI_SCOREBOARD_CNTL__TIME_STAMP_FLUSH_RB0_MASK                                                        0x00000080L
-#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_EN_MASK                                                    0x00000100L
-#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE_MASK                                     0x001FFE00L
-//RMI_SCOREBOARD_STATUS0
-#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID__SHIFT                                                     0x0
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG__SHIFT                                                    0x1
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID__SHIFT                                                   0x2
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE__SHIFT                                                   0x12
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE__SHIFT                                                       0x13
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE__SHIFT                                                 0x14
-#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE__SHIFT                                                    0x15
-#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID_MASK                                                       0x00000001L
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG_MASK                                                      0x00000002L
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID_MASK                                                     0x0003FFFCL
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE_MASK                                                     0x00040000L
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE_MASK                                                         0x00080000L
-#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE_MASK                                                   0x00100000L
-#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE_MASK                                                      0x00200000L
-//RMI_SCOREBOARD_STATUS1
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0__SHIFT                                                        0x0
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0__SHIFT                                              0xc
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0__SHIFT                                               0xd
-#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED__SHIFT                                      0xe
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1__SHIFT                                                        0xf
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1__SHIFT                                              0x1b
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1__SHIFT                                               0x1c
-#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1__SHIFT                                                  0x1d
-#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0__SHIFT                                                  0x1e
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0_MASK                                                          0x00000FFFL
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0_MASK                                                0x00001000L
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0_MASK                                                 0x00002000L
-#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED_MASK                                        0x00004000L
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1_MASK                                                          0x07FF8000L
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1_MASK                                                0x08000000L
-#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1_MASK                                                 0x10000000L
-#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1_MASK                                                    0x20000000L
-#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0_MASK                                                    0x40000000L
-//RMI_SCOREBOARD_STATUS2
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0__SHIFT                                                       0x0
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0__SHIFT                                             0xc
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1__SHIFT                                                       0xd
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1__SHIFT                                             0x19
-#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1__SHIFT                                                     0x1a
-#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0__SHIFT                                                     0x1b
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0__SHIFT                                           0x1c
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1__SHIFT                                           0x1d
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0__SHIFT                                              0x1e
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1__SHIFT                                              0x1f
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0_MASK                                                         0x00000FFFL
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0_MASK                                               0x00001000L
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1_MASK                                                         0x01FFE000L
-#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1_MASK                                               0x02000000L
-#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1_MASK                                                       0x04000000L
-#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0_MASK                                                       0x08000000L
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0_MASK                                             0x10000000L
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1_MASK                                             0x20000000L
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0_MASK                                                0x40000000L
-#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1_MASK                                                0x80000000L
-//RMI_XBAR_ARBITER_CONFIG
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE__SHIFT                                                        0x0
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x2
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL__SHIFT                                                       0x3
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x4
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE__SHIFT                                        0x6
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE__SHIFT                                     0x8
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE__SHIFT                                                        0x10
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x12
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL__SHIFT                                                       0x13
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x14
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE__SHIFT                                        0x16
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE__SHIFT                                     0x18
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_MASK                                                          0x00000003L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00000004L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_MASK                                                         0x00000008L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN_MASK                                           0x00000010L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE_MASK                                          0x000000C0L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE_MASK                                       0x0000FF00L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_MASK                                                          0x00030000L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00040000L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_MASK                                                         0x00080000L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN_MASK                                           0x00100000L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE_MASK                                          0x00C00000L
-#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE_MASK                                       0xFF000000L
-//RMI_XBAR_ARBITER_CONFIG_1
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD__SHIFT                                  0x0
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR__SHIFT                                  0x8
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_RD__SHIFT                                  0x10
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_WR__SHIFT                                  0x18
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD_MASK                                    0x000000FFL
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR_MASK                                    0x0000FF00L
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_RD_MASK                                    0x00FF0000L
-#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB1_WR_MASK                                    0xFF000000L
-//RMI_CLOCK_CNTRL
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK__SHIFT                                                         0x0
-#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK__SHIFT                                                         0x5
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK__SHIFT                                                       0xa
-#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK__SHIFT                                                       0xf
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_BUSY_MASK__SHIFT                                                         0x14
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_WAKEUP_MASK__SHIFT                                                       0x19
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK_MASK                                                           0x0000001FL
-#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK_MASK                                                           0x000003E0L
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK_MASK                                                         0x00007C00L
-#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK_MASK                                                         0x000F8000L
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_BUSY_MASK_MASK                                                           0x01F00000L
-#define RMI_CLOCK_CNTRL__DYN_CLK_RB1_WAKEUP_MASK_MASK                                                         0x3E000000L
-//RMI_UTCL1_STATUS
-#define RMI_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
-#define RMI_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
-#define RMI_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
-#define RMI_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
-#define RMI_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
-#define RMI_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
-//RMI_SPARE
-#define RMI_SPARE__RMI_ARBITER_STALL_TIMER_ENABLED_ALLOW_STREAMING__SHIFT                                     0x0
-#define RMI_SPARE__SPARE_BIT_1__SHIFT                                                                         0x1
-#define RMI_SPARE__SPARE_BIT_2__SHIFT                                                                         0x2
-#define RMI_SPARE__SPARE_BIT_3__SHIFT                                                                         0x3
-#define RMI_SPARE__SPARE_BIT_4__SHIFT                                                                         0x4
-#define RMI_SPARE__SPARE_BIT_5__SHIFT                                                                         0x5
-#define RMI_SPARE__SPARE_BIT_6__SHIFT                                                                         0x6
-#define RMI_SPARE__SPARE_BIT_7__SHIFT                                                                         0x7
-#define RMI_SPARE__SPARE_BIT_8_0__SHIFT                                                                       0x8
-#define RMI_SPARE__SPARE_BIT_16_0__SHIFT                                                                      0x10
-#define RMI_SPARE__RMI_ARBITER_STALL_TIMER_ENABLED_ALLOW_STREAMING_MASK                                       0x00000001L
-#define RMI_SPARE__SPARE_BIT_1_MASK                                                                           0x00000002L
-#define RMI_SPARE__SPARE_BIT_2_MASK                                                                           0x00000004L
-#define RMI_SPARE__SPARE_BIT_3_MASK                                                                           0x00000008L
-#define RMI_SPARE__SPARE_BIT_4_MASK                                                                           0x00000010L
-#define RMI_SPARE__SPARE_BIT_5_MASK                                                                           0x00000020L
-#define RMI_SPARE__SPARE_BIT_6_MASK                                                                           0x00000040L
-#define RMI_SPARE__SPARE_BIT_7_MASK                                                                           0x00000080L
-#define RMI_SPARE__SPARE_BIT_8_0_MASK                                                                         0x0000FF00L
-#define RMI_SPARE__SPARE_BIT_16_0_MASK                                                                        0xFFFF0000L
-//RMI_SPARE_1
-#define RMI_SPARE_1__SPARE_BIT_8__SHIFT                                                                       0x0
-#define RMI_SPARE_1__SPARE_BIT_9__SHIFT                                                                       0x1
-#define RMI_SPARE_1__SPARE_BIT_10__SHIFT                                                                      0x2
-#define RMI_SPARE_1__SPARE_BIT_11__SHIFT                                                                      0x3
-#define RMI_SPARE_1__SPARE_BIT_12__SHIFT                                                                      0x4
-#define RMI_SPARE_1__SPARE_BIT_13__SHIFT                                                                      0x5
-#define RMI_SPARE_1__SPARE_BIT_14__SHIFT                                                                      0x6
-#define RMI_SPARE_1__SPARE_BIT_15__SHIFT                                                                      0x7
-#define RMI_SPARE_1__SPARE_BIT_8_1__SHIFT                                                                     0x8
-#define RMI_SPARE_1__SPARE_BIT_16_1__SHIFT                                                                    0x10
-#define RMI_SPARE_1__SPARE_BIT_8_MASK                                                                         0x00000001L
-#define RMI_SPARE_1__SPARE_BIT_9_MASK                                                                         0x00000002L
-#define RMI_SPARE_1__SPARE_BIT_10_MASK                                                                        0x00000004L
-#define RMI_SPARE_1__SPARE_BIT_11_MASK                                                                        0x00000008L
-#define RMI_SPARE_1__SPARE_BIT_12_MASK                                                                        0x00000010L
-#define RMI_SPARE_1__SPARE_BIT_13_MASK                                                                        0x00000020L
-#define RMI_SPARE_1__SPARE_BIT_14_MASK                                                                        0x00000040L
-#define RMI_SPARE_1__SPARE_BIT_15_MASK                                                                        0x00000080L
-#define RMI_SPARE_1__SPARE_BIT_8_1_MASK                                                                       0x0000FF00L
-#define RMI_SPARE_1__SPARE_BIT_16_1_MASK                                                                      0xFFFF0000L
-//RMI_SPARE_2
-#define RMI_SPARE_2__SPARE_BIT_16__SHIFT                                                                      0x0
-#define RMI_SPARE_2__SPARE_BIT_17__SHIFT                                                                      0x1
-#define RMI_SPARE_2__SPARE_BIT_18__SHIFT                                                                      0x2
-#define RMI_SPARE_2__SPARE_BIT_19__SHIFT                                                                      0x3
-#define RMI_SPARE_2__SPARE_BIT_20__SHIFT                                                                      0x4
-#define RMI_SPARE_2__SPARE_BIT_21__SHIFT                                                                      0x5
-#define RMI_SPARE_2__SPARE_BIT_22__SHIFT                                                                      0x6
-#define RMI_SPARE_2__SPARE_BIT_23__SHIFT                                                                      0x7
-#define RMI_SPARE_2__SPARE_BIT_4_0__SHIFT                                                                     0x8
-#define RMI_SPARE_2__SPARE_BIT_4_1__SHIFT                                                                     0xc
-#define RMI_SPARE_2__SPARE_BIT_8_2__SHIFT                                                                     0x10
-#define RMI_SPARE_2__SPARE_BIT_8_3__SHIFT                                                                     0x18
-#define RMI_SPARE_2__SPARE_BIT_16_MASK                                                                        0x00000001L
-#define RMI_SPARE_2__SPARE_BIT_17_MASK                                                                        0x00000002L
-#define RMI_SPARE_2__SPARE_BIT_18_MASK                                                                        0x00000004L
-#define RMI_SPARE_2__SPARE_BIT_19_MASK                                                                        0x00000008L
-#define RMI_SPARE_2__SPARE_BIT_20_MASK                                                                        0x00000010L
-#define RMI_SPARE_2__SPARE_BIT_21_MASK                                                                        0x00000020L
-#define RMI_SPARE_2__SPARE_BIT_22_MASK                                                                        0x00000040L
-#define RMI_SPARE_2__SPARE_BIT_23_MASK                                                                        0x00000080L
-#define RMI_SPARE_2__SPARE_BIT_4_0_MASK                                                                       0x00000F00L
-#define RMI_SPARE_2__SPARE_BIT_4_1_MASK                                                                       0x0000F000L
-#define RMI_SPARE_2__SPARE_BIT_8_2_MASK                                                                       0x00FF0000L
-#define RMI_SPARE_2__SPARE_BIT_8_3_MASK                                                                       0xFF000000L
-
-
-// addressBlock: gc_dbgu_gfx_dbgudec
-//port_a_addr
-#define port_a_addr__Index__SHIFT                                                                             0x0
-#define port_a_addr__Reserved__SHIFT                                                                          0x8
-#define port_a_addr__ReadEnable__SHIFT                                                                        0x1f
-#define port_a_addr__Index_MASK                                                                               0x000000FFL
-#define port_a_addr__Reserved_MASK                                                                            0x7FFFFF00L
-#define port_a_addr__ReadEnable_MASK                                                                          0x80000000L
-//port_a_data_lo
-#define port_a_data_lo__Data__SHIFT                                                                           0x0
-#define port_a_data_lo__Data_MASK                                                                             0xFFFFFFFFL
-//port_a_data_hi
-#define port_a_data_hi__Data__SHIFT                                                                           0x0
-#define port_a_data_hi__Data_MASK                                                                             0xFFFFFFFFL
-//port_b_addr
-#define port_b_addr__Index__SHIFT                                                                             0x0
-#define port_b_addr__Reserved__SHIFT                                                                          0x8
-#define port_b_addr__ReadEnable__SHIFT                                                                        0x1f
-#define port_b_addr__Index_MASK                                                                               0x000000FFL
-#define port_b_addr__Reserved_MASK                                                                            0x7FFFFF00L
-#define port_b_addr__ReadEnable_MASK                                                                          0x80000000L
-//port_b_data_lo
-#define port_b_data_lo__Data__SHIFT                                                                           0x0
-#define port_b_data_lo__Data_MASK                                                                             0xFFFFFFFFL
-//port_b_data_hi
-#define port_b_data_hi__Data__SHIFT                                                                           0x0
-#define port_b_data_hi__Data_MASK                                                                             0xFFFFFFFFL
-//port_c_addr
-#define port_c_addr__Index__SHIFT                                                                             0x0
-#define port_c_addr__Reserved__SHIFT                                                                          0x8
-#define port_c_addr__ReadEnable__SHIFT                                                                        0x1f
-#define port_c_addr__Index_MASK                                                                               0x000000FFL
-#define port_c_addr__Reserved_MASK                                                                            0x7FFFFF00L
-#define port_c_addr__ReadEnable_MASK                                                                          0x80000000L
-//port_c_data_lo
-#define port_c_data_lo__Data__SHIFT                                                                           0x0
-#define port_c_data_lo__Data_MASK                                                                             0xFFFFFFFFL
-//port_c_data_hi
-#define port_c_data_hi__Data__SHIFT                                                                           0x0
-#define port_c_data_hi__Data_MASK                                                                             0xFFFFFFFFL
-//port_d_addr
-#define port_d_addr__Index__SHIFT                                                                             0x0
-#define port_d_addr__Reserved__SHIFT                                                                          0x8
-#define port_d_addr__ReadEnable__SHIFT                                                                        0x1f
-#define port_d_addr__Index_MASK                                                                               0x000000FFL
-#define port_d_addr__Reserved_MASK                                                                            0x7FFFFF00L
-#define port_d_addr__ReadEnable_MASK                                                                          0x80000000L
-//port_d_data_lo
-#define port_d_data_lo__Data__SHIFT                                                                           0x0
-#define port_d_data_lo__Data_MASK                                                                             0xFFFFFFFFL
-//port_d_data_hi
-#define port_d_data_hi__Data__SHIFT                                                                           0x0
-#define port_d_data_hi__Data_MASK                                                                             0xFFFFFFFFL
-
-
-// addressBlock: gc_utcl2_atcl2dec
-//ATC_L2_CNTL
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS__SHIFT                                               0x0
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS__SHIFT                                              0x3
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                                   0x6
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                                  0x7
-#define ATC_L2_CNTL__CACHE_INVALIDATE_MODE__SHIFT                                                             0x8
-#define ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                          0xb
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS_MASK                                                 0x00000003L
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS_MASK                                                0x00000018L
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                     0x00000040L
-#define ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                                    0x00000080L
-#define ATC_L2_CNTL__CACHE_INVALIDATE_MODE_MASK                                                               0x00000700L
-#define ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                            0x00000800L
-//ATC_L2_CNTL2
-#define ATC_L2_CNTL2__BANK_SELECT__SHIFT                                                                      0x0
-#define ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE__SHIFT                                                             0x6
-#define ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                              0x8
-#define ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS__SHIFT                                                     0x9
-#define ATC_L2_CNTL2__L2_CACHE_VMID_MODE__SHIFT                                                               0xc
-#define ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                         0xf
-#define ATC_L2_CNTL2__BANK_SELECT_MASK                                                                        0x0000003FL
-#define ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE_MASK                                                               0x000000C0L
-#define ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE_MASK                                                0x00000100L
-#define ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS_MASK                                                       0x00000E00L
-#define ATC_L2_CNTL2__L2_CACHE_VMID_MODE_MASK                                                                 0x00007000L
-#define ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                           0x001F8000L
-//ATC_L2_CACHE_DATA0
-#define ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID__SHIFT                                                        0x0
-#define ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID__SHIFT                                                          0x1
-#define ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES__SHIFT                                                          0x2
-#define ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH__SHIFT                                                  0x17
-#define ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID_MASK                                                          0x00000001L
-#define ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID_MASK                                                            0x00000002L
-#define ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES_MASK                                                            0x007FFFFCL
-#define ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH_MASK                                                    0x07800000L
-//ATC_L2_CACHE_DATA1
-#define ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW__SHIFT                                                   0x0
-#define ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW_MASK                                                     0xFFFFFFFFL
-//ATC_L2_CACHE_DATA2
-#define ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS__SHIFT                                                      0x0
-#define ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS_MASK                                                        0xFFFFFFFFL
-//ATC_L2_CNTL3
-#define ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST__SHIFT                                                  0x0
-#define ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1__SHIFT                                                        0x3
-#define ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST_MASK                                                    0x00000007L
-#define ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1_MASK                                                          0x000001F8L
-//ATC_L2_STATUS
-#define ATC_L2_STATUS__BUSY__SHIFT                                                                            0x0
-#define ATC_L2_STATUS__PARITY_ERROR_INFO__SHIFT                                                               0x1
-#define ATC_L2_STATUS__BUSY_MASK                                                                              0x00000001L
-#define ATC_L2_STATUS__PARITY_ERROR_INFO_MASK                                                                 0x3FFFFFFEL
-//ATC_L2_STATUS2
-#define ATC_L2_STATUS2__IFIFO_NON_FATAL_PARITY_ERROR_INFO__SHIFT                                              0x0
-#define ATC_L2_STATUS2__IFIFO_FATAL_PARITY_ERROR_INFO__SHIFT                                                  0x8
-#define ATC_L2_STATUS2__IFIFO_NON_FATAL_PARITY_ERROR_INFO_MASK                                                0x000000FFL
-#define ATC_L2_STATUS2__IFIFO_FATAL_PARITY_ERROR_INFO_MASK                                                    0x0000FF00L
-//ATC_L2_MISC_CG
-#define ATC_L2_MISC_CG__OFFDLY__SHIFT                                                                         0x6
-#define ATC_L2_MISC_CG__ENABLE__SHIFT                                                                         0x12
-#define ATC_L2_MISC_CG__MEM_LS_ENABLE__SHIFT                                                                  0x13
-#define ATC_L2_MISC_CG__OFFDLY_MASK                                                                           0x00000FC0L
-#define ATC_L2_MISC_CG__ENABLE_MASK                                                                           0x00040000L
-#define ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK                                                                    0x00080000L
-//ATC_L2_MEM_POWER_LS
-#define ATC_L2_MEM_POWER_LS__LS_SETUP__SHIFT                                                                  0x0
-#define ATC_L2_MEM_POWER_LS__LS_HOLD__SHIFT                                                                   0x6
-#define ATC_L2_MEM_POWER_LS__LS_SETUP_MASK                                                                    0x0000003FL
-#define ATC_L2_MEM_POWER_LS__LS_HOLD_MASK                                                                     0x00000FC0L
-//ATC_L2_CGTT_CLK_CTRL
-#define ATC_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
-#define ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
-#define ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                            0xf
-#define ATC_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                      0x10
-#define ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                            0x18
-#define ATC_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
-#define ATC_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
-#define ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                              0x00008000L
-#define ATC_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                        0x00FF0000L
-#define ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                              0xFF000000L
-
-
-// addressBlock: gc_utcl2_vml2pfdec
-//VM_L2_CNTL
-#define VM_L2_CNTL__ENABLE_L2_CACHE__SHIFT                                                                    0x0
-#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING__SHIFT                                                      0x1
-#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE__SHIFT                                                      0x2
-#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE__SHIFT                                                      0x4
-#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE__SHIFT                                                  0x8
-#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                            0x9
-#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                           0xa
-#define VM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                           0xb
-#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE__SHIFT                                                           0xc
-#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT                                                            0xf
-#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION__SHIFT                                                           0x12
-#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT                                                      0x13
-#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE__SHIFT                                                        0x15
-#define VM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE__SHIFT                                                             0x1a
-#define VM_L2_CNTL__ENABLE_L2_CACHE_MASK                                                                      0x00000001L
-#define VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK                                                        0x00000002L
-#define VM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE_MASK                                                        0x0000000CL
-#define VM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE_MASK                                                        0x00000030L
-#define VM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE_MASK                                                    0x00000100L
-#define VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK                                              0x00000200L
-#define VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK                                             0x00000400L
-#define VM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                             0x00000800L
-#define VM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE_MASK                                                             0x00007000L
-#define VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE_MASK                                                              0x00038000L
-#define VM_L2_CNTL__PDE_FAULT_CLASSIFICATION_MASK                                                             0x00040000L
-#define VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE_MASK                                                        0x00180000L
-#define VM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE_MASK                                                          0x03E00000L
-#define VM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE_MASK                                                               0x0C000000L
-//VM_L2_CNTL2
-#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS__SHIFT                                                            0x0
-#define VM_L2_CNTL2__INVALIDATE_L2_CACHE__SHIFT                                                               0x1
-#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN__SHIFT                                                     0x15
-#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION__SHIFT                                                   0x16
-#define VM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE__SHIFT                                                            0x17
-#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE__SHIFT                                                             0x1a
-#define VM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE__SHIFT                                                          0x1c
-#define VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK                                                              0x00000001L
-#define VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK                                                                 0x00000002L
-#define VM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN_MASK                                                       0x00200000L
-#define VM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION_MASK                                                     0x00400000L
-#define VM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE_MASK                                                              0x03800000L
-#define VM_L2_CNTL2__INVALIDATE_CACHE_MODE_MASK                                                               0x0C000000L
-#define VM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE_MASK                                                            0x70000000L
-//VM_L2_CNTL3
-#define VM_L2_CNTL3__BANK_SELECT__SHIFT                                                                       0x0
-#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE__SHIFT                                                              0x6
-#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                          0x8
-#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                                                       0xf
-#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY__SHIFT                                                       0x14
-#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE__SHIFT                                                        0x15
-#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE__SHIFT                                                      0x18
-#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS__SHIFT                                                            0x1c
-#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS__SHIFT                                                          0x1d
-#define VM_L2_CNTL3__PDE_CACHE_FORCE_MISS__SHIFT                                                              0x1e
-#define VM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY__SHIFT                                                         0x1f
-#define VM_L2_CNTL3__BANK_SELECT_MASK                                                                         0x0000003FL
-#define VM_L2_CNTL3__L2_CACHE_UPDATE_MODE_MASK                                                                0x000000C0L
-#define VM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                            0x00001F00L
-#define VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                                                         0x000F8000L
-#define VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK                                                         0x00100000L
-#define VM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE_MASK                                                          0x00E00000L
-#define VM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE_MASK                                                        0x0F000000L
-#define VM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS_MASK                                                              0x10000000L
-#define VM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS_MASK                                                            0x20000000L
-#define VM_L2_CNTL3__PDE_CACHE_FORCE_MISS_MASK                                                                0x40000000L
-#define VM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY_MASK                                                           0x80000000L
-//VM_L2_STATUS
-#define VM_L2_STATUS__L2_BUSY__SHIFT                                                                          0x0
-#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY__SHIFT                                                              0x1
-#define VM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS__SHIFT                                                 0x11
-#define VM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS__SHIFT                                               0x12
-#define VM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS__SHIFT                                                   0x13
-#define VM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS__SHIFT                                                   0x14
-#define VM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS__SHIFT                                                   0x15
-#define VM_L2_STATUS__L2_BUSY_MASK                                                                            0x00000001L
-#define VM_L2_STATUS__CONTEXT_DOMAIN_BUSY_MASK                                                                0x0001FFFEL
-#define VM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS_MASK                                                   0x00020000L
-#define VM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS_MASK                                                 0x00040000L
-#define VM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS_MASK                                                     0x00080000L
-#define VM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS_MASK                                                     0x00100000L
-#define VM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS_MASK                                                     0x00200000L
-//VM_DUMMY_PAGE_FAULT_CNTL
-#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE__SHIFT                                              0x0
-#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL__SHIFT                                           0x1
-#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS__SHIFT                                              0x2
-#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE_MASK                                                0x00000001L
-#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL_MASK                                             0x00000002L
-#define VM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS_MASK                                                0x000000FCL
-//VM_DUMMY_PAGE_FAULT_ADDR_LO32
-#define VM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32__SHIFT                                            0x0
-#define VM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32_MASK                                              0xFFFFFFFFL
-//VM_DUMMY_PAGE_FAULT_ADDR_HI32
-#define VM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4__SHIFT                                             0x0
-#define VM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4_MASK                                               0x0000000FL
-//VM_L2_PROTECTION_FAULT_CNTL
-#define VM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                0x0
-#define VM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT             0x1
-#define VM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0x2
-#define VM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x3
-#define VM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x4
-#define VM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x5
-#define VM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                 0x6
-#define VM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0x7
-#define VM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                        0x8
-#define VM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0x9
-#define VM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                              0xa
-#define VM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                             0xb
-#define VM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                           0xc
-#define VM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                                0xd
-#define VM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                          0x1d
-#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT__SHIFT                                           0x1e
-#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT__SHIFT                                              0x1f
-#define VM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                  0x00000001L
-#define VM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK               0x00000002L
-#define VM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000004L
-#define VM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000008L
-#define VM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000010L
-#define VM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000020L
-#define VM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                   0x00000040L
-#define VM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000080L
-#define VM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                          0x00000100L
-#define VM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000200L
-#define VM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                0x00000400L
-#define VM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                               0x00000800L
-#define VM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                             0x00001000L
-#define VM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                                  0x1FFFE000L
-#define VM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                            0x20000000L
-#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT_MASK                                             0x40000000L
-#define VM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT_MASK                                                0x80000000L
-//VM_L2_PROTECTION_FAULT_CNTL2
-#define VM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                                    0x0
-#define VM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                              0x10
-#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE__SHIFT                                        0x11
-#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY__SHIFT                             0x12
-#define VM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT__SHIFT                                     0x13
-#define VM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                      0x0000FFFFL
-#define VM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                0x00010000L
-#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_MASK                                          0x00020000L
-#define VM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY_MASK                               0x00040000L
-#define VM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT_MASK                                       0x00080000L
-//VM_L2_PROTECTION_FAULT_MM_CNTL3
-#define VM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                  0x0
-#define VM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                    0xFFFFFFFFL
-//VM_L2_PROTECTION_FAULT_MM_CNTL4
-#define VM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                 0x0
-#define VM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                   0xFFFFFFFFL
-//VM_L2_PROTECTION_FAULT_STATUS
-#define VM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS__SHIFT                                                     0x0
-#define VM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR__SHIFT                                                    0x1
-#define VM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS__SHIFT                                               0x4
-#define VM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR__SHIFT                                                   0x8
-#define VM_L2_PROTECTION_FAULT_STATUS__CID__SHIFT                                                             0x9
-#define VM_L2_PROTECTION_FAULT_STATUS__RW__SHIFT                                                              0x12
-#define VM_L2_PROTECTION_FAULT_STATUS__ATOMIC__SHIFT                                                          0x13
-#define VM_L2_PROTECTION_FAULT_STATUS__VMID__SHIFT                                                            0x14
-#define VM_L2_PROTECTION_FAULT_STATUS__VF__SHIFT                                                              0x18
-#define VM_L2_PROTECTION_FAULT_STATUS__VFID__SHIFT                                                            0x19
-#define VM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS_MASK                                                       0x00000001L
-#define VM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR_MASK                                                      0x0000000EL
-#define VM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS_MASK                                                 0x000000F0L
-#define VM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR_MASK                                                     0x00000100L
-#define VM_L2_PROTECTION_FAULT_STATUS__CID_MASK                                                               0x0003FE00L
-#define VM_L2_PROTECTION_FAULT_STATUS__RW_MASK                                                                0x00040000L
-#define VM_L2_PROTECTION_FAULT_STATUS__ATOMIC_MASK                                                            0x00080000L
-#define VM_L2_PROTECTION_FAULT_STATUS__VMID_MASK                                                              0x00F00000L
-#define VM_L2_PROTECTION_FAULT_STATUS__VF_MASK                                                                0x01000000L
-#define VM_L2_PROTECTION_FAULT_STATUS__VFID_MASK                                                              0x1E000000L
-//VM_L2_PROTECTION_FAULT_ADDR_LO32
-#define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32__SHIFT                                       0x0
-#define VM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32_MASK                                         0xFFFFFFFFL
-//VM_L2_PROTECTION_FAULT_ADDR_HI32
-#define VM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4__SHIFT                                        0x0
-#define VM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4_MASK                                          0x0000000FL
-//VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32
-#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32__SHIFT                              0x0
-#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32_MASK                                0xFFFFFFFFL
-//VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32
-#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4__SHIFT                               0x0
-#define VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4_MASK                                 0x0000000FL
-//VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                       0x0
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                         0xFFFFFFFFL
-//VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                        0x0
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                          0x0000000FL
-//VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                      0x0
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                        0xFFFFFFFFL
-//VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                       0x0
-#define VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                         0x0000000FL
-//VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32
-#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32__SHIFT                         0x0
-#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32_MASK                           0xFFFFFFFFL
-//VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32
-#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4__SHIFT                          0x0
-#define VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4_MASK                            0x0000000FL
-//VM_L2_CNTL4
-#define VM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT__SHIFT                                                       0x0
-#define VM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL__SHIFT                                                      0x6
-#define VM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL__SHIFT                                                      0x7
-#define VM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                           0x8
-#define VM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                          0x12
-#define VM_L2_CNTL4__BPM_CGCGLS_OVERRIDE__SHIFT                                                               0x1c
-#define VM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT_MASK                                                         0x0000003FL
-#define VM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL_MASK                                                        0x00000040L
-#define VM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL_MASK                                                        0x00000080L
-#define VM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                             0x0003FF00L
-#define VM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                            0x0FFC0000L
-#define VM_L2_CNTL4__BPM_CGCGLS_OVERRIDE_MASK                                                                 0x10000000L
-//VM_L2_MM_GROUP_RT_CLASSES
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS__SHIFT                                                    0x0
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS__SHIFT                                                    0x1
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS__SHIFT                                                    0x2
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS__SHIFT                                                    0x3
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS__SHIFT                                                    0x4
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS__SHIFT                                                    0x5
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS__SHIFT                                                    0x6
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS__SHIFT                                                    0x7
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS__SHIFT                                                    0x8
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS__SHIFT                                                    0x9
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS__SHIFT                                                   0xa
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS__SHIFT                                                   0xb
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS__SHIFT                                                   0xc
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS__SHIFT                                                   0xd
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS__SHIFT                                                   0xe
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS__SHIFT                                                   0xf
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS__SHIFT                                                   0x10
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS__SHIFT                                                   0x11
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS__SHIFT                                                   0x12
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS__SHIFT                                                   0x13
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS__SHIFT                                                   0x14
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS__SHIFT                                                   0x15
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS__SHIFT                                                   0x16
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS__SHIFT                                                   0x17
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS__SHIFT                                                   0x18
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS__SHIFT                                                   0x19
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS__SHIFT                                                   0x1a
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS__SHIFT                                                   0x1b
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS__SHIFT                                                   0x1c
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS__SHIFT                                                   0x1d
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS__SHIFT                                                   0x1e
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS__SHIFT                                                   0x1f
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS_MASK                                                      0x00000001L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS_MASK                                                      0x00000002L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS_MASK                                                      0x00000004L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS_MASK                                                      0x00000008L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS_MASK                                                      0x00000010L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS_MASK                                                      0x00000020L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS_MASK                                                      0x00000040L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS_MASK                                                      0x00000080L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS_MASK                                                      0x00000100L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS_MASK                                                      0x00000200L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS_MASK                                                     0x00000400L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS_MASK                                                     0x00000800L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS_MASK                                                     0x00001000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS_MASK                                                     0x00002000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS_MASK                                                     0x00004000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS_MASK                                                     0x00008000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS_MASK                                                     0x00010000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS_MASK                                                     0x00020000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS_MASK                                                     0x00040000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS_MASK                                                     0x00080000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS_MASK                                                     0x00100000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS_MASK                                                     0x00200000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS_MASK                                                     0x00400000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS_MASK                                                     0x00800000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS_MASK                                                     0x01000000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS_MASK                                                     0x02000000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS_MASK                                                     0x04000000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS_MASK                                                     0x08000000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS_MASK                                                     0x10000000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS_MASK                                                     0x20000000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS_MASK                                                     0x40000000L
-#define VM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS_MASK                                                     0x80000000L
-//VM_L2_BANK_SELECT_RESERVED_CID
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID__SHIFT                                        0x0
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID__SHIFT                                       0xa
-#define VM_L2_BANK_SELECT_RESERVED_CID__ENABLE__SHIFT                                                         0x14
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                               0x18
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                            0x19
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID_MASK                                          0x000001FFL
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID_MASK                                         0x0007FC00L
-#define VM_L2_BANK_SELECT_RESERVED_CID__ENABLE_MASK                                                           0x00100000L
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE_MASK                                 0x01000000L
-#define VM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                              0x02000000L
-//VM_L2_BANK_SELECT_RESERVED_CID2
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID__SHIFT                                       0x0
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID__SHIFT                                      0xa
-#define VM_L2_BANK_SELECT_RESERVED_CID2__ENABLE__SHIFT                                                        0x14
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                              0x18
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                           0x19
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID_MASK                                         0x000001FFL
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID_MASK                                        0x0007FC00L
-#define VM_L2_BANK_SELECT_RESERVED_CID2__ENABLE_MASK                                                          0x00100000L
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE_MASK                                0x01000000L
-#define VM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                             0x02000000L
-//VM_L2_CACHE_PARITY_CNTL
-#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES__SHIFT                                 0x0
-#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES__SHIFT                               0x1
-#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES__SHIFT                                    0x2
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE__SHIFT                                 0x3
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE__SHIFT                               0x4
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE__SHIFT                                    0x5
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK__SHIFT                                                      0x6
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER__SHIFT                                                    0x9
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC__SHIFT                                                     0xc
-#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES_MASK                                   0x00000001L
-#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES_MASK                                 0x00000002L
-#define VM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES_MASK                                      0x00000004L
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE_MASK                                   0x00000008L
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE_MASK                                 0x00000010L
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE_MASK                                      0x00000020L
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK_MASK                                                        0x000001C0L
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER_MASK                                                      0x00000E00L
-#define VM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC_MASK                                                       0x0000F000L
-//VM_L2_CGTT_CLK_CTRL
-#define VM_L2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
-#define VM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
-#define VM_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                             0xf
-#define VM_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                       0x10
-#define VM_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                             0x18
-#define VM_L2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
-#define VM_L2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
-#define VM_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                               0x00008000L
-#define VM_L2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                         0x00FF0000L
-#define VM_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                               0xFF000000L
-
-
-// addressBlock: gc_utcl2_vml2vcdec
-//VM_CONTEXT0_CNTL
-#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT0_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT0_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT1_CNTL
-#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT1_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT1_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT2_CNTL
-#define VM_CONTEXT2_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT2_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT2_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT2_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT3_CNTL
-#define VM_CONTEXT3_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT3_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT3_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT3_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT4_CNTL
-#define VM_CONTEXT4_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT4_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT4_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT4_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT5_CNTL
-#define VM_CONTEXT5_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT5_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT5_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT5_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT6_CNTL
-#define VM_CONTEXT6_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT6_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT6_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT6_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT7_CNTL
-#define VM_CONTEXT7_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT7_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT7_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT7_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT8_CNTL
-#define VM_CONTEXT8_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT8_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT8_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT8_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT9_CNTL
-#define VM_CONTEXT9_CNTL__ENABLE_CONTEXT__SHIFT                                                               0x0
-#define VM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                             0x1
-#define VM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                        0x3
-#define VM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                       0x7
-#define VM_CONTEXT9_CNTL__RETRY_OTHER_FAULT__SHIFT                                                            0x8
-#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x9
-#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xa
-#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0xb
-#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0xc
-#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0xd
-#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0xe
-#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xf
-#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x10
-#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                       0x11
-#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                         0x12
-#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x13
-#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x14
-#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x15
-#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x16
-#define VM_CONTEXT9_CNTL__ENABLE_CONTEXT_MASK                                                                 0x00000001L
-#define VM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH_MASK                                                               0x00000006L
-#define VM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                          0x00000078L
-#define VM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                         0x00000080L
-#define VM_CONTEXT9_CNTL__RETRY_OTHER_FAULT_MASK                                                              0x00000100L
-#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00000200L
-#define VM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00000400L
-#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00000800L
-#define VM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00001000L
-#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00002000L
-#define VM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00004000L
-#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00008000L
-#define VM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00010000L
-#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                         0x00020000L
-#define VM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                           0x00040000L
-#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00080000L
-#define VM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00100000L
-#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00200000L
-#define VM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00400000L
-//VM_CONTEXT10_CNTL
-#define VM_CONTEXT10_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
-#define VM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
-#define VM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
-#define VM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
-#define VM_CONTEXT10_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
-#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
-#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
-#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
-#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
-#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
-#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
-#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
-#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
-#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
-#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
-#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
-#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
-#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
-#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
-#define VM_CONTEXT10_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
-#define VM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
-#define VM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
-#define VM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
-#define VM_CONTEXT10_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
-#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
-#define VM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
-#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
-#define VM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
-#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
-#define VM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
-#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
-#define VM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
-#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
-#define VM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
-#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
-#define VM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
-#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
-#define VM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
-//VM_CONTEXT11_CNTL
-#define VM_CONTEXT11_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
-#define VM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
-#define VM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
-#define VM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
-#define VM_CONTEXT11_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
-#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
-#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
-#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
-#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
-#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
-#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
-#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
-#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
-#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
-#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
-#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
-#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
-#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
-#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
-#define VM_CONTEXT11_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
-#define VM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
-#define VM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
-#define VM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
-#define VM_CONTEXT11_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
-#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
-#define VM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
-#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
-#define VM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
-#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
-#define VM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
-#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
-#define VM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
-#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
-#define VM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
-#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
-#define VM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
-#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
-#define VM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
-//VM_CONTEXT12_CNTL
-#define VM_CONTEXT12_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
-#define VM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
-#define VM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
-#define VM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
-#define VM_CONTEXT12_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
-#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
-#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
-#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
-#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
-#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
-#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
-#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
-#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
-#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
-#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
-#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
-#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
-#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
-#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
-#define VM_CONTEXT12_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
-#define VM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
-#define VM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
-#define VM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
-#define VM_CONTEXT12_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
-#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
-#define VM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
-#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
-#define VM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
-#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
-#define VM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
-#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
-#define VM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
-#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
-#define VM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
-#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
-#define VM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
-#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
-#define VM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
-//VM_CONTEXT13_CNTL
-#define VM_CONTEXT13_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
-#define VM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
-#define VM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
-#define VM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
-#define VM_CONTEXT13_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
-#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
-#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
-#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
-#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
-#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
-#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
-#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
-#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
-#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
-#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
-#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
-#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
-#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
-#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
-#define VM_CONTEXT13_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
-#define VM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
-#define VM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
-#define VM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
-#define VM_CONTEXT13_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
-#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
-#define VM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
-#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
-#define VM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
-#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
-#define VM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
-#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
-#define VM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
-#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
-#define VM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
-#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
-#define VM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
-#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
-#define VM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
-//VM_CONTEXT14_CNTL
-#define VM_CONTEXT14_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
-#define VM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
-#define VM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
-#define VM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
-#define VM_CONTEXT14_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
-#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
-#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
-#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
-#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
-#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
-#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
-#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
-#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
-#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
-#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
-#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
-#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
-#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
-#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
-#define VM_CONTEXT14_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
-#define VM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
-#define VM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
-#define VM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
-#define VM_CONTEXT14_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
-#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
-#define VM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
-#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
-#define VM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
-#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
-#define VM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
-#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
-#define VM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
-#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
-#define VM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
-#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
-#define VM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
-#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
-#define VM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
-//VM_CONTEXT15_CNTL
-#define VM_CONTEXT15_CNTL__ENABLE_CONTEXT__SHIFT                                                              0x0
-#define VM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                            0x1
-#define VM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                       0x3
-#define VM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                      0x7
-#define VM_CONTEXT15_CNTL__RETRY_OTHER_FAULT__SHIFT                                                           0x8
-#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x9
-#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xa
-#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                0xb
-#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                  0xc
-#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0xd
-#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0xe
-#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xf
-#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x10
-#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                      0x11
-#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                        0x12
-#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x13
-#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x14
-#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x15
-#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x16
-#define VM_CONTEXT15_CNTL__ENABLE_CONTEXT_MASK                                                                0x00000001L
-#define VM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH_MASK                                                              0x00000006L
-#define VM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                         0x00000078L
-#define VM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                        0x00000080L
-#define VM_CONTEXT15_CNTL__RETRY_OTHER_FAULT_MASK                                                             0x00000100L
-#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00000200L
-#define VM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00000400L
-#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                  0x00000800L
-#define VM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                    0x00001000L
-#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00002000L
-#define VM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00004000L
-#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00008000L
-#define VM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00010000L
-#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                        0x00020000L
-#define VM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                          0x00040000L
-#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00080000L
-#define VM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00100000L
-#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00200000L
-#define VM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00400000L
-//VM_CONTEXTS_DISABLE
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0__SHIFT                                                         0x0
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1__SHIFT                                                         0x1
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2__SHIFT                                                         0x2
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3__SHIFT                                                         0x3
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4__SHIFT                                                         0x4
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5__SHIFT                                                         0x5
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6__SHIFT                                                         0x6
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7__SHIFT                                                         0x7
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8__SHIFT                                                         0x8
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9__SHIFT                                                         0x9
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10__SHIFT                                                        0xa
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11__SHIFT                                                        0xb
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12__SHIFT                                                        0xc
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13__SHIFT                                                        0xd
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14__SHIFT                                                        0xe
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15__SHIFT                                                        0xf
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0_MASK                                                           0x00000001L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1_MASK                                                           0x00000002L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2_MASK                                                           0x00000004L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3_MASK                                                           0x00000008L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4_MASK                                                           0x00000010L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5_MASK                                                           0x00000020L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6_MASK                                                           0x00000040L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7_MASK                                                           0x00000080L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8_MASK                                                           0x00000100L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9_MASK                                                           0x00000200L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10_MASK                                                          0x00000400L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11_MASK                                                          0x00000800L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12_MASK                                                          0x00001000L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13_MASK                                                          0x00002000L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14_MASK                                                          0x00004000L
-#define VM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15_MASK                                                          0x00008000L
-//VM_INVALIDATE_ENG0_SEM
-#define VM_INVALIDATE_ENG0_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG0_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG1_SEM
-#define VM_INVALIDATE_ENG1_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG1_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG2_SEM
-#define VM_INVALIDATE_ENG2_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG2_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG3_SEM
-#define VM_INVALIDATE_ENG3_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG3_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG4_SEM
-#define VM_INVALIDATE_ENG4_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG4_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG5_SEM
-#define VM_INVALIDATE_ENG5_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG5_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG6_SEM
-#define VM_INVALIDATE_ENG6_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG6_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG7_SEM
-#define VM_INVALIDATE_ENG7_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG7_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG8_SEM
-#define VM_INVALIDATE_ENG8_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG8_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG9_SEM
-#define VM_INVALIDATE_ENG9_SEM__SEMAPHORE__SHIFT                                                              0x0
-#define VM_INVALIDATE_ENG9_SEM__SEMAPHORE_MASK                                                                0x00000001L
-//VM_INVALIDATE_ENG10_SEM
-#define VM_INVALIDATE_ENG10_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG10_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG11_SEM
-#define VM_INVALIDATE_ENG11_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG11_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG12_SEM
-#define VM_INVALIDATE_ENG12_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG12_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG13_SEM
-#define VM_INVALIDATE_ENG13_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG13_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG14_SEM
-#define VM_INVALIDATE_ENG14_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG14_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG15_SEM
-#define VM_INVALIDATE_ENG15_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG15_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG16_SEM
-#define VM_INVALIDATE_ENG16_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG16_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG17_SEM
-#define VM_INVALIDATE_ENG17_SEM__SEMAPHORE__SHIFT                                                             0x0
-#define VM_INVALIDATE_ENG17_SEM__SEMAPHORE_MASK                                                               0x00000001L
-//VM_INVALIDATE_ENG0_REQ
-#define VM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG0_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG1_REQ
-#define VM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG1_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG1_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG2_REQ
-#define VM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG2_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG2_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG3_REQ
-#define VM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG3_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG3_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG4_REQ
-#define VM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG4_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG4_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG5_REQ
-#define VM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG5_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG5_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG6_REQ
-#define VM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG6_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG6_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG7_REQ
-#define VM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG7_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG7_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG8_REQ
-#define VM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG8_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG8_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG9_REQ
-#define VM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG9_REQ__FLUSH_TYPE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES__SHIFT                                                     0x12
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0__SHIFT                                                     0x13
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1__SHIFT                                                     0x14
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2__SHIFT                                                     0x15
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES__SHIFT                                                     0x16
-#define VM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                     0x17
-#define VM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG9_REQ__FLUSH_TYPE_MASK                                                               0x00030000L
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES_MASK                                                       0x00040000L
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0_MASK                                                       0x00080000L
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1_MASK                                                       0x00100000L
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2_MASK                                                       0x00200000L
-#define VM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES_MASK                                                       0x00400000L
-#define VM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                       0x00800000L
-//VM_INVALIDATE_ENG10_REQ
-#define VM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG10_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG10_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG11_REQ
-#define VM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG11_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG11_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG12_REQ
-#define VM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG12_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG12_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG13_REQ
-#define VM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG13_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG13_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG14_REQ
-#define VM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG14_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG14_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG15_REQ
-#define VM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG15_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG15_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG16_REQ
-#define VM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG16_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG17_REQ
-#define VM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG17_REQ__FLUSH_TYPE__SHIFT                                                            0x10
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES__SHIFT                                                    0x12
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0__SHIFT                                                    0x13
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1__SHIFT                                                    0x14
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2__SHIFT                                                    0x15
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES__SHIFT                                                    0x16
-#define VM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                    0x17
-#define VM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG17_REQ__FLUSH_TYPE_MASK                                                              0x00030000L
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES_MASK                                                      0x00040000L
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0_MASK                                                      0x00080000L
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1_MASK                                                      0x00100000L
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2_MASK                                                      0x00200000L
-#define VM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES_MASK                                                      0x00400000L
-#define VM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                      0x00800000L
-//VM_INVALIDATE_ENG0_ACK
-#define VM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG0_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG0_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG1_ACK
-#define VM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG1_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG1_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG2_ACK
-#define VM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG2_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG2_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG3_ACK
-#define VM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG3_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG3_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG4_ACK
-#define VM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG4_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG4_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG5_ACK
-#define VM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG5_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG5_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG6_ACK
-#define VM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG6_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG6_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG7_ACK
-#define VM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG7_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG7_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG8_ACK
-#define VM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG8_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG8_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG9_ACK
-#define VM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                                0x0
-#define VM_INVALIDATE_ENG9_ACK__SEMAPHORE__SHIFT                                                              0x10
-#define VM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                  0x0000FFFFL
-#define VM_INVALIDATE_ENG9_ACK__SEMAPHORE_MASK                                                                0x00010000L
-//VM_INVALIDATE_ENG10_ACK
-#define VM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG10_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG10_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG11_ACK
-#define VM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG11_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG11_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG12_ACK
-#define VM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG12_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG12_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG13_ACK
-#define VM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG13_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG13_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG14_ACK
-#define VM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG14_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG14_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG15_ACK
-#define VM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG15_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG15_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG16_ACK
-#define VM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG16_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG16_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG17_ACK
-#define VM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                               0x0
-#define VM_INVALIDATE_ENG17_ACK__SEMAPHORE__SHIFT                                                             0x10
-#define VM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                 0x0000FFFFL
-#define VM_INVALIDATE_ENG17_ACK__SEMAPHORE_MASK                                                               0x00010000L
-//VM_INVALIDATE_ENG0_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG0_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG1_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG1_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG2_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG2_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG3_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG3_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG4_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG4_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG5_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG5_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG6_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG6_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG7_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG7_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG8_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG8_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG9_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT__SHIFT                                                      0x0
-#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                  0x1
-#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT_MASK                                                        0x00000001L
-#define VM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                    0xFFFFFFFEL
-//VM_INVALIDATE_ENG9_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                   0x0
-#define VM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                     0x0000001FL
-//VM_INVALIDATE_ENG10_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG10_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_INVALIDATE_ENG11_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG11_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_INVALIDATE_ENG12_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG12_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_INVALIDATE_ENG13_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG13_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_INVALIDATE_ENG14_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG14_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_INVALIDATE_ENG15_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG15_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_INVALIDATE_ENG16_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG16_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_INVALIDATE_ENG17_ADDR_RANGE_LO32
-#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT__SHIFT                                                     0x0
-#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                 0x1
-#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT_MASK                                                       0x00000001L
-#define VM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                   0xFFFFFFFEL
-//VM_INVALIDATE_ENG17_ADDR_RANGE_HI32
-#define VM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                  0x0
-#define VM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                    0x0000001FL
-//VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                               0x0
-#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                               0x0
-#define VM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
-#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
-#define VM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
-#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
-#define VM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
-#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
-#define VM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
-#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
-#define VM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
-#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
-#define VM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32
-#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                              0x0
-#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32
-#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                              0x0
-#define VM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
-#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
-//VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
-#define VM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
-//VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
-#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
-#define VM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
-//VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
-#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
-#define VM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
-//VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
-#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
-#define VM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
-//VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
-#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
-#define VM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
-//VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
-#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
-#define VM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
-//VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32
-#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
-#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
-//VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32
-#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
-#define VM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
-//VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                 0x0
-#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                   0xFFFFFFFFL
-//VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                  0x0
-#define VM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                    0x0000000FL
-//VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
-#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
-//VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
-#define VM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
-//VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
-#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
-//VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
-#define VM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
-//VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
-#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
-//VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
-#define VM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
-//VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
-#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
-//VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
-#define VM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
-//VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
-#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
-//VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
-#define VM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
-//VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32
-#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                                0x0
-#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                  0xFFFFFFFFL
-//VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32
-#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                 0x0
-#define VM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                   0x0000000FL
-
-
-// addressBlock: gc_utcl2_vmsharedpfdec
-//MC_VM_NB_MMIOBASE
-#define MC_VM_NB_MMIOBASE__MMIOBASE__SHIFT                                                                    0x0
-#define MC_VM_NB_MMIOBASE__MMIOBASE_MASK                                                                      0xFFFFFFFFL
-//MC_VM_NB_MMIOLIMIT
-#define MC_VM_NB_MMIOLIMIT__MMIOLIMIT__SHIFT                                                                  0x0
-#define MC_VM_NB_MMIOLIMIT__MMIOLIMIT_MASK                                                                    0xFFFFFFFFL
-//MC_VM_NB_PCI_CTRL
-#define MC_VM_NB_PCI_CTRL__MMIOENABLE__SHIFT                                                                  0x17
-#define MC_VM_NB_PCI_CTRL__MMIOENABLE_MASK                                                                    0x00800000L
-//MC_VM_NB_PCI_ARB
-#define MC_VM_NB_PCI_ARB__VGA_HOLE__SHIFT                                                                     0x3
-#define MC_VM_NB_PCI_ARB__VGA_HOLE_MASK                                                                       0x00000008L
-//MC_VM_NB_TOP_OF_DRAM_SLOT1
-#define MC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM__SHIFT                                                        0x17
-#define MC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM_MASK                                                          0xFF800000L
-//MC_VM_NB_LOWER_TOP_OF_DRAM2
-#define MC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE__SHIFT                                                            0x0
-#define MC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2__SHIFT                                                        0x17
-#define MC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE_MASK                                                              0x00000001L
-#define MC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2_MASK                                                          0xFF800000L
-//MC_VM_NB_UPPER_TOP_OF_DRAM2
-#define MC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2__SHIFT                                                        0x0
-#define MC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2_MASK                                                          0x00000FFFL
-//MC_VM_FB_OFFSET
-#define MC_VM_FB_OFFSET__FB_OFFSET__SHIFT                                                                     0x0
-#define MC_VM_FB_OFFSET__FB_OFFSET_MASK                                                                       0x00FFFFFFL
-//MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB
-#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB__SHIFT                               0x0
-#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB_MASK                                 0xFFFFFFFFL
-//MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB
-#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB__SHIFT                               0x0
-#define MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB_MASK                                 0x0000000FL
-//MC_VM_STEERING
-#define MC_VM_STEERING__DEFAULT_STEERING__SHIFT                                                               0x0
-#define MC_VM_STEERING__DEFAULT_STEERING_MASK                                                                 0x00000003L
-//MC_SHARED_VIRT_RESET_REQ
-#define MC_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                   0x0
-#define MC_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                   0x1f
-#define MC_SHARED_VIRT_RESET_REQ__VF_MASK                                                                     0x0000FFFFL
-#define MC_SHARED_VIRT_RESET_REQ__PF_MASK                                                                     0x80000000L
-//MC_MEM_POWER_LS
-#define MC_MEM_POWER_LS__LS_SETUP__SHIFT                                                                      0x0
-#define MC_MEM_POWER_LS__LS_HOLD__SHIFT                                                                       0x6
-#define MC_MEM_POWER_LS__LS_SETUP_MASK                                                                        0x0000003FL
-#define MC_MEM_POWER_LS__LS_HOLD_MASK                                                                         0x00000FC0L
-//MC_VM_CACHEABLE_DRAM_ADDRESS_START
-#define MC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS__SHIFT                                                    0x0
-#define MC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS_MASK                                                      0x000FFFFFL
-//MC_VM_CACHEABLE_DRAM_ADDRESS_END
-#define MC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS__SHIFT                                                      0x0
-#define MC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS_MASK                                                        0x000FFFFFL
-//MC_VM_APT_CNTL
-#define MC_VM_APT_CNTL__FORCE_MTYPE_UC__SHIFT                                                                 0x0
-#define MC_VM_APT_CNTL__DIRECT_SYSTEM_EN__SHIFT                                                               0x1
-#define MC_VM_APT_CNTL__FORCE_MTYPE_UC_MASK                                                                   0x00000001L
-#define MC_VM_APT_CNTL__DIRECT_SYSTEM_EN_MASK                                                                 0x00000002L
-//MC_VM_LOCAL_HBM_ADDRESS_START
-#define MC_VM_LOCAL_HBM_ADDRESS_START__ADDRESS__SHIFT                                                         0x0
-#define MC_VM_LOCAL_HBM_ADDRESS_START__ADDRESS_MASK                                                           0x000FFFFFL
-//MC_VM_LOCAL_HBM_ADDRESS_END
-#define MC_VM_LOCAL_HBM_ADDRESS_END__ADDRESS__SHIFT                                                           0x0
-#define MC_VM_LOCAL_HBM_ADDRESS_END__ADDRESS_MASK                                                             0x000FFFFFL
-//MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL
-#define MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK__SHIFT                                                        0x0
-#define MC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL__LOCK_MASK                                                          0x00000001L
-
-
-// addressBlock: gc_utcl2_vmsharedvcdec
-//MC_VM_FB_LOCATION_BASE
-#define MC_VM_FB_LOCATION_BASE__FB_BASE__SHIFT                                                                0x0
-#define MC_VM_FB_LOCATION_BASE__FB_BASE_MASK                                                                  0x00FFFFFFL
-//MC_VM_FB_LOCATION_TOP
-#define MC_VM_FB_LOCATION_TOP__FB_TOP__SHIFT                                                                  0x0
-#define MC_VM_FB_LOCATION_TOP__FB_TOP_MASK                                                                    0x00FFFFFFL
-//MC_VM_AGP_TOP
-#define MC_VM_AGP_TOP__AGP_TOP__SHIFT                                                                         0x0
-#define MC_VM_AGP_TOP__AGP_TOP_MASK                                                                           0x00FFFFFFL
-//MC_VM_AGP_BOT
-#define MC_VM_AGP_BOT__AGP_BOT__SHIFT                                                                         0x0
-#define MC_VM_AGP_BOT__AGP_BOT_MASK                                                                           0x00FFFFFFL
-//MC_VM_AGP_BASE
-#define MC_VM_AGP_BASE__AGP_BASE__SHIFT                                                                       0x0
-#define MC_VM_AGP_BASE__AGP_BASE_MASK                                                                         0x00FFFFFFL
-//MC_VM_SYSTEM_APERTURE_LOW_ADDR
-#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR__SHIFT                                                   0x0
-#define MC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR_MASK                                                     0x3FFFFFFFL
-//MC_VM_SYSTEM_APERTURE_HIGH_ADDR
-#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR__SHIFT                                                  0x0
-#define MC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR_MASK                                                    0x3FFFFFFFL
-//MC_VM_MX_L1_TLB_CNTL
-#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                            0x0
-#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                                       0x3
-#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                          0x5
-#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                             0x6
-#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS__SHIFT                                                                 0x7
-#define MC_VM_MX_L1_TLB_CNTL__MTYPE__SHIFT                                                                    0xb
-#define MC_VM_MX_L1_TLB_CNTL__ATC_EN__SHIFT                                                                   0xd
-#define MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                              0x00000001L
-#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                                         0x00000018L
-#define MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                            0x00000020L
-#define MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                               0x00000040L
-#define MC_VM_MX_L1_TLB_CNTL__ECO_BITS_MASK                                                                   0x00000780L
-#define MC_VM_MX_L1_TLB_CNTL__MTYPE_MASK                                                                      0x00001800L
-#define MC_VM_MX_L1_TLB_CNTL__ATC_EN_MASK                                                                     0x00002000L
-
-
-// addressBlock: gc_ea_gceadec
-//GCEA_DRAM_RD_CLI2GRP_MAP0
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
-//GCEA_DRAM_RD_CLI2GRP_MAP1
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
-#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
-//GCEA_DRAM_WR_CLI2GRP_MAP0
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
-//GCEA_DRAM_WR_CLI2GRP_MAP1
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
-#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
-//GCEA_DRAM_RD_GRP2VC_MAP
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
-#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
-//GCEA_DRAM_WR_GRP2VC_MAP
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
-#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
-//GCEA_DRAM_RD_LAZY
-#define GCEA_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
-#define GCEA_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
-#define GCEA_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
-#define GCEA_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
-#define GCEA_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
-#define GCEA_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
-#define GCEA_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
-#define GCEA_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
-//GCEA_DRAM_WR_LAZY
-#define GCEA_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
-#define GCEA_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
-#define GCEA_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
-#define GCEA_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
-#define GCEA_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
-#define GCEA_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
-#define GCEA_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
-#define GCEA_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
-//GCEA_DRAM_RD_CAM_CNTL
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
-#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
-#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
-//GCEA_DRAM_WR_CAM_CNTL
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
-#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
-#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
-//GCEA_DRAM_PAGE_BURST
-#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
-#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
-#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
-#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
-#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
-#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
-#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
-#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
-//GCEA_DRAM_RD_PRI_AGE
-#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
-#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
-#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
-#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
-#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
-#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
-#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
-#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
-#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
-#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
-#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
-#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
-#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
-#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
-#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
-#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
-//GCEA_DRAM_WR_PRI_AGE
-#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
-#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
-#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
-#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
-#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
-#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
-#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
-#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
-#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
-#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
-#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
-#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
-#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
-#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
-#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
-#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
-//GCEA_DRAM_RD_PRI_QUEUING
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
-#define GCEA_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
-//GCEA_DRAM_WR_PRI_QUEUING
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
-#define GCEA_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
-//GCEA_DRAM_RD_PRI_FIXED
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
-#define GCEA_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
-//GCEA_DRAM_WR_PRI_FIXED
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
-#define GCEA_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
-//GCEA_DRAM_RD_PRI_URGENCY
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
-#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
-//GCEA_DRAM_WR_PRI_URGENCY
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
-#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
-//GCEA_DRAM_RD_PRI_QUANT_PRI1
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
-#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
-//GCEA_DRAM_RD_PRI_QUANT_PRI2
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
-#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
-//GCEA_DRAM_RD_PRI_QUANT_PRI3
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
-#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
-//GCEA_DRAM_WR_PRI_QUANT_PRI1
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
-#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
-//GCEA_DRAM_WR_PRI_QUANT_PRI2
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
-#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
-//GCEA_DRAM_WR_PRI_QUANT_PRI3
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
-#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
-//GCEA_ADDRNORM_BASE_ADDR0
-#define GCEA_ADDRNORM_BASE_ADDR0__ADDR_RNG_VAL__SHIFT                                                         0x0
-#define GCEA_ADDRNORM_BASE_ADDR0__LGCY_MMIO_HOLE_EN__SHIFT                                                    0x1
-#define GCEA_ADDRNORM_BASE_ADDR0__INTLV_NUM_CHAN__SHIFT                                                       0x4
-#define GCEA_ADDRNORM_BASE_ADDR0__INTLV_ADDR_SEL__SHIFT                                                       0x8
-#define GCEA_ADDRNORM_BASE_ADDR0__BASE_ADDR__SHIFT                                                            0xc
-#define GCEA_ADDRNORM_BASE_ADDR0__ADDR_RNG_VAL_MASK                                                           0x00000001L
-#define GCEA_ADDRNORM_BASE_ADDR0__LGCY_MMIO_HOLE_EN_MASK                                                      0x00000002L
-#define GCEA_ADDRNORM_BASE_ADDR0__INTLV_NUM_CHAN_MASK                                                         0x000000F0L
-#define GCEA_ADDRNORM_BASE_ADDR0__INTLV_ADDR_SEL_MASK                                                         0x00000700L
-#define GCEA_ADDRNORM_BASE_ADDR0__BASE_ADDR_MASK                                                              0xFFFFF000L
-//GCEA_ADDRNORM_LIMIT_ADDR0
-#define GCEA_ADDRNORM_LIMIT_ADDR0__DST_FABRIC_ID__SHIFT                                                       0x0
-#define GCEA_ADDRNORM_LIMIT_ADDR0__INTLV_NUM_SOCKETS__SHIFT                                                   0x8
-#define GCEA_ADDRNORM_LIMIT_ADDR0__INTLV_NUM_DIES__SHIFT                                                      0xa
-#define GCEA_ADDRNORM_LIMIT_ADDR0__LIMIT_ADDR__SHIFT                                                          0xc
-#define GCEA_ADDRNORM_LIMIT_ADDR0__DST_FABRIC_ID_MASK                                                         0x0000000FL
-#define GCEA_ADDRNORM_LIMIT_ADDR0__INTLV_NUM_SOCKETS_MASK                                                     0x00000100L
-#define GCEA_ADDRNORM_LIMIT_ADDR0__INTLV_NUM_DIES_MASK                                                        0x00000C00L
-#define GCEA_ADDRNORM_LIMIT_ADDR0__LIMIT_ADDR_MASK                                                            0xFFFFF000L
-//GCEA_ADDRNORM_BASE_ADDR1
-#define GCEA_ADDRNORM_BASE_ADDR1__ADDR_RNG_VAL__SHIFT                                                         0x0
-#define GCEA_ADDRNORM_BASE_ADDR1__LGCY_MMIO_HOLE_EN__SHIFT                                                    0x1
-#define GCEA_ADDRNORM_BASE_ADDR1__INTLV_NUM_CHAN__SHIFT                                                       0x4
-#define GCEA_ADDRNORM_BASE_ADDR1__INTLV_ADDR_SEL__SHIFT                                                       0x8
-#define GCEA_ADDRNORM_BASE_ADDR1__BASE_ADDR__SHIFT                                                            0xc
-#define GCEA_ADDRNORM_BASE_ADDR1__ADDR_RNG_VAL_MASK                                                           0x00000001L
-#define GCEA_ADDRNORM_BASE_ADDR1__LGCY_MMIO_HOLE_EN_MASK                                                      0x00000002L
-#define GCEA_ADDRNORM_BASE_ADDR1__INTLV_NUM_CHAN_MASK                                                         0x000000F0L
-#define GCEA_ADDRNORM_BASE_ADDR1__INTLV_ADDR_SEL_MASK                                                         0x00000700L
-#define GCEA_ADDRNORM_BASE_ADDR1__BASE_ADDR_MASK                                                              0xFFFFF000L
-//GCEA_ADDRNORM_LIMIT_ADDR1
-#define GCEA_ADDRNORM_LIMIT_ADDR1__DST_FABRIC_ID__SHIFT                                                       0x0
-#define GCEA_ADDRNORM_LIMIT_ADDR1__INTLV_NUM_SOCKETS__SHIFT                                                   0x8
-#define GCEA_ADDRNORM_LIMIT_ADDR1__INTLV_NUM_DIES__SHIFT                                                      0xa
-#define GCEA_ADDRNORM_LIMIT_ADDR1__LIMIT_ADDR__SHIFT                                                          0xc
-#define GCEA_ADDRNORM_LIMIT_ADDR1__DST_FABRIC_ID_MASK                                                         0x0000000FL
-#define GCEA_ADDRNORM_LIMIT_ADDR1__INTLV_NUM_SOCKETS_MASK                                                     0x00000100L
-#define GCEA_ADDRNORM_LIMIT_ADDR1__INTLV_NUM_DIES_MASK                                                        0x00000C00L
-#define GCEA_ADDRNORM_LIMIT_ADDR1__LIMIT_ADDR_MASK                                                            0xFFFFF000L
-//GCEA_ADDRNORM_OFFSET_ADDR1
-#define GCEA_ADDRNORM_OFFSET_ADDR1__HI_ADDR_OFFSET_EN__SHIFT                                                  0x0
-#define GCEA_ADDRNORM_OFFSET_ADDR1__HI_ADDR_OFFSET__SHIFT                                                     0x14
-#define GCEA_ADDRNORM_OFFSET_ADDR1__HI_ADDR_OFFSET_EN_MASK                                                    0x00000001L
-#define GCEA_ADDRNORM_OFFSET_ADDR1__HI_ADDR_OFFSET_MASK                                                       0xFFF00000L
-//GCEA_ADDRNORM_HOLE_CNTL
-#define GCEA_ADDRNORM_HOLE_CNTL__DRAM_HOLE_VALID__SHIFT                                                       0x0
-#define GCEA_ADDRNORM_HOLE_CNTL__DRAM_HOLE_OFFSET__SHIFT                                                      0x7
-#define GCEA_ADDRNORM_HOLE_CNTL__DRAM_HOLE_VALID_MASK                                                         0x00000001L
-#define GCEA_ADDRNORM_HOLE_CNTL__DRAM_HOLE_OFFSET_MASK                                                        0x0000FF80L
-//GCEA_ADDRDEC_BANK_CFG
-#define GCEA_ADDRDEC_BANK_CFG__BANK_MASK_DRAM__SHIFT                                                          0x0
-#define GCEA_ADDRDEC_BANK_CFG__BANK_MASK_GMI__SHIFT                                                           0x5
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_SEL_DRAM__SHIFT                                                      0xa
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_SEL_GMI__SHIFT                                                       0xd
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_INTERLEAVE_DRAM__SHIFT                                               0x10
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_INTERLEAVE_GMI__SHIFT                                                0x11
-#define GCEA_ADDRDEC_BANK_CFG__BANK_MASK_DRAM_MASK                                                            0x0000001FL
-#define GCEA_ADDRDEC_BANK_CFG__BANK_MASK_GMI_MASK                                                             0x000003E0L
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_SEL_DRAM_MASK                                                        0x00001C00L
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_SEL_GMI_MASK                                                         0x0000E000L
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_INTERLEAVE_DRAM_MASK                                                 0x00010000L
-#define GCEA_ADDRDEC_BANK_CFG__BANKGROUP_INTERLEAVE_GMI_MASK                                                  0x00020000L
-//GCEA_ADDRDEC_MISC_CFG
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN0__SHIFT                                                                 0x0
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN1__SHIFT                                                                 0x1
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN2__SHIFT                                                                 0x2
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN3__SHIFT                                                                 0x3
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN4__SHIFT                                                                 0x4
-#define GCEA_ADDRDEC_MISC_CFG__PCH_MASK_DRAM__SHIFT                                                           0x8
-#define GCEA_ADDRDEC_MISC_CFG__PCH_MASK_GMI__SHIFT                                                            0x9
-#define GCEA_ADDRDEC_MISC_CFG__CH_MASK_DRAM__SHIFT                                                            0xc
-#define GCEA_ADDRDEC_MISC_CFG__CH_MASK_GMI__SHIFT                                                             0x10
-#define GCEA_ADDRDEC_MISC_CFG__CS_MASK_DRAM__SHIFT                                                            0x14
-#define GCEA_ADDRDEC_MISC_CFG__CS_MASK_GMI__SHIFT                                                             0x16
-#define GCEA_ADDRDEC_MISC_CFG__RM_MASK_DRAM__SHIFT                                                            0x18
-#define GCEA_ADDRDEC_MISC_CFG__RM_MASK_GMI__SHIFT                                                             0x1b
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN0_MASK                                                                   0x00000001L
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN1_MASK                                                                   0x00000002L
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN2_MASK                                                                   0x00000004L
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN3_MASK                                                                   0x00000008L
-#define GCEA_ADDRDEC_MISC_CFG__VCM_EN4_MASK                                                                   0x00000010L
-#define GCEA_ADDRDEC_MISC_CFG__PCH_MASK_DRAM_MASK                                                             0x00000100L
-#define GCEA_ADDRDEC_MISC_CFG__PCH_MASK_GMI_MASK                                                              0x00000200L
-#define GCEA_ADDRDEC_MISC_CFG__CH_MASK_DRAM_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC_MISC_CFG__CH_MASK_GMI_MASK                                                               0x000F0000L
-#define GCEA_ADDRDEC_MISC_CFG__CS_MASK_DRAM_MASK                                                              0x00300000L
-#define GCEA_ADDRDEC_MISC_CFG__CS_MASK_GMI_MASK                                                               0x00C00000L
-#define GCEA_ADDRDEC_MISC_CFG__RM_MASK_DRAM_MASK                                                              0x07000000L
-#define GCEA_ADDRDEC_MISC_CFG__RM_MASK_GMI_MASK                                                               0x38000000L
-//GCEA_ADDRDECDRAM_ADDR_HASH_BANK0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK0__XOR_ENABLE__SHIFT                                                   0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK0__COL_XOR__SHIFT                                                      0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK0__ROW_XOR__SHIFT                                                      0xe
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK0__XOR_ENABLE_MASK                                                     0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK0__COL_XOR_MASK                                                        0x00003FFEL
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK0__ROW_XOR_MASK                                                        0xFFFFC000L
-//GCEA_ADDRDECDRAM_ADDR_HASH_BANK1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK1__XOR_ENABLE__SHIFT                                                   0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK1__COL_XOR__SHIFT                                                      0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK1__ROW_XOR__SHIFT                                                      0xe
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK1__XOR_ENABLE_MASK                                                     0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK1__COL_XOR_MASK                                                        0x00003FFEL
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK1__ROW_XOR_MASK                                                        0xFFFFC000L
-//GCEA_ADDRDECDRAM_ADDR_HASH_BANK2
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK2__XOR_ENABLE__SHIFT                                                   0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK2__COL_XOR__SHIFT                                                      0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK2__ROW_XOR__SHIFT                                                      0xe
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK2__XOR_ENABLE_MASK                                                     0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK2__COL_XOR_MASK                                                        0x00003FFEL
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK2__ROW_XOR_MASK                                                        0xFFFFC000L
-//GCEA_ADDRDECDRAM_ADDR_HASH_BANK3
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK3__XOR_ENABLE__SHIFT                                                   0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK3__COL_XOR__SHIFT                                                      0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK3__ROW_XOR__SHIFT                                                      0xe
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK3__XOR_ENABLE_MASK                                                     0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK3__COL_XOR_MASK                                                        0x00003FFEL
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK3__ROW_XOR_MASK                                                        0xFFFFC000L
-//GCEA_ADDRDECDRAM_ADDR_HASH_BANK4
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK4__XOR_ENABLE__SHIFT                                                   0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK4__COL_XOR__SHIFT                                                      0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK4__ROW_XOR__SHIFT                                                      0xe
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK4__XOR_ENABLE_MASK                                                     0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK4__COL_XOR_MASK                                                        0x00003FFEL
-#define GCEA_ADDRDECDRAM_ADDR_HASH_BANK4__ROW_XOR_MASK                                                        0xFFFFC000L
-//GCEA_ADDRDECDRAM_ADDR_HASH_PC
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC__XOR_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC__COL_XOR__SHIFT                                                         0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC__ROW_XOR__SHIFT                                                         0xe
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC__XOR_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC__COL_XOR_MASK                                                           0x00003FFEL
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC__ROW_XOR_MASK                                                           0xFFFFC000L
-//GCEA_ADDRDECDRAM_ADDR_HASH_PC2
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC2__BANK_XOR__SHIFT                                                       0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_PC2__BANK_XOR_MASK                                                         0x0000001FL
-//GCEA_ADDRDECDRAM_ADDR_HASH_CS0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS0__XOR_ENABLE__SHIFT                                                     0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS0__NA_XOR__SHIFT                                                         0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS0__XOR_ENABLE_MASK                                                       0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS0__NA_XOR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDECDRAM_ADDR_HASH_CS1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS1__XOR_ENABLE__SHIFT                                                     0x0
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS1__NA_XOR__SHIFT                                                         0x1
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS1__XOR_ENABLE_MASK                                                       0x00000001L
-#define GCEA_ADDRDECDRAM_ADDR_HASH_CS1__NA_XOR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDECDRAM_HARVEST_ENABLE
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B3_EN__SHIFT                                                   0x0
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B3_VAL__SHIFT                                                  0x1
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B4_EN__SHIFT                                                   0x2
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B4_VAL__SHIFT                                                  0x3
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B3_EN_MASK                                                     0x00000001L
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B3_VAL_MASK                                                    0x00000002L
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B4_EN_MASK                                                     0x00000004L
-#define GCEA_ADDRDECDRAM_HARVEST_ENABLE__FORCE_B4_VAL_MASK                                                    0x00000008L
-//GCEA_ADDRDEC0_BASE_ADDR_CS0
-#define GCEA_ADDRDEC0_BASE_ADDR_CS0__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_CS0__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_CS0__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_CS0__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC0_BASE_ADDR_CS1
-#define GCEA_ADDRDEC0_BASE_ADDR_CS1__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_CS1__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_CS1__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_CS1__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC0_BASE_ADDR_CS2
-#define GCEA_ADDRDEC0_BASE_ADDR_CS2__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_CS2__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_CS2__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_CS2__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC0_BASE_ADDR_CS3
-#define GCEA_ADDRDEC0_BASE_ADDR_CS3__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_CS3__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_CS3__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_CS3__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC0_BASE_ADDR_SECCS0
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS0__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS0__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS0__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS0__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC0_BASE_ADDR_SECCS1
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS1__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS1__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS1__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS1__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC0_BASE_ADDR_SECCS2
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS2__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS2__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS2__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS2__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC0_BASE_ADDR_SECCS3
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS3__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS3__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS3__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC0_BASE_ADDR_SECCS3__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC0_ADDR_MASK_CS01
-#define GCEA_ADDRDEC0_ADDR_MASK_CS01__ADDR_MASK__SHIFT                                                        0x1
-#define GCEA_ADDRDEC0_ADDR_MASK_CS01__ADDR_MASK_MASK                                                          0xFFFFFFFEL
-//GCEA_ADDRDEC0_ADDR_MASK_CS23
-#define GCEA_ADDRDEC0_ADDR_MASK_CS23__ADDR_MASK__SHIFT                                                        0x1
-#define GCEA_ADDRDEC0_ADDR_MASK_CS23__ADDR_MASK_MASK                                                          0xFFFFFFFEL
-//GCEA_ADDRDEC0_ADDR_MASK_SECCS01
-#define GCEA_ADDRDEC0_ADDR_MASK_SECCS01__ADDR_MASK__SHIFT                                                     0x1
-#define GCEA_ADDRDEC0_ADDR_MASK_SECCS01__ADDR_MASK_MASK                                                       0xFFFFFFFEL
-//GCEA_ADDRDEC0_ADDR_MASK_SECCS23
-#define GCEA_ADDRDEC0_ADDR_MASK_SECCS23__ADDR_MASK__SHIFT                                                     0x1
-#define GCEA_ADDRDEC0_ADDR_MASK_SECCS23__ADDR_MASK_MASK                                                       0xFFFFFFFEL
-//GCEA_ADDRDEC0_ADDR_CFG_CS01
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_BANK_GROUPS__SHIFT                                                   0x2
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_RM__SHIFT                                                            0x4
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_ROW_LO__SHIFT                                                        0x8
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_ROW_HI__SHIFT                                                        0xc
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_COL__SHIFT                                                           0x10
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_BANKS__SHIFT                                                         0x14
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_BANK_GROUPS_MASK                                                     0x0000000CL
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_RM_MASK                                                              0x00000030L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_ROW_LO_MASK                                                          0x00000F00L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_ROW_HI_MASK                                                          0x0000F000L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_COL_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS01__NUM_BANKS_MASK                                                           0x00300000L
-//GCEA_ADDRDEC0_ADDR_CFG_CS23
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_BANK_GROUPS__SHIFT                                                   0x2
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_RM__SHIFT                                                            0x4
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_ROW_LO__SHIFT                                                        0x8
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_ROW_HI__SHIFT                                                        0xc
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_COL__SHIFT                                                           0x10
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_BANKS__SHIFT                                                         0x14
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_BANK_GROUPS_MASK                                                     0x0000000CL
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_RM_MASK                                                              0x00000030L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_ROW_LO_MASK                                                          0x00000F00L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_ROW_HI_MASK                                                          0x0000F000L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_COL_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC0_ADDR_CFG_CS23__NUM_BANKS_MASK                                                           0x00300000L
-//GCEA_ADDRDEC0_ADDR_SEL_CS01
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK0__SHIFT                                                             0x0
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK1__SHIFT                                                             0x4
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK2__SHIFT                                                             0x8
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK3__SHIFT                                                             0xc
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK4__SHIFT                                                             0x10
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__ROW_LO__SHIFT                                                            0x18
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__ROW_HI__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK0_MASK                                                               0x0000000FL
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK1_MASK                                                               0x000000F0L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK2_MASK                                                               0x00000F00L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK3_MASK                                                               0x0000F000L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__BANK4_MASK                                                               0x000F0000L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__ROW_LO_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS01__ROW_HI_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC0_ADDR_SEL_CS23
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK0__SHIFT                                                             0x0
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK1__SHIFT                                                             0x4
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK2__SHIFT                                                             0x8
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK3__SHIFT                                                             0xc
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK4__SHIFT                                                             0x10
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__ROW_LO__SHIFT                                                            0x18
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__ROW_HI__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK0_MASK                                                               0x0000000FL
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK1_MASK                                                               0x000000F0L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK2_MASK                                                               0x00000F00L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK3_MASK                                                               0x0000F000L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__BANK4_MASK                                                               0x000F0000L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__ROW_LO_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC0_ADDR_SEL_CS23__ROW_HI_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC0_COL_SEL_LO_CS01
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL0__SHIFT                                                            0x0
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL1__SHIFT                                                            0x4
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL2__SHIFT                                                            0x8
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL3__SHIFT                                                            0xc
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL4__SHIFT                                                            0x10
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL5__SHIFT                                                            0x14
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL6__SHIFT                                                            0x18
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL7__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL0_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL1_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL2_MASK                                                              0x00000F00L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL3_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL4_MASK                                                              0x000F0000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL5_MASK                                                              0x00F00000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL6_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS01__COL7_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC0_COL_SEL_LO_CS23
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL0__SHIFT                                                            0x0
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL1__SHIFT                                                            0x4
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL2__SHIFT                                                            0x8
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL3__SHIFT                                                            0xc
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL4__SHIFT                                                            0x10
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL5__SHIFT                                                            0x14
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL6__SHIFT                                                            0x18
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL7__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL0_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL1_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL2_MASK                                                              0x00000F00L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL3_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL4_MASK                                                              0x000F0000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL5_MASK                                                              0x00F00000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL6_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC0_COL_SEL_LO_CS23__COL7_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC0_COL_SEL_HI_CS01
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL8__SHIFT                                                            0x0
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL9__SHIFT                                                            0x4
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL10__SHIFT                                                           0x8
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL11__SHIFT                                                           0xc
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL12__SHIFT                                                           0x10
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL13__SHIFT                                                           0x14
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL14__SHIFT                                                           0x18
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL15__SHIFT                                                           0x1c
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL8_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL9_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL10_MASK                                                             0x00000F00L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL11_MASK                                                             0x0000F000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL12_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL13_MASK                                                             0x00F00000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL14_MASK                                                             0x0F000000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS01__COL15_MASK                                                             0xF0000000L
-//GCEA_ADDRDEC0_COL_SEL_HI_CS23
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL8__SHIFT                                                            0x0
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL9__SHIFT                                                            0x4
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL10__SHIFT                                                           0x8
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL11__SHIFT                                                           0xc
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL12__SHIFT                                                           0x10
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL13__SHIFT                                                           0x14
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL14__SHIFT                                                           0x18
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL15__SHIFT                                                           0x1c
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL8_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL9_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL10_MASK                                                             0x00000F00L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL11_MASK                                                             0x0000F000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL12_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL13_MASK                                                             0x00F00000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL14_MASK                                                             0x0F000000L
-#define GCEA_ADDRDEC0_COL_SEL_HI_CS23__COL15_MASK                                                             0xF0000000L
-//GCEA_ADDRDEC0_RM_SEL_CS01
-#define GCEA_ADDRDEC0_RM_SEL_CS01__RM0__SHIFT                                                                 0x0
-#define GCEA_ADDRDEC0_RM_SEL_CS01__RM1__SHIFT                                                                 0x4
-#define GCEA_ADDRDEC0_RM_SEL_CS01__RM2__SHIFT                                                                 0x8
-#define GCEA_ADDRDEC0_RM_SEL_CS01__CHAN_BIT__SHIFT                                                            0xc
-#define GCEA_ADDRDEC0_RM_SEL_CS01__INVERT_ROW_MSBS_EVEN__SHIFT                                                0x10
-#define GCEA_ADDRDEC0_RM_SEL_CS01__INVERT_ROW_MSBS_ODD__SHIFT                                                 0x12
-#define GCEA_ADDRDEC0_RM_SEL_CS01__RM0_MASK                                                                   0x0000000FL
-#define GCEA_ADDRDEC0_RM_SEL_CS01__RM1_MASK                                                                   0x000000F0L
-#define GCEA_ADDRDEC0_RM_SEL_CS01__RM2_MASK                                                                   0x00000F00L
-#define GCEA_ADDRDEC0_RM_SEL_CS01__CHAN_BIT_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC0_RM_SEL_CS01__INVERT_ROW_MSBS_EVEN_MASK                                                  0x00030000L
-#define GCEA_ADDRDEC0_RM_SEL_CS01__INVERT_ROW_MSBS_ODD_MASK                                                   0x000C0000L
-//GCEA_ADDRDEC0_RM_SEL_CS23
-#define GCEA_ADDRDEC0_RM_SEL_CS23__RM0__SHIFT                                                                 0x0
-#define GCEA_ADDRDEC0_RM_SEL_CS23__RM1__SHIFT                                                                 0x4
-#define GCEA_ADDRDEC0_RM_SEL_CS23__RM2__SHIFT                                                                 0x8
-#define GCEA_ADDRDEC0_RM_SEL_CS23__CHAN_BIT__SHIFT                                                            0xc
-#define GCEA_ADDRDEC0_RM_SEL_CS23__INVERT_ROW_MSBS_EVEN__SHIFT                                                0x10
-#define GCEA_ADDRDEC0_RM_SEL_CS23__INVERT_ROW_MSBS_ODD__SHIFT                                                 0x12
-#define GCEA_ADDRDEC0_RM_SEL_CS23__RM0_MASK                                                                   0x0000000FL
-#define GCEA_ADDRDEC0_RM_SEL_CS23__RM1_MASK                                                                   0x000000F0L
-#define GCEA_ADDRDEC0_RM_SEL_CS23__RM2_MASK                                                                   0x00000F00L
-#define GCEA_ADDRDEC0_RM_SEL_CS23__CHAN_BIT_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC0_RM_SEL_CS23__INVERT_ROW_MSBS_EVEN_MASK                                                  0x00030000L
-#define GCEA_ADDRDEC0_RM_SEL_CS23__INVERT_ROW_MSBS_ODD_MASK                                                   0x000C0000L
-//GCEA_ADDRDEC0_RM_SEL_SECCS01
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__RM0__SHIFT                                                              0x0
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__RM1__SHIFT                                                              0x4
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__RM2__SHIFT                                                              0x8
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__CHAN_BIT__SHIFT                                                         0xc
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__INVERT_ROW_MSBS_EVEN__SHIFT                                             0x10
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__INVERT_ROW_MSBS_ODD__SHIFT                                              0x12
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__RM0_MASK                                                                0x0000000FL
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__RM1_MASK                                                                0x000000F0L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__RM2_MASK                                                                0x00000F00L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__CHAN_BIT_MASK                                                           0x0000F000L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__INVERT_ROW_MSBS_EVEN_MASK                                               0x00030000L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS01__INVERT_ROW_MSBS_ODD_MASK                                                0x000C0000L
-//GCEA_ADDRDEC0_RM_SEL_SECCS23
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__RM0__SHIFT                                                              0x0
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__RM1__SHIFT                                                              0x4
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__RM2__SHIFT                                                              0x8
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__CHAN_BIT__SHIFT                                                         0xc
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__INVERT_ROW_MSBS_EVEN__SHIFT                                             0x10
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__INVERT_ROW_MSBS_ODD__SHIFT                                              0x12
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__RM0_MASK                                                                0x0000000FL
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__RM1_MASK                                                                0x000000F0L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__RM2_MASK                                                                0x00000F00L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__CHAN_BIT_MASK                                                           0x0000F000L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__INVERT_ROW_MSBS_EVEN_MASK                                               0x00030000L
-#define GCEA_ADDRDEC0_RM_SEL_SECCS23__INVERT_ROW_MSBS_ODD_MASK                                                0x000C0000L
-//GCEA_ADDRDEC1_BASE_ADDR_CS0
-#define GCEA_ADDRDEC1_BASE_ADDR_CS0__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_CS0__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_CS0__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_CS0__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC1_BASE_ADDR_CS1
-#define GCEA_ADDRDEC1_BASE_ADDR_CS1__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_CS1__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_CS1__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_CS1__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC1_BASE_ADDR_CS2
-#define GCEA_ADDRDEC1_BASE_ADDR_CS2__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_CS2__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_CS2__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_CS2__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC1_BASE_ADDR_CS3
-#define GCEA_ADDRDEC1_BASE_ADDR_CS3__CS_ENABLE__SHIFT                                                         0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_CS3__BASE_ADDR__SHIFT                                                         0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_CS3__CS_ENABLE_MASK                                                           0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_CS3__BASE_ADDR_MASK                                                           0xFFFFFFFEL
-//GCEA_ADDRDEC1_BASE_ADDR_SECCS0
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS0__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS0__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS0__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS0__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC1_BASE_ADDR_SECCS1
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS1__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS1__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS1__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS1__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC1_BASE_ADDR_SECCS2
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS2__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS2__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS2__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS2__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC1_BASE_ADDR_SECCS3
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS3__CS_ENABLE__SHIFT                                                      0x0
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS3__BASE_ADDR__SHIFT                                                      0x1
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS3__CS_ENABLE_MASK                                                        0x00000001L
-#define GCEA_ADDRDEC1_BASE_ADDR_SECCS3__BASE_ADDR_MASK                                                        0xFFFFFFFEL
-//GCEA_ADDRDEC1_ADDR_MASK_CS01
-#define GCEA_ADDRDEC1_ADDR_MASK_CS01__ADDR_MASK__SHIFT                                                        0x1
-#define GCEA_ADDRDEC1_ADDR_MASK_CS01__ADDR_MASK_MASK                                                          0xFFFFFFFEL
-//GCEA_ADDRDEC1_ADDR_MASK_CS23
-#define GCEA_ADDRDEC1_ADDR_MASK_CS23__ADDR_MASK__SHIFT                                                        0x1
-#define GCEA_ADDRDEC1_ADDR_MASK_CS23__ADDR_MASK_MASK                                                          0xFFFFFFFEL
-//GCEA_ADDRDEC1_ADDR_MASK_SECCS01
-#define GCEA_ADDRDEC1_ADDR_MASK_SECCS01__ADDR_MASK__SHIFT                                                     0x1
-#define GCEA_ADDRDEC1_ADDR_MASK_SECCS01__ADDR_MASK_MASK                                                       0xFFFFFFFEL
-//GCEA_ADDRDEC1_ADDR_MASK_SECCS23
-#define GCEA_ADDRDEC1_ADDR_MASK_SECCS23__ADDR_MASK__SHIFT                                                     0x1
-#define GCEA_ADDRDEC1_ADDR_MASK_SECCS23__ADDR_MASK_MASK                                                       0xFFFFFFFEL
-//GCEA_ADDRDEC1_ADDR_CFG_CS01
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_BANK_GROUPS__SHIFT                                                   0x2
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_RM__SHIFT                                                            0x4
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_ROW_LO__SHIFT                                                        0x8
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_ROW_HI__SHIFT                                                        0xc
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_COL__SHIFT                                                           0x10
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_BANKS__SHIFT                                                         0x14
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_BANK_GROUPS_MASK                                                     0x0000000CL
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_RM_MASK                                                              0x00000030L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_ROW_LO_MASK                                                          0x00000F00L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_ROW_HI_MASK                                                          0x0000F000L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_COL_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS01__NUM_BANKS_MASK                                                           0x00300000L
-//GCEA_ADDRDEC1_ADDR_CFG_CS23
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_BANK_GROUPS__SHIFT                                                   0x2
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_RM__SHIFT                                                            0x4
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_ROW_LO__SHIFT                                                        0x8
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_ROW_HI__SHIFT                                                        0xc
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_COL__SHIFT                                                           0x10
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_BANKS__SHIFT                                                         0x14
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_BANK_GROUPS_MASK                                                     0x0000000CL
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_RM_MASK                                                              0x00000030L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_ROW_LO_MASK                                                          0x00000F00L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_ROW_HI_MASK                                                          0x0000F000L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_COL_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC1_ADDR_CFG_CS23__NUM_BANKS_MASK                                                           0x00300000L
-//GCEA_ADDRDEC1_ADDR_SEL_CS01
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK0__SHIFT                                                             0x0
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK1__SHIFT                                                             0x4
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK2__SHIFT                                                             0x8
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK3__SHIFT                                                             0xc
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK4__SHIFT                                                             0x10
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__ROW_LO__SHIFT                                                            0x18
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__ROW_HI__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK0_MASK                                                               0x0000000FL
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK1_MASK                                                               0x000000F0L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK2_MASK                                                               0x00000F00L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK3_MASK                                                               0x0000F000L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__BANK4_MASK                                                               0x000F0000L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__ROW_LO_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS01__ROW_HI_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC1_ADDR_SEL_CS23
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK0__SHIFT                                                             0x0
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK1__SHIFT                                                             0x4
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK2__SHIFT                                                             0x8
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK3__SHIFT                                                             0xc
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK4__SHIFT                                                             0x10
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__ROW_LO__SHIFT                                                            0x18
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__ROW_HI__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK0_MASK                                                               0x0000000FL
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK1_MASK                                                               0x000000F0L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK2_MASK                                                               0x00000F00L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK3_MASK                                                               0x0000F000L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__BANK4_MASK                                                               0x000F0000L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__ROW_LO_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC1_ADDR_SEL_CS23__ROW_HI_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC1_COL_SEL_LO_CS01
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL0__SHIFT                                                            0x0
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL1__SHIFT                                                            0x4
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL2__SHIFT                                                            0x8
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL3__SHIFT                                                            0xc
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL4__SHIFT                                                            0x10
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL5__SHIFT                                                            0x14
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL6__SHIFT                                                            0x18
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL7__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL0_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL1_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL2_MASK                                                              0x00000F00L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL3_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL4_MASK                                                              0x000F0000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL5_MASK                                                              0x00F00000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL6_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS01__COL7_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC1_COL_SEL_LO_CS23
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL0__SHIFT                                                            0x0
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL1__SHIFT                                                            0x4
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL2__SHIFT                                                            0x8
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL3__SHIFT                                                            0xc
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL4__SHIFT                                                            0x10
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL5__SHIFT                                                            0x14
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL6__SHIFT                                                            0x18
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL7__SHIFT                                                            0x1c
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL0_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL1_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL2_MASK                                                              0x00000F00L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL3_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL4_MASK                                                              0x000F0000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL5_MASK                                                              0x00F00000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL6_MASK                                                              0x0F000000L
-#define GCEA_ADDRDEC1_COL_SEL_LO_CS23__COL7_MASK                                                              0xF0000000L
-//GCEA_ADDRDEC1_COL_SEL_HI_CS01
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL8__SHIFT                                                            0x0
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL9__SHIFT                                                            0x4
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL10__SHIFT                                                           0x8
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL11__SHIFT                                                           0xc
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL12__SHIFT                                                           0x10
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL13__SHIFT                                                           0x14
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL14__SHIFT                                                           0x18
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL15__SHIFT                                                           0x1c
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL8_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL9_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL10_MASK                                                             0x00000F00L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL11_MASK                                                             0x0000F000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL12_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL13_MASK                                                             0x00F00000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL14_MASK                                                             0x0F000000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS01__COL15_MASK                                                             0xF0000000L
-//GCEA_ADDRDEC1_COL_SEL_HI_CS23
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL8__SHIFT                                                            0x0
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL9__SHIFT                                                            0x4
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL10__SHIFT                                                           0x8
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL11__SHIFT                                                           0xc
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL12__SHIFT                                                           0x10
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL13__SHIFT                                                           0x14
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL14__SHIFT                                                           0x18
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL15__SHIFT                                                           0x1c
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL8_MASK                                                              0x0000000FL
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL9_MASK                                                              0x000000F0L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL10_MASK                                                             0x00000F00L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL11_MASK                                                             0x0000F000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL12_MASK                                                             0x000F0000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL13_MASK                                                             0x00F00000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL14_MASK                                                             0x0F000000L
-#define GCEA_ADDRDEC1_COL_SEL_HI_CS23__COL15_MASK                                                             0xF0000000L
-//GCEA_ADDRDEC1_RM_SEL_CS01
-#define GCEA_ADDRDEC1_RM_SEL_CS01__RM0__SHIFT                                                                 0x0
-#define GCEA_ADDRDEC1_RM_SEL_CS01__RM1__SHIFT                                                                 0x4
-#define GCEA_ADDRDEC1_RM_SEL_CS01__RM2__SHIFT                                                                 0x8
-#define GCEA_ADDRDEC1_RM_SEL_CS01__CHAN_BIT__SHIFT                                                            0xc
-#define GCEA_ADDRDEC1_RM_SEL_CS01__INVERT_ROW_MSBS_EVEN__SHIFT                                                0x10
-#define GCEA_ADDRDEC1_RM_SEL_CS01__INVERT_ROW_MSBS_ODD__SHIFT                                                 0x12
-#define GCEA_ADDRDEC1_RM_SEL_CS01__RM0_MASK                                                                   0x0000000FL
-#define GCEA_ADDRDEC1_RM_SEL_CS01__RM1_MASK                                                                   0x000000F0L
-#define GCEA_ADDRDEC1_RM_SEL_CS01__RM2_MASK                                                                   0x00000F00L
-#define GCEA_ADDRDEC1_RM_SEL_CS01__CHAN_BIT_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC1_RM_SEL_CS01__INVERT_ROW_MSBS_EVEN_MASK                                                  0x00030000L
-#define GCEA_ADDRDEC1_RM_SEL_CS01__INVERT_ROW_MSBS_ODD_MASK                                                   0x000C0000L
-//GCEA_ADDRDEC1_RM_SEL_CS23
-#define GCEA_ADDRDEC1_RM_SEL_CS23__RM0__SHIFT                                                                 0x0
-#define GCEA_ADDRDEC1_RM_SEL_CS23__RM1__SHIFT                                                                 0x4
-#define GCEA_ADDRDEC1_RM_SEL_CS23__RM2__SHIFT                                                                 0x8
-#define GCEA_ADDRDEC1_RM_SEL_CS23__CHAN_BIT__SHIFT                                                            0xc
-#define GCEA_ADDRDEC1_RM_SEL_CS23__INVERT_ROW_MSBS_EVEN__SHIFT                                                0x10
-#define GCEA_ADDRDEC1_RM_SEL_CS23__INVERT_ROW_MSBS_ODD__SHIFT                                                 0x12
-#define GCEA_ADDRDEC1_RM_SEL_CS23__RM0_MASK                                                                   0x0000000FL
-#define GCEA_ADDRDEC1_RM_SEL_CS23__RM1_MASK                                                                   0x000000F0L
-#define GCEA_ADDRDEC1_RM_SEL_CS23__RM2_MASK                                                                   0x00000F00L
-#define GCEA_ADDRDEC1_RM_SEL_CS23__CHAN_BIT_MASK                                                              0x0000F000L
-#define GCEA_ADDRDEC1_RM_SEL_CS23__INVERT_ROW_MSBS_EVEN_MASK                                                  0x00030000L
-#define GCEA_ADDRDEC1_RM_SEL_CS23__INVERT_ROW_MSBS_ODD_MASK                                                   0x000C0000L
-//GCEA_ADDRDEC1_RM_SEL_SECCS01
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__RM0__SHIFT                                                              0x0
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__RM1__SHIFT                                                              0x4
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__RM2__SHIFT                                                              0x8
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__CHAN_BIT__SHIFT                                                         0xc
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__INVERT_ROW_MSBS_EVEN__SHIFT                                             0x10
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__INVERT_ROW_MSBS_ODD__SHIFT                                              0x12
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__RM0_MASK                                                                0x0000000FL
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__RM1_MASK                                                                0x000000F0L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__RM2_MASK                                                                0x00000F00L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__CHAN_BIT_MASK                                                           0x0000F000L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__INVERT_ROW_MSBS_EVEN_MASK                                               0x00030000L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS01__INVERT_ROW_MSBS_ODD_MASK                                                0x000C0000L
-//GCEA_ADDRDEC1_RM_SEL_SECCS23
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__RM0__SHIFT                                                              0x0
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__RM1__SHIFT                                                              0x4
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__RM2__SHIFT                                                              0x8
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__CHAN_BIT__SHIFT                                                         0xc
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__INVERT_ROW_MSBS_EVEN__SHIFT                                             0x10
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__INVERT_ROW_MSBS_ODD__SHIFT                                              0x12
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__RM0_MASK                                                                0x0000000FL
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__RM1_MASK                                                                0x000000F0L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__RM2_MASK                                                                0x00000F00L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__CHAN_BIT_MASK                                                           0x0000F000L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__INVERT_ROW_MSBS_EVEN_MASK                                               0x00030000L
-#define GCEA_ADDRDEC1_RM_SEL_SECCS23__INVERT_ROW_MSBS_ODD_MASK                                                0x000C0000L
-//GCEA_IO_RD_CLI2GRP_MAP0
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                            0x0
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                            0x2
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                            0x4
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                            0x6
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                            0x8
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                            0xa
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                            0xc
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                            0xe
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                            0x10
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                            0x12
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                           0x14
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                           0x16
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                           0x18
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                           0x1a
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                           0x1c
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                           0x1e
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                              0x00000003L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                              0x0000000CL
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                              0x00000030L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                              0x000000C0L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                              0x00000300L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                              0x00000C00L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                              0x00003000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                              0x0000C000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                              0x00030000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                              0x000C0000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                             0x00300000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                             0x00C00000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                             0x03000000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                             0x0C000000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                             0x30000000L
-#define GCEA_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                             0xC0000000L
-//GCEA_IO_RD_CLI2GRP_MAP1
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                           0x0
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                           0x2
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                           0x4
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                           0x6
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                           0x8
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                           0xa
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                           0xc
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                           0xe
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                           0x10
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                           0x12
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                           0x14
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                           0x16
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                           0x18
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                           0x1a
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                           0x1c
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                           0x1e
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                             0x00000003L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                             0x0000000CL
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                             0x00000030L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                             0x000000C0L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                             0x00000300L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                             0x00000C00L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                             0x00003000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                             0x0000C000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                             0x00030000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                             0x000C0000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                             0x00300000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                             0x00C00000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                             0x03000000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                             0x0C000000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                             0x30000000L
-#define GCEA_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                             0xC0000000L
-//GCEA_IO_WR_CLI2GRP_MAP0
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                            0x0
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                            0x2
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                            0x4
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                            0x6
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                            0x8
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                            0xa
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                            0xc
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                            0xe
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                            0x10
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                            0x12
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                           0x14
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                           0x16
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                           0x18
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                           0x1a
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                           0x1c
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                           0x1e
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                              0x00000003L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                              0x0000000CL
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                              0x00000030L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                              0x000000C0L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                              0x00000300L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                              0x00000C00L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                              0x00003000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                              0x0000C000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                              0x00030000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                              0x000C0000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                             0x00300000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                             0x00C00000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                             0x03000000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                             0x0C000000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                             0x30000000L
-#define GCEA_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                             0xC0000000L
-//GCEA_IO_WR_CLI2GRP_MAP1
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                           0x0
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                           0x2
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                           0x4
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                           0x6
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                           0x8
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                           0xa
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                           0xc
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                           0xe
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                           0x10
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                           0x12
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                           0x14
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                           0x16
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                           0x18
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                           0x1a
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                           0x1c
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                           0x1e
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                             0x00000003L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                             0x0000000CL
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                             0x00000030L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                             0x000000C0L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                             0x00000300L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                             0x00000C00L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                             0x00003000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                             0x0000C000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                             0x00030000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                             0x000C0000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                             0x00300000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                             0x00C00000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                             0x03000000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                             0x0C000000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                             0x30000000L
-#define GCEA_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                             0xC0000000L
-//GCEA_IO_RD_COMBINE_FLUSH
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                         0x0
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                         0x4
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                         0x8
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                         0xc
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                           0x0000000FL
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                           0x000000F0L
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                           0x00000F00L
-#define GCEA_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                           0x0000F000L
-//GCEA_IO_WR_COMBINE_FLUSH
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                         0x0
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                         0x4
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                         0x8
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                         0xc
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                           0x0000000FL
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                           0x000000F0L
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                           0x00000F00L
-#define GCEA_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                           0x0000F000L
-//GCEA_IO_GROUP_BURST
-#define GCEA_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                               0x0
-#define GCEA_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                               0x8
-#define GCEA_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                               0x10
-#define GCEA_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                               0x18
-#define GCEA_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                 0x000000FFL
-#define GCEA_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                 0x0000FF00L
-#define GCEA_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                 0x00FF0000L
-#define GCEA_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                 0xFF000000L
-//GCEA_IO_RD_PRI_AGE
-#define GCEA_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                          0x0
-#define GCEA_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                          0x3
-#define GCEA_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                          0x6
-#define GCEA_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                          0x9
-#define GCEA_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                     0xc
-#define GCEA_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                     0xf
-#define GCEA_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                     0x12
-#define GCEA_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                     0x15
-#define GCEA_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                            0x00000007L
-#define GCEA_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                            0x00000038L
-#define GCEA_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                            0x000001C0L
-#define GCEA_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                            0x00000E00L
-#define GCEA_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                       0x00007000L
-#define GCEA_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                       0x00038000L
-#define GCEA_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                       0x001C0000L
-#define GCEA_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                       0x00E00000L
-//GCEA_IO_WR_PRI_AGE
-#define GCEA_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                          0x0
-#define GCEA_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                          0x3
-#define GCEA_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                          0x6
-#define GCEA_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                          0x9
-#define GCEA_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                     0xc
-#define GCEA_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                     0xf
-#define GCEA_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                     0x12
-#define GCEA_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                     0x15
-#define GCEA_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                            0x00000007L
-#define GCEA_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                            0x00000038L
-#define GCEA_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                            0x000001C0L
-#define GCEA_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                            0x00000E00L
-#define GCEA_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                       0x00007000L
-#define GCEA_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                       0x00038000L
-#define GCEA_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                       0x001C0000L
-#define GCEA_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                       0x00E00000L
-//GCEA_IO_RD_PRI_QUEUING
-#define GCEA_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                             0x0
-#define GCEA_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                             0x3
-#define GCEA_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                             0x6
-#define GCEA_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                             0x9
-#define GCEA_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                               0x00000007L
-#define GCEA_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                               0x00000038L
-#define GCEA_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                               0x000001C0L
-#define GCEA_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                               0x00000E00L
-//GCEA_IO_WR_PRI_QUEUING
-#define GCEA_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                             0x0
-#define GCEA_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                             0x3
-#define GCEA_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                             0x6
-#define GCEA_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                             0x9
-#define GCEA_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                               0x00000007L
-#define GCEA_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                               0x00000038L
-#define GCEA_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                               0x000001C0L
-#define GCEA_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                               0x00000E00L
-//GCEA_IO_RD_PRI_FIXED
-#define GCEA_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                 0x0
-#define GCEA_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                 0x3
-#define GCEA_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                 0x6
-#define GCEA_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                 0x9
-#define GCEA_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                   0x00000007L
-#define GCEA_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                   0x00000038L
-#define GCEA_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                   0x000001C0L
-#define GCEA_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                   0x00000E00L
-//GCEA_IO_WR_PRI_FIXED
-#define GCEA_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                 0x0
-#define GCEA_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                 0x3
-#define GCEA_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                 0x6
-#define GCEA_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                 0x9
-#define GCEA_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                   0x00000007L
-#define GCEA_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                   0x00000038L
-#define GCEA_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                   0x000001C0L
-#define GCEA_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                   0x00000E00L
-//GCEA_IO_RD_PRI_URGENCY
-#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                             0x0
-#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                             0x3
-#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                             0x6
-#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                             0x9
-#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                    0xc
-#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                    0xd
-#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                    0xe
-#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                    0xf
-#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                               0x00000007L
-#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                               0x00000038L
-#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                               0x000001C0L
-#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                               0x00000E00L
-#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                      0x00001000L
-#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                      0x00002000L
-#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                      0x00004000L
-#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                      0x00008000L
-//GCEA_IO_WR_PRI_URGENCY
-#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                             0x0
-#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                             0x3
-#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                             0x6
-#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                             0x9
-#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                    0xc
-#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                    0xd
-#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                    0xe
-#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                    0xf
-#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                               0x00000007L
-#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                               0x00000038L
-#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                               0x000001C0L
-#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                               0x00000E00L
-#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                      0x00001000L
-#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                      0x00002000L
-#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                      0x00004000L
-#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                      0x00008000L
-//GCEA_IO_RD_PRI_URGENCY_MASK
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID0_MASK__SHIFT                                                         0x0
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID1_MASK__SHIFT                                                         0x1
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID2_MASK__SHIFT                                                         0x2
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID3_MASK__SHIFT                                                         0x3
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID4_MASK__SHIFT                                                         0x4
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID5_MASK__SHIFT                                                         0x5
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID6_MASK__SHIFT                                                         0x6
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID7_MASK__SHIFT                                                         0x7
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID8_MASK__SHIFT                                                         0x8
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID9_MASK__SHIFT                                                         0x9
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID10_MASK__SHIFT                                                        0xa
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID11_MASK__SHIFT                                                        0xb
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID12_MASK__SHIFT                                                        0xc
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID13_MASK__SHIFT                                                        0xd
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID14_MASK__SHIFT                                                        0xe
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID15_MASK__SHIFT                                                        0xf
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID16_MASK__SHIFT                                                        0x10
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID17_MASK__SHIFT                                                        0x11
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID18_MASK__SHIFT                                                        0x12
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID19_MASK__SHIFT                                                        0x13
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID20_MASK__SHIFT                                                        0x14
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID21_MASK__SHIFT                                                        0x15
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID22_MASK__SHIFT                                                        0x16
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID23_MASK__SHIFT                                                        0x17
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID24_MASK__SHIFT                                                        0x18
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID25_MASK__SHIFT                                                        0x19
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID26_MASK__SHIFT                                                        0x1a
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID27_MASK__SHIFT                                                        0x1b
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID28_MASK__SHIFT                                                        0x1c
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID29_MASK__SHIFT                                                        0x1d
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID30_MASK__SHIFT                                                        0x1e
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID31_MASK__SHIFT                                                        0x1f
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID0_MASK_MASK                                                           0x00000001L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID1_MASK_MASK                                                           0x00000002L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID2_MASK_MASK                                                           0x00000004L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID3_MASK_MASK                                                           0x00000008L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID4_MASK_MASK                                                           0x00000010L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID5_MASK_MASK                                                           0x00000020L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID6_MASK_MASK                                                           0x00000040L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID7_MASK_MASK                                                           0x00000080L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID8_MASK_MASK                                                           0x00000100L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID9_MASK_MASK                                                           0x00000200L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID10_MASK_MASK                                                          0x00000400L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID11_MASK_MASK                                                          0x00000800L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID12_MASK_MASK                                                          0x00001000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID13_MASK_MASK                                                          0x00002000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID14_MASK_MASK                                                          0x00004000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID15_MASK_MASK                                                          0x00008000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID16_MASK_MASK                                                          0x00010000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID17_MASK_MASK                                                          0x00020000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID18_MASK_MASK                                                          0x00040000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID19_MASK_MASK                                                          0x00080000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID20_MASK_MASK                                                          0x00100000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID21_MASK_MASK                                                          0x00200000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID22_MASK_MASK                                                          0x00400000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID23_MASK_MASK                                                          0x00800000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID24_MASK_MASK                                                          0x01000000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID25_MASK_MASK                                                          0x02000000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID26_MASK_MASK                                                          0x04000000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID27_MASK_MASK                                                          0x08000000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID28_MASK_MASK                                                          0x10000000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID29_MASK_MASK                                                          0x20000000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID30_MASK_MASK                                                          0x40000000L
-#define GCEA_IO_RD_PRI_URGENCY_MASK__CID31_MASK_MASK                                                          0x80000000L
-//GCEA_IO_WR_PRI_URGENCY_MASK
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID0_MASK__SHIFT                                                         0x0
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID1_MASK__SHIFT                                                         0x1
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID2_MASK__SHIFT                                                         0x2
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID3_MASK__SHIFT                                                         0x3
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID4_MASK__SHIFT                                                         0x4
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID5_MASK__SHIFT                                                         0x5
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID6_MASK__SHIFT                                                         0x6
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID7_MASK__SHIFT                                                         0x7
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID8_MASK__SHIFT                                                         0x8
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID9_MASK__SHIFT                                                         0x9
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID10_MASK__SHIFT                                                        0xa
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID11_MASK__SHIFT                                                        0xb
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID12_MASK__SHIFT                                                        0xc
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID13_MASK__SHIFT                                                        0xd
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID14_MASK__SHIFT                                                        0xe
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID15_MASK__SHIFT                                                        0xf
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID16_MASK__SHIFT                                                        0x10
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID17_MASK__SHIFT                                                        0x11
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID18_MASK__SHIFT                                                        0x12
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID19_MASK__SHIFT                                                        0x13
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID20_MASK__SHIFT                                                        0x14
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID21_MASK__SHIFT                                                        0x15
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID22_MASK__SHIFT                                                        0x16
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID23_MASK__SHIFT                                                        0x17
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID24_MASK__SHIFT                                                        0x18
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID25_MASK__SHIFT                                                        0x19
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID26_MASK__SHIFT                                                        0x1a
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID27_MASK__SHIFT                                                        0x1b
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID28_MASK__SHIFT                                                        0x1c
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID29_MASK__SHIFT                                                        0x1d
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID30_MASK__SHIFT                                                        0x1e
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID31_MASK__SHIFT                                                        0x1f
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID0_MASK_MASK                                                           0x00000001L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID1_MASK_MASK                                                           0x00000002L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID2_MASK_MASK                                                           0x00000004L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID3_MASK_MASK                                                           0x00000008L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID4_MASK_MASK                                                           0x00000010L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID5_MASK_MASK                                                           0x00000020L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID6_MASK_MASK                                                           0x00000040L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID7_MASK_MASK                                                           0x00000080L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID8_MASK_MASK                                                           0x00000100L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID9_MASK_MASK                                                           0x00000200L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID10_MASK_MASK                                                          0x00000400L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID11_MASK_MASK                                                          0x00000800L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID12_MASK_MASK                                                          0x00001000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID13_MASK_MASK                                                          0x00002000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID14_MASK_MASK                                                          0x00004000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID15_MASK_MASK                                                          0x00008000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID16_MASK_MASK                                                          0x00010000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID17_MASK_MASK                                                          0x00020000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID18_MASK_MASK                                                          0x00040000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID19_MASK_MASK                                                          0x00080000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID20_MASK_MASK                                                          0x00100000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID21_MASK_MASK                                                          0x00200000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID22_MASK_MASK                                                          0x00400000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID23_MASK_MASK                                                          0x00800000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID24_MASK_MASK                                                          0x01000000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID25_MASK_MASK                                                          0x02000000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID26_MASK_MASK                                                          0x04000000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID27_MASK_MASK                                                          0x08000000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID28_MASK_MASK                                                          0x10000000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID29_MASK_MASK                                                          0x20000000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID30_MASK_MASK                                                          0x40000000L
-#define GCEA_IO_WR_PRI_URGENCY_MASK__CID31_MASK_MASK                                                          0x80000000L
-//GCEA_IO_RD_PRI_QUANT_PRI1
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                    0x0
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                    0x8
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                    0x10
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                    0x18
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
-#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
-//GCEA_IO_RD_PRI_QUANT_PRI2
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                    0x0
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                    0x8
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                    0x10
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                    0x18
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
-#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
-//GCEA_IO_RD_PRI_QUANT_PRI3
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                    0x0
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                    0x8
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                    0x10
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                    0x18
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
-#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
-//GCEA_IO_WR_PRI_QUANT_PRI1
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                    0x0
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                    0x8
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                    0x10
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                    0x18
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
-#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
-//GCEA_IO_WR_PRI_QUANT_PRI2
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                    0x0
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                    0x8
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                    0x10
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                    0x18
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
-#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
-//GCEA_IO_WR_PRI_QUANT_PRI3
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                    0x0
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                    0x8
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                    0x10
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                    0x18
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
-#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
-//GCEA_SDP_ARB_DRAM
-#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
-#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
-#define GCEA_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                               0x14
-#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
-#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
-#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
-#define GCEA_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
-//GCEA_SDP_ARB_FINAL
-#define GCEA_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                           0x0
-#define GCEA_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                            0x5
-#define GCEA_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                             0xa
-#define GCEA_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                     0xf
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                 0x11
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                 0x12
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                 0x13
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                 0x14
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                 0x15
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                 0x16
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                 0x17
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                 0x18
-#define GCEA_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                          0x19
-#define GCEA_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                           0x1a
-#define GCEA_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                             0x0000001FL
-#define GCEA_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                              0x000003E0L
-#define GCEA_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                               0x00007C00L
-#define GCEA_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                       0x00018000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                   0x00020000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                   0x00040000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                   0x00080000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                   0x00100000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                   0x00200000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                   0x00400000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                   0x00800000L
-#define GCEA_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                   0x01000000L
-#define GCEA_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                            0x02000000L
-#define GCEA_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                             0x04000000L
-//GCEA_SDP_DRAM_PRIORITY
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
-#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
-#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
-//GCEA_SDP_IO_PRIORITY
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                       0x0
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                       0x4
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                       0x8
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                       0xc
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                       0x10
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                       0x14
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                       0x18
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                       0x1c
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                         0x0000000FL
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                         0x000000F0L
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                         0x00000F00L
-#define GCEA_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                         0x0000F000L
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                         0x000F0000L
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                         0x00F00000L
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                         0x0F000000L
-#define GCEA_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                         0xF0000000L
-//GCEA_SDP_CREDITS
-#define GCEA_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                    0x0
-#define GCEA_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                              0x8
-#define GCEA_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                              0x10
-#define GCEA_SDP_CREDITS__PRB_REQ_CREDITS__SHIFT                                                              0x18
-#define GCEA_SDP_CREDITS__TAG_LIMIT_MASK                                                                      0x000000FFL
-#define GCEA_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                                0x00007F00L
-#define GCEA_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                                0x007F0000L
-#define GCEA_SDP_CREDITS__PRB_REQ_CREDITS_MASK                                                                0x3F000000L
-//GCEA_SDP_TAG_RESERVE0
-#define GCEA_SDP_TAG_RESERVE0__VC0__SHIFT                                                                     0x0
-#define GCEA_SDP_TAG_RESERVE0__VC1__SHIFT                                                                     0x8
-#define GCEA_SDP_TAG_RESERVE0__VC2__SHIFT                                                                     0x10
-#define GCEA_SDP_TAG_RESERVE0__VC3__SHIFT                                                                     0x18
-#define GCEA_SDP_TAG_RESERVE0__VC0_MASK                                                                       0x000000FFL
-#define GCEA_SDP_TAG_RESERVE0__VC1_MASK                                                                       0x0000FF00L
-#define GCEA_SDP_TAG_RESERVE0__VC2_MASK                                                                       0x00FF0000L
-#define GCEA_SDP_TAG_RESERVE0__VC3_MASK                                                                       0xFF000000L
-//GCEA_SDP_TAG_RESERVE1
-#define GCEA_SDP_TAG_RESERVE1__VC4__SHIFT                                                                     0x0
-#define GCEA_SDP_TAG_RESERVE1__VC5__SHIFT                                                                     0x8
-#define GCEA_SDP_TAG_RESERVE1__VC6__SHIFT                                                                     0x10
-#define GCEA_SDP_TAG_RESERVE1__VC7__SHIFT                                                                     0x18
-#define GCEA_SDP_TAG_RESERVE1__VC4_MASK                                                                       0x000000FFL
-#define GCEA_SDP_TAG_RESERVE1__VC5_MASK                                                                       0x0000FF00L
-#define GCEA_SDP_TAG_RESERVE1__VC6_MASK                                                                       0x00FF0000L
-#define GCEA_SDP_TAG_RESERVE1__VC7_MASK                                                                       0xFF000000L
-//GCEA_SDP_VCC_RESERVE0
-#define GCEA_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                             0x0
-#define GCEA_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                             0x6
-#define GCEA_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                             0xc
-#define GCEA_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                             0x12
-#define GCEA_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                             0x18
-#define GCEA_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                               0x0000003FL
-#define GCEA_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                               0x00000FC0L
-#define GCEA_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                               0x0003F000L
-#define GCEA_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                               0x00FC0000L
-#define GCEA_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                               0x3F000000L
-//GCEA_SDP_VCC_RESERVE1
-#define GCEA_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                             0x0
-#define GCEA_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                             0x6
-#define GCEA_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                             0xc
-#define GCEA_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                         0x1f
-#define GCEA_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                               0x0000003FL
-#define GCEA_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                               0x00000FC0L
-#define GCEA_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                               0x0003F000L
-#define GCEA_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                           0x80000000L
-//GCEA_SDP_VCD_RESERVE0
-#define GCEA_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                             0x0
-#define GCEA_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                             0x6
-#define GCEA_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                             0xc
-#define GCEA_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                             0x12
-#define GCEA_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                             0x18
-#define GCEA_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                               0x0000003FL
-#define GCEA_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                               0x00000FC0L
-#define GCEA_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                               0x0003F000L
-#define GCEA_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                               0x00FC0000L
-#define GCEA_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                               0x3F000000L
-//GCEA_SDP_VCD_RESERVE1
-#define GCEA_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                             0x0
-#define GCEA_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                             0x6
-#define GCEA_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                             0xc
-#define GCEA_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                         0x1f
-#define GCEA_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                               0x0000003FL
-#define GCEA_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                               0x00000FC0L
-#define GCEA_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                               0x0003F000L
-#define GCEA_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                           0x80000000L
-//GCEA_SDP_REQ_CNTL
-#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                   0x0
-#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                  0x1
-#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                 0x2
-#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                     0x3
-#define GCEA_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                           0x4
-#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                     0x00000001L
-#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                    0x00000002L
-#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                   0x00000004L
-#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                       0x00000008L
-#define GCEA_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                             0x00000010L
-//GCEA_MISC
-#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                         0x0
-#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                         0x1
-#define GCEA_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                          0x2
-#define GCEA_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                          0x3
-#define GCEA_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                           0x4
-#define GCEA_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                           0x5
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                               0x6
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                               0x7
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                               0x8
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                               0x9
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                               0xa
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                               0xb
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                               0xc
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                               0xd
-#define GCEA_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                  0xe
-#define GCEA_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                                0xf
-#define GCEA_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                              0x11
-#define GCEA_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                             0x13
-#define GCEA_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                              0x15
-#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                      0x1a
-#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                       0x1b
-#define GCEA_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                          0x1c
-#define GCEA_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                           0x1d
-#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                        0x1e
-#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                         0x1f
-#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                           0x00000001L
-#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                           0x00000002L
-#define GCEA_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                            0x00000004L
-#define GCEA_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                            0x00000008L
-#define GCEA_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                             0x00000010L
-#define GCEA_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                             0x00000020L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                 0x00000040L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                 0x00000080L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                 0x00000100L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                 0x00000200L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                 0x00000400L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                 0x00000800L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                 0x00001000L
-#define GCEA_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                 0x00002000L
-#define GCEA_MISC__EARLY_SDP_ORIGDATA_MASK                                                                    0x00004000L
-#define GCEA_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                  0x00018000L
-#define GCEA_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                                0x00060000L
-#define GCEA_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                               0x00180000L
-#define GCEA_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                                0x03E00000L
-#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                        0x04000000L
-#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                         0x08000000L
-#define GCEA_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                            0x10000000L
-#define GCEA_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                             0x20000000L
-#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                          0x40000000L
-#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                           0x80000000L
-//GCEA_LATENCY_SAMPLING
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                           0x0
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                           0x1
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                            0x2
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                            0x3
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                             0x4
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                             0x5
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                           0x6
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                           0x7
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                          0x8
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                          0x9
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                     0xa
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                     0xb
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                   0xc
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                   0xd
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                             0xe
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                             0x16
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                             0x00000001L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                             0x00000002L
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                              0x00000004L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                              0x00000008L
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                               0x00000010L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                               0x00000020L
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                             0x00000040L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                             0x00000080L
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                            0x00000100L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                            0x00000200L
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                       0x00000400L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                       0x00000800L
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                     0x00001000L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                     0x00002000L
-#define GCEA_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                               0x003FC000L
-#define GCEA_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                               0x3FC00000L
-//GCEA_PERFCOUNTER_LO
-#define GCEA_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                0x0
-#define GCEA_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                  0xFFFFFFFFL
-//GCEA_PERFCOUNTER_HI
-#define GCEA_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                0x0
-#define GCEA_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                             0x10
-#define GCEA_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                  0x0000FFFFL
-#define GCEA_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                               0xFFFF0000L
-//GCEA_PERFCOUNTER0_CFG
-#define GCEA_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                0x0
-#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                            0x8
-#define GCEA_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                               0x18
-#define GCEA_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                  0x1c
-#define GCEA_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                   0x1d
-#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                  0x000000FFL
-#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                              0x0000FF00L
-#define GCEA_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                 0x0F000000L
-#define GCEA_PERFCOUNTER0_CFG__ENABLE_MASK                                                                    0x10000000L
-#define GCEA_PERFCOUNTER0_CFG__CLEAR_MASK                                                                     0x20000000L
-//GCEA_PERFCOUNTER1_CFG
-#define GCEA_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                0x0
-#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                            0x8
-#define GCEA_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                               0x18
-#define GCEA_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                  0x1c
-#define GCEA_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                   0x1d
-#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                  0x000000FFL
-#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                              0x0000FF00L
-#define GCEA_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                 0x0F000000L
-#define GCEA_PERFCOUNTER1_CFG__ENABLE_MASK                                                                    0x10000000L
-#define GCEA_PERFCOUNTER1_CFG__CLEAR_MASK                                                                     0x20000000L
-//GCEA_PERFCOUNTER_RSLT_CNTL
-#define GCEA_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                0x0
-#define GCEA_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                      0x8
-#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                       0x10
-#define GCEA_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                         0x18
-#define GCEA_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                          0x19
-#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                               0x1a
-#define GCEA_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                  0x0000000FL
-#define GCEA_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                        0x0000FF00L
-#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                         0x00FF0000L
-#define GCEA_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                           0x01000000L
-#define GCEA_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                            0x02000000L
-#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                 0x04000000L
-
-
-// addressBlock: gc_tcdec
-//TCP_INVALIDATE
-#define TCP_INVALIDATE__START__SHIFT                                                                          0x0
-#define TCP_INVALIDATE__START_MASK                                                                            0x00000001L
-//TCP_STATUS
-#define TCP_STATUS__TCP_BUSY__SHIFT                                                                           0x0
-#define TCP_STATUS__INPUT_BUSY__SHIFT                                                                         0x1
-#define TCP_STATUS__ADRS_BUSY__SHIFT                                                                          0x2
-#define TCP_STATUS__TAGRAMS_BUSY__SHIFT                                                                       0x3
-#define TCP_STATUS__CNTRL_BUSY__SHIFT                                                                         0x4
-#define TCP_STATUS__LFIFO_BUSY__SHIFT                                                                         0x5
-#define TCP_STATUS__READ_BUSY__SHIFT                                                                          0x6
-#define TCP_STATUS__FORMAT_BUSY__SHIFT                                                                        0x7
-#define TCP_STATUS__VM_BUSY__SHIFT                                                                            0x8
-#define TCP_STATUS__TCP_BUSY_MASK                                                                             0x00000001L
-#define TCP_STATUS__INPUT_BUSY_MASK                                                                           0x00000002L
-#define TCP_STATUS__ADRS_BUSY_MASK                                                                            0x00000004L
-#define TCP_STATUS__TAGRAMS_BUSY_MASK                                                                         0x00000008L
-#define TCP_STATUS__CNTRL_BUSY_MASK                                                                           0x00000010L
-#define TCP_STATUS__LFIFO_BUSY_MASK                                                                           0x00000020L
-#define TCP_STATUS__READ_BUSY_MASK                                                                            0x00000040L
-#define TCP_STATUS__FORMAT_BUSY_MASK                                                                          0x00000080L
-#define TCP_STATUS__VM_BUSY_MASK                                                                              0x00000100L
-//TCP_CNTL
-#define TCP_CNTL__FORCE_HIT__SHIFT                                                                            0x0
-#define TCP_CNTL__FORCE_MISS__SHIFT                                                                           0x1
-#define TCP_CNTL__L1_SIZE__SHIFT                                                                              0x2
-#define TCP_CNTL__FLAT_BUF_HASH_ENABLE__SHIFT                                                                 0x4
-#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE__SHIFT                                                               0x5
-#define TCP_CNTL__FORCE_EOW_TOTAL_CNT__SHIFT                                                                  0xf
-#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT__SHIFT                                                                 0x16
-#define TCP_CNTL__DISABLE_Z_MAP__SHIFT                                                                        0x1c
-#define TCP_CNTL__INV_ALL_VMIDS__SHIFT                                                                        0x1d
-#define TCP_CNTL__ASTC_VE_MSB_TOLERANT__SHIFT                                                                 0x1e
-#define TCP_CNTL__FORCE_HIT_MASK                                                                              0x00000001L
-#define TCP_CNTL__FORCE_MISS_MASK                                                                             0x00000002L
-#define TCP_CNTL__L1_SIZE_MASK                                                                                0x0000000CL
-#define TCP_CNTL__FLAT_BUF_HASH_ENABLE_MASK                                                                   0x00000010L
-#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE_MASK                                                                 0x00000020L
-#define TCP_CNTL__FORCE_EOW_TOTAL_CNT_MASK                                                                    0x001F8000L
-#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT_MASK                                                                   0x0FC00000L
-#define TCP_CNTL__DISABLE_Z_MAP_MASK                                                                          0x10000000L
-#define TCP_CNTL__INV_ALL_VMIDS_MASK                                                                          0x20000000L
-#define TCP_CNTL__ASTC_VE_MSB_TOLERANT_MASK                                                                   0x40000000L
-//TCP_CHAN_STEER_LO
-#define TCP_CHAN_STEER_LO__CHAN0__SHIFT                                                                       0x0
-#define TCP_CHAN_STEER_LO__CHAN1__SHIFT                                                                       0x4
-#define TCP_CHAN_STEER_LO__CHAN2__SHIFT                                                                       0x8
-#define TCP_CHAN_STEER_LO__CHAN3__SHIFT                                                                       0xc
-#define TCP_CHAN_STEER_LO__CHAN4__SHIFT                                                                       0x10
-#define TCP_CHAN_STEER_LO__CHAN5__SHIFT                                                                       0x14
-#define TCP_CHAN_STEER_LO__CHAN6__SHIFT                                                                       0x18
-#define TCP_CHAN_STEER_LO__CHAN7__SHIFT                                                                       0x1c
-#define TCP_CHAN_STEER_LO__CHAN0_MASK                                                                         0x0000000FL
-#define TCP_CHAN_STEER_LO__CHAN1_MASK                                                                         0x000000F0L
-#define TCP_CHAN_STEER_LO__CHAN2_MASK                                                                         0x00000F00L
-#define TCP_CHAN_STEER_LO__CHAN3_MASK                                                                         0x0000F000L
-#define TCP_CHAN_STEER_LO__CHAN4_MASK                                                                         0x000F0000L
-#define TCP_CHAN_STEER_LO__CHAN5_MASK                                                                         0x00F00000L
-#define TCP_CHAN_STEER_LO__CHAN6_MASK                                                                         0x0F000000L
-#define TCP_CHAN_STEER_LO__CHAN7_MASK                                                                         0xF0000000L
-//TCP_CHAN_STEER_HI
-#define TCP_CHAN_STEER_HI__CHAN8__SHIFT                                                                       0x0
-#define TCP_CHAN_STEER_HI__CHAN9__SHIFT                                                                       0x4
-#define TCP_CHAN_STEER_HI__CHANA__SHIFT                                                                       0x8
-#define TCP_CHAN_STEER_HI__CHANB__SHIFT                                                                       0xc
-#define TCP_CHAN_STEER_HI__CHANC__SHIFT                                                                       0x10
-#define TCP_CHAN_STEER_HI__CHAND__SHIFT                                                                       0x14
-#define TCP_CHAN_STEER_HI__CHANE__SHIFT                                                                       0x18
-#define TCP_CHAN_STEER_HI__CHANF__SHIFT                                                                       0x1c
-#define TCP_CHAN_STEER_HI__CHAN8_MASK                                                                         0x0000000FL
-#define TCP_CHAN_STEER_HI__CHAN9_MASK                                                                         0x000000F0L
-#define TCP_CHAN_STEER_HI__CHANA_MASK                                                                         0x00000F00L
-#define TCP_CHAN_STEER_HI__CHANB_MASK                                                                         0x0000F000L
-#define TCP_CHAN_STEER_HI__CHANC_MASK                                                                         0x000F0000L
-#define TCP_CHAN_STEER_HI__CHAND_MASK                                                                         0x00F00000L
-#define TCP_CHAN_STEER_HI__CHANE_MASK                                                                         0x0F000000L
-#define TCP_CHAN_STEER_HI__CHANF_MASK                                                                         0xF0000000L
-//TCP_ADDR_CONFIG
-#define TCP_ADDR_CONFIG__NUM_TCC_BANKS__SHIFT                                                                 0x0
-#define TCP_ADDR_CONFIG__NUM_BANKS__SHIFT                                                                     0x4
-#define TCP_ADDR_CONFIG__COLHI_WIDTH__SHIFT                                                                   0x6
-#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI__SHIFT                                                                0x9
-#define TCP_ADDR_CONFIG__NUM_TCC_BANKS_MASK                                                                   0x0000000FL
-#define TCP_ADDR_CONFIG__NUM_BANKS_MASK                                                                       0x00000030L
-#define TCP_ADDR_CONFIG__COLHI_WIDTH_MASK                                                                     0x000001C0L
-#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI_MASK                                                                  0x00000200L
-//TCP_CREDIT
-#define TCP_CREDIT__LFIFO_CREDIT__SHIFT                                                                       0x0
-#define TCP_CREDIT__REQ_FIFO_CREDIT__SHIFT                                                                    0x10
-#define TCP_CREDIT__TD_CREDIT__SHIFT                                                                          0x1d
-#define TCP_CREDIT__LFIFO_CREDIT_MASK                                                                         0x000003FFL
-#define TCP_CREDIT__REQ_FIFO_CREDIT_MASK                                                                      0x007F0000L
-#define TCP_CREDIT__TD_CREDIT_MASK                                                                            0xE0000000L
-//TCP_BUFFER_ADDR_HASH_CNTL
-#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS__SHIFT                                                        0x0
-#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS__SHIFT                                                           0x8
-#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT__SHIFT                                                   0x10
-#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT__SHIFT                                                      0x18
-#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS_MASK                                                          0x00000007L
-#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS_MASK                                                             0x00000700L
-#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT_MASK                                                     0x00070000L
-#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT_MASK                                                        0x07000000L
-//TCP_EDC_CNT
-#define TCP_EDC_CNT__SEC_COUNT__SHIFT                                                                         0x0
-#define TCP_EDC_CNT__LFIFO_SED_COUNT__SHIFT                                                                   0x8
-#define TCP_EDC_CNT__DED_COUNT__SHIFT                                                                         0x10
-#define TCP_EDC_CNT__SEC_COUNT_MASK                                                                           0x000000FFL
-#define TCP_EDC_CNT__LFIFO_SED_COUNT_MASK                                                                     0x0000FF00L
-#define TCP_EDC_CNT__DED_COUNT_MASK                                                                           0x00FF0000L
-//TC_CFG_L1_LOAD_POLICY0
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_0__SHIFT                                                               0x0
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_1__SHIFT                                                               0x2
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_2__SHIFT                                                               0x4
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_3__SHIFT                                                               0x6
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_4__SHIFT                                                               0x8
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_5__SHIFT                                                               0xa
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_6__SHIFT                                                               0xc
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_7__SHIFT                                                               0xe
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_8__SHIFT                                                               0x10
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_9__SHIFT                                                               0x12
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_10__SHIFT                                                              0x14
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_11__SHIFT                                                              0x16
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_12__SHIFT                                                              0x18
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_13__SHIFT                                                              0x1a
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_14__SHIFT                                                              0x1c
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_15__SHIFT                                                              0x1e
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_0_MASK                                                                 0x00000003L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_1_MASK                                                                 0x0000000CL
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_2_MASK                                                                 0x00000030L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_3_MASK                                                                 0x000000C0L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_4_MASK                                                                 0x00000300L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_5_MASK                                                                 0x00000C00L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_6_MASK                                                                 0x00003000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_7_MASK                                                                 0x0000C000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_8_MASK                                                                 0x00030000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_9_MASK                                                                 0x000C0000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_10_MASK                                                                0x00300000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_11_MASK                                                                0x00C00000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_12_MASK                                                                0x03000000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_13_MASK                                                                0x0C000000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_14_MASK                                                                0x30000000L
-#define TC_CFG_L1_LOAD_POLICY0__POLICY_15_MASK                                                                0xC0000000L
-//TC_CFG_L1_LOAD_POLICY1
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_16__SHIFT                                                              0x0
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_17__SHIFT                                                              0x2
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_18__SHIFT                                                              0x4
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_19__SHIFT                                                              0x6
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_20__SHIFT                                                              0x8
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_21__SHIFT                                                              0xa
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_22__SHIFT                                                              0xc
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_23__SHIFT                                                              0xe
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_24__SHIFT                                                              0x10
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_25__SHIFT                                                              0x12
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_26__SHIFT                                                              0x14
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_27__SHIFT                                                              0x16
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_28__SHIFT                                                              0x18
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_29__SHIFT                                                              0x1a
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_30__SHIFT                                                              0x1c
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_31__SHIFT                                                              0x1e
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_16_MASK                                                                0x00000003L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_17_MASK                                                                0x0000000CL
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_18_MASK                                                                0x00000030L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_19_MASK                                                                0x000000C0L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_20_MASK                                                                0x00000300L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_21_MASK                                                                0x00000C00L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_22_MASK                                                                0x00003000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_23_MASK                                                                0x0000C000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_24_MASK                                                                0x00030000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_25_MASK                                                                0x000C0000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_26_MASK                                                                0x00300000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_27_MASK                                                                0x00C00000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_28_MASK                                                                0x03000000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_29_MASK                                                                0x0C000000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_30_MASK                                                                0x30000000L
-#define TC_CFG_L1_LOAD_POLICY1__POLICY_31_MASK                                                                0xC0000000L
-//TC_CFG_L1_STORE_POLICY
-#define TC_CFG_L1_STORE_POLICY__POLICY_0__SHIFT                                                               0x0
-#define TC_CFG_L1_STORE_POLICY__POLICY_1__SHIFT                                                               0x1
-#define TC_CFG_L1_STORE_POLICY__POLICY_2__SHIFT                                                               0x2
-#define TC_CFG_L1_STORE_POLICY__POLICY_3__SHIFT                                                               0x3
-#define TC_CFG_L1_STORE_POLICY__POLICY_4__SHIFT                                                               0x4
-#define TC_CFG_L1_STORE_POLICY__POLICY_5__SHIFT                                                               0x5
-#define TC_CFG_L1_STORE_POLICY__POLICY_6__SHIFT                                                               0x6
-#define TC_CFG_L1_STORE_POLICY__POLICY_7__SHIFT                                                               0x7
-#define TC_CFG_L1_STORE_POLICY__POLICY_8__SHIFT                                                               0x8
-#define TC_CFG_L1_STORE_POLICY__POLICY_9__SHIFT                                                               0x9
-#define TC_CFG_L1_STORE_POLICY__POLICY_10__SHIFT                                                              0xa
-#define TC_CFG_L1_STORE_POLICY__POLICY_11__SHIFT                                                              0xb
-#define TC_CFG_L1_STORE_POLICY__POLICY_12__SHIFT                                                              0xc
-#define TC_CFG_L1_STORE_POLICY__POLICY_13__SHIFT                                                              0xd
-#define TC_CFG_L1_STORE_POLICY__POLICY_14__SHIFT                                                              0xe
-#define TC_CFG_L1_STORE_POLICY__POLICY_15__SHIFT                                                              0xf
-#define TC_CFG_L1_STORE_POLICY__POLICY_16__SHIFT                                                              0x10
-#define TC_CFG_L1_STORE_POLICY__POLICY_17__SHIFT                                                              0x11
-#define TC_CFG_L1_STORE_POLICY__POLICY_18__SHIFT                                                              0x12
-#define TC_CFG_L1_STORE_POLICY__POLICY_19__SHIFT                                                              0x13
-#define TC_CFG_L1_STORE_POLICY__POLICY_20__SHIFT                                                              0x14
-#define TC_CFG_L1_STORE_POLICY__POLICY_21__SHIFT                                                              0x15
-#define TC_CFG_L1_STORE_POLICY__POLICY_22__SHIFT                                                              0x16
-#define TC_CFG_L1_STORE_POLICY__POLICY_23__SHIFT                                                              0x17
-#define TC_CFG_L1_STORE_POLICY__POLICY_24__SHIFT                                                              0x18
-#define TC_CFG_L1_STORE_POLICY__POLICY_25__SHIFT                                                              0x19
-#define TC_CFG_L1_STORE_POLICY__POLICY_26__SHIFT                                                              0x1a
-#define TC_CFG_L1_STORE_POLICY__POLICY_27__SHIFT                                                              0x1b
-#define TC_CFG_L1_STORE_POLICY__POLICY_28__SHIFT                                                              0x1c
-#define TC_CFG_L1_STORE_POLICY__POLICY_29__SHIFT                                                              0x1d
-#define TC_CFG_L1_STORE_POLICY__POLICY_30__SHIFT                                                              0x1e
-#define TC_CFG_L1_STORE_POLICY__POLICY_31__SHIFT                                                              0x1f
-#define TC_CFG_L1_STORE_POLICY__POLICY_0_MASK                                                                 0x00000001L
-#define TC_CFG_L1_STORE_POLICY__POLICY_1_MASK                                                                 0x00000002L
-#define TC_CFG_L1_STORE_POLICY__POLICY_2_MASK                                                                 0x00000004L
-#define TC_CFG_L1_STORE_POLICY__POLICY_3_MASK                                                                 0x00000008L
-#define TC_CFG_L1_STORE_POLICY__POLICY_4_MASK                                                                 0x00000010L
-#define TC_CFG_L1_STORE_POLICY__POLICY_5_MASK                                                                 0x00000020L
-#define TC_CFG_L1_STORE_POLICY__POLICY_6_MASK                                                                 0x00000040L
-#define TC_CFG_L1_STORE_POLICY__POLICY_7_MASK                                                                 0x00000080L
-#define TC_CFG_L1_STORE_POLICY__POLICY_8_MASK                                                                 0x00000100L
-#define TC_CFG_L1_STORE_POLICY__POLICY_9_MASK                                                                 0x00000200L
-#define TC_CFG_L1_STORE_POLICY__POLICY_10_MASK                                                                0x00000400L
-#define TC_CFG_L1_STORE_POLICY__POLICY_11_MASK                                                                0x00000800L
-#define TC_CFG_L1_STORE_POLICY__POLICY_12_MASK                                                                0x00001000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_13_MASK                                                                0x00002000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_14_MASK                                                                0x00004000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_15_MASK                                                                0x00008000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_16_MASK                                                                0x00010000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_17_MASK                                                                0x00020000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_18_MASK                                                                0x00040000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_19_MASK                                                                0x00080000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_20_MASK                                                                0x00100000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_21_MASK                                                                0x00200000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_22_MASK                                                                0x00400000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_23_MASK                                                                0x00800000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_24_MASK                                                                0x01000000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_25_MASK                                                                0x02000000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_26_MASK                                                                0x04000000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_27_MASK                                                                0x08000000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_28_MASK                                                                0x10000000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_29_MASK                                                                0x20000000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_30_MASK                                                                0x40000000L
-#define TC_CFG_L1_STORE_POLICY__POLICY_31_MASK                                                                0x80000000L
-//TC_CFG_L2_LOAD_POLICY0
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_0__SHIFT                                                               0x0
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_1__SHIFT                                                               0x2
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_2__SHIFT                                                               0x4
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_3__SHIFT                                                               0x6
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_4__SHIFT                                                               0x8
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_5__SHIFT                                                               0xa
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_6__SHIFT                                                               0xc
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_7__SHIFT                                                               0xe
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_8__SHIFT                                                               0x10
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_9__SHIFT                                                               0x12
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_10__SHIFT                                                              0x14
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_11__SHIFT                                                              0x16
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_12__SHIFT                                                              0x18
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_13__SHIFT                                                              0x1a
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_14__SHIFT                                                              0x1c
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_15__SHIFT                                                              0x1e
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_0_MASK                                                                 0x00000003L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_1_MASK                                                                 0x0000000CL
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_2_MASK                                                                 0x00000030L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_3_MASK                                                                 0x000000C0L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_4_MASK                                                                 0x00000300L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_5_MASK                                                                 0x00000C00L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_6_MASK                                                                 0x00003000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_7_MASK                                                                 0x0000C000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_8_MASK                                                                 0x00030000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_9_MASK                                                                 0x000C0000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_10_MASK                                                                0x00300000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_11_MASK                                                                0x00C00000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_12_MASK                                                                0x03000000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_13_MASK                                                                0x0C000000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_14_MASK                                                                0x30000000L
-#define TC_CFG_L2_LOAD_POLICY0__POLICY_15_MASK                                                                0xC0000000L
-//TC_CFG_L2_LOAD_POLICY1
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_16__SHIFT                                                              0x0
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_17__SHIFT                                                              0x2
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_18__SHIFT                                                              0x4
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_19__SHIFT                                                              0x6
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_20__SHIFT                                                              0x8
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_21__SHIFT                                                              0xa
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_22__SHIFT                                                              0xc
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_23__SHIFT                                                              0xe
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_24__SHIFT                                                              0x10
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_25__SHIFT                                                              0x12
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_26__SHIFT                                                              0x14
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_27__SHIFT                                                              0x16
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_28__SHIFT                                                              0x18
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_29__SHIFT                                                              0x1a
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_30__SHIFT                                                              0x1c
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_31__SHIFT                                                              0x1e
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_16_MASK                                                                0x00000003L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_17_MASK                                                                0x0000000CL
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_18_MASK                                                                0x00000030L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_19_MASK                                                                0x000000C0L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_20_MASK                                                                0x00000300L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_21_MASK                                                                0x00000C00L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_22_MASK                                                                0x00003000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_23_MASK                                                                0x0000C000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_24_MASK                                                                0x00030000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_25_MASK                                                                0x000C0000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_26_MASK                                                                0x00300000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_27_MASK                                                                0x00C00000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_28_MASK                                                                0x03000000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_29_MASK                                                                0x0C000000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_30_MASK                                                                0x30000000L
-#define TC_CFG_L2_LOAD_POLICY1__POLICY_31_MASK                                                                0xC0000000L
-//TC_CFG_L2_STORE_POLICY0
-#define TC_CFG_L2_STORE_POLICY0__POLICY_0__SHIFT                                                              0x0
-#define TC_CFG_L2_STORE_POLICY0__POLICY_1__SHIFT                                                              0x2
-#define TC_CFG_L2_STORE_POLICY0__POLICY_2__SHIFT                                                              0x4
-#define TC_CFG_L2_STORE_POLICY0__POLICY_3__SHIFT                                                              0x6
-#define TC_CFG_L2_STORE_POLICY0__POLICY_4__SHIFT                                                              0x8
-#define TC_CFG_L2_STORE_POLICY0__POLICY_5__SHIFT                                                              0xa
-#define TC_CFG_L2_STORE_POLICY0__POLICY_6__SHIFT                                                              0xc
-#define TC_CFG_L2_STORE_POLICY0__POLICY_7__SHIFT                                                              0xe
-#define TC_CFG_L2_STORE_POLICY0__POLICY_8__SHIFT                                                              0x10
-#define TC_CFG_L2_STORE_POLICY0__POLICY_9__SHIFT                                                              0x12
-#define TC_CFG_L2_STORE_POLICY0__POLICY_10__SHIFT                                                             0x14
-#define TC_CFG_L2_STORE_POLICY0__POLICY_11__SHIFT                                                             0x16
-#define TC_CFG_L2_STORE_POLICY0__POLICY_12__SHIFT                                                             0x18
-#define TC_CFG_L2_STORE_POLICY0__POLICY_13__SHIFT                                                             0x1a
-#define TC_CFG_L2_STORE_POLICY0__POLICY_14__SHIFT                                                             0x1c
-#define TC_CFG_L2_STORE_POLICY0__POLICY_15__SHIFT                                                             0x1e
-#define TC_CFG_L2_STORE_POLICY0__POLICY_0_MASK                                                                0x00000003L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_1_MASK                                                                0x0000000CL
-#define TC_CFG_L2_STORE_POLICY0__POLICY_2_MASK                                                                0x00000030L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_3_MASK                                                                0x000000C0L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_4_MASK                                                                0x00000300L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_5_MASK                                                                0x00000C00L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_6_MASK                                                                0x00003000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_7_MASK                                                                0x0000C000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_8_MASK                                                                0x00030000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_9_MASK                                                                0x000C0000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_10_MASK                                                               0x00300000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_11_MASK                                                               0x00C00000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_12_MASK                                                               0x03000000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_13_MASK                                                               0x0C000000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_14_MASK                                                               0x30000000L
-#define TC_CFG_L2_STORE_POLICY0__POLICY_15_MASK                                                               0xC0000000L
-//TC_CFG_L2_STORE_POLICY1
-#define TC_CFG_L2_STORE_POLICY1__POLICY_16__SHIFT                                                             0x0
-#define TC_CFG_L2_STORE_POLICY1__POLICY_17__SHIFT                                                             0x2
-#define TC_CFG_L2_STORE_POLICY1__POLICY_18__SHIFT                                                             0x4
-#define TC_CFG_L2_STORE_POLICY1__POLICY_19__SHIFT                                                             0x6
-#define TC_CFG_L2_STORE_POLICY1__POLICY_20__SHIFT                                                             0x8
-#define TC_CFG_L2_STORE_POLICY1__POLICY_21__SHIFT                                                             0xa
-#define TC_CFG_L2_STORE_POLICY1__POLICY_22__SHIFT                                                             0xc
-#define TC_CFG_L2_STORE_POLICY1__POLICY_23__SHIFT                                                             0xe
-#define TC_CFG_L2_STORE_POLICY1__POLICY_24__SHIFT                                                             0x10
-#define TC_CFG_L2_STORE_POLICY1__POLICY_25__SHIFT                                                             0x12
-#define TC_CFG_L2_STORE_POLICY1__POLICY_26__SHIFT                                                             0x14
-#define TC_CFG_L2_STORE_POLICY1__POLICY_27__SHIFT                                                             0x16
-#define TC_CFG_L2_STORE_POLICY1__POLICY_28__SHIFT                                                             0x18
-#define TC_CFG_L2_STORE_POLICY1__POLICY_29__SHIFT                                                             0x1a
-#define TC_CFG_L2_STORE_POLICY1__POLICY_30__SHIFT                                                             0x1c
-#define TC_CFG_L2_STORE_POLICY1__POLICY_31__SHIFT                                                             0x1e
-#define TC_CFG_L2_STORE_POLICY1__POLICY_16_MASK                                                               0x00000003L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_17_MASK                                                               0x0000000CL
-#define TC_CFG_L2_STORE_POLICY1__POLICY_18_MASK                                                               0x00000030L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_19_MASK                                                               0x000000C0L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_20_MASK                                                               0x00000300L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_21_MASK                                                               0x00000C00L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_22_MASK                                                               0x00003000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_23_MASK                                                               0x0000C000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_24_MASK                                                               0x00030000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_25_MASK                                                               0x000C0000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_26_MASK                                                               0x00300000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_27_MASK                                                               0x00C00000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_28_MASK                                                               0x03000000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_29_MASK                                                               0x0C000000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_30_MASK                                                               0x30000000L
-#define TC_CFG_L2_STORE_POLICY1__POLICY_31_MASK                                                               0xC0000000L
-//TC_CFG_L2_ATOMIC_POLICY
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0__SHIFT                                                              0x0
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1__SHIFT                                                              0x2
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2__SHIFT                                                              0x4
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3__SHIFT                                                              0x6
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4__SHIFT                                                              0x8
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5__SHIFT                                                              0xa
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6__SHIFT                                                              0xc
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7__SHIFT                                                              0xe
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8__SHIFT                                                              0x10
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9__SHIFT                                                              0x12
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10__SHIFT                                                             0x14
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11__SHIFT                                                             0x16
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12__SHIFT                                                             0x18
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13__SHIFT                                                             0x1a
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14__SHIFT                                                             0x1c
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15__SHIFT                                                             0x1e
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0_MASK                                                                0x00000003L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1_MASK                                                                0x0000000CL
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2_MASK                                                                0x00000030L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3_MASK                                                                0x000000C0L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4_MASK                                                                0x00000300L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5_MASK                                                                0x00000C00L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6_MASK                                                                0x00003000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7_MASK                                                                0x0000C000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8_MASK                                                                0x00030000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9_MASK                                                                0x000C0000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10_MASK                                                               0x00300000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11_MASK                                                               0x00C00000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12_MASK                                                               0x03000000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13_MASK                                                               0x0C000000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14_MASK                                                               0x30000000L
-#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15_MASK                                                               0xC0000000L
-//TC_CFG_L1_VOLATILE
-#define TC_CFG_L1_VOLATILE__VOL__SHIFT                                                                        0x0
-#define TC_CFG_L1_VOLATILE__VOL_MASK                                                                          0x0000000FL
-//TC_CFG_L2_VOLATILE
-#define TC_CFG_L2_VOLATILE__VOL__SHIFT                                                                        0x0
-#define TC_CFG_L2_VOLATILE__VOL_MASK                                                                          0x0000000FL
-//TCI_STATUS
-#define TCI_STATUS__TCI_BUSY__SHIFT                                                                           0x0
-#define TCI_STATUS__TCI_BUSY_MASK                                                                             0x00000001L
-//TCI_CNTL_1
-#define TCI_CNTL_1__WBINVL1_NUM_CYCLES__SHIFT                                                                 0x0
-#define TCI_CNTL_1__REQ_FIFO_DEPTH__SHIFT                                                                     0x10
-#define TCI_CNTL_1__WDATA_RAM_DEPTH__SHIFT                                                                    0x18
-#define TCI_CNTL_1__WBINVL1_NUM_CYCLES_MASK                                                                   0x0000FFFFL
-#define TCI_CNTL_1__REQ_FIFO_DEPTH_MASK                                                                       0x00FF0000L
-#define TCI_CNTL_1__WDATA_RAM_DEPTH_MASK                                                                      0xFF000000L
-//TCI_CNTL_2
-#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2__SHIFT                                                                0x0
-#define TCI_CNTL_2__TCA_MAX_CREDIT__SHIFT                                                                     0x1
-#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2_MASK                                                                  0x00000001L
-#define TCI_CNTL_2__TCA_MAX_CREDIT_MASK                                                                       0x000001FEL
-//TCC_CTRL
-#define TCC_CTRL__CACHE_SIZE__SHIFT                                                                           0x0
-#define TCC_CTRL__RATE__SHIFT                                                                                 0x2
-#define TCC_CTRL__WRITEBACK_MARGIN__SHIFT                                                                     0x4
-#define TCC_CTRL__METADATA_LATENCY_FIFO_SIZE__SHIFT                                                           0x8
-#define TCC_CTRL__SRC_FIFO_SIZE__SHIFT                                                                        0xc
-#define TCC_CTRL__LATENCY_FIFO_SIZE__SHIFT                                                                    0x10
-#define TCC_CTRL__LINEAR_SET_HASH__SHIFT                                                                      0x15
-#define TCC_CTRL__MDC_SIZE__SHIFT                                                                             0x18
-#define TCC_CTRL__MDC_SECTOR_SIZE__SHIFT                                                                      0x1a
-#define TCC_CTRL__MDC_SIDEBAND_FIFO_SIZE__SHIFT                                                               0x1c
-#define TCC_CTRL__CACHE_SIZE_MASK                                                                             0x00000003L
-#define TCC_CTRL__RATE_MASK                                                                                   0x0000000CL
-#define TCC_CTRL__WRITEBACK_MARGIN_MASK                                                                       0x000000F0L
-#define TCC_CTRL__METADATA_LATENCY_FIFO_SIZE_MASK                                                             0x00000F00L
-#define TCC_CTRL__SRC_FIFO_SIZE_MASK                                                                          0x0000F000L
-#define TCC_CTRL__LATENCY_FIFO_SIZE_MASK                                                                      0x000F0000L
-#define TCC_CTRL__LINEAR_SET_HASH_MASK                                                                        0x00200000L
-#define TCC_CTRL__MDC_SIZE_MASK                                                                               0x03000000L
-#define TCC_CTRL__MDC_SECTOR_SIZE_MASK                                                                        0x0C000000L
-#define TCC_CTRL__MDC_SIDEBAND_FIFO_SIZE_MASK                                                                 0xF0000000L
-//TCC_CTRL2
-#define TCC_CTRL2__PROBE_FIFO_SIZE__SHIFT                                                                     0x0
-#define TCC_CTRL2__PROBE_FIFO_SIZE_MASK                                                                       0x0000000FL
-//TCC_EDC_CNT
-#define TCC_EDC_CNT__CACHE_DATA_SEC_COUNT__SHIFT                                                              0x0
-#define TCC_EDC_CNT__CACHE_DATA_DED_COUNT__SHIFT                                                              0x2
-#define TCC_EDC_CNT__CACHE_DIRTY_SEC_COUNT__SHIFT                                                             0x4
-#define TCC_EDC_CNT__CACHE_DIRTY_DED_COUNT__SHIFT                                                             0x6
-#define TCC_EDC_CNT__HIGH_RATE_TAG_SEC_COUNT__SHIFT                                                           0x8
-#define TCC_EDC_CNT__HIGH_RATE_TAG_DED_COUNT__SHIFT                                                           0xa
-#define TCC_EDC_CNT__LOW_RATE_TAG_SEC_COUNT__SHIFT                                                            0xc
-#define TCC_EDC_CNT__LOW_RATE_TAG_DED_COUNT__SHIFT                                                            0xe
-#define TCC_EDC_CNT__SRC_FIFO_SEC_COUNT__SHIFT                                                                0x10
-#define TCC_EDC_CNT__SRC_FIFO_DED_COUNT__SHIFT                                                                0x12
-#define TCC_EDC_CNT__IN_USE_DEC_SED_COUNT__SHIFT                                                              0x14
-#define TCC_EDC_CNT__IN_USE_TRANSFER_SED_COUNT__SHIFT                                                         0x16
-#define TCC_EDC_CNT__LATENCY_FIFO_SED_COUNT__SHIFT                                                            0x18
-#define TCC_EDC_CNT__RETURN_DATA_SED_COUNT__SHIFT                                                             0x1a
-#define TCC_EDC_CNT__RETURN_CONTROL_SED_COUNT__SHIFT                                                          0x1c
-#define TCC_EDC_CNT__UC_ATOMIC_FIFO_SED_COUNT__SHIFT                                                          0x1e
-#define TCC_EDC_CNT__CACHE_DATA_SEC_COUNT_MASK                                                                0x00000003L
-#define TCC_EDC_CNT__CACHE_DATA_DED_COUNT_MASK                                                                0x0000000CL
-#define TCC_EDC_CNT__CACHE_DIRTY_SEC_COUNT_MASK                                                               0x00000030L
-#define TCC_EDC_CNT__CACHE_DIRTY_DED_COUNT_MASK                                                               0x000000C0L
-#define TCC_EDC_CNT__HIGH_RATE_TAG_SEC_COUNT_MASK                                                             0x00000300L
-#define TCC_EDC_CNT__HIGH_RATE_TAG_DED_COUNT_MASK                                                             0x00000C00L
-#define TCC_EDC_CNT__LOW_RATE_TAG_SEC_COUNT_MASK                                                              0x00003000L
-#define TCC_EDC_CNT__LOW_RATE_TAG_DED_COUNT_MASK                                                              0x0000C000L
-#define TCC_EDC_CNT__SRC_FIFO_SEC_COUNT_MASK                                                                  0x00030000L
-#define TCC_EDC_CNT__SRC_FIFO_DED_COUNT_MASK                                                                  0x000C0000L
-#define TCC_EDC_CNT__IN_USE_DEC_SED_COUNT_MASK                                                                0x00300000L
-#define TCC_EDC_CNT__IN_USE_TRANSFER_SED_COUNT_MASK                                                           0x00C00000L
-#define TCC_EDC_CNT__LATENCY_FIFO_SED_COUNT_MASK                                                              0x03000000L
-#define TCC_EDC_CNT__RETURN_DATA_SED_COUNT_MASK                                                               0x0C000000L
-#define TCC_EDC_CNT__RETURN_CONTROL_SED_COUNT_MASK                                                            0x30000000L
-#define TCC_EDC_CNT__UC_ATOMIC_FIFO_SED_COUNT_MASK                                                            0xC0000000L
-//TCC_EDC_CNT2
-#define TCC_EDC_CNT2__WRITE_RETURN_SED_COUNT__SHIFT                                                           0x0
-#define TCC_EDC_CNT2__WRITE_CACHE_READ_SED_COUNT__SHIFT                                                       0x2
-#define TCC_EDC_CNT2__SRC_FIFO_NEXT_RAM_SED_COUNT__SHIFT                                                      0x4
-#define TCC_EDC_CNT2__LATENCY_FIFO_NEXT_RAM_SED_COUNT__SHIFT                                                  0x6
-#define TCC_EDC_CNT2__CACHE_TAG_PROBE_FIFO_SED_COUNT__SHIFT                                                   0x8
-#define TCC_EDC_CNT2__WRITE_RETURN_SED_COUNT_MASK                                                             0x00000003L
-#define TCC_EDC_CNT2__WRITE_CACHE_READ_SED_COUNT_MASK                                                         0x0000000CL
-#define TCC_EDC_CNT2__SRC_FIFO_NEXT_RAM_SED_COUNT_MASK                                                        0x00000030L
-#define TCC_EDC_CNT2__LATENCY_FIFO_NEXT_RAM_SED_COUNT_MASK                                                    0x000000C0L
-#define TCC_EDC_CNT2__CACHE_TAG_PROBE_FIFO_SED_COUNT_MASK                                                     0x00000300L
-//TCC_REDUNDANCY
-#define TCC_REDUNDANCY__MC_SEL0__SHIFT                                                                        0x0
-#define TCC_REDUNDANCY__MC_SEL1__SHIFT                                                                        0x1
-#define TCC_REDUNDANCY__MC_SEL0_MASK                                                                          0x00000001L
-#define TCC_REDUNDANCY__MC_SEL1_MASK                                                                          0x00000002L
-//TCC_EXE_DISABLE
-#define TCC_EXE_DISABLE__EXE_DISABLE__SHIFT                                                                   0x1
-#define TCC_EXE_DISABLE__EXE_DISABLE_MASK                                                                     0x00000002L
-//TCC_DSM_CNTL
-#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_DATA_SEL__SHIFT                                                    0x0
-#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_SINGLE_WRITE__SHIFT                                                0x2
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_DATA_SEL__SHIFT                                           0x3
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_SINGLE_WRITE__SHIFT                                       0x5
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_DATA_SEL__SHIFT                                           0x6
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_SINGLE_WRITE__SHIFT                                       0x8
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_DATA_SEL__SHIFT                                           0x9
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_SINGLE_WRITE__SHIFT                                       0xb
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_DATA_SEL__SHIFT                                            0xc
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_SINGLE_WRITE__SHIFT                                        0xe
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_DATA_SEL__SHIFT                                            0xf
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_SINGLE_WRITE__SHIFT                                        0x11
-#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_DATA_SEL__SHIFT                                                 0x12
-#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x14
-#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_DATA_SEL__SHIFT                                                  0x15
-#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x17
-#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_DATA_SEL__SHIFT                                                    0x18
-#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_SINGLE_WRITE__SHIFT                                                0x1a
-#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_DATA_SEL__SHIFT                                               0x1b
-#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_SINGLE_WRITE__SHIFT                                           0x1d
-#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_DATA_SEL_MASK                                                      0x00000003L
-#define TCC_DSM_CNTL__CACHE_DATA_IRRITATOR_SINGLE_WRITE_MASK                                                  0x00000004L
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_DATA_SEL_MASK                                             0x00000018L
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_0_1_IRRITATOR_SINGLE_WRITE_MASK                                         0x00000020L
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_DATA_SEL_MASK                                             0x000000C0L
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_0_IRRITATOR_SINGLE_WRITE_MASK                                         0x00000100L
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_DATA_SEL_MASK                                             0x00000600L
-#define TCC_DSM_CNTL__CACHE_DATA_BANK_1_1_IRRITATOR_SINGLE_WRITE_MASK                                         0x00000800L
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_DATA_SEL_MASK                                              0x00003000L
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_0_IRRITATOR_SINGLE_WRITE_MASK                                          0x00004000L
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_DATA_SEL_MASK                                              0x00018000L
-#define TCC_DSM_CNTL__CACHE_DIRTY_BANK_1_IRRITATOR_SINGLE_WRITE_MASK                                          0x00020000L
-#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_DATA_SEL_MASK                                                   0x000C0000L
-#define TCC_DSM_CNTL__HIGH_RATE_TAG_IRRITATOR_SINGLE_WRITE_MASK                                               0x00100000L
-#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_DATA_SEL_MASK                                                    0x00600000L
-#define TCC_DSM_CNTL__LOW_RATE_TAG_IRRITATOR_SINGLE_WRITE_MASK                                                0x00800000L
-#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_DATA_SEL_MASK                                                      0x03000000L
-#define TCC_DSM_CNTL__IN_USE_DEC_IRRITATOR_SINGLE_WRITE_MASK                                                  0x04000000L
-#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_DATA_SEL_MASK                                                 0x18000000L
-#define TCC_DSM_CNTL__IN_USE_TRANSFER_IRRITATOR_SINGLE_WRITE_MASK                                             0x20000000L
-//TCC_DSM_CNTLA
-#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_DATA_SEL__SHIFT                                                     0x0
-#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                                 0x2
-#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_DATA_SEL__SHIFT                                               0x3
-#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                           0x5
-#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_DATA_SEL__SHIFT                                                 0x6
-#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x8
-#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_DATA_SEL__SHIFT                                             0x9
-#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_SINGLE_WRITE__SHIFT                                         0xb
-#define TCC_DSM_CNTLA__SRC_FIFO_NEXT_RAM_IRRITATOR_DATA_SEL__SHIFT                                            0xc
-#define TCC_DSM_CNTLA__SRC_FIFO_NEXT_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                        0xe
-#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_DATA_SEL__SHIFT                                        0xf
-#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_SINGLE_WRITE__SHIFT                                    0x11
-#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_DATA_SEL__SHIFT                                         0x12
-#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                     0x14
-#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_DATA_SEL__SHIFT                                                 0x15
-#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x17
-#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_DATA_SEL__SHIFT                                                  0x18
-#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x1a
-#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_DATA_SEL__SHIFT                                               0x1b
-#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_SINGLE_WRITE__SHIFT                                           0x1d
-#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_DATA_SEL_MASK                                                       0x00000003L
-#define TCC_DSM_CNTLA__SRC_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                                   0x00000004L
-#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_DATA_SEL_MASK                                                 0x00000018L
-#define TCC_DSM_CNTLA__UC_ATOMIC_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                             0x00000020L
-#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_DATA_SEL_MASK                                                   0x000000C0L
-#define TCC_DSM_CNTLA__WRITE_RETURN_IRRITATOR_SINGLE_WRITE_MASK                                               0x00000100L
-#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_DATA_SEL_MASK                                               0x00000600L
-#define TCC_DSM_CNTLA__WRITE_CACHE_READ_IRRITATOR_SINGLE_WRITE_MASK                                           0x00000800L
-#define TCC_DSM_CNTLA__SRC_FIFO_NEXT_RAM_IRRITATOR_DATA_SEL_MASK                                              0x00003000L
-#define TCC_DSM_CNTLA__SRC_FIFO_NEXT_RAM_IRRITATOR_SINGLE_WRITE_MASK                                          0x00004000L
-#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_DATA_SEL_MASK                                          0x00018000L
-#define TCC_DSM_CNTLA__LATENCY_FIFO_NEXT_RAM_IRRITATOR_SINGLE_WRITE_MASK                                      0x00020000L
-#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_DATA_SEL_MASK                                           0x000C0000L
-#define TCC_DSM_CNTLA__CACHE_TAG_PROBE_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                       0x00100000L
-#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_DATA_SEL_MASK                                                   0x00600000L
-#define TCC_DSM_CNTLA__LATENCY_FIFO_IRRITATOR_SINGLE_WRITE_MASK                                               0x00800000L
-#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_DATA_SEL_MASK                                                    0x03000000L
-#define TCC_DSM_CNTLA__RETURN_DATA_IRRITATOR_SINGLE_WRITE_MASK                                                0x04000000L
-#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_DATA_SEL_MASK                                                 0x18000000L
-#define TCC_DSM_CNTLA__RETURN_CONTROL_IRRITATOR_SINGLE_WRITE_MASK                                             0x20000000L
-//TCC_DSM_CNTL2
-#define TCC_DSM_CNTL2__CACHE_DATA_ENABLE_ERROR_INJECT__SHIFT                                                  0x0
-#define TCC_DSM_CNTL2__CACHE_DATA_SELECT_INJECT_DELAY__SHIFT                                                  0x2
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_ENABLE_ERROR_INJECT__SHIFT                                         0x3
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_SELECT_INJECT_DELAY__SHIFT                                         0x5
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_ENABLE_ERROR_INJECT__SHIFT                                         0x6
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_SELECT_INJECT_DELAY__SHIFT                                         0x8
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_ENABLE_ERROR_INJECT__SHIFT                                         0x9
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_SELECT_INJECT_DELAY__SHIFT                                         0xb
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_ENABLE_ERROR_INJECT__SHIFT                                          0xc
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_SELECT_INJECT_DELAY__SHIFT                                          0xe
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_ENABLE_ERROR_INJECT__SHIFT                                          0xf
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_SELECT_INJECT_DELAY__SHIFT                                          0x11
-#define TCC_DSM_CNTL2__HIGH_RATE_TAG_ENABLE_ERROR_INJECT__SHIFT                                               0x12
-#define TCC_DSM_CNTL2__HIGH_RATE_TAG_SELECT_INJECT_DELAY__SHIFT                                               0x14
-#define TCC_DSM_CNTL2__LOW_RATE_TAG_ENABLE_ERROR_INJECT__SHIFT                                                0x15
-#define TCC_DSM_CNTL2__LOW_RATE_TAG_SELECT_INJECT_DELAY__SHIFT                                                0x17
-#define TCC_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
-#define TCC_DSM_CNTL2__CACHE_DATA_ENABLE_ERROR_INJECT_MASK                                                    0x00000003L
-#define TCC_DSM_CNTL2__CACHE_DATA_SELECT_INJECT_DELAY_MASK                                                    0x00000004L
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_ENABLE_ERROR_INJECT_MASK                                           0x00000018L
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_0_1_SELECT_INJECT_DELAY_MASK                                           0x00000020L
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_ENABLE_ERROR_INJECT_MASK                                           0x000000C0L
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_0_SELECT_INJECT_DELAY_MASK                                           0x00000100L
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_ENABLE_ERROR_INJECT_MASK                                           0x00000600L
-#define TCC_DSM_CNTL2__CACHE_DATA_BANK_1_1_SELECT_INJECT_DELAY_MASK                                           0x00000800L
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_ENABLE_ERROR_INJECT_MASK                                            0x00003000L
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_0_SELECT_INJECT_DELAY_MASK                                            0x00004000L
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_ENABLE_ERROR_INJECT_MASK                                            0x00018000L
-#define TCC_DSM_CNTL2__CACHE_DIRTY_BANK_1_SELECT_INJECT_DELAY_MASK                                            0x00020000L
-#define TCC_DSM_CNTL2__HIGH_RATE_TAG_ENABLE_ERROR_INJECT_MASK                                                 0x000C0000L
-#define TCC_DSM_CNTL2__HIGH_RATE_TAG_SELECT_INJECT_DELAY_MASK                                                 0x00100000L
-#define TCC_DSM_CNTL2__LOW_RATE_TAG_ENABLE_ERROR_INJECT_MASK                                                  0x00600000L
-#define TCC_DSM_CNTL2__LOW_RATE_TAG_SELECT_INJECT_DELAY_MASK                                                  0x00800000L
-#define TCC_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
-//TCC_DSM_CNTL2A
-#define TCC_DSM_CNTL2A__IN_USE_DEC_ENABLE_ERROR_INJECT__SHIFT                                                 0x0
-#define TCC_DSM_CNTL2A__IN_USE_DEC_SELECT_INJECT_DELAY__SHIFT                                                 0x2
-#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_ENABLE_ERROR_INJECT__SHIFT                                            0x3
-#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_SELECT_INJECT_DELAY__SHIFT                                            0x5
-#define TCC_DSM_CNTL2A__RETURN_DATA_ENABLE_ERROR_INJECT__SHIFT                                                0x6
-#define TCC_DSM_CNTL2A__RETURN_DATA_SELECT_INJECT_DELAY__SHIFT                                                0x8
-#define TCC_DSM_CNTL2A__RETURN_CONTROL_ENABLE_ERROR_INJECT__SHIFT                                             0x9
-#define TCC_DSM_CNTL2A__RETURN_CONTROL_SELECT_INJECT_DELAY__SHIFT                                             0xb
-#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_ENABLE_ERROR_INJECT__SHIFT                                             0xc
-#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_SELECT_INJECT_DELAY__SHIFT                                             0xe
-#define TCC_DSM_CNTL2A__WRITE_RETURN_ENABLE_ERROR_INJECT__SHIFT                                               0xf
-#define TCC_DSM_CNTL2A__WRITE_RETURN_SELECT_INJECT_DELAY__SHIFT                                               0x11
-#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_ENABLE_ERROR_INJECT__SHIFT                                           0x12
-#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_SELECT_INJECT_DELAY__SHIFT                                           0x14
-#define TCC_DSM_CNTL2A__SRC_FIFO_ENABLE_ERROR_INJECT__SHIFT                                                   0x15
-#define TCC_DSM_CNTL2A__SRC_FIFO_SELECT_INJECT_DELAY__SHIFT                                                   0x17
-#define TCC_DSM_CNTL2A__SRC_FIFO_NEXT_RAM_ENABLE_ERROR_INJECT__SHIFT                                          0x18
-#define TCC_DSM_CNTL2A__SRC_FIFO_NEXT_RAM_SELECT_INJECT_DELAY__SHIFT                                          0x1a
-#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_ENABLE_ERROR_INJECT__SHIFT                                       0x1b
-#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_SELECT_INJECT_DELAY__SHIFT                                       0x1d
-#define TCC_DSM_CNTL2A__IN_USE_DEC_ENABLE_ERROR_INJECT_MASK                                                   0x00000003L
-#define TCC_DSM_CNTL2A__IN_USE_DEC_SELECT_INJECT_DELAY_MASK                                                   0x00000004L
-#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_ENABLE_ERROR_INJECT_MASK                                              0x00000018L
-#define TCC_DSM_CNTL2A__IN_USE_TRANSFER_SELECT_INJECT_DELAY_MASK                                              0x00000020L
-#define TCC_DSM_CNTL2A__RETURN_DATA_ENABLE_ERROR_INJECT_MASK                                                  0x000000C0L
-#define TCC_DSM_CNTL2A__RETURN_DATA_SELECT_INJECT_DELAY_MASK                                                  0x00000100L
-#define TCC_DSM_CNTL2A__RETURN_CONTROL_ENABLE_ERROR_INJECT_MASK                                               0x00000600L
-#define TCC_DSM_CNTL2A__RETURN_CONTROL_SELECT_INJECT_DELAY_MASK                                               0x00000800L
-#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_ENABLE_ERROR_INJECT_MASK                                               0x00003000L
-#define TCC_DSM_CNTL2A__UC_ATOMIC_FIFO_SELECT_INJECT_DELAY_MASK                                               0x00004000L
-#define TCC_DSM_CNTL2A__WRITE_RETURN_ENABLE_ERROR_INJECT_MASK                                                 0x00018000L
-#define TCC_DSM_CNTL2A__WRITE_RETURN_SELECT_INJECT_DELAY_MASK                                                 0x00020000L
-#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_ENABLE_ERROR_INJECT_MASK                                             0x000C0000L
-#define TCC_DSM_CNTL2A__WRITE_CACHE_READ_SELECT_INJECT_DELAY_MASK                                             0x00100000L
-#define TCC_DSM_CNTL2A__SRC_FIFO_ENABLE_ERROR_INJECT_MASK                                                     0x00600000L
-#define TCC_DSM_CNTL2A__SRC_FIFO_SELECT_INJECT_DELAY_MASK                                                     0x00800000L
-#define TCC_DSM_CNTL2A__SRC_FIFO_NEXT_RAM_ENABLE_ERROR_INJECT_MASK                                            0x03000000L
-#define TCC_DSM_CNTL2A__SRC_FIFO_NEXT_RAM_SELECT_INJECT_DELAY_MASK                                            0x04000000L
-#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_ENABLE_ERROR_INJECT_MASK                                         0x18000000L
-#define TCC_DSM_CNTL2A__CACHE_TAG_PROBE_FIFO_SELECT_INJECT_DELAY_MASK                                         0x20000000L
-//TCC_DSM_CNTL2B
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_ENABLE_ERROR_INJECT__SHIFT                                               0x0
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_SELECT_INJECT_DELAY__SHIFT                                               0x2
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_ENABLE_ERROR_INJECT__SHIFT                                      0x3
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_SELECT_INJECT_DELAY__SHIFT                                      0x5
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_ENABLE_ERROR_INJECT_MASK                                        0x00000018L
-#define TCC_DSM_CNTL2B__LATENCY_FIFO_NEXT_RAM_SELECT_INJECT_DELAY_MASK                                        0x00000020L
-//TCC_WBINVL2
-#define TCC_WBINVL2__DONE__SHIFT                                                                              0x4
-#define TCC_WBINVL2__DONE_MASK                                                                                0x00000010L
-//TCC_SOFT_RESET
-#define TCC_SOFT_RESET__HALT_FOR_RESET__SHIFT                                                                 0x0
-#define TCC_SOFT_RESET__HALT_FOR_RESET_MASK                                                                   0x00000001L
-//TCA_CTRL
-#define TCA_CTRL__HOLE_TIMEOUT__SHIFT                                                                         0x0
-#define TCA_CTRL__RB_STILL_4_PHASE__SHIFT                                                                     0x4
-#define TCA_CTRL__RB_AS_TCI__SHIFT                                                                            0x5
-#define TCA_CTRL__DISABLE_UTCL2_PRIORITY__SHIFT                                                               0x6
-#define TCA_CTRL__DISABLE_RB_ONLY_TCA_ARBITER__SHIFT                                                          0x7
-#define TCA_CTRL__HOLE_TIMEOUT_MASK                                                                           0x0000000FL
-#define TCA_CTRL__RB_STILL_4_PHASE_MASK                                                                       0x00000010L
-#define TCA_CTRL__RB_AS_TCI_MASK                                                                              0x00000020L
-#define TCA_CTRL__DISABLE_UTCL2_PRIORITY_MASK                                                                 0x00000040L
-#define TCA_CTRL__DISABLE_RB_ONLY_TCA_ARBITER_MASK                                                            0x00000080L
-//TCA_BURST_MASK
-#define TCA_BURST_MASK__ADDR_MASK__SHIFT                                                                      0x0
-#define TCA_BURST_MASK__ADDR_MASK_MASK                                                                        0xFFFFFFFFL
-//TCA_BURST_CTRL
-#define TCA_BURST_CTRL__MAX_BURST__SHIFT                                                                      0x0
-#define TCA_BURST_CTRL__RB_DISABLE__SHIFT                                                                     0x3
-#define TCA_BURST_CTRL__TCP_DISABLE__SHIFT                                                                    0x4
-#define TCA_BURST_CTRL__SQC_DISABLE__SHIFT                                                                    0x5
-#define TCA_BURST_CTRL__CPF_DISABLE__SHIFT                                                                    0x6
-#define TCA_BURST_CTRL__CPG_DISABLE__SHIFT                                                                    0x7
-#define TCA_BURST_CTRL__IA_DISABLE__SHIFT                                                                     0x8
-#define TCA_BURST_CTRL__WD_DISABLE__SHIFT                                                                     0x9
-#define TCA_BURST_CTRL__SQG_DISABLE__SHIFT                                                                    0xa
-#define TCA_BURST_CTRL__UTCL2_DISABLE__SHIFT                                                                  0xb
-#define TCA_BURST_CTRL__TPI_DISABLE__SHIFT                                                                    0xc
-#define TCA_BURST_CTRL__RLC_DISABLE__SHIFT                                                                    0xd
-#define TCA_BURST_CTRL__PA_DISABLE__SHIFT                                                                     0xe
-#define TCA_BURST_CTRL__MAX_BURST_MASK                                                                        0x00000007L
-#define TCA_BURST_CTRL__RB_DISABLE_MASK                                                                       0x00000008L
-#define TCA_BURST_CTRL__TCP_DISABLE_MASK                                                                      0x00000010L
-#define TCA_BURST_CTRL__SQC_DISABLE_MASK                                                                      0x00000020L
-#define TCA_BURST_CTRL__CPF_DISABLE_MASK                                                                      0x00000040L
-#define TCA_BURST_CTRL__CPG_DISABLE_MASK                                                                      0x00000080L
-#define TCA_BURST_CTRL__IA_DISABLE_MASK                                                                       0x00000100L
-#define TCA_BURST_CTRL__WD_DISABLE_MASK                                                                       0x00000200L
-#define TCA_BURST_CTRL__SQG_DISABLE_MASK                                                                      0x00000400L
-#define TCA_BURST_CTRL__UTCL2_DISABLE_MASK                                                                    0x00000800L
-#define TCA_BURST_CTRL__TPI_DISABLE_MASK                                                                      0x00001000L
-#define TCA_BURST_CTRL__RLC_DISABLE_MASK                                                                      0x00002000L
-#define TCA_BURST_CTRL__PA_DISABLE_MASK                                                                       0x00004000L
-//TCA_DSM_CNTL
-#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_DATA_SEL__SHIFT                                                 0x0
-#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_SINGLE_WRITE__SHIFT                                             0x2
-#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_DATA_SEL__SHIFT                                                  0x3
-#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_SINGLE_WRITE__SHIFT                                              0x5
-#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_DATA_SEL_MASK                                                   0x00000003L
-#define TCA_DSM_CNTL__HOLE_FIFO_SED_IRRITATOR_SINGLE_WRITE_MASK                                               0x00000004L
-#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_DATA_SEL_MASK                                                    0x00000018L
-#define TCA_DSM_CNTL__REQ_FIFO_SED_IRRITATOR_SINGLE_WRITE_MASK                                                0x00000020L
-//TCA_DSM_CNTL2
-#define TCA_DSM_CNTL2__HOLE_FIFO_SED_ENABLE_ERROR_INJECT__SHIFT                                               0x0
-#define TCA_DSM_CNTL2__HOLE_FIFO_SED_SELECT_INJECT_DELAY__SHIFT                                               0x2
-#define TCA_DSM_CNTL2__REQ_FIFO_SED_ENABLE_ERROR_INJECT__SHIFT                                                0x3
-#define TCA_DSM_CNTL2__REQ_FIFO_SED_SELECT_INJECT_DELAY__SHIFT                                                0x5
-#define TCA_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                    0x1a
-#define TCA_DSM_CNTL2__HOLE_FIFO_SED_ENABLE_ERROR_INJECT_MASK                                                 0x00000003L
-#define TCA_DSM_CNTL2__HOLE_FIFO_SED_SELECT_INJECT_DELAY_MASK                                                 0x00000004L
-#define TCA_DSM_CNTL2__REQ_FIFO_SED_ENABLE_ERROR_INJECT_MASK                                                  0x00000018L
-#define TCA_DSM_CNTL2__REQ_FIFO_SED_SELECT_INJECT_DELAY_MASK                                                  0x00000020L
-#define TCA_DSM_CNTL2__INJECT_DELAY_MASK                                                                      0xFC000000L
-//TCA_EDC_CNT
-#define TCA_EDC_CNT__HOLE_FIFO_SED_COUNT__SHIFT                                                               0x0
-#define TCA_EDC_CNT__REQ_FIFO_SED_COUNT__SHIFT                                                                0x2
-#define TCA_EDC_CNT__HOLE_FIFO_SED_COUNT_MASK                                                                 0x00000003L
-#define TCA_EDC_CNT__REQ_FIFO_SED_COUNT_MASK                                                                  0x0000000CL
-
-
-// addressBlock: gc_shdec
-//SPI_SHADER_PGM_RSRC3_PS
-#define SPI_SHADER_PGM_RSRC3_PS__CU_EN__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT__SHIFT                                                            0x10
-#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x16
-#define SPI_SHADER_PGM_RSRC3_PS__SIMD_DISABLE__SHIFT                                                          0x1a
-#define SPI_SHADER_PGM_RSRC3_PS__CU_EN_MASK                                                                   0x0000FFFFL
-#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT_MASK                                                              0x003F0000L
-#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD_MASK                                                      0x03C00000L
-#define SPI_SHADER_PGM_RSRC3_PS__SIMD_DISABLE_MASK                                                            0x3C000000L
-//SPI_SHADER_PGM_LO_PS
-#define SPI_SHADER_PGM_LO_PS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_LO_PS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
-//SPI_SHADER_PGM_HI_PS
-#define SPI_SHADER_PGM_HI_PS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_HI_PS__MEM_BASE_MASK                                                                   0xFFL
-//SPI_SHADER_PGM_RSRC1_PS
-#define SPI_SHADER_PGM_RSRC1_PS__VGPRS__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_RSRC1_PS__SGPRS__SHIFT                                                                 0x6
-#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY__SHIFT                                                              0xa
-#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE__SHIFT                                                            0xc
-#define SPI_SHADER_PGM_RSRC1_PS__PRIV__SHIFT                                                                  0x14
-#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP__SHIFT                                                            0x15
-#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE__SHIFT                                                             0x17
-#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE__SHIFT                                                      0x18
-#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL__SHIFT                                                             0x1d
-#define SPI_SHADER_PGM_RSRC1_PS__VGPRS_MASK                                                                   0x0000003FL
-#define SPI_SHADER_PGM_RSRC1_PS__SGPRS_MASK                                                                   0x000003C0L
-#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY_MASK                                                                0x00000C00L
-#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE_MASK                                                              0x000FF000L
-#define SPI_SHADER_PGM_RSRC1_PS__PRIV_MASK                                                                    0x00100000L
-#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP_MASK                                                              0x00200000L
-#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE_MASK                                                               0x00800000L
-#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE_MASK                                                        0x01000000L
-#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL_MASK                                                               0x20000000L
-//SPI_SHADER_PGM_RSRC2_PS
-#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN__SHIFT                                                            0x0
-#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR__SHIFT                                                             0x1
-#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT__SHIFT                                                          0x6
-#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN__SHIFT                                                           0x7
-#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE__SHIFT                                                        0x8
-#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN__SHIFT                                                               0x10
-#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID__SHIFT                                                 0x19
-#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION__SHIFT                                              0x1a
-#define SPI_SHADER_PGM_RSRC2_PS__SKIP_USGPR0__SHIFT                                                           0x1b
-#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB__SHIFT                                                         0x1c
-#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN_MASK                                                              0x00000001L
-#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MASK                                                               0x0000003EL
-#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT_MASK                                                            0x00000040L
-#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN_MASK                                                             0x00000080L
-#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE_MASK                                                          0x0000FF00L
-#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN_MASK                                                                 0x01FF0000L
-#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID_MASK                                                   0x02000000L
-#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION_MASK                                                0x04000000L
-#define SPI_SHADER_PGM_RSRC2_PS__SKIP_USGPR0_MASK                                                             0x08000000L
-#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB_MASK                                                           0x10000000L
-//SPI_SHADER_USER_DATA_PS_0
-#define SPI_SHADER_USER_DATA_PS_0__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_0__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_1
-#define SPI_SHADER_USER_DATA_PS_1__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_1__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_2
-#define SPI_SHADER_USER_DATA_PS_2__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_2__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_3
-#define SPI_SHADER_USER_DATA_PS_3__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_3__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_4
-#define SPI_SHADER_USER_DATA_PS_4__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_4__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_5
-#define SPI_SHADER_USER_DATA_PS_5__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_5__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_6
-#define SPI_SHADER_USER_DATA_PS_6__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_6__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_7
-#define SPI_SHADER_USER_DATA_PS_7__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_7__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_8
-#define SPI_SHADER_USER_DATA_PS_8__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_8__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_9
-#define SPI_SHADER_USER_DATA_PS_9__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_PS_9__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_10
-#define SPI_SHADER_USER_DATA_PS_10__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_10__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_11
-#define SPI_SHADER_USER_DATA_PS_11__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_11__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_12
-#define SPI_SHADER_USER_DATA_PS_12__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_12__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_13
-#define SPI_SHADER_USER_DATA_PS_13__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_13__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_14
-#define SPI_SHADER_USER_DATA_PS_14__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_14__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_15
-#define SPI_SHADER_USER_DATA_PS_15__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_15__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_16
-#define SPI_SHADER_USER_DATA_PS_16__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_16__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_17
-#define SPI_SHADER_USER_DATA_PS_17__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_17__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_18
-#define SPI_SHADER_USER_DATA_PS_18__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_18__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_19
-#define SPI_SHADER_USER_DATA_PS_19__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_19__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_20
-#define SPI_SHADER_USER_DATA_PS_20__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_20__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_21
-#define SPI_SHADER_USER_DATA_PS_21__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_21__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_22
-#define SPI_SHADER_USER_DATA_PS_22__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_22__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_23
-#define SPI_SHADER_USER_DATA_PS_23__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_23__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_24
-#define SPI_SHADER_USER_DATA_PS_24__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_24__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_25
-#define SPI_SHADER_USER_DATA_PS_25__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_25__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_26
-#define SPI_SHADER_USER_DATA_PS_26__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_26__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_27
-#define SPI_SHADER_USER_DATA_PS_27__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_27__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_28
-#define SPI_SHADER_USER_DATA_PS_28__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_28__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_29
-#define SPI_SHADER_USER_DATA_PS_29__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_29__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_30
-#define SPI_SHADER_USER_DATA_PS_30__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_30__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_PS_31
-#define SPI_SHADER_USER_DATA_PS_31__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_PS_31__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_PGM_RSRC3_VS
-#define SPI_SHADER_PGM_RSRC3_VS__CU_EN__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT__SHIFT                                                            0x10
-#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x16
-#define SPI_SHADER_PGM_RSRC3_VS__SIMD_DISABLE__SHIFT                                                          0x1a
-#define SPI_SHADER_PGM_RSRC3_VS__CU_EN_MASK                                                                   0x0000FFFFL
-#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT_MASK                                                              0x003F0000L
-#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD_MASK                                                      0x03C00000L
-#define SPI_SHADER_PGM_RSRC3_VS__SIMD_DISABLE_MASK                                                            0x3C000000L
-//SPI_SHADER_LATE_ALLOC_VS
-#define SPI_SHADER_LATE_ALLOC_VS__LIMIT__SHIFT                                                                0x0
-#define SPI_SHADER_LATE_ALLOC_VS__LIMIT_MASK                                                                  0x0000003FL
-//SPI_SHADER_PGM_LO_VS
-#define SPI_SHADER_PGM_LO_VS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_LO_VS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
-//SPI_SHADER_PGM_HI_VS
-#define SPI_SHADER_PGM_HI_VS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_HI_VS__MEM_BASE_MASK                                                                   0xFFL
-//SPI_SHADER_PGM_RSRC1_VS
-#define SPI_SHADER_PGM_RSRC1_VS__VGPRS__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_RSRC1_VS__SGPRS__SHIFT                                                                 0x6
-#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY__SHIFT                                                              0xa
-#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE__SHIFT                                                            0xc
-#define SPI_SHADER_PGM_RSRC1_VS__PRIV__SHIFT                                                                  0x14
-#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP__SHIFT                                                            0x15
-#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE__SHIFT                                                             0x17
-#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT__SHIFT                                                         0x18
-#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE__SHIFT                                                       0x1a
-#define SPI_SHADER_PGM_RSRC1_VS__FP16_OVFL__SHIFT                                                             0x1f
-#define SPI_SHADER_PGM_RSRC1_VS__VGPRS_MASK                                                                   0x0000003FL
-#define SPI_SHADER_PGM_RSRC1_VS__SGPRS_MASK                                                                   0x000003C0L
-#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY_MASK                                                                0x00000C00L
-#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE_MASK                                                              0x000FF000L
-#define SPI_SHADER_PGM_RSRC1_VS__PRIV_MASK                                                                    0x00100000L
-#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP_MASK                                                              0x00200000L
-#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE_MASK                                                               0x00800000L
-#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT_MASK                                                           0x03000000L
-#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE_MASK                                                         0x04000000L
-#define SPI_SHADER_PGM_RSRC1_VS__FP16_OVFL_MASK                                                               0x80000000L
-//SPI_SHADER_PGM_RSRC2_VS
-#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN__SHIFT                                                            0x0
-#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR__SHIFT                                                             0x1
-#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT__SHIFT                                                          0x6
-#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN__SHIFT                                                             0x7
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN__SHIFT                                                           0x8
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN__SHIFT                                                           0x9
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN__SHIFT                                                           0xa
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN__SHIFT                                                           0xb
-#define SPI_SHADER_PGM_RSRC2_VS__SO_EN__SHIFT                                                                 0xc
-#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN__SHIFT                                                               0xd
-#define SPI_SHADER_PGM_RSRC2_VS__PC_BASE_EN__SHIFT                                                            0x16
-#define SPI_SHADER_PGM_RSRC2_VS__DISPATCH_DRAW_EN__SHIFT                                                      0x18
-#define SPI_SHADER_PGM_RSRC2_VS__SKIP_USGPR0__SHIFT                                                           0x1b
-#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MSB__SHIFT                                                         0x1c
-#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN_MASK                                                              0x00000001L
-#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MASK                                                               0x0000003EL
-#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT_MASK                                                            0x00000040L
-#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN_MASK                                                               0x00000080L
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN_MASK                                                             0x00000100L
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN_MASK                                                             0x00000200L
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN_MASK                                                             0x00000400L
-#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN_MASK                                                             0x00000800L
-#define SPI_SHADER_PGM_RSRC2_VS__SO_EN_MASK                                                                   0x00001000L
-#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN_MASK                                                                 0x003FE000L
-#define SPI_SHADER_PGM_RSRC2_VS__PC_BASE_EN_MASK                                                              0x00400000L
-#define SPI_SHADER_PGM_RSRC2_VS__DISPATCH_DRAW_EN_MASK                                                        0x01000000L
-#define SPI_SHADER_PGM_RSRC2_VS__SKIP_USGPR0_MASK                                                             0x08000000L
-#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MSB_MASK                                                           0x10000000L
-//SPI_SHADER_USER_DATA_VS_0
-#define SPI_SHADER_USER_DATA_VS_0__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_0__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_1
-#define SPI_SHADER_USER_DATA_VS_1__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_1__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_2
-#define SPI_SHADER_USER_DATA_VS_2__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_2__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_3
-#define SPI_SHADER_USER_DATA_VS_3__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_3__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_4
-#define SPI_SHADER_USER_DATA_VS_4__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_4__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_5
-#define SPI_SHADER_USER_DATA_VS_5__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_5__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_6
-#define SPI_SHADER_USER_DATA_VS_6__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_6__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_7
-#define SPI_SHADER_USER_DATA_VS_7__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_7__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_8
-#define SPI_SHADER_USER_DATA_VS_8__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_8__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_9
-#define SPI_SHADER_USER_DATA_VS_9__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_VS_9__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_10
-#define SPI_SHADER_USER_DATA_VS_10__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_10__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_11
-#define SPI_SHADER_USER_DATA_VS_11__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_11__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_12
-#define SPI_SHADER_USER_DATA_VS_12__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_12__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_13
-#define SPI_SHADER_USER_DATA_VS_13__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_13__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_14
-#define SPI_SHADER_USER_DATA_VS_14__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_14__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_15
-#define SPI_SHADER_USER_DATA_VS_15__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_15__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_16
-#define SPI_SHADER_USER_DATA_VS_16__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_16__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_17
-#define SPI_SHADER_USER_DATA_VS_17__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_17__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_18
-#define SPI_SHADER_USER_DATA_VS_18__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_18__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_19
-#define SPI_SHADER_USER_DATA_VS_19__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_19__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_20
-#define SPI_SHADER_USER_DATA_VS_20__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_20__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_21
-#define SPI_SHADER_USER_DATA_VS_21__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_21__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_22
-#define SPI_SHADER_USER_DATA_VS_22__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_22__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_23
-#define SPI_SHADER_USER_DATA_VS_23__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_23__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_24
-#define SPI_SHADER_USER_DATA_VS_24__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_24__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_25
-#define SPI_SHADER_USER_DATA_VS_25__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_25__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_26
-#define SPI_SHADER_USER_DATA_VS_26__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_26__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_27
-#define SPI_SHADER_USER_DATA_VS_27__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_27__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_28
-#define SPI_SHADER_USER_DATA_VS_28__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_28__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_29
-#define SPI_SHADER_USER_DATA_VS_29__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_29__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_30
-#define SPI_SHADER_USER_DATA_VS_30__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_30__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_VS_31
-#define SPI_SHADER_USER_DATA_VS_31__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_VS_31__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_PGM_RSRC2_GS_VS
-#define SPI_SHADER_PGM_RSRC2_GS_VS__SCRATCH_EN__SHIFT                                                         0x0
-#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR__SHIFT                                                          0x1
-#define SPI_SHADER_PGM_RSRC2_GS_VS__TRAP_PRESENT__SHIFT                                                       0x6
-#define SPI_SHADER_PGM_RSRC2_GS_VS__EXCP_EN__SHIFT                                                            0x7
-#define SPI_SHADER_PGM_RSRC2_GS_VS__VGPR_COMP_CNT__SHIFT                                                      0x10
-#define SPI_SHADER_PGM_RSRC2_GS_VS__OC_LDS_EN__SHIFT                                                          0x12
-#define SPI_SHADER_PGM_RSRC2_GS_VS__LDS_SIZE__SHIFT                                                           0x13
-#define SPI_SHADER_PGM_RSRC2_GS_VS__SKIP_USGPR0__SHIFT                                                        0x1b
-#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR_MSB__SHIFT                                                      0x1c
-#define SPI_SHADER_PGM_RSRC2_GS_VS__SCRATCH_EN_MASK                                                           0x00000001L
-#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR_MASK                                                            0x0000003EL
-#define SPI_SHADER_PGM_RSRC2_GS_VS__TRAP_PRESENT_MASK                                                         0x00000040L
-#define SPI_SHADER_PGM_RSRC2_GS_VS__EXCP_EN_MASK                                                              0x0000FF80L
-#define SPI_SHADER_PGM_RSRC2_GS_VS__VGPR_COMP_CNT_MASK                                                        0x00030000L
-#define SPI_SHADER_PGM_RSRC2_GS_VS__OC_LDS_EN_MASK                                                            0x00040000L
-#define SPI_SHADER_PGM_RSRC2_GS_VS__LDS_SIZE_MASK                                                             0x07F80000L
-#define SPI_SHADER_PGM_RSRC2_GS_VS__SKIP_USGPR0_MASK                                                          0x08000000L
-#define SPI_SHADER_PGM_RSRC2_GS_VS__USER_SGPR_MSB_MASK                                                        0x10000000L
-//SPI_SHADER_PGM_RSRC4_GS
-#define SPI_SHADER_PGM_RSRC4_GS__GROUP_FIFO_DEPTH__SHIFT                                                      0x0
-#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS__SHIFT                                              0x7
-#define SPI_SHADER_PGM_RSRC4_GS__GROUP_FIFO_DEPTH_MASK                                                        0x0000007FL
-#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS_MASK                                                0x00003F80L
-//SPI_SHADER_USER_DATA_ADDR_LO_GS
-#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE__SHIFT                                                      0x0
-#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ADDR_HI_GS
-#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE__SHIFT                                                      0x0
-#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
-//SPI_SHADER_PGM_LO_ES
-#define SPI_SHADER_PGM_LO_ES__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_LO_ES__MEM_BASE_MASK                                                                   0xFFFFFFFFL
-//SPI_SHADER_PGM_HI_ES
-#define SPI_SHADER_PGM_HI_ES__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_HI_ES__MEM_BASE_MASK                                                                   0xFFL
-//SPI_SHADER_PGM_RSRC3_GS
-#define SPI_SHADER_PGM_RSRC3_GS__CU_EN__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT__SHIFT                                                            0x10
-#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x16
-#define SPI_SHADER_PGM_RSRC3_GS__SIMD_DISABLE__SHIFT                                                          0x1a
-#define SPI_SHADER_PGM_RSRC3_GS__CU_EN_MASK                                                                   0x0000FFFFL
-#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT_MASK                                                              0x003F0000L
-#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD_MASK                                                      0x03C00000L
-#define SPI_SHADER_PGM_RSRC3_GS__SIMD_DISABLE_MASK                                                            0x3C000000L
-//SPI_SHADER_PGM_LO_GS
-#define SPI_SHADER_PGM_LO_GS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_LO_GS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
-//SPI_SHADER_PGM_HI_GS
-#define SPI_SHADER_PGM_HI_GS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_HI_GS__MEM_BASE_MASK                                                                   0xFFL
-//SPI_SHADER_PGM_RSRC1_GS
-#define SPI_SHADER_PGM_RSRC1_GS__VGPRS__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_RSRC1_GS__SGPRS__SHIFT                                                                 0x6
-#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY__SHIFT                                                              0xa
-#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE__SHIFT                                                            0xc
-#define SPI_SHADER_PGM_RSRC1_GS__PRIV__SHIFT                                                                  0x14
-#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP__SHIFT                                                            0x15
-#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE__SHIFT                                                             0x17
-#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE__SHIFT                                                       0x18
-#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT__SHIFT                                                      0x1d
-#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL__SHIFT                                                             0x1f
-#define SPI_SHADER_PGM_RSRC1_GS__VGPRS_MASK                                                                   0x0000003FL
-#define SPI_SHADER_PGM_RSRC1_GS__SGPRS_MASK                                                                   0x000003C0L
-#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY_MASK                                                                0x00000C00L
-#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE_MASK                                                              0x000FF000L
-#define SPI_SHADER_PGM_RSRC1_GS__PRIV_MASK                                                                    0x00100000L
-#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP_MASK                                                              0x00200000L
-#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE_MASK                                                               0x00800000L
-#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE_MASK                                                         0x01000000L
-#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT_MASK                                                        0x60000000L
-#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL_MASK                                                               0x80000000L
-//SPI_SHADER_PGM_RSRC2_GS
-#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN__SHIFT                                                            0x0
-#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR__SHIFT                                                             0x1
-#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT__SHIFT                                                          0x6
-#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN__SHIFT                                                               0x7
-#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT__SHIFT                                                      0x10
-#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN__SHIFT                                                             0x12
-#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE__SHIFT                                                              0x13
-#define SPI_SHADER_PGM_RSRC2_GS__SKIP_USGPR0__SHIFT                                                           0x1b
-#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB__SHIFT                                                         0x1c
-#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN_MASK                                                              0x00000001L
-#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MASK                                                               0x0000003EL
-#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT_MASK                                                            0x00000040L
-#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN_MASK                                                                 0x0000FF80L
-#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT_MASK                                                        0x00030000L
-#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN_MASK                                                               0x00040000L
-#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE_MASK                                                                0x07F80000L
-#define SPI_SHADER_PGM_RSRC2_GS__SKIP_USGPR0_MASK                                                             0x08000000L
-#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB_MASK                                                           0x10000000L
-//SPI_SHADER_USER_DATA_ES_0
-#define SPI_SHADER_USER_DATA_ES_0__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_0__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_1
-#define SPI_SHADER_USER_DATA_ES_1__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_1__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_2
-#define SPI_SHADER_USER_DATA_ES_2__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_2__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_3
-#define SPI_SHADER_USER_DATA_ES_3__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_3__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_4
-#define SPI_SHADER_USER_DATA_ES_4__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_4__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_5
-#define SPI_SHADER_USER_DATA_ES_5__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_5__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_6
-#define SPI_SHADER_USER_DATA_ES_6__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_6__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_7
-#define SPI_SHADER_USER_DATA_ES_7__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_7__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_8
-#define SPI_SHADER_USER_DATA_ES_8__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_8__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_9
-#define SPI_SHADER_USER_DATA_ES_9__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_ES_9__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_10
-#define SPI_SHADER_USER_DATA_ES_10__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_10__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_11
-#define SPI_SHADER_USER_DATA_ES_11__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_11__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_12
-#define SPI_SHADER_USER_DATA_ES_12__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_12__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_13
-#define SPI_SHADER_USER_DATA_ES_13__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_13__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_14
-#define SPI_SHADER_USER_DATA_ES_14__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_14__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_15
-#define SPI_SHADER_USER_DATA_ES_15__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_15__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_16
-#define SPI_SHADER_USER_DATA_ES_16__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_16__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_17
-#define SPI_SHADER_USER_DATA_ES_17__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_17__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_18
-#define SPI_SHADER_USER_DATA_ES_18__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_18__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_19
-#define SPI_SHADER_USER_DATA_ES_19__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_19__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_20
-#define SPI_SHADER_USER_DATA_ES_20__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_20__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_21
-#define SPI_SHADER_USER_DATA_ES_21__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_21__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_22
-#define SPI_SHADER_USER_DATA_ES_22__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_22__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_23
-#define SPI_SHADER_USER_DATA_ES_23__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_23__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_24
-#define SPI_SHADER_USER_DATA_ES_24__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_24__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_25
-#define SPI_SHADER_USER_DATA_ES_25__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_25__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_26
-#define SPI_SHADER_USER_DATA_ES_26__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_26__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_27
-#define SPI_SHADER_USER_DATA_ES_27__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_27__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_28
-#define SPI_SHADER_USER_DATA_ES_28__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_28__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_29
-#define SPI_SHADER_USER_DATA_ES_29__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_29__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_30
-#define SPI_SHADER_USER_DATA_ES_30__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_30__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ES_31
-#define SPI_SHADER_USER_DATA_ES_31__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_ES_31__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_PGM_RSRC4_HS
-#define SPI_SHADER_PGM_RSRC4_HS__GROUP_FIFO_DEPTH__SHIFT                                                      0x0
-#define SPI_SHADER_PGM_RSRC4_HS__GROUP_FIFO_DEPTH_MASK                                                        0x0000007FL
-//SPI_SHADER_USER_DATA_ADDR_LO_HS
-#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE__SHIFT                                                      0x0
-#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_ADDR_HI_HS
-#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE__SHIFT                                                      0x0
-#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
-//SPI_SHADER_PGM_LO_LS
-#define SPI_SHADER_PGM_LO_LS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_LO_LS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
-//SPI_SHADER_PGM_HI_LS
-#define SPI_SHADER_PGM_HI_LS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_HI_LS__MEM_BASE_MASK                                                                   0xFFL
-//SPI_SHADER_PGM_RSRC3_HS
-#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT__SHIFT                                                            0x0
-#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x6
-#define SPI_SHADER_PGM_RSRC3_HS__SIMD_DISABLE__SHIFT                                                          0xa
-#define SPI_SHADER_PGM_RSRC3_HS__CU_EN__SHIFT                                                                 0x10
-#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT_MASK                                                              0x0000003FL
-#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD_MASK                                                      0x000003C0L
-#define SPI_SHADER_PGM_RSRC3_HS__SIMD_DISABLE_MASK                                                            0x00003C00L
-#define SPI_SHADER_PGM_RSRC3_HS__CU_EN_MASK                                                                   0xFFFF0000L
-//SPI_SHADER_PGM_LO_HS
-#define SPI_SHADER_PGM_LO_HS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_LO_HS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
-//SPI_SHADER_PGM_HI_HS
-#define SPI_SHADER_PGM_HI_HS__MEM_BASE__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_HI_HS__MEM_BASE_MASK                                                                   0xFFL
-//SPI_SHADER_PGM_RSRC1_HS
-#define SPI_SHADER_PGM_RSRC1_HS__VGPRS__SHIFT                                                                 0x0
-#define SPI_SHADER_PGM_RSRC1_HS__SGPRS__SHIFT                                                                 0x6
-#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY__SHIFT                                                              0xa
-#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE__SHIFT                                                            0xc
-#define SPI_SHADER_PGM_RSRC1_HS__PRIV__SHIFT                                                                  0x14
-#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP__SHIFT                                                            0x15
-#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE__SHIFT                                                             0x17
-#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT__SHIFT                                                      0x1c
-#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL__SHIFT                                                             0x1e
-#define SPI_SHADER_PGM_RSRC1_HS__VGPRS_MASK                                                                   0x0000003FL
-#define SPI_SHADER_PGM_RSRC1_HS__SGPRS_MASK                                                                   0x000003C0L
-#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY_MASK                                                                0x00000C00L
-#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE_MASK                                                              0x000FF000L
-#define SPI_SHADER_PGM_RSRC1_HS__PRIV_MASK                                                                    0x00100000L
-#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP_MASK                                                              0x00200000L
-#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE_MASK                                                               0x00800000L
-#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT_MASK                                                        0x30000000L
-#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL_MASK                                                               0x40000000L
-//SPI_SHADER_PGM_RSRC2_HS
-#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN__SHIFT                                                            0x0
-#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR__SHIFT                                                             0x1
-#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT__SHIFT                                                          0x6
-#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN__SHIFT                                                               0x7
-#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE__SHIFT                                                              0x10
-#define SPI_SHADER_PGM_RSRC2_HS__SKIP_USGPR0__SHIFT                                                           0x1b
-#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB__SHIFT                                                         0x1c
-#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN_MASK                                                              0x00000001L
-#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MASK                                                               0x0000003EL
-#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT_MASK                                                            0x00000040L
-#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN_MASK                                                                 0x0000FF80L
-#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE_MASK                                                                0x01FF0000L
-#define SPI_SHADER_PGM_RSRC2_HS__SKIP_USGPR0_MASK                                                             0x08000000L
-#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB_MASK                                                           0x10000000L
-//SPI_SHADER_USER_DATA_LS_0
-#define SPI_SHADER_USER_DATA_LS_0__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_0__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_1
-#define SPI_SHADER_USER_DATA_LS_1__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_1__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_2
-#define SPI_SHADER_USER_DATA_LS_2__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_2__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_3
-#define SPI_SHADER_USER_DATA_LS_3__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_3__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_4
-#define SPI_SHADER_USER_DATA_LS_4__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_4__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_5
-#define SPI_SHADER_USER_DATA_LS_5__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_5__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_6
-#define SPI_SHADER_USER_DATA_LS_6__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_6__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_7
-#define SPI_SHADER_USER_DATA_LS_7__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_7__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_8
-#define SPI_SHADER_USER_DATA_LS_8__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_8__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_9
-#define SPI_SHADER_USER_DATA_LS_9__DATA__SHIFT                                                                0x0
-#define SPI_SHADER_USER_DATA_LS_9__DATA_MASK                                                                  0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_10
-#define SPI_SHADER_USER_DATA_LS_10__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_10__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_11
-#define SPI_SHADER_USER_DATA_LS_11__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_11__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_12
-#define SPI_SHADER_USER_DATA_LS_12__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_12__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_13
-#define SPI_SHADER_USER_DATA_LS_13__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_13__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_14
-#define SPI_SHADER_USER_DATA_LS_14__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_14__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_15
-#define SPI_SHADER_USER_DATA_LS_15__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_15__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_16
-#define SPI_SHADER_USER_DATA_LS_16__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_16__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_17
-#define SPI_SHADER_USER_DATA_LS_17__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_17__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_18
-#define SPI_SHADER_USER_DATA_LS_18__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_18__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_19
-#define SPI_SHADER_USER_DATA_LS_19__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_19__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_20
-#define SPI_SHADER_USER_DATA_LS_20__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_20__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_21
-#define SPI_SHADER_USER_DATA_LS_21__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_21__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_22
-#define SPI_SHADER_USER_DATA_LS_22__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_22__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_23
-#define SPI_SHADER_USER_DATA_LS_23__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_23__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_24
-#define SPI_SHADER_USER_DATA_LS_24__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_24__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_25
-#define SPI_SHADER_USER_DATA_LS_25__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_25__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_26
-#define SPI_SHADER_USER_DATA_LS_26__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_26__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_27
-#define SPI_SHADER_USER_DATA_LS_27__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_27__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_28
-#define SPI_SHADER_USER_DATA_LS_28__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_28__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_29
-#define SPI_SHADER_USER_DATA_LS_29__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_29__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_30
-#define SPI_SHADER_USER_DATA_LS_30__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_30__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_LS_31
-#define SPI_SHADER_USER_DATA_LS_31__DATA__SHIFT                                                               0x0
-#define SPI_SHADER_USER_DATA_LS_31__DATA_MASK                                                                 0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_0
-#define SPI_SHADER_USER_DATA_COMMON_0__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_0__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_1
-#define SPI_SHADER_USER_DATA_COMMON_1__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_1__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_2
-#define SPI_SHADER_USER_DATA_COMMON_2__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_2__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_3
-#define SPI_SHADER_USER_DATA_COMMON_3__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_3__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_4
-#define SPI_SHADER_USER_DATA_COMMON_4__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_4__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_5
-#define SPI_SHADER_USER_DATA_COMMON_5__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_5__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_6
-#define SPI_SHADER_USER_DATA_COMMON_6__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_6__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_7
-#define SPI_SHADER_USER_DATA_COMMON_7__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_7__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_8
-#define SPI_SHADER_USER_DATA_COMMON_8__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_8__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_9
-#define SPI_SHADER_USER_DATA_COMMON_9__DATA__SHIFT                                                            0x0
-#define SPI_SHADER_USER_DATA_COMMON_9__DATA_MASK                                                              0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_10
-#define SPI_SHADER_USER_DATA_COMMON_10__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_10__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_11
-#define SPI_SHADER_USER_DATA_COMMON_11__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_11__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_12
-#define SPI_SHADER_USER_DATA_COMMON_12__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_12__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_13
-#define SPI_SHADER_USER_DATA_COMMON_13__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_13__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_14
-#define SPI_SHADER_USER_DATA_COMMON_14__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_14__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_15
-#define SPI_SHADER_USER_DATA_COMMON_15__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_15__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_16
-#define SPI_SHADER_USER_DATA_COMMON_16__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_16__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_17
-#define SPI_SHADER_USER_DATA_COMMON_17__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_17__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_18
-#define SPI_SHADER_USER_DATA_COMMON_18__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_18__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_19
-#define SPI_SHADER_USER_DATA_COMMON_19__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_19__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_20
-#define SPI_SHADER_USER_DATA_COMMON_20__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_20__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_21
-#define SPI_SHADER_USER_DATA_COMMON_21__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_21__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_22
-#define SPI_SHADER_USER_DATA_COMMON_22__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_22__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_23
-#define SPI_SHADER_USER_DATA_COMMON_23__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_23__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_24
-#define SPI_SHADER_USER_DATA_COMMON_24__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_24__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_25
-#define SPI_SHADER_USER_DATA_COMMON_25__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_25__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_26
-#define SPI_SHADER_USER_DATA_COMMON_26__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_26__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_27
-#define SPI_SHADER_USER_DATA_COMMON_27__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_27__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_28
-#define SPI_SHADER_USER_DATA_COMMON_28__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_28__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_29
-#define SPI_SHADER_USER_DATA_COMMON_29__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_29__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_30
-#define SPI_SHADER_USER_DATA_COMMON_30__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_30__DATA_MASK                                                             0xFFFFFFFFL
-//SPI_SHADER_USER_DATA_COMMON_31
-#define SPI_SHADER_USER_DATA_COMMON_31__DATA__SHIFT                                                           0x0
-#define SPI_SHADER_USER_DATA_COMMON_31__DATA_MASK                                                             0xFFFFFFFFL
-//COMPUTE_DISPATCH_INITIATOR
-#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT                                                  0x0
-#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT                                                      0x1
-#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT                                                 0x2
-#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT                                                0x3
-#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT                                                0x4
-#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT                                              0x5
-#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT                                                         0x6
-#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL__SHIFT                                                  0xa
-#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL__SHIFT                                                  0xb
-#define COMPUTE_DISPATCH_INITIATOR__RESERVED__SHIFT                                                           0xc
-#define COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT                                                            0xe
-#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK                                                    0x00000001L
-#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK                                                        0x00000002L
-#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK                                                   0x00000004L
-#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK                                                  0x00000008L
-#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK                                                  0x00000010L
-#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK                                                0x00000020L
-#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK                                                           0x00000040L
-#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL_MASK                                                    0x00000400L
-#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL_MASK                                                    0x00000800L
-#define COMPUTE_DISPATCH_INITIATOR__RESERVED_MASK                                                             0x00001000L
-#define COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK                                                              0x00004000L
-//COMPUTE_DIM_X
-#define COMPUTE_DIM_X__SIZE__SHIFT                                                                            0x0
-#define COMPUTE_DIM_X__SIZE_MASK                                                                              0xFFFFFFFFL
-//COMPUTE_DIM_Y
-#define COMPUTE_DIM_Y__SIZE__SHIFT                                                                            0x0
-#define COMPUTE_DIM_Y__SIZE_MASK                                                                              0xFFFFFFFFL
-//COMPUTE_DIM_Z
-#define COMPUTE_DIM_Z__SIZE__SHIFT                                                                            0x0
-#define COMPUTE_DIM_Z__SIZE_MASK                                                                              0xFFFFFFFFL
-//COMPUTE_START_X
-#define COMPUTE_START_X__START__SHIFT                                                                         0x0
-#define COMPUTE_START_X__START_MASK                                                                           0xFFFFFFFFL
-//COMPUTE_START_Y
-#define COMPUTE_START_Y__START__SHIFT                                                                         0x0
-#define COMPUTE_START_Y__START_MASK                                                                           0xFFFFFFFFL
-//COMPUTE_START_Z
-#define COMPUTE_START_Z__START__SHIFT                                                                         0x0
-#define COMPUTE_START_Z__START_MASK                                                                           0xFFFFFFFFL
-//COMPUTE_NUM_THREAD_X
-#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT                                                          0x0
-#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
-#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
-#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
-//COMPUTE_NUM_THREAD_Y
-#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT                                                          0x0
-#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
-#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
-#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
-//COMPUTE_NUM_THREAD_Z
-#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT                                                          0x0
-#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
-#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
-#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
-//COMPUTE_PIPELINESTAT_ENABLE
-#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE__SHIFT                                               0x0
-#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE_MASK                                                 0x00000001L
-//COMPUTE_PERFCOUNT_ENABLE
-#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE__SHIFT                                                     0x0
-#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE_MASK                                                       0x00000001L
-//COMPUTE_PGM_LO
-#define COMPUTE_PGM_LO__DATA__SHIFT                                                                           0x0
-#define COMPUTE_PGM_LO__DATA_MASK                                                                             0xFFFFFFFFL
-//COMPUTE_PGM_HI
-#define COMPUTE_PGM_HI__DATA__SHIFT                                                                           0x0
-#define COMPUTE_PGM_HI__DATA_MASK                                                                             0x000000FFL
-//COMPUTE_DISPATCH_PKT_ADDR_LO
-#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA__SHIFT                                                             0x0
-#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA_MASK                                                               0xFFFFFFFFL
-//COMPUTE_DISPATCH_PKT_ADDR_HI
-#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA__SHIFT                                                             0x0
-#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA_MASK                                                               0x000000FFL
-//COMPUTE_DISPATCH_SCRATCH_BASE_LO
-#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA__SHIFT                                                         0x0
-#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA_MASK                                                           0xFFFFFFFFL
-//COMPUTE_DISPATCH_SCRATCH_BASE_HI
-#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA__SHIFT                                                         0x0
-#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA_MASK                                                           0x000000FFL
-//COMPUTE_PGM_RSRC1
-#define COMPUTE_PGM_RSRC1__VGPRS__SHIFT                                                                       0x0
-#define COMPUTE_PGM_RSRC1__SGPRS__SHIFT                                                                       0x6
-#define COMPUTE_PGM_RSRC1__PRIORITY__SHIFT                                                                    0xa
-#define COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT                                                                  0xc
-#define COMPUTE_PGM_RSRC1__PRIV__SHIFT                                                                        0x14
-#define COMPUTE_PGM_RSRC1__DX10_CLAMP__SHIFT                                                                  0x15
-#define COMPUTE_PGM_RSRC1__IEEE_MODE__SHIFT                                                                   0x17
-#define COMPUTE_PGM_RSRC1__BULKY__SHIFT                                                                       0x18
-#define COMPUTE_PGM_RSRC1__FP16_OVFL__SHIFT                                                                   0x1a
-#define COMPUTE_PGM_RSRC1__VGPRS_MASK                                                                         0x0000003FL
-#define COMPUTE_PGM_RSRC1__SGPRS_MASK                                                                         0x000003C0L
-#define COMPUTE_PGM_RSRC1__PRIORITY_MASK                                                                      0x00000C00L
-#define COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK                                                                    0x000FF000L
-#define COMPUTE_PGM_RSRC1__PRIV_MASK                                                                          0x00100000L
-#define COMPUTE_PGM_RSRC1__DX10_CLAMP_MASK                                                                    0x00200000L
-#define COMPUTE_PGM_RSRC1__IEEE_MODE_MASK                                                                     0x00800000L
-#define COMPUTE_PGM_RSRC1__BULKY_MASK                                                                         0x01000000L
-#define COMPUTE_PGM_RSRC1__FP16_OVFL_MASK                                                                     0x04000000L
-//COMPUTE_PGM_RSRC2
-#define COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT                                                                  0x0
-#define COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT                                                                   0x1
-#define COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT                                                                0x6
-#define COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT                                                                   0x7
-#define COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT                                                                   0x8
-#define COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT                                                                   0x9
-#define COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT                                                                  0xa
-#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT                                                              0xb
-#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT                                                                 0xd
-#define COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT                                                                    0xf
-#define COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT                                                                     0x18
-#define COMPUTE_PGM_RSRC2__SKIP_USGPR0__SHIFT                                                                 0x1f
-#define COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK                                                                    0x00000001L
-#define COMPUTE_PGM_RSRC2__USER_SGPR_MASK                                                                     0x0000003EL
-#define COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK                                                                  0x00000040L
-#define COMPUTE_PGM_RSRC2__TGID_X_EN_MASK                                                                     0x00000080L
-#define COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK                                                                     0x00000100L
-#define COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK                                                                     0x00000200L
-#define COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK                                                                    0x00000400L
-#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK                                                                0x00001800L
-#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK                                                                   0x00006000L
-#define COMPUTE_PGM_RSRC2__LDS_SIZE_MASK                                                                      0x00FF8000L
-#define COMPUTE_PGM_RSRC2__EXCP_EN_MASK                                                                       0x7F000000L
-#define COMPUTE_PGM_RSRC2__SKIP_USGPR0_MASK                                                                   0x80000000L
-//COMPUTE_VMID
-#define COMPUTE_VMID__DATA__SHIFT                                                                             0x0
-#define COMPUTE_VMID__DATA_MASK                                                                               0x0000000FL
-//COMPUTE_RESOURCE_LIMITS
-#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT                                                          0x0
-#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT                                                             0xc
-#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT                                                        0x10
-#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT                                                        0x16
-#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT                                                       0x17
-#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT                                                        0x18
-#define COMPUTE_RESOURCE_LIMITS__SIMD_DISABLE__SHIFT                                                          0x1b
-#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK                                                            0x000003FFL
-#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK                                                               0x0000F000L
-#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK                                                          0x003F0000L
-#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK                                                          0x00400000L
-#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK                                                         0x00800000L
-#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK                                                          0x07000000L
-#define COMPUTE_RESOURCE_LIMITS__SIMD_DISABLE_MASK                                                            0x78000000L
-//COMPUTE_STATIC_THREAD_MGMT_SE0
-#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN__SHIFT                                                      0x0
-#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN__SHIFT                                                      0x10
-#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN_MASK                                                        0x0000FFFFL
-#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN_MASK                                                        0xFFFF0000L
-//COMPUTE_STATIC_THREAD_MGMT_SE1
-#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN__SHIFT                                                      0x0
-#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN__SHIFT                                                      0x10
-#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN_MASK                                                        0x0000FFFFL
-#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN_MASK                                                        0xFFFF0000L
-//COMPUTE_TMPRING_SIZE
-#define COMPUTE_TMPRING_SIZE__WAVES__SHIFT                                                                    0x0
-#define COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT                                                                 0xc
-#define COMPUTE_TMPRING_SIZE__WAVES_MASK                                                                      0x00000FFFL
-#define COMPUTE_TMPRING_SIZE__WAVESIZE_MASK                                                                   0x01FFF000L
-//COMPUTE_STATIC_THREAD_MGMT_SE2
-#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN__SHIFT                                                      0x0
-#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN__SHIFT                                                      0x10
-#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN_MASK                                                        0x0000FFFFL
-#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN_MASK                                                        0xFFFF0000L
-//COMPUTE_STATIC_THREAD_MGMT_SE3
-#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN__SHIFT                                                      0x0
-#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN__SHIFT                                                      0x10
-#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN_MASK                                                        0x0000FFFFL
-#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN_MASK                                                        0xFFFF0000L
-//COMPUTE_RESTART_X
-#define COMPUTE_RESTART_X__RESTART__SHIFT                                                                     0x0
-#define COMPUTE_RESTART_X__RESTART_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_RESTART_Y
-#define COMPUTE_RESTART_Y__RESTART__SHIFT                                                                     0x0
-#define COMPUTE_RESTART_Y__RESTART_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_RESTART_Z
-#define COMPUTE_RESTART_Z__RESTART__SHIFT                                                                     0x0
-#define COMPUTE_RESTART_Z__RESTART_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_THREAD_TRACE_ENABLE
-#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE__SHIFT                                               0x0
-#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE_MASK                                                 0x00000001L
-//COMPUTE_MISC_RESERVED
-#define COMPUTE_MISC_RESERVED__SEND_SEID__SHIFT                                                               0x0
-#define COMPUTE_MISC_RESERVED__RESERVED2__SHIFT                                                               0x2
-#define COMPUTE_MISC_RESERVED__RESERVED3__SHIFT                                                               0x3
-#define COMPUTE_MISC_RESERVED__RESERVED4__SHIFT                                                               0x4
-#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE__SHIFT                                                            0x5
-#define COMPUTE_MISC_RESERVED__SEND_SEID_MASK                                                                 0x00000003L
-#define COMPUTE_MISC_RESERVED__RESERVED2_MASK                                                                 0x00000004L
-#define COMPUTE_MISC_RESERVED__RESERVED3_MASK                                                                 0x00000008L
-#define COMPUTE_MISC_RESERVED__RESERVED4_MASK                                                                 0x00000010L
-#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE_MASK                                                              0x0001FFE0L
-//COMPUTE_DISPATCH_ID
-#define COMPUTE_DISPATCH_ID__DISPATCH_ID__SHIFT                                                               0x0
-#define COMPUTE_DISPATCH_ID__DISPATCH_ID_MASK                                                                 0xFFFFFFFFL
-//COMPUTE_THREADGROUP_ID
-#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID__SHIFT                                                         0x0
-#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID_MASK                                                           0xFFFFFFFFL
-//COMPUTE_RELAUNCH
-#define COMPUTE_RELAUNCH__PAYLOAD__SHIFT                                                                      0x0
-#define COMPUTE_RELAUNCH__IS_EVENT__SHIFT                                                                     0x1e
-#define COMPUTE_RELAUNCH__IS_STATE__SHIFT                                                                     0x1f
-#define COMPUTE_RELAUNCH__PAYLOAD_MASK                                                                        0x3FFFFFFFL
-#define COMPUTE_RELAUNCH__IS_EVENT_MASK                                                                       0x40000000L
-#define COMPUTE_RELAUNCH__IS_STATE_MASK                                                                       0x80000000L
-//COMPUTE_WAVE_RESTORE_ADDR_LO
-#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR__SHIFT                                                             0x0
-#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFFL
-//COMPUTE_WAVE_RESTORE_ADDR_HI
-#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR__SHIFT                                                             0x0
-#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR_MASK                                                               0xFFFFL
-//COMPUTE_USER_DATA_0
-#define COMPUTE_USER_DATA_0__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_0__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_1
-#define COMPUTE_USER_DATA_1__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_1__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_2
-#define COMPUTE_USER_DATA_2__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_2__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_3
-#define COMPUTE_USER_DATA_3__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_3__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_4
-#define COMPUTE_USER_DATA_4__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_4__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_5
-#define COMPUTE_USER_DATA_5__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_5__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_6
-#define COMPUTE_USER_DATA_6__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_6__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_7
-#define COMPUTE_USER_DATA_7__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_7__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_8
-#define COMPUTE_USER_DATA_8__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_8__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_9
-#define COMPUTE_USER_DATA_9__DATA__SHIFT                                                                      0x0
-#define COMPUTE_USER_DATA_9__DATA_MASK                                                                        0xFFFFFFFFL
-//COMPUTE_USER_DATA_10
-#define COMPUTE_USER_DATA_10__DATA__SHIFT                                                                     0x0
-#define COMPUTE_USER_DATA_10__DATA_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_USER_DATA_11
-#define COMPUTE_USER_DATA_11__DATA__SHIFT                                                                     0x0
-#define COMPUTE_USER_DATA_11__DATA_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_USER_DATA_12
-#define COMPUTE_USER_DATA_12__DATA__SHIFT                                                                     0x0
-#define COMPUTE_USER_DATA_12__DATA_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_USER_DATA_13
-#define COMPUTE_USER_DATA_13__DATA__SHIFT                                                                     0x0
-#define COMPUTE_USER_DATA_13__DATA_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_USER_DATA_14
-#define COMPUTE_USER_DATA_14__DATA__SHIFT                                                                     0x0
-#define COMPUTE_USER_DATA_14__DATA_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_USER_DATA_15
-#define COMPUTE_USER_DATA_15__DATA__SHIFT                                                                     0x0
-#define COMPUTE_USER_DATA_15__DATA_MASK                                                                       0xFFFFFFFFL
-//COMPUTE_NOWHERE
-#define COMPUTE_NOWHERE__DATA__SHIFT                                                                          0x0
-#define COMPUTE_NOWHERE__DATA_MASK                                                                            0xFFFFFFFFL
-
-
-// addressBlock: gc_cppdec
-//CP_DFY_CNTL
-#define CP_DFY_CNTL__POLICY__SHIFT                                                                            0x0
-#define CP_DFY_CNTL__MTYPE__SHIFT                                                                             0x2
-#define CP_DFY_CNTL__TPI_SDP_SEL__SHIFT                                                                       0x1a
-#define CP_DFY_CNTL__LFSR_RESET__SHIFT                                                                        0x1c
-#define CP_DFY_CNTL__MODE__SHIFT                                                                              0x1d
-#define CP_DFY_CNTL__ENABLE__SHIFT                                                                            0x1f
-#define CP_DFY_CNTL__POLICY_MASK                                                                              0x00000001L
-#define CP_DFY_CNTL__MTYPE_MASK                                                                               0x0000000CL
-#define CP_DFY_CNTL__TPI_SDP_SEL_MASK                                                                         0x04000000L
-#define CP_DFY_CNTL__LFSR_RESET_MASK                                                                          0x10000000L
-#define CP_DFY_CNTL__MODE_MASK                                                                                0x60000000L
-#define CP_DFY_CNTL__ENABLE_MASK                                                                              0x80000000L
-//CP_DFY_STAT
-#define CP_DFY_STAT__BURST_COUNT__SHIFT                                                                       0x0
-#define CP_DFY_STAT__TAGS_PENDING__SHIFT                                                                      0x10
-#define CP_DFY_STAT__BUSY__SHIFT                                                                              0x1f
-#define CP_DFY_STAT__BURST_COUNT_MASK                                                                         0x0000FFFFL
-#define CP_DFY_STAT__TAGS_PENDING_MASK                                                                        0x07FF0000L
-#define CP_DFY_STAT__BUSY_MASK                                                                                0x80000000L
-//CP_DFY_ADDR_HI
-#define CP_DFY_ADDR_HI__ADDR_HI__SHIFT                                                                        0x0
-#define CP_DFY_ADDR_HI__ADDR_HI_MASK                                                                          0xFFFFFFFFL
-//CP_DFY_ADDR_LO
-#define CP_DFY_ADDR_LO__ADDR_LO__SHIFT                                                                        0x5
-#define CP_DFY_ADDR_LO__ADDR_LO_MASK                                                                          0xFFFFFFE0L
-//CP_DFY_DATA_0
-#define CP_DFY_DATA_0__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_0__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_1
-#define CP_DFY_DATA_1__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_1__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_2
-#define CP_DFY_DATA_2__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_2__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_3
-#define CP_DFY_DATA_3__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_3__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_4
-#define CP_DFY_DATA_4__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_4__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_5
-#define CP_DFY_DATA_5__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_5__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_6
-#define CP_DFY_DATA_6__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_6__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_7
-#define CP_DFY_DATA_7__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_7__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_8
-#define CP_DFY_DATA_8__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_8__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_9
-#define CP_DFY_DATA_9__DATA__SHIFT                                                                            0x0
-#define CP_DFY_DATA_9__DATA_MASK                                                                              0xFFFFFFFFL
-//CP_DFY_DATA_10
-#define CP_DFY_DATA_10__DATA__SHIFT                                                                           0x0
-#define CP_DFY_DATA_10__DATA_MASK                                                                             0xFFFFFFFFL
-//CP_DFY_DATA_11
-#define CP_DFY_DATA_11__DATA__SHIFT                                                                           0x0
-#define CP_DFY_DATA_11__DATA_MASK                                                                             0xFFFFFFFFL
-//CP_DFY_DATA_12
-#define CP_DFY_DATA_12__DATA__SHIFT                                                                           0x0
-#define CP_DFY_DATA_12__DATA_MASK                                                                             0xFFFFFFFFL
-//CP_DFY_DATA_13
-#define CP_DFY_DATA_13__DATA__SHIFT                                                                           0x0
-#define CP_DFY_DATA_13__DATA_MASK                                                                             0xFFFFFFFFL
-//CP_DFY_DATA_14
-#define CP_DFY_DATA_14__DATA__SHIFT                                                                           0x0
-#define CP_DFY_DATA_14__DATA_MASK                                                                             0xFFFFFFFFL
-//CP_DFY_DATA_15
-#define CP_DFY_DATA_15__DATA__SHIFT                                                                           0x0
-#define CP_DFY_DATA_15__DATA_MASK                                                                             0xFFFFFFFFL
-//CP_DFY_CMD
-#define CP_DFY_CMD__OFFSET__SHIFT                                                                             0x0
-#define CP_DFY_CMD__SIZE__SHIFT                                                                               0x10
-#define CP_DFY_CMD__OFFSET_MASK                                                                               0x000001FFL
-#define CP_DFY_CMD__SIZE_MASK                                                                                 0xFFFF0000L
-//CP_EOPQ_WAIT_TIME
-#define CP_EOPQ_WAIT_TIME__WAIT_TIME__SHIFT                                                                   0x0
-#define CP_EOPQ_WAIT_TIME__SCALE_COUNT__SHIFT                                                                 0xa
-#define CP_EOPQ_WAIT_TIME__WAIT_TIME_MASK                                                                     0x000003FFL
-#define CP_EOPQ_WAIT_TIME__SCALE_COUNT_MASK                                                                   0x0003FC00L
-//CP_CPC_MGCG_SYNC_CNTL
-#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD__SHIFT                                                         0x0
-#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD__SHIFT                                                           0x8
-#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD_MASK                                                           0x000000FFL
-#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD_MASK                                                             0x0000FF00L
-//CPC_INT_INFO
-#define CPC_INT_INFO__ADDR_HI__SHIFT                                                                          0x0
-#define CPC_INT_INFO__TYPE__SHIFT                                                                             0x10
-#define CPC_INT_INFO__VMID__SHIFT                                                                             0x14
-#define CPC_INT_INFO__QUEUE_ID__SHIFT                                                                         0x1c
-#define CPC_INT_INFO__ADDR_HI_MASK                                                                            0x0000FFFFL
-#define CPC_INT_INFO__TYPE_MASK                                                                               0x00010000L
-#define CPC_INT_INFO__VMID_MASK                                                                               0x00F00000L
-#define CPC_INT_INFO__QUEUE_ID_MASK                                                                           0x70000000L
-//CP_VIRT_STATUS
-#define CP_VIRT_STATUS__VIRT_STATUS__SHIFT                                                                    0x0
-#define CP_VIRT_STATUS__VIRT_STATUS_MASK                                                                      0xFFFFFFFFL
-//CPC_INT_ADDR
-#define CPC_INT_ADDR__ADDR__SHIFT                                                                             0x0
-#define CPC_INT_ADDR__ADDR_MASK                                                                               0xFFFFFFFFL
-//CPC_INT_PASID
-#define CPC_INT_PASID__PASID__SHIFT                                                                           0x0
-#define CPC_INT_PASID__PASID_MASK                                                                             0x0000FFFFL
-//CP_GFX_ERROR
-#define CP_GFX_ERROR__EDC_ERROR_ID__SHIFT                                                                     0x0
-#define CP_GFX_ERROR__SUA_ERROR__SHIFT                                                                        0x4
-#define CP_GFX_ERROR__RSVD1_ERROR__SHIFT                                                                      0x5
-#define CP_GFX_ERROR__RSVD2_ERROR__SHIFT                                                                      0x6
-#define CP_GFX_ERROR__SEM_UTCL1_ERROR__SHIFT                                                                  0x7
-#define CP_GFX_ERROR__QU_STRM_UTCL1_ERROR__SHIFT                                                              0x8
-#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR__SHIFT                                                               0x9
-#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR__SHIFT                                                              0xa
-#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR__SHIFT                                                              0xb
-#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR__SHIFT                                                           0xc
-#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR__SHIFT                                                           0xd
-#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR__SHIFT                                                               0xe
-#define CP_GFX_ERROR__APPEND_UTCL1_ERROR__SHIFT                                                               0xf
-#define CP_GFX_ERROR__CE_DMA_UTCL1_ERROR__SHIFT                                                               0x10
-#define CP_GFX_ERROR__PFP_VGTDMA_UTCL1_ERROR__SHIFT                                                           0x11
-#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0x12
-#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x13
-#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR__SHIFT                                                               0x14
-#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR__SHIFT                                                                0x15
-#define CP_GFX_ERROR__CE_TC_UTCL1_ERROR__SHIFT                                                                0x16
-#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR__SHIFT                                                              0x17
-#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR__SHIFT                                                            0x18
-#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR__SHIFT                                                           0x19
-#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1a
-#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1b
-#define CP_GFX_ERROR__C1_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1c
-#define CP_GFX_ERROR__C2_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1d
-#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1e
-#define CP_GFX_ERROR__CE_INIT_UTCL1_ERROR__SHIFT                                                              0x1f
-#define CP_GFX_ERROR__EDC_ERROR_ID_MASK                                                                       0x0000000FL
-#define CP_GFX_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
-#define CP_GFX_ERROR__RSVD1_ERROR_MASK                                                                        0x00000020L
-#define CP_GFX_ERROR__RSVD2_ERROR_MASK                                                                        0x00000040L
-#define CP_GFX_ERROR__SEM_UTCL1_ERROR_MASK                                                                    0x00000080L
-#define CP_GFX_ERROR__QU_STRM_UTCL1_ERROR_MASK                                                                0x00000100L
-#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR_MASK                                                                 0x00000200L
-#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR_MASK                                                                0x00000400L
-#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR_MASK                                                                0x00000800L
-#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR_MASK                                                             0x00001000L
-#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR_MASK                                                             0x00002000L
-#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR_MASK                                                                 0x00004000L
-#define CP_GFX_ERROR__APPEND_UTCL1_ERROR_MASK                                                                 0x00008000L
-#define CP_GFX_ERROR__CE_DMA_UTCL1_ERROR_MASK                                                                 0x00010000L
-#define CP_GFX_ERROR__PFP_VGTDMA_UTCL1_ERROR_MASK                                                             0x00020000L
-#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00040000L
-#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00080000L
-#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR_MASK                                                                 0x00100000L
-#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR_MASK                                                                  0x00200000L
-#define CP_GFX_ERROR__CE_TC_UTCL1_ERROR_MASK                                                                  0x00400000L
-#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR_MASK                                                                0x00800000L
-#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR_MASK                                                              0x01000000L
-#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR_MASK                                                             0x02000000L
-#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR_MASK                                                             0x04000000L
-#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR_MASK                                                             0x08000000L
-#define CP_GFX_ERROR__C1_FETCHER_UTCL1_ERROR_MASK                                                             0x10000000L
-#define CP_GFX_ERROR__C2_FETCHER_UTCL1_ERROR_MASK                                                             0x20000000L
-#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR_MASK                                                             0x40000000L
-#define CP_GFX_ERROR__CE_INIT_UTCL1_ERROR_MASK                                                                0x80000000L
-//CPG_UTCL1_CNTL
-#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
-#define CPG_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
-#define CPG_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
-#define CPG_UTCL1_CNTL__BYPASS__SHIFT                                                                         0x19
-#define CPG_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
-#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
-#define CPG_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
-#define CPG_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                            0x1d
-#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
-#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
-#define CPG_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
-#define CPG_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
-#define CPG_UTCL1_CNTL__BYPASS_MASK                                                                           0x02000000L
-#define CPG_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
-#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
-#define CPG_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
-#define CPG_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                              0x20000000L
-#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
-//CPC_UTCL1_CNTL
-#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
-#define CPC_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
-#define CPC_UTCL1_CNTL__BYPASS__SHIFT                                                                         0x19
-#define CPC_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
-#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
-#define CPC_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
-#define CPC_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                            0x1d
-#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
-#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
-#define CPC_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
-#define CPC_UTCL1_CNTL__BYPASS_MASK                                                                           0x02000000L
-#define CPC_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
-#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
-#define CPC_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
-#define CPC_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                              0x20000000L
-#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
-//CPF_UTCL1_CNTL
-#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
-#define CPF_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
-#define CPF_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
-#define CPF_UTCL1_CNTL__BYPASS__SHIFT                                                                         0x19
-#define CPF_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
-#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
-#define CPF_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
-#define CPF_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                            0x1d
-#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
-#define CPF_UTCL1_CNTL__FORCE_NO_EXE__SHIFT                                                                   0x1f
-#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
-#define CPF_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
-#define CPF_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
-#define CPF_UTCL1_CNTL__BYPASS_MASK                                                                           0x02000000L
-#define CPF_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
-#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
-#define CPF_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
-#define CPF_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                              0x20000000L
-#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
-#define CPF_UTCL1_CNTL__FORCE_NO_EXE_MASK                                                                     0x80000000L
-//CP_AQL_SMM_STATUS
-#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM__SHIFT                                                               0x0
-#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM_MASK                                                                 0xFFFFFFFFL
-//CP_RB0_BASE
-#define CP_RB0_BASE__RB_BASE__SHIFT                                                                           0x0
-#define CP_RB0_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
-//CP_RB_BASE
-#define CP_RB_BASE__RB_BASE__SHIFT                                                                            0x0
-#define CP_RB_BASE__RB_BASE_MASK                                                                              0xFFFFFFFFL
-//CP_RB0_CNTL
-#define CP_RB0_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
-#define CP_RB0_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
-#define CP_RB0_CNTL__BUF_SWAP__SHIFT                                                                          0x11
-#define CP_RB0_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
-#define CP_RB0_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
-#define CP_RB0_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
-#define CP_RB0_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
-#define CP_RB0_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
-#define CP_RB0_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
-#define CP_RB0_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
-#define CP_RB0_CNTL__BUF_SWAP_MASK                                                                            0x00060000L
-#define CP_RB0_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
-#define CP_RB0_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
-#define CP_RB0_CNTL__CACHE_POLICY_MASK                                                                        0x01000000L
-#define CP_RB0_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
-#define CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
-//CP_RB_CNTL
-#define CP_RB_CNTL__RB_BUFSZ__SHIFT                                                                           0x0
-#define CP_RB_CNTL__RB_BLKSZ__SHIFT                                                                           0x8
-#define CP_RB_CNTL__MIN_AVAILSZ__SHIFT                                                                        0x14
-#define CP_RB_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                     0x16
-#define CP_RB_CNTL__CACHE_POLICY__SHIFT                                                                       0x18
-#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT                                                                       0x1b
-#define CP_RB_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                     0x1f
-#define CP_RB_CNTL__RB_BUFSZ_MASK                                                                             0x0000003FL
-#define CP_RB_CNTL__RB_BLKSZ_MASK                                                                             0x00003F00L
-#define CP_RB_CNTL__MIN_AVAILSZ_MASK                                                                          0x00300000L
-#define CP_RB_CNTL__MIN_IB_AVAILSZ_MASK                                                                       0x00C00000L
-#define CP_RB_CNTL__CACHE_POLICY_MASK                                                                         0x01000000L
-#define CP_RB_CNTL__RB_NO_UPDATE_MASK                                                                         0x08000000L
-#define CP_RB_CNTL__RB_RPTR_WR_ENA_MASK                                                                       0x80000000L
-//CP_RB_RPTR_WR
-#define CP_RB_RPTR_WR__RB_RPTR_WR__SHIFT                                                                      0x0
-#define CP_RB_RPTR_WR__RB_RPTR_WR_MASK                                                                        0x000FFFFFL
-//CP_RB0_RPTR_ADDR
-#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
-#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
-//CP_RB_RPTR_ADDR
-#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                  0x2
-#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                    0xFFFFFFFCL
-//CP_RB0_RPTR_ADDR_HI
-#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
-#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
-//CP_RB_RPTR_ADDR_HI
-#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                            0x0
-#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                              0x0000FFFFL
-//CP_RB0_BUFSZ_MASK
-#define CP_RB0_BUFSZ_MASK__DATA__SHIFT                                                                        0x0
-#define CP_RB0_BUFSZ_MASK__DATA_MASK                                                                          0x000FFFFFL
-//CP_RB_BUFSZ_MASK
-#define CP_RB_BUFSZ_MASK__DATA__SHIFT                                                                         0x0
-#define CP_RB_BUFSZ_MASK__DATA_MASK                                                                           0x000FFFFFL
-//CP_RB_WPTR_POLL_ADDR_LO
-#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO__SHIFT                                                  0x2
-#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO_MASK                                                    0xFFFFFFFCL
-//CP_RB_WPTR_POLL_ADDR_HI
-#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI__SHIFT                                                  0x0
-#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI_MASK                                                    0x0000FFFFL
-//GC_PRIV_MODE
-//CP_INT_CNTL
-#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                      0xb
-#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                           0xe
-#define CP_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                    0x10
-#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                       0x11
-#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE__SHIFT                                                               0x12
-#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE__SHIFT                                                              0x13
-#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE__SHIFT                                                             0x14
-#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE__SHIFT                                                               0x15
-#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE__SHIFT                                                             0x16
-#define CP_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                               0x17
-#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                           0x18
-#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                             0x1a
-#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                     0x1b
-#define CP_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                               0x1d
-#define CP_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                               0x1e
-#define CP_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                               0x1f
-#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                        0x00000800L
-#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                             0x00004000L
-#define CP_INT_CNTL__GPF_INT_ENABLE_MASK                                                                      0x00010000L
-#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                         0x00020000L
-#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE_MASK                                                                 0x00040000L
-#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK                                                                0x00080000L
-#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK                                                               0x00100000L
-#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE_MASK                                                                 0x00200000L
-#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE_MASK                                                               0x00400000L
-#define CP_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                 0x00800000L
-#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                             0x01000000L
-#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                               0x04000000L
-#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                       0x08000000L
-#define CP_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                 0x20000000L
-#define CP_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                 0x40000000L
-#define CP_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                 0x80000000L
-//CP_INT_STATUS
-#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                      0xb
-#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT__SHIFT                                                           0xe
-#define CP_INT_STATUS__GPF_INT_STAT__SHIFT                                                                    0x10
-#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                       0x11
-#define CP_INT_STATUS__CMP_BUSY_INT_STAT__SHIFT                                                               0x12
-#define CP_INT_STATUS__CNTX_BUSY_INT_STAT__SHIFT                                                              0x13
-#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT__SHIFT                                                             0x14
-#define CP_INT_STATUS__GFX_IDLE_INT_STAT__SHIFT                                                               0x15
-#define CP_INT_STATUS__PRIV_INSTR_INT_STAT__SHIFT                                                             0x16
-#define CP_INT_STATUS__PRIV_REG_INT_STAT__SHIFT                                                               0x17
-#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT__SHIFT                                                           0x18
-#define CP_INT_STATUS__TIME_STAMP_INT_STAT__SHIFT                                                             0x1a
-#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                                     0x1b
-#define CP_INT_STATUS__GENERIC2_INT_STAT__SHIFT                                                               0x1d
-#define CP_INT_STATUS__GENERIC1_INT_STAT__SHIFT                                                               0x1e
-#define CP_INT_STATUS__GENERIC0_INT_STAT__SHIFT                                                               0x1f
-#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                        0x00000800L
-#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT_MASK                                                             0x00004000L
-#define CP_INT_STATUS__GPF_INT_STAT_MASK                                                                      0x00010000L
-#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                         0x00020000L
-#define CP_INT_STATUS__CMP_BUSY_INT_STAT_MASK                                                                 0x00040000L
-#define CP_INT_STATUS__CNTX_BUSY_INT_STAT_MASK                                                                0x00080000L
-#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT_MASK                                                               0x00100000L
-#define CP_INT_STATUS__GFX_IDLE_INT_STAT_MASK                                                                 0x00200000L
-#define CP_INT_STATUS__PRIV_INSTR_INT_STAT_MASK                                                               0x00400000L
-#define CP_INT_STATUS__PRIV_REG_INT_STAT_MASK                                                                 0x00800000L
-#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT_MASK                                                             0x01000000L
-#define CP_INT_STATUS__TIME_STAMP_INT_STAT_MASK                                                               0x04000000L
-#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT_MASK                                                       0x08000000L
-#define CP_INT_STATUS__GENERIC2_INT_STAT_MASK                                                                 0x20000000L
-#define CP_INT_STATUS__GENERIC1_INT_STAT_MASK                                                                 0x40000000L
-#define CP_INT_STATUS__GENERIC0_INT_STAT_MASK                                                                 0x80000000L
-//CP_DEVICE_ID
-#define CP_DEVICE_ID__DEVICE_ID__SHIFT                                                                        0x0
-#define CP_DEVICE_ID__DEVICE_ID_MASK                                                                          0x000000FFL
-//CP_ME0_PIPE_PRIORITY_CNTS
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
-#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
-//CP_RING_PRIORITY_CNTS
-#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                           0x0
-#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                          0x8
-#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                          0x10
-#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                           0x18
-#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                             0x000000FFL
-#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                            0x0000FF00L
-#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                            0x00FF0000L
-#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                             0xFF000000L
-//CP_ME0_PIPE0_PRIORITY
-#define CP_ME0_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME0_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_RING0_PRIORITY
-#define CP_RING0_PRIORITY__PRIORITY__SHIFT                                                                    0x0
-#define CP_RING0_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
-//CP_ME0_PIPE1_PRIORITY
-#define CP_ME0_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME0_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_RING1_PRIORITY
-#define CP_RING1_PRIORITY__PRIORITY__SHIFT                                                                    0x0
-#define CP_RING1_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
-//CP_ME0_PIPE2_PRIORITY
-#define CP_ME0_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME0_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_RING2_PRIORITY
-#define CP_RING2_PRIORITY__PRIORITY__SHIFT                                                                    0x0
-#define CP_RING2_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
-//CP_FATAL_ERROR
-#define CP_FATAL_ERROR__CPF_FATAL_ERROR__SHIFT                                                                0x0
-#define CP_FATAL_ERROR__CPG_FATAL_ERROR__SHIFT                                                                0x1
-#define CP_FATAL_ERROR__GFX_HALT_PROC__SHIFT                                                                  0x2
-#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR__SHIFT                                                            0x3
-#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN__SHIFT                                                         0x4
-#define CP_FATAL_ERROR__CPF_FATAL_ERROR_MASK                                                                  0x00000001L
-#define CP_FATAL_ERROR__CPG_FATAL_ERROR_MASK                                                                  0x00000002L
-#define CP_FATAL_ERROR__GFX_HALT_PROC_MASK                                                                    0x00000004L
-#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR_MASK                                                              0x00000008L
-#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN_MASK                                                           0x00000010L
-//CP_RB_VMID
-#define CP_RB_VMID__RB0_VMID__SHIFT                                                                           0x0
-#define CP_RB_VMID__RB1_VMID__SHIFT                                                                           0x8
-#define CP_RB_VMID__RB2_VMID__SHIFT                                                                           0x10
-#define CP_RB_VMID__RB0_VMID_MASK                                                                             0x0000000FL
-#define CP_RB_VMID__RB1_VMID_MASK                                                                             0x00000F00L
-#define CP_RB_VMID__RB2_VMID_MASK                                                                             0x000F0000L
-//CP_ME0_PIPE0_VMID
-#define CP_ME0_PIPE0_VMID__VMID__SHIFT                                                                        0x0
-#define CP_ME0_PIPE0_VMID__VMID_MASK                                                                          0x0000000FL
-//CP_ME0_PIPE1_VMID
-#define CP_ME0_PIPE1_VMID__VMID__SHIFT                                                                        0x0
-#define CP_ME0_PIPE1_VMID__VMID_MASK                                                                          0x0000000FL
-//CP_RB0_WPTR
-#define CP_RB0_WPTR__RB_WPTR__SHIFT                                                                           0x0
-#define CP_RB0_WPTR__RB_WPTR_MASK                                                                             0xFFFFFFFFL
-//CP_RB_WPTR
-#define CP_RB_WPTR__RB_WPTR__SHIFT                                                                            0x0
-#define CP_RB_WPTR__RB_WPTR_MASK                                                                              0xFFFFFFFFL
-//CP_RB0_WPTR_HI
-#define CP_RB0_WPTR_HI__RB_WPTR__SHIFT                                                                        0x0
-#define CP_RB0_WPTR_HI__RB_WPTR_MASK                                                                          0xFFFFFFFFL
-//CP_RB_WPTR_HI
-#define CP_RB_WPTR_HI__RB_WPTR__SHIFT                                                                         0x0
-#define CP_RB_WPTR_HI__RB_WPTR_MASK                                                                           0xFFFFFFFFL
-//CP_RB1_WPTR
-#define CP_RB1_WPTR__RB_WPTR__SHIFT                                                                           0x0
-#define CP_RB1_WPTR__RB_WPTR_MASK                                                                             0xFFFFFFFFL
-//CP_RB1_WPTR_HI
-#define CP_RB1_WPTR_HI__RB_WPTR__SHIFT                                                                        0x0
-#define CP_RB1_WPTR_HI__RB_WPTR_MASK                                                                          0xFFFFFFFFL
-//CP_RB2_WPTR
-#define CP_RB2_WPTR__RB_WPTR__SHIFT                                                                           0x0
-#define CP_RB2_WPTR__RB_WPTR_MASK                                                                             0x000FFFFFL
-//CP_RB_DOORBELL_CONTROL
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                      0x1
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                        0x2
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                            0x1e
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                           0x1f
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                        0x00000002L
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                          0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                              0x40000000L
-#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                             0x80000000L
-//CP_RB_DOORBELL_RANGE_LOWER
-#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                               0x2
-#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                 0x0FFFFFFCL
-//CP_RB_DOORBELL_RANGE_UPPER
-#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                               0x2
-#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                 0x0FFFFFFCL
-//CP_MEC_DOORBELL_RANGE_LOWER
-#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                              0x2
-#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                0x0FFFFFFCL
-//CP_MEC_DOORBELL_RANGE_UPPER
-#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                              0x2
-#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                0x0FFFFFFCL
-//CPG_UTCL1_ERROR
-#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
-#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
-//CPC_UTCL1_ERROR
-#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
-#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
-//CP_RB1_BASE
-#define CP_RB1_BASE__RB_BASE__SHIFT                                                                           0x0
-#define CP_RB1_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
-//CP_RB1_CNTL
-#define CP_RB1_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
-#define CP_RB1_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
-#define CP_RB1_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
-#define CP_RB1_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
-#define CP_RB1_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
-#define CP_RB1_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
-#define CP_RB1_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
-#define CP_RB1_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
-#define CP_RB1_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
-#define CP_RB1_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
-#define CP_RB1_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
-#define CP_RB1_CNTL__CACHE_POLICY_MASK                                                                        0x01000000L
-#define CP_RB1_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
-#define CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
-//CP_RB1_RPTR_ADDR
-#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
-#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
-//CP_RB1_RPTR_ADDR_HI
-#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
-#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
-//CP_RB2_BASE
-#define CP_RB2_BASE__RB_BASE__SHIFT                                                                           0x0
-#define CP_RB2_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
-//CP_RB2_CNTL
-#define CP_RB2_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
-#define CP_RB2_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
-#define CP_RB2_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
-#define CP_RB2_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
-#define CP_RB2_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
-#define CP_RB2_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
-#define CP_RB2_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
-#define CP_RB2_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
-#define CP_RB2_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
-#define CP_RB2_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
-#define CP_RB2_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
-#define CP_RB2_CNTL__CACHE_POLICY_MASK                                                                        0x01000000L
-#define CP_RB2_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
-#define CP_RB2_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
-//CP_RB2_RPTR_ADDR
-#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
-#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
-//CP_RB2_RPTR_ADDR_HI
-#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
-#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
-//CP_RB0_ACTIVE
-#define CP_RB0_ACTIVE__ACTIVE__SHIFT                                                                          0x0
-#define CP_RB0_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
-//CP_RB_ACTIVE
-#define CP_RB_ACTIVE__ACTIVE__SHIFT                                                                           0x0
-#define CP_RB_ACTIVE__ACTIVE_MASK                                                                             0x00000001L
-//CP_INT_CNTL_RING0
-#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
-#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
-#define CP_INT_CNTL_RING0__GPF_INT_ENABLE__SHIFT                                                              0x10
-#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
-#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
-#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
-#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
-#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
-#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
-#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
-#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
-#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
-#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
-#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
-#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
-#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
-#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
-#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
-#define CP_INT_CNTL_RING0__GPF_INT_ENABLE_MASK                                                                0x00010000L
-#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
-#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
-#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
-#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
-#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
-#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
-#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
-#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
-#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
-#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
-#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
-#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
-#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
-//CP_INT_CNTL_RING1
-#define CP_INT_CNTL_RING1__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
-#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
-#define CP_INT_CNTL_RING1__GPF_INT_ENABLE__SHIFT                                                              0x10
-#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
-#define CP_INT_CNTL_RING1__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
-#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
-#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
-#define CP_INT_CNTL_RING1__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
-#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
-#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
-#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
-#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
-#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
-#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
-#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
-#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
-#define CP_INT_CNTL_RING1__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
-#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
-#define CP_INT_CNTL_RING1__GPF_INT_ENABLE_MASK                                                                0x00010000L
-#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
-#define CP_INT_CNTL_RING1__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
-#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
-#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
-#define CP_INT_CNTL_RING1__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
-#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
-#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
-#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
-#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
-#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
-#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
-#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
-#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
-//CP_INT_CNTL_RING2
-#define CP_INT_CNTL_RING2__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
-#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
-#define CP_INT_CNTL_RING2__GPF_INT_ENABLE__SHIFT                                                              0x10
-#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
-#define CP_INT_CNTL_RING2__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
-#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
-#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
-#define CP_INT_CNTL_RING2__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
-#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
-#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
-#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
-#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
-#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
-#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
-#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
-#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
-#define CP_INT_CNTL_RING2__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
-#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
-#define CP_INT_CNTL_RING2__GPF_INT_ENABLE_MASK                                                                0x00010000L
-#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
-#define CP_INT_CNTL_RING2__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
-#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
-#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
-#define CP_INT_CNTL_RING2__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
-#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
-#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
-#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
-#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
-#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
-#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
-#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
-#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
-//CP_INT_STATUS_RING0
-#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
-#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
-#define CP_INT_STATUS_RING0__GPF_INT_STAT__SHIFT                                                              0x10
-#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
-#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
-#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT__SHIFT                                                       0x13
-#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
-#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
-#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
-#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT__SHIFT                                                         0x17
-#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
-#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
-#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
-#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT__SHIFT                                                         0x1d
-#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT__SHIFT                                                         0x1e
-#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT__SHIFT                                                         0x1f
-#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
-#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
-#define CP_INT_STATUS_RING0__GPF_INT_STAT_MASK                                                                0x00010000L
-#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
-#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
-#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT_MASK                                                         0x00080000L
-#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
-#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
-#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
-#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
-#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
-#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
-#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
-#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT_MASK                                                           0x20000000L
-#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT_MASK                                                           0x40000000L
-#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT_MASK                                                           0x80000000L
-//CP_INT_STATUS_RING1
-#define CP_INT_STATUS_RING1__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
-#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
-#define CP_INT_STATUS_RING1__GPF_INT_STAT__SHIFT                                                              0x10
-#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
-#define CP_INT_STATUS_RING1__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
-#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT__SHIFT                                                        0x13
-#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
-#define CP_INT_STATUS_RING1__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
-#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
-#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT__SHIFT                                                         0x17
-#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
-#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
-#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
-#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT__SHIFT                                                         0x1d
-#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT__SHIFT                                                         0x1e
-#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT__SHIFT                                                         0x1f
-#define CP_INT_STATUS_RING1__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
-#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
-#define CP_INT_STATUS_RING1__GPF_INT_STAT_MASK                                                                0x00010000L
-#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
-#define CP_INT_STATUS_RING1__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
-#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT_MASK                                                          0x00080000L
-#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
-#define CP_INT_STATUS_RING1__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
-#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
-#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
-#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
-#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
-#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
-#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT_MASK                                                           0x20000000L
-#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT_MASK                                                           0x40000000L
-#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT_MASK                                                           0x80000000L
-//CP_INT_STATUS_RING2
-#define CP_INT_STATUS_RING2__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
-#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
-#define CP_INT_STATUS_RING2__GPF_INT_STAT__SHIFT                                                              0x10
-#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
-#define CP_INT_STATUS_RING2__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
-#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT__SHIFT                                                        0x13
-#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
-#define CP_INT_STATUS_RING2__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
-#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
-#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT__SHIFT                                                         0x17
-#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
-#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
-#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
-#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT__SHIFT                                                         0x1d
-#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT__SHIFT                                                         0x1e
-#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT__SHIFT                                                         0x1f
-#define CP_INT_STATUS_RING2__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
-#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
-#define CP_INT_STATUS_RING2__GPF_INT_STAT_MASK                                                                0x00010000L
-#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
-#define CP_INT_STATUS_RING2__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
-#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT_MASK                                                          0x00080000L
-#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
-#define CP_INT_STATUS_RING2__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
-#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
-#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
-#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
-#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
-#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
-#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT_MASK                                                           0x20000000L
-#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT_MASK                                                           0x40000000L
-#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT_MASK                                                           0x80000000L
-#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                             0x1
-#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT_MASK                                                               0x00000002L
-#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                            0x1
-#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT_MASK                                                              0x00000002L
-#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                            0x1
-#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT_MASK                                                              0x00000002L
-//CP_PWR_CNTL
-#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0__SHIFT                                                            0x0
-#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1__SHIFT                                                            0x1
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0__SHIFT                                                            0x8
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1__SHIFT                                                            0x9
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2__SHIFT                                                            0xa
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3__SHIFT                                                            0xb
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0__SHIFT                                                            0x10
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1__SHIFT                                                            0x11
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2__SHIFT                                                            0x12
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3__SHIFT                                                            0x13
-#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0_MASK                                                              0x00000001L
-#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1_MASK                                                              0x00000002L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0_MASK                                                              0x00000100L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1_MASK                                                              0x00000200L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2_MASK                                                              0x00000400L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3_MASK                                                              0x00000800L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0_MASK                                                              0x00010000L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1_MASK                                                              0x00020000L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2_MASK                                                              0x00040000L
-#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3_MASK                                                              0x00080000L
-//CP_MEM_SLP_CNTL
-#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN__SHIFT                                                                  0x0
-#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN__SHIFT                                                                  0x1
-#define CP_MEM_SLP_CNTL__RESERVED__SHIFT                                                                      0x2
-#define CP_MEM_SLP_CNTL__CP_LS_DS_BUSY_OVERRIDE__SHIFT                                                        0x7
-#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY__SHIFT                                                            0x8
-#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY__SHIFT                                                           0x10
-#define CP_MEM_SLP_CNTL__RESERVED1__SHIFT                                                                     0x18
-#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK                                                                    0x00000001L
-#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN_MASK                                                                    0x00000002L
-#define CP_MEM_SLP_CNTL__RESERVED_MASK                                                                        0x0000007CL
-#define CP_MEM_SLP_CNTL__CP_LS_DS_BUSY_OVERRIDE_MASK                                                          0x00000080L
-#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY_MASK                                                              0x0000FF00L
-#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY_MASK                                                             0x00FF0000L
-#define CP_MEM_SLP_CNTL__RESERVED1_MASK                                                                       0xFF000000L
-//CP_ECC_FIRSTOCCURRENCE
-#define CP_ECC_FIRSTOCCURRENCE__INTERFACE__SHIFT                                                              0x0
-#define CP_ECC_FIRSTOCCURRENCE__CLIENT__SHIFT                                                                 0x4
-#define CP_ECC_FIRSTOCCURRENCE__ME__SHIFT                                                                     0x8
-#define CP_ECC_FIRSTOCCURRENCE__PIPE__SHIFT                                                                   0xa
-#define CP_ECC_FIRSTOCCURRENCE__QUEUE__SHIFT                                                                  0xc
-#define CP_ECC_FIRSTOCCURRENCE__VMID__SHIFT                                                                   0x10
-#define CP_ECC_FIRSTOCCURRENCE__INTERFACE_MASK                                                                0x00000003L
-#define CP_ECC_FIRSTOCCURRENCE__CLIENT_MASK                                                                   0x000000F0L
-#define CP_ECC_FIRSTOCCURRENCE__ME_MASK                                                                       0x00000300L
-#define CP_ECC_FIRSTOCCURRENCE__PIPE_MASK                                                                     0x00000C00L
-#define CP_ECC_FIRSTOCCURRENCE__QUEUE_MASK                                                                    0x00007000L
-#define CP_ECC_FIRSTOCCURRENCE__VMID_MASK                                                                     0x000F0000L
-//CP_ECC_FIRSTOCCURRENCE_RING0
-#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE__SHIFT                                                         0x0
-#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE_MASK                                                           0xFFFFFFFFL
-//CP_ECC_FIRSTOCCURRENCE_RING1
-#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE__SHIFT                                                         0x0
-#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE_MASK                                                           0xFFFFFFFFL
-//CP_ECC_FIRSTOCCURRENCE_RING2
-#define CP_ECC_FIRSTOCCURRENCE_RING2__OBSOLETE__SHIFT                                                         0x0
-#define CP_ECC_FIRSTOCCURRENCE_RING2__OBSOLETE_MASK                                                           0xFFFFFFFFL
-//GB_EDC_MODE
-#define GB_EDC_MODE__FORCE_SEC_ON_DED__SHIFT                                                                  0xf
-#define GB_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                     0x10
-#define GB_EDC_MODE__GATE_FUE__SHIFT                                                                          0x11
-#define GB_EDC_MODE__DED_MODE__SHIFT                                                                          0x14
-#define GB_EDC_MODE__PROP_FED__SHIFT                                                                          0x1d
-#define GB_EDC_MODE__BYPASS__SHIFT                                                                            0x1f
-#define GB_EDC_MODE__FORCE_SEC_ON_DED_MASK                                                                    0x00008000L
-#define GB_EDC_MODE__COUNT_FED_OUT_MASK                                                                       0x00010000L
-#define GB_EDC_MODE__GATE_FUE_MASK                                                                            0x00020000L
-#define GB_EDC_MODE__DED_MODE_MASK                                                                            0x00300000L
-#define GB_EDC_MODE__PROP_FED_MASK                                                                            0x20000000L
-#define GB_EDC_MODE__BYPASS_MASK                                                                              0x80000000L
-//CP_PQ_WPTR_POLL_CNTL
-#define CP_PQ_WPTR_POLL_CNTL__PERIOD__SHIFT                                                                   0x0
-#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT__SHIFT                                                0x1d
-#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE__SHIFT                                                              0x1e
-#define CP_PQ_WPTR_POLL_CNTL__EN__SHIFT                                                                       0x1f
-#define CP_PQ_WPTR_POLL_CNTL__PERIOD_MASK                                                                     0x000000FFL
-#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT_MASK                                                  0x20000000L
-#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE_MASK                                                                0x40000000L
-#define CP_PQ_WPTR_POLL_CNTL__EN_MASK                                                                         0x80000000L
-//CP_PQ_WPTR_POLL_CNTL1
-#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK__SHIFT                                                              0x0
-#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK_MASK                                                                0xFFFFFFFFL
-//CP_ME1_PIPE0_INT_CNTL
-#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME1_PIPE1_INT_CNTL
-#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME1_PIPE2_INT_CNTL
-#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME1_PIPE3_INT_CNTL
-#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME2_PIPE0_INT_CNTL
-#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME2_PIPE0_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME2_PIPE0_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME2_PIPE1_INT_CNTL
-#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME2_PIPE1_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME2_PIPE1_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME2_PIPE2_INT_CNTL
-#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME2_PIPE2_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME2_PIPE2_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME2_PIPE3_INT_CNTL
-#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
-#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
-#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
-#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
-#define CP_ME2_PIPE3_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
-#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
-#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
-#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
-#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
-#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
-#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
-#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
-#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
-#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
-#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
-#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
-#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
-#define CP_ME2_PIPE3_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
-#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
-#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
-#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
-#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
-#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
-#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
-#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
-#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
-//CP_ME1_PIPE0_INT_STATUS
-#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-//CP_ME1_PIPE1_INT_STATUS
-#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-//CP_ME1_PIPE2_INT_STATUS
-#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-//CP_ME1_PIPE3_INT_STATUS
-#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-//CP_ME2_PIPE0_INT_STATUS
-#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME2_PIPE0_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME2_PIPE0_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-//CP_ME2_PIPE1_INT_STATUS
-#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME2_PIPE1_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME2_PIPE1_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-//CP_ME2_PIPE2_INT_STATUS
-#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME2_PIPE2_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME2_PIPE2_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-//CP_ME2_PIPE3_INT_STATUS
-#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
-#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
-#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
-#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
-#define CP_ME2_PIPE3_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
-#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
-#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
-#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
-#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
-#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
-#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
-#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
-#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
-#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
-#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
-#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
-#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
-#define CP_ME2_PIPE3_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
-#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
-#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
-#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
-#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
-#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
-#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
-#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
-#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
-#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
-#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
-#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
-#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
-//CC_GC_EDC_CONFIG
-#define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
-#define CC_GC_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
-//CP_ME1_PIPE_PRIORITY_CNTS
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
-#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
-//CP_ME1_PIPE0_PRIORITY
-#define CP_ME1_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME1_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_ME1_PIPE1_PRIORITY
-#define CP_ME1_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME1_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_ME1_PIPE2_PRIORITY
-#define CP_ME1_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME1_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_ME1_PIPE3_PRIORITY
-#define CP_ME1_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME1_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_ME2_PIPE_PRIORITY_CNTS
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
-#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
-//CP_ME2_PIPE0_PRIORITY
-#define CP_ME2_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME2_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_ME2_PIPE1_PRIORITY
-#define CP_ME2_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME2_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_ME2_PIPE2_PRIORITY
-#define CP_ME2_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME2_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_ME2_PIPE3_PRIORITY
-#define CP_ME2_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
-#define CP_ME2_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
-//CP_CE_PRGRM_CNTR_START
-#define CP_CE_PRGRM_CNTR_START__IP_START__SHIFT                                                               0x0
-#define CP_CE_PRGRM_CNTR_START__IP_START_MASK                                                                 0x000007FFL
-//CP_PFP_PRGRM_CNTR_START
-#define CP_PFP_PRGRM_CNTR_START__IP_START__SHIFT                                                              0x0
-#define CP_PFP_PRGRM_CNTR_START__IP_START_MASK                                                                0x00001FFFL
-//CP_ME_PRGRM_CNTR_START
-#define CP_ME_PRGRM_CNTR_START__IP_START__SHIFT                                                               0x0
-#define CP_ME_PRGRM_CNTR_START__IP_START_MASK                                                                 0x00000FFFL
-//CP_MEC1_PRGRM_CNTR_START
-#define CP_MEC1_PRGRM_CNTR_START__IP_START__SHIFT                                                             0x0
-#define CP_MEC1_PRGRM_CNTR_START__IP_START_MASK                                                               0x0000FFFFL
-//CP_MEC2_PRGRM_CNTR_START
-#define CP_MEC2_PRGRM_CNTR_START__IP_START__SHIFT                                                             0x0
-#define CP_MEC2_PRGRM_CNTR_START__IP_START_MASK                                                               0x0000FFFFL
-//CP_CE_INTR_ROUTINE_START
-#define CP_CE_INTR_ROUTINE_START__IR_START__SHIFT                                                             0x0
-#define CP_CE_INTR_ROUTINE_START__IR_START_MASK                                                               0x000007FFL
-//CP_PFP_INTR_ROUTINE_START
-#define CP_PFP_INTR_ROUTINE_START__IR_START__SHIFT                                                            0x0
-#define CP_PFP_INTR_ROUTINE_START__IR_START_MASK                                                              0x00001FFFL
-//CP_ME_INTR_ROUTINE_START
-#define CP_ME_INTR_ROUTINE_START__IR_START__SHIFT                                                             0x0
-#define CP_ME_INTR_ROUTINE_START__IR_START_MASK                                                               0x00000FFFL
-//CP_MEC1_INTR_ROUTINE_START
-#define CP_MEC1_INTR_ROUTINE_START__IR_START__SHIFT                                                           0x0
-#define CP_MEC1_INTR_ROUTINE_START__IR_START_MASK                                                             0x0000FFFFL
-//CP_MEC2_INTR_ROUTINE_START
-#define CP_MEC2_INTR_ROUTINE_START__IR_START__SHIFT                                                           0x0
-#define CP_MEC2_INTR_ROUTINE_START__IR_START_MASK                                                             0x0000FFFFL
-//CP_CONTEXT_CNTL
-#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX__SHIFT                                                          0x0
-#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX__SHIFT                                                        0x4
-#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX__SHIFT                                                          0x10
-#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX__SHIFT                                                        0x14
-#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX_MASK                                                            0x00000007L
-#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX_MASK                                                          0x00000070L
-#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX_MASK                                                            0x00070000L
-#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX_MASK                                                          0x00700000L
-//CP_MAX_CONTEXT
-#define CP_MAX_CONTEXT__MAX_CONTEXT__SHIFT                                                                    0x0
-#define CP_MAX_CONTEXT__MAX_CONTEXT_MASK                                                                      0x00000007L
-//CP_IQ_WAIT_TIME1
-#define CP_IQ_WAIT_TIME1__IB_OFFLOAD__SHIFT                                                                   0x0
-#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD__SHIFT                                                               0x8
-#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD__SHIFT                                                                  0x10
-#define CP_IQ_WAIT_TIME1__GWS__SHIFT                                                                          0x18
-#define CP_IQ_WAIT_TIME1__IB_OFFLOAD_MASK                                                                     0x000000FFL
-#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD_MASK                                                                 0x0000FF00L
-#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD_MASK                                                                    0x00FF0000L
-#define CP_IQ_WAIT_TIME1__GWS_MASK                                                                            0xFF000000L
-//CP_IQ_WAIT_TIME2
-#define CP_IQ_WAIT_TIME2__QUE_SLEEP__SHIFT                                                                    0x0
-#define CP_IQ_WAIT_TIME2__SCH_WAVE__SHIFT                                                                     0x8
-#define CP_IQ_WAIT_TIME2__SEM_REARM__SHIFT                                                                    0x10
-#define CP_IQ_WAIT_TIME2__DEQ_RETRY__SHIFT                                                                    0x18
-#define CP_IQ_WAIT_TIME2__QUE_SLEEP_MASK                                                                      0x000000FFL
-#define CP_IQ_WAIT_TIME2__SCH_WAVE_MASK                                                                       0x0000FF00L
-#define CP_IQ_WAIT_TIME2__SEM_REARM_MASK                                                                      0x00FF0000L
-#define CP_IQ_WAIT_TIME2__DEQ_RETRY_MASK                                                                      0xFF000000L
-//CP_RB0_BASE_HI
-#define CP_RB0_BASE_HI__RB_BASE_HI__SHIFT                                                                     0x0
-#define CP_RB0_BASE_HI__RB_BASE_HI_MASK                                                                       0x000000FFL
-//CP_RB1_BASE_HI
-#define CP_RB1_BASE_HI__RB_BASE_HI__SHIFT                                                                     0x0
-#define CP_RB1_BASE_HI__RB_BASE_HI_MASK                                                                       0x000000FFL
-//CP_VMID_RESET
-#define CP_VMID_RESET__RESET_REQUEST__SHIFT                                                                   0x0
-#define CP_VMID_RESET__RESET_REQUEST_MASK                                                                     0x0000FFFFL
-//CPC_INT_CNTL
-#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                                      0xc
-#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                                       0xd
-#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                          0xe
-#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                         0xf
-#define CPC_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                   0x10
-#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                      0x11
-#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                              0x17
-#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                          0x18
-#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                            0x1a
-#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                    0x1b
-#define CPC_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                              0x1d
-#define CPC_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                              0x1e
-#define CPC_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                              0x1f
-#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                                        0x00001000L
-#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                         0x00002000L
-#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                            0x00004000L
-#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                           0x00008000L
-#define CPC_INT_CNTL__GPF_INT_ENABLE_MASK                                                                     0x00010000L
-#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                        0x00020000L
-#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                0x00800000L
-#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                            0x01000000L
-#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                              0x04000000L
-#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                      0x08000000L
-#define CPC_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                0x20000000L
-#define CPC_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                0x40000000L
-#define CPC_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                0x80000000L
-//CPC_INT_STATUS
-#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                                    0xc
-#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                                     0xd
-#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                                        0xe
-#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                                       0xf
-#define CPC_INT_STATUS__GPF_INT_STATUS__SHIFT                                                                 0x10
-#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                                    0x11
-#define CPC_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                            0x17
-#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                                        0x18
-#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                          0x1a
-#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                                  0x1b
-#define CPC_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                            0x1d
-#define CPC_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                            0x1e
-#define CPC_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                            0x1f
-#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                                      0x00001000L
-#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                                       0x00002000L
-#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                          0x00004000L
-#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                         0x00008000L
-#define CPC_INT_STATUS__GPF_INT_STATUS_MASK                                                                   0x00010000L
-#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                                      0x00020000L
-#define CPC_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                              0x00800000L
-#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                          0x01000000L
-#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                            0x04000000L
-#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                                    0x08000000L
-#define CPC_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                              0x20000000L
-#define CPC_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                              0x40000000L
-#define CPC_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                              0x80000000L
-//CP_VMID_PREEMPT
-#define CP_VMID_PREEMPT__PREEMPT_REQUEST__SHIFT                                                               0x0
-#define CP_VMID_PREEMPT__VIRT_COMMAND__SHIFT                                                                  0x10
-#define CP_VMID_PREEMPT__PREEMPT_REQUEST_MASK                                                                 0x0000FFFFL
-#define CP_VMID_PREEMPT__VIRT_COMMAND_MASK                                                                    0x000F0000L
-//CPC_INT_CNTX_ID
-#define CPC_INT_CNTX_ID__CNTX_ID__SHIFT                                                                       0x0
-#define CPC_INT_CNTX_ID__CNTX_ID_MASK                                                                         0xFFFFFFFFL
-//CP_PQ_STATUS
-#define CP_PQ_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
-#define CP_PQ_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
-#define CP_PQ_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
-#define CP_PQ_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
-//CP_CPC_IC_BASE_LO
-#define CP_CPC_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
-#define CP_CPC_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
-//CP_CPC_IC_BASE_HI
-#define CP_CPC_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
-#define CP_CPC_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x0000FFFFL
-//CP_CPC_IC_BASE_CNTL
-#define CP_CPC_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
-#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
-#define CP_CPC_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
-#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x01000000L
-//CP_CPC_IC_OP_CNTL
-#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
-#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
-#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
-#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
-#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
-#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
-//CP_MEC1_F32_INT_DIS
-#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT                                                           0x0
-#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT__SHIFT                                                              0x1
-#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT                                                      0x2
-#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT__SHIFT                                                            0x3
-#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT                                                           0x4
-#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT                                                       0x5
-#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT                                                          0x6
-#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT                                                         0x7
-#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT                                                           0x8
-#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT__SHIFT                                                              0x9
-#define CP_MEC1_F32_INT_DIS__GPF_INT_CPF__SHIFT                                                               0xa
-#define CP_MEC1_F32_INT_DIS__GPF_INT_DMA__SHIFT                                                               0xb
-#define CP_MEC1_F32_INT_DIS__GPF_INT_CPC__SHIFT                                                               0xc
-#define CP_MEC1_F32_INT_DIS__EDC_SR_MEM_FED_INT__SHIFT                                                        0xd
-#define CP_MEC1_F32_INT_DIS__QUEUE_MESSAGE_INT__SHIFT                                                         0xe
-#define CP_MEC1_F32_INT_DIS__FATAL_EDC_ERROR_INT__SHIFT                                                       0xf
-#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT_MASK                                                             0x00000001L
-#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT_MASK                                                                0x00000002L
-#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK                                                        0x00000004L
-#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT_MASK                                                              0x00000008L
-#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT_MASK                                                             0x00000010L
-#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK                                                         0x00000020L
-#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT_MASK                                                            0x00000040L
-#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT_MASK                                                           0x00000080L
-#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT_MASK                                                             0x00000100L
-#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT_MASK                                                                0x00000200L
-#define CP_MEC1_F32_INT_DIS__GPF_INT_CPF_MASK                                                                 0x00000400L
-#define CP_MEC1_F32_INT_DIS__GPF_INT_DMA_MASK                                                                 0x00000800L
-#define CP_MEC1_F32_INT_DIS__GPF_INT_CPC_MASK                                                                 0x00001000L
-#define CP_MEC1_F32_INT_DIS__EDC_SR_MEM_FED_INT_MASK                                                          0x00002000L
-#define CP_MEC1_F32_INT_DIS__QUEUE_MESSAGE_INT_MASK                                                           0x00004000L
-#define CP_MEC1_F32_INT_DIS__FATAL_EDC_ERROR_INT_MASK                                                         0x00008000L
-//CP_MEC2_F32_INT_DIS
-#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT                                                           0x0
-#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT__SHIFT                                                              0x1
-#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT                                                      0x2
-#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT__SHIFT                                                            0x3
-#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT                                                           0x4
-#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT                                                       0x5
-#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT                                                          0x6
-#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT                                                         0x7
-#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT                                                           0x8
-#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT__SHIFT                                                              0x9
-#define CP_MEC2_F32_INT_DIS__GPF_INT_CPF__SHIFT                                                               0xa
-#define CP_MEC2_F32_INT_DIS__GPF_INT_DMA__SHIFT                                                               0xb
-#define CP_MEC2_F32_INT_DIS__GPF_INT_CPC__SHIFT                                                               0xc
-#define CP_MEC2_F32_INT_DIS__EDC_SR_MEM_FED_INT__SHIFT                                                        0xd
-#define CP_MEC2_F32_INT_DIS__QUEUE_MESSAGE_INT__SHIFT                                                         0xe
-#define CP_MEC2_F32_INT_DIS__FATAL_EDC_ERROR_INT__SHIFT                                                       0xf
-#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT_MASK                                                             0x00000001L
-#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT_MASK                                                                0x00000002L
-#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK                                                        0x00000004L
-#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT_MASK                                                              0x00000008L
-#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT_MASK                                                             0x00000010L
-#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK                                                         0x00000020L
-#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT_MASK                                                            0x00000040L
-#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT_MASK                                                           0x00000080L
-#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT_MASK                                                             0x00000100L
-#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT_MASK                                                                0x00000200L
-#define CP_MEC2_F32_INT_DIS__GPF_INT_CPF_MASK                                                                 0x00000400L
-#define CP_MEC2_F32_INT_DIS__GPF_INT_DMA_MASK                                                                 0x00000800L
-#define CP_MEC2_F32_INT_DIS__GPF_INT_CPC_MASK                                                                 0x00001000L
-#define CP_MEC2_F32_INT_DIS__EDC_SR_MEM_FED_INT_MASK                                                          0x00002000L
-#define CP_MEC2_F32_INT_DIS__QUEUE_MESSAGE_INT_MASK                                                           0x00004000L
-#define CP_MEC2_F32_INT_DIS__FATAL_EDC_ERROR_INT_MASK                                                         0x00008000L
-//CP_VMID_STATUS
-#define CP_VMID_STATUS__PREEMPT_DE_STATUS__SHIFT                                                              0x0
-#define CP_VMID_STATUS__PREEMPT_CE_STATUS__SHIFT                                                              0x10
-#define CP_VMID_STATUS__PREEMPT_DE_STATUS_MASK                                                                0x0000FFFFL
-#define CP_VMID_STATUS__PREEMPT_CE_STATUS_MASK                                                                0xFFFF0000L
-
-
-// addressBlock: gc_cppdec2
-//CP_RB_DOORBELL_CONTROL_SCH_0
-#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_0__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CONTROL_SCH_1
-#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_1__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CONTROL_SCH_2
-#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_2__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CONTROL_SCH_3
-#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_3__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CONTROL_SCH_4
-#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_4__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CONTROL_SCH_5
-#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_5__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CONTROL_SCH_6
-#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_6__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CONTROL_SCH_7
-#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_OFFSET__SHIFT                                                  0x2
-#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_EN__SHIFT                                                      0x1e
-#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_HIT__SHIFT                                                     0x1f
-#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_OFFSET_MASK                                                    0x0FFFFFFCL
-#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_EN_MASK                                                        0x40000000L
-#define CP_RB_DOORBELL_CONTROL_SCH_7__DOORBELL_HIT_MASK                                                       0x80000000L
-//CP_RB_DOORBELL_CLEAR
-#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE__SHIFT                                                             0x0
-#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR__SHIFT                                             0x8
-#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR__SHIFT                                            0x9
-#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR__SHIFT                                                 0xa
-#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR__SHIFT                                                0xb
-#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR__SHIFT                                                 0xc
-#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR__SHIFT                                                0xd
-#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE_MASK                                                               0x00000007L
-#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR_MASK                                               0x00000100L
-#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR_MASK                                              0x00000200L
-#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR_MASK                                                   0x00000400L
-#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR_MASK                                                  0x00000800L
-#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR_MASK                                                   0x00001000L
-#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR_MASK                                                  0x00002000L
-//CP_GFX_MQD_CONTROL
-#define CP_GFX_MQD_CONTROL__VMID__SHIFT                                                                       0x0
-#define CP_GFX_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
-#define CP_GFX_MQD_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
-#define CP_GFX_MQD_CONTROL__VMID_MASK                                                                         0x0000000FL
-#define CP_GFX_MQD_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
-#define CP_GFX_MQD_CONTROL__CACHE_POLICY_MASK                                                                 0x01000000L
-//CP_GFX_MQD_BASE_ADDR
-#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x2
-#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFCL
-//CP_GFX_MQD_BASE_ADDR_HI
-#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
-#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x0000FFFFL
-//CP_RB_STATUS
-#define CP_RB_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
-#define CP_RB_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
-#define CP_RB_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
-#define CP_RB_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
-//CPG_UTCL1_STATUS
-#define CPG_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
-#define CPG_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
-#define CPG_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
-#define CPG_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
-#define CPG_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
-#define CPG_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
-#define CPG_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
-#define CPG_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
-#define CPG_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
-#define CPG_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
-#define CPG_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
-#define CPG_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
-//CPC_UTCL1_STATUS
-#define CPC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
-#define CPC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
-#define CPC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
-#define CPC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
-#define CPC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
-#define CPC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
-#define CPC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
-#define CPC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
-#define CPC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
-#define CPC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
-#define CPC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
-#define CPC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
-//CPF_UTCL1_STATUS
-#define CPF_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
-#define CPF_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
-#define CPF_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
-#define CPF_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
-#define CPF_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
-#define CPF_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
-#define CPF_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
-#define CPF_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
-#define CPF_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
-#define CPF_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
-#define CPF_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
-#define CPF_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
-//CP_SD_CNTL
-#define CP_SD_CNTL__CPF_EN__SHIFT                                                                             0x0
-#define CP_SD_CNTL__CPG_EN__SHIFT                                                                             0x1
-#define CP_SD_CNTL__CPC_EN__SHIFT                                                                             0x2
-#define CP_SD_CNTL__RLC_EN__SHIFT                                                                             0x3
-#define CP_SD_CNTL__SPI_EN__SHIFT                                                                             0x4
-#define CP_SD_CNTL__WD_EN__SHIFT                                                                              0x5
-#define CP_SD_CNTL__IA_EN__SHIFT                                                                              0x6
-#define CP_SD_CNTL__PA_EN__SHIFT                                                                              0x7
-#define CP_SD_CNTL__RMI_EN__SHIFT                                                                             0x8
-#define CP_SD_CNTL__EA_EN__SHIFT                                                                              0x9
-#define CP_SD_CNTL__CPF_EN_MASK                                                                               0x00000001L
-#define CP_SD_CNTL__CPG_EN_MASK                                                                               0x00000002L
-#define CP_SD_CNTL__CPC_EN_MASK                                                                               0x00000004L
-#define CP_SD_CNTL__RLC_EN_MASK                                                                               0x00000008L
-#define CP_SD_CNTL__SPI_EN_MASK                                                                               0x00000010L
-#define CP_SD_CNTL__WD_EN_MASK                                                                                0x00000020L
-#define CP_SD_CNTL__IA_EN_MASK                                                                                0x00000040L
-#define CP_SD_CNTL__PA_EN_MASK                                                                                0x00000080L
-#define CP_SD_CNTL__RMI_EN_MASK                                                                               0x00000100L
-#define CP_SD_CNTL__EA_EN_MASK                                                                                0x00000200L
-//CP_SOFT_RESET_CNTL
-#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET__SHIFT                                                        0x0
-#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET__SHIFT                                                        0x1
-#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET__SHIFT                                                          0x2
-#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET__SHIFT                                                         0x3
-#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET__SHIFT                                               0x4
-#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET__SHIFT                                                      0x5
-#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET__SHIFT                                                         0x6
-#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET_MASK                                                          0x00000001L
-#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET_MASK                                                          0x00000002L
-#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET_MASK                                                            0x00000004L
-#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET_MASK                                                           0x00000008L
-#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET_MASK                                                 0x00000010L
-#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET_MASK                                                        0x00000020L
-#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET_MASK                                                           0x00000040L
-//CP_CPC_GFX_CNTL
-#define CP_CPC_GFX_CNTL__QUEUEID__SHIFT                                                                       0x0
-#define CP_CPC_GFX_CNTL__PIPEID__SHIFT                                                                        0x3
-#define CP_CPC_GFX_CNTL__MEID__SHIFT                                                                          0x5
-#define CP_CPC_GFX_CNTL__VALID__SHIFT                                                                         0x7
-#define CP_CPC_GFX_CNTL__QUEUEID_MASK                                                                         0x00000007L
-#define CP_CPC_GFX_CNTL__PIPEID_MASK                                                                          0x00000018L
-#define CP_CPC_GFX_CNTL__MEID_MASK                                                                            0x00000060L
-#define CP_CPC_GFX_CNTL__VALID_MASK                                                                           0x00000080L
-
-
-// addressBlock: gc_spipdec
-//SPI_ARB_PRIORITY
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0__SHIFT                                                               0x0
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1__SHIFT                                                               0x3
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2__SHIFT                                                               0x6
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3__SHIFT                                                               0x9
-#define SPI_ARB_PRIORITY__TS0_DUR_MULT__SHIFT                                                                 0xc
-#define SPI_ARB_PRIORITY__TS1_DUR_MULT__SHIFT                                                                 0xe
-#define SPI_ARB_PRIORITY__TS2_DUR_MULT__SHIFT                                                                 0x10
-#define SPI_ARB_PRIORITY__TS3_DUR_MULT__SHIFT                                                                 0x12
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0_MASK                                                                 0x00000007L
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1_MASK                                                                 0x00000038L
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2_MASK                                                                 0x000001C0L
-#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3_MASK                                                                 0x00000E00L
-#define SPI_ARB_PRIORITY__TS0_DUR_MULT_MASK                                                                   0x00003000L
-#define SPI_ARB_PRIORITY__TS1_DUR_MULT_MASK                                                                   0x0000C000L
-#define SPI_ARB_PRIORITY__TS2_DUR_MULT_MASK                                                                   0x00030000L
-#define SPI_ARB_PRIORITY__TS3_DUR_MULT_MASK                                                                   0x000C0000L
-//SPI_ARB_CYCLES_0
-#define SPI_ARB_CYCLES_0__TS0_DURATION__SHIFT                                                                 0x0
-#define SPI_ARB_CYCLES_0__TS1_DURATION__SHIFT                                                                 0x10
-#define SPI_ARB_CYCLES_0__TS0_DURATION_MASK                                                                   0x0000FFFFL
-#define SPI_ARB_CYCLES_0__TS1_DURATION_MASK                                                                   0xFFFF0000L
-//SPI_ARB_CYCLES_1
-#define SPI_ARB_CYCLES_1__TS2_DURATION__SHIFT                                                                 0x0
-#define SPI_ARB_CYCLES_1__TS3_DURATION__SHIFT                                                                 0x10
-#define SPI_ARB_CYCLES_1__TS2_DURATION_MASK                                                                   0x0000FFFFL
-#define SPI_ARB_CYCLES_1__TS3_DURATION_MASK                                                                   0xFFFF0000L
-//SPI_WCL_PIPE_PERCENT_GFX
-#define SPI_WCL_PIPE_PERCENT_GFX__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_GFX__LS_GRP_VALUE__SHIFT                                                         0x7
-#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE__SHIFT                                                         0xc
-#define SPI_WCL_PIPE_PERCENT_GFX__ES_GRP_VALUE__SHIFT                                                         0x11
-#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE__SHIFT                                                         0x16
-#define SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK                                                                  0x0000007FL
-#define SPI_WCL_PIPE_PERCENT_GFX__LS_GRP_VALUE_MASK                                                           0x00000F80L
-#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE_MASK                                                           0x0001F000L
-#define SPI_WCL_PIPE_PERCENT_GFX__ES_GRP_VALUE_MASK                                                           0x003E0000L
-#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE_MASK                                                           0x07C00000L
-//SPI_WCL_PIPE_PERCENT_HP3D
-#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE__SHIFT                                                               0x0
-#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE__SHIFT                                                        0xc
-#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE__SHIFT                                                        0x16
-#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE_MASK                                                                 0x0000007FL
-#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE_MASK                                                          0x0001F000L
-#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE_MASK                                                          0x07C00000L
-//SPI_WCL_PIPE_PERCENT_CS0
-#define SPI_WCL_PIPE_PERCENT_CS0__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS0__VALUE_MASK                                                                  0x7FL
-//SPI_WCL_PIPE_PERCENT_CS1
-#define SPI_WCL_PIPE_PERCENT_CS1__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS1__VALUE_MASK                                                                  0x7FL
-//SPI_WCL_PIPE_PERCENT_CS2
-#define SPI_WCL_PIPE_PERCENT_CS2__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS2__VALUE_MASK                                                                  0x7FL
-//SPI_WCL_PIPE_PERCENT_CS3
-#define SPI_WCL_PIPE_PERCENT_CS3__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS3__VALUE_MASK                                                                  0x7FL
-//SPI_WCL_PIPE_PERCENT_CS4
-#define SPI_WCL_PIPE_PERCENT_CS4__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS4__VALUE_MASK                                                                  0x7FL
-//SPI_WCL_PIPE_PERCENT_CS5
-#define SPI_WCL_PIPE_PERCENT_CS5__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS5__VALUE_MASK                                                                  0x7FL
-//SPI_WCL_PIPE_PERCENT_CS6
-#define SPI_WCL_PIPE_PERCENT_CS6__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS6__VALUE_MASK                                                                  0x7FL
-//SPI_WCL_PIPE_PERCENT_CS7
-#define SPI_WCL_PIPE_PERCENT_CS7__VALUE__SHIFT                                                                0x0
-#define SPI_WCL_PIPE_PERCENT_CS7__VALUE_MASK                                                                  0x7FL
-//SPI_COMPUTE_QUEUE_RESET
-#define SPI_COMPUTE_QUEUE_RESET__RESET__SHIFT                                                                 0x0
-#define SPI_COMPUTE_QUEUE_RESET__RESET_MASK                                                                   0x01L
-//SPI_RESOURCE_RESERVE_CU_0
-#define SPI_RESOURCE_RESERVE_CU_0__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_0__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_0__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_0__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_0__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_0__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_0__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_0__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_1
-#define SPI_RESOURCE_RESERVE_CU_1__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_1__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_1__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_1__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_1__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_1__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_1__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_1__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_2
-#define SPI_RESOURCE_RESERVE_CU_2__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_2__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_2__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_2__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_2__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_2__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_2__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_2__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_3
-#define SPI_RESOURCE_RESERVE_CU_3__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_3__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_3__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_3__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_3__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_3__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_3__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_3__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_4
-#define SPI_RESOURCE_RESERVE_CU_4__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_4__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_4__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_4__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_4__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_4__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_4__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_4__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_5
-#define SPI_RESOURCE_RESERVE_CU_5__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_5__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_5__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_5__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_5__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_5__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_5__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_5__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_6
-#define SPI_RESOURCE_RESERVE_CU_6__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_6__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_6__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_6__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_6__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_6__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_6__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_6__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_7
-#define SPI_RESOURCE_RESERVE_CU_7__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_7__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_7__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_7__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_7__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_7__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_7__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_7__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_8
-#define SPI_RESOURCE_RESERVE_CU_8__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_8__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_8__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_8__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_8__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_8__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_8__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_8__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_CU_9
-#define SPI_RESOURCE_RESERVE_CU_9__VGPR__SHIFT                                                                0x0
-#define SPI_RESOURCE_RESERVE_CU_9__SGPR__SHIFT                                                                0x4
-#define SPI_RESOURCE_RESERVE_CU_9__LDS__SHIFT                                                                 0x8
-#define SPI_RESOURCE_RESERVE_CU_9__WAVES__SHIFT                                                               0xc
-#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS__SHIFT                                                            0xf
-#define SPI_RESOURCE_RESERVE_CU_9__VGPR_MASK                                                                  0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_9__SGPR_MASK                                                                  0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_9__LDS_MASK                                                                   0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_9__WAVES_MASK                                                                 0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS_MASK                                                              0x00078000L
-//SPI_RESOURCE_RESERVE_EN_CU_0
-#define SPI_RESOURCE_RESERVE_EN_CU_0__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_0__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_1
-#define SPI_RESOURCE_RESERVE_EN_CU_1__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_1__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_2
-#define SPI_RESOURCE_RESERVE_EN_CU_2__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_2__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_3
-#define SPI_RESOURCE_RESERVE_EN_CU_3__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_3__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_4
-#define SPI_RESOURCE_RESERVE_EN_CU_4__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_4__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_5
-#define SPI_RESOURCE_RESERVE_EN_CU_5__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_5__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_6
-#define SPI_RESOURCE_RESERVE_EN_CU_6__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_6__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_7
-#define SPI_RESOURCE_RESERVE_EN_CU_7__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_7__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_8
-#define SPI_RESOURCE_RESERVE_EN_CU_8__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_8__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_9
-#define SPI_RESOURCE_RESERVE_EN_CU_9__EN__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK__SHIFT                                                        0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK__SHIFT                                                       0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY__SHIFT                                               0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_9__EN_MASK                                                                 0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK_MASK                                                          0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK_MASK                                                         0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY_MASK                                                 0x01000000L
-//SPI_RESOURCE_RESERVE_CU_10
-#define SPI_RESOURCE_RESERVE_CU_10__VGPR__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_CU_10__SGPR__SHIFT                                                               0x4
-#define SPI_RESOURCE_RESERVE_CU_10__LDS__SHIFT                                                                0x8
-#define SPI_RESOURCE_RESERVE_CU_10__WAVES__SHIFT                                                              0xc
-#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS__SHIFT                                                           0xf
-#define SPI_RESOURCE_RESERVE_CU_10__VGPR_MASK                                                                 0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_10__SGPR_MASK                                                                 0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_10__LDS_MASK                                                                  0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_10__WAVES_MASK                                                                0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS_MASK                                                             0x00078000L
-//SPI_RESOURCE_RESERVE_CU_11
-#define SPI_RESOURCE_RESERVE_CU_11__VGPR__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_CU_11__SGPR__SHIFT                                                               0x4
-#define SPI_RESOURCE_RESERVE_CU_11__LDS__SHIFT                                                                0x8
-#define SPI_RESOURCE_RESERVE_CU_11__WAVES__SHIFT                                                              0xc
-#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS__SHIFT                                                           0xf
-#define SPI_RESOURCE_RESERVE_CU_11__VGPR_MASK                                                                 0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_11__SGPR_MASK                                                                 0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_11__LDS_MASK                                                                  0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_11__WAVES_MASK                                                                0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS_MASK                                                             0x00078000L
-//SPI_RESOURCE_RESERVE_EN_CU_10
-#define SPI_RESOURCE_RESERVE_EN_CU_10__EN__SHIFT                                                              0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK__SHIFT                                                       0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK__SHIFT                                                      0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY__SHIFT                                              0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_10__EN_MASK                                                                0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK_MASK                                                         0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK_MASK                                                        0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_11
-#define SPI_RESOURCE_RESERVE_EN_CU_11__EN__SHIFT                                                              0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK__SHIFT                                                       0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK__SHIFT                                                      0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY__SHIFT                                              0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_11__EN_MASK                                                                0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK_MASK                                                         0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK_MASK                                                        0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
-//SPI_RESOURCE_RESERVE_CU_12
-#define SPI_RESOURCE_RESERVE_CU_12__VGPR__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_CU_12__SGPR__SHIFT                                                               0x4
-#define SPI_RESOURCE_RESERVE_CU_12__LDS__SHIFT                                                                0x8
-#define SPI_RESOURCE_RESERVE_CU_12__WAVES__SHIFT                                                              0xc
-#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS__SHIFT                                                           0xf
-#define SPI_RESOURCE_RESERVE_CU_12__VGPR_MASK                                                                 0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_12__SGPR_MASK                                                                 0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_12__LDS_MASK                                                                  0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_12__WAVES_MASK                                                                0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS_MASK                                                             0x00078000L
-//SPI_RESOURCE_RESERVE_CU_13
-#define SPI_RESOURCE_RESERVE_CU_13__VGPR__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_CU_13__SGPR__SHIFT                                                               0x4
-#define SPI_RESOURCE_RESERVE_CU_13__LDS__SHIFT                                                                0x8
-#define SPI_RESOURCE_RESERVE_CU_13__WAVES__SHIFT                                                              0xc
-#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS__SHIFT                                                           0xf
-#define SPI_RESOURCE_RESERVE_CU_13__VGPR_MASK                                                                 0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_13__SGPR_MASK                                                                 0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_13__LDS_MASK                                                                  0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_13__WAVES_MASK                                                                0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS_MASK                                                             0x00078000L
-//SPI_RESOURCE_RESERVE_CU_14
-#define SPI_RESOURCE_RESERVE_CU_14__VGPR__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_CU_14__SGPR__SHIFT                                                               0x4
-#define SPI_RESOURCE_RESERVE_CU_14__LDS__SHIFT                                                                0x8
-#define SPI_RESOURCE_RESERVE_CU_14__WAVES__SHIFT                                                              0xc
-#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS__SHIFT                                                           0xf
-#define SPI_RESOURCE_RESERVE_CU_14__VGPR_MASK                                                                 0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_14__SGPR_MASK                                                                 0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_14__LDS_MASK                                                                  0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_14__WAVES_MASK                                                                0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS_MASK                                                             0x00078000L
-//SPI_RESOURCE_RESERVE_CU_15
-#define SPI_RESOURCE_RESERVE_CU_15__VGPR__SHIFT                                                               0x0
-#define SPI_RESOURCE_RESERVE_CU_15__SGPR__SHIFT                                                               0x4
-#define SPI_RESOURCE_RESERVE_CU_15__LDS__SHIFT                                                                0x8
-#define SPI_RESOURCE_RESERVE_CU_15__WAVES__SHIFT                                                              0xc
-#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS__SHIFT                                                           0xf
-#define SPI_RESOURCE_RESERVE_CU_15__VGPR_MASK                                                                 0x0000000FL
-#define SPI_RESOURCE_RESERVE_CU_15__SGPR_MASK                                                                 0x000000F0L
-#define SPI_RESOURCE_RESERVE_CU_15__LDS_MASK                                                                  0x00000F00L
-#define SPI_RESOURCE_RESERVE_CU_15__WAVES_MASK                                                                0x00007000L
-#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS_MASK                                                             0x00078000L
-//SPI_RESOURCE_RESERVE_EN_CU_12
-#define SPI_RESOURCE_RESERVE_EN_CU_12__EN__SHIFT                                                              0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK__SHIFT                                                       0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK__SHIFT                                                      0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_12__RESERVE_SPACE_ONLY__SHIFT                                              0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_12__EN_MASK                                                                0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK_MASK                                                         0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK_MASK                                                        0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_12__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_13
-#define SPI_RESOURCE_RESERVE_EN_CU_13__EN__SHIFT                                                              0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK__SHIFT                                                       0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK__SHIFT                                                      0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_13__RESERVE_SPACE_ONLY__SHIFT                                              0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_13__EN_MASK                                                                0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK_MASK                                                         0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK_MASK                                                        0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_13__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_14
-#define SPI_RESOURCE_RESERVE_EN_CU_14__EN__SHIFT                                                              0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK__SHIFT                                                       0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK__SHIFT                                                      0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_14__RESERVE_SPACE_ONLY__SHIFT                                              0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_14__EN_MASK                                                                0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK_MASK                                                         0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK_MASK                                                        0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_14__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
-//SPI_RESOURCE_RESERVE_EN_CU_15
-#define SPI_RESOURCE_RESERVE_EN_CU_15__EN__SHIFT                                                              0x0
-#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK__SHIFT                                                       0x1
-#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK__SHIFT                                                      0x10
-#define SPI_RESOURCE_RESERVE_EN_CU_15__RESERVE_SPACE_ONLY__SHIFT                                              0x18
-#define SPI_RESOURCE_RESERVE_EN_CU_15__EN_MASK                                                                0x00000001L
-#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK_MASK                                                         0x0000FFFEL
-#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK_MASK                                                        0x00FF0000L
-#define SPI_RESOURCE_RESERVE_EN_CU_15__RESERVE_SPACE_ONLY_MASK                                                0x01000000L
-//SPI_COMPUTE_WF_CTX_SAVE
-#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE__SHIFT                                                              0x0
-#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN__SHIFT                                                      0x1
-#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN__SHIFT                                                     0x2
-#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY__SHIFT                                                          0x1e
-#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY__SHIFT                                                             0x1f
-#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE_MASK                                                                0x00000001L
-#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN_MASK                                                        0x00000002L
-#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN_MASK                                                       0x00000004L
-#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY_MASK                                                            0x40000000L
-#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY_MASK                                                               0x80000000L
-//SPI_ARB_CNTL_0
-#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT__SHIFT                                                                 0x0
-#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT__SHIFT                                                                 0x4
-#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT__SHIFT                                                                 0x8
-#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT_MASK                                                                   0x0000000FL
-#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT_MASK                                                                   0x000000F0L
-#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT_MASK                                                                   0x00000F00L
-
-
-// addressBlock: gc_cpphqddec
-//CP_HQD_GFX_CONTROL
-#define CP_HQD_GFX_CONTROL__MESSAGE__SHIFT                                                                    0x0
-#define CP_HQD_GFX_CONTROL__MISC__SHIFT                                                                       0x4
-#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT                                                          0xf
-#define CP_HQD_GFX_CONTROL__MESSAGE_MASK                                                                      0x0000000FL
-#define CP_HQD_GFX_CONTROL__MISC_MASK                                                                         0x00007FF0L
-#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN_MASK                                                            0x00008000L
-//CP_HQD_GFX_STATUS
-#define CP_HQD_GFX_STATUS__STATUS__SHIFT                                                                      0x0
-#define CP_HQD_GFX_STATUS__STATUS_MASK                                                                        0x0000FFFFL
-//CP_HPD_ROQ_OFFSETS
-#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET__SHIFT                                                                  0x0
-#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET__SHIFT                                                                  0x8
-#define CP_HPD_ROQ_OFFSETS__IB_OFFSET__SHIFT                                                                  0x10
-#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET_MASK                                                                    0x00000007L
-#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET_MASK                                                                    0x00003F00L
-#define CP_HPD_ROQ_OFFSETS__IB_OFFSET_MASK                                                                    0x003F0000L
-//CP_HPD_STATUS0
-#define CP_HPD_STATUS0__QUEUE_STATE__SHIFT                                                                    0x0
-#define CP_HPD_STATUS0__MAPPED_QUEUE__SHIFT                                                                   0x5
-#define CP_HPD_STATUS0__QUEUE_AVAILABLE__SHIFT                                                                0x8
-#define CP_HPD_STATUS0__FETCHING_MQD__SHIFT                                                                   0x10
-#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB__SHIFT                                                           0x11
-#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ__SHIFT                                                             0x12
-#define CP_HPD_STATUS0__FORCE_QUEUE_STATE__SHIFT                                                              0x14
-#define CP_HPD_STATUS0__FORCE_QUEUE__SHIFT                                                                    0x1f
-#define CP_HPD_STATUS0__QUEUE_STATE_MASK                                                                      0x0000001FL
-#define CP_HPD_STATUS0__MAPPED_QUEUE_MASK                                                                     0x000000E0L
-#define CP_HPD_STATUS0__QUEUE_AVAILABLE_MASK                                                                  0x0000FF00L
-#define CP_HPD_STATUS0__FETCHING_MQD_MASK                                                                     0x00010000L
-#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB_MASK                                                             0x00020000L
-#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ_MASK                                                               0x00040000L
-#define CP_HPD_STATUS0__FORCE_QUEUE_STATE_MASK                                                                0x01F00000L
-#define CP_HPD_STATUS0__FORCE_QUEUE_MASK                                                                      0x80000000L
-//CP_HPD_UTCL1_CNTL
-#define CP_HPD_UTCL1_CNTL__SELECT__SHIFT                                                                      0x0
-#define CP_HPD_UTCL1_CNTL__SELECT_MASK                                                                        0x0000000FL
-//CP_HPD_UTCL1_ERROR
-#define CP_HPD_UTCL1_ERROR__ADDR_HI__SHIFT                                                                    0x0
-#define CP_HPD_UTCL1_ERROR__TYPE__SHIFT                                                                       0x10
-#define CP_HPD_UTCL1_ERROR__VMID__SHIFT                                                                       0x14
-#define CP_HPD_UTCL1_ERROR__ADDR_HI_MASK                                                                      0x0000FFFFL
-#define CP_HPD_UTCL1_ERROR__TYPE_MASK                                                                         0x00010000L
-#define CP_HPD_UTCL1_ERROR__VMID_MASK                                                                         0x00F00000L
-//CP_HPD_UTCL1_ERROR_ADDR
-#define CP_HPD_UTCL1_ERROR_ADDR__ADDR__SHIFT                                                                  0xc
-#define CP_HPD_UTCL1_ERROR_ADDR__ADDR_MASK                                                                    0xFFFFF000L
-//CP_MQD_BASE_ADDR
-#define CP_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                    0x2
-#define CP_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                      0xFFFFFFFCL
-//CP_MQD_BASE_ADDR_HI
-#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                              0x0
-#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                                0x0000FFFFL
-//CP_HQD_ACTIVE
-#define CP_HQD_ACTIVE__ACTIVE__SHIFT                                                                          0x0
-#define CP_HQD_ACTIVE__BUSY_GATE__SHIFT                                                                       0x1
-#define CP_HQD_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
-#define CP_HQD_ACTIVE__BUSY_GATE_MASK                                                                         0x00000002L
-//CP_HQD_VMID
-#define CP_HQD_VMID__VMID__SHIFT                                                                              0x0
-#define CP_HQD_VMID__IB_VMID__SHIFT                                                                           0x8
-#define CP_HQD_VMID__VQID__SHIFT                                                                              0x10
-#define CP_HQD_VMID__VMID_MASK                                                                                0x0000000FL
-#define CP_HQD_VMID__IB_VMID_MASK                                                                             0x00000F00L
-#define CP_HQD_VMID__VQID_MASK                                                                                0x03FF0000L
-//CP_HQD_PERSISTENT_STATE
-#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ__SHIFT                                                           0x0
-#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT                                                          0x8
-#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN__SHIFT                                                     0x15
-#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN__SHIFT                                                      0x16
-#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN__SHIFT                                                      0x17
-#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN__SHIFT                                                     0x18
-#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN__SHIFT                                                      0x19
-#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN__SHIFT                                                     0x1a
-#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN__SHIFT                                                  0x1b
-#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE__SHIFT                                                        0x1c
-#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES__SHIFT                                                        0x1d
-#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT                                                          0x1e
-#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE__SHIFT                                                           0x1f
-#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK                                                             0x00000001L
-#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE_MASK                                                            0x0003FF00L
-#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN_MASK                                                       0x00200000L
-#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN_MASK                                                        0x00400000L
-#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN_MASK                                                        0x00800000L
-#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN_MASK                                                       0x01000000L
-#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN_MASK                                                        0x02000000L
-#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN_MASK                                                       0x04000000L
-#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN_MASK                                                    0x08000000L
-#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE_MASK                                                          0x10000000L
-#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES_MASK                                                          0x20000000L
-#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE_MASK                                                            0x40000000L
-#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE_MASK                                                             0x80000000L
-//CP_HQD_PIPE_PRIORITY
-#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY__SHIFT                                                            0x0
-#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY_MASK                                                              0x00000003L
-//CP_HQD_QUEUE_PRIORITY
-#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT                                                          0x0
-#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK                                                            0x0000000FL
-//CP_HQD_QUANTUM
-#define CP_HQD_QUANTUM__QUANTUM_EN__SHIFT                                                                     0x0
-#define CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT                                                                  0x4
-#define CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT                                                               0x8
-#define CP_HQD_QUANTUM__QUANTUM_ACTIVE__SHIFT                                                                 0x1f
-#define CP_HQD_QUANTUM__QUANTUM_EN_MASK                                                                       0x00000001L
-#define CP_HQD_QUANTUM__QUANTUM_SCALE_MASK                                                                    0x00000010L
-#define CP_HQD_QUANTUM__QUANTUM_DURATION_MASK                                                                 0x00003F00L
-#define CP_HQD_QUANTUM__QUANTUM_ACTIVE_MASK                                                                   0x80000000L
-//CP_HQD_PQ_BASE
-#define CP_HQD_PQ_BASE__ADDR__SHIFT                                                                           0x0
-#define CP_HQD_PQ_BASE__ADDR_MASK                                                                             0xFFFFFFFFL
-//CP_HQD_PQ_BASE_HI
-#define CP_HQD_PQ_BASE_HI__ADDR_HI__SHIFT                                                                     0x0
-#define CP_HQD_PQ_BASE_HI__ADDR_HI_MASK                                                                       0x000000FFL
-//CP_HQD_PQ_RPTR
-#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
-#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET_MASK                                                                  0xFFFFFFFFL
-//CP_HQD_PQ_RPTR_REPORT_ADDR
-#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR__SHIFT                                                   0x2
-#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR_MASK                                                     0xFFFFFFFCL
-//CP_HQD_PQ_RPTR_REPORT_ADDR_HI
-#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI__SHIFT                                             0x0
-#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI_MASK                                               0x0000FFFFL
-//CP_HQD_PQ_WPTR_POLL_ADDR
-#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR__SHIFT                                                            0x3
-#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR_MASK                                                              0xFFFFFFF8L
-//CP_HQD_PQ_WPTR_POLL_ADDR_HI
-#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI__SHIFT                                                      0x0
-#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI_MASK                                                        0x0000FFFFL
-//CP_HQD_PQ_DOORBELL_CONTROL
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE__SHIFT                                                      0x0
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                  0x1
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                    0x2
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE__SHIFT                                                    0x1c
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT__SHIFT                                                  0x1d
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                        0x1e
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                       0x1f
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE_MASK                                                        0x00000001L
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                    0x00000002L
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                      0x0FFFFFFCL
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK                                                      0x10000000L
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT_MASK                                                    0x20000000L
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                          0x40000000L
-#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                         0x80000000L
-//CP_HQD_PQ_CONTROL
-#define CP_HQD_PQ_CONTROL__QUEUE_SIZE__SHIFT                                                                  0x0
-#define CP_HQD_PQ_CONTROL__WPTR_CARRY__SHIFT                                                                  0x6
-#define CP_HQD_PQ_CONTROL__RPTR_CARRY__SHIFT                                                                  0x7
-#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT                                                             0x8
-#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT                                                               0xe
-#define CP_HQD_PQ_CONTROL__PQ_EMPTY__SHIFT                                                                    0xf
-#define CP_HQD_PQ_CONTROL__WPP_CLAMP_EN__SHIFT                                                                0x10
-#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP__SHIFT                                                                 0x11
-#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE__SHIFT                                                              0x14
-#define CP_HQD_PQ_CONTROL__EXE_DISABLE__SHIFT                                                                 0x17
-#define CP_HQD_PQ_CONTROL__CACHE_POLICY__SHIFT                                                                0x18
-#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT                                                             0x19
-#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR__SHIFT                                                              0x1b
-#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH__SHIFT                                                              0x1c
-#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP__SHIFT                                                              0x1d
-#define CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT                                                                  0x1e
-#define CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT                                                                   0x1f
-#define CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK                                                                    0x0000003FL
-#define CP_HQD_PQ_CONTROL__WPTR_CARRY_MASK                                                                    0x00000040L
-#define CP_HQD_PQ_CONTROL__RPTR_CARRY_MASK                                                                    0x00000080L
-#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK                                                               0x00003F00L
-#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN_MASK                                                                 0x00004000L
-#define CP_HQD_PQ_CONTROL__PQ_EMPTY_MASK                                                                      0x00008000L
-#define CP_HQD_PQ_CONTROL__WPP_CLAMP_EN_MASK                                                                  0x00010000L
-#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP_MASK                                                                   0x00060000L
-#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE_MASK                                                                0x00300000L
-#define CP_HQD_PQ_CONTROL__EXE_DISABLE_MASK                                                                   0x00800000L
-#define CP_HQD_PQ_CONTROL__CACHE_POLICY_MASK                                                                  0x01000000L
-#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR_MASK                                                               0x06000000L
-#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK                                                                0x08000000L
-#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK                                                                0x10000000L
-#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP_MASK                                                                0x20000000L
-#define CP_HQD_PQ_CONTROL__PRIV_STATE_MASK                                                                    0x40000000L
-#define CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK                                                                     0x80000000L
-//CP_HQD_IB_BASE_ADDR
-#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR__SHIFT                                                              0x2
-#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR_MASK                                                                0xFFFFFFFCL
-//CP_HQD_IB_BASE_ADDR_HI
-#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI__SHIFT                                                        0x0
-#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI_MASK                                                          0x0000FFFFL
-//CP_HQD_IB_RPTR
-#define CP_HQD_IB_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
-#define CP_HQD_IB_RPTR__CONSUMED_OFFSET_MASK                                                                  0x000FFFFFL
-//CP_HQD_IB_CONTROL
-#define CP_HQD_IB_CONTROL__IB_SIZE__SHIFT                                                                     0x0
-#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT                                                           0x14
-#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT                                                              0x17
-#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY__SHIFT                                                             0x18
-#define CP_HQD_IB_CONTROL__PROCESSING_IB__SHIFT                                                               0x1f
-#define CP_HQD_IB_CONTROL__IB_SIZE_MASK                                                                       0x000FFFFFL
-#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE_MASK                                                             0x00300000L
-#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE_MASK                                                                0x00800000L
-#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY_MASK                                                               0x01000000L
-#define CP_HQD_IB_CONTROL__PROCESSING_IB_MASK                                                                 0x80000000L
-//CP_HQD_IQ_TIMER
-#define CP_HQD_IQ_TIMER__WAIT_TIME__SHIFT                                                                     0x0
-#define CP_HQD_IQ_TIMER__RETRY_TYPE__SHIFT                                                                    0x8
-#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT                                                              0xb
-#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT                                                                0xc
-#define CP_HQD_IQ_TIMER__CLOCK_COUNT__SHIFT                                                                   0xe
-#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE__SHIFT                                                                0x10
-#define CP_HQD_IQ_TIMER__QUANTUM_TIMER__SHIFT                                                                 0x16
-#define CP_HQD_IQ_TIMER__EXE_DISABLE__SHIFT                                                                   0x17
-#define CP_HQD_IQ_TIMER__CACHE_POLICY__SHIFT                                                                  0x18
-#define CP_HQD_IQ_TIMER__QUEUE_TYPE__SHIFT                                                                    0x19
-#define CP_HQD_IQ_TIMER__REARM_TIMER__SHIFT                                                                   0x1c
-#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN__SHIFT                                                                 0x1d
-#define CP_HQD_IQ_TIMER__PROCESSING_IQ__SHIFT                                                                 0x1e
-#define CP_HQD_IQ_TIMER__ACTIVE__SHIFT                                                                        0x1f
-#define CP_HQD_IQ_TIMER__WAIT_TIME_MASK                                                                       0x000000FFL
-#define CP_HQD_IQ_TIMER__RETRY_TYPE_MASK                                                                      0x00000700L
-#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE_MASK                                                                0x00000800L
-#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK                                                                  0x00003000L
-#define CP_HQD_IQ_TIMER__CLOCK_COUNT_MASK                                                                     0x0000C000L
-#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE_MASK                                                                  0x003F0000L
-#define CP_HQD_IQ_TIMER__QUANTUM_TIMER_MASK                                                                   0x00400000L
-#define CP_HQD_IQ_TIMER__EXE_DISABLE_MASK                                                                     0x00800000L
-#define CP_HQD_IQ_TIMER__CACHE_POLICY_MASK                                                                    0x01000000L
-#define CP_HQD_IQ_TIMER__QUEUE_TYPE_MASK                                                                      0x02000000L
-#define CP_HQD_IQ_TIMER__REARM_TIMER_MASK                                                                     0x10000000L
-#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN_MASK                                                                   0x20000000L
-#define CP_HQD_IQ_TIMER__PROCESSING_IQ_MASK                                                                   0x40000000L
-#define CP_HQD_IQ_TIMER__ACTIVE_MASK                                                                          0x80000000L
-//CP_HQD_IQ_RPTR
-#define CP_HQD_IQ_RPTR__OFFSET__SHIFT                                                                         0x0
-#define CP_HQD_IQ_RPTR__OFFSET_MASK                                                                           0x0000003FL
-//CP_HQD_DEQUEUE_REQUEST
-#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT                                                            0x0
-#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT                                                            0x4
-#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT__SHIFT                                                            0x8
-#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN__SHIFT                                                         0x9
-#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN__SHIFT                                                         0xa
-#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK                                                              0x00000007L
-#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK                                                              0x00000010L
-#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT_MASK                                                              0x00000100L
-#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN_MASK                                                           0x00000200L
-#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN_MASK                                                           0x00000400L
-//CP_HQD_DMA_OFFLOAD
-#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                0x0
-#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_MASK                                                                  0x00000001L
-//CP_HQD_OFFLOAD
-#define CP_HQD_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                    0x0
-#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN__SHIFT                                                                 0x1
-#define CP_HQD_OFFLOAD__AQL_OFFLOAD__SHIFT                                                                    0x2
-#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN__SHIFT                                                                 0x3
-#define CP_HQD_OFFLOAD__EOP_OFFLOAD__SHIFT                                                                    0x4
-#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN__SHIFT                                                                 0x5
-#define CP_HQD_OFFLOAD__DMA_OFFLOAD_MASK                                                                      0x00000001L
-#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN_MASK                                                                   0x00000002L
-#define CP_HQD_OFFLOAD__AQL_OFFLOAD_MASK                                                                      0x00000004L
-#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN_MASK                                                                   0x00000008L
-#define CP_HQD_OFFLOAD__EOP_OFFLOAD_MASK                                                                      0x00000010L
-#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN_MASK                                                                   0x00000020L
-//CP_HQD_SEMA_CMD
-#define CP_HQD_SEMA_CMD__RETRY__SHIFT                                                                         0x0
-#define CP_HQD_SEMA_CMD__RESULT__SHIFT                                                                        0x1
-#define CP_HQD_SEMA_CMD__RETRY_MASK                                                                           0x00000001L
-#define CP_HQD_SEMA_CMD__RESULT_MASK                                                                          0x00000006L
-//CP_HQD_MSG_TYPE
-#define CP_HQD_MSG_TYPE__ACTION__SHIFT                                                                        0x0
-#define CP_HQD_MSG_TYPE__SAVE_STATE__SHIFT                                                                    0x4
-#define CP_HQD_MSG_TYPE__ACTION_MASK                                                                          0x00000007L
-#define CP_HQD_MSG_TYPE__SAVE_STATE_MASK                                                                      0x00000070L
-//CP_HQD_ATOMIC0_PREOP_LO
-#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO__SHIFT                                                      0x0
-#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO_MASK                                                        0xFFFFFFFFL
-//CP_HQD_ATOMIC0_PREOP_HI
-#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI__SHIFT                                                      0x0
-#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI_MASK                                                        0xFFFFFFFFL
-//CP_HQD_ATOMIC1_PREOP_LO
-#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO__SHIFT                                                      0x0
-#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO_MASK                                                        0xFFFFFFFFL
-//CP_HQD_ATOMIC1_PREOP_HI
-#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI__SHIFT                                                      0x0
-#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI_MASK                                                        0xFFFFFFFFL
-//CP_HQD_HQ_SCHEDULER0
-#define CP_HQD_HQ_SCHEDULER0__SCHEDULER__SHIFT                                                                0x0
-#define CP_HQD_HQ_SCHEDULER0__SCHEDULER_MASK                                                                  0xFFFFFFFFL
-//CP_HQD_HQ_STATUS0
-#define CP_HQD_HQ_STATUS0__DEQUEUE_STATUS__SHIFT                                                              0x0
-#define CP_HQD_HQ_STATUS0__DEQUEUE_RETRY_CNT__SHIFT                                                           0x2
-#define CP_HQD_HQ_STATUS0__RSV_6_4__SHIFT                                                                     0x4
-#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT__SHIFT                                                            0x7
-#define CP_HQD_HQ_STATUS0__TCL2_DIRTY__SHIFT                                                                  0x8
-#define CP_HQD_HQ_STATUS0__PG_ACTIVATED__SHIFT                                                                0x9
-#define CP_HQD_HQ_STATUS0__RSVR_29_10__SHIFT                                                                  0xa
-#define CP_HQD_HQ_STATUS0__QUEUE_IDLE__SHIFT                                                                  0x1e
-#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN__SHIFT                                                           0x1f
-#define CP_HQD_HQ_STATUS0__DEQUEUE_STATUS_MASK                                                                0x00000003L
-#define CP_HQD_HQ_STATUS0__DEQUEUE_RETRY_CNT_MASK                                                             0x0000000CL
-#define CP_HQD_HQ_STATUS0__RSV_6_4_MASK                                                                       0x00000070L
-#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT_MASK                                                              0x00000080L
-#define CP_HQD_HQ_STATUS0__TCL2_DIRTY_MASK                                                                    0x00000100L
-#define CP_HQD_HQ_STATUS0__PG_ACTIVATED_MASK                                                                  0x00000200L
-#define CP_HQD_HQ_STATUS0__RSVR_29_10_MASK                                                                    0x3FFFFC00L
-#define CP_HQD_HQ_STATUS0__QUEUE_IDLE_MASK                                                                    0x40000000L
-#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN_MASK                                                             0x80000000L
-//CP_HQD_HQ_CONTROL0
-#define CP_HQD_HQ_CONTROL0__CONTROL__SHIFT                                                                    0x0
-#define CP_HQD_HQ_CONTROL0__CONTROL_MASK                                                                      0xFFFFFFFFL
-//CP_HQD_HQ_SCHEDULER1
-#define CP_HQD_HQ_SCHEDULER1__SCHEDULER__SHIFT                                                                0x0
-#define CP_HQD_HQ_SCHEDULER1__SCHEDULER_MASK                                                                  0xFFFFFFFFL
-//CP_MQD_CONTROL
-#define CP_MQD_CONTROL__VMID__SHIFT                                                                           0x0
-#define CP_MQD_CONTROL__PRIV_STATE__SHIFT                                                                     0x8
-#define CP_MQD_CONTROL__PROCESSING_MQD__SHIFT                                                                 0xc
-#define CP_MQD_CONTROL__PROCESSING_MQD_EN__SHIFT                                                              0xd
-#define CP_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                    0x17
-#define CP_MQD_CONTROL__CACHE_POLICY__SHIFT                                                                   0x18
-#define CP_MQD_CONTROL__VMID_MASK                                                                             0x0000000FL
-#define CP_MQD_CONTROL__PRIV_STATE_MASK                                                                       0x00000100L
-#define CP_MQD_CONTROL__PROCESSING_MQD_MASK                                                                   0x00001000L
-#define CP_MQD_CONTROL__PROCESSING_MQD_EN_MASK                                                                0x00002000L
-#define CP_MQD_CONTROL__EXE_DISABLE_MASK                                                                      0x00800000L
-#define CP_MQD_CONTROL__CACHE_POLICY_MASK                                                                     0x01000000L
-//CP_HQD_HQ_STATUS1
-#define CP_HQD_HQ_STATUS1__STATUS__SHIFT                                                                      0x0
-#define CP_HQD_HQ_STATUS1__STATUS_MASK                                                                        0xFFFFFFFFL
-//CP_HQD_HQ_CONTROL1
-#define CP_HQD_HQ_CONTROL1__CONTROL__SHIFT                                                                    0x0
-#define CP_HQD_HQ_CONTROL1__CONTROL_MASK                                                                      0xFFFFFFFFL
-//CP_HQD_EOP_BASE_ADDR
-#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x0
-#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFFL
-//CP_HQD_EOP_BASE_ADDR_HI
-#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
-#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x000000FFL
-//CP_HQD_EOP_CONTROL
-#define CP_HQD_EOP_CONTROL__EOP_SIZE__SHIFT                                                                   0x0
-#define CP_HQD_EOP_CONTROL__PROCESSING_EOP__SHIFT                                                             0x8
-#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN__SHIFT                                                             0xc
-#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB__SHIFT                                                           0xd
-#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN__SHIFT                                                           0xe
-#define CP_HQD_EOP_CONTROL__HALT_FETCHER__SHIFT                                                               0x15
-#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN__SHIFT                                                            0x16
-#define CP_HQD_EOP_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
-#define CP_HQD_EOP_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
-#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT__SHIFT                                                             0x1d
-#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM__SHIFT                                                               0x1f
-#define CP_HQD_EOP_CONTROL__EOP_SIZE_MASK                                                                     0x0000003FL
-#define CP_HQD_EOP_CONTROL__PROCESSING_EOP_MASK                                                               0x00000100L
-#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN_MASK                                                               0x00001000L
-#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB_MASK                                                             0x00002000L
-#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN_MASK                                                             0x00004000L
-#define CP_HQD_EOP_CONTROL__HALT_FETCHER_MASK                                                                 0x00200000L
-#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN_MASK                                                              0x00400000L
-#define CP_HQD_EOP_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
-#define CP_HQD_EOP_CONTROL__CACHE_POLICY_MASK                                                                 0x01000000L
-#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT_MASK                                                               0x60000000L
-#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM_MASK                                                                 0x80000000L
-//CP_HQD_EOP_RPTR
-#define CP_HQD_EOP_RPTR__RPTR__SHIFT                                                                          0x0
-#define CP_HQD_EOP_RPTR__RESET_FETCHER__SHIFT                                                                 0x1c
-#define CP_HQD_EOP_RPTR__DEQUEUE_PEND__SHIFT                                                                  0x1d
-#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR__SHIFT                                                             0x1e
-#define CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT                                                                  0x1f
-#define CP_HQD_EOP_RPTR__RPTR_MASK                                                                            0x00001FFFL
-#define CP_HQD_EOP_RPTR__RESET_FETCHER_MASK                                                                   0x10000000L
-#define CP_HQD_EOP_RPTR__DEQUEUE_PEND_MASK                                                                    0x20000000L
-#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR_MASK                                                               0x40000000L
-#define CP_HQD_EOP_RPTR__INIT_FETCHER_MASK                                                                    0x80000000L
-//CP_HQD_EOP_WPTR
-#define CP_HQD_EOP_WPTR__WPTR__SHIFT                                                                          0x0
-#define CP_HQD_EOP_WPTR__EOP_EMPTY__SHIFT                                                                     0xf
-#define CP_HQD_EOP_WPTR__EOP_AVAIL__SHIFT                                                                     0x10
-#define CP_HQD_EOP_WPTR__WPTR_MASK                                                                            0x00001FFFL
-#define CP_HQD_EOP_WPTR__EOP_EMPTY_MASK                                                                       0x00008000L
-#define CP_HQD_EOP_WPTR__EOP_AVAIL_MASK                                                                       0x1FFF0000L
-//CP_HQD_EOP_EVENTS
-#define CP_HQD_EOP_EVENTS__EVENT_COUNT__SHIFT                                                                 0x0
-#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND__SHIFT                                                       0x10
-#define CP_HQD_EOP_EVENTS__EVENT_COUNT_MASK                                                                   0x00000FFFL
-#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND_MASK                                                         0x00010000L
-//CP_HQD_CTX_SAVE_BASE_ADDR_LO
-#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT                                                             0xc
-#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK                                                               0xFFFFF000L
-//CP_HQD_CTX_SAVE_BASE_ADDR_HI
-#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
-#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x0000FFFFL
-//CP_HQD_CTX_SAVE_CONTROL
-#define CP_HQD_CTX_SAVE_CONTROL__POLICY__SHIFT                                                                0x3
-#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE__SHIFT                                                           0x17
-#define CP_HQD_CTX_SAVE_CONTROL__POLICY_MASK                                                                  0x00000008L
-#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE_MASK                                                             0x00800000L
-//CP_HQD_CNTL_STACK_OFFSET
-#define CP_HQD_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                               0x2
-#define CP_HQD_CNTL_STACK_OFFSET__OFFSET_MASK                                                                 0x00007FFCL
-//CP_HQD_CNTL_STACK_SIZE
-#define CP_HQD_CNTL_STACK_SIZE__SIZE__SHIFT                                                                   0xc
-#define CP_HQD_CNTL_STACK_SIZE__SIZE_MASK                                                                     0x00007000L
-//CP_HQD_WG_STATE_OFFSET
-#define CP_HQD_WG_STATE_OFFSET__OFFSET__SHIFT                                                                 0x2
-#define CP_HQD_WG_STATE_OFFSET__OFFSET_MASK                                                                   0x01FFFFFCL
-//CP_HQD_CTX_SAVE_SIZE
-#define CP_HQD_CTX_SAVE_SIZE__SIZE__SHIFT                                                                     0xc
-#define CP_HQD_CTX_SAVE_SIZE__SIZE_MASK                                                                       0x01FFF000L
-//CP_HQD_GDS_RESOURCE_STATE
-#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED__SHIFT                                                         0x0
-#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED__SHIFT                                                         0x1
-#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE__SHIFT                                                            0x4
-#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR__SHIFT                                                            0xc
-#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED_MASK                                                           0x00000001L
-#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED_MASK                                                           0x00000002L
-#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE_MASK                                                              0x000003F0L
-#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR_MASK                                                              0x0003F000L
-//CP_HQD_ERROR
-#define CP_HQD_ERROR__EDC_ERROR_ID__SHIFT                                                                     0x0
-#define CP_HQD_ERROR__SUA_ERROR__SHIFT                                                                        0x4
-#define CP_HQD_ERROR__AQL_ERROR__SHIFT                                                                        0x5
-#define CP_HQD_ERROR__PQ_UTCL1_ERROR__SHIFT                                                                   0x8
-#define CP_HQD_ERROR__IB_UTCL1_ERROR__SHIFT                                                                   0x9
-#define CP_HQD_ERROR__EOP_UTCL1_ERROR__SHIFT                                                                  0xa
-#define CP_HQD_ERROR__IQ_UTCL1_ERROR__SHIFT                                                                   0xb
-#define CP_HQD_ERROR__RRPT_UTCL1_ERROR__SHIFT                                                                 0xc
-#define CP_HQD_ERROR__WPP_UTCL1_ERROR__SHIFT                                                                  0xd
-#define CP_HQD_ERROR__SEM_UTCL1_ERROR__SHIFT                                                                  0xe
-#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0xf
-#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x10
-#define CP_HQD_ERROR__SR_UTCL1_ERROR__SHIFT                                                                   0x11
-#define CP_HQD_ERROR__QU_UTCL1_ERROR__SHIFT                                                                   0x12
-#define CP_HQD_ERROR__TC_UTCL1_ERROR__SHIFT                                                                   0x13
-#define CP_HQD_ERROR__EDC_ERROR_ID_MASK                                                                       0x0000000FL
-#define CP_HQD_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
-#define CP_HQD_ERROR__AQL_ERROR_MASK                                                                          0x00000020L
-#define CP_HQD_ERROR__PQ_UTCL1_ERROR_MASK                                                                     0x00000100L
-#define CP_HQD_ERROR__IB_UTCL1_ERROR_MASK                                                                     0x00000200L
-#define CP_HQD_ERROR__EOP_UTCL1_ERROR_MASK                                                                    0x00000400L
-#define CP_HQD_ERROR__IQ_UTCL1_ERROR_MASK                                                                     0x00000800L
-#define CP_HQD_ERROR__RRPT_UTCL1_ERROR_MASK                                                                   0x00001000L
-#define CP_HQD_ERROR__WPP_UTCL1_ERROR_MASK                                                                    0x00002000L
-#define CP_HQD_ERROR__SEM_UTCL1_ERROR_MASK                                                                    0x00004000L
-#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00008000L
-#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00010000L
-#define CP_HQD_ERROR__SR_UTCL1_ERROR_MASK                                                                     0x00020000L
-#define CP_HQD_ERROR__QU_UTCL1_ERROR_MASK                                                                     0x00040000L
-#define CP_HQD_ERROR__TC_UTCL1_ERROR_MASK                                                                     0x00080000L
-//CP_HQD_EOP_WPTR_MEM
-#define CP_HQD_EOP_WPTR_MEM__WPTR__SHIFT                                                                      0x0
-#define CP_HQD_EOP_WPTR_MEM__WPTR_MASK                                                                        0x00001FFFL
-//CP_HQD_AQL_CONTROL
-#define CP_HQD_AQL_CONTROL__CONTROL0__SHIFT                                                                   0x0
-#define CP_HQD_AQL_CONTROL__CONTROL0_EN__SHIFT                                                                0xf
-#define CP_HQD_AQL_CONTROL__CONTROL1__SHIFT                                                                   0x10
-#define CP_HQD_AQL_CONTROL__CONTROL1_EN__SHIFT                                                                0x1f
-#define CP_HQD_AQL_CONTROL__CONTROL0_MASK                                                                     0x00007FFFL
-#define CP_HQD_AQL_CONTROL__CONTROL0_EN_MASK                                                                  0x00008000L
-#define CP_HQD_AQL_CONTROL__CONTROL1_MASK                                                                     0x7FFF0000L
-#define CP_HQD_AQL_CONTROL__CONTROL1_EN_MASK                                                                  0x80000000L
-//CP_HQD_PQ_WPTR_LO
-#define CP_HQD_PQ_WPTR_LO__OFFSET__SHIFT                                                                      0x0
-#define CP_HQD_PQ_WPTR_LO__OFFSET_MASK                                                                        0xFFFFFFFFL
-//CP_HQD_PQ_WPTR_HI
-#define CP_HQD_PQ_WPTR_HI__DATA__SHIFT                                                                        0x0
-#define CP_HQD_PQ_WPTR_HI__DATA_MASK                                                                          0xFFFFFFFFL
-
-
-// addressBlock: gc_didtdec
-//DIDT_IND_INDEX
-#define DIDT_IND_INDEX__DIDT_IND_INDEX__SHIFT                                                                 0x0
-#define DIDT_IND_INDEX__DIDT_IND_INDEX_MASK                                                                   0xFFFFFFFFL
-//DIDT_IND_DATA
-#define DIDT_IND_DATA__DIDT_IND_DATA__SHIFT                                                                   0x0
-#define DIDT_IND_DATA__DIDT_IND_DATA_MASK                                                                     0xFFFFFFFFL
-
-
-// addressBlock: gc_gccacdec
-//GC_CAC_CTRL_1
-#define GC_CAC_CTRL_1__CAC_WINDOW__SHIFT                                                                      0x0
-#define GC_CAC_CTRL_1__TDP_WINDOW__SHIFT                                                                      0x18
-#define GC_CAC_CTRL_1__CAC_WINDOW_MASK                                                                        0x00FFFFFFL
-#define GC_CAC_CTRL_1__TDP_WINDOW_MASK                                                                        0xFF000000L
-//GC_CAC_CTRL_2
-#define GC_CAC_CTRL_2__CAC_ENABLE__SHIFT                                                                      0x0
-#define GC_CAC_CTRL_2__CAC_SOFT_CTRL_ENABLE__SHIFT                                                            0x1
-#define GC_CAC_CTRL_2__UNUSED_0__SHIFT                                                                        0x2
-#define GC_CAC_CTRL_2__CAC_ENABLE_MASK                                                                        0x00000001L
-#define GC_CAC_CTRL_2__CAC_SOFT_CTRL_ENABLE_MASK                                                              0x00000002L
-#define GC_CAC_CTRL_2__UNUSED_0_MASK                                                                          0xFFFFFFFCL
-//GC_CAC_CGTT_CLK_CTRL
-#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
-#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
-#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                        0x1e
-#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                        0x1f
-#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
-#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
-#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                          0x40000000L
-#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                          0x80000000L
-//GC_CAC_AGGR_LOWER
-#define GC_CAC_AGGR_LOWER__AGGR_31_0__SHIFT                                                                   0x0
-#define GC_CAC_AGGR_LOWER__AGGR_31_0_MASK                                                                     0xFFFFFFFFL
-//GC_CAC_AGGR_UPPER
-#define GC_CAC_AGGR_UPPER__AGGR_63_32__SHIFT                                                                  0x0
-#define GC_CAC_AGGR_UPPER__AGGR_63_32_MASK                                                                    0xFFFFFFFFL
-//GC_CAC_PG_AGGR_LOWER
-#define GC_CAC_PG_AGGR_LOWER__LKG_AGGR_31_0__SHIFT                                                            0x0
-#define GC_CAC_PG_AGGR_LOWER__LKG_AGGR_31_0_MASK                                                              0xFFFFFFFFL
-//GC_CAC_PG_AGGR_UPPER
-#define GC_CAC_PG_AGGR_UPPER__LKG_AGGR_63_32__SHIFT                                                           0x0
-#define GC_CAC_PG_AGGR_UPPER__LKG_AGGR_63_32_MASK                                                             0xFFFFFFFFL
-//GC_CAC_SOFT_CTRL
-#define GC_CAC_SOFT_CTRL__SOFT_SNAP__SHIFT                                                                    0x0
-#define GC_CAC_SOFT_CTRL__UNUSED__SHIFT                                                                       0x1
-#define GC_CAC_SOFT_CTRL__SOFT_SNAP_MASK                                                                      0x00000001L
-#define GC_CAC_SOFT_CTRL__UNUSED_MASK                                                                         0xFFFFFFFEL
-//GC_DIDT_CTRL0
-#define GC_DIDT_CTRL0__DIDT_CTRL_EN__SHIFT                                                                    0x0
-#define GC_DIDT_CTRL0__PHASE_OFFSET__SHIFT                                                                    0x1
-#define GC_DIDT_CTRL0__DIDT_SW_RST__SHIFT                                                                     0x3
-#define GC_DIDT_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT                                                            0x4
-#define GC_DIDT_CTRL0__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                    0x5
-#define GC_DIDT_CTRL0__DIDT_CTRL_EN_MASK                                                                      0x00000001L
-#define GC_DIDT_CTRL0__PHASE_OFFSET_MASK                                                                      0x00000006L
-#define GC_DIDT_CTRL0__DIDT_SW_RST_MASK                                                                       0x00000008L
-#define GC_DIDT_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK                                                              0x00000010L
-#define GC_DIDT_CTRL0__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK                                                      0x000001E0L
-//GC_DIDT_CTRL1
-#define GC_DIDT_CTRL1__MIN_POWER__SHIFT                                                                       0x0
-#define GC_DIDT_CTRL1__MAX_POWER__SHIFT                                                                       0x10
-#define GC_DIDT_CTRL1__MIN_POWER_MASK                                                                         0x0000FFFFL
-#define GC_DIDT_CTRL1__MAX_POWER_MASK                                                                         0xFFFF0000L
-//GC_DIDT_CTRL2
-#define GC_DIDT_CTRL2__MAX_POWER_DELTA__SHIFT                                                                 0x0
-#define GC_DIDT_CTRL2__UNUSED_0__SHIFT                                                                        0xe
-#define GC_DIDT_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                        0x10
-#define GC_DIDT_CTRL2__UNUSED_1__SHIFT                                                                        0x1a
-#define GC_DIDT_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                        0x1b
-#define GC_DIDT_CTRL2__UNUSED_2__SHIFT                                                                        0x1f
-#define GC_DIDT_CTRL2__MAX_POWER_DELTA_MASK                                                                   0x00003FFFL
-#define GC_DIDT_CTRL2__UNUSED_0_MASK                                                                          0x0000C000L
-#define GC_DIDT_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK                                                          0x03FF0000L
-#define GC_DIDT_CTRL2__UNUSED_1_MASK                                                                          0x04000000L
-#define GC_DIDT_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK                                                          0x78000000L
-#define GC_DIDT_CTRL2__UNUSED_2_MASK                                                                          0x80000000L
-//GC_DIDT_WEIGHT
-#define GC_DIDT_WEIGHT__SQ_WEIGHT__SHIFT                                                                      0x0
-#define GC_DIDT_WEIGHT__DB_WEIGHT__SHIFT                                                                      0x8
-#define GC_DIDT_WEIGHT__TD_WEIGHT__SHIFT                                                                      0x10
-#define GC_DIDT_WEIGHT__TCP_WEIGHT__SHIFT                                                                     0x18
-#define GC_DIDT_WEIGHT__SQ_WEIGHT_MASK                                                                        0x000000FFL
-#define GC_DIDT_WEIGHT__DB_WEIGHT_MASK                                                                        0x0000FF00L
-#define GC_DIDT_WEIGHT__TD_WEIGHT_MASK                                                                        0x00FF0000L
-#define GC_DIDT_WEIGHT__TCP_WEIGHT_MASK                                                                       0xFF000000L
-//GC_EDC_CTRL
-#define GC_EDC_CTRL__EDC_EN__SHIFT                                                                            0x0
-#define GC_EDC_CTRL__EDC_SW_RST__SHIFT                                                                        0x1
-#define GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                               0x2
-#define GC_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                                   0x3
-#define GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                       0x4
-#define GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                          0x9
-#define GC_EDC_CTRL__UNUSED_0__SHIFT                                                                          0xa
-#define GC_EDC_CTRL__EDC_EN_MASK                                                                              0x00000001L
-#define GC_EDC_CTRL__EDC_SW_RST_MASK                                                                          0x00000002L
-#define GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                                 0x00000004L
-#define GC_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                     0x00000008L
-#define GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                         0x000001F0L
-#define GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                            0x00000200L
-#define GC_EDC_CTRL__UNUSED_0_MASK                                                                            0xFFFFFC00L
-//GC_EDC_THRESHOLD
-#define GC_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                                0x0
-#define GC_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                                  0xFFFFFFFFL
-//GC_EDC_STATUS
-#define GC_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                              0x0
-#define GC_EDC_STATUS__EDC_ROLLING_DROOP_DELTA__SHIFT                                                         0x3
-#define GC_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                                0x00000007L
-#define GC_EDC_STATUS__EDC_ROLLING_DROOP_DELTA_MASK                                                           0x03FFFFF8L
-//GC_EDC_OVERFLOW
-#define GC_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                              0x0
-#define GC_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                           0x1
-#define GC_EDC_OVERFLOW__EDC_DROOP_LEVEL_OVERFLOW__SHIFT                                                      0x11
-#define GC_EDC_OVERFLOW__PSM_COUNTER__SHIFT                                                                   0x12
-#define GC_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                                0x00000001L
-#define GC_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                             0x0001FFFEL
-#define GC_EDC_OVERFLOW__EDC_DROOP_LEVEL_OVERFLOW_MASK                                                        0x00020000L
-#define GC_EDC_OVERFLOW__PSM_COUNTER_MASK                                                                     0xFFFC0000L
-//GC_EDC_ROLLING_POWER_DELTA
-#define GC_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                            0x0
-#define GC_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                              0xFFFFFFFFL
-//GC_DIDT_DROOP_CTRL
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_EN__SHIFT                                                        0x0
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_THRESHOLD__SHIFT                                                       0x1
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_INDEX__SHIFT                                                     0xf
-#define GC_DIDT_DROOP_CTRL__DIDT_LEVEL_SEL__SHIFT                                                             0x13
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_OVERFLOW__SHIFT                                                  0x1f
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_EN_MASK                                                          0x00000001L
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_THRESHOLD_MASK                                                         0x00007FFEL
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_INDEX_MASK                                                       0x00078000L
-#define GC_DIDT_DROOP_CTRL__DIDT_LEVEL_SEL_MASK                                                               0x00080000L
-#define GC_DIDT_DROOP_CTRL__DIDT_DROOP_LEVEL_OVERFLOW_MASK                                                    0x80000000L
-//GC_EDC_DROOP_CTRL
-#define GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_EN__SHIFT                                                          0x0
-#define GC_EDC_DROOP_CTRL__EDC_DROOP_THRESHOLD__SHIFT                                                         0x1
-#define GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_INDEX__SHIFT                                                       0xf
-#define GC_EDC_DROOP_CTRL__AVG_PSM_SEL__SHIFT                                                                 0x14
-#define GC_EDC_DROOP_CTRL__EDC_LEVEL_SEL__SHIFT                                                               0x15
-#define GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_EN_MASK                                                            0x00000001L
-#define GC_EDC_DROOP_CTRL__EDC_DROOP_THRESHOLD_MASK                                                           0x00007FFEL
-#define GC_EDC_DROOP_CTRL__EDC_DROOP_LEVEL_INDEX_MASK                                                         0x000F8000L
-#define GC_EDC_DROOP_CTRL__AVG_PSM_SEL_MASK                                                                   0x00100000L
-#define GC_EDC_DROOP_CTRL__EDC_LEVEL_SEL_MASK                                                                 0x00200000L
-//GC_CAC_IND_INDEX
-#define GC_CAC_IND_INDEX__GC_CAC_IND_ADDR__SHIFT                                                              0x0
-#define GC_CAC_IND_INDEX__GC_CAC_IND_ADDR_MASK                                                                0xFFFFFFFFL
-//GC_CAC_IND_DATA
-#define GC_CAC_IND_DATA__GC_CAC_IND_DATA__SHIFT                                                               0x0
-#define GC_CAC_IND_DATA__GC_CAC_IND_DATA_MASK                                                                 0xFFFFFFFFL
-//SE_CAC_CGTT_CLK_CTRL
-#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
-#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
-#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                        0x1e
-#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                        0x1f
-#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
-#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
-#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                          0x40000000L
-#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                          0x80000000L
-//SE_CAC_IND_INDEX
-#define SE_CAC_IND_INDEX__SE_CAC_IND_ADDR__SHIFT                                                              0x0
-#define SE_CAC_IND_INDEX__SE_CAC_IND_ADDR_MASK                                                                0xFFFFFFFFL
-//SE_CAC_IND_DATA
-#define SE_CAC_IND_DATA__SE_CAC_IND_DATA__SHIFT                                                               0x0
-#define SE_CAC_IND_DATA__SE_CAC_IND_DATA_MASK                                                                 0xFFFFFFFFL
-
-
-// addressBlock: gc_tcpdec
-//TCP_WATCH0_ADDR_H
-#define TCP_WATCH0_ADDR_H__ADDR__SHIFT                                                                        0x0
-#define TCP_WATCH0_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
-//TCP_WATCH0_ADDR_L
-#define TCP_WATCH0_ADDR_L__ADDR__SHIFT                                                                        0x6
-#define TCP_WATCH0_ADDR_L__ADDR_MASK                                                                          0xFFFFFFC0L
-//TCP_WATCH0_CNTL
-#define TCP_WATCH0_CNTL__MASK__SHIFT                                                                          0x0
-#define TCP_WATCH0_CNTL__VMID__SHIFT                                                                          0x18
-#define TCP_WATCH0_CNTL__ATC__SHIFT                                                                           0x1c
-#define TCP_WATCH0_CNTL__MODE__SHIFT                                                                          0x1d
-#define TCP_WATCH0_CNTL__VALID__SHIFT                                                                         0x1f
-#define TCP_WATCH0_CNTL__MASK_MASK                                                                            0x00FFFFFFL
-#define TCP_WATCH0_CNTL__VMID_MASK                                                                            0x0F000000L
-#define TCP_WATCH0_CNTL__ATC_MASK                                                                             0x10000000L
-#define TCP_WATCH0_CNTL__MODE_MASK                                                                            0x60000000L
-#define TCP_WATCH0_CNTL__VALID_MASK                                                                           0x80000000L
-//TCP_WATCH1_ADDR_H
-#define TCP_WATCH1_ADDR_H__ADDR__SHIFT                                                                        0x0
-#define TCP_WATCH1_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
-//TCP_WATCH1_ADDR_L
-#define TCP_WATCH1_ADDR_L__ADDR__SHIFT                                                                        0x6
-#define TCP_WATCH1_ADDR_L__ADDR_MASK                                                                          0xFFFFFFC0L
-//TCP_WATCH1_CNTL
-#define TCP_WATCH1_CNTL__MASK__SHIFT                                                                          0x0
-#define TCP_WATCH1_CNTL__VMID__SHIFT                                                                          0x18
-#define TCP_WATCH1_CNTL__ATC__SHIFT                                                                           0x1c
-#define TCP_WATCH1_CNTL__MODE__SHIFT                                                                          0x1d
-#define TCP_WATCH1_CNTL__VALID__SHIFT                                                                         0x1f
-#define TCP_WATCH1_CNTL__MASK_MASK                                                                            0x00FFFFFFL
-#define TCP_WATCH1_CNTL__VMID_MASK                                                                            0x0F000000L
-#define TCP_WATCH1_CNTL__ATC_MASK                                                                             0x10000000L
-#define TCP_WATCH1_CNTL__MODE_MASK                                                                            0x60000000L
-#define TCP_WATCH1_CNTL__VALID_MASK                                                                           0x80000000L
-//TCP_WATCH2_ADDR_H
-#define TCP_WATCH2_ADDR_H__ADDR__SHIFT                                                                        0x0
-#define TCP_WATCH2_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
-//TCP_WATCH2_ADDR_L
-#define TCP_WATCH2_ADDR_L__ADDR__SHIFT                                                                        0x6
-#define TCP_WATCH2_ADDR_L__ADDR_MASK                                                                          0xFFFFFFC0L
-//TCP_WATCH2_CNTL
-#define TCP_WATCH2_CNTL__MASK__SHIFT                                                                          0x0
-#define TCP_WATCH2_CNTL__VMID__SHIFT                                                                          0x18
-#define TCP_WATCH2_CNTL__ATC__SHIFT                                                                           0x1c
-#define TCP_WATCH2_CNTL__MODE__SHIFT                                                                          0x1d
-#define TCP_WATCH2_CNTL__VALID__SHIFT                                                                         0x1f
-#define TCP_WATCH2_CNTL__MASK_MASK                                                                            0x00FFFFFFL
-#define TCP_WATCH2_CNTL__VMID_MASK                                                                            0x0F000000L
-#define TCP_WATCH2_CNTL__ATC_MASK                                                                             0x10000000L
-#define TCP_WATCH2_CNTL__MODE_MASK                                                                            0x60000000L
-#define TCP_WATCH2_CNTL__VALID_MASK                                                                           0x80000000L
-//TCP_WATCH3_ADDR_H
-#define TCP_WATCH3_ADDR_H__ADDR__SHIFT                                                                        0x0
-#define TCP_WATCH3_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
-//TCP_WATCH3_ADDR_L
-#define TCP_WATCH3_ADDR_L__ADDR__SHIFT                                                                        0x6
-#define TCP_WATCH3_ADDR_L__ADDR_MASK                                                                          0xFFFFFFC0L
-//TCP_WATCH3_CNTL
-#define TCP_WATCH3_CNTL__MASK__SHIFT                                                                          0x0
-#define TCP_WATCH3_CNTL__VMID__SHIFT                                                                          0x18
-#define TCP_WATCH3_CNTL__ATC__SHIFT                                                                           0x1c
-#define TCP_WATCH3_CNTL__MODE__SHIFT                                                                          0x1d
-#define TCP_WATCH3_CNTL__VALID__SHIFT                                                                         0x1f
-#define TCP_WATCH3_CNTL__MASK_MASK                                                                            0x00FFFFFFL
-#define TCP_WATCH3_CNTL__VMID_MASK                                                                            0x0F000000L
-#define TCP_WATCH3_CNTL__ATC_MASK                                                                             0x10000000L
-#define TCP_WATCH3_CNTL__MODE_MASK                                                                            0x60000000L
-#define TCP_WATCH3_CNTL__VALID_MASK                                                                           0x80000000L
-//TCP_GATCL1_CNTL
-#define TCP_GATCL1_CNTL__INVALIDATE_ALL_VMID__SHIFT                                                           0x19
-#define TCP_GATCL1_CNTL__FORCE_MISS__SHIFT                                                                    0x1a
-#define TCP_GATCL1_CNTL__FORCE_IN_ORDER__SHIFT                                                                0x1b
-#define TCP_GATCL1_CNTL__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
-#define TCP_GATCL1_CNTL__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
-#define TCP_GATCL1_CNTL__INVALIDATE_ALL_VMID_MASK                                                             0x02000000L
-#define TCP_GATCL1_CNTL__FORCE_MISS_MASK                                                                      0x04000000L
-#define TCP_GATCL1_CNTL__FORCE_IN_ORDER_MASK                                                                  0x08000000L
-#define TCP_GATCL1_CNTL__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
-#define TCP_GATCL1_CNTL__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
-//TCP_ATC_EDC_GATCL1_CNT
-#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC__SHIFT                                                               0x0
-#define TCP_ATC_EDC_GATCL1_CNT__DATA_SEC_MASK                                                                 0x000000FFL
-//TCP_GATCL1_DSM_CNTL
-#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A0__SHIFT                                      0x0
-#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A1__SHIFT                                      0x1
-#define TCP_GATCL1_DSM_CNTL__TCP_GATCL1_ENABLE_SINGLE_WRITE_A__SHIFT                                          0x2
-#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A0_MASK                                        0x00000001L
-#define TCP_GATCL1_DSM_CNTL__SEL_DSM_TCP_GATCL1_IRRITATOR_DATA_A1_MASK                                        0x00000002L
-#define TCP_GATCL1_DSM_CNTL__TCP_GATCL1_ENABLE_SINGLE_WRITE_A_MASK                                            0x00000004L
-//TCP_CNTL2
-#define TCP_CNTL2__LS_DISABLE_CLOCKS__SHIFT                                                                   0x0
-#define TCP_CNTL2__LS_DISABLE_CLOCKS_MASK                                                                     0x000000FFL
-//TCP_UTCL1_CNTL1
-#define TCP_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
-#define TCP_UTCL1_CNTL1__GPUVM_64K_DEFAULT__SHIFT                                                             0x1
-#define TCP_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
-#define TCP_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                     0x3
-#define TCP_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
-#define TCP_UTCL1_CNTL1__CLIENTID__SHIFT                                                                      0x7
-#define TCP_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
-#define TCP_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                              0x17
-#define TCP_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
-#define TCP_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                                    0x19
-#define TCP_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
-#define TCP_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
-#define TCP_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
-#define TCP_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
-#define TCP_UTCL1_CNTL1__GPUVM_64K_DEFAULT_MASK                                                               0x00000002L
-#define TCP_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
-#define TCP_UTCL1_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
-#define TCP_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
-#define TCP_UTCL1_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
-#define TCP_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
-#define TCP_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                0x00800000L
-#define TCP_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
-#define TCP_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                                      0x02000000L
-#define TCP_UTCL1_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
-#define TCP_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
-#define TCP_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
-//TCP_UTCL1_CNTL2
-#define TCP_UTCL1_CNTL2__SPARE__SHIFT                                                                         0x0
-#define TCP_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
-#define TCP_UTCL1_CNTL2__ANY_LINE_VALID__SHIFT                                                                0xa
-#define TCP_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                0xc
-#define TCP_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
-#define TCP_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                           0xf
-#define TCP_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                          0x1a
-#define TCP_UTCL1_CNTL2__SPARE_MASK                                                                           0x000000FFL
-#define TCP_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
-#define TCP_UTCL1_CNTL2__ANY_LINE_VALID_MASK                                                                  0x00000400L
-#define TCP_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                  0x00001000L
-#define TCP_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
-#define TCP_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                             0x00008000L
-#define TCP_UTCL1_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                            0x04000000L
-//TCP_UTCL1_STATUS
-#define TCP_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
-#define TCP_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
-#define TCP_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
-#define TCP_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
-#define TCP_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
-#define TCP_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
-//TCP_PERFCOUNTER_FILTER
-#define TCP_PERFCOUNTER_FILTER__BUFFER__SHIFT                                                                 0x0
-#define TCP_PERFCOUNTER_FILTER__FLAT__SHIFT                                                                   0x1
-#define TCP_PERFCOUNTER_FILTER__DIM__SHIFT                                                                    0x2
-#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT__SHIFT                                                            0x5
-#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT__SHIFT                                                             0xb
-#define TCP_PERFCOUNTER_FILTER__SW_MODE__SHIFT                                                                0xf
-#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES__SHIFT                                                            0x14
-#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE__SHIFT                                                            0x16
-#define TCP_PERFCOUNTER_FILTER__GLC__SHIFT                                                                    0x19
-#define TCP_PERFCOUNTER_FILTER__SLC__SHIFT                                                                    0x1a
-#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE__SHIFT                                                     0x1b
-#define TCP_PERFCOUNTER_FILTER__ADDR_MODE__SHIFT                                                              0x1c
-#define TCP_PERFCOUNTER_FILTER__BUFFER_MASK                                                                   0x00000001L
-#define TCP_PERFCOUNTER_FILTER__FLAT_MASK                                                                     0x00000002L
-#define TCP_PERFCOUNTER_FILTER__DIM_MASK                                                                      0x0000001CL
-#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT_MASK                                                              0x000007E0L
-#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT_MASK                                                               0x00007800L
-#define TCP_PERFCOUNTER_FILTER__SW_MODE_MASK                                                                  0x000F8000L
-#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES_MASK                                                              0x00300000L
-#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE_MASK                                                              0x01C00000L
-#define TCP_PERFCOUNTER_FILTER__GLC_MASK                                                                      0x02000000L
-#define TCP_PERFCOUNTER_FILTER__SLC_MASK                                                                      0x04000000L
-#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE_MASK                                                       0x08000000L
-#define TCP_PERFCOUNTER_FILTER__ADDR_MODE_MASK                                                                0x70000000L
-//TCP_PERFCOUNTER_FILTER_EN
-#define TCP_PERFCOUNTER_FILTER_EN__BUFFER__SHIFT                                                              0x0
-#define TCP_PERFCOUNTER_FILTER_EN__FLAT__SHIFT                                                                0x1
-#define TCP_PERFCOUNTER_FILTER_EN__DIM__SHIFT                                                                 0x2
-#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT__SHIFT                                                         0x3
-#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT__SHIFT                                                          0x4
-#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE__SHIFT                                                             0x5
-#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES__SHIFT                                                         0x6
-#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE__SHIFT                                                         0x7
-#define TCP_PERFCOUNTER_FILTER_EN__GLC__SHIFT                                                                 0x8
-#define TCP_PERFCOUNTER_FILTER_EN__SLC__SHIFT                                                                 0x9
-#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE__SHIFT                                                  0xa
-#define TCP_PERFCOUNTER_FILTER_EN__ADDR_MODE__SHIFT                                                           0xb
-#define TCP_PERFCOUNTER_FILTER_EN__BUFFER_MASK                                                                0x00000001L
-#define TCP_PERFCOUNTER_FILTER_EN__FLAT_MASK                                                                  0x00000002L
-#define TCP_PERFCOUNTER_FILTER_EN__DIM_MASK                                                                   0x00000004L
-#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT_MASK                                                           0x00000008L
-#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT_MASK                                                            0x00000010L
-#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE_MASK                                                               0x00000020L
-#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES_MASK                                                           0x00000040L
-#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE_MASK                                                           0x00000080L
-#define TCP_PERFCOUNTER_FILTER_EN__GLC_MASK                                                                   0x00000100L
-#define TCP_PERFCOUNTER_FILTER_EN__SLC_MASK                                                                   0x00000200L
-#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE_MASK                                                    0x00000400L
-#define TCP_PERFCOUNTER_FILTER_EN__ADDR_MODE_MASK                                                             0x00000800L
-
-
-// addressBlock: gc_gdspdec
-//GDS_VMID0_BASE
-#define GDS_VMID0_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID0_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID0_SIZE
-#define GDS_VMID0_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID0_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID1_BASE
-#define GDS_VMID1_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID1_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID1_SIZE
-#define GDS_VMID1_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID1_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID2_BASE
-#define GDS_VMID2_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID2_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID2_SIZE
-#define GDS_VMID2_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID2_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID3_BASE
-#define GDS_VMID3_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID3_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID3_SIZE
-#define GDS_VMID3_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID3_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID4_BASE
-#define GDS_VMID4_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID4_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID4_SIZE
-#define GDS_VMID4_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID4_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID5_BASE
-#define GDS_VMID5_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID5_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID5_SIZE
-#define GDS_VMID5_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID5_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID6_BASE
-#define GDS_VMID6_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID6_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID6_SIZE
-#define GDS_VMID6_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID6_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID7_BASE
-#define GDS_VMID7_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID7_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID7_SIZE
-#define GDS_VMID7_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID7_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID8_BASE
-#define GDS_VMID8_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID8_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID8_SIZE
-#define GDS_VMID8_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID8_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID9_BASE
-#define GDS_VMID9_BASE__BASE__SHIFT                                                                           0x0
-#define GDS_VMID9_BASE__BASE_MASK                                                                             0x0000FFFFL
-//GDS_VMID9_SIZE
-#define GDS_VMID9_SIZE__SIZE__SHIFT                                                                           0x0
-#define GDS_VMID9_SIZE__SIZE_MASK                                                                             0x0001FFFFL
-//GDS_VMID10_BASE
-#define GDS_VMID10_BASE__BASE__SHIFT                                                                          0x0
-#define GDS_VMID10_BASE__BASE_MASK                                                                            0x0000FFFFL
-//GDS_VMID10_SIZE
-#define GDS_VMID10_SIZE__SIZE__SHIFT                                                                          0x0
-#define GDS_VMID10_SIZE__SIZE_MASK                                                                            0x0001FFFFL
-//GDS_VMID11_BASE
-#define GDS_VMID11_BASE__BASE__SHIFT                                                                          0x0
-#define GDS_VMID11_BASE__BASE_MASK                                                                            0x0000FFFFL
-//GDS_VMID11_SIZE
-#define GDS_VMID11_SIZE__SIZE__SHIFT                                                                          0x0
-#define GDS_VMID11_SIZE__SIZE_MASK                                                                            0x0001FFFFL
-//GDS_VMID12_BASE
-#define GDS_VMID12_BASE__BASE__SHIFT                                                                          0x0
-#define GDS_VMID12_BASE__BASE_MASK                                                                            0x0000FFFFL
-//GDS_VMID12_SIZE
-#define GDS_VMID12_SIZE__SIZE__SHIFT                                                                          0x0
-#define GDS_VMID12_SIZE__SIZE_MASK                                                                            0x0001FFFFL
-//GDS_VMID13_BASE
-#define GDS_VMID13_BASE__BASE__SHIFT                                                                          0x0
-#define GDS_VMID13_BASE__BASE_MASK                                                                            0x0000FFFFL
-//GDS_VMID13_SIZE
-#define GDS_VMID13_SIZE__SIZE__SHIFT                                                                          0x0
-#define GDS_VMID13_SIZE__SIZE_MASK                                                                            0x0001FFFFL
-//GDS_VMID14_BASE
-#define GDS_VMID14_BASE__BASE__SHIFT                                                                          0x0
-#define GDS_VMID14_BASE__BASE_MASK                                                                            0x0000FFFFL
-//GDS_VMID14_SIZE
-#define GDS_VMID14_SIZE__SIZE__SHIFT                                                                          0x0
-#define GDS_VMID14_SIZE__SIZE_MASK                                                                            0x0001FFFFL
-//GDS_VMID15_BASE
-#define GDS_VMID15_BASE__BASE__SHIFT                                                                          0x0
-#define GDS_VMID15_BASE__BASE_MASK                                                                            0x0000FFFFL
-//GDS_VMID15_SIZE
-#define GDS_VMID15_SIZE__SIZE__SHIFT                                                                          0x0
-#define GDS_VMID15_SIZE__SIZE_MASK                                                                            0x0001FFFFL
-//GDS_GWS_VMID0
-#define GDS_GWS_VMID0__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID0__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID0__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID0__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID1
-#define GDS_GWS_VMID1__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID1__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID1__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID1__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID2
-#define GDS_GWS_VMID2__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID2__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID2__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID2__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID3
-#define GDS_GWS_VMID3__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID3__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID3__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID3__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID4
-#define GDS_GWS_VMID4__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID4__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID4__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID4__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID5
-#define GDS_GWS_VMID5__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID5__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID5__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID5__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID6
-#define GDS_GWS_VMID6__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID6__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID6__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID6__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID7
-#define GDS_GWS_VMID7__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID7__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID7__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID7__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID8
-#define GDS_GWS_VMID8__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID8__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID8__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID8__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID9
-#define GDS_GWS_VMID9__BASE__SHIFT                                                                            0x0
-#define GDS_GWS_VMID9__SIZE__SHIFT                                                                            0x10
-#define GDS_GWS_VMID9__BASE_MASK                                                                              0x0000003FL
-#define GDS_GWS_VMID9__SIZE_MASK                                                                              0x007F0000L
-//GDS_GWS_VMID10
-#define GDS_GWS_VMID10__BASE__SHIFT                                                                           0x0
-#define GDS_GWS_VMID10__SIZE__SHIFT                                                                           0x10
-#define GDS_GWS_VMID10__BASE_MASK                                                                             0x0000003FL
-#define GDS_GWS_VMID10__SIZE_MASK                                                                             0x007F0000L
-//GDS_GWS_VMID11
-#define GDS_GWS_VMID11__BASE__SHIFT                                                                           0x0
-#define GDS_GWS_VMID11__SIZE__SHIFT                                                                           0x10
-#define GDS_GWS_VMID11__BASE_MASK                                                                             0x0000003FL
-#define GDS_GWS_VMID11__SIZE_MASK                                                                             0x007F0000L
-//GDS_GWS_VMID12
-#define GDS_GWS_VMID12__BASE__SHIFT                                                                           0x0
-#define GDS_GWS_VMID12__SIZE__SHIFT                                                                           0x10
-#define GDS_GWS_VMID12__BASE_MASK                                                                             0x0000003FL
-#define GDS_GWS_VMID12__SIZE_MASK                                                                             0x007F0000L
-//GDS_GWS_VMID13
-#define GDS_GWS_VMID13__BASE__SHIFT                                                                           0x0
-#define GDS_GWS_VMID13__SIZE__SHIFT                                                                           0x10
-#define GDS_GWS_VMID13__BASE_MASK                                                                             0x0000003FL
-#define GDS_GWS_VMID13__SIZE_MASK                                                                             0x007F0000L
-//GDS_GWS_VMID14
-#define GDS_GWS_VMID14__BASE__SHIFT                                                                           0x0
-#define GDS_GWS_VMID14__SIZE__SHIFT                                                                           0x10
-#define GDS_GWS_VMID14__BASE_MASK                                                                             0x0000003FL
-#define GDS_GWS_VMID14__SIZE_MASK                                                                             0x007F0000L
-//GDS_GWS_VMID15
-#define GDS_GWS_VMID15__BASE__SHIFT                                                                           0x0
-#define GDS_GWS_VMID15__SIZE__SHIFT                                                                           0x10
-#define GDS_GWS_VMID15__BASE_MASK                                                                             0x0000003FL
-#define GDS_GWS_VMID15__SIZE_MASK                                                                             0x007F0000L
-//GDS_OA_VMID0
-#define GDS_OA_VMID0__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID0__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID0__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID0__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID1
-#define GDS_OA_VMID1__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID1__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID1__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID1__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID2
-#define GDS_OA_VMID2__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID2__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID2__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID2__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID3
-#define GDS_OA_VMID3__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID3__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID3__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID3__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID4
-#define GDS_OA_VMID4__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID4__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID4__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID4__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID5
-#define GDS_OA_VMID5__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID5__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID5__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID5__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID6
-#define GDS_OA_VMID6__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID6__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID6__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID6__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID7
-#define GDS_OA_VMID7__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID7__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID7__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID7__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID8
-#define GDS_OA_VMID8__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID8__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID8__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID8__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID9
-#define GDS_OA_VMID9__MASK__SHIFT                                                                             0x0
-#define GDS_OA_VMID9__UNUSED__SHIFT                                                                           0x10
-#define GDS_OA_VMID9__MASK_MASK                                                                               0x0000FFFFL
-#define GDS_OA_VMID9__UNUSED_MASK                                                                             0xFFFF0000L
-//GDS_OA_VMID10
-#define GDS_OA_VMID10__MASK__SHIFT                                                                            0x0
-#define GDS_OA_VMID10__UNUSED__SHIFT                                                                          0x10
-#define GDS_OA_VMID10__MASK_MASK                                                                              0x0000FFFFL
-#define GDS_OA_VMID10__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_OA_VMID11
-#define GDS_OA_VMID11__MASK__SHIFT                                                                            0x0
-#define GDS_OA_VMID11__UNUSED__SHIFT                                                                          0x10
-#define GDS_OA_VMID11__MASK_MASK                                                                              0x0000FFFFL
-#define GDS_OA_VMID11__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_OA_VMID12
-#define GDS_OA_VMID12__MASK__SHIFT                                                                            0x0
-#define GDS_OA_VMID12__UNUSED__SHIFT                                                                          0x10
-#define GDS_OA_VMID12__MASK_MASK                                                                              0x0000FFFFL
-#define GDS_OA_VMID12__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_OA_VMID13
-#define GDS_OA_VMID13__MASK__SHIFT                                                                            0x0
-#define GDS_OA_VMID13__UNUSED__SHIFT                                                                          0x10
-#define GDS_OA_VMID13__MASK_MASK                                                                              0x0000FFFFL
-#define GDS_OA_VMID13__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_OA_VMID14
-#define GDS_OA_VMID14__MASK__SHIFT                                                                            0x0
-#define GDS_OA_VMID14__UNUSED__SHIFT                                                                          0x10
-#define GDS_OA_VMID14__MASK_MASK                                                                              0x0000FFFFL
-#define GDS_OA_VMID14__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_OA_VMID15
-#define GDS_OA_VMID15__MASK__SHIFT                                                                            0x0
-#define GDS_OA_VMID15__UNUSED__SHIFT                                                                          0x10
-#define GDS_OA_VMID15__MASK_MASK                                                                              0x0000FFFFL
-#define GDS_OA_VMID15__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_GWS_RESET0
-#define GDS_GWS_RESET0__RESOURCE0_RESET__SHIFT                                                                0x0
-#define GDS_GWS_RESET0__RESOURCE1_RESET__SHIFT                                                                0x1
-#define GDS_GWS_RESET0__RESOURCE2_RESET__SHIFT                                                                0x2
-#define GDS_GWS_RESET0__RESOURCE3_RESET__SHIFT                                                                0x3
-#define GDS_GWS_RESET0__RESOURCE4_RESET__SHIFT                                                                0x4
-#define GDS_GWS_RESET0__RESOURCE5_RESET__SHIFT                                                                0x5
-#define GDS_GWS_RESET0__RESOURCE6_RESET__SHIFT                                                                0x6
-#define GDS_GWS_RESET0__RESOURCE7_RESET__SHIFT                                                                0x7
-#define GDS_GWS_RESET0__RESOURCE8_RESET__SHIFT                                                                0x8
-#define GDS_GWS_RESET0__RESOURCE9_RESET__SHIFT                                                                0x9
-#define GDS_GWS_RESET0__RESOURCE10_RESET__SHIFT                                                               0xa
-#define GDS_GWS_RESET0__RESOURCE11_RESET__SHIFT                                                               0xb
-#define GDS_GWS_RESET0__RESOURCE12_RESET__SHIFT                                                               0xc
-#define GDS_GWS_RESET0__RESOURCE13_RESET__SHIFT                                                               0xd
-#define GDS_GWS_RESET0__RESOURCE14_RESET__SHIFT                                                               0xe
-#define GDS_GWS_RESET0__RESOURCE15_RESET__SHIFT                                                               0xf
-#define GDS_GWS_RESET0__RESOURCE16_RESET__SHIFT                                                               0x10
-#define GDS_GWS_RESET0__RESOURCE17_RESET__SHIFT                                                               0x11
-#define GDS_GWS_RESET0__RESOURCE18_RESET__SHIFT                                                               0x12
-#define GDS_GWS_RESET0__RESOURCE19_RESET__SHIFT                                                               0x13
-#define GDS_GWS_RESET0__RESOURCE20_RESET__SHIFT                                                               0x14
-#define GDS_GWS_RESET0__RESOURCE21_RESET__SHIFT                                                               0x15
-#define GDS_GWS_RESET0__RESOURCE22_RESET__SHIFT                                                               0x16
-#define GDS_GWS_RESET0__RESOURCE23_RESET__SHIFT                                                               0x17
-#define GDS_GWS_RESET0__RESOURCE24_RESET__SHIFT                                                               0x18
-#define GDS_GWS_RESET0__RESOURCE25_RESET__SHIFT                                                               0x19
-#define GDS_GWS_RESET0__RESOURCE26_RESET__SHIFT                                                               0x1a
-#define GDS_GWS_RESET0__RESOURCE27_RESET__SHIFT                                                               0x1b
-#define GDS_GWS_RESET0__RESOURCE28_RESET__SHIFT                                                               0x1c
-#define GDS_GWS_RESET0__RESOURCE29_RESET__SHIFT                                                               0x1d
-#define GDS_GWS_RESET0__RESOURCE30_RESET__SHIFT                                                               0x1e
-#define GDS_GWS_RESET0__RESOURCE31_RESET__SHIFT                                                               0x1f
-#define GDS_GWS_RESET0__RESOURCE0_RESET_MASK                                                                  0x00000001L
-#define GDS_GWS_RESET0__RESOURCE1_RESET_MASK                                                                  0x00000002L
-#define GDS_GWS_RESET0__RESOURCE2_RESET_MASK                                                                  0x00000004L
-#define GDS_GWS_RESET0__RESOURCE3_RESET_MASK                                                                  0x00000008L
-#define GDS_GWS_RESET0__RESOURCE4_RESET_MASK                                                                  0x00000010L
-#define GDS_GWS_RESET0__RESOURCE5_RESET_MASK                                                                  0x00000020L
-#define GDS_GWS_RESET0__RESOURCE6_RESET_MASK                                                                  0x00000040L
-#define GDS_GWS_RESET0__RESOURCE7_RESET_MASK                                                                  0x00000080L
-#define GDS_GWS_RESET0__RESOURCE8_RESET_MASK                                                                  0x00000100L
-#define GDS_GWS_RESET0__RESOURCE9_RESET_MASK                                                                  0x00000200L
-#define GDS_GWS_RESET0__RESOURCE10_RESET_MASK                                                                 0x00000400L
-#define GDS_GWS_RESET0__RESOURCE11_RESET_MASK                                                                 0x00000800L
-#define GDS_GWS_RESET0__RESOURCE12_RESET_MASK                                                                 0x00001000L
-#define GDS_GWS_RESET0__RESOURCE13_RESET_MASK                                                                 0x00002000L
-#define GDS_GWS_RESET0__RESOURCE14_RESET_MASK                                                                 0x00004000L
-#define GDS_GWS_RESET0__RESOURCE15_RESET_MASK                                                                 0x00008000L
-#define GDS_GWS_RESET0__RESOURCE16_RESET_MASK                                                                 0x00010000L
-#define GDS_GWS_RESET0__RESOURCE17_RESET_MASK                                                                 0x00020000L
-#define GDS_GWS_RESET0__RESOURCE18_RESET_MASK                                                                 0x00040000L
-#define GDS_GWS_RESET0__RESOURCE19_RESET_MASK                                                                 0x00080000L
-#define GDS_GWS_RESET0__RESOURCE20_RESET_MASK                                                                 0x00100000L
-#define GDS_GWS_RESET0__RESOURCE21_RESET_MASK                                                                 0x00200000L
-#define GDS_GWS_RESET0__RESOURCE22_RESET_MASK                                                                 0x00400000L
-#define GDS_GWS_RESET0__RESOURCE23_RESET_MASK                                                                 0x00800000L
-#define GDS_GWS_RESET0__RESOURCE24_RESET_MASK                                                                 0x01000000L
-#define GDS_GWS_RESET0__RESOURCE25_RESET_MASK                                                                 0x02000000L
-#define GDS_GWS_RESET0__RESOURCE26_RESET_MASK                                                                 0x04000000L
-#define GDS_GWS_RESET0__RESOURCE27_RESET_MASK                                                                 0x08000000L
-#define GDS_GWS_RESET0__RESOURCE28_RESET_MASK                                                                 0x10000000L
-#define GDS_GWS_RESET0__RESOURCE29_RESET_MASK                                                                 0x20000000L
-#define GDS_GWS_RESET0__RESOURCE30_RESET_MASK                                                                 0x40000000L
-#define GDS_GWS_RESET0__RESOURCE31_RESET_MASK                                                                 0x80000000L
-//GDS_GWS_RESET1
-#define GDS_GWS_RESET1__RESOURCE32_RESET__SHIFT                                                               0x0
-#define GDS_GWS_RESET1__RESOURCE33_RESET__SHIFT                                                               0x1
-#define GDS_GWS_RESET1__RESOURCE34_RESET__SHIFT                                                               0x2
-#define GDS_GWS_RESET1__RESOURCE35_RESET__SHIFT                                                               0x3
-#define GDS_GWS_RESET1__RESOURCE36_RESET__SHIFT                                                               0x4
-#define GDS_GWS_RESET1__RESOURCE37_RESET__SHIFT                                                               0x5
-#define GDS_GWS_RESET1__RESOURCE38_RESET__SHIFT                                                               0x6
-#define GDS_GWS_RESET1__RESOURCE39_RESET__SHIFT                                                               0x7
-#define GDS_GWS_RESET1__RESOURCE40_RESET__SHIFT                                                               0x8
-#define GDS_GWS_RESET1__RESOURCE41_RESET__SHIFT                                                               0x9
-#define GDS_GWS_RESET1__RESOURCE42_RESET__SHIFT                                                               0xa
-#define GDS_GWS_RESET1__RESOURCE43_RESET__SHIFT                                                               0xb
-#define GDS_GWS_RESET1__RESOURCE44_RESET__SHIFT                                                               0xc
-#define GDS_GWS_RESET1__RESOURCE45_RESET__SHIFT                                                               0xd
-#define GDS_GWS_RESET1__RESOURCE46_RESET__SHIFT                                                               0xe
-#define GDS_GWS_RESET1__RESOURCE47_RESET__SHIFT                                                               0xf
-#define GDS_GWS_RESET1__RESOURCE48_RESET__SHIFT                                                               0x10
-#define GDS_GWS_RESET1__RESOURCE49_RESET__SHIFT                                                               0x11
-#define GDS_GWS_RESET1__RESOURCE50_RESET__SHIFT                                                               0x12
-#define GDS_GWS_RESET1__RESOURCE51_RESET__SHIFT                                                               0x13
-#define GDS_GWS_RESET1__RESOURCE52_RESET__SHIFT                                                               0x14
-#define GDS_GWS_RESET1__RESOURCE53_RESET__SHIFT                                                               0x15
-#define GDS_GWS_RESET1__RESOURCE54_RESET__SHIFT                                                               0x16
-#define GDS_GWS_RESET1__RESOURCE55_RESET__SHIFT                                                               0x17
-#define GDS_GWS_RESET1__RESOURCE56_RESET__SHIFT                                                               0x18
-#define GDS_GWS_RESET1__RESOURCE57_RESET__SHIFT                                                               0x19
-#define GDS_GWS_RESET1__RESOURCE58_RESET__SHIFT                                                               0x1a
-#define GDS_GWS_RESET1__RESOURCE59_RESET__SHIFT                                                               0x1b
-#define GDS_GWS_RESET1__RESOURCE60_RESET__SHIFT                                                               0x1c
-#define GDS_GWS_RESET1__RESOURCE61_RESET__SHIFT                                                               0x1d
-#define GDS_GWS_RESET1__RESOURCE62_RESET__SHIFT                                                               0x1e
-#define GDS_GWS_RESET1__RESOURCE63_RESET__SHIFT                                                               0x1f
-#define GDS_GWS_RESET1__RESOURCE32_RESET_MASK                                                                 0x00000001L
-#define GDS_GWS_RESET1__RESOURCE33_RESET_MASK                                                                 0x00000002L
-#define GDS_GWS_RESET1__RESOURCE34_RESET_MASK                                                                 0x00000004L
-#define GDS_GWS_RESET1__RESOURCE35_RESET_MASK                                                                 0x00000008L
-#define GDS_GWS_RESET1__RESOURCE36_RESET_MASK                                                                 0x00000010L
-#define GDS_GWS_RESET1__RESOURCE37_RESET_MASK                                                                 0x00000020L
-#define GDS_GWS_RESET1__RESOURCE38_RESET_MASK                                                                 0x00000040L
-#define GDS_GWS_RESET1__RESOURCE39_RESET_MASK                                                                 0x00000080L
-#define GDS_GWS_RESET1__RESOURCE40_RESET_MASK                                                                 0x00000100L
-#define GDS_GWS_RESET1__RESOURCE41_RESET_MASK                                                                 0x00000200L
-#define GDS_GWS_RESET1__RESOURCE42_RESET_MASK                                                                 0x00000400L
-#define GDS_GWS_RESET1__RESOURCE43_RESET_MASK                                                                 0x00000800L
-#define GDS_GWS_RESET1__RESOURCE44_RESET_MASK                                                                 0x00001000L
-#define GDS_GWS_RESET1__RESOURCE45_RESET_MASK                                                                 0x00002000L
-#define GDS_GWS_RESET1__RESOURCE46_RESET_MASK                                                                 0x00004000L
-#define GDS_GWS_RESET1__RESOURCE47_RESET_MASK                                                                 0x00008000L
-#define GDS_GWS_RESET1__RESOURCE48_RESET_MASK                                                                 0x00010000L
-#define GDS_GWS_RESET1__RESOURCE49_RESET_MASK                                                                 0x00020000L
-#define GDS_GWS_RESET1__RESOURCE50_RESET_MASK                                                                 0x00040000L
-#define GDS_GWS_RESET1__RESOURCE51_RESET_MASK                                                                 0x00080000L
-#define GDS_GWS_RESET1__RESOURCE52_RESET_MASK                                                                 0x00100000L
-#define GDS_GWS_RESET1__RESOURCE53_RESET_MASK                                                                 0x00200000L
-#define GDS_GWS_RESET1__RESOURCE54_RESET_MASK                                                                 0x00400000L
-#define GDS_GWS_RESET1__RESOURCE55_RESET_MASK                                                                 0x00800000L
-#define GDS_GWS_RESET1__RESOURCE56_RESET_MASK                                                                 0x01000000L
-#define GDS_GWS_RESET1__RESOURCE57_RESET_MASK                                                                 0x02000000L
-#define GDS_GWS_RESET1__RESOURCE58_RESET_MASK                                                                 0x04000000L
-#define GDS_GWS_RESET1__RESOURCE59_RESET_MASK                                                                 0x08000000L
-#define GDS_GWS_RESET1__RESOURCE60_RESET_MASK                                                                 0x10000000L
-#define GDS_GWS_RESET1__RESOURCE61_RESET_MASK                                                                 0x20000000L
-#define GDS_GWS_RESET1__RESOURCE62_RESET_MASK                                                                 0x40000000L
-#define GDS_GWS_RESET1__RESOURCE63_RESET_MASK                                                                 0x80000000L
-//GDS_GWS_RESOURCE_RESET
-#define GDS_GWS_RESOURCE_RESET__RESET__SHIFT                                                                  0x0
-#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID__SHIFT                                                            0x8
-#define GDS_GWS_RESOURCE_RESET__RESET_MASK                                                                    0x00000001L
-#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID_MASK                                                              0x0000FF00L
-//GDS_COMPUTE_MAX_WAVE_ID
-#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                           0x0
-#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                             0x00000FFFL
-//GDS_OA_RESET_MASK
-#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET__SHIFT                                                       0x0
-#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET__SHIFT                                                       0x1
-#define GDS_OA_RESET_MASK__ME0_CS_RESET__SHIFT                                                                0x2
-#define GDS_OA_RESET_MASK__ME0_GFXHP3D_GS_RESET__SHIFT                                                        0x3
-#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET__SHIFT                                                             0x4
-#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET__SHIFT                                                             0x5
-#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET__SHIFT                                                             0x6
-#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET__SHIFT                                                             0x7
-#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET__SHIFT                                                             0x8
-#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET__SHIFT                                                             0x9
-#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET__SHIFT                                                             0xa
-#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET__SHIFT                                                             0xb
-#define GDS_OA_RESET_MASK__UNUSED1__SHIFT                                                                     0xc
-#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET_MASK                                                         0x00000001L
-#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET_MASK                                                         0x00000002L
-#define GDS_OA_RESET_MASK__ME0_CS_RESET_MASK                                                                  0x00000004L
-#define GDS_OA_RESET_MASK__ME0_GFXHP3D_GS_RESET_MASK                                                          0x00000008L
-#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET_MASK                                                               0x00000010L
-#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET_MASK                                                               0x00000020L
-#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET_MASK                                                               0x00000040L
-#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET_MASK                                                               0x00000080L
-#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET_MASK                                                               0x00000100L
-#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET_MASK                                                               0x00000200L
-#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET_MASK                                                               0x00000400L
-#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET_MASK                                                               0x00000800L
-#define GDS_OA_RESET_MASK__UNUSED1_MASK                                                                       0xFFFFF000L
-//GDS_OA_RESET
-#define GDS_OA_RESET__RESET__SHIFT                                                                            0x0
-#define GDS_OA_RESET__PIPE_ID__SHIFT                                                                          0x8
-#define GDS_OA_RESET__RESET_MASK                                                                              0x00000001L
-#define GDS_OA_RESET__PIPE_ID_MASK                                                                            0x0000FF00L
-//GDS_ENHANCE
-#define GDS_ENHANCE__MISC__SHIFT                                                                              0x0
-#define GDS_ENHANCE__AUTO_INC_INDEX__SHIFT                                                                    0x10
-#define GDS_ENHANCE__CGPG_RESTORE__SHIFT                                                                      0x11
-#define GDS_ENHANCE__RD_BUF_TAG_MISS__SHIFT                                                                   0x12
-#define GDS_ENHANCE__GDSA_PC_CGTS_DIS__SHIFT                                                                  0x13
-#define GDS_ENHANCE__GDSO_PC_CGTS_DIS__SHIFT                                                                  0x14
-#define GDS_ENHANCE__WD_GDS_CSB_OVERRIDE__SHIFT                                                               0x15
-#define GDS_ENHANCE__UNUSED__SHIFT                                                                            0x16
-#define GDS_ENHANCE__MISC_MASK                                                                                0x0000FFFFL
-#define GDS_ENHANCE__AUTO_INC_INDEX_MASK                                                                      0x00010000L
-#define GDS_ENHANCE__CGPG_RESTORE_MASK                                                                        0x00020000L
-#define GDS_ENHANCE__RD_BUF_TAG_MISS_MASK                                                                     0x00040000L
-#define GDS_ENHANCE__GDSA_PC_CGTS_DIS_MASK                                                                    0x00080000L
-#define GDS_ENHANCE__GDSO_PC_CGTS_DIS_MASK                                                                    0x00100000L
-#define GDS_ENHANCE__WD_GDS_CSB_OVERRIDE_MASK                                                                 0x00200000L
-#define GDS_ENHANCE__UNUSED_MASK                                                                              0xFFC00000L
-//GDS_OA_CGPG_RESTORE
-#define GDS_OA_CGPG_RESTORE__VMID__SHIFT                                                                      0x0
-#define GDS_OA_CGPG_RESTORE__MEID__SHIFT                                                                      0x8
-#define GDS_OA_CGPG_RESTORE__PIPEID__SHIFT                                                                    0xc
-#define GDS_OA_CGPG_RESTORE__QUEUEID__SHIFT                                                                   0x10
-#define GDS_OA_CGPG_RESTORE__UNUSED__SHIFT                                                                    0x14
-#define GDS_OA_CGPG_RESTORE__VMID_MASK                                                                        0x000000FFL
-#define GDS_OA_CGPG_RESTORE__MEID_MASK                                                                        0x00000F00L
-#define GDS_OA_CGPG_RESTORE__PIPEID_MASK                                                                      0x0000F000L
-#define GDS_OA_CGPG_RESTORE__QUEUEID_MASK                                                                     0x000F0000L
-#define GDS_OA_CGPG_RESTORE__UNUSED_MASK                                                                      0xFFF00000L
-//GDS_CS_CTXSW_STATUS
-#define GDS_CS_CTXSW_STATUS__R__SHIFT                                                                         0x0
-#define GDS_CS_CTXSW_STATUS__W__SHIFT                                                                         0x1
-#define GDS_CS_CTXSW_STATUS__UNUSED__SHIFT                                                                    0x2
-#define GDS_CS_CTXSW_STATUS__R_MASK                                                                           0x00000001L
-#define GDS_CS_CTXSW_STATUS__W_MASK                                                                           0x00000002L
-#define GDS_CS_CTXSW_STATUS__UNUSED_MASK                                                                      0xFFFFFFFCL
-//GDS_CS_CTXSW_CNT0
-#define GDS_CS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
-#define GDS_CS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
-#define GDS_CS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_CS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
-//GDS_CS_CTXSW_CNT1
-#define GDS_CS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
-#define GDS_CS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
-#define GDS_CS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_CS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
-//GDS_CS_CTXSW_CNT2
-#define GDS_CS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
-#define GDS_CS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
-#define GDS_CS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_CS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
-//GDS_CS_CTXSW_CNT3
-#define GDS_CS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
-#define GDS_CS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
-#define GDS_CS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_CS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
-//GDS_GFX_CTXSW_STATUS
-#define GDS_GFX_CTXSW_STATUS__R__SHIFT                                                                        0x0
-#define GDS_GFX_CTXSW_STATUS__W__SHIFT                                                                        0x1
-#define GDS_GFX_CTXSW_STATUS__UNUSED__SHIFT                                                                   0x2
-#define GDS_GFX_CTXSW_STATUS__R_MASK                                                                          0x00000001L
-#define GDS_GFX_CTXSW_STATUS__W_MASK                                                                          0x00000002L
-#define GDS_GFX_CTXSW_STATUS__UNUSED_MASK                                                                     0xFFFFFFFCL
-//GDS_VS_CTXSW_CNT0
-#define GDS_VS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
-#define GDS_VS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
-#define GDS_VS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_VS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
-//GDS_VS_CTXSW_CNT1
-#define GDS_VS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
-#define GDS_VS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
-#define GDS_VS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_VS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
-//GDS_VS_CTXSW_CNT2
-#define GDS_VS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
-#define GDS_VS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
-#define GDS_VS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_VS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
-//GDS_VS_CTXSW_CNT3
-#define GDS_VS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
-#define GDS_VS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
-#define GDS_VS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_VS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
-//GDS_PS0_CTXSW_CNT0
-#define GDS_PS0_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS0_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS0_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS0_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS0_CTXSW_CNT1
-#define GDS_PS0_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS0_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS0_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS0_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS0_CTXSW_CNT2
-#define GDS_PS0_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS0_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS0_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS0_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS0_CTXSW_CNT3
-#define GDS_PS0_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS0_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS0_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS0_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS1_CTXSW_CNT0
-#define GDS_PS1_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS1_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS1_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS1_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS1_CTXSW_CNT1
-#define GDS_PS1_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS1_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS1_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS1_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS1_CTXSW_CNT2
-#define GDS_PS1_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS1_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS1_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS1_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS1_CTXSW_CNT3
-#define GDS_PS1_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS1_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS1_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS1_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS2_CTXSW_CNT0
-#define GDS_PS2_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS2_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS2_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS2_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS2_CTXSW_CNT1
-#define GDS_PS2_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS2_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS2_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS2_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS2_CTXSW_CNT2
-#define GDS_PS2_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS2_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS2_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS2_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS2_CTXSW_CNT3
-#define GDS_PS2_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS2_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS2_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS2_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS3_CTXSW_CNT0
-#define GDS_PS3_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS3_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS3_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS3_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS3_CTXSW_CNT1
-#define GDS_PS3_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS3_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS3_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS3_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS3_CTXSW_CNT2
-#define GDS_PS3_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS3_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS3_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS3_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS3_CTXSW_CNT3
-#define GDS_PS3_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS3_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS3_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS3_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS4_CTXSW_CNT0
-#define GDS_PS4_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS4_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS4_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS4_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS4_CTXSW_CNT1
-#define GDS_PS4_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS4_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS4_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS4_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS4_CTXSW_CNT2
-#define GDS_PS4_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS4_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS4_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS4_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS4_CTXSW_CNT3
-#define GDS_PS4_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS4_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS4_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS4_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS5_CTXSW_CNT0
-#define GDS_PS5_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS5_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS5_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS5_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS5_CTXSW_CNT1
-#define GDS_PS5_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS5_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS5_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS5_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS5_CTXSW_CNT2
-#define GDS_PS5_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS5_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS5_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS5_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS5_CTXSW_CNT3
-#define GDS_PS5_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS5_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS5_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS5_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS6_CTXSW_CNT0
-#define GDS_PS6_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS6_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS6_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS6_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS6_CTXSW_CNT1
-#define GDS_PS6_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS6_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS6_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS6_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS6_CTXSW_CNT2
-#define GDS_PS6_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS6_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS6_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS6_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS6_CTXSW_CNT3
-#define GDS_PS6_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS6_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS6_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS6_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS7_CTXSW_CNT0
-#define GDS_PS7_CTXSW_CNT0__UPDN__SHIFT                                                                       0x0
-#define GDS_PS7_CTXSW_CNT0__PTR__SHIFT                                                                        0x10
-#define GDS_PS7_CTXSW_CNT0__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS7_CTXSW_CNT0__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS7_CTXSW_CNT1
-#define GDS_PS7_CTXSW_CNT1__UPDN__SHIFT                                                                       0x0
-#define GDS_PS7_CTXSW_CNT1__PTR__SHIFT                                                                        0x10
-#define GDS_PS7_CTXSW_CNT1__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS7_CTXSW_CNT1__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS7_CTXSW_CNT2
-#define GDS_PS7_CTXSW_CNT2__UPDN__SHIFT                                                                       0x0
-#define GDS_PS7_CTXSW_CNT2__PTR__SHIFT                                                                        0x10
-#define GDS_PS7_CTXSW_CNT2__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS7_CTXSW_CNT2__PTR_MASK                                                                          0xFFFF0000L
-//GDS_PS7_CTXSW_CNT3
-#define GDS_PS7_CTXSW_CNT3__UPDN__SHIFT                                                                       0x0
-#define GDS_PS7_CTXSW_CNT3__PTR__SHIFT                                                                        0x10
-#define GDS_PS7_CTXSW_CNT3__UPDN_MASK                                                                         0x0000FFFFL
-#define GDS_PS7_CTXSW_CNT3__PTR_MASK                                                                          0xFFFF0000L
-//GDS_GS_CTXSW_CNT0
-#define GDS_GS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
-#define GDS_GS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
-#define GDS_GS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_GS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
-//GDS_GS_CTXSW_CNT1
-#define GDS_GS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
-#define GDS_GS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
-#define GDS_GS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_GS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
-//GDS_GS_CTXSW_CNT2
-#define GDS_GS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
-#define GDS_GS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
-#define GDS_GS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_GS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
-//GDS_GS_CTXSW_CNT3
-#define GDS_GS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
-#define GDS_GS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
-#define GDS_GS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
-#define GDS_GS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
-
-
-// addressBlock: gc_rasdec
-//RAS_SIGNATURE_CONTROL
-#define RAS_SIGNATURE_CONTROL__ENABLE__SHIFT                                                                  0x0
-#define RAS_SIGNATURE_CONTROL__ENABLE_MASK                                                                    0x00000001L
-//RAS_SIGNATURE_MASK
-#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK__SHIFT                                                             0x0
-#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK_MASK                                                               0xFFFFFFFFL
-//RAS_SX_SIGNATURE0
-#define RAS_SX_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SX_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SX_SIGNATURE1
-#define RAS_SX_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SX_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SX_SIGNATURE2
-#define RAS_SX_SIGNATURE2__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SX_SIGNATURE2__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SX_SIGNATURE3
-#define RAS_SX_SIGNATURE3__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SX_SIGNATURE3__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_DB_SIGNATURE0
-#define RAS_DB_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_DB_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_PA_SIGNATURE0
-#define RAS_PA_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_PA_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_VGT_SIGNATURE0
-#define RAS_VGT_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
-#define RAS_VGT_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
-//RAS_SQ_SIGNATURE0
-#define RAS_SQ_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SQ_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE0
-#define RAS_SC_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE1
-#define RAS_SC_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE2
-#define RAS_SC_SIGNATURE2__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE2__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE3
-#define RAS_SC_SIGNATURE3__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE3__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE4
-#define RAS_SC_SIGNATURE4__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE4__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE5
-#define RAS_SC_SIGNATURE5__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE5__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE6
-#define RAS_SC_SIGNATURE6__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE6__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SC_SIGNATURE7
-#define RAS_SC_SIGNATURE7__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_SC_SIGNATURE7__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_IA_SIGNATURE0
-#define RAS_IA_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_IA_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_IA_SIGNATURE1
-#define RAS_IA_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_IA_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_SPI_SIGNATURE0
-#define RAS_SPI_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
-#define RAS_SPI_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
-//RAS_SPI_SIGNATURE1
-#define RAS_SPI_SIGNATURE1__SIGNATURE__SHIFT                                                                  0x0
-#define RAS_SPI_SIGNATURE1__SIGNATURE_MASK                                                                    0xFFFFFFFFL
-//RAS_TA_SIGNATURE0
-#define RAS_TA_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_TA_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_TD_SIGNATURE0
-#define RAS_TD_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_TD_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_CB_SIGNATURE0
-#define RAS_CB_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_CB_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-//RAS_BCI_SIGNATURE0
-#define RAS_BCI_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
-#define RAS_BCI_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
-//RAS_BCI_SIGNATURE1
-#define RAS_BCI_SIGNATURE1__SIGNATURE__SHIFT                                                                  0x0
-#define RAS_BCI_SIGNATURE1__SIGNATURE_MASK                                                                    0xFFFFFFFFL
-//RAS_TA_SIGNATURE1
-#define RAS_TA_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
-#define RAS_TA_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
-
-
-// addressBlock: gc_gfxdec0
-//DB_RENDER_CONTROL
-#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE__SHIFT                                                          0x0
-#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE__SHIFT                                                        0x1
-#define DB_RENDER_CONTROL__DEPTH_COPY__SHIFT                                                                  0x2
-#define DB_RENDER_CONTROL__STENCIL_COPY__SHIFT                                                                0x3
-#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE__SHIFT                                                          0x4
-#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE__SHIFT                                                    0x5
-#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE__SHIFT                                                      0x6
-#define DB_RENDER_CONTROL__COPY_CENTROID__SHIFT                                                               0x7
-#define DB_RENDER_CONTROL__COPY_SAMPLE__SHIFT                                                                 0x8
-#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE__SHIFT                                                           0xc
-#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE_MASK                                                            0x00000001L
-#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define DB_RENDER_CONTROL__DEPTH_COPY_MASK                                                                    0x00000004L
-#define DB_RENDER_CONTROL__STENCIL_COPY_MASK                                                                  0x00000008L
-#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE_MASK                                                            0x00000010L
-#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE_MASK                                                      0x00000020L
-#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE_MASK                                                        0x00000040L
-#define DB_RENDER_CONTROL__COPY_CENTROID_MASK                                                                 0x00000080L
-#define DB_RENDER_CONTROL__COPY_SAMPLE_MASK                                                                   0x00000F00L
-#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE_MASK                                                             0x00001000L
-//DB_COUNT_CONTROL
-#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE__SHIFT                                                      0x0
-#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS__SHIFT                                                         0x1
-#define DB_COUNT_CONTROL__SAMPLE_RATE__SHIFT                                                                  0x4
-#define DB_COUNT_CONTROL__ZPASS_ENABLE__SHIFT                                                                 0x8
-#define DB_COUNT_CONTROL__ZFAIL_ENABLE__SHIFT                                                                 0xc
-#define DB_COUNT_CONTROL__SFAIL_ENABLE__SHIFT                                                                 0x10
-#define DB_COUNT_CONTROL__DBFAIL_ENABLE__SHIFT                                                                0x14
-#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                            0x18
-#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                             0x1c
-#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE_MASK                                                        0x00000001L
-#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS_MASK                                                           0x00000002L
-#define DB_COUNT_CONTROL__SAMPLE_RATE_MASK                                                                    0x00000070L
-#define DB_COUNT_CONTROL__ZPASS_ENABLE_MASK                                                                   0x00000F00L
-#define DB_COUNT_CONTROL__ZFAIL_ENABLE_MASK                                                                   0x0000F000L
-#define DB_COUNT_CONTROL__SFAIL_ENABLE_MASK                                                                   0x000F0000L
-#define DB_COUNT_CONTROL__DBFAIL_ENABLE_MASK                                                                  0x00F00000L
-#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                              0x0F000000L
-#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE_MASK                                                               0xF0000000L
-//DB_DEPTH_VIEW
-#define DB_DEPTH_VIEW__SLICE_START__SHIFT                                                                     0x0
-#define DB_DEPTH_VIEW__SLICE_MAX__SHIFT                                                                       0xd
-#define DB_DEPTH_VIEW__Z_READ_ONLY__SHIFT                                                                     0x18
-#define DB_DEPTH_VIEW__STENCIL_READ_ONLY__SHIFT                                                               0x19
-#define DB_DEPTH_VIEW__MIPID__SHIFT                                                                           0x1a
-#define DB_DEPTH_VIEW__SLICE_START_MASK                                                                       0x000007FFL
-#define DB_DEPTH_VIEW__SLICE_MAX_MASK                                                                         0x00FFE000L
-#define DB_DEPTH_VIEW__Z_READ_ONLY_MASK                                                                       0x01000000L
-#define DB_DEPTH_VIEW__STENCIL_READ_ONLY_MASK                                                                 0x02000000L
-#define DB_DEPTH_VIEW__MIPID_MASK                                                                             0x3C000000L
-//DB_RENDER_OVERRIDE
-#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT                                                           0x0
-#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0__SHIFT                                                          0x2
-#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1__SHIFT                                                          0x4
-#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER__SHIFT                                                       0x6
-#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT                                                             0x7
-#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT                                                       0x8
-#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE__SHIFT                                                          0x9
-#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL__SHIFT                                                           0xa
-#define DB_RENDER_OVERRIDE__FORCE_Z_READ__SHIFT                                                               0xb
-#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ__SHIFT                                                         0xc
-#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT                                                         0xd
-#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT__SHIFT                                                    0xf
-#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP__SHIFT                                                     0x10
-#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE__SHIFT                                                           0x11
-#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED__SHIFT                                                      0x12
-#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM__SHIFT                                                         0x13
-#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT__SHIFT                                                           0x15
-#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT                                                    0x1a
-#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY__SHIFT                                                              0x1b
-#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY__SHIFT                                                        0x1c
-#define DB_RENDER_OVERRIDE__FORCE_Z_VALID__SHIFT                                                              0x1d
-#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID__SHIFT                                                        0x1e
-#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION__SHIFT                                                       0x1f
-#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK                                                             0x00000003L
-#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0_MASK                                                            0x0000000CL
-#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1_MASK                                                            0x00000030L
-#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER_MASK                                                         0x00000040L
-#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK                                                               0x00000080L
-#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK                                                         0x00000100L
-#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE_MASK                                                            0x00000200L
-#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL_MASK                                                             0x00000400L
-#define DB_RENDER_OVERRIDE__FORCE_Z_READ_MASK                                                                 0x00000800L
-#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ_MASK                                                           0x00001000L
-#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK                                                           0x00006000L
-#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT_MASK                                                      0x00008000L
-#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP_MASK                                                       0x00010000L
-#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE_MASK                                                             0x00020000L
-#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED_MASK                                                        0x00040000L
-#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM_MASK                                                           0x00180000L
-#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT_MASK                                                             0x03E00000L
-#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK                                                      0x04000000L
-#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY_MASK                                                                0x08000000L
-#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY_MASK                                                          0x10000000L
-#define DB_RENDER_OVERRIDE__FORCE_Z_VALID_MASK                                                                0x20000000L
-#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID_MASK                                                          0x40000000L
-#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION_MASK                                                         0x80000000L
-//DB_RENDER_OVERRIDE2
-#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL__SHIFT                                              0x0
-#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN__SHIFT                                            0x2
-#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION__SHIFT                                       0x5
-#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION__SHIFT                                        0x6
-#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION__SHIFT                                               0x7
-#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH__SHIFT                                                     0x8
-#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP__SHIFT                                                         0x9
-#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT                                           0xa
-#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE__SHIFT                                                 0xb
-#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC__SHIFT                                                                 0xc
-#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF__SHIFT                                                              0xf
-#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF__SHIFT                                                              0x12
-#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE__SHIFT                                                           0x15
-#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS__SHIFT                                                         0x16
-#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT                                                         0x17
-#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL__SHIFT                                               0x19
-#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK                                                0x00000003L
-#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK                                              0x0000001CL
-#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION_MASK                                         0x00000020L
-#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION_MASK                                          0x00000040L
-#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION_MASK                                                 0x00000080L
-#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH_MASK                                                       0x00000100L
-#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP_MASK                                                           0x00000200L
-#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK                                             0x00000400L
-#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE_MASK                                                   0x00000800L
-#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC_MASK                                                                   0x00007000L
-#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF_MASK                                                                0x00038000L
-#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF_MASK                                                                0x001C0000L
-#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE_MASK                                                             0x00200000L
-#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS_MASK                                                           0x00400000L
-#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK                                                           0x00800000L
-#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL_MASK                                                 0x02000000L
-//DB_HTILE_DATA_BASE
-#define DB_HTILE_DATA_BASE__BASE_256B__SHIFT                                                                  0x0
-#define DB_HTILE_DATA_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//DB_HTILE_DATA_BASE_HI
-#define DB_HTILE_DATA_BASE_HI__BASE_HI__SHIFT                                                                 0x0
-#define DB_HTILE_DATA_BASE_HI__BASE_HI_MASK                                                                   0x000000FFL
-//DB_DEPTH_SIZE
-#define DB_DEPTH_SIZE__X_MAX__SHIFT                                                                           0x0
-#define DB_DEPTH_SIZE__Y_MAX__SHIFT                                                                           0x10
-#define DB_DEPTH_SIZE__X_MAX_MASK                                                                             0x00003FFFL
-#define DB_DEPTH_SIZE__Y_MAX_MASK                                                                             0x3FFF0000L
-//DB_DEPTH_BOUNDS_MIN
-#define DB_DEPTH_BOUNDS_MIN__MIN__SHIFT                                                                       0x0
-#define DB_DEPTH_BOUNDS_MIN__MIN_MASK                                                                         0xFFFFFFFFL
-//DB_DEPTH_BOUNDS_MAX
-#define DB_DEPTH_BOUNDS_MAX__MAX__SHIFT                                                                       0x0
-#define DB_DEPTH_BOUNDS_MAX__MAX_MASK                                                                         0xFFFFFFFFL
-//DB_STENCIL_CLEAR
-#define DB_STENCIL_CLEAR__CLEAR__SHIFT                                                                        0x0
-#define DB_STENCIL_CLEAR__CLEAR_MASK                                                                          0x000000FFL
-//DB_DEPTH_CLEAR
-#define DB_DEPTH_CLEAR__DEPTH_CLEAR__SHIFT                                                                    0x0
-#define DB_DEPTH_CLEAR__DEPTH_CLEAR_MASK                                                                      0xFFFFFFFFL
-//PA_SC_SCREEN_SCISSOR_TL
-#define PA_SC_SCREEN_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
-#define PA_SC_SCREEN_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
-#define PA_SC_SCREEN_SCISSOR_TL__TL_X_MASK                                                                    0x0000FFFFL
-#define PA_SC_SCREEN_SCISSOR_TL__TL_Y_MASK                                                                    0xFFFF0000L
-//PA_SC_SCREEN_SCISSOR_BR
-#define PA_SC_SCREEN_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
-#define PA_SC_SCREEN_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
-#define PA_SC_SCREEN_SCISSOR_BR__BR_X_MASK                                                                    0x0000FFFFL
-#define PA_SC_SCREEN_SCISSOR_BR__BR_Y_MASK                                                                    0xFFFF0000L
-//DB_Z_INFO
-#define DB_Z_INFO__FORMAT__SHIFT                                                                              0x0
-#define DB_Z_INFO__NUM_SAMPLES__SHIFT                                                                         0x2
-#define DB_Z_INFO__SW_MODE__SHIFT                                                                             0x4
-#define DB_Z_INFO__PARTIALLY_RESIDENT__SHIFT                                                                  0xc
-#define DB_Z_INFO__FAULT_BEHAVIOR__SHIFT                                                                      0xd
-#define DB_Z_INFO__ITERATE_FLUSH__SHIFT                                                                       0xf
-#define DB_Z_INFO__MAXMIP__SHIFT                                                                              0x10
-#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES__SHIFT                                                             0x17
-#define DB_Z_INFO__ALLOW_EXPCLEAR__SHIFT                                                                      0x1b
-#define DB_Z_INFO__READ_SIZE__SHIFT                                                                           0x1c
-#define DB_Z_INFO__TILE_SURFACE_ENABLE__SHIFT                                                                 0x1d
-#define DB_Z_INFO__CLEAR_DISALLOWED__SHIFT                                                                    0x1e
-#define DB_Z_INFO__ZRANGE_PRECISION__SHIFT                                                                    0x1f
-#define DB_Z_INFO__FORMAT_MASK                                                                                0x00000003L
-#define DB_Z_INFO__NUM_SAMPLES_MASK                                                                           0x0000000CL
-#define DB_Z_INFO__SW_MODE_MASK                                                                               0x000001F0L
-#define DB_Z_INFO__PARTIALLY_RESIDENT_MASK                                                                    0x00001000L
-#define DB_Z_INFO__FAULT_BEHAVIOR_MASK                                                                        0x00006000L
-#define DB_Z_INFO__ITERATE_FLUSH_MASK                                                                         0x00008000L
-#define DB_Z_INFO__MAXMIP_MASK                                                                                0x000F0000L
-#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES_MASK                                                               0x07800000L
-#define DB_Z_INFO__ALLOW_EXPCLEAR_MASK                                                                        0x08000000L
-#define DB_Z_INFO__READ_SIZE_MASK                                                                             0x10000000L
-#define DB_Z_INFO__TILE_SURFACE_ENABLE_MASK                                                                   0x20000000L
-#define DB_Z_INFO__CLEAR_DISALLOWED_MASK                                                                      0x40000000L
-#define DB_Z_INFO__ZRANGE_PRECISION_MASK                                                                      0x80000000L
-//DB_STENCIL_INFO
-#define DB_STENCIL_INFO__FORMAT__SHIFT                                                                        0x0
-#define DB_STENCIL_INFO__SW_MODE__SHIFT                                                                       0x4
-#define DB_STENCIL_INFO__PARTIALLY_RESIDENT__SHIFT                                                            0xc
-#define DB_STENCIL_INFO__FAULT_BEHAVIOR__SHIFT                                                                0xd
-#define DB_STENCIL_INFO__ITERATE_FLUSH__SHIFT                                                                 0xf
-#define DB_STENCIL_INFO__ALLOW_EXPCLEAR__SHIFT                                                                0x1b
-#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE__SHIFT                                                          0x1d
-#define DB_STENCIL_INFO__CLEAR_DISALLOWED__SHIFT                                                              0x1e
-#define DB_STENCIL_INFO__FORMAT_MASK                                                                          0x00000001L
-#define DB_STENCIL_INFO__SW_MODE_MASK                                                                         0x000001F0L
-#define DB_STENCIL_INFO__PARTIALLY_RESIDENT_MASK                                                              0x00001000L
-#define DB_STENCIL_INFO__FAULT_BEHAVIOR_MASK                                                                  0x00006000L
-#define DB_STENCIL_INFO__ITERATE_FLUSH_MASK                                                                   0x00008000L
-#define DB_STENCIL_INFO__ALLOW_EXPCLEAR_MASK                                                                  0x08000000L
-#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE_MASK                                                            0x20000000L
-#define DB_STENCIL_INFO__CLEAR_DISALLOWED_MASK                                                                0x40000000L
-//DB_Z_READ_BASE
-#define DB_Z_READ_BASE__BASE_256B__SHIFT                                                                      0x0
-#define DB_Z_READ_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//DB_Z_READ_BASE_HI
-#define DB_Z_READ_BASE_HI__BASE_HI__SHIFT                                                                     0x0
-#define DB_Z_READ_BASE_HI__BASE_HI_MASK                                                                       0x000000FFL
-//DB_STENCIL_READ_BASE
-#define DB_STENCIL_READ_BASE__BASE_256B__SHIFT                                                                0x0
-#define DB_STENCIL_READ_BASE__BASE_256B_MASK                                                                  0xFFFFFFFFL
-//DB_STENCIL_READ_BASE_HI
-#define DB_STENCIL_READ_BASE_HI__BASE_HI__SHIFT                                                               0x0
-#define DB_STENCIL_READ_BASE_HI__BASE_HI_MASK                                                                 0x000000FFL
-//DB_Z_WRITE_BASE
-#define DB_Z_WRITE_BASE__BASE_256B__SHIFT                                                                     0x0
-#define DB_Z_WRITE_BASE__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//DB_Z_WRITE_BASE_HI
-#define DB_Z_WRITE_BASE_HI__BASE_HI__SHIFT                                                                    0x0
-#define DB_Z_WRITE_BASE_HI__BASE_HI_MASK                                                                      0x000000FFL
-//DB_STENCIL_WRITE_BASE
-#define DB_STENCIL_WRITE_BASE__BASE_256B__SHIFT                                                               0x0
-#define DB_STENCIL_WRITE_BASE__BASE_256B_MASK                                                                 0xFFFFFFFFL
-//DB_STENCIL_WRITE_BASE_HI
-#define DB_STENCIL_WRITE_BASE_HI__BASE_HI__SHIFT                                                              0x0
-#define DB_STENCIL_WRITE_BASE_HI__BASE_HI_MASK                                                                0x000000FFL
-//DB_DFSM_CONTROL
-#define DB_DFSM_CONTROL__PUNCHOUT_MODE__SHIFT                                                                 0x0
-#define DB_DFSM_CONTROL__POPS_DRAIN_PS_ON_OVERLAP__SHIFT                                                      0x2
-#define DB_DFSM_CONTROL__DISALLOW_OVERFLOW__SHIFT                                                             0x3
-#define DB_DFSM_CONTROL__PUNCHOUT_MODE_MASK                                                                   0x00000003L
-#define DB_DFSM_CONTROL__POPS_DRAIN_PS_ON_OVERLAP_MASK                                                        0x00000004L
-#define DB_DFSM_CONTROL__DISALLOW_OVERFLOW_MASK                                                               0x00000008L
-//DB_RENDER_FILTER
-#define DB_RENDER_FILTER__PS_INVOKE_MASK__SHIFT                                                               0x0
-#define DB_RENDER_FILTER__PS_INVOKE_MASK_MASK                                                                 0x0000FFFFL
-//DB_Z_INFO2
-#define DB_Z_INFO2__EPITCH__SHIFT                                                                             0x0
-#define DB_Z_INFO2__EPITCH_MASK                                                                               0x0000FFFFL
-//DB_STENCIL_INFO2
-#define DB_STENCIL_INFO2__EPITCH__SHIFT                                                                       0x0
-#define DB_STENCIL_INFO2__EPITCH_MASK                                                                         0x0000FFFFL
-//TA_BC_BASE_ADDR
-#define TA_BC_BASE_ADDR__ADDRESS__SHIFT                                                                       0x0
-#define TA_BC_BASE_ADDR__ADDRESS_MASK                                                                         0xFFFFFFFFL
-//TA_BC_BASE_ADDR_HI
-#define TA_BC_BASE_ADDR_HI__ADDRESS__SHIFT                                                                    0x0
-#define TA_BC_BASE_ADDR_HI__ADDRESS_MASK                                                                      0x000000FFL
-//COHER_DEST_BASE_HI_0
-#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B__SHIFT                                                        0x0
-#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
-//COHER_DEST_BASE_HI_1
-#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B__SHIFT                                                        0x0
-#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
-//COHER_DEST_BASE_HI_2
-#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B__SHIFT                                                        0x0
-#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
-//COHER_DEST_BASE_HI_3
-#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B__SHIFT                                                        0x0
-#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
-//COHER_DEST_BASE_2
-#define COHER_DEST_BASE_2__DEST_BASE_256B__SHIFT                                                              0x0
-#define COHER_DEST_BASE_2__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
-//COHER_DEST_BASE_3
-#define COHER_DEST_BASE_3__DEST_BASE_256B__SHIFT                                                              0x0
-#define COHER_DEST_BASE_3__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
-//PA_SC_WINDOW_OFFSET
-#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET__SHIFT                                                           0x0
-#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET__SHIFT                                                           0x10
-#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET_MASK                                                             0x0000FFFFL
-#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET_MASK                                                             0xFFFF0000L
-//PA_SC_WINDOW_SCISSOR_TL
-#define PA_SC_WINDOW_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
-#define PA_SC_WINDOW_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
-#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                 0x1f
-#define PA_SC_WINDOW_SCISSOR_TL__TL_X_MASK                                                                    0x00007FFFL
-#define PA_SC_WINDOW_SCISSOR_TL__TL_Y_MASK                                                                    0x7FFF0000L
-#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK                                                   0x80000000L
-//PA_SC_WINDOW_SCISSOR_BR
-#define PA_SC_WINDOW_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
-#define PA_SC_WINDOW_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
-#define PA_SC_WINDOW_SCISSOR_BR__BR_X_MASK                                                                    0x00007FFFL
-#define PA_SC_WINDOW_SCISSOR_BR__BR_Y_MASK                                                                    0x7FFF0000L
-//PA_SC_CLIPRECT_RULE
-#define PA_SC_CLIPRECT_RULE__CLIP_RULE__SHIFT                                                                 0x0
-#define PA_SC_CLIPRECT_RULE__CLIP_RULE_MASK                                                                   0x0000FFFFL
-//PA_SC_CLIPRECT_0_TL
-#define PA_SC_CLIPRECT_0_TL__TL_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_0_TL__TL_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_0_TL__TL_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_0_TL__TL_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_CLIPRECT_0_BR
-#define PA_SC_CLIPRECT_0_BR__BR_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_0_BR__BR_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_0_BR__BR_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_0_BR__BR_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_CLIPRECT_1_TL
-#define PA_SC_CLIPRECT_1_TL__TL_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_1_TL__TL_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_1_TL__TL_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_1_TL__TL_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_CLIPRECT_1_BR
-#define PA_SC_CLIPRECT_1_BR__BR_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_1_BR__BR_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_1_BR__BR_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_1_BR__BR_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_CLIPRECT_2_TL
-#define PA_SC_CLIPRECT_2_TL__TL_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_2_TL__TL_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_2_TL__TL_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_2_TL__TL_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_CLIPRECT_2_BR
-#define PA_SC_CLIPRECT_2_BR__BR_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_2_BR__BR_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_2_BR__BR_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_2_BR__BR_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_CLIPRECT_3_TL
-#define PA_SC_CLIPRECT_3_TL__TL_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_3_TL__TL_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_3_TL__TL_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_3_TL__TL_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_CLIPRECT_3_BR
-#define PA_SC_CLIPRECT_3_BR__BR_X__SHIFT                                                                      0x0
-#define PA_SC_CLIPRECT_3_BR__BR_Y__SHIFT                                                                      0x10
-#define PA_SC_CLIPRECT_3_BR__BR_X_MASK                                                                        0x00007FFFL
-#define PA_SC_CLIPRECT_3_BR__BR_Y_MASK                                                                        0x7FFF0000L
-//PA_SC_EDGERULE
-#define PA_SC_EDGERULE__ER_TRI__SHIFT                                                                         0x0
-#define PA_SC_EDGERULE__ER_POINT__SHIFT                                                                       0x4
-#define PA_SC_EDGERULE__ER_RECT__SHIFT                                                                        0x8
-#define PA_SC_EDGERULE__ER_LINE_LR__SHIFT                                                                     0xc
-#define PA_SC_EDGERULE__ER_LINE_RL__SHIFT                                                                     0x12
-#define PA_SC_EDGERULE__ER_LINE_TB__SHIFT                                                                     0x18
-#define PA_SC_EDGERULE__ER_LINE_BT__SHIFT                                                                     0x1c
-#define PA_SC_EDGERULE__ER_TRI_MASK                                                                           0x0000000FL
-#define PA_SC_EDGERULE__ER_POINT_MASK                                                                         0x000000F0L
-#define PA_SC_EDGERULE__ER_RECT_MASK                                                                          0x00000F00L
-#define PA_SC_EDGERULE__ER_LINE_LR_MASK                                                                       0x0003F000L
-#define PA_SC_EDGERULE__ER_LINE_RL_MASK                                                                       0x00FC0000L
-#define PA_SC_EDGERULE__ER_LINE_TB_MASK                                                                       0x0F000000L
-#define PA_SC_EDGERULE__ER_LINE_BT_MASK                                                                       0xF0000000L
-//PA_SU_HARDWARE_SCREEN_OFFSET
-#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X__SHIFT                                               0x0
-#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y__SHIFT                                               0x10
-#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X_MASK                                                 0x000001FFL
-#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y_MASK                                                 0x01FF0000L
-//CB_TARGET_MASK
-#define CB_TARGET_MASK__TARGET0_ENABLE__SHIFT                                                                 0x0
-#define CB_TARGET_MASK__TARGET1_ENABLE__SHIFT                                                                 0x4
-#define CB_TARGET_MASK__TARGET2_ENABLE__SHIFT                                                                 0x8
-#define CB_TARGET_MASK__TARGET3_ENABLE__SHIFT                                                                 0xc
-#define CB_TARGET_MASK__TARGET4_ENABLE__SHIFT                                                                 0x10
-#define CB_TARGET_MASK__TARGET5_ENABLE__SHIFT                                                                 0x14
-#define CB_TARGET_MASK__TARGET6_ENABLE__SHIFT                                                                 0x18
-#define CB_TARGET_MASK__TARGET7_ENABLE__SHIFT                                                                 0x1c
-#define CB_TARGET_MASK__TARGET0_ENABLE_MASK                                                                   0x0000000FL
-#define CB_TARGET_MASK__TARGET1_ENABLE_MASK                                                                   0x000000F0L
-#define CB_TARGET_MASK__TARGET2_ENABLE_MASK                                                                   0x00000F00L
-#define CB_TARGET_MASK__TARGET3_ENABLE_MASK                                                                   0x0000F000L
-#define CB_TARGET_MASK__TARGET4_ENABLE_MASK                                                                   0x000F0000L
-#define CB_TARGET_MASK__TARGET5_ENABLE_MASK                                                                   0x00F00000L
-#define CB_TARGET_MASK__TARGET6_ENABLE_MASK                                                                   0x0F000000L
-#define CB_TARGET_MASK__TARGET7_ENABLE_MASK                                                                   0xF0000000L
-//CB_SHADER_MASK
-#define CB_SHADER_MASK__OUTPUT0_ENABLE__SHIFT                                                                 0x0
-#define CB_SHADER_MASK__OUTPUT1_ENABLE__SHIFT                                                                 0x4
-#define CB_SHADER_MASK__OUTPUT2_ENABLE__SHIFT                                                                 0x8
-#define CB_SHADER_MASK__OUTPUT3_ENABLE__SHIFT                                                                 0xc
-#define CB_SHADER_MASK__OUTPUT4_ENABLE__SHIFT                                                                 0x10
-#define CB_SHADER_MASK__OUTPUT5_ENABLE__SHIFT                                                                 0x14
-#define CB_SHADER_MASK__OUTPUT6_ENABLE__SHIFT                                                                 0x18
-#define CB_SHADER_MASK__OUTPUT7_ENABLE__SHIFT                                                                 0x1c
-#define CB_SHADER_MASK__OUTPUT0_ENABLE_MASK                                                                   0x0000000FL
-#define CB_SHADER_MASK__OUTPUT1_ENABLE_MASK                                                                   0x000000F0L
-#define CB_SHADER_MASK__OUTPUT2_ENABLE_MASK                                                                   0x00000F00L
-#define CB_SHADER_MASK__OUTPUT3_ENABLE_MASK                                                                   0x0000F000L
-#define CB_SHADER_MASK__OUTPUT4_ENABLE_MASK                                                                   0x000F0000L
-#define CB_SHADER_MASK__OUTPUT5_ENABLE_MASK                                                                   0x00F00000L
-#define CB_SHADER_MASK__OUTPUT6_ENABLE_MASK                                                                   0x0F000000L
-#define CB_SHADER_MASK__OUTPUT7_ENABLE_MASK                                                                   0xF0000000L
-//PA_SC_GENERIC_SCISSOR_TL
-#define PA_SC_GENERIC_SCISSOR_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_GENERIC_SCISSOR_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_GENERIC_SCISSOR_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_GENERIC_SCISSOR_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_GENERIC_SCISSOR_BR
-#define PA_SC_GENERIC_SCISSOR_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_GENERIC_SCISSOR_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_GENERIC_SCISSOR_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_GENERIC_SCISSOR_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//COHER_DEST_BASE_0
-#define COHER_DEST_BASE_0__DEST_BASE_256B__SHIFT                                                              0x0
-#define COHER_DEST_BASE_0__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
-//COHER_DEST_BASE_1
-#define COHER_DEST_BASE_1__DEST_BASE_256B__SHIFT                                                              0x0
-#define COHER_DEST_BASE_1__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
-//PA_SC_VPORT_SCISSOR_0_TL
-#define PA_SC_VPORT_SCISSOR_0_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_0_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_0_BR
-#define PA_SC_VPORT_SCISSOR_0_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_0_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_1_TL
-#define PA_SC_VPORT_SCISSOR_1_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_1_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_1_BR
-#define PA_SC_VPORT_SCISSOR_1_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_1_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_2_TL
-#define PA_SC_VPORT_SCISSOR_2_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_2_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_2_BR
-#define PA_SC_VPORT_SCISSOR_2_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_2_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_3_TL
-#define PA_SC_VPORT_SCISSOR_3_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_3_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_3_BR
-#define PA_SC_VPORT_SCISSOR_3_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_3_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_4_TL
-#define PA_SC_VPORT_SCISSOR_4_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_4_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_4_BR
-#define PA_SC_VPORT_SCISSOR_4_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_4_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_5_TL
-#define PA_SC_VPORT_SCISSOR_5_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_5_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_5_BR
-#define PA_SC_VPORT_SCISSOR_5_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_5_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_6_TL
-#define PA_SC_VPORT_SCISSOR_6_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_6_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_6_BR
-#define PA_SC_VPORT_SCISSOR_6_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_6_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_7_TL
-#define PA_SC_VPORT_SCISSOR_7_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_7_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_7_BR
-#define PA_SC_VPORT_SCISSOR_7_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_7_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_8_TL
-#define PA_SC_VPORT_SCISSOR_8_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_8_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_8_BR
-#define PA_SC_VPORT_SCISSOR_8_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_8_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_9_TL
-#define PA_SC_VPORT_SCISSOR_9_TL__TL_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
-#define PA_SC_VPORT_SCISSOR_9_TL__TL_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y_MASK                                                                   0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
-//PA_SC_VPORT_SCISSOR_9_BR
-#define PA_SC_VPORT_SCISSOR_9_BR__BR_X__SHIFT                                                                 0x0
-#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y__SHIFT                                                                 0x10
-#define PA_SC_VPORT_SCISSOR_9_BR__BR_X_MASK                                                                   0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y_MASK                                                                   0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_10_TL
-#define PA_SC_VPORT_SCISSOR_10_TL__TL_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
-#define PA_SC_VPORT_SCISSOR_10_TL__TL_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y_MASK                                                                  0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
-//PA_SC_VPORT_SCISSOR_10_BR
-#define PA_SC_VPORT_SCISSOR_10_BR__BR_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_10_BR__BR_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y_MASK                                                                  0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_11_TL
-#define PA_SC_VPORT_SCISSOR_11_TL__TL_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
-#define PA_SC_VPORT_SCISSOR_11_TL__TL_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y_MASK                                                                  0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
-//PA_SC_VPORT_SCISSOR_11_BR
-#define PA_SC_VPORT_SCISSOR_11_BR__BR_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_11_BR__BR_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y_MASK                                                                  0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_12_TL
-#define PA_SC_VPORT_SCISSOR_12_TL__TL_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
-#define PA_SC_VPORT_SCISSOR_12_TL__TL_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y_MASK                                                                  0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
-//PA_SC_VPORT_SCISSOR_12_BR
-#define PA_SC_VPORT_SCISSOR_12_BR__BR_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_12_BR__BR_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y_MASK                                                                  0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_13_TL
-#define PA_SC_VPORT_SCISSOR_13_TL__TL_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
-#define PA_SC_VPORT_SCISSOR_13_TL__TL_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y_MASK                                                                  0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
-//PA_SC_VPORT_SCISSOR_13_BR
-#define PA_SC_VPORT_SCISSOR_13_BR__BR_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_13_BR__BR_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y_MASK                                                                  0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_14_TL
-#define PA_SC_VPORT_SCISSOR_14_TL__TL_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
-#define PA_SC_VPORT_SCISSOR_14_TL__TL_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y_MASK                                                                  0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
-//PA_SC_VPORT_SCISSOR_14_BR
-#define PA_SC_VPORT_SCISSOR_14_BR__BR_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_14_BR__BR_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y_MASK                                                                  0x7FFF0000L
-//PA_SC_VPORT_SCISSOR_15_TL
-#define PA_SC_VPORT_SCISSOR_15_TL__TL_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
-#define PA_SC_VPORT_SCISSOR_15_TL__TL_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y_MASK                                                                  0x7FFF0000L
-#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
-//PA_SC_VPORT_SCISSOR_15_BR
-#define PA_SC_VPORT_SCISSOR_15_BR__BR_X__SHIFT                                                                0x0
-#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y__SHIFT                                                                0x10
-#define PA_SC_VPORT_SCISSOR_15_BR__BR_X_MASK                                                                  0x00007FFFL
-#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y_MASK                                                                  0x7FFF0000L
-//PA_SC_VPORT_ZMIN_0
-#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_0
-#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_1
-#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_1
-#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_2
-#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_2
-#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_3
-#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_3
-#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_4
-#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_4
-#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_5
-#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_5
-#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_6
-#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_6
-#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_7
-#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_7
-#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_8
-#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_8
-#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_9
-#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_9
-#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX__SHIFT                                                                 0x0
-#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_10
-#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_10
-#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_11
-#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_11
-#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_12
-#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_12
-#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_13
-#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_13
-#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_14
-#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_14
-#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMIN_15
-#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
-//PA_SC_VPORT_ZMAX_15
-#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX__SHIFT                                                                0x0
-#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
-//PA_SC_RASTER_CONFIG
-#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT                                                               0x0
-#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT                                                               0x2
-#define PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT                                                                  0x4
-#define PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT                                                                   0x6
-#define PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT                                                                   0x7
-#define PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT                                                                   0x8
-#define PA_SC_RASTER_CONFIG__PKR_XSEL__SHIFT                                                                  0xa
-#define PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT                                                                  0xc
-#define PA_SC_RASTER_CONFIG__PKR_XSEL2__SHIFT                                                                 0xe
-#define PA_SC_RASTER_CONFIG__SC_MAP__SHIFT                                                                    0x10
-#define PA_SC_RASTER_CONFIG__SC_XSEL__SHIFT                                                                   0x12
-#define PA_SC_RASTER_CONFIG__SC_YSEL__SHIFT                                                                   0x14
-#define PA_SC_RASTER_CONFIG__SE_MAP__SHIFT                                                                    0x18
-#define PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT                                                                   0x1a
-#define PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT                                                                   0x1d
-#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK                                                                 0x00000003L
-#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK                                                                 0x0000000CL
-#define PA_SC_RASTER_CONFIG__RB_XSEL2_MASK                                                                    0x00000030L
-#define PA_SC_RASTER_CONFIG__RB_XSEL_MASK                                                                     0x00000040L
-#define PA_SC_RASTER_CONFIG__RB_YSEL_MASK                                                                     0x00000080L
-#define PA_SC_RASTER_CONFIG__PKR_MAP_MASK                                                                     0x00000300L
-#define PA_SC_RASTER_CONFIG__PKR_XSEL_MASK                                                                    0x00000C00L
-#define PA_SC_RASTER_CONFIG__PKR_YSEL_MASK                                                                    0x00003000L
-#define PA_SC_RASTER_CONFIG__PKR_XSEL2_MASK                                                                   0x0000C000L
-#define PA_SC_RASTER_CONFIG__SC_MAP_MASK                                                                      0x00030000L
-#define PA_SC_RASTER_CONFIG__SC_XSEL_MASK                                                                     0x000C0000L
-#define PA_SC_RASTER_CONFIG__SC_YSEL_MASK                                                                     0x00300000L
-#define PA_SC_RASTER_CONFIG__SE_MAP_MASK                                                                      0x03000000L
-#define PA_SC_RASTER_CONFIG__SE_XSEL_MASK                                                                     0x1C000000L
-#define PA_SC_RASTER_CONFIG__SE_YSEL_MASK                                                                     0xE0000000L
-//PA_SC_RASTER_CONFIG_1
-#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP__SHIFT                                                             0x0
-#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL__SHIFT                                                            0x2
-#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL__SHIFT                                                            0x5
-#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP_MASK                                                               0x00000003L
-#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL_MASK                                                              0x0000001CL
-#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL_MASK                                                              0x000000E0L
-//PA_SC_SCREEN_EXTENT_CONTROL
-#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                 0x0
-#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                  0x2
-#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                   0x00000003L
-#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE_MASK                                                    0x0000000CL
-//PA_SC_TILE_STEERING_OVERRIDE
-#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE__SHIFT                                                           0x0
-#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SE__SHIFT                                                           0x1
-#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SE__SHIFT                                                    0x5
-#define PA_SC_TILE_STEERING_OVERRIDE__DISABLE_SRBSL_DB_OPTIMIZED_PACKING__SHIFT                               0x8
-#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE_MASK                                                             0x00000001L
-#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SE_MASK                                                             0x00000006L
-#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SE_MASK                                                      0x00000060L
-#define PA_SC_TILE_STEERING_OVERRIDE__DISABLE_SRBSL_DB_OPTIMIZED_PACKING_MASK                                 0x00000100L
-//CP_PERFMON_CNTX_CNTL
-#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT                                                           0x1f
-#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK                                                             0x80000000L
-//CP_PIPEID
-#define CP_PIPEID__PIPE_ID__SHIFT                                                                             0x0
-#define CP_PIPEID__PIPE_ID_MASK                                                                               0x00000003L
-//CP_RINGID
-#define CP_RINGID__RINGID__SHIFT                                                                              0x0
-#define CP_RINGID__RINGID_MASK                                                                                0x00000003L
-//CP_VMID
-#define CP_VMID__VMID__SHIFT                                                                                  0x0
-#define CP_VMID__VMID_MASK                                                                                    0x0000000FL
-//PA_SC_RIGHT_VERT_GRID
-#define PA_SC_RIGHT_VERT_GRID__LEFT_QTR__SHIFT                                                                0x0
-#define PA_SC_RIGHT_VERT_GRID__LEFT_HALF__SHIFT                                                               0x8
-#define PA_SC_RIGHT_VERT_GRID__RIGHT_HALF__SHIFT                                                              0x10
-#define PA_SC_RIGHT_VERT_GRID__RIGHT_QTR__SHIFT                                                               0x18
-#define PA_SC_RIGHT_VERT_GRID__LEFT_QTR_MASK                                                                  0x000000FFL
-#define PA_SC_RIGHT_VERT_GRID__LEFT_HALF_MASK                                                                 0x0000FF00L
-#define PA_SC_RIGHT_VERT_GRID__RIGHT_HALF_MASK                                                                0x00FF0000L
-#define PA_SC_RIGHT_VERT_GRID__RIGHT_QTR_MASK                                                                 0xFF000000L
-//PA_SC_LEFT_VERT_GRID
-#define PA_SC_LEFT_VERT_GRID__LEFT_QTR__SHIFT                                                                 0x0
-#define PA_SC_LEFT_VERT_GRID__LEFT_HALF__SHIFT                                                                0x8
-#define PA_SC_LEFT_VERT_GRID__RIGHT_HALF__SHIFT                                                               0x10
-#define PA_SC_LEFT_VERT_GRID__RIGHT_QTR__SHIFT                                                                0x18
-#define PA_SC_LEFT_VERT_GRID__LEFT_QTR_MASK                                                                   0x000000FFL
-#define PA_SC_LEFT_VERT_GRID__LEFT_HALF_MASK                                                                  0x0000FF00L
-#define PA_SC_LEFT_VERT_GRID__RIGHT_HALF_MASK                                                                 0x00FF0000L
-#define PA_SC_LEFT_VERT_GRID__RIGHT_QTR_MASK                                                                  0xFF000000L
-//PA_SC_HORIZ_GRID
-#define PA_SC_HORIZ_GRID__TOP_QTR__SHIFT                                                                      0x0
-#define PA_SC_HORIZ_GRID__TOP_HALF__SHIFT                                                                     0x8
-#define PA_SC_HORIZ_GRID__BOT_HALF__SHIFT                                                                     0x10
-#define PA_SC_HORIZ_GRID__BOT_QTR__SHIFT                                                                      0x18
-#define PA_SC_HORIZ_GRID__TOP_QTR_MASK                                                                        0x000000FFL
-#define PA_SC_HORIZ_GRID__TOP_HALF_MASK                                                                       0x0000FF00L
-#define PA_SC_HORIZ_GRID__BOT_HALF_MASK                                                                       0x00FF0000L
-#define PA_SC_HORIZ_GRID__BOT_QTR_MASK                                                                        0xFF000000L
-//PA_SC_FOV_WINDOW_LR
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_LEFT__SHIFT                                                         0x0
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_RIGHT__SHIFT                                                        0x8
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_LEFT__SHIFT                                                        0x10
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_RIGHT__SHIFT                                                       0x18
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_LEFT_MASK                                                           0x000000FFL
-#define PA_SC_FOV_WINDOW_LR__LEFT_EYE_FOV_RIGHT_MASK                                                          0x0000FF00L
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_LEFT_MASK                                                          0x00FF0000L
-#define PA_SC_FOV_WINDOW_LR__RIGHT_EYE_FOV_RIGHT_MASK                                                         0xFF000000L
-//PA_SC_FOV_WINDOW_TB
-#define PA_SC_FOV_WINDOW_TB__FOV_TOP__SHIFT                                                                   0x0
-#define PA_SC_FOV_WINDOW_TB__FOV_BOT__SHIFT                                                                   0x8
-#define PA_SC_FOV_WINDOW_TB__FOV_TOP_MASK                                                                     0x000000FFL
-#define PA_SC_FOV_WINDOW_TB__FOV_BOT_MASK                                                                     0x0000FF00L
-//VGT_MULTI_PRIM_IB_RESET_INDX
-#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT                                                       0x0
-#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK                                                         0xFFFFFFFFL
-//CB_BLEND_RED
-#define CB_BLEND_RED__BLEND_RED__SHIFT                                                                        0x0
-#define CB_BLEND_RED__BLEND_RED_MASK                                                                          0xFFFFFFFFL
-//CB_BLEND_GREEN
-#define CB_BLEND_GREEN__BLEND_GREEN__SHIFT                                                                    0x0
-#define CB_BLEND_GREEN__BLEND_GREEN_MASK                                                                      0xFFFFFFFFL
-//CB_BLEND_BLUE
-#define CB_BLEND_BLUE__BLEND_BLUE__SHIFT                                                                      0x0
-#define CB_BLEND_BLUE__BLEND_BLUE_MASK                                                                        0xFFFFFFFFL
-//CB_BLEND_ALPHA
-#define CB_BLEND_ALPHA__BLEND_ALPHA__SHIFT                                                                    0x0
-#define CB_BLEND_ALPHA__BLEND_ALPHA_MASK                                                                      0xFFFFFFFFL
-//CB_DCC_CONTROL
-#define CB_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                                     0x0
-#define CB_DCC_CONTROL__OVERWRITE_COMBINER_MRT_SHARING_DISABLE__SHIFT                                         0x1
-#define CB_DCC_CONTROL__OVERWRITE_COMBINER_WATERMARK__SHIFT                                                   0x2
-#define CB_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                       0x00000001L
-#define CB_DCC_CONTROL__OVERWRITE_COMBINER_MRT_SHARING_DISABLE_MASK                                           0x00000002L
-#define CB_DCC_CONTROL__OVERWRITE_COMBINER_WATERMARK_MASK                                                     0x0000007CL
-//DB_STENCIL_CONTROL
-#define DB_STENCIL_CONTROL__STENCILFAIL__SHIFT                                                                0x0
-#define DB_STENCIL_CONTROL__STENCILZPASS__SHIFT                                                               0x4
-#define DB_STENCIL_CONTROL__STENCILZFAIL__SHIFT                                                               0x8
-#define DB_STENCIL_CONTROL__STENCILFAIL_BF__SHIFT                                                             0xc
-#define DB_STENCIL_CONTROL__STENCILZPASS_BF__SHIFT                                                            0x10
-#define DB_STENCIL_CONTROL__STENCILZFAIL_BF__SHIFT                                                            0x14
-#define DB_STENCIL_CONTROL__STENCILFAIL_MASK                                                                  0x0000000FL
-#define DB_STENCIL_CONTROL__STENCILZPASS_MASK                                                                 0x000000F0L
-#define DB_STENCIL_CONTROL__STENCILZFAIL_MASK                                                                 0x00000F00L
-#define DB_STENCIL_CONTROL__STENCILFAIL_BF_MASK                                                               0x0000F000L
-#define DB_STENCIL_CONTROL__STENCILZPASS_BF_MASK                                                              0x000F0000L
-#define DB_STENCIL_CONTROL__STENCILZFAIL_BF_MASK                                                              0x00F00000L
-//DB_STENCILREFMASK
-#define DB_STENCILREFMASK__STENCILTESTVAL__SHIFT                                                              0x0
-#define DB_STENCILREFMASK__STENCILMASK__SHIFT                                                                 0x8
-#define DB_STENCILREFMASK__STENCILWRITEMASK__SHIFT                                                            0x10
-#define DB_STENCILREFMASK__STENCILOPVAL__SHIFT                                                                0x18
-#define DB_STENCILREFMASK__STENCILTESTVAL_MASK                                                                0x000000FFL
-#define DB_STENCILREFMASK__STENCILMASK_MASK                                                                   0x0000FF00L
-#define DB_STENCILREFMASK__STENCILWRITEMASK_MASK                                                              0x00FF0000L
-#define DB_STENCILREFMASK__STENCILOPVAL_MASK                                                                  0xFF000000L
-//DB_STENCILREFMASK_BF
-#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF__SHIFT                                                        0x0
-#define DB_STENCILREFMASK_BF__STENCILMASK_BF__SHIFT                                                           0x8
-#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF__SHIFT                                                      0x10
-#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF__SHIFT                                                          0x18
-#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF_MASK                                                          0x000000FFL
-#define DB_STENCILREFMASK_BF__STENCILMASK_BF_MASK                                                             0x0000FF00L
-#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF_MASK                                                        0x00FF0000L
-#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF_MASK                                                            0xFF000000L
-//PA_CL_VPORT_XSCALE
-#define PA_CL_VPORT_XSCALE__VPORT_XSCALE__SHIFT                                                               0x0
-#define PA_CL_VPORT_XSCALE__VPORT_XSCALE_MASK                                                                 0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET
-#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET__SHIFT                                                             0x0
-#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE
-#define PA_CL_VPORT_YSCALE__VPORT_YSCALE__SHIFT                                                               0x0
-#define PA_CL_VPORT_YSCALE__VPORT_YSCALE_MASK                                                                 0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET
-#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET__SHIFT                                                             0x0
-#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE
-#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE__SHIFT                                                               0x0
-#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE_MASK                                                                 0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET
-#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_1
-#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_1
-#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_1
-#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_1
-#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_1
-#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_1
-#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_2
-#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_2
-#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_2
-#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_2
-#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_2
-#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_2
-#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_3
-#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_3
-#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_3
-#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_3
-#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_3
-#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_3
-#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_4
-#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_4
-#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_4
-#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_4
-#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_4
-#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_4
-#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_5
-#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_5
-#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_5
-#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_5
-#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_5
-#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_5
-#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_6
-#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_6
-#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_6
-#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_6
-#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_6
-#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_6
-#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_7
-#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_7
-#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_7
-#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_7
-#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_7
-#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_7
-#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_8
-#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_8
-#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_8
-#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_8
-#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_8
-#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_8
-#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_9
-#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_9
-#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_9
-#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_9
-#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_9
-#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE__SHIFT                                                             0x0
-#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_9
-#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET__SHIFT                                                           0x0
-#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_10
-#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_10
-#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_10
-#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_10
-#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_10
-#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_10
-#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_11
-#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_11
-#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_11
-#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_11
-#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_11
-#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_11
-#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_12
-#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_12
-#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_12
-#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_12
-#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_12
-#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_12
-#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_13
-#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_13
-#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_13
-#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_13
-#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_13
-#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_13
-#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_14
-#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_14
-#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_14
-#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_14
-#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_14
-#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_14
-#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_XSCALE_15
-#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_XOFFSET_15
-#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_YSCALE_15
-#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_YOFFSET_15
-#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_VPORT_ZSCALE_15
-#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE__SHIFT                                                            0x0
-#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
-//PA_CL_VPORT_ZOFFSET_15
-#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET__SHIFT                                                          0x0
-#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
-//PA_CL_UCP_0_X
-#define PA_CL_UCP_0_X__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_0_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_0_Y
-#define PA_CL_UCP_0_Y__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_0_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_0_Z
-#define PA_CL_UCP_0_Z__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_0_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_0_W
-#define PA_CL_UCP_0_W__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_0_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_1_X
-#define PA_CL_UCP_1_X__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_1_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_1_Y
-#define PA_CL_UCP_1_Y__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_1_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_1_Z
-#define PA_CL_UCP_1_Z__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_1_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_1_W
-#define PA_CL_UCP_1_W__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_1_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_2_X
-#define PA_CL_UCP_2_X__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_2_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_2_Y
-#define PA_CL_UCP_2_Y__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_2_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_2_Z
-#define PA_CL_UCP_2_Z__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_2_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_2_W
-#define PA_CL_UCP_2_W__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_2_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_3_X
-#define PA_CL_UCP_3_X__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_3_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_3_Y
-#define PA_CL_UCP_3_Y__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_3_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_3_Z
-#define PA_CL_UCP_3_Z__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_3_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_3_W
-#define PA_CL_UCP_3_W__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_3_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_4_X
-#define PA_CL_UCP_4_X__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_4_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_4_Y
-#define PA_CL_UCP_4_Y__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_4_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_4_Z
-#define PA_CL_UCP_4_Z__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_4_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_4_W
-#define PA_CL_UCP_4_W__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_4_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_5_X
-#define PA_CL_UCP_5_X__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_5_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_5_Y
-#define PA_CL_UCP_5_Y__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_5_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_5_Z
-#define PA_CL_UCP_5_Z__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_5_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//PA_CL_UCP_5_W
-#define PA_CL_UCP_5_W__DATA_REGISTER__SHIFT                                                                   0x0
-#define PA_CL_UCP_5_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
-//SPI_PS_INPUT_CNTL_0
-#define SPI_PS_INPUT_CNTL_0__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_0__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_0__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_0__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_0__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_0__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_1
-#define SPI_PS_INPUT_CNTL_1__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_1__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_1__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_1__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_1__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_1__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_2
-#define SPI_PS_INPUT_CNTL_2__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_2__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_2__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_2__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_2__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_2__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_3
-#define SPI_PS_INPUT_CNTL_3__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_3__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_3__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_3__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_3__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_3__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_4
-#define SPI_PS_INPUT_CNTL_4__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_4__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_4__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_4__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_4__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_4__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_5
-#define SPI_PS_INPUT_CNTL_5__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_5__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_5__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_5__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_5__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_5__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_6
-#define SPI_PS_INPUT_CNTL_6__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_6__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_6__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_6__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_6__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_6__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_7
-#define SPI_PS_INPUT_CNTL_7__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_7__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_7__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_7__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_7__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_7__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_8
-#define SPI_PS_INPUT_CNTL_8__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_8__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_8__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_8__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_8__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_8__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_9
-#define SPI_PS_INPUT_CNTL_9__OFFSET__SHIFT                                                                    0x0
-#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL__SHIFT                                                               0x8
-#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE__SHIFT                                                                0xa
-#define SPI_PS_INPUT_CNTL_9__CYL_WRAP__SHIFT                                                                  0xd
-#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX__SHIFT                                                             0x11
-#define SPI_PS_INPUT_CNTL_9__DUP__SHIFT                                                                       0x12
-#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE__SHIFT                                                          0x13
-#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
-#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
-#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
-#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID__SHIFT                                                               0x18
-#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID__SHIFT                                                               0x19
-#define SPI_PS_INPUT_CNTL_9__OFFSET_MASK                                                                      0x0000003FL
-#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_MASK                                                                 0x00000300L
-#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE_MASK                                                                  0x00000400L
-#define SPI_PS_INPUT_CNTL_9__CYL_WRAP_MASK                                                                    0x0001E000L
-#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_MASK                                                               0x00020000L
-#define SPI_PS_INPUT_CNTL_9__DUP_MASK                                                                         0x00040000L
-#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE_MASK                                                            0x00080000L
-#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
-#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
-#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
-#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID_MASK                                                                 0x01000000L
-#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID_MASK                                                                 0x02000000L
-//SPI_PS_INPUT_CNTL_10
-#define SPI_PS_INPUT_CNTL_10__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_10__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_10__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_10__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_10__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_10__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_11
-#define SPI_PS_INPUT_CNTL_11__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_11__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_11__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_11__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_11__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_11__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_12
-#define SPI_PS_INPUT_CNTL_12__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_12__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_12__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_12__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_12__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_12__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_13
-#define SPI_PS_INPUT_CNTL_13__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_13__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_13__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_13__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_13__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_13__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_14
-#define SPI_PS_INPUT_CNTL_14__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_14__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_14__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_14__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_14__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_14__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_15
-#define SPI_PS_INPUT_CNTL_15__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_15__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_15__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_15__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_15__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_15__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_16
-#define SPI_PS_INPUT_CNTL_16__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_16__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_16__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_16__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_16__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_16__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_17
-#define SPI_PS_INPUT_CNTL_17__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_17__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_17__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_17__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_17__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_17__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_18
-#define SPI_PS_INPUT_CNTL_18__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_18__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_18__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_18__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_18__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_18__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_19
-#define SPI_PS_INPUT_CNTL_19__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_19__CYL_WRAP__SHIFT                                                                 0xd
-#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX__SHIFT                                                            0x11
-#define SPI_PS_INPUT_CNTL_19__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
-#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_19__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_19__CYL_WRAP_MASK                                                                   0x0001E000L
-#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_MASK                                                              0x00020000L
-#define SPI_PS_INPUT_CNTL_19__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
-#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_20
-#define SPI_PS_INPUT_CNTL_20__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_20__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_20__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_20__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_21
-#define SPI_PS_INPUT_CNTL_21__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_21__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_21__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_21__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_22
-#define SPI_PS_INPUT_CNTL_22__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_22__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_22__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_22__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_23
-#define SPI_PS_INPUT_CNTL_23__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_23__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_23__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_23__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_24
-#define SPI_PS_INPUT_CNTL_24__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_24__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_24__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_24__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_25
-#define SPI_PS_INPUT_CNTL_25__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_25__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_25__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_25__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_26
-#define SPI_PS_INPUT_CNTL_26__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_26__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_26__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_26__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_27
-#define SPI_PS_INPUT_CNTL_27__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_27__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_27__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_27__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_28
-#define SPI_PS_INPUT_CNTL_28__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_28__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_28__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_28__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_29
-#define SPI_PS_INPUT_CNTL_29__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_29__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_29__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_29__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_30
-#define SPI_PS_INPUT_CNTL_30__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_30__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_30__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_30__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_PS_INPUT_CNTL_31
-#define SPI_PS_INPUT_CNTL_31__OFFSET__SHIFT                                                                   0x0
-#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL__SHIFT                                                              0x8
-#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE__SHIFT                                                               0xa
-#define SPI_PS_INPUT_CNTL_31__DUP__SHIFT                                                                      0x12
-#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE__SHIFT                                                         0x13
-#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
-#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
-#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID__SHIFT                                                              0x18
-#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID__SHIFT                                                              0x19
-#define SPI_PS_INPUT_CNTL_31__OFFSET_MASK                                                                     0x0000003FL
-#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_MASK                                                                0x00000300L
-#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE_MASK                                                                 0x00000400L
-#define SPI_PS_INPUT_CNTL_31__DUP_MASK                                                                        0x00040000L
-#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE_MASK                                                           0x00080000L
-#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
-#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
-#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID_MASK                                                                0x01000000L
-#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID_MASK                                                                0x02000000L
-//SPI_VS_OUT_CONFIG
-#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT__SHIFT                                                             0x1
-#define SPI_VS_OUT_CONFIG__VS_HALF_PACK__SHIFT                                                                0x6
-#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT_MASK                                                               0x0000003EL
-#define SPI_VS_OUT_CONFIG__VS_HALF_PACK_MASK                                                                  0x00000040L
-//SPI_PS_INPUT_ENA
-#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA__SHIFT                                                             0x0
-#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA__SHIFT                                                             0x1
-#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA__SHIFT                                                           0x2
-#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA__SHIFT                                                         0x3
-#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA__SHIFT                                                            0x4
-#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA__SHIFT                                                            0x5
-#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA__SHIFT                                                          0x6
-#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA__SHIFT                                                         0x7
-#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA__SHIFT                                                              0x8
-#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA__SHIFT                                                              0x9
-#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA__SHIFT                                                              0xa
-#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA__SHIFT                                                              0xb
-#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA__SHIFT                                                               0xc
-#define SPI_PS_INPUT_ENA__ANCILLARY_ENA__SHIFT                                                                0xd
-#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA__SHIFT                                                          0xe
-#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA__SHIFT                                                             0xf
-#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA_MASK                                                               0x00000001L
-#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA_MASK                                                               0x00000002L
-#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA_MASK                                                             0x00000004L
-#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA_MASK                                                           0x00000008L
-#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA_MASK                                                              0x00000010L
-#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA_MASK                                                              0x00000020L
-#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA_MASK                                                            0x00000040L
-#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA_MASK                                                           0x00000080L
-#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA_MASK                                                                0x00000100L
-#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA_MASK                                                                0x00000200L
-#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA_MASK                                                                0x00000400L
-#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA_MASK                                                                0x00000800L
-#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA_MASK                                                                 0x00001000L
-#define SPI_PS_INPUT_ENA__ANCILLARY_ENA_MASK                                                                  0x00002000L
-#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA_MASK                                                            0x00004000L
-#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA_MASK                                                               0x00008000L
-//SPI_PS_INPUT_ADDR
-#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA__SHIFT                                                            0x0
-#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA__SHIFT                                                            0x1
-#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA__SHIFT                                                          0x2
-#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA__SHIFT                                                        0x3
-#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA__SHIFT                                                           0x4
-#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA__SHIFT                                                           0x5
-#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA__SHIFT                                                         0x6
-#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA__SHIFT                                                        0x7
-#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA__SHIFT                                                             0x8
-#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA__SHIFT                                                             0x9
-#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA__SHIFT                                                             0xa
-#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA__SHIFT                                                             0xb
-#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA__SHIFT                                                              0xc
-#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA__SHIFT                                                               0xd
-#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA__SHIFT                                                         0xe
-#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA__SHIFT                                                            0xf
-#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA_MASK                                                              0x00000001L
-#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA_MASK                                                              0x00000002L
-#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA_MASK                                                            0x00000004L
-#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA_MASK                                                          0x00000008L
-#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA_MASK                                                             0x00000010L
-#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA_MASK                                                             0x00000020L
-#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA_MASK                                                           0x00000040L
-#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA_MASK                                                          0x00000080L
-#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA_MASK                                                               0x00000100L
-#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA_MASK                                                               0x00000200L
-#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA_MASK                                                               0x00000400L
-#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA_MASK                                                               0x00000800L
-#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA_MASK                                                                0x00001000L
-#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA_MASK                                                                 0x00002000L
-#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA_MASK                                                           0x00004000L
-#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA_MASK                                                              0x00008000L
-//SPI_INTERP_CONTROL_0
-#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA__SHIFT                                                           0x0
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA__SHIFT                                                           0x1
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X__SHIFT                                                        0x2
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y__SHIFT                                                        0x5
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z__SHIFT                                                        0x8
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W__SHIFT                                                        0xb
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1__SHIFT                                                         0xe
-#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA_MASK                                                             0x00000001L
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA_MASK                                                             0x00000002L
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X_MASK                                                          0x0000001CL
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y_MASK                                                          0x000000E0L
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z_MASK                                                          0x00000700L
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W_MASK                                                          0x00003800L
-#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1_MASK                                                           0x00004000L
-//SPI_PS_IN_CONTROL
-#define SPI_PS_IN_CONTROL__NUM_INTERP__SHIFT                                                                  0x0
-#define SPI_PS_IN_CONTROL__PARAM_GEN__SHIFT                                                                   0x6
-#define SPI_PS_IN_CONTROL__OFFCHIP_PARAM_EN__SHIFT                                                            0x7
-#define SPI_PS_IN_CONTROL__LATE_PC_DEALLOC__SHIFT                                                             0x8
-#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE__SHIFT                                                         0xe
-#define SPI_PS_IN_CONTROL__NUM_INTERP_MASK                                                                    0x0000003FL
-#define SPI_PS_IN_CONTROL__PARAM_GEN_MASK                                                                     0x00000040L
-#define SPI_PS_IN_CONTROL__OFFCHIP_PARAM_EN_MASK                                                              0x00000080L
-#define SPI_PS_IN_CONTROL__LATE_PC_DEALLOC_MASK                                                               0x00000100L
-#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE_MASK                                                           0x00004000L
-//SPI_BARYC_CNTL
-#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL__SHIFT                                                              0x0
-#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL__SHIFT                                                            0x4
-#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL__SHIFT                                                             0x8
-#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL__SHIFT                                                           0xc
-#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION__SHIFT                                                             0x10
-#define SPI_BARYC_CNTL__POS_FLOAT_ULC__SHIFT                                                                  0x14
-#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS__SHIFT                                                            0x18
-#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL_MASK                                                                0x00000001L
-#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL_MASK                                                              0x00000010L
-#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL_MASK                                                               0x00000100L
-#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL_MASK                                                             0x00001000L
-#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION_MASK                                                               0x00030000L
-#define SPI_BARYC_CNTL__POS_FLOAT_ULC_MASK                                                                    0x00100000L
-#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS_MASK                                                              0x01000000L
-//SPI_TMPRING_SIZE
-#define SPI_TMPRING_SIZE__WAVES__SHIFT                                                                        0x0
-#define SPI_TMPRING_SIZE__WAVESIZE__SHIFT                                                                     0xc
-#define SPI_TMPRING_SIZE__WAVES_MASK                                                                          0x00000FFFL
-#define SPI_TMPRING_SIZE__WAVESIZE_MASK                                                                       0x01FFF000L
-//SPI_SHADER_POS_FORMAT
-#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT__SHIFT                                                      0x0
-#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT__SHIFT                                                      0x4
-#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT__SHIFT                                                      0x8
-#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT__SHIFT                                                      0xc
-#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT_MASK                                                        0x0000000FL
-#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT_MASK                                                        0x000000F0L
-#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT_MASK                                                        0x00000F00L
-#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT_MASK                                                        0x0000F000L
-//SPI_SHADER_Z_FORMAT
-#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT__SHIFT                                                           0x0
-#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT_MASK                                                             0x0000000FL
-//SPI_SHADER_COL_FORMAT
-#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT__SHIFT                                                      0x0
-#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT__SHIFT                                                      0x4
-#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT__SHIFT                                                      0x8
-#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT__SHIFT                                                      0xc
-#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT__SHIFT                                                      0x10
-#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT__SHIFT                                                      0x14
-#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT__SHIFT                                                      0x18
-#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT__SHIFT                                                      0x1c
-#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT_MASK                                                        0x0000000FL
-#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT_MASK                                                        0x000000F0L
-#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT_MASK                                                        0x00000F00L
-#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT_MASK                                                        0x0000F000L
-#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT_MASK                                                        0x000F0000L
-#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT_MASK                                                        0x00F00000L
-#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT_MASK                                                        0x0F000000L
-#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT_MASK                                                        0xF0000000L
-//SX_PS_DOWNCONVERT
-#define SX_PS_DOWNCONVERT__MRT0__SHIFT                                                                        0x0
-#define SX_PS_DOWNCONVERT__MRT1__SHIFT                                                                        0x4
-#define SX_PS_DOWNCONVERT__MRT2__SHIFT                                                                        0x8
-#define SX_PS_DOWNCONVERT__MRT3__SHIFT                                                                        0xc
-#define SX_PS_DOWNCONVERT__MRT4__SHIFT                                                                        0x10
-#define SX_PS_DOWNCONVERT__MRT5__SHIFT                                                                        0x14
-#define SX_PS_DOWNCONVERT__MRT6__SHIFT                                                                        0x18
-#define SX_PS_DOWNCONVERT__MRT7__SHIFT                                                                        0x1c
-#define SX_PS_DOWNCONVERT__MRT0_MASK                                                                          0x0000000FL
-#define SX_PS_DOWNCONVERT__MRT1_MASK                                                                          0x000000F0L
-#define SX_PS_DOWNCONVERT__MRT2_MASK                                                                          0x00000F00L
-#define SX_PS_DOWNCONVERT__MRT3_MASK                                                                          0x0000F000L
-#define SX_PS_DOWNCONVERT__MRT4_MASK                                                                          0x000F0000L
-#define SX_PS_DOWNCONVERT__MRT5_MASK                                                                          0x00F00000L
-#define SX_PS_DOWNCONVERT__MRT6_MASK                                                                          0x0F000000L
-#define SX_PS_DOWNCONVERT__MRT7_MASK                                                                          0xF0000000L
-//SX_BLEND_OPT_EPSILON
-#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON__SHIFT                                                             0x0
-#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON__SHIFT                                                             0x4
-#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON__SHIFT                                                             0x8
-#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON__SHIFT                                                             0xc
-#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON__SHIFT                                                             0x10
-#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON__SHIFT                                                             0x14
-#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON__SHIFT                                                             0x18
-#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON__SHIFT                                                             0x1c
-#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON_MASK                                                               0x0000000FL
-#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON_MASK                                                               0x000000F0L
-#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON_MASK                                                               0x00000F00L
-#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON_MASK                                                               0x0000F000L
-#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON_MASK                                                               0x000F0000L
-#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON_MASK                                                               0x00F00000L
-#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON_MASK                                                               0x0F000000L
-#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON_MASK                                                               0xF0000000L
-//SX_BLEND_OPT_CONTROL
-#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE__SHIFT                                                   0x0
-#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE__SHIFT                                                   0x1
-#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE__SHIFT                                                   0x4
-#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE__SHIFT                                                   0x5
-#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE__SHIFT                                                   0x8
-#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE__SHIFT                                                   0x9
-#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE__SHIFT                                                   0xc
-#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE__SHIFT                                                   0xd
-#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE__SHIFT                                                   0x10
-#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE__SHIFT                                                   0x11
-#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE__SHIFT                                                   0x14
-#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE__SHIFT                                                   0x15
-#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE__SHIFT                                                   0x18
-#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE__SHIFT                                                   0x19
-#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE__SHIFT                                                   0x1c
-#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE__SHIFT                                                   0x1d
-#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE__SHIFT                                                   0x1f
-#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE_MASK                                                     0x00000001L
-#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE_MASK                                                     0x00000002L
-#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE_MASK                                                     0x00000010L
-#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE_MASK                                                     0x00000020L
-#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE_MASK                                                     0x00000100L
-#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE_MASK                                                     0x00000200L
-#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE_MASK                                                     0x00001000L
-#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE_MASK                                                     0x00002000L
-#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE_MASK                                                     0x00010000L
-#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE_MASK                                                     0x00020000L
-#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE_MASK                                                     0x00100000L
-#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE_MASK                                                     0x00200000L
-#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE_MASK                                                     0x01000000L
-#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE_MASK                                                     0x02000000L
-#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE_MASK                                                     0x10000000L
-#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE_MASK                                                     0x20000000L
-#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE_MASK                                                     0x80000000L
-//SX_MRT0_BLEND_OPT
-#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//SX_MRT1_BLEND_OPT
-#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//SX_MRT2_BLEND_OPT
-#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//SX_MRT3_BLEND_OPT
-#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//SX_MRT4_BLEND_OPT
-#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//SX_MRT5_BLEND_OPT
-#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//SX_MRT6_BLEND_OPT
-#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//SX_MRT7_BLEND_OPT
-#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
-#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
-#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
-#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
-#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
-#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
-#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
-#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
-#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
-#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
-#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
-#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
-//CB_BLEND0_CONTROL
-#define CB_BLEND0_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND0_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND0_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND0_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND0_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND0_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND0_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND0_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND0_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND0_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_BLEND1_CONTROL
-#define CB_BLEND1_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND1_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND1_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND1_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND1_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND1_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND1_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND1_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND1_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND1_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_BLEND2_CONTROL
-#define CB_BLEND2_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND2_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND2_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND2_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND2_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND2_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND2_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND2_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND2_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND2_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_BLEND3_CONTROL
-#define CB_BLEND3_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND3_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND3_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND3_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND3_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND3_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND3_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND3_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND3_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND3_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_BLEND4_CONTROL
-#define CB_BLEND4_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND4_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND4_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND4_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND4_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND4_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND4_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND4_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND4_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND4_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_BLEND5_CONTROL
-#define CB_BLEND5_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND5_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND5_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND5_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND5_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND5_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND5_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND5_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND5_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND5_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_BLEND6_CONTROL
-#define CB_BLEND6_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND6_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND6_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND6_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND6_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND6_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND6_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND6_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND6_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND6_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_BLEND7_CONTROL
-#define CB_BLEND7_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
-#define CB_BLEND7_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
-#define CB_BLEND7_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
-#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
-#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
-#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
-#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
-#define CB_BLEND7_CONTROL__ENABLE__SHIFT                                                                      0x1e
-#define CB_BLEND7_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
-#define CB_BLEND7_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
-#define CB_BLEND7_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
-#define CB_BLEND7_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
-#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
-#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
-#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
-#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
-#define CB_BLEND7_CONTROL__ENABLE_MASK                                                                        0x40000000L
-#define CB_BLEND7_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
-//CB_MRT0_EPITCH
-#define CB_MRT0_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT0_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CB_MRT1_EPITCH
-#define CB_MRT1_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT1_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CB_MRT2_EPITCH
-#define CB_MRT2_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT2_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CB_MRT3_EPITCH
-#define CB_MRT3_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT3_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CB_MRT4_EPITCH
-#define CB_MRT4_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT4_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CB_MRT5_EPITCH
-#define CB_MRT5_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT5_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CB_MRT6_EPITCH
-#define CB_MRT6_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT6_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CB_MRT7_EPITCH
-#define CB_MRT7_EPITCH__EPITCH__SHIFT                                                                         0x0
-#define CB_MRT7_EPITCH__EPITCH_MASK                                                                           0x0000FFFFL
-//CS_COPY_STATE
-#define CS_COPY_STATE__SRC_STATE_ID__SHIFT                                                                    0x0
-#define CS_COPY_STATE__SRC_STATE_ID_MASK                                                                      0x00000007L
-//GFX_COPY_STATE
-#define GFX_COPY_STATE__SRC_STATE_ID__SHIFT                                                                   0x0
-#define GFX_COPY_STATE__SRC_STATE_ID_MASK                                                                     0x00000007L
-//PA_CL_POINT_X_RAD
-#define PA_CL_POINT_X_RAD__DATA_REGISTER__SHIFT                                                               0x0
-#define PA_CL_POINT_X_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
-//PA_CL_POINT_Y_RAD
-#define PA_CL_POINT_Y_RAD__DATA_REGISTER__SHIFT                                                               0x0
-#define PA_CL_POINT_Y_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
-//PA_CL_POINT_SIZE
-#define PA_CL_POINT_SIZE__DATA_REGISTER__SHIFT                                                                0x0
-#define PA_CL_POINT_SIZE__DATA_REGISTER_MASK                                                                  0xFFFFFFFFL
-//PA_CL_POINT_CULL_RAD
-#define PA_CL_POINT_CULL_RAD__DATA_REGISTER__SHIFT                                                            0x0
-#define PA_CL_POINT_CULL_RAD__DATA_REGISTER_MASK                                                              0xFFFFFFFFL
-//VGT_DMA_BASE_HI
-#define VGT_DMA_BASE_HI__BASE_ADDR__SHIFT                                                                     0x0
-#define VGT_DMA_BASE_HI__BASE_ADDR_MASK                                                                       0x0000FFFFL
-//VGT_DMA_BASE
-#define VGT_DMA_BASE__BASE_ADDR__SHIFT                                                                        0x0
-#define VGT_DMA_BASE__BASE_ADDR_MASK                                                                          0xFFFFFFFFL
-//VGT_DRAW_INITIATOR
-#define VGT_DRAW_INITIATOR__SOURCE_SELECT__SHIFT                                                              0x0
-#define VGT_DRAW_INITIATOR__MAJOR_MODE__SHIFT                                                                 0x2
-#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX__SHIFT                                                             0x4
-#define VGT_DRAW_INITIATOR__NOT_EOP__SHIFT                                                                    0x5
-#define VGT_DRAW_INITIATOR__USE_OPAQUE__SHIFT                                                                 0x6
-#define VGT_DRAW_INITIATOR__UNROLLED_INST__SHIFT                                                              0x7
-#define VGT_DRAW_INITIATOR__GRBM_SKEW_NO_DEC__SHIFT                                                           0x8
-#define VGT_DRAW_INITIATOR__REG_RT_INDEX__SHIFT                                                               0x1d
-#define VGT_DRAW_INITIATOR__SOURCE_SELECT_MASK                                                                0x00000003L
-#define VGT_DRAW_INITIATOR__MAJOR_MODE_MASK                                                                   0x0000000CL
-#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX_MASK                                                               0x00000010L
-#define VGT_DRAW_INITIATOR__NOT_EOP_MASK                                                                      0x00000020L
-#define VGT_DRAW_INITIATOR__USE_OPAQUE_MASK                                                                   0x00000040L
-#define VGT_DRAW_INITIATOR__UNROLLED_INST_MASK                                                                0x00000080L
-#define VGT_DRAW_INITIATOR__GRBM_SKEW_NO_DEC_MASK                                                             0x00000100L
-#define VGT_DRAW_INITIATOR__REG_RT_INDEX_MASK                                                                 0xE0000000L
-//VGT_IMMED_DATA
-#define VGT_IMMED_DATA__DATA__SHIFT                                                                           0x0
-#define VGT_IMMED_DATA__DATA_MASK                                                                             0xFFFFFFFFL
-//VGT_EVENT_ADDRESS_REG
-#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW__SHIFT                                                             0x0
-#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW_MASK                                                               0x0FFFFFFFL
-//DB_DEPTH_CONTROL
-#define DB_DEPTH_CONTROL__STENCIL_ENABLE__SHIFT                                                               0x0
-#define DB_DEPTH_CONTROL__Z_ENABLE__SHIFT                                                                     0x1
-#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE__SHIFT                                                               0x2
-#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE__SHIFT                                                          0x3
-#define DB_DEPTH_CONTROL__ZFUNC__SHIFT                                                                        0x4
-#define DB_DEPTH_CONTROL__BACKFACE_ENABLE__SHIFT                                                              0x7
-#define DB_DEPTH_CONTROL__STENCILFUNC__SHIFT                                                                  0x8
-#define DB_DEPTH_CONTROL__STENCILFUNC_BF__SHIFT                                                               0x14
-#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL__SHIFT                                            0x1e
-#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS__SHIFT                                           0x1f
-#define DB_DEPTH_CONTROL__STENCIL_ENABLE_MASK                                                                 0x00000001L
-#define DB_DEPTH_CONTROL__Z_ENABLE_MASK                                                                       0x00000002L
-#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE_MASK                                                                 0x00000004L
-#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE_MASK                                                            0x00000008L
-#define DB_DEPTH_CONTROL__ZFUNC_MASK                                                                          0x00000070L
-#define DB_DEPTH_CONTROL__BACKFACE_ENABLE_MASK                                                                0x00000080L
-#define DB_DEPTH_CONTROL__STENCILFUNC_MASK                                                                    0x00000700L
-#define DB_DEPTH_CONTROL__STENCILFUNC_BF_MASK                                                                 0x00700000L
-#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_MASK                                              0x40000000L
-#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS_MASK                                             0x80000000L
-//DB_EQAA
-#define DB_EQAA__MAX_ANCHOR_SAMPLES__SHIFT                                                                    0x0
-#define DB_EQAA__PS_ITER_SAMPLES__SHIFT                                                                       0x4
-#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES__SHIFT                                                               0x8
-#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES__SHIFT                                                             0xc
-#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS__SHIFT                                                            0x10
-#define DB_EQAA__INCOHERENT_EQAA_READS__SHIFT                                                                 0x11
-#define DB_EQAA__INTERPOLATE_COMP_Z__SHIFT                                                                    0x12
-#define DB_EQAA__INTERPOLATE_SRC_Z__SHIFT                                                                     0x13
-#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS__SHIFT                                                            0x14
-#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE__SHIFT                                                            0x15
-#define DB_EQAA__OVERRASTERIZATION_AMOUNT__SHIFT                                                              0x18
-#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION__SHIFT                                                        0x1b
-#define DB_EQAA__MAX_ANCHOR_SAMPLES_MASK                                                                      0x00000007L
-#define DB_EQAA__PS_ITER_SAMPLES_MASK                                                                         0x00000070L
-#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES_MASK                                                                 0x00000700L
-#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES_MASK                                                               0x00007000L
-#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS_MASK                                                              0x00010000L
-#define DB_EQAA__INCOHERENT_EQAA_READS_MASK                                                                   0x00020000L
-#define DB_EQAA__INTERPOLATE_COMP_Z_MASK                                                                      0x00040000L
-#define DB_EQAA__INTERPOLATE_SRC_Z_MASK                                                                       0x00080000L
-#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS_MASK                                                              0x00100000L
-#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE_MASK                                                              0x00200000L
-#define DB_EQAA__OVERRASTERIZATION_AMOUNT_MASK                                                                0x07000000L
-#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION_MASK                                                          0x08000000L
-//CB_COLOR_CONTROL
-#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD__SHIFT                                                            0x0
-#define CB_COLOR_CONTROL__DEGAMMA_ENABLE__SHIFT                                                               0x3
-#define CB_COLOR_CONTROL__MODE__SHIFT                                                                         0x4
-#define CB_COLOR_CONTROL__ROP3__SHIFT                                                                         0x10
-#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD_MASK                                                              0x00000001L
-#define CB_COLOR_CONTROL__DEGAMMA_ENABLE_MASK                                                                 0x00000008L
-#define CB_COLOR_CONTROL__MODE_MASK                                                                           0x00000070L
-#define CB_COLOR_CONTROL__ROP3_MASK                                                                           0x00FF0000L
-//DB_SHADER_CONTROL
-#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE__SHIFT                                                             0x0
-#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE__SHIFT                                              0x1
-#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE__SHIFT                                                0x2
-#define DB_SHADER_CONTROL__Z_ORDER__SHIFT                                                                     0x4
-#define DB_SHADER_CONTROL__KILL_ENABLE__SHIFT                                                                 0x6
-#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE__SHIFT                                                     0x7
-#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE__SHIFT                                                          0x8
-#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL__SHIFT                                                           0x9
-#define DB_SHADER_CONTROL__EXEC_ON_NOOP__SHIFT                                                                0xa
-#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE__SHIFT                                                       0xb
-#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER__SHIFT                                                         0xc
-#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT                                                       0xd
-#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE__SHIFT                                                           0xf
-#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER__SHIFT                                              0x10
-#define DB_SHADER_CONTROL__EXEC_IF_OVERLAPPED__SHIFT                                                          0x11
-#define DB_SHADER_CONTROL__POPS_OVERLAP_NUM_SAMPLES__SHIFT                                                    0x14
-#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE_MASK                                                               0x00000001L
-#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE_MASK                                                0x00000002L
-#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE_MASK                                                  0x00000004L
-#define DB_SHADER_CONTROL__Z_ORDER_MASK                                                                       0x00000030L
-#define DB_SHADER_CONTROL__KILL_ENABLE_MASK                                                                   0x00000040L
-#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE_MASK                                                       0x00000080L
-#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE_MASK                                                            0x00000100L
-#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL_MASK                                                             0x00000200L
-#define DB_SHADER_CONTROL__EXEC_ON_NOOP_MASK                                                                  0x00000400L
-#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE_MASK                                                         0x00000800L
-#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER_MASK                                                           0x00001000L
-#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT_MASK                                                         0x00006000L
-#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE_MASK                                                             0x00008000L
-#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER_MASK                                                0x00010000L
-#define DB_SHADER_CONTROL__EXEC_IF_OVERLAPPED_MASK                                                            0x00020000L
-#define DB_SHADER_CONTROL__POPS_OVERLAP_NUM_SAMPLES_MASK                                                      0x00700000L
-//PA_CL_CLIP_CNTL
-#define PA_CL_CLIP_CNTL__UCP_ENA_0__SHIFT                                                                     0x0
-#define PA_CL_CLIP_CNTL__UCP_ENA_1__SHIFT                                                                     0x1
-#define PA_CL_CLIP_CNTL__UCP_ENA_2__SHIFT                                                                     0x2
-#define PA_CL_CLIP_CNTL__UCP_ENA_3__SHIFT                                                                     0x3
-#define PA_CL_CLIP_CNTL__UCP_ENA_4__SHIFT                                                                     0x4
-#define PA_CL_CLIP_CNTL__UCP_ENA_5__SHIFT                                                                     0x5
-#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG__SHIFT                                                            0xd
-#define PA_CL_CLIP_CNTL__PS_UCP_MODE__SHIFT                                                                   0xe
-#define PA_CL_CLIP_CNTL__CLIP_DISABLE__SHIFT                                                                  0x10
-#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA__SHIFT                                                             0x11
-#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA__SHIFT                                                        0x12
-#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF__SHIFT                                                             0x13
-#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT__SHIFT                                                           0x14
-#define PA_CL_CLIP_CNTL__VTX_KILL_OR__SHIFT                                                                   0x15
-#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL__SHIFT                                                         0x16
-#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA__SHIFT                                                       0x18
-#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE__SHIFT                                                     0x19
-#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE__SHIFT                                                            0x1a
-#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE__SHIFT                                                             0x1b
-#define PA_CL_CLIP_CNTL__UCP_ENA_0_MASK                                                                       0x00000001L
-#define PA_CL_CLIP_CNTL__UCP_ENA_1_MASK                                                                       0x00000002L
-#define PA_CL_CLIP_CNTL__UCP_ENA_2_MASK                                                                       0x00000004L
-#define PA_CL_CLIP_CNTL__UCP_ENA_3_MASK                                                                       0x00000008L
-#define PA_CL_CLIP_CNTL__UCP_ENA_4_MASK                                                                       0x00000010L
-#define PA_CL_CLIP_CNTL__UCP_ENA_5_MASK                                                                       0x00000020L
-#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG_MASK                                                              0x00002000L
-#define PA_CL_CLIP_CNTL__PS_UCP_MODE_MASK                                                                     0x0000C000L
-#define PA_CL_CLIP_CNTL__CLIP_DISABLE_MASK                                                                    0x00010000L
-#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA_MASK                                                               0x00020000L
-#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA_MASK                                                          0x00040000L
-#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF_MASK                                                               0x00080000L
-#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT_MASK                                                             0x00100000L
-#define PA_CL_CLIP_CNTL__VTX_KILL_OR_MASK                                                                     0x00200000L
-#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL_MASK                                                           0x00400000L
-#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA_MASK                                                         0x01000000L
-#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE_MASK                                                       0x02000000L
-#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE_MASK                                                              0x04000000L
-#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE_MASK                                                               0x08000000L
-//PA_SU_SC_MODE_CNTL
-#define PA_SU_SC_MODE_CNTL__CULL_FRONT__SHIFT                                                                 0x0
-#define PA_SU_SC_MODE_CNTL__CULL_BACK__SHIFT                                                                  0x1
-#define PA_SU_SC_MODE_CNTL__FACE__SHIFT                                                                       0x2
-#define PA_SU_SC_MODE_CNTL__POLY_MODE__SHIFT                                                                  0x3
-#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE__SHIFT                                                       0x5
-#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE__SHIFT                                                        0x8
-#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE__SHIFT                                                   0xb
-#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE__SHIFT                                                    0xc
-#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE__SHIFT                                                    0xd
-#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE__SHIFT                                                   0x10
-#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST__SHIFT                                                         0x13
-#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS__SHIFT                                                             0x14
-#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA__SHIFT                                                          0x15
-#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF__SHIFT                                      0x16
-#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION__SHIFT                                                     0x17
-#define PA_SU_SC_MODE_CNTL__CULL_FRONT_MASK                                                                   0x00000001L
-#define PA_SU_SC_MODE_CNTL__CULL_BACK_MASK                                                                    0x00000002L
-#define PA_SU_SC_MODE_CNTL__FACE_MASK                                                                         0x00000004L
-#define PA_SU_SC_MODE_CNTL__POLY_MODE_MASK                                                                    0x00000018L
-#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE_MASK                                                         0x000000E0L
-#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE_MASK                                                          0x00000700L
-#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE_MASK                                                     0x00000800L
-#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE_MASK                                                      0x00001000L
-#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE_MASK                                                      0x00002000L
-#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE_MASK                                                     0x00010000L
-#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST_MASK                                                           0x00080000L
-#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS_MASK                                                               0x00100000L
-#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA_MASK                                                            0x00200000L
-#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF_MASK                                        0x00400000L
-#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION_MASK                                                       0x00800000L
-//PA_CL_VTE_CNTL
-#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA__SHIFT                                                              0x0
-#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA__SHIFT                                                             0x1
-#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA__SHIFT                                                              0x2
-#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA__SHIFT                                                             0x3
-#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA__SHIFT                                                              0x4
-#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA__SHIFT                                                             0x5
-#define PA_CL_VTE_CNTL__VTX_XY_FMT__SHIFT                                                                     0x8
-#define PA_CL_VTE_CNTL__VTX_Z_FMT__SHIFT                                                                      0x9
-#define PA_CL_VTE_CNTL__VTX_W0_FMT__SHIFT                                                                     0xa
-#define PA_CL_VTE_CNTL__PERFCOUNTER_REF__SHIFT                                                                0xb
-#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA_MASK                                                                0x00000001L
-#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA_MASK                                                               0x00000002L
-#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA_MASK                                                                0x00000004L
-#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA_MASK                                                               0x00000008L
-#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA_MASK                                                                0x00000010L
-#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA_MASK                                                               0x00000020L
-#define PA_CL_VTE_CNTL__VTX_XY_FMT_MASK                                                                       0x00000100L
-#define PA_CL_VTE_CNTL__VTX_Z_FMT_MASK                                                                        0x00000200L
-#define PA_CL_VTE_CNTL__VTX_W0_FMT_MASK                                                                       0x00000400L
-#define PA_CL_VTE_CNTL__PERFCOUNTER_REF_MASK                                                                  0x00000800L
-//PA_CL_VS_OUT_CNTL
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0__SHIFT                                                             0x0
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1__SHIFT                                                             0x1
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2__SHIFT                                                             0x2
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3__SHIFT                                                             0x3
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4__SHIFT                                                             0x4
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5__SHIFT                                                             0x5
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6__SHIFT                                                             0x6
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7__SHIFT                                                             0x7
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0__SHIFT                                                             0x8
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1__SHIFT                                                             0x9
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2__SHIFT                                                             0xa
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3__SHIFT                                                             0xb
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4__SHIFT                                                             0xc
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5__SHIFT                                                             0xd
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6__SHIFT                                                             0xe
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7__SHIFT                                                             0xf
-#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE__SHIFT                                                          0x10
-#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG__SHIFT                                                           0x11
-#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX__SHIFT                                                  0x12
-#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX__SHIFT                                                       0x13
-#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG__SHIFT                                                           0x14
-#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA__SHIFT                                                         0x15
-#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA__SHIFT                                                      0x16
-#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA__SHIFT                                                      0x17
-#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT                                                    0x18
-#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG__SHIFT                                                         0x19
-#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH__SHIFT                                                          0x1a
-#define PA_CL_VS_OUT_CNTL__USE_VTX_SHD_OBJPRIM_ID__SHIFT                                                      0x1b
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK                                                               0x00000001L
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1_MASK                                                               0x00000002L
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2_MASK                                                               0x00000004L
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3_MASK                                                               0x00000008L
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4_MASK                                                               0x00000010L
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5_MASK                                                               0x00000020L
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6_MASK                                                               0x00000040L
-#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7_MASK                                                               0x00000080L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0_MASK                                                               0x00000100L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1_MASK                                                               0x00000200L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2_MASK                                                               0x00000400L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3_MASK                                                               0x00000800L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4_MASK                                                               0x00001000L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5_MASK                                                               0x00002000L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6_MASK                                                               0x00004000L
-#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7_MASK                                                               0x00008000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE_MASK                                                            0x00010000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG_MASK                                                             0x00020000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX_MASK                                                    0x00040000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX_MASK                                                         0x00080000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG_MASK                                                             0x00100000L
-#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA_MASK                                                           0x00200000L
-#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA_MASK                                                        0x00400000L
-#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA_MASK                                                        0x00800000L
-#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK                                                      0x01000000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG_MASK                                                           0x02000000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH_MASK                                                            0x04000000L
-#define PA_CL_VS_OUT_CNTL__USE_VTX_SHD_OBJPRIM_ID_MASK                                                        0x08000000L
-//PA_CL_NANINF_CNTL
-#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD__SHIFT                                                          0x0
-#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD__SHIFT                                                           0x1
-#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD__SHIFT                                                           0x2
-#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0__SHIFT                                                           0x3
-#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN__SHIFT                                                           0x4
-#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN__SHIFT                                                            0x5
-#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN__SHIFT                                                            0x6
-#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0__SHIFT                                                        0x7
-#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF__SHIFT                                                            0x8
-#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN__SHIFT                                                            0x9
-#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF__SHIFT                                                             0xa
-#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN__SHIFT                                                             0xb
-#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF__SHIFT                                                             0xc
-#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN__SHIFT                                                             0xd
-#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD__SHIFT                                                    0xe
-#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0__SHIFT                                                         0x14
-#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD_MASK                                                            0x00000001L
-#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD_MASK                                                             0x00000002L
-#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD_MASK                                                             0x00000004L
-#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0_MASK                                                             0x00000008L
-#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN_MASK                                                             0x00000010L
-#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN_MASK                                                              0x00000020L
-#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN_MASK                                                              0x00000040L
-#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0_MASK                                                          0x00000080L
-#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF_MASK                                                              0x00000100L
-#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN_MASK                                                              0x00000200L
-#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF_MASK                                                               0x00000400L
-#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN_MASK                                                               0x00000800L
-#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF_MASK                                                               0x00001000L
-#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN_MASK                                                               0x00002000L
-#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD_MASK                                                      0x00004000L
-#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0_MASK                                                           0x00100000L
-//PA_SU_LINE_STIPPLE_CNTL
-#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET__SHIFT                                                    0x0
-#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH__SHIFT                                                    0x2
-#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM__SHIFT                                                      0x3
-#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST__SHIFT                                                        0x4
-#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET_MASK                                                      0x00000003L
-#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH_MASK                                                      0x00000004L
-#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM_MASK                                                        0x00000008L
-#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST_MASK                                                          0x00000010L
-//PA_SU_LINE_STIPPLE_SCALE
-#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE__SHIFT                                                   0x0
-#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE_MASK                                                     0xFFFFFFFFL
-//PA_SU_PRIM_FILTER_CNTL
-#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                                0x0
-#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                                    0x1
-#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                                   0x2
-#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                               0x3
-#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA__SHIFT                                                    0x4
-#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA__SHIFT                                                        0x5
-#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA__SHIFT                                                       0x6
-#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA__SHIFT                                                   0x7
-#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT__SHIFT                                                   0x8
-#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION__SHIFT                                                   0x1e
-#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION__SHIFT                                                  0x1f
-#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                                  0x00000001L
-#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                      0x00000002L
-#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                                     0x00000004L
-#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                                 0x00000008L
-#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA_MASK                                                      0x00000010L
-#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA_MASK                                                          0x00000020L
-#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA_MASK                                                         0x00000040L
-#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA_MASK                                                     0x00000080L
-#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT_MASK                                                     0x0000FF00L
-#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION_MASK                                                     0x40000000L
-#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION_MASK                                                    0x80000000L
-//PA_SU_SMALL_PRIM_FILTER_CNTL
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE__SHIFT                                         0x0
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                          0x1
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                              0x2
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                             0x3
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                         0x4
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__SRBSL_ENABLE__SHIFT                                                     0x5
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE_MASK                                           0x00000001L
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                            0x00000002L
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                0x00000004L
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                               0x00000008L
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                           0x00000010L
-#define PA_SU_SMALL_PRIM_FILTER_CNTL__SRBSL_ENABLE_MASK                                                       0x00000020L
-//PA_CL_OBJPRIM_ID_CNTL
-#define PA_CL_OBJPRIM_ID_CNTL__OBJ_ID_SEL__SHIFT                                                              0x0
-#define PA_CL_OBJPRIM_ID_CNTL__ADD_PIPED_PRIM_ID__SHIFT                                                       0x1
-#define PA_CL_OBJPRIM_ID_CNTL__EN_32BIT_OBJPRIMID__SHIFT                                                      0x2
-#define PA_CL_OBJPRIM_ID_CNTL__OBJ_ID_SEL_MASK                                                                0x00000001L
-#define PA_CL_OBJPRIM_ID_CNTL__ADD_PIPED_PRIM_ID_MASK                                                         0x00000002L
-#define PA_CL_OBJPRIM_ID_CNTL__EN_32BIT_OBJPRIMID_MASK                                                        0x00000004L
-//PA_CL_NGG_CNTL
-#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF__SHIFT                                                               0x0
-#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA__SHIFT                                                        0x1
-#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF_MASK                                                                 0x00000001L
-#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA_MASK                                                          0x00000002L
-//PA_SU_OVER_RASTERIZATION_CNTL
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES__SHIFT                                        0x0
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES__SHIFT                                            0x1
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS__SHIFT                                           0x2
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES__SHIFT                                       0x3
-#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW__SHIFT                                                0x4
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES_MASK                                          0x00000001L
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES_MASK                                              0x00000002L
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS_MASK                                             0x00000004L
-#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES_MASK                                         0x00000008L
-#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW_MASK                                                  0x00000010L
-//PA_SU_POINT_SIZE
-#define PA_SU_POINT_SIZE__HEIGHT__SHIFT                                                                       0x0
-#define PA_SU_POINT_SIZE__WIDTH__SHIFT                                                                        0x10
-#define PA_SU_POINT_SIZE__HEIGHT_MASK                                                                         0x0000FFFFL
-#define PA_SU_POINT_SIZE__WIDTH_MASK                                                                          0xFFFF0000L
-//PA_SU_POINT_MINMAX
-#define PA_SU_POINT_MINMAX__MIN_SIZE__SHIFT                                                                   0x0
-#define PA_SU_POINT_MINMAX__MAX_SIZE__SHIFT                                                                   0x10
-#define PA_SU_POINT_MINMAX__MIN_SIZE_MASK                                                                     0x0000FFFFL
-#define PA_SU_POINT_MINMAX__MAX_SIZE_MASK                                                                     0xFFFF0000L
-//PA_SU_LINE_CNTL
-#define PA_SU_LINE_CNTL__WIDTH__SHIFT                                                                         0x0
-#define PA_SU_LINE_CNTL__WIDTH_MASK                                                                           0x0000FFFFL
-//PA_SC_LINE_STIPPLE
-#define PA_SC_LINE_STIPPLE__LINE_PATTERN__SHIFT                                                               0x0
-#define PA_SC_LINE_STIPPLE__REPEAT_COUNT__SHIFT                                                               0x10
-#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER__SHIFT                                                          0x1c
-#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL__SHIFT                                                            0x1d
-#define PA_SC_LINE_STIPPLE__LINE_PATTERN_MASK                                                                 0x0000FFFFL
-#define PA_SC_LINE_STIPPLE__REPEAT_COUNT_MASK                                                                 0x00FF0000L
-#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER_MASK                                                            0x10000000L
-#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL_MASK                                                              0x60000000L
-//VGT_OUTPUT_PATH_CNTL
-#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT__SHIFT                                                              0x0
-#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT_MASK                                                                0x00000007L
-//VGT_HOS_CNTL
-#define VGT_HOS_CNTL__TESS_MODE__SHIFT                                                                        0x0
-#define VGT_HOS_CNTL__TESS_MODE_MASK                                                                          0x00000003L
-//VGT_HOS_MAX_TESS_LEVEL
-#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS__SHIFT                                                               0x0
-#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS_MASK                                                                 0xFFFFFFFFL
-//VGT_HOS_MIN_TESS_LEVEL
-#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS__SHIFT                                                               0x0
-#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS_MASK                                                                 0xFFFFFFFFL
-//VGT_HOS_REUSE_DEPTH
-#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH__SHIFT                                                               0x0
-#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH_MASK                                                                 0x000000FFL
-//VGT_GROUP_PRIM_TYPE
-#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE__SHIFT                                                                 0x0
-#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER__SHIFT                                                              0xe
-#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS__SHIFT                                                              0xf
-#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER__SHIFT                                                                0x10
-#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE_MASK                                                                   0x0000001FL
-#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER_MASK                                                                0x00004000L
-#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS_MASK                                                                0x00008000L
-#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER_MASK                                                                  0x00070000L
-//VGT_GROUP_FIRST_DECR
-#define VGT_GROUP_FIRST_DECR__FIRST_DECR__SHIFT                                                               0x0
-#define VGT_GROUP_FIRST_DECR__FIRST_DECR_MASK                                                                 0x0000000FL
-//VGT_GROUP_DECR
-#define VGT_GROUP_DECR__DECR__SHIFT                                                                           0x0
-#define VGT_GROUP_DECR__DECR_MASK                                                                             0x0000000FL
-//VGT_GROUP_VECT_0_CNTL
-#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN__SHIFT                                                               0x0
-#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN__SHIFT                                                               0x1
-#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN__SHIFT                                                               0x2
-#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN__SHIFT                                                               0x3
-#define VGT_GROUP_VECT_0_CNTL__STRIDE__SHIFT                                                                  0x8
-#define VGT_GROUP_VECT_0_CNTL__SHIFT__SHIFT                                                                   0x10
-#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN_MASK                                                                 0x00000001L
-#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN_MASK                                                                 0x00000002L
-#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN_MASK                                                                 0x00000004L
-#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN_MASK                                                                 0x00000008L
-#define VGT_GROUP_VECT_0_CNTL__STRIDE_MASK                                                                    0x0000FF00L
-#define VGT_GROUP_VECT_0_CNTL__SHIFT_MASK                                                                     0x00FF0000L
-//VGT_GROUP_VECT_1_CNTL
-#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN__SHIFT                                                               0x0
-#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN__SHIFT                                                               0x1
-#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN__SHIFT                                                               0x2
-#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN__SHIFT                                                               0x3
-#define VGT_GROUP_VECT_1_CNTL__STRIDE__SHIFT                                                                  0x8
-#define VGT_GROUP_VECT_1_CNTL__SHIFT__SHIFT                                                                   0x10
-#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN_MASK                                                                 0x00000001L
-#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN_MASK                                                                 0x00000002L
-#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN_MASK                                                                 0x00000004L
-#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN_MASK                                                                 0x00000008L
-#define VGT_GROUP_VECT_1_CNTL__STRIDE_MASK                                                                    0x0000FF00L
-#define VGT_GROUP_VECT_1_CNTL__SHIFT_MASK                                                                     0x00FF0000L
-//VGT_GROUP_VECT_0_FMT_CNTL
-#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV__SHIFT                                                              0x0
-#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET__SHIFT                                                            0x4
-#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV__SHIFT                                                              0x8
-#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET__SHIFT                                                            0xc
-#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV__SHIFT                                                              0x10
-#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET__SHIFT                                                            0x14
-#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV__SHIFT                                                              0x18
-#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET__SHIFT                                                            0x1c
-#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV_MASK                                                                0x0000000FL
-#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET_MASK                                                              0x000000F0L
-#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV_MASK                                                                0x00000F00L
-#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET_MASK                                                              0x0000F000L
-#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV_MASK                                                                0x000F0000L
-#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET_MASK                                                              0x00F00000L
-#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV_MASK                                                                0x0F000000L
-#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET_MASK                                                              0xF0000000L
-//VGT_GROUP_VECT_1_FMT_CNTL
-#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV__SHIFT                                                              0x0
-#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET__SHIFT                                                            0x4
-#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV__SHIFT                                                              0x8
-#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET__SHIFT                                                            0xc
-#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV__SHIFT                                                              0x10
-#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET__SHIFT                                                            0x14
-#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV__SHIFT                                                              0x18
-#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET__SHIFT                                                            0x1c
-#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV_MASK                                                                0x0000000FL
-#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET_MASK                                                              0x000000F0L
-#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV_MASK                                                                0x00000F00L
-#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET_MASK                                                              0x0000F000L
-#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV_MASK                                                                0x000F0000L
-#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET_MASK                                                              0x00F00000L
-#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV_MASK                                                                0x0F000000L
-#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET_MASK                                                              0xF0000000L
-//VGT_GS_MODE
-#define VGT_GS_MODE__MODE__SHIFT                                                                              0x0
-#define VGT_GS_MODE__RESERVED_0__SHIFT                                                                        0x3
-#define VGT_GS_MODE__CUT_MODE__SHIFT                                                                          0x4
-#define VGT_GS_MODE__RESERVED_1__SHIFT                                                                        0x6
-#define VGT_GS_MODE__GS_C_PACK_EN__SHIFT                                                                      0xb
-#define VGT_GS_MODE__RESERVED_2__SHIFT                                                                        0xc
-#define VGT_GS_MODE__ES_PASSTHRU__SHIFT                                                                       0xd
-#define VGT_GS_MODE__RESERVED_3__SHIFT                                                                        0xe
-#define VGT_GS_MODE__RESERVED_4__SHIFT                                                                        0xf
-#define VGT_GS_MODE__RESERVED_5__SHIFT                                                                        0x10
-#define VGT_GS_MODE__PARTIAL_THD_AT_EOI__SHIFT                                                                0x11
-#define VGT_GS_MODE__SUPPRESS_CUTS__SHIFT                                                                     0x12
-#define VGT_GS_MODE__ES_WRITE_OPTIMIZE__SHIFT                                                                 0x13
-#define VGT_GS_MODE__GS_WRITE_OPTIMIZE__SHIFT                                                                 0x14
-#define VGT_GS_MODE__ONCHIP__SHIFT                                                                            0x15
-#define VGT_GS_MODE__MODE_MASK                                                                                0x00000007L
-#define VGT_GS_MODE__RESERVED_0_MASK                                                                          0x00000008L
-#define VGT_GS_MODE__CUT_MODE_MASK                                                                            0x00000030L
-#define VGT_GS_MODE__RESERVED_1_MASK                                                                          0x000007C0L
-#define VGT_GS_MODE__GS_C_PACK_EN_MASK                                                                        0x00000800L
-#define VGT_GS_MODE__RESERVED_2_MASK                                                                          0x00001000L
-#define VGT_GS_MODE__ES_PASSTHRU_MASK                                                                         0x00002000L
-#define VGT_GS_MODE__RESERVED_3_MASK                                                                          0x00004000L
-#define VGT_GS_MODE__RESERVED_4_MASK                                                                          0x00008000L
-#define VGT_GS_MODE__RESERVED_5_MASK                                                                          0x00010000L
-#define VGT_GS_MODE__PARTIAL_THD_AT_EOI_MASK                                                                  0x00020000L
-#define VGT_GS_MODE__SUPPRESS_CUTS_MASK                                                                       0x00040000L
-#define VGT_GS_MODE__ES_WRITE_OPTIMIZE_MASK                                                                   0x00080000L
-#define VGT_GS_MODE__GS_WRITE_OPTIMIZE_MASK                                                                   0x00100000L
-#define VGT_GS_MODE__ONCHIP_MASK                                                                              0x00600000L
-//VGT_GS_ONCHIP_CNTL
-#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP__SHIFT                                                        0x0
-#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP__SHIFT                                                        0xb
-#define VGT_GS_ONCHIP_CNTL__GS_INST_PRIMS_IN_SUBGRP__SHIFT                                                    0x16
-#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP_MASK                                                          0x000007FFL
-#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP_MASK                                                          0x003FF800L
-#define VGT_GS_ONCHIP_CNTL__GS_INST_PRIMS_IN_SUBGRP_MASK                                                      0xFFC00000L
-//PA_SC_MODE_CNTL_0
-#define PA_SC_MODE_CNTL_0__MSAA_ENABLE__SHIFT                                                                 0x0
-#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE__SHIFT                                                        0x1
-#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE__SHIFT                                                         0x2
-#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR__SHIFT                                                    0x3
-#define PA_SC_MODE_CNTL_0__SCALE_LINE_WIDTH_PAD__SHIFT                                                        0x4
-#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE__SHIFT                                                      0x5
-#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB__SHIFT                                               0x6
-#define PA_SC_MODE_CNTL_0__MSAA_ENABLE_MASK                                                                   0x00000001L
-#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE_MASK                                                          0x00000002L
-#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE_MASK                                                           0x00000004L
-#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR_MASK                                                      0x00000008L
-#define PA_SC_MODE_CNTL_0__SCALE_LINE_WIDTH_PAD_MASK                                                          0x00000010L
-#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE_MASK                                                        0x00000020L
-#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB_MASK                                                 0x00000040L
-//PA_SC_MODE_CNTL_1
-#define PA_SC_MODE_CNTL_1__WALK_SIZE__SHIFT                                                                   0x0
-#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT__SHIFT                                                              0x1
-#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST__SHIFT                                                    0x2
-#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE__SHIFT                                                           0x3
-#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE__SHIFT                                                             0x4
-#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE__SHIFT                                                 0x7
-#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE__SHIFT                                                      0x8
-#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE__SHIFT                                                          0x9
-#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR__SHIFT                                                       0xa
-#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT__SHIFT                                                             0xb
-#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET__SHIFT                                                             0xc
-#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT__SHIFT                                                             0xd
-#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z__SHIFT                                                          0xe
-#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK__SHIFT                                                   0xf
-#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE__SHIFT                                                              0x10
-#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE__SHIFT                                     0x11
-#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE__SHIFT                                                  0x12
-#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE__SHIFT                                                      0x13
-#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE__SHIFT                                                             0x14
-#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE__SHIFT                                               0x18
-#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE__SHIFT                                                     0x19
-#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE__SHIFT                                                        0x1a
-#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE__SHIFT                                               0x1b
-#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK__SHIFT                                                     0x1c
-#define PA_SC_MODE_CNTL_1__WALK_SIZE_MASK                                                                     0x00000001L
-#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT_MASK                                                                0x00000002L
-#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST_MASK                                                      0x00000004L
-#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE_MASK                                                             0x00000008L
-#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE_MASK                                                               0x00000070L
-#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE_MASK                                                   0x00000080L
-#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE_MASK                                                        0x00000100L
-#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE_MASK                                                            0x00000200L
-#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR_MASK                                                         0x00000400L
-#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT_MASK                                                               0x00000800L
-#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET_MASK                                                               0x00001000L
-#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT_MASK                                                               0x00002000L
-#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z_MASK                                                            0x00004000L
-#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK_MASK                                                     0x00008000L
-#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE_MASK                                                                0x00010000L
-#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE_MASK                                       0x00020000L
-#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE_MASK                                                    0x00040000L
-#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE_MASK                                                        0x00080000L
-#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_MASK                                                               0x00F00000L
-#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE_MASK                                                 0x01000000L
-#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE_MASK                                                       0x02000000L
-#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE_MASK                                                          0x04000000L
-#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE_MASK                                                 0x08000000L
-#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK_MASK                                                       0x70000000L
-//VGT_ENHANCE
-#define VGT_ENHANCE__MISC__SHIFT                                                                              0x0
-#define VGT_ENHANCE__MISC_MASK                                                                                0xFFFFFFFFL
-//VGT_GS_PER_ES
-#define VGT_GS_PER_ES__GS_PER_ES__SHIFT                                                                       0x0
-#define VGT_GS_PER_ES__GS_PER_ES_MASK                                                                         0x000007FFL
-//VGT_ES_PER_GS
-#define VGT_ES_PER_GS__ES_PER_GS__SHIFT                                                                       0x0
-#define VGT_ES_PER_GS__ES_PER_GS_MASK                                                                         0x000007FFL
-//VGT_GS_PER_VS
-#define VGT_GS_PER_VS__GS_PER_VS__SHIFT                                                                       0x0
-#define VGT_GS_PER_VS__GS_PER_VS_MASK                                                                         0x0000000FL
-//VGT_GSVS_RING_OFFSET_1
-#define VGT_GSVS_RING_OFFSET_1__OFFSET__SHIFT                                                                 0x0
-#define VGT_GSVS_RING_OFFSET_1__OFFSET_MASK                                                                   0x00007FFFL
-//VGT_GSVS_RING_OFFSET_2
-#define VGT_GSVS_RING_OFFSET_2__OFFSET__SHIFT                                                                 0x0
-#define VGT_GSVS_RING_OFFSET_2__OFFSET_MASK                                                                   0x00007FFFL
-//VGT_GSVS_RING_OFFSET_3
-#define VGT_GSVS_RING_OFFSET_3__OFFSET__SHIFT                                                                 0x0
-#define VGT_GSVS_RING_OFFSET_3__OFFSET_MASK                                                                   0x00007FFFL
-//VGT_GS_OUT_PRIM_TYPE
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE__SHIFT                                                             0x0
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1__SHIFT                                                           0x8
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2__SHIFT                                                           0x10
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3__SHIFT                                                           0x16
-#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM__SHIFT                                                   0x1f
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_MASK                                                               0x0000003FL
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1_MASK                                                             0x00003F00L
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2_MASK                                                             0x003F0000L
-#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3_MASK                                                             0x0FC00000L
-#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM_MASK                                                     0x80000000L
-//IA_ENHANCE
-#define IA_ENHANCE__MISC__SHIFT                                                                               0x0
-#define IA_ENHANCE__MISC_MASK                                                                                 0xFFFFFFFFL
-//VGT_DMA_SIZE
-#define VGT_DMA_SIZE__NUM_INDICES__SHIFT                                                                      0x0
-#define VGT_DMA_SIZE__NUM_INDICES_MASK                                                                        0xFFFFFFFFL
-//VGT_DMA_MAX_SIZE
-#define VGT_DMA_MAX_SIZE__MAX_SIZE__SHIFT                                                                     0x0
-#define VGT_DMA_MAX_SIZE__MAX_SIZE_MASK                                                                       0xFFFFFFFFL
-//VGT_DMA_INDEX_TYPE
-#define VGT_DMA_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                 0x0
-#define VGT_DMA_INDEX_TYPE__SWAP_MODE__SHIFT                                                                  0x2
-#define VGT_DMA_INDEX_TYPE__BUF_TYPE__SHIFT                                                                   0x4
-#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY__SHIFT                                                               0x6
-#define VGT_DMA_INDEX_TYPE__PRIMGEN_EN__SHIFT                                                                 0x8
-#define VGT_DMA_INDEX_TYPE__NOT_EOP__SHIFT                                                                    0x9
-#define VGT_DMA_INDEX_TYPE__REQ_PATH__SHIFT                                                                   0xa
-#define VGT_DMA_INDEX_TYPE__INDEX_TYPE_MASK                                                                   0x00000003L
-#define VGT_DMA_INDEX_TYPE__SWAP_MODE_MASK                                                                    0x0000000CL
-#define VGT_DMA_INDEX_TYPE__BUF_TYPE_MASK                                                                     0x00000030L
-#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY_MASK                                                                 0x00000040L
-#define VGT_DMA_INDEX_TYPE__PRIMGEN_EN_MASK                                                                   0x00000100L
-#define VGT_DMA_INDEX_TYPE__NOT_EOP_MASK                                                                      0x00000200L
-#define VGT_DMA_INDEX_TYPE__REQ_PATH_MASK                                                                     0x00000400L
-//WD_ENHANCE
-#define WD_ENHANCE__MISC__SHIFT                                                                               0x0
-#define WD_ENHANCE__MISC_MASK                                                                                 0xFFFFFFFFL
-//VGT_PRIMITIVEID_EN
-#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN__SHIFT                                                             0x0
-#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI__SHIFT                                                       0x1
-#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE__SHIFT                                                   0x2
-#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN_MASK                                                               0x00000001L
-#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI_MASK                                                         0x00000002L
-#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE_MASK                                                     0x00000004L
-//VGT_DMA_NUM_INSTANCES
-#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                           0x0
-#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES_MASK                                                             0xFFFFFFFFL
-//VGT_PRIMITIVEID_RESET
-#define VGT_PRIMITIVEID_RESET__VALUE__SHIFT                                                                   0x0
-#define VGT_PRIMITIVEID_RESET__VALUE_MASK                                                                     0xFFFFFFFFL
-//VGT_EVENT_INITIATOR
-#define VGT_EVENT_INITIATOR__EVENT_TYPE__SHIFT                                                                0x0
-#define VGT_EVENT_INITIATOR__ADDRESS_HI__SHIFT                                                                0xa
-#define VGT_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT                                                            0x1b
-#define VGT_EVENT_INITIATOR__EVENT_TYPE_MASK                                                                  0x0000003FL
-#define VGT_EVENT_INITIATOR__ADDRESS_HI_MASK                                                                  0x07FFFC00L
-#define VGT_EVENT_INITIATOR__EXTENDED_EVENT_MASK                                                              0x08000000L
-//VGT_GS_MAX_PRIMS_PER_SUBGROUP
-#define VGT_GS_MAX_PRIMS_PER_SUBGROUP__MAX_PRIMS_PER_SUBGROUP__SHIFT                                          0x0
-#define VGT_GS_MAX_PRIMS_PER_SUBGROUP__MAX_PRIMS_PER_SUBGROUP_MASK                                            0x0000FFFFL
-//VGT_DRAW_PAYLOAD_CNTL
-#define VGT_DRAW_PAYLOAD_CNTL__OBJPRIM_ID_EN__SHIFT                                                           0x0
-#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX__SHIFT                                                         0x1
-#define VGT_DRAW_PAYLOAD_CNTL__EN_PIPELINE_PRIMID__SHIFT                                                      0x2
-#define VGT_DRAW_PAYLOAD_CNTL__OBJECT_ID_INST_EN__SHIFT                                                       0x3
-#define VGT_DRAW_PAYLOAD_CNTL__OBJPRIM_ID_EN_MASK                                                             0x00000001L
-#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX_MASK                                                           0x00000002L
-#define VGT_DRAW_PAYLOAD_CNTL__EN_PIPELINE_PRIMID_MASK                                                        0x00000004L
-#define VGT_DRAW_PAYLOAD_CNTL__OBJECT_ID_INST_EN_MASK                                                         0x00000008L
-//VGT_INDEX_PAYLOAD_CNTL
-#define VGT_INDEX_PAYLOAD_CNTL__COMPOUND_INDEX_EN__SHIFT                                                      0x0
-#define VGT_INDEX_PAYLOAD_CNTL__COMPOUND_INDEX_EN_MASK                                                        0x00000001L
-//VGT_INSTANCE_STEP_RATE_0
-#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE__SHIFT                                                            0x0
-#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE_MASK                                                              0xFFFFFFFFL
-//VGT_INSTANCE_STEP_RATE_1
-#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE__SHIFT                                                            0x0
-#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE_MASK                                                              0xFFFFFFFFL
-//VGT_ESGS_RING_ITEMSIZE
-#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE__SHIFT                                                               0x0
-#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE_MASK                                                                 0x00007FFFL
-//VGT_GSVS_RING_ITEMSIZE
-#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE__SHIFT                                                               0x0
-#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE_MASK                                                                 0x00007FFFL
-//VGT_REUSE_OFF
-#define VGT_REUSE_OFF__REUSE_OFF__SHIFT                                                                       0x0
-#define VGT_REUSE_OFF__REUSE_OFF_MASK                                                                         0x00000001L
-//VGT_VTX_CNT_EN
-#define VGT_VTX_CNT_EN__VTX_CNT_EN__SHIFT                                                                     0x0
-#define VGT_VTX_CNT_EN__VTX_CNT_EN_MASK                                                                       0x00000001L
-//DB_HTILE_SURFACE
-#define DB_HTILE_SURFACE__FULL_CACHE__SHIFT                                                                   0x1
-#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN__SHIFT                                                       0x2
-#define DB_HTILE_SURFACE__PRELOAD__SHIFT                                                                      0x3
-#define DB_HTILE_SURFACE__PREFETCH_WIDTH__SHIFT                                                               0x4
-#define DB_HTILE_SURFACE__PREFETCH_HEIGHT__SHIFT                                                              0xa
-#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT                                                      0x10
-#define DB_HTILE_SURFACE__PIPE_ALIGNED__SHIFT                                                                 0x12
-#define DB_HTILE_SURFACE__RB_ALIGNED__SHIFT                                                                   0x13
-#define DB_HTILE_SURFACE__FULL_CACHE_MASK                                                                     0x00000002L
-#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN_MASK                                                         0x00000004L
-#define DB_HTILE_SURFACE__PRELOAD_MASK                                                                        0x00000008L
-#define DB_HTILE_SURFACE__PREFETCH_WIDTH_MASK                                                                 0x000003F0L
-#define DB_HTILE_SURFACE__PREFETCH_HEIGHT_MASK                                                                0x0000FC00L
-#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK                                                        0x00010000L
-#define DB_HTILE_SURFACE__PIPE_ALIGNED_MASK                                                                   0x00040000L
-#define DB_HTILE_SURFACE__RB_ALIGNED_MASK                                                                     0x00080000L
-//DB_SRESULTS_COMPARE_STATE0
-#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT                                                       0x0
-#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT                                                      0x4
-#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0__SHIFT                                                       0xc
-#define DB_SRESULTS_COMPARE_STATE0__ENABLE0__SHIFT                                                            0x18
-#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0_MASK                                                         0x00000007L
-#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0_MASK                                                        0x00000FF0L
-#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0_MASK                                                         0x000FF000L
-#define DB_SRESULTS_COMPARE_STATE0__ENABLE0_MASK                                                              0x01000000L
-//DB_SRESULTS_COMPARE_STATE1
-#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1__SHIFT                                                       0x0
-#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1__SHIFT                                                      0x4
-#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1__SHIFT                                                       0xc
-#define DB_SRESULTS_COMPARE_STATE1__ENABLE1__SHIFT                                                            0x18
-#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1_MASK                                                         0x00000007L
-#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1_MASK                                                        0x00000FF0L
-#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1_MASK                                                         0x000FF000L
-#define DB_SRESULTS_COMPARE_STATE1__ENABLE1_MASK                                                              0x01000000L
-//DB_PRELOAD_CONTROL
-#define DB_PRELOAD_CONTROL__START_X__SHIFT                                                                    0x0
-#define DB_PRELOAD_CONTROL__START_Y__SHIFT                                                                    0x8
-#define DB_PRELOAD_CONTROL__MAX_X__SHIFT                                                                      0x10
-#define DB_PRELOAD_CONTROL__MAX_Y__SHIFT                                                                      0x18
-#define DB_PRELOAD_CONTROL__START_X_MASK                                                                      0x000000FFL
-#define DB_PRELOAD_CONTROL__START_Y_MASK                                                                      0x0000FF00L
-#define DB_PRELOAD_CONTROL__MAX_X_MASK                                                                        0x00FF0000L
-#define DB_PRELOAD_CONTROL__MAX_Y_MASK                                                                        0xFF000000L
-//VGT_STRMOUT_BUFFER_SIZE_0
-#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE__SHIFT                                                                0x0
-#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE_MASK                                                                  0xFFFFFFFFL
-//VGT_STRMOUT_VTX_STRIDE_0
-#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE__SHIFT                                                               0x0
-#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE_MASK                                                                 0x000003FFL
-//VGT_STRMOUT_BUFFER_OFFSET_0
-#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET__SHIFT                                                            0x0
-#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET_MASK                                                              0xFFFFFFFFL
-//VGT_STRMOUT_BUFFER_SIZE_1
-#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE__SHIFT                                                                0x0
-#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE_MASK                                                                  0xFFFFFFFFL
-//VGT_STRMOUT_VTX_STRIDE_1
-#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE__SHIFT                                                               0x0
-#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE_MASK                                                                 0x000003FFL
-//VGT_STRMOUT_BUFFER_OFFSET_1
-#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET__SHIFT                                                            0x0
-#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET_MASK                                                              0xFFFFFFFFL
-//VGT_STRMOUT_BUFFER_SIZE_2
-#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE__SHIFT                                                                0x0
-#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE_MASK                                                                  0xFFFFFFFFL
-//VGT_STRMOUT_VTX_STRIDE_2
-#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE__SHIFT                                                               0x0
-#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE_MASK                                                                 0x000003FFL
-//VGT_STRMOUT_BUFFER_OFFSET_2
-#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET__SHIFT                                                            0x0
-#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET_MASK                                                              0xFFFFFFFFL
-//VGT_STRMOUT_BUFFER_SIZE_3
-#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE__SHIFT                                                                0x0
-#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE_MASK                                                                  0xFFFFFFFFL
-//VGT_STRMOUT_VTX_STRIDE_3
-#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE__SHIFT                                                               0x0
-#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE_MASK                                                                 0x000003FFL
-//VGT_STRMOUT_BUFFER_OFFSET_3
-#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET__SHIFT                                                            0x0
-#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET_MASK                                                              0xFFFFFFFFL
-//VGT_STRMOUT_DRAW_OPAQUE_OFFSET
-#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET__SHIFT                                                         0x0
-#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
-//VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
-#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE__SHIFT                                               0x0
-#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE_MASK                                                 0xFFFFFFFFL
-//VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
-#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE__SHIFT                                           0x0
-#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE_MASK                                             0x000001FFL
-//VGT_GS_MAX_VERT_OUT
-#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT__SHIFT                                                              0x0
-#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT_MASK                                                                0x000007FFL
-//VGT_TESS_DISTRIBUTION
-#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE__SHIFT                                                           0x0
-#define VGT_TESS_DISTRIBUTION__ACCUM_TRI__SHIFT                                                               0x8
-#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD__SHIFT                                                              0x10
-#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT__SHIFT                                                             0x18
-#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT__SHIFT                                                              0x1d
-#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE_MASK                                                             0x000000FFL
-#define VGT_TESS_DISTRIBUTION__ACCUM_TRI_MASK                                                                 0x0000FF00L
-#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD_MASK                                                                0x00FF0000L
-#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT_MASK                                                               0x1F000000L
-#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT_MASK                                                                0xE0000000L
-//VGT_SHADER_STAGES_EN
-#define VGT_SHADER_STAGES_EN__LS_EN__SHIFT                                                                    0x0
-#define VGT_SHADER_STAGES_EN__HS_EN__SHIFT                                                                    0x2
-#define VGT_SHADER_STAGES_EN__ES_EN__SHIFT                                                                    0x3
-#define VGT_SHADER_STAGES_EN__GS_EN__SHIFT                                                                    0x5
-#define VGT_SHADER_STAGES_EN__VS_EN__SHIFT                                                                    0x6
-#define VGT_SHADER_STAGES_EN__DISPATCH_DRAW_EN__SHIFT                                                         0x9
-#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_0__SHIFT                                                      0xa
-#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_1__SHIFT                                                      0xb
-#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN__SHIFT                                                            0xc
-#define VGT_SHADER_STAGES_EN__PRIMGEN_EN__SHIFT                                                               0xd
-#define VGT_SHADER_STAGES_EN__ORDERED_ID_MODE__SHIFT                                                          0xe
-#define VGT_SHADER_STAGES_EN__MAX_PRIMGRP_IN_WAVE__SHIFT                                                      0xf
-#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH__SHIFT                                                           0x13
-#define VGT_SHADER_STAGES_EN__LS_EN_MASK                                                                      0x00000003L
-#define VGT_SHADER_STAGES_EN__HS_EN_MASK                                                                      0x00000004L
-#define VGT_SHADER_STAGES_EN__ES_EN_MASK                                                                      0x00000018L
-#define VGT_SHADER_STAGES_EN__GS_EN_MASK                                                                      0x00000020L
-#define VGT_SHADER_STAGES_EN__VS_EN_MASK                                                                      0x000000C0L
-#define VGT_SHADER_STAGES_EN__DISPATCH_DRAW_EN_MASK                                                           0x00000200L
-#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_0_MASK                                                        0x00000400L
-#define VGT_SHADER_STAGES_EN__DIS_DEALLOC_ACCUM_1_MASK                                                        0x00000800L
-#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN_MASK                                                              0x00001000L
-#define VGT_SHADER_STAGES_EN__PRIMGEN_EN_MASK                                                                 0x00002000L
-#define VGT_SHADER_STAGES_EN__ORDERED_ID_MODE_MASK                                                            0x00004000L
-#define VGT_SHADER_STAGES_EN__MAX_PRIMGRP_IN_WAVE_MASK                                                        0x00078000L
-#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH_MASK                                                             0x00080000L
-//VGT_LS_HS_CONFIG
-#define VGT_LS_HS_CONFIG__NUM_PATCHES__SHIFT                                                                  0x0
-#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT                                                              0x8
-#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP__SHIFT                                                             0xe
-#define VGT_LS_HS_CONFIG__NUM_PATCHES_MASK                                                                    0x000000FFL
-#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK                                                                0x00003F00L
-#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP_MASK                                                               0x000FC000L
-//VGT_GS_VERT_ITEMSIZE
-#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE__SHIFT                                                                 0x0
-#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE_MASK                                                                   0x00007FFFL
-//VGT_GS_VERT_ITEMSIZE_1
-#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE__SHIFT                                                               0x0
-#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE_MASK                                                                 0x00007FFFL
-//VGT_GS_VERT_ITEMSIZE_2
-#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE__SHIFT                                                               0x0
-#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE_MASK                                                                 0x00007FFFL
-//VGT_GS_VERT_ITEMSIZE_3
-#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE__SHIFT                                                               0x0
-#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE_MASK                                                                 0x00007FFFL
-//VGT_TF_PARAM
-#define VGT_TF_PARAM__TYPE__SHIFT                                                                             0x0
-#define VGT_TF_PARAM__PARTITIONING__SHIFT                                                                     0x2
-#define VGT_TF_PARAM__TOPOLOGY__SHIFT                                                                         0x5
-#define VGT_TF_PARAM__RESERVED_REDUC_AXIS__SHIFT                                                              0x8
-#define VGT_TF_PARAM__DEPRECATED__SHIFT                                                                       0x9
-#define VGT_TF_PARAM__DISABLE_DONUTS__SHIFT                                                                   0xe
-#define VGT_TF_PARAM__RDREQ_POLICY__SHIFT                                                                     0xf
-#define VGT_TF_PARAM__DISTRIBUTION_MODE__SHIFT                                                                0x11
-#define VGT_TF_PARAM__TYPE_MASK                                                                               0x00000003L
-#define VGT_TF_PARAM__PARTITIONING_MASK                                                                       0x0000001CL
-#define VGT_TF_PARAM__TOPOLOGY_MASK                                                                           0x000000E0L
-#define VGT_TF_PARAM__RESERVED_REDUC_AXIS_MASK                                                                0x00000100L
-#define VGT_TF_PARAM__DEPRECATED_MASK                                                                         0x00000200L
-#define VGT_TF_PARAM__DISABLE_DONUTS_MASK                                                                     0x00004000L
-#define VGT_TF_PARAM__RDREQ_POLICY_MASK                                                                       0x00008000L
-#define VGT_TF_PARAM__DISTRIBUTION_MODE_MASK                                                                  0x00060000L
-//DB_ALPHA_TO_MASK
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE__SHIFT                                                         0x0
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0__SHIFT                                                        0x8
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1__SHIFT                                                        0xa
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2__SHIFT                                                        0xc
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3__SHIFT                                                        0xe
-#define DB_ALPHA_TO_MASK__OFFSET_ROUND__SHIFT                                                                 0x10
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE_MASK                                                           0x00000001L
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0_MASK                                                          0x00000300L
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1_MASK                                                          0x00000C00L
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2_MASK                                                          0x00003000L
-#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3_MASK                                                          0x0000C000L
-#define DB_ALPHA_TO_MASK__OFFSET_ROUND_MASK                                                                   0x00010000L
-//VGT_DISPATCH_DRAW_INDEX
-#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX__SHIFT                                                           0x0
-#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX_MASK                                                             0xFFFFFFFFL
-//PA_SU_POLY_OFFSET_DB_FMT_CNTL
-#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS__SHIFT                                     0x0
-#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT__SHIFT                                     0x8
-#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS_MASK                                       0x000000FFL
-#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT_MASK                                       0x00000100L
-//PA_SU_POLY_OFFSET_CLAMP
-#define PA_SU_POLY_OFFSET_CLAMP__CLAMP__SHIFT                                                                 0x0
-#define PA_SU_POLY_OFFSET_CLAMP__CLAMP_MASK                                                                   0xFFFFFFFFL
-//PA_SU_POLY_OFFSET_FRONT_SCALE
-#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE__SHIFT                                                           0x0
-#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE_MASK                                                             0xFFFFFFFFL
-//PA_SU_POLY_OFFSET_FRONT_OFFSET
-#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET__SHIFT                                                         0x0
-#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
-//PA_SU_POLY_OFFSET_BACK_SCALE
-#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE__SHIFT                                                            0x0
-#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE_MASK                                                              0xFFFFFFFFL
-//PA_SU_POLY_OFFSET_BACK_OFFSET
-#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET__SHIFT                                                          0x0
-#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET_MASK                                                            0xFFFFFFFFL
-//VGT_GS_INSTANCE_CNT
-#define VGT_GS_INSTANCE_CNT__ENABLE__SHIFT                                                                    0x0
-#define VGT_GS_INSTANCE_CNT__CNT__SHIFT                                                                       0x2
-#define VGT_GS_INSTANCE_CNT__ENABLE_MASK                                                                      0x00000001L
-#define VGT_GS_INSTANCE_CNT__CNT_MASK                                                                         0x000001FCL
-//VGT_STRMOUT_CONFIG
-#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN__SHIFT                                                             0x0
-#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN__SHIFT                                                             0x1
-#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN__SHIFT                                                             0x2
-#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN__SHIFT                                                             0x3
-#define VGT_STRMOUT_CONFIG__RAST_STREAM__SHIFT                                                                0x4
-#define VGT_STRMOUT_CONFIG__EN_PRIMS_NEEDED_CNT__SHIFT                                                        0x7
-#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK__SHIFT                                                           0x8
-#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK__SHIFT                                                       0x1f
-#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN_MASK                                                               0x00000001L
-#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN_MASK                                                               0x00000002L
-#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN_MASK                                                               0x00000004L
-#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN_MASK                                                               0x00000008L
-#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK                                                                  0x00000070L
-#define VGT_STRMOUT_CONFIG__EN_PRIMS_NEEDED_CNT_MASK                                                          0x00000080L
-#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK_MASK                                                             0x00000F00L
-#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK_MASK                                                         0x80000000L
-//VGT_STRMOUT_BUFFER_CONFIG
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN__SHIFT                                                  0x0
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN__SHIFT                                                  0x4
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN__SHIFT                                                  0x8
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN__SHIFT                                                  0xc
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN_MASK                                                    0x0000000FL
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN_MASK                                                    0x000000F0L
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN_MASK                                                    0x00000F00L
-#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN_MASK                                                    0x0000F000L
-//VGT_DMA_EVENT_INITIATOR
-#define VGT_DMA_EVENT_INITIATOR__EVENT_TYPE__SHIFT                                                            0x0
-#define VGT_DMA_EVENT_INITIATOR__ADDRESS_HI__SHIFT                                                            0xa
-#define VGT_DMA_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT                                                        0x1b
-#define VGT_DMA_EVENT_INITIATOR__EVENT_TYPE_MASK                                                              0x0000003FL
-#define VGT_DMA_EVENT_INITIATOR__ADDRESS_HI_MASK                                                              0x07FFFC00L
-#define VGT_DMA_EVENT_INITIATOR__EXTENDED_EVENT_MASK                                                          0x08000000L
-//PA_SC_CENTROID_PRIORITY_0
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0__SHIFT                                                          0x0
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1__SHIFT                                                          0x4
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2__SHIFT                                                          0x8
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3__SHIFT                                                          0xc
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4__SHIFT                                                          0x10
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5__SHIFT                                                          0x14
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6__SHIFT                                                          0x18
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7__SHIFT                                                          0x1c
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0_MASK                                                            0x0000000FL
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1_MASK                                                            0x000000F0L
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2_MASK                                                            0x00000F00L
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3_MASK                                                            0x0000F000L
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4_MASK                                                            0x000F0000L
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5_MASK                                                            0x00F00000L
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6_MASK                                                            0x0F000000L
-#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7_MASK                                                            0xF0000000L
-//PA_SC_CENTROID_PRIORITY_1
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8__SHIFT                                                          0x0
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9__SHIFT                                                          0x4
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10__SHIFT                                                         0x8
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11__SHIFT                                                         0xc
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12__SHIFT                                                         0x10
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13__SHIFT                                                         0x14
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14__SHIFT                                                         0x18
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15__SHIFT                                                         0x1c
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8_MASK                                                            0x0000000FL
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9_MASK                                                            0x000000F0L
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10_MASK                                                           0x00000F00L
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11_MASK                                                           0x0000F000L
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12_MASK                                                           0x000F0000L
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13_MASK                                                           0x00F00000L
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14_MASK                                                           0x0F000000L
-#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15_MASK                                                           0xF0000000L
-//PA_SC_LINE_CNTL
-#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH__SHIFT                                                             0x9
-#define PA_SC_LINE_CNTL__LAST_PIXEL__SHIFT                                                                    0xa
-#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA__SHIFT                                                      0xb
-#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA__SHIFT                                                         0xc
-#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH_MASK                                                               0x00000200L
-#define PA_SC_LINE_CNTL__LAST_PIXEL_MASK                                                                      0x00000400L
-#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA_MASK                                                        0x00000800L
-#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA_MASK                                                           0x00001000L
-//PA_SC_AA_CONFIG
-#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES__SHIFT                                                              0x0
-#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN__SHIFT                                                         0x4
-#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST__SHIFT                                                               0xd
-#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES__SHIFT                                                          0x14
-#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE__SHIFT                                                        0x18
-#define PA_SC_AA_CONFIG__COVERAGE_TO_SHADER_SELECT__SHIFT                                                     0x1a
-#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES_MASK                                                                0x00000007L
-#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN_MASK                                                           0x00000010L
-#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST_MASK                                                                 0x0001E000L
-#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES_MASK                                                            0x00700000L
-#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE_MASK                                                          0x03000000L
-#define PA_SC_AA_CONFIG__COVERAGE_TO_SHADER_SELECT_MASK                                                       0x0C000000L
-//PA_SU_VTX_CNTL
-#define PA_SU_VTX_CNTL__PIX_CENTER__SHIFT                                                                     0x0
-#define PA_SU_VTX_CNTL__ROUND_MODE__SHIFT                                                                     0x1
-#define PA_SU_VTX_CNTL__QUANT_MODE__SHIFT                                                                     0x3
-#define PA_SU_VTX_CNTL__PIX_CENTER_MASK                                                                       0x00000001L
-#define PA_SU_VTX_CNTL__ROUND_MODE_MASK                                                                       0x00000006L
-#define PA_SU_VTX_CNTL__QUANT_MODE_MASK                                                                       0x00000038L
-//PA_CL_GB_VERT_CLIP_ADJ
-#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
-#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
-//PA_CL_GB_VERT_DISC_ADJ
-#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
-#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
-//PA_CL_GB_HORZ_CLIP_ADJ
-#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
-#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
-//PA_CL_GB_HORZ_DISC_ADJ
-#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
-#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X__SHIFT                                                       0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y__SHIFT                                                       0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X__SHIFT                                                       0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y__SHIFT                                                       0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X_MASK                                                         0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y_MASK                                                         0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X_MASK                                                         0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y_MASK                                                         0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X__SHIFT                                                       0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y__SHIFT                                                       0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X__SHIFT                                                       0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y__SHIFT                                                       0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X_MASK                                                         0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y_MASK                                                         0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X_MASK                                                         0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y_MASK                                                         0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X__SHIFT                                                       0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y__SHIFT                                                       0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X__SHIFT                                                       0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y__SHIFT                                                       0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X_MASK                                                         0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y_MASK                                                         0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X_MASK                                                         0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y_MASK                                                         0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X__SHIFT                                                        0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y__SHIFT                                                        0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X__SHIFT                                                        0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y__SHIFT                                                        0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X_MASK                                                          0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y_MASK                                                          0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X_MASK                                                          0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y_MASK                                                          0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X__SHIFT                                                        0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y__SHIFT                                                        0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X__SHIFT                                                        0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y__SHIFT                                                        0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X_MASK                                                          0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y_MASK                                                          0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X_MASK                                                          0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y_MASK                                                          0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X__SHIFT                                                       0x0
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y__SHIFT                                                       0x4
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X__SHIFT                                                       0x8
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y__SHIFT                                                       0xc
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X__SHIFT                                                       0x10
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y__SHIFT                                                       0x14
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X__SHIFT                                                       0x18
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y__SHIFT                                                       0x1c
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X_MASK                                                         0x0000000FL
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y_MASK                                                         0x000000F0L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X_MASK                                                         0x00000F00L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y_MASK                                                         0x0000F000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X_MASK                                                         0x000F0000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y_MASK                                                         0x00F00000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X_MASK                                                         0x0F000000L
-#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y_MASK                                                         0xF0000000L
-//PA_SC_AA_MASK_X0Y0_X1Y0
-#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0__SHIFT                                                          0x0
-#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0__SHIFT                                                          0x10
-#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0_MASK                                                            0x0000FFFFL
-#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0_MASK                                                            0xFFFF0000L
-//PA_SC_AA_MASK_X0Y1_X1Y1
-#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1__SHIFT                                                          0x0
-#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1__SHIFT                                                          0x10
-#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1_MASK                                                            0x0000FFFFL
-#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1_MASK                                                            0xFFFF0000L
-//PA_SC_SHADER_CONTROL
-#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES__SHIFT                                             0x0
-#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID__SHIFT                                                    0x2
-#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION__SHIFT                                                 0x3
-#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES_MASK                                               0x00000003L
-#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID_MASK                                                      0x00000004L
-#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION_MASK                                                   0x00000008L
-//PA_SC_BINNER_CNTL_0
-#define PA_SC_BINNER_CNTL_0__BINNING_MODE__SHIFT                                                              0x0
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X__SHIFT                                                                0x2
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y__SHIFT                                                                0x3
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND__SHIFT                                                         0x4
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND__SHIFT                                                         0x7
-#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN__SHIFT                                                    0xa
-#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN__SHIFT                                                 0xd
-#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM__SHIFT                                                     0x12
-#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH__SHIFT                                                           0x13
-#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION__SHIFT                                                     0x1b
-#define PA_SC_BINNER_CNTL_0__BINNING_MODE_MASK                                                                0x00000003L
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_MASK                                                                  0x00000004L
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_MASK                                                                  0x00000008L
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND_MASK                                                           0x00000070L
-#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND_MASK                                                           0x00000380L
-#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN_MASK                                                      0x00001C00L
-#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN_MASK                                                   0x0003E000L
-#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM_MASK                                                       0x00040000L
-#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH_MASK                                                             0x07F80000L
-#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION_MASK                                                       0x08000000L
-//PA_SC_BINNER_CNTL_1
-#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT__SHIFT                                                           0x0
-#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH__SHIFT                                                        0x10
-#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT_MASK                                                             0x0000FFFFL
-#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH_MASK                                                          0xFFFF0000L
-//PA_SC_CONSERVATIVE_RASTERIZATION_CNTL
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE__SHIFT                                        0x0
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT__SHIFT                                 0x1
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE__SHIFT                                       0x5
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT__SHIFT                                0x6
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE__SHIFT                           0xa
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT__SHIFT                                          0xb
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET__SHIFT                                          0xc
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL__SHIFT                      0xd
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL__SHIFT                     0xe
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE__SHIFT             0xf
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE__SHIFT                                 0x10
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x12
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x13
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE__SHIFT                               0x14
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE__SHIFT                                 0x15
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE__SHIFT                                     0x16
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE__SHIFT                                    0x17
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE__SHIFT                                0x18
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE_MASK                                          0x00000001L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT_MASK                                   0x0000001EL
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE_MASK                                         0x00000020L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT_MASK                                  0x000003C0L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE_MASK                             0x00000400L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT_MASK                                            0x00000800L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET_MASK                                            0x00001000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL_MASK                        0x00002000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL_MASK                       0x00004000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE_MASK               0x00008000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE_MASK                                   0x00030000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00040000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00080000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE_MASK                                 0x00100000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE_MASK                                   0x00200000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE_MASK                                       0x00400000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE_MASK                                      0x00800000L
-#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE_MASK                                  0x01000000L
-//PA_SC_NGG_MODE_CNTL
-#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE__SHIFT                                                      0x0
-#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE_MASK                                                        0x000007FFL
-//VGT_VERTEX_REUSE_BLOCK_CNTL
-#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH__SHIFT                                                   0x0
-#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH_MASK                                                     0x000000FFL
-//VGT_OUT_DEALLOC_CNTL
-#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST__SHIFT                                                             0x0
-#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST_MASK                                                               0x0000007FL
-//CB_COLOR0_BASE
-#define CB_COLOR0_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR0_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR0_BASE_EXT
-#define CB_COLOR0_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR0_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR0_ATTRIB2
-#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR0_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR0_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR0_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR0_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR0_VIEW
-#define CB_COLOR0_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR0_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR0_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR0_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR0_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR0_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR0_INFO
-#define CB_COLOR0_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR0_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR0_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR0_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR0_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR0_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR0_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR0_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR0_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR0_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR0_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR0_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR0_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR0_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR0_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR0_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR0_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR0_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR0_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR0_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR0_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR0_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR0_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR0_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR0_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR0_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR0_ATTRIB
-#define CB_COLOR0_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR0_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR0_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR0_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR0_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR0_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR0_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR0_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR0_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR0_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR0_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR0_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR0_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR0_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR0_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR0_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR0_DCC_CONTROL
-#define CB_COLOR0_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR0_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR0_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR0_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR0_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR0_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR0_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR0_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR0_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR0_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR0_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR0_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR0_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR0_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR0_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR0_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR0_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR0_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR0_CMASK
-#define CB_COLOR0_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR0_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR0_CMASK_BASE_EXT
-#define CB_COLOR0_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR0_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR0_FMASK
-#define CB_COLOR0_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR0_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR0_FMASK_BASE_EXT
-#define CB_COLOR0_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR0_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR0_CLEAR_WORD0
-#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR0_CLEAR_WORD1
-#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR0_DCC_BASE
-#define CB_COLOR0_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR0_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR0_DCC_BASE_EXT
-#define CB_COLOR0_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR0_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-//CB_COLOR1_BASE
-#define CB_COLOR1_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR1_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR1_BASE_EXT
-#define CB_COLOR1_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR1_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR1_ATTRIB2
-#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR1_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR1_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR1_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR1_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR1_VIEW
-#define CB_COLOR1_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR1_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR1_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR1_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR1_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR1_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR1_INFO
-#define CB_COLOR1_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR1_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR1_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR1_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR1_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR1_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR1_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR1_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR1_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR1_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR1_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR1_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR1_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR1_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR1_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR1_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR1_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR1_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR1_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR1_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR1_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR1_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR1_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR1_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR1_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR1_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR1_ATTRIB
-#define CB_COLOR1_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR1_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR1_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR1_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR1_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR1_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR1_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR1_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR1_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR1_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR1_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR1_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR1_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR1_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR1_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR1_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR1_DCC_CONTROL
-#define CB_COLOR1_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR1_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR1_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR1_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR1_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR1_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR1_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR1_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR1_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR1_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR1_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR1_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR1_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR1_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR1_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR1_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR1_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR1_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR1_CMASK
-#define CB_COLOR1_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR1_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR1_CMASK_BASE_EXT
-#define CB_COLOR1_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR1_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR1_FMASK
-#define CB_COLOR1_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR1_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR1_FMASK_BASE_EXT
-#define CB_COLOR1_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR1_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR1_CLEAR_WORD0
-#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR1_CLEAR_WORD1
-#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR1_DCC_BASE
-#define CB_COLOR1_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR1_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR1_DCC_BASE_EXT
-#define CB_COLOR1_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR1_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-//CB_COLOR2_BASE
-#define CB_COLOR2_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR2_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR2_BASE_EXT
-#define CB_COLOR2_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR2_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR2_ATTRIB2
-#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR2_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR2_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR2_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR2_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR2_VIEW
-#define CB_COLOR2_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR2_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR2_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR2_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR2_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR2_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR2_INFO
-#define CB_COLOR2_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR2_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR2_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR2_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR2_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR2_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR2_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR2_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR2_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR2_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR2_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR2_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR2_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR2_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR2_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR2_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR2_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR2_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR2_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR2_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR2_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR2_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR2_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR2_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR2_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR2_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR2_ATTRIB
-#define CB_COLOR2_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR2_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR2_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR2_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR2_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR2_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR2_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR2_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR2_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR2_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR2_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR2_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR2_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR2_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR2_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR2_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR2_DCC_CONTROL
-#define CB_COLOR2_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR2_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR2_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR2_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR2_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR2_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR2_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR2_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR2_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR2_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR2_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR2_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR2_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR2_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR2_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR2_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR2_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR2_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR2_CMASK
-#define CB_COLOR2_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR2_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR2_CMASK_BASE_EXT
-#define CB_COLOR2_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR2_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR2_FMASK
-#define CB_COLOR2_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR2_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR2_FMASK_BASE_EXT
-#define CB_COLOR2_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR2_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR2_CLEAR_WORD0
-#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR2_CLEAR_WORD1
-#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR2_DCC_BASE
-#define CB_COLOR2_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR2_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR2_DCC_BASE_EXT
-#define CB_COLOR2_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR2_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-//CB_COLOR3_BASE
-#define CB_COLOR3_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR3_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR3_BASE_EXT
-#define CB_COLOR3_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR3_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR3_ATTRIB2
-#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR3_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR3_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR3_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR3_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR3_VIEW
-#define CB_COLOR3_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR3_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR3_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR3_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR3_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR3_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR3_INFO
-#define CB_COLOR3_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR3_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR3_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR3_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR3_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR3_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR3_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR3_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR3_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR3_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR3_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR3_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR3_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR3_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR3_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR3_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR3_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR3_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR3_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR3_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR3_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR3_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR3_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR3_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR3_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR3_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR3_ATTRIB
-#define CB_COLOR3_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR3_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR3_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR3_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR3_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR3_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR3_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR3_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR3_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR3_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR3_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR3_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR3_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR3_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR3_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR3_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR3_DCC_CONTROL
-#define CB_COLOR3_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR3_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR3_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR3_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR3_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR3_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR3_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR3_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR3_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR3_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR3_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR3_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR3_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR3_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR3_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR3_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR3_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR3_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR3_CMASK
-#define CB_COLOR3_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR3_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR3_CMASK_BASE_EXT
-#define CB_COLOR3_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR3_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR3_FMASK
-#define CB_COLOR3_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR3_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR3_FMASK_BASE_EXT
-#define CB_COLOR3_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR3_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR3_CLEAR_WORD0
-#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR3_CLEAR_WORD1
-#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR3_DCC_BASE
-#define CB_COLOR3_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR3_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR3_DCC_BASE_EXT
-#define CB_COLOR3_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR3_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-//CB_COLOR4_BASE
-#define CB_COLOR4_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR4_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR4_BASE_EXT
-#define CB_COLOR4_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR4_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR4_ATTRIB2
-#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR4_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR4_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR4_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR4_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR4_VIEW
-#define CB_COLOR4_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR4_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR4_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR4_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR4_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR4_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR4_INFO
-#define CB_COLOR4_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR4_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR4_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR4_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR4_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR4_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR4_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR4_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR4_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR4_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR4_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR4_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR4_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR4_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR4_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR4_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR4_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR4_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR4_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR4_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR4_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR4_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR4_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR4_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR4_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR4_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR4_ATTRIB
-#define CB_COLOR4_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR4_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR4_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR4_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR4_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR4_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR4_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR4_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR4_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR4_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR4_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR4_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR4_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR4_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR4_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR4_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR4_DCC_CONTROL
-#define CB_COLOR4_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR4_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR4_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR4_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR4_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR4_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR4_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR4_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR4_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR4_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR4_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR4_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR4_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR4_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR4_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR4_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR4_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR4_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR4_CMASK
-#define CB_COLOR4_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR4_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR4_CMASK_BASE_EXT
-#define CB_COLOR4_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR4_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR4_FMASK
-#define CB_COLOR4_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR4_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR4_FMASK_BASE_EXT
-#define CB_COLOR4_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR4_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR4_CLEAR_WORD0
-#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR4_CLEAR_WORD1
-#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR4_DCC_BASE
-#define CB_COLOR4_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR4_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR4_DCC_BASE_EXT
-#define CB_COLOR4_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR4_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-//CB_COLOR5_BASE
-#define CB_COLOR5_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR5_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR5_BASE_EXT
-#define CB_COLOR5_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR5_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR5_ATTRIB2
-#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR5_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR5_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR5_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR5_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR5_VIEW
-#define CB_COLOR5_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR5_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR5_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR5_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR5_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR5_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR5_INFO
-#define CB_COLOR5_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR5_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR5_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR5_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR5_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR5_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR5_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR5_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR5_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR5_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR5_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR5_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR5_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR5_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR5_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR5_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR5_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR5_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR5_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR5_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR5_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR5_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR5_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR5_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR5_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR5_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR5_ATTRIB
-#define CB_COLOR5_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR5_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR5_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR5_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR5_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR5_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR5_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR5_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR5_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR5_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR5_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR5_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR5_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR5_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR5_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR5_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR5_DCC_CONTROL
-#define CB_COLOR5_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR5_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR5_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR5_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR5_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR5_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR5_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR5_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR5_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR5_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR5_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR5_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR5_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR5_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR5_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR5_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR5_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR5_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR5_CMASK
-#define CB_COLOR5_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR5_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR5_CMASK_BASE_EXT
-#define CB_COLOR5_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR5_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR5_FMASK
-#define CB_COLOR5_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR5_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR5_FMASK_BASE_EXT
-#define CB_COLOR5_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR5_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR5_CLEAR_WORD0
-#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR5_CLEAR_WORD1
-#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR5_DCC_BASE
-#define CB_COLOR5_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR5_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR5_DCC_BASE_EXT
-#define CB_COLOR5_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR5_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-//CB_COLOR6_BASE
-#define CB_COLOR6_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR6_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR6_BASE_EXT
-#define CB_COLOR6_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR6_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR6_ATTRIB2
-#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR6_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR6_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR6_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR6_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR6_VIEW
-#define CB_COLOR6_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR6_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR6_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR6_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR6_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR6_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR6_INFO
-#define CB_COLOR6_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR6_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR6_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR6_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR6_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR6_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR6_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR6_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR6_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR6_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR6_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR6_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR6_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR6_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR6_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR6_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR6_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR6_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR6_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR6_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR6_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR6_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR6_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR6_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR6_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR6_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR6_ATTRIB
-#define CB_COLOR6_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR6_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR6_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR6_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR6_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR6_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR6_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR6_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR6_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR6_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR6_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR6_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR6_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR6_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR6_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR6_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR6_DCC_CONTROL
-#define CB_COLOR6_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR6_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR6_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR6_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR6_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR6_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR6_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR6_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR6_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR6_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR6_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR6_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR6_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR6_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR6_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR6_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR6_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR6_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR6_CMASK
-#define CB_COLOR6_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR6_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR6_CMASK_BASE_EXT
-#define CB_COLOR6_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR6_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR6_FMASK
-#define CB_COLOR6_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR6_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR6_FMASK_BASE_EXT
-#define CB_COLOR6_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR6_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR6_CLEAR_WORD0
-#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR6_CLEAR_WORD1
-#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR6_DCC_BASE
-#define CB_COLOR6_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR6_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR6_DCC_BASE_EXT
-#define CB_COLOR6_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR6_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-//CB_COLOR7_BASE
-#define CB_COLOR7_BASE__BASE_256B__SHIFT                                                                      0x0
-#define CB_COLOR7_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
-//CB_COLOR7_BASE_EXT
-#define CB_COLOR7_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR7_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
-//CB_COLOR7_ATTRIB2
-#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
-#define CB_COLOR7_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
-#define CB_COLOR7_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
-#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
-#define CB_COLOR7_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
-#define CB_COLOR7_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
-//CB_COLOR7_VIEW
-#define CB_COLOR7_VIEW__SLICE_START__SHIFT                                                                    0x0
-#define CB_COLOR7_VIEW__SLICE_MAX__SHIFT                                                                      0xd
-#define CB_COLOR7_VIEW__MIP_LEVEL__SHIFT                                                                      0x18
-#define CB_COLOR7_VIEW__SLICE_START_MASK                                                                      0x000007FFL
-#define CB_COLOR7_VIEW__SLICE_MAX_MASK                                                                        0x00FFE000L
-#define CB_COLOR7_VIEW__MIP_LEVEL_MASK                                                                        0x0F000000L
-//CB_COLOR7_INFO
-#define CB_COLOR7_INFO__ENDIAN__SHIFT                                                                         0x0
-#define CB_COLOR7_INFO__FORMAT__SHIFT                                                                         0x2
-#define CB_COLOR7_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
-#define CB_COLOR7_INFO__COMP_SWAP__SHIFT                                                                      0xb
-#define CB_COLOR7_INFO__FAST_CLEAR__SHIFT                                                                     0xd
-#define CB_COLOR7_INFO__COMPRESSION__SHIFT                                                                    0xe
-#define CB_COLOR7_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
-#define CB_COLOR7_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
-#define CB_COLOR7_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
-#define CB_COLOR7_INFO__ROUND_MODE__SHIFT                                                                     0x12
-#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
-#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
-#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE__SHIFT                                                      0x1a
-#define CB_COLOR7_INFO__FMASK_COMPRESS_1FRAG_ONLY__SHIFT                                                      0x1b
-#define CB_COLOR7_INFO__DCC_ENABLE__SHIFT                                                                     0x1c
-#define CB_COLOR7_INFO__CMASK_ADDR_TYPE__SHIFT                                                                0x1d
-#define CB_COLOR7_INFO__ENDIAN_MASK                                                                           0x00000003L
-#define CB_COLOR7_INFO__FORMAT_MASK                                                                           0x0000007CL
-#define CB_COLOR7_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
-#define CB_COLOR7_INFO__COMP_SWAP_MASK                                                                        0x00001800L
-#define CB_COLOR7_INFO__FAST_CLEAR_MASK                                                                       0x00002000L
-#define CB_COLOR7_INFO__COMPRESSION_MASK                                                                      0x00004000L
-#define CB_COLOR7_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
-#define CB_COLOR7_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
-#define CB_COLOR7_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
-#define CB_COLOR7_INFO__ROUND_MODE_MASK                                                                       0x00040000L
-#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
-#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
-#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE_MASK                                                        0x04000000L
-#define CB_COLOR7_INFO__FMASK_COMPRESS_1FRAG_ONLY_MASK                                                        0x08000000L
-#define CB_COLOR7_INFO__DCC_ENABLE_MASK                                                                       0x10000000L
-#define CB_COLOR7_INFO__CMASK_ADDR_TYPE_MASK                                                                  0x60000000L
-//CB_COLOR7_ATTRIB
-#define CB_COLOR7_ATTRIB__MIP0_DEPTH__SHIFT                                                                   0x0
-#define CB_COLOR7_ATTRIB__META_LINEAR__SHIFT                                                                  0xb
-#define CB_COLOR7_ATTRIB__NUM_SAMPLES__SHIFT                                                                  0xc
-#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0xf
-#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x11
-#define CB_COLOR7_ATTRIB__COLOR_SW_MODE__SHIFT                                                                0x12
-#define CB_COLOR7_ATTRIB__FMASK_SW_MODE__SHIFT                                                                0x17
-#define CB_COLOR7_ATTRIB__RESOURCE_TYPE__SHIFT                                                                0x1c
-#define CB_COLOR7_ATTRIB__RB_ALIGNED__SHIFT                                                                   0x1e
-#define CB_COLOR7_ATTRIB__PIPE_ALIGNED__SHIFT                                                                 0x1f
-#define CB_COLOR7_ATTRIB__MIP0_DEPTH_MASK                                                                     0x000007FFL
-#define CB_COLOR7_ATTRIB__META_LINEAR_MASK                                                                    0x00000800L
-#define CB_COLOR7_ATTRIB__NUM_SAMPLES_MASK                                                                    0x00007000L
-#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00018000L
-#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00020000L
-#define CB_COLOR7_ATTRIB__COLOR_SW_MODE_MASK                                                                  0x007C0000L
-#define CB_COLOR7_ATTRIB__FMASK_SW_MODE_MASK                                                                  0x0F800000L
-#define CB_COLOR7_ATTRIB__RESOURCE_TYPE_MASK                                                                  0x30000000L
-#define CB_COLOR7_ATTRIB__RB_ALIGNED_MASK                                                                     0x40000000L
-#define CB_COLOR7_ATTRIB__PIPE_ALIGNED_MASK                                                                   0x80000000L
-//CB_COLOR7_DCC_CONTROL
-#define CB_COLOR7_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE__SHIFT                                              0x0
-#define CB_COLOR7_DCC_CONTROL__KEY_CLEAR_ENABLE__SHIFT                                                        0x1
-#define CB_COLOR7_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                             0x2
-#define CB_COLOR7_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x4
-#define CB_COLOR7_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                               0x5
-#define CB_COLOR7_DCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                         0x7
-#define CB_COLOR7_DCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                  0x9
-#define CB_COLOR7_DCC_CONTROL__LOSSY_RGB_PRECISION__SHIFT                                                     0xa
-#define CB_COLOR7_DCC_CONTROL__LOSSY_ALPHA_PRECISION__SHIFT                                                   0xe
-#define CB_COLOR7_DCC_CONTROL__OVERWRITE_COMBINER_DISABLE_MASK                                                0x00000001L
-#define CB_COLOR7_DCC_CONTROL__KEY_CLEAR_ENABLE_MASK                                                          0x00000002L
-#define CB_COLOR7_DCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                               0x0000000CL
-#define CB_COLOR7_DCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000010L
-#define CB_COLOR7_DCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                 0x00000060L
-#define CB_COLOR7_DCC_CONTROL__COLOR_TRANSFORM_MASK                                                           0x00000180L
-#define CB_COLOR7_DCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                    0x00000200L
-#define CB_COLOR7_DCC_CONTROL__LOSSY_RGB_PRECISION_MASK                                                       0x00003C00L
-#define CB_COLOR7_DCC_CONTROL__LOSSY_ALPHA_PRECISION_MASK                                                     0x0003C000L
-//CB_COLOR7_CMASK
-#define CB_COLOR7_CMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR7_CMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR7_CMASK_BASE_EXT
-#define CB_COLOR7_CMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR7_CMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR7_FMASK
-#define CB_COLOR7_FMASK__BASE_256B__SHIFT                                                                     0x0
-#define CB_COLOR7_FMASK__BASE_256B_MASK                                                                       0xFFFFFFFFL
-//CB_COLOR7_FMASK_BASE_EXT
-#define CB_COLOR7_FMASK_BASE_EXT__BASE_256B__SHIFT                                                            0x0
-#define CB_COLOR7_FMASK_BASE_EXT__BASE_256B_MASK                                                              0x000000FFL
-//CB_COLOR7_CLEAR_WORD0
-#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0__SHIFT                                                             0x0
-#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0_MASK                                                               0xFFFFFFFFL
-//CB_COLOR7_CLEAR_WORD1
-#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1__SHIFT                                                             0x0
-#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1_MASK                                                               0xFFFFFFFFL
-//CB_COLOR7_DCC_BASE
-#define CB_COLOR7_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
-#define CB_COLOR7_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
-//CB_COLOR7_DCC_BASE_EXT
-#define CB_COLOR7_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
-#define CB_COLOR7_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
-
-
-// addressBlock: gc_gfxudec
-//CP_EOP_DONE_ADDR_LO
-#define CP_EOP_DONE_ADDR_LO__ADDR_LO__SHIFT                                                                   0x2
-#define CP_EOP_DONE_ADDR_LO__ADDR_LO_MASK                                                                     0xFFFFFFFCL
-//CP_EOP_DONE_ADDR_HI
-#define CP_EOP_DONE_ADDR_HI__ADDR_HI__SHIFT                                                                   0x0
-#define CP_EOP_DONE_ADDR_HI__ADDR_HI_MASK                                                                     0x0000FFFFL
-//CP_EOP_DONE_DATA_LO
-#define CP_EOP_DONE_DATA_LO__DATA_LO__SHIFT                                                                   0x0
-#define CP_EOP_DONE_DATA_LO__DATA_LO_MASK                                                                     0xFFFFFFFFL
-//CP_EOP_DONE_DATA_HI
-#define CP_EOP_DONE_DATA_HI__DATA_HI__SHIFT                                                                   0x0
-#define CP_EOP_DONE_DATA_HI__DATA_HI_MASK                                                                     0xFFFFFFFFL
-//CP_EOP_LAST_FENCE_LO
-#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO__SHIFT                                                            0x0
-#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO_MASK                                                              0xFFFFFFFFL
-//CP_EOP_LAST_FENCE_HI
-#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI__SHIFT                                                            0x0
-#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI_MASK                                                              0xFFFFFFFFL
-//CP_STREAM_OUT_ADDR_LO
-#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO__SHIFT                                                      0x2
-#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO_MASK                                                        0xFFFFFFFCL
-//CP_STREAM_OUT_ADDR_HI
-#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI__SHIFT                                                      0x0
-#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI_MASK                                                        0x0000FFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT0_LO
-#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT0_HI
-#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT0_LO
-#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO_MASK                                            0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT0_HI
-#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI_MASK                                            0xFFFFFFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT1_LO
-#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT1_HI
-#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT1_LO
-#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO_MASK                                            0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT1_HI
-#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI_MASK                                            0xFFFFFFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT2_LO
-#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT2_HI
-#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT2_LO
-#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO_MASK                                            0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT2_HI
-#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI_MASK                                            0xFFFFFFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT3_LO
-#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_WRITTEN_COUNT3_HI
-#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI__SHIFT                                        0x0
-#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI_MASK                                          0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT3_LO
-#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO_MASK                                            0xFFFFFFFFL
-//CP_NUM_PRIM_NEEDED_COUNT3_HI
-#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI__SHIFT                                          0x0
-#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI_MASK                                            0xFFFFFFFFL
-//CP_PIPE_STATS_ADDR_LO
-#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO__SHIFT                                                      0x2
-#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO_MASK                                                        0xFFFFFFFCL
-//CP_PIPE_STATS_ADDR_HI
-#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI__SHIFT                                                      0x0
-#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI_MASK                                                        0x0000FFFFL
-//CP_VGT_IAVERT_COUNT_LO
-#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO__SHIFT                                                        0x0
-#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO_MASK                                                          0xFFFFFFFFL
-//CP_VGT_IAVERT_COUNT_HI
-#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI__SHIFT                                                        0x0
-#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI_MASK                                                          0xFFFFFFFFL
-//CP_VGT_IAPRIM_COUNT_LO
-#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO__SHIFT                                                        0x0
-#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
-//CP_VGT_IAPRIM_COUNT_HI
-#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI__SHIFT                                                        0x0
-#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
-//CP_VGT_GSPRIM_COUNT_LO
-#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO__SHIFT                                                        0x0
-#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
-//CP_VGT_GSPRIM_COUNT_HI
-#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI__SHIFT                                                        0x0
-#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
-//CP_VGT_VSINVOC_COUNT_LO
-#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO__SHIFT                                                      0x0
-#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
-//CP_VGT_VSINVOC_COUNT_HI
-#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI__SHIFT                                                      0x0
-#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
-//CP_VGT_GSINVOC_COUNT_LO
-#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO__SHIFT                                                      0x0
-#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
-//CP_VGT_GSINVOC_COUNT_HI
-#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI__SHIFT                                                      0x0
-#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
-//CP_VGT_HSINVOC_COUNT_LO
-#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO__SHIFT                                                      0x0
-#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
-//CP_VGT_HSINVOC_COUNT_HI
-#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI__SHIFT                                                      0x0
-#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
-//CP_VGT_DSINVOC_COUNT_LO
-#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO__SHIFT                                                      0x0
-#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
-//CP_VGT_DSINVOC_COUNT_HI
-#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI__SHIFT                                                      0x0
-#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
-//CP_PA_CINVOC_COUNT_LO
-#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO__SHIFT                                                         0x0
-#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO_MASK                                                           0xFFFFFFFFL
-//CP_PA_CINVOC_COUNT_HI
-#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI__SHIFT                                                         0x0
-#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI_MASK                                                           0xFFFFFFFFL
-//CP_PA_CPRIM_COUNT_LO
-#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO__SHIFT                                                           0x0
-#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO_MASK                                                             0xFFFFFFFFL
-//CP_PA_CPRIM_COUNT_HI
-#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI__SHIFT                                                           0x0
-#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI_MASK                                                             0xFFFFFFFFL
-//CP_SC_PSINVOC_COUNT0_LO
-#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO__SHIFT                                                     0x0
-#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO_MASK                                                       0xFFFFFFFFL
-//CP_SC_PSINVOC_COUNT0_HI
-#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI__SHIFT                                                     0x0
-#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI_MASK                                                       0xFFFFFFFFL
-//CP_SC_PSINVOC_COUNT1_LO
-#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE__SHIFT                                                              0x0
-#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE_MASK                                                                0xFFFFFFFFL
-//CP_SC_PSINVOC_COUNT1_HI
-#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE__SHIFT                                                              0x0
-#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE_MASK                                                                0xFFFFFFFFL
-//CP_VGT_CSINVOC_COUNT_LO
-#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO__SHIFT                                                      0x0
-#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
-//CP_VGT_CSINVOC_COUNT_HI
-#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI__SHIFT                                                      0x0
-#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
-//CP_PIPE_STATS_CONTROL
-#define CP_PIPE_STATS_CONTROL__CACHE_POLICY__SHIFT                                                            0x19
-#define CP_PIPE_STATS_CONTROL__CACHE_POLICY_MASK                                                              0x02000000L
-//CP_STREAM_OUT_CONTROL
-#define CP_STREAM_OUT_CONTROL__CACHE_POLICY__SHIFT                                                            0x19
-#define CP_STREAM_OUT_CONTROL__CACHE_POLICY_MASK                                                              0x02000000L
-//CP_STRMOUT_CNTL
-#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE__SHIFT                                                            0x0
-#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE_MASK                                                              0x00000001L
-//SCRATCH_REG0
-#define SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                     0x0
-#define SCRATCH_REG0__SCRATCH_REG0_MASK                                                                       0xFFFFFFFFL
-//SCRATCH_REG1
-#define SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                     0x0
-#define SCRATCH_REG1__SCRATCH_REG1_MASK                                                                       0xFFFFFFFFL
-//SCRATCH_REG2
-#define SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                     0x0
-#define SCRATCH_REG2__SCRATCH_REG2_MASK                                                                       0xFFFFFFFFL
-//SCRATCH_REG3
-#define SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                     0x0
-#define SCRATCH_REG3__SCRATCH_REG3_MASK                                                                       0xFFFFFFFFL
-//SCRATCH_REG4
-#define SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                     0x0
-#define SCRATCH_REG4__SCRATCH_REG4_MASK                                                                       0xFFFFFFFFL
-//SCRATCH_REG5
-#define SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                     0x0
-#define SCRATCH_REG5__SCRATCH_REG5_MASK                                                                       0xFFFFFFFFL
-//SCRATCH_REG6
-#define SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                     0x0
-#define SCRATCH_REG6__SCRATCH_REG6_MASK                                                                       0xFFFFFFFFL
-//SCRATCH_REG7
-#define SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                     0x0
-#define SCRATCH_REG7__SCRATCH_REG7_MASK                                                                       0xFFFFFFFFL
-//CP_APPEND_DATA_HI
-#define CP_APPEND_DATA_HI__DATA__SHIFT                                                                        0x0
-#define CP_APPEND_DATA_HI__DATA_MASK                                                                          0xFFFFFFFFL
-//CP_APPEND_LAST_CS_FENCE_HI
-#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
-#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
-//CP_APPEND_LAST_PS_FENCE_HI
-#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
-#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
-//SCRATCH_UMSK
-#define SCRATCH_UMSK__OBSOLETE_UMSK__SHIFT                                                                    0x0
-#define SCRATCH_UMSK__OBSOLETE_SWAP__SHIFT                                                                    0x10
-#define SCRATCH_UMSK__OBSOLETE_UMSK_MASK                                                                      0x000000FFL
-#define SCRATCH_UMSK__OBSOLETE_SWAP_MASK                                                                      0x00030000L
-//SCRATCH_ADDR
-#define SCRATCH_ADDR__OBSOLETE_ADDR__SHIFT                                                                    0x0
-#define SCRATCH_ADDR__OBSOLETE_ADDR_MASK                                                                      0xFFFFFFFFL
-//CP_PFP_ATOMIC_PREOP_LO
-#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                        0x0
-#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                          0xFFFFFFFFL
-//CP_PFP_ATOMIC_PREOP_HI
-#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                        0x0
-#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                          0xFFFFFFFFL
-//CP_PFP_GDS_ATOMIC0_PREOP_LO
-#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                              0x0
-#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                0xFFFFFFFFL
-//CP_PFP_GDS_ATOMIC0_PREOP_HI
-#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                              0x0
-#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                0xFFFFFFFFL
-//CP_PFP_GDS_ATOMIC1_PREOP_LO
-#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                              0x0
-#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                0xFFFFFFFFL
-//CP_PFP_GDS_ATOMIC1_PREOP_HI
-#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                              0x0
-#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                0xFFFFFFFFL
-//CP_APPEND_ADDR_LO
-#define CP_APPEND_ADDR_LO__MEM_ADDR_LO__SHIFT                                                                 0x2
-#define CP_APPEND_ADDR_LO__MEM_ADDR_LO_MASK                                                                   0xFFFFFFFCL
-//CP_APPEND_ADDR_HI
-#define CP_APPEND_ADDR_HI__MEM_ADDR_HI__SHIFT                                                                 0x0
-#define CP_APPEND_ADDR_HI__CS_PS_SEL__SHIFT                                                                   0x10
-#define CP_APPEND_ADDR_HI__CACHE_POLICY__SHIFT                                                                0x19
-#define CP_APPEND_ADDR_HI__COMMAND__SHIFT                                                                     0x1d
-#define CP_APPEND_ADDR_HI__MEM_ADDR_HI_MASK                                                                   0x0000FFFFL
-#define CP_APPEND_ADDR_HI__CS_PS_SEL_MASK                                                                     0x00010000L
-#define CP_APPEND_ADDR_HI__CACHE_POLICY_MASK                                                                  0x02000000L
-#define CP_APPEND_ADDR_HI__COMMAND_MASK                                                                       0xE0000000L
-//CP_APPEND_DATA_LO
-#define CP_APPEND_DATA_LO__DATA__SHIFT                                                                        0x0
-#define CP_APPEND_DATA_LO__DATA_MASK                                                                          0xFFFFFFFFL
-//CP_APPEND_LAST_CS_FENCE_LO
-#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
-#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
-//CP_APPEND_LAST_PS_FENCE_LO
-#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
-#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
-//CP_ATOMIC_PREOP_LO
-#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                            0x0
-#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                              0xFFFFFFFFL
-//CP_ME_ATOMIC_PREOP_LO
-#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                         0x0
-#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                           0xFFFFFFFFL
-//CP_ATOMIC_PREOP_HI
-#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                            0x0
-#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                              0xFFFFFFFFL
-//CP_ME_ATOMIC_PREOP_HI
-#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                         0x0
-#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                           0xFFFFFFFFL
-//CP_GDS_ATOMIC0_PREOP_LO
-#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                                  0x0
-#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                    0xFFFFFFFFL
-//CP_ME_GDS_ATOMIC0_PREOP_LO
-#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                               0x0
-#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                 0xFFFFFFFFL
-//CP_GDS_ATOMIC0_PREOP_HI
-#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                                  0x0
-#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                    0xFFFFFFFFL
-//CP_ME_GDS_ATOMIC0_PREOP_HI
-#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                               0x0
-#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                 0xFFFFFFFFL
-//CP_GDS_ATOMIC1_PREOP_LO
-#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                                  0x0
-#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                    0xFFFFFFFFL
-//CP_ME_GDS_ATOMIC1_PREOP_LO
-#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                               0x0
-#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                 0xFFFFFFFFL
-//CP_GDS_ATOMIC1_PREOP_HI
-#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                                  0x0
-#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                    0xFFFFFFFFL
-//CP_ME_GDS_ATOMIC1_PREOP_HI
-#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                               0x0
-#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                 0xFFFFFFFFL
-//CP_ME_MC_WADDR_LO
-#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO__SHIFT                                                              0x2
-#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO_MASK                                                                0xFFFFFFFCL
-//CP_ME_MC_WADDR_HI
-#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI__SHIFT                                                              0x0
-#define CP_ME_MC_WADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
-#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI_MASK                                                                0x0000FFFFL
-#define CP_ME_MC_WADDR_HI__CACHE_POLICY_MASK                                                                  0x00400000L
-//CP_ME_MC_WDATA_LO
-#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO__SHIFT                                                              0x0
-#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO_MASK                                                                0xFFFFFFFFL
-//CP_ME_MC_WDATA_HI
-#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI__SHIFT                                                              0x0
-#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI_MASK                                                                0xFFFFFFFFL
-//CP_ME_MC_RADDR_LO
-#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO__SHIFT                                                              0x2
-#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO_MASK                                                                0xFFFFFFFCL
-//CP_ME_MC_RADDR_HI
-#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI__SHIFT                                                              0x0
-#define CP_ME_MC_RADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
-#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI_MASK                                                                0x0000FFFFL
-#define CP_ME_MC_RADDR_HI__CACHE_POLICY_MASK                                                                  0x00400000L
-//CP_SEM_WAIT_TIMER
-#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER__SHIFT                                                              0x0
-#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER_MASK                                                                0xFFFFFFFFL
-//CP_SIG_SEM_ADDR_LO
-#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT                                                              0x0
-#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT                                                                0x3
-#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK                                                                0x00000003L
-#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO_MASK                                                                  0xFFFFFFF8L
-//CP_SIG_SEM_ADDR_HI
-#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT                                                                0x0
-#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT                                                            0x10
-#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT                                                            0x14
-#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT                                                            0x18
-#define CP_SIG_SEM_ADDR_HI__SEM_SELECT__SHIFT                                                                 0x1d
-#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI_MASK                                                                  0x0000FFFFL
-#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK                                                              0x00010000L
-#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK                                                              0x00100000L
-#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK                                                              0x03000000L
-#define CP_SIG_SEM_ADDR_HI__SEM_SELECT_MASK                                                                   0xE0000000L
-//CP_WAIT_REG_MEM_TIMEOUT
-#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT__SHIFT                                                  0x0
-#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT_MASK                                                    0xFFFFFFFFL
-//CP_WAIT_SEM_ADDR_LO
-#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT                                                             0x0
-#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT                                                               0x3
-#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK                                                               0x00000003L
-#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO_MASK                                                                 0xFFFFFFF8L
-//CP_WAIT_SEM_ADDR_HI
-#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT                                                               0x0
-#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT                                                           0x10
-#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT                                                           0x14
-#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT                                                           0x18
-#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT__SHIFT                                                                0x1d
-#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI_MASK                                                                 0x0000FFFFL
-#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK                                                             0x00010000L
-#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK                                                             0x00100000L
-#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK                                                             0x03000000L
-#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT_MASK                                                                  0xE0000000L
-//CP_DMA_PFP_CONTROL
-#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR__SHIFT                                                               0xa
-#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY__SHIFT                                                           0xd
-#define CP_DMA_PFP_CONTROL__DST_SELECT__SHIFT                                                                 0x14
-#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY__SHIFT                                                           0x19
-#define CP_DMA_PFP_CONTROL__SRC_SELECT__SHIFT                                                                 0x1d
-#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR_MASK                                                                 0x00000400L
-#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY_MASK                                                             0x00002000L
-#define CP_DMA_PFP_CONTROL__DST_SELECT_MASK                                                                   0x00300000L
-#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY_MASK                                                             0x02000000L
-#define CP_DMA_PFP_CONTROL__SRC_SELECT_MASK                                                                   0x60000000L
-//CP_DMA_ME_CONTROL
-#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR__SHIFT                                                                0xa
-#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY__SHIFT                                                            0xd
-#define CP_DMA_ME_CONTROL__DST_SELECT__SHIFT                                                                  0x14
-#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY__SHIFT                                                            0x19
-#define CP_DMA_ME_CONTROL__SRC_SELECT__SHIFT                                                                  0x1d
-#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR_MASK                                                                  0x00000400L
-#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY_MASK                                                              0x00002000L
-#define CP_DMA_ME_CONTROL__DST_SELECT_MASK                                                                    0x00300000L
-#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY_MASK                                                              0x02000000L
-#define CP_DMA_ME_CONTROL__SRC_SELECT_MASK                                                                    0x60000000L
-//CP_COHER_BASE_HI
-#define CP_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT                                                           0x0
-#define CP_COHER_BASE_HI__COHER_BASE_HI_256B_MASK                                                             0x000000FFL
-//CP_COHER_START_DELAY
-#define CP_COHER_START_DELAY__START_DELAY_COUNT__SHIFT                                                        0x0
-#define CP_COHER_START_DELAY__START_DELAY_COUNT_MASK                                                          0x0000003FL
-//CP_COHER_CNTL
-#define CP_COHER_CNTL__TC_NC_ACTION_ENA__SHIFT                                                                0x3
-#define CP_COHER_CNTL__TC_WC_ACTION_ENA__SHIFT                                                                0x4
-#define CP_COHER_CNTL__TC_INV_METADATA_ACTION_ENA__SHIFT                                                      0x5
-#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA__SHIFT                                                             0xf
-#define CP_COHER_CNTL__TC_WB_ACTION_ENA__SHIFT                                                                0x12
-#define CP_COHER_CNTL__TCL1_ACTION_ENA__SHIFT                                                                 0x16
-#define CP_COHER_CNTL__TC_ACTION_ENA__SHIFT                                                                   0x17
-#define CP_COHER_CNTL__CB_ACTION_ENA__SHIFT                                                                   0x19
-#define CP_COHER_CNTL__DB_ACTION_ENA__SHIFT                                                                   0x1a
-#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA__SHIFT                                                            0x1b
-#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA__SHIFT                                                        0x1c
-#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA__SHIFT                                                            0x1d
-#define CP_COHER_CNTL__SH_KCACHE_WB_ACTION_ENA__SHIFT                                                         0x1e
-#define CP_COHER_CNTL__TC_NC_ACTION_ENA_MASK                                                                  0x00000008L
-#define CP_COHER_CNTL__TC_WC_ACTION_ENA_MASK                                                                  0x00000010L
-#define CP_COHER_CNTL__TC_INV_METADATA_ACTION_ENA_MASK                                                        0x00000020L
-#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA_MASK                                                               0x00008000L
-#define CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK                                                                  0x00040000L
-#define CP_COHER_CNTL__TCL1_ACTION_ENA_MASK                                                                   0x00400000L
-#define CP_COHER_CNTL__TC_ACTION_ENA_MASK                                                                     0x00800000L
-#define CP_COHER_CNTL__CB_ACTION_ENA_MASK                                                                     0x02000000L
-#define CP_COHER_CNTL__DB_ACTION_ENA_MASK                                                                     0x04000000L
-#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK                                                              0x08000000L
-#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA_MASK                                                          0x10000000L
-#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK                                                              0x20000000L
-#define CP_COHER_CNTL__SH_KCACHE_WB_ACTION_ENA_MASK                                                           0x40000000L
-//CP_COHER_SIZE
-#define CP_COHER_SIZE__COHER_SIZE_256B__SHIFT                                                                 0x0
-#define CP_COHER_SIZE__COHER_SIZE_256B_MASK                                                                   0xFFFFFFFFL
-//CP_COHER_BASE
-#define CP_COHER_BASE__COHER_BASE_256B__SHIFT                                                                 0x0
-#define CP_COHER_BASE__COHER_BASE_256B_MASK                                                                   0xFFFFFFFFL
-//CP_COHER_STATUS
-#define CP_COHER_STATUS__MEID__SHIFT                                                                          0x18
-#define CP_COHER_STATUS__STATUS__SHIFT                                                                        0x1f
-#define CP_COHER_STATUS__MEID_MASK                                                                            0x03000000L
-#define CP_COHER_STATUS__STATUS_MASK                                                                          0x80000000L
-//CP_DMA_ME_SRC_ADDR
-#define CP_DMA_ME_SRC_ADDR__SRC_ADDR__SHIFT                                                                   0x0
-#define CP_DMA_ME_SRC_ADDR__SRC_ADDR_MASK                                                                     0xFFFFFFFFL
-//CP_DMA_ME_SRC_ADDR_HI
-#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                             0x0
-#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                               0x0000FFFFL
-//CP_DMA_ME_DST_ADDR
-#define CP_DMA_ME_DST_ADDR__DST_ADDR__SHIFT                                                                   0x0
-#define CP_DMA_ME_DST_ADDR__DST_ADDR_MASK                                                                     0xFFFFFFFFL
-//CP_DMA_ME_DST_ADDR_HI
-#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                             0x0
-#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI_MASK                                                               0x0000FFFFL
-//CP_DMA_ME_COMMAND
-#define CP_DMA_ME_COMMAND__BYTE_COUNT__SHIFT                                                                  0x0
-#define CP_DMA_ME_COMMAND__SAS__SHIFT                                                                         0x1a
-#define CP_DMA_ME_COMMAND__DAS__SHIFT                                                                         0x1b
-#define CP_DMA_ME_COMMAND__SAIC__SHIFT                                                                        0x1c
-#define CP_DMA_ME_COMMAND__DAIC__SHIFT                                                                        0x1d
-#define CP_DMA_ME_COMMAND__RAW_WAIT__SHIFT                                                                    0x1e
-#define CP_DMA_ME_COMMAND__DIS_WC__SHIFT                                                                      0x1f
-#define CP_DMA_ME_COMMAND__BYTE_COUNT_MASK                                                                    0x03FFFFFFL
-#define CP_DMA_ME_COMMAND__SAS_MASK                                                                           0x04000000L
-#define CP_DMA_ME_COMMAND__DAS_MASK                                                                           0x08000000L
-#define CP_DMA_ME_COMMAND__SAIC_MASK                                                                          0x10000000L
-#define CP_DMA_ME_COMMAND__DAIC_MASK                                                                          0x20000000L
-#define CP_DMA_ME_COMMAND__RAW_WAIT_MASK                                                                      0x40000000L
-#define CP_DMA_ME_COMMAND__DIS_WC_MASK                                                                        0x80000000L
-//CP_DMA_PFP_SRC_ADDR
-#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR__SHIFT                                                                  0x0
-#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR_MASK                                                                    0xFFFFFFFFL
-//CP_DMA_PFP_SRC_ADDR_HI
-#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                            0x0
-#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                              0x0000FFFFL
-//CP_DMA_PFP_DST_ADDR
-#define CP_DMA_PFP_DST_ADDR__DST_ADDR__SHIFT                                                                  0x0
-#define CP_DMA_PFP_DST_ADDR__DST_ADDR_MASK                                                                    0xFFFFFFFFL
-//CP_DMA_PFP_DST_ADDR_HI
-#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                            0x0
-#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI_MASK                                                              0x0000FFFFL
-//CP_DMA_PFP_COMMAND
-#define CP_DMA_PFP_COMMAND__BYTE_COUNT__SHIFT                                                                 0x0
-#define CP_DMA_PFP_COMMAND__SAS__SHIFT                                                                        0x1a
-#define CP_DMA_PFP_COMMAND__DAS__SHIFT                                                                        0x1b
-#define CP_DMA_PFP_COMMAND__SAIC__SHIFT                                                                       0x1c
-#define CP_DMA_PFP_COMMAND__DAIC__SHIFT                                                                       0x1d
-#define CP_DMA_PFP_COMMAND__RAW_WAIT__SHIFT                                                                   0x1e
-#define CP_DMA_PFP_COMMAND__DIS_WC__SHIFT                                                                     0x1f
-#define CP_DMA_PFP_COMMAND__BYTE_COUNT_MASK                                                                   0x03FFFFFFL
-#define CP_DMA_PFP_COMMAND__SAS_MASK                                                                          0x04000000L
-#define CP_DMA_PFP_COMMAND__DAS_MASK                                                                          0x08000000L
-#define CP_DMA_PFP_COMMAND__SAIC_MASK                                                                         0x10000000L
-#define CP_DMA_PFP_COMMAND__DAIC_MASK                                                                         0x20000000L
-#define CP_DMA_PFP_COMMAND__RAW_WAIT_MASK                                                                     0x40000000L
-#define CP_DMA_PFP_COMMAND__DIS_WC_MASK                                                                       0x80000000L
-//CP_DMA_CNTL
-#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL__SHIFT                                                               0x0
-#define CP_DMA_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x4
-#define CP_DMA_CNTL__BUFFER_DEPTH__SHIFT                                                                      0x10
-#define CP_DMA_CNTL__PIO_FIFO_EMPTY__SHIFT                                                                    0x1c
-#define CP_DMA_CNTL__PIO_FIFO_FULL__SHIFT                                                                     0x1d
-#define CP_DMA_CNTL__PIO_COUNT__SHIFT                                                                         0x1e
-#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL_MASK                                                                 0x00000001L
-#define CP_DMA_CNTL__MIN_AVAILSZ_MASK                                                                         0x00000030L
-#define CP_DMA_CNTL__BUFFER_DEPTH_MASK                                                                        0x000F0000L
-#define CP_DMA_CNTL__PIO_FIFO_EMPTY_MASK                                                                      0x10000000L
-#define CP_DMA_CNTL__PIO_FIFO_FULL_MASK                                                                       0x20000000L
-#define CP_DMA_CNTL__PIO_COUNT_MASK                                                                           0xC0000000L
-//CP_DMA_READ_TAGS
-#define CP_DMA_READ_TAGS__DMA_READ_TAG__SHIFT                                                                 0x0
-#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID__SHIFT                                                           0x1c
-#define CP_DMA_READ_TAGS__DMA_READ_TAG_MASK                                                                   0x03FFFFFFL
-#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID_MASK                                                             0x10000000L
-//CP_COHER_SIZE_HI
-#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT                                                           0x0
-#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK                                                             0x000000FFL
-//CP_PFP_IB_CONTROL
-#define CP_PFP_IB_CONTROL__IB_EN__SHIFT                                                                       0x0
-#define CP_PFP_IB_CONTROL__IB_EN_MASK                                                                         0x000000FFL
-//CP_PFP_LOAD_CONTROL
-#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN__SHIFT                                                             0x0
-#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN__SHIFT                                                               0x1
-#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN__SHIFT                                                             0x10
-#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN__SHIFT                                                              0x18
-#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN_MASK                                                               0x00000001L
-#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN_MASK                                                                 0x00000002L
-#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN_MASK                                                               0x00010000L
-#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN_MASK                                                                0x01000000L
-//CP_SCRATCH_INDEX
-#define CP_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                                0x0
-#define CP_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                                  0x000000FFL
-//CP_SCRATCH_DATA
-#define CP_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                                  0x0
-#define CP_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                    0xFFFFFFFFL
-//CP_RB_OFFSET
-#define CP_RB_OFFSET__RB_OFFSET__SHIFT                                                                        0x0
-#define CP_RB_OFFSET__RB_OFFSET_MASK                                                                          0x000FFFFFL
-//CP_IB1_OFFSET
-#define CP_IB1_OFFSET__IB1_OFFSET__SHIFT                                                                      0x0
-#define CP_IB1_OFFSET__IB1_OFFSET_MASK                                                                        0x000FFFFFL
-//CP_IB2_OFFSET
-#define CP_IB2_OFFSET__IB2_OFFSET__SHIFT                                                                      0x0
-#define CP_IB2_OFFSET__IB2_OFFSET_MASK                                                                        0x000FFFFFL
-//CP_IB1_PREAMBLE_BEGIN
-#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN__SHIFT                                                      0x0
-#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
-//CP_IB1_PREAMBLE_END
-#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END__SHIFT                                                          0x0
-#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END_MASK                                                            0x000FFFFFL
-//CP_IB2_PREAMBLE_BEGIN
-#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN__SHIFT                                                      0x0
-#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
-//CP_IB2_PREAMBLE_END
-#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END__SHIFT                                                          0x0
-#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END_MASK                                                            0x000FFFFFL
-//CP_CE_IB1_OFFSET
-#define CP_CE_IB1_OFFSET__IB1_OFFSET__SHIFT                                                                   0x0
-#define CP_CE_IB1_OFFSET__IB1_OFFSET_MASK                                                                     0x000FFFFFL
-//CP_CE_IB2_OFFSET
-#define CP_CE_IB2_OFFSET__IB2_OFFSET__SHIFT                                                                   0x0
-#define CP_CE_IB2_OFFSET__IB2_OFFSET_MASK                                                                     0x000FFFFFL
-//CP_CE_COUNTER
-#define CP_CE_COUNTER__CONST_ENGINE_COUNT__SHIFT                                                              0x0
-#define CP_CE_COUNTER__CONST_ENGINE_COUNT_MASK                                                                0xFFFFFFFFL
-//CP_CE_RB_OFFSET
-#define CP_CE_RB_OFFSET__RB_OFFSET__SHIFT                                                                     0x0
-#define CP_CE_RB_OFFSET__RB_OFFSET_MASK                                                                       0x000FFFFFL
-//CP_CE_INIT_CMD_BUFSZ
-#define CP_CE_INIT_CMD_BUFSZ__INIT_CMD_REQSZ__SHIFT                                                           0x0
-#define CP_CE_INIT_CMD_BUFSZ__INIT_CMD_REQSZ_MASK                                                             0x00000FFFL
-//CP_CE_IB1_CMD_BUFSZ
-#define CP_CE_IB1_CMD_BUFSZ__IB1_CMD_REQSZ__SHIFT                                                             0x0
-#define CP_CE_IB1_CMD_BUFSZ__IB1_CMD_REQSZ_MASK                                                               0x000FFFFFL
-//CP_CE_IB2_CMD_BUFSZ
-#define CP_CE_IB2_CMD_BUFSZ__IB2_CMD_REQSZ__SHIFT                                                             0x0
-#define CP_CE_IB2_CMD_BUFSZ__IB2_CMD_REQSZ_MASK                                                               0x000FFFFFL
-//CP_IB1_CMD_BUFSZ
-#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ__SHIFT                                                                0x0
-#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ_MASK                                                                  0x000FFFFFL
-//CP_IB2_CMD_BUFSZ
-#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ__SHIFT                                                                0x0
-#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ_MASK                                                                  0x000FFFFFL
-//CP_ST_CMD_BUFSZ
-#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ__SHIFT                                                                  0x0
-#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ_MASK                                                                    0x000FFFFFL
-//CP_CE_INIT_BASE_LO
-#define CP_CE_INIT_BASE_LO__INIT_BASE_LO__SHIFT                                                               0x5
-#define CP_CE_INIT_BASE_LO__INIT_BASE_LO_MASK                                                                 0xFFFFFFE0L
-//CP_CE_INIT_BASE_HI
-#define CP_CE_INIT_BASE_HI__INIT_BASE_HI__SHIFT                                                               0x0
-#define CP_CE_INIT_BASE_HI__INIT_BASE_HI_MASK                                                                 0x0000FFFFL
-//CP_CE_INIT_BUFSZ
-#define CP_CE_INIT_BUFSZ__INIT_BUFSZ__SHIFT                                                                   0x0
-#define CP_CE_INIT_BUFSZ__INIT_BUFSZ_MASK                                                                     0x00000FFFL
-//CP_CE_IB1_BASE_LO
-#define CP_CE_IB1_BASE_LO__IB1_BASE_LO__SHIFT                                                                 0x2
-#define CP_CE_IB1_BASE_LO__IB1_BASE_LO_MASK                                                                   0xFFFFFFFCL
-//CP_CE_IB1_BASE_HI
-#define CP_CE_IB1_BASE_HI__IB1_BASE_HI__SHIFT                                                                 0x0
-#define CP_CE_IB1_BASE_HI__IB1_BASE_HI_MASK                                                                   0x0000FFFFL
-//CP_CE_IB1_BUFSZ
-#define CP_CE_IB1_BUFSZ__IB1_BUFSZ__SHIFT                                                                     0x0
-#define CP_CE_IB1_BUFSZ__IB1_BUFSZ_MASK                                                                       0x000FFFFFL
-//CP_CE_IB2_BASE_LO
-#define CP_CE_IB2_BASE_LO__IB2_BASE_LO__SHIFT                                                                 0x2
-#define CP_CE_IB2_BASE_LO__IB2_BASE_LO_MASK                                                                   0xFFFFFFFCL
-//CP_CE_IB2_BASE_HI
-#define CP_CE_IB2_BASE_HI__IB2_BASE_HI__SHIFT                                                                 0x0
-#define CP_CE_IB2_BASE_HI__IB2_BASE_HI_MASK                                                                   0x0000FFFFL
-//CP_CE_IB2_BUFSZ
-#define CP_CE_IB2_BUFSZ__IB2_BUFSZ__SHIFT                                                                     0x0
-#define CP_CE_IB2_BUFSZ__IB2_BUFSZ_MASK                                                                       0x000FFFFFL
-//CP_IB1_BASE_LO
-#define CP_IB1_BASE_LO__IB1_BASE_LO__SHIFT                                                                    0x2
-#define CP_IB1_BASE_LO__IB1_BASE_LO_MASK                                                                      0xFFFFFFFCL
-//CP_IB1_BASE_HI
-#define CP_IB1_BASE_HI__IB1_BASE_HI__SHIFT                                                                    0x0
-#define CP_IB1_BASE_HI__IB1_BASE_HI_MASK                                                                      0x0000FFFFL
-//CP_IB1_BUFSZ
-#define CP_IB1_BUFSZ__IB1_BUFSZ__SHIFT                                                                        0x0
-#define CP_IB1_BUFSZ__IB1_BUFSZ_MASK                                                                          0x000FFFFFL
-//CP_IB2_BASE_LO
-#define CP_IB2_BASE_LO__IB2_BASE_LO__SHIFT                                                                    0x2
-#define CP_IB2_BASE_LO__IB2_BASE_LO_MASK                                                                      0xFFFFFFFCL
-//CP_IB2_BASE_HI
-#define CP_IB2_BASE_HI__IB2_BASE_HI__SHIFT                                                                    0x0
-#define CP_IB2_BASE_HI__IB2_BASE_HI_MASK                                                                      0x0000FFFFL
-//CP_IB2_BUFSZ
-#define CP_IB2_BUFSZ__IB2_BUFSZ__SHIFT                                                                        0x0
-#define CP_IB2_BUFSZ__IB2_BUFSZ_MASK                                                                          0x000FFFFFL
-//CP_ST_BASE_LO
-#define CP_ST_BASE_LO__ST_BASE_LO__SHIFT                                                                      0x2
-#define CP_ST_BASE_LO__ST_BASE_LO_MASK                                                                        0xFFFFFFFCL
-//CP_ST_BASE_HI
-#define CP_ST_BASE_HI__ST_BASE_HI__SHIFT                                                                      0x0
-#define CP_ST_BASE_HI__ST_BASE_HI_MASK                                                                        0x0000FFFFL
-//CP_ST_BUFSZ
-#define CP_ST_BUFSZ__ST_BUFSZ__SHIFT                                                                          0x0
-#define CP_ST_BUFSZ__ST_BUFSZ_MASK                                                                            0x000FFFFFL
-//CP_EOP_DONE_EVENT_CNTL
-#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP__SHIFT                                                            0x0
-#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA__SHIFT                                                       0xc
-#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY__SHIFT                                                           0x19
-#define CP_EOP_DONE_EVENT_CNTL__EXECUTE__SHIFT                                                                0x1c
-#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP_MASK                                                              0x0000007FL
-#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA_MASK                                                         0x0003F000L
-#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY_MASK                                                             0x02000000L
-#define CP_EOP_DONE_EVENT_CNTL__EXECUTE_MASK                                                                  0x10000000L
-//CP_EOP_DONE_DATA_CNTL
-#define CP_EOP_DONE_DATA_CNTL__DST_SEL__SHIFT                                                                 0x10
-#define CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT                                                                 0x18
-#define CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT                                                                0x1d
-#define CP_EOP_DONE_DATA_CNTL__DST_SEL_MASK                                                                   0x00030000L
-#define CP_EOP_DONE_DATA_CNTL__INT_SEL_MASK                                                                   0x07000000L
-#define CP_EOP_DONE_DATA_CNTL__DATA_SEL_MASK                                                                  0xE0000000L
-//CP_EOP_DONE_CNTX_ID
-#define CP_EOP_DONE_CNTX_ID__CNTX_ID__SHIFT                                                                   0x0
-#define CP_EOP_DONE_CNTX_ID__CNTX_ID_MASK                                                                     0xFFFFFFFFL
-//CP_PFP_COMPLETION_STATUS
-#define CP_PFP_COMPLETION_STATUS__STATUS__SHIFT                                                               0x0
-#define CP_PFP_COMPLETION_STATUS__STATUS_MASK                                                                 0x00000003L
-//CP_CE_COMPLETION_STATUS
-#define CP_CE_COMPLETION_STATUS__STATUS__SHIFT                                                                0x0
-#define CP_CE_COMPLETION_STATUS__STATUS_MASK                                                                  0x00000003L
-//CP_PRED_NOT_VISIBLE
-#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE__SHIFT                                                               0x0
-#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE_MASK                                                                 0x00000001L
-//CP_PFP_METADATA_BASE_ADDR
-#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO__SHIFT                                                             0x0
-#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO_MASK                                                               0xFFFFFFFFL
-//CP_PFP_METADATA_BASE_ADDR_HI
-#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
-#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x0000FFFFL
-//CP_CE_METADATA_BASE_ADDR
-#define CP_CE_METADATA_BASE_ADDR__ADDR_LO__SHIFT                                                              0x0
-#define CP_CE_METADATA_BASE_ADDR__ADDR_LO_MASK                                                                0xFFFFFFFFL
-//CP_CE_METADATA_BASE_ADDR_HI
-#define CP_CE_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT                                                           0x0
-#define CP_CE_METADATA_BASE_ADDR_HI__ADDR_HI_MASK                                                             0x0000FFFFL
-//CP_DRAW_INDX_INDR_ADDR
-#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO__SHIFT                                                                0x0
-#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO_MASK                                                                  0xFFFFFFFFL
-//CP_DRAW_INDX_INDR_ADDR_HI
-#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI__SHIFT                                                             0x0
-#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI_MASK                                                               0x0000FFFFL
-//CP_DISPATCH_INDR_ADDR
-#define CP_DISPATCH_INDR_ADDR__ADDR_LO__SHIFT                                                                 0x0
-#define CP_DISPATCH_INDR_ADDR__ADDR_LO_MASK                                                                   0xFFFFFFFFL
-//CP_DISPATCH_INDR_ADDR_HI
-#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI__SHIFT                                                              0x0
-#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI_MASK                                                                0x0000FFFFL
-//CP_INDEX_BASE_ADDR
-#define CP_INDEX_BASE_ADDR__ADDR_LO__SHIFT                                                                    0x0
-#define CP_INDEX_BASE_ADDR__ADDR_LO_MASK                                                                      0xFFFFFFFFL
-//CP_INDEX_BASE_ADDR_HI
-#define CP_INDEX_BASE_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
-#define CP_INDEX_BASE_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
-//CP_INDEX_TYPE
-#define CP_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                      0x0
-#define CP_INDEX_TYPE__INDEX_TYPE_MASK                                                                        0x00000003L
-//CP_GDS_BKUP_ADDR
-#define CP_GDS_BKUP_ADDR__ADDR_LO__SHIFT                                                                      0x0
-#define CP_GDS_BKUP_ADDR__ADDR_LO_MASK                                                                        0xFFFFFFFFL
-//CP_GDS_BKUP_ADDR_HI
-#define CP_GDS_BKUP_ADDR_HI__ADDR_HI__SHIFT                                                                   0x0
-#define CP_GDS_BKUP_ADDR_HI__ADDR_HI_MASK                                                                     0x0000FFFFL
-//CP_SAMPLE_STATUS
-#define CP_SAMPLE_STATUS__Z_PASS_ACITVE__SHIFT                                                                0x0
-#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE__SHIFT                                                             0x1
-#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE__SHIFT                                                              0x2
-#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE__SHIFT                                                               0x3
-#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE__SHIFT                                                           0x4
-#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE__SHIFT                                                            0x5
-#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE__SHIFT                                                         0x6
-#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE__SHIFT                                                         0x7
-#define CP_SAMPLE_STATUS__Z_PASS_ACITVE_MASK                                                                  0x00000001L
-#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE_MASK                                                               0x00000002L
-#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE_MASK                                                                0x00000004L
-#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE_MASK                                                                 0x00000008L
-#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE_MASK                                                             0x00000010L
-#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE_MASK                                                              0x00000020L
-#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE_MASK                                                           0x00000040L
-#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE_MASK                                                           0x00000080L
-//CP_ME_COHER_CNTL
-#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA__SHIFT                                                              0x0
-#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA__SHIFT                                                              0x1
-#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA__SHIFT                                                            0x6
-#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA__SHIFT                                                            0x7
-#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA__SHIFT                                                            0x8
-#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA__SHIFT                                                            0x9
-#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA__SHIFT                                                            0xa
-#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA__SHIFT                                                            0xb
-#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA__SHIFT                                                            0xc
-#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA__SHIFT                                                            0xd
-#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA__SHIFT                                                             0xe
-#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA__SHIFT                                                              0x13
-#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA__SHIFT                                                              0x15
-#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA_MASK                                                                0x00000001L
-#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA_MASK                                                                0x00000002L
-#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA_MASK                                                              0x00000040L
-#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA_MASK                                                              0x00000080L
-#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA_MASK                                                              0x00000100L
-#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA_MASK                                                              0x00000200L
-#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA_MASK                                                              0x00000400L
-#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA_MASK                                                              0x00000800L
-#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA_MASK                                                              0x00001000L
-#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA_MASK                                                              0x00002000L
-#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA_MASK                                                               0x00004000L
-#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA_MASK                                                                0x00080000L
-#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA_MASK                                                                0x00200000L
-//CP_ME_COHER_SIZE
-#define CP_ME_COHER_SIZE__COHER_SIZE_256B__SHIFT                                                              0x0
-#define CP_ME_COHER_SIZE__COHER_SIZE_256B_MASK                                                                0xFFFFFFFFL
-//CP_ME_COHER_SIZE_HI
-#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT                                                        0x0
-#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK                                                          0x000000FFL
-//CP_ME_COHER_BASE
-#define CP_ME_COHER_BASE__COHER_BASE_256B__SHIFT                                                              0x0
-#define CP_ME_COHER_BASE__COHER_BASE_256B_MASK                                                                0xFFFFFFFFL
-//CP_ME_COHER_BASE_HI
-#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT                                                        0x0
-#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B_MASK                                                          0x000000FFL
-//CP_ME_COHER_STATUS
-#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX__SHIFT                                                          0x0
-#define CP_ME_COHER_STATUS__STATUS__SHIFT                                                                     0x1f
-#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX_MASK                                                            0x000000FFL
-#define CP_ME_COHER_STATUS__STATUS_MASK                                                                       0x80000000L
-//RLC_GPM_PERF_COUNT_0
-#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL__SHIFT                                                              0x0
-#define RLC_GPM_PERF_COUNT_0__SE_INDEX__SHIFT                                                                 0x4
-#define RLC_GPM_PERF_COUNT_0__SH_INDEX__SHIFT                                                                 0x8
-#define RLC_GPM_PERF_COUNT_0__CU_INDEX__SHIFT                                                                 0xc
-#define RLC_GPM_PERF_COUNT_0__EVENT_SEL__SHIFT                                                                0x10
-#define RLC_GPM_PERF_COUNT_0__UNUSED__SHIFT                                                                   0x12
-#define RLC_GPM_PERF_COUNT_0__ENABLE__SHIFT                                                                   0x14
-#define RLC_GPM_PERF_COUNT_0__RESERVED__SHIFT                                                                 0x15
-#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL_MASK                                                                0x0000000FL
-#define RLC_GPM_PERF_COUNT_0__SE_INDEX_MASK                                                                   0x000000F0L
-#define RLC_GPM_PERF_COUNT_0__SH_INDEX_MASK                                                                   0x00000F00L
-#define RLC_GPM_PERF_COUNT_0__CU_INDEX_MASK                                                                   0x0000F000L
-#define RLC_GPM_PERF_COUNT_0__EVENT_SEL_MASK                                                                  0x00030000L
-#define RLC_GPM_PERF_COUNT_0__UNUSED_MASK                                                                     0x000C0000L
-#define RLC_GPM_PERF_COUNT_0__ENABLE_MASK                                                                     0x00100000L
-#define RLC_GPM_PERF_COUNT_0__RESERVED_MASK                                                                   0xFFE00000L
-//RLC_GPM_PERF_COUNT_1
-#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL__SHIFT                                                              0x0
-#define RLC_GPM_PERF_COUNT_1__SE_INDEX__SHIFT                                                                 0x4
-#define RLC_GPM_PERF_COUNT_1__SH_INDEX__SHIFT                                                                 0x8
-#define RLC_GPM_PERF_COUNT_1__CU_INDEX__SHIFT                                                                 0xc
-#define RLC_GPM_PERF_COUNT_1__EVENT_SEL__SHIFT                                                                0x10
-#define RLC_GPM_PERF_COUNT_1__UNUSED__SHIFT                                                                   0x12
-#define RLC_GPM_PERF_COUNT_1__ENABLE__SHIFT                                                                   0x14
-#define RLC_GPM_PERF_COUNT_1__RESERVED__SHIFT                                                                 0x15
-#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL_MASK                                                                0x0000000FL
-#define RLC_GPM_PERF_COUNT_1__SE_INDEX_MASK                                                                   0x000000F0L
-#define RLC_GPM_PERF_COUNT_1__SH_INDEX_MASK                                                                   0x00000F00L
-#define RLC_GPM_PERF_COUNT_1__CU_INDEX_MASK                                                                   0x0000F000L
-#define RLC_GPM_PERF_COUNT_1__EVENT_SEL_MASK                                                                  0x00030000L
-#define RLC_GPM_PERF_COUNT_1__UNUSED_MASK                                                                     0x000C0000L
-#define RLC_GPM_PERF_COUNT_1__ENABLE_MASK                                                                     0x00100000L
-#define RLC_GPM_PERF_COUNT_1__RESERVED_MASK                                                                   0xFFE00000L
-//GRBM_GFX_INDEX
-#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT                                                                 0x0
-#define GRBM_GFX_INDEX__SH_INDEX__SHIFT                                                                       0x8
-#define GRBM_GFX_INDEX__SE_INDEX__SHIFT                                                                       0x10
-#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES__SHIFT                                                            0x1d
-#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT                                                      0x1e
-#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT                                                            0x1f
-#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK                                                                   0x000000FFL
-#define GRBM_GFX_INDEX__SH_INDEX_MASK                                                                         0x0000FF00L
-#define GRBM_GFX_INDEX__SE_INDEX_MASK                                                                         0x00FF0000L
-#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK                                                              0x20000000L
-#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK                                                        0x40000000L
-#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK                                                              0x80000000L
-//VGT_GSVS_RING_SIZE
-#define VGT_GSVS_RING_SIZE__MEM_SIZE__SHIFT                                                                   0x0
-#define VGT_GSVS_RING_SIZE__MEM_SIZE_MASK                                                                     0xFFFFFFFFL
-//VGT_PRIMITIVE_TYPE
-#define VGT_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT                                                                  0x0
-#define VGT_PRIMITIVE_TYPE__PRIM_TYPE_MASK                                                                    0x0000003FL
-//VGT_INDEX_TYPE
-#define VGT_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                     0x0
-#define VGT_INDEX_TYPE__PRIMGEN_EN__SHIFT                                                                     0x8
-#define VGT_INDEX_TYPE__INDEX_TYPE_MASK                                                                       0x00000003L
-#define VGT_INDEX_TYPE__PRIMGEN_EN_MASK                                                                       0x00000100L
-//VGT_STRMOUT_BUFFER_FILLED_SIZE_0
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE__SHIFT                                                         0x0
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE_MASK                                                           0xFFFFFFFFL
-//VGT_STRMOUT_BUFFER_FILLED_SIZE_1
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE__SHIFT                                                         0x0
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE_MASK                                                           0xFFFFFFFFL
-//VGT_STRMOUT_BUFFER_FILLED_SIZE_2
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE__SHIFT                                                         0x0
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE_MASK                                                           0xFFFFFFFFL
-//VGT_STRMOUT_BUFFER_FILLED_SIZE_3
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE__SHIFT                                                         0x0
-#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE_MASK                                                           0xFFFFFFFFL
-//VGT_MAX_VTX_INDX
-#define VGT_MAX_VTX_INDX__MAX_INDX__SHIFT                                                                     0x0
-#define VGT_MAX_VTX_INDX__MAX_INDX_MASK                                                                       0xFFFFFFFFL
-//VGT_MIN_VTX_INDX
-#define VGT_MIN_VTX_INDX__MIN_INDX__SHIFT                                                                     0x0
-#define VGT_MIN_VTX_INDX__MIN_INDX_MASK                                                                       0xFFFFFFFFL
-//VGT_INDX_OFFSET
-#define VGT_INDX_OFFSET__INDX_OFFSET__SHIFT                                                                   0x0
-#define VGT_INDX_OFFSET__INDX_OFFSET_MASK                                                                     0xFFFFFFFFL
-//VGT_MULTI_PRIM_IB_RESET_EN
-#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN__SHIFT                                                           0x0
-#define VGT_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS__SHIFT                                                     0x1
-#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN_MASK                                                             0x00000001L
-#define VGT_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS_MASK                                                       0x00000002L
-//VGT_NUM_INDICES
-#define VGT_NUM_INDICES__NUM_INDICES__SHIFT                                                                   0x0
-#define VGT_NUM_INDICES__NUM_INDICES_MASK                                                                     0xFFFFFFFFL
-//VGT_NUM_INSTANCES
-#define VGT_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                               0x0
-#define VGT_NUM_INSTANCES__NUM_INSTANCES_MASK                                                                 0xFFFFFFFFL
-//VGT_TF_RING_SIZE
-#define VGT_TF_RING_SIZE__SIZE__SHIFT                                                                         0x0
-#define VGT_TF_RING_SIZE__SIZE_MASK                                                                           0x0000FFFFL
-//VGT_HS_OFFCHIP_PARAM
-#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING__SHIFT                                                        0x0
-#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY__SHIFT                                                      0x9
-#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING_MASK                                                          0x000001FFL
-#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY_MASK                                                        0x00000600L
-//VGT_TF_MEMORY_BASE
-#define VGT_TF_MEMORY_BASE__BASE__SHIFT                                                                       0x0
-#define VGT_TF_MEMORY_BASE__BASE_MASK                                                                         0xFFFFFFFFL
-//VGT_TF_MEMORY_BASE_HI
-#define VGT_TF_MEMORY_BASE_HI__BASE_HI__SHIFT                                                                 0x0
-#define VGT_TF_MEMORY_BASE_HI__BASE_HI_MASK                                                                   0x000000FFL
-//WD_POS_BUF_BASE
-#define WD_POS_BUF_BASE__BASE__SHIFT                                                                          0x0
-#define WD_POS_BUF_BASE__BASE_MASK                                                                            0xFFFFFFFFL
-//WD_POS_BUF_BASE_HI
-#define WD_POS_BUF_BASE_HI__BASE_HI__SHIFT                                                                    0x0
-#define WD_POS_BUF_BASE_HI__BASE_HI_MASK                                                                      0x000000FFL
-//WD_CNTL_SB_BUF_BASE
-#define WD_CNTL_SB_BUF_BASE__BASE__SHIFT                                                                      0x0
-#define WD_CNTL_SB_BUF_BASE__BASE_MASK                                                                        0xFFFFFFFFL
-//WD_CNTL_SB_BUF_BASE_HI
-#define WD_CNTL_SB_BUF_BASE_HI__BASE_HI__SHIFT                                                                0x0
-#define WD_CNTL_SB_BUF_BASE_HI__BASE_HI_MASK                                                                  0x000000FFL
-//WD_INDEX_BUF_BASE
-#define WD_INDEX_BUF_BASE__BASE__SHIFT                                                                        0x0
-#define WD_INDEX_BUF_BASE__BASE_MASK                                                                          0xFFFFFFFFL
-//WD_INDEX_BUF_BASE_HI
-#define WD_INDEX_BUF_BASE_HI__BASE_HI__SHIFT                                                                  0x0
-#define WD_INDEX_BUF_BASE_HI__BASE_HI_MASK                                                                    0x000000FFL
-//IA_MULTI_VGT_PARAM
-#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE__SHIFT                                                             0x0
-#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON__SHIFT                                                         0x10
-#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP__SHIFT                                                              0x11
-#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON__SHIFT                                                         0x12
-#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI__SHIFT                                                              0x13
-#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP__SHIFT                                                           0x14
-#define IA_MULTI_VGT_PARAM__EN_INST_OPT_BASIC__SHIFT                                                          0x15
-#define IA_MULTI_VGT_PARAM__EN_INST_OPT_ADV__SHIFT                                                            0x16
-#define IA_MULTI_VGT_PARAM__HW_USE_ONLY__SHIFT                                                                0x17
-#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE_MASK                                                               0x0000FFFFL
-#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON_MASK                                                           0x00010000L
-#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP_MASK                                                                0x00020000L
-#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON_MASK                                                           0x00040000L
-#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI_MASK                                                                0x00080000L
-#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP_MASK                                                             0x00100000L
-#define IA_MULTI_VGT_PARAM__EN_INST_OPT_BASIC_MASK                                                            0x00200000L
-#define IA_MULTI_VGT_PARAM__EN_INST_OPT_ADV_MASK                                                              0x00400000L
-#define IA_MULTI_VGT_PARAM__HW_USE_ONLY_MASK                                                                  0x00800000L
-//VGT_OBJECT_ID
-#define VGT_OBJECT_ID__REG_OBJ_ID__SHIFT                                                                      0x0
-#define VGT_OBJECT_ID__REG_OBJ_ID_MASK                                                                        0xFFFFFFFFL
-//VGT_INSTANCE_BASE_ID
-#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID__SHIFT                                                         0x0
-#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID_MASK                                                           0xFFFFFFFFL
-//PA_SU_LINE_STIPPLE_VALUE
-#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE__SHIFT                                                   0x0
-#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE_MASK                                                     0x00FFFFFFL
-//PA_SC_LINE_STIPPLE_STATE
-#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR__SHIFT                                                          0x0
-#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT__SHIFT                                                        0x8
-#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR_MASK                                                            0x0000000FL
-#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT_MASK                                                          0x0000FF00L
-//PA_SC_SCREEN_EXTENT_MIN_0
-#define PA_SC_SCREEN_EXTENT_MIN_0__X__SHIFT                                                                   0x0
-#define PA_SC_SCREEN_EXTENT_MIN_0__Y__SHIFT                                                                   0x10
-#define PA_SC_SCREEN_EXTENT_MIN_0__X_MASK                                                                     0x0000FFFFL
-#define PA_SC_SCREEN_EXTENT_MIN_0__Y_MASK                                                                     0xFFFF0000L
-//PA_SC_SCREEN_EXTENT_MAX_0
-#define PA_SC_SCREEN_EXTENT_MAX_0__X__SHIFT                                                                   0x0
-#define PA_SC_SCREEN_EXTENT_MAX_0__Y__SHIFT                                                                   0x10
-#define PA_SC_SCREEN_EXTENT_MAX_0__X_MASK                                                                     0x0000FFFFL
-#define PA_SC_SCREEN_EXTENT_MAX_0__Y_MASK                                                                     0xFFFF0000L
-//PA_SC_SCREEN_EXTENT_MIN_1
-#define PA_SC_SCREEN_EXTENT_MIN_1__X__SHIFT                                                                   0x0
-#define PA_SC_SCREEN_EXTENT_MIN_1__Y__SHIFT                                                                   0x10
-#define PA_SC_SCREEN_EXTENT_MIN_1__X_MASK                                                                     0x0000FFFFL
-#define PA_SC_SCREEN_EXTENT_MIN_1__Y_MASK                                                                     0xFFFF0000L
-//PA_SC_SCREEN_EXTENT_MAX_1
-#define PA_SC_SCREEN_EXTENT_MAX_1__X__SHIFT                                                                   0x0
-#define PA_SC_SCREEN_EXTENT_MAX_1__Y__SHIFT                                                                   0x10
-#define PA_SC_SCREEN_EXTENT_MAX_1__X_MASK                                                                     0x0000FFFFL
-#define PA_SC_SCREEN_EXTENT_MAX_1__Y_MASK                                                                     0xFFFF0000L
-//PA_SC_P3D_TRAP_SCREEN_HV_EN
-#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                              0x0
-#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                       0x1
-#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                0x00000001L
-#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                         0x00000002L
-//PA_SC_P3D_TRAP_SCREEN_H
-#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                               0x0
-#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD_MASK                                                                 0x00003FFFL
-//PA_SC_P3D_TRAP_SCREEN_V
-#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                               0x0
-#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                 0x00003FFFL
-//PA_SC_P3D_TRAP_SCREEN_OCCURRENCE
-#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                        0x0
-#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                          0x0000FFFFL
-//PA_SC_P3D_TRAP_SCREEN_COUNT
-#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                             0x0
-#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                               0x0000FFFFL
-//PA_SC_HP3D_TRAP_SCREEN_HV_EN
-#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                             0x0
-#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                      0x1
-#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                               0x00000001L
-#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                        0x00000002L
-//PA_SC_HP3D_TRAP_SCREEN_H
-#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                              0x0
-#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD_MASK                                                                0x00003FFFL
-//PA_SC_HP3D_TRAP_SCREEN_V
-#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                              0x0
-#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                0x00003FFFL
-//PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE
-#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                       0x0
-#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                         0x0000FFFFL
-//PA_SC_HP3D_TRAP_SCREEN_COUNT
-#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                            0x0
-#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                              0x0000FFFFL
-//PA_SC_TRAP_SCREEN_HV_EN
-#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                                  0x0
-#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                           0x1
-#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                    0x00000001L
-#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                             0x00000002L
-//PA_SC_TRAP_SCREEN_H
-#define PA_SC_TRAP_SCREEN_H__X_COORD__SHIFT                                                                   0x0
-#define PA_SC_TRAP_SCREEN_H__X_COORD_MASK                                                                     0x00003FFFL
-//PA_SC_TRAP_SCREEN_V
-#define PA_SC_TRAP_SCREEN_V__Y_COORD__SHIFT                                                                   0x0
-#define PA_SC_TRAP_SCREEN_V__Y_COORD_MASK                                                                     0x00003FFFL
-//PA_SC_TRAP_SCREEN_OCCURRENCE
-#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                            0x0
-#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                              0x0000FFFFL
-//PA_SC_TRAP_SCREEN_COUNT
-#define PA_SC_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                                 0x0
-#define PA_SC_TRAP_SCREEN_COUNT__COUNT_MASK                                                                   0x0000FFFFL
-//SQ_THREAD_TRACE_BASE
-#define SQ_THREAD_TRACE_BASE__ADDR__SHIFT                                                                     0x0
-#define SQ_THREAD_TRACE_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
-//SQ_THREAD_TRACE_SIZE
-#define SQ_THREAD_TRACE_SIZE__SIZE__SHIFT                                                                     0x0
-#define SQ_THREAD_TRACE_SIZE__SIZE_MASK                                                                       0x003FFFFFL
-//SQ_THREAD_TRACE_MASK
-#define SQ_THREAD_TRACE_MASK__CU_SEL__SHIFT                                                                   0x0
-#define SQ_THREAD_TRACE_MASK__SH_SEL__SHIFT                                                                   0x5
-#define SQ_THREAD_TRACE_MASK__REG_STALL_EN__SHIFT                                                             0x7
-#define SQ_THREAD_TRACE_MASK__SIMD_EN__SHIFT                                                                  0x8
-#define SQ_THREAD_TRACE_MASK__VM_ID_MASK__SHIFT                                                               0xc
-#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN__SHIFT                                                             0xe
-#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN__SHIFT                                                              0xf
-#define SQ_THREAD_TRACE_MASK__CU_SEL_MASK                                                                     0x0000001FL
-#define SQ_THREAD_TRACE_MASK__SH_SEL_MASK                                                                     0x00000020L
-#define SQ_THREAD_TRACE_MASK__REG_STALL_EN_MASK                                                               0x00000080L
-#define SQ_THREAD_TRACE_MASK__SIMD_EN_MASK                                                                    0x00000F00L
-#define SQ_THREAD_TRACE_MASK__VM_ID_MASK_MASK                                                                 0x00003000L
-#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN_MASK                                                               0x00004000L
-#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN_MASK                                                                0x00008000L
-//SQ_THREAD_TRACE_TOKEN_MASK
-#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK__SHIFT                                                         0x0
-#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK__SHIFT                                                           0x10
-#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL__SHIFT                                                  0x18
-#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK_MASK                                                           0x0000FFFFL
-#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK_MASK                                                             0x00FF0000L
-#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL_MASK                                                    0x01000000L
-//SQ_THREAD_TRACE_PERF_MASK
-#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK__SHIFT                                                            0x0
-#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK__SHIFT                                                            0x10
-#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK_MASK                                                              0x0000FFFFL
-#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK_MASK                                                              0xFFFF0000L
-//SQ_THREAD_TRACE_CTRL
-#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER__SHIFT                                                             0x1f
-#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER_MASK                                                               0x80000000L
-//SQ_THREAD_TRACE_MODE
-#define SQ_THREAD_TRACE_MODE__MASK_PS__SHIFT                                                                  0x0
-#define SQ_THREAD_TRACE_MODE__MASK_VS__SHIFT                                                                  0x3
-#define SQ_THREAD_TRACE_MODE__MASK_GS__SHIFT                                                                  0x6
-#define SQ_THREAD_TRACE_MODE__MASK_ES__SHIFT                                                                  0x9
-#define SQ_THREAD_TRACE_MODE__MASK_HS__SHIFT                                                                  0xc
-#define SQ_THREAD_TRACE_MODE__MASK_LS__SHIFT                                                                  0xf
-#define SQ_THREAD_TRACE_MODE__MASK_CS__SHIFT                                                                  0x12
-#define SQ_THREAD_TRACE_MODE__MODE__SHIFT                                                                     0x15
-#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE__SHIFT                                                             0x17
-#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN__SHIFT                                                             0x19
-#define SQ_THREAD_TRACE_MODE__TC_PERF_EN__SHIFT                                                               0x1a
-#define SQ_THREAD_TRACE_MODE__ISSUE_MASK__SHIFT                                                               0x1b
-#define SQ_THREAD_TRACE_MODE__TEST_MODE__SHIFT                                                                0x1d
-#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN__SHIFT                                                             0x1e
-#define SQ_THREAD_TRACE_MODE__WRAP__SHIFT                                                                     0x1f
-#define SQ_THREAD_TRACE_MODE__MASK_PS_MASK                                                                    0x00000007L
-#define SQ_THREAD_TRACE_MODE__MASK_VS_MASK                                                                    0x00000038L
-#define SQ_THREAD_TRACE_MODE__MASK_GS_MASK                                                                    0x000001C0L
-#define SQ_THREAD_TRACE_MODE__MASK_ES_MASK                                                                    0x00000E00L
-#define SQ_THREAD_TRACE_MODE__MASK_HS_MASK                                                                    0x00007000L
-#define SQ_THREAD_TRACE_MODE__MASK_LS_MASK                                                                    0x00038000L
-#define SQ_THREAD_TRACE_MODE__MASK_CS_MASK                                                                    0x001C0000L
-#define SQ_THREAD_TRACE_MODE__MODE_MASK                                                                       0x00600000L
-#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE_MASK                                                               0x01800000L
-#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN_MASK                                                               0x02000000L
-#define SQ_THREAD_TRACE_MODE__TC_PERF_EN_MASK                                                                 0x04000000L
-#define SQ_THREAD_TRACE_MODE__ISSUE_MASK_MASK                                                                 0x18000000L
-#define SQ_THREAD_TRACE_MODE__TEST_MODE_MASK                                                                  0x20000000L
-#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN_MASK                                                               0x40000000L
-#define SQ_THREAD_TRACE_MODE__WRAP_MASK                                                                       0x80000000L
-//SQ_THREAD_TRACE_BASE2
-#define SQ_THREAD_TRACE_BASE2__ADDR_HI__SHIFT                                                                 0x0
-#define SQ_THREAD_TRACE_BASE2__ADDR_HI_MASK                                                                   0x0000000FL
-//SQ_THREAD_TRACE_TOKEN_MASK2
-#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK__SHIFT                                                         0x0
-#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK_MASK                                                           0xFFFFFFFFL
-//SQ_THREAD_TRACE_WPTR
-#define SQ_THREAD_TRACE_WPTR__WPTR__SHIFT                                                                     0x0
-#define SQ_THREAD_TRACE_WPTR__READ_OFFSET__SHIFT                                                              0x1e
-#define SQ_THREAD_TRACE_WPTR__WPTR_MASK                                                                       0x3FFFFFFFL
-#define SQ_THREAD_TRACE_WPTR__READ_OFFSET_MASK                                                                0xC0000000L
-//SQ_THREAD_TRACE_STATUS
-#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING__SHIFT                                                         0x0
-#define SQ_THREAD_TRACE_STATUS__FINISH_DONE__SHIFT                                                            0x10
-#define SQ_THREAD_TRACE_STATUS__UTC_ERROR__SHIFT                                                              0x1c
-#define SQ_THREAD_TRACE_STATUS__NEW_BUF__SHIFT                                                                0x1d
-#define SQ_THREAD_TRACE_STATUS__BUSY__SHIFT                                                                   0x1e
-#define SQ_THREAD_TRACE_STATUS__FULL__SHIFT                                                                   0x1f
-#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK                                                           0x000003FFL
-#define SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK                                                              0x03FF0000L
-#define SQ_THREAD_TRACE_STATUS__UTC_ERROR_MASK                                                                0x10000000L
-#define SQ_THREAD_TRACE_STATUS__NEW_BUF_MASK                                                                  0x20000000L
-#define SQ_THREAD_TRACE_STATUS__BUSY_MASK                                                                     0x40000000L
-#define SQ_THREAD_TRACE_STATUS__FULL_MASK                                                                     0x80000000L
-//SQ_THREAD_TRACE_HIWATER
-#define SQ_THREAD_TRACE_HIWATER__HIWATER__SHIFT                                                               0x0
-#define SQ_THREAD_TRACE_HIWATER__HIWATER_MASK                                                                 0x00000007L
-//SQ_THREAD_TRACE_CNTR
-#define SQ_THREAD_TRACE_CNTR__CNTR__SHIFT                                                                     0x0
-#define SQ_THREAD_TRACE_CNTR__CNTR_MASK                                                                       0xFFFFFFFFL
-//SQ_THREAD_TRACE_USERDATA_0
-#define SQ_THREAD_TRACE_USERDATA_0__DATA__SHIFT                                                               0x0
-#define SQ_THREAD_TRACE_USERDATA_0__DATA_MASK                                                                 0xFFFFFFFFL
-//SQ_THREAD_TRACE_USERDATA_1
-#define SQ_THREAD_TRACE_USERDATA_1__DATA__SHIFT                                                               0x0
-#define SQ_THREAD_TRACE_USERDATA_1__DATA_MASK                                                                 0xFFFFFFFFL
-//SQ_THREAD_TRACE_USERDATA_2
-#define SQ_THREAD_TRACE_USERDATA_2__DATA__SHIFT                                                               0x0
-#define SQ_THREAD_TRACE_USERDATA_2__DATA_MASK                                                                 0xFFFFFFFFL
-//SQ_THREAD_TRACE_USERDATA_3
-#define SQ_THREAD_TRACE_USERDATA_3__DATA__SHIFT                                                               0x0
-#define SQ_THREAD_TRACE_USERDATA_3__DATA_MASK                                                                 0xFFFFFFFFL
-//SQC_CACHES
-#define SQC_CACHES__TARGET_INST__SHIFT                                                                        0x0
-#define SQC_CACHES__TARGET_DATA__SHIFT                                                                        0x1
-#define SQC_CACHES__INVALIDATE__SHIFT                                                                         0x2
-#define SQC_CACHES__WRITEBACK__SHIFT                                                                          0x3
-#define SQC_CACHES__VOL__SHIFT                                                                                0x4
-#define SQC_CACHES__COMPLETE__SHIFT                                                                           0x10
-#define SQC_CACHES__TARGET_INST_MASK                                                                          0x00000001L
-#define SQC_CACHES__TARGET_DATA_MASK                                                                          0x00000002L
-#define SQC_CACHES__INVALIDATE_MASK                                                                           0x00000004L
-#define SQC_CACHES__WRITEBACK_MASK                                                                            0x00000008L
-#define SQC_CACHES__VOL_MASK                                                                                  0x00000010L
-#define SQC_CACHES__COMPLETE_MASK                                                                             0x00010000L
-//SQC_WRITEBACK
-#define SQC_WRITEBACK__DWB__SHIFT                                                                             0x0
-#define SQC_WRITEBACK__DIRTY__SHIFT                                                                           0x1
-#define SQC_WRITEBACK__DWB_MASK                                                                               0x00000001L
-#define SQC_WRITEBACK__DIRTY_MASK                                                                             0x00000002L
-//TA_CS_BC_BASE_ADDR
-#define TA_CS_BC_BASE_ADDR__ADDRESS__SHIFT                                                                    0x0
-#define TA_CS_BC_BASE_ADDR__ADDRESS_MASK                                                                      0xFFFFFFFFL
-//TA_CS_BC_BASE_ADDR_HI
-#define TA_CS_BC_BASE_ADDR_HI__ADDRESS__SHIFT                                                                 0x0
-#define TA_CS_BC_BASE_ADDR_HI__ADDRESS_MASK                                                                   0x000000FFL
-//TA_GRAD_ADJ_UCONFIG
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_0__SHIFT                                                                0x0
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_1__SHIFT                                                                0x8
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_2__SHIFT                                                                0x10
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_3__SHIFT                                                                0x18
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_0_MASK                                                                  0x000000FFL
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_1_MASK                                                                  0x0000FF00L
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_2_MASK                                                                  0x00FF0000L
-#define TA_GRAD_ADJ_UCONFIG__GRAD_ADJ_3_MASK                                                                  0xFF000000L
-//DB_OCCLUSION_COUNT0_LOW
-#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW__SHIFT                                                             0x0
-#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
-//DB_OCCLUSION_COUNT0_HI
-#define DB_OCCLUSION_COUNT0_HI__COUNT_HI__SHIFT                                                               0x0
-#define DB_OCCLUSION_COUNT0_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
-//DB_OCCLUSION_COUNT1_LOW
-#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW__SHIFT                                                             0x0
-#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
-//DB_OCCLUSION_COUNT1_HI
-#define DB_OCCLUSION_COUNT1_HI__COUNT_HI__SHIFT                                                               0x0
-#define DB_OCCLUSION_COUNT1_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
-//DB_OCCLUSION_COUNT2_LOW
-#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW__SHIFT                                                             0x0
-#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
-//DB_OCCLUSION_COUNT2_HI
-#define DB_OCCLUSION_COUNT2_HI__COUNT_HI__SHIFT                                                               0x0
-#define DB_OCCLUSION_COUNT2_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
-//DB_OCCLUSION_COUNT3_LOW
-#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW__SHIFT                                                             0x0
-#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
-//DB_OCCLUSION_COUNT3_HI
-#define DB_OCCLUSION_COUNT3_HI__COUNT_HI__SHIFT                                                               0x0
-#define DB_OCCLUSION_COUNT3_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
-//DB_ZPASS_COUNT_LOW
-#define DB_ZPASS_COUNT_LOW__COUNT_LOW__SHIFT                                                                  0x0
-#define DB_ZPASS_COUNT_LOW__COUNT_LOW_MASK                                                                    0xFFFFFFFFL
-//DB_ZPASS_COUNT_HI
-#define DB_ZPASS_COUNT_HI__COUNT_HI__SHIFT                                                                    0x0
-#define DB_ZPASS_COUNT_HI__COUNT_HI_MASK                                                                      0x7FFFFFFFL
-//GDS_RD_ADDR
-#define GDS_RD_ADDR__READ_ADDR__SHIFT                                                                         0x0
-#define GDS_RD_ADDR__READ_ADDR_MASK                                                                           0xFFFFFFFFL
-//GDS_RD_DATA
-#define GDS_RD_DATA__READ_DATA__SHIFT                                                                         0x0
-#define GDS_RD_DATA__READ_DATA_MASK                                                                           0xFFFFFFFFL
-//GDS_RD_BURST_ADDR
-#define GDS_RD_BURST_ADDR__BURST_ADDR__SHIFT                                                                  0x0
-#define GDS_RD_BURST_ADDR__BURST_ADDR_MASK                                                                    0xFFFFFFFFL
-//GDS_RD_BURST_COUNT
-#define GDS_RD_BURST_COUNT__BURST_COUNT__SHIFT                                                                0x0
-#define GDS_RD_BURST_COUNT__BURST_COUNT_MASK                                                                  0xFFFFFFFFL
-//GDS_RD_BURST_DATA
-#define GDS_RD_BURST_DATA__BURST_DATA__SHIFT                                                                  0x0
-#define GDS_RD_BURST_DATA__BURST_DATA_MASK                                                                    0xFFFFFFFFL
-//GDS_WR_ADDR
-#define GDS_WR_ADDR__WRITE_ADDR__SHIFT                                                                        0x0
-#define GDS_WR_ADDR__WRITE_ADDR_MASK                                                                          0xFFFFFFFFL
-//GDS_WR_DATA
-#define GDS_WR_DATA__WRITE_DATA__SHIFT                                                                        0x0
-#define GDS_WR_DATA__WRITE_DATA_MASK                                                                          0xFFFFFFFFL
-//GDS_WR_BURST_ADDR
-#define GDS_WR_BURST_ADDR__WRITE_ADDR__SHIFT                                                                  0x0
-#define GDS_WR_BURST_ADDR__WRITE_ADDR_MASK                                                                    0xFFFFFFFFL
-//GDS_WR_BURST_DATA
-#define GDS_WR_BURST_DATA__WRITE_DATA__SHIFT                                                                  0x0
-#define GDS_WR_BURST_DATA__WRITE_DATA_MASK                                                                    0xFFFFFFFFL
-//GDS_WRITE_COMPLETE
-#define GDS_WRITE_COMPLETE__WRITE_COMPLETE__SHIFT                                                             0x0
-#define GDS_WRITE_COMPLETE__WRITE_COMPLETE_MASK                                                               0xFFFFFFFFL
-//GDS_ATOM_CNTL
-#define GDS_ATOM_CNTL__AINC__SHIFT                                                                            0x0
-#define GDS_ATOM_CNTL__UNUSED1__SHIFT                                                                         0x6
-#define GDS_ATOM_CNTL__DMODE__SHIFT                                                                           0x8
-#define GDS_ATOM_CNTL__UNUSED2__SHIFT                                                                         0xa
-#define GDS_ATOM_CNTL__AINC_MASK                                                                              0x0000003FL
-#define GDS_ATOM_CNTL__UNUSED1_MASK                                                                           0x000000C0L
-#define GDS_ATOM_CNTL__DMODE_MASK                                                                             0x00000300L
-#define GDS_ATOM_CNTL__UNUSED2_MASK                                                                           0xFFFFFC00L
-//GDS_ATOM_COMPLETE
-#define GDS_ATOM_COMPLETE__COMPLETE__SHIFT                                                                    0x0
-#define GDS_ATOM_COMPLETE__UNUSED__SHIFT                                                                      0x1
-#define GDS_ATOM_COMPLETE__COMPLETE_MASK                                                                      0x00000001L
-#define GDS_ATOM_COMPLETE__UNUSED_MASK                                                                        0xFFFFFFFEL
-//GDS_ATOM_BASE
-#define GDS_ATOM_BASE__BASE__SHIFT                                                                            0x0
-#define GDS_ATOM_BASE__UNUSED__SHIFT                                                                          0x10
-#define GDS_ATOM_BASE__BASE_MASK                                                                              0x0000FFFFL
-#define GDS_ATOM_BASE__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_ATOM_SIZE
-#define GDS_ATOM_SIZE__SIZE__SHIFT                                                                            0x0
-#define GDS_ATOM_SIZE__UNUSED__SHIFT                                                                          0x10
-#define GDS_ATOM_SIZE__SIZE_MASK                                                                              0x0000FFFFL
-#define GDS_ATOM_SIZE__UNUSED_MASK                                                                            0xFFFF0000L
-//GDS_ATOM_OFFSET0
-#define GDS_ATOM_OFFSET0__OFFSET0__SHIFT                                                                      0x0
-#define GDS_ATOM_OFFSET0__UNUSED__SHIFT                                                                       0x8
-#define GDS_ATOM_OFFSET0__OFFSET0_MASK                                                                        0x000000FFL
-#define GDS_ATOM_OFFSET0__UNUSED_MASK                                                                         0xFFFFFF00L
-//GDS_ATOM_OFFSET1
-#define GDS_ATOM_OFFSET1__OFFSET1__SHIFT                                                                      0x0
-#define GDS_ATOM_OFFSET1__UNUSED__SHIFT                                                                       0x8
-#define GDS_ATOM_OFFSET1__OFFSET1_MASK                                                                        0x000000FFL
-#define GDS_ATOM_OFFSET1__UNUSED_MASK                                                                         0xFFFFFF00L
-//GDS_ATOM_DST
-#define GDS_ATOM_DST__DST__SHIFT                                                                              0x0
-#define GDS_ATOM_DST__DST_MASK                                                                                0xFFFFFFFFL
-//GDS_ATOM_OP
-#define GDS_ATOM_OP__OP__SHIFT                                                                                0x0
-#define GDS_ATOM_OP__UNUSED__SHIFT                                                                            0x8
-#define GDS_ATOM_OP__OP_MASK                                                                                  0x000000FFL
-#define GDS_ATOM_OP__UNUSED_MASK                                                                              0xFFFFFF00L
-//GDS_ATOM_SRC0
-#define GDS_ATOM_SRC0__DATA__SHIFT                                                                            0x0
-#define GDS_ATOM_SRC0__DATA_MASK                                                                              0xFFFFFFFFL
-//GDS_ATOM_SRC0_U
-#define GDS_ATOM_SRC0_U__DATA__SHIFT                                                                          0x0
-#define GDS_ATOM_SRC0_U__DATA_MASK                                                                            0xFFFFFFFFL
-//GDS_ATOM_SRC1
-#define GDS_ATOM_SRC1__DATA__SHIFT                                                                            0x0
-#define GDS_ATOM_SRC1__DATA_MASK                                                                              0xFFFFFFFFL
-//GDS_ATOM_SRC1_U
-#define GDS_ATOM_SRC1_U__DATA__SHIFT                                                                          0x0
-#define GDS_ATOM_SRC1_U__DATA_MASK                                                                            0xFFFFFFFFL
-//GDS_ATOM_READ0
-#define GDS_ATOM_READ0__DATA__SHIFT                                                                           0x0
-#define GDS_ATOM_READ0__DATA_MASK                                                                             0xFFFFFFFFL
-//GDS_ATOM_READ0_U
-#define GDS_ATOM_READ0_U__DATA__SHIFT                                                                         0x0
-#define GDS_ATOM_READ0_U__DATA_MASK                                                                           0xFFFFFFFFL
-//GDS_ATOM_READ1
-#define GDS_ATOM_READ1__DATA__SHIFT                                                                           0x0
-#define GDS_ATOM_READ1__DATA_MASK                                                                             0xFFFFFFFFL
-//GDS_ATOM_READ1_U
-#define GDS_ATOM_READ1_U__DATA__SHIFT                                                                         0x0
-#define GDS_ATOM_READ1_U__DATA_MASK                                                                           0xFFFFFFFFL
-//GDS_GWS_RESOURCE_CNTL
-#define GDS_GWS_RESOURCE_CNTL__INDEX__SHIFT                                                                   0x0
-#define GDS_GWS_RESOURCE_CNTL__UNUSED__SHIFT                                                                  0x6
-#define GDS_GWS_RESOURCE_CNTL__INDEX_MASK                                                                     0x0000003FL
-#define GDS_GWS_RESOURCE_CNTL__UNUSED_MASK                                                                    0xFFFFFFC0L
-//GDS_GWS_RESOURCE
-#define GDS_GWS_RESOURCE__FLAG__SHIFT                                                                         0x0
-#define GDS_GWS_RESOURCE__COUNTER__SHIFT                                                                      0x1
-#define GDS_GWS_RESOURCE__TYPE__SHIFT                                                                         0xd
-#define GDS_GWS_RESOURCE__DED__SHIFT                                                                          0xe
-#define GDS_GWS_RESOURCE__RELEASE_ALL__SHIFT                                                                  0xf
-#define GDS_GWS_RESOURCE__HEAD_QUEUE__SHIFT                                                                   0x10
-#define GDS_GWS_RESOURCE__HEAD_VALID__SHIFT                                                                   0x1c
-#define GDS_GWS_RESOURCE__HEAD_FLAG__SHIFT                                                                    0x1d
-#define GDS_GWS_RESOURCE__HALTED__SHIFT                                                                       0x1e
-#define GDS_GWS_RESOURCE__UNUSED1__SHIFT                                                                      0x1f
-#define GDS_GWS_RESOURCE__FLAG_MASK                                                                           0x00000001L
-#define GDS_GWS_RESOURCE__COUNTER_MASK                                                                        0x00001FFEL
-#define GDS_GWS_RESOURCE__TYPE_MASK                                                                           0x00002000L
-#define GDS_GWS_RESOURCE__DED_MASK                                                                            0x00004000L
-#define GDS_GWS_RESOURCE__RELEASE_ALL_MASK                                                                    0x00008000L
-#define GDS_GWS_RESOURCE__HEAD_QUEUE_MASK                                                                     0x0FFF0000L
-#define GDS_GWS_RESOURCE__HEAD_VALID_MASK                                                                     0x10000000L
-#define GDS_GWS_RESOURCE__HEAD_FLAG_MASK                                                                      0x20000000L
-#define GDS_GWS_RESOURCE__HALTED_MASK                                                                         0x40000000L
-#define GDS_GWS_RESOURCE__UNUSED1_MASK                                                                        0x80000000L
-//GDS_GWS_RESOURCE_CNT
-#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT__SHIFT                                                             0x0
-#define GDS_GWS_RESOURCE_CNT__UNUSED__SHIFT                                                                   0x10
-#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT_MASK                                                               0x0000FFFFL
-#define GDS_GWS_RESOURCE_CNT__UNUSED_MASK                                                                     0xFFFF0000L
-//GDS_OA_CNTL
-#define GDS_OA_CNTL__INDEX__SHIFT                                                                             0x0
-#define GDS_OA_CNTL__UNUSED__SHIFT                                                                            0x4
-#define GDS_OA_CNTL__INDEX_MASK                                                                               0x0000000FL
-#define GDS_OA_CNTL__UNUSED_MASK                                                                              0xFFFFFFF0L
-//GDS_OA_COUNTER
-#define GDS_OA_COUNTER__SPACE_AVAILABLE__SHIFT                                                                0x0
-#define GDS_OA_COUNTER__SPACE_AVAILABLE_MASK                                                                  0xFFFFFFFFL
-//GDS_OA_ADDRESS
-#define GDS_OA_ADDRESS__DS_ADDRESS__SHIFT                                                                     0x0
-#define GDS_OA_ADDRESS__CRAWLER__SHIFT                                                                        0x10
-#define GDS_OA_ADDRESS__CRAWLER_TYPE__SHIFT                                                                   0x14
-#define GDS_OA_ADDRESS__UNUSED__SHIFT                                                                         0x16
-#define GDS_OA_ADDRESS__NO_ALLOC__SHIFT                                                                       0x1e
-#define GDS_OA_ADDRESS__ENABLE__SHIFT                                                                         0x1f
-#define GDS_OA_ADDRESS__DS_ADDRESS_MASK                                                                       0x0000FFFFL
-#define GDS_OA_ADDRESS__CRAWLER_MASK                                                                          0x000F0000L
-#define GDS_OA_ADDRESS__CRAWLER_TYPE_MASK                                                                     0x00300000L
-#define GDS_OA_ADDRESS__UNUSED_MASK                                                                           0x3FC00000L
-#define GDS_OA_ADDRESS__NO_ALLOC_MASK                                                                         0x40000000L
-#define GDS_OA_ADDRESS__ENABLE_MASK                                                                           0x80000000L
-//GDS_OA_INCDEC
-#define GDS_OA_INCDEC__VALUE__SHIFT                                                                           0x0
-#define GDS_OA_INCDEC__INCDEC__SHIFT                                                                          0x1f
-#define GDS_OA_INCDEC__VALUE_MASK                                                                             0x7FFFFFFFL
-#define GDS_OA_INCDEC__INCDEC_MASK                                                                            0x80000000L
-//GDS_OA_RING_SIZE
-#define GDS_OA_RING_SIZE__RING_SIZE__SHIFT                                                                    0x0
-#define GDS_OA_RING_SIZE__RING_SIZE_MASK                                                                      0xFFFFFFFFL
-//SPI_CONFIG_CNTL
-#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY__SHIFT                                                            0x0
-#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER__SHIFT                                                            0x15
-#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS__SHIFT                                                         0x18
-#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS__SHIFT                                                         0x19
-#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET__SHIFT                                                               0x1a
-#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL__SHIFT                                                              0x1b
-#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA__SHIFT                                                             0x1c
-#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA__SHIFT                                                               0x1d
-#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL__SHIFT                                                          0x1e
-#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY_MASK                                                              0x001FFFFFL
-#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER_MASK                                                              0x00E00000L
-#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS_MASK                                                           0x01000000L
-#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS_MASK                                                           0x02000000L
-#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET_MASK                                                                 0x04000000L
-#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL_MASK                                                                0x08000000L
-#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA_MASK                                                               0x10000000L
-#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA_MASK                                                                 0x20000000L
-#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL_MASK                                                            0xC0000000L
-//SPI_CONFIG_CNTL_1
-#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT                                                              0x0
-#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW__SHIFT                                                     0x4
-#define SPI_CONFIG_CNTL_1__BATON_RESET_DISABLE__SHIFT                                                         0x5
-#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE__SHIFT                                                             0x6
-#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT__SHIFT                                                             0x7
-#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE__SHIFT                                                   0x8
-#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE__SHIFT                                                            0x9
-#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT__SHIFT                                                             0xa
-#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE__SHIFT                                                        0xe
-#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE__SHIFT                                                        0xf
-#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE__SHIFT                                                               0x10
-#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY_MASK                                                                0x0000000FL
-#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW_MASK                                                       0x00000010L
-#define SPI_CONFIG_CNTL_1__BATON_RESET_DISABLE_MASK                                                           0x00000020L
-#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE_MASK                                                               0x00000040L
-#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT_MASK                                                               0x00000080L
-#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE_MASK                                                     0x00000100L
-#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE_MASK                                                              0x00000200L
-#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT_MASK                                                               0x00003C00L
-#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE_MASK                                                          0x00004000L
-#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE_MASK                                                          0x00008000L
-#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE_MASK                                                                 0xFFFF0000L
-//SPI_CONFIG_CNTL_2
-#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD__SHIFT                                    0x0
-#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD__SHIFT                                      0x4
-#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD_MASK                                      0x0000000FL
-#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD_MASK                                        0x000000F0L
-
-
-// addressBlock: gc_perfddec
-//CPG_PERFCOUNTER1_LO
-#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//CPG_PERFCOUNTER1_HI
-#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//CPG_PERFCOUNTER0_LO
-#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//CPG_PERFCOUNTER0_HI
-#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//CPC_PERFCOUNTER1_LO
-#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//CPC_PERFCOUNTER1_HI
-#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//CPC_PERFCOUNTER0_LO
-#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//CPC_PERFCOUNTER0_HI
-#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//CPF_PERFCOUNTER1_LO
-#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//CPF_PERFCOUNTER1_HI
-#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//CPF_PERFCOUNTER0_LO
-#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//CPF_PERFCOUNTER0_HI
-#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//CPF_LATENCY_STATS_DATA
-#define CPF_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
-#define CPF_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
-//CPG_LATENCY_STATS_DATA
-#define CPG_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
-#define CPG_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
-//CPC_LATENCY_STATS_DATA
-#define CPC_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
-#define CPC_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
-//GRBM_PERFCOUNTER0_LO
-#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
-#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
-//GRBM_PERFCOUNTER0_HI
-#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
-#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
-//GRBM_PERFCOUNTER1_LO
-#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
-#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
-//GRBM_PERFCOUNTER1_HI
-#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
-#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
-//GRBM_SE0_PERFCOUNTER_LO
-#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
-#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
-//GRBM_SE0_PERFCOUNTER_HI
-#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
-#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
-//GRBM_SE1_PERFCOUNTER_LO
-#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
-#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
-//GRBM_SE1_PERFCOUNTER_HI
-#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
-#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
-//GRBM_SE2_PERFCOUNTER_LO
-#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
-#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
-//GRBM_SE2_PERFCOUNTER_HI
-#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
-#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
-//GRBM_SE3_PERFCOUNTER_LO
-#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
-#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
-//GRBM_SE3_PERFCOUNTER_HI
-#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
-#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
-//WD_PERFCOUNTER0_LO
-#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//WD_PERFCOUNTER0_HI
-#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//WD_PERFCOUNTER1_LO
-#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//WD_PERFCOUNTER1_HI
-#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//WD_PERFCOUNTER2_LO
-#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//WD_PERFCOUNTER2_HI
-#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//WD_PERFCOUNTER3_LO
-#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//WD_PERFCOUNTER3_HI
-#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER0_LO
-#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER0_HI
-#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER1_LO
-#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER1_HI
-#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER2_LO
-#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER2_HI
-#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER3_LO
-#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//IA_PERFCOUNTER3_HI
-#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//VGT_PERFCOUNTER0_LO
-#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//VGT_PERFCOUNTER0_HI
-#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//VGT_PERFCOUNTER1_LO
-#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//VGT_PERFCOUNTER1_HI
-#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//VGT_PERFCOUNTER2_LO
-#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//VGT_PERFCOUNTER2_HI
-#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//VGT_PERFCOUNTER3_LO
-#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//VGT_PERFCOUNTER3_HI
-#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//PA_SU_PERFCOUNTER0_LO
-#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SU_PERFCOUNTER0_HI
-#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
-//PA_SU_PERFCOUNTER1_LO
-#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SU_PERFCOUNTER1_HI
-#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
-//PA_SU_PERFCOUNTER2_LO
-#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SU_PERFCOUNTER2_HI
-#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
-//PA_SU_PERFCOUNTER3_LO
-#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SU_PERFCOUNTER3_HI
-#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0x0000FFFFL
-//PA_SC_PERFCOUNTER0_LO
-#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER0_HI
-#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER1_LO
-#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER1_HI
-#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER2_LO
-#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER2_HI
-#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER3_LO
-#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER3_HI
-#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER4_LO
-#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER4_HI
-#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER5_LO
-#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER5_HI
-#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER6_LO
-#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER6_HI
-#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER7_LO
-#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
-//PA_SC_PERFCOUNTER7_HI
-#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
-//SPI_PERFCOUNTER0_HI
-#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER0_LO
-#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER1_HI
-#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER1_LO
-#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER2_HI
-#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER2_LO
-#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER3_HI
-#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER3_LO
-#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER4_HI
-#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER4_LO
-#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER5_HI
-#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SPI_PERFCOUNTER5_LO
-#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER0_LO
-#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER0_HI
-#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER1_LO
-#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER1_HI
-#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER2_LO
-#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER2_HI
-#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER3_LO
-#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER3_HI
-#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER4_LO
-#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER4_HI
-#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER5_LO
-#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER5_HI
-#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER6_LO
-#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER6_HI
-#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER7_LO
-#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER7_HI
-#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER8_LO
-#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER8_HI
-#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER9_LO
-#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER9_HI
-#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SQ_PERFCOUNTER10_LO
-#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER10_HI
-#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER11_LO
-#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER11_HI
-#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER12_LO
-#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER12_HI
-#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER13_LO
-#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER13_HI
-#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER14_LO
-#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER14_HI
-#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER15_LO
-#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//SQ_PERFCOUNTER15_HI
-#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//SX_PERFCOUNTER0_LO
-#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SX_PERFCOUNTER0_HI
-#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SX_PERFCOUNTER1_LO
-#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SX_PERFCOUNTER1_HI
-#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SX_PERFCOUNTER2_LO
-#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SX_PERFCOUNTER2_HI
-#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//SX_PERFCOUNTER3_LO
-#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//SX_PERFCOUNTER3_HI
-#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//GDS_PERFCOUNTER0_LO
-#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//GDS_PERFCOUNTER0_HI
-#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//GDS_PERFCOUNTER1_LO
-#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//GDS_PERFCOUNTER1_HI
-#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//GDS_PERFCOUNTER2_LO
-#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//GDS_PERFCOUNTER2_HI
-#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//GDS_PERFCOUNTER3_LO
-#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//GDS_PERFCOUNTER3_HI
-#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TA_PERFCOUNTER0_LO
-#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//TA_PERFCOUNTER0_HI
-#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//TA_PERFCOUNTER1_LO
-#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//TA_PERFCOUNTER1_HI
-#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//TD_PERFCOUNTER0_LO
-#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//TD_PERFCOUNTER0_HI
-#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//TD_PERFCOUNTER1_LO
-#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//TD_PERFCOUNTER1_HI
-#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//TCP_PERFCOUNTER0_LO
-#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCP_PERFCOUNTER0_HI
-#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCP_PERFCOUNTER1_LO
-#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCP_PERFCOUNTER1_HI
-#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCP_PERFCOUNTER2_LO
-#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCP_PERFCOUNTER2_HI
-#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCP_PERFCOUNTER3_LO
-#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCP_PERFCOUNTER3_HI
-#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER0_LO
-#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER0_HI
-#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER1_LO
-#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER1_HI
-#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER2_LO
-#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER2_HI
-#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER3_LO
-#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCC_PERFCOUNTER3_HI
-#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER0_LO
-#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER0_HI
-#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER1_LO
-#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER1_HI
-#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER2_LO
-#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER2_HI
-#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER3_LO
-#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//TCA_PERFCOUNTER3_HI
-#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//CB_PERFCOUNTER0_LO
-#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//CB_PERFCOUNTER0_HI
-#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//CB_PERFCOUNTER1_LO
-#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//CB_PERFCOUNTER1_HI
-#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//CB_PERFCOUNTER2_LO
-#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//CB_PERFCOUNTER2_HI
-#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//CB_PERFCOUNTER3_LO
-#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//CB_PERFCOUNTER3_HI
-#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER0_LO
-#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER0_HI
-#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER1_LO
-#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER1_HI
-#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER2_LO
-#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER2_HI
-#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER3_LO
-#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
-#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
-//DB_PERFCOUNTER3_HI
-#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
-#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
-//RLC_PERFCOUNTER0_LO
-#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//RLC_PERFCOUNTER0_HI
-#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//RLC_PERFCOUNTER1_LO
-#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//RLC_PERFCOUNTER1_HI
-#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER0_LO
-#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER0_HI
-#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER1_LO
-#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER1_HI
-#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER2_LO
-#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER2_HI
-#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER3_LO
-#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
-//RMI_PERFCOUNTER3_HI
-#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
-
-
-// addressBlock: gc_utcl2_atcl2pfcntrdec
-//ATC_L2_PERFCOUNTER_LO
-#define ATC_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                              0x0
-#define ATC_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                0xFFFFFFFFL
-//ATC_L2_PERFCOUNTER_HI
-#define ATC_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                              0x0
-#define ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                           0x10
-#define ATC_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                0x0000FFFFL
-#define ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                             0xFFFF0000L
-
-
-// addressBlock: gc_utcl2_vml2prdec
-//MC_VM_L2_PERFCOUNTER_LO
-#define MC_VM_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                              0xFFFFFFFFL
-//MC_VM_L2_PERFCOUNTER_HI
-#define MC_VM_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                         0x10
-#define MC_VM_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                              0x0000FFFFL
-#define MC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                           0xFFFF0000L
-
-
-// addressBlock: gc_perfsdec
-//CPG_PERFCOUNTER1_SELECT
-#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL0__SHIFT                                                             0x0
-#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL1__SHIFT                                                             0xa
-#define CPG_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE1__SHIFT                                                            0x18
-#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
-#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
-#define CPG_PERFCOUNTER1_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
-#define CPG_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
-#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
-//CPG_PERFCOUNTER0_SELECT1
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL2__SHIFT                                                            0x0
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL3__SHIFT                                                            0xa
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL2_MASK                                                              0x000003FFL
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_SEL3_MASK                                                              0x000FFC00L
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
-#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
-//CPG_PERFCOUNTER0_SELECT
-#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL0__SHIFT                                                             0x0
-#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL1__SHIFT                                                             0xa
-#define CPG_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
-#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
-#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
-#define CPG_PERFCOUNTER0_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
-#define CPG_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
-#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
-//CPC_PERFCOUNTER1_SELECT
-#define CPC_PERFCOUNTER1_SELECT__CNTR_SEL0__SHIFT                                                             0x0
-#define CPC_PERFCOUNTER1_SELECT__CNTR_SEL1__SHIFT                                                             0xa
-#define CPC_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE1__SHIFT                                                            0x18
-#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
-#define CPC_PERFCOUNTER1_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
-#define CPC_PERFCOUNTER1_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
-#define CPC_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
-#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
-//CPC_PERFCOUNTER0_SELECT1
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL2__SHIFT                                                            0x0
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL3__SHIFT                                                            0xa
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL2_MASK                                                              0x000003FFL
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_SEL3_MASK                                                              0x000FFC00L
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
-#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
-//CPF_PERFCOUNTER1_SELECT
-#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL0__SHIFT                                                             0x0
-#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL1__SHIFT                                                             0xa
-#define CPF_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE1__SHIFT                                                            0x18
-#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
-#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
-#define CPF_PERFCOUNTER1_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
-#define CPF_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
-#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
-//CPF_PERFCOUNTER0_SELECT1
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL2__SHIFT                                                            0x0
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL3__SHIFT                                                            0xa
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL2_MASK                                                              0x000003FFL
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_SEL3_MASK                                                              0x000FFC00L
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
-#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
-//CPF_PERFCOUNTER0_SELECT
-#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL0__SHIFT                                                             0x0
-#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL1__SHIFT                                                             0xa
-#define CPF_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
-#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
-#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
-#define CPF_PERFCOUNTER0_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
-#define CPF_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
-#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
-//CP_PERFMON_CNTL
-#define CP_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                 0x0
-#define CP_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT                                                             0x4
-#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT                                                           0x8
-#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                         0xa
-#define CP_PERFMON_CNTL__PERFMON_STATE_MASK                                                                   0x0000000FL
-#define CP_PERFMON_CNTL__SPM_PERFMON_STATE_MASK                                                               0x000000F0L
-#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK                                                             0x00000300L
-#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                           0x00000400L
-//CPC_PERFCOUNTER0_SELECT
-#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL0__SHIFT                                                             0x0
-#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL1__SHIFT                                                             0xa
-#define CPC_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
-#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
-#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL0_MASK                                                               0x000003FFL
-#define CPC_PERFCOUNTER0_SELECT__CNTR_SEL1_MASK                                                               0x000FFC00L
-#define CPC_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
-#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
-//CPF_TC_PERF_COUNTER_WINDOW_SELECT
-#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
-#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
-#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
-#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x00000007L
-#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
-#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
-//CPG_TC_PERF_COUNTER_WINDOW_SELECT
-#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
-#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
-#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
-#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x0000001FL
-#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
-#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
-//CPF_LATENCY_STATS_SELECT
-#define CPF_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
-#define CPF_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
-#define CPF_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
-#define CPF_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000000FL
-#define CPF_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
-#define CPF_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
-//CPG_LATENCY_STATS_SELECT
-#define CPG_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
-#define CPG_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
-#define CPG_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
-#define CPG_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000001FL
-#define CPG_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
-#define CPG_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
-//CPC_LATENCY_STATS_SELECT
-#define CPC_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
-#define CPC_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
-#define CPC_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
-#define CPC_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x00000007L
-#define CPC_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
-#define CPC_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
-//CP_DRAW_OBJECT
-#define CP_DRAW_OBJECT__OBJECT__SHIFT                                                                         0x0
-#define CP_DRAW_OBJECT__OBJECT_MASK                                                                           0xFFFFFFFFL
-//CP_DRAW_OBJECT_COUNTER
-#define CP_DRAW_OBJECT_COUNTER__COUNT__SHIFT                                                                  0x0
-#define CP_DRAW_OBJECT_COUNTER__COUNT_MASK                                                                    0x0000FFFFL
-//CP_DRAW_WINDOW_MASK_HI
-#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI__SHIFT                                                         0x0
-#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI_MASK                                                           0xFFFFFFFFL
-//CP_DRAW_WINDOW_HI
-#define CP_DRAW_WINDOW_HI__WINDOW_HI__SHIFT                                                                   0x0
-#define CP_DRAW_WINDOW_HI__WINDOW_HI_MASK                                                                     0xFFFFFFFFL
-//CP_DRAW_WINDOW_LO
-#define CP_DRAW_WINDOW_LO__MIN__SHIFT                                                                         0x0
-#define CP_DRAW_WINDOW_LO__MAX__SHIFT                                                                         0x10
-#define CP_DRAW_WINDOW_LO__MIN_MASK                                                                           0x0000FFFFL
-#define CP_DRAW_WINDOW_LO__MAX_MASK                                                                           0xFFFF0000L
-//CP_DRAW_WINDOW_CNTL
-#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX__SHIFT                                                0x0
-#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN__SHIFT                                                0x1
-#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI__SHIFT                                                    0x2
-#define CP_DRAW_WINDOW_CNTL__MODE__SHIFT                                                                      0x8
-#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX_MASK                                                  0x00000001L
-#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN_MASK                                                  0x00000002L
-#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI_MASK                                                      0x00000004L
-#define CP_DRAW_WINDOW_CNTL__MODE_MASK                                                                        0x00000100L
-//GRBM_PERFCOUNTER0_SELECT
-#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
-#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
-#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
-#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                           0xc
-#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
-#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
-#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x10
-#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x11
-#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x12
-#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
-#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
-#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
-#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
-#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x17
-#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT                                           0x18
-#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
-#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
-#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1b
-#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1c
-#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
-#define GRBM_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1e
-#define GRBM_PERFCOUNTER0_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1f
-#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x0000003FL
-#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
-#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
-#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                             0x00001000L
-#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
-#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
-#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x00010000L
-#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                              0x00020000L
-#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                              0x00040000L
-#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
-#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
-#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
-#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
-#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK                                              0x00800000L
-#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK                                             0x01000000L
-#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
-#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
-#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK                                              0x08000000L
-#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK                                              0x10000000L
-#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
-#define GRBM_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                              0x40000000L
-#define GRBM_PERFCOUNTER0_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                             0x80000000L
-//GRBM_PERFCOUNTER1_SELECT
-#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
-#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
-#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
-#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                           0xc
-#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
-#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
-#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x10
-#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x11
-#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x12
-#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
-#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
-#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
-#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
-#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x17
-#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT                                           0x18
-#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
-#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
-#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1b
-#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1c
-#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
-#define GRBM_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1e
-#define GRBM_PERFCOUNTER1_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1f
-#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x0000003FL
-#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
-#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
-#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                             0x00001000L
-#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
-#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
-#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x00010000L
-#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                              0x00020000L
-#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                              0x00040000L
-#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
-#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
-#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
-#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
-#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK                                              0x00800000L
-#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK                                             0x01000000L
-#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
-#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
-#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK                                              0x08000000L
-#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK                                              0x10000000L
-#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
-#define GRBM_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                              0x40000000L
-#define GRBM_PERFCOUNTER1_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                             0x80000000L
-//GRBM_SE0_PERFCOUNTER_SELECT
-#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
-#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
-#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
-#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
-#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
-#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
-#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
-#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
-#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
-#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
-#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
-#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
-#define GRBM_SE0_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
-#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
-#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
-#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
-#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
-#define GRBM_SE0_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
-//GRBM_SE1_PERFCOUNTER_SELECT
-#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
-#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
-#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
-#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
-#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
-#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
-#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
-#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
-#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
-#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
-#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
-#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
-#define GRBM_SE1_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
-#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
-#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
-#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
-#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
-#define GRBM_SE1_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
-//GRBM_SE2_PERFCOUNTER_SELECT
-#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
-#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
-#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
-#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
-#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
-#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
-#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
-#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
-#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
-#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
-#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
-#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
-#define GRBM_SE2_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
-#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
-#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
-#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
-#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
-#define GRBM_SE2_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
-//GRBM_SE3_PERFCOUNTER_SELECT
-#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
-#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
-#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
-#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
-#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
-#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
-#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
-#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
-#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
-#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT                                        0x13
-#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
-#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
-#define GRBM_SE3_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
-#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
-#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
-#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
-#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK                                          0x00080000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
-#define GRBM_SE3_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
-//WD_PERFCOUNTER0_SELECT
-#define WD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define WD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define WD_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define WD_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//WD_PERFCOUNTER1_SELECT
-#define WD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define WD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define WD_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define WD_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//WD_PERFCOUNTER2_SELECT
-#define WD_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define WD_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define WD_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define WD_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//WD_PERFCOUNTER3_SELECT
-#define WD_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define WD_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define WD_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define WD_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//IA_PERFCOUNTER0_SELECT
-#define IA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define IA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define IA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define IA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define IA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define IA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
-#define IA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
-#define IA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define IA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define IA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//IA_PERFCOUNTER1_SELECT
-#define IA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define IA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define IA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define IA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//IA_PERFCOUNTER2_SELECT
-#define IA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define IA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define IA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define IA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//IA_PERFCOUNTER3_SELECT
-#define IA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define IA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define IA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define IA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//IA_PERFCOUNTER0_SELECT1
-#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
-#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
-#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
-#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
-#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
-#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
-#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
-#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
-//VGT_PERFCOUNTER0_SELECT
-#define VGT_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define VGT_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define VGT_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define VGT_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//VGT_PERFCOUNTER1_SELECT
-#define VGT_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define VGT_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define VGT_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define VGT_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//VGT_PERFCOUNTER2_SELECT
-#define VGT_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define VGT_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define VGT_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000000FFL
-#define VGT_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//VGT_PERFCOUNTER3_SELECT
-#define VGT_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define VGT_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define VGT_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000000FFL
-#define VGT_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//VGT_PERFCOUNTER0_SELECT1
-#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//VGT_PERFCOUNTER1_SELECT1
-#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//VGT_PERFCOUNTER_SEID_MASK
-#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK__SHIFT                                               0x0
-#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK_MASK                                                 0x000000FFL
-//PA_SU_PERFCOUNTER0_SELECT
-#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
-#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
-#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
-#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
-//PA_SU_PERFCOUNTER0_SELECT1
-#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
-#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
-#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
-//PA_SU_PERFCOUNTER1_SELECT
-#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
-#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
-#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
-#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
-//PA_SU_PERFCOUNTER1_SELECT1
-#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
-#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
-#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
-#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
-//PA_SU_PERFCOUNTER2_SELECT
-#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
-#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
-//PA_SU_PERFCOUNTER3_SELECT
-#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
-#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
-//PA_SC_PERFCOUNTER0_SELECT
-#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
-#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
-#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
-#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
-//PA_SC_PERFCOUNTER0_SELECT1
-#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
-#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
-#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
-#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
-//PA_SC_PERFCOUNTER1_SELECT
-#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-//PA_SC_PERFCOUNTER2_SELECT
-#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-//PA_SC_PERFCOUNTER3_SELECT
-#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-//PA_SC_PERFCOUNTER4_SELECT
-#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-//PA_SC_PERFCOUNTER5_SELECT
-#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-//PA_SC_PERFCOUNTER6_SELECT
-#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-//PA_SC_PERFCOUNTER7_SELECT
-#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                            0x0
-#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                              0x000003FFL
-//SPI_PERFCOUNTER0_SELECT
-#define SPI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define SPI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SPI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define SPI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SPI_PERFCOUNTER1_SELECT
-#define SPI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define SPI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SPI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define SPI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SPI_PERFCOUNTER2_SELECT
-#define SPI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define SPI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SPI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define SPI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SPI_PERFCOUNTER3_SELECT
-#define SPI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define SPI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SPI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define SPI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SPI_PERFCOUNTER0_SELECT1
-#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//SPI_PERFCOUNTER1_SELECT1
-#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//SPI_PERFCOUNTER2_SELECT1
-#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//SPI_PERFCOUNTER3_SELECT1
-#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//SPI_PERFCOUNTER4_SELECT
-#define SPI_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SPI_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                0x000000FFL
-//SPI_PERFCOUNTER5_SELECT
-#define SPI_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SPI_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                0x000000FFL
-//SPI_PERFCOUNTER_BINS
-#define SPI_PERFCOUNTER_BINS__BIN0_MIN__SHIFT                                                                 0x0
-#define SPI_PERFCOUNTER_BINS__BIN0_MAX__SHIFT                                                                 0x4
-#define SPI_PERFCOUNTER_BINS__BIN1_MIN__SHIFT                                                                 0x8
-#define SPI_PERFCOUNTER_BINS__BIN1_MAX__SHIFT                                                                 0xc
-#define SPI_PERFCOUNTER_BINS__BIN2_MIN__SHIFT                                                                 0x10
-#define SPI_PERFCOUNTER_BINS__BIN2_MAX__SHIFT                                                                 0x14
-#define SPI_PERFCOUNTER_BINS__BIN3_MIN__SHIFT                                                                 0x18
-#define SPI_PERFCOUNTER_BINS__BIN3_MAX__SHIFT                                                                 0x1c
-#define SPI_PERFCOUNTER_BINS__BIN0_MIN_MASK                                                                   0x0000000FL
-#define SPI_PERFCOUNTER_BINS__BIN0_MAX_MASK                                                                   0x000000F0L
-#define SPI_PERFCOUNTER_BINS__BIN1_MIN_MASK                                                                   0x00000F00L
-#define SPI_PERFCOUNTER_BINS__BIN1_MAX_MASK                                                                   0x0000F000L
-#define SPI_PERFCOUNTER_BINS__BIN2_MIN_MASK                                                                   0x000F0000L
-#define SPI_PERFCOUNTER_BINS__BIN2_MAX_MASK                                                                   0x00F00000L
-#define SPI_PERFCOUNTER_BINS__BIN3_MIN_MASK                                                                   0x0F000000L
-#define SPI_PERFCOUNTER_BINS__BIN3_MAX_MASK                                                                   0xF0000000L
-//SQ_PERFCOUNTER0_SELECT
-#define SQ_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER1_SELECT
-#define SQ_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER2_SELECT
-#define SQ_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER3_SELECT
-#define SQ_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER4_SELECT
-#define SQ_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER5_SELECT
-#define SQ_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER6_SELECT
-#define SQ_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER6_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER6_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER7_SELECT
-#define SQ_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER7_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER7_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER8_SELECT
-#define SQ_PERFCOUNTER8_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER8_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER8_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER8_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER8_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER8_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER9_SELECT
-#define SQ_PERFCOUNTER9_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK__SHIFT                                                          0xc
-#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK__SHIFT                                                        0x10
-#define SQ_PERFCOUNTER9_SELECT__SPM_MODE__SHIFT                                                               0x14
-#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK__SHIFT                                                              0x18
-#define SQ_PERFCOUNTER9_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define SQ_PERFCOUNTER9_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK_MASK                                                            0x0000F000L
-#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK_MASK                                                          0x000F0000L
-#define SQ_PERFCOUNTER9_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
-#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK_MASK                                                                0x0F000000L
-#define SQ_PERFCOUNTER9_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//SQ_PERFCOUNTER10_SELECT
-#define SQ_PERFCOUNTER10_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
-#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
-#define SQ_PERFCOUNTER10_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK__SHIFT                                                             0x18
-#define SQ_PERFCOUNTER10_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SQ_PERFCOUNTER10_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
-#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
-#define SQ_PERFCOUNTER10_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
-#define SQ_PERFCOUNTER10_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SQ_PERFCOUNTER11_SELECT
-#define SQ_PERFCOUNTER11_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
-#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
-#define SQ_PERFCOUNTER11_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK__SHIFT                                                             0x18
-#define SQ_PERFCOUNTER11_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SQ_PERFCOUNTER11_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
-#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
-#define SQ_PERFCOUNTER11_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
-#define SQ_PERFCOUNTER11_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SQ_PERFCOUNTER12_SELECT
-#define SQ_PERFCOUNTER12_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
-#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
-#define SQ_PERFCOUNTER12_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK__SHIFT                                                             0x18
-#define SQ_PERFCOUNTER12_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SQ_PERFCOUNTER12_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
-#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
-#define SQ_PERFCOUNTER12_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
-#define SQ_PERFCOUNTER12_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SQ_PERFCOUNTER13_SELECT
-#define SQ_PERFCOUNTER13_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
-#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
-#define SQ_PERFCOUNTER13_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK__SHIFT                                                             0x18
-#define SQ_PERFCOUNTER13_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SQ_PERFCOUNTER13_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
-#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
-#define SQ_PERFCOUNTER13_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
-#define SQ_PERFCOUNTER13_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SQ_PERFCOUNTER14_SELECT
-#define SQ_PERFCOUNTER14_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
-#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
-#define SQ_PERFCOUNTER14_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK__SHIFT                                                             0x18
-#define SQ_PERFCOUNTER14_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SQ_PERFCOUNTER14_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
-#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
-#define SQ_PERFCOUNTER14_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
-#define SQ_PERFCOUNTER14_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SQ_PERFCOUNTER15_SELECT
-#define SQ_PERFCOUNTER15_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK__SHIFT                                                         0xc
-#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK__SHIFT                                                       0x10
-#define SQ_PERFCOUNTER15_SELECT__SPM_MODE__SHIFT                                                              0x14
-#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK__SHIFT                                                             0x18
-#define SQ_PERFCOUNTER15_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define SQ_PERFCOUNTER15_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK_MASK                                                           0x0000F000L
-#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK_MASK                                                         0x000F0000L
-#define SQ_PERFCOUNTER15_SELECT__SPM_MODE_MASK                                                                0x00F00000L
-#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK_MASK                                                               0x0F000000L
-#define SQ_PERFCOUNTER15_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//SQ_PERFCOUNTER_CTRL
-#define SQ_PERFCOUNTER_CTRL__PS_EN__SHIFT                                                                     0x0
-#define SQ_PERFCOUNTER_CTRL__VS_EN__SHIFT                                                                     0x1
-#define SQ_PERFCOUNTER_CTRL__GS_EN__SHIFT                                                                     0x2
-#define SQ_PERFCOUNTER_CTRL__ES_EN__SHIFT                                                                     0x3
-#define SQ_PERFCOUNTER_CTRL__HS_EN__SHIFT                                                                     0x4
-#define SQ_PERFCOUNTER_CTRL__LS_EN__SHIFT                                                                     0x5
-#define SQ_PERFCOUNTER_CTRL__CS_EN__SHIFT                                                                     0x6
-#define SQ_PERFCOUNTER_CTRL__CNTR_RATE__SHIFT                                                                 0x8
-#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH__SHIFT                                                             0xd
-#define SQ_PERFCOUNTER_CTRL__PS_EN_MASK                                                                       0x00000001L
-#define SQ_PERFCOUNTER_CTRL__VS_EN_MASK                                                                       0x00000002L
-#define SQ_PERFCOUNTER_CTRL__GS_EN_MASK                                                                       0x00000004L
-#define SQ_PERFCOUNTER_CTRL__ES_EN_MASK                                                                       0x00000008L
-#define SQ_PERFCOUNTER_CTRL__HS_EN_MASK                                                                       0x00000010L
-#define SQ_PERFCOUNTER_CTRL__LS_EN_MASK                                                                       0x00000020L
-#define SQ_PERFCOUNTER_CTRL__CS_EN_MASK                                                                       0x00000040L
-#define SQ_PERFCOUNTER_CTRL__CNTR_RATE_MASK                                                                   0x00001F00L
-#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH_MASK                                                               0x00002000L
-//SQ_PERFCOUNTER_MASK
-#define SQ_PERFCOUNTER_MASK__SH0_MASK__SHIFT                                                                  0x0
-#define SQ_PERFCOUNTER_MASK__SH1_MASK__SHIFT                                                                  0x10
-#define SQ_PERFCOUNTER_MASK__SH0_MASK_MASK                                                                    0x0000FFFFL
-#define SQ_PERFCOUNTER_MASK__SH1_MASK_MASK                                                                    0xFFFF0000L
-//SQ_PERFCOUNTER_CTRL2
-#define SQ_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT                                                                 0x0
-#define SQ_PERFCOUNTER_CTRL2__FORCE_EN_MASK                                                                   0x00000001L
-//SX_PERFCOUNTER0_SELECT
-#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT                                                     0x0
-#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                    0xa
-#define SX_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK                                                       0x000003FFL
-#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK                                                      0x000FFC00L
-#define SX_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-//SX_PERFCOUNTER1_SELECT
-#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT                                                     0x0
-#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                    0xa
-#define SX_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK                                                       0x000003FFL
-#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK                                                      0x000FFC00L
-#define SX_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-//SX_PERFCOUNTER2_SELECT
-#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT                                                     0x0
-#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                    0xa
-#define SX_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK                                                       0x000003FFL
-#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK                                                      0x000FFC00L
-#define SX_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-//SX_PERFCOUNTER3_SELECT
-#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT                                                     0x0
-#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                    0xa
-#define SX_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK                                                       0x000003FFL
-#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK                                                      0x000FFC00L
-#define SX_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-//SX_PERFCOUNTER0_SELECT1
-#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT                                                   0x0
-#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT                                                   0xa
-#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK                                                     0x000003FFL
-#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK                                                     0x000FFC00L
-//SX_PERFCOUNTER1_SELECT1
-#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2__SHIFT                                                   0x0
-#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3__SHIFT                                                   0xa
-#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2_MASK                                                     0x000003FFL
-#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3_MASK                                                     0x000FFC00L
-//GDS_PERFCOUNTER0_SELECT
-#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
-#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                   0xa
-#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000003FFL
-#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK                                                     0x000FFC00L
-#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-//GDS_PERFCOUNTER1_SELECT
-#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
-#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                   0xa
-#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000003FFL
-#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK                                                     0x000FFC00L
-#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-//GDS_PERFCOUNTER2_SELECT
-#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
-#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                   0xa
-#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000003FFL
-#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK                                                     0x000FFC00L
-#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-//GDS_PERFCOUNTER3_SELECT
-#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
-#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT                                                   0xa
-#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000003FFL
-#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK                                                     0x000FFC00L
-#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-//GDS_PERFCOUNTER0_SELECT1
-#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT                                                  0x0
-#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT                                                  0xa
-#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK                                                    0x000003FFL
-#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK                                                    0x000FFC00L
-//TA_PERFCOUNTER0_SELECT
-#define TA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define TA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define TA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define TA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define TA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define TA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define TA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x0003FC00L
-#define TA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define TA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define TA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//TA_PERFCOUNTER0_SELECT1
-#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
-#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
-#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
-#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
-#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000000FFL
-#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x0003FC00L
-#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
-#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
-//TA_PERFCOUNTER1_SELECT
-#define TA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define TA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define TA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define TA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define TA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define TA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define TA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x0003FC00L
-#define TA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define TA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define TA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//TD_PERFCOUNTER0_SELECT
-#define TD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define TD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define TD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define TD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define TD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define TD_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define TD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x0003FC00L
-#define TD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define TD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define TD_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//TD_PERFCOUNTER0_SELECT1
-#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
-#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
-#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
-#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
-#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000000FFL
-#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x0003FC00L
-#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
-#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
-//TD_PERFCOUNTER1_SELECT
-#define TD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define TD_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define TD_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define TD_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define TD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define TD_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000000FFL
-#define TD_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x0003FC00L
-#define TD_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define TD_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define TD_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//TCP_PERFCOUNTER0_SELECT
-#define TCP_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define TCP_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCP_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define TCP_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCP_PERFCOUNTER0_SELECT1
-#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//TCP_PERFCOUNTER1_SELECT
-#define TCP_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define TCP_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCP_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define TCP_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCP_PERFCOUNTER1_SELECT1
-#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//TCP_PERFCOUNTER2_SELECT
-#define TCP_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCP_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCP_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCP_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCP_PERFCOUNTER3_SELECT
-#define TCP_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCP_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCP_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCP_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCC_PERFCOUNTER0_SELECT
-#define TCC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define TCC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define TCC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCC_PERFCOUNTER0_SELECT1
-#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x18
-#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
-#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
-#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
-//TCC_PERFCOUNTER1_SELECT
-#define TCC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define TCC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define TCC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCC_PERFCOUNTER1_SELECT1
-#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x18
-#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
-#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
-#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
-//TCC_PERFCOUNTER2_SELECT
-#define TCC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCC_PERFCOUNTER3_SELECT
-#define TCC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCA_PERFCOUNTER0_SELECT
-#define TCA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define TCA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define TCA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCA_PERFCOUNTER0_SELECT1
-#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x18
-#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
-#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
-#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
-//TCA_PERFCOUNTER1_SELECT
-#define TCA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define TCA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
-#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define TCA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCA_PERFCOUNTER1_SELECT1
-#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x18
-#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
-#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
-#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
-#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
-#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
-//TCA_PERFCOUNTER2_SELECT
-#define TCA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//TCA_PERFCOUNTER3_SELECT
-#define TCA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define TCA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define TCA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
-#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define TCA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//CB_PERFCOUNTER_FILTER
-#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE__SHIFT                                                        0x0
-#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL__SHIFT                                                           0x1
-#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE__SHIFT                                                    0x4
-#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL__SHIFT                                                       0x5
-#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE__SHIFT                                                     0xa
-#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL__SHIFT                                                        0xb
-#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE__SHIFT                                                       0xc
-#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL__SHIFT                                                          0xd
-#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE__SHIFT                                               0x11
-#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL__SHIFT                                                  0x12
-#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE__SHIFT                                             0x15
-#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL__SHIFT                                                0x16
-#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE_MASK                                                          0x00000001L
-#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL_MASK                                                             0x0000000EL
-#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE_MASK                                                      0x00000010L
-#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL_MASK                                                         0x000003E0L
-#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE_MASK                                                       0x00000400L
-#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL_MASK                                                          0x00000800L
-#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE_MASK                                                         0x00001000L
-#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL_MASK                                                            0x0000E000L
-#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE_MASK                                                 0x00020000L
-#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL_MASK                                                    0x001C0000L
-#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE_MASK                                               0x00200000L
-#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL_MASK                                                  0x00C00000L
-//CB_PERFCOUNTER0_SELECT
-#define CB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define CB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define CB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define CB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define CB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define CB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define CB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x0007FC00L
-#define CB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define CB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define CB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//CB_PERFCOUNTER0_SELECT1
-#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
-#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
-#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
-#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
-#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000001FFL
-#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x0007FC00L
-#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
-#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
-//CB_PERFCOUNTER1_SELECT
-#define CB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define CB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define CB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define CB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//CB_PERFCOUNTER2_SELECT
-#define CB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define CB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define CB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define CB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//CB_PERFCOUNTER3_SELECT
-#define CB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define CB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define CB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
-#define CB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//DB_PERFCOUNTER0_SELECT
-#define DB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define DB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define DB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define DB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define DB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define DB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
-#define DB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
-#define DB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define DB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define DB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//DB_PERFCOUNTER0_SELECT1
-#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
-#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
-#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
-#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
-#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
-#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
-#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
-#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
-//DB_PERFCOUNTER1_SELECT
-#define DB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define DB_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define DB_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define DB_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define DB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define DB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
-#define DB_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
-#define DB_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define DB_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define DB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//DB_PERFCOUNTER1_SELECT1
-#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
-#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
-#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
-#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
-#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
-#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
-#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
-#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
-//DB_PERFCOUNTER2_SELECT
-#define DB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define DB_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define DB_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define DB_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define DB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define DB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
-#define DB_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
-#define DB_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define DB_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define DB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//DB_PERFCOUNTER3_SELECT
-#define DB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
-#define DB_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                              0xa
-#define DB_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
-#define DB_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                             0x18
-#define DB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
-#define DB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
-#define DB_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
-#define DB_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
-#define DB_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
-#define DB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
-//RLC_SPM_PERFMON_CNTL
-#define RLC_SPM_PERFMON_CNTL__RESERVED1__SHIFT                                                                0x2
-#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE__SHIFT                                                        0xc
-#define RLC_SPM_PERFMON_CNTL__RESERVED__SHIFT                                                                 0xe
-#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL__SHIFT                                                  0x10
-#define RLC_SPM_PERFMON_CNTL__RESERVED1_MASK                                                                  0x00000FFCL
-#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE_MASK                                                          0x00003000L
-#define RLC_SPM_PERFMON_CNTL__RESERVED_MASK                                                                   0x0000C000L
-#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_MASK                                                    0xFFFF0000L
-//RLC_SPM_PERFMON_RING_BASE_LO
-#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO__SHIFT                                                     0x0
-#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO_MASK                                                       0xFFFFFFFFL
-//RLC_SPM_PERFMON_RING_BASE_HI
-#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI__SHIFT                                                     0x0
-#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED__SHIFT                                                         0x10
-#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI_MASK                                                       0x0000FFFFL
-#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED_MASK                                                           0xFFFF0000L
-//RLC_SPM_PERFMON_RING_SIZE
-#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE__SHIFT                                                      0x0
-#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE_MASK                                                        0xFFFFFFFFL
-//RLC_SPM_PERFMON_SEGMENT_SIZE
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE__SHIFT                                             0x0
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1__SHIFT                                                        0x8
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE__SHIFT                                                  0xb
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE__SHIFT                                                     0x10
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE__SHIFT                                                     0x15
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE__SHIFT                                                     0x1a
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED__SHIFT                                                         0x1f
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE_MASK                                               0x000000FFL
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1_MASK                                                          0x00000700L
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE_MASK                                                    0x0000F800L
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE_MASK                                                       0x001F0000L
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE_MASK                                                       0x03E00000L
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE_MASK                                                       0x7C000000L
-#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED_MASK                                                           0x80000000L
-//RLC_SPM_SE_MUXSEL_ADDR
-#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT                                                       0x0
-#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK                                                         0xFFFFFFFFL
-//RLC_SPM_SE_MUXSEL_DATA
-#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT                                                       0x0
-#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA_MASK                                                         0xFFFFFFFFL
-//RLC_SPM_CPG_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_CPC_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_CPF_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_CB_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_DB_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_PA_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_GDS_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_IA_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_SC_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_TCC_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_TCA_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_TCP_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_TA_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_TD_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_VGT_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_SPI_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_SQG_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_SPM_SX_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                          0x0
-#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                      0x8
-#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                            0x000000FFL
-#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                        0xFFFFFF00L
-//RLC_SPM_GLOBAL_MUXSEL_ADDR
-#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT                                                   0x0
-#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK                                                     0xFFFFFFFFL
-//RLC_SPM_GLOBAL_MUXSEL_DATA
-#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT                                                   0x0
-#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA_MASK                                                     0xFFFFFFFFL
-//RLC_SPM_RING_RDPTR
-#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR__SHIFT                                                         0x0
-#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR_MASK                                                           0xFFFFFFFFL
-//RLC_SPM_SEGMENT_THRESHOLD
-#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD__SHIFT                                               0x0
-#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD_MASK                                                 0xFFFFFFFFL
-//RLC_SPM_RMI_PERFMON_SAMPLE_DELAY
-#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT                                         0x0
-#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT                                                     0x8
-#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK                                           0x000000FFL
-#define RLC_SPM_RMI_PERFMON_SAMPLE_DELAY__RESERVED_MASK                                                       0xFFFFFF00L
-//RLC_PERFMON_CLK_CNTL
-#define RLC_PERFMON_CLK_CNTL__PERFMON_CLOCK_STATE__SHIFT                                                      0x0
-#define RLC_PERFMON_CLK_CNTL__PERFMON_CLOCK_STATE_MASK                                                        0x00000001L
-//RLC_PERFMON_CNTL
-#define RLC_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                0x0
-#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                        0xa
-#define RLC_PERFMON_CNTL__PERFMON_STATE_MASK                                                                  0x00000007L
-#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                          0x00000400L
-//RLC_PERFCOUNTER0_SELECT
-#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
-#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x00FFL
-//RLC_PERFCOUNTER1_SELECT
-#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
-#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x00FFL
-//RLC_GPU_IOV_PERF_CNT_CNTL
-#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE__SHIFT                                                              0x0
-#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT__SHIFT                                                         0x1
-#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET__SHIFT                                                               0x2
-#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED__SHIFT                                                            0x3
-#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE_MASK                                                                0x00000001L
-#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT_MASK                                                           0x00000002L
-#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET_MASK                                                                 0x00000004L
-#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED_MASK                                                              0xFFFFFFF8L
-//RLC_GPU_IOV_PERF_CNT_WR_ADDR
-#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID__SHIFT                                                             0x0
-#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID__SHIFT                                                           0x4
-#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED__SHIFT                                                         0x6
-#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID_MASK                                                               0x0000000FL
-#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID_MASK                                                             0x00000030L
-#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED_MASK                                                           0xFFFFFFC0L
-//RLC_GPU_IOV_PERF_CNT_WR_DATA
-#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA__SHIFT                                                             0x0
-#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA_MASK                                                               0x0000000FL
-//RLC_GPU_IOV_PERF_CNT_RD_ADDR
-#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID__SHIFT                                                             0x0
-#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID__SHIFT                                                           0x4
-#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED__SHIFT                                                         0x6
-#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID_MASK                                                               0x0000000FL
-#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID_MASK                                                             0x00000030L
-#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED_MASK                                                           0xFFFFFFC0L
-//RLC_GPU_IOV_PERF_CNT_RD_DATA
-#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA__SHIFT                                                             0x0
-#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA_MASK                                                               0x0000000FL
-//RMI_PERFCOUNTER0_SELECT
-#define RMI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define RMI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define RMI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x0007FC00L
-#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define RMI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//RMI_PERFCOUNTER0_SELECT1
-#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000001FFL
-#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x0007FC00L
-#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//RMI_PERFCOUNTER1_SELECT
-#define RMI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define RMI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define RMI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define RMI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//RMI_PERFCOUNTER2_SELECT
-#define RMI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
-#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
-#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
-#define RMI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define RMI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x0007FC00L
-#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
-#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
-#define RMI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//RMI_PERFCOUNTER2_SELECT1
-#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
-#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
-#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
-#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
-#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000001FFL
-#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x0007FC00L
-#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
-#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
-//RMI_PERFCOUNTER3_SELECT
-#define RMI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
-#define RMI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
-#define RMI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000001FFL
-#define RMI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
-//RMI_PERF_COUNTER_CNTL
-#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL__SHIFT                                                 0x0
-#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL__SHIFT                                                 0x2
-#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL__SHIFT                                                          0x4
-#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0__SHIFT                                                 0x6
-#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1__SHIFT                                                 0x8
-#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID__SHIFT                                                        0xa
-#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID__SHIFT                                                       0xe
-#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD__SHIFT                                     0x13
-#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET__SHIFT                                                         0x19
-#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL__SHIFT                                                       0x1a
-#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL_MASK                                                   0x00000003L
-#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL_MASK                                                   0x0000000CL
-#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL_MASK                                                            0x00000030L
-#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0_MASK                                                   0x000000C0L
-#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1_MASK                                                   0x00000300L
-#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID_MASK                                                          0x00003C00L
-#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID_MASK                                                         0x0007C000L
-#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD_MASK                                       0x01F80000L
-#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET_MASK                                                           0x02000000L
-#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL_MASK                                                         0x04000000L
-
-
-// addressBlock: gc_utcl2_atcl2pfcntldec
-//ATC_L2_PERFCOUNTER0_CFG
-#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                              0x0
-#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                          0x8
-#define ATC_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                             0x18
-#define ATC_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                0x1c
-#define ATC_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                 0x1d
-#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                0x000000FFL
-#define ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                            0x0000FF00L
-#define ATC_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                               0x0F000000L
-#define ATC_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                  0x10000000L
-#define ATC_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                   0x20000000L
-//ATC_L2_PERFCOUNTER1_CFG
-#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                              0x0
-#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                          0x8
-#define ATC_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                             0x18
-#define ATC_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                0x1c
-#define ATC_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                 0x1d
-#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                0x000000FFL
-#define ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                            0x0000FF00L
-#define ATC_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                               0x0F000000L
-#define ATC_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                  0x10000000L
-#define ATC_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                   0x20000000L
-//ATC_L2_PERFCOUNTER_RSLT_CNTL
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                              0x0
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                    0x8
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                     0x10
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                       0x18
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                        0x19
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                             0x1a
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                0x0000000FL
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                      0x0000FF00L
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                       0x00FF0000L
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                         0x01000000L
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                          0x02000000L
-#define ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                               0x04000000L
-
-
-// addressBlock: gc_utcl2_vml2pldec
-//MC_VM_L2_PERFCOUNTER0_CFG
-#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER1_CFG
-#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER2_CFG
-#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER2_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER3_CFG
-#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER3_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER3_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER4_CFG
-#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER4_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER4_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER5_CFG
-#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER5_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER5_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER6_CFG
-#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER6_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER6_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER7_CFG
-#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                            0x0
-#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                        0x8
-#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                           0x18
-#define MC_VM_L2_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                              0x1c
-#define MC_VM_L2_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                               0x1d
-#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                              0x000000FFL
-#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                          0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                             0x0F000000L
-#define MC_VM_L2_PERFCOUNTER7_CFG__ENABLE_MASK                                                                0x10000000L
-#define MC_VM_L2_PERFCOUNTER7_CFG__CLEAR_MASK                                                                 0x20000000L
-//MC_VM_L2_PERFCOUNTER_RSLT_CNTL
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                            0x0
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                  0x8
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                   0x10
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                     0x18
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                      0x19
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                           0x1a
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                              0x0000000FL
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                    0x0000FF00L
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                     0x00FF0000L
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                       0x01000000L
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                        0x02000000L
-#define MC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                             0x04000000L
-
-
-// addressBlock: gc_rlcpdec
-//RLC_CNTL
-#define RLC_CNTL__RLC_ENABLE_F32__SHIFT                                                                       0x0
-#define RLC_CNTL__FORCE_RETRY__SHIFT                                                                          0x1
-#define RLC_CNTL__READ_CACHE_DISABLE__SHIFT                                                                   0x2
-#define RLC_CNTL__RLC_STEP_F32__SHIFT                                                                         0x3
-#define RLC_CNTL__RESERVED__SHIFT                                                                             0x4
-#define RLC_CNTL__RLC_ENABLE_F32_MASK                                                                         0x00000001L
-#define RLC_CNTL__FORCE_RETRY_MASK                                                                            0x00000002L
-#define RLC_CNTL__READ_CACHE_DISABLE_MASK                                                                     0x00000004L
-#define RLC_CNTL__RLC_STEP_F32_MASK                                                                           0x00000008L
-#define RLC_CNTL__RESERVED_MASK                                                                               0xFFFFFFF0L
-//RLC_STAT
-#define RLC_STAT__RLC_BUSY__SHIFT                                                                             0x0
-#define RLC_STAT__RLC_GPM_BUSY__SHIFT                                                                         0x1
-#define RLC_STAT__RLC_SPM_BUSY__SHIFT                                                                         0x2
-#define RLC_STAT__RLC_SRM_BUSY__SHIFT                                                                         0x3
-#define RLC_STAT__MC_BUSY__SHIFT                                                                              0x4
-#define RLC_STAT__RLC_THREAD_0_BUSY__SHIFT                                                                    0x5
-#define RLC_STAT__RLC_THREAD_1_BUSY__SHIFT                                                                    0x6
-#define RLC_STAT__RLC_THREAD_2_BUSY__SHIFT                                                                    0x7
-#define RLC_STAT__RESERVED__SHIFT                                                                             0x8
-#define RLC_STAT__RLC_BUSY_MASK                                                                               0x00000001L
-#define RLC_STAT__RLC_GPM_BUSY_MASK                                                                           0x00000002L
-#define RLC_STAT__RLC_SPM_BUSY_MASK                                                                           0x00000004L
-#define RLC_STAT__RLC_SRM_BUSY_MASK                                                                           0x00000008L
-#define RLC_STAT__MC_BUSY_MASK                                                                                0x00000010L
-#define RLC_STAT__RLC_THREAD_0_BUSY_MASK                                                                      0x00000020L
-#define RLC_STAT__RLC_THREAD_1_BUSY_MASK                                                                      0x00000040L
-#define RLC_STAT__RLC_THREAD_2_BUSY_MASK                                                                      0x00000080L
-#define RLC_STAT__RESERVED_MASK                                                                               0xFFFFFF00L
-//RLC_SAFE_MODE
-#define RLC_SAFE_MODE__CMD__SHIFT                                                                             0x0
-#define RLC_SAFE_MODE__MESSAGE__SHIFT                                                                         0x1
-#define RLC_SAFE_MODE__RESERVED1__SHIFT                                                                       0x5
-#define RLC_SAFE_MODE__RESPONSE__SHIFT                                                                        0x8
-#define RLC_SAFE_MODE__RESERVED__SHIFT                                                                        0xc
-#define RLC_SAFE_MODE__CMD_MASK                                                                               0x00000001L
-#define RLC_SAFE_MODE__MESSAGE_MASK                                                                           0x0000001EL
-#define RLC_SAFE_MODE__RESERVED1_MASK                                                                         0x000000E0L
-#define RLC_SAFE_MODE__RESPONSE_MASK                                                                          0x00000F00L
-#define RLC_SAFE_MODE__RESERVED_MASK                                                                          0xFFFFF000L
-//RLC_MEM_SLP_CNTL
-#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN__SHIFT                                                                0x0
-#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN__SHIFT                                                                0x1
-#define RLC_MEM_SLP_CNTL__RESERVED__SHIFT                                                                     0x2
-#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE__SHIFT                                                      0x7
-#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY__SHIFT                                                          0x8
-#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY__SHIFT                                                         0x10
-#define RLC_MEM_SLP_CNTL__RESERVED1__SHIFT                                                                    0x18
-#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK                                                                  0x00000001L
-#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN_MASK                                                                  0x00000002L
-#define RLC_MEM_SLP_CNTL__RESERVED_MASK                                                                       0x0000007CL
-#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE_MASK                                                        0x00000080L
-#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY_MASK                                                            0x0000FF00L
-#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY_MASK                                                           0x00FF0000L
-#define RLC_MEM_SLP_CNTL__RESERVED1_MASK                                                                      0xFF000000L
-//SMU_RLC_RESPONSE
-#define SMU_RLC_RESPONSE__RESP__SHIFT                                                                         0x0
-#define SMU_RLC_RESPONSE__RESP_MASK                                                                           0xFFFFFFFFL
-//RLC_RLCV_SAFE_MODE
-#define RLC_RLCV_SAFE_MODE__CMD__SHIFT                                                                        0x0
-#define RLC_RLCV_SAFE_MODE__MESSAGE__SHIFT                                                                    0x1
-#define RLC_RLCV_SAFE_MODE__RESERVED1__SHIFT                                                                  0x5
-#define RLC_RLCV_SAFE_MODE__RESPONSE__SHIFT                                                                   0x8
-#define RLC_RLCV_SAFE_MODE__RESERVED__SHIFT                                                                   0xc
-#define RLC_RLCV_SAFE_MODE__CMD_MASK                                                                          0x00000001L
-#define RLC_RLCV_SAFE_MODE__MESSAGE_MASK                                                                      0x0000001EL
-#define RLC_RLCV_SAFE_MODE__RESERVED1_MASK                                                                    0x000000E0L
-#define RLC_RLCV_SAFE_MODE__RESPONSE_MASK                                                                     0x00000F00L
-#define RLC_RLCV_SAFE_MODE__RESERVED_MASK                                                                     0xFFFFF000L
-//RLC_SMU_SAFE_MODE
-#define RLC_SMU_SAFE_MODE__CMD__SHIFT                                                                         0x0
-#define RLC_SMU_SAFE_MODE__MESSAGE__SHIFT                                                                     0x1
-#define RLC_SMU_SAFE_MODE__RESERVED1__SHIFT                                                                   0x5
-#define RLC_SMU_SAFE_MODE__RESPONSE__SHIFT                                                                    0x8
-#define RLC_SMU_SAFE_MODE__RESERVED__SHIFT                                                                    0xc
-#define RLC_SMU_SAFE_MODE__CMD_MASK                                                                           0x00000001L
-#define RLC_SMU_SAFE_MODE__MESSAGE_MASK                                                                       0x0000001EL
-#define RLC_SMU_SAFE_MODE__RESERVED1_MASK                                                                     0x000000E0L
-#define RLC_SMU_SAFE_MODE__RESPONSE_MASK                                                                      0x00000F00L
-#define RLC_SMU_SAFE_MODE__RESERVED_MASK                                                                      0xFFFFF000L
-//RLC_RLCV_COMMAND
-#define RLC_RLCV_COMMAND__CMD__SHIFT                                                                          0x0
-#define RLC_RLCV_COMMAND__RESERVED__SHIFT                                                                     0x4
-#define RLC_RLCV_COMMAND__CMD_MASK                                                                            0x0000000FL
-#define RLC_RLCV_COMMAND__RESERVED_MASK                                                                       0xFFFFFFF0L
-//RLC_REFCLOCK_TIMESTAMP_LSB
-#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB__SHIFT                                                      0x0
-#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB_MASK                                                        0xFFFFFFFFL
-//RLC_REFCLOCK_TIMESTAMP_MSB
-#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB__SHIFT                                                      0x0
-#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB_MASK                                                        0xFFFFFFFFL
-//RLC_GPM_TIMER_INT_0
-#define RLC_GPM_TIMER_INT_0__TIMER__SHIFT                                                                     0x0
-#define RLC_GPM_TIMER_INT_0__TIMER_MASK                                                                       0xFFFFFFFFL
-//RLC_GPM_TIMER_INT_1
-#define RLC_GPM_TIMER_INT_1__TIMER__SHIFT                                                                     0x0
-#define RLC_GPM_TIMER_INT_1__TIMER_MASK                                                                       0xFFFFFFFFL
-//RLC_GPM_TIMER_INT_2
-#define RLC_GPM_TIMER_INT_2__TIMER__SHIFT                                                                     0x0
-#define RLC_GPM_TIMER_INT_2__TIMER_MASK                                                                       0xFFFFFFFFL
-//RLC_GPM_TIMER_CTRL
-#define RLC_GPM_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                 0x0
-#define RLC_GPM_TIMER_CTRL__TIMER_1_EN__SHIFT                                                                 0x1
-#define RLC_GPM_TIMER_CTRL__TIMER_2_EN__SHIFT                                                                 0x2
-#define RLC_GPM_TIMER_CTRL__TIMER_3_EN__SHIFT                                                                 0x3
-#define RLC_GPM_TIMER_CTRL__RESERVED__SHIFT                                                                   0x4
-#define RLC_GPM_TIMER_CTRL__TIMER_0_EN_MASK                                                                   0x00000001L
-#define RLC_GPM_TIMER_CTRL__TIMER_1_EN_MASK                                                                   0x00000002L
-#define RLC_GPM_TIMER_CTRL__TIMER_2_EN_MASK                                                                   0x00000004L
-#define RLC_GPM_TIMER_CTRL__TIMER_3_EN_MASK                                                                   0x00000008L
-#define RLC_GPM_TIMER_CTRL__RESERVED_MASK                                                                     0xFFFFFFF0L
-//RLC_LB_CNTR_MAX
-#define RLC_LB_CNTR_MAX__LB_CNTR_MAX__SHIFT                                                                   0x0
-#define RLC_LB_CNTR_MAX__LB_CNTR_MAX_MASK                                                                     0xFFFFFFFFL
-//RLC_GPM_TIMER_STAT
-#define RLC_GPM_TIMER_STAT__TIMER_0_STAT__SHIFT                                                               0x0
-#define RLC_GPM_TIMER_STAT__TIMER_1_STAT__SHIFT                                                               0x1
-#define RLC_GPM_TIMER_STAT__TIMER_2_STAT__SHIFT                                                               0x2
-#define RLC_GPM_TIMER_STAT__TIMER_3_STAT__SHIFT                                                               0x3
-#define RLC_GPM_TIMER_STAT__RESERVED__SHIFT                                                                   0x4
-#define RLC_GPM_TIMER_STAT__TIMER_0_STAT_MASK                                                                 0x00000001L
-#define RLC_GPM_TIMER_STAT__TIMER_1_STAT_MASK                                                                 0x00000002L
-#define RLC_GPM_TIMER_STAT__TIMER_2_STAT_MASK                                                                 0x00000004L
-#define RLC_GPM_TIMER_STAT__TIMER_3_STAT_MASK                                                                 0x00000008L
-#define RLC_GPM_TIMER_STAT__RESERVED_MASK                                                                     0xFFFFFFF0L
-//RLC_GPM_TIMER_INT_3
-#define RLC_GPM_TIMER_INT_3__TIMER__SHIFT                                                                     0x0
-#define RLC_GPM_TIMER_INT_3__TIMER_MASK                                                                       0xFFFFFFFFL
-//RLC_SERDES_WR_NONCU_MASTER_MASK_1
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SE_MASTER_MASK_1__SHIFT                                            0x0
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_MASTER_MASK_1__SHIFT                                            0x10
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_GFX_MASTER_MASK_1__SHIFT                                        0x11
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__TC0_1_MASTER_MASK__SHIFT                                           0x12
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED_1__SHIFT                                                  0x13
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE4_MASTER_MASK__SHIFT                                          0x14
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE5_MASTER_MASK__SHIFT                                          0x15
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE6_MASTER_MASK__SHIFT                                          0x16
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE7_MASTER_MASK__SHIFT                                          0x17
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__EA_1_MASTER_MASK__SHIFT                                            0x18
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED__SHIFT                                                    0x19
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SE_MASTER_MASK_1_MASK                                              0x0000FFFFL
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_MASTER_MASK_1_MASK                                              0x00010000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__GC_GFX_MASTER_MASK_1_MASK                                          0x00020000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__TC0_1_MASTER_MASK_MASK                                             0x00040000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED_1_MASK                                                    0x00080000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE4_MASTER_MASK_MASK                                            0x00100000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE5_MASTER_MASK_MASK                                            0x00200000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE6_MASTER_MASK_MASK                                            0x00400000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__SPARE7_MASTER_MASK_MASK                                            0x00800000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__EA_1_MASTER_MASK_MASK                                              0x01000000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK_1__RESERVED_MASK                                                      0xFE000000L
-//RLC_SERDES_NONCU_MASTER_BUSY_1
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SE_MASTER_BUSY_1__SHIFT                                               0x0
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_MASTER_BUSY_1__SHIFT                                               0x10
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_GFX_MASTER_BUSY_1__SHIFT                                           0x11
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__TC0_MASTER_BUSY_1__SHIFT                                              0x12
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED_1__SHIFT                                                     0x13
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE4_MASTER_BUSY__SHIFT                                             0x14
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE5_MASTER_BUSY__SHIFT                                             0x15
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE6_MASTER_BUSY__SHIFT                                             0x16
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE7_MASTER_BUSY__SHIFT                                             0x17
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__EA_1_MASTER_BUSY__SHIFT                                               0x18
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED__SHIFT                                                       0x19
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SE_MASTER_BUSY_1_MASK                                                 0x0000FFFFL
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_MASTER_BUSY_1_MASK                                                 0x00010000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__GC_GFX_MASTER_BUSY_1_MASK                                             0x00020000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__TC0_MASTER_BUSY_1_MASK                                                0x00040000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED_1_MASK                                                       0x00080000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE4_MASTER_BUSY_MASK                                               0x00100000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE5_MASTER_BUSY_MASK                                               0x00200000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE6_MASTER_BUSY_MASK                                               0x00400000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__SPARE7_MASTER_BUSY_MASK                                               0x00800000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__EA_1_MASTER_BUSY_MASK                                                 0x01000000L
-#define RLC_SERDES_NONCU_MASTER_BUSY_1__RESERVED_MASK                                                         0xFE000000L
-//RLC_INT_STAT
-#define RLC_INT_STAT__LAST_CP_RLC_INT_ID__SHIFT                                                               0x0
-#define RLC_INT_STAT__CP_RLC_INT_PENDING__SHIFT                                                               0x8
-#define RLC_INT_STAT__RESERVED__SHIFT                                                                         0x9
-#define RLC_INT_STAT__LAST_CP_RLC_INT_ID_MASK                                                                 0x000000FFL
-#define RLC_INT_STAT__CP_RLC_INT_PENDING_MASK                                                                 0x00000100L
-#define RLC_INT_STAT__RESERVED_MASK                                                                           0xFFFFFE00L
-//RLC_LB_CNTL
-#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE__SHIFT                                                               0x0
-#define RLC_LB_CNTL__LB_CNT_CP_BUSY__SHIFT                                                                    0x1
-#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE__SHIFT                                                                0x2
-#define RLC_LB_CNTL__LB_CNT_REG_INC__SHIFT                                                                    0x3
-#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST__SHIFT                                                             0x4
-#define RLC_LB_CNTL__RESERVED__SHIFT                                                                          0xc
-#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK                                                                 0x00000001L
-#define RLC_LB_CNTL__LB_CNT_CP_BUSY_MASK                                                                      0x00000002L
-#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK                                                                  0x00000004L
-#define RLC_LB_CNTL__LB_CNT_REG_INC_MASK                                                                      0x00000008L
-#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST_MASK                                                               0x00000FF0L
-#define RLC_LB_CNTL__RESERVED_MASK                                                                            0xFFFFF000L
-//RLC_MGCG_CTRL
-#define RLC_MGCG_CTRL__MGCG_EN__SHIFT                                                                         0x0
-#define RLC_MGCG_CTRL__SILICON_EN__SHIFT                                                                      0x1
-#define RLC_MGCG_CTRL__SIMULATION_EN__SHIFT                                                                   0x2
-#define RLC_MGCG_CTRL__ON_DELAY__SHIFT                                                                        0x3
-#define RLC_MGCG_CTRL__OFF_HYSTERESIS__SHIFT                                                                  0x7
-#define RLC_MGCG_CTRL__GC_CAC_MGCG_CLK_CNTL__SHIFT                                                            0xf
-#define RLC_MGCG_CTRL__SE_CAC_MGCG_CLK_CNTL__SHIFT                                                            0x10
-#define RLC_MGCG_CTRL__SPARE__SHIFT                                                                           0x11
-#define RLC_MGCG_CTRL__MGCG_EN_MASK                                                                           0x00000001L
-#define RLC_MGCG_CTRL__SILICON_EN_MASK                                                                        0x00000002L
-#define RLC_MGCG_CTRL__SIMULATION_EN_MASK                                                                     0x00000004L
-#define RLC_MGCG_CTRL__ON_DELAY_MASK                                                                          0x00000078L
-#define RLC_MGCG_CTRL__OFF_HYSTERESIS_MASK                                                                    0x00007F80L
-#define RLC_MGCG_CTRL__GC_CAC_MGCG_CLK_CNTL_MASK                                                              0x00008000L
-#define RLC_MGCG_CTRL__SE_CAC_MGCG_CLK_CNTL_MASK                                                              0x00010000L
-#define RLC_MGCG_CTRL__SPARE_MASK                                                                             0xFFFE0000L
-//RLC_LB_CNTR_INIT
-#define RLC_LB_CNTR_INIT__LB_CNTR_INIT__SHIFT                                                                 0x0
-#define RLC_LB_CNTR_INIT__LB_CNTR_INIT_MASK                                                                   0xFFFFFFFFL
-//RLC_LOAD_BALANCE_CNTR
-#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR__SHIFT                                                   0x0
-#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR_MASK                                                     0xFFFFFFFFL
-//RLC_JUMP_TABLE_RESTORE
-#define RLC_JUMP_TABLE_RESTORE__ADDR__SHIFT                                                                   0x0
-#define RLC_JUMP_TABLE_RESTORE__ADDR_MASK                                                                     0xFFFFFFFFL
-//RLC_PG_DELAY_2
-#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE__SHIFT                                                           0x0
-#define RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT                                                               0x8
-#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE__SHIFT                                                            0x10
-#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE_MASK                                                             0x000000FFL
-#define RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK                                                                 0x0000FF00L
-#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE_MASK                                                              0xFFFF0000L
-//RLC_GPU_CLOCK_COUNT_LSB
-#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB__SHIFT                                                        0x0
-#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB_MASK                                                          0xFFFFFFFFL
-//RLC_GPU_CLOCK_COUNT_MSB
-#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB__SHIFT                                                        0x0
-#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB_MASK                                                          0xFFFFFFFFL
-//RLC_CAPTURE_GPU_CLOCK_COUNT
-#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE__SHIFT                                                           0x0
-#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED__SHIFT                                                          0x1
-#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE_MASK                                                             0x00000001L
-#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED_MASK                                                            0xFFFFFFFEL
-//RLC_UCODE_CNTL
-#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS__SHIFT                                                                0x0
-#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS_MASK                                                                  0xFFFFFFFFL
-//RLC_GPM_THREAD_RESET
-#define RLC_GPM_THREAD_RESET__THREAD0_RESET__SHIFT                                                            0x0
-#define RLC_GPM_THREAD_RESET__THREAD1_RESET__SHIFT                                                            0x1
-#define RLC_GPM_THREAD_RESET__THREAD2_RESET__SHIFT                                                            0x2
-#define RLC_GPM_THREAD_RESET__THREAD3_RESET__SHIFT                                                            0x3
-#define RLC_GPM_THREAD_RESET__RESERVED__SHIFT                                                                 0x4
-#define RLC_GPM_THREAD_RESET__THREAD0_RESET_MASK                                                              0x00000001L
-#define RLC_GPM_THREAD_RESET__THREAD1_RESET_MASK                                                              0x00000002L
-#define RLC_GPM_THREAD_RESET__THREAD2_RESET_MASK                                                              0x00000004L
-#define RLC_GPM_THREAD_RESET__THREAD3_RESET_MASK                                                              0x00000008L
-#define RLC_GPM_THREAD_RESET__RESERVED_MASK                                                                   0xFFFFFFF0L
-//RLC_GPM_CP_DMA_COMPLETE_T0
-#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA__SHIFT                                                               0x0
-#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED__SHIFT                                                           0x1
-#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA_MASK                                                                 0x00000001L
-#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED_MASK                                                             0xFFFFFFFEL
-//RLC_GPM_CP_DMA_COMPLETE_T1
-#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA__SHIFT                                                               0x0
-#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED__SHIFT                                                           0x1
-#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA_MASK                                                                 0x00000001L
-#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED_MASK                                                             0xFFFFFFFEL
-//RLC_FIREWALL_VIOLATION
-#define RLC_FIREWALL_VIOLATION__ADDR__SHIFT                                                                   0x0
-#define RLC_FIREWALL_VIOLATION__ADDR_MASK                                                                     0xFFFFFFFFL
-//RLC_GPM_STAT
-#define RLC_GPM_STAT__RLC_BUSY__SHIFT                                                                         0x0
-#define RLC_GPM_STAT__GFX_POWER_STATUS__SHIFT                                                                 0x1
-#define RLC_GPM_STAT__GFX_CLOCK_STATUS__SHIFT                                                                 0x2
-#define RLC_GPM_STAT__GFX_LS_STATUS__SHIFT                                                                    0x3
-#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS__SHIFT                                                        0x4
-#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED__SHIFT                                                        0x5
-#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED__SHIFT                                                        0x6
-#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED__SHIFT                                                         0x7
-#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED__SHIFT                                                         0x8
-#define RLC_GPM_STAT__SAVING_REGISTERS__SHIFT                                                                 0x9
-#define RLC_GPM_STAT__RESTORING_REGISTERS__SHIFT                                                              0xa
-#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                0xb
-#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                  0xc
-#define RLC_GPM_STAT__STATIC_CU_POWERING_UP__SHIFT                                                            0xd
-#define RLC_GPM_STAT__STATIC_CU_POWERING_DOWN__SHIFT                                                          0xe
-#define RLC_GPM_STAT__DYN_CU_POWERING_UP__SHIFT                                                               0xf
-#define RLC_GPM_STAT__DYN_CU_POWERING_DOWN__SHIFT                                                             0x10
-#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT                                                              0x11
-#define RLC_GPM_STAT__CMP_power_status__SHIFT                                                                 0x12
-#define RLC_GPM_STAT__GFX_LS_STATUS_3D__SHIFT                                                                 0x13
-#define RLC_GPM_STAT__GFX_CLOCK_STATUS_3D__SHIFT                                                              0x14
-#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS__SHIFT                                                             0x15
-#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE__SHIFT                                                                0x16
-#define RLC_GPM_STAT__RESERVED__SHIFT                                                                         0x17
-#define RLC_GPM_STAT__PG_ERROR_STATUS__SHIFT                                                                  0x18
-#define RLC_GPM_STAT__RLC_BUSY_MASK                                                                           0x00000001L
-#define RLC_GPM_STAT__GFX_POWER_STATUS_MASK                                                                   0x00000002L
-#define RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK                                                                   0x00000004L
-#define RLC_GPM_STAT__GFX_LS_STATUS_MASK                                                                      0x00000008L
-#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS_MASK                                                          0x00000010L
-#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED_MASK                                                          0x00000020L
-#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED_MASK                                                          0x00000040L
-#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED_MASK                                                           0x00000080L
-#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED_MASK                                                           0x00000100L
-#define RLC_GPM_STAT__SAVING_REGISTERS_MASK                                                                   0x00000200L
-#define RLC_GPM_STAT__RESTORING_REGISTERS_MASK                                                                0x00000400L
-#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE_MASK                                                  0x00000800L
-#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE_MASK                                                    0x00001000L
-#define RLC_GPM_STAT__STATIC_CU_POWERING_UP_MASK                                                              0x00002000L
-#define RLC_GPM_STAT__STATIC_CU_POWERING_DOWN_MASK                                                            0x00004000L
-#define RLC_GPM_STAT__DYN_CU_POWERING_UP_MASK                                                                 0x00008000L
-#define RLC_GPM_STAT__DYN_CU_POWERING_DOWN_MASK                                                               0x00010000L
-#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE_MASK                                                                0x00020000L
-#define RLC_GPM_STAT__CMP_power_status_MASK                                                                   0x00040000L
-#define RLC_GPM_STAT__GFX_LS_STATUS_3D_MASK                                                                   0x00080000L
-#define RLC_GPM_STAT__GFX_CLOCK_STATUS_3D_MASK                                                                0x00100000L
-#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS_MASK                                                               0x00200000L
-#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE_MASK                                                                  0x00400000L
-#define RLC_GPM_STAT__RESERVED_MASK                                                                           0x00800000L
-#define RLC_GPM_STAT__PG_ERROR_STATUS_MASK                                                                    0xFF000000L
-//RLC_GPU_CLOCK_32_RES_SEL
-#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL__SHIFT                                                              0x0
-#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED__SHIFT                                                             0x6
-#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK                                                                0x0000003FL
-#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED_MASK                                                               0xFFFFFFC0L
-//RLC_GPU_CLOCK_32
-#define RLC_GPU_CLOCK_32__GPU_CLOCK_32__SHIFT                                                                 0x0
-#define RLC_GPU_CLOCK_32__GPU_CLOCK_32_MASK                                                                   0xFFFFFFFFL
-//RLC_PG_CNTL
-#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE__SHIFT                                                           0x0
-#define RLC_PG_CNTL__GFX_POWER_GATING_SRC__SHIFT                                                              0x1
-#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE__SHIFT                                                              0x2
-#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE__SHIFT                                                           0x3
-#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE__SHIFT                                                            0x4
-#define RLC_PG_CNTL__RESERVED__SHIFT                                                                          0x5
-#define RLC_PG_CNTL__PG_OVERRIDE__SHIFT                                                                       0xe
-#define RLC_PG_CNTL__CP_PG_DISABLE__SHIFT                                                                     0xf
-#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE__SHIFT                                                             0x10
-#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE__SHIFT                                                     0x11
-#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT                                                     0x12
-#define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE__SHIFT                                                              0x13
-#define RLC_PG_CNTL__RESERVED1__SHIFT                                                                         0x14
-#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK                                                             0x00000001L
-#define RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK                                                                0x00000002L
-#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK                                                                0x00000004L
-#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK                                                             0x00000008L
-#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK                                                              0x00000010L
-#define RLC_PG_CNTL__RESERVED_MASK                                                                            0x00003FE0L
-#define RLC_PG_CNTL__PG_OVERRIDE_MASK                                                                         0x00004000L
-#define RLC_PG_CNTL__CP_PG_DISABLE_MASK                                                                       0x00008000L
-#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE_MASK                                                               0x00010000L
-#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK                                                       0x00020000L
-#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK                                                       0x00040000L
-#define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE_MASK                                                                0x00080000L
-#define RLC_PG_CNTL__RESERVED1_MASK                                                                           0x00F00000L
-//RLC_GPM_THREAD_PRIORITY
-#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT                                                      0x0
-#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY__SHIFT                                                      0x8
-#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY__SHIFT                                                      0x10
-#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY__SHIFT                                                      0x18
-#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK                                                        0x000000FFL
-#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK                                                        0x0000FF00L
-#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY_MASK                                                        0x00FF0000L
-#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY_MASK                                                        0xFF000000L
-//RLC_GPM_THREAD_ENABLE
-#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE__SHIFT                                                          0x0
-#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE__SHIFT                                                          0x1
-#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE__SHIFT                                                          0x2
-#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE__SHIFT                                                          0x3
-#define RLC_GPM_THREAD_ENABLE__RESERVED__SHIFT                                                                0x4
-#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE_MASK                                                            0x00000001L
-#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE_MASK                                                            0x00000002L
-#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE_MASK                                                            0x00000004L
-#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE_MASK                                                            0x00000008L
-#define RLC_GPM_THREAD_ENABLE__RESERVED_MASK                                                                  0xFFFFFFF0L
-//RLC_CGTT_MGCG_OVERRIDE
-#define RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE__SHIFT                                                 0x0
-#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE__SHIFT                                                 0x1
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE__SHIFT                                                    0x2
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE__SHIFT                                                    0x3
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE__SHIFT                                                    0x4
-#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE__SHIFT                                                0x5
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE__SHIFT                                                    0x6
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE__SHIFT                                                0x7
-#define RLC_CGTT_MGCG_OVERRIDE__RESERVED__SHIFT                                                               0x8
-#define RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK                                                   0x00000001L
-#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK                                                   0x00000002L
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK                                                      0x00000004L
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK                                                      0x00000008L
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK                                                      0x00000010L
-#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK                                                  0x00000020L
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK                                                      0x00000040L
-#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK                                                  0x00000080L
-#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_MASK                                                                 0xFFFFFF00L
-//RLC_CGCG_CGLS_CTRL
-#define RLC_CGCG_CGLS_CTRL__CGCG_EN__SHIFT                                                                    0x0
-#define RLC_CGCG_CGLS_CTRL__CGLS_EN__SHIFT                                                                    0x1
-#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT                                                   0x2
-#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT                                                    0x8
-#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER__SHIFT                                                            0x1b
-#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL__SHIFT                                                              0x1c
-#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE__SHIFT                                                                 0x1d
-#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN__SHIFT                                                             0x1f
-#define RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK                                                                      0x00000001L
-#define RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK                                                                      0x00000002L
-#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK                                                     0x000000FCL
-#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK                                                      0x07FFFF00L
-#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER_MASK                                                              0x08000000L
-#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL_MASK                                                                0x10000000L
-#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE_MASK                                                                   0x60000000L
-#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN_MASK                                                               0x80000000L
-//RLC_CGCG_RAMP_CTRL
-#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT__SHIFT                                                        0x0
-#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT__SHIFT                                                         0x4
-#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT__SHIFT                                                          0x8
-#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT__SHIFT                                                           0xc
-#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT__SHIFT                                                             0x10
-#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT__SHIFT                                                            0x1c
-#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT_MASK                                                          0x0000000FL
-#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT_MASK                                                           0x000000F0L
-#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT_MASK                                                            0x00000F00L
-#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT_MASK                                                             0x0000F000L
-#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT_MASK                                                               0x0FFF0000L
-#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT_MASK                                                              0xF0000000L
-//RLC_DYN_PG_STATUS
-#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK__SHIFT                                                           0x0
-#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK_MASK                                                             0xFFFFFFFFL
-//RLC_DYN_PG_REQUEST
-#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK__SHIFT                                                         0x0
-#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK_MASK                                                           0xFFFFFFFFL
-//RLC_PG_DELAY
-#define RLC_PG_DELAY__POWER_UP_DELAY__SHIFT                                                                   0x0
-#define RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT                                                                 0x8
-#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT                                                              0x10
-#define RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT                                                                  0x18
-#define RLC_PG_DELAY__POWER_UP_DELAY_MASK                                                                     0x000000FFL
-#define RLC_PG_DELAY__POWER_DOWN_DELAY_MASK                                                                   0x0000FF00L
-#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY_MASK                                                                0x00FF0000L
-#define RLC_PG_DELAY__MEM_SLEEP_DELAY_MASK                                                                    0xFF000000L
-//RLC_CU_STATUS
-#define RLC_CU_STATUS__WORK_PENDING__SHIFT                                                                    0x0
-#define RLC_CU_STATUS__WORK_PENDING_MASK                                                                      0xFFFFFFFFL
-//RLC_LB_INIT_CU_MASK
-#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK__SHIFT                                                              0x0
-#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK_MASK                                                                0xFFFFFFFFL
-//RLC_LB_ALWAYS_ACTIVE_CU_MASK
-#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK__SHIFT                                            0x0
-#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK_MASK                                              0xFFFFFFFFL
-//RLC_LB_PARAMS
-#define RLC_LB_PARAMS__SKIP_L2_CHECK__SHIFT                                                                   0x0
-#define RLC_LB_PARAMS__FIFO_SAMPLES__SHIFT                                                                    0x1
-#define RLC_LB_PARAMS__PG_IDLE_SAMPLES__SHIFT                                                                 0x8
-#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL__SHIFT                                                         0x10
-#define RLC_LB_PARAMS__SKIP_L2_CHECK_MASK                                                                     0x00000001L
-#define RLC_LB_PARAMS__FIFO_SAMPLES_MASK                                                                      0x000000FEL
-#define RLC_LB_PARAMS__PG_IDLE_SAMPLES_MASK                                                                   0x0000FF00L
-#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL_MASK                                                           0xFFFF0000L
-//RLC_THREAD1_DELAY
-#define RLC_THREAD1_DELAY__CU_IDEL_DELAY__SHIFT                                                               0x0
-#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY__SHIFT                                                       0x8
-#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY__SHIFT                                                       0x10
-#define RLC_THREAD1_DELAY__SPARE__SHIFT                                                                       0x18
-#define RLC_THREAD1_DELAY__CU_IDEL_DELAY_MASK                                                                 0x000000FFL
-#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY_MASK                                                         0x0000FF00L
-#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY_MASK                                                         0x00FF0000L
-#define RLC_THREAD1_DELAY__SPARE_MASK                                                                         0xFF000000L
-//RLC_PG_ALWAYS_ON_CU_MASK
-#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK__SHIFT                                                          0x0
-#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK_MASK                                                            0xFFFFFFFFL
-//RLC_MAX_PG_CU
-#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT                                                               0x0
-#define RLC_MAX_PG_CU__SPARE__SHIFT                                                                           0x8
-#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK                                                                 0x000000FFL
-#define RLC_MAX_PG_CU__SPARE_MASK                                                                             0xFFFFFF00L
-//RLC_AUTO_PG_CTRL
-#define RLC_AUTO_PG_CTRL__AUTO_PG_EN__SHIFT                                                                   0x0
-#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN__SHIFT                                                0x1
-#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN__SHIFT                                                              0x2
-#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT                                             0x3
-#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD__SHIFT                                             0x13
-#define RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK                                                                     0x00000001L
-#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN_MASK                                                  0x00000002L
-#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN_MASK                                                                0x00000004L
-#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK                                               0x0007FFF8L
-#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK                                               0xFFF80000L
-//RLC_SMU_GRBM_REG_SAVE_CTRL
-#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE__SHIFT                                                0x0
-#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE__SHIFT                                                              0x1
-#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE_MASK                                                  0x00000001L
-#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE_MASK                                                                0xFFFFFFFEL
-//RLC_SERDES_RD_MASTER_INDEX
-#define RLC_SERDES_RD_MASTER_INDEX__CU_ID__SHIFT                                                              0x0
-#define RLC_SERDES_RD_MASTER_INDEX__SH_ID__SHIFT                                                              0x4
-#define RLC_SERDES_RD_MASTER_INDEX__SE_ID__SHIFT                                                              0x6
-#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID__SHIFT                                                        0x9
-#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU__SHIFT                                                           0xc
-#define RLC_SERDES_RD_MASTER_INDEX__NON_SE__SHIFT                                                             0xd
-#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID__SHIFT                                                        0x11
-#define RLC_SERDES_RD_MASTER_INDEX__SPARE__SHIFT                                                              0x13
-#define RLC_SERDES_RD_MASTER_INDEX__CU_ID_MASK                                                                0x0000000FL
-#define RLC_SERDES_RD_MASTER_INDEX__SH_ID_MASK                                                                0x00000030L
-#define RLC_SERDES_RD_MASTER_INDEX__SE_ID_MASK                                                                0x000001C0L
-#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID_MASK                                                          0x00000E00L
-#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_MASK                                                             0x00001000L
-#define RLC_SERDES_RD_MASTER_INDEX__NON_SE_MASK                                                               0x0001E000L
-#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID_MASK                                                          0x00060000L
-#define RLC_SERDES_RD_MASTER_INDEX__SPARE_MASK                                                                0xFFF80000L
-//RLC_SERDES_RD_DATA_0
-#define RLC_SERDES_RD_DATA_0__DATA__SHIFT                                                                     0x0
-#define RLC_SERDES_RD_DATA_0__DATA_MASK                                                                       0xFFFFFFFFL
-//RLC_SERDES_RD_DATA_1
-#define RLC_SERDES_RD_DATA_1__DATA__SHIFT                                                                     0x0
-#define RLC_SERDES_RD_DATA_1__DATA_MASK                                                                       0xFFFFFFFFL
-//RLC_SERDES_RD_DATA_2
-#define RLC_SERDES_RD_DATA_2__DATA__SHIFT                                                                     0x0
-#define RLC_SERDES_RD_DATA_2__DATA_MASK                                                                       0xFFFFFFFFL
-//RLC_SERDES_WR_CU_MASTER_MASK
-#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK__SHIFT                                                      0x0
-#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK_MASK                                                        0xFFFFFFFFL
-//RLC_SERDES_WR_NONCU_MASTER_MASK
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK__SHIFT                                                0x0
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK__SHIFT                                                0x10
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_GFX_MASTER_MASK__SHIFT                                            0x11
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK__SHIFT                                               0x12
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK__SHIFT                                               0x13
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK__SHIFT                                            0x14
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK__SHIFT                                            0x15
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK__SHIFT                                            0x16
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK__SHIFT                                            0x17
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__EA_0_MASTER_MASK__SHIFT                                              0x18
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC2_MASTER_MASK__SHIFT                                               0x19
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED__SHIFT                                                      0x1a
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK_MASK                                                  0x0000FFFFL
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK_MASK                                                  0x00010000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_GFX_MASTER_MASK_MASK                                              0x00020000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK_MASK                                                 0x00040000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK_MASK                                                 0x00080000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK_MASK                                              0x00100000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK_MASK                                              0x00200000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK_MASK                                              0x00400000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK_MASK                                              0x00800000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__EA_0_MASTER_MASK_MASK                                                0x01000000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC2_MASTER_MASK_MASK                                                 0x02000000L
-#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED_MASK                                                        0xFC000000L
-//RLC_SERDES_WR_CTRL
-#define RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT                                                                   0x0
-#define RLC_SERDES_WR_CTRL__POWER_DOWN__SHIFT                                                                 0x8
-#define RLC_SERDES_WR_CTRL__POWER_UP__SHIFT                                                                   0x9
-#define RLC_SERDES_WR_CTRL__P1_SELECT__SHIFT                                                                  0xa
-#define RLC_SERDES_WR_CTRL__P2_SELECT__SHIFT                                                                  0xb
-#define RLC_SERDES_WR_CTRL__WRITE_COMMAND__SHIFT                                                              0xc
-#define RLC_SERDES_WR_CTRL__READ_COMMAND__SHIFT                                                               0xd
-#define RLC_SERDES_WR_CTRL__RDDATA_RESET__SHIFT                                                               0xe
-#define RLC_SERDES_WR_CTRL__SHORT_FORMAT__SHIFT                                                               0xf
-#define RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT                                                                   0x10
-#define RLC_SERDES_WR_CTRL__SRBM_OVERRIDE__SHIFT                                                              0x1a
-#define RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR__SHIFT                                                              0x1b
-#define RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT                                                                   0x1c
-#define RLC_SERDES_WR_CTRL__BPM_ADDR_MASK                                                                     0x000000FFL
-#define RLC_SERDES_WR_CTRL__POWER_DOWN_MASK                                                                   0x00000100L
-#define RLC_SERDES_WR_CTRL__POWER_UP_MASK                                                                     0x00000200L
-#define RLC_SERDES_WR_CTRL__P1_SELECT_MASK                                                                    0x00000400L
-#define RLC_SERDES_WR_CTRL__P2_SELECT_MASK                                                                    0x00000800L
-#define RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK                                                                0x00001000L
-#define RLC_SERDES_WR_CTRL__READ_COMMAND_MASK                                                                 0x00002000L
-#define RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK                                                                 0x00004000L
-#define RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK                                                                 0x00008000L
-#define RLC_SERDES_WR_CTRL__BPM_DATA_MASK                                                                     0x03FF0000L
-#define RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK                                                                0x04000000L
-#define RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK                                                                0x08000000L
-#define RLC_SERDES_WR_CTRL__REG_ADDR_MASK                                                                     0xF0000000L
-//RLC_SERDES_WR_DATA
-#define RLC_SERDES_WR_DATA__DATA__SHIFT                                                                       0x0
-#define RLC_SERDES_WR_DATA__DATA_MASK                                                                         0xFFFFFFFFL
-//RLC_SERDES_CU_MASTER_BUSY
-#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY__SHIFT                                                           0x0
-#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY_MASK                                                             0xFFFFFFFFL
-//RLC_SERDES_NONCU_MASTER_BUSY
-#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY__SHIFT                                                   0x0
-#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY__SHIFT                                                   0x10
-#define RLC_SERDES_NONCU_MASTER_BUSY__GC_GFX_MASTER_BUSY__SHIFT                                               0x11
-#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY__SHIFT                                                  0x12
-#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY__SHIFT                                                  0x13
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY__SHIFT                                               0x14
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY__SHIFT                                               0x15
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY__SHIFT                                               0x16
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY__SHIFT                                               0x17
-#define RLC_SERDES_NONCU_MASTER_BUSY__EA_0_MASTER_BUSY__SHIFT                                                 0x18
-#define RLC_SERDES_NONCU_MASTER_BUSY__TC2_MASTER_BUSY__SHIFT                                                  0x19
-#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED__SHIFT                                                         0x1a
-#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK                                                     0x0000FFFFL
-#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK                                                     0x00010000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__GC_GFX_MASTER_BUSY_MASK                                                 0x00020000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK                                                    0x00040000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK                                                    0x00080000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY_MASK                                                 0x00100000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY_MASK                                                 0x00200000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY_MASK                                                 0x00400000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY_MASK                                                 0x00800000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__EA_0_MASTER_BUSY_MASK                                                   0x01000000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__TC2_MASTER_BUSY_MASK                                                    0x02000000L
-#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED_MASK                                                           0xFC000000L
-//RLC_GPM_GENERAL_0
-#define RLC_GPM_GENERAL_0__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_0__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_1
-#define RLC_GPM_GENERAL_1__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_1__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_2
-#define RLC_GPM_GENERAL_2__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_2__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_3
-#define RLC_GPM_GENERAL_3__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_3__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_4
-#define RLC_GPM_GENERAL_4__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_4__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_5
-#define RLC_GPM_GENERAL_5__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_5__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_6
-#define RLC_GPM_GENERAL_6__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_6__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_7
-#define RLC_GPM_GENERAL_7__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_7__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_SCRATCH_ADDR
-#define RLC_GPM_SCRATCH_ADDR__ADDR__SHIFT                                                                     0x0
-#define RLC_GPM_SCRATCH_ADDR__RESERVED__SHIFT                                                                 0x9
-#define RLC_GPM_SCRATCH_ADDR__ADDR_MASK                                                                       0x000001FFL
-#define RLC_GPM_SCRATCH_ADDR__RESERVED_MASK                                                                   0xFFFFFE00L
-//RLC_GPM_SCRATCH_DATA
-#define RLC_GPM_SCRATCH_DATA__DATA__SHIFT                                                                     0x0
-#define RLC_GPM_SCRATCH_DATA__DATA_MASK                                                                       0xFFFFFFFFL
-//RLC_STATIC_PG_STATUS
-#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK__SHIFT                                                        0x0
-#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK_MASK                                                          0xFFFFFFFFL
-//RLC_SPM_MC_CNTL
-#define RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT                                                                  0x0
-#define RLC_SPM_MC_CNTL__RLC_SPM_POLICY__SHIFT                                                                0x4
-#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR__SHIFT                                                             0x5
-#define RLC_SPM_MC_CNTL__RLC_SPM_FED__SHIFT                                                                   0x6
-#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER__SHIFT                                                            0x7
-#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE__SHIFT                                                                 0x8
-#define RLC_SPM_MC_CNTL__RESERVED__SHIFT                                                                      0xa
-#define RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK                                                                    0x0000000FL
-#define RLC_SPM_MC_CNTL__RLC_SPM_POLICY_MASK                                                                  0x00000010L
-#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR_MASK                                                               0x00000020L
-#define RLC_SPM_MC_CNTL__RLC_SPM_FED_MASK                                                                     0x00000040L
-#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER_MASK                                                              0x00000080L
-#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_MASK                                                                   0x00000300L
-#define RLC_SPM_MC_CNTL__RESERVED_MASK                                                                        0xFFFFFC00L
-//RLC_SPM_INT_CNTL
-#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL__SHIFT                                                             0x0
-#define RLC_SPM_INT_CNTL__RESERVED__SHIFT                                                                     0x1
-#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL_MASK                                                               0x00000001L
-#define RLC_SPM_INT_CNTL__RESERVED_MASK                                                                       0xFFFFFFFEL
-//RLC_SPM_INT_STATUS
-#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS__SHIFT                                                         0x0
-#define RLC_SPM_INT_STATUS__RESERVED__SHIFT                                                                   0x1
-#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS_MASK                                                           0x00000001L
-#define RLC_SPM_INT_STATUS__RESERVED_MASK                                                                     0xFFFFFFFEL
-//RLC_SMU_MESSAGE
-#define RLC_SMU_MESSAGE__CMD__SHIFT                                                                           0x0
-#define RLC_SMU_MESSAGE__CMD_MASK                                                                             0xFFFFFFFFL
-//RLC_GPM_LOG_SIZE
-#define RLC_GPM_LOG_SIZE__SIZE__SHIFT                                                                         0x0
-#define RLC_GPM_LOG_SIZE__SIZE_MASK                                                                           0xFFFFFFFFL
-//RLC_PG_DELAY_3
-#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT                                                        0x0
-#define RLC_PG_DELAY_3__RESERVED__SHIFT                                                                       0x8
-#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK                                                          0x000000FFL
-#define RLC_PG_DELAY_3__RESERVED_MASK                                                                         0xFFFFFF00L
-//RLC_GPR_REG1
-#define RLC_GPR_REG1__DATA__SHIFT                                                                             0x0
-#define RLC_GPR_REG1__DATA_MASK                                                                               0xFFFFFFFFL
-//RLC_GPR_REG2
-#define RLC_GPR_REG2__DATA__SHIFT                                                                             0x0
-#define RLC_GPR_REG2__DATA_MASK                                                                               0xFFFFFFFFL
-//RLC_GPM_LOG_CONT
-#define RLC_GPM_LOG_CONT__CONT__SHIFT                                                                         0x0
-#define RLC_GPM_LOG_CONT__CONT_MASK                                                                           0xFFFFFFFFL
-//RLC_GPM_INT_DISABLE_TH0
-#define RLC_GPM_INT_DISABLE_TH0__DISABLE__SHIFT                                                               0x0
-#define RLC_GPM_INT_DISABLE_TH0__DISABLE_MASK                                                                 0xFFFFFFFFL
-//RLC_GPM_INT_DISABLE_TH1
-#define RLC_GPM_INT_DISABLE_TH1__DISABLE__SHIFT                                                               0x0
-#define RLC_GPM_INT_DISABLE_TH1__DISABLE_MASK                                                                 0xFFFFFFFFL
-//RLC_GPM_INT_FORCE_TH0
-#define RLC_GPM_INT_FORCE_TH0__FORCE__SHIFT                                                                   0x0
-#define RLC_GPM_INT_FORCE_TH0__FORCE_MASK                                                                     0xFFFFFFFFL
-//RLC_GPM_INT_FORCE_TH1
-#define RLC_GPM_INT_FORCE_TH1__FORCE__SHIFT                                                                   0x0
-#define RLC_GPM_INT_FORCE_TH1__FORCE_MASK                                                                     0xFFFFFFFFL
-//RLC_SRM_CNTL
-#define RLC_SRM_CNTL__SRM_ENABLE__SHIFT                                                                       0x0
-#define RLC_SRM_CNTL__AUTO_INCR_ADDR__SHIFT                                                                   0x1
-#define RLC_SRM_CNTL__RESERVED__SHIFT                                                                         0x2
-#define RLC_SRM_CNTL__SRM_ENABLE_MASK                                                                         0x00000001L
-#define RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK                                                                     0x00000002L
-#define RLC_SRM_CNTL__RESERVED_MASK                                                                           0xFFFFFFFCL
-//RLC_SRM_ARAM_ADDR
-#define RLC_SRM_ARAM_ADDR__ADDR__SHIFT                                                                        0x0
-#define RLC_SRM_ARAM_ADDR__RESERVED__SHIFT                                                                    0xc
-#define RLC_SRM_ARAM_ADDR__ADDR_MASK                                                                          0x00000FFFL
-#define RLC_SRM_ARAM_ADDR__RESERVED_MASK                                                                      0xFFFFF000L
-//RLC_SRM_ARAM_DATA
-#define RLC_SRM_ARAM_DATA__DATA__SHIFT                                                                        0x0
-#define RLC_SRM_ARAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_SRM_DRAM_ADDR
-#define RLC_SRM_DRAM_ADDR__ADDR__SHIFT                                                                        0x0
-#define RLC_SRM_DRAM_ADDR__RESERVED__SHIFT                                                                    0xc
-#define RLC_SRM_DRAM_ADDR__ADDR_MASK                                                                          0x00000FFFL
-#define RLC_SRM_DRAM_ADDR__RESERVED_MASK                                                                      0xFFFFF000L
-//RLC_SRM_DRAM_DATA
-#define RLC_SRM_DRAM_DATA__DATA__SHIFT                                                                        0x0
-#define RLC_SRM_DRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_SRM_GPM_COMMAND
-#define RLC_SRM_GPM_COMMAND__OP__SHIFT                                                                        0x0
-#define RLC_SRM_GPM_COMMAND__INDEX_CNTL__SHIFT                                                                0x1
-#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM__SHIFT                                                            0x2
-#define RLC_SRM_GPM_COMMAND__SIZE__SHIFT                                                                      0x5
-#define RLC_SRM_GPM_COMMAND__START_OFFSET__SHIFT                                                              0x11
-#define RLC_SRM_GPM_COMMAND__RESERVED1__SHIFT                                                                 0x1d
-#define RLC_SRM_GPM_COMMAND__DEST_MEMORY__SHIFT                                                               0x1f
-#define RLC_SRM_GPM_COMMAND__OP_MASK                                                                          0x00000001L
-#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_MASK                                                                  0x00000002L
-#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM_MASK                                                              0x0000001CL
-#define RLC_SRM_GPM_COMMAND__SIZE_MASK                                                                        0x0001FFE0L
-#define RLC_SRM_GPM_COMMAND__START_OFFSET_MASK                                                                0x1FFE0000L
-#define RLC_SRM_GPM_COMMAND__RESERVED1_MASK                                                                   0x60000000L
-#define RLC_SRM_GPM_COMMAND__DEST_MEMORY_MASK                                                                 0x80000000L
-//RLC_SRM_GPM_COMMAND_STATUS
-#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY__SHIFT                                                         0x0
-#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL__SHIFT                                                          0x1
-#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED__SHIFT                                                           0x2
-#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY_MASK                                                           0x00000001L
-#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL_MASK                                                            0x00000002L
-#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED_MASK                                                             0xFFFFFFFCL
-//RLC_SRM_RLCV_COMMAND
-#define RLC_SRM_RLCV_COMMAND__OP__SHIFT                                                                       0x0
-#define RLC_SRM_RLCV_COMMAND__RESERVED__SHIFT                                                                 0x1
-#define RLC_SRM_RLCV_COMMAND__SIZE__SHIFT                                                                     0x4
-#define RLC_SRM_RLCV_COMMAND__START_OFFSET__SHIFT                                                             0x10
-#define RLC_SRM_RLCV_COMMAND__RESERVED1__SHIFT                                                                0x1c
-#define RLC_SRM_RLCV_COMMAND__DEST_MEMORY__SHIFT                                                              0x1f
-#define RLC_SRM_RLCV_COMMAND__OP_MASK                                                                         0x00000001L
-#define RLC_SRM_RLCV_COMMAND__RESERVED_MASK                                                                   0x0000000EL
-#define RLC_SRM_RLCV_COMMAND__SIZE_MASK                                                                       0x0000FFF0L
-#define RLC_SRM_RLCV_COMMAND__START_OFFSET_MASK                                                               0x0FFF0000L
-#define RLC_SRM_RLCV_COMMAND__RESERVED1_MASK                                                                  0x70000000L
-#define RLC_SRM_RLCV_COMMAND__DEST_MEMORY_MASK                                                                0x80000000L
-//RLC_SRM_RLCV_COMMAND_STATUS
-#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_EMPTY__SHIFT                                                        0x0
-#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_FULL__SHIFT                                                         0x1
-#define RLC_SRM_RLCV_COMMAND_STATUS__RESERVED__SHIFT                                                          0x2
-#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_EMPTY_MASK                                                          0x00000001L
-#define RLC_SRM_RLCV_COMMAND_STATUS__FIFO_FULL_MASK                                                           0x00000002L
-#define RLC_SRM_RLCV_COMMAND_STATUS__RESERVED_MASK                                                            0xFFFFFFFCL
-//RLC_SRM_INDEX_CNTL_ADDR_0
-#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_0__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_0__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_ADDR_1
-#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_1__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_1__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_ADDR_2
-#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_2__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_2__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_ADDR_3
-#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_3__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_3__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_ADDR_4
-#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_4__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_4__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_ADDR_5
-#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_5__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_5__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_ADDR_6
-#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_6__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_6__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_ADDR_7
-#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS__SHIFT                                                             0x0
-#define RLC_SRM_INDEX_CNTL_ADDR_7__RESERVED__SHIFT                                                            0x10
-#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS_MASK                                                               0x0000FFFFL
-#define RLC_SRM_INDEX_CNTL_ADDR_7__RESERVED_MASK                                                              0xFFFF0000L
-//RLC_SRM_INDEX_CNTL_DATA_0
-#define RLC_SRM_INDEX_CNTL_DATA_0__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_0__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_INDEX_CNTL_DATA_1
-#define RLC_SRM_INDEX_CNTL_DATA_1__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_1__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_INDEX_CNTL_DATA_2
-#define RLC_SRM_INDEX_CNTL_DATA_2__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_2__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_INDEX_CNTL_DATA_3
-#define RLC_SRM_INDEX_CNTL_DATA_3__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_3__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_INDEX_CNTL_DATA_4
-#define RLC_SRM_INDEX_CNTL_DATA_4__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_4__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_INDEX_CNTL_DATA_5
-#define RLC_SRM_INDEX_CNTL_DATA_5__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_5__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_INDEX_CNTL_DATA_6
-#define RLC_SRM_INDEX_CNTL_DATA_6__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_6__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_INDEX_CNTL_DATA_7
-#define RLC_SRM_INDEX_CNTL_DATA_7__DATA__SHIFT                                                                0x0
-#define RLC_SRM_INDEX_CNTL_DATA_7__DATA_MASK                                                                  0xFFFFFFFFL
-//RLC_SRM_STAT
-#define RLC_SRM_STAT__SRM_BUSY__SHIFT                                                                         0x0
-#define RLC_SRM_STAT__SRM_BUSY_DELAY__SHIFT                                                                   0x1
-#define RLC_SRM_STAT__RESERVED__SHIFT                                                                         0x2
-#define RLC_SRM_STAT__SRM_BUSY_MASK                                                                           0x00000001L
-#define RLC_SRM_STAT__SRM_BUSY_DELAY_MASK                                                                     0x00000002L
-#define RLC_SRM_STAT__RESERVED_MASK                                                                           0xFFFFFFFCL
-//RLC_SRM_GPM_ABORT
-#define RLC_SRM_GPM_ABORT__ABORT__SHIFT                                                                       0x0
-#define RLC_SRM_GPM_ABORT__RESERVED__SHIFT                                                                    0x1
-#define RLC_SRM_GPM_ABORT__ABORT_MASK                                                                         0x00000001L
-#define RLC_SRM_GPM_ABORT__RESERVED_MASK                                                                      0xFFFFFFFEL
-//RLC_CSIB_ADDR_LO
-#define RLC_CSIB_ADDR_LO__ADDRESS__SHIFT                                                                      0x0
-#define RLC_CSIB_ADDR_LO__ADDRESS_MASK                                                                        0xFFFFFFFFL
-//RLC_CSIB_ADDR_HI
-#define RLC_CSIB_ADDR_HI__ADDRESS__SHIFT                                                                      0x0
-#define RLC_CSIB_ADDR_HI__ADDRESS_MASK                                                                        0x0000FFFFL
-//RLC_CSIB_LENGTH
-#define RLC_CSIB_LENGTH__LENGTH__SHIFT                                                                        0x0
-#define RLC_CSIB_LENGTH__LENGTH_MASK                                                                          0xFFFFFFFFL
-//RLC_SMU_COMMAND
-#define RLC_SMU_COMMAND__CMD__SHIFT                                                                           0x0
-#define RLC_SMU_COMMAND__CMD_MASK                                                                             0xFFFFFFFFL
-//RLC_CP_SCHEDULERS
-#define RLC_CP_SCHEDULERS__scheduler0__SHIFT                                                                  0x0
-#define RLC_CP_SCHEDULERS__scheduler1__SHIFT                                                                  0x8
-#define RLC_CP_SCHEDULERS__scheduler2__SHIFT                                                                  0x10
-#define RLC_CP_SCHEDULERS__scheduler3__SHIFT                                                                  0x18
-#define RLC_CP_SCHEDULERS__scheduler0_MASK                                                                    0x000000FFL
-#define RLC_CP_SCHEDULERS__scheduler1_MASK                                                                    0x0000FF00L
-#define RLC_CP_SCHEDULERS__scheduler2_MASK                                                                    0x00FF0000L
-#define RLC_CP_SCHEDULERS__scheduler3_MASK                                                                    0xFF000000L
-//RLC_SMU_ARGUMENT_1
-#define RLC_SMU_ARGUMENT_1__ARG__SHIFT                                                                        0x0
-#define RLC_SMU_ARGUMENT_1__ARG_MASK                                                                          0xFFFFFFFFL
-//RLC_SMU_ARGUMENT_2
-#define RLC_SMU_ARGUMENT_2__ARG__SHIFT                                                                        0x0
-#define RLC_SMU_ARGUMENT_2__ARG_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_8
-#define RLC_GPM_GENERAL_8__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_8__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_9
-#define RLC_GPM_GENERAL_9__DATA__SHIFT                                                                        0x0
-#define RLC_GPM_GENERAL_9__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPM_GENERAL_10
-#define RLC_GPM_GENERAL_10__DATA__SHIFT                                                                       0x0
-#define RLC_GPM_GENERAL_10__DATA_MASK                                                                         0xFFFFFFFFL
-//RLC_GPM_GENERAL_11
-#define RLC_GPM_GENERAL_11__DATA__SHIFT                                                                       0x0
-#define RLC_GPM_GENERAL_11__DATA_MASK                                                                         0xFFFFFFFFL
-//RLC_GPM_GENERAL_12
-#define RLC_GPM_GENERAL_12__DATA__SHIFT                                                                       0x0
-#define RLC_GPM_GENERAL_12__DATA_MASK                                                                         0xFFFFFFFFL
-//RLC_GPM_UTCL1_CNTL_0
-#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
-#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE__SHIFT                                                                0x18
-#define RLC_GPM_UTCL1_CNTL_0__BYPASS__SHIFT                                                                   0x19
-#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE__SHIFT                                                               0x1a
-#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
-#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP__SHIFT                                                              0x1c
-#define RLC_GPM_UTCL1_CNTL_0__FORCE_SD_VMID_DIRTY__SHIFT                                                      0x1d
-#define RLC_GPM_UTCL1_CNTL_0__RESERVED__SHIFT                                                                 0x1e
-#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
-#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE_MASK                                                                  0x01000000L
-#define RLC_GPM_UTCL1_CNTL_0__BYPASS_MASK                                                                     0x02000000L
-#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE_MASK                                                                 0x04000000L
-#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
-#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP_MASK                                                                0x10000000L
-#define RLC_GPM_UTCL1_CNTL_0__FORCE_SD_VMID_DIRTY_MASK                                                        0x20000000L
-#define RLC_GPM_UTCL1_CNTL_0__RESERVED_MASK                                                                   0xC0000000L
-//RLC_GPM_UTCL1_CNTL_1
-#define RLC_GPM_UTCL1_CNTL_1__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
-#define RLC_GPM_UTCL1_CNTL_1__DROP_MODE__SHIFT                                                                0x18
-#define RLC_GPM_UTCL1_CNTL_1__BYPASS__SHIFT                                                                   0x19
-#define RLC_GPM_UTCL1_CNTL_1__INVALIDATE__SHIFT                                                               0x1a
-#define RLC_GPM_UTCL1_CNTL_1__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
-#define RLC_GPM_UTCL1_CNTL_1__FORCE_SNOOP__SHIFT                                                              0x1c
-#define RLC_GPM_UTCL1_CNTL_1__FORCE_SD_VMID_DIRTY__SHIFT                                                      0x1d
-#define RLC_GPM_UTCL1_CNTL_1__RESERVED__SHIFT                                                                 0x1e
-#define RLC_GPM_UTCL1_CNTL_1__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
-#define RLC_GPM_UTCL1_CNTL_1__DROP_MODE_MASK                                                                  0x01000000L
-#define RLC_GPM_UTCL1_CNTL_1__BYPASS_MASK                                                                     0x02000000L
-#define RLC_GPM_UTCL1_CNTL_1__INVALIDATE_MASK                                                                 0x04000000L
-#define RLC_GPM_UTCL1_CNTL_1__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
-#define RLC_GPM_UTCL1_CNTL_1__FORCE_SNOOP_MASK                                                                0x10000000L
-#define RLC_GPM_UTCL1_CNTL_1__FORCE_SD_VMID_DIRTY_MASK                                                        0x20000000L
-#define RLC_GPM_UTCL1_CNTL_1__RESERVED_MASK                                                                   0xC0000000L
-//RLC_GPM_UTCL1_CNTL_2
-#define RLC_GPM_UTCL1_CNTL_2__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
-#define RLC_GPM_UTCL1_CNTL_2__DROP_MODE__SHIFT                                                                0x18
-#define RLC_GPM_UTCL1_CNTL_2__BYPASS__SHIFT                                                                   0x19
-#define RLC_GPM_UTCL1_CNTL_2__INVALIDATE__SHIFT                                                               0x1a
-#define RLC_GPM_UTCL1_CNTL_2__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
-#define RLC_GPM_UTCL1_CNTL_2__FORCE_SNOOP__SHIFT                                                              0x1c
-#define RLC_GPM_UTCL1_CNTL_2__FORCE_SD_VMID_DIRTY__SHIFT                                                      0x1d
-#define RLC_GPM_UTCL1_CNTL_2__RESERVED__SHIFT                                                                 0x1e
-#define RLC_GPM_UTCL1_CNTL_2__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
-#define RLC_GPM_UTCL1_CNTL_2__DROP_MODE_MASK                                                                  0x01000000L
-#define RLC_GPM_UTCL1_CNTL_2__BYPASS_MASK                                                                     0x02000000L
-#define RLC_GPM_UTCL1_CNTL_2__INVALIDATE_MASK                                                                 0x04000000L
-#define RLC_GPM_UTCL1_CNTL_2__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
-#define RLC_GPM_UTCL1_CNTL_2__FORCE_SNOOP_MASK                                                                0x10000000L
-#define RLC_GPM_UTCL1_CNTL_2__FORCE_SD_VMID_DIRTY_MASK                                                        0x20000000L
-#define RLC_GPM_UTCL1_CNTL_2__RESERVED_MASK                                                                   0xC0000000L
-//RLC_SPM_UTCL1_CNTL
-#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                       0x0
-#define RLC_SPM_UTCL1_CNTL__DROP_MODE__SHIFT                                                                  0x18
-#define RLC_SPM_UTCL1_CNTL__BYPASS__SHIFT                                                                     0x19
-#define RLC_SPM_UTCL1_CNTL__INVALIDATE__SHIFT                                                                 0x1a
-#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                            0x1b
-#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                0x1c
-#define RLC_SPM_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                        0x1d
-#define RLC_SPM_UTCL1_CNTL__RESERVED__SHIFT                                                                   0x1e
-#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                         0x000FFFFFL
-#define RLC_SPM_UTCL1_CNTL__DROP_MODE_MASK                                                                    0x01000000L
-#define RLC_SPM_UTCL1_CNTL__BYPASS_MASK                                                                       0x02000000L
-#define RLC_SPM_UTCL1_CNTL__INVALIDATE_MASK                                                                   0x04000000L
-#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                              0x08000000L
-#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                  0x10000000L
-#define RLC_SPM_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                          0x20000000L
-#define RLC_SPM_UTCL1_CNTL__RESERVED_MASK                                                                     0xC0000000L
-//RLC_UTCL1_STATUS_2
-#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY__SHIFT                                                         0x0
-#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY__SHIFT                                                         0x1
-#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY__SHIFT                                                         0x2
-#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY__SHIFT                                                             0x3
-#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_BUSY__SHIFT                                                       0x4
-#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans__SHIFT                                                 0x5
-#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans__SHIFT                                                 0x6
-#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans__SHIFT                                                 0x7
-#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans__SHIFT                                                     0x8
-#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_StallOnTrans__SHIFT                                               0x9
-#define RLC_UTCL1_STATUS_2__RESERVED__SHIFT                                                                   0xa
-#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY_MASK                                                           0x00000001L
-#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY_MASK                                                           0x00000002L
-#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY_MASK                                                           0x00000004L
-#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY_MASK                                                               0x00000008L
-#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_BUSY_MASK                                                         0x00000010L
-#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans_MASK                                                   0x00000020L
-#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans_MASK                                                   0x00000040L
-#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans_MASK                                                   0x00000080L
-#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans_MASK                                                       0x00000100L
-#define RLC_UTCL1_STATUS_2__PREWALKER_UTCL1_StallOnTrans_MASK                                                 0x00000200L
-#define RLC_UTCL1_STATUS_2__RESERVED_MASK                                                                     0xFFFFFC00L
-//RLC_LB_THR_CONFIG_2
-#define RLC_LB_THR_CONFIG_2__DATA__SHIFT                                                                      0x0
-#define RLC_LB_THR_CONFIG_2__DATA_MASK                                                                        0xFFFFFFFFL
-//RLC_LB_THR_CONFIG_3
-#define RLC_LB_THR_CONFIG_3__DATA__SHIFT                                                                      0x0
-#define RLC_LB_THR_CONFIG_3__DATA_MASK                                                                        0xFFFFFFFFL
-//RLC_LB_THR_CONFIG_4
-#define RLC_LB_THR_CONFIG_4__DATA__SHIFT                                                                      0x0
-#define RLC_LB_THR_CONFIG_4__DATA_MASK                                                                        0xFFFFFFFFL
-//RLC_SPM_UTCL1_ERROR_1
-#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError__SHIFT                                                     0x0
-#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                                 0x2
-#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                             0x6
-#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError_MASK                                                       0x00000003L
-#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid_MASK                                                   0x0000003CL
-#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                               0x000003C0L
-//RLC_SPM_UTCL1_ERROR_2
-#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
-#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
-//RLC_GPM_UTCL1_TH0_ERROR_1
-#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
-#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
-#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
-#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
-#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
-#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
-//RLC_LB_THR_CONFIG_1
-#define RLC_LB_THR_CONFIG_1__DATA__SHIFT                                                                      0x0
-#define RLC_LB_THR_CONFIG_1__DATA_MASK                                                                        0xFFFFFFFFL
-//RLC_GPM_UTCL1_TH0_ERROR_2
-#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
-#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
-//RLC_GPM_UTCL1_TH1_ERROR_1
-#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
-#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
-#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
-#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
-#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
-#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
-//RLC_GPM_UTCL1_TH1_ERROR_2
-#define RLC_GPM_UTCL1_TH1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
-#define RLC_GPM_UTCL1_TH1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
-//RLC_GPM_UTCL1_TH2_ERROR_1
-#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
-#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
-#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
-#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
-#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
-#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
-//RLC_GPM_UTCL1_TH2_ERROR_2
-#define RLC_GPM_UTCL1_TH2_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
-#define RLC_GPM_UTCL1_TH2_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
-//RLC_CGCG_CGLS_CTRL_3D
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_EN__SHIFT                                                                 0x0
-#define RLC_CGCG_CGLS_CTRL_3D__CGLS_EN__SHIFT                                                                 0x1
-#define RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT                                                0x2
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT                                                 0x8
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_CONTROLLER__SHIFT                                                         0x1b
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_REG_CTRL__SHIFT                                                           0x1c
-#define RLC_CGCG_CGLS_CTRL_3D__SLEEP_MODE__SHIFT                                                              0x1d
-#define RLC_CGCG_CGLS_CTRL_3D__SIM_SILICON_EN__SHIFT                                                          0x1f
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK                                                                   0x00000001L
-#define RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK                                                                   0x00000002L
-#define RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK                                                  0x000000FCL
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK                                                   0x07FFFF00L
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_CONTROLLER_MASK                                                           0x08000000L
-#define RLC_CGCG_CGLS_CTRL_3D__CGCG_REG_CTRL_MASK                                                             0x10000000L
-#define RLC_CGCG_CGLS_CTRL_3D__SLEEP_MODE_MASK                                                                0x60000000L
-#define RLC_CGCG_CGLS_CTRL_3D__SIM_SILICON_EN_MASK                                                            0x80000000L
-//RLC_CGCG_RAMP_CTRL_3D
-#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_START_UNIT__SHIFT                                                     0x0
-#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_STEP_UNIT__SHIFT                                                      0x4
-#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_START_UNIT__SHIFT                                                       0x8
-#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_STEP_UNIT__SHIFT                                                        0xc
-#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_CNT__SHIFT                                                          0x10
-#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_UNIT__SHIFT                                                         0x1c
-#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_START_UNIT_MASK                                                       0x0000000FL
-#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_STEP_UNIT_MASK                                                        0x000000F0L
-#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_START_UNIT_MASK                                                         0x00000F00L
-#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_STEP_UNIT_MASK                                                          0x0000F000L
-#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_CNT_MASK                                                            0x0FFF0000L
-#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_UNIT_MASK                                                           0xF0000000L
-//RLC_SEMAPHORE_0
-#define RLC_SEMAPHORE_0__CLIENT_ID__SHIFT                                                                     0x0
-#define RLC_SEMAPHORE_0__RESERVED__SHIFT                                                                      0x5
-#define RLC_SEMAPHORE_0__CLIENT_ID_MASK                                                                       0x0000001FL
-#define RLC_SEMAPHORE_0__RESERVED_MASK                                                                        0xFFFFFFE0L
-//RLC_SEMAPHORE_1
-#define RLC_SEMAPHORE_1__CLIENT_ID__SHIFT                                                                     0x0
-#define RLC_SEMAPHORE_1__RESERVED__SHIFT                                                                      0x5
-#define RLC_SEMAPHORE_1__CLIENT_ID_MASK                                                                       0x0000001FL
-#define RLC_SEMAPHORE_1__RESERVED_MASK                                                                        0xFFFFFFE0L
-//RLC_CP_EOF_INT
-#define RLC_CP_EOF_INT__INTERRUPT__SHIFT                                                                      0x0
-#define RLC_CP_EOF_INT__RESERVED__SHIFT                                                                       0x1
-#define RLC_CP_EOF_INT__INTERRUPT_MASK                                                                        0x00000001L
-#define RLC_CP_EOF_INT__RESERVED_MASK                                                                         0xFFFFFFFEL
-//RLC_CP_EOF_INT_CNT
-#define RLC_CP_EOF_INT_CNT__CNT__SHIFT                                                                        0x0
-#define RLC_CP_EOF_INT_CNT__CNT_MASK                                                                          0xFFFFFFFFL
-//RLC_SPARE_INT
-#define RLC_SPARE_INT__INTERRUPT__SHIFT                                                                       0x0
-#define RLC_SPARE_INT__RESERVED__SHIFT                                                                        0x1
-#define RLC_SPARE_INT__INTERRUPT_MASK                                                                         0x00000001L
-#define RLC_SPARE_INT__RESERVED_MASK                                                                          0xFFFFFFFEL
-//RLC_PREWALKER_UTCL1_CNTL
-#define RLC_PREWALKER_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                 0x0
-#define RLC_PREWALKER_UTCL1_CNTL__DROP_MODE__SHIFT                                                            0x18
-#define RLC_PREWALKER_UTCL1_CNTL__BYPASS__SHIFT                                                               0x19
-#define RLC_PREWALKER_UTCL1_CNTL__INVALIDATE__SHIFT                                                           0x1a
-#define RLC_PREWALKER_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                      0x1b
-#define RLC_PREWALKER_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                          0x1c
-#define RLC_PREWALKER_UTCL1_CNTL__FORCE_SD_VMID_DIRTY__SHIFT                                                  0x1d
-#define RLC_PREWALKER_UTCL1_CNTL__RESERVED__SHIFT                                                             0x1e
-#define RLC_PREWALKER_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                   0x000FFFFFL
-#define RLC_PREWALKER_UTCL1_CNTL__DROP_MODE_MASK                                                              0x01000000L
-#define RLC_PREWALKER_UTCL1_CNTL__BYPASS_MASK                                                                 0x02000000L
-#define RLC_PREWALKER_UTCL1_CNTL__INVALIDATE_MASK                                                             0x04000000L
-#define RLC_PREWALKER_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                        0x08000000L
-#define RLC_PREWALKER_UTCL1_CNTL__FORCE_SNOOP_MASK                                                            0x10000000L
-#define RLC_PREWALKER_UTCL1_CNTL__FORCE_SD_VMID_DIRTY_MASK                                                    0x20000000L
-#define RLC_PREWALKER_UTCL1_CNTL__RESERVED_MASK                                                               0xC0000000L
-//RLC_PREWALKER_UTCL1_TRIG
-#define RLC_PREWALKER_UTCL1_TRIG__VALID__SHIFT                                                                0x0
-#define RLC_PREWALKER_UTCL1_TRIG__VMID__SHIFT                                                                 0x1
-#define RLC_PREWALKER_UTCL1_TRIG__PRIME_MODE__SHIFT                                                           0x5
-#define RLC_PREWALKER_UTCL1_TRIG__READ_PERM__SHIFT                                                            0x6
-#define RLC_PREWALKER_UTCL1_TRIG__WRITE_PERM__SHIFT                                                           0x7
-#define RLC_PREWALKER_UTCL1_TRIG__EXEC_PERM__SHIFT                                                            0x8
-#define RLC_PREWALKER_UTCL1_TRIG__RESERVED__SHIFT                                                             0x9
-#define RLC_PREWALKER_UTCL1_TRIG__READY__SHIFT                                                                0x1f
-#define RLC_PREWALKER_UTCL1_TRIG__VALID_MASK                                                                  0x00000001L
-#define RLC_PREWALKER_UTCL1_TRIG__VMID_MASK                                                                   0x0000001EL
-#define RLC_PREWALKER_UTCL1_TRIG__PRIME_MODE_MASK                                                             0x00000020L
-#define RLC_PREWALKER_UTCL1_TRIG__READ_PERM_MASK                                                              0x00000040L
-#define RLC_PREWALKER_UTCL1_TRIG__WRITE_PERM_MASK                                                             0x00000080L
-#define RLC_PREWALKER_UTCL1_TRIG__EXEC_PERM_MASK                                                              0x00000100L
-#define RLC_PREWALKER_UTCL1_TRIG__RESERVED_MASK                                                               0x7FFFFE00L
-#define RLC_PREWALKER_UTCL1_TRIG__READY_MASK                                                                  0x80000000L
-//RLC_PREWALKER_UTCL1_ADDR_LSB
-#define RLC_PREWALKER_UTCL1_ADDR_LSB__ADDR_LSB__SHIFT                                                         0x0
-#define RLC_PREWALKER_UTCL1_ADDR_LSB__ADDR_LSB_MASK                                                           0xFFFFFFFFL
-//RLC_PREWALKER_UTCL1_ADDR_MSB
-#define RLC_PREWALKER_UTCL1_ADDR_MSB__ADDR_MSB__SHIFT                                                         0x0
-#define RLC_PREWALKER_UTCL1_ADDR_MSB__ADDR_MSB_MASK                                                           0x0000FFFFL
-//RLC_PREWALKER_UTCL1_SIZE_LSB
-#define RLC_PREWALKER_UTCL1_SIZE_LSB__SIZE_LSB__SHIFT                                                         0x0
-#define RLC_PREWALKER_UTCL1_SIZE_LSB__SIZE_LSB_MASK                                                           0xFFFFFFFFL
-//RLC_PREWALKER_UTCL1_SIZE_MSB
-#define RLC_PREWALKER_UTCL1_SIZE_MSB__SIZE_MSB__SHIFT                                                         0x0
-#define RLC_PREWALKER_UTCL1_SIZE_MSB__SIZE_MSB_MASK                                                           0x00000003L
-//RLC_DSM_TRIG
-//RLC_UTCL1_STATUS
-#define RLC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
-#define RLC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
-#define RLC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
-#define RLC_UTCL1_STATUS__RESERVED__SHIFT                                                                     0x3
-#define RLC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
-#define RLC_UTCL1_STATUS__RESERVED_1__SHIFT                                                                   0xe
-#define RLC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
-#define RLC_UTCL1_STATUS__RESERVED_2__SHIFT                                                                   0x16
-#define RLC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
-#define RLC_UTCL1_STATUS__RESERVED_3__SHIFT                                                                   0x1e
-#define RLC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
-#define RLC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
-#define RLC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
-#define RLC_UTCL1_STATUS__RESERVED_MASK                                                                       0x000000F8L
-#define RLC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
-#define RLC_UTCL1_STATUS__RESERVED_1_MASK                                                                     0x0000C000L
-#define RLC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
-#define RLC_UTCL1_STATUS__RESERVED_2_MASK                                                                     0x00C00000L
-#define RLC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
-#define RLC_UTCL1_STATUS__RESERVED_3_MASK                                                                     0xC0000000L
-//RLC_R2I_CNTL_0
-#define RLC_R2I_CNTL_0__Data__SHIFT                                                                           0x0
-#define RLC_R2I_CNTL_0__Data_MASK                                                                             0xFFFFFFFFL
-//RLC_R2I_CNTL_1
-#define RLC_R2I_CNTL_1__Data__SHIFT                                                                           0x0
-#define RLC_R2I_CNTL_1__Data_MASK                                                                             0xFFFFFFFFL
-//RLC_R2I_CNTL_2
-#define RLC_R2I_CNTL_2__Data__SHIFT                                                                           0x0
-#define RLC_R2I_CNTL_2__Data_MASK                                                                             0xFFFFFFFFL
-//RLC_R2I_CNTL_3
-#define RLC_R2I_CNTL_3__Data__SHIFT                                                                           0x0
-#define RLC_R2I_CNTL_3__Data_MASK                                                                             0xFFFFFFFFL
-//RLC_UTCL2_CNTL
-#define RLC_UTCL2_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x0
-#define RLC_UTCL2_CNTL__RESERVED__SHIFT                                                                       0x1
-#define RLC_UTCL2_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x00000001L
-#define RLC_UTCL2_CNTL__RESERVED_MASK                                                                         0xFFFFFFFEL
-//RLC_LBPW_CU_STAT
-#define RLC_LBPW_CU_STAT__MAX_CU__SHIFT                                                                       0x0
-#define RLC_LBPW_CU_STAT__ON_CU__SHIFT                                                                        0x10
-#define RLC_LBPW_CU_STAT__MAX_CU_MASK                                                                         0x0000FFFFL
-#define RLC_LBPW_CU_STAT__ON_CU_MASK                                                                          0xFFFF0000L
-//RLC_DS_CNTL
-#define RLC_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK__SHIFT                                                          0x0
-#define RLC_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK__SHIFT                                                           0x1
-#define RLC_DS_CNTL__RESRVED__SHIFT                                                                           0x2
-#define RLC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK__SHIFT                                                          0x10
-#define RLC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK__SHIFT                                                           0x11
-#define RLC_DS_CNTL__RESRVED_1__SHIFT                                                                         0x12
-#define RLC_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK_MASK                                                            0x00000001L
-#define RLC_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK_MASK                                                             0x00000002L
-#define RLC_DS_CNTL__RESRVED_MASK                                                                             0x0000FFFCL
-#define RLC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK_MASK                                                            0x00010000L
-#define RLC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK_MASK                                                             0x00020000L
-#define RLC_DS_CNTL__RESRVED_1_MASK                                                                           0xFFFC0000L
-//RLC_RLCV_SPARE_INT
-#define RLC_RLCV_SPARE_INT__INTERRUPT__SHIFT                                                                  0x0
-#define RLC_RLCV_SPARE_INT__RESERVED__SHIFT                                                                   0x1
-#define RLC_RLCV_SPARE_INT__INTERRUPT_MASK                                                                    0x00000001L
-#define RLC_RLCV_SPARE_INT__RESERVED_MASK                                                                     0xFFFFFFFEL
-
-
-// addressBlock: gc_pwrdec
-//CGTS_SM_CTRL_REG
-#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY__SHIFT                                                                 0x0
-#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY__SHIFT                                                                0x4
-#define CGTS_SM_CTRL_REG__MGCG_ENABLED__SHIFT                                                                 0xc
-#define CGTS_SM_CTRL_REG__BASE_MODE__SHIFT                                                                    0x10
-#define CGTS_SM_CTRL_REG__SM_MODE__SHIFT                                                                      0x11
-#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE__SHIFT                                                               0x14
-#define CGTS_SM_CTRL_REG__OVERRIDE__SHIFT                                                                     0x15
-#define CGTS_SM_CTRL_REG__LS_OVERRIDE__SHIFT                                                                  0x16
-#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN__SHIFT                                                            0x17
-#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT                                                               0x18
-#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY_MASK                                                                   0x0000000FL
-#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY_MASK                                                                  0x00000FF0L
-#define CGTS_SM_CTRL_REG__MGCG_ENABLED_MASK                                                                   0x00001000L
-#define CGTS_SM_CTRL_REG__BASE_MODE_MASK                                                                      0x00010000L
-#define CGTS_SM_CTRL_REG__SM_MODE_MASK                                                                        0x000E0000L
-#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK                                                                 0x00100000L
-#define CGTS_SM_CTRL_REG__OVERRIDE_MASK                                                                       0x00200000L
-#define CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK                                                                    0x00400000L
-#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK                                                              0x00800000L
-#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_MASK                                                                 0xFF000000L
-//CGTS_RD_CTRL_REG
-#define CGTS_RD_CTRL_REG__ROW_MUX_SEL__SHIFT                                                                  0x0
-#define CGTS_RD_CTRL_REG__REG_MUX_SEL__SHIFT                                                                  0x8
-#define CGTS_RD_CTRL_REG__ROW_MUX_SEL_MASK                                                                    0x0000001FL
-#define CGTS_RD_CTRL_REG__REG_MUX_SEL_MASK                                                                    0x00001F00L
-//CGTS_RD_REG
-#define CGTS_RD_REG__READ_DATA__SHIFT                                                                         0x0
-#define CGTS_RD_REG__READ_DATA_MASK                                                                           0x00003FFFL
-//CGTS_TCC_DISABLE
-#define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT                                                                  0x10
-#define CGTS_TCC_DISABLE__TCC_DISABLE_MASK                                                                    0xFFFF0000L
-//CGTS_USER_TCC_DISABLE
-#define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT                                                             0x10
-#define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK                                                               0xFFFF0000L
-//CGTS_CU0_SP0_CTRL_REG
-#define CGTS_CU0_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU0_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU0_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU0_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU0_LDS_SQ_CTRL_REG
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU0_TA_SQC_CTRL_REG
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU0_SP1_CTRL_REG
-#define CGTS_CU0_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU0_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU0_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU0_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU0_TD_TCP_CTRL_REG
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU0_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU1_SP0_CTRL_REG
-#define CGTS_CU1_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU1_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU1_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU1_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU1_LDS_SQ_CTRL_REG
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU1_TA_SQC_CTRL_REG
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU1_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-//CGTS_CU1_SP1_CTRL_REG
-#define CGTS_CU1_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU1_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU1_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU1_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU1_TD_TCP_CTRL_REG
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU1_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU2_SP0_CTRL_REG
-#define CGTS_CU2_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU2_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU2_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU2_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU2_LDS_SQ_CTRL_REG
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU2_TA_SQC_CTRL_REG
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU2_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-//CGTS_CU2_SP1_CTRL_REG
-#define CGTS_CU2_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU2_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU2_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU2_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU2_TD_TCP_CTRL_REG
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU2_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU3_SP0_CTRL_REG
-#define CGTS_CU3_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU3_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU3_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU3_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU3_LDS_SQ_CTRL_REG
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU3_TA_SQC_CTRL_REG
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU3_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU3_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU3_SP1_CTRL_REG
-#define CGTS_CU3_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU3_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU3_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU3_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU3_TD_TCP_CTRL_REG
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU3_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU4_SP0_CTRL_REG
-#define CGTS_CU4_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU4_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU4_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU4_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU4_LDS_SQ_CTRL_REG
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU4_TA_SQC_CTRL_REG
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-//CGTS_CU4_SP1_CTRL_REG
-#define CGTS_CU4_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU4_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU4_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU4_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU4_TD_TCP_CTRL_REG
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU4_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU5_SP0_CTRL_REG
-#define CGTS_CU5_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU5_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU5_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU5_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU5_LDS_SQ_CTRL_REG
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU5_TA_SQC_CTRL_REG
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU5_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-//CGTS_CU5_SP1_CTRL_REG
-#define CGTS_CU5_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU5_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU5_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU5_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU5_TD_TCP_CTRL_REG
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU5_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU6_SP0_CTRL_REG
-#define CGTS_CU6_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU6_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU6_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU6_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU6_LDS_SQ_CTRL_REG
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU6_TA_SQC_CTRL_REG
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU6_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU6_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU6_SP1_CTRL_REG
-#define CGTS_CU6_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU6_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU6_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU6_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU6_TD_TCP_CTRL_REG
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU6_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU7_SP0_CTRL_REG
-#define CGTS_CU7_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU7_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU7_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU7_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU7_LDS_SQ_CTRL_REG
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU7_TA_SQC_CTRL_REG
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU7_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-//CGTS_CU7_SP1_CTRL_REG
-#define CGTS_CU7_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU7_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU7_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU7_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU7_TD_TCP_CTRL_REG
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU7_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU8_SP0_CTRL_REG
-#define CGTS_CU8_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU8_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU8_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU8_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU8_LDS_SQ_CTRL_REG
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU8_TA_SQC_CTRL_REG
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-//CGTS_CU8_SP1_CTRL_REG
-#define CGTS_CU8_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU8_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU8_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU8_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU8_TD_TCP_CTRL_REG
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU8_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU9_SP0_CTRL_REG
-#define CGTS_CU9_SP0_CTRL_REG__SP00__SHIFT                                                                    0x0
-#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU9_SP0_CTRL_REG__SP01__SHIFT                                                                    0x10
-#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU9_SP0_CTRL_REG__SP00_MASK                                                                      0x0000007FL
-#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU9_SP0_CTRL_REG__SP01_MASK                                                                      0x007F0000L
-#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU9_LDS_SQ_CTRL_REG
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                  0x0
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                   0x10
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_MASK                                                                    0x0000007FL
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_MASK                                                                     0x007F0000L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU9_TA_SQC_CTRL_REG
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA__SHIFT                                                                   0x0
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC__SHIFT                                                                  0x10
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_MASK                                                                     0x0000007FL
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU9_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_MASK                                                                    0x007F0000L
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU9_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU9_SP1_CTRL_REG
-#define CGTS_CU9_SP1_CTRL_REG__SP10__SHIFT                                                                    0x0
-#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                           0x7
-#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                      0x8
-#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                        0xa
-#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                  0xb
-#define CGTS_CU9_SP1_CTRL_REG__SP11__SHIFT                                                                    0x10
-#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                           0x17
-#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                      0x18
-#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                        0x1a
-#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                  0x1b
-#define CGTS_CU9_SP1_CTRL_REG__SP10_MASK                                                                      0x0000007FL
-#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                             0x00000080L
-#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                        0x00000300L
-#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                          0x00000400L
-#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                    0x00000800L
-#define CGTS_CU9_SP1_CTRL_REG__SP11_MASK                                                                      0x007F0000L
-#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                             0x00800000L
-#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                        0x03000000L
-#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                          0x04000000L
-#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                    0x08000000L
-//CGTS_CU9_TD_TCP_CTRL_REG
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD__SHIFT                                                                   0x0
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                 0x10
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_MASK                                                                     0x0000007FL
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_MASK                                                                   0x007F0000L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU9_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU10_SP0_CTRL_REG
-#define CGTS_CU10_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
-#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU10_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
-#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU10_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
-#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU10_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
-#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU10_LDS_SQ_CTRL_REG
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU10_TA_SQC_CTRL_REG
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU10_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-//CGTS_CU10_SP1_CTRL_REG
-#define CGTS_CU10_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
-#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU10_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
-#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU10_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
-#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU10_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
-#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU10_TD_TCP_CTRL_REG
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
-#define CGTS_CU10_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
-//CGTS_CU11_SP0_CTRL_REG
-#define CGTS_CU11_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
-#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU11_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
-#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU11_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
-#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU11_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
-#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU11_LDS_SQ_CTRL_REG
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU11_TA_SQC_CTRL_REG
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU11_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-//CGTS_CU11_SP1_CTRL_REG
-#define CGTS_CU11_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
-#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU11_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
-#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU11_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
-#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU11_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
-#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU11_TD_TCP_CTRL_REG
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
-#define CGTS_CU11_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
-//CGTS_CU12_SP0_CTRL_REG
-#define CGTS_CU12_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
-#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU12_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
-#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU12_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
-#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU12_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
-#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU12_LDS_SQ_CTRL_REG
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU12_TA_SQC_CTRL_REG
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC__SHIFT                                                                 0x10
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_MASK                                                                   0x007F0000L
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU12_SP1_CTRL_REG
-#define CGTS_CU12_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
-#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU12_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
-#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU12_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
-#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU12_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
-#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU12_TD_TCP_CTRL_REG
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
-#define CGTS_CU12_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
-//CGTS_CU13_SP0_CTRL_REG
-#define CGTS_CU13_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
-#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU13_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
-#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU13_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
-#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU13_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
-#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU13_LDS_SQ_CTRL_REG
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU13_TA_SQC_CTRL_REG
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU13_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-//CGTS_CU13_SP1_CTRL_REG
-#define CGTS_CU13_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
-#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU13_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
-#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU13_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
-#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU13_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
-#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU13_TD_TCP_CTRL_REG
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
-#define CGTS_CU13_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
-//CGTS_CU14_SP0_CTRL_REG
-#define CGTS_CU14_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
-#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU14_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
-#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU14_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
-#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU14_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
-#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU14_LDS_SQ_CTRL_REG
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU14_TA_SQC_CTRL_REG
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU14_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-//CGTS_CU14_SP1_CTRL_REG
-#define CGTS_CU14_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
-#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU14_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
-#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU14_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
-#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU14_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
-#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU14_TD_TCP_CTRL_REG
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
-#define CGTS_CU14_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
-//CGTS_CU15_SP0_CTRL_REG
-#define CGTS_CU15_SP0_CTRL_REG__SP00__SHIFT                                                                   0x0
-#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU15_SP0_CTRL_REG__SP01__SHIFT                                                                   0x10
-#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU15_SP0_CTRL_REG__SP00_MASK                                                                     0x0000007FL
-#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU15_SP0_CTRL_REG__SP01_MASK                                                                     0x007F0000L
-#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU15_LDS_SQ_CTRL_REG
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS__SHIFT                                                                 0x0
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT                                                        0x7
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT                                                   0x8
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT                                                     0xa
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT                                               0xb
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ__SHIFT                                                                  0x10
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT                                                         0x17
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT                                                    0x18
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT                                                      0x1a
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT                                                0x1b
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_MASK                                                                   0x0000007FL
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK                                                          0x00000080L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK                                                     0x00000300L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK                                                       0x00000400L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK                                                 0x00000800L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_MASK                                                                    0x007F0000L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK                                                           0x00800000L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK                                                      0x03000000L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK                                                        0x04000000L
-#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK                                                  0x08000000L
-//CGTS_CU15_TA_SQC_CTRL_REG
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA__SHIFT                                                                  0x0
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC__SHIFT                                                                 0x10
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT                                                        0x17
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT                                                   0x18
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT                                                     0x1a
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT                                               0x1b
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_MASK                                                                    0x0000007FL
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU15_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_MASK                                                                   0x007F0000L
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK                                                          0x00800000L
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK                                                     0x03000000L
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK                                                       0x04000000L
-#define CGTS_CU15_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK                                                 0x08000000L
-//CGTS_CU15_SP1_CTRL_REG
-#define CGTS_CU15_SP1_CTRL_REG__SP10__SHIFT                                                                   0x0
-#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU15_SP1_CTRL_REG__SP11__SHIFT                                                                   0x10
-#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT                                                          0x17
-#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT                                                     0x18
-#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT                                                       0x1a
-#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT                                                 0x1b
-#define CGTS_CU15_SP1_CTRL_REG__SP10_MASK                                                                     0x0000007FL
-#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU15_SP1_CTRL_REG__SP11_MASK                                                                     0x007F0000L
-#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE_MASK                                                            0x00800000L
-#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK                                                       0x03000000L
-#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK                                                         0x04000000L
-#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK                                                   0x08000000L
-//CGTS_CU15_TD_TCP_CTRL_REG
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD__SHIFT                                                                  0x0
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF__SHIFT                                                                0x10
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_OVERRIDE__SHIFT                                                       0x17
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE__SHIFT                                                  0x18
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE__SHIFT                                                    0x1a
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE__SHIFT                                              0x1b
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_MASK                                                                    0x0000007FL
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_MASK                                                                  0x007F0000L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_OVERRIDE_MASK                                                         0x00800000L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_BUSY_OVERRIDE_MASK                                                    0x03000000L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_LS_OVERRIDE_MASK                                                      0x04000000L
-#define CGTS_CU15_TD_TCP_CTRL_REG__TCPF_SIMDBUSY_OVERRIDE_MASK                                                0x08000000L
-//CGTS_CU0_TCPI_CTRL_REG
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU0_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU0_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU0_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU1_TCPI_CTRL_REG
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU1_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU1_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU1_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU2_TCPI_CTRL_REG
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU2_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU2_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU2_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU3_TCPI_CTRL_REG
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU3_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU3_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU3_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU4_TCPI_CTRL_REG
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU4_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU4_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU4_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU5_TCPI_CTRL_REG
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU5_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU5_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU5_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU6_TCPI_CTRL_REG
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU6_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU6_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU6_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU7_TCPI_CTRL_REG
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU7_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU7_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU7_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU8_TCPI_CTRL_REG
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU8_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU8_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU8_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU9_TCPI_CTRL_REG
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI__SHIFT                                                                   0x0
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                          0x7
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                     0x8
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                       0xa
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                 0xb
-#define CGTS_CU9_TCPI_CTRL_REG__RESERVED__SHIFT                                                               0xc
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_MASK                                                                     0x0000007FL
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                            0x00000080L
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                       0x00000300L
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                         0x00000400L
-#define CGTS_CU9_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                   0x00000800L
-#define CGTS_CU9_TCPI_CTRL_REG__RESERVED_MASK                                                                 0xFFFFF000L
-//CGTS_CU10_TCPI_CTRL_REG
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU10_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU10_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU10_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
-//CGTS_CU11_TCPI_CTRL_REG
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU11_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU11_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU11_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
-//CGTS_CU12_TCPI_CTRL_REG
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU12_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU12_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU12_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
-//CGTS_CU13_TCPI_CTRL_REG
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU13_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU13_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU13_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
-//CGTS_CU14_TCPI_CTRL_REG
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU14_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU14_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU14_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
-//CGTS_CU15_TCPI_CTRL_REG
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI__SHIFT                                                                  0x0
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_OVERRIDE__SHIFT                                                         0x7
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE__SHIFT                                                    0x8
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_LS_OVERRIDE__SHIFT                                                      0xa
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE__SHIFT                                                0xb
-#define CGTS_CU15_TCPI_CTRL_REG__RESERVED__SHIFT                                                              0xc
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_MASK                                                                    0x0000007FL
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_OVERRIDE_MASK                                                           0x00000080L
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_BUSY_OVERRIDE_MASK                                                      0x00000300L
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_LS_OVERRIDE_MASK                                                        0x00000400L
-#define CGTS_CU15_TCPI_CTRL_REG__TCPI_SIMDBUSY_OVERRIDE_MASK                                                  0x00000800L
-#define CGTS_CU15_TCPI_CTRL_REG__RESERVED_MASK                                                                0xFFFFF000L
-//CGTT_SPI_CLK_CTRL
-#define CGTT_SPI_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT                                                            0x12
-#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT                                                            0x18
-#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE__SHIFT                                                         0x1a
-#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE__SHIFT                                                               0x1b
-#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                               0x1c
-#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                               0x1d
-#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                               0x1e
-#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
-#define CGTT_SPI_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST_MASK                                                              0x00FC0000L
-#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE_MASK                                                              0x01000000L
-#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE_MASK                                                           0x04000000L
-#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE_MASK                                                                 0x08000000L
-#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE_MASK                                                                 0x10000000L
-#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE_MASK                                                                 0x20000000L
-#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE_MASK                                                                 0x40000000L
-#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
-//CGTT_PC_CLK_CTRL
-#define CGTT_PC_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
-#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
-#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT                                                             0x12
-#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT                                                             0x18
-#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE__SHIFT                                                     0x19
-#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE__SHIFT                                                      0x1a
-#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE__SHIFT                                                               0x1b
-#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE__SHIFT                                                               0x1c
-#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE__SHIFT                                                               0x1d
-#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE__SHIFT                                                               0x1e
-#define CGTT_PC_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
-#define CGTT_PC_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
-#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
-#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST_MASK                                                               0x00FC0000L
-#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE_MASK                                                               0x01000000L
-#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE_MASK                                                       0x02000000L
-#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE_MASK                                                        0x04000000L
-#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE_MASK                                                                 0x08000000L
-#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE_MASK                                                                 0x10000000L
-#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE_MASK                                                                 0x20000000L
-#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE_MASK                                                                 0x40000000L
-#define CGTT_PC_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
-//CGTT_BCI_CLK_CTRL
-#define CGTT_BCI_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_BCI_CLK_CTRL__RESERVED__SHIFT                                                                    0xc
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE__SHIFT                                                              0x18
-#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE__SHIFT                                                              0x19
-#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE__SHIFT                                                              0x1a
-#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE__SHIFT                                                              0x1b
-#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE__SHIFT                                                              0x1c
-#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE__SHIFT                                                              0x1d
-#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE__SHIFT                                                              0x1e
-#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
-#define CGTT_BCI_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_BCI_CLK_CTRL__RESERVED_MASK                                                                      0x0000F000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE_MASK                                                                0x01000000L
-#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE_MASK                                                                0x02000000L
-#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE_MASK                                                                0x04000000L
-#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE_MASK                                                                0x08000000L
-#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE_MASK                                                                0x10000000L
-#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE_MASK                                                                0x20000000L
-#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE_MASK                                                                0x40000000L
-#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
-//CGTT_VGT_CLK_CTRL
-#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT                                                                 0xf
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE9__SHIFT                                                              0x18
-#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE8__SHIFT                                                              0x19
-#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x1a
-#define CGTT_VGT_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT                                                            0x1b
-#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE__SHIFT                                                               0x1c
-#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE__SHIFT                                                                 0x1d
-#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1e
-#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
-#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK                                                                   0x00008000L
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE9_MASK                                                                0x01000000L
-#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE8_MASK                                                                0x02000000L
-#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x04000000L
-#define CGTT_VGT_CLK_CTRL__PRIMGEN_OVERRIDE_MASK                                                              0x08000000L
-#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE_MASK                                                                 0x10000000L
-#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE_MASK                                                                   0x20000000L
-#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x40000000L
-#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
-//CGTT_IA_CLK_CTRL
-#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
-#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
-#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0x19
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                               0x1b
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                               0x1c
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                               0x1d
-#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
-#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
-#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
-#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
-#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
-#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x02000000L
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                 0x08000000L
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                 0x10000000L
-#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                 0x20000000L
-#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
-#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
-//CGTT_WD_CLK_CTRL
-#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
-#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
-#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0xf
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
-#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE8__SHIFT                                                               0x19
-#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x1a
-#define CGTT_WD_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT                                                             0x1b
-#define CGTT_WD_CLK_CTRL__TESS_OVERRIDE__SHIFT                                                                0x1c
-#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1d
-#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT                                                          0x1e
-#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
-#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
-#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
-#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x00008000L
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
-#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
-#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE8_MASK                                                                 0x02000000L
-#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x04000000L
-#define CGTT_WD_CLK_CTRL__PRIMGEN_OVERRIDE_MASK                                                               0x08000000L
-#define CGTT_WD_CLK_CTRL__TESS_OVERRIDE_MASK                                                                  0x10000000L
-#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x20000000L
-#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK                                                            0x40000000L
-#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
-//CGTT_PA_CLK_CTRL
-#define CGTT_PA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
-#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                               0x19
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                               0x1a
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                               0x1b
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                               0x1c
-#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT                                                              0x1d
-#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT                                                              0x1e
-#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE__SHIFT                                                             0x1f
-#define CGTT_PA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
-#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
-#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                 0x02000000L
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                 0x04000000L
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                 0x08000000L
-#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                 0x10000000L
-#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK                                                                0x20000000L
-#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK                                                                0x40000000L
-#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE_MASK                                                               0x80000000L
-//CGTT_SC_CLK_CTRL0
-#define CGTT_SC_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE__SHIFT                                              0x10
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x11
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x12
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x13
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x14
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x15
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x16
-#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE__SHIFT                                                      0x17
-#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE__SHIFT                                                    0x18
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x19
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1a
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1b
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1c
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                              0x1d
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                              0x1e
-#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
-#define CGTT_SC_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE_MASK                                                0x00010000L
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK                                                          0x00020000L
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK                                                          0x00040000L
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK                                                          0x00080000L
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK                                                          0x00100000L
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK                                                          0x00200000L
-#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK                                                          0x00400000L
-#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE_MASK                                                        0x00800000L
-#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE_MASK                                                      0x01000000L
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x02000000L
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x04000000L
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x08000000L
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x10000000L
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                                0x20000000L
-#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                                0x40000000L
-#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
-//CGTT_SC_CLK_CTRL1
-#define CGTT_SC_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE__SHIFT                                              0x11
-#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE__SHIFT                                              0x12
-#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE__SHIFT                                     0x13
-#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE__SHIFT                                           0x14
-#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE__SHIFT                                            0x15
-#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE__SHIFT                                                      0x16
-#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE__SHIFT                                                    0x19
-#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE__SHIFT                                                    0x1a
-#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE__SHIFT                                           0x1b
-#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE__SHIFT                                                 0x1c
-#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE__SHIFT                                                  0x1d
-#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE__SHIFT                                                            0x1e
-#define CGTT_SC_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE_MASK                                                0x00020000L
-#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE_MASK                                                0x00040000L
-#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE_MASK                                       0x00080000L
-#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE_MASK                                             0x00100000L
-#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE_MASK                                              0x00200000L
-#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE_MASK                                                        0x00400000L
-#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE_MASK                                                      0x02000000L
-#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE_MASK                                                      0x04000000L
-#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE_MASK                                             0x08000000L
-#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE_MASK                                                   0x10000000L
-#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE_MASK                                                    0x20000000L
-#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE_MASK                                                              0x40000000L
-//CGTT_SQ_CLK_CTRL
-#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
-#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
-#define CGTT_SQ_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                             0x1d
-#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
-#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
-#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
-#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
-#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
-#define CGTT_SQ_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                               0x20000000L
-#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
-#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
-//CGTT_SQG_CLK_CTRL
-#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE__SHIFT                                                             0x1c
-#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                            0x1d
-#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1e
-#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
-#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE_MASK                                                               0x10000000L
-#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                              0x20000000L
-#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x40000000L
-#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
-//SQ_ALU_CLK_CTRL
-#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT                                                               0x0
-#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT                                                               0x10
-#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0_MASK                                                                 0x0000FFFFL
-#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1_MASK                                                                 0xFFFF0000L
-//SQ_TEX_CLK_CTRL
-#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT                                                               0x0
-#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT                                                               0x10
-#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0_MASK                                                                 0x0000FFFFL
-#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1_MASK                                                                 0xFFFF0000L
-//SQ_LDS_CLK_CTRL
-#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT                                                               0x0
-#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT                                                               0x10
-#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0_MASK                                                                 0x0000FFFFL
-#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1_MASK                                                                 0xFFFF0000L
-//SQ_POWER_THROTTLE
-#define SQ_POWER_THROTTLE__MIN_POWER__SHIFT                                                                   0x0
-#define SQ_POWER_THROTTLE__MAX_POWER__SHIFT                                                                   0x10
-#define SQ_POWER_THROTTLE__PHASE_OFFSET__SHIFT                                                                0x1e
-#define SQ_POWER_THROTTLE__MIN_POWER_MASK                                                                     0x00003FFFL
-#define SQ_POWER_THROTTLE__MAX_POWER_MASK                                                                     0x3FFF0000L
-#define SQ_POWER_THROTTLE__PHASE_OFFSET_MASK                                                                  0xC0000000L
-//SQ_POWER_THROTTLE2
-#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA__SHIFT                                                            0x0
-#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                   0x10
-#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                   0x1b
-#define SQ_POWER_THROTTLE2__USE_REF_CLOCK__SHIFT                                                              0x1f
-#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA_MASK                                                              0x00003FFFL
-#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE_MASK                                                     0x03FF0000L
-#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO_MASK                                                     0x78000000L
-#define SQ_POWER_THROTTLE2__USE_REF_CLOCK_MASK                                                                0x80000000L
-//CGTT_SX_CLK_CTRL0
-#define CGTT_SX_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SX_CLK_CTRL0__RESERVED__SHIFT                                                                    0xc
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT                                                              0x18
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CGTT_SX_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SX_CLK_CTRL0__RESERVED_MASK                                                                      0x0000F000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7_MASK                                                                0x01000000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//CGTT_SX_CLK_CTRL1
-#define CGTT_SX_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SX_CLK_CTRL1__RESERVED__SHIFT                                                                    0xc
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CGTT_SX_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SX_CLK_CTRL1__RESERVED_MASK                                                                      0x0000F000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//CGTT_SX_CLK_CTRL2
-#define CGTT_SX_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SX_CLK_CTRL2__RESERVED__SHIFT                                                                    0xd
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CGTT_SX_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SX_CLK_CTRL2__RESERVED_MASK                                                                      0x0000E000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//CGTT_SX_CLK_CTRL3
-#define CGTT_SX_CLK_CTRL3__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SX_CLK_CTRL3__RESERVED__SHIFT                                                                    0xd
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CGTT_SX_CLK_CTRL3__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SX_CLK_CTRL3__RESERVED_MASK                                                                      0x0000E000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//CGTT_SX_CLK_CTRL4
-#define CGTT_SX_CLK_CTRL4__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_SX_CLK_CTRL4__RESERVED__SHIFT                                                                    0xc
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CGTT_SX_CLK_CTRL4__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_SX_CLK_CTRL4__RESERVED_MASK                                                                      0x0000F000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//TD_CGTT_CTRL
-#define TD_CGTT_CTRL__ON_DELAY__SHIFT                                                                         0x0
-#define TD_CGTT_CTRL__OFF_HYSTERESIS__SHIFT                                                                   0x4
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                             0x10
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                             0x11
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                             0x12
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                             0x13
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                             0x14
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                             0x15
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                             0x16
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                             0x17
-#define TD_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT                                                                   0x18
-#define TD_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT                                                                   0x19
-#define TD_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT                                                                   0x1a
-#define TD_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT                                                                   0x1b
-#define TD_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT                                                                   0x1c
-#define TD_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT                                                                   0x1d
-#define TD_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT                                                                   0x1e
-#define TD_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT                                                                   0x1f
-#define TD_CGTT_CTRL__ON_DELAY_MASK                                                                           0x0000000FL
-#define TD_CGTT_CTRL__OFF_HYSTERESIS_MASK                                                                     0x00000FF0L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                               0x00010000L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                               0x00020000L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                               0x00040000L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                               0x00080000L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                               0x00100000L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                               0x00200000L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                               0x00400000L
-#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                               0x00800000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE7_MASK                                                                     0x01000000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE6_MASK                                                                     0x02000000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE5_MASK                                                                     0x04000000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE4_MASK                                                                     0x08000000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE3_MASK                                                                     0x10000000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE2_MASK                                                                     0x20000000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE1_MASK                                                                     0x40000000L
-#define TD_CGTT_CTRL__SOFT_OVERRIDE0_MASK                                                                     0x80000000L
-//TA_CGTT_CTRL
-#define TA_CGTT_CTRL__ON_DELAY__SHIFT                                                                         0x0
-#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT                                                                   0x4
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                             0x10
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                             0x11
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                             0x12
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                             0x13
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                             0x14
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                             0x15
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                             0x16
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                             0x17
-#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT                                                                   0x18
-#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT                                                                   0x19
-#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT                                                                   0x1a
-#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT                                                                   0x1b
-#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT                                                                   0x1c
-#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT                                                                   0x1d
-#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT                                                                   0x1e
-#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT                                                                   0x1f
-#define TA_CGTT_CTRL__ON_DELAY_MASK                                                                           0x0000000FL
-#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK                                                                     0x00000FF0L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                               0x00010000L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                               0x00020000L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                               0x00040000L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                               0x00080000L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                               0x00100000L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                               0x00200000L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                               0x00400000L
-#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                               0x00800000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK                                                                     0x01000000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK                                                                     0x02000000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK                                                                     0x04000000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK                                                                     0x08000000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK                                                                     0x10000000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK                                                                     0x20000000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK                                                                     0x40000000L
-#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK                                                                     0x80000000L
-//CGTT_TCPI_CLK_CTRL
-#define CGTT_TCPI_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
-#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define CGTT_TCPI_CLK_CTRL__SPARE__SHIFT                                                                      0xc
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x18
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
-#define CGTT_TCPI_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
-#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define CGTT_TCPI_CLK_CTRL__SPARE_MASK                                                                        0x0000F000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x01000000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
-#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
-//CGTT_TCI_CLK_CTRL
-#define CGTT_TCI_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x18
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CGTT_TCI_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_TCI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x01000000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//CGTT_GDS_CLK_CTRL
-#define CGTT_GDS_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x18
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CGTT_GDS_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x01000000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//DB_CGTT_CLK_CTRL_0
-#define DB_CGTT_CLK_CTRL_0__ON_DELAY__SHIFT                                                                   0x0
-#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT                                                                   0xc
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT                                                             0x18
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT                                                             0x19
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT                                                             0x1a
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT                                                             0x1b
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT                                                             0x1c
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT                                                             0x1d
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT                                                             0x1e
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT                                                             0x1f
-#define DB_CGTT_CLK_CTRL_0__ON_DELAY_MASK                                                                     0x0000000FL
-#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK                                                                     0x0000F000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK                                                               0x01000000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK                                                               0x02000000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK                                                               0x04000000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK                                                               0x08000000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK                                                               0x10000000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK                                                               0x20000000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK                                                               0x40000000L
-#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK                                                               0x80000000L
-//CB_CGTT_SCLK_CTRL
-#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x18
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                              0x19
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                              0x1a
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                              0x1b
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                              0x1c
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                              0x1d
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                              0x1e
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                              0x1f
-#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x01000000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                                0x02000000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                                0x04000000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                                0x08000000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                                0x10000000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                                0x20000000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                                0x40000000L
-#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                                0x80000000L
-//TCC_CGTT_SCLK_CTRL
-#define TCC_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
-#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x18
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
-#define TCC_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
-#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x01000000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
-#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
-//TCA_CGTT_SCLK_CTRL
-#define TCA_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
-#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x18
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
-#define TCA_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
-#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x01000000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
-#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
-//CGTT_CP_CLK_CTRL
-#define CGTT_CP_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
-#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
-#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                                0xf
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
-#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                        0x1d
-#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                            0x1e
-#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                            0x1f
-#define CGTT_CP_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
-#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
-#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                  0x00008000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
-#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
-#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                          0x20000000L
-#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                              0x40000000L
-#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                              0x80000000L
-//CGTT_CPF_CLK_CTRL
-#define CGTT_CPF_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
-#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
-#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
-#define CGTT_CPF_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
-#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
-#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
-//CGTT_CPC_CLK_CTRL
-#define CGTT_CPC_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
-#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
-#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
-#define CGTT_CPC_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
-#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
-#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
-//RLC_PWR_CTRL
-#define RLC_PWR_CTRL__MON_CGPG_RTN_EN__SHIFT                                                                  0x0
-#define RLC_PWR_CTRL__RESERVED__SHIFT                                                                         0x1
-#define RLC_PWR_CTRL__DLDO_STATUS__SHIFT                                                                      0x8
-#define RLC_PWR_CTRL__MON_CGPG_RTN_EN_MASK                                                                    0x00000001L
-#define RLC_PWR_CTRL__RESERVED_MASK                                                                           0x000000FEL
-#define RLC_PWR_CTRL__DLDO_STATUS_MASK                                                                        0x00000100L
-//CGTT_RLC_CLK_CTRL
-#define CGTT_RLC_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
-#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
-#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
-#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
-#define CGTT_RLC_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
-#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
-#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
-#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
-#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
-//RLC_GFX_RM_CNTL
-#define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID__SHIFT                                                              0x0
-#define RLC_GFX_RM_CNTL__RESERVED__SHIFT                                                                      0x1
-#define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID_MASK                                                                0x00000001L
-#define RLC_GFX_RM_CNTL__RESERVED_MASK                                                                        0xFFFFFFFEL
-//RMI_CGTT_SCLK_CTRL
-#define RMI_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
-#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
-#define RMI_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
-#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
-#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
-//CGTT_TCPF_CLK_CTRL
-#define CGTT_TCPF_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
-#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define CGTT_TCPF_CLK_CTRL__SPARE__SHIFT                                                                      0xc
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x18
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
-#define CGTT_TCPF_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
-#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define CGTT_TCPF_CLK_CTRL__SPARE_MASK                                                                        0x0000F000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x01000000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
-#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
-
-
-// addressBlock: gc_ea_pwrdec
-//GCEA_CGTT_CLK_CTRL
-#define GCEA_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
-#define GCEA_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                        0x16
-#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                       0x1e
-#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                     0x1f
-#define GCEA_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
-#define GCEA_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                          0x00400000L
-#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                         0x40000000L
-#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                       0x80000000L
-
-
-// addressBlock: gc_utcl2_vmsharedhvdec
-//MC_VM_FB_SIZE_OFFSET_VF0
-#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF1
-#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF2
-#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF3
-#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF4
-#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF5
-#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF6
-#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF7
-#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF8
-#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF9
-#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE__SHIFT                                                           0x0
-#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET__SHIFT                                                         0x10
-#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE_MASK                                                             0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET_MASK                                                           0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF10
-#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE__SHIFT                                                          0x0
-#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET__SHIFT                                                        0x10
-#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE_MASK                                                            0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF11
-#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE__SHIFT                                                          0x0
-#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET__SHIFT                                                        0x10
-#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE_MASK                                                            0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF12
-#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE__SHIFT                                                          0x0
-#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET__SHIFT                                                        0x10
-#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE_MASK                                                            0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF13
-#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE__SHIFT                                                          0x0
-#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET__SHIFT                                                        0x10
-#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE_MASK                                                            0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF14
-#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE__SHIFT                                                          0x0
-#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET__SHIFT                                                        0x10
-#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE_MASK                                                            0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
-//MC_VM_FB_SIZE_OFFSET_VF15
-#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE__SHIFT                                                          0x0
-#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET__SHIFT                                                        0x10
-#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE_MASK                                                            0x0000FFFFL
-#define MC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET_MASK                                                          0xFFFF0000L
-//VM_IOMMU_MMIO_CNTRL_1
-#define VM_IOMMU_MMIO_CNTRL_1__MARC_EN__SHIFT                                                                 0x8
-#define VM_IOMMU_MMIO_CNTRL_1__MARC_EN_MASK                                                                   0x00000100L
-//MC_VM_MARC_BASE_LO_0
-#define MC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0__SHIFT                                                           0xc
-#define MC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0_MASK                                                             0xFFFFF000L
-//MC_VM_MARC_BASE_LO_1
-#define MC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1__SHIFT                                                           0xc
-#define MC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1_MASK                                                             0xFFFFF000L
-//MC_VM_MARC_BASE_LO_2
-#define MC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2__SHIFT                                                           0xc
-#define MC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2_MASK                                                             0xFFFFF000L
-//MC_VM_MARC_BASE_LO_3
-#define MC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3__SHIFT                                                           0xc
-#define MC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3_MASK                                                             0xFFFFF000L
-//MC_VM_MARC_BASE_HI_0
-#define MC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0__SHIFT                                                           0x0
-#define MC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0_MASK                                                             0x000FFFFFL
-//MC_VM_MARC_BASE_HI_1
-#define MC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1__SHIFT                                                           0x0
-#define MC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1_MASK                                                             0x000FFFFFL
-//MC_VM_MARC_BASE_HI_2
-#define MC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2__SHIFT                                                           0x0
-#define MC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2_MASK                                                             0x000FFFFFL
-//MC_VM_MARC_BASE_HI_3
-#define MC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3__SHIFT                                                           0x0
-#define MC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3_MASK                                                             0x000FFFFFL
-//MC_VM_MARC_RELOC_LO_0
-#define MC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0__SHIFT                                                           0x0
-#define MC_VM_MARC_RELOC_LO_0__MARC_READONLY_0__SHIFT                                                         0x1
-#define MC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0__SHIFT                                                         0xc
-#define MC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0_MASK                                                             0x00000001L
-#define MC_VM_MARC_RELOC_LO_0__MARC_READONLY_0_MASK                                                           0x00000002L
-#define MC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0_MASK                                                           0xFFFFF000L
-//MC_VM_MARC_RELOC_LO_1
-#define MC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1__SHIFT                                                           0x0
-#define MC_VM_MARC_RELOC_LO_1__MARC_READONLY_1__SHIFT                                                         0x1
-#define MC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1__SHIFT                                                         0xc
-#define MC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1_MASK                                                             0x00000001L
-#define MC_VM_MARC_RELOC_LO_1__MARC_READONLY_1_MASK                                                           0x00000002L
-#define MC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1_MASK                                                           0xFFFFF000L
-//MC_VM_MARC_RELOC_LO_2
-#define MC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2__SHIFT                                                           0x0
-#define MC_VM_MARC_RELOC_LO_2__MARC_READONLY_2__SHIFT                                                         0x1
-#define MC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2__SHIFT                                                         0xc
-#define MC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2_MASK                                                             0x00000001L
-#define MC_VM_MARC_RELOC_LO_2__MARC_READONLY_2_MASK                                                           0x00000002L
-#define MC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2_MASK                                                           0xFFFFF000L
-//MC_VM_MARC_RELOC_LO_3
-#define MC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3__SHIFT                                                           0x0
-#define MC_VM_MARC_RELOC_LO_3__MARC_READONLY_3__SHIFT                                                         0x1
-#define MC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3__SHIFT                                                         0xc
-#define MC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3_MASK                                                             0x00000001L
-#define MC_VM_MARC_RELOC_LO_3__MARC_READONLY_3_MASK                                                           0x00000002L
-#define MC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3_MASK                                                           0xFFFFF000L
-//MC_VM_MARC_RELOC_HI_0
-#define MC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0__SHIFT                                                         0x0
-#define MC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0_MASK                                                           0x000FFFFFL
-//MC_VM_MARC_RELOC_HI_1
-#define MC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1__SHIFT                                                         0x0
-#define MC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1_MASK                                                           0x000FFFFFL
-//MC_VM_MARC_RELOC_HI_2
-#define MC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2__SHIFT                                                         0x0
-#define MC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2_MASK                                                           0x000FFFFFL
-//MC_VM_MARC_RELOC_HI_3
-#define MC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3__SHIFT                                                         0x0
-#define MC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3_MASK                                                           0x000FFFFFL
-//MC_VM_MARC_LEN_LO_0
-#define MC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0__SHIFT                                                             0xc
-#define MC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0_MASK                                                               0xFFFFF000L
-//MC_VM_MARC_LEN_LO_1
-#define MC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1__SHIFT                                                             0xc
-#define MC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1_MASK                                                               0xFFFFF000L
-//MC_VM_MARC_LEN_LO_2
-#define MC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2__SHIFT                                                             0xc
-#define MC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2_MASK                                                               0xFFFFF000L
-//MC_VM_MARC_LEN_LO_3
-#define MC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3__SHIFT                                                             0xc
-#define MC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3_MASK                                                               0xFFFFF000L
-//MC_VM_MARC_LEN_HI_0
-#define MC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0__SHIFT                                                             0x0
-#define MC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0_MASK                                                               0x000FFFFFL
-//MC_VM_MARC_LEN_HI_1
-#define MC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1__SHIFT                                                             0x0
-#define MC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1_MASK                                                               0x000FFFFFL
-//MC_VM_MARC_LEN_HI_2
-#define MC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2__SHIFT                                                             0x0
-#define MC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2_MASK                                                               0x000FFFFFL
-//MC_VM_MARC_LEN_HI_3
-#define MC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3__SHIFT                                                             0x0
-#define MC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3_MASK                                                               0x000FFFFFL
-//VM_IOMMU_CONTROL_REGISTER
-#define VM_IOMMU_CONTROL_REGISTER__IOMMUEN__SHIFT                                                             0x0
-#define VM_IOMMU_CONTROL_REGISTER__IOMMUEN_MASK                                                               0x00000001L
-//VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER
-#define VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN__SHIFT                                  0xd
-#define VM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN_MASK                                    0x00002000L
-//VM_PCIE_ATS_CNTL
-#define VM_PCIE_ATS_CNTL__STU__SHIFT                                                                          0x10
-#define VM_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                   0x1f
-#define VM_PCIE_ATS_CNTL__STU_MASK                                                                            0x001F0000L
-#define VM_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                     0x80000000L
-//VM_PCIE_ATS_CNTL_VF_0
-#define VM_PCIE_ATS_CNTL_VF_0__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_0__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_1
-#define VM_PCIE_ATS_CNTL_VF_1__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_1__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_2
-#define VM_PCIE_ATS_CNTL_VF_2__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_2__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_3
-#define VM_PCIE_ATS_CNTL_VF_3__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_3__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_4
-#define VM_PCIE_ATS_CNTL_VF_4__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_4__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_5
-#define VM_PCIE_ATS_CNTL_VF_5__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_5__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_6
-#define VM_PCIE_ATS_CNTL_VF_6__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_6__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_7
-#define VM_PCIE_ATS_CNTL_VF_7__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_7__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_8
-#define VM_PCIE_ATS_CNTL_VF_8__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_8__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_9
-#define VM_PCIE_ATS_CNTL_VF_9__ATC_ENABLE__SHIFT                                                              0x1f
-#define VM_PCIE_ATS_CNTL_VF_9__ATC_ENABLE_MASK                                                                0x80000000L
-//VM_PCIE_ATS_CNTL_VF_10
-#define VM_PCIE_ATS_CNTL_VF_10__ATC_ENABLE__SHIFT                                                             0x1f
-#define VM_PCIE_ATS_CNTL_VF_10__ATC_ENABLE_MASK                                                               0x80000000L
-//VM_PCIE_ATS_CNTL_VF_11
-#define VM_PCIE_ATS_CNTL_VF_11__ATC_ENABLE__SHIFT                                                             0x1f
-#define VM_PCIE_ATS_CNTL_VF_11__ATC_ENABLE_MASK                                                               0x80000000L
-//VM_PCIE_ATS_CNTL_VF_12
-#define VM_PCIE_ATS_CNTL_VF_12__ATC_ENABLE__SHIFT                                                             0x1f
-#define VM_PCIE_ATS_CNTL_VF_12__ATC_ENABLE_MASK                                                               0x80000000L
-//VM_PCIE_ATS_CNTL_VF_13
-#define VM_PCIE_ATS_CNTL_VF_13__ATC_ENABLE__SHIFT                                                             0x1f
-#define VM_PCIE_ATS_CNTL_VF_13__ATC_ENABLE_MASK                                                               0x80000000L
-//VM_PCIE_ATS_CNTL_VF_14
-#define VM_PCIE_ATS_CNTL_VF_14__ATC_ENABLE__SHIFT                                                             0x1f
-#define VM_PCIE_ATS_CNTL_VF_14__ATC_ENABLE_MASK                                                               0x80000000L
-//VM_PCIE_ATS_CNTL_VF_15
-#define VM_PCIE_ATS_CNTL_VF_15__ATC_ENABLE__SHIFT                                                             0x1f
-#define VM_PCIE_ATS_CNTL_VF_15__ATC_ENABLE_MASK                                                               0x80000000L
-//UTCL2_CGTT_CLK_CTRL
-#define UTCL2_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
-#define UTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
-#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_EXTRA__SHIFT                                                       0xc
-#define UTCL2_CGTT_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                             0xf
-#define UTCL2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE__SHIFT                                                       0x10
-#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE__SHIFT                                                             0x18
-#define UTCL2_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
-#define UTCL2_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
-#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_EXTRA_MASK                                                         0x00007000L
-#define UTCL2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK                                                               0x00008000L
-#define UTCL2_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_MASK                                                         0x00FF0000L
-#define UTCL2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK                                                               0xFF000000L
-
-
-// addressBlock: gc_hypdec
-//CP_HYP_PFP_UCODE_ADDR
-#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
-#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x00003FFFL
-//CP_PFP_UCODE_ADDR
-#define CP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                  0x0
-#define CP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                    0x00003FFFL
-//CP_HYP_PFP_UCODE_DATA
-#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
-#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
-//CP_PFP_UCODE_DATA
-#define CP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                                  0x0
-#define CP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                    0xFFFFFFFFL
-//CP_HYP_ME_UCODE_ADDR
-#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR__SHIFT                                                               0x0
-#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR_MASK                                                                 0x00001FFFL
-//CP_ME_RAM_RADDR
-#define CP_ME_RAM_RADDR__ME_RAM_RADDR__SHIFT                                                                  0x0
-#define CP_ME_RAM_RADDR__ME_RAM_RADDR_MASK                                                                    0x00001FFFL
-//CP_ME_RAM_WADDR
-#define CP_ME_RAM_WADDR__ME_RAM_WADDR__SHIFT                                                                  0x0
-#define CP_ME_RAM_WADDR__ME_RAM_WADDR_MASK                                                                    0x00001FFFL
-//CP_HYP_ME_UCODE_DATA
-#define CP_HYP_ME_UCODE_DATA__UCODE_DATA__SHIFT                                                               0x0
-#define CP_HYP_ME_UCODE_DATA__UCODE_DATA_MASK                                                                 0xFFFFFFFFL
-//CP_ME_RAM_DATA
-#define CP_ME_RAM_DATA__ME_RAM_DATA__SHIFT                                                                    0x0
-#define CP_ME_RAM_DATA__ME_RAM_DATA_MASK                                                                      0xFFFFFFFFL
-//CP_CE_UCODE_ADDR
-#define CP_CE_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                   0x0
-#define CP_CE_UCODE_ADDR__UCODE_ADDR_MASK                                                                     0x00000FFFL
-//CP_HYP_CE_UCODE_ADDR
-#define CP_HYP_CE_UCODE_ADDR__UCODE_ADDR__SHIFT                                                               0x0
-#define CP_HYP_CE_UCODE_ADDR__UCODE_ADDR_MASK                                                                 0x00000FFFL
-//CP_CE_UCODE_DATA
-#define CP_CE_UCODE_DATA__UCODE_DATA__SHIFT                                                                   0x0
-#define CP_CE_UCODE_DATA__UCODE_DATA_MASK                                                                     0xFFFFFFFFL
-//CP_HYP_CE_UCODE_DATA
-#define CP_HYP_CE_UCODE_DATA__UCODE_DATA__SHIFT                                                               0x0
-#define CP_HYP_CE_UCODE_DATA__UCODE_DATA_MASK                                                                 0xFFFFFFFFL
-//CP_HYP_MEC1_UCODE_ADDR
-#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
-#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x0001FFFFL
-//CP_MEC_ME1_UCODE_ADDR
-#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
-#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x0001FFFFL
-//CP_HYP_MEC1_UCODE_DATA
-#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
-#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
-//CP_MEC_ME1_UCODE_DATA
-#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
-#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
-//CP_HYP_MEC2_UCODE_ADDR
-#define CP_HYP_MEC2_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
-#define CP_HYP_MEC2_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x0001FFFFL
-//CP_MEC_ME2_UCODE_ADDR
-#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
-#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x0001FFFFL
-//CP_HYP_MEC2_UCODE_DATA
-#define CP_HYP_MEC2_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
-#define CP_HYP_MEC2_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
-//CP_MEC_ME2_UCODE_DATA
-#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
-#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
-//RLC_GPM_UCODE_ADDR
-#define RLC_GPM_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                 0x0
-#define RLC_GPM_UCODE_ADDR__RESERVED__SHIFT                                                                   0xe
-#define RLC_GPM_UCODE_ADDR__UCODE_ADDR_MASK                                                                   0x00003FFFL
-#define RLC_GPM_UCODE_ADDR__RESERVED_MASK                                                                     0xFFFFC000L
-//RLC_GPM_UCODE_DATA
-#define RLC_GPM_UCODE_DATA__UCODE_DATA__SHIFT                                                                 0x0
-#define RLC_GPM_UCODE_DATA__UCODE_DATA_MASK                                                                   0xFFFFFFFFL
-//GRBM_GFX_INDEX_SR_SELECT
-#define GRBM_GFX_INDEX_SR_SELECT__INDEX__SHIFT                                                                0x0
-#define GRBM_GFX_INDEX_SR_SELECT__INDEX_MASK                                                                  0x00000007L
-//GRBM_GFX_INDEX_SR_DATA
-#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX__SHIFT                                                         0x0
-#define GRBM_GFX_INDEX_SR_DATA__SH_INDEX__SHIFT                                                               0x8
-#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX__SHIFT                                                               0x10
-#define GRBM_GFX_INDEX_SR_DATA__SH_BROADCAST_WRITES__SHIFT                                                    0x1d
-#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES__SHIFT                                              0x1e
-#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES__SHIFT                                                    0x1f
-#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX_MASK                                                           0x000000FFL
-#define GRBM_GFX_INDEX_SR_DATA__SH_INDEX_MASK                                                                 0x0000FF00L
-#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX_MASK                                                                 0x00FF0000L
-#define GRBM_GFX_INDEX_SR_DATA__SH_BROADCAST_WRITES_MASK                                                      0x20000000L
-#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES_MASK                                                0x40000000L
-#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES_MASK                                                      0x80000000L
-//GRBM_GFX_CNTL_SR_SELECT
-#define GRBM_GFX_CNTL_SR_SELECT__INDEX__SHIFT                                                                 0x0
-#define GRBM_GFX_CNTL_SR_SELECT__INDEX_MASK                                                                   0x00000007L
-//GRBM_GFX_CNTL_SR_DATA
-#define GRBM_GFX_CNTL_SR_DATA__PIPEID__SHIFT                                                                  0x0
-#define GRBM_GFX_CNTL_SR_DATA__MEID__SHIFT                                                                    0x2
-#define GRBM_GFX_CNTL_SR_DATA__VMID__SHIFT                                                                    0x4
-#define GRBM_GFX_CNTL_SR_DATA__QUEUEID__SHIFT                                                                 0x8
-#define GRBM_GFX_CNTL_SR_DATA__PIPEID_MASK                                                                    0x00000003L
-#define GRBM_GFX_CNTL_SR_DATA__MEID_MASK                                                                      0x0000000CL
-#define GRBM_GFX_CNTL_SR_DATA__VMID_MASK                                                                      0x000000F0L
-#define GRBM_GFX_CNTL_SR_DATA__QUEUEID_MASK                                                                   0x00000700L
-//GRBM_CAM_INDEX
-#define GRBM_CAM_INDEX__CAM_INDEX__SHIFT                                                                      0x0
-#define GRBM_CAM_INDEX__CAM_INDEX_MASK                                                                        0x00000007L
-//GRBM_HYP_CAM_INDEX
-#define GRBM_HYP_CAM_INDEX__CAM_INDEX__SHIFT                                                                  0x0
-#define GRBM_HYP_CAM_INDEX__CAM_INDEX_MASK                                                                    0x00000007L
-//GRBM_CAM_DATA
-#define GRBM_CAM_DATA__CAM_ADDR__SHIFT                                                                        0x0
-#define GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT                                                                   0x10
-#define GRBM_CAM_DATA__CAM_ADDR_MASK                                                                          0x0000FFFFL
-#define GRBM_CAM_DATA__CAM_REMAPADDR_MASK                                                                     0xFFFF0000L
-//GRBM_HYP_CAM_DATA
-#define GRBM_HYP_CAM_DATA__CAM_ADDR__SHIFT                                                                    0x0
-#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR__SHIFT                                                               0x10
-#define GRBM_HYP_CAM_DATA__CAM_ADDR_MASK                                                                      0x0000FFFFL
-#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR_MASK                                                                 0xFFFF0000L
-//RLC_GPU_IOV_VF_ENABLE
-#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE__SHIFT                                                               0x0
-#define RLC_GPU_IOV_VF_ENABLE__RESERVED__SHIFT                                                                0x1
-#define RLC_GPU_IOV_VF_ENABLE__VF_NUM__SHIFT                                                                  0x10
-#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE_MASK                                                                 0x00000001L
-#define RLC_GPU_IOV_VF_ENABLE__RESERVED_MASK                                                                  0x0000FFFEL
-#define RLC_GPU_IOV_VF_ENABLE__VF_NUM_MASK                                                                    0xFFFF0000L
-//RLC_GFX_RM_CNTL_ADJ
-#define RLC_GFX_RM_CNTL_ADJ__RLC_GFX_RM_VALID__SHIFT                                                          0x0
-#define RLC_GFX_RM_CNTL_ADJ__RESERVED__SHIFT                                                                  0x1
-#define RLC_GFX_RM_CNTL_ADJ__RLC_GFX_RM_VALID_MASK                                                            0x00000001L
-#define RLC_GFX_RM_CNTL_ADJ__RESERVED_MASK                                                                    0xFFFFFFFEL
-//RLC_GPU_IOV_CFG_REG6
-#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE__SHIFT                                                               0x0
-#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION__SHIFT                                                           0x7
-#define RLC_GPU_IOV_CFG_REG6__RESERVED__SHIFT                                                                 0x8
-#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET__SHIFT                                                             0xa
-#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE_MASK                                                                 0x0000007FL
-#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION_MASK                                                             0x00000080L
-#define RLC_GPU_IOV_CFG_REG6__RESERVED_MASK                                                                   0x00000300L
-#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET_MASK                                                               0xFFFFFC00L
-//RLC_GPU_IOV_CFG_REG8
-#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS__SHIFT                                                           0x0
-#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS_MASK                                                             0xFFFFFFFFL
-//RLC_RLCV_TIMER_INT_0
-#define RLC_RLCV_TIMER_INT_0__TIMER__SHIFT                                                                    0x0
-#define RLC_RLCV_TIMER_INT_0__TIMER_MASK                                                                      0xFFFFFFFFL
-//RLC_RLCV_TIMER_CTRL
-#define RLC_RLCV_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                0x0
-#define RLC_RLCV_TIMER_CTRL__RESERVED__SHIFT                                                                  0x1
-#define RLC_RLCV_TIMER_CTRL__TIMER_0_EN_MASK                                                                  0x00000001L
-#define RLC_RLCV_TIMER_CTRL__RESERVED_MASK                                                                    0xFFFFFFFEL
-//RLC_RLCV_TIMER_STAT
-#define RLC_RLCV_TIMER_STAT__TIMER_0_STAT__SHIFT                                                              0x0
-#define RLC_RLCV_TIMER_STAT__RESERVED__SHIFT                                                                  0x1
-#define RLC_RLCV_TIMER_STAT__TIMER_0_STAT_MASK                                                                0x00000001L
-#define RLC_RLCV_TIMER_STAT__RESERVED_MASK                                                                    0xFFFFFFFEL
-//RLC_GPU_IOV_VF_DOORBELL_STATUS
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS__SHIFT                                             0x0
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS__RESERVED__SHIFT                                                       0x10
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS__SHIFT                                             0x1f
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_MASK                                               0x0000FFFFL
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS__RESERVED_MASK                                                         0x7FFF0000L
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS_MASK                                               0x80000000L
-//RLC_GPU_IOV_VF_DOORBELL_STATUS_SET
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET__SHIFT                                     0x0
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__RESERVED__SHIFT                                                   0x10
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET__SHIFT                                     0x1f
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET_MASK                                       0x0000FFFFL
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__RESERVED_MASK                                                     0x7FFF0000L
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET_MASK                                       0x80000000L
-//RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR__SHIFT                                     0x0
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__RESERVED__SHIFT                                                   0x10
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR__SHIFT                                     0x1f
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR_MASK                                       0x0000FFFFL
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__RESERVED_MASK                                                     0x7FFF0000L
-#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR_MASK                                       0x80000000L
-//RLC_GPU_IOV_VF_MASK
-#define RLC_GPU_IOV_VF_MASK__VF_MASK__SHIFT                                                                   0x0
-#define RLC_GPU_IOV_VF_MASK__RESERVED__SHIFT                                                                  0x10
-#define RLC_GPU_IOV_VF_MASK__VF_MASK_MASK                                                                     0x0000FFFFL
-#define RLC_GPU_IOV_VF_MASK__RESERVED_MASK                                                                    0xFFFF0000L
-//RLC_HYP_SEMAPHORE_2
-#define RLC_HYP_SEMAPHORE_2__CLIENT_ID__SHIFT                                                                 0x0
-#define RLC_HYP_SEMAPHORE_2__RESERVED__SHIFT                                                                  0x5
-#define RLC_HYP_SEMAPHORE_2__CLIENT_ID_MASK                                                                   0x0000001FL
-#define RLC_HYP_SEMAPHORE_2__RESERVED_MASK                                                                    0xFFFFFFE0L
-//RLC_HYP_SEMAPHORE_3
-#define RLC_HYP_SEMAPHORE_3__CLIENT_ID__SHIFT                                                                 0x0
-#define RLC_HYP_SEMAPHORE_3__RESERVED__SHIFT                                                                  0x5
-#define RLC_HYP_SEMAPHORE_3__CLIENT_ID_MASK                                                                   0x0000001FL
-#define RLC_HYP_SEMAPHORE_3__RESERVED_MASK                                                                    0xFFFFFFE0L
-//RLC_CLK_CNTL
-#define RLC_CLK_CNTL__RLC_SRM_CLK_CNTL__SHIFT                                                                 0x0
-#define RLC_CLK_CNTL__RLC_SPM_CLK_CNTL__SHIFT                                                                 0x1
-#define RLC_CLK_CNTL__RESERVED__SHIFT                                                                         0x2
-#define RLC_CLK_CNTL__RLC_SRM_CLK_CNTL_MASK                                                                   0x00000001L
-#define RLC_CLK_CNTL__RLC_SPM_CLK_CNTL_MASK                                                                   0x00000002L
-#define RLC_CLK_CNTL__RESERVED_MASK                                                                           0xFFFFFFFCL
-//RLC_GPU_IOV_SCH_BLOCK
-#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID__SHIFT                                                            0x0
-#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver__SHIFT                                                           0x4
-#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size__SHIFT                                                          0x8
-#define RLC_GPU_IOV_SCH_BLOCK__RESERVED__SHIFT                                                                0x10
-#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID_MASK                                                              0x0000000FL
-#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver_MASK                                                             0x000000F0L
-#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size_MASK                                                            0x00007F00L
-#define RLC_GPU_IOV_SCH_BLOCK__RESERVED_MASK                                                                  0x7FFF0000L
-//RLC_GPU_IOV_CFG_REG1
-#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE__SHIFT                                                                 0x0
-#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE__SHIFT                                                              0x4
-#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN__SHIFT                                                      0x5
-#define RLC_GPU_IOV_CFG_REG1__RESERVED__SHIFT                                                                 0x6
-#define RLC_GPU_IOV_CFG_REG1__FCN_ID__SHIFT                                                                   0x8
-#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID__SHIFT                                                              0x10
-#define RLC_GPU_IOV_CFG_REG1__RESERVED1__SHIFT                                                                0x18
-#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE_MASK                                                                   0x0000000FL
-#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_MASK                                                                0x00000010L
-#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN_MASK                                                        0x00000020L
-#define RLC_GPU_IOV_CFG_REG1__RESERVED_MASK                                                                   0x000000C0L
-#define RLC_GPU_IOV_CFG_REG1__FCN_ID_MASK                                                                     0x0000FF00L
-#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID_MASK                                                                0x00FF0000L
-#define RLC_GPU_IOV_CFG_REG1__RESERVED1_MASK                                                                  0xFF000000L
-//RLC_GPU_IOV_CFG_REG2
-#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS__SHIFT                                                               0x0
-#define RLC_GPU_IOV_CFG_REG2__RESERVED__SHIFT                                                                 0x4
-#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS_MASK                                                                 0x0000000FL
-#define RLC_GPU_IOV_CFG_REG2__RESERVED_MASK                                                                   0xFFFFFFF0L
-//RLC_GPU_IOV_VM_BUSY_STATUS
-#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                     0x0
-#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                       0xFFFFFFFFL
-//RLC_GPU_IOV_SCH_0
-#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS__SHIFT                                                            0x0
-#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS_MASK                                                              0xFFFFFFFFL
-//RLC_GPU_IOV_ACTIVE_FCN_ID
-#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID__SHIFT                                                               0x0
-#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED__SHIFT                                                            0x4
-#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF__SHIFT                                                               0x1f
-#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID_MASK                                                                 0x0000000FL
-#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_MASK                                                              0x7FFFFFF0L
-#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF_MASK                                                                 0x80000000L
-//RLC_GPU_IOV_SCH_3
-#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def__SHIFT                                                             0x0
-#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def_MASK                                                               0xFFFFFFFFL
-//RLC_GPU_IOV_SCH_1
-#define RLC_GPU_IOV_SCH_1__DATA__SHIFT                                                                        0x0
-#define RLC_GPU_IOV_SCH_1__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPU_IOV_SCH_2
-#define RLC_GPU_IOV_SCH_2__DATA__SHIFT                                                                        0x0
-#define RLC_GPU_IOV_SCH_2__DATA_MASK                                                                          0xFFFFFFFFL
-//RLC_GPU_IOV_UCODE_ADDR
-#define RLC_GPU_IOV_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
-#define RLC_GPU_IOV_UCODE_ADDR__RESERVED__SHIFT                                                               0xc
-#define RLC_GPU_IOV_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x00000FFFL
-#define RLC_GPU_IOV_UCODE_ADDR__RESERVED_MASK                                                                 0xFFFFF000L
-//RLC_GPU_IOV_UCODE_DATA
-#define RLC_GPU_IOV_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
-#define RLC_GPU_IOV_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
-//RLC_GPU_IOV_SCRATCH_ADDR
-#define RLC_GPU_IOV_SCRATCH_ADDR__ADDR__SHIFT                                                                 0x0
-#define RLC_GPU_IOV_SCRATCH_ADDR__RESERVED__SHIFT                                                             0x9
-#define RLC_GPU_IOV_SCRATCH_ADDR__ADDR_MASK                                                                   0x000001FFL
-#define RLC_GPU_IOV_SCRATCH_ADDR__RESERVED_MASK                                                               0xFFFFFE00L
-//RLC_GPU_IOV_SCRATCH_DATA
-#define RLC_GPU_IOV_SCRATCH_DATA__DATA__SHIFT                                                                 0x0
-#define RLC_GPU_IOV_SCRATCH_DATA__DATA_MASK                                                                   0xFFFFFFFFL
-//RLC_GPU_IOV_F32_CNTL
-#define RLC_GPU_IOV_F32_CNTL__ENABLE__SHIFT                                                                   0x0
-#define RLC_GPU_IOV_F32_CNTL__RESERVED__SHIFT                                                                 0x1
-#define RLC_GPU_IOV_F32_CNTL__ENABLE_MASK                                                                     0x00000001L
-#define RLC_GPU_IOV_F32_CNTL__RESERVED_MASK                                                                   0xFFFFFFFEL
-//RLC_GPU_IOV_F32_RESET
-#define RLC_GPU_IOV_F32_RESET__RESET__SHIFT                                                                   0x0
-#define RLC_GPU_IOV_F32_RESET__RESERVED__SHIFT                                                                0x1
-#define RLC_GPU_IOV_F32_RESET__RESET_MASK                                                                     0x00000001L
-#define RLC_GPU_IOV_F32_RESET__RESERVED_MASK                                                                  0xFFFFFFFEL
-//RLC_GPU_IOV_SDMA0_STATUS
-#define RLC_GPU_IOV_SDMA0_STATUS__PREEMPTED__SHIFT                                                            0x0
-#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED__SHIFT                                                             0x1
-#define RLC_GPU_IOV_SDMA0_STATUS__SAVED__SHIFT                                                                0x8
-#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED1__SHIFT                                                            0x9
-#define RLC_GPU_IOV_SDMA0_STATUS__RESTORED__SHIFT                                                             0xc
-#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED2__SHIFT                                                            0xd
-#define RLC_GPU_IOV_SDMA0_STATUS__PREEMPTED_MASK                                                              0x00000001L
-#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED_MASK                                                               0x000000FEL
-#define RLC_GPU_IOV_SDMA0_STATUS__SAVED_MASK                                                                  0x00000100L
-#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED1_MASK                                                              0x00000E00L
-#define RLC_GPU_IOV_SDMA0_STATUS__RESTORED_MASK                                                               0x00001000L
-#define RLC_GPU_IOV_SDMA0_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
-//RLC_GPU_IOV_SDMA1_STATUS
-#define RLC_GPU_IOV_SDMA1_STATUS__PREEMPTED__SHIFT                                                            0x0
-#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED__SHIFT                                                             0x1
-#define RLC_GPU_IOV_SDMA1_STATUS__SAVED__SHIFT                                                                0x8
-#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED1__SHIFT                                                            0x9
-#define RLC_GPU_IOV_SDMA1_STATUS__RESTORED__SHIFT                                                             0xc
-#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED2__SHIFT                                                            0xd
-#define RLC_GPU_IOV_SDMA1_STATUS__PREEMPTED_MASK                                                              0x00000001L
-#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED_MASK                                                               0x000000FEL
-#define RLC_GPU_IOV_SDMA1_STATUS__SAVED_MASK                                                                  0x00000100L
-#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED1_MASK                                                              0x00000E00L
-#define RLC_GPU_IOV_SDMA1_STATUS__RESTORED_MASK                                                               0x00001000L
-#define RLC_GPU_IOV_SDMA1_STATUS__RESERVED2_MASK                                                              0xFFFFE000L
-//RLC_GPU_IOV_SMU_RESPONSE
-#define RLC_GPU_IOV_SMU_RESPONSE__RESP__SHIFT                                                                 0x0
-#define RLC_GPU_IOV_SMU_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
-//RLC_GPU_IOV_VIRT_RESET_REQ
-#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR__SHIFT                                                             0x0
-#define RLC_GPU_IOV_VIRT_RESET_REQ__RESERVED__SHIFT                                                           0x10
-#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR__SHIFT                                                        0x1f
-#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR_MASK                                                               0x0000FFFFL
-#define RLC_GPU_IOV_VIRT_RESET_REQ__RESERVED_MASK                                                             0x7FFF0000L
-#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR_MASK                                                          0x80000000L
-//RLC_GPU_IOV_RLC_RESPONSE
-#define RLC_GPU_IOV_RLC_RESPONSE__RESP__SHIFT                                                                 0x0
-#define RLC_GPU_IOV_RLC_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
-//RLC_GPU_IOV_INT_DISABLE
-#define RLC_GPU_IOV_INT_DISABLE__DISABLE__SHIFT                                                               0x0
-#define RLC_GPU_IOV_INT_DISABLE__DISABLE_MASK                                                                 0xFFFFFFFFL
-//RLC_GPU_IOV_INT_FORCE
-#define RLC_GPU_IOV_INT_FORCE__FORCE__SHIFT                                                                   0x0
-#define RLC_GPU_IOV_INT_FORCE__FORCE_MASK                                                                     0xFFFFFFFFL
-//RLC_GPU_IOV_SDMA0_BUSY_STATUS
-#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
-#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
-//RLC_GPU_IOV_SDMA1_BUSY_STATUS
-#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
-#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
-
-
-// addressBlock: gccacind
-//GC_CAC_CNTL
-#define GC_CAC_CNTL__CAC_ENABLE__SHIFT                                                                        0x0
-#define GC_CAC_CNTL__CAC_THRESHOLD__SHIFT                                                                     0x1
-#define GC_CAC_CNTL__CAC_BLOCK_ID__SHIFT                                                                      0x11
-#define GC_CAC_CNTL__CAC_SIGNAL_ID__SHIFT                                                                     0x17
-#define GC_CAC_CNTL__UNUSED_0__SHIFT                                                                          0x1f
-#define GC_CAC_CNTL__CAC_ENABLE_MASK                                                                          0x00000001L
-#define GC_CAC_CNTL__CAC_THRESHOLD_MASK                                                                       0x0001FFFEL
-#define GC_CAC_CNTL__CAC_BLOCK_ID_MASK                                                                        0x007E0000L
-#define GC_CAC_CNTL__CAC_SIGNAL_ID_MASK                                                                       0x7F800000L
-#define GC_CAC_CNTL__UNUSED_0_MASK                                                                            0x80000000L
-//GC_CAC_OVR_SEL
-#define GC_CAC_OVR_SEL__CAC_OVR_SEL__SHIFT                                                                    0x0
-#define GC_CAC_OVR_SEL__CAC_OVR_SEL_MASK                                                                      0xFFFFFFFFL
-//GC_CAC_OVR_VAL
-#define GC_CAC_OVR_VAL__CAC_OVR_VAL__SHIFT                                                                    0x0
-#define GC_CAC_OVR_VAL__CAC_OVR_VAL_MASK                                                                      0xFFFFFFFFL
-//GC_CAC_WEIGHT_BCI_0
-#define GC_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG1__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG1_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_CB_0
-#define GC_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CB_1
-#define GC_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG2__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG3__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG2_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG3_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CP_0
-#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CP_1
-#define GC_CAC_WEIGHT_CP_1__WEIGHT_CP_SIG2__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CP_1__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_CP_1__WEIGHT_CP_SIG2_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CP_1__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_DB_0
-#define GC_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_DB_1
-#define GC_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG2__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG3__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG2_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG3_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_GDS_0
-#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG1__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG1_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_GDS_1
-#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG2__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG3__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG2_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG3_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_IA_0
-#define GC_CAC_WEIGHT_IA_0__WEIGHT_IA_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_IA_0__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_IA_0__WEIGHT_IA_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_IA_0__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_LDS_0
-#define GC_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG1__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG1_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_LDS_1
-#define GC_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG2__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG3__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG2_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG3_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_PA_0
-#define GC_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_PC_0
-#define GC_CAC_WEIGHT_PC_0__WEIGHT_PC_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_PC_0__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_PC_0__WEIGHT_PC_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_PC_0__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_SC_0
-#define GC_CAC_WEIGHT_SC_0__WEIGHT_SC_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_SC_0__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_SC_0__WEIGHT_SC_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_SC_0__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_SPI_0
-#define GC_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG1__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG1_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_SPI_1
-#define GC_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG2__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG3__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG2_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG3_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_SPI_2
-#define GC_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG4__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG5__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG4_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG5_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_SQ_0
-#define GC_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_SQ_1
-#define GC_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG2__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG3__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG2_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG3_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_SQ_2
-#define GC_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG4__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG5__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG4_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG5_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_SQ_3
-#define GC_CAC_WEIGHT_SQ_3__WEIGHT_SQ_SIG6__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_SQ_3__WEIGHT_SQ_SIG7__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_SQ_3__WEIGHT_SQ_SIG6_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_SQ_3__WEIGHT_SQ_SIG7_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_SQ_4
-#define GC_CAC_WEIGHT_SQ_4__WEIGHT_SQ_SIG8__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_SQ_4__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_SQ_4__WEIGHT_SQ_SIG8_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_SQ_4__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_SX_0
-#define GC_CAC_WEIGHT_SX_0__WEIGHT_SX_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_SX_0__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_SX_0__WEIGHT_SX_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_SX_0__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_SXRB_0
-#define GC_CAC_WEIGHT_SXRB_0__WEIGHT_SXRB_SIG0__SHIFT                                                         0x0
-#define GC_CAC_WEIGHT_SXRB_0__WEIGHT_SXRB_SIG1__SHIFT                                                         0x10
-#define GC_CAC_WEIGHT_SXRB_0__WEIGHT_SXRB_SIG0_MASK                                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_SXRB_0__WEIGHT_SXRB_SIG1_MASK                                                           0xFFFF0000L
-//GC_CAC_WEIGHT_TA_0
-#define GC_CAC_WEIGHT_TA_0__WEIGHT_TA_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_TA_0__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_TA_0__WEIGHT_TA_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_TA_0__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_TCC_0
-#define GC_CAC_WEIGHT_TCC_0__WEIGHT_TCC_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_TCC_0__WEIGHT_TCC_SIG1__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_TCC_0__WEIGHT_TCC_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_TCC_0__WEIGHT_TCC_SIG1_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_TCC_1
-#define GC_CAC_WEIGHT_TCC_1__WEIGHT_TCC_SIG2__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_TCC_1__WEIGHT_TCC_SIG3__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_TCC_1__WEIGHT_TCC_SIG2_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_TCC_1__WEIGHT_TCC_SIG3_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_TCC_2
-#define GC_CAC_WEIGHT_TCC_2__WEIGHT_TCC_SIG4__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_TCC_2__UNUSED_0__SHIFT                                                                  0x10
-#define GC_CAC_WEIGHT_TCC_2__WEIGHT_TCC_SIG4_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_TCC_2__UNUSED_0_MASK                                                                    0xFFFF0000L
-//GC_CAC_WEIGHT_TCP_0
-#define GC_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG1__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG1_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_TCP_1
-#define GC_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG2__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG3__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG2_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG3_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_TCP_2
-#define GC_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG4__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_TCP_2__UNUSED_0__SHIFT                                                                  0x10
-#define GC_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG4_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_TCP_2__UNUSED_0_MASK                                                                    0xFFFF0000L
-//GC_CAC_WEIGHT_TD_0
-#define GC_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_TD_1
-#define GC_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG2__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG3__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG2_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG3_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_TD_2
-#define GC_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG4__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG5__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG4_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG5_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_VGT_0
-#define GC_CAC_WEIGHT_VGT_0__WEIGHT_VGT_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_VGT_0__WEIGHT_VGT_SIG1__SHIFT                                                           0x10
-#define GC_CAC_WEIGHT_VGT_0__WEIGHT_VGT_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_VGT_0__WEIGHT_VGT_SIG1_MASK                                                             0xFFFF0000L
-//GC_CAC_WEIGHT_VGT_1
-#define GC_CAC_WEIGHT_VGT_1__WEIGHT_VGT_SIG2__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_VGT_1__UNUSED_0__SHIFT                                                                  0x10
-#define GC_CAC_WEIGHT_VGT_1__WEIGHT_VGT_SIG2_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_VGT_1__UNUSED_0_MASK                                                                    0xFFFF0000L
-//GC_CAC_WEIGHT_WD_0
-#define GC_CAC_WEIGHT_WD_0__WEIGHT_WD_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_WD_0__UNUSED_0__SHIFT                                                                   0x10
-#define GC_CAC_WEIGHT_WD_0__WEIGHT_WD_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_WD_0__UNUSED_0_MASK                                                                     0xFFFF0000L
-//GC_CAC_WEIGHT_CU_0
-#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CU_1
-#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG2__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG3__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG2_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG3_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CU_2
-#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG4__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG5__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG4_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG5_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CU_3
-#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG6__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG7__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG6_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG7_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CU_4
-#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG8__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG9__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG8_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG9_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_CU_5
-#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG10__SHIFT                                                            0x0
-#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG11__SHIFT                                                            0x10
-#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG10_MASK                                                              0x0000FFFFL
-#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG11_MASK                                                              0xFFFF0000L
-//GC_CAC_ACC_BCI0
-#define GC_CAC_ACC_BCI0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_BCI0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_CB0
-#define GC_CAC_ACC_CB0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CB0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CB1
-#define GC_CAC_ACC_CB1__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CB1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CB2
-#define GC_CAC_ACC_CB2__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CB2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CB3
-#define GC_CAC_ACC_CB3__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CB3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CP0
-#define GC_CAC_ACC_CP0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CP0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CP1
-#define GC_CAC_ACC_CP1__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CP1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CP2
-#define GC_CAC_ACC_CP2__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CP2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_DB0
-#define GC_CAC_ACC_DB0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_DB0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_DB1
-#define GC_CAC_ACC_DB1__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_DB1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_DB2
-#define GC_CAC_ACC_DB2__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_DB2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_DB3
-#define GC_CAC_ACC_DB3__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_DB3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_GDS0
-#define GC_CAC_ACC_GDS0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_GDS0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_GDS1
-#define GC_CAC_ACC_GDS1__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_GDS1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_GDS2
-#define GC_CAC_ACC_GDS2__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_GDS2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_GDS3
-#define GC_CAC_ACC_GDS3__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_GDS3__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_IA0
-#define GC_CAC_ACC_IA0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_IA0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_LDS0
-#define GC_CAC_ACC_LDS0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_LDS0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_LDS1
-#define GC_CAC_ACC_LDS1__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_LDS1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_LDS2
-#define GC_CAC_ACC_LDS2__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_LDS2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_LDS3
-#define GC_CAC_ACC_LDS3__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_LDS3__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_PA0
-#define GC_CAC_ACC_PA0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_PA0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_PA1
-#define GC_CAC_ACC_PA1__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_PA1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_PC0
-#define GC_CAC_ACC_PC0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_PC0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_SC0
-#define GC_CAC_ACC_SC0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_SC0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_SPI0
-#define GC_CAC_ACC_SPI0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_SPI0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_SPI1
-#define GC_CAC_ACC_SPI1__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_SPI1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_SPI2
-#define GC_CAC_ACC_SPI2__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_SPI2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_SPI3
-#define GC_CAC_ACC_SPI3__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_SPI3__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_SPI4
-#define GC_CAC_ACC_SPI4__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_SPI4__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_SPI5
-#define GC_CAC_ACC_SPI5__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_SPI5__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_WEIGHT_PG_0
-#define GC_CAC_WEIGHT_PG_0__WEIGHT_PG_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_PG_0__unused__SHIFT                                                                     0x10
-#define GC_CAC_WEIGHT_PG_0__WEIGHT_PG_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_PG_0__unused_MASK                                                                       0xFFFF0000L
-//GC_CAC_ACC_PG0
-#define GC_CAC_ACC_PG0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_PG0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_OVRD_PG
-#define GC_CAC_OVRD_PG__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_PG__OVRRD_VALUE__SHIFT                                                                    0x10
-#define GC_CAC_OVRD_PG__OVRRD_SELECT_MASK                                                                     0x0000FFFFL
-#define GC_CAC_OVRD_PG__OVRRD_VALUE_MASK                                                                      0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_ATCL2_0
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_0__WEIGHT_UTCL2_ATCL2_SIG0__SHIFT                                           0x0
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_0__WEIGHT_UTCL2_ATCL2_SIG1__SHIFT                                           0x10
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_0__WEIGHT_UTCL2_ATCL2_SIG0_MASK                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_0__WEIGHT_UTCL2_ATCL2_SIG1_MASK                                             0xFFFF0000L
-//GC_CAC_ACC_EA0
-#define GC_CAC_ACC_EA0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_EA0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_EA1
-#define GC_CAC_ACC_EA1__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_EA1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_EA2
-#define GC_CAC_ACC_EA2__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_EA2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_EA3
-#define GC_CAC_ACC_EA3__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_EA3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ATCL20
-#define GC_CAC_ACC_UTCL2_ATCL20__ACCUMULATOR_31_0__SHIFT                                                      0x0
-#define GC_CAC_ACC_UTCL2_ATCL20__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
-//GC_CAC_OVRD_EA
-#define GC_CAC_OVRD_EA__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_EA__OVRRD_VALUE__SHIFT                                                                    0x6
-#define GC_CAC_OVRD_EA__OVRRD_SELECT_MASK                                                                     0x0000003FL
-#define GC_CAC_OVRD_EA__OVRRD_VALUE_MASK                                                                      0x00000FC0L
-//GC_CAC_OVRD_UTCL2_ATCL2
-#define GC_CAC_OVRD_UTCL2_ATCL2__OVRRD_SELECT__SHIFT                                                          0x0
-#define GC_CAC_OVRD_UTCL2_ATCL2__OVRRD_VALUE__SHIFT                                                           0x5
-#define GC_CAC_OVRD_UTCL2_ATCL2__OVRRD_SELECT_MASK                                                            0x0000001FL
-#define GC_CAC_OVRD_UTCL2_ATCL2__OVRRD_VALUE_MASK                                                             0x000003E0L
-//GC_CAC_WEIGHT_EA_0
-#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG0__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG1__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG0_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG1_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_EA_1
-#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG2__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG3__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG2_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG3_MASK                                                               0xFFFF0000L
-//GC_CAC_WEIGHT_RMI_0
-#define GC_CAC_WEIGHT_RMI_0__WEIGHT_RMI_SIG0__SHIFT                                                           0x0
-#define GC_CAC_WEIGHT_RMI_0__UNUSED__SHIFT                                                                    0x10
-#define GC_CAC_WEIGHT_RMI_0__WEIGHT_RMI_SIG0_MASK                                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_RMI_0__UNUSED_MASK                                                                      0xFFFF0000L
-//GC_CAC_ACC_RMI0
-#define GC_CAC_ACC_RMI0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_RMI0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_OVRD_RMI
-#define GC_CAC_OVRD_RMI__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_RMI__OVRRD_VALUE__SHIFT                                                                   0x1
-#define GC_CAC_OVRD_RMI__OVRRD_SELECT_MASK                                                                    0x00000001L
-#define GC_CAC_OVRD_RMI__OVRRD_VALUE_MASK                                                                     0x00000002L
-//GC_CAC_WEIGHT_UTCL2_ATCL2_1
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_1__WEIGHT_UTCL2_ATCL2_SIG2__SHIFT                                           0x0
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_1__WEIGHT_UTCL2_ATCL2_SIG3__SHIFT                                           0x10
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_1__WEIGHT_UTCL2_ATCL2_SIG2_MASK                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_1__WEIGHT_UTCL2_ATCL2_SIG3_MASK                                             0xFFFF0000L
-//GC_CAC_ACC_UTCL2_ATCL21
-#define GC_CAC_ACC_UTCL2_ATCL21__ACCUMULATOR_31_0__SHIFT                                                      0x0
-#define GC_CAC_ACC_UTCL2_ATCL21__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ATCL22
-#define GC_CAC_ACC_UTCL2_ATCL22__ACCUMULATOR_31_0__SHIFT                                                      0x0
-#define GC_CAC_ACC_UTCL2_ATCL22__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ATCL23
-#define GC_CAC_ACC_UTCL2_ATCL23__ACCUMULATOR_31_0__SHIFT                                                      0x0
-#define GC_CAC_ACC_UTCL2_ATCL23__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
-//GC_CAC_ACC_EA4
-#define GC_CAC_ACC_EA4__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_EA4__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_EA5
-#define GC_CAC_ACC_EA5__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_EA5__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_WEIGHT_EA_2
-#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG4__SHIFT                                                             0x0
-#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG5__SHIFT                                                             0x10
-#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG4_MASK                                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG5_MASK                                                               0xFFFF0000L
-//GC_CAC_ACC_SQ0_LOWER
-#define GC_CAC_ACC_SQ0_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ0_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ0_UPPER
-#define GC_CAC_ACC_SQ0_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ0_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ0_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ0_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ1_LOWER
-#define GC_CAC_ACC_SQ1_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ1_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ1_UPPER
-#define GC_CAC_ACC_SQ1_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ1_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ1_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ1_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ2_LOWER
-#define GC_CAC_ACC_SQ2_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ2_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ2_UPPER
-#define GC_CAC_ACC_SQ2_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ2_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ2_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ2_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ3_LOWER
-#define GC_CAC_ACC_SQ3_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ3_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ3_UPPER
-#define GC_CAC_ACC_SQ3_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ3_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ3_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ3_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ4_LOWER
-#define GC_CAC_ACC_SQ4_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ4_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ4_UPPER
-#define GC_CAC_ACC_SQ4_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ4_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ4_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ4_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ5_LOWER
-#define GC_CAC_ACC_SQ5_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ5_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ5_UPPER
-#define GC_CAC_ACC_SQ5_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ5_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ5_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ5_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ6_LOWER
-#define GC_CAC_ACC_SQ6_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ6_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ6_UPPER
-#define GC_CAC_ACC_SQ6_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ6_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ6_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ6_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ7_LOWER
-#define GC_CAC_ACC_SQ7_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ7_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ7_UPPER
-#define GC_CAC_ACC_SQ7_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ7_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ7_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ7_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SQ8_LOWER
-#define GC_CAC_ACC_SQ8_LOWER__ACCUMULATOR_31_0__SHIFT                                                         0x0
-#define GC_CAC_ACC_SQ8_LOWER__ACCUMULATOR_31_0_MASK                                                           0xFFFFFFFFL
-//GC_CAC_ACC_SQ8_UPPER
-#define GC_CAC_ACC_SQ8_UPPER__ACCUMULATOR_39_32__SHIFT                                                        0x0
-#define GC_CAC_ACC_SQ8_UPPER__UNUSED_0__SHIFT                                                                 0x8
-#define GC_CAC_ACC_SQ8_UPPER__ACCUMULATOR_39_32_MASK                                                          0x000000FFL
-#define GC_CAC_ACC_SQ8_UPPER__UNUSED_0_MASK                                                                   0xFFFFFF00L
-//GC_CAC_ACC_SX0
-#define GC_CAC_ACC_SX0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_SX0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_SXRB0
-#define GC_CAC_ACC_SXRB0__ACCUMULATOR_31_0__SHIFT                                                             0x0
-#define GC_CAC_ACC_SXRB0__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
-//GC_CAC_ACC_SXRB1
-#define GC_CAC_ACC_SXRB1__ACCUMULATOR_31_0__SHIFT                                                             0x0
-#define GC_CAC_ACC_SXRB1__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
-//GC_CAC_ACC_TA0
-#define GC_CAC_ACC_TA0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_TA0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_TCC0
-#define GC_CAC_ACC_TCC0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCC0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCC1
-#define GC_CAC_ACC_TCC1__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCC1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCC2
-#define GC_CAC_ACC_TCC2__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCC2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCC3
-#define GC_CAC_ACC_TCC3__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCC3__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCC4
-#define GC_CAC_ACC_TCC4__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCC4__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCP0
-#define GC_CAC_ACC_TCP0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCP0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCP1
-#define GC_CAC_ACC_TCP1__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCP1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCP2
-#define GC_CAC_ACC_TCP2__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCP2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCP3
-#define GC_CAC_ACC_TCP3__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCP3__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TCP4
-#define GC_CAC_ACC_TCP4__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_TCP4__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_TD0
-#define GC_CAC_ACC_TD0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_TD0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_TD1
-#define GC_CAC_ACC_TD1__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_TD1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_TD2
-#define GC_CAC_ACC_TD2__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_TD2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_TD3
-#define GC_CAC_ACC_TD3__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_TD3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_TD4
-#define GC_CAC_ACC_TD4__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_TD4__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_TD5
-#define GC_CAC_ACC_TD5__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_TD5__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_VGT0
-#define GC_CAC_ACC_VGT0__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_VGT0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_VGT1
-#define GC_CAC_ACC_VGT1__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_VGT1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_VGT2
-#define GC_CAC_ACC_VGT2__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_VGT2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_ACC_WD0
-#define GC_CAC_ACC_WD0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_WD0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU0
-#define GC_CAC_ACC_CU0__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU1
-#define GC_CAC_ACC_CU1__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU2
-#define GC_CAC_ACC_CU2__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU3
-#define GC_CAC_ACC_CU3__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU4
-#define GC_CAC_ACC_CU4__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU4__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU5
-#define GC_CAC_ACC_CU5__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU5__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU6
-#define GC_CAC_ACC_CU6__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU6__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU7
-#define GC_CAC_ACC_CU7__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU7__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU8
-#define GC_CAC_ACC_CU8__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU8__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU9
-#define GC_CAC_ACC_CU9__ACCUMULATOR_31_0__SHIFT                                                               0x0
-#define GC_CAC_ACC_CU9__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
-//GC_CAC_ACC_CU10
-#define GC_CAC_ACC_CU10__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_CU10__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_OVRD_BCI
-#define GC_CAC_OVRD_BCI__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_BCI__OVRRD_VALUE__SHIFT                                                                   0x2
-#define GC_CAC_OVRD_BCI__OVRRD_SELECT_MASK                                                                    0x00000003L
-#define GC_CAC_OVRD_BCI__OVRRD_VALUE_MASK                                                                     0x0000000CL
-//GC_CAC_OVRD_CB
-#define GC_CAC_OVRD_CB__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_CB__OVRRD_VALUE__SHIFT                                                                    0x4
-#define GC_CAC_OVRD_CB__OVRRD_SELECT_MASK                                                                     0x0000000FL
-#define GC_CAC_OVRD_CB__OVRRD_VALUE_MASK                                                                      0x000000F0L
-//GC_CAC_OVRD_CP
-#define GC_CAC_OVRD_CP__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_CP__OVRRD_VALUE__SHIFT                                                                    0x3
-#define GC_CAC_OVRD_CP__OVRRD_SELECT_MASK                                                                     0x00000007L
-#define GC_CAC_OVRD_CP__OVRRD_VALUE_MASK                                                                      0x00000038L
-//GC_CAC_OVRD_DB
-#define GC_CAC_OVRD_DB__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_DB__OVRRD_VALUE__SHIFT                                                                    0x4
-#define GC_CAC_OVRD_DB__OVRRD_SELECT_MASK                                                                     0x0000000FL
-#define GC_CAC_OVRD_DB__OVRRD_VALUE_MASK                                                                      0x000000F0L
-//GC_CAC_OVRD_GDS
-#define GC_CAC_OVRD_GDS__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_GDS__OVRRD_VALUE__SHIFT                                                                   0x4
-#define GC_CAC_OVRD_GDS__OVRRD_SELECT_MASK                                                                    0x0000000FL
-#define GC_CAC_OVRD_GDS__OVRRD_VALUE_MASK                                                                     0x000000F0L
-//GC_CAC_OVRD_IA
-#define GC_CAC_OVRD_IA__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_IA__OVRRD_VALUE__SHIFT                                                                    0x1
-#define GC_CAC_OVRD_IA__OVRRD_SELECT_MASK                                                                     0x00000001L
-#define GC_CAC_OVRD_IA__OVRRD_VALUE_MASK                                                                      0x00000002L
-//GC_CAC_OVRD_LDS
-#define GC_CAC_OVRD_LDS__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_LDS__OVRRD_VALUE__SHIFT                                                                   0x4
-#define GC_CAC_OVRD_LDS__OVRRD_SELECT_MASK                                                                    0x0000000FL
-#define GC_CAC_OVRD_LDS__OVRRD_VALUE_MASK                                                                     0x000000F0L
-//GC_CAC_OVRD_PA
-#define GC_CAC_OVRD_PA__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_PA__OVRRD_VALUE__SHIFT                                                                    0x2
-#define GC_CAC_OVRD_PA__OVRRD_SELECT_MASK                                                                     0x00000003L
-#define GC_CAC_OVRD_PA__OVRRD_VALUE_MASK                                                                      0x0000000CL
-//GC_CAC_OVRD_PC
-#define GC_CAC_OVRD_PC__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_PC__OVRRD_VALUE__SHIFT                                                                    0x1
-#define GC_CAC_OVRD_PC__OVRRD_SELECT_MASK                                                                     0x00000001L
-#define GC_CAC_OVRD_PC__OVRRD_VALUE_MASK                                                                      0x00000002L
-//GC_CAC_OVRD_SC
-#define GC_CAC_OVRD_SC__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_SC__OVRRD_VALUE__SHIFT                                                                    0x1
-#define GC_CAC_OVRD_SC__OVRRD_SELECT_MASK                                                                     0x00000001L
-#define GC_CAC_OVRD_SC__OVRRD_VALUE_MASK                                                                      0x00000002L
-//GC_CAC_OVRD_SPI
-#define GC_CAC_OVRD_SPI__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_SPI__OVRRD_VALUE__SHIFT                                                                   0x6
-#define GC_CAC_OVRD_SPI__OVRRD_SELECT_MASK                                                                    0x0000003FL
-#define GC_CAC_OVRD_SPI__OVRRD_VALUE_MASK                                                                     0x00000FC0L
-//GC_CAC_OVRD_CU
-#define GC_CAC_OVRD_CU__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_CU__OVRRD_VALUE__SHIFT                                                                    0x1
-#define GC_CAC_OVRD_CU__OVRRD_SELECT_MASK                                                                     0x00000001L
-#define GC_CAC_OVRD_CU__OVRRD_VALUE_MASK                                                                      0x00000002L
-//GC_CAC_OVRD_SQ
-#define GC_CAC_OVRD_SQ__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_SQ__OVRRD_VALUE__SHIFT                                                                    0x9
-#define GC_CAC_OVRD_SQ__OVRRD_SELECT_MASK                                                                     0x000001FFL
-#define GC_CAC_OVRD_SQ__OVRRD_VALUE_MASK                                                                      0x0003FE00L
-//GC_CAC_OVRD_SX
-#define GC_CAC_OVRD_SX__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_SX__OVRRD_VALUE__SHIFT                                                                    0x1
-#define GC_CAC_OVRD_SX__OVRRD_SELECT_MASK                                                                     0x00000001L
-#define GC_CAC_OVRD_SX__OVRRD_VALUE_MASK                                                                      0x00000002L
-//GC_CAC_OVRD_SXRB
-#define GC_CAC_OVRD_SXRB__OVRRD_SELECT__SHIFT                                                                 0x0
-#define GC_CAC_OVRD_SXRB__OVRRD_VALUE__SHIFT                                                                  0x1
-#define GC_CAC_OVRD_SXRB__OVRRD_SELECT_MASK                                                                   0x00000001L
-#define GC_CAC_OVRD_SXRB__OVRRD_VALUE_MASK                                                                    0x00000002L
-//GC_CAC_OVRD_TA
-#define GC_CAC_OVRD_TA__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_TA__OVRRD_VALUE__SHIFT                                                                    0x1
-#define GC_CAC_OVRD_TA__OVRRD_SELECT_MASK                                                                     0x00000001L
-#define GC_CAC_OVRD_TA__OVRRD_VALUE_MASK                                                                      0x00000002L
-//GC_CAC_OVRD_TCC
-#define GC_CAC_OVRD_TCC__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_TCC__OVRRD_VALUE__SHIFT                                                                   0x5
-#define GC_CAC_OVRD_TCC__OVRRD_SELECT_MASK                                                                    0x0000001FL
-#define GC_CAC_OVRD_TCC__OVRRD_VALUE_MASK                                                                     0x000003E0L
-//GC_CAC_OVRD_TCP
-#define GC_CAC_OVRD_TCP__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_TCP__OVRRD_VALUE__SHIFT                                                                   0x5
-#define GC_CAC_OVRD_TCP__OVRRD_SELECT_MASK                                                                    0x0000001FL
-#define GC_CAC_OVRD_TCP__OVRRD_VALUE_MASK                                                                     0x000003E0L
-//GC_CAC_OVRD_TD
-#define GC_CAC_OVRD_TD__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_TD__OVRRD_VALUE__SHIFT                                                                    0x6
-#define GC_CAC_OVRD_TD__OVRRD_SELECT_MASK                                                                     0x0000003FL
-#define GC_CAC_OVRD_TD__OVRRD_VALUE_MASK                                                                      0x00000FC0L
-//GC_CAC_OVRD_VGT
-#define GC_CAC_OVRD_VGT__OVRRD_SELECT__SHIFT                                                                  0x0
-#define GC_CAC_OVRD_VGT__OVRRD_VALUE__SHIFT                                                                   0x3
-#define GC_CAC_OVRD_VGT__OVRRD_SELECT_MASK                                                                    0x00000007L
-#define GC_CAC_OVRD_VGT__OVRRD_VALUE_MASK                                                                     0x00000038L
-//GC_CAC_OVRD_WD
-#define GC_CAC_OVRD_WD__OVRRD_SELECT__SHIFT                                                                   0x0
-#define GC_CAC_OVRD_WD__OVRRD_VALUE__SHIFT                                                                    0x1
-#define GC_CAC_OVRD_WD__OVRRD_SELECT_MASK                                                                     0x00000001L
-#define GC_CAC_OVRD_WD__OVRRD_VALUE_MASK                                                                      0x00000002L
-//GC_CAC_ACC_BCI1
-#define GC_CAC_ACC_BCI1__ACCUMULATOR_31_0__SHIFT                                                              0x0
-#define GC_CAC_ACC_BCI1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
-//GC_CAC_WEIGHT_UTCL2_ATCL2_2
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_2__WEIGHT_UTCL2_ATCL2_SIG4__SHIFT                                           0x0
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_2__WEIGHT_UTCL2_ATCL2_SIG5__SHIFT                                           0x10
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_2__WEIGHT_UTCL2_ATCL2_SIG4_MASK                                             0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ATCL2_2__WEIGHT_UTCL2_ATCL2_SIG5_MASK                                             0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_ROUTER_0
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG0__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG1__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG0_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG1_MASK                                           0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_ROUTER_1
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG2__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG3__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG2_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG3_MASK                                           0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_ROUTER_2
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG4__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG5__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG4_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG5_MASK                                           0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_ROUTER_3
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG6__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG7__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG6_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG7_MASK                                           0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_ROUTER_4
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG8__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG9__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG8_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG9_MASK                                           0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_VML2_0
-#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG0__SHIFT                                             0x0
-#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG1__SHIFT                                             0x10
-#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG0_MASK                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG1_MASK                                               0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_VML2_1
-#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG2__SHIFT                                             0x0
-#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG3__SHIFT                                             0x10
-#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG2_MASK                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG3_MASK                                               0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_VML2_2
-#define GC_CAC_WEIGHT_UTCL2_VML2_2__WEIGHT_UTCL2_VML2_SIG4__SHIFT                                             0x0
-#define GC_CAC_WEIGHT_UTCL2_VML2_2__WEIGHT_UTCL2_VML2_SIG5__SHIFT                                             0x10
-#define GC_CAC_WEIGHT_UTCL2_VML2_2__WEIGHT_UTCL2_VML2_SIG4_MASK                                               0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_VML2_2__WEIGHT_UTCL2_VML2_SIG5_MASK                                               0xFFFF0000L
-//GC_CAC_ACC_UTCL2_ATCL24
-#define GC_CAC_ACC_UTCL2_ATCL24__ACCUMULATOR_31_0__SHIFT                                                      0x0
-#define GC_CAC_ACC_UTCL2_ATCL24__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER0
-#define GC_CAC_ACC_UTCL2_ROUTER0__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER0__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER1
-#define GC_CAC_ACC_UTCL2_ROUTER1__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER1__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER2
-#define GC_CAC_ACC_UTCL2_ROUTER2__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER2__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER3
-#define GC_CAC_ACC_UTCL2_ROUTER3__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER3__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER4
-#define GC_CAC_ACC_UTCL2_ROUTER4__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER4__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER5
-#define GC_CAC_ACC_UTCL2_ROUTER5__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER5__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER6
-#define GC_CAC_ACC_UTCL2_ROUTER6__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER6__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER7
-#define GC_CAC_ACC_UTCL2_ROUTER7__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER7__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER8
-#define GC_CAC_ACC_UTCL2_ROUTER8__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER8__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_ROUTER9
-#define GC_CAC_ACC_UTCL2_ROUTER9__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_ROUTER9__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_VML20
-#define GC_CAC_ACC_UTCL2_VML20__ACCUMULATOR_31_0__SHIFT                                                       0x0
-#define GC_CAC_ACC_UTCL2_VML20__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_VML21
-#define GC_CAC_ACC_UTCL2_VML21__ACCUMULATOR_31_0__SHIFT                                                       0x0
-#define GC_CAC_ACC_UTCL2_VML21__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_VML22
-#define GC_CAC_ACC_UTCL2_VML22__ACCUMULATOR_31_0__SHIFT                                                       0x0
-#define GC_CAC_ACC_UTCL2_VML22__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_VML23
-#define GC_CAC_ACC_UTCL2_VML23__ACCUMULATOR_31_0__SHIFT                                                       0x0
-#define GC_CAC_ACC_UTCL2_VML23__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_VML24
-#define GC_CAC_ACC_UTCL2_VML24__ACCUMULATOR_31_0__SHIFT                                                       0x0
-#define GC_CAC_ACC_UTCL2_VML24__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
-//GC_CAC_OVRD_UTCL2_ROUTER
-#define GC_CAC_OVRD_UTCL2_ROUTER__OVRRD_SELECT__SHIFT                                                         0x0
-#define GC_CAC_OVRD_UTCL2_ROUTER__OVRRD_VALUE__SHIFT                                                          0xa
-#define GC_CAC_OVRD_UTCL2_ROUTER__OVRRD_SELECT_MASK                                                           0x000003FFL
-#define GC_CAC_OVRD_UTCL2_ROUTER__OVRRD_VALUE_MASK                                                            0x000FFC00L
-//GC_CAC_OVRD_UTCL2_VML2
-#define GC_CAC_OVRD_UTCL2_VML2__OVRRD_SELECT__SHIFT                                                           0x0
-#define GC_CAC_OVRD_UTCL2_VML2__OVRRD_VALUE__SHIFT                                                            0x5
-#define GC_CAC_OVRD_UTCL2_VML2__OVRRD_SELECT_MASK                                                             0x0000001FL
-#define GC_CAC_OVRD_UTCL2_VML2__OVRRD_VALUE_MASK                                                              0x000003E0L
-//GC_CAC_WEIGHT_UTCL2_WALKER_0
-#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG0__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG1__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG0_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG1_MASK                                           0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_WALKER_1
-#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG2__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG3__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG2_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG3_MASK                                           0xFFFF0000L
-//GC_CAC_WEIGHT_UTCL2_WALKER_2
-#define GC_CAC_WEIGHT_UTCL2_WALKER_2__WEIGHT_UTCL2_WALKER_SIG4__SHIFT                                         0x0
-#define GC_CAC_WEIGHT_UTCL2_WALKER_2__WEIGHT_UTCL2_WALKER_SIG5__SHIFT                                         0x10
-#define GC_CAC_WEIGHT_UTCL2_WALKER_2__WEIGHT_UTCL2_WALKER_SIG4_MASK                                           0x0000FFFFL
-#define GC_CAC_WEIGHT_UTCL2_WALKER_2__WEIGHT_UTCL2_WALKER_SIG5_MASK                                           0xFFFF0000L
-//GC_CAC_ACC_UTCL2_WALKER0
-#define GC_CAC_ACC_UTCL2_WALKER0__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_WALKER0__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_WALKER1
-#define GC_CAC_ACC_UTCL2_WALKER1__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_WALKER1__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_WALKER2
-#define GC_CAC_ACC_UTCL2_WALKER2__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_WALKER2__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_WALKER3
-#define GC_CAC_ACC_UTCL2_WALKER3__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_WALKER3__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_ACC_UTCL2_WALKER4
-#define GC_CAC_ACC_UTCL2_WALKER4__ACCUMULATOR_31_0__SHIFT                                                     0x0
-#define GC_CAC_ACC_UTCL2_WALKER4__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
-//GC_CAC_OVRD_UTCL2_WALKER
-#define GC_CAC_OVRD_UTCL2_WALKER__OVRRD_SELECT__SHIFT                                                         0x0
-#define GC_CAC_OVRD_UTCL2_WALKER__OVRRD_VALUE__SHIFT                                                          0x5
-#define GC_CAC_OVRD_UTCL2_WALKER__OVRRD_SELECT_MASK                                                           0x0000001FL
-#define GC_CAC_OVRD_UTCL2_WALKER__OVRRD_VALUE_MASK                                                            0x000003E0L
-
-
-// addressBlock: secacind
-//SE_CAC_CNTL
-#define SE_CAC_CNTL__CAC_ENABLE__SHIFT                                                                        0x0
-#define SE_CAC_CNTL__CAC_THRESHOLD__SHIFT                                                                     0x1
-#define SE_CAC_CNTL__CAC_BLOCK_ID__SHIFT                                                                      0x11
-#define SE_CAC_CNTL__CAC_SIGNAL_ID__SHIFT                                                                     0x17
-#define SE_CAC_CNTL__UNUSED_0__SHIFT                                                                          0x1f
-#define SE_CAC_CNTL__CAC_ENABLE_MASK                                                                          0x00000001L
-#define SE_CAC_CNTL__CAC_THRESHOLD_MASK                                                                       0x0001FFFEL
-#define SE_CAC_CNTL__CAC_BLOCK_ID_MASK                                                                        0x007E0000L
-#define SE_CAC_CNTL__CAC_SIGNAL_ID_MASK                                                                       0x7F800000L
-#define SE_CAC_CNTL__UNUSED_0_MASK                                                                            0x80000000L
-//SE_CAC_OVR_SEL
-#define SE_CAC_OVR_SEL__CAC_OVR_SEL__SHIFT                                                                    0x0
-#define SE_CAC_OVR_SEL__CAC_OVR_SEL_MASK                                                                      0xFFFFFFFFL
-//SE_CAC_OVR_VAL
-#define SE_CAC_OVR_VAL__CAC_OVR_VAL__SHIFT                                                                    0x0
-#define SE_CAC_OVR_VAL__CAC_OVR_VAL_MASK                                                                      0xFFFFFFFFL
-
-
-// addressBlock: sqind
-//SQ_WAVE_MODE
-#define SQ_WAVE_MODE__FP_ROUND__SHIFT                                                                         0x0
-#define SQ_WAVE_MODE__FP_DENORM__SHIFT                                                                        0x4
-#define SQ_WAVE_MODE__DX10_CLAMP__SHIFT                                                                       0x8
-#define SQ_WAVE_MODE__IEEE__SHIFT                                                                             0x9
-#define SQ_WAVE_MODE__LOD_CLAMPED__SHIFT                                                                      0xa
-#define SQ_WAVE_MODE__EXCP_EN__SHIFT                                                                          0xc
-#define SQ_WAVE_MODE__FP16_OVFL__SHIFT                                                                        0x17
-#define SQ_WAVE_MODE__POPS_PACKER0__SHIFT                                                                     0x18
-#define SQ_WAVE_MODE__POPS_PACKER1__SHIFT                                                                     0x19
-#define SQ_WAVE_MODE__DISABLE_PERF__SHIFT                                                                     0x1a
-#define SQ_WAVE_MODE__GPR_IDX_EN__SHIFT                                                                       0x1b
-#define SQ_WAVE_MODE__VSKIP__SHIFT                                                                            0x1c
-#define SQ_WAVE_MODE__CSP__SHIFT                                                                              0x1d
-#define SQ_WAVE_MODE__FP_ROUND_MASK                                                                           0x0000000FL
-#define SQ_WAVE_MODE__FP_DENORM_MASK                                                                          0x000000F0L
-#define SQ_WAVE_MODE__DX10_CLAMP_MASK                                                                         0x00000100L
-#define SQ_WAVE_MODE__IEEE_MASK                                                                               0x00000200L
-#define SQ_WAVE_MODE__LOD_CLAMPED_MASK                                                                        0x00000400L
-#define SQ_WAVE_MODE__EXCP_EN_MASK                                                                            0x001FF000L
-#define SQ_WAVE_MODE__FP16_OVFL_MASK                                                                          0x00800000L
-#define SQ_WAVE_MODE__POPS_PACKER0_MASK                                                                       0x01000000L
-#define SQ_WAVE_MODE__POPS_PACKER1_MASK                                                                       0x02000000L
-#define SQ_WAVE_MODE__DISABLE_PERF_MASK                                                                       0x04000000L
-#define SQ_WAVE_MODE__GPR_IDX_EN_MASK                                                                         0x08000000L
-#define SQ_WAVE_MODE__VSKIP_MASK                                                                              0x10000000L
-#define SQ_WAVE_MODE__CSP_MASK                                                                                0xE0000000L
-//SQ_WAVE_STATUS
-#define SQ_WAVE_STATUS__SCC__SHIFT                                                                            0x0
-#define SQ_WAVE_STATUS__SPI_PRIO__SHIFT                                                                       0x1
-#define SQ_WAVE_STATUS__USER_PRIO__SHIFT                                                                      0x3
-#define SQ_WAVE_STATUS__PRIV__SHIFT                                                                           0x5
-#define SQ_WAVE_STATUS__TRAP_EN__SHIFT                                                                        0x6
-#define SQ_WAVE_STATUS__TTRACE_EN__SHIFT                                                                      0x7
-#define SQ_WAVE_STATUS__EXPORT_RDY__SHIFT                                                                     0x8
-#define SQ_WAVE_STATUS__EXECZ__SHIFT                                                                          0x9
-#define SQ_WAVE_STATUS__VCCZ__SHIFT                                                                           0xa
-#define SQ_WAVE_STATUS__IN_TG__SHIFT                                                                          0xb
-#define SQ_WAVE_STATUS__IN_BARRIER__SHIFT                                                                     0xc
-#define SQ_WAVE_STATUS__HALT__SHIFT                                                                           0xd
-#define SQ_WAVE_STATUS__TRAP__SHIFT                                                                           0xe
-#define SQ_WAVE_STATUS__TTRACE_CU_EN__SHIFT                                                                   0xf
-#define SQ_WAVE_STATUS__VALID__SHIFT                                                                          0x10
-#define SQ_WAVE_STATUS__ECC_ERR__SHIFT                                                                        0x11
-#define SQ_WAVE_STATUS__SKIP_EXPORT__SHIFT                                                                    0x12
-#define SQ_WAVE_STATUS__PERF_EN__SHIFT                                                                        0x13
-#define SQ_WAVE_STATUS__ALLOW_REPLAY__SHIFT                                                                   0x16
-#define SQ_WAVE_STATUS__FATAL_HALT__SHIFT                                                                     0x17
-#define SQ_WAVE_STATUS__MUST_EXPORT__SHIFT                                                                    0x1b
-#define SQ_WAVE_STATUS__SCC_MASK                                                                              0x00000001L
-#define SQ_WAVE_STATUS__SPI_PRIO_MASK                                                                         0x00000006L
-#define SQ_WAVE_STATUS__USER_PRIO_MASK                                                                        0x00000018L
-#define SQ_WAVE_STATUS__PRIV_MASK                                                                             0x00000020L
-#define SQ_WAVE_STATUS__TRAP_EN_MASK                                                                          0x00000040L
-#define SQ_WAVE_STATUS__TTRACE_EN_MASK                                                                        0x00000080L
-#define SQ_WAVE_STATUS__EXPORT_RDY_MASK                                                                       0x00000100L
-#define SQ_WAVE_STATUS__EXECZ_MASK                                                                            0x00000200L
-#define SQ_WAVE_STATUS__VCCZ_MASK                                                                             0x00000400L
-#define SQ_WAVE_STATUS__IN_TG_MASK                                                                            0x00000800L
-#define SQ_WAVE_STATUS__IN_BARRIER_MASK                                                                       0x00001000L
-#define SQ_WAVE_STATUS__HALT_MASK                                                                             0x00002000L
-#define SQ_WAVE_STATUS__TRAP_MASK                                                                             0x00004000L
-#define SQ_WAVE_STATUS__TTRACE_CU_EN_MASK                                                                     0x00008000L
-#define SQ_WAVE_STATUS__VALID_MASK                                                                            0x00010000L
-#define SQ_WAVE_STATUS__ECC_ERR_MASK                                                                          0x00020000L
-#define SQ_WAVE_STATUS__SKIP_EXPORT_MASK                                                                      0x00040000L
-#define SQ_WAVE_STATUS__PERF_EN_MASK                                                                          0x00080000L
-#define SQ_WAVE_STATUS__ALLOW_REPLAY_MASK                                                                     0x00400000L
-#define SQ_WAVE_STATUS__FATAL_HALT_MASK                                                                       0x00800000L
-#define SQ_WAVE_STATUS__MUST_EXPORT_MASK                                                                      0x08000000L
-//SQ_WAVE_TRAPSTS
-#define SQ_WAVE_TRAPSTS__EXCP__SHIFT                                                                          0x0
-#define SQ_WAVE_TRAPSTS__SAVECTX__SHIFT                                                                       0xa
-#define SQ_WAVE_TRAPSTS__ILLEGAL_INST__SHIFT                                                                  0xb
-#define SQ_WAVE_TRAPSTS__EXCP_HI__SHIFT                                                                       0xc
-#define SQ_WAVE_TRAPSTS__EXCP_CYCLE__SHIFT                                                                    0x10
-#define SQ_WAVE_TRAPSTS__XNACK_ERROR__SHIFT                                                                   0x1c
-#define SQ_WAVE_TRAPSTS__DP_RATE__SHIFT                                                                       0x1d
-#define SQ_WAVE_TRAPSTS__EXCP_MASK                                                                            0x000001FFL
-#define SQ_WAVE_TRAPSTS__SAVECTX_MASK                                                                         0x00000400L
-#define SQ_WAVE_TRAPSTS__ILLEGAL_INST_MASK                                                                    0x00000800L
-#define SQ_WAVE_TRAPSTS__EXCP_HI_MASK                                                                         0x00007000L
-#define SQ_WAVE_TRAPSTS__EXCP_CYCLE_MASK                                                                      0x003F0000L
-#define SQ_WAVE_TRAPSTS__XNACK_ERROR_MASK                                                                     0x10000000L
-#define SQ_WAVE_TRAPSTS__DP_RATE_MASK                                                                         0xE0000000L
-//SQ_WAVE_HW_ID
-#define SQ_WAVE_HW_ID__WAVE_ID__SHIFT                                                                         0x0
-#define SQ_WAVE_HW_ID__SIMD_ID__SHIFT                                                                         0x4
-#define SQ_WAVE_HW_ID__PIPE_ID__SHIFT                                                                         0x6
-#define SQ_WAVE_HW_ID__CU_ID__SHIFT                                                                           0x8
-#define SQ_WAVE_HW_ID__SH_ID__SHIFT                                                                           0xc
-#define SQ_WAVE_HW_ID__SE_ID__SHIFT                                                                           0xd
-#define SQ_WAVE_HW_ID__TG_ID__SHIFT                                                                           0x10
-#define SQ_WAVE_HW_ID__VM_ID__SHIFT                                                                           0x14
-#define SQ_WAVE_HW_ID__QUEUE_ID__SHIFT                                                                        0x18
-#define SQ_WAVE_HW_ID__STATE_ID__SHIFT                                                                        0x1b
-#define SQ_WAVE_HW_ID__ME_ID__SHIFT                                                                           0x1e
-#define SQ_WAVE_HW_ID__WAVE_ID_MASK                                                                           0x0000000FL
-#define SQ_WAVE_HW_ID__SIMD_ID_MASK                                                                           0x00000030L
-#define SQ_WAVE_HW_ID__PIPE_ID_MASK                                                                           0x000000C0L
-#define SQ_WAVE_HW_ID__CU_ID_MASK                                                                             0x00000F00L
-#define SQ_WAVE_HW_ID__SH_ID_MASK                                                                             0x00001000L
-#define SQ_WAVE_HW_ID__SE_ID_MASK                                                                             0x00006000L
-#define SQ_WAVE_HW_ID__TG_ID_MASK                                                                             0x000F0000L
-#define SQ_WAVE_HW_ID__VM_ID_MASK                                                                             0x00F00000L
-#define SQ_WAVE_HW_ID__QUEUE_ID_MASK                                                                          0x07000000L
-#define SQ_WAVE_HW_ID__STATE_ID_MASK                                                                          0x38000000L
-#define SQ_WAVE_HW_ID__ME_ID_MASK                                                                             0xC0000000L
-//SQ_WAVE_GPR_ALLOC
-#define SQ_WAVE_GPR_ALLOC__VGPR_BASE__SHIFT                                                                   0x0
-#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE__SHIFT                                                                   0x8
-#define SQ_WAVE_GPR_ALLOC__SGPR_BASE__SHIFT                                                                   0x10
-#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE__SHIFT                                                                   0x18
-#define SQ_WAVE_GPR_ALLOC__VGPR_BASE_MASK                                                                     0x0000003FL
-#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE_MASK                                                                     0x00003F00L
-#define SQ_WAVE_GPR_ALLOC__SGPR_BASE_MASK                                                                     0x003F0000L
-#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE_MASK                                                                     0x0F000000L
-//SQ_WAVE_LDS_ALLOC
-#define SQ_WAVE_LDS_ALLOC__LDS_BASE__SHIFT                                                                    0x0
-#define SQ_WAVE_LDS_ALLOC__LDS_SIZE__SHIFT                                                                    0xc
-#define SQ_WAVE_LDS_ALLOC__LDS_BASE_MASK                                                                      0x000000FFL
-#define SQ_WAVE_LDS_ALLOC__LDS_SIZE_MASK                                                                      0x001FF000L
-//SQ_WAVE_IB_STS
-#define SQ_WAVE_IB_STS__VM_CNT__SHIFT                                                                         0x0
-#define SQ_WAVE_IB_STS__EXP_CNT__SHIFT                                                                        0x4
-#define SQ_WAVE_IB_STS__LGKM_CNT__SHIFT                                                                       0x8
-#define SQ_WAVE_IB_STS__VALU_CNT__SHIFT                                                                       0xc
-#define SQ_WAVE_IB_STS__FIRST_REPLAY__SHIFT                                                                   0xf
-#define SQ_WAVE_IB_STS__RCNT__SHIFT                                                                           0x10
-#define SQ_WAVE_IB_STS__VM_CNT_HI__SHIFT                                                                      0x16
-#define SQ_WAVE_IB_STS__VM_CNT_MASK                                                                           0x0000000FL
-#define SQ_WAVE_IB_STS__EXP_CNT_MASK                                                                          0x00000070L
-#define SQ_WAVE_IB_STS__LGKM_CNT_MASK                                                                         0x00000F00L
-#define SQ_WAVE_IB_STS__VALU_CNT_MASK                                                                         0x00007000L
-#define SQ_WAVE_IB_STS__FIRST_REPLAY_MASK                                                                     0x00008000L
-#define SQ_WAVE_IB_STS__RCNT_MASK                                                                             0x001F0000L
-#define SQ_WAVE_IB_STS__VM_CNT_HI_MASK                                                                        0x00C00000L
-//SQ_WAVE_PC_LO
-#define SQ_WAVE_PC_LO__PC_LO__SHIFT                                                                           0x0
-#define SQ_WAVE_PC_LO__PC_LO_MASK                                                                             0xFFFFFFFFL
-//SQ_WAVE_PC_HI
-#define SQ_WAVE_PC_HI__PC_HI__SHIFT                                                                           0x0
-#define SQ_WAVE_PC_HI__PC_HI_MASK                                                                             0x0000FFFFL
-//SQ_WAVE_INST_DW0
-#define SQ_WAVE_INST_DW0__INST_DW0__SHIFT                                                                     0x0
-#define SQ_WAVE_INST_DW0__INST_DW0_MASK                                                                       0xFFFFFFFFL
-//SQ_WAVE_INST_DW1
-#define SQ_WAVE_INST_DW1__INST_DW1__SHIFT                                                                     0x0
-#define SQ_WAVE_INST_DW1__INST_DW1_MASK                                                                       0xFFFFFFFFL
-//SQ_WAVE_IB_DBG0
-#define SQ_WAVE_IB_DBG0__IBUF_ST__SHIFT                                                                       0x0
-#define SQ_WAVE_IB_DBG0__PC_INVALID__SHIFT                                                                    0x3
-#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW__SHIFT                                                                  0x4
-#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT__SHIFT                                                               0x5
-#define SQ_WAVE_IB_DBG0__IBUF_RPTR__SHIFT                                                                     0x8
-#define SQ_WAVE_IB_DBG0__IBUF_WPTR__SHIFT                                                                     0xa
-#define SQ_WAVE_IB_DBG0__INST_STR_ST__SHIFT                                                                   0x10
-#define SQ_WAVE_IB_DBG0__ECC_ST__SHIFT                                                                        0x18
-#define SQ_WAVE_IB_DBG0__IS_HYB__SHIFT                                                                        0x1a
-#define SQ_WAVE_IB_DBG0__HYB_CNT__SHIFT                                                                       0x1b
-#define SQ_WAVE_IB_DBG0__KILL__SHIFT                                                                          0x1d
-#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH__SHIFT                                                              0x1e
-#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_HI__SHIFT                                                            0x1f
-#define SQ_WAVE_IB_DBG0__IBUF_ST_MASK                                                                         0x00000007L
-#define SQ_WAVE_IB_DBG0__PC_INVALID_MASK                                                                      0x00000008L
-#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW_MASK                                                                    0x00000010L
-#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_MASK                                                                 0x000000E0L
-#define SQ_WAVE_IB_DBG0__IBUF_RPTR_MASK                                                                       0x00000300L
-#define SQ_WAVE_IB_DBG0__IBUF_WPTR_MASK                                                                       0x00000C00L
-#define SQ_WAVE_IB_DBG0__INST_STR_ST_MASK                                                                     0x000F0000L
-#define SQ_WAVE_IB_DBG0__ECC_ST_MASK                                                                          0x03000000L
-#define SQ_WAVE_IB_DBG0__IS_HYB_MASK                                                                          0x04000000L
-#define SQ_WAVE_IB_DBG0__HYB_CNT_MASK                                                                         0x18000000L
-#define SQ_WAVE_IB_DBG0__KILL_MASK                                                                            0x20000000L
-#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH_MASK                                                                0x40000000L
-#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_HI_MASK                                                              0x80000000L
-//SQ_WAVE_IB_DBG1
-#define SQ_WAVE_IB_DBG1__IXNACK__SHIFT                                                                        0x0
-#define SQ_WAVE_IB_DBG1__XNACK__SHIFT                                                                         0x1
-#define SQ_WAVE_IB_DBG1__TA_NEED_RESET__SHIFT                                                                 0x2
-#define SQ_WAVE_IB_DBG1__XCNT__SHIFT                                                                          0x4
-#define SQ_WAVE_IB_DBG1__QCNT__SHIFT                                                                          0xb
-#define SQ_WAVE_IB_DBG1__RCNT__SHIFT                                                                          0x12
-#define SQ_WAVE_IB_DBG1__MISC_CNT__SHIFT                                                                      0x19
-#define SQ_WAVE_IB_DBG1__IXNACK_MASK                                                                          0x00000001L
-#define SQ_WAVE_IB_DBG1__XNACK_MASK                                                                           0x00000002L
-#define SQ_WAVE_IB_DBG1__TA_NEED_RESET_MASK                                                                   0x00000004L
-#define SQ_WAVE_IB_DBG1__XCNT_MASK                                                                            0x000001F0L
-#define SQ_WAVE_IB_DBG1__QCNT_MASK                                                                            0x0000F800L
-#define SQ_WAVE_IB_DBG1__RCNT_MASK                                                                            0x007C0000L
-#define SQ_WAVE_IB_DBG1__MISC_CNT_MASK                                                                        0xFE000000L
-//SQ_WAVE_FLUSH_IB
-#define SQ_WAVE_FLUSH_IB__UNUSED__SHIFT                                                                       0x0
-#define SQ_WAVE_FLUSH_IB__UNUSED_MASK                                                                         0xFFFFFFFFL
-//SQ_WAVE_TTMP0
-#define SQ_WAVE_TTMP0__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP0__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP1
-#define SQ_WAVE_TTMP1__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP1__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP2
-#define SQ_WAVE_TTMP2__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP2__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP3
-#define SQ_WAVE_TTMP3__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP3__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP4
-#define SQ_WAVE_TTMP4__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP4__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP5
-#define SQ_WAVE_TTMP5__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP5__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP6
-#define SQ_WAVE_TTMP6__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP6__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP7
-#define SQ_WAVE_TTMP7__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP7__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP8
-#define SQ_WAVE_TTMP8__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP8__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP9
-#define SQ_WAVE_TTMP9__DATA__SHIFT                                                                            0x0
-#define SQ_WAVE_TTMP9__DATA_MASK                                                                              0xFFFFFFFFL
-//SQ_WAVE_TTMP10
-#define SQ_WAVE_TTMP10__DATA__SHIFT                                                                           0x0
-#define SQ_WAVE_TTMP10__DATA_MASK                                                                             0xFFFFFFFFL
-//SQ_WAVE_TTMP11
-#define SQ_WAVE_TTMP11__DATA__SHIFT                                                                           0x0
-#define SQ_WAVE_TTMP11__DATA_MASK                                                                             0xFFFFFFFFL
-//SQ_WAVE_TTMP12
-#define SQ_WAVE_TTMP12__DATA__SHIFT                                                                           0x0
-#define SQ_WAVE_TTMP12__DATA_MASK                                                                             0xFFFFFFFFL
-//SQ_WAVE_TTMP13
-#define SQ_WAVE_TTMP13__DATA__SHIFT                                                                           0x0
-#define SQ_WAVE_TTMP13__DATA_MASK                                                                             0xFFFFFFFFL
-//SQ_WAVE_TTMP14
-#define SQ_WAVE_TTMP14__DATA__SHIFT                                                                           0x0
-#define SQ_WAVE_TTMP14__DATA_MASK                                                                             0xFFFFFFFFL
-//SQ_WAVE_TTMP15
-#define SQ_WAVE_TTMP15__DATA__SHIFT                                                                           0x0
-#define SQ_WAVE_TTMP15__DATA_MASK                                                                             0xFFFFFFFFL
-//SQ_WAVE_M0
-#define SQ_WAVE_M0__M0__SHIFT                                                                                 0x0
-#define SQ_WAVE_M0__M0_MASK                                                                                   0xFFFFFFFFL
-//SQ_WAVE_EXEC_LO
-#define SQ_WAVE_EXEC_LO__EXEC_LO__SHIFT                                                                       0x0
-#define SQ_WAVE_EXEC_LO__EXEC_LO_MASK                                                                         0xFFFFFFFFL
-//SQ_WAVE_EXEC_HI
-#define SQ_WAVE_EXEC_HI__EXEC_HI__SHIFT                                                                       0x0
-#define SQ_WAVE_EXEC_HI__EXEC_HI_MASK                                                                         0xFFFFFFFFL
-//SQ_INTERRUPT_WORD_AUTO_CTXID
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE__SHIFT                                                     0x0
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT__SHIFT                                                              0x1
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL__SHIFT                                            0x2
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP__SHIFT                                                    0x3
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP__SHIFT                                                    0x4
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW__SHIFT                                                0x5
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW__SHIFT                                                0x6
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW__SHIFT                                                   0x7
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR__SHIFT                                           0x8
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID__SHIFT                                                            0x18
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING__SHIFT                                                         0x1a
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_MASK                                                       0x0000001L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__WLT_MASK                                                                0x0000002L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_BUF_FULL_MASK                                              0x0000004L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__REG_TIMESTAMP_MASK                                                      0x0000008L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__CMD_TIMESTAMP_MASK                                                      0x0000010L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_CMD_OVERFLOW_MASK                                                  0x0000020L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__HOST_REG_OVERFLOW_MASK                                                  0x0000040L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__IMMED_OVERFLOW_MASK                                                     0x0000080L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__THREAD_TRACE_UTC_ERROR_MASK                                             0x0000100L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__SE_ID_MASK                                                              0x3000000L
-#define SQ_INTERRUPT_WORD_AUTO_CTXID__ENCODING_MASK                                                           0xC000000L
-//SQ_INTERRUPT_WORD_AUTO_HI
-#define SQ_INTERRUPT_WORD_AUTO_HI__SE_ID__SHIFT                                                               0x8
-#define SQ_INTERRUPT_WORD_AUTO_HI__ENCODING__SHIFT                                                            0xa
-#define SQ_INTERRUPT_WORD_AUTO_HI__SE_ID_MASK                                                                 0x300L
-#define SQ_INTERRUPT_WORD_AUTO_HI__ENCODING_MASK                                                              0xC00L
-//SQ_INTERRUPT_WORD_AUTO_LO
-#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE__SHIFT                                                        0x0
-#define SQ_INTERRUPT_WORD_AUTO_LO__WLT__SHIFT                                                                 0x1
-#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_BUF_FULL__SHIFT                                               0x2
-#define SQ_INTERRUPT_WORD_AUTO_LO__REG_TIMESTAMP__SHIFT                                                       0x3
-#define SQ_INTERRUPT_WORD_AUTO_LO__CMD_TIMESTAMP__SHIFT                                                       0x4
-#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_CMD_OVERFLOW__SHIFT                                                   0x5
-#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_REG_OVERFLOW__SHIFT                                                   0x6
-#define SQ_INTERRUPT_WORD_AUTO_LO__IMMED_OVERFLOW__SHIFT                                                      0x7
-#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_UTC_ERROR__SHIFT                                              0x8
-#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_MASK                                                          0x001L
-#define SQ_INTERRUPT_WORD_AUTO_LO__WLT_MASK                                                                   0x002L
-#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_BUF_FULL_MASK                                                 0x004L
-#define SQ_INTERRUPT_WORD_AUTO_LO__REG_TIMESTAMP_MASK                                                         0x008L
-#define SQ_INTERRUPT_WORD_AUTO_LO__CMD_TIMESTAMP_MASK                                                         0x010L
-#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_CMD_OVERFLOW_MASK                                                     0x020L
-#define SQ_INTERRUPT_WORD_AUTO_LO__HOST_REG_OVERFLOW_MASK                                                     0x040L
-#define SQ_INTERRUPT_WORD_AUTO_LO__IMMED_OVERFLOW_MASK                                                        0x080L
-#define SQ_INTERRUPT_WORD_AUTO_LO__THREAD_TRACE_UTC_ERROR_MASK                                                0x100L
-//SQ_INTERRUPT_WORD_CMN_CTXID
-#define SQ_INTERRUPT_WORD_CMN_CTXID__SE_ID__SHIFT                                                             0x18
-#define SQ_INTERRUPT_WORD_CMN_CTXID__ENCODING__SHIFT                                                          0x1a
-#define SQ_INTERRUPT_WORD_CMN_CTXID__SE_ID_MASK                                                               0x3000000L
-#define SQ_INTERRUPT_WORD_CMN_CTXID__ENCODING_MASK                                                            0xC000000L
-//SQ_INTERRUPT_WORD_CMN_HI
-#define SQ_INTERRUPT_WORD_CMN_HI__SE_ID__SHIFT                                                                0x8
-#define SQ_INTERRUPT_WORD_CMN_HI__ENCODING__SHIFT                                                             0xa
-#define SQ_INTERRUPT_WORD_CMN_HI__SE_ID_MASK                                                                  0x300L
-#define SQ_INTERRUPT_WORD_CMN_HI__ENCODING_MASK                                                               0xC00L
-//SQ_INTERRUPT_WORD_WAVE_CTXID
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA__SHIFT                                                             0x0
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID__SHIFT                                                            0xc
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV__SHIFT                                                             0xd
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID__SHIFT                                                          0xe
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID__SHIFT                                                          0x12
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID__SHIFT                                                            0x14
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID__SHIFT                                                            0x18
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING__SHIFT                                                         0x1a
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__DATA_MASK                                                               0x0000FFFL
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__SH_ID_MASK                                                              0x0001000L
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__PRIV_MASK                                                               0x0002000L
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__WAVE_ID_MASK                                                            0x003C000L
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__SIMD_ID_MASK                                                            0x00C0000L
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__CU_ID_MASK                                                              0x0F00000L
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__SE_ID_MASK                                                              0x3000000L
-#define SQ_INTERRUPT_WORD_WAVE_CTXID__ENCODING_MASK                                                           0xC000000L
-//SQ_INTERRUPT_WORD_WAVE_HI
-#define SQ_INTERRUPT_WORD_WAVE_HI__CU_ID__SHIFT                                                               0x0
-#define SQ_INTERRUPT_WORD_WAVE_HI__VM_ID__SHIFT                                                               0x4
-#define SQ_INTERRUPT_WORD_WAVE_HI__SE_ID__SHIFT                                                               0x8
-#define SQ_INTERRUPT_WORD_WAVE_HI__ENCODING__SHIFT                                                            0xa
-#define SQ_INTERRUPT_WORD_WAVE_HI__CU_ID_MASK                                                                 0x00FL
-#define SQ_INTERRUPT_WORD_WAVE_HI__VM_ID_MASK                                                                 0x0F0L
-#define SQ_INTERRUPT_WORD_WAVE_HI__SE_ID_MASK                                                                 0x300L
-#define SQ_INTERRUPT_WORD_WAVE_HI__ENCODING_MASK                                                              0xC00L
-//SQ_INTERRUPT_WORD_WAVE_LO
-#define SQ_INTERRUPT_WORD_WAVE_LO__DATA__SHIFT                                                                0x0
-#define SQ_INTERRUPT_WORD_WAVE_LO__SH_ID__SHIFT                                                               0x18
-#define SQ_INTERRUPT_WORD_WAVE_LO__PRIV__SHIFT                                                                0x19
-#define SQ_INTERRUPT_WORD_WAVE_LO__WAVE_ID__SHIFT                                                             0x1a
-#define SQ_INTERRUPT_WORD_WAVE_LO__SIMD_ID__SHIFT                                                             0x1e
-#define SQ_INTERRUPT_WORD_WAVE_LO__DATA_MASK                                                                  0x00FFFFFFL
-#define SQ_INTERRUPT_WORD_WAVE_LO__SH_ID_MASK                                                                 0x01000000L
-#define SQ_INTERRUPT_WORD_WAVE_LO__PRIV_MASK                                                                  0x02000000L
-#define SQ_INTERRUPT_WORD_WAVE_LO__WAVE_ID_MASK                                                               0x3C000000L
-#define SQ_INTERRUPT_WORD_WAVE_LO__SIMD_ID_MASK                                                               0xC0000000L
-
-
-
-
-
-
-
-
-// addressBlock: didtind
-//DIDT_SQ_CTRL0
-#define DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT                                                                    0x0
-#define DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT                                                                    0x1
-#define DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT                                                                   0x3
-#define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT                                                            0x4
-#define DIDT_SQ_CTRL0__DIDT_STALL_CTRL_EN__SHIFT                                                              0x5
-#define DIDT_SQ_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT                                                             0x6
-#define DIDT_SQ_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT                                                      0x7
-#define DIDT_SQ_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT                                                         0x8
-#define DIDT_SQ_CTRL0__DIDT_AUTO_MPD_EN__SHIFT                                                                0x18
-#define DIDT_SQ_CTRL0__DIDT_STALL_EVENT_EN__SHIFT                                                             0x19
-#define DIDT_SQ_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT                                                  0x1a
-#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT                                                                        0x1b
-#define DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK                                                                      0x00000001L
-#define DIDT_SQ_CTRL0__PHASE_OFFSET_MASK                                                                      0x00000006L
-#define DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK                                                                     0x00000008L
-#define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK                                                              0x00000010L
-#define DIDT_SQ_CTRL0__DIDT_STALL_CTRL_EN_MASK                                                                0x00000020L
-#define DIDT_SQ_CTRL0__DIDT_TUNING_CTRL_EN_MASK                                                               0x00000040L
-#define DIDT_SQ_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK                                                        0x00000080L
-#define DIDT_SQ_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK                                                           0x00FFFF00L
-#define DIDT_SQ_CTRL0__DIDT_AUTO_MPD_EN_MASK                                                                  0x01000000L
-#define DIDT_SQ_CTRL0__DIDT_STALL_EVENT_EN_MASK                                                               0x02000000L
-#define DIDT_SQ_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK                                                    0x04000000L
-#define DIDT_SQ_CTRL0__UNUSED_0_MASK                                                                          0xF8000000L
-//DIDT_SQ_CTRL1
-#define DIDT_SQ_CTRL1__MIN_POWER__SHIFT                                                                       0x0
-#define DIDT_SQ_CTRL1__MAX_POWER__SHIFT                                                                       0x10
-#define DIDT_SQ_CTRL1__MIN_POWER_MASK                                                                         0x0000FFFFL
-#define DIDT_SQ_CTRL1__MAX_POWER_MASK                                                                         0xFFFF0000L
-//DIDT_SQ_CTRL2
-#define DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT                                                                 0x0
-#define DIDT_SQ_CTRL2__UNUSED_0__SHIFT                                                                        0xe
-#define DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                        0x10
-#define DIDT_SQ_CTRL2__UNUSED_1__SHIFT                                                                        0x1a
-#define DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                        0x1b
-#define DIDT_SQ_CTRL2__UNUSED_2__SHIFT                                                                        0x1f
-#define DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK                                                                   0x00003FFFL
-#define DIDT_SQ_CTRL2__UNUSED_0_MASK                                                                          0x0000C000L
-#define DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK                                                          0x03FF0000L
-#define DIDT_SQ_CTRL2__UNUSED_1_MASK                                                                          0x04000000L
-#define DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK                                                          0x78000000L
-#define DIDT_SQ_CTRL2__UNUSED_2_MASK                                                                          0x80000000L
-//DIDT_SQ_STALL_CTRL
-#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT                                                        0x0
-#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT                                                        0x6
-#define DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT                                                 0xc
-#define DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT                                                 0x12
-#define DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT                                                                   0x18
-#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK                                                          0x0000003FL
-#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK                                                          0x00000FC0L
-#define DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK                                                   0x0003F000L
-#define DIDT_SQ_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK                                                   0x00FC0000L
-#define DIDT_SQ_STALL_CTRL__UNUSED_0_MASK                                                                     0xFF000000L
-//DIDT_SQ_TUNING_CTRL
-#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT                                                        0x0
-#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT                                                        0xe
-#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK                                                          0x00003FFFL
-#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK                                                          0x0FFFC000L
-//DIDT_SQ_STALL_AUTO_RELEASE_CTRL
-#define DIDT_SQ_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME__SHIFT                                  0x0
-#define DIDT_SQ_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME_MASK                                    0x00FFFFFFL
-//DIDT_SQ_CTRL3
-#define DIDT_SQ_CTRL3__GC_DIDT_ENABLE__SHIFT                                                                  0x0
-#define DIDT_SQ_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT                                                         0x1
-#define DIDT_SQ_CTRL3__THROTTLE_POLICY__SHIFT                                                                 0x2
-#define DIDT_SQ_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                    0x4
-#define DIDT_SQ_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT                                                         0x9
-#define DIDT_SQ_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT                                                     0xe
-#define DIDT_SQ_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT                                                           0x16
-#define DIDT_SQ_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT                                                           0x17
-#define DIDT_SQ_CTRL3__QUALIFY_STALL_EN__SHIFT                                                                0x18
-#define DIDT_SQ_CTRL3__DIDT_STALL_SEL__SHIFT                                                                  0x19
-#define DIDT_SQ_CTRL3__DIDT_FORCE_STALL__SHIFT                                                                0x1b
-#define DIDT_SQ_CTRL3__DIDT_STALL_DELAY_EN__SHIFT                                                             0x1c
-#define DIDT_SQ_CTRL3__GC_DIDT_ENABLE_MASK                                                                    0x00000001L
-#define DIDT_SQ_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK                                                           0x00000002L
-#define DIDT_SQ_CTRL3__THROTTLE_POLICY_MASK                                                                   0x0000000CL
-#define DIDT_SQ_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK                                                      0x000001F0L
-#define DIDT_SQ_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK                                                           0x00003E00L
-#define DIDT_SQ_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK                                                       0x003FC000L
-#define DIDT_SQ_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK                                                             0x00400000L
-#define DIDT_SQ_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK                                                             0x00800000L
-#define DIDT_SQ_CTRL3__QUALIFY_STALL_EN_MASK                                                                  0x01000000L
-#define DIDT_SQ_CTRL3__DIDT_STALL_SEL_MASK                                                                    0x06000000L
-#define DIDT_SQ_CTRL3__DIDT_FORCE_STALL_MASK                                                                  0x08000000L
-#define DIDT_SQ_CTRL3__DIDT_STALL_DELAY_EN_MASK                                                               0x10000000L
-//DIDT_SQ_STALL_PATTERN_1_2
-#define DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT                                                0x0
-#define DIDT_SQ_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT                                                0x10
-#define DIDT_SQ_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK                                                  0x00007FFFL
-#define DIDT_SQ_STALL_PATTERN_1_2__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_SQ_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK                                                  0x7FFF0000L
-#define DIDT_SQ_STALL_PATTERN_1_2__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_SQ_STALL_PATTERN_3_4
-#define DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT                                                0x0
-#define DIDT_SQ_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT                                                0x10
-#define DIDT_SQ_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK                                                  0x00007FFFL
-#define DIDT_SQ_STALL_PATTERN_3_4__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_SQ_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK                                                  0x7FFF0000L
-#define DIDT_SQ_STALL_PATTERN_3_4__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_SQ_STALL_PATTERN_5_6
-#define DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT                                                0x0
-#define DIDT_SQ_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT                                                0x10
-#define DIDT_SQ_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK                                                  0x00007FFFL
-#define DIDT_SQ_STALL_PATTERN_5_6__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_SQ_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK                                                  0x7FFF0000L
-#define DIDT_SQ_STALL_PATTERN_5_6__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_SQ_STALL_PATTERN_7
-#define DIDT_SQ_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT                                                  0x0
-#define DIDT_SQ_STALL_PATTERN_7__UNUSED_0__SHIFT                                                              0xf
-#define DIDT_SQ_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK                                                    0x00007FFFL
-#define DIDT_SQ_STALL_PATTERN_7__UNUSED_0_MASK                                                                0xFFFF8000L
-//DIDT_SQ_WEIGHT0_3
-#define DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT                                                                     0x0
-#define DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT                                                                     0x8
-#define DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT                                                                     0x10
-#define DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT                                                                     0x18
-#define DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK                                                                       0x000000FFL
-#define DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK                                                                       0x0000FF00L
-#define DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK                                                                       0x00FF0000L
-#define DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK                                                                       0xFF000000L
-//DIDT_SQ_WEIGHT4_7
-#define DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT                                                                     0x0
-#define DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT                                                                     0x8
-#define DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT                                                                     0x10
-#define DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT                                                                     0x18
-#define DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK                                                                       0x000000FFL
-#define DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK                                                                       0x0000FF00L
-#define DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK                                                                       0x00FF0000L
-#define DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK                                                                       0xFF000000L
-//DIDT_SQ_WEIGHT8_11
-#define DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT                                                                    0x0
-#define DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT                                                                    0x8
-#define DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT                                                                   0x10
-#define DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT                                                                   0x18
-#define DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK                                                                      0x000000FFL
-#define DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK                                                                      0x0000FF00L
-#define DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK                                                                     0x00FF0000L
-#define DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK                                                                     0xFF000000L
-//DIDT_SQ_EDC_CTRL
-#define DIDT_SQ_EDC_CTRL__EDC_EN__SHIFT                                                                       0x0
-#define DIDT_SQ_EDC_CTRL__EDC_SW_RST__SHIFT                                                                   0x1
-#define DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                          0x2
-#define DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                              0x3
-#define DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                  0x4
-#define DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT                                                   0x9
-#define DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                     0x11
-#define DIDT_SQ_EDC_CTRL__GC_EDC_EN__SHIFT                                                                    0x12
-#define DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT                                                          0x13
-#define DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT                                                         0x15
-#define DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT                                                         0x16
-#define DIDT_SQ_EDC_CTRL__UNUSED_0__SHIFT                                                                     0x17
-#define DIDT_SQ_EDC_CTRL__EDC_EN_MASK                                                                         0x00000001L
-#define DIDT_SQ_EDC_CTRL__EDC_SW_RST_MASK                                                                     0x00000002L
-#define DIDT_SQ_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                            0x00000004L
-#define DIDT_SQ_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                0x00000008L
-#define DIDT_SQ_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                    0x000001F0L
-#define DIDT_SQ_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK                                                     0x0001FE00L
-#define DIDT_SQ_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                       0x00020000L
-#define DIDT_SQ_EDC_CTRL__GC_EDC_EN_MASK                                                                      0x00040000L
-#define DIDT_SQ_EDC_CTRL__GC_EDC_STALL_POLICY_MASK                                                            0x00180000L
-#define DIDT_SQ_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK                                                           0x00200000L
-#define DIDT_SQ_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK                                                           0x00400000L
-#define DIDT_SQ_EDC_CTRL__UNUSED_0_MASK                                                                       0xFF800000L
-//DIDT_SQ_EDC_THRESHOLD
-#define DIDT_SQ_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                           0x0
-#define DIDT_SQ_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                             0xFFFFFFFFL
-//DIDT_SQ_EDC_STALL_PATTERN_1_2
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                             0x0
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                             0x10
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                               0x00007FFFL
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                               0x7FFF0000L
-#define DIDT_SQ_EDC_STALL_PATTERN_1_2__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_SQ_EDC_STALL_PATTERN_3_4
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                             0x0
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                             0x10
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                               0x00007FFFL
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                               0x7FFF0000L
-#define DIDT_SQ_EDC_STALL_PATTERN_3_4__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_SQ_EDC_STALL_PATTERN_5_6
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                             0x0
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                             0x10
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                               0x00007FFFL
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                               0x7FFF0000L
-#define DIDT_SQ_EDC_STALL_PATTERN_5_6__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_SQ_EDC_STALL_PATTERN_7
-#define DIDT_SQ_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                               0x0
-#define DIDT_SQ_EDC_STALL_PATTERN_7__UNUSED_0__SHIFT                                                          0xf
-#define DIDT_SQ_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                 0x00007FFFL
-#define DIDT_SQ_EDC_STALL_PATTERN_7__UNUSED_0_MASK                                                            0xFFFF8000L
-//DIDT_SQ_EDC_STATUS
-#define DIDT_SQ_EDC_STATUS__EDC_FSM_STATE__SHIFT                                                              0x0
-#define DIDT_SQ_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                         0x1
-#define DIDT_SQ_EDC_STATUS__EDC_FSM_STATE_MASK                                                                0x00000001L
-#define DIDT_SQ_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                           0x0000000EL
-//DIDT_SQ_EDC_STALL_DELAY_1
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ0__SHIFT                                                 0x0
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ1__SHIFT                                                 0x6
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ2__SHIFT                                                 0xc
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ3__SHIFT                                                 0x12
-#define DIDT_SQ_EDC_STALL_DELAY_1__UNUSED__SHIFT                                                              0x18
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ0_MASK                                                   0x0000003FL
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ1_MASK                                                   0x00000FC0L
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ2_MASK                                                   0x0003F000L
-#define DIDT_SQ_EDC_STALL_DELAY_1__EDC_STALL_DELAY_SQ3_MASK                                                   0x00FC0000L
-#define DIDT_SQ_EDC_STALL_DELAY_1__UNUSED_MASK                                                                0xFF000000L
-//DIDT_SQ_EDC_STALL_DELAY_2
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ4__SHIFT                                                 0x0
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ5__SHIFT                                                 0x6
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ6__SHIFT                                                 0xc
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ7__SHIFT                                                 0x12
-#define DIDT_SQ_EDC_STALL_DELAY_2__UNUSED__SHIFT                                                              0x18
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ4_MASK                                                   0x0000003FL
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ5_MASK                                                   0x00000FC0L
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ6_MASK                                                   0x0003F000L
-#define DIDT_SQ_EDC_STALL_DELAY_2__EDC_STALL_DELAY_SQ7_MASK                                                   0x00FC0000L
-#define DIDT_SQ_EDC_STALL_DELAY_2__UNUSED_MASK                                                                0xFF000000L
-//DIDT_SQ_EDC_STALL_DELAY_3
-#define DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ8__SHIFT                                                 0x0
-#define DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ9__SHIFT                                                 0x6
-#define DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ10__SHIFT                                                0xc
-#define DIDT_SQ_EDC_STALL_DELAY_3__UNUSED__SHIFT                                                              0x12
-#define DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ8_MASK                                                   0x0000003FL
-#define DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ9_MASK                                                   0x00000FC0L
-#define DIDT_SQ_EDC_STALL_DELAY_3__EDC_STALL_DELAY_SQ10_MASK                                                  0x0003F000L
-#define DIDT_SQ_EDC_STALL_DELAY_3__UNUSED_MASK                                                                0xFFFC0000L
-//DIDT_SQ_EDC_OVERFLOW
-#define DIDT_SQ_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                         0x0
-#define DIDT_SQ_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                      0x1
-#define DIDT_SQ_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                           0x00000001L
-#define DIDT_SQ_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                        0x0001FFFEL
-//DIDT_SQ_EDC_ROLLING_POWER_DELTA
-#define DIDT_SQ_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                       0x0
-#define DIDT_SQ_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                         0xFFFFFFFFL
-//DIDT_DB_CTRL0
-#define DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT                                                                    0x0
-#define DIDT_DB_CTRL0__PHASE_OFFSET__SHIFT                                                                    0x1
-#define DIDT_DB_CTRL0__DIDT_CTRL_RST__SHIFT                                                                   0x3
-#define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT                                                            0x4
-#define DIDT_DB_CTRL0__DIDT_STALL_CTRL_EN__SHIFT                                                              0x5
-#define DIDT_DB_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT                                                             0x6
-#define DIDT_DB_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT                                                      0x7
-#define DIDT_DB_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT                                                         0x8
-#define DIDT_DB_CTRL0__DIDT_AUTO_MPD_EN__SHIFT                                                                0x18
-#define DIDT_DB_CTRL0__DIDT_STALL_EVENT_EN__SHIFT                                                             0x19
-#define DIDT_DB_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT                                                  0x1a
-#define DIDT_DB_CTRL0__UNUSED_0__SHIFT                                                                        0x1b
-#define DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK                                                                      0x00000001L
-#define DIDT_DB_CTRL0__PHASE_OFFSET_MASK                                                                      0x00000006L
-#define DIDT_DB_CTRL0__DIDT_CTRL_RST_MASK                                                                     0x00000008L
-#define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK                                                              0x00000010L
-#define DIDT_DB_CTRL0__DIDT_STALL_CTRL_EN_MASK                                                                0x00000020L
-#define DIDT_DB_CTRL0__DIDT_TUNING_CTRL_EN_MASK                                                               0x00000040L
-#define DIDT_DB_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK                                                        0x00000080L
-#define DIDT_DB_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK                                                           0x00FFFF00L
-#define DIDT_DB_CTRL0__DIDT_AUTO_MPD_EN_MASK                                                                  0x01000000L
-#define DIDT_DB_CTRL0__DIDT_STALL_EVENT_EN_MASK                                                               0x02000000L
-#define DIDT_DB_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK                                                    0x04000000L
-#define DIDT_DB_CTRL0__UNUSED_0_MASK                                                                          0xF8000000L
-//DIDT_DB_CTRL1
-#define DIDT_DB_CTRL1__MIN_POWER__SHIFT                                                                       0x0
-#define DIDT_DB_CTRL1__MAX_POWER__SHIFT                                                                       0x10
-#define DIDT_DB_CTRL1__MIN_POWER_MASK                                                                         0x0000FFFFL
-#define DIDT_DB_CTRL1__MAX_POWER_MASK                                                                         0xFFFF0000L
-//DIDT_DB_CTRL2
-#define DIDT_DB_CTRL2__MAX_POWER_DELTA__SHIFT                                                                 0x0
-#define DIDT_DB_CTRL2__UNUSED_0__SHIFT                                                                        0xe
-#define DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                        0x10
-#define DIDT_DB_CTRL2__UNUSED_1__SHIFT                                                                        0x1a
-#define DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                        0x1b
-#define DIDT_DB_CTRL2__UNUSED_2__SHIFT                                                                        0x1f
-#define DIDT_DB_CTRL2__MAX_POWER_DELTA_MASK                                                                   0x00003FFFL
-#define DIDT_DB_CTRL2__UNUSED_0_MASK                                                                          0x0000C000L
-#define DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK                                                          0x03FF0000L
-#define DIDT_DB_CTRL2__UNUSED_1_MASK                                                                          0x04000000L
-#define DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK                                                          0x78000000L
-#define DIDT_DB_CTRL2__UNUSED_2_MASK                                                                          0x80000000L
-//DIDT_DB_STALL_CTRL
-#define DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT                                                        0x0
-#define DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT                                                        0x6
-#define DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT                                                 0xc
-#define DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT                                                 0x12
-#define DIDT_DB_STALL_CTRL__UNUSED_0__SHIFT                                                                   0x18
-#define DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK                                                          0x0000003FL
-#define DIDT_DB_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK                                                          0x00000FC0L
-#define DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK                                                   0x0003F000L
-#define DIDT_DB_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK                                                   0x00FC0000L
-#define DIDT_DB_STALL_CTRL__UNUSED_0_MASK                                                                     0xFF000000L
-//DIDT_DB_TUNING_CTRL
-#define DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT                                                        0x0
-#define DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT                                                        0xe
-#define DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK                                                          0x00003FFFL
-#define DIDT_DB_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK                                                          0x0FFFC000L
-//DIDT_DB_STALL_AUTO_RELEASE_CTRL
-#define DIDT_DB_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME__SHIFT                                  0x0
-#define DIDT_DB_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME_MASK                                    0x00FFFFFFL
-//DIDT_DB_CTRL3
-#define DIDT_DB_CTRL3__GC_DIDT_ENABLE__SHIFT                                                                  0x0
-#define DIDT_DB_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT                                                         0x1
-#define DIDT_DB_CTRL3__THROTTLE_POLICY__SHIFT                                                                 0x2
-#define DIDT_DB_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                    0x4
-#define DIDT_DB_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT                                                         0x9
-#define DIDT_DB_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT                                                     0xe
-#define DIDT_DB_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT                                                           0x16
-#define DIDT_DB_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT                                                           0x17
-#define DIDT_DB_CTRL3__QUALIFY_STALL_EN__SHIFT                                                                0x18
-#define DIDT_DB_CTRL3__DIDT_STALL_SEL__SHIFT                                                                  0x19
-#define DIDT_DB_CTRL3__DIDT_FORCE_STALL__SHIFT                                                                0x1b
-#define DIDT_DB_CTRL3__DIDT_STALL_DELAY_EN__SHIFT                                                             0x1c
-#define DIDT_DB_CTRL3__GC_DIDT_ENABLE_MASK                                                                    0x00000001L
-#define DIDT_DB_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK                                                           0x00000002L
-#define DIDT_DB_CTRL3__THROTTLE_POLICY_MASK                                                                   0x0000000CL
-#define DIDT_DB_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK                                                      0x000001F0L
-#define DIDT_DB_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK                                                           0x00003E00L
-#define DIDT_DB_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK                                                       0x003FC000L
-#define DIDT_DB_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK                                                             0x00400000L
-#define DIDT_DB_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK                                                             0x00800000L
-#define DIDT_DB_CTRL3__QUALIFY_STALL_EN_MASK                                                                  0x01000000L
-#define DIDT_DB_CTRL3__DIDT_STALL_SEL_MASK                                                                    0x06000000L
-#define DIDT_DB_CTRL3__DIDT_FORCE_STALL_MASK                                                                  0x08000000L
-#define DIDT_DB_CTRL3__DIDT_STALL_DELAY_EN_MASK                                                               0x10000000L
-//DIDT_DB_STALL_PATTERN_1_2
-#define DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT                                                0x0
-#define DIDT_DB_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT                                                0x10
-#define DIDT_DB_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK                                                  0x00007FFFL
-#define DIDT_DB_STALL_PATTERN_1_2__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_DB_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK                                                  0x7FFF0000L
-#define DIDT_DB_STALL_PATTERN_1_2__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_DB_STALL_PATTERN_3_4
-#define DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT                                                0x0
-#define DIDT_DB_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT                                                0x10
-#define DIDT_DB_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK                                                  0x00007FFFL
-#define DIDT_DB_STALL_PATTERN_3_4__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_DB_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK                                                  0x7FFF0000L
-#define DIDT_DB_STALL_PATTERN_3_4__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_DB_STALL_PATTERN_5_6
-#define DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT                                                0x0
-#define DIDT_DB_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT                                                0x10
-#define DIDT_DB_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK                                                  0x00007FFFL
-#define DIDT_DB_STALL_PATTERN_5_6__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_DB_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK                                                  0x7FFF0000L
-#define DIDT_DB_STALL_PATTERN_5_6__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_DB_STALL_PATTERN_7
-#define DIDT_DB_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT                                                  0x0
-#define DIDT_DB_STALL_PATTERN_7__UNUSED_0__SHIFT                                                              0xf
-#define DIDT_DB_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK                                                    0x00007FFFL
-#define DIDT_DB_STALL_PATTERN_7__UNUSED_0_MASK                                                                0xFFFF8000L
-//DIDT_DB_WEIGHT0_3
-#define DIDT_DB_WEIGHT0_3__WEIGHT0__SHIFT                                                                     0x0
-#define DIDT_DB_WEIGHT0_3__WEIGHT1__SHIFT                                                                     0x8
-#define DIDT_DB_WEIGHT0_3__WEIGHT2__SHIFT                                                                     0x10
-#define DIDT_DB_WEIGHT0_3__WEIGHT3__SHIFT                                                                     0x18
-#define DIDT_DB_WEIGHT0_3__WEIGHT0_MASK                                                                       0x000000FFL
-#define DIDT_DB_WEIGHT0_3__WEIGHT1_MASK                                                                       0x0000FF00L
-#define DIDT_DB_WEIGHT0_3__WEIGHT2_MASK                                                                       0x00FF0000L
-#define DIDT_DB_WEIGHT0_3__WEIGHT3_MASK                                                                       0xFF000000L
-//DIDT_DB_WEIGHT4_7
-#define DIDT_DB_WEIGHT4_7__WEIGHT4__SHIFT                                                                     0x0
-#define DIDT_DB_WEIGHT4_7__WEIGHT5__SHIFT                                                                     0x8
-#define DIDT_DB_WEIGHT4_7__WEIGHT6__SHIFT                                                                     0x10
-#define DIDT_DB_WEIGHT4_7__WEIGHT7__SHIFT                                                                     0x18
-#define DIDT_DB_WEIGHT4_7__WEIGHT4_MASK                                                                       0x000000FFL
-#define DIDT_DB_WEIGHT4_7__WEIGHT5_MASK                                                                       0x0000FF00L
-#define DIDT_DB_WEIGHT4_7__WEIGHT6_MASK                                                                       0x00FF0000L
-#define DIDT_DB_WEIGHT4_7__WEIGHT7_MASK                                                                       0xFF000000L
-//DIDT_DB_WEIGHT8_11
-#define DIDT_DB_WEIGHT8_11__WEIGHT8__SHIFT                                                                    0x0
-#define DIDT_DB_WEIGHT8_11__WEIGHT9__SHIFT                                                                    0x8
-#define DIDT_DB_WEIGHT8_11__WEIGHT10__SHIFT                                                                   0x10
-#define DIDT_DB_WEIGHT8_11__WEIGHT11__SHIFT                                                                   0x18
-#define DIDT_DB_WEIGHT8_11__WEIGHT8_MASK                                                                      0x000000FFL
-#define DIDT_DB_WEIGHT8_11__WEIGHT9_MASK                                                                      0x0000FF00L
-#define DIDT_DB_WEIGHT8_11__WEIGHT10_MASK                                                                     0x00FF0000L
-#define DIDT_DB_WEIGHT8_11__WEIGHT11_MASK                                                                     0xFF000000L
-//DIDT_DB_EDC_CTRL
-#define DIDT_DB_EDC_CTRL__EDC_EN__SHIFT                                                                       0x0
-#define DIDT_DB_EDC_CTRL__EDC_SW_RST__SHIFT                                                                   0x1
-#define DIDT_DB_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                          0x2
-#define DIDT_DB_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                              0x3
-#define DIDT_DB_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                  0x4
-#define DIDT_DB_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT                                                   0x9
-#define DIDT_DB_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                     0x11
-#define DIDT_DB_EDC_CTRL__GC_EDC_EN__SHIFT                                                                    0x12
-#define DIDT_DB_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT                                                          0x13
-#define DIDT_DB_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT                                                         0x15
-#define DIDT_DB_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT                                                         0x16
-#define DIDT_DB_EDC_CTRL__UNUSED_0__SHIFT                                                                     0x17
-#define DIDT_DB_EDC_CTRL__EDC_EN_MASK                                                                         0x00000001L
-#define DIDT_DB_EDC_CTRL__EDC_SW_RST_MASK                                                                     0x00000002L
-#define DIDT_DB_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                            0x00000004L
-#define DIDT_DB_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                0x00000008L
-#define DIDT_DB_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                    0x000001F0L
-#define DIDT_DB_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK                                                     0x0001FE00L
-#define DIDT_DB_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                       0x00020000L
-#define DIDT_DB_EDC_CTRL__GC_EDC_EN_MASK                                                                      0x00040000L
-#define DIDT_DB_EDC_CTRL__GC_EDC_STALL_POLICY_MASK                                                            0x00180000L
-#define DIDT_DB_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK                                                           0x00200000L
-#define DIDT_DB_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK                                                           0x00400000L
-#define DIDT_DB_EDC_CTRL__UNUSED_0_MASK                                                                       0xFF800000L
-//DIDT_DB_EDC_THRESHOLD
-#define DIDT_DB_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                           0x0
-#define DIDT_DB_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                             0xFFFFFFFFL
-//DIDT_DB_EDC_STALL_PATTERN_1_2
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                             0x0
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                             0x10
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                               0x00007FFFL
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                               0x7FFF0000L
-#define DIDT_DB_EDC_STALL_PATTERN_1_2__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_DB_EDC_STALL_PATTERN_3_4
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                             0x0
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                             0x10
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                               0x00007FFFL
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                               0x7FFF0000L
-#define DIDT_DB_EDC_STALL_PATTERN_3_4__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_DB_EDC_STALL_PATTERN_5_6
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                             0x0
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                             0x10
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                               0x00007FFFL
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                               0x7FFF0000L
-#define DIDT_DB_EDC_STALL_PATTERN_5_6__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_DB_EDC_STALL_PATTERN_7
-#define DIDT_DB_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                               0x0
-#define DIDT_DB_EDC_STALL_PATTERN_7__UNUSED_0__SHIFT                                                          0xf
-#define DIDT_DB_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                 0x00007FFFL
-#define DIDT_DB_EDC_STALL_PATTERN_7__UNUSED_0_MASK                                                            0xFFFF8000L
-//DIDT_DB_EDC_STATUS
-#define DIDT_DB_EDC_STATUS__EDC_FSM_STATE__SHIFT                                                              0x0
-#define DIDT_DB_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                         0x1
-#define DIDT_DB_EDC_STATUS__EDC_FSM_STATE_MASK                                                                0x00000001L
-#define DIDT_DB_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                           0x0000000EL
-//DIDT_DB_EDC_STALL_DELAY_1
-#define DIDT_DB_EDC_STALL_DELAY_1__EDC_STALL_DELAY_DB0__SHIFT                                                 0x0
-#define DIDT_DB_EDC_STALL_DELAY_1__EDC_STALL_DELAY_DB1__SHIFT                                                 0x3
-#define DIDT_DB_EDC_STALL_DELAY_1__UNUSED__SHIFT                                                              0x6
-#define DIDT_DB_EDC_STALL_DELAY_1__EDC_STALL_DELAY_DB0_MASK                                                   0x00000007L
-#define DIDT_DB_EDC_STALL_DELAY_1__EDC_STALL_DELAY_DB1_MASK                                                   0x00000038L
-#define DIDT_DB_EDC_STALL_DELAY_1__UNUSED_MASK                                                                0xFFFFFFC0L
-//DIDT_DB_EDC_OVERFLOW
-#define DIDT_DB_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                         0x0
-#define DIDT_DB_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                      0x1
-#define DIDT_DB_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                           0x00000001L
-#define DIDT_DB_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                        0x0001FFFEL
-//DIDT_DB_EDC_ROLLING_POWER_DELTA
-#define DIDT_DB_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                       0x0
-#define DIDT_DB_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                         0xFFFFFFFFL
-//DIDT_TD_CTRL0
-#define DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT                                                                    0x0
-#define DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT                                                                    0x1
-#define DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT                                                                   0x3
-#define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT                                                            0x4
-#define DIDT_TD_CTRL0__DIDT_STALL_CTRL_EN__SHIFT                                                              0x5
-#define DIDT_TD_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT                                                             0x6
-#define DIDT_TD_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT                                                      0x7
-#define DIDT_TD_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT                                                         0x8
-#define DIDT_TD_CTRL0__DIDT_AUTO_MPD_EN__SHIFT                                                                0x18
-#define DIDT_TD_CTRL0__DIDT_STALL_EVENT_EN__SHIFT                                                             0x19
-#define DIDT_TD_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT                                                  0x1a
-#define DIDT_TD_CTRL0__UNUSED_0__SHIFT                                                                        0x1b
-#define DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK                                                                      0x00000001L
-#define DIDT_TD_CTRL0__PHASE_OFFSET_MASK                                                                      0x00000006L
-#define DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK                                                                     0x00000008L
-#define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK                                                              0x00000010L
-#define DIDT_TD_CTRL0__DIDT_STALL_CTRL_EN_MASK                                                                0x00000020L
-#define DIDT_TD_CTRL0__DIDT_TUNING_CTRL_EN_MASK                                                               0x00000040L
-#define DIDT_TD_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK                                                        0x00000080L
-#define DIDT_TD_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK                                                           0x00FFFF00L
-#define DIDT_TD_CTRL0__DIDT_AUTO_MPD_EN_MASK                                                                  0x01000000L
-#define DIDT_TD_CTRL0__DIDT_STALL_EVENT_EN_MASK                                                               0x02000000L
-#define DIDT_TD_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK                                                    0x04000000L
-#define DIDT_TD_CTRL0__UNUSED_0_MASK                                                                          0xF8000000L
-//DIDT_TD_CTRL1
-#define DIDT_TD_CTRL1__MIN_POWER__SHIFT                                                                       0x0
-#define DIDT_TD_CTRL1__MAX_POWER__SHIFT                                                                       0x10
-#define DIDT_TD_CTRL1__MIN_POWER_MASK                                                                         0x0000FFFFL
-#define DIDT_TD_CTRL1__MAX_POWER_MASK                                                                         0xFFFF0000L
-//DIDT_TD_CTRL2
-#define DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT                                                                 0x0
-#define DIDT_TD_CTRL2__UNUSED_0__SHIFT                                                                        0xe
-#define DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                        0x10
-#define DIDT_TD_CTRL2__UNUSED_1__SHIFT                                                                        0x1a
-#define DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                        0x1b
-#define DIDT_TD_CTRL2__UNUSED_2__SHIFT                                                                        0x1f
-#define DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK                                                                   0x00003FFFL
-#define DIDT_TD_CTRL2__UNUSED_0_MASK                                                                          0x0000C000L
-#define DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK                                                          0x03FF0000L
-#define DIDT_TD_CTRL2__UNUSED_1_MASK                                                                          0x04000000L
-#define DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK                                                          0x78000000L
-#define DIDT_TD_CTRL2__UNUSED_2_MASK                                                                          0x80000000L
-//DIDT_TD_STALL_CTRL
-#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT                                                        0x0
-#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT                                                        0x6
-#define DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT                                                 0xc
-#define DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT                                                 0x12
-#define DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT                                                                   0x18
-#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK                                                          0x0000003FL
-#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK                                                          0x00000FC0L
-#define DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK                                                   0x0003F000L
-#define DIDT_TD_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK                                                   0x00FC0000L
-#define DIDT_TD_STALL_CTRL__UNUSED_0_MASK                                                                     0xFF000000L
-//DIDT_TD_TUNING_CTRL
-#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT                                                        0x0
-#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT                                                        0xe
-#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK                                                          0x00003FFFL
-#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK                                                          0x0FFFC000L
-//DIDT_TD_STALL_AUTO_RELEASE_CTRL
-#define DIDT_TD_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME__SHIFT                                  0x0
-#define DIDT_TD_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME_MASK                                    0x00FFFFFFL
-//DIDT_TD_CTRL3
-#define DIDT_TD_CTRL3__GC_DIDT_ENABLE__SHIFT                                                                  0x0
-#define DIDT_TD_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT                                                         0x1
-#define DIDT_TD_CTRL3__THROTTLE_POLICY__SHIFT                                                                 0x2
-#define DIDT_TD_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                    0x4
-#define DIDT_TD_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT                                                         0x9
-#define DIDT_TD_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT                                                     0xe
-#define DIDT_TD_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT                                                           0x16
-#define DIDT_TD_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT                                                           0x17
-#define DIDT_TD_CTRL3__QUALIFY_STALL_EN__SHIFT                                                                0x18
-#define DIDT_TD_CTRL3__DIDT_STALL_SEL__SHIFT                                                                  0x19
-#define DIDT_TD_CTRL3__DIDT_FORCE_STALL__SHIFT                                                                0x1b
-#define DIDT_TD_CTRL3__DIDT_STALL_DELAY_EN__SHIFT                                                             0x1c
-#define DIDT_TD_CTRL3__GC_DIDT_ENABLE_MASK                                                                    0x00000001L
-#define DIDT_TD_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK                                                           0x00000002L
-#define DIDT_TD_CTRL3__THROTTLE_POLICY_MASK                                                                   0x0000000CL
-#define DIDT_TD_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK                                                      0x000001F0L
-#define DIDT_TD_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK                                                           0x00003E00L
-#define DIDT_TD_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK                                                       0x003FC000L
-#define DIDT_TD_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK                                                             0x00400000L
-#define DIDT_TD_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK                                                             0x00800000L
-#define DIDT_TD_CTRL3__QUALIFY_STALL_EN_MASK                                                                  0x01000000L
-#define DIDT_TD_CTRL3__DIDT_STALL_SEL_MASK                                                                    0x06000000L
-#define DIDT_TD_CTRL3__DIDT_FORCE_STALL_MASK                                                                  0x08000000L
-#define DIDT_TD_CTRL3__DIDT_STALL_DELAY_EN_MASK                                                               0x10000000L
-//DIDT_TD_STALL_PATTERN_1_2
-#define DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT                                                0x0
-#define DIDT_TD_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT                                                0x10
-#define DIDT_TD_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK                                                  0x00007FFFL
-#define DIDT_TD_STALL_PATTERN_1_2__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_TD_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK                                                  0x7FFF0000L
-#define DIDT_TD_STALL_PATTERN_1_2__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_TD_STALL_PATTERN_3_4
-#define DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT                                                0x0
-#define DIDT_TD_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT                                                0x10
-#define DIDT_TD_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK                                                  0x00007FFFL
-#define DIDT_TD_STALL_PATTERN_3_4__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_TD_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK                                                  0x7FFF0000L
-#define DIDT_TD_STALL_PATTERN_3_4__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_TD_STALL_PATTERN_5_6
-#define DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT                                                0x0
-#define DIDT_TD_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                            0xf
-#define DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT                                                0x10
-#define DIDT_TD_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                            0x1f
-#define DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK                                                  0x00007FFFL
-#define DIDT_TD_STALL_PATTERN_5_6__UNUSED_0_MASK                                                              0x00008000L
-#define DIDT_TD_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK                                                  0x7FFF0000L
-#define DIDT_TD_STALL_PATTERN_5_6__UNUSED_1_MASK                                                              0x80000000L
-//DIDT_TD_STALL_PATTERN_7
-#define DIDT_TD_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT                                                  0x0
-#define DIDT_TD_STALL_PATTERN_7__UNUSED_0__SHIFT                                                              0xf
-#define DIDT_TD_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK                                                    0x00007FFFL
-#define DIDT_TD_STALL_PATTERN_7__UNUSED_0_MASK                                                                0xFFFF8000L
-//DIDT_TD_WEIGHT0_3
-#define DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT                                                                     0x0
-#define DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT                                                                     0x8
-#define DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT                                                                     0x10
-#define DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT                                                                     0x18
-#define DIDT_TD_WEIGHT0_3__WEIGHT0_MASK                                                                       0x000000FFL
-#define DIDT_TD_WEIGHT0_3__WEIGHT1_MASK                                                                       0x0000FF00L
-#define DIDT_TD_WEIGHT0_3__WEIGHT2_MASK                                                                       0x00FF0000L
-#define DIDT_TD_WEIGHT0_3__WEIGHT3_MASK                                                                       0xFF000000L
-//DIDT_TD_WEIGHT4_7
-#define DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT                                                                     0x0
-#define DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT                                                                     0x8
-#define DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT                                                                     0x10
-#define DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT                                                                     0x18
-#define DIDT_TD_WEIGHT4_7__WEIGHT4_MASK                                                                       0x000000FFL
-#define DIDT_TD_WEIGHT4_7__WEIGHT5_MASK                                                                       0x0000FF00L
-#define DIDT_TD_WEIGHT4_7__WEIGHT6_MASK                                                                       0x00FF0000L
-#define DIDT_TD_WEIGHT4_7__WEIGHT7_MASK                                                                       0xFF000000L
-//DIDT_TD_WEIGHT8_11
-#define DIDT_TD_WEIGHT8_11__WEIGHT8__SHIFT                                                                    0x0
-#define DIDT_TD_WEIGHT8_11__WEIGHT9__SHIFT                                                                    0x8
-#define DIDT_TD_WEIGHT8_11__WEIGHT10__SHIFT                                                                   0x10
-#define DIDT_TD_WEIGHT8_11__WEIGHT11__SHIFT                                                                   0x18
-#define DIDT_TD_WEIGHT8_11__WEIGHT8_MASK                                                                      0x000000FFL
-#define DIDT_TD_WEIGHT8_11__WEIGHT9_MASK                                                                      0x0000FF00L
-#define DIDT_TD_WEIGHT8_11__WEIGHT10_MASK                                                                     0x00FF0000L
-#define DIDT_TD_WEIGHT8_11__WEIGHT11_MASK                                                                     0xFF000000L
-//DIDT_TD_EDC_CTRL
-#define DIDT_TD_EDC_CTRL__EDC_EN__SHIFT                                                                       0x0
-#define DIDT_TD_EDC_CTRL__EDC_SW_RST__SHIFT                                                                   0x1
-#define DIDT_TD_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                          0x2
-#define DIDT_TD_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                              0x3
-#define DIDT_TD_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                  0x4
-#define DIDT_TD_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT                                                   0x9
-#define DIDT_TD_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                     0x11
-#define DIDT_TD_EDC_CTRL__GC_EDC_EN__SHIFT                                                                    0x12
-#define DIDT_TD_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT                                                          0x13
-#define DIDT_TD_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT                                                         0x15
-#define DIDT_TD_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT                                                         0x16
-#define DIDT_TD_EDC_CTRL__UNUSED_0__SHIFT                                                                     0x17
-#define DIDT_TD_EDC_CTRL__EDC_EN_MASK                                                                         0x00000001L
-#define DIDT_TD_EDC_CTRL__EDC_SW_RST_MASK                                                                     0x00000002L
-#define DIDT_TD_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                            0x00000004L
-#define DIDT_TD_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                0x00000008L
-#define DIDT_TD_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                    0x000001F0L
-#define DIDT_TD_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK                                                     0x0001FE00L
-#define DIDT_TD_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                       0x00020000L
-#define DIDT_TD_EDC_CTRL__GC_EDC_EN_MASK                                                                      0x00040000L
-#define DIDT_TD_EDC_CTRL__GC_EDC_STALL_POLICY_MASK                                                            0x00180000L
-#define DIDT_TD_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK                                                           0x00200000L
-#define DIDT_TD_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK                                                           0x00400000L
-#define DIDT_TD_EDC_CTRL__UNUSED_0_MASK                                                                       0xFF800000L
-//DIDT_TD_EDC_THRESHOLD
-#define DIDT_TD_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                           0x0
-#define DIDT_TD_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                             0xFFFFFFFFL
-//DIDT_TD_EDC_STALL_PATTERN_1_2
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                             0x0
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                             0x10
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                               0x00007FFFL
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                               0x7FFF0000L
-#define DIDT_TD_EDC_STALL_PATTERN_1_2__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_TD_EDC_STALL_PATTERN_3_4
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                             0x0
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                             0x10
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                               0x00007FFFL
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                               0x7FFF0000L
-#define DIDT_TD_EDC_STALL_PATTERN_3_4__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_TD_EDC_STALL_PATTERN_5_6
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                             0x0
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                        0xf
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                             0x10
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                        0x1f
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                               0x00007FFFL
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__UNUSED_0_MASK                                                          0x00008000L
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                               0x7FFF0000L
-#define DIDT_TD_EDC_STALL_PATTERN_5_6__UNUSED_1_MASK                                                          0x80000000L
-//DIDT_TD_EDC_STALL_PATTERN_7
-#define DIDT_TD_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                               0x0
-#define DIDT_TD_EDC_STALL_PATTERN_7__UNUSED_0__SHIFT                                                          0xf
-#define DIDT_TD_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                 0x00007FFFL
-#define DIDT_TD_EDC_STALL_PATTERN_7__UNUSED_0_MASK                                                            0xFFFF8000L
-//DIDT_TD_EDC_STATUS
-#define DIDT_TD_EDC_STATUS__EDC_FSM_STATE__SHIFT                                                              0x0
-#define DIDT_TD_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                         0x1
-#define DIDT_TD_EDC_STATUS__EDC_FSM_STATE_MASK                                                                0x00000001L
-#define DIDT_TD_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                           0x0000000EL
-//DIDT_TD_EDC_STALL_DELAY_1
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD0__SHIFT                                                 0x0
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD1__SHIFT                                                 0x6
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD2__SHIFT                                                 0xc
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD3__SHIFT                                                 0x12
-#define DIDT_TD_EDC_STALL_DELAY_1__UNUSED__SHIFT                                                              0x18
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD0_MASK                                                   0x0000003FL
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD1_MASK                                                   0x00000FC0L
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD2_MASK                                                   0x0003F000L
-#define DIDT_TD_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TD3_MASK                                                   0x00FC0000L
-#define DIDT_TD_EDC_STALL_DELAY_1__UNUSED_MASK                                                                0xFF000000L
-//DIDT_TD_EDC_STALL_DELAY_2
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD4__SHIFT                                                 0x0
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD5__SHIFT                                                 0x6
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD6__SHIFT                                                 0xc
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD7__SHIFT                                                 0x12
-#define DIDT_TD_EDC_STALL_DELAY_2__UNUSED__SHIFT                                                              0x18
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD4_MASK                                                   0x0000003FL
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD5_MASK                                                   0x00000FC0L
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD6_MASK                                                   0x0003F000L
-#define DIDT_TD_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TD7_MASK                                                   0x00FC0000L
-#define DIDT_TD_EDC_STALL_DELAY_2__UNUSED_MASK                                                                0xFF000000L
-//DIDT_TD_EDC_STALL_DELAY_3
-#define DIDT_TD_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TD8__SHIFT                                                 0x0
-#define DIDT_TD_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TD9__SHIFT                                                 0x6
-#define DIDT_TD_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TD10__SHIFT                                                0xc
-#define DIDT_TD_EDC_STALL_DELAY_3__UNUSED__SHIFT                                                              0x12
-#define DIDT_TD_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TD8_MASK                                                   0x0000003FL
-#define DIDT_TD_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TD9_MASK                                                   0x00000FC0L
-#define DIDT_TD_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TD10_MASK                                                  0x0003F000L
-#define DIDT_TD_EDC_STALL_DELAY_3__UNUSED_MASK                                                                0xFFFC0000L
-//DIDT_TD_EDC_OVERFLOW
-#define DIDT_TD_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                         0x0
-#define DIDT_TD_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                      0x1
-#define DIDT_TD_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                           0x00000001L
-#define DIDT_TD_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                        0x0001FFFEL
-//DIDT_TD_EDC_ROLLING_POWER_DELTA
-#define DIDT_TD_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                       0x0
-#define DIDT_TD_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                         0xFFFFFFFFL
-//DIDT_TCP_CTRL0
-#define DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT                                                                   0x0
-#define DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT                                                                   0x1
-#define DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT                                                                  0x3
-#define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT                                                           0x4
-#define DIDT_TCP_CTRL0__DIDT_STALL_CTRL_EN__SHIFT                                                             0x5
-#define DIDT_TCP_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT                                                            0x6
-#define DIDT_TCP_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT                                                     0x7
-#define DIDT_TCP_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT                                                        0x8
-#define DIDT_TCP_CTRL0__DIDT_AUTO_MPD_EN__SHIFT                                                               0x18
-#define DIDT_TCP_CTRL0__DIDT_STALL_EVENT_EN__SHIFT                                                            0x19
-#define DIDT_TCP_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT                                                 0x1a
-#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT                                                                       0x1b
-#define DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK                                                                     0x00000001L
-#define DIDT_TCP_CTRL0__PHASE_OFFSET_MASK                                                                     0x00000006L
-#define DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK                                                                    0x00000008L
-#define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK                                                             0x00000010L
-#define DIDT_TCP_CTRL0__DIDT_STALL_CTRL_EN_MASK                                                               0x00000020L
-#define DIDT_TCP_CTRL0__DIDT_TUNING_CTRL_EN_MASK                                                              0x00000040L
-#define DIDT_TCP_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK                                                       0x00000080L
-#define DIDT_TCP_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK                                                          0x00FFFF00L
-#define DIDT_TCP_CTRL0__DIDT_AUTO_MPD_EN_MASK                                                                 0x01000000L
-#define DIDT_TCP_CTRL0__DIDT_STALL_EVENT_EN_MASK                                                              0x02000000L
-#define DIDT_TCP_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK                                                   0x04000000L
-#define DIDT_TCP_CTRL0__UNUSED_0_MASK                                                                         0xF8000000L
-//DIDT_TCP_CTRL1
-#define DIDT_TCP_CTRL1__MIN_POWER__SHIFT                                                                      0x0
-#define DIDT_TCP_CTRL1__MAX_POWER__SHIFT                                                                      0x10
-#define DIDT_TCP_CTRL1__MIN_POWER_MASK                                                                        0x0000FFFFL
-#define DIDT_TCP_CTRL1__MAX_POWER_MASK                                                                        0xFFFF0000L
-//DIDT_TCP_CTRL2
-#define DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT                                                                0x0
-#define DIDT_TCP_CTRL2__UNUSED_0__SHIFT                                                                       0xe
-#define DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                       0x10
-#define DIDT_TCP_CTRL2__UNUSED_1__SHIFT                                                                       0x1a
-#define DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                       0x1b
-#define DIDT_TCP_CTRL2__UNUSED_2__SHIFT                                                                       0x1f
-#define DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK                                                                  0x00003FFFL
-#define DIDT_TCP_CTRL2__UNUSED_0_MASK                                                                         0x0000C000L
-#define DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK                                                         0x03FF0000L
-#define DIDT_TCP_CTRL2__UNUSED_1_MASK                                                                         0x04000000L
-#define DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK                                                         0x78000000L
-#define DIDT_TCP_CTRL2__UNUSED_2_MASK                                                                         0x80000000L
-//DIDT_TCP_STALL_CTRL
-#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT                                                       0x0
-#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT                                                       0x6
-#define DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT                                                0xc
-#define DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT                                                0x12
-#define DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT                                                                  0x18
-#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK                                                         0x0000003FL
-#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK                                                         0x00000FC0L
-#define DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK                                                  0x0003F000L
-#define DIDT_TCP_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK                                                  0x00FC0000L
-#define DIDT_TCP_STALL_CTRL__UNUSED_0_MASK                                                                    0xFF000000L
-//DIDT_TCP_TUNING_CTRL
-#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT                                                       0x0
-#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT                                                       0xe
-#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK                                                         0x00003FFFL
-#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK                                                         0x0FFFC000L
-//DIDT_TCP_STALL_AUTO_RELEASE_CTRL
-#define DIDT_TCP_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME__SHIFT                                 0x0
-#define DIDT_TCP_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME_MASK                                   0x00FFFFFFL
-//DIDT_TCP_CTRL3
-#define DIDT_TCP_CTRL3__GC_DIDT_ENABLE__SHIFT                                                                 0x0
-#define DIDT_TCP_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT                                                        0x1
-#define DIDT_TCP_CTRL3__THROTTLE_POLICY__SHIFT                                                                0x2
-#define DIDT_TCP_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                   0x4
-#define DIDT_TCP_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT                                                        0x9
-#define DIDT_TCP_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT                                                    0xe
-#define DIDT_TCP_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT                                                          0x16
-#define DIDT_TCP_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT                                                          0x17
-#define DIDT_TCP_CTRL3__QUALIFY_STALL_EN__SHIFT                                                               0x18
-#define DIDT_TCP_CTRL3__DIDT_STALL_SEL__SHIFT                                                                 0x19
-#define DIDT_TCP_CTRL3__DIDT_FORCE_STALL__SHIFT                                                               0x1b
-#define DIDT_TCP_CTRL3__DIDT_STALL_DELAY_EN__SHIFT                                                            0x1c
-#define DIDT_TCP_CTRL3__GC_DIDT_ENABLE_MASK                                                                   0x00000001L
-#define DIDT_TCP_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK                                                          0x00000002L
-#define DIDT_TCP_CTRL3__THROTTLE_POLICY_MASK                                                                  0x0000000CL
-#define DIDT_TCP_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK                                                     0x000001F0L
-#define DIDT_TCP_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK                                                          0x00003E00L
-#define DIDT_TCP_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK                                                      0x003FC000L
-#define DIDT_TCP_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK                                                            0x00400000L
-#define DIDT_TCP_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK                                                            0x00800000L
-#define DIDT_TCP_CTRL3__QUALIFY_STALL_EN_MASK                                                                 0x01000000L
-#define DIDT_TCP_CTRL3__DIDT_STALL_SEL_MASK                                                                   0x06000000L
-#define DIDT_TCP_CTRL3__DIDT_FORCE_STALL_MASK                                                                 0x08000000L
-#define DIDT_TCP_CTRL3__DIDT_STALL_DELAY_EN_MASK                                                              0x10000000L
-//DIDT_TCP_STALL_PATTERN_1_2
-#define DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT                                               0x0
-#define DIDT_TCP_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                           0xf
-#define DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT                                               0x10
-#define DIDT_TCP_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                           0x1f
-#define DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK                                                 0x00007FFFL
-#define DIDT_TCP_STALL_PATTERN_1_2__UNUSED_0_MASK                                                             0x00008000L
-#define DIDT_TCP_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK                                                 0x7FFF0000L
-#define DIDT_TCP_STALL_PATTERN_1_2__UNUSED_1_MASK                                                             0x80000000L
-//DIDT_TCP_STALL_PATTERN_3_4
-#define DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT                                               0x0
-#define DIDT_TCP_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                           0xf
-#define DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT                                               0x10
-#define DIDT_TCP_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                           0x1f
-#define DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK                                                 0x00007FFFL
-#define DIDT_TCP_STALL_PATTERN_3_4__UNUSED_0_MASK                                                             0x00008000L
-#define DIDT_TCP_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK                                                 0x7FFF0000L
-#define DIDT_TCP_STALL_PATTERN_3_4__UNUSED_1_MASK                                                             0x80000000L
-//DIDT_TCP_STALL_PATTERN_5_6
-#define DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT                                               0x0
-#define DIDT_TCP_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                           0xf
-#define DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT                                               0x10
-#define DIDT_TCP_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                           0x1f
-#define DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK                                                 0x00007FFFL
-#define DIDT_TCP_STALL_PATTERN_5_6__UNUSED_0_MASK                                                             0x00008000L
-#define DIDT_TCP_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK                                                 0x7FFF0000L
-#define DIDT_TCP_STALL_PATTERN_5_6__UNUSED_1_MASK                                                             0x80000000L
-//DIDT_TCP_STALL_PATTERN_7
-#define DIDT_TCP_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT                                                 0x0
-#define DIDT_TCP_STALL_PATTERN_7__UNUSED_0__SHIFT                                                             0xf
-#define DIDT_TCP_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK                                                   0x00007FFFL
-#define DIDT_TCP_STALL_PATTERN_7__UNUSED_0_MASK                                                               0xFFFF8000L
-//DIDT_TCP_WEIGHT0_3
-#define DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT                                                                    0x0
-#define DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT                                                                    0x8
-#define DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT                                                                    0x10
-#define DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT                                                                    0x18
-#define DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK                                                                      0x000000FFL
-#define DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK                                                                      0x0000FF00L
-#define DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK                                                                      0x00FF0000L
-#define DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK                                                                      0xFF000000L
-//DIDT_TCP_WEIGHT4_7
-#define DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT                                                                    0x0
-#define DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT                                                                    0x8
-#define DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT                                                                    0x10
-#define DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT                                                                    0x18
-#define DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK                                                                      0x000000FFL
-#define DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK                                                                      0x0000FF00L
-#define DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK                                                                      0x00FF0000L
-#define DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK                                                                      0xFF000000L
-//DIDT_TCP_WEIGHT8_11
-#define DIDT_TCP_WEIGHT8_11__WEIGHT8__SHIFT                                                                   0x0
-#define DIDT_TCP_WEIGHT8_11__WEIGHT9__SHIFT                                                                   0x8
-#define DIDT_TCP_WEIGHT8_11__WEIGHT10__SHIFT                                                                  0x10
-#define DIDT_TCP_WEIGHT8_11__WEIGHT11__SHIFT                                                                  0x18
-#define DIDT_TCP_WEIGHT8_11__WEIGHT8_MASK                                                                     0x000000FFL
-#define DIDT_TCP_WEIGHT8_11__WEIGHT9_MASK                                                                     0x0000FF00L
-#define DIDT_TCP_WEIGHT8_11__WEIGHT10_MASK                                                                    0x00FF0000L
-#define DIDT_TCP_WEIGHT8_11__WEIGHT11_MASK                                                                    0xFF000000L
-//DIDT_TCP_EDC_CTRL
-#define DIDT_TCP_EDC_CTRL__EDC_EN__SHIFT                                                                      0x0
-#define DIDT_TCP_EDC_CTRL__EDC_SW_RST__SHIFT                                                                  0x1
-#define DIDT_TCP_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                         0x2
-#define DIDT_TCP_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                             0x3
-#define DIDT_TCP_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                 0x4
-#define DIDT_TCP_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT                                                  0x9
-#define DIDT_TCP_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                    0x11
-#define DIDT_TCP_EDC_CTRL__GC_EDC_EN__SHIFT                                                                   0x12
-#define DIDT_TCP_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT                                                         0x13
-#define DIDT_TCP_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT                                                        0x15
-#define DIDT_TCP_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT                                                        0x16
-#define DIDT_TCP_EDC_CTRL__UNUSED_0__SHIFT                                                                    0x17
-#define DIDT_TCP_EDC_CTRL__EDC_EN_MASK                                                                        0x00000001L
-#define DIDT_TCP_EDC_CTRL__EDC_SW_RST_MASK                                                                    0x00000002L
-#define DIDT_TCP_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                           0x00000004L
-#define DIDT_TCP_EDC_CTRL__EDC_FORCE_STALL_MASK                                                               0x00000008L
-#define DIDT_TCP_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                   0x000001F0L
-#define DIDT_TCP_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK                                                    0x0001FE00L
-#define DIDT_TCP_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                      0x00020000L
-#define DIDT_TCP_EDC_CTRL__GC_EDC_EN_MASK                                                                     0x00040000L
-#define DIDT_TCP_EDC_CTRL__GC_EDC_STALL_POLICY_MASK                                                           0x00180000L
-#define DIDT_TCP_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK                                                          0x00200000L
-#define DIDT_TCP_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK                                                          0x00400000L
-#define DIDT_TCP_EDC_CTRL__UNUSED_0_MASK                                                                      0xFF800000L
-//DIDT_TCP_EDC_THRESHOLD
-#define DIDT_TCP_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                          0x0
-#define DIDT_TCP_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                            0xFFFFFFFFL
-//DIDT_TCP_EDC_STALL_PATTERN_1_2
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                            0x0
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                       0xf
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                            0x10
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                       0x1f
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                              0x00007FFFL
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__UNUSED_0_MASK                                                         0x00008000L
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                              0x7FFF0000L
-#define DIDT_TCP_EDC_STALL_PATTERN_1_2__UNUSED_1_MASK                                                         0x80000000L
-//DIDT_TCP_EDC_STALL_PATTERN_3_4
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                            0x0
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                       0xf
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                            0x10
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                       0x1f
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                              0x00007FFFL
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__UNUSED_0_MASK                                                         0x00008000L
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                              0x7FFF0000L
-#define DIDT_TCP_EDC_STALL_PATTERN_3_4__UNUSED_1_MASK                                                         0x80000000L
-//DIDT_TCP_EDC_STALL_PATTERN_5_6
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                            0x0
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                       0xf
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                            0x10
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                       0x1f
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                              0x00007FFFL
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__UNUSED_0_MASK                                                         0x00008000L
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                              0x7FFF0000L
-#define DIDT_TCP_EDC_STALL_PATTERN_5_6__UNUSED_1_MASK                                                         0x80000000L
-//DIDT_TCP_EDC_STALL_PATTERN_7
-#define DIDT_TCP_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                              0x0
-#define DIDT_TCP_EDC_STALL_PATTERN_7__UNUSED_0__SHIFT                                                         0xf
-#define DIDT_TCP_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                0x00007FFFL
-#define DIDT_TCP_EDC_STALL_PATTERN_7__UNUSED_0_MASK                                                           0xFFFF8000L
-//DIDT_TCP_EDC_STATUS
-#define DIDT_TCP_EDC_STATUS__EDC_FSM_STATE__SHIFT                                                             0x0
-#define DIDT_TCP_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                        0x1
-#define DIDT_TCP_EDC_STATUS__EDC_FSM_STATE_MASK                                                               0x00000001L
-#define DIDT_TCP_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                          0x0000000EL
-//DIDT_TCP_EDC_STALL_DELAY_1
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP0__SHIFT                                               0x0
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP1__SHIFT                                               0x6
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP2__SHIFT                                               0xc
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP3__SHIFT                                               0x12
-#define DIDT_TCP_EDC_STALL_DELAY_1__UNUSED__SHIFT                                                             0x18
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP0_MASK                                                 0x0000003FL
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP1_MASK                                                 0x00000FC0L
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP2_MASK                                                 0x0003F000L
-#define DIDT_TCP_EDC_STALL_DELAY_1__EDC_STALL_DELAY_TCP3_MASK                                                 0x00FC0000L
-#define DIDT_TCP_EDC_STALL_DELAY_1__UNUSED_MASK                                                               0xFF000000L
-//DIDT_TCP_EDC_STALL_DELAY_2
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP4__SHIFT                                               0x0
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP5__SHIFT                                               0x6
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP6__SHIFT                                               0xc
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP7__SHIFT                                               0x12
-#define DIDT_TCP_EDC_STALL_DELAY_2__UNUSED__SHIFT                                                             0x18
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP4_MASK                                                 0x0000003FL
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP5_MASK                                                 0x00000FC0L
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP6_MASK                                                 0x0003F000L
-#define DIDT_TCP_EDC_STALL_DELAY_2__EDC_STALL_DELAY_TCP7_MASK                                                 0x00FC0000L
-#define DIDT_TCP_EDC_STALL_DELAY_2__UNUSED_MASK                                                               0xFF000000L
-//DIDT_TCP_EDC_STALL_DELAY_3
-#define DIDT_TCP_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TCP8__SHIFT                                               0x0
-#define DIDT_TCP_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TCP9__SHIFT                                               0x6
-#define DIDT_TCP_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TCP10__SHIFT                                              0xc
-#define DIDT_TCP_EDC_STALL_DELAY_3__UNUSED__SHIFT                                                             0x12
-#define DIDT_TCP_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TCP8_MASK                                                 0x0000003FL
-#define DIDT_TCP_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TCP9_MASK                                                 0x00000FC0L
-#define DIDT_TCP_EDC_STALL_DELAY_3__EDC_STALL_DELAY_TCP10_MASK                                                0x0003F000L
-#define DIDT_TCP_EDC_STALL_DELAY_3__UNUSED_MASK                                                               0xFFFC0000L
-//DIDT_TCP_EDC_OVERFLOW
-#define DIDT_TCP_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                        0x0
-#define DIDT_TCP_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                     0x1
-#define DIDT_TCP_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                          0x00000001L
-#define DIDT_TCP_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                       0x0001FFFEL
-//DIDT_TCP_EDC_ROLLING_POWER_DELTA
-#define DIDT_TCP_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                      0x0
-#define DIDT_TCP_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                        0xFFFFFFFFL
-//DIDT_DBR_CTRL0
-#define DIDT_DBR_CTRL0__DIDT_CTRL_EN__SHIFT                                                                   0x0
-#define DIDT_DBR_CTRL0__PHASE_OFFSET__SHIFT                                                                   0x1
-#define DIDT_DBR_CTRL0__DIDT_CTRL_RST__SHIFT                                                                  0x3
-#define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT                                                           0x4
-#define DIDT_DBR_CTRL0__DIDT_STALL_CTRL_EN__SHIFT                                                             0x5
-#define DIDT_DBR_CTRL0__DIDT_TUNING_CTRL_EN__SHIFT                                                            0x6
-#define DIDT_DBR_CTRL0__DIDT_STALL_AUTO_RELEASE_EN__SHIFT                                                     0x7
-#define DIDT_DBR_CTRL0__DIDT_HI_POWER_THRESHOLD__SHIFT                                                        0x8
-#define DIDT_DBR_CTRL0__DIDT_AUTO_MPD_EN__SHIFT                                                               0x18
-#define DIDT_DBR_CTRL0__DIDT_STALL_EVENT_EN__SHIFT                                                            0x19
-#define DIDT_DBR_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR__SHIFT                                                 0x1a
-#define DIDT_DBR_CTRL0__UNUSED_0__SHIFT                                                                       0x1b
-#define DIDT_DBR_CTRL0__DIDT_CTRL_EN_MASK                                                                     0x00000001L
-#define DIDT_DBR_CTRL0__PHASE_OFFSET_MASK                                                                     0x00000006L
-#define DIDT_DBR_CTRL0__DIDT_CTRL_RST_MASK                                                                    0x00000008L
-#define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK                                                             0x00000010L
-#define DIDT_DBR_CTRL0__DIDT_STALL_CTRL_EN_MASK                                                               0x00000020L
-#define DIDT_DBR_CTRL0__DIDT_TUNING_CTRL_EN_MASK                                                              0x00000040L
-#define DIDT_DBR_CTRL0__DIDT_STALL_AUTO_RELEASE_EN_MASK                                                       0x00000080L
-#define DIDT_DBR_CTRL0__DIDT_HI_POWER_THRESHOLD_MASK                                                          0x00FFFF00L
-#define DIDT_DBR_CTRL0__DIDT_AUTO_MPD_EN_MASK                                                                 0x01000000L
-#define DIDT_DBR_CTRL0__DIDT_STALL_EVENT_EN_MASK                                                              0x02000000L
-#define DIDT_DBR_CTRL0__DIDT_STALL_EVENT_COUNTER_CLEAR_MASK                                                   0x04000000L
-#define DIDT_DBR_CTRL0__UNUSED_0_MASK                                                                         0xF8000000L
-//DIDT_DBR_CTRL1
-#define DIDT_DBR_CTRL1__MIN_POWER__SHIFT                                                                      0x0
-#define DIDT_DBR_CTRL1__MAX_POWER__SHIFT                                                                      0x10
-#define DIDT_DBR_CTRL1__MIN_POWER_MASK                                                                        0x0000FFFFL
-#define DIDT_DBR_CTRL1__MAX_POWER_MASK                                                                        0xFFFF0000L
-//DIDT_DBR_CTRL2
-#define DIDT_DBR_CTRL2__MAX_POWER_DELTA__SHIFT                                                                0x0
-#define DIDT_DBR_CTRL2__UNUSED_0__SHIFT                                                                       0xe
-#define DIDT_DBR_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT                                                       0x10
-#define DIDT_DBR_CTRL2__UNUSED_1__SHIFT                                                                       0x1a
-#define DIDT_DBR_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT                                                       0x1b
-#define DIDT_DBR_CTRL2__UNUSED_2__SHIFT                                                                       0x1f
-#define DIDT_DBR_CTRL2__MAX_POWER_DELTA_MASK                                                                  0x00003FFFL
-#define DIDT_DBR_CTRL2__UNUSED_0_MASK                                                                         0x0000C000L
-#define DIDT_DBR_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK                                                         0x03FF0000L
-#define DIDT_DBR_CTRL2__UNUSED_1_MASK                                                                         0x04000000L
-#define DIDT_DBR_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK                                                         0x78000000L
-#define DIDT_DBR_CTRL2__UNUSED_2_MASK                                                                         0x80000000L
-//DIDT_DBR_STALL_CTRL
-#define DIDT_DBR_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT                                                       0x0
-#define DIDT_DBR_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT                                                       0x6
-#define DIDT_DBR_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT                                                0xc
-#define DIDT_DBR_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT                                                0x12
-#define DIDT_DBR_STALL_CTRL__UNUSED_0__SHIFT                                                                  0x18
-#define DIDT_DBR_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK                                                         0x0000003FL
-#define DIDT_DBR_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK                                                         0x00000FC0L
-#define DIDT_DBR_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_HI_MASK                                                  0x0003F000L
-#define DIDT_DBR_STALL_CTRL__DIDT_MAX_STALLS_ALLOWED_LO_MASK                                                  0x00FC0000L
-#define DIDT_DBR_STALL_CTRL__UNUSED_0_MASK                                                                    0xFF000000L
-//DIDT_DBR_TUNING_CTRL
-#define DIDT_DBR_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT                                                       0x0
-#define DIDT_DBR_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT                                                       0xe
-#define DIDT_DBR_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK                                                         0x00003FFFL
-#define DIDT_DBR_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK                                                         0x0FFFC000L
-//DIDT_DBR_STALL_AUTO_RELEASE_CTRL
-#define DIDT_DBR_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME__SHIFT                                 0x0
-#define DIDT_DBR_STALL_AUTO_RELEASE_CTRL__DIDT_STALL_AUTO_RELEASE_TIME_MASK                                   0x00FFFFFFL
-//DIDT_DBR_CTRL3
-#define DIDT_DBR_CTRL3__GC_DIDT_ENABLE__SHIFT                                                                 0x0
-#define DIDT_DBR_CTRL3__GC_DIDT_CLK_EN_OVERRIDE__SHIFT                                                        0x1
-#define DIDT_DBR_CTRL3__THROTTLE_POLICY__SHIFT                                                                0x2
-#define DIDT_DBR_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                   0x4
-#define DIDT_DBR_CTRL3__DIDT_POWER_LEVEL_LOWBIT__SHIFT                                                        0x9
-#define DIDT_DBR_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT                                                    0xe
-#define DIDT_DBR_CTRL3__GC_DIDT_LEVEL_COMB_EN__SHIFT                                                          0x16
-#define DIDT_DBR_CTRL3__SE_DIDT_LEVEL_COMB_EN__SHIFT                                                          0x17
-#define DIDT_DBR_CTRL3__QUALIFY_STALL_EN__SHIFT                                                               0x18
-#define DIDT_DBR_CTRL3__DIDT_STALL_SEL__SHIFT                                                                 0x19
-#define DIDT_DBR_CTRL3__DIDT_FORCE_STALL__SHIFT                                                               0x1b
-#define DIDT_DBR_CTRL3__DIDT_STALL_DELAY_EN__SHIFT                                                            0x1c
-#define DIDT_DBR_CTRL3__GC_DIDT_ENABLE_MASK                                                                   0x00000001L
-#define DIDT_DBR_CTRL3__GC_DIDT_CLK_EN_OVERRIDE_MASK                                                          0x00000002L
-#define DIDT_DBR_CTRL3__THROTTLE_POLICY_MASK                                                                  0x0000000CL
-#define DIDT_DBR_CTRL3__DIDT_TRIGGER_THROTTLE_LOWBIT_MASK                                                     0x000001F0L
-#define DIDT_DBR_CTRL3__DIDT_POWER_LEVEL_LOWBIT_MASK                                                          0x00003E00L
-#define DIDT_DBR_CTRL3__DIDT_STALL_PATTERN_BIT_NUMS_MASK                                                      0x003FC000L
-#define DIDT_DBR_CTRL3__GC_DIDT_LEVEL_COMB_EN_MASK                                                            0x00400000L
-#define DIDT_DBR_CTRL3__SE_DIDT_LEVEL_COMB_EN_MASK                                                            0x00800000L
-#define DIDT_DBR_CTRL3__QUALIFY_STALL_EN_MASK                                                                 0x01000000L
-#define DIDT_DBR_CTRL3__DIDT_STALL_SEL_MASK                                                                   0x06000000L
-#define DIDT_DBR_CTRL3__DIDT_FORCE_STALL_MASK                                                                 0x08000000L
-#define DIDT_DBR_CTRL3__DIDT_STALL_DELAY_EN_MASK                                                              0x10000000L
-//DIDT_DBR_STALL_PATTERN_1_2
-#define DIDT_DBR_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT                                               0x0
-#define DIDT_DBR_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                           0xf
-#define DIDT_DBR_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT                                               0x10
-#define DIDT_DBR_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                           0x1f
-#define DIDT_DBR_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK                                                 0x00007FFFL
-#define DIDT_DBR_STALL_PATTERN_1_2__UNUSED_0_MASK                                                             0x00008000L
-#define DIDT_DBR_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK                                                 0x7FFF0000L
-#define DIDT_DBR_STALL_PATTERN_1_2__UNUSED_1_MASK                                                             0x80000000L
-//DIDT_DBR_STALL_PATTERN_3_4
-#define DIDT_DBR_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT                                               0x0
-#define DIDT_DBR_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                           0xf
-#define DIDT_DBR_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT                                               0x10
-#define DIDT_DBR_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                           0x1f
-#define DIDT_DBR_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK                                                 0x00007FFFL
-#define DIDT_DBR_STALL_PATTERN_3_4__UNUSED_0_MASK                                                             0x00008000L
-#define DIDT_DBR_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK                                                 0x7FFF0000L
-#define DIDT_DBR_STALL_PATTERN_3_4__UNUSED_1_MASK                                                             0x80000000L
-//DIDT_DBR_STALL_PATTERN_5_6
-#define DIDT_DBR_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT                                               0x0
-#define DIDT_DBR_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                           0xf
-#define DIDT_DBR_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT                                               0x10
-#define DIDT_DBR_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                           0x1f
-#define DIDT_DBR_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK                                                 0x00007FFFL
-#define DIDT_DBR_STALL_PATTERN_5_6__UNUSED_0_MASK                                                             0x00008000L
-#define DIDT_DBR_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK                                                 0x7FFF0000L
-#define DIDT_DBR_STALL_PATTERN_5_6__UNUSED_1_MASK                                                             0x80000000L
-//DIDT_DBR_STALL_PATTERN_7
-#define DIDT_DBR_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT                                                 0x0
-#define DIDT_DBR_STALL_PATTERN_7__UNUSED_0__SHIFT                                                             0xf
-#define DIDT_DBR_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK                                                   0x00007FFFL
-#define DIDT_DBR_STALL_PATTERN_7__UNUSED_0_MASK                                                               0xFFFF8000L
-//DIDT_DBR_WEIGHT0_3
-#define DIDT_DBR_WEIGHT0_3__WEIGHT0__SHIFT                                                                    0x0
-#define DIDT_DBR_WEIGHT0_3__WEIGHT1__SHIFT                                                                    0x8
-#define DIDT_DBR_WEIGHT0_3__WEIGHT2__SHIFT                                                                    0x10
-#define DIDT_DBR_WEIGHT0_3__WEIGHT3__SHIFT                                                                    0x18
-#define DIDT_DBR_WEIGHT0_3__WEIGHT0_MASK                                                                      0x000000FFL
-#define DIDT_DBR_WEIGHT0_3__WEIGHT1_MASK                                                                      0x0000FF00L
-#define DIDT_DBR_WEIGHT0_3__WEIGHT2_MASK                                                                      0x00FF0000L
-#define DIDT_DBR_WEIGHT0_3__WEIGHT3_MASK                                                                      0xFF000000L
-//DIDT_DBR_WEIGHT4_7
-#define DIDT_DBR_WEIGHT4_7__WEIGHT4__SHIFT                                                                    0x0
-#define DIDT_DBR_WEIGHT4_7__WEIGHT5__SHIFT                                                                    0x8
-#define DIDT_DBR_WEIGHT4_7__WEIGHT6__SHIFT                                                                    0x10
-#define DIDT_DBR_WEIGHT4_7__WEIGHT7__SHIFT                                                                    0x18
-#define DIDT_DBR_WEIGHT4_7__WEIGHT4_MASK                                                                      0x000000FFL
-#define DIDT_DBR_WEIGHT4_7__WEIGHT5_MASK                                                                      0x0000FF00L
-#define DIDT_DBR_WEIGHT4_7__WEIGHT6_MASK                                                                      0x00FF0000L
-#define DIDT_DBR_WEIGHT4_7__WEIGHT7_MASK                                                                      0xFF000000L
-//DIDT_DBR_WEIGHT8_11
-#define DIDT_DBR_WEIGHT8_11__WEIGHT8__SHIFT                                                                   0x0
-#define DIDT_DBR_WEIGHT8_11__WEIGHT9__SHIFT                                                                   0x8
-#define DIDT_DBR_WEIGHT8_11__WEIGHT10__SHIFT                                                                  0x10
-#define DIDT_DBR_WEIGHT8_11__WEIGHT11__SHIFT                                                                  0x18
-#define DIDT_DBR_WEIGHT8_11__WEIGHT8_MASK                                                                     0x000000FFL
-#define DIDT_DBR_WEIGHT8_11__WEIGHT9_MASK                                                                     0x0000FF00L
-#define DIDT_DBR_WEIGHT8_11__WEIGHT10_MASK                                                                    0x00FF0000L
-#define DIDT_DBR_WEIGHT8_11__WEIGHT11_MASK                                                                    0xFF000000L
-//DIDT_DBR_EDC_CTRL
-#define DIDT_DBR_EDC_CTRL__EDC_EN__SHIFT                                                                      0x0
-#define DIDT_DBR_EDC_CTRL__EDC_SW_RST__SHIFT                                                                  0x1
-#define DIDT_DBR_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                         0x2
-#define DIDT_DBR_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                             0x3
-#define DIDT_DBR_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                 0x4
-#define DIDT_DBR_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT                                                  0x9
-#define DIDT_DBR_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                    0x11
-#define DIDT_DBR_EDC_CTRL__GC_EDC_EN__SHIFT                                                                   0x12
-#define DIDT_DBR_EDC_CTRL__GC_EDC_STALL_POLICY__SHIFT                                                         0x13
-#define DIDT_DBR_EDC_CTRL__GC_EDC_LEVEL_COMB_EN__SHIFT                                                        0x15
-#define DIDT_DBR_EDC_CTRL__SE_EDC_LEVEL_COMB_EN__SHIFT                                                        0x16
-#define DIDT_DBR_EDC_CTRL__UNUSED_0__SHIFT                                                                    0x17
-#define DIDT_DBR_EDC_CTRL__EDC_EN_MASK                                                                        0x00000001L
-#define DIDT_DBR_EDC_CTRL__EDC_SW_RST_MASK                                                                    0x00000002L
-#define DIDT_DBR_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                           0x00000004L
-#define DIDT_DBR_EDC_CTRL__EDC_FORCE_STALL_MASK                                                               0x00000008L
-#define DIDT_DBR_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                   0x000001F0L
-#define DIDT_DBR_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK                                                    0x0001FE00L
-#define DIDT_DBR_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                      0x00020000L
-#define DIDT_DBR_EDC_CTRL__GC_EDC_EN_MASK                                                                     0x00040000L
-#define DIDT_DBR_EDC_CTRL__GC_EDC_STALL_POLICY_MASK                                                           0x00180000L
-#define DIDT_DBR_EDC_CTRL__GC_EDC_LEVEL_COMB_EN_MASK                                                          0x00200000L
-#define DIDT_DBR_EDC_CTRL__SE_EDC_LEVEL_COMB_EN_MASK                                                          0x00400000L
-#define DIDT_DBR_EDC_CTRL__UNUSED_0_MASK                                                                      0xFF800000L
-//DIDT_DBR_EDC_THRESHOLD
-#define DIDT_DBR_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                          0x0
-#define DIDT_DBR_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                            0xFFFFFFFFL
-//DIDT_DBR_EDC_STALL_PATTERN_1_2
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                            0x0
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__UNUSED_0__SHIFT                                                       0xf
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                            0x10
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__UNUSED_1__SHIFT                                                       0x1f
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                              0x00007FFFL
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__UNUSED_0_MASK                                                         0x00008000L
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                              0x7FFF0000L
-#define DIDT_DBR_EDC_STALL_PATTERN_1_2__UNUSED_1_MASK                                                         0x80000000L
-//DIDT_DBR_EDC_STALL_PATTERN_3_4
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                            0x0
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__UNUSED_0__SHIFT                                                       0xf
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                            0x10
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__UNUSED_1__SHIFT                                                       0x1f
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                              0x00007FFFL
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__UNUSED_0_MASK                                                         0x00008000L
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                              0x7FFF0000L
-#define DIDT_DBR_EDC_STALL_PATTERN_3_4__UNUSED_1_MASK                                                         0x80000000L
-//DIDT_DBR_EDC_STALL_PATTERN_5_6
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                            0x0
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__UNUSED_0__SHIFT                                                       0xf
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                            0x10
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__UNUSED_1__SHIFT                                                       0x1f
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                              0x00007FFFL
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__UNUSED_0_MASK                                                         0x00008000L
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                              0x7FFF0000L
-#define DIDT_DBR_EDC_STALL_PATTERN_5_6__UNUSED_1_MASK                                                         0x80000000L
-//DIDT_DBR_EDC_STALL_PATTERN_7
-#define DIDT_DBR_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                              0x0
-#define DIDT_DBR_EDC_STALL_PATTERN_7__UNUSED_0__SHIFT                                                         0xf
-#define DIDT_DBR_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                0x00007FFFL
-#define DIDT_DBR_EDC_STALL_PATTERN_7__UNUSED_0_MASK                                                           0xFFFF8000L
-//DIDT_DBR_EDC_STATUS
-#define DIDT_DBR_EDC_STATUS__EDC_FSM_STATE__SHIFT                                                             0x0
-#define DIDT_DBR_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                        0x1
-#define DIDT_DBR_EDC_STATUS__UNUSED_0__SHIFT                                                                  0x4
-#define DIDT_DBR_EDC_STATUS__EDC_FSM_STATE_MASK                                                               0x00000001L
-#define DIDT_DBR_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                          0x0000000EL
-#define DIDT_DBR_EDC_STATUS__UNUSED_0_MASK                                                                    0xFFFFFFF0L
-//DIDT_DBR_EDC_STALL_DELAY_1
-#define DIDT_DBR_EDC_STALL_DELAY_1__EDC_STALL_DELAY_DBR0__SHIFT                                               0x0
-#define DIDT_DBR_EDC_STALL_DELAY_1__UNUSED__SHIFT                                                             0x1
-#define DIDT_DBR_EDC_STALL_DELAY_1__EDC_STALL_DELAY_DBR0_MASK                                                 0x00000001L
-#define DIDT_DBR_EDC_STALL_DELAY_1__UNUSED_MASK                                                               0xFFFFFFFEL
-//DIDT_DBR_EDC_OVERFLOW
-#define DIDT_DBR_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                        0x0
-#define DIDT_DBR_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                     0x1
-#define DIDT_DBR_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                          0x00000001L
-#define DIDT_DBR_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                       0x0001FFFEL
-//DIDT_DBR_EDC_ROLLING_POWER_DELTA
-#define DIDT_DBR_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                      0x0
-#define DIDT_DBR_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                        0xFFFFFFFFL
-//DIDT_SQ_STALL_EVENT_COUNTER
-#define DIDT_SQ_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER__SHIFT                                          0x0
-#define DIDT_SQ_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER_MASK                                            0xFFFFFFFFL
-//DIDT_DB_STALL_EVENT_COUNTER
-#define DIDT_DB_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER__SHIFT                                          0x0
-#define DIDT_DB_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER_MASK                                            0xFFFFFFFFL
-//DIDT_TD_STALL_EVENT_COUNTER
-#define DIDT_TD_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER__SHIFT                                          0x0
-#define DIDT_TD_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER_MASK                                            0xFFFFFFFFL
-//DIDT_TCP_STALL_EVENT_COUNTER
-#define DIDT_TCP_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER__SHIFT                                         0x0
-#define DIDT_TCP_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER_MASK                                           0xFFFFFFFFL
-//DIDT_DBR_STALL_EVENT_COUNTER
-#define DIDT_DBR_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER__SHIFT                                         0x0
-#define DIDT_DBR_STALL_EVENT_COUNTER__DIDT_STALL_EVENT_COUNTER_MASK                                           0xFFFFFFFFL
-
-
-
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_default.h b/drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_default.h
deleted file mode 100644
index 392ef7721f53..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/raven1/MMHUB/mmhub_9_1_default.h
+++ /dev/null
@@ -1,1028 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _mmhub_9_1_DEFAULT_HEADER
-#define _mmhub_9_1_DEFAULT_HEADER
-
-
-// addressBlock: mmhub_dagbdec
-#define mmDAGB0_RDCLI0_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI1_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI2_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI3_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI4_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI5_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI6_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI7_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI8_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI9_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_RDCLI10_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI11_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI12_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI13_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI14_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI15_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI16_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI17_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI18_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI19_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI20_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI21_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI22_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI23_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI24_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI25_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI26_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI27_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI28_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI29_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI30_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RDCLI31_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_RD_CNTL_DEFAULT                                                  0x03527df8
-#define mmDAGB0_RD_GMI_CNTL_DEFAULT                                              0x0000304f
-#define mmDAGB0_RD_ADDR_DAGB_DEFAULT                                             0x00000039
-#define mmDAGB0_RD_OUTPUT_DAGB_MAX_BURST_DEFAULT                                 0x88888888
-#define mmDAGB0_RD_OUTPUT_DAGB_LAZY_TIMER_DEFAULT                                0x11111111
-#define mmDAGB0_RD_CGTT_CLK_CTRL_DEFAULT                                         0x00000100
-#define mmDAGB0_L1TLB_RD_CGTT_CLK_CTRL_DEFAULT                                   0x00000100
-#define mmDAGB0_ATCVM_RD_CGTT_CLK_CTRL_DEFAULT                                   0x00000100
-#define mmDAGB0_RD_ADDR_DAGB_MAX_BURST0_DEFAULT                                  0x88888888
-#define mmDAGB0_RD_ADDR_DAGB_LAZY_TIMER0_DEFAULT                                 0x11111111
-#define mmDAGB0_RD_ADDR_DAGB_MAX_BURST1_DEFAULT                                  0x88888888
-#define mmDAGB0_RD_ADDR_DAGB_LAZY_TIMER1_DEFAULT                                 0x11111111
-#define mmDAGB0_RD_ADDR_DAGB_MAX_BURST2_DEFAULT                                  0x88888888
-#define mmDAGB0_RD_ADDR_DAGB_LAZY_TIMER2_DEFAULT                                 0x11111111
-#define mmDAGB0_RD_ADDR_DAGB_MAX_BURST3_DEFAULT                                  0x88888888
-#define mmDAGB0_RD_ADDR_DAGB_LAZY_TIMER3_DEFAULT                                 0x11111111
-#define mmDAGB0_RD_VC0_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_VC1_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_VC2_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_VC3_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_VC4_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_VC5_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_VC6_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_VC7_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_RD_CNTL_MISC_DEFAULT                                             0x01a10408
-#define mmDAGB0_RD_TLB_CREDIT_DEFAULT                                            0x2f7bdef7
-#define mmDAGB0_RDCLI_ASK_PENDING_DEFAULT                                        0x00000000
-#define mmDAGB0_RDCLI_GO_PENDING_DEFAULT                                         0x00000000
-#define mmDAGB0_RDCLI_GBLSEND_PENDING_DEFAULT                                    0x00000000
-#define mmDAGB0_RDCLI_TLB_PENDING_DEFAULT                                        0x00000000
-#define mmDAGB0_RDCLI_OARB_PENDING_DEFAULT                                       0x00000000
-#define mmDAGB0_RDCLI_OSD_PENDING_DEFAULT                                        0x00000000
-#define mmDAGB0_WRCLI0_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI1_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI2_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI3_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI4_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI5_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI6_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI7_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI8_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI9_DEFAULT                                                   0xfe5fe0f9
-#define mmDAGB0_WRCLI10_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI11_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI12_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI13_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI14_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI15_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI16_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI17_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI18_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI19_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI20_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI21_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI22_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI23_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI24_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI25_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI26_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI27_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI28_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI29_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI30_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WRCLI31_DEFAULT                                                  0xfe5fe0f9
-#define mmDAGB0_WR_CNTL_DEFAULT                                                  0x03527df8
-#define mmDAGB0_WR_GMI_CNTL_DEFAULT                                              0x0000304f
-#define mmDAGB0_WR_ADDR_DAGB_DEFAULT                                             0x00000039
-#define mmDAGB0_WR_OUTPUT_DAGB_MAX_BURST_DEFAULT                                 0x88888888
-#define mmDAGB0_WR_OUTPUT_DAGB_LAZY_TIMER_DEFAULT                                0x11111111
-#define mmDAGB0_WR_CGTT_CLK_CTRL_DEFAULT                                         0x00000100
-#define mmDAGB0_L1TLB_WR_CGTT_CLK_CTRL_DEFAULT                                   0x00000100
-#define mmDAGB0_ATCVM_WR_CGTT_CLK_CTRL_DEFAULT                                   0x00000100
-#define mmDAGB0_WR_ADDR_DAGB_MAX_BURST0_DEFAULT                                  0x88888888
-#define mmDAGB0_WR_ADDR_DAGB_LAZY_TIMER0_DEFAULT                                 0x11111111
-#define mmDAGB0_WR_ADDR_DAGB_MAX_BURST1_DEFAULT                                  0x88888888
-#define mmDAGB0_WR_ADDR_DAGB_LAZY_TIMER1_DEFAULT                                 0x11111111
-#define mmDAGB0_WR_ADDR_DAGB_MAX_BURST2_DEFAULT                                  0x88888888
-#define mmDAGB0_WR_ADDR_DAGB_LAZY_TIMER2_DEFAULT                                 0x11111111
-#define mmDAGB0_WR_ADDR_DAGB_MAX_BURST3_DEFAULT                                  0x88888888
-#define mmDAGB0_WR_ADDR_DAGB_LAZY_TIMER3_DEFAULT                                 0x11111111
-#define mmDAGB0_WR_DATA_DAGB_DEFAULT                                             0x00000001
-#define mmDAGB0_WR_DATA_DAGB_MAX_BURST0_DEFAULT                                  0x11111111
-#define mmDAGB0_WR_DATA_DAGB_LAZY_TIMER0_DEFAULT                                 0x00000000
-#define mmDAGB0_WR_DATA_DAGB_MAX_BURST1_DEFAULT                                  0x11111111
-#define mmDAGB0_WR_DATA_DAGB_LAZY_TIMER1_DEFAULT                                 0x00000000
-#define mmDAGB0_WR_DATA_DAGB_MAX_BURST2_DEFAULT                                  0x11111111
-#define mmDAGB0_WR_DATA_DAGB_LAZY_TIMER2_DEFAULT                                 0x00000000
-#define mmDAGB0_WR_DATA_DAGB_MAX_BURST3_DEFAULT                                  0x11111111
-#define mmDAGB0_WR_DATA_DAGB_LAZY_TIMER3_DEFAULT                                 0x00000000
-#define mmDAGB0_WR_VC0_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_VC1_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_VC2_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_VC3_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_VC4_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_VC5_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_VC6_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_VC7_CNTL_DEFAULT                                              0xff2ff082
-#define mmDAGB0_WR_CNTL_MISC_DEFAULT                                             0x01a10408
-#define mmDAGB0_WR_TLB_CREDIT_DEFAULT                                            0x2f7bdef7
-#define mmDAGB0_WR_DATA_CREDIT_DEFAULT                                           0x5c626870
-#define mmDAGB0_WR_MISC_CREDIT_DEFAULT                                           0x0078dc88
-#define mmDAGB0_WRCLI_ASK_PENDING_DEFAULT                                        0x00000000
-#define mmDAGB0_WRCLI_GO_PENDING_DEFAULT                                         0x00000000
-#define mmDAGB0_WRCLI_GBLSEND_PENDING_DEFAULT                                    0x00000000
-#define mmDAGB0_WRCLI_TLB_PENDING_DEFAULT                                        0x00000000
-#define mmDAGB0_WRCLI_OARB_PENDING_DEFAULT                                       0x00000000
-#define mmDAGB0_WRCLI_OSD_PENDING_DEFAULT                                        0x00000000
-#define mmDAGB0_WRCLI_DBUS_ASK_PENDING_DEFAULT                                   0x00000000
-#define mmDAGB0_WRCLI_DBUS_GO_PENDING_DEFAULT                                    0x00000000
-#define mmDAGB0_DAGB_DLY_DEFAULT                                                 0x00000000
-#define mmDAGB0_CNTL_MISC_DEFAULT                                                0xcf7c1ffa
-#define mmDAGB0_CNTL_MISC2_DEFAULT                                               0x00000000
-#define mmDAGB0_FIFO_EMPTY_DEFAULT                                               0x00ffffff
-#define mmDAGB0_FIFO_FULL_DEFAULT                                                0x00000000
-#define mmDAGB0_WR_CREDITS_FULL_DEFAULT                                          0x0007ffff
-#define mmDAGB0_RD_CREDITS_FULL_DEFAULT                                          0x0003ffff
-#define mmDAGB0_PERFCOUNTER_LO_DEFAULT                                           0x00000000
-#define mmDAGB0_PERFCOUNTER_HI_DEFAULT                                           0x00000000
-#define mmDAGB0_PERFCOUNTER0_CFG_DEFAULT                                         0x00000000
-#define mmDAGB0_PERFCOUNTER1_CFG_DEFAULT                                         0x00000000
-#define mmDAGB0_PERFCOUNTER2_CFG_DEFAULT                                         0x00000000
-#define mmDAGB0_PERFCOUNTER_RSLT_CNTL_DEFAULT                                    0x04000000
-#define mmDAGB0_RESERVE0_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE1_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE2_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE3_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE4_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE5_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE6_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE7_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE8_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE9_DEFAULT                                                 0x00000000
-#define mmDAGB0_RESERVE10_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE11_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE12_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE13_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE14_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE15_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE16_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE17_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE18_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE19_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE20_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE21_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE22_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE23_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE24_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE25_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE26_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE27_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE28_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE29_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE30_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE31_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE32_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE33_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE34_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE35_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE36_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE37_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE38_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE39_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE40_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE41_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE42_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE43_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE44_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE45_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE46_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE47_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE48_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE49_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE50_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE51_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE52_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE53_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE54_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE55_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE56_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE57_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE58_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE59_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE60_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE61_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE62_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE63_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE64_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE65_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE66_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE67_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE68_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE69_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE70_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE71_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE72_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE73_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE74_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE75_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE76_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE77_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE78_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE79_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE80_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE81_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE82_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE83_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE84_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE85_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE86_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE87_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE88_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE89_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE90_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE91_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE92_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE93_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE94_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE95_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE96_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE97_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE98_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE99_DEFAULT                                                0x00000000
-#define mmDAGB0_RESERVE100_DEFAULT                                               0x00000000
-#define mmDAGB0_RESERVE101_DEFAULT                                               0x00000000
-
-
-// addressBlock: mmhub_ea_mmeadec
-#define mmMMEA0_DRAM_RD_CLI2GRP_MAP0_DEFAULT                                     0x55555555
-#define mmMMEA0_DRAM_RD_CLI2GRP_MAP1_DEFAULT                                     0x55555555
-#define mmMMEA0_DRAM_WR_CLI2GRP_MAP0_DEFAULT                                     0x55555555
-#define mmMMEA0_DRAM_WR_CLI2GRP_MAP1_DEFAULT                                     0x55555555
-#define mmMMEA0_DRAM_RD_GRP2VC_MAP_DEFAULT                                       0x00000e25
-#define mmMMEA0_DRAM_WR_GRP2VC_MAP_DEFAULT                                       0x00000e25
-#define mmMMEA0_DRAM_RD_LAZY_DEFAULT                                             0x00000924
-#define mmMMEA0_DRAM_WR_LAZY_DEFAULT                                             0x00000924
-#define mmMMEA0_DRAM_RD_CAM_CNTL_DEFAULT                                         0x06db3333
-#define mmMMEA0_DRAM_WR_CAM_CNTL_DEFAULT                                         0x06db3333
-#define mmMMEA0_DRAM_PAGE_BURST_DEFAULT                                          0x20002000
-#define mmMMEA0_DRAM_RD_PRI_AGE_DEFAULT                                          0x00db6249
-#define mmMMEA0_DRAM_WR_PRI_AGE_DEFAULT                                          0x00db6249
-#define mmMMEA0_DRAM_RD_PRI_QUEUING_DEFAULT                                      0x00000db6
-#define mmMMEA0_DRAM_WR_PRI_QUEUING_DEFAULT                                      0x00000db6
-#define mmMMEA0_DRAM_RD_PRI_FIXED_DEFAULT                                        0x00000924
-#define mmMMEA0_DRAM_WR_PRI_FIXED_DEFAULT                                        0x00000924
-#define mmMMEA0_DRAM_RD_PRI_URGENCY_DEFAULT                                      0x0000fdb6
-#define mmMMEA0_DRAM_WR_PRI_URGENCY_DEFAULT                                      0x0000fdb6
-#define mmMMEA0_DRAM_RD_PRI_QUANT_PRI1_DEFAULT                                   0x3f3f3f3f
-#define mmMMEA0_DRAM_RD_PRI_QUANT_PRI2_DEFAULT                                   0x7f7f7f7f
-#define mmMMEA0_DRAM_RD_PRI_QUANT_PRI3_DEFAULT                                   0xffffffff
-#define mmMMEA0_DRAM_WR_PRI_QUANT_PRI1_DEFAULT                                   0x3f3f3f3f
-#define mmMMEA0_DRAM_WR_PRI_QUANT_PRI2_DEFAULT                                   0x7f7f7f7f
-#define mmMMEA0_DRAM_WR_PRI_QUANT_PRI3_DEFAULT                                   0xffffffff
-#define mmMMEA0_ADDRNORM_BASE_ADDR0_DEFAULT                                      0x00000000
-#define mmMMEA0_ADDRNORM_LIMIT_ADDR0_DEFAULT                                     0x00000000
-#define mmMMEA0_ADDRNORM_BASE_ADDR1_DEFAULT                                      0x00000000
-#define mmMMEA0_ADDRNORM_LIMIT_ADDR1_DEFAULT                                     0x00000000
-#define mmMMEA0_ADDRNORM_OFFSET_ADDR1_DEFAULT                                    0x00000000
-#define mmMMEA0_ADDRNORM_HOLE_CNTL_DEFAULT                                       0x00000000
-#define mmMMEA0_ADDRDEC_BANK_CFG_DEFAULT                                         0x000001ef
-#define mmMMEA0_ADDRDEC_MISC_CFG_DEFAULT                                         0x3ffff000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_BANK0_DEFAULT                              0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_BANK1_DEFAULT                              0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_BANK2_DEFAULT                              0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_BANK3_DEFAULT                              0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_BANK4_DEFAULT                              0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_PC_DEFAULT                                 0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_PC2_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_CS0_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDECDRAM_ADDR_HASH_CS1_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDECDRAM_HARVEST_ENABLE_DEFAULT                               0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_CS0_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_CS1_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_CS2_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_CS3_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_SECCS0_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_SECCS1_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_SECCS2_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC0_BASE_ADDR_SECCS3_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC0_ADDR_MASK_CS01_DEFAULT                                  0xfffffffe
-#define mmMMEA0_ADDRDEC0_ADDR_MASK_CS23_DEFAULT                                  0xfffffffe
-#define mmMMEA0_ADDRDEC0_ADDR_MASK_SECCS01_DEFAULT                               0xfffffffe
-#define mmMMEA0_ADDRDEC0_ADDR_MASK_SECCS23_DEFAULT                               0xfffffffe
-#define mmMMEA0_ADDRDEC0_ADDR_CFG_CS01_DEFAULT                                   0x00050408
-#define mmMMEA0_ADDRDEC0_ADDR_CFG_CS23_DEFAULT                                   0x00050408
-#define mmMMEA0_ADDRDEC0_ADDR_SEL_CS01_DEFAULT                                   0x04076543
-#define mmMMEA0_ADDRDEC0_ADDR_SEL_CS23_DEFAULT                                   0x04076543
-#define mmMMEA0_ADDRDEC0_COL_SEL_LO_CS01_DEFAULT                                 0x87654321
-#define mmMMEA0_ADDRDEC0_COL_SEL_LO_CS23_DEFAULT                                 0x87654321
-#define mmMMEA0_ADDRDEC0_COL_SEL_HI_CS01_DEFAULT                                 0xa9876543
-#define mmMMEA0_ADDRDEC0_COL_SEL_HI_CS23_DEFAULT                                 0xa9876543
-#define mmMMEA0_ADDRDEC0_RM_SEL_CS01_DEFAULT                                     0x00000000
-#define mmMMEA0_ADDRDEC0_RM_SEL_CS23_DEFAULT                                     0x00000000
-#define mmMMEA0_ADDRDEC0_RM_SEL_SECCS01_DEFAULT                                  0x00000000
-#define mmMMEA0_ADDRDEC0_RM_SEL_SECCS23_DEFAULT                                  0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_CS0_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_CS1_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_CS2_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_CS3_DEFAULT                                   0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_SECCS0_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_SECCS1_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_SECCS2_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC1_BASE_ADDR_SECCS3_DEFAULT                                0x00000000
-#define mmMMEA0_ADDRDEC1_ADDR_MASK_CS01_DEFAULT                                  0xfffffffe
-#define mmMMEA0_ADDRDEC1_ADDR_MASK_CS23_DEFAULT                                  0xfffffffe
-#define mmMMEA0_ADDRDEC1_ADDR_MASK_SECCS01_DEFAULT                               0xfffffffe
-#define mmMMEA0_ADDRDEC1_ADDR_MASK_SECCS23_DEFAULT                               0xfffffffe
-#define mmMMEA0_ADDRDEC1_ADDR_CFG_CS01_DEFAULT                                   0x00050408
-#define mmMMEA0_ADDRDEC1_ADDR_CFG_CS23_DEFAULT                                   0x00050408
-#define mmMMEA0_ADDRDEC1_ADDR_SEL_CS01_DEFAULT                                   0x04076543
-#define mmMMEA0_ADDRDEC1_ADDR_SEL_CS23_DEFAULT                                   0x04076543
-#define mmMMEA0_ADDRDEC1_COL_SEL_LO_CS01_DEFAULT                                 0x87654321
-#define mmMMEA0_ADDRDEC1_COL_SEL_LO_CS23_DEFAULT                                 0x87654321
-#define mmMMEA0_ADDRDEC1_COL_SEL_HI_CS01_DEFAULT                                 0xa9876543
-#define mmMMEA0_ADDRDEC1_COL_SEL_HI_CS23_DEFAULT                                 0xa9876543
-#define mmMMEA0_ADDRDEC1_RM_SEL_CS01_DEFAULT                                     0x00000000
-#define mmMMEA0_ADDRDEC1_RM_SEL_CS23_DEFAULT                                     0x00000000
-#define mmMMEA0_ADDRDEC1_RM_SEL_SECCS01_DEFAULT                                  0x00000000
-#define mmMMEA0_ADDRDEC1_RM_SEL_SECCS23_DEFAULT                                  0x00000000
-#define mmMMEA0_IO_RD_CLI2GRP_MAP0_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA0_IO_RD_CLI2GRP_MAP1_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA0_IO_WR_CLI2GRP_MAP0_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA0_IO_WR_CLI2GRP_MAP1_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA0_IO_RD_COMBINE_FLUSH_DEFAULT                                      0x00007777
-#define mmMMEA0_IO_WR_COMBINE_FLUSH_DEFAULT                                      0x00007777
-#define mmMMEA0_IO_GROUP_BURST_DEFAULT                                           0x1f031f03
-#define mmMMEA0_IO_RD_PRI_AGE_DEFAULT                                            0x00db6249
-#define mmMMEA0_IO_WR_PRI_AGE_DEFAULT                                            0x00db6249
-#define mmMMEA0_IO_RD_PRI_QUEUING_DEFAULT                                        0x00000db6
-#define mmMMEA0_IO_WR_PRI_QUEUING_DEFAULT                                        0x00000db6
-#define mmMMEA0_IO_RD_PRI_FIXED_DEFAULT                                          0x00000924
-#define mmMMEA0_IO_WR_PRI_FIXED_DEFAULT                                          0x00000924
-#define mmMMEA0_IO_RD_PRI_URGENCY_DEFAULT                                        0x00000492
-#define mmMMEA0_IO_WR_PRI_URGENCY_DEFAULT                                        0x00000492
-#define mmMMEA0_IO_RD_PRI_URGENCY_MASK_DEFAULT                                   0xffffffff
-#define mmMMEA0_IO_WR_PRI_URGENCY_MASK_DEFAULT                                   0xffffffff
-#define mmMMEA0_IO_RD_PRI_QUANT_PRI1_DEFAULT                                     0x3f3f3f3f
-#define mmMMEA0_IO_RD_PRI_QUANT_PRI2_DEFAULT                                     0x7f7f7f7f
-#define mmMMEA0_IO_RD_PRI_QUANT_PRI3_DEFAULT                                     0xffffffff
-#define mmMMEA0_IO_WR_PRI_QUANT_PRI1_DEFAULT                                     0x3f3f3f3f
-#define mmMMEA0_IO_WR_PRI_QUANT_PRI2_DEFAULT                                     0x7f7f7f7f
-#define mmMMEA0_IO_WR_PRI_QUANT_PRI3_DEFAULT                                     0xffffffff
-#define mmMMEA0_SDP_ARB_DRAM_DEFAULT                                             0x00102040
-#define mmMMEA0_SDP_ARB_FINAL_DEFAULT                                            0x00007fff
-#define mmMMEA0_SDP_DRAM_PRIORITY_DEFAULT                                        0x00000000
-#define mmMMEA0_SDP_IO_PRIORITY_DEFAULT                                          0x00000000
-#define mmMMEA0_SDP_CREDITS_DEFAULT                                              0x000100bf
-#define mmMMEA0_SDP_TAG_RESERVE0_DEFAULT                                         0x00000000
-#define mmMMEA0_SDP_TAG_RESERVE1_DEFAULT                                         0x00000000
-#define mmMMEA0_SDP_VCC_RESERVE0_DEFAULT                                         0x00000000
-#define mmMMEA0_SDP_VCC_RESERVE1_DEFAULT                                         0x00000000
-#define mmMMEA0_SDP_VCD_RESERVE0_DEFAULT                                         0x00000000
-#define mmMMEA0_SDP_VCD_RESERVE1_DEFAULT                                         0x00000000
-#define mmMMEA0_SDP_REQ_CNTL_DEFAULT                                             0x0000000f
-#define mmMMEA0_MISC_DEFAULT                                                     0x00180130
-#define mmMMEA0_LATENCY_SAMPLING_DEFAULT                                         0x00000000
-#define mmMMEA0_PERFCOUNTER_LO_DEFAULT                                           0x00000000
-#define mmMMEA0_PERFCOUNTER_HI_DEFAULT                                           0x00000000
-#define mmMMEA0_PERFCOUNTER0_CFG_DEFAULT                                         0x00000000
-#define mmMMEA0_PERFCOUNTER1_CFG_DEFAULT                                         0x00000000
-#define mmMMEA0_PERFCOUNTER_RSLT_CNTL_DEFAULT                                    0x04000000
-#define mmMMEA0_EDC_CNT_DEFAULT                                                  0x00000000
-#define mmMMEA0_EDC_CNT2_DEFAULT                                                 0x00000000
-#define mmMMEA0_DSM_CNTL_DEFAULT                                                 0x00000000
-#define mmMMEA0_DSM_CNTLA_DEFAULT                                                0x00000000
-#define mmMMEA0_DSM_CNTLB_DEFAULT                                                0x00000000
-#define mmMMEA0_DSM_CNTL2_DEFAULT                                                0x00000000
-#define mmMMEA0_DSM_CNTL2A_DEFAULT                                               0x00000000
-#define mmMMEA0_DSM_CNTL2B_DEFAULT                                               0x00000000
-#define mmMMEA0_CGTT_CLK_CTRL_DEFAULT                                            0x00000100
-#define mmMMEA0_EDC_MODE_DEFAULT                                                 0x00000000
-#define mmMMEA0_ERR_STATUS_DEFAULT                                               0x00000000
-#define mmMMEA0_MISC2_DEFAULT                                                    0x00000000
-#define mmMMEA1_DRAM_RD_CLI2GRP_MAP0_DEFAULT                                     0x55555555
-#define mmMMEA1_DRAM_RD_CLI2GRP_MAP1_DEFAULT                                     0x55555555
-#define mmMMEA1_DRAM_WR_CLI2GRP_MAP0_DEFAULT                                     0x55555555
-#define mmMMEA1_DRAM_WR_CLI2GRP_MAP1_DEFAULT                                     0x55555555
-#define mmMMEA1_DRAM_RD_GRP2VC_MAP_DEFAULT                                       0x00000e25
-#define mmMMEA1_DRAM_WR_GRP2VC_MAP_DEFAULT                                       0x00000e25
-#define mmMMEA1_DRAM_RD_LAZY_DEFAULT                                             0x00000924
-#define mmMMEA1_DRAM_WR_LAZY_DEFAULT                                             0x00000924
-#define mmMMEA1_DRAM_RD_CAM_CNTL_DEFAULT                                         0x06db3333
-#define mmMMEA1_DRAM_WR_CAM_CNTL_DEFAULT                                         0x06db3333
-#define mmMMEA1_DRAM_PAGE_BURST_DEFAULT                                          0x20002000
-#define mmMMEA1_DRAM_RD_PRI_AGE_DEFAULT                                          0x00db6249
-#define mmMMEA1_DRAM_WR_PRI_AGE_DEFAULT                                          0x00db6249
-#define mmMMEA1_DRAM_RD_PRI_QUEUING_DEFAULT                                      0x00000db6
-#define mmMMEA1_DRAM_WR_PRI_QUEUING_DEFAULT                                      0x00000db6
-#define mmMMEA1_DRAM_RD_PRI_FIXED_DEFAULT                                        0x00000924
-#define mmMMEA1_DRAM_WR_PRI_FIXED_DEFAULT                                        0x00000924
-#define mmMMEA1_DRAM_RD_PRI_URGENCY_DEFAULT                                      0x0000fdb6
-#define mmMMEA1_DRAM_WR_PRI_URGENCY_DEFAULT                                      0x0000fdb6
-#define mmMMEA1_DRAM_RD_PRI_QUANT_PRI1_DEFAULT                                   0x3f3f3f3f
-#define mmMMEA1_DRAM_RD_PRI_QUANT_PRI2_DEFAULT                                   0x7f7f7f7f
-#define mmMMEA1_DRAM_RD_PRI_QUANT_PRI3_DEFAULT                                   0xffffffff
-#define mmMMEA1_DRAM_WR_PRI_QUANT_PRI1_DEFAULT                                   0x3f3f3f3f
-#define mmMMEA1_DRAM_WR_PRI_QUANT_PRI2_DEFAULT                                   0x7f7f7f7f
-#define mmMMEA1_DRAM_WR_PRI_QUANT_PRI3_DEFAULT                                   0xffffffff
-#define mmMMEA1_ADDRNORM_BASE_ADDR0_DEFAULT                                      0x00000000
-#define mmMMEA1_ADDRNORM_LIMIT_ADDR0_DEFAULT                                     0x00000000
-#define mmMMEA1_ADDRNORM_BASE_ADDR1_DEFAULT                                      0x00000000
-#define mmMMEA1_ADDRNORM_LIMIT_ADDR1_DEFAULT                                     0x00000000
-#define mmMMEA1_ADDRNORM_OFFSET_ADDR1_DEFAULT                                    0x00000000
-#define mmMMEA1_ADDRNORM_HOLE_CNTL_DEFAULT                                       0x00000000
-#define mmMMEA1_ADDRDEC_BANK_CFG_DEFAULT                                         0x000001ef
-#define mmMMEA1_ADDRDEC_MISC_CFG_DEFAULT                                         0x3ffff000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_BANK0_DEFAULT                              0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_BANK1_DEFAULT                              0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_BANK2_DEFAULT                              0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_BANK3_DEFAULT                              0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_BANK4_DEFAULT                              0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_PC_DEFAULT                                 0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_PC2_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_CS0_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDECDRAM_ADDR_HASH_CS1_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDECDRAM_HARVEST_ENABLE_DEFAULT                               0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_CS0_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_CS1_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_CS2_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_CS3_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_SECCS0_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_SECCS1_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_SECCS2_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC0_BASE_ADDR_SECCS3_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC0_ADDR_MASK_CS01_DEFAULT                                  0xfffffffe
-#define mmMMEA1_ADDRDEC0_ADDR_MASK_CS23_DEFAULT                                  0xfffffffe
-#define mmMMEA1_ADDRDEC0_ADDR_MASK_SECCS01_DEFAULT                               0xfffffffe
-#define mmMMEA1_ADDRDEC0_ADDR_MASK_SECCS23_DEFAULT                               0xfffffffe
-#define mmMMEA1_ADDRDEC0_ADDR_CFG_CS01_DEFAULT                                   0x00050408
-#define mmMMEA1_ADDRDEC0_ADDR_CFG_CS23_DEFAULT                                   0x00050408
-#define mmMMEA1_ADDRDEC0_ADDR_SEL_CS01_DEFAULT                                   0x04076543
-#define mmMMEA1_ADDRDEC0_ADDR_SEL_CS23_DEFAULT                                   0x04076543
-#define mmMMEA1_ADDRDEC0_COL_SEL_LO_CS01_DEFAULT                                 0x87654321
-#define mmMMEA1_ADDRDEC0_COL_SEL_LO_CS23_DEFAULT                                 0x87654321
-#define mmMMEA1_ADDRDEC0_COL_SEL_HI_CS01_DEFAULT                                 0xa9876543
-#define mmMMEA1_ADDRDEC0_COL_SEL_HI_CS23_DEFAULT                                 0xa9876543
-#define mmMMEA1_ADDRDEC0_RM_SEL_CS01_DEFAULT                                     0x00000000
-#define mmMMEA1_ADDRDEC0_RM_SEL_CS23_DEFAULT                                     0x00000000
-#define mmMMEA1_ADDRDEC0_RM_SEL_SECCS01_DEFAULT                                  0x00000000
-#define mmMMEA1_ADDRDEC0_RM_SEL_SECCS23_DEFAULT                                  0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_CS0_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_CS1_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_CS2_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_CS3_DEFAULT                                   0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_SECCS0_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_SECCS1_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_SECCS2_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC1_BASE_ADDR_SECCS3_DEFAULT                                0x00000000
-#define mmMMEA1_ADDRDEC1_ADDR_MASK_CS01_DEFAULT                                  0xfffffffe
-#define mmMMEA1_ADDRDEC1_ADDR_MASK_CS23_DEFAULT                                  0xfffffffe
-#define mmMMEA1_ADDRDEC1_ADDR_MASK_SECCS01_DEFAULT                               0xfffffffe
-#define mmMMEA1_ADDRDEC1_ADDR_MASK_SECCS23_DEFAULT                               0xfffffffe
-#define mmMMEA1_ADDRDEC1_ADDR_CFG_CS01_DEFAULT                                   0x00050408
-#define mmMMEA1_ADDRDEC1_ADDR_CFG_CS23_DEFAULT                                   0x00050408
-#define mmMMEA1_ADDRDEC1_ADDR_SEL_CS01_DEFAULT                                   0x04076543
-#define mmMMEA1_ADDRDEC1_ADDR_SEL_CS23_DEFAULT                                   0x04076543
-#define mmMMEA1_ADDRDEC1_COL_SEL_LO_CS01_DEFAULT                                 0x87654321
-#define mmMMEA1_ADDRDEC1_COL_SEL_LO_CS23_DEFAULT                                 0x87654321
-#define mmMMEA1_ADDRDEC1_COL_SEL_HI_CS01_DEFAULT                                 0xa9876543
-#define mmMMEA1_ADDRDEC1_COL_SEL_HI_CS23_DEFAULT                                 0xa9876543
-#define mmMMEA1_ADDRDEC1_RM_SEL_CS01_DEFAULT                                     0x00000000
-#define mmMMEA1_ADDRDEC1_RM_SEL_CS23_DEFAULT                                     0x00000000
-#define mmMMEA1_ADDRDEC1_RM_SEL_SECCS01_DEFAULT                                  0x00000000
-#define mmMMEA1_ADDRDEC1_RM_SEL_SECCS23_DEFAULT                                  0x00000000
-#define mmMMEA1_IO_RD_CLI2GRP_MAP0_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA1_IO_RD_CLI2GRP_MAP1_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA1_IO_WR_CLI2GRP_MAP0_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA1_IO_WR_CLI2GRP_MAP1_DEFAULT                                       0xe4e4e4e4
-#define mmMMEA1_IO_RD_COMBINE_FLUSH_DEFAULT                                      0x00007777
-#define mmMMEA1_IO_WR_COMBINE_FLUSH_DEFAULT                                      0x00007777
-#define mmMMEA1_IO_GROUP_BURST_DEFAULT                                           0x1f031f03
-#define mmMMEA1_IO_RD_PRI_AGE_DEFAULT                                            0x00db6249
-#define mmMMEA1_IO_WR_PRI_AGE_DEFAULT                                            0x00db6249
-#define mmMMEA1_IO_RD_PRI_QUEUING_DEFAULT                                        0x00000db6
-#define mmMMEA1_IO_WR_PRI_QUEUING_DEFAULT                                        0x00000db6
-#define mmMMEA1_IO_RD_PRI_FIXED_DEFAULT                                          0x00000924
-#define mmMMEA1_IO_WR_PRI_FIXED_DEFAULT                                          0x00000924
-#define mmMMEA1_IO_RD_PRI_URGENCY_DEFAULT                                        0x00000492
-#define mmMMEA1_IO_WR_PRI_URGENCY_DEFAULT                                        0x00000492
-#define mmMMEA1_IO_RD_PRI_URGENCY_MASK_DEFAULT                                   0xffffffff
-#define mmMMEA1_IO_WR_PRI_URGENCY_MASK_DEFAULT                                   0xffffffff
-#define mmMMEA1_IO_RD_PRI_QUANT_PRI1_DEFAULT                                     0x3f3f3f3f
-#define mmMMEA1_IO_RD_PRI_QUANT_PRI2_DEFAULT                                     0x7f7f7f7f
-#define mmMMEA1_IO_RD_PRI_QUANT_PRI3_DEFAULT                                     0xffffffff
-#define mmMMEA1_IO_WR_PRI_QUANT_PRI1_DEFAULT                                     0x3f3f3f3f
-#define mmMMEA1_IO_WR_PRI_QUANT_PRI2_DEFAULT                                     0x7f7f7f7f
-#define mmMMEA1_IO_WR_PRI_QUANT_PRI3_DEFAULT                                     0xffffffff
-#define mmMMEA1_SDP_ARB_DRAM_DEFAULT                                             0x00102040
-#define mmMMEA1_SDP_ARB_FINAL_DEFAULT                                            0x00007fff
-#define mmMMEA1_SDP_DRAM_PRIORITY_DEFAULT                                        0x00000000
-#define mmMMEA1_SDP_IO_PRIORITY_DEFAULT                                          0x00000000
-#define mmMMEA1_SDP_CREDITS_DEFAULT                                              0x000100bf
-#define mmMMEA1_SDP_TAG_RESERVE0_DEFAULT                                         0x00000000
-#define mmMMEA1_SDP_TAG_RESERVE1_DEFAULT                                         0x00000000
-#define mmMMEA1_SDP_VCC_RESERVE0_DEFAULT                                         0x00000000
-#define mmMMEA1_SDP_VCC_RESERVE1_DEFAULT                                         0x00000000
-#define mmMMEA1_SDP_VCD_RESERVE0_DEFAULT                                         0x00000000
-#define mmMMEA1_SDP_VCD_RESERVE1_DEFAULT                                         0x00000000
-#define mmMMEA1_SDP_REQ_CNTL_DEFAULT                                             0x0000000f
-#define mmMMEA1_MISC_DEFAULT                                                     0x00180130
-#define mmMMEA1_LATENCY_SAMPLING_DEFAULT                                         0x00000000
-#define mmMMEA1_PERFCOUNTER_LO_DEFAULT                                           0x00000000
-#define mmMMEA1_PERFCOUNTER_HI_DEFAULT                                           0x00000000
-#define mmMMEA1_PERFCOUNTER0_CFG_DEFAULT                                         0x00000000
-#define mmMMEA1_PERFCOUNTER1_CFG_DEFAULT                                         0x00000000
-#define mmMMEA1_PERFCOUNTER_RSLT_CNTL_DEFAULT                                    0x04000000
-#define mmMMEA1_EDC_CNT_DEFAULT                                                  0x00000000
-#define mmMMEA1_EDC_CNT2_DEFAULT                                                 0x00000000
-#define mmMMEA1_DSM_CNTL_DEFAULT                                                 0x00000000
-#define mmMMEA1_DSM_CNTLA_DEFAULT                                                0x00000000
-#define mmMMEA1_DSM_CNTLB_DEFAULT                                                0x00000000
-#define mmMMEA1_DSM_CNTL2_DEFAULT                                                0x00000000
-#define mmMMEA1_DSM_CNTL2A_DEFAULT                                               0x00000000
-#define mmMMEA1_DSM_CNTL2B_DEFAULT                                               0x00000000
-#define mmMMEA1_CGTT_CLK_CTRL_DEFAULT                                            0x00000100
-#define mmMMEA1_EDC_MODE_DEFAULT                                                 0x00000000
-#define mmMMEA1_ERR_STATUS_DEFAULT                                               0x00000000
-#define mmMMEA1_MISC2_DEFAULT                                                    0x00000000
-
-
-// addressBlock: mmhub_pctldec
-#define mmPCTL_MISC_DEFAULT                                                      0x00000889
-#define mmPCTL_MMHUB_DEEPSLEEP_DEFAULT                                           0x00000000
-#define mmPCTL_MMHUB_DEEPSLEEP_OVERRIDE_DEFAULT                                  0x00000000
-#define mmPCTL_PG_IGNORE_DEEPSLEEP_DEFAULT                                       0x00000000
-#define mmPCTL_PG_DAGB_DEFAULT                                                   0x00000000
-#define mmPCTL0_RENG_RAM_INDEX_DEFAULT                                           0x00000000
-#define mmPCTL0_RENG_RAM_DATA_DEFAULT                                            0x00000000
-#define mmPCTL0_RENG_EXECUTE_DEFAULT                                             0x00000000
-#define mmPCTL0_MISC_DEFAULT                                                     0x00001000
-#define mmPCTL0_STCTRL_REGISTER_SAVE_RANGE0_DEFAULT                              0x00000000
-#define mmPCTL0_STCTRL_REGISTER_SAVE_RANGE1_DEFAULT                              0x00000000
-#define mmPCTL0_STCTRL_REGISTER_SAVE_RANGE2_DEFAULT                              0x00000000
-#define mmPCTL0_STCTRL_REGISTER_SAVE_EXCL_SET_DEFAULT                            0xffffffff
-#define mmPCTL0_STCTRL_REGISTER_SAVE_EXCL_SET1_DEFAULT                           0xffffffff
-#define mmPCTL1_RENG_RAM_INDEX_DEFAULT                                           0x00000000
-#define mmPCTL1_RENG_RAM_DATA_DEFAULT                                            0x00000000
-#define mmPCTL1_RENG_EXECUTE_DEFAULT                                             0x00000000
-#define mmPCTL1_MISC_DEFAULT                                                     0x00000800
-#define mmPCTL1_STCTRL_REGISTER_SAVE_RANGE0_DEFAULT                              0x061f05a0
-#define mmPCTL1_STCTRL_REGISTER_SAVE_RANGE1_DEFAULT                              0x08590800
-#define mmPCTL1_STCTRL_REGISTER_SAVE_RANGE2_DEFAULT                              0x00000000
-#define mmPCTL1_STCTRL_REGISTER_SAVE_EXCL_SET_DEFAULT                            0xffffffff
-#define mmPCTL1_STCTRL_REGISTER_SAVE_EXCL_SET1_DEFAULT                           0xffffffff
-#define mmPCTL2_RENG_RAM_INDEX_DEFAULT                                           0x00000000
-#define mmPCTL2_RENG_RAM_DATA_DEFAULT                                            0x00000000
-#define mmPCTL2_RENG_EXECUTE_DEFAULT                                             0x00000000
-#define mmPCTL2_MISC_DEFAULT                                                     0x00000800
-#define mmPCTL2_STCTRL_REGISTER_SAVE_RANGE0_DEFAULT                              0x069f0620
-#define mmPCTL2_STCTRL_REGISTER_SAVE_RANGE1_DEFAULT                              0x08b3085a
-#define mmPCTL2_STCTRL_REGISTER_SAVE_RANGE2_DEFAULT                              0x00000000
-#define mmPCTL2_STCTRL_REGISTER_SAVE_EXCL_SET_DEFAULT                            0xffffffff
-#define mmPCTL2_STCTRL_REGISTER_SAVE_EXCL_SET1_DEFAULT                           0xffffffff
-
-
-// addressBlock: mmhub_l1tlb_vml1dec
-#define mmMC_VM_MX_L1_TLB0_STATUS_DEFAULT                                        0x00000000
-#define mmMC_VM_MX_L1_TLB1_STATUS_DEFAULT                                        0x00000000
-#define mmMC_VM_MX_L1_TLB2_STATUS_DEFAULT                                        0x00000000
-#define mmMC_VM_MX_L1_TLB3_STATUS_DEFAULT                                        0x00000000
-#define mmMC_VM_MX_L1_TLB4_STATUS_DEFAULT                                        0x00000000
-#define mmMC_VM_MX_L1_TLB5_STATUS_DEFAULT                                        0x00000000
-#define mmMC_VM_MX_L1_TLB6_STATUS_DEFAULT                                        0x00000000
-#define mmMC_VM_MX_L1_TLB7_STATUS_DEFAULT                                        0x00000000
-
-
-// addressBlock: mmhub_l1tlb_vml1pldec
-#define mmMC_VM_MX_L1_PERFCOUNTER0_CFG_DEFAULT                                   0x00000000
-#define mmMC_VM_MX_L1_PERFCOUNTER1_CFG_DEFAULT                                   0x00000000
-#define mmMC_VM_MX_L1_PERFCOUNTER2_CFG_DEFAULT                                   0x00000000
-#define mmMC_VM_MX_L1_PERFCOUNTER3_CFG_DEFAULT                                   0x00000000
-#define mmMC_VM_MX_L1_PERFCOUNTER_RSLT_CNTL_DEFAULT                              0x04000000
-
-
-// addressBlock: mmhub_l1tlb_vml1prdec
-#define mmMC_VM_MX_L1_PERFCOUNTER_LO_DEFAULT                                     0x00000000
-#define mmMC_VM_MX_L1_PERFCOUNTER_HI_DEFAULT                                     0x00000000
-
-
-// addressBlock: mmhub_l1tlb_vmtlspfdec
-#define mmVM_L2_SAW_CNTL_DEFAULT                                                 0x0c0b8602
-#define mmVM_L2_SAW_CNTL2_DEFAULT                                                0x00000000
-#define mmVM_L2_SAW_CNTL3_DEFAULT                                                0x80100004
-#define mmVM_L2_SAW_CNTL4_DEFAULT                                                0x00000001
-#define mmVM_L2_SAW_CONTEXT0_CNTL_DEFAULT                                        0x00fffed8
-#define mmVM_L2_SAW_CONTEXT0_CNTL2_DEFAULT                                       0x00000000
-#define mmVM_L2_SAW_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                   0x00000000
-#define mmVM_L2_SAW_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                   0x00000000
-#define mmVM_L2_SAW_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_DEFAULT                  0x00000000
-#define mmVM_L2_SAW_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_DEFAULT                  0x00000000
-#define mmVM_L2_SAW_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_DEFAULT                    0x00000000
-#define mmVM_L2_SAW_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_DEFAULT                    0x00000000
-#define mmVM_L2_SAW_CONTEXTS_DISABLE_DEFAULT                                     0x00000000
-#define mmVM_L2_SAW_PIPES_BUSY_DEFAULT                                           0x00000000
-
-
-// addressBlock: mmhub_utcl2_atcl2dec
-#define mmATC_L2_CNTL_DEFAULT                                                    0x000001c9
-#define mmATC_L2_CNTL2_DEFAULT                                                   0x00000100
-#define mmATC_L2_CACHE_DATA0_DEFAULT                                             0x00000000
-#define mmATC_L2_CACHE_DATA1_DEFAULT                                             0x00000000
-#define mmATC_L2_CACHE_DATA2_DEFAULT                                             0x00000000
-#define mmATC_L2_CNTL3_DEFAULT                                                   0x000001f8
-#define mmATC_L2_STATUS_DEFAULT                                                  0x00000000
-#define mmATC_L2_STATUS2_DEFAULT                                                 0x00000000
-#define mmATC_L2_MISC_CG_DEFAULT                                                 0x00000200
-#define mmATC_L2_MEM_POWER_LS_DEFAULT                                            0x00000208
-#define mmATC_L2_CGTT_CLK_CTRL_DEFAULT                                           0x00000080
-
-
-// addressBlock: mmhub_utcl2_vml2pfdec
-#define mmVM_L2_CNTL_DEFAULT                                                     0x00080602
-#define mmVM_L2_CNTL2_DEFAULT                                                    0x00000000
-#define mmVM_L2_CNTL3_DEFAULT                                                    0x80100007
-#define mmVM_L2_STATUS_DEFAULT                                                   0x00000000
-#define mmVM_DUMMY_PAGE_FAULT_CNTL_DEFAULT                                       0x00000090
-#define mmVM_DUMMY_PAGE_FAULT_ADDR_LO32_DEFAULT                                  0x00000000
-#define mmVM_DUMMY_PAGE_FAULT_ADDR_HI32_DEFAULT                                  0x00000000
-#define mmVM_L2_PROTECTION_FAULT_CNTL_DEFAULT                                    0x3ffffffc
-#define mmVM_L2_PROTECTION_FAULT_CNTL2_DEFAULT                                   0x000a0000
-#define mmVM_L2_PROTECTION_FAULT_MM_CNTL3_DEFAULT                                0xffffffff
-#define mmVM_L2_PROTECTION_FAULT_MM_CNTL4_DEFAULT                                0xffffffff
-#define mmVM_L2_PROTECTION_FAULT_STATUS_DEFAULT                                  0x00000000
-#define mmVM_L2_PROTECTION_FAULT_ADDR_LO32_DEFAULT                               0x00000000
-#define mmVM_L2_PROTECTION_FAULT_ADDR_HI32_DEFAULT                               0x00000000
-#define mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_DEFAULT                       0x00000000
-#define mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_DEFAULT                       0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_DEFAULT                 0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32_DEFAULT                 0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32_DEFAULT                0x00000000
-#define mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32_DEFAULT                0x00000000
-#define mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32_DEFAULT                    0x00000000
-#define mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32_DEFAULT                    0x00000000
-#define mmVM_L2_CNTL4_DEFAULT                                                    0x000000c1
-#define mmVM_L2_MM_GROUP_RT_CLASSES_DEFAULT                                      0x00000000
-#define mmVM_L2_BANK_SELECT_RESERVED_CID_DEFAULT                                 0x00000000
-#define mmVM_L2_BANK_SELECT_RESERVED_CID2_DEFAULT                                0x00000000
-#define mmVM_L2_CACHE_PARITY_CNTL_DEFAULT                                        0x00000000
-#define mmVM_L2_CGTT_CLK_CTRL_DEFAULT                                            0x00000080
-
-
-// addressBlock: mmhub_utcl2_vml2vcdec
-#define mmVM_CONTEXT0_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT1_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT2_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT3_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT4_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT5_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT6_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT7_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT8_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT9_CNTL_DEFAULT                                               0x007ffe80
-#define mmVM_CONTEXT10_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT11_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT12_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT13_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT14_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXT15_CNTL_DEFAULT                                              0x007ffe80
-#define mmVM_CONTEXTS_DISABLE_DEFAULT                                            0x00000000
-#define mmVM_INVALIDATE_ENG0_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG1_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG2_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG3_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG4_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG5_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG6_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG7_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG8_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG9_SEM_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG10_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG11_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG12_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG13_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG14_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG15_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG16_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG17_SEM_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG0_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG1_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG2_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG3_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG4_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG5_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG6_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG7_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG8_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG9_REQ_DEFAULT                                         0x017c0000
-#define mmVM_INVALIDATE_ENG10_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG11_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG12_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG13_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG14_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG15_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG16_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG17_REQ_DEFAULT                                        0x017c0000
-#define mmVM_INVALIDATE_ENG0_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG1_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG2_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG3_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG4_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG5_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG6_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG7_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG8_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG9_ACK_DEFAULT                                         0x00000000
-#define mmVM_INVALIDATE_ENG10_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG11_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG12_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG13_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG14_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG15_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG16_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG17_ACK_DEFAULT                                        0x00000000
-#define mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG1_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG1_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG2_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG2_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG3_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG3_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG4_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG4_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG5_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG5_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG6_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG6_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG7_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG7_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG8_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG8_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG9_ADDR_RANGE_LO32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG9_ADDR_RANGE_HI32_DEFAULT                             0x00000000
-#define mmVM_INVALIDATE_ENG10_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG10_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG11_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG11_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG12_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG12_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG13_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG13_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG14_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG14_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG15_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG15_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG16_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG16_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG17_ADDR_RANGE_LO32_DEFAULT                            0x00000000
-#define mmVM_INVALIDATE_ENG17_ADDR_RANGE_HI32_DEFAULT                            0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_DEFAULT                         0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_DEFAULT                        0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_DEFAULT                           0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_DEFAULT                          0x00000000
-#define mmVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_DEFAULT                          0x00000000
-
-
-// addressBlock: mmhub_utcl2_vml2pldec
-#define mmMC_VM_L2_PERFCOUNTER0_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER1_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER2_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER3_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER4_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER5_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER6_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER7_CFG_DEFAULT                                      0x00000000
-#define mmMC_VM_L2_PERFCOUNTER_RSLT_CNTL_DEFAULT                                 0x04000000
-
-
-// addressBlock: mmhub_utcl2_vml2prdec
-#define mmMC_VM_L2_PERFCOUNTER_LO_DEFAULT                                        0x00000000
-#define mmMC_VM_L2_PERFCOUNTER_HI_DEFAULT                                        0x00000000
-
-
-// addressBlock: mmhub_utcl2_vmsharedhvdec
-#define mmMC_VM_FB_SIZE_OFFSET_VF0_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF1_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF2_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF3_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF4_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF5_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF6_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF7_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF8_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF9_DEFAULT                                       0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF10_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF11_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF12_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF13_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF14_DEFAULT                                      0x00000000
-#define mmMC_VM_FB_SIZE_OFFSET_VF15_DEFAULT                                      0x00000000
-#define mmVM_IOMMU_MMIO_CNTRL_1_DEFAULT                                          0x00000100
-#define mmMC_VM_MARC_BASE_LO_0_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_LO_1_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_LO_2_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_LO_3_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_0_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_1_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_2_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_BASE_HI_3_DEFAULT                                           0x00000000
-#define mmMC_VM_MARC_RELOC_LO_0_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_LO_1_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_LO_2_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_LO_3_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_0_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_1_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_2_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_RELOC_HI_3_DEFAULT                                          0x00000000
-#define mmMC_VM_MARC_LEN_LO_0_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_LO_1_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_LO_2_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_LO_3_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_0_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_1_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_2_DEFAULT                                            0x00000000
-#define mmMC_VM_MARC_LEN_HI_3_DEFAULT                                            0x00000000
-#define mmVM_IOMMU_CONTROL_REGISTER_DEFAULT                                      0x00000000
-#define mmVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER_DEFAULT             0x00000000
-#define mmVM_PCIE_ATS_CNTL_DEFAULT                                               0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_0_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_1_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_2_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_3_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_4_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_5_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_6_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_7_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_8_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_9_DEFAULT                                          0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_10_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_11_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_12_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_13_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_14_DEFAULT                                         0x00000000
-#define mmVM_PCIE_ATS_CNTL_VF_15_DEFAULT                                         0x00000000
-#define mmUTCL2_CGTT_CLK_CTRL_DEFAULT                                            0x00000080
-
-
-// addressBlock: mmhub_utcl2_vmsharedpfdec
-#define mmMC_VM_NB_MMIOBASE_DEFAULT                                              0x00000000
-#define mmMC_VM_NB_MMIOLIMIT_DEFAULT                                             0x00000000
-#define mmMC_VM_NB_PCI_CTRL_DEFAULT                                              0x00000000
-#define mmMC_VM_NB_PCI_ARB_DEFAULT                                               0x00000008
-#define mmMC_VM_NB_TOP_OF_DRAM_SLOT1_DEFAULT                                     0x00000000
-#define mmMC_VM_NB_LOWER_TOP_OF_DRAM2_DEFAULT                                    0x00000000
-#define mmMC_VM_NB_UPPER_TOP_OF_DRAM2_DEFAULT                                    0x00000000
-#define mmMC_VM_FB_OFFSET_DEFAULT                                                0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_DEFAULT                         0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_DEFAULT                         0x00000000
-#define mmMC_VM_STEERING_DEFAULT                                                 0x00000001
-#define mmMC_SHARED_VIRT_RESET_REQ_DEFAULT                                       0x00000000
-#define mmMC_MEM_POWER_LS_DEFAULT                                                0x00000208
-#define mmMC_VM_CACHEABLE_DRAM_ADDRESS_START_DEFAULT                             0x00000000
-#define mmMC_VM_CACHEABLE_DRAM_ADDRESS_END_DEFAULT                               0x00000000
-#define mmMC_VM_APT_CNTL_DEFAULT                                                 0x00000000
-#define mmMC_VM_LOCAL_HBM_ADDRESS_START_DEFAULT                                  0x00000000
-#define mmMC_VM_LOCAL_HBM_ADDRESS_END_DEFAULT                                    0x000fffff
-#define mmMC_VM_LOCAL_HBM_ADDRESS_LOCK_CNTL_DEFAULT                              0x00000000
-
-
-// addressBlock: mmhub_utcl2_vmsharedvcdec
-#define mmMC_VM_FB_LOCATION_BASE_DEFAULT                                         0x00000000
-#define mmMC_VM_FB_LOCATION_TOP_DEFAULT                                          0x00000000
-#define mmMC_VM_AGP_TOP_DEFAULT                                                  0x00000000
-#define mmMC_VM_AGP_BOT_DEFAULT                                                  0x00000000
-#define mmMC_VM_AGP_BASE_DEFAULT                                                 0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_LOW_ADDR_DEFAULT                                 0x00000000
-#define mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR_DEFAULT                                0x00000000
-#define mmMC_VM_MX_L1_TLB_CNTL_DEFAULT                                           0x00002501
-
-
-// addressBlock: mmhub_utcl2_atcl2pfcntrdec
-#define mmATC_L2_PERFCOUNTER_LO_DEFAULT                                          0x00000000
-#define mmATC_L2_PERFCOUNTER_HI_DEFAULT                                          0x00000000
-
-
-// addressBlock: mmhub_utcl2_atcl2pfcntldec
-#define mmATC_L2_PERFCOUNTER0_CFG_DEFAULT                                        0x00000000
-#define mmATC_L2_PERFCOUNTER1_CFG_DEFAULT                                        0x00000000
-#define mmATC_L2_PERFCOUNTER_RSLT_CNTL_DEFAULT                                   0x04000000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_sh_mask.h
deleted file mode 100644
index 1445bba8f41f..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_sh_mask.h
+++ /dev/null
@@ -1,1658 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _sdma0_4_1_SH_MASK_HEADER
-#define _sdma0_4_1_SH_MASK_HEADER
-
-
-// addressBlock: sdma0_sdma0dec
-//SDMA0_UCODE_ADDR
-#define SDMA0_UCODE_ADDR__VALUE__SHIFT                                                                        0x0
-#define SDMA0_UCODE_ADDR__VALUE_MASK                                                                          0x00001FFFL
-//SDMA0_UCODE_DATA
-#define SDMA0_UCODE_DATA__VALUE__SHIFT                                                                        0x0
-#define SDMA0_UCODE_DATA__VALUE_MASK                                                                          0xFFFFFFFFL
-//SDMA0_VM_CNTL
-#define SDMA0_VM_CNTL__CMD__SHIFT                                                                             0x0
-#define SDMA0_VM_CNTL__CMD_MASK                                                                               0x0000000FL
-//SDMA0_VM_CTX_LO
-#define SDMA0_VM_CTX_LO__ADDR__SHIFT                                                                          0x2
-#define SDMA0_VM_CTX_LO__ADDR_MASK                                                                            0xFFFFFFFCL
-//SDMA0_VM_CTX_HI
-#define SDMA0_VM_CTX_HI__ADDR__SHIFT                                                                          0x0
-#define SDMA0_VM_CTX_HI__ADDR_MASK                                                                            0xFFFFFFFFL
-//SDMA0_ACTIVE_FCN_ID
-#define SDMA0_ACTIVE_FCN_ID__VFID__SHIFT                                                                      0x0
-#define SDMA0_ACTIVE_FCN_ID__RESERVED__SHIFT                                                                  0x4
-#define SDMA0_ACTIVE_FCN_ID__VF__SHIFT                                                                        0x1f
-#define SDMA0_ACTIVE_FCN_ID__VFID_MASK                                                                        0x0000000FL
-#define SDMA0_ACTIVE_FCN_ID__RESERVED_MASK                                                                    0x7FFFFFF0L
-#define SDMA0_ACTIVE_FCN_ID__VF_MASK                                                                          0x80000000L
-//SDMA0_VM_CTX_CNTL
-#define SDMA0_VM_CTX_CNTL__PRIV__SHIFT                                                                        0x0
-#define SDMA0_VM_CTX_CNTL__VMID__SHIFT                                                                        0x4
-#define SDMA0_VM_CTX_CNTL__PRIV_MASK                                                                          0x00000001L
-#define SDMA0_VM_CTX_CNTL__VMID_MASK                                                                          0x000000F0L
-//SDMA0_VIRT_RESET_REQ
-#define SDMA0_VIRT_RESET_REQ__VF__SHIFT                                                                       0x0
-#define SDMA0_VIRT_RESET_REQ__PF__SHIFT                                                                       0x1f
-#define SDMA0_VIRT_RESET_REQ__VF_MASK                                                                         0x0000FFFFL
-#define SDMA0_VIRT_RESET_REQ__PF_MASK                                                                         0x80000000L
-//SDMA0_CONTEXT_REG_TYPE0
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_CNTL__SHIFT                                                     0x0
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE__SHIFT                                                     0x1
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_HI__SHIFT                                                  0x2
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR__SHIFT                                                     0x3
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_HI__SHIFT                                                  0x4
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR__SHIFT                                                     0x5
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_HI__SHIFT                                                  0x6
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_POLL_CNTL__SHIFT                                           0x7
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_HI__SHIFT                                             0x8
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_LO__SHIFT                                             0x9
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_CNTL__SHIFT                                                     0xa
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_RPTR__SHIFT                                                     0xb
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_OFFSET__SHIFT                                                   0xc
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_LO__SHIFT                                                  0xd
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_HI__SHIFT                                                  0xe
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_SIZE__SHIFT                                                     0xf
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_SKIP_CNTL__SHIFT                                                   0x10
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_STATUS__SHIFT                                              0x11
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_DOORBELL__SHIFT                                                    0x12
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_CNTL__SHIFT                                                0x13
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_CNTL_MASK                                                       0x00000001L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_MASK                                                       0x00000002L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_HI_MASK                                                    0x00000004L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_MASK                                                       0x00000008L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_HI_MASK                                                    0x00000010L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_MASK                                                       0x00000020L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_HI_MASK                                                    0x00000040L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_POLL_CNTL_MASK                                             0x00000080L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_HI_MASK                                               0x00000100L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_LO_MASK                                               0x00000200L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_CNTL_MASK                                                       0x00000400L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_RPTR_MASK                                                       0x00000800L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_OFFSET_MASK                                                     0x00001000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_LO_MASK                                                    0x00002000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_HI_MASK                                                    0x00004000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_SIZE_MASK                                                       0x00008000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_SKIP_CNTL_MASK                                                     0x00010000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_STATUS_MASK                                                0x00020000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_DOORBELL_MASK                                                      0x00040000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_CNTL_MASK                                                  0x00080000L
-//SDMA0_CONTEXT_REG_TYPE1
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_STATUS__SHIFT                                                      0x8
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_LOG__SHIFT                                                0x9
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_WATERMARK__SHIFT                                                   0xa
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_OFFSET__SHIFT                                             0xb
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_LO__SHIFT                                                 0xc
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_HI__SHIFT                                                 0xd
-#define SDMA0_CONTEXT_REG_TYPE1__VOID_REG2__SHIFT                                                             0xe
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_IB_SUB_REMAIN__SHIFT                                               0xf
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_PREEMPT__SHIFT                                                     0x10
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DUMMY_REG__SHIFT                                                   0x11
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__SHIFT                                        0x12
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__SHIFT                                        0x13
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_AQL_CNTL__SHIFT                                                 0x14
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_MINOR_PTR_UPDATE__SHIFT                                            0x15
-#define SDMA0_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                              0x16
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_STATUS_MASK                                                        0x00000100L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_LOG_MASK                                                  0x00000200L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_WATERMARK_MASK                                                     0x00000400L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_OFFSET_MASK                                               0x00000800L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_LO_MASK                                                   0x00001000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_HI_MASK                                                   0x00002000L
-#define SDMA0_CONTEXT_REG_TYPE1__VOID_REG2_MASK                                                               0x00004000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_IB_SUB_REMAIN_MASK                                                 0x00008000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_PREEMPT_MASK                                                       0x00010000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DUMMY_REG_MASK                                                     0x00020000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_HI_MASK                                          0x00040000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_LO_MASK                                          0x00080000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_AQL_CNTL_MASK                                                   0x00100000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_MINOR_PTR_UPDATE_MASK                                              0x00200000L
-#define SDMA0_CONTEXT_REG_TYPE1__RESERVED_MASK                                                                0xFFC00000L
-//SDMA0_CONTEXT_REG_TYPE2
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA0__SHIFT                                                0x0
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA1__SHIFT                                                0x1
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA2__SHIFT                                                0x2
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA3__SHIFT                                                0x3
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA4__SHIFT                                                0x4
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA5__SHIFT                                                0x5
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA6__SHIFT                                                0x6
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA7__SHIFT                                                0x7
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA8__SHIFT                                                0x8
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_CNTL__SHIFT                                                 0x9
-#define SDMA0_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                              0xa
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA0_MASK                                                  0x00000001L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA1_MASK                                                  0x00000002L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA2_MASK                                                  0x00000004L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA3_MASK                                                  0x00000008L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA4_MASK                                                  0x00000010L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA5_MASK                                                  0x00000020L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA6_MASK                                                  0x00000040L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA7_MASK                                                  0x00000080L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA8_MASK                                                  0x00000100L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_CNTL_MASK                                                   0x00000200L
-#define SDMA0_CONTEXT_REG_TYPE2__RESERVED_MASK                                                                0xFFFFFC00L
-//SDMA0_CONTEXT_REG_TYPE3
-#define SDMA0_CONTEXT_REG_TYPE3__RESERVED__SHIFT                                                              0x0
-#define SDMA0_CONTEXT_REG_TYPE3__RESERVED_MASK                                                                0xFFFFFFFFL
-//SDMA0_PUB_REG_TYPE0
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_ADDR__SHIFT                                                          0x0
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_DATA__SHIFT                                                          0x1
-#define SDMA0_PUB_REG_TYPE0__RESERVED3__SHIFT                                                                 0x3
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CNTL__SHIFT                                                             0x4
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_LO__SHIFT                                                           0x5
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_HI__SHIFT                                                           0x6
-#define SDMA0_PUB_REG_TYPE0__SDMA0_ACTIVE_FCN_ID__SHIFT                                                       0x7
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_CNTL__SHIFT                                                         0x8
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VIRT_RESET_REQ__SHIFT                                                      0x9
-#define SDMA0_PUB_REG_TYPE0__RESERVED10__SHIFT                                                                0xa
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE0__SHIFT                                                   0xb
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE1__SHIFT                                                   0xc
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE2__SHIFT                                                   0xd
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE3__SHIFT                                                   0xe
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE0__SHIFT                                                       0xf
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE1__SHIFT                                                       0x10
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE2__SHIFT                                                       0x11
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE3__SHIFT                                                       0x12
-#define SDMA0_PUB_REG_TYPE0__SDMA0_MMHUB_CNTL__SHIFT                                                          0x13
-#define SDMA0_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY__SHIFT                                           0x14
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_GROUP_BOUNDARY__SHIFT                                              0x19
-#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL__SHIFT                                                          0x1a
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CLK_CTRL__SHIFT                                                            0x1b
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL__SHIFT                                                                0x1c
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS__SHIFT                                                        0x1d
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG__SHIFT                                                      0x1e
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ__SHIFT                                                 0x1f
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_ADDR_MASK                                                            0x00000001L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_DATA_MASK                                                            0x00000002L
-#define SDMA0_PUB_REG_TYPE0__RESERVED3_MASK                                                                   0x00000008L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CNTL_MASK                                                               0x00000010L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_LO_MASK                                                             0x00000020L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_HI_MASK                                                             0x00000040L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_ACTIVE_FCN_ID_MASK                                                         0x00000080L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_CNTL_MASK                                                           0x00000100L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VIRT_RESET_REQ_MASK                                                        0x00000200L
-#define SDMA0_PUB_REG_TYPE0__RESERVED10_MASK                                                                  0x00000400L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE0_MASK                                                     0x00000800L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE1_MASK                                                     0x00001000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE2_MASK                                                     0x00002000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE3_MASK                                                     0x00004000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE0_MASK                                                         0x00008000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE1_MASK                                                         0x00010000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE2_MASK                                                         0x00020000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE3_MASK                                                         0x00040000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_MMHUB_CNTL_MASK                                                            0x00080000L
-#define SDMA0_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY_MASK                                             0x01F00000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_GROUP_BOUNDARY_MASK                                                0x02000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL_MASK                                                            0x04000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CLK_CTRL_MASK                                                              0x08000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL_MASK                                                                  0x10000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS_MASK                                                          0x20000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_MASK                                                        0x40000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ_MASK                                                   0x80000000L
-//SDMA0_PUB_REG_TYPE1
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI__SHIFT                                                    0x0
-#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT                                            0x1
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH__SHIFT                                                       0x2
-#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH__SHIFT                                                     0x3
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM__SHIFT                                                             0x4
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG__SHIFT                                                          0x5
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG__SHIFT                                                         0x6
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RD_BURST_CNTL__SHIFT                                                       0x7
-#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG__SHIFT                                                     0x8
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM__SHIFT                                                      0x9
-#define SDMA0_PUB_REG_TYPE1__SDMA0_F32_CNTL__SHIFT                                                            0xa
-#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE__SHIFT                                                              0xb
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE0_QUANTUM__SHIFT                                                      0xc
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE1_QUANTUM__SHIFT                                                      0xd
-#define SDMA0_PUB_REG_TYPE1__SDMA_POWER_GATING__SHIFT                                                         0xe
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG__SHIFT                                                         0xf
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_WRITE__SHIFT                                                          0x10
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_READ__SHIFT                                                           0x11
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG__SHIFT                                                          0x12
-#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD__SHIFT                                                        0x13
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ID__SHIFT                                                                  0x14
-#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION__SHIFT                                                             0x15
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER__SHIFT                                                         0x16
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR__SHIFT                                                   0x17
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG__SHIFT                                                         0x18
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL__SHIFT                                                         0x19
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO__SHIFT                                                     0x1a
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI__SHIFT                                                     0x1b
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL__SHIFT                                                          0x1c
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK__SHIFT                                                       0x1d
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_RD_STATUS__SHIFT                                                     0x1e
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WR_STATUS__SHIFT                                                     0x1f
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI_MASK                                                      0x00000001L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL_MASK                                              0x00000002L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_MASK                                                         0x00000004L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH_MASK                                                       0x00000008L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM_MASK                                                               0x00000010L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG_MASK                                                            0x00000020L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG_MASK                                                           0x00000040L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RD_BURST_CNTL_MASK                                                         0x00000080L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG_MASK                                                       0x00000100L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM_MASK                                                        0x00000200L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_F32_CNTL_MASK                                                              0x00000400L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE_MASK                                                                0x00000800L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE0_QUANTUM_MASK                                                        0x00001000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE1_QUANTUM_MASK                                                        0x00002000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_POWER_GATING_MASK                                                           0x00004000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG_MASK                                                           0x00008000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_WRITE_MASK                                                            0x00010000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_READ_MASK                                                             0x00020000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG_MASK                                                            0x00040000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD_MASK                                                          0x00080000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ID_MASK                                                                    0x00100000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION_MASK                                                               0x00200000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_MASK                                                           0x00400000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR_MASK                                                     0x00800000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG_MASK                                                           0x01000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL_MASK                                                           0x02000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO_MASK                                                       0x04000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI_MASK                                                       0x08000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL_MASK                                                            0x10000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK_MASK                                                         0x20000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_RD_STATUS_MASK                                                       0x40000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WR_STATUS_MASK                                                       0x80000000L
-//SDMA0_PUB_REG_TYPE2
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0__SHIFT                                                          0x0
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1__SHIFT                                                          0x1
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2__SHIFT                                                          0x2
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0__SHIFT                                                     0x3
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1__SHIFT                                                     0x4
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0__SHIFT                                                     0x5
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1__SHIFT                                                     0x6
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_TIMEOUT__SHIFT                                                       0x7
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_PAGE__SHIFT                                                          0x8
-#define SDMA0_PUB_REG_TYPE2__SDMA0_POWER_CNTL_IDLE__SHIFT                                                     0x9
-#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT__SHIFT                                                  0xa
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2__SHIFT                                                      0xb
-#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG__SHIFT                                                         0xc
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO__SHIFT                                                    0xd
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI__SHIFT                                                    0xe
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG__SHIFT                                                           0x10
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0__SHIFT                                                      0x11
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1__SHIFT                                                      0x12
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2__SHIFT                                                      0x13
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3__SHIFT                                                      0x14
-#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER__SHIFT                                                         0x15
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UNBREAKABLE__SHIFT                                                         0x16
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFMON_CNTL__SHIFT                                                        0x17
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER0_RESULT__SHIFT                                                 0x18
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER1_RESULT__SHIFT                                                 0x19
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SHIFT                                         0x1a
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL__SHIFT                                                            0x1b
-#define SDMA0_PUB_REG_TYPE2__SDMA0_MMHUB_TRUSTLVL__SHIFT                                                      0x1c
-#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG__SHIFT                                               0x1d
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ULV_CNTL__SHIFT                                                            0x1e
-#define SDMA0_PUB_REG_TYPE2__RESERVED__SHIFT                                                                  0x1f
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0_MASK                                                            0x00000001L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1_MASK                                                            0x00000002L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2_MASK                                                            0x00000004L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0_MASK                                                       0x00000008L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1_MASK                                                       0x00000010L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0_MASK                                                       0x00000020L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1_MASK                                                       0x00000040L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_TIMEOUT_MASK                                                         0x00000080L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_PAGE_MASK                                                            0x00000100L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_POWER_CNTL_IDLE_MASK                                                       0x00000200L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT_MASK                                                    0x00000400L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2_MASK                                                        0x00000800L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG_MASK                                                           0x00001000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO_MASK                                                      0x00002000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI_MASK                                                      0x00004000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG_MASK                                                             0x00010000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0_MASK                                                        0x00020000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1_MASK                                                        0x00040000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2_MASK                                                        0x00080000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3_MASK                                                        0x00100000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER_MASK                                                           0x00200000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UNBREAKABLE_MASK                                                           0x00400000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFMON_CNTL_MASK                                                          0x00800000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER0_RESULT_MASK                                                   0x01000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER1_RESULT_MASK                                                   0x02000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER_TAG_DELAY_RANGE_MASK                                           0x04000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL_MASK                                                              0x08000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_MMHUB_TRUSTLVL_MASK                                                        0x10000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG_MASK                                                 0x20000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ULV_CNTL_MASK                                                              0x40000000L
-#define SDMA0_PUB_REG_TYPE2__RESERVED_MASK                                                                    0x80000000L
-//SDMA0_PUB_REG_TYPE3
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA__SHIFT                                                   0x0
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX__SHIFT                                                  0x1
-#define SDMA0_PUB_REG_TYPE3__RESERVED__SHIFT                                                                  0x2
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA_MASK                                                     0x00000001L
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX_MASK                                                    0x00000002L
-#define SDMA0_PUB_REG_TYPE3__RESERVED_MASK                                                                    0xFFFFFFFCL
-//SDMA0_MMHUB_CNTL
-#define SDMA0_MMHUB_CNTL__UNIT_ID__SHIFT                                                                      0x0
-#define SDMA0_MMHUB_CNTL__UNIT_ID_MASK                                                                        0x0000003FL
-//SDMA0_CONTEXT_GROUP_BOUNDARY
-#define SDMA0_CONTEXT_GROUP_BOUNDARY__RESERVED__SHIFT                                                         0x0
-#define SDMA0_CONTEXT_GROUP_BOUNDARY__RESERVED_MASK                                                           0xFFFFFFFFL
-//SDMA0_POWER_CNTL
-#define SDMA0_POWER_CNTL__PG_CNTL_ENABLE__SHIFT                                                               0x0
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_ON_REQ__SHIFT                                                          0x1
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_OFF_REQ__SHIFT                                                         0x2
-#define SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME__SHIFT                                                   0x3
-#define SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE__SHIFT                                                           0x8
-#define SDMA0_POWER_CNTL__MEM_POWER_LS_EN__SHIFT                                                              0x9
-#define SDMA0_POWER_CNTL__MEM_POWER_DS_EN__SHIFT                                                              0xa
-#define SDMA0_POWER_CNTL__MEM_POWER_SD_EN__SHIFT                                                              0xb
-#define SDMA0_POWER_CNTL__MEM_POWER_DELAY__SHIFT                                                              0xc
-#define SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME__SHIFT                                                  0x1a
-#define SDMA0_POWER_CNTL__PG_CNTL_ENABLE_MASK                                                                 0x00000001L
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_ON_REQ_MASK                                                            0x00000002L
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_OFF_REQ_MASK                                                           0x00000004L
-#define SDMA0_POWER_CNTL__ON_OFF_CONDITION_HOLD_TIME_MASK                                                     0x000000F8L
-#define SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK                                                             0x00000100L
-#define SDMA0_POWER_CNTL__MEM_POWER_LS_EN_MASK                                                                0x00000200L
-#define SDMA0_POWER_CNTL__MEM_POWER_DS_EN_MASK                                                                0x00000400L
-#define SDMA0_POWER_CNTL__MEM_POWER_SD_EN_MASK                                                                0x00000800L
-#define SDMA0_POWER_CNTL__MEM_POWER_DELAY_MASK                                                                0x003FF000L
-#define SDMA0_POWER_CNTL__ON_OFF_STATUS_DURATION_TIME_MASK                                                    0xFC000000L
-//SDMA0_CLK_CTRL
-#define SDMA0_CLK_CTRL__ON_DELAY__SHIFT                                                                       0x0
-#define SDMA0_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                                 0x4
-#define SDMA0_CLK_CTRL__RESERVED__SHIFT                                                                       0xc
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                                 0x18
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                                 0x19
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                                 0x1a
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                                 0x1b
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                                 0x1c
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                                 0x1d
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                                 0x1e
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                                 0x1f
-#define SDMA0_CLK_CTRL__ON_DELAY_MASK                                                                         0x0000000FL
-#define SDMA0_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                   0x00000FF0L
-#define SDMA0_CLK_CTRL__RESERVED_MASK                                                                         0x00FFF000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                   0x01000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                   0x02000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                   0x04000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                   0x08000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                   0x10000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                   0x20000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                                   0x40000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                                   0x80000000L
-//SDMA0_CNTL
-#define SDMA0_CNTL__TRAP_ENABLE__SHIFT                                                                        0x0
-#define SDMA0_CNTL__UTC_L1_ENABLE__SHIFT                                                                      0x1
-#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE__SHIFT                                                                0x2
-#define SDMA0_CNTL__DATA_SWAP_ENABLE__SHIFT                                                                   0x3
-#define SDMA0_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                                  0x4
-#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                              0x5
-#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                          0x11
-#define SDMA0_CNTL__AUTO_CTXSW_ENABLE__SHIFT                                                                  0x12
-#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                                0x1c
-#define SDMA0_CNTL__FROZEN_INT_ENABLE__SHIFT                                                                  0x1d
-#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                              0x1e
-#define SDMA0_CNTL__TRAP_ENABLE_MASK                                                                          0x00000001L
-#define SDMA0_CNTL__UTC_L1_ENABLE_MASK                                                                        0x00000002L
-#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE_MASK                                                                  0x00000004L
-#define SDMA0_CNTL__DATA_SWAP_ENABLE_MASK                                                                     0x00000008L
-#define SDMA0_CNTL__FENCE_SWAP_ENABLE_MASK                                                                    0x00000010L
-#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                                0x00000020L
-#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                            0x00020000L
-#define SDMA0_CNTL__AUTO_CTXSW_ENABLE_MASK                                                                    0x00040000L
-#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                                  0x10000000L
-#define SDMA0_CNTL__FROZEN_INT_ENABLE_MASK                                                                    0x20000000L
-#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                                0x40000000L
-//SDMA0_CHICKEN_BITS
-#define SDMA0_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE__SHIFT                                                     0x0
-#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT                                                 0x1
-#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT                                        0x2
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_LENGTH__SHIFT                                                         0x8
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE__SHIFT                                                     0xa
-#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT                                                        0x10
-#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT                                                           0x11
-#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                         0x14
-#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                           0x17
-#define SDMA0_CHICKEN_BITS__TIME_BASED_QOS__SHIFT                                                             0x19
-#define SDMA0_CHICKEN_BITS__CE_AFIFO_WATERMARK__SHIFT                                                         0x1a
-#define SDMA0_CHICKEN_BITS__CE_DFIFO_WATERMARK__SHIFT                                                         0x1c
-#define SDMA0_CHICKEN_BITS__CE_LFIFO_WATERMARK__SHIFT                                                         0x1e
-#define SDMA0_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE_MASK                                                       0x00000001L
-#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK                                                   0x00000002L
-#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK                                          0x00000004L
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_LENGTH_MASK                                                           0x00000300L
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE_MASK                                                       0x00001C00L
-#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK                                                          0x00010000L
-#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK                                                             0x00020000L
-#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                           0x00100000L
-#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                             0x00800000L
-#define SDMA0_CHICKEN_BITS__TIME_BASED_QOS_MASK                                                               0x02000000L
-#define SDMA0_CHICKEN_BITS__CE_AFIFO_WATERMARK_MASK                                                           0x0C000000L
-#define SDMA0_CHICKEN_BITS__CE_DFIFO_WATERMARK_MASK                                                           0x30000000L
-#define SDMA0_CHICKEN_BITS__CE_LFIFO_WATERMARK_MASK                                                           0xC0000000L
-//SDMA0_GB_ADDR_CONFIG
-#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                0x0
-#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                     0x3
-#define SDMA0_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT                                                     0x8
-#define SDMA0_GB_ADDR_CONFIG__NUM_BANKS__SHIFT                                                                0xc
-#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                       0x13
-#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                  0x00000007L
-#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                       0x00000038L
-#define SDMA0_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK                                                       0x00000700L
-#define SDMA0_GB_ADDR_CONFIG__NUM_BANKS_MASK                                                                  0x00007000L
-#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                         0x00180000L
-//SDMA0_GB_ADDR_CONFIG_READ
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                           0x0
-#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                0x3
-#define SDMA0_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT                                                0x8
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT                                                           0xc
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                  0x13
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                             0x00000007L
-#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                  0x00000038L
-#define SDMA0_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK                                                  0x00000700L
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_BANKS_MASK                                                             0x00007000L
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                    0x00180000L
-//SDMA0_RB_RPTR_FETCH_HI
-#define SDMA0_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                                 0x0
-#define SDMA0_RB_RPTR_FETCH_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
-//SDMA0_SEM_WAIT_FAIL_TIMER_CNTL
-#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT                                                          0x0
-#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK                                                            0xFFFFFFFFL
-//SDMA0_RB_RPTR_FETCH
-#define SDMA0_RB_RPTR_FETCH__OFFSET__SHIFT                                                                    0x2
-#define SDMA0_RB_RPTR_FETCH__OFFSET_MASK                                                                      0xFFFFFFFCL
-//SDMA0_IB_OFFSET_FETCH
-#define SDMA0_IB_OFFSET_FETCH__OFFSET__SHIFT                                                                  0x2
-#define SDMA0_IB_OFFSET_FETCH__OFFSET_MASK                                                                    0x003FFFFCL
-//SDMA0_PROGRAM
-#define SDMA0_PROGRAM__STREAM__SHIFT                                                                          0x0
-#define SDMA0_PROGRAM__STREAM_MASK                                                                            0xFFFFFFFFL
-//SDMA0_STATUS_REG
-#define SDMA0_STATUS_REG__IDLE__SHIFT                                                                         0x0
-#define SDMA0_STATUS_REG__REG_IDLE__SHIFT                                                                     0x1
-#define SDMA0_STATUS_REG__RB_EMPTY__SHIFT                                                                     0x2
-#define SDMA0_STATUS_REG__RB_FULL__SHIFT                                                                      0x3
-#define SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT                                                                  0x4
-#define SDMA0_STATUS_REG__RB_CMD_FULL__SHIFT                                                                  0x5
-#define SDMA0_STATUS_REG__IB_CMD_IDLE__SHIFT                                                                  0x6
-#define SDMA0_STATUS_REG__IB_CMD_FULL__SHIFT                                                                  0x7
-#define SDMA0_STATUS_REG__BLOCK_IDLE__SHIFT                                                                   0x8
-#define SDMA0_STATUS_REG__INSIDE_IB__SHIFT                                                                    0x9
-#define SDMA0_STATUS_REG__EX_IDLE__SHIFT                                                                      0xa
-#define SDMA0_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE__SHIFT                                                    0xb
-#define SDMA0_STATUS_REG__PACKET_READY__SHIFT                                                                 0xc
-#define SDMA0_STATUS_REG__MC_WR_IDLE__SHIFT                                                                   0xd
-#define SDMA0_STATUS_REG__SRBM_IDLE__SHIFT                                                                    0xe
-#define SDMA0_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                                0xf
-#define SDMA0_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                              0x10
-#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                              0x11
-#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                              0x12
-#define SDMA0_STATUS_REG__MC_RD_IDLE__SHIFT                                                                   0x13
-#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                             0x14
-#define SDMA0_STATUS_REG__MC_RD_RET_STALL__SHIFT                                                              0x15
-#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT                                                           0x16
-#define SDMA0_STATUS_REG__PREV_CMD_IDLE__SHIFT                                                                0x19
-#define SDMA0_STATUS_REG__SEM_IDLE__SHIFT                                                                     0x1a
-#define SDMA0_STATUS_REG__SEM_REQ_STALL__SHIFT                                                                0x1b
-#define SDMA0_STATUS_REG__SEM_RESP_STATE__SHIFT                                                               0x1c
-#define SDMA0_STATUS_REG__INT_IDLE__SHIFT                                                                     0x1e
-#define SDMA0_STATUS_REG__INT_REQ_STALL__SHIFT                                                                0x1f
-#define SDMA0_STATUS_REG__IDLE_MASK                                                                           0x00000001L
-#define SDMA0_STATUS_REG__REG_IDLE_MASK                                                                       0x00000002L
-#define SDMA0_STATUS_REG__RB_EMPTY_MASK                                                                       0x00000004L
-#define SDMA0_STATUS_REG__RB_FULL_MASK                                                                        0x00000008L
-#define SDMA0_STATUS_REG__RB_CMD_IDLE_MASK                                                                    0x00000010L
-#define SDMA0_STATUS_REG__RB_CMD_FULL_MASK                                                                    0x00000020L
-#define SDMA0_STATUS_REG__IB_CMD_IDLE_MASK                                                                    0x00000040L
-#define SDMA0_STATUS_REG__IB_CMD_FULL_MASK                                                                    0x00000080L
-#define SDMA0_STATUS_REG__BLOCK_IDLE_MASK                                                                     0x00000100L
-#define SDMA0_STATUS_REG__INSIDE_IB_MASK                                                                      0x00000200L
-#define SDMA0_STATUS_REG__EX_IDLE_MASK                                                                        0x00000400L
-#define SDMA0_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE_MASK                                                      0x00000800L
-#define SDMA0_STATUS_REG__PACKET_READY_MASK                                                                   0x00001000L
-#define SDMA0_STATUS_REG__MC_WR_IDLE_MASK                                                                     0x00002000L
-#define SDMA0_STATUS_REG__SRBM_IDLE_MASK                                                                      0x00004000L
-#define SDMA0_STATUS_REG__CONTEXT_EMPTY_MASK                                                                  0x00008000L
-#define SDMA0_STATUS_REG__DELTA_RPTR_FULL_MASK                                                                0x00010000L
-#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                                0x00020000L
-#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                                0x00040000L
-#define SDMA0_STATUS_REG__MC_RD_IDLE_MASK                                                                     0x00080000L
-#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                               0x00100000L
-#define SDMA0_STATUS_REG__MC_RD_RET_STALL_MASK                                                                0x00200000L
-#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK                                                             0x00400000L
-#define SDMA0_STATUS_REG__PREV_CMD_IDLE_MASK                                                                  0x02000000L
-#define SDMA0_STATUS_REG__SEM_IDLE_MASK                                                                       0x04000000L
-#define SDMA0_STATUS_REG__SEM_REQ_STALL_MASK                                                                  0x08000000L
-#define SDMA0_STATUS_REG__SEM_RESP_STATE_MASK                                                                 0x30000000L
-#define SDMA0_STATUS_REG__INT_IDLE_MASK                                                                       0x40000000L
-#define SDMA0_STATUS_REG__INT_REQ_STALL_MASK                                                                  0x80000000L
-//SDMA0_STATUS1_REG
-#define SDMA0_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                                0x0
-#define SDMA0_STATUS1_REG__CE_WR_IDLE__SHIFT                                                                  0x1
-#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE__SHIFT                                                               0x2
-#define SDMA0_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                                0x3
-#define SDMA0_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                                 0x4
-#define SDMA0_STATUS1_REG__CE_IN_IDLE__SHIFT                                                                  0x5
-#define SDMA0_STATUS1_REG__CE_DST_IDLE__SHIFT                                                                 0x6
-#define SDMA0_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                                 0x9
-#define SDMA0_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                               0xa
-#define SDMA0_STATUS1_REG__CE_INFO_FULL__SHIFT                                                                0xd
-#define SDMA0_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                               0xe
-#define SDMA0_STATUS1_REG__EX_START__SHIFT                                                                    0xf
-#define SDMA0_STATUS1_REG__CE_RD_STALL__SHIFT                                                                 0x11
-#define SDMA0_STATUS1_REG__CE_WR_STALL__SHIFT                                                                 0x12
-#define SDMA0_STATUS1_REG__CE_WREQ_IDLE_MASK                                                                  0x00000001L
-#define SDMA0_STATUS1_REG__CE_WR_IDLE_MASK                                                                    0x00000002L
-#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE_MASK                                                                 0x00000004L
-#define SDMA0_STATUS1_REG__CE_RREQ_IDLE_MASK                                                                  0x00000008L
-#define SDMA0_STATUS1_REG__CE_OUT_IDLE_MASK                                                                   0x00000010L
-#define SDMA0_STATUS1_REG__CE_IN_IDLE_MASK                                                                    0x00000020L
-#define SDMA0_STATUS1_REG__CE_DST_IDLE_MASK                                                                   0x00000040L
-#define SDMA0_STATUS1_REG__CE_CMD_IDLE_MASK                                                                   0x00000200L
-#define SDMA0_STATUS1_REG__CE_AFIFO_FULL_MASK                                                                 0x00000400L
-#define SDMA0_STATUS1_REG__CE_INFO_FULL_MASK                                                                  0x00002000L
-#define SDMA0_STATUS1_REG__CE_INFO1_FULL_MASK                                                                 0x00004000L
-#define SDMA0_STATUS1_REG__EX_START_MASK                                                                      0x00008000L
-#define SDMA0_STATUS1_REG__CE_RD_STALL_MASK                                                                   0x00020000L
-#define SDMA0_STATUS1_REG__CE_WR_STALL_MASK                                                                   0x00040000L
-//SDMA0_RD_BURST_CNTL
-#define SDMA0_RD_BURST_CNTL__RD_BURST__SHIFT                                                                  0x0
-#define SDMA0_RD_BURST_CNTL__RD_BURST_MASK                                                                    0x00000003L
-//SDMA0_HBM_PAGE_CONFIG
-#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                      0x0
-#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                        0x00000003L
-//SDMA0_UCODE_CHECKSUM
-#define SDMA0_UCODE_CHECKSUM__DATA__SHIFT                                                                     0x0
-#define SDMA0_UCODE_CHECKSUM__DATA_MASK                                                                       0xFFFFFFFFL
-//SDMA0_F32_CNTL
-#define SDMA0_F32_CNTL__HALT__SHIFT                                                                           0x0
-#define SDMA0_F32_CNTL__STEP__SHIFT                                                                           0x1
-#define SDMA0_F32_CNTL__HALT_MASK                                                                             0x00000001L
-#define SDMA0_F32_CNTL__STEP_MASK                                                                             0x00000002L
-//SDMA0_FREEZE
-#define SDMA0_FREEZE__PREEMPT__SHIFT                                                                          0x0
-#define SDMA0_FREEZE__FREEZE__SHIFT                                                                           0x4
-#define SDMA0_FREEZE__FROZEN__SHIFT                                                                           0x5
-#define SDMA0_FREEZE__F32_FREEZE__SHIFT                                                                       0x6
-#define SDMA0_FREEZE__PREEMPT_MASK                                                                            0x00000001L
-#define SDMA0_FREEZE__FREEZE_MASK                                                                             0x00000010L
-#define SDMA0_FREEZE__FROZEN_MASK                                                                             0x00000020L
-#define SDMA0_FREEZE__F32_FREEZE_MASK                                                                         0x00000040L
-//SDMA0_PHASE0_QUANTUM
-#define SDMA0_PHASE0_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA0_PHASE0_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA0_PHASE0_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA0_PHASE0_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA0_PHASE0_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA0_PHASE0_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA0_PHASE1_QUANTUM
-#define SDMA0_PHASE1_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA0_PHASE1_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA0_PHASE1_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA0_PHASE1_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA0_PHASE1_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA0_PHASE1_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA_POWER_GATING
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_CONDITION__SHIFT                                                   0x0
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_CONDITION__SHIFT                                                    0x1
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_REQ__SHIFT                                                         0x2
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_REQ__SHIFT                                                          0x3
-#define SDMA_POWER_GATING__PG_CNTL_STATUS__SHIFT                                                              0x4
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_CONDITION_MASK                                                     0x00000001L
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_CONDITION_MASK                                                      0x00000002L
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_REQ_MASK                                                           0x00000004L
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_REQ_MASK                                                            0x00000008L
-#define SDMA_POWER_GATING__PG_CNTL_STATUS_MASK                                                                0x00000030L
-//SDMA_PGFSM_CONFIG
-#define SDMA_PGFSM_CONFIG__FSM_ADDR__SHIFT                                                                    0x0
-#define SDMA_PGFSM_CONFIG__POWER_DOWN__SHIFT                                                                  0x8
-#define SDMA_PGFSM_CONFIG__POWER_UP__SHIFT                                                                    0x9
-#define SDMA_PGFSM_CONFIG__P1_SELECT__SHIFT                                                                   0xa
-#define SDMA_PGFSM_CONFIG__P2_SELECT__SHIFT                                                                   0xb
-#define SDMA_PGFSM_CONFIG__WRITE__SHIFT                                                                       0xc
-#define SDMA_PGFSM_CONFIG__READ__SHIFT                                                                        0xd
-#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE__SHIFT                                                               0x1b
-#define SDMA_PGFSM_CONFIG__REG_ADDR__SHIFT                                                                    0x1c
-#define SDMA_PGFSM_CONFIG__FSM_ADDR_MASK                                                                      0x000000FFL
-#define SDMA_PGFSM_CONFIG__POWER_DOWN_MASK                                                                    0x00000100L
-#define SDMA_PGFSM_CONFIG__POWER_UP_MASK                                                                      0x00000200L
-#define SDMA_PGFSM_CONFIG__P1_SELECT_MASK                                                                     0x00000400L
-#define SDMA_PGFSM_CONFIG__P2_SELECT_MASK                                                                     0x00000800L
-#define SDMA_PGFSM_CONFIG__WRITE_MASK                                                                         0x00001000L
-#define SDMA_PGFSM_CONFIG__READ_MASK                                                                          0x00002000L
-#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE_MASK                                                                 0x08000000L
-#define SDMA_PGFSM_CONFIG__REG_ADDR_MASK                                                                      0xF0000000L
-//SDMA_PGFSM_WRITE
-#define SDMA_PGFSM_WRITE__VALUE__SHIFT                                                                        0x0
-#define SDMA_PGFSM_WRITE__VALUE_MASK                                                                          0xFFFFFFFFL
-//SDMA_PGFSM_READ
-#define SDMA_PGFSM_READ__VALUE__SHIFT                                                                         0x0
-#define SDMA_PGFSM_READ__VALUE_MASK                                                                           0x00FFFFFFL
-//SDMA0_EDC_CONFIG
-#define SDMA0_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
-#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE__SHIFT                                                               0x2
-#define SDMA0_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
-#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE_MASK                                                                 0x00000004L
-//SDMA0_BA_THRESHOLD
-#define SDMA0_BA_THRESHOLD__READ_THRES__SHIFT                                                                 0x0
-#define SDMA0_BA_THRESHOLD__WRITE_THRES__SHIFT                                                                0x10
-#define SDMA0_BA_THRESHOLD__READ_THRES_MASK                                                                   0x000003FFL
-#define SDMA0_BA_THRESHOLD__WRITE_THRES_MASK                                                                  0x03FF0000L
-//SDMA0_ID
-#define SDMA0_ID__DEVICE_ID__SHIFT                                                                            0x0
-#define SDMA0_ID__DEVICE_ID_MASK                                                                              0x000000FFL
-//SDMA0_VERSION
-#define SDMA0_VERSION__MINVER__SHIFT                                                                          0x0
-#define SDMA0_VERSION__MAJVER__SHIFT                                                                          0x8
-#define SDMA0_VERSION__REV__SHIFT                                                                             0x10
-#define SDMA0_VERSION__MINVER_MASK                                                                            0x0000007FL
-#define SDMA0_VERSION__MAJVER_MASK                                                                            0x00007F00L
-#define SDMA0_VERSION__REV_MASK                                                                               0x003F0000L
-//SDMA0_EDC_COUNTER
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED__SHIFT                                                          0x0
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC__SHIFT                                                          0x1
-#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED__SHIFT                                                         0x2
-#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED__SHIFT                                                         0x3
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED__SHIFT                                                      0x4
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT                                                   0x5
-#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED__SHIFT                                                      0x6
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT                                                    0x7
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT                                                    0x8
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT                                                    0x9
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT                                                    0xa
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT                                                    0xb
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT                                                    0xc
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT                                                    0xd
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT                                                    0xe
-#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED__SHIFT                                                      0xf
-#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT                                                    0x10
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED_MASK                                                            0x00000001L
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC_MASK                                                            0x00000002L
-#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED_MASK                                                           0x00000004L
-#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED_MASK                                                           0x00000008L
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED_MASK                                                        0x00000010L
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED_MASK                                                     0x00000020L
-#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED_MASK                                                        0x00000040L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK                                                      0x00000080L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK                                                      0x00000100L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK                                                      0x00000200L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK                                                      0x00000400L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK                                                      0x00000800L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK                                                      0x00001000L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK                                                      0x00002000L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK                                                      0x00004000L
-#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED_MASK                                                        0x00008000L
-#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED_MASK                                                      0x00010000L
-//SDMA0_EDC_COUNTER_CLEAR
-#define SDMA0_EDC_COUNTER_CLEAR__DUMMY__SHIFT                                                                 0x0
-#define SDMA0_EDC_COUNTER_CLEAR__DUMMY_MASK                                                                   0x00000001L
-//SDMA0_STATUS2_REG
-#define SDMA0_STATUS2_REG__ID__SHIFT                                                                          0x0
-#define SDMA0_STATUS2_REG__F32_INSTR_PTR__SHIFT                                                               0x2
-#define SDMA0_STATUS2_REG__CMD_OP__SHIFT                                                                      0x10
-#define SDMA0_STATUS2_REG__ID_MASK                                                                            0x00000003L
-#define SDMA0_STATUS2_REG__F32_INSTR_PTR_MASK                                                                 0x00000FFCL
-#define SDMA0_STATUS2_REG__CMD_OP_MASK                                                                        0xFFFF0000L
-//SDMA0_ATOMIC_CNTL
-#define SDMA0_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                                  0x0
-#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                       0x1f
-#define SDMA0_ATOMIC_CNTL__LOOP_TIMER_MASK                                                                    0x7FFFFFFFL
-#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                         0x80000000L
-//SDMA0_ATOMIC_PREOP_LO
-#define SDMA0_ATOMIC_PREOP_LO__DATA__SHIFT                                                                    0x0
-#define SDMA0_ATOMIC_PREOP_LO__DATA_MASK                                                                      0xFFFFFFFFL
-//SDMA0_ATOMIC_PREOP_HI
-#define SDMA0_ATOMIC_PREOP_HI__DATA__SHIFT                                                                    0x0
-#define SDMA0_ATOMIC_PREOP_HI__DATA_MASK                                                                      0xFFFFFFFFL
-//SDMA0_UTCL1_CNTL
-#define SDMA0_UTCL1_CNTL__REDO_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_UTCL1_CNTL__REDO_DELAY__SHIFT                                                                   0x1
-#define SDMA0_UTCL1_CNTL__REDO_WATERMK__SHIFT                                                                 0xb
-#define SDMA0_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                                 0xe
-#define SDMA0_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                                 0x18
-#define SDMA0_UTCL1_CNTL__VADDR_WATERMK__SHIFT                                                                0x1d
-#define SDMA0_UTCL1_CNTL__REDO_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_UTCL1_CNTL__REDO_DELAY_MASK                                                                     0x000007FEL
-#define SDMA0_UTCL1_CNTL__REDO_WATERMK_MASK                                                                   0x00003800L
-#define SDMA0_UTCL1_CNTL__INVACK_DELAY_MASK                                                                   0x00FFC000L
-#define SDMA0_UTCL1_CNTL__REQL2_CREDIT_MASK                                                                   0x1F000000L
-#define SDMA0_UTCL1_CNTL__VADDR_WATERMK_MASK                                                                  0xE0000000L
-//SDMA0_UTCL1_WATERMK
-#define SDMA0_UTCL1_WATERMK__REQMC_WATERMK__SHIFT                                                             0x0
-#define SDMA0_UTCL1_WATERMK__REQPG_WATERMK__SHIFT                                                             0xa
-#define SDMA0_UTCL1_WATERMK__INVREQ_WATERMK__SHIFT                                                            0x12
-#define SDMA0_UTCL1_WATERMK__XNACK_WATERMK__SHIFT                                                             0x1a
-#define SDMA0_UTCL1_WATERMK__REQMC_WATERMK_MASK                                                               0x000003FFL
-#define SDMA0_UTCL1_WATERMK__REQPG_WATERMK_MASK                                                               0x0003FC00L
-#define SDMA0_UTCL1_WATERMK__INVREQ_WATERMK_MASK                                                              0x03FC0000L
-#define SDMA0_UTCL1_WATERMK__XNACK_WATERMK_MASK                                                               0xFC000000L
-//SDMA0_UTCL1_RD_STATUS
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                0x0
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                     0x1
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                      0x2
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                   0x3
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                               0x4
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                    0x5
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                 0x6
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT                                                   0x7
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT                                                  0x8
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                 0x9
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                      0xa
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                       0xb
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                    0xc
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                0xd
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                     0xe
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                  0xf
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT                                                    0x10
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT                                                   0x11
-#define SDMA0_UTCL1_RD_STATUS__PAGE_FAULT__SHIFT                                                              0x12
-#define SDMA0_UTCL1_RD_STATUS__PAGE_NULL__SHIFT                                                               0x13
-#define SDMA0_UTCL1_RD_STATUS__REQL2_IDLE__SHIFT                                                              0x14
-#define SDMA0_UTCL1_RD_STATUS__CE_L1_STALL__SHIFT                                                             0x15
-#define SDMA0_UTCL1_RD_STATUS__NEXT_RD_VECTOR__SHIFT                                                          0x16
-#define SDMA0_UTCL1_RD_STATUS__MERGE_STATE__SHIFT                                                             0x1a
-#define SDMA0_UTCL1_RD_STATUS__ADDR_RD_RTR__SHIFT                                                             0x1d
-#define SDMA0_UTCL1_RD_STATUS__WPTR_POLLING__SHIFT                                                            0x1e
-#define SDMA0_UTCL1_RD_STATUS__INVREQ_SIZE__SHIFT                                                             0x1f
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                  0x00000001L
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                       0x00000002L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                        0x00000004L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                     0x00000008L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                 0x00000010L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                      0x00000020L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                   0x00000040L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK                                                     0x00000080L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK                                                    0x00000100L
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                   0x00000200L
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                        0x00000400L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL_MASK                                                         0x00000800L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                      0x00001000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                  0x00002000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                       0x00004000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                    0x00008000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL_MASK                                                      0x00010000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL_MASK                                                     0x00020000L
-#define SDMA0_UTCL1_RD_STATUS__PAGE_FAULT_MASK                                                                0x00040000L
-#define SDMA0_UTCL1_RD_STATUS__PAGE_NULL_MASK                                                                 0x00080000L
-#define SDMA0_UTCL1_RD_STATUS__REQL2_IDLE_MASK                                                                0x00100000L
-#define SDMA0_UTCL1_RD_STATUS__CE_L1_STALL_MASK                                                               0x00200000L
-#define SDMA0_UTCL1_RD_STATUS__NEXT_RD_VECTOR_MASK                                                            0x03C00000L
-#define SDMA0_UTCL1_RD_STATUS__MERGE_STATE_MASK                                                               0x1C000000L
-#define SDMA0_UTCL1_RD_STATUS__ADDR_RD_RTR_MASK                                                               0x20000000L
-#define SDMA0_UTCL1_RD_STATUS__WPTR_POLLING_MASK                                                              0x40000000L
-#define SDMA0_UTCL1_RD_STATUS__INVREQ_SIZE_MASK                                                               0x80000000L
-//SDMA0_UTCL1_WR_STATUS
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                0x0
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                     0x1
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                      0x2
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                   0x3
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                               0x4
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                    0x5
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                 0x6
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT                                                   0x7
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT                                                  0x8
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                 0x9
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                      0xa
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                       0xb
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                    0xc
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                0xd
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                     0xe
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                  0xf
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT                                                    0x10
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT                                                   0x11
-#define SDMA0_UTCL1_WR_STATUS__PAGE_FAULT__SHIFT                                                              0x12
-#define SDMA0_UTCL1_WR_STATUS__PAGE_NULL__SHIFT                                                               0x13
-#define SDMA0_UTCL1_WR_STATUS__REQL2_IDLE__SHIFT                                                              0x14
-#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT                                                              0x15
-#define SDMA0_UTCL1_WR_STATUS__NEXT_WR_VECTOR__SHIFT                                                          0x16
-#define SDMA0_UTCL1_WR_STATUS__MERGE_STATE__SHIFT                                                             0x19
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY__SHIFT                                                    0x1c
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL__SHIFT                                                     0x1d
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY__SHIFT                                                   0x1e
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL__SHIFT                                                    0x1f
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                  0x00000001L
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                       0x00000002L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                        0x00000004L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                     0x00000008L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                 0x00000010L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                      0x00000020L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                   0x00000040L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK                                                     0x00000080L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK                                                    0x00000100L
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                   0x00000200L
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                        0x00000400L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL_MASK                                                         0x00000800L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                      0x00001000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                  0x00002000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                       0x00004000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                    0x00008000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL_MASK                                                      0x00010000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL_MASK                                                     0x00020000L
-#define SDMA0_UTCL1_WR_STATUS__PAGE_FAULT_MASK                                                                0x00040000L
-#define SDMA0_UTCL1_WR_STATUS__PAGE_NULL_MASK                                                                 0x00080000L
-#define SDMA0_UTCL1_WR_STATUS__REQL2_IDLE_MASK                                                                0x00100000L
-#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR_MASK                                                                0x00200000L
-#define SDMA0_UTCL1_WR_STATUS__NEXT_WR_VECTOR_MASK                                                            0x01C00000L
-#define SDMA0_UTCL1_WR_STATUS__MERGE_STATE_MASK                                                               0x0E000000L
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY_MASK                                                      0x10000000L
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL_MASK                                                       0x20000000L
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY_MASK                                                     0x40000000L
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL_MASK                                                      0x80000000L
-//SDMA0_UTCL1_INV0
-#define SDMA0_UTCL1_INV0__INV_MIDDLE__SHIFT                                                                   0x0
-#define SDMA0_UTCL1_INV0__RD_TIMEOUT__SHIFT                                                                   0x1
-#define SDMA0_UTCL1_INV0__WR_TIMEOUT__SHIFT                                                                   0x2
-#define SDMA0_UTCL1_INV0__RD_IN_INVADR__SHIFT                                                                 0x3
-#define SDMA0_UTCL1_INV0__WR_IN_INVADR__SHIFT                                                                 0x4
-#define SDMA0_UTCL1_INV0__PAGE_NULL_SW__SHIFT                                                                 0x5
-#define SDMA0_UTCL1_INV0__XNACK_IS_INVADR__SHIFT                                                              0x6
-#define SDMA0_UTCL1_INV0__INVREQ_ENABLE__SHIFT                                                                0x7
-#define SDMA0_UTCL1_INV0__NACK_TIMEOUT_SW__SHIFT                                                              0x8
-#define SDMA0_UTCL1_INV0__NFLUSH_INV_IDLE__SHIFT                                                              0x9
-#define SDMA0_UTCL1_INV0__FLUSH_INV_IDLE__SHIFT                                                               0xa
-#define SDMA0_UTCL1_INV0__INV_FLUSHTYPE__SHIFT                                                                0xb
-#define SDMA0_UTCL1_INV0__INV_VMID_VEC__SHIFT                                                                 0xc
-#define SDMA0_UTCL1_INV0__INV_ADDR_HI__SHIFT                                                                  0x1c
-#define SDMA0_UTCL1_INV0__INV_MIDDLE_MASK                                                                     0x00000001L
-#define SDMA0_UTCL1_INV0__RD_TIMEOUT_MASK                                                                     0x00000002L
-#define SDMA0_UTCL1_INV0__WR_TIMEOUT_MASK                                                                     0x00000004L
-#define SDMA0_UTCL1_INV0__RD_IN_INVADR_MASK                                                                   0x00000008L
-#define SDMA0_UTCL1_INV0__WR_IN_INVADR_MASK                                                                   0x00000010L
-#define SDMA0_UTCL1_INV0__PAGE_NULL_SW_MASK                                                                   0x00000020L
-#define SDMA0_UTCL1_INV0__XNACK_IS_INVADR_MASK                                                                0x00000040L
-#define SDMA0_UTCL1_INV0__INVREQ_ENABLE_MASK                                                                  0x00000080L
-#define SDMA0_UTCL1_INV0__NACK_TIMEOUT_SW_MASK                                                                0x00000100L
-#define SDMA0_UTCL1_INV0__NFLUSH_INV_IDLE_MASK                                                                0x00000200L
-#define SDMA0_UTCL1_INV0__FLUSH_INV_IDLE_MASK                                                                 0x00000400L
-#define SDMA0_UTCL1_INV0__INV_FLUSHTYPE_MASK                                                                  0x00000800L
-#define SDMA0_UTCL1_INV0__INV_VMID_VEC_MASK                                                                   0x0FFFF000L
-#define SDMA0_UTCL1_INV0__INV_ADDR_HI_MASK                                                                    0xF0000000L
-//SDMA0_UTCL1_INV1
-#define SDMA0_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                                  0x0
-#define SDMA0_UTCL1_INV1__INV_ADDR_LO_MASK                                                                    0xFFFFFFFFL
-//SDMA0_UTCL1_INV2
-#define SDMA0_UTCL1_INV2__INV_NFLUSH_VMID_VEC__SHIFT                                                          0x0
-#define SDMA0_UTCL1_INV2__INV_NFLUSH_VMID_VEC_MASK                                                            0xFFFFFFFFL
-//SDMA0_UTCL1_RD_XNACK0
-#define SDMA0_UTCL1_RD_XNACK0__XNACK_ADDR_LO__SHIFT                                                           0x0
-#define SDMA0_UTCL1_RD_XNACK0__XNACK_ADDR_LO_MASK                                                             0xFFFFFFFFL
-//SDMA0_UTCL1_RD_XNACK1
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_ADDR_HI__SHIFT                                                           0x0
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VMID__SHIFT                                                              0x4
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VECTOR__SHIFT                                                            0x8
-#define SDMA0_UTCL1_RD_XNACK1__IS_XNACK__SHIFT                                                                0x1a
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_ADDR_HI_MASK                                                             0x0000000FL
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VMID_MASK                                                                0x000000F0L
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VECTOR_MASK                                                              0x03FFFF00L
-#define SDMA0_UTCL1_RD_XNACK1__IS_XNACK_MASK                                                                  0x0C000000L
-//SDMA0_UTCL1_WR_XNACK0
-#define SDMA0_UTCL1_WR_XNACK0__XNACK_ADDR_LO__SHIFT                                                           0x0
-#define SDMA0_UTCL1_WR_XNACK0__XNACK_ADDR_LO_MASK                                                             0xFFFFFFFFL
-//SDMA0_UTCL1_WR_XNACK1
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_ADDR_HI__SHIFT                                                           0x0
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VMID__SHIFT                                                              0x4
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VECTOR__SHIFT                                                            0x8
-#define SDMA0_UTCL1_WR_XNACK1__IS_XNACK__SHIFT                                                                0x1a
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_ADDR_HI_MASK                                                             0x0000000FL
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VMID_MASK                                                                0x000000F0L
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VECTOR_MASK                                                              0x03FFFF00L
-#define SDMA0_UTCL1_WR_XNACK1__IS_XNACK_MASK                                                                  0x0C000000L
-//SDMA0_UTCL1_TIMEOUT
-#define SDMA0_UTCL1_TIMEOUT__RD_XNACK_LIMIT__SHIFT                                                            0x0
-#define SDMA0_UTCL1_TIMEOUT__WR_XNACK_LIMIT__SHIFT                                                            0x10
-#define SDMA0_UTCL1_TIMEOUT__RD_XNACK_LIMIT_MASK                                                              0x0000FFFFL
-#define SDMA0_UTCL1_TIMEOUT__WR_XNACK_LIMIT_MASK                                                              0xFFFF0000L
-//SDMA0_UTCL1_PAGE
-#define SDMA0_UTCL1_PAGE__VM_HOLE__SHIFT                                                                      0x0
-#define SDMA0_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                     0x1
-#define SDMA0_UTCL1_PAGE__USE_MTYPE__SHIFT                                                                    0x6
-#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                                 0x9
-#define SDMA0_UTCL1_PAGE__VM_HOLE_MASK                                                                        0x00000001L
-#define SDMA0_UTCL1_PAGE__REQ_TYPE_MASK                                                                       0x0000001EL
-#define SDMA0_UTCL1_PAGE__USE_MTYPE_MASK                                                                      0x000001C0L
-#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                                   0x00000200L
-//SDMA0_POWER_CNTL_IDLE
-#define SDMA0_POWER_CNTL_IDLE__DELAY0__SHIFT                                                                  0x0
-#define SDMA0_POWER_CNTL_IDLE__DELAY1__SHIFT                                                                  0x10
-#define SDMA0_POWER_CNTL_IDLE__DELAY2__SHIFT                                                                  0x18
-#define SDMA0_POWER_CNTL_IDLE__DELAY0_MASK                                                                    0x0000FFFFL
-#define SDMA0_POWER_CNTL_IDLE__DELAY1_MASK                                                                    0x00FF0000L
-#define SDMA0_POWER_CNTL_IDLE__DELAY2_MASK                                                                    0xFF000000L
-//SDMA0_RELAX_ORDERING_LUT
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED0__SHIFT                                                            0x0
-#define SDMA0_RELAX_ORDERING_LUT__COPY__SHIFT                                                                 0x1
-#define SDMA0_RELAX_ORDERING_LUT__WRITE__SHIFT                                                                0x2
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED3__SHIFT                                                            0x3
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED4__SHIFT                                                            0x4
-#define SDMA0_RELAX_ORDERING_LUT__FENCE__SHIFT                                                                0x5
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED76__SHIFT                                                           0x6
-#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM__SHIFT                                                             0x8
-#define SDMA0_RELAX_ORDERING_LUT__COND_EXE__SHIFT                                                             0x9
-#define SDMA0_RELAX_ORDERING_LUT__ATOMIC__SHIFT                                                               0xa
-#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL__SHIFT                                                           0xb
-#define SDMA0_RELAX_ORDERING_LUT__PTEPDE__SHIFT                                                               0xc
-#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT                                                            0xd
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED__SHIFT                                                             0xe
-#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT                                                         0x1b
-#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT                                                             0x1c
-#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT                                                            0x1d
-#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH__SHIFT                                                             0x1e
-#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH__SHIFT                                                             0x1f
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED0_MASK                                                              0x00000001L
-#define SDMA0_RELAX_ORDERING_LUT__COPY_MASK                                                                   0x00000002L
-#define SDMA0_RELAX_ORDERING_LUT__WRITE_MASK                                                                  0x00000004L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED3_MASK                                                              0x00000008L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED4_MASK                                                              0x00000010L
-#define SDMA0_RELAX_ORDERING_LUT__FENCE_MASK                                                                  0x00000020L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED76_MASK                                                             0x000000C0L
-#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM_MASK                                                               0x00000100L
-#define SDMA0_RELAX_ORDERING_LUT__COND_EXE_MASK                                                               0x00000200L
-#define SDMA0_RELAX_ORDERING_LUT__ATOMIC_MASK                                                                 0x00000400L
-#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL_MASK                                                             0x00000800L
-#define SDMA0_RELAX_ORDERING_LUT__PTEPDE_MASK                                                                 0x00001000L
-#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP_MASK                                                              0x00002000L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED_MASK                                                               0x07FFC000L
-#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK                                                           0x08000000L
-#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB_MASK                                                               0x10000000L
-#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL_MASK                                                              0x20000000L
-#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH_MASK                                                               0x40000000L
-#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH_MASK                                                               0x80000000L
-//SDMA0_CHICKEN_BITS_2
-#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT                                                       0x0
-#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK                                                         0x0000000FL
-//SDMA0_STATUS3_REG
-#define SDMA0_STATUS3_REG__CMD_OP_STATUS__SHIFT                                                               0x0
-#define SDMA0_STATUS3_REG__PREV_VM_CMD__SHIFT                                                                 0x10
-#define SDMA0_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                              0x14
-#define SDMA0_STATUS3_REG__CMD_OP_STATUS_MASK                                                                 0x0000FFFFL
-#define SDMA0_STATUS3_REG__PREV_VM_CMD_MASK                                                                   0x000F0000L
-#define SDMA0_STATUS3_REG__EXCEPTION_IDLE_MASK                                                                0x00100000L
-//SDMA0_PHYSICAL_ADDR_LO
-#define SDMA0_PHYSICAL_ADDR_LO__D_VALID__SHIFT                                                                0x0
-#define SDMA0_PHYSICAL_ADDR_LO__DIRTY__SHIFT                                                                  0x1
-#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT                                                              0x2
-#define SDMA0_PHYSICAL_ADDR_LO__ADDR__SHIFT                                                                   0xc
-#define SDMA0_PHYSICAL_ADDR_LO__D_VALID_MASK                                                                  0x00000001L
-#define SDMA0_PHYSICAL_ADDR_LO__DIRTY_MASK                                                                    0x00000002L
-#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID_MASK                                                                0x00000004L
-#define SDMA0_PHYSICAL_ADDR_LO__ADDR_MASK                                                                     0xFFFFF000L
-//SDMA0_PHYSICAL_ADDR_HI
-#define SDMA0_PHYSICAL_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA0_PHYSICAL_ADDR_HI__ADDR_MASK                                                                     0x0000FFFFL
-//SDMA0_ERROR_LOG
-#define SDMA0_ERROR_LOG__OVERRIDE__SHIFT                                                                      0x0
-#define SDMA0_ERROR_LOG__STATUS__SHIFT                                                                        0x10
-#define SDMA0_ERROR_LOG__OVERRIDE_MASK                                                                        0x0000FFFFL
-#define SDMA0_ERROR_LOG__STATUS_MASK                                                                          0xFFFF0000L
-//SDMA0_PUB_DUMMY_REG0
-#define SDMA0_PUB_DUMMY_REG0__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG0__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PUB_DUMMY_REG1
-#define SDMA0_PUB_DUMMY_REG1__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG1__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PUB_DUMMY_REG2
-#define SDMA0_PUB_DUMMY_REG2__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG2__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PUB_DUMMY_REG3
-#define SDMA0_PUB_DUMMY_REG3__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG3__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_F32_COUNTER
-#define SDMA0_F32_COUNTER__VALUE__SHIFT                                                                       0x0
-#define SDMA0_F32_COUNTER__VALUE_MASK                                                                         0xFFFFFFFFL
-//SDMA0_UNBREAKABLE
-#define SDMA0_UNBREAKABLE__VALUE__SHIFT                                                                       0x0
-#define SDMA0_UNBREAKABLE__VALUE_MASK                                                                         0x00000001L
-//SDMA0_PERFMON_CNTL
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE0__SHIFT                                                               0x0
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR0__SHIFT                                                                0x1
-#define SDMA0_PERFMON_CNTL__PERF_SEL0__SHIFT                                                                  0x2
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE1__SHIFT                                                               0xa
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR1__SHIFT                                                                0xb
-#define SDMA0_PERFMON_CNTL__PERF_SEL1__SHIFT                                                                  0xc
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE0_MASK                                                                 0x00000001L
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR0_MASK                                                                  0x00000002L
-#define SDMA0_PERFMON_CNTL__PERF_SEL0_MASK                                                                    0x000003FCL
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE1_MASK                                                                 0x00000400L
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR1_MASK                                                                  0x00000800L
-#define SDMA0_PERFMON_CNTL__PERF_SEL1_MASK                                                                    0x000FF000L
-//SDMA0_PERFCOUNTER0_RESULT
-#define SDMA0_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT                                                          0x0
-#define SDMA0_PERFCOUNTER0_RESULT__PERF_COUNT_MASK                                                            0xFFFFFFFFL
-//SDMA0_PERFCOUNTER1_RESULT
-#define SDMA0_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT                                                          0x0
-#define SDMA0_PERFCOUNTER1_RESULT__PERF_COUNT_MASK                                                            0xFFFFFFFFL
-//SDMA0_PERFCOUNTER_TAG_DELAY_RANGE
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW__SHIFT                                                   0x0
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH__SHIFT                                                  0xe
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW__SHIFT                                                   0x1c
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW_MASK                                                     0x00003FFFL
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH_MASK                                                    0x0FFFC000L
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW_MASK                                                     0x10000000L
-//SDMA0_CRD_CNTL
-#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT                                                                0x7
-#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT                                                                0xd
-#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT_MASK                                                                  0x00001F80L
-#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT_MASK                                                                  0x0007E000L
-//SDMA0_MMHUB_TRUSTLVL
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG0__SHIFT                                                                 0x0
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG1__SHIFT                                                                 0x3
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG2__SHIFT                                                                 0x6
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG3__SHIFT                                                                 0x9
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG4__SHIFT                                                                 0xc
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG5__SHIFT                                                                 0xf
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG6__SHIFT                                                                 0x12
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG7__SHIFT                                                                 0x15
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG0_MASK                                                                   0x00000007L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG1_MASK                                                                   0x00000038L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG2_MASK                                                                   0x000001C0L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG3_MASK                                                                   0x00000E00L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG4_MASK                                                                   0x00007000L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG5_MASK                                                                   0x00038000L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG6_MASK                                                                   0x001C0000L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG7_MASK                                                                   0x00E00000L
-//SDMA0_GPU_IOV_VIOLATION_LOG
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                  0x0
-#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                         0x1
-#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                           0x2
-#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                                   0x12
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                0x13
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                              0x14
-#define SDMA0_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT                                                      0x18
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                    0x00000001L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                           0x00000002L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                             0x0003FFFCL
-#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                     0x00040000L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                  0x00080000L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                                0x00F00000L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK                                                        0xFF000000L
-//SDMA0_ULV_CNTL
-#define SDMA0_ULV_CNTL__HYSTERESIS__SHIFT                                                                     0x0
-#define SDMA0_ULV_CNTL__ENTER_ULV_INT__SHIFT                                                                  0x1d
-#define SDMA0_ULV_CNTL__EXIT_ULV_INT__SHIFT                                                                   0x1e
-#define SDMA0_ULV_CNTL__ULV_STATUS__SHIFT                                                                     0x1f
-#define SDMA0_ULV_CNTL__HYSTERESIS_MASK                                                                       0x0000001FL
-#define SDMA0_ULV_CNTL__ENTER_ULV_INT_MASK                                                                    0x20000000L
-#define SDMA0_ULV_CNTL__EXIT_ULV_INT_MASK                                                                     0x40000000L
-#define SDMA0_ULV_CNTL__ULV_STATUS_MASK                                                                       0x80000000L
-//SDMA0_EA_DBIT_ADDR_DATA
-#define SDMA0_EA_DBIT_ADDR_DATA__VALUE__SHIFT                                                                 0x0
-#define SDMA0_EA_DBIT_ADDR_DATA__VALUE_MASK                                                                   0xFFFFFFFFL
-//SDMA0_EA_DBIT_ADDR_INDEX
-#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE__SHIFT                                                                0x0
-#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE_MASK                                                                  0x00000007L
-//SDMA0_GFX_RB_CNTL
-#define SDMA0_GFX_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
-#define SDMA0_GFX_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
-#define SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
-#define SDMA0_GFX_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
-#define SDMA0_GFX_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
-#define SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
-#define SDMA0_GFX_RB_CNTL__RB_SIZE_MASK                                                                       0x0000007EL
-#define SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
-#define SDMA0_GFX_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
-#define SDMA0_GFX_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
-//SDMA0_GFX_RB_BASE
-#define SDMA0_GFX_RB_BASE__ADDR__SHIFT                                                                        0x0
-#define SDMA0_GFX_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
-//SDMA0_GFX_RB_BASE_HI
-#define SDMA0_GFX_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
-#define SDMA0_GFX_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
-//SDMA0_GFX_RB_RPTR
-#define SDMA0_GFX_RB_RPTR__OFFSET__SHIFT                                                                      0x0
-#define SDMA0_GFX_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
-//SDMA0_GFX_RB_RPTR_HI
-#define SDMA0_GFX_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
-#define SDMA0_GFX_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR
-#define SDMA0_GFX_RB_WPTR__OFFSET__SHIFT                                                                      0x0
-#define SDMA0_GFX_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_HI
-#define SDMA0_GFX_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
-#define SDMA0_GFX_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_POLL_CNTL
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
-//SDMA0_GFX_RB_RPTR_ADDR_HI
-#define SDMA0_GFX_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
-#define SDMA0_GFX_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
-//SDMA0_GFX_RB_RPTR_ADDR_LO
-#define SDMA0_GFX_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
-#define SDMA0_GFX_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
-//SDMA0_GFX_IB_CNTL
-#define SDMA0_GFX_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
-#define SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
-#define SDMA0_GFX_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
-#define SDMA0_GFX_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
-#define SDMA0_GFX_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
-#define SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
-#define SDMA0_GFX_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
-#define SDMA0_GFX_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
-//SDMA0_GFX_IB_RPTR
-#define SDMA0_GFX_IB_RPTR__OFFSET__SHIFT                                                                      0x2
-#define SDMA0_GFX_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
-//SDMA0_GFX_IB_OFFSET
-#define SDMA0_GFX_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
-#define SDMA0_GFX_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
-//SDMA0_GFX_IB_BASE_LO
-#define SDMA0_GFX_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
-#define SDMA0_GFX_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
-//SDMA0_GFX_IB_BASE_HI
-#define SDMA0_GFX_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
-#define SDMA0_GFX_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
-//SDMA0_GFX_IB_SIZE
-#define SDMA0_GFX_IB_SIZE__SIZE__SHIFT                                                                        0x0
-#define SDMA0_GFX_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
-//SDMA0_GFX_SKIP_CNTL
-#define SDMA0_GFX_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
-#define SDMA0_GFX_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x00003FFFL
-//SDMA0_GFX_CONTEXT_STATUS
-#define SDMA0_GFX_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
-#define SDMA0_GFX_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
-#define SDMA0_GFX_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
-#define SDMA0_GFX_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
-#define SDMA0_GFX_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
-#define SDMA0_GFX_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
-#define SDMA0_GFX_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
-#define SDMA0_GFX_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
-//SDMA0_GFX_DOORBELL
-#define SDMA0_GFX_DOORBELL__ENABLE__SHIFT                                                                     0x1c
-#define SDMA0_GFX_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
-#define SDMA0_GFX_DOORBELL__ENABLE_MASK                                                                       0x10000000L
-#define SDMA0_GFX_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
-//SDMA0_GFX_CONTEXT_CNTL
-#define SDMA0_GFX_CONTEXT_CNTL__RESUME_CTX__SHIFT                                                             0x10
-#define SDMA0_GFX_CONTEXT_CNTL__RESUME_CTX_MASK                                                               0x00010000L
-//SDMA0_GFX_STATUS
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
-//SDMA0_GFX_DOORBELL_LOG
-#define SDMA0_GFX_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
-#define SDMA0_GFX_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
-#define SDMA0_GFX_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
-#define SDMA0_GFX_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
-//SDMA0_GFX_WATERMARK
-#define SDMA0_GFX_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
-#define SDMA0_GFX_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
-#define SDMA0_GFX_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
-#define SDMA0_GFX_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
-//SDMA0_GFX_DOORBELL_OFFSET
-#define SDMA0_GFX_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
-#define SDMA0_GFX_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
-//SDMA0_GFX_CSA_ADDR_LO
-#define SDMA0_GFX_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
-#define SDMA0_GFX_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
-//SDMA0_GFX_CSA_ADDR_HI
-#define SDMA0_GFX_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_GFX_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA0_GFX_IB_SUB_REMAIN
-#define SDMA0_GFX_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
-#define SDMA0_GFX_IB_SUB_REMAIN__SIZE_MASK                                                                    0x00003FFFL
-//SDMA0_GFX_PREEMPT
-#define SDMA0_GFX_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
-#define SDMA0_GFX_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
-//SDMA0_GFX_DUMMY_REG
-#define SDMA0_GFX_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
-#define SDMA0_GFX_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_POLL_ADDR_HI
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_POLL_ADDR_LO
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
-//SDMA0_GFX_RB_AQL_CNTL
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
-#define SDMA0_GFX_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
-#define SDMA0_GFX_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
-//SDMA0_GFX_MINOR_PTR_UPDATE
-#define SDMA0_GFX_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
-#define SDMA0_GFX_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
-//SDMA0_GFX_MIDCMD_DATA0
-#define SDMA0_GFX_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA1
-#define SDMA0_GFX_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA2
-#define SDMA0_GFX_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA3
-#define SDMA0_GFX_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA4
-#define SDMA0_GFX_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA5
-#define SDMA0_GFX_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA6
-#define SDMA0_GFX_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA7
-#define SDMA0_GFX_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA8
-#define SDMA0_GFX_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_CNTL
-#define SDMA0_GFX_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
-#define SDMA0_GFX_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
-#define SDMA0_GFX_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
-#define SDMA0_GFX_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
-#define SDMA0_GFX_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
-#define SDMA0_GFX_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
-#define SDMA0_GFX_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
-#define SDMA0_GFX_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
-//SDMA0_RLC0_RB_CNTL
-#define SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA0_RLC0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC0_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA0_RLC0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA0_RLC0_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA0_RLC0_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA0_RLC0_RB_BASE
-#define SDMA0_RLC0_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA0_RLC0_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA0_RLC0_RB_BASE_HI
-#define SDMA0_RLC0_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC0_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA0_RLC0_RB_RPTR
-#define SDMA0_RLC0_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC0_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC0_RB_RPTR_HI
-#define SDMA0_RLC0_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC0_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR
-#define SDMA0_RLC0_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC0_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_HI
-#define SDMA0_RLC0_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC0_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_POLL_CNTL
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA0_RLC0_RB_RPTR_ADDR_HI
-#define SDMA0_RLC0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA0_RLC0_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA0_RLC0_RB_RPTR_ADDR_LO
-#define SDMA0_RLC0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA0_RLC0_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA0_RLC0_IB_CNTL
-#define SDMA0_RLC0_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA0_RLC0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA0_RLC0_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA0_RLC0_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA0_RLC0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA0_RLC0_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA0_RLC0_IB_RPTR
-#define SDMA0_RLC0_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA0_RLC0_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA0_RLC0_IB_OFFSET
-#define SDMA0_RLC0_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA0_RLC0_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA0_RLC0_IB_BASE_LO
-#define SDMA0_RLC0_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA0_RLC0_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA0_RLC0_IB_BASE_HI
-#define SDMA0_RLC0_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC0_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC0_IB_SIZE
-#define SDMA0_RLC0_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA0_RLC0_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA0_RLC0_SKIP_CNTL
-#define SDMA0_RLC0_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA0_RLC0_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA0_RLC0_CONTEXT_STATUS
-#define SDMA0_RLC0_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA0_RLC0_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA0_RLC0_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA0_RLC0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA0_RLC0_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA0_RLC0_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA0_RLC0_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA0_RLC0_DOORBELL
-#define SDMA0_RLC0_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA0_RLC0_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA0_RLC0_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA0_RLC0_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA0_RLC0_STATUS
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA0_RLC0_DOORBELL_LOG
-#define SDMA0_RLC0_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA0_RLC0_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA0_RLC0_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA0_RLC0_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA0_RLC0_WATERMARK
-#define SDMA0_RLC0_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA0_RLC0_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA0_RLC0_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA0_RLC0_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA0_RLC0_DOORBELL_OFFSET
-#define SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA0_RLC0_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA0_RLC0_CSA_ADDR_LO
-#define SDMA0_RLC0_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA0_RLC0_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA0_RLC0_CSA_ADDR_HI
-#define SDMA0_RLC0_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA0_RLC0_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA0_RLC0_IB_SUB_REMAIN
-#define SDMA0_RLC0_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA0_RLC0_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA0_RLC0_PREEMPT
-#define SDMA0_RLC0_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA0_RLC0_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA0_RLC0_DUMMY_REG
-#define SDMA0_RLC0_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA0_RLC0_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA0_RLC0_RB_AQL_CNTL
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA0_RLC0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA0_RLC0_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA0_RLC0_MINOR_PTR_UPDATE
-#define SDMA0_RLC0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA0_RLC0_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA0_RLC0_MIDCMD_DATA0
-#define SDMA0_RLC0_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA1
-#define SDMA0_RLC0_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA2
-#define SDMA0_RLC0_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA3
-#define SDMA0_RLC0_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA4
-#define SDMA0_RLC0_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA5
-#define SDMA0_RLC0_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA6
-#define SDMA0_RLC0_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA7
-#define SDMA0_RLC0_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA8
-#define SDMA0_RLC0_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_CNTL
-#define SDMA0_RLC0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA0_RLC0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA0_RLC0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA0_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA0_RLC0_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA0_RLC0_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA0_RLC0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA0_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-//SDMA0_RLC1_RB_CNTL
-#define SDMA0_RLC1_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC1_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA0_RLC1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA0_RLC1_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA0_RLC1_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA0_RLC1_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC1_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA0_RLC1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA0_RLC1_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA0_RLC1_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA0_RLC1_RB_BASE
-#define SDMA0_RLC1_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA0_RLC1_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA0_RLC1_RB_BASE_HI
-#define SDMA0_RLC1_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC1_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA0_RLC1_RB_RPTR
-#define SDMA0_RLC1_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC1_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC1_RB_RPTR_HI
-#define SDMA0_RLC1_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC1_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR
-#define SDMA0_RLC1_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC1_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_HI
-#define SDMA0_RLC1_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC1_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_POLL_CNTL
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA0_RLC1_RB_RPTR_ADDR_HI
-#define SDMA0_RLC1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA0_RLC1_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA0_RLC1_RB_RPTR_ADDR_LO
-#define SDMA0_RLC1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA0_RLC1_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA0_RLC1_IB_CNTL
-#define SDMA0_RLC1_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA0_RLC1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA0_RLC1_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA0_RLC1_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA0_RLC1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA0_RLC1_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA0_RLC1_IB_RPTR
-#define SDMA0_RLC1_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA0_RLC1_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA0_RLC1_IB_OFFSET
-#define SDMA0_RLC1_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA0_RLC1_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA0_RLC1_IB_BASE_LO
-#define SDMA0_RLC1_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA0_RLC1_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA0_RLC1_IB_BASE_HI
-#define SDMA0_RLC1_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC1_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC1_IB_SIZE
-#define SDMA0_RLC1_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA0_RLC1_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA0_RLC1_SKIP_CNTL
-#define SDMA0_RLC1_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA0_RLC1_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA0_RLC1_CONTEXT_STATUS
-#define SDMA0_RLC1_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA0_RLC1_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA0_RLC1_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA0_RLC1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA0_RLC1_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA0_RLC1_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA0_RLC1_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA0_RLC1_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA0_RLC1_DOORBELL
-#define SDMA0_RLC1_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA0_RLC1_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA0_RLC1_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA0_RLC1_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA0_RLC1_STATUS
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA0_RLC1_DOORBELL_LOG
-#define SDMA0_RLC1_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA0_RLC1_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA0_RLC1_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA0_RLC1_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA0_RLC1_WATERMARK
-#define SDMA0_RLC1_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA0_RLC1_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA0_RLC1_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA0_RLC1_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA0_RLC1_DOORBELL_OFFSET
-#define SDMA0_RLC1_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA0_RLC1_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA0_RLC1_CSA_ADDR_LO
-#define SDMA0_RLC1_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA0_RLC1_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA0_RLC1_CSA_ADDR_HI
-#define SDMA0_RLC1_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA0_RLC1_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA0_RLC1_IB_SUB_REMAIN
-#define SDMA0_RLC1_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA0_RLC1_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA0_RLC1_PREEMPT
-#define SDMA0_RLC1_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA0_RLC1_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA0_RLC1_DUMMY_REG
-#define SDMA0_RLC1_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA0_RLC1_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA0_RLC1_RB_AQL_CNTL
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA0_RLC1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA0_RLC1_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA0_RLC1_MINOR_PTR_UPDATE
-#define SDMA0_RLC1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA0_RLC1_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA0_RLC1_MIDCMD_DATA0
-#define SDMA0_RLC1_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA1
-#define SDMA0_RLC1_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA2
-#define SDMA0_RLC1_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA3
-#define SDMA0_RLC1_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA4
-#define SDMA0_RLC1_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA5
-#define SDMA0_RLC1_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA6
-#define SDMA0_RLC1_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA7
-#define SDMA0_RLC1_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA8
-#define SDMA0_RLC1_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_CNTL
-#define SDMA0_RLC1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA0_RLC1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA0_RLC1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA0_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA0_RLC1_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA0_RLC1_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA0_RLC1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA0_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_default.h
deleted file mode 100644
index 5793a10e3dc2..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_default.h
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _vcn_1_0_DEFAULT_HEADER
-#define _vcn_1_0_DEFAULT_HEADER
-
-
-// addressBlock: uvd_uvd_pg_dec
-#define mmUVD_PGFSM_CONFIG_DEFAULT                                               0x00000000
-#define mmUVD_PGFSM_STATUS_DEFAULT                                               0x002aaaaa
-#define mmUVD_POWER_STATUS_DEFAULT                                               0x00000801
-#define mmCC_UVD_HARVESTING_DEFAULT                                              0x00000000
-#define mmUVD_SCRATCH1_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH2_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH3_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH4_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH5_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH6_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH7_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH8_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH9_DEFAULT                                                   0x00000000
-#define mmUVD_SCRATCH10_DEFAULT                                                  0x00000000
-#define mmUVD_SCRATCH11_DEFAULT                                                  0x00000000
-#define mmUVD_SCRATCH12_DEFAULT                                                  0x00000000
-#define mmUVD_SCRATCH13_DEFAULT                                                  0x00000000
-#define mmUVD_SCRATCH14_DEFAULT                                                  0x00000000
-#define mmUVD_DPG_LMI_VCPU_CACHE_64BIT_BAR_LOW_DEFAULT                           0x00000000
-#define mmUVD_DPG_LMI_VCPU_CACHE_64BIT_BAR_HIGH_DEFAULT                          0x00000000
-#define mmUVD_DPG_VCPU_CACHE_OFFSET0_DEFAULT                                     0x00000000
-
-
-// addressBlock: uvd_uvdgendec
-#define mmUVD_LCM_CGC_CNTRL_DEFAULT                                              0xa0f00000
-
-
-// addressBlock: uvd_uvdnpdec
-#define mmUVD_JPEG_CNTL_DEFAULT                                                  0x00000004
-#define mmUVD_JPEG_RB_BASE_DEFAULT                                               0x00000000
-#define mmUVD_JPEG_RB_WPTR_DEFAULT                                               0x00000000
-#define mmUVD_JPEG_RB_RPTR_DEFAULT                                               0x00000000
-#define mmUVD_JPEG_RB_SIZE_DEFAULT                                               0x00000000
-#define mmUVD_JPEG_UV_TILING_CTRL_DEFAULT                                        0x02104800
-#define mmUVD_JPEG_TILING_CTRL_DEFAULT                                           0x02104800
-#define mmUVD_JPEG_ADDR_CONFIG_DEFAULT                                           0x22010010
-#define mmUVD_JPEG_GPCOM_CMD_DEFAULT                                             0x00000000
-#define mmUVD_JPEG_GPCOM_DATA0_DEFAULT                                           0x00000000
-#define mmUVD_JPEG_GPCOM_DATA1_DEFAULT                                           0x00000000
-#define mmUVD_JPEG_JRB_BASE_LO_DEFAULT                                           0x00000000
-#define mmUVD_JPEG_JRB_BASE_HI_DEFAULT                                           0x00000000
-#define mmUVD_JPEG_JRB_SIZE_DEFAULT                                              0x00000000
-#define mmUVD_JPEG_JRB_RPTR_DEFAULT                                              0x00000000
-#define mmUVD_JPEG_JRB_WPTR_DEFAULT                                              0x00000000
-#define mmUVD_JPEG_UV_ADDR_CONFIG_DEFAULT                                        0x22010010
-#define mmUVD_SEMA_ADDR_LOW_DEFAULT                                              0x00000000
-#define mmUVD_SEMA_ADDR_HIGH_DEFAULT                                             0x00000000
-#define mmUVD_SEMA_CMD_DEFAULT                                                   0x00000080
-#define mmUVD_GPCOM_VCPU_CMD_DEFAULT                                             0x00000000
-#define mmUVD_GPCOM_VCPU_DATA0_DEFAULT                                           0x00000000
-#define mmUVD_GPCOM_VCPU_DATA1_DEFAULT                                           0x00000000
-#define mmUVD_UDEC_DBW_UV_ADDR_CONFIG_DEFAULT                                    0x22010010
-#define mmUVD_UDEC_ADDR_CONFIG_DEFAULT                                           0x22010010
-#define mmUVD_UDEC_DB_ADDR_CONFIG_DEFAULT                                        0x22010010
-#define mmUVD_UDEC_DBW_ADDR_CONFIG_DEFAULT                                       0x22010010
-#define mmUVD_SUVD_CGC_GATE_DEFAULT                                              0x00000000
-#define mmUVD_SUVD_CGC_STATUS_DEFAULT                                            0x00000000
-#define mmUVD_SUVD_CGC_CTRL_DEFAULT                                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW_DEFAULT                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH_DEFAULT                             0x00000000
-#define mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW_DEFAULT                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH_DEFAULT                             0x00000000
-#define mmUVD_NO_OP_DEFAULT                                                      0x00000000
-#define mmUVD_JPEG_CNTL2_DEFAULT                                                 0x00000000
-#define mmUVD_VERSION_DEFAULT                                                    0x00010000
-#define mmUVD_GP_SCRATCH8_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH9_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH10_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH11_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH12_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH13_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH14_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH15_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH16_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH17_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH18_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH19_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH20_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH21_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH22_DEFAULT                                               0x00000000
-#define mmUVD_GP_SCRATCH23_DEFAULT                                               0x00000000
-#define mmUVD_RB_BASE_LO2_DEFAULT                                                0x00000000
-#define mmUVD_RB_BASE_HI2_DEFAULT                                                0x00000000
-#define mmUVD_RB_SIZE2_DEFAULT                                                   0x00000000
-#define mmUVD_RB_RPTR2_DEFAULT                                                   0x00000000
-#define mmUVD_RB_WPTR2_DEFAULT                                                   0x00000000
-#define mmUVD_RB_BASE_LO_DEFAULT                                                 0x00000000
-#define mmUVD_RB_BASE_HI_DEFAULT                                                 0x00000000
-#define mmUVD_RB_SIZE_DEFAULT                                                    0x00000000
-#define mmUVD_RB_RPTR_DEFAULT                                                    0x00000000
-#define mmUVD_RB_WPTR_DEFAULT                                                    0x00000000
-#define mmUVD_RB_WPTR4_DEFAULT                                                   0x00000000
-#define mmUVD_JRBC_RB_RPTR_DEFAULT                                               0x00000000
-#define mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH_DEFAULT                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW_DEFAULT                               0x00000000
-
-
-// addressBlock: uvd_uvddec
-#define mmUVD_SEMA_CNTL_DEFAULT                                                  0x00000003
-#define mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW_DEFAULT                                  0x00000000
-#define mmUVD_LMI_JRBC_RB_64BIT_BAR_HIGH_DEFAULT                                 0x00000000
-#define mmUVD_LMI_JRBC_IB_64BIT_BAR_LOW_DEFAULT                                  0x00000000
-#define mmUVD_LMI_JRBC_IB_64BIT_BAR_HIGH_DEFAULT                                 0x00000000
-#define mmUVD_LMI_JRBC_IB_VMID_DEFAULT                                           0x00000000
-#define mmUVD_JRBC_RB_WPTR_DEFAULT                                               0x00000000
-#define mmUVD_JRBC_RB_CNTL_DEFAULT                                               0x00000100
-#define mmUVD_JRBC_IB_SIZE_DEFAULT                                               0x00000000
-#define mmUVD_JRBC_LMI_SWAP_CNTL_DEFAULT                                         0x00000000
-#define mmUVD_JRBC_SOFT_RESET_DEFAULT                                            0x00000000
-#define mmUVD_JRBC_STATUS_DEFAULT                                                0x00000003
-#define mmUVD_RB_RPTR3_DEFAULT                                                   0x00000000
-#define mmUVD_RB_WPTR3_DEFAULT                                                   0x00000000
-#define mmUVD_RB_BASE_LO3_DEFAULT                                                0x00000000
-#define mmUVD_RB_BASE_HI3_DEFAULT                                                0x00000000
-#define mmUVD_RB_SIZE3_DEFAULT                                                   0x00000000
-#define mmJPEG_CGC_GATE_DEFAULT                                                  0x00300000
-#define mmUVD_CTX_INDEX_DEFAULT                                                  0x00000000
-#define mmUVD_CTX_DATA_DEFAULT                                                   0x00000000
-#define mmUVD_CGC_GATE_DEFAULT                                                   0x000fffff
-#define mmUVD_CGC_STATUS_DEFAULT                                                 0x00000000
-#define mmUVD_CGC_CTRL_DEFAULT                                                   0x1fff018d
-#define mmUVD_GP_SCRATCH0_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH1_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH2_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH3_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH4_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH5_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH6_DEFAULT                                                0x00000000
-#define mmUVD_GP_SCRATCH7_DEFAULT                                                0x00000000
-#define mmUVD_LMI_VCPU_CACHE_VMID_DEFAULT                                        0x00000000
-#define mmUVD_LMI_CTRL2_DEFAULT                                                  0x003e0000
-#define mmUVD_MASTINT_EN_DEFAULT                                                 0x00000000
-#define mmJPEG_CGC_CTRL_DEFAULT                                                  0x0000018d
-#define mmUVD_LMI_CTRL_DEFAULT                                                   0x00104340
-#define mmUVD_LMI_STATUS_DEFAULT                                                 0x003fff7f
-#define mmUVD_LMI_VM_CTRL_DEFAULT                                                0x00000000
-#define mmUVD_LMI_SWAP_CNTL_DEFAULT                                              0x00000000
-#define mmUVD_MPC_SET_MUXA0_DEFAULT                                              0x00002040
-#define mmUVD_MPC_SET_MUXA1_DEFAULT                                              0x00000000
-#define mmUVD_MPC_SET_MUXB0_DEFAULT                                              0x00002040
-#define mmUVD_MPC_SET_MUXB1_DEFAULT                                              0x00000000
-#define mmUVD_MPC_SET_MUX_DEFAULT                                                0x00000088
-#define mmUVD_MPC_SET_ALU_DEFAULT                                                0x00000000
-#define mmUVD_GPCOM_SYS_CMD_DEFAULT                                              0x00000000
-#define mmUVD_GPCOM_SYS_DATA0_DEFAULT                                            0x00000000
-#define mmUVD_GPCOM_SYS_DATA1_DEFAULT                                            0x00000000
-#define mmUVD_VCPU_CACHE_OFFSET0_DEFAULT                                         0x00000000
-#define mmUVD_VCPU_CACHE_SIZE0_DEFAULT                                           0x00000000
-#define mmUVD_VCPU_CACHE_OFFSET1_DEFAULT                                         0x00000000
-#define mmUVD_VCPU_CACHE_SIZE1_DEFAULT                                           0x00000000
-#define mmUVD_VCPU_CACHE_OFFSET2_DEFAULT                                         0x00000000
-#define mmUVD_VCPU_CACHE_SIZE2_DEFAULT                                           0x00000000
-#define mmUVD_VCPU_CNTL_DEFAULT                                                  0x0ff20000
-#define mmUVD_SOFT_RESET_DEFAULT                                                 0x00000008
-#define mmUVD_LMI_RBC_IB_VMID_DEFAULT                                            0x00000000
-#define mmUVD_RBC_IB_SIZE_DEFAULT                                                0x00000000
-#define mmUVD_RBC_RB_RPTR_DEFAULT                                                0x00000000
-#define mmUVD_RBC_RB_WPTR_DEFAULT                                                0x00000000
-#define mmUVD_RBC_RB_WPTR_CNTL_DEFAULT                                           0x00000000
-#define mmUVD_RBC_RB_CNTL_DEFAULT                                                0x01000101
-#define mmUVD_RBC_RB_RPTR_ADDR_DEFAULT                                           0x00000000
-#define mmUVD_STATUS_DEFAULT                                                     0x00000000
-#define mmUVD_SEMA_TIMEOUT_STATUS_DEFAULT                                        0x00000000
-#define mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL_DEFAULT                          0x02000000
-#define mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL_DEFAULT                               0x02000000
-#define mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL_DEFAULT                        0x02000000
-#define mmUVD_CONTEXT_ID_DEFAULT                                                 0x00000000
-#define mmUVD_CONTEXT_ID2_DEFAULT                                                0x00000000
-#define mmUVD_RBC_WPTR_POLL_CNTL_DEFAULT                                         0x00400100
-#define mmUVD_RBC_WPTR_POLL_ADDR_DEFAULT                                         0x00000000
-#define mmUVD_RB_BASE_LO4_DEFAULT                                                0x00000000
-#define mmUVD_RB_BASE_HI4_DEFAULT                                                0x00000000
-#define mmUVD_RB_SIZE4_DEFAULT                                                   0x00000000
-#define mmUVD_RB_RPTR4_DEFAULT                                                   0x00000000
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_default.h
new file mode 100644
index 000000000000..4be3cb5c4556
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_default.h
@@ -0,0 +1,286 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _sdma0_4_0_DEFAULT_HEADER
+#define _sdma0_4_0_DEFAULT_HEADER
+
+
+// addressBlock: sdma0_sdma0dec
+#define mmSDMA0_UCODE_ADDR_DEFAULT	0x00000000
+#define mmSDMA0_UCODE_DATA_DEFAULT	0x00000000
+#define mmSDMA0_VM_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_VM_CTX_LO_DEFAULT	0x00000000
+#define mmSDMA0_VM_CTX_HI_DEFAULT	0x00000000
+#define mmSDMA0_ACTIVE_FCN_ID_DEFAULT	0x00000000
+#define mmSDMA0_VM_CTX_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_VIRT_RESET_REQ_DEFAULT	0x00000000
+#define mmSDMA0_VF_ENABLE_DEFAULT	0x00000000
+#define mmSDMA0_CONTEXT_REG_TYPE0_DEFAULT	0xfffdf79f
+#define mmSDMA0_CONTEXT_REG_TYPE1_DEFAULT	0x003fbcff
+#define mmSDMA0_CONTEXT_REG_TYPE2_DEFAULT	0x000003ff
+#define mmSDMA0_CONTEXT_REG_TYPE3_DEFAULT	0x00000000
+#define mmSDMA0_PUB_REG_TYPE0_DEFAULT	0x3c000000
+#define mmSDMA0_PUB_REG_TYPE1_DEFAULT	0x30003882
+#define mmSDMA0_PUB_REG_TYPE2_DEFAULT	0x0fc6e880
+#define mmSDMA0_PUB_REG_TYPE3_DEFAULT	0x00000000
+#define mmSDMA0_MMHUB_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_CONTEXT_GROUP_BOUNDARY_DEFAULT	0x00000000
+#define mmSDMA0_POWER_CNTL_DEFAULT	0x0003c000
+#define mmSDMA0_CLK_CTRL_DEFAULT	0xff000100
+#define mmSDMA0_CNTL_DEFAULT	0x00000002
+#define mmSDMA0_CHICKEN_BITS_DEFAULT	0x00831f07
+#define mmSDMA0_GB_ADDR_CONFIG_DEFAULT	0x00100012
+#define mmSDMA0_GB_ADDR_CONFIG_READ_DEFAULT	0x00100012
+#define mmSDMA0_RB_RPTR_FETCH_HI_DEFAULT	0x00000000
+#define mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_RB_RPTR_FETCH_DEFAULT	0x00000000
+#define mmSDMA0_IB_OFFSET_FETCH_DEFAULT	0x00000000
+#define mmSDMA0_PROGRAM_DEFAULT	0x00000000
+#define mmSDMA0_STATUS_REG_DEFAULT	0x46dee557
+#define mmSDMA0_STATUS1_REG_DEFAULT	0x000003ff
+#define mmSDMA0_RD_BURST_CNTL_DEFAULT	0x00000003
+#define mmSDMA0_HBM_PAGE_CONFIG_DEFAULT	0x00000000
+#define mmSDMA0_UCODE_CHECKSUM_DEFAULT	0x00000000
+#define mmSDMA0_F32_CNTL_DEFAULT	0x00000001
+#define mmSDMA0_FREEZE_DEFAULT	0x00000000
+#define mmSDMA0_PHASE0_QUANTUM_DEFAULT	0x00010002
+#define mmSDMA0_PHASE1_QUANTUM_DEFAULT	0x00010002
+#define mmSDMA_POWER_GATING_DEFAULT	0x00000000
+#define mmSDMA_PGFSM_CONFIG_DEFAULT	0x00000000
+#define mmSDMA_PGFSM_WRITE_DEFAULT	0x00000000
+#define mmSDMA_PGFSM_READ_DEFAULT	0x00000000
+#define mmSDMA0_EDC_CONFIG_DEFAULT	0x00000002
+#define mmSDMA0_BA_THRESHOLD_DEFAULT	0x03ff03ff
+#define mmSDMA0_ID_DEFAULT	0x00000001
+#define mmSDMA0_VERSION_DEFAULT	0x00000400
+#define mmSDMA0_EDC_COUNTER_DEFAULT	0x00000000
+#define mmSDMA0_EDC_COUNTER_CLEAR_DEFAULT	0x00000000
+#define mmSDMA0_STATUS2_REG_DEFAULT	0x00000000
+#define mmSDMA0_ATOMIC_CNTL_DEFAULT	0x00000200
+#define mmSDMA0_ATOMIC_PREOP_LO_DEFAULT	0x00000000
+#define mmSDMA0_ATOMIC_PREOP_HI_DEFAULT	0x00000000
+#define mmSDMA0_UTCL1_CNTL_DEFAULT	0xd0003019
+#define mmSDMA0_UTCL1_WATERMK_DEFAULT	0xfffbe1fe
+#define mmSDMA0_UTCL1_RD_STATUS_DEFAULT	0x201001ff
+#define mmSDMA0_UTCL1_WR_STATUS_DEFAULT	0x503001ff
+#define mmSDMA0_UTCL1_INV0_DEFAULT	0x00000600
+#define mmSDMA0_UTCL1_INV1_DEFAULT	0x00000000
+#define mmSDMA0_UTCL1_INV2_DEFAULT	0x00000000
+#define mmSDMA0_UTCL1_RD_XNACK0_DEFAULT	0x00000000
+#define mmSDMA0_UTCL1_RD_XNACK1_DEFAULT	0x00000000
+#define mmSDMA0_UTCL1_WR_XNACK0_DEFAULT	0x00000000
+#define mmSDMA0_UTCL1_WR_XNACK1_DEFAULT	0x00000000
+#define mmSDMA0_UTCL1_TIMEOUT_DEFAULT	0x00010001
+#define mmSDMA0_UTCL1_PAGE_DEFAULT	0x000003e0
+#define mmSDMA0_POWER_CNTL_IDLE_DEFAULT	0x06060200
+#define mmSDMA0_RELAX_ORDERING_LUT_DEFAULT	0xc0000006
+#define mmSDMA0_CHICKEN_BITS_2_DEFAULT	0x00000005
+#define mmSDMA0_STATUS3_REG_DEFAULT	0x00100000
+#define mmSDMA0_PHYSICAL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_PHYSICAL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_PHASE2_QUANTUM_DEFAULT	0x00010002
+#define mmSDMA0_ERROR_LOG_DEFAULT	0x0000000f
+#define mmSDMA0_PUB_DUMMY_REG0_DEFAULT	0x00000000
+#define mmSDMA0_PUB_DUMMY_REG1_DEFAULT	0x00000000
+#define mmSDMA0_PUB_DUMMY_REG2_DEFAULT	0x00000000
+#define mmSDMA0_PUB_DUMMY_REG3_DEFAULT	0x00000000
+#define mmSDMA0_F32_COUNTER_DEFAULT	0x00000000
+#define mmSDMA0_UNBREAKABLE_DEFAULT	0x00000000
+#define mmSDMA0_PERFMON_CNTL_DEFAULT	0x000ff7fd
+#define mmSDMA0_PERFCOUNTER0_RESULT_DEFAULT	0x00000000
+#define mmSDMA0_PERFCOUNTER1_RESULT_DEFAULT	0x00000000
+#define mmSDMA0_PERFCOUNTER_TAG_DELAY_RANGE_DEFAULT	0x00640000
+#define mmSDMA0_CRD_CNTL_DEFAULT	0x000085c0
+#define mmSDMA0_MMHUB_TRUSTLVL_DEFAULT	0x00000000
+#define mmSDMA0_GPU_IOV_VIOLATION_LOG_DEFAULT	0x00000000
+#define mmSDMA0_ULV_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_EA_DBIT_ADDR_DATA_DEFAULT	0x00000000
+#define mmSDMA0_EA_DBIT_ADDR_INDEX_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA0_GFX_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA0_GFX_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_GFX_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA0_GFX_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_GFX_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_GFX_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA0_GFX_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_GFX_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA0_GFX_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_GFX_CONTEXT_STATUS_DEFAULT	0x00000005
+#define mmSDMA0_GFX_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA0_GFX_CONTEXT_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_GFX_STATUS_DEFAULT	0x00000000
+#define mmSDMA0_GFX_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA0_GFX_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA0_GFX_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_GFX_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_GFX_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_GFX_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA0_GFX_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA0_GFX_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_GFX_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA0_GFX_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA0_GFX_MIDCMD_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA0_PAGE_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA0_PAGE_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA0_PAGE_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_CONTEXT_STATUS_DEFAULT	0x00000004
+#define mmSDMA0_PAGE_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_STATUS_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA0_PAGE_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA0_PAGE_MIDCMD_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA0_RLC0_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA0_RLC0_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA0_RLC0_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_CONTEXT_STATUS_DEFAULT	0x00000004
+#define mmSDMA0_RLC0_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_STATUS_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA0_RLC0_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA0_RLC0_MIDCMD_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA0_RLC1_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA0_RLC1_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA0_RLC1_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_CONTEXT_STATUS_DEFAULT	0x00000004
+#define mmSDMA0_RLC1_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_STATUS_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA0_RLC1_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA0_RLC1_MIDCMD_CNTL_DEFAULT	0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_offset.h
new file mode 100644
index 000000000000..99758695f019
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_offset.h
@@ -0,0 +1,547 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _sdma0_4_0_OFFSET_HEADER
+#define _sdma0_4_0_OFFSET_HEADER
+
+
+
+// addressBlock: sdma0_sdma0dec
+// base address:	0x4980
+#define mmSDMA0_UCODE_ADDR	0x0000
+#define mmSDMA0_UCODE_ADDR_BASE_IDX	0
+#define mmSDMA0_UCODE_DATA	0x0001
+#define mmSDMA0_UCODE_DATA_BASE_IDX	0
+#define mmSDMA0_VM_CNTL	0x0004
+#define mmSDMA0_VM_CNTL_BASE_IDX	0
+#define mmSDMA0_VM_CTX_LO	0x0005
+#define mmSDMA0_VM_CTX_LO_BASE_IDX	0
+#define mmSDMA0_VM_CTX_HI	0x0006
+#define mmSDMA0_VM_CTX_HI_BASE_IDX	0
+#define mmSDMA0_ACTIVE_FCN_ID	0x0007
+#define mmSDMA0_ACTIVE_FCN_ID_BASE_IDX	0
+#define mmSDMA0_VM_CTX_CNTL	0x0008
+#define mmSDMA0_VM_CTX_CNTL_BASE_IDX	0
+#define mmSDMA0_VIRT_RESET_REQ	0x0009
+#define mmSDMA0_VIRT_RESET_REQ_BASE_IDX	0
+#define mmSDMA0_VF_ENABLE	0x000a
+#define mmSDMA0_VF_ENABLE_BASE_IDX	0
+#define mmSDMA0_CONTEXT_REG_TYPE0	0x000b
+#define mmSDMA0_CONTEXT_REG_TYPE0_BASE_IDX	0
+#define mmSDMA0_CONTEXT_REG_TYPE1	0x000c
+#define mmSDMA0_CONTEXT_REG_TYPE1_BASE_IDX	0
+#define mmSDMA0_CONTEXT_REG_TYPE2	0x000d
+#define mmSDMA0_CONTEXT_REG_TYPE2_BASE_IDX	0
+#define mmSDMA0_CONTEXT_REG_TYPE3	0x000e
+#define mmSDMA0_CONTEXT_REG_TYPE3_BASE_IDX	0
+#define mmSDMA0_PUB_REG_TYPE0	0x000f
+#define mmSDMA0_PUB_REG_TYPE0_BASE_IDX	0
+#define mmSDMA0_PUB_REG_TYPE1	0x0010
+#define mmSDMA0_PUB_REG_TYPE1_BASE_IDX	0
+#define mmSDMA0_PUB_REG_TYPE2	0x0011
+#define mmSDMA0_PUB_REG_TYPE2_BASE_IDX	0
+#define mmSDMA0_PUB_REG_TYPE3	0x0012
+#define mmSDMA0_PUB_REG_TYPE3_BASE_IDX	0
+#define mmSDMA0_MMHUB_CNTL	0x0013
+#define mmSDMA0_MMHUB_CNTL_BASE_IDX	0
+#define mmSDMA0_CONTEXT_GROUP_BOUNDARY	0x0019
+#define mmSDMA0_CONTEXT_GROUP_BOUNDARY_BASE_IDX	0
+#define mmSDMA0_POWER_CNTL	0x001a
+#define mmSDMA0_POWER_CNTL_BASE_IDX	0
+#define mmSDMA0_CLK_CTRL	0x001b
+#define mmSDMA0_CLK_CTRL_BASE_IDX	0
+#define mmSDMA0_CNTL	0x001c
+#define mmSDMA0_CNTL_BASE_IDX	0
+#define mmSDMA0_CHICKEN_BITS	0x001d
+#define mmSDMA0_CHICKEN_BITS_BASE_IDX	0
+#define mmSDMA0_GB_ADDR_CONFIG	0x001e
+#define mmSDMA0_GB_ADDR_CONFIG_BASE_IDX	0
+#define mmSDMA0_GB_ADDR_CONFIG_READ	0x001f
+#define mmSDMA0_GB_ADDR_CONFIG_READ_BASE_IDX	0
+#define mmSDMA0_RB_RPTR_FETCH_HI	0x0020
+#define mmSDMA0_RB_RPTR_FETCH_HI_BASE_IDX	0
+#define mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL	0x0021
+#define mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL_BASE_IDX	0
+#define mmSDMA0_RB_RPTR_FETCH	0x0022
+#define mmSDMA0_RB_RPTR_FETCH_BASE_IDX	0
+#define mmSDMA0_IB_OFFSET_FETCH	0x0023
+#define mmSDMA0_IB_OFFSET_FETCH_BASE_IDX	0
+#define mmSDMA0_PROGRAM	0x0024
+#define mmSDMA0_PROGRAM_BASE_IDX	0
+#define mmSDMA0_STATUS_REG	0x0025
+#define mmSDMA0_STATUS_REG_BASE_IDX	0
+#define mmSDMA0_STATUS1_REG	0x0026
+#define mmSDMA0_STATUS1_REG_BASE_IDX	0
+#define mmSDMA0_RD_BURST_CNTL	0x0027
+#define mmSDMA0_RD_BURST_CNTL_BASE_IDX	0
+#define mmSDMA0_HBM_PAGE_CONFIG	0x0028
+#define mmSDMA0_HBM_PAGE_CONFIG_BASE_IDX	0
+#define mmSDMA0_UCODE_CHECKSUM	0x0029
+#define mmSDMA0_UCODE_CHECKSUM_BASE_IDX	0
+#define mmSDMA0_F32_CNTL	0x002a
+#define mmSDMA0_F32_CNTL_BASE_IDX	0
+#define mmSDMA0_FREEZE	0x002b
+#define mmSDMA0_FREEZE_BASE_IDX	0
+#define mmSDMA0_PHASE0_QUANTUM	0x002c
+#define mmSDMA0_PHASE0_QUANTUM_BASE_IDX	0
+#define mmSDMA0_PHASE1_QUANTUM	0x002d
+#define mmSDMA0_PHASE1_QUANTUM_BASE_IDX	0
+#define mmSDMA_POWER_GATING	0x002e
+#define mmSDMA_POWER_GATING_BASE_IDX	0
+#define mmSDMA_PGFSM_CONFIG	0x002f
+#define mmSDMA_PGFSM_CONFIG_BASE_IDX	0
+#define mmSDMA_PGFSM_WRITE	0x0030
+#define mmSDMA_PGFSM_WRITE_BASE_IDX	0
+#define mmSDMA_PGFSM_READ	0x0031
+#define mmSDMA_PGFSM_READ_BASE_IDX	0
+#define mmSDMA0_EDC_CONFIG	0x0032
+#define mmSDMA0_EDC_CONFIG_BASE_IDX	0
+#define mmSDMA0_BA_THRESHOLD	0x0033
+#define mmSDMA0_BA_THRESHOLD_BASE_IDX	0
+#define mmSDMA0_ID	0x0034
+#define mmSDMA0_ID_BASE_IDX	0
+#define mmSDMA0_VERSION	0x0035
+#define mmSDMA0_VERSION_BASE_IDX	0
+#define mmSDMA0_EDC_COUNTER	0x0036
+#define mmSDMA0_EDC_COUNTER_BASE_IDX	0
+#define mmSDMA0_EDC_COUNTER_CLEAR	0x0037
+#define mmSDMA0_EDC_COUNTER_CLEAR_BASE_IDX	0
+#define mmSDMA0_STATUS2_REG	0x0038
+#define mmSDMA0_STATUS2_REG_BASE_IDX	0
+#define mmSDMA0_ATOMIC_CNTL	0x0039
+#define mmSDMA0_ATOMIC_CNTL_BASE_IDX	0
+#define mmSDMA0_ATOMIC_PREOP_LO	0x003a
+#define mmSDMA0_ATOMIC_PREOP_LO_BASE_IDX	0
+#define mmSDMA0_ATOMIC_PREOP_HI	0x003b
+#define mmSDMA0_ATOMIC_PREOP_HI_BASE_IDX	0
+#define mmSDMA0_UTCL1_CNTL	0x003c
+#define mmSDMA0_UTCL1_CNTL_BASE_IDX	0
+#define mmSDMA0_UTCL1_WATERMK	0x003d
+#define mmSDMA0_UTCL1_WATERMK_BASE_IDX	0
+#define mmSDMA0_UTCL1_RD_STATUS	0x003e
+#define mmSDMA0_UTCL1_RD_STATUS_BASE_IDX	0
+#define mmSDMA0_UTCL1_WR_STATUS	0x003f
+#define mmSDMA0_UTCL1_WR_STATUS_BASE_IDX	0
+#define mmSDMA0_UTCL1_INV0	0x0040
+#define mmSDMA0_UTCL1_INV0_BASE_IDX	0
+#define mmSDMA0_UTCL1_INV1	0x0041
+#define mmSDMA0_UTCL1_INV1_BASE_IDX	0
+#define mmSDMA0_UTCL1_INV2	0x0042
+#define mmSDMA0_UTCL1_INV2_BASE_IDX	0
+#define mmSDMA0_UTCL1_RD_XNACK0	0x0043
+#define mmSDMA0_UTCL1_RD_XNACK0_BASE_IDX	0
+#define mmSDMA0_UTCL1_RD_XNACK1	0x0044
+#define mmSDMA0_UTCL1_RD_XNACK1_BASE_IDX	0
+#define mmSDMA0_UTCL1_WR_XNACK0	0x0045
+#define mmSDMA0_UTCL1_WR_XNACK0_BASE_IDX	0
+#define mmSDMA0_UTCL1_WR_XNACK1	0x0046
+#define mmSDMA0_UTCL1_WR_XNACK1_BASE_IDX	0
+#define mmSDMA0_UTCL1_TIMEOUT	0x0047
+#define mmSDMA0_UTCL1_TIMEOUT_BASE_IDX	0
+#define mmSDMA0_UTCL1_PAGE	0x0048
+#define mmSDMA0_UTCL1_PAGE_BASE_IDX	0
+#define mmSDMA0_POWER_CNTL_IDLE	0x0049
+#define mmSDMA0_POWER_CNTL_IDLE_BASE_IDX	0
+#define mmSDMA0_RELAX_ORDERING_LUT	0x004a
+#define mmSDMA0_RELAX_ORDERING_LUT_BASE_IDX	0
+#define mmSDMA0_CHICKEN_BITS_2	0x004b
+#define mmSDMA0_CHICKEN_BITS_2_BASE_IDX	0
+#define mmSDMA0_STATUS3_REG	0x004c
+#define mmSDMA0_STATUS3_REG_BASE_IDX	0
+#define mmSDMA0_PHYSICAL_ADDR_LO	0x004d
+#define mmSDMA0_PHYSICAL_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_PHYSICAL_ADDR_HI	0x004e
+#define mmSDMA0_PHYSICAL_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_PHASE2_QUANTUM	0x004f
+#define mmSDMA0_PHASE2_QUANTUM_BASE_IDX	0
+#define mmSDMA0_ERROR_LOG	0x0050
+#define mmSDMA0_ERROR_LOG_BASE_IDX	0
+#define mmSDMA0_PUB_DUMMY_REG0	0x0051
+#define mmSDMA0_PUB_DUMMY_REG0_BASE_IDX	0
+#define mmSDMA0_PUB_DUMMY_REG1	0x0052
+#define mmSDMA0_PUB_DUMMY_REG1_BASE_IDX	0
+#define mmSDMA0_PUB_DUMMY_REG2	0x0053
+#define mmSDMA0_PUB_DUMMY_REG2_BASE_IDX	0
+#define mmSDMA0_PUB_DUMMY_REG3	0x0054
+#define mmSDMA0_PUB_DUMMY_REG3_BASE_IDX	0
+#define mmSDMA0_F32_COUNTER	0x0055
+#define mmSDMA0_F32_COUNTER_BASE_IDX	0
+#define mmSDMA0_UNBREAKABLE	0x0056
+#define mmSDMA0_UNBREAKABLE_BASE_IDX	0
+#define mmSDMA0_PERFMON_CNTL	0x0057
+#define mmSDMA0_PERFMON_CNTL_BASE_IDX	0
+#define mmSDMA0_PERFCOUNTER0_RESULT	0x0058
+#define mmSDMA0_PERFCOUNTER0_RESULT_BASE_IDX	0
+#define mmSDMA0_PERFCOUNTER1_RESULT	0x0059
+#define mmSDMA0_PERFCOUNTER1_RESULT_BASE_IDX	0
+#define mmSDMA0_PERFCOUNTER_TAG_DELAY_RANGE	0x005a
+#define mmSDMA0_PERFCOUNTER_TAG_DELAY_RANGE_BASE_IDX	0
+#define mmSDMA0_CRD_CNTL	0x005b
+#define mmSDMA0_CRD_CNTL_BASE_IDX	0
+#define mmSDMA0_MMHUB_TRUSTLVL	0x005c
+#define mmSDMA0_MMHUB_TRUSTLVL_BASE_IDX	0
+#define mmSDMA0_GPU_IOV_VIOLATION_LOG	0x005d
+#define mmSDMA0_GPU_IOV_VIOLATION_LOG_BASE_IDX	0
+#define mmSDMA0_ULV_CNTL	0x005e
+#define mmSDMA0_ULV_CNTL_BASE_IDX	0
+#define mmSDMA0_EA_DBIT_ADDR_DATA	0x0060
+#define mmSDMA0_EA_DBIT_ADDR_DATA_BASE_IDX	0
+#define mmSDMA0_EA_DBIT_ADDR_INDEX	0x0061
+#define mmSDMA0_EA_DBIT_ADDR_INDEX_BASE_IDX	0
+#define mmSDMA0_GFX_RB_CNTL	0x0080
+#define mmSDMA0_GFX_RB_CNTL_BASE_IDX	0
+#define mmSDMA0_GFX_RB_BASE	0x0081
+#define mmSDMA0_GFX_RB_BASE_BASE_IDX	0
+#define mmSDMA0_GFX_RB_BASE_HI	0x0082
+#define mmSDMA0_GFX_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_GFX_RB_RPTR	0x0083
+#define mmSDMA0_GFX_RB_RPTR_BASE_IDX	0
+#define mmSDMA0_GFX_RB_RPTR_HI	0x0084
+#define mmSDMA0_GFX_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA0_GFX_RB_WPTR	0x0085
+#define mmSDMA0_GFX_RB_WPTR_BASE_IDX	0
+#define mmSDMA0_GFX_RB_WPTR_HI	0x0086
+#define mmSDMA0_GFX_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA0_GFX_RB_WPTR_POLL_CNTL	0x0087
+#define mmSDMA0_GFX_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA0_GFX_RB_RPTR_ADDR_HI	0x0088
+#define mmSDMA0_GFX_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_GFX_RB_RPTR_ADDR_LO	0x0089
+#define mmSDMA0_GFX_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_GFX_IB_CNTL	0x008a
+#define mmSDMA0_GFX_IB_CNTL_BASE_IDX	0
+#define mmSDMA0_GFX_IB_RPTR	0x008b
+#define mmSDMA0_GFX_IB_RPTR_BASE_IDX	0
+#define mmSDMA0_GFX_IB_OFFSET	0x008c
+#define mmSDMA0_GFX_IB_OFFSET_BASE_IDX	0
+#define mmSDMA0_GFX_IB_BASE_LO	0x008d
+#define mmSDMA0_GFX_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA0_GFX_IB_BASE_HI	0x008e
+#define mmSDMA0_GFX_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_GFX_IB_SIZE	0x008f
+#define mmSDMA0_GFX_IB_SIZE_BASE_IDX	0
+#define mmSDMA0_GFX_SKIP_CNTL	0x0090
+#define mmSDMA0_GFX_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA0_GFX_CONTEXT_STATUS	0x0091
+#define mmSDMA0_GFX_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA0_GFX_DOORBELL	0x0092
+#define mmSDMA0_GFX_DOORBELL_BASE_IDX	0
+#define mmSDMA0_GFX_CONTEXT_CNTL	0x0093
+#define mmSDMA0_GFX_CONTEXT_CNTL_BASE_IDX	0
+#define mmSDMA0_GFX_STATUS	0x00a8
+#define mmSDMA0_GFX_STATUS_BASE_IDX	0
+#define mmSDMA0_GFX_DOORBELL_LOG	0x00a9
+#define mmSDMA0_GFX_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA0_GFX_WATERMARK	0x00aa
+#define mmSDMA0_GFX_WATERMARK_BASE_IDX	0
+#define mmSDMA0_GFX_DOORBELL_OFFSET	0x00ab
+#define mmSDMA0_GFX_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA0_GFX_CSA_ADDR_LO	0x00ac
+#define mmSDMA0_GFX_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_GFX_CSA_ADDR_HI	0x00ad
+#define mmSDMA0_GFX_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_GFX_IB_SUB_REMAIN	0x00af
+#define mmSDMA0_GFX_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA0_GFX_PREEMPT	0x00b0
+#define mmSDMA0_GFX_PREEMPT_BASE_IDX	0
+#define mmSDMA0_GFX_DUMMY_REG	0x00b1
+#define mmSDMA0_GFX_DUMMY_REG_BASE_IDX	0
+#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI	0x00b2
+#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO	0x00b3
+#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_GFX_RB_AQL_CNTL	0x00b4
+#define mmSDMA0_GFX_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA0_GFX_MINOR_PTR_UPDATE	0x00b5
+#define mmSDMA0_GFX_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA0	0x00c0
+#define mmSDMA0_GFX_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA1	0x00c1
+#define mmSDMA0_GFX_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA2	0x00c2
+#define mmSDMA0_GFX_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA3	0x00c3
+#define mmSDMA0_GFX_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA4	0x00c4
+#define mmSDMA0_GFX_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA5	0x00c5
+#define mmSDMA0_GFX_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA6	0x00c6
+#define mmSDMA0_GFX_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA7	0x00c7
+#define mmSDMA0_GFX_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_DATA8	0x00c8
+#define mmSDMA0_GFX_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA0_GFX_MIDCMD_CNTL	0x00c9
+#define mmSDMA0_GFX_MIDCMD_CNTL_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_CNTL	0x00e0
+#define mmSDMA0_PAGE_RB_CNTL_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_BASE	0x00e1
+#define mmSDMA0_PAGE_RB_BASE_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_BASE_HI	0x00e2
+#define mmSDMA0_PAGE_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_RPTR	0x00e3
+#define mmSDMA0_PAGE_RB_RPTR_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_RPTR_HI	0x00e4
+#define mmSDMA0_PAGE_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_WPTR	0x00e5
+#define mmSDMA0_PAGE_RB_WPTR_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_WPTR_HI	0x00e6
+#define mmSDMA0_PAGE_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_WPTR_POLL_CNTL	0x00e7
+#define mmSDMA0_PAGE_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_RPTR_ADDR_HI	0x00e8
+#define mmSDMA0_PAGE_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_RPTR_ADDR_LO	0x00e9
+#define mmSDMA0_PAGE_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_PAGE_IB_CNTL	0x00ea
+#define mmSDMA0_PAGE_IB_CNTL_BASE_IDX	0
+#define mmSDMA0_PAGE_IB_RPTR	0x00eb
+#define mmSDMA0_PAGE_IB_RPTR_BASE_IDX	0
+#define mmSDMA0_PAGE_IB_OFFSET	0x00ec
+#define mmSDMA0_PAGE_IB_OFFSET_BASE_IDX	0
+#define mmSDMA0_PAGE_IB_BASE_LO	0x00ed
+#define mmSDMA0_PAGE_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA0_PAGE_IB_BASE_HI	0x00ee
+#define mmSDMA0_PAGE_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_PAGE_IB_SIZE	0x00ef
+#define mmSDMA0_PAGE_IB_SIZE_BASE_IDX	0
+#define mmSDMA0_PAGE_SKIP_CNTL	0x00f0
+#define mmSDMA0_PAGE_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA0_PAGE_CONTEXT_STATUS	0x00f1
+#define mmSDMA0_PAGE_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA0_PAGE_DOORBELL	0x00f2
+#define mmSDMA0_PAGE_DOORBELL_BASE_IDX	0
+#define mmSDMA0_PAGE_STATUS	0x0108
+#define mmSDMA0_PAGE_STATUS_BASE_IDX	0
+#define mmSDMA0_PAGE_DOORBELL_LOG	0x0109
+#define mmSDMA0_PAGE_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA0_PAGE_WATERMARK	0x010a
+#define mmSDMA0_PAGE_WATERMARK_BASE_IDX	0
+#define mmSDMA0_PAGE_DOORBELL_OFFSET	0x010b
+#define mmSDMA0_PAGE_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA0_PAGE_CSA_ADDR_LO	0x010c
+#define mmSDMA0_PAGE_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_PAGE_CSA_ADDR_HI	0x010d
+#define mmSDMA0_PAGE_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_PAGE_IB_SUB_REMAIN	0x010f
+#define mmSDMA0_PAGE_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA0_PAGE_PREEMPT	0x0110
+#define mmSDMA0_PAGE_PREEMPT_BASE_IDX	0
+#define mmSDMA0_PAGE_DUMMY_REG	0x0111
+#define mmSDMA0_PAGE_DUMMY_REG_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI	0x0112
+#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO	0x0113
+#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_PAGE_RB_AQL_CNTL	0x0114
+#define mmSDMA0_PAGE_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA0_PAGE_MINOR_PTR_UPDATE	0x0115
+#define mmSDMA0_PAGE_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA0	0x0120
+#define mmSDMA0_PAGE_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA1	0x0121
+#define mmSDMA0_PAGE_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA2	0x0122
+#define mmSDMA0_PAGE_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA3	0x0123
+#define mmSDMA0_PAGE_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA4	0x0124
+#define mmSDMA0_PAGE_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA5	0x0125
+#define mmSDMA0_PAGE_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA6	0x0126
+#define mmSDMA0_PAGE_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA7	0x0127
+#define mmSDMA0_PAGE_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_DATA8	0x0128
+#define mmSDMA0_PAGE_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA0_PAGE_MIDCMD_CNTL	0x0129
+#define mmSDMA0_PAGE_MIDCMD_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_CNTL	0x0140
+#define mmSDMA0_RLC0_RB_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_BASE	0x0141
+#define mmSDMA0_RLC0_RB_BASE_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_BASE_HI	0x0142
+#define mmSDMA0_RLC0_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_RPTR	0x0143
+#define mmSDMA0_RLC0_RB_RPTR_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_RPTR_HI	0x0144
+#define mmSDMA0_RLC0_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_WPTR	0x0145
+#define mmSDMA0_RLC0_RB_WPTR_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_WPTR_HI	0x0146
+#define mmSDMA0_RLC0_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_WPTR_POLL_CNTL	0x0147
+#define mmSDMA0_RLC0_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_RPTR_ADDR_HI	0x0148
+#define mmSDMA0_RLC0_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_RPTR_ADDR_LO	0x0149
+#define mmSDMA0_RLC0_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_RLC0_IB_CNTL	0x014a
+#define mmSDMA0_RLC0_IB_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC0_IB_RPTR	0x014b
+#define mmSDMA0_RLC0_IB_RPTR_BASE_IDX	0
+#define mmSDMA0_RLC0_IB_OFFSET	0x014c
+#define mmSDMA0_RLC0_IB_OFFSET_BASE_IDX	0
+#define mmSDMA0_RLC0_IB_BASE_LO	0x014d
+#define mmSDMA0_RLC0_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA0_RLC0_IB_BASE_HI	0x014e
+#define mmSDMA0_RLC0_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_RLC0_IB_SIZE	0x014f
+#define mmSDMA0_RLC0_IB_SIZE_BASE_IDX	0
+#define mmSDMA0_RLC0_SKIP_CNTL	0x0150
+#define mmSDMA0_RLC0_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC0_CONTEXT_STATUS	0x0151
+#define mmSDMA0_RLC0_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA0_RLC0_DOORBELL	0x0152
+#define mmSDMA0_RLC0_DOORBELL_BASE_IDX	0
+#define mmSDMA0_RLC0_STATUS	0x0168
+#define mmSDMA0_RLC0_STATUS_BASE_IDX	0
+#define mmSDMA0_RLC0_DOORBELL_LOG	0x0169
+#define mmSDMA0_RLC0_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA0_RLC0_WATERMARK	0x016a
+#define mmSDMA0_RLC0_WATERMARK_BASE_IDX	0
+#define mmSDMA0_RLC0_DOORBELL_OFFSET	0x016b
+#define mmSDMA0_RLC0_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA0_RLC0_CSA_ADDR_LO	0x016c
+#define mmSDMA0_RLC0_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_RLC0_CSA_ADDR_HI	0x016d
+#define mmSDMA0_RLC0_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_RLC0_IB_SUB_REMAIN	0x016f
+#define mmSDMA0_RLC0_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA0_RLC0_PREEMPT	0x0170
+#define mmSDMA0_RLC0_PREEMPT_BASE_IDX	0
+#define mmSDMA0_RLC0_DUMMY_REG	0x0171
+#define mmSDMA0_RLC0_DUMMY_REG_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_HI	0x0172
+#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_LO	0x0173
+#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_RLC0_RB_AQL_CNTL	0x0174
+#define mmSDMA0_RLC0_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC0_MINOR_PTR_UPDATE	0x0175
+#define mmSDMA0_RLC0_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA0	0x0180
+#define mmSDMA0_RLC0_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA1	0x0181
+#define mmSDMA0_RLC0_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA2	0x0182
+#define mmSDMA0_RLC0_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA3	0x0183
+#define mmSDMA0_RLC0_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA4	0x0184
+#define mmSDMA0_RLC0_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA5	0x0185
+#define mmSDMA0_RLC0_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA6	0x0186
+#define mmSDMA0_RLC0_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA7	0x0187
+#define mmSDMA0_RLC0_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_DATA8	0x0188
+#define mmSDMA0_RLC0_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA0_RLC0_MIDCMD_CNTL	0x0189
+#define mmSDMA0_RLC0_MIDCMD_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_CNTL	0x01a0
+#define mmSDMA0_RLC1_RB_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_BASE	0x01a1
+#define mmSDMA0_RLC1_RB_BASE_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_BASE_HI	0x01a2
+#define mmSDMA0_RLC1_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_RPTR	0x01a3
+#define mmSDMA0_RLC1_RB_RPTR_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_RPTR_HI	0x01a4
+#define mmSDMA0_RLC1_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_WPTR	0x01a5
+#define mmSDMA0_RLC1_RB_WPTR_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_WPTR_HI	0x01a6
+#define mmSDMA0_RLC1_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_WPTR_POLL_CNTL	0x01a7
+#define mmSDMA0_RLC1_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_RPTR_ADDR_HI	0x01a8
+#define mmSDMA0_RLC1_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_RPTR_ADDR_LO	0x01a9
+#define mmSDMA0_RLC1_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_RLC1_IB_CNTL	0x01aa
+#define mmSDMA0_RLC1_IB_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC1_IB_RPTR	0x01ab
+#define mmSDMA0_RLC1_IB_RPTR_BASE_IDX	0
+#define mmSDMA0_RLC1_IB_OFFSET	0x01ac
+#define mmSDMA0_RLC1_IB_OFFSET_BASE_IDX	0
+#define mmSDMA0_RLC1_IB_BASE_LO	0x01ad
+#define mmSDMA0_RLC1_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA0_RLC1_IB_BASE_HI	0x01ae
+#define mmSDMA0_RLC1_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA0_RLC1_IB_SIZE	0x01af
+#define mmSDMA0_RLC1_IB_SIZE_BASE_IDX	0
+#define mmSDMA0_RLC1_SKIP_CNTL	0x01b0
+#define mmSDMA0_RLC1_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC1_CONTEXT_STATUS	0x01b1
+#define mmSDMA0_RLC1_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA0_RLC1_DOORBELL	0x01b2
+#define mmSDMA0_RLC1_DOORBELL_BASE_IDX	0
+#define mmSDMA0_RLC1_STATUS	0x01c8
+#define mmSDMA0_RLC1_STATUS_BASE_IDX	0
+#define mmSDMA0_RLC1_DOORBELL_LOG	0x01c9
+#define mmSDMA0_RLC1_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA0_RLC1_WATERMARK	0x01ca
+#define mmSDMA0_RLC1_WATERMARK_BASE_IDX	0
+#define mmSDMA0_RLC1_DOORBELL_OFFSET	0x01cb
+#define mmSDMA0_RLC1_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA0_RLC1_CSA_ADDR_LO	0x01cc
+#define mmSDMA0_RLC1_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_RLC1_CSA_ADDR_HI	0x01cd
+#define mmSDMA0_RLC1_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_RLC1_IB_SUB_REMAIN	0x01cf
+#define mmSDMA0_RLC1_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA0_RLC1_PREEMPT	0x01d0
+#define mmSDMA0_RLC1_PREEMPT_BASE_IDX	0
+#define mmSDMA0_RLC1_DUMMY_REG	0x01d1
+#define mmSDMA0_RLC1_DUMMY_REG_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_HI	0x01d2
+#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_LO	0x01d3
+#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA0_RLC1_RB_AQL_CNTL	0x01d4
+#define mmSDMA0_RLC1_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA0_RLC1_MINOR_PTR_UPDATE	0x01d5
+#define mmSDMA0_RLC1_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA0	0x01e0
+#define mmSDMA0_RLC1_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA1	0x01e1
+#define mmSDMA0_RLC1_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA2	0x01e2
+#define mmSDMA0_RLC1_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA3	0x01e3
+#define mmSDMA0_RLC1_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA4	0x01e4
+#define mmSDMA0_RLC1_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA5	0x01e5
+#define mmSDMA0_RLC1_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA6	0x01e6
+#define mmSDMA0_RLC1_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA7	0x01e7
+#define mmSDMA0_RLC1_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_DATA8	0x01e8
+#define mmSDMA0_RLC1_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA0_RLC1_MIDCMD_CNTL	0x01e9
+#define mmSDMA0_RLC1_MIDCMD_CNTL_BASE_IDX	0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_sh_mask.h
new file mode 100644
index 000000000000..f846cc8268d8
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_0_sh_mask.h
@@ -0,0 +1,1852 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _sdma0_4_0_SH_MASK_HEADER
+#define _sdma0_4_0_SH_MASK_HEADER
+
+
+// addressBlock: sdma0_sdma0dec
+//SDMA0_UCODE_ADDR
+#define SDMA0_UCODE_ADDR__VALUE__SHIFT	0x0
+#define SDMA0_UCODE_ADDR__VALUE_MASK	0x00001FFFL
+//SDMA0_UCODE_DATA
+#define SDMA0_UCODE_DATA__VALUE__SHIFT	0x0
+#define SDMA0_UCODE_DATA__VALUE_MASK	0xFFFFFFFFL
+//SDMA0_VM_CNTL
+#define SDMA0_VM_CNTL__CMD__SHIFT	0x0
+#define SDMA0_VM_CNTL__CMD_MASK	0x0000000FL
+//SDMA0_VM_CTX_LO
+#define SDMA0_VM_CTX_LO__ADDR__SHIFT	0x2
+#define SDMA0_VM_CTX_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_VM_CTX_HI
+#define SDMA0_VM_CTX_HI__ADDR__SHIFT	0x0
+#define SDMA0_VM_CTX_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_ACTIVE_FCN_ID
+#define SDMA0_ACTIVE_FCN_ID__VFID__SHIFT	0x0
+#define SDMA0_ACTIVE_FCN_ID__RESERVED__SHIFT	0x4
+#define SDMA0_ACTIVE_FCN_ID__VF__SHIFT	0x1f
+#define SDMA0_ACTIVE_FCN_ID__VFID_MASK	0x0000000FL
+#define SDMA0_ACTIVE_FCN_ID__RESERVED_MASK	0x7FFFFFF0L
+#define SDMA0_ACTIVE_FCN_ID__VF_MASK	0x80000000L
+//SDMA0_VM_CTX_CNTL
+#define SDMA0_VM_CTX_CNTL__PRIV__SHIFT	0x0
+#define SDMA0_VM_CTX_CNTL__VMID__SHIFT	0x4
+#define SDMA0_VM_CTX_CNTL__PRIV_MASK	0x00000001L
+#define SDMA0_VM_CTX_CNTL__VMID_MASK	0x000000F0L
+//SDMA0_VIRT_RESET_REQ
+#define SDMA0_VIRT_RESET_REQ__VF__SHIFT	0x0
+#define SDMA0_VIRT_RESET_REQ__PF__SHIFT	0x1f
+#define SDMA0_VIRT_RESET_REQ__VF_MASK	0x0000FFFFL
+#define SDMA0_VIRT_RESET_REQ__PF_MASK	0x80000000L
+//SDMA0_VF_ENABLE
+#define SDMA0_VF_ENABLE__VF_ENABLE__SHIFT	0x0
+#define SDMA0_VF_ENABLE__VF_ENABLE_MASK	0x00000001L
+//SDMA0_CONTEXT_REG_TYPE0
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_CNTL__SHIFT	0x0
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE__SHIFT	0x1
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_HI__SHIFT	0x2
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR__SHIFT	0x3
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_HI__SHIFT	0x4
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR__SHIFT	0x5
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_HI__SHIFT	0x6
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_POLL_CNTL__SHIFT	0x7
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_HI__SHIFT	0x8
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_LO__SHIFT	0x9
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_CNTL__SHIFT	0xa
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_RPTR__SHIFT	0xb
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_OFFSET__SHIFT	0xc
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_LO__SHIFT	0xd
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_HI__SHIFT	0xe
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_SIZE__SHIFT	0xf
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_SKIP_CNTL__SHIFT	0x10
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_STATUS__SHIFT	0x11
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_DOORBELL__SHIFT	0x12
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_CNTL__SHIFT	0x13
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_CNTL_MASK	0x00000001L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_MASK	0x00000002L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_HI_MASK	0x00000004L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_MASK	0x00000008L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_HI_MASK	0x00000010L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_MASK	0x00000020L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_HI_MASK	0x00000040L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_POLL_CNTL_MASK	0x00000080L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_HI_MASK	0x00000100L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_LO_MASK	0x00000200L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_CNTL_MASK	0x00000400L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_RPTR_MASK	0x00000800L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_OFFSET_MASK	0x00001000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_LO_MASK	0x00002000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_HI_MASK	0x00004000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_SIZE_MASK	0x00008000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_SKIP_CNTL_MASK	0x00010000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_STATUS_MASK	0x00020000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_DOORBELL_MASK	0x00040000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_CNTL_MASK	0x00080000L
+//SDMA0_CONTEXT_REG_TYPE1
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_STATUS__SHIFT	0x8
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_LOG__SHIFT	0x9
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_WATERMARK__SHIFT	0xa
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_OFFSET__SHIFT	0xb
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_LO__SHIFT	0xc
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_HI__SHIFT	0xd
+#define SDMA0_CONTEXT_REG_TYPE1__VOID_REG2__SHIFT	0xe
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_IB_SUB_REMAIN__SHIFT	0xf
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_PREEMPT__SHIFT	0x10
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DUMMY_REG__SHIFT	0x11
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__SHIFT	0x12
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__SHIFT	0x13
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_AQL_CNTL__SHIFT	0x14
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_MINOR_PTR_UPDATE__SHIFT	0x15
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED__SHIFT	0x16
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_STATUS_MASK	0x00000100L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_LOG_MASK	0x00000200L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_WATERMARK_MASK	0x00000400L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_OFFSET_MASK	0x00000800L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_LO_MASK	0x00001000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_HI_MASK	0x00002000L
+#define SDMA0_CONTEXT_REG_TYPE1__VOID_REG2_MASK	0x00004000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_IB_SUB_REMAIN_MASK	0x00008000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_PREEMPT_MASK	0x00010000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DUMMY_REG_MASK	0x00020000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_HI_MASK	0x00040000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_LO_MASK	0x00080000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_AQL_CNTL_MASK	0x00100000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_MINOR_PTR_UPDATE_MASK	0x00200000L
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED_MASK	0xFFC00000L
+//SDMA0_CONTEXT_REG_TYPE2
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA0__SHIFT	0x0
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA1__SHIFT	0x1
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA2__SHIFT	0x2
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA3__SHIFT	0x3
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA4__SHIFT	0x4
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA5__SHIFT	0x5
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA6__SHIFT	0x6
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA7__SHIFT	0x7
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA8__SHIFT	0x8
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_CNTL__SHIFT	0x9
+#define SDMA0_CONTEXT_REG_TYPE2__RESERVED__SHIFT	0xa
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA0_MASK	0x00000001L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA1_MASK	0x00000002L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA2_MASK	0x00000004L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA3_MASK	0x00000008L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA4_MASK	0x00000010L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA5_MASK	0x00000020L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA6_MASK	0x00000040L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA7_MASK	0x00000080L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA8_MASK	0x00000100L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_CNTL_MASK	0x00000200L
+#define SDMA0_CONTEXT_REG_TYPE2__RESERVED_MASK	0xFFFFFC00L
+//SDMA0_CONTEXT_REG_TYPE3
+#define SDMA0_CONTEXT_REG_TYPE3__RESERVED__SHIFT	0x0
+#define SDMA0_CONTEXT_REG_TYPE3__RESERVED_MASK	0xFFFFFFFFL
+//SDMA0_PUB_REG_TYPE0
+#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_ADDR__SHIFT	0x0
+#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_DATA__SHIFT	0x1
+#define SDMA0_PUB_REG_TYPE0__RESERVED3__SHIFT	0x3
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CNTL__SHIFT	0x4
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_LO__SHIFT	0x5
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_HI__SHIFT	0x6
+#define SDMA0_PUB_REG_TYPE0__SDMA0_ACTIVE_FCN_ID__SHIFT	0x7
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_CNTL__SHIFT	0x8
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VIRT_RESET_REQ__SHIFT	0x9
+#define SDMA0_PUB_REG_TYPE0__RESERVED10__SHIFT	0xa
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE0__SHIFT	0xb
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE1__SHIFT	0xc
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE2__SHIFT	0xd
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE3__SHIFT	0xe
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE0__SHIFT	0xf
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE1__SHIFT	0x10
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE2__SHIFT	0x11
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE3__SHIFT	0x12
+#define SDMA0_PUB_REG_TYPE0__SDMA0_MMHUB_CNTL__SHIFT	0x13
+#define SDMA0_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY__SHIFT	0x14
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_GROUP_BOUNDARY__SHIFT	0x19
+#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL__SHIFT	0x1a
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CLK_CTRL__SHIFT	0x1b
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL__SHIFT	0x1c
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS__SHIFT	0x1d
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG__SHIFT	0x1e
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ__SHIFT	0x1f
+#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_ADDR_MASK	0x00000001L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_DATA_MASK	0x00000002L
+#define SDMA0_PUB_REG_TYPE0__RESERVED3_MASK	0x00000008L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CNTL_MASK	0x00000010L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_LO_MASK	0x00000020L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_HI_MASK	0x00000040L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_ACTIVE_FCN_ID_MASK	0x00000080L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_CNTL_MASK	0x00000100L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_VIRT_RESET_REQ_MASK	0x00000200L
+#define SDMA0_PUB_REG_TYPE0__RESERVED10_MASK	0x00000400L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE0_MASK	0x00000800L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE1_MASK	0x00001000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE2_MASK	0x00002000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE3_MASK	0x00004000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE0_MASK	0x00008000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE1_MASK	0x00010000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE2_MASK	0x00020000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE3_MASK	0x00040000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_MMHUB_CNTL_MASK	0x00080000L
+#define SDMA0_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY_MASK	0x01F00000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_GROUP_BOUNDARY_MASK	0x02000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL_MASK	0x04000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CLK_CTRL_MASK	0x08000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL_MASK	0x10000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS_MASK	0x20000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_MASK	0x40000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ_MASK	0x80000000L
+//SDMA0_PUB_REG_TYPE1
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI__SHIFT	0x0
+#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT	0x1
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH__SHIFT	0x2
+#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH__SHIFT	0x3
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM__SHIFT	0x4
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG__SHIFT	0x5
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG__SHIFT	0x6
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RD_BURST_CNTL__SHIFT	0x7
+#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG__SHIFT	0x8
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM__SHIFT	0x9
+#define SDMA0_PUB_REG_TYPE1__SDMA0_F32_CNTL__SHIFT	0xa
+#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE__SHIFT	0xb
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE0_QUANTUM__SHIFT	0xc
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE1_QUANTUM__SHIFT	0xd
+#define SDMA0_PUB_REG_TYPE1__SDMA_POWER_GATING__SHIFT	0xe
+#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG__SHIFT	0xf
+#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_WRITE__SHIFT	0x10
+#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_READ__SHIFT	0x11
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG__SHIFT	0x12
+#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD__SHIFT	0x13
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ID__SHIFT	0x14
+#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION__SHIFT	0x15
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER__SHIFT	0x16
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR__SHIFT	0x17
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG__SHIFT	0x18
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL__SHIFT	0x19
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO__SHIFT	0x1a
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI__SHIFT	0x1b
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL__SHIFT	0x1c
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK__SHIFT	0x1d
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_RD_STATUS__SHIFT	0x1e
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WR_STATUS__SHIFT	0x1f
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI_MASK	0x00000001L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL_MASK	0x00000002L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_MASK	0x00000004L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH_MASK	0x00000008L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM_MASK	0x00000010L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG_MASK	0x00000020L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG_MASK	0x00000040L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RD_BURST_CNTL_MASK	0x00000080L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG_MASK	0x00000100L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM_MASK	0x00000200L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_F32_CNTL_MASK	0x00000400L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE_MASK	0x00000800L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE0_QUANTUM_MASK	0x00001000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE1_QUANTUM_MASK	0x00002000L
+#define SDMA0_PUB_REG_TYPE1__SDMA_POWER_GATING_MASK	0x00004000L
+#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG_MASK	0x00008000L
+#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_WRITE_MASK	0x00010000L
+#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_READ_MASK	0x00020000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG_MASK	0x00040000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD_MASK	0x00080000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ID_MASK	0x00100000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION_MASK	0x00200000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_MASK	0x00400000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR_MASK	0x00800000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG_MASK	0x01000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL_MASK	0x02000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO_MASK	0x04000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI_MASK	0x08000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL_MASK	0x10000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK_MASK	0x20000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_RD_STATUS_MASK	0x40000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WR_STATUS_MASK	0x80000000L
+//SDMA0_PUB_REG_TYPE2
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0__SHIFT	0x0
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1__SHIFT	0x1
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2__SHIFT	0x2
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0__SHIFT	0x3
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1__SHIFT	0x4
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0__SHIFT	0x5
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1__SHIFT	0x6
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_TIMEOUT__SHIFT	0x7
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_PAGE__SHIFT	0x8
+#define SDMA0_PUB_REG_TYPE2__SDMA0_POWER_CNTL_IDLE__SHIFT	0x9
+#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT__SHIFT	0xa
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2__SHIFT	0xb
+#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG__SHIFT	0xc
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO__SHIFT	0xd
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI__SHIFT	0xe
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHASE2_QUANTUM__SHIFT	0xf
+#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG__SHIFT	0x10
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0__SHIFT	0x11
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1__SHIFT	0x12
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2__SHIFT	0x13
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3__SHIFT	0x14
+#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER__SHIFT	0x15
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UNBREAKABLE__SHIFT	0x16
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFMON_CNTL__SHIFT	0x17
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER0_RESULT__SHIFT	0x18
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER1_RESULT__SHIFT	0x19
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SHIFT	0x1a
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL__SHIFT	0x1b
+#define SDMA0_PUB_REG_TYPE2__SDMA0_MMHUB_TRUSTLVL__SHIFT	0x1c
+#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG__SHIFT	0x1d
+#define SDMA0_PUB_REG_TYPE2__SDMA0_ULV_CNTL__SHIFT	0x1e
+#define SDMA0_PUB_REG_TYPE2__RESERVED__SHIFT	0x1f
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0_MASK	0x00000001L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1_MASK	0x00000002L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2_MASK	0x00000004L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0_MASK	0x00000008L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1_MASK	0x00000010L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0_MASK	0x00000020L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1_MASK	0x00000040L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_TIMEOUT_MASK	0x00000080L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_PAGE_MASK	0x00000100L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_POWER_CNTL_IDLE_MASK	0x00000200L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT_MASK	0x00000400L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2_MASK	0x00000800L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG_MASK	0x00001000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO_MASK	0x00002000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI_MASK	0x00004000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHASE2_QUANTUM_MASK	0x00008000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG_MASK	0x00010000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0_MASK	0x00020000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1_MASK	0x00040000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2_MASK	0x00080000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3_MASK	0x00100000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER_MASK	0x00200000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UNBREAKABLE_MASK	0x00400000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFMON_CNTL_MASK	0x00800000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER0_RESULT_MASK	0x01000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER1_RESULT_MASK	0x02000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER_TAG_DELAY_RANGE_MASK	0x04000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL_MASK	0x08000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_MMHUB_TRUSTLVL_MASK	0x10000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG_MASK	0x20000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_ULV_CNTL_MASK	0x40000000L
+#define SDMA0_PUB_REG_TYPE2__RESERVED_MASK	0x80000000L
+//SDMA0_PUB_REG_TYPE3
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA__SHIFT	0x0
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX__SHIFT	0x1
+#define SDMA0_PUB_REG_TYPE3__RESERVED__SHIFT	0x2
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA_MASK	0x00000001L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX_MASK	0x00000002L
+#define SDMA0_PUB_REG_TYPE3__RESERVED_MASK	0xFFFFFFFCL
+//SDMA0_MMHUB_CNTL
+#define SDMA0_MMHUB_CNTL__UNIT_ID__SHIFT	0x0
+#define SDMA0_MMHUB_CNTL__UNIT_ID_MASK	0x0000003FL
+//SDMA0_CONTEXT_GROUP_BOUNDARY
+#define SDMA0_CONTEXT_GROUP_BOUNDARY__RESERVED__SHIFT	0x0
+#define SDMA0_CONTEXT_GROUP_BOUNDARY__RESERVED_MASK	0xFFFFFFFFL
+//SDMA0_POWER_CNTL
+#define SDMA0_POWER_CNTL__PG_CNTL_ENABLE__SHIFT	0x0
+#define SDMA0_POWER_CNTL__EXT_PG_POWER_ON_REQ__SHIFT	0x1
+#define SDMA0_POWER_CNTL__EXT_PG_POWER_OFF_REQ__SHIFT	0x2
+#define SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE__SHIFT	0x8
+#define SDMA0_POWER_CNTL__MEM_POWER_LS_EN__SHIFT	0x9
+#define SDMA0_POWER_CNTL__MEM_POWER_DS_EN__SHIFT	0xa
+#define SDMA0_POWER_CNTL__MEM_POWER_SD_EN__SHIFT	0xb
+#define SDMA0_POWER_CNTL__MEM_POWER_DELAY__SHIFT	0xc
+#define SDMA0_POWER_CNTL__PG_CNTL_ENABLE_MASK	0x00000001L
+#define SDMA0_POWER_CNTL__EXT_PG_POWER_ON_REQ_MASK	0x00000002L
+#define SDMA0_POWER_CNTL__EXT_PG_POWER_OFF_REQ_MASK	0x00000004L
+#define SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK	0x00000100L
+#define SDMA0_POWER_CNTL__MEM_POWER_LS_EN_MASK	0x00000200L
+#define SDMA0_POWER_CNTL__MEM_POWER_DS_EN_MASK	0x00000400L
+#define SDMA0_POWER_CNTL__MEM_POWER_SD_EN_MASK	0x00000800L
+#define SDMA0_POWER_CNTL__MEM_POWER_DELAY_MASK	0x003FF000L
+//SDMA0_CLK_CTRL
+#define SDMA0_CLK_CTRL__ON_DELAY__SHIFT	0x0
+#define SDMA0_CLK_CTRL__OFF_HYSTERESIS__SHIFT	0x4
+#define SDMA0_CLK_CTRL__RESERVED__SHIFT	0xc
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE7__SHIFT	0x18
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE6__SHIFT	0x19
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE5__SHIFT	0x1a
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE4__SHIFT	0x1b
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE3__SHIFT	0x1c
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE2__SHIFT	0x1d
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE1__SHIFT	0x1e
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE0__SHIFT	0x1f
+#define SDMA0_CLK_CTRL__ON_DELAY_MASK	0x0000000FL
+#define SDMA0_CLK_CTRL__OFF_HYSTERESIS_MASK	0x00000FF0L
+#define SDMA0_CLK_CTRL__RESERVED_MASK	0x00FFF000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK	0x01000000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK	0x02000000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK	0x04000000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK	0x08000000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK	0x10000000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK	0x20000000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK	0x40000000L
+#define SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK	0x80000000L
+//SDMA0_CNTL
+#define SDMA0_CNTL__TRAP_ENABLE__SHIFT	0x0
+#define SDMA0_CNTL__UTC_L1_ENABLE__SHIFT	0x1
+#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE__SHIFT	0x2
+#define SDMA0_CNTL__DATA_SWAP_ENABLE__SHIFT	0x3
+#define SDMA0_CNTL__FENCE_SWAP_ENABLE__SHIFT	0x4
+#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT	0x5
+#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT	0x11
+#define SDMA0_CNTL__AUTO_CTXSW_ENABLE__SHIFT	0x12
+#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE__SHIFT	0x1c
+#define SDMA0_CNTL__FROZEN_INT_ENABLE__SHIFT	0x1d
+#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT	0x1e
+#define SDMA0_CNTL__TRAP_ENABLE_MASK	0x00000001L
+#define SDMA0_CNTL__UTC_L1_ENABLE_MASK	0x00000002L
+#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE_MASK	0x00000004L
+#define SDMA0_CNTL__DATA_SWAP_ENABLE_MASK	0x00000008L
+#define SDMA0_CNTL__FENCE_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE_MASK	0x00000020L
+#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK	0x00020000L
+#define SDMA0_CNTL__AUTO_CTXSW_ENABLE_MASK	0x00040000L
+#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK	0x10000000L
+#define SDMA0_CNTL__FROZEN_INT_ENABLE_MASK	0x20000000L
+#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE_MASK	0x40000000L
+//SDMA0_CHICKEN_BITS
+#define SDMA0_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE__SHIFT	0x0
+#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT	0x1
+#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT	0x2
+#define SDMA0_CHICKEN_BITS__WRITE_BURST_LENGTH__SHIFT	0x8
+#define SDMA0_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE__SHIFT	0xa
+#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT	0x10
+#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT	0x11
+#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT	0x14
+#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT	0x17
+#define SDMA0_CHICKEN_BITS__TIME_BASED_QOS__SHIFT	0x19
+#define SDMA0_CHICKEN_BITS__CE_AFIFO_WATERMARK__SHIFT	0x1a
+#define SDMA0_CHICKEN_BITS__CE_DFIFO_WATERMARK__SHIFT	0x1c
+#define SDMA0_CHICKEN_BITS__CE_LFIFO_WATERMARK__SHIFT	0x1e
+#define SDMA0_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE_MASK	0x00000001L
+#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK	0x00000002L
+#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK	0x00000004L
+#define SDMA0_CHICKEN_BITS__WRITE_BURST_LENGTH_MASK	0x00000300L
+#define SDMA0_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE_MASK	0x00001C00L
+#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK	0x00010000L
+#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK	0x00020000L
+#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK	0x00100000L
+#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK	0x00800000L
+#define SDMA0_CHICKEN_BITS__TIME_BASED_QOS_MASK	0x02000000L
+#define SDMA0_CHICKEN_BITS__CE_AFIFO_WATERMARK_MASK	0x0C000000L
+#define SDMA0_CHICKEN_BITS__CE_DFIFO_WATERMARK_MASK	0x30000000L
+#define SDMA0_CHICKEN_BITS__CE_LFIFO_WATERMARK_MASK	0xC0000000L
+//SDMA0_GB_ADDR_CONFIG
+#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES__SHIFT	0x0
+#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT	0x3
+#define SDMA0_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT	0x8
+#define SDMA0_GB_ADDR_CONFIG__NUM_BANKS__SHIFT	0xc
+#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT	0x13
+#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES_MASK	0x00000007L
+#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK	0x00000038L
+#define SDMA0_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK	0x00000700L
+#define SDMA0_GB_ADDR_CONFIG__NUM_BANKS_MASK	0x00007000L
+#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK	0x00180000L
+//SDMA0_GB_ADDR_CONFIG_READ
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT	0x0
+#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT	0x3
+#define SDMA0_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT	0x8
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT	0xc
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT	0x13
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK	0x00000007L
+#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK	0x00000038L
+#define SDMA0_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK	0x00000700L
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_BANKS_MASK	0x00007000L
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK	0x00180000L
+//SDMA0_RB_RPTR_FETCH_HI
+#define SDMA0_RB_RPTR_FETCH_HI__OFFSET__SHIFT	0x0
+#define SDMA0_RB_RPTR_FETCH_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_SEM_WAIT_FAIL_TIMER_CNTL
+#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT	0x0
+#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK	0xFFFFFFFFL
+//SDMA0_RB_RPTR_FETCH
+#define SDMA0_RB_RPTR_FETCH__OFFSET__SHIFT	0x2
+#define SDMA0_RB_RPTR_FETCH__OFFSET_MASK	0xFFFFFFFCL
+//SDMA0_IB_OFFSET_FETCH
+#define SDMA0_IB_OFFSET_FETCH__OFFSET__SHIFT	0x2
+#define SDMA0_IB_OFFSET_FETCH__OFFSET_MASK	0x003FFFFCL
+//SDMA0_PROGRAM
+#define SDMA0_PROGRAM__STREAM__SHIFT	0x0
+#define SDMA0_PROGRAM__STREAM_MASK	0xFFFFFFFFL
+//SDMA0_STATUS_REG
+#define SDMA0_STATUS_REG__IDLE__SHIFT	0x0
+#define SDMA0_STATUS_REG__REG_IDLE__SHIFT	0x1
+#define SDMA0_STATUS_REG__RB_EMPTY__SHIFT	0x2
+#define SDMA0_STATUS_REG__RB_FULL__SHIFT	0x3
+#define SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT	0x4
+#define SDMA0_STATUS_REG__RB_CMD_FULL__SHIFT	0x5
+#define SDMA0_STATUS_REG__IB_CMD_IDLE__SHIFT	0x6
+#define SDMA0_STATUS_REG__IB_CMD_FULL__SHIFT	0x7
+#define SDMA0_STATUS_REG__BLOCK_IDLE__SHIFT	0x8
+#define SDMA0_STATUS_REG__INSIDE_IB__SHIFT	0x9
+#define SDMA0_STATUS_REG__EX_IDLE__SHIFT	0xa
+#define SDMA0_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE__SHIFT	0xb
+#define SDMA0_STATUS_REG__PACKET_READY__SHIFT	0xc
+#define SDMA0_STATUS_REG__MC_WR_IDLE__SHIFT	0xd
+#define SDMA0_STATUS_REG__SRBM_IDLE__SHIFT	0xe
+#define SDMA0_STATUS_REG__CONTEXT_EMPTY__SHIFT	0xf
+#define SDMA0_STATUS_REG__DELTA_RPTR_FULL__SHIFT	0x10
+#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT	0x11
+#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT	0x12
+#define SDMA0_STATUS_REG__MC_RD_IDLE__SHIFT	0x13
+#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT	0x14
+#define SDMA0_STATUS_REG__MC_RD_RET_STALL__SHIFT	0x15
+#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT	0x16
+#define SDMA0_STATUS_REG__PREV_CMD_IDLE__SHIFT	0x19
+#define SDMA0_STATUS_REG__SEM_IDLE__SHIFT	0x1a
+#define SDMA0_STATUS_REG__SEM_REQ_STALL__SHIFT	0x1b
+#define SDMA0_STATUS_REG__SEM_RESP_STATE__SHIFT	0x1c
+#define SDMA0_STATUS_REG__INT_IDLE__SHIFT	0x1e
+#define SDMA0_STATUS_REG__INT_REQ_STALL__SHIFT	0x1f
+#define SDMA0_STATUS_REG__IDLE_MASK	0x00000001L
+#define SDMA0_STATUS_REG__REG_IDLE_MASK	0x00000002L
+#define SDMA0_STATUS_REG__RB_EMPTY_MASK	0x00000004L
+#define SDMA0_STATUS_REG__RB_FULL_MASK	0x00000008L
+#define SDMA0_STATUS_REG__RB_CMD_IDLE_MASK	0x00000010L
+#define SDMA0_STATUS_REG__RB_CMD_FULL_MASK	0x00000020L
+#define SDMA0_STATUS_REG__IB_CMD_IDLE_MASK	0x00000040L
+#define SDMA0_STATUS_REG__IB_CMD_FULL_MASK	0x00000080L
+#define SDMA0_STATUS_REG__BLOCK_IDLE_MASK	0x00000100L
+#define SDMA0_STATUS_REG__INSIDE_IB_MASK	0x00000200L
+#define SDMA0_STATUS_REG__EX_IDLE_MASK	0x00000400L
+#define SDMA0_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE_MASK	0x00000800L
+#define SDMA0_STATUS_REG__PACKET_READY_MASK	0x00001000L
+#define SDMA0_STATUS_REG__MC_WR_IDLE_MASK	0x00002000L
+#define SDMA0_STATUS_REG__SRBM_IDLE_MASK	0x00004000L
+#define SDMA0_STATUS_REG__CONTEXT_EMPTY_MASK	0x00008000L
+#define SDMA0_STATUS_REG__DELTA_RPTR_FULL_MASK	0x00010000L
+#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE_MASK	0x00020000L
+#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE_MASK	0x00040000L
+#define SDMA0_STATUS_REG__MC_RD_IDLE_MASK	0x00080000L
+#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY_MASK	0x00100000L
+#define SDMA0_STATUS_REG__MC_RD_RET_STALL_MASK	0x00200000L
+#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK	0x00400000L
+#define SDMA0_STATUS_REG__PREV_CMD_IDLE_MASK	0x02000000L
+#define SDMA0_STATUS_REG__SEM_IDLE_MASK	0x04000000L
+#define SDMA0_STATUS_REG__SEM_REQ_STALL_MASK	0x08000000L
+#define SDMA0_STATUS_REG__SEM_RESP_STATE_MASK	0x30000000L
+#define SDMA0_STATUS_REG__INT_IDLE_MASK	0x40000000L
+#define SDMA0_STATUS_REG__INT_REQ_STALL_MASK	0x80000000L
+//SDMA0_STATUS1_REG
+#define SDMA0_STATUS1_REG__CE_WREQ_IDLE__SHIFT	0x0
+#define SDMA0_STATUS1_REG__CE_WR_IDLE__SHIFT	0x1
+#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE__SHIFT	0x2
+#define SDMA0_STATUS1_REG__CE_RREQ_IDLE__SHIFT	0x3
+#define SDMA0_STATUS1_REG__CE_OUT_IDLE__SHIFT	0x4
+#define SDMA0_STATUS1_REG__CE_IN_IDLE__SHIFT	0x5
+#define SDMA0_STATUS1_REG__CE_DST_IDLE__SHIFT	0x6
+#define SDMA0_STATUS1_REG__CE_CMD_IDLE__SHIFT	0x9
+#define SDMA0_STATUS1_REG__CE_AFIFO_FULL__SHIFT	0xa
+#define SDMA0_STATUS1_REG__CE_INFO_FULL__SHIFT	0xd
+#define SDMA0_STATUS1_REG__CE_INFO1_FULL__SHIFT	0xe
+#define SDMA0_STATUS1_REG__EX_START__SHIFT	0xf
+#define SDMA0_STATUS1_REG__CE_RD_STALL__SHIFT	0x11
+#define SDMA0_STATUS1_REG__CE_WR_STALL__SHIFT	0x12
+#define SDMA0_STATUS1_REG__CE_WREQ_IDLE_MASK	0x00000001L
+#define SDMA0_STATUS1_REG__CE_WR_IDLE_MASK	0x00000002L
+#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE_MASK	0x00000004L
+#define SDMA0_STATUS1_REG__CE_RREQ_IDLE_MASK	0x00000008L
+#define SDMA0_STATUS1_REG__CE_OUT_IDLE_MASK	0x00000010L
+#define SDMA0_STATUS1_REG__CE_IN_IDLE_MASK	0x00000020L
+#define SDMA0_STATUS1_REG__CE_DST_IDLE_MASK	0x00000040L
+#define SDMA0_STATUS1_REG__CE_CMD_IDLE_MASK	0x00000200L
+#define SDMA0_STATUS1_REG__CE_AFIFO_FULL_MASK	0x00000400L
+#define SDMA0_STATUS1_REG__CE_INFO_FULL_MASK	0x00002000L
+#define SDMA0_STATUS1_REG__CE_INFO1_FULL_MASK	0x00004000L
+#define SDMA0_STATUS1_REG__EX_START_MASK	0x00008000L
+#define SDMA0_STATUS1_REG__CE_RD_STALL_MASK	0x00020000L
+#define SDMA0_STATUS1_REG__CE_WR_STALL_MASK	0x00040000L
+//SDMA0_RD_BURST_CNTL
+#define SDMA0_RD_BURST_CNTL__RD_BURST__SHIFT	0x0
+#define SDMA0_RD_BURST_CNTL__RD_BURST_MASK	0x00000003L
+//SDMA0_HBM_PAGE_CONFIG
+#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT	0x0
+#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK	0x00000003L
+//SDMA0_UCODE_CHECKSUM
+#define SDMA0_UCODE_CHECKSUM__DATA__SHIFT	0x0
+#define SDMA0_UCODE_CHECKSUM__DATA_MASK	0xFFFFFFFFL
+//SDMA0_F32_CNTL
+#define SDMA0_F32_CNTL__HALT__SHIFT	0x0
+#define SDMA0_F32_CNTL__STEP__SHIFT	0x1
+#define SDMA0_F32_CNTL__HALT_MASK	0x00000001L
+#define SDMA0_F32_CNTL__STEP_MASK	0x00000002L
+//SDMA0_FREEZE
+#define SDMA0_FREEZE__PREEMPT__SHIFT	0x0
+#define SDMA0_FREEZE__FREEZE__SHIFT	0x4
+#define SDMA0_FREEZE__FROZEN__SHIFT	0x5
+#define SDMA0_FREEZE__F32_FREEZE__SHIFT	0x6
+#define SDMA0_FREEZE__PREEMPT_MASK	0x00000001L
+#define SDMA0_FREEZE__FREEZE_MASK	0x00000010L
+#define SDMA0_FREEZE__FROZEN_MASK	0x00000020L
+#define SDMA0_FREEZE__F32_FREEZE_MASK	0x00000040L
+//SDMA0_PHASE0_QUANTUM
+#define SDMA0_PHASE0_QUANTUM__UNIT__SHIFT	0x0
+#define SDMA0_PHASE0_QUANTUM__VALUE__SHIFT	0x8
+#define SDMA0_PHASE0_QUANTUM__PREFER__SHIFT	0x1e
+#define SDMA0_PHASE0_QUANTUM__UNIT_MASK	0x0000000FL
+#define SDMA0_PHASE0_QUANTUM__VALUE_MASK	0x00FFFF00L
+#define SDMA0_PHASE0_QUANTUM__PREFER_MASK	0x40000000L
+//SDMA0_PHASE1_QUANTUM
+#define SDMA0_PHASE1_QUANTUM__UNIT__SHIFT	0x0
+#define SDMA0_PHASE1_QUANTUM__VALUE__SHIFT	0x8
+#define SDMA0_PHASE1_QUANTUM__PREFER__SHIFT	0x1e
+#define SDMA0_PHASE1_QUANTUM__UNIT_MASK	0x0000000FL
+#define SDMA0_PHASE1_QUANTUM__VALUE_MASK	0x00FFFF00L
+#define SDMA0_PHASE1_QUANTUM__PREFER_MASK	0x40000000L
+//SDMA_POWER_GATING
+#define SDMA_POWER_GATING__SDMA0_POWER_OFF_CONDITION__SHIFT	0x0
+#define SDMA_POWER_GATING__SDMA0_POWER_ON_CONDITION__SHIFT	0x1
+#define SDMA_POWER_GATING__SDMA0_POWER_OFF_REQ__SHIFT	0x2
+#define SDMA_POWER_GATING__SDMA0_POWER_ON_REQ__SHIFT	0x3
+#define SDMA_POWER_GATING__PG_CNTL_STATUS__SHIFT	0x4
+#define SDMA_POWER_GATING__SDMA0_POWER_OFF_CONDITION_MASK	0x00000001L
+#define SDMA_POWER_GATING__SDMA0_POWER_ON_CONDITION_MASK	0x00000002L
+#define SDMA_POWER_GATING__SDMA0_POWER_OFF_REQ_MASK	0x00000004L
+#define SDMA_POWER_GATING__SDMA0_POWER_ON_REQ_MASK	0x00000008L
+#define SDMA_POWER_GATING__PG_CNTL_STATUS_MASK	0x00000030L
+//SDMA_PGFSM_CONFIG
+#define SDMA_PGFSM_CONFIG__FSM_ADDR__SHIFT	0x0
+#define SDMA_PGFSM_CONFIG__POWER_DOWN__SHIFT	0x8
+#define SDMA_PGFSM_CONFIG__POWER_UP__SHIFT	0x9
+#define SDMA_PGFSM_CONFIG__P1_SELECT__SHIFT	0xa
+#define SDMA_PGFSM_CONFIG__P2_SELECT__SHIFT	0xb
+#define SDMA_PGFSM_CONFIG__WRITE__SHIFT	0xc
+#define SDMA_PGFSM_CONFIG__READ__SHIFT	0xd
+#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE__SHIFT	0x1b
+#define SDMA_PGFSM_CONFIG__REG_ADDR__SHIFT	0x1c
+#define SDMA_PGFSM_CONFIG__FSM_ADDR_MASK	0x000000FFL
+#define SDMA_PGFSM_CONFIG__POWER_DOWN_MASK	0x00000100L
+#define SDMA_PGFSM_CONFIG__POWER_UP_MASK	0x00000200L
+#define SDMA_PGFSM_CONFIG__P1_SELECT_MASK	0x00000400L
+#define SDMA_PGFSM_CONFIG__P2_SELECT_MASK	0x00000800L
+#define SDMA_PGFSM_CONFIG__WRITE_MASK	0x00001000L
+#define SDMA_PGFSM_CONFIG__READ_MASK	0x00002000L
+#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE_MASK	0x08000000L
+#define SDMA_PGFSM_CONFIG__REG_ADDR_MASK	0xF0000000L
+//SDMA_PGFSM_WRITE
+#define SDMA_PGFSM_WRITE__VALUE__SHIFT	0x0
+#define SDMA_PGFSM_WRITE__VALUE_MASK	0xFFFFFFFFL
+//SDMA_PGFSM_READ
+#define SDMA_PGFSM_READ__VALUE__SHIFT	0x0
+#define SDMA_PGFSM_READ__VALUE_MASK	0x00FFFFFFL
+//SDMA0_EDC_CONFIG
+#define SDMA0_EDC_CONFIG__DIS_EDC__SHIFT	0x1
+#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE__SHIFT	0x2
+#define SDMA0_EDC_CONFIG__DIS_EDC_MASK	0x00000002L
+#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE_MASK	0x00000004L
+//SDMA0_BA_THRESHOLD
+#define SDMA0_BA_THRESHOLD__READ_THRES__SHIFT	0x0
+#define SDMA0_BA_THRESHOLD__WRITE_THRES__SHIFT	0x10
+#define SDMA0_BA_THRESHOLD__READ_THRES_MASK	0x000003FFL
+#define SDMA0_BA_THRESHOLD__WRITE_THRES_MASK	0x03FF0000L
+//SDMA0_ID
+#define SDMA0_ID__DEVICE_ID__SHIFT	0x0
+#define SDMA0_ID__DEVICE_ID_MASK	0x000000FFL
+//SDMA0_VERSION
+#define SDMA0_VERSION__MINVER__SHIFT	0x0
+#define SDMA0_VERSION__MAJVER__SHIFT	0x8
+#define SDMA0_VERSION__REV__SHIFT	0x10
+#define SDMA0_VERSION__MINVER_MASK	0x0000007FL
+#define SDMA0_VERSION__MAJVER_MASK	0x00007F00L
+#define SDMA0_VERSION__REV_MASK	0x003F0000L
+//SDMA0_EDC_COUNTER
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED__SHIFT	0x0
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC__SHIFT	0x1
+#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED__SHIFT	0x2
+#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED__SHIFT	0x3
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED__SHIFT	0x4
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT	0x5
+#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED__SHIFT	0x6
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT	0x7
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT	0x8
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT	0x9
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT	0xa
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT	0xb
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT	0xc
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT	0xd
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT	0xe
+#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED__SHIFT	0xf
+#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT	0x10
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED_MASK	0x00000001L
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC_MASK	0x00000002L
+#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED_MASK	0x00000004L
+#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED_MASK	0x00000008L
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED_MASK	0x00000010L
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED_MASK	0x00000020L
+#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED_MASK	0x00000040L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK	0x00000080L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK	0x00000100L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK	0x00000200L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK	0x00000400L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK	0x00000800L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK	0x00001000L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK	0x00002000L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK	0x00004000L
+#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED_MASK	0x00008000L
+#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED_MASK	0x00010000L
+//SDMA0_EDC_COUNTER_CLEAR
+#define SDMA0_EDC_COUNTER_CLEAR__DUMMY__SHIFT	0x0
+#define SDMA0_EDC_COUNTER_CLEAR__DUMMY_MASK	0x00000001L
+//SDMA0_STATUS2_REG
+#define SDMA0_STATUS2_REG__ID__SHIFT	0x0
+#define SDMA0_STATUS2_REG__F32_INSTR_PTR__SHIFT	0x2
+#define SDMA0_STATUS2_REG__CMD_OP__SHIFT	0x10
+#define SDMA0_STATUS2_REG__ID_MASK	0x00000003L
+#define SDMA0_STATUS2_REG__F32_INSTR_PTR_MASK	0x00000FFCL
+#define SDMA0_STATUS2_REG__CMD_OP_MASK	0xFFFF0000L
+//SDMA0_ATOMIC_CNTL
+#define SDMA0_ATOMIC_CNTL__LOOP_TIMER__SHIFT	0x0
+#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT	0x1f
+#define SDMA0_ATOMIC_CNTL__LOOP_TIMER_MASK	0x7FFFFFFFL
+#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK	0x80000000L
+//SDMA0_ATOMIC_PREOP_LO
+#define SDMA0_ATOMIC_PREOP_LO__DATA__SHIFT	0x0
+#define SDMA0_ATOMIC_PREOP_LO__DATA_MASK	0xFFFFFFFFL
+//SDMA0_ATOMIC_PREOP_HI
+#define SDMA0_ATOMIC_PREOP_HI__DATA__SHIFT	0x0
+#define SDMA0_ATOMIC_PREOP_HI__DATA_MASK	0xFFFFFFFFL
+//SDMA0_UTCL1_CNTL
+#define SDMA0_UTCL1_CNTL__REDO_ENABLE__SHIFT	0x0
+#define SDMA0_UTCL1_CNTL__REDO_DELAY__SHIFT	0x1
+#define SDMA0_UTCL1_CNTL__REDO_WATERMK__SHIFT	0xb
+#define SDMA0_UTCL1_CNTL__INVACK_DELAY__SHIFT	0xe
+#define SDMA0_UTCL1_CNTL__REQL2_CREDIT__SHIFT	0x18
+#define SDMA0_UTCL1_CNTL__VADDR_WATERMK__SHIFT	0x1d
+#define SDMA0_UTCL1_CNTL__REDO_ENABLE_MASK	0x00000001L
+#define SDMA0_UTCL1_CNTL__REDO_DELAY_MASK	0x000007FEL
+#define SDMA0_UTCL1_CNTL__REDO_WATERMK_MASK	0x00003800L
+#define SDMA0_UTCL1_CNTL__INVACK_DELAY_MASK	0x00FFC000L
+#define SDMA0_UTCL1_CNTL__REQL2_CREDIT_MASK	0x1F000000L
+#define SDMA0_UTCL1_CNTL__VADDR_WATERMK_MASK	0xE0000000L
+//SDMA0_UTCL1_WATERMK
+#define SDMA0_UTCL1_WATERMK__REQMC_WATERMK__SHIFT	0x0
+#define SDMA0_UTCL1_WATERMK__REQPG_WATERMK__SHIFT	0xa
+#define SDMA0_UTCL1_WATERMK__INVREQ_WATERMK__SHIFT	0x12
+#define SDMA0_UTCL1_WATERMK__XNACK_WATERMK__SHIFT	0x1a
+#define SDMA0_UTCL1_WATERMK__REQMC_WATERMK_MASK	0x000003FFL
+#define SDMA0_UTCL1_WATERMK__REQPG_WATERMK_MASK	0x0003FC00L
+#define SDMA0_UTCL1_WATERMK__INVREQ_WATERMK_MASK	0x03FC0000L
+#define SDMA0_UTCL1_WATERMK__XNACK_WATERMK_MASK	0xFC000000L
+//SDMA0_UTCL1_RD_STATUS
+#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT	0x0
+#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT	0x1
+#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY__SHIFT	0x2
+#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT	0x3
+#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT	0x4
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT	0x5
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT	0x6
+#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT	0x7
+#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT	0x8
+#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT	0x9
+#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL__SHIFT	0xa
+#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL__SHIFT	0xb
+#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT	0xc
+#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT	0xd
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL__SHIFT	0xe
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT	0xf
+#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT	0x10
+#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT	0x11
+#define SDMA0_UTCL1_RD_STATUS__PAGE_FAULT__SHIFT	0x12
+#define SDMA0_UTCL1_RD_STATUS__PAGE_NULL__SHIFT	0x13
+#define SDMA0_UTCL1_RD_STATUS__REQL2_IDLE__SHIFT	0x14
+#define SDMA0_UTCL1_RD_STATUS__CE_L1_STALL__SHIFT	0x15
+#define SDMA0_UTCL1_RD_STATUS__NEXT_RD_VECTOR__SHIFT	0x16
+#define SDMA0_UTCL1_RD_STATUS__MERGE_STATE__SHIFT	0x1a
+#define SDMA0_UTCL1_RD_STATUS__ADDR_RD_RTR__SHIFT	0x1d
+#define SDMA0_UTCL1_RD_STATUS__WPTR_POLLING__SHIFT	0x1e
+#define SDMA0_UTCL1_RD_STATUS__INVREQ_SIZE__SHIFT	0x1f
+#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK	0x00000001L
+#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY_MASK	0x00000002L
+#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY_MASK	0x00000004L
+#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK	0x00000008L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK	0x00000010L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY_MASK	0x00000020L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK	0x00000040L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK	0x00000080L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK	0x00000100L
+#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK	0x00000200L
+#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL_MASK	0x00000400L
+#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL_MASK	0x00000800L
+#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL_MASK	0x00001000L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK	0x00002000L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL_MASK	0x00004000L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK	0x00008000L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL_MASK	0x00010000L
+#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL_MASK	0x00020000L
+#define SDMA0_UTCL1_RD_STATUS__PAGE_FAULT_MASK	0x00040000L
+#define SDMA0_UTCL1_RD_STATUS__PAGE_NULL_MASK	0x00080000L
+#define SDMA0_UTCL1_RD_STATUS__REQL2_IDLE_MASK	0x00100000L
+#define SDMA0_UTCL1_RD_STATUS__CE_L1_STALL_MASK	0x00200000L
+#define SDMA0_UTCL1_RD_STATUS__NEXT_RD_VECTOR_MASK	0x03C00000L
+#define SDMA0_UTCL1_RD_STATUS__MERGE_STATE_MASK	0x1C000000L
+#define SDMA0_UTCL1_RD_STATUS__ADDR_RD_RTR_MASK	0x20000000L
+#define SDMA0_UTCL1_RD_STATUS__WPTR_POLLING_MASK	0x40000000L
+#define SDMA0_UTCL1_RD_STATUS__INVREQ_SIZE_MASK	0x80000000L
+//SDMA0_UTCL1_WR_STATUS
+#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT	0x0
+#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT	0x1
+#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY__SHIFT	0x2
+#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT	0x3
+#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT	0x4
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT	0x5
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT	0x6
+#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT	0x7
+#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT	0x8
+#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT	0x9
+#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL__SHIFT	0xa
+#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL__SHIFT	0xb
+#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT	0xc
+#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT	0xd
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL__SHIFT	0xe
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT	0xf
+#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT	0x10
+#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT	0x11
+#define SDMA0_UTCL1_WR_STATUS__PAGE_FAULT__SHIFT	0x12
+#define SDMA0_UTCL1_WR_STATUS__PAGE_NULL__SHIFT	0x13
+#define SDMA0_UTCL1_WR_STATUS__REQL2_IDLE__SHIFT	0x14
+#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT	0x15
+#define SDMA0_UTCL1_WR_STATUS__NEXT_WR_VECTOR__SHIFT	0x16
+#define SDMA0_UTCL1_WR_STATUS__MERGE_STATE__SHIFT	0x19
+#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY__SHIFT	0x1c
+#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL__SHIFT	0x1d
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY__SHIFT	0x1e
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL__SHIFT	0x1f
+#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK	0x00000001L
+#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY_MASK	0x00000002L
+#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY_MASK	0x00000004L
+#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK	0x00000008L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK	0x00000010L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY_MASK	0x00000020L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK	0x00000040L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK	0x00000080L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK	0x00000100L
+#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK	0x00000200L
+#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL_MASK	0x00000400L
+#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL_MASK	0x00000800L
+#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL_MASK	0x00001000L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK	0x00002000L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL_MASK	0x00004000L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK	0x00008000L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL_MASK	0x00010000L
+#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL_MASK	0x00020000L
+#define SDMA0_UTCL1_WR_STATUS__PAGE_FAULT_MASK	0x00040000L
+#define SDMA0_UTCL1_WR_STATUS__PAGE_NULL_MASK	0x00080000L
+#define SDMA0_UTCL1_WR_STATUS__REQL2_IDLE_MASK	0x00100000L
+#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR_MASK	0x00200000L
+#define SDMA0_UTCL1_WR_STATUS__NEXT_WR_VECTOR_MASK	0x01C00000L
+#define SDMA0_UTCL1_WR_STATUS__MERGE_STATE_MASK	0x0E000000L
+#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY_MASK	0x10000000L
+#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL_MASK	0x20000000L
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY_MASK	0x40000000L
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL_MASK	0x80000000L
+//SDMA0_UTCL1_INV0
+#define SDMA0_UTCL1_INV0__INV_MIDDLE__SHIFT	0x0
+#define SDMA0_UTCL1_INV0__RD_TIMEOUT__SHIFT	0x1
+#define SDMA0_UTCL1_INV0__WR_TIMEOUT__SHIFT	0x2
+#define SDMA0_UTCL1_INV0__RD_IN_INVADR__SHIFT	0x3
+#define SDMA0_UTCL1_INV0__WR_IN_INVADR__SHIFT	0x4
+#define SDMA0_UTCL1_INV0__PAGE_NULL_SW__SHIFT	0x5
+#define SDMA0_UTCL1_INV0__XNACK_IS_INVADR__SHIFT	0x6
+#define SDMA0_UTCL1_INV0__INVREQ_ENABLE__SHIFT	0x7
+#define SDMA0_UTCL1_INV0__NACK_TIMEOUT_SW__SHIFT	0x8
+#define SDMA0_UTCL1_INV0__NFLUSH_INV_IDLE__SHIFT	0x9
+#define SDMA0_UTCL1_INV0__FLUSH_INV_IDLE__SHIFT	0xa
+#define SDMA0_UTCL1_INV0__INV_FLUSHTYPE__SHIFT	0xb
+#define SDMA0_UTCL1_INV0__INV_VMID_VEC__SHIFT	0xc
+#define SDMA0_UTCL1_INV0__INV_ADDR_HI__SHIFT	0x1c
+#define SDMA0_UTCL1_INV0__INV_MIDDLE_MASK	0x00000001L
+#define SDMA0_UTCL1_INV0__RD_TIMEOUT_MASK	0x00000002L
+#define SDMA0_UTCL1_INV0__WR_TIMEOUT_MASK	0x00000004L
+#define SDMA0_UTCL1_INV0__RD_IN_INVADR_MASK	0x00000008L
+#define SDMA0_UTCL1_INV0__WR_IN_INVADR_MASK	0x00000010L
+#define SDMA0_UTCL1_INV0__PAGE_NULL_SW_MASK	0x00000020L
+#define SDMA0_UTCL1_INV0__XNACK_IS_INVADR_MASK	0x00000040L
+#define SDMA0_UTCL1_INV0__INVREQ_ENABLE_MASK	0x00000080L
+#define SDMA0_UTCL1_INV0__NACK_TIMEOUT_SW_MASK	0x00000100L
+#define SDMA0_UTCL1_INV0__NFLUSH_INV_IDLE_MASK	0x00000200L
+#define SDMA0_UTCL1_INV0__FLUSH_INV_IDLE_MASK	0x00000400L
+#define SDMA0_UTCL1_INV0__INV_FLUSHTYPE_MASK	0x00000800L
+#define SDMA0_UTCL1_INV0__INV_VMID_VEC_MASK	0x0FFFF000L
+#define SDMA0_UTCL1_INV0__INV_ADDR_HI_MASK	0xF0000000L
+//SDMA0_UTCL1_INV1
+#define SDMA0_UTCL1_INV1__INV_ADDR_LO__SHIFT	0x0
+#define SDMA0_UTCL1_INV1__INV_ADDR_LO_MASK	0xFFFFFFFFL
+//SDMA0_UTCL1_INV2
+#define SDMA0_UTCL1_INV2__INV_NFLUSH_VMID_VEC__SHIFT	0x0
+#define SDMA0_UTCL1_INV2__INV_NFLUSH_VMID_VEC_MASK	0xFFFFFFFFL
+//SDMA0_UTCL1_RD_XNACK0
+#define SDMA0_UTCL1_RD_XNACK0__XNACK_ADDR_LO__SHIFT	0x0
+#define SDMA0_UTCL1_RD_XNACK0__XNACK_ADDR_LO_MASK	0xFFFFFFFFL
+//SDMA0_UTCL1_RD_XNACK1
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_ADDR_HI__SHIFT	0x0
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_VMID__SHIFT	0x4
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_VECTOR__SHIFT	0x8
+#define SDMA0_UTCL1_RD_XNACK1__IS_XNACK__SHIFT	0x1a
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_ADDR_HI_MASK	0x0000000FL
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_VMID_MASK	0x000000F0L
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_VECTOR_MASK	0x03FFFF00L
+#define SDMA0_UTCL1_RD_XNACK1__IS_XNACK_MASK	0x0C000000L
+//SDMA0_UTCL1_WR_XNACK0
+#define SDMA0_UTCL1_WR_XNACK0__XNACK_ADDR_LO__SHIFT	0x0
+#define SDMA0_UTCL1_WR_XNACK0__XNACK_ADDR_LO_MASK	0xFFFFFFFFL
+//SDMA0_UTCL1_WR_XNACK1
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_ADDR_HI__SHIFT	0x0
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_VMID__SHIFT	0x4
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_VECTOR__SHIFT	0x8
+#define SDMA0_UTCL1_WR_XNACK1__IS_XNACK__SHIFT	0x1a
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_ADDR_HI_MASK	0x0000000FL
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_VMID_MASK	0x000000F0L
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_VECTOR_MASK	0x03FFFF00L
+#define SDMA0_UTCL1_WR_XNACK1__IS_XNACK_MASK	0x0C000000L
+//SDMA0_UTCL1_TIMEOUT
+#define SDMA0_UTCL1_TIMEOUT__RD_XNACK_LIMIT__SHIFT	0x0
+#define SDMA0_UTCL1_TIMEOUT__WR_XNACK_LIMIT__SHIFT	0x10
+#define SDMA0_UTCL1_TIMEOUT__RD_XNACK_LIMIT_MASK	0x0000FFFFL
+#define SDMA0_UTCL1_TIMEOUT__WR_XNACK_LIMIT_MASK	0xFFFF0000L
+//SDMA0_UTCL1_PAGE
+#define SDMA0_UTCL1_PAGE__VM_HOLE__SHIFT	0x0
+#define SDMA0_UTCL1_PAGE__REQ_TYPE__SHIFT	0x1
+#define SDMA0_UTCL1_PAGE__USE_MTYPE__SHIFT	0x6
+#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP__SHIFT	0x9
+#define SDMA0_UTCL1_PAGE__VM_HOLE_MASK	0x00000001L
+#define SDMA0_UTCL1_PAGE__REQ_TYPE_MASK	0x0000001EL
+#define SDMA0_UTCL1_PAGE__USE_MTYPE_MASK	0x000001C0L
+#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP_MASK	0x00000200L
+//SDMA0_POWER_CNTL_IDLE
+#define SDMA0_POWER_CNTL_IDLE__DELAY0__SHIFT	0x0
+#define SDMA0_POWER_CNTL_IDLE__DELAY1__SHIFT	0x10
+#define SDMA0_POWER_CNTL_IDLE__DELAY2__SHIFT	0x18
+#define SDMA0_POWER_CNTL_IDLE__DELAY0_MASK	0x0000FFFFL
+#define SDMA0_POWER_CNTL_IDLE__DELAY1_MASK	0x00FF0000L
+#define SDMA0_POWER_CNTL_IDLE__DELAY2_MASK	0xFF000000L
+//SDMA0_RELAX_ORDERING_LUT
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED0__SHIFT	0x0
+#define SDMA0_RELAX_ORDERING_LUT__COPY__SHIFT	0x1
+#define SDMA0_RELAX_ORDERING_LUT__WRITE__SHIFT	0x2
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED3__SHIFT	0x3
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED4__SHIFT	0x4
+#define SDMA0_RELAX_ORDERING_LUT__FENCE__SHIFT	0x5
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED76__SHIFT	0x6
+#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM__SHIFT	0x8
+#define SDMA0_RELAX_ORDERING_LUT__COND_EXE__SHIFT	0x9
+#define SDMA0_RELAX_ORDERING_LUT__ATOMIC__SHIFT	0xa
+#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL__SHIFT	0xb
+#define SDMA0_RELAX_ORDERING_LUT__PTEPDE__SHIFT	0xc
+#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT	0xd
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED__SHIFT	0xe
+#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT	0x1b
+#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT	0x1c
+#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT	0x1d
+#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH__SHIFT	0x1e
+#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH__SHIFT	0x1f
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED0_MASK	0x00000001L
+#define SDMA0_RELAX_ORDERING_LUT__COPY_MASK	0x00000002L
+#define SDMA0_RELAX_ORDERING_LUT__WRITE_MASK	0x00000004L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED3_MASK	0x00000008L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED4_MASK	0x00000010L
+#define SDMA0_RELAX_ORDERING_LUT__FENCE_MASK	0x00000020L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED76_MASK	0x000000C0L
+#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM_MASK	0x00000100L
+#define SDMA0_RELAX_ORDERING_LUT__COND_EXE_MASK	0x00000200L
+#define SDMA0_RELAX_ORDERING_LUT__ATOMIC_MASK	0x00000400L
+#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL_MASK	0x00000800L
+#define SDMA0_RELAX_ORDERING_LUT__PTEPDE_MASK	0x00001000L
+#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP_MASK	0x00002000L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED_MASK	0x07FFC000L
+#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK	0x08000000L
+#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB_MASK	0x10000000L
+#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL_MASK	0x20000000L
+#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH_MASK	0x40000000L
+#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH_MASK	0x80000000L
+//SDMA0_CHICKEN_BITS_2
+#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT	0x0
+#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK	0x0000000FL
+//SDMA0_STATUS3_REG
+#define SDMA0_STATUS3_REG__CMD_OP_STATUS__SHIFT	0x0
+#define SDMA0_STATUS3_REG__PREV_VM_CMD__SHIFT	0x10
+#define SDMA0_STATUS3_REG__EXCEPTION_IDLE__SHIFT	0x14
+#define SDMA0_STATUS3_REG__CMD_OP_STATUS_MASK	0x0000FFFFL
+#define SDMA0_STATUS3_REG__PREV_VM_CMD_MASK	0x000F0000L
+#define SDMA0_STATUS3_REG__EXCEPTION_IDLE_MASK	0x00100000L
+//SDMA0_PHYSICAL_ADDR_LO
+#define SDMA0_PHYSICAL_ADDR_LO__D_VALID__SHIFT	0x0
+#define SDMA0_PHYSICAL_ADDR_LO__DIRTY__SHIFT	0x1
+#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT	0x2
+#define SDMA0_PHYSICAL_ADDR_LO__ADDR__SHIFT	0xc
+#define SDMA0_PHYSICAL_ADDR_LO__D_VALID_MASK	0x00000001L
+#define SDMA0_PHYSICAL_ADDR_LO__DIRTY_MASK	0x00000002L
+#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID_MASK	0x00000004L
+#define SDMA0_PHYSICAL_ADDR_LO__ADDR_MASK	0xFFFFF000L
+//SDMA0_PHYSICAL_ADDR_HI
+#define SDMA0_PHYSICAL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_PHYSICAL_ADDR_HI__ADDR_MASK	0x0000FFFFL
+//SDMA0_PHASE2_QUANTUM
+#define SDMA0_PHASE2_QUANTUM__UNIT__SHIFT	0x0
+#define SDMA0_PHASE2_QUANTUM__VALUE__SHIFT	0x8
+#define SDMA0_PHASE2_QUANTUM__PREFER__SHIFT	0x1e
+#define SDMA0_PHASE2_QUANTUM__UNIT_MASK	0x0000000FL
+#define SDMA0_PHASE2_QUANTUM__VALUE_MASK	0x00FFFF00L
+#define SDMA0_PHASE2_QUANTUM__PREFER_MASK	0x40000000L
+//SDMA0_ERROR_LOG
+#define SDMA0_ERROR_LOG__OVERRIDE__SHIFT	0x0
+#define SDMA0_ERROR_LOG__STATUS__SHIFT	0x10
+#define SDMA0_ERROR_LOG__OVERRIDE_MASK	0x0000FFFFL
+#define SDMA0_ERROR_LOG__STATUS_MASK	0xFFFF0000L
+//SDMA0_PUB_DUMMY_REG0
+#define SDMA0_PUB_DUMMY_REG0__VALUE__SHIFT	0x0
+#define SDMA0_PUB_DUMMY_REG0__VALUE_MASK	0xFFFFFFFFL
+//SDMA0_PUB_DUMMY_REG1
+#define SDMA0_PUB_DUMMY_REG1__VALUE__SHIFT	0x0
+#define SDMA0_PUB_DUMMY_REG1__VALUE_MASK	0xFFFFFFFFL
+//SDMA0_PUB_DUMMY_REG2
+#define SDMA0_PUB_DUMMY_REG2__VALUE__SHIFT	0x0
+#define SDMA0_PUB_DUMMY_REG2__VALUE_MASK	0xFFFFFFFFL
+//SDMA0_PUB_DUMMY_REG3
+#define SDMA0_PUB_DUMMY_REG3__VALUE__SHIFT	0x0
+#define SDMA0_PUB_DUMMY_REG3__VALUE_MASK	0xFFFFFFFFL
+//SDMA0_F32_COUNTER
+#define SDMA0_F32_COUNTER__VALUE__SHIFT	0x0
+#define SDMA0_F32_COUNTER__VALUE_MASK	0xFFFFFFFFL
+//SDMA0_UNBREAKABLE
+#define SDMA0_UNBREAKABLE__VALUE__SHIFT	0x0
+#define SDMA0_UNBREAKABLE__VALUE_MASK	0x00000001L
+//SDMA0_PERFMON_CNTL
+#define SDMA0_PERFMON_CNTL__PERF_ENABLE0__SHIFT	0x0
+#define SDMA0_PERFMON_CNTL__PERF_CLEAR0__SHIFT	0x1
+#define SDMA0_PERFMON_CNTL__PERF_SEL0__SHIFT	0x2
+#define SDMA0_PERFMON_CNTL__PERF_ENABLE1__SHIFT	0xa
+#define SDMA0_PERFMON_CNTL__PERF_CLEAR1__SHIFT	0xb
+#define SDMA0_PERFMON_CNTL__PERF_SEL1__SHIFT	0xc
+#define SDMA0_PERFMON_CNTL__PERF_ENABLE0_MASK	0x00000001L
+#define SDMA0_PERFMON_CNTL__PERF_CLEAR0_MASK	0x00000002L
+#define SDMA0_PERFMON_CNTL__PERF_SEL0_MASK	0x000003FCL
+#define SDMA0_PERFMON_CNTL__PERF_ENABLE1_MASK	0x00000400L
+#define SDMA0_PERFMON_CNTL__PERF_CLEAR1_MASK	0x00000800L
+#define SDMA0_PERFMON_CNTL__PERF_SEL1_MASK	0x000FF000L
+//SDMA0_PERFCOUNTER0_RESULT
+#define SDMA0_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT	0x0
+#define SDMA0_PERFCOUNTER0_RESULT__PERF_COUNT_MASK	0xFFFFFFFFL
+//SDMA0_PERFCOUNTER1_RESULT
+#define SDMA0_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT	0x0
+#define SDMA0_PERFCOUNTER1_RESULT__PERF_COUNT_MASK	0xFFFFFFFFL
+//SDMA0_PERFCOUNTER_TAG_DELAY_RANGE
+#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW__SHIFT	0x0
+#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH__SHIFT	0xe
+#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW__SHIFT	0x1c
+#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW_MASK	0x00003FFFL
+#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH_MASK	0x0FFFC000L
+#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW_MASK	0x10000000L
+//SDMA0_CRD_CNTL
+#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT	0x7
+#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT	0xd
+#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT_MASK	0x00001F80L
+#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT_MASK	0x0007E000L
+//SDMA0_MMHUB_TRUSTLVL
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG0__SHIFT	0x0
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG1__SHIFT	0x3
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG2__SHIFT	0x6
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG3__SHIFT	0x9
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG4__SHIFT	0xc
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG5__SHIFT	0xf
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG6__SHIFT	0x12
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG7__SHIFT	0x15
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG0_MASK	0x00000007L
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG1_MASK	0x00000038L
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG2_MASK	0x000001C0L
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG3_MASK	0x00000E00L
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG4_MASK	0x00007000L
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG5_MASK	0x00038000L
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG6_MASK	0x001C0000L
+#define SDMA0_MMHUB_TRUSTLVL__SECFLAG7_MASK	0x00E00000L
+//SDMA0_GPU_IOV_VIOLATION_LOG
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT	0x0
+#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT	0x1
+#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT	0x2
+#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT	0x12
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VF__SHIFT	0x13
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID__SHIFT	0x14
+#define SDMA0_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT	0x18
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK	0x00000001L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK	0x00000002L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK	0x0003FFFCL
+#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK	0x00040000L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VF_MASK	0x00080000L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID_MASK	0x00F00000L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK	0xFF000000L
+//SDMA0_ULV_CNTL
+#define SDMA0_ULV_CNTL__HYSTERESIS__SHIFT	0x0
+#define SDMA0_ULV_CNTL__ENTER_ULV_INT__SHIFT	0x1d
+#define SDMA0_ULV_CNTL__EXIT_ULV_INT__SHIFT	0x1e
+#define SDMA0_ULV_CNTL__ULV_STATUS__SHIFT	0x1f
+#define SDMA0_ULV_CNTL__HYSTERESIS_MASK	0x0000001FL
+#define SDMA0_ULV_CNTL__ENTER_ULV_INT_MASK	0x20000000L
+#define SDMA0_ULV_CNTL__EXIT_ULV_INT_MASK	0x40000000L
+#define SDMA0_ULV_CNTL__ULV_STATUS_MASK	0x80000000L
+//SDMA0_EA_DBIT_ADDR_DATA
+#define SDMA0_EA_DBIT_ADDR_DATA__VALUE__SHIFT	0x0
+#define SDMA0_EA_DBIT_ADDR_DATA__VALUE_MASK	0xFFFFFFFFL
+//SDMA0_EA_DBIT_ADDR_INDEX
+#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE__SHIFT	0x0
+#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE_MASK	0x00000007L
+//SDMA0_GFX_RB_CNTL
+#define SDMA0_GFX_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA0_GFX_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA0_GFX_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA0_GFX_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA0_GFX_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA0_GFX_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA0_GFX_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA0_GFX_RB_BASE
+#define SDMA0_GFX_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA0_GFX_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_BASE_HI
+#define SDMA0_GFX_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_GFX_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA0_GFX_RB_RPTR
+#define SDMA0_GFX_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA0_GFX_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_RPTR_HI
+#define SDMA0_GFX_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_GFX_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_WPTR
+#define SDMA0_GFX_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA0_GFX_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_WPTR_HI
+#define SDMA0_GFX_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_GFX_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_WPTR_POLL_CNTL
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA0_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA0_GFX_RB_RPTR_ADDR_HI
+#define SDMA0_GFX_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_GFX_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_RPTR_ADDR_LO
+#define SDMA0_GFX_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_GFX_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_GFX_IB_CNTL
+#define SDMA0_GFX_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA0_GFX_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA0_GFX_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA0_GFX_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA0_GFX_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA0_GFX_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA0_GFX_IB_RPTR
+#define SDMA0_GFX_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA0_GFX_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA0_GFX_IB_OFFSET
+#define SDMA0_GFX_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_GFX_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA0_GFX_IB_BASE_LO
+#define SDMA0_GFX_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA0_GFX_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA0_GFX_IB_BASE_HI
+#define SDMA0_GFX_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_GFX_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_GFX_IB_SIZE
+#define SDMA0_GFX_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA0_GFX_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA0_GFX_SKIP_CNTL
+#define SDMA0_GFX_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA0_GFX_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA0_GFX_CONTEXT_STATUS
+#define SDMA0_GFX_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA0_GFX_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA0_GFX_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA0_GFX_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA0_GFX_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA0_GFX_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA0_GFX_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA0_GFX_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA0_GFX_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA0_GFX_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA0_GFX_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA0_GFX_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA0_GFX_DOORBELL
+#define SDMA0_GFX_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA0_GFX_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA0_GFX_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA0_GFX_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA0_GFX_CONTEXT_CNTL
+#define SDMA0_GFX_CONTEXT_CNTL__RESUME_CTX__SHIFT	0x10
+#define SDMA0_GFX_CONTEXT_CNTL__RESUME_CTX_MASK	0x00010000L
+//SDMA0_GFX_STATUS
+#define SDMA0_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA0_GFX_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA0_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA0_GFX_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA0_GFX_DOORBELL_LOG
+#define SDMA0_GFX_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA0_GFX_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA0_GFX_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA0_GFX_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA0_GFX_WATERMARK
+#define SDMA0_GFX_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA0_GFX_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA0_GFX_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA0_GFX_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA0_GFX_DOORBELL_OFFSET
+#define SDMA0_GFX_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_GFX_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA0_GFX_CSA_ADDR_LO
+#define SDMA0_GFX_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_GFX_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_GFX_CSA_ADDR_HI
+#define SDMA0_GFX_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_GFX_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_GFX_IB_SUB_REMAIN
+#define SDMA0_GFX_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA0_GFX_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA0_GFX_PREEMPT
+#define SDMA0_GFX_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA0_GFX_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA0_GFX_DUMMY_REG
+#define SDMA0_GFX_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA0_GFX_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_GFX_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_GFX_RB_AQL_CNTL
+#define SDMA0_GFX_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA0_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA0_GFX_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA0_GFX_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA0_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA0_GFX_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA0_GFX_MINOR_PTR_UPDATE
+#define SDMA0_GFX_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA0_GFX_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA0_GFX_MIDCMD_DATA0
+#define SDMA0_GFX_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA1
+#define SDMA0_GFX_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA2
+#define SDMA0_GFX_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA3
+#define SDMA0_GFX_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA4
+#define SDMA0_GFX_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA5
+#define SDMA0_GFX_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA6
+#define SDMA0_GFX_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA7
+#define SDMA0_GFX_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_DATA8
+#define SDMA0_GFX_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA0_GFX_MIDCMD_CNTL
+#define SDMA0_GFX_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA0_GFX_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA0_GFX_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA0_GFX_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA0_GFX_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA0_GFX_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA0_GFX_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA0_GFX_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+//SDMA0_PAGE_RB_CNTL
+#define SDMA0_PAGE_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA0_PAGE_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA0_PAGE_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA0_PAGE_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA0_PAGE_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA0_PAGE_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA0_PAGE_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA0_PAGE_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA0_PAGE_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA0_PAGE_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA0_PAGE_RB_BASE
+#define SDMA0_PAGE_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA0_PAGE_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_BASE_HI
+#define SDMA0_PAGE_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_PAGE_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA0_PAGE_RB_RPTR
+#define SDMA0_PAGE_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA0_PAGE_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_RPTR_HI
+#define SDMA0_PAGE_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_PAGE_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_WPTR
+#define SDMA0_PAGE_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA0_PAGE_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_WPTR_HI
+#define SDMA0_PAGE_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_PAGE_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_WPTR_POLL_CNTL
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA0_PAGE_RB_RPTR_ADDR_HI
+#define SDMA0_PAGE_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_PAGE_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_RPTR_ADDR_LO
+#define SDMA0_PAGE_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_PAGE_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_PAGE_IB_CNTL
+#define SDMA0_PAGE_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA0_PAGE_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA0_PAGE_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA0_PAGE_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA0_PAGE_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA0_PAGE_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA0_PAGE_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA0_PAGE_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA0_PAGE_IB_RPTR
+#define SDMA0_PAGE_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA0_PAGE_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA0_PAGE_IB_OFFSET
+#define SDMA0_PAGE_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_PAGE_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA0_PAGE_IB_BASE_LO
+#define SDMA0_PAGE_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA0_PAGE_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA0_PAGE_IB_BASE_HI
+#define SDMA0_PAGE_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_PAGE_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_IB_SIZE
+#define SDMA0_PAGE_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA0_PAGE_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA0_PAGE_SKIP_CNTL
+#define SDMA0_PAGE_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA0_PAGE_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA0_PAGE_CONTEXT_STATUS
+#define SDMA0_PAGE_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA0_PAGE_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA0_PAGE_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA0_PAGE_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA0_PAGE_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA0_PAGE_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA0_PAGE_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA0_PAGE_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA0_PAGE_DOORBELL
+#define SDMA0_PAGE_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA0_PAGE_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA0_PAGE_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA0_PAGE_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA0_PAGE_STATUS
+#define SDMA0_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA0_PAGE_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA0_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA0_PAGE_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA0_PAGE_DOORBELL_LOG
+#define SDMA0_PAGE_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA0_PAGE_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA0_PAGE_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA0_PAGE_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA0_PAGE_WATERMARK
+#define SDMA0_PAGE_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA0_PAGE_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA0_PAGE_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA0_PAGE_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA0_PAGE_DOORBELL_OFFSET
+#define SDMA0_PAGE_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_PAGE_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA0_PAGE_CSA_ADDR_LO
+#define SDMA0_PAGE_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_PAGE_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_PAGE_CSA_ADDR_HI
+#define SDMA0_PAGE_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_PAGE_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_IB_SUB_REMAIN
+#define SDMA0_PAGE_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA0_PAGE_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA0_PAGE_PREEMPT
+#define SDMA0_PAGE_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA0_PAGE_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA0_PAGE_DUMMY_REG
+#define SDMA0_PAGE_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA0_PAGE_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_PAGE_RB_AQL_CNTL
+#define SDMA0_PAGE_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA0_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA0_PAGE_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA0_PAGE_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA0_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA0_PAGE_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA0_PAGE_MINOR_PTR_UPDATE
+#define SDMA0_PAGE_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA0_PAGE_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA0_PAGE_MIDCMD_DATA0
+#define SDMA0_PAGE_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA1
+#define SDMA0_PAGE_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA2
+#define SDMA0_PAGE_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA3
+#define SDMA0_PAGE_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA4
+#define SDMA0_PAGE_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA5
+#define SDMA0_PAGE_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA6
+#define SDMA0_PAGE_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA7
+#define SDMA0_PAGE_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_DATA8
+#define SDMA0_PAGE_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA0_PAGE_MIDCMD_CNTL
+#define SDMA0_PAGE_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA0_PAGE_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA0_PAGE_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA0_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA0_PAGE_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA0_PAGE_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA0_PAGE_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA0_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+//SDMA0_RLC0_RB_CNTL
+#define SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA0_RLC0_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA0_RLC0_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA0_RLC0_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA0_RLC0_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA0_RLC0_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA0_RLC0_RB_BASE
+#define SDMA0_RLC0_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA0_RLC0_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_BASE_HI
+#define SDMA0_RLC0_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC0_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA0_RLC0_RB_RPTR
+#define SDMA0_RLC0_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA0_RLC0_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_RPTR_HI
+#define SDMA0_RLC0_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_RLC0_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_WPTR
+#define SDMA0_RLC0_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA0_RLC0_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_WPTR_HI
+#define SDMA0_RLC0_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_RLC0_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_WPTR_POLL_CNTL
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA0_RLC0_RB_RPTR_ADDR_HI
+#define SDMA0_RLC0_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC0_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_RPTR_ADDR_LO
+#define SDMA0_RLC0_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_RLC0_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_RLC0_IB_CNTL
+#define SDMA0_RLC0_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA0_RLC0_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA0_RLC0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA0_RLC0_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA0_RLC0_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA0_RLC0_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA0_RLC0_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA0_RLC0_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA0_RLC0_IB_RPTR
+#define SDMA0_RLC0_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA0_RLC0_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA0_RLC0_IB_OFFSET
+#define SDMA0_RLC0_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_RLC0_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA0_RLC0_IB_BASE_LO
+#define SDMA0_RLC0_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA0_RLC0_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA0_RLC0_IB_BASE_HI
+#define SDMA0_RLC0_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC0_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_IB_SIZE
+#define SDMA0_RLC0_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA0_RLC0_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA0_RLC0_SKIP_CNTL
+#define SDMA0_RLC0_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA0_RLC0_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA0_RLC0_CONTEXT_STATUS
+#define SDMA0_RLC0_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA0_RLC0_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA0_RLC0_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA0_RLC0_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA0_RLC0_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA0_RLC0_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA0_RLC0_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA0_RLC0_DOORBELL
+#define SDMA0_RLC0_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA0_RLC0_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA0_RLC0_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA0_RLC0_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA0_RLC0_STATUS
+#define SDMA0_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA0_RLC0_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA0_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA0_RLC0_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA0_RLC0_DOORBELL_LOG
+#define SDMA0_RLC0_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA0_RLC0_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA0_RLC0_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA0_RLC0_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA0_RLC0_WATERMARK
+#define SDMA0_RLC0_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA0_RLC0_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA0_RLC0_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA0_RLC0_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA0_RLC0_DOORBELL_OFFSET
+#define SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_RLC0_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA0_RLC0_CSA_ADDR_LO
+#define SDMA0_RLC0_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_RLC0_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_RLC0_CSA_ADDR_HI
+#define SDMA0_RLC0_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC0_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_IB_SUB_REMAIN
+#define SDMA0_RLC0_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA0_RLC0_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA0_RLC0_PREEMPT
+#define SDMA0_RLC0_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA0_RLC0_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA0_RLC0_DUMMY_REG
+#define SDMA0_RLC0_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA0_RLC0_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_RLC0_RB_AQL_CNTL
+#define SDMA0_RLC0_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA0_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA0_RLC0_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA0_RLC0_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA0_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA0_RLC0_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA0_RLC0_MINOR_PTR_UPDATE
+#define SDMA0_RLC0_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA0_RLC0_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA0_RLC0_MIDCMD_DATA0
+#define SDMA0_RLC0_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA1
+#define SDMA0_RLC0_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA2
+#define SDMA0_RLC0_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA3
+#define SDMA0_RLC0_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA4
+#define SDMA0_RLC0_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA5
+#define SDMA0_RLC0_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA6
+#define SDMA0_RLC0_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA7
+#define SDMA0_RLC0_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_DATA8
+#define SDMA0_RLC0_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA0_RLC0_MIDCMD_CNTL
+#define SDMA0_RLC0_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA0_RLC0_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA0_RLC0_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA0_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA0_RLC0_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA0_RLC0_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA0_RLC0_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA0_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+//SDMA0_RLC1_RB_CNTL
+#define SDMA0_RLC1_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA0_RLC1_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA0_RLC1_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA0_RLC1_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA0_RLC1_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA0_RLC1_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA0_RLC1_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA0_RLC1_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA0_RLC1_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA0_RLC1_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA0_RLC1_RB_BASE
+#define SDMA0_RLC1_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA0_RLC1_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_BASE_HI
+#define SDMA0_RLC1_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC1_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA0_RLC1_RB_RPTR
+#define SDMA0_RLC1_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA0_RLC1_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_RPTR_HI
+#define SDMA0_RLC1_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_RLC1_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_WPTR
+#define SDMA0_RLC1_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA0_RLC1_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_WPTR_HI
+#define SDMA0_RLC1_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA0_RLC1_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_WPTR_POLL_CNTL
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA0_RLC1_RB_RPTR_ADDR_HI
+#define SDMA0_RLC1_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC1_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_RPTR_ADDR_LO
+#define SDMA0_RLC1_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_RLC1_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_RLC1_IB_CNTL
+#define SDMA0_RLC1_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA0_RLC1_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA0_RLC1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA0_RLC1_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA0_RLC1_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA0_RLC1_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA0_RLC1_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA0_RLC1_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA0_RLC1_IB_RPTR
+#define SDMA0_RLC1_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA0_RLC1_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA0_RLC1_IB_OFFSET
+#define SDMA0_RLC1_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_RLC1_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA0_RLC1_IB_BASE_LO
+#define SDMA0_RLC1_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA0_RLC1_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA0_RLC1_IB_BASE_HI
+#define SDMA0_RLC1_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC1_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_IB_SIZE
+#define SDMA0_RLC1_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA0_RLC1_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA0_RLC1_SKIP_CNTL
+#define SDMA0_RLC1_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA0_RLC1_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA0_RLC1_CONTEXT_STATUS
+#define SDMA0_RLC1_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA0_RLC1_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA0_RLC1_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA0_RLC1_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA0_RLC1_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA0_RLC1_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA0_RLC1_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA0_RLC1_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA0_RLC1_DOORBELL
+#define SDMA0_RLC1_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA0_RLC1_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA0_RLC1_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA0_RLC1_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA0_RLC1_STATUS
+#define SDMA0_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA0_RLC1_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA0_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA0_RLC1_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA0_RLC1_DOORBELL_LOG
+#define SDMA0_RLC1_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA0_RLC1_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA0_RLC1_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA0_RLC1_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA0_RLC1_WATERMARK
+#define SDMA0_RLC1_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA0_RLC1_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA0_RLC1_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA0_RLC1_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA0_RLC1_DOORBELL_OFFSET
+#define SDMA0_RLC1_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA0_RLC1_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA0_RLC1_CSA_ADDR_LO
+#define SDMA0_RLC1_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_RLC1_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_RLC1_CSA_ADDR_HI
+#define SDMA0_RLC1_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC1_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_IB_SUB_REMAIN
+#define SDMA0_RLC1_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA0_RLC1_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA0_RLC1_PREEMPT
+#define SDMA0_RLC1_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA0_RLC1_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA0_RLC1_DUMMY_REG
+#define SDMA0_RLC1_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA0_RLC1_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA0_RLC1_RB_AQL_CNTL
+#define SDMA0_RLC1_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA0_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA0_RLC1_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA0_RLC1_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA0_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA0_RLC1_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA0_RLC1_MINOR_PTR_UPDATE
+#define SDMA0_RLC1_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA0_RLC1_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA0_RLC1_MIDCMD_DATA0
+#define SDMA0_RLC1_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA1
+#define SDMA0_RLC1_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA2
+#define SDMA0_RLC1_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA3
+#define SDMA0_RLC1_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA4
+#define SDMA0_RLC1_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA5
+#define SDMA0_RLC1_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA6
+#define SDMA0_RLC1_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA7
+#define SDMA0_RLC1_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_DATA8
+#define SDMA0_RLC1_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA0_RLC1_MIDCMD_CNTL
+#define SDMA0_RLC1_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA0_RLC1_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA0_RLC1_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA0_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA0_RLC1_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA0_RLC1_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA0_RLC1_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA0_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_default.h b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_1_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_1_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_1_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/SDMA0/sdma0_4_1_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/sdma0/sdma0_4_1_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_default.h
new file mode 100644
index 000000000000..934733762ddf
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_default.h
@@ -0,0 +1,282 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _sdma1_4_0_DEFAULT_HEADER
+#define _sdma1_4_0_DEFAULT_HEADER
+
+
+// addressBlock: sdma1_sdma1dec
+#define mmSDMA1_UCODE_ADDR_DEFAULT	0x00000000
+#define mmSDMA1_UCODE_DATA_DEFAULT	0x00000000
+#define mmSDMA1_VM_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_VM_CTX_LO_DEFAULT	0x00000000
+#define mmSDMA1_VM_CTX_HI_DEFAULT	0x00000000
+#define mmSDMA1_ACTIVE_FCN_ID_DEFAULT	0x00000000
+#define mmSDMA1_VM_CTX_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_VIRT_RESET_REQ_DEFAULT	0x00000000
+#define mmSDMA1_VF_ENABLE_DEFAULT	0x00000000
+#define mmSDMA1_CONTEXT_REG_TYPE0_DEFAULT	0xfffdf79f
+#define mmSDMA1_CONTEXT_REG_TYPE1_DEFAULT	0x003fbcff
+#define mmSDMA1_CONTEXT_REG_TYPE2_DEFAULT	0x000003ff
+#define mmSDMA1_CONTEXT_REG_TYPE3_DEFAULT	0x00000000
+#define mmSDMA1_PUB_REG_TYPE0_DEFAULT	0x3c000000
+#define mmSDMA1_PUB_REG_TYPE1_DEFAULT	0x30003882
+#define mmSDMA1_PUB_REG_TYPE2_DEFAULT	0x0fc6e880
+#define mmSDMA1_PUB_REG_TYPE3_DEFAULT	0x00000000
+#define mmSDMA1_MMHUB_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_CONTEXT_GROUP_BOUNDARY_DEFAULT	0x00000000
+#define mmSDMA1_POWER_CNTL_DEFAULT	0x0003c000
+#define mmSDMA1_CLK_CTRL_DEFAULT	0xff000100
+#define mmSDMA1_CNTL_DEFAULT	0x00000002
+#define mmSDMA1_CHICKEN_BITS_DEFAULT	0x00831f07
+#define mmSDMA1_GB_ADDR_CONFIG_DEFAULT	0x00100012
+#define mmSDMA1_GB_ADDR_CONFIG_READ_DEFAULT	0x00100012
+#define mmSDMA1_RB_RPTR_FETCH_HI_DEFAULT	0x00000000
+#define mmSDMA1_SEM_WAIT_FAIL_TIMER_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_RB_RPTR_FETCH_DEFAULT	0x00000000
+#define mmSDMA1_IB_OFFSET_FETCH_DEFAULT	0x00000000
+#define mmSDMA1_PROGRAM_DEFAULT	0x00000000
+#define mmSDMA1_STATUS_REG_DEFAULT	0x46dee557
+#define mmSDMA1_STATUS1_REG_DEFAULT	0x000003ff
+#define mmSDMA1_RD_BURST_CNTL_DEFAULT	0x00000003
+#define mmSDMA1_HBM_PAGE_CONFIG_DEFAULT	0x00000000
+#define mmSDMA1_UCODE_CHECKSUM_DEFAULT	0x00000000
+#define mmSDMA1_F32_CNTL_DEFAULT	0x00000001
+#define mmSDMA1_FREEZE_DEFAULT	0x00000000
+#define mmSDMA1_PHASE0_QUANTUM_DEFAULT	0x00010002
+#define mmSDMA1_PHASE1_QUANTUM_DEFAULT	0x00010002
+#define mmSDMA1_EDC_CONFIG_DEFAULT	0x00000002
+#define mmSDMA1_BA_THRESHOLD_DEFAULT	0x03ff03ff
+#define mmSDMA1_ID_DEFAULT	0x00000001
+#define mmSDMA1_VERSION_DEFAULT	0x00000400
+#define mmSDMA1_EDC_COUNTER_DEFAULT	0x00000000
+#define mmSDMA1_EDC_COUNTER_CLEAR_DEFAULT	0x00000000
+#define mmSDMA1_STATUS2_REG_DEFAULT	0x00000001
+#define mmSDMA1_ATOMIC_CNTL_DEFAULT	0x00000200
+#define mmSDMA1_ATOMIC_PREOP_LO_DEFAULT	0x00000000
+#define mmSDMA1_ATOMIC_PREOP_HI_DEFAULT	0x00000000
+#define mmSDMA1_UTCL1_CNTL_DEFAULT	0xd0003019
+#define mmSDMA1_UTCL1_WATERMK_DEFAULT	0xfffbe1fe
+#define mmSDMA1_UTCL1_RD_STATUS_DEFAULT	0x201001ff
+#define mmSDMA1_UTCL1_WR_STATUS_DEFAULT	0x503001ff
+#define mmSDMA1_UTCL1_INV0_DEFAULT	0x00000600
+#define mmSDMA1_UTCL1_INV1_DEFAULT	0x00000000
+#define mmSDMA1_UTCL1_INV2_DEFAULT	0x00000000
+#define mmSDMA1_UTCL1_RD_XNACK0_DEFAULT	0x00000000
+#define mmSDMA1_UTCL1_RD_XNACK1_DEFAULT	0x00000000
+#define mmSDMA1_UTCL1_WR_XNACK0_DEFAULT	0x00000000
+#define mmSDMA1_UTCL1_WR_XNACK1_DEFAULT	0x00000000
+#define mmSDMA1_UTCL1_TIMEOUT_DEFAULT	0x00010001
+#define mmSDMA1_UTCL1_PAGE_DEFAULT	0x000003e0
+#define mmSDMA1_POWER_CNTL_IDLE_DEFAULT	0x06060200
+#define mmSDMA1_RELAX_ORDERING_LUT_DEFAULT	0xc0000006
+#define mmSDMA1_CHICKEN_BITS_2_DEFAULT	0x00000005
+#define mmSDMA1_STATUS3_REG_DEFAULT	0x00100000
+#define mmSDMA1_PHYSICAL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_PHYSICAL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_PHASE2_QUANTUM_DEFAULT	0x00010002
+#define mmSDMA1_ERROR_LOG_DEFAULT	0x0000000f
+#define mmSDMA1_PUB_DUMMY_REG0_DEFAULT	0x00000000
+#define mmSDMA1_PUB_DUMMY_REG1_DEFAULT	0x00000000
+#define mmSDMA1_PUB_DUMMY_REG2_DEFAULT	0x00000000
+#define mmSDMA1_PUB_DUMMY_REG3_DEFAULT	0x00000000
+#define mmSDMA1_F32_COUNTER_DEFAULT	0x00000000
+#define mmSDMA1_UNBREAKABLE_DEFAULT	0x00000000
+#define mmSDMA1_PERFMON_CNTL_DEFAULT	0x000ff7fd
+#define mmSDMA1_PERFCOUNTER0_RESULT_DEFAULT	0x00000000
+#define mmSDMA1_PERFCOUNTER1_RESULT_DEFAULT	0x00000000
+#define mmSDMA1_PERFCOUNTER_TAG_DELAY_RANGE_DEFAULT	0x00640000
+#define mmSDMA1_CRD_CNTL_DEFAULT	0x000085c0
+#define mmSDMA1_MMHUB_TRUSTLVL_DEFAULT	0x00000000
+#define mmSDMA1_GPU_IOV_VIOLATION_LOG_DEFAULT	0x00000000
+#define mmSDMA1_ULV_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_EA_DBIT_ADDR_DATA_DEFAULT	0x00000000
+#define mmSDMA1_EA_DBIT_ADDR_INDEX_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA1_GFX_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA1_GFX_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_GFX_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA1_GFX_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_GFX_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_GFX_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA1_GFX_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_GFX_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA1_GFX_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_GFX_CONTEXT_STATUS_DEFAULT	0x00000005
+#define mmSDMA1_GFX_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA1_GFX_CONTEXT_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_GFX_STATUS_DEFAULT	0x00000000
+#define mmSDMA1_GFX_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA1_GFX_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA1_GFX_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_GFX_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_GFX_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_GFX_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA1_GFX_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA1_GFX_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_GFX_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA1_GFX_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA1_GFX_MIDCMD_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA1_PAGE_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA1_PAGE_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA1_PAGE_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_CONTEXT_STATUS_DEFAULT	0x00000004
+#define mmSDMA1_PAGE_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_STATUS_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA1_PAGE_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA1_PAGE_MIDCMD_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA1_RLC0_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA1_RLC0_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA1_RLC0_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_CONTEXT_STATUS_DEFAULT	0x00000004
+#define mmSDMA1_RLC0_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_STATUS_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA1_RLC0_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA1_RLC0_MIDCMD_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_CNTL_DEFAULT	0x00040000
+#define mmSDMA1_RLC1_RB_BASE_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_RPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_WPTR_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_WPTR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_WPTR_POLL_CNTL_DEFAULT	0x00401000
+#define mmSDMA1_RLC1_RB_RPTR_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_RPTR_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_IB_CNTL_DEFAULT	0x00000100
+#define mmSDMA1_RLC1_IB_RPTR_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_IB_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_IB_BASE_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_IB_BASE_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_IB_SIZE_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_SKIP_CNTL_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_CONTEXT_STATUS_DEFAULT	0x00000004
+#define mmSDMA1_RLC1_DOORBELL_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_STATUS_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_DOORBELL_LOG_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_WATERMARK_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_DOORBELL_OFFSET_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_CSA_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_CSA_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_IB_SUB_REMAIN_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_PREEMPT_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_DUMMY_REG_DEFAULT	0x0000000f
+#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_HI_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_LO_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_RB_AQL_CNTL_DEFAULT	0x00004000
+#define mmSDMA1_RLC1_MINOR_PTR_UPDATE_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA0_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA1_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA2_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA3_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA4_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA5_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA6_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA7_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_DATA8_DEFAULT	0x00000000
+#define mmSDMA1_RLC1_MIDCMD_CNTL_DEFAULT	0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_offset.h
new file mode 100644
index 000000000000..f2c151a7935d
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_offset.h
@@ -0,0 +1,539 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _sdma1_4_0_OFFSET_HEADER
+#define _sdma1_4_0_OFFSET_HEADER
+
+
+
+// addressBlock: sdma1_sdma1dec
+// base address:	0x5180
+#define mmSDMA1_UCODE_ADDR	0x0000
+#define mmSDMA1_UCODE_ADDR_BASE_IDX	0
+#define mmSDMA1_UCODE_DATA	0x0001
+#define mmSDMA1_UCODE_DATA_BASE_IDX	0
+#define mmSDMA1_VM_CNTL	0x0004
+#define mmSDMA1_VM_CNTL_BASE_IDX	0
+#define mmSDMA1_VM_CTX_LO	0x0005
+#define mmSDMA1_VM_CTX_LO_BASE_IDX	0
+#define mmSDMA1_VM_CTX_HI	0x0006
+#define mmSDMA1_VM_CTX_HI_BASE_IDX	0
+#define mmSDMA1_ACTIVE_FCN_ID	0x0007
+#define mmSDMA1_ACTIVE_FCN_ID_BASE_IDX	0
+#define mmSDMA1_VM_CTX_CNTL	0x0008
+#define mmSDMA1_VM_CTX_CNTL_BASE_IDX	0
+#define mmSDMA1_VIRT_RESET_REQ	0x0009
+#define mmSDMA1_VIRT_RESET_REQ_BASE_IDX	0
+#define mmSDMA1_VF_ENABLE	0x000a
+#define mmSDMA1_VF_ENABLE_BASE_IDX	0
+#define mmSDMA1_CONTEXT_REG_TYPE0	0x000b
+#define mmSDMA1_CONTEXT_REG_TYPE0_BASE_IDX	0
+#define mmSDMA1_CONTEXT_REG_TYPE1	0x000c
+#define mmSDMA1_CONTEXT_REG_TYPE1_BASE_IDX	0
+#define mmSDMA1_CONTEXT_REG_TYPE2	0x000d
+#define mmSDMA1_CONTEXT_REG_TYPE2_BASE_IDX	0
+#define mmSDMA1_CONTEXT_REG_TYPE3	0x000e
+#define mmSDMA1_CONTEXT_REG_TYPE3_BASE_IDX	0
+#define mmSDMA1_PUB_REG_TYPE0	0x000f
+#define mmSDMA1_PUB_REG_TYPE0_BASE_IDX	0
+#define mmSDMA1_PUB_REG_TYPE1	0x0010
+#define mmSDMA1_PUB_REG_TYPE1_BASE_IDX	0
+#define mmSDMA1_PUB_REG_TYPE2	0x0011
+#define mmSDMA1_PUB_REG_TYPE2_BASE_IDX	0
+#define mmSDMA1_PUB_REG_TYPE3	0x0012
+#define mmSDMA1_PUB_REG_TYPE3_BASE_IDX	0
+#define mmSDMA1_MMHUB_CNTL	0x0013
+#define mmSDMA1_MMHUB_CNTL_BASE_IDX	0
+#define mmSDMA1_CONTEXT_GROUP_BOUNDARY	0x0019
+#define mmSDMA1_CONTEXT_GROUP_BOUNDARY_BASE_IDX	0
+#define mmSDMA1_POWER_CNTL	0x001a
+#define mmSDMA1_POWER_CNTL_BASE_IDX	0
+#define mmSDMA1_CLK_CTRL	0x001b
+#define mmSDMA1_CLK_CTRL_BASE_IDX	0
+#define mmSDMA1_CNTL	0x001c
+#define mmSDMA1_CNTL_BASE_IDX	0
+#define mmSDMA1_CHICKEN_BITS	0x001d
+#define mmSDMA1_CHICKEN_BITS_BASE_IDX	0
+#define mmSDMA1_GB_ADDR_CONFIG	0x001e
+#define mmSDMA1_GB_ADDR_CONFIG_BASE_IDX	0
+#define mmSDMA1_GB_ADDR_CONFIG_READ	0x001f
+#define mmSDMA1_GB_ADDR_CONFIG_READ_BASE_IDX	0
+#define mmSDMA1_RB_RPTR_FETCH_HI	0x0020
+#define mmSDMA1_RB_RPTR_FETCH_HI_BASE_IDX	0
+#define mmSDMA1_SEM_WAIT_FAIL_TIMER_CNTL	0x0021
+#define mmSDMA1_SEM_WAIT_FAIL_TIMER_CNTL_BASE_IDX	0
+#define mmSDMA1_RB_RPTR_FETCH	0x0022
+#define mmSDMA1_RB_RPTR_FETCH_BASE_IDX	0
+#define mmSDMA1_IB_OFFSET_FETCH	0x0023
+#define mmSDMA1_IB_OFFSET_FETCH_BASE_IDX	0
+#define mmSDMA1_PROGRAM	0x0024
+#define mmSDMA1_PROGRAM_BASE_IDX	0
+#define mmSDMA1_STATUS_REG	0x0025
+#define mmSDMA1_STATUS_REG_BASE_IDX	0
+#define mmSDMA1_STATUS1_REG	0x0026
+#define mmSDMA1_STATUS1_REG_BASE_IDX	0
+#define mmSDMA1_RD_BURST_CNTL	0x0027
+#define mmSDMA1_RD_BURST_CNTL_BASE_IDX	0
+#define mmSDMA1_HBM_PAGE_CONFIG	0x0028
+#define mmSDMA1_HBM_PAGE_CONFIG_BASE_IDX	0
+#define mmSDMA1_UCODE_CHECKSUM	0x0029
+#define mmSDMA1_UCODE_CHECKSUM_BASE_IDX	0
+#define mmSDMA1_F32_CNTL	0x002a
+#define mmSDMA1_F32_CNTL_BASE_IDX	0
+#define mmSDMA1_FREEZE	0x002b
+#define mmSDMA1_FREEZE_BASE_IDX	0
+#define mmSDMA1_PHASE0_QUANTUM	0x002c
+#define mmSDMA1_PHASE0_QUANTUM_BASE_IDX	0
+#define mmSDMA1_PHASE1_QUANTUM	0x002d
+#define mmSDMA1_PHASE1_QUANTUM_BASE_IDX	0
+#define mmSDMA1_EDC_CONFIG	0x0032
+#define mmSDMA1_EDC_CONFIG_BASE_IDX	0
+#define mmSDMA1_BA_THRESHOLD	0x0033
+#define mmSDMA1_BA_THRESHOLD_BASE_IDX	0
+#define mmSDMA1_ID	0x0034
+#define mmSDMA1_ID_BASE_IDX	0
+#define mmSDMA1_VERSION	0x0035
+#define mmSDMA1_VERSION_BASE_IDX	0
+#define mmSDMA1_EDC_COUNTER	0x0036
+#define mmSDMA1_EDC_COUNTER_BASE_IDX	0
+#define mmSDMA1_EDC_COUNTER_CLEAR	0x0037
+#define mmSDMA1_EDC_COUNTER_CLEAR_BASE_IDX	0
+#define mmSDMA1_STATUS2_REG	0x0038
+#define mmSDMA1_STATUS2_REG_BASE_IDX	0
+#define mmSDMA1_ATOMIC_CNTL	0x0039
+#define mmSDMA1_ATOMIC_CNTL_BASE_IDX	0
+#define mmSDMA1_ATOMIC_PREOP_LO	0x003a
+#define mmSDMA1_ATOMIC_PREOP_LO_BASE_IDX	0
+#define mmSDMA1_ATOMIC_PREOP_HI	0x003b
+#define mmSDMA1_ATOMIC_PREOP_HI_BASE_IDX	0
+#define mmSDMA1_UTCL1_CNTL	0x003c
+#define mmSDMA1_UTCL1_CNTL_BASE_IDX	0
+#define mmSDMA1_UTCL1_WATERMK	0x003d
+#define mmSDMA1_UTCL1_WATERMK_BASE_IDX	0
+#define mmSDMA1_UTCL1_RD_STATUS	0x003e
+#define mmSDMA1_UTCL1_RD_STATUS_BASE_IDX	0
+#define mmSDMA1_UTCL1_WR_STATUS	0x003f
+#define mmSDMA1_UTCL1_WR_STATUS_BASE_IDX	0
+#define mmSDMA1_UTCL1_INV0	0x0040
+#define mmSDMA1_UTCL1_INV0_BASE_IDX	0
+#define mmSDMA1_UTCL1_INV1	0x0041
+#define mmSDMA1_UTCL1_INV1_BASE_IDX	0
+#define mmSDMA1_UTCL1_INV2	0x0042
+#define mmSDMA1_UTCL1_INV2_BASE_IDX	0
+#define mmSDMA1_UTCL1_RD_XNACK0	0x0043
+#define mmSDMA1_UTCL1_RD_XNACK0_BASE_IDX	0
+#define mmSDMA1_UTCL1_RD_XNACK1	0x0044
+#define mmSDMA1_UTCL1_RD_XNACK1_BASE_IDX	0
+#define mmSDMA1_UTCL1_WR_XNACK0	0x0045
+#define mmSDMA1_UTCL1_WR_XNACK0_BASE_IDX	0
+#define mmSDMA1_UTCL1_WR_XNACK1	0x0046
+#define mmSDMA1_UTCL1_WR_XNACK1_BASE_IDX	0
+#define mmSDMA1_UTCL1_TIMEOUT	0x0047
+#define mmSDMA1_UTCL1_TIMEOUT_BASE_IDX	0
+#define mmSDMA1_UTCL1_PAGE	0x0048
+#define mmSDMA1_UTCL1_PAGE_BASE_IDX	0
+#define mmSDMA1_POWER_CNTL_IDLE	0x0049
+#define mmSDMA1_POWER_CNTL_IDLE_BASE_IDX	0
+#define mmSDMA1_RELAX_ORDERING_LUT	0x004a
+#define mmSDMA1_RELAX_ORDERING_LUT_BASE_IDX	0
+#define mmSDMA1_CHICKEN_BITS_2	0x004b
+#define mmSDMA1_CHICKEN_BITS_2_BASE_IDX	0
+#define mmSDMA1_STATUS3_REG	0x004c
+#define mmSDMA1_STATUS3_REG_BASE_IDX	0
+#define mmSDMA1_PHYSICAL_ADDR_LO	0x004d
+#define mmSDMA1_PHYSICAL_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_PHYSICAL_ADDR_HI	0x004e
+#define mmSDMA1_PHYSICAL_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_PHASE2_QUANTUM	0x004f
+#define mmSDMA1_PHASE2_QUANTUM_BASE_IDX	0
+#define mmSDMA1_ERROR_LOG	0x0050
+#define mmSDMA1_ERROR_LOG_BASE_IDX	0
+#define mmSDMA1_PUB_DUMMY_REG0	0x0051
+#define mmSDMA1_PUB_DUMMY_REG0_BASE_IDX	0
+#define mmSDMA1_PUB_DUMMY_REG1	0x0052
+#define mmSDMA1_PUB_DUMMY_REG1_BASE_IDX	0
+#define mmSDMA1_PUB_DUMMY_REG2	0x0053
+#define mmSDMA1_PUB_DUMMY_REG2_BASE_IDX	0
+#define mmSDMA1_PUB_DUMMY_REG3	0x0054
+#define mmSDMA1_PUB_DUMMY_REG3_BASE_IDX	0
+#define mmSDMA1_F32_COUNTER	0x0055
+#define mmSDMA1_F32_COUNTER_BASE_IDX	0
+#define mmSDMA1_UNBREAKABLE	0x0056
+#define mmSDMA1_UNBREAKABLE_BASE_IDX	0
+#define mmSDMA1_PERFMON_CNTL	0x0057
+#define mmSDMA1_PERFMON_CNTL_BASE_IDX	0
+#define mmSDMA1_PERFCOUNTER0_RESULT	0x0058
+#define mmSDMA1_PERFCOUNTER0_RESULT_BASE_IDX	0
+#define mmSDMA1_PERFCOUNTER1_RESULT	0x0059
+#define mmSDMA1_PERFCOUNTER1_RESULT_BASE_IDX	0
+#define mmSDMA1_PERFCOUNTER_TAG_DELAY_RANGE	0x005a
+#define mmSDMA1_PERFCOUNTER_TAG_DELAY_RANGE_BASE_IDX	0
+#define mmSDMA1_CRD_CNTL	0x005b
+#define mmSDMA1_CRD_CNTL_BASE_IDX	0
+#define mmSDMA1_MMHUB_TRUSTLVL	0x005c
+#define mmSDMA1_MMHUB_TRUSTLVL_BASE_IDX	0
+#define mmSDMA1_GPU_IOV_VIOLATION_LOG	0x005d
+#define mmSDMA1_GPU_IOV_VIOLATION_LOG_BASE_IDX	0
+#define mmSDMA1_ULV_CNTL	0x005e
+#define mmSDMA1_ULV_CNTL_BASE_IDX	0
+#define mmSDMA1_EA_DBIT_ADDR_DATA	0x0060
+#define mmSDMA1_EA_DBIT_ADDR_DATA_BASE_IDX	0
+#define mmSDMA1_EA_DBIT_ADDR_INDEX	0x0061
+#define mmSDMA1_EA_DBIT_ADDR_INDEX_BASE_IDX	0
+#define mmSDMA1_GFX_RB_CNTL	0x0080
+#define mmSDMA1_GFX_RB_CNTL_BASE_IDX	0
+#define mmSDMA1_GFX_RB_BASE	0x0081
+#define mmSDMA1_GFX_RB_BASE_BASE_IDX	0
+#define mmSDMA1_GFX_RB_BASE_HI	0x0082
+#define mmSDMA1_GFX_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_GFX_RB_RPTR	0x0083
+#define mmSDMA1_GFX_RB_RPTR_BASE_IDX	0
+#define mmSDMA1_GFX_RB_RPTR_HI	0x0084
+#define mmSDMA1_GFX_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA1_GFX_RB_WPTR	0x0085
+#define mmSDMA1_GFX_RB_WPTR_BASE_IDX	0
+#define mmSDMA1_GFX_RB_WPTR_HI	0x0086
+#define mmSDMA1_GFX_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA1_GFX_RB_WPTR_POLL_CNTL	0x0087
+#define mmSDMA1_GFX_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA1_GFX_RB_RPTR_ADDR_HI	0x0088
+#define mmSDMA1_GFX_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_GFX_RB_RPTR_ADDR_LO	0x0089
+#define mmSDMA1_GFX_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_GFX_IB_CNTL	0x008a
+#define mmSDMA1_GFX_IB_CNTL_BASE_IDX	0
+#define mmSDMA1_GFX_IB_RPTR	0x008b
+#define mmSDMA1_GFX_IB_RPTR_BASE_IDX	0
+#define mmSDMA1_GFX_IB_OFFSET	0x008c
+#define mmSDMA1_GFX_IB_OFFSET_BASE_IDX	0
+#define mmSDMA1_GFX_IB_BASE_LO	0x008d
+#define mmSDMA1_GFX_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA1_GFX_IB_BASE_HI	0x008e
+#define mmSDMA1_GFX_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_GFX_IB_SIZE	0x008f
+#define mmSDMA1_GFX_IB_SIZE_BASE_IDX	0
+#define mmSDMA1_GFX_SKIP_CNTL	0x0090
+#define mmSDMA1_GFX_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA1_GFX_CONTEXT_STATUS	0x0091
+#define mmSDMA1_GFX_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA1_GFX_DOORBELL	0x0092
+#define mmSDMA1_GFX_DOORBELL_BASE_IDX	0
+#define mmSDMA1_GFX_CONTEXT_CNTL	0x0093
+#define mmSDMA1_GFX_CONTEXT_CNTL_BASE_IDX	0
+#define mmSDMA1_GFX_STATUS	0x00a8
+#define mmSDMA1_GFX_STATUS_BASE_IDX	0
+#define mmSDMA1_GFX_DOORBELL_LOG	0x00a9
+#define mmSDMA1_GFX_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA1_GFX_WATERMARK	0x00aa
+#define mmSDMA1_GFX_WATERMARK_BASE_IDX	0
+#define mmSDMA1_GFX_DOORBELL_OFFSET	0x00ab
+#define mmSDMA1_GFX_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA1_GFX_CSA_ADDR_LO	0x00ac
+#define mmSDMA1_GFX_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_GFX_CSA_ADDR_HI	0x00ad
+#define mmSDMA1_GFX_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_GFX_IB_SUB_REMAIN	0x00af
+#define mmSDMA1_GFX_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA1_GFX_PREEMPT	0x00b0
+#define mmSDMA1_GFX_PREEMPT_BASE_IDX	0
+#define mmSDMA1_GFX_DUMMY_REG	0x00b1
+#define mmSDMA1_GFX_DUMMY_REG_BASE_IDX	0
+#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_HI	0x00b2
+#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_LO	0x00b3
+#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_GFX_RB_AQL_CNTL	0x00b4
+#define mmSDMA1_GFX_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA1_GFX_MINOR_PTR_UPDATE	0x00b5
+#define mmSDMA1_GFX_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA0	0x00c0
+#define mmSDMA1_GFX_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA1	0x00c1
+#define mmSDMA1_GFX_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA2	0x00c2
+#define mmSDMA1_GFX_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA3	0x00c3
+#define mmSDMA1_GFX_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA4	0x00c4
+#define mmSDMA1_GFX_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA5	0x00c5
+#define mmSDMA1_GFX_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA6	0x00c6
+#define mmSDMA1_GFX_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA7	0x00c7
+#define mmSDMA1_GFX_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_DATA8	0x00c8
+#define mmSDMA1_GFX_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA1_GFX_MIDCMD_CNTL	0x00c9
+#define mmSDMA1_GFX_MIDCMD_CNTL_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_CNTL	0x00e0
+#define mmSDMA1_PAGE_RB_CNTL_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_BASE	0x00e1
+#define mmSDMA1_PAGE_RB_BASE_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_BASE_HI	0x00e2
+#define mmSDMA1_PAGE_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_RPTR	0x00e3
+#define mmSDMA1_PAGE_RB_RPTR_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_RPTR_HI	0x00e4
+#define mmSDMA1_PAGE_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_WPTR	0x00e5
+#define mmSDMA1_PAGE_RB_WPTR_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_WPTR_HI	0x00e6
+#define mmSDMA1_PAGE_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_WPTR_POLL_CNTL	0x00e7
+#define mmSDMA1_PAGE_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_RPTR_ADDR_HI	0x00e8
+#define mmSDMA1_PAGE_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_RPTR_ADDR_LO	0x00e9
+#define mmSDMA1_PAGE_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_PAGE_IB_CNTL	0x00ea
+#define mmSDMA1_PAGE_IB_CNTL_BASE_IDX	0
+#define mmSDMA1_PAGE_IB_RPTR	0x00eb
+#define mmSDMA1_PAGE_IB_RPTR_BASE_IDX	0
+#define mmSDMA1_PAGE_IB_OFFSET	0x00ec
+#define mmSDMA1_PAGE_IB_OFFSET_BASE_IDX	0
+#define mmSDMA1_PAGE_IB_BASE_LO	0x00ed
+#define mmSDMA1_PAGE_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA1_PAGE_IB_BASE_HI	0x00ee
+#define mmSDMA1_PAGE_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_PAGE_IB_SIZE	0x00ef
+#define mmSDMA1_PAGE_IB_SIZE_BASE_IDX	0
+#define mmSDMA1_PAGE_SKIP_CNTL	0x00f0
+#define mmSDMA1_PAGE_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA1_PAGE_CONTEXT_STATUS	0x00f1
+#define mmSDMA1_PAGE_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA1_PAGE_DOORBELL	0x00f2
+#define mmSDMA1_PAGE_DOORBELL_BASE_IDX	0
+#define mmSDMA1_PAGE_STATUS	0x0108
+#define mmSDMA1_PAGE_STATUS_BASE_IDX	0
+#define mmSDMA1_PAGE_DOORBELL_LOG	0x0109
+#define mmSDMA1_PAGE_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA1_PAGE_WATERMARK	0x010a
+#define mmSDMA1_PAGE_WATERMARK_BASE_IDX	0
+#define mmSDMA1_PAGE_DOORBELL_OFFSET	0x010b
+#define mmSDMA1_PAGE_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA1_PAGE_CSA_ADDR_LO	0x010c
+#define mmSDMA1_PAGE_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_PAGE_CSA_ADDR_HI	0x010d
+#define mmSDMA1_PAGE_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_PAGE_IB_SUB_REMAIN	0x010f
+#define mmSDMA1_PAGE_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA1_PAGE_PREEMPT	0x0110
+#define mmSDMA1_PAGE_PREEMPT_BASE_IDX	0
+#define mmSDMA1_PAGE_DUMMY_REG	0x0111
+#define mmSDMA1_PAGE_DUMMY_REG_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_HI	0x0112
+#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_LO	0x0113
+#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_PAGE_RB_AQL_CNTL	0x0114
+#define mmSDMA1_PAGE_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA1_PAGE_MINOR_PTR_UPDATE	0x0115
+#define mmSDMA1_PAGE_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA0	0x0120
+#define mmSDMA1_PAGE_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA1	0x0121
+#define mmSDMA1_PAGE_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA2	0x0122
+#define mmSDMA1_PAGE_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA3	0x0123
+#define mmSDMA1_PAGE_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA4	0x0124
+#define mmSDMA1_PAGE_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA5	0x0125
+#define mmSDMA1_PAGE_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA6	0x0126
+#define mmSDMA1_PAGE_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA7	0x0127
+#define mmSDMA1_PAGE_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_DATA8	0x0128
+#define mmSDMA1_PAGE_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA1_PAGE_MIDCMD_CNTL	0x0129
+#define mmSDMA1_PAGE_MIDCMD_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_CNTL	0x0140
+#define mmSDMA1_RLC0_RB_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_BASE	0x0141
+#define mmSDMA1_RLC0_RB_BASE_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_BASE_HI	0x0142
+#define mmSDMA1_RLC0_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_RPTR	0x0143
+#define mmSDMA1_RLC0_RB_RPTR_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_RPTR_HI	0x0144
+#define mmSDMA1_RLC0_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_WPTR	0x0145
+#define mmSDMA1_RLC0_RB_WPTR_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_WPTR_HI	0x0146
+#define mmSDMA1_RLC0_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_WPTR_POLL_CNTL	0x0147
+#define mmSDMA1_RLC0_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_RPTR_ADDR_HI	0x0148
+#define mmSDMA1_RLC0_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_RPTR_ADDR_LO	0x0149
+#define mmSDMA1_RLC0_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_RLC0_IB_CNTL	0x014a
+#define mmSDMA1_RLC0_IB_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC0_IB_RPTR	0x014b
+#define mmSDMA1_RLC0_IB_RPTR_BASE_IDX	0
+#define mmSDMA1_RLC0_IB_OFFSET	0x014c
+#define mmSDMA1_RLC0_IB_OFFSET_BASE_IDX	0
+#define mmSDMA1_RLC0_IB_BASE_LO	0x014d
+#define mmSDMA1_RLC0_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA1_RLC0_IB_BASE_HI	0x014e
+#define mmSDMA1_RLC0_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_RLC0_IB_SIZE	0x014f
+#define mmSDMA1_RLC0_IB_SIZE_BASE_IDX	0
+#define mmSDMA1_RLC0_SKIP_CNTL	0x0150
+#define mmSDMA1_RLC0_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC0_CONTEXT_STATUS	0x0151
+#define mmSDMA1_RLC0_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA1_RLC0_DOORBELL	0x0152
+#define mmSDMA1_RLC0_DOORBELL_BASE_IDX	0
+#define mmSDMA1_RLC0_STATUS	0x0168
+#define mmSDMA1_RLC0_STATUS_BASE_IDX	0
+#define mmSDMA1_RLC0_DOORBELL_LOG	0x0169
+#define mmSDMA1_RLC0_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA1_RLC0_WATERMARK	0x016a
+#define mmSDMA1_RLC0_WATERMARK_BASE_IDX	0
+#define mmSDMA1_RLC0_DOORBELL_OFFSET	0x016b
+#define mmSDMA1_RLC0_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA1_RLC0_CSA_ADDR_LO	0x016c
+#define mmSDMA1_RLC0_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_RLC0_CSA_ADDR_HI	0x016d
+#define mmSDMA1_RLC0_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_RLC0_IB_SUB_REMAIN	0x016f
+#define mmSDMA1_RLC0_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA1_RLC0_PREEMPT	0x0170
+#define mmSDMA1_RLC0_PREEMPT_BASE_IDX	0
+#define mmSDMA1_RLC0_DUMMY_REG	0x0171
+#define mmSDMA1_RLC0_DUMMY_REG_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_HI	0x0172
+#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_LO	0x0173
+#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_RLC0_RB_AQL_CNTL	0x0174
+#define mmSDMA1_RLC0_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC0_MINOR_PTR_UPDATE	0x0175
+#define mmSDMA1_RLC0_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA0	0x0180
+#define mmSDMA1_RLC0_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA1	0x0181
+#define mmSDMA1_RLC0_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA2	0x0182
+#define mmSDMA1_RLC0_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA3	0x0183
+#define mmSDMA1_RLC0_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA4	0x0184
+#define mmSDMA1_RLC0_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA5	0x0185
+#define mmSDMA1_RLC0_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA6	0x0186
+#define mmSDMA1_RLC0_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA7	0x0187
+#define mmSDMA1_RLC0_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_DATA8	0x0188
+#define mmSDMA1_RLC0_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA1_RLC0_MIDCMD_CNTL	0x0189
+#define mmSDMA1_RLC0_MIDCMD_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_CNTL	0x01a0
+#define mmSDMA1_RLC1_RB_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_BASE	0x01a1
+#define mmSDMA1_RLC1_RB_BASE_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_BASE_HI	0x01a2
+#define mmSDMA1_RLC1_RB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_RPTR	0x01a3
+#define mmSDMA1_RLC1_RB_RPTR_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_RPTR_HI	0x01a4
+#define mmSDMA1_RLC1_RB_RPTR_HI_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_WPTR	0x01a5
+#define mmSDMA1_RLC1_RB_WPTR_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_WPTR_HI	0x01a6
+#define mmSDMA1_RLC1_RB_WPTR_HI_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_WPTR_POLL_CNTL	0x01a7
+#define mmSDMA1_RLC1_RB_WPTR_POLL_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_RPTR_ADDR_HI	0x01a8
+#define mmSDMA1_RLC1_RB_RPTR_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_RPTR_ADDR_LO	0x01a9
+#define mmSDMA1_RLC1_RB_RPTR_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_RLC1_IB_CNTL	0x01aa
+#define mmSDMA1_RLC1_IB_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC1_IB_RPTR	0x01ab
+#define mmSDMA1_RLC1_IB_RPTR_BASE_IDX	0
+#define mmSDMA1_RLC1_IB_OFFSET	0x01ac
+#define mmSDMA1_RLC1_IB_OFFSET_BASE_IDX	0
+#define mmSDMA1_RLC1_IB_BASE_LO	0x01ad
+#define mmSDMA1_RLC1_IB_BASE_LO_BASE_IDX	0
+#define mmSDMA1_RLC1_IB_BASE_HI	0x01ae
+#define mmSDMA1_RLC1_IB_BASE_HI_BASE_IDX	0
+#define mmSDMA1_RLC1_IB_SIZE	0x01af
+#define mmSDMA1_RLC1_IB_SIZE_BASE_IDX	0
+#define mmSDMA1_RLC1_SKIP_CNTL	0x01b0
+#define mmSDMA1_RLC1_SKIP_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC1_CONTEXT_STATUS	0x01b1
+#define mmSDMA1_RLC1_CONTEXT_STATUS_BASE_IDX	0
+#define mmSDMA1_RLC1_DOORBELL	0x01b2
+#define mmSDMA1_RLC1_DOORBELL_BASE_IDX	0
+#define mmSDMA1_RLC1_STATUS	0x01c8
+#define mmSDMA1_RLC1_STATUS_BASE_IDX	0
+#define mmSDMA1_RLC1_DOORBELL_LOG	0x01c9
+#define mmSDMA1_RLC1_DOORBELL_LOG_BASE_IDX	0
+#define mmSDMA1_RLC1_WATERMARK	0x01ca
+#define mmSDMA1_RLC1_WATERMARK_BASE_IDX	0
+#define mmSDMA1_RLC1_DOORBELL_OFFSET	0x01cb
+#define mmSDMA1_RLC1_DOORBELL_OFFSET_BASE_IDX	0
+#define mmSDMA1_RLC1_CSA_ADDR_LO	0x01cc
+#define mmSDMA1_RLC1_CSA_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_RLC1_CSA_ADDR_HI	0x01cd
+#define mmSDMA1_RLC1_CSA_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_RLC1_IB_SUB_REMAIN	0x01cf
+#define mmSDMA1_RLC1_IB_SUB_REMAIN_BASE_IDX	0
+#define mmSDMA1_RLC1_PREEMPT	0x01d0
+#define mmSDMA1_RLC1_PREEMPT_BASE_IDX	0
+#define mmSDMA1_RLC1_DUMMY_REG	0x01d1
+#define mmSDMA1_RLC1_DUMMY_REG_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_HI	0x01d2
+#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_HI_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_LO	0x01d3
+#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_LO_BASE_IDX	0
+#define mmSDMA1_RLC1_RB_AQL_CNTL	0x01d4
+#define mmSDMA1_RLC1_RB_AQL_CNTL_BASE_IDX	0
+#define mmSDMA1_RLC1_MINOR_PTR_UPDATE	0x01d5
+#define mmSDMA1_RLC1_MINOR_PTR_UPDATE_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA0	0x01e0
+#define mmSDMA1_RLC1_MIDCMD_DATA0_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA1	0x01e1
+#define mmSDMA1_RLC1_MIDCMD_DATA1_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA2	0x01e2
+#define mmSDMA1_RLC1_MIDCMD_DATA2_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA3	0x01e3
+#define mmSDMA1_RLC1_MIDCMD_DATA3_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA4	0x01e4
+#define mmSDMA1_RLC1_MIDCMD_DATA4_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA5	0x01e5
+#define mmSDMA1_RLC1_MIDCMD_DATA5_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA6	0x01e6
+#define mmSDMA1_RLC1_MIDCMD_DATA6_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA7	0x01e7
+#define mmSDMA1_RLC1_MIDCMD_DATA7_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_DATA8	0x01e8
+#define mmSDMA1_RLC1_MIDCMD_DATA8_BASE_IDX	0
+#define mmSDMA1_RLC1_MIDCMD_CNTL	0x01e9
+#define mmSDMA1_RLC1_MIDCMD_CNTL_BASE_IDX	0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_sh_mask.h
new file mode 100644
index 000000000000..99849e0dde5e
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/sdma1/sdma1_4_0_sh_mask.h
@@ -0,0 +1,1810 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef _sdma1_4_0_SH_MASK_HEADER
+#define _sdma1_4_0_SH_MASK_HEADER
+
+
+// addressBlock: sdma1_sdma1dec
+//SDMA1_UCODE_ADDR
+#define SDMA1_UCODE_ADDR__VALUE__SHIFT	0x0
+#define SDMA1_UCODE_ADDR__VALUE_MASK	0x00001FFFL
+//SDMA1_UCODE_DATA
+#define SDMA1_UCODE_DATA__VALUE__SHIFT	0x0
+#define SDMA1_UCODE_DATA__VALUE_MASK	0xFFFFFFFFL
+//SDMA1_VM_CNTL
+#define SDMA1_VM_CNTL__CMD__SHIFT	0x0
+#define SDMA1_VM_CNTL__CMD_MASK	0x0000000FL
+//SDMA1_VM_CTX_LO
+#define SDMA1_VM_CTX_LO__ADDR__SHIFT	0x2
+#define SDMA1_VM_CTX_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_VM_CTX_HI
+#define SDMA1_VM_CTX_HI__ADDR__SHIFT	0x0
+#define SDMA1_VM_CTX_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_ACTIVE_FCN_ID
+#define SDMA1_ACTIVE_FCN_ID__VFID__SHIFT	0x0
+#define SDMA1_ACTIVE_FCN_ID__RESERVED__SHIFT	0x4
+#define SDMA1_ACTIVE_FCN_ID__VF__SHIFT	0x1f
+#define SDMA1_ACTIVE_FCN_ID__VFID_MASK	0x0000000FL
+#define SDMA1_ACTIVE_FCN_ID__RESERVED_MASK	0x7FFFFFF0L
+#define SDMA1_ACTIVE_FCN_ID__VF_MASK	0x80000000L
+//SDMA1_VM_CTX_CNTL
+#define SDMA1_VM_CTX_CNTL__PRIV__SHIFT	0x0
+#define SDMA1_VM_CTX_CNTL__VMID__SHIFT	0x4
+#define SDMA1_VM_CTX_CNTL__PRIV_MASK	0x00000001L
+#define SDMA1_VM_CTX_CNTL__VMID_MASK	0x000000F0L
+//SDMA1_VIRT_RESET_REQ
+#define SDMA1_VIRT_RESET_REQ__VF__SHIFT	0x0
+#define SDMA1_VIRT_RESET_REQ__PF__SHIFT	0x1f
+#define SDMA1_VIRT_RESET_REQ__VF_MASK	0x0000FFFFL
+#define SDMA1_VIRT_RESET_REQ__PF_MASK	0x80000000L
+//SDMA1_VF_ENABLE
+#define SDMA1_VF_ENABLE__VF_ENABLE__SHIFT	0x0
+#define SDMA1_VF_ENABLE__VF_ENABLE_MASK	0x00000001L
+//SDMA1_CONTEXT_REG_TYPE0
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_CNTL__SHIFT	0x0
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE__SHIFT	0x1
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE_HI__SHIFT	0x2
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR__SHIFT	0x3
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_HI__SHIFT	0x4
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR__SHIFT	0x5
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_HI__SHIFT	0x6
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_POLL_CNTL__SHIFT	0x7
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_HI__SHIFT	0x8
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_LO__SHIFT	0x9
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_CNTL__SHIFT	0xa
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_RPTR__SHIFT	0xb
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_OFFSET__SHIFT	0xc
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_LO__SHIFT	0xd
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_HI__SHIFT	0xe
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_SIZE__SHIFT	0xf
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_SKIP_CNTL__SHIFT	0x10
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_STATUS__SHIFT	0x11
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_DOORBELL__SHIFT	0x12
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_CNTL__SHIFT	0x13
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_CNTL_MASK	0x00000001L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE_MASK	0x00000002L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE_HI_MASK	0x00000004L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_MASK	0x00000008L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_HI_MASK	0x00000010L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_MASK	0x00000020L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_HI_MASK	0x00000040L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_POLL_CNTL_MASK	0x00000080L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_HI_MASK	0x00000100L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_LO_MASK	0x00000200L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_CNTL_MASK	0x00000400L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_RPTR_MASK	0x00000800L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_OFFSET_MASK	0x00001000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_LO_MASK	0x00002000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_HI_MASK	0x00004000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_SIZE_MASK	0x00008000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_SKIP_CNTL_MASK	0x00010000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_STATUS_MASK	0x00020000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_DOORBELL_MASK	0x00040000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_CNTL_MASK	0x00080000L
+//SDMA1_CONTEXT_REG_TYPE1
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_STATUS__SHIFT	0x8
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_LOG__SHIFT	0x9
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_WATERMARK__SHIFT	0xa
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_OFFSET__SHIFT	0xb
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_LO__SHIFT	0xc
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_HI__SHIFT	0xd
+#define SDMA1_CONTEXT_REG_TYPE1__VOID_REG2__SHIFT	0xe
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_IB_SUB_REMAIN__SHIFT	0xf
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_PREEMPT__SHIFT	0x10
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DUMMY_REG__SHIFT	0x11
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_HI__SHIFT	0x12
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_LO__SHIFT	0x13
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_AQL_CNTL__SHIFT	0x14
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_MINOR_PTR_UPDATE__SHIFT	0x15
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED__SHIFT	0x16
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_STATUS_MASK	0x00000100L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_LOG_MASK	0x00000200L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_WATERMARK_MASK	0x00000400L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_OFFSET_MASK	0x00000800L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_LO_MASK	0x00001000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_HI_MASK	0x00002000L
+#define SDMA1_CONTEXT_REG_TYPE1__VOID_REG2_MASK	0x00004000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_IB_SUB_REMAIN_MASK	0x00008000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_PREEMPT_MASK	0x00010000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DUMMY_REG_MASK	0x00020000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_HI_MASK	0x00040000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_LO_MASK	0x00080000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_AQL_CNTL_MASK	0x00100000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_MINOR_PTR_UPDATE_MASK	0x00200000L
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED_MASK	0xFFC00000L
+//SDMA1_CONTEXT_REG_TYPE2
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA0__SHIFT	0x0
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA1__SHIFT	0x1
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA2__SHIFT	0x2
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA3__SHIFT	0x3
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA4__SHIFT	0x4
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA5__SHIFT	0x5
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA6__SHIFT	0x6
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA7__SHIFT	0x7
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA8__SHIFT	0x8
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_CNTL__SHIFT	0x9
+#define SDMA1_CONTEXT_REG_TYPE2__RESERVED__SHIFT	0xa
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA0_MASK	0x00000001L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA1_MASK	0x00000002L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA2_MASK	0x00000004L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA3_MASK	0x00000008L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA4_MASK	0x00000010L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA5_MASK	0x00000020L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA6_MASK	0x00000040L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA7_MASK	0x00000080L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA8_MASK	0x00000100L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_CNTL_MASK	0x00000200L
+#define SDMA1_CONTEXT_REG_TYPE2__RESERVED_MASK	0xFFFFFC00L
+//SDMA1_CONTEXT_REG_TYPE3
+#define SDMA1_CONTEXT_REG_TYPE3__RESERVED__SHIFT	0x0
+#define SDMA1_CONTEXT_REG_TYPE3__RESERVED_MASK	0xFFFFFFFFL
+//SDMA1_PUB_REG_TYPE0
+#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_ADDR__SHIFT	0x0
+#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_DATA__SHIFT	0x1
+#define SDMA1_PUB_REG_TYPE0__RESERVED3__SHIFT	0x3
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CNTL__SHIFT	0x4
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_LO__SHIFT	0x5
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_HI__SHIFT	0x6
+#define SDMA1_PUB_REG_TYPE0__SDMA1_ACTIVE_FCN_ID__SHIFT	0x7
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_CNTL__SHIFT	0x8
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VIRT_RESET_REQ__SHIFT	0x9
+#define SDMA1_PUB_REG_TYPE0__RESERVED10__SHIFT	0xa
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE0__SHIFT	0xb
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE1__SHIFT	0xc
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE2__SHIFT	0xd
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE3__SHIFT	0xe
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE0__SHIFT	0xf
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE1__SHIFT	0x10
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE2__SHIFT	0x11
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE3__SHIFT	0x12
+#define SDMA1_PUB_REG_TYPE0__SDMA1_MMHUB_CNTL__SHIFT	0x13
+#define SDMA1_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY__SHIFT	0x14
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_GROUP_BOUNDARY__SHIFT	0x19
+#define SDMA1_PUB_REG_TYPE0__SDMA1_POWER_CNTL__SHIFT	0x1a
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CLK_CTRL__SHIFT	0x1b
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CNTL__SHIFT	0x1c
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CHICKEN_BITS__SHIFT	0x1d
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG__SHIFT	0x1e
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_READ__SHIFT	0x1f
+#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_ADDR_MASK	0x00000001L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_DATA_MASK	0x00000002L
+#define SDMA1_PUB_REG_TYPE0__RESERVED3_MASK	0x00000008L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CNTL_MASK	0x00000010L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_LO_MASK	0x00000020L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_HI_MASK	0x00000040L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_ACTIVE_FCN_ID_MASK	0x00000080L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_CNTL_MASK	0x00000100L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_VIRT_RESET_REQ_MASK	0x00000200L
+#define SDMA1_PUB_REG_TYPE0__RESERVED10_MASK	0x00000400L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE0_MASK	0x00000800L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE1_MASK	0x00001000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE2_MASK	0x00002000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE3_MASK	0x00004000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE0_MASK	0x00008000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE1_MASK	0x00010000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE2_MASK	0x00020000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE3_MASK	0x00040000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_MMHUB_CNTL_MASK	0x00080000L
+#define SDMA1_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY_MASK	0x01F00000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_GROUP_BOUNDARY_MASK	0x02000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_POWER_CNTL_MASK	0x04000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CLK_CTRL_MASK	0x08000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CNTL_MASK	0x10000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CHICKEN_BITS_MASK	0x20000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_MASK	0x40000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_READ_MASK	0x80000000L
+//SDMA1_PUB_REG_TYPE1
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_HI__SHIFT	0x0
+#define SDMA1_PUB_REG_TYPE1__SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT	0x1
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH__SHIFT	0x2
+#define SDMA1_PUB_REG_TYPE1__SDMA1_IB_OFFSET_FETCH__SHIFT	0x3
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROGRAM__SHIFT	0x4
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS_REG__SHIFT	0x5
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS1_REG__SHIFT	0x6
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RD_BURST_CNTL__SHIFT	0x7
+#define SDMA1_PUB_REG_TYPE1__SDMA1_HBM_PAGE_CONFIG__SHIFT	0x8
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UCODE_CHECKSUM__SHIFT	0x9
+#define SDMA1_PUB_REG_TYPE1__SDMA1_F32_CNTL__SHIFT	0xa
+#define SDMA1_PUB_REG_TYPE1__SDMA1_FREEZE__SHIFT	0xb
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE0_QUANTUM__SHIFT	0xc
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE1_QUANTUM__SHIFT	0xd
+#define SDMA1_PUB_REG_TYPE1__SDMA_POWER_GATING__SHIFT	0xe
+#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG__SHIFT	0xf
+#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_WRITE__SHIFT	0x10
+#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_READ__SHIFT	0x11
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_CONFIG__SHIFT	0x12
+#define SDMA1_PUB_REG_TYPE1__SDMA1_BA_THRESHOLD__SHIFT	0x13
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ID__SHIFT	0x14
+#define SDMA1_PUB_REG_TYPE1__SDMA1_VERSION__SHIFT	0x15
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER__SHIFT	0x16
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_CLEAR__SHIFT	0x17
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS2_REG__SHIFT	0x18
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_CNTL__SHIFT	0x19
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_LO__SHIFT	0x1a
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_HI__SHIFT	0x1b
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_CNTL__SHIFT	0x1c
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WATERMK__SHIFT	0x1d
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_RD_STATUS__SHIFT	0x1e
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WR_STATUS__SHIFT	0x1f
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_HI_MASK	0x00000001L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_SEM_WAIT_FAIL_TIMER_CNTL_MASK	0x00000002L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_MASK	0x00000004L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_IB_OFFSET_FETCH_MASK	0x00000008L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROGRAM_MASK	0x00000010L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS_REG_MASK	0x00000020L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS1_REG_MASK	0x00000040L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RD_BURST_CNTL_MASK	0x00000080L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_HBM_PAGE_CONFIG_MASK	0x00000100L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UCODE_CHECKSUM_MASK	0x00000200L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_F32_CNTL_MASK	0x00000400L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_FREEZE_MASK	0x00000800L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE0_QUANTUM_MASK	0x00001000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE1_QUANTUM_MASK	0x00002000L
+#define SDMA1_PUB_REG_TYPE1__SDMA_POWER_GATING_MASK	0x00004000L
+#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG_MASK	0x00008000L
+#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_WRITE_MASK	0x00010000L
+#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_READ_MASK	0x00020000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_CONFIG_MASK	0x00040000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_BA_THRESHOLD_MASK	0x00080000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ID_MASK	0x00100000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_VERSION_MASK	0x00200000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_MASK	0x00400000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_CLEAR_MASK	0x00800000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS2_REG_MASK	0x01000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_CNTL_MASK	0x02000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_LO_MASK	0x04000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_HI_MASK	0x08000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_CNTL_MASK	0x10000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WATERMK_MASK	0x20000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_RD_STATUS_MASK	0x40000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WR_STATUS_MASK	0x80000000L
+//SDMA1_PUB_REG_TYPE2
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV0__SHIFT	0x0
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV1__SHIFT	0x1
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV2__SHIFT	0x2
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK0__SHIFT	0x3
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK1__SHIFT	0x4
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK0__SHIFT	0x5
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK1__SHIFT	0x6
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_TIMEOUT__SHIFT	0x7
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_PAGE__SHIFT	0x8
+#define SDMA1_PUB_REG_TYPE2__SDMA1_POWER_CNTL_IDLE__SHIFT	0x9
+#define SDMA1_PUB_REG_TYPE2__SDMA1_RELAX_ORDERING_LUT__SHIFT	0xa
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CHICKEN_BITS_2__SHIFT	0xb
+#define SDMA1_PUB_REG_TYPE2__SDMA1_STATUS3_REG__SHIFT	0xc
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_LO__SHIFT	0xd
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_HI__SHIFT	0xe
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHASE2_QUANTUM__SHIFT	0xf
+#define SDMA1_PUB_REG_TYPE2__SDMA1_ERROR_LOG__SHIFT	0x10
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG0__SHIFT	0x11
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG1__SHIFT	0x12
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG2__SHIFT	0x13
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG3__SHIFT	0x14
+#define SDMA1_PUB_REG_TYPE2__SDMA1_F32_COUNTER__SHIFT	0x15
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UNBREAKABLE__SHIFT	0x16
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFMON_CNTL__SHIFT	0x17
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER0_RESULT__SHIFT	0x18
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER1_RESULT__SHIFT	0x19
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__SHIFT	0x1a
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CRD_CNTL__SHIFT	0x1b
+#define SDMA1_PUB_REG_TYPE2__SDMA1_MMHUB_TRUSTLVL__SHIFT	0x1c
+#define SDMA1_PUB_REG_TYPE2__SDMA1_GPU_IOV_VIOLATION_LOG__SHIFT	0x1d
+#define SDMA1_PUB_REG_TYPE2__SDMA1_ULV_CNTL__SHIFT	0x1e
+#define SDMA1_PUB_REG_TYPE2__RESERVED__SHIFT	0x1f
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV0_MASK	0x00000001L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV1_MASK	0x00000002L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV2_MASK	0x00000004L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK0_MASK	0x00000008L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK1_MASK	0x00000010L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK0_MASK	0x00000020L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK1_MASK	0x00000040L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_TIMEOUT_MASK	0x00000080L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_PAGE_MASK	0x00000100L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_POWER_CNTL_IDLE_MASK	0x00000200L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_RELAX_ORDERING_LUT_MASK	0x00000400L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CHICKEN_BITS_2_MASK	0x00000800L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_STATUS3_REG_MASK	0x00001000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_LO_MASK	0x00002000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_HI_MASK	0x00004000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHASE2_QUANTUM_MASK	0x00008000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_ERROR_LOG_MASK	0x00010000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG0_MASK	0x00020000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG1_MASK	0x00040000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG2_MASK	0x00080000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG3_MASK	0x00100000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_F32_COUNTER_MASK	0x00200000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UNBREAKABLE_MASK	0x00400000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFMON_CNTL_MASK	0x00800000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER0_RESULT_MASK	0x01000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER1_RESULT_MASK	0x02000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER_TAG_DELAY_RANGE_MASK	0x04000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CRD_CNTL_MASK	0x08000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_MMHUB_TRUSTLVL_MASK	0x10000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_GPU_IOV_VIOLATION_LOG_MASK	0x20000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_ULV_CNTL_MASK	0x40000000L
+#define SDMA1_PUB_REG_TYPE2__RESERVED_MASK	0x80000000L
+//SDMA1_PUB_REG_TYPE3
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_DATA__SHIFT	0x0
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_INDEX__SHIFT	0x1
+#define SDMA1_PUB_REG_TYPE3__RESERVED__SHIFT	0x2
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_DATA_MASK	0x00000001L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_INDEX_MASK	0x00000002L
+#define SDMA1_PUB_REG_TYPE3__RESERVED_MASK	0xFFFFFFFCL
+//SDMA1_MMHUB_CNTL
+#define SDMA1_MMHUB_CNTL__UNIT_ID__SHIFT	0x0
+#define SDMA1_MMHUB_CNTL__UNIT_ID_MASK	0x0000003FL
+//SDMA1_CONTEXT_GROUP_BOUNDARY
+#define SDMA1_CONTEXT_GROUP_BOUNDARY__RESERVED__SHIFT	0x0
+#define SDMA1_CONTEXT_GROUP_BOUNDARY__RESERVED_MASK	0xFFFFFFFFL
+//SDMA1_POWER_CNTL
+#define SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE__SHIFT	0x8
+#define SDMA1_POWER_CNTL__MEM_POWER_LS_EN__SHIFT	0x9
+#define SDMA1_POWER_CNTL__MEM_POWER_DS_EN__SHIFT	0xa
+#define SDMA1_POWER_CNTL__MEM_POWER_SD_EN__SHIFT	0xb
+#define SDMA1_POWER_CNTL__MEM_POWER_DELAY__SHIFT	0xc
+#define SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE_MASK	0x00000100L
+#define SDMA1_POWER_CNTL__MEM_POWER_LS_EN_MASK	0x00000200L
+#define SDMA1_POWER_CNTL__MEM_POWER_DS_EN_MASK	0x00000400L
+#define SDMA1_POWER_CNTL__MEM_POWER_SD_EN_MASK	0x00000800L
+#define SDMA1_POWER_CNTL__MEM_POWER_DELAY_MASK	0x003FF000L
+//SDMA1_CLK_CTRL
+#define SDMA1_CLK_CTRL__ON_DELAY__SHIFT	0x0
+#define SDMA1_CLK_CTRL__OFF_HYSTERESIS__SHIFT	0x4
+#define SDMA1_CLK_CTRL__RESERVED__SHIFT	0xc
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE7__SHIFT	0x18
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE6__SHIFT	0x19
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE5__SHIFT	0x1a
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE4__SHIFT	0x1b
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE3__SHIFT	0x1c
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE2__SHIFT	0x1d
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE1__SHIFT	0x1e
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE0__SHIFT	0x1f
+#define SDMA1_CLK_CTRL__ON_DELAY_MASK	0x0000000FL
+#define SDMA1_CLK_CTRL__OFF_HYSTERESIS_MASK	0x00000FF0L
+#define SDMA1_CLK_CTRL__RESERVED_MASK	0x00FFF000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE7_MASK	0x01000000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE6_MASK	0x02000000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE5_MASK	0x04000000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE4_MASK	0x08000000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE3_MASK	0x10000000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE2_MASK	0x20000000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE1_MASK	0x40000000L
+#define SDMA1_CLK_CTRL__SOFT_OVERRIDE0_MASK	0x80000000L
+//SDMA1_CNTL
+#define SDMA1_CNTL__TRAP_ENABLE__SHIFT	0x0
+#define SDMA1_CNTL__UTC_L1_ENABLE__SHIFT	0x1
+#define SDMA1_CNTL__SEM_WAIT_INT_ENABLE__SHIFT	0x2
+#define SDMA1_CNTL__DATA_SWAP_ENABLE__SHIFT	0x3
+#define SDMA1_CNTL__FENCE_SWAP_ENABLE__SHIFT	0x4
+#define SDMA1_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT	0x5
+#define SDMA1_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT	0x11
+#define SDMA1_CNTL__AUTO_CTXSW_ENABLE__SHIFT	0x12
+#define SDMA1_CNTL__CTXEMPTY_INT_ENABLE__SHIFT	0x1c
+#define SDMA1_CNTL__FROZEN_INT_ENABLE__SHIFT	0x1d
+#define SDMA1_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT	0x1e
+#define SDMA1_CNTL__TRAP_ENABLE_MASK	0x00000001L
+#define SDMA1_CNTL__UTC_L1_ENABLE_MASK	0x00000002L
+#define SDMA1_CNTL__SEM_WAIT_INT_ENABLE_MASK	0x00000004L
+#define SDMA1_CNTL__DATA_SWAP_ENABLE_MASK	0x00000008L
+#define SDMA1_CNTL__FENCE_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA1_CNTL__MIDCMD_PREEMPT_ENABLE_MASK	0x00000020L
+#define SDMA1_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK	0x00020000L
+#define SDMA1_CNTL__AUTO_CTXSW_ENABLE_MASK	0x00040000L
+#define SDMA1_CNTL__CTXEMPTY_INT_ENABLE_MASK	0x10000000L
+#define SDMA1_CNTL__FROZEN_INT_ENABLE_MASK	0x20000000L
+#define SDMA1_CNTL__IB_PREEMPT_INT_ENABLE_MASK	0x40000000L
+//SDMA1_CHICKEN_BITS
+#define SDMA1_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE__SHIFT	0x0
+#define SDMA1_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT	0x1
+#define SDMA1_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT	0x2
+#define SDMA1_CHICKEN_BITS__WRITE_BURST_LENGTH__SHIFT	0x8
+#define SDMA1_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE__SHIFT	0xa
+#define SDMA1_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT	0x10
+#define SDMA1_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT	0x11
+#define SDMA1_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT	0x14
+#define SDMA1_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT	0x17
+#define SDMA1_CHICKEN_BITS__TIME_BASED_QOS__SHIFT	0x19
+#define SDMA1_CHICKEN_BITS__CE_AFIFO_WATERMARK__SHIFT	0x1a
+#define SDMA1_CHICKEN_BITS__CE_DFIFO_WATERMARK__SHIFT	0x1c
+#define SDMA1_CHICKEN_BITS__CE_LFIFO_WATERMARK__SHIFT	0x1e
+#define SDMA1_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE_MASK	0x00000001L
+#define SDMA1_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK	0x00000002L
+#define SDMA1_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK	0x00000004L
+#define SDMA1_CHICKEN_BITS__WRITE_BURST_LENGTH_MASK	0x00000300L
+#define SDMA1_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE_MASK	0x00001C00L
+#define SDMA1_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK	0x00010000L
+#define SDMA1_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK	0x00020000L
+#define SDMA1_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK	0x00100000L
+#define SDMA1_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK	0x00800000L
+#define SDMA1_CHICKEN_BITS__TIME_BASED_QOS_MASK	0x02000000L
+#define SDMA1_CHICKEN_BITS__CE_AFIFO_WATERMARK_MASK	0x0C000000L
+#define SDMA1_CHICKEN_BITS__CE_DFIFO_WATERMARK_MASK	0x30000000L
+#define SDMA1_CHICKEN_BITS__CE_LFIFO_WATERMARK_MASK	0xC0000000L
+//SDMA1_GB_ADDR_CONFIG
+#define SDMA1_GB_ADDR_CONFIG__NUM_PIPES__SHIFT	0x0
+#define SDMA1_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT	0x3
+#define SDMA1_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT	0x8
+#define SDMA1_GB_ADDR_CONFIG__NUM_BANKS__SHIFT	0xc
+#define SDMA1_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT	0x13
+#define SDMA1_GB_ADDR_CONFIG__NUM_PIPES_MASK	0x00000007L
+#define SDMA1_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK	0x00000038L
+#define SDMA1_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK	0x00000700L
+#define SDMA1_GB_ADDR_CONFIG__NUM_BANKS_MASK	0x00007000L
+#define SDMA1_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK	0x00180000L
+//SDMA1_GB_ADDR_CONFIG_READ
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT	0x0
+#define SDMA1_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT	0x3
+#define SDMA1_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT	0x8
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT	0xc
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT	0x13
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK	0x00000007L
+#define SDMA1_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK	0x00000038L
+#define SDMA1_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK	0x00000700L
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_BANKS_MASK	0x00007000L
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK	0x00180000L
+//SDMA1_RB_RPTR_FETCH_HI
+#define SDMA1_RB_RPTR_FETCH_HI__OFFSET__SHIFT	0x0
+#define SDMA1_RB_RPTR_FETCH_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_SEM_WAIT_FAIL_TIMER_CNTL
+#define SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT	0x0
+#define SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK	0xFFFFFFFFL
+//SDMA1_RB_RPTR_FETCH
+#define SDMA1_RB_RPTR_FETCH__OFFSET__SHIFT	0x2
+#define SDMA1_RB_RPTR_FETCH__OFFSET_MASK	0xFFFFFFFCL
+//SDMA1_IB_OFFSET_FETCH
+#define SDMA1_IB_OFFSET_FETCH__OFFSET__SHIFT	0x2
+#define SDMA1_IB_OFFSET_FETCH__OFFSET_MASK	0x003FFFFCL
+//SDMA1_PROGRAM
+#define SDMA1_PROGRAM__STREAM__SHIFT	0x0
+#define SDMA1_PROGRAM__STREAM_MASK	0xFFFFFFFFL
+//SDMA1_STATUS_REG
+#define SDMA1_STATUS_REG__IDLE__SHIFT	0x0
+#define SDMA1_STATUS_REG__REG_IDLE__SHIFT	0x1
+#define SDMA1_STATUS_REG__RB_EMPTY__SHIFT	0x2
+#define SDMA1_STATUS_REG__RB_FULL__SHIFT	0x3
+#define SDMA1_STATUS_REG__RB_CMD_IDLE__SHIFT	0x4
+#define SDMA1_STATUS_REG__RB_CMD_FULL__SHIFT	0x5
+#define SDMA1_STATUS_REG__IB_CMD_IDLE__SHIFT	0x6
+#define SDMA1_STATUS_REG__IB_CMD_FULL__SHIFT	0x7
+#define SDMA1_STATUS_REG__BLOCK_IDLE__SHIFT	0x8
+#define SDMA1_STATUS_REG__INSIDE_IB__SHIFT	0x9
+#define SDMA1_STATUS_REG__EX_IDLE__SHIFT	0xa
+#define SDMA1_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE__SHIFT	0xb
+#define SDMA1_STATUS_REG__PACKET_READY__SHIFT	0xc
+#define SDMA1_STATUS_REG__MC_WR_IDLE__SHIFT	0xd
+#define SDMA1_STATUS_REG__SRBM_IDLE__SHIFT	0xe
+#define SDMA1_STATUS_REG__CONTEXT_EMPTY__SHIFT	0xf
+#define SDMA1_STATUS_REG__DELTA_RPTR_FULL__SHIFT	0x10
+#define SDMA1_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT	0x11
+#define SDMA1_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT	0x12
+#define SDMA1_STATUS_REG__MC_RD_IDLE__SHIFT	0x13
+#define SDMA1_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT	0x14
+#define SDMA1_STATUS_REG__MC_RD_RET_STALL__SHIFT	0x15
+#define SDMA1_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT	0x16
+#define SDMA1_STATUS_REG__PREV_CMD_IDLE__SHIFT	0x19
+#define SDMA1_STATUS_REG__SEM_IDLE__SHIFT	0x1a
+#define SDMA1_STATUS_REG__SEM_REQ_STALL__SHIFT	0x1b
+#define SDMA1_STATUS_REG__SEM_RESP_STATE__SHIFT	0x1c
+#define SDMA1_STATUS_REG__INT_IDLE__SHIFT	0x1e
+#define SDMA1_STATUS_REG__INT_REQ_STALL__SHIFT	0x1f
+#define SDMA1_STATUS_REG__IDLE_MASK	0x00000001L
+#define SDMA1_STATUS_REG__REG_IDLE_MASK	0x00000002L
+#define SDMA1_STATUS_REG__RB_EMPTY_MASK	0x00000004L
+#define SDMA1_STATUS_REG__RB_FULL_MASK	0x00000008L
+#define SDMA1_STATUS_REG__RB_CMD_IDLE_MASK	0x00000010L
+#define SDMA1_STATUS_REG__RB_CMD_FULL_MASK	0x00000020L
+#define SDMA1_STATUS_REG__IB_CMD_IDLE_MASK	0x00000040L
+#define SDMA1_STATUS_REG__IB_CMD_FULL_MASK	0x00000080L
+#define SDMA1_STATUS_REG__BLOCK_IDLE_MASK	0x00000100L
+#define SDMA1_STATUS_REG__INSIDE_IB_MASK	0x00000200L
+#define SDMA1_STATUS_REG__EX_IDLE_MASK	0x00000400L
+#define SDMA1_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE_MASK	0x00000800L
+#define SDMA1_STATUS_REG__PACKET_READY_MASK	0x00001000L
+#define SDMA1_STATUS_REG__MC_WR_IDLE_MASK	0x00002000L
+#define SDMA1_STATUS_REG__SRBM_IDLE_MASK	0x00004000L
+#define SDMA1_STATUS_REG__CONTEXT_EMPTY_MASK	0x00008000L
+#define SDMA1_STATUS_REG__DELTA_RPTR_FULL_MASK	0x00010000L
+#define SDMA1_STATUS_REG__RB_MC_RREQ_IDLE_MASK	0x00020000L
+#define SDMA1_STATUS_REG__IB_MC_RREQ_IDLE_MASK	0x00040000L
+#define SDMA1_STATUS_REG__MC_RD_IDLE_MASK	0x00080000L
+#define SDMA1_STATUS_REG__DELTA_RPTR_EMPTY_MASK	0x00100000L
+#define SDMA1_STATUS_REG__MC_RD_RET_STALL_MASK	0x00200000L
+#define SDMA1_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK	0x00400000L
+#define SDMA1_STATUS_REG__PREV_CMD_IDLE_MASK	0x02000000L
+#define SDMA1_STATUS_REG__SEM_IDLE_MASK	0x04000000L
+#define SDMA1_STATUS_REG__SEM_REQ_STALL_MASK	0x08000000L
+#define SDMA1_STATUS_REG__SEM_RESP_STATE_MASK	0x30000000L
+#define SDMA1_STATUS_REG__INT_IDLE_MASK	0x40000000L
+#define SDMA1_STATUS_REG__INT_REQ_STALL_MASK	0x80000000L
+//SDMA1_STATUS1_REG
+#define SDMA1_STATUS1_REG__CE_WREQ_IDLE__SHIFT	0x0
+#define SDMA1_STATUS1_REG__CE_WR_IDLE__SHIFT	0x1
+#define SDMA1_STATUS1_REG__CE_SPLIT_IDLE__SHIFT	0x2
+#define SDMA1_STATUS1_REG__CE_RREQ_IDLE__SHIFT	0x3
+#define SDMA1_STATUS1_REG__CE_OUT_IDLE__SHIFT	0x4
+#define SDMA1_STATUS1_REG__CE_IN_IDLE__SHIFT	0x5
+#define SDMA1_STATUS1_REG__CE_DST_IDLE__SHIFT	0x6
+#define SDMA1_STATUS1_REG__CE_CMD_IDLE__SHIFT	0x9
+#define SDMA1_STATUS1_REG__CE_AFIFO_FULL__SHIFT	0xa
+#define SDMA1_STATUS1_REG__CE_INFO_FULL__SHIFT	0xd
+#define SDMA1_STATUS1_REG__CE_INFO1_FULL__SHIFT	0xe
+#define SDMA1_STATUS1_REG__EX_START__SHIFT	0xf
+#define SDMA1_STATUS1_REG__CE_RD_STALL__SHIFT	0x11
+#define SDMA1_STATUS1_REG__CE_WR_STALL__SHIFT	0x12
+#define SDMA1_STATUS1_REG__CE_WREQ_IDLE_MASK	0x00000001L
+#define SDMA1_STATUS1_REG__CE_WR_IDLE_MASK	0x00000002L
+#define SDMA1_STATUS1_REG__CE_SPLIT_IDLE_MASK	0x00000004L
+#define SDMA1_STATUS1_REG__CE_RREQ_IDLE_MASK	0x00000008L
+#define SDMA1_STATUS1_REG__CE_OUT_IDLE_MASK	0x00000010L
+#define SDMA1_STATUS1_REG__CE_IN_IDLE_MASK	0x00000020L
+#define SDMA1_STATUS1_REG__CE_DST_IDLE_MASK	0x00000040L
+#define SDMA1_STATUS1_REG__CE_CMD_IDLE_MASK	0x00000200L
+#define SDMA1_STATUS1_REG__CE_AFIFO_FULL_MASK	0x00000400L
+#define SDMA1_STATUS1_REG__CE_INFO_FULL_MASK	0x00002000L
+#define SDMA1_STATUS1_REG__CE_INFO1_FULL_MASK	0x00004000L
+#define SDMA1_STATUS1_REG__EX_START_MASK	0x00008000L
+#define SDMA1_STATUS1_REG__CE_RD_STALL_MASK	0x00020000L
+#define SDMA1_STATUS1_REG__CE_WR_STALL_MASK	0x00040000L
+//SDMA1_RD_BURST_CNTL
+#define SDMA1_RD_BURST_CNTL__RD_BURST__SHIFT	0x0
+#define SDMA1_RD_BURST_CNTL__RD_BURST_MASK	0x00000003L
+//SDMA1_HBM_PAGE_CONFIG
+#define SDMA1_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT	0x0
+#define SDMA1_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK	0x00000001L
+//SDMA1_UCODE_CHECKSUM
+#define SDMA1_UCODE_CHECKSUM__DATA__SHIFT	0x0
+#define SDMA1_UCODE_CHECKSUM__DATA_MASK	0xFFFFFFFFL
+//SDMA1_F32_CNTL
+#define SDMA1_F32_CNTL__HALT__SHIFT	0x0
+#define SDMA1_F32_CNTL__STEP__SHIFT	0x1
+#define SDMA1_F32_CNTL__HALT_MASK	0x00000001L
+#define SDMA1_F32_CNTL__STEP_MASK	0x00000002L
+//SDMA1_FREEZE
+#define SDMA1_FREEZE__PREEMPT__SHIFT	0x0
+#define SDMA1_FREEZE__FREEZE__SHIFT	0x4
+#define SDMA1_FREEZE__FROZEN__SHIFT	0x5
+#define SDMA1_FREEZE__F32_FREEZE__SHIFT	0x6
+#define SDMA1_FREEZE__PREEMPT_MASK	0x00000001L
+#define SDMA1_FREEZE__FREEZE_MASK	0x00000010L
+#define SDMA1_FREEZE__FROZEN_MASK	0x00000020L
+#define SDMA1_FREEZE__F32_FREEZE_MASK	0x00000040L
+//SDMA1_PHASE0_QUANTUM
+#define SDMA1_PHASE0_QUANTUM__UNIT__SHIFT	0x0
+#define SDMA1_PHASE0_QUANTUM__VALUE__SHIFT	0x8
+#define SDMA1_PHASE0_QUANTUM__PREFER__SHIFT	0x1e
+#define SDMA1_PHASE0_QUANTUM__UNIT_MASK	0x0000000FL
+#define SDMA1_PHASE0_QUANTUM__VALUE_MASK	0x00FFFF00L
+#define SDMA1_PHASE0_QUANTUM__PREFER_MASK	0x40000000L
+//SDMA1_PHASE1_QUANTUM
+#define SDMA1_PHASE1_QUANTUM__UNIT__SHIFT	0x0
+#define SDMA1_PHASE1_QUANTUM__VALUE__SHIFT	0x8
+#define SDMA1_PHASE1_QUANTUM__PREFER__SHIFT	0x1e
+#define SDMA1_PHASE1_QUANTUM__UNIT_MASK	0x0000000FL
+#define SDMA1_PHASE1_QUANTUM__VALUE_MASK	0x00FFFF00L
+#define SDMA1_PHASE1_QUANTUM__PREFER_MASK	0x40000000L
+//SDMA1_EDC_CONFIG
+#define SDMA1_EDC_CONFIG__DIS_EDC__SHIFT	0x1
+#define SDMA1_EDC_CONFIG__ECC_INT_ENABLE__SHIFT	0x2
+#define SDMA1_EDC_CONFIG__DIS_EDC_MASK	0x00000002L
+#define SDMA1_EDC_CONFIG__ECC_INT_ENABLE_MASK	0x00000004L
+//SDMA1_BA_THRESHOLD
+#define SDMA1_BA_THRESHOLD__READ_THRES__SHIFT	0x0
+#define SDMA1_BA_THRESHOLD__WRITE_THRES__SHIFT	0x10
+#define SDMA1_BA_THRESHOLD__READ_THRES_MASK	0x000003FFL
+#define SDMA1_BA_THRESHOLD__WRITE_THRES_MASK	0x03FF0000L
+//SDMA1_ID
+#define SDMA1_ID__DEVICE_ID__SHIFT	0x0
+#define SDMA1_ID__DEVICE_ID_MASK	0x000000FFL
+//SDMA1_VERSION
+#define SDMA1_VERSION__MINVER__SHIFT	0x0
+#define SDMA1_VERSION__MAJVER__SHIFT	0x8
+#define SDMA1_VERSION__REV__SHIFT	0x10
+#define SDMA1_VERSION__MINVER_MASK	0x0000007FL
+#define SDMA1_VERSION__MAJVER_MASK	0x00007F00L
+#define SDMA1_VERSION__REV_MASK	0x003F0000L
+//SDMA1_EDC_COUNTER
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_DED__SHIFT	0x0
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_SEC__SHIFT	0x1
+#define SDMA1_EDC_COUNTER__SDMA_RB_CMD_BUF_SED__SHIFT	0x2
+#define SDMA1_EDC_COUNTER__SDMA_IB_CMD_BUF_SED__SHIFT	0x3
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED__SHIFT	0x4
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT	0x5
+#define SDMA1_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED__SHIFT	0x6
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT	0x7
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT	0x8
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT	0x9
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT	0xa
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT	0xb
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT	0xc
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT	0xd
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT	0xe
+#define SDMA1_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED__SHIFT	0xf
+#define SDMA1_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT	0x10
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_DED_MASK	0x00000001L
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_SEC_MASK	0x00000002L
+#define SDMA1_EDC_COUNTER__SDMA_RB_CMD_BUF_SED_MASK	0x00000004L
+#define SDMA1_EDC_COUNTER__SDMA_IB_CMD_BUF_SED_MASK	0x00000008L
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED_MASK	0x00000010L
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED_MASK	0x00000020L
+#define SDMA1_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED_MASK	0x00000040L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK	0x00000080L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK	0x00000100L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK	0x00000200L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK	0x00000400L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK	0x00000800L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK	0x00001000L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK	0x00002000L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK	0x00004000L
+#define SDMA1_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED_MASK	0x00008000L
+#define SDMA1_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED_MASK	0x00010000L
+//SDMA1_EDC_COUNTER_CLEAR
+#define SDMA1_EDC_COUNTER_CLEAR__DUMMY__SHIFT	0x0
+#define SDMA1_EDC_COUNTER_CLEAR__DUMMY_MASK	0x00000001L
+//SDMA1_STATUS2_REG
+#define SDMA1_STATUS2_REG__ID__SHIFT	0x0
+#define SDMA1_STATUS2_REG__F32_INSTR_PTR__SHIFT	0x2
+#define SDMA1_STATUS2_REG__CMD_OP__SHIFT	0x10
+#define SDMA1_STATUS2_REG__ID_MASK	0x00000003L
+#define SDMA1_STATUS2_REG__F32_INSTR_PTR_MASK	0x00000FFCL
+#define SDMA1_STATUS2_REG__CMD_OP_MASK	0xFFFF0000L
+//SDMA1_ATOMIC_CNTL
+#define SDMA1_ATOMIC_CNTL__LOOP_TIMER__SHIFT	0x0
+#define SDMA1_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT	0x1f
+#define SDMA1_ATOMIC_CNTL__LOOP_TIMER_MASK	0x7FFFFFFFL
+#define SDMA1_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK	0x80000000L
+//SDMA1_ATOMIC_PREOP_LO
+#define SDMA1_ATOMIC_PREOP_LO__DATA__SHIFT	0x0
+#define SDMA1_ATOMIC_PREOP_LO__DATA_MASK	0xFFFFFFFFL
+//SDMA1_ATOMIC_PREOP_HI
+#define SDMA1_ATOMIC_PREOP_HI__DATA__SHIFT	0x0
+#define SDMA1_ATOMIC_PREOP_HI__DATA_MASK	0xFFFFFFFFL
+//SDMA1_UTCL1_CNTL
+#define SDMA1_UTCL1_CNTL__REDO_ENABLE__SHIFT	0x0
+#define SDMA1_UTCL1_CNTL__REDO_DELAY__SHIFT	0x1
+#define SDMA1_UTCL1_CNTL__REDO_WATERMK__SHIFT	0xb
+#define SDMA1_UTCL1_CNTL__INVACK_DELAY__SHIFT	0xe
+#define SDMA1_UTCL1_CNTL__REQL2_CREDIT__SHIFT	0x18
+#define SDMA1_UTCL1_CNTL__VADDR_WATERMK__SHIFT	0x1d
+#define SDMA1_UTCL1_CNTL__REDO_ENABLE_MASK	0x00000001L
+#define SDMA1_UTCL1_CNTL__REDO_DELAY_MASK	0x000007FEL
+#define SDMA1_UTCL1_CNTL__REDO_WATERMK_MASK	0x00003800L
+#define SDMA1_UTCL1_CNTL__INVACK_DELAY_MASK	0x00FFC000L
+#define SDMA1_UTCL1_CNTL__REQL2_CREDIT_MASK	0x1F000000L
+#define SDMA1_UTCL1_CNTL__VADDR_WATERMK_MASK	0xE0000000L
+//SDMA1_UTCL1_WATERMK
+#define SDMA1_UTCL1_WATERMK__REQMC_WATERMK__SHIFT	0x0
+#define SDMA1_UTCL1_WATERMK__REQPG_WATERMK__SHIFT	0xa
+#define SDMA1_UTCL1_WATERMK__INVREQ_WATERMK__SHIFT	0x12
+#define SDMA1_UTCL1_WATERMK__XNACK_WATERMK__SHIFT	0x1a
+#define SDMA1_UTCL1_WATERMK__REQMC_WATERMK_MASK	0x000003FFL
+#define SDMA1_UTCL1_WATERMK__REQPG_WATERMK_MASK	0x0003FC00L
+#define SDMA1_UTCL1_WATERMK__INVREQ_WATERMK_MASK	0x03FC0000L
+#define SDMA1_UTCL1_WATERMK__XNACK_WATERMK_MASK	0xFC000000L
+//SDMA1_UTCL1_RD_STATUS
+#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT	0x0
+#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT	0x1
+#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY__SHIFT	0x2
+#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT	0x3
+#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT	0x4
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT	0x5
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT	0x6
+#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT	0x7
+#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT	0x8
+#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT	0x9
+#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL__SHIFT	0xa
+#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL__SHIFT	0xb
+#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT	0xc
+#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT	0xd
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL__SHIFT	0xe
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT	0xf
+#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT	0x10
+#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT	0x11
+#define SDMA1_UTCL1_RD_STATUS__PAGE_FAULT__SHIFT	0x12
+#define SDMA1_UTCL1_RD_STATUS__PAGE_NULL__SHIFT	0x13
+#define SDMA1_UTCL1_RD_STATUS__REQL2_IDLE__SHIFT	0x14
+#define SDMA1_UTCL1_RD_STATUS__CE_L1_STALL__SHIFT	0x15
+#define SDMA1_UTCL1_RD_STATUS__NEXT_RD_VECTOR__SHIFT	0x16
+#define SDMA1_UTCL1_RD_STATUS__MERGE_STATE__SHIFT	0x1a
+#define SDMA1_UTCL1_RD_STATUS__ADDR_RD_RTR__SHIFT	0x1d
+#define SDMA1_UTCL1_RD_STATUS__WPTR_POLLING__SHIFT	0x1e
+#define SDMA1_UTCL1_RD_STATUS__INVREQ_SIZE__SHIFT	0x1f
+#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK	0x00000001L
+#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY_MASK	0x00000002L
+#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY_MASK	0x00000004L
+#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK	0x00000008L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK	0x00000010L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY_MASK	0x00000020L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK	0x00000040L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK	0x00000080L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK	0x00000100L
+#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK	0x00000200L
+#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL_MASK	0x00000400L
+#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL_MASK	0x00000800L
+#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL_MASK	0x00001000L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK	0x00002000L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL_MASK	0x00004000L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK	0x00008000L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL_MASK	0x00010000L
+#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL_MASK	0x00020000L
+#define SDMA1_UTCL1_RD_STATUS__PAGE_FAULT_MASK	0x00040000L
+#define SDMA1_UTCL1_RD_STATUS__PAGE_NULL_MASK	0x00080000L
+#define SDMA1_UTCL1_RD_STATUS__REQL2_IDLE_MASK	0x00100000L
+#define SDMA1_UTCL1_RD_STATUS__CE_L1_STALL_MASK	0x00200000L
+#define SDMA1_UTCL1_RD_STATUS__NEXT_RD_VECTOR_MASK	0x03C00000L
+#define SDMA1_UTCL1_RD_STATUS__MERGE_STATE_MASK	0x1C000000L
+#define SDMA1_UTCL1_RD_STATUS__ADDR_RD_RTR_MASK	0x20000000L
+#define SDMA1_UTCL1_RD_STATUS__WPTR_POLLING_MASK	0x40000000L
+#define SDMA1_UTCL1_RD_STATUS__INVREQ_SIZE_MASK	0x80000000L
+//SDMA1_UTCL1_WR_STATUS
+#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT	0x0
+#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT	0x1
+#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY__SHIFT	0x2
+#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT	0x3
+#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT	0x4
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT	0x5
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT	0x6
+#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT	0x7
+#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT	0x8
+#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT	0x9
+#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL__SHIFT	0xa
+#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL__SHIFT	0xb
+#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT	0xc
+#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT	0xd
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL__SHIFT	0xe
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT	0xf
+#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT	0x10
+#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT	0x11
+#define SDMA1_UTCL1_WR_STATUS__PAGE_FAULT__SHIFT	0x12
+#define SDMA1_UTCL1_WR_STATUS__PAGE_NULL__SHIFT	0x13
+#define SDMA1_UTCL1_WR_STATUS__REQL2_IDLE__SHIFT	0x14
+#define SDMA1_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT	0x15
+#define SDMA1_UTCL1_WR_STATUS__NEXT_WR_VECTOR__SHIFT	0x16
+#define SDMA1_UTCL1_WR_STATUS__MERGE_STATE__SHIFT	0x19
+#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY__SHIFT	0x1c
+#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL__SHIFT	0x1d
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY__SHIFT	0x1e
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL__SHIFT	0x1f
+#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK	0x00000001L
+#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY_MASK	0x00000002L
+#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY_MASK	0x00000004L
+#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK	0x00000008L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK	0x00000010L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY_MASK	0x00000020L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK	0x00000040L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK	0x00000080L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK	0x00000100L
+#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK	0x00000200L
+#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL_MASK	0x00000400L
+#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL_MASK	0x00000800L
+#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL_MASK	0x00001000L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK	0x00002000L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL_MASK	0x00004000L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK	0x00008000L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL_MASK	0x00010000L
+#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL_MASK	0x00020000L
+#define SDMA1_UTCL1_WR_STATUS__PAGE_FAULT_MASK	0x00040000L
+#define SDMA1_UTCL1_WR_STATUS__PAGE_NULL_MASK	0x00080000L
+#define SDMA1_UTCL1_WR_STATUS__REQL2_IDLE_MASK	0x00100000L
+#define SDMA1_UTCL1_WR_STATUS__F32_WR_RTR_MASK	0x00200000L
+#define SDMA1_UTCL1_WR_STATUS__NEXT_WR_VECTOR_MASK	0x01C00000L
+#define SDMA1_UTCL1_WR_STATUS__MERGE_STATE_MASK	0x0E000000L
+#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY_MASK	0x10000000L
+#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL_MASK	0x20000000L
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY_MASK	0x40000000L
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL_MASK	0x80000000L
+//SDMA1_UTCL1_INV0
+#define SDMA1_UTCL1_INV0__INV_MIDDLE__SHIFT	0x0
+#define SDMA1_UTCL1_INV0__RD_TIMEOUT__SHIFT	0x1
+#define SDMA1_UTCL1_INV0__WR_TIMEOUT__SHIFT	0x2
+#define SDMA1_UTCL1_INV0__RD_IN_INVADR__SHIFT	0x3
+#define SDMA1_UTCL1_INV0__WR_IN_INVADR__SHIFT	0x4
+#define SDMA1_UTCL1_INV0__PAGE_NULL_SW__SHIFT	0x5
+#define SDMA1_UTCL1_INV0__XNACK_IS_INVADR__SHIFT	0x6
+#define SDMA1_UTCL1_INV0__INVREQ_ENABLE__SHIFT	0x7
+#define SDMA1_UTCL1_INV0__NACK_TIMEOUT_SW__SHIFT	0x8
+#define SDMA1_UTCL1_INV0__NFLUSH_INV_IDLE__SHIFT	0x9
+#define SDMA1_UTCL1_INV0__FLUSH_INV_IDLE__SHIFT	0xa
+#define SDMA1_UTCL1_INV0__INV_FLUSHTYPE__SHIFT	0xb
+#define SDMA1_UTCL1_INV0__INV_VMID_VEC__SHIFT	0xc
+#define SDMA1_UTCL1_INV0__INV_ADDR_HI__SHIFT	0x1c
+#define SDMA1_UTCL1_INV0__INV_MIDDLE_MASK	0x00000001L
+#define SDMA1_UTCL1_INV0__RD_TIMEOUT_MASK	0x00000002L
+#define SDMA1_UTCL1_INV0__WR_TIMEOUT_MASK	0x00000004L
+#define SDMA1_UTCL1_INV0__RD_IN_INVADR_MASK	0x00000008L
+#define SDMA1_UTCL1_INV0__WR_IN_INVADR_MASK	0x00000010L
+#define SDMA1_UTCL1_INV0__PAGE_NULL_SW_MASK	0x00000020L
+#define SDMA1_UTCL1_INV0__XNACK_IS_INVADR_MASK	0x00000040L
+#define SDMA1_UTCL1_INV0__INVREQ_ENABLE_MASK	0x00000080L
+#define SDMA1_UTCL1_INV0__NACK_TIMEOUT_SW_MASK	0x00000100L
+#define SDMA1_UTCL1_INV0__NFLUSH_INV_IDLE_MASK	0x00000200L
+#define SDMA1_UTCL1_INV0__FLUSH_INV_IDLE_MASK	0x00000400L
+#define SDMA1_UTCL1_INV0__INV_FLUSHTYPE_MASK	0x00000800L
+#define SDMA1_UTCL1_INV0__INV_VMID_VEC_MASK	0x0FFFF000L
+#define SDMA1_UTCL1_INV0__INV_ADDR_HI_MASK	0xF0000000L
+//SDMA1_UTCL1_INV1
+#define SDMA1_UTCL1_INV1__INV_ADDR_LO__SHIFT	0x0
+#define SDMA1_UTCL1_INV1__INV_ADDR_LO_MASK	0xFFFFFFFFL
+//SDMA1_UTCL1_INV2
+#define SDMA1_UTCL1_INV2__INV_NFLUSH_VMID_VEC__SHIFT	0x0
+#define SDMA1_UTCL1_INV2__INV_NFLUSH_VMID_VEC_MASK	0xFFFFFFFFL
+//SDMA1_UTCL1_RD_XNACK0
+#define SDMA1_UTCL1_RD_XNACK0__XNACK_ADDR_LO__SHIFT	0x0
+#define SDMA1_UTCL1_RD_XNACK0__XNACK_ADDR_LO_MASK	0xFFFFFFFFL
+//SDMA1_UTCL1_RD_XNACK1
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_ADDR_HI__SHIFT	0x0
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_VMID__SHIFT	0x4
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_VECTOR__SHIFT	0x8
+#define SDMA1_UTCL1_RD_XNACK1__IS_XNACK__SHIFT	0x1a
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_ADDR_HI_MASK	0x0000000FL
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_VMID_MASK	0x000000F0L
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_VECTOR_MASK	0x03FFFF00L
+#define SDMA1_UTCL1_RD_XNACK1__IS_XNACK_MASK	0x0C000000L
+//SDMA1_UTCL1_WR_XNACK0
+#define SDMA1_UTCL1_WR_XNACK0__XNACK_ADDR_LO__SHIFT	0x0
+#define SDMA1_UTCL1_WR_XNACK0__XNACK_ADDR_LO_MASK	0xFFFFFFFFL
+//SDMA1_UTCL1_WR_XNACK1
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_ADDR_HI__SHIFT	0x0
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_VMID__SHIFT	0x4
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_VECTOR__SHIFT	0x8
+#define SDMA1_UTCL1_WR_XNACK1__IS_XNACK__SHIFT	0x1a
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_ADDR_HI_MASK	0x0000000FL
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_VMID_MASK	0x000000F0L
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_VECTOR_MASK	0x03FFFF00L
+#define SDMA1_UTCL1_WR_XNACK1__IS_XNACK_MASK	0x0C000000L
+//SDMA1_UTCL1_TIMEOUT
+#define SDMA1_UTCL1_TIMEOUT__RD_XNACK_LIMIT__SHIFT	0x0
+#define SDMA1_UTCL1_TIMEOUT__WR_XNACK_LIMIT__SHIFT	0x10
+#define SDMA1_UTCL1_TIMEOUT__RD_XNACK_LIMIT_MASK	0x0000FFFFL
+#define SDMA1_UTCL1_TIMEOUT__WR_XNACK_LIMIT_MASK	0xFFFF0000L
+//SDMA1_UTCL1_PAGE
+#define SDMA1_UTCL1_PAGE__VM_HOLE__SHIFT	0x0
+#define SDMA1_UTCL1_PAGE__REQ_TYPE__SHIFT	0x1
+#define SDMA1_UTCL1_PAGE__USE_MTYPE__SHIFT	0x6
+#define SDMA1_UTCL1_PAGE__USE_PT_SNOOP__SHIFT	0x9
+#define SDMA1_UTCL1_PAGE__VM_HOLE_MASK	0x00000001L
+#define SDMA1_UTCL1_PAGE__REQ_TYPE_MASK	0x0000001EL
+#define SDMA1_UTCL1_PAGE__USE_MTYPE_MASK	0x000001C0L
+#define SDMA1_UTCL1_PAGE__USE_PT_SNOOP_MASK	0x00000200L
+//SDMA1_POWER_CNTL_IDLE
+#define SDMA1_POWER_CNTL_IDLE__DELAY0__SHIFT	0x0
+#define SDMA1_POWER_CNTL_IDLE__DELAY1__SHIFT	0x10
+#define SDMA1_POWER_CNTL_IDLE__DELAY2__SHIFT	0x18
+#define SDMA1_POWER_CNTL_IDLE__DELAY0_MASK	0x0000FFFFL
+#define SDMA1_POWER_CNTL_IDLE__DELAY1_MASK	0x00FF0000L
+#define SDMA1_POWER_CNTL_IDLE__DELAY2_MASK	0xFF000000L
+//SDMA1_RELAX_ORDERING_LUT
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED0__SHIFT	0x0
+#define SDMA1_RELAX_ORDERING_LUT__COPY__SHIFT	0x1
+#define SDMA1_RELAX_ORDERING_LUT__WRITE__SHIFT	0x2
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED3__SHIFT	0x3
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED4__SHIFT	0x4
+#define SDMA1_RELAX_ORDERING_LUT__FENCE__SHIFT	0x5
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED76__SHIFT	0x6
+#define SDMA1_RELAX_ORDERING_LUT__POLL_MEM__SHIFT	0x8
+#define SDMA1_RELAX_ORDERING_LUT__COND_EXE__SHIFT	0x9
+#define SDMA1_RELAX_ORDERING_LUT__ATOMIC__SHIFT	0xa
+#define SDMA1_RELAX_ORDERING_LUT__CONST_FILL__SHIFT	0xb
+#define SDMA1_RELAX_ORDERING_LUT__PTEPDE__SHIFT	0xc
+#define SDMA1_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT	0xd
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED__SHIFT	0xe
+#define SDMA1_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT	0x1b
+#define SDMA1_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT	0x1c
+#define SDMA1_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT	0x1d
+#define SDMA1_RELAX_ORDERING_LUT__IB_FETCH__SHIFT	0x1e
+#define SDMA1_RELAX_ORDERING_LUT__RB_FETCH__SHIFT	0x1f
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED0_MASK	0x00000001L
+#define SDMA1_RELAX_ORDERING_LUT__COPY_MASK	0x00000002L
+#define SDMA1_RELAX_ORDERING_LUT__WRITE_MASK	0x00000004L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED3_MASK	0x00000008L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED4_MASK	0x00000010L
+#define SDMA1_RELAX_ORDERING_LUT__FENCE_MASK	0x00000020L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED76_MASK	0x000000C0L
+#define SDMA1_RELAX_ORDERING_LUT__POLL_MEM_MASK	0x00000100L
+#define SDMA1_RELAX_ORDERING_LUT__COND_EXE_MASK	0x00000200L
+#define SDMA1_RELAX_ORDERING_LUT__ATOMIC_MASK	0x00000400L
+#define SDMA1_RELAX_ORDERING_LUT__CONST_FILL_MASK	0x00000800L
+#define SDMA1_RELAX_ORDERING_LUT__PTEPDE_MASK	0x00001000L
+#define SDMA1_RELAX_ORDERING_LUT__TIMESTAMP_MASK	0x00002000L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED_MASK	0x07FFC000L
+#define SDMA1_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK	0x08000000L
+#define SDMA1_RELAX_ORDERING_LUT__RPTR_WRB_MASK	0x10000000L
+#define SDMA1_RELAX_ORDERING_LUT__WPTR_POLL_MASK	0x20000000L
+#define SDMA1_RELAX_ORDERING_LUT__IB_FETCH_MASK	0x40000000L
+#define SDMA1_RELAX_ORDERING_LUT__RB_FETCH_MASK	0x80000000L
+//SDMA1_CHICKEN_BITS_2
+#define SDMA1_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT	0x0
+#define SDMA1_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK	0x0000000FL
+//SDMA1_STATUS3_REG
+#define SDMA1_STATUS3_REG__CMD_OP_STATUS__SHIFT	0x0
+#define SDMA1_STATUS3_REG__PREV_VM_CMD__SHIFT	0x10
+#define SDMA1_STATUS3_REG__EXCEPTION_IDLE__SHIFT	0x14
+#define SDMA1_STATUS3_REG__CMD_OP_STATUS_MASK	0x0000FFFFL
+#define SDMA1_STATUS3_REG__PREV_VM_CMD_MASK	0x000F0000L
+#define SDMA1_STATUS3_REG__EXCEPTION_IDLE_MASK	0x00100000L
+//SDMA1_PHYSICAL_ADDR_LO
+#define SDMA1_PHYSICAL_ADDR_LO__D_VALID__SHIFT	0x0
+#define SDMA1_PHYSICAL_ADDR_LO__DIRTY__SHIFT	0x1
+#define SDMA1_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT	0x2
+#define SDMA1_PHYSICAL_ADDR_LO__ADDR__SHIFT	0xc
+#define SDMA1_PHYSICAL_ADDR_LO__D_VALID_MASK	0x00000001L
+#define SDMA1_PHYSICAL_ADDR_LO__DIRTY_MASK	0x00000002L
+#define SDMA1_PHYSICAL_ADDR_LO__PHY_VALID_MASK	0x00000004L
+#define SDMA1_PHYSICAL_ADDR_LO__ADDR_MASK	0xFFFFF000L
+//SDMA1_PHYSICAL_ADDR_HI
+#define SDMA1_PHYSICAL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_PHYSICAL_ADDR_HI__ADDR_MASK	0x0000FFFFL
+//SDMA1_PHASE2_QUANTUM
+#define SDMA1_PHASE2_QUANTUM__UNIT__SHIFT	0x0
+#define SDMA1_PHASE2_QUANTUM__VALUE__SHIFT	0x8
+#define SDMA1_PHASE2_QUANTUM__PREFER__SHIFT	0x1e
+#define SDMA1_PHASE2_QUANTUM__UNIT_MASK	0x0000000FL
+#define SDMA1_PHASE2_QUANTUM__VALUE_MASK	0x00FFFF00L
+#define SDMA1_PHASE2_QUANTUM__PREFER_MASK	0x40000000L
+//SDMA1_ERROR_LOG
+#define SDMA1_ERROR_LOG__OVERRIDE__SHIFT	0x0
+#define SDMA1_ERROR_LOG__STATUS__SHIFT	0x10
+#define SDMA1_ERROR_LOG__OVERRIDE_MASK	0x0000FFFFL
+#define SDMA1_ERROR_LOG__STATUS_MASK	0xFFFF0000L
+//SDMA1_PUB_DUMMY_REG0
+#define SDMA1_PUB_DUMMY_REG0__VALUE__SHIFT	0x0
+#define SDMA1_PUB_DUMMY_REG0__VALUE_MASK	0xFFFFFFFFL
+//SDMA1_PUB_DUMMY_REG1
+#define SDMA1_PUB_DUMMY_REG1__VALUE__SHIFT	0x0
+#define SDMA1_PUB_DUMMY_REG1__VALUE_MASK	0xFFFFFFFFL
+//SDMA1_PUB_DUMMY_REG2
+#define SDMA1_PUB_DUMMY_REG2__VALUE__SHIFT	0x0
+#define SDMA1_PUB_DUMMY_REG2__VALUE_MASK	0xFFFFFFFFL
+//SDMA1_PUB_DUMMY_REG3
+#define SDMA1_PUB_DUMMY_REG3__VALUE__SHIFT	0x0
+#define SDMA1_PUB_DUMMY_REG3__VALUE_MASK	0xFFFFFFFFL
+//SDMA1_F32_COUNTER
+#define SDMA1_F32_COUNTER__VALUE__SHIFT	0x0
+#define SDMA1_F32_COUNTER__VALUE_MASK	0xFFFFFFFFL
+//SDMA1_UNBREAKABLE
+#define SDMA1_UNBREAKABLE__VALUE__SHIFT	0x0
+#define SDMA1_UNBREAKABLE__VALUE_MASK	0x00000001L
+//SDMA1_PERFMON_CNTL
+#define SDMA1_PERFMON_CNTL__PERF_ENABLE0__SHIFT	0x0
+#define SDMA1_PERFMON_CNTL__PERF_CLEAR0__SHIFT	0x1
+#define SDMA1_PERFMON_CNTL__PERF_SEL0__SHIFT	0x2
+#define SDMA1_PERFMON_CNTL__PERF_ENABLE1__SHIFT	0xa
+#define SDMA1_PERFMON_CNTL__PERF_CLEAR1__SHIFT	0xb
+#define SDMA1_PERFMON_CNTL__PERF_SEL1__SHIFT	0xc
+#define SDMA1_PERFMON_CNTL__PERF_ENABLE0_MASK	0x00000001L
+#define SDMA1_PERFMON_CNTL__PERF_CLEAR0_MASK	0x00000002L
+#define SDMA1_PERFMON_CNTL__PERF_SEL0_MASK	0x000003FCL
+#define SDMA1_PERFMON_CNTL__PERF_ENABLE1_MASK	0x00000400L
+#define SDMA1_PERFMON_CNTL__PERF_CLEAR1_MASK	0x00000800L
+#define SDMA1_PERFMON_CNTL__PERF_SEL1_MASK	0x000FF000L
+//SDMA1_PERFCOUNTER0_RESULT
+#define SDMA1_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT	0x0
+#define SDMA1_PERFCOUNTER0_RESULT__PERF_COUNT_MASK	0xFFFFFFFFL
+//SDMA1_PERFCOUNTER1_RESULT
+#define SDMA1_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT	0x0
+#define SDMA1_PERFCOUNTER1_RESULT__PERF_COUNT_MASK	0xFFFFFFFFL
+//SDMA1_PERFCOUNTER_TAG_DELAY_RANGE
+#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW__SHIFT	0x0
+#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH__SHIFT	0xe
+#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW__SHIFT	0x1c
+#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW_MASK	0x00003FFFL
+#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH_MASK	0x0FFFC000L
+#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW_MASK	0x10000000L
+//SDMA1_CRD_CNTL
+#define SDMA1_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT	0x7
+#define SDMA1_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT	0xd
+#define SDMA1_CRD_CNTL__MC_WRREQ_CREDIT_MASK	0x00001F80L
+#define SDMA1_CRD_CNTL__MC_RDREQ_CREDIT_MASK	0x0007E000L
+//SDMA1_MMHUB_TRUSTLVL
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG0__SHIFT	0x0
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG1__SHIFT	0x3
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG2__SHIFT	0x6
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG3__SHIFT	0x9
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG4__SHIFT	0xc
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG5__SHIFT	0xf
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG6__SHIFT	0x12
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG7__SHIFT	0x15
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG0_MASK	0x00000007L
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG1_MASK	0x00000038L
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG2_MASK	0x000001C0L
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG3_MASK	0x00000E00L
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG4_MASK	0x00007000L
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG5_MASK	0x00038000L
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG6_MASK	0x001C0000L
+#define SDMA1_MMHUB_TRUSTLVL__SECFLAG7_MASK	0x00E00000L
+//SDMA1_GPU_IOV_VIOLATION_LOG
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT	0x0
+#define SDMA1_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT	0x1
+#define SDMA1_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT	0x2
+#define SDMA1_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT	0x12
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VF__SHIFT	0x13
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VFID__SHIFT	0x14
+#define SDMA1_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT	0x18
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK	0x00000001L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK	0x00000002L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK	0x0003FFFCL
+#define SDMA1_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK	0x00040000L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VF_MASK	0x00080000L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VFID_MASK	0x00F00000L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK	0xFF000000L
+//SDMA1_ULV_CNTL
+#define SDMA1_ULV_CNTL__HYSTERESIS__SHIFT	0x0
+#define SDMA1_ULV_CNTL__ENTER_ULV_INT__SHIFT	0x1d
+#define SDMA1_ULV_CNTL__EXIT_ULV_INT__SHIFT	0x1e
+#define SDMA1_ULV_CNTL__ULV_STATUS__SHIFT	0x1f
+#define SDMA1_ULV_CNTL__HYSTERESIS_MASK	0x0000001FL
+#define SDMA1_ULV_CNTL__ENTER_ULV_INT_MASK	0x20000000L
+#define SDMA1_ULV_CNTL__EXIT_ULV_INT_MASK	0x40000000L
+#define SDMA1_ULV_CNTL__ULV_STATUS_MASK	0x80000000L
+//SDMA1_EA_DBIT_ADDR_DATA
+#define SDMA1_EA_DBIT_ADDR_DATA__VALUE__SHIFT	0x0
+#define SDMA1_EA_DBIT_ADDR_DATA__VALUE_MASK	0xFFFFFFFFL
+//SDMA1_EA_DBIT_ADDR_INDEX
+#define SDMA1_EA_DBIT_ADDR_INDEX__VALUE__SHIFT	0x0
+#define SDMA1_EA_DBIT_ADDR_INDEX__VALUE_MASK	0x00000007L
+//SDMA1_GFX_RB_CNTL
+#define SDMA1_GFX_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA1_GFX_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA1_GFX_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA1_GFX_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA1_GFX_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA1_GFX_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA1_GFX_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA1_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA1_GFX_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA1_GFX_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA1_GFX_RB_BASE
+#define SDMA1_GFX_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA1_GFX_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_BASE_HI
+#define SDMA1_GFX_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_GFX_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA1_GFX_RB_RPTR
+#define SDMA1_GFX_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA1_GFX_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_RPTR_HI
+#define SDMA1_GFX_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_GFX_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_WPTR
+#define SDMA1_GFX_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA1_GFX_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_WPTR_HI
+#define SDMA1_GFX_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_GFX_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_WPTR_POLL_CNTL
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA1_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA1_GFX_RB_RPTR_ADDR_HI
+#define SDMA1_GFX_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_GFX_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_RPTR_ADDR_LO
+#define SDMA1_GFX_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_GFX_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_GFX_IB_CNTL
+#define SDMA1_GFX_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA1_GFX_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA1_GFX_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA1_GFX_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA1_GFX_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA1_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA1_GFX_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA1_GFX_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA1_GFX_IB_RPTR
+#define SDMA1_GFX_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA1_GFX_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA1_GFX_IB_OFFSET
+#define SDMA1_GFX_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_GFX_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA1_GFX_IB_BASE_LO
+#define SDMA1_GFX_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA1_GFX_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA1_GFX_IB_BASE_HI
+#define SDMA1_GFX_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_GFX_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_GFX_IB_SIZE
+#define SDMA1_GFX_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA1_GFX_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA1_GFX_SKIP_CNTL
+#define SDMA1_GFX_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA1_GFX_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA1_GFX_CONTEXT_STATUS
+#define SDMA1_GFX_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA1_GFX_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA1_GFX_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA1_GFX_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA1_GFX_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA1_GFX_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA1_GFX_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA1_GFX_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA1_GFX_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA1_GFX_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA1_GFX_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA1_GFX_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA1_GFX_DOORBELL
+#define SDMA1_GFX_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA1_GFX_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA1_GFX_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA1_GFX_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA1_GFX_CONTEXT_CNTL
+#define SDMA1_GFX_CONTEXT_CNTL__RESUME_CTX__SHIFT	0x10
+#define SDMA1_GFX_CONTEXT_CNTL__RESUME_CTX_MASK	0x00010000L
+//SDMA1_GFX_STATUS
+#define SDMA1_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA1_GFX_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA1_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA1_GFX_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA1_GFX_DOORBELL_LOG
+#define SDMA1_GFX_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA1_GFX_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA1_GFX_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA1_GFX_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA1_GFX_WATERMARK
+#define SDMA1_GFX_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA1_GFX_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA1_GFX_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA1_GFX_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA1_GFX_DOORBELL_OFFSET
+#define SDMA1_GFX_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_GFX_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA1_GFX_CSA_ADDR_LO
+#define SDMA1_GFX_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_GFX_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_GFX_CSA_ADDR_HI
+#define SDMA1_GFX_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_GFX_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_GFX_IB_SUB_REMAIN
+#define SDMA1_GFX_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA1_GFX_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA1_GFX_PREEMPT
+#define SDMA1_GFX_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA1_GFX_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA1_GFX_DUMMY_REG
+#define SDMA1_GFX_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA1_GFX_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_GFX_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_GFX_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_GFX_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_GFX_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_GFX_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_GFX_RB_AQL_CNTL
+#define SDMA1_GFX_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA1_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA1_GFX_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA1_GFX_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA1_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA1_GFX_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA1_GFX_MINOR_PTR_UPDATE
+#define SDMA1_GFX_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA1_GFX_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA1_GFX_MIDCMD_DATA0
+#define SDMA1_GFX_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA1
+#define SDMA1_GFX_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA2
+#define SDMA1_GFX_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA3
+#define SDMA1_GFX_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA4
+#define SDMA1_GFX_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA5
+#define SDMA1_GFX_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA6
+#define SDMA1_GFX_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA7
+#define SDMA1_GFX_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_DATA8
+#define SDMA1_GFX_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA1_GFX_MIDCMD_CNTL
+#define SDMA1_GFX_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA1_GFX_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA1_GFX_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA1_GFX_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA1_GFX_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA1_GFX_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA1_GFX_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA1_GFX_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+//SDMA1_PAGE_RB_CNTL
+#define SDMA1_PAGE_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA1_PAGE_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA1_PAGE_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA1_PAGE_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA1_PAGE_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA1_PAGE_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA1_PAGE_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA1_PAGE_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA1_PAGE_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA1_PAGE_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA1_PAGE_RB_BASE
+#define SDMA1_PAGE_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA1_PAGE_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_BASE_HI
+#define SDMA1_PAGE_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_PAGE_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA1_PAGE_RB_RPTR
+#define SDMA1_PAGE_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA1_PAGE_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_RPTR_HI
+#define SDMA1_PAGE_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_PAGE_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_WPTR
+#define SDMA1_PAGE_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA1_PAGE_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_WPTR_HI
+#define SDMA1_PAGE_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_PAGE_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_WPTR_POLL_CNTL
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA1_PAGE_RB_RPTR_ADDR_HI
+#define SDMA1_PAGE_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_PAGE_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_RPTR_ADDR_LO
+#define SDMA1_PAGE_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_PAGE_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_PAGE_IB_CNTL
+#define SDMA1_PAGE_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA1_PAGE_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA1_PAGE_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA1_PAGE_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA1_PAGE_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA1_PAGE_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA1_PAGE_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA1_PAGE_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA1_PAGE_IB_RPTR
+#define SDMA1_PAGE_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA1_PAGE_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA1_PAGE_IB_OFFSET
+#define SDMA1_PAGE_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_PAGE_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA1_PAGE_IB_BASE_LO
+#define SDMA1_PAGE_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA1_PAGE_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA1_PAGE_IB_BASE_HI
+#define SDMA1_PAGE_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_PAGE_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_IB_SIZE
+#define SDMA1_PAGE_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA1_PAGE_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA1_PAGE_SKIP_CNTL
+#define SDMA1_PAGE_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA1_PAGE_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA1_PAGE_CONTEXT_STATUS
+#define SDMA1_PAGE_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA1_PAGE_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA1_PAGE_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA1_PAGE_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA1_PAGE_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA1_PAGE_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA1_PAGE_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA1_PAGE_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA1_PAGE_DOORBELL
+#define SDMA1_PAGE_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA1_PAGE_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA1_PAGE_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA1_PAGE_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA1_PAGE_STATUS
+#define SDMA1_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA1_PAGE_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA1_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA1_PAGE_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA1_PAGE_DOORBELL_LOG
+#define SDMA1_PAGE_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA1_PAGE_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA1_PAGE_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA1_PAGE_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA1_PAGE_WATERMARK
+#define SDMA1_PAGE_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA1_PAGE_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA1_PAGE_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA1_PAGE_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA1_PAGE_DOORBELL_OFFSET
+#define SDMA1_PAGE_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_PAGE_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA1_PAGE_CSA_ADDR_LO
+#define SDMA1_PAGE_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_PAGE_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_PAGE_CSA_ADDR_HI
+#define SDMA1_PAGE_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_PAGE_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_IB_SUB_REMAIN
+#define SDMA1_PAGE_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA1_PAGE_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA1_PAGE_PREEMPT
+#define SDMA1_PAGE_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA1_PAGE_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA1_PAGE_DUMMY_REG
+#define SDMA1_PAGE_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA1_PAGE_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_PAGE_RB_AQL_CNTL
+#define SDMA1_PAGE_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA1_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA1_PAGE_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA1_PAGE_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA1_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA1_PAGE_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA1_PAGE_MINOR_PTR_UPDATE
+#define SDMA1_PAGE_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA1_PAGE_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA1_PAGE_MIDCMD_DATA0
+#define SDMA1_PAGE_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA1
+#define SDMA1_PAGE_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA2
+#define SDMA1_PAGE_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA3
+#define SDMA1_PAGE_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA4
+#define SDMA1_PAGE_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA5
+#define SDMA1_PAGE_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA6
+#define SDMA1_PAGE_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA7
+#define SDMA1_PAGE_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_DATA8
+#define SDMA1_PAGE_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA1_PAGE_MIDCMD_CNTL
+#define SDMA1_PAGE_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA1_PAGE_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA1_PAGE_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA1_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA1_PAGE_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA1_PAGE_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA1_PAGE_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA1_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+//SDMA1_RLC0_RB_CNTL
+#define SDMA1_RLC0_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA1_RLC0_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA1_RLC0_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA1_RLC0_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA1_RLC0_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA1_RLC0_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA1_RLC0_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA1_RLC0_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA1_RLC0_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA1_RLC0_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA1_RLC0_RB_BASE
+#define SDMA1_RLC0_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA1_RLC0_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_BASE_HI
+#define SDMA1_RLC0_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC0_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA1_RLC0_RB_RPTR
+#define SDMA1_RLC0_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA1_RLC0_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_RPTR_HI
+#define SDMA1_RLC0_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_RLC0_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_WPTR
+#define SDMA1_RLC0_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA1_RLC0_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_WPTR_HI
+#define SDMA1_RLC0_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_RLC0_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_WPTR_POLL_CNTL
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA1_RLC0_RB_RPTR_ADDR_HI
+#define SDMA1_RLC0_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC0_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_RPTR_ADDR_LO
+#define SDMA1_RLC0_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_RLC0_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_RLC0_IB_CNTL
+#define SDMA1_RLC0_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA1_RLC0_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA1_RLC0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA1_RLC0_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA1_RLC0_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA1_RLC0_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA1_RLC0_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA1_RLC0_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA1_RLC0_IB_RPTR
+#define SDMA1_RLC0_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA1_RLC0_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA1_RLC0_IB_OFFSET
+#define SDMA1_RLC0_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_RLC0_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA1_RLC0_IB_BASE_LO
+#define SDMA1_RLC0_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA1_RLC0_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA1_RLC0_IB_BASE_HI
+#define SDMA1_RLC0_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC0_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_IB_SIZE
+#define SDMA1_RLC0_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA1_RLC0_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA1_RLC0_SKIP_CNTL
+#define SDMA1_RLC0_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA1_RLC0_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA1_RLC0_CONTEXT_STATUS
+#define SDMA1_RLC0_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA1_RLC0_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA1_RLC0_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA1_RLC0_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA1_RLC0_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA1_RLC0_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA1_RLC0_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA1_RLC0_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA1_RLC0_DOORBELL
+#define SDMA1_RLC0_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA1_RLC0_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA1_RLC0_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA1_RLC0_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA1_RLC0_STATUS
+#define SDMA1_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA1_RLC0_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA1_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA1_RLC0_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA1_RLC0_DOORBELL_LOG
+#define SDMA1_RLC0_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA1_RLC0_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA1_RLC0_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA1_RLC0_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA1_RLC0_WATERMARK
+#define SDMA1_RLC0_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA1_RLC0_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA1_RLC0_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA1_RLC0_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA1_RLC0_DOORBELL_OFFSET
+#define SDMA1_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_RLC0_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA1_RLC0_CSA_ADDR_LO
+#define SDMA1_RLC0_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_RLC0_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_RLC0_CSA_ADDR_HI
+#define SDMA1_RLC0_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC0_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_IB_SUB_REMAIN
+#define SDMA1_RLC0_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA1_RLC0_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA1_RLC0_PREEMPT
+#define SDMA1_RLC0_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA1_RLC0_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA1_RLC0_DUMMY_REG
+#define SDMA1_RLC0_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA1_RLC0_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_RLC0_RB_AQL_CNTL
+#define SDMA1_RLC0_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA1_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA1_RLC0_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA1_RLC0_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA1_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA1_RLC0_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA1_RLC0_MINOR_PTR_UPDATE
+#define SDMA1_RLC0_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA1_RLC0_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA1_RLC0_MIDCMD_DATA0
+#define SDMA1_RLC0_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA1
+#define SDMA1_RLC0_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA2
+#define SDMA1_RLC0_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA3
+#define SDMA1_RLC0_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA4
+#define SDMA1_RLC0_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA5
+#define SDMA1_RLC0_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA6
+#define SDMA1_RLC0_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA7
+#define SDMA1_RLC0_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_DATA8
+#define SDMA1_RLC0_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA1_RLC0_MIDCMD_CNTL
+#define SDMA1_RLC0_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA1_RLC0_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA1_RLC0_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA1_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA1_RLC0_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA1_RLC0_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA1_RLC0_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA1_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+//SDMA1_RLC1_RB_CNTL
+#define SDMA1_RLC1_RB_CNTL__RB_ENABLE__SHIFT	0x0
+#define SDMA1_RLC1_RB_CNTL__RB_SIZE__SHIFT	0x1
+#define SDMA1_RLC1_RB_CNTL__RB_SWAP_ENABLE__SHIFT	0x9
+#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT	0xc
+#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT	0xd
+#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT	0x10
+#define SDMA1_RLC1_RB_CNTL__RB_PRIV__SHIFT	0x17
+#define SDMA1_RLC1_RB_CNTL__RB_VMID__SHIFT	0x18
+#define SDMA1_RLC1_RB_CNTL__RB_ENABLE_MASK	0x00000001L
+#define SDMA1_RLC1_RB_CNTL__RB_SIZE_MASK	0x0000007EL
+#define SDMA1_RLC1_RB_CNTL__RB_SWAP_ENABLE_MASK	0x00000200L
+#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK	0x00001000L
+#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK	0x00002000L
+#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK	0x001F0000L
+#define SDMA1_RLC1_RB_CNTL__RB_PRIV_MASK	0x00800000L
+#define SDMA1_RLC1_RB_CNTL__RB_VMID_MASK	0x0F000000L
+//SDMA1_RLC1_RB_BASE
+#define SDMA1_RLC1_RB_BASE__ADDR__SHIFT	0x0
+#define SDMA1_RLC1_RB_BASE__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_BASE_HI
+#define SDMA1_RLC1_RB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC1_RB_BASE_HI__ADDR_MASK	0x00FFFFFFL
+//SDMA1_RLC1_RB_RPTR
+#define SDMA1_RLC1_RB_RPTR__OFFSET__SHIFT	0x0
+#define SDMA1_RLC1_RB_RPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_RPTR_HI
+#define SDMA1_RLC1_RB_RPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_RLC1_RB_RPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_WPTR
+#define SDMA1_RLC1_RB_WPTR__OFFSET__SHIFT	0x0
+#define SDMA1_RLC1_RB_WPTR__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_WPTR_HI
+#define SDMA1_RLC1_RB_WPTR_HI__OFFSET__SHIFT	0x0
+#define SDMA1_RLC1_RB_WPTR_HI__OFFSET_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_WPTR_POLL_CNTL
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__ENABLE__SHIFT	0x0
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT	0x1
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT	0x2
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT	0x4
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT	0x10
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__ENABLE_MASK	0x00000001L
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK	0x00000002L
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK	0x00000004L
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY_MASK	0x0000FFF0L
+#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK	0xFFFF0000L
+//SDMA1_RLC1_RB_RPTR_ADDR_HI
+#define SDMA1_RLC1_RB_RPTR_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC1_RB_RPTR_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_RPTR_ADDR_LO
+#define SDMA1_RLC1_RB_RPTR_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_RLC1_RB_RPTR_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_RLC1_IB_CNTL
+#define SDMA1_RLC1_IB_CNTL__IB_ENABLE__SHIFT	0x0
+#define SDMA1_RLC1_IB_CNTL__IB_SWAP_ENABLE__SHIFT	0x4
+#define SDMA1_RLC1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT	0x8
+#define SDMA1_RLC1_IB_CNTL__CMD_VMID__SHIFT	0x10
+#define SDMA1_RLC1_IB_CNTL__IB_ENABLE_MASK	0x00000001L
+#define SDMA1_RLC1_IB_CNTL__IB_SWAP_ENABLE_MASK	0x00000010L
+#define SDMA1_RLC1_IB_CNTL__SWITCH_INSIDE_IB_MASK	0x00000100L
+#define SDMA1_RLC1_IB_CNTL__CMD_VMID_MASK	0x000F0000L
+//SDMA1_RLC1_IB_RPTR
+#define SDMA1_RLC1_IB_RPTR__OFFSET__SHIFT	0x2
+#define SDMA1_RLC1_IB_RPTR__OFFSET_MASK	0x003FFFFCL
+//SDMA1_RLC1_IB_OFFSET
+#define SDMA1_RLC1_IB_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_RLC1_IB_OFFSET__OFFSET_MASK	0x003FFFFCL
+//SDMA1_RLC1_IB_BASE_LO
+#define SDMA1_RLC1_IB_BASE_LO__ADDR__SHIFT	0x5
+#define SDMA1_RLC1_IB_BASE_LO__ADDR_MASK	0xFFFFFFE0L
+//SDMA1_RLC1_IB_BASE_HI
+#define SDMA1_RLC1_IB_BASE_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC1_IB_BASE_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_IB_SIZE
+#define SDMA1_RLC1_IB_SIZE__SIZE__SHIFT	0x0
+#define SDMA1_RLC1_IB_SIZE__SIZE_MASK	0x000FFFFFL
+//SDMA1_RLC1_SKIP_CNTL
+#define SDMA1_RLC1_SKIP_CNTL__SKIP_COUNT__SHIFT	0x0
+#define SDMA1_RLC1_SKIP_CNTL__SKIP_COUNT_MASK	0x00003FFFL
+//SDMA1_RLC1_CONTEXT_STATUS
+#define SDMA1_RLC1_CONTEXT_STATUS__SELECTED__SHIFT	0x0
+#define SDMA1_RLC1_CONTEXT_STATUS__IDLE__SHIFT	0x2
+#define SDMA1_RLC1_CONTEXT_STATUS__EXPIRED__SHIFT	0x3
+#define SDMA1_RLC1_CONTEXT_STATUS__EXCEPTION__SHIFT	0x4
+#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT	0x7
+#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_READY__SHIFT	0x8
+#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPTED__SHIFT	0x9
+#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT	0xa
+#define SDMA1_RLC1_CONTEXT_STATUS__SELECTED_MASK	0x00000001L
+#define SDMA1_RLC1_CONTEXT_STATUS__IDLE_MASK	0x00000004L
+#define SDMA1_RLC1_CONTEXT_STATUS__EXPIRED_MASK	0x00000008L
+#define SDMA1_RLC1_CONTEXT_STATUS__EXCEPTION_MASK	0x00000070L
+#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_ABLE_MASK	0x00000080L
+#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_READY_MASK	0x00000100L
+#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPTED_MASK	0x00000200L
+#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK	0x00000400L
+//SDMA1_RLC1_DOORBELL
+#define SDMA1_RLC1_DOORBELL__ENABLE__SHIFT	0x1c
+#define SDMA1_RLC1_DOORBELL__CAPTURED__SHIFT	0x1e
+#define SDMA1_RLC1_DOORBELL__ENABLE_MASK	0x10000000L
+#define SDMA1_RLC1_DOORBELL__CAPTURED_MASK	0x40000000L
+//SDMA1_RLC1_STATUS
+#define SDMA1_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT	0x0
+#define SDMA1_RLC1_STATUS__WPTR_UPDATE_PENDING__SHIFT	0x8
+#define SDMA1_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK	0x000000FFL
+#define SDMA1_RLC1_STATUS__WPTR_UPDATE_PENDING_MASK	0x00000100L
+//SDMA1_RLC1_DOORBELL_LOG
+#define SDMA1_RLC1_DOORBELL_LOG__BE_ERROR__SHIFT	0x0
+#define SDMA1_RLC1_DOORBELL_LOG__DATA__SHIFT	0x2
+#define SDMA1_RLC1_DOORBELL_LOG__BE_ERROR_MASK	0x00000001L
+#define SDMA1_RLC1_DOORBELL_LOG__DATA_MASK	0xFFFFFFFCL
+//SDMA1_RLC1_WATERMARK
+#define SDMA1_RLC1_WATERMARK__RD_OUTSTANDING__SHIFT	0x0
+#define SDMA1_RLC1_WATERMARK__WR_OUTSTANDING__SHIFT	0x10
+#define SDMA1_RLC1_WATERMARK__RD_OUTSTANDING_MASK	0x00000FFFL
+#define SDMA1_RLC1_WATERMARK__WR_OUTSTANDING_MASK	0x03FF0000L
+//SDMA1_RLC1_DOORBELL_OFFSET
+#define SDMA1_RLC1_DOORBELL_OFFSET__OFFSET__SHIFT	0x2
+#define SDMA1_RLC1_DOORBELL_OFFSET__OFFSET_MASK	0x0FFFFFFCL
+//SDMA1_RLC1_CSA_ADDR_LO
+#define SDMA1_RLC1_CSA_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_RLC1_CSA_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_RLC1_CSA_ADDR_HI
+#define SDMA1_RLC1_CSA_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC1_CSA_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_IB_SUB_REMAIN
+#define SDMA1_RLC1_IB_SUB_REMAIN__SIZE__SHIFT	0x0
+#define SDMA1_RLC1_IB_SUB_REMAIN__SIZE_MASK	0x00003FFFL
+//SDMA1_RLC1_PREEMPT
+#define SDMA1_RLC1_PREEMPT__IB_PREEMPT__SHIFT	0x0
+#define SDMA1_RLC1_PREEMPT__IB_PREEMPT_MASK	0x00000001L
+//SDMA1_RLC1_DUMMY_REG
+#define SDMA1_RLC1_DUMMY_REG__DUMMY__SHIFT	0x0
+#define SDMA1_RLC1_DUMMY_REG__DUMMY_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT	0x0
+#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT	0x2
+#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK	0xFFFFFFFCL
+//SDMA1_RLC1_RB_AQL_CNTL
+#define SDMA1_RLC1_RB_AQL_CNTL__AQL_ENABLE__SHIFT	0x0
+#define SDMA1_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT	0x1
+#define SDMA1_RLC1_RB_AQL_CNTL__PACKET_STEP__SHIFT	0x8
+#define SDMA1_RLC1_RB_AQL_CNTL__AQL_ENABLE_MASK	0x00000001L
+#define SDMA1_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK	0x000000FEL
+#define SDMA1_RLC1_RB_AQL_CNTL__PACKET_STEP_MASK	0x0000FF00L
+//SDMA1_RLC1_MINOR_PTR_UPDATE
+#define SDMA1_RLC1_MINOR_PTR_UPDATE__ENABLE__SHIFT	0x0
+#define SDMA1_RLC1_MINOR_PTR_UPDATE__ENABLE_MASK	0x00000001L
+//SDMA1_RLC1_MIDCMD_DATA0
+#define SDMA1_RLC1_MIDCMD_DATA0__DATA0__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA0__DATA0_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA1
+#define SDMA1_RLC1_MIDCMD_DATA1__DATA1__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA1__DATA1_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA2
+#define SDMA1_RLC1_MIDCMD_DATA2__DATA2__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA2__DATA2_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA3
+#define SDMA1_RLC1_MIDCMD_DATA3__DATA3__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA3__DATA3_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA4
+#define SDMA1_RLC1_MIDCMD_DATA4__DATA4__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA4__DATA4_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA5
+#define SDMA1_RLC1_MIDCMD_DATA5__DATA5__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA5__DATA5_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA6
+#define SDMA1_RLC1_MIDCMD_DATA6__DATA6__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA6__DATA6_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA7
+#define SDMA1_RLC1_MIDCMD_DATA7__DATA7__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA7__DATA7_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_DATA8
+#define SDMA1_RLC1_MIDCMD_DATA8__DATA8__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_DATA8__DATA8_MASK	0xFFFFFFFFL
+//SDMA1_RLC1_MIDCMD_CNTL
+#define SDMA1_RLC1_MIDCMD_CNTL__DATA_VALID__SHIFT	0x0
+#define SDMA1_RLC1_MIDCMD_CNTL__COPY_MODE__SHIFT	0x1
+#define SDMA1_RLC1_MIDCMD_CNTL__SPLIT_STATE__SHIFT	0x4
+#define SDMA1_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT	0x8
+#define SDMA1_RLC1_MIDCMD_CNTL__DATA_VALID_MASK	0x00000001L
+#define SDMA1_RLC1_MIDCMD_CNTL__COPY_MODE_MASK	0x00000002L
+#define SDMA1_RLC1_MIDCMD_CNTL__SPLIT_STATE_MASK	0x000000F0L
+#define SDMA1_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK	0x00000100L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_9_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_9_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_9_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_9_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/THM/thm_10_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_10_0_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/THM/thm_10_0_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/thm/thm_10_0_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/THM/thm_10_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_10_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/THM/thm_10_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/thm/thm_10_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/THM/thm_10_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_10_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/THM/thm_10_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/thm/thm_10_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/THM/thm_9_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_9_0_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/THM/thm_9_0_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/thm/thm_9_0_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/THM/thm_9_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_9_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/THM/thm_9_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/thm/thm_9_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/THM/thm_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/thm/thm_9_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/THM/thm_9_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/thm/thm_9_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_default.h
new file mode 100644
index 000000000000..128a18f1e362
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_default.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _umc_6_0_DEFAULT_HEADER
+#define _umc_6_0_DEFAULT_HEADER
+
+#define mmUMCCH0_0_EccCtrl_DEFAULT				0x00000000
+
+#define mmUMCCH0_0_UMC_CONFIG_DEFAULT				0x00000203
+
+#define mmUMCCH0_0_UmcLocalCap_DEFAULT				0x00000000
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_offset.h
new file mode 100644
index 000000000000..6985dbba39f5
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_offset.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _umc_6_0_OFFSET_H_
+#define _umc_6_0_OFFSET_H_
+
+#define mmUMCCH0_0_EccCtrl				0x0053
+#define mmUMCCH0_0_EccCtrl_BASE_IDX			0
+#define mmUMCCH1_0_EccCtrl				0x0853
+#define mmUMCCH1_0_EccCtrl_BASE_IDX			0
+#define mmUMCCH2_0_EccCtrl				0x1053
+#define mmUMCCH2_0_EccCtrl_BASE_IDX			0
+#define mmUMCCH3_0_EccCtrl				0x1853
+#define mmUMCCH3_0_EccCtrl_BASE_IDX			0
+
+#define mmUMCCH0_0_UMC_CONFIG				0x0040
+#define mmUMCCH0_0_UMC_CONFIG_BASE_IDX			0
+#define mmUMCCH1_0_UMC_CONFIG				0x0840
+#define mmUMCCH1_0_UMC_CONFIG_BASE_IDX			0
+#define mmUMCCH2_0_UMC_CONFIG				0x1040
+#define mmUMCCH2_0_UMC_CONFIG_BASE_IDX			0
+#define mmUMCCH3_0_UMC_CONFIG				0x1840
+#define mmUMCCH3_0_UMC_CONFIG_BASE_IDX			0
+
+#define mmUMCCH0_0_UmcLocalCap				0x0306
+#define mmUMCCH0_0_UmcLocalCap_BASE_IDX			0
+#define mmUMCCH1_0_UmcLocalCap				0x0b06
+#define mmUMCCH1_0_UmcLocalCap_BASE_IDX			0
+#define mmUMCCH2_0_UmcLocalCap				0x1306
+#define mmUMCCH2_0_UmcLocalCap_BASE_IDX			0
+#define mmUMCCH3_0_UmcLocalCap				0x1b06
+#define mmUMCCH3_0_UmcLocalCap_BASE_IDX			0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_sh_mask.h
new file mode 100644
index 000000000000..3e857d1613f0
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_6_0_sh_mask.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2017  Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _umc_6_0_SH_MASK_HEADER
+#define _umc_6_0_SH_MASK_HEADER
+
+#define UMCCH0_0_EccCtrl__RdEccEn_MASK   0x00000400L
+#define UMCCH0_0_EccCtrl__RdEccEn__SHIFT   0xa
+#define UMCCH0_0_EccCtrl__WrEccEn_MASK   0x00000001L
+#define UMCCH0_0_EccCtrl__WrEccEn__SHIFT   0x0
+
+#define UMCCH0_0_UMC_CONFIG__DramReady_MASK   0x80000000L
+#define UMCCH0_0_UMC_CONFIG__DramReady__SHIFT   0x1f
+
+#define UMCCH0_0_UmcLocalCap__EccDis_MASK   0x00000001L
+#define UMCCH0_0_UmcLocalCap__EccDis__SHIFT   0x0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_7_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_7_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_7_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_7_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/VCE/vce_4_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vce/vce_4_0_default.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/VCE/vce_4_0_default.h
rename to drivers/gpu/drm/amd/include/asic_reg/vce/vce_4_0_default.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/VCE/vce_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vce/vce_4_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/VCE/vce_4_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/vce/vce_4_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/VCE/vce_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vce/vce_4_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/VCE/vce_4_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/vce/vce_4_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_offset.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_offset.h
rename to drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_offset.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_sh_mask.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/raven1/VCN/vcn_1_0_sh_mask.h
rename to drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_sh_mask.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_default.h
deleted file mode 100644
index 1650dc369f7d..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_default.h
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _athub_1_0_DEFAULT_HEADER
-#define _athub_1_0_DEFAULT_HEADER
-
-
-// addressBlock: athub_atsdec
-#define mmATC_ATS_CNTL_DEFAULT                                                   0x009a0800
-#define mmATC_ATS_STATUS_DEFAULT                                                 0x00000000
-#define mmATC_ATS_FAULT_CNTL_DEFAULT                                             0x000001ff
-#define mmATC_ATS_FAULT_STATUS_INFO_DEFAULT                                      0x00000000
-#define mmATC_ATS_FAULT_STATUS_ADDR_DEFAULT                                      0x00000000
-#define mmATC_ATS_DEFAULT_PAGE_LOW_DEFAULT                                       0x00000000
-#define mmATC_TRANS_FAULT_RSPCNTRL_DEFAULT                                       0xffffffff
-#define mmATC_ATS_FAULT_STATUS_INFO2_DEFAULT                                     0x00000000
-#define mmATHUB_MISC_CNTL_DEFAULT                                                0x00040200
-#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS_DEFAULT                           0x00000000
-#define mmATC_VMID0_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID1_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID2_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID3_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID4_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID5_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID6_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID7_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID8_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID9_PASID_MAPPING_DEFAULT                                        0x00000000
-#define mmATC_VMID10_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID11_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID12_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID13_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID14_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID15_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_ATS_VMID_STATUS_DEFAULT                                            0x00000000
-#define mmATC_ATS_GFX_ATCL2_STATUS_DEFAULT                                       0x00000000
-#define mmATC_PERFCOUNTER0_CFG_DEFAULT                                           0x00000000
-#define mmATC_PERFCOUNTER1_CFG_DEFAULT                                           0x00000000
-#define mmATC_PERFCOUNTER2_CFG_DEFAULT                                           0x00000000
-#define mmATC_PERFCOUNTER3_CFG_DEFAULT                                           0x00000000
-#define mmATC_PERFCOUNTER_RSLT_CNTL_DEFAULT                                      0x04000000
-#define mmATC_PERFCOUNTER_LO_DEFAULT                                             0x00000000
-#define mmATC_PERFCOUNTER_HI_DEFAULT                                             0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_DEFAULT                                            0x00000000
-#define mmATHUB_PCIE_PASID_CNTL_DEFAULT                                          0x00000000
-#define mmATHUB_PCIE_PAGE_REQ_CNTL_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC_DEFAULT                             0x00000000
-#define mmATHUB_COMMAND_DEFAULT                                                  0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_0_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_1_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_2_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_3_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_4_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_5_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_6_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_7_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_8_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_9_DEFAULT                                       0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_10_DEFAULT                                      0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_11_DEFAULT                                      0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_12_DEFAULT                                      0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_13_DEFAULT                                      0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_14_DEFAULT                                      0x00000000
-#define mmATHUB_PCIE_ATS_CNTL_VF_15_DEFAULT                                      0x00000000
-#define mmATHUB_MEM_POWER_LS_DEFAULT                                             0x00000208
-#define mmATS_IH_CREDIT_DEFAULT                                                  0x00150002
-#define mmATHUB_IH_CREDIT_DEFAULT                                                0x00020002
-#define mmATC_VMID16_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID17_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID18_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID19_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID20_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID21_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID22_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID23_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID24_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID25_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID26_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID27_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID28_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID29_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID30_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_VMID31_PASID_MAPPING_DEFAULT                                       0x00000000
-#define mmATC_ATS_MMHUB_ATCL2_STATUS_DEFAULT                                     0x00000000
-#define mmATHUB_SHARED_VIRT_RESET_REQ_DEFAULT                                    0x00000000
-#define mmATHUB_SHARED_ACTIVE_FCN_ID_DEFAULT                                     0x00000000
-#define mmATC_ATS_SDPPORT_CNTL_DEFAULT                                           0x03ffa210
-#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT_DEFAULT                                 0x00000000
-#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT_DEFAULT                               0x00000000
-
-
-// addressBlock: athub_xpbdec
-#define mmXPB_RTR_SRC_APRTR0_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR1_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR2_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR3_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR4_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR5_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR6_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR7_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR8_DEFAULT                                             0x00000000
-#define mmXPB_RTR_SRC_APRTR9_DEFAULT                                             0x00000000
-#define mmXPB_XDMA_RTR_SRC_APRTR0_DEFAULT                                        0x00000000
-#define mmXPB_XDMA_RTR_SRC_APRTR1_DEFAULT                                        0x00000000
-#define mmXPB_XDMA_RTR_SRC_APRTR2_DEFAULT                                        0x00000000
-#define mmXPB_XDMA_RTR_SRC_APRTR3_DEFAULT                                        0x00000000
-#define mmXPB_RTR_DEST_MAP0_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP1_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP2_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP3_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP4_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP5_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP6_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP7_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP8_DEFAULT                                              0x00000000
-#define mmXPB_RTR_DEST_MAP9_DEFAULT                                              0x00000000
-#define mmXPB_XDMA_RTR_DEST_MAP0_DEFAULT                                         0x00000000
-#define mmXPB_XDMA_RTR_DEST_MAP1_DEFAULT                                         0x00000000
-#define mmXPB_XDMA_RTR_DEST_MAP2_DEFAULT                                         0x00000000
-#define mmXPB_XDMA_RTR_DEST_MAP3_DEFAULT                                         0x00000000
-#define mmXPB_CLG_CFG0_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_CFG1_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_CFG2_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_CFG3_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_CFG4_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_CFG5_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_CFG6_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_CFG7_DEFAULT                                                   0x00000000
-#define mmXPB_CLG_EXTRA_DEFAULT                                                  0x00000000
-#define mmXPB_CLG_EXTRA_MSK_DEFAULT                                              0x00000000
-#define mmXPB_LB_ADDR_DEFAULT                                                    0x00000000
-#define mmXPB_WCB_STS_DEFAULT                                                    0x00000000
-#define mmXPB_HST_CFG_DEFAULT                                                    0x00000000
-#define mmXPB_P2P_BAR_CFG_DEFAULT                                                0x0000000f
-#define mmXPB_P2P_BAR0_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR1_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR2_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR3_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR4_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR5_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR6_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR7_DEFAULT                                                   0x00000000
-#define mmXPB_P2P_BAR_SETUP_DEFAULT                                              0x00000000
-#define mmXPB_P2P_BAR_DELTA_ABOVE_DEFAULT                                        0x00000000
-#define mmXPB_P2P_BAR_DELTA_BELOW_DEFAULT                                        0x00000000
-#define mmXPB_PEER_SYS_BAR0_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR1_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR2_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR3_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR4_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR5_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR6_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR7_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR8_DEFAULT                                              0x00000000
-#define mmXPB_PEER_SYS_BAR9_DEFAULT                                              0x00000000
-#define mmXPB_XDMA_PEER_SYS_BAR0_DEFAULT                                         0x00000000
-#define mmXPB_XDMA_PEER_SYS_BAR1_DEFAULT                                         0x00000000
-#define mmXPB_XDMA_PEER_SYS_BAR2_DEFAULT                                         0x00000000
-#define mmXPB_XDMA_PEER_SYS_BAR3_DEFAULT                                         0x00000000
-#define mmXPB_CLK_GAT_DEFAULT                                                    0x00040400
-#define mmXPB_INTF_CFG_DEFAULT                                                   0x000f1040
-#define mmXPB_INTF_STS_DEFAULT                                                   0x00000000
-#define mmXPB_PIPE_STS_DEFAULT                                                   0x00000000
-#define mmXPB_SUB_CTRL_DEFAULT                                                   0x00000000
-#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB_DEFAULT                                   0x00000000
-#define mmXPB_PERF_KNOBS_DEFAULT                                                 0x00000000
-#define mmXPB_STICKY_DEFAULT                                                     0x00000000
-#define mmXPB_STICKY_W1C_DEFAULT                                                 0x00000000
-#define mmXPB_MISC_CFG_DEFAULT                                                   0x4d585042
-#define mmXPB_INTF_CFG2_DEFAULT                                                  0x00000040
-#define mmXPB_CLG_EXTRA_RD_DEFAULT                                               0x00000000
-#define mmXPB_CLG_EXTRA_MSK_RD_DEFAULT                                           0x00000000
-#define mmXPB_CLG_GFX_MATCH_DEFAULT                                              0x03000000
-#define mmXPB_CLG_GFX_MATCH_MSK_DEFAULT                                          0x00000000
-#define mmXPB_CLG_MM_MATCH_DEFAULT                                               0x03000000
-#define mmXPB_CLG_MM_MATCH_MSK_DEFAULT                                           0x00000000
-#define mmXPB_CLG_GFX_UNITID_MAPPING0_DEFAULT                                    0x00000000
-#define mmXPB_CLG_GFX_UNITID_MAPPING1_DEFAULT                                    0x00000040
-#define mmXPB_CLG_GFX_UNITID_MAPPING2_DEFAULT                                    0x00000080
-#define mmXPB_CLG_GFX_UNITID_MAPPING3_DEFAULT                                    0x000000c0
-#define mmXPB_CLG_GFX_UNITID_MAPPING4_DEFAULT                                    0x00000100
-#define mmXPB_CLG_GFX_UNITID_MAPPING5_DEFAULT                                    0x00000140
-#define mmXPB_CLG_GFX_UNITID_MAPPING6_DEFAULT                                    0x00000000
-#define mmXPB_CLG_GFX_UNITID_MAPPING7_DEFAULT                                    0x000001c0
-#define mmXPB_CLG_MM_UNITID_MAPPING0_DEFAULT                                     0x00000000
-#define mmXPB_CLG_MM_UNITID_MAPPING1_DEFAULT                                     0x00000040
-#define mmXPB_CLG_MM_UNITID_MAPPING2_DEFAULT                                     0x00000080
-#define mmXPB_CLG_MM_UNITID_MAPPING3_DEFAULT                                     0x000000c0
-
-
-// addressBlock: athub_rpbdec
-#define mmRPB_PASSPW_CONF_DEFAULT                                                0x00000230
-#define mmRPB_BLOCKLEVEL_CONF_DEFAULT                                            0x000000f0
-#define mmRPB_TAG_CONF_DEFAULT                                                   0x00204020
-#define mmRPB_EFF_CNTL_DEFAULT                                                   0x00001010
-#define mmRPB_ARB_CNTL_DEFAULT                                                   0x00040404
-#define mmRPB_ARB_CNTL2_DEFAULT                                                  0x00040104
-#define mmRPB_BIF_CNTL_DEFAULT                                                   0x01000404
-#define mmRPB_WR_SWITCH_CNTL_DEFAULT                                             0x02040810
-#define mmRPB_RD_SWITCH_CNTL_DEFAULT                                             0x02040810
-#define mmRPB_CID_QUEUE_WR_DEFAULT                                               0x00000000
-#define mmRPB_CID_QUEUE_RD_DEFAULT                                               0x00000000
-#define mmRPB_CID_QUEUE_EX_DEFAULT                                               0x00000000
-#define mmRPB_CID_QUEUE_EX_DATA_DEFAULT                                          0x00000000
-#define mmRPB_SWITCH_CNTL2_DEFAULT                                               0x02040810
-#define mmRPB_DEINTRLV_COMBINE_CNTL_DEFAULT                                      0x00000004
-#define mmRPB_VC_SWITCH_RDWR_DEFAULT                                             0x00004040
-#define mmRPB_PERFCOUNTER_LO_DEFAULT                                             0x00000000
-#define mmRPB_PERFCOUNTER_HI_DEFAULT                                             0x00000000
-#define mmRPB_PERFCOUNTER0_CFG_DEFAULT                                           0x00000000
-#define mmRPB_PERFCOUNTER1_CFG_DEFAULT                                           0x00000000
-#define mmRPB_PERFCOUNTER2_CFG_DEFAULT                                           0x00000000
-#define mmRPB_PERFCOUNTER3_CFG_DEFAULT                                           0x00000000
-#define mmRPB_PERFCOUNTER_RSLT_CNTL_DEFAULT                                      0x04000000
-#define mmRPB_RD_QUEUE_CNTL_DEFAULT                                              0x00000000
-#define mmRPB_RD_QUEUE_CNTL2_DEFAULT                                             0x00000000
-#define mmRPB_WR_QUEUE_CNTL_DEFAULT                                              0x00000000
-#define mmRPB_WR_QUEUE_CNTL2_DEFAULT                                             0x00000000
-#define mmRPB_EA_QUEUE_WR_DEFAULT                                                0x00000000
-#define mmRPB_ATS_CNTL_DEFAULT                                                   0x58088422
-#define mmRPB_ATS_CNTL2_DEFAULT                                                  0x00050b13
-#define mmRPB_SDPPORT_CNTL_DEFAULT                                               0x0fd14814
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_offset.h
deleted file mode 100644
index 80042e1c8770..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_offset.h
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _athub_1_0_OFFSET_HEADER
-#define _athub_1_0_OFFSET_HEADER
-
-
-
-// addressBlock: athub_atsdec
-// base address: 0x3080
-#define mmATC_ATS_CNTL                                                                                 0x0000
-#define mmATC_ATS_CNTL_BASE_IDX                                                                        0
-#define mmATC_ATS_STATUS                                                                               0x0003
-#define mmATC_ATS_STATUS_BASE_IDX                                                                      0
-#define mmATC_ATS_FAULT_CNTL                                                                           0x0004
-#define mmATC_ATS_FAULT_CNTL_BASE_IDX                                                                  0
-#define mmATC_ATS_FAULT_STATUS_INFO                                                                    0x0005
-#define mmATC_ATS_FAULT_STATUS_INFO_BASE_IDX                                                           0
-#define mmATC_ATS_FAULT_STATUS_ADDR                                                                    0x0006
-#define mmATC_ATS_FAULT_STATUS_ADDR_BASE_IDX                                                           0
-#define mmATC_ATS_DEFAULT_PAGE_LOW                                                                     0x0007
-#define mmATC_ATS_DEFAULT_PAGE_LOW_BASE_IDX                                                            0
-#define mmATC_TRANS_FAULT_RSPCNTRL                                                                     0x0008
-#define mmATC_TRANS_FAULT_RSPCNTRL_BASE_IDX                                                            0
-#define mmATC_ATS_FAULT_STATUS_INFO2                                                                   0x0009
-#define mmATC_ATS_FAULT_STATUS_INFO2_BASE_IDX                                                          0
-#define mmATHUB_MISC_CNTL                                                                              0x000a
-#define mmATHUB_MISC_CNTL_BASE_IDX                                                                     0
-#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS                                                         0x000b
-#define mmATC_VMID_PASID_MAPPING_UPDATE_STATUS_BASE_IDX                                                0
-#define mmATC_VMID0_PASID_MAPPING                                                                      0x000c
-#define mmATC_VMID0_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID1_PASID_MAPPING                                                                      0x000d
-#define mmATC_VMID1_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID2_PASID_MAPPING                                                                      0x000e
-#define mmATC_VMID2_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID3_PASID_MAPPING                                                                      0x000f
-#define mmATC_VMID3_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID4_PASID_MAPPING                                                                      0x0010
-#define mmATC_VMID4_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID5_PASID_MAPPING                                                                      0x0011
-#define mmATC_VMID5_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID6_PASID_MAPPING                                                                      0x0012
-#define mmATC_VMID6_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID7_PASID_MAPPING                                                                      0x0013
-#define mmATC_VMID7_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID8_PASID_MAPPING                                                                      0x0014
-#define mmATC_VMID8_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID9_PASID_MAPPING                                                                      0x0015
-#define mmATC_VMID9_PASID_MAPPING_BASE_IDX                                                             0
-#define mmATC_VMID10_PASID_MAPPING                                                                     0x0016
-#define mmATC_VMID10_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID11_PASID_MAPPING                                                                     0x0017
-#define mmATC_VMID11_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID12_PASID_MAPPING                                                                     0x0018
-#define mmATC_VMID12_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID13_PASID_MAPPING                                                                     0x0019
-#define mmATC_VMID13_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID14_PASID_MAPPING                                                                     0x001a
-#define mmATC_VMID14_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID15_PASID_MAPPING                                                                     0x001b
-#define mmATC_VMID15_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_ATS_VMID_STATUS                                                                          0x001c
-#define mmATC_ATS_VMID_STATUS_BASE_IDX                                                                 0
-#define mmATC_ATS_GFX_ATCL2_STATUS                                                                     0x001d
-#define mmATC_ATS_GFX_ATCL2_STATUS_BASE_IDX                                                            0
-#define mmATC_PERFCOUNTER0_CFG                                                                         0x001e
-#define mmATC_PERFCOUNTER0_CFG_BASE_IDX                                                                0
-#define mmATC_PERFCOUNTER1_CFG                                                                         0x001f
-#define mmATC_PERFCOUNTER1_CFG_BASE_IDX                                                                0
-#define mmATC_PERFCOUNTER2_CFG                                                                         0x0020
-#define mmATC_PERFCOUNTER2_CFG_BASE_IDX                                                                0
-#define mmATC_PERFCOUNTER3_CFG                                                                         0x0021
-#define mmATC_PERFCOUNTER3_CFG_BASE_IDX                                                                0
-#define mmATC_PERFCOUNTER_RSLT_CNTL                                                                    0x0022
-#define mmATC_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                           0
-#define mmATC_PERFCOUNTER_LO                                                                           0x0023
-#define mmATC_PERFCOUNTER_LO_BASE_IDX                                                                  0
-#define mmATC_PERFCOUNTER_HI                                                                           0x0024
-#define mmATC_PERFCOUNTER_HI_BASE_IDX                                                                  0
-#define mmATHUB_PCIE_ATS_CNTL                                                                          0x0025
-#define mmATHUB_PCIE_ATS_CNTL_BASE_IDX                                                                 0
-#define mmATHUB_PCIE_PASID_CNTL                                                                        0x0026
-#define mmATHUB_PCIE_PASID_CNTL_BASE_IDX                                                               0
-#define mmATHUB_PCIE_PAGE_REQ_CNTL                                                                     0x0027
-#define mmATHUB_PCIE_PAGE_REQ_CNTL_BASE_IDX                                                            0
-#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC                                                           0x0028
-#define mmATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC_BASE_IDX                                                  0
-#define mmATHUB_COMMAND                                                                                0x0029
-#define mmATHUB_COMMAND_BASE_IDX                                                                       0
-#define mmATHUB_PCIE_ATS_CNTL_VF_0                                                                     0x002a
-#define mmATHUB_PCIE_ATS_CNTL_VF_0_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_1                                                                     0x002b
-#define mmATHUB_PCIE_ATS_CNTL_VF_1_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_2                                                                     0x002c
-#define mmATHUB_PCIE_ATS_CNTL_VF_2_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_3                                                                     0x002d
-#define mmATHUB_PCIE_ATS_CNTL_VF_3_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_4                                                                     0x002e
-#define mmATHUB_PCIE_ATS_CNTL_VF_4_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_5                                                                     0x002f
-#define mmATHUB_PCIE_ATS_CNTL_VF_5_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_6                                                                     0x0030
-#define mmATHUB_PCIE_ATS_CNTL_VF_6_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_7                                                                     0x0031
-#define mmATHUB_PCIE_ATS_CNTL_VF_7_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_8                                                                     0x0032
-#define mmATHUB_PCIE_ATS_CNTL_VF_8_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_9                                                                     0x0033
-#define mmATHUB_PCIE_ATS_CNTL_VF_9_BASE_IDX                                                            0
-#define mmATHUB_PCIE_ATS_CNTL_VF_10                                                                    0x0034
-#define mmATHUB_PCIE_ATS_CNTL_VF_10_BASE_IDX                                                           0
-#define mmATHUB_PCIE_ATS_CNTL_VF_11                                                                    0x0035
-#define mmATHUB_PCIE_ATS_CNTL_VF_11_BASE_IDX                                                           0
-#define mmATHUB_PCIE_ATS_CNTL_VF_12                                                                    0x0036
-#define mmATHUB_PCIE_ATS_CNTL_VF_12_BASE_IDX                                                           0
-#define mmATHUB_PCIE_ATS_CNTL_VF_13                                                                    0x0037
-#define mmATHUB_PCIE_ATS_CNTL_VF_13_BASE_IDX                                                           0
-#define mmATHUB_PCIE_ATS_CNTL_VF_14                                                                    0x0038
-#define mmATHUB_PCIE_ATS_CNTL_VF_14_BASE_IDX                                                           0
-#define mmATHUB_PCIE_ATS_CNTL_VF_15                                                                    0x0039
-#define mmATHUB_PCIE_ATS_CNTL_VF_15_BASE_IDX                                                           0
-#define mmATHUB_MEM_POWER_LS                                                                           0x003a
-#define mmATHUB_MEM_POWER_LS_BASE_IDX                                                                  0
-#define mmATS_IH_CREDIT                                                                                0x003b
-#define mmATS_IH_CREDIT_BASE_IDX                                                                       0
-#define mmATHUB_IH_CREDIT                                                                              0x003c
-#define mmATHUB_IH_CREDIT_BASE_IDX                                                                     0
-#define mmATC_VMID16_PASID_MAPPING                                                                     0x003d
-#define mmATC_VMID16_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID17_PASID_MAPPING                                                                     0x003e
-#define mmATC_VMID17_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID18_PASID_MAPPING                                                                     0x003f
-#define mmATC_VMID18_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID19_PASID_MAPPING                                                                     0x0040
-#define mmATC_VMID19_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID20_PASID_MAPPING                                                                     0x0041
-#define mmATC_VMID20_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID21_PASID_MAPPING                                                                     0x0042
-#define mmATC_VMID21_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID22_PASID_MAPPING                                                                     0x0043
-#define mmATC_VMID22_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID23_PASID_MAPPING                                                                     0x0044
-#define mmATC_VMID23_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID24_PASID_MAPPING                                                                     0x0045
-#define mmATC_VMID24_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID25_PASID_MAPPING                                                                     0x0046
-#define mmATC_VMID25_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID26_PASID_MAPPING                                                                     0x0047
-#define mmATC_VMID26_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID27_PASID_MAPPING                                                                     0x0048
-#define mmATC_VMID27_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID28_PASID_MAPPING                                                                     0x0049
-#define mmATC_VMID28_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID29_PASID_MAPPING                                                                     0x004a
-#define mmATC_VMID29_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID30_PASID_MAPPING                                                                     0x004b
-#define mmATC_VMID30_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_VMID31_PASID_MAPPING                                                                     0x004c
-#define mmATC_VMID31_PASID_MAPPING_BASE_IDX                                                            0
-#define mmATC_ATS_MMHUB_ATCL2_STATUS                                                                   0x004d
-#define mmATC_ATS_MMHUB_ATCL2_STATUS_BASE_IDX                                                          0
-#define mmATHUB_SHARED_VIRT_RESET_REQ                                                                  0x004e
-#define mmATHUB_SHARED_VIRT_RESET_REQ_BASE_IDX                                                         0
-#define mmATHUB_SHARED_ACTIVE_FCN_ID                                                                   0x004f
-#define mmATHUB_SHARED_ACTIVE_FCN_ID_BASE_IDX                                                          0
-#define mmATC_ATS_SDPPORT_CNTL                                                                         0x0050
-#define mmATC_ATS_SDPPORT_CNTL_BASE_IDX                                                                0
-#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT                                                               0x0052
-#define mmATC_ATS_VMID_SNAPSHOT_GFX_STAT_BASE_IDX                                                      0
-#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT                                                             0x0053
-#define mmATC_ATS_VMID_SNAPSHOT_MMHUB_STAT_BASE_IDX                                                    0
-
-
-// addressBlock: athub_xpbdec
-// base address: 0x31f0
-#define mmXPB_RTR_SRC_APRTR0                                                                           0x005c
-#define mmXPB_RTR_SRC_APRTR0_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR1                                                                           0x005d
-#define mmXPB_RTR_SRC_APRTR1_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR2                                                                           0x005e
-#define mmXPB_RTR_SRC_APRTR2_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR3                                                                           0x005f
-#define mmXPB_RTR_SRC_APRTR3_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR4                                                                           0x0060
-#define mmXPB_RTR_SRC_APRTR4_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR5                                                                           0x0061
-#define mmXPB_RTR_SRC_APRTR5_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR6                                                                           0x0062
-#define mmXPB_RTR_SRC_APRTR6_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR7                                                                           0x0063
-#define mmXPB_RTR_SRC_APRTR7_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR8                                                                           0x0064
-#define mmXPB_RTR_SRC_APRTR8_BASE_IDX                                                                  0
-#define mmXPB_RTR_SRC_APRTR9                                                                           0x0065
-#define mmXPB_RTR_SRC_APRTR9_BASE_IDX                                                                  0
-#define mmXPB_XDMA_RTR_SRC_APRTR0                                                                      0x0066
-#define mmXPB_XDMA_RTR_SRC_APRTR0_BASE_IDX                                                             0
-#define mmXPB_XDMA_RTR_SRC_APRTR1                                                                      0x0067
-#define mmXPB_XDMA_RTR_SRC_APRTR1_BASE_IDX                                                             0
-#define mmXPB_XDMA_RTR_SRC_APRTR2                                                                      0x0068
-#define mmXPB_XDMA_RTR_SRC_APRTR2_BASE_IDX                                                             0
-#define mmXPB_XDMA_RTR_SRC_APRTR3                                                                      0x0069
-#define mmXPB_XDMA_RTR_SRC_APRTR3_BASE_IDX                                                             0
-#define mmXPB_RTR_DEST_MAP0                                                                            0x006a
-#define mmXPB_RTR_DEST_MAP0_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP1                                                                            0x006b
-#define mmXPB_RTR_DEST_MAP1_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP2                                                                            0x006c
-#define mmXPB_RTR_DEST_MAP2_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP3                                                                            0x006d
-#define mmXPB_RTR_DEST_MAP3_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP4                                                                            0x006e
-#define mmXPB_RTR_DEST_MAP4_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP5                                                                            0x006f
-#define mmXPB_RTR_DEST_MAP5_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP6                                                                            0x0070
-#define mmXPB_RTR_DEST_MAP6_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP7                                                                            0x0071
-#define mmXPB_RTR_DEST_MAP7_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP8                                                                            0x0072
-#define mmXPB_RTR_DEST_MAP8_BASE_IDX                                                                   0
-#define mmXPB_RTR_DEST_MAP9                                                                            0x0073
-#define mmXPB_RTR_DEST_MAP9_BASE_IDX                                                                   0
-#define mmXPB_XDMA_RTR_DEST_MAP0                                                                       0x0074
-#define mmXPB_XDMA_RTR_DEST_MAP0_BASE_IDX                                                              0
-#define mmXPB_XDMA_RTR_DEST_MAP1                                                                       0x0075
-#define mmXPB_XDMA_RTR_DEST_MAP1_BASE_IDX                                                              0
-#define mmXPB_XDMA_RTR_DEST_MAP2                                                                       0x0076
-#define mmXPB_XDMA_RTR_DEST_MAP2_BASE_IDX                                                              0
-#define mmXPB_XDMA_RTR_DEST_MAP3                                                                       0x0077
-#define mmXPB_XDMA_RTR_DEST_MAP3_BASE_IDX                                                              0
-#define mmXPB_CLG_CFG0                                                                                 0x0078
-#define mmXPB_CLG_CFG0_BASE_IDX                                                                        0
-#define mmXPB_CLG_CFG1                                                                                 0x0079
-#define mmXPB_CLG_CFG1_BASE_IDX                                                                        0
-#define mmXPB_CLG_CFG2                                                                                 0x007a
-#define mmXPB_CLG_CFG2_BASE_IDX                                                                        0
-#define mmXPB_CLG_CFG3                                                                                 0x007b
-#define mmXPB_CLG_CFG3_BASE_IDX                                                                        0
-#define mmXPB_CLG_CFG4                                                                                 0x007c
-#define mmXPB_CLG_CFG4_BASE_IDX                                                                        0
-#define mmXPB_CLG_CFG5                                                                                 0x007d
-#define mmXPB_CLG_CFG5_BASE_IDX                                                                        0
-#define mmXPB_CLG_CFG6                                                                                 0x007e
-#define mmXPB_CLG_CFG6_BASE_IDX                                                                        0
-#define mmXPB_CLG_CFG7                                                                                 0x007f
-#define mmXPB_CLG_CFG7_BASE_IDX                                                                        0
-#define mmXPB_CLG_EXTRA                                                                                0x0080
-#define mmXPB_CLG_EXTRA_BASE_IDX                                                                       0
-#define mmXPB_CLG_EXTRA_MSK                                                                            0x0081
-#define mmXPB_CLG_EXTRA_MSK_BASE_IDX                                                                   0
-#define mmXPB_LB_ADDR                                                                                  0x0082
-#define mmXPB_LB_ADDR_BASE_IDX                                                                         0
-#define mmXPB_WCB_STS                                                                                  0x0083
-#define mmXPB_WCB_STS_BASE_IDX                                                                         0
-#define mmXPB_HST_CFG                                                                                  0x0084
-#define mmXPB_HST_CFG_BASE_IDX                                                                         0
-#define mmXPB_P2P_BAR_CFG                                                                              0x0085
-#define mmXPB_P2P_BAR_CFG_BASE_IDX                                                                     0
-#define mmXPB_P2P_BAR0                                                                                 0x0086
-#define mmXPB_P2P_BAR0_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR1                                                                                 0x0087
-#define mmXPB_P2P_BAR1_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR2                                                                                 0x0088
-#define mmXPB_P2P_BAR2_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR3                                                                                 0x0089
-#define mmXPB_P2P_BAR3_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR4                                                                                 0x008a
-#define mmXPB_P2P_BAR4_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR5                                                                                 0x008b
-#define mmXPB_P2P_BAR5_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR6                                                                                 0x008c
-#define mmXPB_P2P_BAR6_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR7                                                                                 0x008d
-#define mmXPB_P2P_BAR7_BASE_IDX                                                                        0
-#define mmXPB_P2P_BAR_SETUP                                                                            0x008e
-#define mmXPB_P2P_BAR_SETUP_BASE_IDX                                                                   0
-#define mmXPB_P2P_BAR_DELTA_ABOVE                                                                      0x0090
-#define mmXPB_P2P_BAR_DELTA_ABOVE_BASE_IDX                                                             0
-#define mmXPB_P2P_BAR_DELTA_BELOW                                                                      0x0091
-#define mmXPB_P2P_BAR_DELTA_BELOW_BASE_IDX                                                             0
-#define mmXPB_PEER_SYS_BAR0                                                                            0x0092
-#define mmXPB_PEER_SYS_BAR0_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR1                                                                            0x0093
-#define mmXPB_PEER_SYS_BAR1_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR2                                                                            0x0094
-#define mmXPB_PEER_SYS_BAR2_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR3                                                                            0x0095
-#define mmXPB_PEER_SYS_BAR3_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR4                                                                            0x0096
-#define mmXPB_PEER_SYS_BAR4_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR5                                                                            0x0097
-#define mmXPB_PEER_SYS_BAR5_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR6                                                                            0x0098
-#define mmXPB_PEER_SYS_BAR6_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR7                                                                            0x0099
-#define mmXPB_PEER_SYS_BAR7_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR8                                                                            0x009a
-#define mmXPB_PEER_SYS_BAR8_BASE_IDX                                                                   0
-#define mmXPB_PEER_SYS_BAR9                                                                            0x009b
-#define mmXPB_PEER_SYS_BAR9_BASE_IDX                                                                   0
-#define mmXPB_XDMA_PEER_SYS_BAR0                                                                       0x009c
-#define mmXPB_XDMA_PEER_SYS_BAR0_BASE_IDX                                                              0
-#define mmXPB_XDMA_PEER_SYS_BAR1                                                                       0x009d
-#define mmXPB_XDMA_PEER_SYS_BAR1_BASE_IDX                                                              0
-#define mmXPB_XDMA_PEER_SYS_BAR2                                                                       0x009e
-#define mmXPB_XDMA_PEER_SYS_BAR2_BASE_IDX                                                              0
-#define mmXPB_XDMA_PEER_SYS_BAR3                                                                       0x009f
-#define mmXPB_XDMA_PEER_SYS_BAR3_BASE_IDX                                                              0
-#define mmXPB_CLK_GAT                                                                                  0x00a0
-#define mmXPB_CLK_GAT_BASE_IDX                                                                         0
-#define mmXPB_INTF_CFG                                                                                 0x00a1
-#define mmXPB_INTF_CFG_BASE_IDX                                                                        0
-#define mmXPB_INTF_STS                                                                                 0x00a2
-#define mmXPB_INTF_STS_BASE_IDX                                                                        0
-#define mmXPB_PIPE_STS                                                                                 0x00a3
-#define mmXPB_PIPE_STS_BASE_IDX                                                                        0
-#define mmXPB_SUB_CTRL                                                                                 0x00a4
-#define mmXPB_SUB_CTRL_BASE_IDX                                                                        0
-#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB                                                                 0x00a5
-#define mmXPB_MAP_INVERT_FLUSH_NUM_LSB_BASE_IDX                                                        0
-#define mmXPB_PERF_KNOBS                                                                               0x00a6
-#define mmXPB_PERF_KNOBS_BASE_IDX                                                                      0
-#define mmXPB_STICKY                                                                                   0x00a7
-#define mmXPB_STICKY_BASE_IDX                                                                          0
-#define mmXPB_STICKY_W1C                                                                               0x00a8
-#define mmXPB_STICKY_W1C_BASE_IDX                                                                      0
-#define mmXPB_MISC_CFG                                                                                 0x00a9
-#define mmXPB_MISC_CFG_BASE_IDX                                                                        0
-#define mmXPB_INTF_CFG2                                                                                0x00aa
-#define mmXPB_INTF_CFG2_BASE_IDX                                                                       0
-#define mmXPB_CLG_EXTRA_RD                                                                             0x00ab
-#define mmXPB_CLG_EXTRA_RD_BASE_IDX                                                                    0
-#define mmXPB_CLG_EXTRA_MSK_RD                                                                         0x00ac
-#define mmXPB_CLG_EXTRA_MSK_RD_BASE_IDX                                                                0
-#define mmXPB_CLG_GFX_MATCH                                                                            0x00ad
-#define mmXPB_CLG_GFX_MATCH_BASE_IDX                                                                   0
-#define mmXPB_CLG_GFX_MATCH_MSK                                                                        0x00ae
-#define mmXPB_CLG_GFX_MATCH_MSK_BASE_IDX                                                               0
-#define mmXPB_CLG_MM_MATCH                                                                             0x00af
-#define mmXPB_CLG_MM_MATCH_BASE_IDX                                                                    0
-#define mmXPB_CLG_MM_MATCH_MSK                                                                         0x00b0
-#define mmXPB_CLG_MM_MATCH_MSK_BASE_IDX                                                                0
-#define mmXPB_CLG_GFX_UNITID_MAPPING0                                                                  0x00b1
-#define mmXPB_CLG_GFX_UNITID_MAPPING0_BASE_IDX                                                         0
-#define mmXPB_CLG_GFX_UNITID_MAPPING1                                                                  0x00b2
-#define mmXPB_CLG_GFX_UNITID_MAPPING1_BASE_IDX                                                         0
-#define mmXPB_CLG_GFX_UNITID_MAPPING2                                                                  0x00b3
-#define mmXPB_CLG_GFX_UNITID_MAPPING2_BASE_IDX                                                         0
-#define mmXPB_CLG_GFX_UNITID_MAPPING3                                                                  0x00b4
-#define mmXPB_CLG_GFX_UNITID_MAPPING3_BASE_IDX                                                         0
-#define mmXPB_CLG_GFX_UNITID_MAPPING4                                                                  0x00b5
-#define mmXPB_CLG_GFX_UNITID_MAPPING4_BASE_IDX                                                         0
-#define mmXPB_CLG_GFX_UNITID_MAPPING5                                                                  0x00b6
-#define mmXPB_CLG_GFX_UNITID_MAPPING5_BASE_IDX                                                         0
-#define mmXPB_CLG_GFX_UNITID_MAPPING6                                                                  0x00b7
-#define mmXPB_CLG_GFX_UNITID_MAPPING6_BASE_IDX                                                         0
-#define mmXPB_CLG_GFX_UNITID_MAPPING7                                                                  0x00b8
-#define mmXPB_CLG_GFX_UNITID_MAPPING7_BASE_IDX                                                         0
-#define mmXPB_CLG_MM_UNITID_MAPPING0                                                                   0x00b9
-#define mmXPB_CLG_MM_UNITID_MAPPING0_BASE_IDX                                                          0
-#define mmXPB_CLG_MM_UNITID_MAPPING1                                                                   0x00ba
-#define mmXPB_CLG_MM_UNITID_MAPPING1_BASE_IDX                                                          0
-#define mmXPB_CLG_MM_UNITID_MAPPING2                                                                   0x00bb
-#define mmXPB_CLG_MM_UNITID_MAPPING2_BASE_IDX                                                          0
-#define mmXPB_CLG_MM_UNITID_MAPPING3                                                                   0x00bc
-#define mmXPB_CLG_MM_UNITID_MAPPING3_BASE_IDX                                                          0
-
-
-// addressBlock: athub_rpbdec
-// base address: 0x33b0
-#define mmRPB_PASSPW_CONF                                                                              0x00cc
-#define mmRPB_PASSPW_CONF_BASE_IDX                                                                     0
-#define mmRPB_BLOCKLEVEL_CONF                                                                          0x00cd
-#define mmRPB_BLOCKLEVEL_CONF_BASE_IDX                                                                 0
-#define mmRPB_TAG_CONF                                                                                 0x00cf
-#define mmRPB_TAG_CONF_BASE_IDX                                                                        0
-#define mmRPB_EFF_CNTL                                                                                 0x00d1
-#define mmRPB_EFF_CNTL_BASE_IDX                                                                        0
-#define mmRPB_ARB_CNTL                                                                                 0x00d2
-#define mmRPB_ARB_CNTL_BASE_IDX                                                                        0
-#define mmRPB_ARB_CNTL2                                                                                0x00d3
-#define mmRPB_ARB_CNTL2_BASE_IDX                                                                       0
-#define mmRPB_BIF_CNTL                                                                                 0x00d4
-#define mmRPB_BIF_CNTL_BASE_IDX                                                                        0
-#define mmRPB_WR_SWITCH_CNTL                                                                           0x00d5
-#define mmRPB_WR_SWITCH_CNTL_BASE_IDX                                                                  0
-#define mmRPB_RD_SWITCH_CNTL                                                                           0x00d7
-#define mmRPB_RD_SWITCH_CNTL_BASE_IDX                                                                  0
-#define mmRPB_CID_QUEUE_WR                                                                             0x00d8
-#define mmRPB_CID_QUEUE_WR_BASE_IDX                                                                    0
-#define mmRPB_CID_QUEUE_RD                                                                             0x00d9
-#define mmRPB_CID_QUEUE_RD_BASE_IDX                                                                    0
-#define mmRPB_CID_QUEUE_EX                                                                             0x00dc
-#define mmRPB_CID_QUEUE_EX_BASE_IDX                                                                    0
-#define mmRPB_CID_QUEUE_EX_DATA                                                                        0x00dd
-#define mmRPB_CID_QUEUE_EX_DATA_BASE_IDX                                                               0
-#define mmRPB_SWITCH_CNTL2                                                                             0x00de
-#define mmRPB_SWITCH_CNTL2_BASE_IDX                                                                    0
-#define mmRPB_DEINTRLV_COMBINE_CNTL                                                                    0x00df
-#define mmRPB_DEINTRLV_COMBINE_CNTL_BASE_IDX                                                           0
-#define mmRPB_VC_SWITCH_RDWR                                                                           0x00e0
-#define mmRPB_VC_SWITCH_RDWR_BASE_IDX                                                                  0
-#define mmRPB_PERFCOUNTER_LO                                                                           0x00e1
-#define mmRPB_PERFCOUNTER_LO_BASE_IDX                                                                  0
-#define mmRPB_PERFCOUNTER_HI                                                                           0x00e2
-#define mmRPB_PERFCOUNTER_HI_BASE_IDX                                                                  0
-#define mmRPB_PERFCOUNTER0_CFG                                                                         0x00e3
-#define mmRPB_PERFCOUNTER0_CFG_BASE_IDX                                                                0
-#define mmRPB_PERFCOUNTER1_CFG                                                                         0x00e4
-#define mmRPB_PERFCOUNTER1_CFG_BASE_IDX                                                                0
-#define mmRPB_PERFCOUNTER2_CFG                                                                         0x00e5
-#define mmRPB_PERFCOUNTER2_CFG_BASE_IDX                                                                0
-#define mmRPB_PERFCOUNTER3_CFG                                                                         0x00e6
-#define mmRPB_PERFCOUNTER3_CFG_BASE_IDX                                                                0
-#define mmRPB_PERFCOUNTER_RSLT_CNTL                                                                    0x00e7
-#define mmRPB_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                           0
-#define mmRPB_RD_QUEUE_CNTL                                                                            0x00e9
-#define mmRPB_RD_QUEUE_CNTL_BASE_IDX                                                                   0
-#define mmRPB_RD_QUEUE_CNTL2                                                                           0x00ea
-#define mmRPB_RD_QUEUE_CNTL2_BASE_IDX                                                                  0
-#define mmRPB_WR_QUEUE_CNTL                                                                            0x00eb
-#define mmRPB_WR_QUEUE_CNTL_BASE_IDX                                                                   0
-#define mmRPB_WR_QUEUE_CNTL2                                                                           0x00ec
-#define mmRPB_WR_QUEUE_CNTL2_BASE_IDX                                                                  0
-#define mmRPB_EA_QUEUE_WR                                                                              0x00ed
-#define mmRPB_EA_QUEUE_WR_BASE_IDX                                                                     0
-#define mmRPB_ATS_CNTL                                                                                 0x00ee
-#define mmRPB_ATS_CNTL_BASE_IDX                                                                        0
-#define mmRPB_ATS_CNTL2                                                                                0x00ef
-#define mmRPB_ATS_CNTL2_BASE_IDX                                                                       0
-#define mmRPB_SDPPORT_CNTL                                                                             0x00f0
-#define mmRPB_SDPPORT_CNTL_BASE_IDX                                                                    0
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_sh_mask.h
deleted file mode 100644
index 777b05c89708..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/ATHUB/athub_1_0_sh_mask.h
+++ /dev/null
@@ -1,2045 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _athub_1_0_SH_MASK_HEADER
-#define _athub_1_0_SH_MASK_HEADER
-
-
-// addressBlock: athub_atsdec
-//ATC_ATS_CNTL
-#define ATC_ATS_CNTL__DISABLE_ATC__SHIFT                                                                      0x0
-#define ATC_ATS_CNTL__DISABLE_PRI__SHIFT                                                                      0x1
-#define ATC_ATS_CNTL__DISABLE_PASID__SHIFT                                                                    0x2
-#define ATC_ATS_CNTL__CREDITS_ATS_RPB__SHIFT                                                                  0x8
-#define ATC_ATS_CNTL__INVALIDATION_LOG_KEEP_ORDER__SHIFT                                                      0x14
-#define ATC_ATS_CNTL__TRANS_LOG_KEEP_ORDER__SHIFT                                                             0x15
-#define ATC_ATS_CNTL__TRANS_EXE_RETURN__SHIFT                                                                 0x16
-#define ATC_ATS_CNTL__DISABLE_ATC_MASK                                                                        0x00000001L
-#define ATC_ATS_CNTL__DISABLE_PRI_MASK                                                                        0x00000002L
-#define ATC_ATS_CNTL__DISABLE_PASID_MASK                                                                      0x00000004L
-#define ATC_ATS_CNTL__CREDITS_ATS_RPB_MASK                                                                    0x00003F00L
-#define ATC_ATS_CNTL__INVALIDATION_LOG_KEEP_ORDER_MASK                                                        0x00100000L
-#define ATC_ATS_CNTL__TRANS_LOG_KEEP_ORDER_MASK                                                               0x00200000L
-#define ATC_ATS_CNTL__TRANS_EXE_RETURN_MASK                                                                   0x00C00000L
-//ATC_ATS_STATUS
-#define ATC_ATS_STATUS__BUSY__SHIFT                                                                           0x0
-#define ATC_ATS_STATUS__CRASHED__SHIFT                                                                        0x1
-#define ATC_ATS_STATUS__DEADLOCK_DETECTION__SHIFT                                                             0x2
-#define ATC_ATS_STATUS__FLUSH_INVALIDATION_OUTSTANDING__SHIFT                                                 0x3
-#define ATC_ATS_STATUS__NONFLUSH_INVALIDATION_OUTSTANDING__SHIFT                                              0x6
-#define ATC_ATS_STATUS__BUSY_MASK                                                                             0x00000001L
-#define ATC_ATS_STATUS__CRASHED_MASK                                                                          0x00000002L
-#define ATC_ATS_STATUS__DEADLOCK_DETECTION_MASK                                                               0x00000004L
-#define ATC_ATS_STATUS__FLUSH_INVALIDATION_OUTSTANDING_MASK                                                   0x00000038L
-#define ATC_ATS_STATUS__NONFLUSH_INVALIDATION_OUTSTANDING_MASK                                                0x000001C0L
-//ATC_ATS_FAULT_CNTL
-#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG__SHIFT                                                         0x0
-#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE__SHIFT                                                      0xa
-#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE__SHIFT                                                          0x14
-#define ATC_ATS_FAULT_CNTL__FAULT_REGISTER_LOG_MASK                                                           0x000001FFL
-#define ATC_ATS_FAULT_CNTL__FAULT_INTERRUPT_TABLE_MASK                                                        0x0007FC00L
-#define ATC_ATS_FAULT_CNTL__FAULT_CRASH_TABLE_MASK                                                            0x1FF00000L
-//ATC_ATS_FAULT_STATUS_INFO
-#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE__SHIFT                                                          0x0
-#define ATC_ATS_FAULT_STATUS_INFO__VMID__SHIFT                                                                0xa
-#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO__SHIFT                                                          0xf
-#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2__SHIFT                                                         0x10
-#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION__SHIFT                                                        0x11
-#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST__SHIFT                                                        0x12
-#define ATC_ATS_FAULT_STATUS_INFO__STATUS__SHIFT                                                              0x13
-#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH__SHIFT                                                      0x18
-#define ATC_ATS_FAULT_STATUS_INFO__FAULT_TYPE_MASK                                                            0x000001FFL
-#define ATC_ATS_FAULT_STATUS_INFO__VMID_MASK                                                                  0x00007C00L
-#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO_MASK                                                            0x00008000L
-#define ATC_ATS_FAULT_STATUS_INFO__EXTRA_INFO2_MASK                                                           0x00010000L
-#define ATC_ATS_FAULT_STATUS_INFO__INVALIDATION_MASK                                                          0x00020000L
-#define ATC_ATS_FAULT_STATUS_INFO__PAGE_REQUEST_MASK                                                          0x00040000L
-#define ATC_ATS_FAULT_STATUS_INFO__STATUS_MASK                                                                0x00F80000L
-#define ATC_ATS_FAULT_STATUS_INFO__PAGE_ADDR_HIGH_MASK                                                        0x0F000000L
-//ATC_ATS_FAULT_STATUS_ADDR
-#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR__SHIFT                                                           0x0
-#define ATC_ATS_FAULT_STATUS_ADDR__PAGE_ADDR_MASK                                                             0xFFFFFFFFL
-//ATC_ATS_DEFAULT_PAGE_LOW
-#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE__SHIFT                                                         0x0
-#define ATC_ATS_DEFAULT_PAGE_LOW__DEFAULT_PAGE_MASK                                                           0xFFFFFFFFL
-//ATC_TRANS_FAULT_RSPCNTRL
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID0__SHIFT                                                                0x0
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID1__SHIFT                                                                0x1
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID2__SHIFT                                                                0x2
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID3__SHIFT                                                                0x3
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID4__SHIFT                                                                0x4
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID5__SHIFT                                                                0x5
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID6__SHIFT                                                                0x6
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID7__SHIFT                                                                0x7
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID8__SHIFT                                                                0x8
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID9__SHIFT                                                                0x9
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID10__SHIFT                                                               0xa
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID11__SHIFT                                                               0xb
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID12__SHIFT                                                               0xc
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID13__SHIFT                                                               0xd
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID14__SHIFT                                                               0xe
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID15__SHIFT                                                               0xf
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID16__SHIFT                                                               0x10
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID17__SHIFT                                                               0x11
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID18__SHIFT                                                               0x12
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID19__SHIFT                                                               0x13
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID20__SHIFT                                                               0x14
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID21__SHIFT                                                               0x15
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID22__SHIFT                                                               0x16
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID23__SHIFT                                                               0x17
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID24__SHIFT                                                               0x18
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID25__SHIFT                                                               0x19
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID26__SHIFT                                                               0x1a
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID27__SHIFT                                                               0x1b
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID28__SHIFT                                                               0x1c
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID29__SHIFT                                                               0x1d
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID30__SHIFT                                                               0x1e
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID31__SHIFT                                                               0x1f
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID0_MASK                                                                  0x00000001L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID1_MASK                                                                  0x00000002L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID2_MASK                                                                  0x00000004L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID3_MASK                                                                  0x00000008L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID4_MASK                                                                  0x00000010L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID5_MASK                                                                  0x00000020L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID6_MASK                                                                  0x00000040L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID7_MASK                                                                  0x00000080L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID8_MASK                                                                  0x00000100L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID9_MASK                                                                  0x00000200L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID10_MASK                                                                 0x00000400L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID11_MASK                                                                 0x00000800L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID12_MASK                                                                 0x00001000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID13_MASK                                                                 0x00002000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID14_MASK                                                                 0x00004000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID15_MASK                                                                 0x00008000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID16_MASK                                                                 0x00010000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID17_MASK                                                                 0x00020000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID18_MASK                                                                 0x00040000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID19_MASK                                                                 0x00080000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID20_MASK                                                                 0x00100000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID21_MASK                                                                 0x00200000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID22_MASK                                                                 0x00400000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID23_MASK                                                                 0x00800000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID24_MASK                                                                 0x01000000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID25_MASK                                                                 0x02000000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID26_MASK                                                                 0x04000000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID27_MASK                                                                 0x08000000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID28_MASK                                                                 0x10000000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID29_MASK                                                                 0x20000000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID30_MASK                                                                 0x40000000L
-#define ATC_TRANS_FAULT_RSPCNTRL__VMID31_MASK                                                                 0x80000000L
-//ATC_ATS_FAULT_STATUS_INFO2
-#define ATC_ATS_FAULT_STATUS_INFO2__VF__SHIFT                                                                 0x0
-#define ATC_ATS_FAULT_STATUS_INFO2__VFID__SHIFT                                                               0x1
-#define ATC_ATS_FAULT_STATUS_INFO2__MMHUB_INV_VMID__SHIFT                                                     0x9
-#define ATC_ATS_FAULT_STATUS_INFO2__VF_MASK                                                                   0x00000001L
-#define ATC_ATS_FAULT_STATUS_INFO2__VFID_MASK                                                                 0x0000001EL
-#define ATC_ATS_FAULT_STATUS_INFO2__MMHUB_INV_VMID_MASK                                                       0x00003E00L
-//ATHUB_MISC_CNTL
-#define ATHUB_MISC_CNTL__CG_OFFDLY__SHIFT                                                                     0x6
-#define ATHUB_MISC_CNTL__CG_ENABLE__SHIFT                                                                     0x12
-#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE__SHIFT                                                              0x13
-#define ATHUB_MISC_CNTL__PG_ENABLE__SHIFT                                                                     0x14
-#define ATHUB_MISC_CNTL__PG_OFFDLY__SHIFT                                                                     0x15
-#define ATHUB_MISC_CNTL__CG_STATUS__SHIFT                                                                     0x1b
-#define ATHUB_MISC_CNTL__PG_STATUS__SHIFT                                                                     0x1c
-#define ATHUB_MISC_CNTL__CG_OFFDLY_MASK                                                                       0x00000FC0L
-#define ATHUB_MISC_CNTL__CG_ENABLE_MASK                                                                       0x00040000L
-#define ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK                                                                0x00080000L
-#define ATHUB_MISC_CNTL__PG_ENABLE_MASK                                                                       0x00100000L
-#define ATHUB_MISC_CNTL__PG_OFFDLY_MASK                                                                       0x07E00000L
-#define ATHUB_MISC_CNTL__CG_STATUS_MASK                                                                       0x08000000L
-#define ATHUB_MISC_CNTL__PG_STATUS_MASK                                                                       0x10000000L
-//ATC_VMID_PASID_MAPPING_UPDATE_STATUS
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED__SHIFT                                 0x0
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED__SHIFT                                 0x1
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED__SHIFT                                 0x2
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED__SHIFT                                 0x3
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED__SHIFT                                 0x4
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED__SHIFT                                 0x5
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED__SHIFT                                 0x6
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED__SHIFT                                 0x7
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED__SHIFT                                 0x8
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED__SHIFT                                 0x9
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED__SHIFT                                0xa
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED__SHIFT                                0xb
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED__SHIFT                                0xc
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED__SHIFT                                0xd
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED__SHIFT                                0xe
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED__SHIFT                                0xf
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID16_REMAPPING_FINISHED__SHIFT                                0x10
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID17_REMAPPING_FINISHED__SHIFT                                0x11
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID18_REMAPPING_FINISHED__SHIFT                                0x12
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID19_REMAPPING_FINISHED__SHIFT                                0x13
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID20_REMAPPING_FINISHED__SHIFT                                0x14
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID21_REMAPPING_FINISHED__SHIFT                                0x15
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID22_REMAPPING_FINISHED__SHIFT                                0x16
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID23_REMAPPING_FINISHED__SHIFT                                0x17
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID24_REMAPPING_FINISHED__SHIFT                                0x18
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID25_REMAPPING_FINISHED__SHIFT                                0x19
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID26_REMAPPING_FINISHED__SHIFT                                0x1a
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID27_REMAPPING_FINISHED__SHIFT                                0x1b
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID28_REMAPPING_FINISHED__SHIFT                                0x1c
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID29_REMAPPING_FINISHED__SHIFT                                0x1d
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID30_REMAPPING_FINISHED__SHIFT                                0x1e
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID31_REMAPPING_FINISHED__SHIFT                                0x1f
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID0_REMAPPING_FINISHED_MASK                                   0x00000001L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID1_REMAPPING_FINISHED_MASK                                   0x00000002L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID2_REMAPPING_FINISHED_MASK                                   0x00000004L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID3_REMAPPING_FINISHED_MASK                                   0x00000008L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID4_REMAPPING_FINISHED_MASK                                   0x00000010L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID5_REMAPPING_FINISHED_MASK                                   0x00000020L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID6_REMAPPING_FINISHED_MASK                                   0x00000040L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID7_REMAPPING_FINISHED_MASK                                   0x00000080L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID8_REMAPPING_FINISHED_MASK                                   0x00000100L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID9_REMAPPING_FINISHED_MASK                                   0x00000200L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID10_REMAPPING_FINISHED_MASK                                  0x00000400L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID11_REMAPPING_FINISHED_MASK                                  0x00000800L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID12_REMAPPING_FINISHED_MASK                                  0x00001000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID13_REMAPPING_FINISHED_MASK                                  0x00002000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID14_REMAPPING_FINISHED_MASK                                  0x00004000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID15_REMAPPING_FINISHED_MASK                                  0x00008000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID16_REMAPPING_FINISHED_MASK                                  0x00010000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID17_REMAPPING_FINISHED_MASK                                  0x00020000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID18_REMAPPING_FINISHED_MASK                                  0x00040000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID19_REMAPPING_FINISHED_MASK                                  0x00080000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID20_REMAPPING_FINISHED_MASK                                  0x00100000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID21_REMAPPING_FINISHED_MASK                                  0x00200000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID22_REMAPPING_FINISHED_MASK                                  0x00400000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID23_REMAPPING_FINISHED_MASK                                  0x00800000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID24_REMAPPING_FINISHED_MASK                                  0x01000000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID25_REMAPPING_FINISHED_MASK                                  0x02000000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID26_REMAPPING_FINISHED_MASK                                  0x04000000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID27_REMAPPING_FINISHED_MASK                                  0x08000000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID28_REMAPPING_FINISHED_MASK                                  0x10000000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID29_REMAPPING_FINISHED_MASK                                  0x20000000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID30_REMAPPING_FINISHED_MASK                                  0x40000000L
-#define ATC_VMID_PASID_MAPPING_UPDATE_STATUS__VMID31_REMAPPING_FINISHED_MASK                                  0x80000000L
-//ATC_VMID0_PASID_MAPPING
-#define ATC_VMID0_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID0_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID0_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID0_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID0_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID1_PASID_MAPPING
-#define ATC_VMID1_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID1_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID1_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID1_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID1_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID2_PASID_MAPPING
-#define ATC_VMID2_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID2_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID2_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID2_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID2_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID3_PASID_MAPPING
-#define ATC_VMID3_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID3_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID3_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID3_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID3_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID4_PASID_MAPPING
-#define ATC_VMID4_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID4_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID4_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID4_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID4_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID5_PASID_MAPPING
-#define ATC_VMID5_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID5_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID5_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID5_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID5_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID6_PASID_MAPPING
-#define ATC_VMID6_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID6_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID6_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID6_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID6_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID7_PASID_MAPPING
-#define ATC_VMID7_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID7_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID7_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID7_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID7_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID8_PASID_MAPPING
-#define ATC_VMID8_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID8_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID8_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID8_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID8_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID9_PASID_MAPPING
-#define ATC_VMID9_PASID_MAPPING__PASID__SHIFT                                                                 0x0
-#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                       0x1e
-#define ATC_VMID9_PASID_MAPPING__VALID__SHIFT                                                                 0x1f
-#define ATC_VMID9_PASID_MAPPING__PASID_MASK                                                                   0x0000FFFFL
-#define ATC_VMID9_PASID_MAPPING__NO_INVALIDATION_MASK                                                         0x40000000L
-#define ATC_VMID9_PASID_MAPPING__VALID_MASK                                                                   0x80000000L
-//ATC_VMID10_PASID_MAPPING
-#define ATC_VMID10_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID10_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID10_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID10_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID10_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID11_PASID_MAPPING
-#define ATC_VMID11_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID11_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID11_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID11_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID11_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID12_PASID_MAPPING
-#define ATC_VMID12_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID12_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID12_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID12_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID12_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID13_PASID_MAPPING
-#define ATC_VMID13_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID13_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID13_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID13_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID13_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID14_PASID_MAPPING
-#define ATC_VMID14_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID14_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID14_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID14_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID14_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID15_PASID_MAPPING
-#define ATC_VMID15_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID15_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID15_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID15_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID15_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_ATS_VMID_STATUS
-#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING__SHIFT                                                         0x0
-#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING__SHIFT                                                         0x1
-#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING__SHIFT                                                         0x2
-#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING__SHIFT                                                         0x3
-#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING__SHIFT                                                         0x4
-#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING__SHIFT                                                         0x5
-#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING__SHIFT                                                         0x6
-#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING__SHIFT                                                         0x7
-#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING__SHIFT                                                         0x8
-#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING__SHIFT                                                         0x9
-#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING__SHIFT                                                        0xa
-#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING__SHIFT                                                        0xb
-#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING__SHIFT                                                        0xc
-#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING__SHIFT                                                        0xd
-#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING__SHIFT                                                        0xe
-#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING__SHIFT                                                        0xf
-#define ATC_ATS_VMID_STATUS__VMID16_OUTSTANDING__SHIFT                                                        0x10
-#define ATC_ATS_VMID_STATUS__VMID17_OUTSTANDING__SHIFT                                                        0x11
-#define ATC_ATS_VMID_STATUS__VMID18_OUTSTANDING__SHIFT                                                        0x12
-#define ATC_ATS_VMID_STATUS__VMID19_OUTSTANDING__SHIFT                                                        0x13
-#define ATC_ATS_VMID_STATUS__VMID20_OUTSTANDING__SHIFT                                                        0x14
-#define ATC_ATS_VMID_STATUS__VMID21_OUTSTANDING__SHIFT                                                        0x15
-#define ATC_ATS_VMID_STATUS__VMID22_OUTSTANDING__SHIFT                                                        0x16
-#define ATC_ATS_VMID_STATUS__VMID23_OUTSTANDING__SHIFT                                                        0x17
-#define ATC_ATS_VMID_STATUS__VMID24_OUTSTANDING__SHIFT                                                        0x18
-#define ATC_ATS_VMID_STATUS__VMID25_OUTSTANDING__SHIFT                                                        0x19
-#define ATC_ATS_VMID_STATUS__VMID26_OUTSTANDING__SHIFT                                                        0x1a
-#define ATC_ATS_VMID_STATUS__VMID27_OUTSTANDING__SHIFT                                                        0x1b
-#define ATC_ATS_VMID_STATUS__VMID28_OUTSTANDING__SHIFT                                                        0x1c
-#define ATC_ATS_VMID_STATUS__VMID29_OUTSTANDING__SHIFT                                                        0x1d
-#define ATC_ATS_VMID_STATUS__VMID30_OUTSTANDING__SHIFT                                                        0x1e
-#define ATC_ATS_VMID_STATUS__VMID31_OUTSTANDING__SHIFT                                                        0x1f
-#define ATC_ATS_VMID_STATUS__VMID0_OUTSTANDING_MASK                                                           0x00000001L
-#define ATC_ATS_VMID_STATUS__VMID1_OUTSTANDING_MASK                                                           0x00000002L
-#define ATC_ATS_VMID_STATUS__VMID2_OUTSTANDING_MASK                                                           0x00000004L
-#define ATC_ATS_VMID_STATUS__VMID3_OUTSTANDING_MASK                                                           0x00000008L
-#define ATC_ATS_VMID_STATUS__VMID4_OUTSTANDING_MASK                                                           0x00000010L
-#define ATC_ATS_VMID_STATUS__VMID5_OUTSTANDING_MASK                                                           0x00000020L
-#define ATC_ATS_VMID_STATUS__VMID6_OUTSTANDING_MASK                                                           0x00000040L
-#define ATC_ATS_VMID_STATUS__VMID7_OUTSTANDING_MASK                                                           0x00000080L
-#define ATC_ATS_VMID_STATUS__VMID8_OUTSTANDING_MASK                                                           0x00000100L
-#define ATC_ATS_VMID_STATUS__VMID9_OUTSTANDING_MASK                                                           0x00000200L
-#define ATC_ATS_VMID_STATUS__VMID10_OUTSTANDING_MASK                                                          0x00000400L
-#define ATC_ATS_VMID_STATUS__VMID11_OUTSTANDING_MASK                                                          0x00000800L
-#define ATC_ATS_VMID_STATUS__VMID12_OUTSTANDING_MASK                                                          0x00001000L
-#define ATC_ATS_VMID_STATUS__VMID13_OUTSTANDING_MASK                                                          0x00002000L
-#define ATC_ATS_VMID_STATUS__VMID14_OUTSTANDING_MASK                                                          0x00004000L
-#define ATC_ATS_VMID_STATUS__VMID15_OUTSTANDING_MASK                                                          0x00008000L
-#define ATC_ATS_VMID_STATUS__VMID16_OUTSTANDING_MASK                                                          0x00010000L
-#define ATC_ATS_VMID_STATUS__VMID17_OUTSTANDING_MASK                                                          0x00020000L
-#define ATC_ATS_VMID_STATUS__VMID18_OUTSTANDING_MASK                                                          0x00040000L
-#define ATC_ATS_VMID_STATUS__VMID19_OUTSTANDING_MASK                                                          0x00080000L
-#define ATC_ATS_VMID_STATUS__VMID20_OUTSTANDING_MASK                                                          0x00100000L
-#define ATC_ATS_VMID_STATUS__VMID21_OUTSTANDING_MASK                                                          0x00200000L
-#define ATC_ATS_VMID_STATUS__VMID22_OUTSTANDING_MASK                                                          0x00400000L
-#define ATC_ATS_VMID_STATUS__VMID23_OUTSTANDING_MASK                                                          0x00800000L
-#define ATC_ATS_VMID_STATUS__VMID24_OUTSTANDING_MASK                                                          0x01000000L
-#define ATC_ATS_VMID_STATUS__VMID25_OUTSTANDING_MASK                                                          0x02000000L
-#define ATC_ATS_VMID_STATUS__VMID26_OUTSTANDING_MASK                                                          0x04000000L
-#define ATC_ATS_VMID_STATUS__VMID27_OUTSTANDING_MASK                                                          0x08000000L
-#define ATC_ATS_VMID_STATUS__VMID28_OUTSTANDING_MASK                                                          0x10000000L
-#define ATC_ATS_VMID_STATUS__VMID29_OUTSTANDING_MASK                                                          0x20000000L
-#define ATC_ATS_VMID_STATUS__VMID30_OUTSTANDING_MASK                                                          0x40000000L
-#define ATC_ATS_VMID_STATUS__VMID31_OUTSTANDING_MASK                                                          0x80000000L
-//ATC_ATS_GFX_ATCL2_STATUS
-#define ATC_ATS_GFX_ATCL2_STATUS__POWERED_DOWN__SHIFT                                                         0x0
-#define ATC_ATS_GFX_ATCL2_STATUS__POWERED_DOWN_MASK                                                           0x00000001L
-//ATC_PERFCOUNTER0_CFG
-#define ATC_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define ATC_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                                0x18
-#define ATC_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                   0x1c
-#define ATC_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                    0x1d
-#define ATC_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define ATC_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define ATC_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define ATC_PERFCOUNTER0_CFG__ENABLE_MASK                                                                     0x10000000L
-#define ATC_PERFCOUNTER0_CFG__CLEAR_MASK                                                                      0x20000000L
-//ATC_PERFCOUNTER1_CFG
-#define ATC_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define ATC_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                                0x18
-#define ATC_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                   0x1c
-#define ATC_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                    0x1d
-#define ATC_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define ATC_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define ATC_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define ATC_PERFCOUNTER1_CFG__ENABLE_MASK                                                                     0x10000000L
-#define ATC_PERFCOUNTER1_CFG__CLEAR_MASK                                                                      0x20000000L
-//ATC_PERFCOUNTER2_CFG
-#define ATC_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define ATC_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                                0x18
-#define ATC_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                   0x1c
-#define ATC_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                    0x1d
-#define ATC_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define ATC_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define ATC_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define ATC_PERFCOUNTER2_CFG__ENABLE_MASK                                                                     0x10000000L
-#define ATC_PERFCOUNTER2_CFG__CLEAR_MASK                                                                      0x20000000L
-//ATC_PERFCOUNTER3_CFG
-#define ATC_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define ATC_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                                0x18
-#define ATC_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                                   0x1c
-#define ATC_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                    0x1d
-#define ATC_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define ATC_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define ATC_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define ATC_PERFCOUNTER3_CFG__ENABLE_MASK                                                                     0x10000000L
-#define ATC_PERFCOUNTER3_CFG__CLEAR_MASK                                                                      0x20000000L
-//ATC_PERFCOUNTER_RSLT_CNTL
-#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                 0x0
-#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                       0x8
-#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                        0x10
-#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                          0x18
-#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                           0x19
-#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                                0x1a
-#define ATC_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                   0x0000000FL
-#define ATC_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                         0x0000FF00L
-#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                          0x00FF0000L
-#define ATC_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                            0x01000000L
-#define ATC_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                             0x02000000L
-#define ATC_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                  0x04000000L
-//ATC_PERFCOUNTER_LO
-#define ATC_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                 0x0
-#define ATC_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                   0xFFFFFFFFL
-//ATC_PERFCOUNTER_HI
-#define ATC_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                 0x0
-#define ATC_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                              0x10
-#define ATC_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                   0x0000FFFFL
-#define ATC_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                                0xFFFF0000L
-//ATHUB_PCIE_ATS_CNTL
-#define ATHUB_PCIE_ATS_CNTL__STU__SHIFT                                                                       0x10
-#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                                0x1f
-#define ATHUB_PCIE_ATS_CNTL__STU_MASK                                                                         0x001F0000L
-#define ATHUB_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                                  0x80000000L
-//ATHUB_PCIE_PASID_CNTL
-#define ATHUB_PCIE_PASID_CNTL__PASID_EN__SHIFT                                                                0x10
-#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                             0x11
-#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                                        0x12
-#define ATHUB_PCIE_PASID_CNTL__PASID_EN_MASK                                                                  0x00010000L
-#define ATHUB_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                               0x00020000L
-#define ATHUB_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                                          0x00040000L
-//ATHUB_PCIE_PAGE_REQ_CNTL
-#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                                           0x0
-#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                                            0x1
-#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                                             0x00000001L
-#define ATHUB_PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                              0x00000002L
-//ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC
-#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                                    0x0
-#define ATHUB_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                                      0xFFFFFFFFL
-//ATHUB_COMMAND
-#define ATHUB_COMMAND__BUS_MASTER_EN__SHIFT                                                                   0x2
-#define ATHUB_COMMAND__BUS_MASTER_EN_MASK                                                                     0x00000004L
-//ATHUB_PCIE_ATS_CNTL_VF_0
-#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_0__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_1
-#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_1__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_2
-#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_2__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_3
-#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_3__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_4
-#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_4__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_5
-#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_5__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_6
-#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_6__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_7
-#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_7__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_8
-#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_8__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_9
-#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE__SHIFT                                                           0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_9__ATC_ENABLE_MASK                                                             0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_10
-#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE__SHIFT                                                          0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_10__ATC_ENABLE_MASK                                                            0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_11
-#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE__SHIFT                                                          0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_11__ATC_ENABLE_MASK                                                            0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_12
-#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE__SHIFT                                                          0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_12__ATC_ENABLE_MASK                                                            0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_13
-#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE__SHIFT                                                          0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_13__ATC_ENABLE_MASK                                                            0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_14
-#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE__SHIFT                                                          0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_14__ATC_ENABLE_MASK                                                            0x80000000L
-//ATHUB_PCIE_ATS_CNTL_VF_15
-#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE__SHIFT                                                          0x1f
-#define ATHUB_PCIE_ATS_CNTL_VF_15__ATC_ENABLE_MASK                                                            0x80000000L
-//ATHUB_MEM_POWER_LS
-#define ATHUB_MEM_POWER_LS__LS_SETUP__SHIFT                                                                   0x0
-#define ATHUB_MEM_POWER_LS__LS_HOLD__SHIFT                                                                    0x6
-#define ATHUB_MEM_POWER_LS__LS_SETUP_MASK                                                                     0x0000003FL
-#define ATHUB_MEM_POWER_LS__LS_HOLD_MASK                                                                      0x00000FC0L
-//ATS_IH_CREDIT
-#define ATS_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                    0x0
-#define ATS_IH_CREDIT__IH_CLIENT_ID__SHIFT                                                                    0x10
-#define ATS_IH_CREDIT__CREDIT_VALUE_MASK                                                                      0x00000003L
-#define ATS_IH_CREDIT__IH_CLIENT_ID_MASK                                                                      0x00FF0000L
-//ATHUB_IH_CREDIT
-#define ATHUB_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                  0x0
-#define ATHUB_IH_CREDIT__IH_CLIENT_ID__SHIFT                                                                  0x10
-#define ATHUB_IH_CREDIT__CREDIT_VALUE_MASK                                                                    0x00000003L
-#define ATHUB_IH_CREDIT__IH_CLIENT_ID_MASK                                                                    0x00FF0000L
-//ATC_VMID16_PASID_MAPPING
-#define ATC_VMID16_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID16_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID16_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID16_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID16_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID16_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID17_PASID_MAPPING
-#define ATC_VMID17_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID17_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID17_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID17_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID17_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID17_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID18_PASID_MAPPING
-#define ATC_VMID18_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID18_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID18_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID18_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID18_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID18_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID19_PASID_MAPPING
-#define ATC_VMID19_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID19_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID19_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID19_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID19_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID19_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID20_PASID_MAPPING
-#define ATC_VMID20_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID20_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID20_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID20_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID20_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID20_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID21_PASID_MAPPING
-#define ATC_VMID21_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID21_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID21_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID21_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID21_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID21_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID22_PASID_MAPPING
-#define ATC_VMID22_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID22_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID22_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID22_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID22_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID22_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID23_PASID_MAPPING
-#define ATC_VMID23_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID23_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID23_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID23_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID23_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID23_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID24_PASID_MAPPING
-#define ATC_VMID24_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID24_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID24_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID24_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID24_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID24_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID25_PASID_MAPPING
-#define ATC_VMID25_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID25_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID25_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID25_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID25_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID25_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID26_PASID_MAPPING
-#define ATC_VMID26_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID26_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID26_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID26_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID26_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID26_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID27_PASID_MAPPING
-#define ATC_VMID27_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID27_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID27_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID27_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID27_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID27_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID28_PASID_MAPPING
-#define ATC_VMID28_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID28_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID28_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID28_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID28_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID28_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID29_PASID_MAPPING
-#define ATC_VMID29_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID29_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID29_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID29_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID29_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID29_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID30_PASID_MAPPING
-#define ATC_VMID30_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID30_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID30_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID30_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID30_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID30_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_VMID31_PASID_MAPPING
-#define ATC_VMID31_PASID_MAPPING__PASID__SHIFT                                                                0x0
-#define ATC_VMID31_PASID_MAPPING__NO_INVALIDATION__SHIFT                                                      0x1e
-#define ATC_VMID31_PASID_MAPPING__VALID__SHIFT                                                                0x1f
-#define ATC_VMID31_PASID_MAPPING__PASID_MASK                                                                  0x0000FFFFL
-#define ATC_VMID31_PASID_MAPPING__NO_INVALIDATION_MASK                                                        0x40000000L
-#define ATC_VMID31_PASID_MAPPING__VALID_MASK                                                                  0x80000000L
-//ATC_ATS_MMHUB_ATCL2_STATUS
-#define ATC_ATS_MMHUB_ATCL2_STATUS__POWERED_DOWN__SHIFT                                                       0x0
-#define ATC_ATS_MMHUB_ATCL2_STATUS__POWERED_DOWN_MASK                                                         0x00000001L
-//ATHUB_SHARED_VIRT_RESET_REQ
-#define ATHUB_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                0x0
-#define ATHUB_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                0x1f
-#define ATHUB_SHARED_VIRT_RESET_REQ__VF_MASK                                                                  0x0000FFFFL
-#define ATHUB_SHARED_VIRT_RESET_REQ__PF_MASK                                                                  0x80000000L
-//ATHUB_SHARED_ACTIVE_FCN_ID
-#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID__SHIFT                                                               0x0
-#define ATHUB_SHARED_ACTIVE_FCN_ID__VF__SHIFT                                                                 0x1f
-#define ATHUB_SHARED_ACTIVE_FCN_ID__VFID_MASK                                                                 0x0000000FL
-#define ATHUB_SHARED_ACTIVE_FCN_ID__VF_MASK                                                                   0x80000000L
-//ATC_ATS_SDPPORT_CNTL
-#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE__SHIFT                                                    0x0
-#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE__SHIFT                                                         0x1
-#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD__SHIFT                                                   0x3
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE__SHIFT                                                0x7
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK__SHIFT                                                 0x8
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD__SHIFT                                               0x9
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE__SHIFT                                                 0xd
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_RDY_MODE__SHIFT                                                       0xe
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_MMHUB_RDY_MODE__SHIFT                                                     0xf
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN__SHIFT                                              0x10
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV__SHIFT                                           0x11
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN__SHIFT                                          0x12
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV__SHIFT                                       0x13
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN__SHIFT                                              0x14
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV__SHIFT                                           0x15
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN__SHIFT                                                0x16
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV__SHIFT                                             0x17
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN__SHIFT                                           0x18
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV__SHIFT                                        0x19
-#define ATC_ATS_SDPPORT_CNTL__ATS_INV_SELF_ACTIVATE_MASK                                                      0x00000001L
-#define ATC_ATS_SDPPORT_CNTL__ATS_INV_CFG_MODE_MASK                                                           0x00000006L
-#define ATC_ATS_SDPPORT_CNTL__ATS_INV_HALT_THRESHOLD_MASK                                                     0x00000078L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_SELF_ACTIVATE_MASK                                                  0x00000080L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_QUICK_COMACK_MASK                                                   0x00000100L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_HALT_THRESHOLD_MASK                                                 0x00001E00L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_TRANS_PASSIVE_MODE_MASK                                                   0x00002000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_RDY_MODE_MASK                                                         0x00004000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_MMHUB_RDY_MODE_MASK                                                       0x00008000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKEN_MASK                                                0x00010000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPCKENRCV_MASK                                             0x00020000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKEN_MASK                                            0x00040000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_RDRSPDATACKENRCV_MASK                                         0x00080000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKEN_MASK                                                0x00100000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_WRRSPCKENRCV_MASK                                             0x00200000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKEN_MASK                                                  0x00400000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_REQCKENRCV_MASK                                               0x00800000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKEN_MASK                                             0x01000000L
-#define ATC_ATS_SDPPORT_CNTL__UTCL2_GFX_SDPVDCI_ORIGDATACKENRCV_MASK                                          0x02000000L
-//ATC_ATS_VMID_SNAPSHOT_GFX_STAT
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID0__SHIFT                                                          0x0
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID1__SHIFT                                                          0x1
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID2__SHIFT                                                          0x2
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID3__SHIFT                                                          0x3
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID4__SHIFT                                                          0x4
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID5__SHIFT                                                          0x5
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID6__SHIFT                                                          0x6
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID7__SHIFT                                                          0x7
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID8__SHIFT                                                          0x8
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID9__SHIFT                                                          0x9
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID10__SHIFT                                                         0xa
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID11__SHIFT                                                         0xb
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID12__SHIFT                                                         0xc
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID13__SHIFT                                                         0xd
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID14__SHIFT                                                         0xe
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID15__SHIFT                                                         0xf
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID0_MASK                                                            0x00000001L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID1_MASK                                                            0x00000002L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID2_MASK                                                            0x00000004L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID3_MASK                                                            0x00000008L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID4_MASK                                                            0x00000010L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID5_MASK                                                            0x00000020L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID6_MASK                                                            0x00000040L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID7_MASK                                                            0x00000080L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID8_MASK                                                            0x00000100L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID9_MASK                                                            0x00000200L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID10_MASK                                                           0x00000400L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID11_MASK                                                           0x00000800L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID12_MASK                                                           0x00001000L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID13_MASK                                                           0x00002000L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID14_MASK                                                           0x00004000L
-#define ATC_ATS_VMID_SNAPSHOT_GFX_STAT__VMID15_MASK                                                           0x00008000L
-//ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID0__SHIFT                                                        0x0
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID1__SHIFT                                                        0x1
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID2__SHIFT                                                        0x2
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID3__SHIFT                                                        0x3
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID4__SHIFT                                                        0x4
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID5__SHIFT                                                        0x5
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID6__SHIFT                                                        0x6
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID7__SHIFT                                                        0x7
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID8__SHIFT                                                        0x8
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID9__SHIFT                                                        0x9
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID10__SHIFT                                                       0xa
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID11__SHIFT                                                       0xb
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID12__SHIFT                                                       0xc
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID13__SHIFT                                                       0xd
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID14__SHIFT                                                       0xe
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID15__SHIFT                                                       0xf
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID0_MASK                                                          0x00000001L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID1_MASK                                                          0x00000002L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID2_MASK                                                          0x00000004L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID3_MASK                                                          0x00000008L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID4_MASK                                                          0x00000010L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID5_MASK                                                          0x00000020L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID6_MASK                                                          0x00000040L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID7_MASK                                                          0x00000080L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID8_MASK                                                          0x00000100L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID9_MASK                                                          0x00000200L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID10_MASK                                                         0x00000400L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID11_MASK                                                         0x00000800L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID12_MASK                                                         0x00001000L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID13_MASK                                                         0x00002000L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID14_MASK                                                         0x00004000L
-#define ATC_ATS_VMID_SNAPSHOT_MMHUB_STAT__VMID15_MASK                                                         0x00008000L
-
-
-// addressBlock: athub_xpbdec
-//XPB_RTR_SRC_APRTR0
-#define XPB_RTR_SRC_APRTR0__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR0__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR1
-#define XPB_RTR_SRC_APRTR1__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR1__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR2
-#define XPB_RTR_SRC_APRTR2__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR2__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR3
-#define XPB_RTR_SRC_APRTR3__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR3__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR4
-#define XPB_RTR_SRC_APRTR4__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR4__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR5
-#define XPB_RTR_SRC_APRTR5__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR5__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR6
-#define XPB_RTR_SRC_APRTR6__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR6__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR7
-#define XPB_RTR_SRC_APRTR7__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR7__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR8
-#define XPB_RTR_SRC_APRTR8__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR8__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_RTR_SRC_APRTR9
-#define XPB_RTR_SRC_APRTR9__BASE_ADDR__SHIFT                                                                  0x0
-#define XPB_RTR_SRC_APRTR9__BASE_ADDR_MASK                                                                    0x7FFFFFFFL
-//XPB_XDMA_RTR_SRC_APRTR0
-#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR__SHIFT                                                             0x0
-#define XPB_XDMA_RTR_SRC_APRTR0__BASE_ADDR_MASK                                                               0x7FFFFFFFL
-//XPB_XDMA_RTR_SRC_APRTR1
-#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR__SHIFT                                                             0x0
-#define XPB_XDMA_RTR_SRC_APRTR1__BASE_ADDR_MASK                                                               0x7FFFFFFFL
-//XPB_XDMA_RTR_SRC_APRTR2
-#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR__SHIFT                                                             0x0
-#define XPB_XDMA_RTR_SRC_APRTR2__BASE_ADDR_MASK                                                               0x7FFFFFFFL
-//XPB_XDMA_RTR_SRC_APRTR3
-#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR__SHIFT                                                             0x0
-#define XPB_XDMA_RTR_SRC_APRTR3__BASE_ADDR_MASK                                                               0x7FFFFFFFL
-//XPB_RTR_DEST_MAP0
-#define XPB_RTR_DEST_MAP0__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP0__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP0__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP0__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP0__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP0__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP0__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP0__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP0__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP1
-#define XPB_RTR_DEST_MAP1__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP1__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP1__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP1__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP1__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP1__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP1__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP1__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP1__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP2
-#define XPB_RTR_DEST_MAP2__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP2__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP2__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP2__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP2__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP2__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP2__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP2__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP2__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP3
-#define XPB_RTR_DEST_MAP3__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP3__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP3__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP3__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP3__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP3__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP3__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP3__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP3__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP4
-#define XPB_RTR_DEST_MAP4__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP4__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP4__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP4__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP4__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP4__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP4__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP4__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP4__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP5
-#define XPB_RTR_DEST_MAP5__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP5__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP5__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP5__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP5__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP5__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP5__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP5__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP5__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP6
-#define XPB_RTR_DEST_MAP6__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP6__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP6__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP6__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP6__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP6__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP6__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP6__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP6__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP7
-#define XPB_RTR_DEST_MAP7__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP7__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP7__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP7__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP7__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP7__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP7__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP7__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP7__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP8
-#define XPB_RTR_DEST_MAP8__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP8__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP8__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP8__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP8__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP8__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP8__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP8__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP8__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_RTR_DEST_MAP9
-#define XPB_RTR_DEST_MAP9__NMR__SHIFT                                                                         0x0
-#define XPB_RTR_DEST_MAP9__DEST_OFFSET__SHIFT                                                                 0x1
-#define XPB_RTR_DEST_MAP9__DEST_SEL__SHIFT                                                                    0x14
-#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB__SHIFT                                                                0x18
-#define XPB_RTR_DEST_MAP9__APRTR_SIZE__SHIFT                                                                  0x1a
-#define XPB_RTR_DEST_MAP9__NMR_MASK                                                                           0x00000001L
-#define XPB_RTR_DEST_MAP9__DEST_OFFSET_MASK                                                                   0x000FFFFEL
-#define XPB_RTR_DEST_MAP9__DEST_SEL_MASK                                                                      0x00F00000L
-#define XPB_RTR_DEST_MAP9__DEST_SEL_RPB_MASK                                                                  0x01000000L
-#define XPB_RTR_DEST_MAP9__APRTR_SIZE_MASK                                                                    0x7C000000L
-//XPB_XDMA_RTR_DEST_MAP0
-#define XPB_XDMA_RTR_DEST_MAP0__NMR__SHIFT                                                                    0x0
-#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET__SHIFT                                                            0x1
-#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL__SHIFT                                                               0x14
-#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB__SHIFT                                                           0x18
-#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE__SHIFT                                                             0x1a
-#define XPB_XDMA_RTR_DEST_MAP0__NMR_MASK                                                                      0x00000001L
-#define XPB_XDMA_RTR_DEST_MAP0__DEST_OFFSET_MASK                                                              0x000FFFFEL
-#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_MASK                                                                 0x00F00000L
-#define XPB_XDMA_RTR_DEST_MAP0__DEST_SEL_RPB_MASK                                                             0x01000000L
-#define XPB_XDMA_RTR_DEST_MAP0__APRTR_SIZE_MASK                                                               0x7C000000L
-//XPB_XDMA_RTR_DEST_MAP1
-#define XPB_XDMA_RTR_DEST_MAP1__NMR__SHIFT                                                                    0x0
-#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET__SHIFT                                                            0x1
-#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL__SHIFT                                                               0x14
-#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB__SHIFT                                                           0x18
-#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE__SHIFT                                                             0x1a
-#define XPB_XDMA_RTR_DEST_MAP1__NMR_MASK                                                                      0x00000001L
-#define XPB_XDMA_RTR_DEST_MAP1__DEST_OFFSET_MASK                                                              0x000FFFFEL
-#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_MASK                                                                 0x00F00000L
-#define XPB_XDMA_RTR_DEST_MAP1__DEST_SEL_RPB_MASK                                                             0x01000000L
-#define XPB_XDMA_RTR_DEST_MAP1__APRTR_SIZE_MASK                                                               0x7C000000L
-//XPB_XDMA_RTR_DEST_MAP2
-#define XPB_XDMA_RTR_DEST_MAP2__NMR__SHIFT                                                                    0x0
-#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET__SHIFT                                                            0x1
-#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL__SHIFT                                                               0x14
-#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB__SHIFT                                                           0x18
-#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE__SHIFT                                                             0x1a
-#define XPB_XDMA_RTR_DEST_MAP2__NMR_MASK                                                                      0x00000001L
-#define XPB_XDMA_RTR_DEST_MAP2__DEST_OFFSET_MASK                                                              0x000FFFFEL
-#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_MASK                                                                 0x00F00000L
-#define XPB_XDMA_RTR_DEST_MAP2__DEST_SEL_RPB_MASK                                                             0x01000000L
-#define XPB_XDMA_RTR_DEST_MAP2__APRTR_SIZE_MASK                                                               0x7C000000L
-//XPB_XDMA_RTR_DEST_MAP3
-#define XPB_XDMA_RTR_DEST_MAP3__NMR__SHIFT                                                                    0x0
-#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET__SHIFT                                                            0x1
-#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL__SHIFT                                                               0x14
-#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB__SHIFT                                                           0x18
-#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE__SHIFT                                                             0x1a
-#define XPB_XDMA_RTR_DEST_MAP3__NMR_MASK                                                                      0x00000001L
-#define XPB_XDMA_RTR_DEST_MAP3__DEST_OFFSET_MASK                                                              0x000FFFFEL
-#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_MASK                                                                 0x00F00000L
-#define XPB_XDMA_RTR_DEST_MAP3__DEST_SEL_RPB_MASK                                                             0x01000000L
-#define XPB_XDMA_RTR_DEST_MAP3__APRTR_SIZE_MASK                                                               0x7C000000L
-//XPB_CLG_CFG0
-#define XPB_CLG_CFG0__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG0__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG0__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG0__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG0__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG0__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_CFG1
-#define XPB_CLG_CFG1__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG1__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG1__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG1__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG1__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG1__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_CFG2
-#define XPB_CLG_CFG2__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG2__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG2__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG2__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG2__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG2__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_CFG3
-#define XPB_CLG_CFG3__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG3__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG3__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG3__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG3__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG3__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_CFG4
-#define XPB_CLG_CFG4__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG4__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG4__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG4__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG4__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG4__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_CFG5
-#define XPB_CLG_CFG5__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG5__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG5__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG5__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG5__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG5__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_CFG6
-#define XPB_CLG_CFG6__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG6__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG6__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG6__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG6__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG6__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_CFG7
-#define XPB_CLG_CFG7__WCB_NUM__SHIFT                                                                          0x0
-#define XPB_CLG_CFG7__P2P_BAR__SHIFT                                                                          0x7
-#define XPB_CLG_CFG7__HOST_FLUSH__SHIFT                                                                       0xa
-#define XPB_CLG_CFG7__WCB_NUM_MASK                                                                            0x0000000FL
-#define XPB_CLG_CFG7__P2P_BAR_MASK                                                                            0x00000380L
-#define XPB_CLG_CFG7__HOST_FLUSH_MASK                                                                         0x00003C00L
-//XPB_CLG_EXTRA
-#define XPB_CLG_EXTRA__CMP0_HIGH__SHIFT                                                                       0x0
-#define XPB_CLG_EXTRA__CMP0_LOW__SHIFT                                                                        0x6
-#define XPB_CLG_EXTRA__VLD0__SHIFT                                                                            0xb
-#define XPB_CLG_EXTRA__CLG0_NUM__SHIFT                                                                        0xc
-#define XPB_CLG_EXTRA__CMP1_HIGH__SHIFT                                                                       0xf
-#define XPB_CLG_EXTRA__CMP1_LOW__SHIFT                                                                        0x15
-#define XPB_CLG_EXTRA__VLD1__SHIFT                                                                            0x1a
-#define XPB_CLG_EXTRA__CLG1_NUM__SHIFT                                                                        0x1b
-#define XPB_CLG_EXTRA__CMP0_HIGH_MASK                                                                         0x0000003FL
-#define XPB_CLG_EXTRA__CMP0_LOW_MASK                                                                          0x000007C0L
-#define XPB_CLG_EXTRA__VLD0_MASK                                                                              0x00000800L
-#define XPB_CLG_EXTRA__CLG0_NUM_MASK                                                                          0x00007000L
-#define XPB_CLG_EXTRA__CMP1_HIGH_MASK                                                                         0x001F8000L
-#define XPB_CLG_EXTRA__CMP1_LOW_MASK                                                                          0x03E00000L
-#define XPB_CLG_EXTRA__VLD1_MASK                                                                              0x04000000L
-#define XPB_CLG_EXTRA__CLG1_NUM_MASK                                                                          0x38000000L
-//XPB_CLG_EXTRA_MSK
-#define XPB_CLG_EXTRA_MSK__MSK0_HIGH__SHIFT                                                                   0x0
-#define XPB_CLG_EXTRA_MSK__MSK0_LOW__SHIFT                                                                    0x6
-#define XPB_CLG_EXTRA_MSK__MSK1_HIGH__SHIFT                                                                   0xb
-#define XPB_CLG_EXTRA_MSK__MSK1_LOW__SHIFT                                                                    0x11
-#define XPB_CLG_EXTRA_MSK__MSK0_HIGH_MASK                                                                     0x0000003FL
-#define XPB_CLG_EXTRA_MSK__MSK0_LOW_MASK                                                                      0x000007C0L
-#define XPB_CLG_EXTRA_MSK__MSK1_HIGH_MASK                                                                     0x0001F800L
-#define XPB_CLG_EXTRA_MSK__MSK1_LOW_MASK                                                                      0x003E0000L
-//XPB_LB_ADDR
-#define XPB_LB_ADDR__CMP0__SHIFT                                                                              0x0
-#define XPB_LB_ADDR__MASK0__SHIFT                                                                             0xa
-#define XPB_LB_ADDR__CMP1__SHIFT                                                                              0x14
-#define XPB_LB_ADDR__MASK1__SHIFT                                                                             0x1a
-#define XPB_LB_ADDR__CMP0_MASK                                                                                0x000003FFL
-#define XPB_LB_ADDR__MASK0_MASK                                                                               0x000FFC00L
-#define XPB_LB_ADDR__CMP1_MASK                                                                                0x03F00000L
-#define XPB_LB_ADDR__MASK1_MASK                                                                               0xFC000000L
-//XPB_WCB_STS
-#define XPB_WCB_STS__PBUF_VLD__SHIFT                                                                          0x0
-#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT__SHIFT                                                              0x10
-#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT__SHIFT                                                              0x17
-#define XPB_WCB_STS__PBUF_VLD_MASK                                                                            0x0000FFFFL
-#define XPB_WCB_STS__WCB_HST_DATA_BUF_CNT_MASK                                                                0x007F0000L
-#define XPB_WCB_STS__WCB_SID_DATA_BUF_CNT_MASK                                                                0x3F800000L
-//XPB_HST_CFG
-#define XPB_HST_CFG__BAR_UP_WR_CMD__SHIFT                                                                     0x0
-#define XPB_HST_CFG__BAR_UP_WR_CMD_MASK                                                                       0x00000001L
-//XPB_P2P_BAR_CFG
-#define XPB_P2P_BAR_CFG__ADDR_SIZE__SHIFT                                                                     0x0
-#define XPB_P2P_BAR_CFG__SEND_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR_CFG__SNOOP__SHIFT                                                                         0x6
-#define XPB_P2P_BAR_CFG__SEND_DIS__SHIFT                                                                      0x7
-#define XPB_P2P_BAR_CFG__COMPRESS_DIS__SHIFT                                                                  0x8
-#define XPB_P2P_BAR_CFG__UPDATE_DIS__SHIFT                                                                    0x9
-#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR__SHIFT                                                            0xa
-#define XPB_P2P_BAR_CFG__RD_EN__SHIFT                                                                         0xb
-#define XPB_P2P_BAR_CFG__ATC_TRANSLATED__SHIFT                                                                0xc
-#define XPB_P2P_BAR_CFG__ADDR_SIZE_MASK                                                                       0x0000000FL
-#define XPB_P2P_BAR_CFG__SEND_BAR_MASK                                                                        0x00000030L
-#define XPB_P2P_BAR_CFG__SNOOP_MASK                                                                           0x00000040L
-#define XPB_P2P_BAR_CFG__SEND_DIS_MASK                                                                        0x00000080L
-#define XPB_P2P_BAR_CFG__COMPRESS_DIS_MASK                                                                    0x00000100L
-#define XPB_P2P_BAR_CFG__UPDATE_DIS_MASK                                                                      0x00000200L
-#define XPB_P2P_BAR_CFG__REGBAR_FROM_SYSBAR_MASK                                                              0x00000400L
-#define XPB_P2P_BAR_CFG__RD_EN_MASK                                                                           0x00000800L
-#define XPB_P2P_BAR_CFG__ATC_TRANSLATED_MASK                                                                  0x00001000L
-//XPB_P2P_BAR0
-#define XPB_P2P_BAR0__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR0__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR0__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR0__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR0__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR0__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR0__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR0__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR0__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR0__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR0__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR0__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR0__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR0__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR0__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR0__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR1
-#define XPB_P2P_BAR1__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR1__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR1__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR1__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR1__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR1__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR1__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR1__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR1__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR1__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR1__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR1__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR1__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR1__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR1__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR1__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR2
-#define XPB_P2P_BAR2__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR2__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR2__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR2__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR2__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR2__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR2__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR2__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR2__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR2__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR2__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR2__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR2__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR2__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR2__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR2__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR3
-#define XPB_P2P_BAR3__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR3__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR3__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR3__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR3__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR3__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR3__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR3__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR3__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR3__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR3__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR3__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR3__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR3__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR3__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR3__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR4
-#define XPB_P2P_BAR4__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR4__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR4__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR4__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR4__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR4__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR4__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR4__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR4__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR4__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR4__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR4__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR4__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR4__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR4__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR4__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR5
-#define XPB_P2P_BAR5__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR5__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR5__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR5__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR5__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR5__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR5__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR5__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR5__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR5__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR5__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR5__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR5__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR5__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR5__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR5__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR6
-#define XPB_P2P_BAR6__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR6__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR6__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR6__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR6__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR6__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR6__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR6__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR6__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR6__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR6__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR6__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR6__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR6__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR6__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR6__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR7
-#define XPB_P2P_BAR7__HOST_FLUSH__SHIFT                                                                       0x0
-#define XPB_P2P_BAR7__REG_SYS_BAR__SHIFT                                                                      0x4
-#define XPB_P2P_BAR7__MEM_SYS_BAR__SHIFT                                                                      0x8
-#define XPB_P2P_BAR7__VALID__SHIFT                                                                            0xc
-#define XPB_P2P_BAR7__SEND_DIS__SHIFT                                                                         0xd
-#define XPB_P2P_BAR7__COMPRESS_DIS__SHIFT                                                                     0xe
-#define XPB_P2P_BAR7__RESERVED__SHIFT                                                                         0xf
-#define XPB_P2P_BAR7__ADDRESS__SHIFT                                                                          0x10
-#define XPB_P2P_BAR7__HOST_FLUSH_MASK                                                                         0x0000000FL
-#define XPB_P2P_BAR7__REG_SYS_BAR_MASK                                                                        0x000000F0L
-#define XPB_P2P_BAR7__MEM_SYS_BAR_MASK                                                                        0x00000F00L
-#define XPB_P2P_BAR7__VALID_MASK                                                                              0x00001000L
-#define XPB_P2P_BAR7__SEND_DIS_MASK                                                                           0x00002000L
-#define XPB_P2P_BAR7__COMPRESS_DIS_MASK                                                                       0x00004000L
-#define XPB_P2P_BAR7__RESERVED_MASK                                                                           0x00008000L
-#define XPB_P2P_BAR7__ADDRESS_MASK                                                                            0xFFFF0000L
-//XPB_P2P_BAR_SETUP
-#define XPB_P2P_BAR_SETUP__SEL__SHIFT                                                                         0x0
-#define XPB_P2P_BAR_SETUP__REG_SYS_BAR__SHIFT                                                                 0x8
-#define XPB_P2P_BAR_SETUP__VALID__SHIFT                                                                       0xc
-#define XPB_P2P_BAR_SETUP__SEND_DIS__SHIFT                                                                    0xd
-#define XPB_P2P_BAR_SETUP__COMPRESS_DIS__SHIFT                                                                0xe
-#define XPB_P2P_BAR_SETUP__RESERVED__SHIFT                                                                    0xf
-#define XPB_P2P_BAR_SETUP__ADDRESS__SHIFT                                                                     0x10
-#define XPB_P2P_BAR_SETUP__SEL_MASK                                                                           0x000000FFL
-#define XPB_P2P_BAR_SETUP__REG_SYS_BAR_MASK                                                                   0x00000F00L
-#define XPB_P2P_BAR_SETUP__VALID_MASK                                                                         0x00001000L
-#define XPB_P2P_BAR_SETUP__SEND_DIS_MASK                                                                      0x00002000L
-#define XPB_P2P_BAR_SETUP__COMPRESS_DIS_MASK                                                                  0x00004000L
-#define XPB_P2P_BAR_SETUP__RESERVED_MASK                                                                      0x00008000L
-#define XPB_P2P_BAR_SETUP__ADDRESS_MASK                                                                       0xFFFF0000L
-//XPB_P2P_BAR_DELTA_ABOVE
-#define XPB_P2P_BAR_DELTA_ABOVE__EN__SHIFT                                                                    0x0
-#define XPB_P2P_BAR_DELTA_ABOVE__DELTA__SHIFT                                                                 0x8
-#define XPB_P2P_BAR_DELTA_ABOVE__EN_MASK                                                                      0x000000FFL
-#define XPB_P2P_BAR_DELTA_ABOVE__DELTA_MASK                                                                   0x0FFFFF00L
-//XPB_P2P_BAR_DELTA_BELOW
-#define XPB_P2P_BAR_DELTA_BELOW__EN__SHIFT                                                                    0x0
-#define XPB_P2P_BAR_DELTA_BELOW__DELTA__SHIFT                                                                 0x8
-#define XPB_P2P_BAR_DELTA_BELOW__EN_MASK                                                                      0x000000FFL
-#define XPB_P2P_BAR_DELTA_BELOW__DELTA_MASK                                                                   0x0FFFFF00L
-//XPB_PEER_SYS_BAR0
-#define XPB_PEER_SYS_BAR0__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR0__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR0__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR0__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR1
-#define XPB_PEER_SYS_BAR1__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR1__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR1__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR1__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR2
-#define XPB_PEER_SYS_BAR2__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR2__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR2__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR2__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR3
-#define XPB_PEER_SYS_BAR3__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR3__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR3__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR3__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR4
-#define XPB_PEER_SYS_BAR4__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR4__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR4__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR4__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR5
-#define XPB_PEER_SYS_BAR5__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR5__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR5__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR5__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR6
-#define XPB_PEER_SYS_BAR6__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR6__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR6__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR6__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR7
-#define XPB_PEER_SYS_BAR7__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR7__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR7__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR7__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR8
-#define XPB_PEER_SYS_BAR8__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR8__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR8__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR8__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_PEER_SYS_BAR9
-#define XPB_PEER_SYS_BAR9__VALID__SHIFT                                                                       0x0
-#define XPB_PEER_SYS_BAR9__ADDR__SHIFT                                                                        0x1
-#define XPB_PEER_SYS_BAR9__VALID_MASK                                                                         0x00000001L
-#define XPB_PEER_SYS_BAR9__ADDR_MASK                                                                          0xFFFFFFFEL
-//XPB_XDMA_PEER_SYS_BAR0
-#define XPB_XDMA_PEER_SYS_BAR0__VALID__SHIFT                                                                  0x0
-#define XPB_XDMA_PEER_SYS_BAR0__ADDR__SHIFT                                                                   0x1
-#define XPB_XDMA_PEER_SYS_BAR0__VALID_MASK                                                                    0x00000001L
-#define XPB_XDMA_PEER_SYS_BAR0__ADDR_MASK                                                                     0xFFFFFFFEL
-//XPB_XDMA_PEER_SYS_BAR1
-#define XPB_XDMA_PEER_SYS_BAR1__VALID__SHIFT                                                                  0x0
-#define XPB_XDMA_PEER_SYS_BAR1__ADDR__SHIFT                                                                   0x1
-#define XPB_XDMA_PEER_SYS_BAR1__VALID_MASK                                                                    0x00000001L
-#define XPB_XDMA_PEER_SYS_BAR1__ADDR_MASK                                                                     0xFFFFFFFEL
-//XPB_XDMA_PEER_SYS_BAR2
-#define XPB_XDMA_PEER_SYS_BAR2__VALID__SHIFT                                                                  0x0
-#define XPB_XDMA_PEER_SYS_BAR2__ADDR__SHIFT                                                                   0x1
-#define XPB_XDMA_PEER_SYS_BAR2__VALID_MASK                                                                    0x00000001L
-#define XPB_XDMA_PEER_SYS_BAR2__ADDR_MASK                                                                     0xFFFFFFFEL
-//XPB_XDMA_PEER_SYS_BAR3
-#define XPB_XDMA_PEER_SYS_BAR3__VALID__SHIFT                                                                  0x0
-#define XPB_XDMA_PEER_SYS_BAR3__ADDR__SHIFT                                                                   0x1
-#define XPB_XDMA_PEER_SYS_BAR3__VALID_MASK                                                                    0x00000001L
-#define XPB_XDMA_PEER_SYS_BAR3__ADDR_MASK                                                                     0xFFFFFFFEL
-//XPB_CLK_GAT
-#define XPB_CLK_GAT__ONDLY__SHIFT                                                                             0x0
-#define XPB_CLK_GAT__OFFDLY__SHIFT                                                                            0x6
-#define XPB_CLK_GAT__RDYDLY__SHIFT                                                                            0xc
-#define XPB_CLK_GAT__ENABLE__SHIFT                                                                            0x12
-#define XPB_CLK_GAT__MEM_LS_ENABLE__SHIFT                                                                     0x13
-#define XPB_CLK_GAT__ONDLY_MASK                                                                               0x0000003FL
-#define XPB_CLK_GAT__OFFDLY_MASK                                                                              0x00000FC0L
-#define XPB_CLK_GAT__RDYDLY_MASK                                                                              0x0003F000L
-#define XPB_CLK_GAT__ENABLE_MASK                                                                              0x00040000L
-#define XPB_CLK_GAT__MEM_LS_ENABLE_MASK                                                                       0x00080000L
-//XPB_INTF_CFG
-#define XPB_INTF_CFG__RPB_WRREQ_CRD__SHIFT                                                                    0x0
-#define XPB_INTF_CFG__MC_WRRET_ASK__SHIFT                                                                     0x8
-#define XPB_INTF_CFG__XSP_REQ_CRD__SHIFT                                                                      0x10
-#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL__SHIFT                                                                0x17
-#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL__SHIFT                                                                0x18
-#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL__SHIFT                                                                0x19
-#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL__SHIFT                                                                0x1a
-#define XPB_INTF_CFG__XSP_SNOOP_SEL__SHIFT                                                                    0x1b
-#define XPB_INTF_CFG__XSP_SNOOP_VAL__SHIFT                                                                    0x1d
-#define XPB_INTF_CFG__XSP_ORDERING_SEL__SHIFT                                                                 0x1e
-#define XPB_INTF_CFG__XSP_ORDERING_VAL__SHIFT                                                                 0x1f
-#define XPB_INTF_CFG__RPB_WRREQ_CRD_MASK                                                                      0x000000FFL
-#define XPB_INTF_CFG__MC_WRRET_ASK_MASK                                                                       0x0000FF00L
-#define XPB_INTF_CFG__XSP_REQ_CRD_MASK                                                                        0x007F0000L
-#define XPB_INTF_CFG__BIF_REG_SNOOP_SEL_MASK                                                                  0x00800000L
-#define XPB_INTF_CFG__BIF_REG_SNOOP_VAL_MASK                                                                  0x01000000L
-#define XPB_INTF_CFG__BIF_MEM_SNOOP_SEL_MASK                                                                  0x02000000L
-#define XPB_INTF_CFG__BIF_MEM_SNOOP_VAL_MASK                                                                  0x04000000L
-#define XPB_INTF_CFG__XSP_SNOOP_SEL_MASK                                                                      0x18000000L
-#define XPB_INTF_CFG__XSP_SNOOP_VAL_MASK                                                                      0x20000000L
-#define XPB_INTF_CFG__XSP_ORDERING_SEL_MASK                                                                   0x40000000L
-#define XPB_INTF_CFG__XSP_ORDERING_VAL_MASK                                                                   0x80000000L
-//XPB_INTF_STS
-#define XPB_INTF_STS__RPB_WRREQ_CRD__SHIFT                                                                    0x0
-#define XPB_INTF_STS__XSP_REQ_CRD__SHIFT                                                                      0x8
-#define XPB_INTF_STS__HOP_DATA_BUF_FULL__SHIFT                                                                0xf
-#define XPB_INTF_STS__HOP_ATTR_BUF_FULL__SHIFT                                                                0x10
-#define XPB_INTF_STS__CNS_BUF_FULL__SHIFT                                                                     0x11
-#define XPB_INTF_STS__CNS_BUF_BUSY__SHIFT                                                                     0x12
-#define XPB_INTF_STS__RPB_RDREQ_CRD__SHIFT                                                                    0x13
-#define XPB_INTF_STS__RPB_WRREQ_CRD_MASK                                                                      0x000000FFL
-#define XPB_INTF_STS__XSP_REQ_CRD_MASK                                                                        0x00007F00L
-#define XPB_INTF_STS__HOP_DATA_BUF_FULL_MASK                                                                  0x00008000L
-#define XPB_INTF_STS__HOP_ATTR_BUF_FULL_MASK                                                                  0x00010000L
-#define XPB_INTF_STS__CNS_BUF_FULL_MASK                                                                       0x00020000L
-#define XPB_INTF_STS__CNS_BUF_BUSY_MASK                                                                       0x00040000L
-#define XPB_INTF_STS__RPB_RDREQ_CRD_MASK                                                                      0x07F80000L
-//XPB_PIPE_STS
-#define XPB_PIPE_STS__WCB_ANY_PBUF__SHIFT                                                                     0x0
-#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT__SHIFT                                                             0x1
-#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT__SHIFT                                                             0x8
-#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL__SHIFT                                                          0xf
-#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL__SHIFT                                                          0x10
-#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL__SHIFT                                                            0x11
-#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL__SHIFT                                                            0x12
-#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL__SHIFT                                                            0x13
-#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL__SHIFT                                                            0x14
-#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL__SHIFT                                                           0x15
-#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL__SHIFT                                                           0x16
-#define XPB_PIPE_STS__RET_BUF_FULL__SHIFT                                                                     0x17
-#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS__SHIFT                                                                0x18
-#define XPB_PIPE_STS__WCB_ANY_PBUF_MASK                                                                       0x00000001L
-#define XPB_PIPE_STS__WCB_HST_DATA_BUF_CNT_MASK                                                               0x000000FEL
-#define XPB_PIPE_STS__WCB_SID_DATA_BUF_CNT_MASK                                                               0x00007F00L
-#define XPB_PIPE_STS__WCB_HST_RD_PTR_BUF_FULL_MASK                                                            0x00008000L
-#define XPB_PIPE_STS__WCB_SID_RD_PTR_BUF_FULL_MASK                                                            0x00010000L
-#define XPB_PIPE_STS__WCB_HST_REQ_FIFO_FULL_MASK                                                              0x00020000L
-#define XPB_PIPE_STS__WCB_SID_REQ_FIFO_FULL_MASK                                                              0x00040000L
-#define XPB_PIPE_STS__WCB_HST_REQ_OBUF_FULL_MASK                                                              0x00080000L
-#define XPB_PIPE_STS__WCB_SID_REQ_OBUF_FULL_MASK                                                              0x00100000L
-#define XPB_PIPE_STS__WCB_HST_DATA_OBUF_FULL_MASK                                                             0x00200000L
-#define XPB_PIPE_STS__WCB_SID_DATA_OBUF_FULL_MASK                                                             0x00400000L
-#define XPB_PIPE_STS__RET_BUF_FULL_MASK                                                                       0x00800000L
-#define XPB_PIPE_STS__XPB_CLK_BUSY_BITS_MASK                                                                  0xFF000000L
-//XPB_SUB_CTRL
-#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB__SHIFT                                                                 0x0
-#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ__SHIFT                                                                0x1
-#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ__SHIFT                                                              0x2
-#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ__SHIFT                                                                0x3
-#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ__SHIFT                                                                0x4
-#define XPB_SUB_CTRL__STALL_WCB_SID_REQ__SHIFT                                                                0x5
-#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND__SHIFT                                                            0x6
-#define XPB_SUB_CTRL__STALL_WCB_HST_REQ__SHIFT                                                                0x7
-#define XPB_SUB_CTRL__STALL_HST_HOP_REQ__SHIFT                                                                0x8
-#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR__SHIFT                                                           0x9
-#define XPB_SUB_CTRL__RESET_CNS__SHIFT                                                                        0xa
-#define XPB_SUB_CTRL__RESET_RTR__SHIFT                                                                        0xb
-#define XPB_SUB_CTRL__RESET_RET__SHIFT                                                                        0xc
-#define XPB_SUB_CTRL__RESET_MAP__SHIFT                                                                        0xd
-#define XPB_SUB_CTRL__RESET_WCB__SHIFT                                                                        0xe
-#define XPB_SUB_CTRL__RESET_HST__SHIFT                                                                        0xf
-#define XPB_SUB_CTRL__RESET_HOP__SHIFT                                                                        0x10
-#define XPB_SUB_CTRL__RESET_SID__SHIFT                                                                        0x11
-#define XPB_SUB_CTRL__RESET_SRB__SHIFT                                                                        0x12
-#define XPB_SUB_CTRL__RESET_CGR__SHIFT                                                                        0x13
-#define XPB_SUB_CTRL__WRREQ_BYPASS_XPB_MASK                                                                   0x00000001L
-#define XPB_SUB_CTRL__STALL_CNS_RTR_REQ_MASK                                                                  0x00000002L
-#define XPB_SUB_CTRL__STALL_RTR_RPB_WRREQ_MASK                                                                0x00000004L
-#define XPB_SUB_CTRL__STALL_RTR_MAP_REQ_MASK                                                                  0x00000008L
-#define XPB_SUB_CTRL__STALL_MAP_WCB_REQ_MASK                                                                  0x00000010L
-#define XPB_SUB_CTRL__STALL_WCB_SID_REQ_MASK                                                                  0x00000020L
-#define XPB_SUB_CTRL__STALL_MC_XSP_REQ_SEND_MASK                                                              0x00000040L
-#define XPB_SUB_CTRL__STALL_WCB_HST_REQ_MASK                                                                  0x00000080L
-#define XPB_SUB_CTRL__STALL_HST_HOP_REQ_MASK                                                                  0x00000100L
-#define XPB_SUB_CTRL__STALL_XPB_RPB_REQ_ATTR_MASK                                                             0x00000200L
-#define XPB_SUB_CTRL__RESET_CNS_MASK                                                                          0x00000400L
-#define XPB_SUB_CTRL__RESET_RTR_MASK                                                                          0x00000800L
-#define XPB_SUB_CTRL__RESET_RET_MASK                                                                          0x00001000L
-#define XPB_SUB_CTRL__RESET_MAP_MASK                                                                          0x00002000L
-#define XPB_SUB_CTRL__RESET_WCB_MASK                                                                          0x00004000L
-#define XPB_SUB_CTRL__RESET_HST_MASK                                                                          0x00008000L
-#define XPB_SUB_CTRL__RESET_HOP_MASK                                                                          0x00010000L
-#define XPB_SUB_CTRL__RESET_SID_MASK                                                                          0x00020000L
-#define XPB_SUB_CTRL__RESET_SRB_MASK                                                                          0x00040000L
-#define XPB_SUB_CTRL__RESET_CGR_MASK                                                                          0x00080000L
-//XPB_MAP_INVERT_FLUSH_NUM_LSB
-#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM__SHIFT                                                  0x0
-#define XPB_MAP_INVERT_FLUSH_NUM_LSB__ALTER_FLUSH_NUM_MASK                                                    0x0000FFFFL
-//XPB_PERF_KNOBS
-#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH__SHIFT                                                                 0x0
-#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH__SHIFT                                                             0x6
-#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH__SHIFT                                                             0xc
-#define XPB_PERF_KNOBS__CNS_FIFO_DEPTH_MASK                                                                   0x0000003FL
-#define XPB_PERF_KNOBS__WCB_HST_FIFO_DEPTH_MASK                                                               0x00000FC0L
-#define XPB_PERF_KNOBS__WCB_SID_FIFO_DEPTH_MASK                                                               0x0003F000L
-//XPB_STICKY
-#define XPB_STICKY__BITS__SHIFT                                                                               0x0
-#define XPB_STICKY__BITS_MASK                                                                                 0xFFFFFFFFL
-//XPB_STICKY_W1C
-#define XPB_STICKY_W1C__BITS__SHIFT                                                                           0x0
-#define XPB_STICKY_W1C__BITS_MASK                                                                             0xFFFFFFFFL
-//XPB_MISC_CFG
-#define XPB_MISC_CFG__FIELDNAME0__SHIFT                                                                       0x0
-#define XPB_MISC_CFG__FIELDNAME1__SHIFT                                                                       0x8
-#define XPB_MISC_CFG__FIELDNAME2__SHIFT                                                                       0x10
-#define XPB_MISC_CFG__FIELDNAME3__SHIFT                                                                       0x18
-#define XPB_MISC_CFG__TRIGGERNAME__SHIFT                                                                      0x1f
-#define XPB_MISC_CFG__FIELDNAME0_MASK                                                                         0x000000FFL
-#define XPB_MISC_CFG__FIELDNAME1_MASK                                                                         0x0000FF00L
-#define XPB_MISC_CFG__FIELDNAME2_MASK                                                                         0x00FF0000L
-#define XPB_MISC_CFG__FIELDNAME3_MASK                                                                         0x7F000000L
-#define XPB_MISC_CFG__TRIGGERNAME_MASK                                                                        0x80000000L
-//XPB_INTF_CFG2
-#define XPB_INTF_CFG2__RPB_RDREQ_CRD__SHIFT                                                                   0x0
-#define XPB_INTF_CFG2__RPB_RDREQ_CRD_MASK                                                                     0x000000FFL
-//XPB_CLG_EXTRA_RD
-#define XPB_CLG_EXTRA_RD__CMP0_HIGH__SHIFT                                                                    0x0
-#define XPB_CLG_EXTRA_RD__CMP0_LOW__SHIFT                                                                     0x6
-#define XPB_CLG_EXTRA_RD__VLD0__SHIFT                                                                         0xb
-#define XPB_CLG_EXTRA_RD__CLG0_NUM__SHIFT                                                                     0xc
-#define XPB_CLG_EXTRA_RD__CMP1_HIGH__SHIFT                                                                    0xf
-#define XPB_CLG_EXTRA_RD__CMP1_LOW__SHIFT                                                                     0x15
-#define XPB_CLG_EXTRA_RD__VLD1__SHIFT                                                                         0x1a
-#define XPB_CLG_EXTRA_RD__CLG1_NUM__SHIFT                                                                     0x1b
-#define XPB_CLG_EXTRA_RD__CMP0_HIGH_MASK                                                                      0x0000003FL
-#define XPB_CLG_EXTRA_RD__CMP0_LOW_MASK                                                                       0x000007C0L
-#define XPB_CLG_EXTRA_RD__VLD0_MASK                                                                           0x00000800L
-#define XPB_CLG_EXTRA_RD__CLG0_NUM_MASK                                                                       0x00007000L
-#define XPB_CLG_EXTRA_RD__CMP1_HIGH_MASK                                                                      0x001F8000L
-#define XPB_CLG_EXTRA_RD__CMP1_LOW_MASK                                                                       0x03E00000L
-#define XPB_CLG_EXTRA_RD__VLD1_MASK                                                                           0x04000000L
-#define XPB_CLG_EXTRA_RD__CLG1_NUM_MASK                                                                       0x38000000L
-//XPB_CLG_EXTRA_MSK_RD
-#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH__SHIFT                                                                0x0
-#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW__SHIFT                                                                 0x6
-#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH__SHIFT                                                                0xb
-#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW__SHIFT                                                                 0x11
-#define XPB_CLG_EXTRA_MSK_RD__MSK0_HIGH_MASK                                                                  0x0000003FL
-#define XPB_CLG_EXTRA_MSK_RD__MSK0_LOW_MASK                                                                   0x000007C0L
-#define XPB_CLG_EXTRA_MSK_RD__MSK1_HIGH_MASK                                                                  0x0001F800L
-#define XPB_CLG_EXTRA_MSK_RD__MSK1_LOW_MASK                                                                   0x003E0000L
-//XPB_CLG_GFX_MATCH
-#define XPB_CLG_GFX_MATCH__FARBIRC0_ID__SHIFT                                                                 0x0
-#define XPB_CLG_GFX_MATCH__FARBIRC1_ID__SHIFT                                                                 0x6
-#define XPB_CLG_GFX_MATCH__FARBIRC2_ID__SHIFT                                                                 0xc
-#define XPB_CLG_GFX_MATCH__FARBIRC3_ID__SHIFT                                                                 0x12
-#define XPB_CLG_GFX_MATCH__FARBIRC0_VLD__SHIFT                                                                0x18
-#define XPB_CLG_GFX_MATCH__FARBIRC1_VLD__SHIFT                                                                0x19
-#define XPB_CLG_GFX_MATCH__FARBIRC2_VLD__SHIFT                                                                0x1a
-#define XPB_CLG_GFX_MATCH__FARBIRC3_VLD__SHIFT                                                                0x1b
-#define XPB_CLG_GFX_MATCH__FARBIRC0_ID_MASK                                                                   0x0000003FL
-#define XPB_CLG_GFX_MATCH__FARBIRC1_ID_MASK                                                                   0x00000FC0L
-#define XPB_CLG_GFX_MATCH__FARBIRC2_ID_MASK                                                                   0x0003F000L
-#define XPB_CLG_GFX_MATCH__FARBIRC3_ID_MASK                                                                   0x00FC0000L
-#define XPB_CLG_GFX_MATCH__FARBIRC0_VLD_MASK                                                                  0x01000000L
-#define XPB_CLG_GFX_MATCH__FARBIRC1_VLD_MASK                                                                  0x02000000L
-#define XPB_CLG_GFX_MATCH__FARBIRC2_VLD_MASK                                                                  0x04000000L
-#define XPB_CLG_GFX_MATCH__FARBIRC3_VLD_MASK                                                                  0x08000000L
-//XPB_CLG_GFX_MATCH_MSK
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT                                                         0x0
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT                                                         0x6
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK__SHIFT                                                         0xc
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK__SHIFT                                                         0x12
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC0_ID_MSK_MASK                                                           0x0000003FL
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC1_ID_MSK_MASK                                                           0x00000FC0L
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC2_ID_MSK_MASK                                                           0x0003F000L
-#define XPB_CLG_GFX_MATCH_MSK__FARBIRC3_ID_MSK_MASK                                                           0x00FC0000L
-//XPB_CLG_MM_MATCH
-#define XPB_CLG_MM_MATCH__FARBIRC0_ID__SHIFT                                                                  0x0
-#define XPB_CLG_MM_MATCH__FARBIRC1_ID__SHIFT                                                                  0x6
-#define XPB_CLG_MM_MATCH__FARBIRC2_ID__SHIFT                                                                  0xc
-#define XPB_CLG_MM_MATCH__FARBIRC3_ID__SHIFT                                                                  0x12
-#define XPB_CLG_MM_MATCH__FARBIRC0_VLD__SHIFT                                                                 0x18
-#define XPB_CLG_MM_MATCH__FARBIRC1_VLD__SHIFT                                                                 0x19
-#define XPB_CLG_MM_MATCH__FARBIRC2_VLD__SHIFT                                                                 0x1a
-#define XPB_CLG_MM_MATCH__FARBIRC3_VLD__SHIFT                                                                 0x1b
-#define XPB_CLG_MM_MATCH__FARBIRC0_ID_MASK                                                                    0x0000003FL
-#define XPB_CLG_MM_MATCH__FARBIRC1_ID_MASK                                                                    0x00000FC0L
-#define XPB_CLG_MM_MATCH__FARBIRC2_ID_MASK                                                                    0x0003F000L
-#define XPB_CLG_MM_MATCH__FARBIRC3_ID_MASK                                                                    0x00FC0000L
-#define XPB_CLG_MM_MATCH__FARBIRC0_VLD_MASK                                                                   0x01000000L
-#define XPB_CLG_MM_MATCH__FARBIRC1_VLD_MASK                                                                   0x02000000L
-#define XPB_CLG_MM_MATCH__FARBIRC2_VLD_MASK                                                                   0x04000000L
-#define XPB_CLG_MM_MATCH__FARBIRC3_VLD_MASK                                                                   0x08000000L
-//XPB_CLG_MM_MATCH_MSK
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK__SHIFT                                                          0x0
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK__SHIFT                                                          0x6
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC2_ID_MSK__SHIFT                                                          0xc
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC3_ID_MSK__SHIFT                                                          0x12
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC0_ID_MSK_MASK                                                            0x0000003FL
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC1_ID_MSK_MASK                                                            0x00000FC0L
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC2_ID_MSK_MASK                                                            0x0003F000L
-#define XPB_CLG_MM_MATCH_MSK__FARBIRC3_ID_MSK_MASK                                                            0x00FC0000L
-//XPB_CLG_GFX_UNITID_MAPPING0
-#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING0__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING0__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_GFX_UNITID_MAPPING1
-#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING1__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING1__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_GFX_UNITID_MAPPING2
-#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING2__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING2__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_GFX_UNITID_MAPPING3
-#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING3__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING3__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_GFX_UNITID_MAPPING4
-#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING4__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING4__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_GFX_UNITID_MAPPING5
-#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING5__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING5__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_GFX_UNITID_MAPPING6
-#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING6__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING6__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_GFX_UNITID_MAPPING7
-#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW__SHIFT                                                        0x0
-#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD__SHIFT                                                        0x5
-#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM__SHIFT                                                      0x6
-#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_LOW_MASK                                                          0x0000001FL
-#define XPB_CLG_GFX_UNITID_MAPPING7__UNITID_VLD_MASK                                                          0x00000020L
-#define XPB_CLG_GFX_UNITID_MAPPING7__DEST_CLG_NUM_MASK                                                        0x000001C0L
-//XPB_CLG_MM_UNITID_MAPPING0
-#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW__SHIFT                                                         0x0
-#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD__SHIFT                                                         0x5
-#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM__SHIFT                                                       0x6
-#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_LOW_MASK                                                           0x0000001FL
-#define XPB_CLG_MM_UNITID_MAPPING0__UNITID_VLD_MASK                                                           0x00000020L
-#define XPB_CLG_MM_UNITID_MAPPING0__DEST_CLG_NUM_MASK                                                         0x000001C0L
-//XPB_CLG_MM_UNITID_MAPPING1
-#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW__SHIFT                                                         0x0
-#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD__SHIFT                                                         0x5
-#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM__SHIFT                                                       0x6
-#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_LOW_MASK                                                           0x0000001FL
-#define XPB_CLG_MM_UNITID_MAPPING1__UNITID_VLD_MASK                                                           0x00000020L
-#define XPB_CLG_MM_UNITID_MAPPING1__DEST_CLG_NUM_MASK                                                         0x000001C0L
-//XPB_CLG_MM_UNITID_MAPPING2
-#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW__SHIFT                                                         0x0
-#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD__SHIFT                                                         0x5
-#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM__SHIFT                                                       0x6
-#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_LOW_MASK                                                           0x0000001FL
-#define XPB_CLG_MM_UNITID_MAPPING2__UNITID_VLD_MASK                                                           0x00000020L
-#define XPB_CLG_MM_UNITID_MAPPING2__DEST_CLG_NUM_MASK                                                         0x000001C0L
-//XPB_CLG_MM_UNITID_MAPPING3
-#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW__SHIFT                                                         0x0
-#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD__SHIFT                                                         0x5
-#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM__SHIFT                                                       0x6
-#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_LOW_MASK                                                           0x0000001FL
-#define XPB_CLG_MM_UNITID_MAPPING3__UNITID_VLD_MASK                                                           0x00000020L
-#define XPB_CLG_MM_UNITID_MAPPING3__DEST_CLG_NUM_MASK                                                         0x000001C0L
-
-
-// addressBlock: athub_rpbdec
-//RPB_PASSPW_CONF
-#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE__SHIFT                                                           0x0
-#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE__SHIFT                                                        0x1
-#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE__SHIFT                                                        0x2
-#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE__SHIFT                                                      0x3
-#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE__SHIFT                                                            0x4
-#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE__SHIFT                                                            0x5
-#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE__SHIFT                                                         0x6
-#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE__SHIFT                                                         0x7
-#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE__SHIFT                                                        0x8
-#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE__SHIFT                                                        0x9
-#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE__SHIFT                                                     0xa
-#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN__SHIFT                                                     0xb
-#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN__SHIFT                                                   0xc
-#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN__SHIFT                                                     0xd
-#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE__SHIFT                                                         0xe
-#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN__SHIFT                                                      0xf
-#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE__SHIFT                                                         0x10
-#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN__SHIFT                                                      0x11
-#define RPB_PASSPW_CONF__XPB_PASSPW_OVERRIDE_MASK                                                             0x00000001L
-#define RPB_PASSPW_CONF__XPB_RSPPASSPW_OVERRIDE_MASK                                                          0x00000002L
-#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_MASK                                                          0x00000004L
-#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_MASK                                                        0x00000008L
-#define RPB_PASSPW_CONF__WR_PASSPW_OVERRIDE_MASK                                                              0x00000010L
-#define RPB_PASSPW_CONF__RD_PASSPW_OVERRIDE_MASK                                                              0x00000020L
-#define RPB_PASSPW_CONF__WR_RSPPASSPW_OVERRIDE_MASK                                                           0x00000040L
-#define RPB_PASSPW_CONF__RD_RSPPASSPW_OVERRIDE_MASK                                                           0x00000080L
-#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_MASK                                                          0x00000100L
-#define RPB_PASSPW_CONF__ATOMIC_PASSPW_OVERRIDE_MASK                                                          0x00000200L
-#define RPB_PASSPW_CONF__ATOMIC_RSPPASSPW_OVERRIDE_MASK                                                       0x00000400L
-#define RPB_PASSPW_CONF__ATC_TR_PASSPW_OVERRIDE_EN_MASK                                                       0x00000800L
-#define RPB_PASSPW_CONF__ATC_PAGE_PASSPW_OVERRIDE_EN_MASK                                                     0x00001000L
-#define RPB_PASSPW_CONF__ATC_RSPPASSPW_OVERRIDE_EN_MASK                                                       0x00002000L
-#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_MASK                                                           0x00004000L
-#define RPB_PASSPW_CONF__WRRSP_PASSPW_OVERRIDE_EN_MASK                                                        0x00008000L
-#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_MASK                                                           0x00010000L
-#define RPB_PASSPW_CONF__RDRSP_PASSPW_OVERRIDE_EN_MASK                                                        0x00020000L
-//RPB_BLOCKLEVEL_CONF
-#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE__SHIFT                                                   0x0
-#define RPB_BLOCKLEVEL_CONF__ATC_TR_BLOCKLEVEL__SHIFT                                                         0x2
-#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL__SHIFT                                                       0x4
-#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL__SHIFT                                                        0x6
-#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE__SHIFT                                                 0x8
-#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE__SHIFT                                                 0xa
-#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE__SHIFT                                                0xc
-#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                                0xe
-#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                              0xf
-#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                              0x10
-#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN__SHIFT                                             0x11
-#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_MASK                                                     0x00000003L
-#define RPB_BLOCKLEVEL_CONF__ATC_TR_BLOCKLEVEL_MASK                                                           0x0000000CL
-#define RPB_BLOCKLEVEL_CONF__ATC_PAGE_BLOCKLEVEL_MASK                                                         0x00000030L
-#define RPB_BLOCKLEVEL_CONF__ATC_INV_BLOCKLEVEL_MASK                                                          0x000000C0L
-#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_MASK                                                   0x00000300L
-#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_MASK                                                   0x00000C00L
-#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_MASK                                                  0x00003000L
-#define RPB_BLOCKLEVEL_CONF__XPB_BLOCKLEVEL_OVERRIDE_EN_MASK                                                  0x00004000L
-#define RPB_BLOCKLEVEL_CONF__IO_WR_BLOCKLEVEL_OVERRIDE_EN_MASK                                                0x00008000L
-#define RPB_BLOCKLEVEL_CONF__IO_RD_BLOCKLEVEL_OVERRIDE_EN_MASK                                                0x00010000L
-#define RPB_BLOCKLEVEL_CONF__ATOMIC_BLOCKLEVEL_OVERRIDE_EN_MASK                                               0x00020000L
-//RPB_TAG_CONF
-#define RPB_TAG_CONF__RPB_ATS_TR__SHIFT                                                                       0x0
-#define RPB_TAG_CONF__RPB_IO_WR__SHIFT                                                                        0x8
-#define RPB_TAG_CONF__RPB_ATS_PR__SHIFT                                                                       0x10
-#define RPB_TAG_CONF__RPB_ATS_TR_MASK                                                                         0x000000FFL
-#define RPB_TAG_CONF__RPB_IO_WR_MASK                                                                          0x0000FF00L
-#define RPB_TAG_CONF__RPB_ATS_PR_MASK                                                                         0x00FF0000L
-//RPB_EFF_CNTL
-#define RPB_EFF_CNTL__WR_LAZY_TIMER__SHIFT                                                                    0x0
-#define RPB_EFF_CNTL__RD_LAZY_TIMER__SHIFT                                                                    0x8
-#define RPB_EFF_CNTL__WR_LAZY_TIMER_MASK                                                                      0x000000FFL
-#define RPB_EFF_CNTL__RD_LAZY_TIMER_MASK                                                                      0x0000FF00L
-//RPB_ARB_CNTL
-#define RPB_ARB_CNTL__RD_SWITCH_NUM__SHIFT                                                                    0x0
-#define RPB_ARB_CNTL__WR_SWITCH_NUM__SHIFT                                                                    0x8
-#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM__SHIFT                                                                0x10
-#define RPB_ARB_CNTL__ARB_MODE__SHIFT                                                                         0x18
-#define RPB_ARB_CNTL__SWITCH_NUM_MODE__SHIFT                                                                  0x19
-#define RPB_ARB_CNTL__RD_SWITCH_NUM_MASK                                                                      0x000000FFL
-#define RPB_ARB_CNTL__WR_SWITCH_NUM_MASK                                                                      0x0000FF00L
-#define RPB_ARB_CNTL__ATC_TR_SWITCH_NUM_MASK                                                                  0x00FF0000L
-#define RPB_ARB_CNTL__ARB_MODE_MASK                                                                           0x01000000L
-#define RPB_ARB_CNTL__SWITCH_NUM_MODE_MASK                                                                    0x02000000L
-//RPB_ARB_CNTL2
-#define RPB_ARB_CNTL2__P2P_SWITCH_NUM__SHIFT                                                                  0x0
-#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM__SHIFT                                                               0x8
-#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM__SHIFT                                                             0x10
-#define RPB_ARB_CNTL2__P2P_SWITCH_NUM_MASK                                                                    0x000000FFL
-#define RPB_ARB_CNTL2__ATOMIC_SWITCH_NUM_MASK                                                                 0x0000FF00L
-#define RPB_ARB_CNTL2__ATC_PAGE_SWITCH_NUM_MASK                                                               0x00FF0000L
-//RPB_BIF_CNTL
-#define RPB_BIF_CNTL__VC0_SWITCH_NUM__SHIFT                                                                   0x0
-#define RPB_BIF_CNTL__VC1_SWITCH_NUM__SHIFT                                                                   0x8
-#define RPB_BIF_CNTL__ARB_MODE__SHIFT                                                                         0x10
-#define RPB_BIF_CNTL__DRAIN_VC_NUM__SHIFT                                                                     0x11
-#define RPB_BIF_CNTL__SWITCH_ENABLE__SHIFT                                                                    0x12
-#define RPB_BIF_CNTL__SWITCH_THRESHOLD__SHIFT                                                                 0x13
-#define RPB_BIF_CNTL__PAGE_PRI_EN__SHIFT                                                                      0x1b
-#define RPB_BIF_CNTL__TR_PRI_EN__SHIFT                                                                        0x1c
-#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE__SHIFT                                                             0x1d
-#define RPB_BIF_CNTL__PARITY_CHECK_EN__SHIFT                                                                  0x1e
-#define RPB_BIF_CNTL__VC0_SWITCH_NUM_MASK                                                                     0x000000FFL
-#define RPB_BIF_CNTL__VC1_SWITCH_NUM_MASK                                                                     0x0000FF00L
-#define RPB_BIF_CNTL__ARB_MODE_MASK                                                                           0x00010000L
-#define RPB_BIF_CNTL__DRAIN_VC_NUM_MASK                                                                       0x00020000L
-#define RPB_BIF_CNTL__SWITCH_ENABLE_MASK                                                                      0x00040000L
-#define RPB_BIF_CNTL__SWITCH_THRESHOLD_MASK                                                                   0x07F80000L
-#define RPB_BIF_CNTL__PAGE_PRI_EN_MASK                                                                        0x08000000L
-#define RPB_BIF_CNTL__TR_PRI_EN_MASK                                                                          0x10000000L
-#define RPB_BIF_CNTL__VC0_CHAINED_OVERRIDE_MASK                                                               0x20000000L
-#define RPB_BIF_CNTL__PARITY_CHECK_EN_MASK                                                                    0x40000000L
-//RPB_WR_SWITCH_CNTL
-#define RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT                                                          0x0
-#define RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT                                                          0x7
-#define RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT                                                          0xe
-#define RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT                                                          0x15
-#define RPB_WR_SWITCH_CNTL__SWITCH_NUM_MODE__SHIFT                                                            0x1c
-#define RPB_WR_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK                                                            0x0000007FL
-#define RPB_WR_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK                                                            0x00003F80L
-#define RPB_WR_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK                                                            0x001FC000L
-#define RPB_WR_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK                                                            0x0FE00000L
-#define RPB_WR_SWITCH_CNTL__SWITCH_NUM_MODE_MASK                                                              0x10000000L
-//RPB_RD_SWITCH_CNTL
-#define RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM__SHIFT                                                          0x0
-#define RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM__SHIFT                                                          0x7
-#define RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM__SHIFT                                                          0xe
-#define RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM__SHIFT                                                          0x15
-#define RPB_RD_SWITCH_CNTL__SWITCH_NUM_MODE__SHIFT                                                            0x1c
-#define RPB_RD_SWITCH_CNTL__QUEUE0_SWITCH_NUM_MASK                                                            0x0000007FL
-#define RPB_RD_SWITCH_CNTL__QUEUE1_SWITCH_NUM_MASK                                                            0x00003F80L
-#define RPB_RD_SWITCH_CNTL__QUEUE2_SWITCH_NUM_MASK                                                            0x001FC000L
-#define RPB_RD_SWITCH_CNTL__QUEUE3_SWITCH_NUM_MASK                                                            0x0FE00000L
-#define RPB_RD_SWITCH_CNTL__SWITCH_NUM_MODE_MASK                                                              0x10000000L
-//RPB_CID_QUEUE_WR
-#define RPB_CID_QUEUE_WR__CLIENT_ID_LOW__SHIFT                                                                0x0
-#define RPB_CID_QUEUE_WR__CLIENT_ID_HIGH__SHIFT                                                               0x5
-#define RPB_CID_QUEUE_WR__UPDATE_MODE__SHIFT                                                                  0xb
-#define RPB_CID_QUEUE_WR__WRITE_QUEUE__SHIFT                                                                  0xc
-#define RPB_CID_QUEUE_WR__READ_QUEUE__SHIFT                                                                   0xf
-#define RPB_CID_QUEUE_WR__UPDATE__SHIFT                                                                       0x12
-#define RPB_CID_QUEUE_WR__CLIENT_ID_LOW_MASK                                                                  0x0000001FL
-#define RPB_CID_QUEUE_WR__CLIENT_ID_HIGH_MASK                                                                 0x000007E0L
-#define RPB_CID_QUEUE_WR__UPDATE_MODE_MASK                                                                    0x00000800L
-#define RPB_CID_QUEUE_WR__WRITE_QUEUE_MASK                                                                    0x00007000L
-#define RPB_CID_QUEUE_WR__READ_QUEUE_MASK                                                                     0x00038000L
-#define RPB_CID_QUEUE_WR__UPDATE_MASK                                                                         0x00040000L
-//RPB_CID_QUEUE_RD
-#define RPB_CID_QUEUE_RD__CLIENT_ID_LOW__SHIFT                                                                0x0
-#define RPB_CID_QUEUE_RD__CLIENT_ID_HIGH__SHIFT                                                               0x5
-#define RPB_CID_QUEUE_RD__WRITE_QUEUE__SHIFT                                                                  0xb
-#define RPB_CID_QUEUE_RD__READ_QUEUE__SHIFT                                                                   0xe
-#define RPB_CID_QUEUE_RD__CLIENT_ID_LOW_MASK                                                                  0x0000001FL
-#define RPB_CID_QUEUE_RD__CLIENT_ID_HIGH_MASK                                                                 0x000007E0L
-#define RPB_CID_QUEUE_RD__WRITE_QUEUE_MASK                                                                    0x00003800L
-#define RPB_CID_QUEUE_RD__READ_QUEUE_MASK                                                                     0x0001C000L
-//RPB_CID_QUEUE_EX
-#define RPB_CID_QUEUE_EX__START__SHIFT                                                                        0x0
-#define RPB_CID_QUEUE_EX__OFFSET__SHIFT                                                                       0x1
-#define RPB_CID_QUEUE_EX__START_MASK                                                                          0x00000001L
-#define RPB_CID_QUEUE_EX__OFFSET_MASK                                                                         0x000001FEL
-//RPB_CID_QUEUE_EX_DATA
-#define RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES__SHIFT                                                           0x0
-#define RPB_CID_QUEUE_EX_DATA__READ_ENTRIES__SHIFT                                                            0x10
-#define RPB_CID_QUEUE_EX_DATA__WRITE_ENTRIES_MASK                                                             0x0000FFFFL
-#define RPB_CID_QUEUE_EX_DATA__READ_ENTRIES_MASK                                                              0xFFFF0000L
-//RPB_SWITCH_CNTL2
-#define RPB_SWITCH_CNTL2__RD_QUEUE4_SWITCH_NUM__SHIFT                                                         0x0
-#define RPB_SWITCH_CNTL2__RD_QUEUE5_SWITCH_NUM__SHIFT                                                         0x7
-#define RPB_SWITCH_CNTL2__WR_QUEUE4_SWITCH_NUM__SHIFT                                                         0xe
-#define RPB_SWITCH_CNTL2__WR_QUEUE5_SWITCH_NUM__SHIFT                                                         0x15
-#define RPB_SWITCH_CNTL2__RD_QUEUE4_SWITCH_NUM_MASK                                                           0x0000007FL
-#define RPB_SWITCH_CNTL2__RD_QUEUE5_SWITCH_NUM_MASK                                                           0x00003F80L
-#define RPB_SWITCH_CNTL2__WR_QUEUE4_SWITCH_NUM_MASK                                                           0x001FC000L
-#define RPB_SWITCH_CNTL2__WR_QUEUE5_SWITCH_NUM_MASK                                                           0x0FE00000L
-//RPB_DEINTRLV_COMBINE_CNTL
-#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER__SHIFT                                              0x0
-#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN__SHIFT                                                 0x4
-#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE__SHIFT                                             0x5
-#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_FLUSH_TIMER_MASK                                                0x0000000FL
-#define RPB_DEINTRLV_COMBINE_CNTL__WC_CHAINED_BREAK_EN_MASK                                                   0x00000010L
-#define RPB_DEINTRLV_COMBINE_CNTL__WC_HANDLE_CHECK_DISABLE_MASK                                               0x00000020L
-//RPB_VC_SWITCH_RDWR
-#define RPB_VC_SWITCH_RDWR__MODE__SHIFT                                                                       0x0
-#define RPB_VC_SWITCH_RDWR__NUM_RD__SHIFT                                                                     0x2
-#define RPB_VC_SWITCH_RDWR__NUM_WR__SHIFT                                                                     0xa
-#define RPB_VC_SWITCH_RDWR__MODE_MASK                                                                         0x00000003L
-#define RPB_VC_SWITCH_RDWR__NUM_RD_MASK                                                                       0x000003FCL
-#define RPB_VC_SWITCH_RDWR__NUM_WR_MASK                                                                       0x0003FC00L
-//RPB_PERFCOUNTER_LO
-#define RPB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                 0x0
-#define RPB_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                   0xFFFFFFFFL
-//RPB_PERFCOUNTER_HI
-#define RPB_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                 0x0
-#define RPB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                              0x10
-#define RPB_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                   0x0000FFFFL
-#define RPB_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                                0xFFFF0000L
-//RPB_PERFCOUNTER0_CFG
-#define RPB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define RPB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                                0x18
-#define RPB_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                   0x1c
-#define RPB_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                    0x1d
-#define RPB_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define RPB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define RPB_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define RPB_PERFCOUNTER0_CFG__ENABLE_MASK                                                                     0x10000000L
-#define RPB_PERFCOUNTER0_CFG__CLEAR_MASK                                                                      0x20000000L
-//RPB_PERFCOUNTER1_CFG
-#define RPB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define RPB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                                0x18
-#define RPB_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                   0x1c
-#define RPB_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                    0x1d
-#define RPB_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define RPB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define RPB_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define RPB_PERFCOUNTER1_CFG__ENABLE_MASK                                                                     0x10000000L
-#define RPB_PERFCOUNTER1_CFG__CLEAR_MASK                                                                      0x20000000L
-//RPB_PERFCOUNTER2_CFG
-#define RPB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define RPB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                                0x18
-#define RPB_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                                   0x1c
-#define RPB_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                    0x1d
-#define RPB_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define RPB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define RPB_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define RPB_PERFCOUNTER2_CFG__ENABLE_MASK                                                                     0x10000000L
-#define RPB_PERFCOUNTER2_CFG__CLEAR_MASK                                                                      0x20000000L
-//RPB_PERFCOUNTER3_CFG
-#define RPB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                                 0x0
-#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                             0x8
-#define RPB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                                0x18
-#define RPB_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                                   0x1c
-#define RPB_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                    0x1d
-#define RPB_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                                   0x000000FFL
-#define RPB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
-#define RPB_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                                  0x0F000000L
-#define RPB_PERFCOUNTER3_CFG__ENABLE_MASK                                                                     0x10000000L
-#define RPB_PERFCOUNTER3_CFG__CLEAR_MASK                                                                      0x20000000L
-//RPB_PERFCOUNTER_RSLT_CNTL
-#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                 0x0
-#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                       0x8
-#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                        0x10
-#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                          0x18
-#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                           0x19
-#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                                0x1a
-#define RPB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                   0x0000000FL
-#define RPB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                         0x0000FF00L
-#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                          0x00FF0000L
-#define RPB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                            0x01000000L
-#define RPB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                             0x02000000L
-#define RPB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                  0x04000000L
-//RPB_RD_QUEUE_CNTL
-#define RPB_RD_QUEUE_CNTL__ARB_MODE__SHIFT                                                                    0x0
-#define RPB_RD_QUEUE_CNTL__Q4_SHARED__SHIFT                                                                   0x1
-#define RPB_RD_QUEUE_CNTL__Q5_SHARED__SHIFT                                                                   0x2
-#define RPB_RD_QUEUE_CNTL__Q4_UNITID_EA_MODE__SHIFT                                                           0x3
-#define RPB_RD_QUEUE_CNTL__Q5_UNITID_EA_MODE__SHIFT                                                           0x4
-#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_LOW__SHIFT                                                              0x5
-#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_HIGH__SHIFT                                                             0xa
-#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_LOW__SHIFT                                                              0x10
-#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_HIGH__SHIFT                                                             0x15
-#define RPB_RD_QUEUE_CNTL__ARB_MODE_MASK                                                                      0x00000001L
-#define RPB_RD_QUEUE_CNTL__Q4_SHARED_MASK                                                                     0x00000002L
-#define RPB_RD_QUEUE_CNTL__Q5_SHARED_MASK                                                                     0x00000004L
-#define RPB_RD_QUEUE_CNTL__Q4_UNITID_EA_MODE_MASK                                                             0x00000008L
-#define RPB_RD_QUEUE_CNTL__Q5_UNITID_EA_MODE_MASK                                                             0x00000010L
-#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_LOW_MASK                                                                0x000003E0L
-#define RPB_RD_QUEUE_CNTL__Q4_PATTERN_HIGH_MASK                                                               0x0000FC00L
-#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_LOW_MASK                                                                0x001F0000L
-#define RPB_RD_QUEUE_CNTL__Q5_PATTERN_HIGH_MASK                                                               0x07E00000L
-//RPB_RD_QUEUE_CNTL2
-#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW__SHIFT                                                        0x0
-#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH__SHIFT                                                       0x5
-#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW__SHIFT                                                        0xb
-#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH__SHIFT                                                       0x10
-#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW_MASK                                                          0x0000001FL
-#define RPB_RD_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH_MASK                                                         0x000007E0L
-#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW_MASK                                                          0x0000F800L
-#define RPB_RD_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH_MASK                                                         0x003F0000L
-//RPB_WR_QUEUE_CNTL
-#define RPB_WR_QUEUE_CNTL__ARB_MODE__SHIFT                                                                    0x0
-#define RPB_WR_QUEUE_CNTL__Q4_SHARED__SHIFT                                                                   0x1
-#define RPB_WR_QUEUE_CNTL__Q5_SHARED__SHIFT                                                                   0x2
-#define RPB_WR_QUEUE_CNTL__Q4_UNITID_EA_MODE__SHIFT                                                           0x3
-#define RPB_WR_QUEUE_CNTL__Q5_UNITID_EA_MODE__SHIFT                                                           0x4
-#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_LOW__SHIFT                                                              0x5
-#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_HIGH__SHIFT                                                             0xa
-#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_LOW__SHIFT                                                              0x10
-#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_HIGH__SHIFT                                                             0x15
-#define RPB_WR_QUEUE_CNTL__ARB_MODE_MASK                                                                      0x00000001L
-#define RPB_WR_QUEUE_CNTL__Q4_SHARED_MASK                                                                     0x00000002L
-#define RPB_WR_QUEUE_CNTL__Q5_SHARED_MASK                                                                     0x00000004L
-#define RPB_WR_QUEUE_CNTL__Q4_UNITID_EA_MODE_MASK                                                             0x00000008L
-#define RPB_WR_QUEUE_CNTL__Q5_UNITID_EA_MODE_MASK                                                             0x00000010L
-#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_LOW_MASK                                                                0x000003E0L
-#define RPB_WR_QUEUE_CNTL__Q4_PATTERN_HIGH_MASK                                                               0x0000FC00L
-#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_LOW_MASK                                                                0x001F0000L
-#define RPB_WR_QUEUE_CNTL__Q5_PATTERN_HIGH_MASK                                                               0x07E00000L
-//RPB_WR_QUEUE_CNTL2
-#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW__SHIFT                                                        0x0
-#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH__SHIFT                                                       0x5
-#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW__SHIFT                                                        0xb
-#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH__SHIFT                                                       0x10
-#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_LOW_MASK                                                          0x0000001FL
-#define RPB_WR_QUEUE_CNTL2__Q4_PATTERN_MASK_HIGH_MASK                                                         0x000007E0L
-#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_LOW_MASK                                                          0x0000F800L
-#define RPB_WR_QUEUE_CNTL2__Q5_PATTERN_MASK_HIGH_MASK                                                         0x003F0000L
-//RPB_EA_QUEUE_WR
-#define RPB_EA_QUEUE_WR__EA_NUMBER__SHIFT                                                                     0x0
-#define RPB_EA_QUEUE_WR__WRITE_QUEUE__SHIFT                                                                   0x5
-#define RPB_EA_QUEUE_WR__READ_QUEUE__SHIFT                                                                    0x8
-#define RPB_EA_QUEUE_WR__UPDATE__SHIFT                                                                        0xb
-#define RPB_EA_QUEUE_WR__EA_NUMBER_MASK                                                                       0x0000001FL
-#define RPB_EA_QUEUE_WR__WRITE_QUEUE_MASK                                                                     0x000000E0L
-#define RPB_EA_QUEUE_WR__READ_QUEUE_MASK                                                                      0x00000700L
-#define RPB_EA_QUEUE_WR__UPDATE_MASK                                                                          0x00000800L
-//RPB_ATS_CNTL
-#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE__SHIFT                                                          0x0
-#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE__SHIFT                                                            0x1
-#define RPB_ATS_CNTL__SWITCH_THRESHOLD__SHIFT                                                                 0x2
-#define RPB_ATS_CNTL__TIME_SLICE__SHIFT                                                                       0x7
-#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM__SHIFT                                                                 0xf
-#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM__SHIFT                                                               0x13
-#define RPB_ATS_CNTL__WR_AT__SHIFT                                                                            0x17
-#define RPB_ATS_CNTL__INVAL_COM_CMD__SHIFT                                                                    0x19
-#define RPB_ATS_CNTL__PAGE_MIN_LATENCY_ENABLE_MASK                                                            0x00000001L
-#define RPB_ATS_CNTL__TR_MIN_LATENCY_ENABLE_MASK                                                              0x00000002L
-#define RPB_ATS_CNTL__SWITCH_THRESHOLD_MASK                                                                   0x0000007CL
-#define RPB_ATS_CNTL__TIME_SLICE_MASK                                                                         0x00007F80L
-#define RPB_ATS_CNTL__ATCTR_SWITCH_NUM_MASK                                                                   0x00078000L
-#define RPB_ATS_CNTL__ATCPAGE_SWITCH_NUM_MASK                                                                 0x00780000L
-#define RPB_ATS_CNTL__WR_AT_MASK                                                                              0x01800000L
-#define RPB_ATS_CNTL__INVAL_COM_CMD_MASK                                                                      0x7E000000L
-//RPB_ATS_CNTL2
-#define RPB_ATS_CNTL2__TRANS_CMD__SHIFT                                                                       0x0
-#define RPB_ATS_CNTL2__PAGE_REQ_CMD__SHIFT                                                                    0x6
-#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE__SHIFT                                                               0xc
-#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE__SHIFT                                                          0xf
-#define RPB_ATS_CNTL2__VENDOR_ID__SHIFT                                                                       0x12
-#define RPB_ATS_CNTL2__TRANS_CMD_MASK                                                                         0x0000003FL
-#define RPB_ATS_CNTL2__PAGE_REQ_CMD_MASK                                                                      0x00000FC0L
-#define RPB_ATS_CNTL2__PAGE_ROUTING_CODE_MASK                                                                 0x00007000L
-#define RPB_ATS_CNTL2__INVAL_COM_ROUTING_CODE_MASK                                                            0x00038000L
-#define RPB_ATS_CNTL2__VENDOR_ID_MASK                                                                         0x000C0000L
-//RPB_SDPPORT_CNTL
-#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE__SHIFT                                                       0x0
-#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE__SHIFT                                                            0x1
-#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT__SHIFT                                               0x3
-#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER__SHIFT                                             0x4
-#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS__SHIFT                                              0x5
-#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD__SHIFT                                                      0x6
-#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE__SHIFT                                                       0xa
-#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE__SHIFT                                                            0xb
-#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT__SHIFT                                               0xd
-#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER__SHIFT                                             0xe
-#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS__SHIFT                                              0xf
-#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD__SHIFT                                                      0x10
-#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE__SHIFT                                                        0x14
-#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK__SHIFT                                                        0x15
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN__SHIFT                                                         0x16
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV__SHIFT                                                      0x17
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN__SHIFT                                                     0x18
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV__SHIFT                                                  0x19
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN__SHIFT                                                         0x1a
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV__SHIFT                                                      0x1b
-#define RPB_SDPPORT_CNTL__NBIF_DMA_SELF_ACTIVATE_MASK                                                         0x00000001L
-#define RPB_SDPPORT_CNTL__NBIF_DMA_CFG_MODE_MASK                                                              0x00000006L
-#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_REISSUE_CREDIT_MASK                                                 0x00000008L
-#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_SATURATE_COUNTER_MASK                                               0x00000010L
-#define RPB_SDPPORT_CNTL__NBIF_DMA_ENABLE_DISRUPT_FULLDIS_MASK                                                0x00000020L
-#define RPB_SDPPORT_CNTL__NBIF_DMA_HALT_THRESHOLD_MASK                                                        0x000003C0L
-#define RPB_SDPPORT_CNTL__NBIF_HST_SELF_ACTIVATE_MASK                                                         0x00000400L
-#define RPB_SDPPORT_CNTL__NBIF_HST_CFG_MODE_MASK                                                              0x00001800L
-#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_REISSUE_CREDIT_MASK                                                 0x00002000L
-#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_SATURATE_COUNTER_MASK                                               0x00004000L
-#define RPB_SDPPORT_CNTL__NBIF_HST_ENABLE_DISRUPT_FULLDIS_MASK                                                0x00008000L
-#define RPB_SDPPORT_CNTL__NBIF_HST_HALT_THRESHOLD_MASK                                                        0x000F0000L
-#define RPB_SDPPORT_CNTL__NBIF_HST_PASSIVE_MODE_MASK                                                          0x00100000L
-#define RPB_SDPPORT_CNTL__NBIF_HST_QUICK_COMACK_MASK                                                          0x00200000L
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKEN_MASK                                                           0x00400000L
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPCKENRCV_MASK                                                        0x00800000L
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKEN_MASK                                                       0x01000000L
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_RDRSPDATACKENRCV_MASK                                                    0x02000000L
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKEN_MASK                                                           0x04000000L
-#define RPB_SDPPORT_CNTL__DF_SDPVDCI_WRRSPCKENRCV_MASK                                                        0x08000000L
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_default.h
deleted file mode 100644
index 8a0007ce43dc..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/DC/dce_12_0_default.h
+++ /dev/null
@@ -1,9868 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _dce_12_0_DEFAULT_HEADER
-#define _dce_12_0_DEFAULT_HEADER
-
-
-// addressBlock: dce_dc_dispdec_VGA_MEM_WRITE_PAGE_ADDR
-#define mmdispdec_VGA_MEM_WRITE_PAGE_ADDR_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dispdec_VGA_MEM_READ_PAGE_ADDR
-#define mmdispdec_VGA_MEM_READ_PAGE_ADDR_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon0_dispdec
-#define mmDC_PERFMON0_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON0_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON0_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON0_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON0_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON0_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON0_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON0_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON0_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon13_dispdec
-#define mmDC_PERFMON13_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON13_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON13_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON13_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON13_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON13_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON13_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON13_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON13_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dc_displaypllregs_dispdec
-#define mmPPLL_VREG_CFG_DEFAULT                                                  0x00000000
-#define mmPPLL_MODE_CNTL_DEFAULT                                                 0x00020100
-#define mmPPLL_FREQ_CTRL0_DEFAULT                                                0x00280000
-#define mmPPLL_FREQ_CTRL1_DEFAULT                                                0x00000000
-#define mmPPLL_FREQ_CTRL2_DEFAULT                                                0x00000000
-#define mmPPLL_FREQ_CTRL3_DEFAULT                                                0x00190040
-#define mmPPLL_BW_CTRL_COARSE_DEFAULT                                            0x0020c4b1
-#define mmPPLL_BW_CTRL_FINE_DEFAULT                                              0x00000001
-#define mmPPLL_CAL_CTRL_DEFAULT                                                  0x64000002
-#define mmPPLL_LOOP_CTRL_DEFAULT                                                 0x00000090
-#define mmPPLL_REFCLK_CNTL_DEFAULT                                               0x00018004
-#define mmPPLL_CLKOUT_CNTL_DEFAULT                                               0x00022500
-#define mmPPLL_DFT_CNTL_DEFAULT                                                  0x00000004
-#define mmPPLL_ANALOG_CNTL_DEFAULT                                               0x00000000
-#define mmPPLL_POSTDIV_DEFAULT                                                   0x00000400
-#define mmPPLL_OBSERVE0_DEFAULT                                                  0x00000000
-#define mmPPLL_OBSERVE1_DEFAULT                                                  0x04b00000
-#define mmPPLL_UPDATE_CNTL_DEFAULT                                               0x00000000
-#define mmPPLL_OBSERVE0_OUT_DEFAULT                                              0x00000000
-
-
-// addressBlock: dce_dc_dccg_pll0_dispdec
-#define mmPLL_MACRO_CNTL_RESERVED0_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED1_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED2_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED3_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED4_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED5_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED6_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED7_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED8_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED9_DEFAULT                                       0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED10_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED11_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED12_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED13_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED14_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED15_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED16_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED17_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED18_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED19_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED20_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED21_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED22_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED23_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED24_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED25_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED26_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED27_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED28_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED29_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED30_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED31_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED32_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED33_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED34_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED35_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED36_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED37_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED38_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED39_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED40_DEFAULT                                      0x00000000
-#define mmPLL_MACRO_CNTL_RESERVED41_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon1_dispdec
-#define mmDC_PERFMON1_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON1_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON1_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON1_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON1_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON1_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON1_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON1_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON1_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_mcif_wb0_dispdec
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_SW_CONTROL_DEFAULT                             0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_CUR_LINE_R_DEFAULT                             0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_STATUS_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_PITCH_DEFAULT                                     0x04000400
-#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB0_MCIF_WB_ARBITRATION_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_SCLK_CHANGE_DEFAULT                                   0x00000008
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_1_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_2_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_3_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUF_4_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL_DEFAULT                            0x000f0000
-#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_DEFAULT                   0x00000000
-#define mmMCIF_WB0_MCIF_WB_NB_PSTATE_CONTROL_DEFAULT                             0x00000040
-#define mmMCIF_WB0_MCIF_WB_WATERMARK_DEFAULT                                     0x00000000
-#define mmMCIF_WB0_MCIF_WB_CLOCK_GATER_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB0_MCIF_WB_WARM_UP_CNTL_DEFAULT                                  0x00001000
-#define mmMCIF_WB0_MCIF_WB_SELF_REFRESH_CONTROL_DEFAULT                          0x00000002
-#define mmMCIF_WB0_MULTI_LEVEL_QOS_CTRL_DEFAULT                                  0x00000080
-#define mmMCIF_WB0_MCIF_WB_BUF_LUMA_SIZE_DEFAULT                                 0x000fffff
-#define mmMCIF_WB0_MCIF_WB_BUF_CHROMA_SIZE_DEFAULT                               0x000fffff
-
-
-// addressBlock: dce_dc_mcif_wb1_dispdec
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_SW_CONTROL_DEFAULT                             0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_CUR_LINE_R_DEFAULT                             0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_STATUS_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_PITCH_DEFAULT                                     0x04000400
-#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB1_MCIF_WB_ARBITRATION_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_SCLK_CHANGE_DEFAULT                                   0x00000008
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_1_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_2_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_3_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUF_4_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_BUFMGR_VCE_CONTROL_DEFAULT                            0x000f0000
-#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_DEFAULT                   0x00000000
-#define mmMCIF_WB1_MCIF_WB_NB_PSTATE_CONTROL_DEFAULT                             0x00000040
-#define mmMCIF_WB1_MCIF_WB_WATERMARK_DEFAULT                                     0x00000000
-#define mmMCIF_WB1_MCIF_WB_CLOCK_GATER_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB1_MCIF_WB_WARM_UP_CNTL_DEFAULT                                  0x00001000
-#define mmMCIF_WB1_MCIF_WB_SELF_REFRESH_CONTROL_DEFAULT                          0x00000002
-#define mmMCIF_WB1_MULTI_LEVEL_QOS_CTRL_DEFAULT                                  0x00000080
-#define mmMCIF_WB1_MCIF_WB_BUF_LUMA_SIZE_DEFAULT                                 0x000fffff
-#define mmMCIF_WB1_MCIF_WB_BUF_CHROMA_SIZE_DEFAULT                               0x000fffff
-
-
-// addressBlock: dce_dc_mcif_wb2_dispdec
-#define mmMCIF_WB2_MCIF_WB_BUFMGR_SW_CONTROL_DEFAULT                             0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUFMGR_CUR_LINE_R_DEFAULT                             0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUFMGR_STATUS_DEFAULT                                 0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_PITCH_DEFAULT                                     0x04000400
-#define mmMCIF_WB2_MCIF_WB_BUF_1_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_1_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_2_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_2_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_3_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_3_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_4_STATUS_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_4_STATUS2_DEFAULT                                 0x00000000
-#define mmMCIF_WB2_MCIF_WB_ARBITRATION_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_SCLK_CHANGE_DEFAULT                                   0x00000008
-#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_1_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_2_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_3_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_Y_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C_DEFAULT                                  0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUF_4_ADDR_C_OFFSET_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_BUFMGR_VCE_CONTROL_DEFAULT                            0x000f0000
-#define mmMCIF_WB2_MCIF_WB_NB_PSTATE_LATENCY_WATERMARK_DEFAULT                   0x00000000
-#define mmMCIF_WB2_MCIF_WB_NB_PSTATE_CONTROL_DEFAULT                             0x00000040
-#define mmMCIF_WB2_MCIF_WB_WATERMARK_DEFAULT                                     0x00000000
-#define mmMCIF_WB2_MCIF_WB_CLOCK_GATER_CONTROL_DEFAULT                           0x00000000
-#define mmMCIF_WB2_MCIF_WB_WARM_UP_CNTL_DEFAULT                                  0x00001000
-#define mmMCIF_WB2_MCIF_WB_SELF_REFRESH_CONTROL_DEFAULT                          0x00000002
-#define mmMCIF_WB2_MULTI_LEVEL_QOS_CTRL_DEFAULT                                  0x00000080
-#define mmMCIF_WB2_MCIF_WB_BUF_LUMA_SIZE_DEFAULT                                 0x000fffff
-#define mmMCIF_WB2_MCIF_WB_BUF_CHROMA_SIZE_DEFAULT                               0x000fffff
-
-
-// addressBlock: dce_dc_cwb0_dispdec
-#define mmCWB0_CWB_CTRL_DEFAULT                                                  0x00000110
-#define mmCWB0_CWB_FENCE_PAR0_DEFAULT                                            0x03ff03ff
-#define mmCWB0_CWB_FENCE_PAR1_DEFAULT                                            0x000102ff
-#define mmCWB0_CWB_CRC_CTRL_DEFAULT                                              0x00000000
-#define mmCWB0_CWB_CRC_RED_GREEN_MASK_DEFAULT                                    0xffffffff
-#define mmCWB0_CWB_CRC_BLUE_MASK_DEFAULT                                         0x0000ffff
-#define mmCWB0_CWB_CRC_RED_GREEN_RESULT_DEFAULT                                  0x00000000
-#define mmCWB0_CWB_CRC_BLUE_RESULT_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_cwb1_dispdec
-#define mmCWB1_CWB_CTRL_DEFAULT                                                  0x00000110
-#define mmCWB1_CWB_FENCE_PAR0_DEFAULT                                            0x03ff03ff
-#define mmCWB1_CWB_FENCE_PAR1_DEFAULT                                            0x000102ff
-#define mmCWB1_CWB_CRC_CTRL_DEFAULT                                              0x00000000
-#define mmCWB1_CWB_CRC_RED_GREEN_MASK_DEFAULT                                    0xffffffff
-#define mmCWB1_CWB_CRC_BLUE_MASK_DEFAULT                                         0x0000ffff
-#define mmCWB1_CWB_CRC_RED_GREEN_RESULT_DEFAULT                                  0x00000000
-#define mmCWB1_CWB_CRC_BLUE_RESULT_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon9_dispdec
-#define mmDC_PERFMON9_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON9_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON9_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON9_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON9_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON9_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON9_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON9_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON9_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dispdec
-#define mmVGA_MEM_WRITE_PAGE_ADDR_DEFAULT                                        0x00000000
-#define mmVGA_MEM_READ_PAGE_ADDR_DEFAULT                                         0x00000000
-#define mmVGA_RENDER_CONTROL_DEFAULT                                             0x0000000f
-#define mmVGA_SEQUENCER_RESET_CONTROL_DEFAULT                                    0x00003f3f
-#define mmVGA_MODE_CONTROL_DEFAULT                                               0x00000000
-#define mmVGA_SURFACE_PITCH_SELECT_DEFAULT                                       0x00000002
-#define mmVGA_MEMORY_BASE_ADDRESS_DEFAULT                                        0x00000000
-#define mmVGA_DISPBUF1_SURFACE_ADDR_DEFAULT                                      0x00000000
-#define mmVGA_DISPBUF2_SURFACE_ADDR_DEFAULT                                      0x00000000
-#define mmVGA_MEMORY_BASE_ADDRESS_HIGH_DEFAULT                                   0x00000000
-#define mmVGA_HDP_CONTROL_DEFAULT                                                0x00000000
-#define mmVGA_CACHE_CONTROL_DEFAULT                                              0x00000000
-#define mmD1VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD2VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmVGA_STATUS_DEFAULT                                                     0x00000000
-#define mmVGA_INTERRUPT_CONTROL_DEFAULT                                          0x00000000
-#define mmVGA_STATUS_CLEAR_DEFAULT                                               0x00000000
-#define mmVGA_INTERRUPT_STATUS_DEFAULT                                           0x00000000
-#define mmVGA_MAIN_CONTROL_DEFAULT                                               0x00005018
-#define mmVGA_TEST_CONTROL_DEFAULT                                               0x00000000
-#define mmVGA_QOS_CTRL_DEFAULT                                                   0x00000000
-#define mmCRTC8_IDX_DEFAULT                                                      0x00000000
-#define mmCRTC8_DATA_DEFAULT                                                     0x00000000
-#define mmGENFC_WT_DEFAULT                                                       0x00000000
-#define mmGENS1_DEFAULT                                                          0x00000000
-#define mmATTRDW_DEFAULT                                                         0x00000000
-#define mmATTRX_DEFAULT                                                          0x00000000
-#define mmATTRDR_DEFAULT                                                         0x00000000
-#define mmGENMO_WT_DEFAULT                                                       0x00000000
-#define mmGENS0_DEFAULT                                                          0x00000000
-#define mmGENENB_DEFAULT                                                         0x00000000
-#define mmSEQ8_IDX_DEFAULT                                                       0x00000000
-#define mmSEQ8_DATA_DEFAULT                                                      0x00000000
-#define mmDAC_MASK_DEFAULT                                                       0x00000000
-#define mmDAC_R_INDEX_DEFAULT                                                    0x00000000
-#define mmDAC_W_INDEX_DEFAULT                                                    0x00000000
-#define mmDAC_DATA_DEFAULT                                                       0x00000000
-#define mmGENFC_RD_DEFAULT                                                       0x00000000
-#define mmGENMO_RD_DEFAULT                                                       0x00000000
-#define mmGRPH8_IDX_DEFAULT                                                      0x00000000
-#define mmGRPH8_DATA_DEFAULT                                                     0x00000000
-#define mmCRTC8_IDX_1_DEFAULT                                                    0x00000000
-#define mmCRTC8_DATA_1_DEFAULT                                                   0x00000000
-#define mmGENFC_WT_1_DEFAULT                                                     0x00000000
-#define mmGENS1_1_DEFAULT                                                        0x00000000
-#define mmD3VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD4VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD5VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmD6VGA_CONTROL_DEFAULT                                                  0x00000000
-#define mmVGA_SOURCE_SELECT_DEFAULT                                              0x00000100
-#define mmPHYPLLA_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmPHYPLLB_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmPHYPLLC_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmPHYPLLD_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDCFEV0_CRTC_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmDCFEV1_CRTC_PIXEL_RATE_CNTL_DEFAULT                                    0x00000000
-#define mmSYMCLKLPA_CLOCK_ENABLE_DEFAULT                                         0x00000000
-#define mmSYMCLKLPB_CLOCK_ENABLE_DEFAULT                                         0x00000100
-#define mmDPREFCLK_CGTT_BLK_CTRL_REG_DEFAULT                                     0x00000200
-#define mmREFCLK_CNTL_DEFAULT                                                    0x00000000
-#define mmMIPI_CLK_CNTL_DEFAULT                                                  0x00000000
-#define mmREFCLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmPHYPLLE_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDCCG_PERFMON_CNTL2_DEFAULT                                             0x00000000
-#define mmDSICLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmDCCG_CBUS_WRCMD_DELAY_DEFAULT                                          0x00000003
-#define mmDCCG_DS_DTO_INCR_DEFAULT                                               0x00000000
-#define mmDCCG_DS_DTO_MODULO_DEFAULT                                             0x00000000
-#define mmDCCG_DS_CNTL_DEFAULT                                                   0x00000000
-#define mmDCCG_DS_HW_CAL_INTERVAL_DEFAULT                                        0x00989680
-#define mmSYMCLKG_CLOCK_ENABLE_DEFAULT                                           0x00000600
-#define mmDPREFCLK_CNTL_DEFAULT                                                  0x00000000
-#define mmAOMCLK0_CNTL_DEFAULT                                                   0x00000000
-#define mmAOMCLK1_CNTL_DEFAULT                                                   0x00000000
-#define mmAOMCLK2_CNTL_DEFAULT                                                   0x00000000
-#define mmDCCG_AUDIO_DTO2_PHASE_DEFAULT                                          0x00000000
-#define mmDCCG_AUDIO_DTO2_MODULO_DEFAULT                                         0x00000001
-#define mmDCE_VERSION_DEFAULT                                                    0x00000000
-#define mmPHYPLLG_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDCCG_GTC_CNTL_DEFAULT                                                  0x00000000
-#define mmDCCG_GTC_DTO_INCR_DEFAULT                                              0x00000000
-#define mmDCCG_GTC_DTO_MODULO_DEFAULT                                            0x00000000
-#define mmDCCG_GTC_CURRENT_DEFAULT                                               0x00000000
-#define mmDENTIST_DISPCLK_CNTL_DEFAULT                                           0x64010064
-#define mmMIPI_DTO_CNTL_DEFAULT                                                  0x00000000
-#define mmMIPI_DTO_PHASE_DEFAULT                                                 0x00000000
-#define mmMIPI_DTO_MODULO_DEFAULT                                                0x00000000
-#define mmDAC_CLK_ENABLE_DEFAULT                                                 0x00000000
-#define mmDVO_CLK_ENABLE_DEFAULT                                                 0x00000000
-#define mmAVSYNC_COUNTER_WRITE_DEFAULT                                           0x00000000
-#define mmAVSYNC_COUNTER_CONTROL_DEFAULT                                         0x00000000
-#define mmDMCU_SMU_INTERRUPT_CNTL_DEFAULT                                        0x00000000
-#define mmSMU_CONTROL_DEFAULT                                                    0x00000000
-#define mmSMU_INTERRUPT_CONTROL_DEFAULT                                          0x00000000
-#define mmAVSYNC_COUNTER_READ_DEFAULT                                            0x00000000
-#define mmMILLISECOND_TIME_BASE_DIV_DEFAULT                                      0x001186a0
-#define mmDISPCLK_FREQ_CHANGE_CNTL_DEFAULT                                       0x08010028
-#define mmDC_MEM_GLOBAL_PWR_REQ_CNTL_DEFAULT                                     0x00000001
-#define mmDCCG_PERFMON_CNTL_DEFAULT                                              0xfffff800
-#define mmDCCG_GATE_DISABLE_CNTL_DEFAULT                                         0x74ee00fd
-#define mmDISPCLK_CGTT_BLK_CTRL_REG_DEFAULT                                      0x00000200
-#define mmSCLK_CGTT_BLK_CTRL_REG_DEFAULT                                         0x00000200
-#define mmDCCG_CAC_STATUS_DEFAULT                                                0x00000000
-#define mmPIXCLK1_RESYNC_CNTL_DEFAULT                                            0x00000000
-#define mmPIXCLK2_RESYNC_CNTL_DEFAULT                                            0x00000000
-#define mmPIXCLK0_RESYNC_CNTL_DEFAULT                                            0x00000000
-#define mmMICROSECOND_TIME_BASE_DIV_DEFAULT                                      0x00120464
-#define mmDCCG_GATE_DISABLE_CNTL2_DEFAULT                                        0x037f037f
-#define mmSYMCLK_CGTT_BLK_CTRL_REG_DEFAULT                                       0x00000200
-#define mmPHYPLLF_PIXCLK_RESYNC_CNTL_DEFAULT                                     0x00000000
-#define mmDCCG_DISP_CNTL_REG_DEFAULT                                             0x00000000
-#define mmCRTC0_PIXEL_RATE_CNTL_DEFAULT                                          0x00000000
-#define mmDP_DTO0_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO0_MODULO_DEFAULT                                                 0x00000000
-#define mmCRTC0_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                   0x00000000
-#define mmCRTC1_PIXEL_RATE_CNTL_DEFAULT                                          0x00000000
-#define mmDP_DTO1_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO1_MODULO_DEFAULT                                                 0x00000000
-#define mmCRTC1_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                   0x00000000
-#define mmCRTC2_PIXEL_RATE_CNTL_DEFAULT                                          0x00000000
-#define mmDP_DTO2_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO2_MODULO_DEFAULT                                                 0x00000000
-#define mmCRTC2_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                   0x00000000
-#define mmCRTC3_PIXEL_RATE_CNTL_DEFAULT                                          0x00000000
-#define mmDP_DTO3_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO3_MODULO_DEFAULT                                                 0x00000000
-#define mmCRTC3_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                   0x00000000
-#define mmCRTC4_PIXEL_RATE_CNTL_DEFAULT                                          0x00000000
-#define mmDP_DTO4_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO4_MODULO_DEFAULT                                                 0x00000000
-#define mmCRTC4_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                   0x00000000
-#define mmCRTC5_PIXEL_RATE_CNTL_DEFAULT                                          0x00000000
-#define mmDP_DTO5_PHASE_DEFAULT                                                  0x00000000
-#define mmDP_DTO5_MODULO_DEFAULT                                                 0x00000000
-#define mmCRTC5_PHYPLL_PIXEL_RATE_CNTL_DEFAULT                                   0x00000000
-#define mmDCCG_SOFT_RESET_DEFAULT                                                0x00000000
-#define mmSYMCLKA_CLOCK_ENABLE_DEFAULT                                           0x00000000
-#define mmSYMCLKB_CLOCK_ENABLE_DEFAULT                                           0x00000100
-#define mmSYMCLKC_CLOCK_ENABLE_DEFAULT                                           0x00000200
-#define mmSYMCLKD_CLOCK_ENABLE_DEFAULT                                           0x00000300
-#define mmSYMCLKE_CLOCK_ENABLE_DEFAULT                                           0x00000400
-#define mmSYMCLKF_CLOCK_ENABLE_DEFAULT                                           0x00000500
-#define mmDVOACLKD_CNTL_DEFAULT                                                  0x00070000
-#define mmDVOACLKC_MVP_CNTL_DEFAULT                                              0x00030000
-#define mmDVOACLKC_CNTL_DEFAULT                                                  0x00030000
-#define mmDCCG_AUDIO_DTO_SOURCE_DEFAULT                                          0x00000030
-#define mmDCCG_AUDIO_DTO0_PHASE_DEFAULT                                          0x00000000
-#define mmDCCG_AUDIO_DTO0_MODULE_DEFAULT                                         0x00000001
-#define mmDCCG_AUDIO_DTO1_PHASE_DEFAULT                                          0x00000000
-#define mmDCCG_AUDIO_DTO1_MODULE_DEFAULT                                         0x00000001
-#define mmDCCG_TEST_CLK_SEL_DEFAULT                                              0x01ff01ff
-#define mmFBC_CNTL_DEFAULT                                                       0x00000500
-#define mmFBC_IDLE_FORCE_CLEAR_MASK_DEFAULT                                      0x00000000
-#define mmFBC_START_STOP_DELAY_DEFAULT                                           0x00000000
-#define mmFBC_COMP_CNTL_DEFAULT                                                  0x0000000f
-#define mmFBC_COMP_MODE_DEFAULT                                                  0x00000000
-#define mmFBC_IND_LUT0_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT1_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT2_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT3_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT4_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT5_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT6_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT7_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT8_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT9_DEFAULT                                                   0x00000000
-#define mmFBC_IND_LUT10_DEFAULT                                                  0x00000000
-#define mmFBC_IND_LUT11_DEFAULT                                                  0x00000000
-#define mmFBC_IND_LUT12_DEFAULT                                                  0x00000000
-#define mmFBC_IND_LUT13_DEFAULT                                                  0x00000000
-#define mmFBC_IND_LUT14_DEFAULT                                                  0x00000000
-#define mmFBC_IND_LUT15_DEFAULT                                                  0x00000000
-#define mmFBC_CSM_REGION_OFFSET_01_DEFAULT                                       0x00000000
-#define mmFBC_CSM_REGION_OFFSET_23_DEFAULT                                       0x00000000
-#define mmFBC_CLIENT_REGION_MASK_DEFAULT                                         0x00000000
-#define mmFBC_DEBUG_COMP_DEFAULT                                                 0x00000000
-#define mmFBC_MISC_DEFAULT                                                       0x0c306008
-#define mmFBC_STATUS_DEFAULT                                                     0x00000000
-#define mmFBC_ALPHA_CNTL_DEFAULT                                                 0x00000000
-#define mmFBC_ALPHA_RGB_OVERRIDE_DEFAULT                                         0x00000000
-#define mmPIPE0_PG_CONFIG_DEFAULT                                                0x00000001
-#define mmPIPE0_PG_ENABLE_DEFAULT                                                0x00000000
-#define mmPIPE0_PG_STATUS_DEFAULT                                                0x00000000
-#define mmPIPE1_PG_CONFIG_DEFAULT                                                0x00000001
-#define mmPIPE1_PG_ENABLE_DEFAULT                                                0x00000000
-#define mmPIPE1_PG_STATUS_DEFAULT                                                0x00000000
-#define mmPIPE2_PG_CONFIG_DEFAULT                                                0x00000001
-#define mmPIPE2_PG_ENABLE_DEFAULT                                                0x00000000
-#define mmPIPE2_PG_STATUS_DEFAULT                                                0x00000000
-#define mmPIPE3_PG_CONFIG_DEFAULT                                                0x00000001
-#define mmPIPE3_PG_ENABLE_DEFAULT                                                0x00000000
-#define mmPIPE3_PG_STATUS_DEFAULT                                                0x00000000
-#define mmPIPE4_PG_CONFIG_DEFAULT                                                0x00000001
-#define mmPIPE4_PG_ENABLE_DEFAULT                                                0x00000000
-#define mmPIPE4_PG_STATUS_DEFAULT                                                0x00000000
-#define mmPIPE5_PG_CONFIG_DEFAULT                                                0x00000001
-#define mmPIPE5_PG_ENABLE_DEFAULT                                                0x00000000
-#define mmPIPE5_PG_STATUS_DEFAULT                                                0x00000000
-#define mmDSI_PG_CONFIG_DEFAULT                                                  0x00000001
-#define mmDSI_PG_ENABLE_DEFAULT                                                  0x00000000
-#define mmDSI_PG_STATUS_DEFAULT                                                  0x00000000
-#define mmDCFEV0_PG_CONFIG_DEFAULT                                               0x00000001
-#define mmDCFEV0_PG_ENABLE_DEFAULT                                               0x00000000
-#define mmDCFEV0_PG_STATUS_DEFAULT                                               0x00000000
-#define mmDCPG_INTERRUPT_STATUS_DEFAULT                                          0x00000000
-#define mmDCPG_INTERRUPT_CONTROL_DEFAULT                                         0x00000000
-#define mmDCPG_INTERRUPT_CONTROL2_DEFAULT                                        0x00000000
-#define mmDCFEV1_PG_CONFIG_DEFAULT                                               0x00000001
-#define mmDCFEV1_PG_ENABLE_DEFAULT                                               0x00000000
-#define mmDCFEV1_PG_STATUS_DEFAULT                                               0x00000000
-#define mmDC_IP_REQUEST_CNTL_DEFAULT                                             0x00000000
-#define mmDC_PGCNTL_STATUS_REG_DEFAULT                                           0x00000000
-#define mmDMIFV_STATUS_DEFAULT                                                   0x00000000
-#define mmDMIF_CONTROL_DEFAULT                                                   0x00000c04
-#define mmDMIF_STATUS_DEFAULT                                                    0x0ff00000
-#define mmDMIF_ARBITRATION_CONTROL_DEFAULT                                       0x00042710
-#define mmPIPE0_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmPIPE1_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmPIPE2_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmPIPE3_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmPIPE4_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmPIPE5_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmDMIF_P_VMID_DEFAULT                                                    0x00000000
-#define mmDMIF_ADDR_CALC_DEFAULT                                                 0x00000000
-#define mmDMIF_STATUS2_DEFAULT                                                   0x00000000
-#define mmPIPE0_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmPIPE1_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmPIPE2_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmPIPE3_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmPIPE4_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmPIPE5_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmLOW_POWER_TILING_CONTROL_DEFAULT                                       0x00001000
-#define mmMCIF_CONTROL_DEFAULT                                                   0x00000000
-#define mmMCIF_WRITE_COMBINE_CONTROL_DEFAULT                                     0x00000080
-#define mmMCIF_PHASE0_OUTSTANDING_COUNTER_DEFAULT                                0x00000000
-#define mmCC_DC_PIPE_DIS_DEFAULT                                                 0x00000000
-#define mmSMU_WM_CONTROL_DEFAULT                                                 0x00000000
-#define mmRBBMIF_TIMEOUT_DEFAULT                                                 0x20000a00
-#define mmRBBMIF_STATUS_DEFAULT                                                  0x80000000
-#define mmRBBMIF_TIMEOUT_DIS_DEFAULT                                             0x00000000
-#define mmDCI_MEM_PWR_STATUS_DEFAULT                                             0x00000000
-#define mmDCI_MEM_PWR_STATUS2_DEFAULT                                            0x00000000
-#define mmDCI_CLK_CNTL_DEFAULT                                                   0x00000000
-#define mmDCI_CLK_CNTL2_DEFAULT                                                  0x00020020
-#define mmDCI_MEM_PWR_CNTL_DEFAULT                                               0x00000000
-#define mmDCI_MEM_PWR_CNTL2_DEFAULT                                              0x00000000
-#define mmDCI_MEM_PWR_CNTL3_DEFAULT                                              0x00000000
-#define mmPIPE0_DMIF_BUFFER_CONTROL_DEFAULT                                      0x00000000
-#define mmPIPE1_DMIF_BUFFER_CONTROL_DEFAULT                                      0x00000000
-#define mmPIPE2_DMIF_BUFFER_CONTROL_DEFAULT                                      0x00000000
-#define mmPIPE3_DMIF_BUFFER_CONTROL_DEFAULT                                      0x00000000
-#define mmPIPE4_DMIF_BUFFER_CONTROL_DEFAULT                                      0x00000000
-#define mmPIPE5_DMIF_BUFFER_CONTROL_DEFAULT                                      0x00000000
-#define mmRBBMIF_STATUS_FLAG_DEFAULT                                             0x00000000
-#define mmDCI_SOFT_RESET_DEFAULT                                                 0x00000000
-#define mmDMIF_URG_OVERRIDE_DEFAULT                                              0x00000000
-#define mmPIPE6_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmPIPE7_ARBITRATION_CONTROL3_DEFAULT                                     0x00000000
-#define mmPIPE6_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmPIPE7_MAX_REQUESTS_DEFAULT                                             0x000003ff
-#define mmDVMM_REG_RD_STATUS_DEFAULT                                             0x00000000
-#define mmDVMM_REG_RD_DATA_DEFAULT                                               0x00000000
-#define mmDVMM_PTE_REQ_DEFAULT                                                   0x000120ff
-#define mmDVMM_CNTL_DEFAULT                                                      0x00000000
-#define mmDVMM_FAULT_STATUS_DEFAULT                                              0x00000000
-#define mmDVMM_FAULT_ADDR_DEFAULT                                                0x00000000
-#define mmFMON_CTRL_DEFAULT                                                      0x0000f040
-#define mmDVMM_PTE_PGMEM_CONTROL_DEFAULT                                         0x00000000
-#define mmDVMM_PTE_PGMEM_STATE_DEFAULT                                           0x00000000
-#define mmMCIF_PHASE1_OUTSTANDING_COUNTER_DEFAULT                                0x00000000
-#define mmMCIF_PHASE2_OUTSTANDING_COUNTER_DEFAULT                                0x00000000
-#define mmMCIF_WB_PHASE0_OUTSTANDING_COUNTER_DEFAULT                             0x00000000
-#define mmMCIF_WB_PHASE1_OUTSTANDING_COUNTER_DEFAULT                             0x00000000
-#define mmDCI_MEM_PWR_CNTL4_DEFAULT                                              0x0000003f
-#define mmMCIF_WB_MISC_CTRL_DEFAULT                                              0x00010001
-#define mmDCI_MEM_PWR_STATUS3_DEFAULT                                            0x00000000
-#define mmDMIF_CURSOR_CONTROL_DEFAULT                                            0x00000000
-#define mmDMIF_CURSOR_MEM_CONTROL_DEFAULT                                        0x00000000
-#define mmDCHUB_FB_LOCATION_DEFAULT                                              0x00000000
-#define mmDCHUB_FB_OFFSET_DEFAULT                                                0x00000000
-#define mmDCHUB_AGP_BASE_DEFAULT                                                 0x00000000
-#define mmDCHUB_AGP_BOT_DEFAULT                                                  0x00000000
-#define mmDCHUB_AGP_TOP_DEFAULT                                                  0x00000000
-#define mmDCHUB_DRAM_APER_BASE_DEFAULT                                           0x00000000
-#define mmDCHUB_DRAM_APER_DEF_DEFAULT                                            0x00000000
-#define mmDCHUB_DRAM_APER_TOP_DEFAULT                                            0x00000000
-#define mmDCHUB_CONTROL_STATUS_DEFAULT                                           0x00c00000
-#define mmWB_ENABLE_DEFAULT                                                      0x00000000
-#define mmWB_EC_CONFIG_DEFAULT                                                   0x55000000
-#define mmCNV_MODE_DEFAULT                                                       0x00000000
-#define mmCNV_WINDOW_START_DEFAULT                                               0x00000000
-#define mmCNV_WINDOW_SIZE_DEFAULT                                                0x00100010
-#define mmCNV_UPDATE_DEFAULT                                                     0x00000000
-#define mmCNV_SOURCE_SIZE_DEFAULT                                                0x00100010
-#define mmCNV_CSC_CONTROL_DEFAULT                                                0x00000000
-#define mmCNV_CSC_C11_C12_DEFAULT                                                0x00000000
-#define mmCNV_CSC_C13_C14_DEFAULT                                                0x00000000
-#define mmCNV_CSC_C21_C22_DEFAULT                                                0x00000000
-#define mmCNV_CSC_C23_C24_DEFAULT                                                0x00000000
-#define mmCNV_CSC_C31_C32_DEFAULT                                                0x00000000
-#define mmCNV_CSC_C33_C34_DEFAULT                                                0x00000000
-#define mmCNV_CSC_ROUND_OFFSET_R_DEFAULT                                         0x00000000
-#define mmCNV_CSC_ROUND_OFFSET_G_DEFAULT                                         0x00000000
-#define mmCNV_CSC_ROUND_OFFSET_B_DEFAULT                                         0x00000000
-#define mmCNV_CSC_CLAMP_R_DEFAULT                                                0x00000fff
-#define mmCNV_CSC_CLAMP_G_DEFAULT                                                0x00000fff
-#define mmCNV_CSC_CLAMP_B_DEFAULT                                                0x00000fff
-#define mmCNV_TEST_CNTL_DEFAULT                                                  0x00000000
-#define mmCNV_TEST_CRC_RED_DEFAULT                                               0x0000fff0
-#define mmCNV_TEST_CRC_GREEN_DEFAULT                                             0x0000fff0
-#define mmCNV_TEST_CRC_BLUE_DEFAULT                                              0x0000fff0
-#define mmCNV_INPUT_SELECT_DEFAULT                                               0x00000000
-#define mmWB_SOFT_RESET_DEFAULT                                                  0x00000000
-#define mmWB_WARM_UP_MODE_CTL1_DEFAULT                                           0x88700100
-#define mmWB_WARM_UP_MODE_CTL2_DEFAULT                                           0x00000100
-#define mmWBSCL_COEF_RAM_SELECT_DEFAULT                                          0x00000000
-#define mmWBSCL_COEF_RAM_TAP_DATA_DEFAULT                                        0x00000000
-#define mmWBSCL_MODE_DEFAULT                                                     0x00000000
-#define mmWBSCL_TAP_CONTROL_DEFAULT                                              0x00001111
-#define mmWBSCL_DEST_SIZE_DEFAULT                                                0x00010001
-#define mmWBSCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                                  0x00080000
-#define mmWBSCL_HORZ_FILTER_INIT_Y_RGB_DEFAULT                                   0x01000000
-#define mmWBSCL_HORZ_FILTER_INIT_CBCR_DEFAULT                                    0x01000000
-#define mmWBSCL_VERT_FILTER_SCALE_RATIO_DEFAULT                                  0x00080000
-#define mmWBSCL_VERT_FILTER_INIT_Y_RGB_DEFAULT                                   0x01000000
-#define mmWBSCL_VERT_FILTER_INIT_CBCR_DEFAULT                                    0x01000000
-#define mmWBSCL_ROUND_OFFSET_DEFAULT                                             0x00800010
-#define mmWBSCL_CLAMP_DEFAULT                                                    0x01fe01fe
-#define mmWBSCL_OVERFLOW_STATUS_DEFAULT                                          0x00000000
-#define mmWBSCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                                 0x00000000
-#define mmWBSCL_OUTSIDE_PIX_STRATEGY_DEFAULT                                     0x80108000
-#define mmWBSCL_TEST_CNTL_DEFAULT                                                0x00000000
-#define mmWBSCL_TEST_CRC_RED_DEFAULT                                             0x0000ff00
-#define mmWBSCL_TEST_CRC_GREEN_DEFAULT                                           0x0000ffff
-#define mmWBSCL_TEST_CRC_BLUE_DEFAULT                                            0x0000ff00
-#define mmWBSCL_BACKPRESSURE_CNT_EN_DEFAULT                                      0x00000000
-#define mmWB_MCIF_BACKPRESSURE_CNT_DEFAULT                                       0x00000000
-#define mmWBSCL_RAM_SHUTDOWN_DEFAULT                                             0x00000000
-#define mmDMCU_CTRL_DEFAULT                                                      0xffff0101
-#define mmDMCU_STATUS_DEFAULT                                                    0x00000001
-#define mmDMCU_PC_START_ADDR_DEFAULT                                             0x00000000
-#define mmDMCU_FW_START_ADDR_DEFAULT                                             0x00000000
-#define mmDMCU_FW_END_ADDR_DEFAULT                                               0x00000000
-#define mmDMCU_FW_ISR_START_ADDR_DEFAULT                                         0x00000004
-#define mmDMCU_FW_CS_HI_DEFAULT                                                  0x00000000
-#define mmDMCU_FW_CS_LO_DEFAULT                                                  0x00000000
-#define mmDMCU_RAM_ACCESS_CTRL_DEFAULT                                           0x00000000
-#define mmDMCU_ERAM_WR_CTRL_DEFAULT                                              0x000f0000
-#define mmDMCU_ERAM_WR_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_ERAM_RD_CTRL_DEFAULT                                              0x000f0000
-#define mmDMCU_ERAM_RD_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_WR_CTRL_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_WR_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_RD_CTRL_DEFAULT                                              0x00000000
-#define mmDMCU_IRAM_RD_DATA_DEFAULT                                              0x00000000
-#define mmDMCU_EVENT_TRIGGER_DEFAULT                                             0x00000000
-#define mmDMCU_UC_INTERNAL_INT_STATUS_DEFAULT                                    0x00000000
-#define mmDMCU_SS_INTERRUPT_CNTL_STATUS_DEFAULT                                  0x00000000
-#define mmDMCU_INTERRUPT_STATUS_DEFAULT                                          0x00000000
-#define mmDMCU_INTERRUPT_TO_HOST_EN_MASK_DEFAULT                                 0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_DEFAULT                                   0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_DEFAULT                              0x00000000
-#define mmDC_DMCU_SCRATCH_DEFAULT                                                0x00000000
-#define mmDMCU_INT_CNT_DEFAULT                                                   0x00000000
-#define mmDMCU_FW_CHECKSUM_SMPL_BYTE_POS_DEFAULT                                 0x00000000
-#define mmDMCU_UC_CLK_GATING_CNTL_DEFAULT                                        0x00010102
-#define mmMASTER_COMM_DATA_REG1_DEFAULT                                          0x00000000
-#define mmMASTER_COMM_DATA_REG2_DEFAULT                                          0x00000000
-#define mmMASTER_COMM_DATA_REG3_DEFAULT                                          0x00000000
-#define mmMASTER_COMM_CMD_REG_DEFAULT                                            0x00000000
-#define mmMASTER_COMM_CNTL_REG_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_DATA_REG1_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_DATA_REG2_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_DATA_REG3_DEFAULT                                           0x00000000
-#define mmSLAVE_COMM_CMD_REG_DEFAULT                                             0x00000000
-#define mmSLAVE_COMM_CNTL_REG_DEFAULT                                            0x00000000
-#define mmBL1_PWM_AMBIENT_LIGHT_LEVEL_DEFAULT                                    0x00000000
-#define mmBL1_PWM_USER_LEVEL_DEFAULT                                             0x00000000
-#define mmBL1_PWM_TARGET_ABM_LEVEL_DEFAULT                                       0x00000000
-#define mmBL1_PWM_CURRENT_ABM_LEVEL_DEFAULT                                      0x00000000
-#define mmBL1_PWM_FINAL_DUTY_CYCLE_DEFAULT                                       0x00000000
-#define mmBL1_PWM_MINIMUM_DUTY_CYCLE_DEFAULT                                     0x00000000
-#define mmBL1_PWM_ABM_CNTL_DEFAULT                                               0x00000000
-#define mmBL1_PWM_BL_UPDATE_SAMPLE_RATE_DEFAULT                                  0x00000000
-#define mmBL1_PWM_GRP2_REG_LOCK_DEFAULT                                          0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_EN_MASK_1_DEFAULT                                 0x00000000
-#define mmDMCU_INTERRUPT_TO_UC_XIRQ_IRQ_SEL_1_DEFAULT                            0x00000000
-#define mmDMCU_INTERRUPT_STATUS_1_DEFAULT                                        0x00000000
-#define mmDMCU_DPRX_INTERRUPT_STATUS1_DEFAULT                                    0x00000000
-#define mmDMCU_DPRX_INTERRUPT_TO_UC_EN_MASK1_DEFAULT                             0x00000000
-#define mmDMCU_DPRX_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1_DEFAULT                        0x00000000
-#define mmDC_ABM1_CNTL_DEFAULT                                                   0x00000000
-#define mmDC_ABM1_IPCSC_COEFF_SEL_DEFAULT                                        0x00000000
-#define mmDC_ABM1_ACE_OFFSET_SLOPE_0_DEFAULT                                     0x00000400
-#define mmDC_ABM1_ACE_OFFSET_SLOPE_1_DEFAULT                                     0x00000400
-#define mmDC_ABM1_ACE_OFFSET_SLOPE_2_DEFAULT                                     0x00000400
-#define mmDC_ABM1_ACE_OFFSET_SLOPE_3_DEFAULT                                     0x00000400
-#define mmDC_ABM1_ACE_OFFSET_SLOPE_4_DEFAULT                                     0x00000400
-#define mmDC_ABM1_ACE_THRES_12_DEFAULT                                           0x00000000
-#define mmDC_ABM1_ACE_THRES_34_DEFAULT                                           0x00000000
-#define mmDC_ABM1_ACE_CNTL_MISC_DEFAULT                                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS5_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK5_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS1_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS2_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS3_DEFAULT                                 0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_STATUS4_DEFAULT                                 0x00000000
-#define mmDC_ABM1_HGLS_REG_READ_PROGRESS_DEFAULT                                 0x00000000
-#define mmDC_ABM1_HG_MISC_CTRL_DEFAULT                                           0x00000000
-#define mmDC_ABM1_LS_SUM_OF_LUMA_DEFAULT                                         0x00000000
-#define mmDC_ABM1_LS_MIN_MAX_LUMA_DEFAULT                                        0x00000000
-#define mmDC_ABM1_LS_FILTERED_MIN_MAX_LUMA_DEFAULT                               0x00000000
-#define mmDC_ABM1_LS_PIXEL_COUNT_DEFAULT                                         0x00000000
-#define mmDC_ABM1_LS_OVR_SCAN_BIN_DEFAULT                                        0x00000000
-#define mmDC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES_DEFAULT                           0x00000000
-#define mmDC_ABM1_LS_MIN_PIXEL_VALUE_COUNT_DEFAULT                               0x00000000
-#define mmDC_ABM1_LS_MAX_PIXEL_VALUE_COUNT_DEFAULT                               0x00000000
-#define mmDC_ABM1_HG_SAMPLE_RATE_DEFAULT                                         0x00000000
-#define mmDC_ABM1_LS_SAMPLE_RATE_DEFAULT                                         0x00000000
-#define mmDC_ABM1_HG_BIN_1_32_SHIFT_FLAG_DEFAULT                                 0x00000000
-#define mmDC_ABM1_HG_BIN_1_8_SHIFT_INDEX_DEFAULT                                 0x00000000
-#define mmDC_ABM1_HG_BIN_9_16_SHIFT_INDEX_DEFAULT                                0x00000000
-#define mmDC_ABM1_HG_BIN_17_24_SHIFT_INDEX_DEFAULT                               0x00000000
-#define mmDC_ABM1_HG_BIN_25_32_SHIFT_INDEX_DEFAULT                               0x00000000
-#define mmDC_ABM1_HG_RESULT_1_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_2_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_3_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_4_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_5_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_6_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_7_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_8_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_9_DEFAULT                                            0x00000000
-#define mmDC_ABM1_HG_RESULT_10_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_11_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_12_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_13_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_14_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_15_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_16_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_17_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_18_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_19_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_20_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_21_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_22_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_23_DEFAULT                                           0x00000000
-#define mmDC_ABM1_HG_RESULT_24_DEFAULT                                           0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL5_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK1_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK2_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK3_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_EN_MASK4_DEFAULT                          0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL1_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL2_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL3_DEFAULT                     0x00000000
-#define mmDMCU_PERFMON_INTERRUPT_TO_UC_XIRQ_IRQ_SEL4_DEFAULT                     0x00000000
-#define mmDC_ABM1_OVERSCAN_PIXEL_VALUE_DEFAULT                                   0x00000000
-#define mmDC_ABM1_BL_MASTER_LOCK_DEFAULT                                         0x00000000
-#define mmAZALIA_CONTROLLER_CLOCK_GATING_DEFAULT                                 0x00000000
-#define mmAZALIA_AUDIO_DTO_DEFAULT                                               0x001b0018
-#define mmAZALIA_AUDIO_DTO_CONTROL_DEFAULT                                       0x00000000
-#define mmAZALIA_SOCCLK_CONTROL_DEFAULT                                          0x00000001
-#define mmAZALIA_UNDERFLOW_FILLER_SAMPLE_DEFAULT                                 0x00000000
-#define mmAZALIA_DATA_DMA_CONTROL_DEFAULT                                        0x0000000a
-#define mmAZALIA_BDL_DMA_CONTROL_DEFAULT                                         0x0000000a
-#define mmAZALIA_RIRB_AND_DP_CONTROL_DEFAULT                                     0x00000000
-#define mmAZALIA_CORB_DMA_CONTROL_DEFAULT                                        0x00000000
-#define mmAZALIA_APPLICATION_POSITION_IN_CYCLIC_BUFFER_DEFAULT                   0x00000000
-#define mmAZALIA_CYCLIC_BUFFER_SYNC_DEFAULT                                      0x00000000
-#define mmAZALIA_GLOBAL_CAPABILITIES_DEFAULT                                     0x00000000
-#define mmAZALIA_OUTPUT_PAYLOAD_CAPABILITY_DEFAULT                               0x00000060
-#define mmAZALIA_OUTPUT_STREAM_ARBITER_CONTROL_DEFAULT                           0x00080008
-#define mmAZALIA_INPUT_PAYLOAD_CAPABILITY_DEFAULT                                0x00000080
-#define mmAZALIA_INPUT_CRC0_CONTROL0_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_CONTROL1_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_CONTROL2_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_CONTROL3_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC0_RESULT_DEFAULT                                       0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL0_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL1_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL2_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_CONTROL3_DEFAULT                                     0x00000000
-#define mmAZALIA_INPUT_CRC1_RESULT_DEFAULT                                       0x00000000
-#define mmAZALIA_CRC0_CONTROL0_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_CONTROL1_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_CONTROL2_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_CONTROL3_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC0_RESULT_DEFAULT                                             0x00000000
-#define mmAZALIA_CRC1_CONTROL0_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_CONTROL1_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_CONTROL2_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_CONTROL3_DEFAULT                                           0x00000000
-#define mmAZALIA_CRC1_RESULT_DEFAULT                                             0x00000000
-#define mmAZALIA_MEM_PWR_CTRL_DEFAULT                                            0x00000000
-#define mmAZALIA_MEM_PWR_STATUS_DEFAULT                                          0x00000000
-#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_DEFAULT            0x1002aa01
-#define mmAZALIA_F0_CODEC_ROOT_PARAMETER_REVISION_ID_DEFAULT                     0x00100700
-#define mmAZALIA_F0_CODEC_CHANNEL_COUNT_CONTROL_DEFAULT                          0x00000000
-#define mmAZALIA_F0_CODEC_RESYNC_FIFO_CONTROL_DEFAULT                            0x0000000d
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_DEFAULT                  0x00000001
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT        0x00020070
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_DEFAULT              0x00000001
-#define mmAZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES_DEFAULT                0xc0000009
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_POWER_STATE_DEFAULT                   0x00000200
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESET_DEFAULT                         0x00000000
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_DEFAULT         0x00aa0100
-#define mmAZALIA_F0_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION_DEFAULT     0x00000000
-#define mmCC_RCU_DC_AUDIO_PORT_CONNECTIVITY_DEFAULT                              0x00000000
-#define mmCC_RCU_DC_AUDIO_INPUT_PORT_CONNECTIVITY_DEFAULT                        0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET0_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET1_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET2_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET3_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET4_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET5_DEFAULT                                    0x00000000
-#define mmAZALIA_F0_GTC_GROUP_OFFSET6_DEFAULT                                    0x00000000
-#define mmREG_DC_AUDIO_PORT_CONNECTIVITY_DEFAULT                                 0x00000000
-#define mmREG_DC_AUDIO_INPUT_PORT_CONNECTIVITY_DEFAULT                           0x00000000
-#define mmDAC_ENABLE_DEFAULT                                                     0x00000004
-#define mmDAC_SOURCE_SELECT_DEFAULT                                              0x00000000
-#define mmDAC_CRC_EN_DEFAULT                                                     0x00000000
-#define mmDAC_CRC_CONTROL_DEFAULT                                                0x00000000
-#define mmDAC_CRC_SIG_RGB_MASK_DEFAULT                                           0x3fffffff
-#define mmDAC_CRC_SIG_CONTROL_MASK_DEFAULT                                       0x0000003f
-#define mmDAC_CRC_SIG_RGB_DEFAULT                                                0x3fffffff
-#define mmDAC_CRC_SIG_CONTROL_DEFAULT                                            0x0000003f
-#define mmDAC_SYNC_TRISTATE_CONTROL_DEFAULT                                      0x00000000
-#define mmDAC_STEREOSYNC_SELECT_DEFAULT                                          0x00000000
-#define mmDAC_AUTODETECT_CONTROL_DEFAULT                                         0x00070000
-#define mmDAC_AUTODETECT_CONTROL2_DEFAULT                                        0x0000000b
-#define mmDAC_AUTODETECT_CONTROL3_DEFAULT                                        0x00000519
-#define mmDAC_AUTODETECT_STATUS_DEFAULT                                          0x00000000
-#define mmDAC_AUTODETECT_INT_CONTROL_DEFAULT                                     0x00000000
-#define mmDAC_FORCE_OUTPUT_CNTL_DEFAULT                                          0x00000000
-#define mmDAC_FORCE_DATA_DEFAULT                                                 0x000001e6
-#define mmDAC_POWERDOWN_DEFAULT                                                  0x01010100
-#define mmDAC_CONTROL_DEFAULT                                                    0x00000000
-#define mmDAC_COMPARATOR_ENABLE_DEFAULT                                          0x00000000
-#define mmDAC_COMPARATOR_OUTPUT_DEFAULT                                          0x00000000
-#define mmDAC_PWR_CNTL_DEFAULT                                                   0x00000000
-#define mmDAC_DFT_CONFIG_DEFAULT                                                 0x00000000
-#define mmDAC_FIFO_STATUS_DEFAULT                                                0x00000000
-#define mmDC_I2C_CONTROL_DEFAULT                                                 0x00000000
-#define mmDC_I2C_ARBITRATION_DEFAULT                                             0x00000001
-#define mmDC_I2C_INTERRUPT_CONTROL_DEFAULT                                       0x00000000
-#define mmDC_I2C_SW_STATUS_DEFAULT                                               0x00000000
-#define mmDC_I2C_DDC1_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC2_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC3_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC4_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC5_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC6_HW_STATUS_DEFAULT                                          0x00000000
-#define mmDC_I2C_DDC1_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC1_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC2_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC2_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC3_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC3_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC4_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC4_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC5_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC5_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_DDC6_SPEED_DEFAULT                                              0x00000002
-#define mmDC_I2C_DDC6_SETUP_DEFAULT                                              0x00000000
-#define mmDC_I2C_TRANSACTION0_DEFAULT                                            0x00000000
-#define mmDC_I2C_TRANSACTION1_DEFAULT                                            0x00000000
-#define mmDC_I2C_TRANSACTION2_DEFAULT                                            0x00000000
-#define mmDC_I2C_TRANSACTION3_DEFAULT                                            0x00000000
-#define mmDC_I2C_DATA_DEFAULT                                                    0x00000000
-#define mmDC_I2C_DDCVGA_HW_STATUS_DEFAULT                                        0x00000000
-#define mmDC_I2C_DDCVGA_SPEED_DEFAULT                                            0x00000002
-#define mmDC_I2C_DDCVGA_SETUP_DEFAULT                                            0x00000000
-#define mmDC_I2C_EDID_DETECT_CTRL_DEFAULT                                        0x004001f4
-#define mmDC_I2C_READ_REQUEST_INTERRUPT_DEFAULT                                  0x40000000
-#define mmGENERIC_I2C_CONTROL_DEFAULT                                            0x00000000
-#define mmGENERIC_I2C_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmGENERIC_I2C_STATUS_DEFAULT                                             0x00000000
-#define mmGENERIC_I2C_SPEED_DEFAULT                                              0x00000002
-#define mmGENERIC_I2C_SETUP_DEFAULT                                              0x00000000
-#define mmGENERIC_I2C_TRANSACTION_DEFAULT                                        0x00000000
-#define mmGENERIC_I2C_DATA_DEFAULT                                               0x00000000
-#define mmGENERIC_I2C_PIN_SELECTION_DEFAULT                                      0x00000000
-#define mmDCO_SCRATCH0_DEFAULT                                                   0x00000000
-#define mmDCO_SCRATCH1_DEFAULT                                                   0x00000000
-#define mmDCO_SCRATCH2_DEFAULT                                                   0x00000000
-#define mmDCO_SCRATCH3_DEFAULT                                                   0x00000000
-#define mmDCO_SCRATCH4_DEFAULT                                                   0x00000000
-#define mmDCO_SCRATCH5_DEFAULT                                                   0x00000000
-#define mmDCO_SCRATCH6_DEFAULT                                                   0x00000000
-#define mmDCO_SCRATCH7_DEFAULT                                                   0x00000000
-#define mmDCE_VCE_CONTROL_DEFAULT                                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_DEFAULT                                          0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE_DEFAULT                                 0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE2_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE3_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE4_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE5_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE6_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE7_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE8_DEFAULT                                0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE9_DEFAULT                                0x00000000
-#define mmDCO_MEM_PWR_STATUS_DEFAULT                                             0x00000000
-#define mmDCO_MEM_PWR_CTRL_DEFAULT                                               0x6db6d800
-#define mmDCO_MEM_PWR_CTRL2_DEFAULT                                              0x001b0000
-#define mmDCO_CLK_CNTL_DEFAULT                                                   0x00000000
-#define mmDCO_POWER_MANAGEMENT_CNTL_DEFAULT                                      0x00000000
-#define mmDIG_SOFT_RESET_2_DEFAULT                                               0x00000000
-#define mmDCO_STEREOSYNC_SEL_DEFAULT                                             0x00000000
-#define mmDCO_SOFT_RESET_DEFAULT                                                 0x00000000
-#define mmDIG_SOFT_RESET_DEFAULT                                                 0x00000000
-#define mmDCO_MEM_PWR_STATUS1_DEFAULT                                            0x00000000
-#define mmDISP_INTERRUPT_STATUS_CONTINUE10_DEFAULT                               0x00000000
-#define mmDCO_CLK_CNTL2_DEFAULT                                                  0x00000000
-#define mmDCO_CLK_CNTL3_DEFAULT                                                  0x00000000
-#define mmDCO_HDMI_RXSTATUS_TIMER_CONTROL_DEFAULT                                0x00000000
-#define mmDCO_PSP_INTERRUPT_STATUS_DEFAULT                                       0x00000000
-#define mmDCO_PSP_INTERRUPT_CLEAR_DEFAULT                                        0x00000000
-#define mmDCO_GENERIC_INTERRUPT_MESSAGE_DEFAULT                                  0x00000000
-#define mmDCO_GENERIC_INTERRUPT_CLEAR_DEFAULT                                    0x00000000
-#define mmFMT_MEMORY0_CONTROL_DEFAULT                                            0x00000030
-#define mmFMT_MEMORY1_CONTROL_DEFAULT                                            0x00000031
-#define mmFMT_MEMORY2_CONTROL_DEFAULT                                            0x00000032
-#define mmFMT_MEMORY3_CONTROL_DEFAULT                                            0x00000033
-#define mmFMT_MEMORY4_CONTROL_DEFAULT                                            0x00000034
-#define mmFMT_MEMORY5_CONTROL_DEFAULT                                            0x00000035
-#define mmDISP_INTERRUPT_STATUS_CONTINUE11_DEFAULT                               0x00000000
-#define mmDC_GENERICA_DEFAULT                                                    0x00000000
-#define mmDC_GENERICB_DEFAULT                                                    0x00000000
-#define mmDC_PAD_EXTERN_SIG_DEFAULT                                              0x00000000
-#define mmDC_REF_CLK_CNTL_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DEBUG_DEFAULT                                                  0x00000101
-#define mmUNIPHYA_LINK_CNTL_DEFAULT                                              0x01100100
-#define mmUNIPHYA_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYB_LINK_CNTL_DEFAULT                                              0x01100100
-#define mmUNIPHYB_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYC_LINK_CNTL_DEFAULT                                              0x01100100
-#define mmUNIPHYC_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYD_LINK_CNTL_DEFAULT                                              0x01100100
-#define mmUNIPHYD_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYE_LINK_CNTL_DEFAULT                                              0x01100100
-#define mmUNIPHYE_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYF_LINK_CNTL_DEFAULT                                              0x01100100
-#define mmUNIPHYF_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmUNIPHYG_LINK_CNTL_DEFAULT                                              0x01100100
-#define mmUNIPHYG_CHANNEL_XBAR_CNTL_DEFAULT                                      0x03020100
-#define mmDCIO_WRCMD_DELAY_DEFAULT                                               0x00033333
-#define mmDC_DVODATA_CONFIG_DEFAULT                                              0x00000000
-#define mmLVTMA_PWRSEQ_CNTL_DEFAULT                                              0x00000000
-#define mmLVTMA_PWRSEQ_STATE_DEFAULT                                             0x00000000
-#define mmLVTMA_PWRSEQ_REF_DIV_DEFAULT                                           0x00010000
-#define mmLVTMA_PWRSEQ_DELAY1_DEFAULT                                            0x00000000
-#define mmLVTMA_PWRSEQ_DELAY2_DEFAULT                                            0x00000000
-#define mmBL_PWM_CNTL_DEFAULT                                                    0x00000000
-#define mmBL_PWM_CNTL2_DEFAULT                                                   0x00000000
-#define mmBL_PWM_PERIOD_CNTL_DEFAULT                                             0x00000001
-#define mmBL_PWM_GRP1_REG_LOCK_DEFAULT                                           0x00000000
-#define mmDCIO_GSL_GENLK_PAD_CNTL_DEFAULT                                        0x00000000
-#define mmDCIO_GSL_SWAPLOCK_PAD_CNTL_DEFAULT                                     0x00000000
-#define mmDCIO_GSL0_CNTL_DEFAULT                                                 0x00000000
-#define mmDCIO_GSL1_CNTL_DEFAULT                                                 0x00000000
-#define mmDCIO_GSL2_CNTL_DEFAULT                                                 0x00000000
-#define mmDC_GPU_TIMER_START_POSITION_V_UPDATE_DEFAULT                           0x00000000
-#define mmDC_GPU_TIMER_START_POSITION_P_FLIP_DEFAULT                             0x00000000
-#define mmDC_GPU_TIMER_READ_DEFAULT                                              0x00000000
-#define mmDC_GPU_TIMER_READ_CNTL_DEFAULT                                         0x00000000
-#define mmDCIO_CLOCK_CNTL_DEFAULT                                                0x00000000
-#define mmDCO_DCFE_EXT_VSYNC_CNTL_DEFAULT                                        0x00000000
-#define mmDCIO_SOFT_RESET_DEFAULT                                                0x00000000
-#define mmDCIO_DPHY_SEL_DEFAULT                                                  0x000000e4
-#define mmUNIPHY_IMPCAL_LINKA_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_LINKB_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_PERIOD_DEFAULT                                           0x00000000
-#define mmAUXP_IMPCAL_DEFAULT                                                    0x0a000000
-#define mmAUXN_IMPCAL_DEFAULT                                                    0x04000000
-#define mmDCIO_IMPCAL_CNTL_DEFAULT                                               0x00000000
-#define mmUNIPHY_IMPCAL_PSW_AB_DEFAULT                                           0x00000000
-#define mmUNIPHY_IMPCAL_LINKC_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_LINKD_DEFAULT                                            0x0f000000
-#define mmDCIO_IMPCAL_CNTL_CD_DEFAULT                                            0x00000000
-#define mmUNIPHY_IMPCAL_PSW_CD_DEFAULT                                           0x00000000
-#define mmUNIPHY_IMPCAL_LINKE_DEFAULT                                            0x0f000000
-#define mmUNIPHY_IMPCAL_LINKF_DEFAULT                                            0x0f000000
-#define mmDCIO_IMPCAL_CNTL_EF_DEFAULT                                            0x00000000
-#define mmUNIPHY_IMPCAL_PSW_EF_DEFAULT                                           0x00000000
-#define mmUNIPHYLPA_LINK_CNTL_DEFAULT                                            0x01100100
-#define mmUNIPHYLPB_LINK_CNTL_DEFAULT                                            0x01100100
-#define mmUNIPHYLPA_CHANNEL_XBAR_CNTL_DEFAULT                                    0x03020100
-#define mmUNIPHYLPB_CHANNEL_XBAR_CNTL_DEFAULT                                    0x03020100
-#define mmDCIO_DPCS_TX_INTERRUPT_DEFAULT                                         0x00000000
-#define mmDCIO_DPCS_RX_INTERRUPT_DEFAULT                                         0x00000000
-#define mmDCIO_SEMAPHORE0_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE1_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE2_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE3_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE4_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE5_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE6_DEFAULT                                                0x00000000
-#define mmDCIO_SEMAPHORE7_DEFAULT                                                0x00000000
-#define mmDC_GPIO_GENERIC_MASK_DEFAULT                                           0x04444444
-#define mmDC_GPIO_GENERIC_A_DEFAULT                                              0x00000000
-#define mmDC_GPIO_GENERIC_EN_DEFAULT                                             0x00000000
-#define mmDC_GPIO_GENERIC_Y_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DVODATA_MASK_DEFAULT                                           0x00000000
-#define mmDC_GPIO_DVODATA_A_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DVODATA_EN_DEFAULT                                             0x00000000
-#define mmDC_GPIO_DVODATA_Y_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DDC1_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC1_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC1_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC1_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC2_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC2_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC2_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC2_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC3_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC3_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC3_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC3_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC4_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC4_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC4_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC4_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC5_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC5_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC5_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC5_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC6_MASK_DEFAULT                                              0xcf400000
-#define mmDC_GPIO_DDC6_A_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDC6_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_DDC6_Y_DEFAULT                                                 0x00000000
-#define mmDC_GPIO_DDCVGA_MASK_DEFAULT                                            0xcf400000
-#define mmDC_GPIO_DDCVGA_A_DEFAULT                                               0x00000000
-#define mmDC_GPIO_DDCVGA_EN_DEFAULT                                              0x00000000
-#define mmDC_GPIO_DDCVGA_Y_DEFAULT                                               0x00000000
-#define mmDC_GPIO_SYNCA_MASK_DEFAULT                                             0x00004040
-#define mmDC_GPIO_SYNCA_A_DEFAULT                                                0x00000000
-#define mmDC_GPIO_SYNCA_EN_DEFAULT                                               0x00000000
-#define mmDC_GPIO_SYNCA_Y_DEFAULT                                                0x00000000
-#define mmDC_GPIO_GENLK_MASK_DEFAULT                                             0x10101a10
-#define mmDC_GPIO_GENLK_A_DEFAULT                                                0x00000000
-#define mmDC_GPIO_GENLK_EN_DEFAULT                                               0x00000000
-#define mmDC_GPIO_GENLK_Y_DEFAULT                                                0x00000000
-#define mmDC_GPIO_HPD_MASK_DEFAULT                                               0x44440440
-#define mmDC_GPIO_HPD_A_DEFAULT                                                  0x00000000
-#define mmDC_GPIO_HPD_EN_DEFAULT                                                 0x22220202
-#define mmDC_GPIO_HPD_Y_DEFAULT                                                  0x00000000
-#define mmDC_GPIO_PWRSEQ_MASK_DEFAULT                                            0x66404040
-#define mmDC_GPIO_PWRSEQ_A_DEFAULT                                               0x00000000
-#define mmDC_GPIO_PWRSEQ_EN_DEFAULT                                              0x00000000
-#define mmDC_GPIO_PWRSEQ_Y_DEFAULT                                               0x00000000
-#define mmDC_GPIO_PAD_STRENGTH_1_DEFAULT                                         0x47ac470f
-#define mmDC_GPIO_PAD_STRENGTH_2_DEFAULT                                         0x00472147
-#define mmPHY_AUX_CNTL_DEFAULT                                                   0x00010001
-#define mmDC_GPIO_I2CPAD_MASK_DEFAULT                                            0x00000000
-#define mmDC_GPIO_I2CPAD_A_DEFAULT                                               0x00000000
-#define mmDC_GPIO_I2CPAD_EN_DEFAULT                                              0x00000000
-#define mmDC_GPIO_I2CPAD_Y_DEFAULT                                               0x00000000
-#define mmDC_GPIO_I2CPAD_STRENGTH_DEFAULT                                        0x0000004c
-#define mmDVO_STRENGTH_CONTROL_DEFAULT                                           0x31116060
-#define mmDVO_VREF_CONTROL_DEFAULT                                               0x00000000
-#define mmDVO_SKEW_ADJUST_DEFAULT                                                0x00000000
-#define mmDC_GPIO_I2S_SPDIF_MASK_DEFAULT                                         0x00000000
-#define mmDC_GPIO_I2S_SPDIF_A_DEFAULT                                            0x00000000
-#define mmDC_GPIO_I2S_SPDIF_EN_DEFAULT                                           0x00008000
-#define mmDC_GPIO_I2S_SPDIF_Y_DEFAULT                                            0x00000000
-#define mmDC_GPIO_I2S_SPDIF_STRENGTH_DEFAULT                                     0x01021202
-#define mmDC_GPIO_TX12_EN_DEFAULT                                                0x00000000
-#define mmDC_GPIO_AUX_CTRL_0_DEFAULT                                             0x00000000
-#define mmDC_GPIO_AUX_CTRL_1_DEFAULT                                             0x00500000
-#define mmDC_GPIO_AUX_CTRL_2_DEFAULT                                             0x00000000
-#define mmDC_GPIO_RXEN_DEFAULT                                                   0x007fff7f
-#define mmBPHYC_DAC_MACRO_CNTL_DEFAULT                                           0x00202002
-#define mmDAC_MACRO_CNTL_RESERVED0_DEFAULT                                       0x00000000
-#define mmBPHYC_DAC_AUTO_CALIB_CONTROL_DEFAULT                                   0x00700255
-#define mmDAC_MACRO_CNTL_RESERVED1_DEFAULT                                       0x00000000
-#define mmDAC_MACRO_CNTL_RESERVED2_DEFAULT                                       0x00000000
-#define mmDAC_MACRO_CNTL_RESERVED3_DEFAULT                                       0x00000000
-#define mmDISP_DSI_DUAL_CTRL_DEFAULT                                             0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED0_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED1_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED2_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED3_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED4_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED5_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED6_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED7_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED8_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED9_DEFAULT                                      0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED10_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED11_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED12_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED13_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED14_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED15_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED16_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED17_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED18_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED19_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED20_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED21_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED22_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED23_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED24_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED25_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED26_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED27_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED28_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED29_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED30_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED31_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED32_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED33_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED34_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED35_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED36_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED37_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED38_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED39_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED40_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED41_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED42_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED43_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED44_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED45_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED46_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED47_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED48_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED49_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED50_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED51_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED52_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED53_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED54_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED55_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED56_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED57_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED58_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED59_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED60_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED61_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED62_DEFAULT                                     0x00000000
-#define mmDPHY_MACRO_CNTL_RESERVED63_DEFAULT                                     0x00000000
-#define mmDPRX_AUX_REFERENCE_PULSE_DIV_DEFAULT                                   0x0a640064
-#define mmDPRX_AUX_CONTROL_DEFAULT                                               0x01012c00
-#define mmDPRX_AUX_HPD_CONTROL1_DEFAULT                                          0x00001407
-#define mmDPRX_AUX_HPD_CONTROL2_DEFAULT                                          0x00000000
-#define mmDPRX_AUX_RX_STATUS_DEFAULT                                             0x00000000
-#define mmDPRX_AUX_RX_ERROR_MASK_DEFAULT                                         0x00000000
-#define mmDPRX_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                   0x00320000
-#define mmDPRX_AUX_DPHY_TX_CONTROL_DEFAULT                                       0x00001002
-#define mmDPRX_AUX_DPHY_RX_CONTROL0_DEFAULT                                      0x203d1210
-#define mmDPRX_AUX_DPHY_RX_CONTROL1_DEFAULT                                      0x0a00fa00
-#define mmDPRX_AUX_DPHY_TX_STATUS_DEFAULT                                        0x00000000
-#define mmDPRX_AUX_DPHY_RX_STATUS_DEFAULT                                        0x00000000
-#define mmDPRX_AUX_DMCU_HW_INT_STATUS_DEFAULT                                    0x00003f00
-#define mmDPRX_AUX_DMCU_HW_INT_ACK_DEFAULT                                       0x00000000
-#define mmDPRX_AUX_CPU_TO_DMCU_INTERRUPT1_DEFAULT                                0x00000000
-#define mmDPRX_AUX_CPU_TO_DMCU_INTERRUPT2_DEFAULT                                0x00000001
-#define mmDPRX_AUX_DMCU_TO_CPU_INTERRUPT1_DEFAULT                                0x00000000
-#define mmDPRX_AUX_DMCU_TO_CPU_INTERRUPT2_DEFAULT                                0x00000000
-#define mmDPRX_AUX_AUX_BUF_INDEX_DEFAULT                                         0x00000000
-#define mmDPRX_AUX_AUX_BUF_DATA_DEFAULT                                          0x00000000
-#define mmDPRX_AUX_EDID_INDEX_DEFAULT                                            0x00000000
-#define mmDPRX_AUX_EDID_DATA_DEFAULT                                             0x00000000
-#define mmDPRX_AUX_DPCD_INDEX1_DEFAULT                                           0x00000000
-#define mmDPRX_AUX_DPCD_DATA1_DEFAULT                                            0x00000000
-#define mmDPRX_AUX_DPCD_INDEX2_DEFAULT                                           0x00000000
-#define mmDPRX_AUX_DPCD_DATA2_DEFAULT                                            0x00000000
-#define mmDPRX_AUX_MSG_INDEX1_DEFAULT                                            0x00000000
-#define mmDPRX_AUX_MSG_DATA1_DEFAULT                                             0x00000000
-#define mmDPRX_AUX_MSG_INDEX2_DEFAULT                                            0x00000000
-#define mmDPRX_AUX_MSG_DATA2_DEFAULT                                             0x00000000
-#define mmDPRX_AUX_KSV_INDEX1_DEFAULT                                            0x00000000
-#define mmDPRX_AUX_KSV_DATA1_DEFAULT                                             0x00000000
-#define mmDPRX_AUX_KSV_INDEX2_DEFAULT                                            0x00000000
-#define mmDPRX_AUX_KSV_DATA2_DEFAULT                                             0x00000000
-#define mmDPRX_AUX_MSG_TIMEOUT_CONTROL_DEFAULT                                   0x00000032
-#define mmDPRX_AUX_MSG_BUF_CONTROL1_DEFAULT                                      0x00000000
-#define mmDPRX_AUX_MSG_BUF_CONTROL2_DEFAULT                                      0x00000000
-#define mmDPRX_AUX_SCRATCH1_DEFAULT                                              0x00000000
-#define mmDPRX_AUX_SCRATCH2_DEFAULT                                              0x00000000
-#define mmDPRX_AUX_MSG1_PENDING_DEFAULT                                          0x00000000
-#define mmDPRX_AUX_MSG2_PENDING_DEFAULT                                          0x00000000
-#define mmDPRX_AUX_MSG3_PENDING_DEFAULT                                          0x00000000
-#define mmDPRX_AUX_MSG4_PENDING_DEFAULT                                          0x00000000
-#define mmDPRX_DPHY_DPCD_LANE_COUNT_SET_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_DPCD_TRAINING_PATTERN_SET_DEFAULT                            0x00000003
-#define mmDPRX_DPHY_DPCD_MSTM_CTRL_DEFAULT                                       0x00000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE0_SET_DEFAULT                             0x00000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE0_STATUS_DEFAULT                          0x20000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE1_SET_DEFAULT                             0x00000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE1_STATUS_DEFAULT                          0x20000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE2_SET_DEFAULT                             0x00000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE2_STATUS_DEFAULT                          0x20000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE3_SET_DEFAULT                             0x00000000
-#define mmDPRX_DPHY_DPCD_LINK_QUAL_LANE3_STATUS_DEFAULT                          0x20000000
-#define mmDPRX_DPHY_READY_DEFAULT                                                0x00000000
-#define mmDPRX_DPHY_COMMA_STATUS_DEFAULT                                         0x00000000
-#define mmDPRX_DPHY_LANE_ALIGN_ERROR_STATUS_UPDATED_DEFAULT                      0x00000000
-#define mmDPRX_DPHY_LANE_ALIGN_STATUS_UPDATED_DEFAULT                            0x00000000
-#define mmDPRX_DPHY_ERROR_THRESH_A_LANE0_DEFAULT                                 0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_A_LANE0_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_B_LANE0_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_C_LANE0_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_THRESH_A_LANE1_DEFAULT                                 0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_A_LANE1_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_B_LANE1_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_C_LANE1_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_THRESH_A_LANE2_DEFAULT                                 0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_A_LANE2_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_B_LANE2_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_C_LANE2_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_THRESH_A_LANE3_DEFAULT                                 0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_A_LANE3_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_B_LANE3_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_ERROR_COUNT_C_LANE3_DEFAULT                                  0x00000000
-#define mmDPRX_DPHY_BS_ERROR_THRESH_GLOBAL_DEFAULT                               0x00000000
-#define mmDPRX_DPHY_SR_ERROR_COUNT_A_DEFAULT                                     0x00000000
-#define mmDPRX_DPHY_BS_ERROR_COUNT_A_DEFAULT                                     0x00000000
-#define mmDPRX_DPHY_BS_ERROR_COUNT_B_DEFAULT                                     0x00000000
-#define mmDPRX_DPHY_LANESETUP0_DEFAULT                                           0x00000000
-#define mmDPRX_DPHY_LANESETUP1_DEFAULT                                           0x00000000
-#define mmDPRX_DPHY_LFSRADV_DEFAULT                                              0x00000039
-#define mmDPRX_DPHY_SEVENSYMBOLWINDOW_ERROR_DETECT_DEFAULT                       0x00000000
-#define mmDPRX_DPHY_SET_ENABLE_DEFAULT                                           0x00000000
-#define mmDPRX_DPHY_ECF_LSB_DEFAULT                                              0x00000000
-#define mmDPRX_DPHY_ECF_MSB_DEFAULT                                              0x00000000
-#define mmDPRX_DPHY_ENHANCED_FRAME_EN_DEFAULT                                    0x00000001
-#define mmDPRX_DPHY_MTP_HEADER_COUNT_FORCE_DEFAULT                               0x000a6800
-#define mmDPRX_DPHY_DYNAMIC_DESKEW_DATA_DEFAULT                                  0xbcbcbcbc
-#define mmDPRX_DPHY_DYNAMIC_DESKEW_CONTROL_DEFAULT                               0x800071c5
-#define mmDPRX_DPHY_BYPASS_DEFAULT                                               0x00000000
-#define mmDPRX_DPHY_INT_RESET_DEFAULT                                            0x00000000
-#define mmDPRX_DPHY_BS_INTERVAL_ERROR_THRESH_EXCEEDED_STATUS_DEFAULT             0x00000000
-#define mmDPRX_DPHY_SYMBOL_ERROR_THRESH_EXCEEDED_STATUS_DEFAULT                  0x00000000
-#define mmDPRX_DPHY_DISPARITY_ERROR_THRESH_EXCEEDED_STATUS_DEFAULT               0x00000000
-#define mmDPRX_DPHY_TEST_PATTERN_ERROR_THRESH_EXCEEDED_STATUS_DEFAULT            0x00000000
-#define mmDPRX_DPHY_DETECT_SR_LOCK_STATUS_DEFAULT                                0x00000000
-#define mmDPRX_DPHY_LOSS_OF_ALIGN_STATUS_DEFAULT                                 0x00000000
-#define mmDPRX_DPHY_LOSS_OF_DESKEW_STATUS_DEFAULT                                0x00000000
-#define mmDPRX_DPHY_EXCESSIVE_ERROR_STATUS_DEFAULT                               0x00000000
-#define mmDPRX_DPHY_DESKEW_FIFO_OVERFLOW_STATUS_DEFAULT                          0x00000000
-#define mmDPRX_DPHY_SPARE_DEFAULT                                                0x00000000
-#define mmDCRX_GATE_DISABLE_CNTL_DEFAULT                                         0x00001f0f
-#define mmDCRX_SOFT_RESET_DEFAULT                                                0x00000000
-#define mmDCRX_LIGHT_SLEEP_CNTL_DEFAULT                                          0x00000101
-#define mmDCRX_DISPCLK_GATE_CNTL_DEFAULT                                         0x00000200
-#define mmDCRX_CLK_CNTL_DEFAULT                                                  0x00000000
-#define mmDCRX_TEST_CLK_CNTL_DEFAULT                                             0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED0_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED1_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED2_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED3_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED4_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED5_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED6_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED7_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED8_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED9_DEFAULT                                  0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED10_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED11_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED12_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED13_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED14_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED15_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED16_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED17_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED18_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED19_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED20_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED21_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED22_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED23_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED24_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED25_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED26_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED27_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED28_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED29_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED30_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED31_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED32_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED33_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED34_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED35_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED36_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED37_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED38_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED39_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED40_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED41_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED42_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED43_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED44_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED45_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED46_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED47_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED48_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED49_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED50_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED51_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED52_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED53_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED54_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED55_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED56_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED57_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED58_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED59_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED60_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED61_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED62_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED63_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED64_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED65_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED66_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED67_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED68_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED69_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED70_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED71_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED72_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED73_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED74_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED75_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED76_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED77_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED78_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED79_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED80_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED81_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED82_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED83_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED84_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED85_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED86_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED87_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED88_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED89_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED90_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED91_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED92_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED93_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED94_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED95_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED96_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED97_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED98_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED99_DEFAULT                                 0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED100_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED101_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED102_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED103_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED104_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED105_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED106_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED107_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED108_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED109_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED110_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED111_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED112_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED113_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED114_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED115_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED116_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED117_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED118_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED119_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED120_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED121_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED122_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED123_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED124_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED125_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED126_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED127_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED128_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED129_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED130_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED131_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED132_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED133_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED134_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED135_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED136_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED137_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED138_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED139_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED140_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED141_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED142_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED143_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED144_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED145_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED146_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED147_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED148_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED149_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED150_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED151_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED152_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED153_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED154_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED155_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED156_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED157_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED158_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED159_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED160_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED161_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED162_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED163_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED164_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED165_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED166_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED167_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED168_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED169_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED170_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED171_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED172_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED173_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED174_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED175_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED176_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED177_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED178_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED179_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED180_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED181_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED182_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED183_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED184_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED185_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED186_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED187_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED188_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED189_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED190_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED191_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED192_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED193_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED194_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED195_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED196_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED197_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED198_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED199_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED200_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED201_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED202_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED203_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED204_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED205_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED206_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED207_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED208_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED209_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED210_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED211_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED212_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED213_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED214_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED215_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED216_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED217_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED218_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED219_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED220_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED221_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED222_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED223_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED224_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED225_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED226_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED227_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED228_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED229_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED230_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED231_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED232_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED233_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED234_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED235_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED236_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED237_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED238_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED239_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED240_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED241_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED242_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED243_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED244_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED245_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED246_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED247_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED248_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED249_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED250_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED251_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED252_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED253_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED254_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED255_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED256_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED257_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED258_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED259_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED260_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED261_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED262_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED263_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED264_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED265_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED266_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED267_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED268_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED269_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED270_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED271_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED272_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED273_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED274_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED275_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED276_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED277_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED278_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED279_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED280_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED281_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED282_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED283_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED284_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED285_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED286_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED287_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED288_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED289_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED290_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED291_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED292_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED293_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED294_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED295_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED296_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED297_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED298_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED299_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED300_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED301_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED302_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED303_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED304_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED305_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED306_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED307_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED308_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED309_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED310_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED311_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED312_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED313_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED314_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED315_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED316_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED317_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED318_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED319_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED320_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED321_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED322_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED323_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED324_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED325_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED326_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED327_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED328_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED329_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED330_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED331_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED332_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED333_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED334_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED335_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED336_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED337_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED338_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED339_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED340_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED341_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED342_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED343_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED344_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED345_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED346_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED347_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED348_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED349_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED350_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED351_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED352_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED353_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED354_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED355_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED356_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED357_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED358_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED359_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED360_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED361_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED362_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED363_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED364_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED365_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED366_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED367_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED368_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED369_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED370_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED371_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED372_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED373_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED374_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED375_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED376_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED377_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED378_DEFAULT                                0x00000000
-#define mmDCRX_PHY_MACRO_CNTL_RESERVED379_DEFAULT                                0x00000000
-#define mmI2S0_CNTL_DEFAULT                                                      0x00010000
-#define mmSPDIF0_CNTL_DEFAULT                                                    0x00000000
-#define mmI2S1_CNTL_DEFAULT                                                      0x00010000
-#define mmSPDIF1_CNTL_DEFAULT                                                    0x00000000
-#define mmI2S0_STATUS_DEFAULT                                                    0x00000000
-#define mmI2S1_STATUS_DEFAULT                                                    0x00000000
-#define mmI2S0_CRC_TEST_CNTL_DEFAULT                                             0x00000100
-#define mmI2S0_CRC_TEST_DATA_01_DEFAULT                                          0x00000000
-#define mmI2S0_CRC_TEST_DATA_23_DEFAULT                                          0x00000000
-#define mmI2S1_CRC_TEST_CNTL_DEFAULT                                             0x00000100
-#define mmI2S1_CRC_TEST_DATA_0_DEFAULT                                           0x00000000
-#define mmSPDIF0_CRC_TEST_CNTL_DEFAULT                                           0x00000100
-#define mmSPDIF0_CRC_TEST_DATA_0_DEFAULT                                         0x00000000
-#define mmSPDIF1_CRC_TEST_CNTL_DEFAULT                                           0x00000100
-#define mmSPDIF1_CRC_TEST_DATA_DEFAULT                                           0x00000000
-#define mmCRC_I2S_CONT_REPEAT_NUM_DEFAULT                                        0x00000000
-#define mmCRC_SPDIF_CONT_REPEAT_NUM_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED0_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED1_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED2_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED3_DEFAULT                                      0x00000000
-#define mmZCAL_MACRO_CNTL_RESERVED4_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_azf0stream0_dispdec
-#define mmAZF0STREAM0_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM0_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream1_dispdec
-#define mmAZF0STREAM1_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM1_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream2_dispdec
-#define mmAZF0STREAM2_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM2_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream3_dispdec
-#define mmAZF0STREAM3_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM3_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream4_dispdec
-#define mmAZF0STREAM4_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM4_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream5_dispdec
-#define mmAZF0STREAM5_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM5_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream6_dispdec
-#define mmAZF0STREAM6_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM6_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream7_dispdec
-#define mmAZF0STREAM7_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM7_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint0_dispdec
-#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint1_dispdec
-#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT1_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint2_dispdec
-#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT2_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint3_dispdec
-#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT3_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint4_dispdec
-#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT4_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint5_dispdec
-#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT5_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint6_dispdec
-#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT6_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0endpoint7_dispdec
-#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_INDEX_DEFAULT                   0x00000000
-#define mmAZF0ENDPOINT7_AZALIA_F0_CODEC_ENDPOINT_DATA_DEFAULT                    0x00000000
-
-
-// addressBlock: dce_dc_azf0stream8_dispdec
-#define mmAZF0STREAM8_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM8_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream9_dispdec
-#define mmAZF0STREAM9_AZALIA_STREAM_INDEX_DEFAULT                                0x00000000
-#define mmAZF0STREAM9_AZALIA_STREAM_DATA_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_azf0stream10_dispdec
-#define mmAZF0STREAM10_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM10_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_azf0stream11_dispdec
-#define mmAZF0STREAM11_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM11_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_azf0stream12_dispdec
-#define mmAZF0STREAM12_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM12_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_azf0stream13_dispdec
-#define mmAZF0STREAM13_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM13_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_azf0stream14_dispdec
-#define mmAZF0STREAM14_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM14_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_azf0stream15_dispdec
-#define mmAZF0STREAM15_AZALIA_STREAM_INDEX_DEFAULT                               0x00000000
-#define mmAZF0STREAM15_AZALIA_STREAM_DATA_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint0_dispdec
-#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint1_dispdec
-#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint2_dispdec
-#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint3_dispdec
-#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint4_dispdec
-#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint5_dispdec
-#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint6_dispdec
-#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_azf0inputendpoint7_dispdec
-#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_INDEX_DEFAULT        0x00000000
-#define mmAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_ENDPOINT_DATA_DEFAULT         0x00000000
-
-
-// addressBlock: dce_dc_dcp0_dispdec
-#define mmDCP0_GRPH_ENABLE_DEFAULT                                               0x00000001
-#define mmDCP0_GRPH_CONTROL_DEFAULT                                              0x20002040
-#define mmDCP0_GRPH_LUT_10BIT_BYPASS_DEFAULT                                     0x00000000
-#define mmDCP0_GRPH_SWAP_CNTL_DEFAULT                                            0x00000000
-#define mmDCP0_GRPH_PRIMARY_SURFACE_ADDRESS_DEFAULT                              0x00000000
-#define mmDCP0_GRPH_SECONDARY_SURFACE_ADDRESS_DEFAULT                            0x00000000
-#define mmDCP0_GRPH_PITCH_DEFAULT                                                0x00000000
-#define mmDCP0_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                         0x00000000
-#define mmDCP0_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                       0x00000000
-#define mmDCP0_GRPH_SURFACE_OFFSET_X_DEFAULT                                     0x00000000
-#define mmDCP0_GRPH_SURFACE_OFFSET_Y_DEFAULT                                     0x00000000
-#define mmDCP0_GRPH_X_START_DEFAULT                                              0x00000000
-#define mmDCP0_GRPH_Y_START_DEFAULT                                              0x00000000
-#define mmDCP0_GRPH_X_END_DEFAULT                                                0x00000000
-#define mmDCP0_GRPH_Y_END_DEFAULT                                                0x00000000
-#define mmDCP0_INPUT_GAMMA_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP0_GRPH_UPDATE_DEFAULT                                               0x00000000
-#define mmDCP0_GRPH_FLIP_CONTROL_DEFAULT                                         0x00000020
-#define mmDCP0_GRPH_SURFACE_ADDRESS_INUSE_DEFAULT                                0x00000000
-#define mmDCP0_GRPH_DFQ_CONTROL_DEFAULT                                          0x00000000
-#define mmDCP0_GRPH_DFQ_STATUS_DEFAULT                                           0x00000000
-#define mmDCP0_GRPH_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDCP0_GRPH_INTERRUPT_CONTROL_DEFAULT                                    0x00000000
-#define mmDCP0_GRPH_SURFACE_ADDRESS_HIGH_INUSE_DEFAULT                           0x00000000
-#define mmDCP0_GRPH_COMPRESS_SURFACE_ADDRESS_DEFAULT                             0x00000000
-#define mmDCP0_GRPH_COMPRESS_PITCH_DEFAULT                                       0x00000000
-#define mmDCP0_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH_DEFAULT                        0x00000000
-#define mmDCP0_GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                       0x000000ff
-#define mmDCP0_PRESCALE_GRPH_CONTROL_DEFAULT                                     0x00000010
-#define mmDCP0_PRESCALE_VALUES_GRPH_R_DEFAULT                                    0x20000000
-#define mmDCP0_PRESCALE_VALUES_GRPH_G_DEFAULT                                    0x20000000
-#define mmDCP0_PRESCALE_VALUES_GRPH_B_DEFAULT                                    0x20000000
-#define mmDCP0_INPUT_CSC_CONTROL_DEFAULT                                         0x00000000
-#define mmDCP0_INPUT_CSC_C11_C12_DEFAULT                                         0x00002000
-#define mmDCP0_INPUT_CSC_C13_C14_DEFAULT                                         0x00000000
-#define mmDCP0_INPUT_CSC_C21_C22_DEFAULT                                         0x20000000
-#define mmDCP0_INPUT_CSC_C23_C24_DEFAULT                                         0x00000000
-#define mmDCP0_INPUT_CSC_C31_C32_DEFAULT                                         0x00000000
-#define mmDCP0_INPUT_CSC_C33_C34_DEFAULT                                         0x00002000
-#define mmDCP0_OUTPUT_CSC_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP0_OUTPUT_CSC_C11_C12_DEFAULT                                        0x00002000
-#define mmDCP0_OUTPUT_CSC_C13_C14_DEFAULT                                        0x00000000
-#define mmDCP0_OUTPUT_CSC_C21_C22_DEFAULT                                        0x20000000
-#define mmDCP0_OUTPUT_CSC_C23_C24_DEFAULT                                        0x00000000
-#define mmDCP0_OUTPUT_CSC_C31_C32_DEFAULT                                        0x00000000
-#define mmDCP0_OUTPUT_CSC_C33_C34_DEFAULT                                        0x00002000
-#define mmDCP0_COMM_MATRIXA_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP0_COMM_MATRIXA_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP0_COMM_MATRIXA_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP0_COMM_MATRIXA_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP0_COMM_MATRIXA_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP0_COMM_MATRIXA_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP0_COMM_MATRIXB_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP0_COMM_MATRIXB_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP0_COMM_MATRIXB_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP0_COMM_MATRIXB_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP0_COMM_MATRIXB_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP0_COMM_MATRIXB_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP0_DENORM_CONTROL_DEFAULT                                            0x00000003
-#define mmDCP0_OUT_ROUND_CONTROL_DEFAULT                                         0x0000000a
-#define mmDCP0_OUT_CLAMP_CONTROL_R_CR_DEFAULT                                    0x00003fff
-#define mmDCP0_OUT_CLAMP_CONTROL_G_Y_DEFAULT                                     0x00003fff
-#define mmDCP0_OUT_CLAMP_CONTROL_B_CB_DEFAULT                                    0x00003fff
-#define mmDCP0_KEY_CONTROL_DEFAULT                                               0x00000000
-#define mmDCP0_KEY_RANGE_ALPHA_DEFAULT                                           0x00000000
-#define mmDCP0_KEY_RANGE_RED_DEFAULT                                             0x00000000
-#define mmDCP0_KEY_RANGE_GREEN_DEFAULT                                           0x00000000
-#define mmDCP0_KEY_RANGE_BLUE_DEFAULT                                            0x00000000
-#define mmDCP0_DEGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP0_GAMUT_REMAP_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP0_GAMUT_REMAP_C11_C12_DEFAULT                                       0x00002000
-#define mmDCP0_GAMUT_REMAP_C13_C14_DEFAULT                                       0x00000000
-#define mmDCP0_GAMUT_REMAP_C21_C22_DEFAULT                                       0x20000000
-#define mmDCP0_GAMUT_REMAP_C23_C24_DEFAULT                                       0x00000000
-#define mmDCP0_GAMUT_REMAP_C31_C32_DEFAULT                                       0x00000000
-#define mmDCP0_GAMUT_REMAP_C33_C34_DEFAULT                                       0x00002000
-#define mmDCP0_DCP_SPATIAL_DITHER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP0_DCP_RANDOM_SEEDS_DEFAULT                                          0x00000000
-#define mmDCP0_DCP_FP_CONVERTED_FIELD_DEFAULT                                    0x00000000
-#define mmDCP0_CUR_CONTROL_DEFAULT                                               0x00000810
-#define mmDCP0_CUR_SURFACE_ADDRESS_DEFAULT                                       0x00000000
-#define mmDCP0_CUR_SIZE_DEFAULT                                                  0x00000000
-#define mmDCP0_CUR_SURFACE_ADDRESS_HIGH_DEFAULT                                  0x00000000
-#define mmDCP0_CUR_POSITION_DEFAULT                                              0x00000000
-#define mmDCP0_CUR_HOT_SPOT_DEFAULT                                              0x00000000
-#define mmDCP0_CUR_COLOR1_DEFAULT                                                0x00000000
-#define mmDCP0_CUR_COLOR2_DEFAULT                                                0x00000000
-#define mmDCP0_CUR_UPDATE_DEFAULT                                                0x00000000
-#define mmDCP0_CUR_REQUEST_FILTER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP0_CUR_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP0_DC_LUT_RW_MODE_DEFAULT                                            0x00000000
-#define mmDCP0_DC_LUT_RW_INDEX_DEFAULT                                           0x00000000
-#define mmDCP0_DC_LUT_SEQ_COLOR_DEFAULT                                          0x00000000
-#define mmDCP0_DC_LUT_PWL_DATA_DEFAULT                                           0x00000000
-#define mmDCP0_DC_LUT_30_COLOR_DEFAULT                                           0x00000000
-#define mmDCP0_DC_LUT_VGA_ACCESS_ENABLE_DEFAULT                                  0x00000000
-#define mmDCP0_DC_LUT_WRITE_EN_MASK_DEFAULT                                      0x00000007
-#define mmDCP0_DC_LUT_AUTOFILL_DEFAULT                                           0x00000000
-#define mmDCP0_DC_LUT_CONTROL_DEFAULT                                            0x00000000
-#define mmDCP0_DC_LUT_BLACK_OFFSET_BLUE_DEFAULT                                  0x00000000
-#define mmDCP0_DC_LUT_BLACK_OFFSET_GREEN_DEFAULT                                 0x00000000
-#define mmDCP0_DC_LUT_BLACK_OFFSET_RED_DEFAULT                                   0x00000000
-#define mmDCP0_DC_LUT_WHITE_OFFSET_BLUE_DEFAULT                                  0x0000ffff
-#define mmDCP0_DC_LUT_WHITE_OFFSET_GREEN_DEFAULT                                 0x0000ffff
-#define mmDCP0_DC_LUT_WHITE_OFFSET_RED_DEFAULT                                   0x0000ffff
-#define mmDCP0_DCP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP0_DCP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmDCP0_DCP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmDCP0_DVMM_PTE_CONTROL_DEFAULT                                          0x00004000
-#define mmDCP0_DCP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmDCP0_DVMM_PTE_ARB_CONTROL_DEFAULT                                      0x00002220
-#define mmDCP0_GRPH_FLIP_RATE_CNTL_DEFAULT                                       0x00000000
-#define mmDCP0_DCP_GSL_CONTROL_DEFAULT                                           0x60000020
-#define mmDCP0_DCP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000035
-#define mmDCP0_GRPH_STEREOSYNC_FLIP_DEFAULT                                      0x00000200
-#define mmDCP0_HW_ROTATION_DEFAULT                                               0x00000000
-#define mmDCP0_GRPH_XDMA_CACHE_UNDERFLOW_DET_CNTL_DEFAULT                        0x00000010
-#define mmDCP0_REGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP0_REGAMMA_LUT_INDEX_DEFAULT                                         0x00000000
-#define mmDCP0_REGAMMA_LUT_DATA_DEFAULT                                          0x00000000
-#define mmDCP0_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                                 0x00000007
-#define mmDCP0_REGAMMA_CNTLA_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLA_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP0_REGAMMA_CNTLA_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP0_REGAMMA_CNTLA_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP0_REGAMMA_CNTLB_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLB_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP0_REGAMMA_CNTLB_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP0_REGAMMA_CNTLB_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP0_ALPHA_CONTROL_DEFAULT                                             0x00000002
-#define mmDCP0_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmDCP0_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmDCP0_GRPH_XDMA_CACHE_UNDERFLOW_DET_STATUS_DEFAULT                      0x00000000
-#define mmDCP0_GRPH_XDMA_FLIP_TIMEOUT_DEFAULT                                    0x00000000
-#define mmDCP0_GRPH_XDMA_FLIP_AVG_DELAY_DEFAULT                                  0x00000000
-#define mmDCP0_GRPH_SURFACE_COUNTER_CONTROL_DEFAULT                              0x00000012
-#define mmDCP0_GRPH_SURFACE_COUNTER_OUTPUT_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_lb0_dispdec
-#define mmLB0_LB_DATA_FORMAT_DEFAULT                                             0x00000000
-#define mmLB0_LB_MEMORY_CTRL_DEFAULT                                             0x000006b0
-#define mmLB0_LB_MEMORY_SIZE_STATUS_DEFAULT                                      0x00000000
-#define mmLB0_LB_DESKTOP_HEIGHT_DEFAULT                                          0x00000000
-#define mmLB0_LB_VLINE_START_END_DEFAULT                                         0x00000000
-#define mmLB0_LB_VLINE2_START_END_DEFAULT                                        0x00000000
-#define mmLB0_LB_V_COUNTER_DEFAULT                                               0x00000000
-#define mmLB0_LB_SNAPSHOT_V_COUNTER_DEFAULT                                      0x00000000
-#define mmLB0_LB_INTERRUPT_MASK_DEFAULT                                          0x00000000
-#define mmLB0_LB_VLINE_STATUS_DEFAULT                                            0x00000000
-#define mmLB0_LB_VLINE2_STATUS_DEFAULT                                           0x00000000
-#define mmLB0_LB_VBLANK_STATUS_DEFAULT                                           0x00000000
-#define mmLB0_LB_SYNC_RESET_SEL_DEFAULT                                          0x00000002
-#define mmLB0_LB_BLACK_KEYER_R_CR_DEFAULT                                        0x00000000
-#define mmLB0_LB_BLACK_KEYER_G_Y_DEFAULT                                         0x00000000
-#define mmLB0_LB_BLACK_KEYER_B_CB_DEFAULT                                        0x00000000
-#define mmLB0_LB_KEYER_COLOR_CTRL_DEFAULT                                        0x00000000
-#define mmLB0_LB_KEYER_COLOR_R_CR_DEFAULT                                        0x00000000
-#define mmLB0_LB_KEYER_COLOR_G_Y_DEFAULT                                         0x00000000
-#define mmLB0_LB_KEYER_COLOR_B_CB_DEFAULT                                        0x00000000
-#define mmLB0_LB_KEYER_COLOR_REP_R_CR_DEFAULT                                    0x00000000
-#define mmLB0_LB_KEYER_COLOR_REP_G_Y_DEFAULT                                     0x00000000
-#define mmLB0_LB_KEYER_COLOR_REP_B_CB_DEFAULT                                    0x00000000
-#define mmLB0_LB_BUFFER_LEVEL_STATUS_DEFAULT                                     0xa0008000
-#define mmLB0_LB_BUFFER_URGENCY_CTRL_DEFAULT                                     0x00200010
-#define mmLB0_LB_BUFFER_URGENCY_STATUS_DEFAULT                                   0x00000000
-#define mmLB0_LB_BUFFER_STATUS_DEFAULT                                           0x00000002
-#define mmLB0_LB_NO_OUTSTANDING_REQ_STATUS_DEFAULT                               0x00000000
-#define mmLB0_MVP_AFR_FLIP_MODE_DEFAULT                                          0x00000000
-#define mmLB0_MVP_AFR_FLIP_FIFO_CNTL_DEFAULT                                     0x00000000
-#define mmLB0_MVP_FLIP_LINE_NUM_INSERT_DEFAULT                                   0x00000002
-#define mmLB0_DC_MVP_LB_CONTROL_DEFAULT                                          0x00000001
-
-
-// addressBlock: dce_dc_dcfe0_dispdec
-#define mmDCFE0_DCFE_CLOCK_CONTROL_DEFAULT                                       0x00000000
-#define mmDCFE0_DCFE_SOFT_RESET_DEFAULT                                          0x00000000
-#define mmDCFE0_DCFE_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDCFE0_DCFE_MEM_PWR_CTRL2_DEFAULT                                       0x00000000
-#define mmDCFE0_DCFE_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDCFE0_DCFE_MISC_DEFAULT                                                0x00000001
-#define mmDCFE0_DCFE_FLUSH_DEFAULT                                               0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon3_dispdec
-#define mmDC_PERFMON3_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON3_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON3_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON3_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON3_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON3_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON3_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON3_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON3_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dmif_pg0_dispdec
-#define mmDMIF_PG0_DPG_PIPE_ARBITRATION_CONTROL1_DEFAULT                         0x00000000
-#define mmDMIF_PG0_DPG_PIPE_ARBITRATION_CONTROL2_DEFAULT                         0x00000000
-#define mmDMIF_PG0_DPG_WATERMARK_MASK_CONTROL_DEFAULT                            0x000bf777
-#define mmDMIF_PG0_DPG_PIPE_URGENCY_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG0_DPG_PIPE_URGENT_LEVEL_CONTROL_DEFAULT                         0x00000000
-#define mmDMIF_PG0_DPG_PIPE_STUTTER_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG0_DPG_PIPE_STUTTER_CONTROL2_DEFAULT                             0x00000000
-#define mmDMIF_PG0_DPG_PIPE_LOW_POWER_CONTROL_DEFAULT                            0x00000000
-#define mmDMIF_PG0_DPG_REPEATER_PROGRAM_DEFAULT                                  0x00000000
-#define mmDMIF_PG0_DPG_CHK_PRE_PROC_CNTL_DEFAULT                                 0x00000000
-#define mmDMIF_PG0_DPG_DVMM_STATUS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_scl0_dispdec
-#define mmSCL0_SCL_COEF_RAM_SELECT_DEFAULT                                       0x00000000
-#define mmSCL0_SCL_COEF_RAM_TAP_DATA_DEFAULT                                     0x00000000
-#define mmSCL0_SCL_MODE_DEFAULT                                                  0x00000000
-#define mmSCL0_SCL_TAP_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL0_SCL_CONTROL_DEFAULT                                               0x00000000
-#define mmSCL0_SCL_BYPASS_CONTROL_DEFAULT                                        0x00000000
-#define mmSCL0_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                              0x00000000
-#define mmSCL0_SCL_AUTOMATIC_MODE_CONTROL_DEFAULT                                0x00000000
-#define mmSCL0_SCL_HORZ_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL0_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL0_SCL_HORZ_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL0_SCL_VERT_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL0_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL0_SCL_VERT_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL0_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                  0x01000000
-#define mmSCL0_SCL_ROUND_OFFSET_DEFAULT                                          0x80000000
-#define mmSCL0_SCL_UPDATE_DEFAULT                                                0x00000000
-#define mmSCL0_SCL_F_SHARP_CONTROL_DEFAULT                                       0x00000000
-#define mmSCL0_SCL_ALU_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL0_SCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                              0x00000000
-#define mmSCL0_VIEWPORT_START_SECONDARY_DEFAULT                                  0x00000000
-#define mmSCL0_VIEWPORT_START_DEFAULT                                            0x00000000
-#define mmSCL0_VIEWPORT_SIZE_DEFAULT                                             0x00000000
-#define mmSCL0_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                                   0x00000000
-#define mmSCL0_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                                   0x00000000
-#define mmSCL0_SCL_MODE_CHANGE_DET1_DEFAULT                                      0x00000000
-#define mmSCL0_SCL_MODE_CHANGE_DET2_DEFAULT                                      0x00000000
-#define mmSCL0_SCL_MODE_CHANGE_DET3_DEFAULT                                      0x00000000
-#define mmSCL0_SCL_MODE_CHANGE_MASK_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_blnd0_dispdec
-#define mmBLND0_BLND_CONTROL_DEFAULT                                             0xff0220ff
-#define mmBLND0_BLND_SM_CONTROL2_DEFAULT                                         0x00000000
-#define mmBLND0_BLND_CONTROL2_DEFAULT                                            0x00000010
-#define mmBLND0_BLND_UPDATE_DEFAULT                                              0x00000000
-#define mmBLND0_BLND_UNDERFLOW_INTERRUPT_DEFAULT                                 0x00000000
-#define mmBLND0_BLND_V_UPDATE_LOCK_DEFAULT                                       0x80000000
-#define mmBLND0_BLND_REG_UPDATE_STATUS_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_crtc0_dispdec
-#define mmCRTC0_CRTC_H_BLANK_EARLY_NUM_DEFAULT                                   0x00000040
-#define mmCRTC0_CRTC_H_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC0_CRTC_H_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_H_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC0_CRTC_H_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_H_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC0_CRTC_H_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_VBI_END_DEFAULT                                             0x00000003
-#define mmCRTC0_CRTC_V_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC0_CRTC_V_TOTAL_MIN_DEFAULT                                         0x00000000
-#define mmCRTC0_CRTC_V_TOTAL_MAX_DEFAULT                                         0x00000000
-#define mmCRTC0_CRTC_V_TOTAL_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTC0_CRTC_V_TOTAL_INT_STATUS_DEFAULT                                  0x00000000
-#define mmCRTC0_CRTC_VSYNC_NOM_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTC0_CRTC_V_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_V_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC0_CRTC_V_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_V_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC0_CRTC_V_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_DTMTEST_CNTL_DEFAULT                                        0x00000000
-#define mmCRTC0_CRTC_DTMTEST_STATUS_POSITION_DEFAULT                             0x00000000
-#define mmCRTC0_CRTC_TRIGA_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC0_CRTC_TRIGA_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_TRIGB_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC0_CRTC_TRIGB_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_FORCE_COUNT_NOW_CNTL_DEFAULT                                0x00000000
-#define mmCRTC0_CRTC_FLOW_CONTROL_DEFAULT                                        0x00000000
-#define mmCRTC0_CRTC_STEREO_FORCE_NEXT_EYE_DEFAULT                               0x00000000
-#define mmCRTC0_CRTC_AVSYNC_COUNTER_DEFAULT                                      0x00000000
-#define mmCRTC0_CRTC_CONTROL_DEFAULT                                             0x80400110
-#define mmCRTC0_CRTC_BLANK_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_INTERLACE_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_INTERLACE_STATUS_DEFAULT                                    0x00000000
-#define mmCRTC0_CRTC_FIELD_INDICATION_CONTROL_DEFAULT                            0x00000000
-#define mmCRTC0_CRTC_PIXEL_DATA_READBACK0_DEFAULT                                0x00000000
-#define mmCRTC0_CRTC_PIXEL_DATA_READBACK1_DEFAULT                                0x00000000
-#define mmCRTC0_CRTC_STATUS_DEFAULT                                              0x00000000
-#define mmCRTC0_CRTC_STATUS_POSITION_DEFAULT                                     0x00000000
-#define mmCRTC0_CRTC_NOM_VERT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_STATUS_FRAME_COUNT_DEFAULT                                  0x00000000
-#define mmCRTC0_CRTC_STATUS_VF_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC0_CRTC_STATUS_HV_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC0_CRTC_COUNT_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_COUNT_RESET_DEFAULT                                         0x00000000
-#define mmCRTC0_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                        0x00000000
-#define mmCRTC0_CRTC_VERT_SYNC_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_STEREO_STATUS_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_STEREO_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTC0_CRTC_SNAPSHOT_STATUS_DEFAULT                                     0x00000000
-#define mmCRTC0_CRTC_SNAPSHOT_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTC0_CRTC_SNAPSHOT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_SNAPSHOT_FRAME_DEFAULT                                      0x00000000
-#define mmCRTC0_CRTC_START_LINE_CONTROL_DEFAULT                                  0x00003002
-#define mmCRTC0_CRTC_INTERRUPT_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC0_CRTC_UPDATE_LOCK_DEFAULT                                         0x00000000
-#define mmCRTC0_CRTC_DOUBLE_BUFFER_CONTROL_DEFAULT                               0x00000000
-#define mmCRTC0_CRTC_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                          0x00000000
-#define mmCRTC0_CRTC_TEST_PATTERN_CONTROL_DEFAULT                                0x00000000
-#define mmCRTC0_CRTC_TEST_PATTERN_PARAMETERS_DEFAULT                             0x00000000
-#define mmCRTC0_CRTC_TEST_PATTERN_COLOR_DEFAULT                                  0x00000000
-#define mmCRTC0_CRTC_MASTER_UPDATE_LOCK_DEFAULT                                  0x00010000
-#define mmCRTC0_CRTC_MASTER_UPDATE_MODE_DEFAULT                                  0x00000000
-#define mmCRTC0_CRTC_MVP_INBAND_CNTL_INSERT_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                        0x00000008
-#define mmCRTC0_CRTC_MVP_STATUS_DEFAULT                                          0x00000000
-#define mmCRTC0_CRTC_MASTER_EN_DEFAULT                                           0x00000000
-#define mmCRTC0_CRTC_ALLOW_STOP_OFF_V_CNT_DEFAULT                                0x00010000
-#define mmCRTC0_CRTC_V_UPDATE_INT_STATUS_DEFAULT                                 0x00000000
-#define mmCRTC0_CRTC_OVERSCAN_COLOR_DEFAULT                                      0x00000000
-#define mmCRTC0_CRTC_OVERSCAN_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmCRTC0_CRTC_BLANK_DATA_COLOR_DEFAULT                                    0x00000000
-#define mmCRTC0_CRTC_BLANK_DATA_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTC0_CRTC_BLACK_COLOR_DEFAULT                                         0x00000000
-#define mmCRTC0_CRTC_BLACK_COLOR_EXT_DEFAULT                                     0x00000000
-#define mmCRTC0_CRTC_VERTICAL_INTERRUPT0_POSITION_DEFAULT                        0x00000000
-#define mmCRTC0_CRTC_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC0_CRTC_VERTICAL_INTERRUPT1_POSITION_DEFAULT                        0x00000000
-#define mmCRTC0_CRTC_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC0_CRTC_VERTICAL_INTERRUPT2_POSITION_DEFAULT                        0x00000000
-#define mmCRTC0_CRTC_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC0_CRTC_CRC_CNTL_DEFAULT                                            0x00000000
-#define mmCRTC0_CRTC_CRC0_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC0_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC0_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC0_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC0_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC0_CRTC_CRC0_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC0_CRTC_CRC1_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC1_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC1_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC1_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC0_CRTC_CRC1_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC0_CRTC_CRC1_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC0_CRTC_EXT_TIMING_SYNC_CONTROL_DEFAULT                             0x00000000
-#define mmCRTC0_CRTC_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                        0x00000000
-#define mmCRTC0_CRTC_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                          0x00000000
-#define mmCRTC0_CRTC_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT              0x00000000
-#define mmCRTC0_CRTC_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                   0x00000000
-#define mmCRTC0_CRTC_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTC0_CRTC_STATIC_SCREEN_CONTROL_DEFAULT                               0x00010000
-#define mmCRTC0_CRTC_3D_STRUCTURE_CONTROL_DEFAULT                                0x00000010
-#define mmCRTC0_CRTC_GSL_VSYNC_GAP_DEFAULT                                       0x00000000
-#define mmCRTC0_CRTC_GSL_WINDOW_DEFAULT                                          0x00000000
-#define mmCRTC0_CRTC_GSL_CONTROL_DEFAULT                                         0x00020000
-#define mmCRTC0_CRTC_RANGE_TIMING_INT_STATUS_DEFAULT                             0x00000000
-#define mmCRTC0_CRTC_DRR_CONTROL_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_fmt0_dispdec
-#define mmFMT0_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT0_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT0_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT0_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT0_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT0_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT0_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT0_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT0_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT0_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT0_FMT_CRC_CNTL_DEFAULT                                              0x01000040
-#define mmFMT0_FMT_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0x00ff00ff
-#define mmFMT0_FMT_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0x000700ff
-#define mmFMT0_FMT_CRC_SIG_RED_GREEN_DEFAULT                                     0x00000000
-#define mmFMT0_FMT_CRC_SIG_BLUE_CONTROL_DEFAULT                                  0x00000000
-#define mmFMT0_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT0_FMT_420_HBLANK_EARLY_START_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcp1_dispdec
-#define mmDCP1_GRPH_ENABLE_DEFAULT                                               0x00000001
-#define mmDCP1_GRPH_CONTROL_DEFAULT                                              0x20002040
-#define mmDCP1_GRPH_LUT_10BIT_BYPASS_DEFAULT                                     0x00000000
-#define mmDCP1_GRPH_SWAP_CNTL_DEFAULT                                            0x00000000
-#define mmDCP1_GRPH_PRIMARY_SURFACE_ADDRESS_DEFAULT                              0x00000000
-#define mmDCP1_GRPH_SECONDARY_SURFACE_ADDRESS_DEFAULT                            0x00000000
-#define mmDCP1_GRPH_PITCH_DEFAULT                                                0x00000000
-#define mmDCP1_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                         0x00000000
-#define mmDCP1_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                       0x00000000
-#define mmDCP1_GRPH_SURFACE_OFFSET_X_DEFAULT                                     0x00000000
-#define mmDCP1_GRPH_SURFACE_OFFSET_Y_DEFAULT                                     0x00000000
-#define mmDCP1_GRPH_X_START_DEFAULT                                              0x00000000
-#define mmDCP1_GRPH_Y_START_DEFAULT                                              0x00000000
-#define mmDCP1_GRPH_X_END_DEFAULT                                                0x00000000
-#define mmDCP1_GRPH_Y_END_DEFAULT                                                0x00000000
-#define mmDCP1_INPUT_GAMMA_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP1_GRPH_UPDATE_DEFAULT                                               0x00000000
-#define mmDCP1_GRPH_FLIP_CONTROL_DEFAULT                                         0x00000020
-#define mmDCP1_GRPH_SURFACE_ADDRESS_INUSE_DEFAULT                                0x00000000
-#define mmDCP1_GRPH_DFQ_CONTROL_DEFAULT                                          0x00000000
-#define mmDCP1_GRPH_DFQ_STATUS_DEFAULT                                           0x00000000
-#define mmDCP1_GRPH_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDCP1_GRPH_INTERRUPT_CONTROL_DEFAULT                                    0x00000000
-#define mmDCP1_GRPH_SURFACE_ADDRESS_HIGH_INUSE_DEFAULT                           0x00000000
-#define mmDCP1_GRPH_COMPRESS_SURFACE_ADDRESS_DEFAULT                             0x00000000
-#define mmDCP1_GRPH_COMPRESS_PITCH_DEFAULT                                       0x00000000
-#define mmDCP1_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH_DEFAULT                        0x00000000
-#define mmDCP1_GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                       0x000000ff
-#define mmDCP1_PRESCALE_GRPH_CONTROL_DEFAULT                                     0x00000010
-#define mmDCP1_PRESCALE_VALUES_GRPH_R_DEFAULT                                    0x20000000
-#define mmDCP1_PRESCALE_VALUES_GRPH_G_DEFAULT                                    0x20000000
-#define mmDCP1_PRESCALE_VALUES_GRPH_B_DEFAULT                                    0x20000000
-#define mmDCP1_INPUT_CSC_CONTROL_DEFAULT                                         0x00000000
-#define mmDCP1_INPUT_CSC_C11_C12_DEFAULT                                         0x00002000
-#define mmDCP1_INPUT_CSC_C13_C14_DEFAULT                                         0x00000000
-#define mmDCP1_INPUT_CSC_C21_C22_DEFAULT                                         0x20000000
-#define mmDCP1_INPUT_CSC_C23_C24_DEFAULT                                         0x00000000
-#define mmDCP1_INPUT_CSC_C31_C32_DEFAULT                                         0x00000000
-#define mmDCP1_INPUT_CSC_C33_C34_DEFAULT                                         0x00002000
-#define mmDCP1_OUTPUT_CSC_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP1_OUTPUT_CSC_C11_C12_DEFAULT                                        0x00002000
-#define mmDCP1_OUTPUT_CSC_C13_C14_DEFAULT                                        0x00000000
-#define mmDCP1_OUTPUT_CSC_C21_C22_DEFAULT                                        0x20000000
-#define mmDCP1_OUTPUT_CSC_C23_C24_DEFAULT                                        0x00000000
-#define mmDCP1_OUTPUT_CSC_C31_C32_DEFAULT                                        0x00000000
-#define mmDCP1_OUTPUT_CSC_C33_C34_DEFAULT                                        0x00002000
-#define mmDCP1_COMM_MATRIXA_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP1_COMM_MATRIXA_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP1_COMM_MATRIXA_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP1_COMM_MATRIXA_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP1_COMM_MATRIXA_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP1_COMM_MATRIXA_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP1_COMM_MATRIXB_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP1_COMM_MATRIXB_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP1_COMM_MATRIXB_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP1_COMM_MATRIXB_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP1_COMM_MATRIXB_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP1_COMM_MATRIXB_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP1_DENORM_CONTROL_DEFAULT                                            0x00000003
-#define mmDCP1_OUT_ROUND_CONTROL_DEFAULT                                         0x0000000a
-#define mmDCP1_OUT_CLAMP_CONTROL_R_CR_DEFAULT                                    0x00003fff
-#define mmDCP1_OUT_CLAMP_CONTROL_G_Y_DEFAULT                                     0x00003fff
-#define mmDCP1_OUT_CLAMP_CONTROL_B_CB_DEFAULT                                    0x00003fff
-#define mmDCP1_KEY_CONTROL_DEFAULT                                               0x00000000
-#define mmDCP1_KEY_RANGE_ALPHA_DEFAULT                                           0x00000000
-#define mmDCP1_KEY_RANGE_RED_DEFAULT                                             0x00000000
-#define mmDCP1_KEY_RANGE_GREEN_DEFAULT                                           0x00000000
-#define mmDCP1_KEY_RANGE_BLUE_DEFAULT                                            0x00000000
-#define mmDCP1_DEGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP1_GAMUT_REMAP_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP1_GAMUT_REMAP_C11_C12_DEFAULT                                       0x00002000
-#define mmDCP1_GAMUT_REMAP_C13_C14_DEFAULT                                       0x00000000
-#define mmDCP1_GAMUT_REMAP_C21_C22_DEFAULT                                       0x20000000
-#define mmDCP1_GAMUT_REMAP_C23_C24_DEFAULT                                       0x00000000
-#define mmDCP1_GAMUT_REMAP_C31_C32_DEFAULT                                       0x00000000
-#define mmDCP1_GAMUT_REMAP_C33_C34_DEFAULT                                       0x00002000
-#define mmDCP1_DCP_SPATIAL_DITHER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP1_DCP_RANDOM_SEEDS_DEFAULT                                          0x00000000
-#define mmDCP1_DCP_FP_CONVERTED_FIELD_DEFAULT                                    0x00000000
-#define mmDCP1_CUR_CONTROL_DEFAULT                                               0x00000810
-#define mmDCP1_CUR_SURFACE_ADDRESS_DEFAULT                                       0x00000000
-#define mmDCP1_CUR_SIZE_DEFAULT                                                  0x00000000
-#define mmDCP1_CUR_SURFACE_ADDRESS_HIGH_DEFAULT                                  0x00000000
-#define mmDCP1_CUR_POSITION_DEFAULT                                              0x00000000
-#define mmDCP1_CUR_HOT_SPOT_DEFAULT                                              0x00000000
-#define mmDCP1_CUR_COLOR1_DEFAULT                                                0x00000000
-#define mmDCP1_CUR_COLOR2_DEFAULT                                                0x00000000
-#define mmDCP1_CUR_UPDATE_DEFAULT                                                0x00000000
-#define mmDCP1_CUR_REQUEST_FILTER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP1_CUR_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP1_DC_LUT_RW_MODE_DEFAULT                                            0x00000000
-#define mmDCP1_DC_LUT_RW_INDEX_DEFAULT                                           0x00000000
-#define mmDCP1_DC_LUT_SEQ_COLOR_DEFAULT                                          0x00000000
-#define mmDCP1_DC_LUT_PWL_DATA_DEFAULT                                           0x00000000
-#define mmDCP1_DC_LUT_30_COLOR_DEFAULT                                           0x00000000
-#define mmDCP1_DC_LUT_VGA_ACCESS_ENABLE_DEFAULT                                  0x00000000
-#define mmDCP1_DC_LUT_WRITE_EN_MASK_DEFAULT                                      0x00000007
-#define mmDCP1_DC_LUT_AUTOFILL_DEFAULT                                           0x00000000
-#define mmDCP1_DC_LUT_CONTROL_DEFAULT                                            0x00000000
-#define mmDCP1_DC_LUT_BLACK_OFFSET_BLUE_DEFAULT                                  0x00000000
-#define mmDCP1_DC_LUT_BLACK_OFFSET_GREEN_DEFAULT                                 0x00000000
-#define mmDCP1_DC_LUT_BLACK_OFFSET_RED_DEFAULT                                   0x00000000
-#define mmDCP1_DC_LUT_WHITE_OFFSET_BLUE_DEFAULT                                  0x0000ffff
-#define mmDCP1_DC_LUT_WHITE_OFFSET_GREEN_DEFAULT                                 0x0000ffff
-#define mmDCP1_DC_LUT_WHITE_OFFSET_RED_DEFAULT                                   0x0000ffff
-#define mmDCP1_DCP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP1_DCP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmDCP1_DCP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmDCP1_DVMM_PTE_CONTROL_DEFAULT                                          0x00004000
-#define mmDCP1_DCP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmDCP1_DVMM_PTE_ARB_CONTROL_DEFAULT                                      0x00002220
-#define mmDCP1_GRPH_FLIP_RATE_CNTL_DEFAULT                                       0x00000000
-#define mmDCP1_DCP_GSL_CONTROL_DEFAULT                                           0x60000020
-#define mmDCP1_DCP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000035
-#define mmDCP1_GRPH_STEREOSYNC_FLIP_DEFAULT                                      0x00000200
-#define mmDCP1_HW_ROTATION_DEFAULT                                               0x00000000
-#define mmDCP1_GRPH_XDMA_CACHE_UNDERFLOW_DET_CNTL_DEFAULT                        0x00000010
-#define mmDCP1_REGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP1_REGAMMA_LUT_INDEX_DEFAULT                                         0x00000000
-#define mmDCP1_REGAMMA_LUT_DATA_DEFAULT                                          0x00000000
-#define mmDCP1_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                                 0x00000007
-#define mmDCP1_REGAMMA_CNTLA_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLA_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP1_REGAMMA_CNTLA_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP1_REGAMMA_CNTLA_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP1_REGAMMA_CNTLB_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLB_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP1_REGAMMA_CNTLB_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP1_REGAMMA_CNTLB_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP1_ALPHA_CONTROL_DEFAULT                                             0x00000002
-#define mmDCP1_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmDCP1_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmDCP1_GRPH_XDMA_CACHE_UNDERFLOW_DET_STATUS_DEFAULT                      0x00000000
-#define mmDCP1_GRPH_XDMA_FLIP_TIMEOUT_DEFAULT                                    0x00000000
-#define mmDCP1_GRPH_XDMA_FLIP_AVG_DELAY_DEFAULT                                  0x00000000
-#define mmDCP1_GRPH_SURFACE_COUNTER_CONTROL_DEFAULT                              0x00000012
-#define mmDCP1_GRPH_SURFACE_COUNTER_OUTPUT_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_lb1_dispdec
-#define mmLB1_LB_DATA_FORMAT_DEFAULT                                             0x00000000
-#define mmLB1_LB_MEMORY_CTRL_DEFAULT                                             0x000006b0
-#define mmLB1_LB_MEMORY_SIZE_STATUS_DEFAULT                                      0x00000000
-#define mmLB1_LB_DESKTOP_HEIGHT_DEFAULT                                          0x00000000
-#define mmLB1_LB_VLINE_START_END_DEFAULT                                         0x00000000
-#define mmLB1_LB_VLINE2_START_END_DEFAULT                                        0x00000000
-#define mmLB1_LB_V_COUNTER_DEFAULT                                               0x00000000
-#define mmLB1_LB_SNAPSHOT_V_COUNTER_DEFAULT                                      0x00000000
-#define mmLB1_LB_INTERRUPT_MASK_DEFAULT                                          0x00000000
-#define mmLB1_LB_VLINE_STATUS_DEFAULT                                            0x00000000
-#define mmLB1_LB_VLINE2_STATUS_DEFAULT                                           0x00000000
-#define mmLB1_LB_VBLANK_STATUS_DEFAULT                                           0x00000000
-#define mmLB1_LB_SYNC_RESET_SEL_DEFAULT                                          0x00000002
-#define mmLB1_LB_BLACK_KEYER_R_CR_DEFAULT                                        0x00000000
-#define mmLB1_LB_BLACK_KEYER_G_Y_DEFAULT                                         0x00000000
-#define mmLB1_LB_BLACK_KEYER_B_CB_DEFAULT                                        0x00000000
-#define mmLB1_LB_KEYER_COLOR_CTRL_DEFAULT                                        0x00000000
-#define mmLB1_LB_KEYER_COLOR_R_CR_DEFAULT                                        0x00000000
-#define mmLB1_LB_KEYER_COLOR_G_Y_DEFAULT                                         0x00000000
-#define mmLB1_LB_KEYER_COLOR_B_CB_DEFAULT                                        0x00000000
-#define mmLB1_LB_KEYER_COLOR_REP_R_CR_DEFAULT                                    0x00000000
-#define mmLB1_LB_KEYER_COLOR_REP_G_Y_DEFAULT                                     0x00000000
-#define mmLB1_LB_KEYER_COLOR_REP_B_CB_DEFAULT                                    0x00000000
-#define mmLB1_LB_BUFFER_LEVEL_STATUS_DEFAULT                                     0xa0008000
-#define mmLB1_LB_BUFFER_URGENCY_CTRL_DEFAULT                                     0x00200010
-#define mmLB1_LB_BUFFER_URGENCY_STATUS_DEFAULT                                   0x00000000
-#define mmLB1_LB_BUFFER_STATUS_DEFAULT                                           0x00000002
-#define mmLB1_LB_NO_OUTSTANDING_REQ_STATUS_DEFAULT                               0x00000000
-#define mmLB1_MVP_AFR_FLIP_MODE_DEFAULT                                          0x00000000
-#define mmLB1_MVP_AFR_FLIP_FIFO_CNTL_DEFAULT                                     0x00000000
-#define mmLB1_MVP_FLIP_LINE_NUM_INSERT_DEFAULT                                   0x00000002
-#define mmLB1_DC_MVP_LB_CONTROL_DEFAULT                                          0x00000001
-
-
-// addressBlock: dce_dc_dcfe1_dispdec
-#define mmDCFE1_DCFE_CLOCK_CONTROL_DEFAULT                                       0x00000000
-#define mmDCFE1_DCFE_SOFT_RESET_DEFAULT                                          0x00000000
-#define mmDCFE1_DCFE_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDCFE1_DCFE_MEM_PWR_CTRL2_DEFAULT                                       0x00000000
-#define mmDCFE1_DCFE_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDCFE1_DCFE_MISC_DEFAULT                                                0x00000001
-#define mmDCFE1_DCFE_FLUSH_DEFAULT                                               0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon4_dispdec
-#define mmDC_PERFMON4_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON4_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON4_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON4_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON4_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON4_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON4_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON4_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON4_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dmif_pg1_dispdec
-#define mmDMIF_PG1_DPG_PIPE_ARBITRATION_CONTROL1_DEFAULT                         0x00000000
-#define mmDMIF_PG1_DPG_PIPE_ARBITRATION_CONTROL2_DEFAULT                         0x00000000
-#define mmDMIF_PG1_DPG_WATERMARK_MASK_CONTROL_DEFAULT                            0x000bf777
-#define mmDMIF_PG1_DPG_PIPE_URGENCY_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG1_DPG_PIPE_URGENT_LEVEL_CONTROL_DEFAULT                         0x00000000
-#define mmDMIF_PG1_DPG_PIPE_STUTTER_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG1_DPG_PIPE_STUTTER_CONTROL2_DEFAULT                             0x00000000
-#define mmDMIF_PG1_DPG_PIPE_LOW_POWER_CONTROL_DEFAULT                            0x00000000
-#define mmDMIF_PG1_DPG_REPEATER_PROGRAM_DEFAULT                                  0x00000000
-#define mmDMIF_PG1_DPG_CHK_PRE_PROC_CNTL_DEFAULT                                 0x00000000
-#define mmDMIF_PG1_DPG_DVMM_STATUS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_scl1_dispdec
-#define mmSCL1_SCL_COEF_RAM_SELECT_DEFAULT                                       0x00000000
-#define mmSCL1_SCL_COEF_RAM_TAP_DATA_DEFAULT                                     0x00000000
-#define mmSCL1_SCL_MODE_DEFAULT                                                  0x00000000
-#define mmSCL1_SCL_TAP_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL1_SCL_CONTROL_DEFAULT                                               0x00000000
-#define mmSCL1_SCL_BYPASS_CONTROL_DEFAULT                                        0x00000000
-#define mmSCL1_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                              0x00000000
-#define mmSCL1_SCL_AUTOMATIC_MODE_CONTROL_DEFAULT                                0x00000000
-#define mmSCL1_SCL_HORZ_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL1_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL1_SCL_HORZ_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL1_SCL_VERT_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL1_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL1_SCL_VERT_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL1_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                  0x01000000
-#define mmSCL1_SCL_ROUND_OFFSET_DEFAULT                                          0x80000000
-#define mmSCL1_SCL_UPDATE_DEFAULT                                                0x00000000
-#define mmSCL1_SCL_F_SHARP_CONTROL_DEFAULT                                       0x00000000
-#define mmSCL1_SCL_ALU_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL1_SCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                              0x00000000
-#define mmSCL1_VIEWPORT_START_SECONDARY_DEFAULT                                  0x00000000
-#define mmSCL1_VIEWPORT_START_DEFAULT                                            0x00000000
-#define mmSCL1_VIEWPORT_SIZE_DEFAULT                                             0x00000000
-#define mmSCL1_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                                   0x00000000
-#define mmSCL1_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                                   0x00000000
-#define mmSCL1_SCL_MODE_CHANGE_DET1_DEFAULT                                      0x00000000
-#define mmSCL1_SCL_MODE_CHANGE_DET2_DEFAULT                                      0x00000000
-#define mmSCL1_SCL_MODE_CHANGE_DET3_DEFAULT                                      0x00000000
-#define mmSCL1_SCL_MODE_CHANGE_MASK_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_blnd1_dispdec
-#define mmBLND1_BLND_CONTROL_DEFAULT                                             0xff0220ff
-#define mmBLND1_BLND_SM_CONTROL2_DEFAULT                                         0x00000000
-#define mmBLND1_BLND_CONTROL2_DEFAULT                                            0x00000010
-#define mmBLND1_BLND_UPDATE_DEFAULT                                              0x00000000
-#define mmBLND1_BLND_UNDERFLOW_INTERRUPT_DEFAULT                                 0x00000000
-#define mmBLND1_BLND_V_UPDATE_LOCK_DEFAULT                                       0x80000000
-#define mmBLND1_BLND_REG_UPDATE_STATUS_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_crtc1_dispdec
-#define mmCRTC1_CRTC_H_BLANK_EARLY_NUM_DEFAULT                                   0x00000040
-#define mmCRTC1_CRTC_H_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC1_CRTC_H_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_H_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC1_CRTC_H_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_H_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC1_CRTC_H_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_VBI_END_DEFAULT                                             0x00000003
-#define mmCRTC1_CRTC_V_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC1_CRTC_V_TOTAL_MIN_DEFAULT                                         0x00000000
-#define mmCRTC1_CRTC_V_TOTAL_MAX_DEFAULT                                         0x00000000
-#define mmCRTC1_CRTC_V_TOTAL_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTC1_CRTC_V_TOTAL_INT_STATUS_DEFAULT                                  0x00000000
-#define mmCRTC1_CRTC_VSYNC_NOM_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTC1_CRTC_V_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_V_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC1_CRTC_V_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_V_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC1_CRTC_V_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_DTMTEST_CNTL_DEFAULT                                        0x00000000
-#define mmCRTC1_CRTC_DTMTEST_STATUS_POSITION_DEFAULT                             0x00000000
-#define mmCRTC1_CRTC_TRIGA_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC1_CRTC_TRIGA_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_TRIGB_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC1_CRTC_TRIGB_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_FORCE_COUNT_NOW_CNTL_DEFAULT                                0x00000000
-#define mmCRTC1_CRTC_FLOW_CONTROL_DEFAULT                                        0x00000000
-#define mmCRTC1_CRTC_STEREO_FORCE_NEXT_EYE_DEFAULT                               0x00000000
-#define mmCRTC1_CRTC_AVSYNC_COUNTER_DEFAULT                                      0x00000000
-#define mmCRTC1_CRTC_CONTROL_DEFAULT                                             0x80400110
-#define mmCRTC1_CRTC_BLANK_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_INTERLACE_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_INTERLACE_STATUS_DEFAULT                                    0x00000000
-#define mmCRTC1_CRTC_FIELD_INDICATION_CONTROL_DEFAULT                            0x00000000
-#define mmCRTC1_CRTC_PIXEL_DATA_READBACK0_DEFAULT                                0x00000000
-#define mmCRTC1_CRTC_PIXEL_DATA_READBACK1_DEFAULT                                0x00000000
-#define mmCRTC1_CRTC_STATUS_DEFAULT                                              0x00000000
-#define mmCRTC1_CRTC_STATUS_POSITION_DEFAULT                                     0x00000000
-#define mmCRTC1_CRTC_NOM_VERT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_STATUS_FRAME_COUNT_DEFAULT                                  0x00000000
-#define mmCRTC1_CRTC_STATUS_VF_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC1_CRTC_STATUS_HV_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC1_CRTC_COUNT_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_COUNT_RESET_DEFAULT                                         0x00000000
-#define mmCRTC1_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                        0x00000000
-#define mmCRTC1_CRTC_VERT_SYNC_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_STEREO_STATUS_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_STEREO_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTC1_CRTC_SNAPSHOT_STATUS_DEFAULT                                     0x00000000
-#define mmCRTC1_CRTC_SNAPSHOT_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTC1_CRTC_SNAPSHOT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_SNAPSHOT_FRAME_DEFAULT                                      0x00000000
-#define mmCRTC1_CRTC_START_LINE_CONTROL_DEFAULT                                  0x00003002
-#define mmCRTC1_CRTC_INTERRUPT_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC1_CRTC_UPDATE_LOCK_DEFAULT                                         0x00000000
-#define mmCRTC1_CRTC_DOUBLE_BUFFER_CONTROL_DEFAULT                               0x00000000
-#define mmCRTC1_CRTC_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                          0x00000000
-#define mmCRTC1_CRTC_TEST_PATTERN_CONTROL_DEFAULT                                0x00000000
-#define mmCRTC1_CRTC_TEST_PATTERN_PARAMETERS_DEFAULT                             0x00000000
-#define mmCRTC1_CRTC_TEST_PATTERN_COLOR_DEFAULT                                  0x00000000
-#define mmCRTC1_CRTC_MASTER_UPDATE_LOCK_DEFAULT                                  0x00010000
-#define mmCRTC1_CRTC_MASTER_UPDATE_MODE_DEFAULT                                  0x00000000
-#define mmCRTC1_CRTC_MVP_INBAND_CNTL_INSERT_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                        0x00000008
-#define mmCRTC1_CRTC_MVP_STATUS_DEFAULT                                          0x00000000
-#define mmCRTC1_CRTC_MASTER_EN_DEFAULT                                           0x00000000
-#define mmCRTC1_CRTC_ALLOW_STOP_OFF_V_CNT_DEFAULT                                0x00010000
-#define mmCRTC1_CRTC_V_UPDATE_INT_STATUS_DEFAULT                                 0x00000000
-#define mmCRTC1_CRTC_OVERSCAN_COLOR_DEFAULT                                      0x00000000
-#define mmCRTC1_CRTC_OVERSCAN_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmCRTC1_CRTC_BLANK_DATA_COLOR_DEFAULT                                    0x00000000
-#define mmCRTC1_CRTC_BLANK_DATA_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTC1_CRTC_BLACK_COLOR_DEFAULT                                         0x00000000
-#define mmCRTC1_CRTC_BLACK_COLOR_EXT_DEFAULT                                     0x00000000
-#define mmCRTC1_CRTC_VERTICAL_INTERRUPT0_POSITION_DEFAULT                        0x00000000
-#define mmCRTC1_CRTC_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC1_CRTC_VERTICAL_INTERRUPT1_POSITION_DEFAULT                        0x00000000
-#define mmCRTC1_CRTC_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC1_CRTC_VERTICAL_INTERRUPT2_POSITION_DEFAULT                        0x00000000
-#define mmCRTC1_CRTC_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC1_CRTC_CRC_CNTL_DEFAULT                                            0x00000000
-#define mmCRTC1_CRTC_CRC0_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC0_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC0_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC0_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC0_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC1_CRTC_CRC0_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC1_CRTC_CRC1_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC1_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC1_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC1_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC1_CRTC_CRC1_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC1_CRTC_CRC1_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC1_CRTC_EXT_TIMING_SYNC_CONTROL_DEFAULT                             0x00000000
-#define mmCRTC1_CRTC_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                        0x00000000
-#define mmCRTC1_CRTC_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                          0x00000000
-#define mmCRTC1_CRTC_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT              0x00000000
-#define mmCRTC1_CRTC_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                   0x00000000
-#define mmCRTC1_CRTC_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTC1_CRTC_STATIC_SCREEN_CONTROL_DEFAULT                               0x00010000
-#define mmCRTC1_CRTC_3D_STRUCTURE_CONTROL_DEFAULT                                0x00000010
-#define mmCRTC1_CRTC_GSL_VSYNC_GAP_DEFAULT                                       0x00000000
-#define mmCRTC1_CRTC_GSL_WINDOW_DEFAULT                                          0x00000000
-#define mmCRTC1_CRTC_GSL_CONTROL_DEFAULT                                         0x00020000
-#define mmCRTC1_CRTC_RANGE_TIMING_INT_STATUS_DEFAULT                             0x00000000
-#define mmCRTC1_CRTC_DRR_CONTROL_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_fmt1_dispdec
-#define mmFMT1_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT1_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT1_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT1_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT1_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT1_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT1_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT1_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT1_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT1_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT1_FMT_CRC_CNTL_DEFAULT                                              0x01000040
-#define mmFMT1_FMT_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0x00ff00ff
-#define mmFMT1_FMT_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0x000700ff
-#define mmFMT1_FMT_CRC_SIG_RED_GREEN_DEFAULT                                     0x00000000
-#define mmFMT1_FMT_CRC_SIG_BLUE_CONTROL_DEFAULT                                  0x00000000
-#define mmFMT1_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT1_FMT_420_HBLANK_EARLY_START_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcp2_dispdec
-#define mmDCP2_GRPH_ENABLE_DEFAULT                                               0x00000001
-#define mmDCP2_GRPH_CONTROL_DEFAULT                                              0x20002040
-#define mmDCP2_GRPH_LUT_10BIT_BYPASS_DEFAULT                                     0x00000000
-#define mmDCP2_GRPH_SWAP_CNTL_DEFAULT                                            0x00000000
-#define mmDCP2_GRPH_PRIMARY_SURFACE_ADDRESS_DEFAULT                              0x00000000
-#define mmDCP2_GRPH_SECONDARY_SURFACE_ADDRESS_DEFAULT                            0x00000000
-#define mmDCP2_GRPH_PITCH_DEFAULT                                                0x00000000
-#define mmDCP2_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                         0x00000000
-#define mmDCP2_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                       0x00000000
-#define mmDCP2_GRPH_SURFACE_OFFSET_X_DEFAULT                                     0x00000000
-#define mmDCP2_GRPH_SURFACE_OFFSET_Y_DEFAULT                                     0x00000000
-#define mmDCP2_GRPH_X_START_DEFAULT                                              0x00000000
-#define mmDCP2_GRPH_Y_START_DEFAULT                                              0x00000000
-#define mmDCP2_GRPH_X_END_DEFAULT                                                0x00000000
-#define mmDCP2_GRPH_Y_END_DEFAULT                                                0x00000000
-#define mmDCP2_INPUT_GAMMA_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP2_GRPH_UPDATE_DEFAULT                                               0x00000000
-#define mmDCP2_GRPH_FLIP_CONTROL_DEFAULT                                         0x00000020
-#define mmDCP2_GRPH_SURFACE_ADDRESS_INUSE_DEFAULT                                0x00000000
-#define mmDCP2_GRPH_DFQ_CONTROL_DEFAULT                                          0x00000000
-#define mmDCP2_GRPH_DFQ_STATUS_DEFAULT                                           0x00000000
-#define mmDCP2_GRPH_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDCP2_GRPH_INTERRUPT_CONTROL_DEFAULT                                    0x00000000
-#define mmDCP2_GRPH_SURFACE_ADDRESS_HIGH_INUSE_DEFAULT                           0x00000000
-#define mmDCP2_GRPH_COMPRESS_SURFACE_ADDRESS_DEFAULT                             0x00000000
-#define mmDCP2_GRPH_COMPRESS_PITCH_DEFAULT                                       0x00000000
-#define mmDCP2_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH_DEFAULT                        0x00000000
-#define mmDCP2_GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                       0x000000ff
-#define mmDCP2_PRESCALE_GRPH_CONTROL_DEFAULT                                     0x00000010
-#define mmDCP2_PRESCALE_VALUES_GRPH_R_DEFAULT                                    0x20000000
-#define mmDCP2_PRESCALE_VALUES_GRPH_G_DEFAULT                                    0x20000000
-#define mmDCP2_PRESCALE_VALUES_GRPH_B_DEFAULT                                    0x20000000
-#define mmDCP2_INPUT_CSC_CONTROL_DEFAULT                                         0x00000000
-#define mmDCP2_INPUT_CSC_C11_C12_DEFAULT                                         0x00002000
-#define mmDCP2_INPUT_CSC_C13_C14_DEFAULT                                         0x00000000
-#define mmDCP2_INPUT_CSC_C21_C22_DEFAULT                                         0x20000000
-#define mmDCP2_INPUT_CSC_C23_C24_DEFAULT                                         0x00000000
-#define mmDCP2_INPUT_CSC_C31_C32_DEFAULT                                         0x00000000
-#define mmDCP2_INPUT_CSC_C33_C34_DEFAULT                                         0x00002000
-#define mmDCP2_OUTPUT_CSC_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP2_OUTPUT_CSC_C11_C12_DEFAULT                                        0x00002000
-#define mmDCP2_OUTPUT_CSC_C13_C14_DEFAULT                                        0x00000000
-#define mmDCP2_OUTPUT_CSC_C21_C22_DEFAULT                                        0x20000000
-#define mmDCP2_OUTPUT_CSC_C23_C24_DEFAULT                                        0x00000000
-#define mmDCP2_OUTPUT_CSC_C31_C32_DEFAULT                                        0x00000000
-#define mmDCP2_OUTPUT_CSC_C33_C34_DEFAULT                                        0x00002000
-#define mmDCP2_COMM_MATRIXA_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP2_COMM_MATRIXA_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP2_COMM_MATRIXA_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP2_COMM_MATRIXA_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP2_COMM_MATRIXA_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP2_COMM_MATRIXA_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP2_COMM_MATRIXB_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP2_COMM_MATRIXB_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP2_COMM_MATRIXB_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP2_COMM_MATRIXB_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP2_COMM_MATRIXB_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP2_COMM_MATRIXB_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP2_DENORM_CONTROL_DEFAULT                                            0x00000003
-#define mmDCP2_OUT_ROUND_CONTROL_DEFAULT                                         0x0000000a
-#define mmDCP2_OUT_CLAMP_CONTROL_R_CR_DEFAULT                                    0x00003fff
-#define mmDCP2_OUT_CLAMP_CONTROL_G_Y_DEFAULT                                     0x00003fff
-#define mmDCP2_OUT_CLAMP_CONTROL_B_CB_DEFAULT                                    0x00003fff
-#define mmDCP2_KEY_CONTROL_DEFAULT                                               0x00000000
-#define mmDCP2_KEY_RANGE_ALPHA_DEFAULT                                           0x00000000
-#define mmDCP2_KEY_RANGE_RED_DEFAULT                                             0x00000000
-#define mmDCP2_KEY_RANGE_GREEN_DEFAULT                                           0x00000000
-#define mmDCP2_KEY_RANGE_BLUE_DEFAULT                                            0x00000000
-#define mmDCP2_DEGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP2_GAMUT_REMAP_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP2_GAMUT_REMAP_C11_C12_DEFAULT                                       0x00002000
-#define mmDCP2_GAMUT_REMAP_C13_C14_DEFAULT                                       0x00000000
-#define mmDCP2_GAMUT_REMAP_C21_C22_DEFAULT                                       0x20000000
-#define mmDCP2_GAMUT_REMAP_C23_C24_DEFAULT                                       0x00000000
-#define mmDCP2_GAMUT_REMAP_C31_C32_DEFAULT                                       0x00000000
-#define mmDCP2_GAMUT_REMAP_C33_C34_DEFAULT                                       0x00002000
-#define mmDCP2_DCP_SPATIAL_DITHER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP2_DCP_RANDOM_SEEDS_DEFAULT                                          0x00000000
-#define mmDCP2_DCP_FP_CONVERTED_FIELD_DEFAULT                                    0x00000000
-#define mmDCP2_CUR_CONTROL_DEFAULT                                               0x00000810
-#define mmDCP2_CUR_SURFACE_ADDRESS_DEFAULT                                       0x00000000
-#define mmDCP2_CUR_SIZE_DEFAULT                                                  0x00000000
-#define mmDCP2_CUR_SURFACE_ADDRESS_HIGH_DEFAULT                                  0x00000000
-#define mmDCP2_CUR_POSITION_DEFAULT                                              0x00000000
-#define mmDCP2_CUR_HOT_SPOT_DEFAULT                                              0x00000000
-#define mmDCP2_CUR_COLOR1_DEFAULT                                                0x00000000
-#define mmDCP2_CUR_COLOR2_DEFAULT                                                0x00000000
-#define mmDCP2_CUR_UPDATE_DEFAULT                                                0x00000000
-#define mmDCP2_CUR_REQUEST_FILTER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP2_CUR_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP2_DC_LUT_RW_MODE_DEFAULT                                            0x00000000
-#define mmDCP2_DC_LUT_RW_INDEX_DEFAULT                                           0x00000000
-#define mmDCP2_DC_LUT_SEQ_COLOR_DEFAULT                                          0x00000000
-#define mmDCP2_DC_LUT_PWL_DATA_DEFAULT                                           0x00000000
-#define mmDCP2_DC_LUT_30_COLOR_DEFAULT                                           0x00000000
-#define mmDCP2_DC_LUT_VGA_ACCESS_ENABLE_DEFAULT                                  0x00000000
-#define mmDCP2_DC_LUT_WRITE_EN_MASK_DEFAULT                                      0x00000007
-#define mmDCP2_DC_LUT_AUTOFILL_DEFAULT                                           0x00000000
-#define mmDCP2_DC_LUT_CONTROL_DEFAULT                                            0x00000000
-#define mmDCP2_DC_LUT_BLACK_OFFSET_BLUE_DEFAULT                                  0x00000000
-#define mmDCP2_DC_LUT_BLACK_OFFSET_GREEN_DEFAULT                                 0x00000000
-#define mmDCP2_DC_LUT_BLACK_OFFSET_RED_DEFAULT                                   0x00000000
-#define mmDCP2_DC_LUT_WHITE_OFFSET_BLUE_DEFAULT                                  0x0000ffff
-#define mmDCP2_DC_LUT_WHITE_OFFSET_GREEN_DEFAULT                                 0x0000ffff
-#define mmDCP2_DC_LUT_WHITE_OFFSET_RED_DEFAULT                                   0x0000ffff
-#define mmDCP2_DCP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP2_DCP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmDCP2_DCP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmDCP2_DVMM_PTE_CONTROL_DEFAULT                                          0x00004000
-#define mmDCP2_DCP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmDCP2_DVMM_PTE_ARB_CONTROL_DEFAULT                                      0x00002220
-#define mmDCP2_GRPH_FLIP_RATE_CNTL_DEFAULT                                       0x00000000
-#define mmDCP2_DCP_GSL_CONTROL_DEFAULT                                           0x60000020
-#define mmDCP2_DCP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000035
-#define mmDCP2_GRPH_STEREOSYNC_FLIP_DEFAULT                                      0x00000200
-#define mmDCP2_HW_ROTATION_DEFAULT                                               0x00000000
-#define mmDCP2_GRPH_XDMA_CACHE_UNDERFLOW_DET_CNTL_DEFAULT                        0x00000010
-#define mmDCP2_REGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP2_REGAMMA_LUT_INDEX_DEFAULT                                         0x00000000
-#define mmDCP2_REGAMMA_LUT_DATA_DEFAULT                                          0x00000000
-#define mmDCP2_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                                 0x00000007
-#define mmDCP2_REGAMMA_CNTLA_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLA_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP2_REGAMMA_CNTLA_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP2_REGAMMA_CNTLA_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP2_REGAMMA_CNTLB_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLB_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP2_REGAMMA_CNTLB_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP2_REGAMMA_CNTLB_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP2_ALPHA_CONTROL_DEFAULT                                             0x00000002
-#define mmDCP2_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmDCP2_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmDCP2_GRPH_XDMA_CACHE_UNDERFLOW_DET_STATUS_DEFAULT                      0x00000000
-#define mmDCP2_GRPH_XDMA_FLIP_TIMEOUT_DEFAULT                                    0x00000000
-#define mmDCP2_GRPH_XDMA_FLIP_AVG_DELAY_DEFAULT                                  0x00000000
-#define mmDCP2_GRPH_SURFACE_COUNTER_CONTROL_DEFAULT                              0x00000012
-#define mmDCP2_GRPH_SURFACE_COUNTER_OUTPUT_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_lb2_dispdec
-#define mmLB2_LB_DATA_FORMAT_DEFAULT                                             0x00000000
-#define mmLB2_LB_MEMORY_CTRL_DEFAULT                                             0x000006b0
-#define mmLB2_LB_MEMORY_SIZE_STATUS_DEFAULT                                      0x00000000
-#define mmLB2_LB_DESKTOP_HEIGHT_DEFAULT                                          0x00000000
-#define mmLB2_LB_VLINE_START_END_DEFAULT                                         0x00000000
-#define mmLB2_LB_VLINE2_START_END_DEFAULT                                        0x00000000
-#define mmLB2_LB_V_COUNTER_DEFAULT                                               0x00000000
-#define mmLB2_LB_SNAPSHOT_V_COUNTER_DEFAULT                                      0x00000000
-#define mmLB2_LB_INTERRUPT_MASK_DEFAULT                                          0x00000000
-#define mmLB2_LB_VLINE_STATUS_DEFAULT                                            0x00000000
-#define mmLB2_LB_VLINE2_STATUS_DEFAULT                                           0x00000000
-#define mmLB2_LB_VBLANK_STATUS_DEFAULT                                           0x00000000
-#define mmLB2_LB_SYNC_RESET_SEL_DEFAULT                                          0x00000002
-#define mmLB2_LB_BLACK_KEYER_R_CR_DEFAULT                                        0x00000000
-#define mmLB2_LB_BLACK_KEYER_G_Y_DEFAULT                                         0x00000000
-#define mmLB2_LB_BLACK_KEYER_B_CB_DEFAULT                                        0x00000000
-#define mmLB2_LB_KEYER_COLOR_CTRL_DEFAULT                                        0x00000000
-#define mmLB2_LB_KEYER_COLOR_R_CR_DEFAULT                                        0x00000000
-#define mmLB2_LB_KEYER_COLOR_G_Y_DEFAULT                                         0x00000000
-#define mmLB2_LB_KEYER_COLOR_B_CB_DEFAULT                                        0x00000000
-#define mmLB2_LB_KEYER_COLOR_REP_R_CR_DEFAULT                                    0x00000000
-#define mmLB2_LB_KEYER_COLOR_REP_G_Y_DEFAULT                                     0x00000000
-#define mmLB2_LB_KEYER_COLOR_REP_B_CB_DEFAULT                                    0x00000000
-#define mmLB2_LB_BUFFER_LEVEL_STATUS_DEFAULT                                     0xa0008000
-#define mmLB2_LB_BUFFER_URGENCY_CTRL_DEFAULT                                     0x00200010
-#define mmLB2_LB_BUFFER_URGENCY_STATUS_DEFAULT                                   0x00000000
-#define mmLB2_LB_BUFFER_STATUS_DEFAULT                                           0x00000002
-#define mmLB2_LB_NO_OUTSTANDING_REQ_STATUS_DEFAULT                               0x00000000
-#define mmLB2_MVP_AFR_FLIP_MODE_DEFAULT                                          0x00000000
-#define mmLB2_MVP_AFR_FLIP_FIFO_CNTL_DEFAULT                                     0x00000000
-#define mmLB2_MVP_FLIP_LINE_NUM_INSERT_DEFAULT                                   0x00000002
-#define mmLB2_DC_MVP_LB_CONTROL_DEFAULT                                          0x00000001
-
-
-// addressBlock: dce_dc_dcfe2_dispdec
-#define mmDCFE2_DCFE_CLOCK_CONTROL_DEFAULT                                       0x00000000
-#define mmDCFE2_DCFE_SOFT_RESET_DEFAULT                                          0x00000000
-#define mmDCFE2_DCFE_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDCFE2_DCFE_MEM_PWR_CTRL2_DEFAULT                                       0x00000000
-#define mmDCFE2_DCFE_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDCFE2_DCFE_MISC_DEFAULT                                                0x00000001
-#define mmDCFE2_DCFE_FLUSH_DEFAULT                                               0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon5_dispdec
-#define mmDC_PERFMON5_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON5_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON5_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON5_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON5_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON5_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON5_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON5_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON5_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dmif_pg2_dispdec
-#define mmDMIF_PG2_DPG_PIPE_ARBITRATION_CONTROL1_DEFAULT                         0x00000000
-#define mmDMIF_PG2_DPG_PIPE_ARBITRATION_CONTROL2_DEFAULT                         0x00000000
-#define mmDMIF_PG2_DPG_WATERMARK_MASK_CONTROL_DEFAULT                            0x000bf777
-#define mmDMIF_PG2_DPG_PIPE_URGENCY_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG2_DPG_PIPE_URGENT_LEVEL_CONTROL_DEFAULT                         0x00000000
-#define mmDMIF_PG2_DPG_PIPE_STUTTER_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG2_DPG_PIPE_STUTTER_CONTROL2_DEFAULT                             0x00000000
-#define mmDMIF_PG2_DPG_PIPE_LOW_POWER_CONTROL_DEFAULT                            0x00000000
-#define mmDMIF_PG2_DPG_REPEATER_PROGRAM_DEFAULT                                  0x00000000
-#define mmDMIF_PG2_DPG_CHK_PRE_PROC_CNTL_DEFAULT                                 0x00000000
-#define mmDMIF_PG2_DPG_DVMM_STATUS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_scl2_dispdec
-#define mmSCL2_SCL_COEF_RAM_SELECT_DEFAULT                                       0x00000000
-#define mmSCL2_SCL_COEF_RAM_TAP_DATA_DEFAULT                                     0x00000000
-#define mmSCL2_SCL_MODE_DEFAULT                                                  0x00000000
-#define mmSCL2_SCL_TAP_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL2_SCL_CONTROL_DEFAULT                                               0x00000000
-#define mmSCL2_SCL_BYPASS_CONTROL_DEFAULT                                        0x00000000
-#define mmSCL2_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                              0x00000000
-#define mmSCL2_SCL_AUTOMATIC_MODE_CONTROL_DEFAULT                                0x00000000
-#define mmSCL2_SCL_HORZ_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL2_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL2_SCL_HORZ_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL2_SCL_VERT_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL2_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL2_SCL_VERT_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL2_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                  0x01000000
-#define mmSCL2_SCL_ROUND_OFFSET_DEFAULT                                          0x80000000
-#define mmSCL2_SCL_UPDATE_DEFAULT                                                0x00000000
-#define mmSCL2_SCL_F_SHARP_CONTROL_DEFAULT                                       0x00000000
-#define mmSCL2_SCL_ALU_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL2_SCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                              0x00000000
-#define mmSCL2_VIEWPORT_START_SECONDARY_DEFAULT                                  0x00000000
-#define mmSCL2_VIEWPORT_START_DEFAULT                                            0x00000000
-#define mmSCL2_VIEWPORT_SIZE_DEFAULT                                             0x00000000
-#define mmSCL2_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                                   0x00000000
-#define mmSCL2_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                                   0x00000000
-#define mmSCL2_SCL_MODE_CHANGE_DET1_DEFAULT                                      0x00000000
-#define mmSCL2_SCL_MODE_CHANGE_DET2_DEFAULT                                      0x00000000
-#define mmSCL2_SCL_MODE_CHANGE_DET3_DEFAULT                                      0x00000000
-#define mmSCL2_SCL_MODE_CHANGE_MASK_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_blnd2_dispdec
-#define mmBLND2_BLND_CONTROL_DEFAULT                                             0xff0220ff
-#define mmBLND2_BLND_SM_CONTROL2_DEFAULT                                         0x00000000
-#define mmBLND2_BLND_CONTROL2_DEFAULT                                            0x00000010
-#define mmBLND2_BLND_UPDATE_DEFAULT                                              0x00000000
-#define mmBLND2_BLND_UNDERFLOW_INTERRUPT_DEFAULT                                 0x00000000
-#define mmBLND2_BLND_V_UPDATE_LOCK_DEFAULT                                       0x80000000
-#define mmBLND2_BLND_REG_UPDATE_STATUS_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_crtc2_dispdec
-#define mmCRTC2_CRTC_H_BLANK_EARLY_NUM_DEFAULT                                   0x00000040
-#define mmCRTC2_CRTC_H_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC2_CRTC_H_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_H_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC2_CRTC_H_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_H_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC2_CRTC_H_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_VBI_END_DEFAULT                                             0x00000003
-#define mmCRTC2_CRTC_V_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC2_CRTC_V_TOTAL_MIN_DEFAULT                                         0x00000000
-#define mmCRTC2_CRTC_V_TOTAL_MAX_DEFAULT                                         0x00000000
-#define mmCRTC2_CRTC_V_TOTAL_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTC2_CRTC_V_TOTAL_INT_STATUS_DEFAULT                                  0x00000000
-#define mmCRTC2_CRTC_VSYNC_NOM_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTC2_CRTC_V_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_V_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC2_CRTC_V_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_V_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC2_CRTC_V_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_DTMTEST_CNTL_DEFAULT                                        0x00000000
-#define mmCRTC2_CRTC_DTMTEST_STATUS_POSITION_DEFAULT                             0x00000000
-#define mmCRTC2_CRTC_TRIGA_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC2_CRTC_TRIGA_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_TRIGB_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC2_CRTC_TRIGB_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_FORCE_COUNT_NOW_CNTL_DEFAULT                                0x00000000
-#define mmCRTC2_CRTC_FLOW_CONTROL_DEFAULT                                        0x00000000
-#define mmCRTC2_CRTC_STEREO_FORCE_NEXT_EYE_DEFAULT                               0x00000000
-#define mmCRTC2_CRTC_AVSYNC_COUNTER_DEFAULT                                      0x00000000
-#define mmCRTC2_CRTC_CONTROL_DEFAULT                                             0x80400110
-#define mmCRTC2_CRTC_BLANK_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_INTERLACE_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_INTERLACE_STATUS_DEFAULT                                    0x00000000
-#define mmCRTC2_CRTC_FIELD_INDICATION_CONTROL_DEFAULT                            0x00000000
-#define mmCRTC2_CRTC_PIXEL_DATA_READBACK0_DEFAULT                                0x00000000
-#define mmCRTC2_CRTC_PIXEL_DATA_READBACK1_DEFAULT                                0x00000000
-#define mmCRTC2_CRTC_STATUS_DEFAULT                                              0x00000000
-#define mmCRTC2_CRTC_STATUS_POSITION_DEFAULT                                     0x00000000
-#define mmCRTC2_CRTC_NOM_VERT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_STATUS_FRAME_COUNT_DEFAULT                                  0x00000000
-#define mmCRTC2_CRTC_STATUS_VF_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC2_CRTC_STATUS_HV_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC2_CRTC_COUNT_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_COUNT_RESET_DEFAULT                                         0x00000000
-#define mmCRTC2_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                        0x00000000
-#define mmCRTC2_CRTC_VERT_SYNC_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_STEREO_STATUS_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_STEREO_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTC2_CRTC_SNAPSHOT_STATUS_DEFAULT                                     0x00000000
-#define mmCRTC2_CRTC_SNAPSHOT_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTC2_CRTC_SNAPSHOT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_SNAPSHOT_FRAME_DEFAULT                                      0x00000000
-#define mmCRTC2_CRTC_START_LINE_CONTROL_DEFAULT                                  0x00003002
-#define mmCRTC2_CRTC_INTERRUPT_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC2_CRTC_UPDATE_LOCK_DEFAULT                                         0x00000000
-#define mmCRTC2_CRTC_DOUBLE_BUFFER_CONTROL_DEFAULT                               0x00000000
-#define mmCRTC2_CRTC_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                          0x00000000
-#define mmCRTC2_CRTC_TEST_PATTERN_CONTROL_DEFAULT                                0x00000000
-#define mmCRTC2_CRTC_TEST_PATTERN_PARAMETERS_DEFAULT                             0x00000000
-#define mmCRTC2_CRTC_TEST_PATTERN_COLOR_DEFAULT                                  0x00000000
-#define mmCRTC2_CRTC_MASTER_UPDATE_LOCK_DEFAULT                                  0x00010000
-#define mmCRTC2_CRTC_MASTER_UPDATE_MODE_DEFAULT                                  0x00000000
-#define mmCRTC2_CRTC_MVP_INBAND_CNTL_INSERT_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                        0x00000008
-#define mmCRTC2_CRTC_MVP_STATUS_DEFAULT                                          0x00000000
-#define mmCRTC2_CRTC_MASTER_EN_DEFAULT                                           0x00000000
-#define mmCRTC2_CRTC_ALLOW_STOP_OFF_V_CNT_DEFAULT                                0x00010000
-#define mmCRTC2_CRTC_V_UPDATE_INT_STATUS_DEFAULT                                 0x00000000
-#define mmCRTC2_CRTC_OVERSCAN_COLOR_DEFAULT                                      0x00000000
-#define mmCRTC2_CRTC_OVERSCAN_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmCRTC2_CRTC_BLANK_DATA_COLOR_DEFAULT                                    0x00000000
-#define mmCRTC2_CRTC_BLANK_DATA_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTC2_CRTC_BLACK_COLOR_DEFAULT                                         0x00000000
-#define mmCRTC2_CRTC_BLACK_COLOR_EXT_DEFAULT                                     0x00000000
-#define mmCRTC2_CRTC_VERTICAL_INTERRUPT0_POSITION_DEFAULT                        0x00000000
-#define mmCRTC2_CRTC_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC2_CRTC_VERTICAL_INTERRUPT1_POSITION_DEFAULT                        0x00000000
-#define mmCRTC2_CRTC_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC2_CRTC_VERTICAL_INTERRUPT2_POSITION_DEFAULT                        0x00000000
-#define mmCRTC2_CRTC_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC2_CRTC_CRC_CNTL_DEFAULT                                            0x00000000
-#define mmCRTC2_CRTC_CRC0_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC0_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC0_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC0_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC0_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC2_CRTC_CRC0_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC2_CRTC_CRC1_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC1_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC1_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC1_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC2_CRTC_CRC1_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC2_CRTC_CRC1_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC2_CRTC_EXT_TIMING_SYNC_CONTROL_DEFAULT                             0x00000000
-#define mmCRTC2_CRTC_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                        0x00000000
-#define mmCRTC2_CRTC_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                          0x00000000
-#define mmCRTC2_CRTC_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT              0x00000000
-#define mmCRTC2_CRTC_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                   0x00000000
-#define mmCRTC2_CRTC_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTC2_CRTC_STATIC_SCREEN_CONTROL_DEFAULT                               0x00010000
-#define mmCRTC2_CRTC_3D_STRUCTURE_CONTROL_DEFAULT                                0x00000010
-#define mmCRTC2_CRTC_GSL_VSYNC_GAP_DEFAULT                                       0x00000000
-#define mmCRTC2_CRTC_GSL_WINDOW_DEFAULT                                          0x00000000
-#define mmCRTC2_CRTC_GSL_CONTROL_DEFAULT                                         0x00020000
-#define mmCRTC2_CRTC_RANGE_TIMING_INT_STATUS_DEFAULT                             0x00000000
-#define mmCRTC2_CRTC_DRR_CONTROL_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_fmt2_dispdec
-#define mmFMT2_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT2_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT2_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT2_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT2_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT2_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT2_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT2_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT2_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT2_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT2_FMT_CRC_CNTL_DEFAULT                                              0x01000040
-#define mmFMT2_FMT_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0x00ff00ff
-#define mmFMT2_FMT_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0x000700ff
-#define mmFMT2_FMT_CRC_SIG_RED_GREEN_DEFAULT                                     0x00000000
-#define mmFMT2_FMT_CRC_SIG_BLUE_CONTROL_DEFAULT                                  0x00000000
-#define mmFMT2_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT2_FMT_420_HBLANK_EARLY_START_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcp3_dispdec
-#define mmDCP3_GRPH_ENABLE_DEFAULT                                               0x00000001
-#define mmDCP3_GRPH_CONTROL_DEFAULT                                              0x20002040
-#define mmDCP3_GRPH_LUT_10BIT_BYPASS_DEFAULT                                     0x00000000
-#define mmDCP3_GRPH_SWAP_CNTL_DEFAULT                                            0x00000000
-#define mmDCP3_GRPH_PRIMARY_SURFACE_ADDRESS_DEFAULT                              0x00000000
-#define mmDCP3_GRPH_SECONDARY_SURFACE_ADDRESS_DEFAULT                            0x00000000
-#define mmDCP3_GRPH_PITCH_DEFAULT                                                0x00000000
-#define mmDCP3_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                         0x00000000
-#define mmDCP3_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                       0x00000000
-#define mmDCP3_GRPH_SURFACE_OFFSET_X_DEFAULT                                     0x00000000
-#define mmDCP3_GRPH_SURFACE_OFFSET_Y_DEFAULT                                     0x00000000
-#define mmDCP3_GRPH_X_START_DEFAULT                                              0x00000000
-#define mmDCP3_GRPH_Y_START_DEFAULT                                              0x00000000
-#define mmDCP3_GRPH_X_END_DEFAULT                                                0x00000000
-#define mmDCP3_GRPH_Y_END_DEFAULT                                                0x00000000
-#define mmDCP3_INPUT_GAMMA_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP3_GRPH_UPDATE_DEFAULT                                               0x00000000
-#define mmDCP3_GRPH_FLIP_CONTROL_DEFAULT                                         0x00000020
-#define mmDCP3_GRPH_SURFACE_ADDRESS_INUSE_DEFAULT                                0x00000000
-#define mmDCP3_GRPH_DFQ_CONTROL_DEFAULT                                          0x00000000
-#define mmDCP3_GRPH_DFQ_STATUS_DEFAULT                                           0x00000000
-#define mmDCP3_GRPH_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDCP3_GRPH_INTERRUPT_CONTROL_DEFAULT                                    0x00000000
-#define mmDCP3_GRPH_SURFACE_ADDRESS_HIGH_INUSE_DEFAULT                           0x00000000
-#define mmDCP3_GRPH_COMPRESS_SURFACE_ADDRESS_DEFAULT                             0x00000000
-#define mmDCP3_GRPH_COMPRESS_PITCH_DEFAULT                                       0x00000000
-#define mmDCP3_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH_DEFAULT                        0x00000000
-#define mmDCP3_GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                       0x000000ff
-#define mmDCP3_PRESCALE_GRPH_CONTROL_DEFAULT                                     0x00000010
-#define mmDCP3_PRESCALE_VALUES_GRPH_R_DEFAULT                                    0x20000000
-#define mmDCP3_PRESCALE_VALUES_GRPH_G_DEFAULT                                    0x20000000
-#define mmDCP3_PRESCALE_VALUES_GRPH_B_DEFAULT                                    0x20000000
-#define mmDCP3_INPUT_CSC_CONTROL_DEFAULT                                         0x00000000
-#define mmDCP3_INPUT_CSC_C11_C12_DEFAULT                                         0x00002000
-#define mmDCP3_INPUT_CSC_C13_C14_DEFAULT                                         0x00000000
-#define mmDCP3_INPUT_CSC_C21_C22_DEFAULT                                         0x20000000
-#define mmDCP3_INPUT_CSC_C23_C24_DEFAULT                                         0x00000000
-#define mmDCP3_INPUT_CSC_C31_C32_DEFAULT                                         0x00000000
-#define mmDCP3_INPUT_CSC_C33_C34_DEFAULT                                         0x00002000
-#define mmDCP3_OUTPUT_CSC_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP3_OUTPUT_CSC_C11_C12_DEFAULT                                        0x00002000
-#define mmDCP3_OUTPUT_CSC_C13_C14_DEFAULT                                        0x00000000
-#define mmDCP3_OUTPUT_CSC_C21_C22_DEFAULT                                        0x20000000
-#define mmDCP3_OUTPUT_CSC_C23_C24_DEFAULT                                        0x00000000
-#define mmDCP3_OUTPUT_CSC_C31_C32_DEFAULT                                        0x00000000
-#define mmDCP3_OUTPUT_CSC_C33_C34_DEFAULT                                        0x00002000
-#define mmDCP3_COMM_MATRIXA_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP3_COMM_MATRIXA_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP3_COMM_MATRIXA_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP3_COMM_MATRIXA_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP3_COMM_MATRIXA_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP3_COMM_MATRIXA_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP3_COMM_MATRIXB_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP3_COMM_MATRIXB_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP3_COMM_MATRIXB_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP3_COMM_MATRIXB_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP3_COMM_MATRIXB_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP3_COMM_MATRIXB_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP3_DENORM_CONTROL_DEFAULT                                            0x00000003
-#define mmDCP3_OUT_ROUND_CONTROL_DEFAULT                                         0x0000000a
-#define mmDCP3_OUT_CLAMP_CONTROL_R_CR_DEFAULT                                    0x00003fff
-#define mmDCP3_OUT_CLAMP_CONTROL_G_Y_DEFAULT                                     0x00003fff
-#define mmDCP3_OUT_CLAMP_CONTROL_B_CB_DEFAULT                                    0x00003fff
-#define mmDCP3_KEY_CONTROL_DEFAULT                                               0x00000000
-#define mmDCP3_KEY_RANGE_ALPHA_DEFAULT                                           0x00000000
-#define mmDCP3_KEY_RANGE_RED_DEFAULT                                             0x00000000
-#define mmDCP3_KEY_RANGE_GREEN_DEFAULT                                           0x00000000
-#define mmDCP3_KEY_RANGE_BLUE_DEFAULT                                            0x00000000
-#define mmDCP3_DEGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP3_GAMUT_REMAP_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP3_GAMUT_REMAP_C11_C12_DEFAULT                                       0x00002000
-#define mmDCP3_GAMUT_REMAP_C13_C14_DEFAULT                                       0x00000000
-#define mmDCP3_GAMUT_REMAP_C21_C22_DEFAULT                                       0x20000000
-#define mmDCP3_GAMUT_REMAP_C23_C24_DEFAULT                                       0x00000000
-#define mmDCP3_GAMUT_REMAP_C31_C32_DEFAULT                                       0x00000000
-#define mmDCP3_GAMUT_REMAP_C33_C34_DEFAULT                                       0x00002000
-#define mmDCP3_DCP_SPATIAL_DITHER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP3_DCP_RANDOM_SEEDS_DEFAULT                                          0x00000000
-#define mmDCP3_DCP_FP_CONVERTED_FIELD_DEFAULT                                    0x00000000
-#define mmDCP3_CUR_CONTROL_DEFAULT                                               0x00000810
-#define mmDCP3_CUR_SURFACE_ADDRESS_DEFAULT                                       0x00000000
-#define mmDCP3_CUR_SIZE_DEFAULT                                                  0x00000000
-#define mmDCP3_CUR_SURFACE_ADDRESS_HIGH_DEFAULT                                  0x00000000
-#define mmDCP3_CUR_POSITION_DEFAULT                                              0x00000000
-#define mmDCP3_CUR_HOT_SPOT_DEFAULT                                              0x00000000
-#define mmDCP3_CUR_COLOR1_DEFAULT                                                0x00000000
-#define mmDCP3_CUR_COLOR2_DEFAULT                                                0x00000000
-#define mmDCP3_CUR_UPDATE_DEFAULT                                                0x00000000
-#define mmDCP3_CUR_REQUEST_FILTER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP3_CUR_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP3_DC_LUT_RW_MODE_DEFAULT                                            0x00000000
-#define mmDCP3_DC_LUT_RW_INDEX_DEFAULT                                           0x00000000
-#define mmDCP3_DC_LUT_SEQ_COLOR_DEFAULT                                          0x00000000
-#define mmDCP3_DC_LUT_PWL_DATA_DEFAULT                                           0x00000000
-#define mmDCP3_DC_LUT_30_COLOR_DEFAULT                                           0x00000000
-#define mmDCP3_DC_LUT_VGA_ACCESS_ENABLE_DEFAULT                                  0x00000000
-#define mmDCP3_DC_LUT_WRITE_EN_MASK_DEFAULT                                      0x00000007
-#define mmDCP3_DC_LUT_AUTOFILL_DEFAULT                                           0x00000000
-#define mmDCP3_DC_LUT_CONTROL_DEFAULT                                            0x00000000
-#define mmDCP3_DC_LUT_BLACK_OFFSET_BLUE_DEFAULT                                  0x00000000
-#define mmDCP3_DC_LUT_BLACK_OFFSET_GREEN_DEFAULT                                 0x00000000
-#define mmDCP3_DC_LUT_BLACK_OFFSET_RED_DEFAULT                                   0x00000000
-#define mmDCP3_DC_LUT_WHITE_OFFSET_BLUE_DEFAULT                                  0x0000ffff
-#define mmDCP3_DC_LUT_WHITE_OFFSET_GREEN_DEFAULT                                 0x0000ffff
-#define mmDCP3_DC_LUT_WHITE_OFFSET_RED_DEFAULT                                   0x0000ffff
-#define mmDCP3_DCP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP3_DCP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmDCP3_DCP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmDCP3_DVMM_PTE_CONTROL_DEFAULT                                          0x00004000
-#define mmDCP3_DCP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmDCP3_DVMM_PTE_ARB_CONTROL_DEFAULT                                      0x00002220
-#define mmDCP3_GRPH_FLIP_RATE_CNTL_DEFAULT                                       0x00000000
-#define mmDCP3_DCP_GSL_CONTROL_DEFAULT                                           0x60000020
-#define mmDCP3_DCP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000035
-#define mmDCP3_GRPH_STEREOSYNC_FLIP_DEFAULT                                      0x00000200
-#define mmDCP3_HW_ROTATION_DEFAULT                                               0x00000000
-#define mmDCP3_GRPH_XDMA_CACHE_UNDERFLOW_DET_CNTL_DEFAULT                        0x00000010
-#define mmDCP3_REGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP3_REGAMMA_LUT_INDEX_DEFAULT                                         0x00000000
-#define mmDCP3_REGAMMA_LUT_DATA_DEFAULT                                          0x00000000
-#define mmDCP3_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                                 0x00000007
-#define mmDCP3_REGAMMA_CNTLA_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLA_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP3_REGAMMA_CNTLA_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP3_REGAMMA_CNTLA_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP3_REGAMMA_CNTLB_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLB_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP3_REGAMMA_CNTLB_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP3_REGAMMA_CNTLB_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP3_ALPHA_CONTROL_DEFAULT                                             0x00000002
-#define mmDCP3_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmDCP3_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmDCP3_GRPH_XDMA_CACHE_UNDERFLOW_DET_STATUS_DEFAULT                      0x00000000
-#define mmDCP3_GRPH_XDMA_FLIP_TIMEOUT_DEFAULT                                    0x00000000
-#define mmDCP3_GRPH_XDMA_FLIP_AVG_DELAY_DEFAULT                                  0x00000000
-#define mmDCP3_GRPH_SURFACE_COUNTER_CONTROL_DEFAULT                              0x00000012
-#define mmDCP3_GRPH_SURFACE_COUNTER_OUTPUT_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_lb3_dispdec
-#define mmLB3_LB_DATA_FORMAT_DEFAULT                                             0x00000000
-#define mmLB3_LB_MEMORY_CTRL_DEFAULT                                             0x000006b0
-#define mmLB3_LB_MEMORY_SIZE_STATUS_DEFAULT                                      0x00000000
-#define mmLB3_LB_DESKTOP_HEIGHT_DEFAULT                                          0x00000000
-#define mmLB3_LB_VLINE_START_END_DEFAULT                                         0x00000000
-#define mmLB3_LB_VLINE2_START_END_DEFAULT                                        0x00000000
-#define mmLB3_LB_V_COUNTER_DEFAULT                                               0x00000000
-#define mmLB3_LB_SNAPSHOT_V_COUNTER_DEFAULT                                      0x00000000
-#define mmLB3_LB_INTERRUPT_MASK_DEFAULT                                          0x00000000
-#define mmLB3_LB_VLINE_STATUS_DEFAULT                                            0x00000000
-#define mmLB3_LB_VLINE2_STATUS_DEFAULT                                           0x00000000
-#define mmLB3_LB_VBLANK_STATUS_DEFAULT                                           0x00000000
-#define mmLB3_LB_SYNC_RESET_SEL_DEFAULT                                          0x00000002
-#define mmLB3_LB_BLACK_KEYER_R_CR_DEFAULT                                        0x00000000
-#define mmLB3_LB_BLACK_KEYER_G_Y_DEFAULT                                         0x00000000
-#define mmLB3_LB_BLACK_KEYER_B_CB_DEFAULT                                        0x00000000
-#define mmLB3_LB_KEYER_COLOR_CTRL_DEFAULT                                        0x00000000
-#define mmLB3_LB_KEYER_COLOR_R_CR_DEFAULT                                        0x00000000
-#define mmLB3_LB_KEYER_COLOR_G_Y_DEFAULT                                         0x00000000
-#define mmLB3_LB_KEYER_COLOR_B_CB_DEFAULT                                        0x00000000
-#define mmLB3_LB_KEYER_COLOR_REP_R_CR_DEFAULT                                    0x00000000
-#define mmLB3_LB_KEYER_COLOR_REP_G_Y_DEFAULT                                     0x00000000
-#define mmLB3_LB_KEYER_COLOR_REP_B_CB_DEFAULT                                    0x00000000
-#define mmLB3_LB_BUFFER_LEVEL_STATUS_DEFAULT                                     0xa0008000
-#define mmLB3_LB_BUFFER_URGENCY_CTRL_DEFAULT                                     0x00200010
-#define mmLB3_LB_BUFFER_URGENCY_STATUS_DEFAULT                                   0x00000000
-#define mmLB3_LB_BUFFER_STATUS_DEFAULT                                           0x00000002
-#define mmLB3_LB_NO_OUTSTANDING_REQ_STATUS_DEFAULT                               0x00000000
-#define mmLB3_MVP_AFR_FLIP_MODE_DEFAULT                                          0x00000000
-#define mmLB3_MVP_AFR_FLIP_FIFO_CNTL_DEFAULT                                     0x00000000
-#define mmLB3_MVP_FLIP_LINE_NUM_INSERT_DEFAULT                                   0x00000002
-#define mmLB3_DC_MVP_LB_CONTROL_DEFAULT                                          0x00000001
-
-
-// addressBlock: dce_dc_dcfe3_dispdec
-#define mmDCFE3_DCFE_CLOCK_CONTROL_DEFAULT                                       0x00000000
-#define mmDCFE3_DCFE_SOFT_RESET_DEFAULT                                          0x00000000
-#define mmDCFE3_DCFE_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDCFE3_DCFE_MEM_PWR_CTRL2_DEFAULT                                       0x00000000
-#define mmDCFE3_DCFE_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDCFE3_DCFE_MISC_DEFAULT                                                0x00000001
-#define mmDCFE3_DCFE_FLUSH_DEFAULT                                               0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon6_dispdec
-#define mmDC_PERFMON6_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON6_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON6_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON6_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON6_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON6_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON6_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON6_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON6_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dmif_pg3_dispdec
-#define mmDMIF_PG3_DPG_PIPE_ARBITRATION_CONTROL1_DEFAULT                         0x00000000
-#define mmDMIF_PG3_DPG_PIPE_ARBITRATION_CONTROL2_DEFAULT                         0x00000000
-#define mmDMIF_PG3_DPG_WATERMARK_MASK_CONTROL_DEFAULT                            0x000bf777
-#define mmDMIF_PG3_DPG_PIPE_URGENCY_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG3_DPG_PIPE_URGENT_LEVEL_CONTROL_DEFAULT                         0x00000000
-#define mmDMIF_PG3_DPG_PIPE_STUTTER_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG3_DPG_PIPE_STUTTER_CONTROL2_DEFAULT                             0x00000000
-#define mmDMIF_PG3_DPG_PIPE_LOW_POWER_CONTROL_DEFAULT                            0x00000000
-#define mmDMIF_PG3_DPG_REPEATER_PROGRAM_DEFAULT                                  0x00000000
-#define mmDMIF_PG3_DPG_CHK_PRE_PROC_CNTL_DEFAULT                                 0x00000000
-#define mmDMIF_PG3_DPG_DVMM_STATUS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_scl3_dispdec
-#define mmSCL3_SCL_COEF_RAM_SELECT_DEFAULT                                       0x00000000
-#define mmSCL3_SCL_COEF_RAM_TAP_DATA_DEFAULT                                     0x00000000
-#define mmSCL3_SCL_MODE_DEFAULT                                                  0x00000000
-#define mmSCL3_SCL_TAP_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL3_SCL_CONTROL_DEFAULT                                               0x00000000
-#define mmSCL3_SCL_BYPASS_CONTROL_DEFAULT                                        0x00000000
-#define mmSCL3_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                              0x00000000
-#define mmSCL3_SCL_AUTOMATIC_MODE_CONTROL_DEFAULT                                0x00000000
-#define mmSCL3_SCL_HORZ_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL3_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL3_SCL_HORZ_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL3_SCL_VERT_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL3_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL3_SCL_VERT_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL3_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                  0x01000000
-#define mmSCL3_SCL_ROUND_OFFSET_DEFAULT                                          0x80000000
-#define mmSCL3_SCL_UPDATE_DEFAULT                                                0x00000000
-#define mmSCL3_SCL_F_SHARP_CONTROL_DEFAULT                                       0x00000000
-#define mmSCL3_SCL_ALU_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL3_SCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                              0x00000000
-#define mmSCL3_VIEWPORT_START_SECONDARY_DEFAULT                                  0x00000000
-#define mmSCL3_VIEWPORT_START_DEFAULT                                            0x00000000
-#define mmSCL3_VIEWPORT_SIZE_DEFAULT                                             0x00000000
-#define mmSCL3_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                                   0x00000000
-#define mmSCL3_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                                   0x00000000
-#define mmSCL3_SCL_MODE_CHANGE_DET1_DEFAULT                                      0x00000000
-#define mmSCL3_SCL_MODE_CHANGE_DET2_DEFAULT                                      0x00000000
-#define mmSCL3_SCL_MODE_CHANGE_DET3_DEFAULT                                      0x00000000
-#define mmSCL3_SCL_MODE_CHANGE_MASK_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_blnd3_dispdec
-#define mmBLND3_BLND_CONTROL_DEFAULT                                             0xff0220ff
-#define mmBLND3_BLND_SM_CONTROL2_DEFAULT                                         0x00000000
-#define mmBLND3_BLND_CONTROL2_DEFAULT                                            0x00000010
-#define mmBLND3_BLND_UPDATE_DEFAULT                                              0x00000000
-#define mmBLND3_BLND_UNDERFLOW_INTERRUPT_DEFAULT                                 0x00000000
-#define mmBLND3_BLND_V_UPDATE_LOCK_DEFAULT                                       0x80000000
-#define mmBLND3_BLND_REG_UPDATE_STATUS_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_crtc3_dispdec
-#define mmCRTC3_CRTC_H_BLANK_EARLY_NUM_DEFAULT                                   0x00000040
-#define mmCRTC3_CRTC_H_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC3_CRTC_H_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_H_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC3_CRTC_H_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_H_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC3_CRTC_H_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_VBI_END_DEFAULT                                             0x00000003
-#define mmCRTC3_CRTC_V_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC3_CRTC_V_TOTAL_MIN_DEFAULT                                         0x00000000
-#define mmCRTC3_CRTC_V_TOTAL_MAX_DEFAULT                                         0x00000000
-#define mmCRTC3_CRTC_V_TOTAL_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTC3_CRTC_V_TOTAL_INT_STATUS_DEFAULT                                  0x00000000
-#define mmCRTC3_CRTC_VSYNC_NOM_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTC3_CRTC_V_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_V_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC3_CRTC_V_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_V_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC3_CRTC_V_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_DTMTEST_CNTL_DEFAULT                                        0x00000000
-#define mmCRTC3_CRTC_DTMTEST_STATUS_POSITION_DEFAULT                             0x00000000
-#define mmCRTC3_CRTC_TRIGA_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC3_CRTC_TRIGA_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_TRIGB_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC3_CRTC_TRIGB_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_FORCE_COUNT_NOW_CNTL_DEFAULT                                0x00000000
-#define mmCRTC3_CRTC_FLOW_CONTROL_DEFAULT                                        0x00000000
-#define mmCRTC3_CRTC_STEREO_FORCE_NEXT_EYE_DEFAULT                               0x00000000
-#define mmCRTC3_CRTC_AVSYNC_COUNTER_DEFAULT                                      0x00000000
-#define mmCRTC3_CRTC_CONTROL_DEFAULT                                             0x80400110
-#define mmCRTC3_CRTC_BLANK_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_INTERLACE_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_INTERLACE_STATUS_DEFAULT                                    0x00000000
-#define mmCRTC3_CRTC_FIELD_INDICATION_CONTROL_DEFAULT                            0x00000000
-#define mmCRTC3_CRTC_PIXEL_DATA_READBACK0_DEFAULT                                0x00000000
-#define mmCRTC3_CRTC_PIXEL_DATA_READBACK1_DEFAULT                                0x00000000
-#define mmCRTC3_CRTC_STATUS_DEFAULT                                              0x00000000
-#define mmCRTC3_CRTC_STATUS_POSITION_DEFAULT                                     0x00000000
-#define mmCRTC3_CRTC_NOM_VERT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_STATUS_FRAME_COUNT_DEFAULT                                  0x00000000
-#define mmCRTC3_CRTC_STATUS_VF_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC3_CRTC_STATUS_HV_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC3_CRTC_COUNT_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_COUNT_RESET_DEFAULT                                         0x00000000
-#define mmCRTC3_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                        0x00000000
-#define mmCRTC3_CRTC_VERT_SYNC_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_STEREO_STATUS_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_STEREO_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTC3_CRTC_SNAPSHOT_STATUS_DEFAULT                                     0x00000000
-#define mmCRTC3_CRTC_SNAPSHOT_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTC3_CRTC_SNAPSHOT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_SNAPSHOT_FRAME_DEFAULT                                      0x00000000
-#define mmCRTC3_CRTC_START_LINE_CONTROL_DEFAULT                                  0x00003002
-#define mmCRTC3_CRTC_INTERRUPT_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC3_CRTC_UPDATE_LOCK_DEFAULT                                         0x00000000
-#define mmCRTC3_CRTC_DOUBLE_BUFFER_CONTROL_DEFAULT                               0x00000000
-#define mmCRTC3_CRTC_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                          0x00000000
-#define mmCRTC3_CRTC_TEST_PATTERN_CONTROL_DEFAULT                                0x00000000
-#define mmCRTC3_CRTC_TEST_PATTERN_PARAMETERS_DEFAULT                             0x00000000
-#define mmCRTC3_CRTC_TEST_PATTERN_COLOR_DEFAULT                                  0x00000000
-#define mmCRTC3_CRTC_MASTER_UPDATE_LOCK_DEFAULT                                  0x00010000
-#define mmCRTC3_CRTC_MASTER_UPDATE_MODE_DEFAULT                                  0x00000000
-#define mmCRTC3_CRTC_MVP_INBAND_CNTL_INSERT_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                        0x00000008
-#define mmCRTC3_CRTC_MVP_STATUS_DEFAULT                                          0x00000000
-#define mmCRTC3_CRTC_MASTER_EN_DEFAULT                                           0x00000000
-#define mmCRTC3_CRTC_ALLOW_STOP_OFF_V_CNT_DEFAULT                                0x00010000
-#define mmCRTC3_CRTC_V_UPDATE_INT_STATUS_DEFAULT                                 0x00000000
-#define mmCRTC3_CRTC_OVERSCAN_COLOR_DEFAULT                                      0x00000000
-#define mmCRTC3_CRTC_OVERSCAN_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmCRTC3_CRTC_BLANK_DATA_COLOR_DEFAULT                                    0x00000000
-#define mmCRTC3_CRTC_BLANK_DATA_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTC3_CRTC_BLACK_COLOR_DEFAULT                                         0x00000000
-#define mmCRTC3_CRTC_BLACK_COLOR_EXT_DEFAULT                                     0x00000000
-#define mmCRTC3_CRTC_VERTICAL_INTERRUPT0_POSITION_DEFAULT                        0x00000000
-#define mmCRTC3_CRTC_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC3_CRTC_VERTICAL_INTERRUPT1_POSITION_DEFAULT                        0x00000000
-#define mmCRTC3_CRTC_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC3_CRTC_VERTICAL_INTERRUPT2_POSITION_DEFAULT                        0x00000000
-#define mmCRTC3_CRTC_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC3_CRTC_CRC_CNTL_DEFAULT                                            0x00000000
-#define mmCRTC3_CRTC_CRC0_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC0_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC0_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC0_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC0_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC3_CRTC_CRC0_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC3_CRTC_CRC1_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC1_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC1_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC1_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC3_CRTC_CRC1_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC3_CRTC_CRC1_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC3_CRTC_EXT_TIMING_SYNC_CONTROL_DEFAULT                             0x00000000
-#define mmCRTC3_CRTC_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                        0x00000000
-#define mmCRTC3_CRTC_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                          0x00000000
-#define mmCRTC3_CRTC_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT              0x00000000
-#define mmCRTC3_CRTC_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                   0x00000000
-#define mmCRTC3_CRTC_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTC3_CRTC_STATIC_SCREEN_CONTROL_DEFAULT                               0x00010000
-#define mmCRTC3_CRTC_3D_STRUCTURE_CONTROL_DEFAULT                                0x00000010
-#define mmCRTC3_CRTC_GSL_VSYNC_GAP_DEFAULT                                       0x00000000
-#define mmCRTC3_CRTC_GSL_WINDOW_DEFAULT                                          0x00000000
-#define mmCRTC3_CRTC_GSL_CONTROL_DEFAULT                                         0x00020000
-#define mmCRTC3_CRTC_RANGE_TIMING_INT_STATUS_DEFAULT                             0x00000000
-#define mmCRTC3_CRTC_DRR_CONTROL_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_fmt3_dispdec
-#define mmFMT3_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT3_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT3_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT3_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT3_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT3_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT3_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT3_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT3_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT3_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT3_FMT_CRC_CNTL_DEFAULT                                              0x01000040
-#define mmFMT3_FMT_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0x00ff00ff
-#define mmFMT3_FMT_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0x000700ff
-#define mmFMT3_FMT_CRC_SIG_RED_GREEN_DEFAULT                                     0x00000000
-#define mmFMT3_FMT_CRC_SIG_BLUE_CONTROL_DEFAULT                                  0x00000000
-#define mmFMT3_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT3_FMT_420_HBLANK_EARLY_START_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcp4_dispdec
-#define mmDCP4_GRPH_ENABLE_DEFAULT                                               0x00000001
-#define mmDCP4_GRPH_CONTROL_DEFAULT                                              0x20002040
-#define mmDCP4_GRPH_LUT_10BIT_BYPASS_DEFAULT                                     0x00000000
-#define mmDCP4_GRPH_SWAP_CNTL_DEFAULT                                            0x00000000
-#define mmDCP4_GRPH_PRIMARY_SURFACE_ADDRESS_DEFAULT                              0x00000000
-#define mmDCP4_GRPH_SECONDARY_SURFACE_ADDRESS_DEFAULT                            0x00000000
-#define mmDCP4_GRPH_PITCH_DEFAULT                                                0x00000000
-#define mmDCP4_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                         0x00000000
-#define mmDCP4_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                       0x00000000
-#define mmDCP4_GRPH_SURFACE_OFFSET_X_DEFAULT                                     0x00000000
-#define mmDCP4_GRPH_SURFACE_OFFSET_Y_DEFAULT                                     0x00000000
-#define mmDCP4_GRPH_X_START_DEFAULT                                              0x00000000
-#define mmDCP4_GRPH_Y_START_DEFAULT                                              0x00000000
-#define mmDCP4_GRPH_X_END_DEFAULT                                                0x00000000
-#define mmDCP4_GRPH_Y_END_DEFAULT                                                0x00000000
-#define mmDCP4_INPUT_GAMMA_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP4_GRPH_UPDATE_DEFAULT                                               0x00000000
-#define mmDCP4_GRPH_FLIP_CONTROL_DEFAULT                                         0x00000020
-#define mmDCP4_GRPH_SURFACE_ADDRESS_INUSE_DEFAULT                                0x00000000
-#define mmDCP4_GRPH_DFQ_CONTROL_DEFAULT                                          0x00000000
-#define mmDCP4_GRPH_DFQ_STATUS_DEFAULT                                           0x00000000
-#define mmDCP4_GRPH_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDCP4_GRPH_INTERRUPT_CONTROL_DEFAULT                                    0x00000000
-#define mmDCP4_GRPH_SURFACE_ADDRESS_HIGH_INUSE_DEFAULT                           0x00000000
-#define mmDCP4_GRPH_COMPRESS_SURFACE_ADDRESS_DEFAULT                             0x00000000
-#define mmDCP4_GRPH_COMPRESS_PITCH_DEFAULT                                       0x00000000
-#define mmDCP4_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH_DEFAULT                        0x00000000
-#define mmDCP4_GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                       0x000000ff
-#define mmDCP4_PRESCALE_GRPH_CONTROL_DEFAULT                                     0x00000010
-#define mmDCP4_PRESCALE_VALUES_GRPH_R_DEFAULT                                    0x20000000
-#define mmDCP4_PRESCALE_VALUES_GRPH_G_DEFAULT                                    0x20000000
-#define mmDCP4_PRESCALE_VALUES_GRPH_B_DEFAULT                                    0x20000000
-#define mmDCP4_INPUT_CSC_CONTROL_DEFAULT                                         0x00000000
-#define mmDCP4_INPUT_CSC_C11_C12_DEFAULT                                         0x00002000
-#define mmDCP4_INPUT_CSC_C13_C14_DEFAULT                                         0x00000000
-#define mmDCP4_INPUT_CSC_C21_C22_DEFAULT                                         0x20000000
-#define mmDCP4_INPUT_CSC_C23_C24_DEFAULT                                         0x00000000
-#define mmDCP4_INPUT_CSC_C31_C32_DEFAULT                                         0x00000000
-#define mmDCP4_INPUT_CSC_C33_C34_DEFAULT                                         0x00002000
-#define mmDCP4_OUTPUT_CSC_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP4_OUTPUT_CSC_C11_C12_DEFAULT                                        0x00002000
-#define mmDCP4_OUTPUT_CSC_C13_C14_DEFAULT                                        0x00000000
-#define mmDCP4_OUTPUT_CSC_C21_C22_DEFAULT                                        0x20000000
-#define mmDCP4_OUTPUT_CSC_C23_C24_DEFAULT                                        0x00000000
-#define mmDCP4_OUTPUT_CSC_C31_C32_DEFAULT                                        0x00000000
-#define mmDCP4_OUTPUT_CSC_C33_C34_DEFAULT                                        0x00002000
-#define mmDCP4_COMM_MATRIXA_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP4_COMM_MATRIXA_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP4_COMM_MATRIXA_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP4_COMM_MATRIXA_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP4_COMM_MATRIXA_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP4_COMM_MATRIXA_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP4_COMM_MATRIXB_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP4_COMM_MATRIXB_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP4_COMM_MATRIXB_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP4_COMM_MATRIXB_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP4_COMM_MATRIXB_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP4_COMM_MATRIXB_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP4_DENORM_CONTROL_DEFAULT                                            0x00000003
-#define mmDCP4_OUT_ROUND_CONTROL_DEFAULT                                         0x0000000a
-#define mmDCP4_OUT_CLAMP_CONTROL_R_CR_DEFAULT                                    0x00003fff
-#define mmDCP4_OUT_CLAMP_CONTROL_G_Y_DEFAULT                                     0x00003fff
-#define mmDCP4_OUT_CLAMP_CONTROL_B_CB_DEFAULT                                    0x00003fff
-#define mmDCP4_KEY_CONTROL_DEFAULT                                               0x00000000
-#define mmDCP4_KEY_RANGE_ALPHA_DEFAULT                                           0x00000000
-#define mmDCP4_KEY_RANGE_RED_DEFAULT                                             0x00000000
-#define mmDCP4_KEY_RANGE_GREEN_DEFAULT                                           0x00000000
-#define mmDCP4_KEY_RANGE_BLUE_DEFAULT                                            0x00000000
-#define mmDCP4_DEGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP4_GAMUT_REMAP_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP4_GAMUT_REMAP_C11_C12_DEFAULT                                       0x00002000
-#define mmDCP4_GAMUT_REMAP_C13_C14_DEFAULT                                       0x00000000
-#define mmDCP4_GAMUT_REMAP_C21_C22_DEFAULT                                       0x20000000
-#define mmDCP4_GAMUT_REMAP_C23_C24_DEFAULT                                       0x00000000
-#define mmDCP4_GAMUT_REMAP_C31_C32_DEFAULT                                       0x00000000
-#define mmDCP4_GAMUT_REMAP_C33_C34_DEFAULT                                       0x00002000
-#define mmDCP4_DCP_SPATIAL_DITHER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP4_DCP_RANDOM_SEEDS_DEFAULT                                          0x00000000
-#define mmDCP4_DCP_FP_CONVERTED_FIELD_DEFAULT                                    0x00000000
-#define mmDCP4_CUR_CONTROL_DEFAULT                                               0x00000810
-#define mmDCP4_CUR_SURFACE_ADDRESS_DEFAULT                                       0x00000000
-#define mmDCP4_CUR_SIZE_DEFAULT                                                  0x00000000
-#define mmDCP4_CUR_SURFACE_ADDRESS_HIGH_DEFAULT                                  0x00000000
-#define mmDCP4_CUR_POSITION_DEFAULT                                              0x00000000
-#define mmDCP4_CUR_HOT_SPOT_DEFAULT                                              0x00000000
-#define mmDCP4_CUR_COLOR1_DEFAULT                                                0x00000000
-#define mmDCP4_CUR_COLOR2_DEFAULT                                                0x00000000
-#define mmDCP4_CUR_UPDATE_DEFAULT                                                0x00000000
-#define mmDCP4_CUR_REQUEST_FILTER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP4_CUR_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP4_DC_LUT_RW_MODE_DEFAULT                                            0x00000000
-#define mmDCP4_DC_LUT_RW_INDEX_DEFAULT                                           0x00000000
-#define mmDCP4_DC_LUT_SEQ_COLOR_DEFAULT                                          0x00000000
-#define mmDCP4_DC_LUT_PWL_DATA_DEFAULT                                           0x00000000
-#define mmDCP4_DC_LUT_30_COLOR_DEFAULT                                           0x00000000
-#define mmDCP4_DC_LUT_VGA_ACCESS_ENABLE_DEFAULT                                  0x00000000
-#define mmDCP4_DC_LUT_WRITE_EN_MASK_DEFAULT                                      0x00000007
-#define mmDCP4_DC_LUT_AUTOFILL_DEFAULT                                           0x00000000
-#define mmDCP4_DC_LUT_CONTROL_DEFAULT                                            0x00000000
-#define mmDCP4_DC_LUT_BLACK_OFFSET_BLUE_DEFAULT                                  0x00000000
-#define mmDCP4_DC_LUT_BLACK_OFFSET_GREEN_DEFAULT                                 0x00000000
-#define mmDCP4_DC_LUT_BLACK_OFFSET_RED_DEFAULT                                   0x00000000
-#define mmDCP4_DC_LUT_WHITE_OFFSET_BLUE_DEFAULT                                  0x0000ffff
-#define mmDCP4_DC_LUT_WHITE_OFFSET_GREEN_DEFAULT                                 0x0000ffff
-#define mmDCP4_DC_LUT_WHITE_OFFSET_RED_DEFAULT                                   0x0000ffff
-#define mmDCP4_DCP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP4_DCP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmDCP4_DCP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmDCP4_DVMM_PTE_CONTROL_DEFAULT                                          0x00004000
-#define mmDCP4_DCP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmDCP4_DVMM_PTE_ARB_CONTROL_DEFAULT                                      0x00002220
-#define mmDCP4_GRPH_FLIP_RATE_CNTL_DEFAULT                                       0x00000000
-#define mmDCP4_DCP_GSL_CONTROL_DEFAULT                                           0x60000020
-#define mmDCP4_DCP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000035
-#define mmDCP4_GRPH_STEREOSYNC_FLIP_DEFAULT                                      0x00000200
-#define mmDCP4_HW_ROTATION_DEFAULT                                               0x00000000
-#define mmDCP4_GRPH_XDMA_CACHE_UNDERFLOW_DET_CNTL_DEFAULT                        0x00000010
-#define mmDCP4_REGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP4_REGAMMA_LUT_INDEX_DEFAULT                                         0x00000000
-#define mmDCP4_REGAMMA_LUT_DATA_DEFAULT                                          0x00000000
-#define mmDCP4_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                                 0x00000007
-#define mmDCP4_REGAMMA_CNTLA_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLA_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP4_REGAMMA_CNTLA_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP4_REGAMMA_CNTLA_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP4_REGAMMA_CNTLB_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLB_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP4_REGAMMA_CNTLB_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP4_REGAMMA_CNTLB_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP4_ALPHA_CONTROL_DEFAULT                                             0x00000002
-#define mmDCP4_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmDCP4_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmDCP4_GRPH_XDMA_CACHE_UNDERFLOW_DET_STATUS_DEFAULT                      0x00000000
-#define mmDCP4_GRPH_XDMA_FLIP_TIMEOUT_DEFAULT                                    0x00000000
-#define mmDCP4_GRPH_XDMA_FLIP_AVG_DELAY_DEFAULT                                  0x00000000
-#define mmDCP4_GRPH_SURFACE_COUNTER_CONTROL_DEFAULT                              0x00000012
-#define mmDCP4_GRPH_SURFACE_COUNTER_OUTPUT_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_lb4_dispdec
-#define mmLB4_LB_DATA_FORMAT_DEFAULT                                             0x00000000
-#define mmLB4_LB_MEMORY_CTRL_DEFAULT                                             0x000006b0
-#define mmLB4_LB_MEMORY_SIZE_STATUS_DEFAULT                                      0x00000000
-#define mmLB4_LB_DESKTOP_HEIGHT_DEFAULT                                          0x00000000
-#define mmLB4_LB_VLINE_START_END_DEFAULT                                         0x00000000
-#define mmLB4_LB_VLINE2_START_END_DEFAULT                                        0x00000000
-#define mmLB4_LB_V_COUNTER_DEFAULT                                               0x00000000
-#define mmLB4_LB_SNAPSHOT_V_COUNTER_DEFAULT                                      0x00000000
-#define mmLB4_LB_INTERRUPT_MASK_DEFAULT                                          0x00000000
-#define mmLB4_LB_VLINE_STATUS_DEFAULT                                            0x00000000
-#define mmLB4_LB_VLINE2_STATUS_DEFAULT                                           0x00000000
-#define mmLB4_LB_VBLANK_STATUS_DEFAULT                                           0x00000000
-#define mmLB4_LB_SYNC_RESET_SEL_DEFAULT                                          0x00000002
-#define mmLB4_LB_BLACK_KEYER_R_CR_DEFAULT                                        0x00000000
-#define mmLB4_LB_BLACK_KEYER_G_Y_DEFAULT                                         0x00000000
-#define mmLB4_LB_BLACK_KEYER_B_CB_DEFAULT                                        0x00000000
-#define mmLB4_LB_KEYER_COLOR_CTRL_DEFAULT                                        0x00000000
-#define mmLB4_LB_KEYER_COLOR_R_CR_DEFAULT                                        0x00000000
-#define mmLB4_LB_KEYER_COLOR_G_Y_DEFAULT                                         0x00000000
-#define mmLB4_LB_KEYER_COLOR_B_CB_DEFAULT                                        0x00000000
-#define mmLB4_LB_KEYER_COLOR_REP_R_CR_DEFAULT                                    0x00000000
-#define mmLB4_LB_KEYER_COLOR_REP_G_Y_DEFAULT                                     0x00000000
-#define mmLB4_LB_KEYER_COLOR_REP_B_CB_DEFAULT                                    0x00000000
-#define mmLB4_LB_BUFFER_LEVEL_STATUS_DEFAULT                                     0xa0008000
-#define mmLB4_LB_BUFFER_URGENCY_CTRL_DEFAULT                                     0x00200010
-#define mmLB4_LB_BUFFER_URGENCY_STATUS_DEFAULT                                   0x00000000
-#define mmLB4_LB_BUFFER_STATUS_DEFAULT                                           0x00000002
-#define mmLB4_LB_NO_OUTSTANDING_REQ_STATUS_DEFAULT                               0x00000000
-#define mmLB4_MVP_AFR_FLIP_MODE_DEFAULT                                          0x00000000
-#define mmLB4_MVP_AFR_FLIP_FIFO_CNTL_DEFAULT                                     0x00000000
-#define mmLB4_MVP_FLIP_LINE_NUM_INSERT_DEFAULT                                   0x00000002
-#define mmLB4_DC_MVP_LB_CONTROL_DEFAULT                                          0x00000001
-
-
-// addressBlock: dce_dc_dcfe4_dispdec
-#define mmDCFE4_DCFE_CLOCK_CONTROL_DEFAULT                                       0x00000000
-#define mmDCFE4_DCFE_SOFT_RESET_DEFAULT                                          0x00000000
-#define mmDCFE4_DCFE_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDCFE4_DCFE_MEM_PWR_CTRL2_DEFAULT                                       0x00000000
-#define mmDCFE4_DCFE_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDCFE4_DCFE_MISC_DEFAULT                                                0x00000001
-#define mmDCFE4_DCFE_FLUSH_DEFAULT                                               0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon7_dispdec
-#define mmDC_PERFMON7_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON7_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON7_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON7_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON7_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON7_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON7_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON7_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON7_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dmif_pg4_dispdec
-#define mmDMIF_PG4_DPG_PIPE_ARBITRATION_CONTROL1_DEFAULT                         0x00000000
-#define mmDMIF_PG4_DPG_PIPE_ARBITRATION_CONTROL2_DEFAULT                         0x00000000
-#define mmDMIF_PG4_DPG_WATERMARK_MASK_CONTROL_DEFAULT                            0x000bf777
-#define mmDMIF_PG4_DPG_PIPE_URGENCY_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG4_DPG_PIPE_URGENT_LEVEL_CONTROL_DEFAULT                         0x00000000
-#define mmDMIF_PG4_DPG_PIPE_STUTTER_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG4_DPG_PIPE_STUTTER_CONTROL2_DEFAULT                             0x00000000
-#define mmDMIF_PG4_DPG_PIPE_LOW_POWER_CONTROL_DEFAULT                            0x00000000
-#define mmDMIF_PG4_DPG_REPEATER_PROGRAM_DEFAULT                                  0x00000000
-#define mmDMIF_PG4_DPG_CHK_PRE_PROC_CNTL_DEFAULT                                 0x00000000
-#define mmDMIF_PG4_DPG_DVMM_STATUS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_scl4_dispdec
-#define mmSCL4_SCL_COEF_RAM_SELECT_DEFAULT                                       0x00000000
-#define mmSCL4_SCL_COEF_RAM_TAP_DATA_DEFAULT                                     0x00000000
-#define mmSCL4_SCL_MODE_DEFAULT                                                  0x00000000
-#define mmSCL4_SCL_TAP_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL4_SCL_CONTROL_DEFAULT                                               0x00000000
-#define mmSCL4_SCL_BYPASS_CONTROL_DEFAULT                                        0x00000000
-#define mmSCL4_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                              0x00000000
-#define mmSCL4_SCL_AUTOMATIC_MODE_CONTROL_DEFAULT                                0x00000000
-#define mmSCL4_SCL_HORZ_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL4_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL4_SCL_HORZ_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL4_SCL_VERT_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL4_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL4_SCL_VERT_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL4_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                  0x01000000
-#define mmSCL4_SCL_ROUND_OFFSET_DEFAULT                                          0x80000000
-#define mmSCL4_SCL_UPDATE_DEFAULT                                                0x00000000
-#define mmSCL4_SCL_F_SHARP_CONTROL_DEFAULT                                       0x00000000
-#define mmSCL4_SCL_ALU_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL4_SCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                              0x00000000
-#define mmSCL4_VIEWPORT_START_SECONDARY_DEFAULT                                  0x00000000
-#define mmSCL4_VIEWPORT_START_DEFAULT                                            0x00000000
-#define mmSCL4_VIEWPORT_SIZE_DEFAULT                                             0x00000000
-#define mmSCL4_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                                   0x00000000
-#define mmSCL4_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                                   0x00000000
-#define mmSCL4_SCL_MODE_CHANGE_DET1_DEFAULT                                      0x00000000
-#define mmSCL4_SCL_MODE_CHANGE_DET2_DEFAULT                                      0x00000000
-#define mmSCL4_SCL_MODE_CHANGE_DET3_DEFAULT                                      0x00000000
-#define mmSCL4_SCL_MODE_CHANGE_MASK_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_blnd4_dispdec
-#define mmBLND4_BLND_CONTROL_DEFAULT                                             0xff0220ff
-#define mmBLND4_BLND_SM_CONTROL2_DEFAULT                                         0x00000000
-#define mmBLND4_BLND_CONTROL2_DEFAULT                                            0x00000010
-#define mmBLND4_BLND_UPDATE_DEFAULT                                              0x00000000
-#define mmBLND4_BLND_UNDERFLOW_INTERRUPT_DEFAULT                                 0x00000000
-#define mmBLND4_BLND_V_UPDATE_LOCK_DEFAULT                                       0x80000000
-#define mmBLND4_BLND_REG_UPDATE_STATUS_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_crtc4_dispdec
-#define mmCRTC4_CRTC_H_BLANK_EARLY_NUM_DEFAULT                                   0x00000040
-#define mmCRTC4_CRTC_H_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC4_CRTC_H_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_H_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC4_CRTC_H_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_H_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC4_CRTC_H_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_VBI_END_DEFAULT                                             0x00000003
-#define mmCRTC4_CRTC_V_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC4_CRTC_V_TOTAL_MIN_DEFAULT                                         0x00000000
-#define mmCRTC4_CRTC_V_TOTAL_MAX_DEFAULT                                         0x00000000
-#define mmCRTC4_CRTC_V_TOTAL_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTC4_CRTC_V_TOTAL_INT_STATUS_DEFAULT                                  0x00000000
-#define mmCRTC4_CRTC_VSYNC_NOM_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTC4_CRTC_V_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_V_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC4_CRTC_V_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_V_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC4_CRTC_V_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_DTMTEST_CNTL_DEFAULT                                        0x00000000
-#define mmCRTC4_CRTC_DTMTEST_STATUS_POSITION_DEFAULT                             0x00000000
-#define mmCRTC4_CRTC_TRIGA_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC4_CRTC_TRIGA_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_TRIGB_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC4_CRTC_TRIGB_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_FORCE_COUNT_NOW_CNTL_DEFAULT                                0x00000000
-#define mmCRTC4_CRTC_FLOW_CONTROL_DEFAULT                                        0x00000000
-#define mmCRTC4_CRTC_STEREO_FORCE_NEXT_EYE_DEFAULT                               0x00000000
-#define mmCRTC4_CRTC_AVSYNC_COUNTER_DEFAULT                                      0x00000000
-#define mmCRTC4_CRTC_CONTROL_DEFAULT                                             0x80400110
-#define mmCRTC4_CRTC_BLANK_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_INTERLACE_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_INTERLACE_STATUS_DEFAULT                                    0x00000000
-#define mmCRTC4_CRTC_FIELD_INDICATION_CONTROL_DEFAULT                            0x00000000
-#define mmCRTC4_CRTC_PIXEL_DATA_READBACK0_DEFAULT                                0x00000000
-#define mmCRTC4_CRTC_PIXEL_DATA_READBACK1_DEFAULT                                0x00000000
-#define mmCRTC4_CRTC_STATUS_DEFAULT                                              0x00000000
-#define mmCRTC4_CRTC_STATUS_POSITION_DEFAULT                                     0x00000000
-#define mmCRTC4_CRTC_NOM_VERT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_STATUS_FRAME_COUNT_DEFAULT                                  0x00000000
-#define mmCRTC4_CRTC_STATUS_VF_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC4_CRTC_STATUS_HV_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC4_CRTC_COUNT_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_COUNT_RESET_DEFAULT                                         0x00000000
-#define mmCRTC4_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                        0x00000000
-#define mmCRTC4_CRTC_VERT_SYNC_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_STEREO_STATUS_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_STEREO_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTC4_CRTC_SNAPSHOT_STATUS_DEFAULT                                     0x00000000
-#define mmCRTC4_CRTC_SNAPSHOT_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTC4_CRTC_SNAPSHOT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_SNAPSHOT_FRAME_DEFAULT                                      0x00000000
-#define mmCRTC4_CRTC_START_LINE_CONTROL_DEFAULT                                  0x00003002
-#define mmCRTC4_CRTC_INTERRUPT_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC4_CRTC_UPDATE_LOCK_DEFAULT                                         0x00000000
-#define mmCRTC4_CRTC_DOUBLE_BUFFER_CONTROL_DEFAULT                               0x00000000
-#define mmCRTC4_CRTC_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                          0x00000000
-#define mmCRTC4_CRTC_TEST_PATTERN_CONTROL_DEFAULT                                0x00000000
-#define mmCRTC4_CRTC_TEST_PATTERN_PARAMETERS_DEFAULT                             0x00000000
-#define mmCRTC4_CRTC_TEST_PATTERN_COLOR_DEFAULT                                  0x00000000
-#define mmCRTC4_CRTC_MASTER_UPDATE_LOCK_DEFAULT                                  0x00010000
-#define mmCRTC4_CRTC_MASTER_UPDATE_MODE_DEFAULT                                  0x00000000
-#define mmCRTC4_CRTC_MVP_INBAND_CNTL_INSERT_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                        0x00000008
-#define mmCRTC4_CRTC_MVP_STATUS_DEFAULT                                          0x00000000
-#define mmCRTC4_CRTC_MASTER_EN_DEFAULT                                           0x00000000
-#define mmCRTC4_CRTC_ALLOW_STOP_OFF_V_CNT_DEFAULT                                0x00010000
-#define mmCRTC4_CRTC_V_UPDATE_INT_STATUS_DEFAULT                                 0x00000000
-#define mmCRTC4_CRTC_OVERSCAN_COLOR_DEFAULT                                      0x00000000
-#define mmCRTC4_CRTC_OVERSCAN_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmCRTC4_CRTC_BLANK_DATA_COLOR_DEFAULT                                    0x00000000
-#define mmCRTC4_CRTC_BLANK_DATA_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTC4_CRTC_BLACK_COLOR_DEFAULT                                         0x00000000
-#define mmCRTC4_CRTC_BLACK_COLOR_EXT_DEFAULT                                     0x00000000
-#define mmCRTC4_CRTC_VERTICAL_INTERRUPT0_POSITION_DEFAULT                        0x00000000
-#define mmCRTC4_CRTC_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC4_CRTC_VERTICAL_INTERRUPT1_POSITION_DEFAULT                        0x00000000
-#define mmCRTC4_CRTC_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC4_CRTC_VERTICAL_INTERRUPT2_POSITION_DEFAULT                        0x00000000
-#define mmCRTC4_CRTC_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC4_CRTC_CRC_CNTL_DEFAULT                                            0x00000000
-#define mmCRTC4_CRTC_CRC0_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC0_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC0_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC0_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC0_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC4_CRTC_CRC0_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC4_CRTC_CRC1_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC1_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC1_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC1_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC4_CRTC_CRC1_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC4_CRTC_CRC1_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC4_CRTC_EXT_TIMING_SYNC_CONTROL_DEFAULT                             0x00000000
-#define mmCRTC4_CRTC_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                        0x00000000
-#define mmCRTC4_CRTC_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                          0x00000000
-#define mmCRTC4_CRTC_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT              0x00000000
-#define mmCRTC4_CRTC_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                   0x00000000
-#define mmCRTC4_CRTC_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTC4_CRTC_STATIC_SCREEN_CONTROL_DEFAULT                               0x00010000
-#define mmCRTC4_CRTC_3D_STRUCTURE_CONTROL_DEFAULT                                0x00000010
-#define mmCRTC4_CRTC_GSL_VSYNC_GAP_DEFAULT                                       0x00000000
-#define mmCRTC4_CRTC_GSL_WINDOW_DEFAULT                                          0x00000000
-#define mmCRTC4_CRTC_GSL_CONTROL_DEFAULT                                         0x00020000
-#define mmCRTC4_CRTC_RANGE_TIMING_INT_STATUS_DEFAULT                             0x00000000
-#define mmCRTC4_CRTC_DRR_CONTROL_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_fmt4_dispdec
-#define mmFMT4_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT4_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT4_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT4_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT4_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT4_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT4_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT4_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT4_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT4_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT4_FMT_CRC_CNTL_DEFAULT                                              0x01000040
-#define mmFMT4_FMT_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0x00ff00ff
-#define mmFMT4_FMT_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0x000700ff
-#define mmFMT4_FMT_CRC_SIG_RED_GREEN_DEFAULT                                     0x00000000
-#define mmFMT4_FMT_CRC_SIG_BLUE_CONTROL_DEFAULT                                  0x00000000
-#define mmFMT4_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT4_FMT_420_HBLANK_EARLY_START_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_dcp5_dispdec
-#define mmDCP5_GRPH_ENABLE_DEFAULT                                               0x00000001
-#define mmDCP5_GRPH_CONTROL_DEFAULT                                              0x20002040
-#define mmDCP5_GRPH_LUT_10BIT_BYPASS_DEFAULT                                     0x00000000
-#define mmDCP5_GRPH_SWAP_CNTL_DEFAULT                                            0x00000000
-#define mmDCP5_GRPH_PRIMARY_SURFACE_ADDRESS_DEFAULT                              0x00000000
-#define mmDCP5_GRPH_SECONDARY_SURFACE_ADDRESS_DEFAULT                            0x00000000
-#define mmDCP5_GRPH_PITCH_DEFAULT                                                0x00000000
-#define mmDCP5_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_DEFAULT                         0x00000000
-#define mmDCP5_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_DEFAULT                       0x00000000
-#define mmDCP5_GRPH_SURFACE_OFFSET_X_DEFAULT                                     0x00000000
-#define mmDCP5_GRPH_SURFACE_OFFSET_Y_DEFAULT                                     0x00000000
-#define mmDCP5_GRPH_X_START_DEFAULT                                              0x00000000
-#define mmDCP5_GRPH_Y_START_DEFAULT                                              0x00000000
-#define mmDCP5_GRPH_X_END_DEFAULT                                                0x00000000
-#define mmDCP5_GRPH_Y_END_DEFAULT                                                0x00000000
-#define mmDCP5_INPUT_GAMMA_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP5_GRPH_UPDATE_DEFAULT                                               0x00000000
-#define mmDCP5_GRPH_FLIP_CONTROL_DEFAULT                                         0x00000020
-#define mmDCP5_GRPH_SURFACE_ADDRESS_INUSE_DEFAULT                                0x00000000
-#define mmDCP5_GRPH_DFQ_CONTROL_DEFAULT                                          0x00000000
-#define mmDCP5_GRPH_DFQ_STATUS_DEFAULT                                           0x00000000
-#define mmDCP5_GRPH_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDCP5_GRPH_INTERRUPT_CONTROL_DEFAULT                                    0x00000000
-#define mmDCP5_GRPH_SURFACE_ADDRESS_HIGH_INUSE_DEFAULT                           0x00000000
-#define mmDCP5_GRPH_COMPRESS_SURFACE_ADDRESS_DEFAULT                             0x00000000
-#define mmDCP5_GRPH_COMPRESS_PITCH_DEFAULT                                       0x00000000
-#define mmDCP5_GRPH_COMPRESS_SURFACE_ADDRESS_HIGH_DEFAULT                        0x00000000
-#define mmDCP5_GRPH_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                       0x000000ff
-#define mmDCP5_PRESCALE_GRPH_CONTROL_DEFAULT                                     0x00000010
-#define mmDCP5_PRESCALE_VALUES_GRPH_R_DEFAULT                                    0x20000000
-#define mmDCP5_PRESCALE_VALUES_GRPH_G_DEFAULT                                    0x20000000
-#define mmDCP5_PRESCALE_VALUES_GRPH_B_DEFAULT                                    0x20000000
-#define mmDCP5_INPUT_CSC_CONTROL_DEFAULT                                         0x00000000
-#define mmDCP5_INPUT_CSC_C11_C12_DEFAULT                                         0x00002000
-#define mmDCP5_INPUT_CSC_C13_C14_DEFAULT                                         0x00000000
-#define mmDCP5_INPUT_CSC_C21_C22_DEFAULT                                         0x20000000
-#define mmDCP5_INPUT_CSC_C23_C24_DEFAULT                                         0x00000000
-#define mmDCP5_INPUT_CSC_C31_C32_DEFAULT                                         0x00000000
-#define mmDCP5_INPUT_CSC_C33_C34_DEFAULT                                         0x00002000
-#define mmDCP5_OUTPUT_CSC_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP5_OUTPUT_CSC_C11_C12_DEFAULT                                        0x00002000
-#define mmDCP5_OUTPUT_CSC_C13_C14_DEFAULT                                        0x00000000
-#define mmDCP5_OUTPUT_CSC_C21_C22_DEFAULT                                        0x20000000
-#define mmDCP5_OUTPUT_CSC_C23_C24_DEFAULT                                        0x00000000
-#define mmDCP5_OUTPUT_CSC_C31_C32_DEFAULT                                        0x00000000
-#define mmDCP5_OUTPUT_CSC_C33_C34_DEFAULT                                        0x00002000
-#define mmDCP5_COMM_MATRIXA_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP5_COMM_MATRIXA_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP5_COMM_MATRIXA_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP5_COMM_MATRIXA_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP5_COMM_MATRIXA_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP5_COMM_MATRIXA_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP5_COMM_MATRIXB_TRANS_C11_C12_DEFAULT                                0x00002000
-#define mmDCP5_COMM_MATRIXB_TRANS_C13_C14_DEFAULT                                0x00000000
-#define mmDCP5_COMM_MATRIXB_TRANS_C21_C22_DEFAULT                                0x20000000
-#define mmDCP5_COMM_MATRIXB_TRANS_C23_C24_DEFAULT                                0x00000000
-#define mmDCP5_COMM_MATRIXB_TRANS_C31_C32_DEFAULT                                0x00000000
-#define mmDCP5_COMM_MATRIXB_TRANS_C33_C34_DEFAULT                                0x00002000
-#define mmDCP5_DENORM_CONTROL_DEFAULT                                            0x00000003
-#define mmDCP5_OUT_ROUND_CONTROL_DEFAULT                                         0x0000000a
-#define mmDCP5_OUT_CLAMP_CONTROL_R_CR_DEFAULT                                    0x00003fff
-#define mmDCP5_OUT_CLAMP_CONTROL_G_Y_DEFAULT                                     0x00003fff
-#define mmDCP5_OUT_CLAMP_CONTROL_B_CB_DEFAULT                                    0x00003fff
-#define mmDCP5_KEY_CONTROL_DEFAULT                                               0x00000000
-#define mmDCP5_KEY_RANGE_ALPHA_DEFAULT                                           0x00000000
-#define mmDCP5_KEY_RANGE_RED_DEFAULT                                             0x00000000
-#define mmDCP5_KEY_RANGE_GREEN_DEFAULT                                           0x00000000
-#define mmDCP5_KEY_RANGE_BLUE_DEFAULT                                            0x00000000
-#define mmDCP5_DEGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP5_GAMUT_REMAP_CONTROL_DEFAULT                                       0x00000000
-#define mmDCP5_GAMUT_REMAP_C11_C12_DEFAULT                                       0x00002000
-#define mmDCP5_GAMUT_REMAP_C13_C14_DEFAULT                                       0x00000000
-#define mmDCP5_GAMUT_REMAP_C21_C22_DEFAULT                                       0x20000000
-#define mmDCP5_GAMUT_REMAP_C23_C24_DEFAULT                                       0x00000000
-#define mmDCP5_GAMUT_REMAP_C31_C32_DEFAULT                                       0x00000000
-#define mmDCP5_GAMUT_REMAP_C33_C34_DEFAULT                                       0x00002000
-#define mmDCP5_DCP_SPATIAL_DITHER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP5_DCP_RANDOM_SEEDS_DEFAULT                                          0x00000000
-#define mmDCP5_DCP_FP_CONVERTED_FIELD_DEFAULT                                    0x00000000
-#define mmDCP5_CUR_CONTROL_DEFAULT                                               0x00000810
-#define mmDCP5_CUR_SURFACE_ADDRESS_DEFAULT                                       0x00000000
-#define mmDCP5_CUR_SIZE_DEFAULT                                                  0x00000000
-#define mmDCP5_CUR_SURFACE_ADDRESS_HIGH_DEFAULT                                  0x00000000
-#define mmDCP5_CUR_POSITION_DEFAULT                                              0x00000000
-#define mmDCP5_CUR_HOT_SPOT_DEFAULT                                              0x00000000
-#define mmDCP5_CUR_COLOR1_DEFAULT                                                0x00000000
-#define mmDCP5_CUR_COLOR2_DEFAULT                                                0x00000000
-#define mmDCP5_CUR_UPDATE_DEFAULT                                                0x00000000
-#define mmDCP5_CUR_REQUEST_FILTER_CNTL_DEFAULT                                   0x00000000
-#define mmDCP5_CUR_STEREO_CONTROL_DEFAULT                                        0x00000000
-#define mmDCP5_DC_LUT_RW_MODE_DEFAULT                                            0x00000000
-#define mmDCP5_DC_LUT_RW_INDEX_DEFAULT                                           0x00000000
-#define mmDCP5_DC_LUT_SEQ_COLOR_DEFAULT                                          0x00000000
-#define mmDCP5_DC_LUT_PWL_DATA_DEFAULT                                           0x00000000
-#define mmDCP5_DC_LUT_30_COLOR_DEFAULT                                           0x00000000
-#define mmDCP5_DC_LUT_VGA_ACCESS_ENABLE_DEFAULT                                  0x00000000
-#define mmDCP5_DC_LUT_WRITE_EN_MASK_DEFAULT                                      0x00000007
-#define mmDCP5_DC_LUT_AUTOFILL_DEFAULT                                           0x00000000
-#define mmDCP5_DC_LUT_CONTROL_DEFAULT                                            0x00000000
-#define mmDCP5_DC_LUT_BLACK_OFFSET_BLUE_DEFAULT                                  0x00000000
-#define mmDCP5_DC_LUT_BLACK_OFFSET_GREEN_DEFAULT                                 0x00000000
-#define mmDCP5_DC_LUT_BLACK_OFFSET_RED_DEFAULT                                   0x00000000
-#define mmDCP5_DC_LUT_WHITE_OFFSET_BLUE_DEFAULT                                  0x0000ffff
-#define mmDCP5_DC_LUT_WHITE_OFFSET_GREEN_DEFAULT                                 0x0000ffff
-#define mmDCP5_DC_LUT_WHITE_OFFSET_RED_DEFAULT                                   0x0000ffff
-#define mmDCP5_DCP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP5_DCP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmDCP5_DCP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmDCP5_DVMM_PTE_CONTROL_DEFAULT                                          0x00004000
-#define mmDCP5_DCP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmDCP5_DVMM_PTE_ARB_CONTROL_DEFAULT                                      0x00002220
-#define mmDCP5_GRPH_FLIP_RATE_CNTL_DEFAULT                                       0x00000000
-#define mmDCP5_DCP_GSL_CONTROL_DEFAULT                                           0x60000020
-#define mmDCP5_DCP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000035
-#define mmDCP5_GRPH_STEREOSYNC_FLIP_DEFAULT                                      0x00000200
-#define mmDCP5_HW_ROTATION_DEFAULT                                               0x00000000
-#define mmDCP5_GRPH_XDMA_CACHE_UNDERFLOW_DET_CNTL_DEFAULT                        0x00000010
-#define mmDCP5_REGAMMA_CONTROL_DEFAULT                                           0x00000000
-#define mmDCP5_REGAMMA_LUT_INDEX_DEFAULT                                         0x00000000
-#define mmDCP5_REGAMMA_LUT_DATA_DEFAULT                                          0x00000000
-#define mmDCP5_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                                 0x00000007
-#define mmDCP5_REGAMMA_CNTLA_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLA_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP5_REGAMMA_CNTLA_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP5_REGAMMA_CNTLA_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP5_REGAMMA_CNTLB_START_CNTL_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLB_END_CNTL1_DEFAULT                                   0x00000000
-#define mmDCP5_REGAMMA_CNTLB_END_CNTL2_DEFAULT                                   0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_0_1_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_2_3_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_4_5_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_6_7_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_8_9_DEFAULT                                  0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_10_11_DEFAULT                                0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_12_13_DEFAULT                                0x00000000
-#define mmDCP5_REGAMMA_CNTLB_REGION_14_15_DEFAULT                                0x00000000
-#define mmDCP5_ALPHA_CONTROL_DEFAULT                                             0x00000002
-#define mmDCP5_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_DEFAULT                        0x00000000
-#define mmDCP5_GRPH_XDMA_RECOVERY_SURFACE_ADDRESS_HIGH_DEFAULT                   0x00000000
-#define mmDCP5_GRPH_XDMA_CACHE_UNDERFLOW_DET_STATUS_DEFAULT                      0x00000000
-#define mmDCP5_GRPH_XDMA_FLIP_TIMEOUT_DEFAULT                                    0x00000000
-#define mmDCP5_GRPH_XDMA_FLIP_AVG_DELAY_DEFAULT                                  0x00000000
-#define mmDCP5_GRPH_SURFACE_COUNTER_CONTROL_DEFAULT                              0x00000012
-#define mmDCP5_GRPH_SURFACE_COUNTER_OUTPUT_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_lb5_dispdec
-#define mmLB5_LB_DATA_FORMAT_DEFAULT                                             0x00000000
-#define mmLB5_LB_MEMORY_CTRL_DEFAULT                                             0x000006b0
-#define mmLB5_LB_MEMORY_SIZE_STATUS_DEFAULT                                      0x00000000
-#define mmLB5_LB_DESKTOP_HEIGHT_DEFAULT                                          0x00000000
-#define mmLB5_LB_VLINE_START_END_DEFAULT                                         0x00000000
-#define mmLB5_LB_VLINE2_START_END_DEFAULT                                        0x00000000
-#define mmLB5_LB_V_COUNTER_DEFAULT                                               0x00000000
-#define mmLB5_LB_SNAPSHOT_V_COUNTER_DEFAULT                                      0x00000000
-#define mmLB5_LB_INTERRUPT_MASK_DEFAULT                                          0x00000000
-#define mmLB5_LB_VLINE_STATUS_DEFAULT                                            0x00000000
-#define mmLB5_LB_VLINE2_STATUS_DEFAULT                                           0x00000000
-#define mmLB5_LB_VBLANK_STATUS_DEFAULT                                           0x00000000
-#define mmLB5_LB_SYNC_RESET_SEL_DEFAULT                                          0x00000002
-#define mmLB5_LB_BLACK_KEYER_R_CR_DEFAULT                                        0x00000000
-#define mmLB5_LB_BLACK_KEYER_G_Y_DEFAULT                                         0x00000000
-#define mmLB5_LB_BLACK_KEYER_B_CB_DEFAULT                                        0x00000000
-#define mmLB5_LB_KEYER_COLOR_CTRL_DEFAULT                                        0x00000000
-#define mmLB5_LB_KEYER_COLOR_R_CR_DEFAULT                                        0x00000000
-#define mmLB5_LB_KEYER_COLOR_G_Y_DEFAULT                                         0x00000000
-#define mmLB5_LB_KEYER_COLOR_B_CB_DEFAULT                                        0x00000000
-#define mmLB5_LB_KEYER_COLOR_REP_R_CR_DEFAULT                                    0x00000000
-#define mmLB5_LB_KEYER_COLOR_REP_G_Y_DEFAULT                                     0x00000000
-#define mmLB5_LB_KEYER_COLOR_REP_B_CB_DEFAULT                                    0x00000000
-#define mmLB5_LB_BUFFER_LEVEL_STATUS_DEFAULT                                     0xa0008000
-#define mmLB5_LB_BUFFER_URGENCY_CTRL_DEFAULT                                     0x00200010
-#define mmLB5_LB_BUFFER_URGENCY_STATUS_DEFAULT                                   0x00000000
-#define mmLB5_LB_BUFFER_STATUS_DEFAULT                                           0x00000002
-#define mmLB5_LB_NO_OUTSTANDING_REQ_STATUS_DEFAULT                               0x00000000
-#define mmLB5_MVP_AFR_FLIP_MODE_DEFAULT                                          0x00000000
-#define mmLB5_MVP_AFR_FLIP_FIFO_CNTL_DEFAULT                                     0x00000000
-#define mmLB5_MVP_FLIP_LINE_NUM_INSERT_DEFAULT                                   0x00000002
-#define mmLB5_DC_MVP_LB_CONTROL_DEFAULT                                          0x00000001
-
-
-// addressBlock: dce_dc_dcfe5_dispdec
-#define mmDCFE5_DCFE_CLOCK_CONTROL_DEFAULT                                       0x00000000
-#define mmDCFE5_DCFE_SOFT_RESET_DEFAULT                                          0x00000000
-#define mmDCFE5_DCFE_MEM_PWR_CTRL_DEFAULT                                        0x00000000
-#define mmDCFE5_DCFE_MEM_PWR_CTRL2_DEFAULT                                       0x00000000
-#define mmDCFE5_DCFE_MEM_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmDCFE5_DCFE_MISC_DEFAULT                                                0x00000001
-#define mmDCFE5_DCFE_FLUSH_DEFAULT                                               0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon8_dispdec
-#define mmDC_PERFMON8_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON8_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON8_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON8_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON8_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON8_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON8_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON8_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON8_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dmif_pg5_dispdec
-#define mmDMIF_PG5_DPG_PIPE_ARBITRATION_CONTROL1_DEFAULT                         0x00000000
-#define mmDMIF_PG5_DPG_PIPE_ARBITRATION_CONTROL2_DEFAULT                         0x00000000
-#define mmDMIF_PG5_DPG_WATERMARK_MASK_CONTROL_DEFAULT                            0x000bf777
-#define mmDMIF_PG5_DPG_PIPE_URGENCY_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG5_DPG_PIPE_URGENT_LEVEL_CONTROL_DEFAULT                         0x00000000
-#define mmDMIF_PG5_DPG_PIPE_STUTTER_CONTROL_DEFAULT                              0x00000000
-#define mmDMIF_PG5_DPG_PIPE_STUTTER_CONTROL2_DEFAULT                             0x00000000
-#define mmDMIF_PG5_DPG_PIPE_LOW_POWER_CONTROL_DEFAULT                            0x00000000
-#define mmDMIF_PG5_DPG_REPEATER_PROGRAM_DEFAULT                                  0x00000000
-#define mmDMIF_PG5_DPG_CHK_PRE_PROC_CNTL_DEFAULT                                 0x00000000
-#define mmDMIF_PG5_DPG_DVMM_STATUS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_scl5_dispdec
-#define mmSCL5_SCL_COEF_RAM_SELECT_DEFAULT                                       0x00000000
-#define mmSCL5_SCL_COEF_RAM_TAP_DATA_DEFAULT                                     0x00000000
-#define mmSCL5_SCL_MODE_DEFAULT                                                  0x00000000
-#define mmSCL5_SCL_TAP_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL5_SCL_CONTROL_DEFAULT                                               0x00000000
-#define mmSCL5_SCL_BYPASS_CONTROL_DEFAULT                                        0x00000000
-#define mmSCL5_SCL_MANUAL_REPLICATE_CONTROL_DEFAULT                              0x00000000
-#define mmSCL5_SCL_AUTOMATIC_MODE_CONTROL_DEFAULT                                0x00000000
-#define mmSCL5_SCL_HORZ_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL5_SCL_HORZ_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL5_SCL_HORZ_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL5_SCL_VERT_FILTER_CONTROL_DEFAULT                                   0x00000000
-#define mmSCL5_SCL_VERT_FILTER_SCALE_RATIO_DEFAULT                               0x00000000
-#define mmSCL5_SCL_VERT_FILTER_INIT_DEFAULT                                      0x01000000
-#define mmSCL5_SCL_VERT_FILTER_INIT_BOT_DEFAULT                                  0x01000000
-#define mmSCL5_SCL_ROUND_OFFSET_DEFAULT                                          0x80000000
-#define mmSCL5_SCL_UPDATE_DEFAULT                                                0x00000000
-#define mmSCL5_SCL_F_SHARP_CONTROL_DEFAULT                                       0x00000000
-#define mmSCL5_SCL_ALU_CONTROL_DEFAULT                                           0x00000000
-#define mmSCL5_SCL_COEF_RAM_CONFLICT_STATUS_DEFAULT                              0x00000000
-#define mmSCL5_VIEWPORT_START_SECONDARY_DEFAULT                                  0x00000000
-#define mmSCL5_VIEWPORT_START_DEFAULT                                            0x00000000
-#define mmSCL5_VIEWPORT_SIZE_DEFAULT                                             0x00000000
-#define mmSCL5_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                                   0x00000000
-#define mmSCL5_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                                   0x00000000
-#define mmSCL5_SCL_MODE_CHANGE_DET1_DEFAULT                                      0x00000000
-#define mmSCL5_SCL_MODE_CHANGE_DET2_DEFAULT                                      0x00000000
-#define mmSCL5_SCL_MODE_CHANGE_DET3_DEFAULT                                      0x00000000
-#define mmSCL5_SCL_MODE_CHANGE_MASK_DEFAULT                                      0x00000000
-
-
-// addressBlock: dce_dc_blnd5_dispdec
-#define mmBLND5_BLND_CONTROL_DEFAULT                                             0xff0220ff
-#define mmBLND5_BLND_SM_CONTROL2_DEFAULT                                         0x00000000
-#define mmBLND5_BLND_CONTROL2_DEFAULT                                            0x00000010
-#define mmBLND5_BLND_UPDATE_DEFAULT                                              0x00000000
-#define mmBLND5_BLND_UNDERFLOW_INTERRUPT_DEFAULT                                 0x00000000
-#define mmBLND5_BLND_V_UPDATE_LOCK_DEFAULT                                       0x80000000
-#define mmBLND5_BLND_REG_UPDATE_STATUS_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_crtc5_dispdec
-#define mmCRTC5_CRTC_H_BLANK_EARLY_NUM_DEFAULT                                   0x00000040
-#define mmCRTC5_CRTC_H_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC5_CRTC_H_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_H_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC5_CRTC_H_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_H_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC5_CRTC_H_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_VBI_END_DEFAULT                                             0x00000003
-#define mmCRTC5_CRTC_V_TOTAL_DEFAULT                                             0x00000000
-#define mmCRTC5_CRTC_V_TOTAL_MIN_DEFAULT                                         0x00000000
-#define mmCRTC5_CRTC_V_TOTAL_MAX_DEFAULT                                         0x00000000
-#define mmCRTC5_CRTC_V_TOTAL_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTC5_CRTC_V_TOTAL_INT_STATUS_DEFAULT                                  0x00000000
-#define mmCRTC5_CRTC_VSYNC_NOM_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTC5_CRTC_V_BLANK_START_END_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_V_SYNC_A_DEFAULT                                            0x00000000
-#define mmCRTC5_CRTC_V_SYNC_A_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_V_SYNC_B_DEFAULT                                            0x00000000
-#define mmCRTC5_CRTC_V_SYNC_B_CNTL_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_DTMTEST_CNTL_DEFAULT                                        0x00000000
-#define mmCRTC5_CRTC_DTMTEST_STATUS_POSITION_DEFAULT                             0x00000000
-#define mmCRTC5_CRTC_TRIGA_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC5_CRTC_TRIGA_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_TRIGB_CNTL_DEFAULT                                          0x00000000
-#define mmCRTC5_CRTC_TRIGB_MANUAL_TRIG_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_FORCE_COUNT_NOW_CNTL_DEFAULT                                0x00000000
-#define mmCRTC5_CRTC_FLOW_CONTROL_DEFAULT                                        0x00000000
-#define mmCRTC5_CRTC_STEREO_FORCE_NEXT_EYE_DEFAULT                               0x00000000
-#define mmCRTC5_CRTC_AVSYNC_COUNTER_DEFAULT                                      0x00000000
-#define mmCRTC5_CRTC_CONTROL_DEFAULT                                             0x80400110
-#define mmCRTC5_CRTC_BLANK_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_INTERLACE_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_INTERLACE_STATUS_DEFAULT                                    0x00000000
-#define mmCRTC5_CRTC_FIELD_INDICATION_CONTROL_DEFAULT                            0x00000000
-#define mmCRTC5_CRTC_PIXEL_DATA_READBACK0_DEFAULT                                0x00000000
-#define mmCRTC5_CRTC_PIXEL_DATA_READBACK1_DEFAULT                                0x00000000
-#define mmCRTC5_CRTC_STATUS_DEFAULT                                              0x00000000
-#define mmCRTC5_CRTC_STATUS_POSITION_DEFAULT                                     0x00000000
-#define mmCRTC5_CRTC_NOM_VERT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_STATUS_FRAME_COUNT_DEFAULT                                  0x00000000
-#define mmCRTC5_CRTC_STATUS_VF_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC5_CRTC_STATUS_HV_COUNT_DEFAULT                                     0x00000000
-#define mmCRTC5_CRTC_COUNT_CONTROL_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_COUNT_RESET_DEFAULT                                         0x00000000
-#define mmCRTC5_CRTC_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                        0x00000000
-#define mmCRTC5_CRTC_VERT_SYNC_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_STEREO_STATUS_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_STEREO_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTC5_CRTC_SNAPSHOT_STATUS_DEFAULT                                     0x00000000
-#define mmCRTC5_CRTC_SNAPSHOT_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTC5_CRTC_SNAPSHOT_POSITION_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_SNAPSHOT_FRAME_DEFAULT                                      0x00000000
-#define mmCRTC5_CRTC_START_LINE_CONTROL_DEFAULT                                  0x00003002
-#define mmCRTC5_CRTC_INTERRUPT_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTC5_CRTC_UPDATE_LOCK_DEFAULT                                         0x00000000
-#define mmCRTC5_CRTC_DOUBLE_BUFFER_CONTROL_DEFAULT                               0x00000000
-#define mmCRTC5_CRTC_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                          0x00000000
-#define mmCRTC5_CRTC_TEST_PATTERN_CONTROL_DEFAULT                                0x00000000
-#define mmCRTC5_CRTC_TEST_PATTERN_PARAMETERS_DEFAULT                             0x00000000
-#define mmCRTC5_CRTC_TEST_PATTERN_COLOR_DEFAULT                                  0x00000000
-#define mmCRTC5_CRTC_MASTER_UPDATE_LOCK_DEFAULT                                  0x00010000
-#define mmCRTC5_CRTC_MASTER_UPDATE_MODE_DEFAULT                                  0x00000000
-#define mmCRTC5_CRTC_MVP_INBAND_CNTL_INSERT_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                        0x00000008
-#define mmCRTC5_CRTC_MVP_STATUS_DEFAULT                                          0x00000000
-#define mmCRTC5_CRTC_MASTER_EN_DEFAULT                                           0x00000000
-#define mmCRTC5_CRTC_ALLOW_STOP_OFF_V_CNT_DEFAULT                                0x00010000
-#define mmCRTC5_CRTC_V_UPDATE_INT_STATUS_DEFAULT                                 0x00000000
-#define mmCRTC5_CRTC_OVERSCAN_COLOR_DEFAULT                                      0x00000000
-#define mmCRTC5_CRTC_OVERSCAN_COLOR_EXT_DEFAULT                                  0x00000000
-#define mmCRTC5_CRTC_BLANK_DATA_COLOR_DEFAULT                                    0x00000000
-#define mmCRTC5_CRTC_BLANK_DATA_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTC5_CRTC_BLACK_COLOR_DEFAULT                                         0x00000000
-#define mmCRTC5_CRTC_BLACK_COLOR_EXT_DEFAULT                                     0x00000000
-#define mmCRTC5_CRTC_VERTICAL_INTERRUPT0_POSITION_DEFAULT                        0x00000000
-#define mmCRTC5_CRTC_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC5_CRTC_VERTICAL_INTERRUPT1_POSITION_DEFAULT                        0x00000000
-#define mmCRTC5_CRTC_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC5_CRTC_VERTICAL_INTERRUPT2_POSITION_DEFAULT                        0x00000000
-#define mmCRTC5_CRTC_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                         0x00000000
-#define mmCRTC5_CRTC_CRC_CNTL_DEFAULT                                            0x00000000
-#define mmCRTC5_CRTC_CRC0_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC0_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC0_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC0_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC0_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC5_CRTC_CRC0_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC5_CRTC_CRC1_WINDOWA_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC1_WINDOWA_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC1_WINDOWB_X_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC1_WINDOWB_Y_CONTROL_DEFAULT                              0x00000000
-#define mmCRTC5_CRTC_CRC1_DATA_RG_DEFAULT                                        0x00000000
-#define mmCRTC5_CRTC_CRC1_DATA_B_DEFAULT                                         0x00000000
-#define mmCRTC5_CRTC_EXT_TIMING_SYNC_CONTROL_DEFAULT                             0x00000000
-#define mmCRTC5_CRTC_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                        0x00000000
-#define mmCRTC5_CRTC_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                          0x00000000
-#define mmCRTC5_CRTC_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT              0x00000000
-#define mmCRTC5_CRTC_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                   0x00000000
-#define mmCRTC5_CRTC_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTC5_CRTC_STATIC_SCREEN_CONTROL_DEFAULT                               0x00010000
-#define mmCRTC5_CRTC_3D_STRUCTURE_CONTROL_DEFAULT                                0x00000010
-#define mmCRTC5_CRTC_GSL_VSYNC_GAP_DEFAULT                                       0x00000000
-#define mmCRTC5_CRTC_GSL_WINDOW_DEFAULT                                          0x00000000
-#define mmCRTC5_CRTC_GSL_CONTROL_DEFAULT                                         0x00020000
-#define mmCRTC5_CRTC_RANGE_TIMING_INT_STATUS_DEFAULT                             0x00000000
-#define mmCRTC5_CRTC_DRR_CONTROL_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_fmt5_dispdec
-#define mmFMT5_FMT_CLAMP_COMPONENT_R_DEFAULT                                     0x00000000
-#define mmFMT5_FMT_CLAMP_COMPONENT_G_DEFAULT                                     0x00000000
-#define mmFMT5_FMT_CLAMP_COMPONENT_B_DEFAULT                                     0x00000000
-#define mmFMT5_FMT_DYNAMIC_EXP_CNTL_DEFAULT                                      0x00000000
-#define mmFMT5_FMT_CONTROL_DEFAULT                                               0x00000000
-#define mmFMT5_FMT_BIT_DEPTH_CONTROL_DEFAULT                                     0x00600000
-#define mmFMT5_FMT_DITHER_RAND_R_SEED_DEFAULT                                    0x00000000
-#define mmFMT5_FMT_DITHER_RAND_G_SEED_DEFAULT                                    0x00000099
-#define mmFMT5_FMT_DITHER_RAND_B_SEED_DEFAULT                                    0x000000dd
-#define mmFMT5_FMT_CLAMP_CNTL_DEFAULT                                            0x00000000
-#define mmFMT5_FMT_CRC_CNTL_DEFAULT                                              0x01000040
-#define mmFMT5_FMT_CRC_SIG_RED_GREEN_MASK_DEFAULT                                0x00ff00ff
-#define mmFMT5_FMT_CRC_SIG_BLUE_CONTROL_MASK_DEFAULT                             0x000700ff
-#define mmFMT5_FMT_CRC_SIG_RED_GREEN_DEFAULT                                     0x00000000
-#define mmFMT5_FMT_CRC_SIG_BLUE_CONTROL_DEFAULT                                  0x00000000
-#define mmFMT5_FMT_SIDE_BY_SIDE_STEREO_CONTROL_DEFAULT                           0x00000000
-#define mmFMT5_FMT_420_HBLANK_EARLY_START_DEFAULT                                0x00000000
-
-
-// addressBlock: dce_dc_unp0_dispdec
-#define mmUNP0_UNP_GRPH_ENABLE_DEFAULT                                           0x00000001
-#define mmUNP0_UNP_GRPH_CONTROL_DEFAULT                                          0x0a008008
-#define mmUNP0_UNP_GRPH_CONTROL_C_DEFAULT                                        0x00008000
-#define mmUNP0_UNP_GRPH_CONTROL_EXP_DEFAULT                                      0x00000000
-#define mmUNP0_UNP_GRPH_SWAP_CNTL_DEFAULT                                        0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_L_DEFAULT                        0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_C_DEFAULT                        0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_L_DEFAULT                   0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                   0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_L_DEFAULT                 0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_C_DEFAULT                 0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_HIGH_L_DEFAULT            0x00000000
-#define mmUNP0_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_HIGH_C_DEFAULT            0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_L_DEFAULT                      0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_C_DEFAULT                      0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_L_DEFAULT                 0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                 0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_L_DEFAULT               0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_C_DEFAULT               0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_HIGH_L_DEFAULT          0x00000000
-#define mmUNP0_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_HIGH_C_DEFAULT          0x00000000
-#define mmUNP0_UNP_GRPH_PITCH_L_DEFAULT                                          0x00000000
-#define mmUNP0_UNP_GRPH_PITCH_C_DEFAULT                                          0x00000000
-#define mmUNP0_UNP_GRPH_SURFACE_OFFSET_X_L_DEFAULT                               0x00000000
-#define mmUNP0_UNP_GRPH_SURFACE_OFFSET_X_C_DEFAULT                               0x00000000
-#define mmUNP0_UNP_GRPH_SURFACE_OFFSET_Y_L_DEFAULT                               0x00000000
-#define mmUNP0_UNP_GRPH_SURFACE_OFFSET_Y_C_DEFAULT                               0x00000000
-#define mmUNP0_UNP_GRPH_X_START_L_DEFAULT                                        0x00000000
-#define mmUNP0_UNP_GRPH_X_START_C_DEFAULT                                        0x00000000
-#define mmUNP0_UNP_GRPH_Y_START_L_DEFAULT                                        0x00000000
-#define mmUNP0_UNP_GRPH_Y_START_C_DEFAULT                                        0x00000000
-#define mmUNP0_UNP_GRPH_X_END_L_DEFAULT                                          0x00000000
-#define mmUNP0_UNP_GRPH_X_END_C_DEFAULT                                          0x00000000
-#define mmUNP0_UNP_GRPH_Y_END_L_DEFAULT                                          0x00000000
-#define mmUNP0_UNP_GRPH_Y_END_C_DEFAULT                                          0x00000000
-#define mmUNP0_UNP_GRPH_UPDATE_DEFAULT                                           0x00000000
-#define mmUNP0_UNP_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                        0x0000ffff
-#define mmUNP0_UNP_GRPH_SURFACE_ADDRESS_INUSE_L_DEFAULT                          0x00000000
-#define mmUNP0_UNP_GRPH_SURFACE_ADDRESS_INUSE_C_DEFAULT                          0x00000000
-#define mmUNP0_UNP_GRPH_SURFACE_ADDRESS_HIGH_INUSE_L_DEFAULT                     0x00000000
-#define mmUNP0_UNP_GRPH_SURFACE_ADDRESS_HIGH_INUSE_C_DEFAULT                     0x00000000
-#define mmUNP0_UNP_DVMM_PTE_CONTROL_DEFAULT                                      0x00004000
-#define mmUNP0_UNP_DVMM_PTE_CONTROL_C_DEFAULT                                    0x00004000
-#define mmUNP0_UNP_DVMM_PTE_ARB_CONTROL_DEFAULT                                  0x00002220
-#define mmUNP0_UNP_DVMM_PTE_ARB_CONTROL_C_DEFAULT                                0x00002220
-#define mmUNP0_UNP_GRPH_INTERRUPT_STATUS_DEFAULT                                 0x00000000
-#define mmUNP0_UNP_GRPH_INTERRUPT_CONTROL_DEFAULT                                0x00000000
-#define mmUNP0_UNP_GRPH_STEREOSYNC_FLIP_DEFAULT                                  0x00002020
-#define mmUNP0_UNP_FLIP_CONTROL_DEFAULT                                          0x00000001
-#define mmUNP0_UNP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmUNP0_UNP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmUNP0_UNP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmUNP0_UNP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmUNP0_UNP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000100
-#define mmUNP0_UNP_HW_ROTATION_DEFAULT                                           0x00000010
-
-
-// addressBlock: dce_dc_lbv0_dispdec
-#define mmLBV0_LBV_DATA_FORMAT_DEFAULT                                           0x00000000
-#define mmLBV0_LBV_MEMORY_CTRL_DEFAULT                                           0x000006b0
-#define mmLBV0_LBV_MEMORY_SIZE_STATUS_DEFAULT                                    0x00000000
-#define mmLBV0_LBV_DESKTOP_HEIGHT_DEFAULT                                        0x00000000
-#define mmLBV0_LBV_VLINE_START_END_DEFAULT                                       0x00000000
-#define mmLBV0_LBV_VLINE2_START_END_DEFAULT                                      0x00000000
-#define mmLBV0_LBV_V_COUNTER_DEFAULT                                             0x00000000
-#define mmLBV0_LBV_SNAPSHOT_V_COUNTER_DEFAULT                                    0x00000000
-#define mmLBV0_LBV_V_COUNTER_CHROMA_DEFAULT                                      0x00000000
-#define mmLBV0_LBV_SNAPSHOT_V_COUNTER_CHROMA_DEFAULT                             0x00000000
-#define mmLBV0_LBV_INTERRUPT_MASK_DEFAULT                                        0x00000000
-#define mmLBV0_LBV_VLINE_STATUS_DEFAULT                                          0x00000000
-#define mmLBV0_LBV_VLINE2_STATUS_DEFAULT                                         0x00000000
-#define mmLBV0_LBV_VBLANK_STATUS_DEFAULT                                         0x00000000
-#define mmLBV0_LBV_SYNC_RESET_SEL_DEFAULT                                        0x00000002
-#define mmLBV0_LBV_BLACK_KEYER_R_CR_DEFAULT                                      0x00000000
-#define mmLBV0_LBV_BLACK_KEYER_G_Y_DEFAULT                                       0x00000000
-#define mmLBV0_LBV_BLACK_KEYER_B_CB_DEFAULT                                      0x00000000
-#define mmLBV0_LBV_KEYER_COLOR_CTRL_DEFAULT                                      0x00000000
-#define mmLBV0_LBV_KEYER_COLOR_R_CR_DEFAULT                                      0x00000000
-#define mmLBV0_LBV_KEYER_COLOR_G_Y_DEFAULT                                       0x00000000
-#define mmLBV0_LBV_KEYER_COLOR_B_CB_DEFAULT                                      0x00000000
-#define mmLBV0_LBV_KEYER_COLOR_REP_R_CR_DEFAULT                                  0x00000000
-#define mmLBV0_LBV_KEYER_COLOR_REP_G_Y_DEFAULT                                   0x00000000
-#define mmLBV0_LBV_KEYER_COLOR_REP_B_CB_DEFAULT                                  0x00000000
-#define mmLBV0_LBV_BUFFER_LEVEL_STATUS_DEFAULT                                   0xa0008000
-#define mmLBV0_LBV_BUFFER_URGENCY_CTRL_DEFAULT                                   0x00200010
-#define mmLBV0_LBV_BUFFER_URGENCY_STATUS_DEFAULT                                 0x00000000
-#define mmLBV0_LBV_BUFFER_STATUS_DEFAULT                                         0x12000002
-#define mmLBV0_LBV_NO_OUTSTANDING_REQ_STATUS_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_sclv0_dispdec
-#define mmSCLV0_SCLV_COEF_RAM_SELECT_DEFAULT                                     0x00000000
-#define mmSCLV0_SCLV_COEF_RAM_TAP_DATA_DEFAULT                                   0x00000000
-#define mmSCLV0_SCLV_MODE_DEFAULT                                                0x00000000
-#define mmSCLV0_SCLV_TAP_CONTROL_DEFAULT                                         0x00000000
-#define mmSCLV0_SCLV_CONTROL_DEFAULT                                             0x00000000
-#define mmSCLV0_SCLV_MANUAL_REPLICATE_CONTROL_DEFAULT                            0x00000000
-#define mmSCLV0_SCLV_AUTOMATIC_MODE_CONTROL_DEFAULT                              0x00000000
-#define mmSCLV0_SCLV_HORZ_FILTER_CONTROL_DEFAULT                                 0x00000000
-#define mmSCLV0_SCLV_HORZ_FILTER_SCALE_RATIO_DEFAULT                             0x00000000
-#define mmSCLV0_SCLV_HORZ_FILTER_INIT_DEFAULT                                    0x01000000
-#define mmSCLV0_SCLV_HORZ_FILTER_SCALE_RATIO_C_DEFAULT                           0x00000000
-#define mmSCLV0_SCLV_HORZ_FILTER_INIT_C_DEFAULT                                  0x01000000
-#define mmSCLV0_SCLV_VERT_FILTER_CONTROL_DEFAULT                                 0x00000000
-#define mmSCLV0_SCLV_VERT_FILTER_SCALE_RATIO_DEFAULT                             0x00000000
-#define mmSCLV0_SCLV_VERT_FILTER_INIT_DEFAULT                                    0x01000000
-#define mmSCLV0_SCLV_VERT_FILTER_INIT_BOT_DEFAULT                                0x01000000
-#define mmSCLV0_SCLV_VERT_FILTER_SCALE_RATIO_C_DEFAULT                           0x00000000
-#define mmSCLV0_SCLV_VERT_FILTER_INIT_C_DEFAULT                                  0x01000000
-#define mmSCLV0_SCLV_VERT_FILTER_INIT_BOT_C_DEFAULT                              0x01000000
-#define mmSCLV0_SCLV_ROUND_OFFSET_DEFAULT                                        0x80000000
-#define mmSCLV0_SCLV_UPDATE_DEFAULT                                              0x00000000
-#define mmSCLV0_SCLV_ALU_CONTROL_DEFAULT                                         0x00000000
-#define mmSCLV0_SCLV_VIEWPORT_START_DEFAULT                                      0x00000000
-#define mmSCLV0_SCLV_VIEWPORT_START_SECONDARY_DEFAULT                            0x00000000
-#define mmSCLV0_SCLV_VIEWPORT_SIZE_DEFAULT                                       0x00000000
-#define mmSCLV0_SCLV_VIEWPORT_START_C_DEFAULT                                    0x00000000
-#define mmSCLV0_SCLV_VIEWPORT_START_SECONDARY_C_DEFAULT                          0x00000000
-#define mmSCLV0_SCLV_VIEWPORT_SIZE_C_DEFAULT                                     0x00000000
-#define mmSCLV0_SCLV_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                             0x00000000
-#define mmSCLV0_SCLV_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                             0x00000000
-#define mmSCLV0_SCLV_MODE_CHANGE_DET1_DEFAULT                                    0x00000000
-#define mmSCLV0_SCLV_MODE_CHANGE_DET2_DEFAULT                                    0x00000000
-#define mmSCLV0_SCLV_MODE_CHANGE_DET3_DEFAULT                                    0x00000000
-#define mmSCLV0_SCLV_MODE_CHANGE_MASK_DEFAULT                                    0x00000000
-#define mmSCLV0_SCLV_HORZ_FILTER_INIT_BOT_DEFAULT                                0x01000000
-#define mmSCLV0_SCLV_HORZ_FILTER_INIT_BOT_C_DEFAULT                              0x01000000
-
-
-// addressBlock: dce_dc_col_man0_dispdec
-#define mmCOL_MAN0_COL_MAN_UPDATE_DEFAULT                                        0x00000000
-#define mmCOL_MAN0_COL_MAN_INPUT_CSC_CONTROL_DEFAULT                             0x00000000
-#define mmCOL_MAN0_INPUT_CSC_C11_C12_A_DEFAULT                                   0x00002000
-#define mmCOL_MAN0_INPUT_CSC_C13_C14_A_DEFAULT                                   0x00000000
-#define mmCOL_MAN0_INPUT_CSC_C21_C22_A_DEFAULT                                   0x20000000
-#define mmCOL_MAN0_INPUT_CSC_C23_C24_A_DEFAULT                                   0x00000000
-#define mmCOL_MAN0_INPUT_CSC_C31_C32_A_DEFAULT                                   0x00000000
-#define mmCOL_MAN0_INPUT_CSC_C33_C34_A_DEFAULT                                   0x00002000
-#define mmCOL_MAN0_INPUT_CSC_C11_C12_B_DEFAULT                                   0x00002000
-#define mmCOL_MAN0_INPUT_CSC_C13_C14_B_DEFAULT                                   0x00000000
-#define mmCOL_MAN0_INPUT_CSC_C21_C22_B_DEFAULT                                   0x20000000
-#define mmCOL_MAN0_INPUT_CSC_C23_C24_B_DEFAULT                                   0x00000000
-#define mmCOL_MAN0_INPUT_CSC_C31_C32_B_DEFAULT                                   0x00000000
-#define mmCOL_MAN0_INPUT_CSC_C33_C34_B_DEFAULT                                   0x00002000
-#define mmCOL_MAN0_PRESCALE_CONTROL_DEFAULT                                      0x00000000
-#define mmCOL_MAN0_PRESCALE_VALUES_R_DEFAULT                                     0x20000000
-#define mmCOL_MAN0_PRESCALE_VALUES_G_DEFAULT                                     0x20000000
-#define mmCOL_MAN0_PRESCALE_VALUES_B_DEFAULT                                     0x20000000
-#define mmCOL_MAN0_COL_MAN_OUTPUT_CSC_CONTROL_DEFAULT                            0x00000000
-#define mmCOL_MAN0_OUTPUT_CSC_C11_C12_A_DEFAULT                                  0x00002000
-#define mmCOL_MAN0_OUTPUT_CSC_C13_C14_A_DEFAULT                                  0x00000000
-#define mmCOL_MAN0_OUTPUT_CSC_C21_C22_A_DEFAULT                                  0x20000000
-#define mmCOL_MAN0_OUTPUT_CSC_C23_C24_A_DEFAULT                                  0x00000000
-#define mmCOL_MAN0_OUTPUT_CSC_C31_C32_A_DEFAULT                                  0x00000000
-#define mmCOL_MAN0_OUTPUT_CSC_C33_C34_A_DEFAULT                                  0x00002000
-#define mmCOL_MAN0_OUTPUT_CSC_C11_C12_B_DEFAULT                                  0x00002000
-#define mmCOL_MAN0_OUTPUT_CSC_C13_C14_B_DEFAULT                                  0x00000000
-#define mmCOL_MAN0_OUTPUT_CSC_C21_C22_B_DEFAULT                                  0x20000000
-#define mmCOL_MAN0_OUTPUT_CSC_C23_C24_B_DEFAULT                                  0x00000000
-#define mmCOL_MAN0_OUTPUT_CSC_C31_C32_B_DEFAULT                                  0x00000000
-#define mmCOL_MAN0_OUTPUT_CSC_C33_C34_B_DEFAULT                                  0x00002000
-#define mmCOL_MAN0_DENORM_CLAMP_CONTROL_DEFAULT                                  0x00000000
-#define mmCOL_MAN0_DENORM_CLAMP_RANGE_R_CR_DEFAULT                               0x00000fff
-#define mmCOL_MAN0_DENORM_CLAMP_RANGE_G_Y_DEFAULT                                0x00000fff
-#define mmCOL_MAN0_DENORM_CLAMP_RANGE_B_CB_DEFAULT                               0x00000fff
-#define mmCOL_MAN0_COL_MAN_FP_CONVERTED_FIELD_DEFAULT                            0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CONTROL_DEFAULT                               0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_LUT_INDEX_DEFAULT                             0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_LUT_DATA_DEFAULT                              0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                     0x00000007
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_START_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_END_CNTL1_DEFAULT                       0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_END_CNTL2_DEFAULT                       0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_0_1_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_2_3_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_4_5_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_6_7_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_8_9_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_10_11_DEFAULT                    0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_12_13_DEFAULT                    0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLA_REGION_14_15_DEFAULT                    0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_START_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_END_CNTL1_DEFAULT                       0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_END_CNTL2_DEFAULT                       0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_0_1_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_2_3_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_4_5_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_6_7_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_8_9_DEFAULT                      0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_10_11_DEFAULT                    0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_12_13_DEFAULT                    0x00000000
-#define mmCOL_MAN0_COL_MAN_REGAMMA_CNTLB_REGION_14_15_DEFAULT                    0x00000000
-#define mmCOL_MAN0_PACK_FIFO_ERROR_DEFAULT                                       0x00000000
-#define mmCOL_MAN0_OUTPUT_FIFO_ERROR_DEFAULT                                     0x00000000
-#define mmCOL_MAN0_INPUT_GAMMA_LUT_AUTOFILL_DEFAULT                              0x00000000
-#define mmCOL_MAN0_INPUT_GAMMA_LUT_RW_INDEX_DEFAULT                              0x00000000
-#define mmCOL_MAN0_INPUT_GAMMA_LUT_SEQ_COLOR_DEFAULT                             0x00000000
-#define mmCOL_MAN0_INPUT_GAMMA_LUT_PWL_DATA_DEFAULT                              0x00000000
-#define mmCOL_MAN0_INPUT_GAMMA_LUT_30_COLOR_DEFAULT                              0x00000000
-#define mmCOL_MAN0_COL_MAN_INPUT_GAMMA_CONTROL1_DEFAULT                          0x00000000
-#define mmCOL_MAN0_COL_MAN_INPUT_GAMMA_CONTROL2_DEFAULT                          0x03800000
-#define mmCOL_MAN0_INPUT_GAMMA_BW_OFFSETS_B_DEFAULT                              0xffff0000
-#define mmCOL_MAN0_INPUT_GAMMA_BW_OFFSETS_G_DEFAULT                              0xffff0000
-#define mmCOL_MAN0_INPUT_GAMMA_BW_OFFSETS_R_DEFAULT                              0xffff0000
-#define mmCOL_MAN0_COL_MAN_DEGAMMA_CONTROL_DEFAULT                               0x00000000
-#define mmCOL_MAN0_COL_MAN_GAMUT_REMAP_CONTROL_DEFAULT                           0x00000000
-#define mmCOL_MAN0_COL_MAN_GAMUT_REMAP_C11_C12_DEFAULT                           0x00002000
-#define mmCOL_MAN0_COL_MAN_GAMUT_REMAP_C13_C14_DEFAULT                           0x00000000
-#define mmCOL_MAN0_COL_MAN_GAMUT_REMAP_C21_C22_DEFAULT                           0x20000000
-#define mmCOL_MAN0_COL_MAN_GAMUT_REMAP_C23_C24_DEFAULT                           0x00000000
-#define mmCOL_MAN0_COL_MAN_GAMUT_REMAP_C31_C32_DEFAULT                           0x00000000
-#define mmCOL_MAN0_COL_MAN_GAMUT_REMAP_C33_C34_DEFAULT                           0x00002000
-
-
-// addressBlock: dce_dc_dcfev0_dispdec
-#define mmDCFEV0_DCFEV_CLOCK_CONTROL_DEFAULT                                     0x00000000
-#define mmDCFEV0_DCFEV_SOFT_RESET_DEFAULT                                        0x00000000
-#define mmDCFEV0_DCFEV_DMIFV_CLOCK_CONTROL_DEFAULT                               0x00000000
-#define mmDCFEV0_DCFEV_DMIFV_MEM_PWR_CTRL_DEFAULT                                0x00000000
-#define mmDCFEV0_DCFEV_DMIFV_MEM_PWR_STATUS_DEFAULT                              0x00000000
-#define mmDCFEV0_DCFEV_MEM_PWR_CTRL_DEFAULT                                      0x00000000
-#define mmDCFEV0_DCFEV_MEM_PWR_CTRL2_DEFAULT                                     0x00000000
-#define mmDCFEV0_DCFEV_MEM_PWR_STATUS_DEFAULT                                    0x00000000
-#define mmDCFEV0_DCFEV_L_FLUSH_DEFAULT                                           0x00000000
-#define mmDCFEV0_DCFEV_C_FLUSH_DEFAULT                                           0x00000000
-#define mmDCFEV0_DCFEV_MISC_DEFAULT                                              0x00000001
-
-
-// addressBlock: dce_dc_dc_perfmon11_dispdec
-#define mmDC_PERFMON11_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON11_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON11_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON11_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON11_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON11_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON11_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON11_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON11_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dmifv_pg0_dispdec
-#define mmDMIFV_PG0_DPGV0_PIPE_ARBITRATION_CONTROL1_DEFAULT                      0x00000000
-#define mmDMIFV_PG0_DPGV0_PIPE_ARBITRATION_CONTROL2_DEFAULT                      0x00000000
-#define mmDMIFV_PG0_DPGV0_WATERMARK_MASK_CONTROL_DEFAULT                         0x00030303
-#define mmDMIFV_PG0_DPGV0_PIPE_URGENCY_CONTROL_DEFAULT                           0x00000000
-#define mmDMIFV_PG0_DPGV0_PIPE_DPM_CONTROL_DEFAULT                               0x00003000
-#define mmDMIFV_PG0_DPGV0_PIPE_STUTTER_CONTROL_DEFAULT                           0x00000200
-#define mmDMIFV_PG0_DPGV0_PIPE_NB_PSTATE_CHANGE_CONTROL_DEFAULT                  0x00000000
-#define mmDMIFV_PG0_DPGV0_PIPE_STUTTER_CONTROL_NONLPTCH_DEFAULT                  0x00000200
-#define mmDMIFV_PG0_DPGV0_REPEATER_PROGRAM_DEFAULT                               0x00000000
-#define mmDMIFV_PG0_DPGV0_CHK_PRE_PROC_CNTL_DEFAULT                              0x00000000
-#define mmDMIFV_PG0_DPGV1_PIPE_ARBITRATION_CONTROL1_DEFAULT                      0x00000000
-#define mmDMIFV_PG0_DPGV1_PIPE_ARBITRATION_CONTROL2_DEFAULT                      0x00000000
-#define mmDMIFV_PG0_DPGV1_WATERMARK_MASK_CONTROL_DEFAULT                         0x00030303
-#define mmDMIFV_PG0_DPGV1_PIPE_URGENCY_CONTROL_DEFAULT                           0x00000000
-#define mmDMIFV_PG0_DPGV1_PIPE_DPM_CONTROL_DEFAULT                               0x00003000
-#define mmDMIFV_PG0_DPGV1_PIPE_STUTTER_CONTROL_DEFAULT                           0x00000200
-#define mmDMIFV_PG0_DPGV1_PIPE_NB_PSTATE_CHANGE_CONTROL_DEFAULT                  0x00000000
-#define mmDMIFV_PG0_DPGV1_PIPE_STUTTER_CONTROL_NONLPTCH_DEFAULT                  0x00000200
-#define mmDMIFV_PG0_DPGV1_REPEATER_PROGRAM_DEFAULT                               0x00000000
-#define mmDMIFV_PG0_DPGV1_CHK_PRE_PROC_CNTL_DEFAULT                              0x00000000
-
-
-// addressBlock: dce_dc_blndv0_dispdec
-#define mmBLNDV0_BLNDV_CONTROL_DEFAULT                                           0xff0220ff
-#define mmBLNDV0_BLNDV_SM_CONTROL2_DEFAULT                                       0x00000000
-#define mmBLNDV0_BLNDV_CONTROL2_DEFAULT                                          0x00000010
-#define mmBLNDV0_BLNDV_UPDATE_DEFAULT                                            0x00000000
-#define mmBLNDV0_BLNDV_UNDERFLOW_INTERRUPT_DEFAULT                               0x00000000
-#define mmBLNDV0_BLNDV_V_UPDATE_LOCK_DEFAULT                                     0x80000000
-#define mmBLNDV0_BLNDV_REG_UPDATE_STATUS_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_crtcv0_dispdec
-#define mmCRTCV0_CRTCV_H_BLANK_EARLY_NUM_DEFAULT                                 0x00000040
-#define mmCRTCV0_CRTCV_H_TOTAL_DEFAULT                                           0x00000000
-#define mmCRTCV0_CRTCV_H_BLANK_START_END_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_H_SYNC_A_DEFAULT                                          0x00000000
-#define mmCRTCV0_CRTCV_H_SYNC_A_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_H_SYNC_B_DEFAULT                                          0x00000000
-#define mmCRTCV0_CRTCV_H_SYNC_B_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_VBI_END_DEFAULT                                           0x00000003
-#define mmCRTCV0_CRTCV_V_TOTAL_DEFAULT                                           0x00000000
-#define mmCRTCV0_CRTCV_V_TOTAL_MIN_DEFAULT                                       0x00000000
-#define mmCRTCV0_CRTCV_V_TOTAL_MAX_DEFAULT                                       0x00000000
-#define mmCRTCV0_CRTCV_V_TOTAL_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTCV0_CRTCV_V_TOTAL_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTCV0_CRTCV_VSYNC_NOM_INT_STATUS_DEFAULT                              0x00000000
-#define mmCRTCV0_CRTCV_V_BLANK_START_END_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_V_SYNC_A_DEFAULT                                          0x00000000
-#define mmCRTCV0_CRTCV_V_SYNC_A_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_V_SYNC_B_DEFAULT                                          0x00000000
-#define mmCRTCV0_CRTCV_V_SYNC_B_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_DTMTEST_CNTL_DEFAULT                                      0x00000000
-#define mmCRTCV0_CRTCV_DTMTEST_STATUS_POSITION_DEFAULT                           0x00000000
-#define mmCRTCV0_CRTCV_TRIGA_CNTL_DEFAULT                                        0x00000000
-#define mmCRTCV0_CRTCV_TRIGA_MANUAL_TRIG_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_TRIGB_CNTL_DEFAULT                                        0x00000000
-#define mmCRTCV0_CRTCV_TRIGB_MANUAL_TRIG_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_FORCE_COUNT_NOW_CNTL_DEFAULT                              0x00000000
-#define mmCRTCV0_CRTCV_FLOW_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTCV0_CRTCV_STEREO_FORCE_NEXT_EYE_DEFAULT                             0x00000000
-#define mmCRTCV0_CRTCV_AVSYNC_COUNTER_DEFAULT                                    0x00000000
-#define mmCRTCV0_CRTCV_CONTROL_DEFAULT                                           0x80400110
-#define mmCRTCV0_CRTCV_BLANK_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_INTERLACE_CONTROL_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_INTERLACE_STATUS_DEFAULT                                  0x00000000
-#define mmCRTCV0_CRTCV_FIELD_INDICATION_CONTROL_DEFAULT                          0x00000000
-#define mmCRTCV0_CRTCV_PIXEL_DATA_READBACK0_DEFAULT                              0x00000000
-#define mmCRTCV0_CRTCV_PIXEL_DATA_READBACK1_DEFAULT                              0x00000000
-#define mmCRTCV0_CRTCV_STATUS_DEFAULT                                            0x00000000
-#define mmCRTCV0_CRTCV_STATUS_POSITION_DEFAULT                                   0x00000000
-#define mmCRTCV0_CRTCV_NOM_VERT_POSITION_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_STATUS_FRAME_COUNT_DEFAULT                                0x00000000
-#define mmCRTCV0_CRTCV_STATUS_VF_COUNT_DEFAULT                                   0x00000000
-#define mmCRTCV0_CRTCV_STATUS_HV_COUNT_DEFAULT                                   0x00000000
-#define mmCRTCV0_CRTCV_COUNT_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_COUNT_RESET_DEFAULT                                       0x00000000
-#define mmCRTCV0_CRTCV_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                      0x00000000
-#define mmCRTCV0_CRTCV_VERT_SYNC_CONTROL_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_STEREO_STATUS_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_STEREO_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTCV0_CRTCV_SNAPSHOT_STATUS_DEFAULT                                   0x00000000
-#define mmCRTCV0_CRTCV_SNAPSHOT_CONTROL_DEFAULT                                  0x00000000
-#define mmCRTCV0_CRTCV_SNAPSHOT_POSITION_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_SNAPSHOT_FRAME_DEFAULT                                    0x00000000
-#define mmCRTCV0_CRTCV_START_LINE_CONTROL_DEFAULT                                0x00003002
-#define mmCRTCV0_CRTCV_INTERRUPT_CONTROL_DEFAULT                                 0x00000000
-#define mmCRTCV0_CRTCV_UPDATE_LOCK_DEFAULT                                       0x00000000
-#define mmCRTCV0_CRTCV_DOUBLE_BUFFER_CONTROL_DEFAULT                             0x00000000
-#define mmCRTCV0_CRTCV_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                        0x00000000
-#define mmCRTCV0_CRTCV_TEST_PATTERN_CONTROL_DEFAULT                              0x00000000
-#define mmCRTCV0_CRTCV_TEST_PATTERN_PARAMETERS_DEFAULT                           0x00000000
-#define mmCRTCV0_CRTCV_TEST_PATTERN_COLOR_DEFAULT                                0x00000000
-#define mmCRTCV0_CRTCV_MASTER_UPDATE_LOCK_DEFAULT                                0x00010000
-#define mmCRTCV0_CRTCV_MASTER_UPDATE_MODE_DEFAULT                                0x00000000
-#define mmCRTCV0_CRTCV_MVP_INBAND_CNTL_INSERT_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                      0x00000008
-#define mmCRTCV0_CRTCV_MVP_STATUS_DEFAULT                                        0x00000000
-#define mmCRTCV0_CRTCV_MASTER_EN_DEFAULT                                         0x00000000
-#define mmCRTCV0_CRTCV_ALLOW_STOP_OFF_V_CNT_DEFAULT                              0x00010000
-#define mmCRTCV0_CRTCV_V_UPDATE_INT_STATUS_DEFAULT                               0x00000000
-#define mmCRTCV0_CRTCV_OVERSCAN_COLOR_DEFAULT                                    0x00000000
-#define mmCRTCV0_CRTCV_OVERSCAN_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTCV0_CRTCV_BLANK_DATA_COLOR_DEFAULT                                  0x00000000
-#define mmCRTCV0_CRTCV_BLANK_DATA_COLOR_EXT_DEFAULT                              0x00000000
-#define mmCRTCV0_CRTCV_BLACK_COLOR_DEFAULT                                       0x00000000
-#define mmCRTCV0_CRTCV_BLACK_COLOR_EXT_DEFAULT                                   0x00000000
-#define mmCRTCV0_CRTCV_VERTICAL_INTERRUPT0_POSITION_DEFAULT                      0x00000000
-#define mmCRTCV0_CRTCV_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                       0x00000000
-#define mmCRTCV0_CRTCV_VERTICAL_INTERRUPT1_POSITION_DEFAULT                      0x00000000
-#define mmCRTCV0_CRTCV_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                       0x00000000
-#define mmCRTCV0_CRTCV_VERTICAL_INTERRUPT2_POSITION_DEFAULT                      0x00000000
-#define mmCRTCV0_CRTCV_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                       0x00000000
-#define mmCRTCV0_CRTCV_CRC_CNTL_DEFAULT                                          0x00000000
-#define mmCRTCV0_CRTCV_CRC0_WINDOWA_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC0_WINDOWA_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC0_WINDOWB_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC0_WINDOWB_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC0_DATA_RG_DEFAULT                                      0x00000000
-#define mmCRTCV0_CRTCV_CRC0_DATA_B_DEFAULT                                       0x00000000
-#define mmCRTCV0_CRTCV_CRC1_WINDOWA_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC1_WINDOWA_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC1_WINDOWB_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC1_WINDOWB_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV0_CRTCV_CRC1_DATA_RG_DEFAULT                                      0x00000000
-#define mmCRTCV0_CRTCV_CRC1_DATA_B_DEFAULT                                       0x00000000
-#define mmCRTCV0_CRTCV_EXT_TIMING_SYNC_CONTROL_DEFAULT                           0x00000000
-#define mmCRTCV0_CRTCV_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                      0x00000000
-#define mmCRTCV0_CRTCV_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                        0x00000000
-#define mmCRTCV0_CRTCV_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTCV0_CRTCV_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                 0x00000000
-#define mmCRTCV0_CRTCV_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT          0x00000000
-#define mmCRTCV0_CRTCV_STATIC_SCREEN_CONTROL_DEFAULT                             0x00010000
-#define mmCRTCV0_CRTCV_3D_STRUCTURE_CONTROL_DEFAULT                              0x00000010
-#define mmCRTCV0_CRTCV_GSL_VSYNC_GAP_DEFAULT                                     0x00000000
-#define mmCRTCV0_CRTCV_GSL_WINDOW_DEFAULT                                        0x00000000
-#define mmCRTCV0_CRTCV_GSL_CONTROL_DEFAULT                                       0x00020000
-
-
-// addressBlock: dce_dc_unp1_dispdec
-#define mmUNP1_UNP_GRPH_ENABLE_DEFAULT                                           0x00000001
-#define mmUNP1_UNP_GRPH_CONTROL_DEFAULT                                          0x0a008008
-#define mmUNP1_UNP_GRPH_CONTROL_C_DEFAULT                                        0x00008000
-#define mmUNP1_UNP_GRPH_CONTROL_EXP_DEFAULT                                      0x00000000
-#define mmUNP1_UNP_GRPH_SWAP_CNTL_DEFAULT                                        0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_L_DEFAULT                        0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_C_DEFAULT                        0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_L_DEFAULT                   0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                   0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_L_DEFAULT                 0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_C_DEFAULT                 0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_HIGH_L_DEFAULT            0x00000000
-#define mmUNP1_UNP_GRPH_PRIMARY_BOTTOM_SURFACE_ADDRESS_HIGH_C_DEFAULT            0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_L_DEFAULT                      0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_C_DEFAULT                      0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_L_DEFAULT                 0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH_C_DEFAULT                 0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_L_DEFAULT               0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_C_DEFAULT               0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_HIGH_L_DEFAULT          0x00000000
-#define mmUNP1_UNP_GRPH_SECONDARY_BOTTOM_SURFACE_ADDRESS_HIGH_C_DEFAULT          0x00000000
-#define mmUNP1_UNP_GRPH_PITCH_L_DEFAULT                                          0x00000000
-#define mmUNP1_UNP_GRPH_PITCH_C_DEFAULT                                          0x00000000
-#define mmUNP1_UNP_GRPH_SURFACE_OFFSET_X_L_DEFAULT                               0x00000000
-#define mmUNP1_UNP_GRPH_SURFACE_OFFSET_X_C_DEFAULT                               0x00000000
-#define mmUNP1_UNP_GRPH_SURFACE_OFFSET_Y_L_DEFAULT                               0x00000000
-#define mmUNP1_UNP_GRPH_SURFACE_OFFSET_Y_C_DEFAULT                               0x00000000
-#define mmUNP1_UNP_GRPH_X_START_L_DEFAULT                                        0x00000000
-#define mmUNP1_UNP_GRPH_X_START_C_DEFAULT                                        0x00000000
-#define mmUNP1_UNP_GRPH_Y_START_L_DEFAULT                                        0x00000000
-#define mmUNP1_UNP_GRPH_Y_START_C_DEFAULT                                        0x00000000
-#define mmUNP1_UNP_GRPH_X_END_L_DEFAULT                                          0x00000000
-#define mmUNP1_UNP_GRPH_X_END_C_DEFAULT                                          0x00000000
-#define mmUNP1_UNP_GRPH_Y_END_L_DEFAULT                                          0x00000000
-#define mmUNP1_UNP_GRPH_Y_END_C_DEFAULT                                          0x00000000
-#define mmUNP1_UNP_GRPH_UPDATE_DEFAULT                                           0x00000000
-#define mmUNP1_UNP_PIPE_OUTSTANDING_REQUEST_LIMIT_DEFAULT                        0x0000ffff
-#define mmUNP1_UNP_GRPH_SURFACE_ADDRESS_INUSE_L_DEFAULT                          0x00000000
-#define mmUNP1_UNP_GRPH_SURFACE_ADDRESS_INUSE_C_DEFAULT                          0x00000000
-#define mmUNP1_UNP_GRPH_SURFACE_ADDRESS_HIGH_INUSE_L_DEFAULT                     0x00000000
-#define mmUNP1_UNP_GRPH_SURFACE_ADDRESS_HIGH_INUSE_C_DEFAULT                     0x00000000
-#define mmUNP1_UNP_DVMM_PTE_CONTROL_DEFAULT                                      0x00004000
-#define mmUNP1_UNP_DVMM_PTE_CONTROL_C_DEFAULT                                    0x00004000
-#define mmUNP1_UNP_DVMM_PTE_ARB_CONTROL_DEFAULT                                  0x00002220
-#define mmUNP1_UNP_DVMM_PTE_ARB_CONTROL_C_DEFAULT                                0x00002220
-#define mmUNP1_UNP_GRPH_INTERRUPT_STATUS_DEFAULT                                 0x00000000
-#define mmUNP1_UNP_GRPH_INTERRUPT_CONTROL_DEFAULT                                0x00000000
-#define mmUNP1_UNP_GRPH_STEREOSYNC_FLIP_DEFAULT                                  0x00002020
-#define mmUNP1_UNP_FLIP_CONTROL_DEFAULT                                          0x00000001
-#define mmUNP1_UNP_CRC_CONTROL_DEFAULT                                           0x00000000
-#define mmUNP1_UNP_CRC_MASK_DEFAULT                                              0x00000000
-#define mmUNP1_UNP_CRC_CURRENT_DEFAULT                                           0x00000000
-#define mmUNP1_UNP_CRC_LAST_DEFAULT                                              0x00000000
-#define mmUNP1_UNP_LB_DATA_GAP_BETWEEN_CHUNK_DEFAULT                             0x00000100
-#define mmUNP1_UNP_HW_ROTATION_DEFAULT                                           0x00000010
-
-
-// addressBlock: dce_dc_lbv1_dispdec
-#define mmLBV1_LBV_DATA_FORMAT_DEFAULT                                           0x00000000
-#define mmLBV1_LBV_MEMORY_CTRL_DEFAULT                                           0x000006b0
-#define mmLBV1_LBV_MEMORY_SIZE_STATUS_DEFAULT                                    0x00000000
-#define mmLBV1_LBV_DESKTOP_HEIGHT_DEFAULT                                        0x00000000
-#define mmLBV1_LBV_VLINE_START_END_DEFAULT                                       0x00000000
-#define mmLBV1_LBV_VLINE2_START_END_DEFAULT                                      0x00000000
-#define mmLBV1_LBV_V_COUNTER_DEFAULT                                             0x00000000
-#define mmLBV1_LBV_SNAPSHOT_V_COUNTER_DEFAULT                                    0x00000000
-#define mmLBV1_LBV_V_COUNTER_CHROMA_DEFAULT                                      0x00000000
-#define mmLBV1_LBV_SNAPSHOT_V_COUNTER_CHROMA_DEFAULT                             0x00000000
-#define mmLBV1_LBV_INTERRUPT_MASK_DEFAULT                                        0x00000000
-#define mmLBV1_LBV_VLINE_STATUS_DEFAULT                                          0x00000000
-#define mmLBV1_LBV_VLINE2_STATUS_DEFAULT                                         0x00000000
-#define mmLBV1_LBV_VBLANK_STATUS_DEFAULT                                         0x00000000
-#define mmLBV1_LBV_SYNC_RESET_SEL_DEFAULT                                        0x00000002
-#define mmLBV1_LBV_BLACK_KEYER_R_CR_DEFAULT                                      0x00000000
-#define mmLBV1_LBV_BLACK_KEYER_G_Y_DEFAULT                                       0x00000000
-#define mmLBV1_LBV_BLACK_KEYER_B_CB_DEFAULT                                      0x00000000
-#define mmLBV1_LBV_KEYER_COLOR_CTRL_DEFAULT                                      0x00000000
-#define mmLBV1_LBV_KEYER_COLOR_R_CR_DEFAULT                                      0x00000000
-#define mmLBV1_LBV_KEYER_COLOR_G_Y_DEFAULT                                       0x00000000
-#define mmLBV1_LBV_KEYER_COLOR_B_CB_DEFAULT                                      0x00000000
-#define mmLBV1_LBV_KEYER_COLOR_REP_R_CR_DEFAULT                                  0x00000000
-#define mmLBV1_LBV_KEYER_COLOR_REP_G_Y_DEFAULT                                   0x00000000
-#define mmLBV1_LBV_KEYER_COLOR_REP_B_CB_DEFAULT                                  0x00000000
-#define mmLBV1_LBV_BUFFER_LEVEL_STATUS_DEFAULT                                   0xa0008000
-#define mmLBV1_LBV_BUFFER_URGENCY_CTRL_DEFAULT                                   0x00200010
-#define mmLBV1_LBV_BUFFER_URGENCY_STATUS_DEFAULT                                 0x00000000
-#define mmLBV1_LBV_BUFFER_STATUS_DEFAULT                                         0x12000002
-#define mmLBV1_LBV_NO_OUTSTANDING_REQ_STATUS_DEFAULT                             0x00000000
-
-
-// addressBlock: dce_dc_sclv1_dispdec
-#define mmSCLV1_SCLV_COEF_RAM_SELECT_DEFAULT                                     0x00000000
-#define mmSCLV1_SCLV_COEF_RAM_TAP_DATA_DEFAULT                                   0x00000000
-#define mmSCLV1_SCLV_MODE_DEFAULT                                                0x00000000
-#define mmSCLV1_SCLV_TAP_CONTROL_DEFAULT                                         0x00000000
-#define mmSCLV1_SCLV_CONTROL_DEFAULT                                             0x00000000
-#define mmSCLV1_SCLV_MANUAL_REPLICATE_CONTROL_DEFAULT                            0x00000000
-#define mmSCLV1_SCLV_AUTOMATIC_MODE_CONTROL_DEFAULT                              0x00000000
-#define mmSCLV1_SCLV_HORZ_FILTER_CONTROL_DEFAULT                                 0x00000000
-#define mmSCLV1_SCLV_HORZ_FILTER_SCALE_RATIO_DEFAULT                             0x00000000
-#define mmSCLV1_SCLV_HORZ_FILTER_INIT_DEFAULT                                    0x01000000
-#define mmSCLV1_SCLV_HORZ_FILTER_SCALE_RATIO_C_DEFAULT                           0x00000000
-#define mmSCLV1_SCLV_HORZ_FILTER_INIT_C_DEFAULT                                  0x01000000
-#define mmSCLV1_SCLV_VERT_FILTER_CONTROL_DEFAULT                                 0x00000000
-#define mmSCLV1_SCLV_VERT_FILTER_SCALE_RATIO_DEFAULT                             0x00000000
-#define mmSCLV1_SCLV_VERT_FILTER_INIT_DEFAULT                                    0x01000000
-#define mmSCLV1_SCLV_VERT_FILTER_INIT_BOT_DEFAULT                                0x01000000
-#define mmSCLV1_SCLV_VERT_FILTER_SCALE_RATIO_C_DEFAULT                           0x00000000
-#define mmSCLV1_SCLV_VERT_FILTER_INIT_C_DEFAULT                                  0x01000000
-#define mmSCLV1_SCLV_VERT_FILTER_INIT_BOT_C_DEFAULT                              0x01000000
-#define mmSCLV1_SCLV_ROUND_OFFSET_DEFAULT                                        0x80000000
-#define mmSCLV1_SCLV_UPDATE_DEFAULT                                              0x00000000
-#define mmSCLV1_SCLV_ALU_CONTROL_DEFAULT                                         0x00000000
-#define mmSCLV1_SCLV_VIEWPORT_START_DEFAULT                                      0x00000000
-#define mmSCLV1_SCLV_VIEWPORT_START_SECONDARY_DEFAULT                            0x00000000
-#define mmSCLV1_SCLV_VIEWPORT_SIZE_DEFAULT                                       0x00000000
-#define mmSCLV1_SCLV_VIEWPORT_START_C_DEFAULT                                    0x00000000
-#define mmSCLV1_SCLV_VIEWPORT_START_SECONDARY_C_DEFAULT                          0x00000000
-#define mmSCLV1_SCLV_VIEWPORT_SIZE_C_DEFAULT                                     0x00000000
-#define mmSCLV1_SCLV_EXT_OVERSCAN_LEFT_RIGHT_DEFAULT                             0x00000000
-#define mmSCLV1_SCLV_EXT_OVERSCAN_TOP_BOTTOM_DEFAULT                             0x00000000
-#define mmSCLV1_SCLV_MODE_CHANGE_DET1_DEFAULT                                    0x00000000
-#define mmSCLV1_SCLV_MODE_CHANGE_DET2_DEFAULT                                    0x00000000
-#define mmSCLV1_SCLV_MODE_CHANGE_DET3_DEFAULT                                    0x00000000
-#define mmSCLV1_SCLV_MODE_CHANGE_MASK_DEFAULT                                    0x00000000
-#define mmSCLV1_SCLV_HORZ_FILTER_INIT_BOT_DEFAULT                                0x01000000
-#define mmSCLV1_SCLV_HORZ_FILTER_INIT_BOT_C_DEFAULT                              0x01000000
-
-
-// addressBlock: dce_dc_col_man1_dispdec
-#define mmCOL_MAN1_COL_MAN_UPDATE_DEFAULT                                        0x00000000
-#define mmCOL_MAN1_COL_MAN_INPUT_CSC_CONTROL_DEFAULT                             0x00000000
-#define mmCOL_MAN1_INPUT_CSC_C11_C12_A_DEFAULT                                   0x00002000
-#define mmCOL_MAN1_INPUT_CSC_C13_C14_A_DEFAULT                                   0x00000000
-#define mmCOL_MAN1_INPUT_CSC_C21_C22_A_DEFAULT                                   0x20000000
-#define mmCOL_MAN1_INPUT_CSC_C23_C24_A_DEFAULT                                   0x00000000
-#define mmCOL_MAN1_INPUT_CSC_C31_C32_A_DEFAULT                                   0x00000000
-#define mmCOL_MAN1_INPUT_CSC_C33_C34_A_DEFAULT                                   0x00002000
-#define mmCOL_MAN1_INPUT_CSC_C11_C12_B_DEFAULT                                   0x00002000
-#define mmCOL_MAN1_INPUT_CSC_C13_C14_B_DEFAULT                                   0x00000000
-#define mmCOL_MAN1_INPUT_CSC_C21_C22_B_DEFAULT                                   0x20000000
-#define mmCOL_MAN1_INPUT_CSC_C23_C24_B_DEFAULT                                   0x00000000
-#define mmCOL_MAN1_INPUT_CSC_C31_C32_B_DEFAULT                                   0x00000000
-#define mmCOL_MAN1_INPUT_CSC_C33_C34_B_DEFAULT                                   0x00002000
-#define mmCOL_MAN1_PRESCALE_CONTROL_DEFAULT                                      0x00000000
-#define mmCOL_MAN1_PRESCALE_VALUES_R_DEFAULT                                     0x20000000
-#define mmCOL_MAN1_PRESCALE_VALUES_G_DEFAULT                                     0x20000000
-#define mmCOL_MAN1_PRESCALE_VALUES_B_DEFAULT                                     0x20000000
-#define mmCOL_MAN1_COL_MAN_OUTPUT_CSC_CONTROL_DEFAULT                            0x00000000
-#define mmCOL_MAN1_OUTPUT_CSC_C11_C12_A_DEFAULT                                  0x00002000
-#define mmCOL_MAN1_OUTPUT_CSC_C13_C14_A_DEFAULT                                  0x00000000
-#define mmCOL_MAN1_OUTPUT_CSC_C21_C22_A_DEFAULT                                  0x20000000
-#define mmCOL_MAN1_OUTPUT_CSC_C23_C24_A_DEFAULT                                  0x00000000
-#define mmCOL_MAN1_OUTPUT_CSC_C31_C32_A_DEFAULT                                  0x00000000
-#define mmCOL_MAN1_OUTPUT_CSC_C33_C34_A_DEFAULT                                  0x00002000
-#define mmCOL_MAN1_OUTPUT_CSC_C11_C12_B_DEFAULT                                  0x00002000
-#define mmCOL_MAN1_OUTPUT_CSC_C13_C14_B_DEFAULT                                  0x00000000
-#define mmCOL_MAN1_OUTPUT_CSC_C21_C22_B_DEFAULT                                  0x20000000
-#define mmCOL_MAN1_OUTPUT_CSC_C23_C24_B_DEFAULT                                  0x00000000
-#define mmCOL_MAN1_OUTPUT_CSC_C31_C32_B_DEFAULT                                  0x00000000
-#define mmCOL_MAN1_OUTPUT_CSC_C33_C34_B_DEFAULT                                  0x00002000
-#define mmCOL_MAN1_DENORM_CLAMP_CONTROL_DEFAULT                                  0x00000000
-#define mmCOL_MAN1_DENORM_CLAMP_RANGE_R_CR_DEFAULT                               0x00000fff
-#define mmCOL_MAN1_DENORM_CLAMP_RANGE_G_Y_DEFAULT                                0x00000fff
-#define mmCOL_MAN1_DENORM_CLAMP_RANGE_B_CB_DEFAULT                               0x00000fff
-#define mmCOL_MAN1_COL_MAN_FP_CONVERTED_FIELD_DEFAULT                            0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CONTROL_DEFAULT                               0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_LUT_INDEX_DEFAULT                             0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_LUT_DATA_DEFAULT                              0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_LUT_WRITE_EN_MASK_DEFAULT                     0x00000007
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_START_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_SLOPE_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_END_CNTL1_DEFAULT                       0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_END_CNTL2_DEFAULT                       0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_0_1_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_2_3_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_4_5_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_6_7_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_8_9_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_10_11_DEFAULT                    0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_12_13_DEFAULT                    0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLA_REGION_14_15_DEFAULT                    0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_START_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_SLOPE_CNTL_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_END_CNTL1_DEFAULT                       0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_END_CNTL2_DEFAULT                       0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_0_1_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_2_3_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_4_5_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_6_7_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_8_9_DEFAULT                      0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_10_11_DEFAULT                    0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_12_13_DEFAULT                    0x00000000
-#define mmCOL_MAN1_COL_MAN_REGAMMA_CNTLB_REGION_14_15_DEFAULT                    0x00000000
-#define mmCOL_MAN1_PACK_FIFO_ERROR_DEFAULT                                       0x00000000
-#define mmCOL_MAN1_OUTPUT_FIFO_ERROR_DEFAULT                                     0x00000000
-#define mmCOL_MAN1_INPUT_GAMMA_LUT_AUTOFILL_DEFAULT                              0x00000000
-#define mmCOL_MAN1_INPUT_GAMMA_LUT_RW_INDEX_DEFAULT                              0x00000000
-#define mmCOL_MAN1_INPUT_GAMMA_LUT_SEQ_COLOR_DEFAULT                             0x00000000
-#define mmCOL_MAN1_INPUT_GAMMA_LUT_PWL_DATA_DEFAULT                              0x00000000
-#define mmCOL_MAN1_INPUT_GAMMA_LUT_30_COLOR_DEFAULT                              0x00000000
-#define mmCOL_MAN1_COL_MAN_INPUT_GAMMA_CONTROL1_DEFAULT                          0x00000000
-#define mmCOL_MAN1_COL_MAN_INPUT_GAMMA_CONTROL2_DEFAULT                          0x03800000
-#define mmCOL_MAN1_INPUT_GAMMA_BW_OFFSETS_B_DEFAULT                              0xffff0000
-#define mmCOL_MAN1_INPUT_GAMMA_BW_OFFSETS_G_DEFAULT                              0xffff0000
-#define mmCOL_MAN1_INPUT_GAMMA_BW_OFFSETS_R_DEFAULT                              0xffff0000
-#define mmCOL_MAN1_COL_MAN_DEGAMMA_CONTROL_DEFAULT                               0x00000000
-#define mmCOL_MAN1_COL_MAN_GAMUT_REMAP_CONTROL_DEFAULT                           0x00000000
-#define mmCOL_MAN1_COL_MAN_GAMUT_REMAP_C11_C12_DEFAULT                           0x00002000
-#define mmCOL_MAN1_COL_MAN_GAMUT_REMAP_C13_C14_DEFAULT                           0x00000000
-#define mmCOL_MAN1_COL_MAN_GAMUT_REMAP_C21_C22_DEFAULT                           0x20000000
-#define mmCOL_MAN1_COL_MAN_GAMUT_REMAP_C23_C24_DEFAULT                           0x00000000
-#define mmCOL_MAN1_COL_MAN_GAMUT_REMAP_C31_C32_DEFAULT                           0x00000000
-#define mmCOL_MAN1_COL_MAN_GAMUT_REMAP_C33_C34_DEFAULT                           0x00002000
-
-
-// addressBlock: dce_dc_dcfev1_dispdec
-#define mmDCFEV1_DCFEV_CLOCK_CONTROL_DEFAULT                                     0x00000000
-#define mmDCFEV1_DCFEV_SOFT_RESET_DEFAULT                                        0x00000000
-#define mmDCFEV1_DCFEV_DMIFV_CLOCK_CONTROL_DEFAULT                               0x00000000
-#define mmDCFEV1_DCFEV_DMIFV_MEM_PWR_CTRL_DEFAULT                                0x00000000
-#define mmDCFEV1_DCFEV_DMIFV_MEM_PWR_STATUS_DEFAULT                              0x00000000
-#define mmDCFEV1_DCFEV_MEM_PWR_CTRL_DEFAULT                                      0x00000000
-#define mmDCFEV1_DCFEV_MEM_PWR_CTRL2_DEFAULT                                     0x00000000
-#define mmDCFEV1_DCFEV_MEM_PWR_STATUS_DEFAULT                                    0x00000000
-#define mmDCFEV1_DCFEV_L_FLUSH_DEFAULT                                           0x00000000
-#define mmDCFEV1_DCFEV_C_FLUSH_DEFAULT                                           0x00000000
-#define mmDCFEV1_DCFEV_MISC_DEFAULT                                              0x00000001
-
-
-// addressBlock: dce_dc_dc_perfmon12_dispdec
-#define mmDC_PERFMON12_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON12_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON12_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON12_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON12_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON12_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON12_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON12_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON12_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dmifv_pg1_dispdec
-#define mmDMIFV_PG1_DPGV0_PIPE_ARBITRATION_CONTROL1_DEFAULT                      0x00000000
-#define mmDMIFV_PG1_DPGV0_PIPE_ARBITRATION_CONTROL2_DEFAULT                      0x00000000
-#define mmDMIFV_PG1_DPGV0_WATERMARK_MASK_CONTROL_DEFAULT                         0x00030303
-#define mmDMIFV_PG1_DPGV0_PIPE_URGENCY_CONTROL_DEFAULT                           0x00000000
-#define mmDMIFV_PG1_DPGV0_PIPE_DPM_CONTROL_DEFAULT                               0x00003000
-#define mmDMIFV_PG1_DPGV0_PIPE_STUTTER_CONTROL_DEFAULT                           0x00000200
-#define mmDMIFV_PG1_DPGV0_PIPE_NB_PSTATE_CHANGE_CONTROL_DEFAULT                  0x00000000
-#define mmDMIFV_PG1_DPGV0_PIPE_STUTTER_CONTROL_NONLPTCH_DEFAULT                  0x00000200
-#define mmDMIFV_PG1_DPGV0_REPEATER_PROGRAM_DEFAULT                               0x00000000
-#define mmDMIFV_PG1_DPGV0_CHK_PRE_PROC_CNTL_DEFAULT                              0x00000000
-#define mmDMIFV_PG1_DPGV1_PIPE_ARBITRATION_CONTROL1_DEFAULT                      0x00000000
-#define mmDMIFV_PG1_DPGV1_PIPE_ARBITRATION_CONTROL2_DEFAULT                      0x00000000
-#define mmDMIFV_PG1_DPGV1_WATERMARK_MASK_CONTROL_DEFAULT                         0x00030303
-#define mmDMIFV_PG1_DPGV1_PIPE_URGENCY_CONTROL_DEFAULT                           0x00000000
-#define mmDMIFV_PG1_DPGV1_PIPE_DPM_CONTROL_DEFAULT                               0x00003000
-#define mmDMIFV_PG1_DPGV1_PIPE_STUTTER_CONTROL_DEFAULT                           0x00000200
-#define mmDMIFV_PG1_DPGV1_PIPE_NB_PSTATE_CHANGE_CONTROL_DEFAULT                  0x00000000
-#define mmDMIFV_PG1_DPGV1_PIPE_STUTTER_CONTROL_NONLPTCH_DEFAULT                  0x00000200
-#define mmDMIFV_PG1_DPGV1_REPEATER_PROGRAM_DEFAULT                               0x00000000
-#define mmDMIFV_PG1_DPGV1_CHK_PRE_PROC_CNTL_DEFAULT                              0x00000000
-
-
-// addressBlock: dce_dc_blndv1_dispdec
-#define mmBLNDV1_BLNDV_CONTROL_DEFAULT                                           0xff0220ff
-#define mmBLNDV1_BLNDV_SM_CONTROL2_DEFAULT                                       0x00000000
-#define mmBLNDV1_BLNDV_CONTROL2_DEFAULT                                          0x00000010
-#define mmBLNDV1_BLNDV_UPDATE_DEFAULT                                            0x00000000
-#define mmBLNDV1_BLNDV_UNDERFLOW_INTERRUPT_DEFAULT                               0x00000000
-#define mmBLNDV1_BLNDV_V_UPDATE_LOCK_DEFAULT                                     0x80000000
-#define mmBLNDV1_BLNDV_REG_UPDATE_STATUS_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_crtcv1_dispdec
-#define mmCRTCV1_CRTCV_H_BLANK_EARLY_NUM_DEFAULT                                 0x00000040
-#define mmCRTCV1_CRTCV_H_TOTAL_DEFAULT                                           0x00000000
-#define mmCRTCV1_CRTCV_H_BLANK_START_END_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_H_SYNC_A_DEFAULT                                          0x00000000
-#define mmCRTCV1_CRTCV_H_SYNC_A_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_H_SYNC_B_DEFAULT                                          0x00000000
-#define mmCRTCV1_CRTCV_H_SYNC_B_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_VBI_END_DEFAULT                                           0x00000003
-#define mmCRTCV1_CRTCV_V_TOTAL_DEFAULT                                           0x00000000
-#define mmCRTCV1_CRTCV_V_TOTAL_MIN_DEFAULT                                       0x00000000
-#define mmCRTCV1_CRTCV_V_TOTAL_MAX_DEFAULT                                       0x00000000
-#define mmCRTCV1_CRTCV_V_TOTAL_CONTROL_DEFAULT                                   0x00000000
-#define mmCRTCV1_CRTCV_V_TOTAL_INT_STATUS_DEFAULT                                0x00000000
-#define mmCRTCV1_CRTCV_VSYNC_NOM_INT_STATUS_DEFAULT                              0x00000000
-#define mmCRTCV1_CRTCV_V_BLANK_START_END_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_V_SYNC_A_DEFAULT                                          0x00000000
-#define mmCRTCV1_CRTCV_V_SYNC_A_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_V_SYNC_B_DEFAULT                                          0x00000000
-#define mmCRTCV1_CRTCV_V_SYNC_B_CNTL_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_DTMTEST_CNTL_DEFAULT                                      0x00000000
-#define mmCRTCV1_CRTCV_DTMTEST_STATUS_POSITION_DEFAULT                           0x00000000
-#define mmCRTCV1_CRTCV_TRIGA_CNTL_DEFAULT                                        0x00000000
-#define mmCRTCV1_CRTCV_TRIGA_MANUAL_TRIG_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_TRIGB_CNTL_DEFAULT                                        0x00000000
-#define mmCRTCV1_CRTCV_TRIGB_MANUAL_TRIG_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_FORCE_COUNT_NOW_CNTL_DEFAULT                              0x00000000
-#define mmCRTCV1_CRTCV_FLOW_CONTROL_DEFAULT                                      0x00000000
-#define mmCRTCV1_CRTCV_STEREO_FORCE_NEXT_EYE_DEFAULT                             0x00000000
-#define mmCRTCV1_CRTCV_AVSYNC_COUNTER_DEFAULT                                    0x00000000
-#define mmCRTCV1_CRTCV_CONTROL_DEFAULT                                           0x80400110
-#define mmCRTCV1_CRTCV_BLANK_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_INTERLACE_CONTROL_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_INTERLACE_STATUS_DEFAULT                                  0x00000000
-#define mmCRTCV1_CRTCV_FIELD_INDICATION_CONTROL_DEFAULT                          0x00000000
-#define mmCRTCV1_CRTCV_PIXEL_DATA_READBACK0_DEFAULT                              0x00000000
-#define mmCRTCV1_CRTCV_PIXEL_DATA_READBACK1_DEFAULT                              0x00000000
-#define mmCRTCV1_CRTCV_STATUS_DEFAULT                                            0x00000000
-#define mmCRTCV1_CRTCV_STATUS_POSITION_DEFAULT                                   0x00000000
-#define mmCRTCV1_CRTCV_NOM_VERT_POSITION_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_STATUS_FRAME_COUNT_DEFAULT                                0x00000000
-#define mmCRTCV1_CRTCV_STATUS_VF_COUNT_DEFAULT                                   0x00000000
-#define mmCRTCV1_CRTCV_STATUS_HV_COUNT_DEFAULT                                   0x00000000
-#define mmCRTCV1_CRTCV_COUNT_CONTROL_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_COUNT_RESET_DEFAULT                                       0x00000000
-#define mmCRTCV1_CRTCV_MANUAL_FORCE_VSYNC_NEXT_LINE_DEFAULT                      0x00000000
-#define mmCRTCV1_CRTCV_VERT_SYNC_CONTROL_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_STEREO_STATUS_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_STEREO_CONTROL_DEFAULT                                    0x00000000
-#define mmCRTCV1_CRTCV_SNAPSHOT_STATUS_DEFAULT                                   0x00000000
-#define mmCRTCV1_CRTCV_SNAPSHOT_CONTROL_DEFAULT                                  0x00000000
-#define mmCRTCV1_CRTCV_SNAPSHOT_POSITION_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_SNAPSHOT_FRAME_DEFAULT                                    0x00000000
-#define mmCRTCV1_CRTCV_START_LINE_CONTROL_DEFAULT                                0x00003002
-#define mmCRTCV1_CRTCV_INTERRUPT_CONTROL_DEFAULT                                 0x00000000
-#define mmCRTCV1_CRTCV_UPDATE_LOCK_DEFAULT                                       0x00000000
-#define mmCRTCV1_CRTCV_DOUBLE_BUFFER_CONTROL_DEFAULT                             0x00000000
-#define mmCRTCV1_CRTCV_VGA_PARAMETER_CAPTURE_MODE_DEFAULT                        0x00000000
-#define mmCRTCV1_CRTCV_TEST_PATTERN_CONTROL_DEFAULT                              0x00000000
-#define mmCRTCV1_CRTCV_TEST_PATTERN_PARAMETERS_DEFAULT                           0x00000000
-#define mmCRTCV1_CRTCV_TEST_PATTERN_COLOR_DEFAULT                                0x00000000
-#define mmCRTCV1_CRTCV_MASTER_UPDATE_LOCK_DEFAULT                                0x00010000
-#define mmCRTCV1_CRTCV_MASTER_UPDATE_MODE_DEFAULT                                0x00000000
-#define mmCRTCV1_CRTCV_MVP_INBAND_CNTL_INSERT_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_MVP_INBAND_CNTL_INSERT_TIMER_DEFAULT                      0x00000008
-#define mmCRTCV1_CRTCV_MVP_STATUS_DEFAULT                                        0x00000000
-#define mmCRTCV1_CRTCV_MASTER_EN_DEFAULT                                         0x00000000
-#define mmCRTCV1_CRTCV_ALLOW_STOP_OFF_V_CNT_DEFAULT                              0x00010000
-#define mmCRTCV1_CRTCV_V_UPDATE_INT_STATUS_DEFAULT                               0x00000000
-#define mmCRTCV1_CRTCV_OVERSCAN_COLOR_DEFAULT                                    0x00000000
-#define mmCRTCV1_CRTCV_OVERSCAN_COLOR_EXT_DEFAULT                                0x00000000
-#define mmCRTCV1_CRTCV_BLANK_DATA_COLOR_DEFAULT                                  0x00000000
-#define mmCRTCV1_CRTCV_BLANK_DATA_COLOR_EXT_DEFAULT                              0x00000000
-#define mmCRTCV1_CRTCV_BLACK_COLOR_DEFAULT                                       0x00000000
-#define mmCRTCV1_CRTCV_BLACK_COLOR_EXT_DEFAULT                                   0x00000000
-#define mmCRTCV1_CRTCV_VERTICAL_INTERRUPT0_POSITION_DEFAULT                      0x00000000
-#define mmCRTCV1_CRTCV_VERTICAL_INTERRUPT0_CONTROL_DEFAULT                       0x00000000
-#define mmCRTCV1_CRTCV_VERTICAL_INTERRUPT1_POSITION_DEFAULT                      0x00000000
-#define mmCRTCV1_CRTCV_VERTICAL_INTERRUPT1_CONTROL_DEFAULT                       0x00000000
-#define mmCRTCV1_CRTCV_VERTICAL_INTERRUPT2_POSITION_DEFAULT                      0x00000000
-#define mmCRTCV1_CRTCV_VERTICAL_INTERRUPT2_CONTROL_DEFAULT                       0x00000000
-#define mmCRTCV1_CRTCV_CRC_CNTL_DEFAULT                                          0x00000000
-#define mmCRTCV1_CRTCV_CRC0_WINDOWA_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC0_WINDOWA_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC0_WINDOWB_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC0_WINDOWB_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC0_DATA_RG_DEFAULT                                      0x00000000
-#define mmCRTCV1_CRTCV_CRC0_DATA_B_DEFAULT                                       0x00000000
-#define mmCRTCV1_CRTCV_CRC1_WINDOWA_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC1_WINDOWA_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC1_WINDOWB_X_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC1_WINDOWB_Y_CONTROL_DEFAULT                            0x00000000
-#define mmCRTCV1_CRTCV_CRC1_DATA_RG_DEFAULT                                      0x00000000
-#define mmCRTCV1_CRTCV_CRC1_DATA_B_DEFAULT                                       0x00000000
-#define mmCRTCV1_CRTCV_EXT_TIMING_SYNC_CONTROL_DEFAULT                           0x00000000
-#define mmCRTCV1_CRTCV_EXT_TIMING_SYNC_WINDOW_START_DEFAULT                      0x00000000
-#define mmCRTCV1_CRTCV_EXT_TIMING_SYNC_WINDOW_END_DEFAULT                        0x00000000
-#define mmCRTCV1_CRTCV_EXT_TIMING_SYNC_LOSS_INTERRUPT_CONTROL_DEFAULT            0x00000000
-#define mmCRTCV1_CRTCV_EXT_TIMING_SYNC_INTERRUPT_CONTROL_DEFAULT                 0x00000000
-#define mmCRTCV1_CRTCV_EXT_TIMING_SYNC_SIGNAL_INTERRUPT_CONTROL_DEFAULT          0x00000000
-#define mmCRTCV1_CRTCV_STATIC_SCREEN_CONTROL_DEFAULT                             0x00010000
-#define mmCRTCV1_CRTCV_3D_STRUCTURE_CONTROL_DEFAULT                              0x00000010
-#define mmCRTCV1_CRTCV_GSL_VSYNC_GAP_DEFAULT                                     0x00000000
-#define mmCRTCV1_CRTCV_GSL_WINDOW_DEFAULT                                        0x00000000
-#define mmCRTCV1_CRTCV_GSL_CONTROL_DEFAULT                                       0x00020000
-
-
-// addressBlock: dce_dc_hpd0_dispdec
-#define mmHPD0_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD0_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD0_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD0_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD0_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_hpd1_dispdec
-#define mmHPD1_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD1_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD1_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD1_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD1_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_hpd2_dispdec
-#define mmHPD2_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD2_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD2_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD2_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD2_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_hpd3_dispdec
-#define mmHPD3_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD3_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD3_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD3_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD3_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_hpd4_dispdec
-#define mmHPD4_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD4_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD4_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD4_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD4_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_hpd5_dispdec
-#define mmHPD5_DC_HPD_INT_STATUS_DEFAULT                                         0x00000000
-#define mmHPD5_DC_HPD_INT_CONTROL_DEFAULT                                        0x00000000
-#define mmHPD5_DC_HPD_CONTROL_DEFAULT                                            0x10fa09c4
-#define mmHPD5_DC_HPD_FAST_TRAIN_CNTL_DEFAULT                                    0x00000000
-#define mmHPD5_DC_HPD_TOGGLE_FILT_CNTL_DEFAULT                                   0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon2_dispdec
-#define mmDC_PERFMON2_PERFCOUNTER_CNTL_DEFAULT                                   0x00000000
-#define mmDC_PERFMON2_PERFCOUNTER_CNTL2_DEFAULT                                  0x00000000
-#define mmDC_PERFMON2_PERFCOUNTER_STATE_DEFAULT                                  0x00000000
-#define mmDC_PERFMON2_PERFMON_CNTL_DEFAULT                                       0x00000100
-#define mmDC_PERFMON2_PERFMON_CNTL2_DEFAULT                                      0x00000000
-#define mmDC_PERFMON2_PERFMON_CVALUE_INT_MISC_DEFAULT                            0x00000000
-#define mmDC_PERFMON2_PERFMON_CVALUE_LOW_DEFAULT                                 0x00000000
-#define mmDC_PERFMON2_PERFMON_HI_DEFAULT                                         0x00000000
-#define mmDC_PERFMON2_PERFMON_LOW_DEFAULT                                        0x00000000
-
-
-// addressBlock: dce_dc_dp_aux0_dispdec
-#define mmDP_AUX0_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX0_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX0_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX0_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX0_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX0_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX0_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX0_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX0_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX0_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX0_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX0_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX0_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX0_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX0_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX0_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX0_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dp_aux1_dispdec
-#define mmDP_AUX1_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX1_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX1_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX1_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX1_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX1_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX1_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX1_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX1_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX1_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX1_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX1_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX1_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX1_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX1_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX1_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX1_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dp_aux2_dispdec
-#define mmDP_AUX2_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX2_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX2_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX2_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX2_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX2_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX2_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX2_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX2_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX2_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX2_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX2_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX2_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX2_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX2_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX2_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX2_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dp_aux3_dispdec
-#define mmDP_AUX3_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX3_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX3_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX3_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX3_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX3_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX3_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX3_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX3_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX3_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX3_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX3_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX3_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX3_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX3_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX3_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX3_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dp_aux4_dispdec
-#define mmDP_AUX4_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX4_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX4_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX4_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX4_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX4_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX4_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX4_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX4_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX4_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX4_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX4_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX4_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX4_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX4_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX4_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX4_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dp_aux5_dispdec
-#define mmDP_AUX5_AUX_CONTROL_DEFAULT                                            0x01040000
-#define mmDP_AUX5_AUX_SW_CONTROL_DEFAULT                                         0x00000000
-#define mmDP_AUX5_AUX_ARB_CONTROL_DEFAULT                                        0x00000000
-#define mmDP_AUX5_AUX_INTERRUPT_CONTROL_DEFAULT                                  0x00000000
-#define mmDP_AUX5_AUX_SW_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX5_AUX_LS_STATUS_DEFAULT                                          0x00000000
-#define mmDP_AUX5_AUX_SW_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX5_AUX_LS_DATA_DEFAULT                                            0x00000000
-#define mmDP_AUX5_AUX_DPHY_TX_REF_CONTROL_DEFAULT                                0x00320000
-#define mmDP_AUX5_AUX_DPHY_TX_CONTROL_DEFAULT                                    0x00021002
-#define mmDP_AUX5_AUX_DPHY_RX_CONTROL0_DEFAULT                                   0x223d1210
-#define mmDP_AUX5_AUX_DPHY_RX_CONTROL1_DEFAULT                                   0x00000000
-#define mmDP_AUX5_AUX_DPHY_TX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX5_AUX_DPHY_RX_STATUS_DEFAULT                                     0x00000000
-#define mmDP_AUX5_AUX_GTC_SYNC_ERROR_CONTROL_DEFAULT                             0x00210000
-#define mmDP_AUX5_AUX_GTC_SYNC_CONTROLLER_STATUS_DEFAULT                         0x00000000
-#define mmDP_AUX5_AUX_GTC_SYNC_STATUS_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dig0_dispdec
-#define mmDIG0_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG0_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG0_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG0_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG0_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG0_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG0_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG0_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG0_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG0_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG0_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG0_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG0_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG0_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG0_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG0_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG0_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_AVI_INFO0_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_AVI_INFO1_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_AVI_INFO2_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_AVI_INFO3_DEFAULT                                            0x02000000
-#define mmDIG0_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG0_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG0_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG0_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG0_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG0_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG0_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG0_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG0_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG0_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG0_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG0_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG0_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG0_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG0_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG0_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG0_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG0_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG0_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG0_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG0_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG0_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG0_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG0_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG0_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG0_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG0_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG0_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG0_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG0_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG0_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG0_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG0_AFMT_CNTL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dp0_dispdec
-#define mmDP0_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP0_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP0_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP0_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP0_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP0_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP0_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP0_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP0_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP0_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP0_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP0_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP0_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP0_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP0_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP0_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP0_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP0_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP0_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP0_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP0_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP0_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP0_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP0_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP0_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP0_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP0_DP_MSA_V_TIMING_OVERRIDE1_DEFAULT                                  0x00000000
-#define mmDP0_DP_MSA_V_TIMING_OVERRIDE2_DEFAULT                                  0x00000000
-#define mmDP0_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP0_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP0_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP0_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP0_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP0_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP0_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP0_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP0_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP0_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP0_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP0_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP0_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP0_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP0_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP0_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP0_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP0_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP0_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP0_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP0_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP0_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP0_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP0_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_dig1_dispdec
-#define mmDIG1_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG1_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG1_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG1_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG1_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG1_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG1_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG1_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG1_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG1_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG1_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG1_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG1_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG1_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG1_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG1_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG1_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_AVI_INFO0_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_AVI_INFO1_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_AVI_INFO2_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_AVI_INFO3_DEFAULT                                            0x02000000
-#define mmDIG1_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG1_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG1_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG1_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG1_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG1_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG1_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG1_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG1_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG1_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG1_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG1_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG1_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG1_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG1_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG1_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG1_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG1_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG1_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG1_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG1_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG1_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG1_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG1_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG1_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG1_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG1_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG1_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG1_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG1_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG1_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG1_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG1_AFMT_CNTL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dp1_dispdec
-#define mmDP1_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP1_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP1_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP1_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP1_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP1_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP1_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP1_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP1_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP1_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP1_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP1_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP1_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP1_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP1_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP1_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP1_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP1_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP1_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP1_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP1_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP1_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP1_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP1_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP1_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP1_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP1_DP_MSA_V_TIMING_OVERRIDE1_DEFAULT                                  0x00000000
-#define mmDP1_DP_MSA_V_TIMING_OVERRIDE2_DEFAULT                                  0x00000000
-#define mmDP1_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP1_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP1_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP1_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP1_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP1_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP1_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP1_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP1_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP1_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP1_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP1_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP1_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP1_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP1_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP1_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP1_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP1_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP1_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP1_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP1_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP1_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP1_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP1_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_dig2_dispdec
-#define mmDIG2_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG2_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG2_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG2_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG2_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG2_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG2_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG2_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG2_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG2_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG2_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG2_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG2_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG2_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG2_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG2_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG2_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_AVI_INFO0_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_AVI_INFO1_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_AVI_INFO2_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_AVI_INFO3_DEFAULT                                            0x02000000
-#define mmDIG2_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG2_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG2_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG2_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG2_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG2_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG2_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG2_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG2_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG2_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG2_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG2_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG2_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG2_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG2_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG2_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG2_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG2_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG2_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG2_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG2_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG2_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG2_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG2_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG2_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG2_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG2_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG2_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG2_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG2_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG2_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG2_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG2_AFMT_CNTL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dp2_dispdec
-#define mmDP2_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP2_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP2_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP2_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP2_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP2_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP2_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP2_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP2_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP2_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP2_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP2_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP2_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP2_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP2_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP2_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP2_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP2_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP2_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP2_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP2_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP2_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP2_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP2_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP2_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP2_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP2_DP_MSA_V_TIMING_OVERRIDE1_DEFAULT                                  0x00000000
-#define mmDP2_DP_MSA_V_TIMING_OVERRIDE2_DEFAULT                                  0x00000000
-#define mmDP2_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP2_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP2_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP2_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP2_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP2_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP2_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP2_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP2_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP2_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP2_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP2_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP2_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP2_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP2_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP2_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP2_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP2_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP2_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP2_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP2_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP2_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP2_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP2_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_dig3_dispdec
-#define mmDIG3_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG3_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG3_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG3_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG3_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG3_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG3_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG3_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG3_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG3_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG3_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG3_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG3_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG3_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG3_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG3_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG3_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_AVI_INFO0_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_AVI_INFO1_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_AVI_INFO2_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_AVI_INFO3_DEFAULT                                            0x02000000
-#define mmDIG3_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG3_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG3_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG3_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG3_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG3_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG3_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG3_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG3_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG3_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG3_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG3_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG3_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG3_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG3_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG3_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG3_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG3_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG3_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG3_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG3_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG3_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG3_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG3_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG3_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG3_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG3_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG3_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG3_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG3_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG3_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG3_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG3_AFMT_CNTL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dp3_dispdec
-#define mmDP3_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP3_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP3_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP3_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP3_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP3_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP3_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP3_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP3_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP3_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP3_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP3_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP3_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP3_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP3_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP3_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP3_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP3_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP3_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP3_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP3_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP3_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP3_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP3_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP3_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP3_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP3_DP_MSA_V_TIMING_OVERRIDE1_DEFAULT                                  0x00000000
-#define mmDP3_DP_MSA_V_TIMING_OVERRIDE2_DEFAULT                                  0x00000000
-#define mmDP3_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP3_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP3_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP3_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP3_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP3_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP3_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP3_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP3_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP3_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP3_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP3_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP3_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP3_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP3_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP3_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP3_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP3_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP3_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP3_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP3_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP3_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP3_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP3_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_dig4_dispdec
-#define mmDIG4_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG4_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG4_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG4_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG4_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG4_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG4_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG4_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG4_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG4_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG4_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG4_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG4_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG4_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG4_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG4_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG4_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_AVI_INFO0_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_AVI_INFO1_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_AVI_INFO2_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_AVI_INFO3_DEFAULT                                            0x02000000
-#define mmDIG4_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG4_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG4_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG4_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG4_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG4_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG4_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG4_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG4_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG4_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG4_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG4_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG4_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG4_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG4_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG4_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG4_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG4_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG4_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG4_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG4_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG4_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG4_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG4_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG4_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG4_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG4_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG4_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG4_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG4_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG4_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG4_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG4_AFMT_CNTL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dp4_dispdec
-#define mmDP4_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP4_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP4_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP4_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP4_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP4_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP4_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP4_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP4_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP4_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP4_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP4_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP4_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP4_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP4_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP4_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP4_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP4_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP4_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP4_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP4_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP4_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP4_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP4_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP4_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP4_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP4_DP_MSA_V_TIMING_OVERRIDE1_DEFAULT                                  0x00000000
-#define mmDP4_DP_MSA_V_TIMING_OVERRIDE2_DEFAULT                                  0x00000000
-#define mmDP4_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP4_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP4_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP4_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP4_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP4_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP4_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP4_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP4_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP4_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP4_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP4_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP4_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP4_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP4_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP4_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP4_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP4_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP4_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP4_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP4_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP4_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP4_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP4_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_dig5_dispdec
-#define mmDIG5_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG5_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG5_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG5_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG5_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG5_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG5_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG5_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG5_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG5_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG5_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG5_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG5_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG5_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG5_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG5_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG5_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_AVI_INFO0_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_AVI_INFO1_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_AVI_INFO2_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_AVI_INFO3_DEFAULT                                            0x02000000
-#define mmDIG5_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG5_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG5_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG5_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG5_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG5_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG5_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG5_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG5_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG5_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG5_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG5_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG5_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG5_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG5_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG5_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG5_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG5_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG5_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG5_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG5_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG5_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG5_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG5_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG5_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG5_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG5_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG5_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG5_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG5_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG5_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG5_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG5_AFMT_CNTL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dp5_dispdec
-#define mmDP5_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP5_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP5_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP5_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP5_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP5_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP5_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP5_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP5_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP5_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP5_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP5_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP5_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP5_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP5_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP5_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP5_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP5_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP5_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP5_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP5_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP5_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP5_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP5_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP5_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP5_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP5_DP_MSA_V_TIMING_OVERRIDE1_DEFAULT                                  0x00000000
-#define mmDP5_DP_MSA_V_TIMING_OVERRIDE2_DEFAULT                                  0x00000000
-#define mmDP5_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP5_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP5_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP5_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP5_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP5_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP5_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP5_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP5_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP5_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP5_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP5_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP5_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP5_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP5_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP5_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP5_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP5_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP5_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP5_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP5_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP5_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP5_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP5_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_dig6_dispdec
-#define mmDIG6_DIG_FE_CNTL_DEFAULT                                               0x00000000
-#define mmDIG6_DIG_OUTPUT_CRC_CNTL_DEFAULT                                       0x00000100
-#define mmDIG6_DIG_OUTPUT_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG6_DIG_CLOCK_PATTERN_DEFAULT                                         0x00000063
-#define mmDIG6_DIG_TEST_PATTERN_DEFAULT                                          0x00000060
-#define mmDIG6_DIG_RANDOM_PATTERN_SEED_DEFAULT                                   0x00222222
-#define mmDIG6_DIG_FIFO_STATUS_DEFAULT                                           0x00000000
-#define mmDIG6_HDMI_CONTROL_DEFAULT                                              0x00010001
-#define mmDIG6_HDMI_STATUS_DEFAULT                                               0x00000000
-#define mmDIG6_HDMI_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000010
-#define mmDIG6_HDMI_ACR_PACKET_CONTROL_DEFAULT                                   0x00010000
-#define mmDIG6_HDMI_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG6_HDMI_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG6_HDMI_INFOFRAME_CONTROL1_DEFAULT                                   0x00000000
-#define mmDIG6_HDMI_GENERIC_PACKET_CONTROL0_DEFAULT                              0x00000000
-#define mmDIG6_AFMT_INTERRUPT_STATUS_DEFAULT                                     0x00000000
-#define mmDIG6_HDMI_GC_DEFAULT                                                   0x00000004
-#define mmDIG6_AFMT_AUDIO_PACKET_CONTROL2_DEFAULT                                0x00000000
-#define mmDIG6_AFMT_ISRC1_0_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_1_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_2_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_3_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC1_4_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_0_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_1_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_2_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_ISRC2_3_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_AVI_INFO0_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_AVI_INFO1_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_AVI_INFO2_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_AVI_INFO3_DEFAULT                                            0x02000000
-#define mmDIG6_AFMT_MPEG_INFO0_DEFAULT                                           0x00000000
-#define mmDIG6_AFMT_MPEG_INFO1_DEFAULT                                           0x00000000
-#define mmDIG6_AFMT_GENERIC_HDR_DEFAULT                                          0x00000000
-#define mmDIG6_AFMT_GENERIC_0_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_1_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_2_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_3_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_4_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_5_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_6_DEFAULT                                            0x00000000
-#define mmDIG6_AFMT_GENERIC_7_DEFAULT                                            0x00000000
-#define mmDIG6_HDMI_GENERIC_PACKET_CONTROL1_DEFAULT                              0x00000000
-#define mmDIG6_HDMI_ACR_32_0_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_32_1_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_44_0_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_44_1_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_48_0_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_48_1_DEFAULT                                             0x00000000
-#define mmDIG6_HDMI_ACR_STATUS_0_DEFAULT                                         0x00000000
-#define mmDIG6_HDMI_ACR_STATUS_1_DEFAULT                                         0x00000000
-#define mmDIG6_AFMT_AUDIO_INFO0_DEFAULT                                          0x00000170
-#define mmDIG6_AFMT_AUDIO_INFO1_DEFAULT                                          0x00000000
-#define mmDIG6_AFMT_60958_0_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_60958_1_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_AUDIO_CRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL0_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL1_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL2_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_RAMP_CONTROL3_DEFAULT                                        0x00000000
-#define mmDIG6_AFMT_60958_2_DEFAULT                                              0x00000000
-#define mmDIG6_AFMT_AUDIO_CRC_RESULT_DEFAULT                                     0x00000000
-#define mmDIG6_AFMT_STATUS_DEFAULT                                               0x00000000
-#define mmDIG6_AFMT_AUDIO_PACKET_CONTROL_DEFAULT                                 0x00000800
-#define mmDIG6_AFMT_VBI_PACKET_CONTROL_DEFAULT                                   0x00000000
-#define mmDIG6_AFMT_INFOFRAME_CONTROL0_DEFAULT                                   0x00000000
-#define mmDIG6_AFMT_AUDIO_SRC_CONTROL_DEFAULT                                    0x00000000
-#define mmDIG6_DIG_BE_CNTL_DEFAULT                                               0x00010000
-#define mmDIG6_DIG_BE_EN_CNTL_DEFAULT                                            0x00000000
-#define mmDIG6_TMDS_CNTL_DEFAULT                                                 0x00000001
-#define mmDIG6_TMDS_CONTROL_CHAR_DEFAULT                                         0x00000000
-#define mmDIG6_TMDS_CONTROL0_FEEDBACK_DEFAULT                                    0x00000000
-#define mmDIG6_TMDS_STEREOSYNC_CTL_SEL_DEFAULT                                   0x00000000
-#define mmDIG6_TMDS_SYNC_CHAR_PATTERN_0_1_DEFAULT                                0x00000000
-#define mmDIG6_TMDS_SYNC_CHAR_PATTERN_2_3_DEFAULT                                0x00000000
-#define mmDIG6_TMDS_CTL_BITS_DEFAULT                                             0x00000000
-#define mmDIG6_TMDS_DCBALANCER_CONTROL_DEFAULT                                   0x00000001
-#define mmDIG6_TMDS_CTL0_1_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG6_TMDS_CTL2_3_GEN_CNTL_DEFAULT                                      0x00000000
-#define mmDIG6_DIG_VERSION_DEFAULT                                               0x00000000
-#define mmDIG6_DIG_LANE_ENABLE_DEFAULT                                           0x00000000
-#define mmDIG6_AFMT_CNTL_DEFAULT                                                 0x00000000
-
-
-// addressBlock: dce_dc_dp6_dispdec
-#define mmDP6_DP_LINK_CNTL_DEFAULT                                               0x00000000
-#define mmDP6_DP_PIXEL_FORMAT_DEFAULT                                            0x00000000
-#define mmDP6_DP_MSA_COLORIMETRY_DEFAULT                                         0x00000000
-#define mmDP6_DP_CONFIG_DEFAULT                                                  0x00000000
-#define mmDP6_DP_VID_STREAM_CNTL_DEFAULT                                         0x00000200
-#define mmDP6_DP_STEER_FIFO_DEFAULT                                              0x00000000
-#define mmDP6_DP_MSA_MISC_DEFAULT                                                0x00000000
-#define mmDP6_DP_VID_TIMING_DEFAULT                                              0x00000000
-#define mmDP6_DP_VID_N_DEFAULT                                                   0x00002000
-#define mmDP6_DP_VID_M_DEFAULT                                                   0x00000000
-#define mmDP6_DP_LINK_FRAMING_CNTL_DEFAULT                                       0x10002000
-#define mmDP6_DP_HBR2_EYE_PATTERN_DEFAULT                                        0x00000000
-#define mmDP6_DP_VID_MSA_VBID_DEFAULT                                            0x01000000
-#define mmDP6_DP_VID_INTERRUPT_CNTL_DEFAULT                                      0x00000000
-#define mmDP6_DP_DPHY_CNTL_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_TRAINING_PATTERN_SEL_DEFAULT                               0x00000000
-#define mmDP6_DP_DPHY_SYM0_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_SYM1_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_SYM2_DEFAULT                                               0x00000000
-#define mmDP6_DP_DPHY_8B10B_CNTL_DEFAULT                                         0x00000000
-#define mmDP6_DP_DPHY_PRBS_CNTL_DEFAULT                                          0x7fffff00
-#define mmDP6_DP_DPHY_SCRAM_CNTL_DEFAULT                                         0x0101ff10
-#define mmDP6_DP_DPHY_CRC_EN_DEFAULT                                             0x00000000
-#define mmDP6_DP_DPHY_CRC_CNTL_DEFAULT                                           0x00ff0000
-#define mmDP6_DP_DPHY_CRC_RESULT_DEFAULT                                         0x00000000
-#define mmDP6_DP_DPHY_CRC_MST_CNTL_DEFAULT                                       0x00000000
-#define mmDP6_DP_DPHY_CRC_MST_STATUS_DEFAULT                                     0x00000000
-#define mmDP6_DP_DPHY_FAST_TRAINING_DEFAULT                                      0x20020000
-#define mmDP6_DP_DPHY_FAST_TRAINING_STATUS_DEFAULT                               0x00000000
-#define mmDP6_DP_MSA_V_TIMING_OVERRIDE1_DEFAULT                                  0x00000000
-#define mmDP6_DP_MSA_V_TIMING_OVERRIDE2_DEFAULT                                  0x00000000
-#define mmDP6_DP_SEC_CNTL_DEFAULT                                                0x00000000
-#define mmDP6_DP_SEC_CNTL1_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_FRAMING1_DEFAULT                                            0x00000000
-#define mmDP6_DP_SEC_FRAMING2_DEFAULT                                            0x00000000
-#define mmDP6_DP_SEC_FRAMING3_DEFAULT                                            0x00000200
-#define mmDP6_DP_SEC_FRAMING4_DEFAULT                                            0x00000000
-#define mmDP6_DP_SEC_AUD_N_DEFAULT                                               0x00008000
-#define mmDP6_DP_SEC_AUD_N_READBACK_DEFAULT                                      0x00000000
-#define mmDP6_DP_SEC_AUD_M_DEFAULT                                               0x00000000
-#define mmDP6_DP_SEC_AUD_M_READBACK_DEFAULT                                      0x00000000
-#define mmDP6_DP_SEC_TIMESTAMP_DEFAULT                                           0x00000000
-#define mmDP6_DP_SEC_PACKET_CNTL_DEFAULT                                         0x00001100
-#define mmDP6_DP_MSE_RATE_CNTL_DEFAULT                                           0x00000000
-#define mmDP6_DP_MSE_RATE_UPDATE_DEFAULT                                         0x00000000
-#define mmDP6_DP_MSE_SAT0_DEFAULT                                                0x00000000
-#define mmDP6_DP_MSE_SAT1_DEFAULT                                                0x00000000
-#define mmDP6_DP_MSE_SAT2_DEFAULT                                                0x00000000
-#define mmDP6_DP_MSE_SAT_UPDATE_DEFAULT                                          0x00000000
-#define mmDP6_DP_MSE_LINK_TIMING_DEFAULT                                         0x000203ff
-#define mmDP6_DP_MSE_MISC_CNTL_DEFAULT                                           0x00000000
-#define mmDP6_DP_DPHY_BS_SR_SWAP_CNTL_DEFAULT                                    0x00000005
-#define mmDP6_DP_DPHY_HBR2_PATTERN_CONTROL_DEFAULT                               0x00000000
-#define mmDP6_DP_MSE_SAT0_STATUS_DEFAULT                                         0x00000000
-#define mmDP6_DP_MSE_SAT1_STATUS_DEFAULT                                         0x00000000
-#define mmDP6_DP_MSE_SAT2_STATUS_DEFAULT                                         0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy0_dispdec
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY0_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs0_dispdec
-#define mmDC_COMBOPHYCMREGS0_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS0_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS0_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS0_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS0_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS0_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs0_dispdec
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS0_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS0_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs0_dispdec
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS0_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS0_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS0_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS0_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS0_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS0_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS0_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS0_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS0_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy1_dispdec
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY1_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs1_dispdec
-#define mmDC_COMBOPHYCMREGS1_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS1_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS1_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS1_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS1_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS1_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs1_dispdec
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS1_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS1_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs1_dispdec
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS1_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS1_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS1_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS1_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS1_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS1_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS1_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS1_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS1_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy2_dispdec
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY2_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs2_dispdec
-#define mmDC_COMBOPHYCMREGS2_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS2_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS2_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS2_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS2_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS2_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs2_dispdec
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS2_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS2_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs2_dispdec
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS2_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS2_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS2_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS2_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS2_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS2_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS2_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS2_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS2_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy3_dispdec
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY3_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs3_dispdec
-#define mmDC_COMBOPHYCMREGS3_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS3_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS3_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS3_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS3_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS3_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs3_dispdec
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS3_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS3_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs3_dispdec
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS3_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS3_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS3_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS3_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS3_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS3_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS3_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS3_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS3_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy4_dispdec
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY4_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs4_dispdec
-#define mmDC_COMBOPHYCMREGS4_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS4_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS4_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS4_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS4_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS4_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs4_dispdec
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS4_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS4_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS4_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS4_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS4_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs4_dispdec
-#define mmDC_COMBOPHYPLLREGS4_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS4_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS4_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS4_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS4_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS4_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS4_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS4_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS4_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS4_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS4_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS4_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy5_dispdec
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY5_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs5_dispdec
-#define mmDC_COMBOPHYCMREGS5_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS5_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS5_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS5_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS5_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS5_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs5_dispdec
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS5_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS5_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS5_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS5_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS5_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs5_dispdec
-#define mmDC_COMBOPHYPLLREGS5_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS5_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS5_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS5_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS5_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS5_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS5_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS5_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS5_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS5_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS5_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS5_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy6_dispdec
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY6_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs6_dispdec
-#define mmDC_COMBOPHYCMREGS6_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS6_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS6_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS6_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS6_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS6_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs6_dispdec
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS6_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS6_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS6_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS6_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS6_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs6_dispdec
-#define mmDC_COMBOPHYPLLREGS6_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS6_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS6_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS6_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS6_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS6_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS6_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS6_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS6_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS6_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS6_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS6_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dcio_uniphy8_dispdec
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED0_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED1_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED2_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED3_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED4_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED5_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED6_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED7_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED8_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED9_DEFAULT                       0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED10_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED11_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED12_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED13_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED14_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED15_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED16_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED17_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED18_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED19_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED20_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED21_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED22_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED23_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED24_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED25_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED26_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED27_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED28_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED29_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED30_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED31_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED32_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED33_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED34_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED35_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED36_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED37_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED38_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED39_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED40_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED41_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED42_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED43_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED44_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED45_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED46_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED47_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED48_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED49_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED50_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED51_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED52_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED53_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED54_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED55_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED56_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED57_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED58_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED59_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED60_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED61_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED62_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED63_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED64_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED65_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED66_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED67_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED68_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED69_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED70_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED71_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED72_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED73_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED74_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED75_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED76_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED77_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED78_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED79_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED80_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED81_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED82_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED83_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED84_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED85_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED86_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED87_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED88_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED89_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED90_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED91_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED92_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED93_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED94_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED95_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED96_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED97_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED98_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED99_DEFAULT                      0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED100_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED101_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED102_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED103_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED104_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED105_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED106_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED107_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED108_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED109_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED110_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED111_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED112_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED113_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED114_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED115_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED116_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED117_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED118_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED119_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED120_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED121_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED122_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED123_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED124_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED125_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED126_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED127_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED128_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED129_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED130_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED131_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED132_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED133_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED134_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED135_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED136_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED137_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED138_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED139_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED140_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED141_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED142_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED143_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED144_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED145_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED146_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED147_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED148_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED149_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED150_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED151_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED152_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED153_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED154_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED155_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED156_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED157_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED158_DEFAULT                     0x00000000
-#define mmDCIO_UNIPHY8_UNIPHY_MACRO_CNTL_RESERVED159_DEFAULT                     0x00000000
-
-
-// addressBlock: dce_dc_dc_combophycmregs8_dispdec
-#define mmDC_COMBOPHYCMREGS8_COMMON_FUSE1_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_FUSE2_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_FUSE3_DEFAULT                                0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_MAR_DEEMPH_NOM_DEFAULT                       0x402a2a00
-#define mmDC_COMBOPHYCMREGS8_COMMON_LANE_PWRMGMT_DEFAULT                         0x00000004
-#define mmDC_COMBOPHYCMREGS8_COMMON_TXCNTRL_DEFAULT                              0x00000007
-#define mmDC_COMBOPHYCMREGS8_COMMON_TMDP_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_LANE_RESETS_DEFAULT                          0x000000ff
-#define mmDC_COMBOPHYCMREGS8_COMMON_ZCALCODE_CTRL_DEFAULT                        0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_DISP_RFU1_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_DISP_RFU2_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_DISP_RFU3_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_DISP_RFU4_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_DISP_RFU5_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_DISP_RFU6_DEFAULT                            0x00000000
-#define mmDC_COMBOPHYCMREGS8_COMMON_DISP_RFU7_DEFAULT                            0x00000000
-
-
-// addressBlock: dce_dc_dc_combophytxregs8_dispdec
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_TX_CONTROL_LANE0_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS8_MARGIN_DEEMPH_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_GLOBAL_FOR_TX_LANE0_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU0_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU1_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU2_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU3_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU4_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU5_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU6_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU7_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU8_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU9_LANE0_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU10_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU11_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU12_LANE0_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_TX_CONTROL_LANE1_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS8_MARGIN_DEEMPH_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_GLOBAL_FOR_TX_LANE1_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU0_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU1_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU2_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU3_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU4_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU5_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU6_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU7_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU8_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU9_LANE1_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU10_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU11_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU12_LANE1_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_TX_CONTROL_LANE2_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS8_MARGIN_DEEMPH_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_GLOBAL_FOR_TX_LANE2_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU0_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU1_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU2_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU3_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU4_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU5_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU6_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU7_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU8_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU9_LANE2_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU10_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU11_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU12_LANE2_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_TX_CONTROL_LANE3_DEFAULT                    0x00000006
-#define mmDC_COMBOPHYTXREGS8_MARGIN_DEEMPH_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_CMD_BUS_GLOBAL_FOR_TX_LANE3_DEFAULT                 0x00000040
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU0_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU1_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU2_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU3_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU4_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU5_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU6_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU7_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU8_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU9_LANE3_DEFAULT                          0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU10_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU11_LANE3_DEFAULT                         0x00000000
-#define mmDC_COMBOPHYTXREGS8_TX_DISP_RFU12_LANE3_DEFAULT                         0x00000000
-
-
-// addressBlock: dce_dc_dc_combophypllregs8_dispdec
-#define mmDC_COMBOPHYPLLREGS8_FREQ_CTRL0_DEFAULT                                 0x00280000
-#define mmDC_COMBOPHYPLLREGS8_FREQ_CTRL1_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS8_FREQ_CTRL2_DEFAULT                                 0x00000000
-#define mmDC_COMBOPHYPLLREGS8_FREQ_CTRL3_DEFAULT                                 0x00e80000
-#define mmDC_COMBOPHYPLLREGS8_BW_CTRL_COARSE_DEFAULT                             0x0020c4b1
-#define mmDC_COMBOPHYPLLREGS8_BW_CTRL_FINE_DEFAULT                               0x00000001
-#define mmDC_COMBOPHYPLLREGS8_CAL_CTRL_DEFAULT                                   0x64000000
-#define mmDC_COMBOPHYPLLREGS8_LOOP_CTRL_DEFAULT                                  0x00000090
-#define mmDC_COMBOPHYPLLREGS8_VREG_CFG_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS8_OBSERVE0_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS8_OBSERVE1_DEFAULT                                   0x00000000
-#define mmDC_COMBOPHYPLLREGS8_DFT_OUT_DEFAULT                                    0x00000000
-
-
-// addressBlock: dce_dc_dsi0_dispdec
-#define mmDSI0_DISP_DSI_CTRL_DEFAULT                                             0x00000000
-#define mmDSI0_DISP_DSI_STATUS_DEFAULT                                           0x00000000
-#define mmDSI0_DISP_DSI_VIDEO_MODE_CTRL_DEFAULT                                  0x00008000
-#define mmDSI0_DISP_DSI_VIDEO_MODE_SYNC_DATATYPE_DEFAULT                         0x31211101
-#define mmDSI0_DISP_DSI_VIDEO_MODE_VSYNC_PAYLOAD_DEFAULT                         0x00000000
-#define mmDSI0_DISP_DSI_VIDEO_MODE_HSYNC_PAYLOAD_DEFAULT                         0x00000000
-#define mmDSI0_DISP_DSI_VIDEO_MODE_PIXEL_DATATYPE_DEFAULT                        0x3e2e1e0e
-#define mmDSI0_DISP_DSI_VIDEO_MODE_BLANKING_DATATYPE_DEFAULT                     0x00001900
-#define mmDSI0_DISP_DSI_VIDEO_MODE_DATA_CTRL_DEFAULT                             0x00000000
-#define mmDSI0_DISP_DSI_COMMAND_MODE_CTRL_DEFAULT                                0x00000000
-#define mmDSI0_DISP_DSI_COMMAND_MODE_DATA_CTRL_DEFAULT                           0x00000066
-#define mmDSI0_DISP_DSI_COMMAND_MODE_DCS_CMD_CTRL_DEFAULT                        0x00003c2c
-#define mmDSI0_DISP_DSI_DMA_CMD_OFFSET_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_DMA_CMD_LENGTH_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_DMA_DATA_OFFSET_0_DEFAULT                                0x00000000
-#define mmDSI0_DISP_DSI_DMA_DATA_OFFSET_1_DEFAULT                                0x00000000
-#define mmDSI0_DISP_DSI_DMA_DATA_PITCH_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_DMA_DATA_WIDTH_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_DMA_DATA_HEIGHT_DEFAULT                                  0x00000000
-#define mmDSI0_DISP_DSI_DMA_FIFO_CTRL_DEFAULT                                    0x00000000
-#define mmDSI0_DISP_DSI_DMA_NULL_PACKET_DATA_DEFAULT                             0x00000900
-#define mmDSI0_DISP_DSI_DENG_DATA_LENGTH_DEFAULT                                 0x00000000
-#define mmDSI0_DISP_DSI_ACK_ERROR_REPORT_DEFAULT                                 0x00000000
-#define mmDSI0_DISP_DSI_RDBK_DATA0_DEFAULT                                       0x00000000
-#define mmDSI0_DISP_DSI_RDBK_DATA1_DEFAULT                                       0x00000000
-#define mmDSI0_DISP_DSI_RDBK_DATA2_DEFAULT                                       0x00000000
-#define mmDSI0_DISP_DSI_RDBK_DATA3_DEFAULT                                       0x00000000
-#define mmDSI0_DISP_DSI_RDBK_DATATYPE0_DEFAULT                                   0x22211211
-#define mmDSI0_DISP_DSI_RDBK_DATATYPE1_DEFAULT                                   0x001c1a02
-#define mmDSI0_DISP_DSI_TRIG_CTRL_DEFAULT                                        0x00000000
-#define mmDSI0_DISP_DSI_EXT_MUX_DEFAULT                                          0x00000000
-#define mmDSI0_DISP_DSI_EXT_TE_PULSE_DETECTION_CTRL_DEFAULT                      0x00000000
-#define mmDSI0_DISP_DSI_CMD_MODE_DMA_SW_TRIGGER_DEFAULT                          0x00000000
-#define mmDSI0_DISP_DSI_CMD_MODE_DENG_SW_TRIGGER_DEFAULT                         0x00000000
-#define mmDSI0_DISP_DSI_CMD_MODE_BTA_SW_TRIGGER_DEFAULT                          0x00000000
-#define mmDSI0_DISP_DSI_RESET_SW_TRIGGER_DEFAULT                                 0x00000000
-#define mmDSI0_DISP_DSI_EXT_RESET_DEFAULT                                        0x00000000
-#define mmDSI0_DISP_DSI_LANE_CRC_HS_MODE_DEFAULT                                 0x00000000
-#define mmDSI0_DISP_DSI_LANE_CRC_LP_MODE_DEFAULT                                 0x00000000
-#define mmDSI0_DISP_DSI_LANE_CRC_CTRL_DEFAULT                                    0x00000000
-#define mmDSI0_DISP_DSI_PIXEL_CRC_CTRL_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_LANE_CTRL_DEFAULT                                        0x00000000
-#define mmDSI0_DISP_DSI_DLN0_PHY_ERROR_DEFAULT                                   0x00088888
-#define mmDSI0_DISP_DSI_LP_TIMER_CTRL_DEFAULT                                    0xffffffff
-#define mmDSI0_DISP_DSI_HS_TIMER_CTRL_DEFAULT                                    0x0000ffff
-#define mmDSI0_DISP_DSI_TIMEOUT_STATUS_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_PHY_CLK_TIMING_CTRL_DEFAULT                              0x00000000
-#define mmDSI0_DISP_DSI_PHY_CLK_TIMING_CTRL2_DEFAULT                             0x00000000
-#define mmDSI0_DISP_DSI_EOT_PACKET_DEFAULT                                       0x010f0f08
-#define mmDSI0_DISP_DSI_EOT_PACKET_CTRL_DEFAULT                                  0x00000000
-#define mmDSI0_DISP_DSI_GENERIC_ESC_TX_TRIGGER_DEFAULT                           0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_CTRL_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_FRAME_SIZE_DEFAULT                             0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_BLOCK_SIZE_DEFAULT                             0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_FRAME_CONFIG_DEFAULT                           0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_LSFR_CTRL_DEFAULT                              0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_LSFR_INIT_DEFAULT                              0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_START_DEFAULT                                  0x00000000
-#define mmDSI0_DISP_DSI_MIPI_BIST_STATUS_DEFAULT                                 0x00000000
-#define mmDSI0_DISP_DSI_ERROR_INTERRUPT_MASK_DEFAULT                             0xfd37377f
-#define mmDSI0_DISP_DSI_INTERRUPT_CTRL_DEFAULT                                   0x02222222
-#define mmDSI0_DISP_DSI_CLK_CTRL_DEFAULT                                         0x00000000
-#define mmDSI0_DISP_DSI_CLK_STATUS_DEFAULT                                       0x00000000
-#define mmDSI0_DISP_DSI_DENG_FIFO_STATUS_DEFAULT                                 0x00000000
-#define mmDSI0_DISP_DSI_DENG_FIFO_CTRL_DEFAULT                                   0x00000000
-#define mmDSI0_DISP_DSI_CMD_FIFO_DATA_DEFAULT                                    0x00000000
-#define mmDSI0_DISP_DSI_CMD_FIFO_CTRL_DEFAULT                                    0x00000001
-#define mmDSI0_DISP_DSI_TE_CTRL_DEFAULT                                          0x00000000
-#define mmDSI0_DISP_DSI_LANE_STATUS_DEFAULT                                      0x00000000
-#define mmDSI0_DISP_DSI_PERF_CTRL_DEFAULT                                        0x00000000
-#define mmDSI0_DISP_DSI_HSYNC_LENGTH_DEFAULT                                     0x00000000
-#define mmDSI0_DISP_DSI_RDBK_NUM_DEFAULT                                         0x00000000
-#define mmDSI0_DISP_DSI_CMD_MEM_PWR_CTRL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_dsi1_dispdec
-#define mmDSI1_DISP_DSI_CTRL_DEFAULT                                             0x00000000
-#define mmDSI1_DISP_DSI_STATUS_DEFAULT                                           0x00000000
-#define mmDSI1_DISP_DSI_VIDEO_MODE_CTRL_DEFAULT                                  0x00008000
-#define mmDSI1_DISP_DSI_VIDEO_MODE_SYNC_DATATYPE_DEFAULT                         0x31211101
-#define mmDSI1_DISP_DSI_VIDEO_MODE_VSYNC_PAYLOAD_DEFAULT                         0x00000000
-#define mmDSI1_DISP_DSI_VIDEO_MODE_HSYNC_PAYLOAD_DEFAULT                         0x00000000
-#define mmDSI1_DISP_DSI_VIDEO_MODE_PIXEL_DATATYPE_DEFAULT                        0x3e2e1e0e
-#define mmDSI1_DISP_DSI_VIDEO_MODE_BLANKING_DATATYPE_DEFAULT                     0x00001900
-#define mmDSI1_DISP_DSI_VIDEO_MODE_DATA_CTRL_DEFAULT                             0x00000000
-#define mmDSI1_DISP_DSI_COMMAND_MODE_CTRL_DEFAULT                                0x00000000
-#define mmDSI1_DISP_DSI_COMMAND_MODE_DATA_CTRL_DEFAULT                           0x00000066
-#define mmDSI1_DISP_DSI_COMMAND_MODE_DCS_CMD_CTRL_DEFAULT                        0x00003c2c
-#define mmDSI1_DISP_DSI_DMA_CMD_OFFSET_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_DMA_CMD_LENGTH_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_DMA_DATA_OFFSET_0_DEFAULT                                0x00000000
-#define mmDSI1_DISP_DSI_DMA_DATA_OFFSET_1_DEFAULT                                0x00000000
-#define mmDSI1_DISP_DSI_DMA_DATA_PITCH_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_DMA_DATA_WIDTH_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_DMA_DATA_HEIGHT_DEFAULT                                  0x00000000
-#define mmDSI1_DISP_DSI_DMA_FIFO_CTRL_DEFAULT                                    0x00000000
-#define mmDSI1_DISP_DSI_DMA_NULL_PACKET_DATA_DEFAULT                             0x00000900
-#define mmDSI1_DISP_DSI_DENG_DATA_LENGTH_DEFAULT                                 0x00000000
-#define mmDSI1_DISP_DSI_ACK_ERROR_REPORT_DEFAULT                                 0x00000000
-#define mmDSI1_DISP_DSI_RDBK_DATA0_DEFAULT                                       0x00000000
-#define mmDSI1_DISP_DSI_RDBK_DATA1_DEFAULT                                       0x00000000
-#define mmDSI1_DISP_DSI_RDBK_DATA2_DEFAULT                                       0x00000000
-#define mmDSI1_DISP_DSI_RDBK_DATA3_DEFAULT                                       0x00000000
-#define mmDSI1_DISP_DSI_RDBK_DATATYPE0_DEFAULT                                   0x22211211
-#define mmDSI1_DISP_DSI_RDBK_DATATYPE1_DEFAULT                                   0x001c1a02
-#define mmDSI1_DISP_DSI_TRIG_CTRL_DEFAULT                                        0x00000000
-#define mmDSI1_DISP_DSI_EXT_MUX_DEFAULT                                          0x00000000
-#define mmDSI1_DISP_DSI_EXT_TE_PULSE_DETECTION_CTRL_DEFAULT                      0x00000000
-#define mmDSI1_DISP_DSI_CMD_MODE_DMA_SW_TRIGGER_DEFAULT                          0x00000000
-#define mmDSI1_DISP_DSI_CMD_MODE_DENG_SW_TRIGGER_DEFAULT                         0x00000000
-#define mmDSI1_DISP_DSI_CMD_MODE_BTA_SW_TRIGGER_DEFAULT                          0x00000000
-#define mmDSI1_DISP_DSI_RESET_SW_TRIGGER_DEFAULT                                 0x00000000
-#define mmDSI1_DISP_DSI_EXT_RESET_DEFAULT                                        0x00000000
-#define mmDSI1_DISP_DSI_LANE_CRC_HS_MODE_DEFAULT                                 0x00000000
-#define mmDSI1_DISP_DSI_LANE_CRC_LP_MODE_DEFAULT                                 0x00000000
-#define mmDSI1_DISP_DSI_LANE_CRC_CTRL_DEFAULT                                    0x00000000
-#define mmDSI1_DISP_DSI_PIXEL_CRC_CTRL_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_LANE_CTRL_DEFAULT                                        0x00000000
-#define mmDSI1_DISP_DSI_DLN0_PHY_ERROR_DEFAULT                                   0x00088888
-#define mmDSI1_DISP_DSI_LP_TIMER_CTRL_DEFAULT                                    0xffffffff
-#define mmDSI1_DISP_DSI_HS_TIMER_CTRL_DEFAULT                                    0x0000ffff
-#define mmDSI1_DISP_DSI_TIMEOUT_STATUS_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_PHY_CLK_TIMING_CTRL_DEFAULT                              0x00000000
-#define mmDSI1_DISP_DSI_PHY_CLK_TIMING_CTRL2_DEFAULT                             0x00000000
-#define mmDSI1_DISP_DSI_EOT_PACKET_DEFAULT                                       0x010f0f08
-#define mmDSI1_DISP_DSI_EOT_PACKET_CTRL_DEFAULT                                  0x00000000
-#define mmDSI1_DISP_DSI_GENERIC_ESC_TX_TRIGGER_DEFAULT                           0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_CTRL_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_FRAME_SIZE_DEFAULT                             0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_BLOCK_SIZE_DEFAULT                             0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_FRAME_CONFIG_DEFAULT                           0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_LSFR_CTRL_DEFAULT                              0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_LSFR_INIT_DEFAULT                              0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_START_DEFAULT                                  0x00000000
-#define mmDSI1_DISP_DSI_MIPI_BIST_STATUS_DEFAULT                                 0x00000000
-#define mmDSI1_DISP_DSI_ERROR_INTERRUPT_MASK_DEFAULT                             0xfd37377f
-#define mmDSI1_DISP_DSI_INTERRUPT_CTRL_DEFAULT                                   0x02222222
-#define mmDSI1_DISP_DSI_CLK_CTRL_DEFAULT                                         0x00000000
-#define mmDSI1_DISP_DSI_CLK_STATUS_DEFAULT                                       0x00000000
-#define mmDSI1_DISP_DSI_DENG_FIFO_STATUS_DEFAULT                                 0x00000000
-#define mmDSI1_DISP_DSI_DENG_FIFO_CTRL_DEFAULT                                   0x00000000
-#define mmDSI1_DISP_DSI_CMD_FIFO_DATA_DEFAULT                                    0x00000000
-#define mmDSI1_DISP_DSI_CMD_FIFO_CTRL_DEFAULT                                    0x00000001
-#define mmDSI1_DISP_DSI_TE_CTRL_DEFAULT                                          0x00000000
-#define mmDSI1_DISP_DSI_LANE_STATUS_DEFAULT                                      0x00000000
-#define mmDSI1_DISP_DSI_PERF_CTRL_DEFAULT                                        0x00000000
-#define mmDSI1_DISP_DSI_HSYNC_LENGTH_DEFAULT                                     0x00000000
-#define mmDSI1_DISP_DSI_RDBK_NUM_DEFAULT                                         0x00000000
-#define mmDSI1_DISP_DSI_CMD_MEM_PWR_CTRL_DEFAULT                                 0x00000000
-
-
-// addressBlock: dce_dc_dprx_sd0_dispdec
-#define mmDPRX_SD0_DPRX_SD_CONTROL_DEFAULT                                       0x00000000
-#define mmDPRX_SD0_DPRX_SD_STREAM_ENABLE_DEFAULT                                 0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA0_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA1_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA2_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA3_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA4_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA5_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA6_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA7_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA8_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_VBID_DEFAULT                                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_CURRENT_LINE_DEFAULT                                  0x00000000
-#define mmDPRX_SD0_DPRX_SD_DISPLAY_TIMER_SNAPSHOT_DEFAULT                        0x00000000
-#define mmDPRX_SD0_DPRX_SD_DISPLAY_TIMER_MODE_DEFAULT                            0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSE_SAT_DEFAULT                                       0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSE_FORCE_UPDATE_DEFAULT                              0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSE_SAT_ACTIVE_DEFAULT                                0x00000000
-#define mmDPRX_SD0_DPRX_SD_V_PARAMETER_DEFAULT                                   0x00000000
-#define mmDPRX_SD0_DPRX_SD_PIXEL_FORMAT_DEFAULT                                  0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSA_RECEIVED_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD0_DPRX_SD_VIDEO_STREAM_STATUS_TOGGLED_DEFAULT                   0x00000000
-#define mmDPRX_SD0_DPRX_SD_LINE_NUMBER0_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD0_DPRX_SD_LINE_NUMBER0_CONTROL_DEFAULT                          0x0000ffff
-#define mmDPRX_SD0_DPRX_SD_LINE_NUMBER1_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD0_DPRX_SD_LINE_NUMBER1_CONTROL_DEFAULT                          0x0000ffff
-#define mmDPRX_SD0_DPRX_SD_MAIN_DEFRAMING_ERROR_DEFAULT                          0x00000000
-#define mmDPRX_SD0_DPRX_SD_VBID_MAJORITY_VOTE_DEFAULT                            0x00000000
-#define mmDPRX_SD0_DPRX_SD_SECONDARY_DEFRAMING_ERROR_DEFAULT                     0x00000000
-#define mmDPRX_SD0_DPRX_SD_VCPF_PHASE_LOCKED_DEFAULT                             0x00000000
-#define mmDPRX_SD0_DPRX_SD_VCPF_PHASE_ERROR_DEFAULT                              0x00000000
-#define mmDPRX_SD0_DPRX_SD_MAJORITY_VOTE_ERROR_DEFAULT                           0x00000000
-#define mmDPRX_SD0_DPRX_SD_PIXEL_FIFO_ERROR_DEFAULT                              0x00000000
-#define mmDPRX_SD0_DPRX_SD_MAXIMUM_SDP_PAYLOAD_LENGTH_DEFAULT                    0x000003ff
-#define mmDPRX_SD0_DPRX_SD_SDP_STEER_DEFAULT                                     0x00000001
-#define mmDPRX_SD0_DPRX_SD_SDP_RECEIVED_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD0_DPRX_SD_SDP_LEVEL_DEFAULT                                     0x00000000
-#define mmDPRX_SD0_DPRX_SD_SDP_DATA_DEFAULT                                      0x00000000
-#define mmDPRX_SD0_DPRX_SD_SDP_ERROR_DEFAULT                                     0x00000000
-#define mmDPRX_SD0_DPRX_SD_AUDIO_HEADER_DEFAULT                                  0x00000000
-#define mmDPRX_SD0_DPRX_SD_AUDIO_FIFO_ERROR_DEFAULT                              0x00000000
-#define mmDPRX_SD0_DPRX_SD_SDP_CONTROL_DEFAULT                                   0x00000001
-#define mmDPRX_SD0_DPRX_SD_V_TOTAL_MEASURED_DEFAULT                              0x00000000
-#define mmDPRX_SD0_DPRX_SD_H_TOTAL_MEASURED_DEFAULT                              0x00000000
-#define mmDPRX_SD0_DPRX_SD_BS_COUNTER_DEFAULT                                    0x00000000
-#define mmDPRX_SD0_DPRX_SD_MSE_ACT_HANDLED_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_dprx_sd1_dispdec
-#define mmDPRX_SD1_DPRX_SD_CONTROL_DEFAULT                                       0x00000000
-#define mmDPRX_SD1_DPRX_SD_STREAM_ENABLE_DEFAULT                                 0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA0_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA1_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA2_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA3_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA4_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA5_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA6_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA7_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA8_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_VBID_DEFAULT                                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_CURRENT_LINE_DEFAULT                                  0x00000000
-#define mmDPRX_SD1_DPRX_SD_DISPLAY_TIMER_SNAPSHOT_DEFAULT                        0x00000000
-#define mmDPRX_SD1_DPRX_SD_DISPLAY_TIMER_MODE_DEFAULT                            0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSE_SAT_DEFAULT                                       0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSE_FORCE_UPDATE_DEFAULT                              0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSE_SAT_ACTIVE_DEFAULT                                0x00000000
-#define mmDPRX_SD1_DPRX_SD_V_PARAMETER_DEFAULT                                   0x00000000
-#define mmDPRX_SD1_DPRX_SD_PIXEL_FORMAT_DEFAULT                                  0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSA_RECEIVED_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD1_DPRX_SD_VIDEO_STREAM_STATUS_TOGGLED_DEFAULT                   0x00000000
-#define mmDPRX_SD1_DPRX_SD_LINE_NUMBER0_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD1_DPRX_SD_LINE_NUMBER0_CONTROL_DEFAULT                          0x0000ffff
-#define mmDPRX_SD1_DPRX_SD_LINE_NUMBER1_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD1_DPRX_SD_LINE_NUMBER1_CONTROL_DEFAULT                          0x0000ffff
-#define mmDPRX_SD1_DPRX_SD_MAIN_DEFRAMING_ERROR_DEFAULT                          0x00000000
-#define mmDPRX_SD1_DPRX_SD_VBID_MAJORITY_VOTE_DEFAULT                            0x00000000
-#define mmDPRX_SD1_DPRX_SD_SECONDARY_DEFRAMING_ERROR_DEFAULT                     0x00000000
-#define mmDPRX_SD1_DPRX_SD_VCPF_PHASE_LOCKED_DEFAULT                             0x00000000
-#define mmDPRX_SD1_DPRX_SD_VCPF_PHASE_ERROR_DEFAULT                              0x00000000
-#define mmDPRX_SD1_DPRX_SD_MAJORITY_VOTE_ERROR_DEFAULT                           0x00000000
-#define mmDPRX_SD1_DPRX_SD_PIXEL_FIFO_ERROR_DEFAULT                              0x00000000
-#define mmDPRX_SD1_DPRX_SD_MAXIMUM_SDP_PAYLOAD_LENGTH_DEFAULT                    0x000003ff
-#define mmDPRX_SD1_DPRX_SD_SDP_STEER_DEFAULT                                     0x00000001
-#define mmDPRX_SD1_DPRX_SD_SDP_RECEIVED_STATUS_DEFAULT                           0x00000000
-#define mmDPRX_SD1_DPRX_SD_SDP_LEVEL_DEFAULT                                     0x00000000
-#define mmDPRX_SD1_DPRX_SD_SDP_DATA_DEFAULT                                      0x00000000
-#define mmDPRX_SD1_DPRX_SD_SDP_ERROR_DEFAULT                                     0x00000000
-#define mmDPRX_SD1_DPRX_SD_AUDIO_HEADER_DEFAULT                                  0x00000000
-#define mmDPRX_SD1_DPRX_SD_AUDIO_FIFO_ERROR_DEFAULT                              0x00000000
-#define mmDPRX_SD1_DPRX_SD_SDP_CONTROL_DEFAULT                                   0x00000001
-#define mmDPRX_SD1_DPRX_SD_V_TOTAL_MEASURED_DEFAULT                              0x00000000
-#define mmDPRX_SD1_DPRX_SD_H_TOTAL_MEASURED_DEFAULT                              0x00000000
-#define mmDPRX_SD1_DPRX_SD_BS_COUNTER_DEFAULT                                    0x00000000
-#define mmDPRX_SD1_DPRX_SD_MSE_ACT_HANDLED_DEFAULT                               0x00000000
-
-
-// addressBlock: dce_dc_dc_perfmon10_dispdec
-#define mmDC_PERFMON10_PERFCOUNTER_CNTL_DEFAULT                                  0x00000000
-#define mmDC_PERFMON10_PERFCOUNTER_CNTL2_DEFAULT                                 0x00000000
-#define mmDC_PERFMON10_PERFCOUNTER_STATE_DEFAULT                                 0x00000000
-#define mmDC_PERFMON10_PERFMON_CNTL_DEFAULT                                      0x00000100
-#define mmDC_PERFMON10_PERFMON_CNTL2_DEFAULT                                     0x00000000
-#define mmDC_PERFMON10_PERFMON_CVALUE_INT_MISC_DEFAULT                           0x00000000
-#define mmDC_PERFMON10_PERFMON_CVALUE_LOW_DEFAULT                                0x00000000
-#define mmDC_PERFMON10_PERFMON_HI_DEFAULT                                        0x00000000
-#define mmDC_PERFMON10_PERFMON_LOW_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_dc_zcalregs_dispdec
-#define mmCOMP_EN_CTL_DEFAULT                                                    0x00080000
-#define mmCOMP_EN_DFX_DEFAULT                                                    0x00000000
-#define mmZCAL_FUSES_DEFAULT                                                     0x00000000
-
-
-// addressBlock: dce_dc_dispdec_VGA_MEM_WRITE_PAGE_ADDR
-
-
-// addressBlock: dce_dc_dispdec_VGA_MEM_READ_PAGE_ADDR
-
-
-// addressBlock: dce_dc_dispdec[948..986]
-
-
-// addressBlock: dce_dc_azdec
-#define mmCORB_WRITE_POINTER_DEFAULT                                             0x00000000
-#define mmCORB_READ_POINTER_DEFAULT                                              0x00000000
-#define mmCORB_CONTROL_DEFAULT                                                   0x00000000
-#define mmCORB_STATUS_DEFAULT                                                    0x00000000
-#define mmCORB_SIZE_DEFAULT                                                      0x00000002
-#define mmRIRB_LOWER_BASE_ADDRESS_DEFAULT                                        0x00000000
-#define mmRIRB_UPPER_BASE_ADDRESS_DEFAULT                                        0x00000000
-#define mmRIRB_WRITE_POINTER_DEFAULT                                             0x00000000
-#define mmRESPONSE_INTERRUPT_COUNT_DEFAULT                                       0x00000000
-#define mmRIRB_CONTROL_DEFAULT                                                   0x00000000
-#define mmRIRB_STATUS_DEFAULT                                                    0x00000000
-#define mmRIRB_SIZE_DEFAULT                                                      0x00000002
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_DATA_DEFAULT              0x00000000
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_INPUT_INTERFACE_INDEX_DEFAULT             0x00000000
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT             0x00000000
-#define mmAZENDPOINT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT            0x00000000
-#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT                 0x00000000
-#define mmAZROOT_IMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT                0x00000000
-#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_DEFAULT                             0x00000000
-#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_DATA_DEFAULT                        0x00000000
-#define mmIMMEDIATE_COMMAND_OUTPUT_INTERFACE_INDEX_DEFAULT                       0x00000000
-#define mmIMMEDIATE_RESPONSE_INPUT_INTERFACE_DEFAULT                             0x00000000
-#define mmIMMEDIATE_COMMAND_STATUS_DEFAULT                                       0x00000000
-#define mmDMA_POSITION_LOWER_BASE_ADDRESS_DEFAULT                                0x00000000
-#define mmDMA_POSITION_UPPER_BASE_ADDRESS_DEFAULT                                0x00000000
-#define mmWALL_CLOCK_COUNTER_ALIAS_DEFAULT                                       0x00000000
-
-
-// addressBlock: dce_dc_azstream0_azdec
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM0_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_azstream1_azdec
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM1_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_azstream2_azdec
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM2_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_azstream3_azdec
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM3_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_azstream4_azdec
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM4_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_azstream5_azdec
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM5_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_azstream6_azdec
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM6_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: dce_dc_azstream7_azdec
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_CONTROL_AND_STATUS_DEFAULT          0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_DEFAULT 0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_CYCLIC_BUFFER_LENGTH_DEFAULT        0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LAST_VALID_INDEX_DEFAULT            0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_FIFO_SIZE_DEFAULT                   0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_FORMAT_DEFAULT                      0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_LOWER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_BDL_POINTER_UPPER_BASE_ADDRESS_DEFAULT 0x00000000
-#define mmAZSTREAM7_OUTPUT_STREAM_DESCRIPTOR_LINK_POSITION_IN_CURRENT_BUFFER_ALIAS_DEFAULT 0x00000000
-
-
-// addressBlock: azf0stream0_streamind
-#define ixAZF0STREAM0_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM0_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM0_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM0_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM0_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream1_streamind
-#define ixAZF0STREAM1_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM1_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM1_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM1_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM1_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream2_streamind
-#define ixAZF0STREAM2_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM2_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM2_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM2_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM2_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream3_streamind
-#define ixAZF0STREAM3_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM3_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM3_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM3_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM3_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream4_streamind
-#define ixAZF0STREAM4_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM4_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM4_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM4_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM4_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream5_streamind
-#define ixAZF0STREAM5_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM5_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM5_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM5_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM5_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream6_streamind
-#define ixAZF0STREAM6_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM6_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM6_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM6_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM6_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream7_streamind
-#define ixAZF0STREAM7_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM7_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM7_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM7_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM7_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream8_streamind
-#define ixAZF0STREAM8_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM8_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM8_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM8_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM8_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream9_streamind
-#define ixAZF0STREAM9_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                           0x00203004
-#define ixAZF0STREAM9_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                     0x00000000
-#define ixAZF0STREAM9_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                     0x00000000
-#define ixAZF0STREAM9_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM9_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                    0x00000000
-
-
-// addressBlock: azf0stream10_streamind
-#define ixAZF0STREAM10_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM10_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM10_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM10_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM10_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream11_streamind
-#define ixAZF0STREAM11_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM11_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM11_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM11_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM11_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream12_streamind
-#define ixAZF0STREAM12_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM12_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM12_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM12_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM12_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream13_streamind
-#define ixAZF0STREAM13_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM13_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM13_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM13_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM13_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream14_streamind
-#define ixAZF0STREAM14_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM14_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM14_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM14_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM14_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0stream15_streamind
-#define ixAZF0STREAM15_AZALIA_FIFO_SIZE_CONTROL_DEFAULT                          0x00203004
-#define ixAZF0STREAM15_AZALIA_LATENCY_COUNTER_CONTROL_DEFAULT                    0x00000000
-#define ixAZF0STREAM15_AZALIA_WORSTCASE_LATENCY_COUNT_DEFAULT                    0x00000000
-#define ixAZF0STREAM15_AZALIA_CUMULATIVE_LATENCY_COUNT_DEFAULT                   0x00000000
-#define ixAZF0STREAM15_AZALIA_CUMULATIVE_REQUEST_COUNT_DEFAULT                   0x00000000
-
-
-// addressBlock: azf0endpoint0_endpointind
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT0_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint1_endpointind
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT1_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint2_endpointind
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT2_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint3_endpointind
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT3_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint4_endpointind
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT4_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint5_endpointind
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT5_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint6_endpointind
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT6_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0endpoint7_endpointind
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000221
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT         0x00300000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MIN_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_CONVERTER_GTC_COUNTER_DELTA_MAX_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400380
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT       0x00000094
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT   0x7fffffff
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CHANNEL_SPEAKER_DEFAULT      0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR0_DEFAULT    0x07010701
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR1_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR2_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR3_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR4_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR5_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR6_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR7_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR8_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR9_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR10_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR11_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR12_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR13_DEFAULT   0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR_DEFAULT         0x00000001
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO0_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO1_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO2_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO3_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO4_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO5_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO6_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO7_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_SINK_INFO8_DEFAULT           0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18560010
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT        0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_ASSOCIATION_INFO_DEFAULT             0xffffffff
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_DEFAULT                 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT  0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT          0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT       0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT 0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT     0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLE_STATUS_DEFAULT                    0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_ENABLED_INT_STATUS_DEFAULT               0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_DISABLED_INT_STATUS_DEFAULT              0x00000000
-#define ixAZF0ENDPOINT7_AZALIA_F0_AUDIO_FORMAT_CHANGED_INT_STATUS_DEFAULT        0x00000000
-
-
-// addressBlock: azf0inputendpoint0_inputendpointind
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT0_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint1_inputendpointind
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT1_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint2_inputendpointind
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT2_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint3_inputendpointind
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT3_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint4_inputendpointind
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT4_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint5_inputendpointind
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT5_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint6_inputendpointind
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT6_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: azf0inputendpoint7_inputendpointind
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00100301
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00020070
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00400280
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT 0x000000a4
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_INPUT_PIN_SENSE_DEFAULT 0x7fffffff
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT 0x00000020
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL_ENABLE2_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_HBR_DEFAULT 0x00000001
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_HOT_PLUG_CONTROL_DEFAULT 0x00000010
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_FORCE_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x18d600f0
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT      0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT 0x00000000
-#define ixAZF0INPUTENDPOINT7_AZALIA_F0_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT 0x00000000
-
-
-// addressBlock: f2codecind
-#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_VENDOR_AND_DEVICE_ID_DEFAULT            0x00000000
-#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_REVISION_ID_DEFAULT                     0x00000000
-#define ixAZALIA_F2_CODEC_ROOT_PARAMETER_SUBORDINATE_NODE_COUNT_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_POWER_STATE_DEFAULT                   0x00000003
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_2_DEFAULT       0x00000001
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_3_DEFAULT       0x000000aa
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESPONSE_SUBSYSTEM_ID_4_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_CONVERTER_SYNCHRONIZATION_DEFAULT     0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_CONTROL_RESET_DEFAULT                         0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUBORDINATE_NODE_COUNT_DEFAULT      0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_GROUP_TYPE_DEFAULT                  0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_FUNCTION_PARAMETER_POWER_STATES_DEFAULT                0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT            0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT            0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_2_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_STRIPE_CONTROL_DEFAULT                       0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_DIGITAL_CONVERTER_3_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_RAMP_RATE_DEFAULT                    0x000000b4
-#define ixAZALIA_F2_CODEC_CONVERTER_CONTROL_GTC_EMBEDDING_DEFAULT                0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT  0x00000020
-#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONNECTION_LIST_ENTRY_DEFAULT     0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIDGET_CONTROL_DEFAULT                     0x00000040
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT                 0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT     0x00000010
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2_DEFAULT   0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3_DEFAULT   0x00000056
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4_DEFAULT   0x00000018
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_RESPONSE_SPEAKER_ALLOCATION_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT                 0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_DOWN_MIX_INFO_DEFAULT                      0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DEFAULT                   0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_DESCRIPTOR_DATA_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL01_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL23_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL45_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL67_ENABLE_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LIPSYNC_DEFAULT                            0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_HBR_DEFAULT                                0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_INDEX_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_AUDIO_SINK_INFO_DATA_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL1_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL3_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL5_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL7_ENABLE_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MULTICHANNEL_MODE_DEFAULT                  0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_0_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_1_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_2_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_3_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_4_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_5_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_6_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_7_DEFAULT                      0x00000000
-#define ixAZALIA_F2_PIN_CONTROL_CODEC_CS_OVERRIDE_8_DEFAULT                      0x00000000
-#define ixAZALIA_F2_CODEC_PIN_ASSOCIATION_INFO_DEFAULT                           0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_DIGITAL_OUTPUT_STATUS_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT              0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_DEFAULT                               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT                0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_CODING_TYPE_DEFAULT                        0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_FORMAT_CHANGED_DEFAULT                     0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_WIRELESS_DISPLAY_IDENTIFICATION_DEFAULT    0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_REMOTE_KEEPALIVE_DEFAULT                   0x00000000
-#define ixAZALIA_F2_CODEC_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CAPABILITIES_DEFAULT                     0x00000000
-#define ixAZALIA_F2_CODEC_PIN_PARAMETER_CONNECTION_LIST_LENGTH_DEFAULT           0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CONVERTER_FORMAT_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_CHANNEL_STREAM_ID_DEFAULT      0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_CONTROL_DIGITAL_CONVERTER_DEFAULT      0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT 0x00000020
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_SUPPORTED_SIZE_RATES_DEFAULT 0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_CONVERTER_PARAMETER_STREAM_FORMATS_DEFAULT       0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_WIDGET_CONTROL_DEFAULT               0x00000020
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_UNSOLICITED_RESPONSE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_PIN_SENSE_DEFAULT           0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_DEFAULT 0x000000f0
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_2_DEFAULT 0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_3_DEFAULT 0x000000d6
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_RESPONSE_CONFIGURATION_DEFAULT_4_DEFAULT 0x00000018
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_ALLOCATION_DEFAULT           0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL0_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL2_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL4_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL6_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_HBR_DEFAULT                          0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL1_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL3_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL5_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_MULTICHANNEL7_ENABLE_DEFAULT         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_SNAPSHOT_CONTROL_DEFAULT        0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_DEFAULT                         0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_LPIB_TIMER_SNAPSHOT_DEFAULT          0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INPUT_STATUS_CONTROL_DEFAULT         0x00000010
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_INFOFRAME_DEFAULT                    0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_L_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_CONTROL_CHANNEL_STATUS_H_DEFAULT             0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_AUDIO_WIDGET_CAPABILITIES_DEFAULT  0x00000000
-#define ixAZALIA_F2_CODEC_INPUT_PIN_PARAMETER_CAPABILITIES_DEFAULT               0x00000000
-
-
-// addressBlock: descriptorind
-#define ixAUDIO_DESCRIPTOR0_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR1_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR2_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR3_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR4_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR5_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR6_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR7_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR8_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR9_DEFAULT                                              0x00000000
-#define ixAUDIO_DESCRIPTOR10_DEFAULT                                             0x00000000
-#define ixAUDIO_DESCRIPTOR11_DEFAULT                                             0x00000000
-#define ixAUDIO_DESCRIPTOR12_DEFAULT                                             0x00000000
-#define ixAUDIO_DESCRIPTOR13_DEFAULT                                             0x00000000
-
-
-// addressBlock: sinkinfoind
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_MANUFACTURER_ID_DEFAULT                    0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_PRODUCT_ID_DEFAULT                         0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_SINK_DESCRIPTION_LEN_DEFAULT               0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID0_DEFAULT                            0x00000000
-#define ixAZALIA_F2_CODEC_PIN_CONTROL_PORTID1_DEFAULT                            0x00000000
-#define ixSINK_DESCRIPTION0_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION1_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION2_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION3_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION4_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION5_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION6_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION7_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION8_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION9_DEFAULT                                              0x00000000
-#define ixSINK_DESCRIPTION10_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION11_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION12_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION13_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION14_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION15_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION16_DEFAULT                                             0x00000000
-#define ixSINK_DESCRIPTION17_DEFAULT                                             0x00000000
-
-
-// addressBlock: azinputcrc0resultind
-#define ixAZALIA_INPUT_CRC0_CHANNEL0_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL1_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL2_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL3_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL4_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL5_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL6_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC0_CHANNEL7_DEFAULT                                     0x00000000
-
-
-// addressBlock: azinputcrc1resultind
-#define ixAZALIA_INPUT_CRC1_CHANNEL0_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL1_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL2_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL3_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL4_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL5_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL6_DEFAULT                                     0x00000000
-#define ixAZALIA_INPUT_CRC1_CHANNEL7_DEFAULT                                     0x00000000
-
-
-// addressBlock: azcrc0resultind
-#define ixAZALIA_CRC0_CHANNEL0_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL1_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL2_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL3_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL4_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL5_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL6_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC0_CHANNEL7_DEFAULT                                           0x00000000
-
-
-// addressBlock: azcrc1resultind
-#define ixAZALIA_CRC1_CHANNEL0_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL1_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL2_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL3_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL4_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL5_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL6_DEFAULT                                           0x00000000
-#define ixAZALIA_CRC1_CHANNEL7_DEFAULT                                           0x00000000
-
-
-// addressBlock: vgaseqind
-#define ixSEQ00_DEFAULT                                                          0x00000003
-#define ixSEQ01_DEFAULT                                                          0x00000021
-#define ixSEQ02_DEFAULT                                                          0x00000000
-#define ixSEQ03_DEFAULT                                                          0x00000000
-#define ixSEQ04_DEFAULT                                                          0x00000000
-
-
-// addressBlock: vgacrtind
-#define ixCRT00_DEFAULT                                                          0x00000000
-#define ixCRT01_DEFAULT                                                          0x00000000
-#define ixCRT02_DEFAULT                                                          0x00000000
-#define ixCRT03_DEFAULT                                                          0x00000000
-#define ixCRT04_DEFAULT                                                          0x00000000
-#define ixCRT05_DEFAULT                                                          0x00000000
-#define ixCRT06_DEFAULT                                                          0x00000000
-#define ixCRT07_DEFAULT                                                          0x00000000
-#define ixCRT08_DEFAULT                                                          0x00000000
-#define ixCRT09_DEFAULT                                                          0x00000000
-#define ixCRT0A_DEFAULT                                                          0x00000000
-#define ixCRT0B_DEFAULT                                                          0x00000000
-#define ixCRT0C_DEFAULT                                                          0x00000000
-#define ixCRT0D_DEFAULT                                                          0x00000000
-#define ixCRT0E_DEFAULT                                                          0x00000000
-#define ixCRT0F_DEFAULT                                                          0x00000000
-#define ixCRT10_DEFAULT                                                          0x00000000
-#define ixCRT11_DEFAULT                                                          0x00000000
-#define ixCRT12_DEFAULT                                                          0x00000000
-#define ixCRT13_DEFAULT                                                          0x00000000
-#define ixCRT14_DEFAULT                                                          0x00000000
-#define ixCRT15_DEFAULT                                                          0x00000000
-#define ixCRT16_DEFAULT                                                          0x00000000
-#define ixCRT17_DEFAULT                                                          0x00000000
-#define ixCRT18_DEFAULT                                                          0x00000000
-#define ixCRT1E_DEFAULT                                                          0x00000000
-#define ixCRT1F_DEFAULT                                                          0x00000000
-#define ixCRT22_DEFAULT                                                          0x00000000
-
-
-// addressBlock: vgagrphind
-#define ixGRA00_DEFAULT                                                          0x00000000
-#define ixGRA01_DEFAULT                                                          0x00000000
-#define ixGRA02_DEFAULT                                                          0x00000000
-#define ixGRA03_DEFAULT                                                          0x00000000
-#define ixGRA04_DEFAULT                                                          0x00000000
-#define ixGRA05_DEFAULT                                                          0x00000000
-#define ixGRA06_DEFAULT                                                          0x00000000
-#define ixGRA07_DEFAULT                                                          0x00000000
-#define ixGRA08_DEFAULT                                                          0x00000000
-
-
-// addressBlock: vgaattrind
-#define ixATTR00_DEFAULT                                                         0x00000000
-#define ixATTR01_DEFAULT                                                         0x00000000
-#define ixATTR02_DEFAULT                                                         0x00000000
-#define ixATTR03_DEFAULT                                                         0x00000000
-#define ixATTR04_DEFAULT                                                         0x00000000
-#define ixATTR05_DEFAULT                                                         0x00000000
-#define ixATTR06_DEFAULT                                                         0x00000000
-#define ixATTR07_DEFAULT                                                         0x00000000
-#define ixATTR08_DEFAULT                                                         0x00000000
-#define ixATTR09_DEFAULT                                                         0x00000000
-#define ixATTR0A_DEFAULT                                                         0x00000000
-#define ixATTR0B_DEFAULT                                                         0x00000000
-#define ixATTR0C_DEFAULT                                                         0x00000000
-#define ixATTR0D_DEFAULT                                                         0x00000000
-#define ixATTR0E_DEFAULT                                                         0x00000000
-#define ixATTR0F_DEFAULT                                                         0x00000000
-#define ixATTR10_DEFAULT                                                         0x00000000
-#define ixATTR11_DEFAULT                                                         0x00000000
-#define ixATTR12_DEFAULT                                                         0x00000000
-#define ixATTR13_DEFAULT                                                         0x00000000
-#define ixATTR14_DEFAULT                                                         0x00000000
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_default.h
deleted file mode 100644
index 864690cc910a..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_default.h
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _hdp_4_0_DEFAULT_HEADER
-#define _hdp_4_0_DEFAULT_HEADER
-
-
-// addressBlock: hdp_hdpdec
-#define mmHDP_MMHUB_TLVL_DEFAULT                                                 0x00006666
-#define mmHDP_MMHUB_UNITID_DEFAULT                                               0x00000000
-#define mmHDP_NONSURFACE_BASE_DEFAULT                                            0x00000000
-#define mmHDP_NONSURFACE_INFO_DEFAULT                                            0x00000000
-#define mmHDP_NONSURFACE_BASE_HI_DEFAULT                                         0x00000000
-#define mmHDP_NONSURF_FLAGS_DEFAULT                                              0x00000000
-#define mmHDP_NONSURF_FLAGS_CLR_DEFAULT                                          0x00000000
-#define mmHDP_HOST_PATH_CNTL_DEFAULT                                             0x00680000
-#define mmHDP_SW_SEMAPHORE_DEFAULT                                               0x00000000
-#define mmHDP_DEBUG0_DEFAULT                                                     0x00000000
-#define mmHDP_LAST_SURFACE_HIT_DEFAULT                                           0x00000003
-#define mmHDP_READ_CACHE_INVALIDATE_DEFAULT                                      0x00000000
-#define mmHDP_OUTSTANDING_REQ_DEFAULT                                            0x00000000
-#define mmHDP_MISC_CNTL_DEFAULT                                                  0x2d200861
-#define mmHDP_MEM_POWER_LS_DEFAULT                                               0x00000901
-#define mmHDP_MMHUB_CNTL_DEFAULT                                                 0x00000000
-#define mmHDP_EDC_CNT_DEFAULT                                                    0x00000000
-#define mmHDP_VERSION_DEFAULT                                                    0x00000400
-#define mmHDP_CLK_CNTL_DEFAULT                                                   0x0000000f
-#define mmHDP_MEMIO_CNTL_DEFAULT                                                 0x00000000
-#define mmHDP_MEMIO_ADDR_DEFAULT                                                 0x00000000
-#define mmHDP_MEMIO_STATUS_DEFAULT                                               0x00000000
-#define mmHDP_MEMIO_WR_DATA_DEFAULT                                              0x00000000
-#define mmHDP_MEMIO_RD_DATA_DEFAULT                                              0xdeadbeef
-#define mmHDP_XDP_DIRECT2HDP_FIRST_DEFAULT                                       0x00000000
-#define mmHDP_XDP_D2H_FLUSH_DEFAULT                                              0x00000000
-#define mmHDP_XDP_D2H_BAR_UPDATE_DEFAULT                                         0x00000000
-#define mmHDP_XDP_D2H_RSVD_3_DEFAULT                                             0x00000000
-#define mmHDP_XDP_D2H_RSVD_4_DEFAULT                                             0x00000000
-#define mmHDP_XDP_D2H_RSVD_5_DEFAULT                                             0x00000000
-#define mmHDP_XDP_D2H_RSVD_6_DEFAULT                                             0x00000000
-#define mmHDP_XDP_D2H_RSVD_7_DEFAULT                                             0x00000000
-#define mmHDP_XDP_D2H_RSVD_8_DEFAULT                                             0x00000000
-#define mmHDP_XDP_D2H_RSVD_9_DEFAULT                                             0x00000000
-#define mmHDP_XDP_D2H_RSVD_10_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_11_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_12_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_13_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_14_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_15_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_16_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_17_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_18_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_19_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_20_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_21_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_22_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_23_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_24_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_25_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_26_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_27_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_28_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_29_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_30_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_31_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_32_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_33_DEFAULT                                            0x00000000
-#define mmHDP_XDP_D2H_RSVD_34_DEFAULT                                            0x00000000
-#define mmHDP_XDP_DIRECT2HDP_LAST_DEFAULT                                        0x00000000
-#define mmHDP_XDP_P2P_BAR_CFG_DEFAULT                                            0x0000000f
-#define mmHDP_XDP_P2P_MBX_OFFSET_DEFAULT                                         0x000011bc
-#define mmHDP_XDP_P2P_MBX_ADDR0_DEFAULT                                          0x00000000
-#define mmHDP_XDP_P2P_MBX_ADDR1_DEFAULT                                          0x00000000
-#define mmHDP_XDP_P2P_MBX_ADDR2_DEFAULT                                          0x00000000
-#define mmHDP_XDP_P2P_MBX_ADDR3_DEFAULT                                          0x00000000
-#define mmHDP_XDP_P2P_MBX_ADDR4_DEFAULT                                          0x00000000
-#define mmHDP_XDP_P2P_MBX_ADDR5_DEFAULT                                          0x00000000
-#define mmHDP_XDP_P2P_MBX_ADDR6_DEFAULT                                          0x00000000
-#define mmHDP_XDP_HDP_MBX_MC_CFG_DEFAULT                                         0x00000000
-#define mmHDP_XDP_HDP_MC_CFG_DEFAULT                                             0x00020000
-#define mmHDP_XDP_HST_CFG_DEFAULT                                                0x0000001b
-#define mmHDP_XDP_HDP_IPH_CFG_DEFAULT                                            0x00000000
-#define mmHDP_XDP_P2P_BAR0_DEFAULT                                               0x00000000
-#define mmHDP_XDP_P2P_BAR1_DEFAULT                                               0x00000000
-#define mmHDP_XDP_P2P_BAR2_DEFAULT                                               0x00000000
-#define mmHDP_XDP_P2P_BAR3_DEFAULT                                               0x00000000
-#define mmHDP_XDP_P2P_BAR4_DEFAULT                                               0x00000000
-#define mmHDP_XDP_P2P_BAR5_DEFAULT                                               0x00000000
-#define mmHDP_XDP_P2P_BAR6_DEFAULT                                               0x00000000
-#define mmHDP_XDP_P2P_BAR7_DEFAULT                                               0x00000000
-#define mmHDP_XDP_FLUSH_ARMED_STS_DEFAULT                                        0x00000000
-#define mmHDP_XDP_FLUSH_CNTR0_STS_DEFAULT                                        0x00000000
-#define mmHDP_XDP_BUSY_STS_DEFAULT                                               0x00000000
-#define mmHDP_XDP_STICKY_DEFAULT                                                 0x00000000
-#define mmHDP_XDP_CHKN_DEFAULT                                                   0x48584450
-#define mmHDP_XDP_BARS_ADDR_39_36_DEFAULT                                        0x00000000
-#define mmHDP_XDP_MC_VM_FB_LOCATION_BASE_DEFAULT                                 0x00000000
-#define mmHDP_XDP_GPU_IOV_VIOLATION_LOG_DEFAULT                                  0x00000000
-#define mmHDP_XDP_MMHUB_ERROR_DEFAULT                                            0x00000000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_offset.h
deleted file mode 100644
index fbad771a569e..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_offset.h
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _hdp_4_0_OFFSET_HEADER
-#define _hdp_4_0_OFFSET_HEADER
-
-
-
-// addressBlock: hdp_hdpdec
-// base address: 0x3c80
-#define mmHDP_MMHUB_TLVL                                                                               0x0000
-#define mmHDP_MMHUB_TLVL_BASE_IDX                                                                      0
-#define mmHDP_MMHUB_UNITID                                                                             0x0001
-#define mmHDP_MMHUB_UNITID_BASE_IDX                                                                    0
-#define mmHDP_NONSURFACE_BASE                                                                          0x0040
-#define mmHDP_NONSURFACE_BASE_BASE_IDX                                                                 0
-#define mmHDP_NONSURFACE_INFO                                                                          0x0041
-#define mmHDP_NONSURFACE_INFO_BASE_IDX                                                                 0
-#define mmHDP_NONSURFACE_BASE_HI                                                                       0x0042
-#define mmHDP_NONSURFACE_BASE_HI_BASE_IDX                                                              0
-#define mmHDP_NONSURF_FLAGS                                                                            0x00c8
-#define mmHDP_NONSURF_FLAGS_BASE_IDX                                                                   0
-#define mmHDP_NONSURF_FLAGS_CLR                                                                        0x00c9
-#define mmHDP_NONSURF_FLAGS_CLR_BASE_IDX                                                               0
-#define mmHDP_HOST_PATH_CNTL                                                                           0x00cc
-#define mmHDP_HOST_PATH_CNTL_BASE_IDX                                                                  0
-#define mmHDP_SW_SEMAPHORE                                                                             0x00cd
-#define mmHDP_SW_SEMAPHORE_BASE_IDX                                                                    0
-#define mmHDP_DEBUG0                                                                                   0x00ce
-#define mmHDP_DEBUG0_BASE_IDX                                                                          0
-#define mmHDP_LAST_SURFACE_HIT                                                                         0x00d0
-#define mmHDP_LAST_SURFACE_HIT_BASE_IDX                                                                0
-#define mmHDP_READ_CACHE_INVALIDATE                                                                    0x00d1
-#define mmHDP_READ_CACHE_INVALIDATE_BASE_IDX                                                           0
-#define mmHDP_OUTSTANDING_REQ                                                                          0x00d2
-#define mmHDP_OUTSTANDING_REQ_BASE_IDX                                                                 0
-#define mmHDP_MISC_CNTL                                                                                0x00d3
-#define mmHDP_MISC_CNTL_BASE_IDX                                                                       0
-#define mmHDP_MEM_POWER_LS                                                                             0x00d4
-#define mmHDP_MEM_POWER_LS_BASE_IDX                                                                    0
-#define mmHDP_MMHUB_CNTL                                                                               0x00d5
-#define mmHDP_MMHUB_CNTL_BASE_IDX                                                                      0
-#define mmHDP_EDC_CNT                                                                                  0x00d6
-#define mmHDP_EDC_CNT_BASE_IDX                                                                         0
-#define mmHDP_VERSION                                                                                  0x00d7
-#define mmHDP_VERSION_BASE_IDX                                                                         0
-#define mmHDP_CLK_CNTL                                                                                 0x00d8
-#define mmHDP_CLK_CNTL_BASE_IDX                                                                        0
-#define mmHDP_MEMIO_CNTL                                                                               0x00f6
-#define mmHDP_MEMIO_CNTL_BASE_IDX                                                                      0
-#define mmHDP_MEMIO_ADDR                                                                               0x00f7
-#define mmHDP_MEMIO_ADDR_BASE_IDX                                                                      0
-#define mmHDP_MEMIO_STATUS                                                                             0x00f8
-#define mmHDP_MEMIO_STATUS_BASE_IDX                                                                    0
-#define mmHDP_MEMIO_WR_DATA                                                                            0x00f9
-#define mmHDP_MEMIO_WR_DATA_BASE_IDX                                                                   0
-#define mmHDP_MEMIO_RD_DATA                                                                            0x00fa
-#define mmHDP_MEMIO_RD_DATA_BASE_IDX                                                                   0
-#define mmHDP_XDP_DIRECT2HDP_FIRST                                                                     0x0100
-#define mmHDP_XDP_DIRECT2HDP_FIRST_BASE_IDX                                                            0
-#define mmHDP_XDP_D2H_FLUSH                                                                            0x0101
-#define mmHDP_XDP_D2H_FLUSH_BASE_IDX                                                                   0
-#define mmHDP_XDP_D2H_BAR_UPDATE                                                                       0x0102
-#define mmHDP_XDP_D2H_BAR_UPDATE_BASE_IDX                                                              0
-#define mmHDP_XDP_D2H_RSVD_3                                                                           0x0103
-#define mmHDP_XDP_D2H_RSVD_3_BASE_IDX                                                                  0
-#define mmHDP_XDP_D2H_RSVD_4                                                                           0x0104
-#define mmHDP_XDP_D2H_RSVD_4_BASE_IDX                                                                  0
-#define mmHDP_XDP_D2H_RSVD_5                                                                           0x0105
-#define mmHDP_XDP_D2H_RSVD_5_BASE_IDX                                                                  0
-#define mmHDP_XDP_D2H_RSVD_6                                                                           0x0106
-#define mmHDP_XDP_D2H_RSVD_6_BASE_IDX                                                                  0
-#define mmHDP_XDP_D2H_RSVD_7                                                                           0x0107
-#define mmHDP_XDP_D2H_RSVD_7_BASE_IDX                                                                  0
-#define mmHDP_XDP_D2H_RSVD_8                                                                           0x0108
-#define mmHDP_XDP_D2H_RSVD_8_BASE_IDX                                                                  0
-#define mmHDP_XDP_D2H_RSVD_9                                                                           0x0109
-#define mmHDP_XDP_D2H_RSVD_9_BASE_IDX                                                                  0
-#define mmHDP_XDP_D2H_RSVD_10                                                                          0x010a
-#define mmHDP_XDP_D2H_RSVD_10_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_11                                                                          0x010b
-#define mmHDP_XDP_D2H_RSVD_11_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_12                                                                          0x010c
-#define mmHDP_XDP_D2H_RSVD_12_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_13                                                                          0x010d
-#define mmHDP_XDP_D2H_RSVD_13_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_14                                                                          0x010e
-#define mmHDP_XDP_D2H_RSVD_14_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_15                                                                          0x010f
-#define mmHDP_XDP_D2H_RSVD_15_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_16                                                                          0x0110
-#define mmHDP_XDP_D2H_RSVD_16_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_17                                                                          0x0111
-#define mmHDP_XDP_D2H_RSVD_17_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_18                                                                          0x0112
-#define mmHDP_XDP_D2H_RSVD_18_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_19                                                                          0x0113
-#define mmHDP_XDP_D2H_RSVD_19_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_20                                                                          0x0114
-#define mmHDP_XDP_D2H_RSVD_20_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_21                                                                          0x0115
-#define mmHDP_XDP_D2H_RSVD_21_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_22                                                                          0x0116
-#define mmHDP_XDP_D2H_RSVD_22_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_23                                                                          0x0117
-#define mmHDP_XDP_D2H_RSVD_23_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_24                                                                          0x0118
-#define mmHDP_XDP_D2H_RSVD_24_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_25                                                                          0x0119
-#define mmHDP_XDP_D2H_RSVD_25_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_26                                                                          0x011a
-#define mmHDP_XDP_D2H_RSVD_26_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_27                                                                          0x011b
-#define mmHDP_XDP_D2H_RSVD_27_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_28                                                                          0x011c
-#define mmHDP_XDP_D2H_RSVD_28_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_29                                                                          0x011d
-#define mmHDP_XDP_D2H_RSVD_29_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_30                                                                          0x011e
-#define mmHDP_XDP_D2H_RSVD_30_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_31                                                                          0x011f
-#define mmHDP_XDP_D2H_RSVD_31_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_32                                                                          0x0120
-#define mmHDP_XDP_D2H_RSVD_32_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_33                                                                          0x0121
-#define mmHDP_XDP_D2H_RSVD_33_BASE_IDX                                                                 0
-#define mmHDP_XDP_D2H_RSVD_34                                                                          0x0122
-#define mmHDP_XDP_D2H_RSVD_34_BASE_IDX                                                                 0
-#define mmHDP_XDP_DIRECT2HDP_LAST                                                                      0x0123
-#define mmHDP_XDP_DIRECT2HDP_LAST_BASE_IDX                                                             0
-#define mmHDP_XDP_P2P_BAR_CFG                                                                          0x0124
-#define mmHDP_XDP_P2P_BAR_CFG_BASE_IDX                                                                 0
-#define mmHDP_XDP_P2P_MBX_OFFSET                                                                       0x0125
-#define mmHDP_XDP_P2P_MBX_OFFSET_BASE_IDX                                                              0
-#define mmHDP_XDP_P2P_MBX_ADDR0                                                                        0x0126
-#define mmHDP_XDP_P2P_MBX_ADDR0_BASE_IDX                                                               0
-#define mmHDP_XDP_P2P_MBX_ADDR1                                                                        0x0127
-#define mmHDP_XDP_P2P_MBX_ADDR1_BASE_IDX                                                               0
-#define mmHDP_XDP_P2P_MBX_ADDR2                                                                        0x0128
-#define mmHDP_XDP_P2P_MBX_ADDR2_BASE_IDX                                                               0
-#define mmHDP_XDP_P2P_MBX_ADDR3                                                                        0x0129
-#define mmHDP_XDP_P2P_MBX_ADDR3_BASE_IDX                                                               0
-#define mmHDP_XDP_P2P_MBX_ADDR4                                                                        0x012a
-#define mmHDP_XDP_P2P_MBX_ADDR4_BASE_IDX                                                               0
-#define mmHDP_XDP_P2P_MBX_ADDR5                                                                        0x012b
-#define mmHDP_XDP_P2P_MBX_ADDR5_BASE_IDX                                                               0
-#define mmHDP_XDP_P2P_MBX_ADDR6                                                                        0x012c
-#define mmHDP_XDP_P2P_MBX_ADDR6_BASE_IDX                                                               0
-#define mmHDP_XDP_HDP_MBX_MC_CFG                                                                       0x012d
-#define mmHDP_XDP_HDP_MBX_MC_CFG_BASE_IDX                                                              0
-#define mmHDP_XDP_HDP_MC_CFG                                                                           0x012e
-#define mmHDP_XDP_HDP_MC_CFG_BASE_IDX                                                                  0
-#define mmHDP_XDP_HST_CFG                                                                              0x012f
-#define mmHDP_XDP_HST_CFG_BASE_IDX                                                                     0
-#define mmHDP_XDP_HDP_IPH_CFG                                                                          0x0131
-#define mmHDP_XDP_HDP_IPH_CFG_BASE_IDX                                                                 0
-#define mmHDP_XDP_P2P_BAR0                                                                             0x0134
-#define mmHDP_XDP_P2P_BAR0_BASE_IDX                                                                    0
-#define mmHDP_XDP_P2P_BAR1                                                                             0x0135
-#define mmHDP_XDP_P2P_BAR1_BASE_IDX                                                                    0
-#define mmHDP_XDP_P2P_BAR2                                                                             0x0136
-#define mmHDP_XDP_P2P_BAR2_BASE_IDX                                                                    0
-#define mmHDP_XDP_P2P_BAR3                                                                             0x0137
-#define mmHDP_XDP_P2P_BAR3_BASE_IDX                                                                    0
-#define mmHDP_XDP_P2P_BAR4                                                                             0x0138
-#define mmHDP_XDP_P2P_BAR4_BASE_IDX                                                                    0
-#define mmHDP_XDP_P2P_BAR5                                                                             0x0139
-#define mmHDP_XDP_P2P_BAR5_BASE_IDX                                                                    0
-#define mmHDP_XDP_P2P_BAR6                                                                             0x013a
-#define mmHDP_XDP_P2P_BAR6_BASE_IDX                                                                    0
-#define mmHDP_XDP_P2P_BAR7                                                                             0x013b
-#define mmHDP_XDP_P2P_BAR7_BASE_IDX                                                                    0
-#define mmHDP_XDP_FLUSH_ARMED_STS                                                                      0x013c
-#define mmHDP_XDP_FLUSH_ARMED_STS_BASE_IDX                                                             0
-#define mmHDP_XDP_FLUSH_CNTR0_STS                                                                      0x013d
-#define mmHDP_XDP_FLUSH_CNTR0_STS_BASE_IDX                                                             0
-#define mmHDP_XDP_BUSY_STS                                                                             0x013e
-#define mmHDP_XDP_BUSY_STS_BASE_IDX                                                                    0
-#define mmHDP_XDP_STICKY                                                                               0x013f
-#define mmHDP_XDP_STICKY_BASE_IDX                                                                      0
-#define mmHDP_XDP_CHKN                                                                                 0x0140
-#define mmHDP_XDP_CHKN_BASE_IDX                                                                        0
-#define mmHDP_XDP_BARS_ADDR_39_36                                                                      0x0144
-#define mmHDP_XDP_BARS_ADDR_39_36_BASE_IDX                                                             0
-#define mmHDP_XDP_MC_VM_FB_LOCATION_BASE                                                               0x0145
-#define mmHDP_XDP_MC_VM_FB_LOCATION_BASE_BASE_IDX                                                      0
-#define mmHDP_XDP_GPU_IOV_VIOLATION_LOG                                                                0x0148
-#define mmHDP_XDP_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                       0
-#define mmHDP_XDP_MMHUB_ERROR                                                                          0x0149
-#define mmHDP_XDP_MMHUB_ERROR_BASE_IDX                                                                 0
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_sh_mask.h
deleted file mode 100644
index 586187576d70..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/HDP/hdp_4_0_sh_mask.h
+++ /dev/null
@@ -1,601 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _hdp_4_0_SH_MASK_HEADER
-#define _hdp_4_0_SH_MASK_HEADER
-
-
-// addressBlock: hdp_hdpdec
-//HDP_MMHUB_TLVL
-#define HDP_MMHUB_TLVL__HDP_WR_TLVL__SHIFT                                                                    0x0
-#define HDP_MMHUB_TLVL__HDP_RD_TLVL__SHIFT                                                                    0x4
-#define HDP_MMHUB_TLVL__XDP_WR_TLVL__SHIFT                                                                    0x8
-#define HDP_MMHUB_TLVL__XDP_RD_TLVL__SHIFT                                                                    0xc
-#define HDP_MMHUB_TLVL__XDP_MBX_WR_TLVL__SHIFT                                                                0x10
-#define HDP_MMHUB_TLVL__HDP_WR_TLVL_MASK                                                                      0x00000007L
-#define HDP_MMHUB_TLVL__HDP_RD_TLVL_MASK                                                                      0x00000070L
-#define HDP_MMHUB_TLVL__XDP_WR_TLVL_MASK                                                                      0x00000700L
-#define HDP_MMHUB_TLVL__XDP_RD_TLVL_MASK                                                                      0x00007000L
-#define HDP_MMHUB_TLVL__XDP_MBX_WR_TLVL_MASK                                                                  0x00070000L
-//HDP_MMHUB_UNITID
-#define HDP_MMHUB_UNITID__HDP_UNITID__SHIFT                                                                   0x0
-#define HDP_MMHUB_UNITID__XDP_UNITID__SHIFT                                                                   0x8
-#define HDP_MMHUB_UNITID__XDP_MBX_UNITID__SHIFT                                                               0x10
-#define HDP_MMHUB_UNITID__HDP_UNITID_MASK                                                                     0x0000003FL
-#define HDP_MMHUB_UNITID__XDP_UNITID_MASK                                                                     0x00003F00L
-#define HDP_MMHUB_UNITID__XDP_MBX_UNITID_MASK                                                                 0x003F0000L
-//HDP_NONSURFACE_BASE
-#define HDP_NONSURFACE_BASE__NONSURF_BASE_39_8__SHIFT                                                         0x0
-#define HDP_NONSURFACE_BASE__NONSURF_BASE_39_8_MASK                                                           0xFFFFFFFFL
-//HDP_NONSURFACE_INFO
-#define HDP_NONSURFACE_INFO__NONSURF_SWAP__SHIFT                                                              0x4
-#define HDP_NONSURFACE_INFO__NONSURF_VMID__SHIFT                                                              0x8
-#define HDP_NONSURFACE_INFO__NONSURF_SWAP_MASK                                                                0x00000030L
-#define HDP_NONSURFACE_INFO__NONSURF_VMID_MASK                                                                0x00000F00L
-//HDP_NONSURFACE_BASE_HI
-#define HDP_NONSURFACE_BASE_HI__NONSURF_BASE_47_40__SHIFT                                                     0x0
-#define HDP_NONSURFACE_BASE_HI__NONSURF_BASE_47_40_MASK                                                       0x000000FFL
-//HDP_NONSURF_FLAGS
-#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG__SHIFT                                                          0x0
-#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG__SHIFT                                                           0x1
-#define HDP_NONSURF_FLAGS__NONSURF_WRITE_FLAG_MASK                                                            0x00000001L
-#define HDP_NONSURF_FLAGS__NONSURF_READ_FLAG_MASK                                                             0x00000002L
-//HDP_NONSURF_FLAGS_CLR
-#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR__SHIFT                                                  0x0
-#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR__SHIFT                                                   0x1
-#define HDP_NONSURF_FLAGS_CLR__NONSURF_WRITE_FLAG_CLR_MASK                                                    0x00000001L
-#define HDP_NONSURF_FLAGS_CLR__NONSURF_READ_FLAG_CLR_MASK                                                     0x00000002L
-//HDP_HOST_PATH_CNTL
-#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER__SHIFT                                                             0x9
-#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER__SHIFT                                                             0xb
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_PRELOAD_CFG__SHIFT                                            0x12
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER__SHIFT                                                        0x13
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN__SHIFT                                                           0x15
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_64B_EN__SHIFT                                                       0x16
-#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS__SHIFT                                                           0x1d
-#define HDP_HOST_PATH_CNTL__WRITE_THROUGH_CACHE_DIS__SHIFT                                                    0x1e
-#define HDP_HOST_PATH_CNTL__LIN_RD_CACHE_DIS__SHIFT                                                           0x1f
-#define HDP_HOST_PATH_CNTL__WR_STALL_TIMER_MASK                                                               0x00000600L
-#define HDP_HOST_PATH_CNTL__RD_STALL_TIMER_MASK                                                               0x00001800L
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_PRELOAD_CFG_MASK                                              0x00040000L
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_TIMER_MASK                                                          0x00180000L
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_EN_MASK                                                             0x00200000L
-#define HDP_HOST_PATH_CNTL__WRITE_COMBINE_64B_EN_MASK                                                         0x00400000L
-#define HDP_HOST_PATH_CNTL__ALL_SURFACES_DIS_MASK                                                             0x20000000L
-#define HDP_HOST_PATH_CNTL__WRITE_THROUGH_CACHE_DIS_MASK                                                      0x40000000L
-#define HDP_HOST_PATH_CNTL__LIN_RD_CACHE_DIS_MASK                                                             0x80000000L
-//HDP_SW_SEMAPHORE
-#define HDP_SW_SEMAPHORE__SW_SEMAPHORE__SHIFT                                                                 0x0
-#define HDP_SW_SEMAPHORE__SW_SEMAPHORE_MASK                                                                   0xFFFFFFFFL
-//HDP_DEBUG0
-#define HDP_DEBUG0__HDP_DEBUG__SHIFT                                                                          0x0
-#define HDP_DEBUG0__HDP_DEBUG_MASK                                                                            0xFFFFFFFFL
-//HDP_LAST_SURFACE_HIT
-#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT__SHIFT                                                         0x0
-#define HDP_LAST_SURFACE_HIT__LAST_SURFACE_HIT_MASK                                                           0x00000003L
-//HDP_READ_CACHE_INVALIDATE
-#define HDP_READ_CACHE_INVALIDATE__READ_CACHE_INVALIDATE__SHIFT                                               0x0
-#define HDP_READ_CACHE_INVALIDATE__READ_CACHE_INVALIDATE_MASK                                                 0x00000001L
-//HDP_OUTSTANDING_REQ
-#define HDP_OUTSTANDING_REQ__WRITE_REQ__SHIFT                                                                 0x0
-#define HDP_OUTSTANDING_REQ__READ_REQ__SHIFT                                                                  0x8
-#define HDP_OUTSTANDING_REQ__WRITE_REQ_MASK                                                                   0x000000FFL
-#define HDP_OUTSTANDING_REQ__READ_REQ_MASK                                                                    0x0000FF00L
-//HDP_MISC_CNTL
-#define HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE__SHIFT                                                          0x0
-#define HDP_MISC_CNTL__IDLE_HYSTERESIS_CNTL__SHIFT                                                            0x2
-#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024__SHIFT                                                    0x5
-#define HDP_MISC_CNTL__MULTIPLE_READS__SHIFT                                                                  0x6
-#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES__SHIFT                                                       0xb
-#define HDP_MISC_CNTL__FED_ENABLE__SHIFT                                                                      0x15
-#define HDP_MISC_CNTL__SYSHUB_CHANNEL_PRIORITY__SHIFT                                                         0x17
-#define HDP_MISC_CNTL__MMHUB_WRBURST_ENABLE__SHIFT                                                            0x18
-#define HDP_MISC_CNTL__ALL_FUNCTION_CACHELINE_INVALID__SHIFT                                                  0x19
-#define HDP_MISC_CNTL__HDP_MMHUB_PENDING_WR_TAG_CHECK__SHIFT                                                  0x1a
-#define HDP_MISC_CNTL__XDP_MMHUB_PENDING_WR_TAG_CHECK__SHIFT                                                  0x1b
-#define HDP_MISC_CNTL__VARIABLE_CACHELINE_SIZE__SHIFT                                                         0x1c
-#define HDP_MISC_CNTL__ADAPTIVE_CACHELINE_SIZE__SHIFT                                                         0x1d
-#define HDP_MISC_CNTL__MMHUB_WRBURST_SIZE__SHIFT                                                              0x1e
-#define HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK                                                            0x00000001L
-#define HDP_MISC_CNTL__IDLE_HYSTERESIS_CNTL_MASK                                                              0x0000000CL
-#define HDP_MISC_CNTL__OUTSTANDING_WRITE_COUNT_1024_MASK                                                      0x00000020L
-#define HDP_MISC_CNTL__MULTIPLE_READS_MASK                                                                    0x00000040L
-#define HDP_MISC_CNTL__SIMULTANEOUS_READS_WRITES_MASK                                                         0x00000800L
-#define HDP_MISC_CNTL__FED_ENABLE_MASK                                                                        0x00200000L
-#define HDP_MISC_CNTL__SYSHUB_CHANNEL_PRIORITY_MASK                                                           0x00800000L
-#define HDP_MISC_CNTL__MMHUB_WRBURST_ENABLE_MASK                                                              0x01000000L
-#define HDP_MISC_CNTL__ALL_FUNCTION_CACHELINE_INVALID_MASK                                                    0x02000000L
-#define HDP_MISC_CNTL__HDP_MMHUB_PENDING_WR_TAG_CHECK_MASK                                                    0x04000000L
-#define HDP_MISC_CNTL__XDP_MMHUB_PENDING_WR_TAG_CHECK_MASK                                                    0x08000000L
-#define HDP_MISC_CNTL__VARIABLE_CACHELINE_SIZE_MASK                                                           0x10000000L
-#define HDP_MISC_CNTL__ADAPTIVE_CACHELINE_SIZE_MASK                                                           0x20000000L
-#define HDP_MISC_CNTL__MMHUB_WRBURST_SIZE_MASK                                                                0x40000000L
-//HDP_MEM_POWER_LS
-#define HDP_MEM_POWER_LS__LS_ENABLE__SHIFT                                                                    0x0
-#define HDP_MEM_POWER_LS__LS_HOLD__SHIFT                                                                      0x7
-#define HDP_MEM_POWER_LS__LS_ENABLE_MASK                                                                      0x00000001L
-#define HDP_MEM_POWER_LS__LS_HOLD_MASK                                                                        0x00001F80L
-//HDP_MMHUB_CNTL
-#define HDP_MMHUB_CNTL__HDP_MMHUB_RO__SHIFT                                                                   0x0
-#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC__SHIFT                                                                  0x1
-#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP__SHIFT                                                                0x2
-#define HDP_MMHUB_CNTL__HDP_MMHUB_RO_MASK                                                                     0x00000001L
-#define HDP_MMHUB_CNTL__HDP_MMHUB_GCC_MASK                                                                    0x00000002L
-#define HDP_MMHUB_CNTL__HDP_MMHUB_SNOOP_MASK                                                                  0x00000004L
-//HDP_EDC_CNT
-#define HDP_EDC_CNT__MEM0_SED_COUNT__SHIFT                                                                    0x0
-#define HDP_EDC_CNT__MEM1_SED_COUNT__SHIFT                                                                    0x2
-#define HDP_EDC_CNT__MEM0_SED_COUNT_MASK                                                                      0x00000003L
-#define HDP_EDC_CNT__MEM1_SED_COUNT_MASK                                                                      0x0000000CL
-//HDP_VERSION
-#define HDP_VERSION__MINVER__SHIFT                                                                            0x0
-#define HDP_VERSION__MAJVER__SHIFT                                                                            0x8
-#define HDP_VERSION__REV__SHIFT                                                                               0x10
-#define HDP_VERSION__MINVER_MASK                                                                              0x000000FFL
-#define HDP_VERSION__MAJVER_MASK                                                                              0x0000FF00L
-#define HDP_VERSION__REV_MASK                                                                                 0x00FF0000L
-//HDP_CLK_CNTL
-#define HDP_CLK_CNTL__REG_CLK_ENABLE_COUNT__SHIFT                                                             0x0
-#define HDP_CLK_CNTL__REG_WAKE_DYN_CLK__SHIFT                                                                 0x4
-#define HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE__SHIFT                                                           0x1c
-#define HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE__SHIFT                                                            0x1d
-#define HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE__SHIFT                                                        0x1e
-#define HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE__SHIFT                                                        0x1f
-#define HDP_CLK_CNTL__REG_CLK_ENABLE_COUNT_MASK                                                               0x0000000FL
-#define HDP_CLK_CNTL__REG_WAKE_DYN_CLK_MASK                                                                   0x00000010L
-#define HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK                                                             0x10000000L
-#define HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK                                                              0x20000000L
-#define HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK                                                          0x40000000L
-#define HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK                                                          0x80000000L
-//HDP_MEMIO_CNTL
-#define HDP_MEMIO_CNTL__MEMIO_SEND__SHIFT                                                                     0x0
-#define HDP_MEMIO_CNTL__MEMIO_OP__SHIFT                                                                       0x1
-#define HDP_MEMIO_CNTL__MEMIO_BE__SHIFT                                                                       0x2
-#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE__SHIFT                                                                0x6
-#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE__SHIFT                                                                0x7
-#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER__SHIFT                                                               0x8
-#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR__SHIFT                                                             0xe
-#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR__SHIFT                                                             0xf
-#define HDP_MEMIO_CNTL__MEMIO_VF__SHIFT                                                                       0x10
-#define HDP_MEMIO_CNTL__MEMIO_VFID__SHIFT                                                                     0x11
-#define HDP_MEMIO_CNTL__MEMIO_SEND_MASK                                                                       0x00000001L
-#define HDP_MEMIO_CNTL__MEMIO_OP_MASK                                                                         0x00000002L
-#define HDP_MEMIO_CNTL__MEMIO_BE_MASK                                                                         0x0000003CL
-#define HDP_MEMIO_CNTL__MEMIO_WR_STROBE_MASK                                                                  0x00000040L
-#define HDP_MEMIO_CNTL__MEMIO_RD_STROBE_MASK                                                                  0x00000080L
-#define HDP_MEMIO_CNTL__MEMIO_ADDR_UPPER_MASK                                                                 0x00003F00L
-#define HDP_MEMIO_CNTL__MEMIO_CLR_WR_ERROR_MASK                                                               0x00004000L
-#define HDP_MEMIO_CNTL__MEMIO_CLR_RD_ERROR_MASK                                                               0x00008000L
-#define HDP_MEMIO_CNTL__MEMIO_VF_MASK                                                                         0x00010000L
-#define HDP_MEMIO_CNTL__MEMIO_VFID_MASK                                                                       0x003E0000L
-//HDP_MEMIO_ADDR
-#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER__SHIFT                                                               0x0
-#define HDP_MEMIO_ADDR__MEMIO_ADDR_LOWER_MASK                                                                 0xFFFFFFFFL
-//HDP_MEMIO_STATUS
-#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS__SHIFT                                                              0x0
-#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS__SHIFT                                                              0x1
-#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR__SHIFT                                                               0x2
-#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR__SHIFT                                                               0x3
-#define HDP_MEMIO_STATUS__MEMIO_WR_STATUS_MASK                                                                0x00000001L
-#define HDP_MEMIO_STATUS__MEMIO_RD_STATUS_MASK                                                                0x00000002L
-#define HDP_MEMIO_STATUS__MEMIO_WR_ERROR_MASK                                                                 0x00000004L
-#define HDP_MEMIO_STATUS__MEMIO_RD_ERROR_MASK                                                                 0x00000008L
-//HDP_MEMIO_WR_DATA
-#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA__SHIFT                                                               0x0
-#define HDP_MEMIO_WR_DATA__MEMIO_WR_DATA_MASK                                                                 0xFFFFFFFFL
-//HDP_MEMIO_RD_DATA
-#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA__SHIFT                                                               0x0
-#define HDP_MEMIO_RD_DATA__MEMIO_RD_DATA_MASK                                                                 0xFFFFFFFFL
-//HDP_XDP_DIRECT2HDP_FIRST
-#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED__SHIFT                                                             0x0
-#define HDP_XDP_DIRECT2HDP_FIRST__RESERVED_MASK                                                               0xFFFFFFFFL
-//HDP_XDP_D2H_FLUSH
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM__SHIFT                                                         0x0
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA__SHIFT                                                      0x4
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL__SHIFT                                                      0x8
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG__SHIFT                                                           0xb
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST__SHIFT                                                         0x10
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM__SHIFT                                                   0x12
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0__SHIFT                                                            0x13
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1__SHIFT                                                            0x14
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_FLUSH_NUM_MASK                                                           0x0000000FL
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ENC_DATA_MASK                                                        0x000000F0L
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_MBX_ADDR_SEL_MASK                                                        0x00000700L
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_XPB_CLG_MASK                                                             0x0000F800L
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_SEND_HOST_MASK                                                           0x00010000L
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_ALTER_FLUSH_NUM_MASK                                                     0x00040000L
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_0_MASK                                                              0x00080000L
-#define HDP_XDP_D2H_FLUSH__D2H_FLUSH_RSVD_1_MASK                                                              0x00100000L
-//HDP_XDP_D2H_BAR_UPDATE
-#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR__SHIFT                                                    0x0
-#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM__SHIFT                                               0x10
-#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM__SHIFT                                                 0x14
-#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_ADDR_MASK                                                      0x0000FFFFL
-#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_FLUSH_NUM_MASK                                                 0x000F0000L
-#define HDP_XDP_D2H_BAR_UPDATE__D2H_BAR_UPDATE_BAR_NUM_MASK                                                   0x00700000L
-//HDP_XDP_D2H_RSVD_3
-#define HDP_XDP_D2H_RSVD_3__RESERVED__SHIFT                                                                   0x0
-#define HDP_XDP_D2H_RSVD_3__RESERVED_MASK                                                                     0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_4
-#define HDP_XDP_D2H_RSVD_4__RESERVED__SHIFT                                                                   0x0
-#define HDP_XDP_D2H_RSVD_4__RESERVED_MASK                                                                     0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_5
-#define HDP_XDP_D2H_RSVD_5__RESERVED__SHIFT                                                                   0x0
-#define HDP_XDP_D2H_RSVD_5__RESERVED_MASK                                                                     0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_6
-#define HDP_XDP_D2H_RSVD_6__RESERVED__SHIFT                                                                   0x0
-#define HDP_XDP_D2H_RSVD_6__RESERVED_MASK                                                                     0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_7
-#define HDP_XDP_D2H_RSVD_7__RESERVED__SHIFT                                                                   0x0
-#define HDP_XDP_D2H_RSVD_7__RESERVED_MASK                                                                     0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_8
-#define HDP_XDP_D2H_RSVD_8__RESERVED__SHIFT                                                                   0x0
-#define HDP_XDP_D2H_RSVD_8__RESERVED_MASK                                                                     0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_9
-#define HDP_XDP_D2H_RSVD_9__RESERVED__SHIFT                                                                   0x0
-#define HDP_XDP_D2H_RSVD_9__RESERVED_MASK                                                                     0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_10
-#define HDP_XDP_D2H_RSVD_10__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_10__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_11
-#define HDP_XDP_D2H_RSVD_11__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_11__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_12
-#define HDP_XDP_D2H_RSVD_12__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_12__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_13
-#define HDP_XDP_D2H_RSVD_13__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_13__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_14
-#define HDP_XDP_D2H_RSVD_14__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_14__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_15
-#define HDP_XDP_D2H_RSVD_15__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_15__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_16
-#define HDP_XDP_D2H_RSVD_16__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_16__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_17
-#define HDP_XDP_D2H_RSVD_17__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_17__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_18
-#define HDP_XDP_D2H_RSVD_18__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_18__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_19
-#define HDP_XDP_D2H_RSVD_19__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_19__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_20
-#define HDP_XDP_D2H_RSVD_20__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_20__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_21
-#define HDP_XDP_D2H_RSVD_21__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_21__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_22
-#define HDP_XDP_D2H_RSVD_22__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_22__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_23
-#define HDP_XDP_D2H_RSVD_23__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_23__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_24
-#define HDP_XDP_D2H_RSVD_24__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_24__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_25
-#define HDP_XDP_D2H_RSVD_25__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_25__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_26
-#define HDP_XDP_D2H_RSVD_26__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_26__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_27
-#define HDP_XDP_D2H_RSVD_27__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_27__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_28
-#define HDP_XDP_D2H_RSVD_28__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_28__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_29
-#define HDP_XDP_D2H_RSVD_29__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_29__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_30
-#define HDP_XDP_D2H_RSVD_30__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_30__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_31
-#define HDP_XDP_D2H_RSVD_31__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_31__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_32
-#define HDP_XDP_D2H_RSVD_32__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_32__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_33
-#define HDP_XDP_D2H_RSVD_33__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_33__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_D2H_RSVD_34
-#define HDP_XDP_D2H_RSVD_34__RESERVED__SHIFT                                                                  0x0
-#define HDP_XDP_D2H_RSVD_34__RESERVED_MASK                                                                    0xFFFFFFFFL
-//HDP_XDP_DIRECT2HDP_LAST
-#define HDP_XDP_DIRECT2HDP_LAST__RESERVED__SHIFT                                                              0x0
-#define HDP_XDP_DIRECT2HDP_LAST__RESERVED_MASK                                                                0xFFFFFFFFL
-//HDP_XDP_P2P_BAR_CFG
-#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE__SHIFT                                                     0x0
-#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM__SHIFT                                                      0x4
-#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_ADDR_SIZE_MASK                                                       0x0000000FL
-#define HDP_XDP_P2P_BAR_CFG__P2P_BAR_CFG_BAR_FROM_MASK                                                        0x00000030L
-//HDP_XDP_P2P_MBX_OFFSET
-#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET__SHIFT                                                         0x0
-#define HDP_XDP_P2P_MBX_OFFSET__P2P_MBX_OFFSET_MASK                                                           0x0001FFFFL
-//HDP_XDP_P2P_MBX_ADDR0
-#define HDP_XDP_P2P_MBX_ADDR0__VALID__SHIFT                                                                   0x0
-#define HDP_XDP_P2P_MBX_ADDR0__ADDR_35_19__SHIFT                                                              0x3
-#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36__SHIFT                                                              0x14
-#define HDP_XDP_P2P_MBX_ADDR0__ADDR_47_40__SHIFT                                                              0x18
-#define HDP_XDP_P2P_MBX_ADDR0__VALID_MASK                                                                     0x00000001L
-#define HDP_XDP_P2P_MBX_ADDR0__ADDR_35_19_MASK                                                                0x000FFFF8L
-#define HDP_XDP_P2P_MBX_ADDR0__ADDR_39_36_MASK                                                                0x00F00000L
-#define HDP_XDP_P2P_MBX_ADDR0__ADDR_47_40_MASK                                                                0xFF000000L
-//HDP_XDP_P2P_MBX_ADDR1
-#define HDP_XDP_P2P_MBX_ADDR1__VALID__SHIFT                                                                   0x0
-#define HDP_XDP_P2P_MBX_ADDR1__ADDR_35_19__SHIFT                                                              0x3
-#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36__SHIFT                                                              0x14
-#define HDP_XDP_P2P_MBX_ADDR1__ADDR_47_40__SHIFT                                                              0x18
-#define HDP_XDP_P2P_MBX_ADDR1__VALID_MASK                                                                     0x00000001L
-#define HDP_XDP_P2P_MBX_ADDR1__ADDR_35_19_MASK                                                                0x000FFFF8L
-#define HDP_XDP_P2P_MBX_ADDR1__ADDR_39_36_MASK                                                                0x00F00000L
-#define HDP_XDP_P2P_MBX_ADDR1__ADDR_47_40_MASK                                                                0xFF000000L
-//HDP_XDP_P2P_MBX_ADDR2
-#define HDP_XDP_P2P_MBX_ADDR2__VALID__SHIFT                                                                   0x0
-#define HDP_XDP_P2P_MBX_ADDR2__ADDR_35_19__SHIFT                                                              0x3
-#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36__SHIFT                                                              0x14
-#define HDP_XDP_P2P_MBX_ADDR2__ADDR_47_40__SHIFT                                                              0x18
-#define HDP_XDP_P2P_MBX_ADDR2__VALID_MASK                                                                     0x00000001L
-#define HDP_XDP_P2P_MBX_ADDR2__ADDR_35_19_MASK                                                                0x000FFFF8L
-#define HDP_XDP_P2P_MBX_ADDR2__ADDR_39_36_MASK                                                                0x00F00000L
-#define HDP_XDP_P2P_MBX_ADDR2__ADDR_47_40_MASK                                                                0xFF000000L
-//HDP_XDP_P2P_MBX_ADDR3
-#define HDP_XDP_P2P_MBX_ADDR3__VALID__SHIFT                                                                   0x0
-#define HDP_XDP_P2P_MBX_ADDR3__ADDR_35_19__SHIFT                                                              0x3
-#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36__SHIFT                                                              0x14
-#define HDP_XDP_P2P_MBX_ADDR3__ADDR_47_40__SHIFT                                                              0x18
-#define HDP_XDP_P2P_MBX_ADDR3__VALID_MASK                                                                     0x00000001L
-#define HDP_XDP_P2P_MBX_ADDR3__ADDR_35_19_MASK                                                                0x000FFFF8L
-#define HDP_XDP_P2P_MBX_ADDR3__ADDR_39_36_MASK                                                                0x00F00000L
-#define HDP_XDP_P2P_MBX_ADDR3__ADDR_47_40_MASK                                                                0xFF000000L
-//HDP_XDP_P2P_MBX_ADDR4
-#define HDP_XDP_P2P_MBX_ADDR4__VALID__SHIFT                                                                   0x0
-#define HDP_XDP_P2P_MBX_ADDR4__ADDR_35_19__SHIFT                                                              0x3
-#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36__SHIFT                                                              0x14
-#define HDP_XDP_P2P_MBX_ADDR4__ADDR_47_40__SHIFT                                                              0x18
-#define HDP_XDP_P2P_MBX_ADDR4__VALID_MASK                                                                     0x00000001L
-#define HDP_XDP_P2P_MBX_ADDR4__ADDR_35_19_MASK                                                                0x000FFFF8L
-#define HDP_XDP_P2P_MBX_ADDR4__ADDR_39_36_MASK                                                                0x00F00000L
-#define HDP_XDP_P2P_MBX_ADDR4__ADDR_47_40_MASK                                                                0xFF000000L
-//HDP_XDP_P2P_MBX_ADDR5
-#define HDP_XDP_P2P_MBX_ADDR5__VALID__SHIFT                                                                   0x0
-#define HDP_XDP_P2P_MBX_ADDR5__ADDR_35_19__SHIFT                                                              0x3
-#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36__SHIFT                                                              0x14
-#define HDP_XDP_P2P_MBX_ADDR5__ADDR_47_40__SHIFT                                                              0x18
-#define HDP_XDP_P2P_MBX_ADDR5__VALID_MASK                                                                     0x00000001L
-#define HDP_XDP_P2P_MBX_ADDR5__ADDR_35_19_MASK                                                                0x000FFFF8L
-#define HDP_XDP_P2P_MBX_ADDR5__ADDR_39_36_MASK                                                                0x00F00000L
-#define HDP_XDP_P2P_MBX_ADDR5__ADDR_47_40_MASK                                                                0xFF000000L
-//HDP_XDP_P2P_MBX_ADDR6
-#define HDP_XDP_P2P_MBX_ADDR6__VALID__SHIFT                                                                   0x0
-#define HDP_XDP_P2P_MBX_ADDR6__ADDR_35_19__SHIFT                                                              0x3
-#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36__SHIFT                                                              0x14
-#define HDP_XDP_P2P_MBX_ADDR6__ADDR_47_40__SHIFT                                                              0x18
-#define HDP_XDP_P2P_MBX_ADDR6__VALID_MASK                                                                     0x00000001L
-#define HDP_XDP_P2P_MBX_ADDR6__ADDR_35_19_MASK                                                                0x000FFFF8L
-#define HDP_XDP_P2P_MBX_ADDR6__ADDR_39_36_MASK                                                                0x00F00000L
-#define HDP_XDP_P2P_MBX_ADDR6__ADDR_47_40_MASK                                                                0xFF000000L
-//HDP_XDP_HDP_MBX_MC_CFG
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_QOS__SHIFT                                           0x0
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP__SHIFT                                          0x4
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID__SHIFT                                          0x8
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_RO__SHIFT                                            0xc
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_GCC__SHIFT                                           0xd
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SNOOP__SHIFT                                         0xe
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_QOS_MASK                                             0x0000000FL
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SWAP_MASK                                            0x00000030L
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_VMID_MASK                                            0x00000F00L
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_RO_MASK                                              0x00001000L
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_GCC_MASK                                             0x00002000L
-#define HDP_XDP_HDP_MBX_MC_CFG__HDP_MBX_MC_CFG_TAP_WRREQ_SNOOP_MASK                                           0x00004000L
-//HDP_XDP_HDP_MC_CFG
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP__SHIFT                                               0x3
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SWAP__SHIFT                                                0x4
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_VMID__SHIFT                                                0x8
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO__SHIFT                                                  0xc
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC__SHIFT                                                 0xd
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH__SHIFT                                           0xe
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SNOOP_MASK                                                 0x00000008L
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_SWAP_MASK                                                  0x00000030L
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_VMID_MASK                                                  0x00000F00L
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_RO_MASK                                                    0x00001000L
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_HST_TAP_REQ_GCC_MASK                                                   0x00002000L
-#define HDP_XDP_HDP_MC_CFG__HDP_MC_CFG_XDP_HIGHER_PRI_THRESH_MASK                                             0x000FC000L
-//HDP_XDP_HST_CFG
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN__SHIFT                                                         0x0
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER__SHIFT                                                      0x1
-#define HDP_XDP_HST_CFG__HST_CFG_WR_BURST_EN__SHIFT                                                           0x3
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_64B_EN__SHIFT                                                     0x4
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_PRELOAD_CFG__SHIFT                                          0x5
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_EN_MASK                                                           0x00000001L
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_MASK                                                        0x00000006L
-#define HDP_XDP_HST_CFG__HST_CFG_WR_BURST_EN_MASK                                                             0x00000008L
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_64B_EN_MASK                                                       0x00000010L
-#define HDP_XDP_HST_CFG__HST_CFG_WR_COMBINE_TIMER_PRELOAD_CFG_MASK                                            0x00000020L
-//HDP_XDP_HDP_IPH_CFG
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE__SHIFT                                       0x0
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE__SHIFT                                       0x6
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING__SHIFT                                     0xc
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN__SHIFT                                                     0xd
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_SYS_FIFO_DEPTH_OVERRIDE_MASK                                         0x0000003FL
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_XDP_FIFO_DEPTH_OVERRIDE_MASK                                         0x00000FC0L
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_INVERSE_PEER_TAG_MATCHING_MASK                                       0x00001000L
-#define HDP_XDP_HDP_IPH_CFG__HDP_IPH_CFG_P2P_RD_EN_MASK                                                       0x00002000L
-//HDP_XDP_P2P_BAR0
-#define HDP_XDP_P2P_BAR0__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR0__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR0__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR0__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR0__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR0__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_P2P_BAR1
-#define HDP_XDP_P2P_BAR1__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR1__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR1__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR1__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR1__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR1__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_P2P_BAR2
-#define HDP_XDP_P2P_BAR2__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR2__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR2__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR2__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR2__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR2__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_P2P_BAR3
-#define HDP_XDP_P2P_BAR3__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR3__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR3__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR3__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR3__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR3__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_P2P_BAR4
-#define HDP_XDP_P2P_BAR4__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR4__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR4__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR4__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR4__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR4__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_P2P_BAR5
-#define HDP_XDP_P2P_BAR5__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR5__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR5__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR5__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR5__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR5__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_P2P_BAR6
-#define HDP_XDP_P2P_BAR6__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR6__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR6__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR6__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR6__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR6__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_P2P_BAR7
-#define HDP_XDP_P2P_BAR7__ADDR__SHIFT                                                                         0x0
-#define HDP_XDP_P2P_BAR7__FLUSH__SHIFT                                                                        0x10
-#define HDP_XDP_P2P_BAR7__VALID__SHIFT                                                                        0x14
-#define HDP_XDP_P2P_BAR7__ADDR_MASK                                                                           0x0000FFFFL
-#define HDP_XDP_P2P_BAR7__FLUSH_MASK                                                                          0x000F0000L
-#define HDP_XDP_P2P_BAR7__VALID_MASK                                                                          0x00100000L
-//HDP_XDP_FLUSH_ARMED_STS
-#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS__SHIFT                                                       0x0
-#define HDP_XDP_FLUSH_ARMED_STS__FLUSH_ARMED_STS_MASK                                                         0xFFFFFFFFL
-//HDP_XDP_FLUSH_CNTR0_STS
-#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS__SHIFT                                                       0x0
-#define HDP_XDP_FLUSH_CNTR0_STS__FLUSH_CNTR0_STS_MASK                                                         0x03FFFFFFL
-//HDP_XDP_BUSY_STS
-#define HDP_XDP_BUSY_STS__BUSY_BITS__SHIFT                                                                    0x0
-#define HDP_XDP_BUSY_STS__BUSY_BITS_MASK                                                                      0x0003FFFFL
-//HDP_XDP_STICKY
-#define HDP_XDP_STICKY__STICKY_STS__SHIFT                                                                     0x0
-#define HDP_XDP_STICKY__STICKY_W1C__SHIFT                                                                     0x10
-#define HDP_XDP_STICKY__STICKY_STS_MASK                                                                       0x0000FFFFL
-#define HDP_XDP_STICKY__STICKY_W1C_MASK                                                                       0xFFFF0000L
-//HDP_XDP_CHKN
-#define HDP_XDP_CHKN__CHKN_0_RSVD__SHIFT                                                                      0x0
-#define HDP_XDP_CHKN__CHKN_1_RSVD__SHIFT                                                                      0x8
-#define HDP_XDP_CHKN__CHKN_2_RSVD__SHIFT                                                                      0x10
-#define HDP_XDP_CHKN__CHKN_3_RSVD__SHIFT                                                                      0x18
-#define HDP_XDP_CHKN__CHKN_0_RSVD_MASK                                                                        0x000000FFL
-#define HDP_XDP_CHKN__CHKN_1_RSVD_MASK                                                                        0x0000FF00L
-#define HDP_XDP_CHKN__CHKN_2_RSVD_MASK                                                                        0x00FF0000L
-#define HDP_XDP_CHKN__CHKN_3_RSVD_MASK                                                                        0xFF000000L
-//HDP_XDP_BARS_ADDR_39_36
-#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36__SHIFT                                                       0x0
-#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36__SHIFT                                                       0x4
-#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36__SHIFT                                                       0x8
-#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36__SHIFT                                                       0xc
-#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36__SHIFT                                                       0x10
-#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36__SHIFT                                                       0x14
-#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36__SHIFT                                                       0x18
-#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36__SHIFT                                                       0x1c
-#define HDP_XDP_BARS_ADDR_39_36__BAR0_ADDR_39_36_MASK                                                         0x0000000FL
-#define HDP_XDP_BARS_ADDR_39_36__BAR1_ADDR_39_36_MASK                                                         0x000000F0L
-#define HDP_XDP_BARS_ADDR_39_36__BAR2_ADDR_39_36_MASK                                                         0x00000F00L
-#define HDP_XDP_BARS_ADDR_39_36__BAR3_ADDR_39_36_MASK                                                         0x0000F000L
-#define HDP_XDP_BARS_ADDR_39_36__BAR4_ADDR_39_36_MASK                                                         0x000F0000L
-#define HDP_XDP_BARS_ADDR_39_36__BAR5_ADDR_39_36_MASK                                                         0x00F00000L
-#define HDP_XDP_BARS_ADDR_39_36__BAR6_ADDR_39_36_MASK                                                         0x0F000000L
-#define HDP_XDP_BARS_ADDR_39_36__BAR7_ADDR_39_36_MASK                                                         0xF0000000L
-//HDP_XDP_MC_VM_FB_LOCATION_BASE
-#define HDP_XDP_MC_VM_FB_LOCATION_BASE__FB_BASE__SHIFT                                                        0x0
-#define HDP_XDP_MC_VM_FB_LOCATION_BASE__FB_BASE_MASK                                                          0x03FFFFFFL
-//HDP_XDP_GPU_IOV_VIOLATION_LOG
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                0x0
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                       0x1
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                         0x2
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT                                                          0x12
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                              0x13
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                            0x14
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT                                                    0x18
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                  0x00000001L
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                         0x00000002L
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                           0x0003FFFCL
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__OPCODE_MASK                                                            0x00040000L
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                0x00080000L
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                              0x00F00000L
-#define HDP_XDP_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK                                                      0xFF000000L
-//HDP_XDP_MMHUB_ERROR
-#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_01__SHIFT                                                              0x1
-#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_10__SHIFT                                                              0x2
-#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_11__SHIFT                                                              0x3
-#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_01__SHIFT                                                         0x5
-#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_10__SHIFT                                                         0x6
-#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_11__SHIFT                                                         0x7
-#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_01__SHIFT                                                              0x9
-#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_10__SHIFT                                                              0xa
-#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_11__SHIFT                                                              0xb
-#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_01__SHIFT                                                         0xd
-#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_10__SHIFT                                                         0xe
-#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_11__SHIFT                                                         0xf
-#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_01__SHIFT                                                              0x11
-#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_10__SHIFT                                                              0x12
-#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_11__SHIFT                                                              0x13
-#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_01__SHIFT                                                         0x15
-#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_10__SHIFT                                                         0x16
-#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_11__SHIFT                                                         0x17
-#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_01_MASK                                                                0x00000002L
-#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_10_MASK                                                                0x00000004L
-#define HDP_XDP_MMHUB_ERROR__HDP_BRESP_11_MASK                                                                0x00000008L
-#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_01_MASK                                                           0x00000020L
-#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_10_MASK                                                           0x00000040L
-#define HDP_XDP_MMHUB_ERROR__HDP_BUSER_NACK_11_MASK                                                           0x00000080L
-#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_01_MASK                                                                0x00000200L
-#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_10_MASK                                                                0x00000400L
-#define HDP_XDP_MMHUB_ERROR__HDP_RRESP_11_MASK                                                                0x00000800L
-#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_01_MASK                                                           0x00002000L
-#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_10_MASK                                                           0x00004000L
-#define HDP_XDP_MMHUB_ERROR__HDP_RUSER_NACK_11_MASK                                                           0x00008000L
-#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_01_MASK                                                                0x00020000L
-#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_10_MASK                                                                0x00040000L
-#define HDP_XDP_MMHUB_ERROR__XDP_BRESP_11_MASK                                                                0x00080000L
-#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_01_MASK                                                           0x00200000L
-#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_10_MASK                                                           0x00400000L
-#define HDP_XDP_MMHUB_ERROR__XDP_BUSER_NACK_11_MASK                                                           0x00800000L
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_default.h
deleted file mode 100644
index 98ba7d832423..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_default.h
+++ /dev/null
@@ -1,342 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _mp_9_0_DEFAULT_HEADER
-#define _mp_9_0_DEFAULT_HEADER
-
-
-// addressBlock: mp_SmuMp0_SmnDec
-#define mmMP0_SMN_C2PMSG_32_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_33_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_34_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_35_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_36_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_37_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_38_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_39_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_40_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_41_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_42_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_43_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_44_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_45_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_46_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_47_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_48_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_49_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_50_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_51_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_52_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_53_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_54_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_55_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_56_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_57_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_58_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_59_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_60_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_61_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_62_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_63_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_64_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_65_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_66_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_67_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_68_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_69_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_70_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_71_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_72_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_73_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_74_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_75_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_76_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_77_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_78_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_79_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_80_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_81_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_82_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_83_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_84_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_85_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_86_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_87_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_88_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_89_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_90_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_91_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_92_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_93_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_94_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_95_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_96_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_97_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_98_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_99_DEFAULT                                              0x00000000
-#define mmMP0_SMN_C2PMSG_100_DEFAULT                                             0x00000000
-#define mmMP0_SMN_C2PMSG_101_DEFAULT                                             0x00000000
-#define mmMP0_SMN_C2PMSG_102_DEFAULT                                             0x00000000
-#define mmMP0_SMN_C2PMSG_103_DEFAULT                                             0x00000000
-#define mmMP0_SMN_ACTIVE_FCN_ID_DEFAULT                                          0x00000000
-#define mmMP0_SMN_IH_CREDIT_DEFAULT                                              0x00000000
-#define mmMP0_SMN_IH_SW_INT_DEFAULT                                              0x00000000
-#define mmMP0_SMN_IH_SW_INT_CTRL_DEFAULT                                         0x00000000
-
-
-// addressBlock: mp_SmuMp1_SmnDec
-#define mmMP1_SMN_ACP2MP_RESP_DEFAULT                                            0x00000000
-#define mmMP1_SMN_DC2MP_RESP_DEFAULT                                             0x00000000
-#define mmMP1_SMN_UVD2MP_RESP_DEFAULT                                            0x00000000
-#define mmMP1_SMN_VCE2MP_RESP_DEFAULT                                            0x00000000
-#define mmMP1_SMN_RLC2MP_RESP_DEFAULT                                            0x00000000
-#define mmMP1_SMN_C2PMSG_32_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_33_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_34_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_35_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_36_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_37_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_38_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_39_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_40_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_41_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_42_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_43_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_44_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_45_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_46_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_47_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_48_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_49_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_50_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_51_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_52_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_53_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_54_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_55_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_56_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_57_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_58_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_59_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_60_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_61_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_62_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_63_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_64_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_65_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_66_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_67_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_68_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_69_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_70_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_71_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_72_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_73_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_74_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_75_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_76_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_77_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_78_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_79_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_80_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_81_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_82_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_83_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_84_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_85_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_86_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_87_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_88_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_89_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_90_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_91_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_92_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_93_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_94_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_95_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_96_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_97_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_98_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_99_DEFAULT                                              0x00000000
-#define mmMP1_SMN_C2PMSG_100_DEFAULT                                             0x00000000
-#define mmMP1_SMN_C2PMSG_101_DEFAULT                                             0x00000000
-#define mmMP1_SMN_C2PMSG_102_DEFAULT                                             0x00000000
-#define mmMP1_SMN_C2PMSG_103_DEFAULT                                             0x00000000
-#define mmMP1_SMN_ACTIVE_FCN_ID_DEFAULT                                          0x00000000
-#define mmMP1_SMN_IH_CREDIT_DEFAULT                                              0x00000000
-#define mmMP1_SMN_IH_SW_INT_DEFAULT                                              0x00000000
-#define mmMP1_SMN_IH_SW_INT_CTRL_DEFAULT                                         0x00000000
-#define mmMP1_SMN_FPS_CNT_DEFAULT                                                0x00000000
-#define mmMP1_SMN_EXT_SCRATCH0_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH1_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH2_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH3_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH4_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH5_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH6_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH7_DEFAULT                                           0x00000000
-#define mmMP1_SMN_EXT_SCRATCH8_DEFAULT                                           0x00000000
-
-
-// addressBlock: mp_SmuMp1Pub_CruDec
-#define mmMP1_SMN_PUB_CTRL_DEFAULT                                               0x00000001
-#define smnMP1_FIRMWARE_FLAGS_DEFAULT                                             0x00000000
-#define smnMP1_PUB_SCRATCH0_DEFAULT                                               0x00000000
-#define smnMP1_PUB_SCRATCH1_DEFAULT                                               0x00000000
-#define smnMP1_PUB_SCRATCH2_DEFAULT                                               0x00000000
-#define smnMP1_PUB_SCRATCH3_DEFAULT                                               0x00000000
-#define smnMP1_C2PMSG_0_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_1_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_2_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_3_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_4_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_5_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_6_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_7_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_8_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_9_DEFAULT                                                   0x00000000
-#define smnMP1_C2PMSG_10_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_11_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_12_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_13_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_14_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_15_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_16_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_17_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_18_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_19_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_20_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_21_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_22_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_23_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_24_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_25_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_26_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_27_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_28_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_29_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_30_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_31_DEFAULT                                                  0x00000000
-#define smnMP1_P2CMSG_0_DEFAULT                                                   0x00000000
-#define smnMP1_P2CMSG_1_DEFAULT                                                   0x00000000
-#define smnMP1_P2CMSG_2_DEFAULT                                                   0x00000000
-#define smnMP1_P2CMSG_3_DEFAULT                                                   0x00000000
-#define smnMP1_P2CMSG_INTEN_DEFAULT                                               0x00000000
-#define smnMP1_P2CMSG_INTSTS_DEFAULT                                              0x00000000
-#define smnMP1_P2SMSG_0_DEFAULT                                                   0x00000000
-#define smnMP1_P2SMSG_1_DEFAULT                                                   0x00000000
-#define smnMP1_P2SMSG_2_DEFAULT                                                   0x00000000
-#define smnMP1_P2SMSG_3_DEFAULT                                                   0x00000000
-#define smnMP1_P2SMSG_INTSTS_DEFAULT                                              0x00000000
-#define smnMP1_S2PMSG_0_DEFAULT                                                   0x00000000
-#define smnMP1_ACP2MP_RESP_DEFAULT                                                0x00000000
-#define smnMP1_DC2MP_RESP_DEFAULT                                                 0x00000000
-#define smnMP1_UVD2MP_RESP_DEFAULT                                                0x00000000
-#define smnMP1_VCE2MP_RESP_DEFAULT                                                0x00000000
-#define smnMP1_RLC2MP_RESP_DEFAULT                                                0x00000000
-#define smnMP1_C2PMSG_32_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_33_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_34_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_35_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_36_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_37_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_38_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_39_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_40_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_41_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_42_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_43_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_44_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_45_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_46_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_47_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_48_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_49_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_50_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_51_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_52_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_53_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_54_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_55_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_56_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_57_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_58_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_59_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_60_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_61_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_62_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_63_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_64_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_65_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_66_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_67_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_68_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_69_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_70_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_71_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_72_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_73_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_74_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_75_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_76_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_77_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_78_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_79_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_80_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_81_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_82_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_83_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_84_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_85_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_86_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_87_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_88_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_89_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_90_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_91_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_92_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_93_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_94_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_95_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_96_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_97_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_98_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_99_DEFAULT                                                  0x00000000
-#define smnMP1_C2PMSG_100_DEFAULT                                                 0x00000000
-#define smnMP1_C2PMSG_101_DEFAULT                                                 0x00000000
-#define smnMP1_C2PMSG_102_DEFAULT                                                 0x00000000
-#define smnMP1_C2PMSG_103_DEFAULT                                                 0x00000000
-#define smnMP1_ACTIVE_FCN_ID_DEFAULT                                              0x00000000
-#define smnMP1_IH_CREDIT_DEFAULT                                                  0x00000000
-#define smnMP1_IH_SW_INT_DEFAULT                                                  0x00000000
-#define smnMP1_IH_SW_INT_CTRL_DEFAULT                                             0x00000000
-#define smnMP1_FPS_CNT_DEFAULT                                                    0x00000000
-#define smnMP1_PUB_CTRL_DEFAULT                                                   0x00000001
-#define smnMP1_EXT_SCRATCH0_DEFAULT                                               0x00000000
-#define smnMP1_EXT_SCRATCH1_DEFAULT                                               0x00000000
-#define smnMP1_EXT_SCRATCH2_DEFAULT                                               0x00000000
-#define smnMP1_EXT_SCRATCH3_DEFAULT                                               0x00000000
-#define smnMP1_EXT_SCRATCH4_DEFAULT                                               0x00000000
-#define smnMP1_EXT_SCRATCH5_DEFAULT                                               0x00000000
-#define smnMP1_EXT_SCRATCH6_DEFAULT                                               0x00000000
-#define smnMP1_EXT_SCRATCH7_DEFAULT                                               0x00000000
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_offset.h
deleted file mode 100644
index 621e8809c867..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_offset.h
+++ /dev/null
@@ -1,375 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _mp_9_0_OFFSET_HEADER
-#define _mp_9_0_OFFSET_HEADER
-
-
-
-// addressBlock: mp_SmuMp0_SmnDec
-// base address: 0x0
-#define mmMP0_SMN_C2PMSG_32                                                                            0x0060
-#define mmMP0_SMN_C2PMSG_32_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_33                                                                            0x0061
-#define mmMP0_SMN_C2PMSG_33_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_34                                                                            0x0062
-#define mmMP0_SMN_C2PMSG_34_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_35                                                                            0x0063
-#define mmMP0_SMN_C2PMSG_35_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_36                                                                            0x0064
-#define mmMP0_SMN_C2PMSG_36_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_37                                                                            0x0065
-#define mmMP0_SMN_C2PMSG_37_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_38                                                                            0x0066
-#define mmMP0_SMN_C2PMSG_38_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_39                                                                            0x0067
-#define mmMP0_SMN_C2PMSG_39_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_40                                                                            0x0068
-#define mmMP0_SMN_C2PMSG_40_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_41                                                                            0x0069
-#define mmMP0_SMN_C2PMSG_41_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_42                                                                            0x006a
-#define mmMP0_SMN_C2PMSG_42_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_43                                                                            0x006b
-#define mmMP0_SMN_C2PMSG_43_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_44                                                                            0x006c
-#define mmMP0_SMN_C2PMSG_44_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_45                                                                            0x006d
-#define mmMP0_SMN_C2PMSG_45_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_46                                                                            0x006e
-#define mmMP0_SMN_C2PMSG_46_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_47                                                                            0x006f
-#define mmMP0_SMN_C2PMSG_47_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_48                                                                            0x0070
-#define mmMP0_SMN_C2PMSG_48_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_49                                                                            0x0071
-#define mmMP0_SMN_C2PMSG_49_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_50                                                                            0x0072
-#define mmMP0_SMN_C2PMSG_50_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_51                                                                            0x0073
-#define mmMP0_SMN_C2PMSG_51_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_52                                                                            0x0074
-#define mmMP0_SMN_C2PMSG_52_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_53                                                                            0x0075
-#define mmMP0_SMN_C2PMSG_53_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_54                                                                            0x0076
-#define mmMP0_SMN_C2PMSG_54_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_55                                                                            0x0077
-#define mmMP0_SMN_C2PMSG_55_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_56                                                                            0x0078
-#define mmMP0_SMN_C2PMSG_56_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_57                                                                            0x0079
-#define mmMP0_SMN_C2PMSG_57_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_58                                                                            0x007a
-#define mmMP0_SMN_C2PMSG_58_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_59                                                                            0x007b
-#define mmMP0_SMN_C2PMSG_59_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_60                                                                            0x007c
-#define mmMP0_SMN_C2PMSG_60_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_61                                                                            0x007d
-#define mmMP0_SMN_C2PMSG_61_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_62                                                                            0x007e
-#define mmMP0_SMN_C2PMSG_62_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_63                                                                            0x007f
-#define mmMP0_SMN_C2PMSG_63_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_64                                                                            0x0080
-#define mmMP0_SMN_C2PMSG_64_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_65                                                                            0x0081
-#define mmMP0_SMN_C2PMSG_65_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_66                                                                            0x0082
-#define mmMP0_SMN_C2PMSG_66_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_67                                                                            0x0083
-#define mmMP0_SMN_C2PMSG_67_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_68                                                                            0x0084
-#define mmMP0_SMN_C2PMSG_68_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_69                                                                            0x0085
-#define mmMP0_SMN_C2PMSG_69_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_70                                                                            0x0086
-#define mmMP0_SMN_C2PMSG_70_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_71                                                                            0x0087
-#define mmMP0_SMN_C2PMSG_71_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_72                                                                            0x0088
-#define mmMP0_SMN_C2PMSG_72_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_73                                                                            0x0089
-#define mmMP0_SMN_C2PMSG_73_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_74                                                                            0x008a
-#define mmMP0_SMN_C2PMSG_74_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_75                                                                            0x008b
-#define mmMP0_SMN_C2PMSG_75_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_76                                                                            0x008c
-#define mmMP0_SMN_C2PMSG_76_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_77                                                                            0x008d
-#define mmMP0_SMN_C2PMSG_77_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_78                                                                            0x008e
-#define mmMP0_SMN_C2PMSG_78_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_79                                                                            0x008f
-#define mmMP0_SMN_C2PMSG_79_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_80                                                                            0x0090
-#define mmMP0_SMN_C2PMSG_80_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_81                                                                            0x0091
-#define mmMP0_SMN_C2PMSG_81_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_82                                                                            0x0092
-#define mmMP0_SMN_C2PMSG_82_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_83                                                                            0x0093
-#define mmMP0_SMN_C2PMSG_83_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_84                                                                            0x0094
-#define mmMP0_SMN_C2PMSG_84_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_85                                                                            0x0095
-#define mmMP0_SMN_C2PMSG_85_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_86                                                                            0x0096
-#define mmMP0_SMN_C2PMSG_86_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_87                                                                            0x0097
-#define mmMP0_SMN_C2PMSG_87_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_88                                                                            0x0098
-#define mmMP0_SMN_C2PMSG_88_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_89                                                                            0x0099
-#define mmMP0_SMN_C2PMSG_89_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_90                                                                            0x009a
-#define mmMP0_SMN_C2PMSG_90_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_91                                                                            0x009b
-#define mmMP0_SMN_C2PMSG_91_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_92                                                                            0x009c
-#define mmMP0_SMN_C2PMSG_92_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_93                                                                            0x009d
-#define mmMP0_SMN_C2PMSG_93_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_94                                                                            0x009e
-#define mmMP0_SMN_C2PMSG_94_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_95                                                                            0x009f
-#define mmMP0_SMN_C2PMSG_95_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_96                                                                            0x00a0
-#define mmMP0_SMN_C2PMSG_96_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_97                                                                            0x00a1
-#define mmMP0_SMN_C2PMSG_97_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_98                                                                            0x00a2
-#define mmMP0_SMN_C2PMSG_98_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_99                                                                            0x00a3
-#define mmMP0_SMN_C2PMSG_99_BASE_IDX                                                                   0
-#define mmMP0_SMN_C2PMSG_100                                                                           0x00a4
-#define mmMP0_SMN_C2PMSG_100_BASE_IDX                                                                  0
-#define mmMP0_SMN_C2PMSG_101                                                                           0x00a5
-#define mmMP0_SMN_C2PMSG_101_BASE_IDX                                                                  0
-#define mmMP0_SMN_C2PMSG_102                                                                           0x00a6
-#define mmMP0_SMN_C2PMSG_102_BASE_IDX                                                                  0
-#define mmMP0_SMN_C2PMSG_103                                                                           0x00a7
-#define mmMP0_SMN_C2PMSG_103_BASE_IDX                                                                  0
-#define mmMP0_SMN_ACTIVE_FCN_ID                                                                        0x00c0
-#define mmMP0_SMN_ACTIVE_FCN_ID_BASE_IDX                                                               0
-#define mmMP0_SMN_IH_CREDIT                                                                            0x00c1
-#define mmMP0_SMN_IH_CREDIT_BASE_IDX                                                                   0
-#define mmMP0_SMN_IH_SW_INT                                                                            0x00c2
-#define mmMP0_SMN_IH_SW_INT_BASE_IDX                                                                   0
-#define mmMP0_SMN_IH_SW_INT_CTRL                                                                       0x00c3
-#define mmMP0_SMN_IH_SW_INT_CTRL_BASE_IDX                                                              0
-
-
-// addressBlock: mp_SmuMp1_SmnDec
-// base address: 0x0
-#define mmMP1_SMN_ACP2MP_RESP                                                                          0x0240
-#define mmMP1_SMN_ACP2MP_RESP_BASE_IDX                                                                 0
-#define mmMP1_SMN_DC2MP_RESP                                                                           0x0241
-#define mmMP1_SMN_DC2MP_RESP_BASE_IDX                                                                  0
-#define mmMP1_SMN_UVD2MP_RESP                                                                          0x0242
-#define mmMP1_SMN_UVD2MP_RESP_BASE_IDX                                                                 0
-#define mmMP1_SMN_VCE2MP_RESP                                                                          0x0243
-#define mmMP1_SMN_VCE2MP_RESP_BASE_IDX                                                                 0
-#define mmMP1_SMN_RLC2MP_RESP                                                                          0x0244
-#define mmMP1_SMN_RLC2MP_RESP_BASE_IDX                                                                 0
-#define mmMP1_SMN_C2PMSG_32                                                                            0x0260
-#define mmMP1_SMN_C2PMSG_32_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_33                                                                            0x0261
-#define mmMP1_SMN_C2PMSG_33_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_34                                                                            0x0262
-#define mmMP1_SMN_C2PMSG_34_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_35                                                                            0x0263
-#define mmMP1_SMN_C2PMSG_35_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_36                                                                            0x0264
-#define mmMP1_SMN_C2PMSG_36_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_37                                                                            0x0265
-#define mmMP1_SMN_C2PMSG_37_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_38                                                                            0x0266
-#define mmMP1_SMN_C2PMSG_38_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_39                                                                            0x0267
-#define mmMP1_SMN_C2PMSG_39_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_40                                                                            0x0268
-#define mmMP1_SMN_C2PMSG_40_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_41                                                                            0x0269
-#define mmMP1_SMN_C2PMSG_41_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_42                                                                            0x026a
-#define mmMP1_SMN_C2PMSG_42_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_43                                                                            0x026b
-#define mmMP1_SMN_C2PMSG_43_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_44                                                                            0x026c
-#define mmMP1_SMN_C2PMSG_44_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_45                                                                            0x026d
-#define mmMP1_SMN_C2PMSG_45_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_46                                                                            0x026e
-#define mmMP1_SMN_C2PMSG_46_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_47                                                                            0x026f
-#define mmMP1_SMN_C2PMSG_47_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_48                                                                            0x0270
-#define mmMP1_SMN_C2PMSG_48_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_49                                                                            0x0271
-#define mmMP1_SMN_C2PMSG_49_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_50                                                                            0x0272
-#define mmMP1_SMN_C2PMSG_50_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_51                                                                            0x0273
-#define mmMP1_SMN_C2PMSG_51_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_52                                                                            0x0274
-#define mmMP1_SMN_C2PMSG_52_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_53                                                                            0x0275
-#define mmMP1_SMN_C2PMSG_53_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_54                                                                            0x0276
-#define mmMP1_SMN_C2PMSG_54_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_55                                                                            0x0277
-#define mmMP1_SMN_C2PMSG_55_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_56                                                                            0x0278
-#define mmMP1_SMN_C2PMSG_56_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_57                                                                            0x0279
-#define mmMP1_SMN_C2PMSG_57_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_58                                                                            0x027a
-#define mmMP1_SMN_C2PMSG_58_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_59                                                                            0x027b
-#define mmMP1_SMN_C2PMSG_59_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_60                                                                            0x027c
-#define mmMP1_SMN_C2PMSG_60_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_61                                                                            0x027d
-#define mmMP1_SMN_C2PMSG_61_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_62                                                                            0x027e
-#define mmMP1_SMN_C2PMSG_62_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_63                                                                            0x027f
-#define mmMP1_SMN_C2PMSG_63_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_64                                                                            0x0280
-#define mmMP1_SMN_C2PMSG_64_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_65                                                                            0x0281
-#define mmMP1_SMN_C2PMSG_65_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_66                                                                            0x0282
-#define mmMP1_SMN_C2PMSG_66_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_67                                                                            0x0283
-#define mmMP1_SMN_C2PMSG_67_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_68                                                                            0x0284
-#define mmMP1_SMN_C2PMSG_68_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_69                                                                            0x0285
-#define mmMP1_SMN_C2PMSG_69_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_70                                                                            0x0286
-#define mmMP1_SMN_C2PMSG_70_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_71                                                                            0x0287
-#define mmMP1_SMN_C2PMSG_71_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_72                                                                            0x0288
-#define mmMP1_SMN_C2PMSG_72_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_73                                                                            0x0289
-#define mmMP1_SMN_C2PMSG_73_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_74                                                                            0x028a
-#define mmMP1_SMN_C2PMSG_74_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_75                                                                            0x028b
-#define mmMP1_SMN_C2PMSG_75_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_76                                                                            0x028c
-#define mmMP1_SMN_C2PMSG_76_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_77                                                                            0x028d
-#define mmMP1_SMN_C2PMSG_77_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_78                                                                            0x028e
-#define mmMP1_SMN_C2PMSG_78_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_79                                                                            0x028f
-#define mmMP1_SMN_C2PMSG_79_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_80                                                                            0x0290
-#define mmMP1_SMN_C2PMSG_80_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_81                                                                            0x0291
-#define mmMP1_SMN_C2PMSG_81_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_82                                                                            0x0292
-#define mmMP1_SMN_C2PMSG_82_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_83                                                                            0x0293
-#define mmMP1_SMN_C2PMSG_83_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_84                                                                            0x0294
-#define mmMP1_SMN_C2PMSG_84_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_85                                                                            0x0295
-#define mmMP1_SMN_C2PMSG_85_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_86                                                                            0x0296
-#define mmMP1_SMN_C2PMSG_86_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_87                                                                            0x0297
-#define mmMP1_SMN_C2PMSG_87_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_88                                                                            0x0298
-#define mmMP1_SMN_C2PMSG_88_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_89                                                                            0x0299
-#define mmMP1_SMN_C2PMSG_89_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_90                                                                            0x029a
-#define mmMP1_SMN_C2PMSG_90_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_91                                                                            0x029b
-#define mmMP1_SMN_C2PMSG_91_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_92                                                                            0x029c
-#define mmMP1_SMN_C2PMSG_92_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_93                                                                            0x029d
-#define mmMP1_SMN_C2PMSG_93_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_94                                                                            0x029e
-#define mmMP1_SMN_C2PMSG_94_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_95                                                                            0x029f
-#define mmMP1_SMN_C2PMSG_95_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_96                                                                            0x02a0
-#define mmMP1_SMN_C2PMSG_96_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_97                                                                            0x02a1
-#define mmMP1_SMN_C2PMSG_97_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_98                                                                            0x02a2
-#define mmMP1_SMN_C2PMSG_98_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_99                                                                            0x02a3
-#define mmMP1_SMN_C2PMSG_99_BASE_IDX                                                                   0
-#define mmMP1_SMN_C2PMSG_100                                                                           0x02a4
-#define mmMP1_SMN_C2PMSG_100_BASE_IDX                                                                  0
-#define mmMP1_SMN_C2PMSG_101                                                                           0x02a5
-#define mmMP1_SMN_C2PMSG_101_BASE_IDX                                                                  0
-#define mmMP1_SMN_C2PMSG_102                                                                           0x02a6
-#define mmMP1_SMN_C2PMSG_102_BASE_IDX                                                                  0
-#define mmMP1_SMN_C2PMSG_103                                                                           0x02a7
-#define mmMP1_SMN_C2PMSG_103_BASE_IDX                                                                  0
-#define mmMP1_SMN_ACTIVE_FCN_ID                                                                        0x02c0
-#define mmMP1_SMN_ACTIVE_FCN_ID_BASE_IDX                                                               0
-#define mmMP1_SMN_IH_CREDIT                                                                            0x02c1
-#define mmMP1_SMN_IH_CREDIT_BASE_IDX                                                                   0
-#define mmMP1_SMN_IH_SW_INT                                                                            0x02c2
-#define mmMP1_SMN_IH_SW_INT_BASE_IDX                                                                   0
-#define mmMP1_SMN_IH_SW_INT_CTRL                                                                       0x02c3
-#define mmMP1_SMN_IH_SW_INT_CTRL_BASE_IDX                                                              0
-#define mmMP1_SMN_FPS_CNT                                                                              0x02c4
-#define mmMP1_SMN_FPS_CNT_BASE_IDX                                                                     0
-#define mmMP1_SMN_EXT_SCRATCH0                                                                         0x03c0
-#define mmMP1_SMN_EXT_SCRATCH0_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH1                                                                         0x03c1
-#define mmMP1_SMN_EXT_SCRATCH1_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH2                                                                         0x03c2
-#define mmMP1_SMN_EXT_SCRATCH2_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH3                                                                         0x03c3
-#define mmMP1_SMN_EXT_SCRATCH3_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH4                                                                         0x03c4
-#define mmMP1_SMN_EXT_SCRATCH4_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH5                                                                         0x03c5
-#define mmMP1_SMN_EXT_SCRATCH5_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH6                                                                         0x03c6
-#define mmMP1_SMN_EXT_SCRATCH6_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH7                                                                         0x03c7
-#define mmMP1_SMN_EXT_SCRATCH7_BASE_IDX                                                                0
-#define mmMP1_SMN_EXT_SCRATCH8                                                                         0x03c8
-#define mmMP1_SMN_EXT_SCRATCH8_BASE_IDX                                                                0
-
-
-// addressBlock: mp_SmuMp1Pub_CruDec
-// base address: 0x0
-#define mmMP1_SMN_PUB_CTRL                                                                             0x02c5
-#define mmMP1_SMN_PUB_CTRL_BASE_IDX                                                                    0
-
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_sh_mask.h
deleted file mode 100644
index ae7b51870322..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/MP/mp_9_0_sh_mask.h
+++ /dev/null
@@ -1,1463 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _mp_9_0_SH_MASK_HEADER
-#define _mp_9_0_SH_MASK_HEADER
-
-
-// addressBlock: mp_SmuMp0_SmnDec
-//MP0_SMN_C2PMSG_32
-#define MP0_SMN_C2PMSG_32__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_32__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_33
-#define MP0_SMN_C2PMSG_33__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_33__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_34
-#define MP0_SMN_C2PMSG_34__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_34__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_35
-#define MP0_SMN_C2PMSG_35__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_35__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_36
-#define MP0_SMN_C2PMSG_36__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_36__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_37
-#define MP0_SMN_C2PMSG_37__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_37__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_38
-#define MP0_SMN_C2PMSG_38__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_38__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_39
-#define MP0_SMN_C2PMSG_39__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_39__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_40
-#define MP0_SMN_C2PMSG_40__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_40__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_41
-#define MP0_SMN_C2PMSG_41__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_41__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_42
-#define MP0_SMN_C2PMSG_42__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_42__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_43
-#define MP0_SMN_C2PMSG_43__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_43__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_44
-#define MP0_SMN_C2PMSG_44__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_44__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_45
-#define MP0_SMN_C2PMSG_45__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_45__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_46
-#define MP0_SMN_C2PMSG_46__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_46__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_47
-#define MP0_SMN_C2PMSG_47__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_47__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_48
-#define MP0_SMN_C2PMSG_48__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_48__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_49
-#define MP0_SMN_C2PMSG_49__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_49__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_50
-#define MP0_SMN_C2PMSG_50__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_50__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_51
-#define MP0_SMN_C2PMSG_51__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_51__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_52
-#define MP0_SMN_C2PMSG_52__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_52__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_53
-#define MP0_SMN_C2PMSG_53__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_53__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_54
-#define MP0_SMN_C2PMSG_54__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_54__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_55
-#define MP0_SMN_C2PMSG_55__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_55__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_56
-#define MP0_SMN_C2PMSG_56__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_56__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_57
-#define MP0_SMN_C2PMSG_57__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_57__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_58
-#define MP0_SMN_C2PMSG_58__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_58__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_59
-#define MP0_SMN_C2PMSG_59__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_59__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_60
-#define MP0_SMN_C2PMSG_60__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_60__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_61
-#define MP0_SMN_C2PMSG_61__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_61__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_62
-#define MP0_SMN_C2PMSG_62__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_62__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_63
-#define MP0_SMN_C2PMSG_63__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_63__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_64
-#define MP0_SMN_C2PMSG_64__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_64__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_65
-#define MP0_SMN_C2PMSG_65__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_65__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_66
-#define MP0_SMN_C2PMSG_66__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_66__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_67
-#define MP0_SMN_C2PMSG_67__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_67__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_68
-#define MP0_SMN_C2PMSG_68__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_68__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_69
-#define MP0_SMN_C2PMSG_69__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_69__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_70
-#define MP0_SMN_C2PMSG_70__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_70__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_71
-#define MP0_SMN_C2PMSG_71__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_71__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_72
-#define MP0_SMN_C2PMSG_72__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_72__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_73
-#define MP0_SMN_C2PMSG_73__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_73__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_74
-#define MP0_SMN_C2PMSG_74__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_74__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_75
-#define MP0_SMN_C2PMSG_75__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_75__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_76
-#define MP0_SMN_C2PMSG_76__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_76__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_77
-#define MP0_SMN_C2PMSG_77__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_77__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_78
-#define MP0_SMN_C2PMSG_78__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_78__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_79
-#define MP0_SMN_C2PMSG_79__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_79__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_80
-#define MP0_SMN_C2PMSG_80__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_80__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_81
-#define MP0_SMN_C2PMSG_81__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_81__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_82
-#define MP0_SMN_C2PMSG_82__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_82__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_83
-#define MP0_SMN_C2PMSG_83__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_83__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_84
-#define MP0_SMN_C2PMSG_84__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_84__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_85
-#define MP0_SMN_C2PMSG_85__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_85__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_86
-#define MP0_SMN_C2PMSG_86__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_86__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_87
-#define MP0_SMN_C2PMSG_87__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_87__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_88
-#define MP0_SMN_C2PMSG_88__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_88__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_89
-#define MP0_SMN_C2PMSG_89__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_89__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_90
-#define MP0_SMN_C2PMSG_90__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_90__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_91
-#define MP0_SMN_C2PMSG_91__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_91__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_92
-#define MP0_SMN_C2PMSG_92__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_92__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_93
-#define MP0_SMN_C2PMSG_93__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_93__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_94
-#define MP0_SMN_C2PMSG_94__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_94__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_95
-#define MP0_SMN_C2PMSG_95__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_95__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_96
-#define MP0_SMN_C2PMSG_96__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_96__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_97
-#define MP0_SMN_C2PMSG_97__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_97__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_98
-#define MP0_SMN_C2PMSG_98__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_98__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_99
-#define MP0_SMN_C2PMSG_99__CONTENT__SHIFT                                                                     0x0
-#define MP0_SMN_C2PMSG_99__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP0_SMN_C2PMSG_100
-#define MP0_SMN_C2PMSG_100__CONTENT__SHIFT                                                                    0x0
-#define MP0_SMN_C2PMSG_100__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP0_SMN_C2PMSG_101
-#define MP0_SMN_C2PMSG_101__CONTENT__SHIFT                                                                    0x0
-#define MP0_SMN_C2PMSG_101__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP0_SMN_C2PMSG_102
-#define MP0_SMN_C2PMSG_102__CONTENT__SHIFT                                                                    0x0
-#define MP0_SMN_C2PMSG_102__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP0_SMN_C2PMSG_103
-#define MP0_SMN_C2PMSG_103__CONTENT__SHIFT                                                                    0x0
-#define MP0_SMN_C2PMSG_103__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP0_SMN_ACTIVE_FCN_ID
-#define MP0_SMN_ACTIVE_FCN_ID__VFID__SHIFT                                                                    0x0
-#define MP0_SMN_ACTIVE_FCN_ID__VF__SHIFT                                                                      0x1f
-#define MP0_SMN_ACTIVE_FCN_ID__VFID_MASK                                                                      0x0000000FL
-#define MP0_SMN_ACTIVE_FCN_ID__VF_MASK                                                                        0x80000000L
-//MP0_SMN_IH_CREDIT
-#define MP0_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                0x0
-#define MP0_SMN_IH_CREDIT__CLIENT_ID__SHIFT                                                                   0x10
-#define MP0_SMN_IH_CREDIT__CREDIT_VALUE_MASK                                                                  0x00000003L
-#define MP0_SMN_IH_CREDIT__CLIENT_ID_MASK                                                                     0x00FF0000L
-//MP0_SMN_IH_SW_INT
-#define MP0_SMN_IH_SW_INT__VALID__SHIFT                                                                       0x0
-#define MP0_SMN_IH_SW_INT__ID__SHIFT                                                                          0x1
-#define MP0_SMN_IH_SW_INT__VALID_MASK                                                                         0x00000001L
-#define MP0_SMN_IH_SW_INT__ID_MASK                                                                            0x000001FEL
-//MP0_SMN_IH_SW_INT_CTRL
-#define MP0_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK__SHIFT                                                           0x0
-#define MP0_SMN_IH_SW_INT_CTRL__SW_INT_ACK__SHIFT                                                             0x8
-#define MP0_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK_MASK                                                             0x00000001L
-#define MP0_SMN_IH_SW_INT_CTRL__SW_INT_ACK_MASK                                                               0x00000100L
-
-
-// addressBlock: mp_SmuMp1_SmnDec
-//MP1_SMN_ACP2MP_RESP
-#define MP1_SMN_ACP2MP_RESP__CONTENT__SHIFT                                                                   0x0
-#define MP1_SMN_ACP2MP_RESP__CONTENT_MASK                                                                     0xFFFFFFFFL
-//MP1_SMN_DC2MP_RESP
-#define MP1_SMN_DC2MP_RESP__CONTENT__SHIFT                                                                    0x0
-#define MP1_SMN_DC2MP_RESP__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP1_SMN_UVD2MP_RESP
-#define MP1_SMN_UVD2MP_RESP__CONTENT__SHIFT                                                                   0x0
-#define MP1_SMN_UVD2MP_RESP__CONTENT_MASK                                                                     0xFFFFFFFFL
-//MP1_SMN_VCE2MP_RESP
-#define MP1_SMN_VCE2MP_RESP__CONTENT__SHIFT                                                                   0x0
-#define MP1_SMN_VCE2MP_RESP__CONTENT_MASK                                                                     0xFFFFFFFFL
-//MP1_SMN_RLC2MP_RESP
-#define MP1_SMN_RLC2MP_RESP__CONTENT__SHIFT                                                                   0x0
-#define MP1_SMN_RLC2MP_RESP__CONTENT_MASK                                                                     0xFFFFFFFFL
-//MP1_SMN_C2PMSG_32
-#define MP1_SMN_C2PMSG_32__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_32__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_33
-#define MP1_SMN_C2PMSG_33__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_33__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_34
-#define MP1_SMN_C2PMSG_34__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_34__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_35
-#define MP1_SMN_C2PMSG_35__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_35__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_36
-#define MP1_SMN_C2PMSG_36__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_36__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_37
-#define MP1_SMN_C2PMSG_37__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_37__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_38
-#define MP1_SMN_C2PMSG_38__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_38__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_39
-#define MP1_SMN_C2PMSG_39__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_39__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_40
-#define MP1_SMN_C2PMSG_40__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_40__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_41
-#define MP1_SMN_C2PMSG_41__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_41__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_42
-#define MP1_SMN_C2PMSG_42__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_42__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_43
-#define MP1_SMN_C2PMSG_43__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_43__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_44
-#define MP1_SMN_C2PMSG_44__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_44__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_45
-#define MP1_SMN_C2PMSG_45__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_45__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_46
-#define MP1_SMN_C2PMSG_46__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_46__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_47
-#define MP1_SMN_C2PMSG_47__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_47__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_48
-#define MP1_SMN_C2PMSG_48__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_48__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_49
-#define MP1_SMN_C2PMSG_49__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_49__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_50
-#define MP1_SMN_C2PMSG_50__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_50__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_51
-#define MP1_SMN_C2PMSG_51__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_51__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_52
-#define MP1_SMN_C2PMSG_52__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_52__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_53
-#define MP1_SMN_C2PMSG_53__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_53__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_54
-#define MP1_SMN_C2PMSG_54__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_54__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_55
-#define MP1_SMN_C2PMSG_55__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_55__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_56
-#define MP1_SMN_C2PMSG_56__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_56__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_57
-#define MP1_SMN_C2PMSG_57__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_57__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_58
-#define MP1_SMN_C2PMSG_58__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_58__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_59
-#define MP1_SMN_C2PMSG_59__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_59__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_60
-#define MP1_SMN_C2PMSG_60__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_60__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_61
-#define MP1_SMN_C2PMSG_61__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_61__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_62
-#define MP1_SMN_C2PMSG_62__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_62__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_63
-#define MP1_SMN_C2PMSG_63__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_63__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_64
-#define MP1_SMN_C2PMSG_64__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_64__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_65
-#define MP1_SMN_C2PMSG_65__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_65__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_66
-#define MP1_SMN_C2PMSG_66__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_66__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_67
-#define MP1_SMN_C2PMSG_67__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_67__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_68
-#define MP1_SMN_C2PMSG_68__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_68__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_69
-#define MP1_SMN_C2PMSG_69__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_69__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_70
-#define MP1_SMN_C2PMSG_70__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_70__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_71
-#define MP1_SMN_C2PMSG_71__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_71__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_72
-#define MP1_SMN_C2PMSG_72__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_72__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_73
-#define MP1_SMN_C2PMSG_73__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_73__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_74
-#define MP1_SMN_C2PMSG_74__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_74__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_75
-#define MP1_SMN_C2PMSG_75__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_75__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_76
-#define MP1_SMN_C2PMSG_76__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_76__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_77
-#define MP1_SMN_C2PMSG_77__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_77__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_78
-#define MP1_SMN_C2PMSG_78__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_78__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_79
-#define MP1_SMN_C2PMSG_79__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_79__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_80
-#define MP1_SMN_C2PMSG_80__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_80__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_81
-#define MP1_SMN_C2PMSG_81__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_81__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_82
-#define MP1_SMN_C2PMSG_82__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_82__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_83
-#define MP1_SMN_C2PMSG_83__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_83__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_84
-#define MP1_SMN_C2PMSG_84__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_84__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_85
-#define MP1_SMN_C2PMSG_85__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_85__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_86
-#define MP1_SMN_C2PMSG_86__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_86__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_87
-#define MP1_SMN_C2PMSG_87__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_87__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_88
-#define MP1_SMN_C2PMSG_88__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_88__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_89
-#define MP1_SMN_C2PMSG_89__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_89__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_90
-#define MP1_SMN_C2PMSG_90__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_90__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_91
-#define MP1_SMN_C2PMSG_91__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_91__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_92
-#define MP1_SMN_C2PMSG_92__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_92__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_93
-#define MP1_SMN_C2PMSG_93__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_93__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_94
-#define MP1_SMN_C2PMSG_94__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_94__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_95
-#define MP1_SMN_C2PMSG_95__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_95__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_96
-#define MP1_SMN_C2PMSG_96__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_96__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_97
-#define MP1_SMN_C2PMSG_97__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_97__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_98
-#define MP1_SMN_C2PMSG_98__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_98__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_99
-#define MP1_SMN_C2PMSG_99__CONTENT__SHIFT                                                                     0x0
-#define MP1_SMN_C2PMSG_99__CONTENT_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_C2PMSG_100
-#define MP1_SMN_C2PMSG_100__CONTENT__SHIFT                                                                    0x0
-#define MP1_SMN_C2PMSG_100__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP1_SMN_C2PMSG_101
-#define MP1_SMN_C2PMSG_101__CONTENT__SHIFT                                                                    0x0
-#define MP1_SMN_C2PMSG_101__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP1_SMN_C2PMSG_102
-#define MP1_SMN_C2PMSG_102__CONTENT__SHIFT                                                                    0x0
-#define MP1_SMN_C2PMSG_102__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP1_SMN_C2PMSG_103
-#define MP1_SMN_C2PMSG_103__CONTENT__SHIFT                                                                    0x0
-#define MP1_SMN_C2PMSG_103__CONTENT_MASK                                                                      0xFFFFFFFFL
-//MP1_SMN_ACTIVE_FCN_ID
-#define MP1_SMN_ACTIVE_FCN_ID__VFID__SHIFT                                                                    0x0
-#define MP1_SMN_ACTIVE_FCN_ID__VF__SHIFT                                                                      0x1f
-#define MP1_SMN_ACTIVE_FCN_ID__VFID_MASK                                                                      0x0000000FL
-#define MP1_SMN_ACTIVE_FCN_ID__VF_MASK                                                                        0x80000000L
-//MP1_SMN_IH_CREDIT
-#define MP1_SMN_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                0x0
-#define MP1_SMN_IH_CREDIT__CLIENT_ID__SHIFT                                                                   0x10
-#define MP1_SMN_IH_CREDIT__CREDIT_VALUE_MASK                                                                  0x00000003L
-#define MP1_SMN_IH_CREDIT__CLIENT_ID_MASK                                                                     0x00FF0000L
-//MP1_SMN_IH_SW_INT
-#define MP1_SMN_IH_SW_INT__VALID__SHIFT                                                                       0x0
-#define MP1_SMN_IH_SW_INT__ID__SHIFT                                                                          0x1
-#define MP1_SMN_IH_SW_INT__VALID_MASK                                                                         0x00000001L
-#define MP1_SMN_IH_SW_INT__ID_MASK                                                                            0x000001FEL
-//MP1_SMN_IH_SW_INT_CTRL
-#define MP1_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK__SHIFT                                                           0x0
-#define MP1_SMN_IH_SW_INT_CTRL__SW_INT_ACK__SHIFT                                                             0x8
-#define MP1_SMN_IH_SW_INT_CTRL__SW_TRIG_MASK_MASK                                                             0x00000001L
-#define MP1_SMN_IH_SW_INT_CTRL__SW_INT_ACK_MASK                                                               0x00000100L
-//MP1_SMN_FPS_CNT
-#define MP1_SMN_FPS_CNT__COUNT__SHIFT                                                                         0x0
-#define MP1_SMN_FPS_CNT__COUNT_MASK                                                                           0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH0
-#define MP1_SMN_EXT_SCRATCH0__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH0__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH1
-#define MP1_SMN_EXT_SCRATCH1__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH1__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH2
-#define MP1_SMN_EXT_SCRATCH2__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH2__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH3
-#define MP1_SMN_EXT_SCRATCH3__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH3__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH4
-#define MP1_SMN_EXT_SCRATCH4__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH4__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH5
-#define MP1_SMN_EXT_SCRATCH5__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH5__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH6
-#define MP1_SMN_EXT_SCRATCH6__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH6__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH7
-#define MP1_SMN_EXT_SCRATCH7__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH7__DATA_MASK                                                                       0xFFFFFFFFL
-//MP1_SMN_EXT_SCRATCH8
-#define MP1_SMN_EXT_SCRATCH8__DATA__SHIFT                                                                     0x0
-#define MP1_SMN_EXT_SCRATCH8__DATA_MASK                                                                       0xFFFFFFFFL
-
-
-
-
-// addressBlock: mp_SmuMp0Pub_CruDec
-//MP0_SOC_INFO
-#define MP0_SOC_INFO__SOC_DIE_ID__SHIFT                                                                       0x0
-#define MP0_SOC_INFO__SOC_PKG_TYPE__SHIFT                                                                     0x2
-#define MP0_SOC_INFO__SOC_DIE_ID_MASK                                                                         0x00000003L
-#define MP0_SOC_INFO__SOC_PKG_TYPE_MASK                                                                       0x0000001CL
-//MP0_PUB_SCRATCH0
-#define MP0_PUB_SCRATCH0__DATA__SHIFT                                                                         0x0
-#define MP0_PUB_SCRATCH0__DATA_MASK                                                                           0xFFFFFFFFL
-//MP0_PUB_SCRATCH1
-#define MP0_PUB_SCRATCH1__DATA__SHIFT                                                                         0x0
-#define MP0_PUB_SCRATCH1__DATA_MASK                                                                           0xFFFFFFFFL
-//MP0_PUB_SCRATCH2
-#define MP0_PUB_SCRATCH2__DATA__SHIFT                                                                         0x0
-#define MP0_PUB_SCRATCH2__DATA_MASK                                                                           0xFFFFFFFFL
-//MP0_PUB_SCRATCH3
-#define MP0_PUB_SCRATCH3__DATA__SHIFT                                                                         0x0
-#define MP0_PUB_SCRATCH3__DATA_MASK                                                                           0xFFFFFFFFL
-//MP0_FW_INTF
-#define MP0_FW_INTF__SS_SECURE__SHIFT                                                                         0x13
-#define MP0_FW_INTF__SS_SECURE_MASK                                                                           0x00080000L
-//MP0_C2PMSG_0
-#define MP0_C2PMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_1
-#define MP0_C2PMSG_1__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_1__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_2
-#define MP0_C2PMSG_2__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_2__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_3
-#define MP0_C2PMSG_3__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_3__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_4
-#define MP0_C2PMSG_4__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_4__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_5
-#define MP0_C2PMSG_5__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_5__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_6
-#define MP0_C2PMSG_6__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_6__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_7
-#define MP0_C2PMSG_7__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_7__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_8
-#define MP0_C2PMSG_8__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_8__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_9
-#define MP0_C2PMSG_9__CONTENT__SHIFT                                                                          0x0
-#define MP0_C2PMSG_9__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_10
-#define MP0_C2PMSG_10__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_10__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_11
-#define MP0_C2PMSG_11__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_11__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_12
-#define MP0_C2PMSG_12__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_12__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_13
-#define MP0_C2PMSG_13__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_13__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_14
-#define MP0_C2PMSG_14__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_14__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_15
-#define MP0_C2PMSG_15__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_15__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_16
-#define MP0_C2PMSG_16__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_16__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_17
-#define MP0_C2PMSG_17__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_17__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_18
-#define MP0_C2PMSG_18__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_18__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_19
-#define MP0_C2PMSG_19__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_19__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_20
-#define MP0_C2PMSG_20__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_20__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_21
-#define MP0_C2PMSG_21__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_21__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_22
-#define MP0_C2PMSG_22__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_22__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_23
-#define MP0_C2PMSG_23__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_23__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_24
-#define MP0_C2PMSG_24__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_24__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_25
-#define MP0_C2PMSG_25__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_25__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_26
-#define MP0_C2PMSG_26__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_26__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_27
-#define MP0_C2PMSG_27__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_27__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_28
-#define MP0_C2PMSG_28__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_28__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_29
-#define MP0_C2PMSG_29__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_29__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_30
-#define MP0_C2PMSG_30__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_30__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_31
-#define MP0_C2PMSG_31__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_31__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_P2CMSG_0
-#define MP0_P2CMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2CMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2CMSG_1
-#define MP0_P2CMSG_1__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2CMSG_1__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2CMSG_2
-#define MP0_P2CMSG_2__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2CMSG_2__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2CMSG_3
-#define MP0_P2CMSG_3__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2CMSG_3__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2CMSG_INTEN
-#define MP0_P2CMSG_INTEN__INTEN__SHIFT                                                                        0x0
-#define MP0_P2CMSG_INTEN__INTEN_MASK                                                                          0x0000000FL
-//MP0_P2CMSG_INTSTS
-#define MP0_P2CMSG_INTSTS__INTSTS0__SHIFT                                                                     0x0
-#define MP0_P2CMSG_INTSTS__INTSTS1__SHIFT                                                                     0x1
-#define MP0_P2CMSG_INTSTS__INTSTS2__SHIFT                                                                     0x2
-#define MP0_P2CMSG_INTSTS__INTSTS3__SHIFT                                                                     0x3
-#define MP0_P2CMSG_INTSTS__INTSTS0_MASK                                                                       0x00000001L
-#define MP0_P2CMSG_INTSTS__INTSTS1_MASK                                                                       0x00000002L
-#define MP0_P2CMSG_INTSTS__INTSTS2_MASK                                                                       0x00000004L
-#define MP0_P2CMSG_INTSTS__INTSTS3_MASK                                                                       0x00000008L
-//MP0_C2PMSG_ATTR_0
-#define MP0_C2PMSG_ATTR_0__MSG_ATTR__SHIFT                                                                    0x0
-#define MP0_C2PMSG_ATTR_0__MSG_ATTR_MASK                                                                      0xFFFFFFFFL
-//MP0_C2PMSG_ATTR_1
-#define MP0_C2PMSG_ATTR_1__MSG_ATTR__SHIFT                                                                    0x0
-#define MP0_C2PMSG_ATTR_1__MSG_ATTR_MASK                                                                      0xFFFFFFFFL
-//MP0_C2PMSG_ATTR_2
-#define MP0_C2PMSG_ATTR_2__MSG_ATTR__SHIFT                                                                    0x0
-#define MP0_C2PMSG_ATTR_2__MSG_ATTR_MASK                                                                      0xFFFFFFFFL
-//MP0_C2PMSG_ATTR_3
-#define MP0_C2PMSG_ATTR_3__MSG_ATTR__SHIFT                                                                    0x0
-#define MP0_C2PMSG_ATTR_3__MSG_ATTR_MASK                                                                      0xFFFFFFFFL
-//MP0_C2PMSG_ATTR_4
-#define MP0_C2PMSG_ATTR_4__MSG_ATTR__SHIFT                                                                    0x0
-#define MP0_C2PMSG_ATTR_4__MSG_ATTR_MASK                                                                      0xFFFFFFFFL
-//MP0_C2PMSG_ATTR_5
-#define MP0_C2PMSG_ATTR_5__MSG_ATTR__SHIFT                                                                    0x0
-#define MP0_C2PMSG_ATTR_5__MSG_ATTR_MASK                                                                      0xFFFFFFFFL
-//MP0_C2PMSG_ATTR_6
-#define MP0_C2PMSG_ATTR_6__MSG_ATTR__SHIFT                                                                    0x0
-#define MP0_C2PMSG_ATTR_6__MSG_ATTR_MASK                                                                      0x0000FFFFL
-//MP0_P2CMSG_ATTR
-#define MP0_P2CMSG_ATTR__MSG_ATTR__SHIFT                                                                      0x0
-#define MP0_P2CMSG_ATTR__MSG_ATTR_MASK                                                                        0x000000FFL
-//MP0_P2SMSG_0
-#define MP0_P2SMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2SMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2SMSG_1
-#define MP0_P2SMSG_1__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2SMSG_1__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2SMSG_2
-#define MP0_P2SMSG_2__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2SMSG_2__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2SMSG_3
-#define MP0_P2SMSG_3__CONTENT__SHIFT                                                                          0x0
-#define MP0_P2SMSG_3__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_P2SMSG_ATTR
-#define MP0_P2SMSG_ATTR__MSG_ATTR__SHIFT                                                                      0x0
-#define MP0_P2SMSG_ATTR__MSG_ATTR_MASK                                                                        0x000000FFL
-//MP0_S2PMSG_ATTR
-#define MP0_S2PMSG_ATTR__MSG_ATTR__SHIFT                                                                      0x0
-#define MP0_S2PMSG_ATTR__MSG_ATTR_MASK                                                                        0x00000003L
-//MP0_P2SMSG_INTSTS
-#define MP0_P2SMSG_INTSTS__INTSTS0__SHIFT                                                                     0x0
-#define MP0_P2SMSG_INTSTS__INTSTS1__SHIFT                                                                     0x1
-#define MP0_P2SMSG_INTSTS__INTSTS2__SHIFT                                                                     0x2
-#define MP0_P2SMSG_INTSTS__INTSTS3__SHIFT                                                                     0x3
-#define MP0_P2SMSG_INTSTS__INTSTS0_MASK                                                                       0x00000001L
-#define MP0_P2SMSG_INTSTS__INTSTS1_MASK                                                                       0x00000002L
-#define MP0_P2SMSG_INTSTS__INTSTS2_MASK                                                                       0x00000004L
-#define MP0_P2SMSG_INTSTS__INTSTS3_MASK                                                                       0x00000008L
-//MP0_S2PMSG_0
-#define MP0_S2PMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP0_S2PMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP0_C2PMSG_32
-#define MP0_C2PMSG_32__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_32__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_33
-#define MP0_C2PMSG_33__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_33__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_34
-#define MP0_C2PMSG_34__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_34__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_35
-#define MP0_C2PMSG_35__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_35__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_36
-#define MP0_C2PMSG_36__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_36__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_37
-#define MP0_C2PMSG_37__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_37__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_38
-#define MP0_C2PMSG_38__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_38__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_39
-#define MP0_C2PMSG_39__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_39__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_40
-#define MP0_C2PMSG_40__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_40__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_41
-#define MP0_C2PMSG_41__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_41__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_42
-#define MP0_C2PMSG_42__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_42__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_43
-#define MP0_C2PMSG_43__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_43__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_44
-#define MP0_C2PMSG_44__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_44__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_45
-#define MP0_C2PMSG_45__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_45__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_46
-#define MP0_C2PMSG_46__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_46__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_47
-#define MP0_C2PMSG_47__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_47__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_48
-#define MP0_C2PMSG_48__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_48__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_49
-#define MP0_C2PMSG_49__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_49__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_50
-#define MP0_C2PMSG_50__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_50__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_51
-#define MP0_C2PMSG_51__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_51__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_52
-#define MP0_C2PMSG_52__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_52__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_53
-#define MP0_C2PMSG_53__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_53__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_54
-#define MP0_C2PMSG_54__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_54__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_55
-#define MP0_C2PMSG_55__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_55__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_56
-#define MP0_C2PMSG_56__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_56__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_57
-#define MP0_C2PMSG_57__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_57__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_58
-#define MP0_C2PMSG_58__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_58__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_59
-#define MP0_C2PMSG_59__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_59__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_60
-#define MP0_C2PMSG_60__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_60__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_61
-#define MP0_C2PMSG_61__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_61__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_62
-#define MP0_C2PMSG_62__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_62__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_63
-#define MP0_C2PMSG_63__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_63__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_64
-#define MP0_C2PMSG_64__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_64__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_65
-#define MP0_C2PMSG_65__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_65__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_66
-#define MP0_C2PMSG_66__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_66__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_67
-#define MP0_C2PMSG_67__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_67__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_68
-#define MP0_C2PMSG_68__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_68__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_69
-#define MP0_C2PMSG_69__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_69__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_70
-#define MP0_C2PMSG_70__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_70__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_71
-#define MP0_C2PMSG_71__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_71__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_72
-#define MP0_C2PMSG_72__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_72__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_73
-#define MP0_C2PMSG_73__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_73__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_74
-#define MP0_C2PMSG_74__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_74__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_75
-#define MP0_C2PMSG_75__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_75__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_76
-#define MP0_C2PMSG_76__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_76__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_77
-#define MP0_C2PMSG_77__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_77__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_78
-#define MP0_C2PMSG_78__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_78__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_79
-#define MP0_C2PMSG_79__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_79__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_80
-#define MP0_C2PMSG_80__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_80__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_81
-#define MP0_C2PMSG_81__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_81__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_82
-#define MP0_C2PMSG_82__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_82__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_83
-#define MP0_C2PMSG_83__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_83__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_84
-#define MP0_C2PMSG_84__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_84__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_85
-#define MP0_C2PMSG_85__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_85__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_86
-#define MP0_C2PMSG_86__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_86__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_87
-#define MP0_C2PMSG_87__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_87__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_88
-#define MP0_C2PMSG_88__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_88__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_89
-#define MP0_C2PMSG_89__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_89__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_90
-#define MP0_C2PMSG_90__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_90__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_91
-#define MP0_C2PMSG_91__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_91__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_92
-#define MP0_C2PMSG_92__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_92__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_93
-#define MP0_C2PMSG_93__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_93__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_94
-#define MP0_C2PMSG_94__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_94__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_95
-#define MP0_C2PMSG_95__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_95__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_96
-#define MP0_C2PMSG_96__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_96__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_97
-#define MP0_C2PMSG_97__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_97__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_98
-#define MP0_C2PMSG_98__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_98__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_99
-#define MP0_C2PMSG_99__CONTENT__SHIFT                                                                         0x0
-#define MP0_C2PMSG_99__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP0_C2PMSG_100
-#define MP0_C2PMSG_100__CONTENT__SHIFT                                                                        0x0
-#define MP0_C2PMSG_100__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP0_C2PMSG_101
-#define MP0_C2PMSG_101__CONTENT__SHIFT                                                                        0x0
-#define MP0_C2PMSG_101__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP0_C2PMSG_102
-#define MP0_C2PMSG_102__CONTENT__SHIFT                                                                        0x0
-#define MP0_C2PMSG_102__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP0_C2PMSG_103
-#define MP0_C2PMSG_103__CONTENT__SHIFT                                                                        0x0
-#define MP0_C2PMSG_103__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP0_ACTIVE_FCN_ID
-#define MP0_ACTIVE_FCN_ID__VFID__SHIFT                                                                        0x0
-#define MP0_ACTIVE_FCN_ID__VF__SHIFT                                                                          0x1f
-#define MP0_ACTIVE_FCN_ID__VFID_MASK                                                                          0x0000000FL
-#define MP0_ACTIVE_FCN_ID__VF_MASK                                                                            0x80000000L
-//MP0_IH_CREDIT
-#define MP0_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                    0x0
-#define MP0_IH_CREDIT__CLIENT_ID__SHIFT                                                                       0x10
-#define MP0_IH_CREDIT__CREDIT_VALUE_MASK                                                                      0x00000003L
-#define MP0_IH_CREDIT__CLIENT_ID_MASK                                                                         0x00FF0000L
-//MP0_IH_SW_INT
-#define MP0_IH_SW_INT__ID__SHIFT                                                                              0x0
-#define MP0_IH_SW_INT__VALID__SHIFT                                                                           0x8
-#define MP0_IH_SW_INT__ID_MASK                                                                                0x000000FFL
-#define MP0_IH_SW_INT__VALID_MASK                                                                             0x00000100L
-//MP0_IH_SW_INT_CTRL
-#define MP0_IH_SW_INT_CTRL__INT_MASK__SHIFT                                                                   0x0
-#define MP0_IH_SW_INT_CTRL__INT_ACK__SHIFT                                                                    0x8
-#define MP0_IH_SW_INT_CTRL__INT_MASK_MASK                                                                     0x00000001L
-#define MP0_IH_SW_INT_CTRL__INT_ACK_MASK                                                                      0x00000100L
-
-
-//CGTT_DRM_CLK_CTRL0
-#define CGTT_DRM_CLK_CTRL0__ON_DELAY__SHIFT                                                                   0x0
-#define CGTT_DRM_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                             0x4
-#define CGTT_DRM_CLK_CTRL0__DIV_ID__SHIFT                                                                     0xc
-#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_0__SHIFT                                                             0x15
-#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_REG__SHIFT                                                           0x16
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT                                                             0x18
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT                                                             0x19
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                             0x1a
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                             0x1b
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                             0x1c
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                             0x1d
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                             0x1e
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                             0x1f
-#define CGTT_DRM_CLK_CTRL0__ON_DELAY_MASK                                                                     0x0000000FL
-#define CGTT_DRM_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
-#define CGTT_DRM_CLK_CTRL0__DIV_ID_MASK                                                                       0x00007000L
-#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_0_MASK                                                               0x00200000L
-#define CGTT_DRM_CLK_CTRL0__RAMP_DIS_CLK_REG_MASK                                                             0x00400000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE7_MASK                                                               0x01000000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE6_MASK                                                               0x02000000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                               0x04000000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                               0x08000000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                               0x10000000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                               0x20000000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                               0x40000000L
-#define CGTT_DRM_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                               0x80000000L
-//DRM_LIGHT_SLEEP_CTRL
-#define DRM_LIGHT_SLEEP_CTRL__MEM_LIGHT_SLEEP_EN__SHIFT                                                       0x0
-#define DRM_LIGHT_SLEEP_CTRL__MEM_LIGHT_SLEEP_EN_MASK                                                         0x00000001L
-
-
-// addressBlock: mp_SmuMp1Pub_CruDec
-//MP1_SMN_PUB_CTRL
-#define MP1_SMN_PUB_CTRL__RESET__SHIFT                                                                        0x0
-#define MP1_SMN_PUB_CTRL__RESET_MASK                                                                          0x00000001L
-//MP1_FIRMWARE_FLAGS
-#define MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT                                                         0x0
-#define MP1_FIRMWARE_FLAGS__RESERVED__SHIFT                                                                   0x1
-#define MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK                                                           0x00000001L
-#define MP1_FIRMWARE_FLAGS__RESERVED_MASK                                                                     0xFFFFFFFEL
-//MP1_PUB_SCRATCH0
-#define MP1_PUB_SCRATCH0__DATA__SHIFT                                                                         0x0
-#define MP1_PUB_SCRATCH0__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_PUB_SCRATCH1
-#define MP1_PUB_SCRATCH1__DATA__SHIFT                                                                         0x0
-#define MP1_PUB_SCRATCH1__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_PUB_SCRATCH2
-#define MP1_PUB_SCRATCH2__DATA__SHIFT                                                                         0x0
-#define MP1_PUB_SCRATCH2__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_PUB_SCRATCH3
-#define MP1_PUB_SCRATCH3__DATA__SHIFT                                                                         0x0
-#define MP1_PUB_SCRATCH3__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_0
-#define MP1_C2PMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_1
-#define MP1_C2PMSG_1__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_1__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_2
-#define MP1_C2PMSG_2__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_2__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_3
-#define MP1_C2PMSG_3__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_3__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_4
-#define MP1_C2PMSG_4__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_4__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_5
-#define MP1_C2PMSG_5__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_5__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_6
-#define MP1_C2PMSG_6__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_6__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_7
-#define MP1_C2PMSG_7__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_7__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_8
-#define MP1_C2PMSG_8__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_8__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_9
-#define MP1_C2PMSG_9__CONTENT__SHIFT                                                                          0x0
-#define MP1_C2PMSG_9__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_C2PMSG_10
-#define MP1_C2PMSG_10__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_10__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_11
-#define MP1_C2PMSG_11__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_11__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_12
-#define MP1_C2PMSG_12__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_12__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_13
-#define MP1_C2PMSG_13__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_13__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_14
-#define MP1_C2PMSG_14__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_14__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_15
-#define MP1_C2PMSG_15__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_15__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_16
-#define MP1_C2PMSG_16__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_16__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_17
-#define MP1_C2PMSG_17__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_17__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_18
-#define MP1_C2PMSG_18__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_18__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_19
-#define MP1_C2PMSG_19__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_19__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_20
-#define MP1_C2PMSG_20__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_20__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_21
-#define MP1_C2PMSG_21__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_21__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_22
-#define MP1_C2PMSG_22__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_22__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_23
-#define MP1_C2PMSG_23__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_23__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_24
-#define MP1_C2PMSG_24__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_24__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_25
-#define MP1_C2PMSG_25__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_25__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_26
-#define MP1_C2PMSG_26__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_26__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_27
-#define MP1_C2PMSG_27__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_27__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_28
-#define MP1_C2PMSG_28__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_28__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_29
-#define MP1_C2PMSG_29__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_29__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_30
-#define MP1_C2PMSG_30__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_30__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_31
-#define MP1_C2PMSG_31__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_31__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_P2CMSG_0
-#define MP1_P2CMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2CMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2CMSG_1
-#define MP1_P2CMSG_1__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2CMSG_1__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2CMSG_2
-#define MP1_P2CMSG_2__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2CMSG_2__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2CMSG_3
-#define MP1_P2CMSG_3__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2CMSG_3__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2CMSG_INTEN
-#define MP1_P2CMSG_INTEN__INTEN__SHIFT                                                                        0x0
-#define MP1_P2CMSG_INTEN__INTEN_MASK                                                                          0x0000000FL
-//MP1_P2CMSG_INTSTS
-#define MP1_P2CMSG_INTSTS__INTSTS0__SHIFT                                                                     0x0
-#define MP1_P2CMSG_INTSTS__INTSTS1__SHIFT                                                                     0x1
-#define MP1_P2CMSG_INTSTS__INTSTS2__SHIFT                                                                     0x2
-#define MP1_P2CMSG_INTSTS__INTSTS3__SHIFT                                                                     0x3
-#define MP1_P2CMSG_INTSTS__INTSTS0_MASK                                                                       0x00000001L
-#define MP1_P2CMSG_INTSTS__INTSTS1_MASK                                                                       0x00000002L
-#define MP1_P2CMSG_INTSTS__INTSTS2_MASK                                                                       0x00000004L
-#define MP1_P2CMSG_INTSTS__INTSTS3_MASK                                                                       0x00000008L
-//MP1_P2SMSG_0
-#define MP1_P2SMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2SMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2SMSG_1
-#define MP1_P2SMSG_1__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2SMSG_1__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2SMSG_2
-#define MP1_P2SMSG_2__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2SMSG_2__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2SMSG_3
-#define MP1_P2SMSG_3__CONTENT__SHIFT                                                                          0x0
-#define MP1_P2SMSG_3__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_P2SMSG_INTSTS
-#define MP1_P2SMSG_INTSTS__INTSTS0__SHIFT                                                                     0x0
-#define MP1_P2SMSG_INTSTS__INTSTS1__SHIFT                                                                     0x1
-#define MP1_P2SMSG_INTSTS__INTSTS2__SHIFT                                                                     0x2
-#define MP1_P2SMSG_INTSTS__INTSTS3__SHIFT                                                                     0x3
-#define MP1_P2SMSG_INTSTS__INTSTS0_MASK                                                                       0x00000001L
-#define MP1_P2SMSG_INTSTS__INTSTS1_MASK                                                                       0x00000002L
-#define MP1_P2SMSG_INTSTS__INTSTS2_MASK                                                                       0x00000004L
-#define MP1_P2SMSG_INTSTS__INTSTS3_MASK                                                                       0x00000008L
-//MP1_S2PMSG_0
-#define MP1_S2PMSG_0__CONTENT__SHIFT                                                                          0x0
-#define MP1_S2PMSG_0__CONTENT_MASK                                                                            0xFFFFFFFFL
-//MP1_ACP2MP_RESP
-#define MP1_ACP2MP_RESP__CONTENT__SHIFT                                                                       0x0
-#define MP1_ACP2MP_RESP__CONTENT_MASK                                                                         0xFFFFFFFFL
-//MP1_DC2MP_RESP
-#define MP1_DC2MP_RESP__CONTENT__SHIFT                                                                        0x0
-#define MP1_DC2MP_RESP__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP1_UVD2MP_RESP
-#define MP1_UVD2MP_RESP__CONTENT__SHIFT                                                                       0x0
-#define MP1_UVD2MP_RESP__CONTENT_MASK                                                                         0xFFFFFFFFL
-//MP1_VCE2MP_RESP
-#define MP1_VCE2MP_RESP__CONTENT__SHIFT                                                                       0x0
-#define MP1_VCE2MP_RESP__CONTENT_MASK                                                                         0xFFFFFFFFL
-//MP1_RLC2MP_RESP
-#define MP1_RLC2MP_RESP__CONTENT__SHIFT                                                                       0x0
-#define MP1_RLC2MP_RESP__CONTENT_MASK                                                                         0xFFFFFFFFL
-//MP1_C2PMSG_32
-#define MP1_C2PMSG_32__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_32__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_33
-#define MP1_C2PMSG_33__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_33__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_34
-#define MP1_C2PMSG_34__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_34__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_35
-#define MP1_C2PMSG_35__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_35__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_36
-#define MP1_C2PMSG_36__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_36__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_37
-#define MP1_C2PMSG_37__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_37__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_38
-#define MP1_C2PMSG_38__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_38__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_39
-#define MP1_C2PMSG_39__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_39__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_40
-#define MP1_C2PMSG_40__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_40__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_41
-#define MP1_C2PMSG_41__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_41__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_42
-#define MP1_C2PMSG_42__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_42__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_43
-#define MP1_C2PMSG_43__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_43__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_44
-#define MP1_C2PMSG_44__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_44__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_45
-#define MP1_C2PMSG_45__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_45__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_46
-#define MP1_C2PMSG_46__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_46__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_47
-#define MP1_C2PMSG_47__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_47__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_48
-#define MP1_C2PMSG_48__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_48__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_49
-#define MP1_C2PMSG_49__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_49__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_50
-#define MP1_C2PMSG_50__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_50__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_51
-#define MP1_C2PMSG_51__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_51__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_52
-#define MP1_C2PMSG_52__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_52__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_53
-#define MP1_C2PMSG_53__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_53__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_54
-#define MP1_C2PMSG_54__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_54__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_55
-#define MP1_C2PMSG_55__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_55__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_56
-#define MP1_C2PMSG_56__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_56__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_57
-#define MP1_C2PMSG_57__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_57__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_58
-#define MP1_C2PMSG_58__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_58__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_59
-#define MP1_C2PMSG_59__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_59__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_60
-#define MP1_C2PMSG_60__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_60__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_61
-#define MP1_C2PMSG_61__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_61__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_62
-#define MP1_C2PMSG_62__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_62__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_63
-#define MP1_C2PMSG_63__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_63__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_64
-#define MP1_C2PMSG_64__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_64__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_65
-#define MP1_C2PMSG_65__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_65__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_66
-#define MP1_C2PMSG_66__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_66__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_67
-#define MP1_C2PMSG_67__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_67__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_68
-#define MP1_C2PMSG_68__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_68__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_69
-#define MP1_C2PMSG_69__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_69__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_70
-#define MP1_C2PMSG_70__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_70__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_71
-#define MP1_C2PMSG_71__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_71__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_72
-#define MP1_C2PMSG_72__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_72__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_73
-#define MP1_C2PMSG_73__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_73__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_74
-#define MP1_C2PMSG_74__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_74__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_75
-#define MP1_C2PMSG_75__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_75__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_76
-#define MP1_C2PMSG_76__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_76__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_77
-#define MP1_C2PMSG_77__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_77__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_78
-#define MP1_C2PMSG_78__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_78__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_79
-#define MP1_C2PMSG_79__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_79__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_80
-#define MP1_C2PMSG_80__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_80__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_81
-#define MP1_C2PMSG_81__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_81__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_82
-#define MP1_C2PMSG_82__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_82__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_83
-#define MP1_C2PMSG_83__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_83__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_84
-#define MP1_C2PMSG_84__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_84__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_85
-#define MP1_C2PMSG_85__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_85__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_86
-#define MP1_C2PMSG_86__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_86__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_87
-#define MP1_C2PMSG_87__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_87__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_88
-#define MP1_C2PMSG_88__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_88__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_89
-#define MP1_C2PMSG_89__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_89__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_90
-#define MP1_C2PMSG_90__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_90__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_91
-#define MP1_C2PMSG_91__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_91__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_92
-#define MP1_C2PMSG_92__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_92__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_93
-#define MP1_C2PMSG_93__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_93__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_94
-#define MP1_C2PMSG_94__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_94__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_95
-#define MP1_C2PMSG_95__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_95__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_96
-#define MP1_C2PMSG_96__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_96__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_97
-#define MP1_C2PMSG_97__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_97__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_98
-#define MP1_C2PMSG_98__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_98__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_99
-#define MP1_C2PMSG_99__CONTENT__SHIFT                                                                         0x0
-#define MP1_C2PMSG_99__CONTENT_MASK                                                                           0xFFFFFFFFL
-//MP1_C2PMSG_100
-#define MP1_C2PMSG_100__CONTENT__SHIFT                                                                        0x0
-#define MP1_C2PMSG_100__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP1_C2PMSG_101
-#define MP1_C2PMSG_101__CONTENT__SHIFT                                                                        0x0
-#define MP1_C2PMSG_101__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP1_C2PMSG_102
-#define MP1_C2PMSG_102__CONTENT__SHIFT                                                                        0x0
-#define MP1_C2PMSG_102__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP1_C2PMSG_103
-#define MP1_C2PMSG_103__CONTENT__SHIFT                                                                        0x0
-#define MP1_C2PMSG_103__CONTENT_MASK                                                                          0xFFFFFFFFL
-//MP1_ACTIVE_FCN_ID
-#define MP1_ACTIVE_FCN_ID__VFID__SHIFT                                                                        0x0
-#define MP1_ACTIVE_FCN_ID__VF__SHIFT                                                                          0x1f
-#define MP1_ACTIVE_FCN_ID__VFID_MASK                                                                          0x0000000FL
-#define MP1_ACTIVE_FCN_ID__VF_MASK                                                                            0x80000000L
-//MP1_IH_CREDIT
-#define MP1_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                    0x0
-#define MP1_IH_CREDIT__CLIENT_ID__SHIFT                                                                       0x10
-#define MP1_IH_CREDIT__CREDIT_VALUE_MASK                                                                      0x00000003L
-#define MP1_IH_CREDIT__CLIENT_ID_MASK                                                                         0x00FF0000L
-//MP1_IH_SW_INT
-#define MP1_IH_SW_INT__ID__SHIFT                                                                              0x0
-#define MP1_IH_SW_INT__VALID__SHIFT                                                                           0x8
-#define MP1_IH_SW_INT__ID_MASK                                                                                0x000000FFL
-#define MP1_IH_SW_INT__VALID_MASK                                                                             0x00000100L
-//MP1_IH_SW_INT_CTRL
-#define MP1_IH_SW_INT_CTRL__INT_MASK__SHIFT                                                                   0x0
-#define MP1_IH_SW_INT_CTRL__INT_ACK__SHIFT                                                                    0x8
-#define MP1_IH_SW_INT_CTRL__INT_MASK_MASK                                                                     0x00000001L
-#define MP1_IH_SW_INT_CTRL__INT_ACK_MASK                                                                      0x00000100L
-//MP1_FPS_CNT
-#define MP1_FPS_CNT__COUNT__SHIFT                                                                             0x0
-#define MP1_FPS_CNT__COUNT_MASK                                                                               0xFFFFFFFFL
-//MP1_PUB_CTRL
-#define MP1_PUB_CTRL__RESET__SHIFT                                                                            0x0
-#define MP1_PUB_CTRL__RESET_MASK                                                                              0x00000001L
-//MP1_EXT_SCRATCH0
-#define MP1_EXT_SCRATCH0__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH0__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_EXT_SCRATCH1
-#define MP1_EXT_SCRATCH1__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH1__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_EXT_SCRATCH2
-#define MP1_EXT_SCRATCH2__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH2__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_EXT_SCRATCH3
-#define MP1_EXT_SCRATCH3__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH3__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_EXT_SCRATCH4
-#define MP1_EXT_SCRATCH4__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH4__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_EXT_SCRATCH5
-#define MP1_EXT_SCRATCH5__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH5__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_EXT_SCRATCH6
-#define MP1_EXT_SCRATCH6__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH6__DATA_MASK                                                                           0xFFFFFFFFL
-//MP1_EXT_SCRATCH7
-#define MP1_EXT_SCRATCH7__DATA__SHIFT                                                                         0x0
-#define MP1_EXT_SCRATCH7__DATA_MASK                                                                           0xFFFFFFFFL
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_default.h
deleted file mode 100644
index daa7eaef01b8..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/NBIF/nbif_6_1_default.h
+++ /dev/null
@@ -1,1271 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _nbif_6_1_DEFAULT_HEADER
-#define _nbif_6_1_DEFAULT_HEADER
-
-
-// addressBlock: bif_cfg_dev0_epf0_bifcfgdecp
-// base address: 0x0
-#define cfgVENDOR_ID_DEFAULT                                                      0x00000000
-#define cfgDEVICE_ID_DEFAULT                                                      0x00000000
-#define cfgCOMMAND_DEFAULT                                                        0x00000000
-#define cfgSTATUS_DEFAULT                                                         0x00000000
-#define cfgREVISION_ID_DEFAULT                                                    0x00000000
-#define cfgPROG_INTERFACE_DEFAULT                                                 0x00000000
-#define cfgSUB_CLASS_DEFAULT                                                      0x00000000
-#define cfgBASE_CLASS_DEFAULT                                                     0x00000000
-#define cfgCACHE_LINE_DEFAULT                                                     0x00000000
-#define cfgLATENCY_DEFAULT                                                        0x00000000
-#define cfgHEADER_DEFAULT                                                         0x00000000
-#define cfgBIST_DEFAULT                                                           0x00000000
-#define cfgBASE_ADDR_1_DEFAULT                                                    0x00000000
-#define cfgBASE_ADDR_2_DEFAULT                                                    0x00000000
-#define cfgBASE_ADDR_3_DEFAULT                                                    0x00000000
-#define cfgBASE_ADDR_4_DEFAULT                                                    0x00000000
-#define cfgBASE_ADDR_5_DEFAULT                                                    0x00000000
-#define cfgBASE_ADDR_6_DEFAULT                                                    0x00000000
-#define cfgADAPTER_ID_DEFAULT                                                     0x00000000
-#define cfgROM_BASE_ADDR_DEFAULT                                                  0x00000000
-#define cfgCAP_PTR_DEFAULT                                                        0x00000000
-#define cfgINTERRUPT_LINE_DEFAULT                                                 0x000000ff
-#define cfgINTERRUPT_PIN_DEFAULT                                                  0x00000000
-#define cfgMIN_GRANT_DEFAULT                                                      0x00000000
-#define cfgMAX_LATENCY_DEFAULT                                                    0x00000000
-#define cfgVENDOR_CAP_LIST_DEFAULT                                                0x00000000
-#define cfgADAPTER_ID_W_DEFAULT                                                   0x00000000
-#define cfgPMI_CAP_LIST_DEFAULT                                                   0x00000000
-#define cfgPMI_CAP_DEFAULT                                                        0x00000000
-#define cfgPMI_STATUS_CNTL_DEFAULT                                                0x00000000
-#define cfgPCIE_CAP_LIST_DEFAULT                                                  0x0000a000
-#define cfgPCIE_CAP_DEFAULT                                                       0x00000002
-#define cfgDEVICE_CAP_DEFAULT                                                     0x10000000
-#define cfgDEVICE_CNTL_DEFAULT                                                    0x00002810
-#define cfgDEVICE_STATUS_DEFAULT                                                  0x00000000
-#define cfgLINK_CAP_DEFAULT                                                       0x00011c03
-#define cfgLINK_CNTL_DEFAULT                                                      0x00000000
-#define cfgLINK_STATUS_DEFAULT                                                    0x00000001
-#define cfgDEVICE_CAP2_DEFAULT                                                    0x00000000
-#define cfgDEVICE_CNTL2_DEFAULT                                                   0x00000000
-#define cfgDEVICE_STATUS2_DEFAULT                                                 0x00000000
-#define cfgLINK_CAP2_DEFAULT                                                      0x0000000e
-#define cfgLINK_CNTL2_DEFAULT                                                     0x00000003
-#define cfgLINK_STATUS2_DEFAULT                                                   0x00000000
-#define cfgSLOT_CAP2_DEFAULT                                                      0x00000000
-#define cfgSLOT_CNTL2_DEFAULT                                                     0x00000000
-#define cfgSLOT_STATUS2_DEFAULT                                                   0x00000000
-#define cfgMSI_CAP_LIST_DEFAULT                                                   0x0000c000
-#define cfgMSI_MSG_CNTL_DEFAULT                                                   0x00000080
-#define cfgMSI_MSG_ADDR_LO_DEFAULT                                                0x00000000
-#define cfgMSI_MSG_ADDR_HI_DEFAULT                                                0x00000000
-#define cfgMSI_MSG_DATA_DEFAULT                                                   0x00000000
-#define cfgMSI_MSG_DATA_64_DEFAULT                                                0x00000000
-#define cfgMSI_MASK_DEFAULT                                                       0x00000000
-#define cfgMSI_PENDING_DEFAULT                                                    0x00000000
-#define cfgMSI_MASK_64_DEFAULT                                                    0x00000000
-#define cfgMSI_PENDING_64_DEFAULT                                                 0x00000000
-#define cfgMSIX_CAP_LIST_DEFAULT                                                  0x00000000
-#define cfgMSIX_MSG_CNTL_DEFAULT                                                  0x00000000
-#define cfgMSIX_TABLE_DEFAULT                                                     0x00000000
-#define cfgMSIX_PBA_DEFAULT                                                       0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_DEFAULT                              0x11000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_DEFAULT                                       0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC1_DEFAULT                                          0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC2_DEFAULT                                          0x00000000
-#define cfgPCIE_VC_ENH_CAP_LIST_DEFAULT                                           0x14000000
-#define cfgPCIE_PORT_VC_CAP_REG1_DEFAULT                                          0x00000000
-#define cfgPCIE_PORT_VC_CAP_REG2_DEFAULT                                          0x00000000
-#define cfgPCIE_PORT_VC_CNTL_DEFAULT                                              0x00000000
-#define cfgPCIE_PORT_VC_STATUS_DEFAULT                                            0x00000000
-#define cfgPCIE_VC0_RESOURCE_CAP_DEFAULT                                          0x00000000
-#define cfgPCIE_VC0_RESOURCE_CNTL_DEFAULT                                         0x000000fe
-#define cfgPCIE_VC0_RESOURCE_STATUS_DEFAULT                                       0x00000002
-#define cfgPCIE_VC1_RESOURCE_CAP_DEFAULT                                          0x00000000
-#define cfgPCIE_VC1_RESOURCE_CNTL_DEFAULT                                         0x00000000
-#define cfgPCIE_VC1_RESOURCE_STATUS_DEFAULT                                       0x00000002
-#define cfgPCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_DEFAULT                               0x15000000
-#define cfgPCIE_DEV_SERIAL_NUM_DW1_DEFAULT                                        0x00000000
-#define cfgPCIE_DEV_SERIAL_NUM_DW2_DEFAULT                                        0x00000000
-#define cfgPCIE_ADV_ERR_RPT_ENH_CAP_LIST_DEFAULT                                  0x20020000
-#define cfgPCIE_UNCORR_ERR_STATUS_DEFAULT                                         0x00000000
-#define cfgPCIE_UNCORR_ERR_MASK_DEFAULT                                           0x00000000
-#define cfgPCIE_UNCORR_ERR_SEVERITY_DEFAULT                                       0x00440010
-#define cfgPCIE_CORR_ERR_STATUS_DEFAULT                                           0x00000000
-#define cfgPCIE_CORR_ERR_MASK_DEFAULT                                             0x00002000
-#define cfgPCIE_ADV_ERR_CAP_CNTL_DEFAULT                                          0x00000000
-#define cfgPCIE_HDR_LOG0_DEFAULT                                                  0x00000000
-#define cfgPCIE_HDR_LOG1_DEFAULT                                                  0x00000000
-#define cfgPCIE_HDR_LOG2_DEFAULT                                                  0x00000000
-#define cfgPCIE_HDR_LOG3_DEFAULT                                                  0x00000000
-#define cfgPCIE_ROOT_ERR_CMD_DEFAULT                                              0x00000000
-#define cfgPCIE_ROOT_ERR_STATUS_DEFAULT                                           0x00000000
-#define cfgPCIE_ERR_SRC_ID_DEFAULT                                                0x00000000
-#define cfgPCIE_TLP_PREFIX_LOG0_DEFAULT                                           0x00000000
-#define cfgPCIE_TLP_PREFIX_LOG1_DEFAULT                                           0x00000000
-#define cfgPCIE_TLP_PREFIX_LOG2_DEFAULT                                           0x00000000
-#define cfgPCIE_TLP_PREFIX_LOG3_DEFAULT                                           0x00000000
-#define cfgPCIE_BAR_ENH_CAP_LIST_DEFAULT                                          0x24000000
-#define cfgPCIE_BAR1_CAP_DEFAULT                                                  0x00000000
-#define cfgPCIE_BAR1_CNTL_DEFAULT                                                 0x00000020
-#define cfgPCIE_BAR2_CAP_DEFAULT                                                  0x00000000
-#define cfgPCIE_BAR2_CNTL_DEFAULT                                                 0x00000000
-#define cfgPCIE_BAR3_CAP_DEFAULT                                                  0x00000000
-#define cfgPCIE_BAR3_CNTL_DEFAULT                                                 0x00000000
-#define cfgPCIE_BAR4_CAP_DEFAULT                                                  0x00000000
-#define cfgPCIE_BAR4_CNTL_DEFAULT                                                 0x00000000
-#define cfgPCIE_BAR5_CAP_DEFAULT                                                  0x00000000
-#define cfgPCIE_BAR5_CNTL_DEFAULT                                                 0x00000000
-#define cfgPCIE_BAR6_CAP_DEFAULT                                                  0x00000000
-#define cfgPCIE_BAR6_CNTL_DEFAULT                                                 0x00000000
-#define cfgPCIE_PWR_BUDGET_ENH_CAP_LIST_DEFAULT                                   0x25000000
-#define cfgPCIE_PWR_BUDGET_DATA_SELECT_DEFAULT                                    0x00000000
-#define cfgPCIE_PWR_BUDGET_DATA_DEFAULT                                           0x00000000
-#define cfgPCIE_PWR_BUDGET_CAP_DEFAULT                                            0x00000000
-#define cfgPCIE_DPA_ENH_CAP_LIST_DEFAULT                                          0x27000000
-#define cfgPCIE_DPA_CAP_DEFAULT                                                   0x00000000
-#define cfgPCIE_DPA_LATENCY_INDICATOR_DEFAULT                                     0x00000000
-#define cfgPCIE_DPA_STATUS_DEFAULT                                                0x00000100
-#define cfgPCIE_DPA_CNTL_DEFAULT                                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_0_DEFAULT                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_1_DEFAULT                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_2_DEFAULT                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_3_DEFAULT                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_4_DEFAULT                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_5_DEFAULT                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_6_DEFAULT                                  0x00000000
-#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_7_DEFAULT                                  0x00000000
-#define cfgPCIE_SECONDARY_ENH_CAP_LIST_DEFAULT                                    0x2a010019
-#define cfgPCIE_LINK_CNTL3_DEFAULT                                                0x00000000
-#define cfgPCIE_LANE_ERROR_STATUS_DEFAULT                                         0x00000000
-#define cfgPCIE_LANE_0_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_1_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_2_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_3_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_4_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_5_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_6_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_7_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_8_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_9_EQUALIZATION_CNTL_DEFAULT                                  0x00007f00
-#define cfgPCIE_LANE_10_EQUALIZATION_CNTL_DEFAULT                                 0x00007f00
-#define cfgPCIE_LANE_11_EQUALIZATION_CNTL_DEFAULT                                 0x00007f00
-#define cfgPCIE_LANE_12_EQUALIZATION_CNTL_DEFAULT                                 0x00007f00
-#define cfgPCIE_LANE_13_EQUALIZATION_CNTL_DEFAULT                                 0x00007f00
-#define cfgPCIE_LANE_14_EQUALIZATION_CNTL_DEFAULT                                 0x00007f00
-#define cfgPCIE_LANE_15_EQUALIZATION_CNTL_DEFAULT                                 0x00007f00
-#define cfgPCIE_ACS_ENH_CAP_LIST_DEFAULT                                          0x2b000000
-#define cfgPCIE_ACS_CAP_DEFAULT                                                   0x00000000
-#define cfgPCIE_ACS_CNTL_DEFAULT                                                  0x00000000
-#define cfgPCIE_ATS_ENH_CAP_LIST_DEFAULT                                          0x2c000000
-#define cfgPCIE_ATS_CAP_DEFAULT                                                   0x00000000
-#define cfgPCIE_ATS_CNTL_DEFAULT                                                  0x00000000
-#define cfgPCIE_PAGE_REQ_ENH_CAP_LIST_DEFAULT                                     0x2d000000
-#define cfgPCIE_PAGE_REQ_CNTL_DEFAULT                                             0x00000000
-#define cfgPCIE_PAGE_REQ_STATUS_DEFAULT                                           0x00000000
-#define cfgPCIE_OUTSTAND_PAGE_REQ_CAPACITY_DEFAULT                                0x00000000
-#define cfgPCIE_OUTSTAND_PAGE_REQ_ALLOC_DEFAULT                                   0x00000000
-#define cfgPCIE_PASID_ENH_CAP_LIST_DEFAULT                                        0x2e000000
-#define cfgPCIE_PASID_CAP_DEFAULT                                                 0x00000000
-#define cfgPCIE_PASID_CNTL_DEFAULT                                                0x00000000
-#define cfgPCIE_TPH_REQR_ENH_CAP_LIST_DEFAULT                                     0x2f000000
-#define cfgPCIE_TPH_REQR_CAP_DEFAULT                                              0x00000000
-#define cfgPCIE_TPH_REQR_CNTL_DEFAULT                                             0x00000000
-#define cfgPCIE_MC_ENH_CAP_LIST_DEFAULT                                           0x32000000
-#define cfgPCIE_MC_CAP_DEFAULT                                                    0x00000000
-#define cfgPCIE_MC_CNTL_DEFAULT                                                   0x00000000
-#define cfgPCIE_MC_ADDR0_DEFAULT                                                  0x00000000
-#define cfgPCIE_MC_ADDR1_DEFAULT                                                  0x00000000
-#define cfgPCIE_MC_RCV0_DEFAULT                                                   0x00000000
-#define cfgPCIE_MC_RCV1_DEFAULT                                                   0x00000000
-#define cfgPCIE_MC_BLOCK_ALL0_DEFAULT                                             0x00000000
-#define cfgPCIE_MC_BLOCK_ALL1_DEFAULT                                             0x00000000
-#define cfgPCIE_MC_BLOCK_UNTRANSLATED_0_DEFAULT                                   0x00000000
-#define cfgPCIE_MC_BLOCK_UNTRANSLATED_1_DEFAULT                                   0x00000000
-#define cfgPCIE_LTR_ENH_CAP_LIST_DEFAULT                                          0x32800000
-#define cfgPCIE_LTR_CAP_DEFAULT                                                   0x00000000
-#define cfgPCIE_ARI_ENH_CAP_LIST_DEFAULT                                          0x33000000
-#define cfgPCIE_ARI_CAP_DEFAULT                                                   0x00000000
-#define cfgPCIE_ARI_CNTL_DEFAULT                                                  0x00000000
-#define cfgPCIE_SRIOV_ENH_CAP_LIST_DEFAULT                                        0x00000000
-#define cfgPCIE_SRIOV_CAP_DEFAULT                                                 0x00000000
-#define cfgPCIE_SRIOV_CONTROL_DEFAULT                                             0x00000000
-#define cfgPCIE_SRIOV_STATUS_DEFAULT                                              0x00000000
-#define cfgPCIE_SRIOV_INITIAL_VFS_DEFAULT                                         0x00000000
-#define cfgPCIE_SRIOV_TOTAL_VFS_DEFAULT                                           0x00000000
-#define cfgPCIE_SRIOV_NUM_VFS_DEFAULT                                             0x00000000
-#define cfgPCIE_SRIOV_FUNC_DEP_LINK_DEFAULT                                       0x00000000
-#define cfgPCIE_SRIOV_FIRST_VF_OFFSET_DEFAULT                                     0x00000000
-#define cfgPCIE_SRIOV_VF_STRIDE_DEFAULT                                           0x00000000
-#define cfgPCIE_SRIOV_VF_DEVICE_ID_DEFAULT                                        0x00000000
-#define cfgPCIE_SRIOV_SUPPORTED_PAGE_SIZE_DEFAULT                                 0x00000000
-#define cfgPCIE_SRIOV_SYSTEM_PAGE_SIZE_DEFAULT                                    0x00000001
-#define cfgPCIE_SRIOV_VF_BASE_ADDR_0_DEFAULT                                      0x00000000
-#define cfgPCIE_SRIOV_VF_BASE_ADDR_1_DEFAULT                                      0x00000000
-#define cfgPCIE_SRIOV_VF_BASE_ADDR_2_DEFAULT                                      0x00000000
-#define cfgPCIE_SRIOV_VF_BASE_ADDR_3_DEFAULT                                      0x00000000
-#define cfgPCIE_SRIOV_VF_BASE_ADDR_4_DEFAULT                                      0x00000000
-#define cfgPCIE_SRIOV_VF_BASE_ADDR_5_DEFAULT                                      0x00000000
-#define cfgPCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV_DEFAULT                       0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_DEFAULT                                0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW_DEFAULT                   0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE_DEFAULT                    0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS_DEFAULT                    0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL_DEFAULT                  0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0_DEFAULT                  0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1_DEFAULT                  0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2_DEFAULT                  0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_CONTEXT_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB_DEFAULT                       0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF0_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF1_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF2_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF3_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF4_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF5_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF6_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF7_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF8_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF9_FB_DEFAULT                         0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF10_FB_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF11_FB_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF12_FB_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF13_FB_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF14_FB_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VF15_FB_DEFAULT                        0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW0_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW1_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW2_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW3_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW4_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW5_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW6_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_UVDSCH_DW7_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW0_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW1_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW2_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW3_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW4_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW5_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW6_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_VCESCH_DW7_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW0_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW1_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW2_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW3_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW4_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW5_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW6_DEFAULT                     0x00000000
-#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_GFXSCH_DW7_DEFAULT                     0x00000000
-
-
-// addressBlock: bif_cfg_dev0_swds_bifcfgdecp
-// base address: 0x0
-#define mmSUB_BUS_NUMBER_LATENCY_DEFAULT                                         0x00000000
-#define mmIO_BASE_LIMIT_DEFAULT                                                  0x00000000
-#define mmSECONDARY_STATUS_DEFAULT                                               0x00000000
-#define mmMEM_BASE_LIMIT_DEFAULT                                                 0x00000000
-#define mmPREF_BASE_LIMIT_DEFAULT                                                0x00000000
-#define mmPREF_BASE_UPPER_DEFAULT                                                0x00000000
-#define mmPREF_LIMIT_UPPER_DEFAULT                                               0x00000000
-#define mmIO_BASE_LIMIT_HI_DEFAULT                                               0x00000000
-#define mmIRQ_BRIDGE_CNTL_DEFAULT                                                0x00000000
-#define mmSLOT_CAP_DEFAULT                                                       0x00000000
-#define mmSLOT_CNTL_DEFAULT                                                      0x00000000
-#define mmSLOT_STATUS_DEFAULT                                                    0x00000000
-#define mmSSID_CAP_LIST_DEFAULT                                                  0x00000000
-#define mmSSID_CAP_DEFAULT                                                       0x00000000
-
-
-// addressBlock: rcc_shadow_reg_shadowdec
-// base address: 0x0
-#define ixSHADOW_COMMAND_DEFAULT                                                 0x00000000
-#define ixSHADOW_BASE_ADDR_1_DEFAULT                                             0x00000000
-#define ixSHADOW_BASE_ADDR_2_DEFAULT                                             0x00000000
-#define ixSHADOW_SUB_BUS_NUMBER_LATENCY_DEFAULT                                  0x00000000
-#define ixSHADOW_IO_BASE_LIMIT_DEFAULT                                           0x00000000
-#define ixSHADOW_MEM_BASE_LIMIT_DEFAULT                                          0x00000000
-#define ixSHADOW_PREF_BASE_LIMIT_DEFAULT                                         0x00000000
-#define ixSHADOW_PREF_BASE_UPPER_DEFAULT                                         0x00000000
-#define ixSHADOW_PREF_LIMIT_UPPER_DEFAULT                                        0x00000000
-#define ixSHADOW_IO_BASE_LIMIT_HI_DEFAULT                                        0x00000000
-#define ixSHADOW_IRQ_BRIDGE_CNTL_DEFAULT                                         0x00000000
-#define ixSUC_INDEX_DEFAULT                                                      0x00000000
-#define ixSUC_DATA_DEFAULT                                                       0x00000000
-
-
-// addressBlock: bif_bx_pf_SUMDEC
-// base address: 0x0
-#define ixSUM_INDEX_DEFAULT                                                      0x00000000
-#define ixSUM_DATA_DEFAULT                                                       0x00000000
-
-
-// addressBlock: gdc_GDCDEC
-// base address: 0x1400000
-#define mmA2S_CNTL_CL0_DEFAULT                                                   0x00280540
-#define mmA2S_CNTL_CL1_DEFAULT                                                   0x00282540
-#define mmA2S_CNTL_CL2_DEFAULT                                                   0x002825a0
-#define mmA2S_CNTL_CL3_DEFAULT                                                   0x00282550
-#define mmA2S_CNTL_CL4_DEFAULT                                                   0x00282550
-#define mmA2S_CNTL_SW0_DEFAULT                                                   0x08080005
-#define mmA2S_CNTL_SW1_DEFAULT                                                   0x08080205
-#define mmA2S_CNTL_SW2_DEFAULT                                                   0x08080200
-#define mmNGDC_MGCG_CTRL_DEFAULT                                                 0x00000080
-#define mmA2S_MISC_CNTL_DEFAULT                                                  0x00000003
-#define mmNGDC_SDP_PORT_CTRL_DEFAULT                                             0x0000000f
-#define mmNGDC_RESERVED_0_DEFAULT                                                0x00000000
-#define mmNGDC_RESERVED_1_DEFAULT                                                0x00000000
-#define mmBIF_SDMA0_DOORBELL_RANGE_DEFAULT                                       0x00000000
-#define mmBIF_SDMA1_DOORBELL_RANGE_DEFAULT                                       0x00000000
-#define mmBIF_IH_DOORBELL_RANGE_DEFAULT                                          0x00000000
-#define mmBIF_MMSCH0_DOORBELL_RANGE_DEFAULT                                      0x00000000
-#define mmBIF_DOORBELL_FENCE_CNTL_DEFAULT                                        0x00000000
-#define mmS2A_MISC_CNTL_DEFAULT                                                  0x00000000
-#define mmA2S_CNTL2_SEC_CL0_DEFAULT                                              0x00000006
-#define mmA2S_CNTL2_SEC_CL1_DEFAULT                                              0x00000006
-#define mmA2S_CNTL2_SEC_CL2_DEFAULT                                              0x00000006
-#define mmA2S_CNTL2_SEC_CL3_DEFAULT                                              0x00000006
-#define mmA2S_CNTL2_SEC_CL4_DEFAULT                                              0x00000006
-
-
-// addressBlock: nbif_sion_SIONDEC
-// base address: 0x1400000
-#define ixSION_CL0_RdRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL0_RdRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL0_RdRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL0_RdRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL0_WrRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL0_WrRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL0_WrRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL0_WrRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL0_Req_BurstTarget_REG0_DEFAULT                                  0x00000000
-#define ixSION_CL0_Req_BurstTarget_REG1_DEFAULT                                  0x00000000
-#define ixSION_CL0_Req_TimeSlot_REG0_DEFAULT                                     0x00000000
-#define ixSION_CL0_Req_TimeSlot_REG1_DEFAULT                                     0x00000000
-#define ixSION_CL0_ReqPoolCredit_Alloc_REG0_DEFAULT                              0x00000000
-#define ixSION_CL0_ReqPoolCredit_Alloc_REG1_DEFAULT                              0x00000000
-#define ixSION_CL0_DataPoolCredit_Alloc_REG0_DEFAULT                             0x00000000
-#define ixSION_CL0_DataPoolCredit_Alloc_REG1_DEFAULT                             0x00000000
-#define ixSION_CL0_RdRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL0_RdRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL0_WrRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL0_WrRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL1_RdRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL1_RdRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL1_RdRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL1_RdRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL1_WrRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL1_WrRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL1_WrRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL1_WrRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL1_Req_BurstTarget_REG0_DEFAULT                                  0x00000000
-#define ixSION_CL1_Req_BurstTarget_REG1_DEFAULT                                  0x00000000
-#define ixSION_CL1_Req_TimeSlot_REG0_DEFAULT                                     0x00000000
-#define ixSION_CL1_Req_TimeSlot_REG1_DEFAULT                                     0x00000000
-#define ixSION_CL1_ReqPoolCredit_Alloc_REG0_DEFAULT                              0x00000000
-#define ixSION_CL1_ReqPoolCredit_Alloc_REG1_DEFAULT                              0x00000000
-#define ixSION_CL1_DataPoolCredit_Alloc_REG0_DEFAULT                             0x00000000
-#define ixSION_CL1_DataPoolCredit_Alloc_REG1_DEFAULT                             0x00000000
-#define ixSION_CL1_RdRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL1_RdRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL1_WrRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL1_WrRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL2_RdRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL2_RdRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL2_RdRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL2_RdRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL2_WrRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL2_WrRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL2_WrRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL2_WrRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL2_Req_BurstTarget_REG0_DEFAULT                                  0x00000000
-#define ixSION_CL2_Req_BurstTarget_REG1_DEFAULT                                  0x00000000
-#define ixSION_CL2_Req_TimeSlot_REG0_DEFAULT                                     0x00000000
-#define ixSION_CL2_Req_TimeSlot_REG1_DEFAULT                                     0x00000000
-#define ixSION_CL2_ReqPoolCredit_Alloc_REG0_DEFAULT                              0x00000000
-#define ixSION_CL2_ReqPoolCredit_Alloc_REG1_DEFAULT                              0x00000000
-#define ixSION_CL2_DataPoolCredit_Alloc_REG0_DEFAULT                             0x00000000
-#define ixSION_CL2_DataPoolCredit_Alloc_REG1_DEFAULT                             0x00000000
-#define ixSION_CL2_RdRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL2_RdRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL2_WrRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL2_WrRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL3_RdRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL3_RdRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL3_RdRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL3_RdRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL3_WrRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL3_WrRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL3_WrRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL3_WrRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL3_Req_BurstTarget_REG0_DEFAULT                                  0x00000000
-#define ixSION_CL3_Req_BurstTarget_REG1_DEFAULT                                  0x00000000
-#define ixSION_CL3_Req_TimeSlot_REG0_DEFAULT                                     0x00000000
-#define ixSION_CL3_Req_TimeSlot_REG1_DEFAULT                                     0x00000000
-#define ixSION_CL3_ReqPoolCredit_Alloc_REG0_DEFAULT                              0x00000000
-#define ixSION_CL3_ReqPoolCredit_Alloc_REG1_DEFAULT                              0x00000000
-#define ixSION_CL3_DataPoolCredit_Alloc_REG0_DEFAULT                             0x00000000
-#define ixSION_CL3_DataPoolCredit_Alloc_REG1_DEFAULT                             0x00000000
-#define ixSION_CL3_RdRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL3_RdRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL3_WrRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL3_WrRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL4_RdRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL4_RdRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL4_RdRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL4_RdRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL4_WrRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL4_WrRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL4_WrRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL4_WrRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL4_Req_BurstTarget_REG0_DEFAULT                                  0x00000000
-#define ixSION_CL4_Req_BurstTarget_REG1_DEFAULT                                  0x00000000
-#define ixSION_CL4_Req_TimeSlot_REG0_DEFAULT                                     0x00000000
-#define ixSION_CL4_Req_TimeSlot_REG1_DEFAULT                                     0x00000000
-#define ixSION_CL4_ReqPoolCredit_Alloc_REG0_DEFAULT                              0x00000000
-#define ixSION_CL4_ReqPoolCredit_Alloc_REG1_DEFAULT                              0x00000000
-#define ixSION_CL4_DataPoolCredit_Alloc_REG0_DEFAULT                             0x00000000
-#define ixSION_CL4_DataPoolCredit_Alloc_REG1_DEFAULT                             0x00000000
-#define ixSION_CL4_RdRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL4_RdRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL4_WrRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL4_WrRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL5_RdRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL5_RdRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL5_RdRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL5_RdRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL5_WrRsp_BurstTarget_REG0_DEFAULT                                0x00000000
-#define ixSION_CL5_WrRsp_BurstTarget_REG1_DEFAULT                                0x00000000
-#define ixSION_CL5_WrRsp_TimeSlot_REG0_DEFAULT                                   0x00000000
-#define ixSION_CL5_WrRsp_TimeSlot_REG1_DEFAULT                                   0x00000000
-#define ixSION_CL5_Req_BurstTarget_REG0_DEFAULT                                  0x00000000
-#define ixSION_CL5_Req_BurstTarget_REG1_DEFAULT                                  0x00000000
-#define ixSION_CL5_Req_TimeSlot_REG0_DEFAULT                                     0x00000000
-#define ixSION_CL5_Req_TimeSlot_REG1_DEFAULT                                     0x00000000
-#define ixSION_CL5_ReqPoolCredit_Alloc_REG0_DEFAULT                              0x00000000
-#define ixSION_CL5_ReqPoolCredit_Alloc_REG1_DEFAULT                              0x00000000
-#define ixSION_CL5_DataPoolCredit_Alloc_REG0_DEFAULT                             0x00000000
-#define ixSION_CL5_DataPoolCredit_Alloc_REG1_DEFAULT                             0x00000000
-#define ixSION_CL5_RdRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL5_RdRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CL5_WrRspPoolCredit_Alloc_REG0_DEFAULT                            0x00000000
-#define ixSION_CL5_WrRspPoolCredit_Alloc_REG1_DEFAULT                            0x00000000
-#define ixSION_CNTL_REG0_DEFAULT                                                 0x00000000
-#define ixSION_CNTL_REG1_DEFAULT                                                 0x00000000
-
-
-// addressBlock: syshub_mmreg_direct_syshubdirect
-// base address: 0x1400000
-#define ixSYSHUB_DS_CTRL_SOCCLK_DEFAULT                                          0x00000000
-#define ixSYSHUB_DS_CTRL2_SOCCLK_DEFAULT                                         0x00000100
-#define ixSYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK_DEFAULT                       0x00000000
-#define ixSYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK_DEFAULT                          0x00000000
-#define ixDMA_CLK0_SW0_SYSHUB_QOS_CNTL_DEFAULT                                   0x0000001e
-#define ixDMA_CLK0_SW1_SYSHUB_QOS_CNTL_DEFAULT                                   0x0000001e
-#define ixDMA_CLK0_SW0_CL0_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK0_SW0_CL1_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK0_SW0_CL2_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK0_SW0_CL3_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK0_SW0_CL4_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK0_SW0_CL5_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK0_SW1_CL0_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK0_SW2_CL0_CNTL_DEFAULT                                          0x20200000
-#define ixSYSHUB_CG_CNTL_DEFAULT                                                 0x00082000
-#define ixSYSHUB_TRANS_IDLE_DEFAULT                                              0x00000000
-#define ixSYSHUB_HP_TIMER_DEFAULT                                                0x00000100
-#define ixSYSHUB_SCRATCH_DEFAULT                                                 0x00000040
-#define ixSYSHUB_DS_CTRL_SHUBCLK_DEFAULT                                         0x00000000
-#define ixSYSHUB_DS_CTRL2_SHUBCLK_DEFAULT                                        0x00000100
-#define ixSYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SHUBCLK_DEFAULT                      0x00000000
-#define ixSYSHUB_BGEN_ENHANCEMENT_IMM_EN_SHUBCLK_DEFAULT                         0x00000000
-#define ixDMA_CLK1_SW0_SYSHUB_QOS_CNTL_DEFAULT                                   0x0000001e
-#define ixDMA_CLK1_SW1_SYSHUB_QOS_CNTL_DEFAULT                                   0x0000001e
-#define ixDMA_CLK1_SW0_CL0_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW0_CL1_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW0_CL2_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW0_CL3_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW0_CL4_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW1_CL0_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW1_CL1_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW1_CL2_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW1_CL3_CNTL_DEFAULT                                          0x20200000
-#define ixDMA_CLK1_SW1_CL4_CNTL_DEFAULT                                          0x20200000
-
-
-// addressBlock: gdc_ras_gdc_ras_regblk
-// base address: 0x1400000
-#define ixGDC_RAS_LEAF0_CTRL_DEFAULT                                             0x00000000
-#define ixGDC_RAS_LEAF1_CTRL_DEFAULT                                             0x00000000
-#define ixGDC_RAS_LEAF2_CTRL_DEFAULT                                             0x00000000
-#define ixGDC_RAS_LEAF3_CTRL_DEFAULT                                             0x00000000
-#define ixGDC_RAS_LEAF4_CTRL_DEFAULT                                             0x00000000
-#define ixGDC_RAS_LEAF5_CTRL_DEFAULT                                             0x00000000
-
-
-// addressBlock: gdc_rst_GDCRST_DEC
-// base address: 0x1400000
-#define ixSHUB_PF_FLR_RST_DEFAULT                                                0x00000000
-#define ixSHUB_GFX_DRV_MODE1_RST_DEFAULT                                         0x00000000
-#define ixSHUB_LINK_RESET_DEFAULT                                                0x00000000
-#define ixSHUB_PF0_VF_FLR_RST_DEFAULT                                            0x00000000
-#define ixSHUB_HARD_RST_CTRL_DEFAULT                                             0x0000001b
-#define ixSHUB_SOFT_RST_CTRL_DEFAULT                                             0x00000009
-#define ixSHUB_SDP_PORT_RST_DEFAULT                                              0x00000000
-
-
-// addressBlock: bif_bx_pf_SYSDEC
-// base address: 0x0
-#define mmSBIOS_SCRATCH_0_DEFAULT                                                0x00000000
-#define mmSBIOS_SCRATCH_1_DEFAULT                                                0x00000000
-#define mmSBIOS_SCRATCH_2_DEFAULT                                                0x00000000
-#define mmSBIOS_SCRATCH_3_DEFAULT                                                0x00000000
-#define mmBIOS_SCRATCH_0_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_1_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_2_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_3_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_4_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_5_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_6_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_7_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_8_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_9_DEFAULT                                                 0x00000000
-#define mmBIOS_SCRATCH_10_DEFAULT                                                0x00000000
-#define mmBIOS_SCRATCH_11_DEFAULT                                                0x00000000
-#define mmBIOS_SCRATCH_12_DEFAULT                                                0x00000000
-#define mmBIOS_SCRATCH_13_DEFAULT                                                0x00000000
-#define mmBIOS_SCRATCH_14_DEFAULT                                                0x00000000
-#define mmBIOS_SCRATCH_15_DEFAULT                                                0x00000000
-#define mmBIF_RLC_INTR_CNTL_DEFAULT                                              0x00000000
-#define mmBIF_VCE_INTR_CNTL_DEFAULT                                              0x00000000
-#define mmBIF_UVD_INTR_CNTL_DEFAULT                                              0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR0_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR0_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR1_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR1_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR2_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR2_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR3_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR3_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR4_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR4_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR5_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR5_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR6_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR6_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_ADDR7_DEFAULT                                          0x00000000
-#define mmGFX_MMIOREG_CAM_REMAP_ADDR7_DEFAULT                                    0x00000000
-#define mmGFX_MMIOREG_CAM_CNTL_DEFAULT                                           0x00000000
-#define mmGFX_MMIOREG_CAM_ZERO_CPL_DEFAULT                                       0x00000000
-#define mmGFX_MMIOREG_CAM_ONE_CPL_DEFAULT                                        0x00000000
-#define mmGFX_MMIOREG_CAM_PROGRAMMABLE_CPL_DEFAULT                               0x00000000
-
-
-// addressBlock: bif_bx_pf_SYSPFVFDEC
-// base address: 0x0
-#define mmMM_INDEX_DEFAULT                                                       0x00000000
-#define mmMM_DATA_DEFAULT                                                        0x00000000
-#define mmMM_INDEX_HI_DEFAULT                                                    0x00000000
-#define mmSYSHUB_INDEX_OVLP_DEFAULT                                              0x00000000
-#define mmSYSHUB_DATA_OVLP_DEFAULT                                               0x00000000
-#define mmPCIE_INDEX_DEFAULT                                                     0x00000000
-#define mmPCIE_DATA_DEFAULT                                                      0x00000000
-#define mmPCIE_INDEX2_DEFAULT                                                    0x00000000
-#define mmPCIE_DATA2_DEFAULT                                                     0x00000000
-
-
-// addressBlock: rcc_dwn_BIFDEC1
-// base address: 0x0
-#define mmDN_PCIE_RESERVED_DEFAULT                                               0x00000000
-#define mmDN_PCIE_SCRATCH_DEFAULT                                                0x00000000
-#define mmDN_PCIE_CNTL_DEFAULT                                                   0x00000000
-#define mmDN_PCIE_CONFIG_CNTL_DEFAULT                                            0x00000000
-#define mmDN_PCIE_RX_CNTL2_DEFAULT                                               0x00000000
-#define mmDN_PCIE_BUS_CNTL_DEFAULT                                               0x00000080
-#define mmDN_PCIE_CFG_CNTL_DEFAULT                                               0x00000000
-#define mmDN_PCIE_STRAP_F0_DEFAULT                                               0x00000001
-#define mmDN_PCIE_STRAP_MISC_DEFAULT                                             0x00000000
-#define mmDN_PCIE_STRAP_MISC2_DEFAULT                                            0x00000000
-
-
-// addressBlock: rcc_dwnp_BIFDEC1
-// base address: 0x0
-#define mmPCIEP_RESERVED_DEFAULT                                                 0x00000000
-#define mmPCIEP_SCRATCH_DEFAULT                                                  0x00000000
-#define mmPCIE_ERR_CNTL_DEFAULT                                                  0x00000500
-#define mmPCIE_RX_CNTL_DEFAULT                                                   0x00000000
-#define mmPCIE_LC_SPEED_CNTL_DEFAULT                                             0x00000000
-#define mmPCIE_LC_CNTL2_DEFAULT                                                  0x00000000
-#define mmPCIEP_STRAP_MISC_DEFAULT                                               0x00000000
-#define mmLTR_MSG_INFO_FROM_EP_DEFAULT                                           0x00000000
-
-
-// addressBlock: rcc_ep_BIFDEC1
-// base address: 0x0
-#define mmEP_PCIE_SCRATCH_DEFAULT                                                0x00000000
-#define mmEP_PCIE_CNTL_DEFAULT                                                   0x00000100
-#define mmEP_PCIE_INT_CNTL_DEFAULT                                               0x00000000
-#define mmEP_PCIE_INT_STATUS_DEFAULT                                             0x00000000
-#define mmEP_PCIE_RX_CNTL2_DEFAULT                                               0x00000000
-#define mmEP_PCIE_BUS_CNTL_DEFAULT                                               0x00000080
-#define mmEP_PCIE_CFG_CNTL_DEFAULT                                               0x00000000
-#define mmEP_PCIE_OBFF_CNTL_DEFAULT                                              0x00012774
-#define mmEP_PCIE_TX_LTR_CNTL_DEFAULT                                            0x00003468
-#define mmEP_PCIE_STRAP_MISC_DEFAULT                                             0x00000000
-#define mmEP_PCIE_STRAP_MISC2_DEFAULT                                            0x00000000
-#define mmEP_PCIE_STRAP_PI_DEFAULT                                               0x00000000
-#define mmEP_PCIE_F0_DPA_CAP_DEFAULT                                             0x190a1000
-#define mmEP_PCIE_F0_DPA_LATENCY_INDICATOR_DEFAULT                               0x000000f0
-#define mmEP_PCIE_F0_DPA_CNTL_DEFAULT                                            0x00000100
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_DEFAULT                               0x000000fa
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_DEFAULT                               0x000000c8
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_DEFAULT                               0x00000096
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_DEFAULT                               0x00000064
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_DEFAULT                               0x0000004b
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_DEFAULT                               0x00000032
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_DEFAULT                               0x00000019
-#define mmPCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_DEFAULT                               0x0000000a
-#define mmEP_PCIE_PME_CONTROL_DEFAULT                                            0x00000000
-#define mmEP_PCIEP_RESERVED_DEFAULT                                              0x00000000
-#define mmEP_PCIE_TX_CNTL_DEFAULT                                                0x00000000
-#define mmEP_PCIE_TX_REQUESTER_ID_DEFAULT                                        0x00000000
-#define mmEP_PCIE_ERR_CNTL_DEFAULT                                               0x00000500
-#define mmEP_PCIE_RX_CNTL_DEFAULT                                                0x01000000
-#define mmEP_PCIE_LC_SPEED_CNTL_DEFAULT                                          0x00000000
-
-
-// addressBlock: bif_bx_pf_BIFDEC1
-// base address: 0x0
-#define mmBIF_MM_INDACCESS_CNTL_DEFAULT                                          0x00000000
-#define mmBUS_CNTL_DEFAULT                                                       0x00000000
-#define mmBIF_SCRATCH0_DEFAULT                                                   0x00000000
-#define mmBIF_SCRATCH1_DEFAULT                                                   0x00000000
-#define mmBX_RESET_EN_DEFAULT                                                    0x00010003
-#define mmMM_CFGREGS_CNTL_DEFAULT                                                0x00000000
-#define mmBX_RESET_CNTL_DEFAULT                                                  0x00000000
-#define mmINTERRUPT_CNTL_DEFAULT                                                 0x00000010
-#define mmINTERRUPT_CNTL2_DEFAULT                                                0x00000000
-#define mmCLKREQB_PAD_CNTL_DEFAULT                                               0x000008e0
-#define mmCLKREQB_PERF_COUNTER_DEFAULT                                           0x00000000
-#define mmBIF_CLK_CTRL_DEFAULT                                                   0x00000000
-#define mmBIF_FEATURES_CONTROL_MISC_DEFAULT                                      0x00000000
-#define mmBIF_DOORBELL_CNTL_DEFAULT                                              0x00000000
-#define mmBIF_DOORBELL_INT_CNTL_DEFAULT                                          0x00000000
-#define mmBIF_SLVARB_MODE_DEFAULT                                                0x00000000
-#define mmBIF_FB_EN_DEFAULT                                                      0x00000000
-#define mmBIF_BUSY_DELAY_CNTR_DEFAULT                                            0x0000003f
-#define mmBIF_PERFMON_CNTL_DEFAULT                                               0x00000000
-#define mmBIF_PERFCOUNTER0_RESULT_DEFAULT                                        0x00000000
-#define mmBIF_PERFCOUNTER1_RESULT_DEFAULT                                        0x00000000
-#define mmBIF_MST_TRANS_PENDING_VF_DEFAULT                                       0x00000000
-#define mmBIF_SLV_TRANS_PENDING_VF_DEFAULT                                       0x00000000
-#define mmBACO_CNTL_DEFAULT                                                      0x00000000
-#define mmBIF_BACO_EXIT_TIME0_DEFAULT                                            0x00000100
-#define mmBIF_BACO_EXIT_TIMER1_DEFAULT                                           0x00000100
-#define mmBIF_BACO_EXIT_TIMER2_DEFAULT                                           0x00000300
-#define mmBIF_BACO_EXIT_TIMER3_DEFAULT                                           0x00000400
-#define mmBIF_BACO_EXIT_TIMER4_DEFAULT                                           0x00000100
-#define mmMEM_TYPE_CNTL_DEFAULT                                                  0x00000000
-#define mmSMU_BIF_VDDGFX_PWR_STATUS_DEFAULT                                      0x00000000
-#define mmBIF_VDDGFX_GFX0_LOWER_DEFAULT                                          0xc0008000
-#define mmBIF_VDDGFX_GFX0_UPPER_DEFAULT                                          0x0000cffc
-#define mmBIF_VDDGFX_GFX1_LOWER_DEFAULT                                          0xc0028000
-#define mmBIF_VDDGFX_GFX1_UPPER_DEFAULT                                          0x00031ffc
-#define mmBIF_VDDGFX_GFX2_LOWER_DEFAULT                                          0xc0034000
-#define mmBIF_VDDGFX_GFX2_UPPER_DEFAULT                                          0x00037ffc
-#define mmBIF_VDDGFX_GFX3_LOWER_DEFAULT                                          0xc003c000
-#define mmBIF_VDDGFX_GFX3_UPPER_DEFAULT                                          0x0003e1fc
-#define mmBIF_VDDGFX_GFX4_LOWER_DEFAULT                                          0xc003ec00
-#define mmBIF_VDDGFX_GFX4_UPPER_DEFAULT                                          0x0003f1fc
-#define mmBIF_VDDGFX_GFX5_LOWER_DEFAULT                                          0xc003fc00
-#define mmBIF_VDDGFX_GFX5_UPPER_DEFAULT                                          0x0003fffc
-#define mmBIF_VDDGFX_RSV1_LOWER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_RSV1_UPPER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_RSV2_LOWER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_RSV2_UPPER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_RSV3_LOWER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_RSV3_UPPER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_RSV4_LOWER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_RSV4_UPPER_DEFAULT                                          0x00000000
-#define mmBIF_VDDGFX_FB_CMP_DEFAULT                                              0x00000000
-#define mmBIF_DOORBELL_GBLAPER1_LOWER_DEFAULT                                    0x80000780
-#define mmBIF_DOORBELL_GBLAPER1_UPPER_DEFAULT                                    0x000007fc
-#define mmBIF_DOORBELL_GBLAPER2_LOWER_DEFAULT                                    0x80000800
-#define mmBIF_DOORBELL_GBLAPER2_UPPER_DEFAULT                                    0x0000087c
-#define mmREMAP_HDP_MEM_FLUSH_CNTL_DEFAULT                                       0x0000385c
-#define mmREMAP_HDP_REG_FLUSH_CNTL_DEFAULT                                       0x00003858
-#define mmBIF_RB_CNTL_DEFAULT                                                    0x00000000
-#define mmBIF_RB_BASE_DEFAULT                                                    0x00000000
-#define mmBIF_RB_RPTR_DEFAULT                                                    0x00000000
-#define mmBIF_RB_WPTR_DEFAULT                                                    0x00000000
-#define mmBIF_RB_WPTR_ADDR_HI_DEFAULT                                            0x00000000
-#define mmBIF_RB_WPTR_ADDR_LO_DEFAULT                                            0x00000000
-#define mmMAILBOX_INDEX_DEFAULT                                                  0x00000000
-#define mmBIF_GPUIOV_RESET_NOTIFICATION_DEFAULT                                  0x00000000
-#define mmBIF_UVD_GPUIOV_CFG_SIZE_DEFAULT                                        0x00000008
-#define mmBIF_VCE_GPUIOV_CFG_SIZE_DEFAULT                                        0x00000008
-#define mmBIF_GFX_SDMA_GPUIOV_CFG_SIZE_DEFAULT                                   0x00000008
-#define mmBIF_GMI_WRR_WEIGHT_DEFAULT                                             0x00202020
-#define mmNBIF_STRAP_WRITE_CTRL_DEFAULT                                          0x00000000
-#define mmBIF_PERSTB_PAD_CNTL_DEFAULT                                            0x000000c0
-#define mmBIF_PX_EN_PAD_CNTL_DEFAULT                                             0x00000031
-#define mmBIF_REFPADKIN_PAD_CNTL_DEFAULT                                         0x00000007
-#define mmBIF_CLKREQB_PAD_CNTL_DEFAULT                                           0x00600100
-
-
-// addressBlock: rcc_pf_0_BIFDEC1
-// base address: 0x0
-#define mmRCC_BACO_CNTL_MISC_DEFAULT                                             0x00000000
-#define mmRCC_RESET_EN_DEFAULT                                                   0x00008000
-#define mmRCC_VDM_SUPPORT_DEFAULT                                                0x00000000
-#define mmRCC_PEER_REG_RANGE0_DEFAULT                                            0xffff0000
-#define mmRCC_PEER_REG_RANGE1_DEFAULT                                            0xffff0000
-#define mmRCC_BUS_CNTL_DEFAULT                                                   0x00000000
-#define mmRCC_CONFIG_CNTL_DEFAULT                                                0x00000000
-#define mmRCC_CONFIG_F0_BASE_DEFAULT                                             0x00000000
-#define mmRCC_CONFIG_APER_SIZE_DEFAULT                                           0x00000000
-#define mmRCC_CONFIG_REG_APER_SIZE_DEFAULT                                       0x00000000
-#define mmRCC_XDMA_LO_DEFAULT                                                    0x00000000
-#define mmRCC_XDMA_HI_DEFAULT                                                    0x00000000
-#define mmRCC_FEATURES_CONTROL_MISC_DEFAULT                                      0x00000000
-#define mmRCC_BUSNUM_CNTL1_DEFAULT                                               0x00000000
-#define mmRCC_BUSNUM_LIST0_DEFAULT                                               0x00000000
-#define mmRCC_BUSNUM_LIST1_DEFAULT                                               0x00000000
-#define mmRCC_BUSNUM_CNTL2_DEFAULT                                               0x00000000
-#define mmRCC_CAPTURE_HOST_BUSNUM_DEFAULT                                        0x00000000
-#define mmRCC_HOST_BUSNUM_DEFAULT                                                0x00000000
-#define mmRCC_PEER0_FB_OFFSET_HI_DEFAULT                                         0x00000000
-#define mmRCC_PEER0_FB_OFFSET_LO_DEFAULT                                         0x00000000
-#define mmRCC_PEER1_FB_OFFSET_HI_DEFAULT                                         0x00000000
-#define mmRCC_PEER1_FB_OFFSET_LO_DEFAULT                                         0x00000000
-#define mmRCC_PEER2_FB_OFFSET_HI_DEFAULT                                         0x00000000
-#define mmRCC_PEER2_FB_OFFSET_LO_DEFAULT                                         0x00000000
-#define mmRCC_PEER3_FB_OFFSET_HI_DEFAULT                                         0x00000000
-#define mmRCC_PEER3_FB_OFFSET_LO_DEFAULT                                         0x00000000
-#define mmRCC_DEVFUNCNUM_LIST0_DEFAULT                                           0x00000000
-#define mmRCC_DEVFUNCNUM_LIST1_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_LINK_CNTL_DEFAULT                                             0x00000000
-#define mmRCC_CMN_LINK_CNTL_DEFAULT                                              0x00000000
-#define mmRCC_EP_REQUESTERID_RESTORE_DEFAULT                                     0x00000000
-#define mmRCC_LTR_LSWITCH_CNTL_DEFAULT                                           0x00000000
-#define mmRCC_MH_ARB_CNTL_DEFAULT                                                0x00000000
-
-
-// addressBlock: rcc_pf_0_BIFDEC2
-// base address: 0x0
-#define mmGFXMSIX_VECT0_ADDR_LO_DEFAULT                                          0x00000000
-#define mmGFXMSIX_VECT0_ADDR_HI_DEFAULT                                          0x00000000
-#define mmGFXMSIX_VECT0_MSG_DATA_DEFAULT                                         0x00000000
-#define mmGFXMSIX_VECT0_CONTROL_DEFAULT                                          0x00000001
-#define mmGFXMSIX_VECT1_ADDR_LO_DEFAULT                                          0x00000000
-#define mmGFXMSIX_VECT1_ADDR_HI_DEFAULT                                          0x00000000
-#define mmGFXMSIX_VECT1_MSG_DATA_DEFAULT                                         0x00000000
-#define mmGFXMSIX_VECT1_CONTROL_DEFAULT                                          0x00000001
-#define mmGFXMSIX_VECT2_ADDR_LO_DEFAULT                                          0x00000000
-#define mmGFXMSIX_VECT2_ADDR_HI_DEFAULT                                          0x00000000
-#define mmGFXMSIX_VECT2_MSG_DATA_DEFAULT                                         0x00000000
-#define mmGFXMSIX_VECT2_CONTROL_DEFAULT                                          0x00000001
-#define mmGFXMSIX_PBA_DEFAULT                                                    0x00000000
-
-
-// addressBlock: rcc_strap_BIFDEC1
-// base address: 0x0
-#define mmRCC_DEV0_PORT_STRAP0_DEFAULT                                           0x54228bc0
-#define mmRCC_DEV0_PORT_STRAP1_DEFAULT                                           0x1022145e
-#define mmRCC_DEV0_PORT_STRAP2_DEFAULT                                           0x1c65e009
-#define mmRCC_DEV0_PORT_STRAP3_DEFAULT                                           0x5ffff849
-#define mmRCC_DEV0_PORT_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_PORT_STRAP5_DEFAULT                                           0xaf800000
-#define mmRCC_DEV0_PORT_STRAP6_DEFAULT                                           0x00000002
-#define mmRCC_DEV0_PORT_STRAP7_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF0_STRAP0_DEFAULT                                           0x30000000
-#define mmRCC_DEV0_EPF0_STRAP1_DEFAULT                                           0x05530000
-#define mmRCC_DEV0_EPF0_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF0_STRAP2_DEFAULT                                           0x02000000
-#define mmRCC_DEV0_EPF0_STRAP3_DEFAULT                                           0x08b40001
-#define mmRCC_DEV0_EPF0_STRAP4_DEFAULT                                           0x1f000042
-#define mmRCC_DEV0_EPF0_STRAP5_DEFAULT                                           0x00001022
-#define mmRCC_DEV0_EPF0_STRAP8_DEFAULT                                           0xc8c73002
-#define mmRCC_DEV0_EPF0_STRAP9_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF1_STRAP0_DEFAULT                                           0x30000000
-#define mmRCC_DEV0_EPF1_STRAP10_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF1_STRAP11_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF1_STRAP12_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF1_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF1_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF1_STRAP3_DEFAULT                                           0x08040001
-#define mmRCC_DEV0_EPF1_STRAP4_DEFAULT                                           0x2f000000
-#define mmRCC_DEV0_EPF1_STRAP5_DEFAULT                                           0x00001022
-#define mmRCC_DEV0_EPF1_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF1_STRAP7_DEFAULT                                           0x00000000
-
-
-// addressBlock: bif_bx_pf_BIFPFVFDEC1
-// base address: 0x0
-#define mmBIF_BME_STATUS_DEFAULT                                                 0x00000000
-#define mmBIF_ATOMIC_ERR_LOG_DEFAULT                                             0x00000000
-#define mmDOORBELL_SELFRING_GPA_APER_BASE_HIGH_DEFAULT                           0x00000000
-#define mmDOORBELL_SELFRING_GPA_APER_BASE_LOW_DEFAULT                            0x00000000
-#define mmDOORBELL_SELFRING_GPA_APER_CNTL_DEFAULT                                0x00000000
-#define mmHDP_REG_COHERENCY_FLUSH_CNTL_DEFAULT                                   0x00000000
-#define mmHDP_MEM_COHERENCY_FLUSH_CNTL_DEFAULT                                   0x00000000
-#define mmGPU_HDP_FLUSH_REQ_DEFAULT                                              0x00000000
-#define mmGPU_HDP_FLUSH_DONE_DEFAULT                                             0x00000000
-#define mmBIF_TRANS_PENDING_DEFAULT                                              0x00000000
-#define mmMAILBOX_MSGBUF_TRN_DW0_DEFAULT                                         0x00000000
-#define mmMAILBOX_MSGBUF_TRN_DW1_DEFAULT                                         0x00000000
-#define mmMAILBOX_MSGBUF_TRN_DW2_DEFAULT                                         0x00000000
-#define mmMAILBOX_MSGBUF_TRN_DW3_DEFAULT                                         0x00000000
-#define mmMAILBOX_MSGBUF_RCV_DW0_DEFAULT                                         0x00000000
-#define mmMAILBOX_MSGBUF_RCV_DW1_DEFAULT                                         0x00000000
-#define mmMAILBOX_MSGBUF_RCV_DW2_DEFAULT                                         0x00000000
-#define mmMAILBOX_MSGBUF_RCV_DW3_DEFAULT                                         0x00000000
-#define mmMAILBOX_CONTROL_DEFAULT                                                0x00000000
-#define mmMAILBOX_INT_CNTL_DEFAULT                                               0x00000000
-#define mmBIF_VMHV_MAILBOX_DEFAULT                                               0x00000000
-
-
-// addressBlock: rcc_pf_0_BIFPFVFDEC1
-// base address: 0x0
-#define mmRCC_DOORBELL_APER_EN_DEFAULT                                           0x00000000
-#define mmRCC_CONFIG_MEMSIZE_DEFAULT                                             0x00000000
-#define mmRCC_CONFIG_RESERVED_DEFAULT                                            0x00000000
-#define mmRCC_IOV_FUNC_IDENTIFIER_DEFAULT                                        0x00000000
-
-
-// addressBlock: syshub_mmreg_ind_syshubdec
-// base address: 0x0
-#define mmSYSHUB_INDEX_DEFAULT                                                   0x00000000
-#define mmSYSHUB_DATA_DEFAULT                                                    0x00000000
-
-
-// addressBlock: rcc_strap_rcc_strap_internal
-// base address: 0x10100000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP0_DEFAULT                          0x54228bc0
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP1_DEFAULT                          0x1022145e
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP2_DEFAULT                          0x1c65e009
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP3_DEFAULT                          0x5ffff849
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP4_DEFAULT                          0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP5_DEFAULT                          0xaf800000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP6_DEFAULT                          0x00000002
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_PORT_STRAP7_DEFAULT                          0x00000000
-#define mmRCC_DEV1_PORT_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_PORT_STRAP1_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_PORT_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_PORT_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_PORT_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_PORT_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_PORT_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_PORT_STRAP7_DEFAULT                                           0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP0_DEFAULT                          0x30000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP1_DEFAULT                          0x05530000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP2_DEFAULT                          0x02000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP3_DEFAULT                          0x08b40001
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP4_DEFAULT                          0x1f000042
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP5_DEFAULT                          0x00001022
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP8_DEFAULT                          0xc8c73002
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP9_DEFAULT                          0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF0_STRAP13_DEFAULT                         0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP0_DEFAULT                          0x30000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP2_DEFAULT                          0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP3_DEFAULT                          0x08040001
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP4_DEFAULT                          0x2f000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP5_DEFAULT                          0x00001022
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP6_DEFAULT                          0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP7_DEFAULT                          0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP10_DEFAULT                         0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP11_DEFAULT                         0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP12_DEFAULT                         0x00000000
-#define mmRCCSTRAPRCCSTRAP_RCC_DEV0_EPF1_STRAP13_DEFAULT                         0x00000000
-#define mmRCC_DEV0_EPF2_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF2_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF2_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF2_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF2_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF2_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF2_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF3_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF3_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF3_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF3_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF3_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF3_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF3_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF4_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF4_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF4_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF4_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF4_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF4_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF4_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF5_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF5_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF5_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF5_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF5_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF5_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF5_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF6_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF6_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF6_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF6_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF6_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF6_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF6_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV0_EPF7_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF7_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF7_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF7_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF7_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF7_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV0_EPF7_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV1_EPF0_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF0_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF0_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF0_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF0_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF0_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF0_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV1_EPF1_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF1_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF1_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF1_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF1_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF1_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF1_STRAP13_DEFAULT                                          0x00000000
-#define mmRCC_DEV1_EPF2_STRAP0_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF2_STRAP2_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF2_STRAP3_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF2_STRAP4_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF2_STRAP5_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF2_STRAP6_DEFAULT                                           0x00000000
-#define mmRCC_DEV1_EPF2_STRAP13_DEFAULT                                          0x00000000
-
-
-// addressBlock: bif_rst_bif_rst_regblk
-// base address: 0x10100000
-#define ixHARD_RST_CTRL_DEFAULT                                                  0xb0000055
-#define ixRSMU_SOFT_RST_CTRL_DEFAULT                                             0x90000000
-#define ixSELF_SOFT_RST_DEFAULT                                                  0x00000000
-#define ixGFX_DRV_MODE1_RST_CTRL_DEFAULT                                         0x000000a9
-#define ixBIF_RST_MISC_CTRL_DEFAULT                                              0x00000644
-#define ixBIF_RST_MISC_CTRL2_DEFAULT                                             0x00000000
-#define ixBIF_RST_MISC_CTRL3_DEFAULT                                             0x00004900
-#define ixBIF_RST_GFXVF_FLR_IDLE_DEFAULT                                         0x00000000
-#define ixDEV0_PF0_FLR_RST_CTRL_DEFAULT                                          0x0206a9a9
-#define ixDEV0_PF1_FLR_RST_CTRL_DEFAULT                                          0x02060009
-#define ixDEV0_PF2_FLR_RST_CTRL_DEFAULT                                          0x02060009
-#define ixDEV0_PF3_FLR_RST_CTRL_DEFAULT                                          0x02060009
-#define ixDEV0_PF4_FLR_RST_CTRL_DEFAULT                                          0x02060009
-#define ixDEV0_PF5_FLR_RST_CTRL_DEFAULT                                          0x02060009
-#define ixDEV0_PF6_FLR_RST_CTRL_DEFAULT                                          0x02060009
-#define ixDEV0_PF7_FLR_RST_CTRL_DEFAULT                                          0x02060009
-#define ixBIF_INST_RESET_INTR_STS_DEFAULT                                        0x00000000
-#define ixBIF_PF_FLR_INTR_STS_DEFAULT                                            0x00000000
-#define ixBIF_D3HOTD0_INTR_STS_DEFAULT                                           0x00000000
-#define ixBIF_POWER_INTR_STS_DEFAULT                                             0x00000000
-#define ixBIF_PF_DSTATE_INTR_STS_DEFAULT                                         0x00000000
-#define ixBIF_PF0_VF_FLR_INTR_STS_DEFAULT                                        0x00000000
-#define ixBIF_INST_RESET_INTR_MASK_DEFAULT                                       0x00000000
-#define ixBIF_PF_FLR_INTR_MASK_DEFAULT                                           0x00000000
-#define ixBIF_D3HOTD0_INTR_MASK_DEFAULT                                          0x000000ff
-#define ixBIF_POWER_INTR_MASK_DEFAULT                                            0x00000000
-#define ixBIF_PF_DSTATE_INTR_MASK_DEFAULT                                        0x00000000
-#define ixBIF_PF0_VF_FLR_INTR_MASK_DEFAULT                                       0x00000000
-#define ixBIF_PF_FLR_RST_DEFAULT                                                 0x00000000
-#define ixBIF_PF0_VF_FLR_RST_DEFAULT                                             0x00000000
-#define ixBIF_DEV0_PF0_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixBIF_DEV0_PF1_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixBIF_DEV0_PF2_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixBIF_DEV0_PF3_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixBIF_DEV0_PF4_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixBIF_DEV0_PF5_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixBIF_DEV0_PF6_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixBIF_DEV0_PF7_DSTATE_VALUE_DEFAULT                                      0x00000000
-#define ixDEV0_PF0_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixDEV0_PF1_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixDEV0_PF2_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixDEV0_PF3_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixDEV0_PF4_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixDEV0_PF5_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixDEV0_PF6_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixDEV0_PF7_D3HOTD0_RST_CTRL_DEFAULT                                      0x0000001b
-#define ixBIF_PORT0_DSTATE_VALUE_DEFAULT                                         0x00000000
-
-
-// addressBlock: bif_misc_bif_misc_regblk
-// base address: 0x10100000
-#define ixMISC_SCRATCH_DEFAULT                                                   0x00000000
-#define ixINTR_LINE_POLARITY_DEFAULT                                             0x00000000
-#define ixINTR_LINE_ENABLE_DEFAULT                                               0x00000000
-#define ixOUTSTANDING_VC_ALLOC_DEFAULT                                           0x6f06c0cf
-#define ixBIFC_MISC_CTRL0_DEFAULT                                                0x08000004
-#define ixBIFC_MISC_CTRL1_DEFAULT                                                0x00008004
-#define ixBIFC_BME_ERR_LOG_DEFAULT                                               0x00000000
-#define ixBIFC_RCCBIH_BME_ERR_LOG_DEFAULT                                        0x00000000
-#define ixBIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1_DEFAULT                              0x00000000
-#define ixBIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3_DEFAULT                              0x00000000
-#define ixBIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5_DEFAULT                              0x00000000
-#define ixBIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7_DEFAULT                              0x00000000
-#define ixNBIF_VWIRE_CTRL_DEFAULT                                                0x00000000
-#define ixNBIF_SMN_VWR_VCHG_DIS_CTRL_DEFAULT                                     0x00000000
-#define ixNBIF_SMN_VWR_VCHG_RST_CTRL0_DEFAULT                                    0x00000000
-#define ixNBIF_SMN_VWR_VCHG_TRIG_DEFAULT                                         0x00000000
-#define ixNBIF_SMN_VWR_WTRIG_CNTL_DEFAULT                                        0x00000000
-#define ixNBIF_SMN_VWR_VCHG_DIS_CTRL_1_DEFAULT                                   0x00000000
-#define ixNBIF_MGCG_CTRL_DEFAULT                                                 0x00000080
-#define ixNBIF_DS_CTRL_LCLK_DEFAULT                                              0x01000000
-#define ixSMN_MST_CNTL0_DEFAULT                                                  0x00000001
-#define ixSMN_MST_EP_CNTL1_DEFAULT                                               0x00000000
-#define ixSMN_MST_EP_CNTL2_DEFAULT                                               0x00000000
-#define ixNBIF_SDP_VWR_VCHG_DIS_CTRL_DEFAULT                                     0x00000000
-#define ixNBIF_SDP_VWR_VCHG_RST_CTRL0_DEFAULT                                    0x00000000
-#define ixNBIF_SDP_VWR_VCHG_RST_CTRL1_DEFAULT                                    0x00000000
-#define ixNBIF_SDP_VWR_VCHG_TRIG_DEFAULT                                         0x00000000
-#define ixBME_DUMMY_CNTL_0_DEFAULT                                               0x0000aaaa
-#define ixBIFC_THT_CNTL_DEFAULT                                                  0x00000222
-#define ixBIFC_HSTARB_CNTL_DEFAULT                                               0x00000000
-#define ixBIFC_GSI_CNTL_DEFAULT                                                  0x000017c0
-#define ixBIFC_PCIEFUNC_CNTL_DEFAULT                                             0x00000000
-#define ixBIFC_SDP_CNTL_0_DEFAULT                                                0x003cf3cf
-#define ixBIFC_PERF_CNTL_0_DEFAULT                                               0x00000000
-#define ixBIFC_PERF_CNTL_1_DEFAULT                                               0x00000000
-#define ixBIFC_PERF_CNT_MMIO_RD_DEFAULT                                          0x00000000
-#define ixBIFC_PERF_CNT_MMIO_WR_DEFAULT                                          0x00000000
-#define ixBIFC_PERF_CNT_DMA_RD_DEFAULT                                           0x00000000
-#define ixBIFC_PERF_CNT_DMA_WR_DEFAULT                                           0x00000000
-#define ixNBIF_REGIF_ERRSET_CTRL_DEFAULT                                         0x00000000
-#define ixSMN_MST_EP_CNTL3_DEFAULT                                               0x00000000
-#define ixSMN_MST_EP_CNTL4_DEFAULT                                               0x00000000
-#define ixBIF_SELFRING_BUFFER_VID_DEFAULT                                        0x0000605f
-#define ixBIF_SELFRING_VECTOR_CNTL_DEFAULT                                       0x00000000
-
-
-// addressBlock: bif_ras_bif_ras_regblk
-// base address: 0x10100000
-#define ixBIF_RAS_LEAF0_CTRL_DEFAULT                                             0x00000000
-#define ixBIF_RAS_LEAF1_CTRL_DEFAULT                                             0x00000000
-#define ixBIF_RAS_LEAF2_CTRL_DEFAULT                                             0x00000000
-#define ixBIF_RAS_MISC_CTRL_DEFAULT                                              0x00000000
-#define ixBIF_IOHUB_RAS_IH_CNTL_DEFAULT                                          0x00000000
-#define ixBIF_RAS_VWR_FROM_IOHUB_DEFAULT                                         0x00000000
-
-
-// addressBlock: rcc_pfc_amdgfx_RCCPFCDEC
-// base address: 0x10134000
-#define ixRCC_PFC_LTR_CNTL_DEFAULT                                               0x00000000
-#define ixRCC_PFC_PME_RESTORE_DEFAULT                                            0x00000000
-#define ixRCC_PFC_STICKY_RESTORE_0_DEFAULT                                       0x00000000
-#define ixRCC_PFC_STICKY_RESTORE_1_DEFAULT                                       0x00000000
-#define ixRCC_PFC_STICKY_RESTORE_2_DEFAULT                                       0x00000000
-#define ixRCC_PFC_STICKY_RESTORE_3_DEFAULT                                       0x00000000
-#define ixRCC_PFC_STICKY_RESTORE_4_DEFAULT                                       0x00000000
-#define ixRCC_PFC_STICKY_RESTORE_5_DEFAULT                                       0x00000000
-#define ixRCC_PFC_AUXPWR_CNTL_DEFAULT                                            0x00000000
-
-
-// addressBlock: rcc_pfc_amdgfxaz_RCCPFCDEC
-// base address: 0x10134200
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_LTR_CNTL_DEFAULT                                0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_PME_RESTORE_DEFAULT                             0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_STICKY_RESTORE_0_DEFAULT                        0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_STICKY_RESTORE_1_DEFAULT                        0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_STICKY_RESTORE_2_DEFAULT                        0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_STICKY_RESTORE_3_DEFAULT                        0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_STICKY_RESTORE_4_DEFAULT                        0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_STICKY_RESTORE_5_DEFAULT                        0x00000000
-#define ixRCCPFCAMDGFXAZ_RCC_PFC_AUXPWR_CNTL_DEFAULT                             0x00000000
-
-
-// addressBlock: pciemsix_amdgfx_MSIXTDEC
-// base address: 0x10170000
-#define ixPCIEMSIX_VECT0_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT0_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT0_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT0_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT1_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT1_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT1_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT1_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT2_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT2_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT2_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT2_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT3_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT3_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT3_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT3_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT4_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT4_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT4_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT4_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT5_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT5_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT5_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT5_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT6_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT6_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT6_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT6_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT7_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT7_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT7_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT7_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT8_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT8_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT8_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT8_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT9_ADDR_LO_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT9_ADDR_HI_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT9_MSG_DATA_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT9_CONTROL_DEFAULT                                         0x00000000
-#define ixPCIEMSIX_VECT10_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT10_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT10_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT10_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT11_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT11_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT11_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT11_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT12_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT12_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT12_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT12_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT13_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT13_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT13_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT13_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT14_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT14_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT14_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT14_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT15_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT15_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT15_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT15_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT16_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT16_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT16_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT16_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT17_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT17_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT17_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT17_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT18_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT18_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT18_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT18_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT19_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT19_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT19_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT19_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT20_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT20_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT20_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT20_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT21_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT21_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT21_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT21_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT22_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT22_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT22_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT22_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT23_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT23_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT23_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT23_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT24_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT24_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT24_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT24_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT25_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT25_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT25_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT25_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT26_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT26_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT26_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT26_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT27_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT27_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT27_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT27_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT28_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT28_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT28_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT28_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT29_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT29_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT29_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT29_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT30_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT30_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT30_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT30_CONTROL_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT31_ADDR_LO_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT31_ADDR_HI_DEFAULT                                        0x00000000
-#define ixPCIEMSIX_VECT31_MSG_DATA_DEFAULT                                       0x00000000
-#define ixPCIEMSIX_VECT31_CONTROL_DEFAULT                                        0x00000000
-
-
-// addressBlock: pciemsix_amdgfx_MSIXPDEC
-// base address: 0x10171000
-#define ixPCIEMSIX_PBA_DEFAULT                                                   0x00000000
-
-
-// addressBlock: syshub_mmreg_ind_syshubind
-// base address: 0x0
-#define ixSYSHUBMMREGIND_SYSHUB_DS_CTRL_SOCCLK_DEFAULT                           0x00000000
-#define ixSYSHUBMMREGIND_SYSHUB_DS_CTRL2_SOCCLK_DEFAULT                          0x00000100
-#define ixSYSHUBMMREGIND_SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK_DEFAULT        0x00000000
-#define ixSYSHUBMMREGIND_SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK_DEFAULT           0x00000000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW0_SYSHUB_QOS_CNTL_DEFAULT                    0x0000001e
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW1_SYSHUB_QOS_CNTL_DEFAULT                    0x0000001e
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW0_CL0_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW0_CL1_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW0_CL2_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW0_CL3_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW0_CL4_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW0_CL5_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW1_CL0_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK0_SW2_CL0_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_SYSHUB_CG_CNTL_DEFAULT                                  0x00082000
-#define ixSYSHUBMMREGIND_SYSHUB_TRANS_IDLE_DEFAULT                               0x00000000
-#define ixSYSHUBMMREGIND_SYSHUB_HP_TIMER_DEFAULT                                 0x00000100
-#define ixSYSHUBMMREGIND_SYSHUB_SCRATCH_DEFAULT                                  0x00000040
-#define ixSYSHUBMMREGIND_SYSHUB_DS_CTRL_SHUBCLK_DEFAULT                          0x00000000
-#define ixSYSHUBMMREGIND_SYSHUB_DS_CTRL2_SHUBCLK_DEFAULT                         0x00000100
-#define ixSYSHUBMMREGIND_SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SHUBCLK_DEFAULT       0x00000000
-#define ixSYSHUBMMREGIND_SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SHUBCLK_DEFAULT          0x00000000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW0_SYSHUB_QOS_CNTL_DEFAULT                    0x0000001e
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW1_SYSHUB_QOS_CNTL_DEFAULT                    0x0000001e
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW0_CL0_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW0_CL1_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW0_CL2_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW0_CL3_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW0_CL4_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW1_CL0_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW1_CL1_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW1_CL2_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW1_CL3_CNTL_DEFAULT                           0x20200000
-#define ixSYSHUBMMREGIND_DMA_CLK1_SW1_CL4_CNTL_DEFAULT                           0x20200000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_default.h
deleted file mode 100644
index 1fddd0f5aaa2..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/OSSSYS/osssys_4_0_default.h
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _osssys_4_0_DEFAULT_HEADER
-#define _osssys_4_0_DEFAULT_HEADER
-
-
-// addressBlock: osssys_osssysdec
-#define mmIH_VMID_0_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_1_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_2_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_3_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_4_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_5_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_6_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_7_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_8_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_9_LUT_DEFAULT                                                  0x00000000
-#define mmIH_VMID_10_LUT_DEFAULT                                                 0x00000000
-#define mmIH_VMID_11_LUT_DEFAULT                                                 0x00000000
-#define mmIH_VMID_12_LUT_DEFAULT                                                 0x00000000
-#define mmIH_VMID_13_LUT_DEFAULT                                                 0x00000000
-#define mmIH_VMID_14_LUT_DEFAULT                                                 0x00000000
-#define mmIH_VMID_15_LUT_DEFAULT                                                 0x00000000
-#define mmIH_VMID_0_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_1_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_2_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_3_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_4_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_5_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_6_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_7_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_8_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_9_LUT_MM_DEFAULT                                               0x00000000
-#define mmIH_VMID_10_LUT_MM_DEFAULT                                              0x00000000
-#define mmIH_VMID_11_LUT_MM_DEFAULT                                              0x00000000
-#define mmIH_VMID_12_LUT_MM_DEFAULT                                              0x00000000
-#define mmIH_VMID_13_LUT_MM_DEFAULT                                              0x00000000
-#define mmIH_VMID_14_LUT_MM_DEFAULT                                              0x00000000
-#define mmIH_VMID_15_LUT_MM_DEFAULT                                              0x00000000
-#define mmIH_COOKIE_0_DEFAULT                                                    0x00000000
-#define mmIH_COOKIE_1_DEFAULT                                                    0x00000000
-#define mmIH_COOKIE_2_DEFAULT                                                    0x00000000
-#define mmIH_COOKIE_3_DEFAULT                                                    0x00000000
-#define mmIH_COOKIE_4_DEFAULT                                                    0x00000000
-#define mmIH_COOKIE_5_DEFAULT                                                    0x00000000
-#define mmIH_COOKIE_6_DEFAULT                                                    0x00000000
-#define mmIH_COOKIE_7_DEFAULT                                                    0x00000000
-#define mmIH_REGISTER_LAST_PART0_DEFAULT                                         0x00000000
-#define mmSEM_REQ_INPUT_0_DEFAULT                                                0x00000000
-#define mmSEM_REQ_INPUT_1_DEFAULT                                                0x00000000
-#define mmSEM_REQ_INPUT_2_DEFAULT                                                0x00000000
-#define mmSEM_REQ_INPUT_3_DEFAULT                                                0x00000000
-#define mmSEM_REGISTER_LAST_PART0_DEFAULT                                        0x00000000
-#define mmIH_RB_CNTL_DEFAULT                                                     0x10610000
-#define mmIH_RB_BASE_DEFAULT                                                     0x00000000
-#define mmIH_RB_BASE_HI_DEFAULT                                                  0x00000000
-#define mmIH_RB_RPTR_DEFAULT                                                     0x00000000
-#define mmIH_RB_WPTR_DEFAULT                                                     0x00000000
-#define mmIH_RB_WPTR_ADDR_HI_DEFAULT                                             0x00000000
-#define mmIH_RB_WPTR_ADDR_LO_DEFAULT                                             0x00000000
-#define mmIH_DOORBELL_RPTR_DEFAULT                                               0x00000000
-#define mmIH_RB_CNTL_RING1_DEFAULT                                               0x10410000
-#define mmIH_RB_BASE_RING1_DEFAULT                                               0x00000000
-#define mmIH_RB_BASE_HI_RING1_DEFAULT                                            0x00000000
-#define mmIH_RB_RPTR_RING1_DEFAULT                                               0x00000000
-#define mmIH_RB_WPTR_RING1_DEFAULT                                               0x00000000
-#define mmIH_DOORBELL_RPTR_RING1_DEFAULT                                         0x00000000
-#define mmIH_RB_CNTL_RING2_DEFAULT                                               0x10410000
-#define mmIH_RB_BASE_RING2_DEFAULT                                               0x00000000
-#define mmIH_RB_BASE_HI_RING2_DEFAULT                                            0x00000000
-#define mmIH_RB_RPTR_RING2_DEFAULT                                               0x00000000
-#define mmIH_RB_WPTR_RING2_DEFAULT                                               0x00000000
-#define mmIH_DOORBELL_RPTR_RING2_DEFAULT                                         0x00000000
-#define mmIH_VERSION_DEFAULT                                                     0x00000400
-#define mmIH_CNTL_DEFAULT                                                        0x01000000
-#define mmIH_CNTL2_DEFAULT                                                       0x000000ff
-#define mmIH_STATUS_DEFAULT                                                      0x00040847
-#define mmIH_PERFMON_CNTL_DEFAULT                                                0x00000000
-#define mmIH_PERFCOUNTER0_RESULT_DEFAULT                                         0x00000000
-#define mmIH_PERFCOUNTER1_RESULT_DEFAULT                                         0x00000000
-#define mmIH_DSM_MATCH_VALUE_BIT_31_0_DEFAULT                                    0x00000000
-#define mmIH_DSM_MATCH_VALUE_BIT_63_32_DEFAULT                                   0x00000000
-#define mmIH_DSM_MATCH_VALUE_BIT_95_64_DEFAULT                                   0x00000000
-#define mmIH_DSM_MATCH_FIELD_CONTROL_DEFAULT                                     0x0000007f
-#define mmIH_DSM_MATCH_DATA_CONTROL_DEFAULT                                      0x0fffffff
-#define mmIH_DSM_MATCH_FCN_ID_DEFAULT                                            0x00000000
-#define mmIH_LIMIT_INT_RATE_CNTL_DEFAULT                                         0x00000000
-#define mmIH_VF_RB_STATUS_DEFAULT                                                0x00000000
-#define mmIH_VF_RB_STATUS2_DEFAULT                                               0x00000000
-#define mmIH_VF_RB1_STATUS_DEFAULT                                               0x00000000
-#define mmIH_VF_RB1_STATUS2_DEFAULT                                              0x00000000
-#define mmIH_VF_RB2_STATUS_DEFAULT                                               0x00000000
-#define mmIH_VF_RB2_STATUS2_DEFAULT                                              0x00000000
-#define mmIH_INT_FLOOD_CNTL_DEFAULT                                              0x00000000
-#define mmIH_RB0_INT_FLOOD_STATUS_DEFAULT                                        0x00000000
-#define mmIH_RB1_INT_FLOOD_STATUS_DEFAULT                                        0x00000000
-#define mmIH_RB2_INT_FLOOD_STATUS_DEFAULT                                        0x00000000
-#define mmIH_INT_FLOOD_STATUS_DEFAULT                                            0x00000000
-#define mmIH_STORM_CLIENT_LIST_CNTL_DEFAULT                                      0x00000000
-#define mmIH_CLK_CTRL_DEFAULT                                                    0x00000000
-#define mmIH_INT_FLAGS_DEFAULT                                                   0x00000000
-#define mmIH_LAST_INT_INFO0_DEFAULT                                              0x00000000
-#define mmIH_LAST_INT_INFO1_DEFAULT                                              0x00000000
-#define mmIH_LAST_INT_INFO2_DEFAULT                                              0x00000000
-#define mmIH_SCRATCH_DEFAULT                                                     0x00000000
-#define mmIH_CLIENT_CREDIT_ERROR_DEFAULT                                         0x00000000
-#define mmIH_GPU_IOV_VIOLATION_LOG_DEFAULT                                       0x00000000
-#define mmIH_COOKIE_REC_VIOLATION_LOG_DEFAULT                                    0x00000000
-#define mmIH_CREDIT_STATUS_DEFAULT                                               0xfffffffe
-#define mmIH_MMHUB_ERROR_DEFAULT                                                 0x00000000
-#define mmIH_REGISTER_LAST_PART2_DEFAULT                                         0x00000000
-#define mmSEM_CLK_CTRL_DEFAULT                                                   0x00000100
-#define mmSEM_UTC_CREDIT_DEFAULT                                                 0x00000510
-#define mmSEM_UTC_CONFIG_DEFAULT                                                 0x00000020
-#define mmSEM_UTCL2_TRAN_EN_LUT_DEFAULT                                          0x800000ff
-#define mmSEM_MCIF_CONFIG_DEFAULT                                                0x00001040
-#define mmSEM_PERFMON_CNTL_DEFAULT                                               0x00000000
-#define mmSEM_PERFCOUNTER0_RESULT_DEFAULT                                        0x00000000
-#define mmSEM_PERFCOUNTER1_RESULT_DEFAULT                                        0x00000000
-#define mmSEM_STATUS_DEFAULT                                                     0x80f90003
-#define mmSEM_MAILBOX_CLIENTCONFIG_DEFAULT                                       0x00fac688
-#define mmSEM_MAILBOX_DEFAULT                                                    0x00000000
-#define mmSEM_MAILBOX_CONTROL_DEFAULT                                            0x00000000
-#define mmSEM_CHICKEN_BITS_DEFAULT                                               0x00084ad6
-#define mmSEM_MAILBOX_CLIENTCONFIG_EXTRA_DEFAULT                                 0x00000008
-#define mmSEM_GPU_IOV_VIOLATION_LOG_DEFAULT                                      0x00000000
-#define mmSEM_OUTSTANDING_THRESHOLD_DEFAULT                                      0x00000010
-#define mmSEM_REGISTER_LAST_PART2_DEFAULT                                        0x00000000
-#define mmIH_ACTIVE_FCN_ID_DEFAULT                                               0x00000000
-#define mmIH_VIRT_RESET_REQ_DEFAULT                                              0x00000000
-#define mmIH_CLIENT_CFG_DEFAULT                                                  0x0000001f
-#define mmIH_CLIENT_CFG_INDEX_DEFAULT                                            0x00000000
-#define mmIH_CLIENT_CFG_DATA_DEFAULT                                             0x00000000
-#define mmIH_CID_REMAP_INDEX_DEFAULT                                             0x00000000
-#define mmIH_CID_REMAP_DATA_DEFAULT                                              0x00000000
-#define mmIH_CHICKEN_DEFAULT                                                     0x00000000
-#define mmIH_MMHUB_CNTL_DEFAULT                                                  0x00000001
-#define mmIH_REGISTER_LAST_PART1_DEFAULT                                         0x00000000
-#define mmSEM_ACTIVE_FCN_ID_DEFAULT                                              0x00000000
-#define mmSEM_VIRT_RESET_REQ_DEFAULT                                             0x00000000
-#define mmSEM_RESP_SDMA0_DEFAULT                                                 0x0004950c
-#define mmSEM_RESP_SDMA1_DEFAULT                                                 0x0004958c
-#define mmSEM_RESP_UVD_DEFAULT                                                   0x0004860c
-#define mmSEM_RESP_VCE_0_DEFAULT                                                 0x0004900c
-#define mmSEM_RESP_ACP_DEFAULT                                                   0x0004870c
-#define mmSEM_RESP_ISP_DEFAULT                                                   0x00000000
-#define mmSEM_RESP_VCE_1_DEFAULT                                                 0x0004908c
-#define mmSEM_RESP_VP8_DEFAULT                                                   0x00000000
-#define mmSEM_RESP_GC_DEFAULT                                                    0x0004858c
-#define mmSEM_CID_REMAP_INDEX_DEFAULT                                            0x00000000
-#define mmSEM_CID_REMAP_DATA_DEFAULT                                             0x00000000
-#define mmSEM_ATOMIC_OP_LUT_DEFAULT                                              0x040a102f
-#define mmSEM_EDC_CONFIG_DEFAULT                                                 0x00000002
-#define mmSEM_CHICKEN_BITS2_DEFAULT                                              0x00000000
-#define mmSEM_MMHUB_CNTL_DEFAULT                                                 0x00000000
-#define mmSEM_REGISTER_LAST_PART1_DEFAULT                                        0x00000000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_default.h
deleted file mode 100644
index afd15bd6a41a..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_default.h
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _sdma0_4_0_DEFAULT_HEADER
-#define _sdma0_4_0_DEFAULT_HEADER
-
-
-// addressBlock: sdma0_sdma0dec
-#define mmSDMA0_UCODE_ADDR_DEFAULT                                               0x00000000
-#define mmSDMA0_UCODE_DATA_DEFAULT                                               0x00000000
-#define mmSDMA0_VM_CNTL_DEFAULT                                                  0x00000000
-#define mmSDMA0_VM_CTX_LO_DEFAULT                                                0x00000000
-#define mmSDMA0_VM_CTX_HI_DEFAULT                                                0x00000000
-#define mmSDMA0_ACTIVE_FCN_ID_DEFAULT                                            0x00000000
-#define mmSDMA0_VM_CTX_CNTL_DEFAULT                                              0x00000000
-#define mmSDMA0_VIRT_RESET_REQ_DEFAULT                                           0x00000000
-#define mmSDMA0_VF_ENABLE_DEFAULT                                                0x00000000
-#define mmSDMA0_CONTEXT_REG_TYPE0_DEFAULT                                        0xfffdf79f
-#define mmSDMA0_CONTEXT_REG_TYPE1_DEFAULT                                        0x003fbcff
-#define mmSDMA0_CONTEXT_REG_TYPE2_DEFAULT                                        0x000003ff
-#define mmSDMA0_CONTEXT_REG_TYPE3_DEFAULT                                        0x00000000
-#define mmSDMA0_PUB_REG_TYPE0_DEFAULT                                            0x3c000000
-#define mmSDMA0_PUB_REG_TYPE1_DEFAULT                                            0x30003882
-#define mmSDMA0_PUB_REG_TYPE2_DEFAULT                                            0x0fc6e880
-#define mmSDMA0_PUB_REG_TYPE3_DEFAULT                                            0x00000000
-#define mmSDMA0_MMHUB_CNTL_DEFAULT                                               0x00000000
-#define mmSDMA0_CONTEXT_GROUP_BOUNDARY_DEFAULT                                   0x00000000
-#define mmSDMA0_POWER_CNTL_DEFAULT                                               0x0003c000
-#define mmSDMA0_CLK_CTRL_DEFAULT                                                 0xff000100
-#define mmSDMA0_CNTL_DEFAULT                                                     0x00000002
-#define mmSDMA0_CHICKEN_BITS_DEFAULT                                             0x00831f07
-#define mmSDMA0_GB_ADDR_CONFIG_DEFAULT                                           0x00100012
-#define mmSDMA0_GB_ADDR_CONFIG_READ_DEFAULT                                      0x00100012
-#define mmSDMA0_RB_RPTR_FETCH_HI_DEFAULT                                         0x00000000
-#define mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL_DEFAULT                                 0x00000000
-#define mmSDMA0_RB_RPTR_FETCH_DEFAULT                                            0x00000000
-#define mmSDMA0_IB_OFFSET_FETCH_DEFAULT                                          0x00000000
-#define mmSDMA0_PROGRAM_DEFAULT                                                  0x00000000
-#define mmSDMA0_STATUS_REG_DEFAULT                                               0x46dee557
-#define mmSDMA0_STATUS1_REG_DEFAULT                                              0x000003ff
-#define mmSDMA0_RD_BURST_CNTL_DEFAULT                                            0x00000003
-#define mmSDMA0_HBM_PAGE_CONFIG_DEFAULT                                          0x00000000
-#define mmSDMA0_UCODE_CHECKSUM_DEFAULT                                           0x00000000
-#define mmSDMA0_F32_CNTL_DEFAULT                                                 0x00000001
-#define mmSDMA0_FREEZE_DEFAULT                                                   0x00000000
-#define mmSDMA0_PHASE0_QUANTUM_DEFAULT                                           0x00010002
-#define mmSDMA0_PHASE1_QUANTUM_DEFAULT                                           0x00010002
-#define mmSDMA_POWER_GATING_DEFAULT                                              0x00000000
-#define mmSDMA_PGFSM_CONFIG_DEFAULT                                              0x00000000
-#define mmSDMA_PGFSM_WRITE_DEFAULT                                               0x00000000
-#define mmSDMA_PGFSM_READ_DEFAULT                                                0x00000000
-#define mmSDMA0_EDC_CONFIG_DEFAULT                                               0x00000002
-#define mmSDMA0_BA_THRESHOLD_DEFAULT                                             0x03ff03ff
-#define mmSDMA0_ID_DEFAULT                                                       0x00000001
-#define mmSDMA0_VERSION_DEFAULT                                                  0x00000400
-#define mmSDMA0_EDC_COUNTER_DEFAULT                                              0x00000000
-#define mmSDMA0_EDC_COUNTER_CLEAR_DEFAULT                                        0x00000000
-#define mmSDMA0_STATUS2_REG_DEFAULT                                              0x00000000
-#define mmSDMA0_ATOMIC_CNTL_DEFAULT                                              0x00000200
-#define mmSDMA0_ATOMIC_PREOP_LO_DEFAULT                                          0x00000000
-#define mmSDMA0_ATOMIC_PREOP_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_UTCL1_CNTL_DEFAULT                                               0xd0003019
-#define mmSDMA0_UTCL1_WATERMK_DEFAULT                                            0xfffbe1fe
-#define mmSDMA0_UTCL1_RD_STATUS_DEFAULT                                          0x201001ff
-#define mmSDMA0_UTCL1_WR_STATUS_DEFAULT                                          0x503001ff
-#define mmSDMA0_UTCL1_INV0_DEFAULT                                               0x00000600
-#define mmSDMA0_UTCL1_INV1_DEFAULT                                               0x00000000
-#define mmSDMA0_UTCL1_INV2_DEFAULT                                               0x00000000
-#define mmSDMA0_UTCL1_RD_XNACK0_DEFAULT                                          0x00000000
-#define mmSDMA0_UTCL1_RD_XNACK1_DEFAULT                                          0x00000000
-#define mmSDMA0_UTCL1_WR_XNACK0_DEFAULT                                          0x00000000
-#define mmSDMA0_UTCL1_WR_XNACK1_DEFAULT                                          0x00000000
-#define mmSDMA0_UTCL1_TIMEOUT_DEFAULT                                            0x00010001
-#define mmSDMA0_UTCL1_PAGE_DEFAULT                                               0x000003e0
-#define mmSDMA0_POWER_CNTL_IDLE_DEFAULT                                          0x06060200
-#define mmSDMA0_RELAX_ORDERING_LUT_DEFAULT                                       0xc0000006
-#define mmSDMA0_CHICKEN_BITS_2_DEFAULT                                           0x00000005
-#define mmSDMA0_STATUS3_REG_DEFAULT                                              0x00100000
-#define mmSDMA0_PHYSICAL_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA0_PHYSICAL_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA0_PHASE2_QUANTUM_DEFAULT                                           0x00010002
-#define mmSDMA0_ERROR_LOG_DEFAULT                                                0x0000000f
-#define mmSDMA0_PUB_DUMMY_REG0_DEFAULT                                           0x00000000
-#define mmSDMA0_PUB_DUMMY_REG1_DEFAULT                                           0x00000000
-#define mmSDMA0_PUB_DUMMY_REG2_DEFAULT                                           0x00000000
-#define mmSDMA0_PUB_DUMMY_REG3_DEFAULT                                           0x00000000
-#define mmSDMA0_F32_COUNTER_DEFAULT                                              0x00000000
-#define mmSDMA0_UNBREAKABLE_DEFAULT                                              0x00000000
-#define mmSDMA0_PERFMON_CNTL_DEFAULT                                             0x000ff7fd
-#define mmSDMA0_PERFCOUNTER0_RESULT_DEFAULT                                      0x00000000
-#define mmSDMA0_PERFCOUNTER1_RESULT_DEFAULT                                      0x00000000
-#define mmSDMA0_PERFCOUNTER_TAG_DELAY_RANGE_DEFAULT                              0x00640000
-#define mmSDMA0_CRD_CNTL_DEFAULT                                                 0x000085c0
-#define mmSDMA0_MMHUB_TRUSTLVL_DEFAULT                                           0x00000000
-#define mmSDMA0_GPU_IOV_VIOLATION_LOG_DEFAULT                                    0x00000000
-#define mmSDMA0_ULV_CNTL_DEFAULT                                                 0x00000000
-#define mmSDMA0_EA_DBIT_ADDR_DATA_DEFAULT                                        0x00000000
-#define mmSDMA0_EA_DBIT_ADDR_INDEX_DEFAULT                                       0x00000000
-#define mmSDMA0_GFX_RB_CNTL_DEFAULT                                              0x00040000
-#define mmSDMA0_GFX_RB_BASE_DEFAULT                                              0x00000000
-#define mmSDMA0_GFX_RB_BASE_HI_DEFAULT                                           0x00000000
-#define mmSDMA0_GFX_RB_RPTR_DEFAULT                                              0x00000000
-#define mmSDMA0_GFX_RB_RPTR_HI_DEFAULT                                           0x00000000
-#define mmSDMA0_GFX_RB_WPTR_DEFAULT                                              0x00000000
-#define mmSDMA0_GFX_RB_WPTR_HI_DEFAULT                                           0x00000000
-#define mmSDMA0_GFX_RB_WPTR_POLL_CNTL_DEFAULT                                    0x00401000
-#define mmSDMA0_GFX_RB_RPTR_ADDR_HI_DEFAULT                                      0x00000000
-#define mmSDMA0_GFX_RB_RPTR_ADDR_LO_DEFAULT                                      0x00000000
-#define mmSDMA0_GFX_IB_CNTL_DEFAULT                                              0x00000100
-#define mmSDMA0_GFX_IB_RPTR_DEFAULT                                              0x00000000
-#define mmSDMA0_GFX_IB_OFFSET_DEFAULT                                            0x00000000
-#define mmSDMA0_GFX_IB_BASE_LO_DEFAULT                                           0x00000000
-#define mmSDMA0_GFX_IB_BASE_HI_DEFAULT                                           0x00000000
-#define mmSDMA0_GFX_IB_SIZE_DEFAULT                                              0x00000000
-#define mmSDMA0_GFX_SKIP_CNTL_DEFAULT                                            0x00000000
-#define mmSDMA0_GFX_CONTEXT_STATUS_DEFAULT                                       0x00000005
-#define mmSDMA0_GFX_DOORBELL_DEFAULT                                             0x00000000
-#define mmSDMA0_GFX_CONTEXT_CNTL_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_STATUS_DEFAULT                                               0x00000000
-#define mmSDMA0_GFX_DOORBELL_LOG_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_WATERMARK_DEFAULT                                            0x00000000
-#define mmSDMA0_GFX_DOORBELL_OFFSET_DEFAULT                                      0x00000000
-#define mmSDMA0_GFX_CSA_ADDR_LO_DEFAULT                                          0x00000000
-#define mmSDMA0_GFX_CSA_ADDR_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_GFX_IB_SUB_REMAIN_DEFAULT                                        0x00000000
-#define mmSDMA0_GFX_PREEMPT_DEFAULT                                              0x00000000
-#define mmSDMA0_GFX_DUMMY_REG_DEFAULT                                            0x0000000f
-#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI_DEFAULT                                 0x00000000
-#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO_DEFAULT                                 0x00000000
-#define mmSDMA0_GFX_RB_AQL_CNTL_DEFAULT                                          0x00004000
-#define mmSDMA0_GFX_MINOR_PTR_UPDATE_DEFAULT                                     0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA0_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA1_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA2_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA3_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA4_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA5_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA6_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA7_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_DATA8_DEFAULT                                         0x00000000
-#define mmSDMA0_GFX_MIDCMD_CNTL_DEFAULT                                          0x00000000
-#define mmSDMA0_PAGE_RB_CNTL_DEFAULT                                             0x00040000
-#define mmSDMA0_PAGE_RB_BASE_DEFAULT                                             0x00000000
-#define mmSDMA0_PAGE_RB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_PAGE_RB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_PAGE_RB_RPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_PAGE_RB_WPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_PAGE_RB_WPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_PAGE_RB_WPTR_POLL_CNTL_DEFAULT                                   0x00401000
-#define mmSDMA0_PAGE_RB_RPTR_ADDR_HI_DEFAULT                                     0x00000000
-#define mmSDMA0_PAGE_RB_RPTR_ADDR_LO_DEFAULT                                     0x00000000
-#define mmSDMA0_PAGE_IB_CNTL_DEFAULT                                             0x00000100
-#define mmSDMA0_PAGE_IB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_PAGE_IB_OFFSET_DEFAULT                                           0x00000000
-#define mmSDMA0_PAGE_IB_BASE_LO_DEFAULT                                          0x00000000
-#define mmSDMA0_PAGE_IB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_PAGE_IB_SIZE_DEFAULT                                             0x00000000
-#define mmSDMA0_PAGE_SKIP_CNTL_DEFAULT                                           0x00000000
-#define mmSDMA0_PAGE_CONTEXT_STATUS_DEFAULT                                      0x00000004
-#define mmSDMA0_PAGE_DOORBELL_DEFAULT                                            0x00000000
-#define mmSDMA0_PAGE_STATUS_DEFAULT                                              0x00000000
-#define mmSDMA0_PAGE_DOORBELL_LOG_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_WATERMARK_DEFAULT                                           0x00000000
-#define mmSDMA0_PAGE_DOORBELL_OFFSET_DEFAULT                                     0x00000000
-#define mmSDMA0_PAGE_CSA_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA0_PAGE_CSA_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA0_PAGE_IB_SUB_REMAIN_DEFAULT                                       0x00000000
-#define mmSDMA0_PAGE_PREEMPT_DEFAULT                                             0x00000000
-#define mmSDMA0_PAGE_DUMMY_REG_DEFAULT                                           0x0000000f
-#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI_DEFAULT                                0x00000000
-#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO_DEFAULT                                0x00000000
-#define mmSDMA0_PAGE_RB_AQL_CNTL_DEFAULT                                         0x00004000
-#define mmSDMA0_PAGE_MINOR_PTR_UPDATE_DEFAULT                                    0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA0_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA1_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA2_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA3_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA4_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA5_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA6_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA7_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_DATA8_DEFAULT                                        0x00000000
-#define mmSDMA0_PAGE_MIDCMD_CNTL_DEFAULT                                         0x00000000
-#define mmSDMA0_RLC0_RB_CNTL_DEFAULT                                             0x00040000
-#define mmSDMA0_RLC0_RB_BASE_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC0_RB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC0_RB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC0_RB_RPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC0_RB_WPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC0_RB_WPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC0_RB_WPTR_POLL_CNTL_DEFAULT                                   0x00401000
-#define mmSDMA0_RLC0_RB_RPTR_ADDR_HI_DEFAULT                                     0x00000000
-#define mmSDMA0_RLC0_RB_RPTR_ADDR_LO_DEFAULT                                     0x00000000
-#define mmSDMA0_RLC0_IB_CNTL_DEFAULT                                             0x00000100
-#define mmSDMA0_RLC0_IB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC0_IB_OFFSET_DEFAULT                                           0x00000000
-#define mmSDMA0_RLC0_IB_BASE_LO_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC0_IB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC0_IB_SIZE_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC0_SKIP_CNTL_DEFAULT                                           0x00000000
-#define mmSDMA0_RLC0_CONTEXT_STATUS_DEFAULT                                      0x00000004
-#define mmSDMA0_RLC0_DOORBELL_DEFAULT                                            0x00000000
-#define mmSDMA0_RLC0_STATUS_DEFAULT                                              0x00000000
-#define mmSDMA0_RLC0_DOORBELL_LOG_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_WATERMARK_DEFAULT                                           0x00000000
-#define mmSDMA0_RLC0_DOORBELL_OFFSET_DEFAULT                                     0x00000000
-#define mmSDMA0_RLC0_CSA_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA0_RLC0_CSA_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA0_RLC0_IB_SUB_REMAIN_DEFAULT                                       0x00000000
-#define mmSDMA0_RLC0_PREEMPT_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC0_DUMMY_REG_DEFAULT                                           0x0000000f
-#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_HI_DEFAULT                                0x00000000
-#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_LO_DEFAULT                                0x00000000
-#define mmSDMA0_RLC0_RB_AQL_CNTL_DEFAULT                                         0x00004000
-#define mmSDMA0_RLC0_MINOR_PTR_UPDATE_DEFAULT                                    0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA0_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA1_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA2_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA3_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA4_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA5_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA6_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA7_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_DATA8_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC0_MIDCMD_CNTL_DEFAULT                                         0x00000000
-#define mmSDMA0_RLC1_RB_CNTL_DEFAULT                                             0x00040000
-#define mmSDMA0_RLC1_RB_BASE_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC1_RB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC1_RB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC1_RB_RPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC1_RB_WPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC1_RB_WPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC1_RB_WPTR_POLL_CNTL_DEFAULT                                   0x00401000
-#define mmSDMA0_RLC1_RB_RPTR_ADDR_HI_DEFAULT                                     0x00000000
-#define mmSDMA0_RLC1_RB_RPTR_ADDR_LO_DEFAULT                                     0x00000000
-#define mmSDMA0_RLC1_IB_CNTL_DEFAULT                                             0x00000100
-#define mmSDMA0_RLC1_IB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC1_IB_OFFSET_DEFAULT                                           0x00000000
-#define mmSDMA0_RLC1_IB_BASE_LO_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC1_IB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA0_RLC1_IB_SIZE_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC1_SKIP_CNTL_DEFAULT                                           0x00000000
-#define mmSDMA0_RLC1_CONTEXT_STATUS_DEFAULT                                      0x00000004
-#define mmSDMA0_RLC1_DOORBELL_DEFAULT                                            0x00000000
-#define mmSDMA0_RLC1_STATUS_DEFAULT                                              0x00000000
-#define mmSDMA0_RLC1_DOORBELL_LOG_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_WATERMARK_DEFAULT                                           0x00000000
-#define mmSDMA0_RLC1_DOORBELL_OFFSET_DEFAULT                                     0x00000000
-#define mmSDMA0_RLC1_CSA_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA0_RLC1_CSA_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA0_RLC1_IB_SUB_REMAIN_DEFAULT                                       0x00000000
-#define mmSDMA0_RLC1_PREEMPT_DEFAULT                                             0x00000000
-#define mmSDMA0_RLC1_DUMMY_REG_DEFAULT                                           0x0000000f
-#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_HI_DEFAULT                                0x00000000
-#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_LO_DEFAULT                                0x00000000
-#define mmSDMA0_RLC1_RB_AQL_CNTL_DEFAULT                                         0x00004000
-#define mmSDMA0_RLC1_MINOR_PTR_UPDATE_DEFAULT                                    0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA0_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA1_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA2_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA3_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA4_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA5_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA6_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA7_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_DATA8_DEFAULT                                        0x00000000
-#define mmSDMA0_RLC1_MIDCMD_CNTL_DEFAULT                                         0x00000000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_offset.h
deleted file mode 100644
index b100c4e5f1ca..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_offset.h
+++ /dev/null
@@ -1,547 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _sdma0_4_0_OFFSET_HEADER
-#define _sdma0_4_0_OFFSET_HEADER
-
-
-
-// addressBlock: sdma0_sdma0dec
-// base address: 0x4980
-#define mmSDMA0_UCODE_ADDR                                                                             0x0000
-#define mmSDMA0_UCODE_ADDR_BASE_IDX                                                                    0
-#define mmSDMA0_UCODE_DATA                                                                             0x0001
-#define mmSDMA0_UCODE_DATA_BASE_IDX                                                                    0
-#define mmSDMA0_VM_CNTL                                                                                0x0004
-#define mmSDMA0_VM_CNTL_BASE_IDX                                                                       0
-#define mmSDMA0_VM_CTX_LO                                                                              0x0005
-#define mmSDMA0_VM_CTX_LO_BASE_IDX                                                                     0
-#define mmSDMA0_VM_CTX_HI                                                                              0x0006
-#define mmSDMA0_VM_CTX_HI_BASE_IDX                                                                     0
-#define mmSDMA0_ACTIVE_FCN_ID                                                                          0x0007
-#define mmSDMA0_ACTIVE_FCN_ID_BASE_IDX                                                                 0
-#define mmSDMA0_VM_CTX_CNTL                                                                            0x0008
-#define mmSDMA0_VM_CTX_CNTL_BASE_IDX                                                                   0
-#define mmSDMA0_VIRT_RESET_REQ                                                                         0x0009
-#define mmSDMA0_VIRT_RESET_REQ_BASE_IDX                                                                0
-#define mmSDMA0_VF_ENABLE                                                                              0x000a
-#define mmSDMA0_VF_ENABLE_BASE_IDX                                                                     0
-#define mmSDMA0_CONTEXT_REG_TYPE0                                                                      0x000b
-#define mmSDMA0_CONTEXT_REG_TYPE0_BASE_IDX                                                             0
-#define mmSDMA0_CONTEXT_REG_TYPE1                                                                      0x000c
-#define mmSDMA0_CONTEXT_REG_TYPE1_BASE_IDX                                                             0
-#define mmSDMA0_CONTEXT_REG_TYPE2                                                                      0x000d
-#define mmSDMA0_CONTEXT_REG_TYPE2_BASE_IDX                                                             0
-#define mmSDMA0_CONTEXT_REG_TYPE3                                                                      0x000e
-#define mmSDMA0_CONTEXT_REG_TYPE3_BASE_IDX                                                             0
-#define mmSDMA0_PUB_REG_TYPE0                                                                          0x000f
-#define mmSDMA0_PUB_REG_TYPE0_BASE_IDX                                                                 0
-#define mmSDMA0_PUB_REG_TYPE1                                                                          0x0010
-#define mmSDMA0_PUB_REG_TYPE1_BASE_IDX                                                                 0
-#define mmSDMA0_PUB_REG_TYPE2                                                                          0x0011
-#define mmSDMA0_PUB_REG_TYPE2_BASE_IDX                                                                 0
-#define mmSDMA0_PUB_REG_TYPE3                                                                          0x0012
-#define mmSDMA0_PUB_REG_TYPE3_BASE_IDX                                                                 0
-#define mmSDMA0_MMHUB_CNTL                                                                             0x0013
-#define mmSDMA0_MMHUB_CNTL_BASE_IDX                                                                    0
-#define mmSDMA0_CONTEXT_GROUP_BOUNDARY                                                                 0x0019
-#define mmSDMA0_CONTEXT_GROUP_BOUNDARY_BASE_IDX                                                        0
-#define mmSDMA0_POWER_CNTL                                                                             0x001a
-#define mmSDMA0_POWER_CNTL_BASE_IDX                                                                    0
-#define mmSDMA0_CLK_CTRL                                                                               0x001b
-#define mmSDMA0_CLK_CTRL_BASE_IDX                                                                      0
-#define mmSDMA0_CNTL                                                                                   0x001c
-#define mmSDMA0_CNTL_BASE_IDX                                                                          0
-#define mmSDMA0_CHICKEN_BITS                                                                           0x001d
-#define mmSDMA0_CHICKEN_BITS_BASE_IDX                                                                  0
-#define mmSDMA0_GB_ADDR_CONFIG                                                                         0x001e
-#define mmSDMA0_GB_ADDR_CONFIG_BASE_IDX                                                                0
-#define mmSDMA0_GB_ADDR_CONFIG_READ                                                                    0x001f
-#define mmSDMA0_GB_ADDR_CONFIG_READ_BASE_IDX                                                           0
-#define mmSDMA0_RB_RPTR_FETCH_HI                                                                       0x0020
-#define mmSDMA0_RB_RPTR_FETCH_HI_BASE_IDX                                                              0
-#define mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL                                                               0x0021
-#define mmSDMA0_SEM_WAIT_FAIL_TIMER_CNTL_BASE_IDX                                                      0
-#define mmSDMA0_RB_RPTR_FETCH                                                                          0x0022
-#define mmSDMA0_RB_RPTR_FETCH_BASE_IDX                                                                 0
-#define mmSDMA0_IB_OFFSET_FETCH                                                                        0x0023
-#define mmSDMA0_IB_OFFSET_FETCH_BASE_IDX                                                               0
-#define mmSDMA0_PROGRAM                                                                                0x0024
-#define mmSDMA0_PROGRAM_BASE_IDX                                                                       0
-#define mmSDMA0_STATUS_REG                                                                             0x0025
-#define mmSDMA0_STATUS_REG_BASE_IDX                                                                    0
-#define mmSDMA0_STATUS1_REG                                                                            0x0026
-#define mmSDMA0_STATUS1_REG_BASE_IDX                                                                   0
-#define mmSDMA0_RD_BURST_CNTL                                                                          0x0027
-#define mmSDMA0_RD_BURST_CNTL_BASE_IDX                                                                 0
-#define mmSDMA0_HBM_PAGE_CONFIG                                                                        0x0028
-#define mmSDMA0_HBM_PAGE_CONFIG_BASE_IDX                                                               0
-#define mmSDMA0_UCODE_CHECKSUM                                                                         0x0029
-#define mmSDMA0_UCODE_CHECKSUM_BASE_IDX                                                                0
-#define mmSDMA0_F32_CNTL                                                                               0x002a
-#define mmSDMA0_F32_CNTL_BASE_IDX                                                                      0
-#define mmSDMA0_FREEZE                                                                                 0x002b
-#define mmSDMA0_FREEZE_BASE_IDX                                                                        0
-#define mmSDMA0_PHASE0_QUANTUM                                                                         0x002c
-#define mmSDMA0_PHASE0_QUANTUM_BASE_IDX                                                                0
-#define mmSDMA0_PHASE1_QUANTUM                                                                         0x002d
-#define mmSDMA0_PHASE1_QUANTUM_BASE_IDX                                                                0
-#define mmSDMA_POWER_GATING                                                                            0x002e
-#define mmSDMA_POWER_GATING_BASE_IDX                                                                   0
-#define mmSDMA_PGFSM_CONFIG                                                                            0x002f
-#define mmSDMA_PGFSM_CONFIG_BASE_IDX                                                                   0
-#define mmSDMA_PGFSM_WRITE                                                                             0x0030
-#define mmSDMA_PGFSM_WRITE_BASE_IDX                                                                    0
-#define mmSDMA_PGFSM_READ                                                                              0x0031
-#define mmSDMA_PGFSM_READ_BASE_IDX                                                                     0
-#define mmSDMA0_EDC_CONFIG                                                                             0x0032
-#define mmSDMA0_EDC_CONFIG_BASE_IDX                                                                    0
-#define mmSDMA0_BA_THRESHOLD                                                                           0x0033
-#define mmSDMA0_BA_THRESHOLD_BASE_IDX                                                                  0
-#define mmSDMA0_ID                                                                                     0x0034
-#define mmSDMA0_ID_BASE_IDX                                                                            0
-#define mmSDMA0_VERSION                                                                                0x0035
-#define mmSDMA0_VERSION_BASE_IDX                                                                       0
-#define mmSDMA0_EDC_COUNTER                                                                            0x0036
-#define mmSDMA0_EDC_COUNTER_BASE_IDX                                                                   0
-#define mmSDMA0_EDC_COUNTER_CLEAR                                                                      0x0037
-#define mmSDMA0_EDC_COUNTER_CLEAR_BASE_IDX                                                             0
-#define mmSDMA0_STATUS2_REG                                                                            0x0038
-#define mmSDMA0_STATUS2_REG_BASE_IDX                                                                   0
-#define mmSDMA0_ATOMIC_CNTL                                                                            0x0039
-#define mmSDMA0_ATOMIC_CNTL_BASE_IDX                                                                   0
-#define mmSDMA0_ATOMIC_PREOP_LO                                                                        0x003a
-#define mmSDMA0_ATOMIC_PREOP_LO_BASE_IDX                                                               0
-#define mmSDMA0_ATOMIC_PREOP_HI                                                                        0x003b
-#define mmSDMA0_ATOMIC_PREOP_HI_BASE_IDX                                                               0
-#define mmSDMA0_UTCL1_CNTL                                                                             0x003c
-#define mmSDMA0_UTCL1_CNTL_BASE_IDX                                                                    0
-#define mmSDMA0_UTCL1_WATERMK                                                                          0x003d
-#define mmSDMA0_UTCL1_WATERMK_BASE_IDX                                                                 0
-#define mmSDMA0_UTCL1_RD_STATUS                                                                        0x003e
-#define mmSDMA0_UTCL1_RD_STATUS_BASE_IDX                                                               0
-#define mmSDMA0_UTCL1_WR_STATUS                                                                        0x003f
-#define mmSDMA0_UTCL1_WR_STATUS_BASE_IDX                                                               0
-#define mmSDMA0_UTCL1_INV0                                                                             0x0040
-#define mmSDMA0_UTCL1_INV0_BASE_IDX                                                                    0
-#define mmSDMA0_UTCL1_INV1                                                                             0x0041
-#define mmSDMA0_UTCL1_INV1_BASE_IDX                                                                    0
-#define mmSDMA0_UTCL1_INV2                                                                             0x0042
-#define mmSDMA0_UTCL1_INV2_BASE_IDX                                                                    0
-#define mmSDMA0_UTCL1_RD_XNACK0                                                                        0x0043
-#define mmSDMA0_UTCL1_RD_XNACK0_BASE_IDX                                                               0
-#define mmSDMA0_UTCL1_RD_XNACK1                                                                        0x0044
-#define mmSDMA0_UTCL1_RD_XNACK1_BASE_IDX                                                               0
-#define mmSDMA0_UTCL1_WR_XNACK0                                                                        0x0045
-#define mmSDMA0_UTCL1_WR_XNACK0_BASE_IDX                                                               0
-#define mmSDMA0_UTCL1_WR_XNACK1                                                                        0x0046
-#define mmSDMA0_UTCL1_WR_XNACK1_BASE_IDX                                                               0
-#define mmSDMA0_UTCL1_TIMEOUT                                                                          0x0047
-#define mmSDMA0_UTCL1_TIMEOUT_BASE_IDX                                                                 0
-#define mmSDMA0_UTCL1_PAGE                                                                             0x0048
-#define mmSDMA0_UTCL1_PAGE_BASE_IDX                                                                    0
-#define mmSDMA0_POWER_CNTL_IDLE                                                                        0x0049
-#define mmSDMA0_POWER_CNTL_IDLE_BASE_IDX                                                               0
-#define mmSDMA0_RELAX_ORDERING_LUT                                                                     0x004a
-#define mmSDMA0_RELAX_ORDERING_LUT_BASE_IDX                                                            0
-#define mmSDMA0_CHICKEN_BITS_2                                                                         0x004b
-#define mmSDMA0_CHICKEN_BITS_2_BASE_IDX                                                                0
-#define mmSDMA0_STATUS3_REG                                                                            0x004c
-#define mmSDMA0_STATUS3_REG_BASE_IDX                                                                   0
-#define mmSDMA0_PHYSICAL_ADDR_LO                                                                       0x004d
-#define mmSDMA0_PHYSICAL_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA0_PHYSICAL_ADDR_HI                                                                       0x004e
-#define mmSDMA0_PHYSICAL_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA0_PHASE2_QUANTUM                                                                         0x004f
-#define mmSDMA0_PHASE2_QUANTUM_BASE_IDX                                                                0
-#define mmSDMA0_ERROR_LOG                                                                              0x0050
-#define mmSDMA0_ERROR_LOG_BASE_IDX                                                                     0
-#define mmSDMA0_PUB_DUMMY_REG0                                                                         0x0051
-#define mmSDMA0_PUB_DUMMY_REG0_BASE_IDX                                                                0
-#define mmSDMA0_PUB_DUMMY_REG1                                                                         0x0052
-#define mmSDMA0_PUB_DUMMY_REG1_BASE_IDX                                                                0
-#define mmSDMA0_PUB_DUMMY_REG2                                                                         0x0053
-#define mmSDMA0_PUB_DUMMY_REG2_BASE_IDX                                                                0
-#define mmSDMA0_PUB_DUMMY_REG3                                                                         0x0054
-#define mmSDMA0_PUB_DUMMY_REG3_BASE_IDX                                                                0
-#define mmSDMA0_F32_COUNTER                                                                            0x0055
-#define mmSDMA0_F32_COUNTER_BASE_IDX                                                                   0
-#define mmSDMA0_UNBREAKABLE                                                                            0x0056
-#define mmSDMA0_UNBREAKABLE_BASE_IDX                                                                   0
-#define mmSDMA0_PERFMON_CNTL                                                                           0x0057
-#define mmSDMA0_PERFMON_CNTL_BASE_IDX                                                                  0
-#define mmSDMA0_PERFCOUNTER0_RESULT                                                                    0x0058
-#define mmSDMA0_PERFCOUNTER0_RESULT_BASE_IDX                                                           0
-#define mmSDMA0_PERFCOUNTER1_RESULT                                                                    0x0059
-#define mmSDMA0_PERFCOUNTER1_RESULT_BASE_IDX                                                           0
-#define mmSDMA0_PERFCOUNTER_TAG_DELAY_RANGE                                                            0x005a
-#define mmSDMA0_PERFCOUNTER_TAG_DELAY_RANGE_BASE_IDX                                                   0
-#define mmSDMA0_CRD_CNTL                                                                               0x005b
-#define mmSDMA0_CRD_CNTL_BASE_IDX                                                                      0
-#define mmSDMA0_MMHUB_TRUSTLVL                                                                         0x005c
-#define mmSDMA0_MMHUB_TRUSTLVL_BASE_IDX                                                                0
-#define mmSDMA0_GPU_IOV_VIOLATION_LOG                                                                  0x005d
-#define mmSDMA0_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                         0
-#define mmSDMA0_ULV_CNTL                                                                               0x005e
-#define mmSDMA0_ULV_CNTL_BASE_IDX                                                                      0
-#define mmSDMA0_EA_DBIT_ADDR_DATA                                                                      0x0060
-#define mmSDMA0_EA_DBIT_ADDR_DATA_BASE_IDX                                                             0
-#define mmSDMA0_EA_DBIT_ADDR_INDEX                                                                     0x0061
-#define mmSDMA0_EA_DBIT_ADDR_INDEX_BASE_IDX                                                            0
-#define mmSDMA0_GFX_RB_CNTL                                                                            0x0080
-#define mmSDMA0_GFX_RB_CNTL_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_RB_BASE                                                                            0x0081
-#define mmSDMA0_GFX_RB_BASE_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_RB_BASE_HI                                                                         0x0082
-#define mmSDMA0_GFX_RB_BASE_HI_BASE_IDX                                                                0
-#define mmSDMA0_GFX_RB_RPTR                                                                            0x0083
-#define mmSDMA0_GFX_RB_RPTR_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_RB_RPTR_HI                                                                         0x0084
-#define mmSDMA0_GFX_RB_RPTR_HI_BASE_IDX                                                                0
-#define mmSDMA0_GFX_RB_WPTR                                                                            0x0085
-#define mmSDMA0_GFX_RB_WPTR_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_RB_WPTR_HI                                                                         0x0086
-#define mmSDMA0_GFX_RB_WPTR_HI_BASE_IDX                                                                0
-#define mmSDMA0_GFX_RB_WPTR_POLL_CNTL                                                                  0x0087
-#define mmSDMA0_GFX_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
-#define mmSDMA0_GFX_RB_RPTR_ADDR_HI                                                                    0x0088
-#define mmSDMA0_GFX_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
-#define mmSDMA0_GFX_RB_RPTR_ADDR_LO                                                                    0x0089
-#define mmSDMA0_GFX_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
-#define mmSDMA0_GFX_IB_CNTL                                                                            0x008a
-#define mmSDMA0_GFX_IB_CNTL_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_IB_RPTR                                                                            0x008b
-#define mmSDMA0_GFX_IB_RPTR_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_IB_OFFSET                                                                          0x008c
-#define mmSDMA0_GFX_IB_OFFSET_BASE_IDX                                                                 0
-#define mmSDMA0_GFX_IB_BASE_LO                                                                         0x008d
-#define mmSDMA0_GFX_IB_BASE_LO_BASE_IDX                                                                0
-#define mmSDMA0_GFX_IB_BASE_HI                                                                         0x008e
-#define mmSDMA0_GFX_IB_BASE_HI_BASE_IDX                                                                0
-#define mmSDMA0_GFX_IB_SIZE                                                                            0x008f
-#define mmSDMA0_GFX_IB_SIZE_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_SKIP_CNTL                                                                          0x0090
-#define mmSDMA0_GFX_SKIP_CNTL_BASE_IDX                                                                 0
-#define mmSDMA0_GFX_CONTEXT_STATUS                                                                     0x0091
-#define mmSDMA0_GFX_CONTEXT_STATUS_BASE_IDX                                                            0
-#define mmSDMA0_GFX_DOORBELL                                                                           0x0092
-#define mmSDMA0_GFX_DOORBELL_BASE_IDX                                                                  0
-#define mmSDMA0_GFX_CONTEXT_CNTL                                                                       0x0093
-#define mmSDMA0_GFX_CONTEXT_CNTL_BASE_IDX                                                              0
-#define mmSDMA0_GFX_STATUS                                                                             0x00a8
-#define mmSDMA0_GFX_STATUS_BASE_IDX                                                                    0
-#define mmSDMA0_GFX_DOORBELL_LOG                                                                       0x00a9
-#define mmSDMA0_GFX_DOORBELL_LOG_BASE_IDX                                                              0
-#define mmSDMA0_GFX_WATERMARK                                                                          0x00aa
-#define mmSDMA0_GFX_WATERMARK_BASE_IDX                                                                 0
-#define mmSDMA0_GFX_DOORBELL_OFFSET                                                                    0x00ab
-#define mmSDMA0_GFX_DOORBELL_OFFSET_BASE_IDX                                                           0
-#define mmSDMA0_GFX_CSA_ADDR_LO                                                                        0x00ac
-#define mmSDMA0_GFX_CSA_ADDR_LO_BASE_IDX                                                               0
-#define mmSDMA0_GFX_CSA_ADDR_HI                                                                        0x00ad
-#define mmSDMA0_GFX_CSA_ADDR_HI_BASE_IDX                                                               0
-#define mmSDMA0_GFX_IB_SUB_REMAIN                                                                      0x00af
-#define mmSDMA0_GFX_IB_SUB_REMAIN_BASE_IDX                                                             0
-#define mmSDMA0_GFX_PREEMPT                                                                            0x00b0
-#define mmSDMA0_GFX_PREEMPT_BASE_IDX                                                                   0
-#define mmSDMA0_GFX_DUMMY_REG                                                                          0x00b1
-#define mmSDMA0_GFX_DUMMY_REG_BASE_IDX                                                                 0
-#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI                                                               0x00b2
-#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
-#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO                                                               0x00b3
-#define mmSDMA0_GFX_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
-#define mmSDMA0_GFX_RB_AQL_CNTL                                                                        0x00b4
-#define mmSDMA0_GFX_RB_AQL_CNTL_BASE_IDX                                                               0
-#define mmSDMA0_GFX_MINOR_PTR_UPDATE                                                                   0x00b5
-#define mmSDMA0_GFX_MINOR_PTR_UPDATE_BASE_IDX                                                          0
-#define mmSDMA0_GFX_MIDCMD_DATA0                                                                       0x00c0
-#define mmSDMA0_GFX_MIDCMD_DATA0_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA1                                                                       0x00c1
-#define mmSDMA0_GFX_MIDCMD_DATA1_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA2                                                                       0x00c2
-#define mmSDMA0_GFX_MIDCMD_DATA2_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA3                                                                       0x00c3
-#define mmSDMA0_GFX_MIDCMD_DATA3_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA4                                                                       0x00c4
-#define mmSDMA0_GFX_MIDCMD_DATA4_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA5                                                                       0x00c5
-#define mmSDMA0_GFX_MIDCMD_DATA5_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA6                                                                       0x00c6
-#define mmSDMA0_GFX_MIDCMD_DATA6_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA7                                                                       0x00c7
-#define mmSDMA0_GFX_MIDCMD_DATA7_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_DATA8                                                                       0x00c8
-#define mmSDMA0_GFX_MIDCMD_DATA8_BASE_IDX                                                              0
-#define mmSDMA0_GFX_MIDCMD_CNTL                                                                        0x00c9
-#define mmSDMA0_GFX_MIDCMD_CNTL_BASE_IDX                                                               0
-#define mmSDMA0_PAGE_RB_CNTL                                                                           0x00e0
-#define mmSDMA0_PAGE_RB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_RB_BASE                                                                           0x00e1
-#define mmSDMA0_PAGE_RB_BASE_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_RB_BASE_HI                                                                        0x00e2
-#define mmSDMA0_PAGE_RB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA0_PAGE_RB_RPTR                                                                           0x00e3
-#define mmSDMA0_PAGE_RB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_RB_RPTR_HI                                                                        0x00e4
-#define mmSDMA0_PAGE_RB_RPTR_HI_BASE_IDX                                                               0
-#define mmSDMA0_PAGE_RB_WPTR                                                                           0x00e5
-#define mmSDMA0_PAGE_RB_WPTR_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_RB_WPTR_HI                                                                        0x00e6
-#define mmSDMA0_PAGE_RB_WPTR_HI_BASE_IDX                                                               0
-#define mmSDMA0_PAGE_RB_WPTR_POLL_CNTL                                                                 0x00e7
-#define mmSDMA0_PAGE_RB_WPTR_POLL_CNTL_BASE_IDX                                                        0
-#define mmSDMA0_PAGE_RB_RPTR_ADDR_HI                                                                   0x00e8
-#define mmSDMA0_PAGE_RB_RPTR_ADDR_HI_BASE_IDX                                                          0
-#define mmSDMA0_PAGE_RB_RPTR_ADDR_LO                                                                   0x00e9
-#define mmSDMA0_PAGE_RB_RPTR_ADDR_LO_BASE_IDX                                                          0
-#define mmSDMA0_PAGE_IB_CNTL                                                                           0x00ea
-#define mmSDMA0_PAGE_IB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_IB_RPTR                                                                           0x00eb
-#define mmSDMA0_PAGE_IB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_IB_OFFSET                                                                         0x00ec
-#define mmSDMA0_PAGE_IB_OFFSET_BASE_IDX                                                                0
-#define mmSDMA0_PAGE_IB_BASE_LO                                                                        0x00ed
-#define mmSDMA0_PAGE_IB_BASE_LO_BASE_IDX                                                               0
-#define mmSDMA0_PAGE_IB_BASE_HI                                                                        0x00ee
-#define mmSDMA0_PAGE_IB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA0_PAGE_IB_SIZE                                                                           0x00ef
-#define mmSDMA0_PAGE_IB_SIZE_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_SKIP_CNTL                                                                         0x00f0
-#define mmSDMA0_PAGE_SKIP_CNTL_BASE_IDX                                                                0
-#define mmSDMA0_PAGE_CONTEXT_STATUS                                                                    0x00f1
-#define mmSDMA0_PAGE_CONTEXT_STATUS_BASE_IDX                                                           0
-#define mmSDMA0_PAGE_DOORBELL                                                                          0x00f2
-#define mmSDMA0_PAGE_DOORBELL_BASE_IDX                                                                 0
-#define mmSDMA0_PAGE_STATUS                                                                            0x0108
-#define mmSDMA0_PAGE_STATUS_BASE_IDX                                                                   0
-#define mmSDMA0_PAGE_DOORBELL_LOG                                                                      0x0109
-#define mmSDMA0_PAGE_DOORBELL_LOG_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_WATERMARK                                                                         0x010a
-#define mmSDMA0_PAGE_WATERMARK_BASE_IDX                                                                0
-#define mmSDMA0_PAGE_DOORBELL_OFFSET                                                                   0x010b
-#define mmSDMA0_PAGE_DOORBELL_OFFSET_BASE_IDX                                                          0
-#define mmSDMA0_PAGE_CSA_ADDR_LO                                                                       0x010c
-#define mmSDMA0_PAGE_CSA_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA0_PAGE_CSA_ADDR_HI                                                                       0x010d
-#define mmSDMA0_PAGE_CSA_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA0_PAGE_IB_SUB_REMAIN                                                                     0x010f
-#define mmSDMA0_PAGE_IB_SUB_REMAIN_BASE_IDX                                                            0
-#define mmSDMA0_PAGE_PREEMPT                                                                           0x0110
-#define mmSDMA0_PAGE_PREEMPT_BASE_IDX                                                                  0
-#define mmSDMA0_PAGE_DUMMY_REG                                                                         0x0111
-#define mmSDMA0_PAGE_DUMMY_REG_BASE_IDX                                                                0
-#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI                                                              0x0112
-#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                     0
-#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO                                                              0x0113
-#define mmSDMA0_PAGE_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                     0
-#define mmSDMA0_PAGE_RB_AQL_CNTL                                                                       0x0114
-#define mmSDMA0_PAGE_RB_AQL_CNTL_BASE_IDX                                                              0
-#define mmSDMA0_PAGE_MINOR_PTR_UPDATE                                                                  0x0115
-#define mmSDMA0_PAGE_MINOR_PTR_UPDATE_BASE_IDX                                                         0
-#define mmSDMA0_PAGE_MIDCMD_DATA0                                                                      0x0120
-#define mmSDMA0_PAGE_MIDCMD_DATA0_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA1                                                                      0x0121
-#define mmSDMA0_PAGE_MIDCMD_DATA1_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA2                                                                      0x0122
-#define mmSDMA0_PAGE_MIDCMD_DATA2_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA3                                                                      0x0123
-#define mmSDMA0_PAGE_MIDCMD_DATA3_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA4                                                                      0x0124
-#define mmSDMA0_PAGE_MIDCMD_DATA4_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA5                                                                      0x0125
-#define mmSDMA0_PAGE_MIDCMD_DATA5_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA6                                                                      0x0126
-#define mmSDMA0_PAGE_MIDCMD_DATA6_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA7                                                                      0x0127
-#define mmSDMA0_PAGE_MIDCMD_DATA7_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_DATA8                                                                      0x0128
-#define mmSDMA0_PAGE_MIDCMD_DATA8_BASE_IDX                                                             0
-#define mmSDMA0_PAGE_MIDCMD_CNTL                                                                       0x0129
-#define mmSDMA0_PAGE_MIDCMD_CNTL_BASE_IDX                                                              0
-#define mmSDMA0_RLC0_RB_CNTL                                                                           0x0140
-#define mmSDMA0_RLC0_RB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_RB_BASE                                                                           0x0141
-#define mmSDMA0_RLC0_RB_BASE_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_RB_BASE_HI                                                                        0x0142
-#define mmSDMA0_RLC0_RB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC0_RB_RPTR                                                                           0x0143
-#define mmSDMA0_RLC0_RB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_RB_RPTR_HI                                                                        0x0144
-#define mmSDMA0_RLC0_RB_RPTR_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC0_RB_WPTR                                                                           0x0145
-#define mmSDMA0_RLC0_RB_WPTR_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_RB_WPTR_HI                                                                        0x0146
-#define mmSDMA0_RLC0_RB_WPTR_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC0_RB_WPTR_POLL_CNTL                                                                 0x0147
-#define mmSDMA0_RLC0_RB_WPTR_POLL_CNTL_BASE_IDX                                                        0
-#define mmSDMA0_RLC0_RB_RPTR_ADDR_HI                                                                   0x0148
-#define mmSDMA0_RLC0_RB_RPTR_ADDR_HI_BASE_IDX                                                          0
-#define mmSDMA0_RLC0_RB_RPTR_ADDR_LO                                                                   0x0149
-#define mmSDMA0_RLC0_RB_RPTR_ADDR_LO_BASE_IDX                                                          0
-#define mmSDMA0_RLC0_IB_CNTL                                                                           0x014a
-#define mmSDMA0_RLC0_IB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_IB_RPTR                                                                           0x014b
-#define mmSDMA0_RLC0_IB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_IB_OFFSET                                                                         0x014c
-#define mmSDMA0_RLC0_IB_OFFSET_BASE_IDX                                                                0
-#define mmSDMA0_RLC0_IB_BASE_LO                                                                        0x014d
-#define mmSDMA0_RLC0_IB_BASE_LO_BASE_IDX                                                               0
-#define mmSDMA0_RLC0_IB_BASE_HI                                                                        0x014e
-#define mmSDMA0_RLC0_IB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC0_IB_SIZE                                                                           0x014f
-#define mmSDMA0_RLC0_IB_SIZE_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_SKIP_CNTL                                                                         0x0150
-#define mmSDMA0_RLC0_SKIP_CNTL_BASE_IDX                                                                0
-#define mmSDMA0_RLC0_CONTEXT_STATUS                                                                    0x0151
-#define mmSDMA0_RLC0_CONTEXT_STATUS_BASE_IDX                                                           0
-#define mmSDMA0_RLC0_DOORBELL                                                                          0x0152
-#define mmSDMA0_RLC0_DOORBELL_BASE_IDX                                                                 0
-#define mmSDMA0_RLC0_STATUS                                                                            0x0168
-#define mmSDMA0_RLC0_STATUS_BASE_IDX                                                                   0
-#define mmSDMA0_RLC0_DOORBELL_LOG                                                                      0x0169
-#define mmSDMA0_RLC0_DOORBELL_LOG_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_WATERMARK                                                                         0x016a
-#define mmSDMA0_RLC0_WATERMARK_BASE_IDX                                                                0
-#define mmSDMA0_RLC0_DOORBELL_OFFSET                                                                   0x016b
-#define mmSDMA0_RLC0_DOORBELL_OFFSET_BASE_IDX                                                          0
-#define mmSDMA0_RLC0_CSA_ADDR_LO                                                                       0x016c
-#define mmSDMA0_RLC0_CSA_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA0_RLC0_CSA_ADDR_HI                                                                       0x016d
-#define mmSDMA0_RLC0_CSA_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA0_RLC0_IB_SUB_REMAIN                                                                     0x016f
-#define mmSDMA0_RLC0_IB_SUB_REMAIN_BASE_IDX                                                            0
-#define mmSDMA0_RLC0_PREEMPT                                                                           0x0170
-#define mmSDMA0_RLC0_PREEMPT_BASE_IDX                                                                  0
-#define mmSDMA0_RLC0_DUMMY_REG                                                                         0x0171
-#define mmSDMA0_RLC0_DUMMY_REG_BASE_IDX                                                                0
-#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_HI                                                              0x0172
-#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                     0
-#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_LO                                                              0x0173
-#define mmSDMA0_RLC0_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                     0
-#define mmSDMA0_RLC0_RB_AQL_CNTL                                                                       0x0174
-#define mmSDMA0_RLC0_RB_AQL_CNTL_BASE_IDX                                                              0
-#define mmSDMA0_RLC0_MINOR_PTR_UPDATE                                                                  0x0175
-#define mmSDMA0_RLC0_MINOR_PTR_UPDATE_BASE_IDX                                                         0
-#define mmSDMA0_RLC0_MIDCMD_DATA0                                                                      0x0180
-#define mmSDMA0_RLC0_MIDCMD_DATA0_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA1                                                                      0x0181
-#define mmSDMA0_RLC0_MIDCMD_DATA1_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA2                                                                      0x0182
-#define mmSDMA0_RLC0_MIDCMD_DATA2_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA3                                                                      0x0183
-#define mmSDMA0_RLC0_MIDCMD_DATA3_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA4                                                                      0x0184
-#define mmSDMA0_RLC0_MIDCMD_DATA4_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA5                                                                      0x0185
-#define mmSDMA0_RLC0_MIDCMD_DATA5_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA6                                                                      0x0186
-#define mmSDMA0_RLC0_MIDCMD_DATA6_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA7                                                                      0x0187
-#define mmSDMA0_RLC0_MIDCMD_DATA7_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_DATA8                                                                      0x0188
-#define mmSDMA0_RLC0_MIDCMD_DATA8_BASE_IDX                                                             0
-#define mmSDMA0_RLC0_MIDCMD_CNTL                                                                       0x0189
-#define mmSDMA0_RLC0_MIDCMD_CNTL_BASE_IDX                                                              0
-#define mmSDMA0_RLC1_RB_CNTL                                                                           0x01a0
-#define mmSDMA0_RLC1_RB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_RB_BASE                                                                           0x01a1
-#define mmSDMA0_RLC1_RB_BASE_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_RB_BASE_HI                                                                        0x01a2
-#define mmSDMA0_RLC1_RB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC1_RB_RPTR                                                                           0x01a3
-#define mmSDMA0_RLC1_RB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_RB_RPTR_HI                                                                        0x01a4
-#define mmSDMA0_RLC1_RB_RPTR_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC1_RB_WPTR                                                                           0x01a5
-#define mmSDMA0_RLC1_RB_WPTR_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_RB_WPTR_HI                                                                        0x01a6
-#define mmSDMA0_RLC1_RB_WPTR_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC1_RB_WPTR_POLL_CNTL                                                                 0x01a7
-#define mmSDMA0_RLC1_RB_WPTR_POLL_CNTL_BASE_IDX                                                        0
-#define mmSDMA0_RLC1_RB_RPTR_ADDR_HI                                                                   0x01a8
-#define mmSDMA0_RLC1_RB_RPTR_ADDR_HI_BASE_IDX                                                          0
-#define mmSDMA0_RLC1_RB_RPTR_ADDR_LO                                                                   0x01a9
-#define mmSDMA0_RLC1_RB_RPTR_ADDR_LO_BASE_IDX                                                          0
-#define mmSDMA0_RLC1_IB_CNTL                                                                           0x01aa
-#define mmSDMA0_RLC1_IB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_IB_RPTR                                                                           0x01ab
-#define mmSDMA0_RLC1_IB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_IB_OFFSET                                                                         0x01ac
-#define mmSDMA0_RLC1_IB_OFFSET_BASE_IDX                                                                0
-#define mmSDMA0_RLC1_IB_BASE_LO                                                                        0x01ad
-#define mmSDMA0_RLC1_IB_BASE_LO_BASE_IDX                                                               0
-#define mmSDMA0_RLC1_IB_BASE_HI                                                                        0x01ae
-#define mmSDMA0_RLC1_IB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA0_RLC1_IB_SIZE                                                                           0x01af
-#define mmSDMA0_RLC1_IB_SIZE_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_SKIP_CNTL                                                                         0x01b0
-#define mmSDMA0_RLC1_SKIP_CNTL_BASE_IDX                                                                0
-#define mmSDMA0_RLC1_CONTEXT_STATUS                                                                    0x01b1
-#define mmSDMA0_RLC1_CONTEXT_STATUS_BASE_IDX                                                           0
-#define mmSDMA0_RLC1_DOORBELL                                                                          0x01b2
-#define mmSDMA0_RLC1_DOORBELL_BASE_IDX                                                                 0
-#define mmSDMA0_RLC1_STATUS                                                                            0x01c8
-#define mmSDMA0_RLC1_STATUS_BASE_IDX                                                                   0
-#define mmSDMA0_RLC1_DOORBELL_LOG                                                                      0x01c9
-#define mmSDMA0_RLC1_DOORBELL_LOG_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_WATERMARK                                                                         0x01ca
-#define mmSDMA0_RLC1_WATERMARK_BASE_IDX                                                                0
-#define mmSDMA0_RLC1_DOORBELL_OFFSET                                                                   0x01cb
-#define mmSDMA0_RLC1_DOORBELL_OFFSET_BASE_IDX                                                          0
-#define mmSDMA0_RLC1_CSA_ADDR_LO                                                                       0x01cc
-#define mmSDMA0_RLC1_CSA_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA0_RLC1_CSA_ADDR_HI                                                                       0x01cd
-#define mmSDMA0_RLC1_CSA_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA0_RLC1_IB_SUB_REMAIN                                                                     0x01cf
-#define mmSDMA0_RLC1_IB_SUB_REMAIN_BASE_IDX                                                            0
-#define mmSDMA0_RLC1_PREEMPT                                                                           0x01d0
-#define mmSDMA0_RLC1_PREEMPT_BASE_IDX                                                                  0
-#define mmSDMA0_RLC1_DUMMY_REG                                                                         0x01d1
-#define mmSDMA0_RLC1_DUMMY_REG_BASE_IDX                                                                0
-#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_HI                                                              0x01d2
-#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                     0
-#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_LO                                                              0x01d3
-#define mmSDMA0_RLC1_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                     0
-#define mmSDMA0_RLC1_RB_AQL_CNTL                                                                       0x01d4
-#define mmSDMA0_RLC1_RB_AQL_CNTL_BASE_IDX                                                              0
-#define mmSDMA0_RLC1_MINOR_PTR_UPDATE                                                                  0x01d5
-#define mmSDMA0_RLC1_MINOR_PTR_UPDATE_BASE_IDX                                                         0
-#define mmSDMA0_RLC1_MIDCMD_DATA0                                                                      0x01e0
-#define mmSDMA0_RLC1_MIDCMD_DATA0_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA1                                                                      0x01e1
-#define mmSDMA0_RLC1_MIDCMD_DATA1_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA2                                                                      0x01e2
-#define mmSDMA0_RLC1_MIDCMD_DATA2_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA3                                                                      0x01e3
-#define mmSDMA0_RLC1_MIDCMD_DATA3_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA4                                                                      0x01e4
-#define mmSDMA0_RLC1_MIDCMD_DATA4_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA5                                                                      0x01e5
-#define mmSDMA0_RLC1_MIDCMD_DATA5_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA6                                                                      0x01e6
-#define mmSDMA0_RLC1_MIDCMD_DATA6_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA7                                                                      0x01e7
-#define mmSDMA0_RLC1_MIDCMD_DATA7_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_DATA8                                                                      0x01e8
-#define mmSDMA0_RLC1_MIDCMD_DATA8_BASE_IDX                                                             0
-#define mmSDMA0_RLC1_MIDCMD_CNTL                                                                       0x01e9
-#define mmSDMA0_RLC1_MIDCMD_CNTL_BASE_IDX                                                              0
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_sh_mask.h
deleted file mode 100644
index 412ae457f7e0..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA0/sdma0_4_0_sh_mask.h
+++ /dev/null
@@ -1,1852 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _sdma0_4_0_SH_MASK_HEADER
-#define _sdma0_4_0_SH_MASK_HEADER
-
-
-// addressBlock: sdma0_sdma0dec
-//SDMA0_UCODE_ADDR
-#define SDMA0_UCODE_ADDR__VALUE__SHIFT                                                                        0x0
-#define SDMA0_UCODE_ADDR__VALUE_MASK                                                                          0x00001FFFL
-//SDMA0_UCODE_DATA
-#define SDMA0_UCODE_DATA__VALUE__SHIFT                                                                        0x0
-#define SDMA0_UCODE_DATA__VALUE_MASK                                                                          0xFFFFFFFFL
-//SDMA0_VM_CNTL
-#define SDMA0_VM_CNTL__CMD__SHIFT                                                                             0x0
-#define SDMA0_VM_CNTL__CMD_MASK                                                                               0x0000000FL
-//SDMA0_VM_CTX_LO
-#define SDMA0_VM_CTX_LO__ADDR__SHIFT                                                                          0x2
-#define SDMA0_VM_CTX_LO__ADDR_MASK                                                                            0xFFFFFFFCL
-//SDMA0_VM_CTX_HI
-#define SDMA0_VM_CTX_HI__ADDR__SHIFT                                                                          0x0
-#define SDMA0_VM_CTX_HI__ADDR_MASK                                                                            0xFFFFFFFFL
-//SDMA0_ACTIVE_FCN_ID
-#define SDMA0_ACTIVE_FCN_ID__VFID__SHIFT                                                                      0x0
-#define SDMA0_ACTIVE_FCN_ID__RESERVED__SHIFT                                                                  0x4
-#define SDMA0_ACTIVE_FCN_ID__VF__SHIFT                                                                        0x1f
-#define SDMA0_ACTIVE_FCN_ID__VFID_MASK                                                                        0x0000000FL
-#define SDMA0_ACTIVE_FCN_ID__RESERVED_MASK                                                                    0x7FFFFFF0L
-#define SDMA0_ACTIVE_FCN_ID__VF_MASK                                                                          0x80000000L
-//SDMA0_VM_CTX_CNTL
-#define SDMA0_VM_CTX_CNTL__PRIV__SHIFT                                                                        0x0
-#define SDMA0_VM_CTX_CNTL__VMID__SHIFT                                                                        0x4
-#define SDMA0_VM_CTX_CNTL__PRIV_MASK                                                                          0x00000001L
-#define SDMA0_VM_CTX_CNTL__VMID_MASK                                                                          0x000000F0L
-//SDMA0_VIRT_RESET_REQ
-#define SDMA0_VIRT_RESET_REQ__VF__SHIFT                                                                       0x0
-#define SDMA0_VIRT_RESET_REQ__PF__SHIFT                                                                       0x1f
-#define SDMA0_VIRT_RESET_REQ__VF_MASK                                                                         0x0000FFFFL
-#define SDMA0_VIRT_RESET_REQ__PF_MASK                                                                         0x80000000L
-//SDMA0_VF_ENABLE
-#define SDMA0_VF_ENABLE__VF_ENABLE__SHIFT                                                                     0x0
-#define SDMA0_VF_ENABLE__VF_ENABLE_MASK                                                                       0x00000001L
-//SDMA0_CONTEXT_REG_TYPE0
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_CNTL__SHIFT                                                     0x0
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE__SHIFT                                                     0x1
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_HI__SHIFT                                                  0x2
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR__SHIFT                                                     0x3
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_HI__SHIFT                                                  0x4
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR__SHIFT                                                     0x5
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_HI__SHIFT                                                  0x6
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_POLL_CNTL__SHIFT                                           0x7
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_HI__SHIFT                                             0x8
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_LO__SHIFT                                             0x9
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_CNTL__SHIFT                                                     0xa
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_RPTR__SHIFT                                                     0xb
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_OFFSET__SHIFT                                                   0xc
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_LO__SHIFT                                                  0xd
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_HI__SHIFT                                                  0xe
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_SIZE__SHIFT                                                     0xf
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_SKIP_CNTL__SHIFT                                                   0x10
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_STATUS__SHIFT                                              0x11
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_DOORBELL__SHIFT                                                    0x12
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_CNTL__SHIFT                                                0x13
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_CNTL_MASK                                                       0x00000001L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_MASK                                                       0x00000002L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_BASE_HI_MASK                                                    0x00000004L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_MASK                                                       0x00000008L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_HI_MASK                                                    0x00000010L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_MASK                                                       0x00000020L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_HI_MASK                                                    0x00000040L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_WPTR_POLL_CNTL_MASK                                             0x00000080L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_HI_MASK                                               0x00000100L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_RB_RPTR_ADDR_LO_MASK                                               0x00000200L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_CNTL_MASK                                                       0x00000400L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_RPTR_MASK                                                       0x00000800L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_OFFSET_MASK                                                     0x00001000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_LO_MASK                                                    0x00002000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_BASE_HI_MASK                                                    0x00004000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_IB_SIZE_MASK                                                       0x00008000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_SKIP_CNTL_MASK                                                     0x00010000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_STATUS_MASK                                                0x00020000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_DOORBELL_MASK                                                      0x00040000L
-#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_GFX_CONTEXT_CNTL_MASK                                                  0x00080000L
-//SDMA0_CONTEXT_REG_TYPE1
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_STATUS__SHIFT                                                      0x8
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_LOG__SHIFT                                                0x9
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_WATERMARK__SHIFT                                                   0xa
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_OFFSET__SHIFT                                             0xb
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_LO__SHIFT                                                 0xc
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_HI__SHIFT                                                 0xd
-#define SDMA0_CONTEXT_REG_TYPE1__VOID_REG2__SHIFT                                                             0xe
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_IB_SUB_REMAIN__SHIFT                                               0xf
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_PREEMPT__SHIFT                                                     0x10
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DUMMY_REG__SHIFT                                                   0x11
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__SHIFT                                        0x12
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__SHIFT                                        0x13
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_AQL_CNTL__SHIFT                                                 0x14
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_MINOR_PTR_UPDATE__SHIFT                                            0x15
-#define SDMA0_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                              0x16
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_STATUS_MASK                                                        0x00000100L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_LOG_MASK                                                  0x00000200L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_WATERMARK_MASK                                                     0x00000400L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DOORBELL_OFFSET_MASK                                               0x00000800L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_LO_MASK                                                   0x00001000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_CSA_ADDR_HI_MASK                                                   0x00002000L
-#define SDMA0_CONTEXT_REG_TYPE1__VOID_REG2_MASK                                                               0x00004000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_IB_SUB_REMAIN_MASK                                                 0x00008000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_PREEMPT_MASK                                                       0x00010000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_DUMMY_REG_MASK                                                     0x00020000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_HI_MASK                                          0x00040000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_WPTR_POLL_ADDR_LO_MASK                                          0x00080000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_RB_AQL_CNTL_MASK                                                   0x00100000L
-#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_GFX_MINOR_PTR_UPDATE_MASK                                              0x00200000L
-#define SDMA0_CONTEXT_REG_TYPE1__RESERVED_MASK                                                                0xFFC00000L
-//SDMA0_CONTEXT_REG_TYPE2
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA0__SHIFT                                                0x0
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA1__SHIFT                                                0x1
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA2__SHIFT                                                0x2
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA3__SHIFT                                                0x3
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA4__SHIFT                                                0x4
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA5__SHIFT                                                0x5
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA6__SHIFT                                                0x6
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA7__SHIFT                                                0x7
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA8__SHIFT                                                0x8
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_CNTL__SHIFT                                                 0x9
-#define SDMA0_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                              0xa
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA0_MASK                                                  0x00000001L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA1_MASK                                                  0x00000002L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA2_MASK                                                  0x00000004L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA3_MASK                                                  0x00000008L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA4_MASK                                                  0x00000010L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA5_MASK                                                  0x00000020L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA6_MASK                                                  0x00000040L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA7_MASK                                                  0x00000080L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_DATA8_MASK                                                  0x00000100L
-#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_GFX_MIDCMD_CNTL_MASK                                                   0x00000200L
-#define SDMA0_CONTEXT_REG_TYPE2__RESERVED_MASK                                                                0xFFFFFC00L
-//SDMA0_CONTEXT_REG_TYPE3
-#define SDMA0_CONTEXT_REG_TYPE3__RESERVED__SHIFT                                                              0x0
-#define SDMA0_CONTEXT_REG_TYPE3__RESERVED_MASK                                                                0xFFFFFFFFL
-//SDMA0_PUB_REG_TYPE0
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_ADDR__SHIFT                                                          0x0
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_DATA__SHIFT                                                          0x1
-#define SDMA0_PUB_REG_TYPE0__RESERVED3__SHIFT                                                                 0x3
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CNTL__SHIFT                                                             0x4
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_LO__SHIFT                                                           0x5
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_HI__SHIFT                                                           0x6
-#define SDMA0_PUB_REG_TYPE0__SDMA0_ACTIVE_FCN_ID__SHIFT                                                       0x7
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_CNTL__SHIFT                                                         0x8
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VIRT_RESET_REQ__SHIFT                                                      0x9
-#define SDMA0_PUB_REG_TYPE0__RESERVED10__SHIFT                                                                0xa
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE0__SHIFT                                                   0xb
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE1__SHIFT                                                   0xc
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE2__SHIFT                                                   0xd
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE3__SHIFT                                                   0xe
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE0__SHIFT                                                       0xf
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE1__SHIFT                                                       0x10
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE2__SHIFT                                                       0x11
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE3__SHIFT                                                       0x12
-#define SDMA0_PUB_REG_TYPE0__SDMA0_MMHUB_CNTL__SHIFT                                                          0x13
-#define SDMA0_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY__SHIFT                                           0x14
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_GROUP_BOUNDARY__SHIFT                                              0x19
-#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL__SHIFT                                                          0x1a
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CLK_CTRL__SHIFT                                                            0x1b
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL__SHIFT                                                                0x1c
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS__SHIFT                                                        0x1d
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG__SHIFT                                                      0x1e
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ__SHIFT                                                 0x1f
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_ADDR_MASK                                                            0x00000001L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_UCODE_DATA_MASK                                                            0x00000002L
-#define SDMA0_PUB_REG_TYPE0__RESERVED3_MASK                                                                   0x00000008L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CNTL_MASK                                                               0x00000010L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_LO_MASK                                                             0x00000020L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_HI_MASK                                                             0x00000040L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_ACTIVE_FCN_ID_MASK                                                         0x00000080L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VM_CTX_CNTL_MASK                                                           0x00000100L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_VIRT_RESET_REQ_MASK                                                        0x00000200L
-#define SDMA0_PUB_REG_TYPE0__RESERVED10_MASK                                                                  0x00000400L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE0_MASK                                                     0x00000800L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE1_MASK                                                     0x00001000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE2_MASK                                                     0x00002000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_REG_TYPE3_MASK                                                     0x00004000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE0_MASK                                                         0x00008000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE1_MASK                                                         0x00010000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE2_MASK                                                         0x00020000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_PUB_REG_TYPE3_MASK                                                         0x00040000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_MMHUB_CNTL_MASK                                                            0x00080000L
-#define SDMA0_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY_MASK                                             0x01F00000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CONTEXT_GROUP_BOUNDARY_MASK                                                0x02000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL_MASK                                                            0x04000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CLK_CTRL_MASK                                                              0x08000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL_MASK                                                                  0x10000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS_MASK                                                          0x20000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_MASK                                                        0x40000000L
-#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ_MASK                                                   0x80000000L
-//SDMA0_PUB_REG_TYPE1
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI__SHIFT                                                    0x0
-#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT                                            0x1
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH__SHIFT                                                       0x2
-#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH__SHIFT                                                     0x3
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM__SHIFT                                                             0x4
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG__SHIFT                                                          0x5
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG__SHIFT                                                         0x6
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RD_BURST_CNTL__SHIFT                                                       0x7
-#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG__SHIFT                                                     0x8
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM__SHIFT                                                      0x9
-#define SDMA0_PUB_REG_TYPE1__SDMA0_F32_CNTL__SHIFT                                                            0xa
-#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE__SHIFT                                                              0xb
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE0_QUANTUM__SHIFT                                                      0xc
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE1_QUANTUM__SHIFT                                                      0xd
-#define SDMA0_PUB_REG_TYPE1__SDMA_POWER_GATING__SHIFT                                                         0xe
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG__SHIFT                                                         0xf
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_WRITE__SHIFT                                                          0x10
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_READ__SHIFT                                                           0x11
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG__SHIFT                                                          0x12
-#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD__SHIFT                                                        0x13
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ID__SHIFT                                                                  0x14
-#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION__SHIFT                                                             0x15
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER__SHIFT                                                         0x16
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR__SHIFT                                                   0x17
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG__SHIFT                                                         0x18
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL__SHIFT                                                         0x19
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO__SHIFT                                                     0x1a
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI__SHIFT                                                     0x1b
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL__SHIFT                                                          0x1c
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK__SHIFT                                                       0x1d
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_RD_STATUS__SHIFT                                                     0x1e
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WR_STATUS__SHIFT                                                     0x1f
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI_MASK                                                      0x00000001L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL_MASK                                              0x00000002L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_MASK                                                         0x00000004L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH_MASK                                                       0x00000008L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM_MASK                                                               0x00000010L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG_MASK                                                            0x00000020L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG_MASK                                                           0x00000040L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_RD_BURST_CNTL_MASK                                                         0x00000080L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG_MASK                                                       0x00000100L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM_MASK                                                        0x00000200L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_F32_CNTL_MASK                                                              0x00000400L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE_MASK                                                                0x00000800L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE0_QUANTUM_MASK                                                        0x00001000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_PHASE1_QUANTUM_MASK                                                        0x00002000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_POWER_GATING_MASK                                                           0x00004000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG_MASK                                                           0x00008000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_WRITE_MASK                                                            0x00010000L
-#define SDMA0_PUB_REG_TYPE1__SDMA_PGFSM_READ_MASK                                                             0x00020000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG_MASK                                                            0x00040000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD_MASK                                                          0x00080000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ID_MASK                                                                    0x00100000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION_MASK                                                               0x00200000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_MASK                                                           0x00400000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR_MASK                                                     0x00800000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG_MASK                                                           0x01000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL_MASK                                                           0x02000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO_MASK                                                       0x04000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI_MASK                                                       0x08000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL_MASK                                                            0x10000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK_MASK                                                         0x20000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_RD_STATUS_MASK                                                       0x40000000L
-#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WR_STATUS_MASK                                                       0x80000000L
-//SDMA0_PUB_REG_TYPE2
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0__SHIFT                                                          0x0
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1__SHIFT                                                          0x1
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2__SHIFT                                                          0x2
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0__SHIFT                                                     0x3
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1__SHIFT                                                     0x4
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0__SHIFT                                                     0x5
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1__SHIFT                                                     0x6
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_TIMEOUT__SHIFT                                                       0x7
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_PAGE__SHIFT                                                          0x8
-#define SDMA0_PUB_REG_TYPE2__SDMA0_POWER_CNTL_IDLE__SHIFT                                                     0x9
-#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT__SHIFT                                                  0xa
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2__SHIFT                                                      0xb
-#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG__SHIFT                                                         0xc
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO__SHIFT                                                    0xd
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI__SHIFT                                                    0xe
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHASE2_QUANTUM__SHIFT                                                      0xf
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG__SHIFT                                                           0x10
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0__SHIFT                                                      0x11
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1__SHIFT                                                      0x12
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2__SHIFT                                                      0x13
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3__SHIFT                                                      0x14
-#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER__SHIFT                                                         0x15
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UNBREAKABLE__SHIFT                                                         0x16
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFMON_CNTL__SHIFT                                                        0x17
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER0_RESULT__SHIFT                                                 0x18
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER1_RESULT__SHIFT                                                 0x19
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SHIFT                                         0x1a
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL__SHIFT                                                            0x1b
-#define SDMA0_PUB_REG_TYPE2__SDMA0_MMHUB_TRUSTLVL__SHIFT                                                      0x1c
-#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG__SHIFT                                               0x1d
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ULV_CNTL__SHIFT                                                            0x1e
-#define SDMA0_PUB_REG_TYPE2__RESERVED__SHIFT                                                                  0x1f
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0_MASK                                                            0x00000001L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1_MASK                                                            0x00000002L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2_MASK                                                            0x00000004L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0_MASK                                                       0x00000008L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1_MASK                                                       0x00000010L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0_MASK                                                       0x00000020L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1_MASK                                                       0x00000040L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_TIMEOUT_MASK                                                         0x00000080L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_PAGE_MASK                                                            0x00000100L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_POWER_CNTL_IDLE_MASK                                                       0x00000200L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT_MASK                                                    0x00000400L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2_MASK                                                        0x00000800L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG_MASK                                                           0x00001000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO_MASK                                                      0x00002000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI_MASK                                                      0x00004000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PHASE2_QUANTUM_MASK                                                        0x00008000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG_MASK                                                             0x00010000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0_MASK                                                        0x00020000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1_MASK                                                        0x00040000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2_MASK                                                        0x00080000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3_MASK                                                        0x00100000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER_MASK                                                           0x00200000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_UNBREAKABLE_MASK                                                           0x00400000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFMON_CNTL_MASK                                                          0x00800000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER0_RESULT_MASK                                                   0x01000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER1_RESULT_MASK                                                   0x02000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_PERFCOUNTER_TAG_DELAY_RANGE_MASK                                           0x04000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL_MASK                                                              0x08000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_MMHUB_TRUSTLVL_MASK                                                        0x10000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG_MASK                                                 0x20000000L
-#define SDMA0_PUB_REG_TYPE2__SDMA0_ULV_CNTL_MASK                                                              0x40000000L
-#define SDMA0_PUB_REG_TYPE2__RESERVED_MASK                                                                    0x80000000L
-//SDMA0_PUB_REG_TYPE3
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA__SHIFT                                                   0x0
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX__SHIFT                                                  0x1
-#define SDMA0_PUB_REG_TYPE3__RESERVED__SHIFT                                                                  0x2
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA_MASK                                                     0x00000001L
-#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX_MASK                                                    0x00000002L
-#define SDMA0_PUB_REG_TYPE3__RESERVED_MASK                                                                    0xFFFFFFFCL
-//SDMA0_MMHUB_CNTL
-#define SDMA0_MMHUB_CNTL__UNIT_ID__SHIFT                                                                      0x0
-#define SDMA0_MMHUB_CNTL__UNIT_ID_MASK                                                                        0x0000003FL
-//SDMA0_CONTEXT_GROUP_BOUNDARY
-#define SDMA0_CONTEXT_GROUP_BOUNDARY__RESERVED__SHIFT                                                         0x0
-#define SDMA0_CONTEXT_GROUP_BOUNDARY__RESERVED_MASK                                                           0xFFFFFFFFL
-//SDMA0_POWER_CNTL
-#define SDMA0_POWER_CNTL__PG_CNTL_ENABLE__SHIFT                                                               0x0
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_ON_REQ__SHIFT                                                          0x1
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_OFF_REQ__SHIFT                                                         0x2
-#define SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE__SHIFT                                                           0x8
-#define SDMA0_POWER_CNTL__MEM_POWER_LS_EN__SHIFT                                                              0x9
-#define SDMA0_POWER_CNTL__MEM_POWER_DS_EN__SHIFT                                                              0xa
-#define SDMA0_POWER_CNTL__MEM_POWER_SD_EN__SHIFT                                                              0xb
-#define SDMA0_POWER_CNTL__MEM_POWER_DELAY__SHIFT                                                              0xc
-#define SDMA0_POWER_CNTL__PG_CNTL_ENABLE_MASK                                                                 0x00000001L
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_ON_REQ_MASK                                                            0x00000002L
-#define SDMA0_POWER_CNTL__EXT_PG_POWER_OFF_REQ_MASK                                                           0x00000004L
-#define SDMA0_POWER_CNTL__MEM_POWER_OVERRIDE_MASK                                                             0x00000100L
-#define SDMA0_POWER_CNTL__MEM_POWER_LS_EN_MASK                                                                0x00000200L
-#define SDMA0_POWER_CNTL__MEM_POWER_DS_EN_MASK                                                                0x00000400L
-#define SDMA0_POWER_CNTL__MEM_POWER_SD_EN_MASK                                                                0x00000800L
-#define SDMA0_POWER_CNTL__MEM_POWER_DELAY_MASK                                                                0x003FF000L
-//SDMA0_CLK_CTRL
-#define SDMA0_CLK_CTRL__ON_DELAY__SHIFT                                                                       0x0
-#define SDMA0_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                                 0x4
-#define SDMA0_CLK_CTRL__RESERVED__SHIFT                                                                       0xc
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                                 0x18
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                                 0x19
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                                 0x1a
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                                 0x1b
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                                 0x1c
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                                 0x1d
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                                 0x1e
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                                 0x1f
-#define SDMA0_CLK_CTRL__ON_DELAY_MASK                                                                         0x0000000FL
-#define SDMA0_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                   0x00000FF0L
-#define SDMA0_CLK_CTRL__RESERVED_MASK                                                                         0x00FFF000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                   0x01000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                   0x02000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                   0x04000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                   0x08000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                   0x10000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                   0x20000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                                   0x40000000L
-#define SDMA0_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                                   0x80000000L
-//SDMA0_CNTL
-#define SDMA0_CNTL__TRAP_ENABLE__SHIFT                                                                        0x0
-#define SDMA0_CNTL__UTC_L1_ENABLE__SHIFT                                                                      0x1
-#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE__SHIFT                                                                0x2
-#define SDMA0_CNTL__DATA_SWAP_ENABLE__SHIFT                                                                   0x3
-#define SDMA0_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                                  0x4
-#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                              0x5
-#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                          0x11
-#define SDMA0_CNTL__AUTO_CTXSW_ENABLE__SHIFT                                                                  0x12
-#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                                0x1c
-#define SDMA0_CNTL__FROZEN_INT_ENABLE__SHIFT                                                                  0x1d
-#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                              0x1e
-#define SDMA0_CNTL__TRAP_ENABLE_MASK                                                                          0x00000001L
-#define SDMA0_CNTL__UTC_L1_ENABLE_MASK                                                                        0x00000002L
-#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE_MASK                                                                  0x00000004L
-#define SDMA0_CNTL__DATA_SWAP_ENABLE_MASK                                                                     0x00000008L
-#define SDMA0_CNTL__FENCE_SWAP_ENABLE_MASK                                                                    0x00000010L
-#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                                0x00000020L
-#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                            0x00020000L
-#define SDMA0_CNTL__AUTO_CTXSW_ENABLE_MASK                                                                    0x00040000L
-#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                                  0x10000000L
-#define SDMA0_CNTL__FROZEN_INT_ENABLE_MASK                                                                    0x20000000L
-#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                                0x40000000L
-//SDMA0_CHICKEN_BITS
-#define SDMA0_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE__SHIFT                                                     0x0
-#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT                                                 0x1
-#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT                                        0x2
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_LENGTH__SHIFT                                                         0x8
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE__SHIFT                                                     0xa
-#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT                                                        0x10
-#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT                                                           0x11
-#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                         0x14
-#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                           0x17
-#define SDMA0_CHICKEN_BITS__TIME_BASED_QOS__SHIFT                                                             0x19
-#define SDMA0_CHICKEN_BITS__CE_AFIFO_WATERMARK__SHIFT                                                         0x1a
-#define SDMA0_CHICKEN_BITS__CE_DFIFO_WATERMARK__SHIFT                                                         0x1c
-#define SDMA0_CHICKEN_BITS__CE_LFIFO_WATERMARK__SHIFT                                                         0x1e
-#define SDMA0_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE_MASK                                                       0x00000001L
-#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK                                                   0x00000002L
-#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK                                          0x00000004L
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_LENGTH_MASK                                                           0x00000300L
-#define SDMA0_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE_MASK                                                       0x00001C00L
-#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK                                                          0x00010000L
-#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK                                                             0x00020000L
-#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                           0x00100000L
-#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                             0x00800000L
-#define SDMA0_CHICKEN_BITS__TIME_BASED_QOS_MASK                                                               0x02000000L
-#define SDMA0_CHICKEN_BITS__CE_AFIFO_WATERMARK_MASK                                                           0x0C000000L
-#define SDMA0_CHICKEN_BITS__CE_DFIFO_WATERMARK_MASK                                                           0x30000000L
-#define SDMA0_CHICKEN_BITS__CE_LFIFO_WATERMARK_MASK                                                           0xC0000000L
-//SDMA0_GB_ADDR_CONFIG
-#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                0x0
-#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                     0x3
-#define SDMA0_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT                                                     0x8
-#define SDMA0_GB_ADDR_CONFIG__NUM_BANKS__SHIFT                                                                0xc
-#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                       0x13
-#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                  0x00000007L
-#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                       0x00000038L
-#define SDMA0_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK                                                       0x00000700L
-#define SDMA0_GB_ADDR_CONFIG__NUM_BANKS_MASK                                                                  0x00007000L
-#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                         0x00180000L
-//SDMA0_GB_ADDR_CONFIG_READ
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                           0x0
-#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                0x3
-#define SDMA0_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT                                                0x8
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT                                                           0xc
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                  0x13
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                             0x00000007L
-#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                  0x00000038L
-#define SDMA0_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK                                                  0x00000700L
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_BANKS_MASK                                                             0x00007000L
-#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                    0x00180000L
-//SDMA0_RB_RPTR_FETCH_HI
-#define SDMA0_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                                 0x0
-#define SDMA0_RB_RPTR_FETCH_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
-//SDMA0_SEM_WAIT_FAIL_TIMER_CNTL
-#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT                                                          0x0
-#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK                                                            0xFFFFFFFFL
-//SDMA0_RB_RPTR_FETCH
-#define SDMA0_RB_RPTR_FETCH__OFFSET__SHIFT                                                                    0x2
-#define SDMA0_RB_RPTR_FETCH__OFFSET_MASK                                                                      0xFFFFFFFCL
-//SDMA0_IB_OFFSET_FETCH
-#define SDMA0_IB_OFFSET_FETCH__OFFSET__SHIFT                                                                  0x2
-#define SDMA0_IB_OFFSET_FETCH__OFFSET_MASK                                                                    0x003FFFFCL
-//SDMA0_PROGRAM
-#define SDMA0_PROGRAM__STREAM__SHIFT                                                                          0x0
-#define SDMA0_PROGRAM__STREAM_MASK                                                                            0xFFFFFFFFL
-//SDMA0_STATUS_REG
-#define SDMA0_STATUS_REG__IDLE__SHIFT                                                                         0x0
-#define SDMA0_STATUS_REG__REG_IDLE__SHIFT                                                                     0x1
-#define SDMA0_STATUS_REG__RB_EMPTY__SHIFT                                                                     0x2
-#define SDMA0_STATUS_REG__RB_FULL__SHIFT                                                                      0x3
-#define SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT                                                                  0x4
-#define SDMA0_STATUS_REG__RB_CMD_FULL__SHIFT                                                                  0x5
-#define SDMA0_STATUS_REG__IB_CMD_IDLE__SHIFT                                                                  0x6
-#define SDMA0_STATUS_REG__IB_CMD_FULL__SHIFT                                                                  0x7
-#define SDMA0_STATUS_REG__BLOCK_IDLE__SHIFT                                                                   0x8
-#define SDMA0_STATUS_REG__INSIDE_IB__SHIFT                                                                    0x9
-#define SDMA0_STATUS_REG__EX_IDLE__SHIFT                                                                      0xa
-#define SDMA0_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE__SHIFT                                                    0xb
-#define SDMA0_STATUS_REG__PACKET_READY__SHIFT                                                                 0xc
-#define SDMA0_STATUS_REG__MC_WR_IDLE__SHIFT                                                                   0xd
-#define SDMA0_STATUS_REG__SRBM_IDLE__SHIFT                                                                    0xe
-#define SDMA0_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                                0xf
-#define SDMA0_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                              0x10
-#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                              0x11
-#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                              0x12
-#define SDMA0_STATUS_REG__MC_RD_IDLE__SHIFT                                                                   0x13
-#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                             0x14
-#define SDMA0_STATUS_REG__MC_RD_RET_STALL__SHIFT                                                              0x15
-#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT                                                           0x16
-#define SDMA0_STATUS_REG__PREV_CMD_IDLE__SHIFT                                                                0x19
-#define SDMA0_STATUS_REG__SEM_IDLE__SHIFT                                                                     0x1a
-#define SDMA0_STATUS_REG__SEM_REQ_STALL__SHIFT                                                                0x1b
-#define SDMA0_STATUS_REG__SEM_RESP_STATE__SHIFT                                                               0x1c
-#define SDMA0_STATUS_REG__INT_IDLE__SHIFT                                                                     0x1e
-#define SDMA0_STATUS_REG__INT_REQ_STALL__SHIFT                                                                0x1f
-#define SDMA0_STATUS_REG__IDLE_MASK                                                                           0x00000001L
-#define SDMA0_STATUS_REG__REG_IDLE_MASK                                                                       0x00000002L
-#define SDMA0_STATUS_REG__RB_EMPTY_MASK                                                                       0x00000004L
-#define SDMA0_STATUS_REG__RB_FULL_MASK                                                                        0x00000008L
-#define SDMA0_STATUS_REG__RB_CMD_IDLE_MASK                                                                    0x00000010L
-#define SDMA0_STATUS_REG__RB_CMD_FULL_MASK                                                                    0x00000020L
-#define SDMA0_STATUS_REG__IB_CMD_IDLE_MASK                                                                    0x00000040L
-#define SDMA0_STATUS_REG__IB_CMD_FULL_MASK                                                                    0x00000080L
-#define SDMA0_STATUS_REG__BLOCK_IDLE_MASK                                                                     0x00000100L
-#define SDMA0_STATUS_REG__INSIDE_IB_MASK                                                                      0x00000200L
-#define SDMA0_STATUS_REG__EX_IDLE_MASK                                                                        0x00000400L
-#define SDMA0_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE_MASK                                                      0x00000800L
-#define SDMA0_STATUS_REG__PACKET_READY_MASK                                                                   0x00001000L
-#define SDMA0_STATUS_REG__MC_WR_IDLE_MASK                                                                     0x00002000L
-#define SDMA0_STATUS_REG__SRBM_IDLE_MASK                                                                      0x00004000L
-#define SDMA0_STATUS_REG__CONTEXT_EMPTY_MASK                                                                  0x00008000L
-#define SDMA0_STATUS_REG__DELTA_RPTR_FULL_MASK                                                                0x00010000L
-#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                                0x00020000L
-#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                                0x00040000L
-#define SDMA0_STATUS_REG__MC_RD_IDLE_MASK                                                                     0x00080000L
-#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                               0x00100000L
-#define SDMA0_STATUS_REG__MC_RD_RET_STALL_MASK                                                                0x00200000L
-#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK                                                             0x00400000L
-#define SDMA0_STATUS_REG__PREV_CMD_IDLE_MASK                                                                  0x02000000L
-#define SDMA0_STATUS_REG__SEM_IDLE_MASK                                                                       0x04000000L
-#define SDMA0_STATUS_REG__SEM_REQ_STALL_MASK                                                                  0x08000000L
-#define SDMA0_STATUS_REG__SEM_RESP_STATE_MASK                                                                 0x30000000L
-#define SDMA0_STATUS_REG__INT_IDLE_MASK                                                                       0x40000000L
-#define SDMA0_STATUS_REG__INT_REQ_STALL_MASK                                                                  0x80000000L
-//SDMA0_STATUS1_REG
-#define SDMA0_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                                0x0
-#define SDMA0_STATUS1_REG__CE_WR_IDLE__SHIFT                                                                  0x1
-#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE__SHIFT                                                               0x2
-#define SDMA0_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                                0x3
-#define SDMA0_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                                 0x4
-#define SDMA0_STATUS1_REG__CE_IN_IDLE__SHIFT                                                                  0x5
-#define SDMA0_STATUS1_REG__CE_DST_IDLE__SHIFT                                                                 0x6
-#define SDMA0_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                                 0x9
-#define SDMA0_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                               0xa
-#define SDMA0_STATUS1_REG__CE_INFO_FULL__SHIFT                                                                0xd
-#define SDMA0_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                               0xe
-#define SDMA0_STATUS1_REG__EX_START__SHIFT                                                                    0xf
-#define SDMA0_STATUS1_REG__CE_RD_STALL__SHIFT                                                                 0x11
-#define SDMA0_STATUS1_REG__CE_WR_STALL__SHIFT                                                                 0x12
-#define SDMA0_STATUS1_REG__CE_WREQ_IDLE_MASK                                                                  0x00000001L
-#define SDMA0_STATUS1_REG__CE_WR_IDLE_MASK                                                                    0x00000002L
-#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE_MASK                                                                 0x00000004L
-#define SDMA0_STATUS1_REG__CE_RREQ_IDLE_MASK                                                                  0x00000008L
-#define SDMA0_STATUS1_REG__CE_OUT_IDLE_MASK                                                                   0x00000010L
-#define SDMA0_STATUS1_REG__CE_IN_IDLE_MASK                                                                    0x00000020L
-#define SDMA0_STATUS1_REG__CE_DST_IDLE_MASK                                                                   0x00000040L
-#define SDMA0_STATUS1_REG__CE_CMD_IDLE_MASK                                                                   0x00000200L
-#define SDMA0_STATUS1_REG__CE_AFIFO_FULL_MASK                                                                 0x00000400L
-#define SDMA0_STATUS1_REG__CE_INFO_FULL_MASK                                                                  0x00002000L
-#define SDMA0_STATUS1_REG__CE_INFO1_FULL_MASK                                                                 0x00004000L
-#define SDMA0_STATUS1_REG__EX_START_MASK                                                                      0x00008000L
-#define SDMA0_STATUS1_REG__CE_RD_STALL_MASK                                                                   0x00020000L
-#define SDMA0_STATUS1_REG__CE_WR_STALL_MASK                                                                   0x00040000L
-//SDMA0_RD_BURST_CNTL
-#define SDMA0_RD_BURST_CNTL__RD_BURST__SHIFT                                                                  0x0
-#define SDMA0_RD_BURST_CNTL__RD_BURST_MASK                                                                    0x00000003L
-//SDMA0_HBM_PAGE_CONFIG
-#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                      0x0
-#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                        0x00000003L
-//SDMA0_UCODE_CHECKSUM
-#define SDMA0_UCODE_CHECKSUM__DATA__SHIFT                                                                     0x0
-#define SDMA0_UCODE_CHECKSUM__DATA_MASK                                                                       0xFFFFFFFFL
-//SDMA0_F32_CNTL
-#define SDMA0_F32_CNTL__HALT__SHIFT                                                                           0x0
-#define SDMA0_F32_CNTL__STEP__SHIFT                                                                           0x1
-#define SDMA0_F32_CNTL__HALT_MASK                                                                             0x00000001L
-#define SDMA0_F32_CNTL__STEP_MASK                                                                             0x00000002L
-//SDMA0_FREEZE
-#define SDMA0_FREEZE__PREEMPT__SHIFT                                                                          0x0
-#define SDMA0_FREEZE__FREEZE__SHIFT                                                                           0x4
-#define SDMA0_FREEZE__FROZEN__SHIFT                                                                           0x5
-#define SDMA0_FREEZE__F32_FREEZE__SHIFT                                                                       0x6
-#define SDMA0_FREEZE__PREEMPT_MASK                                                                            0x00000001L
-#define SDMA0_FREEZE__FREEZE_MASK                                                                             0x00000010L
-#define SDMA0_FREEZE__FROZEN_MASK                                                                             0x00000020L
-#define SDMA0_FREEZE__F32_FREEZE_MASK                                                                         0x00000040L
-//SDMA0_PHASE0_QUANTUM
-#define SDMA0_PHASE0_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA0_PHASE0_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA0_PHASE0_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA0_PHASE0_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA0_PHASE0_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA0_PHASE0_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA0_PHASE1_QUANTUM
-#define SDMA0_PHASE1_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA0_PHASE1_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA0_PHASE1_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA0_PHASE1_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA0_PHASE1_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA0_PHASE1_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA_POWER_GATING
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_CONDITION__SHIFT                                                   0x0
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_CONDITION__SHIFT                                                    0x1
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_REQ__SHIFT                                                         0x2
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_REQ__SHIFT                                                          0x3
-#define SDMA_POWER_GATING__PG_CNTL_STATUS__SHIFT                                                              0x4
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_CONDITION_MASK                                                     0x00000001L
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_CONDITION_MASK                                                      0x00000002L
-#define SDMA_POWER_GATING__SDMA0_POWER_OFF_REQ_MASK                                                           0x00000004L
-#define SDMA_POWER_GATING__SDMA0_POWER_ON_REQ_MASK                                                            0x00000008L
-#define SDMA_POWER_GATING__PG_CNTL_STATUS_MASK                                                                0x00000030L
-//SDMA_PGFSM_CONFIG
-#define SDMA_PGFSM_CONFIG__FSM_ADDR__SHIFT                                                                    0x0
-#define SDMA_PGFSM_CONFIG__POWER_DOWN__SHIFT                                                                  0x8
-#define SDMA_PGFSM_CONFIG__POWER_UP__SHIFT                                                                    0x9
-#define SDMA_PGFSM_CONFIG__P1_SELECT__SHIFT                                                                   0xa
-#define SDMA_PGFSM_CONFIG__P2_SELECT__SHIFT                                                                   0xb
-#define SDMA_PGFSM_CONFIG__WRITE__SHIFT                                                                       0xc
-#define SDMA_PGFSM_CONFIG__READ__SHIFT                                                                        0xd
-#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE__SHIFT                                                               0x1b
-#define SDMA_PGFSM_CONFIG__REG_ADDR__SHIFT                                                                    0x1c
-#define SDMA_PGFSM_CONFIG__FSM_ADDR_MASK                                                                      0x000000FFL
-#define SDMA_PGFSM_CONFIG__POWER_DOWN_MASK                                                                    0x00000100L
-#define SDMA_PGFSM_CONFIG__POWER_UP_MASK                                                                      0x00000200L
-#define SDMA_PGFSM_CONFIG__P1_SELECT_MASK                                                                     0x00000400L
-#define SDMA_PGFSM_CONFIG__P2_SELECT_MASK                                                                     0x00000800L
-#define SDMA_PGFSM_CONFIG__WRITE_MASK                                                                         0x00001000L
-#define SDMA_PGFSM_CONFIG__READ_MASK                                                                          0x00002000L
-#define SDMA_PGFSM_CONFIG__SRBM_OVERRIDE_MASK                                                                 0x08000000L
-#define SDMA_PGFSM_CONFIG__REG_ADDR_MASK                                                                      0xF0000000L
-//SDMA_PGFSM_WRITE
-#define SDMA_PGFSM_WRITE__VALUE__SHIFT                                                                        0x0
-#define SDMA_PGFSM_WRITE__VALUE_MASK                                                                          0xFFFFFFFFL
-//SDMA_PGFSM_READ
-#define SDMA_PGFSM_READ__VALUE__SHIFT                                                                         0x0
-#define SDMA_PGFSM_READ__VALUE_MASK                                                                           0x00FFFFFFL
-//SDMA0_EDC_CONFIG
-#define SDMA0_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
-#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE__SHIFT                                                               0x2
-#define SDMA0_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
-#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE_MASK                                                                 0x00000004L
-//SDMA0_BA_THRESHOLD
-#define SDMA0_BA_THRESHOLD__READ_THRES__SHIFT                                                                 0x0
-#define SDMA0_BA_THRESHOLD__WRITE_THRES__SHIFT                                                                0x10
-#define SDMA0_BA_THRESHOLD__READ_THRES_MASK                                                                   0x000003FFL
-#define SDMA0_BA_THRESHOLD__WRITE_THRES_MASK                                                                  0x03FF0000L
-//SDMA0_ID
-#define SDMA0_ID__DEVICE_ID__SHIFT                                                                            0x0
-#define SDMA0_ID__DEVICE_ID_MASK                                                                              0x000000FFL
-//SDMA0_VERSION
-#define SDMA0_VERSION__MINVER__SHIFT                                                                          0x0
-#define SDMA0_VERSION__MAJVER__SHIFT                                                                          0x8
-#define SDMA0_VERSION__REV__SHIFT                                                                             0x10
-#define SDMA0_VERSION__MINVER_MASK                                                                            0x0000007FL
-#define SDMA0_VERSION__MAJVER_MASK                                                                            0x00007F00L
-#define SDMA0_VERSION__REV_MASK                                                                               0x003F0000L
-//SDMA0_EDC_COUNTER
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED__SHIFT                                                          0x0
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC__SHIFT                                                          0x1
-#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED__SHIFT                                                         0x2
-#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED__SHIFT                                                         0x3
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED__SHIFT                                                      0x4
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT                                                   0x5
-#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED__SHIFT                                                      0x6
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT                                                    0x7
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT                                                    0x8
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT                                                    0x9
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT                                                    0xa
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT                                                    0xb
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT                                                    0xc
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT                                                    0xd
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT                                                    0xe
-#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED__SHIFT                                                      0xf
-#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT                                                    0x10
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED_MASK                                                            0x00000001L
-#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC_MASK                                                            0x00000002L
-#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED_MASK                                                           0x00000004L
-#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED_MASK                                                           0x00000008L
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED_MASK                                                        0x00000010L
-#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED_MASK                                                     0x00000020L
-#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED_MASK                                                        0x00000040L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK                                                      0x00000080L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK                                                      0x00000100L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK                                                      0x00000200L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK                                                      0x00000400L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK                                                      0x00000800L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK                                                      0x00001000L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK                                                      0x00002000L
-#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK                                                      0x00004000L
-#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED_MASK                                                        0x00008000L
-#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED_MASK                                                      0x00010000L
-//SDMA0_EDC_COUNTER_CLEAR
-#define SDMA0_EDC_COUNTER_CLEAR__DUMMY__SHIFT                                                                 0x0
-#define SDMA0_EDC_COUNTER_CLEAR__DUMMY_MASK                                                                   0x00000001L
-//SDMA0_STATUS2_REG
-#define SDMA0_STATUS2_REG__ID__SHIFT                                                                          0x0
-#define SDMA0_STATUS2_REG__F32_INSTR_PTR__SHIFT                                                               0x2
-#define SDMA0_STATUS2_REG__CMD_OP__SHIFT                                                                      0x10
-#define SDMA0_STATUS2_REG__ID_MASK                                                                            0x00000003L
-#define SDMA0_STATUS2_REG__F32_INSTR_PTR_MASK                                                                 0x00000FFCL
-#define SDMA0_STATUS2_REG__CMD_OP_MASK                                                                        0xFFFF0000L
-//SDMA0_ATOMIC_CNTL
-#define SDMA0_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                                  0x0
-#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                       0x1f
-#define SDMA0_ATOMIC_CNTL__LOOP_TIMER_MASK                                                                    0x7FFFFFFFL
-#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                         0x80000000L
-//SDMA0_ATOMIC_PREOP_LO
-#define SDMA0_ATOMIC_PREOP_LO__DATA__SHIFT                                                                    0x0
-#define SDMA0_ATOMIC_PREOP_LO__DATA_MASK                                                                      0xFFFFFFFFL
-//SDMA0_ATOMIC_PREOP_HI
-#define SDMA0_ATOMIC_PREOP_HI__DATA__SHIFT                                                                    0x0
-#define SDMA0_ATOMIC_PREOP_HI__DATA_MASK                                                                      0xFFFFFFFFL
-//SDMA0_UTCL1_CNTL
-#define SDMA0_UTCL1_CNTL__REDO_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_UTCL1_CNTL__REDO_DELAY__SHIFT                                                                   0x1
-#define SDMA0_UTCL1_CNTL__REDO_WATERMK__SHIFT                                                                 0xb
-#define SDMA0_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                                 0xe
-#define SDMA0_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                                 0x18
-#define SDMA0_UTCL1_CNTL__VADDR_WATERMK__SHIFT                                                                0x1d
-#define SDMA0_UTCL1_CNTL__REDO_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_UTCL1_CNTL__REDO_DELAY_MASK                                                                     0x000007FEL
-#define SDMA0_UTCL1_CNTL__REDO_WATERMK_MASK                                                                   0x00003800L
-#define SDMA0_UTCL1_CNTL__INVACK_DELAY_MASK                                                                   0x00FFC000L
-#define SDMA0_UTCL1_CNTL__REQL2_CREDIT_MASK                                                                   0x1F000000L
-#define SDMA0_UTCL1_CNTL__VADDR_WATERMK_MASK                                                                  0xE0000000L
-//SDMA0_UTCL1_WATERMK
-#define SDMA0_UTCL1_WATERMK__REQMC_WATERMK__SHIFT                                                             0x0
-#define SDMA0_UTCL1_WATERMK__REQPG_WATERMK__SHIFT                                                             0xa
-#define SDMA0_UTCL1_WATERMK__INVREQ_WATERMK__SHIFT                                                            0x12
-#define SDMA0_UTCL1_WATERMK__XNACK_WATERMK__SHIFT                                                             0x1a
-#define SDMA0_UTCL1_WATERMK__REQMC_WATERMK_MASK                                                               0x000003FFL
-#define SDMA0_UTCL1_WATERMK__REQPG_WATERMK_MASK                                                               0x0003FC00L
-#define SDMA0_UTCL1_WATERMK__INVREQ_WATERMK_MASK                                                              0x03FC0000L
-#define SDMA0_UTCL1_WATERMK__XNACK_WATERMK_MASK                                                               0xFC000000L
-//SDMA0_UTCL1_RD_STATUS
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                0x0
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                     0x1
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                      0x2
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                   0x3
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                               0x4
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                    0x5
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                 0x6
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT                                                   0x7
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT                                                  0x8
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                 0x9
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                      0xa
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                       0xb
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                    0xc
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                0xd
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                     0xe
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                  0xf
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT                                                    0x10
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT                                                   0x11
-#define SDMA0_UTCL1_RD_STATUS__PAGE_FAULT__SHIFT                                                              0x12
-#define SDMA0_UTCL1_RD_STATUS__PAGE_NULL__SHIFT                                                               0x13
-#define SDMA0_UTCL1_RD_STATUS__REQL2_IDLE__SHIFT                                                              0x14
-#define SDMA0_UTCL1_RD_STATUS__CE_L1_STALL__SHIFT                                                             0x15
-#define SDMA0_UTCL1_RD_STATUS__NEXT_RD_VECTOR__SHIFT                                                          0x16
-#define SDMA0_UTCL1_RD_STATUS__MERGE_STATE__SHIFT                                                             0x1a
-#define SDMA0_UTCL1_RD_STATUS__ADDR_RD_RTR__SHIFT                                                             0x1d
-#define SDMA0_UTCL1_RD_STATUS__WPTR_POLLING__SHIFT                                                            0x1e
-#define SDMA0_UTCL1_RD_STATUS__INVREQ_SIZE__SHIFT                                                             0x1f
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                  0x00000001L
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                       0x00000002L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                        0x00000004L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                     0x00000008L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                 0x00000010L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                      0x00000020L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                   0x00000040L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK                                                     0x00000080L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK                                                    0x00000100L
-#define SDMA0_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                   0x00000200L
-#define SDMA0_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                        0x00000400L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL_MASK                                                         0x00000800L
-#define SDMA0_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                      0x00001000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                  0x00002000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                       0x00004000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                    0x00008000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL_MASK                                                      0x00010000L
-#define SDMA0_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL_MASK                                                     0x00020000L
-#define SDMA0_UTCL1_RD_STATUS__PAGE_FAULT_MASK                                                                0x00040000L
-#define SDMA0_UTCL1_RD_STATUS__PAGE_NULL_MASK                                                                 0x00080000L
-#define SDMA0_UTCL1_RD_STATUS__REQL2_IDLE_MASK                                                                0x00100000L
-#define SDMA0_UTCL1_RD_STATUS__CE_L1_STALL_MASK                                                               0x00200000L
-#define SDMA0_UTCL1_RD_STATUS__NEXT_RD_VECTOR_MASK                                                            0x03C00000L
-#define SDMA0_UTCL1_RD_STATUS__MERGE_STATE_MASK                                                               0x1C000000L
-#define SDMA0_UTCL1_RD_STATUS__ADDR_RD_RTR_MASK                                                               0x20000000L
-#define SDMA0_UTCL1_RD_STATUS__WPTR_POLLING_MASK                                                              0x40000000L
-#define SDMA0_UTCL1_RD_STATUS__INVREQ_SIZE_MASK                                                               0x80000000L
-//SDMA0_UTCL1_WR_STATUS
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                0x0
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                     0x1
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                      0x2
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                   0x3
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                               0x4
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                    0x5
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                 0x6
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT                                                   0x7
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT                                                  0x8
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                 0x9
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                      0xa
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                       0xb
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                    0xc
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                0xd
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                     0xe
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                  0xf
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT                                                    0x10
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT                                                   0x11
-#define SDMA0_UTCL1_WR_STATUS__PAGE_FAULT__SHIFT                                                              0x12
-#define SDMA0_UTCL1_WR_STATUS__PAGE_NULL__SHIFT                                                               0x13
-#define SDMA0_UTCL1_WR_STATUS__REQL2_IDLE__SHIFT                                                              0x14
-#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT                                                              0x15
-#define SDMA0_UTCL1_WR_STATUS__NEXT_WR_VECTOR__SHIFT                                                          0x16
-#define SDMA0_UTCL1_WR_STATUS__MERGE_STATE__SHIFT                                                             0x19
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY__SHIFT                                                    0x1c
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL__SHIFT                                                     0x1d
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY__SHIFT                                                   0x1e
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL__SHIFT                                                    0x1f
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                  0x00000001L
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                       0x00000002L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                        0x00000004L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                     0x00000008L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                 0x00000010L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                      0x00000020L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                   0x00000040L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK                                                     0x00000080L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK                                                    0x00000100L
-#define SDMA0_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                   0x00000200L
-#define SDMA0_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                        0x00000400L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL_MASK                                                         0x00000800L
-#define SDMA0_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                      0x00001000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                  0x00002000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                       0x00004000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                    0x00008000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL_MASK                                                      0x00010000L
-#define SDMA0_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL_MASK                                                     0x00020000L
-#define SDMA0_UTCL1_WR_STATUS__PAGE_FAULT_MASK                                                                0x00040000L
-#define SDMA0_UTCL1_WR_STATUS__PAGE_NULL_MASK                                                                 0x00080000L
-#define SDMA0_UTCL1_WR_STATUS__REQL2_IDLE_MASK                                                                0x00100000L
-#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR_MASK                                                                0x00200000L
-#define SDMA0_UTCL1_WR_STATUS__NEXT_WR_VECTOR_MASK                                                            0x01C00000L
-#define SDMA0_UTCL1_WR_STATUS__MERGE_STATE_MASK                                                               0x0E000000L
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY_MASK                                                      0x10000000L
-#define SDMA0_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL_MASK                                                       0x20000000L
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY_MASK                                                     0x40000000L
-#define SDMA0_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL_MASK                                                      0x80000000L
-//SDMA0_UTCL1_INV0
-#define SDMA0_UTCL1_INV0__INV_MIDDLE__SHIFT                                                                   0x0
-#define SDMA0_UTCL1_INV0__RD_TIMEOUT__SHIFT                                                                   0x1
-#define SDMA0_UTCL1_INV0__WR_TIMEOUT__SHIFT                                                                   0x2
-#define SDMA0_UTCL1_INV0__RD_IN_INVADR__SHIFT                                                                 0x3
-#define SDMA0_UTCL1_INV0__WR_IN_INVADR__SHIFT                                                                 0x4
-#define SDMA0_UTCL1_INV0__PAGE_NULL_SW__SHIFT                                                                 0x5
-#define SDMA0_UTCL1_INV0__XNACK_IS_INVADR__SHIFT                                                              0x6
-#define SDMA0_UTCL1_INV0__INVREQ_ENABLE__SHIFT                                                                0x7
-#define SDMA0_UTCL1_INV0__NACK_TIMEOUT_SW__SHIFT                                                              0x8
-#define SDMA0_UTCL1_INV0__NFLUSH_INV_IDLE__SHIFT                                                              0x9
-#define SDMA0_UTCL1_INV0__FLUSH_INV_IDLE__SHIFT                                                               0xa
-#define SDMA0_UTCL1_INV0__INV_FLUSHTYPE__SHIFT                                                                0xb
-#define SDMA0_UTCL1_INV0__INV_VMID_VEC__SHIFT                                                                 0xc
-#define SDMA0_UTCL1_INV0__INV_ADDR_HI__SHIFT                                                                  0x1c
-#define SDMA0_UTCL1_INV0__INV_MIDDLE_MASK                                                                     0x00000001L
-#define SDMA0_UTCL1_INV0__RD_TIMEOUT_MASK                                                                     0x00000002L
-#define SDMA0_UTCL1_INV0__WR_TIMEOUT_MASK                                                                     0x00000004L
-#define SDMA0_UTCL1_INV0__RD_IN_INVADR_MASK                                                                   0x00000008L
-#define SDMA0_UTCL1_INV0__WR_IN_INVADR_MASK                                                                   0x00000010L
-#define SDMA0_UTCL1_INV0__PAGE_NULL_SW_MASK                                                                   0x00000020L
-#define SDMA0_UTCL1_INV0__XNACK_IS_INVADR_MASK                                                                0x00000040L
-#define SDMA0_UTCL1_INV0__INVREQ_ENABLE_MASK                                                                  0x00000080L
-#define SDMA0_UTCL1_INV0__NACK_TIMEOUT_SW_MASK                                                                0x00000100L
-#define SDMA0_UTCL1_INV0__NFLUSH_INV_IDLE_MASK                                                                0x00000200L
-#define SDMA0_UTCL1_INV0__FLUSH_INV_IDLE_MASK                                                                 0x00000400L
-#define SDMA0_UTCL1_INV0__INV_FLUSHTYPE_MASK                                                                  0x00000800L
-#define SDMA0_UTCL1_INV0__INV_VMID_VEC_MASK                                                                   0x0FFFF000L
-#define SDMA0_UTCL1_INV0__INV_ADDR_HI_MASK                                                                    0xF0000000L
-//SDMA0_UTCL1_INV1
-#define SDMA0_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                                  0x0
-#define SDMA0_UTCL1_INV1__INV_ADDR_LO_MASK                                                                    0xFFFFFFFFL
-//SDMA0_UTCL1_INV2
-#define SDMA0_UTCL1_INV2__INV_NFLUSH_VMID_VEC__SHIFT                                                          0x0
-#define SDMA0_UTCL1_INV2__INV_NFLUSH_VMID_VEC_MASK                                                            0xFFFFFFFFL
-//SDMA0_UTCL1_RD_XNACK0
-#define SDMA0_UTCL1_RD_XNACK0__XNACK_ADDR_LO__SHIFT                                                           0x0
-#define SDMA0_UTCL1_RD_XNACK0__XNACK_ADDR_LO_MASK                                                             0xFFFFFFFFL
-//SDMA0_UTCL1_RD_XNACK1
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_ADDR_HI__SHIFT                                                           0x0
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VMID__SHIFT                                                              0x4
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VECTOR__SHIFT                                                            0x8
-#define SDMA0_UTCL1_RD_XNACK1__IS_XNACK__SHIFT                                                                0x1a
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_ADDR_HI_MASK                                                             0x0000000FL
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VMID_MASK                                                                0x000000F0L
-#define SDMA0_UTCL1_RD_XNACK1__XNACK_VECTOR_MASK                                                              0x03FFFF00L
-#define SDMA0_UTCL1_RD_XNACK1__IS_XNACK_MASK                                                                  0x0C000000L
-//SDMA0_UTCL1_WR_XNACK0
-#define SDMA0_UTCL1_WR_XNACK0__XNACK_ADDR_LO__SHIFT                                                           0x0
-#define SDMA0_UTCL1_WR_XNACK0__XNACK_ADDR_LO_MASK                                                             0xFFFFFFFFL
-//SDMA0_UTCL1_WR_XNACK1
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_ADDR_HI__SHIFT                                                           0x0
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VMID__SHIFT                                                              0x4
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VECTOR__SHIFT                                                            0x8
-#define SDMA0_UTCL1_WR_XNACK1__IS_XNACK__SHIFT                                                                0x1a
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_ADDR_HI_MASK                                                             0x0000000FL
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VMID_MASK                                                                0x000000F0L
-#define SDMA0_UTCL1_WR_XNACK1__XNACK_VECTOR_MASK                                                              0x03FFFF00L
-#define SDMA0_UTCL1_WR_XNACK1__IS_XNACK_MASK                                                                  0x0C000000L
-//SDMA0_UTCL1_TIMEOUT
-#define SDMA0_UTCL1_TIMEOUT__RD_XNACK_LIMIT__SHIFT                                                            0x0
-#define SDMA0_UTCL1_TIMEOUT__WR_XNACK_LIMIT__SHIFT                                                            0x10
-#define SDMA0_UTCL1_TIMEOUT__RD_XNACK_LIMIT_MASK                                                              0x0000FFFFL
-#define SDMA0_UTCL1_TIMEOUT__WR_XNACK_LIMIT_MASK                                                              0xFFFF0000L
-//SDMA0_UTCL1_PAGE
-#define SDMA0_UTCL1_PAGE__VM_HOLE__SHIFT                                                                      0x0
-#define SDMA0_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                     0x1
-#define SDMA0_UTCL1_PAGE__USE_MTYPE__SHIFT                                                                    0x6
-#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                                 0x9
-#define SDMA0_UTCL1_PAGE__VM_HOLE_MASK                                                                        0x00000001L
-#define SDMA0_UTCL1_PAGE__REQ_TYPE_MASK                                                                       0x0000001EL
-#define SDMA0_UTCL1_PAGE__USE_MTYPE_MASK                                                                      0x000001C0L
-#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                                   0x00000200L
-//SDMA0_POWER_CNTL_IDLE
-#define SDMA0_POWER_CNTL_IDLE__DELAY0__SHIFT                                                                  0x0
-#define SDMA0_POWER_CNTL_IDLE__DELAY1__SHIFT                                                                  0x10
-#define SDMA0_POWER_CNTL_IDLE__DELAY2__SHIFT                                                                  0x18
-#define SDMA0_POWER_CNTL_IDLE__DELAY0_MASK                                                                    0x0000FFFFL
-#define SDMA0_POWER_CNTL_IDLE__DELAY1_MASK                                                                    0x00FF0000L
-#define SDMA0_POWER_CNTL_IDLE__DELAY2_MASK                                                                    0xFF000000L
-//SDMA0_RELAX_ORDERING_LUT
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED0__SHIFT                                                            0x0
-#define SDMA0_RELAX_ORDERING_LUT__COPY__SHIFT                                                                 0x1
-#define SDMA0_RELAX_ORDERING_LUT__WRITE__SHIFT                                                                0x2
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED3__SHIFT                                                            0x3
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED4__SHIFT                                                            0x4
-#define SDMA0_RELAX_ORDERING_LUT__FENCE__SHIFT                                                                0x5
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED76__SHIFT                                                           0x6
-#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM__SHIFT                                                             0x8
-#define SDMA0_RELAX_ORDERING_LUT__COND_EXE__SHIFT                                                             0x9
-#define SDMA0_RELAX_ORDERING_LUT__ATOMIC__SHIFT                                                               0xa
-#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL__SHIFT                                                           0xb
-#define SDMA0_RELAX_ORDERING_LUT__PTEPDE__SHIFT                                                               0xc
-#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT                                                            0xd
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED__SHIFT                                                             0xe
-#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT                                                         0x1b
-#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT                                                             0x1c
-#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT                                                            0x1d
-#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH__SHIFT                                                             0x1e
-#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH__SHIFT                                                             0x1f
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED0_MASK                                                              0x00000001L
-#define SDMA0_RELAX_ORDERING_LUT__COPY_MASK                                                                   0x00000002L
-#define SDMA0_RELAX_ORDERING_LUT__WRITE_MASK                                                                  0x00000004L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED3_MASK                                                              0x00000008L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED4_MASK                                                              0x00000010L
-#define SDMA0_RELAX_ORDERING_LUT__FENCE_MASK                                                                  0x00000020L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED76_MASK                                                             0x000000C0L
-#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM_MASK                                                               0x00000100L
-#define SDMA0_RELAX_ORDERING_LUT__COND_EXE_MASK                                                               0x00000200L
-#define SDMA0_RELAX_ORDERING_LUT__ATOMIC_MASK                                                                 0x00000400L
-#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL_MASK                                                             0x00000800L
-#define SDMA0_RELAX_ORDERING_LUT__PTEPDE_MASK                                                                 0x00001000L
-#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP_MASK                                                              0x00002000L
-#define SDMA0_RELAX_ORDERING_LUT__RESERVED_MASK                                                               0x07FFC000L
-#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK                                                           0x08000000L
-#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB_MASK                                                               0x10000000L
-#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL_MASK                                                              0x20000000L
-#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH_MASK                                                               0x40000000L
-#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH_MASK                                                               0x80000000L
-//SDMA0_CHICKEN_BITS_2
-#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT                                                       0x0
-#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK                                                         0x0000000FL
-//SDMA0_STATUS3_REG
-#define SDMA0_STATUS3_REG__CMD_OP_STATUS__SHIFT                                                               0x0
-#define SDMA0_STATUS3_REG__PREV_VM_CMD__SHIFT                                                                 0x10
-#define SDMA0_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                              0x14
-#define SDMA0_STATUS3_REG__CMD_OP_STATUS_MASK                                                                 0x0000FFFFL
-#define SDMA0_STATUS3_REG__PREV_VM_CMD_MASK                                                                   0x000F0000L
-#define SDMA0_STATUS3_REG__EXCEPTION_IDLE_MASK                                                                0x00100000L
-//SDMA0_PHYSICAL_ADDR_LO
-#define SDMA0_PHYSICAL_ADDR_LO__D_VALID__SHIFT                                                                0x0
-#define SDMA0_PHYSICAL_ADDR_LO__DIRTY__SHIFT                                                                  0x1
-#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT                                                              0x2
-#define SDMA0_PHYSICAL_ADDR_LO__ADDR__SHIFT                                                                   0xc
-#define SDMA0_PHYSICAL_ADDR_LO__D_VALID_MASK                                                                  0x00000001L
-#define SDMA0_PHYSICAL_ADDR_LO__DIRTY_MASK                                                                    0x00000002L
-#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID_MASK                                                                0x00000004L
-#define SDMA0_PHYSICAL_ADDR_LO__ADDR_MASK                                                                     0xFFFFF000L
-//SDMA0_PHYSICAL_ADDR_HI
-#define SDMA0_PHYSICAL_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA0_PHYSICAL_ADDR_HI__ADDR_MASK                                                                     0x0000FFFFL
-//SDMA0_PHASE2_QUANTUM
-#define SDMA0_PHASE2_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA0_PHASE2_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA0_PHASE2_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA0_PHASE2_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA0_PHASE2_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA0_PHASE2_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA0_ERROR_LOG
-#define SDMA0_ERROR_LOG__OVERRIDE__SHIFT                                                                      0x0
-#define SDMA0_ERROR_LOG__STATUS__SHIFT                                                                        0x10
-#define SDMA0_ERROR_LOG__OVERRIDE_MASK                                                                        0x0000FFFFL
-#define SDMA0_ERROR_LOG__STATUS_MASK                                                                          0xFFFF0000L
-//SDMA0_PUB_DUMMY_REG0
-#define SDMA0_PUB_DUMMY_REG0__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG0__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PUB_DUMMY_REG1
-#define SDMA0_PUB_DUMMY_REG1__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG1__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PUB_DUMMY_REG2
-#define SDMA0_PUB_DUMMY_REG2__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG2__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PUB_DUMMY_REG3
-#define SDMA0_PUB_DUMMY_REG3__VALUE__SHIFT                                                                    0x0
-#define SDMA0_PUB_DUMMY_REG3__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA0_F32_COUNTER
-#define SDMA0_F32_COUNTER__VALUE__SHIFT                                                                       0x0
-#define SDMA0_F32_COUNTER__VALUE_MASK                                                                         0xFFFFFFFFL
-//SDMA0_UNBREAKABLE
-#define SDMA0_UNBREAKABLE__VALUE__SHIFT                                                                       0x0
-#define SDMA0_UNBREAKABLE__VALUE_MASK                                                                         0x00000001L
-//SDMA0_PERFMON_CNTL
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE0__SHIFT                                                               0x0
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR0__SHIFT                                                                0x1
-#define SDMA0_PERFMON_CNTL__PERF_SEL0__SHIFT                                                                  0x2
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE1__SHIFT                                                               0xa
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR1__SHIFT                                                                0xb
-#define SDMA0_PERFMON_CNTL__PERF_SEL1__SHIFT                                                                  0xc
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE0_MASK                                                                 0x00000001L
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR0_MASK                                                                  0x00000002L
-#define SDMA0_PERFMON_CNTL__PERF_SEL0_MASK                                                                    0x000003FCL
-#define SDMA0_PERFMON_CNTL__PERF_ENABLE1_MASK                                                                 0x00000400L
-#define SDMA0_PERFMON_CNTL__PERF_CLEAR1_MASK                                                                  0x00000800L
-#define SDMA0_PERFMON_CNTL__PERF_SEL1_MASK                                                                    0x000FF000L
-//SDMA0_PERFCOUNTER0_RESULT
-#define SDMA0_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT                                                          0x0
-#define SDMA0_PERFCOUNTER0_RESULT__PERF_COUNT_MASK                                                            0xFFFFFFFFL
-//SDMA0_PERFCOUNTER1_RESULT
-#define SDMA0_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT                                                          0x0
-#define SDMA0_PERFCOUNTER1_RESULT__PERF_COUNT_MASK                                                            0xFFFFFFFFL
-//SDMA0_PERFCOUNTER_TAG_DELAY_RANGE
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW__SHIFT                                                   0x0
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH__SHIFT                                                  0xe
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW__SHIFT                                                   0x1c
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW_MASK                                                     0x00003FFFL
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH_MASK                                                    0x0FFFC000L
-#define SDMA0_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW_MASK                                                     0x10000000L
-//SDMA0_CRD_CNTL
-#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT                                                                0x7
-#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT                                                                0xd
-#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT_MASK                                                                  0x00001F80L
-#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT_MASK                                                                  0x0007E000L
-//SDMA0_MMHUB_TRUSTLVL
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG0__SHIFT                                                                 0x0
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG1__SHIFT                                                                 0x3
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG2__SHIFT                                                                 0x6
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG3__SHIFT                                                                 0x9
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG4__SHIFT                                                                 0xc
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG5__SHIFT                                                                 0xf
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG6__SHIFT                                                                 0x12
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG7__SHIFT                                                                 0x15
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG0_MASK                                                                   0x00000007L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG1_MASK                                                                   0x00000038L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG2_MASK                                                                   0x000001C0L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG3_MASK                                                                   0x00000E00L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG4_MASK                                                                   0x00007000L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG5_MASK                                                                   0x00038000L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG6_MASK                                                                   0x001C0000L
-#define SDMA0_MMHUB_TRUSTLVL__SECFLAG7_MASK                                                                   0x00E00000L
-//SDMA0_GPU_IOV_VIOLATION_LOG
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                  0x0
-#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                         0x1
-#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                           0x2
-#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                                   0x12
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                0x13
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                              0x14
-#define SDMA0_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT                                                      0x18
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                    0x00000001L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                           0x00000002L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                             0x0003FFFCL
-#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                     0x00040000L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                  0x00080000L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                                0x00F00000L
-#define SDMA0_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK                                                        0xFF000000L
-//SDMA0_ULV_CNTL
-#define SDMA0_ULV_CNTL__HYSTERESIS__SHIFT                                                                     0x0
-#define SDMA0_ULV_CNTL__ENTER_ULV_INT__SHIFT                                                                  0x1d
-#define SDMA0_ULV_CNTL__EXIT_ULV_INT__SHIFT                                                                   0x1e
-#define SDMA0_ULV_CNTL__ULV_STATUS__SHIFT                                                                     0x1f
-#define SDMA0_ULV_CNTL__HYSTERESIS_MASK                                                                       0x0000001FL
-#define SDMA0_ULV_CNTL__ENTER_ULV_INT_MASK                                                                    0x20000000L
-#define SDMA0_ULV_CNTL__EXIT_ULV_INT_MASK                                                                     0x40000000L
-#define SDMA0_ULV_CNTL__ULV_STATUS_MASK                                                                       0x80000000L
-//SDMA0_EA_DBIT_ADDR_DATA
-#define SDMA0_EA_DBIT_ADDR_DATA__VALUE__SHIFT                                                                 0x0
-#define SDMA0_EA_DBIT_ADDR_DATA__VALUE_MASK                                                                   0xFFFFFFFFL
-//SDMA0_EA_DBIT_ADDR_INDEX
-#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE__SHIFT                                                                0x0
-#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE_MASK                                                                  0x00000007L
-//SDMA0_GFX_RB_CNTL
-#define SDMA0_GFX_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
-#define SDMA0_GFX_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
-#define SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
-#define SDMA0_GFX_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
-#define SDMA0_GFX_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
-#define SDMA0_GFX_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
-#define SDMA0_GFX_RB_CNTL__RB_SIZE_MASK                                                                       0x0000007EL
-#define SDMA0_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
-#define SDMA0_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
-#define SDMA0_GFX_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
-#define SDMA0_GFX_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
-//SDMA0_GFX_RB_BASE
-#define SDMA0_GFX_RB_BASE__ADDR__SHIFT                                                                        0x0
-#define SDMA0_GFX_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
-//SDMA0_GFX_RB_BASE_HI
-#define SDMA0_GFX_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
-#define SDMA0_GFX_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
-//SDMA0_GFX_RB_RPTR
-#define SDMA0_GFX_RB_RPTR__OFFSET__SHIFT                                                                      0x0
-#define SDMA0_GFX_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
-//SDMA0_GFX_RB_RPTR_HI
-#define SDMA0_GFX_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
-#define SDMA0_GFX_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR
-#define SDMA0_GFX_RB_WPTR__OFFSET__SHIFT                                                                      0x0
-#define SDMA0_GFX_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_HI
-#define SDMA0_GFX_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
-#define SDMA0_GFX_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_POLL_CNTL
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
-#define SDMA0_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
-//SDMA0_GFX_RB_RPTR_ADDR_HI
-#define SDMA0_GFX_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
-#define SDMA0_GFX_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
-//SDMA0_GFX_RB_RPTR_ADDR_LO
-#define SDMA0_GFX_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
-#define SDMA0_GFX_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
-//SDMA0_GFX_IB_CNTL
-#define SDMA0_GFX_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
-#define SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
-#define SDMA0_GFX_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
-#define SDMA0_GFX_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
-#define SDMA0_GFX_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
-#define SDMA0_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
-#define SDMA0_GFX_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
-#define SDMA0_GFX_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
-//SDMA0_GFX_IB_RPTR
-#define SDMA0_GFX_IB_RPTR__OFFSET__SHIFT                                                                      0x2
-#define SDMA0_GFX_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
-//SDMA0_GFX_IB_OFFSET
-#define SDMA0_GFX_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
-#define SDMA0_GFX_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
-//SDMA0_GFX_IB_BASE_LO
-#define SDMA0_GFX_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
-#define SDMA0_GFX_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
-//SDMA0_GFX_IB_BASE_HI
-#define SDMA0_GFX_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
-#define SDMA0_GFX_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
-//SDMA0_GFX_IB_SIZE
-#define SDMA0_GFX_IB_SIZE__SIZE__SHIFT                                                                        0x0
-#define SDMA0_GFX_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
-//SDMA0_GFX_SKIP_CNTL
-#define SDMA0_GFX_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
-#define SDMA0_GFX_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x00003FFFL
-//SDMA0_GFX_CONTEXT_STATUS
-#define SDMA0_GFX_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
-#define SDMA0_GFX_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
-#define SDMA0_GFX_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
-#define SDMA0_GFX_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
-#define SDMA0_GFX_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
-#define SDMA0_GFX_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
-#define SDMA0_GFX_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
-#define SDMA0_GFX_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
-#define SDMA0_GFX_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
-#define SDMA0_GFX_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
-//SDMA0_GFX_DOORBELL
-#define SDMA0_GFX_DOORBELL__ENABLE__SHIFT                                                                     0x1c
-#define SDMA0_GFX_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
-#define SDMA0_GFX_DOORBELL__ENABLE_MASK                                                                       0x10000000L
-#define SDMA0_GFX_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
-//SDMA0_GFX_CONTEXT_CNTL
-#define SDMA0_GFX_CONTEXT_CNTL__RESUME_CTX__SHIFT                                                             0x10
-#define SDMA0_GFX_CONTEXT_CNTL__RESUME_CTX_MASK                                                               0x00010000L
-//SDMA0_GFX_STATUS
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
-#define SDMA0_GFX_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
-//SDMA0_GFX_DOORBELL_LOG
-#define SDMA0_GFX_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
-#define SDMA0_GFX_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
-#define SDMA0_GFX_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
-#define SDMA0_GFX_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
-//SDMA0_GFX_WATERMARK
-#define SDMA0_GFX_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
-#define SDMA0_GFX_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
-#define SDMA0_GFX_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
-#define SDMA0_GFX_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
-//SDMA0_GFX_DOORBELL_OFFSET
-#define SDMA0_GFX_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
-#define SDMA0_GFX_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
-//SDMA0_GFX_CSA_ADDR_LO
-#define SDMA0_GFX_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
-#define SDMA0_GFX_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
-//SDMA0_GFX_CSA_ADDR_HI
-#define SDMA0_GFX_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_GFX_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA0_GFX_IB_SUB_REMAIN
-#define SDMA0_GFX_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
-#define SDMA0_GFX_IB_SUB_REMAIN__SIZE_MASK                                                                    0x00003FFFL
-//SDMA0_GFX_PREEMPT
-#define SDMA0_GFX_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
-#define SDMA0_GFX_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
-//SDMA0_GFX_DUMMY_REG
-#define SDMA0_GFX_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
-#define SDMA0_GFX_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_POLL_ADDR_HI
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
-//SDMA0_GFX_RB_WPTR_POLL_ADDR_LO
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
-#define SDMA0_GFX_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
-//SDMA0_GFX_RB_AQL_CNTL
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
-#define SDMA0_GFX_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
-#define SDMA0_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
-#define SDMA0_GFX_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
-//SDMA0_GFX_MINOR_PTR_UPDATE
-#define SDMA0_GFX_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
-#define SDMA0_GFX_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
-//SDMA0_GFX_MIDCMD_DATA0
-#define SDMA0_GFX_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA1
-#define SDMA0_GFX_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA2
-#define SDMA0_GFX_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA3
-#define SDMA0_GFX_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA4
-#define SDMA0_GFX_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA5
-#define SDMA0_GFX_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA6
-#define SDMA0_GFX_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA7
-#define SDMA0_GFX_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_DATA8
-#define SDMA0_GFX_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
-#define SDMA0_GFX_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
-//SDMA0_GFX_MIDCMD_CNTL
-#define SDMA0_GFX_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
-#define SDMA0_GFX_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
-#define SDMA0_GFX_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
-#define SDMA0_GFX_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
-#define SDMA0_GFX_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
-#define SDMA0_GFX_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
-#define SDMA0_GFX_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
-#define SDMA0_GFX_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
-//SDMA0_PAGE_RB_CNTL
-#define SDMA0_PAGE_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_PAGE_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA0_PAGE_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA0_PAGE_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA0_PAGE_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA0_PAGE_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_PAGE_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA0_PAGE_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA0_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA0_PAGE_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA0_PAGE_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA0_PAGE_RB_BASE
-#define SDMA0_PAGE_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA0_PAGE_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA0_PAGE_RB_BASE_HI
-#define SDMA0_PAGE_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_PAGE_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA0_PAGE_RB_RPTR
-#define SDMA0_PAGE_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_PAGE_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_PAGE_RB_RPTR_HI
-#define SDMA0_PAGE_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_PAGE_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_PAGE_RB_WPTR
-#define SDMA0_PAGE_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_PAGE_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_PAGE_RB_WPTR_HI
-#define SDMA0_PAGE_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_PAGE_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_PAGE_RB_WPTR_POLL_CNTL
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA0_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA0_PAGE_RB_RPTR_ADDR_HI
-#define SDMA0_PAGE_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA0_PAGE_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA0_PAGE_RB_RPTR_ADDR_LO
-#define SDMA0_PAGE_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA0_PAGE_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA0_PAGE_IB_CNTL
-#define SDMA0_PAGE_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_PAGE_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA0_PAGE_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA0_PAGE_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA0_PAGE_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_PAGE_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA0_PAGE_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA0_PAGE_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA0_PAGE_IB_RPTR
-#define SDMA0_PAGE_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA0_PAGE_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA0_PAGE_IB_OFFSET
-#define SDMA0_PAGE_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA0_PAGE_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA0_PAGE_IB_BASE_LO
-#define SDMA0_PAGE_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA0_PAGE_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA0_PAGE_IB_BASE_HI
-#define SDMA0_PAGE_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_PAGE_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PAGE_IB_SIZE
-#define SDMA0_PAGE_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA0_PAGE_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA0_PAGE_SKIP_CNTL
-#define SDMA0_PAGE_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA0_PAGE_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA0_PAGE_CONTEXT_STATUS
-#define SDMA0_PAGE_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA0_PAGE_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA0_PAGE_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA0_PAGE_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA0_PAGE_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA0_PAGE_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA0_PAGE_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA0_PAGE_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA0_PAGE_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA0_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA0_PAGE_DOORBELL
-#define SDMA0_PAGE_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA0_PAGE_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA0_PAGE_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA0_PAGE_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA0_PAGE_STATUS
-#define SDMA0_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA0_PAGE_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA0_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA0_PAGE_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA0_PAGE_DOORBELL_LOG
-#define SDMA0_PAGE_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA0_PAGE_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA0_PAGE_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA0_PAGE_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA0_PAGE_WATERMARK
-#define SDMA0_PAGE_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA0_PAGE_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA0_PAGE_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA0_PAGE_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA0_PAGE_DOORBELL_OFFSET
-#define SDMA0_PAGE_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA0_PAGE_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA0_PAGE_CSA_ADDR_LO
-#define SDMA0_PAGE_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA0_PAGE_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA0_PAGE_CSA_ADDR_HI
-#define SDMA0_PAGE_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA0_PAGE_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA0_PAGE_IB_SUB_REMAIN
-#define SDMA0_PAGE_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA0_PAGE_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA0_PAGE_PREEMPT
-#define SDMA0_PAGE_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA0_PAGE_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA0_PAGE_DUMMY_REG
-#define SDMA0_PAGE_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA0_PAGE_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA0_PAGE_RB_WPTR_POLL_ADDR_HI
-#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA0_PAGE_RB_WPTR_POLL_ADDR_LO
-#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA0_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA0_PAGE_RB_AQL_CNTL
-#define SDMA0_PAGE_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA0_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA0_PAGE_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA0_PAGE_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA0_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA0_PAGE_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA0_PAGE_MINOR_PTR_UPDATE
-#define SDMA0_PAGE_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA0_PAGE_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA0_PAGE_MIDCMD_DATA0
-#define SDMA0_PAGE_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA1
-#define SDMA0_PAGE_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA2
-#define SDMA0_PAGE_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA3
-#define SDMA0_PAGE_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA4
-#define SDMA0_PAGE_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA5
-#define SDMA0_PAGE_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA6
-#define SDMA0_PAGE_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA7
-#define SDMA0_PAGE_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_DATA8
-#define SDMA0_PAGE_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA0_PAGE_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA0_PAGE_MIDCMD_CNTL
-#define SDMA0_PAGE_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA0_PAGE_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA0_PAGE_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA0_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA0_PAGE_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA0_PAGE_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA0_PAGE_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA0_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-//SDMA0_RLC0_RB_CNTL
-#define SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA0_RLC0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA0_RLC0_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC0_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA0_RLC0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA0_RLC0_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA0_RLC0_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA0_RLC0_RB_BASE
-#define SDMA0_RLC0_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA0_RLC0_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA0_RLC0_RB_BASE_HI
-#define SDMA0_RLC0_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC0_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA0_RLC0_RB_RPTR
-#define SDMA0_RLC0_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC0_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC0_RB_RPTR_HI
-#define SDMA0_RLC0_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC0_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR
-#define SDMA0_RLC0_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC0_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_HI
-#define SDMA0_RLC0_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC0_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_POLL_CNTL
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA0_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA0_RLC0_RB_RPTR_ADDR_HI
-#define SDMA0_RLC0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA0_RLC0_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA0_RLC0_RB_RPTR_ADDR_LO
-#define SDMA0_RLC0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA0_RLC0_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA0_RLC0_IB_CNTL
-#define SDMA0_RLC0_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA0_RLC0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA0_RLC0_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA0_RLC0_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA0_RLC0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA0_RLC0_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA0_RLC0_IB_RPTR
-#define SDMA0_RLC0_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA0_RLC0_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA0_RLC0_IB_OFFSET
-#define SDMA0_RLC0_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA0_RLC0_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA0_RLC0_IB_BASE_LO
-#define SDMA0_RLC0_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA0_RLC0_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA0_RLC0_IB_BASE_HI
-#define SDMA0_RLC0_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC0_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC0_IB_SIZE
-#define SDMA0_RLC0_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA0_RLC0_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA0_RLC0_SKIP_CNTL
-#define SDMA0_RLC0_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA0_RLC0_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA0_RLC0_CONTEXT_STATUS
-#define SDMA0_RLC0_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA0_RLC0_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA0_RLC0_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA0_RLC0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA0_RLC0_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA0_RLC0_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA0_RLC0_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA0_RLC0_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA0_RLC0_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA0_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA0_RLC0_DOORBELL
-#define SDMA0_RLC0_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA0_RLC0_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA0_RLC0_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA0_RLC0_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA0_RLC0_STATUS
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA0_RLC0_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA0_RLC0_DOORBELL_LOG
-#define SDMA0_RLC0_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA0_RLC0_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA0_RLC0_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA0_RLC0_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA0_RLC0_WATERMARK
-#define SDMA0_RLC0_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA0_RLC0_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA0_RLC0_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA0_RLC0_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA0_RLC0_DOORBELL_OFFSET
-#define SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA0_RLC0_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA0_RLC0_CSA_ADDR_LO
-#define SDMA0_RLC0_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA0_RLC0_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA0_RLC0_CSA_ADDR_HI
-#define SDMA0_RLC0_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA0_RLC0_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA0_RLC0_IB_SUB_REMAIN
-#define SDMA0_RLC0_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA0_RLC0_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA0_RLC0_PREEMPT
-#define SDMA0_RLC0_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA0_RLC0_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA0_RLC0_DUMMY_REG
-#define SDMA0_RLC0_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA0_RLC0_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA0_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA0_RLC0_RB_AQL_CNTL
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA0_RLC0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA0_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA0_RLC0_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA0_RLC0_MINOR_PTR_UPDATE
-#define SDMA0_RLC0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA0_RLC0_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA0_RLC0_MIDCMD_DATA0
-#define SDMA0_RLC0_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA1
-#define SDMA0_RLC0_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA2
-#define SDMA0_RLC0_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA3
-#define SDMA0_RLC0_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA4
-#define SDMA0_RLC0_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA5
-#define SDMA0_RLC0_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA6
-#define SDMA0_RLC0_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA7
-#define SDMA0_RLC0_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_DATA8
-#define SDMA0_RLC0_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA0_RLC0_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC0_MIDCMD_CNTL
-#define SDMA0_RLC0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA0_RLC0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA0_RLC0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA0_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA0_RLC0_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA0_RLC0_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA0_RLC0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA0_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-//SDMA0_RLC1_RB_CNTL
-#define SDMA0_RLC1_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC1_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA0_RLC1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA0_RLC1_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA0_RLC1_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA0_RLC1_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC1_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA0_RLC1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA0_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA0_RLC1_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA0_RLC1_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA0_RLC1_RB_BASE
-#define SDMA0_RLC1_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA0_RLC1_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA0_RLC1_RB_BASE_HI
-#define SDMA0_RLC1_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC1_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA0_RLC1_RB_RPTR
-#define SDMA0_RLC1_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC1_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC1_RB_RPTR_HI
-#define SDMA0_RLC1_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC1_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR
-#define SDMA0_RLC1_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA0_RLC1_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_HI
-#define SDMA0_RLC1_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA0_RLC1_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_POLL_CNTL
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA0_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA0_RLC1_RB_RPTR_ADDR_HI
-#define SDMA0_RLC1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA0_RLC1_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA0_RLC1_RB_RPTR_ADDR_LO
-#define SDMA0_RLC1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA0_RLC1_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA0_RLC1_IB_CNTL
-#define SDMA0_RLC1_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA0_RLC1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA0_RLC1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA0_RLC1_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA0_RLC1_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA0_RLC1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA0_RLC1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA0_RLC1_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA0_RLC1_IB_RPTR
-#define SDMA0_RLC1_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA0_RLC1_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA0_RLC1_IB_OFFSET
-#define SDMA0_RLC1_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA0_RLC1_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA0_RLC1_IB_BASE_LO
-#define SDMA0_RLC1_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA0_RLC1_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA0_RLC1_IB_BASE_HI
-#define SDMA0_RLC1_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA0_RLC1_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC1_IB_SIZE
-#define SDMA0_RLC1_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA0_RLC1_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA0_RLC1_SKIP_CNTL
-#define SDMA0_RLC1_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA0_RLC1_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA0_RLC1_CONTEXT_STATUS
-#define SDMA0_RLC1_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA0_RLC1_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA0_RLC1_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA0_RLC1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA0_RLC1_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA0_RLC1_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA0_RLC1_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA0_RLC1_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA0_RLC1_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA0_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA0_RLC1_DOORBELL
-#define SDMA0_RLC1_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA0_RLC1_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA0_RLC1_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA0_RLC1_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA0_RLC1_STATUS
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA0_RLC1_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA0_RLC1_DOORBELL_LOG
-#define SDMA0_RLC1_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA0_RLC1_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA0_RLC1_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA0_RLC1_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA0_RLC1_WATERMARK
-#define SDMA0_RLC1_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA0_RLC1_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA0_RLC1_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA0_RLC1_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA0_RLC1_DOORBELL_OFFSET
-#define SDMA0_RLC1_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA0_RLC1_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA0_RLC1_CSA_ADDR_LO
-#define SDMA0_RLC1_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA0_RLC1_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA0_RLC1_CSA_ADDR_HI
-#define SDMA0_RLC1_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA0_RLC1_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA0_RLC1_IB_SUB_REMAIN
-#define SDMA0_RLC1_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA0_RLC1_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA0_RLC1_PREEMPT
-#define SDMA0_RLC1_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA0_RLC1_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA0_RLC1_DUMMY_REG
-#define SDMA0_RLC1_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA0_RLC1_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA0_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA0_RLC1_RB_AQL_CNTL
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA0_RLC1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA0_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA0_RLC1_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA0_RLC1_MINOR_PTR_UPDATE
-#define SDMA0_RLC1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA0_RLC1_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA0_RLC1_MIDCMD_DATA0
-#define SDMA0_RLC1_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA1
-#define SDMA0_RLC1_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA2
-#define SDMA0_RLC1_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA3
-#define SDMA0_RLC1_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA4
-#define SDMA0_RLC1_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA5
-#define SDMA0_RLC1_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA6
-#define SDMA0_RLC1_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA7
-#define SDMA0_RLC1_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_DATA8
-#define SDMA0_RLC1_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA0_RLC1_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA0_RLC1_MIDCMD_CNTL
-#define SDMA0_RLC1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA0_RLC1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA0_RLC1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA0_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA0_RLC1_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA0_RLC1_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA0_RLC1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA0_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_default.h
deleted file mode 100644
index 85c5c5e3ce7d..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_default.h
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _sdma1_4_0_DEFAULT_HEADER
-#define _sdma1_4_0_DEFAULT_HEADER
-
-
-// addressBlock: sdma1_sdma1dec
-#define mmSDMA1_UCODE_ADDR_DEFAULT                                               0x00000000
-#define mmSDMA1_UCODE_DATA_DEFAULT                                               0x00000000
-#define mmSDMA1_VM_CNTL_DEFAULT                                                  0x00000000
-#define mmSDMA1_VM_CTX_LO_DEFAULT                                                0x00000000
-#define mmSDMA1_VM_CTX_HI_DEFAULT                                                0x00000000
-#define mmSDMA1_ACTIVE_FCN_ID_DEFAULT                                            0x00000000
-#define mmSDMA1_VM_CTX_CNTL_DEFAULT                                              0x00000000
-#define mmSDMA1_VIRT_RESET_REQ_DEFAULT                                           0x00000000
-#define mmSDMA1_VF_ENABLE_DEFAULT                                                0x00000000
-#define mmSDMA1_CONTEXT_REG_TYPE0_DEFAULT                                        0xfffdf79f
-#define mmSDMA1_CONTEXT_REG_TYPE1_DEFAULT                                        0x003fbcff
-#define mmSDMA1_CONTEXT_REG_TYPE2_DEFAULT                                        0x000003ff
-#define mmSDMA1_CONTEXT_REG_TYPE3_DEFAULT                                        0x00000000
-#define mmSDMA1_PUB_REG_TYPE0_DEFAULT                                            0x3c000000
-#define mmSDMA1_PUB_REG_TYPE1_DEFAULT                                            0x30003882
-#define mmSDMA1_PUB_REG_TYPE2_DEFAULT                                            0x0fc6e880
-#define mmSDMA1_PUB_REG_TYPE3_DEFAULT                                            0x00000000
-#define mmSDMA1_MMHUB_CNTL_DEFAULT                                               0x00000000
-#define mmSDMA1_CONTEXT_GROUP_BOUNDARY_DEFAULT                                   0x00000000
-#define mmSDMA1_POWER_CNTL_DEFAULT                                               0x0003c000
-#define mmSDMA1_CLK_CTRL_DEFAULT                                                 0xff000100
-#define mmSDMA1_CNTL_DEFAULT                                                     0x00000002
-#define mmSDMA1_CHICKEN_BITS_DEFAULT                                             0x00831f07
-#define mmSDMA1_GB_ADDR_CONFIG_DEFAULT                                           0x00100012
-#define mmSDMA1_GB_ADDR_CONFIG_READ_DEFAULT                                      0x00100012
-#define mmSDMA1_RB_RPTR_FETCH_HI_DEFAULT                                         0x00000000
-#define mmSDMA1_SEM_WAIT_FAIL_TIMER_CNTL_DEFAULT                                 0x00000000
-#define mmSDMA1_RB_RPTR_FETCH_DEFAULT                                            0x00000000
-#define mmSDMA1_IB_OFFSET_FETCH_DEFAULT                                          0x00000000
-#define mmSDMA1_PROGRAM_DEFAULT                                                  0x00000000
-#define mmSDMA1_STATUS_REG_DEFAULT                                               0x46dee557
-#define mmSDMA1_STATUS1_REG_DEFAULT                                              0x000003ff
-#define mmSDMA1_RD_BURST_CNTL_DEFAULT                                            0x00000003
-#define mmSDMA1_HBM_PAGE_CONFIG_DEFAULT                                          0x00000000
-#define mmSDMA1_UCODE_CHECKSUM_DEFAULT                                           0x00000000
-#define mmSDMA1_F32_CNTL_DEFAULT                                                 0x00000001
-#define mmSDMA1_FREEZE_DEFAULT                                                   0x00000000
-#define mmSDMA1_PHASE0_QUANTUM_DEFAULT                                           0x00010002
-#define mmSDMA1_PHASE1_QUANTUM_DEFAULT                                           0x00010002
-#define mmSDMA1_EDC_CONFIG_DEFAULT                                               0x00000002
-#define mmSDMA1_BA_THRESHOLD_DEFAULT                                             0x03ff03ff
-#define mmSDMA1_ID_DEFAULT                                                       0x00000001
-#define mmSDMA1_VERSION_DEFAULT                                                  0x00000400
-#define mmSDMA1_EDC_COUNTER_DEFAULT                                              0x00000000
-#define mmSDMA1_EDC_COUNTER_CLEAR_DEFAULT                                        0x00000000
-#define mmSDMA1_STATUS2_REG_DEFAULT                                              0x00000001
-#define mmSDMA1_ATOMIC_CNTL_DEFAULT                                              0x00000200
-#define mmSDMA1_ATOMIC_PREOP_LO_DEFAULT                                          0x00000000
-#define mmSDMA1_ATOMIC_PREOP_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_UTCL1_CNTL_DEFAULT                                               0xd0003019
-#define mmSDMA1_UTCL1_WATERMK_DEFAULT                                            0xfffbe1fe
-#define mmSDMA1_UTCL1_RD_STATUS_DEFAULT                                          0x201001ff
-#define mmSDMA1_UTCL1_WR_STATUS_DEFAULT                                          0x503001ff
-#define mmSDMA1_UTCL1_INV0_DEFAULT                                               0x00000600
-#define mmSDMA1_UTCL1_INV1_DEFAULT                                               0x00000000
-#define mmSDMA1_UTCL1_INV2_DEFAULT                                               0x00000000
-#define mmSDMA1_UTCL1_RD_XNACK0_DEFAULT                                          0x00000000
-#define mmSDMA1_UTCL1_RD_XNACK1_DEFAULT                                          0x00000000
-#define mmSDMA1_UTCL1_WR_XNACK0_DEFAULT                                          0x00000000
-#define mmSDMA1_UTCL1_WR_XNACK1_DEFAULT                                          0x00000000
-#define mmSDMA1_UTCL1_TIMEOUT_DEFAULT                                            0x00010001
-#define mmSDMA1_UTCL1_PAGE_DEFAULT                                               0x000003e0
-#define mmSDMA1_POWER_CNTL_IDLE_DEFAULT                                          0x06060200
-#define mmSDMA1_RELAX_ORDERING_LUT_DEFAULT                                       0xc0000006
-#define mmSDMA1_CHICKEN_BITS_2_DEFAULT                                           0x00000005
-#define mmSDMA1_STATUS3_REG_DEFAULT                                              0x00100000
-#define mmSDMA1_PHYSICAL_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA1_PHYSICAL_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA1_PHASE2_QUANTUM_DEFAULT                                           0x00010002
-#define mmSDMA1_ERROR_LOG_DEFAULT                                                0x0000000f
-#define mmSDMA1_PUB_DUMMY_REG0_DEFAULT                                           0x00000000
-#define mmSDMA1_PUB_DUMMY_REG1_DEFAULT                                           0x00000000
-#define mmSDMA1_PUB_DUMMY_REG2_DEFAULT                                           0x00000000
-#define mmSDMA1_PUB_DUMMY_REG3_DEFAULT                                           0x00000000
-#define mmSDMA1_F32_COUNTER_DEFAULT                                              0x00000000
-#define mmSDMA1_UNBREAKABLE_DEFAULT                                              0x00000000
-#define mmSDMA1_PERFMON_CNTL_DEFAULT                                             0x000ff7fd
-#define mmSDMA1_PERFCOUNTER0_RESULT_DEFAULT                                      0x00000000
-#define mmSDMA1_PERFCOUNTER1_RESULT_DEFAULT                                      0x00000000
-#define mmSDMA1_PERFCOUNTER_TAG_DELAY_RANGE_DEFAULT                              0x00640000
-#define mmSDMA1_CRD_CNTL_DEFAULT                                                 0x000085c0
-#define mmSDMA1_MMHUB_TRUSTLVL_DEFAULT                                           0x00000000
-#define mmSDMA1_GPU_IOV_VIOLATION_LOG_DEFAULT                                    0x00000000
-#define mmSDMA1_ULV_CNTL_DEFAULT                                                 0x00000000
-#define mmSDMA1_EA_DBIT_ADDR_DATA_DEFAULT                                        0x00000000
-#define mmSDMA1_EA_DBIT_ADDR_INDEX_DEFAULT                                       0x00000000
-#define mmSDMA1_GFX_RB_CNTL_DEFAULT                                              0x00040000
-#define mmSDMA1_GFX_RB_BASE_DEFAULT                                              0x00000000
-#define mmSDMA1_GFX_RB_BASE_HI_DEFAULT                                           0x00000000
-#define mmSDMA1_GFX_RB_RPTR_DEFAULT                                              0x00000000
-#define mmSDMA1_GFX_RB_RPTR_HI_DEFAULT                                           0x00000000
-#define mmSDMA1_GFX_RB_WPTR_DEFAULT                                              0x00000000
-#define mmSDMA1_GFX_RB_WPTR_HI_DEFAULT                                           0x00000000
-#define mmSDMA1_GFX_RB_WPTR_POLL_CNTL_DEFAULT                                    0x00401000
-#define mmSDMA1_GFX_RB_RPTR_ADDR_HI_DEFAULT                                      0x00000000
-#define mmSDMA1_GFX_RB_RPTR_ADDR_LO_DEFAULT                                      0x00000000
-#define mmSDMA1_GFX_IB_CNTL_DEFAULT                                              0x00000100
-#define mmSDMA1_GFX_IB_RPTR_DEFAULT                                              0x00000000
-#define mmSDMA1_GFX_IB_OFFSET_DEFAULT                                            0x00000000
-#define mmSDMA1_GFX_IB_BASE_LO_DEFAULT                                           0x00000000
-#define mmSDMA1_GFX_IB_BASE_HI_DEFAULT                                           0x00000000
-#define mmSDMA1_GFX_IB_SIZE_DEFAULT                                              0x00000000
-#define mmSDMA1_GFX_SKIP_CNTL_DEFAULT                                            0x00000000
-#define mmSDMA1_GFX_CONTEXT_STATUS_DEFAULT                                       0x00000005
-#define mmSDMA1_GFX_DOORBELL_DEFAULT                                             0x00000000
-#define mmSDMA1_GFX_CONTEXT_CNTL_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_STATUS_DEFAULT                                               0x00000000
-#define mmSDMA1_GFX_DOORBELL_LOG_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_WATERMARK_DEFAULT                                            0x00000000
-#define mmSDMA1_GFX_DOORBELL_OFFSET_DEFAULT                                      0x00000000
-#define mmSDMA1_GFX_CSA_ADDR_LO_DEFAULT                                          0x00000000
-#define mmSDMA1_GFX_CSA_ADDR_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_GFX_IB_SUB_REMAIN_DEFAULT                                        0x00000000
-#define mmSDMA1_GFX_PREEMPT_DEFAULT                                              0x00000000
-#define mmSDMA1_GFX_DUMMY_REG_DEFAULT                                            0x0000000f
-#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_HI_DEFAULT                                 0x00000000
-#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_LO_DEFAULT                                 0x00000000
-#define mmSDMA1_GFX_RB_AQL_CNTL_DEFAULT                                          0x00004000
-#define mmSDMA1_GFX_MINOR_PTR_UPDATE_DEFAULT                                     0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA0_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA1_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA2_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA3_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA4_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA5_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA6_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA7_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_DATA8_DEFAULT                                         0x00000000
-#define mmSDMA1_GFX_MIDCMD_CNTL_DEFAULT                                          0x00000000
-#define mmSDMA1_PAGE_RB_CNTL_DEFAULT                                             0x00040000
-#define mmSDMA1_PAGE_RB_BASE_DEFAULT                                             0x00000000
-#define mmSDMA1_PAGE_RB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_PAGE_RB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_PAGE_RB_RPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_PAGE_RB_WPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_PAGE_RB_WPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_PAGE_RB_WPTR_POLL_CNTL_DEFAULT                                   0x00401000
-#define mmSDMA1_PAGE_RB_RPTR_ADDR_HI_DEFAULT                                     0x00000000
-#define mmSDMA1_PAGE_RB_RPTR_ADDR_LO_DEFAULT                                     0x00000000
-#define mmSDMA1_PAGE_IB_CNTL_DEFAULT                                             0x00000100
-#define mmSDMA1_PAGE_IB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_PAGE_IB_OFFSET_DEFAULT                                           0x00000000
-#define mmSDMA1_PAGE_IB_BASE_LO_DEFAULT                                          0x00000000
-#define mmSDMA1_PAGE_IB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_PAGE_IB_SIZE_DEFAULT                                             0x00000000
-#define mmSDMA1_PAGE_SKIP_CNTL_DEFAULT                                           0x00000000
-#define mmSDMA1_PAGE_CONTEXT_STATUS_DEFAULT                                      0x00000004
-#define mmSDMA1_PAGE_DOORBELL_DEFAULT                                            0x00000000
-#define mmSDMA1_PAGE_STATUS_DEFAULT                                              0x00000000
-#define mmSDMA1_PAGE_DOORBELL_LOG_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_WATERMARK_DEFAULT                                           0x00000000
-#define mmSDMA1_PAGE_DOORBELL_OFFSET_DEFAULT                                     0x00000000
-#define mmSDMA1_PAGE_CSA_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA1_PAGE_CSA_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA1_PAGE_IB_SUB_REMAIN_DEFAULT                                       0x00000000
-#define mmSDMA1_PAGE_PREEMPT_DEFAULT                                             0x00000000
-#define mmSDMA1_PAGE_DUMMY_REG_DEFAULT                                           0x0000000f
-#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_HI_DEFAULT                                0x00000000
-#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_LO_DEFAULT                                0x00000000
-#define mmSDMA1_PAGE_RB_AQL_CNTL_DEFAULT                                         0x00004000
-#define mmSDMA1_PAGE_MINOR_PTR_UPDATE_DEFAULT                                    0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA0_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA1_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA2_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA3_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA4_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA5_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA6_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA7_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_DATA8_DEFAULT                                        0x00000000
-#define mmSDMA1_PAGE_MIDCMD_CNTL_DEFAULT                                         0x00000000
-#define mmSDMA1_RLC0_RB_CNTL_DEFAULT                                             0x00040000
-#define mmSDMA1_RLC0_RB_BASE_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC0_RB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC0_RB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC0_RB_RPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC0_RB_WPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC0_RB_WPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC0_RB_WPTR_POLL_CNTL_DEFAULT                                   0x00401000
-#define mmSDMA1_RLC0_RB_RPTR_ADDR_HI_DEFAULT                                     0x00000000
-#define mmSDMA1_RLC0_RB_RPTR_ADDR_LO_DEFAULT                                     0x00000000
-#define mmSDMA1_RLC0_IB_CNTL_DEFAULT                                             0x00000100
-#define mmSDMA1_RLC0_IB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC0_IB_OFFSET_DEFAULT                                           0x00000000
-#define mmSDMA1_RLC0_IB_BASE_LO_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC0_IB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC0_IB_SIZE_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC0_SKIP_CNTL_DEFAULT                                           0x00000000
-#define mmSDMA1_RLC0_CONTEXT_STATUS_DEFAULT                                      0x00000004
-#define mmSDMA1_RLC0_DOORBELL_DEFAULT                                            0x00000000
-#define mmSDMA1_RLC0_STATUS_DEFAULT                                              0x00000000
-#define mmSDMA1_RLC0_DOORBELL_LOG_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_WATERMARK_DEFAULT                                           0x00000000
-#define mmSDMA1_RLC0_DOORBELL_OFFSET_DEFAULT                                     0x00000000
-#define mmSDMA1_RLC0_CSA_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA1_RLC0_CSA_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA1_RLC0_IB_SUB_REMAIN_DEFAULT                                       0x00000000
-#define mmSDMA1_RLC0_PREEMPT_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC0_DUMMY_REG_DEFAULT                                           0x0000000f
-#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_HI_DEFAULT                                0x00000000
-#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_LO_DEFAULT                                0x00000000
-#define mmSDMA1_RLC0_RB_AQL_CNTL_DEFAULT                                         0x00004000
-#define mmSDMA1_RLC0_MINOR_PTR_UPDATE_DEFAULT                                    0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA0_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA1_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA2_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA3_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA4_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA5_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA6_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA7_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_DATA8_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC0_MIDCMD_CNTL_DEFAULT                                         0x00000000
-#define mmSDMA1_RLC1_RB_CNTL_DEFAULT                                             0x00040000
-#define mmSDMA1_RLC1_RB_BASE_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC1_RB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC1_RB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC1_RB_RPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC1_RB_WPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC1_RB_WPTR_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC1_RB_WPTR_POLL_CNTL_DEFAULT                                   0x00401000
-#define mmSDMA1_RLC1_RB_RPTR_ADDR_HI_DEFAULT                                     0x00000000
-#define mmSDMA1_RLC1_RB_RPTR_ADDR_LO_DEFAULT                                     0x00000000
-#define mmSDMA1_RLC1_IB_CNTL_DEFAULT                                             0x00000100
-#define mmSDMA1_RLC1_IB_RPTR_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC1_IB_OFFSET_DEFAULT                                           0x00000000
-#define mmSDMA1_RLC1_IB_BASE_LO_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC1_IB_BASE_HI_DEFAULT                                          0x00000000
-#define mmSDMA1_RLC1_IB_SIZE_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC1_SKIP_CNTL_DEFAULT                                           0x00000000
-#define mmSDMA1_RLC1_CONTEXT_STATUS_DEFAULT                                      0x00000004
-#define mmSDMA1_RLC1_DOORBELL_DEFAULT                                            0x00000000
-#define mmSDMA1_RLC1_STATUS_DEFAULT                                              0x00000000
-#define mmSDMA1_RLC1_DOORBELL_LOG_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_WATERMARK_DEFAULT                                           0x00000000
-#define mmSDMA1_RLC1_DOORBELL_OFFSET_DEFAULT                                     0x00000000
-#define mmSDMA1_RLC1_CSA_ADDR_LO_DEFAULT                                         0x00000000
-#define mmSDMA1_RLC1_CSA_ADDR_HI_DEFAULT                                         0x00000000
-#define mmSDMA1_RLC1_IB_SUB_REMAIN_DEFAULT                                       0x00000000
-#define mmSDMA1_RLC1_PREEMPT_DEFAULT                                             0x00000000
-#define mmSDMA1_RLC1_DUMMY_REG_DEFAULT                                           0x0000000f
-#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_HI_DEFAULT                                0x00000000
-#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_LO_DEFAULT                                0x00000000
-#define mmSDMA1_RLC1_RB_AQL_CNTL_DEFAULT                                         0x00004000
-#define mmSDMA1_RLC1_MINOR_PTR_UPDATE_DEFAULT                                    0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA0_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA1_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA2_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA3_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA4_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA5_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA6_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA7_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_DATA8_DEFAULT                                        0x00000000
-#define mmSDMA1_RLC1_MIDCMD_CNTL_DEFAULT                                         0x00000000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_offset.h
deleted file mode 100644
index 92150d6b65b8..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_offset.h
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _sdma1_4_0_OFFSET_HEADER
-#define _sdma1_4_0_OFFSET_HEADER
-
-
-
-// addressBlock: sdma1_sdma1dec
-// base address: 0x5180
-#define mmSDMA1_UCODE_ADDR                                                                             0x0000
-#define mmSDMA1_UCODE_ADDR_BASE_IDX                                                                    0
-#define mmSDMA1_UCODE_DATA                                                                             0x0001
-#define mmSDMA1_UCODE_DATA_BASE_IDX                                                                    0
-#define mmSDMA1_VM_CNTL                                                                                0x0004
-#define mmSDMA1_VM_CNTL_BASE_IDX                                                                       0
-#define mmSDMA1_VM_CTX_LO                                                                              0x0005
-#define mmSDMA1_VM_CTX_LO_BASE_IDX                                                                     0
-#define mmSDMA1_VM_CTX_HI                                                                              0x0006
-#define mmSDMA1_VM_CTX_HI_BASE_IDX                                                                     0
-#define mmSDMA1_ACTIVE_FCN_ID                                                                          0x0007
-#define mmSDMA1_ACTIVE_FCN_ID_BASE_IDX                                                                 0
-#define mmSDMA1_VM_CTX_CNTL                                                                            0x0008
-#define mmSDMA1_VM_CTX_CNTL_BASE_IDX                                                                   0
-#define mmSDMA1_VIRT_RESET_REQ                                                                         0x0009
-#define mmSDMA1_VIRT_RESET_REQ_BASE_IDX                                                                0
-#define mmSDMA1_VF_ENABLE                                                                              0x000a
-#define mmSDMA1_VF_ENABLE_BASE_IDX                                                                     0
-#define mmSDMA1_CONTEXT_REG_TYPE0                                                                      0x000b
-#define mmSDMA1_CONTEXT_REG_TYPE0_BASE_IDX                                                             0
-#define mmSDMA1_CONTEXT_REG_TYPE1                                                                      0x000c
-#define mmSDMA1_CONTEXT_REG_TYPE1_BASE_IDX                                                             0
-#define mmSDMA1_CONTEXT_REG_TYPE2                                                                      0x000d
-#define mmSDMA1_CONTEXT_REG_TYPE2_BASE_IDX                                                             0
-#define mmSDMA1_CONTEXT_REG_TYPE3                                                                      0x000e
-#define mmSDMA1_CONTEXT_REG_TYPE3_BASE_IDX                                                             0
-#define mmSDMA1_PUB_REG_TYPE0                                                                          0x000f
-#define mmSDMA1_PUB_REG_TYPE0_BASE_IDX                                                                 0
-#define mmSDMA1_PUB_REG_TYPE1                                                                          0x0010
-#define mmSDMA1_PUB_REG_TYPE1_BASE_IDX                                                                 0
-#define mmSDMA1_PUB_REG_TYPE2                                                                          0x0011
-#define mmSDMA1_PUB_REG_TYPE2_BASE_IDX                                                                 0
-#define mmSDMA1_PUB_REG_TYPE3                                                                          0x0012
-#define mmSDMA1_PUB_REG_TYPE3_BASE_IDX                                                                 0
-#define mmSDMA1_MMHUB_CNTL                                                                             0x0013
-#define mmSDMA1_MMHUB_CNTL_BASE_IDX                                                                    0
-#define mmSDMA1_CONTEXT_GROUP_BOUNDARY                                                                 0x0019
-#define mmSDMA1_CONTEXT_GROUP_BOUNDARY_BASE_IDX                                                        0
-#define mmSDMA1_POWER_CNTL                                                                             0x001a
-#define mmSDMA1_POWER_CNTL_BASE_IDX                                                                    0
-#define mmSDMA1_CLK_CTRL                                                                               0x001b
-#define mmSDMA1_CLK_CTRL_BASE_IDX                                                                      0
-#define mmSDMA1_CNTL                                                                                   0x001c
-#define mmSDMA1_CNTL_BASE_IDX                                                                          0
-#define mmSDMA1_CHICKEN_BITS                                                                           0x001d
-#define mmSDMA1_CHICKEN_BITS_BASE_IDX                                                                  0
-#define mmSDMA1_GB_ADDR_CONFIG                                                                         0x001e
-#define mmSDMA1_GB_ADDR_CONFIG_BASE_IDX                                                                0
-#define mmSDMA1_GB_ADDR_CONFIG_READ                                                                    0x001f
-#define mmSDMA1_GB_ADDR_CONFIG_READ_BASE_IDX                                                           0
-#define mmSDMA1_RB_RPTR_FETCH_HI                                                                       0x0020
-#define mmSDMA1_RB_RPTR_FETCH_HI_BASE_IDX                                                              0
-#define mmSDMA1_SEM_WAIT_FAIL_TIMER_CNTL                                                               0x0021
-#define mmSDMA1_SEM_WAIT_FAIL_TIMER_CNTL_BASE_IDX                                                      0
-#define mmSDMA1_RB_RPTR_FETCH                                                                          0x0022
-#define mmSDMA1_RB_RPTR_FETCH_BASE_IDX                                                                 0
-#define mmSDMA1_IB_OFFSET_FETCH                                                                        0x0023
-#define mmSDMA1_IB_OFFSET_FETCH_BASE_IDX                                                               0
-#define mmSDMA1_PROGRAM                                                                                0x0024
-#define mmSDMA1_PROGRAM_BASE_IDX                                                                       0
-#define mmSDMA1_STATUS_REG                                                                             0x0025
-#define mmSDMA1_STATUS_REG_BASE_IDX                                                                    0
-#define mmSDMA1_STATUS1_REG                                                                            0x0026
-#define mmSDMA1_STATUS1_REG_BASE_IDX                                                                   0
-#define mmSDMA1_RD_BURST_CNTL                                                                          0x0027
-#define mmSDMA1_RD_BURST_CNTL_BASE_IDX                                                                 0
-#define mmSDMA1_HBM_PAGE_CONFIG                                                                        0x0028
-#define mmSDMA1_HBM_PAGE_CONFIG_BASE_IDX                                                               0
-#define mmSDMA1_UCODE_CHECKSUM                                                                         0x0029
-#define mmSDMA1_UCODE_CHECKSUM_BASE_IDX                                                                0
-#define mmSDMA1_F32_CNTL                                                                               0x002a
-#define mmSDMA1_F32_CNTL_BASE_IDX                                                                      0
-#define mmSDMA1_FREEZE                                                                                 0x002b
-#define mmSDMA1_FREEZE_BASE_IDX                                                                        0
-#define mmSDMA1_PHASE0_QUANTUM                                                                         0x002c
-#define mmSDMA1_PHASE0_QUANTUM_BASE_IDX                                                                0
-#define mmSDMA1_PHASE1_QUANTUM                                                                         0x002d
-#define mmSDMA1_PHASE1_QUANTUM_BASE_IDX                                                                0
-#define mmSDMA1_EDC_CONFIG                                                                             0x0032
-#define mmSDMA1_EDC_CONFIG_BASE_IDX                                                                    0
-#define mmSDMA1_BA_THRESHOLD                                                                           0x0033
-#define mmSDMA1_BA_THRESHOLD_BASE_IDX                                                                  0
-#define mmSDMA1_ID                                                                                     0x0034
-#define mmSDMA1_ID_BASE_IDX                                                                            0
-#define mmSDMA1_VERSION                                                                                0x0035
-#define mmSDMA1_VERSION_BASE_IDX                                                                       0
-#define mmSDMA1_EDC_COUNTER                                                                            0x0036
-#define mmSDMA1_EDC_COUNTER_BASE_IDX                                                                   0
-#define mmSDMA1_EDC_COUNTER_CLEAR                                                                      0x0037
-#define mmSDMA1_EDC_COUNTER_CLEAR_BASE_IDX                                                             0
-#define mmSDMA1_STATUS2_REG                                                                            0x0038
-#define mmSDMA1_STATUS2_REG_BASE_IDX                                                                   0
-#define mmSDMA1_ATOMIC_CNTL                                                                            0x0039
-#define mmSDMA1_ATOMIC_CNTL_BASE_IDX                                                                   0
-#define mmSDMA1_ATOMIC_PREOP_LO                                                                        0x003a
-#define mmSDMA1_ATOMIC_PREOP_LO_BASE_IDX                                                               0
-#define mmSDMA1_ATOMIC_PREOP_HI                                                                        0x003b
-#define mmSDMA1_ATOMIC_PREOP_HI_BASE_IDX                                                               0
-#define mmSDMA1_UTCL1_CNTL                                                                             0x003c
-#define mmSDMA1_UTCL1_CNTL_BASE_IDX                                                                    0
-#define mmSDMA1_UTCL1_WATERMK                                                                          0x003d
-#define mmSDMA1_UTCL1_WATERMK_BASE_IDX                                                                 0
-#define mmSDMA1_UTCL1_RD_STATUS                                                                        0x003e
-#define mmSDMA1_UTCL1_RD_STATUS_BASE_IDX                                                               0
-#define mmSDMA1_UTCL1_WR_STATUS                                                                        0x003f
-#define mmSDMA1_UTCL1_WR_STATUS_BASE_IDX                                                               0
-#define mmSDMA1_UTCL1_INV0                                                                             0x0040
-#define mmSDMA1_UTCL1_INV0_BASE_IDX                                                                    0
-#define mmSDMA1_UTCL1_INV1                                                                             0x0041
-#define mmSDMA1_UTCL1_INV1_BASE_IDX                                                                    0
-#define mmSDMA1_UTCL1_INV2                                                                             0x0042
-#define mmSDMA1_UTCL1_INV2_BASE_IDX                                                                    0
-#define mmSDMA1_UTCL1_RD_XNACK0                                                                        0x0043
-#define mmSDMA1_UTCL1_RD_XNACK0_BASE_IDX                                                               0
-#define mmSDMA1_UTCL1_RD_XNACK1                                                                        0x0044
-#define mmSDMA1_UTCL1_RD_XNACK1_BASE_IDX                                                               0
-#define mmSDMA1_UTCL1_WR_XNACK0                                                                        0x0045
-#define mmSDMA1_UTCL1_WR_XNACK0_BASE_IDX                                                               0
-#define mmSDMA1_UTCL1_WR_XNACK1                                                                        0x0046
-#define mmSDMA1_UTCL1_WR_XNACK1_BASE_IDX                                                               0
-#define mmSDMA1_UTCL1_TIMEOUT                                                                          0x0047
-#define mmSDMA1_UTCL1_TIMEOUT_BASE_IDX                                                                 0
-#define mmSDMA1_UTCL1_PAGE                                                                             0x0048
-#define mmSDMA1_UTCL1_PAGE_BASE_IDX                                                                    0
-#define mmSDMA1_POWER_CNTL_IDLE                                                                        0x0049
-#define mmSDMA1_POWER_CNTL_IDLE_BASE_IDX                                                               0
-#define mmSDMA1_RELAX_ORDERING_LUT                                                                     0x004a
-#define mmSDMA1_RELAX_ORDERING_LUT_BASE_IDX                                                            0
-#define mmSDMA1_CHICKEN_BITS_2                                                                         0x004b
-#define mmSDMA1_CHICKEN_BITS_2_BASE_IDX                                                                0
-#define mmSDMA1_STATUS3_REG                                                                            0x004c
-#define mmSDMA1_STATUS3_REG_BASE_IDX                                                                   0
-#define mmSDMA1_PHYSICAL_ADDR_LO                                                                       0x004d
-#define mmSDMA1_PHYSICAL_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA1_PHYSICAL_ADDR_HI                                                                       0x004e
-#define mmSDMA1_PHYSICAL_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA1_PHASE2_QUANTUM                                                                         0x004f
-#define mmSDMA1_PHASE2_QUANTUM_BASE_IDX                                                                0
-#define mmSDMA1_ERROR_LOG                                                                              0x0050
-#define mmSDMA1_ERROR_LOG_BASE_IDX                                                                     0
-#define mmSDMA1_PUB_DUMMY_REG0                                                                         0x0051
-#define mmSDMA1_PUB_DUMMY_REG0_BASE_IDX                                                                0
-#define mmSDMA1_PUB_DUMMY_REG1                                                                         0x0052
-#define mmSDMA1_PUB_DUMMY_REG1_BASE_IDX                                                                0
-#define mmSDMA1_PUB_DUMMY_REG2                                                                         0x0053
-#define mmSDMA1_PUB_DUMMY_REG2_BASE_IDX                                                                0
-#define mmSDMA1_PUB_DUMMY_REG3                                                                         0x0054
-#define mmSDMA1_PUB_DUMMY_REG3_BASE_IDX                                                                0
-#define mmSDMA1_F32_COUNTER                                                                            0x0055
-#define mmSDMA1_F32_COUNTER_BASE_IDX                                                                   0
-#define mmSDMA1_UNBREAKABLE                                                                            0x0056
-#define mmSDMA1_UNBREAKABLE_BASE_IDX                                                                   0
-#define mmSDMA1_PERFMON_CNTL                                                                           0x0057
-#define mmSDMA1_PERFMON_CNTL_BASE_IDX                                                                  0
-#define mmSDMA1_PERFCOUNTER0_RESULT                                                                    0x0058
-#define mmSDMA1_PERFCOUNTER0_RESULT_BASE_IDX                                                           0
-#define mmSDMA1_PERFCOUNTER1_RESULT                                                                    0x0059
-#define mmSDMA1_PERFCOUNTER1_RESULT_BASE_IDX                                                           0
-#define mmSDMA1_PERFCOUNTER_TAG_DELAY_RANGE                                                            0x005a
-#define mmSDMA1_PERFCOUNTER_TAG_DELAY_RANGE_BASE_IDX                                                   0
-#define mmSDMA1_CRD_CNTL                                                                               0x005b
-#define mmSDMA1_CRD_CNTL_BASE_IDX                                                                      0
-#define mmSDMA1_MMHUB_TRUSTLVL                                                                         0x005c
-#define mmSDMA1_MMHUB_TRUSTLVL_BASE_IDX                                                                0
-#define mmSDMA1_GPU_IOV_VIOLATION_LOG                                                                  0x005d
-#define mmSDMA1_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                         0
-#define mmSDMA1_ULV_CNTL                                                                               0x005e
-#define mmSDMA1_ULV_CNTL_BASE_IDX                                                                      0
-#define mmSDMA1_EA_DBIT_ADDR_DATA                                                                      0x0060
-#define mmSDMA1_EA_DBIT_ADDR_DATA_BASE_IDX                                                             0
-#define mmSDMA1_EA_DBIT_ADDR_INDEX                                                                     0x0061
-#define mmSDMA1_EA_DBIT_ADDR_INDEX_BASE_IDX                                                            0
-#define mmSDMA1_GFX_RB_CNTL                                                                            0x0080
-#define mmSDMA1_GFX_RB_CNTL_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_RB_BASE                                                                            0x0081
-#define mmSDMA1_GFX_RB_BASE_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_RB_BASE_HI                                                                         0x0082
-#define mmSDMA1_GFX_RB_BASE_HI_BASE_IDX                                                                0
-#define mmSDMA1_GFX_RB_RPTR                                                                            0x0083
-#define mmSDMA1_GFX_RB_RPTR_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_RB_RPTR_HI                                                                         0x0084
-#define mmSDMA1_GFX_RB_RPTR_HI_BASE_IDX                                                                0
-#define mmSDMA1_GFX_RB_WPTR                                                                            0x0085
-#define mmSDMA1_GFX_RB_WPTR_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_RB_WPTR_HI                                                                         0x0086
-#define mmSDMA1_GFX_RB_WPTR_HI_BASE_IDX                                                                0
-#define mmSDMA1_GFX_RB_WPTR_POLL_CNTL                                                                  0x0087
-#define mmSDMA1_GFX_RB_WPTR_POLL_CNTL_BASE_IDX                                                         0
-#define mmSDMA1_GFX_RB_RPTR_ADDR_HI                                                                    0x0088
-#define mmSDMA1_GFX_RB_RPTR_ADDR_HI_BASE_IDX                                                           0
-#define mmSDMA1_GFX_RB_RPTR_ADDR_LO                                                                    0x0089
-#define mmSDMA1_GFX_RB_RPTR_ADDR_LO_BASE_IDX                                                           0
-#define mmSDMA1_GFX_IB_CNTL                                                                            0x008a
-#define mmSDMA1_GFX_IB_CNTL_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_IB_RPTR                                                                            0x008b
-#define mmSDMA1_GFX_IB_RPTR_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_IB_OFFSET                                                                          0x008c
-#define mmSDMA1_GFX_IB_OFFSET_BASE_IDX                                                                 0
-#define mmSDMA1_GFX_IB_BASE_LO                                                                         0x008d
-#define mmSDMA1_GFX_IB_BASE_LO_BASE_IDX                                                                0
-#define mmSDMA1_GFX_IB_BASE_HI                                                                         0x008e
-#define mmSDMA1_GFX_IB_BASE_HI_BASE_IDX                                                                0
-#define mmSDMA1_GFX_IB_SIZE                                                                            0x008f
-#define mmSDMA1_GFX_IB_SIZE_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_SKIP_CNTL                                                                          0x0090
-#define mmSDMA1_GFX_SKIP_CNTL_BASE_IDX                                                                 0
-#define mmSDMA1_GFX_CONTEXT_STATUS                                                                     0x0091
-#define mmSDMA1_GFX_CONTEXT_STATUS_BASE_IDX                                                            0
-#define mmSDMA1_GFX_DOORBELL                                                                           0x0092
-#define mmSDMA1_GFX_DOORBELL_BASE_IDX                                                                  0
-#define mmSDMA1_GFX_CONTEXT_CNTL                                                                       0x0093
-#define mmSDMA1_GFX_CONTEXT_CNTL_BASE_IDX                                                              0
-#define mmSDMA1_GFX_STATUS                                                                             0x00a8
-#define mmSDMA1_GFX_STATUS_BASE_IDX                                                                    0
-#define mmSDMA1_GFX_DOORBELL_LOG                                                                       0x00a9
-#define mmSDMA1_GFX_DOORBELL_LOG_BASE_IDX                                                              0
-#define mmSDMA1_GFX_WATERMARK                                                                          0x00aa
-#define mmSDMA1_GFX_WATERMARK_BASE_IDX                                                                 0
-#define mmSDMA1_GFX_DOORBELL_OFFSET                                                                    0x00ab
-#define mmSDMA1_GFX_DOORBELL_OFFSET_BASE_IDX                                                           0
-#define mmSDMA1_GFX_CSA_ADDR_LO                                                                        0x00ac
-#define mmSDMA1_GFX_CSA_ADDR_LO_BASE_IDX                                                               0
-#define mmSDMA1_GFX_CSA_ADDR_HI                                                                        0x00ad
-#define mmSDMA1_GFX_CSA_ADDR_HI_BASE_IDX                                                               0
-#define mmSDMA1_GFX_IB_SUB_REMAIN                                                                      0x00af
-#define mmSDMA1_GFX_IB_SUB_REMAIN_BASE_IDX                                                             0
-#define mmSDMA1_GFX_PREEMPT                                                                            0x00b0
-#define mmSDMA1_GFX_PREEMPT_BASE_IDX                                                                   0
-#define mmSDMA1_GFX_DUMMY_REG                                                                          0x00b1
-#define mmSDMA1_GFX_DUMMY_REG_BASE_IDX                                                                 0
-#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_HI                                                               0x00b2
-#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                      0
-#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_LO                                                               0x00b3
-#define mmSDMA1_GFX_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                      0
-#define mmSDMA1_GFX_RB_AQL_CNTL                                                                        0x00b4
-#define mmSDMA1_GFX_RB_AQL_CNTL_BASE_IDX                                                               0
-#define mmSDMA1_GFX_MINOR_PTR_UPDATE                                                                   0x00b5
-#define mmSDMA1_GFX_MINOR_PTR_UPDATE_BASE_IDX                                                          0
-#define mmSDMA1_GFX_MIDCMD_DATA0                                                                       0x00c0
-#define mmSDMA1_GFX_MIDCMD_DATA0_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA1                                                                       0x00c1
-#define mmSDMA1_GFX_MIDCMD_DATA1_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA2                                                                       0x00c2
-#define mmSDMA1_GFX_MIDCMD_DATA2_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA3                                                                       0x00c3
-#define mmSDMA1_GFX_MIDCMD_DATA3_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA4                                                                       0x00c4
-#define mmSDMA1_GFX_MIDCMD_DATA4_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA5                                                                       0x00c5
-#define mmSDMA1_GFX_MIDCMD_DATA5_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA6                                                                       0x00c6
-#define mmSDMA1_GFX_MIDCMD_DATA6_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA7                                                                       0x00c7
-#define mmSDMA1_GFX_MIDCMD_DATA7_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_DATA8                                                                       0x00c8
-#define mmSDMA1_GFX_MIDCMD_DATA8_BASE_IDX                                                              0
-#define mmSDMA1_GFX_MIDCMD_CNTL                                                                        0x00c9
-#define mmSDMA1_GFX_MIDCMD_CNTL_BASE_IDX                                                               0
-#define mmSDMA1_PAGE_RB_CNTL                                                                           0x00e0
-#define mmSDMA1_PAGE_RB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_RB_BASE                                                                           0x00e1
-#define mmSDMA1_PAGE_RB_BASE_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_RB_BASE_HI                                                                        0x00e2
-#define mmSDMA1_PAGE_RB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA1_PAGE_RB_RPTR                                                                           0x00e3
-#define mmSDMA1_PAGE_RB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_RB_RPTR_HI                                                                        0x00e4
-#define mmSDMA1_PAGE_RB_RPTR_HI_BASE_IDX                                                               0
-#define mmSDMA1_PAGE_RB_WPTR                                                                           0x00e5
-#define mmSDMA1_PAGE_RB_WPTR_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_RB_WPTR_HI                                                                        0x00e6
-#define mmSDMA1_PAGE_RB_WPTR_HI_BASE_IDX                                                               0
-#define mmSDMA1_PAGE_RB_WPTR_POLL_CNTL                                                                 0x00e7
-#define mmSDMA1_PAGE_RB_WPTR_POLL_CNTL_BASE_IDX                                                        0
-#define mmSDMA1_PAGE_RB_RPTR_ADDR_HI                                                                   0x00e8
-#define mmSDMA1_PAGE_RB_RPTR_ADDR_HI_BASE_IDX                                                          0
-#define mmSDMA1_PAGE_RB_RPTR_ADDR_LO                                                                   0x00e9
-#define mmSDMA1_PAGE_RB_RPTR_ADDR_LO_BASE_IDX                                                          0
-#define mmSDMA1_PAGE_IB_CNTL                                                                           0x00ea
-#define mmSDMA1_PAGE_IB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_IB_RPTR                                                                           0x00eb
-#define mmSDMA1_PAGE_IB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_IB_OFFSET                                                                         0x00ec
-#define mmSDMA1_PAGE_IB_OFFSET_BASE_IDX                                                                0
-#define mmSDMA1_PAGE_IB_BASE_LO                                                                        0x00ed
-#define mmSDMA1_PAGE_IB_BASE_LO_BASE_IDX                                                               0
-#define mmSDMA1_PAGE_IB_BASE_HI                                                                        0x00ee
-#define mmSDMA1_PAGE_IB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA1_PAGE_IB_SIZE                                                                           0x00ef
-#define mmSDMA1_PAGE_IB_SIZE_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_SKIP_CNTL                                                                         0x00f0
-#define mmSDMA1_PAGE_SKIP_CNTL_BASE_IDX                                                                0
-#define mmSDMA1_PAGE_CONTEXT_STATUS                                                                    0x00f1
-#define mmSDMA1_PAGE_CONTEXT_STATUS_BASE_IDX                                                           0
-#define mmSDMA1_PAGE_DOORBELL                                                                          0x00f2
-#define mmSDMA1_PAGE_DOORBELL_BASE_IDX                                                                 0
-#define mmSDMA1_PAGE_STATUS                                                                            0x0108
-#define mmSDMA1_PAGE_STATUS_BASE_IDX                                                                   0
-#define mmSDMA1_PAGE_DOORBELL_LOG                                                                      0x0109
-#define mmSDMA1_PAGE_DOORBELL_LOG_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_WATERMARK                                                                         0x010a
-#define mmSDMA1_PAGE_WATERMARK_BASE_IDX                                                                0
-#define mmSDMA1_PAGE_DOORBELL_OFFSET                                                                   0x010b
-#define mmSDMA1_PAGE_DOORBELL_OFFSET_BASE_IDX                                                          0
-#define mmSDMA1_PAGE_CSA_ADDR_LO                                                                       0x010c
-#define mmSDMA1_PAGE_CSA_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA1_PAGE_CSA_ADDR_HI                                                                       0x010d
-#define mmSDMA1_PAGE_CSA_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA1_PAGE_IB_SUB_REMAIN                                                                     0x010f
-#define mmSDMA1_PAGE_IB_SUB_REMAIN_BASE_IDX                                                            0
-#define mmSDMA1_PAGE_PREEMPT                                                                           0x0110
-#define mmSDMA1_PAGE_PREEMPT_BASE_IDX                                                                  0
-#define mmSDMA1_PAGE_DUMMY_REG                                                                         0x0111
-#define mmSDMA1_PAGE_DUMMY_REG_BASE_IDX                                                                0
-#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_HI                                                              0x0112
-#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                     0
-#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_LO                                                              0x0113
-#define mmSDMA1_PAGE_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                     0
-#define mmSDMA1_PAGE_RB_AQL_CNTL                                                                       0x0114
-#define mmSDMA1_PAGE_RB_AQL_CNTL_BASE_IDX                                                              0
-#define mmSDMA1_PAGE_MINOR_PTR_UPDATE                                                                  0x0115
-#define mmSDMA1_PAGE_MINOR_PTR_UPDATE_BASE_IDX                                                         0
-#define mmSDMA1_PAGE_MIDCMD_DATA0                                                                      0x0120
-#define mmSDMA1_PAGE_MIDCMD_DATA0_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA1                                                                      0x0121
-#define mmSDMA1_PAGE_MIDCMD_DATA1_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA2                                                                      0x0122
-#define mmSDMA1_PAGE_MIDCMD_DATA2_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA3                                                                      0x0123
-#define mmSDMA1_PAGE_MIDCMD_DATA3_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA4                                                                      0x0124
-#define mmSDMA1_PAGE_MIDCMD_DATA4_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA5                                                                      0x0125
-#define mmSDMA1_PAGE_MIDCMD_DATA5_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA6                                                                      0x0126
-#define mmSDMA1_PAGE_MIDCMD_DATA6_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA7                                                                      0x0127
-#define mmSDMA1_PAGE_MIDCMD_DATA7_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_DATA8                                                                      0x0128
-#define mmSDMA1_PAGE_MIDCMD_DATA8_BASE_IDX                                                             0
-#define mmSDMA1_PAGE_MIDCMD_CNTL                                                                       0x0129
-#define mmSDMA1_PAGE_MIDCMD_CNTL_BASE_IDX                                                              0
-#define mmSDMA1_RLC0_RB_CNTL                                                                           0x0140
-#define mmSDMA1_RLC0_RB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_RB_BASE                                                                           0x0141
-#define mmSDMA1_RLC0_RB_BASE_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_RB_BASE_HI                                                                        0x0142
-#define mmSDMA1_RLC0_RB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC0_RB_RPTR                                                                           0x0143
-#define mmSDMA1_RLC0_RB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_RB_RPTR_HI                                                                        0x0144
-#define mmSDMA1_RLC0_RB_RPTR_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC0_RB_WPTR                                                                           0x0145
-#define mmSDMA1_RLC0_RB_WPTR_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_RB_WPTR_HI                                                                        0x0146
-#define mmSDMA1_RLC0_RB_WPTR_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC0_RB_WPTR_POLL_CNTL                                                                 0x0147
-#define mmSDMA1_RLC0_RB_WPTR_POLL_CNTL_BASE_IDX                                                        0
-#define mmSDMA1_RLC0_RB_RPTR_ADDR_HI                                                                   0x0148
-#define mmSDMA1_RLC0_RB_RPTR_ADDR_HI_BASE_IDX                                                          0
-#define mmSDMA1_RLC0_RB_RPTR_ADDR_LO                                                                   0x0149
-#define mmSDMA1_RLC0_RB_RPTR_ADDR_LO_BASE_IDX                                                          0
-#define mmSDMA1_RLC0_IB_CNTL                                                                           0x014a
-#define mmSDMA1_RLC0_IB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_IB_RPTR                                                                           0x014b
-#define mmSDMA1_RLC0_IB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_IB_OFFSET                                                                         0x014c
-#define mmSDMA1_RLC0_IB_OFFSET_BASE_IDX                                                                0
-#define mmSDMA1_RLC0_IB_BASE_LO                                                                        0x014d
-#define mmSDMA1_RLC0_IB_BASE_LO_BASE_IDX                                                               0
-#define mmSDMA1_RLC0_IB_BASE_HI                                                                        0x014e
-#define mmSDMA1_RLC0_IB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC0_IB_SIZE                                                                           0x014f
-#define mmSDMA1_RLC0_IB_SIZE_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_SKIP_CNTL                                                                         0x0150
-#define mmSDMA1_RLC0_SKIP_CNTL_BASE_IDX                                                                0
-#define mmSDMA1_RLC0_CONTEXT_STATUS                                                                    0x0151
-#define mmSDMA1_RLC0_CONTEXT_STATUS_BASE_IDX                                                           0
-#define mmSDMA1_RLC0_DOORBELL                                                                          0x0152
-#define mmSDMA1_RLC0_DOORBELL_BASE_IDX                                                                 0
-#define mmSDMA1_RLC0_STATUS                                                                            0x0168
-#define mmSDMA1_RLC0_STATUS_BASE_IDX                                                                   0
-#define mmSDMA1_RLC0_DOORBELL_LOG                                                                      0x0169
-#define mmSDMA1_RLC0_DOORBELL_LOG_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_WATERMARK                                                                         0x016a
-#define mmSDMA1_RLC0_WATERMARK_BASE_IDX                                                                0
-#define mmSDMA1_RLC0_DOORBELL_OFFSET                                                                   0x016b
-#define mmSDMA1_RLC0_DOORBELL_OFFSET_BASE_IDX                                                          0
-#define mmSDMA1_RLC0_CSA_ADDR_LO                                                                       0x016c
-#define mmSDMA1_RLC0_CSA_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA1_RLC0_CSA_ADDR_HI                                                                       0x016d
-#define mmSDMA1_RLC0_CSA_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA1_RLC0_IB_SUB_REMAIN                                                                     0x016f
-#define mmSDMA1_RLC0_IB_SUB_REMAIN_BASE_IDX                                                            0
-#define mmSDMA1_RLC0_PREEMPT                                                                           0x0170
-#define mmSDMA1_RLC0_PREEMPT_BASE_IDX                                                                  0
-#define mmSDMA1_RLC0_DUMMY_REG                                                                         0x0171
-#define mmSDMA1_RLC0_DUMMY_REG_BASE_IDX                                                                0
-#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_HI                                                              0x0172
-#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                     0
-#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_LO                                                              0x0173
-#define mmSDMA1_RLC0_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                     0
-#define mmSDMA1_RLC0_RB_AQL_CNTL                                                                       0x0174
-#define mmSDMA1_RLC0_RB_AQL_CNTL_BASE_IDX                                                              0
-#define mmSDMA1_RLC0_MINOR_PTR_UPDATE                                                                  0x0175
-#define mmSDMA1_RLC0_MINOR_PTR_UPDATE_BASE_IDX                                                         0
-#define mmSDMA1_RLC0_MIDCMD_DATA0                                                                      0x0180
-#define mmSDMA1_RLC0_MIDCMD_DATA0_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA1                                                                      0x0181
-#define mmSDMA1_RLC0_MIDCMD_DATA1_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA2                                                                      0x0182
-#define mmSDMA1_RLC0_MIDCMD_DATA2_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA3                                                                      0x0183
-#define mmSDMA1_RLC0_MIDCMD_DATA3_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA4                                                                      0x0184
-#define mmSDMA1_RLC0_MIDCMD_DATA4_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA5                                                                      0x0185
-#define mmSDMA1_RLC0_MIDCMD_DATA5_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA6                                                                      0x0186
-#define mmSDMA1_RLC0_MIDCMD_DATA6_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA7                                                                      0x0187
-#define mmSDMA1_RLC0_MIDCMD_DATA7_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_DATA8                                                                      0x0188
-#define mmSDMA1_RLC0_MIDCMD_DATA8_BASE_IDX                                                             0
-#define mmSDMA1_RLC0_MIDCMD_CNTL                                                                       0x0189
-#define mmSDMA1_RLC0_MIDCMD_CNTL_BASE_IDX                                                              0
-#define mmSDMA1_RLC1_RB_CNTL                                                                           0x01a0
-#define mmSDMA1_RLC1_RB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_RB_BASE                                                                           0x01a1
-#define mmSDMA1_RLC1_RB_BASE_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_RB_BASE_HI                                                                        0x01a2
-#define mmSDMA1_RLC1_RB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC1_RB_RPTR                                                                           0x01a3
-#define mmSDMA1_RLC1_RB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_RB_RPTR_HI                                                                        0x01a4
-#define mmSDMA1_RLC1_RB_RPTR_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC1_RB_WPTR                                                                           0x01a5
-#define mmSDMA1_RLC1_RB_WPTR_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_RB_WPTR_HI                                                                        0x01a6
-#define mmSDMA1_RLC1_RB_WPTR_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC1_RB_WPTR_POLL_CNTL                                                                 0x01a7
-#define mmSDMA1_RLC1_RB_WPTR_POLL_CNTL_BASE_IDX                                                        0
-#define mmSDMA1_RLC1_RB_RPTR_ADDR_HI                                                                   0x01a8
-#define mmSDMA1_RLC1_RB_RPTR_ADDR_HI_BASE_IDX                                                          0
-#define mmSDMA1_RLC1_RB_RPTR_ADDR_LO                                                                   0x01a9
-#define mmSDMA1_RLC1_RB_RPTR_ADDR_LO_BASE_IDX                                                          0
-#define mmSDMA1_RLC1_IB_CNTL                                                                           0x01aa
-#define mmSDMA1_RLC1_IB_CNTL_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_IB_RPTR                                                                           0x01ab
-#define mmSDMA1_RLC1_IB_RPTR_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_IB_OFFSET                                                                         0x01ac
-#define mmSDMA1_RLC1_IB_OFFSET_BASE_IDX                                                                0
-#define mmSDMA1_RLC1_IB_BASE_LO                                                                        0x01ad
-#define mmSDMA1_RLC1_IB_BASE_LO_BASE_IDX                                                               0
-#define mmSDMA1_RLC1_IB_BASE_HI                                                                        0x01ae
-#define mmSDMA1_RLC1_IB_BASE_HI_BASE_IDX                                                               0
-#define mmSDMA1_RLC1_IB_SIZE                                                                           0x01af
-#define mmSDMA1_RLC1_IB_SIZE_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_SKIP_CNTL                                                                         0x01b0
-#define mmSDMA1_RLC1_SKIP_CNTL_BASE_IDX                                                                0
-#define mmSDMA1_RLC1_CONTEXT_STATUS                                                                    0x01b1
-#define mmSDMA1_RLC1_CONTEXT_STATUS_BASE_IDX                                                           0
-#define mmSDMA1_RLC1_DOORBELL                                                                          0x01b2
-#define mmSDMA1_RLC1_DOORBELL_BASE_IDX                                                                 0
-#define mmSDMA1_RLC1_STATUS                                                                            0x01c8
-#define mmSDMA1_RLC1_STATUS_BASE_IDX                                                                   0
-#define mmSDMA1_RLC1_DOORBELL_LOG                                                                      0x01c9
-#define mmSDMA1_RLC1_DOORBELL_LOG_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_WATERMARK                                                                         0x01ca
-#define mmSDMA1_RLC1_WATERMARK_BASE_IDX                                                                0
-#define mmSDMA1_RLC1_DOORBELL_OFFSET                                                                   0x01cb
-#define mmSDMA1_RLC1_DOORBELL_OFFSET_BASE_IDX                                                          0
-#define mmSDMA1_RLC1_CSA_ADDR_LO                                                                       0x01cc
-#define mmSDMA1_RLC1_CSA_ADDR_LO_BASE_IDX                                                              0
-#define mmSDMA1_RLC1_CSA_ADDR_HI                                                                       0x01cd
-#define mmSDMA1_RLC1_CSA_ADDR_HI_BASE_IDX                                                              0
-#define mmSDMA1_RLC1_IB_SUB_REMAIN                                                                     0x01cf
-#define mmSDMA1_RLC1_IB_SUB_REMAIN_BASE_IDX                                                            0
-#define mmSDMA1_RLC1_PREEMPT                                                                           0x01d0
-#define mmSDMA1_RLC1_PREEMPT_BASE_IDX                                                                  0
-#define mmSDMA1_RLC1_DUMMY_REG                                                                         0x01d1
-#define mmSDMA1_RLC1_DUMMY_REG_BASE_IDX                                                                0
-#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_HI                                                              0x01d2
-#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                     0
-#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_LO                                                              0x01d3
-#define mmSDMA1_RLC1_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                     0
-#define mmSDMA1_RLC1_RB_AQL_CNTL                                                                       0x01d4
-#define mmSDMA1_RLC1_RB_AQL_CNTL_BASE_IDX                                                              0
-#define mmSDMA1_RLC1_MINOR_PTR_UPDATE                                                                  0x01d5
-#define mmSDMA1_RLC1_MINOR_PTR_UPDATE_BASE_IDX                                                         0
-#define mmSDMA1_RLC1_MIDCMD_DATA0                                                                      0x01e0
-#define mmSDMA1_RLC1_MIDCMD_DATA0_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA1                                                                      0x01e1
-#define mmSDMA1_RLC1_MIDCMD_DATA1_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA2                                                                      0x01e2
-#define mmSDMA1_RLC1_MIDCMD_DATA2_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA3                                                                      0x01e3
-#define mmSDMA1_RLC1_MIDCMD_DATA3_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA4                                                                      0x01e4
-#define mmSDMA1_RLC1_MIDCMD_DATA4_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA5                                                                      0x01e5
-#define mmSDMA1_RLC1_MIDCMD_DATA5_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA6                                                                      0x01e6
-#define mmSDMA1_RLC1_MIDCMD_DATA6_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA7                                                                      0x01e7
-#define mmSDMA1_RLC1_MIDCMD_DATA7_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_DATA8                                                                      0x01e8
-#define mmSDMA1_RLC1_MIDCMD_DATA8_BASE_IDX                                                             0
-#define mmSDMA1_RLC1_MIDCMD_CNTL                                                                       0x01e9
-#define mmSDMA1_RLC1_MIDCMD_CNTL_BASE_IDX                                                              0
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_sh_mask.h
deleted file mode 100644
index 25decdf96d16..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/SDMA1/sdma1_4_0_sh_mask.h
+++ /dev/null
@@ -1,1810 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _sdma1_4_0_SH_MASK_HEADER
-#define _sdma1_4_0_SH_MASK_HEADER
-
-
-// addressBlock: sdma1_sdma1dec
-//SDMA1_UCODE_ADDR
-#define SDMA1_UCODE_ADDR__VALUE__SHIFT                                                                        0x0
-#define SDMA1_UCODE_ADDR__VALUE_MASK                                                                          0x00001FFFL
-//SDMA1_UCODE_DATA
-#define SDMA1_UCODE_DATA__VALUE__SHIFT                                                                        0x0
-#define SDMA1_UCODE_DATA__VALUE_MASK                                                                          0xFFFFFFFFL
-//SDMA1_VM_CNTL
-#define SDMA1_VM_CNTL__CMD__SHIFT                                                                             0x0
-#define SDMA1_VM_CNTL__CMD_MASK                                                                               0x0000000FL
-//SDMA1_VM_CTX_LO
-#define SDMA1_VM_CTX_LO__ADDR__SHIFT                                                                          0x2
-#define SDMA1_VM_CTX_LO__ADDR_MASK                                                                            0xFFFFFFFCL
-//SDMA1_VM_CTX_HI
-#define SDMA1_VM_CTX_HI__ADDR__SHIFT                                                                          0x0
-#define SDMA1_VM_CTX_HI__ADDR_MASK                                                                            0xFFFFFFFFL
-//SDMA1_ACTIVE_FCN_ID
-#define SDMA1_ACTIVE_FCN_ID__VFID__SHIFT                                                                      0x0
-#define SDMA1_ACTIVE_FCN_ID__RESERVED__SHIFT                                                                  0x4
-#define SDMA1_ACTIVE_FCN_ID__VF__SHIFT                                                                        0x1f
-#define SDMA1_ACTIVE_FCN_ID__VFID_MASK                                                                        0x0000000FL
-#define SDMA1_ACTIVE_FCN_ID__RESERVED_MASK                                                                    0x7FFFFFF0L
-#define SDMA1_ACTIVE_FCN_ID__VF_MASK                                                                          0x80000000L
-//SDMA1_VM_CTX_CNTL
-#define SDMA1_VM_CTX_CNTL__PRIV__SHIFT                                                                        0x0
-#define SDMA1_VM_CTX_CNTL__VMID__SHIFT                                                                        0x4
-#define SDMA1_VM_CTX_CNTL__PRIV_MASK                                                                          0x00000001L
-#define SDMA1_VM_CTX_CNTL__VMID_MASK                                                                          0x000000F0L
-//SDMA1_VIRT_RESET_REQ
-#define SDMA1_VIRT_RESET_REQ__VF__SHIFT                                                                       0x0
-#define SDMA1_VIRT_RESET_REQ__PF__SHIFT                                                                       0x1f
-#define SDMA1_VIRT_RESET_REQ__VF_MASK                                                                         0x0000FFFFL
-#define SDMA1_VIRT_RESET_REQ__PF_MASK                                                                         0x80000000L
-//SDMA1_VF_ENABLE
-#define SDMA1_VF_ENABLE__VF_ENABLE__SHIFT                                                                     0x0
-#define SDMA1_VF_ENABLE__VF_ENABLE_MASK                                                                       0x00000001L
-//SDMA1_CONTEXT_REG_TYPE0
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_CNTL__SHIFT                                                     0x0
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE__SHIFT                                                     0x1
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE_HI__SHIFT                                                  0x2
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR__SHIFT                                                     0x3
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_HI__SHIFT                                                  0x4
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR__SHIFT                                                     0x5
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_HI__SHIFT                                                  0x6
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_POLL_CNTL__SHIFT                                           0x7
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_HI__SHIFT                                             0x8
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_LO__SHIFT                                             0x9
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_CNTL__SHIFT                                                     0xa
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_RPTR__SHIFT                                                     0xb
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_OFFSET__SHIFT                                                   0xc
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_LO__SHIFT                                                  0xd
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_HI__SHIFT                                                  0xe
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_SIZE__SHIFT                                                     0xf
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_SKIP_CNTL__SHIFT                                                   0x10
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_STATUS__SHIFT                                              0x11
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_DOORBELL__SHIFT                                                    0x12
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_CNTL__SHIFT                                                0x13
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_CNTL_MASK                                                       0x00000001L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE_MASK                                                       0x00000002L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_BASE_HI_MASK                                                    0x00000004L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_MASK                                                       0x00000008L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_HI_MASK                                                    0x00000010L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_MASK                                                       0x00000020L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_HI_MASK                                                    0x00000040L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_WPTR_POLL_CNTL_MASK                                             0x00000080L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_HI_MASK                                               0x00000100L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_RB_RPTR_ADDR_LO_MASK                                               0x00000200L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_CNTL_MASK                                                       0x00000400L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_RPTR_MASK                                                       0x00000800L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_OFFSET_MASK                                                     0x00001000L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_LO_MASK                                                    0x00002000L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_BASE_HI_MASK                                                    0x00004000L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_IB_SIZE_MASK                                                       0x00008000L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_SKIP_CNTL_MASK                                                     0x00010000L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_STATUS_MASK                                                0x00020000L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_DOORBELL_MASK                                                      0x00040000L
-#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_GFX_CONTEXT_CNTL_MASK                                                  0x00080000L
-//SDMA1_CONTEXT_REG_TYPE1
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_STATUS__SHIFT                                                      0x8
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_LOG__SHIFT                                                0x9
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_WATERMARK__SHIFT                                                   0xa
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_OFFSET__SHIFT                                             0xb
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_LO__SHIFT                                                 0xc
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_HI__SHIFT                                                 0xd
-#define SDMA1_CONTEXT_REG_TYPE1__VOID_REG2__SHIFT                                                             0xe
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_IB_SUB_REMAIN__SHIFT                                               0xf
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_PREEMPT__SHIFT                                                     0x10
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DUMMY_REG__SHIFT                                                   0x11
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_HI__SHIFT                                        0x12
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_LO__SHIFT                                        0x13
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_AQL_CNTL__SHIFT                                                 0x14
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_MINOR_PTR_UPDATE__SHIFT                                            0x15
-#define SDMA1_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                              0x16
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_STATUS_MASK                                                        0x00000100L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_LOG_MASK                                                  0x00000200L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_WATERMARK_MASK                                                     0x00000400L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DOORBELL_OFFSET_MASK                                               0x00000800L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_LO_MASK                                                   0x00001000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_CSA_ADDR_HI_MASK                                                   0x00002000L
-#define SDMA1_CONTEXT_REG_TYPE1__VOID_REG2_MASK                                                               0x00004000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_IB_SUB_REMAIN_MASK                                                 0x00008000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_PREEMPT_MASK                                                       0x00010000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_DUMMY_REG_MASK                                                     0x00020000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_HI_MASK                                          0x00040000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_WPTR_POLL_ADDR_LO_MASK                                          0x00080000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_RB_AQL_CNTL_MASK                                                   0x00100000L
-#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_GFX_MINOR_PTR_UPDATE_MASK                                              0x00200000L
-#define SDMA1_CONTEXT_REG_TYPE1__RESERVED_MASK                                                                0xFFC00000L
-//SDMA1_CONTEXT_REG_TYPE2
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA0__SHIFT                                                0x0
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA1__SHIFT                                                0x1
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA2__SHIFT                                                0x2
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA3__SHIFT                                                0x3
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA4__SHIFT                                                0x4
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA5__SHIFT                                                0x5
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA6__SHIFT                                                0x6
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA7__SHIFT                                                0x7
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA8__SHIFT                                                0x8
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_CNTL__SHIFT                                                 0x9
-#define SDMA1_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                              0xa
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA0_MASK                                                  0x00000001L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA1_MASK                                                  0x00000002L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA2_MASK                                                  0x00000004L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA3_MASK                                                  0x00000008L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA4_MASK                                                  0x00000010L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA5_MASK                                                  0x00000020L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA6_MASK                                                  0x00000040L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA7_MASK                                                  0x00000080L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_DATA8_MASK                                                  0x00000100L
-#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_GFX_MIDCMD_CNTL_MASK                                                   0x00000200L
-#define SDMA1_CONTEXT_REG_TYPE2__RESERVED_MASK                                                                0xFFFFFC00L
-//SDMA1_CONTEXT_REG_TYPE3
-#define SDMA1_CONTEXT_REG_TYPE3__RESERVED__SHIFT                                                              0x0
-#define SDMA1_CONTEXT_REG_TYPE3__RESERVED_MASK                                                                0xFFFFFFFFL
-//SDMA1_PUB_REG_TYPE0
-#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_ADDR__SHIFT                                                          0x0
-#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_DATA__SHIFT                                                          0x1
-#define SDMA1_PUB_REG_TYPE0__RESERVED3__SHIFT                                                                 0x3
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CNTL__SHIFT                                                             0x4
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_LO__SHIFT                                                           0x5
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_HI__SHIFT                                                           0x6
-#define SDMA1_PUB_REG_TYPE0__SDMA1_ACTIVE_FCN_ID__SHIFT                                                       0x7
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_CNTL__SHIFT                                                         0x8
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VIRT_RESET_REQ__SHIFT                                                      0x9
-#define SDMA1_PUB_REG_TYPE0__RESERVED10__SHIFT                                                                0xa
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE0__SHIFT                                                   0xb
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE1__SHIFT                                                   0xc
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE2__SHIFT                                                   0xd
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE3__SHIFT                                                   0xe
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE0__SHIFT                                                       0xf
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE1__SHIFT                                                       0x10
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE2__SHIFT                                                       0x11
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE3__SHIFT                                                       0x12
-#define SDMA1_PUB_REG_TYPE0__SDMA1_MMHUB_CNTL__SHIFT                                                          0x13
-#define SDMA1_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY__SHIFT                                           0x14
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_GROUP_BOUNDARY__SHIFT                                              0x19
-#define SDMA1_PUB_REG_TYPE0__SDMA1_POWER_CNTL__SHIFT                                                          0x1a
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CLK_CTRL__SHIFT                                                            0x1b
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CNTL__SHIFT                                                                0x1c
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CHICKEN_BITS__SHIFT                                                        0x1d
-#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG__SHIFT                                                      0x1e
-#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_READ__SHIFT                                                 0x1f
-#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_ADDR_MASK                                                            0x00000001L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_UCODE_DATA_MASK                                                            0x00000002L
-#define SDMA1_PUB_REG_TYPE0__RESERVED3_MASK                                                                   0x00000008L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CNTL_MASK                                                               0x00000010L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_LO_MASK                                                             0x00000020L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_HI_MASK                                                             0x00000040L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_ACTIVE_FCN_ID_MASK                                                         0x00000080L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VM_CTX_CNTL_MASK                                                           0x00000100L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_VIRT_RESET_REQ_MASK                                                        0x00000200L
-#define SDMA1_PUB_REG_TYPE0__RESERVED10_MASK                                                                  0x00000400L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE0_MASK                                                     0x00000800L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE1_MASK                                                     0x00001000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE2_MASK                                                     0x00002000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_REG_TYPE3_MASK                                                     0x00004000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE0_MASK                                                         0x00008000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE1_MASK                                                         0x00010000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE2_MASK                                                         0x00020000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_PUB_REG_TYPE3_MASK                                                         0x00040000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_MMHUB_CNTL_MASK                                                            0x00080000L
-#define SDMA1_PUB_REG_TYPE0__RESERVED_FOR_PSPSMU_ACCESS_ONLY_MASK                                             0x01F00000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CONTEXT_GROUP_BOUNDARY_MASK                                                0x02000000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_POWER_CNTL_MASK                                                            0x04000000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CLK_CTRL_MASK                                                              0x08000000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CNTL_MASK                                                                  0x10000000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_CHICKEN_BITS_MASK                                                          0x20000000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_MASK                                                        0x40000000L
-#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_READ_MASK                                                   0x80000000L
-//SDMA1_PUB_REG_TYPE1
-#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_HI__SHIFT                                                    0x0
-#define SDMA1_PUB_REG_TYPE1__SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT                                            0x1
-#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH__SHIFT                                                       0x2
-#define SDMA1_PUB_REG_TYPE1__SDMA1_IB_OFFSET_FETCH__SHIFT                                                     0x3
-#define SDMA1_PUB_REG_TYPE1__SDMA1_PROGRAM__SHIFT                                                             0x4
-#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS_REG__SHIFT                                                          0x5
-#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS1_REG__SHIFT                                                         0x6
-#define SDMA1_PUB_REG_TYPE1__SDMA1_RD_BURST_CNTL__SHIFT                                                       0x7
-#define SDMA1_PUB_REG_TYPE1__SDMA1_HBM_PAGE_CONFIG__SHIFT                                                     0x8
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UCODE_CHECKSUM__SHIFT                                                      0x9
-#define SDMA1_PUB_REG_TYPE1__SDMA1_F32_CNTL__SHIFT                                                            0xa
-#define SDMA1_PUB_REG_TYPE1__SDMA1_FREEZE__SHIFT                                                              0xb
-#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE0_QUANTUM__SHIFT                                                      0xc
-#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE1_QUANTUM__SHIFT                                                      0xd
-#define SDMA1_PUB_REG_TYPE1__SDMA_POWER_GATING__SHIFT                                                         0xe
-#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG__SHIFT                                                         0xf
-#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_WRITE__SHIFT                                                          0x10
-#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_READ__SHIFT                                                           0x11
-#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_CONFIG__SHIFT                                                          0x12
-#define SDMA1_PUB_REG_TYPE1__SDMA1_BA_THRESHOLD__SHIFT                                                        0x13
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ID__SHIFT                                                                  0x14
-#define SDMA1_PUB_REG_TYPE1__SDMA1_VERSION__SHIFT                                                             0x15
-#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER__SHIFT                                                         0x16
-#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_CLEAR__SHIFT                                                   0x17
-#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS2_REG__SHIFT                                                         0x18
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_CNTL__SHIFT                                                         0x19
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_LO__SHIFT                                                     0x1a
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_HI__SHIFT                                                     0x1b
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_CNTL__SHIFT                                                          0x1c
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WATERMK__SHIFT                                                       0x1d
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_RD_STATUS__SHIFT                                                     0x1e
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WR_STATUS__SHIFT                                                     0x1f
-#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_HI_MASK                                                      0x00000001L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_SEM_WAIT_FAIL_TIMER_CNTL_MASK                                              0x00000002L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_MASK                                                         0x00000004L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_IB_OFFSET_FETCH_MASK                                                       0x00000008L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_PROGRAM_MASK                                                               0x00000010L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS_REG_MASK                                                            0x00000020L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS1_REG_MASK                                                           0x00000040L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_RD_BURST_CNTL_MASK                                                         0x00000080L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_HBM_PAGE_CONFIG_MASK                                                       0x00000100L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UCODE_CHECKSUM_MASK                                                        0x00000200L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_F32_CNTL_MASK                                                              0x00000400L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_FREEZE_MASK                                                                0x00000800L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE0_QUANTUM_MASK                                                        0x00001000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_PHASE1_QUANTUM_MASK                                                        0x00002000L
-#define SDMA1_PUB_REG_TYPE1__SDMA_POWER_GATING_MASK                                                           0x00004000L
-#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_CONFIG_MASK                                                           0x00008000L
-#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_WRITE_MASK                                                            0x00010000L
-#define SDMA1_PUB_REG_TYPE1__SDMA_PGFSM_READ_MASK                                                             0x00020000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_CONFIG_MASK                                                            0x00040000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_BA_THRESHOLD_MASK                                                          0x00080000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ID_MASK                                                                    0x00100000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_VERSION_MASK                                                               0x00200000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_MASK                                                           0x00400000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_CLEAR_MASK                                                     0x00800000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS2_REG_MASK                                                           0x01000000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_CNTL_MASK                                                           0x02000000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_LO_MASK                                                       0x04000000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_HI_MASK                                                       0x08000000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_CNTL_MASK                                                            0x10000000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WATERMK_MASK                                                         0x20000000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_RD_STATUS_MASK                                                       0x40000000L
-#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WR_STATUS_MASK                                                       0x80000000L
-//SDMA1_PUB_REG_TYPE2
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV0__SHIFT                                                          0x0
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV1__SHIFT                                                          0x1
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV2__SHIFT                                                          0x2
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK0__SHIFT                                                     0x3
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK1__SHIFT                                                     0x4
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK0__SHIFT                                                     0x5
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK1__SHIFT                                                     0x6
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_TIMEOUT__SHIFT                                                       0x7
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_PAGE__SHIFT                                                          0x8
-#define SDMA1_PUB_REG_TYPE2__SDMA1_POWER_CNTL_IDLE__SHIFT                                                     0x9
-#define SDMA1_PUB_REG_TYPE2__SDMA1_RELAX_ORDERING_LUT__SHIFT                                                  0xa
-#define SDMA1_PUB_REG_TYPE2__SDMA1_CHICKEN_BITS_2__SHIFT                                                      0xb
-#define SDMA1_PUB_REG_TYPE2__SDMA1_STATUS3_REG__SHIFT                                                         0xc
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_LO__SHIFT                                                    0xd
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_HI__SHIFT                                                    0xe
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PHASE2_QUANTUM__SHIFT                                                      0xf
-#define SDMA1_PUB_REG_TYPE2__SDMA1_ERROR_LOG__SHIFT                                                           0x10
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG0__SHIFT                                                      0x11
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG1__SHIFT                                                      0x12
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG2__SHIFT                                                      0x13
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG3__SHIFT                                                      0x14
-#define SDMA1_PUB_REG_TYPE2__SDMA1_F32_COUNTER__SHIFT                                                         0x15
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UNBREAKABLE__SHIFT                                                         0x16
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFMON_CNTL__SHIFT                                                        0x17
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER0_RESULT__SHIFT                                                 0x18
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER1_RESULT__SHIFT                                                 0x19
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__SHIFT                                         0x1a
-#define SDMA1_PUB_REG_TYPE2__SDMA1_CRD_CNTL__SHIFT                                                            0x1b
-#define SDMA1_PUB_REG_TYPE2__SDMA1_MMHUB_TRUSTLVL__SHIFT                                                      0x1c
-#define SDMA1_PUB_REG_TYPE2__SDMA1_GPU_IOV_VIOLATION_LOG__SHIFT                                               0x1d
-#define SDMA1_PUB_REG_TYPE2__SDMA1_ULV_CNTL__SHIFT                                                            0x1e
-#define SDMA1_PUB_REG_TYPE2__RESERVED__SHIFT                                                                  0x1f
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV0_MASK                                                            0x00000001L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV1_MASK                                                            0x00000002L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV2_MASK                                                            0x00000004L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK0_MASK                                                       0x00000008L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK1_MASK                                                       0x00000010L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK0_MASK                                                       0x00000020L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK1_MASK                                                       0x00000040L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_TIMEOUT_MASK                                                         0x00000080L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_PAGE_MASK                                                            0x00000100L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_POWER_CNTL_IDLE_MASK                                                       0x00000200L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_RELAX_ORDERING_LUT_MASK                                                    0x00000400L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_CHICKEN_BITS_2_MASK                                                        0x00000800L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_STATUS3_REG_MASK                                                           0x00001000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_LO_MASK                                                      0x00002000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_HI_MASK                                                      0x00004000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PHASE2_QUANTUM_MASK                                                        0x00008000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_ERROR_LOG_MASK                                                             0x00010000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG0_MASK                                                        0x00020000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG1_MASK                                                        0x00040000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG2_MASK                                                        0x00080000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG3_MASK                                                        0x00100000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_F32_COUNTER_MASK                                                           0x00200000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_UNBREAKABLE_MASK                                                           0x00400000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFMON_CNTL_MASK                                                          0x00800000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER0_RESULT_MASK                                                   0x01000000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER1_RESULT_MASK                                                   0x02000000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_PERFCOUNTER_TAG_DELAY_RANGE_MASK                                           0x04000000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_CRD_CNTL_MASK                                                              0x08000000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_MMHUB_TRUSTLVL_MASK                                                        0x10000000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_GPU_IOV_VIOLATION_LOG_MASK                                                 0x20000000L
-#define SDMA1_PUB_REG_TYPE2__SDMA1_ULV_CNTL_MASK                                                              0x40000000L
-#define SDMA1_PUB_REG_TYPE2__RESERVED_MASK                                                                    0x80000000L
-//SDMA1_PUB_REG_TYPE3
-#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_DATA__SHIFT                                                   0x0
-#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_INDEX__SHIFT                                                  0x1
-#define SDMA1_PUB_REG_TYPE3__RESERVED__SHIFT                                                                  0x2
-#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_DATA_MASK                                                     0x00000001L
-#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_INDEX_MASK                                                    0x00000002L
-#define SDMA1_PUB_REG_TYPE3__RESERVED_MASK                                                                    0xFFFFFFFCL
-//SDMA1_MMHUB_CNTL
-#define SDMA1_MMHUB_CNTL__UNIT_ID__SHIFT                                                                      0x0
-#define SDMA1_MMHUB_CNTL__UNIT_ID_MASK                                                                        0x0000003FL
-//SDMA1_CONTEXT_GROUP_BOUNDARY
-#define SDMA1_CONTEXT_GROUP_BOUNDARY__RESERVED__SHIFT                                                         0x0
-#define SDMA1_CONTEXT_GROUP_BOUNDARY__RESERVED_MASK                                                           0xFFFFFFFFL
-//SDMA1_POWER_CNTL
-#define SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE__SHIFT                                                           0x8
-#define SDMA1_POWER_CNTL__MEM_POWER_LS_EN__SHIFT                                                              0x9
-#define SDMA1_POWER_CNTL__MEM_POWER_DS_EN__SHIFT                                                              0xa
-#define SDMA1_POWER_CNTL__MEM_POWER_SD_EN__SHIFT                                                              0xb
-#define SDMA1_POWER_CNTL__MEM_POWER_DELAY__SHIFT                                                              0xc
-#define SDMA1_POWER_CNTL__MEM_POWER_OVERRIDE_MASK                                                             0x00000100L
-#define SDMA1_POWER_CNTL__MEM_POWER_LS_EN_MASK                                                                0x00000200L
-#define SDMA1_POWER_CNTL__MEM_POWER_DS_EN_MASK                                                                0x00000400L
-#define SDMA1_POWER_CNTL__MEM_POWER_SD_EN_MASK                                                                0x00000800L
-#define SDMA1_POWER_CNTL__MEM_POWER_DELAY_MASK                                                                0x003FF000L
-//SDMA1_CLK_CTRL
-#define SDMA1_CLK_CTRL__ON_DELAY__SHIFT                                                                       0x0
-#define SDMA1_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                                 0x4
-#define SDMA1_CLK_CTRL__RESERVED__SHIFT                                                                       0xc
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                                 0x18
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                                 0x19
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                                 0x1a
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                                 0x1b
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                                 0x1c
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                                 0x1d
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                                 0x1e
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                                 0x1f
-#define SDMA1_CLK_CTRL__ON_DELAY_MASK                                                                         0x0000000FL
-#define SDMA1_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                   0x00000FF0L
-#define SDMA1_CLK_CTRL__RESERVED_MASK                                                                         0x00FFF000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                   0x01000000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                   0x02000000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                   0x04000000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                   0x08000000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                   0x10000000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                   0x20000000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                                   0x40000000L
-#define SDMA1_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                                   0x80000000L
-//SDMA1_CNTL
-#define SDMA1_CNTL__TRAP_ENABLE__SHIFT                                                                        0x0
-#define SDMA1_CNTL__UTC_L1_ENABLE__SHIFT                                                                      0x1
-#define SDMA1_CNTL__SEM_WAIT_INT_ENABLE__SHIFT                                                                0x2
-#define SDMA1_CNTL__DATA_SWAP_ENABLE__SHIFT                                                                   0x3
-#define SDMA1_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                                  0x4
-#define SDMA1_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                              0x5
-#define SDMA1_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                          0x11
-#define SDMA1_CNTL__AUTO_CTXSW_ENABLE__SHIFT                                                                  0x12
-#define SDMA1_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                                0x1c
-#define SDMA1_CNTL__FROZEN_INT_ENABLE__SHIFT                                                                  0x1d
-#define SDMA1_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                              0x1e
-#define SDMA1_CNTL__TRAP_ENABLE_MASK                                                                          0x00000001L
-#define SDMA1_CNTL__UTC_L1_ENABLE_MASK                                                                        0x00000002L
-#define SDMA1_CNTL__SEM_WAIT_INT_ENABLE_MASK                                                                  0x00000004L
-#define SDMA1_CNTL__DATA_SWAP_ENABLE_MASK                                                                     0x00000008L
-#define SDMA1_CNTL__FENCE_SWAP_ENABLE_MASK                                                                    0x00000010L
-#define SDMA1_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                                0x00000020L
-#define SDMA1_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                            0x00020000L
-#define SDMA1_CNTL__AUTO_CTXSW_ENABLE_MASK                                                                    0x00040000L
-#define SDMA1_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                                  0x10000000L
-#define SDMA1_CNTL__FROZEN_INT_ENABLE_MASK                                                                    0x20000000L
-#define SDMA1_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                                0x40000000L
-//SDMA1_CHICKEN_BITS
-#define SDMA1_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE__SHIFT                                                     0x0
-#define SDMA1_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT                                                 0x1
-#define SDMA1_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT                                        0x2
-#define SDMA1_CHICKEN_BITS__WRITE_BURST_LENGTH__SHIFT                                                         0x8
-#define SDMA1_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE__SHIFT                                                     0xa
-#define SDMA1_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT                                                        0x10
-#define SDMA1_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT                                                           0x11
-#define SDMA1_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                         0x14
-#define SDMA1_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                           0x17
-#define SDMA1_CHICKEN_BITS__TIME_BASED_QOS__SHIFT                                                             0x19
-#define SDMA1_CHICKEN_BITS__CE_AFIFO_WATERMARK__SHIFT                                                         0x1a
-#define SDMA1_CHICKEN_BITS__CE_DFIFO_WATERMARK__SHIFT                                                         0x1c
-#define SDMA1_CHICKEN_BITS__CE_LFIFO_WATERMARK__SHIFT                                                         0x1e
-#define SDMA1_CHICKEN_BITS__COPY_EFFICIENCY_ENABLE_MASK                                                       0x00000001L
-#define SDMA1_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK                                                   0x00000002L
-#define SDMA1_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK                                          0x00000004L
-#define SDMA1_CHICKEN_BITS__WRITE_BURST_LENGTH_MASK                                                           0x00000300L
-#define SDMA1_CHICKEN_BITS__WRITE_BURST_WAIT_CYCLE_MASK                                                       0x00001C00L
-#define SDMA1_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK                                                          0x00010000L
-#define SDMA1_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK                                                             0x00020000L
-#define SDMA1_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                           0x00100000L
-#define SDMA1_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                             0x00800000L
-#define SDMA1_CHICKEN_BITS__TIME_BASED_QOS_MASK                                                               0x02000000L
-#define SDMA1_CHICKEN_BITS__CE_AFIFO_WATERMARK_MASK                                                           0x0C000000L
-#define SDMA1_CHICKEN_BITS__CE_DFIFO_WATERMARK_MASK                                                           0x30000000L
-#define SDMA1_CHICKEN_BITS__CE_LFIFO_WATERMARK_MASK                                                           0xC0000000L
-//SDMA1_GB_ADDR_CONFIG
-#define SDMA1_GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                0x0
-#define SDMA1_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                     0x3
-#define SDMA1_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT                                                     0x8
-#define SDMA1_GB_ADDR_CONFIG__NUM_BANKS__SHIFT                                                                0xc
-#define SDMA1_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                       0x13
-#define SDMA1_GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                  0x00000007L
-#define SDMA1_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                       0x00000038L
-#define SDMA1_GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK                                                       0x00000700L
-#define SDMA1_GB_ADDR_CONFIG__NUM_BANKS_MASK                                                                  0x00007000L
-#define SDMA1_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                         0x00180000L
-//SDMA1_GB_ADDR_CONFIG_READ
-#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                           0x0
-#define SDMA1_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                0x3
-#define SDMA1_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE__SHIFT                                                0x8
-#define SDMA1_GB_ADDR_CONFIG_READ__NUM_BANKS__SHIFT                                                           0xc
-#define SDMA1_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                  0x13
-#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                             0x00000007L
-#define SDMA1_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                  0x00000038L
-#define SDMA1_GB_ADDR_CONFIG_READ__BANK_INTERLEAVE_SIZE_MASK                                                  0x00000700L
-#define SDMA1_GB_ADDR_CONFIG_READ__NUM_BANKS_MASK                                                             0x00007000L
-#define SDMA1_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                    0x00180000L
-//SDMA1_RB_RPTR_FETCH_HI
-#define SDMA1_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                                 0x0
-#define SDMA1_RB_RPTR_FETCH_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
-//SDMA1_SEM_WAIT_FAIL_TIMER_CNTL
-#define SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT                                                          0x0
-#define SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK                                                            0xFFFFFFFFL
-//SDMA1_RB_RPTR_FETCH
-#define SDMA1_RB_RPTR_FETCH__OFFSET__SHIFT                                                                    0x2
-#define SDMA1_RB_RPTR_FETCH__OFFSET_MASK                                                                      0xFFFFFFFCL
-//SDMA1_IB_OFFSET_FETCH
-#define SDMA1_IB_OFFSET_FETCH__OFFSET__SHIFT                                                                  0x2
-#define SDMA1_IB_OFFSET_FETCH__OFFSET_MASK                                                                    0x003FFFFCL
-//SDMA1_PROGRAM
-#define SDMA1_PROGRAM__STREAM__SHIFT                                                                          0x0
-#define SDMA1_PROGRAM__STREAM_MASK                                                                            0xFFFFFFFFL
-//SDMA1_STATUS_REG
-#define SDMA1_STATUS_REG__IDLE__SHIFT                                                                         0x0
-#define SDMA1_STATUS_REG__REG_IDLE__SHIFT                                                                     0x1
-#define SDMA1_STATUS_REG__RB_EMPTY__SHIFT                                                                     0x2
-#define SDMA1_STATUS_REG__RB_FULL__SHIFT                                                                      0x3
-#define SDMA1_STATUS_REG__RB_CMD_IDLE__SHIFT                                                                  0x4
-#define SDMA1_STATUS_REG__RB_CMD_FULL__SHIFT                                                                  0x5
-#define SDMA1_STATUS_REG__IB_CMD_IDLE__SHIFT                                                                  0x6
-#define SDMA1_STATUS_REG__IB_CMD_FULL__SHIFT                                                                  0x7
-#define SDMA1_STATUS_REG__BLOCK_IDLE__SHIFT                                                                   0x8
-#define SDMA1_STATUS_REG__INSIDE_IB__SHIFT                                                                    0x9
-#define SDMA1_STATUS_REG__EX_IDLE__SHIFT                                                                      0xa
-#define SDMA1_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE__SHIFT                                                    0xb
-#define SDMA1_STATUS_REG__PACKET_READY__SHIFT                                                                 0xc
-#define SDMA1_STATUS_REG__MC_WR_IDLE__SHIFT                                                                   0xd
-#define SDMA1_STATUS_REG__SRBM_IDLE__SHIFT                                                                    0xe
-#define SDMA1_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                                0xf
-#define SDMA1_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                              0x10
-#define SDMA1_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                              0x11
-#define SDMA1_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                              0x12
-#define SDMA1_STATUS_REG__MC_RD_IDLE__SHIFT                                                                   0x13
-#define SDMA1_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                             0x14
-#define SDMA1_STATUS_REG__MC_RD_RET_STALL__SHIFT                                                              0x15
-#define SDMA1_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT                                                           0x16
-#define SDMA1_STATUS_REG__PREV_CMD_IDLE__SHIFT                                                                0x19
-#define SDMA1_STATUS_REG__SEM_IDLE__SHIFT                                                                     0x1a
-#define SDMA1_STATUS_REG__SEM_REQ_STALL__SHIFT                                                                0x1b
-#define SDMA1_STATUS_REG__SEM_RESP_STATE__SHIFT                                                               0x1c
-#define SDMA1_STATUS_REG__INT_IDLE__SHIFT                                                                     0x1e
-#define SDMA1_STATUS_REG__INT_REQ_STALL__SHIFT                                                                0x1f
-#define SDMA1_STATUS_REG__IDLE_MASK                                                                           0x00000001L
-#define SDMA1_STATUS_REG__REG_IDLE_MASK                                                                       0x00000002L
-#define SDMA1_STATUS_REG__RB_EMPTY_MASK                                                                       0x00000004L
-#define SDMA1_STATUS_REG__RB_FULL_MASK                                                                        0x00000008L
-#define SDMA1_STATUS_REG__RB_CMD_IDLE_MASK                                                                    0x00000010L
-#define SDMA1_STATUS_REG__RB_CMD_FULL_MASK                                                                    0x00000020L
-#define SDMA1_STATUS_REG__IB_CMD_IDLE_MASK                                                                    0x00000040L
-#define SDMA1_STATUS_REG__IB_CMD_FULL_MASK                                                                    0x00000080L
-#define SDMA1_STATUS_REG__BLOCK_IDLE_MASK                                                                     0x00000100L
-#define SDMA1_STATUS_REG__INSIDE_IB_MASK                                                                      0x00000200L
-#define SDMA1_STATUS_REG__EX_IDLE_MASK                                                                        0x00000400L
-#define SDMA1_STATUS_REG__EX_IDLE_POLL_TIMER_EXPIRE_MASK                                                      0x00000800L
-#define SDMA1_STATUS_REG__PACKET_READY_MASK                                                                   0x00001000L
-#define SDMA1_STATUS_REG__MC_WR_IDLE_MASK                                                                     0x00002000L
-#define SDMA1_STATUS_REG__SRBM_IDLE_MASK                                                                      0x00004000L
-#define SDMA1_STATUS_REG__CONTEXT_EMPTY_MASK                                                                  0x00008000L
-#define SDMA1_STATUS_REG__DELTA_RPTR_FULL_MASK                                                                0x00010000L
-#define SDMA1_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                                0x00020000L
-#define SDMA1_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                                0x00040000L
-#define SDMA1_STATUS_REG__MC_RD_IDLE_MASK                                                                     0x00080000L
-#define SDMA1_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                               0x00100000L
-#define SDMA1_STATUS_REG__MC_RD_RET_STALL_MASK                                                                0x00200000L
-#define SDMA1_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK                                                             0x00400000L
-#define SDMA1_STATUS_REG__PREV_CMD_IDLE_MASK                                                                  0x02000000L
-#define SDMA1_STATUS_REG__SEM_IDLE_MASK                                                                       0x04000000L
-#define SDMA1_STATUS_REG__SEM_REQ_STALL_MASK                                                                  0x08000000L
-#define SDMA1_STATUS_REG__SEM_RESP_STATE_MASK                                                                 0x30000000L
-#define SDMA1_STATUS_REG__INT_IDLE_MASK                                                                       0x40000000L
-#define SDMA1_STATUS_REG__INT_REQ_STALL_MASK                                                                  0x80000000L
-//SDMA1_STATUS1_REG
-#define SDMA1_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                                0x0
-#define SDMA1_STATUS1_REG__CE_WR_IDLE__SHIFT                                                                  0x1
-#define SDMA1_STATUS1_REG__CE_SPLIT_IDLE__SHIFT                                                               0x2
-#define SDMA1_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                                0x3
-#define SDMA1_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                                 0x4
-#define SDMA1_STATUS1_REG__CE_IN_IDLE__SHIFT                                                                  0x5
-#define SDMA1_STATUS1_REG__CE_DST_IDLE__SHIFT                                                                 0x6
-#define SDMA1_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                                 0x9
-#define SDMA1_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                               0xa
-#define SDMA1_STATUS1_REG__CE_INFO_FULL__SHIFT                                                                0xd
-#define SDMA1_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                               0xe
-#define SDMA1_STATUS1_REG__EX_START__SHIFT                                                                    0xf
-#define SDMA1_STATUS1_REG__CE_RD_STALL__SHIFT                                                                 0x11
-#define SDMA1_STATUS1_REG__CE_WR_STALL__SHIFT                                                                 0x12
-#define SDMA1_STATUS1_REG__CE_WREQ_IDLE_MASK                                                                  0x00000001L
-#define SDMA1_STATUS1_REG__CE_WR_IDLE_MASK                                                                    0x00000002L
-#define SDMA1_STATUS1_REG__CE_SPLIT_IDLE_MASK                                                                 0x00000004L
-#define SDMA1_STATUS1_REG__CE_RREQ_IDLE_MASK                                                                  0x00000008L
-#define SDMA1_STATUS1_REG__CE_OUT_IDLE_MASK                                                                   0x00000010L
-#define SDMA1_STATUS1_REG__CE_IN_IDLE_MASK                                                                    0x00000020L
-#define SDMA1_STATUS1_REG__CE_DST_IDLE_MASK                                                                   0x00000040L
-#define SDMA1_STATUS1_REG__CE_CMD_IDLE_MASK                                                                   0x00000200L
-#define SDMA1_STATUS1_REG__CE_AFIFO_FULL_MASK                                                                 0x00000400L
-#define SDMA1_STATUS1_REG__CE_INFO_FULL_MASK                                                                  0x00002000L
-#define SDMA1_STATUS1_REG__CE_INFO1_FULL_MASK                                                                 0x00004000L
-#define SDMA1_STATUS1_REG__EX_START_MASK                                                                      0x00008000L
-#define SDMA1_STATUS1_REG__CE_RD_STALL_MASK                                                                   0x00020000L
-#define SDMA1_STATUS1_REG__CE_WR_STALL_MASK                                                                   0x00040000L
-//SDMA1_RD_BURST_CNTL
-#define SDMA1_RD_BURST_CNTL__RD_BURST__SHIFT                                                                  0x0
-#define SDMA1_RD_BURST_CNTL__RD_BURST_MASK                                                                    0x00000003L
-//SDMA1_HBM_PAGE_CONFIG
-#define SDMA1_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                      0x0
-#define SDMA1_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                        0x00000001L
-//SDMA1_UCODE_CHECKSUM
-#define SDMA1_UCODE_CHECKSUM__DATA__SHIFT                                                                     0x0
-#define SDMA1_UCODE_CHECKSUM__DATA_MASK                                                                       0xFFFFFFFFL
-//SDMA1_F32_CNTL
-#define SDMA1_F32_CNTL__HALT__SHIFT                                                                           0x0
-#define SDMA1_F32_CNTL__STEP__SHIFT                                                                           0x1
-#define SDMA1_F32_CNTL__HALT_MASK                                                                             0x00000001L
-#define SDMA1_F32_CNTL__STEP_MASK                                                                             0x00000002L
-//SDMA1_FREEZE
-#define SDMA1_FREEZE__PREEMPT__SHIFT                                                                          0x0
-#define SDMA1_FREEZE__FREEZE__SHIFT                                                                           0x4
-#define SDMA1_FREEZE__FROZEN__SHIFT                                                                           0x5
-#define SDMA1_FREEZE__F32_FREEZE__SHIFT                                                                       0x6
-#define SDMA1_FREEZE__PREEMPT_MASK                                                                            0x00000001L
-#define SDMA1_FREEZE__FREEZE_MASK                                                                             0x00000010L
-#define SDMA1_FREEZE__FROZEN_MASK                                                                             0x00000020L
-#define SDMA1_FREEZE__F32_FREEZE_MASK                                                                         0x00000040L
-//SDMA1_PHASE0_QUANTUM
-#define SDMA1_PHASE0_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA1_PHASE0_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA1_PHASE0_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA1_PHASE0_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA1_PHASE0_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA1_PHASE0_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA1_PHASE1_QUANTUM
-#define SDMA1_PHASE1_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA1_PHASE1_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA1_PHASE1_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA1_PHASE1_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA1_PHASE1_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA1_PHASE1_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA1_EDC_CONFIG
-#define SDMA1_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
-#define SDMA1_EDC_CONFIG__ECC_INT_ENABLE__SHIFT                                                               0x2
-#define SDMA1_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
-#define SDMA1_EDC_CONFIG__ECC_INT_ENABLE_MASK                                                                 0x00000004L
-//SDMA1_BA_THRESHOLD
-#define SDMA1_BA_THRESHOLD__READ_THRES__SHIFT                                                                 0x0
-#define SDMA1_BA_THRESHOLD__WRITE_THRES__SHIFT                                                                0x10
-#define SDMA1_BA_THRESHOLD__READ_THRES_MASK                                                                   0x000003FFL
-#define SDMA1_BA_THRESHOLD__WRITE_THRES_MASK                                                                  0x03FF0000L
-//SDMA1_ID
-#define SDMA1_ID__DEVICE_ID__SHIFT                                                                            0x0
-#define SDMA1_ID__DEVICE_ID_MASK                                                                              0x000000FFL
-//SDMA1_VERSION
-#define SDMA1_VERSION__MINVER__SHIFT                                                                          0x0
-#define SDMA1_VERSION__MAJVER__SHIFT                                                                          0x8
-#define SDMA1_VERSION__REV__SHIFT                                                                             0x10
-#define SDMA1_VERSION__MINVER_MASK                                                                            0x0000007FL
-#define SDMA1_VERSION__MAJVER_MASK                                                                            0x00007F00L
-#define SDMA1_VERSION__REV_MASK                                                                               0x003F0000L
-//SDMA1_EDC_COUNTER
-#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_DED__SHIFT                                                          0x0
-#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_SEC__SHIFT                                                          0x1
-#define SDMA1_EDC_COUNTER__SDMA_RB_CMD_BUF_SED__SHIFT                                                         0x2
-#define SDMA1_EDC_COUNTER__SDMA_IB_CMD_BUF_SED__SHIFT                                                         0x3
-#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED__SHIFT                                                      0x4
-#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT                                                   0x5
-#define SDMA1_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED__SHIFT                                                      0x6
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT                                                    0x7
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT                                                    0x8
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT                                                    0x9
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT                                                    0xa
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT                                                    0xb
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT                                                    0xc
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT                                                    0xd
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT                                                    0xe
-#define SDMA1_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED__SHIFT                                                      0xf
-#define SDMA1_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT                                                    0x10
-#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_DED_MASK                                                            0x00000001L
-#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_SEC_MASK                                                            0x00000002L
-#define SDMA1_EDC_COUNTER__SDMA_RB_CMD_BUF_SED_MASK                                                           0x00000004L
-#define SDMA1_EDC_COUNTER__SDMA_IB_CMD_BUF_SED_MASK                                                           0x00000008L
-#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED_MASK                                                        0x00000010L
-#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED_MASK                                                     0x00000020L
-#define SDMA1_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED_MASK                                                        0x00000040L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK                                                      0x00000080L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK                                                      0x00000100L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK                                                      0x00000200L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK                                                      0x00000400L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK                                                      0x00000800L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK                                                      0x00001000L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK                                                      0x00002000L
-#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK                                                      0x00004000L
-#define SDMA1_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED_MASK                                                        0x00008000L
-#define SDMA1_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED_MASK                                                      0x00010000L
-//SDMA1_EDC_COUNTER_CLEAR
-#define SDMA1_EDC_COUNTER_CLEAR__DUMMY__SHIFT                                                                 0x0
-#define SDMA1_EDC_COUNTER_CLEAR__DUMMY_MASK                                                                   0x00000001L
-//SDMA1_STATUS2_REG
-#define SDMA1_STATUS2_REG__ID__SHIFT                                                                          0x0
-#define SDMA1_STATUS2_REG__F32_INSTR_PTR__SHIFT                                                               0x2
-#define SDMA1_STATUS2_REG__CMD_OP__SHIFT                                                                      0x10
-#define SDMA1_STATUS2_REG__ID_MASK                                                                            0x00000003L
-#define SDMA1_STATUS2_REG__F32_INSTR_PTR_MASK                                                                 0x00000FFCL
-#define SDMA1_STATUS2_REG__CMD_OP_MASK                                                                        0xFFFF0000L
-//SDMA1_ATOMIC_CNTL
-#define SDMA1_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                                  0x0
-#define SDMA1_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                       0x1f
-#define SDMA1_ATOMIC_CNTL__LOOP_TIMER_MASK                                                                    0x7FFFFFFFL
-#define SDMA1_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                         0x80000000L
-//SDMA1_ATOMIC_PREOP_LO
-#define SDMA1_ATOMIC_PREOP_LO__DATA__SHIFT                                                                    0x0
-#define SDMA1_ATOMIC_PREOP_LO__DATA_MASK                                                                      0xFFFFFFFFL
-//SDMA1_ATOMIC_PREOP_HI
-#define SDMA1_ATOMIC_PREOP_HI__DATA__SHIFT                                                                    0x0
-#define SDMA1_ATOMIC_PREOP_HI__DATA_MASK                                                                      0xFFFFFFFFL
-//SDMA1_UTCL1_CNTL
-#define SDMA1_UTCL1_CNTL__REDO_ENABLE__SHIFT                                                                  0x0
-#define SDMA1_UTCL1_CNTL__REDO_DELAY__SHIFT                                                                   0x1
-#define SDMA1_UTCL1_CNTL__REDO_WATERMK__SHIFT                                                                 0xb
-#define SDMA1_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                                 0xe
-#define SDMA1_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                                 0x18
-#define SDMA1_UTCL1_CNTL__VADDR_WATERMK__SHIFT                                                                0x1d
-#define SDMA1_UTCL1_CNTL__REDO_ENABLE_MASK                                                                    0x00000001L
-#define SDMA1_UTCL1_CNTL__REDO_DELAY_MASK                                                                     0x000007FEL
-#define SDMA1_UTCL1_CNTL__REDO_WATERMK_MASK                                                                   0x00003800L
-#define SDMA1_UTCL1_CNTL__INVACK_DELAY_MASK                                                                   0x00FFC000L
-#define SDMA1_UTCL1_CNTL__REQL2_CREDIT_MASK                                                                   0x1F000000L
-#define SDMA1_UTCL1_CNTL__VADDR_WATERMK_MASK                                                                  0xE0000000L
-//SDMA1_UTCL1_WATERMK
-#define SDMA1_UTCL1_WATERMK__REQMC_WATERMK__SHIFT                                                             0x0
-#define SDMA1_UTCL1_WATERMK__REQPG_WATERMK__SHIFT                                                             0xa
-#define SDMA1_UTCL1_WATERMK__INVREQ_WATERMK__SHIFT                                                            0x12
-#define SDMA1_UTCL1_WATERMK__XNACK_WATERMK__SHIFT                                                             0x1a
-#define SDMA1_UTCL1_WATERMK__REQMC_WATERMK_MASK                                                               0x000003FFL
-#define SDMA1_UTCL1_WATERMK__REQPG_WATERMK_MASK                                                               0x0003FC00L
-#define SDMA1_UTCL1_WATERMK__INVREQ_WATERMK_MASK                                                              0x03FC0000L
-#define SDMA1_UTCL1_WATERMK__XNACK_WATERMK_MASK                                                               0xFC000000L
-//SDMA1_UTCL1_RD_STATUS
-#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                0x0
-#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                     0x1
-#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                      0x2
-#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                   0x3
-#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                               0x4
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                    0x5
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                 0x6
-#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT                                                   0x7
-#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT                                                  0x8
-#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                 0x9
-#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                      0xa
-#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                       0xb
-#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                    0xc
-#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                0xd
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                     0xe
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                  0xf
-#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT                                                    0x10
-#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT                                                   0x11
-#define SDMA1_UTCL1_RD_STATUS__PAGE_FAULT__SHIFT                                                              0x12
-#define SDMA1_UTCL1_RD_STATUS__PAGE_NULL__SHIFT                                                               0x13
-#define SDMA1_UTCL1_RD_STATUS__REQL2_IDLE__SHIFT                                                              0x14
-#define SDMA1_UTCL1_RD_STATUS__CE_L1_STALL__SHIFT                                                             0x15
-#define SDMA1_UTCL1_RD_STATUS__NEXT_RD_VECTOR__SHIFT                                                          0x16
-#define SDMA1_UTCL1_RD_STATUS__MERGE_STATE__SHIFT                                                             0x1a
-#define SDMA1_UTCL1_RD_STATUS__ADDR_RD_RTR__SHIFT                                                             0x1d
-#define SDMA1_UTCL1_RD_STATUS__WPTR_POLLING__SHIFT                                                            0x1e
-#define SDMA1_UTCL1_RD_STATUS__INVREQ_SIZE__SHIFT                                                             0x1f
-#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                  0x00000001L
-#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                       0x00000002L
-#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                        0x00000004L
-#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                     0x00000008L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                 0x00000010L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                      0x00000020L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                   0x00000040L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK                                                     0x00000080L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK                                                    0x00000100L
-#define SDMA1_UTCL1_RD_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                   0x00000200L
-#define SDMA1_UTCL1_RD_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                        0x00000400L
-#define SDMA1_UTCL1_RD_STATUS__RTPG_RET_BUF_FULL_MASK                                                         0x00000800L
-#define SDMA1_UTCL1_RD_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                      0x00001000L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                  0x00002000L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                       0x00004000L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                    0x00008000L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_XNACK_FIFO_FULL_MASK                                                      0x00010000L
-#define SDMA1_UTCL1_RD_STATUS__RQPG_INVREQ_FIFO_FULL_MASK                                                     0x00020000L
-#define SDMA1_UTCL1_RD_STATUS__PAGE_FAULT_MASK                                                                0x00040000L
-#define SDMA1_UTCL1_RD_STATUS__PAGE_NULL_MASK                                                                 0x00080000L
-#define SDMA1_UTCL1_RD_STATUS__REQL2_IDLE_MASK                                                                0x00100000L
-#define SDMA1_UTCL1_RD_STATUS__CE_L1_STALL_MASK                                                               0x00200000L
-#define SDMA1_UTCL1_RD_STATUS__NEXT_RD_VECTOR_MASK                                                            0x03C00000L
-#define SDMA1_UTCL1_RD_STATUS__MERGE_STATE_MASK                                                               0x1C000000L
-#define SDMA1_UTCL1_RD_STATUS__ADDR_RD_RTR_MASK                                                               0x20000000L
-#define SDMA1_UTCL1_RD_STATUS__WPTR_POLLING_MASK                                                              0x40000000L
-#define SDMA1_UTCL1_RD_STATUS__INVREQ_SIZE_MASK                                                               0x80000000L
-//SDMA1_UTCL1_WR_STATUS
-#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY__SHIFT                                                0x0
-#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY__SHIFT                                                     0x1
-#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY__SHIFT                                                      0x2
-#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY__SHIFT                                                   0x3
-#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY__SHIFT                                               0x4
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY__SHIFT                                                    0x5
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY__SHIFT                                                 0x6
-#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY__SHIFT                                                   0x7
-#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY__SHIFT                                                  0x8
-#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL__SHIFT                                                 0x9
-#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL__SHIFT                                                      0xa
-#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL__SHIFT                                                       0xb
-#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL__SHIFT                                                    0xc
-#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL__SHIFT                                                0xd
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL__SHIFT                                                     0xe
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL__SHIFT                                                  0xf
-#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL__SHIFT                                                    0x10
-#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL__SHIFT                                                   0x11
-#define SDMA1_UTCL1_WR_STATUS__PAGE_FAULT__SHIFT                                                              0x12
-#define SDMA1_UTCL1_WR_STATUS__PAGE_NULL__SHIFT                                                               0x13
-#define SDMA1_UTCL1_WR_STATUS__REQL2_IDLE__SHIFT                                                              0x14
-#define SDMA1_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT                                                              0x15
-#define SDMA1_UTCL1_WR_STATUS__NEXT_WR_VECTOR__SHIFT                                                          0x16
-#define SDMA1_UTCL1_WR_STATUS__MERGE_STATE__SHIFT                                                             0x19
-#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY__SHIFT                                                    0x1c
-#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL__SHIFT                                                     0x1d
-#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY__SHIFT                                                   0x1e
-#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL__SHIFT                                                    0x1f
-#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_EMPTY_MASK                                                  0x00000001L
-#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_EMPTY_MASK                                                       0x00000002L
-#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_EMPTY_MASK                                                        0x00000004L
-#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_EMPTY_MASK                                                     0x00000008L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_EMPTY_MASK                                                 0x00000010L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_EMPTY_MASK                                                      0x00000020L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_EMPTY_MASK                                                   0x00000040L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_EMPTY_MASK                                                     0x00000080L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_EMPTY_MASK                                                    0x00000100L
-#define SDMA1_UTCL1_WR_STATUS__RQMC_RET_ADDR_FIFO_FULL_MASK                                                   0x00000200L
-#define SDMA1_UTCL1_WR_STATUS__RQMC_REQ_FIFO_FULL_MASK                                                        0x00000400L
-#define SDMA1_UTCL1_WR_STATUS__RTPG_RET_BUF_FULL_MASK                                                         0x00000800L
-#define SDMA1_UTCL1_WR_STATUS__RTPG_VADDR_FIFO_FULL_MASK                                                      0x00001000L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_HEAD_VIRT_FIFO_FULL_MASK                                                  0x00002000L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REDO_FIFO_FULL_MASK                                                       0x00004000L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_REQPAGE_FIFO_FULL_MASK                                                    0x00008000L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_XNACK_FIFO_FULL_MASK                                                      0x00010000L
-#define SDMA1_UTCL1_WR_STATUS__RQPG_INVREQ_FIFO_FULL_MASK                                                     0x00020000L
-#define SDMA1_UTCL1_WR_STATUS__PAGE_FAULT_MASK                                                                0x00040000L
-#define SDMA1_UTCL1_WR_STATUS__PAGE_NULL_MASK                                                                 0x00080000L
-#define SDMA1_UTCL1_WR_STATUS__REQL2_IDLE_MASK                                                                0x00100000L
-#define SDMA1_UTCL1_WR_STATUS__F32_WR_RTR_MASK                                                                0x00200000L
-#define SDMA1_UTCL1_WR_STATUS__NEXT_WR_VECTOR_MASK                                                            0x01C00000L
-#define SDMA1_UTCL1_WR_STATUS__MERGE_STATE_MASK                                                               0x0E000000L
-#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_EMPTY_MASK                                                      0x10000000L
-#define SDMA1_UTCL1_WR_STATUS__RPTR_DATA_FIFO_FULL_MASK                                                       0x20000000L
-#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_EMPTY_MASK                                                     0x40000000L
-#define SDMA1_UTCL1_WR_STATUS__WRREQ_DATA_FIFO_FULL_MASK                                                      0x80000000L
-//SDMA1_UTCL1_INV0
-#define SDMA1_UTCL1_INV0__INV_MIDDLE__SHIFT                                                                   0x0
-#define SDMA1_UTCL1_INV0__RD_TIMEOUT__SHIFT                                                                   0x1
-#define SDMA1_UTCL1_INV0__WR_TIMEOUT__SHIFT                                                                   0x2
-#define SDMA1_UTCL1_INV0__RD_IN_INVADR__SHIFT                                                                 0x3
-#define SDMA1_UTCL1_INV0__WR_IN_INVADR__SHIFT                                                                 0x4
-#define SDMA1_UTCL1_INV0__PAGE_NULL_SW__SHIFT                                                                 0x5
-#define SDMA1_UTCL1_INV0__XNACK_IS_INVADR__SHIFT                                                              0x6
-#define SDMA1_UTCL1_INV0__INVREQ_ENABLE__SHIFT                                                                0x7
-#define SDMA1_UTCL1_INV0__NACK_TIMEOUT_SW__SHIFT                                                              0x8
-#define SDMA1_UTCL1_INV0__NFLUSH_INV_IDLE__SHIFT                                                              0x9
-#define SDMA1_UTCL1_INV0__FLUSH_INV_IDLE__SHIFT                                                               0xa
-#define SDMA1_UTCL1_INV0__INV_FLUSHTYPE__SHIFT                                                                0xb
-#define SDMA1_UTCL1_INV0__INV_VMID_VEC__SHIFT                                                                 0xc
-#define SDMA1_UTCL1_INV0__INV_ADDR_HI__SHIFT                                                                  0x1c
-#define SDMA1_UTCL1_INV0__INV_MIDDLE_MASK                                                                     0x00000001L
-#define SDMA1_UTCL1_INV0__RD_TIMEOUT_MASK                                                                     0x00000002L
-#define SDMA1_UTCL1_INV0__WR_TIMEOUT_MASK                                                                     0x00000004L
-#define SDMA1_UTCL1_INV0__RD_IN_INVADR_MASK                                                                   0x00000008L
-#define SDMA1_UTCL1_INV0__WR_IN_INVADR_MASK                                                                   0x00000010L
-#define SDMA1_UTCL1_INV0__PAGE_NULL_SW_MASK                                                                   0x00000020L
-#define SDMA1_UTCL1_INV0__XNACK_IS_INVADR_MASK                                                                0x00000040L
-#define SDMA1_UTCL1_INV0__INVREQ_ENABLE_MASK                                                                  0x00000080L
-#define SDMA1_UTCL1_INV0__NACK_TIMEOUT_SW_MASK                                                                0x00000100L
-#define SDMA1_UTCL1_INV0__NFLUSH_INV_IDLE_MASK                                                                0x00000200L
-#define SDMA1_UTCL1_INV0__FLUSH_INV_IDLE_MASK                                                                 0x00000400L
-#define SDMA1_UTCL1_INV0__INV_FLUSHTYPE_MASK                                                                  0x00000800L
-#define SDMA1_UTCL1_INV0__INV_VMID_VEC_MASK                                                                   0x0FFFF000L
-#define SDMA1_UTCL1_INV0__INV_ADDR_HI_MASK                                                                    0xF0000000L
-//SDMA1_UTCL1_INV1
-#define SDMA1_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                                  0x0
-#define SDMA1_UTCL1_INV1__INV_ADDR_LO_MASK                                                                    0xFFFFFFFFL
-//SDMA1_UTCL1_INV2
-#define SDMA1_UTCL1_INV2__INV_NFLUSH_VMID_VEC__SHIFT                                                          0x0
-#define SDMA1_UTCL1_INV2__INV_NFLUSH_VMID_VEC_MASK                                                            0xFFFFFFFFL
-//SDMA1_UTCL1_RD_XNACK0
-#define SDMA1_UTCL1_RD_XNACK0__XNACK_ADDR_LO__SHIFT                                                           0x0
-#define SDMA1_UTCL1_RD_XNACK0__XNACK_ADDR_LO_MASK                                                             0xFFFFFFFFL
-//SDMA1_UTCL1_RD_XNACK1
-#define SDMA1_UTCL1_RD_XNACK1__XNACK_ADDR_HI__SHIFT                                                           0x0
-#define SDMA1_UTCL1_RD_XNACK1__XNACK_VMID__SHIFT                                                              0x4
-#define SDMA1_UTCL1_RD_XNACK1__XNACK_VECTOR__SHIFT                                                            0x8
-#define SDMA1_UTCL1_RD_XNACK1__IS_XNACK__SHIFT                                                                0x1a
-#define SDMA1_UTCL1_RD_XNACK1__XNACK_ADDR_HI_MASK                                                             0x0000000FL
-#define SDMA1_UTCL1_RD_XNACK1__XNACK_VMID_MASK                                                                0x000000F0L
-#define SDMA1_UTCL1_RD_XNACK1__XNACK_VECTOR_MASK                                                              0x03FFFF00L
-#define SDMA1_UTCL1_RD_XNACK1__IS_XNACK_MASK                                                                  0x0C000000L
-//SDMA1_UTCL1_WR_XNACK0
-#define SDMA1_UTCL1_WR_XNACK0__XNACK_ADDR_LO__SHIFT                                                           0x0
-#define SDMA1_UTCL1_WR_XNACK0__XNACK_ADDR_LO_MASK                                                             0xFFFFFFFFL
-//SDMA1_UTCL1_WR_XNACK1
-#define SDMA1_UTCL1_WR_XNACK1__XNACK_ADDR_HI__SHIFT                                                           0x0
-#define SDMA1_UTCL1_WR_XNACK1__XNACK_VMID__SHIFT                                                              0x4
-#define SDMA1_UTCL1_WR_XNACK1__XNACK_VECTOR__SHIFT                                                            0x8
-#define SDMA1_UTCL1_WR_XNACK1__IS_XNACK__SHIFT                                                                0x1a
-#define SDMA1_UTCL1_WR_XNACK1__XNACK_ADDR_HI_MASK                                                             0x0000000FL
-#define SDMA1_UTCL1_WR_XNACK1__XNACK_VMID_MASK                                                                0x000000F0L
-#define SDMA1_UTCL1_WR_XNACK1__XNACK_VECTOR_MASK                                                              0x03FFFF00L
-#define SDMA1_UTCL1_WR_XNACK1__IS_XNACK_MASK                                                                  0x0C000000L
-//SDMA1_UTCL1_TIMEOUT
-#define SDMA1_UTCL1_TIMEOUT__RD_XNACK_LIMIT__SHIFT                                                            0x0
-#define SDMA1_UTCL1_TIMEOUT__WR_XNACK_LIMIT__SHIFT                                                            0x10
-#define SDMA1_UTCL1_TIMEOUT__RD_XNACK_LIMIT_MASK                                                              0x0000FFFFL
-#define SDMA1_UTCL1_TIMEOUT__WR_XNACK_LIMIT_MASK                                                              0xFFFF0000L
-//SDMA1_UTCL1_PAGE
-#define SDMA1_UTCL1_PAGE__VM_HOLE__SHIFT                                                                      0x0
-#define SDMA1_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                     0x1
-#define SDMA1_UTCL1_PAGE__USE_MTYPE__SHIFT                                                                    0x6
-#define SDMA1_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                                 0x9
-#define SDMA1_UTCL1_PAGE__VM_HOLE_MASK                                                                        0x00000001L
-#define SDMA1_UTCL1_PAGE__REQ_TYPE_MASK                                                                       0x0000001EL
-#define SDMA1_UTCL1_PAGE__USE_MTYPE_MASK                                                                      0x000001C0L
-#define SDMA1_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                                   0x00000200L
-//SDMA1_POWER_CNTL_IDLE
-#define SDMA1_POWER_CNTL_IDLE__DELAY0__SHIFT                                                                  0x0
-#define SDMA1_POWER_CNTL_IDLE__DELAY1__SHIFT                                                                  0x10
-#define SDMA1_POWER_CNTL_IDLE__DELAY2__SHIFT                                                                  0x18
-#define SDMA1_POWER_CNTL_IDLE__DELAY0_MASK                                                                    0x0000FFFFL
-#define SDMA1_POWER_CNTL_IDLE__DELAY1_MASK                                                                    0x00FF0000L
-#define SDMA1_POWER_CNTL_IDLE__DELAY2_MASK                                                                    0xFF000000L
-//SDMA1_RELAX_ORDERING_LUT
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED0__SHIFT                                                            0x0
-#define SDMA1_RELAX_ORDERING_LUT__COPY__SHIFT                                                                 0x1
-#define SDMA1_RELAX_ORDERING_LUT__WRITE__SHIFT                                                                0x2
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED3__SHIFT                                                            0x3
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED4__SHIFT                                                            0x4
-#define SDMA1_RELAX_ORDERING_LUT__FENCE__SHIFT                                                                0x5
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED76__SHIFT                                                           0x6
-#define SDMA1_RELAX_ORDERING_LUT__POLL_MEM__SHIFT                                                             0x8
-#define SDMA1_RELAX_ORDERING_LUT__COND_EXE__SHIFT                                                             0x9
-#define SDMA1_RELAX_ORDERING_LUT__ATOMIC__SHIFT                                                               0xa
-#define SDMA1_RELAX_ORDERING_LUT__CONST_FILL__SHIFT                                                           0xb
-#define SDMA1_RELAX_ORDERING_LUT__PTEPDE__SHIFT                                                               0xc
-#define SDMA1_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT                                                            0xd
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED__SHIFT                                                             0xe
-#define SDMA1_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT                                                         0x1b
-#define SDMA1_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT                                                             0x1c
-#define SDMA1_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT                                                            0x1d
-#define SDMA1_RELAX_ORDERING_LUT__IB_FETCH__SHIFT                                                             0x1e
-#define SDMA1_RELAX_ORDERING_LUT__RB_FETCH__SHIFT                                                             0x1f
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED0_MASK                                                              0x00000001L
-#define SDMA1_RELAX_ORDERING_LUT__COPY_MASK                                                                   0x00000002L
-#define SDMA1_RELAX_ORDERING_LUT__WRITE_MASK                                                                  0x00000004L
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED3_MASK                                                              0x00000008L
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED4_MASK                                                              0x00000010L
-#define SDMA1_RELAX_ORDERING_LUT__FENCE_MASK                                                                  0x00000020L
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED76_MASK                                                             0x000000C0L
-#define SDMA1_RELAX_ORDERING_LUT__POLL_MEM_MASK                                                               0x00000100L
-#define SDMA1_RELAX_ORDERING_LUT__COND_EXE_MASK                                                               0x00000200L
-#define SDMA1_RELAX_ORDERING_LUT__ATOMIC_MASK                                                                 0x00000400L
-#define SDMA1_RELAX_ORDERING_LUT__CONST_FILL_MASK                                                             0x00000800L
-#define SDMA1_RELAX_ORDERING_LUT__PTEPDE_MASK                                                                 0x00001000L
-#define SDMA1_RELAX_ORDERING_LUT__TIMESTAMP_MASK                                                              0x00002000L
-#define SDMA1_RELAX_ORDERING_LUT__RESERVED_MASK                                                               0x07FFC000L
-#define SDMA1_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK                                                           0x08000000L
-#define SDMA1_RELAX_ORDERING_LUT__RPTR_WRB_MASK                                                               0x10000000L
-#define SDMA1_RELAX_ORDERING_LUT__WPTR_POLL_MASK                                                              0x20000000L
-#define SDMA1_RELAX_ORDERING_LUT__IB_FETCH_MASK                                                               0x40000000L
-#define SDMA1_RELAX_ORDERING_LUT__RB_FETCH_MASK                                                               0x80000000L
-//SDMA1_CHICKEN_BITS_2
-#define SDMA1_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT                                                       0x0
-#define SDMA1_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK                                                         0x0000000FL
-//SDMA1_STATUS3_REG
-#define SDMA1_STATUS3_REG__CMD_OP_STATUS__SHIFT                                                               0x0
-#define SDMA1_STATUS3_REG__PREV_VM_CMD__SHIFT                                                                 0x10
-#define SDMA1_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                              0x14
-#define SDMA1_STATUS3_REG__CMD_OP_STATUS_MASK                                                                 0x0000FFFFL
-#define SDMA1_STATUS3_REG__PREV_VM_CMD_MASK                                                                   0x000F0000L
-#define SDMA1_STATUS3_REG__EXCEPTION_IDLE_MASK                                                                0x00100000L
-//SDMA1_PHYSICAL_ADDR_LO
-#define SDMA1_PHYSICAL_ADDR_LO__D_VALID__SHIFT                                                                0x0
-#define SDMA1_PHYSICAL_ADDR_LO__DIRTY__SHIFT                                                                  0x1
-#define SDMA1_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT                                                              0x2
-#define SDMA1_PHYSICAL_ADDR_LO__ADDR__SHIFT                                                                   0xc
-#define SDMA1_PHYSICAL_ADDR_LO__D_VALID_MASK                                                                  0x00000001L
-#define SDMA1_PHYSICAL_ADDR_LO__DIRTY_MASK                                                                    0x00000002L
-#define SDMA1_PHYSICAL_ADDR_LO__PHY_VALID_MASK                                                                0x00000004L
-#define SDMA1_PHYSICAL_ADDR_LO__ADDR_MASK                                                                     0xFFFFF000L
-//SDMA1_PHYSICAL_ADDR_HI
-#define SDMA1_PHYSICAL_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA1_PHYSICAL_ADDR_HI__ADDR_MASK                                                                     0x0000FFFFL
-//SDMA1_PHASE2_QUANTUM
-#define SDMA1_PHASE2_QUANTUM__UNIT__SHIFT                                                                     0x0
-#define SDMA1_PHASE2_QUANTUM__VALUE__SHIFT                                                                    0x8
-#define SDMA1_PHASE2_QUANTUM__PREFER__SHIFT                                                                   0x1e
-#define SDMA1_PHASE2_QUANTUM__UNIT_MASK                                                                       0x0000000FL
-#define SDMA1_PHASE2_QUANTUM__VALUE_MASK                                                                      0x00FFFF00L
-#define SDMA1_PHASE2_QUANTUM__PREFER_MASK                                                                     0x40000000L
-//SDMA1_ERROR_LOG
-#define SDMA1_ERROR_LOG__OVERRIDE__SHIFT                                                                      0x0
-#define SDMA1_ERROR_LOG__STATUS__SHIFT                                                                        0x10
-#define SDMA1_ERROR_LOG__OVERRIDE_MASK                                                                        0x0000FFFFL
-#define SDMA1_ERROR_LOG__STATUS_MASK                                                                          0xFFFF0000L
-//SDMA1_PUB_DUMMY_REG0
-#define SDMA1_PUB_DUMMY_REG0__VALUE__SHIFT                                                                    0x0
-#define SDMA1_PUB_DUMMY_REG0__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA1_PUB_DUMMY_REG1
-#define SDMA1_PUB_DUMMY_REG1__VALUE__SHIFT                                                                    0x0
-#define SDMA1_PUB_DUMMY_REG1__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA1_PUB_DUMMY_REG2
-#define SDMA1_PUB_DUMMY_REG2__VALUE__SHIFT                                                                    0x0
-#define SDMA1_PUB_DUMMY_REG2__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA1_PUB_DUMMY_REG3
-#define SDMA1_PUB_DUMMY_REG3__VALUE__SHIFT                                                                    0x0
-#define SDMA1_PUB_DUMMY_REG3__VALUE_MASK                                                                      0xFFFFFFFFL
-//SDMA1_F32_COUNTER
-#define SDMA1_F32_COUNTER__VALUE__SHIFT                                                                       0x0
-#define SDMA1_F32_COUNTER__VALUE_MASK                                                                         0xFFFFFFFFL
-//SDMA1_UNBREAKABLE
-#define SDMA1_UNBREAKABLE__VALUE__SHIFT                                                                       0x0
-#define SDMA1_UNBREAKABLE__VALUE_MASK                                                                         0x00000001L
-//SDMA1_PERFMON_CNTL
-#define SDMA1_PERFMON_CNTL__PERF_ENABLE0__SHIFT                                                               0x0
-#define SDMA1_PERFMON_CNTL__PERF_CLEAR0__SHIFT                                                                0x1
-#define SDMA1_PERFMON_CNTL__PERF_SEL0__SHIFT                                                                  0x2
-#define SDMA1_PERFMON_CNTL__PERF_ENABLE1__SHIFT                                                               0xa
-#define SDMA1_PERFMON_CNTL__PERF_CLEAR1__SHIFT                                                                0xb
-#define SDMA1_PERFMON_CNTL__PERF_SEL1__SHIFT                                                                  0xc
-#define SDMA1_PERFMON_CNTL__PERF_ENABLE0_MASK                                                                 0x00000001L
-#define SDMA1_PERFMON_CNTL__PERF_CLEAR0_MASK                                                                  0x00000002L
-#define SDMA1_PERFMON_CNTL__PERF_SEL0_MASK                                                                    0x000003FCL
-#define SDMA1_PERFMON_CNTL__PERF_ENABLE1_MASK                                                                 0x00000400L
-#define SDMA1_PERFMON_CNTL__PERF_CLEAR1_MASK                                                                  0x00000800L
-#define SDMA1_PERFMON_CNTL__PERF_SEL1_MASK                                                                    0x000FF000L
-//SDMA1_PERFCOUNTER0_RESULT
-#define SDMA1_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT                                                          0x0
-#define SDMA1_PERFCOUNTER0_RESULT__PERF_COUNT_MASK                                                            0xFFFFFFFFL
-//SDMA1_PERFCOUNTER1_RESULT
-#define SDMA1_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT                                                          0x0
-#define SDMA1_PERFCOUNTER1_RESULT__PERF_COUNT_MASK                                                            0xFFFFFFFFL
-//SDMA1_PERFCOUNTER_TAG_DELAY_RANGE
-#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW__SHIFT                                                   0x0
-#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH__SHIFT                                                  0xe
-#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW__SHIFT                                                   0x1c
-#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_LOW_MASK                                                     0x00003FFFL
-#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__RANGE_HIGH_MASK                                                    0x0FFFC000L
-#define SDMA1_PERFCOUNTER_TAG_DELAY_RANGE__SELECT_RW_MASK                                                     0x10000000L
-//SDMA1_CRD_CNTL
-#define SDMA1_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT                                                                0x7
-#define SDMA1_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT                                                                0xd
-#define SDMA1_CRD_CNTL__MC_WRREQ_CREDIT_MASK                                                                  0x00001F80L
-#define SDMA1_CRD_CNTL__MC_RDREQ_CREDIT_MASK                                                                  0x0007E000L
-//SDMA1_MMHUB_TRUSTLVL
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG0__SHIFT                                                                 0x0
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG1__SHIFT                                                                 0x3
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG2__SHIFT                                                                 0x6
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG3__SHIFT                                                                 0x9
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG4__SHIFT                                                                 0xc
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG5__SHIFT                                                                 0xf
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG6__SHIFT                                                                 0x12
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG7__SHIFT                                                                 0x15
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG0_MASK                                                                   0x00000007L
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG1_MASK                                                                   0x00000038L
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG2_MASK                                                                   0x000001C0L
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG3_MASK                                                                   0x00000E00L
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG4_MASK                                                                   0x00007000L
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG5_MASK                                                                   0x00038000L
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG6_MASK                                                                   0x001C0000L
-#define SDMA1_MMHUB_TRUSTLVL__SECFLAG7_MASK                                                                   0x00E00000L
-//SDMA1_GPU_IOV_VIOLATION_LOG
-#define SDMA1_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                  0x0
-#define SDMA1_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                         0x1
-#define SDMA1_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                           0x2
-#define SDMA1_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                                   0x12
-#define SDMA1_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                0x13
-#define SDMA1_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                              0x14
-#define SDMA1_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT                                                      0x18
-#define SDMA1_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                    0x00000001L
-#define SDMA1_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                           0x00000002L
-#define SDMA1_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                             0x0003FFFCL
-#define SDMA1_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                     0x00040000L
-#define SDMA1_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                  0x00080000L
-#define SDMA1_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                                0x00F00000L
-#define SDMA1_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK                                                        0xFF000000L
-//SDMA1_ULV_CNTL
-#define SDMA1_ULV_CNTL__HYSTERESIS__SHIFT                                                                     0x0
-#define SDMA1_ULV_CNTL__ENTER_ULV_INT__SHIFT                                                                  0x1d
-#define SDMA1_ULV_CNTL__EXIT_ULV_INT__SHIFT                                                                   0x1e
-#define SDMA1_ULV_CNTL__ULV_STATUS__SHIFT                                                                     0x1f
-#define SDMA1_ULV_CNTL__HYSTERESIS_MASK                                                                       0x0000001FL
-#define SDMA1_ULV_CNTL__ENTER_ULV_INT_MASK                                                                    0x20000000L
-#define SDMA1_ULV_CNTL__EXIT_ULV_INT_MASK                                                                     0x40000000L
-#define SDMA1_ULV_CNTL__ULV_STATUS_MASK                                                                       0x80000000L
-//SDMA1_EA_DBIT_ADDR_DATA
-#define SDMA1_EA_DBIT_ADDR_DATA__VALUE__SHIFT                                                                 0x0
-#define SDMA1_EA_DBIT_ADDR_DATA__VALUE_MASK                                                                   0xFFFFFFFFL
-//SDMA1_EA_DBIT_ADDR_INDEX
-#define SDMA1_EA_DBIT_ADDR_INDEX__VALUE__SHIFT                                                                0x0
-#define SDMA1_EA_DBIT_ADDR_INDEX__VALUE_MASK                                                                  0x00000007L
-//SDMA1_GFX_RB_CNTL
-#define SDMA1_GFX_RB_CNTL__RB_ENABLE__SHIFT                                                                   0x0
-#define SDMA1_GFX_RB_CNTL__RB_SIZE__SHIFT                                                                     0x1
-#define SDMA1_GFX_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                              0x9
-#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                       0xc
-#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                  0xd
-#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                        0x10
-#define SDMA1_GFX_RB_CNTL__RB_PRIV__SHIFT                                                                     0x17
-#define SDMA1_GFX_RB_CNTL__RB_VMID__SHIFT                                                                     0x18
-#define SDMA1_GFX_RB_CNTL__RB_ENABLE_MASK                                                                     0x00000001L
-#define SDMA1_GFX_RB_CNTL__RB_SIZE_MASK                                                                       0x0000007EL
-#define SDMA1_GFX_RB_CNTL__RB_SWAP_ENABLE_MASK                                                                0x00000200L
-#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                         0x00001000L
-#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                    0x00002000L
-#define SDMA1_GFX_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                          0x001F0000L
-#define SDMA1_GFX_RB_CNTL__RB_PRIV_MASK                                                                       0x00800000L
-#define SDMA1_GFX_RB_CNTL__RB_VMID_MASK                                                                       0x0F000000L
-//SDMA1_GFX_RB_BASE
-#define SDMA1_GFX_RB_BASE__ADDR__SHIFT                                                                        0x0
-#define SDMA1_GFX_RB_BASE__ADDR_MASK                                                                          0xFFFFFFFFL
-//SDMA1_GFX_RB_BASE_HI
-#define SDMA1_GFX_RB_BASE_HI__ADDR__SHIFT                                                                     0x0
-#define SDMA1_GFX_RB_BASE_HI__ADDR_MASK                                                                       0x00FFFFFFL
-//SDMA1_GFX_RB_RPTR
-#define SDMA1_GFX_RB_RPTR__OFFSET__SHIFT                                                                      0x0
-#define SDMA1_GFX_RB_RPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
-//SDMA1_GFX_RB_RPTR_HI
-#define SDMA1_GFX_RB_RPTR_HI__OFFSET__SHIFT                                                                   0x0
-#define SDMA1_GFX_RB_RPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
-//SDMA1_GFX_RB_WPTR
-#define SDMA1_GFX_RB_WPTR__OFFSET__SHIFT                                                                      0x0
-#define SDMA1_GFX_RB_WPTR__OFFSET_MASK                                                                        0xFFFFFFFFL
-//SDMA1_GFX_RB_WPTR_HI
-#define SDMA1_GFX_RB_WPTR_HI__OFFSET__SHIFT                                                                   0x0
-#define SDMA1_GFX_RB_WPTR_HI__OFFSET_MASK                                                                     0xFFFFFFFFL
-//SDMA1_GFX_RB_WPTR_POLL_CNTL
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                            0x0
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                       0x1
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                   0x2
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                         0x4
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                   0x10
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                              0x00000001L
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                         0x00000002L
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                     0x00000004L
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                           0x0000FFF0L
-#define SDMA1_GFX_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                     0xFFFF0000L
-//SDMA1_GFX_RB_RPTR_ADDR_HI
-#define SDMA1_GFX_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                                0x0
-#define SDMA1_GFX_RB_RPTR_ADDR_HI__ADDR_MASK                                                                  0xFFFFFFFFL
-//SDMA1_GFX_RB_RPTR_ADDR_LO
-#define SDMA1_GFX_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                                0x2
-#define SDMA1_GFX_RB_RPTR_ADDR_LO__ADDR_MASK                                                                  0xFFFFFFFCL
-//SDMA1_GFX_IB_CNTL
-#define SDMA1_GFX_IB_CNTL__IB_ENABLE__SHIFT                                                                   0x0
-#define SDMA1_GFX_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                              0x4
-#define SDMA1_GFX_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                            0x8
-#define SDMA1_GFX_IB_CNTL__CMD_VMID__SHIFT                                                                    0x10
-#define SDMA1_GFX_IB_CNTL__IB_ENABLE_MASK                                                                     0x00000001L
-#define SDMA1_GFX_IB_CNTL__IB_SWAP_ENABLE_MASK                                                                0x00000010L
-#define SDMA1_GFX_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                              0x00000100L
-#define SDMA1_GFX_IB_CNTL__CMD_VMID_MASK                                                                      0x000F0000L
-//SDMA1_GFX_IB_RPTR
-#define SDMA1_GFX_IB_RPTR__OFFSET__SHIFT                                                                      0x2
-#define SDMA1_GFX_IB_RPTR__OFFSET_MASK                                                                        0x003FFFFCL
-//SDMA1_GFX_IB_OFFSET
-#define SDMA1_GFX_IB_OFFSET__OFFSET__SHIFT                                                                    0x2
-#define SDMA1_GFX_IB_OFFSET__OFFSET_MASK                                                                      0x003FFFFCL
-//SDMA1_GFX_IB_BASE_LO
-#define SDMA1_GFX_IB_BASE_LO__ADDR__SHIFT                                                                     0x5
-#define SDMA1_GFX_IB_BASE_LO__ADDR_MASK                                                                       0xFFFFFFE0L
-//SDMA1_GFX_IB_BASE_HI
-#define SDMA1_GFX_IB_BASE_HI__ADDR__SHIFT                                                                     0x0
-#define SDMA1_GFX_IB_BASE_HI__ADDR_MASK                                                                       0xFFFFFFFFL
-//SDMA1_GFX_IB_SIZE
-#define SDMA1_GFX_IB_SIZE__SIZE__SHIFT                                                                        0x0
-#define SDMA1_GFX_IB_SIZE__SIZE_MASK                                                                          0x000FFFFFL
-//SDMA1_GFX_SKIP_CNTL
-#define SDMA1_GFX_SKIP_CNTL__SKIP_COUNT__SHIFT                                                                0x0
-#define SDMA1_GFX_SKIP_CNTL__SKIP_COUNT_MASK                                                                  0x00003FFFL
-//SDMA1_GFX_CONTEXT_STATUS
-#define SDMA1_GFX_CONTEXT_STATUS__SELECTED__SHIFT                                                             0x0
-#define SDMA1_GFX_CONTEXT_STATUS__IDLE__SHIFT                                                                 0x2
-#define SDMA1_GFX_CONTEXT_STATUS__EXPIRED__SHIFT                                                              0x3
-#define SDMA1_GFX_CONTEXT_STATUS__EXCEPTION__SHIFT                                                            0x4
-#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                           0x7
-#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                          0x8
-#define SDMA1_GFX_CONTEXT_STATUS__PREEMPTED__SHIFT                                                            0x9
-#define SDMA1_GFX_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                      0xa
-#define SDMA1_GFX_CONTEXT_STATUS__SELECTED_MASK                                                               0x00000001L
-#define SDMA1_GFX_CONTEXT_STATUS__IDLE_MASK                                                                   0x00000004L
-#define SDMA1_GFX_CONTEXT_STATUS__EXPIRED_MASK                                                                0x00000008L
-#define SDMA1_GFX_CONTEXT_STATUS__EXCEPTION_MASK                                                              0x00000070L
-#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                             0x00000080L
-#define SDMA1_GFX_CONTEXT_STATUS__CTXSW_READY_MASK                                                            0x00000100L
-#define SDMA1_GFX_CONTEXT_STATUS__PREEMPTED_MASK                                                              0x00000200L
-#define SDMA1_GFX_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                        0x00000400L
-//SDMA1_GFX_DOORBELL
-#define SDMA1_GFX_DOORBELL__ENABLE__SHIFT                                                                     0x1c
-#define SDMA1_GFX_DOORBELL__CAPTURED__SHIFT                                                                   0x1e
-#define SDMA1_GFX_DOORBELL__ENABLE_MASK                                                                       0x10000000L
-#define SDMA1_GFX_DOORBELL__CAPTURED_MASK                                                                     0x40000000L
-//SDMA1_GFX_CONTEXT_CNTL
-#define SDMA1_GFX_CONTEXT_CNTL__RESUME_CTX__SHIFT                                                             0x10
-#define SDMA1_GFX_CONTEXT_CNTL__RESUME_CTX_MASK                                                               0x00010000L
-//SDMA1_GFX_STATUS
-#define SDMA1_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                       0x0
-#define SDMA1_GFX_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                          0x8
-#define SDMA1_GFX_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                         0x000000FFL
-#define SDMA1_GFX_STATUS__WPTR_UPDATE_PENDING_MASK                                                            0x00000100L
-//SDMA1_GFX_DOORBELL_LOG
-#define SDMA1_GFX_DOORBELL_LOG__BE_ERROR__SHIFT                                                               0x0
-#define SDMA1_GFX_DOORBELL_LOG__DATA__SHIFT                                                                   0x2
-#define SDMA1_GFX_DOORBELL_LOG__BE_ERROR_MASK                                                                 0x00000001L
-#define SDMA1_GFX_DOORBELL_LOG__DATA_MASK                                                                     0xFFFFFFFCL
-//SDMA1_GFX_WATERMARK
-#define SDMA1_GFX_WATERMARK__RD_OUTSTANDING__SHIFT                                                            0x0
-#define SDMA1_GFX_WATERMARK__WR_OUTSTANDING__SHIFT                                                            0x10
-#define SDMA1_GFX_WATERMARK__RD_OUTSTANDING_MASK                                                              0x00000FFFL
-#define SDMA1_GFX_WATERMARK__WR_OUTSTANDING_MASK                                                              0x03FF0000L
-//SDMA1_GFX_DOORBELL_OFFSET
-#define SDMA1_GFX_DOORBELL_OFFSET__OFFSET__SHIFT                                                              0x2
-#define SDMA1_GFX_DOORBELL_OFFSET__OFFSET_MASK                                                                0x0FFFFFFCL
-//SDMA1_GFX_CSA_ADDR_LO
-#define SDMA1_GFX_CSA_ADDR_LO__ADDR__SHIFT                                                                    0x2
-#define SDMA1_GFX_CSA_ADDR_LO__ADDR_MASK                                                                      0xFFFFFFFCL
-//SDMA1_GFX_CSA_ADDR_HI
-#define SDMA1_GFX_CSA_ADDR_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA1_GFX_CSA_ADDR_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA1_GFX_IB_SUB_REMAIN
-#define SDMA1_GFX_IB_SUB_REMAIN__SIZE__SHIFT                                                                  0x0
-#define SDMA1_GFX_IB_SUB_REMAIN__SIZE_MASK                                                                    0x00003FFFL
-//SDMA1_GFX_PREEMPT
-#define SDMA1_GFX_PREEMPT__IB_PREEMPT__SHIFT                                                                  0x0
-#define SDMA1_GFX_PREEMPT__IB_PREEMPT_MASK                                                                    0x00000001L
-//SDMA1_GFX_DUMMY_REG
-#define SDMA1_GFX_DUMMY_REG__DUMMY__SHIFT                                                                     0x0
-#define SDMA1_GFX_DUMMY_REG__DUMMY_MASK                                                                       0xFFFFFFFFL
-//SDMA1_GFX_RB_WPTR_POLL_ADDR_HI
-#define SDMA1_GFX_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                           0x0
-#define SDMA1_GFX_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                             0xFFFFFFFFL
-//SDMA1_GFX_RB_WPTR_POLL_ADDR_LO
-#define SDMA1_GFX_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                           0x2
-#define SDMA1_GFX_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                             0xFFFFFFFCL
-//SDMA1_GFX_RB_AQL_CNTL
-#define SDMA1_GFX_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                              0x0
-#define SDMA1_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                         0x1
-#define SDMA1_GFX_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                             0x8
-#define SDMA1_GFX_RB_AQL_CNTL__AQL_ENABLE_MASK                                                                0x00000001L
-#define SDMA1_GFX_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                           0x000000FEL
-#define SDMA1_GFX_RB_AQL_CNTL__PACKET_STEP_MASK                                                               0x0000FF00L
-//SDMA1_GFX_MINOR_PTR_UPDATE
-#define SDMA1_GFX_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                             0x0
-#define SDMA1_GFX_MINOR_PTR_UPDATE__ENABLE_MASK                                                               0x00000001L
-//SDMA1_GFX_MIDCMD_DATA0
-#define SDMA1_GFX_MIDCMD_DATA0__DATA0__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA0__DATA0_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA1
-#define SDMA1_GFX_MIDCMD_DATA1__DATA1__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA1__DATA1_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA2
-#define SDMA1_GFX_MIDCMD_DATA2__DATA2__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA2__DATA2_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA3
-#define SDMA1_GFX_MIDCMD_DATA3__DATA3__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA3__DATA3_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA4
-#define SDMA1_GFX_MIDCMD_DATA4__DATA4__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA4__DATA4_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA5
-#define SDMA1_GFX_MIDCMD_DATA5__DATA5__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA5__DATA5_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA6
-#define SDMA1_GFX_MIDCMD_DATA6__DATA6__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA6__DATA6_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA7
-#define SDMA1_GFX_MIDCMD_DATA7__DATA7__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA7__DATA7_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_DATA8
-#define SDMA1_GFX_MIDCMD_DATA8__DATA8__SHIFT                                                                  0x0
-#define SDMA1_GFX_MIDCMD_DATA8__DATA8_MASK                                                                    0xFFFFFFFFL
-//SDMA1_GFX_MIDCMD_CNTL
-#define SDMA1_GFX_MIDCMD_CNTL__DATA_VALID__SHIFT                                                              0x0
-#define SDMA1_GFX_MIDCMD_CNTL__COPY_MODE__SHIFT                                                               0x1
-#define SDMA1_GFX_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                             0x4
-#define SDMA1_GFX_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                           0x8
-#define SDMA1_GFX_MIDCMD_CNTL__DATA_VALID_MASK                                                                0x00000001L
-#define SDMA1_GFX_MIDCMD_CNTL__COPY_MODE_MASK                                                                 0x00000002L
-#define SDMA1_GFX_MIDCMD_CNTL__SPLIT_STATE_MASK                                                               0x000000F0L
-#define SDMA1_GFX_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                             0x00000100L
-//SDMA1_PAGE_RB_CNTL
-#define SDMA1_PAGE_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA1_PAGE_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA1_PAGE_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA1_PAGE_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA1_PAGE_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA1_PAGE_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA1_PAGE_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA1_PAGE_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA1_PAGE_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA1_PAGE_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA1_PAGE_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA1_PAGE_RB_BASE
-#define SDMA1_PAGE_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA1_PAGE_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA1_PAGE_RB_BASE_HI
-#define SDMA1_PAGE_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA1_PAGE_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA1_PAGE_RB_RPTR
-#define SDMA1_PAGE_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA1_PAGE_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA1_PAGE_RB_RPTR_HI
-#define SDMA1_PAGE_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA1_PAGE_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA1_PAGE_RB_WPTR
-#define SDMA1_PAGE_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA1_PAGE_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA1_PAGE_RB_WPTR_HI
-#define SDMA1_PAGE_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA1_PAGE_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA1_PAGE_RB_WPTR_POLL_CNTL
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA1_PAGE_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA1_PAGE_RB_RPTR_ADDR_HI
-#define SDMA1_PAGE_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA1_PAGE_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA1_PAGE_RB_RPTR_ADDR_LO
-#define SDMA1_PAGE_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA1_PAGE_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA1_PAGE_IB_CNTL
-#define SDMA1_PAGE_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA1_PAGE_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA1_PAGE_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA1_PAGE_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA1_PAGE_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA1_PAGE_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA1_PAGE_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA1_PAGE_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA1_PAGE_IB_RPTR
-#define SDMA1_PAGE_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA1_PAGE_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA1_PAGE_IB_OFFSET
-#define SDMA1_PAGE_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA1_PAGE_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA1_PAGE_IB_BASE_LO
-#define SDMA1_PAGE_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA1_PAGE_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA1_PAGE_IB_BASE_HI
-#define SDMA1_PAGE_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA1_PAGE_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA1_PAGE_IB_SIZE
-#define SDMA1_PAGE_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA1_PAGE_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA1_PAGE_SKIP_CNTL
-#define SDMA1_PAGE_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA1_PAGE_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA1_PAGE_CONTEXT_STATUS
-#define SDMA1_PAGE_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA1_PAGE_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA1_PAGE_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA1_PAGE_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA1_PAGE_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA1_PAGE_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA1_PAGE_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA1_PAGE_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA1_PAGE_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA1_PAGE_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA1_PAGE_DOORBELL
-#define SDMA1_PAGE_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA1_PAGE_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA1_PAGE_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA1_PAGE_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA1_PAGE_STATUS
-#define SDMA1_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA1_PAGE_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA1_PAGE_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA1_PAGE_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA1_PAGE_DOORBELL_LOG
-#define SDMA1_PAGE_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA1_PAGE_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA1_PAGE_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA1_PAGE_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA1_PAGE_WATERMARK
-#define SDMA1_PAGE_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA1_PAGE_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA1_PAGE_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA1_PAGE_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA1_PAGE_DOORBELL_OFFSET
-#define SDMA1_PAGE_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA1_PAGE_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA1_PAGE_CSA_ADDR_LO
-#define SDMA1_PAGE_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA1_PAGE_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA1_PAGE_CSA_ADDR_HI
-#define SDMA1_PAGE_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA1_PAGE_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA1_PAGE_IB_SUB_REMAIN
-#define SDMA1_PAGE_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA1_PAGE_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA1_PAGE_PREEMPT
-#define SDMA1_PAGE_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA1_PAGE_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA1_PAGE_DUMMY_REG
-#define SDMA1_PAGE_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA1_PAGE_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA1_PAGE_RB_WPTR_POLL_ADDR_HI
-#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA1_PAGE_RB_WPTR_POLL_ADDR_LO
-#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA1_PAGE_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA1_PAGE_RB_AQL_CNTL
-#define SDMA1_PAGE_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA1_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA1_PAGE_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA1_PAGE_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA1_PAGE_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA1_PAGE_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA1_PAGE_MINOR_PTR_UPDATE
-#define SDMA1_PAGE_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA1_PAGE_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA1_PAGE_MIDCMD_DATA0
-#define SDMA1_PAGE_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA1
-#define SDMA1_PAGE_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA2
-#define SDMA1_PAGE_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA3
-#define SDMA1_PAGE_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA4
-#define SDMA1_PAGE_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA5
-#define SDMA1_PAGE_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA6
-#define SDMA1_PAGE_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA7
-#define SDMA1_PAGE_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_DATA8
-#define SDMA1_PAGE_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA1_PAGE_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA1_PAGE_MIDCMD_CNTL
-#define SDMA1_PAGE_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA1_PAGE_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA1_PAGE_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA1_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA1_PAGE_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA1_PAGE_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA1_PAGE_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA1_PAGE_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-//SDMA1_RLC0_RB_CNTL
-#define SDMA1_RLC0_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA1_RLC0_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA1_RLC0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA1_RLC0_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA1_RLC0_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA1_RLC0_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA1_RLC0_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA1_RLC0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA1_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA1_RLC0_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA1_RLC0_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA1_RLC0_RB_BASE
-#define SDMA1_RLC0_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA1_RLC0_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA1_RLC0_RB_BASE_HI
-#define SDMA1_RLC0_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA1_RLC0_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA1_RLC0_RB_RPTR
-#define SDMA1_RLC0_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA1_RLC0_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA1_RLC0_RB_RPTR_HI
-#define SDMA1_RLC0_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA1_RLC0_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA1_RLC0_RB_WPTR
-#define SDMA1_RLC0_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA1_RLC0_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA1_RLC0_RB_WPTR_HI
-#define SDMA1_RLC0_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA1_RLC0_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA1_RLC0_RB_WPTR_POLL_CNTL
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA1_RLC0_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA1_RLC0_RB_RPTR_ADDR_HI
-#define SDMA1_RLC0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA1_RLC0_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA1_RLC0_RB_RPTR_ADDR_LO
-#define SDMA1_RLC0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA1_RLC0_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA1_RLC0_IB_CNTL
-#define SDMA1_RLC0_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA1_RLC0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA1_RLC0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA1_RLC0_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA1_RLC0_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA1_RLC0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA1_RLC0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA1_RLC0_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA1_RLC0_IB_RPTR
-#define SDMA1_RLC0_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA1_RLC0_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA1_RLC0_IB_OFFSET
-#define SDMA1_RLC0_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA1_RLC0_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA1_RLC0_IB_BASE_LO
-#define SDMA1_RLC0_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA1_RLC0_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA1_RLC0_IB_BASE_HI
-#define SDMA1_RLC0_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA1_RLC0_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA1_RLC0_IB_SIZE
-#define SDMA1_RLC0_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA1_RLC0_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA1_RLC0_SKIP_CNTL
-#define SDMA1_RLC0_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA1_RLC0_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA1_RLC0_CONTEXT_STATUS
-#define SDMA1_RLC0_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA1_RLC0_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA1_RLC0_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA1_RLC0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA1_RLC0_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA1_RLC0_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA1_RLC0_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA1_RLC0_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA1_RLC0_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA1_RLC0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA1_RLC0_DOORBELL
-#define SDMA1_RLC0_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA1_RLC0_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA1_RLC0_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA1_RLC0_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA1_RLC0_STATUS
-#define SDMA1_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA1_RLC0_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA1_RLC0_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA1_RLC0_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA1_RLC0_DOORBELL_LOG
-#define SDMA1_RLC0_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA1_RLC0_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA1_RLC0_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA1_RLC0_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA1_RLC0_WATERMARK
-#define SDMA1_RLC0_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA1_RLC0_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA1_RLC0_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA1_RLC0_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA1_RLC0_DOORBELL_OFFSET
-#define SDMA1_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA1_RLC0_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA1_RLC0_CSA_ADDR_LO
-#define SDMA1_RLC0_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA1_RLC0_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA1_RLC0_CSA_ADDR_HI
-#define SDMA1_RLC0_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA1_RLC0_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA1_RLC0_IB_SUB_REMAIN
-#define SDMA1_RLC0_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA1_RLC0_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA1_RLC0_PREEMPT
-#define SDMA1_RLC0_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA1_RLC0_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA1_RLC0_DUMMY_REG
-#define SDMA1_RLC0_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA1_RLC0_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA1_RLC0_RB_WPTR_POLL_ADDR_HI
-#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA1_RLC0_RB_WPTR_POLL_ADDR_LO
-#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA1_RLC0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA1_RLC0_RB_AQL_CNTL
-#define SDMA1_RLC0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA1_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA1_RLC0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA1_RLC0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA1_RLC0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA1_RLC0_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA1_RLC0_MINOR_PTR_UPDATE
-#define SDMA1_RLC0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA1_RLC0_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA1_RLC0_MIDCMD_DATA0
-#define SDMA1_RLC0_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA1
-#define SDMA1_RLC0_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA2
-#define SDMA1_RLC0_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA3
-#define SDMA1_RLC0_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA4
-#define SDMA1_RLC0_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA5
-#define SDMA1_RLC0_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA6
-#define SDMA1_RLC0_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA7
-#define SDMA1_RLC0_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_DATA8
-#define SDMA1_RLC0_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA1_RLC0_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC0_MIDCMD_CNTL
-#define SDMA1_RLC0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA1_RLC0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA1_RLC0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA1_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA1_RLC0_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA1_RLC0_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA1_RLC0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA1_RLC0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-//SDMA1_RLC1_RB_CNTL
-#define SDMA1_RLC1_RB_CNTL__RB_ENABLE__SHIFT                                                                  0x0
-#define SDMA1_RLC1_RB_CNTL__RB_SIZE__SHIFT                                                                    0x1
-#define SDMA1_RLC1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                             0x9
-#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                      0xc
-#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                 0xd
-#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                       0x10
-#define SDMA1_RLC1_RB_CNTL__RB_PRIV__SHIFT                                                                    0x17
-#define SDMA1_RLC1_RB_CNTL__RB_VMID__SHIFT                                                                    0x18
-#define SDMA1_RLC1_RB_CNTL__RB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA1_RLC1_RB_CNTL__RB_SIZE_MASK                                                                      0x0000007EL
-#define SDMA1_RLC1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                               0x00000200L
-#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                        0x00001000L
-#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                   0x00002000L
-#define SDMA1_RLC1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                         0x001F0000L
-#define SDMA1_RLC1_RB_CNTL__RB_PRIV_MASK                                                                      0x00800000L
-#define SDMA1_RLC1_RB_CNTL__RB_VMID_MASK                                                                      0x0F000000L
-//SDMA1_RLC1_RB_BASE
-#define SDMA1_RLC1_RB_BASE__ADDR__SHIFT                                                                       0x0
-#define SDMA1_RLC1_RB_BASE__ADDR_MASK                                                                         0xFFFFFFFFL
-//SDMA1_RLC1_RB_BASE_HI
-#define SDMA1_RLC1_RB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA1_RLC1_RB_BASE_HI__ADDR_MASK                                                                      0x00FFFFFFL
-//SDMA1_RLC1_RB_RPTR
-#define SDMA1_RLC1_RB_RPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA1_RLC1_RB_RPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA1_RLC1_RB_RPTR_HI
-#define SDMA1_RLC1_RB_RPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA1_RLC1_RB_RPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA1_RLC1_RB_WPTR
-#define SDMA1_RLC1_RB_WPTR__OFFSET__SHIFT                                                                     0x0
-#define SDMA1_RLC1_RB_WPTR__OFFSET_MASK                                                                       0xFFFFFFFFL
-//SDMA1_RLC1_RB_WPTR_HI
-#define SDMA1_RLC1_RB_WPTR_HI__OFFSET__SHIFT                                                                  0x0
-#define SDMA1_RLC1_RB_WPTR_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
-//SDMA1_RLC1_RB_WPTR_POLL_CNTL
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__ENABLE__SHIFT                                                           0x0
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE__SHIFT                                                      0x1
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE__SHIFT                                                  0x2
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY__SHIFT                                                        0x4
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                  0x10
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__ENABLE_MASK                                                             0x00000001L
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__SWAP_ENABLE_MASK                                                        0x00000002L
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__F32_POLL_ENABLE_MASK                                                    0x00000004L
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__FREQUENCY_MASK                                                          0x0000FFF0L
-#define SDMA1_RLC1_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                    0xFFFF0000L
-//SDMA1_RLC1_RB_RPTR_ADDR_HI
-#define SDMA1_RLC1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                               0x0
-#define SDMA1_RLC1_RB_RPTR_ADDR_HI__ADDR_MASK                                                                 0xFFFFFFFFL
-//SDMA1_RLC1_RB_RPTR_ADDR_LO
-#define SDMA1_RLC1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                               0x2
-#define SDMA1_RLC1_RB_RPTR_ADDR_LO__ADDR_MASK                                                                 0xFFFFFFFCL
-//SDMA1_RLC1_IB_CNTL
-#define SDMA1_RLC1_IB_CNTL__IB_ENABLE__SHIFT                                                                  0x0
-#define SDMA1_RLC1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                             0x4
-#define SDMA1_RLC1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                           0x8
-#define SDMA1_RLC1_IB_CNTL__CMD_VMID__SHIFT                                                                   0x10
-#define SDMA1_RLC1_IB_CNTL__IB_ENABLE_MASK                                                                    0x00000001L
-#define SDMA1_RLC1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                               0x00000010L
-#define SDMA1_RLC1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                             0x00000100L
-#define SDMA1_RLC1_IB_CNTL__CMD_VMID_MASK                                                                     0x000F0000L
-//SDMA1_RLC1_IB_RPTR
-#define SDMA1_RLC1_IB_RPTR__OFFSET__SHIFT                                                                     0x2
-#define SDMA1_RLC1_IB_RPTR__OFFSET_MASK                                                                       0x003FFFFCL
-//SDMA1_RLC1_IB_OFFSET
-#define SDMA1_RLC1_IB_OFFSET__OFFSET__SHIFT                                                                   0x2
-#define SDMA1_RLC1_IB_OFFSET__OFFSET_MASK                                                                     0x003FFFFCL
-//SDMA1_RLC1_IB_BASE_LO
-#define SDMA1_RLC1_IB_BASE_LO__ADDR__SHIFT                                                                    0x5
-#define SDMA1_RLC1_IB_BASE_LO__ADDR_MASK                                                                      0xFFFFFFE0L
-//SDMA1_RLC1_IB_BASE_HI
-#define SDMA1_RLC1_IB_BASE_HI__ADDR__SHIFT                                                                    0x0
-#define SDMA1_RLC1_IB_BASE_HI__ADDR_MASK                                                                      0xFFFFFFFFL
-//SDMA1_RLC1_IB_SIZE
-#define SDMA1_RLC1_IB_SIZE__SIZE__SHIFT                                                                       0x0
-#define SDMA1_RLC1_IB_SIZE__SIZE_MASK                                                                         0x000FFFFFL
-//SDMA1_RLC1_SKIP_CNTL
-#define SDMA1_RLC1_SKIP_CNTL__SKIP_COUNT__SHIFT                                                               0x0
-#define SDMA1_RLC1_SKIP_CNTL__SKIP_COUNT_MASK                                                                 0x00003FFFL
-//SDMA1_RLC1_CONTEXT_STATUS
-#define SDMA1_RLC1_CONTEXT_STATUS__SELECTED__SHIFT                                                            0x0
-#define SDMA1_RLC1_CONTEXT_STATUS__IDLE__SHIFT                                                                0x2
-#define SDMA1_RLC1_CONTEXT_STATUS__EXPIRED__SHIFT                                                             0x3
-#define SDMA1_RLC1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                           0x4
-#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                          0x7
-#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_READY__SHIFT                                                         0x8
-#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPTED__SHIFT                                                           0x9
-#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                     0xa
-#define SDMA1_RLC1_CONTEXT_STATUS__SELECTED_MASK                                                              0x00000001L
-#define SDMA1_RLC1_CONTEXT_STATUS__IDLE_MASK                                                                  0x00000004L
-#define SDMA1_RLC1_CONTEXT_STATUS__EXPIRED_MASK                                                               0x00000008L
-#define SDMA1_RLC1_CONTEXT_STATUS__EXCEPTION_MASK                                                             0x00000070L
-#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                            0x00000080L
-#define SDMA1_RLC1_CONTEXT_STATUS__CTXSW_READY_MASK                                                           0x00000100L
-#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPTED_MASK                                                             0x00000200L
-#define SDMA1_RLC1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                       0x00000400L
-//SDMA1_RLC1_DOORBELL
-#define SDMA1_RLC1_DOORBELL__ENABLE__SHIFT                                                                    0x1c
-#define SDMA1_RLC1_DOORBELL__CAPTURED__SHIFT                                                                  0x1e
-#define SDMA1_RLC1_DOORBELL__ENABLE_MASK                                                                      0x10000000L
-#define SDMA1_RLC1_DOORBELL__CAPTURED_MASK                                                                    0x40000000L
-//SDMA1_RLC1_STATUS
-#define SDMA1_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                                      0x0
-#define SDMA1_RLC1_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                         0x8
-#define SDMA1_RLC1_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                                        0x000000FFL
-#define SDMA1_RLC1_STATUS__WPTR_UPDATE_PENDING_MASK                                                           0x00000100L
-//SDMA1_RLC1_DOORBELL_LOG
-#define SDMA1_RLC1_DOORBELL_LOG__BE_ERROR__SHIFT                                                              0x0
-#define SDMA1_RLC1_DOORBELL_LOG__DATA__SHIFT                                                                  0x2
-#define SDMA1_RLC1_DOORBELL_LOG__BE_ERROR_MASK                                                                0x00000001L
-#define SDMA1_RLC1_DOORBELL_LOG__DATA_MASK                                                                    0xFFFFFFFCL
-//SDMA1_RLC1_WATERMARK
-#define SDMA1_RLC1_WATERMARK__RD_OUTSTANDING__SHIFT                                                           0x0
-#define SDMA1_RLC1_WATERMARK__WR_OUTSTANDING__SHIFT                                                           0x10
-#define SDMA1_RLC1_WATERMARK__RD_OUTSTANDING_MASK                                                             0x00000FFFL
-#define SDMA1_RLC1_WATERMARK__WR_OUTSTANDING_MASK                                                             0x03FF0000L
-//SDMA1_RLC1_DOORBELL_OFFSET
-#define SDMA1_RLC1_DOORBELL_OFFSET__OFFSET__SHIFT                                                             0x2
-#define SDMA1_RLC1_DOORBELL_OFFSET__OFFSET_MASK                                                               0x0FFFFFFCL
-//SDMA1_RLC1_CSA_ADDR_LO
-#define SDMA1_RLC1_CSA_ADDR_LO__ADDR__SHIFT                                                                   0x2
-#define SDMA1_RLC1_CSA_ADDR_LO__ADDR_MASK                                                                     0xFFFFFFFCL
-//SDMA1_RLC1_CSA_ADDR_HI
-#define SDMA1_RLC1_CSA_ADDR_HI__ADDR__SHIFT                                                                   0x0
-#define SDMA1_RLC1_CSA_ADDR_HI__ADDR_MASK                                                                     0xFFFFFFFFL
-//SDMA1_RLC1_IB_SUB_REMAIN
-#define SDMA1_RLC1_IB_SUB_REMAIN__SIZE__SHIFT                                                                 0x0
-#define SDMA1_RLC1_IB_SUB_REMAIN__SIZE_MASK                                                                   0x00003FFFL
-//SDMA1_RLC1_PREEMPT
-#define SDMA1_RLC1_PREEMPT__IB_PREEMPT__SHIFT                                                                 0x0
-#define SDMA1_RLC1_PREEMPT__IB_PREEMPT_MASK                                                                   0x00000001L
-//SDMA1_RLC1_DUMMY_REG
-#define SDMA1_RLC1_DUMMY_REG__DUMMY__SHIFT                                                                    0x0
-#define SDMA1_RLC1_DUMMY_REG__DUMMY_MASK                                                                      0xFFFFFFFFL
-//SDMA1_RLC1_RB_WPTR_POLL_ADDR_HI
-#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                          0x0
-#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                            0xFFFFFFFFL
-//SDMA1_RLC1_RB_WPTR_POLL_ADDR_LO
-#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                          0x2
-#define SDMA1_RLC1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                            0xFFFFFFFCL
-//SDMA1_RLC1_RB_AQL_CNTL
-#define SDMA1_RLC1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                             0x0
-#define SDMA1_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                        0x1
-#define SDMA1_RLC1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                            0x8
-#define SDMA1_RLC1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                               0x00000001L
-#define SDMA1_RLC1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                          0x000000FEL
-#define SDMA1_RLC1_RB_AQL_CNTL__PACKET_STEP_MASK                                                              0x0000FF00L
-//SDMA1_RLC1_MINOR_PTR_UPDATE
-#define SDMA1_RLC1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                            0x0
-#define SDMA1_RLC1_MINOR_PTR_UPDATE__ENABLE_MASK                                                              0x00000001L
-//SDMA1_RLC1_MIDCMD_DATA0
-#define SDMA1_RLC1_MIDCMD_DATA0__DATA0__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA0__DATA0_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA1
-#define SDMA1_RLC1_MIDCMD_DATA1__DATA1__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA1__DATA1_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA2
-#define SDMA1_RLC1_MIDCMD_DATA2__DATA2__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA2__DATA2_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA3
-#define SDMA1_RLC1_MIDCMD_DATA3__DATA3__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA3__DATA3_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA4
-#define SDMA1_RLC1_MIDCMD_DATA4__DATA4__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA4__DATA4_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA5
-#define SDMA1_RLC1_MIDCMD_DATA5__DATA5__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA5__DATA5_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA6
-#define SDMA1_RLC1_MIDCMD_DATA6__DATA6__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA6__DATA6_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA7
-#define SDMA1_RLC1_MIDCMD_DATA7__DATA7__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA7__DATA7_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_DATA8
-#define SDMA1_RLC1_MIDCMD_DATA8__DATA8__SHIFT                                                                 0x0
-#define SDMA1_RLC1_MIDCMD_DATA8__DATA8_MASK                                                                   0xFFFFFFFFL
-//SDMA1_RLC1_MIDCMD_CNTL
-#define SDMA1_RLC1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                             0x0
-#define SDMA1_RLC1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                              0x1
-#define SDMA1_RLC1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                            0x4
-#define SDMA1_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                          0x8
-#define SDMA1_RLC1_MIDCMD_CNTL__DATA_VALID_MASK                                                               0x00000001L
-#define SDMA1_RLC1_MIDCMD_CNTL__COPY_MODE_MASK                                                                0x00000002L
-#define SDMA1_RLC1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                              0x000000F0L
-#define SDMA1_RLC1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                            0x00000100L
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_default.h
deleted file mode 100644
index 5c186c2e8739..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/SMUIO/smuio_9_0_default.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _smuio_9_0_DEFAULT_HEADER
-#define _smuio_9_0_DEFAULT_HEADER
-
-
-// addressBlock: smuio_smuio_SmuSmuioDec
-#define mmROM_CNTL_DEFAULT                                                       0x00000000
-#define mmROM_STATUS_DEFAULT                                                     0x00000000
-#define mmCGTT_ROM_CLK_CTRL0_DEFAULT                                             0xc0000100
-#define mmROM_INDEX_DEFAULT                                                      0x00000000
-#define mmROM_DATA_DEFAULT                                                       0x00000000
-#define mmROM_START_DEFAULT                                                      0x00000000
-#define mmROM_SW_CNTL_DEFAULT                                                    0x00000000
-#define mmROM_SW_STATUS_DEFAULT                                                  0x00000000
-#define mmROM_SW_COMMAND_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_1_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_2_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_3_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_4_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_5_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_6_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_7_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_8_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_9_DEFAULT                                                  0x00000000
-#define mmROM_SW_DATA_10_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_11_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_12_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_13_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_14_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_15_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_16_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_17_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_18_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_19_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_20_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_21_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_22_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_23_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_24_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_25_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_26_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_27_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_28_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_29_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_30_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_31_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_32_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_33_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_34_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_35_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_36_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_37_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_38_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_39_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_40_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_41_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_42_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_43_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_44_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_45_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_46_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_47_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_48_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_49_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_50_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_51_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_52_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_53_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_54_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_55_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_56_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_57_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_58_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_59_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_60_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_61_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_62_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_63_DEFAULT                                                 0x00000000
-#define mmROM_SW_DATA_64_DEFAULT                                                 0x00000000
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_default.h b/drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_default.h
deleted file mode 100644
index 48963caac534..000000000000
--- a/drivers/gpu/drm/amd/include/asic_reg/vega10/UVD/uvd_7_0_default.h
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright (C) 2017  Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
- * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-#ifndef _uvd_7_0_DEFAULT_HEADER
-#define _uvd_7_0_DEFAULT_HEADER
-
-
-// addressBlock: uvd0_uvd_pg_dec
-#define mmUVD_POWER_STATUS_DEFAULT                                               0x00000000
-#define mmUVD_DPG_RBC_RB_CNTL_DEFAULT                                            0x01000101
-#define mmUVD_DPG_RBC_RB_BASE_LOW_DEFAULT                                        0x00000000
-#define mmUVD_DPG_RBC_RB_BASE_HIGH_DEFAULT                                       0x00000000
-#define mmUVD_DPG_RBC_RB_WPTR_CNTL_DEFAULT                                       0x00000000
-#define mmUVD_DPG_RBC_RB_RPTR_DEFAULT                                            0x00000000
-#define mmUVD_DPG_RBC_RB_WPTR_DEFAULT                                            0x00000000
-#define mmUVD_DPG_LMI_VCPU_CACHE_64BIT_BAR_LOW_DEFAULT                           0x00000000
-#define mmUVD_DPG_LMI_VCPU_CACHE_64BIT_BAR_HIGH_DEFAULT                          0x00000000
-#define mmUVD_DPG_VCPU_CACHE_OFFSET0_DEFAULT                                     0x00000000
-
-
-// addressBlock: uvd0_uvdnpdec
-#define mmUVD_JPEG_ADDR_CONFIG_DEFAULT                                           0x22010010
-#define mmUVD_GPCOM_VCPU_CMD_DEFAULT                                             0x00000000
-#define mmUVD_GPCOM_VCPU_DATA0_DEFAULT                                           0x00000000
-#define mmUVD_GPCOM_VCPU_DATA1_DEFAULT                                           0x00000000
-#define mmUVD_UDEC_ADDR_CONFIG_DEFAULT                                           0x22010010
-#define mmUVD_UDEC_DB_ADDR_CONFIG_DEFAULT                                        0x22010010
-#define mmUVD_UDEC_DBW_ADDR_CONFIG_DEFAULT                                       0x22010010
-#define mmUVD_SUVD_CGC_GATE_DEFAULT                                              0x00000000
-#define mmUVD_SUVD_CGC_CTRL_DEFAULT                                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW_DEFAULT                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH_DEFAULT                             0x00000000
-#define mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW_DEFAULT                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH_DEFAULT                             0x00000000
-#define mmUVD_POWER_STATUS_U_DEFAULT                                             0x00000000
-#define mmUVD_NO_OP_DEFAULT                                                      0x00000000
-#define mmUVD_GP_SCRATCH8_DEFAULT                                                0x00000000
-#define mmUVD_RB_BASE_LO2_DEFAULT                                                0x00000000
-#define mmUVD_RB_BASE_HI2_DEFAULT                                                0x00000000
-#define mmUVD_RB_SIZE2_DEFAULT                                                   0x00000000
-#define mmUVD_RB_RPTR2_DEFAULT                                                   0x00000000
-#define mmUVD_RB_WPTR2_DEFAULT                                                   0x00000000
-#define mmUVD_RB_BASE_LO_DEFAULT                                                 0x00000000
-#define mmUVD_RB_BASE_HI_DEFAULT                                                 0x00000000
-#define mmUVD_RB_SIZE_DEFAULT                                                    0x00000000
-#define mmUVD_RB_RPTR_DEFAULT                                                    0x00000000
-#define mmUVD_RB_WPTR_DEFAULT                                                    0x00000000
-#define mmUVD_JRBC_RB_RPTR_DEFAULT                                               0x00000000
-#define mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH_DEFAULT                              0x00000000
-#define mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW_DEFAULT                               0x00000000
-#define mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH_DEFAULT                                  0x00000000
-#define mmUVD_LMI_RBC_IB_64BIT_BAR_LOW_DEFAULT                                   0x00000000
-#define mmUVD_LMI_RBC_RB_64BIT_BAR_HIGH_DEFAULT                                  0x00000000
-#define mmUVD_LMI_RBC_RB_64BIT_BAR_LOW_DEFAULT                                   0x00000000
-
-
-// addressBlock: uvd0_uvddec
-#define mmUVD_SEMA_CNTL_DEFAULT                                                  0x00000003
-#define mmUVD_LMI_JRBC_RB_64BIT_BAR_LOW_DEFAULT                                  0x00000000
-#define mmUVD_JRBC_RB_WPTR_DEFAULT                                               0x00000000
-#define mmUVD_RB_RPTR3_DEFAULT                                                   0x00000000
-#define mmUVD_RB_WPTR3_DEFAULT                                                   0x00000000
-#define mmUVD_RB_BASE_LO3_DEFAULT                                                0x00000000
-#define mmUVD_RB_BASE_HI3_DEFAULT                                                0x00000000
-#define mmUVD_RB_SIZE3_DEFAULT                                                   0x00000000
-#define mmJPEG_CGC_GATE_DEFAULT                                                  0x00300000
-#define mmUVD_CTX_INDEX_DEFAULT                                                  0x00000000
-#define mmUVD_CTX_DATA_DEFAULT                                                   0x00000000
-#define mmUVD_CGC_GATE_DEFAULT                                                   0x000fffff
-#define mmUVD_CGC_CTRL_DEFAULT                                                   0x1fff018d
-#define mmUVD_GP_SCRATCH4_DEFAULT                                                0x00000000
-#define mmUVD_LMI_CTRL2_DEFAULT                                                  0x003e0000
-#define mmUVD_MASTINT_EN_DEFAULT                                                 0x00000000
-#define mmJPEG_CGC_CTRL_DEFAULT                                                  0x0000018d
-#define mmUVD_LMI_CTRL_DEFAULT                                                   0x00104340
-#define mmUVD_LMI_VM_CTRL_DEFAULT                                                0x00000000
-#define mmUVD_LMI_SWAP_CNTL_DEFAULT                                              0x00000000
-#define mmUVD_MP_SWAP_CNTL_DEFAULT                                               0x00000000
-#define mmUVD_MPC_SET_MUXA0_DEFAULT                                              0x00002040
-#define mmUVD_MPC_SET_MUXA1_DEFAULT                                              0x00000000
-#define mmUVD_MPC_SET_MUXB0_DEFAULT                                              0x00002040
-#define mmUVD_MPC_SET_MUXB1_DEFAULT                                              0x00000000
-#define mmUVD_MPC_SET_MUX_DEFAULT                                                0x00000088
-#define mmUVD_MPC_SET_ALU_DEFAULT                                                0x00000000
-#define mmUVD_VCPU_CACHE_OFFSET0_DEFAULT                                         0x00000000
-#define mmUVD_VCPU_CACHE_SIZE0_DEFAULT                                           0x00000000
-#define mmUVD_VCPU_CACHE_OFFSET1_DEFAULT                                         0x00000000
-#define mmUVD_VCPU_CACHE_SIZE1_DEFAULT                                           0x00000000
-#define mmUVD_VCPU_CACHE_OFFSET2_DEFAULT                                         0x00000000
-#define mmUVD_VCPU_CACHE_SIZE2_DEFAULT                                           0x00000000
-#define mmUVD_VCPU_CNTL_DEFAULT                                                  0x0ff20000
-#define mmUVD_SOFT_RESET_DEFAULT                                                 0x00000008
-#define mmUVD_LMI_RBC_IB_VMID_DEFAULT                                            0x00000000
-#define mmUVD_RBC_IB_SIZE_DEFAULT                                                0x00000000
-#define mmUVD_LMI_RBC_RB_VMID_DEFAULT                                            0x00000000
-#define mmUVD_RBC_RB_RPTR_DEFAULT                                                0x00000000
-#define mmUVD_RBC_RB_WPTR_DEFAULT                                                0x00000000
-#define mmUVD_RBC_RB_WPTR_CNTL_DEFAULT                                           0x00000000
-#define mmUVD_RBC_RB_CNTL_DEFAULT                                                0x01000101
-#define mmUVD_RBC_RB_RPTR_ADDR_DEFAULT                                           0x00000000
-#define mmUVD_STATUS_DEFAULT                                                     0x00000000
-#define mmUVD_SEMA_TIMEOUT_STATUS_DEFAULT                                        0x00000000
-#define mmUVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL_DEFAULT                          0x02000000
-#define mmUVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL_DEFAULT                               0x02000000
-#define mmUVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL_DEFAULT                        0x02000000
-#define mmUVD_CONTEXT_ID_DEFAULT                                                 0x00000000
-#define mmUVD_CONTEXT_ID2_DEFAULT                                                0x00000000
-
-
-#endif
diff --git a/drivers/gpu/drm/amd/include/dm_pp_interface.h b/drivers/gpu/drm/amd/include/dm_pp_interface.h
index 7343aed4d019..721473199921 100644
--- a/drivers/gpu/drm/amd/include/dm_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/dm_pp_interface.h
@@ -25,6 +25,145 @@
 
 #define PP_MAX_CLOCK_LEVELS 8
 
+enum amd_pp_display_config_type{
+	AMD_PP_DisplayConfigType_None = 0,
+	AMD_PP_DisplayConfigType_DP54 ,
+	AMD_PP_DisplayConfigType_DP432 ,
+	AMD_PP_DisplayConfigType_DP324 ,
+	AMD_PP_DisplayConfigType_DP27,
+	AMD_PP_DisplayConfigType_DP243,
+	AMD_PP_DisplayConfigType_DP216,
+	AMD_PP_DisplayConfigType_DP162,
+	AMD_PP_DisplayConfigType_HDMI6G ,
+	AMD_PP_DisplayConfigType_HDMI297 ,
+	AMD_PP_DisplayConfigType_HDMI162,
+	AMD_PP_DisplayConfigType_LVDS,
+	AMD_PP_DisplayConfigType_DVI,
+	AMD_PP_DisplayConfigType_WIRELESS,
+	AMD_PP_DisplayConfigType_VGA
+};
+
+struct single_display_configuration
+{
+	uint32_t controller_index;
+	uint32_t controller_id;
+	uint32_t signal_type;
+	uint32_t display_state;
+	/* phy id for the primary internal transmitter */
+	uint8_t primary_transmitter_phyi_d;
+	/* bitmap with the active lanes */
+	uint8_t primary_transmitter_active_lanemap;
+	/* phy id for the secondary internal transmitter (for dual-link dvi) */
+	uint8_t secondary_transmitter_phy_id;
+	/* bitmap with the active lanes */
+	uint8_t secondary_transmitter_active_lanemap;
+	/* misc phy settings for SMU. */
+	uint32_t config_flags;
+	uint32_t display_type;
+	uint32_t view_resolution_cx;
+	uint32_t view_resolution_cy;
+	enum amd_pp_display_config_type displayconfigtype;
+	uint32_t vertical_refresh; /* for active display */
+};
+
+#define MAX_NUM_DISPLAY 32
+
+struct amd_pp_display_configuration {
+	bool nb_pstate_switch_disable;/* controls NB PState switch */
+	bool cpu_cc6_disable; /* controls CPU CState switch ( on or off) */
+	bool cpu_pstate_disable;
+	uint32_t cpu_pstate_separation_time;
+
+	uint32_t num_display;  /* total number of display*/
+	uint32_t num_path_including_non_display;
+	uint32_t crossfire_display_index;
+	uint32_t min_mem_set_clock;
+	uint32_t min_core_set_clock;
+	/* unit 10KHz x bit*/
+	uint32_t min_bus_bandwidth;
+	/* minimum required stutter sclk, in 10khz uint32_t ulMinCoreSetClk;*/
+	uint32_t min_core_set_clock_in_sr;
+
+	struct single_display_configuration displays[MAX_NUM_DISPLAY];
+
+	uint32_t vrefresh; /* for active display*/
+
+	uint32_t min_vblank_time; /* for active display*/
+	bool multi_monitor_in_sync;
+	/* Controller Index of primary display - used in MCLK SMC switching hang
+	 * SW Workaround*/
+	uint32_t crtc_index;
+	/* htotal*1000/pixelclk - used in MCLK SMC switching hang SW Workaround*/
+	uint32_t line_time_in_us;
+	bool invalid_vblank_time;
+
+	uint32_t display_clk;
+	/*
+	 * for given display configuration if multimonitormnsync == false then
+	 * Memory clock DPMS with this latency or below is allowed, DPMS with
+	 * higher latency not allowed.
+	 */
+	uint32_t dce_tolerable_mclk_in_active_latency;
+	uint32_t min_dcef_set_clk;
+	uint32_t min_dcef_deep_sleep_set_clk;
+};
+
+struct amd_pp_simple_clock_info {
+	uint32_t	engine_max_clock;
+	uint32_t	memory_max_clock;
+	uint32_t	level;
+};
+
+enum PP_DAL_POWERLEVEL {
+	PP_DAL_POWERLEVEL_INVALID = 0,
+	PP_DAL_POWERLEVEL_ULTRALOW,
+	PP_DAL_POWERLEVEL_LOW,
+	PP_DAL_POWERLEVEL_NOMINAL,
+	PP_DAL_POWERLEVEL_PERFORMANCE,
+
+	PP_DAL_POWERLEVEL_0 = PP_DAL_POWERLEVEL_ULTRALOW,
+	PP_DAL_POWERLEVEL_1 = PP_DAL_POWERLEVEL_LOW,
+	PP_DAL_POWERLEVEL_2 = PP_DAL_POWERLEVEL_NOMINAL,
+	PP_DAL_POWERLEVEL_3 = PP_DAL_POWERLEVEL_PERFORMANCE,
+	PP_DAL_POWERLEVEL_4 = PP_DAL_POWERLEVEL_3+1,
+	PP_DAL_POWERLEVEL_5 = PP_DAL_POWERLEVEL_4+1,
+	PP_DAL_POWERLEVEL_6 = PP_DAL_POWERLEVEL_5+1,
+	PP_DAL_POWERLEVEL_7 = PP_DAL_POWERLEVEL_6+1,
+};
+
+struct amd_pp_clock_info {
+	uint32_t min_engine_clock;
+	uint32_t max_engine_clock;
+	uint32_t min_memory_clock;
+	uint32_t max_memory_clock;
+	uint32_t min_bus_bandwidth;
+	uint32_t max_bus_bandwidth;
+	uint32_t max_engine_clock_in_sr;
+	uint32_t min_engine_clock_in_sr;
+	enum PP_DAL_POWERLEVEL max_clocks_state;
+};
+
+enum amd_pp_clock_type {
+	amd_pp_disp_clock = 1,
+	amd_pp_sys_clock,
+	amd_pp_mem_clock,
+	amd_pp_dcef_clock,
+	amd_pp_soc_clock,
+	amd_pp_pixel_clock,
+	amd_pp_phy_clock,
+	amd_pp_dcf_clock,
+	amd_pp_dpp_clock,
+	amd_pp_f_clock = amd_pp_dcef_clock,
+};
+
+#define MAX_NUM_CLOCKS 16
+
+struct amd_pp_clocks {
+	uint32_t count;
+	uint32_t clock[MAX_NUM_CLOCKS];
+	uint32_t latency[MAX_NUM_CLOCKS];
+};
+
 struct pp_clock_with_latency {
 	uint32_t clocks_in_khz;
 	uint32_t latency_in_us;
@@ -45,6 +184,11 @@ struct pp_clock_levels_with_voltage {
 	struct pp_clock_with_voltage data[PP_MAX_CLOCK_LEVELS];
 };
 
+struct pp_display_clock_request {
+	enum amd_pp_clock_type clock_type;
+	uint32_t clock_freq_in_khz;
+};
+
 #define PP_MAX_WM_SETS 4
 
 enum pp_wm_set_id {
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index f516fd10e6ba..a6752bd0c871 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -46,6 +46,28 @@ enum kfd_preempt_type {
 	KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 };
 
+struct kfd_cu_info {
+	uint32_t num_shader_engines;
+	uint32_t num_shader_arrays_per_engine;
+	uint32_t num_cu_per_sh;
+	uint32_t cu_active_number;
+	uint32_t cu_ao_mask;
+	uint32_t simd_per_cu;
+	uint32_t max_waves_per_simd;
+	uint32_t wave_front_size;
+	uint32_t max_scratch_slots_per_cu;
+	uint32_t lds_size;
+	uint32_t cu_bitmap[4][4];
+};
+
+/* For getting GPU local memory information from KGD */
+struct kfd_local_mem_info {
+	uint64_t local_mem_size_private;
+	uint64_t local_mem_size_public;
+	uint32_t vram_width;
+	uint32_t mem_clk_max;
+};
+
 enum kgd_memory_pool {
 	KGD_POOL_SYSTEM_CACHEABLE = 1,
 	KGD_POOL_SYSTEM_WRITECOMBINE = 2,
@@ -106,7 +128,7 @@ struct tile_config {
  *
  * @free_gtt_mem: Frees a buffer that was allocated on the gart aperture
  *
- * @get_vmem_size: Retrieves (physical) size of VRAM
+ * @get_local_mem_info: Retrieves information about GPU local memory
  *
  * @get_gpu_clock_counter: Retrieves GPU clock counter
  *
@@ -131,6 +153,12 @@ struct tile_config {
  * @hqd_sdma_load: Loads the SDMA mqd structure to a H/W SDMA hqd slot.
  * used only for no HWS mode.
  *
+ * @hqd_dump: Dumps CPC HQD registers to an array of address-value pairs.
+ * Array is allocated with kmalloc, needs to be freed with kfree by caller.
+ *
+ * @hqd_sdma_dump: Dumps SDMA HQD registers to an array of address-value pairs.
+ * Array is allocated with kmalloc, needs to be freed with kfree by caller.
+ *
  * @hqd_is_occupies: Checks if a hqd slot is occupied.
  *
  * @hqd_destroy: Destructs and preempts the queue assigned to that hqd slot.
@@ -147,6 +175,10 @@ struct tile_config {
  *
  * @get_tile_config: Returns GPU-specific tiling mode information
  *
+ * @get_cu_info: Retrieves activated cu info
+ *
+ * @get_vram_usage: Returns current VRAM usage
+ *
  * This structure contains function pointers to services that the kgd driver
  * provides to amdkfd driver.
  *
@@ -158,7 +190,8 @@ struct kfd2kgd_calls {
 
 	void (*free_gtt_mem)(struct kgd_dev *kgd, void *mem_obj);
 
-	uint64_t (*get_vmem_size)(struct kgd_dev *kgd);
+	void (*get_local_mem_info)(struct kgd_dev *kgd,
+			struct kfd_local_mem_info *mem_info);
 	uint64_t (*get_gpu_clock_counter)(struct kgd_dev *kgd);
 
 	uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd);
@@ -184,7 +217,16 @@ struct kfd2kgd_calls {
 			uint32_t wptr_shift, uint32_t wptr_mask,
 			struct mm_struct *mm);
 
-	int (*hqd_sdma_load)(struct kgd_dev *kgd, void *mqd);
+	int (*hqd_sdma_load)(struct kgd_dev *kgd, void *mqd,
+			     uint32_t __user *wptr, struct mm_struct *mm);
+
+	int (*hqd_dump)(struct kgd_dev *kgd,
+			uint32_t pipe_id, uint32_t queue_id,
+			uint32_t (**dump)[2], uint32_t *n_regs);
+
+	int (*hqd_sdma_dump)(struct kgd_dev *kgd,
+			     uint32_t engine_id, uint32_t queue_id,
+			     uint32_t (**dump)[2], uint32_t *n_regs);
 
 	bool (*hqd_is_occupied)(struct kgd_dev *kgd, uint64_t queue_address,
 				uint32_t pipe_id, uint32_t queue_id);
@@ -224,6 +266,10 @@ struct kfd2kgd_calls {
 	void (*set_scratch_backing_va)(struct kgd_dev *kgd,
 				uint64_t va, uint32_t vmid);
 	int (*get_tile_config)(struct kgd_dev *kgd, struct tile_config *config);
+
+	void (*get_cu_info)(struct kgd_dev *kgd,
+			struct kfd_cu_info *cu_info);
+	uint64_t (*get_vram_usage)(struct kgd_dev *kgd);
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
new file mode 100644
index 000000000000..ed27626dff14
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2017 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __KGD_PP_INTERFACE_H__
+#define __KGD_PP_INTERFACE_H__
+
+extern const struct amd_ip_funcs pp_ip_funcs;
+extern const struct amd_pm_funcs pp_dpm_funcs;
+
+struct amd_vce_state {
+	/* vce clocks */
+	u32 evclk;
+	u32 ecclk;
+	/* gpu clocks */
+	u32 sclk;
+	u32 mclk;
+	u8 clk_idx;
+	u8 pstate;
+};
+
+
+enum amd_dpm_forced_level {
+	AMD_DPM_FORCED_LEVEL_AUTO = 0x1,
+	AMD_DPM_FORCED_LEVEL_MANUAL = 0x2,
+	AMD_DPM_FORCED_LEVEL_LOW = 0x4,
+	AMD_DPM_FORCED_LEVEL_HIGH = 0x8,
+	AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD = 0x10,
+	AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK = 0x20,
+	AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK = 0x40,
+	AMD_DPM_FORCED_LEVEL_PROFILE_PEAK = 0x80,
+	AMD_DPM_FORCED_LEVEL_PROFILE_EXIT = 0x100,
+};
+
+enum amd_pm_state_type {
+	/* not used for dpm */
+	POWER_STATE_TYPE_DEFAULT,
+	POWER_STATE_TYPE_POWERSAVE,
+	/* user selectable states */
+	POWER_STATE_TYPE_BATTERY,
+	POWER_STATE_TYPE_BALANCED,
+	POWER_STATE_TYPE_PERFORMANCE,
+	/* internal states */
+	POWER_STATE_TYPE_INTERNAL_UVD,
+	POWER_STATE_TYPE_INTERNAL_UVD_SD,
+	POWER_STATE_TYPE_INTERNAL_UVD_HD,
+	POWER_STATE_TYPE_INTERNAL_UVD_HD2,
+	POWER_STATE_TYPE_INTERNAL_UVD_MVC,
+	POWER_STATE_TYPE_INTERNAL_BOOT,
+	POWER_STATE_TYPE_INTERNAL_THERMAL,
+	POWER_STATE_TYPE_INTERNAL_ACPI,
+	POWER_STATE_TYPE_INTERNAL_ULV,
+	POWER_STATE_TYPE_INTERNAL_3DPERF,
+};
+
+#define AMD_MAX_VCE_LEVELS 6
+
+enum amd_vce_level {
+	AMD_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
+	AMD_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
+	AMD_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
+	AMD_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
+	AMD_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
+	AMD_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
+};
+
+enum amd_pp_profile_type {
+	AMD_PP_GFX_PROFILE,
+	AMD_PP_COMPUTE_PROFILE,
+};
+
+struct amd_pp_profile {
+	enum amd_pp_profile_type type;
+	uint32_t min_sclk;
+	uint32_t min_mclk;
+	uint16_t activity_threshold;
+	uint8_t up_hyst;
+	uint8_t down_hyst;
+};
+
+enum amd_fan_ctrl_mode {
+	AMD_FAN_CTRL_NONE = 0,
+	AMD_FAN_CTRL_MANUAL = 1,
+	AMD_FAN_CTRL_AUTO = 2,
+};
+
+enum pp_clock_type {
+	PP_SCLK,
+	PP_MCLK,
+	PP_PCIE,
+};
+
+enum amd_pp_sensors {
+	AMDGPU_PP_SENSOR_GFX_SCLK = 0,
+	AMDGPU_PP_SENSOR_VDDNB,
+	AMDGPU_PP_SENSOR_VDDGFX,
+	AMDGPU_PP_SENSOR_UVD_VCLK,
+	AMDGPU_PP_SENSOR_UVD_DCLK,
+	AMDGPU_PP_SENSOR_VCE_ECCLK,
+	AMDGPU_PP_SENSOR_GPU_LOAD,
+	AMDGPU_PP_SENSOR_GFX_MCLK,
+	AMDGPU_PP_SENSOR_GPU_TEMP,
+	AMDGPU_PP_SENSOR_VCE_POWER,
+	AMDGPU_PP_SENSOR_UVD_POWER,
+	AMDGPU_PP_SENSOR_GPU_POWER,
+};
+
+enum amd_pp_task {
+	AMD_PP_TASK_DISPLAY_CONFIG_CHANGE,
+	AMD_PP_TASK_ENABLE_USER_STATE,
+	AMD_PP_TASK_READJUST_POWER_STATE,
+	AMD_PP_TASK_COMPLETE_INIT,
+	AMD_PP_TASK_MAX
+};
+
+struct amd_pp_init {
+	struct cgs_device *device;
+	uint32_t chip_family;
+	uint32_t chip_id;
+	bool pm_en;
+	uint32_t feature_mask;
+};
+
+
+
+enum {
+	PP_GROUP_UNKNOWN = 0,
+	PP_GROUP_GFX = 1,
+	PP_GROUP_SYS,
+	PP_GROUP_MAX
+};
+
+struct pp_states_info {
+	uint32_t nums;
+	uint32_t states[16];
+};
+
+struct pp_gpu_power {
+	uint32_t vddc_power;
+	uint32_t vddci_power;
+	uint32_t max_gpu_power;
+	uint32_t average_gpu_power;
+};
+
+#define PP_GROUP_MASK        0xF0000000
+#define PP_GROUP_SHIFT       28
+
+#define PP_BLOCK_MASK        0x0FFFFF00
+#define PP_BLOCK_SHIFT       8
+
+#define PP_BLOCK_GFX_CG         0x01
+#define PP_BLOCK_GFX_MG         0x02
+#define PP_BLOCK_GFX_3D         0x04
+#define PP_BLOCK_GFX_RLC        0x08
+#define PP_BLOCK_GFX_CP         0x10
+#define PP_BLOCK_SYS_BIF        0x01
+#define PP_BLOCK_SYS_MC         0x02
+#define PP_BLOCK_SYS_ROM        0x04
+#define PP_BLOCK_SYS_DRM        0x08
+#define PP_BLOCK_SYS_HDP        0x10
+#define PP_BLOCK_SYS_SDMA       0x20
+
+#define PP_STATE_MASK           0x0000000F
+#define PP_STATE_SHIFT          0
+#define PP_STATE_SUPPORT_MASK   0x000000F0
+#define PP_STATE_SUPPORT_SHIFT  0
+
+#define PP_STATE_CG             0x01
+#define PP_STATE_LS             0x02
+#define PP_STATE_DS             0x04
+#define PP_STATE_SD             0x08
+#define PP_STATE_SUPPORT_CG     0x10
+#define PP_STATE_SUPPORT_LS     0x20
+#define PP_STATE_SUPPORT_DS     0x40
+#define PP_STATE_SUPPORT_SD     0x80
+
+#define PP_CG_MSG_ID(group, block, support, state) \
+		((group) << PP_GROUP_SHIFT | (block) << PP_BLOCK_SHIFT | \
+		(support) << PP_STATE_SUPPORT_SHIFT | (state) << PP_STATE_SHIFT)
+
+struct seq_file;
+enum amd_pp_clock_type;
+struct amd_pp_simple_clock_info;
+struct amd_pp_display_configuration;
+struct amd_pp_clock_info;
+struct pp_display_clock_request;
+struct pp_wm_sets_with_clock_ranges_soc15;
+struct pp_clock_levels_with_voltage;
+struct pp_clock_levels_with_latency;
+struct amd_pp_clocks;
+
+struct amd_pm_funcs {
+/* export for dpm on ci and si */
+	int (*pre_set_power_state)(void *handle);
+	int (*set_power_state)(void *handle);
+	void (*post_set_power_state)(void *handle);
+	void (*display_configuration_changed)(void *handle);
+	void (*print_power_state)(void *handle, void *ps);
+	bool (*vblank_too_short)(void *handle);
+	void (*enable_bapm)(void *handle, bool enable);
+	int (*check_state_equal)(void *handle,
+				void  *cps,
+				void  *rps,
+				bool  *equal);
+/* export for sysfs */
+	int (*get_temperature)(void *handle);
+	void (*set_fan_control_mode)(void *handle, u32 mode);
+	u32 (*get_fan_control_mode)(void *handle);
+	int (*set_fan_speed_percent)(void *handle, u32 speed);
+	int (*get_fan_speed_percent)(void *handle, u32 *speed);
+	int (*force_clock_level)(void *handle, enum pp_clock_type type, uint32_t mask);
+	int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf);
+	int (*force_performance_level)(void *handle, enum amd_dpm_forced_level level);
+	int (*get_sclk_od)(void *handle);
+	int (*set_sclk_od)(void *handle, uint32_t value);
+	int (*get_mclk_od)(void *handle);
+	int (*set_mclk_od)(void *handle, uint32_t value);
+	int (*read_sensor)(void *handle, int idx, void *value, int *size);
+	enum amd_dpm_forced_level (*get_performance_level)(void *handle);
+	enum amd_pm_state_type (*get_current_power_state)(void *handle);
+	int (*get_fan_speed_rpm)(void *handle, uint32_t *rpm);
+	int (*get_pp_num_states)(void *handle, struct pp_states_info *data);
+	int (*get_pp_table)(void *handle, char **table);
+	int (*set_pp_table)(void *handle, const char *buf, size_t size);
+	void (*debugfs_print_current_performance_level)(void *handle, struct seq_file *m);
+
+	int (*reset_power_profile_state)(void *handle,
+			struct amd_pp_profile *request);
+	int (*get_power_profile_state)(void *handle,
+			struct amd_pp_profile *query);
+	int (*set_power_profile_state)(void *handle,
+			struct amd_pp_profile *request);
+	int (*switch_power_profile)(void *handle,
+			enum amd_pp_profile_type type);
+/* export to amdgpu */
+	void (*powergate_uvd)(void *handle, bool gate);
+	void (*powergate_vce)(void *handle, bool gate);
+	struct amd_vce_state *(*get_vce_clock_state)(void *handle, u32 idx);
+	int (*dispatch_tasks)(void *handle, enum amd_pp_task task_id,
+				   void *input, void *output);
+	int (*load_firmware)(void *handle);
+	int (*wait_for_fw_loading_complete)(void *handle);
+	int (*set_clockgating_by_smu)(void *handle, uint32_t msg_id);
+	int (*notify_smu_memory_info)(void *handle, uint32_t virtual_addr_low,
+					uint32_t virtual_addr_hi,
+					uint32_t mc_addr_low,
+					uint32_t mc_addr_hi,
+					uint32_t size);
+/* export to DC */
+	u32 (*get_sclk)(void *handle, bool low);
+	u32 (*get_mclk)(void *handle, bool low);
+	int (*display_configuration_change)(void *handle,
+		const struct amd_pp_display_configuration *input);
+	int (*get_display_power_level)(void *handle,
+		struct amd_pp_simple_clock_info *output);
+	int (*get_current_clocks)(void *handle,
+		struct amd_pp_clock_info *clocks);
+	int (*get_clock_by_type)(void *handle,
+		enum amd_pp_clock_type type,
+		struct amd_pp_clocks *clocks);
+	int (*get_clock_by_type_with_latency)(void *handle,
+		enum amd_pp_clock_type type,
+		struct pp_clock_levels_with_latency *clocks);
+	int (*get_clock_by_type_with_voltage)(void *handle,
+		enum amd_pp_clock_type type,
+		struct pp_clock_levels_with_voltage *clocks);
+	int (*set_watermarks_for_clocks_ranges)(void *handle,
+		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
+	int (*display_clock_voltage_request)(void *handle,
+				struct pp_display_clock_request *clock);
+	int (*get_display_mode_validation_clocks)(void *handle,
+		struct amd_pp_simple_clock_info *clocks);
+};
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/soc15ip.h b/drivers/gpu/drm/amd/include/soc15ip.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/soc15ip.h
rename to drivers/gpu/drm/amd/include/soc15ip.h
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vega10/vega10_enum.h b/drivers/gpu/drm/amd/include/vega10_enum.h
similarity index 100%
rename from drivers/gpu/drm/amd/include/asic_reg/vega10/vega10_enum.h
rename to drivers/gpu/drm/amd/include/vega10_enum.h
diff --git a/drivers/gpu/drm/amd/include/vi_structs.h b/drivers/gpu/drm/amd/include/vi_structs.h
index 20234820194b..717fbae1d362 100644
--- a/drivers/gpu/drm/amd/include/vi_structs.h
+++ b/drivers/gpu/drm/amd/include/vi_structs.h
@@ -153,6 +153,8 @@ struct vi_sdma_mqd {
 	uint32_t reserved_125;
 	uint32_t reserved_126;
 	uint32_t reserved_127;
+	uint32_t sdma_engine_id;
+	uint32_t sdma_queue_id;
 };
 
 struct vi_mqd {
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index c7e34128cbde..adb6e7b9280c 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -122,7 +122,7 @@ static int pp_sw_init(void *handle)
 
 		ret = hwmgr->smumgr_funcs->smu_init(hwmgr);
 
-		pr_info("amdgpu: powerplay sw initialized\n");
+		pr_debug("amdgpu: powerplay sw initialized\n");
 	}
 	return ret;
 }
@@ -162,7 +162,7 @@ static int pp_hw_init(void *handle)
 		if(hwmgr->smumgr_funcs->start_smu(pp_handle->hwmgr)) {
 			pr_err("smc start failed\n");
 			hwmgr->smumgr_funcs->smu_fini(pp_handle->hwmgr);
-			return -EINVAL;;
+			return -EINVAL;
 		}
 		if (ret == PP_DPM_DISABLED)
 			goto exit;
@@ -389,20 +389,12 @@ static int pp_dpm_force_performance_level(void *handle,
 	if (level == hwmgr->dpm_level)
 		return 0;
 
-	if (hwmgr->hwmgr_func->force_dpm_level == NULL) {
-		pr_info("%s was not implemented.\n", __func__);
-		return 0;
-	}
-
 	mutex_lock(&pp_handle->pp_lock);
 	pp_dpm_en_umd_pstate(hwmgr, &level);
 	hwmgr->request_dpm_level = level;
 	hwmgr_handle_task(pp_handle, AMD_PP_TASK_READJUST_POWER_STATE, NULL, NULL);
-	ret = hwmgr->hwmgr_func->force_dpm_level(hwmgr, level);
-	if (!ret)
-		hwmgr->dpm_level = hwmgr->request_dpm_level;
-
 	mutex_unlock(&pp_handle->pp_lock);
+
 	return 0;
 }
 
@@ -726,6 +718,8 @@ static int pp_dpm_get_pp_num_states(void *handle,
 	struct pp_instance *pp_handle = (struct pp_instance *)handle;
 	int ret = 0;
 
+	memset(data, 0, sizeof(*data));
+
 	ret = pp_check(pp_handle);
 
 	if (ret)
@@ -788,6 +782,26 @@ static int pp_dpm_get_pp_table(void *handle, char **table)
 	return size;
 }
 
+static int amd_powerplay_reset(void *handle)
+{
+	struct pp_instance *instance = (struct pp_instance *)handle;
+	int ret;
+
+	ret = pp_check(instance);
+	if (ret)
+		return ret;
+
+	ret = pp_hw_fini(instance);
+	if (ret)
+		return ret;
+
+	ret = hwmgr_hw_init(instance);
+	if (ret)
+		return ret;
+
+	return hwmgr_handle_task(instance, AMD_PP_TASK_COMPLETE_INIT, NULL, NULL);
+}
+
 static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
 {
 	struct pp_hwmgr *hwmgr;
@@ -1146,64 +1160,41 @@ static int pp_dpm_switch_power_profile(void *handle,
 	return 0;
 }
 
-const struct amd_pm_funcs pp_dpm_funcs = {
-	.get_temperature = pp_dpm_get_temperature,
-	.load_firmware = pp_dpm_load_fw,
-	.wait_for_fw_loading_complete = pp_dpm_fw_loading_complete,
-	.force_performance_level = pp_dpm_force_performance_level,
-	.get_performance_level = pp_dpm_get_performance_level,
-	.get_current_power_state = pp_dpm_get_current_power_state,
-	.get_sclk = pp_dpm_get_sclk,
-	.get_mclk = pp_dpm_get_mclk,
-	.powergate_vce = pp_dpm_powergate_vce,
-	.powergate_uvd = pp_dpm_powergate_uvd,
-	.dispatch_tasks = pp_dpm_dispatch_tasks,
-	.set_fan_control_mode = pp_dpm_set_fan_control_mode,
-	.get_fan_control_mode = pp_dpm_get_fan_control_mode,
-	.set_fan_speed_percent = pp_dpm_set_fan_speed_percent,
-	.get_fan_speed_percent = pp_dpm_get_fan_speed_percent,
-	.get_fan_speed_rpm = pp_dpm_get_fan_speed_rpm,
-	.get_pp_num_states = pp_dpm_get_pp_num_states,
-	.get_pp_table = pp_dpm_get_pp_table,
-	.set_pp_table = pp_dpm_set_pp_table,
-	.force_clock_level = pp_dpm_force_clock_level,
-	.print_clock_levels = pp_dpm_print_clock_levels,
-	.get_sclk_od = pp_dpm_get_sclk_od,
-	.set_sclk_od = pp_dpm_set_sclk_od,
-	.get_mclk_od = pp_dpm_get_mclk_od,
-	.set_mclk_od = pp_dpm_set_mclk_od,
-	.read_sensor = pp_dpm_read_sensor,
-	.get_vce_clock_state = pp_dpm_get_vce_clock_state,
-	.reset_power_profile_state = pp_dpm_reset_power_profile_state,
-	.get_power_profile_state = pp_dpm_get_power_profile_state,
-	.set_power_profile_state = pp_dpm_set_power_profile_state,
-	.switch_power_profile = pp_dpm_switch_power_profile,
-	.set_clockgating_by_smu = pp_set_clockgating_by_smu,
-};
-
-int amd_powerplay_reset(void *handle)
+static int pp_dpm_notify_smu_memory_info(void *handle,
+					uint32_t virtual_addr_low,
+					uint32_t virtual_addr_hi,
+					uint32_t mc_addr_low,
+					uint32_t mc_addr_hi,
+					uint32_t size)
 {
-	struct pp_instance *instance = (struct pp_instance *)handle;
-	int ret;
+	struct pp_hwmgr  *hwmgr;
+	struct pp_instance *pp_handle = (struct pp_instance *)handle;
+	int ret = 0;
 
-	ret = pp_check(instance);
-	if (ret)
-		return ret;
+	ret = pp_check(pp_handle);
 
-	ret = pp_hw_fini(instance);
 	if (ret)
 		return ret;
 
-	ret = hwmgr_hw_init(instance);
-	if (ret)
-		return ret;
+	hwmgr = pp_handle->hwmgr;
 
-	return hwmgr_handle_task(instance, AMD_PP_TASK_COMPLETE_INIT, NULL, NULL);
-}
+	if (hwmgr->hwmgr_func->notify_cac_buffer_info == NULL) {
+		pr_info("%s was not implemented.\n", __func__);
+		return -EINVAL;
+	}
 
-/* export this function to DAL */
+	mutex_lock(&pp_handle->pp_lock);
+
+	ret = hwmgr->hwmgr_func->notify_cac_buffer_info(hwmgr, virtual_addr_low,
+					virtual_addr_hi, mc_addr_low, mc_addr_hi,
+					size);
+
+	mutex_unlock(&pp_handle->pp_lock);
 
-int amd_powerplay_display_configuration_change(void *handle,
+	return ret;
+}
+
+static int pp_display_configuration_change(void *handle,
 	const struct amd_pp_display_configuration *display_config)
 {
 	struct pp_hwmgr  *hwmgr;
@@ -1222,7 +1213,7 @@ int amd_powerplay_display_configuration_change(void *handle,
 	return 0;
 }
 
-int amd_powerplay_get_display_power_level(void *handle,
+static int pp_get_display_power_level(void *handle,
 		struct amd_pp_simple_clock_info *output)
 {
 	struct pp_hwmgr  *hwmgr;
@@ -1245,7 +1236,7 @@ int amd_powerplay_get_display_power_level(void *handle,
 	return ret;
 }
 
-int amd_powerplay_get_current_clocks(void *handle,
+static int pp_get_current_clocks(void *handle,
 		struct amd_pp_clock_info *clocks)
 {
 	struct amd_pp_simple_clock_info simple_clocks;
@@ -1299,7 +1290,7 @@ int amd_powerplay_get_current_clocks(void *handle,
 	return 0;
 }
 
-int amd_powerplay_get_clock_by_type(void *handle, enum amd_pp_clock_type type, struct amd_pp_clocks *clocks)
+static int pp_get_clock_by_type(void *handle, enum amd_pp_clock_type type, struct amd_pp_clocks *clocks)
 {
 	struct pp_hwmgr  *hwmgr;
 	struct pp_instance *pp_handle = (struct pp_instance *)handle;
@@ -1321,7 +1312,7 @@ int amd_powerplay_get_clock_by_type(void *handle, enum amd_pp_clock_type type, s
 	return ret;
 }
 
-int amd_powerplay_get_clock_by_type_with_latency(void *handle,
+static int pp_get_clock_by_type_with_latency(void *handle,
 		enum amd_pp_clock_type type,
 		struct pp_clock_levels_with_latency *clocks)
 {
@@ -1343,7 +1334,7 @@ int amd_powerplay_get_clock_by_type_with_latency(void *handle,
 	return ret;
 }
 
-int amd_powerplay_get_clock_by_type_with_voltage(void *handle,
+static int pp_get_clock_by_type_with_voltage(void *handle,
 		enum amd_pp_clock_type type,
 		struct pp_clock_levels_with_voltage *clocks)
 {
@@ -1368,7 +1359,7 @@ int amd_powerplay_get_clock_by_type_with_voltage(void *handle,
 	return ret;
 }
 
-int amd_powerplay_set_watermarks_for_clocks_ranges(void *handle,
+static int pp_set_watermarks_for_clocks_ranges(void *handle,
 		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges)
 {
 	struct pp_hwmgr *hwmgr;
@@ -1392,7 +1383,7 @@ int amd_powerplay_set_watermarks_for_clocks_ranges(void *handle,
 	return ret;
 }
 
-int amd_powerplay_display_clock_voltage_request(void *handle,
+static int pp_display_clock_voltage_request(void *handle,
 		struct pp_display_clock_request *clock)
 {
 	struct pp_hwmgr *hwmgr;
@@ -1415,7 +1406,7 @@ int amd_powerplay_display_clock_voltage_request(void *handle,
 	return ret;
 }
 
-int amd_powerplay_get_display_mode_validation_clocks(void *handle,
+static int pp_get_display_mode_validation_clocks(void *handle,
 		struct amd_pp_simple_clock_info *clocks)
 {
 	struct pp_hwmgr  *hwmgr;
@@ -1441,3 +1432,48 @@ int amd_powerplay_get_display_mode_validation_clocks(void *handle,
 	return ret;
 }
 
+const struct amd_pm_funcs pp_dpm_funcs = {
+	.get_temperature = pp_dpm_get_temperature,
+	.load_firmware = pp_dpm_load_fw,
+	.wait_for_fw_loading_complete = pp_dpm_fw_loading_complete,
+	.force_performance_level = pp_dpm_force_performance_level,
+	.get_performance_level = pp_dpm_get_performance_level,
+	.get_current_power_state = pp_dpm_get_current_power_state,
+	.powergate_vce = pp_dpm_powergate_vce,
+	.powergate_uvd = pp_dpm_powergate_uvd,
+	.dispatch_tasks = pp_dpm_dispatch_tasks,
+	.set_fan_control_mode = pp_dpm_set_fan_control_mode,
+	.get_fan_control_mode = pp_dpm_get_fan_control_mode,
+	.set_fan_speed_percent = pp_dpm_set_fan_speed_percent,
+	.get_fan_speed_percent = pp_dpm_get_fan_speed_percent,
+	.get_fan_speed_rpm = pp_dpm_get_fan_speed_rpm,
+	.get_pp_num_states = pp_dpm_get_pp_num_states,
+	.get_pp_table = pp_dpm_get_pp_table,
+	.set_pp_table = pp_dpm_set_pp_table,
+	.force_clock_level = pp_dpm_force_clock_level,
+	.print_clock_levels = pp_dpm_print_clock_levels,
+	.get_sclk_od = pp_dpm_get_sclk_od,
+	.set_sclk_od = pp_dpm_set_sclk_od,
+	.get_mclk_od = pp_dpm_get_mclk_od,
+	.set_mclk_od = pp_dpm_set_mclk_od,
+	.read_sensor = pp_dpm_read_sensor,
+	.get_vce_clock_state = pp_dpm_get_vce_clock_state,
+	.reset_power_profile_state = pp_dpm_reset_power_profile_state,
+	.get_power_profile_state = pp_dpm_get_power_profile_state,
+	.set_power_profile_state = pp_dpm_set_power_profile_state,
+	.switch_power_profile = pp_dpm_switch_power_profile,
+	.set_clockgating_by_smu = pp_set_clockgating_by_smu,
+	.notify_smu_memory_info = pp_dpm_notify_smu_memory_info,
+/* export to DC */
+	.get_sclk = pp_dpm_get_sclk,
+	.get_mclk = pp_dpm_get_mclk,
+	.display_configuration_change = pp_display_configuration_change,
+	.get_display_power_level = pp_get_display_power_level,
+	.get_current_clocks = pp_get_current_clocks,
+	.get_clock_by_type = pp_get_clock_by_type,
+	.get_clock_by_type_with_latency = pp_get_clock_by_type_with_latency,
+	.get_clock_by_type_with_voltage = pp_get_clock_by_type_with_voltage,
+	.set_watermarks_for_clocks_ranges = pp_set_watermarks_for_clocks_ranges,
+	.display_clock_voltage_request = pp_display_clock_voltage_request,
+	.get_display_mode_validation_clocks = pp_get_display_mode_validation_clocks,
+};
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index ad1f6b57884b..b314d09d41af 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -728,9 +728,6 @@ static int cz_update_sclk_limit(struct pp_hwmgr *hwmgr)
 
 		if (clock < stable_pstate_sclk)
 			clock = stable_pstate_sclk;
-	} else {
-		if (clock < hwmgr->gfx_arbiter.sclk)
-			clock = hwmgr->gfx_arbiter.sclk;
 	}
 
 	if (cz_hwmgr->sclk_dpm.soft_min_clk != clock) {
@@ -1085,14 +1082,8 @@ static int cz_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 	uint32_t  num_of_active_displays = 0;
 	struct cgs_display_info info = {0};
 
-	cz_ps->evclk = hwmgr->vce_arbiter.evclk;
-	cz_ps->ecclk = hwmgr->vce_arbiter.ecclk;
-
 	cz_ps->need_dfs_bypass = true;
 
-	cz_hwmgr->video_start = (hwmgr->uvd_arbiter.vclk != 0 || hwmgr->uvd_arbiter.dclk != 0 ||
-				hwmgr->vce_arbiter.evclk != 0 || hwmgr->vce_arbiter.ecclk != 0);
-
 	cz_hwmgr->battery_state = (PP_StateUILabel_Battery == prequest_ps->classification.ui_label);
 
 	clocks.memoryClock = hwmgr->display_config.min_mem_set_clock != 0 ?
@@ -1105,9 +1096,6 @@ static int cz_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
 		clocks.memoryClock = hwmgr->dyn_state.max_clock_voltage_on_ac.mclk;
 
-	if (clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
-		clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
-
 	force_high = (clocks.memoryClock > cz_hwmgr->sys_info.nbp_memory_clock[CZ_NUM_NBPMEMORYCLOCK - 1])
 			|| (num_of_active_displays >= 3);
 
@@ -1339,22 +1327,13 @@ int  cz_dpm_update_vce_dpm(struct pp_hwmgr *hwmgr)
 				cz_hwmgr->vce_dpm.hard_min_clk,
 				PPSMC_MSG_SetEclkHardMin));
 	} else {
-		/*Program HardMin based on the vce_arbiter.ecclk */
-		if (hwmgr->vce_arbiter.ecclk == 0) {
-			smum_send_msg_to_smc_with_parameter(hwmgr,
-					    PPSMC_MSG_SetEclkHardMin, 0);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+					PPSMC_MSG_SetEclkHardMin, 0);
 		/* disable ECLK DPM 0. Otherwise VCE could hang if
 		 * switching SCLK from DPM 0 to 6/7 */
-			smum_send_msg_to_smc_with_parameter(hwmgr,
+		smum_send_msg_to_smc_with_parameter(hwmgr,
 					PPSMC_MSG_SetEclkSoftMin, 1);
-		} else {
-			cz_hwmgr->vce_dpm.hard_min_clk = hwmgr->vce_arbiter.ecclk;
-			smum_send_msg_to_smc_with_parameter(hwmgr,
-				PPSMC_MSG_SetEclkHardMin,
-				cz_get_eclk_level(hwmgr,
-					cz_hwmgr->vce_dpm.hard_min_clk,
-					PPSMC_MSG_SetEclkHardMin));
-		}
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index 623cff90233d..2b0c53fe4c8d 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -112,26 +112,29 @@ int phm_force_dpm_levels(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level
 
 	PHM_FUNC_CHECK(hwmgr);
 
-	if (hwmgr->hwmgr_func->force_dpm_level != NULL) {
+	if (hwmgr->hwmgr_func->force_dpm_level != NULL)
 		ret = hwmgr->hwmgr_func->force_dpm_level(hwmgr, level);
-		if (ret)
-			return ret;
-
-		if (hwmgr->hwmgr_func->set_power_profile_state) {
-			if (hwmgr->current_power_profile == AMD_PP_GFX_PROFILE)
-				ret = hwmgr->hwmgr_func->set_power_profile_state(
-						hwmgr,
-						&hwmgr->gfx_power_profile);
-			else if (hwmgr->current_power_profile == AMD_PP_COMPUTE_PROFILE)
-				ret = hwmgr->hwmgr_func->set_power_profile_state(
-						hwmgr,
-						&hwmgr->compute_power_profile);
-		}
-	}
 
 	return ret;
 }
 
+int phm_reset_power_profile_state(struct pp_hwmgr *hwmgr)
+{
+	int ret = 0;
+
+	if (hwmgr->hwmgr_func->set_power_profile_state) {
+		if (hwmgr->current_power_profile == AMD_PP_GFX_PROFILE)
+			ret = hwmgr->hwmgr_func->set_power_profile_state(
+					hwmgr,
+					&hwmgr->gfx_power_profile);
+		else if (hwmgr->current_power_profile == AMD_PP_COMPUTE_PROFILE)
+			ret = hwmgr->hwmgr_func->set_power_profile_state(
+					hwmgr,
+					&hwmgr->compute_power_profile);
+	}
+	return ret;
+}
+
 int phm_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 				   struct pp_power_state *adjusted_ps,
 			     const struct pp_power_state *current_ps)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index ce59e0e67cb2..0229f774f7a9 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -149,6 +149,7 @@ int hwmgr_early_init(struct pp_instance *handle)
 	hwmgr->power_source = PP_PowerSource_AC;
 	hwmgr->pp_table_version = PP_TABLE_V1;
 	hwmgr->dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
+	hwmgr->request_dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
 	hwmgr_init_default_caps(hwmgr);
 	hwmgr_set_user_specify_caps(hwmgr);
 	hwmgr->fan_ctrl_is_in_default_mode = true;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h
index c6ba0d64cfb7..4112a9398163 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_overdriver.h
@@ -43,4 +43,4 @@ struct phm_fuses_default {
 extern int pp_override_get_default_fuse_value(uint64_t key,
 			struct phm_fuses_default *result);
 
-#endif
\ No newline at end of file
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c
index ffa44bbb218e..95ab772e0c3e 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_psm.c
@@ -244,6 +244,10 @@ int psm_adjust_power_state_dynamic(struct pp_hwmgr *hwmgr, bool skip,
 	}
 
 	phm_notify_smc_display_config_after_ps_adjustment(hwmgr);
+	if (!phm_force_dpm_levels(hwmgr, hwmgr->request_dpm_level))
+		hwmgr->dpm_level = hwmgr->request_dpm_level;
+
+	phm_reset_power_profile_state(hwmgr);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
index c062844b15f3..560c1c159fcc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomfwctrl.c
@@ -542,4 +542,4 @@ int pp_atomfwctrl_get_vbios_bootup_values(struct pp_hwmgr *hwmgr,
 		boot_values->ulDCEFClk   = frequency;
 
 	return 0;
-}
\ No newline at end of file
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
index a651ebcf44fd..b49d65c3e984 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c
@@ -523,8 +523,7 @@ static int get_pcie_table(
 		if ((uint32_t)atom_pcie_table->ucNumEntries <= pcie_count)
 			pcie_count = (uint32_t)atom_pcie_table->ucNumEntries;
 		else
-			pr_err("Number of Pcie Entries exceed the number of SCLK Dpm Levels! \
-			Disregarding the excess entries... \n");
+			pr_err("Number of Pcie Entries exceed the number of SCLK Dpm Levels! Disregarding the excess entries...\n");
 
 		pcie_table->count = pcie_count;
 		for (i = 0; i < pcie_count; i++) {
@@ -563,8 +562,7 @@ static int get_pcie_table(
 		if ((uint32_t)atom_pcie_table->ucNumEntries <= pcie_count)
 			pcie_count = (uint32_t)atom_pcie_table->ucNumEntries;
 		else
-			pr_err("Number of Pcie Entries exceed the number of SCLK Dpm Levels! \
-			Disregarding the excess entries... \n");
+			pr_err("Number of Pcie Entries exceed the number of SCLK Dpm Levels! Disregarding the excess entries...\n");
 
 		pcie_table->count = pcie_count;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
index afae32ee2b0d..c3e7e34535e8 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
@@ -394,8 +394,8 @@ static int get_clock_voltage_dependency_table(struct pp_hwmgr *hwmgr,
 		dep_table->entries[i].clk =
 			((unsigned long)table->entries[i].ucClockHigh << 16) |
 			le16_to_cpu(table->entries[i].usClockLow);
-			dep_table->entries[i].v =
-				(unsigned long)le16_to_cpu(table->entries[i].usVoltage);
+		dep_table->entries[i].v =
+			(unsigned long)le16_to_cpu(table->entries[i].usVoltage);
 	}
 
 	*ptable = dep_table;
@@ -1042,7 +1042,7 @@ static int init_overdrive_limits_V2_1(struct pp_hwmgr *hwmgr,
 static int init_overdrive_limits(struct pp_hwmgr *hwmgr,
 			const ATOM_PPLIB_POWERPLAYTABLE *powerplay_table)
 {
-	int result;
+	int result = 0;
 	uint8_t frev, crev;
 	uint16_t size;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c
index 3e0b267c74a8..569073e3a5a1 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.c
@@ -159,7 +159,6 @@ static int rv_construct_boot_state(struct pp_hwmgr *hwmgr)
 
 static int rv_set_clock_limit(struct pp_hwmgr *hwmgr, const void *input)
 {
-	struct rv_hwmgr *rv_data = (struct rv_hwmgr *)(hwmgr->backend);
 	struct PP_Clocks clocks = {0};
 	struct pp_display_clock_request clock_req;
 
@@ -170,39 +169,6 @@ static int rv_set_clock_limit(struct pp_hwmgr *hwmgr, const void *input)
 	PP_ASSERT_WITH_CODE(!rv_display_clock_voltage_request(hwmgr, &clock_req),
 				"Attempt to set DCF Clock Failed!", return -EINVAL);
 
-	if (((hwmgr->uvd_arbiter.vclk_soft_min / 100) != rv_data->vclk_soft_min) ||
-	    ((hwmgr->uvd_arbiter.dclk_soft_min / 100) != rv_data->dclk_soft_min)) {
-		rv_data->vclk_soft_min = hwmgr->uvd_arbiter.vclk_soft_min / 100;
-		rv_data->dclk_soft_min = hwmgr->uvd_arbiter.dclk_soft_min / 100;
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-			PPSMC_MSG_SetSoftMinVcn,
-			(rv_data->vclk_soft_min << 16) | rv_data->vclk_soft_min);
-	}
-
-	if((hwmgr->gfx_arbiter.sclk_hard_min != 0) &&
-		((hwmgr->gfx_arbiter.sclk_hard_min / 100) != rv_data->soc_actual_hard_min_freq)) {
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-					PPSMC_MSG_SetHardMinSocclkByFreq,
-					hwmgr->gfx_arbiter.sclk_hard_min / 100);
-		rv_read_arg_from_smc(hwmgr, &rv_data->soc_actual_hard_min_freq);
-	}
-
-	if ((hwmgr->gfx_arbiter.gfxclk != 0) &&
-		(rv_data->gfx_actual_soft_min_freq != (hwmgr->gfx_arbiter.gfxclk))) {
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-					PPSMC_MSG_SetMinVideoGfxclkFreq,
-					hwmgr->gfx_arbiter.gfxclk / 100);
-		rv_read_arg_from_smc(hwmgr, &rv_data->gfx_actual_soft_min_freq);
-	}
-
-	if ((hwmgr->gfx_arbiter.fclk != 0) &&
-		(rv_data->fabric_actual_soft_min_freq != (hwmgr->gfx_arbiter.fclk / 100))) {
-		smum_send_msg_to_smc_with_parameter(hwmgr,
-					PPSMC_MSG_SetMinVideoFclkFreq,
-					hwmgr->gfx_arbiter.fclk / 100);
-		rv_read_arg_from_smc(hwmgr, &rv_data->fabric_actual_soft_min_freq);
-	}
-
 	return 0;
 }
 
@@ -518,17 +484,161 @@ static int rv_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 static int rv_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 				enum amd_dpm_forced_level level)
 {
+	if (hwmgr->smu_version < 0x1E3700) {
+		pr_info("smu firmware version too old, can not set dpm level\n");
+		return 0;
+	}
+
+	switch (level) {
+	case AMD_DPM_FORCED_LEVEL_HIGH:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinGfxClk,
+						RAVEN_UMD_PSTATE_PEAK_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinFclkByFreq,
+						RAVEN_UMD_PSTATE_PEAK_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinSocclkByFreq,
+						RAVEN_UMD_PSTATE_PEAK_SOCCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinVcn,
+						RAVEN_UMD_PSTATE_VCE);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxGfxClk,
+						RAVEN_UMD_PSTATE_PEAK_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxFclkByFreq,
+						RAVEN_UMD_PSTATE_PEAK_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxSocclkByFreq,
+						RAVEN_UMD_PSTATE_PEAK_SOCCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxVcn,
+						RAVEN_UMD_PSTATE_VCE);
+		break;
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinGfxClk,
+						RAVEN_UMD_PSTATE_MIN_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxGfxClk,
+						RAVEN_UMD_PSTATE_MIN_GFXCLK);
+		break;
+	case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinFclkByFreq,
+						RAVEN_UMD_PSTATE_MIN_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxFclkByFreq,
+						RAVEN_UMD_PSTATE_MIN_FCLK);
+		break;
+	case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinGfxClk,
+						RAVEN_UMD_PSTATE_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinFclkByFreq,
+						RAVEN_UMD_PSTATE_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinSocclkByFreq,
+						RAVEN_UMD_PSTATE_SOCCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinVcn,
+						RAVEN_UMD_PSTATE_VCE);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxGfxClk,
+						RAVEN_UMD_PSTATE_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxFclkByFreq,
+						RAVEN_UMD_PSTATE_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxSocclkByFreq,
+						RAVEN_UMD_PSTATE_SOCCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxVcn,
+						RAVEN_UMD_PSTATE_VCE);
+		break;
+	case AMD_DPM_FORCED_LEVEL_AUTO:
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinGfxClk,
+						RAVEN_UMD_PSTATE_MIN_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinFclkByFreq,
+						RAVEN_UMD_PSTATE_MIN_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinSocclkByFreq,
+						RAVEN_UMD_PSTATE_MIN_SOCCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinVcn,
+						RAVEN_UMD_PSTATE_MIN_VCE);
+
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxGfxClk,
+						RAVEN_UMD_PSTATE_PEAK_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxFclkByFreq,
+						RAVEN_UMD_PSTATE_PEAK_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxSocclkByFreq,
+						RAVEN_UMD_PSTATE_PEAK_SOCCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxVcn,
+						RAVEN_UMD_PSTATE_VCE);
+		break;
+	case AMD_DPM_FORCED_LEVEL_LOW:
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinGfxClk,
+						RAVEN_UMD_PSTATE_MIN_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxGfxClk,
+						RAVEN_UMD_PSTATE_MIN_GFXCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetHardMinFclkByFreq,
+						RAVEN_UMD_PSTATE_MIN_FCLK);
+		smum_send_msg_to_smc_with_parameter(hwmgr,
+						PPSMC_MSG_SetSoftMaxFclkByFreq,
+						RAVEN_UMD_PSTATE_MIN_FCLK);
+		break;
+	case AMD_DPM_FORCED_LEVEL_MANUAL:
+	case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT:
+	default:
+		break;
+	}
 	return 0;
 }
 
 static uint32_t rv_dpm_get_mclk(struct pp_hwmgr *hwmgr, bool low)
 {
-	return 0;
+	struct rv_hwmgr *data;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	data = (struct rv_hwmgr *)(hwmgr->backend);
+
+	if (low)
+		return data->clock_vol_info.vdd_dep_on_fclk->entries[0].clk;
+	else
+		return data->clock_vol_info.vdd_dep_on_fclk->entries[
+			data->clock_vol_info.vdd_dep_on_fclk->count - 1].clk;
 }
 
 static uint32_t rv_dpm_get_sclk(struct pp_hwmgr *hwmgr, bool low)
 {
-	return 0;
+	struct rv_hwmgr *data;
+
+	if (hwmgr == NULL)
+		return -EINVAL;
+
+	data = (struct rv_hwmgr *)(hwmgr->backend);
+
+	if (low)
+		return data->gfx_min_freq_limit;
+	else
+		return data->gfx_max_freq_limit;
 }
 
 static int rv_dpm_patch_boot_state(struct pp_hwmgr *hwmgr,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h
index 9dc503055394..c3bc311dc59f 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_hwmgr.h
@@ -304,4 +304,19 @@ struct pp_hwmgr;
 
 int rv_init_function_pointers(struct pp_hwmgr *hwmgr);
 
+/* UMD PState Raven Msg Parameters in MHz */
+#define RAVEN_UMD_PSTATE_GFXCLK                 700
+#define RAVEN_UMD_PSTATE_SOCCLK                 626
+#define RAVEN_UMD_PSTATE_FCLK                   933
+#define RAVEN_UMD_PSTATE_VCE                    0x03C00320
+
+#define RAVEN_UMD_PSTATE_PEAK_GFXCLK            1100
+#define RAVEN_UMD_PSTATE_PEAK_SOCCLK            757
+#define RAVEN_UMD_PSTATE_PEAK_FCLK              1200
+
+#define RAVEN_UMD_PSTATE_MIN_GFXCLK             200
+#define RAVEN_UMD_PSTATE_MIN_FCLK               400
+#define RAVEN_UMD_PSTATE_MIN_SOCCLK             200
+#define RAVEN_UMD_PSTATE_MIN_VCE                0x0190012C
+
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_inc.h b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_inc.h
index 9a0149370d26..ae59a3fdea8a 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/rv_inc.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/rv_inc.h
@@ -25,17 +25,17 @@
 #define RAVEN_INC_H
 
 
-#include "asic_reg/raven1/MP/mp_10_0_default.h"
-#include "asic_reg/raven1/MP/mp_10_0_offset.h"
-#include "asic_reg/raven1/MP/mp_10_0_sh_mask.h"
+#include "asic_reg/mp/mp_10_0_default.h"
+#include "asic_reg/mp/mp_10_0_offset.h"
+#include "asic_reg/mp/mp_10_0_sh_mask.h"
 
-#include "asic_reg/raven1/NBIO/nbio_7_0_default.h"
-#include "asic_reg/raven1/NBIO/nbio_7_0_offset.h"
-#include "asic_reg/raven1/NBIO/nbio_7_0_sh_mask.h"
+#include "asic_reg/nbio/nbio_7_0_default.h"
+#include "asic_reg/nbio/nbio_7_0_offset.h"
+#include "asic_reg/nbio/nbio_7_0_sh_mask.h"
 
-#include "asic_reg/raven1/THM/thm_10_0_default.h"
-#include "asic_reg/raven1/THM/thm_10_0_offset.h"
-#include "asic_reg/raven1/THM/thm_10_0_sh_mask.h"
+#include "asic_reg/thm/thm_10_0_default.h"
+#include "asic_reg/thm/thm_10_0_offset.h"
+#include "asic_reg/thm/thm_10_0_sh_mask.h"
 
 
 #define ixDDI_PHY_GEN_STATUS                       0x3FCE8
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index e33ec7fc5d09..08e8a793714f 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -2722,9 +2722,6 @@ static int smu7_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		}
 	}
 
-	smu7_ps->vce_clks.evclk = hwmgr->vce_arbiter.evclk;
-	smu7_ps->vce_clks.ecclk = hwmgr->vce_arbiter.ecclk;
-
 	cgs_get_active_displays_info(hwmgr->device, &info);
 
 	minimum_clocks.engineClock = hwmgr->display_config.min_core_set_clock;
@@ -2754,47 +2751,18 @@ static int smu7_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		minimum_clocks.memoryClock = stable_pstate_mclk;
 	}
 
-	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
-		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
-
-	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
-		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
-
-	smu7_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
-
-	if (0 != hwmgr->gfx_arbiter.sclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.engineClock),
-				"Overdrive sclk exceeds limit",
-				hwmgr->gfx_arbiter.sclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.engineClock);
-
-		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
-			smu7_ps->performance_levels[1].engine_clock =
-					hwmgr->gfx_arbiter.sclk_over_drive;
-	}
-
-	if (0 != hwmgr->gfx_arbiter.mclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.memoryClock),
-				"Overdrive mclk exceeds limit",
-				hwmgr->gfx_arbiter.mclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.memoryClock);
-
-		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
-			smu7_ps->performance_levels[1].memory_clock =
-					hwmgr->gfx_arbiter.mclk_over_drive;
-	}
-
 	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
 				    hwmgr->platform_descriptor.platformCaps,
 				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
 
 
-	disable_mclk_switching = ((1 < info.display_count) ||
-				  disable_mclk_switching_for_frame_lock ||
-				  smu7_vblank_too_short(hwmgr, mode_info.vblank_time_us) ||
-				  (mode_info.refresh_rate > 120));
+	if (info.display_count == 0)
+		disable_mclk_switching = false;
+	else
+		disable_mclk_switching = ((1 < info.display_count) ||
+					  disable_mclk_switching_for_frame_lock ||
+					  smu7_vblank_too_short(hwmgr, mode_info.vblank_time_us) ||
+					  (mode_info.refresh_rate > 120));
 
 	sclk = smu7_ps->performance_levels[0].engine_clock;
 	mclk = smu7_ps->performance_levels[0].memory_clock;
@@ -4339,9 +4307,9 @@ static int smu7_print_clock_levels(struct pp_hwmgr *hwmgr,
 
 		for (i = 0; i < pcie_table->count; i++)
 			size += sprintf(buf + size, "%d: %s %s\n", i,
-					(pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x8" :
-					(pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" :
-					(pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "",
+					(pcie_table->dpm_levels[i].value == 0) ? "2.5GT/s, x8" :
+					(pcie_table->dpm_levels[i].value == 1) ? "5.0GT/s, x16" :
+					(pcie_table->dpm_levels[i].value == 2) ? "8.0GT/s, x16" : "",
 					(i == now) ? "*" : "");
 		break;
 	default:
@@ -4569,13 +4537,6 @@ static int smu7_set_power_profile_state(struct pp_hwmgr *hwmgr,
 	int tmp_result, result = 0;
 	uint32_t sclk_mask = 0, mclk_mask = 0;
 
-	if (hwmgr->chip_id == CHIP_FIJI) {
-		if (request->type == AMD_PP_GFX_PROFILE)
-			smu7_enable_power_containment(hwmgr);
-		else if (request->type == AMD_PP_COMPUTE_PROFILE)
-			smu7_disable_power_containment(hwmgr);
-	}
-
 	if (hwmgr->dpm_level != AMD_DPM_FORCED_LEVEL_AUTO)
 		return -EINVAL;
 
@@ -4686,6 +4647,25 @@ static int smu7_notify_cac_buffer_info(struct pp_hwmgr *hwmgr,
 	return 0;
 }
 
+static int smu7_get_max_high_clocks(struct pp_hwmgr *hwmgr,
+					struct amd_pp_simple_clock_info *clocks)
+{
+	struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend);
+	struct smu7_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+	struct smu7_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+
+	if (clocks == NULL)
+		return -EINVAL;
+
+	clocks->memory_max_clock = mclk_table->count > 1 ?
+				mclk_table->dpm_levels[mclk_table->count-1].value :
+				mclk_table->dpm_levels[0].value;
+	clocks->engine_max_clock = sclk_table->count > 1 ?
+				sclk_table->dpm_levels[sclk_table->count-1].value :
+				sclk_table->dpm_levels[0].value;
+	return 0;
+}
+
 static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
 	.backend_init = &smu7_hwmgr_backend_init,
 	.backend_fini = &smu7_hwmgr_backend_fini,
@@ -4738,6 +4718,7 @@ static const struct pp_hwmgr_func smu7_hwmgr_funcs = {
 	.disable_smc_firmware_ctf = smu7_thermal_disable_alert,
 	.start_thermal_controller = smu7_start_thermal_controller,
 	.notify_cac_buffer_info = smu7_notify_cac_buffer_info,
+	.get_max_high_clocks = smu7_get_max_high_clocks,
 };
 
 uint8_t smu7_get_sleep_divider_id_from_clock(uint32_t clock,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
index f8d838c2c8ee..5f9c3efb532f 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
@@ -426,9 +426,9 @@ static void vega10_init_dpm_defaults(struct pp_hwmgr *hwmgr)
 		data->smu_features[GNLD_VR0HOT].supported = true;
 
 	smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetSmuVersion);
-	vega10_read_arg_from_smc(hwmgr, &(data->smu_version));
+	vega10_read_arg_from_smc(hwmgr, &(hwmgr->smu_version));
 		/* ACG firmware has major version 5 */
-	if ((data->smu_version & 0xff000000) == 0x5000000)
+	if ((hwmgr->smu_version & 0xff000000) == 0x5000000)
 		data->smu_features[GNLD_ACG].supported = true;
 
 	if (data->registry_data.didt_support)
@@ -546,8 +546,7 @@ static void vega10_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr,
 	}
 
 	if (*voltage > ATOM_VIRTUAL_VOLTAGE_ID0)
-		pr_info("Voltage value looks like a Leakage ID \
-				but it's not patched\n");
+		pr_info("Voltage value looks like a Leakage ID but it's not patched\n");
 }
 
 /**
@@ -701,18 +700,14 @@ static int vega10_set_private_data_based_on_pptable(struct pp_hwmgr *hwmgr)
 			table_info->vdd_dep_on_mclk;
 
 	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table,
-		"VDD dependency on SCLK table is missing. \
-		This table is mandatory", return -EINVAL);
+		"VDD dependency on SCLK table is missing. This table is mandatory", return -EINVAL);
 	PP_ASSERT_WITH_CODE(allowed_sclk_vdd_table->count >= 1,
-		"VDD dependency on SCLK table is empty. \
-		This table is mandatory", return -EINVAL);
+		"VDD dependency on SCLK table is empty. This table is mandatory", return -EINVAL);
 
 	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table,
-		"VDD dependency on MCLK table is missing. \
-		This table is mandatory", return -EINVAL);
+		"VDD dependency on MCLK table is missing.  This table is mandatory", return -EINVAL);
 	PP_ASSERT_WITH_CODE(allowed_mclk_vdd_table->count >= 1,
-		"VDD dependency on MCLK table is empty. \
-		This table is mandatory", return -EINVAL);
+		"VDD dependency on MCLK table is empty.  This table is mandatory", return -EINVAL);
 
 	table_info->max_clock_voltage_on_ac.sclk =
 		allowed_sclk_vdd_table->entries[allowed_sclk_vdd_table->count - 1].clk;
@@ -2884,8 +2879,8 @@ static int vega10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 			"DPM is already running right , skipping re-enablement!",
 			return 0);
 
-	if ((data->smu_version == 0x001c2c00) ||
-			(data->smu_version == 0x001c2d00)) {
+	if ((hwmgr->smu_version == 0x001c2c00) ||
+			(hwmgr->smu_version == 0x001c2d00)) {
 		tmp_result = smum_send_msg_to_smc_with_parameter(hwmgr,
 				PPSMC_MSG_UpdatePkgPwrPidAlpha, 1);
 		PP_ASSERT_WITH_CODE(!tmp_result,
@@ -3129,9 +3124,6 @@ static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		}
 	}
 
-	vega10_ps->vce_clks.evclk = hwmgr->vce_arbiter.evclk;
-	vega10_ps->vce_clks.ecclk = hwmgr->vce_arbiter.ecclk;
-
 	cgs_get_active_displays_info(hwmgr->device, &info);
 
 	/* result = PHM_CheckVBlankTime(hwmgr, &vblankTooShort);*/
@@ -3170,48 +3162,19 @@ static int vega10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 		minimum_clocks.memoryClock = stable_pstate_mclk;
 	}
 
-	if (minimum_clocks.engineClock < hwmgr->gfx_arbiter.sclk)
-		minimum_clocks.engineClock = hwmgr->gfx_arbiter.sclk;
-
-	if (minimum_clocks.memoryClock < hwmgr->gfx_arbiter.mclk)
-		minimum_clocks.memoryClock = hwmgr->gfx_arbiter.mclk;
-
-	vega10_ps->sclk_threshold = hwmgr->gfx_arbiter.sclk_threshold;
-
-	if (hwmgr->gfx_arbiter.sclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.sclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.engineClock),
-				"Overdrive sclk exceeds limit",
-				hwmgr->gfx_arbiter.sclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.engineClock);
-
-		if (hwmgr->gfx_arbiter.sclk_over_drive >= hwmgr->gfx_arbiter.sclk)
-			vega10_ps->performance_levels[1].gfx_clock =
-					hwmgr->gfx_arbiter.sclk_over_drive;
-	}
-
-	if (hwmgr->gfx_arbiter.mclk_over_drive) {
-		PP_ASSERT_WITH_CODE((hwmgr->gfx_arbiter.mclk_over_drive <=
-				hwmgr->platform_descriptor.overdriveLimit.memoryClock),
-				"Overdrive mclk exceeds limit",
-				hwmgr->gfx_arbiter.mclk_over_drive =
-						hwmgr->platform_descriptor.overdriveLimit.memoryClock);
-
-		if (hwmgr->gfx_arbiter.mclk_over_drive >= hwmgr->gfx_arbiter.mclk)
-			vega10_ps->performance_levels[1].mem_clock =
-					hwmgr->gfx_arbiter.mclk_over_drive;
-	}
-
 	disable_mclk_switching_for_frame_lock = phm_cap_enabled(
 				    hwmgr->platform_descriptor.platformCaps,
 				    PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
 	disable_mclk_switching_for_vr = PP_CAP(PHM_PlatformCaps_DisableMclkSwitchForVR);
 	force_mclk_high = PP_CAP(PHM_PlatformCaps_ForceMclkHigh);
 
-	disable_mclk_switching = (info.display_count > 1) ||
-				    disable_mclk_switching_for_frame_lock ||
-				    disable_mclk_switching_for_vr ||
-				    force_mclk_high;
+	if (info.display_count == 0)
+		disable_mclk_switching = false;
+	else
+		disable_mclk_switching = (info.display_count > 1) ||
+			disable_mclk_switching_for_frame_lock ||
+			disable_mclk_switching_for_vr ||
+			force_mclk_high;
 
 	sclk = vega10_ps->performance_levels[0].gfx_clock;
 	mclk = vega10_ps->performance_levels[0].mem_clock;
@@ -3416,8 +3379,7 @@ static int vega10_populate_and_upload_sclk_mclk_dpm_levels(
 					DPMTABLE_OD_UPDATE_SCLK)) {
 			result = vega10_populate_all_graphic_levels(hwmgr);
 			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate SCLK during \
-					PopulateNewDPMClocksStates Function!",
+					"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
 					return result);
 		}
 
@@ -3426,8 +3388,7 @@ static int vega10_populate_and_upload_sclk_mclk_dpm_levels(
 					DPMTABLE_OD_UPDATE_MCLK)){
 			result = vega10_populate_all_memory_levels(hwmgr);
 			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate MCLK during \
-					PopulateNewDPMClocksStates Function!",
+					"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
 					return result);
 		}
 	} else {
@@ -3544,8 +3505,7 @@ static int vega10_populate_and_upload_sclk_mclk_dpm_levels(
 			data->apply_optimized_settings) {
 			result = vega10_populate_all_graphic_levels(hwmgr);
 			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate SCLK during \
-					PopulateNewDPMClocksStates Function!",
+					"Failed to populate SCLK during PopulateNewDPMClocksStates Function!",
 					return result);
 		}
 
@@ -3553,8 +3513,7 @@ static int vega10_populate_and_upload_sclk_mclk_dpm_levels(
 				(DPMTABLE_OD_UPDATE_MCLK + DPMTABLE_UPDATE_MCLK)) {
 			result = vega10_populate_all_memory_levels(hwmgr);
 			PP_ASSERT_WITH_CODE(!result,
-					"Failed to populate MCLK during \
-					PopulateNewDPMClocksStates Function!",
+					"Failed to populate MCLK during PopulateNewDPMClocksStates Function!",
 					return result);
 		}
 	}
@@ -3828,10 +3787,7 @@ static int vega10_update_sclk_threshold(struct pp_hwmgr *hwmgr)
 	uint32_t low_sclk_interrupt_threshold = 0;
 
 	if (PP_CAP(PHM_PlatformCaps_SclkThrottleLowNotification) &&
-	    (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
+		(data->low_sclk_interrupt_threshold != 0)) {
 		low_sclk_interrupt_threshold =
 				data->low_sclk_interrupt_threshold;
 
@@ -4654,9 +4610,9 @@ static int vega10_print_clock_levels(struct pp_hwmgr *hwmgr,
 
 		for (i = 0; i < pcie_table->count; i++)
 			size += sprintf(buf + size, "%d: %s %s\n", i,
-					(pcie_table->pcie_gen[i] == 0) ? "2.5GB, x1" :
-					(pcie_table->pcie_gen[i] == 1) ? "5.0GB, x16" :
-					(pcie_table->pcie_gen[i] == 2) ? "8.0GB, x16" : "",
+					(pcie_table->pcie_gen[i] == 0) ? "2.5GT/s, x1" :
+					(pcie_table->pcie_gen[i] == 1) ? "5.0GT/s, x16" :
+					(pcie_table->pcie_gen[i] == 2) ? "8.0GT/s, x16" : "",
 					(i == now) ? "*" : "");
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
index 8f7358cc3327..e8507ff8dbb3 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.h
@@ -387,7 +387,6 @@ struct vega10_hwmgr {
 	struct vega10_smc_state_table  smc_state_table;
 
 	uint32_t                       config_telemetry;
-	uint32_t                       smu_version;
 	uint32_t                       acg_loop_state;
 	uint32_t                       mem_channels;
 };
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_inc.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_inc.h
index 8c55eaa3c32b..faf7ac044348 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_inc.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_inc.h
@@ -24,21 +24,20 @@
 #ifndef VEGA10_INC_H
 #define VEGA10_INC_H
 
-#include "asic_reg/vega10/THM/thm_9_0_default.h"
-#include "asic_reg/vega10/THM/thm_9_0_offset.h"
-#include "asic_reg/vega10/THM/thm_9_0_sh_mask.h"
+#include "asic_reg/thm/thm_9_0_default.h"
+#include "asic_reg/thm/thm_9_0_offset.h"
+#include "asic_reg/thm/thm_9_0_sh_mask.h"
 
-#include "asic_reg/vega10/MP/mp_9_0_default.h"
-#include "asic_reg/vega10/MP/mp_9_0_offset.h"
-#include "asic_reg/vega10/MP/mp_9_0_sh_mask.h"
+#include "asic_reg/mp/mp_9_0_offset.h"
+#include "asic_reg/mp/mp_9_0_sh_mask.h"
 
-#include "asic_reg/vega10/GC/gc_9_0_default.h"
-#include "asic_reg/vega10/GC/gc_9_0_offset.h"
-#include "asic_reg/vega10/GC/gc_9_0_sh_mask.h"
+#include "asic_reg/gc/gc_9_0_default.h"
+#include "asic_reg/gc/gc_9_0_offset.h"
+#include "asic_reg/gc/gc_9_0_sh_mask.h"
 
-#include "asic_reg/vega10/NBIO/nbio_6_1_default.h"
-#include "asic_reg/vega10/NBIO/nbio_6_1_offset.h"
-#include "asic_reg/vega10/NBIO/nbio_6_1_sh_mask.h"
+#include "asic_reg/nbio/nbio_6_1_default.h"
+#include "asic_reg/nbio/nbio_6_1_offset.h"
+#include "asic_reg/nbio/nbio_6_1_sh_mask.h"
 
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
index 95932cc88460..152e70db4a81 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
@@ -29,280 +29,7 @@
 #include "amd_shared.h"
 #include "cgs_common.h"
 #include "dm_pp_interface.h"
-
-extern const struct amd_ip_funcs pp_ip_funcs;
-extern const struct amd_pm_funcs pp_dpm_funcs;
-
-enum amd_pp_sensors {
-	AMDGPU_PP_SENSOR_GFX_SCLK = 0,
-	AMDGPU_PP_SENSOR_VDDNB,
-	AMDGPU_PP_SENSOR_VDDGFX,
-	AMDGPU_PP_SENSOR_UVD_VCLK,
-	AMDGPU_PP_SENSOR_UVD_DCLK,
-	AMDGPU_PP_SENSOR_VCE_ECCLK,
-	AMDGPU_PP_SENSOR_GPU_LOAD,
-	AMDGPU_PP_SENSOR_GFX_MCLK,
-	AMDGPU_PP_SENSOR_GPU_TEMP,
-	AMDGPU_PP_SENSOR_VCE_POWER,
-	AMDGPU_PP_SENSOR_UVD_POWER,
-	AMDGPU_PP_SENSOR_GPU_POWER,
-};
-
-enum amd_pp_task {
-	AMD_PP_TASK_DISPLAY_CONFIG_CHANGE,
-	AMD_PP_TASK_ENABLE_USER_STATE,
-	AMD_PP_TASK_READJUST_POWER_STATE,
-	AMD_PP_TASK_COMPLETE_INIT,
-	AMD_PP_TASK_MAX
-};
-
-struct amd_pp_init {
-	struct cgs_device *device;
-	uint32_t chip_family;
-	uint32_t chip_id;
-	bool pm_en;
-	uint32_t feature_mask;
-};
-
-enum amd_pp_display_config_type{
-	AMD_PP_DisplayConfigType_None = 0,
-	AMD_PP_DisplayConfigType_DP54 ,
-	AMD_PP_DisplayConfigType_DP432 ,
-	AMD_PP_DisplayConfigType_DP324 ,
-	AMD_PP_DisplayConfigType_DP27,
-	AMD_PP_DisplayConfigType_DP243,
-	AMD_PP_DisplayConfigType_DP216,
-	AMD_PP_DisplayConfigType_DP162,
-	AMD_PP_DisplayConfigType_HDMI6G ,
-	AMD_PP_DisplayConfigType_HDMI297 ,
-	AMD_PP_DisplayConfigType_HDMI162,
-	AMD_PP_DisplayConfigType_LVDS,
-	AMD_PP_DisplayConfigType_DVI,
-	AMD_PP_DisplayConfigType_WIRELESS,
-	AMD_PP_DisplayConfigType_VGA
-};
-
-struct single_display_configuration
-{
-	uint32_t controller_index;
-	uint32_t controller_id;
-	uint32_t signal_type;
-	uint32_t display_state;
-	/* phy id for the primary internal transmitter */
-	uint8_t primary_transmitter_phyi_d;
-	/* bitmap with the active lanes */
-	uint8_t primary_transmitter_active_lanemap;
-	/* phy id for the secondary internal transmitter (for dual-link dvi) */
-	uint8_t secondary_transmitter_phy_id;
-	/* bitmap with the active lanes */
-	uint8_t secondary_transmitter_active_lanemap;
-	/* misc phy settings for SMU. */
-	uint32_t config_flags;
-	uint32_t display_type;
-	uint32_t view_resolution_cx;
-	uint32_t view_resolution_cy;
-	enum amd_pp_display_config_type displayconfigtype;
-	uint32_t vertical_refresh; /* for active display */
-};
-
-#define MAX_NUM_DISPLAY 32
-
-struct amd_pp_display_configuration {
-	bool nb_pstate_switch_disable;/* controls NB PState switch */
-	bool cpu_cc6_disable; /* controls CPU CState switch ( on or off) */
-	bool cpu_pstate_disable;
-	uint32_t cpu_pstate_separation_time;
-
-	uint32_t num_display;  /* total number of display*/
-	uint32_t num_path_including_non_display;
-	uint32_t crossfire_display_index;
-	uint32_t min_mem_set_clock;
-	uint32_t min_core_set_clock;
-	/* unit 10KHz x bit*/
-	uint32_t min_bus_bandwidth;
-	/* minimum required stutter sclk, in 10khz uint32_t ulMinCoreSetClk;*/
-	uint32_t min_core_set_clock_in_sr;
-
-	struct single_display_configuration displays[MAX_NUM_DISPLAY];
-
-	uint32_t vrefresh; /* for active display*/
-
-	uint32_t min_vblank_time; /* for active display*/
-	bool multi_monitor_in_sync;
-	/* Controller Index of primary display - used in MCLK SMC switching hang
-	 * SW Workaround*/
-	uint32_t crtc_index;
-	/* htotal*1000/pixelclk - used in MCLK SMC switching hang SW Workaround*/
-	uint32_t line_time_in_us;
-	bool invalid_vblank_time;
-
-	uint32_t display_clk;
-	/*
-	 * for given display configuration if multimonitormnsync == false then
-	 * Memory clock DPMS with this latency or below is allowed, DPMS with
-	 * higher latency not allowed.
-	 */
-	uint32_t dce_tolerable_mclk_in_active_latency;
-	uint32_t min_dcef_set_clk;
-	uint32_t min_dcef_deep_sleep_set_clk;
-};
-
-struct amd_pp_simple_clock_info {
-	uint32_t	engine_max_clock;
-	uint32_t	memory_max_clock;
-	uint32_t	level;
-};
-
-enum PP_DAL_POWERLEVEL {
-	PP_DAL_POWERLEVEL_INVALID = 0,
-	PP_DAL_POWERLEVEL_ULTRALOW,
-	PP_DAL_POWERLEVEL_LOW,
-	PP_DAL_POWERLEVEL_NOMINAL,
-	PP_DAL_POWERLEVEL_PERFORMANCE,
-
-	PP_DAL_POWERLEVEL_0 = PP_DAL_POWERLEVEL_ULTRALOW,
-	PP_DAL_POWERLEVEL_1 = PP_DAL_POWERLEVEL_LOW,
-	PP_DAL_POWERLEVEL_2 = PP_DAL_POWERLEVEL_NOMINAL,
-	PP_DAL_POWERLEVEL_3 = PP_DAL_POWERLEVEL_PERFORMANCE,
-	PP_DAL_POWERLEVEL_4 = PP_DAL_POWERLEVEL_3+1,
-	PP_DAL_POWERLEVEL_5 = PP_DAL_POWERLEVEL_4+1,
-	PP_DAL_POWERLEVEL_6 = PP_DAL_POWERLEVEL_5+1,
-	PP_DAL_POWERLEVEL_7 = PP_DAL_POWERLEVEL_6+1,
-};
-
-struct amd_pp_clock_info {
-	uint32_t min_engine_clock;
-	uint32_t max_engine_clock;
-	uint32_t min_memory_clock;
-	uint32_t max_memory_clock;
-	uint32_t min_bus_bandwidth;
-	uint32_t max_bus_bandwidth;
-	uint32_t max_engine_clock_in_sr;
-	uint32_t min_engine_clock_in_sr;
-	enum PP_DAL_POWERLEVEL max_clocks_state;
-};
-
-enum amd_pp_clock_type {
-	amd_pp_disp_clock = 1,
-	amd_pp_sys_clock,
-	amd_pp_mem_clock,
-	amd_pp_dcef_clock,
-	amd_pp_soc_clock,
-	amd_pp_pixel_clock,
-	amd_pp_phy_clock,
-	amd_pp_dcf_clock,
-	amd_pp_dpp_clock,
-	amd_pp_f_clock = amd_pp_dcef_clock,
-};
-
-#define MAX_NUM_CLOCKS 16
-
-struct amd_pp_clocks {
-	uint32_t count;
-	uint32_t clock[MAX_NUM_CLOCKS];
-	uint32_t latency[MAX_NUM_CLOCKS];
-};
-
-
-enum {
-	PP_GROUP_UNKNOWN = 0,
-	PP_GROUP_GFX = 1,
-	PP_GROUP_SYS,
-	PP_GROUP_MAX
-};
-
-struct pp_states_info {
-	uint32_t nums;
-	uint32_t states[16];
-};
-
-struct pp_gpu_power {
-	uint32_t vddc_power;
-	uint32_t vddci_power;
-	uint32_t max_gpu_power;
-	uint32_t average_gpu_power;
-};
-
-struct pp_display_clock_request {
-	enum amd_pp_clock_type clock_type;
-	uint32_t clock_freq_in_khz;
-};
-
-#define PP_GROUP_MASK        0xF0000000
-#define PP_GROUP_SHIFT       28
-
-#define PP_BLOCK_MASK        0x0FFFFF00
-#define PP_BLOCK_SHIFT       8
-
-#define PP_BLOCK_GFX_CG         0x01
-#define PP_BLOCK_GFX_MG         0x02
-#define PP_BLOCK_GFX_3D         0x04
-#define PP_BLOCK_GFX_RLC        0x08
-#define PP_BLOCK_GFX_CP         0x10
-#define PP_BLOCK_SYS_BIF        0x01
-#define PP_BLOCK_SYS_MC         0x02
-#define PP_BLOCK_SYS_ROM        0x04
-#define PP_BLOCK_SYS_DRM        0x08
-#define PP_BLOCK_SYS_HDP        0x10
-#define PP_BLOCK_SYS_SDMA       0x20
-
-#define PP_STATE_MASK           0x0000000F
-#define PP_STATE_SHIFT          0
-#define PP_STATE_SUPPORT_MASK   0x000000F0
-#define PP_STATE_SUPPORT_SHIFT  0
-
-#define PP_STATE_CG             0x01
-#define PP_STATE_LS             0x02
-#define PP_STATE_DS             0x04
-#define PP_STATE_SD             0x08
-#define PP_STATE_SUPPORT_CG     0x10
-#define PP_STATE_SUPPORT_LS     0x20
-#define PP_STATE_SUPPORT_DS     0x40
-#define PP_STATE_SUPPORT_SD     0x80
-
-#define PP_CG_MSG_ID(group, block, support, state) (group << PP_GROUP_SHIFT |\
-								block << PP_BLOCK_SHIFT |\
-								support << PP_STATE_SUPPORT_SHIFT |\
-								state << PP_STATE_SHIFT)
-
-struct amd_powerplay {
-	struct cgs_device *cgs_device;
-	void *pp_handle;
-	const struct amd_ip_funcs *ip_funcs;
-	const struct amd_pm_funcs *pp_funcs;
-};
-
-int amd_powerplay_reset(void *handle);
-
-int amd_powerplay_display_configuration_change(void *handle,
-		const struct amd_pp_display_configuration *input);
-
-int amd_powerplay_get_display_power_level(void *handle,
-		struct amd_pp_simple_clock_info *output);
-
-int amd_powerplay_get_current_clocks(void *handle,
-		struct amd_pp_clock_info *output);
-
-int amd_powerplay_get_clock_by_type(void *handle,
-		enum amd_pp_clock_type type,
-		struct amd_pp_clocks *clocks);
-
-int amd_powerplay_get_clock_by_type_with_latency(void *handle,
-		enum amd_pp_clock_type type,
-		struct pp_clock_levels_with_latency *clocks);
-
-int amd_powerplay_get_clock_by_type_with_voltage(void *handle,
-		enum amd_pp_clock_type type,
-		struct pp_clock_levels_with_voltage *clocks);
-
-int amd_powerplay_set_watermarks_for_clocks_ranges(void *handle,
-		struct pp_wm_sets_with_clock_ranges_soc15 *wm_with_clock_ranges);
-
-int amd_powerplay_display_clock_voltage_request(void *handle,
-		struct pp_display_clock_request *clock);
-
-int amd_powerplay_get_display_mode_validation_clocks(void *handle,
-		struct amd_pp_simple_clock_info *output);
+#include "kgd_pp_interface.h"
 
 
 #endif /* _AMD_POWERPLAY_H_ */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
index 57a0467b7267..5716b937a6ad 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -437,5 +437,6 @@ extern int phm_display_clock_voltage_request(struct pp_hwmgr *hwmgr,
 
 extern int phm_get_max_high_clocks(struct pp_hwmgr *hwmgr, struct amd_pp_simple_clock_info *clocks);
 extern int phm_disable_smc_firmware_ctf(struct pp_hwmgr *hwmgr);
+extern int phm_reset_power_profile_state(struct pp_hwmgr *hwmgr);
 #endif /* _HARDWARE_MANAGER_H_ */
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index 004a40e88bde..565fe0832f41 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -105,36 +105,6 @@ struct phm_set_power_state_input {
 	const struct pp_hw_power_state *pnew_state;
 };
 
-struct phm_acp_arbiter {
-	uint32_t acpclk;
-};
-
-struct phm_uvd_arbiter {
-	uint32_t vclk;
-	uint32_t dclk;
-	uint32_t vclk_ceiling;
-	uint32_t dclk_ceiling;
-	uint32_t vclk_soft_min;
-	uint32_t dclk_soft_min;
-};
-
-struct phm_vce_arbiter {
-	uint32_t   evclk;
-	uint32_t   ecclk;
-};
-
-struct phm_gfx_arbiter {
-	uint32_t sclk;
-	uint32_t sclk_hard_min;
-	uint32_t mclk;
-	uint32_t sclk_over_drive;
-	uint32_t mclk_over_drive;
-	uint32_t sclk_threshold;
-	uint32_t num_cus;
-	uint32_t gfxclk;
-	uint32_t fclk;
-};
-
 struct phm_clock_array {
 	uint32_t count;
 	uint32_t values[1];
@@ -722,6 +692,7 @@ enum PP_TABLE_VERSION {
 struct pp_hwmgr {
 	uint32_t chip_family;
 	uint32_t chip_id;
+	uint32_t smu_version;
 
 	uint32_t pp_table_version;
 	void *device;
@@ -737,10 +708,6 @@ struct pp_hwmgr {
 	enum amd_dpm_forced_level dpm_level;
 	enum amd_dpm_forced_level saved_dpm_level;
 	enum amd_dpm_forced_level request_dpm_level;
-	struct phm_gfx_arbiter gfx_arbiter;
-	struct phm_acp_arbiter acp_arbiter;
-	struct phm_uvd_arbiter uvd_arbiter;
-	struct phm_vce_arbiter vce_arbiter;
 	uint32_t usec_timeout;
 	void *pptable;
 	struct phm_platform_descriptor platform_descriptor;
diff --git a/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h b/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
index a511611ec7e0..b7ab69e4c254 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/pp_soc15.h
@@ -23,7 +23,7 @@
 #ifndef PP_SOC15_H
 #define PP_SOC15_H
 
-#include "vega10/soc15ip.h"
+#include "soc15ip.h"
 
 inline static uint32_t soc15_get_register_offset(
 		uint32_t hw_id,
diff --git a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
index 2b3497135bbd..f15f4df9d0a9 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
@@ -75,7 +75,12 @@
 #define PPSMC_MSG_GetMinGfxclkFrequency         0x2C
 #define PPSMC_MSG_GetMaxGfxclkFrequency         0x2D
 #define PPSMC_MSG_SoftReset                     0x2E
-#define PPSMC_Message_Count                     0x2F
+#define PPSMC_MSG_SetSoftMaxGfxClk              0x30
+#define PPSMC_MSG_SetHardMinGfxClk              0x31
+#define PPSMC_MSG_SetSoftMaxSocclkByFreq        0x32
+#define PPSMC_MSG_SetSoftMaxFclkByFreq          0x33
+#define PPSMC_MSG_SetSoftMaxVcn                 0x34
+#define PPSMC_Message_Count                     0x35
 
 
 typedef uint16_t PPSMC_Result;
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
index 4d672cd15785..0b4a55660de4 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/ci_smumgr.c
@@ -1732,8 +1732,7 @@ static int ci_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
 	if (0 != result) {
 		smu_data->smc_state_table.GraphicsBootLevel = 0;
-		pr_err("VBIOS did not find boot engine clock value \
-			in dependency table. Using Graphics DPM level 0!");
+		pr_err("VBIOS did not find boot engine clock value in dependency table. Using Graphics DPM level 0!\n");
 		result = 0;
 	}
 
@@ -1743,8 +1742,7 @@ static int ci_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
 	if (0 != result) {
 		smu_data->smc_state_table.MemoryBootLevel = 0;
-		pr_err("VBIOS did not find boot engine clock value \
-			in dependency table. Using Memory DPM level 0!");
+		pr_err("VBIOS did not find boot engine clock value in dependency table. Using Memory DPM level 0!\n");
 		result = 0;
 	}
 
@@ -2220,10 +2218,7 @@ static int ci_update_sclk_threshold(struct pp_hwmgr *hwmgr)
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
+		&& (data->low_sclk_interrupt_threshold != 0)) {
 		low_sclk_interrupt_threshold =
 				data->low_sclk_interrupt_threshold;
 
@@ -2321,6 +2316,7 @@ static int ci_load_smc_ucode(struct pp_hwmgr *hwmgr)
 	cgs_get_firmware_info(hwmgr->device, CGS_UCODE_ID_SMU, &info);
 
 	hwmgr->is_kicker = info.is_kicker;
+	hwmgr->smu_version = info.version;
 	byte_count = info.image_size;
 	src = (uint8_t *)info.kptr;
 	start_addr = info.ucode_start_address;
@@ -2602,9 +2598,9 @@ static int ci_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
 			}
 			j++;
+
 			PP_ASSERT_WITH_CODE((j < SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE),
 				"Invalid VramInfo table.", return -EINVAL);
-
 			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS);
 			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
 			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
@@ -2617,10 +2613,10 @@ static int ci_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
 			}
 			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -EINVAL);
 
-			if (!data->is_memory_gddr5 && j < SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) {
+			if (!data->is_memory_gddr5) {
+				PP_ASSERT_WITH_CODE((j < SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+					"Invalid VramInfo table.", return -EINVAL);
 				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
 				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
 				for (k = 0; k < table->num_entries; k++) {
@@ -2628,8 +2624,6 @@ static int ci_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
 				}
 				j++;
-				PP_ASSERT_WITH_CODE((j <= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-					"Invalid VramInfo table.", return -EINVAL);
 			}
 
 			break;
@@ -2644,8 +2638,6 @@ static int ci_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
 			}
 			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -EINVAL);
 			break;
 
 		default:
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
index 78ab0556e48f..4d3aff381bca 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/cz_smumgr.c
@@ -709,6 +709,19 @@ static int cz_start_smu(struct pp_hwmgr *hwmgr)
 {
 	int ret = 0;
 	uint32_t fw_to_check = 0;
+	struct cgs_firmware_info info = {0};
+	uint32_t index = SMN_MP1_SRAM_START_ADDR +
+			 SMU8_FIRMWARE_HEADER_LOCATION +
+			 offsetof(struct SMU8_Firmware_Header, Version);
+
+
+	if (hwmgr == NULL || hwmgr->device == NULL)
+		return -EINVAL;
+
+	cgs_write_register(hwmgr->device, mmMP0PUB_IND_INDEX, index);
+	hwmgr->smu_version = cgs_read_register(hwmgr->device, mmMP0PUB_IND_DATA);
+	info.version = hwmgr->smu_version >> 8;
+	cgs_get_firmware_info(hwmgr->device, CGS_UCODE_ID_SMU, &info);
 
 	fw_to_check = UCODE_ID_RLC_G_MASK |
 			UCODE_ID_SDMA0_MASK |
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
index f572beff197f..085d81c8b332 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/fiji_smumgr.c
@@ -2385,10 +2385,7 @@ static int fiji_update_sclk_threshold(struct pp_hwmgr *hwmgr)
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
+		&& (data->low_sclk_interrupt_threshold != 0)) {
 		low_sclk_interrupt_threshold =
 				data->low_sclk_interrupt_threshold;
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
index 34128822b8fb..125312691f75 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/iceland_smumgr.c
@@ -204,7 +204,7 @@ static int iceland_smu_upload_firmware_image(struct pp_hwmgr *hwmgr)
 		pr_err("[ powerplay ] SMC address is beyond the SMC RAM area\n");
 		return -EINVAL;
 	}
-
+	hwmgr->smu_version = info.version;
 	/* wait for smc boot up */
 	PHM_WAIT_INDIRECT_FIELD_UNEQUAL(hwmgr, SMC_IND,
 					 RCU_UC_EVENTS, boot_seq_done, 0);
@@ -911,8 +911,7 @@ static int iceland_populate_single_graphic_level(struct pp_hwmgr *hwmgr,
 		hwmgr->dyn_state.vddc_dependency_on_sclk, engine_clock,
 		&graphic_level->MinVddc);
 	PP_ASSERT_WITH_CODE((0 == result),
-		"can not find VDDC voltage value for VDDC	\
-		engine clock dependency table", return result);
+		"can not find VDDC voltage value for VDDC engine clock dependency table", return result);
 
 	/* SCLK frequency in units of 10KHz*/
 	graphic_level->SclkFrequency = engine_clock;
@@ -1678,8 +1677,7 @@ static int iceland_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
 	if (0 != result) {
 		smu_data->smc_state_table.GraphicsBootLevel = 0;
-		pr_err("VBIOS did not find boot engine clock value \
-			in dependency table. Using Graphics DPM level 0!");
+		pr_err("VBIOS did not find boot engine clock value in dependency table. Using Graphics DPM level 0!\n");
 		result = 0;
 	}
 
@@ -1689,8 +1687,7 @@ static int iceland_populate_smc_boot_level(struct pp_hwmgr *hwmgr,
 
 	if (0 != result) {
 		smu_data->smc_state_table.MemoryBootLevel = 0;
-		pr_err("VBIOS did not find boot engine clock value \
-			in dependency table. Using Memory DPM level 0!");
+		pr_err("VBIOS did not find boot engine clock value in dependency table. Using Memory DPM level 0!\n");
 		result = 0;
 	}
 
@@ -2205,10 +2202,7 @@ static int iceland_update_sclk_threshold(struct pp_hwmgr *hwmgr)
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
+		&& (data->low_sclk_interrupt_threshold != 0)) {
 		low_sclk_interrupt_threshold =
 				data->low_sclk_interrupt_threshold;
 
@@ -2552,9 +2546,9 @@ static int iceland_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
 			}
 			j++;
+
 			PP_ASSERT_WITH_CODE((j < SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
 				"Invalid VramInfo table.", return -EINVAL);
-
 			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS);
 			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
 			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
@@ -2568,10 +2562,10 @@ static int iceland_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 				}
 			}
 			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -EINVAL);
 
-			if (!data->is_memory_gddr5 && j < SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE) {
+			if (!data->is_memory_gddr5) {
+				PP_ASSERT_WITH_CODE((j < SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+					"Invalid VramInfo table.", return -EINVAL);
 				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
 				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
 				for (k = 0; k < table->num_entries; k++) {
@@ -2579,8 +2573,6 @@ static int iceland_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
 				}
 				j++;
-				PP_ASSERT_WITH_CODE((j <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-					"Invalid VramInfo table.", return -EINVAL);
 			}
 
 			break;
@@ -2595,8 +2587,6 @@ static int iceland_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
 			}
 			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU71_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -EINVAL);
 			break;
 
 		default:
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
index bd6be7793ca7..cdb47657b567 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/polaris10_smumgr.c
@@ -2369,10 +2369,7 @@ static int polaris10_update_sclk_threshold(struct pp_hwmgr *hwmgr)
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
+		&& (data->low_sclk_interrupt_threshold != 0)) {
 		low_sclk_interrupt_threshold =
 				data->low_sclk_interrupt_threshold;
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.c
index b98ade676d12..2d662b44af54 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.c
@@ -305,6 +305,14 @@ static int rv_smu_fini(struct pp_hwmgr *hwmgr)
 
 static int rv_start_smu(struct pp_hwmgr *hwmgr)
 {
+	struct cgs_firmware_info info = {0};
+
+	smum_send_msg_to_smc(hwmgr, PPSMC_MSG_GetSmuVersion);
+	rv_read_arg_from_smc(hwmgr, &hwmgr->smu_version);
+	info.version = hwmgr->smu_version >> 8;
+
+	cgs_get_firmware_info(hwmgr->device, CGS_UCODE_ID_SMU, &info);
+
 	if (rv_verify_smc_interface(hwmgr))
 		return -EINVAL;
 	if (rv_smc_enable_sdma(hwmgr))
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.h
index 58888400f1b8..caebdbebdcd8 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/rv_smumgr.h
@@ -40,7 +40,7 @@ struct smu_table_entry {
 	uint32_t table_addr_high;
 	uint32_t table_addr_low;
 	uint8_t *table;
-	uint32_t handle;
+	unsigned long handle;
 };
 
 struct smu_table_array {
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
index 7f5359a97ef2..311ff3718618 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu7_smumgr.c
@@ -535,7 +535,7 @@ int smu7_upload_smu_firmware_image(struct pp_hwmgr *hwmgr)
 			smu7_convert_fw_type_to_cgs(UCODE_ID_SMU_SK), &info);
 
 	hwmgr->is_kicker = info.is_kicker;
-
+	hwmgr->smu_version = info.version;
 	result = smu7_upload_smc_firmware_data(hwmgr, info.image_size, (uint32_t *)info.kptr, SMU7_SMC_SIZE);
 
 	return result;
@@ -648,6 +648,12 @@ int smu7_init(struct pp_hwmgr *hwmgr)
 
 int smu7_smu_fini(struct pp_hwmgr *hwmgr)
 {
+	struct smu7_smumgr *smu_data = (struct smu7_smumgr *)(hwmgr->smu_backend);
+
+	smu_free_memory(hwmgr->device, (void *) smu_data->header_buffer.handle);
+	if (!cgs_is_virtualization_enabled(hwmgr->device))
+		smu_free_memory(hwmgr->device, (void *) smu_data->smu_buffer.handle);
+
 	kfree(hwmgr->smu_backend);
 	hwmgr->smu_backend = NULL;
 	cgs_rel_firmware(hwmgr->device, CGS_UCODE_ID_SMU);
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
index 0a8e48bff219..79e5c05571bc 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
@@ -2654,10 +2654,7 @@ static int tonga_update_sclk_threshold(struct pp_hwmgr *hwmgr)
 
 	if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
 			PHM_PlatformCaps_SclkThrottleLowNotification)
-		&& (hwmgr->gfx_arbiter.sclk_threshold !=
-				data->low_sclk_interrupt_threshold)) {
-		data->low_sclk_interrupt_threshold =
-				hwmgr->gfx_arbiter.sclk_threshold;
+		&& (data->low_sclk_interrupt_threshold != 0)) {
 		low_sclk_interrupt_threshold =
 				data->low_sclk_interrupt_threshold;
 
@@ -3106,9 +3103,9 @@ static int tonga_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
 			}
 			j++;
+
 			PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
 				"Invalid VramInfo table.", return -EINVAL);
-
 			temp_reg = cgs_read_register(hwmgr->device, mmMC_PMG_CMD_MRS);
 			table->mc_reg_address[j].s1 = mmMC_PMG_CMD_MRS;
 			table->mc_reg_address[j].s0 = mmMC_SEQ_PMG_CMD_MRS_LP;
@@ -3121,18 +3118,16 @@ static int tonga_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
 			}
 			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -EINVAL);
 
 			if (!data->is_memory_gddr5) {
+				PP_ASSERT_WITH_CODE((j < SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
+					"Invalid VramInfo table.", return -EINVAL);
 				table->mc_reg_address[j].s1 = mmMC_PMG_AUTO_CMD;
 				table->mc_reg_address[j].s0 = mmMC_PMG_AUTO_CMD;
 				for (k = 0; k < table->num_entries; k++)
 					table->mc_reg_table_entry[k].mc_data[j] =
 						(table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
 				j++;
-				PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-					"Invalid VramInfo table.", return -EINVAL);
 			}
 
 			break;
@@ -3147,8 +3142,6 @@ static int tonga_set_mc_special_registers(struct pp_hwmgr *hwmgr,
 					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
 			}
 			j++;
-			PP_ASSERT_WITH_CODE((j <= SMU72_DISCRETE_MC_REGISTER_ARRAY_SIZE),
-				"Invalid VramInfo table.", return -EINVAL);
 			break;
 
 		default:
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
index 2f979fb86824..f6f39d01d227 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega10_smumgr.c
@@ -381,10 +381,8 @@ static int vega10_verify_smc_interface(struct pp_hwmgr *hwmgr)
 		(rev_id == 0xc1) ||
 		(rev_id == 0xc3)))) {
 		if (smc_driver_if_version != SMU9_DRIVER_IF_VERSION) {
-			pr_err("Your firmware(0x%x) doesn't match \
-				SMU9_DRIVER_IF_VERSION(0x%x). \
-				Please update your firmware!\n",
-				smc_driver_if_version, SMU9_DRIVER_IF_VERSION);
+			pr_err("Your firmware(0x%x) doesn't match SMU9_DRIVER_IF_VERSION(0x%x). Please update your firmware!\n",
+			       smc_driver_if_version, SMU9_DRIVER_IF_VERSION);
 			return -EINVAL;
 		}
 	}
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
deleted file mode 100644
index 52c8e5447624..000000000000
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Copyright 2015 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef _GPU_SCHEDULER_H_
-#define _GPU_SCHEDULER_H_
-
-#include <linux/kfifo.h>
-#include <linux/dma-fence.h>
-
-struct amd_gpu_scheduler;
-struct amd_sched_rq;
-
-/**
- * A scheduler entity is a wrapper around a job queue or a group
- * of other entities. Entities take turns emitting jobs from their
- * job queues to corresponding hardware ring based on scheduling
- * policy.
-*/
-struct amd_sched_entity {
-	struct list_head		list;
-	struct amd_sched_rq		*rq;
-	spinlock_t			rq_lock;
-	struct amd_gpu_scheduler	*sched;
-
-	spinlock_t			queue_lock;
-	struct kfifo                    job_queue;
-
-	atomic_t			fence_seq;
-	uint64_t                        fence_context;
-
-	struct dma_fence		*dependency;
-	struct dma_fence_cb		cb;
-};
-
-/**
- * Run queue is a set of entities scheduling command submissions for
- * one specific ring. It implements the scheduling policy that selects
- * the next entity to emit commands from.
-*/
-struct amd_sched_rq {
-	spinlock_t		lock;
-	struct list_head	entities;
-	struct amd_sched_entity	*current_entity;
-};
-
-struct amd_sched_fence {
-	struct dma_fence                scheduled;
-	struct dma_fence                finished;
-	struct dma_fence_cb             cb;
-	struct dma_fence                *parent;
-	struct amd_gpu_scheduler	*sched;
-	spinlock_t			lock;
-	void                            *owner;
-};
-
-struct amd_sched_job {
-	struct amd_gpu_scheduler        *sched;
-	struct amd_sched_entity         *s_entity;
-	struct amd_sched_fence          *s_fence;
-	struct dma_fence_cb		finish_cb;
-	struct work_struct		finish_work;
-	struct list_head		node;
-	struct delayed_work		work_tdr;
-	uint64_t			id;
-	atomic_t karma;
-};
-
-extern const struct dma_fence_ops amd_sched_fence_ops_scheduled;
-extern const struct dma_fence_ops amd_sched_fence_ops_finished;
-static inline struct amd_sched_fence *to_amd_sched_fence(struct dma_fence *f)
-{
-	if (f->ops == &amd_sched_fence_ops_scheduled)
-		return container_of(f, struct amd_sched_fence, scheduled);
-
-	if (f->ops == &amd_sched_fence_ops_finished)
-		return container_of(f, struct amd_sched_fence, finished);
-
-	return NULL;
-}
-
-static inline bool amd_sched_invalidate_job(struct amd_sched_job *s_job, int threshold)
-{
-	return (s_job && atomic_inc_return(&s_job->karma) > threshold);
-}
-
-/**
- * Define the backend operations called by the scheduler,
- * these functions should be implemented in driver side
-*/
-struct amd_sched_backend_ops {
-	struct dma_fence *(*dependency)(struct amd_sched_job *sched_job);
-	struct dma_fence *(*run_job)(struct amd_sched_job *sched_job);
-	void (*timedout_job)(struct amd_sched_job *sched_job);
-	void (*free_job)(struct amd_sched_job *sched_job);
-};
-
-enum amd_sched_priority {
-	AMD_SCHED_PRIORITY_MIN,
-	AMD_SCHED_PRIORITY_LOW = AMD_SCHED_PRIORITY_MIN,
-	AMD_SCHED_PRIORITY_NORMAL,
-	AMD_SCHED_PRIORITY_HIGH_SW,
-	AMD_SCHED_PRIORITY_HIGH_HW,
-	AMD_SCHED_PRIORITY_KERNEL,
-	AMD_SCHED_PRIORITY_MAX,
-	AMD_SCHED_PRIORITY_INVALID = -1,
-	AMD_SCHED_PRIORITY_UNSET = -2
-};
-
-/**
- * One scheduler is implemented for each hardware ring
-*/
-struct amd_gpu_scheduler {
-	const struct amd_sched_backend_ops	*ops;
-	uint32_t			hw_submission_limit;
-	long				timeout;
-	const char			*name;
-	struct amd_sched_rq		sched_rq[AMD_SCHED_PRIORITY_MAX];
-	wait_queue_head_t		wake_up_worker;
-	wait_queue_head_t		job_scheduled;
-	atomic_t			hw_rq_count;
-	atomic64_t			job_id_count;
-	struct task_struct		*thread;
-	struct list_head	ring_mirror_list;
-	spinlock_t			job_list_lock;
-};
-
-int amd_sched_init(struct amd_gpu_scheduler *sched,
-		   const struct amd_sched_backend_ops *ops,
-		   uint32_t hw_submission, long timeout, const char *name);
-void amd_sched_fini(struct amd_gpu_scheduler *sched);
-
-int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
-			  struct amd_sched_entity *entity,
-			  struct amd_sched_rq *rq,
-			  uint32_t jobs);
-void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
-			   struct amd_sched_entity *entity);
-void amd_sched_entity_push_job(struct amd_sched_job *sched_job);
-void amd_sched_entity_set_rq(struct amd_sched_entity *entity,
-			     struct amd_sched_rq *rq);
-
-int amd_sched_fence_slab_init(void);
-void amd_sched_fence_slab_fini(void);
-
-struct amd_sched_fence *amd_sched_fence_create(
-	struct amd_sched_entity *s_entity, void *owner);
-void amd_sched_fence_scheduled(struct amd_sched_fence *fence);
-void amd_sched_fence_finished(struct amd_sched_fence *fence);
-int amd_sched_job_init(struct amd_sched_job *job,
-		       struct amd_gpu_scheduler *sched,
-		       struct amd_sched_entity *entity,
-		       void *owner);
-void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched);
-void amd_sched_job_recovery(struct amd_gpu_scheduler *sched);
-bool amd_sched_dependency_optimized(struct dma_fence* fence,
-				    struct amd_sched_entity *entity);
-void amd_sched_job_kickout(struct amd_sched_job *s_job);
-
-static inline enum amd_sched_priority
-amd_sched_get_job_priority(struct amd_sched_job *job)
-{
-	return (job->s_entity->rq - job->sched->sched_rq);
-}
-
-#endif
diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c
index 074fd4ea7ece..f067de4e1e82 100644
--- a/drivers/gpu/drm/arc/arcpgu_drv.c
+++ b/drivers/gpu/drm/arc/arcpgu_drv.c
@@ -155,7 +155,6 @@ static int arcpgu_show_pxlclock(struct seq_file *m, void *arg)
 
 static struct drm_info_list arcpgu_debugfs_list[] = {
 	{ "clocks", arcpgu_show_pxlclock, 0 },
-	{ "fb", drm_fb_cma_debugfs_show, 0 },
 };
 
 static int arcpgu_debugfs_init(struct drm_minor *minor)
@@ -180,6 +179,7 @@ static struct drm_driver arcpgu_drm_driver = {
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
+	.gem_print_info = drm_gem_cma_print_info,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.gem_prime_export = drm_gem_prime_export,
 	.gem_prime_import = drm_gem_prime_import,
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index 5a5427bbd70e..630721f429f7 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -252,10 +252,10 @@ static int hdlcd_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
 
-	return drm_plane_helper_check_state(state, &clip,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    false, true);
+	return drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   false, true);
 }
 
 static void hdlcd_plane_atomic_update(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index 0afb53b1f4e9..feaa8bc3d7b7 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -231,7 +231,6 @@ static int hdlcd_show_pxlclock(struct seq_file *m, void *arg)
 static struct drm_info_list hdlcd_debugfs_list[] = {
 	{ "interrupt_count", hdlcd_show_underrun_count, 0 },
 	{ "clocks", hdlcd_show_pxlclock, 0 },
-	{ "fb", drm_fb_cma_debugfs_show, 0 },
 };
 
 static int hdlcd_debugfs_init(struct drm_minor *minor)
@@ -253,6 +252,7 @@ static struct drm_driver hdlcd_driver = {
 	.irq_postinstall = hdlcd_irq_postinstall,
 	.irq_uninstall = hdlcd_irq_uninstall,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
+	.gem_print_info = drm_gem_cma_print_info,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index 91f2b0191368..3d82712d8002 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -13,7 +13,6 @@
 #include <linux/module.h>
 #include <linux/clk.h>
 #include <linux/component.h>
-#include <linux/console.h>
 #include <linux/of_device.h>
 #include <linux/of_graph.h>
 #include <linux/of_reserved_mem.h>
@@ -24,9 +23,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/drm_of.h>
 
 #include "malidp_drv.h"
@@ -182,13 +183,6 @@ static int malidp_set_and_wait_config_valid(struct drm_device *drm)
 	return (ret > 0) ? 0 : -ETIMEDOUT;
 }
 
-static void malidp_output_poll_changed(struct drm_device *drm)
-{
-	struct malidp_drm *malidp = drm->dev_private;
-
-	drm_fbdev_cma_hotplug_event(malidp->fbdev);
-}
-
 static void malidp_atomic_commit_hw_done(struct drm_atomic_state *state)
 {
 	struct drm_pending_vblank_event *event;
@@ -251,7 +245,7 @@ static const struct drm_mode_config_helper_funcs malidp_mode_config_helpers = {
 
 static const struct drm_mode_config_funcs malidp_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
-	.output_poll_changed = malidp_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -316,19 +310,12 @@ static int malidp_irq_init(struct platform_device *pdev)
 	return 0;
 }
 
-static void malidp_lastclose(struct drm_device *drm)
-{
-	struct malidp_drm *malidp = drm->dev_private;
-
-	drm_fbdev_cma_restore_mode(malidp->fbdev);
-}
-
 DEFINE_DRM_GEM_CMA_FOPS(fops);
 
 static struct drm_driver malidp_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC |
 			   DRIVER_PRIME,
-	.lastclose = malidp_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
@@ -622,14 +609,9 @@ static int malidp_bind(struct device *dev)
 
 	drm_mode_config_reset(drm);
 
-	malidp->fbdev = drm_fbdev_cma_init(drm, 32,
-					   drm->mode_config.num_connector);
-
-	if (IS_ERR(malidp->fbdev)) {
-		ret = PTR_ERR(malidp->fbdev);
-		malidp->fbdev = NULL;
+	ret = drm_fb_cma_fbdev_init(drm, 32, 0);
+	if (ret)
 		goto fbdev_fail;
-	}
 
 	drm_kms_helper_poll_init(drm);
 
@@ -640,10 +622,7 @@ static int malidp_bind(struct device *dev)
 	return 0;
 
 register_fail:
-	if (malidp->fbdev) {
-		drm_fbdev_cma_fini(malidp->fbdev);
-		malidp->fbdev = NULL;
-	}
+	drm_fb_cma_fbdev_fini(drm);
 	drm_kms_helper_poll_fini(drm);
 fbdev_fail:
 	pm_runtime_get_sync(dev);
@@ -680,10 +659,7 @@ static void malidp_unbind(struct device *dev)
 	struct malidp_drm *malidp = drm->dev_private;
 
 	drm_dev_unregister(drm);
-	if (malidp->fbdev) {
-		drm_fbdev_cma_fini(malidp->fbdev);
-		malidp->fbdev = NULL;
-	}
+	drm_fb_cma_fbdev_fini(drm);
 	drm_kms_helper_poll_fini(drm);
 	pm_runtime_get_sync(dev);
 	malidp_se_irq_fini(drm);
@@ -745,34 +721,15 @@ static int malidp_platform_remove(struct platform_device *pdev)
 static int __maybe_unused malidp_pm_suspend(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
-	struct malidp_drm *malidp = drm->dev_private;
-
-	drm_kms_helper_poll_disable(drm);
-	console_lock();
-	drm_fbdev_cma_set_suspend(malidp->fbdev, 1);
-	console_unlock();
-	malidp->pm_state = drm_atomic_helper_suspend(drm);
-	if (IS_ERR(malidp->pm_state)) {
-		console_lock();
-		drm_fbdev_cma_set_suspend(malidp->fbdev, 0);
-		console_unlock();
-		drm_kms_helper_poll_enable(drm);
-		return PTR_ERR(malidp->pm_state);
-	}
 
-	return 0;
+	return drm_mode_config_helper_suspend(drm);
 }
 
 static int __maybe_unused malidp_pm_resume(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
-	struct malidp_drm *malidp = drm->dev_private;
 
-	drm_atomic_helper_resume(drm, malidp->pm_state);
-	console_lock();
-	drm_fbdev_cma_set_suspend(malidp->fbdev, 0);
-	console_unlock();
-	drm_kms_helper_poll_enable(drm);
+	drm_mode_config_helper_resume(drm);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/arm/malidp_drv.h b/drivers/gpu/drm/arm/malidp_drv.h
index 2e2033140efc..e0d12c9fc6b8 100644
--- a/drivers/gpu/drm/arm/malidp_drv.h
+++ b/drivers/gpu/drm/arm/malidp_drv.h
@@ -20,11 +20,9 @@
 
 struct malidp_drm {
 	struct malidp_hw_device *dev;
-	struct drm_fbdev_cma *fbdev;
 	struct drm_crtc crtc;
 	wait_queue_head_t wq;
 	atomic_t config_valid;
-	struct drm_atomic_state *pm_state;
 	u32 core_id;
 };
 
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index e7419797bbd1..33c5ef96ced0 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -150,7 +150,8 @@ static int malidp_se_check_scaling(struct malidp_plane *mp,
 
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
-	ret = drm_plane_helper_check_state(state, &clip, 0, INT_MAX, true, true);
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  0, INT_MAX, true, true);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index a0f4d2a2a481..e2adfbef7d6b 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -13,6 +13,7 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_plane_helper.h>
+#include <drm/drm_atomic_helper.h>
 #include "armada_crtc.h"
 #include "armada_drm.h"
 #include "armada_fb.h"
@@ -20,13 +21,6 @@
 #include "armada_hw.h"
 #include "armada_trace.h"
 
-struct armada_frame_work {
-	struct armada_plane_work work;
-	struct drm_pending_vblank_event *event;
-	struct armada_regs regs[4];
-	struct drm_framebuffer *old_fb;
-};
-
 enum csc_mode {
 	CSC_AUTO = 0,
 	CSC_YUV_CCIR601 = 1,
@@ -216,6 +210,38 @@ static unsigned armada_drm_crtc_calc_fb(struct drm_framebuffer *fb,
 	return i;
 }
 
+static void armada_drm_plane_work_call(struct armada_crtc *dcrtc,
+	struct armada_plane_work *work,
+	void (*fn)(struct armada_crtc *, struct armada_plane_work *))
+{
+	struct armada_plane *dplane = drm_to_armada_plane(work->plane);
+	struct drm_pending_vblank_event *event;
+	struct drm_framebuffer *fb;
+
+	if (fn)
+		fn(dcrtc, work);
+	drm_crtc_vblank_put(&dcrtc->crtc);
+
+	event = work->event;
+	fb = work->old_fb;
+	if (event || fb) {
+		struct drm_device *dev = dcrtc->crtc.dev;
+		unsigned long flags;
+
+		spin_lock_irqsave(&dev->event_lock, flags);
+		if (event)
+			drm_crtc_send_vblank_event(&dcrtc->crtc, event);
+		if (fb)
+			__armada_drm_queue_unref_work(dev, fb);
+		spin_unlock_irqrestore(&dev->event_lock, flags);
+	}
+
+	if (work->need_kfree)
+		kfree(work);
+
+	wake_up(&dplane->frame_wait);
+}
+
 static void armada_drm_plane_work_run(struct armada_crtc *dcrtc,
 	struct drm_plane *plane)
 {
@@ -223,24 +249,19 @@ static void armada_drm_plane_work_run(struct armada_crtc *dcrtc,
 	struct armada_plane_work *work = xchg(&dplane->work, NULL);
 
 	/* Handle any pending frame work. */
-	if (work) {
-		work->fn(dcrtc, dplane, work);
-		drm_crtc_vblank_put(&dcrtc->crtc);
-	}
-
-	wake_up(&dplane->frame_wait);
+	if (work)
+		armada_drm_plane_work_call(dcrtc, work, work->fn);
 }
 
 int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
-	struct armada_plane *plane, struct armada_plane_work *work)
+	struct armada_plane_work *work)
 {
+	struct armada_plane *plane = drm_to_armada_plane(work->plane);
 	int ret;
 
 	ret = drm_crtc_vblank_get(&dcrtc->crtc);
-	if (ret) {
-		DRM_ERROR("failed to acquire vblank counter\n");
+	if (ret)
 		return ret;
-	}
 
 	ret = cmpxchg(&plane->work, NULL, work) ? -EBUSY : 0;
 	if (ret)
@@ -254,51 +275,60 @@ int armada_drm_plane_work_wait(struct armada_plane *plane, long timeout)
 	return wait_event_timeout(plane->frame_wait, !plane->work, timeout);
 }
 
-struct armada_plane_work *armada_drm_plane_work_cancel(
-	struct armada_crtc *dcrtc, struct armada_plane *plane)
+void armada_drm_plane_work_cancel(struct armada_crtc *dcrtc,
+	struct armada_plane *dplane)
 {
-	struct armada_plane_work *work = xchg(&plane->work, NULL);
+	struct armada_plane_work *work = xchg(&dplane->work, NULL);
 
 	if (work)
-		drm_crtc_vblank_put(&dcrtc->crtc);
-
-	return work;
+		armada_drm_plane_work_call(dcrtc, work, work->cancel);
 }
 
-static int armada_drm_crtc_queue_frame_work(struct armada_crtc *dcrtc,
-	struct armada_frame_work *work)
+static void armada_drm_crtc_complete_frame_work(struct armada_crtc *dcrtc,
+	struct armada_plane_work *work)
 {
-	struct armada_plane *plane = drm_to_armada_plane(dcrtc->crtc.primary);
+	unsigned long flags;
 
-	return armada_drm_plane_work_queue(dcrtc, plane, &work->work);
+	spin_lock_irqsave(&dcrtc->irq_lock, flags);
+	armada_drm_crtc_update_regs(dcrtc, work->regs);
+	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
 }
 
-static void armada_drm_crtc_complete_frame_work(struct armada_crtc *dcrtc,
-	struct armada_plane *plane, struct armada_plane_work *work)
+static void armada_drm_crtc_complete_disable_work(struct armada_crtc *dcrtc,
+	struct armada_plane_work *work)
 {
-	struct armada_frame_work *fwork = container_of(work, struct armada_frame_work, work);
-	struct drm_device *dev = dcrtc->crtc.dev;
 	unsigned long flags;
 
+	if (dcrtc->plane == work->plane)
+		dcrtc->plane = NULL;
+
 	spin_lock_irqsave(&dcrtc->irq_lock, flags);
-	armada_drm_crtc_update_regs(dcrtc, fwork->regs);
+	armada_drm_crtc_update_regs(dcrtc, work->regs);
 	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
+}
 
-	if (fwork->event) {
-		spin_lock_irqsave(&dev->event_lock, flags);
-		drm_crtc_send_vblank_event(&dcrtc->crtc, fwork->event);
-		spin_unlock_irqrestore(&dev->event_lock, flags);
-	}
+static struct armada_plane_work *
+armada_drm_crtc_alloc_plane_work(struct drm_plane *plane)
+{
+	struct armada_plane_work *work;
+	int i = 0;
+
+	work = kzalloc(sizeof(*work), GFP_KERNEL);
+	if (!work)
+		return NULL;
 
-	/* Finally, queue the process-half of the cleanup. */
-	__armada_drm_queue_unref_work(dcrtc->crtc.dev, fwork->old_fb);
-	kfree(fwork);
+	work->plane = plane;
+	work->fn = armada_drm_crtc_complete_frame_work;
+	work->need_kfree = true;
+	armada_reg_queue_end(work->regs, i);
+
+	return work;
 }
 
 static void armada_drm_crtc_finish_fb(struct armada_crtc *dcrtc,
 	struct drm_framebuffer *fb, bool force)
 {
-	struct armada_frame_work *work;
+	struct armada_plane_work *work;
 
 	if (!fb)
 		return;
@@ -309,15 +339,11 @@ static void armada_drm_crtc_finish_fb(struct armada_crtc *dcrtc,
 		return;
 	}
 
-	work = kmalloc(sizeof(*work), GFP_KERNEL);
+	work = armada_drm_crtc_alloc_plane_work(dcrtc->crtc.primary);
 	if (work) {
-		int i = 0;
-		work->work.fn = armada_drm_crtc_complete_frame_work;
-		work->event = NULL;
 		work->old_fb = fb;
-		armada_reg_queue_end(work->regs, i);
 
-		if (armada_drm_crtc_queue_frame_work(dcrtc, work) == 0)
+		if (armada_drm_plane_work_queue(dcrtc, work) == 0)
 			return;
 
 		kfree(work);
@@ -380,8 +406,11 @@ static void armada_drm_crtc_prepare(struct drm_crtc *crtc)
 	 * the new mode parameters.
 	 */
 	plane = dcrtc->plane;
-	if (plane)
+	if (plane) {
 		drm_plane_force_disable(plane);
+		WARN_ON(!armada_drm_plane_work_wait(drm_to_armada_plane(plane),
+						    HZ));
+	}
 }
 
 /* The mode_config.mutex will be held for this call */
@@ -447,11 +476,11 @@ static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
 	if (stat & VSYNC_IRQ)
 		drm_crtc_handle_vblank(&dcrtc->crtc);
 
-	spin_lock(&dcrtc->irq_lock);
 	ovl_plane = dcrtc->plane;
 	if (ovl_plane)
 		armada_drm_plane_work_run(dcrtc, ovl_plane);
 
+	spin_lock(&dcrtc->irq_lock);
 	if (stat & GRA_FRAME_IRQ && dcrtc->interlaced) {
 		int i = stat & GRA_FRAME_IRQ0 ? 0 : 1;
 		uint32_t val;
@@ -543,18 +572,14 @@ static uint32_t armada_drm_crtc_calculate_csc(struct armada_crtc *dcrtc)
 	return val;
 }
 
-static void armada_drm_primary_set(struct drm_crtc *crtc,
-	struct drm_plane *plane, int x, int y)
+static void armada_drm_gra_plane_regs(struct armada_regs *regs,
+	struct drm_framebuffer *fb, struct armada_plane_state *state,
+	int x, int y, bool interlaced)
 {
-	struct armada_plane_state *state = &drm_to_armada_plane(plane)->state;
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct armada_regs regs[8];
-	bool interlaced = dcrtc->interlaced;
-	unsigned i;
+	unsigned int i;
 	u32 ctrl0;
 
-	i = armada_drm_crtc_calc_fb(plane->fb, x, y, regs, interlaced);
-
+	i = armada_drm_crtc_calc_fb(fb, x, y, regs, interlaced);
 	armada_reg_queue_set(regs, i, state->dst_yx, LCD_SPU_GRA_OVSA_HPXL_VLN);
 	armada_reg_queue_set(regs, i, state->src_hw, LCD_SPU_GRA_HPXL_VLN);
 	armada_reg_queue_set(regs, i, state->dst_hw, LCD_SPU_GZM_HPXL_VLN);
@@ -566,9 +591,21 @@ static void armada_drm_primary_set(struct drm_crtc *crtc,
 	armada_reg_queue_mod(regs, i, ctrl0, CFG_GRAFORMAT |
 			     CFG_GRA_MOD(CFG_SWAPRB | CFG_SWAPUV |
 					 CFG_SWAPYU | CFG_YUV2RGB) |
-			     CFG_PALETTE_ENA | CFG_GRA_FTOGGLE,
+			     CFG_PALETTE_ENA | CFG_GRA_FTOGGLE |
+			     CFG_GRA_HSMOOTH | CFG_GRA_ENA,
 			     LCD_SPU_DMA_CTRL0);
 	armada_reg_queue_end(regs, i);
+}
+
+static void armada_drm_primary_set(struct drm_crtc *crtc,
+	struct drm_plane *plane, int x, int y)
+{
+	struct armada_plane_state *state = &drm_to_armada_plane(plane)->state;
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	struct armada_regs regs[8];
+	bool interlaced = dcrtc->interlaced;
+
+	armada_drm_gra_plane_regs(regs, plane->fb, state, x, y, interlaced);
 	armada_drm_crtc_update_regs(dcrtc, regs);
 }
 
@@ -588,7 +625,7 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 
 	interlaced = !!(adj->flags & DRM_MODE_FLAG_INTERLACE);
 
-	val = CFG_GRA_ENA | CFG_GRA_HSMOOTH;
+	val = CFG_GRA_ENA;
 	val |= CFG_GRA_FMT(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt);
 	val |= CFG_GRA_MOD(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->mod);
 
@@ -640,8 +677,6 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 	/* Now compute the divider for real */
 	dcrtc->variant->compute_clock(dcrtc, adj, &sclk);
 
-	/* Ensure graphic fifo is enabled */
-	armada_reg_queue_mod(regs, i, 0, CFG_PDWN64x66, LCD_SPU_SRAM_PARA1);
 	armada_reg_queue_set(regs, i, sclk, LCD_CFG_SCLK_DIV);
 
 	if (interlaced ^ dcrtc->interlaced) {
@@ -654,6 +689,9 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 
 	spin_lock_irqsave(&dcrtc->irq_lock, flags);
 
+	/* Ensure graphic fifo is enabled */
+	armada_reg_queue_mod(regs, i, 0, CFG_PDWN64x66, LCD_SPU_SRAM_PARA1);
+
 	/* Even interlaced/progressive frame */
 	dcrtc->v[1].spu_v_h_total = adj->crtc_vtotal << 16 |
 				    adj->crtc_htotal;
@@ -736,46 +774,13 @@ static int armada_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
 	return 0;
 }
 
-void armada_drm_crtc_plane_disable(struct armada_crtc *dcrtc,
-	struct drm_plane *plane)
-{
-	u32 sram_para1, dma_ctrl0_mask;
-
-	/*
-	 * Drop our reference on any framebuffer attached to this plane.
-	 * We don't need to NULL this out as drm_plane_force_disable(),
-	 * and __setplane_internal() will do so for an overlay plane, and
-	 * __drm_helper_disable_unused_functions() will do so for the
-	 * primary plane.
-	 */
-	if (plane->fb)
-		drm_framebuffer_put(plane->fb);
-
-	/* Power down most RAMs and FIFOs if this is the primary plane */
-	if (plane->type == DRM_PLANE_TYPE_PRIMARY) {
-		sram_para1 = CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
-			     CFG_PDWN32x32 | CFG_PDWN64x66;
-		dma_ctrl0_mask = CFG_GRA_ENA;
-	} else {
-		/* Power down the Y/U/V FIFOs */
-		sram_para1 = CFG_PDWN16x66 | CFG_PDWN32x66;
-		dma_ctrl0_mask = CFG_DMA_ENA;
-	}
-
-	spin_lock_irq(&dcrtc->irq_lock);
-	armada_updatel(0, dma_ctrl0_mask, dcrtc->base + LCD_SPU_DMA_CTRL0);
-	spin_unlock_irq(&dcrtc->irq_lock);
-
-	armada_updatel(sram_para1, 0, dcrtc->base + LCD_SPU_SRAM_PARA1);
-}
-
 /* The mode_config.mutex will be held for this call */
 static void armada_drm_crtc_disable(struct drm_crtc *crtc)
 {
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-
 	armada_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
-	armada_drm_crtc_plane_disable(dcrtc, crtc->primary);
+
+	/* Disable our primary plane when we disable the CRTC. */
+	crtc->primary->funcs->disable_plane(crtc->primary, NULL);
 }
 
 static const struct drm_crtc_helper_funcs armada_crtc_helper_funcs = {
@@ -885,9 +890,11 @@ static int armada_drm_crtc_cursor_update(struct armada_crtc *dcrtc, bool reload)
 		return 0;
 	}
 
+	spin_lock_irq(&dcrtc->irq_lock);
 	para1 = readl_relaxed(dcrtc->base + LCD_SPU_SRAM_PARA1);
 	armada_updatel(CFG_CSB_256x32, CFG_CSB_256x32 | CFG_PDWN256x32,
 		       dcrtc->base + LCD_SPU_SRAM_PARA1);
+	spin_unlock_irq(&dcrtc->irq_lock);
 
 	/*
 	 * Initialize the transparency if the SRAM was powered down.
@@ -1027,7 +1034,7 @@ static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
 	struct drm_modeset_acquire_ctx *ctx)
 {
 	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
-	struct armada_frame_work *work;
+	struct armada_plane_work *work;
 	unsigned i;
 	int ret;
 
@@ -1035,11 +1042,10 @@ static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
 	if (fb->format != crtc->primary->fb->format)
 		return -EINVAL;
 
-	work = kmalloc(sizeof(*work), GFP_KERNEL);
+	work = armada_drm_crtc_alloc_plane_work(dcrtc->crtc.primary);
 	if (!work)
 		return -ENOMEM;
 
-	work->work.fn = armada_drm_crtc_complete_frame_work;
 	work->event = event;
 	work->old_fb = dcrtc->crtc.primary->fb;
 
@@ -1053,7 +1059,7 @@ static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
 	 */
 	drm_framebuffer_get(fb);
 
-	ret = armada_drm_crtc_queue_frame_work(dcrtc, work);
+	ret = armada_drm_plane_work_queue(dcrtc, work);
 	if (ret) {
 		/* Undo our reference above */
 		drm_framebuffer_put(fb);
@@ -1133,14 +1139,195 @@ static const struct drm_crtc_funcs armada_crtc_funcs = {
 	.disable_vblank	= armada_drm_crtc_disable_vblank,
 };
 
+static void armada_drm_primary_update_state(struct drm_plane_state *state,
+	struct armada_regs *regs)
+{
+	struct armada_plane *dplane = drm_to_armada_plane(state->plane);
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(state->crtc);
+	struct armada_framebuffer *dfb = drm_fb_to_armada_fb(state->fb);
+	bool was_disabled;
+	unsigned int idx = 0;
+	u32 val;
+
+	val = CFG_GRA_FMT(dfb->fmt) | CFG_GRA_MOD(dfb->mod);
+	if (dfb->fmt > CFG_420)
+		val |= CFG_PALETTE_ENA;
+	if (state->visible)
+		val |= CFG_GRA_ENA;
+	if (drm_rect_width(&state->src) >> 16 != drm_rect_width(&state->dst))
+		val |= CFG_GRA_HSMOOTH;
+
+	was_disabled = !(dplane->state.ctrl0 & CFG_GRA_ENA);
+	if (was_disabled)
+		armada_reg_queue_mod(regs, idx,
+				     0, CFG_PDWN64x66, LCD_SPU_SRAM_PARA1);
+
+	dplane->state.ctrl0 = val;
+	dplane->state.src_hw = (drm_rect_height(&state->src) & 0xffff0000) |
+				drm_rect_width(&state->src) >> 16;
+	dplane->state.dst_hw = drm_rect_height(&state->dst) << 16 |
+			       drm_rect_width(&state->dst);
+	dplane->state.dst_yx = state->dst.y1 << 16 | state->dst.x1;
+
+	armada_drm_gra_plane_regs(regs + idx, &dfb->fb, &dplane->state,
+				  state->src.x1 >> 16, state->src.y1 >> 16,
+				  dcrtc->interlaced);
+
+	dplane->state.vsync_update = !was_disabled;
+	dplane->state.changed = true;
+}
+
+static int armada_drm_primary_update(struct drm_plane *plane,
+	struct drm_crtc *crtc, struct drm_framebuffer *fb,
+	int crtc_x, int crtc_y, unsigned int crtc_w, unsigned int crtc_h,
+	uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h,
+	struct drm_modeset_acquire_ctx *ctx)
+{
+	struct armada_plane *dplane = drm_to_armada_plane(plane);
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	struct armada_plane_work *work;
+	struct drm_plane_state state = {
+		.plane = plane,
+		.crtc = crtc,
+		.fb = fb,
+		.src_x = src_x,
+		.src_y = src_y,
+		.src_w = src_w,
+		.src_h = src_h,
+		.crtc_x = crtc_x,
+		.crtc_y = crtc_y,
+		.crtc_w = crtc_w,
+		.crtc_h = crtc_h,
+		.rotation = DRM_MODE_ROTATE_0,
+	};
+	const struct drm_rect clip = {
+		.x2 = crtc->mode.hdisplay,
+		.y2 = crtc->mode.vdisplay,
+	};
+	int ret;
+
+	ret = drm_atomic_helper_check_plane_state(&state, crtc->state, &clip, 0,
+						  INT_MAX, true, false);
+	if (ret)
+		return ret;
+
+	work = &dplane->works[dplane->next_work];
+	work->fn = armada_drm_crtc_complete_frame_work;
+
+	if (plane->fb != fb) {
+		/*
+		 * Take a reference on the new framebuffer - we want to
+		 * hold on to it while the hardware is displaying it.
+		 */
+		drm_framebuffer_reference(fb);
+
+		work->old_fb = plane->fb;
+	} else {
+		work->old_fb = NULL;
+	}
+
+	armada_drm_primary_update_state(&state, work->regs);
+
+	if (!dplane->state.changed)
+		return 0;
+
+	/* Wait for pending work to complete */
+	if (armada_drm_plane_work_wait(dplane, HZ / 10) == 0)
+		armada_drm_plane_work_cancel(dcrtc, dplane);
+
+	if (!dplane->state.vsync_update) {
+		work->fn(dcrtc, work);
+		if (work->old_fb)
+			drm_framebuffer_unreference(work->old_fb);
+		return 0;
+	}
+
+	/* Queue it for update on the next interrupt if we are enabled */
+	ret = armada_drm_plane_work_queue(dcrtc, work);
+	if (ret) {
+		work->fn(dcrtc, work);
+		if (work->old_fb)
+			drm_framebuffer_unreference(work->old_fb);
+	}
+
+	dplane->next_work = !dplane->next_work;
+
+	return 0;
+}
+
+int armada_drm_plane_disable(struct drm_plane *plane,
+			     struct drm_modeset_acquire_ctx *ctx)
+{
+	struct armada_plane *dplane = drm_to_armada_plane(plane);
+	struct armada_crtc *dcrtc;
+	struct armada_plane_work *work;
+	unsigned int idx = 0;
+	u32 sram_para1, enable_mask;
+
+	if (!plane->crtc)
+		return 0;
+
+	/*
+	 * Arrange to power down most RAMs and FIFOs if this is the primary
+	 * plane, otherwise just the YUV FIFOs for the overlay plane.
+	 */
+	if (plane->type == DRM_PLANE_TYPE_PRIMARY) {
+		sram_para1 = CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
+			     CFG_PDWN32x32 | CFG_PDWN64x66;
+		enable_mask = CFG_GRA_ENA;
+	} else {
+		sram_para1 = CFG_PDWN16x66 | CFG_PDWN32x66;
+		enable_mask = CFG_DMA_ENA;
+	}
+
+	dplane->state.ctrl0 &= ~enable_mask;
+
+	dcrtc = drm_to_armada_crtc(plane->crtc);
+
+	/*
+	 * Try to disable the plane and drop our ref on the framebuffer
+	 * at the next frame update. If we fail for any reason, disable
+	 * the plane immediately.
+	 */
+	work = &dplane->works[dplane->next_work];
+	work->fn = armada_drm_crtc_complete_disable_work;
+	work->cancel = armada_drm_crtc_complete_disable_work;
+	work->old_fb = plane->fb;
+
+	armada_reg_queue_mod(work->regs, idx,
+			     0, enable_mask, LCD_SPU_DMA_CTRL0);
+	armada_reg_queue_mod(work->regs, idx,
+			     sram_para1, 0, LCD_SPU_SRAM_PARA1);
+	armada_reg_queue_end(work->regs, idx);
+
+	/* Wait for any preceding work to complete, but don't wedge */
+	if (WARN_ON(!armada_drm_plane_work_wait(dplane, HZ)))
+		armada_drm_plane_work_cancel(dcrtc, dplane);
+
+	if (armada_drm_plane_work_queue(dcrtc, work)) {
+		work->fn(dcrtc, work);
+		if (work->old_fb)
+			drm_framebuffer_unreference(work->old_fb);
+	}
+
+	dplane->next_work = !dplane->next_work;
+
+	return 0;
+}
+
 static const struct drm_plane_funcs armada_primary_plane_funcs = {
-	.update_plane	= drm_primary_helper_update,
-	.disable_plane	= drm_primary_helper_disable,
+	.update_plane	= armada_drm_primary_update,
+	.disable_plane	= armada_drm_plane_disable,
 	.destroy	= drm_primary_helper_destroy,
 };
 
 int armada_drm_plane_init(struct armada_plane *plane)
 {
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(plane->works); i++)
+		plane->works[i].plane = &plane->base;
+
 	init_waitqueue_head(&plane->frame_wait);
 
 	return 0;
diff --git a/drivers/gpu/drm/armada/armada_crtc.h b/drivers/gpu/drm/armada/armada_crtc.h
index bfd3514fbe9b..445829b8877a 100644
--- a/drivers/gpu/drm/armada/armada_crtc.h
+++ b/drivers/gpu/drm/armada/armada_crtc.h
@@ -36,9 +36,13 @@ struct armada_plane;
 struct armada_variant;
 
 struct armada_plane_work {
-	void			(*fn)(struct armada_crtc *,
-				      struct armada_plane *,
-				      struct armada_plane_work *);
+	void (*fn)(struct armada_crtc *, struct armada_plane_work *);
+	void (*cancel)(struct armada_crtc *, struct armada_plane_work *);
+	bool need_kfree;
+	struct drm_plane *plane;
+	struct drm_framebuffer *old_fb;
+	struct drm_pending_vblank_event *event;
+	struct armada_regs regs[14];
 };
 
 struct armada_plane_state {
@@ -48,11 +52,15 @@ struct armada_plane_state {
 	u32 dst_hw;
 	u32 dst_yx;
 	u32 ctrl0;
+	bool changed;
+	bool vsync_update;
 };
 
 struct armada_plane {
 	struct drm_plane	base;
 	wait_queue_head_t	frame_wait;
+	bool			next_work;
+	struct armada_plane_work works[2];
 	struct armada_plane_work *work;
 	struct armada_plane_state state;
 };
@@ -60,10 +68,10 @@ struct armada_plane {
 
 int armada_drm_plane_init(struct armada_plane *plane);
 int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
-	struct armada_plane *plane, struct armada_plane_work *work);
+	struct armada_plane_work *work);
 int armada_drm_plane_work_wait(struct armada_plane *plane, long timeout);
-struct armada_plane_work *armada_drm_plane_work_cancel(
-	struct armada_crtc *dcrtc, struct armada_plane *plane);
+void armada_drm_plane_work_cancel(struct armada_crtc *dcrtc,
+	struct armada_plane *plane);
 void armada_drm_plane_calc_addrs(u32 *addrs, struct drm_framebuffer *fb,
 	int x, int y);
 
@@ -106,8 +114,8 @@ struct armada_crtc {
 
 void armada_drm_crtc_update_regs(struct armada_crtc *, struct armada_regs *);
 
-void armada_drm_crtc_plane_disable(struct armada_crtc *dcrtc,
-	struct drm_plane *plane);
+int armada_drm_plane_disable(struct drm_plane *plane,
+			     struct drm_modeset_acquire_ctx *ctx);
 
 extern struct platform_driver armada_lcd_platform_driver;
 
diff --git a/drivers/gpu/drm/armada/armada_drm.h b/drivers/gpu/drm/armada/armada_drm.h
index b064879ecdbd..cc4c557c9f66 100644
--- a/drivers/gpu/drm/armada/armada_drm.h
+++ b/drivers/gpu/drm/armada/armada_drm.h
@@ -84,7 +84,6 @@ void armada_drm_queue_unref_work(struct drm_device *,
 extern const struct drm_mode_config_funcs armada_drm_mode_config_funcs;
 
 int armada_fbdev_init(struct drm_device *);
-void armada_fbdev_lastclose(struct drm_device *);
 void armada_fbdev_fini(struct drm_device *);
 
 int armada_overlay_plane_create(struct drm_device *, unsigned long);
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index e857b88a9799..4b11b6b52f1d 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/of_graph.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_of.h>
 #include "armada_crtc.h"
 #include "armada_drm.h"
@@ -54,15 +55,10 @@ static struct drm_ioctl_desc armada_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(ARMADA_GEM_PWRITE, armada_gem_pwrite_ioctl, 0),
 };
 
-static void armada_drm_lastclose(struct drm_device *dev)
-{
-	armada_fbdev_lastclose(dev);
-}
-
 DEFINE_DRM_GEM_FOPS(armada_drm_fops);
 
 static struct drm_driver armada_drm_driver = {
-	.lastclose		= armada_drm_lastclose,
+	.lastclose		= drm_fb_helper_lastclose,
 	.gem_free_object_unlocked = armada_gem_free_object,
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
diff --git a/drivers/gpu/drm/armada/armada_fb.c b/drivers/gpu/drm/armada/armada_fb.c
index a38d5a0892a9..ac92bce07ecd 100644
--- a/drivers/gpu/drm/armada/armada_fb.c
+++ b/drivers/gpu/drm/armada/armada_fb.c
@@ -154,16 +154,7 @@ static struct drm_framebuffer *armada_fb_create(struct drm_device *dev,
 	return ERR_PTR(ret);
 }
 
-static void armada_output_poll_changed(struct drm_device *dev)
-{
-	struct armada_private *priv = dev->dev_private;
-	struct drm_fb_helper *fbh = priv->fbdev;
-
-	if (fbh)
-		drm_fb_helper_hotplug_event(fbh);
-}
-
 const struct drm_mode_config_funcs armada_drm_mode_config_funcs = {
 	.fb_create		= armada_fb_create,
-	.output_poll_changed	= armada_output_poll_changed,
+	.output_poll_changed	= drm_fb_helper_output_poll_changed,
 };
diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c
index a2ce83f84800..2a59db0994b2 100644
--- a/drivers/gpu/drm/armada/armada_fbdev.c
+++ b/drivers/gpu/drm/armada/armada_fbdev.c
@@ -159,14 +159,6 @@ int armada_fbdev_init(struct drm_device *dev)
 	return ret;
 }
 
-void armada_fbdev_lastclose(struct drm_device *dev)
-{
-	struct armada_private *priv = dev->dev_private;
-
-	if (priv->fbdev)
-		drm_fb_helper_restore_fbdev_mode_unlocked(priv->fbdev);
-}
-
 void armada_fbdev_fini(struct drm_device *dev)
 {
 	struct armada_private *priv = dev->dev_private;
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index aba947696178..77b55adaa2ac 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -7,7 +7,7 @@
  * published by the Free Software Foundation.
  */
 #include <drm/drmP.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_atomic_helper.h>
 #include "armada_crtc.h"
 #include "armada_drm.h"
 #include "armada_fb.h"
@@ -32,11 +32,6 @@ struct armada_ovl_plane_properties {
 
 struct armada_ovl_plane {
 	struct armada_plane base;
-	struct drm_framebuffer *old_fb;
-	struct {
-		struct armada_plane_work work;
-		struct armada_regs regs[13];
-	} vbl;
 	struct armada_ovl_plane_properties prop;
 };
 #define drm_to_armada_ovl_plane(p) \
@@ -67,218 +62,204 @@ armada_ovl_update_attr(struct armada_ovl_plane_properties *prop,
 	spin_unlock_irq(&dcrtc->irq_lock);
 }
 
-static void armada_ovl_retire_fb(struct armada_ovl_plane *dplane,
-	struct drm_framebuffer *fb)
-{
-	struct drm_framebuffer *old_fb;
-
-	old_fb = xchg(&dplane->old_fb, fb);
-
-	if (old_fb)
-		armada_drm_queue_unref_work(dplane->base.base.dev, old_fb);
-}
-
 /* === Plane support === */
 static void armada_ovl_plane_work(struct armada_crtc *dcrtc,
-	struct armada_plane *plane, struct armada_plane_work *work)
+	struct armada_plane_work *work)
 {
-	struct armada_ovl_plane *dplane = container_of(plane, struct armada_ovl_plane, base);
+	unsigned long flags;
 
-	trace_armada_ovl_plane_work(&dcrtc->crtc, &plane->base);
+	trace_armada_ovl_plane_work(&dcrtc->crtc, work->plane);
 
-	armada_drm_crtc_update_regs(dcrtc, dplane->vbl.regs);
-	armada_ovl_retire_fb(dplane, NULL);
+	spin_lock_irqsave(&dcrtc->irq_lock, flags);
+	armada_drm_crtc_update_regs(dcrtc, work->regs);
+	spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
 }
 
-static int
-armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
-	struct drm_framebuffer *fb,
-	int crtc_x, int crtc_y, unsigned crtc_w, unsigned crtc_h,
-	uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h,
-	struct drm_modeset_acquire_ctx *ctx)
+static void armada_ovl_plane_update_state(struct drm_plane_state *state,
+	struct armada_regs *regs)
 {
-	struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
-	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(state->plane);
+	struct armada_framebuffer *dfb = drm_fb_to_armada_fb(state->fb);
 	const struct drm_format_info *format;
-	struct drm_rect src = {
-		.x1 = src_x,
-		.y1 = src_y,
-		.x2 = src_x + src_w,
-		.y2 = src_y + src_h,
-	};
-	struct drm_rect dest = {
-		.x1 = crtc_x,
-		.y1 = crtc_y,
-		.x2 = crtc_x + crtc_w,
-		.y2 = crtc_y + crtc_h,
-	};
-	const struct drm_rect clip = {
-		.x2 = crtc->mode.hdisplay,
-		.y2 = crtc->mode.vdisplay,
-	};
-	uint32_t val, ctrl0;
-	unsigned idx = 0;
-	bool visible, fb_changed;
-	int ret;
+	unsigned int idx = 0;
+	bool fb_changed;
+	u32 val, ctrl0;
+	u16 src_x, src_y;
 
-	trace_armada_ovl_plane_update(plane, crtc, fb,
-				 crtc_x, crtc_y, crtc_w, crtc_h,
-				 src_x, src_y, src_w, src_h);
-
-	ret = drm_plane_helper_check_update(plane, crtc, fb, &src, &dest, &clip,
-					    DRM_MODE_ROTATE_0,
-					    0, INT_MAX, true, false, &visible);
-	if (ret)
-		return ret;
-
-	ctrl0 = CFG_DMA_FMT(drm_fb_to_armada_fb(fb)->fmt) |
-		CFG_DMA_MOD(drm_fb_to_armada_fb(fb)->mod) |
-		CFG_CBSH_ENA | CFG_DMA_HSMOOTH | CFG_DMA_ENA;
-
-	/* Does the position/size result in nothing to display? */
-	if (!visible)
-		ctrl0 &= ~CFG_DMA_ENA;
+	ctrl0 = CFG_DMA_FMT(dfb->fmt) | CFG_DMA_MOD(dfb->mod) | CFG_CBSH_ENA;
+	if (state->visible)
+		ctrl0 |= CFG_DMA_ENA;
+	if (drm_rect_width(&state->src) >> 16 != drm_rect_width(&state->dst))
+		ctrl0 |= CFG_DMA_HSMOOTH;
 
 	/*
 	 * Shifting a YUV packed format image by one pixel causes the U/V
 	 * planes to swap.  Compensate for it by also toggling the UV swap.
 	 */
-	format = fb->format;
-	if (format->num_planes == 1 && src.x1 >> 16 & (format->hsub - 1))
+	format = dfb->fb.format;
+	if (format->num_planes == 1 && state->src.x1 >> 16 & (format->hsub - 1))
 		ctrl0 ^= CFG_DMA_MOD(CFG_SWAPUV);
 
-	fb_changed = plane->fb != fb ||
-		     dplane->base.state.src_x != src.x1 >> 16 ||
-	             dplane->base.state.src_y != src.y1 >> 16;
-
-	if (!dcrtc->plane) {
-		dcrtc->plane = plane;
-		armada_ovl_update_attr(&dplane->prop, dcrtc);
-	}
-
-	/* FIXME: overlay on an interlaced display */
-	/* Just updating the position/size? */
-	if (!fb_changed && dplane->base.state.ctrl0 == ctrl0) {
-		val = (drm_rect_height(&src) & 0xffff0000) |
-		      drm_rect_width(&src) >> 16;
-		dplane->base.state.src_hw = val;
-		writel_relaxed(val, dcrtc->base + LCD_SPU_DMA_HPXL_VLN);
-
-		val = drm_rect_height(&dest) << 16 | drm_rect_width(&dest);
-		dplane->base.state.dst_hw = val;
-		writel_relaxed(val, dcrtc->base + LCD_SPU_DZM_HPXL_VLN);
-
-		val = dest.y1 << 16 | dest.x1;
-		dplane->base.state.dst_yx = val;
-		writel_relaxed(val, dcrtc->base + LCD_SPU_DMA_OVSA_HPXL_VLN);
-
-		return 0;
-	} else if (~dplane->base.state.ctrl0 & ctrl0 & CFG_DMA_ENA) {
+	if (~dplane->base.state.ctrl0 & ctrl0 & CFG_DMA_ENA) {
 		/* Power up the Y/U/V FIFOs on ENA 0->1 transitions */
-		armada_updatel(0, CFG_PDWN16x66 | CFG_PDWN32x66,
-			       dcrtc->base + LCD_SPU_SRAM_PARA1);
+		armada_reg_queue_mod(regs, idx,
+				     0, CFG_PDWN16x66 | CFG_PDWN32x66,
+				     LCD_SPU_SRAM_PARA1);
 	}
 
-	if (armada_drm_plane_work_wait(&dplane->base, HZ / 25) == 0)
-		armada_drm_plane_work_cancel(dcrtc, &dplane->base);
+	fb_changed = dplane->base.base.fb != &dfb->fb ||
+		     dplane->base.state.src_x != state->src.x1 >> 16 ||
+	             dplane->base.state.src_y != state->src.y1 >> 16;
+
+	dplane->base.state.vsync_update = fb_changed;
 
+	/* FIXME: overlay on an interlaced display */
 	if (fb_changed) {
 		u32 addrs[3];
 
-		/*
-		 * Take a reference on the new framebuffer - we want to
-		 * hold on to it while the hardware is displaying it.
-		 */
-		drm_framebuffer_get(fb);
-
-		if (plane->fb)
-			armada_ovl_retire_fb(dplane, plane->fb);
+		dplane->base.state.src_y = src_y = state->src.y1 >> 16;
+		dplane->base.state.src_x = src_x = state->src.x1 >> 16;
 
-		dplane->base.state.src_y = src_y = src.y1 >> 16;
-		dplane->base.state.src_x = src_x = src.x1 >> 16;
+		armada_drm_plane_calc_addrs(addrs, &dfb->fb, src_x, src_y);
 
-		armada_drm_plane_calc_addrs(addrs, fb, src_x, src_y);
-
-		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[0],
+		armada_reg_queue_set(regs, idx, addrs[0],
 				     LCD_SPU_DMA_START_ADDR_Y0);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[1],
+		armada_reg_queue_set(regs, idx, addrs[1],
 				     LCD_SPU_DMA_START_ADDR_U0);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[2],
+		armada_reg_queue_set(regs, idx, addrs[2],
 				     LCD_SPU_DMA_START_ADDR_V0);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[0],
+		armada_reg_queue_set(regs, idx, addrs[0],
 				     LCD_SPU_DMA_START_ADDR_Y1);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[1],
+		armada_reg_queue_set(regs, idx, addrs[1],
 				     LCD_SPU_DMA_START_ADDR_U1);
-		armada_reg_queue_set(dplane->vbl.regs, idx, addrs[2],
+		armada_reg_queue_set(regs, idx, addrs[2],
 				     LCD_SPU_DMA_START_ADDR_V1);
 
-		val = fb->pitches[0] << 16 | fb->pitches[0];
-		armada_reg_queue_set(dplane->vbl.regs, idx, val,
+		val = dfb->fb.pitches[0] << 16 | dfb->fb.pitches[0];
+		armada_reg_queue_set(regs, idx, val,
 				     LCD_SPU_DMA_PITCH_YC);
-		val = fb->pitches[1] << 16 | fb->pitches[2];
-		armada_reg_queue_set(dplane->vbl.regs, idx, val,
+		val = dfb->fb.pitches[1] << 16 | dfb->fb.pitches[2];
+		armada_reg_queue_set(regs, idx, val,
 				     LCD_SPU_DMA_PITCH_UV);
 	}
 
-	val = (drm_rect_height(&src) & 0xffff0000) | drm_rect_width(&src) >> 16;
+	val = (drm_rect_height(&state->src) & 0xffff0000) |
+	       drm_rect_width(&state->src) >> 16;
 	if (dplane->base.state.src_hw != val) {
 		dplane->base.state.src_hw = val;
-		armada_reg_queue_set(dplane->vbl.regs, idx, val,
+		armada_reg_queue_set(regs, idx, val,
 				     LCD_SPU_DMA_HPXL_VLN);
 	}
 
-	val = drm_rect_height(&dest) << 16 | drm_rect_width(&dest);
+	val = drm_rect_height(&state->dst) << 16 | drm_rect_width(&state->dst);
 	if (dplane->base.state.dst_hw != val) {
 		dplane->base.state.dst_hw = val;
-		armada_reg_queue_set(dplane->vbl.regs, idx, val,
+		armada_reg_queue_set(regs, idx, val,
 				     LCD_SPU_DZM_HPXL_VLN);
 	}
 
-	val = dest.y1 << 16 | dest.x1;
+	val = state->dst.y1 << 16 | state->dst.x1;
 	if (dplane->base.state.dst_yx != val) {
 		dplane->base.state.dst_yx = val;
-		armada_reg_queue_set(dplane->vbl.regs, idx, val,
+		armada_reg_queue_set(regs, idx, val,
 				     LCD_SPU_DMA_OVSA_HPXL_VLN);
 	}
 
 	if (dplane->base.state.ctrl0 != ctrl0) {
 		dplane->base.state.ctrl0 = ctrl0;
-		armada_reg_queue_mod(dplane->vbl.regs, idx, ctrl0,
+		armada_reg_queue_mod(regs, idx, ctrl0,
 			CFG_CBSH_ENA | CFG_DMAFORMAT | CFG_DMA_FTOGGLE |
 			CFG_DMA_HSMOOTH | CFG_DMA_TSTMODE |
 			CFG_DMA_MOD(CFG_SWAPRB | CFG_SWAPUV | CFG_SWAPYU |
 			CFG_YUV2RGB) | CFG_DMA_ENA,
 			LCD_SPU_DMA_CTRL0);
+		dplane->base.state.vsync_update = true;
 	}
-	if (idx) {
-		armada_reg_queue_end(dplane->vbl.regs, idx);
-		armada_drm_plane_work_queue(dcrtc, &dplane->base,
-					    &dplane->vbl.work);
-	}
-	return 0;
+
+	dplane->base.state.changed = idx != 0;
+
+	armada_reg_queue_end(regs, idx);
 }
 
-static int armada_ovl_plane_disable(struct drm_plane *plane,
-				    struct drm_modeset_acquire_ctx *ctx)
+static int
+armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
+	struct drm_framebuffer *fb,
+	int crtc_x, int crtc_y, unsigned crtc_w, unsigned crtc_h,
+	uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h,
+	struct drm_modeset_acquire_ctx *ctx)
 {
 	struct armada_ovl_plane *dplane = drm_to_armada_ovl_plane(plane);
-	struct drm_framebuffer *fb;
-	struct armada_crtc *dcrtc;
+	struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+	struct armada_plane_work *work;
+	struct drm_plane_state state = {
+		.plane = plane,
+		.crtc = crtc,
+		.fb = fb,
+		.src_x = src_x,
+		.src_y = src_y,
+		.src_w = src_w,
+		.src_h = src_h,
+		.crtc_x = crtc_x,
+		.crtc_y = crtc_y,
+		.crtc_w = crtc_w,
+		.crtc_h = crtc_h,
+		.rotation = DRM_MODE_ROTATE_0,
+	};
+	const struct drm_rect clip = {
+		.x2 = crtc->mode.hdisplay,
+		.y2 = crtc->mode.vdisplay,
+	};
+	int ret;
+
+	trace_armada_ovl_plane_update(plane, crtc, fb,
+				 crtc_x, crtc_y, crtc_w, crtc_h,
+				 src_x, src_y, src_w, src_h);
+
+	ret = drm_atomic_helper_check_plane_state(&state, crtc->state, &clip, 0,
+						  INT_MAX, true, false);
+	if (ret)
+		return ret;
+
+	work = &dplane->base.works[dplane->base.next_work];
+
+	if (plane->fb != fb) {
+		/*
+		 * Take a reference on the new framebuffer - we want to
+		 * hold on to it while the hardware is displaying it.
+		 */
+		drm_framebuffer_reference(fb);
+
+		work->old_fb = plane->fb;
+	} else {
+		work->old_fb = NULL;
+	}
 
-	if (!dplane->base.base.crtc)
+	armada_ovl_plane_update_state(&state, work->regs);
+
+	if (!dplane->base.state.changed)
 		return 0;
 
-	dcrtc = drm_to_armada_crtc(dplane->base.base.crtc);
+	/* Wait for pending work to complete */
+	if (armada_drm_plane_work_wait(&dplane->base, HZ / 25) == 0)
+		armada_drm_plane_work_cancel(dcrtc, &dplane->base);
+
+	/* Just updating the position/size? */
+	if (!dplane->base.state.vsync_update) {
+		armada_ovl_plane_work(dcrtc, work);
+		return 0;
+	}
 
-	armada_drm_plane_work_cancel(dcrtc, &dplane->base);
-	armada_drm_crtc_plane_disable(dcrtc, plane);
+	if (!dcrtc->plane) {
+		dcrtc->plane = plane;
+		armada_ovl_update_attr(&dplane->prop, dcrtc);
+	}
 
-	dcrtc->plane = NULL;
-	dplane->base.state.ctrl0 = 0;
+	/* Queue it for update on the next interrupt if we are enabled */
+	ret = armada_drm_plane_work_queue(dcrtc, work);
+	if (ret)
+		DRM_ERROR("failed to queue plane work: %d\n", ret);
 
-	fb = xchg(&dplane->old_fb, NULL);
-	if (fb)
-		drm_framebuffer_put(fb);
+	dplane->base.next_work = !dplane->base.next_work;
 
 	return 0;
 }
@@ -362,7 +343,7 @@ static int armada_ovl_plane_set_property(struct drm_plane *plane,
 
 static const struct drm_plane_funcs armada_ovl_plane_funcs = {
 	.update_plane	= armada_ovl_plane_update,
-	.disable_plane	= armada_ovl_plane_disable,
+	.disable_plane	= armada_drm_plane_disable,
 	.destroy	= armada_ovl_plane_destroy,
 	.set_property	= armada_ovl_plane_set_property,
 };
@@ -454,7 +435,8 @@ int armada_overlay_plane_create(struct drm_device *dev, unsigned long crtcs)
 		return ret;
 	}
 
-	dplane->vbl.work.fn = armada_ovl_plane_work;
+	dplane->base.works[0].fn = armada_ovl_plane_work;
+	dplane->base.works[1].fn = armada_ovl_plane_work;
 
 	ret = drm_universal_plane_init(dev, &dplane->base.base, crtcs,
 				       &armada_ovl_plane_funcs,
diff --git a/drivers/gpu/drm/armada/armada_trace.h b/drivers/gpu/drm/armada/armada_trace.h
index 8dbfea7a00fe..f03a56bda596 100644
--- a/drivers/gpu/drm/armada/armada_trace.h
+++ b/drivers/gpu/drm/armada/armada_trace.h
@@ -34,14 +34,34 @@ TRACE_EVENT(armada_ovl_plane_update,
 		__field(struct drm_plane *, plane)
 		__field(struct drm_crtc *, crtc)
 		__field(struct drm_framebuffer *, fb)
+		__field(int, crtc_x)
+		__field(int, crtc_y)
+		__field(unsigned int, crtc_w)
+		__field(unsigned int, crtc_h)
+		__field(u32, src_x)
+		__field(u32, src_y)
+		__field(u32, src_w)
+		__field(u32, src_h)
 	),
 	TP_fast_assign(
 		__entry->plane = plane;
 		__entry->crtc = crtc;
 		__entry->fb = fb;
+		__entry->crtc_x = crtc_x;
+		__entry->crtc_y = crtc_y;
+		__entry->crtc_w = crtc_w;
+		__entry->crtc_h = crtc_h;
+		__entry->src_x = src_x;
+		__entry->src_y = src_y;
+		__entry->src_w = src_w;
+		__entry->src_h = src_h;
 	),
-	TP_printk("plane %p crtc %p fb %p",
-		__entry->plane, __entry->crtc, __entry->fb)
+	TP_printk("plane %p crtc %p fb %p crtc @ (%d,%d, %ux%u) src @ (%u,%u, %ux%u)",
+		__entry->plane, __entry->crtc, __entry->fb,
+		__entry->crtc_x, __entry->crtc_y,
+		__entry->crtc_w, __entry->crtc_h,
+		__entry->src_x >> 16, __entry->src_y >> 16,
+		__entry->src_w >> 16, __entry->src_h >> 16)
 );
 
 TRACE_EVENT(armada_ovl_plane_work,
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 9555a3542022..831b73392d82 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -644,6 +644,7 @@ static void ast_crtc_commit(struct drm_crtc *crtc)
 {
 	struct ast_private *ast = crtc->dev->dev_private;
 	ast_set_index_reg_mask(ast, AST_IO_SEQ_PORT, 0x1, 0xdf, 0);
+	ast_crtc_load_lut(crtc);
 }
 
 
diff --git a/drivers/gpu/drm/ast/ast_tables.h b/drivers/gpu/drm/ast/ast_tables.h
index 5f4c2e833a65..d665dd5af5dd 100644
--- a/drivers/gpu/drm/ast/ast_tables.h
+++ b/drivers/gpu/drm/ast/ast_tables.h
@@ -97,7 +97,7 @@ static const struct ast_vbios_dclk_info dclk_table[] = {
 	{0x67, 0x22, 0x00},			/* 0E: VCLK157_5	*/
 	{0x6A, 0x22, 0x00},			/* 0F: VCLK162		*/
 	{0x4d, 0x4c, 0x80},			/* 10: VCLK154		*/
-	{0xa7, 0x78, 0x80},			/* 11: VCLK83.5		*/
+	{0x68, 0x6f, 0x80},			/* 11: VCLK83.5		*/
 	{0x28, 0x49, 0x80},			/* 12: VCLK106.5	*/
 	{0x37, 0x49, 0x80},			/* 13: VCLK146.25	*/
 	{0x1f, 0x45, 0x80},			/* 14: VCLK148.5	*/
@@ -127,7 +127,7 @@ static const struct ast_vbios_dclk_info dclk_table_ast2500[] = {
 	{0x67, 0x22, 0x00},			/* 0E: VCLK157_5	*/
 	{0x6A, 0x22, 0x00},			/* 0F: VCLK162		*/
 	{0x4d, 0x4c, 0x80},			/* 10: VCLK154		*/
-	{0xa7, 0x78, 0x80},			/* 11: VCLK83.5		*/
+	{0x68, 0x6f, 0x80},			/* 11: VCLK83.5		*/
 	{0x28, 0x49, 0x80},			/* 12: VCLK106.5	*/
 	{0x37, 0x49, 0x80},			/* 13: VCLK146.25	*/
 	{0x1f, 0x45, 0x80},			/* 14: VCLK148.5	*/
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 696a15dc2f3f..7b784d91e258 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -216,9 +216,10 @@ static struct ttm_tt *ast_ttm_tt_create(struct ttm_bo_device *bdev,
 	return tt;
 }
 
-static int ast_ttm_tt_populate(struct ttm_tt *ttm)
+static int ast_ttm_tt_populate(struct ttm_tt *ttm,
+			struct ttm_operation_ctx *ctx)
 {
-	return ttm_pool_populate(ttm);
+	return ttm_pool_populate(ttm, ctx);
 }
 
 static void ast_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -237,7 +238,6 @@ struct ttm_bo_driver ast_bo_driver = {
 	.verify_access = ast_bo_verify_access,
 	.io_mem_reserve = &ast_ttm_io_mem_reserve,
 	.io_mem_free = &ast_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 };
 
 int ast_mm_init(struct ast_private *ast)
@@ -354,6 +354,7 @@ static inline u64 ast_bo_gpu_offset(struct ast_bo *bo)
 
 int ast_bo_pin(struct ast_bo *bo, u32 pl_flag, u64 *gpu_addr)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 
 	if (bo->pin_count) {
@@ -365,7 +366,7 @@ int ast_bo_pin(struct ast_bo *bo, u32 pl_flag, u64 *gpu_addr)
 	ast_ttm_placement(bo, pl_flag);
 	for (i = 0; i < bo->placement.num_placement; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret)
 		return ret;
 
@@ -377,6 +378,7 @@ int ast_bo_pin(struct ast_bo *bo, u32 pl_flag, u64 *gpu_addr)
 
 int ast_bo_unpin(struct ast_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i;
 	if (!bo->pin_count) {
 		DRM_ERROR("unpin bad %p\n", bo);
@@ -388,11 +390,12 @@ int ast_bo_unpin(struct ast_bo *bo)
 
 	for (i = 0; i < bo->placement.num_placement ; i++)
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
-	return ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	return ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 }
 
 int ast_bo_push_sysram(struct ast_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 	if (!bo->pin_count) {
 		DRM_ERROR("unpin bad %p\n", bo);
@@ -409,7 +412,7 @@ int ast_bo_push_sysram(struct ast_bo *bo)
 	for (i = 0; i < bo->placement.num_placement ; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret) {
 		DRM_ERROR("pushing to VRAM failed\n");
 		return ret;
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index c6e8061ffcfc..c1ea5c36b006 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -461,13 +461,6 @@ static struct drm_framebuffer *atmel_hlcdc_fb_create(struct drm_device *dev,
 	return drm_gem_fb_create(dev, file_priv, mode_cmd);
 }
 
-static void atmel_hlcdc_fb_output_poll_changed(struct drm_device *dev)
-{
-	struct atmel_hlcdc_dc *dc = dev->dev_private;
-
-	drm_fbdev_cma_hotplug_event(dc->fbdev);
-}
-
 struct atmel_hlcdc_dc_commit {
 	struct work_struct work;
 	struct drm_device *dev;
@@ -563,7 +556,7 @@ static int atmel_hlcdc_dc_atomic_commit(struct drm_device *dev,
 
 static const struct drm_mode_config_funcs mode_config_funcs = {
 	.fb_create = atmel_hlcdc_fb_create,
-	.output_poll_changed = atmel_hlcdc_fb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = atmel_hlcdc_dc_atomic_commit,
 };
@@ -665,10 +658,7 @@ static int atmel_hlcdc_dc_load(struct drm_device *dev)
 
 	platform_set_drvdata(pdev, dev);
 
-	dc->fbdev = drm_fbdev_cma_init(dev, 24,
-			dev->mode_config.num_connector);
-	if (IS_ERR(dc->fbdev))
-		dc->fbdev = NULL;
+	drm_fb_cma_fbdev_init(dev, 24, 0);
 
 	drm_kms_helper_poll_init(dev);
 
@@ -688,8 +678,7 @@ static void atmel_hlcdc_dc_unload(struct drm_device *dev)
 {
 	struct atmel_hlcdc_dc *dc = dev->dev_private;
 
-	if (dc->fbdev)
-		drm_fbdev_cma_fini(dc->fbdev);
+	drm_fb_cma_fbdev_fini(dev);
 	flush_workqueue(dc->wq);
 	drm_kms_helper_poll_fini(dev);
 	drm_mode_config_cleanup(dev);
@@ -705,13 +694,6 @@ static void atmel_hlcdc_dc_unload(struct drm_device *dev)
 	destroy_workqueue(dc->wq);
 }
 
-static void atmel_hlcdc_dc_lastclose(struct drm_device *dev)
-{
-	struct atmel_hlcdc_dc *dc = dev->dev_private;
-
-	drm_fbdev_cma_restore_mode(dc->fbdev);
-}
-
 static int atmel_hlcdc_dc_irq_postinstall(struct drm_device *dev)
 {
 	struct atmel_hlcdc_dc *dc = dev->dev_private;
@@ -744,7 +726,7 @@ static struct drm_driver atmel_hlcdc_dc_driver = {
 	.driver_features = DRIVER_HAVE_IRQ | DRIVER_GEM |
 			   DRIVER_MODESET | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
-	.lastclose = atmel_hlcdc_dc_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.irq_handler = atmel_hlcdc_dc_irq_handler,
 	.irq_preinstall = atmel_hlcdc_dc_irq_uninstall,
 	.irq_postinstall = atmel_hlcdc_dc_irq_postinstall,
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
index 6833ee253cfa..ab32d5b268d2 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
@@ -32,6 +32,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -374,7 +375,6 @@ struct atmel_hlcdc_dc {
 	const struct atmel_hlcdc_dc_desc *desc;
 	struct dma_pool *dscrpool;
 	struct atmel_hlcdc *hlcdc;
-	struct drm_fbdev_cma *fbdev;
 	struct drm_crtc *crtc;
 	struct atmel_hlcdc_layer *layers[ATMEL_HLCDC_MAX_LAYERS];
 	struct workqueue_struct *wq;
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index c4cadb638460..704e879711e4 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -205,7 +205,6 @@ struct ttm_bo_driver bochs_bo_driver = {
 	.verify_access = bochs_bo_verify_access,
 	.io_mem_reserve = &bochs_ttm_io_mem_reserve,
 	.io_mem_free = &bochs_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 };
 
 int bochs_mm_init(struct bochs_device *bochs)
@@ -283,6 +282,7 @@ static inline u64 bochs_bo_gpu_offset(struct bochs_bo *bo)
 
 int bochs_bo_pin(struct bochs_bo *bo, u32 pl_flag, u64 *gpu_addr)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 
 	if (bo->pin_count) {
@@ -295,7 +295,7 @@ int bochs_bo_pin(struct bochs_bo *bo, u32 pl_flag, u64 *gpu_addr)
 	bochs_ttm_placement(bo, pl_flag);
 	for (i = 0; i < bo->placement.num_placement; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret)
 		return ret;
 
@@ -307,6 +307,7 @@ int bochs_bo_pin(struct bochs_bo *bo, u32 pl_flag, u64 *gpu_addr)
 
 int bochs_bo_unpin(struct bochs_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 
 	if (!bo->pin_count) {
@@ -320,7 +321,7 @@ int bochs_bo_unpin(struct bochs_bo *bo)
 
 	for (i = 0; i < bo->placement.num_placement; i++)
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/bridge/analogix-anx78xx.c b/drivers/gpu/drm/bridge/analogix-anx78xx.c
index 9385eb0b1ee4..ed12a7ddd64a 100644
--- a/drivers/gpu/drm/bridge/analogix-anx78xx.c
+++ b/drivers/gpu/drm/bridge/analogix-anx78xx.c
@@ -977,8 +977,6 @@ static int anx78xx_get_modes(struct drm_connector *connector)
 	}
 
 	num_modes = drm_add_edid_modes(connector, anx78xx->edid);
-	/* Store the ELD */
-	drm_edid_to_eld(connector, anx78xx->edid);
 
 unlock:
 	mutex_unlock(&anx78xx->lock);
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
index b7eb704d0a8a..86789f8918a4 100644
--- a/drivers/gpu/drm/bridge/sil-sii8620.c
+++ b/drivers/gpu/drm/bridge/sil-sii8620.c
@@ -1169,8 +1169,18 @@ static void sii8620_set_infoframes(struct sii8620 *ctx)
 	sii8620_write_buf(ctx, REG_TPI_INFO_B0, buf, ret);
 }
 
-static void sii8620_start_hdmi(struct sii8620 *ctx)
+static void sii8620_start_video(struct sii8620 *ctx)
 {
+	if (!sii8620_is_mhl3(ctx))
+		sii8620_stop_video(ctx);
+
+	if (ctx->sink_type == SINK_DVI && !sii8620_is_mhl3(ctx)) {
+		sii8620_write(ctx, REG_RX_HDMI_CTRL2,
+			      VAL_RX_HDMI_CTRL2_DEFVAL);
+		sii8620_write(ctx, REG_TPI_SC, 0);
+		return;
+	}
+
 	sii8620_write_seq_static(ctx,
 		REG_RX_HDMI_CTRL2, VAL_RX_HDMI_CTRL2_DEFVAL
 			| BIT_RX_HDMI_CTRL2_USE_AV_MUTE,
@@ -1229,21 +1239,6 @@ static void sii8620_start_hdmi(struct sii8620 *ctx)
 	sii8620_set_infoframes(ctx);
 }
 
-static void sii8620_start_video(struct sii8620 *ctx)
-{
-	if (!sii8620_is_mhl3(ctx))
-		sii8620_stop_video(ctx);
-
-	switch (ctx->sink_type) {
-	case SINK_HDMI:
-		sii8620_start_hdmi(ctx);
-		break;
-	case SINK_DVI:
-	default:
-		break;
-	}
-}
-
 static void sii8620_disable_hpd(struct sii8620 *ctx)
 {
 	sii8620_setbits(ctx, REG_EDID_CTRL, BIT_EDID_CTRL_EDID_PRIME_VALID, 0);
@@ -1945,8 +1940,13 @@ static void sii8620_irq_scdt(struct sii8620 *ctx)
 	if (stat & BIT_INTR_SCDT_CHANGE) {
 		u8 cstat = sii8620_readb(ctx, REG_TMDS_CSTAT_P3);
 
-		if (cstat & BIT_TMDS_CSTAT_P3_SCDT)
-			sii8620_scdt_high(ctx);
+		if (cstat & BIT_TMDS_CSTAT_P3_SCDT) {
+			if (ctx->sink_type == SINK_HDMI)
+				/* enable infoframe interrupt */
+				sii8620_scdt_high(ctx);
+			else
+				sii8620_start_video(ctx);
+		}
 	}
 
 	sii8620_write(ctx, REG_INTR5, stat);
@@ -2191,6 +2191,19 @@ static void sii8620_detach(struct drm_bridge *bridge)
 	rc_unregister_device(ctx->rc_dev);
 }
 
+static enum drm_mode_status sii8620_mode_valid(struct drm_bridge *bridge,
+					 const struct drm_display_mode *mode)
+{
+	struct sii8620 *ctx = bridge_to_sii8620(bridge);
+	bool can_pack = ctx->devcap[MHL_DCAP_VID_LINK_MODE] &
+			MHL_DCAP_VID_LINK_PPIXEL;
+	unsigned int max_pclk = sii8620_is_mhl3(ctx) ? MHL3_MAX_LCLK :
+						       MHL1_MAX_LCLK;
+	max_pclk /= can_pack ? 2 : 3;
+
+	return (mode->clock > max_pclk) ? MODE_CLOCK_HIGH : MODE_OK;
+}
+
 static bool sii8620_mode_fixup(struct drm_bridge *bridge,
 			       const struct drm_display_mode *mode,
 			       struct drm_display_mode *adjusted_mode)
@@ -2220,8 +2233,9 @@ static bool sii8620_mode_fixup(struct drm_bridge *bridge,
 			union hdmi_infoframe frm;
 			u8 mhl_vic[] = { 0, 95, 94, 93, 98 };
 
+			/* FIXME: We need the connector here */
 			drm_hdmi_vendor_infoframe_from_display_mode(
-				&frm.vendor.hdmi, adjusted_mode);
+				&frm.vendor.hdmi, NULL, adjusted_mode);
 			vic = frm.vendor.hdmi.vic;
 			if (vic >= ARRAY_SIZE(mhl_vic))
 				vic = 0;
@@ -2238,6 +2252,7 @@ static const struct drm_bridge_funcs sii8620_bridge_funcs = {
 	.attach = sii8620_attach,
 	.detach = sii8620_detach,
 	.mode_fixup = sii8620_mode_fixup,
+	.mode_valid = sii8620_mode_valid,
 };
 
 static int sii8620_probe(struct i2c_client *client,
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index b72259bf6e2f..a38db40ce990 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -1438,7 +1438,9 @@ static void hdmi_config_vendor_specific_infoframe(struct dw_hdmi *hdmi,
 	u8 buffer[10];
 	ssize_t err;
 
-	err = drm_hdmi_vendor_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_vendor_infoframe_from_display_mode(&frame,
+							  &hdmi->connector,
+							  mode);
 	if (err < 0)
 		/*
 		 * Going into that statement does not means vendor infoframe
@@ -1911,8 +1913,6 @@ static int dw_hdmi_connector_get_modes(struct drm_connector *connector)
 		drm_mode_connector_update_edid_property(connector, edid);
 		cec_notifier_set_phys_addr_from_edid(hdmi->cec_notifier, edid);
 		ret = drm_add_edid_modes(connector, edid);
-		/* Store the ELD */
-		drm_edid_to_eld(connector, edid);
 		kfree(edid);
 	} else {
 		dev_dbg(hdmi->dev, "failed to get edid\n");
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 8636e7eeb731..08ab7d6aea65 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -6,6 +6,8 @@
  *
  * Copyright (C) 2016 Pengutronix, Philipp Zabel <p.zabel@pengutronix.de>
  *
+ * Copyright (C) 2016 Zodiac Inflight Innovations
+ *
  * Initially based on: drivers/gpu/drm/i2c/tda998x_drv.c
  *
  * Copyright (C) 2012 Texas Instruments
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
index b5f528543956..26df1e8cd490 100644
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ b/drivers/gpu/drm/cirrus/cirrus_main.c
@@ -13,6 +13,14 @@
 
 #include "cirrus_drv.h"
 
+static int cirrus_create_handle(struct drm_framebuffer *fb,
+				struct drm_file* file_priv,
+				unsigned int* handle)
+{
+	struct cirrus_framebuffer *cirrus_fb = to_cirrus_framebuffer(fb);
+
+	return drm_gem_handle_create(file_priv, cirrus_fb->obj, handle);
+}
 
 static void cirrus_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
@@ -24,6 +32,7 @@ static void cirrus_user_framebuffer_destroy(struct drm_framebuffer *fb)
 }
 
 static const struct drm_framebuffer_funcs cirrus_fb_funcs = {
+	.create_handle = cirrus_create_handle,
 	.destroy = cirrus_user_framebuffer_destroy,
 };
 
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index cd23b1b28259..c91b9b054e3f 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -294,22 +294,7 @@ static void cirrus_crtc_prepare(struct drm_crtc *crtc)
 {
 }
 
-/*
- * This is called after a mode is programmed. It should reverse anything done
- * by the prepare function
- */
-static void cirrus_crtc_commit(struct drm_crtc *crtc)
-{
-}
-
-/*
- * The core can pass us a set of gamma values to program. We actually only
- * use this for 8-bit mode so can't perform smooth fades on deeper modes,
- * but it's a requirement that we provide the function
- */
-static int cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-				 u16 *blue, uint32_t size,
-				 struct drm_modeset_acquire_ctx *ctx)
+static void cirrus_crtc_load_lut(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	struct cirrus_device *cdev = dev->dev_private;
@@ -317,7 +302,7 @@ static int cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 	int i;
 
 	if (!crtc->enabled)
-		return 0;
+		return;
 
 	r = crtc->gamma_store;
 	g = r + crtc->gamma_size;
@@ -330,6 +315,27 @@ static int cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
 		WREG8(PALETTE_DATA, *g++ >> 8);
 		WREG8(PALETTE_DATA, *b++ >> 8);
 	}
+}
+
+/*
+ * This is called after a mode is programmed. It should reverse anything done
+ * by the prepare function
+ */
+static void cirrus_crtc_commit(struct drm_crtc *crtc)
+{
+	cirrus_crtc_load_lut(crtc);
+}
+
+/*
+ * The core can pass us a set of gamma values to program. We actually only
+ * use this for 8-bit mode so can't perform smooth fades on deeper modes,
+ * but it's a requirement that we provide the function
+ */
+static int cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+				 u16 *blue, uint32_t size,
+				 struct drm_modeset_acquire_ctx *ctx)
+{
+	cirrus_crtc_load_lut(crtc);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index 1ff1838c0d44..a8e31ea07382 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -216,9 +216,10 @@ static struct ttm_tt *cirrus_ttm_tt_create(struct ttm_bo_device *bdev,
 	return tt;
 }
 
-static int cirrus_ttm_tt_populate(struct ttm_tt *ttm)
+static int cirrus_ttm_tt_populate(struct ttm_tt *ttm,
+		struct ttm_operation_ctx *ctx)
 {
-	return ttm_pool_populate(ttm);
+	return ttm_pool_populate(ttm, ctx);
 }
 
 static void cirrus_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -237,7 +238,6 @@ struct ttm_bo_driver cirrus_bo_driver = {
 	.verify_access = cirrus_bo_verify_access,
 	.io_mem_reserve = &cirrus_ttm_io_mem_reserve,
 	.io_mem_free = &cirrus_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 };
 
 int cirrus_mm_init(struct cirrus_device *cirrus)
@@ -358,6 +358,7 @@ static inline u64 cirrus_bo_gpu_offset(struct cirrus_bo *bo)
 
 int cirrus_bo_pin(struct cirrus_bo *bo, u32 pl_flag, u64 *gpu_addr)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 
 	if (bo->pin_count) {
@@ -369,7 +370,7 @@ int cirrus_bo_pin(struct cirrus_bo *bo, u32 pl_flag, u64 *gpu_addr)
 	cirrus_ttm_placement(bo, pl_flag);
 	for (i = 0; i < bo->placement.num_placement; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret)
 		return ret;
 
@@ -381,6 +382,7 @@ int cirrus_bo_pin(struct cirrus_bo *bo, u32 pl_flag, u64 *gpu_addr)
 
 int cirrus_bo_push_sysram(struct cirrus_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 	if (!bo->pin_count) {
 		DRM_ERROR("unpin bad %p\n", bo);
@@ -397,7 +399,7 @@ int cirrus_bo_push_sysram(struct cirrus_bo *bo)
 	for (i = 0; i < bo->placement.num_placement ; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret) {
 		DRM_ERROR("pushing to VRAM failed\n");
 		return ret;
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index c2da5585e201..b76d49218cf1 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -33,6 +33,7 @@
 #include <linux/sync_file.h>
 
 #include "drm_crtc_internal.h"
+#include "drm_internal.h"
 
 void __drm_crtc_commit_free(struct kref *kref)
 {
@@ -49,7 +50,8 @@ EXPORT_SYMBOL(__drm_crtc_commit_free);
  * @state: atomic state
  *
  * Free all the memory allocated by drm_atomic_state_init.
- * This is useful for drivers that subclass the atomic state.
+ * This should only be used by drivers which are still subclassing
+ * &drm_atomic_state and haven't switched to &drm_private_state yet.
  */
 void drm_atomic_state_default_release(struct drm_atomic_state *state)
 {
@@ -66,7 +68,8 @@ EXPORT_SYMBOL(drm_atomic_state_default_release);
  * @state: atomic state
  *
  * Default implementation for filling in a new atomic state.
- * This is useful for drivers that subclass the atomic state.
+ * This should only be used by drivers which are still subclassing
+ * &drm_atomic_state and haven't switched to &drm_private_state yet.
  */
 int
 drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state)
@@ -131,7 +134,8 @@ EXPORT_SYMBOL(drm_atomic_state_alloc);
  * @state: atomic state
  *
  * Default implementation for clearing atomic state.
- * This is useful for drivers that subclass the atomic state.
+ * This should only be used by drivers which are still subclassing
+ * &drm_atomic_state and haven't switched to &drm_private_state yet.
  */
 void drm_atomic_state_default_clear(struct drm_atomic_state *state)
 {
@@ -907,11 +911,12 @@ static int drm_atomic_plane_check(struct drm_plane *plane,
 	    state->src_h > fb_height ||
 	    state->src_y > fb_height - state->src_h) {
 		DRM_DEBUG_ATOMIC("Invalid source coordinates "
-				 "%u.%06ux%u.%06u+%u.%06u+%u.%06u\n",
+				 "%u.%06ux%u.%06u+%u.%06u+%u.%06u (fb %ux%u)\n",
 				 state->src_w >> 16, ((state->src_w & 0xffff) * 15625) >> 10,
 				 state->src_h >> 16, ((state->src_h & 0xffff) * 15625) >> 10,
 				 state->src_x >> 16, ((state->src_x & 0xffff) * 15625) >> 10,
-				 state->src_y >> 16, ((state->src_y & 0xffff) * 15625) >> 10);
+				 state->src_y >> 16, ((state->src_y & 0xffff) * 15625) >> 10,
+				 state->fb->width, state->fb->height);
 		return -ENOSPC;
 	}
 
@@ -934,21 +939,8 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
 	drm_printf(p, "plane[%u]: %s\n", plane->base.id, plane->name);
 	drm_printf(p, "\tcrtc=%s\n", state->crtc ? state->crtc->name : "(null)");
 	drm_printf(p, "\tfb=%u\n", state->fb ? state->fb->base.id : 0);
-	if (state->fb) {
-		struct drm_framebuffer *fb = state->fb;
-		int i, n = fb->format->num_planes;
-		struct drm_format_name_buf format_name;
-
-		drm_printf(p, "\t\tformat=%s\n",
-		              drm_get_format_name(fb->format->format, &format_name));
-		drm_printf(p, "\t\t\tmodifier=0x%llx\n", fb->modifier);
-		drm_printf(p, "\t\tsize=%dx%d\n", fb->width, fb->height);
-		drm_printf(p, "\t\tlayers:\n");
-		for (i = 0; i < n; i++) {
-			drm_printf(p, "\t\t\tpitch[%d]=%u\n", i, fb->pitches[i]);
-			drm_printf(p, "\t\t\toffset[%d]=%u\n", i, fb->offsets[i]);
-		}
-	}
+	if (state->fb)
+		drm_framebuffer_print_info(p, 2, state->fb);
 	drm_printf(p, "\tcrtc-pos=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&dest));
 	drm_printf(p, "\tsrc-pos=" DRM_RECT_FP_FMT "\n", DRM_RECT_FP_ARG(&src));
 	drm_printf(p, "\trotation=%x\n", state->rotation);
@@ -957,6 +949,42 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
 		plane->funcs->atomic_print_state(p, state);
 }
 
+/**
+ * DOC: handling driver private state
+ *
+ * Very often the DRM objects exposed to userspace in the atomic modeset api
+ * (&drm_connector, &drm_crtc and &drm_plane) do not map neatly to the
+ * underlying hardware. Especially for any kind of shared resources (e.g. shared
+ * clocks, scaler units, bandwidth and fifo limits shared among a group of
+ * planes or CRTCs, and so on) it makes sense to model these as independent
+ * objects. Drivers then need to do similar state tracking and commit ordering for
+ * such private (since not exposed to userpace) objects as the atomic core and
+ * helpers already provide for connectors, planes and CRTCs.
+ *
+ * To make this easier on drivers the atomic core provides some support to track
+ * driver private state objects using struct &drm_private_obj, with the
+ * associated state struct &drm_private_state.
+ *
+ * Similar to userspace-exposed objects, private state structures can be
+ * acquired by calling drm_atomic_get_private_obj_state(). Since this function
+ * does not take care of locking, drivers should wrap it for each type of
+ * private state object they have with the required call to drm_modeset_lock()
+ * for the corresponding &drm_modeset_lock.
+ *
+ * All private state structures contained in a &drm_atomic_state update can be
+ * iterated using for_each_oldnew_private_obj_in_state(),
+ * for_each_new_private_obj_in_state() and for_each_old_private_obj_in_state().
+ * Drivers are recommended to wrap these for each type of driver private state
+ * object they have, filtering on &drm_private_obj.funcs using for_each_if(), at
+ * least if they want to iterate over all objects of a given type.
+ *
+ * An earlier way to handle driver private state was by subclassing struct
+ * &drm_atomic_state. But since that encourages non-standard ways to implement
+ * the check/commit split atomic requires (by using e.g. "check and rollback or
+ * commit instead" of "duplicate state, check, then either commit or release
+ * duplicated state) it is deprecated in favour of using &drm_private_state.
+ */
+
 /**
  * drm_atomic_private_obj_init - initialize private object
  * @obj: private object
@@ -1808,7 +1836,7 @@ int drm_atomic_debugfs_init(struct drm_minor *minor)
 #endif
 
 /*
- * The big monstor ioctl
+ * The big monster ioctl
  */
 
 static struct drm_pending_vblank_event *create_vblank_event(
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index b16f1d69a0bb..ae3cbfe9e01c 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -695,6 +695,100 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_atomic_helper_check_modeset);
 
+/**
+ * drm_atomic_helper_check_plane_state() - Check plane state for validity
+ * @plane_state: plane state to check
+ * @crtc_state: crtc state to check
+ * @clip: integer clipping coordinates
+ * @min_scale: minimum @src:@dest scaling factor in 16.16 fixed point
+ * @max_scale: maximum @src:@dest scaling factor in 16.16 fixed point
+ * @can_position: is it legal to position the plane such that it
+ *                doesn't cover the entire crtc?  This will generally
+ *                only be false for primary planes.
+ * @can_update_disabled: can the plane be updated while the crtc
+ *                       is disabled?
+ *
+ * Checks that a desired plane update is valid, and updates various
+ * bits of derived state (clipped coordinates etc.). Drivers that provide
+ * their own plane handling rather than helper-provided implementations may
+ * still wish to call this function to avoid duplication of error checking
+ * code.
+ *
+ * RETURNS:
+ * Zero if update appears valid, error code on failure
+ */
+int drm_atomic_helper_check_plane_state(struct drm_plane_state *plane_state,
+					const struct drm_crtc_state *crtc_state,
+					const struct drm_rect *clip,
+					int min_scale,
+					int max_scale,
+					bool can_position,
+					bool can_update_disabled)
+{
+	struct drm_framebuffer *fb = plane_state->fb;
+	struct drm_rect *src = &plane_state->src;
+	struct drm_rect *dst = &plane_state->dst;
+	unsigned int rotation = plane_state->rotation;
+	int hscale, vscale;
+
+	WARN_ON(plane_state->crtc && plane_state->crtc != crtc_state->crtc);
+
+	*src = drm_plane_state_src(plane_state);
+	*dst = drm_plane_state_dest(plane_state);
+
+	if (!fb) {
+		plane_state->visible = false;
+		return 0;
+	}
+
+	/* crtc should only be NULL when disabling (i.e., !fb) */
+	if (WARN_ON(!plane_state->crtc)) {
+		plane_state->visible = false;
+		return 0;
+	}
+
+	if (!crtc_state->enable && !can_update_disabled) {
+		DRM_DEBUG_KMS("Cannot update plane of a disabled CRTC.\n");
+		return -EINVAL;
+	}
+
+	drm_rect_rotate(src, fb->width << 16, fb->height << 16, rotation);
+
+	/* Check scaling */
+	hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale);
+	vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale);
+	if (hscale < 0 || vscale < 0) {
+		DRM_DEBUG_KMS("Invalid scaling of plane\n");
+		drm_rect_debug_print("src: ", &plane_state->src, true);
+		drm_rect_debug_print("dst: ", &plane_state->dst, false);
+		return -ERANGE;
+	}
+
+	plane_state->visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale);
+
+	drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation);
+
+	if (!plane_state->visible)
+		/*
+		 * Plane isn't visible; some drivers can handle this
+		 * so we just return success here.  Drivers that can't
+		 * (including those that use the primary plane helper's
+		 * update function) will return an error from their
+		 * update_plane handler.
+		 */
+		return 0;
+
+	if (!can_position && !drm_rect_equals(dst, clip)) {
+		DRM_DEBUG_KMS("Plane must cover entire CRTC\n");
+		drm_rect_debug_print("dst: ", dst, false);
+		drm_rect_debug_print("clip: ", clip, false);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_atomic_helper_check_plane_state);
+
 /**
  * drm_atomic_helper_check_planes - validate state object for planes changes
  * @dev: DRM device
@@ -907,6 +1001,12 @@ disable_outputs(struct drm_device *dev, struct drm_atomic_state *old_state)
  *
  * Drivers can use this for building their own atomic commit if they don't have
  * a pure helper-based modeset implementation.
+ *
+ * Since these updates are not synchronized with lockings, only code paths
+ * called from &drm_mode_config_helper_funcs.atomic_commit_tail can look at the
+ * legacy state filled out by this helper. Defacto this means this helper and
+ * the legacy state pointers are only really useful for transitioning an
+ * existing driver to the atomic world.
  */
 void
 drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev,
@@ -1778,6 +1878,8 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
 		new_crtc_state->event->base.completion = &commit->flip_done;
 		new_crtc_state->event->base.completion_release = release_crtc_commit;
 		drm_crtc_commit_get(commit);
+
+		commit->abort_completion = true;
 	}
 
 	for_each_oldnew_connector_in_state(state, conn, old_conn_state, new_conn_state, i) {
@@ -1787,11 +1889,8 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
 		    !try_wait_for_completion(&old_conn_state->commit->flip_done))
 			return -EBUSY;
 
-		/* commit tracked through new_crtc_state->commit, no need to do it explicitly */
-		if (new_conn_state->crtc)
-			continue;
-
-		commit = crtc_or_fake_commit(state, old_conn_state->crtc);
+		/* Always track connectors explicitly for e.g. link retraining. */
+		commit = crtc_or_fake_commit(state, new_conn_state->crtc ?: old_conn_state->crtc);
 		if (!commit)
 			return -ENOMEM;
 
@@ -1805,10 +1904,7 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
 		    !try_wait_for_completion(&old_plane_state->commit->flip_done))
 			return -EBUSY;
 
-		/*
-		 * Unlike connectors, always track planes explicitly for
-		 * async pageflip support.
-		 */
+		/* Always track planes explicitly for async pageflip support. */
 		commit = crtc_or_fake_commit(state, new_plane_state->crtc ?: old_plane_state->crtc);
 		if (!commit)
 			return -ENOMEM;
@@ -3327,8 +3423,21 @@ EXPORT_SYMBOL(drm_atomic_helper_crtc_duplicate_state);
 void __drm_atomic_helper_crtc_destroy_state(struct drm_crtc_state *state)
 {
 	if (state->commit) {
+		/*
+		 * In the event that a non-blocking commit returns
+		 * -ERESTARTSYS before the commit_tail work is queued, we will
+		 * have an extra reference to the commit object. Release it, if
+		 * the event has not been consumed by the worker.
+		 *
+		 * state->event may be freed, so we can't directly look at
+		 * state->event->base.completion.
+		 */
+		if (state->event && state->commit->abort_completion)
+			drm_crtc_commit_put(state->commit);
+
 		kfree(state->commit->event);
 		state->commit->event = NULL;
+
 		drm_crtc_commit_put(state->commit);
 	}
 
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index aad468d170a7..d9c0f7573905 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -230,6 +230,12 @@ int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
 	if (!dev->master)
 		goto out_unlock;
 
+	if (file_priv->master->lessor != NULL) {
+		DRM_DEBUG_LEASE("Attempt to drop lessee %d as master\n", file_priv->master->lessee_id);
+		ret = -EINVAL;
+		goto out_unlock;
+	}
+
 	ret = 0;
 	drm_drop_master(dev, file_priv);
 out_unlock:
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index 2e5e089dd912..4c62dff14893 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -214,9 +214,11 @@ EXPORT_SYMBOL(drm_rotation_simplify);
  * This function initializes generic mutable zpos property and enables support
  * for it in drm core. Drivers can then attach this property to planes to enable
  * support for configurable planes arrangement during blending operation.
- * Once mutable zpos property has been enabled, the DRM core will automatically
- * calculate &drm_plane_state.normalized_zpos values. Usually min should be set
- * to 0 and max to maximal number of planes for given crtc - 1.
+ * Drivers that attach a mutable zpos property to any plane should call the
+ * drm_atomic_normalize_zpos() helper during their implementation of
+ * &drm_mode_config_funcs.atomic_check(), which will update the normalized zpos
+ * values and store them in &drm_plane_state.normalized_zpos. Usually min
+ * should be set to 0 and max to maximal number of planes for given crtc - 1.
  *
  * If zpos of some planes cannot be changed (like fixed background or
  * cursor/topmost planes), driver should adjust min/max values and assign those
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 9ae236036e32..e6a21e69059c 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -24,6 +24,7 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_encoder.h>
+#include <drm/drm_utils.h>
 
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
@@ -231,6 +232,8 @@ int drm_connector_init(struct drm_device *dev,
 	mutex_init(&connector->mutex);
 	connector->edid_blob_ptr = NULL;
 	connector->status = connector_status_unknown;
+	connector->display_info.panel_orientation =
+		DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
 
 	drm_connector_get_cmdline_mode(connector);
 
@@ -712,6 +715,13 @@ static const struct drm_prop_enum_list drm_aspect_ratio_enum_list[] = {
 	{ DRM_MODE_PICTURE_ASPECT_16_9, "16:9" },
 };
 
+static const struct drm_prop_enum_list drm_panel_orientation_enum_list[] = {
+	{ DRM_MODE_PANEL_ORIENTATION_NORMAL,	"Normal"	},
+	{ DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP,	"Upside Down"	},
+	{ DRM_MODE_PANEL_ORIENTATION_LEFT_UP,	"Left Side Up"	},
+	{ DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,	"Right Side Up"	},
+};
+
 static const struct drm_prop_enum_list drm_dvi_i_select_enum_list[] = {
 	{ DRM_MODE_SUBCONNECTOR_Automatic, "Automatic" }, /* DVI-I and TV-out */
 	{ DRM_MODE_SUBCONNECTOR_DVID,      "DVI-D"     }, /* DVI-I  */
@@ -820,6 +830,18 @@ DRM_ENUM_NAME_FN(drm_get_tv_subconnector_name,
  *
  * CRTC_ID:
  * 	Mode object ID of the &drm_crtc this connector should be connected to.
+ *
+ * Connectors for LCD panels may also have one standardized property:
+ *
+ * panel orientation:
+ *	On some devices the LCD panel is mounted in the casing in such a way
+ *	that the up/top side of the panel does not match with the top side of
+ *	the device. Userspace can use this property to check for this.
+ *	Note that input coordinates from touchscreens (input devices with
+ *	INPUT_PROP_DIRECT) will still map 1:1 to the actual LCD panel
+ *	coordinates, so if userspace rotates the picture to adjust for
+ *	the orientation it must also apply the same transformation to the
+ *	touchscreen input coordinates.
  */
 
 int drm_connector_create_standard_properties(struct drm_device *dev)
@@ -1308,6 +1330,57 @@ void drm_mode_connector_set_link_status_property(struct drm_connector *connector
 }
 EXPORT_SYMBOL(drm_mode_connector_set_link_status_property);
 
+/**
+ * drm_connector_init_panel_orientation_property -
+ *	initialize the connecters panel_orientation property
+ * @connector: connector for which to init the panel-orientation property.
+ * @width: width in pixels of the panel, used for panel quirk detection
+ * @height: height in pixels of the panel, used for panel quirk detection
+ *
+ * This function should only be called for built-in panels, after setting
+ * connector->display_info.panel_orientation first (if known).
+ *
+ * This function will check for platform specific (e.g. DMI based) quirks
+ * overriding display_info.panel_orientation first, then if panel_orientation
+ * is not DRM_MODE_PANEL_ORIENTATION_UNKNOWN it will attach the
+ * "panel orientation" property to the connector.
+ *
+ * Returns:
+ * Zero on success, negative errno on failure.
+ */
+int drm_connector_init_panel_orientation_property(
+	struct drm_connector *connector, int width, int height)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_display_info *info = &connector->display_info;
+	struct drm_property *prop;
+	int orientation_quirk;
+
+	orientation_quirk = drm_get_panel_orientation_quirk(width, height);
+	if (orientation_quirk != DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
+		info->panel_orientation = orientation_quirk;
+
+	if (info->panel_orientation == DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
+		return 0;
+
+	prop = dev->mode_config.panel_orientation_property;
+	if (!prop) {
+		prop = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
+				"panel orientation",
+				drm_panel_orientation_enum_list,
+				ARRAY_SIZE(drm_panel_orientation_enum_list));
+		if (!prop)
+			return -ENOMEM;
+
+		dev->mode_config.panel_orientation_property = prop;
+	}
+
+	drm_object_attach_property(&connector->base, prop,
+				   info->panel_orientation);
+	return 0;
+}
+EXPORT_SYMBOL(drm_connector_init_panel_orientation_property);
+
 int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj,
 				    struct drm_property *property,
 				    uint64_t value)
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index c1807d5754b2..b2482818fee8 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -158,6 +158,14 @@ int drm_debugfs_init(struct drm_minor *minor, int minor_id,
 		}
 	}
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+		ret = drm_framebuffer_debugfs_init(minor);
+		if (ret) {
+			DRM_ERROR("Failed to create framebuffer debugfs file\n");
+			return ret;
+		}
+	}
+
 	if (dev->driver->debugfs_init) {
 		ret = dev->driver->debugfs_init(minor);
 		if (ret) {
diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
index b3d68964b407..adf79be42c1e 100644
--- a/drivers/gpu/drm/drm_dp_helper.c
+++ b/drivers/gpu/drm/drm_dp_helper.c
@@ -1097,7 +1097,6 @@ int drm_dp_aux_register(struct drm_dp_aux *aux)
 	aux->ddc.class = I2C_CLASS_DDC;
 	aux->ddc.owner = THIS_MODULE;
 	aux->ddc.dev.parent = aux->dev;
-	aux->ddc.dev.of_node = aux->dev->of_node;
 
 	strlcpy(aux->ddc.name, aux->name ? aux->name : dev_name(aux->dev),
 		sizeof(aux->ddc.name));
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index a934fd5e7e55..9acc1e157813 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -75,53 +75,6 @@ static bool drm_core_init_complete = false;
 
 static struct dentry *drm_debugfs_root;
 
-#define DRM_PRINTK_FMT "[" DRM_NAME ":%s]%s %pV"
-
-void drm_dev_printk(const struct device *dev, const char *level,
-		    unsigned int category, const char *function_name,
-		    const char *prefix, const char *format, ...)
-{
-	struct va_format vaf;
-	va_list args;
-
-	if (category != DRM_UT_NONE && !(drm_debug & category))
-		return;
-
-	va_start(args, format);
-	vaf.fmt = format;
-	vaf.va = &args;
-
-	if (dev)
-		dev_printk(level, dev, DRM_PRINTK_FMT, function_name, prefix,
-			   &vaf);
-	else
-		printk("%s" DRM_PRINTK_FMT, level, function_name, prefix, &vaf);
-
-	va_end(args);
-}
-EXPORT_SYMBOL(drm_dev_printk);
-
-void drm_printk(const char *level, unsigned int category,
-		const char *format, ...)
-{
-	struct va_format vaf;
-	va_list args;
-
-	if (category != DRM_UT_NONE && !(drm_debug & category))
-		return;
-
-	va_start(args, format);
-	vaf.fmt = format;
-	vaf.va = &args;
-
-	printk("%s" "[" DRM_NAME ":%ps]%s %pV",
-	       level, __builtin_return_address(0),
-	       strcmp(level, KERN_ERR) == 0 ? " *ERROR*" : "", &vaf);
-
-	va_end(args);
-}
-EXPORT_SYMBOL(drm_printk);
-
 /*
  * DRM Minors
  * A DRM device can provide several char-dev interfaces on the DRM-Major. Each
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index cb487148359a..4f751a9d71a3 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -113,6 +113,9 @@ static const struct edid_quirk {
 	/* AEO model 0 reports 8 bpc, but is a 6 bpc panel */
 	{ "AEO", 0, EDID_QUIRK_FORCE_6BPC },
 
+	/* CPT panel of Asus UX303LA reports 8 bpc, but is a 6 bpc panel */
+	{ "CPT", 0x17df, EDID_QUIRK_FORCE_6BPC },
+
 	/* Belinea 10 15 55 */
 	{ "MAX", 1516, EDID_QUIRK_PREFER_LARGE_60 },
 	{ "MAX", 0x77e, EDID_QUIRK_PREFER_LARGE_60 },
@@ -162,6 +165,24 @@ static const struct edid_quirk {
 
 	/* HTC Vive VR Headset */
 	{ "HVR", 0xaa01, EDID_QUIRK_NON_DESKTOP },
+
+	/* Oculus Rift DK1, DK2, and CV1 VR Headsets */
+	{ "OVR", 0x0001, EDID_QUIRK_NON_DESKTOP },
+	{ "OVR", 0x0003, EDID_QUIRK_NON_DESKTOP },
+	{ "OVR", 0x0004, EDID_QUIRK_NON_DESKTOP },
+
+	/* Windows Mixed Reality Headsets */
+	{ "ACR", 0x7fce, EDID_QUIRK_NON_DESKTOP },
+	{ "HPN", 0x3515, EDID_QUIRK_NON_DESKTOP },
+	{ "LEN", 0x0408, EDID_QUIRK_NON_DESKTOP },
+	{ "LEN", 0xb800, EDID_QUIRK_NON_DESKTOP },
+	{ "FUJ", 0x1970, EDID_QUIRK_NON_DESKTOP },
+	{ "DEL", 0x7fce, EDID_QUIRK_NON_DESKTOP },
+	{ "SEC", 0x144a, EDID_QUIRK_NON_DESKTOP },
+	{ "AUS", 0xc102, EDID_QUIRK_NON_DESKTOP },
+
+	/* Sony PlayStation VR Headset */
+	{ "SNY", 0x0704, EDID_QUIRK_NON_DESKTOP },
 };
 
 /*
@@ -3398,6 +3419,7 @@ static int
 do_hdmi_vsdb_modes(struct drm_connector *connector, const u8 *db, u8 len,
 		   const u8 *video_db, u8 video_len)
 {
+	struct drm_display_info *info = &connector->display_info;
 	int modes = 0, offset = 0, i, multi_present = 0, multi_len;
 	u8 vic_len, hdmi_3d_len = 0;
 	u16 mask;
@@ -3525,6 +3547,8 @@ do_hdmi_vsdb_modes(struct drm_connector *connector, const u8 *db, u8 len,
 	}
 
 out:
+	if (modes > 0)
+		info->has_hdmi_infoframe = true;
 	return modes;
 }
 
@@ -3761,8 +3785,8 @@ drm_parse_hdmi_vsdb_audio(struct drm_connector *connector, const u8 *db)
 {
 	u8 len = cea_db_payload_len(db);
 
-	if (len >= 6)
-		connector->eld[5] |= (db[6] >> 7) << 1;  /* Supports_AI */
+	if (len >= 6 && (db[6] & (1 << 7)))
+		connector->eld[DRM_ELD_SAD_COUNT_CONN_TYPE] |= DRM_ELD_SUPPORTS_AI;
 	if (len >= 8) {
 		connector->latency_present[0] = db[8] >> 7;
 		connector->latency_present[1] = (db[8] >> 6) & 1;
@@ -3834,16 +3858,27 @@ void drm_edid_get_monitor_name(struct edid *edid, char *name, int bufsize)
 }
 EXPORT_SYMBOL(drm_edid_get_monitor_name);
 
-/**
+static void clear_eld(struct drm_connector *connector)
+{
+	memset(connector->eld, 0, sizeof(connector->eld));
+
+	connector->latency_present[0] = false;
+	connector->latency_present[1] = false;
+	connector->video_latency[0] = 0;
+	connector->audio_latency[0] = 0;
+	connector->video_latency[1] = 0;
+	connector->audio_latency[1] = 0;
+}
+
+/*
  * drm_edid_to_eld - build ELD from EDID
  * @connector: connector corresponding to the HDMI/DP sink
  * @edid: EDID to parse
  *
  * Fill the ELD (EDID-Like Data) buffer for passing to the audio driver. The
- * Conn_Type, HDCP and Port_ID ELD fields are left for the graphics driver to
- * fill in.
+ * HDCP and Port_ID ELD fields are left for the graphics driver to fill in.
  */
-void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
+static void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 {
 	uint8_t *eld = connector->eld;
 	u8 *cea;
@@ -3852,14 +3887,7 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 	int mnl;
 	int dbl;
 
-	memset(eld, 0, sizeof(connector->eld));
-
-	connector->latency_present[0] = false;
-	connector->latency_present[1] = false;
-	connector->video_latency[0] = 0;
-	connector->audio_latency[0] = 0;
-	connector->video_latency[1] = 0;
-	connector->audio_latency[1] = 0;
+	clear_eld(connector);
 
 	if (!edid)
 		return;
@@ -3870,17 +3898,18 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 		return;
 	}
 
-	mnl = get_monitor_name(edid, eld + 20);
+	mnl = get_monitor_name(edid, &eld[DRM_ELD_MONITOR_NAME_STRING]);
+	DRM_DEBUG_KMS("ELD monitor %s\n", &eld[DRM_ELD_MONITOR_NAME_STRING]);
 
-	eld[4] = (cea[1] << 5) | mnl;
-	DRM_DEBUG_KMS("ELD monitor %s\n", eld + 20);
+	eld[DRM_ELD_CEA_EDID_VER_MNL] = cea[1] << DRM_ELD_CEA_EDID_VER_SHIFT;
+	eld[DRM_ELD_CEA_EDID_VER_MNL] |= mnl;
 
-	eld[0] = 2 << 3;		/* ELD version: 2 */
+	eld[DRM_ELD_VER] = DRM_ELD_VER_CEA861D;
 
-	eld[16] = edid->mfg_id[0];
-	eld[17] = edid->mfg_id[1];
-	eld[18] = edid->prod_code[0];
-	eld[19] = edid->prod_code[1];
+	eld[DRM_ELD_MANUFACTURER_NAME0] = edid->mfg_id[0];
+	eld[DRM_ELD_MANUFACTURER_NAME1] = edid->mfg_id[1];
+	eld[DRM_ELD_PRODUCT_CODE0] = edid->prod_code[0];
+	eld[DRM_ELD_PRODUCT_CODE1] = edid->prod_code[1];
 
 	if (cea_revision(cea) >= 3) {
 		int i, start, end;
@@ -3901,14 +3930,14 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 				/* Audio Data Block, contains SADs */
 				sad_count = min(dbl / 3, 15 - total_sad_count);
 				if (sad_count >= 1)
-					memcpy(eld + 20 + mnl + total_sad_count * 3,
+					memcpy(&eld[DRM_ELD_CEA_SAD(mnl, total_sad_count)],
 					       &db[1], sad_count * 3);
 				total_sad_count += sad_count;
 				break;
 			case SPEAKER_BLOCK:
 				/* Speaker Allocation Data Block */
 				if (dbl >= 1)
-					eld[7] = db[1];
+					eld[DRM_ELD_SPEAKER] = db[1];
 				break;
 			case VENDOR_BLOCK:
 				/* HDMI Vendor-Specific Data Block */
@@ -3920,7 +3949,13 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 			}
 		}
 	}
-	eld[5] |= total_sad_count << 4;
+	eld[DRM_ELD_SAD_COUNT_CONN_TYPE] |= total_sad_count << DRM_ELD_SAD_COUNT_SHIFT;
+
+	if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort ||
+	    connector->connector_type == DRM_MODE_CONNECTOR_eDP)
+		eld[DRM_ELD_SAD_COUNT_CONN_TYPE] |= DRM_ELD_CONN_TYPE_DP;
+	else
+		eld[DRM_ELD_SAD_COUNT_CONN_TYPE] |= DRM_ELD_CONN_TYPE_HDMI;
 
 	eld[DRM_ELD_BASELINE_ELD_LEN] =
 		DIV_ROUND_UP(drm_eld_calc_baseline_block_size(eld), 4);
@@ -3928,7 +3963,6 @@ void drm_edid_to_eld(struct drm_connector *connector, struct edid *edid)
 	DRM_DEBUG_KMS("ELD size %d, SAD count %d\n",
 		      drm_eld_size(eld), total_sad_count);
 }
-EXPORT_SYMBOL(drm_edid_to_eld);
 
 /**
  * drm_edid_to_sad - extracts SADs from EDID
@@ -4238,6 +4272,8 @@ static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector,
 	struct drm_display_info *display = &connector->display_info;
 	struct drm_hdmi_info *hdmi = &display->hdmi;
 
+	display->has_hdmi_infoframe = true;
+
 	if (hf_vsdb[6] & 0x80) {
 		hdmi->scdc.supported = true;
 		if (hf_vsdb[6] & 0x40)
@@ -4413,6 +4449,7 @@ drm_reset_display_info(struct drm_connector *connector)
 	info->cea_rev = 0;
 	info->max_tmds_clock = 0;
 	info->dvi_dual = false;
+	info->has_hdmi_infoframe = false;
 
 	info->non_desktop = 0;
 }
@@ -4433,6 +4470,7 @@ u32 drm_add_display_info(struct drm_connector *connector, const struct edid *edi
 	info->cea_rev = 0;
 	info->max_tmds_clock = 0;
 	info->dvi_dual = false;
+	info->has_hdmi_infoframe = false;
 
 	info->non_desktop = !!(quirks & EDID_QUIRK_NON_DESKTOP);
 
@@ -4634,8 +4672,8 @@ static int add_displayid_detailed_modes(struct drm_connector *connector,
  * @edid: EDID data
  *
  * Add the specified modes to the connector's mode list. Also fills out the
- * &drm_display_info structure in @connector with any information which can be
- * derived from the edid.
+ * &drm_display_info structure and ELD in @connector with any information which
+ * can be derived from the edid.
  *
  * Return: The number of modes added or 0 if we couldn't find any.
  */
@@ -4645,14 +4683,18 @@ int drm_add_edid_modes(struct drm_connector *connector, struct edid *edid)
 	u32 quirks;
 
 	if (edid == NULL) {
+		clear_eld(connector);
 		return 0;
 	}
 	if (!drm_edid_is_valid(edid)) {
+		clear_eld(connector);
 		dev_warn(connector->dev->dev, "%s: EDID invalid.\n",
 			 connector->name);
 		return 0;
 	}
 
+	drm_edid_to_eld(connector, edid);
+
 	/*
 	 * CEA-861-F adds ycbcr capability map block, for HDMI 2.0 sinks.
 	 * To avoid multiple parsing of same block, lets parse that map
@@ -4850,6 +4892,11 @@ EXPORT_SYMBOL(drm_hdmi_avi_infoframe_from_display_mode);
  * @mode: DRM display mode
  * @rgb_quant_range: RGB quantization range (Q)
  * @rgb_quant_range_selectable: Sink support selectable RGB quantization range (QS)
+ * @is_hdmi2_sink: HDMI 2.0 sink, which has different default recommendations
+ *
+ * Note that @is_hdmi2_sink can be derived by looking at the
+ * &drm_scdc.supported flag stored in &drm_hdmi_info.scdc,
+ * &drm_display_info.hdmi, which can be found in &drm_connector.display_info.
  */
 void
 drm_hdmi_avi_infoframe_quant_range(struct hdmi_avi_infoframe *frame,
@@ -4928,6 +4975,7 @@ s3d_structure_from_display_mode(const struct drm_display_mode *mode)
  * drm_hdmi_vendor_infoframe_from_display_mode() - fill an HDMI infoframe with
  * data from a DRM display mode
  * @frame: HDMI vendor infoframe
+ * @connector: the connector
  * @mode: DRM display mode
  *
  * Note that there's is a need to send HDMI vendor infoframes only when using a
@@ -4938,8 +4986,15 @@ s3d_structure_from_display_mode(const struct drm_display_mode *mode)
  */
 int
 drm_hdmi_vendor_infoframe_from_display_mode(struct hdmi_vendor_infoframe *frame,
+					    struct drm_connector *connector,
 					    const struct drm_display_mode *mode)
 {
+	/*
+	 * FIXME: sil-sii8620 doesn't have a connector around when
+	 * we need one, so we have to be prepared for a NULL connector.
+	 */
+	bool has_hdmi_infoframe = connector ?
+		connector->display_info.has_hdmi_infoframe : false;
 	int err;
 	u32 s3d_flags;
 	u8 vic;
@@ -4947,11 +5002,21 @@ drm_hdmi_vendor_infoframe_from_display_mode(struct hdmi_vendor_infoframe *frame,
 	if (!frame || !mode)
 		return -EINVAL;
 
+	if (!has_hdmi_infoframe)
+		return -EINVAL;
+
 	vic = drm_match_hdmi_mode(mode);
 	s3d_flags = mode->flags & DRM_MODE_FLAG_3D_MASK;
 
-	if (!vic && !s3d_flags)
-		return -EINVAL;
+	/*
+	 * Even if it's not absolutely necessary to send the infoframe
+	 * (ie.vic==0 and s3d_struct==0) we will still send it if we
+	 * know that the sink can handle it. This is based on a
+	 * suggestion in HDMI 2.0 Appendix F. Apparently some sinks
+	 * have trouble realizing that they shuld switch from 3D to 2D
+	 * mode if the source simply stops sending the infoframe when
+	 * it wants to switch from 3D to 2D.
+	 */
 
 	if (vic && s3d_flags)
 		return -EINVAL;
@@ -4960,10 +5025,8 @@ drm_hdmi_vendor_infoframe_from_display_mode(struct hdmi_vendor_infoframe *frame,
 	if (err < 0)
 		return err;
 
-	if (vic)
-		frame->vic = vic;
-	else
-		frame->s3d_struct = s3d_structure_from_display_mode(mode);
+	frame->vic = vic;
+	frame->s3d_struct = s3d_structure_from_display_mode(mode);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c
index 0e3c14174d08..186d00adfb5f 100644
--- a/drivers/gpu/drm/drm_fb_cma_helper.c
+++ b/drivers/gpu/drm/drm_fb_cma_helper.c
@@ -23,6 +23,7 @@
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_print.h>
 #include <linux/module.h>
 
 #define DEFAULT_FBDEFIO_DELAY_MS 50
@@ -42,7 +43,7 @@ struct drm_fbdev_cma {
  * callback function to create a cma backed framebuffer.
  *
  * An fbdev framebuffer backed by cma is also available by calling
- * drm_fbdev_cma_init(). drm_fbdev_cma_fini() tears it down.
+ * drm_fb_cma_fbdev_init(). drm_fb_cma_fbdev_fini() tears it down.
  * If the &drm_framebuffer_funcs.dirty callback is set, fb_deferred_io will be
  * set up automatically. &drm_framebuffer_funcs.dirty is called by
  * drm_fb_helper_deferred_io() in process context (&struct delayed_work).
@@ -68,7 +69,7 @@ struct drm_fbdev_cma {
  *
  * Initialize::
  *
- *     fbdev = drm_fbdev_cma_init_with_funcs(dev, 16,
+ *     fbdev = drm_fb_cma_fbdev_init_with_funcs(dev, 16,
  *                                           dev->mode_config.num_crtc,
  *                                           dev->mode_config.num_connector,
  *                                           &driver_fb_funcs);
@@ -130,43 +131,6 @@ dma_addr_t drm_fb_cma_get_gem_addr(struct drm_framebuffer *fb,
 }
 EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_addr);
 
-#ifdef CONFIG_DEBUG_FS
-static void drm_fb_cma_describe(struct drm_framebuffer *fb, struct seq_file *m)
-{
-	int i;
-
-	seq_printf(m, "fb: %dx%d@%4.4s\n", fb->width, fb->height,
-			(char *)&fb->format->format);
-
-	for (i = 0; i < fb->format->num_planes; i++) {
-		seq_printf(m, "   %d: offset=%d pitch=%d, obj: ",
-				i, fb->offsets[i], fb->pitches[i]);
-		drm_gem_cma_describe(drm_fb_cma_get_gem_obj(fb, i), m);
-	}
-}
-
-/**
- * drm_fb_cma_debugfs_show() - Helper to list CMA framebuffer objects
- *			       in debugfs.
- * @m: output file
- * @arg: private data for the callback
- */
-int drm_fb_cma_debugfs_show(struct seq_file *m, void *arg)
-{
-	struct drm_info_node *node = (struct drm_info_node *) m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct drm_framebuffer *fb;
-
-	mutex_lock(&dev->mode_config.fb_lock);
-	drm_for_each_fb(fb, dev)
-		drm_fb_cma_describe(fb, m);
-	mutex_unlock(&dev->mode_config.fb_lock);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(drm_fb_cma_debugfs_show);
-#endif
-
 static int drm_fb_cma_mmap(struct fb_info *info, struct vm_area_struct *vma)
 {
 	return dma_mmap_writecombine(info->device, vma, info->screen_base,
@@ -293,7 +257,7 @@ drm_fbdev_cma_create(struct drm_fb_helper *helper,
 	fbi->screen_size = size;
 	fbi->fix.smem_len = size;
 
-	if (fbdev_cma->fb_funcs->dirty) {
+	if (fb->funcs->dirty) {
 		ret = drm_fbdev_cma_defio_init(fbi, obj);
 		if (ret)
 			goto err_cma_destroy;
@@ -314,6 +278,118 @@ static const struct drm_fb_helper_funcs drm_fb_cma_helper_funcs = {
 	.fb_probe = drm_fbdev_cma_create,
 };
 
+/**
+ * drm_fb_cma_fbdev_init_with_funcs() - Allocate and initialize fbdev emulation
+ * @dev: DRM device
+ * @preferred_bpp: Preferred bits per pixel for the device.
+ *                 @dev->mode_config.preferred_depth is used if this is zero.
+ * @max_conn_count: Maximum number of connectors.
+ *                  @dev->mode_config.num_connector is used if this is zero.
+ * @funcs: Framebuffer functions, in particular a custom dirty() callback.
+ *         Can be NULL.
+ *
+ * Returns:
+ * Zero on success or negative error code on failure.
+ */
+int drm_fb_cma_fbdev_init_with_funcs(struct drm_device *dev,
+	unsigned int preferred_bpp, unsigned int max_conn_count,
+	const struct drm_framebuffer_funcs *funcs)
+{
+	struct drm_fbdev_cma *fbdev_cma;
+	struct drm_fb_helper *fb_helper;
+	int ret;
+
+	if (!preferred_bpp)
+		preferred_bpp = dev->mode_config.preferred_depth;
+	if (!preferred_bpp)
+		preferred_bpp = 32;
+
+	if (!max_conn_count)
+		max_conn_count = dev->mode_config.num_connector;
+
+	fbdev_cma = kzalloc(sizeof(*fbdev_cma), GFP_KERNEL);
+	if (!fbdev_cma)
+		return -ENOMEM;
+
+	fbdev_cma->fb_funcs = funcs;
+	fb_helper = &fbdev_cma->fb_helper;
+
+	drm_fb_helper_prepare(dev, fb_helper, &drm_fb_cma_helper_funcs);
+
+	ret = drm_fb_helper_init(dev, fb_helper, max_conn_count);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev->dev, "Failed to initialize fbdev helper.\n");
+		goto err_free;
+	}
+
+	ret = drm_fb_helper_single_add_all_connectors(fb_helper);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev->dev, "Failed to add connectors.\n");
+		goto err_drm_fb_helper_fini;
+	}
+
+	ret = drm_fb_helper_initial_config(fb_helper, preferred_bpp);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev->dev, "Failed to set fbdev configuration.\n");
+		goto err_drm_fb_helper_fini;
+	}
+
+	return 0;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+err_free:
+	kfree(fbdev_cma);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_fb_cma_fbdev_init_with_funcs);
+
+/**
+ * drm_fb_cma_fbdev_init() - Allocate and initialize fbdev emulation
+ * @dev: DRM device
+ * @preferred_bpp: Preferred bits per pixel for the device.
+ *                 @dev->mode_config.preferred_depth is used if this is zero.
+ * @max_conn_count: Maximum number of connectors.
+ *                  @dev->mode_config.num_connector is used if this is zero.
+ *
+ * Returns:
+ * Zero on success or negative error code on failure.
+ */
+int drm_fb_cma_fbdev_init(struct drm_device *dev, unsigned int preferred_bpp,
+			  unsigned int max_conn_count)
+{
+	return drm_fb_cma_fbdev_init_with_funcs(dev, preferred_bpp,
+						max_conn_count, NULL);
+}
+EXPORT_SYMBOL_GPL(drm_fb_cma_fbdev_init);
+
+/**
+ * drm_fb_cma_fbdev_fini() - Teardown fbdev emulation
+ * @dev: DRM device
+ */
+void drm_fb_cma_fbdev_fini(struct drm_device *dev)
+{
+	struct drm_fb_helper *fb_helper = dev->fb_helper;
+
+	if (!fb_helper)
+		return;
+
+	/* Unregister if it hasn't been done already */
+	if (fb_helper->fbdev && fb_helper->fbdev->dev)
+		drm_fb_helper_unregister_fbi(fb_helper);
+
+	if (fb_helper->fbdev)
+		drm_fbdev_cma_defio_fini(fb_helper->fbdev);
+
+	if (fb_helper->fb)
+		drm_framebuffer_remove(fb_helper->fb);
+
+	drm_fb_helper_fini(fb_helper);
+	kfree(to_fbdev_cma(fb_helper));
+}
+EXPORT_SYMBOL_GPL(drm_fb_cma_fbdev_fini);
+
 /**
  * drm_fbdev_cma_init_with_funcs() - Allocate and initializes a drm_fbdev_cma struct
  * @dev: DRM device
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index e56166334455..035784ddd133 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -41,6 +41,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 
+#include "drm_crtc_internal.h"
 #include "drm_crtc_helper_internal.h"
 
 static bool drm_fbdev_emulation = true;
@@ -65,19 +66,23 @@ static DEFINE_MUTEX(kernel_fb_helper_lock);
  * helper functions used by many drivers to implement the kernel mode setting
  * interfaces.
  *
- * Initialization is done as a four-step process with drm_fb_helper_prepare(),
- * drm_fb_helper_init(), drm_fb_helper_single_add_all_connectors() and
- * drm_fb_helper_initial_config(). Drivers with fancier requirements than the
- * default behaviour can override the third step with their own code.
- * Teardown is done with drm_fb_helper_fini() after the fbdev device is
- * unregisters using drm_fb_helper_unregister_fbi().
+ * Setup fbdev emulation by calling drm_fb_helper_fbdev_setup() and tear it
+ * down by calling drm_fb_helper_fbdev_teardown().
  *
- * At runtime drivers should restore the fbdev console by calling
- * drm_fb_helper_restore_fbdev_mode_unlocked() from their &drm_driver.lastclose
- * callback.  They should also notify the fb helper code from updates to the
- * output configuration by calling drm_fb_helper_hotplug_event(). For easier
- * integration with the output polling code in drm_crtc_helper.c the modeset
- * code provides a &drm_mode_config_funcs.output_poll_changed callback.
+ * Drivers that need to handle connector hotplugging (e.g. dp mst) can't use
+ * the setup helper and will need to do the whole four-step setup process with
+ * drm_fb_helper_prepare(), drm_fb_helper_init(),
+ * drm_fb_helper_single_add_all_connectors(), enable hotplugging and
+ * drm_fb_helper_initial_config() to avoid a possible race window.
+ *
+ * At runtime drivers should restore the fbdev console by using
+ * drm_fb_helper_lastclose() as their &drm_driver.lastclose callback.
+ * They should also notify the fb helper code from updates to the output
+ * configuration by using drm_fb_helper_output_poll_changed() as their
+ * &drm_mode_config_funcs.output_poll_changed callback.
+ *
+ * For suspend/resume consider using drm_mode_config_helper_suspend() and
+ * drm_mode_config_helper_resume() which takes care of fbdev as well.
  *
  * All other functions exported by the fb helper library can be used to
  * implement the fbdev driver interface by the driver.
@@ -102,7 +107,8 @@ static DEFINE_MUTEX(kernel_fb_helper_lock);
  * always run in process context since the fb_*() function could be running in
  * atomic context. If drm_fb_helper_deferred_io() is used as the deferred_io
  * callback it will also schedule dirty_work with the damage collected from the
- * mmap page writes.
+ * mmap page writes. Drivers can use drm_fb_helper_defio_init() to setup
+ * deferred I/O (coupled with drm_fb_helper_fbdev_teardown()).
  */
 
 #define drm_fb_helper_for_each_connector(fbh, i__) \
@@ -150,6 +156,9 @@ int drm_fb_helper_add_one_connector(struct drm_fb_helper *fb_helper,
 {
 	int err;
 
+	if (!fb_helper)
+		return 0;
+
 	mutex_lock(&fb_helper->lock);
 	err = __drm_fb_helper_add_one_connector(fb_helper, connector);
 	mutex_unlock(&fb_helper->lock);
@@ -161,7 +170,7 @@ EXPORT_SYMBOL(drm_fb_helper_add_one_connector);
 /**
  * drm_fb_helper_single_add_all_connectors() - add all connectors to fbdev
  * 					       emulation helper
- * @fb_helper: fbdev initialized with drm_fb_helper_init
+ * @fb_helper: fbdev initialized with drm_fb_helper_init, can be NULL
  *
  * This functions adds all the available connectors for use with the given
  * fb_helper. This is a separate step to allow drivers to freely assign
@@ -174,14 +183,16 @@ EXPORT_SYMBOL(drm_fb_helper_add_one_connector);
  */
 int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper)
 {
-	struct drm_device *dev = fb_helper->dev;
+	struct drm_device *dev;
 	struct drm_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	int i, ret = 0;
 
-	if (!drm_fbdev_emulation)
+	if (!drm_fbdev_emulation || !fb_helper)
 		return 0;
 
+	dev = fb_helper->dev;
+
 	mutex_lock(&fb_helper->lock);
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	drm_for_each_connector_iter(connector, &conn_iter) {
@@ -245,6 +256,9 @@ int drm_fb_helper_remove_one_connector(struct drm_fb_helper *fb_helper,
 {
 	int err;
 
+	if (!fb_helper)
+		return 0;
+
 	mutex_lock(&fb_helper->lock);
 	err = __drm_fb_helper_remove_one_connector(fb_helper, connector);
 	mutex_unlock(&fb_helper->lock);
@@ -350,6 +364,7 @@ EXPORT_SYMBOL(drm_fb_helper_debug_leave);
 static int restore_fbdev_mode_atomic(struct drm_fb_helper *fb_helper, bool active)
 {
 	struct drm_device *dev = fb_helper->dev;
+	struct drm_plane_state *plane_state;
 	struct drm_plane *plane;
 	struct drm_atomic_state *state;
 	int i, ret;
@@ -368,8 +383,6 @@ static int restore_fbdev_mode_atomic(struct drm_fb_helper *fb_helper, bool activ
 retry:
 	plane_mask = 0;
 	drm_for_each_plane(plane, dev) {
-		struct drm_plane_state *plane_state;
-
 		plane_state = drm_atomic_get_plane_state(state, plane);
 		if (IS_ERR(plane_state)) {
 			ret = PTR_ERR(plane_state);
@@ -392,6 +405,11 @@ static int restore_fbdev_mode_atomic(struct drm_fb_helper *fb_helper, bool activ
 
 	for (i = 0; i < fb_helper->crtc_count; i++) {
 		struct drm_mode_set *mode_set = &fb_helper->crtc_info[i].mode_set;
+		struct drm_plane *primary = mode_set->crtc->primary;
+
+		/* Cannot fail as we've already gotten the plane state above */
+		plane_state = drm_atomic_get_new_plane_state(state, primary);
+		plane_state->rotation = fb_helper->crtc_info[i].rotation;
 
 		ret = __drm_atomic_helper_set_config(mode_set, state);
 		if (ret != 0)
@@ -484,7 +502,7 @@ static int restore_fbdev_mode(struct drm_fb_helper *fb_helper)
 
 /**
  * drm_fb_helper_restore_fbdev_mode_unlocked - restore fbdev configuration
- * @fb_helper: fbcon to restore
+ * @fb_helper: driver-allocated fbdev helper, can be NULL
  *
  * This should be called from driver's drm &drm_driver.lastclose callback
  * when implementing an fbcon on top of kms using this helper. This ensures that
@@ -498,7 +516,7 @@ int drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper)
 	bool do_delayed;
 	int ret;
 
-	if (!drm_fbdev_emulation)
+	if (!drm_fbdev_emulation || !fb_helper)
 		return -ENODEV;
 
 	if (READ_ONCE(fb_helper->deferred_setup))
@@ -793,8 +811,10 @@ int drm_fb_helper_init(struct drm_device *dev,
 	struct drm_mode_config *config = &dev->mode_config;
 	int i;
 
-	if (!drm_fbdev_emulation)
+	if (!drm_fbdev_emulation) {
+		dev->fb_helper = fb_helper;
 		return 0;
+	}
 
 	if (!max_conn_count)
 		return -EINVAL;
@@ -821,6 +841,7 @@ int drm_fb_helper_init(struct drm_device *dev,
 		if (!fb_helper->crtc_info[i].mode_set.connectors)
 			goto out_free;
 		fb_helper->crtc_info[i].mode_set.num_connectors = 0;
+		fb_helper->crtc_info[i].rotation = DRM_MODE_ROTATE_0;
 	}
 
 	i = 0;
@@ -829,6 +850,8 @@ int drm_fb_helper_init(struct drm_device *dev,
 		i++;
 	}
 
+	dev->fb_helper = fb_helper;
+
 	return 0;
 out_free:
 	drm_fb_helper_crtc_free(fb_helper);
@@ -883,7 +906,7 @@ EXPORT_SYMBOL(drm_fb_helper_alloc_fbi);
 
 /**
  * drm_fb_helper_unregister_fbi - unregister fb_info framebuffer device
- * @fb_helper: driver-allocated fbdev helper
+ * @fb_helper: driver-allocated fbdev helper, can be NULL
  *
  * A wrapper around unregister_framebuffer, to release the fb_info
  * framebuffer device. This must be called before releasing all resources for
@@ -898,7 +921,7 @@ EXPORT_SYMBOL(drm_fb_helper_unregister_fbi);
 
 /**
  * drm_fb_helper_fini - finialize a &struct drm_fb_helper
- * @fb_helper: driver-allocated fbdev helper
+ * @fb_helper: driver-allocated fbdev helper, can be NULL
  *
  * This cleans up all remaining resources associated with @fb_helper. Must be
  * called after drm_fb_helper_unlink_fbi() was called.
@@ -907,7 +930,12 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper)
 {
 	struct fb_info *info;
 
-	if (!drm_fbdev_emulation || !fb_helper)
+	if (!fb_helper)
+		return;
+
+	fb_helper->dev->fb_helper = NULL;
+
+	if (!drm_fbdev_emulation)
 		return;
 
 	cancel_work_sync(&fb_helper->resume_work);
@@ -937,7 +965,7 @@ EXPORT_SYMBOL(drm_fb_helper_fini);
 
 /**
  * drm_fb_helper_unlink_fbi - wrapper around unlink_framebuffer
- * @fb_helper: driver-allocated fbdev helper
+ * @fb_helper: driver-allocated fbdev helper, can be NULL
  *
  * A wrapper around unlink_framebuffer implemented by fbdev core
  */
@@ -1001,6 +1029,49 @@ void drm_fb_helper_deferred_io(struct fb_info *info,
 }
 EXPORT_SYMBOL(drm_fb_helper_deferred_io);
 
+/**
+ * drm_fb_helper_defio_init - fbdev deferred I/O initialization
+ * @fb_helper: driver-allocated fbdev helper
+ *
+ * This function allocates &fb_deferred_io, sets callback to
+ * drm_fb_helper_deferred_io(), delay to 50ms and calls fb_deferred_io_init().
+ * It should be called from the &drm_fb_helper_funcs->fb_probe callback.
+ * drm_fb_helper_fbdev_teardown() cleans up deferred I/O.
+ *
+ * NOTE: A copy of &fb_ops is made and assigned to &info->fbops. This is done
+ * because fb_deferred_io_cleanup() clears &fbops->fb_mmap and would thereby
+ * affect other instances of that &fb_ops.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_fb_helper_defio_init(struct drm_fb_helper *fb_helper)
+{
+	struct fb_info *info = fb_helper->fbdev;
+	struct fb_deferred_io *fbdefio;
+	struct fb_ops *fbops;
+
+	fbdefio = kzalloc(sizeof(*fbdefio), GFP_KERNEL);
+	fbops = kzalloc(sizeof(*fbops), GFP_KERNEL);
+	if (!fbdefio || !fbops) {
+		kfree(fbdefio);
+		kfree(fbops);
+		return -ENOMEM;
+	}
+
+	info->fbdefio = fbdefio;
+	fbdefio->delay = msecs_to_jiffies(50);
+	fbdefio->deferred_io = drm_fb_helper_deferred_io;
+
+	*fbops = *info->fbops;
+	info->fbops = fbops;
+
+	fb_deferred_io_init(info);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_fb_helper_defio_init);
+
 /**
  * drm_fb_helper_sys_read - wrapper around fb_sys_read
  * @info: fb_info struct pointer
@@ -1138,7 +1209,7 @@ EXPORT_SYMBOL(drm_fb_helper_cfb_imageblit);
 
 /**
  * drm_fb_helper_set_suspend - wrapper around fb_set_suspend
- * @fb_helper: driver-allocated fbdev helper
+ * @fb_helper: driver-allocated fbdev helper, can be NULL
  * @suspend: whether to suspend or resume
  *
  * A wrapper around fb_set_suspend implemented by fbdev core.
@@ -1155,7 +1226,7 @@ EXPORT_SYMBOL(drm_fb_helper_set_suspend);
 /**
  * drm_fb_helper_set_suspend_unlocked - wrapper around fb_set_suspend that also
  *                                      takes the console lock
- * @fb_helper: driver-allocated fbdev helper
+ * @fb_helper: driver-allocated fbdev helper, can be NULL
  * @suspend: whether to suspend or resume
  *
  * A wrapper around fb_set_suspend() that takes the console lock. If the lock
@@ -1825,6 +1896,7 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	if (ret < 0)
 		return ret;
 
+	strcpy(fb_helper->fb->comm, "[fbcon]");
 	return 0;
 }
 
@@ -2342,6 +2414,62 @@ static int drm_pick_crtcs(struct drm_fb_helper *fb_helper,
 	return best_score;
 }
 
+/*
+ * This function checks if rotation is necessary because of panel orientation
+ * and if it is, if it is supported.
+ * If rotation is necessary and supported, its gets set in fb_crtc.rotation.
+ * If rotation is necessary but not supported, a DRM_MODE_ROTATE_* flag gets
+ * or-ed into fb_helper->sw_rotations. In drm_setup_crtcs_fb() we check if only
+ * one bit is set and then we set fb_info.fbcon_rotate_hint to make fbcon do
+ * the unsupported rotation.
+ */
+static void drm_setup_crtc_rotation(struct drm_fb_helper *fb_helper,
+				    struct drm_fb_helper_crtc *fb_crtc,
+				    struct drm_connector *connector)
+{
+	struct drm_plane *plane = fb_crtc->mode_set.crtc->primary;
+	uint64_t valid_mask = 0;
+	int i, rotation;
+
+	fb_crtc->rotation = DRM_MODE_ROTATE_0;
+
+	switch (connector->display_info.panel_orientation) {
+	case DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP:
+		rotation = DRM_MODE_ROTATE_180;
+		break;
+	case DRM_MODE_PANEL_ORIENTATION_LEFT_UP:
+		rotation = DRM_MODE_ROTATE_90;
+		break;
+	case DRM_MODE_PANEL_ORIENTATION_RIGHT_UP:
+		rotation = DRM_MODE_ROTATE_270;
+		break;
+	default:
+		rotation = DRM_MODE_ROTATE_0;
+	}
+
+	/*
+	 * TODO: support 90 / 270 degree hardware rotation,
+	 * depending on the hardware this may require the framebuffer
+	 * to be in a specific tiling format.
+	 */
+	if (rotation != DRM_MODE_ROTATE_180 || !plane->rotation_property) {
+		fb_helper->sw_rotations |= rotation;
+		return;
+	}
+
+	for (i = 0; i < plane->rotation_property->num_values; i++)
+		valid_mask |= (1ULL << plane->rotation_property->values[i]);
+
+	if (!(rotation & valid_mask)) {
+		fb_helper->sw_rotations |= rotation;
+		return;
+	}
+
+	fb_crtc->rotation = rotation;
+	/* Rotating in hardware, fbcon should not rotate */
+	fb_helper->sw_rotations |= DRM_MODE_ROTATE_0;
+}
+
 static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 			    u32 width, u32 height)
 {
@@ -2401,6 +2529,7 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 		drm_fb_helper_modeset_release(fb_helper,
 					      &fb_helper->crtc_info[i].mode_set);
 
+	fb_helper->sw_rotations = 0;
 	drm_fb_helper_for_each_connector(fb_helper, i) {
 		struct drm_display_mode *mode = modes[i];
 		struct drm_fb_helper_crtc *fb_crtc = crtcs[i];
@@ -2420,6 +2549,7 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 			modeset->mode = drm_mode_duplicate(dev,
 							   fb_crtc->desired_mode);
 			drm_connector_get(connector);
+			drm_setup_crtc_rotation(fb_helper, fb_crtc, connector);
 			modeset->connectors[modeset->num_connectors++] = connector;
 			modeset->x = offset->x;
 			modeset->y = offset->y;
@@ -2461,6 +2591,28 @@ static void drm_setup_crtcs_fb(struct drm_fb_helper *fb_helper)
 		}
 	}
 	mutex_unlock(&fb_helper->dev->mode_config.mutex);
+
+	switch (fb_helper->sw_rotations) {
+	case DRM_MODE_ROTATE_0:
+		info->fbcon_rotate_hint = FB_ROTATE_UR;
+		break;
+	case DRM_MODE_ROTATE_90:
+		info->fbcon_rotate_hint = FB_ROTATE_CCW;
+		break;
+	case DRM_MODE_ROTATE_180:
+		info->fbcon_rotate_hint = FB_ROTATE_UD;
+		break;
+	case DRM_MODE_ROTATE_270:
+		info->fbcon_rotate_hint = FB_ROTATE_CW;
+		break;
+	default:
+		/*
+		 * Multiple bits are set / multiple rotations requested
+		 * fbcon cannot handle separate rotation settings per
+		 * output, so fallback to unrotated.
+		 */
+		info->fbcon_rotate_hint = FB_ROTATE_UR;
+	}
 }
 
 /* Note: Drops fb_helper->lock before returning. */
@@ -2576,7 +2728,7 @@ EXPORT_SYMBOL(drm_fb_helper_initial_config);
 /**
  * drm_fb_helper_hotplug_event - respond to a hotplug notification by
  *                               probing all the outputs attached to the fb
- * @fb_helper: the drm_fb_helper
+ * @fb_helper: driver-allocated fbdev helper, can be NULL
  *
  * Scan the connectors attached to the fb_helper and try to put together a
  * setup after notification of a change in output configuration.
@@ -2598,7 +2750,7 @@ int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 {
 	int err = 0;
 
-	if (!drm_fbdev_emulation)
+	if (!drm_fbdev_emulation || !fb_helper)
 		return 0;
 
 	mutex_lock(&fb_helper->lock);
@@ -2626,6 +2778,148 @@ int drm_fb_helper_hotplug_event(struct drm_fb_helper *fb_helper)
 }
 EXPORT_SYMBOL(drm_fb_helper_hotplug_event);
 
+/**
+ * drm_fb_helper_fbdev_setup() - Setup fbdev emulation
+ * @dev: DRM device
+ * @fb_helper: fbdev helper structure to set up
+ * @funcs: fbdev helper functions
+ * @preferred_bpp: Preferred bits per pixel for the device.
+ *                 @dev->mode_config.preferred_depth is used if this is zero.
+ * @max_conn_count: Maximum number of connectors.
+ *                  @dev->mode_config.num_connector is used if this is zero.
+ *
+ * This function sets up fbdev emulation and registers fbdev for access by
+ * userspace. If all connectors are disconnected, setup is deferred to the next
+ * time drm_fb_helper_hotplug_event() is called.
+ * The caller must to provide a &drm_fb_helper_funcs->fb_probe callback
+ * function.
+ *
+ * See also: drm_fb_helper_initial_config()
+ *
+ * Returns:
+ * Zero on success or negative error code on failure.
+ */
+int drm_fb_helper_fbdev_setup(struct drm_device *dev,
+			      struct drm_fb_helper *fb_helper,
+			      const struct drm_fb_helper_funcs *funcs,
+			      unsigned int preferred_bpp,
+			      unsigned int max_conn_count)
+{
+	int ret;
+
+	if (!preferred_bpp)
+		preferred_bpp = dev->mode_config.preferred_depth;
+	if (!preferred_bpp)
+		preferred_bpp = 32;
+
+	if (!max_conn_count)
+		max_conn_count = dev->mode_config.num_connector;
+	if (!max_conn_count) {
+		DRM_DEV_ERROR(dev->dev, "No connectors\n");
+		return -EINVAL;
+	}
+
+	drm_fb_helper_prepare(dev, fb_helper, funcs);
+
+	ret = drm_fb_helper_init(dev, fb_helper, max_conn_count);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev->dev, "Failed to initialize fbdev helper\n");
+		return ret;
+	}
+
+	ret = drm_fb_helper_single_add_all_connectors(fb_helper);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev->dev, "Failed to add connectors\n");
+		goto err_drm_fb_helper_fini;
+	}
+
+	if (!drm_drv_uses_atomic_modeset(dev))
+		drm_helper_disable_unused_functions(dev);
+
+	ret = drm_fb_helper_initial_config(fb_helper, preferred_bpp);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev->dev, "Failed to set fbdev configuration\n");
+		goto err_drm_fb_helper_fini;
+	}
+
+	return 0;
+
+err_drm_fb_helper_fini:
+	drm_fb_helper_fini(fb_helper);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_fb_helper_fbdev_setup);
+
+/**
+ * drm_fb_helper_fbdev_teardown - Tear down fbdev emulation
+ * @dev: DRM device
+ *
+ * This function unregisters fbdev if not already done and cleans up the
+ * associated resources including the &drm_framebuffer.
+ * The driver is responsible for freeing the &drm_fb_helper structure which is
+ * stored in &drm_device->fb_helper. Do note that this pointer has been cleared
+ * when this function returns.
+ *
+ * In order to support device removal/unplug while file handles are still open,
+ * drm_fb_helper_unregister_fbi() should be called on device removal and
+ * drm_fb_helper_fbdev_teardown() in the &drm_driver->release callback when
+ * file handles are closed.
+ */
+void drm_fb_helper_fbdev_teardown(struct drm_device *dev)
+{
+	struct drm_fb_helper *fb_helper = dev->fb_helper;
+	struct fb_ops *fbops = NULL;
+
+	if (!fb_helper)
+		return;
+
+	/* Unregister if it hasn't been done already */
+	if (fb_helper->fbdev && fb_helper->fbdev->dev)
+		drm_fb_helper_unregister_fbi(fb_helper);
+
+	if (fb_helper->fbdev && fb_helper->fbdev->fbdefio) {
+		fb_deferred_io_cleanup(fb_helper->fbdev);
+		kfree(fb_helper->fbdev->fbdefio);
+		fbops = fb_helper->fbdev->fbops;
+	}
+
+	drm_fb_helper_fini(fb_helper);
+	kfree(fbops);
+
+	if (fb_helper->fb)
+		drm_framebuffer_remove(fb_helper->fb);
+}
+EXPORT_SYMBOL(drm_fb_helper_fbdev_teardown);
+
+/**
+ * drm_fb_helper_lastclose - DRM driver lastclose helper for fbdev emulation
+ * @dev: DRM device
+ *
+ * This function can be used as the &drm_driver->lastclose callback for drivers
+ * that only need to call drm_fb_helper_restore_fbdev_mode_unlocked().
+ */
+void drm_fb_helper_lastclose(struct drm_device *dev)
+{
+	drm_fb_helper_restore_fbdev_mode_unlocked(dev->fb_helper);
+}
+EXPORT_SYMBOL(drm_fb_helper_lastclose);
+
+/**
+ * drm_fb_helper_output_poll_changed - DRM mode config \.output_poll_changed
+ *                                     helper for fbdev emulation
+ * @dev: DRM device
+ *
+ * This function can be used as the
+ * &drm_mode_config_funcs.output_poll_changed callback for drivers that only
+ * need to call drm_fb_helper_hotplug_event().
+ */
+void drm_fb_helper_output_poll_changed(struct drm_device *dev)
+{
+	drm_fb_helper_hotplug_event(dev->fb_helper);
+}
+EXPORT_SYMBOL(drm_fb_helper_output_poll_changed);
+
 /* The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EXPERT)
  * but the module doesn't depend on any fb console symbols.  At least
  * attempt to load fbcon to avoid leaving the system without a usable console.
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index b3c6e997ccdb..e394799979a6 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -559,15 +559,15 @@ EXPORT_SYMBOL(drm_read);
  *
  * Mask of POLL flags indicating the current status of the file.
  */
-unsigned int drm_poll(struct file *filp, struct poll_table_struct *wait)
+__poll_t drm_poll(struct file *filp, struct poll_table_struct *wait)
 {
 	struct drm_file *file_priv = filp->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(filp, &file_priv->event_wait, wait);
 
 	if (!list_empty(&file_priv->event_list))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index 279c1035c12d..2dc5e8bed172 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -25,7 +25,9 @@
 #include <drm/drm_auth.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_atomic.h>
+#include <drm/drm_print.h>
 
+#include "drm_internal.h"
 #include "drm_crtc_internal.h"
 
 /**
@@ -78,11 +80,12 @@ int drm_framebuffer_check_src_coords(uint32_t src_x, uint32_t src_y,
 	    src_h > fb_height ||
 	    src_y > fb_height - src_h) {
 		DRM_DEBUG_KMS("Invalid source coordinates "
-			      "%u.%06ux%u.%06u+%u.%06u+%u.%06u\n",
+			      "%u.%06ux%u.%06u+%u.%06u+%u.%06u (fb %ux%u)\n",
 			      src_w >> 16, ((src_w & 0xffff) * 15625) >> 10,
 			      src_h >> 16, ((src_h & 0xffff) * 15625) >> 10,
 			      src_x >> 16, ((src_x & 0xffff) * 15625) >> 10,
-			      src_y >> 16, ((src_y & 0xffff) * 15625) >> 10);
+			      src_y >> 16, ((src_y & 0xffff) * 15625) >> 10,
+			      fb->width, fb->height);
 		return -ENOSPC;
 	}
 
@@ -118,6 +121,10 @@ int drm_mode_addfb(struct drm_device *dev,
 	r.pixel_format = drm_mode_legacy_fb_format(or->bpp, or->depth);
 	r.handles[0] = or->handle;
 
+	if (r.pixel_format == DRM_FORMAT_XRGB2101010 &&
+	    dev->driver->driver_features & DRIVER_PREFER_XBGR_30BPP)
+		r.pixel_format = DRM_FORMAT_XBGR2101010;
+
 	ret = drm_mode_addfb2(dev, &r, file_priv);
 	if (ret)
 		return ret;
@@ -454,6 +461,12 @@ int drm_mode_getfb(struct drm_device *dev,
 	if (!fb)
 		return -ENOENT;
 
+	/* Multi-planar framebuffers need getfb2. */
+	if (fb->format->num_planes > 1) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	r->height = fb->height;
 	r->width = fb->width;
 	r->depth = fb->format->depth;
@@ -477,6 +490,7 @@ int drm_mode_getfb(struct drm_device *dev,
 		ret = -ENODEV;
 	}
 
+out:
 	drm_framebuffer_put(fb);
 
 	return ret;
@@ -661,6 +675,7 @@ int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb,
 	INIT_LIST_HEAD(&fb->filp_head);
 
 	fb->funcs = funcs;
+	strcpy(fb->comm, current->comm);
 
 	ret = __drm_mode_object_add(dev, &fb->base, DRM_MODE_OBJECT_FB,
 				    false, drm_framebuffer_free);
@@ -766,14 +781,18 @@ static int atomic_remove_fb(struct drm_framebuffer *fb)
 	struct drm_plane *plane;
 	struct drm_connector *conn;
 	struct drm_connector_state *conn_state;
-	int i, ret = 0;
+	int i, ret;
 	unsigned plane_mask;
+	bool disable_crtcs = false;
 
-	state = drm_atomic_state_alloc(dev);
-	if (!state)
-		return -ENOMEM;
-
+retry_disable:
 	drm_modeset_acquire_init(&ctx, 0);
+
+	state = drm_atomic_state_alloc(dev);
+	if (!state) {
+		ret = -ENOMEM;
+		goto out;
+	}
 	state->acquire_ctx = &ctx;
 
 retry:
@@ -794,7 +813,7 @@ static int atomic_remove_fb(struct drm_framebuffer *fb)
 			goto unlock;
 		}
 
-		if (plane_state->crtc->primary == plane) {
+		if (disable_crtcs && plane_state->crtc->primary == plane) {
 			struct drm_crtc_state *crtc_state;
 
 			crtc_state = drm_atomic_get_existing_crtc_state(state, plane_state->crtc);
@@ -819,6 +838,7 @@ static int atomic_remove_fb(struct drm_framebuffer *fb)
 		plane->old_fb = plane->fb;
 	}
 
+	/* This list is only filled when disable_crtcs is set. */
 	for_each_new_connector_in_state(state, conn, conn_state, i) {
 		ret = drm_atomic_set_crtc_for_connector(conn_state, NULL);
 
@@ -841,9 +861,15 @@ static int atomic_remove_fb(struct drm_framebuffer *fb)
 
 	drm_atomic_state_put(state);
 
+out:
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
 
+	if (ret == -EINVAL && !disable_crtcs) {
+		disable_crtcs = true;
+		goto retry_disable;
+	}
+
 	return ret;
 }
 
@@ -957,3 +983,61 @@ int drm_framebuffer_plane_height(int height,
 	return fb_plane_height(height, fb->format, plane);
 }
 EXPORT_SYMBOL(drm_framebuffer_plane_height);
+
+void drm_framebuffer_print_info(struct drm_printer *p, unsigned int indent,
+				const struct drm_framebuffer *fb)
+{
+	struct drm_format_name_buf format_name;
+	unsigned int i;
+
+	drm_printf_indent(p, indent, "allocated by = %s\n", fb->comm);
+	drm_printf_indent(p, indent, "refcount=%u\n",
+			  drm_framebuffer_read_refcount(fb));
+	drm_printf_indent(p, indent, "format=%s\n",
+			  drm_get_format_name(fb->format->format, &format_name));
+	drm_printf_indent(p, indent, "modifier=0x%llx\n", fb->modifier);
+	drm_printf_indent(p, indent, "size=%ux%u\n", fb->width, fb->height);
+	drm_printf_indent(p, indent, "layers:\n");
+
+	for (i = 0; i < fb->format->num_planes; i++) {
+		drm_printf_indent(p, indent + 1, "size[%u]=%dx%d\n", i,
+				  drm_framebuffer_plane_width(fb->width, fb, i),
+				  drm_framebuffer_plane_height(fb->height, fb, i));
+		drm_printf_indent(p, indent + 1, "pitch[%u]=%u\n", i, fb->pitches[i]);
+		drm_printf_indent(p, indent + 1, "offset[%u]=%u\n", i, fb->offsets[i]);
+		drm_printf_indent(p, indent + 1, "obj[%u]:%s\n", i,
+				  fb->obj[i] ? "" : "(null)");
+		if (fb->obj[i])
+			drm_gem_print_info(p, indent + 2, fb->obj[i]);
+	}
+}
+
+#ifdef CONFIG_DEBUG_FS
+static int drm_framebuffer_info(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct drm_printer p = drm_seq_file_printer(m);
+	struct drm_framebuffer *fb;
+
+	mutex_lock(&dev->mode_config.fb_lock);
+	drm_for_each_fb(fb, dev) {
+		drm_printf(&p, "framebuffer[%u]:\n", fb->base.id);
+		drm_framebuffer_print_info(&p, 1, fb);
+	}
+	mutex_unlock(&dev->mode_config.fb_lock);
+
+	return 0;
+}
+
+static const struct drm_info_list drm_framebuffer_debugfs_list[] = {
+	{ "framebuffer", drm_framebuffer_info, 0 },
+};
+
+int drm_framebuffer_debugfs_init(struct drm_minor *minor)
+{
+	return drm_debugfs_create_files(drm_framebuffer_debugfs_list,
+				ARRAY_SIZE(drm_framebuffer_debugfs_list),
+				minor->debugfs_root, minor);
+}
+#endif
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 55d6182555c7..01f8d9481211 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -40,6 +40,7 @@
 #include <drm/drmP.h>
 #include <drm/drm_vma_manager.h>
 #include <drm/drm_gem.h>
+#include <drm/drm_print.h>
 #include "drm_internal.h"
 
 /** @file drm_gem.c
@@ -348,7 +349,7 @@ EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
  * @file: drm file-private structure to remove the dumb handle from
  * @dev: corresponding drm_device
  * @handle: the dumb handle to remove
- * 
+ *
  * This implements the &drm_driver.dumb_destroy kms driver callback for drivers
  * which use gem to manage their backing storage.
  */
@@ -365,7 +366,7 @@ EXPORT_SYMBOL(drm_gem_dumb_destroy);
  * @file_priv: drm file-private structure to register the handle for
  * @obj: object to register
  * @handlep: pointer to return the created handle to the caller
- * 
+ *
  * This expects the &drm_device.object_name_lock to be held already and will
  * drop it before returning. Used to avoid races in establishing new handles
  * when importing an object from either an flink name or a dma-buf.
@@ -1040,3 +1041,19 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 	return ret;
 }
 EXPORT_SYMBOL(drm_gem_mmap);
+
+void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
+			const struct drm_gem_object *obj)
+{
+	drm_printf_indent(p, indent, "name=%d\n", obj->name);
+	drm_printf_indent(p, indent, "refcount=%u\n",
+			  kref_read(&obj->refcount));
+	drm_printf_indent(p, indent, "start=%08lx\n",
+			  drm_vma_node_start(&obj->vma_node));
+	drm_printf_indent(p, indent, "size=%zu\n", obj->size);
+	drm_printf_indent(p, indent, "imported=%s\n",
+			  obj->import_attach ? "yes" : "no");
+
+	if (obj->dev->driver->gem_print_info)
+		obj->dev->driver->gem_print_info(p, indent, obj);
+}
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
index 020e7668dfab..80a5115c3846 100644
--- a/drivers/gpu/drm/drm_gem_cma_helper.c
+++ b/drivers/gpu/drm/drm_gem_cma_helper.c
@@ -397,31 +397,24 @@ unsigned long drm_gem_cma_get_unmapped_area(struct file *filp,
 EXPORT_SYMBOL_GPL(drm_gem_cma_get_unmapped_area);
 #endif
 
-#ifdef CONFIG_DEBUG_FS
 /**
- * drm_gem_cma_describe - describe a CMA GEM object for debugfs
- * @cma_obj: CMA GEM object
- * @m: debugfs file handle
+ * drm_gem_cma_print_info() - Print &drm_gem_cma_object info for debugfs
+ * @p: DRM printer
+ * @indent: Tab indentation level
+ * @obj: GEM object
  *
- * This function can be used to dump a human-readable representation of the
- * CMA GEM object into a synthetic file.
+ * This function can be used as the &drm_driver->gem_print_info callback.
+ * It prints paddr and vaddr for use in e.g. debugfs output.
  */
-void drm_gem_cma_describe(struct drm_gem_cma_object *cma_obj,
-			  struct seq_file *m)
+void drm_gem_cma_print_info(struct drm_printer *p, unsigned int indent,
+			    const struct drm_gem_object *obj)
 {
-	struct drm_gem_object *obj = &cma_obj->base;
-	uint64_t off;
-
-	off = drm_vma_node_start(&obj->vma_node);
-
-	seq_printf(m, "%2d (%2d) %08llx %pad %p %zu",
-			obj->name, kref_read(&obj->refcount),
-			off, &cma_obj->paddr, cma_obj->vaddr, obj->size);
+	const struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
 
-	seq_printf(m, "\n");
+	drm_printf_indent(p, indent, "paddr=%pad\n", &cma_obj->paddr);
+	drm_printf_indent(p, indent, "vaddr=%p\n", cma_obj->vaddr);
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_describe);
-#endif
+EXPORT_SYMBOL(drm_gem_cma_print_info);
 
 /**
  * drm_gem_cma_prime_get_sg_table - provide a scatter/gather table of pinned
@@ -482,8 +475,26 @@ drm_gem_cma_prime_import_sg_table(struct drm_device *dev,
 {
 	struct drm_gem_cma_object *cma_obj;
 
-	if (sgt->nents != 1)
-		return ERR_PTR(-EINVAL);
+	if (sgt->nents != 1) {
+		/* check if the entries in the sg_table are contiguous */
+		dma_addr_t next_addr = sg_dma_address(sgt->sgl);
+		struct scatterlist *s;
+		unsigned int i;
+
+		for_each_sg(sgt->sgl, s, sgt->nents, i) {
+			/*
+			 * sg_dma_address(s) is only valid for entries
+			 * that have sg_dma_len(s) != 0
+			 */
+			if (!sg_dma_len(s))
+				continue;
+
+			if (sg_dma_address(s) != next_addr)
+				return ERR_PTR(-EINVAL);
+
+			next_addr = sg_dma_address(s) + sg_dma_len(s);
+		}
+	}
 
 	/* Create a CMA GEM buffer. */
 	cma_obj = __drm_gem_cma_create(dev, attach->dmabuf->size);
diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
index aa8cb9bfa499..4d682a6e8bcb 100644
--- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c
+++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
@@ -272,7 +272,8 @@ EXPORT_SYMBOL_GPL(drm_gem_fb_prepare_fb);
  * @sizes: fbdev size description
  * @pitch_align: Optional pitch alignment
  * @obj: GEM object backing the framebuffer
- * @funcs: vtable to be used for the new framebuffer object
+ * @funcs: Optional vtable to be used for the new framebuffer object when the
+ *         dirty callback is needed.
  *
  * This function creates a framebuffer from a &drm_fb_helper_surface_size
  * description for use in the &drm_fb_helper_funcs.fb_probe callback.
@@ -300,6 +301,9 @@ drm_gem_fbdev_fb_create(struct drm_device *dev,
 	if (obj->size < mode_cmd.pitches[0] * mode_cmd.height)
 		return ERR_PTR(-EINVAL);
 
+	if (!funcs)
+		funcs = &drm_gem_fb_funcs;
+
 	return drm_gem_fb_alloc(dev, &mode_cmd, &obj, 1, funcs);
 }
 EXPORT_SYMBOL(drm_gem_fbdev_fb_create);
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index c9d5a6cd4d41..b72242e93ea4 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -111,6 +111,8 @@ int drm_gem_open_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *file_priv);
 void drm_gem_open(struct drm_device *dev, struct drm_file *file_private);
 void drm_gem_release(struct drm_device *dev, struct drm_file *file_private);
+void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
+			const struct drm_gem_object *obj);
 
 /* drm_debugfs.c drm_debugfs_crc.c */
 #if defined(CONFIG_DEBUG_FS)
@@ -178,3 +180,8 @@ int drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_private);
 int drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_private);
+
+/* drm_framebuffer.c */
+void drm_framebuffer_print_info(struct drm_printer *p, unsigned int indent,
+				const struct drm_framebuffer *fb);
+int drm_framebuffer_debugfs_init(struct drm_minor *minor);
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index c3c79ee6119e..89eef1bb4ddc 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -92,7 +92,7 @@
  * some basic allocator dumpers for debugging.
  *
  * Note that this range allocator is not thread-safe, drivers need to protect
- * modifications with their on locking. The idea behind this is that for a full
+ * modifications with their own locking. The idea behind this is that for a full
  * memory manager additional data needs to be protected anyway, hence internal
  * locking would be fully redundant.
  */
@@ -836,9 +836,24 @@ struct drm_mm_node *drm_mm_scan_color_evict(struct drm_mm_scan *scan)
 	if (!mm->color_adjust)
 		return NULL;
 
-	hole = list_first_entry(&mm->hole_stack, typeof(*hole), hole_stack);
-	hole_start = __drm_mm_hole_node_start(hole);
-	hole_end = hole_start + hole->hole_size;
+	/*
+	 * The hole found during scanning should ideally be the first element
+	 * in the hole_stack list, but due to side-effects in the driver it
+	 * may not be.
+	 */
+	list_for_each_entry(hole, &mm->hole_stack, hole_stack) {
+		hole_start = __drm_mm_hole_node_start(hole);
+		hole_end = hole_start + hole->hole_size;
+
+		if (hole_start <= scan->hit_start &&
+		    hole_end >= scan->hit_end)
+			break;
+	}
+
+	/* We should only be called after we found the hole previously */
+	DRM_MM_BUG_ON(&hole->hole_stack == &mm->hole_stack);
+	if (unlikely(&hole->hole_stack == &mm->hole_stack))
+		return NULL;
 
 	DRM_MM_BUG_ON(hole_start > scan->hit_start);
 	DRM_MM_BUG_ON(hole_end < scan->hit_end);
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index 256de7313612..e5c653357024 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -472,6 +472,9 @@ void drm_mode_config_cleanup(struct drm_device *dev)
 	 */
 	WARN_ON(!list_empty(&dev->mode_config.fb_list));
 	list_for_each_entry_safe(fb, fbt, &dev->mode_config.fb_list, head) {
+		struct drm_printer p = drm_debug_printer("[leaked fb]");
+		drm_printf(&p, "framebuffer[%u]:\n", fb->base.id);
+		drm_framebuffer_print_info(&p, 1, fb);
 		drm_framebuffer_free(&fb->base.refcount);
 	}
 
diff --git a/drivers/gpu/drm/drm_modeset_helper.c b/drivers/gpu/drm/drm_modeset_helper.c
index 9cb1eede0b4d..f1c24ab0ef09 100644
--- a/drivers/gpu/drm/drm_modeset_helper.c
+++ b/drivers/gpu/drm/drm_modeset_helper.c
@@ -20,6 +20,9 @@
  * OF THIS SOFTWARE.
  */
 
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_plane_helper.h>
 
@@ -156,3 +159,76 @@ int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 					 NULL);
 }
 EXPORT_SYMBOL(drm_crtc_init);
+
+/**
+ * drm_mode_config_helper_suspend - Modeset suspend helper
+ * @dev: DRM device
+ *
+ * This helper function takes care of suspending the modeset side. It disables
+ * output polling if initialized, suspends fbdev if used and finally calls
+ * drm_atomic_helper_suspend().
+ * If suspending fails, fbdev and polling is re-enabled.
+ *
+ * Returns:
+ * Zero on success, negative error code on error.
+ *
+ * See also:
+ * drm_kms_helper_poll_disable() and drm_fb_helper_set_suspend_unlocked().
+ */
+int drm_mode_config_helper_suspend(struct drm_device *dev)
+{
+	struct drm_atomic_state *state;
+
+	if (!dev)
+		return 0;
+
+	drm_kms_helper_poll_disable(dev);
+	drm_fb_helper_set_suspend_unlocked(dev->fb_helper, 1);
+	state = drm_atomic_helper_suspend(dev);
+	if (IS_ERR(state)) {
+		drm_fb_helper_set_suspend_unlocked(dev->fb_helper, 0);
+		drm_kms_helper_poll_enable(dev);
+		return PTR_ERR(state);
+	}
+
+	dev->mode_config.suspend_state = state;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_config_helper_suspend);
+
+/**
+ * drm_mode_config_helper_resume - Modeset resume helper
+ * @dev: DRM device
+ *
+ * This helper function takes care of resuming the modeset side. It calls
+ * drm_atomic_helper_resume(), resumes fbdev if used and enables output polling
+ * if initiaized.
+ *
+ * Returns:
+ * Zero on success, negative error code on error.
+ *
+ * See also:
+ * drm_fb_helper_set_suspend_unlocked() and drm_kms_helper_poll_enable().
+ */
+int drm_mode_config_helper_resume(struct drm_device *dev)
+{
+	int ret;
+
+	if (!dev)
+		return 0;
+
+	if (WARN_ON(!dev->mode_config.suspend_state))
+		return -EINVAL;
+
+	ret = drm_atomic_helper_resume(dev, dev->mode_config.suspend_state);
+	if (ret)
+		DRM_ERROR("Failed to resume (%d)\n", ret);
+	dev->mode_config.suspend_state = NULL;
+
+	drm_fb_helper_set_suspend_unlocked(dev->fb_helper, 0);
+	drm_kms_helper_poll_enable(dev);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_mode_config_helper_resume);
diff --git a/drivers/video/fbdev/core/fbcon_dmi_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c
similarity index 50%
rename from drivers/video/fbdev/core/fbcon_dmi_quirks.c
rename to drivers/gpu/drm/drm_panel_orientation_quirks.c
index 6904e47d1e51..1f2af707ce03 100644
--- a/drivers/video/fbdev/core/fbcon_dmi_quirks.c
+++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c
@@ -1,17 +1,18 @@
+/* SPDX-License-Identifier: MIT */
 /*
- *  fbcon_dmi_quirks.c -- DMI based quirk detection for fbcon
+ * drm_panel_orientation_quirks.c -- Quirks for non-normal panel orientation
  *
- *	Copyright (C) 2017 Hans de Goede <hdegoede@redhat.com>
+ * Copyright (C) 2017 Hans de Goede <hdegoede@redhat.com>
  *
- *  This file is subject to the terms and conditions of the GNU General Public
- *  License.  See the file COPYING in the main directory of this archive for
- *  more details.
+ * Note the quirks in this file are shared with fbdev/efifb and as such
+ * must not depend on other drm code.
  */
 
 #include <linux/dmi.h>
-#include <linux/fb.h>
-#include <linux/kernel.h>
-#include "fbcon.h"
+#include <linux/module.h>
+#include <drm/drm_connector.h>
+
+#ifdef CONFIG_DMI
 
 /*
  * Some x86 clamshell design devices use portrait tablet screens and a display
@@ -21,56 +22,56 @@
  * and a list of known BIOS dates to avoid false positives.
  */
 
-struct fbcon_dmi_rotate_data {
+struct drm_dmi_panel_orientation_data {
 	int width;
 	int height;
 	const char * const *bios_dates;
-	int rotate;
+	int orientation;
 };
 
-static const struct fbcon_dmi_rotate_data rotate_data_asus_t100ha = {
+static const struct drm_dmi_panel_orientation_data asus_t100ha = {
 	.width = 800,
 	.height = 1280,
-	.rotate = FB_ROTATE_CCW,
+	.orientation = DRM_MODE_PANEL_ORIENTATION_LEFT_UP,
 };
 
-static const struct fbcon_dmi_rotate_data rotate_data_gpd_pocket = {
+static const struct drm_dmi_panel_orientation_data gpd_pocket = {
 	.width = 1200,
 	.height = 1920,
 	.bios_dates = (const char * const []){ "05/26/2017", "06/28/2017",
 		"07/05/2017", "08/07/2017", NULL },
-	.rotate = FB_ROTATE_CW,
+	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
-static const struct fbcon_dmi_rotate_data rotate_data_gpd_win = {
+static const struct drm_dmi_panel_orientation_data gpd_win = {
 	.width = 720,
 	.height = 1280,
 	.bios_dates = (const char * const []){
 		"10/25/2016", "11/18/2016", "12/23/2016", "12/26/2016",
 		"02/21/2017", "03/20/2017", "05/25/2017", NULL },
-	.rotate = FB_ROTATE_CW,
+	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
-static const struct fbcon_dmi_rotate_data rotate_data_itworks_tw891 = {
+static const struct drm_dmi_panel_orientation_data itworks_tw891 = {
 	.width = 800,
 	.height = 1280,
 	.bios_dates = (const char * const []){ "10/16/2015", NULL },
-	.rotate = FB_ROTATE_CW,
+	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
-static const struct fbcon_dmi_rotate_data rotate_data_vios_lth17 = {
+static const struct drm_dmi_panel_orientation_data vios_lth17 = {
 	.width = 800,
 	.height = 1280,
-	.rotate = FB_ROTATE_CW,
+	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
-static const struct dmi_system_id rotate_data[] = {
+static const struct dmi_system_id orientation_data[] = {
 	{	/* Asus T100HA */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
 		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "T100HAN"),
 		},
-		.driver_data = (void *)&rotate_data_asus_t100ha,
+		.driver_data = (void *)&asus_t100ha,
 	}, {	/*
 		 * GPD Pocket, note that the the DMI data is less generic then
 		 * it seems, devices with a board-vendor of "AMI Corporation"
@@ -83,7 +84,7 @@ static const struct dmi_system_id rotate_data[] = {
 		  DMI_EXACT_MATCH(DMI_BOARD_SERIAL, "Default string"),
 		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Default string"),
 		},
-		.driver_data = (void *)&rotate_data_gpd_pocket,
+		.driver_data = (void *)&gpd_pocket,
 	}, {	/* GPD Win (same note on DMI match as GPD Pocket) */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
@@ -91,7 +92,7 @@ static const struct dmi_system_id rotate_data[] = {
 		  DMI_EXACT_MATCH(DMI_BOARD_SERIAL, "Default string"),
 		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Default string"),
 		},
-		.driver_data = (void *)&rotate_data_gpd_win,
+		.driver_data = (void *)&gpd_win,
 	}, {	/* I.T.Works TW891 */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "To be filled by O.E.M."),
@@ -99,37 +100,54 @@ static const struct dmi_system_id rotate_data[] = {
 		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "To be filled by O.E.M."),
 		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "TW891"),
 		},
-		.driver_data = (void *)&rotate_data_itworks_tw891,
+		.driver_data = (void *)&itworks_tw891,
 	}, {	/* VIOS LTH17 */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "VIOS"),
 		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "LTH17"),
-		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "VIOS"),
-		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "LTH17"),
 		},
-		.driver_data = (void *)&rotate_data_vios_lth17,
+		.driver_data = (void *)&vios_lth17,
 	},
 	{}
 };
 
-int fbcon_platform_get_rotate(struct fb_info *info)
+/**
+ * drm_get_panel_orientation_quirk - Check for panel orientation quirks
+ * @width: width in pixels of the panel
+ * @height: height in pixels of the panel
+ *
+ * This function checks for platform specific (e.g. DMI based) quirks
+ * providing info on panel_orientation for systems where this cannot be
+ * probed from the hard-/firm-ware. To avoid false-positive this function
+ * takes the panel resolution as argument and checks that against the
+ * resolution expected by the quirk-table entry.
+ *
+ * Note this function is also used outside of the drm-subsys, by for example
+ * the efifb code. Because of this this function gets compiled into its own
+ * kernel-module when built as a module.
+ *
+ * Returns:
+ * A DRM_MODE_PANEL_ORIENTATION_* value if there is a quirk for this system,
+ * or DRM_MODE_PANEL_ORIENTATION_UNKNOWN if there is no quirk.
+ */
+int drm_get_panel_orientation_quirk(int width, int height)
 {
 	const struct dmi_system_id *match;
-	const struct fbcon_dmi_rotate_data *data;
+	const struct drm_dmi_panel_orientation_data *data;
 	const char *bios_date;
 	int i;
 
-	for (match = dmi_first_match(rotate_data);
+	for (match = dmi_first_match(orientation_data);
 	     match;
 	     match = dmi_first_match(match + 1)) {
 		data = match->driver_data;
 
-		if (data->width != info->var.xres ||
-		    data->height != info->var.yres)
+		if (data->width != width ||
+		    data->height != height)
 			continue;
 
 		if (!data->bios_dates)
-			return data->rotate;
+			return data->orientation;
 
 		bios_date = dmi_get_system_info(DMI_BIOS_DATE);
 		if (!bios_date)
@@ -137,9 +155,23 @@ int fbcon_platform_get_rotate(struct fb_info *info)
 
 		for (i = 0; data->bios_dates[i]; i++) {
 			if (!strcmp(data->bios_dates[i], bios_date))
-				return data->rotate;
+				return data->orientation;
 		}
 	}
 
-	return FB_ROTATE_UR;
+	return DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
 }
+EXPORT_SYMBOL(drm_get_panel_orientation_quirk);
+
+#else
+
+/* There are no quirks for non x86 devices yet */
+int drm_get_panel_orientation_quirk(int width, int height)
+{
+	return DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
+}
+EXPORT_SYMBOL(drm_get_panel_orientation_quirk);
+
+#endif
+
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
index 759ed93f4ba8..f1be8cd4e387 100644
--- a/drivers/gpu/drm/drm_plane_helper.c
+++ b/drivers/gpu/drm/drm_plane_helper.c
@@ -99,97 +99,6 @@ static int get_connectors_for_crtc(struct drm_crtc *crtc,
 	return count;
 }
 
-/**
- * drm_plane_helper_check_state() - Check plane state for validity
- * @state: plane state to check
- * @clip: integer clipping coordinates
- * @min_scale: minimum @src:@dest scaling factor in 16.16 fixed point
- * @max_scale: maximum @src:@dest scaling factor in 16.16 fixed point
- * @can_position: is it legal to position the plane such that it
- *                doesn't cover the entire crtc?  This will generally
- *                only be false for primary planes.
- * @can_update_disabled: can the plane be updated while the crtc
- *                       is disabled?
- *
- * Checks that a desired plane update is valid, and updates various
- * bits of derived state (clipped coordinates etc.). Drivers that provide
- * their own plane handling rather than helper-provided implementations may
- * still wish to call this function to avoid duplication of error checking
- * code.
- *
- * RETURNS:
- * Zero if update appears valid, error code on failure
- */
-int drm_plane_helper_check_state(struct drm_plane_state *state,
-				 const struct drm_rect *clip,
-				 int min_scale,
-				 int max_scale,
-				 bool can_position,
-				 bool can_update_disabled)
-{
-	struct drm_crtc *crtc = state->crtc;
-	struct drm_framebuffer *fb = state->fb;
-	struct drm_rect *src = &state->src;
-	struct drm_rect *dst = &state->dst;
-	unsigned int rotation = state->rotation;
-	int hscale, vscale;
-
-	*src = drm_plane_state_src(state);
-	*dst = drm_plane_state_dest(state);
-
-	if (!fb) {
-		state->visible = false;
-		return 0;
-	}
-
-	/* crtc should only be NULL when disabling (i.e., !fb) */
-	if (WARN_ON(!crtc)) {
-		state->visible = false;
-		return 0;
-	}
-
-	if (!crtc->enabled && !can_update_disabled) {
-		DRM_DEBUG_KMS("Cannot update plane of a disabled CRTC.\n");
-		return -EINVAL;
-	}
-
-	drm_rect_rotate(src, fb->width << 16, fb->height << 16, rotation);
-
-	/* Check scaling */
-	hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale);
-	vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale);
-	if (hscale < 0 || vscale < 0) {
-		DRM_DEBUG_KMS("Invalid scaling of plane\n");
-		drm_rect_debug_print("src: ", &state->src, true);
-		drm_rect_debug_print("dst: ", &state->dst, false);
-		return -ERANGE;
-	}
-
-	state->visible = drm_rect_clip_scaled(src, dst, clip, hscale, vscale);
-
-	drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation);
-
-	if (!state->visible)
-		/*
-		 * Plane isn't visible; some drivers can handle this
-		 * so we just return success here.  Drivers that can't
-		 * (including those that use the primary plane helper's
-		 * update function) will return an error from their
-		 * update_plane handler.
-		 */
-		return 0;
-
-	if (!can_position && !drm_rect_equals(dst, clip)) {
-		DRM_DEBUG_KMS("Plane must cover entire CRTC\n");
-		drm_rect_debug_print("dst: ", dst, false);
-		drm_rect_debug_print("clip: ", clip, false);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(drm_plane_helper_check_state);
-
 /**
  * drm_plane_helper_check_update() - Check plane update for validity
  * @plane: plane object to update
@@ -230,7 +139,7 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
 				  bool can_update_disabled,
 				  bool *visible)
 {
-	struct drm_plane_state state = {
+	struct drm_plane_state plane_state = {
 		.plane = plane,
 		.crtc = crtc,
 		.fb = fb,
@@ -245,18 +154,22 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
 		.rotation = rotation,
 		.visible = *visible,
 	};
+	struct drm_crtc_state crtc_state = {
+		.crtc = crtc,
+		.enable = crtc->enabled,
+	};
 	int ret;
 
-	ret = drm_plane_helper_check_state(&state, clip,
-					   min_scale, max_scale,
-					   can_position,
-					   can_update_disabled);
+	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
+						  clip, min_scale, max_scale,
+						  can_position,
+						  can_update_disabled);
 	if (ret)
 		return ret;
 
-	*src = state.src;
-	*dst = state.dst;
-	*visible = state.visible;
+	*src = plane_state.src;
+	*dst = plane_state.dst;
+	*visible = plane_state.visible;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 8de93a226c24..9a17725b0f7a 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -218,8 +218,9 @@ static void drm_gem_map_detach(struct dma_buf *dma_buf,
 	sgt = prime_attach->sgt;
 	if (sgt) {
 		if (prime_attach->dir != DMA_NONE)
-			dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
-					prime_attach->dir);
+			dma_unmap_sg_attrs(attach->dev, sgt->sgl, sgt->nents,
+					   prime_attach->dir,
+					   DMA_ATTR_SKIP_CPU_SYNC);
 		sg_free_table(sgt);
 	}
 
@@ -277,7 +278,8 @@ static struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach,
 	sgt = obj->dev->driver->gem_prime_get_sg_table(obj);
 
 	if (!IS_ERR(sgt)) {
-		if (!dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir)) {
+		if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir,
+				      DMA_ATTR_SKIP_CPU_SYNC)) {
 			sg_free_table(sgt);
 			kfree(sgt);
 			sgt = ERR_PTR(-ENOMEM);
diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c
index 74c466aca622..781518fd88e3 100644
--- a/drivers/gpu/drm/drm_print.c
+++ b/drivers/gpu/drm/drm_print.c
@@ -23,6 +23,8 @@
  * Rob Clark <robdclark@gmail.com>
  */
 
+#define DEBUG /* for pr_debug() */
+
 #include <stdarg.h>
 #include <linux/seq_file.h>
 #include <drm/drmP.h>
@@ -53,13 +55,57 @@ EXPORT_SYMBOL(__drm_printfn_debug);
  */
 void drm_printf(struct drm_printer *p, const char *f, ...)
 {
-	struct va_format vaf;
 	va_list args;
 
 	va_start(args, f);
-	vaf.fmt = f;
-	vaf.va = &args;
-	p->printfn(p, &vaf);
+	drm_vprintf(p, f, &args);
 	va_end(args);
 }
 EXPORT_SYMBOL(drm_printf);
+
+#define DRM_PRINTK_FMT "[" DRM_NAME ":%s]%s %pV"
+
+void drm_dev_printk(const struct device *dev, const char *level,
+		    unsigned int category, const char *function_name,
+		    const char *prefix, const char *format, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	if (category != DRM_UT_NONE && !(drm_debug & category))
+		return;
+
+	va_start(args, format);
+	vaf.fmt = format;
+	vaf.va = &args;
+
+	if (dev)
+		dev_printk(level, dev, DRM_PRINTK_FMT, function_name, prefix,
+			   &vaf);
+	else
+		printk("%s" DRM_PRINTK_FMT, level, function_name, prefix, &vaf);
+
+	va_end(args);
+}
+EXPORT_SYMBOL(drm_dev_printk);
+
+void drm_printk(const char *level, unsigned int category,
+		const char *format, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	if (category != DRM_UT_NONE && !(drm_debug & category))
+		return;
+
+	va_start(args, format);
+	vaf.fmt = format;
+	vaf.va = &args;
+
+	printk("%s" "[" DRM_NAME ":%ps]%s %pV",
+	       level, __builtin_return_address(0),
+	       strcmp(level, KERN_ERR) == 0 ? " *ERROR*" : "", &vaf);
+
+	va_end(args);
+}
+EXPORT_SYMBOL(drm_printk);
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index 6dc2dde5b672..00b8445ba819 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -216,8 +216,7 @@ enum drm_mode_status drm_connector_mode_valid(struct drm_connector *connector,
  * suspend/resume.
  *
  * Drivers can call this helper from their device resume implementation. It is
- * an error to call this when the output polling support has not yet been set
- * up.
+ * not an error to call this even when output polling isn't enabled.
  *
  * Note that calls to enable and disable polling must be strictly ordered, which
  * is automatically the case when they're only call from suspend/resume
@@ -654,6 +653,26 @@ static void output_poll_execute(struct work_struct *work)
 		schedule_delayed_work(delayed_work, DRM_OUTPUT_POLL_PERIOD);
 }
 
+/**
+ * drm_kms_helper_is_poll_worker - is %current task an output poll worker?
+ *
+ * Determine if %current task is an output poll worker.  This can be used
+ * to select distinct code paths for output polling versus other contexts.
+ *
+ * One use case is to avoid a deadlock between the output poll worker and
+ * the autosuspend worker wherein the latter waits for polling to finish
+ * upon calling drm_kms_helper_poll_disable(), while the former waits for
+ * runtime suspend to finish upon calling pm_runtime_get_sync() in a
+ * connector ->detect hook.
+ */
+bool drm_kms_helper_is_poll_worker(void)
+{
+	struct work_struct *work = current_work();
+
+	return work && work->func == output_poll_execute;
+}
+EXPORT_SYMBOL(drm_kms_helper_is_poll_worker);
+
 /**
  * drm_kms_helper_poll_disable - disable output polling
  * @dev: drm_device
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
index dc9fd109de14..9f3b1c94802b 100644
--- a/drivers/gpu/drm/drm_simple_kms_helper.c
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -103,10 +103,11 @@ static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
 
-	ret = drm_plane_helper_check_state(plane_state, &clip,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   false, true);
+	ret = drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						  &clip,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, true);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c
index cb4d09c70fd4..0b7b0d1ad2d5 100644
--- a/drivers/gpu/drm/drm_syncobj.c
+++ b/drivers/gpu/drm/drm_syncobj.c
@@ -29,9 +29,9 @@
 /**
  * DOC: Overview
  *
- * DRM synchronisation objects (syncobj) are a persistent objects,
- * that contain an optional fence. The fence can be updated with a new
- * fence, or be NULL.
+ * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
+ * persistent objects that contain an optional fence. The fence can be updated
+ * with a new fence, or be NULL.
  *
  * syncobj's can be waited upon, where it will wait for the underlying
  * fence.
@@ -61,7 +61,8 @@
  * @file_private: drm file private pointer
  * @handle: sync object handle to lookup.
  *
- * Returns a reference to the syncobj pointed to by handle or NULL.
+ * Returns a reference to the syncobj pointed to by handle or NULL. The
+ * reference must be released by calling drm_syncobj_put().
  */
 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 				     u32 handle)
@@ -106,7 +107,8 @@ static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
 	 * callback when a fence has already been set.
 	 */
 	if (syncobj->fence) {
-		*fence = dma_fence_get(syncobj->fence);
+		*fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
+								 lockdep_is_held(&syncobj->lock)));
 		ret = 1;
 	} else {
 		*fence = NULL;
@@ -168,8 +170,9 @@ void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 
 	spin_lock(&syncobj->lock);
 
-	old_fence = syncobj->fence;
-	syncobj->fence = fence;
+	old_fence = rcu_dereference_protected(syncobj->fence,
+					      lockdep_is_held(&syncobj->lock));
+	rcu_assign_pointer(syncobj->fence, fence);
 
 	if (fence != old_fence) {
 		list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
@@ -227,6 +230,19 @@ static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 	return 0;
 }
 
+/**
+ * drm_syncobj_find_fence - lookup and reference the fence in a sync object
+ * @file_private: drm file private pointer
+ * @handle: sync object handle to lookup.
+ * @fence: out parameter for the fence
+ *
+ * This is just a convenience function that combines drm_syncobj_find() and
+ * drm_syncobj_fence_get().
+ *
+ * Returns 0 on success or a negative error value on failure. On success @fence
+ * contains a reference to the fence, which must be released by calling
+ * dma_fence_put().
+ */
 int drm_syncobj_find_fence(struct drm_file *file_private,
 			   u32 handle,
 			   struct dma_fence **fence)
@@ -267,6 +283,12 @@ EXPORT_SYMBOL(drm_syncobj_free);
  * @out_syncobj: returned syncobj
  * @flags: DRM_SYNCOBJ_* flags
  * @fence: if non-NULL, the syncobj will represent this fence
+ *
+ * This is the first function to create a sync object. After creating, drivers
+ * probably want to make it available to userspace, either through
+ * drm_syncobj_get_handle() or drm_syncobj_get_fd().
+ *
+ * Returns 0 on success or a negative error value on failure.
  */
 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 		       struct dma_fence *fence)
@@ -300,6 +322,14 @@ EXPORT_SYMBOL(drm_syncobj_create);
 
 /**
  * drm_syncobj_get_handle - get a handle from a syncobj
+ * @file_private: drm file private pointer
+ * @syncobj: Sync object to export
+ * @handle: out parameter with the new handle
+ *
+ * Exports a sync object created with drm_syncobj_create() as a handle on
+ * @file_private to userspace.
+ *
+ * Returns 0 on success or a negative error value on failure.
  */
 int drm_syncobj_get_handle(struct drm_file *file_private,
 			   struct drm_syncobj *syncobj, u32 *handle)
@@ -369,6 +399,15 @@ static const struct file_operations drm_syncobj_file_fops = {
 	.release = drm_syncobj_file_release,
 };
 
+/**
+ * drm_syncobj_get_fd - get a file descriptor from a syncobj
+ * @syncobj: Sync object to export
+ * @p_fd: out parameter with the new file descriptor
+ *
+ * Exports a sync object created with drm_syncobj_create() as a file descriptor.
+ *
+ * Returns 0 on success or a negative error value on failure.
+ */
 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 {
 	struct file *file;
@@ -640,7 +679,8 @@ static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 		container_of(cb, struct syncobj_wait_entry, syncobj_cb);
 
 	/* This happens inside the syncobj lock */
-	wait->fence = dma_fence_get(syncobj->fence);
+	wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
+							      lockdep_is_held(&syncobj->lock)));
 	wake_up_process(wait->task);
 }
 
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c
index 3717b3df34a4..32d9bcf5be7f 100644
--- a/drivers/gpu/drm/drm_vblank.c
+++ b/drivers/gpu/drm/drm_vblank.c
@@ -663,14 +663,16 @@ bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
 	delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
 			   mode->crtc_clock);
 
-	/* save this only for debugging purposes */
-	ts_etime = ktime_to_timespec64(etime);
-	ts_vblank_time = ktime_to_timespec64(*vblank_time);
 	/* Subtract time delta from raw timestamp to get final
 	 * vblank_time timestamp for end of vblank.
 	 */
-	etime = ktime_sub_ns(etime, delta_ns);
-	*vblank_time = etime;
+	*vblank_time = ktime_sub_ns(etime, delta_ns);
+
+	if ((drm_debug & DRM_UT_VBL) == 0)
+		return true;
+
+	ts_etime = ktime_to_timespec64(etime);
+	ts_vblank_time = ktime_to_timespec64(*vblank_time);
 
 	DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
 		      pipe, hpos, vpos,
diff --git a/drivers/gpu/drm/drm_vma_manager.c b/drivers/gpu/drm/drm_vma_manager.c
index 28f1226576f8..23c749c05b5a 100644
--- a/drivers/gpu/drm/drm_vma_manager.c
+++ b/drivers/gpu/drm/drm_vma_manager.c
@@ -203,21 +203,16 @@ EXPORT_SYMBOL(drm_vma_offset_lookup_locked);
 int drm_vma_offset_add(struct drm_vma_offset_manager *mgr,
 		       struct drm_vma_offset_node *node, unsigned long pages)
 {
-	int ret;
+	int ret = 0;
 
 	write_lock(&mgr->vm_lock);
 
-	if (drm_mm_node_allocated(&node->vm_node)) {
-		ret = 0;
-		goto out_unlock;
-	}
+	if (!drm_mm_node_allocated(&node->vm_node))
+		ret = drm_mm_insert_node(&mgr->vm_addr_space_mm,
+					 &node->vm_node, pages);
 
-	ret = drm_mm_insert_node(&mgr->vm_addr_space_mm, &node->vm_node, pages);
-	if (ret)
-		goto out_unlock;
-
-out_unlock:
 	write_unlock(&mgr->vm_lock);
+
 	return ret;
 }
 EXPORT_SYMBOL(drm_vma_offset_add);
diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig
index a29b8f59eb15..3f58b4077767 100644
--- a/drivers/gpu/drm/etnaviv/Kconfig
+++ b/drivers/gpu/drm/etnaviv/Kconfig
@@ -6,6 +6,7 @@ config DRM_ETNAVIV
 	depends on MMU
 	select SHMEM
 	select SYNC_FILE
+	select THERMAL if DRM_ETNAVIV_THERMAL
 	select TMPFS
 	select WANT_DEV_COREDUMP
 	select CMA if HAVE_DMA_CONTIGUOUS
@@ -13,6 +14,14 @@ config DRM_ETNAVIV
 	help
 	  DRM driver for Vivante GPUs.
 
+config DRM_ETNAVIV_THERMAL
+	bool "enable ETNAVIV thermal throttling"
+	depends on DRM_ETNAVIV
+	default y
+	help
+	  Compile in support for thermal throttling.
+	  Say Y unless you want to risk burning your SoC.
+
 config DRM_ETNAVIV_REGISTER_LOGGING
 	bool "enable ETNAVIV register logging"
 	depends on DRM_ETNAVIV
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
index 9e7098e3207f..99ad2f073c6e 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c
@@ -100,6 +100,8 @@ static void etnaviv_cmd_select_pipe(struct etnaviv_gpu *gpu,
 {
 	u32 flush = 0;
 
+	lockdep_assert_held(&gpu->lock);
+
 	/*
 	 * This assumes that if we're switching to 2D, we're switching
 	 * away from 3D, and vice versa.  Hence, if we're switching to
@@ -164,7 +166,9 @@ static u32 etnaviv_buffer_reserve(struct etnaviv_gpu *gpu,
 
 u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)
 {
-	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+	struct etnaviv_cmdbuf *buffer = &gpu->buffer;
+
+	lockdep_assert_held(&gpu->lock);
 
 	/* initialize buffer */
 	buffer->user_size = 0;
@@ -178,7 +182,9 @@ u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)
 
 u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe_addr)
 {
-	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+	struct etnaviv_cmdbuf *buffer = &gpu->buffer;
+
+	lockdep_assert_held(&gpu->lock);
 
 	buffer->user_size = 0;
 
@@ -211,10 +217,12 @@ u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe
 
 void etnaviv_buffer_end(struct etnaviv_gpu *gpu)
 {
-	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+	struct etnaviv_cmdbuf *buffer = &gpu->buffer;
 	unsigned int waitlink_offset = buffer->user_size - 16;
 	u32 link_target, flush = 0;
 
+	lockdep_assert_held(&gpu->lock);
+
 	if (gpu->exec_state == ETNA_PIPE_2D)
 		flush = VIVS_GL_FLUSH_CACHE_PE2D;
 	else if (gpu->exec_state == ETNA_PIPE_3D)
@@ -253,10 +261,12 @@ void etnaviv_buffer_end(struct etnaviv_gpu *gpu)
 /* Append a 'sync point' to the ring buffer. */
 void etnaviv_sync_point_queue(struct etnaviv_gpu *gpu, unsigned int event)
 {
-	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+	struct etnaviv_cmdbuf *buffer = &gpu->buffer;
 	unsigned int waitlink_offset = buffer->user_size - 16;
 	u32 dwords, target;
 
+	lockdep_assert_held(&gpu->lock);
+
 	/*
 	 * We need at most 3 dwords in the return target:
 	 * 1 event + 1 end + 1 wait + 1 link.
@@ -287,13 +297,16 @@ void etnaviv_sync_point_queue(struct etnaviv_gpu *gpu, unsigned int event)
 }
 
 /* Append a command buffer to the ring buffer. */
-void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
-	struct etnaviv_cmdbuf *cmdbuf)
+void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state,
+	unsigned int event, struct etnaviv_cmdbuf *cmdbuf)
 {
-	struct etnaviv_cmdbuf *buffer = gpu->buffer;
+	struct etnaviv_cmdbuf *buffer = &gpu->buffer;
 	unsigned int waitlink_offset = buffer->user_size - 16;
 	u32 return_target, return_dwords;
 	u32 link_target, link_dwords;
+	bool switch_context = gpu->exec_state != exec_state;
+
+	lockdep_assert_held(&gpu->lock);
 
 	if (drm_debug & DRM_UT_DRIVER)
 		etnaviv_buffer_dump(gpu, buffer, 0, 0x50);
@@ -306,7 +319,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 	 * need to append a mmu flush load state, followed by a new
 	 * link to this buffer - a total of four additional words.
 	 */
-	if (gpu->mmu->need_flush || gpu->switch_context) {
+	if (gpu->mmu->need_flush || switch_context) {
 		u32 target, extra_dwords;
 
 		/* link command */
@@ -321,7 +334,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 		}
 
 		/* pipe switch commands */
-		if (gpu->switch_context)
+		if (switch_context)
 			extra_dwords += 4;
 
 		target = etnaviv_buffer_reserve(gpu, buffer, extra_dwords);
@@ -349,10 +362,9 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 			gpu->mmu->need_flush = false;
 		}
 
-		if (gpu->switch_context) {
-			etnaviv_cmd_select_pipe(gpu, buffer, cmdbuf->exec_state);
-			gpu->exec_state = cmdbuf->exec_state;
-			gpu->switch_context = false;
+		if (switch_context) {
+			etnaviv_cmd_select_pipe(gpu, buffer, exec_state);
+			gpu->exec_state = exec_state;
 		}
 
 		/* And the link to the submitted buffer */
@@ -421,4 +433,6 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
 
 	if (drm_debug & DRM_UT_DRIVER)
 		etnaviv_buffer_dump(gpu, buffer, 0, 0x50);
+
+	gpu->lastctx = cmdbuf->ctx;
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
index 6e3bbcf24160..68e6d3772ad8 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmd_parser.c
@@ -78,6 +78,7 @@ static const struct {
 	ST(0x17c0, 8),
 	ST(0x17e0, 8),
 	ST(0x2400, 14 * 16),
+	ST(0x3824, 1),
 	ST(0x10800, 32 * 16),
 	ST(0x14600, 16),
 	ST(0x14800, 8 * 8),
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c
index 66ac79558bbd..3746827f45eb 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c
@@ -86,26 +86,11 @@ void etnaviv_cmdbuf_suballoc_destroy(struct etnaviv_cmdbuf_suballoc *suballoc)
 	kfree(suballoc);
 }
 
-struct etnaviv_cmdbuf *
-etnaviv_cmdbuf_new(struct etnaviv_cmdbuf_suballoc *suballoc, u32 size,
-		   size_t nr_bos, size_t nr_pmrs)
+int etnaviv_cmdbuf_init(struct etnaviv_cmdbuf_suballoc *suballoc,
+			struct etnaviv_cmdbuf *cmdbuf, u32 size)
 {
-	struct etnaviv_cmdbuf *cmdbuf;
-	struct etnaviv_perfmon_request *pmrs;
-	size_t sz = size_vstruct(nr_bos, sizeof(cmdbuf->bo_map[0]),
-				 sizeof(*cmdbuf));
 	int granule_offs, order, ret;
 
-	cmdbuf = kzalloc(sz, GFP_KERNEL);
-	if (!cmdbuf)
-		return NULL;
-
-	sz = sizeof(*pmrs) * nr_pmrs;
-	pmrs = kzalloc(sz, GFP_KERNEL);
-	if (!pmrs)
-		goto out_free_cmdbuf;
-
-	cmdbuf->pmrs = pmrs;
 	cmdbuf->suballoc = suballoc;
 	cmdbuf->size = size;
 
@@ -123,7 +108,7 @@ etnaviv_cmdbuf_new(struct etnaviv_cmdbuf_suballoc *suballoc, u32 size,
 		if (!ret) {
 			dev_err(suballoc->gpu->dev,
 				"Timeout waiting for cmdbuf space\n");
-			return NULL;
+			return -ETIMEDOUT;
 		}
 		goto retry;
 	}
@@ -131,11 +116,7 @@ etnaviv_cmdbuf_new(struct etnaviv_cmdbuf_suballoc *suballoc, u32 size,
 	cmdbuf->suballoc_offset = granule_offs * SUBALLOC_GRANULE;
 	cmdbuf->vaddr = suballoc->vaddr + cmdbuf->suballoc_offset;
 
-	return cmdbuf;
-
-out_free_cmdbuf:
-	kfree(cmdbuf);
-	return NULL;
+	return 0;
 }
 
 void etnaviv_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf)
@@ -151,8 +132,6 @@ void etnaviv_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf)
 	suballoc->free_space = 1;
 	mutex_unlock(&suballoc->lock);
 	wake_up_all(&suballoc->free_event);
-	kfree(cmdbuf->pmrs);
-	kfree(cmdbuf);
 }
 
 u32 etnaviv_cmdbuf_get_va(struct etnaviv_cmdbuf *buf)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h
index b6348b9f2a9d..ddc3f7ea169c 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h
@@ -33,27 +33,15 @@ struct etnaviv_cmdbuf {
 	void *vaddr;
 	u32 size;
 	u32 user_size;
-	/* fence after which this buffer is to be disposed */
-	struct dma_fence *fence;
-	/* target exec state */
-	u32 exec_state;
-	/* per GPU in-flight list */
-	struct list_head node;
-	/* perfmon requests */
-	unsigned int nr_pmrs;
-	struct etnaviv_perfmon_request *pmrs;
-	/* BOs attached to this command buffer */
-	unsigned int nr_bos;
-	struct etnaviv_vram_mapping *bo_map[0];
 };
 
 struct etnaviv_cmdbuf_suballoc *
 etnaviv_cmdbuf_suballoc_new(struct etnaviv_gpu * gpu);
 void etnaviv_cmdbuf_suballoc_destroy(struct etnaviv_cmdbuf_suballoc *suballoc);
 
-struct etnaviv_cmdbuf *
-etnaviv_cmdbuf_new(struct etnaviv_cmdbuf_suballoc *suballoc, u32 size,
-		   size_t nr_bos, size_t nr_pmrs);
+
+int etnaviv_cmdbuf_init(struct etnaviv_cmdbuf_suballoc *suballoc,
+		struct etnaviv_cmdbuf *cmdbuf, u32 size);
 void etnaviv_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf);
 
 u32 etnaviv_cmdbuf_get_va(struct etnaviv_cmdbuf *buf);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 491eddf9b150..6faf4042db23 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -172,7 +172,7 @@ static int etnaviv_mmu_show(struct etnaviv_gpu *gpu, struct seq_file *m)
 
 static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu, struct seq_file *m)
 {
-	struct etnaviv_cmdbuf *buf = gpu->buffer;
+	struct etnaviv_cmdbuf *buf = &gpu->buffer;
 	u32 size = buf->size;
 	u32 *ptr = buf->vaddr;
 	u32 i;
@@ -459,9 +459,6 @@ static int etnaviv_ioctl_pm_query_dom(struct drm_device *dev, void *data,
 	struct drm_etnaviv_pm_domain *args = data;
 	struct etnaviv_gpu *gpu;
 
-	/* reject as long as the feature isn't stable */
-	return -EINVAL;
-
 	if (args->pipe >= ETNA_MAX_PIPES)
 		return -EINVAL;
 
@@ -479,9 +476,6 @@ static int etnaviv_ioctl_pm_query_sig(struct drm_device *dev, void *data,
 	struct drm_etnaviv_pm_signal *args = data;
 	struct etnaviv_gpu *gpu;
 
-	/* reject as long as the feature isn't stable */
-	return -EINVAL;
-
 	if (args->pipe >= ETNA_MAX_PIPES)
 		return -EINVAL;
 
@@ -556,7 +550,7 @@ static struct drm_driver etnaviv_drm_driver = {
 	.desc               = "etnaviv DRM",
 	.date               = "20151214",
 	.major              = 1,
-	.minor              = 1,
+	.minor              = 2,
 };
 
 /*
@@ -580,12 +574,6 @@ static int etnaviv_bind(struct device *dev)
 	}
 	drm->dev_private = priv;
 
-	priv->wq = alloc_ordered_workqueue("etnaviv", 0);
-	if (!priv->wq) {
-		ret = -ENOMEM;
-		goto out_wq;
-	}
-
 	mutex_init(&priv->gem_lock);
 	INIT_LIST_HEAD(&priv->gem_list);
 	priv->num_gpus = 0;
@@ -607,9 +595,6 @@ static int etnaviv_bind(struct device *dev)
 out_register:
 	component_unbind_all(dev, drm);
 out_bind:
-	flush_workqueue(priv->wq);
-	destroy_workqueue(priv->wq);
-out_wq:
 	kfree(priv);
 out_unref:
 	drm_dev_unref(drm);
@@ -624,9 +609,6 @@ static void etnaviv_unbind(struct device *dev)
 
 	drm_dev_unregister(drm);
 
-	flush_workqueue(priv->wq);
-	destroy_workqueue(priv->wq);
-
 	component_unbind_all(dev, drm);
 
 	drm->dev_private = NULL;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index d249acb6da08..a54f0b758a5c 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -56,18 +56,8 @@ struct etnaviv_drm_private {
 	/* list of GEM objects: */
 	struct mutex gem_lock;
 	struct list_head gem_list;
-
-	struct workqueue_struct *wq;
 };
 
-static inline void etnaviv_queue_work(struct drm_device *dev,
-	struct work_struct *w)
-{
-	struct etnaviv_drm_private *priv = dev->dev_private;
-
-	queue_work(priv->wq, w);
-}
-
 int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 		struct drm_file *file);
 
@@ -97,8 +87,8 @@ u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu);
 u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe_addr);
 void etnaviv_buffer_end(struct etnaviv_gpu *gpu);
 void etnaviv_sync_point_queue(struct etnaviv_gpu *gpu, unsigned int event);
-void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
-	struct etnaviv_cmdbuf *cmdbuf);
+void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state,
+	unsigned int event, struct etnaviv_cmdbuf *cmdbuf);
 void etnaviv_validate_init(void);
 bool etnaviv_cmd_validate_one(struct etnaviv_gpu *gpu,
 	u32 *stream, unsigned int size,
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
index 2d955d7d7b6d..6d0909c589d1 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c
@@ -120,7 +120,7 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu)
 	struct core_dump_iterator iter;
 	struct etnaviv_vram_mapping *vram;
 	struct etnaviv_gem_object *obj;
-	struct etnaviv_cmdbuf *cmd;
+	struct etnaviv_gem_submit *submit;
 	unsigned int n_obj, n_bomap_pages;
 	size_t file_size, mmu_size;
 	__le64 *bomap, *bomap_start;
@@ -132,11 +132,11 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu)
 	n_bomap_pages = 0;
 	file_size = ARRAY_SIZE(etnaviv_dump_registers) *
 			sizeof(struct etnaviv_dump_registers) +
-		    mmu_size + gpu->buffer->size;
+		    mmu_size + gpu->buffer.size;
 
 	/* Add in the active command buffers */
-	list_for_each_entry(cmd, &gpu->active_cmd_list, node) {
-		file_size += cmd->size;
+	list_for_each_entry(submit, &gpu->active_submit_list, node) {
+		file_size += submit->cmdbuf.size;
 		n_obj++;
 	}
 
@@ -176,13 +176,14 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu)
 
 	etnaviv_core_dump_registers(&iter, gpu);
 	etnaviv_core_dump_mmu(&iter, gpu, mmu_size);
-	etnaviv_core_dump_mem(&iter, ETDUMP_BUF_RING, gpu->buffer->vaddr,
-			      gpu->buffer->size,
-			      etnaviv_cmdbuf_get_va(gpu->buffer));
-
-	list_for_each_entry(cmd, &gpu->active_cmd_list, node)
-		etnaviv_core_dump_mem(&iter, ETDUMP_BUF_CMD, cmd->vaddr,
-				      cmd->size, etnaviv_cmdbuf_get_va(cmd));
+	etnaviv_core_dump_mem(&iter, ETDUMP_BUF_RING, gpu->buffer.vaddr,
+			      gpu->buffer.size,
+			      etnaviv_cmdbuf_get_va(&gpu->buffer));
+
+	list_for_each_entry(submit, &gpu->active_submit_list, node)
+		etnaviv_core_dump_mem(&iter, ETDUMP_BUF_CMD,
+				      submit->cmdbuf.vaddr, submit->cmdbuf.size,
+				      etnaviv_cmdbuf_get_va(&submit->cmdbuf));
 
 	/* Reserve space for the bomap */
 	if (n_bomap_pages) {
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index daee3f1196df..fcc969fa0e69 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -24,6 +24,9 @@
 #include "etnaviv_gpu.h"
 #include "etnaviv_mmu.h"
 
+static struct lock_class_key etnaviv_shm_lock_class;
+static struct lock_class_key etnaviv_userptr_lock_class;
+
 static void etnaviv_gem_scatter_map(struct etnaviv_gem_object *etnaviv_obj)
 {
 	struct drm_device *dev = etnaviv_obj->base.dev;
@@ -583,7 +586,7 @@ void etnaviv_gem_free_object(struct drm_gem_object *obj)
 	kfree(etnaviv_obj);
 }
 
-int etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj)
+void etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj)
 {
 	struct etnaviv_drm_private *priv = dev->dev_private;
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
@@ -591,8 +594,6 @@ int etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj)
 	mutex_lock(&priv->gem_lock);
 	list_add_tail(&etnaviv_obj->gem_node, &priv->gem_list);
 	mutex_unlock(&priv->gem_lock);
-
-	return 0;
 }
 
 static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags,
@@ -640,8 +641,9 @@ static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags,
 	return 0;
 }
 
-static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
-		u32 size, u32 flags)
+/* convenience method to construct a GEM buffer object, and userspace handle */
+int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file,
+	u32 size, u32 flags, u32 *handle)
 {
 	struct drm_gem_object *obj = NULL;
 	int ret;
@@ -653,6 +655,8 @@ static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
+	lockdep_set_class(&to_etnaviv_bo(obj)->lock, &etnaviv_shm_lock_class);
+
 	ret = drm_gem_object_init(dev, obj, size);
 	if (ret == 0) {
 		struct address_space *mapping;
@@ -660,7 +664,7 @@ static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
 		/*
 		 * Our buffers are kept pinned, so allocating them
 		 * from the MOVABLE zone is a really bad idea, and
-		 * conflicts with CMA.  See coments above new_inode()
+		 * conflicts with CMA. See comments above new_inode()
 		 * why this is required _and_ expected if you're
 		 * going to pin these pages.
 		 */
@@ -672,33 +676,12 @@ static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
-	return obj;
-
-fail:
-	drm_gem_object_put_unlocked(obj);
-	return ERR_PTR(ret);
-}
-
-/* convenience method to construct a GEM buffer object, and userspace handle */
-int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file,
-		u32 size, u32 flags, u32 *handle)
-{
-	struct drm_gem_object *obj;
-	int ret;
-
-	obj = __etnaviv_gem_new(dev, size, flags);
-	if (IS_ERR(obj))
-		return PTR_ERR(obj);
-
-	ret = etnaviv_gem_obj_add(dev, obj);
-	if (ret < 0) {
-		drm_gem_object_put_unlocked(obj);
-		return ret;
-	}
+	etnaviv_gem_obj_add(dev, obj);
 
 	ret = drm_gem_handle_create(file, obj, handle);
 
 	/* drop reference from allocate - handle holds it now */
+fail:
 	drm_gem_object_put_unlocked(obj);
 
 	return ret;
@@ -722,139 +705,41 @@ int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags,
 	return 0;
 }
 
-struct get_pages_work {
-	struct work_struct work;
-	struct mm_struct *mm;
-	struct task_struct *task;
-	struct etnaviv_gem_object *etnaviv_obj;
-};
-
-static struct page **etnaviv_gem_userptr_do_get_pages(
-	struct etnaviv_gem_object *etnaviv_obj, struct mm_struct *mm, struct task_struct *task)
-{
-	int ret = 0, pinned, npages = etnaviv_obj->base.size >> PAGE_SHIFT;
-	struct page **pvec;
-	uintptr_t ptr;
-	unsigned int flags = 0;
-
-	pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
-	if (!pvec)
-		return ERR_PTR(-ENOMEM);
-
-	if (!etnaviv_obj->userptr.ro)
-		flags |= FOLL_WRITE;
-
-	pinned = 0;
-	ptr = etnaviv_obj->userptr.ptr;
-
-	down_read(&mm->mmap_sem);
-	while (pinned < npages) {
-		ret = get_user_pages_remote(task, mm, ptr, npages - pinned,
-					    flags, pvec + pinned, NULL, NULL);
-		if (ret < 0)
-			break;
-
-		ptr += ret * PAGE_SIZE;
-		pinned += ret;
-	}
-	up_read(&mm->mmap_sem);
-
-	if (ret < 0) {
-		release_pages(pvec, pinned);
-		kvfree(pvec);
-		return ERR_PTR(ret);
-	}
-
-	return pvec;
-}
-
-static void __etnaviv_gem_userptr_get_pages(struct work_struct *_work)
-{
-	struct get_pages_work *work = container_of(_work, typeof(*work), work);
-	struct etnaviv_gem_object *etnaviv_obj = work->etnaviv_obj;
-	struct page **pvec;
-
-	pvec = etnaviv_gem_userptr_do_get_pages(etnaviv_obj, work->mm, work->task);
-
-	mutex_lock(&etnaviv_obj->lock);
-	if (IS_ERR(pvec)) {
-		etnaviv_obj->userptr.work = ERR_CAST(pvec);
-	} else {
-		etnaviv_obj->userptr.work = NULL;
-		etnaviv_obj->pages = pvec;
-	}
-
-	mutex_unlock(&etnaviv_obj->lock);
-	drm_gem_object_put_unlocked(&etnaviv_obj->base);
-
-	mmput(work->mm);
-	put_task_struct(work->task);
-	kfree(work);
-}
-
 static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj)
 {
 	struct page **pvec = NULL;
-	struct get_pages_work *work;
-	struct mm_struct *mm;
-	int ret, pinned, npages = etnaviv_obj->base.size >> PAGE_SHIFT;
-
-	if (etnaviv_obj->userptr.work) {
-		if (IS_ERR(etnaviv_obj->userptr.work)) {
-			ret = PTR_ERR(etnaviv_obj->userptr.work);
-			etnaviv_obj->userptr.work = NULL;
-		} else {
-			ret = -EAGAIN;
-		}
-		return ret;
-	}
+	struct etnaviv_gem_userptr *userptr = &etnaviv_obj->userptr;
+	int ret, pinned = 0, npages = etnaviv_obj->base.size >> PAGE_SHIFT;
 
-	mm = get_task_mm(etnaviv_obj->userptr.task);
-	pinned = 0;
-	if (mm == current->mm) {
-		pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
-		if (!pvec) {
-			mmput(mm);
-			return -ENOMEM;
-		}
-
-		pinned = __get_user_pages_fast(etnaviv_obj->userptr.ptr, npages,
-					       !etnaviv_obj->userptr.ro, pvec);
-		if (pinned < 0) {
-			kvfree(pvec);
-			mmput(mm);
-			return pinned;
-		}
-
-		if (pinned == npages) {
-			etnaviv_obj->pages = pvec;
-			mmput(mm);
-			return 0;
-		}
-	}
+	might_lock_read(&current->mm->mmap_sem);
 
-	release_pages(pvec, pinned);
-	kvfree(pvec);
+	if (userptr->mm != current->mm)
+		return -EPERM;
 
-	work = kmalloc(sizeof(*work), GFP_KERNEL);
-	if (!work) {
-		mmput(mm);
+	pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
+	if (!pvec)
 		return -ENOMEM;
-	}
 
-	get_task_struct(current);
-	drm_gem_object_get(&etnaviv_obj->base);
+	do {
+		unsigned num_pages = npages - pinned;
+		uint64_t ptr = userptr->ptr + pinned * PAGE_SIZE;
+		struct page **pages = pvec + pinned;
 
-	work->mm = mm;
-	work->task = current;
-	work->etnaviv_obj = etnaviv_obj;
+		ret = get_user_pages_fast(ptr, num_pages,
+					  !userptr->ro ? FOLL_WRITE : 0, pages);
+		if (ret < 0) {
+			release_pages(pvec, pinned);
+			kvfree(pvec);
+			return ret;
+		}
+
+		pinned += ret;
 
-	etnaviv_obj->userptr.work = &work->work;
-	INIT_WORK(&work->work, __etnaviv_gem_userptr_get_pages);
+	} while (pinned < npages);
 
-	etnaviv_queue_work(etnaviv_obj->base.dev, &work->work);
+	etnaviv_obj->pages = pvec;
 
-	return -EAGAIN;
+	return 0;
 }
 
 static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj)
@@ -870,7 +755,6 @@ static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj)
 		release_pages(etnaviv_obj->pages, npages);
 		kvfree(etnaviv_obj->pages);
 	}
-	put_task_struct(etnaviv_obj->userptr.task);
 }
 
 static int etnaviv_gem_userptr_mmap_obj(struct etnaviv_gem_object *etnaviv_obj,
@@ -897,17 +781,16 @@ int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file,
 	if (ret)
 		return ret;
 
+	lockdep_set_class(&etnaviv_obj->lock, &etnaviv_userptr_lock_class);
+
 	etnaviv_obj->userptr.ptr = ptr;
-	etnaviv_obj->userptr.task = current;
+	etnaviv_obj->userptr.mm = current->mm;
 	etnaviv_obj->userptr.ro = !(flags & ETNA_USERPTR_WRITE);
-	get_task_struct(current);
 
-	ret = etnaviv_gem_obj_add(dev, &etnaviv_obj->base);
-	if (ret)
-		goto unreference;
+	etnaviv_gem_obj_add(dev, &etnaviv_obj->base);
 
 	ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle);
-unreference:
+
 	/* drop reference from allocate - handle holds it now */
 	drm_gem_object_put_unlocked(&etnaviv_obj->base);
 	return ret;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
index e437fba1209d..be72a9833f2b 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
@@ -18,6 +18,7 @@
 #define __ETNAVIV_GEM_H__
 
 #include <linux/reservation.h>
+#include "etnaviv_cmdbuf.h"
 #include "etnaviv_drv.h"
 
 struct dma_fence;
@@ -26,8 +27,7 @@ struct etnaviv_gem_object;
 
 struct etnaviv_gem_userptr {
 	uintptr_t ptr;
-	struct task_struct *task;
-	struct work_struct *work;
+	struct mm_struct *mm;
 	bool ro;
 };
 
@@ -98,26 +98,32 @@ struct etnaviv_gem_submit_bo {
 
 /* Created per submit-ioctl, to track bo's and cmdstream bufs, etc,
  * associated with the cmdstream submission for synchronization (and
- * make it easier to unwind when things go wrong, etc).  This only
- * lasts for the duration of the submit-ioctl.
+ * make it easier to unwind when things go wrong, etc).
  */
 struct etnaviv_gem_submit {
-	struct drm_device *dev;
+	struct kref refcount;
 	struct etnaviv_gpu *gpu;
-	struct ww_acquire_ctx ticket;
-	struct dma_fence *fence;
+	struct dma_fence *out_fence, *in_fence;
+	struct list_head node; /* GPU active submit list */
+	struct etnaviv_cmdbuf cmdbuf;
+	bool runtime_resumed;
+	u32 exec_state;
 	u32 flags;
+	unsigned int nr_pmrs;
+	struct etnaviv_perfmon_request *pmrs;
 	unsigned int nr_bos;
 	struct etnaviv_gem_submit_bo bos[0];
 	/* No new members here, the previous one is variable-length! */
 };
 
+void etnaviv_submit_put(struct etnaviv_gem_submit * submit);
+
 int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj,
 	struct timespec *timeout);
 int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags,
 	struct reservation_object *robj, const struct etnaviv_gem_ops *ops,
 	struct etnaviv_gem_object **res);
-int etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj);
+void etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj);
 struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *obj);
 void etnaviv_gem_put_pages(struct etnaviv_gem_object *obj);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
index ae884723e9b1..5704305d41e6 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -19,6 +19,7 @@
 #include "etnaviv_drv.h"
 #include "etnaviv_gem.h"
 
+static struct lock_class_key etnaviv_prime_lock_class;
 
 struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj)
 {
@@ -125,6 +126,8 @@ struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
 	if (ret < 0)
 		return ERR_PTR(ret);
 
+	lockdep_set_class(&etnaviv_obj->lock, &etnaviv_prime_lock_class);
+
 	npages = size / PAGE_SIZE;
 
 	etnaviv_obj->sgt = sgt;
@@ -139,9 +142,7 @@ struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
 	if (ret)
 		goto fail;
 
-	ret = etnaviv_gem_obj_add(dev, &etnaviv_obj->base);
-	if (ret)
-		goto fail;
+	etnaviv_gem_obj_add(dev, &etnaviv_obj->base);
 
 	return &etnaviv_obj->base;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
index ff911541a190..1f8202bca061 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -33,22 +33,25 @@
 #define BO_PINNED   0x2000
 
 static struct etnaviv_gem_submit *submit_create(struct drm_device *dev,
-		struct etnaviv_gpu *gpu, size_t nr)
+		struct etnaviv_gpu *gpu, size_t nr_bos, size_t nr_pmrs)
 {
 	struct etnaviv_gem_submit *submit;
-	size_t sz = size_vstruct(nr, sizeof(submit->bos[0]), sizeof(*submit));
+	size_t sz = size_vstruct(nr_bos, sizeof(submit->bos[0]), sizeof(*submit));
 
-	submit = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
-	if (submit) {
-		submit->dev = dev;
-		submit->gpu = gpu;
+	submit = kzalloc(sz, GFP_KERNEL);
+	if (!submit)
+		return NULL;
 
-		/* initially, until copy_from_user() and bo lookup succeeds: */
-		submit->nr_bos = 0;
-		submit->fence = NULL;
-
-		ww_acquire_init(&submit->ticket, &reservation_ww_class);
+	submit->pmrs = kcalloc(nr_pmrs, sizeof(struct etnaviv_perfmon_request),
+			       GFP_KERNEL);
+	if (!submit->pmrs) {
+		kfree(submit);
+		return NULL;
 	}
+	submit->nr_pmrs = nr_pmrs;
+
+	submit->gpu = gpu;
+	kref_init(&submit->refcount);
 
 	return submit;
 }
@@ -111,7 +114,8 @@ static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i)
 	}
 }
 
-static int submit_lock_objects(struct etnaviv_gem_submit *submit)
+static int submit_lock_objects(struct etnaviv_gem_submit *submit,
+		struct ww_acquire_ctx *ticket)
 {
 	int contended, slow_locked = -1, i, ret = 0;
 
@@ -126,7 +130,7 @@ static int submit_lock_objects(struct etnaviv_gem_submit *submit)
 
 		if (!(submit->bos[i].flags & BO_LOCKED)) {
 			ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock,
-					&submit->ticket);
+							  ticket);
 			if (ret == -EALREADY)
 				DRM_ERROR("BO at index %u already on submit list\n",
 					  i);
@@ -136,7 +140,7 @@ static int submit_lock_objects(struct etnaviv_gem_submit *submit)
 		}
 	}
 
-	ww_acquire_done(&submit->ticket);
+	ww_acquire_done(ticket);
 
 	return 0;
 
@@ -154,7 +158,7 @@ static int submit_lock_objects(struct etnaviv_gem_submit *submit)
 
 		/* we lost out in a seqno race, lock and retry.. */
 		ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock,
-				&submit->ticket);
+						       ticket);
 		if (!ret) {
 			submit->bos[contended].flags |= BO_LOCKED;
 			slow_locked = contended;
@@ -181,19 +185,33 @@ static int submit_fence_sync(const struct etnaviv_gem_submit *submit)
 			break;
 	}
 
+	if (submit->flags & ETNA_SUBMIT_FENCE_FD_IN) {
+		/*
+		 * Wait if the fence is from a foreign context, or if the fence
+		 * array contains any fence from a foreign context.
+		 */
+		if (!dma_fence_match_context(submit->in_fence, context))
+			ret = dma_fence_wait(submit->in_fence, true);
+	}
+
 	return ret;
 }
 
-static void submit_unpin_objects(struct etnaviv_gem_submit *submit)
+static void submit_attach_object_fences(struct etnaviv_gem_submit *submit)
 {
 	int i;
 
 	for (i = 0; i < submit->nr_bos; i++) {
-		if (submit->bos[i].flags & BO_PINNED)
-			etnaviv_gem_mapping_unreference(submit->bos[i].mapping);
+		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+
+		if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE)
+			reservation_object_add_excl_fence(etnaviv_obj->resv,
+							  submit->out_fence);
+		else
+			reservation_object_add_shared_fence(etnaviv_obj->resv,
+							    submit->out_fence);
 
-		submit->bos[i].mapping = NULL;
-		submit->bos[i].flags &= ~BO_PINNED;
+		submit_unlock_object(submit, i);
 	}
 }
 
@@ -211,6 +229,7 @@ static int submit_pin_objects(struct etnaviv_gem_submit *submit)
 			ret = PTR_ERR(mapping);
 			break;
 		}
+		atomic_inc(&etnaviv_obj->gpu_active);
 
 		submit->bos[i].flags |= BO_PINNED;
 		submit->bos[i].mapping = mapping;
@@ -285,13 +304,11 @@ static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream,
 }
 
 static int submit_perfmon_validate(struct etnaviv_gem_submit *submit,
-		struct etnaviv_cmdbuf *cmdbuf,
-		const struct drm_etnaviv_gem_submit_pmr *pmrs,
-		u32 nr_pms)
+		u32 exec_state, const struct drm_etnaviv_gem_submit_pmr *pmrs)
 {
 	u32 i;
 
-	for (i = 0; i < nr_pms; i++) {
+	for (i = 0; i < submit->nr_pmrs; i++) {
 		const struct drm_etnaviv_gem_submit_pmr *r = pmrs + i;
 		struct etnaviv_gem_submit_bo *bo;
 		int ret;
@@ -316,39 +333,65 @@ static int submit_perfmon_validate(struct etnaviv_gem_submit *submit,
 			return -EINVAL;
 		}
 
-		if (etnaviv_pm_req_validate(r, cmdbuf->exec_state)) {
+		if (etnaviv_pm_req_validate(r, exec_state)) {
 			DRM_ERROR("perfmon request: domain or signal not valid");
 			return -EINVAL;
 		}
 
-		cmdbuf->pmrs[i].flags = r->flags;
-		cmdbuf->pmrs[i].domain = r->domain;
-		cmdbuf->pmrs[i].signal = r->signal;
-		cmdbuf->pmrs[i].sequence = r->sequence;
-		cmdbuf->pmrs[i].offset = r->read_offset;
-		cmdbuf->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base);
+		submit->pmrs[i].flags = r->flags;
+		submit->pmrs[i].domain = r->domain;
+		submit->pmrs[i].signal = r->signal;
+		submit->pmrs[i].sequence = r->sequence;
+		submit->pmrs[i].offset = r->read_offset;
+		submit->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base);
 	}
 
 	return 0;
 }
 
-static void submit_cleanup(struct etnaviv_gem_submit *submit)
+static void submit_cleanup(struct kref *kref)
 {
+	struct etnaviv_gem_submit *submit =
+			container_of(kref, struct etnaviv_gem_submit, refcount);
 	unsigned i;
 
+	if (submit->runtime_resumed)
+		pm_runtime_put_autosuspend(submit->gpu->dev);
+
+	if (submit->cmdbuf.suballoc)
+		etnaviv_cmdbuf_free(&submit->cmdbuf);
+
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
 
+		/* unpin all objects */
+		if (submit->bos[i].flags & BO_PINNED) {
+			etnaviv_gem_mapping_unreference(submit->bos[i].mapping);
+			atomic_dec(&etnaviv_obj->gpu_active);
+			submit->bos[i].mapping = NULL;
+			submit->bos[i].flags &= ~BO_PINNED;
+		}
+
+		/* if the GPU submit failed, objects might still be locked */
 		submit_unlock_object(submit, i);
 		drm_gem_object_put_unlocked(&etnaviv_obj->base);
 	}
 
-	ww_acquire_fini(&submit->ticket);
-	if (submit->fence)
-		dma_fence_put(submit->fence);
+	wake_up_all(&submit->gpu->fence_event);
+
+	if (submit->in_fence)
+		dma_fence_put(submit->in_fence);
+	if (submit->out_fence)
+		dma_fence_put(submit->out_fence);
+	kfree(submit->pmrs);
 	kfree(submit);
 }
 
+void etnaviv_submit_put(struct etnaviv_gem_submit *submit)
+{
+	kref_put(&submit->refcount, submit_cleanup);
+}
+
 int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 		struct drm_file *file)
 {
@@ -358,10 +401,9 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 	struct drm_etnaviv_gem_submit_pmr *pmrs;
 	struct drm_etnaviv_gem_submit_bo *bos;
 	struct etnaviv_gem_submit *submit;
-	struct etnaviv_cmdbuf *cmdbuf;
 	struct etnaviv_gpu *gpu;
-	struct dma_fence *in_fence = NULL;
 	struct sync_file *sync_file = NULL;
+	struct ww_acquire_ctx ticket;
 	int out_fence_fd = -1;
 	void *stream;
 	int ret;
@@ -399,17 +441,11 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 	relocs = kvmalloc_array(args->nr_relocs, sizeof(*relocs), GFP_KERNEL);
 	pmrs = kvmalloc_array(args->nr_pmrs, sizeof(*pmrs), GFP_KERNEL);
 	stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL);
-	cmdbuf = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc,
-				    ALIGN(args->stream_size, 8) + 8,
-				    args->nr_bos, args->nr_pmrs);
-	if (!bos || !relocs || !pmrs || !stream || !cmdbuf) {
+	if (!bos || !relocs || !pmrs || !stream) {
 		ret = -ENOMEM;
 		goto err_submit_cmds;
 	}
 
-	cmdbuf->exec_state = args->exec_state;
-	cmdbuf->ctx = file->driver_priv;
-
 	ret = copy_from_user(bos, u64_to_user_ptr(args->bos),
 			     args->nr_bos * sizeof(*bos));
 	if (ret) {
@@ -430,7 +466,6 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 		ret = -EFAULT;
 		goto err_submit_cmds;
 	}
-	cmdbuf->nr_pmrs = args->nr_pmrs;
 
 	ret = copy_from_user(stream, u64_to_user_ptr(args->stream),
 			     args->stream_size);
@@ -447,19 +482,28 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 		}
 	}
 
-	submit = submit_create(dev, gpu, args->nr_bos);
+	ww_acquire_init(&ticket, &reservation_ww_class);
+
+	submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs);
 	if (!submit) {
 		ret = -ENOMEM;
-		goto err_submit_cmds;
+		goto err_submit_ww_acquire;
 	}
 
+	ret = etnaviv_cmdbuf_init(gpu->cmdbuf_suballoc, &submit->cmdbuf,
+				  ALIGN(args->stream_size, 8) + 8);
+	if (ret)
+		goto err_submit_objects;
+
+	submit->cmdbuf.ctx = file->driver_priv;
+	submit->exec_state = args->exec_state;
 	submit->flags = args->flags;
 
 	ret = submit_lookup_objects(submit, file, bos, args->nr_bos);
 	if (ret)
 		goto err_submit_objects;
 
-	ret = submit_lock_objects(submit);
+	ret = submit_lock_objects(submit, &ticket);
 	if (ret)
 		goto err_submit_objects;
 
@@ -470,21 +514,11 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 	}
 
 	if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) {
-		in_fence = sync_file_get_fence(args->fence_fd);
-		if (!in_fence) {
+		submit->in_fence = sync_file_get_fence(args->fence_fd);
+		if (!submit->in_fence) {
 			ret = -EINVAL;
 			goto err_submit_objects;
 		}
-
-		/*
-		 * Wait if the fence is from a foreign context, or if the fence
-		 * array contains any fence from a foreign context.
-		 */
-		if (!dma_fence_match_context(in_fence, gpu->fence_context)) {
-			ret = dma_fence_wait(in_fence, true);
-			if (ret)
-				goto err_submit_objects;
-		}
 	}
 
 	ret = submit_fence_sync(submit);
@@ -493,25 +527,25 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 
 	ret = submit_pin_objects(submit);
 	if (ret)
-		goto out;
+		goto err_submit_objects;
 
 	ret = submit_reloc(submit, stream, args->stream_size / 4,
 			   relocs, args->nr_relocs);
 	if (ret)
-		goto out;
+		goto err_submit_objects;
 
-	ret = submit_perfmon_validate(submit, cmdbuf, pmrs, args->nr_pmrs);
+	ret = submit_perfmon_validate(submit, args->exec_state, pmrs);
 	if (ret)
-		goto out;
+		goto err_submit_objects;
 
-	memcpy(cmdbuf->vaddr, stream, args->stream_size);
-	cmdbuf->user_size = ALIGN(args->stream_size, 8);
+	memcpy(submit->cmdbuf.vaddr, stream, args->stream_size);
+	submit->cmdbuf.user_size = ALIGN(args->stream_size, 8);
 
-	ret = etnaviv_gpu_submit(gpu, submit, cmdbuf);
+	ret = etnaviv_gpu_submit(gpu, submit);
 	if (ret)
-		goto out;
+		goto err_submit_objects;
 
-	cmdbuf = NULL;
+	submit_attach_object_fences(submit);
 
 	if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) {
 		/*
@@ -520,39 +554,26 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
 		 * fence to the sync file here, eliminating the ENOMEM
 		 * possibility at this stage.
 		 */
-		sync_file = sync_file_create(submit->fence);
+		sync_file = sync_file_create(submit->out_fence);
 		if (!sync_file) {
 			ret = -ENOMEM;
-			goto out;
+			goto err_submit_objects;
 		}
 		fd_install(out_fence_fd, sync_file->file);
 	}
 
 	args->fence_fd = out_fence_fd;
-	args->fence = submit->fence->seqno;
-
-out:
-	submit_unpin_objects(submit);
-
-	/*
-	 * If we're returning -EAGAIN, it may be due to the userptr code
-	 * wanting to run its workqueue outside of any locks. Flush our
-	 * workqueue to ensure that it is run in a timely manner.
-	 */
-	if (ret == -EAGAIN)
-		flush_workqueue(priv->wq);
+	args->fence = submit->out_fence->seqno;
 
 err_submit_objects:
-	if (in_fence)
-		dma_fence_put(in_fence);
-	submit_cleanup(submit);
+	etnaviv_submit_put(submit);
+
+err_submit_ww_acquire:
+	ww_acquire_fini(&ticket);
 
 err_submit_cmds:
 	if (ret && (out_fence_fd >= 0))
 		put_unused_fd(out_fence_fd);
-	/* if we still own the cmdbuf */
-	if (cmdbuf)
-		etnaviv_cmdbuf_free(cmdbuf);
 	if (stream)
 		kvfree(stream);
 	if (bos)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index e19cbe05da2a..21d0d22f1168 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -644,7 +644,7 @@ static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu)
 	prefetch = etnaviv_buffer_init(gpu);
 
 	gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U);
-	etnaviv_gpu_start_fe(gpu, etnaviv_cmdbuf_get_va(gpu->buffer),
+	etnaviv_gpu_start_fe(gpu, etnaviv_cmdbuf_get_va(&gpu->buffer),
 			     prefetch);
 }
 
@@ -717,15 +717,15 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
 	}
 
 	/* Create buffer: */
-	gpu->buffer = etnaviv_cmdbuf_new(gpu->cmdbuf_suballoc, PAGE_SIZE, 0, 0);
-	if (!gpu->buffer) {
-		ret = -ENOMEM;
+	ret = etnaviv_cmdbuf_init(gpu->cmdbuf_suballoc, &gpu->buffer,
+				  PAGE_SIZE);
+	if (ret) {
 		dev_err(gpu->dev, "could not create command buffer\n");
 		goto destroy_iommu;
 	}
 
 	if (gpu->mmu->version == ETNAVIV_IOMMU_V1 &&
-	    etnaviv_cmdbuf_get_va(gpu->buffer) > 0x80000000) {
+	    etnaviv_cmdbuf_get_va(&gpu->buffer) > 0x80000000) {
 		ret = -EINVAL;
 		dev_err(gpu->dev,
 			"command buffer outside valid memory window\n");
@@ -751,8 +751,7 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
 	return 0;
 
 free_buffer:
-	etnaviv_cmdbuf_free(gpu->buffer);
-	gpu->buffer = NULL;
+	etnaviv_cmdbuf_free(&gpu->buffer);
 destroy_iommu:
 	etnaviv_iommu_destroy(gpu->mmu);
 	gpu->mmu = NULL;
@@ -958,7 +957,7 @@ static void recover_worker(struct work_struct *work)
 	pm_runtime_put_autosuspend(gpu->dev);
 
 	/* Retire the buffer objects in a work */
-	etnaviv_queue_work(gpu->drm, &gpu->retire_work);
+	queue_work(gpu->wq, &gpu->retire_work);
 }
 
 static void hangcheck_timer_reset(struct etnaviv_gpu *gpu)
@@ -994,7 +993,7 @@ static void hangcheck_handler(struct timer_list *t)
 		dev_err(gpu->dev, "     completed fence: %u\n", fence);
 		dev_err(gpu->dev, "     active fence: %u\n",
 			gpu->active_fence);
-		etnaviv_queue_work(gpu->drm, &gpu->recover_work);
+		queue_work(gpu->wq, &gpu->recover_work);
 	}
 
 	/* if still more pending work, reset the hangcheck timer: */
@@ -1201,42 +1200,23 @@ static void retire_worker(struct work_struct *work)
 	struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu,
 					       retire_work);
 	u32 fence = gpu->completed_fence;
-	struct etnaviv_cmdbuf *cmdbuf, *tmp;
-	unsigned int i;
+	struct etnaviv_gem_submit *submit, *tmp;
+	LIST_HEAD(retire_list);
 
 	mutex_lock(&gpu->lock);
-	list_for_each_entry_safe(cmdbuf, tmp, &gpu->active_cmd_list, node) {
-		if (!dma_fence_is_signaled(cmdbuf->fence))
+	list_for_each_entry_safe(submit, tmp, &gpu->active_submit_list, node) {
+		if (!dma_fence_is_signaled(submit->out_fence))
 			break;
 
-		list_del(&cmdbuf->node);
-		dma_fence_put(cmdbuf->fence);
-
-		for (i = 0; i < cmdbuf->nr_bos; i++) {
-			struct etnaviv_vram_mapping *mapping = cmdbuf->bo_map[i];
-			struct etnaviv_gem_object *etnaviv_obj = mapping->object;
-
-			atomic_dec(&etnaviv_obj->gpu_active);
-			/* drop the refcount taken in etnaviv_gpu_submit */
-			etnaviv_gem_mapping_unreference(mapping);
-		}
-
-		etnaviv_cmdbuf_free(cmdbuf);
-		/*
-		 * We need to balance the runtime PM count caused by
-		 * each submission.  Upon submission, we increment
-		 * the runtime PM counter, and allocate one event.
-		 * So here, we put the runtime PM count for each
-		 * completed event.
-		 */
-		pm_runtime_put_autosuspend(gpu->dev);
+		list_move(&submit->node, &retire_list);
 	}
 
 	gpu->retired_fence = fence;
 
 	mutex_unlock(&gpu->lock);
 
-	wake_up_all(&gpu->fence_event);
+	list_for_each_entry_safe(submit, tmp, &retire_list, node)
+		etnaviv_submit_put(submit);
 }
 
 int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu,
@@ -1295,41 +1275,25 @@ int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu,
 	ret = wait_event_interruptible_timeout(gpu->fence_event,
 					       !is_active(etnaviv_obj),
 					       remaining);
-	if (ret > 0) {
-		struct etnaviv_drm_private *priv = gpu->drm->dev_private;
-
-		/* Synchronise with the retire worker */
-		flush_workqueue(priv->wq);
+	if (ret > 0)
 		return 0;
-	} else if (ret == -ERESTARTSYS) {
+	else if (ret == -ERESTARTSYS)
 		return -ERESTARTSYS;
-	} else {
+	else
 		return -ETIMEDOUT;
-	}
-}
-
-int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu)
-{
-	return pm_runtime_get_sync(gpu->dev);
-}
-
-void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu)
-{
-	pm_runtime_mark_last_busy(gpu->dev);
-	pm_runtime_put_autosuspend(gpu->dev);
 }
 
 static void sync_point_perfmon_sample(struct etnaviv_gpu *gpu,
 	struct etnaviv_event *event, unsigned int flags)
 {
-	const struct etnaviv_cmdbuf *cmdbuf = event->cmdbuf;
+	const struct etnaviv_gem_submit *submit = event->submit;
 	unsigned int i;
 
-	for (i = 0; i < cmdbuf->nr_pmrs; i++) {
-		const struct etnaviv_perfmon_request *pmr = cmdbuf->pmrs + i;
+	for (i = 0; i < submit->nr_pmrs; i++) {
+		const struct etnaviv_perfmon_request *pmr = submit->pmrs + i;
 
 		if (pmr->flags == flags)
-			etnaviv_perfmon_process(gpu, pmr);
+			etnaviv_perfmon_process(gpu, pmr, submit->exec_state);
 	}
 }
 
@@ -1354,14 +1318,14 @@ static void sync_point_perfmon_sample_pre(struct etnaviv_gpu *gpu,
 static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu,
 	struct etnaviv_event *event)
 {
-	const struct etnaviv_cmdbuf *cmdbuf = event->cmdbuf;
+	const struct etnaviv_gem_submit *submit = event->submit;
 	unsigned int i;
 	u32 val;
 
 	sync_point_perfmon_sample(gpu, event, ETNA_PM_PROCESS_POST);
 
-	for (i = 0; i < cmdbuf->nr_pmrs; i++) {
-		const struct etnaviv_perfmon_request *pmr = cmdbuf->pmrs + i;
+	for (i = 0; i < submit->nr_pmrs; i++) {
+		const struct etnaviv_perfmon_request *pmr = submit->pmrs + i;
 
 		*pmr->bo_vma = pmr->sequence;
 	}
@@ -1380,24 +1344,15 @@ static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu,
 
 /* add bo's to gpu's ring, and kick gpu: */
 int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
-	struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf)
+	struct etnaviv_gem_submit *submit)
 {
-	struct dma_fence *fence;
 	unsigned int i, nr_events = 1, event[3];
 	int ret;
 
-	ret = etnaviv_gpu_pm_get_sync(gpu);
+	ret = pm_runtime_get_sync(gpu->dev);
 	if (ret < 0)
 		return ret;
-
-	/*
-	 * TODO
-	 *
-	 * - flush
-	 * - data endian
-	 * - prefetch
-	 *
-	 */
+	submit->runtime_resumed = true;
 
 	/*
 	 * if there are performance monitor requests we need to have
@@ -1406,19 +1361,19 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
 	 * - a sync point to re-configure gpu, process ETNA_PM_PROCESS_POST requests
 	 *   and update the sequence number for userspace.
 	 */
-	if (cmdbuf->nr_pmrs)
+	if (submit->nr_pmrs)
 		nr_events = 3;
 
 	ret = event_alloc(gpu, nr_events, event);
 	if (ret) {
 		DRM_ERROR("no free events\n");
-		goto out_pm_put;
+		return ret;
 	}
 
 	mutex_lock(&gpu->lock);
 
-	fence = etnaviv_gpu_fence_alloc(gpu);
-	if (!fence) {
+	submit->out_fence = etnaviv_gpu_fence_alloc(gpu);
+	if (!submit->out_fence) {
 		for (i = 0; i < nr_events; i++)
 			event_free(gpu, event[i]);
 
@@ -1426,80 +1381,51 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
 		goto out_unlock;
 	}
 
-	gpu->event[event[0]].fence = fence;
-	submit->fence = dma_fence_get(fence);
-	gpu->active_fence = submit->fence->seqno;
+	gpu->active_fence = submit->out_fence->seqno;
 
-	if (gpu->lastctx != cmdbuf->ctx) {
-		gpu->mmu->need_flush = true;
-		gpu->switch_context = true;
-		gpu->lastctx = cmdbuf->ctx;
-	}
-
-	if (cmdbuf->nr_pmrs) {
+	if (submit->nr_pmrs) {
 		gpu->event[event[1]].sync_point = &sync_point_perfmon_sample_pre;
-		gpu->event[event[1]].cmdbuf = cmdbuf;
+		kref_get(&submit->refcount);
+		gpu->event[event[1]].submit = submit;
 		etnaviv_sync_point_queue(gpu, event[1]);
 	}
 
-	etnaviv_buffer_queue(gpu, event[0], cmdbuf);
+	kref_get(&submit->refcount);
+	gpu->event[event[0]].fence = submit->out_fence;
+	etnaviv_buffer_queue(gpu, submit->exec_state, event[0],
+			     &submit->cmdbuf);
 
-	if (cmdbuf->nr_pmrs) {
+	if (submit->nr_pmrs) {
 		gpu->event[event[2]].sync_point = &sync_point_perfmon_sample_post;
-		gpu->event[event[2]].cmdbuf = cmdbuf;
+		kref_get(&submit->refcount);
+		gpu->event[event[2]].submit = submit;
 		etnaviv_sync_point_queue(gpu, event[2]);
 	}
 
-	cmdbuf->fence = fence;
-	list_add_tail(&cmdbuf->node, &gpu->active_cmd_list);
-
-	/* We're committed to adding this command buffer, hold a PM reference */
-	pm_runtime_get_noresume(gpu->dev);
-
-	for (i = 0; i < submit->nr_bos; i++) {
-		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+	list_add_tail(&submit->node, &gpu->active_submit_list);
 
-		/* Each cmdbuf takes a refcount on the mapping */
-		etnaviv_gem_mapping_reference(submit->bos[i].mapping);
-		cmdbuf->bo_map[i] = submit->bos[i].mapping;
-		atomic_inc(&etnaviv_obj->gpu_active);
-
-		if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE)
-			reservation_object_add_excl_fence(etnaviv_obj->resv,
-							  fence);
-		else
-			reservation_object_add_shared_fence(etnaviv_obj->resv,
-							    fence);
-	}
-	cmdbuf->nr_bos = submit->nr_bos;
 	hangcheck_timer_reset(gpu);
 	ret = 0;
 
 out_unlock:
 	mutex_unlock(&gpu->lock);
 
-out_pm_put:
-	etnaviv_gpu_pm_put(gpu);
-
 	return ret;
 }
 
-static void etnaviv_process_sync_point(struct etnaviv_gpu *gpu,
-	struct etnaviv_event *event)
-{
-	u32 addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
-
-	event->sync_point(gpu, event);
-	etnaviv_gpu_start_fe(gpu, addr + 2, 2);
-}
-
 static void sync_point_worker(struct work_struct *work)
 {
 	struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu,
 					       sync_point_work);
+	struct etnaviv_event *event = &gpu->event[gpu->sync_point_event];
+	u32 addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
 
-	etnaviv_process_sync_point(gpu, &gpu->event[gpu->sync_point_event]);
+	event->sync_point(gpu, event);
+	etnaviv_submit_put(event->submit);
 	event_free(gpu, gpu->sync_point_event);
+
+	/* restart FE last to avoid GPU and IRQ racing against this worker */
+	etnaviv_gpu_start_fe(gpu, addr + 2, 2);
 }
 
 /*
@@ -1550,7 +1476,7 @@ static irqreturn_t irq_handler(int irq, void *data)
 
 			if (gpu->event[event].sync_point) {
 				gpu->sync_point_event = event;
-				etnaviv_queue_work(gpu->drm, &gpu->sync_point_work);
+				queue_work(gpu->wq, &gpu->sync_point_work);
 			}
 
 			fence = gpu->event[event].fence;
@@ -1576,7 +1502,7 @@ static irqreturn_t irq_handler(int irq, void *data)
 		}
 
 		/* Retire the buffer objects in a work */
-		etnaviv_queue_work(gpu->drm, &gpu->retire_work);
+		queue_work(gpu->wq, &gpu->retire_work);
 
 		ret = IRQ_HANDLED;
 	}
@@ -1653,9 +1579,11 @@ int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms)
 
 static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu *gpu)
 {
-	if (gpu->buffer) {
+	if (gpu->buffer.suballoc) {
 		/* Replace the last WAIT with END */
+		mutex_lock(&gpu->lock);
 		etnaviv_buffer_end(gpu);
+		mutex_unlock(&gpu->lock);
 
 		/*
 		 * We know that only the FE is busy here, this should
@@ -1680,7 +1608,7 @@ static int etnaviv_gpu_hw_resume(struct etnaviv_gpu *gpu)
 	etnaviv_gpu_update_clock(gpu);
 	etnaviv_gpu_hw_init(gpu);
 
-	gpu->switch_context = true;
+	gpu->lastctx = NULL;
 	gpu->exec_state = -1;
 
 	mutex_unlock(&gpu->lock);
@@ -1738,20 +1666,29 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
 	struct etnaviv_gpu *gpu = dev_get_drvdata(dev);
 	int ret;
 
-	if (IS_ENABLED(CONFIG_THERMAL)) {
+	if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL)) {
 		gpu->cooling = thermal_of_cooling_device_register(dev->of_node,
 				(char *)dev_name(dev), gpu, &cooling_ops);
 		if (IS_ERR(gpu->cooling))
 			return PTR_ERR(gpu->cooling);
 	}
 
+	gpu->wq = alloc_ordered_workqueue(dev_name(dev), 0);
+	if (!gpu->wq) {
+		if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL))
+			thermal_cooling_device_unregister(gpu->cooling);
+		return -ENOMEM;
+	}
+
 #ifdef CONFIG_PM
 	ret = pm_runtime_get_sync(gpu->dev);
 #else
 	ret = etnaviv_gpu_clk_enable(gpu);
 #endif
 	if (ret < 0) {
-		thermal_cooling_device_unregister(gpu->cooling);
+		destroy_workqueue(gpu->wq);
+		if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL))
+			thermal_cooling_device_unregister(gpu->cooling);
 		return ret;
 	}
 
@@ -1759,7 +1696,7 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master,
 	gpu->fence_context = dma_fence_context_alloc(1);
 	spin_lock_init(&gpu->fence_spinlock);
 
-	INIT_LIST_HEAD(&gpu->active_cmd_list);
+	INIT_LIST_HEAD(&gpu->active_submit_list);
 	INIT_WORK(&gpu->retire_work, retire_worker);
 	INIT_WORK(&gpu->sync_point_work, sync_point_worker);
 	INIT_WORK(&gpu->recover_work, recover_worker);
@@ -1784,6 +1721,9 @@ static void etnaviv_gpu_unbind(struct device *dev, struct device *master,
 
 	hangcheck_disable(gpu);
 
+	flush_workqueue(gpu->wq);
+	destroy_workqueue(gpu->wq);
+
 #ifdef CONFIG_PM
 	pm_runtime_get_sync(gpu->dev);
 	pm_runtime_put_sync_suspend(gpu->dev);
@@ -1791,10 +1731,8 @@ static void etnaviv_gpu_unbind(struct device *dev, struct device *master,
 	etnaviv_gpu_hw_suspend(gpu);
 #endif
 
-	if (gpu->buffer) {
-		etnaviv_cmdbuf_free(gpu->buffer);
-		gpu->buffer = NULL;
-	}
+	if (gpu->buffer.suballoc)
+		etnaviv_cmdbuf_free(&gpu->buffer);
 
 	if (gpu->cmdbuf_suballoc) {
 		etnaviv_cmdbuf_suballoc_destroy(gpu->cmdbuf_suballoc);
@@ -1808,7 +1746,8 @@ static void etnaviv_gpu_unbind(struct device *dev, struct device *master,
 
 	gpu->drm = NULL;
 
-	thermal_cooling_device_unregister(gpu->cooling);
+	if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL))
+		thermal_cooling_device_unregister(gpu->cooling);
 	gpu->cooling = NULL;
 }
 
@@ -1931,7 +1870,7 @@ static int etnaviv_gpu_rpm_resume(struct device *dev)
 		return ret;
 
 	/* Re-initialise the basic hardware state */
-	if (gpu->drm && gpu->buffer) {
+	if (gpu->drm && gpu->buffer.suballoc) {
 		ret = etnaviv_gpu_hw_resume(gpu);
 		if (ret) {
 			etnaviv_gpu_clk_disable(gpu);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index 4f10f147297a..7623905210dc 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -20,6 +20,7 @@
 #include <linux/clk.h>
 #include <linux/regulator/consumer.h>
 
+#include "etnaviv_cmdbuf.h"
 #include "etnaviv_drv.h"
 
 struct etnaviv_gem_submit;
@@ -89,7 +90,7 @@ struct etnaviv_chip_identity {
 
 struct etnaviv_event {
 	struct dma_fence *fence;
-	struct etnaviv_cmdbuf *cmdbuf;
+	struct etnaviv_gem_submit *submit;
 
 	void (*sync_point)(struct etnaviv_gpu *gpu, struct etnaviv_event *event);
 };
@@ -106,10 +107,10 @@ struct etnaviv_gpu {
 	struct mutex lock;
 	struct etnaviv_chip_identity identity;
 	struct etnaviv_file_private *lastctx;
-	bool switch_context;
+	struct workqueue_struct *wq;
 
 	/* 'ring'-buffer: */
-	struct etnaviv_cmdbuf *buffer;
+	struct etnaviv_cmdbuf buffer;
 	int exec_state;
 
 	/* bus base address of memory  */
@@ -122,7 +123,7 @@ struct etnaviv_gpu {
 	spinlock_t event_spinlock;
 
 	/* list of currently in-flight command buffers */
-	struct list_head active_cmd_list;
+	struct list_head active_submit_list;
 
 	u32 idle_mask;
 
@@ -202,7 +203,7 @@ int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu,
 int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu,
 	struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout);
 int etnaviv_gpu_submit(struct etnaviv_gpu *gpu,
-	struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf);
+	struct etnaviv_gem_submit *submit);
 int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu);
 void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu);
 int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
index 14e24ac6573f..7a8c94731748 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c
@@ -70,9 +70,8 @@ static int __etnaviv_iommu_init(struct etnaviv_iommuv1_domain *etnaviv_domain)
 		return -ENOMEM;
 	}
 
-	for (i = 0; i < PT_ENTRIES; i++)
-		etnaviv_domain->pgtable_cpu[i] =
-				etnaviv_domain->base.bad_page_dma;
+	memset32(etnaviv_domain->pgtable_cpu, etnaviv_domain->base.bad_page_dma,
+		 PT_ENTRIES);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
index fc60fc8ddbf0..1e956e266aa3 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c
@@ -229,7 +229,7 @@ void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu)
 	prefetch = etnaviv_buffer_config_mmuv2(gpu,
 				(u32)etnaviv_domain->mtlb_dma,
 				(u32)etnaviv_domain->base.bad_page_dma);
-	etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(gpu->buffer),
+	etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
 			     prefetch);
 	etnaviv_gpu_wait_idle(gpu, 100);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
index 35074b944778..d113fe06e6b5 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c
@@ -263,18 +263,16 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
 		if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) {
 			mapping->iova = iova;
 			list_add_tail(&mapping->mmu_node, &mmu->mappings);
-			mutex_unlock(&mmu->lock);
-			return 0;
+			ret = 0;
+			goto unlock;
 		}
 	}
 
 	node = &mapping->vram_node;
 
 	ret = etnaviv_iommu_find_iova(mmu, node, etnaviv_obj->base.size);
-	if (ret < 0) {
-		mutex_unlock(&mmu->lock);
-		return ret;
-	}
+	if (ret < 0)
+		goto unlock;
 
 	mmu->last_iova = node->start + etnaviv_obj->base.size;
 	mapping->iova = node->start;
@@ -283,12 +281,12 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
 
 	if (ret < 0) {
 		drm_mm_remove_node(node);
-		mutex_unlock(&mmu->lock);
-		return ret;
+		goto unlock;
 	}
 
 	list_add_tail(&mapping->mmu_node, &mmu->mappings);
 	mmu->need_flush = true;
+unlock:
 	mutex_unlock(&mmu->lock);
 
 	return ret;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
index 768f5aafdd18..26dddfc41aac 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.c
@@ -479,9 +479,9 @@ int etnaviv_pm_req_validate(const struct drm_etnaviv_gem_submit_pmr *r,
 }
 
 void etnaviv_perfmon_process(struct etnaviv_gpu *gpu,
-	const struct etnaviv_perfmon_request *pmr)
+	const struct etnaviv_perfmon_request *pmr, u32 exec_state)
 {
-	const struct etnaviv_pm_domain_meta *meta = &doms_meta[gpu->exec_state];
+	const struct etnaviv_pm_domain_meta *meta = &doms_meta[exec_state];
 	const struct etnaviv_pm_domain *dom;
 	const struct etnaviv_pm_signal *sig;
 	u32 *bo = pmr->bo_vma;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h
index 35dce194cb00..c1653c64ab6b 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_perfmon.h
@@ -44,6 +44,6 @@ int etnaviv_pm_req_validate(const struct drm_etnaviv_gem_submit_pmr *r,
 	u32 exec_state);
 
 void etnaviv_perfmon_process(struct etnaviv_gpu *gpu,
-	const struct etnaviv_perfmon_request *pmr);
+	const struct etnaviv_perfmon_request *pmr, u32 exec_state);
 
 #endif /* __ETNAVIV_PERFMON_H__ */
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 5a7c9d8abd6b..735ce47688f9 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -95,26 +95,21 @@ config DRM_EXYNOS_G2D
 	help
 	  Choose this option if you want to use Exynos G2D for DRM.
 
-config DRM_EXYNOS_IPP
-	bool "Image Post Processor"
-	help
-	  Choose this option if you want to use IPP feature for DRM.
-
 config DRM_EXYNOS_FIMC
 	bool "FIMC"
-	depends on DRM_EXYNOS_IPP && MFD_SYSCON
+	depends on BROKEN && MFD_SYSCON
 	help
 	  Choose this option if you want to use Exynos FIMC for DRM.
 
 config DRM_EXYNOS_ROTATOR
 	bool "Rotator"
-	depends on DRM_EXYNOS_IPP
+	depends on BROKEN
 	help
 	  Choose this option if you want to use Exynos Rotator for DRM.
 
 config DRM_EXYNOS_GSC
 	bool "GScaler"
-	depends on DRM_EXYNOS_IPP && ARCH_EXYNOS5 && VIDEO_SAMSUNG_EXYNOS_GSC=n
+	depends on BROKEN && ARCH_EXYNOS5 && VIDEO_SAMSUNG_EXYNOS_GSC=n
 	help
 	  Choose this option if you want to use Exynos GSC for DRM.
 
diff --git a/drivers/gpu/drm/exynos/Makefile b/drivers/gpu/drm/exynos/Makefile
index bdf4212dde7b..a51c5459bb13 100644
--- a/drivers/gpu/drm/exynos/Makefile
+++ b/drivers/gpu/drm/exynos/Makefile
@@ -18,7 +18,6 @@ exynosdrm-$(CONFIG_DRM_EXYNOS_MIXER)	+= exynos_mixer.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_HDMI)	+= exynos_hdmi.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_VIDI)	+= exynos_drm_vidi.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_G2D)	+= exynos_drm_g2d.o
-exynosdrm-$(CONFIG_DRM_EXYNOS_IPP)	+= exynos_drm_ipp.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_FIMC)	+= exynos_drm_fimc.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_ROTATOR)	+= exynos_drm_rotator.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_GSC)	+= exynos_drm_gsc.o
diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
index 6be5b53c3b27..1c330f2a7a5d 100644
--- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
@@ -21,13 +21,12 @@
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 
-#include <video/exynos5433_decon.h>
-
 #include "exynos_drm_drv.h"
 #include "exynos_drm_crtc.h"
 #include "exynos_drm_fb.h"
 #include "exynos_drm_plane.h"
 #include "exynos_drm_iommu.h"
+#include "regs-decon5433.h"
 
 #define DSD_CFG_MUX 0x1004
 #define DSD_CFG_MUX_TE_UNMASK_GLOBAL BIT(13)
@@ -744,11 +743,6 @@ static int exynos5433_decon_probe(struct platform_device *pdev)
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(dev, "cannot find IO resource\n");
-		return -ENXIO;
-	}
-
 	ctx->addr = devm_ioremap_resource(dev, res);
 	if (IS_ERR(ctx->addr)) {
 		dev_err(dev, "ioremap failed\n");
diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
index 615efcf7782a..3931d5e33fe0 100644
--- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
@@ -25,13 +25,13 @@
 
 #include <video/of_display_timing.h>
 #include <video/of_videomode.h>
-#include <video/exynos7_decon.h>
 
 #include "exynos_drm_crtc.h"
 #include "exynos_drm_plane.h"
 #include "exynos_drm_drv.h"
 #include "exynos_drm_fb.h"
 #include "exynos_drm_iommu.h"
+#include "regs-decon7.h"
 
 /*
  * DECON stands for Display and Enhancement controller.
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index 27e423b87266..a518e9c6d6cc 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -16,6 +16,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 
 #include <linux/component.h>
 
@@ -28,7 +29,6 @@
 #include "exynos_drm_plane.h"
 #include "exynos_drm_vidi.h"
 #include "exynos_drm_g2d.h"
-#include "exynos_drm_ipp.h"
 #include "exynos_drm_iommu.h"
 
 #define DRIVER_NAME	"exynos"
@@ -87,11 +87,6 @@ static void exynos_drm_postclose(struct drm_device *dev, struct drm_file *file)
 	file->driver_priv = NULL;
 }
 
-static void exynos_drm_lastclose(struct drm_device *dev)
-{
-	exynos_drm_fbdev_restore_mode(dev);
-}
-
 static const struct vm_operations_struct exynos_drm_gem_vm_ops = {
 	.fault = exynos_drm_gem_fault,
 	.open = drm_gem_vm_open,
@@ -113,14 +108,6 @@ static const struct drm_ioctl_desc exynos_ioctls[] = {
 			DRM_AUTH | DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(EXYNOS_G2D_EXEC, exynos_g2d_exec_ioctl,
 			DRM_AUTH | DRM_RENDER_ALLOW),
-	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_GET_PROPERTY, exynos_drm_ipp_get_property,
-			DRM_AUTH | DRM_RENDER_ALLOW),
-	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_SET_PROPERTY, exynos_drm_ipp_set_property,
-			DRM_AUTH | DRM_RENDER_ALLOW),
-	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_QUEUE_BUF, exynos_drm_ipp_queue_buf,
-			DRM_AUTH | DRM_RENDER_ALLOW),
-	DRM_IOCTL_DEF_DRV(EXYNOS_IPP_CMD_CTRL, exynos_drm_ipp_cmd_ctrl,
-			DRM_AUTH | DRM_RENDER_ALLOW),
 };
 
 static const struct file_operations exynos_drm_driver_fops = {
@@ -138,7 +125,7 @@ static struct drm_driver exynos_drm_driver = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME
 				  | DRIVER_ATOMIC | DRIVER_RENDER,
 	.open			= exynos_drm_open,
-	.lastclose		= exynos_drm_lastclose,
+	.lastclose		= drm_fb_helper_lastclose,
 	.postclose		= exynos_drm_postclose,
 	.gem_free_object_unlocked = exynos_drm_gem_free_object,
 	.gem_vm_ops		= &exynos_drm_gem_vm_ops,
@@ -260,9 +247,6 @@ static struct exynos_drm_driver_info exynos_drm_drivers[] = {
 		DRV_PTR(rotator_driver, CONFIG_DRM_EXYNOS_ROTATOR),
 	}, {
 		DRV_PTR(gsc_driver, CONFIG_DRM_EXYNOS_GSC),
-	}, {
-		DRV_PTR(ipp_driver, CONFIG_DRM_EXYNOS_IPP),
-		DRM_VIRTUAL_DEVICE
 	}, {
 		&exynos_drm_platform_driver,
 		DRM_VIRTUAL_DEVICE
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index 589d465a7f88..df2262f70d91 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -188,7 +188,6 @@ struct exynos_drm_g2d_private {
 
 struct drm_exynos_file_private {
 	struct exynos_drm_g2d_private	*g2d_priv;
-	struct device			*ipp_dev;
 };
 
 /*
@@ -291,6 +290,5 @@ extern struct platform_driver g2d_driver;
 extern struct platform_driver fimc_driver;
 extern struct platform_driver rotator_driver;
 extern struct platform_driver gsc_driver;
-extern struct platform_driver ipp_driver;
 extern struct platform_driver mic_driver;
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index 8208df56a88f..0faaf829f5bf 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -205,7 +205,7 @@ static struct drm_mode_config_helper_funcs exynos_drm_mode_config_helpers = {
 
 static const struct drm_mode_config_funcs exynos_drm_mode_config_funcs = {
 	.fb_create = exynos_user_fb_create,
-	.output_poll_changed = exynos_drm_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = exynos_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index dfb66ecf417b..132dd52d0ac7 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -270,24 +270,6 @@ void exynos_drm_fbdev_fini(struct drm_device *dev)
 	private->fb_helper = NULL;
 }
 
-void exynos_drm_fbdev_restore_mode(struct drm_device *dev)
-{
-	struct exynos_drm_private *private = dev->dev_private;
-
-	if (!private || !private->fb_helper)
-		return;
-
-	drm_fb_helper_restore_fbdev_mode_unlocked(private->fb_helper);
-}
-
-void exynos_drm_output_poll_changed(struct drm_device *dev)
-{
-	struct exynos_drm_private *private = dev->dev_private;
-	struct drm_fb_helper *fb_helper = private->fb_helper;
-
-	drm_fb_helper_hotplug_event(fb_helper);
-}
-
 void exynos_drm_fbdev_suspend(struct drm_device *dev)
 {
 	struct exynos_drm_private *private = dev->dev_private;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.h b/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
index 645d1bb7f665..b33847223a85 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.h
@@ -19,8 +19,6 @@
 
 int exynos_drm_fbdev_init(struct drm_device *dev);
 void exynos_drm_fbdev_fini(struct drm_device *dev);
-void exynos_drm_fbdev_restore_mode(struct drm_device *dev);
-void exynos_drm_output_poll_changed(struct drm_device *dev);
 void exynos_drm_fbdev_suspend(struct drm_device *drm);
 void exynos_drm_fbdev_resume(struct drm_device *drm);
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
index 2b8bf2dd6387..f68ef1b3a28c 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
@@ -286,7 +286,6 @@ static int g2d_init_cmdlist(struct g2d_data *g2d)
 
 	node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
 	if (!node) {
-		dev_err(dev, "failed to allocate memory\n");
 		ret = -ENOMEM;
 		goto err;
 	}
@@ -926,7 +925,7 @@ static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
 	struct drm_device *drm_dev = g2d->subdrv.drm_dev;
 	struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
 	struct drm_exynos_pending_g2d_event *e;
-	struct timeval now;
+	struct timespec64 now;
 
 	if (list_empty(&runqueue_node->event_list))
 		return;
@@ -934,9 +933,9 @@ static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
 	e = list_first_entry(&runqueue_node->event_list,
 			     struct drm_exynos_pending_g2d_event, base.link);
 
-	do_gettimeofday(&now);
+	ktime_get_ts64(&now);
 	e->event.tv_sec = now.tv_sec;
-	e->event.tv_usec = now.tv_usec;
+	e->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
 	e->event.cmdlist_no = cmdlist_no;
 
 	drm_send_event(drm_dev, &e->base);
@@ -1358,10 +1357,9 @@ int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
 		return -EFAULT;
 
 	runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
-	if (!runqueue_node) {
-		dev_err(dev, "failed to allocate memory\n");
+	if (!runqueue_node)
 		return -ENOMEM;
-	}
+
 	run_cmdlist = &runqueue_node->run_cmdlist;
 	event_list = &runqueue_node->event_list;
 	INIT_LIST_HEAD(run_cmdlist);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.c b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
deleted file mode 100644
index 3edda18cc2d2..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_ipp.c
+++ /dev/null
@@ -1,1806 +0,0 @@
-/*
- * Copyright (C) 2012 Samsung Electronics Co.Ltd
- * Authors:
- *	Eunchul Kim <chulspro.kim@samsung.com>
- *	Jinyoung Jeon <jy0.jeon@samsung.com>
- *	Sangmin Lee <lsmin.lee@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- */
-#include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/types.h>
-#include <linux/clk.h>
-#include <linux/pm_runtime.h>
-
-#include <drm/drmP.h>
-#include <drm/exynos_drm.h>
-#include "exynos_drm_drv.h"
-#include "exynos_drm_gem.h"
-#include "exynos_drm_ipp.h"
-#include "exynos_drm_iommu.h"
-
-/*
- * IPP stands for Image Post Processing and
- * supports image scaler/rotator and input/output DMA operations.
- * using FIMC, GSC, Rotator, so on.
- * IPP is integration device driver of same attribute h/w
- */
-
-/*
- * TODO
- * 1. expand command control id.
- * 2. integrate	property and config.
- * 3. removed send_event id check routine.
- * 4. compare send_event id if needed.
- * 5. free subdrv_remove notifier callback list if needed.
- * 6. need to check subdrv_open about multi-open.
- * 7. need to power_on implement power and sysmmu ctrl.
- */
-
-#define get_ipp_context(dev)	platform_get_drvdata(to_platform_device(dev))
-#define ipp_is_m2m_cmd(c)	(c == IPP_CMD_M2M)
-
-/*
- * A structure of event.
- *
- * @base: base of event.
- * @event: ipp event.
- */
-struct drm_exynos_ipp_send_event {
-	struct drm_pending_event	base;
-	struct drm_exynos_ipp_event	event;
-};
-
-/*
- * A structure of memory node.
- *
- * @list: list head to memory queue information.
- * @ops_id: id of operations.
- * @prop_id: id of property.
- * @buf_id: id of buffer.
- * @buf_info: gem objects and dma address, size.
- * @filp: a pointer to drm_file.
- */
-struct drm_exynos_ipp_mem_node {
-	struct list_head	list;
-	enum drm_exynos_ops_id	ops_id;
-	u32	prop_id;
-	u32	buf_id;
-	struct drm_exynos_ipp_buf_info	buf_info;
-};
-
-/*
- * A structure of ipp context.
- *
- * @subdrv: prepare initialization using subdrv.
- * @ipp_lock: lock for synchronization of access to ipp_idr.
- * @prop_lock: lock for synchronization of access to prop_idr.
- * @ipp_idr: ipp driver idr.
- * @prop_idr: property idr.
- * @event_workq: event work queue.
- * @cmd_workq: command work queue.
- */
-struct ipp_context {
-	struct exynos_drm_subdrv	subdrv;
-	struct mutex	ipp_lock;
-	struct mutex	prop_lock;
-	struct idr	ipp_idr;
-	struct idr	prop_idr;
-	struct workqueue_struct	*event_workq;
-	struct workqueue_struct	*cmd_workq;
-};
-
-static LIST_HEAD(exynos_drm_ippdrv_list);
-static DEFINE_MUTEX(exynos_drm_ippdrv_lock);
-static BLOCKING_NOTIFIER_HEAD(exynos_drm_ippnb_list);
-
-int exynos_drm_ippdrv_register(struct exynos_drm_ippdrv *ippdrv)
-{
-	mutex_lock(&exynos_drm_ippdrv_lock);
-	list_add_tail(&ippdrv->drv_list, &exynos_drm_ippdrv_list);
-	mutex_unlock(&exynos_drm_ippdrv_lock);
-
-	return 0;
-}
-
-int exynos_drm_ippdrv_unregister(struct exynos_drm_ippdrv *ippdrv)
-{
-	mutex_lock(&exynos_drm_ippdrv_lock);
-	list_del(&ippdrv->drv_list);
-	mutex_unlock(&exynos_drm_ippdrv_lock);
-
-	return 0;
-}
-
-static int ipp_create_id(struct idr *id_idr, struct mutex *lock, void *obj)
-{
-	int ret;
-
-	mutex_lock(lock);
-	ret = idr_alloc(id_idr, obj, 1, 0, GFP_KERNEL);
-	mutex_unlock(lock);
-
-	return ret;
-}
-
-static void ipp_remove_id(struct idr *id_idr, struct mutex *lock, u32 id)
-{
-	mutex_lock(lock);
-	idr_remove(id_idr, id);
-	mutex_unlock(lock);
-}
-
-static void *ipp_find_obj(struct idr *id_idr, struct mutex *lock, u32 id)
-{
-	void *obj;
-
-	mutex_lock(lock);
-	obj = idr_find(id_idr, id);
-	mutex_unlock(lock);
-
-	return obj;
-}
-
-static int ipp_check_driver(struct exynos_drm_ippdrv *ippdrv,
-			    struct drm_exynos_ipp_property *property)
-{
-	if (ippdrv->dedicated || (!ipp_is_m2m_cmd(property->cmd) &&
-				  !pm_runtime_suspended(ippdrv->dev)))
-		return -EBUSY;
-
-	if (ippdrv->check_property &&
-	    ippdrv->check_property(ippdrv->dev, property))
-		return -EINVAL;
-
-	return 0;
-}
-
-static struct exynos_drm_ippdrv *ipp_find_driver(struct ipp_context *ctx,
-		struct drm_exynos_ipp_property *property)
-{
-	struct exynos_drm_ippdrv *ippdrv;
-	u32 ipp_id = property->ipp_id;
-	int ret;
-
-	if (ipp_id) {
-		ippdrv = ipp_find_obj(&ctx->ipp_idr, &ctx->ipp_lock, ipp_id);
-		if (!ippdrv) {
-			DRM_DEBUG("ipp%d driver not found\n", ipp_id);
-			return ERR_PTR(-ENODEV);
-		}
-
-		ret = ipp_check_driver(ippdrv, property);
-		if (ret < 0) {
-			DRM_DEBUG("ipp%d driver check error %d\n", ipp_id, ret);
-			return ERR_PTR(ret);
-		}
-
-		return ippdrv;
-	} else {
-		list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
-			ret = ipp_check_driver(ippdrv, property);
-			if (ret == 0)
-				return ippdrv;
-		}
-
-		DRM_DEBUG("cannot find driver suitable for given property.\n");
-	}
-
-	return ERR_PTR(-ENODEV);
-}
-
-static struct exynos_drm_ippdrv *ipp_find_drv_by_handle(u32 prop_id)
-{
-	struct exynos_drm_ippdrv *ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node;
-	int count = 0;
-
-	DRM_DEBUG_KMS("prop_id[%d]\n", prop_id);
-
-	/*
-	 * This case is search ipp driver by prop_id handle.
-	 * sometimes, ipp subsystem find driver by prop_id.
-	 * e.g PAUSE state, queue buf, command control.
-	 */
-	list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
-		DRM_DEBUG_KMS("count[%d]ippdrv[%pK]\n", count++, ippdrv);
-
-		mutex_lock(&ippdrv->cmd_lock);
-		list_for_each_entry(c_node, &ippdrv->cmd_list, list) {
-			if (c_node->property.prop_id == prop_id) {
-				mutex_unlock(&ippdrv->cmd_lock);
-				return ippdrv;
-			}
-		}
-		mutex_unlock(&ippdrv->cmd_lock);
-	}
-
-	return ERR_PTR(-ENODEV);
-}
-
-int exynos_drm_ipp_get_property(struct drm_device *drm_dev, void *data,
-		struct drm_file *file)
-{
-	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct device *dev = file_priv->ipp_dev;
-	struct ipp_context *ctx = get_ipp_context(dev);
-	struct drm_exynos_ipp_prop_list *prop_list = data;
-	struct exynos_drm_ippdrv *ippdrv;
-	int count = 0;
-
-	if (!ctx) {
-		DRM_ERROR("invalid context.\n");
-		return -EINVAL;
-	}
-
-	if (!prop_list) {
-		DRM_ERROR("invalid property parameter.\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG_KMS("ipp_id[%d]\n", prop_list->ipp_id);
-
-	if (!prop_list->ipp_id) {
-		list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list)
-			count++;
-
-		/*
-		 * Supports ippdrv list count for user application.
-		 * First step user application getting ippdrv count.
-		 * and second step getting ippdrv capability using ipp_id.
-		 */
-		prop_list->count = count;
-	} else {
-		/*
-		 * Getting ippdrv capability by ipp_id.
-		 * some device not supported wb, output interface.
-		 * so, user application detect correct ipp driver
-		 * using this ioctl.
-		 */
-		ippdrv = ipp_find_obj(&ctx->ipp_idr, &ctx->ipp_lock,
-						prop_list->ipp_id);
-		if (!ippdrv) {
-			DRM_ERROR("not found ipp%d driver.\n",
-					prop_list->ipp_id);
-			return -ENODEV;
-		}
-
-		*prop_list = ippdrv->prop_list;
-	}
-
-	return 0;
-}
-
-static void ipp_print_property(struct drm_exynos_ipp_property *property,
-		int idx)
-{
-	struct drm_exynos_ipp_config *config = &property->config[idx];
-	struct drm_exynos_pos *pos = &config->pos;
-	struct drm_exynos_sz *sz = &config->sz;
-
-	DRM_DEBUG_KMS("prop_id[%d]ops[%s]fmt[0x%x]\n",
-		property->prop_id, idx ? "dst" : "src", config->fmt);
-
-	DRM_DEBUG_KMS("pos[%d %d %d %d]sz[%d %d]f[%d]r[%d]\n",
-		pos->x, pos->y, pos->w, pos->h,
-		sz->hsize, sz->vsize, config->flip, config->degree);
-}
-
-static struct drm_exynos_ipp_cmd_work *ipp_create_cmd_work(void)
-{
-	struct drm_exynos_ipp_cmd_work *cmd_work;
-
-	cmd_work = kzalloc(sizeof(*cmd_work), GFP_KERNEL);
-	if (!cmd_work)
-		return ERR_PTR(-ENOMEM);
-
-	INIT_WORK((struct work_struct *)cmd_work, ipp_sched_cmd);
-
-	return cmd_work;
-}
-
-static struct drm_exynos_ipp_event_work *ipp_create_event_work(void)
-{
-	struct drm_exynos_ipp_event_work *event_work;
-
-	event_work = kzalloc(sizeof(*event_work), GFP_KERNEL);
-	if (!event_work)
-		return ERR_PTR(-ENOMEM);
-
-	INIT_WORK(&event_work->work, ipp_sched_event);
-
-	return event_work;
-}
-
-int exynos_drm_ipp_set_property(struct drm_device *drm_dev, void *data,
-		struct drm_file *file)
-{
-	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct device *dev = file_priv->ipp_dev;
-	struct ipp_context *ctx = get_ipp_context(dev);
-	struct drm_exynos_ipp_property *property = data;
-	struct exynos_drm_ippdrv *ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node;
-	u32 prop_id;
-	int ret, i;
-
-	if (!ctx) {
-		DRM_ERROR("invalid context.\n");
-		return -EINVAL;
-	}
-
-	if (!property) {
-		DRM_ERROR("invalid property parameter.\n");
-		return -EINVAL;
-	}
-
-	prop_id = property->prop_id;
-
-	/*
-	 * This is log print for user application property.
-	 * user application set various property.
-	 */
-	for_each_ipp_ops(i)
-		ipp_print_property(property, i);
-
-	/*
-	 * In case prop_id is not zero try to set existing property.
-	 */
-	if (prop_id) {
-		c_node = ipp_find_obj(&ctx->prop_idr, &ctx->prop_lock, prop_id);
-
-		if (!c_node || c_node->filp != file) {
-			DRM_DEBUG_KMS("prop_id[%d] not found\n", prop_id);
-			return -EINVAL;
-		}
-
-		if (c_node->state != IPP_STATE_STOP) {
-			DRM_DEBUG_KMS("prop_id[%d] not stopped\n", prop_id);
-			return -EINVAL;
-		}
-
-		c_node->property = *property;
-
-		return 0;
-	}
-
-	/* find ipp driver using ipp id */
-	ippdrv = ipp_find_driver(ctx, property);
-	if (IS_ERR(ippdrv)) {
-		DRM_ERROR("failed to get ipp driver.\n");
-		return -EINVAL;
-	}
-
-	/* allocate command node */
-	c_node = kzalloc(sizeof(*c_node), GFP_KERNEL);
-	if (!c_node)
-		return -ENOMEM;
-
-	ret = ipp_create_id(&ctx->prop_idr, &ctx->prop_lock, c_node);
-	if (ret < 0) {
-		DRM_ERROR("failed to create id.\n");
-		goto err_clear;
-	}
-	property->prop_id = ret;
-
-	DRM_DEBUG_KMS("created prop_id[%d]cmd[%d]ippdrv[%pK]\n",
-		property->prop_id, property->cmd, ippdrv);
-
-	/* stored property information and ippdrv in private data */
-	c_node->property = *property;
-	c_node->state = IPP_STATE_IDLE;
-	c_node->filp = file;
-
-	c_node->start_work = ipp_create_cmd_work();
-	if (IS_ERR(c_node->start_work)) {
-		DRM_ERROR("failed to create start work.\n");
-		ret = PTR_ERR(c_node->start_work);
-		goto err_remove_id;
-	}
-
-	c_node->stop_work = ipp_create_cmd_work();
-	if (IS_ERR(c_node->stop_work)) {
-		DRM_ERROR("failed to create stop work.\n");
-		ret = PTR_ERR(c_node->stop_work);
-		goto err_free_start;
-	}
-
-	c_node->event_work = ipp_create_event_work();
-	if (IS_ERR(c_node->event_work)) {
-		DRM_ERROR("failed to create event work.\n");
-		ret = PTR_ERR(c_node->event_work);
-		goto err_free_stop;
-	}
-
-	mutex_init(&c_node->lock);
-	mutex_init(&c_node->mem_lock);
-	mutex_init(&c_node->event_lock);
-
-	init_completion(&c_node->start_complete);
-	init_completion(&c_node->stop_complete);
-
-	for_each_ipp_ops(i)
-		INIT_LIST_HEAD(&c_node->mem_list[i]);
-
-	INIT_LIST_HEAD(&c_node->event_list);
-	mutex_lock(&ippdrv->cmd_lock);
-	list_add_tail(&c_node->list, &ippdrv->cmd_list);
-	mutex_unlock(&ippdrv->cmd_lock);
-
-	/* make dedicated state without m2m */
-	if (!ipp_is_m2m_cmd(property->cmd))
-		ippdrv->dedicated = true;
-
-	return 0;
-
-err_free_stop:
-	kfree(c_node->stop_work);
-err_free_start:
-	kfree(c_node->start_work);
-err_remove_id:
-	ipp_remove_id(&ctx->prop_idr, &ctx->prop_lock, property->prop_id);
-err_clear:
-	kfree(c_node);
-	return ret;
-}
-
-static int ipp_validate_mem_node(struct drm_device *drm_dev,
-				 struct drm_exynos_ipp_mem_node *m_node,
-				 struct drm_exynos_ipp_cmd_node *c_node)
-{
-	struct drm_exynos_ipp_config *ipp_cfg;
-	unsigned int num_plane;
-	unsigned long size, buf_size = 0, plane_size, img_size = 0;
-	unsigned int bpp, width, height;
-	int i;
-
-	ipp_cfg = &c_node->property.config[m_node->ops_id];
-	num_plane = drm_format_num_planes(ipp_cfg->fmt);
-
-	/**
-	 * This is a rather simplified validation of a memory node.
-	 * It basically verifies provided gem object handles
-	 * and the buffer sizes with respect to current configuration.
-	 * This is not the best that can be done
-	 * but it seems more than enough
-	 */
-	for (i = 0; i < num_plane; ++i) {
-		width = ipp_cfg->sz.hsize;
-		height = ipp_cfg->sz.vsize;
-		bpp = drm_format_plane_cpp(ipp_cfg->fmt, i);
-
-		/*
-		 * The result of drm_format_plane_cpp() for chroma planes must
-		 * be used with drm_format_xxxx_chroma_subsampling() for
-		 * correct result.
-		 */
-		if (i > 0) {
-			width /= drm_format_horz_chroma_subsampling(
-								ipp_cfg->fmt);
-			height /= drm_format_vert_chroma_subsampling(
-								ipp_cfg->fmt);
-		}
-		plane_size = width * height * bpp;
-		img_size += plane_size;
-
-		if (m_node->buf_info.handles[i]) {
-			size = exynos_drm_gem_get_size(drm_dev,
-					m_node->buf_info.handles[i],
-					c_node->filp);
-			if (plane_size > size) {
-				DRM_ERROR(
-					"buffer %d is smaller than required\n",
-					i);
-				return -EINVAL;
-			}
-
-			buf_size += size;
-		}
-	}
-
-	if (buf_size < img_size) {
-		DRM_ERROR("size of buffers(%lu) is smaller than image(%lu)\n",
-			buf_size, img_size);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static int ipp_put_mem_node(struct drm_device *drm_dev,
-		struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_mem_node *m_node)
-{
-	int i;
-
-	DRM_DEBUG_KMS("node[%pK]\n", m_node);
-
-	if (!m_node) {
-		DRM_ERROR("invalid dequeue node.\n");
-		return -EFAULT;
-	}
-
-	DRM_DEBUG_KMS("ops_id[%d]\n", m_node->ops_id);
-
-	/* put gem buffer */
-	for_each_ipp_planar(i) {
-		unsigned long handle = m_node->buf_info.handles[i];
-		if (handle)
-			exynos_drm_gem_put_dma_addr(drm_dev, handle,
-							c_node->filp);
-	}
-
-	list_del(&m_node->list);
-	kfree(m_node);
-
-	return 0;
-}
-
-static struct drm_exynos_ipp_mem_node
-		*ipp_get_mem_node(struct drm_device *drm_dev,
-		struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_queue_buf *qbuf)
-{
-	struct drm_exynos_ipp_mem_node *m_node;
-	struct drm_exynos_ipp_buf_info *buf_info;
-	int i;
-
-	m_node = kzalloc(sizeof(*m_node), GFP_KERNEL);
-	if (!m_node)
-		return ERR_PTR(-ENOMEM);
-
-	buf_info = &m_node->buf_info;
-
-	/* operations, buffer id */
-	m_node->ops_id = qbuf->ops_id;
-	m_node->prop_id = qbuf->prop_id;
-	m_node->buf_id = qbuf->buf_id;
-	INIT_LIST_HEAD(&m_node->list);
-
-	DRM_DEBUG_KMS("m_node[%pK]ops_id[%d]\n", m_node, qbuf->ops_id);
-	DRM_DEBUG_KMS("prop_id[%d]buf_id[%d]\n", qbuf->prop_id, m_node->buf_id);
-
-	for_each_ipp_planar(i) {
-		DRM_DEBUG_KMS("i[%d]handle[0x%x]\n", i, qbuf->handle[i]);
-
-		/* get dma address by handle */
-		if (qbuf->handle[i]) {
-			dma_addr_t *addr;
-
-			addr = exynos_drm_gem_get_dma_addr(drm_dev,
-					qbuf->handle[i], c_node->filp);
-			if (IS_ERR(addr)) {
-				DRM_ERROR("failed to get addr.\n");
-				ipp_put_mem_node(drm_dev, c_node, m_node);
-				return ERR_PTR(-EFAULT);
-			}
-
-			buf_info->handles[i] = qbuf->handle[i];
-			buf_info->base[i] = *addr;
-			DRM_DEBUG_KMS("i[%d]base[%pad]hd[0x%lx]\n", i,
-				      &buf_info->base[i], buf_info->handles[i]);
-		}
-	}
-
-	mutex_lock(&c_node->mem_lock);
-	if (ipp_validate_mem_node(drm_dev, m_node, c_node)) {
-		ipp_put_mem_node(drm_dev, c_node, m_node);
-		mutex_unlock(&c_node->mem_lock);
-		return ERR_PTR(-EFAULT);
-	}
-	list_add_tail(&m_node->list, &c_node->mem_list[qbuf->ops_id]);
-	mutex_unlock(&c_node->mem_lock);
-
-	return m_node;
-}
-
-static void ipp_clean_mem_nodes(struct drm_device *drm_dev,
-			       struct drm_exynos_ipp_cmd_node *c_node, int ops)
-{
-	struct drm_exynos_ipp_mem_node *m_node, *tm_node;
-	struct list_head *head = &c_node->mem_list[ops];
-
-	mutex_lock(&c_node->mem_lock);
-
-	list_for_each_entry_safe(m_node, tm_node, head, list) {
-		int ret;
-
-		ret = ipp_put_mem_node(drm_dev, c_node, m_node);
-		if (ret)
-			DRM_ERROR("failed to put m_node.\n");
-	}
-
-	mutex_unlock(&c_node->mem_lock);
-}
-
-static int ipp_get_event(struct drm_device *drm_dev,
-		struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_queue_buf *qbuf)
-{
-	struct drm_exynos_ipp_send_event *e;
-	int ret;
-
-	DRM_DEBUG_KMS("ops_id[%d]buf_id[%d]\n", qbuf->ops_id, qbuf->buf_id);
-
-	e = kzalloc(sizeof(*e), GFP_KERNEL);
-	if (!e)
-		return -ENOMEM;
-
-	/* make event */
-	e->event.base.type = DRM_EXYNOS_IPP_EVENT;
-	e->event.base.length = sizeof(e->event);
-	e->event.user_data = qbuf->user_data;
-	e->event.prop_id = qbuf->prop_id;
-	e->event.buf_id[EXYNOS_DRM_OPS_DST] = qbuf->buf_id;
-
-	ret = drm_event_reserve_init(drm_dev, c_node->filp, &e->base, &e->event.base);
-	if (ret) {
-		kfree(e);
-		return ret;
-	}
-
-	mutex_lock(&c_node->event_lock);
-	list_add_tail(&e->base.link, &c_node->event_list);
-	mutex_unlock(&c_node->event_lock);
-
-	return 0;
-}
-
-static void ipp_put_event(struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_queue_buf *qbuf)
-{
-	struct drm_exynos_ipp_send_event *e, *te;
-	int count = 0;
-
-	mutex_lock(&c_node->event_lock);
-	list_for_each_entry_safe(e, te, &c_node->event_list, base.link) {
-		DRM_DEBUG_KMS("count[%d]e[%pK]\n", count++, e);
-
-		/*
-		 * qbuf == NULL condition means all event deletion.
-		 * stop operations want to delete all event list.
-		 * another case delete only same buf id.
-		 */
-		if (!qbuf) {
-			/* delete list */
-			list_del(&e->base.link);
-			kfree(e);
-		}
-
-		/* compare buffer id */
-		if (qbuf && (qbuf->buf_id ==
-		    e->event.buf_id[EXYNOS_DRM_OPS_DST])) {
-			/* delete list */
-			list_del(&e->base.link);
-			kfree(e);
-			goto out_unlock;
-		}
-	}
-
-out_unlock:
-	mutex_unlock(&c_node->event_lock);
-	return;
-}
-
-static void ipp_clean_cmd_node(struct ipp_context *ctx,
-				struct drm_exynos_ipp_cmd_node *c_node)
-{
-	int i;
-
-	/* cancel works */
-	cancel_work_sync(&c_node->start_work->work);
-	cancel_work_sync(&c_node->stop_work->work);
-	cancel_work_sync(&c_node->event_work->work);
-
-	/* put event */
-	ipp_put_event(c_node, NULL);
-
-	for_each_ipp_ops(i)
-		ipp_clean_mem_nodes(ctx->subdrv.drm_dev, c_node, i);
-
-	/* delete list */
-	list_del(&c_node->list);
-
-	ipp_remove_id(&ctx->prop_idr, &ctx->prop_lock,
-			c_node->property.prop_id);
-
-	/* destroy mutex */
-	mutex_destroy(&c_node->lock);
-	mutex_destroy(&c_node->mem_lock);
-	mutex_destroy(&c_node->event_lock);
-
-	/* free command node */
-	kfree(c_node->start_work);
-	kfree(c_node->stop_work);
-	kfree(c_node->event_work);
-	kfree(c_node);
-}
-
-static bool ipp_check_mem_list(struct drm_exynos_ipp_cmd_node *c_node)
-{
-	switch (c_node->property.cmd) {
-	case IPP_CMD_WB:
-		return !list_empty(&c_node->mem_list[EXYNOS_DRM_OPS_DST]);
-	case IPP_CMD_OUTPUT:
-		return !list_empty(&c_node->mem_list[EXYNOS_DRM_OPS_SRC]);
-	case IPP_CMD_M2M:
-	default:
-		return !list_empty(&c_node->mem_list[EXYNOS_DRM_OPS_SRC]) &&
-		       !list_empty(&c_node->mem_list[EXYNOS_DRM_OPS_DST]);
-	}
-}
-
-static struct drm_exynos_ipp_mem_node
-		*ipp_find_mem_node(struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_queue_buf *qbuf)
-{
-	struct drm_exynos_ipp_mem_node *m_node;
-	struct list_head *head;
-	int count = 0;
-
-	DRM_DEBUG_KMS("buf_id[%d]\n", qbuf->buf_id);
-
-	/* source/destination memory list */
-	head = &c_node->mem_list[qbuf->ops_id];
-
-	/* find memory node from memory list */
-	list_for_each_entry(m_node, head, list) {
-		DRM_DEBUG_KMS("count[%d]m_node[%pK]\n", count++, m_node);
-
-		/* compare buffer id */
-		if (m_node->buf_id == qbuf->buf_id)
-			return m_node;
-	}
-
-	return NULL;
-}
-
-static int ipp_set_mem_node(struct exynos_drm_ippdrv *ippdrv,
-		struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_mem_node *m_node)
-{
-	struct exynos_drm_ipp_ops *ops = NULL;
-	int ret = 0;
-
-	DRM_DEBUG_KMS("node[%pK]\n", m_node);
-
-	if (!m_node) {
-		DRM_ERROR("invalid queue node.\n");
-		return -EFAULT;
-	}
-
-	DRM_DEBUG_KMS("ops_id[%d]\n", m_node->ops_id);
-
-	/* get operations callback */
-	ops = ippdrv->ops[m_node->ops_id];
-	if (!ops) {
-		DRM_ERROR("not support ops.\n");
-		return -EFAULT;
-	}
-
-	/* set address and enable irq */
-	if (ops->set_addr) {
-		ret = ops->set_addr(ippdrv->dev, &m_node->buf_info,
-			m_node->buf_id, IPP_BUF_ENQUEUE);
-		if (ret) {
-			DRM_ERROR("failed to set addr.\n");
-			return ret;
-		}
-	}
-
-	return ret;
-}
-
-static void ipp_handle_cmd_work(struct device *dev,
-		struct exynos_drm_ippdrv *ippdrv,
-		struct drm_exynos_ipp_cmd_work *cmd_work,
-		struct drm_exynos_ipp_cmd_node *c_node)
-{
-	struct ipp_context *ctx = get_ipp_context(dev);
-
-	cmd_work->ippdrv = ippdrv;
-	cmd_work->c_node = c_node;
-	queue_work(ctx->cmd_workq, &cmd_work->work);
-}
-
-static int ipp_queue_buf_with_run(struct device *dev,
-		struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_mem_node *m_node,
-		struct drm_exynos_ipp_queue_buf *qbuf)
-{
-	struct exynos_drm_ippdrv *ippdrv;
-	struct drm_exynos_ipp_property *property;
-	struct exynos_drm_ipp_ops *ops;
-	int ret;
-
-	ippdrv = ipp_find_drv_by_handle(qbuf->prop_id);
-	if (IS_ERR(ippdrv)) {
-		DRM_ERROR("failed to get ipp driver.\n");
-		return -EFAULT;
-	}
-
-	ops = ippdrv->ops[qbuf->ops_id];
-	if (!ops) {
-		DRM_ERROR("failed to get ops.\n");
-		return -EFAULT;
-	}
-
-	property = &c_node->property;
-
-	if (c_node->state != IPP_STATE_START) {
-		DRM_DEBUG_KMS("bypass for invalid state.\n");
-		return 0;
-	}
-
-	mutex_lock(&c_node->mem_lock);
-	if (!ipp_check_mem_list(c_node)) {
-		mutex_unlock(&c_node->mem_lock);
-		DRM_DEBUG_KMS("empty memory.\n");
-		return 0;
-	}
-
-	/*
-	 * If set destination buffer and enabled clock,
-	 * then m2m operations need start operations at queue_buf
-	 */
-	if (ipp_is_m2m_cmd(property->cmd)) {
-		struct drm_exynos_ipp_cmd_work *cmd_work = c_node->start_work;
-
-		cmd_work->ctrl = IPP_CTRL_PLAY;
-		ipp_handle_cmd_work(dev, ippdrv, cmd_work, c_node);
-	} else {
-		ret = ipp_set_mem_node(ippdrv, c_node, m_node);
-		if (ret) {
-			mutex_unlock(&c_node->mem_lock);
-			DRM_ERROR("failed to set m node.\n");
-			return ret;
-		}
-	}
-	mutex_unlock(&c_node->mem_lock);
-
-	return 0;
-}
-
-static void ipp_clean_queue_buf(struct drm_device *drm_dev,
-		struct drm_exynos_ipp_cmd_node *c_node,
-		struct drm_exynos_ipp_queue_buf *qbuf)
-{
-	struct drm_exynos_ipp_mem_node *m_node, *tm_node;
-
-	/* delete list */
-	mutex_lock(&c_node->mem_lock);
-	list_for_each_entry_safe(m_node, tm_node,
-		&c_node->mem_list[qbuf->ops_id], list) {
-		if (m_node->buf_id == qbuf->buf_id &&
-		    m_node->ops_id == qbuf->ops_id)
-			ipp_put_mem_node(drm_dev, c_node, m_node);
-	}
-	mutex_unlock(&c_node->mem_lock);
-}
-
-int exynos_drm_ipp_queue_buf(struct drm_device *drm_dev, void *data,
-		struct drm_file *file)
-{
-	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct device *dev = file_priv->ipp_dev;
-	struct ipp_context *ctx = get_ipp_context(dev);
-	struct drm_exynos_ipp_queue_buf *qbuf = data;
-	struct drm_exynos_ipp_cmd_node *c_node;
-	struct drm_exynos_ipp_mem_node *m_node;
-	int ret;
-
-	if (!qbuf) {
-		DRM_ERROR("invalid buf parameter.\n");
-		return -EINVAL;
-	}
-
-	if (qbuf->ops_id >= EXYNOS_DRM_OPS_MAX) {
-		DRM_ERROR("invalid ops parameter.\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG_KMS("prop_id[%d]ops_id[%s]buf_id[%d]buf_type[%d]\n",
-		qbuf->prop_id, qbuf->ops_id ? "dst" : "src",
-		qbuf->buf_id, qbuf->buf_type);
-
-	/* find command node */
-	c_node = ipp_find_obj(&ctx->prop_idr, &ctx->prop_lock,
-		qbuf->prop_id);
-	if (!c_node || c_node->filp != file) {
-		DRM_ERROR("failed to get command node.\n");
-		return -ENODEV;
-	}
-
-	/* buffer control */
-	switch (qbuf->buf_type) {
-	case IPP_BUF_ENQUEUE:
-		/* get memory node */
-		m_node = ipp_get_mem_node(drm_dev, c_node, qbuf);
-		if (IS_ERR(m_node)) {
-			DRM_ERROR("failed to get m_node.\n");
-			return PTR_ERR(m_node);
-		}
-
-		/*
-		 * first step get event for destination buffer.
-		 * and second step when M2M case run with destination buffer
-		 * if needed.
-		 */
-		if (qbuf->ops_id == EXYNOS_DRM_OPS_DST) {
-			/* get event for destination buffer */
-			ret = ipp_get_event(drm_dev, c_node, qbuf);
-			if (ret) {
-				DRM_ERROR("failed to get event.\n");
-				goto err_clean_node;
-			}
-
-			/*
-			 * M2M case run play control for streaming feature.
-			 * other case set address and waiting.
-			 */
-			ret = ipp_queue_buf_with_run(dev, c_node, m_node, qbuf);
-			if (ret) {
-				DRM_ERROR("failed to run command.\n");
-				goto err_clean_node;
-			}
-		}
-		break;
-	case IPP_BUF_DEQUEUE:
-		mutex_lock(&c_node->lock);
-
-		/* put event for destination buffer */
-		if (qbuf->ops_id == EXYNOS_DRM_OPS_DST)
-			ipp_put_event(c_node, qbuf);
-
-		ipp_clean_queue_buf(drm_dev, c_node, qbuf);
-
-		mutex_unlock(&c_node->lock);
-		break;
-	default:
-		DRM_ERROR("invalid buffer control.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-
-err_clean_node:
-	DRM_ERROR("clean memory nodes.\n");
-
-	ipp_clean_queue_buf(drm_dev, c_node, qbuf);
-	return ret;
-}
-
-static bool exynos_drm_ipp_check_valid(struct device *dev,
-		enum drm_exynos_ipp_ctrl ctrl, enum drm_exynos_ipp_state state)
-{
-	if (ctrl != IPP_CTRL_PLAY) {
-		if (pm_runtime_suspended(dev)) {
-			DRM_ERROR("pm:runtime_suspended.\n");
-			goto err_status;
-		}
-	}
-
-	switch (ctrl) {
-	case IPP_CTRL_PLAY:
-		if (state != IPP_STATE_IDLE)
-			goto err_status;
-		break;
-	case IPP_CTRL_STOP:
-		if (state == IPP_STATE_STOP)
-			goto err_status;
-		break;
-	case IPP_CTRL_PAUSE:
-		if (state != IPP_STATE_START)
-			goto err_status;
-		break;
-	case IPP_CTRL_RESUME:
-		if (state != IPP_STATE_STOP)
-			goto err_status;
-		break;
-	default:
-		DRM_ERROR("invalid state.\n");
-		goto err_status;
-	}
-
-	return true;
-
-err_status:
-	DRM_ERROR("invalid status:ctrl[%d]state[%d]\n", ctrl, state);
-	return false;
-}
-
-int exynos_drm_ipp_cmd_ctrl(struct drm_device *drm_dev, void *data,
-		struct drm_file *file)
-{
-	struct drm_exynos_file_private *file_priv = file->driver_priv;
-	struct exynos_drm_ippdrv *ippdrv = NULL;
-	struct device *dev = file_priv->ipp_dev;
-	struct ipp_context *ctx = get_ipp_context(dev);
-	struct drm_exynos_ipp_cmd_ctrl *cmd_ctrl = data;
-	struct drm_exynos_ipp_cmd_work *cmd_work;
-	struct drm_exynos_ipp_cmd_node *c_node;
-
-	if (!ctx) {
-		DRM_ERROR("invalid context.\n");
-		return -EINVAL;
-	}
-
-	if (!cmd_ctrl) {
-		DRM_ERROR("invalid control parameter.\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG_KMS("ctrl[%d]prop_id[%d]\n",
-		cmd_ctrl->ctrl, cmd_ctrl->prop_id);
-
-	ippdrv = ipp_find_drv_by_handle(cmd_ctrl->prop_id);
-	if (IS_ERR(ippdrv)) {
-		DRM_ERROR("failed to get ipp driver.\n");
-		return PTR_ERR(ippdrv);
-	}
-
-	c_node = ipp_find_obj(&ctx->prop_idr, &ctx->prop_lock,
-		cmd_ctrl->prop_id);
-	if (!c_node || c_node->filp != file) {
-		DRM_ERROR("invalid command node list.\n");
-		return -ENODEV;
-	}
-
-	if (!exynos_drm_ipp_check_valid(ippdrv->dev, cmd_ctrl->ctrl,
-	    c_node->state)) {
-		DRM_ERROR("invalid state.\n");
-		return -EINVAL;
-	}
-
-	switch (cmd_ctrl->ctrl) {
-	case IPP_CTRL_PLAY:
-		if (pm_runtime_suspended(ippdrv->dev))
-			pm_runtime_get_sync(ippdrv->dev);
-
-		c_node->state = IPP_STATE_START;
-
-		cmd_work = c_node->start_work;
-		cmd_work->ctrl = cmd_ctrl->ctrl;
-		ipp_handle_cmd_work(dev, ippdrv, cmd_work, c_node);
-		break;
-	case IPP_CTRL_STOP:
-		cmd_work = c_node->stop_work;
-		cmd_work->ctrl = cmd_ctrl->ctrl;
-		ipp_handle_cmd_work(dev, ippdrv, cmd_work, c_node);
-
-		if (!wait_for_completion_timeout(&c_node->stop_complete,
-		    msecs_to_jiffies(300))) {
-			DRM_ERROR("timeout stop:prop_id[%d]\n",
-				c_node->property.prop_id);
-		}
-
-		c_node->state = IPP_STATE_STOP;
-		ippdrv->dedicated = false;
-		mutex_lock(&ippdrv->cmd_lock);
-		ipp_clean_cmd_node(ctx, c_node);
-
-		if (list_empty(&ippdrv->cmd_list))
-			pm_runtime_put_sync(ippdrv->dev);
-		mutex_unlock(&ippdrv->cmd_lock);
-		break;
-	case IPP_CTRL_PAUSE:
-		cmd_work = c_node->stop_work;
-		cmd_work->ctrl = cmd_ctrl->ctrl;
-		ipp_handle_cmd_work(dev, ippdrv, cmd_work, c_node);
-
-		if (!wait_for_completion_timeout(&c_node->stop_complete,
-		    msecs_to_jiffies(200))) {
-			DRM_ERROR("timeout stop:prop_id[%d]\n",
-				c_node->property.prop_id);
-		}
-
-		c_node->state = IPP_STATE_STOP;
-		break;
-	case IPP_CTRL_RESUME:
-		c_node->state = IPP_STATE_START;
-		cmd_work = c_node->start_work;
-		cmd_work->ctrl = cmd_ctrl->ctrl;
-		ipp_handle_cmd_work(dev, ippdrv, cmd_work, c_node);
-		break;
-	default:
-		DRM_ERROR("could not support this state currently.\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG_KMS("done ctrl[%d]prop_id[%d]\n",
-		cmd_ctrl->ctrl, cmd_ctrl->prop_id);
-
-	return 0;
-}
-
-int exynos_drm_ippnb_register(struct notifier_block *nb)
-{
-	return blocking_notifier_chain_register(
-		&exynos_drm_ippnb_list, nb);
-}
-
-int exynos_drm_ippnb_unregister(struct notifier_block *nb)
-{
-	return blocking_notifier_chain_unregister(
-		&exynos_drm_ippnb_list, nb);
-}
-
-int exynos_drm_ippnb_send_event(unsigned long val, void *v)
-{
-	return blocking_notifier_call_chain(
-		&exynos_drm_ippnb_list, val, v);
-}
-
-static int ipp_set_property(struct exynos_drm_ippdrv *ippdrv,
-		struct drm_exynos_ipp_property *property)
-{
-	struct exynos_drm_ipp_ops *ops = NULL;
-	bool swap = false;
-	int ret, i;
-
-	if (!property) {
-		DRM_ERROR("invalid property parameter.\n");
-		return -EINVAL;
-	}
-
-	DRM_DEBUG_KMS("prop_id[%d]\n", property->prop_id);
-
-	/* reset h/w block */
-	if (ippdrv->reset &&
-	    ippdrv->reset(ippdrv->dev)) {
-		return -EINVAL;
-	}
-
-	/* set source,destination operations */
-	for_each_ipp_ops(i) {
-		struct drm_exynos_ipp_config *config =
-			&property->config[i];
-
-		ops = ippdrv->ops[i];
-		if (!ops || !config) {
-			DRM_ERROR("not support ops and config.\n");
-			return -EINVAL;
-		}
-
-		/* set format */
-		if (ops->set_fmt) {
-			ret = ops->set_fmt(ippdrv->dev, config->fmt);
-			if (ret)
-				return ret;
-		}
-
-		/* set transform for rotation, flip */
-		if (ops->set_transf) {
-			ret = ops->set_transf(ippdrv->dev, config->degree,
-				config->flip, &swap);
-			if (ret)
-				return ret;
-		}
-
-		/* set size */
-		if (ops->set_size) {
-			ret = ops->set_size(ippdrv->dev, swap, &config->pos,
-				&config->sz);
-			if (ret)
-				return ret;
-		}
-	}
-
-	return 0;
-}
-
-static int ipp_start_property(struct exynos_drm_ippdrv *ippdrv,
-		struct drm_exynos_ipp_cmd_node *c_node)
-{
-	struct drm_exynos_ipp_mem_node *m_node;
-	struct drm_exynos_ipp_property *property = &c_node->property;
-	struct list_head *head;
-	int ret, i;
-
-	DRM_DEBUG_KMS("prop_id[%d]\n", property->prop_id);
-
-	/* store command info in ippdrv */
-	ippdrv->c_node = c_node;
-
-	mutex_lock(&c_node->mem_lock);
-	if (!ipp_check_mem_list(c_node)) {
-		DRM_DEBUG_KMS("empty memory.\n");
-		ret = -ENOMEM;
-		goto err_unlock;
-	}
-
-	/* set current property in ippdrv */
-	ret = ipp_set_property(ippdrv, property);
-	if (ret) {
-		DRM_ERROR("failed to set property.\n");
-		ippdrv->c_node = NULL;
-		goto err_unlock;
-	}
-
-	/* check command */
-	switch (property->cmd) {
-	case IPP_CMD_M2M:
-		for_each_ipp_ops(i) {
-			/* source/destination memory list */
-			head = &c_node->mem_list[i];
-
-			m_node = list_first_entry(head,
-				struct drm_exynos_ipp_mem_node, list);
-
-			DRM_DEBUG_KMS("m_node[%pK]\n", m_node);
-
-			ret = ipp_set_mem_node(ippdrv, c_node, m_node);
-			if (ret) {
-				DRM_ERROR("failed to set m node.\n");
-				goto err_unlock;
-			}
-		}
-		break;
-	case IPP_CMD_WB:
-		/* destination memory list */
-		head = &c_node->mem_list[EXYNOS_DRM_OPS_DST];
-
-		list_for_each_entry(m_node, head, list) {
-			ret = ipp_set_mem_node(ippdrv, c_node, m_node);
-			if (ret) {
-				DRM_ERROR("failed to set m node.\n");
-				goto err_unlock;
-			}
-		}
-		break;
-	case IPP_CMD_OUTPUT:
-		/* source memory list */
-		head = &c_node->mem_list[EXYNOS_DRM_OPS_SRC];
-
-		list_for_each_entry(m_node, head, list) {
-			ret = ipp_set_mem_node(ippdrv, c_node, m_node);
-			if (ret) {
-				DRM_ERROR("failed to set m node.\n");
-				goto err_unlock;
-			}
-		}
-		break;
-	default:
-		DRM_ERROR("invalid operations.\n");
-		ret = -EINVAL;
-		goto err_unlock;
-	}
-	mutex_unlock(&c_node->mem_lock);
-
-	DRM_DEBUG_KMS("cmd[%d]\n", property->cmd);
-
-	/* start operations */
-	if (ippdrv->start) {
-		ret = ippdrv->start(ippdrv->dev, property->cmd);
-		if (ret) {
-			DRM_ERROR("failed to start ops.\n");
-			ippdrv->c_node = NULL;
-			return ret;
-		}
-	}
-
-	return 0;
-
-err_unlock:
-	mutex_unlock(&c_node->mem_lock);
-	ippdrv->c_node = NULL;
-	return ret;
-}
-
-static int ipp_stop_property(struct drm_device *drm_dev,
-		struct exynos_drm_ippdrv *ippdrv,
-		struct drm_exynos_ipp_cmd_node *c_node)
-{
-	struct drm_exynos_ipp_property *property = &c_node->property;
-	int i;
-
-	DRM_DEBUG_KMS("prop_id[%d]\n", property->prop_id);
-
-	/* stop operations */
-	if (ippdrv->stop)
-		ippdrv->stop(ippdrv->dev, property->cmd);
-
-	/* check command */
-	switch (property->cmd) {
-	case IPP_CMD_M2M:
-		for_each_ipp_ops(i)
-			ipp_clean_mem_nodes(drm_dev, c_node, i);
-		break;
-	case IPP_CMD_WB:
-		ipp_clean_mem_nodes(drm_dev, c_node, EXYNOS_DRM_OPS_DST);
-		break;
-	case IPP_CMD_OUTPUT:
-		ipp_clean_mem_nodes(drm_dev, c_node, EXYNOS_DRM_OPS_SRC);
-		break;
-	default:
-		DRM_ERROR("invalid operations.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-void ipp_sched_cmd(struct work_struct *work)
-{
-	struct drm_exynos_ipp_cmd_work *cmd_work =
-		container_of(work, struct drm_exynos_ipp_cmd_work, work);
-	struct exynos_drm_ippdrv *ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node;
-	struct drm_exynos_ipp_property *property;
-	int ret;
-
-	ippdrv = cmd_work->ippdrv;
-	if (!ippdrv) {
-		DRM_ERROR("invalid ippdrv list.\n");
-		return;
-	}
-
-	c_node = cmd_work->c_node;
-	if (!c_node) {
-		DRM_ERROR("invalid command node list.\n");
-		return;
-	}
-
-	mutex_lock(&c_node->lock);
-
-	property = &c_node->property;
-
-	switch (cmd_work->ctrl) {
-	case IPP_CTRL_PLAY:
-	case IPP_CTRL_RESUME:
-		ret = ipp_start_property(ippdrv, c_node);
-		if (ret) {
-			DRM_ERROR("failed to start property:prop_id[%d]\n",
-				c_node->property.prop_id);
-			goto err_unlock;
-		}
-
-		/*
-		 * M2M case supports wait_completion of transfer.
-		 * because M2M case supports single unit operation
-		 * with multiple queue.
-		 * M2M need to wait completion of data transfer.
-		 */
-		if (ipp_is_m2m_cmd(property->cmd)) {
-			if (!wait_for_completion_timeout
-			    (&c_node->start_complete, msecs_to_jiffies(200))) {
-				DRM_ERROR("timeout event:prop_id[%d]\n",
-					c_node->property.prop_id);
-				goto err_unlock;
-			}
-		}
-		break;
-	case IPP_CTRL_STOP:
-	case IPP_CTRL_PAUSE:
-		ret = ipp_stop_property(ippdrv->drm_dev, ippdrv,
-			c_node);
-		if (ret) {
-			DRM_ERROR("failed to stop property.\n");
-			goto err_unlock;
-		}
-
-		complete(&c_node->stop_complete);
-		break;
-	default:
-		DRM_ERROR("unknown control type\n");
-		break;
-	}
-
-	DRM_DEBUG_KMS("ctrl[%d] done.\n", cmd_work->ctrl);
-
-err_unlock:
-	mutex_unlock(&c_node->lock);
-}
-
-static int ipp_send_event(struct exynos_drm_ippdrv *ippdrv,
-		struct drm_exynos_ipp_cmd_node *c_node, int *buf_id)
-{
-	struct drm_device *drm_dev = ippdrv->drm_dev;
-	struct drm_exynos_ipp_property *property = &c_node->property;
-	struct drm_exynos_ipp_mem_node *m_node;
-	struct drm_exynos_ipp_queue_buf qbuf;
-	struct drm_exynos_ipp_send_event *e;
-	struct list_head *head;
-	struct timeval now;
-	u32 tbuf_id[EXYNOS_DRM_OPS_MAX] = {0, };
-	int ret, i;
-
-	for_each_ipp_ops(i)
-		DRM_DEBUG_KMS("%s buf_id[%d]\n", i ? "dst" : "src", buf_id[i]);
-
-	if (!drm_dev) {
-		DRM_ERROR("failed to get drm_dev.\n");
-		return -EINVAL;
-	}
-
-	if (!property) {
-		DRM_ERROR("failed to get property.\n");
-		return -EINVAL;
-	}
-
-	mutex_lock(&c_node->event_lock);
-	if (list_empty(&c_node->event_list)) {
-		DRM_DEBUG_KMS("event list is empty.\n");
-		ret = 0;
-		goto err_event_unlock;
-	}
-
-	mutex_lock(&c_node->mem_lock);
-	if (!ipp_check_mem_list(c_node)) {
-		DRM_DEBUG_KMS("empty memory.\n");
-		ret = 0;
-		goto err_mem_unlock;
-	}
-
-	/* check command */
-	switch (property->cmd) {
-	case IPP_CMD_M2M:
-		for_each_ipp_ops(i) {
-			/* source/destination memory list */
-			head = &c_node->mem_list[i];
-
-			m_node = list_first_entry(head,
-				struct drm_exynos_ipp_mem_node, list);
-
-			tbuf_id[i] = m_node->buf_id;
-			DRM_DEBUG_KMS("%s buf_id[%d]\n",
-				i ? "dst" : "src", tbuf_id[i]);
-
-			ret = ipp_put_mem_node(drm_dev, c_node, m_node);
-			if (ret)
-				DRM_ERROR("failed to put m_node.\n");
-		}
-		break;
-	case IPP_CMD_WB:
-		/* clear buf for finding */
-		memset(&qbuf, 0x0, sizeof(qbuf));
-		qbuf.ops_id = EXYNOS_DRM_OPS_DST;
-		qbuf.buf_id = buf_id[EXYNOS_DRM_OPS_DST];
-
-		/* get memory node entry */
-		m_node = ipp_find_mem_node(c_node, &qbuf);
-		if (!m_node) {
-			DRM_ERROR("empty memory node.\n");
-			ret = -ENOMEM;
-			goto err_mem_unlock;
-		}
-
-		tbuf_id[EXYNOS_DRM_OPS_DST] = m_node->buf_id;
-
-		ret = ipp_put_mem_node(drm_dev, c_node, m_node);
-		if (ret)
-			DRM_ERROR("failed to put m_node.\n");
-		break;
-	case IPP_CMD_OUTPUT:
-		/* source memory list */
-		head = &c_node->mem_list[EXYNOS_DRM_OPS_SRC];
-
-		m_node = list_first_entry(head,
-			struct drm_exynos_ipp_mem_node, list);
-
-		tbuf_id[EXYNOS_DRM_OPS_SRC] = m_node->buf_id;
-
-		ret = ipp_put_mem_node(drm_dev, c_node, m_node);
-		if (ret)
-			DRM_ERROR("failed to put m_node.\n");
-		break;
-	default:
-		DRM_ERROR("invalid operations.\n");
-		ret = -EINVAL;
-		goto err_mem_unlock;
-	}
-	mutex_unlock(&c_node->mem_lock);
-
-	if (tbuf_id[EXYNOS_DRM_OPS_DST] != buf_id[EXYNOS_DRM_OPS_DST])
-		DRM_ERROR("failed to match buf_id[%d %d]prop_id[%d]\n",
-			tbuf_id[1], buf_id[1], property->prop_id);
-
-	/*
-	 * command node have event list of destination buffer
-	 * If destination buffer enqueue to mem list,
-	 * then we make event and link to event list tail.
-	 * so, we get first event for first enqueued buffer.
-	 */
-	e = list_first_entry(&c_node->event_list,
-		struct drm_exynos_ipp_send_event, base.link);
-
-	do_gettimeofday(&now);
-	DRM_DEBUG_KMS("tv_sec[%ld]tv_usec[%ld]\n", now.tv_sec, now.tv_usec);
-	e->event.tv_sec = now.tv_sec;
-	e->event.tv_usec = now.tv_usec;
-	e->event.prop_id = property->prop_id;
-
-	/* set buffer id about source destination */
-	for_each_ipp_ops(i)
-		e->event.buf_id[i] = tbuf_id[i];
-
-	drm_send_event(drm_dev, &e->base);
-	mutex_unlock(&c_node->event_lock);
-
-	DRM_DEBUG_KMS("done cmd[%d]prop_id[%d]buf_id[%d]\n",
-		property->cmd, property->prop_id, tbuf_id[EXYNOS_DRM_OPS_DST]);
-
-	return 0;
-
-err_mem_unlock:
-	mutex_unlock(&c_node->mem_lock);
-err_event_unlock:
-	mutex_unlock(&c_node->event_lock);
-	return ret;
-}
-
-void ipp_sched_event(struct work_struct *work)
-{
-	struct drm_exynos_ipp_event_work *event_work =
-		container_of(work, struct drm_exynos_ipp_event_work, work);
-	struct exynos_drm_ippdrv *ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node;
-	int ret;
-
-	if (!event_work) {
-		DRM_ERROR("failed to get event_work.\n");
-		return;
-	}
-
-	DRM_DEBUG_KMS("buf_id[%d]\n", event_work->buf_id[EXYNOS_DRM_OPS_DST]);
-
-	ippdrv = event_work->ippdrv;
-	if (!ippdrv) {
-		DRM_ERROR("failed to get ipp driver.\n");
-		return;
-	}
-
-	c_node = ippdrv->c_node;
-	if (!c_node) {
-		DRM_ERROR("failed to get command node.\n");
-		return;
-	}
-
-	/*
-	 * IPP supports command thread, event thread synchronization.
-	 * If IPP close immediately from user land, then IPP make
-	 * synchronization with command thread, so make complete event.
-	 * or going out operations.
-	 */
-	if (c_node->state != IPP_STATE_START) {
-		DRM_DEBUG_KMS("bypass state[%d]prop_id[%d]\n",
-			c_node->state, c_node->property.prop_id);
-		goto err_completion;
-	}
-
-	ret = ipp_send_event(ippdrv, c_node, event_work->buf_id);
-	if (ret) {
-		DRM_ERROR("failed to send event.\n");
-		goto err_completion;
-	}
-
-err_completion:
-	if (ipp_is_m2m_cmd(c_node->property.cmd))
-		complete(&c_node->start_complete);
-}
-
-static int ipp_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
-{
-	struct ipp_context *ctx = get_ipp_context(dev);
-	struct exynos_drm_ippdrv *ippdrv;
-	int ret, count = 0;
-
-	/* get ipp driver entry */
-	list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
-		ippdrv->drm_dev = drm_dev;
-
-		ret = ipp_create_id(&ctx->ipp_idr, &ctx->ipp_lock, ippdrv);
-		if (ret < 0) {
-			DRM_ERROR("failed to create id.\n");
-			goto err;
-		}
-		ippdrv->prop_list.ipp_id = ret;
-
-		DRM_DEBUG_KMS("count[%d]ippdrv[%pK]ipp_id[%d]\n",
-			count++, ippdrv, ret);
-
-		/* store parent device for node */
-		ippdrv->parent_dev = dev;
-
-		/* store event work queue and handler */
-		ippdrv->event_workq = ctx->event_workq;
-		ippdrv->sched_event = ipp_sched_event;
-		INIT_LIST_HEAD(&ippdrv->cmd_list);
-		mutex_init(&ippdrv->cmd_lock);
-
-		ret = drm_iommu_attach_device(drm_dev, ippdrv->dev);
-		if (ret) {
-			DRM_ERROR("failed to activate iommu\n");
-			goto err;
-		}
-	}
-
-	return 0;
-
-err:
-	/* get ipp driver entry */
-	list_for_each_entry_continue_reverse(ippdrv, &exynos_drm_ippdrv_list,
-						drv_list) {
-		drm_iommu_detach_device(drm_dev, ippdrv->dev);
-
-		ipp_remove_id(&ctx->ipp_idr, &ctx->ipp_lock,
-				ippdrv->prop_list.ipp_id);
-	}
-
-	return ret;
-}
-
-static void ipp_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
-{
-	struct exynos_drm_ippdrv *ippdrv, *t;
-	struct ipp_context *ctx = get_ipp_context(dev);
-
-	/* get ipp driver entry */
-	list_for_each_entry_safe(ippdrv, t, &exynos_drm_ippdrv_list, drv_list) {
-		drm_iommu_detach_device(drm_dev, ippdrv->dev);
-
-		ipp_remove_id(&ctx->ipp_idr, &ctx->ipp_lock,
-				ippdrv->prop_list.ipp_id);
-
-		ippdrv->drm_dev = NULL;
-		exynos_drm_ippdrv_unregister(ippdrv);
-	}
-}
-
-static int ipp_subdrv_open(struct drm_device *drm_dev, struct device *dev,
-		struct drm_file *file)
-{
-	struct drm_exynos_file_private *file_priv = file->driver_priv;
-
-	file_priv->ipp_dev = dev;
-
-	DRM_DEBUG_KMS("done priv[%pK]\n", dev);
-
-	return 0;
-}
-
-static void ipp_subdrv_close(struct drm_device *drm_dev, struct device *dev,
-		struct drm_file *file)
-{
-	struct exynos_drm_ippdrv *ippdrv = NULL;
-	struct ipp_context *ctx = get_ipp_context(dev);
-	struct drm_exynos_ipp_cmd_node *c_node, *tc_node;
-	int count = 0;
-
-	list_for_each_entry(ippdrv, &exynos_drm_ippdrv_list, drv_list) {
-		mutex_lock(&ippdrv->cmd_lock);
-		list_for_each_entry_safe(c_node, tc_node,
-			&ippdrv->cmd_list, list) {
-			DRM_DEBUG_KMS("count[%d]ippdrv[%pK]\n",
-				count++, ippdrv);
-
-			if (c_node->filp == file) {
-				/*
-				 * userland goto unnormal state. process killed.
-				 * and close the file.
-				 * so, IPP didn't called stop cmd ctrl.
-				 * so, we are make stop operation in this state.
-				 */
-				if (c_node->state == IPP_STATE_START) {
-					ipp_stop_property(drm_dev, ippdrv,
-						c_node);
-					c_node->state = IPP_STATE_STOP;
-				}
-
-				ippdrv->dedicated = false;
-				ipp_clean_cmd_node(ctx, c_node);
-				if (list_empty(&ippdrv->cmd_list))
-					pm_runtime_put_sync(ippdrv->dev);
-			}
-		}
-		mutex_unlock(&ippdrv->cmd_lock);
-	}
-
-	return;
-}
-
-static int ipp_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct ipp_context *ctx;
-	struct exynos_drm_subdrv *subdrv;
-	int ret;
-
-	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
-
-	mutex_init(&ctx->ipp_lock);
-	mutex_init(&ctx->prop_lock);
-
-	idr_init(&ctx->ipp_idr);
-	idr_init(&ctx->prop_idr);
-
-	/*
-	 * create single thread for ipp event
-	 * IPP supports event thread for IPP drivers.
-	 * IPP driver send event_work to this thread.
-	 * and IPP event thread send event to user process.
-	 */
-	ctx->event_workq = create_singlethread_workqueue("ipp_event");
-	if (!ctx->event_workq) {
-		dev_err(dev, "failed to create event workqueue\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * create single thread for ipp command
-	 * IPP supports command thread for user process.
-	 * user process make command node using set property ioctl.
-	 * and make start_work and send this work to command thread.
-	 * and then this command thread start property.
-	 */
-	ctx->cmd_workq = create_singlethread_workqueue("ipp_cmd");
-	if (!ctx->cmd_workq) {
-		dev_err(dev, "failed to create cmd workqueue\n");
-		ret = -EINVAL;
-		goto err_event_workq;
-	}
-
-	/* set sub driver informations */
-	subdrv = &ctx->subdrv;
-	subdrv->dev = dev;
-	subdrv->probe = ipp_subdrv_probe;
-	subdrv->remove = ipp_subdrv_remove;
-	subdrv->open = ipp_subdrv_open;
-	subdrv->close = ipp_subdrv_close;
-
-	platform_set_drvdata(pdev, ctx);
-
-	ret = exynos_drm_subdrv_register(subdrv);
-	if (ret < 0) {
-		DRM_ERROR("failed to register drm ipp device.\n");
-		goto err_cmd_workq;
-	}
-
-	dev_info(dev, "drm ipp registered successfully.\n");
-
-	return 0;
-
-err_cmd_workq:
-	destroy_workqueue(ctx->cmd_workq);
-err_event_workq:
-	destroy_workqueue(ctx->event_workq);
-	return ret;
-}
-
-static int ipp_remove(struct platform_device *pdev)
-{
-	struct ipp_context *ctx = platform_get_drvdata(pdev);
-
-	/* unregister sub driver */
-	exynos_drm_subdrv_unregister(&ctx->subdrv);
-
-	/* remove,destroy ipp idr */
-	idr_destroy(&ctx->ipp_idr);
-	idr_destroy(&ctx->prop_idr);
-
-	mutex_destroy(&ctx->ipp_lock);
-	mutex_destroy(&ctx->prop_lock);
-
-	/* destroy command, event work queue */
-	destroy_workqueue(ctx->cmd_workq);
-	destroy_workqueue(ctx->event_workq);
-
-	return 0;
-}
-
-struct platform_driver ipp_driver = {
-	.probe		= ipp_probe,
-	.remove		= ipp_remove,
-	.driver		= {
-		.name	= "exynos-drm-ipp",
-		.owner	= THIS_MODULE,
-	},
-};
-
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.h b/drivers/gpu/drm/exynos/exynos_drm_ipp.h
deleted file mode 100644
index 2a61547a39d0..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_ipp.h
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Copyright (c) 2012 Samsung Electronics Co., Ltd.
- *
- * Authors:
- *	Eunchul Kim <chulspro.kim@samsung.com>
- *	Jinyoung Jeon <jy0.jeon@samsung.com>
- *	Sangmin Lee <lsmin.lee@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#ifndef _EXYNOS_DRM_IPP_H_
-#define _EXYNOS_DRM_IPP_H_
-
-#define for_each_ipp_ops(pos)	\
-	for (pos = 0; pos < EXYNOS_DRM_OPS_MAX; pos++)
-#define for_each_ipp_planar(pos)	\
-	for (pos = 0; pos < EXYNOS_DRM_PLANAR_MAX; pos++)
-
-#define IPP_GET_LCD_WIDTH	_IOR('F', 302, int)
-#define IPP_GET_LCD_HEIGHT	_IOR('F', 303, int)
-#define IPP_SET_WRITEBACK	_IOW('F', 304, u32)
-
-/* definition of state */
-enum drm_exynos_ipp_state {
-	IPP_STATE_IDLE,
-	IPP_STATE_START,
-	IPP_STATE_STOP,
-};
-
-/*
- * A structure of command work information.
- * @work: work structure.
- * @ippdrv: current work ippdrv.
- * @c_node: command node information.
- * @ctrl: command control.
- */
-struct drm_exynos_ipp_cmd_work {
-	struct work_struct	work;
-	struct exynos_drm_ippdrv	*ippdrv;
-	struct drm_exynos_ipp_cmd_node *c_node;
-	enum drm_exynos_ipp_ctrl	ctrl;
-};
-
-/*
- * A structure of command node.
- *
- * @list: list head to command queue information.
- * @event_list: list head of event.
- * @mem_list: list head to source,destination memory queue information.
- * @lock: lock for synchronization of access to ioctl.
- * @mem_lock: lock for synchronization of access to memory nodes.
- * @event_lock: lock for synchronization of access to scheduled event.
- * @start_complete: completion of start of command.
- * @stop_complete: completion of stop of command.
- * @property: property information.
- * @start_work: start command work structure.
- * @stop_work: stop command work structure.
- * @event_work: event work structure.
- * @state: state of command node.
- * @filp: associated file pointer.
- */
-struct drm_exynos_ipp_cmd_node {
-	struct list_head	list;
-	struct list_head	event_list;
-	struct list_head	mem_list[EXYNOS_DRM_OPS_MAX];
-	struct mutex	lock;
-	struct mutex	mem_lock;
-	struct mutex	event_lock;
-	struct completion	start_complete;
-	struct completion	stop_complete;
-	struct drm_exynos_ipp_property	property;
-	struct drm_exynos_ipp_cmd_work *start_work;
-	struct drm_exynos_ipp_cmd_work *stop_work;
-	struct drm_exynos_ipp_event_work *event_work;
-	enum drm_exynos_ipp_state	state;
-	struct drm_file	*filp;
-};
-
-/*
- * A structure of buffer information.
- *
- * @handles: Y, Cb, Cr each gem object handle.
- * @base: Y, Cb, Cr each planar address.
- */
-struct drm_exynos_ipp_buf_info {
-	unsigned long	handles[EXYNOS_DRM_PLANAR_MAX];
-	dma_addr_t	base[EXYNOS_DRM_PLANAR_MAX];
-};
-
-/*
- * A structure of wb setting information.
- *
- * @enable: enable flag for wb.
- * @refresh: HZ of the refresh rate.
- */
-struct drm_exynos_ipp_set_wb {
-	__u32	enable;
-	__u32	refresh;
-};
-
-/*
- * A structure of event work information.
- *
- * @work: work structure.
- * @ippdrv: current work ippdrv.
- * @buf_id: id of src, dst buffer.
- */
-struct drm_exynos_ipp_event_work {
-	struct work_struct	work;
-	struct exynos_drm_ippdrv *ippdrv;
-	u32	buf_id[EXYNOS_DRM_OPS_MAX];
-};
-
-/*
- * A structure of source,destination operations.
- *
- * @set_fmt: set format of image.
- * @set_transf: set transform(rotations, flip).
- * @set_size: set size of region.
- * @set_addr: set address for dma.
- */
-struct exynos_drm_ipp_ops {
-	int (*set_fmt)(struct device *dev, u32 fmt);
-	int (*set_transf)(struct device *dev,
-		enum drm_exynos_degree degree,
-		enum drm_exynos_flip flip, bool *swap);
-	int (*set_size)(struct device *dev, int swap,
-		struct drm_exynos_pos *pos, struct drm_exynos_sz *sz);
-	int (*set_addr)(struct device *dev,
-		 struct drm_exynos_ipp_buf_info *buf_info, u32 buf_id,
-		enum drm_exynos_ipp_buf_type buf_type);
-};
-
-/*
- * A structure of ipp driver.
- *
- * @drv_list: list head for registed sub driver information.
- * @parent_dev: parent device information.
- * @dev: platform device.
- * @drm_dev: drm device.
- * @dedicated: dedicated ipp device.
- * @ops: source, destination operations.
- * @event_workq: event work queue.
- * @c_node: current command information.
- * @cmd_list: list head for command information.
- * @cmd_lock: lock for synchronization of access to cmd_list.
- * @prop_list: property informations of current ipp driver.
- * @check_property: check property about format, size, buffer.
- * @reset: reset ipp block.
- * @start: ipp each device start.
- * @stop: ipp each device stop.
- * @sched_event: work schedule handler.
- */
-struct exynos_drm_ippdrv {
-	struct list_head	drv_list;
-	struct device	*parent_dev;
-	struct device	*dev;
-	struct drm_device	*drm_dev;
-	bool	dedicated;
-	struct exynos_drm_ipp_ops	*ops[EXYNOS_DRM_OPS_MAX];
-	struct workqueue_struct	*event_workq;
-	struct drm_exynos_ipp_cmd_node *c_node;
-	struct list_head	cmd_list;
-	struct mutex	cmd_lock;
-	struct drm_exynos_ipp_prop_list prop_list;
-
-	int (*check_property)(struct device *dev,
-		struct drm_exynos_ipp_property *property);
-	int (*reset)(struct device *dev);
-	int (*start)(struct device *dev, enum drm_exynos_ipp_cmd cmd);
-	void (*stop)(struct device *dev, enum drm_exynos_ipp_cmd cmd);
-	void (*sched_event)(struct work_struct *work);
-};
-
-#ifdef CONFIG_DRM_EXYNOS_IPP
-extern int exynos_drm_ippdrv_register(struct exynos_drm_ippdrv *ippdrv);
-extern int exynos_drm_ippdrv_unregister(struct exynos_drm_ippdrv *ippdrv);
-extern int exynos_drm_ipp_get_property(struct drm_device *drm_dev, void *data,
-					 struct drm_file *file);
-extern int exynos_drm_ipp_set_property(struct drm_device *drm_dev, void *data,
-					 struct drm_file *file);
-extern int exynos_drm_ipp_queue_buf(struct drm_device *drm_dev, void *data,
-					 struct drm_file *file);
-extern int exynos_drm_ipp_cmd_ctrl(struct drm_device *drm_dev, void *data,
-					 struct drm_file *file);
-extern int exynos_drm_ippnb_register(struct notifier_block *nb);
-extern int exynos_drm_ippnb_unregister(struct notifier_block *nb);
-extern int exynos_drm_ippnb_send_event(unsigned long val, void *v);
-extern void ipp_sched_cmd(struct work_struct *work);
-extern void ipp_sched_event(struct work_struct *work);
-
-#else
-static inline int exynos_drm_ippdrv_register(struct exynos_drm_ippdrv *ippdrv)
-{
-	return -ENODEV;
-}
-
-static inline int exynos_drm_ippdrv_unregister(struct exynos_drm_ippdrv *ippdrv)
-{
-	return -ENODEV;
-}
-
-static inline int exynos_drm_ipp_get_property(struct drm_device *drm_dev,
-						void *data,
-						struct drm_file *file_priv)
-{
-	return -ENOTTY;
-}
-
-static inline int exynos_drm_ipp_set_property(struct drm_device *drm_dev,
-						void *data,
-						struct drm_file *file_priv)
-{
-	return -ENOTTY;
-}
-
-static inline int exynos_drm_ipp_queue_buf(struct drm_device *drm_dev,
-						void *data,
-						struct drm_file *file)
-{
-	return -ENOTTY;
-}
-
-static inline int exynos_drm_ipp_cmd_ctrl(struct drm_device *drm_dev,
-						void *data,
-						struct drm_file *file)
-{
-	return -ENOTTY;
-}
-
-static inline int exynos_drm_ippnb_register(struct notifier_block *nb)
-{
-	return -ENODEV;
-}
-
-static inline int exynos_drm_ippnb_unregister(struct notifier_block *nb)
-{
-	return -ENODEV;
-}
-
-static inline int exynos_drm_ippnb_send_event(unsigned long val, void *v)
-{
-	return -ENOTTY;
-}
-#endif
-
-#endif /* _EXYNOS_DRM_IPP_H_ */
-
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.h b/drivers/gpu/drm/exynos/exynos_drm_rotator.h
deleted file mode 100644
index 71a0b4c0c1e8..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.h
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright (c) 2012 Samsung Electronics Co., Ltd.
- *
- * Authors:
- *	YoungJun Cho <yj44.cho@samsung.com>
- *	Eunchul Kim <chulspro.kim@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#ifndef	_EXYNOS_DRM_ROTATOR_H_
-#define	_EXYNOS_DRM_ROTATOR_H_
-
-/* TODO */
-
-#endif
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 82d1b7e2febe..abd84cbcf1c2 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -829,7 +829,8 @@ static void hdmi_reg_infoframes(struct hdmi_context *hdata)
 		DRM_INFO("%s: invalid AVI infoframe (%d)\n", __func__, ret);
 	}
 
-	ret = drm_hdmi_vendor_infoframe_from_display_mode(&frm.vendor.hdmi, m);
+	ret = drm_hdmi_vendor_infoframe_from_display_mode(&frm.vendor.hdmi,
+							  &hdata->connector, m);
 	if (!ret)
 		ret = hdmi_vendor_infoframe_pack(&frm.vendor.hdmi, buf,
 				sizeof(buf));
@@ -1067,10 +1068,13 @@ static void hdmi_audio_config(struct hdmi_context *hdata)
 	/* Configuration I2S input ports. Configure I2S_PIN_SEL_0~4 */
 	hdmi_reg_writeb(hdata, HDMI_I2S_PIN_SEL_0, HDMI_I2S_SEL_SCLK(5)
 			| HDMI_I2S_SEL_LRCK(6));
-	hdmi_reg_writeb(hdata, HDMI_I2S_PIN_SEL_1, HDMI_I2S_SEL_SDATA1(1)
-			| HDMI_I2S_SEL_SDATA2(4));
+
+	hdmi_reg_writeb(hdata, HDMI_I2S_PIN_SEL_1, HDMI_I2S_SEL_SDATA1(3)
+			| HDMI_I2S_SEL_SDATA0(4));
+
 	hdmi_reg_writeb(hdata, HDMI_I2S_PIN_SEL_2, HDMI_I2S_SEL_SDATA3(1)
 			| HDMI_I2S_SEL_SDATA2(2));
+
 	hdmi_reg_writeb(hdata, HDMI_I2S_PIN_SEL_3, HDMI_I2S_SEL_DSD(0));
 
 	/* I2S_CON_1 & 2 */
diff --git a/include/video/exynos5433_decon.h b/drivers/gpu/drm/exynos/regs-decon5433.h
similarity index 98%
rename from include/video/exynos5433_decon.h
rename to drivers/gpu/drm/exynos/regs-decon5433.h
index 78957c9626f5..19ad9e47945e 100644
--- a/include/video/exynos5433_decon.h
+++ b/drivers/gpu/drm/exynos/regs-decon5433.h
@@ -6,8 +6,8 @@
  * published by the Free Software Foundationr
  */
 
-#ifndef EXYNOS_REGS_DECON_H
-#define EXYNOS_REGS_DECON_H
+#ifndef EXYNOS_REGS_DECON5433_H
+#define EXYNOS_REGS_DECON5433_H
 
 /* Exynos543X DECON */
 #define DECON_VIDCON0			0x0000
@@ -206,4 +206,4 @@
 #define CRCCTRL_CRCEN			(0x1 << 0)
 #define CRCCTRL_MASK			(0x7)
 
-#endif /* EXYNOS_REGS_DECON_H */
+#endif /* EXYNOS_REGS_DECON5433_H */
diff --git a/include/video/exynos7_decon.h b/drivers/gpu/drm/exynos/regs-decon7.h
similarity index 99%
rename from include/video/exynos7_decon.h
rename to drivers/gpu/drm/exynos/regs-decon7.h
index a62b11b613f6..5df7765d2397 100644
--- a/include/video/exynos7_decon.h
+++ b/drivers/gpu/drm/exynos/regs-decon7.h
@@ -1,5 +1,4 @@
-/* include/video/exynos7_decon.h
- *
+/*
  * Copyright (c) 2014 Samsung Electronics Co., Ltd.
  * Author: Ajay Kumar <ajaykumar.rs@samsung.com>
  *
@@ -9,6 +8,9 @@
  * option) any later version.
  */
 
+#ifndef EXYNOS_REGS_DECON7_H
+#define EXYNOS_REGS_DECON7_H
+
 /* VIDCON0 */
 #define VIDCON0					0x00
 
@@ -347,3 +349,5 @@
 
 #define DECON_UPDATE_SLAVE_SYNC			(1 << 4)
 #define DECON_UPDATE_STANDALONE_F		(1 << 0)
+
+#endif /* EXYNOS_REGS_DECON7_H */
diff --git a/drivers/gpu/drm/exynos/regs-fimc.h b/drivers/gpu/drm/exynos/regs-fimc.h
index 30496134a3d0..d7cbe53c4c01 100644
--- a/drivers/gpu/drm/exynos/regs-fimc.h
+++ b/drivers/gpu/drm/exynos/regs-fimc.h
@@ -569,7 +569,7 @@
 #define EXYNOS_CIIMGEFF_FIN_EMBOSSING		(4 << 26)
 #define EXYNOS_CIIMGEFF_FIN_SILHOUETTE		(5 << 26)
 #define EXYNOS_CIIMGEFF_FIN_MASK			(7 << 26)
-#define EXYNOS_CIIMGEFF_PAT_CBCR_MASK		((0xff < 13) | (0xff < 0))
+#define EXYNOS_CIIMGEFF_PAT_CBCR_MASK		((0xff << 13) | (0xff << 0))
 
 /* Real input DMA size register */
 #define EXYNOS_CIREAL_ISIZE_AUTOLOAD_ENABLE	(1 << 31)
diff --git a/drivers/gpu/drm/exynos/regs-hdmi.h b/drivers/gpu/drm/exynos/regs-hdmi.h
index 04be0f7e8193..4420c203ac85 100644
--- a/drivers/gpu/drm/exynos/regs-hdmi.h
+++ b/drivers/gpu/drm/exynos/regs-hdmi.h
@@ -464,7 +464,7 @@
 
 /* I2S_PIN_SEL_1 */
 #define HDMI_I2S_SEL_SDATA1(x)		(((x) & 0x7) << 4)
-#define HDMI_I2S_SEL_SDATA2(x)		((x) & 0x7)
+#define HDMI_I2S_SEL_SDATA0(x)		((x) & 0x7)
 
 /* I2S_PIN_SEL_2 */
 #define HDMI_I2S_SEL_SDATA3(x)		(((x) & 0x7) << 4)
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index faf17b83b910..80232321a244 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -27,6 +27,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_modeset_helper.h>
 
 #include "fsl_dcu_drm_crtc.h"
 #include "fsl_dcu_drm_drv.h"
@@ -188,26 +189,17 @@ static struct drm_driver fsl_dcu_drm_driver = {
 static int fsl_dcu_drm_pm_suspend(struct device *dev)
 {
 	struct fsl_dcu_drm_device *fsl_dev = dev_get_drvdata(dev);
+	int ret;
 
 	if (!fsl_dev)
 		return 0;
 
 	disable_irq(fsl_dev->irq);
-	drm_kms_helper_poll_disable(fsl_dev->drm);
-
-	console_lock();
-	drm_fbdev_cma_set_suspend(fsl_dev->fbdev, 1);
-	console_unlock();
-
-	fsl_dev->state = drm_atomic_helper_suspend(fsl_dev->drm);
-	if (IS_ERR(fsl_dev->state)) {
-		console_lock();
-		drm_fbdev_cma_set_suspend(fsl_dev->fbdev, 0);
-		console_unlock();
 
-		drm_kms_helper_poll_enable(fsl_dev->drm);
+	ret = drm_mode_config_helper_suspend(fsl_dev->drm);
+	if (ret) {
 		enable_irq(fsl_dev->irq);
-		return PTR_ERR(fsl_dev->state);
+		return ret;
 	}
 
 	clk_disable_unprepare(fsl_dev->clk);
@@ -233,13 +225,8 @@ static int fsl_dcu_drm_pm_resume(struct device *dev)
 		fsl_tcon_bypass_enable(fsl_dev->tcon);
 	fsl_dcu_drm_init_planes(fsl_dev->drm);
 	enable_irq(fsl_dev->irq);
-	drm_atomic_helper_resume(fsl_dev->drm, fsl_dev->state);
-
-	console_lock();
-	drm_fbdev_cma_set_suspend(fsl_dev->fbdev, 0);
-	console_unlock();
 
-	drm_kms_helper_poll_enable(fsl_dev->drm);
+	drm_mode_config_helper_resume(fsl_dev->drm);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
index da9bfd432ca6..93bfb98012d4 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
@@ -196,7 +196,6 @@ struct fsl_dcu_drm_device {
 	struct drm_encoder encoder;
 	struct fsl_dcu_drm_connector connector;
 	const struct fsl_dcu_soc_data *soc;
-	struct drm_atomic_state *state;
 };
 
 int fsl_dcu_drm_modeset_init(struct fsl_dcu_drm_device *fsl_dev);
diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c
index 8745971a7680..3a3bf752e03a 100644
--- a/drivers/gpu/drm/gma500/cdv_device.c
+++ b/drivers/gpu/drm/gma500/cdv_device.c
@@ -185,21 +185,22 @@ static int cdv_backlight_init(struct drm_device *dev)
  *	for this and the MID devices.
  */
 
-static inline u32 CDV_MSG_READ32(uint port, uint offset)
+static inline u32 CDV_MSG_READ32(int domain, uint port, uint offset)
 {
 	int mcr = (0x10<<24) | (port << 16) | (offset << 8);
 	uint32_t ret_val = 0;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root = pci_get_domain_bus_and_slot(domain, 0, 0);
 	pci_write_config_dword(pci_root, 0xD0, mcr);
 	pci_read_config_dword(pci_root, 0xD4, &ret_val);
 	pci_dev_put(pci_root);
 	return ret_val;
 }
 
-static inline void CDV_MSG_WRITE32(uint port, uint offset, u32 value)
+static inline void CDV_MSG_WRITE32(int domain, uint port, uint offset,
+				   u32 value)
 {
 	int mcr = (0x11<<24) | (port << 16) | (offset << 8) | 0xF0;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root = pci_get_domain_bus_and_slot(domain, 0, 0);
 	pci_write_config_dword(pci_root, 0xD4, value);
 	pci_write_config_dword(pci_root, 0xD0, mcr);
 	pci_dev_put(pci_root);
@@ -216,11 +217,12 @@ static void cdv_init_pm(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = dev->dev_private;
 	u32 pwr_cnt;
+	int domain = pci_domain_nr(dev->pdev->bus);
 	int i;
 
-	dev_priv->apm_base = CDV_MSG_READ32(PSB_PUNIT_PORT,
+	dev_priv->apm_base = CDV_MSG_READ32(domain, PSB_PUNIT_PORT,
 							PSB_APMBA) & 0xFFFF;
-	dev_priv->ospm_base = CDV_MSG_READ32(PSB_PUNIT_PORT,
+	dev_priv->ospm_base = CDV_MSG_READ32(domain, PSB_PUNIT_PORT,
 							PSB_OSPMBA) & 0xFFFF;
 
 	/* Power status */
@@ -251,7 +253,7 @@ static void cdv_errata(struct drm_device *dev)
 	 *	Bonus Launch to work around the issue, by degrading
 	 *	performance.
 	 */
-	 CDV_MSG_WRITE32(3, 0x30, 0x08027108);
+	 CDV_MSG_WRITE32(pci_domain_nr(dev->pdev->bus), 3, 0x30, 0x08027108);
 }
 
 /**
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 2570c7f647a6..cb0a2ae916e0 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -576,13 +576,6 @@ static void psb_fbdev_fini(struct drm_device *dev)
 	dev_priv->fbdev = NULL;
 }
 
-static void psbfb_output_poll_changed(struct drm_device *dev)
-{
-	struct drm_psb_private *dev_priv = dev->dev_private;
-	struct psb_fbdev *fbdev = (struct psb_fbdev *)dev_priv->fbdev;
-	drm_fb_helper_hotplug_event(&fbdev->psb_fb_helper);
-}
-
 /**
  *	psb_user_framebuffer_create_handle - add hamdle to a framebuffer
  *	@fb: framebuffer
@@ -623,7 +616,7 @@ static void psb_user_framebuffer_destroy(struct drm_framebuffer *fb)
 
 static const struct drm_mode_config_funcs psb_mode_funcs = {
 	.fb_create = psb_user_framebuffer_create,
-	.output_poll_changed = psbfb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 };
 
 static void psb_setup_outputs(struct drm_device *dev)
diff --git a/drivers/gpu/drm/gma500/gma_device.c b/drivers/gpu/drm/gma500/gma_device.c
index 4a295f9ba067..a7fb6de4dd15 100644
--- a/drivers/gpu/drm/gma500/gma_device.c
+++ b/drivers/gpu/drm/gma500/gma_device.c
@@ -19,7 +19,9 @@
 void gma_get_core_freq(struct drm_device *dev)
 {
 	uint32_t clock;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root =
+		pci_get_domain_bus_and_slot(pci_domain_nr(dev->pdev->bus),
+					    0, 0);
 	struct drm_psb_private *dev_priv = dev->dev_private;
 
 	/*pci_write_config_dword(pci_root, 0xD4, 0x00C32004);*/
diff --git a/drivers/gpu/drm/gma500/mid_bios.c b/drivers/gpu/drm/gma500/mid_bios.c
index 1fa163373a47..7171b7475f58 100644
--- a/drivers/gpu/drm/gma500/mid_bios.c
+++ b/drivers/gpu/drm/gma500/mid_bios.c
@@ -32,7 +32,9 @@
 static void mid_get_fuse_settings(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = dev->dev_private;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root =
+		pci_get_domain_bus_and_slot(pci_domain_nr(dev->pdev->bus),
+					    0, 0);
 	uint32_t fuse_value = 0;
 	uint32_t fuse_value_tmp = 0;
 
@@ -104,7 +106,9 @@ static void mid_get_fuse_settings(struct drm_device *dev)
 static void mid_get_pci_revID(struct drm_psb_private *dev_priv)
 {
 	uint32_t platform_rev_id = 0;
-	struct pci_dev *pci_gfx_root = pci_get_bus_and_slot(0, PCI_DEVFN(2, 0));
+	int domain = pci_domain_nr(dev_priv->dev->pdev->bus);
+	struct pci_dev *pci_gfx_root =
+		pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(2, 0));
 
 	if (pci_gfx_root == NULL) {
 		WARN_ON(1);
@@ -281,7 +285,9 @@ static void mid_get_vbt_data(struct drm_psb_private *dev_priv)
 	u32 addr;
 	u8 __iomem *vbt_virtual;
 	struct mid_vbt_header vbt_header;
-	struct pci_dev *pci_gfx_root = pci_get_bus_and_slot(0, PCI_DEVFN(2, 0));
+	struct pci_dev *pci_gfx_root =
+		pci_get_domain_bus_and_slot(pci_domain_nr(dev->pdev->bus),
+					    0, PCI_DEVFN(2, 0));
 	int ret = -1;
 
 	/* Get the address of the platform config vbt */
diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c
index 0eaf11c19939..ccb161c73a59 100644
--- a/drivers/gpu/drm/gma500/mmu.c
+++ b/drivers/gpu/drm/gma500/mmu.c
@@ -395,7 +395,7 @@ static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
 			psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
 			atomic_set(&pd->driver->needs_tlbflush, 1);
 		}
-		kunmap_atomic(pt->v);
+		kunmap_atomic(v);
 		spin_unlock(&pd->driver->lock);
 		psb_mmu_free_pt(pt);
 		return;
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 37a3be71acd9..ac32ab5aa002 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -107,19 +107,6 @@ MODULE_DEVICE_TABLE(pci, pciidlist);
 static const struct drm_ioctl_desc psb_ioctls[] = {
 };
 
-static void psb_driver_lastclose(struct drm_device *dev)
-{
-	int ret;
-	struct drm_psb_private *dev_priv = dev->dev_private;
-	struct psb_fbdev *fbdev = dev_priv->fbdev;
-
-	ret = drm_fb_helper_restore_fbdev_mode_unlocked(&fbdev->psb_fb_helper);
-	if (ret)
-		DRM_DEBUG("failed to restore crtc mode\n");
-
-	return;
-}
-
 static int psb_do_init(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = dev->dev_private;
@@ -261,7 +248,11 @@ static int psb_driver_load(struct drm_device *dev, unsigned long flags)
 		goto out_err;
 
 	if (IS_MRST(dev)) {
-		dev_priv->aux_pdev = pci_get_bus_and_slot(0, PCI_DEVFN(3, 0));
+		int domain = pci_domain_nr(dev->pdev->bus);
+
+		dev_priv->aux_pdev =
+			pci_get_domain_bus_and_slot(domain, 0,
+						    PCI_DEVFN(3, 0));
 
 		if (dev_priv->aux_pdev) {
 			resource_start = pci_resource_start(dev_priv->aux_pdev,
@@ -281,7 +272,9 @@ static int psb_driver_load(struct drm_device *dev, unsigned long flags)
 		}
 		dev_priv->gmbus_reg = dev_priv->aux_reg;
 
-		dev_priv->lpc_pdev = pci_get_bus_and_slot(0, PCI_DEVFN(31, 0));
+		dev_priv->lpc_pdev =
+			pci_get_domain_bus_and_slot(domain, 0,
+						    PCI_DEVFN(31, 0));
 		if (dev_priv->lpc_pdev) {
 			pci_read_config_word(dev_priv->lpc_pdev, PSB_LPC_GBA,
 				&dev_priv->lpc_gpio_base);
@@ -479,7 +472,7 @@ static struct drm_driver driver = {
 			   DRIVER_MODESET | DRIVER_GEM,
 	.load = psb_driver_load,
 	.unload = psb_driver_unload,
-	.lastclose = psb_driver_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 
 	.num_ioctls = ARRAY_SIZE(psb_ioctls),
 	.irq_preinstall = psb_irq_preinstall,
@@ -527,4 +520,4 @@ module_exit(psb_exit);
 
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index 821497dbd3fc..e8300f509023 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -36,7 +36,6 @@
 #include "mmu.h"
 
 #define DRIVER_AUTHOR "Alan Cox <alan@linux.intel.com> and others"
-#define DRIVER_LICENSE "GPL"
 
 #define DRIVER_NAME "gma500"
 #define DRIVER_DESC "DRM driver for the Intel GMA500, GMA600, GMA3600, GMA3650"
@@ -781,38 +780,40 @@ extern const struct psb_ops cdv_chip_ops;
 extern int drm_idle_check_interval;
 
 /* Utilities */
-static inline u32 MRST_MSG_READ32(uint port, uint offset)
+static inline u32 MRST_MSG_READ32(int domain, uint port, uint offset)
 {
 	int mcr = (0xD0<<24) | (port << 16) | (offset << 8);
 	uint32_t ret_val = 0;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root = pci_get_domain_bus_and_slot(domain, 0, 0);
 	pci_write_config_dword(pci_root, 0xD0, mcr);
 	pci_read_config_dword(pci_root, 0xD4, &ret_val);
 	pci_dev_put(pci_root);
 	return ret_val;
 }
-static inline void MRST_MSG_WRITE32(uint port, uint offset, u32 value)
+static inline void MRST_MSG_WRITE32(int domain, uint port, uint offset,
+				    u32 value)
 {
 	int mcr = (0xE0<<24) | (port << 16) | (offset << 8) | 0xF0;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root = pci_get_domain_bus_and_slot(domain, 0, 0);
 	pci_write_config_dword(pci_root, 0xD4, value);
 	pci_write_config_dword(pci_root, 0xD0, mcr);
 	pci_dev_put(pci_root);
 }
-static inline u32 MDFLD_MSG_READ32(uint port, uint offset)
+static inline u32 MDFLD_MSG_READ32(int domain, uint port, uint offset)
 {
 	int mcr = (0x10<<24) | (port << 16) | (offset << 8);
 	uint32_t ret_val = 0;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root = pci_get_domain_bus_and_slot(domain, 0, 0);
 	pci_write_config_dword(pci_root, 0xD0, mcr);
 	pci_read_config_dword(pci_root, 0xD4, &ret_val);
 	pci_dev_put(pci_root);
 	return ret_val;
 }
-static inline void MDFLD_MSG_WRITE32(uint port, uint offset, u32 value)
+static inline void MDFLD_MSG_WRITE32(int domain, uint port, uint offset,
+				     u32 value)
 {
 	int mcr = (0x11<<24) | (port << 16) | (offset << 8) | 0xF0;
-	struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+	struct pci_dev *pci_root = pci_get_domain_bus_and_slot(domain, 0, 0);
 	pci_write_config_dword(pci_root, 0xD4, value);
 	pci_write_config_dword(pci_root, 0xD0, mcr);
 	pci_dev_put(pci_root);
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
index 3518167a7dc4..8516e005643f 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
@@ -223,9 +223,10 @@ static struct ttm_tt *hibmc_ttm_tt_create(struct ttm_bo_device *bdev,
 	return tt;
 }
 
-static int hibmc_ttm_tt_populate(struct ttm_tt *ttm)
+static int hibmc_ttm_tt_populate(struct ttm_tt *ttm,
+		struct ttm_operation_ctx *ctx)
 {
-	return ttm_pool_populate(ttm);
+	return ttm_pool_populate(ttm, ctx);
 }
 
 static void hibmc_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -344,6 +345,7 @@ int hibmc_bo_create(struct drm_device *dev, int size, int align,
 
 int hibmc_bo_pin(struct hibmc_bo *bo, u32 pl_flag, u64 *gpu_addr)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 
 	if (bo->pin_count) {
@@ -356,7 +358,7 @@ int hibmc_bo_pin(struct hibmc_bo *bo, u32 pl_flag, u64 *gpu_addr)
 	hibmc_ttm_placement(bo, pl_flag);
 	for (i = 0; i < bo->placement.num_placement; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret)
 		return ret;
 
@@ -368,6 +370,7 @@ int hibmc_bo_pin(struct hibmc_bo *bo, u32 pl_flag, u64 *gpu_addr)
 
 int hibmc_bo_unpin(struct hibmc_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 
 	if (!bo->pin_count) {
@@ -380,7 +383,7 @@ int hibmc_bo_unpin(struct hibmc_bo *bo)
 
 	for (i = 0; i < bo->placement.num_placement ; i++)
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret) {
 		DRM_ERROR("validate failed for unpin: %d\n", ret);
 		return ret;
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index 127815253a84..cd3f0873bbdd 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -1100,7 +1100,6 @@ static int tda998x_connector_get_modes(struct drm_connector *connector)
 
 	drm_mode_connector_update_edid_property(connector, edid);
 	n = drm_add_edid_modes(connector, edid);
-	drm_edid_to_eld(connector, edid);
 
 	kfree(edid);
 
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index aed7d207ea84..108d21f34777 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -18,6 +18,7 @@ config DRM_I915_WERROR
 config DRM_I915_DEBUG
         bool "Enable additional driver debugging"
         depends on DRM_I915
+        select DEBUG_FS
         select PREEMPT_COUNT
         select I2C_CHARDEV
         select DRM_DP_AUX_CHARDEV
@@ -49,6 +50,20 @@ config DRM_I915_DEBUG_GEM
 
           If in doubt, say "N".
 
+config DRM_I915_TRACE_GEM
+	bool "Insert extra ftrace output from the GEM internals"
+	depends on DRM_I915_DEBUG_GEM
+	select TRACING
+	default n
+	help
+	  Enable additional and verbose debugging output that will spam
+	  ordinary tests, but may be vital for post-mortem debugging when
+	  used with /proc/sys/kernel/ftrace_dump_on_oops
+
+	  Recommended for driver developers only.
+
+	  If in doubt, say "N".
+
 config DRM_I915_SW_FENCE_DEBUG_OBJECTS
         bool "Enable additional driver debugging for fence objects"
         depends on DRM_I915
@@ -90,6 +105,20 @@ config DRM_I915_SELFTEST
 
 	  If in doubt, say "N".
 
+config DRM_I915_SELFTEST_BROKEN
+	bool "Enable broken and dangerous selftests"
+	depends on DRM_I915_SELFTEST
+	depends on BROKEN
+	default n
+	help
+	  This option enables the execution of selftests that are "dangerous"
+	  and may trigger unintended HW side-effects as they break strict
+	  rules given in the HW specification. For science.
+
+	  Recommended for masochistic driver developers only.
+
+	  If in doubt, say "N".
+
 config DRM_I915_LOW_LEVEL_TRACEPOINTS
         bool "Enable low level request tracing events"
         depends on DRM_I915
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 2acf3b3c5f9d..091aef281963 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -3,7 +3,26 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-subdir-ccflags-$(CONFIG_DRM_I915_WERROR) := -Werror
+# Add a set of useful warning flags and enable -Werror for CI to prevent
+# trivial mistakes from creeping in. We have to do this piecemeal as we reject
+# any patch that isn't warning clean, so turning on -Wall -Wextra (or W=1) we
+# need to filter out dubious warnings.  Still it is our interest
+# to keep running locally with W=1 C=1 until we are completely clean.
+#
+# Note the danger in using -Wall -Wextra is that when CI updates gcc we
+# will most likely get a sudden build breakage... Hopefully we will fix
+# new warnings before CI updates!
+subdir-ccflags-y := -Wall -Wextra
+subdir-ccflags-y += $(call cc-disable-warning, unused-parameter)
+subdir-ccflags-y += $(call cc-disable-warning, type-limits)
+subdir-ccflags-y += $(call cc-disable-warning, missing-field-initializers)
+subdir-ccflags-y += $(call cc-disable-warning, implicit-fallthrough)
+subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror
+
+# Fine grained warnings disable
+CFLAGS_i915_pci.o = $(call cc-disable-warning, override-init)
+CFLAGS_intel_fbdev.o = $(call cc-disable-warning, override-init)
+
 subdir-ccflags-y += \
 	$(call as-instr,movntdqa (%eax)$(comma)%xmm0,-DCONFIG_AS_MOVNTDQA)
 
@@ -27,6 +46,7 @@ i915-y := i915_drv.o \
 
 i915-$(CONFIG_COMPAT)   += i915_ioc32.o
 i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o intel_pipe_crc.o
+i915-$(CONFIG_PERF_EVENTS) += i915_pmu.o
 
 # GEM code
 i915-y += i915_cmd_parser.o \
@@ -64,10 +84,10 @@ i915-y += intel_uc.o \
 	  intel_uc_fw.o \
 	  intel_guc.o \
 	  intel_guc_ct.o \
-	  intel_guc_log.o \
 	  intel_guc_fw.o \
-	  intel_huc.o \
-	  i915_guc_submission.o
+	  intel_guc_log.o \
+	  intel_guc_submission.o \
+	  intel_huc.o
 
 # autogenerated null render state
 i915-y += intel_renderstate_gen6.o \
@@ -144,7 +164,9 @@ i915-y += i915_perf.o \
 	  i915_oa_kblgt2.o \
 	  i915_oa_kblgt3.o \
 	  i915_oa_glk.o \
-	  i915_oa_cflgt2.o
+	  i915_oa_cflgt2.o \
+	  i915_oa_cflgt3.o \
+	  i915_oa_cnl.o
 
 ifeq ($(CONFIG_DRM_I915_GVT),y)
 i915-y += intel_gvt.o
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index 2641ba510a61..347116faa558 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -2,7 +2,8 @@
 GVT_DIR := gvt
 GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
 	interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \
-	execlist.o scheduler.o sched_policy.o render.o cmd_parser.o
+	execlist.o scheduler.o sched_policy.o mmio_context.o cmd_parser.o debugfs.o \
+	fb_decoder.o dmabuf.o
 
 ccflags-y				+= -I$(src) -I$(src)/$(GVT_DIR)
 i915-y					+= $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
index 4ce2e6bd0680..c62346fdc05d 100644
--- a/drivers/gpu/drm/i915/gvt/cfg_space.c
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -119,16 +119,6 @@ static int map_aperture(struct intel_vgpu *vgpu, bool map)
 	if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked)
 		return 0;
 
-	if (map) {
-		vgpu->gm.aperture_va = memremap(aperture_pa, aperture_sz,
-						MEMREMAP_WC);
-		if (!vgpu->gm.aperture_va)
-			return -ENOMEM;
-	} else {
-		memunmap(vgpu->gm.aperture_va);
-		vgpu->gm.aperture_va = NULL;
-	}
-
 	val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2];
 	if (val & PCI_BASE_ADDRESS_MEM_TYPE_64)
 		val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2);
@@ -141,11 +131,8 @@ static int map_aperture(struct intel_vgpu *vgpu, bool map)
 						  aperture_pa >> PAGE_SHIFT,
 						  aperture_sz >> PAGE_SHIFT,
 						  map);
-	if (ret) {
-		memunmap(vgpu->gm.aperture_va);
-		vgpu->gm.aperture_va = NULL;
+	if (ret)
 		return ret;
-	}
 
 	vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map;
 	return 0;
@@ -335,7 +322,8 @@ int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
 	case INTEL_GVT_PCI_OPREGION:
 		if (WARN_ON(!IS_ALIGNED(offset, 4)))
 			return -EINVAL;
-		ret = intel_vgpu_init_opregion(vgpu, *(u32 *)p_data);
+		ret = intel_vgpu_opregion_base_write_handler(vgpu,
+						   *(u32 *)p_data);
 		if (ret)
 			return ret;
 
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 49af94627c8a..db6b94dda5df 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -471,6 +471,7 @@ struct parser_exec_state {
 	 * used when ret from 2nd level batch buffer
 	 */
 	int saved_buf_addr_type;
+	bool is_ctx_wa;
 
 	struct cmd_info *info;
 
@@ -709,18 +710,13 @@ static void parser_exec_state_dump(struct parser_exec_state *s)
 
 	print_opcode(cmd_val(s, 0), s->ring_id);
 
-	/* print the whole page to trace */
-	pr_err("    ip_va=%p: %08x %08x %08x %08x\n",
-			s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
-			cmd_val(s, 2), cmd_val(s, 3));
-
 	s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12);
 
 	while (cnt < 1024) {
-		pr_err("ip_va=%p: ", s->ip_va);
+		gvt_dbg_cmd("ip_va=%p: ", s->ip_va);
 		for (i = 0; i < 8; i++)
-			pr_err("%08x ", cmd_val(s, i));
-		pr_err("\n");
+			gvt_dbg_cmd("%08x ", cmd_val(s, i));
+		gvt_dbg_cmd("\n");
 
 		s->ip_va += 8 * sizeof(u32);
 		cnt += 8;
@@ -825,11 +821,26 @@ static int force_nonpriv_reg_handler(struct parser_exec_state *s,
 	if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data)) {
 		gvt_err("Unexpected forcenonpriv 0x%x LRI write, value=0x%x\n",
 			offset, data);
-		return -EINVAL;
+		return -EPERM;
 	}
 	return 0;
 }
 
+static inline bool is_mocs_mmio(unsigned int offset)
+{
+	return ((offset >= 0xc800) && (offset <= 0xcff8)) ||
+		((offset >= 0xb020) && (offset <= 0xb0a0));
+}
+
+static int mocs_cmd_reg_handler(struct parser_exec_state *s,
+				unsigned int offset, unsigned int index)
+{
+	if (!is_mocs_mmio(offset))
+		return -EINVAL;
+	vgpu_vreg(s->vgpu, offset) = cmd_val(s, index + 1);
+	return 0;
+}
+
 static int cmd_reg_handler(struct parser_exec_state *s,
 	unsigned int offset, unsigned int index, char *cmd)
 {
@@ -839,7 +850,7 @@ static int cmd_reg_handler(struct parser_exec_state *s,
 	if (offset + 4 > gvt->device_info.mmio_size) {
 		gvt_vgpu_err("%s access to (%x) outside of MMIO range\n",
 				cmd, offset);
-		return -EINVAL;
+		return -EFAULT;
 	}
 
 	if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) {
@@ -853,10 +864,14 @@ static int cmd_reg_handler(struct parser_exec_state *s,
 		return 0;
 	}
 
-	if (is_force_nonpriv_mmio(offset) &&
-	    force_nonpriv_reg_handler(s, offset, index))
+	if (is_mocs_mmio(offset) &&
+	    mocs_cmd_reg_handler(s, offset, index))
 		return -EINVAL;
 
+	if (is_force_nonpriv_mmio(offset) &&
+		force_nonpriv_reg_handler(s, offset, index))
+		return -EPERM;
+
 	if (offset == i915_mmio_reg_offset(DERRMR) ||
 		offset == i915_mmio_reg_offset(FORCEWAKE_MT)) {
 		/* Writing to HW VGT_PVINFO_PAGE offset will be discarded */
@@ -894,11 +909,14 @@ static int cmd_handler_lri(struct parser_exec_state *s)
 					i915_mmio_reg_offset(DERRMR))
 				ret |= 0;
 			else
-				ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+				ret |= (cmd_reg_inhibit(s, i)) ?
+					-EBADRQC : 0;
 		}
 		if (ret)
 			break;
 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri");
+		if (ret)
+			break;
 	}
 	return ret;
 }
@@ -912,11 +930,15 @@ static int cmd_handler_lrr(struct parser_exec_state *s)
 		if (IS_BROADWELL(s->vgpu->gvt->dev_priv))
 			ret |= ((cmd_reg_inhibit(s, i) ||
 					(cmd_reg_inhibit(s, i + 1)))) ?
-				-EINVAL : 0;
+				-EBADRQC : 0;
 		if (ret)
 			break;
 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src");
+		if (ret)
+			break;
 		ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst");
+		if (ret)
+			break;
 	}
 	return ret;
 }
@@ -934,15 +956,19 @@ static int cmd_handler_lrm(struct parser_exec_state *s)
 
 	for (i = 1; i < cmd_len;) {
 		if (IS_BROADWELL(gvt->dev_priv))
-			ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+			ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0;
 		if (ret)
 			break;
 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm");
+		if (ret)
+			break;
 		if (cmd_val(s, 0) & (1 << 22)) {
 			gma = cmd_gma(s, i + 1);
 			if (gmadr_bytes == 8)
 				gma |= (cmd_gma_hi(s, i + 2)) << 32;
 			ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+			if (ret)
+				break;
 		}
 		i += gmadr_dw_number(s) + 1;
 	}
@@ -958,11 +984,15 @@ static int cmd_handler_srm(struct parser_exec_state *s)
 
 	for (i = 1; i < cmd_len;) {
 		ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm");
+		if (ret)
+			break;
 		if (cmd_val(s, 0) & (1 << 22)) {
 			gma = cmd_gma(s, i + 1);
 			if (gmadr_bytes == 8)
 				gma |= (cmd_gma_hi(s, i + 2)) << 32;
 			ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+			if (ret)
+				break;
 		}
 		i += gmadr_dw_number(s) + 1;
 	}
@@ -1116,7 +1146,7 @@ static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
 
 	v = (dword0 & GENMASK(21, 19)) >> 19;
 	if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code)))
-		return -EINVAL;
+		return -EBADRQC;
 
 	info->pipe = gen8_plane_code[v].pipe;
 	info->plane = gen8_plane_code[v].plane;
@@ -1136,7 +1166,7 @@ static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
 		info->surf_reg = SPRSURF(info->pipe);
 	} else {
 		WARN_ON(1);
-		return -EINVAL;
+		return -EBADRQC;
 	}
 	return 0;
 }
@@ -1185,7 +1215,7 @@ static int skl_decode_mi_display_flip(struct parser_exec_state *s,
 
 	default:
 		gvt_vgpu_err("unknown plane code %d\n", plane);
-		return -EINVAL;
+		return -EBADRQC;
 	}
 
 	info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
@@ -1210,13 +1240,13 @@ static int gen8_check_mi_display_flip(struct parser_exec_state *s,
 		return 0;
 
 	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
-		stride = vgpu_vreg(s->vgpu, info->stride_reg) & GENMASK(9, 0);
-		tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) &
+		stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0);
+		tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) &
 				GENMASK(12, 10)) >> 10;
 	} else {
-		stride = (vgpu_vreg(s->vgpu, info->stride_reg) &
+		stride = (vgpu_vreg_t(s->vgpu, info->stride_reg) &
 				GENMASK(15, 6)) >> 6;
-		tile = (vgpu_vreg(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10;
+		tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10;
 	}
 
 	if (stride != info->stride_val)
@@ -1235,21 +1265,21 @@ static int gen8_update_plane_mmio_from_mi_display_flip(
 	struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv;
 	struct intel_vgpu *vgpu = s->vgpu;
 
-	set_mask_bits(&vgpu_vreg(vgpu, info->surf_reg), GENMASK(31, 12),
+	set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12),
 		      info->surf_val << 12);
 	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
-		set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(9, 0),
+		set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0),
 			      info->stride_val);
-		set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(12, 10),
+		set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10),
 			      info->tile_val << 10);
 	} else {
-		set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(15, 6),
+		set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(15, 6),
 			      info->stride_val << 6);
-		set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(10, 10),
+		set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(10, 10),
 			      info->tile_val << 10);
 	}
 
-	vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++;
+	vgpu_vreg_t(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++;
 	intel_vgpu_trigger_virtual_event(vgpu, info->event);
 	return 0;
 }
@@ -1348,10 +1378,13 @@ static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index)
 {
 	unsigned long addr;
 	unsigned long gma_high, gma_low;
-	int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+	struct intel_vgpu *vgpu = s->vgpu;
+	int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd;
 
-	if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8))
+	if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) {
+		gvt_vgpu_err("invalid gma bytes %d\n", gmadr_bytes);
 		return INTEL_GVT_INVALID_ADDR;
+	}
 
 	gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK;
 	if (gmadr_bytes == 4) {
@@ -1374,16 +1407,16 @@ static inline int cmd_address_audit(struct parser_exec_state *s,
 	if (op_size > max_surface_size) {
 		gvt_vgpu_err("command address audit fail name %s\n",
 			s->info->name);
-		return -EINVAL;
+		return -EFAULT;
 	}
 
 	if (index_mode)	{
-		if (guest_gma >= GTT_PAGE_SIZE / sizeof(u64)) {
-			ret = -EINVAL;
+		if (guest_gma >= I915_GTT_PAGE_SIZE / sizeof(u64)) {
+			ret = -EFAULT;
 			goto err;
 		}
 	} else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) {
-		ret = -EINVAL;
+		ret = -EFAULT;
 		goto err;
 	}
 
@@ -1439,7 +1472,7 @@ static inline int unexpected_cmd(struct parser_exec_state *s)
 
 	gvt_vgpu_err("Unexpected %s in command buffer!\n", s->info->name);
 
-	return -EINVAL;
+	return -EBADRQC;
 }
 
 static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s)
@@ -1545,10 +1578,10 @@ static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm,
 			return -EFAULT;
 		}
 
-		offset = gma & (GTT_PAGE_SIZE - 1);
+		offset = gma & (I915_GTT_PAGE_SIZE - 1);
 
-		copy_len = (end_gma - gma) >= (GTT_PAGE_SIZE - offset) ?
-			GTT_PAGE_SIZE - offset : end_gma - gma;
+		copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ?
+			I915_GTT_PAGE_SIZE - offset : end_gma - gma;
 
 		intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len);
 
@@ -1576,110 +1609,118 @@ static int batch_buffer_needs_scan(struct parser_exec_state *s)
 	return 1;
 }
 
-static int find_bb_size(struct parser_exec_state *s)
+static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size)
 {
 	unsigned long gma = 0;
 	struct cmd_info *info;
-	int bb_size = 0;
 	uint32_t cmd_len = 0;
-	bool met_bb_end = false;
+	bool bb_end = false;
 	struct intel_vgpu *vgpu = s->vgpu;
 	u32 cmd;
 
+	*bb_size = 0;
+
 	/* get the start gm address of the batch buffer */
 	gma = get_gma_bb_from_cmd(s, 1);
-	cmd = cmd_val(s, 0);
+	if (gma == INTEL_GVT_INVALID_ADDR)
+		return -EFAULT;
 
+	cmd = cmd_val(s, 0);
 	info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
 	if (info == NULL) {
 		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
 				cmd, get_opcode(cmd, s->ring_id));
-		return -EINVAL;
+		return -EBADRQC;
 	}
 	do {
-		copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
-				gma, gma + 4, &cmd);
+		if (copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+				gma, gma + 4, &cmd) < 0)
+			return -EFAULT;
 		info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
 		if (info == NULL) {
 			gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
 				cmd, get_opcode(cmd, s->ring_id));
-			return -EINVAL;
+			return -EBADRQC;
 		}
 
 		if (info->opcode == OP_MI_BATCH_BUFFER_END) {
-			met_bb_end = true;
+			bb_end = true;
 		} else if (info->opcode == OP_MI_BATCH_BUFFER_START) {
-			if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) {
+			if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0)
 				/* chained batch buffer */
-				met_bb_end = true;
-			}
+				bb_end = true;
 		}
 		cmd_len = get_cmd_length(info, cmd) << 2;
-		bb_size += cmd_len;
+		*bb_size += cmd_len;
 		gma += cmd_len;
+	} while (!bb_end);
 
-	} while (!met_bb_end);
-
-	return bb_size;
+	return 0;
 }
 
 static int perform_bb_shadow(struct parser_exec_state *s)
 {
-	struct intel_shadow_bb_entry *entry_obj;
 	struct intel_vgpu *vgpu = s->vgpu;
+	struct intel_vgpu_shadow_bb *bb;
 	unsigned long gma = 0;
-	int bb_size;
-	void *dst = NULL;
+	unsigned long bb_size;
 	int ret = 0;
 
 	/* get the start gm address of the batch buffer */
 	gma = get_gma_bb_from_cmd(s, 1);
+	if (gma == INTEL_GVT_INVALID_ADDR)
+		return -EFAULT;
 
-	/* get the size of the batch buffer */
-	bb_size = find_bb_size(s);
-	if (bb_size < 0)
-		return -EINVAL;
+	ret = find_bb_size(s, &bb_size);
+	if (ret)
+		return ret;
 
-	/* allocate shadow batch buffer */
-	entry_obj = kmalloc(sizeof(*entry_obj), GFP_KERNEL);
-	if (entry_obj == NULL)
+	bb = kzalloc(sizeof(*bb), GFP_KERNEL);
+	if (!bb)
 		return -ENOMEM;
 
-	entry_obj->obj =
-		i915_gem_object_create(s->vgpu->gvt->dev_priv,
-				       roundup(bb_size, PAGE_SIZE));
-	if (IS_ERR(entry_obj->obj)) {
-		ret = PTR_ERR(entry_obj->obj);
-		goto free_entry;
+	bb->obj = i915_gem_object_create(s->vgpu->gvt->dev_priv,
+					 roundup(bb_size, PAGE_SIZE));
+	if (IS_ERR(bb->obj)) {
+		ret = PTR_ERR(bb->obj);
+		goto err_free_bb;
 	}
-	entry_obj->len = bb_size;
-	INIT_LIST_HEAD(&entry_obj->list);
 
-	dst = i915_gem_object_pin_map(entry_obj->obj, I915_MAP_WB);
-	if (IS_ERR(dst)) {
-		ret = PTR_ERR(dst);
-		goto put_obj;
-	}
+	ret = i915_gem_obj_prepare_shmem_write(bb->obj, &bb->clflush);
+	if (ret)
+		goto err_free_obj;
 
-	ret = i915_gem_object_set_to_cpu_domain(entry_obj->obj, false);
-	if (ret) {
-		gvt_vgpu_err("failed to set shadow batch to CPU\n");
-		goto unmap_src;
+	bb->va = i915_gem_object_pin_map(bb->obj, I915_MAP_WB);
+	if (IS_ERR(bb->va)) {
+		ret = PTR_ERR(bb->va);
+		goto err_finish_shmem_access;
 	}
 
-	entry_obj->va = dst;
-	entry_obj->bb_start_cmd_va = s->ip_va;
+	if (bb->clflush & CLFLUSH_BEFORE) {
+		drm_clflush_virt_range(bb->va, bb->obj->base.size);
+		bb->clflush &= ~CLFLUSH_BEFORE;
+	}
 
-	/* copy batch buffer to shadow batch buffer*/
 	ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
 			      gma, gma + bb_size,
-			      dst);
+			      bb->va);
 	if (ret < 0) {
 		gvt_vgpu_err("fail to copy guest ring buffer\n");
-		goto unmap_src;
+		ret = -EFAULT;
+		goto err_unmap;
 	}
 
-	list_add(&entry_obj->list, &s->workload->shadow_bb);
+	INIT_LIST_HEAD(&bb->list);
+	list_add(&bb->list, &s->workload->shadow_bb);
+
+	bb->accessing = true;
+	bb->bb_start_cmd_va = s->ip_va;
+
+	if ((s->buf_type == BATCH_BUFFER_INSTRUCTION) && (!s->is_ctx_wa))
+		bb->bb_offset = s->ip_va - s->rb_va;
+	else
+		bb->bb_offset = 0;
+
 	/*
 	 * ip_va saves the virtual address of the shadow batch buffer, while
 	 * ip_gma saves the graphics address of the original batch buffer.
@@ -1688,17 +1729,17 @@ static int perform_bb_shadow(struct parser_exec_state *s)
 	 * buffer's gma in pair. After all, we don't want to pin the shadow
 	 * buffer here (too early).
 	 */
-	s->ip_va = dst;
+	s->ip_va = bb->va;
 	s->ip_gma = gma;
-
 	return 0;
-
-unmap_src:
-	i915_gem_object_unpin_map(entry_obj->obj);
-put_obj:
-	i915_gem_object_put(entry_obj->obj);
-free_entry:
-	kfree(entry_obj);
+err_unmap:
+	i915_gem_object_unpin_map(bb->obj);
+err_finish_shmem_access:
+	i915_gem_obj_finish_shmem_access(bb->obj);
+err_free_obj:
+	i915_gem_object_put(bb->obj);
+err_free_bb:
+	kfree(bb);
 	return ret;
 }
 
@@ -1710,13 +1751,13 @@ static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
 
 	if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
 		gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n");
-		return -EINVAL;
+		return -EFAULT;
 	}
 
 	second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1;
 	if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) {
 		gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n");
-		return -EINVAL;
+		return -EFAULT;
 	}
 
 	s->saved_buf_addr_type = s->buf_addr_type;
@@ -1740,7 +1781,6 @@ static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
 		if (ret < 0)
 			return ret;
 	}
-
 	return ret;
 }
 
@@ -2430,7 +2470,7 @@ static int cmd_parser_exec(struct parser_exec_state *s)
 	if (info == NULL) {
 		gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
 				cmd, get_opcode(cmd, s->ring_id));
-		return -EINVAL;
+		return -EBADRQC;
 	}
 
 	s->info = info;
@@ -2465,6 +2505,10 @@ static inline bool gma_out_of_range(unsigned long gma,
 		return (gma > gma_tail) && (gma < gma_head);
 }
 
+/* Keep the consistent return type, e.g EBADRQC for unknown
+ * cmd, EFAULT for invalid address, EPERM for nonpriv. later
+ * works as the input of VM healthy status.
+ */
 static int command_scan(struct parser_exec_state *s,
 		unsigned long rb_head, unsigned long rb_tail,
 		unsigned long rb_start, unsigned long rb_len)
@@ -2487,7 +2531,7 @@ static int command_scan(struct parser_exec_state *s,
 					s->ip_gma, rb_start,
 					gma_bottom);
 				parser_exec_state_dump(s);
-				return -EINVAL;
+				return -EFAULT;
 			}
 			if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) {
 				gvt_vgpu_err("ip_gma %lx out of range."
@@ -2516,7 +2560,7 @@ static int scan_workload(struct intel_vgpu_workload *workload)
 	int ret = 0;
 
 	/* ring base is page aligned */
-	if (WARN_ON(!IS_ALIGNED(workload->rb_start, GTT_PAGE_SIZE)))
+	if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE)))
 		return -EINVAL;
 
 	gma_head = workload->rb_start + workload->rb_head;
@@ -2533,6 +2577,7 @@ static int scan_workload(struct intel_vgpu_workload *workload)
 	s.ring_tail = gma_tail;
 	s.rb_va = workload->shadow_ring_buffer_va;
 	s.workload = workload;
+	s.is_ctx_wa = false;
 
 	if ((bypass_scan_mask & (1 << workload->ring_id)) ||
 		gma_head == gma_tail)
@@ -2565,7 +2610,8 @@ static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 				wa_ctx);
 
 	/* ring base is page aligned */
-	if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, GTT_PAGE_SIZE)))
+	if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma,
+					I915_GTT_PAGE_SIZE)))
 		return -EINVAL;
 
 	ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(uint32_t);
@@ -2585,6 +2631,7 @@ static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 	s.ring_tail = gma_tail;
 	s.rb_va = wa_ctx->indirect_ctx.shadow_va;
 	s.workload = workload;
+	s.is_ctx_wa = true;
 
 	if (!intel_gvt_ggtt_validate_range(s.vgpu, s.ring_start, s.ring_size)) {
 		ret = -EINVAL;
@@ -2604,6 +2651,7 @@ static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
 {
 	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_vgpu_submission *s = &vgpu->submission;
 	unsigned long gma_head, gma_tail, gma_top, guest_rb_size;
 	void *shadow_ring_buffer_va;
 	int ring_id = workload->ring_id;
@@ -2619,19 +2667,21 @@ static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
 	gma_tail = workload->rb_start + workload->rb_tail;
 	gma_top = workload->rb_start + guest_rb_size;
 
-	if (workload->rb_len > vgpu->reserve_ring_buffer_size[ring_id]) {
-		void *va = vgpu->reserve_ring_buffer_va[ring_id];
+	if (workload->rb_len > s->ring_scan_buffer_size[ring_id]) {
+		void *p;
+
 		/* realloc the new ring buffer if needed */
-		vgpu->reserve_ring_buffer_va[ring_id] =
-			krealloc(va, workload->rb_len, GFP_KERNEL);
-		if (!vgpu->reserve_ring_buffer_va[ring_id]) {
-			gvt_vgpu_err("fail to alloc reserve ring buffer\n");
+		p = krealloc(s->ring_scan_buffer[ring_id], workload->rb_len,
+				GFP_KERNEL);
+		if (!p) {
+			gvt_vgpu_err("fail to re-alloc ring scan buffer\n");
 			return -ENOMEM;
 		}
-		vgpu->reserve_ring_buffer_size[ring_id] = workload->rb_len;
+		s->ring_scan_buffer[ring_id] = p;
+		s->ring_scan_buffer_size[ring_id] = workload->rb_len;
 	}
 
-	shadow_ring_buffer_va = vgpu->reserve_ring_buffer_va[ring_id];
+	shadow_ring_buffer_va = s->ring_scan_buffer[ring_id];
 
 	/* get shadow ring buffer va */
 	workload->shadow_ring_buffer_va = shadow_ring_buffer_va;
diff --git a/drivers/gpu/drm/i915/gvt/debug.h b/drivers/gpu/drm/i915/gvt/debug.h
index b0cff4dc2684..c6027125c1ec 100644
--- a/drivers/gpu/drm/i915/gvt/debug.h
+++ b/drivers/gpu/drm/i915/gvt/debug.h
@@ -25,41 +25,41 @@
 #define __GVT_DEBUG_H__
 
 #define gvt_err(fmt, args...) \
-	DRM_ERROR("gvt: "fmt, ##args)
+	pr_err("gvt: "fmt, ##args)
 
 #define gvt_vgpu_err(fmt, args...)					\
 do {									\
 	if (IS_ERR_OR_NULL(vgpu))					\
-		DRM_DEBUG_DRIVER("gvt: "fmt, ##args);			\
+		pr_err("gvt: "fmt, ##args);			\
 	else								\
-		DRM_DEBUG_DRIVER("gvt: vgpu %d: "fmt, vgpu->id, ##args);\
+		pr_err("gvt: vgpu %d: "fmt, vgpu->id, ##args);\
 } while (0)
 
 #define gvt_dbg_core(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: core: "fmt, ##args)
+	pr_debug("gvt: core: "fmt, ##args)
 
 #define gvt_dbg_irq(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: irq: "fmt, ##args)
+	pr_debug("gvt: irq: "fmt, ##args)
 
 #define gvt_dbg_mm(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: mm: "fmt, ##args)
+	pr_debug("gvt: mm: "fmt, ##args)
 
 #define gvt_dbg_mmio(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: mmio: "fmt, ##args)
+	pr_debug("gvt: mmio: "fmt, ##args)
 
 #define gvt_dbg_dpy(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: dpy: "fmt, ##args)
+	pr_debug("gvt: dpy: "fmt, ##args)
 
 #define gvt_dbg_el(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: el: "fmt, ##args)
+	pr_debug("gvt: el: "fmt, ##args)
 
 #define gvt_dbg_sched(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: sched: "fmt, ##args)
+	pr_debug("gvt: sched: "fmt, ##args)
 
 #define gvt_dbg_render(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: render: "fmt, ##args)
+	pr_debug("gvt: render: "fmt, ##args)
 
 #define gvt_dbg_cmd(fmt, args...) \
-	DRM_DEBUG_DRIVER("gvt: cmd: "fmt, ##args)
+	pr_debug("gvt: cmd: "fmt, ##args)
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/debugfs.c b/drivers/gpu/drm/i915/gvt/debugfs.c
new file mode 100644
index 000000000000..32a66dfdf112
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/debugfs.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright(c) 2011-2017 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/debugfs.h>
+#include <linux/list_sort.h>
+#include "i915_drv.h"
+#include "gvt.h"
+
+struct mmio_diff_param {
+	struct intel_vgpu *vgpu;
+	int total;
+	int diff;
+	struct list_head diff_mmio_list;
+};
+
+struct diff_mmio {
+	struct list_head node;
+	u32 offset;
+	u32 preg;
+	u32 vreg;
+};
+
+/* Compare two diff_mmio items. */
+static int mmio_offset_compare(void *priv,
+	struct list_head *a, struct list_head *b)
+{
+	struct diff_mmio *ma;
+	struct diff_mmio *mb;
+
+	ma = container_of(a, struct diff_mmio, node);
+	mb = container_of(b, struct diff_mmio, node);
+	if (ma->offset < mb->offset)
+		return -1;
+	else if (ma->offset > mb->offset)
+		return 1;
+	return 0;
+}
+
+static inline int mmio_diff_handler(struct intel_gvt *gvt,
+				    u32 offset, void *data)
+{
+	struct drm_i915_private *dev_priv = gvt->dev_priv;
+	struct mmio_diff_param *param = data;
+	struct diff_mmio *node;
+	u32 preg, vreg;
+
+	preg = I915_READ_NOTRACE(_MMIO(offset));
+	vreg = vgpu_vreg(param->vgpu, offset);
+
+	if (preg != vreg) {
+		node = kmalloc(sizeof(*node), GFP_KERNEL);
+		if (!node)
+			return -ENOMEM;
+
+		node->offset = offset;
+		node->preg = preg;
+		node->vreg = vreg;
+		list_add(&node->node, &param->diff_mmio_list);
+		param->diff++;
+	}
+	param->total++;
+	return 0;
+}
+
+/* Show the all the different values of tracked mmio. */
+static int vgpu_mmio_diff_show(struct seq_file *s, void *unused)
+{
+	struct intel_vgpu *vgpu = s->private;
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct mmio_diff_param param = {
+		.vgpu = vgpu,
+		.total = 0,
+		.diff = 0,
+	};
+	struct diff_mmio *node, *next;
+
+	INIT_LIST_HEAD(&param.diff_mmio_list);
+
+	mutex_lock(&gvt->lock);
+	spin_lock_bh(&gvt->scheduler.mmio_context_lock);
+
+	mmio_hw_access_pre(gvt->dev_priv);
+	/* Recognize all the diff mmios to list. */
+	intel_gvt_for_each_tracked_mmio(gvt, mmio_diff_handler, &param);
+	mmio_hw_access_post(gvt->dev_priv);
+
+	spin_unlock_bh(&gvt->scheduler.mmio_context_lock);
+	mutex_unlock(&gvt->lock);
+
+	/* In an ascending order by mmio offset. */
+	list_sort(NULL, &param.diff_mmio_list, mmio_offset_compare);
+
+	seq_printf(s, "%-8s %-8s %-8s %-8s\n", "Offset", "HW", "vGPU", "Diff");
+	list_for_each_entry_safe(node, next, &param.diff_mmio_list, node) {
+		u32 diff = node->preg ^ node->vreg;
+
+		seq_printf(s, "%08x %08x %08x %*pbl\n",
+			   node->offset, node->preg, node->vreg,
+			   32, &diff);
+		list_del(&node->node);
+		kfree(node);
+	}
+	seq_printf(s, "Total: %d, Diff: %d\n", param.total, param.diff);
+	return 0;
+}
+
+static int vgpu_mmio_diff_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, vgpu_mmio_diff_show, inode->i_private);
+}
+
+static const struct file_operations vgpu_mmio_diff_fops = {
+	.open		= vgpu_mmio_diff_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+/**
+ * intel_gvt_debugfs_add_vgpu - register debugfs entries for a vGPU
+ * @vgpu: a vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu)
+{
+	struct dentry *ent;
+	char name[10] = "";
+
+	sprintf(name, "vgpu%d", vgpu->id);
+	vgpu->debugfs = debugfs_create_dir(name, vgpu->gvt->debugfs_root);
+	if (!vgpu->debugfs)
+		return -ENOMEM;
+
+	ent = debugfs_create_bool("active", 0444, vgpu->debugfs,
+				  &vgpu->active);
+	if (!ent)
+		return -ENOMEM;
+
+	ent = debugfs_create_file("mmio_diff", 0444, vgpu->debugfs,
+				  vgpu, &vgpu_mmio_diff_fops);
+	if (!ent)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * intel_gvt_debugfs_remove_vgpu - remove debugfs entries of a vGPU
+ * @vgpu: a vGPU
+ */
+void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu)
+{
+	debugfs_remove_recursive(vgpu->debugfs);
+	vgpu->debugfs = NULL;
+}
+
+/**
+ * intel_gvt_debugfs_init - register gvt debugfs root entry
+ * @gvt: GVT device
+ *
+ * Returns:
+ * zero on success, negative if failed.
+ */
+int intel_gvt_debugfs_init(struct intel_gvt *gvt)
+{
+	struct drm_minor *minor = gvt->dev_priv->drm.primary;
+	struct dentry *ent;
+
+	gvt->debugfs_root = debugfs_create_dir("gvt", minor->debugfs_root);
+	if (!gvt->debugfs_root) {
+		gvt_err("Cannot create debugfs dir\n");
+		return -ENOMEM;
+	}
+
+	ent = debugfs_create_ulong("num_tracked_mmio", 0444, gvt->debugfs_root,
+				   &gvt->mmio.num_tracked_mmio);
+	if (!ent)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/**
+ * intel_gvt_debugfs_clean - remove debugfs entries
+ * @gvt: GVT device
+ */
+void intel_gvt_debugfs_clean(struct intel_gvt *gvt)
+{
+	debugfs_remove_recursive(gvt->debugfs_root);
+	gvt->debugfs_root = NULL;
+}
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
index 309f3fa6794a..dd96ffc878ac 100644
--- a/drivers/gpu/drm/i915/gvt/display.c
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -59,7 +59,7 @@ static int edp_pipe_is_enabled(struct intel_vgpu *vgpu)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 
-	if (!(vgpu_vreg(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE))
+	if (!(vgpu_vreg_t(vgpu, PIPECONF(_PIPE_EDP)) & PIPECONF_ENABLE))
 		return 0;
 
 	if (!(vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP) & TRANS_DDI_FUNC_ENABLE))
@@ -67,14 +67,14 @@ static int edp_pipe_is_enabled(struct intel_vgpu *vgpu)
 	return 1;
 }
 
-static int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe)
+int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 
 	if (WARN_ON(pipe < PIPE_A || pipe >= I915_MAX_PIPES))
 		return -EINVAL;
 
-	if (vgpu_vreg(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE)
+	if (vgpu_vreg_t(vgpu, PIPECONF(pipe)) & PIPECONF_ENABLE)
 		return 1;
 
 	if (edp_pipe_is_enabled(vgpu) &&
@@ -169,105 +169,105 @@ static u8 dpcd_fix_data[DPCD_HEADER_SIZE] = {
 static void emulate_monitor_status_change(struct intel_vgpu *vgpu)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTB_HOTPLUG_CPT |
+	vgpu_vreg_t(vgpu, SDEISR) &= ~(SDE_PORTB_HOTPLUG_CPT |
 			SDE_PORTC_HOTPLUG_CPT |
 			SDE_PORTD_HOTPLUG_CPT);
 
 	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
-		vgpu_vreg(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT |
+		vgpu_vreg_t(vgpu, SDEISR) &= ~(SDE_PORTA_HOTPLUG_SPT |
 				SDE_PORTE_HOTPLUG_SPT);
-		vgpu_vreg(vgpu, SKL_FUSE_STATUS) |=
+		vgpu_vreg_t(vgpu, SKL_FUSE_STATUS) |=
 				SKL_FUSE_DOWNLOAD_STATUS |
 				SKL_FUSE_PG_DIST_STATUS(SKL_PG0) |
 				SKL_FUSE_PG_DIST_STATUS(SKL_PG1) |
 				SKL_FUSE_PG_DIST_STATUS(SKL_PG2);
-		vgpu_vreg(vgpu, LCPLL1_CTL) |=
+		vgpu_vreg_t(vgpu, LCPLL1_CTL) |=
 				LCPLL_PLL_ENABLE |
 				LCPLL_PLL_LOCK;
-		vgpu_vreg(vgpu, LCPLL2_CTL) |= LCPLL_PLL_ENABLE;
+		vgpu_vreg_t(vgpu, LCPLL2_CTL) |= LCPLL_PLL_ENABLE;
 
 	}
 
 	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_B)) {
-		vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIB_DETECTED;
-		vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &=
+		vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIB_DETECTED;
+		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &=
 			~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK |
 			TRANS_DDI_PORT_MASK);
-		vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
+		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
 			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST |
 			(PORT_B << TRANS_DDI_PORT_SHIFT) |
 			TRANS_DDI_FUNC_ENABLE);
 		if (IS_BROADWELL(dev_priv)) {
-			vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_B)) &=
+			vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_B)) &=
 				~PORT_CLK_SEL_MASK;
-			vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_B)) |=
+			vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_B)) |=
 				PORT_CLK_SEL_LCPLL_810;
 		}
-		vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_B)) |= DDI_BUF_CTL_ENABLE;
-		vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_B)) &= ~DDI_BUF_IS_IDLE;
-		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT;
+		vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) |= DDI_BUF_CTL_ENABLE;
+		vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_B)) &= ~DDI_BUF_IS_IDLE;
+		vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTB_HOTPLUG_CPT;
 	}
 
 	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_C)) {
-		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT;
-		vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &=
+		vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTC_HOTPLUG_CPT;
+		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &=
 			~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK |
 			TRANS_DDI_PORT_MASK);
-		vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
+		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
 			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST |
 			(PORT_C << TRANS_DDI_PORT_SHIFT) |
 			TRANS_DDI_FUNC_ENABLE);
 		if (IS_BROADWELL(dev_priv)) {
-			vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_C)) &=
+			vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_C)) &=
 				~PORT_CLK_SEL_MASK;
-			vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_C)) |=
+			vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_C)) |=
 				PORT_CLK_SEL_LCPLL_810;
 		}
-		vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_C)) |= DDI_BUF_CTL_ENABLE;
-		vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_C)) &= ~DDI_BUF_IS_IDLE;
-		vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIC_DETECTED;
+		vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) |= DDI_BUF_CTL_ENABLE;
+		vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_C)) &= ~DDI_BUF_IS_IDLE;
+		vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDIC_DETECTED;
 	}
 
 	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_D)) {
-		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT;
-		vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &=
+		vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTD_HOTPLUG_CPT;
+		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) &=
 			~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK |
 			TRANS_DDI_PORT_MASK);
-		vgpu_vreg(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
+		vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |=
 			(TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST |
 			(PORT_D << TRANS_DDI_PORT_SHIFT) |
 			TRANS_DDI_FUNC_ENABLE);
 		if (IS_BROADWELL(dev_priv)) {
-			vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_D)) &=
+			vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_D)) &=
 				~PORT_CLK_SEL_MASK;
-			vgpu_vreg(vgpu, PORT_CLK_SEL(PORT_D)) |=
+			vgpu_vreg_t(vgpu, PORT_CLK_SEL(PORT_D)) |=
 				PORT_CLK_SEL_LCPLL_810;
 		}
-		vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_D)) |= DDI_BUF_CTL_ENABLE;
-		vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_D)) &= ~DDI_BUF_IS_IDLE;
-		vgpu_vreg(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDID_DETECTED;
+		vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_D)) |= DDI_BUF_CTL_ENABLE;
+		vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_D)) &= ~DDI_BUF_IS_IDLE;
+		vgpu_vreg_t(vgpu, SFUSE_STRAP) |= SFUSE_STRAP_DDID_DETECTED;
 	}
 
 	if ((IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) &&
 			intel_vgpu_has_monitor_on_port(vgpu, PORT_E)) {
-		vgpu_vreg(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT;
+		vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTE_HOTPLUG_SPT;
 	}
 
 	if (intel_vgpu_has_monitor_on_port(vgpu, PORT_A)) {
 		if (IS_BROADWELL(dev_priv))
-			vgpu_vreg(vgpu, GEN8_DE_PORT_ISR) |=
+			vgpu_vreg_t(vgpu, GEN8_DE_PORT_ISR) |=
 				GEN8_PORT_DP_A_HOTPLUG;
 		else
-			vgpu_vreg(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT;
+			vgpu_vreg_t(vgpu, SDEISR) |= SDE_PORTA_HOTPLUG_SPT;
 
-		vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_A)) |= DDI_INIT_DISPLAY_DETECTED;
+		vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_A)) |= DDI_INIT_DISPLAY_DETECTED;
 	}
 
 	/* Clear host CRT status, so guest couldn't detect this host CRT. */
 	if (IS_BROADWELL(dev_priv))
-		vgpu_vreg(vgpu, PCH_ADPA) &= ~ADPA_CRT_HOTPLUG_MONITOR_MASK;
+		vgpu_vreg_t(vgpu, PCH_ADPA) &= ~ADPA_CRT_HOTPLUG_MONITOR_MASK;
 
-	vgpu_vreg(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE;
+	vgpu_vreg_t(vgpu, PIPECONF(PIPE_A)) |= PIPECONF_ENABLE;
 }
 
 static void clean_virtual_dp_monitor(struct intel_vgpu *vgpu, int port_num)
@@ -369,12 +369,12 @@ static void emulate_vblank_on_pipe(struct intel_vgpu *vgpu, int pipe)
 		if (!pipe_is_enabled(vgpu, pipe))
 			continue;
 
-		vgpu_vreg(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++;
+		vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++;
 		intel_vgpu_trigger_virtual_event(vgpu, event);
 	}
 
 	if (pipe_is_enabled(vgpu, pipe)) {
-		vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(pipe))++;
+		vgpu_vreg_t(vgpu, PIPE_FRMCOUNT_G4X(pipe))++;
 		intel_vgpu_trigger_virtual_event(vgpu, vblank_event[pipe]);
 	}
 }
diff --git a/drivers/gpu/drm/i915/gvt/display.h b/drivers/gpu/drm/i915/gvt/display.h
index d73de22102e2..b46b86892d58 100644
--- a/drivers/gpu/drm/i915/gvt/display.h
+++ b/drivers/gpu/drm/i915/gvt/display.h
@@ -179,4 +179,6 @@ int intel_vgpu_init_display(struct intel_vgpu *vgpu, u64 resolution);
 void intel_vgpu_reset_display(struct intel_vgpu *vgpu);
 void intel_vgpu_clean_display(struct intel_vgpu *vgpu);
 
+int pipe_is_enabled(struct intel_vgpu *vgpu, int pipe);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c
new file mode 100644
index 000000000000..2fb7b34ef561
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/dmabuf.c
@@ -0,0 +1,536 @@
+/*
+ * Copyright 2017 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Xiaoguang Chen
+ *    Tina Zhang <tina.zhang@intel.com>
+ */
+
+#include <linux/dma-buf.h>
+#include <drm/drmP.h>
+#include <linux/vfio.h>
+
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define GEN8_DECODE_PTE(pte) (pte & GENMASK_ULL(63, 12))
+
+static int vgpu_gem_get_pages(
+		struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	struct sg_table *st;
+	struct scatterlist *sg;
+	int i, ret;
+	gen8_pte_t __iomem *gtt_entries;
+	struct intel_vgpu_fb_info *fb_info;
+
+	fb_info = (struct intel_vgpu_fb_info *)obj->gvt_info;
+	if (WARN_ON(!fb_info))
+		return -ENODEV;
+
+	st = kmalloc(sizeof(*st), GFP_KERNEL);
+	if (unlikely(!st))
+		return -ENOMEM;
+
+	ret = sg_alloc_table(st, fb_info->size, GFP_KERNEL);
+	if (ret) {
+		kfree(st);
+		return ret;
+	}
+	gtt_entries = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm +
+		(fb_info->start >> PAGE_SHIFT);
+	for_each_sg(st->sgl, sg, fb_info->size, i) {
+		sg->offset = 0;
+		sg->length = PAGE_SIZE;
+		sg_dma_address(sg) =
+			GEN8_DECODE_PTE(readq(&gtt_entries[i]));
+		sg_dma_len(sg) = PAGE_SIZE;
+	}
+
+	__i915_gem_object_set_pages(obj, st, PAGE_SIZE);
+
+	return 0;
+}
+
+static void vgpu_gem_put_pages(struct drm_i915_gem_object *obj,
+		struct sg_table *pages)
+{
+	sg_free_table(pages);
+	kfree(pages);
+}
+
+static void dmabuf_gem_object_free(struct kref *kref)
+{
+	struct intel_vgpu_dmabuf_obj *obj =
+		container_of(kref, struct intel_vgpu_dmabuf_obj, kref);
+	struct intel_vgpu *vgpu = obj->vgpu;
+	struct list_head *pos;
+	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
+
+	if (vgpu && vgpu->active && !list_empty(&vgpu->dmabuf_obj_list_head)) {
+		list_for_each(pos, &vgpu->dmabuf_obj_list_head) {
+			dmabuf_obj = container_of(pos,
+					struct intel_vgpu_dmabuf_obj, list);
+			if (dmabuf_obj == obj) {
+				intel_gvt_hypervisor_put_vfio_device(vgpu);
+				idr_remove(&vgpu->object_idr,
+					   dmabuf_obj->dmabuf_id);
+				kfree(dmabuf_obj->info);
+				kfree(dmabuf_obj);
+				list_del(pos);
+				break;
+			}
+		}
+	} else {
+		/* Free the orphan dmabuf_objs here */
+		kfree(obj->info);
+		kfree(obj);
+	}
+}
+
+
+static inline void dmabuf_obj_get(struct intel_vgpu_dmabuf_obj *obj)
+{
+	kref_get(&obj->kref);
+}
+
+static inline void dmabuf_obj_put(struct intel_vgpu_dmabuf_obj *obj)
+{
+	kref_put(&obj->kref, dmabuf_gem_object_free);
+}
+
+static void vgpu_gem_release(struct drm_i915_gem_object *gem_obj)
+{
+
+	struct intel_vgpu_fb_info *fb_info = gem_obj->gvt_info;
+	struct intel_vgpu_dmabuf_obj *obj = fb_info->obj;
+	struct intel_vgpu *vgpu = obj->vgpu;
+
+	if (vgpu) {
+		mutex_lock(&vgpu->dmabuf_lock);
+		gem_obj->base.dma_buf = NULL;
+		dmabuf_obj_put(obj);
+		mutex_unlock(&vgpu->dmabuf_lock);
+	} else {
+		/* vgpu is NULL, as it has been removed already */
+		gem_obj->base.dma_buf = NULL;
+		dmabuf_obj_put(obj);
+	}
+}
+
+static const struct drm_i915_gem_object_ops intel_vgpu_gem_ops = {
+	.flags = I915_GEM_OBJECT_IS_PROXY,
+	.get_pages = vgpu_gem_get_pages,
+	.put_pages = vgpu_gem_put_pages,
+	.release = vgpu_gem_release,
+};
+
+static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev,
+		struct intel_vgpu_fb_info *info)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_gem_object *obj;
+
+	obj = i915_gem_object_alloc(dev_priv);
+	if (obj == NULL)
+		return NULL;
+
+	drm_gem_private_object_init(dev, &obj->base,
+		info->size << PAGE_SHIFT);
+	i915_gem_object_init(obj, &intel_vgpu_gem_ops);
+
+	obj->base.read_domains = I915_GEM_DOMAIN_GTT;
+	obj->base.write_domain = 0;
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		unsigned int tiling_mode = 0;
+		unsigned int stride = 0;
+
+		switch (info->drm_format_mod << 10) {
+		case PLANE_CTL_TILED_LINEAR:
+			tiling_mode = I915_TILING_NONE;
+			break;
+		case PLANE_CTL_TILED_X:
+			tiling_mode = I915_TILING_X;
+			stride = info->stride;
+			break;
+		case PLANE_CTL_TILED_Y:
+			tiling_mode = I915_TILING_Y;
+			stride = info->stride;
+			break;
+		default:
+			gvt_dbg_core("not supported tiling mode\n");
+		}
+		obj->tiling_and_stride = tiling_mode | stride;
+	} else {
+		obj->tiling_and_stride = info->drm_format_mod ?
+					I915_TILING_X : 0;
+	}
+
+	return obj;
+}
+
+static int vgpu_get_plane_info(struct drm_device *dev,
+		struct intel_vgpu *vgpu,
+		struct intel_vgpu_fb_info *info,
+		int plane_id)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_vgpu_primary_plane_format p;
+	struct intel_vgpu_cursor_plane_format c;
+	int ret;
+
+	if (plane_id == DRM_PLANE_TYPE_PRIMARY) {
+		ret = intel_vgpu_decode_primary_plane(vgpu, &p);
+		if (ret)
+			return ret;
+		info->start = p.base;
+		info->start_gpa = p.base_gpa;
+		info->width = p.width;
+		info->height = p.height;
+		info->stride = p.stride;
+		info->drm_format = p.drm_format;
+		info->drm_format_mod = p.tiled;
+		info->size = (((p.stride * p.height * p.bpp) / 8) +
+				(PAGE_SIZE - 1)) >> PAGE_SHIFT;
+	} else if (plane_id == DRM_PLANE_TYPE_CURSOR) {
+		ret = intel_vgpu_decode_cursor_plane(vgpu, &c);
+		if (ret)
+			return ret;
+		info->start = c.base;
+		info->start_gpa = c.base_gpa;
+		info->width = c.width;
+		info->height = c.height;
+		info->stride = c.width * (c.bpp / 8);
+		info->drm_format = c.drm_format;
+		info->drm_format_mod = 0;
+		info->x_pos = c.x_pos;
+		info->y_pos = c.y_pos;
+
+		/* The invalid cursor hotspot value is delivered to host
+		 * until we find a way to get the cursor hotspot info of
+		 * guest OS.
+		 */
+		info->x_hot = UINT_MAX;
+		info->y_hot = UINT_MAX;
+		info->size = (((info->stride * c.height * c.bpp) / 8)
+				+ (PAGE_SIZE - 1)) >> PAGE_SHIFT;
+	} else {
+		gvt_vgpu_err("invalid plane id:%d\n", plane_id);
+		return -EINVAL;
+	}
+
+	if (info->size == 0) {
+		gvt_vgpu_err("fb size is zero\n");
+		return -EINVAL;
+	}
+
+	if (info->start & (PAGE_SIZE - 1)) {
+		gvt_vgpu_err("Not aligned fb address:0x%llx\n", info->start);
+		return -EFAULT;
+	}
+	if (((info->start >> PAGE_SHIFT) + info->size) >
+		ggtt_total_entries(&dev_priv->ggtt)) {
+		gvt_vgpu_err("Invalid GTT offset or size\n");
+		return -EFAULT;
+	}
+
+	if (!intel_gvt_ggtt_validate_range(vgpu, info->start, info->size)) {
+		gvt_vgpu_err("invalid gma addr\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static struct intel_vgpu_dmabuf_obj *
+pick_dmabuf_by_info(struct intel_vgpu *vgpu,
+		    struct intel_vgpu_fb_info *latest_info)
+{
+	struct list_head *pos;
+	struct intel_vgpu_fb_info *fb_info;
+	struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL;
+	struct intel_vgpu_dmabuf_obj *ret = NULL;
+
+	list_for_each(pos, &vgpu->dmabuf_obj_list_head) {
+		dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj,
+						list);
+		if ((dmabuf_obj == NULL) ||
+		    (dmabuf_obj->info == NULL))
+			continue;
+
+		fb_info = (struct intel_vgpu_fb_info *)dmabuf_obj->info;
+		if ((fb_info->start == latest_info->start) &&
+		    (fb_info->start_gpa == latest_info->start_gpa) &&
+		    (fb_info->size == latest_info->size) &&
+		    (fb_info->drm_format_mod == latest_info->drm_format_mod) &&
+		    (fb_info->drm_format == latest_info->drm_format) &&
+		    (fb_info->width == latest_info->width) &&
+		    (fb_info->height == latest_info->height)) {
+			ret = dmabuf_obj;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static struct intel_vgpu_dmabuf_obj *
+pick_dmabuf_by_num(struct intel_vgpu *vgpu, u32 id)
+{
+	struct list_head *pos;
+	struct intel_vgpu_dmabuf_obj *dmabuf_obj = NULL;
+	struct intel_vgpu_dmabuf_obj *ret = NULL;
+
+	list_for_each(pos, &vgpu->dmabuf_obj_list_head) {
+		dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj,
+						list);
+		if (!dmabuf_obj)
+			continue;
+
+		if (dmabuf_obj->dmabuf_id == id) {
+			ret = dmabuf_obj;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static void update_fb_info(struct vfio_device_gfx_plane_info *gvt_dmabuf,
+		      struct intel_vgpu_fb_info *fb_info)
+{
+	gvt_dmabuf->drm_format = fb_info->drm_format;
+	gvt_dmabuf->width = fb_info->width;
+	gvt_dmabuf->height = fb_info->height;
+	gvt_dmabuf->stride = fb_info->stride;
+	gvt_dmabuf->size = fb_info->size;
+	gvt_dmabuf->x_pos = fb_info->x_pos;
+	gvt_dmabuf->y_pos = fb_info->y_pos;
+	gvt_dmabuf->x_hot = fb_info->x_hot;
+	gvt_dmabuf->y_hot = fb_info->y_hot;
+}
+
+int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args)
+{
+	struct drm_device *dev = &vgpu->gvt->dev_priv->drm;
+	struct vfio_device_gfx_plane_info *gfx_plane_info = args;
+	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
+	struct intel_vgpu_fb_info fb_info;
+	int ret = 0;
+
+	if (gfx_plane_info->flags == (VFIO_GFX_PLANE_TYPE_DMABUF |
+				       VFIO_GFX_PLANE_TYPE_PROBE))
+		return ret;
+	else if ((gfx_plane_info->flags & ~VFIO_GFX_PLANE_TYPE_DMABUF) ||
+			(!gfx_plane_info->flags))
+		return -EINVAL;
+
+	ret = vgpu_get_plane_info(dev, vgpu, &fb_info,
+					gfx_plane_info->drm_plane_type);
+	if (ret != 0)
+		goto out;
+
+	mutex_lock(&vgpu->dmabuf_lock);
+	/* If exists, pick up the exposed dmabuf_obj */
+	dmabuf_obj = pick_dmabuf_by_info(vgpu, &fb_info);
+	if (dmabuf_obj) {
+		update_fb_info(gfx_plane_info, &fb_info);
+		gfx_plane_info->dmabuf_id = dmabuf_obj->dmabuf_id;
+
+		/* This buffer may be released between query_plane ioctl and
+		 * get_dmabuf ioctl. Add the refcount to make sure it won't
+		 * be released between the two ioctls.
+		 */
+		if (!dmabuf_obj->initref) {
+			dmabuf_obj->initref = true;
+			dmabuf_obj_get(dmabuf_obj);
+		}
+		ret = 0;
+		gvt_dbg_dpy("vgpu%d: re-use dmabuf_obj ref %d, id %d\n",
+			    vgpu->id, kref_read(&dmabuf_obj->kref),
+			    gfx_plane_info->dmabuf_id);
+		mutex_unlock(&vgpu->dmabuf_lock);
+		goto out;
+	}
+
+	mutex_unlock(&vgpu->dmabuf_lock);
+
+	/* Need to allocate a new one*/
+	dmabuf_obj = kmalloc(sizeof(struct intel_vgpu_dmabuf_obj), GFP_KERNEL);
+	if (unlikely(!dmabuf_obj)) {
+		gvt_vgpu_err("alloc dmabuf_obj failed\n");
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	dmabuf_obj->info = kmalloc(sizeof(struct intel_vgpu_fb_info),
+				   GFP_KERNEL);
+	if (unlikely(!dmabuf_obj->info)) {
+		gvt_vgpu_err("allocate intel vgpu fb info failed\n");
+		ret = -ENOMEM;
+		goto out_free_dmabuf;
+	}
+	memcpy(dmabuf_obj->info, &fb_info, sizeof(struct intel_vgpu_fb_info));
+
+	((struct intel_vgpu_fb_info *)dmabuf_obj->info)->obj = dmabuf_obj;
+
+	dmabuf_obj->vgpu = vgpu;
+
+	ret = idr_alloc(&vgpu->object_idr, dmabuf_obj, 1, 0, GFP_NOWAIT);
+	if (ret < 0)
+		goto out_free_info;
+	gfx_plane_info->dmabuf_id = ret;
+	dmabuf_obj->dmabuf_id = ret;
+
+	dmabuf_obj->initref = true;
+
+	kref_init(&dmabuf_obj->kref);
+
+	mutex_lock(&vgpu->dmabuf_lock);
+	if (intel_gvt_hypervisor_get_vfio_device(vgpu)) {
+		gvt_vgpu_err("get vfio device failed\n");
+		mutex_unlock(&vgpu->dmabuf_lock);
+		goto out_free_info;
+	}
+	mutex_unlock(&vgpu->dmabuf_lock);
+
+	update_fb_info(gfx_plane_info, &fb_info);
+
+	INIT_LIST_HEAD(&dmabuf_obj->list);
+	mutex_lock(&vgpu->dmabuf_lock);
+	list_add_tail(&dmabuf_obj->list, &vgpu->dmabuf_obj_list_head);
+	mutex_unlock(&vgpu->dmabuf_lock);
+
+	gvt_dbg_dpy("vgpu%d: %s new dmabuf_obj ref %d, id %d\n", vgpu->id,
+		    __func__, kref_read(&dmabuf_obj->kref), ret);
+
+	return 0;
+
+out_free_info:
+	kfree(dmabuf_obj->info);
+out_free_dmabuf:
+	kfree(dmabuf_obj);
+out:
+	/* ENODEV means plane isn't ready, which might be a normal case. */
+	return (ret == -ENODEV) ? 0 : ret;
+}
+
+/* To associate an exposed dmabuf with the dmabuf_obj */
+int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id)
+{
+	struct drm_device *dev = &vgpu->gvt->dev_priv->drm;
+	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
+	struct drm_i915_gem_object *obj;
+	struct dma_buf *dmabuf;
+	int dmabuf_fd;
+	int ret = 0;
+
+	mutex_lock(&vgpu->dmabuf_lock);
+
+	dmabuf_obj = pick_dmabuf_by_num(vgpu, dmabuf_id);
+	if (dmabuf_obj == NULL) {
+		gvt_vgpu_err("invalid dmabuf id:%d\n", dmabuf_id);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	obj = vgpu_create_gem(dev, dmabuf_obj->info);
+	if (obj == NULL) {
+		gvt_vgpu_err("create gvt gem obj failed:%d\n", vgpu->id);
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	obj->gvt_info = dmabuf_obj->info;
+
+	dmabuf = i915_gem_prime_export(dev, &obj->base, DRM_CLOEXEC | DRM_RDWR);
+	if (IS_ERR(dmabuf)) {
+		gvt_vgpu_err("export dma-buf failed\n");
+		ret = PTR_ERR(dmabuf);
+		goto out_free_gem;
+	}
+
+	i915_gem_object_put(obj);
+
+	ret = dma_buf_fd(dmabuf, DRM_CLOEXEC | DRM_RDWR);
+	if (ret < 0) {
+		gvt_vgpu_err("create dma-buf fd failed ret:%d\n", ret);
+		goto out_free_dmabuf;
+	}
+	dmabuf_fd = ret;
+
+	dmabuf_obj_get(dmabuf_obj);
+
+	if (dmabuf_obj->initref) {
+		dmabuf_obj->initref = false;
+		dmabuf_obj_put(dmabuf_obj);
+	}
+
+	mutex_unlock(&vgpu->dmabuf_lock);
+
+	gvt_dbg_dpy("vgpu%d: dmabuf:%d, dmabuf ref %d, fd:%d\n"
+		    "        file count: %ld, GEM ref: %d\n",
+		    vgpu->id, dmabuf_obj->dmabuf_id,
+		    kref_read(&dmabuf_obj->kref),
+		    dmabuf_fd,
+		    file_count(dmabuf->file),
+		    kref_read(&obj->base.refcount));
+
+	return dmabuf_fd;
+
+out_free_dmabuf:
+	dma_buf_put(dmabuf);
+out_free_gem:
+	i915_gem_object_put(obj);
+out:
+	mutex_unlock(&vgpu->dmabuf_lock);
+	return ret;
+}
+
+void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu)
+{
+	struct list_head *pos, *n;
+	struct intel_vgpu_dmabuf_obj *dmabuf_obj;
+
+	mutex_lock(&vgpu->dmabuf_lock);
+	list_for_each_safe(pos, n, &vgpu->dmabuf_obj_list_head) {
+		dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj,
+						list);
+		dmabuf_obj->vgpu = NULL;
+
+		idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id);
+		intel_gvt_hypervisor_put_vfio_device(vgpu);
+		list_del(pos);
+
+		/* dmabuf_obj might be freed in dmabuf_obj_put */
+		if (dmabuf_obj->initref) {
+			dmabuf_obj->initref = false;
+			dmabuf_obj_put(dmabuf_obj);
+		}
+
+	}
+	mutex_unlock(&vgpu->dmabuf_lock);
+}
diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.h b/drivers/gpu/drm/i915/gvt/dmabuf.h
new file mode 100644
index 000000000000..5f8f03fb1d1b
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/dmabuf.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Zhiyuan Lv <zhiyuan.lv@intel.com>
+ *
+ * Contributors:
+ *    Xiaoguang Chen
+ *    Tina Zhang <tina.zhang@intel.com>
+ */
+
+#ifndef _GVT_DMABUF_H_
+#define _GVT_DMABUF_H_
+#include <linux/vfio.h>
+
+struct intel_vgpu_fb_info {
+	__u64 start;
+	__u64 start_gpa;
+	__u64 drm_format_mod;
+	__u32 drm_format;	/* drm format of plane */
+	__u32 width;	/* width of plane */
+	__u32 height;	/* height of plane */
+	__u32 stride;	/* stride of plane */
+	__u32 size;	/* size of plane in bytes, align on page */
+	__u32 x_pos;	/* horizontal position of cursor plane */
+	__u32 y_pos;	/* vertical position of cursor plane */
+	__u32 x_hot;    /* horizontal position of cursor hotspot */
+	__u32 y_hot;    /* vertical position of cursor hotspot */
+	struct intel_vgpu_dmabuf_obj *obj;
+};
+
+/**
+ * struct intel_vgpu_dmabuf_obj- Intel vGPU device buffer object
+ */
+struct intel_vgpu_dmabuf_obj {
+	struct intel_vgpu *vgpu;
+	struct intel_vgpu_fb_info *info;
+	__u32 dmabuf_id;
+	struct kref kref;
+	bool initref;
+	struct list_head list;
+};
+
+int intel_vgpu_query_plane(struct intel_vgpu *vgpu, void *args);
+int intel_vgpu_get_dmabuf(struct intel_vgpu *vgpu, unsigned int dmabuf_id);
+void intel_vgpu_dmabuf_cleanup(struct intel_vgpu *vgpu);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
index 42cd09ec63fa..f61337632969 100644
--- a/drivers/gpu/drm/i915/gvt/edid.c
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -95,9 +95,9 @@ static inline int get_port_from_gmbus0(u32 gmbus0)
 
 static void reset_gmbus_controller(struct intel_vgpu *vgpu)
 {
-	vgpu_vreg(vgpu, PCH_GMBUS2) = GMBUS_HW_RDY;
+	vgpu_vreg_t(vgpu, PCH_GMBUS2) = GMBUS_HW_RDY;
 	if (!vgpu->display.i2c_edid.edid_available)
-		vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
+		vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
 	vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE;
 }
 
@@ -123,16 +123,16 @@ static int gmbus0_mmio_write(struct intel_vgpu *vgpu,
 	vgpu->display.i2c_edid.state = I2C_GMBUS;
 	vgpu->display.i2c_edid.gmbus.phase = GMBUS_IDLE_PHASE;
 
-	vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
-	vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY | GMBUS_HW_WAIT_PHASE;
+	vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
+	vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY | GMBUS_HW_WAIT_PHASE;
 
 	if (intel_vgpu_has_monitor_on_port(vgpu, port) &&
 			!intel_vgpu_port_is_dp(vgpu, port)) {
 		vgpu->display.i2c_edid.port = port;
 		vgpu->display.i2c_edid.edid_available = true;
-		vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_SATOER;
+		vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_SATOER;
 	} else
-		vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
+		vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_SATOER;
 	return 0;
 }
 
@@ -159,8 +159,8 @@ static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 		 * 2) HW_RDY bit asserted
 		 */
 		if (wvalue & GMBUS_SW_CLR_INT) {
-			vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_INT;
-			vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY;
+			vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_INT;
+			vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_HW_RDY;
 		}
 
 		/* For virtualization, we suppose that HW is always ready,
@@ -208,7 +208,7 @@ static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 				 * visible in gmbus interface)
 				 */
 				i2c_edid->gmbus.phase = GMBUS_IDLE_PHASE;
-				vgpu_vreg(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
+				vgpu_vreg_t(vgpu, PCH_GMBUS2) &= ~GMBUS_ACTIVE;
 			}
 			break;
 		case NIDX_NS_W:
@@ -220,7 +220,7 @@ static int gmbus1_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 			 * START (-->INDEX) -->DATA
 			 */
 			i2c_edid->gmbus.phase = GMBUS_DATA_PHASE;
-			vgpu_vreg(vgpu, PCH_GMBUS2) |= GMBUS_ACTIVE;
+			vgpu_vreg_t(vgpu, PCH_GMBUS2) |= GMBUS_ACTIVE;
 			break;
 		default:
 			gvt_vgpu_err("Unknown/reserved GMBUS cycle detected!\n");
@@ -256,7 +256,7 @@ static int gmbus3_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
 	u32 reg_data = 0;
 
 	/* Data can only be recevied if previous settings correct */
-	if (vgpu_vreg(vgpu, PCH_GMBUS1) & GMBUS_SLAVE_READ) {
+	if (vgpu_vreg_t(vgpu, PCH_GMBUS1) & GMBUS_SLAVE_READ) {
 		if (byte_left <= 0) {
 			memcpy(p_data, &vgpu_vreg(vgpu, offset), bytes);
 			return 0;
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
index 940cdaaa3f24..70494e394d2c 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.c
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -46,8 +46,6 @@
 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
 		((a)->lrca == (b)->lrca))
 
-static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask);
-
 static int context_switch_events[] = {
 	[RCS] = RCS_AS_CONTEXT_SWITCH,
 	[BCS] = BCS_AS_CONTEXT_SWITCH,
@@ -135,6 +133,8 @@ static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
 	struct execlist_context_status_pointer_format ctx_status_ptr;
 	u32 write_pointer;
 	u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset;
+	unsigned long hwsp_gpa;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 
 	ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
 			_EL_OFFSET_STATUS_PTR);
@@ -160,6 +160,20 @@ static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
 	ctx_status_ptr.write_ptr = write_pointer;
 	vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
 
+	/* Update the CSB and CSB write pointer in HWSP */
+	hwsp_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+					 vgpu->hws_pga[ring_id]);
+	if (hwsp_gpa != INTEL_GVT_INVALID_ADDR) {
+		intel_gvt_hypervisor_write_gpa(vgpu,
+			hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 +
+			write_pointer * 8,
+			status, 8);
+		intel_gvt_hypervisor_write_gpa(vgpu,
+			hwsp_gpa +
+			intel_hws_csb_write_index(dev_priv) * 4,
+			&write_pointer, 4);
+	}
+
 	gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n",
 		vgpu->id, write_pointer, offset, status->ldw, status->udw);
 
@@ -358,218 +372,47 @@ static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist,
 	return 0;
 }
 
-static void free_workload(struct intel_vgpu_workload *workload)
-{
-	intel_vgpu_unpin_mm(workload->shadow_mm);
-	intel_gvt_mm_unreference(workload->shadow_mm);
-	kmem_cache_free(workload->vgpu->workloads, workload);
-}
-
 #define get_desc_from_elsp_dwords(ed, i) \
 	((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
 
-static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
-{
-	const int gmadr_bytes = workload->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
-	struct intel_shadow_bb_entry *entry_obj;
-
-	/* pin the gem object to ggtt */
-	list_for_each_entry(entry_obj, &workload->shadow_bb, list) {
-		struct i915_vma *vma;
-
-		vma = i915_gem_object_ggtt_pin(entry_obj->obj, NULL, 0, 4, 0);
-		if (IS_ERR(vma)) {
-			return PTR_ERR(vma);
-		}
-
-		/* FIXME: we are not tracking our pinned VMA leaving it
-		 * up to the core to fix up the stray pin_count upon
-		 * free.
-		 */
-
-		/* update the relocate gma with shadow batch buffer*/
-		entry_obj->bb_start_cmd_va[1] = i915_ggtt_offset(vma);
-		if (gmadr_bytes == 8)
-			entry_obj->bb_start_cmd_va[2] = 0;
-	}
-	return 0;
-}
-
-static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
-{
-	struct intel_vgpu_workload *workload = container_of(wa_ctx,
-					struct intel_vgpu_workload,
-					wa_ctx);
-	int ring_id = workload->ring_id;
-	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
-	struct drm_i915_gem_object *ctx_obj =
-		shadow_ctx->engine[ring_id].state->obj;
-	struct execlist_ring_context *shadow_ring_context;
-	struct page *page;
-
-	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
-	shadow_ring_context = kmap_atomic(page);
-
-	shadow_ring_context->bb_per_ctx_ptr.val =
-		(shadow_ring_context->bb_per_ctx_ptr.val &
-		(~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
-	shadow_ring_context->rcs_indirect_ctx.val =
-		(shadow_ring_context->rcs_indirect_ctx.val &
-		(~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
-
-	kunmap_atomic(shadow_ring_context);
-	return 0;
-}
-
-static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
-{
-	struct i915_vma *vma;
-	unsigned char *per_ctx_va =
-		(unsigned char *)wa_ctx->indirect_ctx.shadow_va +
-		wa_ctx->indirect_ctx.size;
-
-	if (wa_ctx->indirect_ctx.size == 0)
-		return 0;
-
-	vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
-				       0, CACHELINE_BYTES, 0);
-	if (IS_ERR(vma)) {
-		return PTR_ERR(vma);
-	}
-
-	/* FIXME: we are not tracking our pinned VMA leaving it
-	 * up to the core to fix up the stray pin_count upon
-	 * free.
-	 */
-
-	wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
-
-	wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
-	memset(per_ctx_va, 0, CACHELINE_BYTES);
-
-	update_wa_ctx_2_shadow_ctx(wa_ctx);
-	return 0;
-}
-
-static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
-{
-	/* release all the shadow batch buffer */
-	if (!list_empty(&workload->shadow_bb)) {
-		struct intel_shadow_bb_entry *entry_obj =
-			list_first_entry(&workload->shadow_bb,
-					 struct intel_shadow_bb_entry,
-					 list);
-		struct intel_shadow_bb_entry *temp;
-
-		list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb,
-					 list) {
-			i915_gem_object_unpin_map(entry_obj->obj);
-			i915_gem_object_put(entry_obj->obj);
-			list_del(&entry_obj->list);
-			kfree(entry_obj);
-		}
-	}
-}
-
 static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
 {
 	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_vgpu_submission *s = &vgpu->submission;
 	struct execlist_ctx_descriptor_format ctx[2];
 	int ring_id = workload->ring_id;
 	int ret;
 
-	ret = intel_vgpu_pin_mm(workload->shadow_mm);
-	if (ret) {
-		gvt_vgpu_err("fail to vgpu pin mm\n");
-		goto out;
-	}
-
-	ret = intel_vgpu_sync_oos_pages(workload->vgpu);
-	if (ret) {
-		gvt_vgpu_err("fail to vgpu sync oos pages\n");
-		goto err_unpin_mm;
-	}
-
-	ret = intel_vgpu_flush_post_shadow(workload->vgpu);
-	if (ret) {
-		gvt_vgpu_err("fail to flush post shadow\n");
-		goto err_unpin_mm;
-	}
-
-	ret = intel_gvt_generate_request(workload);
-	if (ret) {
-		gvt_vgpu_err("fail to generate request\n");
-		goto err_unpin_mm;
-	}
-
-	ret = prepare_shadow_batch_buffer(workload);
-	if (ret) {
-		gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
-		goto err_unpin_mm;
-	}
-
-	ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
-	if (ret) {
-		gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
-		goto err_shadow_batch;
-	}
-
 	if (!workload->emulate_schedule_in)
 		return 0;
 
-	ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
-	ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
+	ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
+	ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
 
-	ret = emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx);
-	if (!ret)
-		goto out;
-	else
+	ret = emulate_execlist_schedule_in(&s->execlist[ring_id], ctx);
+	if (ret) {
 		gvt_vgpu_err("fail to emulate execlist schedule in\n");
-
-	release_shadow_wa_ctx(&workload->wa_ctx);
-err_shadow_batch:
-	release_shadow_batch_buffer(workload);
-err_unpin_mm:
-	intel_vgpu_unpin_mm(workload->shadow_mm);
-out:
-	return ret;
+		return ret;
+	}
+	return 0;
 }
 
 static int complete_execlist_workload(struct intel_vgpu_workload *workload)
 {
 	struct intel_vgpu *vgpu = workload->vgpu;
 	int ring_id = workload->ring_id;
-	struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct intel_vgpu_execlist *execlist = &s->execlist[ring_id];
 	struct intel_vgpu_workload *next_workload;
 	struct list_head *next = workload_q_head(vgpu, ring_id)->next;
 	bool lite_restore = false;
-	int ret;
+	int ret = 0;
 
 	gvt_dbg_el("complete workload %p status %d\n", workload,
 			workload->status);
 
-	if (!workload->status) {
-		release_shadow_batch_buffer(workload);
-		release_shadow_wa_ctx(&workload->wa_ctx);
-	}
-
-	if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
-		/* if workload->status is not successful means HW GPU
-		 * has occurred GPU hang or something wrong with i915/GVT,
-		 * and GVT won't inject context switch interrupt to guest.
-		 * So this error is a vGPU hang actually to the guest.
-		 * According to this we should emunlate a vGPU hang. If
-		 * there are pending workloads which are already submitted
-		 * from guest, we should clean them up like HW GPU does.
-		 *
-		 * if it is in middle of engine resetting, the pending
-		 * workloads won't be submitted to HW GPU and will be
-		 * cleaned up during the resetting process later, so doing
-		 * the workload clean up here doesn't have any impact.
-		 **/
-		clean_workloads(vgpu, ENGINE_MASK(ring_id));
+	if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id)))
 		goto out;
-	}
 
 	if (!list_empty(workload_q_head(vgpu, ring_id))) {
 		struct execlist_ctx_descriptor_format *this_desc, *next_desc;
@@ -584,213 +427,60 @@ static int complete_execlist_workload(struct intel_vgpu_workload *workload)
 
 	if (lite_restore) {
 		gvt_dbg_el("next context == current - no schedule-out\n");
-		free_workload(workload);
-		return 0;
+		goto out;
 	}
 
 	ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc);
-	if (ret)
-		goto err;
 out:
-	free_workload(workload);
-	return 0;
-err:
-	free_workload(workload);
+	intel_vgpu_unpin_mm(workload->shadow_mm);
+	intel_vgpu_destroy_workload(workload);
 	return ret;
 }
 
-#define RING_CTX_OFF(x) \
-	offsetof(struct execlist_ring_context, x)
-
-static void read_guest_pdps(struct intel_vgpu *vgpu,
-		u64 ring_context_gpa, u32 pdp[8])
-{
-	u64 gpa;
-	int i;
-
-	gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val);
-
-	for (i = 0; i < 8; i++)
-		intel_gvt_hypervisor_read_gpa(vgpu,
-				gpa + i * 8, &pdp[7 - i], 4);
-}
-
-static int prepare_mm(struct intel_vgpu_workload *workload)
-{
-	struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
-	struct intel_vgpu_mm *mm;
-	struct intel_vgpu *vgpu = workload->vgpu;
-	int page_table_level;
-	u32 pdp[8];
-
-	if (desc->addressing_mode == 1) { /* legacy 32-bit */
-		page_table_level = 3;
-	} else if (desc->addressing_mode == 3) { /* legacy 64 bit */
-		page_table_level = 4;
-	} else {
-		gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
-		return -EINVAL;
-	}
-
-	read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp);
-
-	mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp);
-	if (mm) {
-		intel_gvt_mm_reference(mm);
-	} else {
-
-		mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT,
-				pdp, page_table_level, 0);
-		if (IS_ERR(mm)) {
-			gvt_vgpu_err("fail to create mm object.\n");
-			return PTR_ERR(mm);
-		}
-	}
-	workload->shadow_mm = mm;
-	return 0;
-}
-
-#define get_last_workload(q) \
-	(list_empty(q) ? NULL : container_of(q->prev, \
-	struct intel_vgpu_workload, list))
-
 static int submit_context(struct intel_vgpu *vgpu, int ring_id,
 		struct execlist_ctx_descriptor_format *desc,
 		bool emulate_schedule_in)
 {
-	struct list_head *q = workload_q_head(vgpu, ring_id);
-	struct intel_vgpu_workload *last_workload = get_last_workload(q);
+	struct intel_vgpu_submission *s = &vgpu->submission;
 	struct intel_vgpu_workload *workload = NULL;
-	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	u64 ring_context_gpa;
-	u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
-	int ret;
-
-	ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
-			(u32)((desc->lrca + 1) << GTT_PAGE_SHIFT));
-	if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
-		gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
-		return -EINVAL;
-	}
 
-	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-			RING_CTX_OFF(ring_header.val), &head, 4);
+	workload = intel_vgpu_create_workload(vgpu, ring_id, desc);
+	if (IS_ERR(workload))
+		return PTR_ERR(workload);
 
-	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-			RING_CTX_OFF(ring_tail.val), &tail, 4);
-
-	head &= RB_HEAD_OFF_MASK;
-	tail &= RB_TAIL_OFF_MASK;
-
-	if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
-		gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
-		gvt_dbg_el("ctx head %x real head %lx\n", head,
-				last_workload->rb_tail);
-		/*
-		 * cannot use guest context head pointer here,
-		 * as it might not be updated at this time
-		 */
-		head = last_workload->rb_tail;
-	}
-
-	gvt_dbg_el("ring id %d begin a new workload\n", ring_id);
-
-	workload = kmem_cache_zalloc(vgpu->workloads, GFP_KERNEL);
-	if (!workload)
-		return -ENOMEM;
-
-	/* record some ring buffer register values for scan and shadow */
-	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-			RING_CTX_OFF(rb_start.val), &start, 4);
-	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-			RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
-	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-			RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
-
-	INIT_LIST_HEAD(&workload->list);
-	INIT_LIST_HEAD(&workload->shadow_bb);
-
-	init_waitqueue_head(&workload->shadow_ctx_status_wq);
-	atomic_set(&workload->shadow_ctx_active, 0);
-
-	workload->vgpu = vgpu;
-	workload->ring_id = ring_id;
-	workload->ctx_desc = *desc;
-	workload->ring_context_gpa = ring_context_gpa;
-	workload->rb_head = head;
-	workload->rb_tail = tail;
-	workload->rb_start = start;
-	workload->rb_ctl = ctl;
 	workload->prepare = prepare_execlist_workload;
 	workload->complete = complete_execlist_workload;
-	workload->status = -EINPROGRESS;
 	workload->emulate_schedule_in = emulate_schedule_in;
-	workload->shadowed = false;
-
-	if (ring_id == RCS) {
-		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-			RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
-		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-			RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
-
-		workload->wa_ctx.indirect_ctx.guest_gma =
-			indirect_ctx & INDIRECT_CTX_ADDR_MASK;
-		workload->wa_ctx.indirect_ctx.size =
-			(indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
-			CACHELINE_BYTES;
-		workload->wa_ctx.per_ctx.guest_gma =
-			per_ctx & PER_CTX_ADDR_MASK;
-		workload->wa_ctx.per_ctx.valid = per_ctx & 1;
-	}
 
 	if (emulate_schedule_in)
-		workload->elsp_dwords = vgpu->execlist[ring_id].elsp_dwords;
-
-	gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
-			workload, ring_id, head, tail, start, ctl);
+		workload->elsp_dwords = s->execlist[ring_id].elsp_dwords;
 
 	gvt_dbg_el("workload %p emulate schedule_in %d\n", workload,
 			emulate_schedule_in);
 
-	ret = prepare_mm(workload);
-	if (ret) {
-		kmem_cache_free(vgpu->workloads, workload);
-		return ret;
-	}
-
-	/* Only scan and shadow the first workload in the queue
-	 * as there is only one pre-allocated buf-obj for shadow.
-	 */
-	if (list_empty(workload_q_head(vgpu, ring_id))) {
-		intel_runtime_pm_get(dev_priv);
-		mutex_lock(&dev_priv->drm.struct_mutex);
-		intel_gvt_scan_and_shadow_workload(workload);
-		mutex_unlock(&dev_priv->drm.struct_mutex);
-		intel_runtime_pm_put(dev_priv);
-	}
-
-	queue_workload(workload);
+	intel_vgpu_queue_workload(workload);
 	return 0;
 }
 
 int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
 {
-	struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
-	struct execlist_ctx_descriptor_format desc[2];
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct intel_vgpu_execlist *execlist = &s->execlist[ring_id];
+	struct execlist_ctx_descriptor_format *desc[2];
 	int i, ret;
 
-	desc[0] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
-	desc[1] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
+	desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
+	desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
 
-	if (!desc[0].valid) {
+	if (!desc[0]->valid) {
 		gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n");
 		goto inv_desc;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(desc); i++) {
-		if (!desc[i].valid)
+		if (!desc[i]->valid)
 			continue;
-		if (!desc[i].privilege_access) {
+		if (!desc[i]->privilege_access) {
 			gvt_vgpu_err("unexpected GGTT elsp submission\n");
 			goto inv_desc;
 		}
@@ -798,9 +488,9 @@ int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
 
 	/* submit workload */
 	for (i = 0; i < ARRAY_SIZE(desc); i++) {
-		if (!desc[i].valid)
+		if (!desc[i]->valid)
 			continue;
-		ret = submit_context(vgpu, ring_id, &desc[i], i == 0);
+		ret = submit_context(vgpu, ring_id, desc[i], i == 0);
 		if (ret) {
 			gvt_vgpu_err("failed to submit desc %d\n", i);
 			return ret;
@@ -811,13 +501,14 @@ int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
 
 inv_desc:
 	gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n",
-		     desc[0].udw, desc[0].ldw, desc[1].udw, desc[1].ldw);
+		     desc[0]->udw, desc[0]->ldw, desc[1]->udw, desc[1]->ldw);
 	return -EINVAL;
 }
 
 static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
 {
-	struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct intel_vgpu_execlist *execlist = &s->execlist[ring_id];
 	struct execlist_context_status_pointer_format ctx_status_ptr;
 	u32 ctx_status_ptr_reg;
 
@@ -830,98 +521,47 @@ static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
 
 	ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
 			_EL_OFFSET_STATUS_PTR);
-
 	ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg);
 	ctx_status_ptr.read_ptr = 0;
 	ctx_status_ptr.write_ptr = 0x7;
 	vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
 }
 
-static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask)
+static void clean_execlist(struct intel_vgpu *vgpu, unsigned long engine_mask)
 {
+	unsigned int tmp;
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine;
-	struct intel_vgpu_workload *pos, *n;
-	unsigned int tmp;
+	struct intel_vgpu_submission *s = &vgpu->submission;
 
-	/* free the unsubmited workloads in the queues. */
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
-		list_for_each_entry_safe(pos, n,
-			&vgpu->workload_q_head[engine->id], list) {
-			list_del_init(&pos->list);
-			free_workload(pos);
-		}
-
-		clear_bit(engine->id, vgpu->shadow_ctx_desc_updated);
-	}
-}
-
-void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu)
-{
-	enum intel_engine_id i;
-	struct intel_engine_cs *engine;
-
-	clean_workloads(vgpu, ALL_ENGINES);
-	kmem_cache_destroy(vgpu->workloads);
-
-	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-		kfree(vgpu->reserve_ring_buffer_va[i]);
-		vgpu->reserve_ring_buffer_va[i] = NULL;
-		vgpu->reserve_ring_buffer_size[i] = 0;
-	}
-
-}
-
-#define RESERVE_RING_BUFFER_SIZE		((1 * PAGE_SIZE)/8)
-int intel_vgpu_init_execlist(struct intel_vgpu *vgpu)
-{
-	enum intel_engine_id i;
-	struct intel_engine_cs *engine;
-
-	/* each ring has a virtual execlist engine */
-	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-		init_vgpu_execlist(vgpu, i);
-		INIT_LIST_HEAD(&vgpu->workload_q_head[i]);
+		kfree(s->ring_scan_buffer[engine->id]);
+		s->ring_scan_buffer[engine->id] = NULL;
+		s->ring_scan_buffer_size[engine->id] = 0;
 	}
-
-	vgpu->workloads = kmem_cache_create("gvt-g_vgpu_workload",
-			sizeof(struct intel_vgpu_workload), 0,
-			SLAB_HWCACHE_ALIGN,
-			NULL);
-
-	if (!vgpu->workloads)
-		return -ENOMEM;
-
-	/* each ring has a shadow ring buffer until vgpu destroyed */
-	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-		vgpu->reserve_ring_buffer_va[i] =
-			kmalloc(RESERVE_RING_BUFFER_SIZE, GFP_KERNEL);
-		if (!vgpu->reserve_ring_buffer_va[i]) {
-			gvt_vgpu_err("fail to alloc reserve ring buffer\n");
-			goto out;
-		}
-		vgpu->reserve_ring_buffer_size[i] = RESERVE_RING_BUFFER_SIZE;
-	}
-	return 0;
-out:
-	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-		if (vgpu->reserve_ring_buffer_size[i]) {
-			kfree(vgpu->reserve_ring_buffer_va[i]);
-			vgpu->reserve_ring_buffer_va[i] = NULL;
-			vgpu->reserve_ring_buffer_size[i] = 0;
-		}
-	}
-	return -ENOMEM;
 }
 
-void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
+static void reset_execlist(struct intel_vgpu *vgpu,
 		unsigned long engine_mask)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine;
 	unsigned int tmp;
 
-	clean_workloads(vgpu, engine_mask);
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
 		init_vgpu_execlist(vgpu, engine->id);
 }
+
+static int init_execlist(struct intel_vgpu *vgpu,
+			 unsigned long engine_mask)
+{
+	reset_execlist(vgpu, engine_mask);
+	return 0;
+}
+
+const struct intel_vgpu_submission_ops intel_vgpu_execlist_submission_ops = {
+	.name = "execlist",
+	.init = init_execlist,
+	.reset = reset_execlist,
+	.clean = clean_execlist,
+};
diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h
index 7eced40a1e30..427e40e64d41 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.h
+++ b/drivers/gpu/drm/i915/gvt/execlist.h
@@ -36,10 +36,6 @@
 #define _GVT_EXECLIST_H_
 
 struct execlist_ctx_descriptor_format {
-	union {
-		u32 udw;
-		u32 context_id;
-	};
 	union {
 		u32 ldw;
 		struct {
@@ -54,6 +50,10 @@ struct execlist_ctx_descriptor_format {
 			u32 lrca                   : 20;
 		};
 	};
+	union {
+		u32 udw;
+		u32 context_id;
+	};
 };
 
 struct execlist_status_format {
diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.c b/drivers/gpu/drm/i915/gvt/fb_decoder.c
new file mode 100644
index 000000000000..6b50fe78dc1b
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/fb_decoder.c
@@ -0,0 +1,514 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Bing Niu <bing.niu@intel.com>
+ *    Xu Han <xu.han@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Xiaoguang Chen <xiaoguang.chen@intel.com>
+ *    Yang Liu <yang2.liu@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *
+ */
+
+#include <uapi/drm/drm_fourcc.h>
+#include "i915_drv.h"
+#include "gvt.h"
+
+#define PRIMARY_FORMAT_NUM	16
+struct pixel_format {
+	int	drm_format;	/* Pixel format in DRM definition */
+	int	bpp;		/* Bits per pixel, 0 indicates invalid */
+	char	*desc;		/* The description */
+};
+
+static struct pixel_format bdw_pixel_formats[] = {
+	{DRM_FORMAT_C8, 8, "8-bit Indexed"},
+	{DRM_FORMAT_RGB565, 16, "16-bit BGRX (5:6:5 MSB-R:G:B)"},
+	{DRM_FORMAT_XRGB8888, 32, "32-bit BGRX (8:8:8:8 MSB-X:R:G:B)"},
+	{DRM_FORMAT_XBGR2101010, 32, "32-bit RGBX (2:10:10:10 MSB-X:B:G:R)"},
+
+	{DRM_FORMAT_XRGB2101010, 32, "32-bit BGRX (2:10:10:10 MSB-X:R:G:B)"},
+	{DRM_FORMAT_XBGR8888, 32, "32-bit RGBX (8:8:8:8 MSB-X:B:G:R)"},
+
+	/* non-supported format has bpp default to 0 */
+	{0, 0, NULL},
+};
+
+static struct pixel_format skl_pixel_formats[] = {
+	{DRM_FORMAT_YUYV, 16, "16-bit packed YUYV (8:8:8:8 MSB-V:Y2:U:Y1)"},
+	{DRM_FORMAT_UYVY, 16, "16-bit packed UYVY (8:8:8:8 MSB-Y2:V:Y1:U)"},
+	{DRM_FORMAT_YVYU, 16, "16-bit packed YVYU (8:8:8:8 MSB-U:Y2:V:Y1)"},
+	{DRM_FORMAT_VYUY, 16, "16-bit packed VYUY (8:8:8:8 MSB-Y2:U:Y1:V)"},
+
+	{DRM_FORMAT_C8, 8, "8-bit Indexed"},
+	{DRM_FORMAT_RGB565, 16, "16-bit BGRX (5:6:5 MSB-R:G:B)"},
+	{DRM_FORMAT_ABGR8888, 32, "32-bit RGBA (8:8:8:8 MSB-A:B:G:R)"},
+	{DRM_FORMAT_XBGR8888, 32, "32-bit RGBX (8:8:8:8 MSB-X:B:G:R)"},
+
+	{DRM_FORMAT_ARGB8888, 32, "32-bit BGRA (8:8:8:8 MSB-A:R:G:B)"},
+	{DRM_FORMAT_XRGB8888, 32, "32-bit BGRX (8:8:8:8 MSB-X:R:G:B)"},
+	{DRM_FORMAT_XBGR2101010, 32, "32-bit RGBX (2:10:10:10 MSB-X:B:G:R)"},
+	{DRM_FORMAT_XRGB2101010, 32, "32-bit BGRX (2:10:10:10 MSB-X:R:G:B)"},
+
+	/* non-supported format has bpp default to 0 */
+	{0, 0, NULL},
+};
+
+static int bdw_format_to_drm(int format)
+{
+	int bdw_pixel_formats_index = 6;
+
+	switch (format) {
+	case DISPPLANE_8BPP:
+		bdw_pixel_formats_index = 0;
+		break;
+	case DISPPLANE_BGRX565:
+		bdw_pixel_formats_index = 1;
+		break;
+	case DISPPLANE_BGRX888:
+		bdw_pixel_formats_index = 2;
+		break;
+	case DISPPLANE_RGBX101010:
+		bdw_pixel_formats_index = 3;
+		break;
+	case DISPPLANE_BGRX101010:
+		bdw_pixel_formats_index = 4;
+		break;
+	case DISPPLANE_RGBX888:
+		bdw_pixel_formats_index = 5;
+		break;
+
+	default:
+		break;
+	}
+
+	return bdw_pixel_formats_index;
+}
+
+static int skl_format_to_drm(int format, bool rgb_order, bool alpha,
+	int yuv_order)
+{
+	int skl_pixel_formats_index = 12;
+
+	switch (format) {
+	case PLANE_CTL_FORMAT_INDEXED:
+		skl_pixel_formats_index = 4;
+		break;
+	case PLANE_CTL_FORMAT_RGB_565:
+		skl_pixel_formats_index = 5;
+		break;
+	case PLANE_CTL_FORMAT_XRGB_8888:
+		if (rgb_order)
+			skl_pixel_formats_index = alpha ? 6 : 7;
+		else
+			skl_pixel_formats_index = alpha ? 8 : 9;
+		break;
+	case PLANE_CTL_FORMAT_XRGB_2101010:
+		skl_pixel_formats_index = rgb_order ? 10 : 11;
+		break;
+	case PLANE_CTL_FORMAT_YUV422:
+		skl_pixel_formats_index = yuv_order >> 16;
+		if (skl_pixel_formats_index > 3)
+			return -EINVAL;
+		break;
+
+	default:
+		break;
+	}
+
+	return skl_pixel_formats_index;
+}
+
+static u32 intel_vgpu_get_stride(struct intel_vgpu *vgpu, int pipe,
+	u32 tiled, int stride_mask, int bpp)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+
+	u32 stride_reg = vgpu_vreg_t(vgpu, DSPSTRIDE(pipe)) & stride_mask;
+	u32 stride = stride_reg;
+
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		switch (tiled) {
+		case PLANE_CTL_TILED_LINEAR:
+			stride = stride_reg * 64;
+			break;
+		case PLANE_CTL_TILED_X:
+			stride = stride_reg * 512;
+			break;
+		case PLANE_CTL_TILED_Y:
+			stride = stride_reg * 128;
+			break;
+		case PLANE_CTL_TILED_YF:
+			if (bpp == 8)
+				stride = stride_reg * 64;
+			else if (bpp == 16 || bpp == 32 || bpp == 64)
+				stride = stride_reg * 128;
+			else
+				gvt_dbg_core("skl: unsupported bpp:%d\n", bpp);
+			break;
+		default:
+			gvt_dbg_core("skl: unsupported tile format:%x\n",
+				tiled);
+		}
+	}
+
+	return stride;
+}
+
+static int get_active_pipe(struct intel_vgpu *vgpu)
+{
+	int i;
+
+	for (i = 0; i < I915_MAX_PIPES; i++)
+		if (pipe_is_enabled(vgpu, i))
+			break;
+
+	return i;
+}
+
+/**
+ * intel_vgpu_decode_primary_plane - Decode primary plane
+ * @vgpu: input vgpu
+ * @plane: primary plane to save decoded info
+ * This function is called for decoding plane
+ *
+ * Returns:
+ * 0 on success, non-zero if failed.
+ */
+int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu,
+	struct intel_vgpu_primary_plane_format *plane)
+{
+	u32 val, fmt;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	int pipe;
+
+	pipe = get_active_pipe(vgpu);
+	if (pipe >= I915_MAX_PIPES)
+		return -ENODEV;
+
+	val = vgpu_vreg_t(vgpu, DSPCNTR(pipe));
+	plane->enabled = !!(val & DISPLAY_PLANE_ENABLE);
+	if (!plane->enabled)
+		return -ENODEV;
+
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+		plane->tiled = (val & PLANE_CTL_TILED_MASK) >>
+		_PLANE_CTL_TILED_SHIFT;
+		fmt = skl_format_to_drm(
+			val & PLANE_CTL_FORMAT_MASK,
+			val & PLANE_CTL_ORDER_RGBX,
+			val & PLANE_CTL_ALPHA_MASK,
+			val & PLANE_CTL_YUV422_ORDER_MASK);
+
+		if (fmt >= ARRAY_SIZE(skl_pixel_formats)) {
+			gvt_vgpu_err("Out-of-bounds pixel format index\n");
+			return -EINVAL;
+		}
+
+		plane->bpp = skl_pixel_formats[fmt].bpp;
+		plane->drm_format = skl_pixel_formats[fmt].drm_format;
+	} else {
+		plane->tiled = !!(val & DISPPLANE_TILED);
+		fmt = bdw_format_to_drm(val & DISPPLANE_PIXFORMAT_MASK);
+		plane->bpp = bdw_pixel_formats[fmt].bpp;
+		plane->drm_format = bdw_pixel_formats[fmt].drm_format;
+	}
+
+	if (!plane->bpp) {
+		gvt_vgpu_err("Non-supported pixel format (0x%x)\n", fmt);
+		return -EINVAL;
+	}
+
+	plane->hw_format = fmt;
+
+	plane->base = vgpu_vreg_t(vgpu, DSPSURF(pipe)) & I915_GTT_PAGE_MASK;
+	if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) {
+		gvt_vgpu_err("invalid gma address: %lx\n",
+			     (unsigned long)plane->base);
+		return  -EINVAL;
+	}
+
+	plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base);
+	if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) {
+		gvt_vgpu_err("invalid gma address: %lx\n",
+				(unsigned long)plane->base);
+		return  -EINVAL;
+	}
+
+	plane->stride = intel_vgpu_get_stride(vgpu, pipe, (plane->tiled << 10),
+		(IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) ?
+			(_PRI_PLANE_STRIDE_MASK >> 6) :
+				_PRI_PLANE_STRIDE_MASK, plane->bpp);
+
+	plane->width = (vgpu_vreg_t(vgpu, PIPESRC(pipe)) & _PIPE_H_SRCSZ_MASK) >>
+		_PIPE_H_SRCSZ_SHIFT;
+	plane->width += 1;
+	plane->height = (vgpu_vreg_t(vgpu, PIPESRC(pipe)) &
+			_PIPE_V_SRCSZ_MASK) >> _PIPE_V_SRCSZ_SHIFT;
+	plane->height += 1;	/* raw height is one minus the real value */
+
+	val = vgpu_vreg_t(vgpu, DSPTILEOFF(pipe));
+	plane->x_offset = (val & _PRI_PLANE_X_OFF_MASK) >>
+		_PRI_PLANE_X_OFF_SHIFT;
+	plane->y_offset = (val & _PRI_PLANE_Y_OFF_MASK) >>
+		_PRI_PLANE_Y_OFF_SHIFT;
+
+	return 0;
+}
+
+#define CURSOR_FORMAT_NUM	(1 << 6)
+struct cursor_mode_format {
+	int	drm_format;	/* Pixel format in DRM definition */
+	u8	bpp;		/* Bits per pixel; 0 indicates invalid */
+	u32	width;		/* In pixel */
+	u32	height;		/* In lines */
+	char	*desc;		/* The description */
+};
+
+static struct cursor_mode_format cursor_pixel_formats[] = {
+	{DRM_FORMAT_ARGB8888, 32, 128, 128, "128x128 32bpp ARGB"},
+	{DRM_FORMAT_ARGB8888, 32, 256, 256, "256x256 32bpp ARGB"},
+	{DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"},
+	{DRM_FORMAT_ARGB8888, 32, 64, 64, "64x64 32bpp ARGB"},
+
+	/* non-supported format has bpp default to 0 */
+	{0, 0, 0, 0, NULL},
+};
+
+static int cursor_mode_to_drm(int mode)
+{
+	int cursor_pixel_formats_index = 4;
+
+	switch (mode) {
+	case CURSOR_MODE_128_ARGB_AX:
+		cursor_pixel_formats_index = 0;
+		break;
+	case CURSOR_MODE_256_ARGB_AX:
+		cursor_pixel_formats_index = 1;
+		break;
+	case CURSOR_MODE_64_ARGB_AX:
+		cursor_pixel_formats_index = 2;
+		break;
+	case CURSOR_MODE_64_32B_AX:
+		cursor_pixel_formats_index = 3;
+		break;
+
+	default:
+		break;
+	}
+
+	return cursor_pixel_formats_index;
+}
+
+/**
+ * intel_vgpu_decode_cursor_plane - Decode sprite plane
+ * @vgpu: input vgpu
+ * @plane: cursor plane to save decoded info
+ * This function is called for decoding plane
+ *
+ * Returns:
+ * 0 on success, non-zero if failed.
+ */
+int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu,
+	struct intel_vgpu_cursor_plane_format *plane)
+{
+	u32 val, mode, index;
+	u32 alpha_plane, alpha_force;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	int pipe;
+
+	pipe = get_active_pipe(vgpu);
+	if (pipe >= I915_MAX_PIPES)
+		return -ENODEV;
+
+	val = vgpu_vreg_t(vgpu, CURCNTR(pipe));
+	mode = val & CURSOR_MODE;
+	plane->enabled = (mode != CURSOR_MODE_DISABLE);
+	if (!plane->enabled)
+		return -ENODEV;
+
+	index = cursor_mode_to_drm(mode);
+
+	if (!cursor_pixel_formats[index].bpp) {
+		gvt_vgpu_err("Non-supported cursor mode (0x%x)\n", mode);
+		return -EINVAL;
+	}
+	plane->mode = mode;
+	plane->bpp = cursor_pixel_formats[index].bpp;
+	plane->drm_format = cursor_pixel_formats[index].drm_format;
+	plane->width = cursor_pixel_formats[index].width;
+	plane->height = cursor_pixel_formats[index].height;
+
+	alpha_plane = (val & _CURSOR_ALPHA_PLANE_MASK) >>
+				_CURSOR_ALPHA_PLANE_SHIFT;
+	alpha_force = (val & _CURSOR_ALPHA_FORCE_MASK) >>
+				_CURSOR_ALPHA_FORCE_SHIFT;
+	if (alpha_plane || alpha_force)
+		gvt_dbg_core("alpha_plane=0x%x, alpha_force=0x%x\n",
+			alpha_plane, alpha_force);
+
+	plane->base = vgpu_vreg_t(vgpu, CURBASE(pipe)) & I915_GTT_PAGE_MASK;
+	if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) {
+		gvt_vgpu_err("invalid gma address: %lx\n",
+			     (unsigned long)plane->base);
+		return  -EINVAL;
+	}
+
+	plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base);
+	if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) {
+		gvt_vgpu_err("invalid gma address: %lx\n",
+				(unsigned long)plane->base);
+		return  -EINVAL;
+	}
+
+	val = vgpu_vreg_t(vgpu, CURPOS(pipe));
+	plane->x_pos = (val & _CURSOR_POS_X_MASK) >> _CURSOR_POS_X_SHIFT;
+	plane->x_sign = (val & _CURSOR_SIGN_X_MASK) >> _CURSOR_SIGN_X_SHIFT;
+	plane->y_pos = (val & _CURSOR_POS_Y_MASK) >> _CURSOR_POS_Y_SHIFT;
+	plane->y_sign = (val & _CURSOR_SIGN_Y_MASK) >> _CURSOR_SIGN_Y_SHIFT;
+
+	return 0;
+}
+
+#define SPRITE_FORMAT_NUM	(1 << 3)
+
+static struct pixel_format sprite_pixel_formats[SPRITE_FORMAT_NUM] = {
+	[0x0] = {DRM_FORMAT_YUV422, 16, "YUV 16-bit 4:2:2 packed"},
+	[0x1] = {DRM_FORMAT_XRGB2101010, 32, "RGB 32-bit 2:10:10:10"},
+	[0x2] = {DRM_FORMAT_XRGB8888, 32, "RGB 32-bit 8:8:8:8"},
+	[0x4] = {DRM_FORMAT_AYUV, 32,
+		"YUV 32-bit 4:4:4 packed (8:8:8:8 MSB-X:Y:U:V)"},
+};
+
+/**
+ * intel_vgpu_decode_sprite_plane - Decode sprite plane
+ * @vgpu: input vgpu
+ * @plane: sprite plane to save decoded info
+ * This function is called for decoding plane
+ *
+ * Returns:
+ * 0 on success, non-zero if failed.
+ */
+int intel_vgpu_decode_sprite_plane(struct intel_vgpu *vgpu,
+	struct intel_vgpu_sprite_plane_format *plane)
+{
+	u32 val, fmt;
+	u32 color_order, yuv_order;
+	int drm_format;
+	int pipe;
+
+	pipe = get_active_pipe(vgpu);
+	if (pipe >= I915_MAX_PIPES)
+		return -ENODEV;
+
+	val = vgpu_vreg_t(vgpu, SPRCTL(pipe));
+	plane->enabled = !!(val & SPRITE_ENABLE);
+	if (!plane->enabled)
+		return -ENODEV;
+
+	plane->tiled = !!(val & SPRITE_TILED);
+	color_order = !!(val & SPRITE_RGB_ORDER_RGBX);
+	yuv_order = (val & SPRITE_YUV_BYTE_ORDER_MASK) >>
+				_SPRITE_YUV_ORDER_SHIFT;
+
+	fmt = (val & SPRITE_PIXFORMAT_MASK) >> _SPRITE_FMT_SHIFT;
+	if (!sprite_pixel_formats[fmt].bpp) {
+		gvt_vgpu_err("Non-supported pixel format (0x%x)\n", fmt);
+		return -EINVAL;
+	}
+	plane->hw_format = fmt;
+	plane->bpp = sprite_pixel_formats[fmt].bpp;
+	drm_format = sprite_pixel_formats[fmt].drm_format;
+
+	/* Order of RGB values in an RGBxxx buffer may be ordered RGB or
+	 * BGR depending on the state of the color_order field
+	 */
+	if (!color_order) {
+		if (drm_format == DRM_FORMAT_XRGB2101010)
+			drm_format = DRM_FORMAT_XBGR2101010;
+		else if (drm_format == DRM_FORMAT_XRGB8888)
+			drm_format = DRM_FORMAT_XBGR8888;
+	}
+
+	if (drm_format == DRM_FORMAT_YUV422) {
+		switch (yuv_order) {
+		case 0:
+			drm_format = DRM_FORMAT_YUYV;
+			break;
+		case 1:
+			drm_format = DRM_FORMAT_UYVY;
+			break;
+		case 2:
+			drm_format = DRM_FORMAT_YVYU;
+			break;
+		case 3:
+			drm_format = DRM_FORMAT_VYUY;
+			break;
+		default:
+			/* yuv_order has only 2 bits */
+			break;
+		}
+	}
+
+	plane->drm_format = drm_format;
+
+	plane->base = vgpu_vreg_t(vgpu, SPRSURF(pipe)) & I915_GTT_PAGE_MASK;
+	if (!intel_gvt_ggtt_validate_range(vgpu, plane->base, 0)) {
+		gvt_vgpu_err("invalid gma address: %lx\n",
+			     (unsigned long)plane->base);
+		return  -EINVAL;
+	}
+
+	plane->base_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, plane->base);
+	if (plane->base_gpa == INTEL_GVT_INVALID_ADDR) {
+		gvt_vgpu_err("invalid gma address: %lx\n",
+				(unsigned long)plane->base);
+		return  -EINVAL;
+	}
+
+	plane->stride = vgpu_vreg_t(vgpu, SPRSTRIDE(pipe)) &
+				_SPRITE_STRIDE_MASK;
+
+	val = vgpu_vreg_t(vgpu, SPRSIZE(pipe));
+	plane->height = (val & _SPRITE_SIZE_HEIGHT_MASK) >>
+		_SPRITE_SIZE_HEIGHT_SHIFT;
+	plane->width = (val & _SPRITE_SIZE_WIDTH_MASK) >>
+		_SPRITE_SIZE_WIDTH_SHIFT;
+	plane->height += 1;	/* raw height is one minus the real value */
+	plane->width += 1;	/* raw width is one minus the real value */
+
+	val = vgpu_vreg_t(vgpu, SPRPOS(pipe));
+	plane->x_pos = (val & _SPRITE_POS_X_MASK) >> _SPRITE_POS_X_SHIFT;
+	plane->y_pos = (val & _SPRITE_POS_Y_MASK) >> _SPRITE_POS_Y_SHIFT;
+
+	val = vgpu_vreg_t(vgpu, SPROFFSET(pipe));
+	plane->x_offset = (val & _SPRITE_OFFSET_START_X_MASK) >>
+			   _SPRITE_OFFSET_START_X_SHIFT;
+	plane->y_offset = (val & _SPRITE_OFFSET_START_Y_MASK) >>
+			   _SPRITE_OFFSET_START_Y_SHIFT;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/fb_decoder.h b/drivers/gpu/drm/i915/gvt/fb_decoder.h
new file mode 100644
index 000000000000..cb055f3c81a2
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/fb_decoder.h
@@ -0,0 +1,169 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Bing Niu <bing.niu@intel.com>
+ *    Xu Han <xu.han@intel.com>
+ *    Ping Gao <ping.a.gao@intel.com>
+ *    Xiaoguang Chen <xiaoguang.chen@intel.com>
+ *    Yang Liu <yang2.liu@intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *
+ */
+
+#ifndef _GVT_FB_DECODER_H_
+#define _GVT_FB_DECODER_H_
+
+#define _PLANE_CTL_FORMAT_SHIFT		24
+#define _PLANE_CTL_TILED_SHIFT		10
+#define _PIPE_V_SRCSZ_SHIFT		0
+#define _PIPE_V_SRCSZ_MASK		(0xfff << _PIPE_V_SRCSZ_SHIFT)
+#define _PIPE_H_SRCSZ_SHIFT		16
+#define _PIPE_H_SRCSZ_MASK		(0x1fff << _PIPE_H_SRCSZ_SHIFT)
+
+#define _PRI_PLANE_FMT_SHIFT		26
+#define _PRI_PLANE_STRIDE_MASK		(0x3ff << 6)
+#define _PRI_PLANE_X_OFF_SHIFT		0
+#define _PRI_PLANE_X_OFF_MASK		(0x1fff << _PRI_PLANE_X_OFF_SHIFT)
+#define _PRI_PLANE_Y_OFF_SHIFT		16
+#define _PRI_PLANE_Y_OFF_MASK		(0xfff << _PRI_PLANE_Y_OFF_SHIFT)
+
+#define _CURSOR_MODE			0x3f
+#define _CURSOR_ALPHA_FORCE_SHIFT	8
+#define _CURSOR_ALPHA_FORCE_MASK	(0x3 << _CURSOR_ALPHA_FORCE_SHIFT)
+#define _CURSOR_ALPHA_PLANE_SHIFT	10
+#define _CURSOR_ALPHA_PLANE_MASK	(0x3 << _CURSOR_ALPHA_PLANE_SHIFT)
+#define _CURSOR_POS_X_SHIFT		0
+#define _CURSOR_POS_X_MASK		(0x1fff << _CURSOR_POS_X_SHIFT)
+#define _CURSOR_SIGN_X_SHIFT		15
+#define _CURSOR_SIGN_X_MASK		(1 << _CURSOR_SIGN_X_SHIFT)
+#define _CURSOR_POS_Y_SHIFT		16
+#define _CURSOR_POS_Y_MASK		(0xfff << _CURSOR_POS_Y_SHIFT)
+#define _CURSOR_SIGN_Y_SHIFT		31
+#define _CURSOR_SIGN_Y_MASK		(1 << _CURSOR_SIGN_Y_SHIFT)
+
+#define _SPRITE_FMT_SHIFT		25
+#define _SPRITE_COLOR_ORDER_SHIFT	20
+#define _SPRITE_YUV_ORDER_SHIFT		16
+#define _SPRITE_STRIDE_SHIFT		6
+#define _SPRITE_STRIDE_MASK		(0x1ff << _SPRITE_STRIDE_SHIFT)
+#define _SPRITE_SIZE_WIDTH_SHIFT	0
+#define _SPRITE_SIZE_HEIGHT_SHIFT	16
+#define _SPRITE_SIZE_WIDTH_MASK		(0x1fff << _SPRITE_SIZE_WIDTH_SHIFT)
+#define _SPRITE_SIZE_HEIGHT_MASK	(0xfff << _SPRITE_SIZE_HEIGHT_SHIFT)
+#define _SPRITE_POS_X_SHIFT		0
+#define _SPRITE_POS_Y_SHIFT		16
+#define _SPRITE_POS_X_MASK		(0x1fff << _SPRITE_POS_X_SHIFT)
+#define _SPRITE_POS_Y_MASK		(0xfff << _SPRITE_POS_Y_SHIFT)
+#define _SPRITE_OFFSET_START_X_SHIFT	0
+#define _SPRITE_OFFSET_START_Y_SHIFT	16
+#define _SPRITE_OFFSET_START_X_MASK	(0x1fff << _SPRITE_OFFSET_START_X_SHIFT)
+#define _SPRITE_OFFSET_START_Y_MASK	(0xfff << _SPRITE_OFFSET_START_Y_SHIFT)
+
+enum GVT_FB_EVENT {
+	FB_MODE_SET_START = 1,
+	FB_MODE_SET_END,
+	FB_DISPLAY_FLIP,
+};
+
+enum DDI_PORT {
+	DDI_PORT_NONE	= 0,
+	DDI_PORT_B	= 1,
+	DDI_PORT_C	= 2,
+	DDI_PORT_D	= 3,
+	DDI_PORT_E	= 4
+};
+
+struct intel_gvt;
+
+/* color space conversion and gamma correction are not included */
+struct intel_vgpu_primary_plane_format {
+	u8	enabled;	/* plane is enabled */
+	u8	tiled;		/* X-tiled */
+	u8	bpp;		/* bits per pixel */
+	u32	hw_format;	/* format field in the PRI_CTL register */
+	u32	drm_format;	/* format in DRM definition */
+	u32	base;		/* framebuffer base in graphics memory */
+	u64     base_gpa;
+	u32	x_offset;	/* in pixels */
+	u32	y_offset;	/* in lines */
+	u32	width;		/* in pixels */
+	u32	height;		/* in lines */
+	u32	stride;		/* in bytes */
+};
+
+struct intel_vgpu_sprite_plane_format {
+	u8	enabled;	/* plane is enabled */
+	u8	tiled;		/* X-tiled */
+	u8	bpp;		/* bits per pixel */
+	u32	hw_format;	/* format field in the SPR_CTL register */
+	u32	drm_format;	/* format in DRM definition */
+	u32	base;		/* sprite base in graphics memory */
+	u64     base_gpa;
+	u32	x_pos;		/* in pixels */
+	u32	y_pos;		/* in lines */
+	u32	x_offset;	/* in pixels */
+	u32	y_offset;	/* in lines */
+	u32	width;		/* in pixels */
+	u32	height;		/* in lines */
+	u32	stride;		/* in bytes */
+};
+
+struct intel_vgpu_cursor_plane_format {
+	u8	enabled;
+	u8	mode;		/* cursor mode select */
+	u8	bpp;		/* bits per pixel */
+	u32	drm_format;	/* format in DRM definition */
+	u32	base;		/* cursor base in graphics memory */
+	u64     base_gpa;
+	u32	x_pos;		/* in pixels */
+	u32	y_pos;		/* in lines */
+	u8	x_sign;		/* X Position Sign */
+	u8	y_sign;		/* Y Position Sign */
+	u32	width;		/* in pixels */
+	u32	height;		/* in lines */
+	u32	x_hot;		/* in pixels */
+	u32	y_hot;		/* in pixels */
+};
+
+struct intel_vgpu_pipe_format {
+	struct intel_vgpu_primary_plane_format	primary;
+	struct intel_vgpu_sprite_plane_format	sprite;
+	struct intel_vgpu_cursor_plane_format	cursor;
+	enum DDI_PORT ddi_port;  /* the DDI port that pipe is connected to */
+};
+
+struct intel_vgpu_fb_format {
+	struct intel_vgpu_pipe_format	pipes[I915_MAX_PIPES];
+};
+
+int intel_vgpu_decode_primary_plane(struct intel_vgpu *vgpu,
+	struct intel_vgpu_primary_plane_format *plane);
+int intel_vgpu_decode_cursor_plane(struct intel_vgpu *vgpu,
+	struct intel_vgpu_cursor_plane_format *plane);
+int intel_vgpu_decode_sprite_plane(struct intel_vgpu *vgpu,
+	struct intel_vgpu_sprite_plane_format *plane);
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c
index a26c1705430e..a73e1d418c22 100644
--- a/drivers/gpu/drm/i915/gvt/firmware.c
+++ b/drivers/gpu/drm/i915/gvt/firmware.c
@@ -66,20 +66,23 @@ static struct bin_attribute firmware_attr = {
 	.mmap = NULL,
 };
 
-static int expose_firmware_sysfs(struct intel_gvt *gvt)
+static int mmio_snapshot_handler(struct intel_gvt *gvt, u32 offset, void *data)
 {
 	struct drm_i915_private *dev_priv = gvt->dev_priv;
+
+	*(u32 *)(data + offset) = I915_READ_NOTRACE(_MMIO(offset));
+	return 0;
+}
+
+static int expose_firmware_sysfs(struct intel_gvt *gvt)
+{
 	struct intel_gvt_device_info *info = &gvt->device_info;
 	struct pci_dev *pdev = gvt->dev_priv->drm.pdev;
-	struct intel_gvt_mmio_info *e;
-	struct gvt_mmio_block *block = gvt->mmio.mmio_block;
-	int num = gvt->mmio.num_mmio_block;
 	struct gvt_firmware_header *h;
 	void *firmware;
 	void *p;
 	unsigned long size, crc32_start;
-	int i, j;
-	int ret;
+	int i, ret;
 
 	size = sizeof(*h) + info->mmio_size + info->cfg_space_size;
 	firmware = vzalloc(size);
@@ -104,15 +107,8 @@ static int expose_firmware_sysfs(struct intel_gvt *gvt)
 
 	p = firmware + h->mmio_offset;
 
-	hash_for_each(gvt->mmio.mmio_info_table, i, e, node)
-		*(u32 *)(p + e->offset) = I915_READ_NOTRACE(_MMIO(e->offset));
-
-	for (i = 0; i < num; i++, block++) {
-		for (j = 0; j < block->size; j += 4)
-			*(u32 *)(p + INTEL_GVT_MMIO_OFFSET(block->offset) + j) =
-				I915_READ_NOTRACE(_MMIO(INTEL_GVT_MMIO_OFFSET(
-							block->offset) + j));
-	}
+	/* Take a snapshot of hw mmio registers. */
+	intel_gvt_for_each_tracked_mmio(gvt, mmio_snapshot_handler, p);
 
 	memcpy(gvt->firmware.mmio, p, info->mmio_size);
 
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 64d67ff9bf08..8d5317d0122d 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -94,12 +94,12 @@ int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
 	u64 h_addr;
 	int ret;
 
-	ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << GTT_PAGE_SHIFT,
+	ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << I915_GTT_PAGE_SHIFT,
 				       &h_addr);
 	if (ret)
 		return ret;
 
-	*h_index = h_addr >> GTT_PAGE_SHIFT;
+	*h_index = h_addr >> I915_GTT_PAGE_SHIFT;
 	return 0;
 }
 
@@ -109,12 +109,12 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
 	u64 g_addr;
 	int ret;
 
-	ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << GTT_PAGE_SHIFT,
+	ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << I915_GTT_PAGE_SHIFT,
 				       &g_addr);
 	if (ret)
 		return ret;
 
-	*g_index = g_addr >> GTT_PAGE_SHIFT;
+	*g_index = g_addr >> I915_GTT_PAGE_SHIFT;
 	return 0;
 }
 
@@ -156,13 +156,15 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
 
 struct gtt_type_table_entry {
 	int entry_type;
+	int pt_type;
 	int next_pt_type;
 	int pse_entry_type;
 };
 
-#define GTT_TYPE_TABLE_ENTRY(type, e_type, npt_type, pse_type) \
+#define GTT_TYPE_TABLE_ENTRY(type, e_type, cpt_type, npt_type, pse_type) \
 	[type] = { \
 		.entry_type = e_type, \
+		.pt_type = cpt_type, \
 		.next_pt_type = npt_type, \
 		.pse_entry_type = pse_type, \
 	}
@@ -170,55 +172,68 @@ struct gtt_type_table_entry {
 static struct gtt_type_table_entry gtt_type_table[] = {
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
 			GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+			GTT_TYPE_INVALID,
 			GTT_TYPE_PPGTT_PML4_PT,
 			GTT_TYPE_INVALID),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_PT,
 			GTT_TYPE_PPGTT_PML4_ENTRY,
+			GTT_TYPE_PPGTT_PML4_PT,
 			GTT_TYPE_PPGTT_PDP_PT,
 			GTT_TYPE_INVALID),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_ENTRY,
 			GTT_TYPE_PPGTT_PML4_ENTRY,
+			GTT_TYPE_PPGTT_PML4_PT,
 			GTT_TYPE_PPGTT_PDP_PT,
 			GTT_TYPE_INVALID),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_PT,
 			GTT_TYPE_PPGTT_PDP_ENTRY,
+			GTT_TYPE_PPGTT_PDP_PT,
 			GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
 			GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+			GTT_TYPE_INVALID,
 			GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_ENTRY,
 			GTT_TYPE_PPGTT_PDP_ENTRY,
+			GTT_TYPE_PPGTT_PDP_PT,
 			GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_PPGTT_PDE_ENTRY,
+			GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_ENTRY,
 			GTT_TYPE_PPGTT_PDE_ENTRY,
+			GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+			GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_INVALID,
 			GTT_TYPE_INVALID),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY,
 			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+			GTT_TYPE_PPGTT_PTE_PT,
 			GTT_TYPE_INVALID,
 			GTT_TYPE_INVALID),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY,
 			GTT_TYPE_PPGTT_PDE_ENTRY,
+			GTT_TYPE_PPGTT_PDE_PT,
 			GTT_TYPE_INVALID,
 			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_1G_ENTRY,
 			GTT_TYPE_PPGTT_PDP_ENTRY,
+			GTT_TYPE_PPGTT_PDP_PT,
 			GTT_TYPE_INVALID,
 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_GGTT_PTE,
 			GTT_TYPE_GGTT_PTE,
 			GTT_TYPE_INVALID,
+			GTT_TYPE_INVALID,
 			GTT_TYPE_INVALID),
 };
 
@@ -227,6 +242,11 @@ static inline int get_next_pt_type(int type)
 	return gtt_type_table[type].next_pt_type;
 }
 
+static inline int get_pt_type(int type)
+{
+	return gtt_type_table[type].pt_type;
+}
+
 static inline int get_entry_type(int type)
 {
 	return gtt_type_table[type].entry_type;
@@ -351,7 +371,7 @@ static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
 		return false;
 
 	e->type = get_entry_type(e->type);
-	if (!(e->val64 & (1 << 7)))
+	if (!(e->val64 & BIT(7)))
 		return false;
 
 	e->type = get_pse_type(e->type);
@@ -369,12 +389,17 @@ static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
 			|| e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
 		return (e->val64 != 0);
 	else
-		return (e->val64 & (1 << 0));
+		return (e->val64 & BIT(0));
 }
 
 static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
 {
-	e->val64 &= ~(1 << 0);
+	e->val64 &= ~BIT(0);
+}
+
+static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e)
+{
+	e->val64 |= BIT(0);
 }
 
 /*
@@ -382,7 +407,7 @@ static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
  */
 static unsigned long gma_to_ggtt_pte_index(unsigned long gma)
 {
-	unsigned long x = (gma >> GTT_PAGE_SHIFT);
+	unsigned long x = (gma >> I915_GTT_PAGE_SHIFT);
 
 	trace_gma_index(__func__, gma, x);
 	return x;
@@ -406,6 +431,7 @@ static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
 	.get_entry = gtt_get_entry64,
 	.set_entry = gtt_set_entry64,
 	.clear_present = gtt_entry_clear_present,
+	.set_present = gtt_entry_set_present,
 	.test_present = gen8_gtt_test_present,
 	.test_pse = gen8_gtt_test_pse,
 	.get_pfn = gen8_gtt_get_pfn,
@@ -494,7 +520,7 @@ static inline int ppgtt_spt_get_entry(
 		return -EINVAL;
 
 	ret = ops->get_entry(page_table, e, index, guest,
-			spt->guest_page.gfn << GTT_PAGE_SHIFT,
+			spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT,
 			spt->vgpu);
 	if (ret)
 		return ret;
@@ -516,7 +542,7 @@ static inline int ppgtt_spt_set_entry(
 		return -EINVAL;
 
 	return ops->set_entry(page_table, e, index, guest,
-			spt->guest_page.gfn << GTT_PAGE_SHIFT,
+			spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT,
 			spt->vgpu);
 }
 
@@ -537,88 +563,103 @@ static inline int ppgtt_spt_set_entry(
 		spt->shadow_page.type, e, index, false)
 
 /**
- * intel_vgpu_init_guest_page - init a guest page data structure
+ * intel_vgpu_init_page_track - init a page track data structure
  * @vgpu: a vGPU
- * @p: a guest page data structure
+ * @t: a page track data structure
  * @gfn: guest memory page frame number
- * @handler: function will be called when target guest memory page has
+ * @handler: the function will be called when target guest memory page has
  * been modified.
  *
- * This function is called when user wants to track a guest memory page.
+ * This function is called when a user wants to prepare a page track data
+ * structure to track a guest memory page.
  *
  * Returns:
  * Zero on success, negative error code if failed.
  */
-int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *p,
+int intel_vgpu_init_page_track(struct intel_vgpu *vgpu,
+		struct intel_vgpu_page_track *t,
 		unsigned long gfn,
 		int (*handler)(void *, u64, void *, int),
 		void *data)
 {
-	INIT_HLIST_NODE(&p->node);
+	INIT_HLIST_NODE(&t->node);
 
-	p->writeprotection = false;
-	p->gfn = gfn;
-	p->handler = handler;
-	p->data = data;
-	p->oos_page = NULL;
-	p->write_cnt = 0;
+	t->tracked = false;
+	t->gfn = gfn;
+	t->handler = handler;
+	t->data = data;
 
-	hash_add(vgpu->gtt.guest_page_hash_table, &p->node, p->gfn);
+	hash_add(vgpu->gtt.tracked_guest_page_hash_table, &t->node, t->gfn);
 	return 0;
 }
 
-static int detach_oos_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_oos_page *oos_page);
-
 /**
- * intel_vgpu_clean_guest_page - release the resource owned by guest page data
- * structure
+ * intel_vgpu_clean_page_track - release a page track data structure
  * @vgpu: a vGPU
- * @p: a tracked guest page
+ * @t: a page track data structure
  *
- * This function is called when user tries to stop tracking a guest memory
- * page.
+ * This function is called before a user frees a page track data structure.
  */
-void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *p)
+void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu,
+		struct intel_vgpu_page_track *t)
 {
-	if (!hlist_unhashed(&p->node))
-		hash_del(&p->node);
+	if (!hlist_unhashed(&t->node))
+		hash_del(&t->node);
 
-	if (p->oos_page)
-		detach_oos_page(vgpu, p->oos_page);
-
-	if (p->writeprotection)
-		intel_gvt_hypervisor_unset_wp_page(vgpu, p);
+	if (t->tracked)
+		intel_gvt_hypervisor_disable_page_track(vgpu, t);
 }
 
 /**
- * intel_vgpu_find_guest_page - find a guest page data structure by GFN.
+ * intel_vgpu_find_tracked_page - find a tracked guest page
  * @vgpu: a vGPU
  * @gfn: guest memory page frame number
  *
- * This function is called when emulation logic wants to know if a trapped GFN
- * is a tracked guest page.
+ * This function is called when the emulation layer wants to figure out if a
+ * trapped GFN is a tracked guest page.
  *
  * Returns:
- * Pointer to guest page data structure, NULL if failed.
+ * Pointer to page track data structure, NULL if not found.
  */
-struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+struct intel_vgpu_page_track *intel_vgpu_find_tracked_page(
 		struct intel_vgpu *vgpu, unsigned long gfn)
 {
-	struct intel_vgpu_guest_page *p;
+	struct intel_vgpu_page_track *t;
 
-	hash_for_each_possible(vgpu->gtt.guest_page_hash_table,
-		p, node, gfn) {
-		if (p->gfn == gfn)
-			return p;
+	hash_for_each_possible(vgpu->gtt.tracked_guest_page_hash_table,
+			t, node, gfn) {
+		if (t->gfn == gfn)
+			return t;
 	}
 	return NULL;
 }
 
+static int init_guest_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *p,
+		unsigned long gfn,
+		int (*handler)(void *, u64, void *, int),
+		void *data)
+{
+	p->oos_page = NULL;
+	p->write_cnt = 0;
+
+	return intel_vgpu_init_page_track(vgpu, &p->track, gfn, handler, data);
+}
+
+static int detach_oos_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_oos_page *oos_page);
+
+static void clean_guest_page(struct intel_vgpu *vgpu,
+		struct intel_vgpu_guest_page *p)
+{
+	if (p->oos_page)
+		detach_oos_page(vgpu, p->oos_page);
+
+	intel_vgpu_clean_page_track(vgpu, &p->track);
+}
+
 static inline int init_shadow_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_shadow_page *p, int type)
+		struct intel_vgpu_shadow_page *p, int type, bool hash)
 {
 	struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 	dma_addr_t daddr;
@@ -634,8 +675,9 @@ static inline int init_shadow_page(struct intel_vgpu *vgpu,
 
 	INIT_HLIST_NODE(&p->node);
 
-	p->mfn = daddr >> GTT_PAGE_SHIFT;
-	hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
+	p->mfn = daddr >> I915_GTT_PAGE_SHIFT;
+	if (hash)
+		hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
 	return 0;
 }
 
@@ -644,7 +686,7 @@ static inline void clean_shadow_page(struct intel_vgpu *vgpu,
 {
 	struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 
-	dma_unmap_page(kdev, p->mfn << GTT_PAGE_SHIFT, 4096,
+	dma_unmap_page(kdev, p->mfn << I915_GTT_PAGE_SHIFT, 4096,
 			PCI_DMA_BIDIRECTIONAL);
 
 	if (!hlist_unhashed(&p->node))
@@ -664,6 +706,9 @@ static inline struct intel_vgpu_shadow_page *find_shadow_page(
 	return NULL;
 }
 
+#define page_track_to_guest_page(ptr) \
+	container_of(ptr, struct intel_vgpu_guest_page, track)
+
 #define guest_page_to_ppgtt_spt(ptr) \
 	container_of(ptr, struct intel_vgpu_ppgtt_spt, guest_page)
 
@@ -697,7 +742,7 @@ static void ppgtt_free_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
 	trace_spt_free(spt->vgpu->id, spt, spt->shadow_page.type);
 
 	clean_shadow_page(spt->vgpu, &spt->shadow_page);
-	intel_vgpu_clean_guest_page(spt->vgpu, &spt->guest_page);
+	clean_guest_page(spt->vgpu, &spt->guest_page);
 	list_del_init(&spt->post_shadow_list);
 
 	free_spt(spt);
@@ -713,22 +758,24 @@ static void ppgtt_free_all_shadow_page(struct intel_vgpu *vgpu)
 		ppgtt_free_shadow_page(shadow_page_to_ppgtt_spt(sp));
 }
 
-static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+static int ppgtt_handle_guest_write_page_table_bytes(
+		struct intel_vgpu_guest_page *gpt,
 		u64 pa, void *p_data, int bytes);
 
-static int ppgtt_write_protection_handler(void *gp, u64 pa,
+static int ppgtt_write_protection_handler(void *data, u64 pa,
 		void *p_data, int bytes)
 {
-	struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
+	struct intel_vgpu_page_track *t = data;
+	struct intel_vgpu_guest_page *p = page_track_to_guest_page(t);
 	int ret;
 
 	if (bytes != 4 && bytes != 8)
 		return -EINVAL;
 
-	if (!gpt->writeprotection)
+	if (!t->tracked)
 		return -EINVAL;
 
-	ret = ppgtt_handle_guest_write_page_table_bytes(gp,
+	ret = ppgtt_handle_guest_write_page_table_bytes(p,
 		pa, p_data, bytes);
 	if (ret)
 		return ret;
@@ -762,13 +809,13 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_shadow_page(
 	 * TODO: guest page type may be different with shadow page type,
 	 *	 when we support PSE page in future.
 	 */
-	ret = init_shadow_page(vgpu, &spt->shadow_page, type);
+	ret = init_shadow_page(vgpu, &spt->shadow_page, type, true);
 	if (ret) {
 		gvt_vgpu_err("fail to initialize shadow page for spt\n");
 		goto err;
 	}
 
-	ret = intel_vgpu_init_guest_page(vgpu, &spt->guest_page,
+	ret = init_guest_page(vgpu, &spt->guest_page,
 			gfn, ppgtt_write_protection_handler, NULL);
 	if (ret) {
 		gvt_vgpu_err("fail to initialize guest page for spt\n");
@@ -798,7 +845,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page(
 	((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
 
 #define pt_entries(spt) \
-	(GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
+	(I915_GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
 
 #define for_each_present_guest_entry(spt, e, i) \
 	for (i = 0; i < pt_entries(spt); i++) \
@@ -856,7 +903,7 @@ static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
 	int v = atomic_read(&spt->refcount);
 
 	trace_spt_change(spt->vgpu->id, "die", spt,
-			spt->guest_page.gfn, spt->shadow_page.type);
+			spt->guest_page.track.gfn, spt->shadow_page.type);
 
 	trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
 
@@ -878,7 +925,7 @@ static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
 	}
 release:
 	trace_spt_change(spt->vgpu->id, "release", spt,
-			spt->guest_page.gfn, spt->shadow_page.type);
+			spt->guest_page.track.gfn, spt->shadow_page.type);
 	ppgtt_free_shadow_page(spt);
 	return 0;
 fail:
@@ -895,6 +942,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
 	struct intel_vgpu_ppgtt_spt *s = NULL;
 	struct intel_vgpu_guest_page *g;
+	struct intel_vgpu_page_track *t;
 	int ret;
 
 	if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(we->type)))) {
@@ -902,8 +950,9 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
 		goto fail;
 	}
 
-	g = intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we));
-	if (g) {
+	t = intel_vgpu_find_tracked_page(vgpu, ops->get_pfn(we));
+	if (t) {
+		g = page_track_to_guest_page(t);
 		s = guest_page_to_ppgtt_spt(g);
 		ppgtt_get_shadow_page(s);
 	} else {
@@ -915,7 +964,8 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
 			goto fail;
 		}
 
-		ret = intel_gvt_hypervisor_set_wp_page(vgpu, &s->guest_page);
+		ret = intel_gvt_hypervisor_enable_page_track(vgpu,
+				&s->guest_page.track);
 		if (ret)
 			goto fail;
 
@@ -923,7 +973,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
 		if (ret)
 			goto fail;
 
-		trace_spt_change(vgpu->id, "new", s, s->guest_page.gfn,
+		trace_spt_change(vgpu->id, "new", s, s->guest_page.track.gfn,
 			s->shadow_page.type);
 	}
 	return s;
@@ -947,19 +997,22 @@ static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
 static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
 {
 	struct intel_vgpu *vgpu = spt->vgpu;
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
 	struct intel_vgpu_ppgtt_spt *s;
 	struct intel_gvt_gtt_entry se, ge;
-	unsigned long i;
+	unsigned long gfn, i;
 	int ret;
 
 	trace_spt_change(spt->vgpu->id, "born", spt,
-			spt->guest_page.gfn, spt->shadow_page.type);
+			spt->guest_page.track.gfn, spt->shadow_page.type);
 
 	if (gtt_type_is_pte_pt(spt->shadow_page.type)) {
 		for_each_present_guest_entry(spt, &ge, i) {
-			ret = gtt_entry_p2m(vgpu, &ge, &se);
-			if (ret)
-				goto fail;
+			gfn = ops->get_pfn(&ge);
+			if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn) ||
+				gtt_entry_p2m(vgpu, &ge, &se))
+				ops->set_pfn(&se, gvt->gtt.scratch_mfn);
 			ppgtt_set_shadow_entry(spt, &se, i);
 		}
 		return 0;
@@ -1078,11 +1131,11 @@ static int sync_oos_page(struct intel_vgpu *vgpu,
 	old.type = new.type = get_entry_type(spt->guest_page_type);
 	old.val64 = new.val64 = 0;
 
-	for (index = 0; index < (GTT_PAGE_SIZE >> info->gtt_entry_size_shift);
-		index++) {
+	for (index = 0; index < (I915_GTT_PAGE_SIZE >>
+				info->gtt_entry_size_shift); index++) {
 		ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu);
 		ops->get_entry(NULL, &new, index, true,
-			oos_page->guest_page->gfn << PAGE_SHIFT, vgpu);
+			oos_page->guest_page->track.gfn << PAGE_SHIFT, vgpu);
 
 		if (old.val64 == new.val64
 			&& !test_and_clear_bit(index, spt->post_shadow_bitmap))
@@ -1132,8 +1185,9 @@ static int attach_oos_page(struct intel_vgpu *vgpu,
 	struct intel_gvt *gvt = vgpu->gvt;
 	int ret;
 
-	ret = intel_gvt_hypervisor_read_gpa(vgpu, gpt->gfn << GTT_PAGE_SHIFT,
-		oos_page->mem, GTT_PAGE_SIZE);
+	ret = intel_gvt_hypervisor_read_gpa(vgpu,
+			gpt->track.gfn << I915_GTT_PAGE_SHIFT,
+			oos_page->mem, I915_GTT_PAGE_SIZE);
 	if (ret)
 		return ret;
 
@@ -1152,7 +1206,7 @@ static int ppgtt_set_guest_page_sync(struct intel_vgpu *vgpu,
 {
 	int ret;
 
-	ret = intel_gvt_hypervisor_set_wp_page(vgpu, gpt);
+	ret = intel_gvt_hypervisor_enable_page_track(vgpu, &gpt->track);
 	if (ret)
 		return ret;
 
@@ -1200,7 +1254,7 @@ static int ppgtt_set_guest_page_oos(struct intel_vgpu *vgpu,
 			gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
 
 	list_add_tail(&oos_page->vm_list, &vgpu->gtt.oos_page_list_head);
-	return intel_gvt_hypervisor_unset_wp_page(vgpu, gpt);
+	return intel_gvt_hypervisor_disable_page_track(vgpu, &gpt->track);
 }
 
 /**
@@ -1335,10 +1389,10 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
 	return 0;
 }
 
-static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+static int ppgtt_handle_guest_write_page_table_bytes(
+		struct intel_vgpu_guest_page *gpt,
 		u64 pa, void *p_data, int bytes)
 {
-	struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
 	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
 	struct intel_vgpu *vgpu = spt->vgpu;
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
@@ -1418,7 +1472,7 @@ static int gen8_mm_alloc_page_table(struct intel_vgpu_mm *mm)
 		mm->shadow_page_table = mem + mm->page_table_entry_size;
 	} else if (mm->type == INTEL_GVT_MM_GGTT) {
 		mm->page_table_entry_cnt =
-			(gvt_ggtt_gm_sz(gvt) >> GTT_PAGE_SHIFT);
+			(gvt_ggtt_gm_sz(gvt) >> I915_GTT_PAGE_SHIFT);
 		mm->page_table_entry_size = mm->page_table_entry_cnt *
 			info->gtt_entry_size;
 		mem = vzalloc(mm->page_table_entry_size);
@@ -1740,8 +1794,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
 				gma_ops->gma_to_ggtt_pte_index(gma));
 		if (ret)
 			goto err;
-		gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
-			+ (gma & ~GTT_PAGE_MASK);
+		gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT)
+			+ (gma & ~I915_GTT_PAGE_MASK);
 
 		trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa);
 		return gpa;
@@ -1793,8 +1847,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
 		}
 	}
 
-	gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
-		+ (gma & ~GTT_PAGE_MASK);
+	gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT)
+		+ (gma & ~I915_GTT_PAGE_MASK);
 
 	trace_gma_translate(vgpu->id, "ppgtt", 0,
 			mm->page_table_level, gma, gpa);
@@ -1855,14 +1909,14 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
 	unsigned long g_gtt_index = off >> info->gtt_entry_size_shift;
-	unsigned long gma;
+	unsigned long gma, gfn;
 	struct intel_gvt_gtt_entry e, m;
 	int ret;
 
 	if (bytes != 4 && bytes != 8)
 		return -EINVAL;
 
-	gma = g_gtt_index << GTT_PAGE_SHIFT;
+	gma = g_gtt_index << I915_GTT_PAGE_SHIFT;
 
 	/* the VM may configure the whole GM space when ballooning is used */
 	if (!vgpu_gmadr_is_valid(vgpu, gma))
@@ -1874,6 +1928,16 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 			bytes);
 
 	if (ops->test_present(&e)) {
+		gfn = ops->get_pfn(&e);
+
+		/* one PTE update may be issued in multiple writes and the
+		 * first write may not construct a valid gfn
+		 */
+		if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) {
+			ops->set_pfn(&m, gvt->gtt.scratch_mfn);
+			goto out;
+		}
+
 		ret = gtt_entry_p2m(vgpu, &e, &m);
 		if (ret) {
 			gvt_vgpu_err("fail to translate guest gtt entry\n");
@@ -1881,13 +1945,14 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 			 * update the entry in this situation p2m will fail
 			 * settting the shadow entry to point to a scratch page
 			 */
-			ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn);
+			ops->set_pfn(&m, gvt->gtt.scratch_mfn);
 		}
 	} else {
 		m = e;
-		ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn);
+		ops->set_pfn(&m, gvt->gtt.scratch_mfn);
 	}
 
+out:
 	ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index);
 	gtt_invalidate(gvt->dev_priv);
 	ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index);
@@ -1920,12 +1985,45 @@ int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
 	return ret;
 }
 
+int intel_vgpu_write_protect_handler(struct intel_vgpu *vgpu, u64 pa,
+				     void *p_data, unsigned int bytes)
+{
+	struct intel_gvt *gvt = vgpu->gvt;
+	int ret = 0;
+
+	if (atomic_read(&vgpu->gtt.n_tracked_guest_page)) {
+		struct intel_vgpu_page_track *t;
+
+		mutex_lock(&gvt->lock);
+
+		t = intel_vgpu_find_tracked_page(vgpu, pa >> PAGE_SHIFT);
+		if (t) {
+			if (unlikely(vgpu->failsafe)) {
+				/* remove write protection to prevent furture traps */
+				intel_vgpu_clean_page_track(vgpu, t);
+			} else {
+				ret = t->handler(t, pa, p_data, bytes);
+				if (ret) {
+					gvt_err("guest page write error %d, "
+						"gfn 0x%lx, pa 0x%llx, "
+						"var 0x%x, len %d\n",
+						ret, t->gfn, pa,
+						*(u32 *)p_data, bytes);
+				}
+			}
+		}
+		mutex_unlock(&gvt->lock);
+	}
+	return ret;
+}
+
+
 static int alloc_scratch_pages(struct intel_vgpu *vgpu,
 		intel_gvt_gtt_type_t type)
 {
 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;
 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
-	int page_entry_num = GTT_PAGE_SIZE >>
+	int page_entry_num = I915_GTT_PAGE_SIZE >>
 				vgpu->gvt->device_info.gtt_entry_size_shift;
 	void *scratch_pt;
 	int i;
@@ -1949,7 +2047,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu,
 		return -ENOMEM;
 	}
 	gtt->scratch_pt[type].page_mfn =
-		(unsigned long)(daddr >> GTT_PAGE_SHIFT);
+		(unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);
 	gtt->scratch_pt[type].page = virt_to_page(scratch_pt);
 	gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n",
 			vgpu->id, type, gtt->scratch_pt[type].page_mfn);
@@ -1992,7 +2090,7 @@ static int release_scratch_page_tree(struct intel_vgpu *vgpu)
 	for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
 		if (vgpu->gtt.scratch_pt[i].page != NULL) {
 			daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<
-					GTT_PAGE_SHIFT);
+					I915_GTT_PAGE_SHIFT);
 			dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 			__free_page(vgpu->gtt.scratch_pt[i].page);
 			vgpu->gtt.scratch_pt[i].page = NULL;
@@ -2035,7 +2133,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;
 	struct intel_vgpu_mm *ggtt_mm;
 
-	hash_init(gtt->guest_page_hash_table);
+	hash_init(gtt->tracked_guest_page_hash_table);
 	hash_init(gtt->shadow_page_hash_table);
 
 	INIT_LIST_HEAD(&gtt->mm_list_head);
@@ -2196,7 +2294,7 @@ struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
 int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu,
 		int page_table_level)
 {
-	u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
+	u64 *pdp = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0]));
 	struct intel_vgpu_mm *mm;
 
 	if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
@@ -2231,7 +2329,7 @@ int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu,
 int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu,
 		int page_table_level)
 {
-	u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
+	u64 *pdp = (u64 *)&vgpu_vreg64_t(vgpu, vgtif_reg(pdp[0]));
 	struct intel_vgpu_mm *mm;
 
 	if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
@@ -2288,15 +2386,16 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
 		__free_page(virt_to_page(page));
 		return -ENOMEM;
 	}
-	gvt->gtt.scratch_ggtt_page = virt_to_page(page);
-	gvt->gtt.scratch_ggtt_mfn = (unsigned long)(daddr >> GTT_PAGE_SHIFT);
+
+	gvt->gtt.scratch_page = virt_to_page(page);
+	gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);
 
 	if (enable_out_of_sync) {
 		ret = setup_spt_oos(gvt);
 		if (ret) {
 			gvt_err("fail to initialize SPT oos\n");
 			dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
-			__free_page(gvt->gtt.scratch_ggtt_page);
+			__free_page(gvt->gtt.scratch_page);
 			return ret;
 		}
 	}
@@ -2315,12 +2414,12 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
 void intel_gvt_clean_gtt(struct intel_gvt *gvt)
 {
 	struct device *dev = &gvt->dev_priv->drm.pdev->dev;
-	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_ggtt_mfn <<
-					GTT_PAGE_SHIFT);
+	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<
+					I915_GTT_PAGE_SHIFT);
 
 	dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 
-	__free_page(gvt->gtt.scratch_ggtt_page);
+	__free_page(gvt->gtt.scratch_page);
 
 	if (enable_out_of_sync)
 		clean_spt_oos(gvt);
@@ -2346,7 +2445,7 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
 
 	memset(&e, 0, sizeof(struct intel_gvt_gtt_entry));
 	e.type = GTT_TYPE_GGTT_PTE;
-	ops->set_pfn(&e, gvt->gtt.scratch_ggtt_mfn);
+	ops->set_pfn(&e, gvt->gtt.scratch_mfn);
 	e.val64 |= _PAGE_PRESENT;
 
 	index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT;
@@ -2372,8 +2471,6 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
  */
 void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
 {
-	int i;
-
 	ppgtt_free_all_shadow_page(vgpu);
 
 	/* Shadow pages are only created when there is no page
@@ -2383,11 +2480,4 @@ void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
 	intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
 
 	intel_vgpu_reset_ggtt(vgpu);
-
-	/* clear scratch page for security */
-	for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
-		if (vgpu->gtt.scratch_pt[i].page != NULL)
-			memset(page_address(vgpu->gtt.scratch_pt[i].page),
-				0, PAGE_SIZE);
-	}
 }
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
index 30a4c8d16026..4cc13b5934f1 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.h
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -34,9 +34,8 @@
 #ifndef _GVT_GTT_H_
 #define _GVT_GTT_H_
 
-#define GTT_PAGE_SHIFT		12
-#define GTT_PAGE_SIZE		(1UL << GTT_PAGE_SHIFT)
-#define GTT_PAGE_MASK		(~(GTT_PAGE_SIZE-1))
+#define I915_GTT_PAGE_SHIFT         12
+#define I915_GTT_PAGE_MASK		(~(I915_GTT_PAGE_SIZE - 1))
 
 struct intel_vgpu_mm;
 
@@ -63,6 +62,7 @@ struct intel_gvt_gtt_pte_ops {
 			 struct intel_vgpu *vgpu);
 	bool (*test_present)(struct intel_gvt_gtt_entry *e);
 	void (*clear_present)(struct intel_gvt_gtt_entry *e);
+	void (*set_present)(struct intel_gvt_gtt_entry *e);
 	bool (*test_pse)(struct intel_gvt_gtt_entry *e);
 	void (*set_pfn)(struct intel_gvt_gtt_entry *e, unsigned long pfn);
 	unsigned long (*get_pfn)(struct intel_gvt_gtt_entry *e);
@@ -86,8 +86,8 @@ struct intel_gvt_gtt {
 	struct list_head oos_page_free_list_head;
 	struct list_head mm_lru_list_head;
 
-	struct page *scratch_ggtt_page;
-	unsigned long scratch_ggtt_mfn;
+	struct page *scratch_page;
+	unsigned long scratch_mfn;
 };
 
 enum {
@@ -193,18 +193,16 @@ struct intel_vgpu_scratch_pt {
 	unsigned long page_mfn;
 };
 
-
 struct intel_vgpu_gtt {
 	struct intel_vgpu_mm *ggtt_mm;
 	unsigned long active_ppgtt_mm_bitmap;
 	struct list_head mm_list_head;
 	DECLARE_HASHTABLE(shadow_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
-	DECLARE_HASHTABLE(guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
-	atomic_t n_write_protected_guest_page;
+	DECLARE_HASHTABLE(tracked_guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
+	atomic_t n_tracked_guest_page;
 	struct list_head oos_page_list_head;
 	struct list_head post_shadow_list_head;
 	struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX];
-
 };
 
 extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu);
@@ -228,12 +226,16 @@ struct intel_vgpu_shadow_page {
 	unsigned long mfn;
 };
 
-struct intel_vgpu_guest_page {
+struct intel_vgpu_page_track {
 	struct hlist_node node;
-	bool writeprotection;
+	bool tracked;
 	unsigned long gfn;
 	int (*handler)(void *, u64, void *, int);
 	void *data;
+};
+
+struct intel_vgpu_guest_page {
+	struct intel_vgpu_page_track track;
 	unsigned long write_cnt;
 	struct intel_vgpu_oos_page *oos_page;
 };
@@ -243,7 +245,7 @@ struct intel_vgpu_oos_page {
 	struct list_head list;
 	struct list_head vm_list;
 	int id;
-	unsigned char mem[GTT_PAGE_SIZE];
+	unsigned char mem[I915_GTT_PAGE_SIZE];
 };
 
 #define GTT_ENTRY_NUM_IN_ONE_PAGE 512
@@ -258,22 +260,16 @@ struct intel_vgpu_ppgtt_spt {
 	struct list_head post_shadow_list;
 };
 
-int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *guest_page,
+int intel_vgpu_init_page_track(struct intel_vgpu *vgpu,
+		struct intel_vgpu_page_track *t,
 		unsigned long gfn,
 		int (*handler)(void *gp, u64, void *, int),
 		void *data);
 
-void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *guest_page);
-
-int intel_vgpu_set_guest_page_writeprotection(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *guest_page);
+void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu,
+		struct intel_vgpu_page_track *t);
 
-void intel_vgpu_clear_guest_page_writeprotection(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *guest_page);
-
-struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+struct intel_vgpu_page_track *intel_vgpu_find_tracked_page(
 		struct intel_vgpu *vgpu, unsigned long gfn);
 
 int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu);
@@ -312,4 +308,7 @@ int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu,
 int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu,
 	unsigned int off, void *p_data, unsigned int bytes);
 
+int intel_vgpu_write_protect_handler(struct intel_vgpu *vgpu, u64 pa,
+				     void *p_data, unsigned int bytes);
+
 #endif /* _GVT_GTT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index aaa347f8620c..fac54f32d33f 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -36,6 +36,8 @@
 
 #include "i915_drv.h"
 #include "gvt.h"
+#include <linux/vfio.h>
+#include <linux/mdev.h>
 
 struct intel_gvt_host intel_gvt_host;
 
@@ -44,6 +46,129 @@ static const char * const supported_hypervisors[] = {
 	[INTEL_GVT_HYPERVISOR_KVM] = "KVM",
 };
 
+static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt,
+		const char *name)
+{
+	int i;
+	struct intel_vgpu_type *t;
+	const char *driver_name = dev_driver_string(
+			&gvt->dev_priv->drm.pdev->dev);
+
+	for (i = 0; i < gvt->num_types; i++) {
+		t = &gvt->types[i];
+		if (!strncmp(t->name, name + strlen(driver_name) + 1,
+			sizeof(t->name)))
+			return t;
+	}
+
+	return NULL;
+}
+
+static ssize_t available_instances_show(struct kobject *kobj,
+					struct device *dev, char *buf)
+{
+	struct intel_vgpu_type *type;
+	unsigned int num = 0;
+	void *gvt = kdev_to_i915(dev)->gvt;
+
+	type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+	if (!type)
+		num = 0;
+	else
+		num = type->avail_instance;
+
+	return sprintf(buf, "%u\n", num);
+}
+
+static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
+		char *buf)
+{
+	return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
+}
+
+static ssize_t description_show(struct kobject *kobj, struct device *dev,
+		char *buf)
+{
+	struct intel_vgpu_type *type;
+	void *gvt = kdev_to_i915(dev)->gvt;
+
+	type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+	if (!type)
+		return 0;
+
+	return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
+		       "fence: %d\nresolution: %s\n"
+		       "weight: %d\n",
+		       BYTES_TO_MB(type->low_gm_size),
+		       BYTES_TO_MB(type->high_gm_size),
+		       type->fence, vgpu_edid_str(type->resolution),
+		       type->weight);
+}
+
+static MDEV_TYPE_ATTR_RO(available_instances);
+static MDEV_TYPE_ATTR_RO(device_api);
+static MDEV_TYPE_ATTR_RO(description);
+
+static struct attribute *gvt_type_attrs[] = {
+	&mdev_type_attr_available_instances.attr,
+	&mdev_type_attr_device_api.attr,
+	&mdev_type_attr_description.attr,
+	NULL,
+};
+
+static struct attribute_group *gvt_vgpu_type_groups[] = {
+	[0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL,
+};
+
+static bool intel_get_gvt_attrs(struct attribute ***type_attrs,
+		struct attribute_group ***intel_vgpu_type_groups)
+{
+	*type_attrs = gvt_type_attrs;
+	*intel_vgpu_type_groups = gvt_vgpu_type_groups;
+	return true;
+}
+
+static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt)
+{
+	int i, j;
+	struct intel_vgpu_type *type;
+	struct attribute_group *group;
+
+	for (i = 0; i < gvt->num_types; i++) {
+		type = &gvt->types[i];
+
+		group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL);
+		if (WARN_ON(!group))
+			goto unwind;
+
+		group->name = type->name;
+		group->attrs = gvt_type_attrs;
+		gvt_vgpu_type_groups[i] = group;
+	}
+
+	return true;
+
+unwind:
+	for (j = 0; j < i; j++) {
+		group = gvt_vgpu_type_groups[j];
+		kfree(group);
+	}
+
+	return false;
+}
+
+static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt)
+{
+	int i;
+	struct attribute_group *group;
+
+	for (i = 0; i < gvt->num_types; i++) {
+		group = gvt_vgpu_type_groups[i];
+		gvt_vgpu_type_groups[i] = NULL;
+		kfree(group);
+	}
+}
+
 static const struct intel_gvt_ops intel_gvt_ops = {
 	.emulate_cfg_read = intel_vgpu_emulate_cfg_read,
 	.emulate_cfg_write = intel_vgpu_emulate_cfg_write,
@@ -54,6 +179,11 @@ static const struct intel_gvt_ops intel_gvt_ops = {
 	.vgpu_reset = intel_gvt_reset_vgpu,
 	.vgpu_activate = intel_gvt_activate_vgpu,
 	.vgpu_deactivate = intel_gvt_deactivate_vgpu,
+	.gvt_find_vgpu_type = intel_gvt_find_vgpu_type,
+	.get_gvt_attrs = intel_get_gvt_attrs,
+	.vgpu_query_plane = intel_vgpu_query_plane,
+	.vgpu_get_dmabuf = intel_vgpu_get_dmabuf,
+	.write_protect_handler = intel_vgpu_write_protect_handler,
 };
 
 /**
@@ -191,17 +321,18 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
 	if (WARN_ON(!gvt))
 		return;
 
+	intel_gvt_debugfs_clean(gvt);
 	clean_service_thread(gvt);
 	intel_gvt_clean_cmd_parser(gvt);
 	intel_gvt_clean_sched_policy(gvt);
 	intel_gvt_clean_workload_scheduler(gvt);
-	intel_gvt_clean_opregion(gvt);
 	intel_gvt_clean_gtt(gvt);
 	intel_gvt_clean_irq(gvt);
 	intel_gvt_clean_mmio_info(gvt);
 	intel_gvt_free_firmware(gvt);
 
 	intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt);
+	intel_gvt_cleanup_vgpu_type_groups(gvt);
 	intel_gvt_clean_vgpu_types(gvt);
 
 	idr_destroy(&gvt->vgpu_idr);
@@ -256,6 +387,8 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 	if (ret)
 		goto out_clean_idr;
 
+	intel_gvt_init_engine_mmio_context(gvt);
+
 	ret = intel_gvt_load_firmware(gvt);
 	if (ret)
 		goto out_clean_mmio_info;
@@ -268,13 +401,9 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 	if (ret)
 		goto out_clean_irq;
 
-	ret = intel_gvt_init_opregion(gvt);
-	if (ret)
-		goto out_clean_gtt;
-
 	ret = intel_gvt_init_workload_scheduler(gvt);
 	if (ret)
-		goto out_clean_opregion;
+		goto out_clean_gtt;
 
 	ret = intel_gvt_init_sched_policy(gvt);
 	if (ret)
@@ -292,6 +421,12 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 	if (ret)
 		goto out_clean_thread;
 
+	ret = intel_gvt_init_vgpu_type_groups(gvt);
+	if (ret == false) {
+		gvt_err("failed to init vgpu type groups: %d\n", ret);
+		goto out_clean_types;
+	}
+
 	ret = intel_gvt_hypervisor_host_init(&dev_priv->drm.pdev->dev, gvt,
 				&intel_gvt_ops);
 	if (ret) {
@@ -307,6 +442,10 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 	}
 	gvt->idle_vgpu = vgpu;
 
+	ret = intel_gvt_debugfs_init(gvt);
+	if (ret)
+		gvt_err("debugfs registeration failed, go on.\n");
+
 	gvt_dbg_core("gvt device initialization is done\n");
 	dev_priv->gvt = gvt;
 	return 0;
@@ -321,8 +460,6 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
 	intel_gvt_clean_sched_policy(gvt);
 out_clean_workload_scheduler:
 	intel_gvt_clean_workload_scheduler(gvt);
-out_clean_opregion:
-	intel_gvt_clean_opregion(gvt);
 out_clean_gtt:
 	intel_gvt_clean_gtt(gvt);
 out_clean_irq:
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 9c2e7c0aa38f..c6197d990818 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -44,8 +44,10 @@
 #include "execlist.h"
 #include "scheduler.h"
 #include "sched_policy.h"
-#include "render.h"
+#include "mmio_context.h"
 #include "cmd_parser.h"
+#include "fb_decoder.h"
+#include "dmabuf.h"
 
 #define GVT_MAX_VGPU 8
 
@@ -80,7 +82,6 @@ struct intel_gvt_device_info {
 struct intel_vgpu_gm {
 	u64 aperture_sz;
 	u64 hidden_sz;
-	void *aperture_va;
 	struct drm_mm_node low_gm_node;
 	struct drm_mm_node high_gm_node;
 };
@@ -123,9 +124,9 @@ struct intel_vgpu_irq {
 };
 
 struct intel_vgpu_opregion {
+	bool mapped;
 	void *va;
 	u32 gfn[INTEL_GVT_OPREGION_PAGES];
-	struct page *pages[INTEL_GVT_OPREGION_PAGES];
 };
 
 #define vgpu_opregion(vgpu) (&(vgpu->opregion))
@@ -142,6 +143,33 @@ struct vgpu_sched_ctl {
 	int weight;
 };
 
+enum {
+	INTEL_VGPU_EXECLIST_SUBMISSION = 1,
+	INTEL_VGPU_GUC_SUBMISSION,
+};
+
+struct intel_vgpu_submission_ops {
+	const char *name;
+	int (*init)(struct intel_vgpu *vgpu, unsigned long engine_mask);
+	void (*clean)(struct intel_vgpu *vgpu, unsigned long engine_mask);
+	void (*reset)(struct intel_vgpu *vgpu, unsigned long engine_mask);
+};
+
+struct intel_vgpu_submission {
+	struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
+	struct list_head workload_q_head[I915_NUM_ENGINES];
+	struct kmem_cache *workloads;
+	atomic_t running_workload_num;
+	struct i915_gem_context *shadow_ctx;
+	DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
+	DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
+	void *ring_scan_buffer[I915_NUM_ENGINES];
+	int ring_scan_buffer_size[I915_NUM_ENGINES];
+	const struct intel_vgpu_submission_ops *ops;
+	int virtual_submission_interface;
+	bool active;
+};
+
 struct intel_vgpu {
 	struct intel_gvt *gvt;
 	int id;
@@ -161,16 +189,10 @@ struct intel_vgpu {
 	struct intel_vgpu_gtt gtt;
 	struct intel_vgpu_opregion opregion;
 	struct intel_vgpu_display display;
-	struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
-	struct list_head workload_q_head[I915_NUM_ENGINES];
-	struct kmem_cache *workloads;
-	atomic_t running_workload_num;
-	/* 1/2K for each reserve ring buffer */
-	void *reserve_ring_buffer_va[I915_NUM_ENGINES];
-	int reserve_ring_buffer_size[I915_NUM_ENGINES];
-	DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
-	struct i915_gem_context *shadow_ctx;
-	DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
+	struct intel_vgpu_submission submission;
+	u32 hws_pga[I915_NUM_ENGINES];
+
+	struct dentry *debugfs;
 
 #if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
 	struct {
@@ -186,10 +208,22 @@ struct intel_vgpu {
 		struct kvm *kvm;
 		struct work_struct release_work;
 		atomic_t released;
+		struct vfio_device *vfio_device;
 	} vdev;
 #endif
+
+	struct list_head dmabuf_obj_list_head;
+	struct mutex dmabuf_lock;
+	struct idr object_idr;
+
+	struct completion vblank_done;
+
 };
 
+/* validating GM healthy status*/
+#define vgpu_is_vm_unhealthy(ret_val) \
+	(((ret_val) == -EBADRQC) || ((ret_val) == -EFAULT))
+
 struct intel_gvt_gm {
 	unsigned long vgpu_allocated_low_gm_size;
 	unsigned long vgpu_allocated_high_gm_size;
@@ -231,7 +265,7 @@ struct intel_gvt_mmio {
 	unsigned int num_mmio_block;
 
 	DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS);
-	unsigned int num_tracked_mmio;
+	unsigned long num_tracked_mmio;
 };
 
 struct intel_gvt_firmware {
@@ -240,11 +274,6 @@ struct intel_gvt_firmware {
 	bool firmware_loaded;
 };
 
-struct intel_gvt_opregion {
-	void *opregion_va;
-	u32 opregion_pa;
-};
-
 #define NR_MAX_INTEL_VGPU_TYPES 20
 struct intel_vgpu_type {
 	char name[16];
@@ -268,7 +297,6 @@ struct intel_gvt {
 	struct intel_gvt_firmware firmware;
 	struct intel_gvt_irq irq;
 	struct intel_gvt_gtt gtt;
-	struct intel_gvt_opregion opregion;
 	struct intel_gvt_workload_scheduler scheduler;
 	struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES];
 	DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
@@ -279,6 +307,10 @@ struct intel_gvt {
 	struct task_struct *service_thread;
 	wait_queue_head_t service_thread_wq;
 	unsigned long service_request;
+
+	struct engine_mmio *engine_mmio_list;
+
+	struct dentry *debugfs_root;
 };
 
 static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915)
@@ -316,7 +348,7 @@ int intel_gvt_load_firmware(struct intel_gvt *gvt);
 
 /* Aperture/GM space definitions for GVT device */
 #define gvt_aperture_sz(gvt)	  (gvt->dev_priv->ggtt.mappable_end)
-#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.mappable_base)
+#define gvt_aperture_pa_base(gvt) (gvt->dev_priv->ggtt.gmadr.start)
 
 #define gvt_ggtt_gm_sz(gvt)	  (gvt->dev_priv->ggtt.base.total)
 #define gvt_ggtt_sz(gvt) \
@@ -378,23 +410,20 @@ void intel_vgpu_free_resource(struct intel_vgpu *vgpu);
 void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
 	u32 fence, u64 value);
 
-/* Macros for easily accessing vGPU virtual/shadow register */
-#define vgpu_vreg(vgpu, reg) \
-	(*(u32 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
-#define vgpu_vreg8(vgpu, reg) \
-	(*(u8 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
-#define vgpu_vreg16(vgpu, reg) \
-	(*(u16 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
-#define vgpu_vreg64(vgpu, reg) \
-	(*(u64 *)(vgpu->mmio.vreg + INTEL_GVT_MMIO_OFFSET(reg)))
-#define vgpu_sreg(vgpu, reg) \
-	(*(u32 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
-#define vgpu_sreg8(vgpu, reg) \
-	(*(u8 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
-#define vgpu_sreg16(vgpu, reg) \
-	(*(u16 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
-#define vgpu_sreg64(vgpu, reg) \
-	(*(u64 *)(vgpu->mmio.sreg + INTEL_GVT_MMIO_OFFSET(reg)))
+/* Macros for easily accessing vGPU virtual/shadow register.
+   Explicitly seperate use for typed MMIO reg or real offset.*/
+#define vgpu_vreg_t(vgpu, reg) \
+	(*(u32 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg)))
+#define vgpu_vreg(vgpu, offset) \
+	(*(u32 *)(vgpu->mmio.vreg + (offset)))
+#define vgpu_vreg64_t(vgpu, reg) \
+	(*(u64 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg)))
+#define vgpu_vreg64(vgpu, offset) \
+	(*(u64 *)(vgpu->mmio.vreg + (offset)))
+#define vgpu_sreg_t(vgpu, reg) \
+	(*(u32 *)(vgpu->mmio.sreg + i915_mmio_reg_offset(reg)))
+#define vgpu_sreg(vgpu, offset) \
+	(*(u32 *)(vgpu->mmio.sreg + (offset)))
 
 #define for_each_active_vgpu(gvt, vgpu, id) \
 	idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \
@@ -484,16 +513,15 @@ static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar)
 			PCI_BASE_ADDRESS_MEM_MASK;
 }
 
-void intel_gvt_clean_opregion(struct intel_gvt *gvt);
-int intel_gvt_init_opregion(struct intel_gvt *gvt);
-
 void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu);
-int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa);
+int intel_vgpu_init_opregion(struct intel_vgpu *vgpu);
+int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa);
 
 int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
 void populate_pvinfo_page(struct intel_vgpu *vgpu);
 
 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
+void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason);
 
 struct intel_gvt_ops {
 	int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *,
@@ -510,12 +538,21 @@ struct intel_gvt_ops {
 	void (*vgpu_reset)(struct intel_vgpu *);
 	void (*vgpu_activate)(struct intel_vgpu *);
 	void (*vgpu_deactivate)(struct intel_vgpu *);
+	struct intel_vgpu_type *(*gvt_find_vgpu_type)(struct intel_gvt *gvt,
+			const char *name);
+	bool (*get_gvt_attrs)(struct attribute ***type_attrs,
+			struct attribute_group ***intel_vgpu_type_groups);
+	int (*vgpu_query_plane)(struct intel_vgpu *vgpu, void *);
+	int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int);
+	int (*write_protect_handler)(struct intel_vgpu *, u64, void *,
+				     unsigned int);
 };
 
 
 enum {
 	GVT_FAILSAFE_UNSUPPORTED_GUEST,
 	GVT_FAILSAFE_INSUFFICIENT_RESOURCE,
+	GVT_FAILSAFE_GUEST_ERR,
 };
 
 static inline void mmio_hw_access_pre(struct drm_i915_private *dev_priv)
@@ -591,6 +628,12 @@ static inline bool intel_gvt_mmio_has_mode_mask(
 	return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK;
 }
 
+int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu);
+void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu);
+int intel_gvt_debugfs_init(struct intel_gvt *gvt);
+void intel_gvt_debugfs_clean(struct intel_gvt *gvt);
+
+
 #include "trace.h"
 #include "mpt.h"
 
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 1f840f6b81bb..9be639aa3b55 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -166,7 +166,7 @@ int intel_gvt_render_mmio_to_ring_id(struct intel_gvt *gvt,
 	(num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0)))
 
 
-static void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
+void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
 {
 	switch (reason) {
 	case GVT_FAILSAFE_UNSUPPORTED_GUEST:
@@ -174,6 +174,10 @@ static void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
 		break;
 	case GVT_FAILSAFE_INSUFFICIENT_RESOURCE:
 		pr_err("Graphics resource is not enough for the guest\n");
+		break;
+	case GVT_FAILSAFE_GUEST_ERR:
+		pr_err("GVT Internal error  for the guest\n");
+		break;
 	default:
 		break;
 	}
@@ -339,13 +343,13 @@ static int pch_pp_control_mmio_write(struct intel_vgpu *vgpu,
 	write_vreg(vgpu, offset, p_data, bytes);
 
 	if (vgpu_vreg(vgpu, offset) & PANEL_POWER_ON) {
-		vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_ON;
-		vgpu_vreg(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE;
-		vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN;
-		vgpu_vreg(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE;
+		vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_ON;
+		vgpu_vreg_t(vgpu, PCH_PP_STATUS) |= PP_SEQUENCE_STATE_ON_IDLE;
+		vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_SEQUENCE_POWER_DOWN;
+		vgpu_vreg_t(vgpu, PCH_PP_STATUS) &= ~PP_CYCLE_DELAY_ACTIVE;
 
 	} else
-		vgpu_vreg(vgpu, PCH_PP_STATUS) &=
+		vgpu_vreg_t(vgpu, PCH_PP_STATUS) &=
 			~(PP_ON | PP_SEQUENCE_POWER_DOWN
 					| PP_CYCLE_DELAY_ACTIVE);
 	return 0;
@@ -499,7 +503,7 @@ static int ddi_buf_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	} else {
 		vgpu_vreg(vgpu, offset) |= DDI_BUF_IS_IDLE;
 		if (offset == i915_mmio_reg_offset(DDI_BUF_CTL(PORT_E)))
-			vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E))
+			vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E))
 				&= ~DP_TP_STATUS_AUTOTRAIN_DONE;
 	}
 	return 0;
@@ -517,9 +521,9 @@ static int fdi_rx_iir_mmio_write(struct intel_vgpu *vgpu,
 
 static int fdi_auto_training_started(struct intel_vgpu *vgpu)
 {
-	u32 ddi_buf_ctl = vgpu_vreg(vgpu, DDI_BUF_CTL(PORT_E));
+	u32 ddi_buf_ctl = vgpu_vreg_t(vgpu, DDI_BUF_CTL(PORT_E));
 	u32 rx_ctl = vgpu_vreg(vgpu, _FDI_RXA_CTL);
-	u32 tx_ctl = vgpu_vreg(vgpu, DP_TP_CTL(PORT_E));
+	u32 tx_ctl = vgpu_vreg_t(vgpu, DP_TP_CTL(PORT_E));
 
 	if ((ddi_buf_ctl & DDI_BUF_CTL_ENABLE) &&
 			(rx_ctl & FDI_RX_ENABLE) &&
@@ -560,12 +564,12 @@ static int check_fdi_rx_train_status(struct intel_vgpu *vgpu,
 	fdi_tx_check_bits = FDI_TX_ENABLE | fdi_tx_train_bits;
 
 	/* If imr bit has been masked */
-	if (vgpu_vreg(vgpu, fdi_rx_imr) & fdi_iir_check_bits)
+	if (vgpu_vreg_t(vgpu, fdi_rx_imr) & fdi_iir_check_bits)
 		return 0;
 
-	if (((vgpu_vreg(vgpu, fdi_tx_ctl) & fdi_tx_check_bits)
+	if (((vgpu_vreg_t(vgpu, fdi_tx_ctl) & fdi_tx_check_bits)
 			== fdi_tx_check_bits)
-		&& ((vgpu_vreg(vgpu, fdi_rx_ctl) & fdi_rx_check_bits)
+		&& ((vgpu_vreg_t(vgpu, fdi_rx_ctl) & fdi_rx_check_bits)
 			== fdi_rx_check_bits))
 		return 1;
 	else
@@ -622,17 +626,17 @@ static int update_fdi_rx_iir_status(struct intel_vgpu *vgpu,
 	if (ret < 0)
 		return ret;
 	if (ret)
-		vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK;
+		vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_BIT_LOCK;
 
 	ret = check_fdi_rx_train_status(vgpu, index, FDI_LINK_TRAIN_PATTERN2);
 	if (ret < 0)
 		return ret;
 	if (ret)
-		vgpu_vreg(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK;
+		vgpu_vreg_t(vgpu, fdi_rx_iir) |= FDI_RX_SYMBOL_LOCK;
 
 	if (offset == _FDI_RXA_CTL)
 		if (fdi_auto_training_started(vgpu))
-			vgpu_vreg(vgpu, DP_TP_STATUS(PORT_E)) |=
+			vgpu_vreg_t(vgpu, DP_TP_STATUS(PORT_E)) |=
 				DP_TP_STATUS_AUTOTRAIN_DONE;
 	return 0;
 }
@@ -653,7 +657,7 @@ static int dp_tp_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	data = (vgpu_vreg(vgpu, offset) & GENMASK(10, 8)) >> 8;
 	if (data == 0x2) {
 		status_reg = DP_TP_STATUS(index);
-		vgpu_vreg(vgpu, status_reg) |= (1 << 25);
+		vgpu_vreg_t(vgpu, status_reg) |= (1 << 25);
 	}
 	return 0;
 }
@@ -717,7 +721,7 @@ static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	};
 
 	write_vreg(vgpu, offset, p_data, bytes);
-	vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+	vgpu_vreg_t(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
 
 	set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
 	return 0;
@@ -738,7 +742,7 @@ static int spr_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	};
 
 	write_vreg(vgpu, offset, p_data, bytes);
-	vgpu_vreg(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+	vgpu_vreg_t(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
 
 	set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
 	return 0;
@@ -1060,9 +1064,9 @@ static void write_virtual_sbi_register(struct intel_vgpu *vgpu,
 static int sbi_data_mmio_read(struct intel_vgpu *vgpu, unsigned int offset,
 		void *p_data, unsigned int bytes)
 {
-	if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
+	if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
 				SBI_OPCODE_SHIFT) == SBI_CMD_CRRD) {
-		unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) &
+		unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) &
 				SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT;
 		vgpu_vreg(vgpu, offset) = read_virtual_sbi_register(vgpu,
 				sbi_offset);
@@ -1087,13 +1091,13 @@ static int sbi_ctl_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 
 	vgpu_vreg(vgpu, offset) = data;
 
-	if (((vgpu_vreg(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
+	if (((vgpu_vreg_t(vgpu, SBI_CTL_STAT) & SBI_OPCODE_MASK) >>
 				SBI_OPCODE_SHIFT) == SBI_CMD_CRWR) {
-		unsigned int sbi_offset = (vgpu_vreg(vgpu, SBI_ADDR) &
+		unsigned int sbi_offset = (vgpu_vreg_t(vgpu, SBI_ADDR) &
 				SBI_ADDR_OFFSET_MASK) >> SBI_ADDR_OFFSET_SHIFT;
 
 		write_virtual_sbi_register(vgpu, sbi_offset,
-				vgpu_vreg(vgpu, SBI_DATA));
+					   vgpu_vreg_t(vgpu, SBI_DATA));
 	}
 	return 0;
 }
@@ -1339,7 +1343,7 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset,
 {
 	u32 value = *(u32 *)p_data;
 	u32 cmd = value & 0xff;
-	u32 *data0 = &vgpu_vreg(vgpu, GEN6_PCODE_DATA);
+	u32 *data0 = &vgpu_vreg_t(vgpu, GEN6_PCODE_DATA);
 
 	switch (cmd) {
 	case GEN9_PCODE_READ_MEM_LATENCY:
@@ -1378,6 +1382,34 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset,
 	return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes);
 }
 
+static int hws_pga_write(struct intel_vgpu *vgpu, unsigned int offset,
+		void *p_data, unsigned int bytes)
+{
+	u32 value = *(u32 *)p_data;
+	int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
+
+	if (!intel_gvt_ggtt_validate_range(vgpu, value, I915_GTT_PAGE_SIZE)) {
+		gvt_vgpu_err("VM(%d) write invalid HWSP address, reg:0x%x, value:0x%x\n",
+			      vgpu->id, offset, value);
+		return -EINVAL;
+	}
+	/*
+	 * Need to emulate all the HWSP register write to ensure host can
+	 * update the VM CSB status correctly. Here listed registers can
+	 * support BDW, SKL or other platforms with same HWSP registers.
+	 */
+	if (unlikely(ring_id < 0 || ring_id >= I915_NUM_ENGINES)) {
+		gvt_vgpu_err("VM(%d) access unknown hardware status page register:0x%x\n",
+			     vgpu->id, offset);
+		return -EINVAL;
+	}
+	vgpu->hws_pga[ring_id] = value;
+	gvt_dbg_mmio("VM(%d) write: 0x%x to HWSP: 0x%x\n",
+		     vgpu->id, value, offset);
+
+	return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes);
+}
+
 static int skl_power_well_ctl_write(struct intel_vgpu *vgpu,
 		unsigned int offset, void *p_data, unsigned int bytes)
 {
@@ -1441,12 +1473,12 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	u32 data = *(u32 *)p_data;
 	int ret = 0;
 
-	if (WARN_ON(ring_id < 0 || ring_id > I915_NUM_ENGINES - 1))
+	if (WARN_ON(ring_id < 0 || ring_id >= I915_NUM_ENGINES))
 		return -EINVAL;
 
-	execlist = &vgpu->execlist[ring_id];
+	execlist = &vgpu->submission.execlist[ring_id];
 
-	execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data;
+	execlist->elsp_dwords.data[3 - execlist->elsp_dwords.index] = data;
 	if (execlist->elsp_dwords.index == 3) {
 		ret = intel_vgpu_submit_execlist(vgpu, ring_id);
 		if(ret)
@@ -1465,6 +1497,7 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	u32 data = *(u32 *)p_data;
 	int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
 	bool enable_execlist;
+	int ret;
 
 	write_vreg(vgpu, offset, p_data, bytes);
 
@@ -1486,8 +1519,16 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 				(enable_execlist ? "enabling" : "disabling"),
 				ring_id);
 
-		if (enable_execlist)
-			intel_vgpu_start_schedule(vgpu);
+		if (!enable_execlist)
+			return 0;
+
+		ret = intel_vgpu_select_submission_ops(vgpu,
+			       ENGINE_MASK(ring_id),
+			       INTEL_VGPU_EXECLIST_SUBMISSION);
+		if (ret)
+			return ret;
+
+		intel_vgpu_start_schedule(vgpu);
 	}
 	return 0;
 }
@@ -1519,7 +1560,7 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
 	default:
 		return -EINVAL;
 	}
-	set_bit(id, (void *)vgpu->tlb_handle_pending);
+	set_bit(id, (void *)vgpu->submission.tlb_handle_pending);
 
 	return 0;
 }
@@ -1542,7 +1583,7 @@ static int ring_reset_ctl_write(struct intel_vgpu *vgpu,
 }
 
 #define MMIO_F(reg, s, f, am, rm, d, r, w) do { \
-	ret = new_mmio_info(gvt, INTEL_GVT_MMIO_OFFSET(reg), \
+	ret = new_mmio_info(gvt, i915_mmio_reg_offset(reg), \
 		f, s, am, rm, d, r, w); \
 	if (ret) \
 		return ret; \
@@ -1610,22 +1651,22 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_GM_RDR(BLT_HWS_PGA_GEN7, D_ALL, NULL, NULL);
 	MMIO_GM_RDR(VEBOX_HWS_PGA_GEN7, D_ALL, NULL, NULL);
 
-#define RING_REG(base) (base + 0x28)
+#define RING_REG(base) _MMIO((base) + 0x28)
 	MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL);
 #undef RING_REG
 
-#define RING_REG(base) (base + 0x134)
+#define RING_REG(base) _MMIO((base) + 0x134)
 	MMIO_RING_DFH(RING_REG, D_ALL, F_CMD_ACCESS, NULL, NULL);
 #undef RING_REG
 
-#define RING_REG(base) (base + 0x6c)
+#define RING_REG(base) _MMIO((base) + 0x6c)
 	MMIO_RING_DFH(RING_REG, D_ALL, 0, mmio_read_from_hw, NULL);
 #undef RING_REG
 	MMIO_DH(GEN7_SC_INSTDONE, D_BDW_PLUS, mmio_read_from_hw, NULL);
 
-	MMIO_GM_RDR(0x2148, D_ALL, NULL, NULL);
+	MMIO_GM_RDR(_MMIO(0x2148), D_ALL, NULL, NULL);
 	MMIO_GM_RDR(CCID, D_ALL, NULL, NULL);
-	MMIO_GM_RDR(0x12198, D_ALL, NULL, NULL);
+	MMIO_GM_RDR(_MMIO(0x12198), D_ALL, NULL, NULL);
 	MMIO_D(GEN7_CXT_SIZE, D_ALL);
 
 	MMIO_RING_DFH(RING_TAIL, D_ALL, F_CMD_ACCESS, NULL, NULL);
@@ -1635,7 +1676,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_RING_GM_RDR(RING_START, D_ALL, NULL, NULL);
 
 	/* RING MODE */
-#define RING_REG(base) (base + 0x29c)
+#define RING_REG(base) _MMIO((base) + 0x29c)
 	MMIO_RING_DFH(RING_REG, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL,
 		ring_mode_mmio_write);
 #undef RING_REG
@@ -1654,37 +1695,37 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 		NULL, NULL);
 	MMIO_DFH(CACHE_MODE_1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(CACHE_MODE_0, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2124, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2124), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 
-	MMIO_DFH(0x20dc, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x20dc), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(_3D_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2088, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x20e4, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2470, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2088), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x20e4), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2470), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(GAM_ECOCHK, D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(GEN7_COMMON_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS,
 		NULL, NULL);
 	MMIO_DFH(COMMON_SLICE_CHICKEN2, D_ALL, F_MODE_MASK | F_CMD_ACCESS,
 		 NULL, NULL);
-	MMIO_DFH(0x9030, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x20a0, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2420, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2430, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2434, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2438, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x243c, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x7018, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x9030), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x20a0), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2420), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2430), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2434), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2438), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x243c), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x7018), D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(HALF_SLICE_CHICKEN3, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(GEN7_HALF_SLICE_CHICKEN1, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 
 	/* display */
-	MMIO_F(0x60220, 0x20, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_D(0x602a0, D_ALL);
+	MMIO_F(_MMIO(0x60220), 0x20, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_D(_MMIO(0x602a0), D_ALL);
 
-	MMIO_D(0x65050, D_ALL);
-	MMIO_D(0x650b4, D_ALL);
+	MMIO_D(_MMIO(0x65050), D_ALL);
+	MMIO_D(_MMIO(0x650b4), D_ALL);
 
-	MMIO_D(0xc4040, D_ALL);
+	MMIO_D(_MMIO(0xc4040), D_ALL);
 	MMIO_D(DERRMR, D_ALL);
 
 	MMIO_D(PIPEDSL(PIPE_A), D_ALL);
@@ -1724,14 +1765,14 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(CURBASE(PIPE_B), D_ALL);
 	MMIO_D(CURBASE(PIPE_C), D_ALL);
 
-	MMIO_D(0x700ac, D_ALL);
-	MMIO_D(0x710ac, D_ALL);
-	MMIO_D(0x720ac, D_ALL);
+	MMIO_D(_MMIO(0x700ac), D_ALL);
+	MMIO_D(_MMIO(0x710ac), D_ALL);
+	MMIO_D(_MMIO(0x720ac), D_ALL);
 
-	MMIO_D(0x70090, D_ALL);
-	MMIO_D(0x70094, D_ALL);
-	MMIO_D(0x70098, D_ALL);
-	MMIO_D(0x7009c, D_ALL);
+	MMIO_D(_MMIO(0x70090), D_ALL);
+	MMIO_D(_MMIO(0x70094), D_ALL);
+	MMIO_D(_MMIO(0x70098), D_ALL);
+	MMIO_D(_MMIO(0x7009c), D_ALL);
 
 	MMIO_D(DSPCNTR(PIPE_A), D_ALL);
 	MMIO_D(DSPADDR(PIPE_A), D_ALL);
@@ -1907,24 +1948,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(BLC_PWM_PCH_CTL1, D_ALL);
 	MMIO_D(BLC_PWM_PCH_CTL2, D_ALL);
 
-	MMIO_D(0x48268, D_ALL);
+	MMIO_D(_MMIO(0x48268), D_ALL);
 
 	MMIO_F(PCH_GMBUS0, 4 * 4, 0, 0, 0, D_ALL, gmbus_mmio_read,
 		gmbus_mmio_write);
 	MMIO_F(PCH_GPIOA, 6 * 4, F_UNALIGN, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0xe4f00, 0x28, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0xe4f00), 0x28, 0, 0, 0, D_ALL, NULL, NULL);
 
-	MMIO_F(_PCH_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+	MMIO_F(_MMIO(_PCH_DPB_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
 		dp_aux_ch_ctl_mmio_write);
-	MMIO_F(_PCH_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+	MMIO_F(_MMIO(_PCH_DPC_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
 		dp_aux_ch_ctl_mmio_write);
-	MMIO_F(_PCH_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
+	MMIO_F(_MMIO(_PCH_DPD_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_PRE_SKL, NULL,
 		dp_aux_ch_ctl_mmio_write);
 
 	MMIO_DH(PCH_ADPA, D_PRE_SKL, NULL, pch_adpa_mmio_write);
 
-	MMIO_DH(_PCH_TRANSACONF, D_ALL, NULL, transconf_mmio_write);
-	MMIO_DH(_PCH_TRANSBCONF, D_ALL, NULL, transconf_mmio_write);
+	MMIO_DH(_MMIO(_PCH_TRANSACONF), D_ALL, NULL, transconf_mmio_write);
+	MMIO_DH(_MMIO(_PCH_TRANSBCONF), D_ALL, NULL, transconf_mmio_write);
 
 	MMIO_DH(FDI_RX_IIR(PIPE_A), D_ALL, NULL, fdi_rx_iir_mmio_write);
 	MMIO_DH(FDI_RX_IIR(PIPE_B), D_ALL, NULL, fdi_rx_iir_mmio_write);
@@ -1936,30 +1977,30 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(FDI_RX_CTL(PIPE_B), D_ALL, NULL, update_fdi_rx_iir_status);
 	MMIO_DH(FDI_RX_CTL(PIPE_C), D_ALL, NULL, update_fdi_rx_iir_status);
 
-	MMIO_D(_PCH_TRANS_HTOTAL_A, D_ALL);
-	MMIO_D(_PCH_TRANS_HBLANK_A, D_ALL);
-	MMIO_D(_PCH_TRANS_HSYNC_A, D_ALL);
-	MMIO_D(_PCH_TRANS_VTOTAL_A, D_ALL);
-	MMIO_D(_PCH_TRANS_VBLANK_A, D_ALL);
-	MMIO_D(_PCH_TRANS_VSYNC_A, D_ALL);
-	MMIO_D(_PCH_TRANS_VSYNCSHIFT_A, D_ALL);
-
-	MMIO_D(_PCH_TRANS_HTOTAL_B, D_ALL);
-	MMIO_D(_PCH_TRANS_HBLANK_B, D_ALL);
-	MMIO_D(_PCH_TRANS_HSYNC_B, D_ALL);
-	MMIO_D(_PCH_TRANS_VTOTAL_B, D_ALL);
-	MMIO_D(_PCH_TRANS_VBLANK_B, D_ALL);
-	MMIO_D(_PCH_TRANS_VSYNC_B, D_ALL);
-	MMIO_D(_PCH_TRANS_VSYNCSHIFT_B, D_ALL);
-
-	MMIO_D(_PCH_TRANSA_DATA_M1, D_ALL);
-	MMIO_D(_PCH_TRANSA_DATA_N1, D_ALL);
-	MMIO_D(_PCH_TRANSA_DATA_M2, D_ALL);
-	MMIO_D(_PCH_TRANSA_DATA_N2, D_ALL);
-	MMIO_D(_PCH_TRANSA_LINK_M1, D_ALL);
-	MMIO_D(_PCH_TRANSA_LINK_N1, D_ALL);
-	MMIO_D(_PCH_TRANSA_LINK_M2, D_ALL);
-	MMIO_D(_PCH_TRANSA_LINK_N2, D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_HTOTAL_A), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_HBLANK_A), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_HSYNC_A), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VTOTAL_A), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VBLANK_A), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VSYNC_A), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VSYNCSHIFT_A), D_ALL);
+
+	MMIO_D(_MMIO(_PCH_TRANS_HTOTAL_B), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_HBLANK_B), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_HSYNC_B), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VTOTAL_B), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VBLANK_B), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VSYNC_B), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANS_VSYNCSHIFT_B), D_ALL);
+
+	MMIO_D(_MMIO(_PCH_TRANSA_DATA_M1), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANSA_DATA_N1), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANSA_DATA_M2), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANSA_DATA_N2), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANSA_LINK_M1), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANSA_LINK_N1), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANSA_LINK_M2), D_ALL);
+	MMIO_D(_MMIO(_PCH_TRANSA_LINK_N2), D_ALL);
 
 	MMIO_D(TRANS_DP_CTL(PIPE_A), D_ALL);
 	MMIO_D(TRANS_DP_CTL(PIPE_B), D_ALL);
@@ -1977,38 +2018,38 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(TVIDEO_DIP_DATA(PIPE_C), D_ALL);
 	MMIO_D(TVIDEO_DIP_GCP(PIPE_C), D_ALL);
 
-	MMIO_D(_FDI_RXA_MISC, D_ALL);
-	MMIO_D(_FDI_RXB_MISC, D_ALL);
-	MMIO_D(_FDI_RXA_TUSIZE1, D_ALL);
-	MMIO_D(_FDI_RXA_TUSIZE2, D_ALL);
-	MMIO_D(_FDI_RXB_TUSIZE1, D_ALL);
-	MMIO_D(_FDI_RXB_TUSIZE2, D_ALL);
+	MMIO_D(_MMIO(_FDI_RXA_MISC), D_ALL);
+	MMIO_D(_MMIO(_FDI_RXB_MISC), D_ALL);
+	MMIO_D(_MMIO(_FDI_RXA_TUSIZE1), D_ALL);
+	MMIO_D(_MMIO(_FDI_RXA_TUSIZE2), D_ALL);
+	MMIO_D(_MMIO(_FDI_RXB_TUSIZE1), D_ALL);
+	MMIO_D(_MMIO(_FDI_RXB_TUSIZE2), D_ALL);
 
 	MMIO_DH(PCH_PP_CONTROL, D_ALL, NULL, pch_pp_control_mmio_write);
 	MMIO_D(PCH_PP_DIVISOR, D_ALL);
 	MMIO_D(PCH_PP_STATUS,  D_ALL);
 	MMIO_D(PCH_LVDS, D_ALL);
-	MMIO_D(_PCH_DPLL_A, D_ALL);
-	MMIO_D(_PCH_DPLL_B, D_ALL);
-	MMIO_D(_PCH_FPA0, D_ALL);
-	MMIO_D(_PCH_FPA1, D_ALL);
-	MMIO_D(_PCH_FPB0, D_ALL);
-	MMIO_D(_PCH_FPB1, D_ALL);
+	MMIO_D(_MMIO(_PCH_DPLL_A), D_ALL);
+	MMIO_D(_MMIO(_PCH_DPLL_B), D_ALL);
+	MMIO_D(_MMIO(_PCH_FPA0), D_ALL);
+	MMIO_D(_MMIO(_PCH_FPA1), D_ALL);
+	MMIO_D(_MMIO(_PCH_FPB0), D_ALL);
+	MMIO_D(_MMIO(_PCH_FPB1), D_ALL);
 	MMIO_D(PCH_DREF_CONTROL, D_ALL);
 	MMIO_D(PCH_RAWCLK_FREQ, D_ALL);
 	MMIO_D(PCH_DPLL_SEL, D_ALL);
 
-	MMIO_D(0x61208, D_ALL);
-	MMIO_D(0x6120c, D_ALL);
+	MMIO_D(_MMIO(0x61208), D_ALL);
+	MMIO_D(_MMIO(0x6120c), D_ALL);
 	MMIO_D(PCH_PP_ON_DELAYS, D_ALL);
 	MMIO_D(PCH_PP_OFF_DELAYS, D_ALL);
 
-	MMIO_DH(0xe651c, D_ALL, dpy_reg_mmio_read, NULL);
-	MMIO_DH(0xe661c, D_ALL, dpy_reg_mmio_read, NULL);
-	MMIO_DH(0xe671c, D_ALL, dpy_reg_mmio_read, NULL);
-	MMIO_DH(0xe681c, D_ALL, dpy_reg_mmio_read, NULL);
-	MMIO_DH(0xe6c04, D_ALL, dpy_reg_mmio_read, NULL);
-	MMIO_DH(0xe6e1c, D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(_MMIO(0xe651c), D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(_MMIO(0xe661c), D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(_MMIO(0xe671c), D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(_MMIO(0xe681c), D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(_MMIO(0xe6c04), D_ALL, dpy_reg_mmio_read, NULL);
+	MMIO_DH(_MMIO(0xe6e1c), D_ALL, dpy_reg_mmio_read, NULL);
 
 	MMIO_RO(PCH_PORT_HOTPLUG, D_ALL, 0,
 		PORTA_HOTPLUG_STATUS_MASK
@@ -2030,11 +2071,11 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 
 	MMIO_D(SOUTH_CHICKEN1, D_ALL);
 	MMIO_DH(SOUTH_CHICKEN2, D_ALL, NULL, south_chicken2_mmio_write);
-	MMIO_D(_TRANSA_CHICKEN1, D_ALL);
-	MMIO_D(_TRANSB_CHICKEN1, D_ALL);
+	MMIO_D(_MMIO(_TRANSA_CHICKEN1), D_ALL);
+	MMIO_D(_MMIO(_TRANSB_CHICKEN1), D_ALL);
 	MMIO_D(SOUTH_DSPCLK_GATE_D, D_ALL);
-	MMIO_D(_TRANSA_CHICKEN2, D_ALL);
-	MMIO_D(_TRANSB_CHICKEN2, D_ALL);
+	MMIO_D(_MMIO(_TRANSA_CHICKEN2), D_ALL);
+	MMIO_D(_MMIO(_TRANSB_CHICKEN2), D_ALL);
 
 	MMIO_D(ILK_DPFC_CB_BASE, D_ALL);
 	MMIO_D(ILK_DPFC_CONTROL, D_ALL);
@@ -2100,24 +2141,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(PREC_PAL_DATA(PIPE_C), D_ALL);
 	MMIO_F(PREC_PAL_GC_MAX(PIPE_C, 0), 4 * 3, 0, 0, 0, D_ALL, NULL, NULL);
 
-	MMIO_D(0x60110, D_ALL);
-	MMIO_D(0x61110, D_ALL);
-	MMIO_F(0x70400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x71400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x72400, 0x40, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x70440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
-	MMIO_F(0x71440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
-	MMIO_F(0x72440, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
-	MMIO_F(0x7044c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
-	MMIO_F(0x7144c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
-	MMIO_F(0x7244c, 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_D(_MMIO(0x60110), D_ALL);
+	MMIO_D(_MMIO(0x61110), D_ALL);
+	MMIO_F(_MMIO(0x70400), 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x71400), 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x72400), 0x40, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x70440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(_MMIO(0x71440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(_MMIO(0x72440), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(_MMIO(0x7044c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(_MMIO(0x7144c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
+	MMIO_F(_MMIO(0x7244c), 0xc, 0, 0, 0, D_PRE_SKL, NULL, NULL);
 
 	MMIO_D(PIPE_WM_LINETIME(PIPE_A), D_ALL);
 	MMIO_D(PIPE_WM_LINETIME(PIPE_B), D_ALL);
 	MMIO_D(PIPE_WM_LINETIME(PIPE_C), D_ALL);
 	MMIO_D(SPLL_CTL, D_ALL);
-	MMIO_D(_WRPLL_CTL1, D_ALL);
-	MMIO_D(_WRPLL_CTL2, D_ALL);
+	MMIO_D(_MMIO(_WRPLL_CTL1), D_ALL);
+	MMIO_D(_MMIO(_WRPLL_CTL2), D_ALL);
 	MMIO_D(PORT_CLK_SEL(PORT_A), D_ALL);
 	MMIO_D(PORT_CLK_SEL(PORT_B), D_ALL);
 	MMIO_D(PORT_CLK_SEL(PORT_C), D_ALL);
@@ -2128,15 +2169,15 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(TRANS_CLK_SEL(TRANSCODER_C), D_ALL);
 
 	MMIO_D(HSW_NDE_RSTWRN_OPT, D_ALL);
-	MMIO_D(0x46508, D_ALL);
+	MMIO_D(_MMIO(0x46508), D_ALL);
 
-	MMIO_D(0x49080, D_ALL);
-	MMIO_D(0x49180, D_ALL);
-	MMIO_D(0x49280, D_ALL);
+	MMIO_D(_MMIO(0x49080), D_ALL);
+	MMIO_D(_MMIO(0x49180), D_ALL);
+	MMIO_D(_MMIO(0x49280), D_ALL);
 
-	MMIO_F(0x49090, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x49190, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x49290, 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x49090), 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x49190), 0x14, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x49290), 0x14, 0, 0, 0, D_ALL, NULL, NULL);
 
 	MMIO_D(GAMMA_MODE(PIPE_A), D_ALL);
 	MMIO_D(GAMMA_MODE(PIPE_B), D_ALL);
@@ -2156,7 +2197,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(SBI_CTL_STAT, D_ALL, NULL, sbi_ctl_mmio_write);
 	MMIO_D(PIXCLK_GATE, D_ALL);
 
-	MMIO_F(_DPA_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_ALL, NULL,
+	MMIO_F(_MMIO(_DPA_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_ALL, NULL,
 		dp_aux_ch_ctl_mmio_write);
 
 	MMIO_DH(DDI_BUF_CTL(PORT_A), D_ALL, NULL, ddi_buf_ctl_mmio_write);
@@ -2177,24 +2218,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(DP_TP_STATUS(PORT_D), D_ALL, NULL, dp_tp_status_mmio_write);
 	MMIO_DH(DP_TP_STATUS(PORT_E), D_ALL, NULL, NULL);
 
-	MMIO_F(_DDI_BUF_TRANS_A, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x64e60, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x64eC0, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x64f20, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x64f80, 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(_DDI_BUF_TRANS_A), 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x64e60), 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x64eC0), 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x64f20), 0x50, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x64f80), 0x50, 0, 0, 0, D_ALL, NULL, NULL);
 
 	MMIO_D(HSW_AUD_CFG(PIPE_A), D_ALL);
 	MMIO_D(HSW_AUD_PIN_ELD_CP_VLD, D_ALL);
 
-	MMIO_DH(_TRANS_DDI_FUNC_CTL_A, D_ALL, NULL, NULL);
-	MMIO_DH(_TRANS_DDI_FUNC_CTL_B, D_ALL, NULL, NULL);
-	MMIO_DH(_TRANS_DDI_FUNC_CTL_C, D_ALL, NULL, NULL);
-	MMIO_DH(_TRANS_DDI_FUNC_CTL_EDP, D_ALL, NULL, NULL);
+	MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_A), D_ALL, NULL, NULL);
+	MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_B), D_ALL, NULL, NULL);
+	MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_C), D_ALL, NULL, NULL);
+	MMIO_DH(_MMIO(_TRANS_DDI_FUNC_CTL_EDP), D_ALL, NULL, NULL);
 
-	MMIO_D(_TRANSA_MSA_MISC, D_ALL);
-	MMIO_D(_TRANSB_MSA_MISC, D_ALL);
-	MMIO_D(_TRANSC_MSA_MISC, D_ALL);
-	MMIO_D(_TRANS_EDP_MSA_MISC, D_ALL);
+	MMIO_D(_MMIO(_TRANSA_MSA_MISC), D_ALL);
+	MMIO_D(_MMIO(_TRANSB_MSA_MISC), D_ALL);
+	MMIO_D(_MMIO(_TRANSC_MSA_MISC), D_ALL);
+	MMIO_D(_MMIO(_TRANS_EDP_MSA_MISC), D_ALL);
 
 	MMIO_DH(FORCEWAKE, D_ALL, NULL, NULL);
 	MMIO_D(FORCEWAKE_ACK, D_ALL);
@@ -2260,101 +2301,101 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(GEN6_UCGCTL1, D_ALL);
 	MMIO_D(GEN6_UCGCTL2, D_ALL);
 
-	MMIO_F(0x4f000, 0x90, 0, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x4f000), 0x90, 0, 0, 0, D_ALL, NULL, NULL);
 
 	MMIO_D(GEN6_PCODE_DATA, D_ALL);
-	MMIO_D(0x13812c, D_ALL);
+	MMIO_D(_MMIO(0x13812c), D_ALL);
 	MMIO_DH(GEN7_ERR_INT, D_ALL, NULL, NULL);
 	MMIO_D(HSW_EDRAM_CAP, D_ALL);
 	MMIO_D(HSW_IDICR, D_ALL);
 	MMIO_DH(GFX_FLSH_CNTL_GEN6, D_ALL, NULL, NULL);
 
-	MMIO_D(0x3c, D_ALL);
-	MMIO_D(0x860, D_ALL);
+	MMIO_D(_MMIO(0x3c), D_ALL);
+	MMIO_D(_MMIO(0x860), D_ALL);
 	MMIO_D(ECOSKPD, D_ALL);
-	MMIO_D(0x121d0, D_ALL);
+	MMIO_D(_MMIO(0x121d0), D_ALL);
 	MMIO_D(GEN6_BLITTER_ECOSKPD, D_ALL);
-	MMIO_D(0x41d0, D_ALL);
+	MMIO_D(_MMIO(0x41d0), D_ALL);
 	MMIO_D(GAC_ECO_BITS, D_ALL);
-	MMIO_D(0x6200, D_ALL);
-	MMIO_D(0x6204, D_ALL);
-	MMIO_D(0x6208, D_ALL);
-	MMIO_D(0x7118, D_ALL);
-	MMIO_D(0x7180, D_ALL);
-	MMIO_D(0x7408, D_ALL);
-	MMIO_D(0x7c00, D_ALL);
+	MMIO_D(_MMIO(0x6200), D_ALL);
+	MMIO_D(_MMIO(0x6204), D_ALL);
+	MMIO_D(_MMIO(0x6208), D_ALL);
+	MMIO_D(_MMIO(0x7118), D_ALL);
+	MMIO_D(_MMIO(0x7180), D_ALL);
+	MMIO_D(_MMIO(0x7408), D_ALL);
+	MMIO_D(_MMIO(0x7c00), D_ALL);
 	MMIO_DH(GEN6_MBCTL, D_ALL, NULL, mbctl_write);
-	MMIO_D(0x911c, D_ALL);
-	MMIO_D(0x9120, D_ALL);
+	MMIO_D(_MMIO(0x911c), D_ALL);
+	MMIO_D(_MMIO(0x9120), D_ALL);
 	MMIO_DFH(GEN7_UCGCTL4, D_ALL, F_CMD_ACCESS, NULL, NULL);
 
 	MMIO_D(GAB_CTL, D_ALL);
-	MMIO_D(0x48800, D_ALL);
-	MMIO_D(0xce044, D_ALL);
-	MMIO_D(0xe6500, D_ALL);
-	MMIO_D(0xe6504, D_ALL);
-	MMIO_D(0xe6600, D_ALL);
-	MMIO_D(0xe6604, D_ALL);
-	MMIO_D(0xe6700, D_ALL);
-	MMIO_D(0xe6704, D_ALL);
-	MMIO_D(0xe6800, D_ALL);
-	MMIO_D(0xe6804, D_ALL);
+	MMIO_D(_MMIO(0x48800), D_ALL);
+	MMIO_D(_MMIO(0xce044), D_ALL);
+	MMIO_D(_MMIO(0xe6500), D_ALL);
+	MMIO_D(_MMIO(0xe6504), D_ALL);
+	MMIO_D(_MMIO(0xe6600), D_ALL);
+	MMIO_D(_MMIO(0xe6604), D_ALL);
+	MMIO_D(_MMIO(0xe6700), D_ALL);
+	MMIO_D(_MMIO(0xe6704), D_ALL);
+	MMIO_D(_MMIO(0xe6800), D_ALL);
+	MMIO_D(_MMIO(0xe6804), D_ALL);
 	MMIO_D(PCH_GMBUS4, D_ALL);
 	MMIO_D(PCH_GMBUS5, D_ALL);
 
-	MMIO_D(0x902c, D_ALL);
-	MMIO_D(0xec008, D_ALL);
-	MMIO_D(0xec00c, D_ALL);
-	MMIO_D(0xec008 + 0x18, D_ALL);
-	MMIO_D(0xec00c + 0x18, D_ALL);
-	MMIO_D(0xec008 + 0x18 * 2, D_ALL);
-	MMIO_D(0xec00c + 0x18 * 2, D_ALL);
-	MMIO_D(0xec008 + 0x18 * 3, D_ALL);
-	MMIO_D(0xec00c + 0x18 * 3, D_ALL);
-	MMIO_D(0xec408, D_ALL);
-	MMIO_D(0xec40c, D_ALL);
-	MMIO_D(0xec408 + 0x18, D_ALL);
-	MMIO_D(0xec40c + 0x18, D_ALL);
-	MMIO_D(0xec408 + 0x18 * 2, D_ALL);
-	MMIO_D(0xec40c + 0x18 * 2, D_ALL);
-	MMIO_D(0xec408 + 0x18 * 3, D_ALL);
-	MMIO_D(0xec40c + 0x18 * 3, D_ALL);
-	MMIO_D(0xfc810, D_ALL);
-	MMIO_D(0xfc81c, D_ALL);
-	MMIO_D(0xfc828, D_ALL);
-	MMIO_D(0xfc834, D_ALL);
-	MMIO_D(0xfcc00, D_ALL);
-	MMIO_D(0xfcc0c, D_ALL);
-	MMIO_D(0xfcc18, D_ALL);
-	MMIO_D(0xfcc24, D_ALL);
-	MMIO_D(0xfd000, D_ALL);
-	MMIO_D(0xfd00c, D_ALL);
-	MMIO_D(0xfd018, D_ALL);
-	MMIO_D(0xfd024, D_ALL);
-	MMIO_D(0xfd034, D_ALL);
+	MMIO_D(_MMIO(0x902c), D_ALL);
+	MMIO_D(_MMIO(0xec008), D_ALL);
+	MMIO_D(_MMIO(0xec00c), D_ALL);
+	MMIO_D(_MMIO(0xec008 + 0x18), D_ALL);
+	MMIO_D(_MMIO(0xec00c + 0x18), D_ALL);
+	MMIO_D(_MMIO(0xec008 + 0x18 * 2), D_ALL);
+	MMIO_D(_MMIO(0xec00c + 0x18 * 2), D_ALL);
+	MMIO_D(_MMIO(0xec008 + 0x18 * 3), D_ALL);
+	MMIO_D(_MMIO(0xec00c + 0x18 * 3), D_ALL);
+	MMIO_D(_MMIO(0xec408), D_ALL);
+	MMIO_D(_MMIO(0xec40c), D_ALL);
+	MMIO_D(_MMIO(0xec408 + 0x18), D_ALL);
+	MMIO_D(_MMIO(0xec40c + 0x18), D_ALL);
+	MMIO_D(_MMIO(0xec408 + 0x18 * 2), D_ALL);
+	MMIO_D(_MMIO(0xec40c + 0x18 * 2), D_ALL);
+	MMIO_D(_MMIO(0xec408 + 0x18 * 3), D_ALL);
+	MMIO_D(_MMIO(0xec40c + 0x18 * 3), D_ALL);
+	MMIO_D(_MMIO(0xfc810), D_ALL);
+	MMIO_D(_MMIO(0xfc81c), D_ALL);
+	MMIO_D(_MMIO(0xfc828), D_ALL);
+	MMIO_D(_MMIO(0xfc834), D_ALL);
+	MMIO_D(_MMIO(0xfcc00), D_ALL);
+	MMIO_D(_MMIO(0xfcc0c), D_ALL);
+	MMIO_D(_MMIO(0xfcc18), D_ALL);
+	MMIO_D(_MMIO(0xfcc24), D_ALL);
+	MMIO_D(_MMIO(0xfd000), D_ALL);
+	MMIO_D(_MMIO(0xfd00c), D_ALL);
+	MMIO_D(_MMIO(0xfd018), D_ALL);
+	MMIO_D(_MMIO(0xfd024), D_ALL);
+	MMIO_D(_MMIO(0xfd034), D_ALL);
 
 	MMIO_DH(FPGA_DBG, D_ALL, NULL, fpga_dbg_mmio_write);
-	MMIO_D(0x2054, D_ALL);
-	MMIO_D(0x12054, D_ALL);
-	MMIO_D(0x22054, D_ALL);
-	MMIO_D(0x1a054, D_ALL);
-
-	MMIO_D(0x44070, D_ALL);
-	MMIO_DFH(0x215c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2178, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x12178, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x1217c, D_ALL, F_CMD_ACCESS, NULL, NULL);
-
-	MMIO_F(0x2290, 8, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL);
-	MMIO_D(0x2b00, D_BDW_PLUS);
-	MMIO_D(0x2360, D_BDW_PLUS);
-	MMIO_F(0x5200, 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x5240, 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
-	MMIO_F(0x5280, 16, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
-
-	MMIO_DFH(0x1c17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x1c178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_D(_MMIO(0x2054), D_ALL);
+	MMIO_D(_MMIO(0x12054), D_ALL);
+	MMIO_D(_MMIO(0x22054), D_ALL);
+	MMIO_D(_MMIO(0x1a054), D_ALL);
+
+	MMIO_D(_MMIO(0x44070), D_ALL);
+	MMIO_DFH(_MMIO(0x215c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2178), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x217c), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x12178), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x1217c), D_ALL, F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_F(_MMIO(0x2290), 8, F_CMD_ACCESS, 0, 0, D_BDW_PLUS, NULL, NULL);
+	MMIO_D(_MMIO(0x2b00), D_BDW_PLUS);
+	MMIO_D(_MMIO(0x2360), D_BDW_PLUS);
+	MMIO_F(_MMIO(0x5200), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x5240), 32, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+	MMIO_F(_MMIO(0x5280), 16, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
+
+	MMIO_DFH(_MMIO(0x1c17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x1c178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(BCS_SWCTRL, D_ALL, F_CMD_ACCESS, NULL, NULL);
 
 	MMIO_F(HS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
@@ -2368,24 +2409,24 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_F(CL_PRIMITIVES_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
 	MMIO_F(PS_INVOCATION_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
 	MMIO_F(PS_DEPTH_COUNT, 8, F_CMD_ACCESS, 0, 0, D_ALL, NULL, NULL);
-	MMIO_DH(0x4260, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
-	MMIO_DH(0x4264, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
-	MMIO_DH(0x4268, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
-	MMIO_DH(0x426c, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
-	MMIO_DH(0x4270, D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
-	MMIO_DFH(0x4094, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DH(_MMIO(0x4260), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(_MMIO(0x4264), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(_MMIO(0x4268), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(_MMIO(0x426c), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DH(_MMIO(0x4270), D_BDW_PLUS, NULL, gvt_reg_tlb_control_handler);
+	MMIO_DFH(_MMIO(0x4094), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
 
 	MMIO_DFH(ARB_MODE, D_ALL, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 	MMIO_RING_GM_RDR(RING_BBADDR, D_ALL, NULL, NULL);
-	MMIO_DFH(0x2220, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x12220, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x22220, D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2220), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x12220), D_ALL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x22220), D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_RING_DFH(RING_SYNC_1, D_ALL, F_CMD_ACCESS, NULL, NULL);
 	MMIO_RING_DFH(RING_SYNC_0, D_ALL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x22178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x1a178, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x1a17c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2217c, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x22178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x1a178), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x1a17c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2217c), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
 	return 0;
 }
 
@@ -2459,40 +2500,40 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
 	MMIO_RING_DFH(RING_ACTHD_UDW, D_BDW_PLUS, F_CMD_ACCESS,
 		mmio_read_from_hw, NULL);
 
-#define RING_REG(base) (base + 0xd0)
+#define RING_REG(base) _MMIO((base) + 0xd0)
 	MMIO_RING_F(RING_REG, 4, F_RO, 0,
 		~_MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET), D_BDW_PLUS, NULL,
 		ring_reset_ctl_write);
 #undef RING_REG
 
-#define RING_REG(base) (base + 0x230)
+#define RING_REG(base) _MMIO((base) + 0x230)
 	MMIO_RING_DFH(RING_REG, D_BDW_PLUS, 0, NULL, elsp_mmio_write);
 #undef RING_REG
 
-#define RING_REG(base) (base + 0x234)
+#define RING_REG(base) _MMIO((base) + 0x234)
 	MMIO_RING_F(RING_REG, 8, F_RO | F_CMD_ACCESS, 0, ~0, D_BDW_PLUS,
 		NULL, NULL);
 #undef RING_REG
 
-#define RING_REG(base) (base + 0x244)
+#define RING_REG(base) _MMIO((base) + 0x244)
 	MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
 #undef RING_REG
 
-#define RING_REG(base) (base + 0x370)
+#define RING_REG(base) _MMIO((base) + 0x370)
 	MMIO_RING_F(RING_REG, 48, F_RO, 0, ~0, D_BDW_PLUS, NULL, NULL);
 #undef RING_REG
 
-#define RING_REG(base) (base + 0x3a0)
+#define RING_REG(base) _MMIO((base) + 0x3a0)
 	MMIO_RING_DFH(RING_REG, D_BDW_PLUS, F_MODE_MASK, NULL, NULL);
 #undef RING_REG
 
 	MMIO_D(PIPEMISC(PIPE_A), D_BDW_PLUS);
 	MMIO_D(PIPEMISC(PIPE_B), D_BDW_PLUS);
 	MMIO_D(PIPEMISC(PIPE_C), D_BDW_PLUS);
-	MMIO_D(0x1c1d0, D_BDW_PLUS);
+	MMIO_D(_MMIO(0x1c1d0), D_BDW_PLUS);
 	MMIO_D(GEN6_MBCUNIT_SNPCR, D_BDW_PLUS);
 	MMIO_D(GEN7_MISCCPCTL, D_BDW_PLUS);
-	MMIO_D(0x1c054, D_BDW_PLUS);
+	MMIO_D(_MMIO(0x1c054), D_BDW_PLUS);
 
 	MMIO_DH(GEN6_PCODE_MAILBOX, D_BDW_PLUS, NULL, mailbox_write);
 
@@ -2501,11 +2542,11 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
 
 	MMIO_D(GAMTARBMODE, D_BDW_PLUS);
 
-#define RING_REG(base) (base + 0x270)
+#define RING_REG(base) _MMIO((base) + 0x270)
 	MMIO_RING_F(RING_REG, 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL);
 #undef RING_REG
 
-	MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL);
+	MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, hws_pga_write);
 
 	MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 
@@ -2514,10 +2555,10 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(CHICKEN_PIPESL_1(PIPE_C), D_BDW_PLUS);
 
 	MMIO_D(WM_MISC, D_BDW);
-	MMIO_D(BDW_EDP_PSR_BASE, D_BDW);
+	MMIO_D(_MMIO(BDW_EDP_PSR_BASE), D_BDW);
 
-	MMIO_D(0x66c00, D_BDW_PLUS);
-	MMIO_D(0x66c04, D_BDW_PLUS);
+	MMIO_D(_MMIO(0x66c00), D_BDW_PLUS);
+	MMIO_D(_MMIO(0x66c04), D_BDW_PLUS);
 
 	MMIO_D(HSW_GTT_CACHE_EN, D_BDW_PLUS);
 
@@ -2525,54 +2566,54 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(GEN8_EU_DISABLE1, D_BDW_PLUS);
 	MMIO_D(GEN8_EU_DISABLE2, D_BDW_PLUS);
 
-	MMIO_D(0xfdc, D_BDW_PLUS);
+	MMIO_D(_MMIO(0xfdc), D_BDW_PLUS);
 	MMIO_DFH(GEN8_ROW_CHICKEN, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS,
 		NULL, NULL);
 	MMIO_DFH(GEN7_ROW_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS,
 		NULL, NULL);
 	MMIO_DFH(GEN8_UCGCTL6, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
 
-	MMIO_DFH(0xb1f0, D_BDW, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xb1c0, D_BDW, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xb1f0), D_BDW, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xb1c0), D_BDW, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(GEN8_L3SQCREG4, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xb100, D_BDW, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xb10c, D_BDW, F_CMD_ACCESS, NULL, NULL);
-	MMIO_D(0xb110, D_BDW);
+	MMIO_DFH(_MMIO(0xb100), D_BDW, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xb10c), D_BDW, F_CMD_ACCESS, NULL, NULL);
+	MMIO_D(_MMIO(0xb110), D_BDW);
 
-	MMIO_F(0x24d0, 48, F_CMD_ACCESS, 0, 0, D_BDW_PLUS,
+	MMIO_F(_MMIO(0x24d0), 48, F_CMD_ACCESS, 0, 0, D_BDW_PLUS,
 		NULL, force_nonpriv_write);
 
-	MMIO_D(0x44484, D_BDW_PLUS);
-	MMIO_D(0x4448c, D_BDW_PLUS);
+	MMIO_D(_MMIO(0x44484), D_BDW_PLUS);
+	MMIO_D(_MMIO(0x4448c), D_BDW_PLUS);
 
-	MMIO_DFH(0x83a4, D_BDW, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x83a4), D_BDW, F_CMD_ACCESS, NULL, NULL);
 	MMIO_D(GEN8_L3_LRA_1_GPGPU, D_BDW_PLUS);
 
-	MMIO_DFH(0x8430, D_BDW, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x8430), D_BDW, F_CMD_ACCESS, NULL, NULL);
 
-	MMIO_D(0x110000, D_BDW_PLUS);
+	MMIO_D(_MMIO(0x110000), D_BDW_PLUS);
 
-	MMIO_D(0x48400, D_BDW_PLUS);
+	MMIO_D(_MMIO(0x48400), D_BDW_PLUS);
 
-	MMIO_D(0x6e570, D_BDW_PLUS);
-	MMIO_D(0x65f10, D_BDW_PLUS);
+	MMIO_D(_MMIO(0x6e570), D_BDW_PLUS);
+	MMIO_D(_MMIO(0x65f10), D_BDW_PLUS);
 
-	MMIO_DFH(0xe194, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe188, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe194), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe188), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(HALF_SLICE_CHICKEN2, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x2580, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
-
-	MMIO_DFH(0x2248, D_BDW, F_CMD_ACCESS, NULL, NULL);
-
-	MMIO_DFH(0xe220, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe230, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe240, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe260, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe270, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe280, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe2a0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe2b0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0xe2c0, D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x2580), D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_DFH(_MMIO(0x2248), D_BDW, F_CMD_ACCESS, NULL, NULL);
+
+	MMIO_DFH(_MMIO(0xe220), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe230), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe240), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe260), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe270), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe280), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe2a0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe2b0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0xe2c0), D_BDW_PLUS, F_CMD_ACCESS, NULL, NULL);
 	return 0;
 }
 
@@ -2588,11 +2629,11 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(FORCEWAKE_MEDIA_GEN9, D_SKL_PLUS, NULL, mul_force_wake_write);
 	MMIO_DH(FORCEWAKE_ACK_MEDIA_GEN9, D_SKL_PLUS, NULL, NULL);
 
-	MMIO_F(_DPB_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL,
+	MMIO_F(_MMIO(_DPB_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL,
 						dp_aux_ch_ctl_mmio_write);
-	MMIO_F(_DPC_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL,
+	MMIO_F(_MMIO(_DPC_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL,
 						dp_aux_ch_ctl_mmio_write);
-	MMIO_F(_DPD_AUX_CH_CTL, 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL,
+	MMIO_F(_MMIO(_DPD_AUX_CH_CTL), 6 * 4, 0, 0, 0, D_SKL_PLUS, NULL,
 						dp_aux_ch_ctl_mmio_write);
 
 	/*
@@ -2603,26 +2644,26 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(HSW_PWR_WELL_CTL_DRIVER(SKL_DISP_PW_MISC_IO), D_SKL_PLUS, NULL,
 		skl_power_well_ctl_write);
 
-	MMIO_D(0xa210, D_SKL_PLUS);
+	MMIO_D(_MMIO(0xa210), D_SKL_PLUS);
 	MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
 	MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
 	MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DH(0x4ddc, D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(0x42080, D_SKL_PLUS, NULL, NULL);
-	MMIO_D(0x45504, D_SKL_PLUS);
-	MMIO_D(0x45520, D_SKL_PLUS);
-	MMIO_D(0x46000, D_SKL_PLUS);
-	MMIO_DH(0x46010, D_SKL | D_KBL, NULL, skl_lcpll_write);
-	MMIO_DH(0x46014, D_SKL | D_KBL, NULL, skl_lcpll_write);
-	MMIO_D(0x6C040, D_SKL | D_KBL);
-	MMIO_D(0x6C048, D_SKL | D_KBL);
-	MMIO_D(0x6C050, D_SKL | D_KBL);
-	MMIO_D(0x6C044, D_SKL | D_KBL);
-	MMIO_D(0x6C04C, D_SKL | D_KBL);
-	MMIO_D(0x6C054, D_SKL | D_KBL);
-	MMIO_D(0x6c058, D_SKL | D_KBL);
-	MMIO_D(0x6c05c, D_SKL | D_KBL);
-	MMIO_DH(0X6c060, D_SKL | D_KBL, dpll_status_read, NULL);
+	MMIO_DH(_MMIO(0x4ddc), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(0x42080), D_SKL_PLUS, NULL, NULL);
+	MMIO_D(_MMIO(0x45504), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x45520), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x46000), D_SKL_PLUS);
+	MMIO_DH(_MMIO(0x46010), D_SKL | D_KBL, NULL, skl_lcpll_write);
+	MMIO_DH(_MMIO(0x46014), D_SKL | D_KBL, NULL, skl_lcpll_write);
+	MMIO_D(_MMIO(0x6C040), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6C048), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6C050), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6C044), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6C04C), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6C054), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6c058), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6c05c), D_SKL | D_KBL);
+	MMIO_DH(_MMIO(0x6c060), D_SKL | D_KBL, dpll_status_read, NULL);
 
 	MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write);
 	MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write);
@@ -2711,105 +2752,108 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 2), D_SKL_PLUS, NULL, NULL);
 	MMIO_DH(PLANE_NV12_BUF_CFG(PIPE_C, 3), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_DH(_REG_701C0(PIPE_A, 1), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_A, 2), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_A, 3), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_A, 4), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_DH(_REG_701C0(PIPE_B, 1), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_B, 2), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_B, 3), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_B, 4), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_DH(_REG_701C0(PIPE_C, 1), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_C, 2), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_C, 3), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C0(PIPE_C, 4), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C0(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_DH(_REG_701C4(PIPE_A, 1), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_A, 2), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_A, 3), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_A, 4), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 1)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 2)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 3)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_A, 4)), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_DH(_REG_701C4(PIPE_B, 1), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_B, 2), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_B, 3), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_B, 4), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 1)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 2)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 3)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_B, 4)), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_DH(_REG_701C4(PIPE_C, 1), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_C, 2), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_C, 3), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_REG_701C4(PIPE_C, 4), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 1)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 2)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_D(0x70380, D_SKL_PLUS);
-	MMIO_D(0x71380, D_SKL_PLUS);
-	MMIO_D(0x72380, D_SKL_PLUS);
-	MMIO_D(0x7039c, D_SKL_PLUS);
+	MMIO_D(_MMIO(0x70380), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x71380), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x72380), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x7039c), D_SKL_PLUS);
 
-	MMIO_D(0x8f074, D_SKL | D_KBL);
-	MMIO_D(0x8f004, D_SKL | D_KBL);
-	MMIO_D(0x8f034, D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x8f074), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x8f004), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x8f034), D_SKL | D_KBL);
 
-	MMIO_D(0xb11c, D_SKL | D_KBL);
+	MMIO_D(_MMIO(0xb11c), D_SKL | D_KBL);
 
-	MMIO_D(0x51000, D_SKL | D_KBL);
-	MMIO_D(0x6c00c, D_SKL_PLUS);
+	MMIO_D(_MMIO(0x51000), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6c00c), D_SKL_PLUS);
 
-	MMIO_F(0xc800, 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL);
-	MMIO_F(0xb020, 0x80, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL);
+	MMIO_F(_MMIO(0xc800), 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL);
+	MMIO_F(_MMIO(0xb020), 0x80, F_CMD_ACCESS, 0, 0, D_SKL | D_KBL, NULL, NULL);
 
-	MMIO_D(0xd08, D_SKL_PLUS);
-	MMIO_DFH(0x20e0, D_SKL_PLUS, F_MODE_MASK, NULL, NULL);
-	MMIO_DFH(0x20ec, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
+	MMIO_D(_MMIO(0xd08), D_SKL_PLUS);
+	MMIO_DFH(_MMIO(0x20e0), D_SKL_PLUS, F_MODE_MASK, NULL, NULL);
+	MMIO_DFH(_MMIO(0x20ec), D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 
 	/* TRTT */
-	MMIO_DFH(0x4de0, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x4de4, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x4de8, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x4dec, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x4df0, D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(0x4df4, D_SKL | D_KBL, F_CMD_ACCESS, NULL, gen9_trtte_write);
-	MMIO_DH(0x4dfc, D_SKL | D_KBL, NULL, gen9_trtt_chicken_write);
+	MMIO_DFH(_MMIO(0x4de0), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x4de4), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x4de8), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x4dec), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x4df0), D_SKL | D_KBL, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(_MMIO(0x4df4), D_SKL | D_KBL, F_CMD_ACCESS, NULL, gen9_trtte_write);
+	MMIO_DH(_MMIO(0x4dfc), D_SKL | D_KBL, NULL, gen9_trtt_chicken_write);
 
-	MMIO_D(0x45008, D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x45008), D_SKL | D_KBL);
 
-	MMIO_D(0x46430, D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x46430), D_SKL | D_KBL);
 
-	MMIO_D(0x46520, D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x46520), D_SKL | D_KBL);
 
-	MMIO_D(0xc403c, D_SKL | D_KBL);
-	MMIO_D(0xb004, D_SKL_PLUS);
+	MMIO_D(_MMIO(0xc403c), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0xb004), D_SKL_PLUS);
 	MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write);
 
-	MMIO_D(0x65900, D_SKL_PLUS);
-	MMIO_D(0x1082c0, D_SKL | D_KBL);
-	MMIO_D(0x4068, D_SKL | D_KBL);
-	MMIO_D(0x67054, D_SKL | D_KBL);
-	MMIO_D(0x6e560, D_SKL | D_KBL);
-	MMIO_D(0x6e554, D_SKL | D_KBL);
-	MMIO_D(0x2b20, D_SKL | D_KBL);
-	MMIO_D(0x65f00, D_SKL | D_KBL);
-	MMIO_D(0x65f08, D_SKL | D_KBL);
-	MMIO_D(0x320f0, D_SKL | D_KBL);
-
-	MMIO_D(0x70034, D_SKL_PLUS);
-	MMIO_D(0x71034, D_SKL_PLUS);
-	MMIO_D(0x72034, D_SKL_PLUS);
-
-	MMIO_D(_PLANE_KEYVAL_1(PIPE_A), D_SKL_PLUS);
-	MMIO_D(_PLANE_KEYVAL_1(PIPE_B), D_SKL_PLUS);
-	MMIO_D(_PLANE_KEYVAL_1(PIPE_C), D_SKL_PLUS);
-	MMIO_D(_PLANE_KEYMSK_1(PIPE_A), D_SKL_PLUS);
-	MMIO_D(_PLANE_KEYMSK_1(PIPE_B), D_SKL_PLUS);
-	MMIO_D(_PLANE_KEYMSK_1(PIPE_C), D_SKL_PLUS);
-
-	MMIO_D(0x44500, D_SKL_PLUS);
+	MMIO_D(_MMIO(0x65900), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x1082c0), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x4068), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x67054), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6e560), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x6e554), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x2b20), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x65f00), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x65f08), D_SKL | D_KBL);
+	MMIO_D(_MMIO(0x320f0), D_SKL | D_KBL);
+
+	MMIO_D(_MMIO(0x70034), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x71034), D_SKL_PLUS);
+	MMIO_D(_MMIO(0x72034), D_SKL_PLUS);
+
+	MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_A)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_B)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYVAL_1(PIPE_C)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_A)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_B)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYMAX_1(PIPE_C)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_A)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_B)), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_KEYMSK_1(PIPE_C)), D_SKL_PLUS);
+
+	MMIO_D(_MMIO(0x44500), D_SKL_PLUS);
 	MMIO_DFH(GEN9_CSFE_CHICKEN1_RCS, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
 	MMIO_DFH(GEN8_HDC_CHICKEN1, D_SKL | D_KBL, F_MODE_MASK | F_CMD_ACCESS,
 		NULL, NULL);
 
-	MMIO_D(0x4ab8, D_KBL);
-	MMIO_D(0x2248, D_SKL_PLUS | D_KBL);
+	MMIO_D(_MMIO(0x4ab8), D_KBL);
+	MMIO_D(_MMIO(0x2248), D_SKL_PLUS | D_KBL);
 
 	return 0;
 }
@@ -2825,8 +2869,8 @@ static struct gvt_mmio_block *find_mmio_block(struct intel_gvt *gvt,
 	for (i = 0; i < num; i++, block++) {
 		if (!(device & block->device))
 			continue;
-		if (offset >= INTEL_GVT_MMIO_OFFSET(block->offset) &&
-		    offset < INTEL_GVT_MMIO_OFFSET(block->offset) + block->size)
+		if (offset >= i915_mmio_reg_offset(block->offset) &&
+		    offset < i915_mmio_reg_offset(block->offset) + block->size)
 			return block;
 	}
 	return NULL;
@@ -2906,14 +2950,46 @@ int intel_gvt_setup_mmio_info(struct intel_gvt *gvt)
 	gvt->mmio.mmio_block = mmio_blocks;
 	gvt->mmio.num_mmio_block = ARRAY_SIZE(mmio_blocks);
 
-	gvt_dbg_mmio("traced %u virtual mmio registers\n",
-		     gvt->mmio.num_tracked_mmio);
 	return 0;
 err:
 	intel_gvt_clean_mmio_info(gvt);
 	return ret;
 }
 
+/**
+ * intel_gvt_for_each_tracked_mmio - iterate each tracked mmio
+ * @gvt: a GVT device
+ * @handler: the handler
+ * @data: private data given to handler
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt,
+	int (*handler)(struct intel_gvt *gvt, u32 offset, void *data),
+	void *data)
+{
+	struct gvt_mmio_block *block = gvt->mmio.mmio_block;
+	struct intel_gvt_mmio_info *e;
+	int i, j, ret;
+
+	hash_for_each(gvt->mmio.mmio_info_table, i, e, node) {
+		ret = handler(gvt, e->offset, data);
+		if (ret)
+			return ret;
+	}
+
+	for (i = 0; i < gvt->mmio.num_mmio_block; i++, block++) {
+		for (j = 0; j < block->size; j += 4) {
+			ret = handler(gvt,
+				      i915_mmio_reg_offset(block->offset) + j,
+				      data);
+			if (ret)
+				return ret;
+		}
+	}
+	return 0;
+}
 
 /**
  * intel_vgpu_default_mmio_read - default MMIO read handler
diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h
index df7f33abd393..f8e77e166246 100644
--- a/drivers/gpu/drm/i915/gvt/hypercall.h
+++ b/drivers/gpu/drm/i915/gvt/hypercall.h
@@ -55,6 +55,10 @@ struct intel_gvt_mpt {
 			      unsigned long mfn, unsigned int nr, bool map);
 	int (*set_trap_area)(unsigned long handle, u64 start, u64 end,
 			     bool map);
+	int (*set_opregion)(void *vgpu);
+	int (*get_vfio_device)(void *vgpu);
+	void (*put_vfio_device)(void *vgpu);
+	bool (*is_valid_gfn)(unsigned long handle, unsigned long gfn);
 };
 
 extern struct intel_gvt_mpt xengt_mpt;
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
index 96060920a6fe..021f722e2481 100644
--- a/drivers/gpu/drm/i915/gvt/kvmgt.c
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -53,11 +53,23 @@ static const struct intel_gvt_ops *intel_gvt_ops;
 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
 #define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
 
+#define OPREGION_SIGNATURE "IntelGraphicsMem"
+
+struct vfio_region;
+struct intel_vgpu_regops {
+	size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
+			size_t count, loff_t *ppos, bool iswrite);
+	void (*release)(struct intel_vgpu *vgpu,
+			struct vfio_region *region);
+};
+
 struct vfio_region {
 	u32				type;
 	u32				subtype;
 	size_t				size;
 	u32				flags;
+	const struct intel_vgpu_regops	*ops;
+	void				*data;
 };
 
 struct kvmgt_pgfn {
@@ -248,120 +260,6 @@ static void gvt_cache_destroy(struct intel_vgpu *vgpu)
 	}
 }
 
-static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt,
-		const char *name)
-{
-	int i;
-	struct intel_vgpu_type *t;
-	const char *driver_name = dev_driver_string(
-			&gvt->dev_priv->drm.pdev->dev);
-
-	for (i = 0; i < gvt->num_types; i++) {
-		t = &gvt->types[i];
-		if (!strncmp(t->name, name + strlen(driver_name) + 1,
-			sizeof(t->name)))
-			return t;
-	}
-
-	return NULL;
-}
-
-static ssize_t available_instances_show(struct kobject *kobj,
-					struct device *dev, char *buf)
-{
-	struct intel_vgpu_type *type;
-	unsigned int num = 0;
-	void *gvt = kdev_to_i915(dev)->gvt;
-
-	type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
-	if (!type)
-		num = 0;
-	else
-		num = type->avail_instance;
-
-	return sprintf(buf, "%u\n", num);
-}
-
-static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
-		char *buf)
-{
-	return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
-}
-
-static ssize_t description_show(struct kobject *kobj, struct device *dev,
-		char *buf)
-{
-	struct intel_vgpu_type *type;
-	void *gvt = kdev_to_i915(dev)->gvt;
-
-	type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
-	if (!type)
-		return 0;
-
-	return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
-		       "fence: %d\nresolution: %s\n"
-		       "weight: %d\n",
-		       BYTES_TO_MB(type->low_gm_size),
-		       BYTES_TO_MB(type->high_gm_size),
-		       type->fence, vgpu_edid_str(type->resolution),
-		       type->weight);
-}
-
-static MDEV_TYPE_ATTR_RO(available_instances);
-static MDEV_TYPE_ATTR_RO(device_api);
-static MDEV_TYPE_ATTR_RO(description);
-
-static struct attribute *type_attrs[] = {
-	&mdev_type_attr_available_instances.attr,
-	&mdev_type_attr_device_api.attr,
-	&mdev_type_attr_description.attr,
-	NULL,
-};
-
-static struct attribute_group *intel_vgpu_type_groups[] = {
-	[0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL,
-};
-
-static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt)
-{
-	int i, j;
-	struct intel_vgpu_type *type;
-	struct attribute_group *group;
-
-	for (i = 0; i < gvt->num_types; i++) {
-		type = &gvt->types[i];
-
-		group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL);
-		if (WARN_ON(!group))
-			goto unwind;
-
-		group->name = type->name;
-		group->attrs = type_attrs;
-		intel_vgpu_type_groups[i] = group;
-	}
-
-	return true;
-
-unwind:
-	for (j = 0; j < i; j++) {
-		group = intel_vgpu_type_groups[j];
-		kfree(group);
-	}
-
-	return false;
-}
-
-static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt)
-{
-	int i;
-	struct attribute_group *group;
-
-	for (i = 0; i < gvt->num_types; i++) {
-		group = intel_vgpu_type_groups[i];
-		kfree(group);
-	}
-}
-
 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
 {
 	hash_init(info->ptable);
@@ -430,6 +328,108 @@ static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
 	}
 }
 
+static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
+		size_t count, loff_t *ppos, bool iswrite)
+{
+	unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
+			VFIO_PCI_NUM_REGIONS;
+	void *base = vgpu->vdev.region[i].data;
+	loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
+
+	if (pos >= vgpu->vdev.region[i].size || iswrite) {
+		gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
+		return -EINVAL;
+	}
+	count = min(count, (size_t)(vgpu->vdev.region[i].size - pos));
+	memcpy(buf, base + pos, count);
+
+	return count;
+}
+
+static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
+		struct vfio_region *region)
+{
+}
+
+static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
+	.rw = intel_vgpu_reg_rw_opregion,
+	.release = intel_vgpu_reg_release_opregion,
+};
+
+static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
+		unsigned int type, unsigned int subtype,
+		const struct intel_vgpu_regops *ops,
+		size_t size, u32 flags, void *data)
+{
+	struct vfio_region *region;
+
+	region = krealloc(vgpu->vdev.region,
+			(vgpu->vdev.num_regions + 1) * sizeof(*region),
+			GFP_KERNEL);
+	if (!region)
+		return -ENOMEM;
+
+	vgpu->vdev.region = region;
+	vgpu->vdev.region[vgpu->vdev.num_regions].type = type;
+	vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype;
+	vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops;
+	vgpu->vdev.region[vgpu->vdev.num_regions].size = size;
+	vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags;
+	vgpu->vdev.region[vgpu->vdev.num_regions].data = data;
+	vgpu->vdev.num_regions++;
+	return 0;
+}
+
+static int kvmgt_get_vfio_device(void *p_vgpu)
+{
+	struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
+
+	vgpu->vdev.vfio_device = vfio_device_get_from_dev(
+		mdev_dev(vgpu->vdev.mdev));
+	if (!vgpu->vdev.vfio_device) {
+		gvt_vgpu_err("failed to get vfio device\n");
+		return -ENODEV;
+	}
+	return 0;
+}
+
+
+static int kvmgt_set_opregion(void *p_vgpu)
+{
+	struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
+	void *base;
+	int ret;
+
+	/* Each vgpu has its own opregion, although VFIO would create another
+	 * one later. This one is used to expose opregion to VFIO. And the
+	 * other one created by VFIO later, is used by guest actually.
+	 */
+	base = vgpu_opregion(vgpu)->va;
+	if (!base)
+		return -ENOMEM;
+
+	if (memcmp(base, OPREGION_SIGNATURE, 16)) {
+		memunmap(base);
+		return -EINVAL;
+	}
+
+	ret = intel_vgpu_register_reg(vgpu,
+			PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
+			VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
+			&intel_vgpu_regops_opregion, OPREGION_SIZE,
+			VFIO_REGION_INFO_FLAG_READ, base);
+
+	return ret;
+}
+
+static void kvmgt_put_vfio_device(void *vgpu)
+{
+	if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device))
+		return;
+
+	vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device);
+}
+
 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
 {
 	struct intel_vgpu *vgpu = NULL;
@@ -441,7 +441,7 @@ static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
 	pdev = mdev_parent_dev(mdev);
 	gvt = kdev_to_i915(pdev)->gvt;
 
-	type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+	type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
 	if (!type) {
 		gvt_vgpu_err("failed to find type %s to create\n",
 						kobject_name(kobj));
@@ -651,6 +651,39 @@ static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
 	return ret;
 }
 
+static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off)
+{
+	return off >= vgpu_aperture_offset(vgpu) &&
+	       off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
+}
+
+static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off,
+		void *buf, unsigned long count, bool is_write)
+{
+	void *aperture_va;
+
+	if (!intel_vgpu_in_aperture(vgpu, off) ||
+	    !intel_vgpu_in_aperture(vgpu, off + count)) {
+		gvt_vgpu_err("Invalid aperture offset %llu\n", off);
+		return -EINVAL;
+	}
+
+	aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap,
+					ALIGN_DOWN(off, PAGE_SIZE),
+					count + offset_in_page(off));
+	if (!aperture_va)
+		return -EIO;
+
+	if (is_write)
+		memcpy(aperture_va + offset_in_page(off), buf, count);
+	else
+		memcpy(buf, aperture_va + offset_in_page(off), count);
+
+	io_mapping_unmap(aperture_va);
+
+	return 0;
+}
+
 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
 			size_t count, loff_t *ppos, bool is_write)
 {
@@ -660,7 +693,7 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
 	int ret = -EINVAL;
 
 
-	if (index >= VFIO_PCI_NUM_REGIONS) {
+	if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) {
 		gvt_vgpu_err("invalid index: %u\n", index);
 		return -EINVAL;
 	}
@@ -679,8 +712,7 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
 					buf, count, is_write);
 		break;
 	case VFIO_PCI_BAR2_REGION_INDEX:
-		ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_2, pos,
-					buf, count, is_write);
+		ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
 		break;
 	case VFIO_PCI_BAR1_REGION_INDEX:
 	case VFIO_PCI_BAR3_REGION_INDEX:
@@ -688,13 +720,38 @@ static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
 	case VFIO_PCI_BAR5_REGION_INDEX:
 	case VFIO_PCI_VGA_REGION_INDEX:
 	case VFIO_PCI_ROM_REGION_INDEX:
+		break;
 	default:
-		gvt_vgpu_err("unsupported region: %u\n", index);
+		if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions)
+			return -EINVAL;
+
+		index -= VFIO_PCI_NUM_REGIONS;
+		return vgpu->vdev.region[index].ops->rw(vgpu, buf, count,
+				ppos, is_write);
 	}
 
 	return ret == 0 ? count : ret;
 }
 
+static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos)
+{
+	struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
+	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
+	struct intel_gvt *gvt = vgpu->gvt;
+	int offset;
+
+	/* Only allow MMIO GGTT entry access */
+	if (index != PCI_BASE_ADDRESS_0)
+		return false;
+
+	offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
+		intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
+
+	return (offset >= gvt->device_info.gtt_start_offset &&
+		offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
+			true : false;
+}
+
 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
 			size_t count, loff_t *ppos)
 {
@@ -704,7 +761,21 @@ static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
 	while (count) {
 		size_t filled;
 
-		if (count >= 4 && !(*ppos % 4)) {
+		/* Only support GGTT entry 8 bytes read */
+		if (count >= 8 && !(*ppos % 8) &&
+			gtt_entry(mdev, ppos)) {
+			u64 val;
+
+			ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
+					ppos, false);
+			if (ret <= 0)
+				goto read_err;
+
+			if (copy_to_user(buf, &val, sizeof(val)))
+				goto read_err;
+
+			filled = 8;
+		} else if (count >= 4 && !(*ppos % 4)) {
 			u32 val;
 
 			ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
@@ -764,7 +835,21 @@ static ssize_t intel_vgpu_write(struct mdev_device *mdev,
 	while (count) {
 		size_t filled;
 
-		if (count >= 4 && !(*ppos % 4)) {
+		/* Only support GGTT entry 8 bytes write */
+		if (count >= 8 && !(*ppos % 8) &&
+			gtt_entry(mdev, ppos)) {
+			u64 val;
+
+			if (copy_from_user(&val, buf, sizeof(val)))
+				goto write_err;
+
+			ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
+					ppos, true);
+			if (ret <= 0)
+				goto write_err;
+
+			filled = 8;
+		} else if (count >= 4 && !(*ppos % 4)) {
 			u32 val;
 
 			if (copy_from_user(&val, buf, sizeof(val)))
@@ -952,7 +1037,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 
 		info.flags = VFIO_DEVICE_FLAGS_PCI;
 		info.flags |= VFIO_DEVICE_FLAGS_RESET;
-		info.num_regions = VFIO_PCI_NUM_REGIONS;
+		info.num_regions = VFIO_PCI_NUM_REGIONS +
+				vgpu->vdev.num_regions;
 		info.num_irqs = VFIO_PCI_NUM_IRQS;
 
 		return copy_to_user((void __user *)arg, &info, minsz) ?
@@ -1012,6 +1098,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 			if (!sparse)
 				return -ENOMEM;
 
+			sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
+			sparse->header.version = 1;
 			sparse->nr_areas = nr_areas;
 			cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
 			sparse->areas[0].offset =
@@ -1022,18 +1110,24 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 		case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
 			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
 			info.size = 0;
-
 			info.flags = 0;
+
 			gvt_dbg_core("get region info bar:%d\n", info.index);
 			break;
 
 		case VFIO_PCI_ROM_REGION_INDEX:
 		case VFIO_PCI_VGA_REGION_INDEX:
+			info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
+			info.size = 0;
+			info.flags = 0;
+
 			gvt_dbg_core("get region info index:%d\n", info.index);
 			break;
 		default:
 			{
-				struct vfio_region_info_cap_type cap_type;
+				struct vfio_region_info_cap_type cap_type = {
+					.header.id = VFIO_REGION_INFO_CAP_TYPE,
+					.header.version = 1 };
 
 				if (info.index >= VFIO_PCI_NUM_REGIONS +
 						vgpu->vdev.num_regions)
@@ -1050,8 +1144,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 				cap_type.subtype = vgpu->vdev.region[i].subtype;
 
 				ret = vfio_info_add_capability(&caps,
-						VFIO_REGION_INFO_CAP_TYPE,
-						&cap_type);
+							&cap_type.header,
+							sizeof(cap_type));
 				if (ret)
 					return ret;
 			}
@@ -1061,8 +1155,9 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 			switch (cap_type_id) {
 			case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
 				ret = vfio_info_add_capability(&caps,
-					VFIO_REGION_INFO_CAP_SPARSE_MMAP,
-					sparse);
+					&sparse->header, sizeof(*sparse) +
+					(sparse->nr_areas *
+						sizeof(*sparse->areas)));
 				kfree(sparse);
 				if (ret)
 					return ret;
@@ -1073,6 +1168,7 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 		}
 
 		if (caps.size) {
+			info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
 			if (info.argsz < sizeof(info) + caps.size) {
 				info.argsz = sizeof(info) + caps.size;
 				info.cap_offset = 0;
@@ -1159,6 +1255,33 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
 	} else if (cmd == VFIO_DEVICE_RESET) {
 		intel_gvt_ops->vgpu_reset(vgpu);
 		return 0;
+	} else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
+		struct vfio_device_gfx_plane_info dmabuf;
+		int ret = 0;
+
+		minsz = offsetofend(struct vfio_device_gfx_plane_info,
+				    dmabuf_id);
+		if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
+			return -EFAULT;
+		if (dmabuf.argsz < minsz)
+			return -EINVAL;
+
+		ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf);
+		if (ret != 0)
+			return ret;
+
+		return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
+								-EFAULT : 0;
+	} else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
+		__u32 dmabuf_id;
+		__s32 dmabuf_fd;
+
+		if (get_user(dmabuf_id, (__u32 __user *)arg))
+			return -EFAULT;
+
+		dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id);
+		return dmabuf_fd;
+
 	}
 
 	return 0;
@@ -1188,7 +1311,7 @@ hw_id_show(struct device *dev, struct device_attribute *attr,
 		struct intel_vgpu *vgpu = (struct intel_vgpu *)
 			mdev_get_drvdata(mdev);
 		return sprintf(buf, "%u\n",
-			       vgpu->shadow_ctx->hw_id);
+			       vgpu->submission.shadow_ctx->hw_id);
 	}
 	return sprintf(buf, "\n");
 }
@@ -1212,8 +1335,7 @@ static const struct attribute_group *intel_vgpu_groups[] = {
 	NULL,
 };
 
-static const struct mdev_parent_ops intel_vgpu_ops = {
-	.supported_type_groups	= intel_vgpu_type_groups,
+static struct mdev_parent_ops intel_vgpu_ops = {
 	.mdev_attr_groups       = intel_vgpu_groups,
 	.create			= intel_vgpu_create,
 	.remove			= intel_vgpu_remove,
@@ -1229,17 +1351,20 @@ static const struct mdev_parent_ops intel_vgpu_ops = {
 
 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
 {
-	if (!intel_gvt_init_vgpu_type_groups(gvt))
-		return -EFAULT;
+	struct attribute **kvm_type_attrs;
+	struct attribute_group **kvm_vgpu_type_groups;
 
 	intel_gvt_ops = ops;
+	if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs,
+			&kvm_vgpu_type_groups))
+		return -EFAULT;
+	intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
 
 	return mdev_register_device(dev, &intel_vgpu_ops);
 }
 
 static void kvmgt_host_exit(struct device *dev, void *gvt)
 {
-	intel_gvt_cleanup_vgpu_type_groups(gvt);
 	mdev_unregister_device(dev);
 }
 
@@ -1319,8 +1444,8 @@ static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
 					struct kvmgt_guest_info, track_node);
 
 	if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
-		intel_gvt_ops->emulate_mmio_write(info->vgpu, gpa,
-					(void *)val, len);
+		intel_gvt_ops->write_protect_handler(info->vgpu, gpa,
+						     (void *)val, len);
 }
 
 static void kvmgt_page_track_flush_slot(struct kvm *kvm,
@@ -1398,6 +1523,9 @@ static int kvmgt_guest_init(struct mdev_device *mdev)
 	kvmgt_protect_table_init(info);
 	gvt_cache_init(vgpu);
 
+	mutex_init(&vgpu->dmabuf_lock);
+	init_completion(&vgpu->vblank_done);
+
 	info->track_node.track_write = kvmgt_page_track_write;
 	info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
 	kvm_page_track_register_notifier(kvm, &info->track_node);
@@ -1526,6 +1654,21 @@ static unsigned long kvmgt_virt_to_pfn(void *addr)
 	return PFN_DOWN(__pa(addr));
 }
 
+static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn)
+{
+	struct kvmgt_guest_info *info;
+	struct kvm *kvm;
+
+	if (!handle_valid(handle))
+		return false;
+
+	info = (struct kvmgt_guest_info *)handle;
+	kvm = info->kvm;
+
+	return kvm_is_visible_gfn(kvm, gfn);
+
+}
+
 struct intel_gvt_mpt kvmgt_mpt = {
 	.host_init = kvmgt_host_init,
 	.host_exit = kvmgt_host_exit,
@@ -1538,6 +1681,10 @@ struct intel_gvt_mpt kvmgt_mpt = {
 	.read_gpa = kvmgt_read_gpa,
 	.write_gpa = kvmgt_write_gpa,
 	.gfn_to_mfn = kvmgt_gfn_to_pfn,
+	.set_opregion = kvmgt_set_opregion,
+	.get_vfio_device = kvmgt_get_vfio_device,
+	.put_vfio_device = kvmgt_put_vfio_device,
+	.is_valid_gfn = kvmgt_is_valid_gfn,
 };
 EXPORT_SYMBOL_GPL(kvmgt_mpt);
 
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 1e1310f50289..5c869e3fdf3b 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -56,38 +56,6 @@ int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
 	(reg >= gvt->device_info.gtt_start_offset \
 	 && reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt))
 
-static bool vgpu_gpa_is_aperture(struct intel_vgpu *vgpu, uint64_t gpa)
-{
-	u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2);
-	u64 aperture_sz = vgpu_aperture_sz(vgpu);
-
-	return gpa >= aperture_gpa && gpa < aperture_gpa + aperture_sz;
-}
-
-static int vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t gpa,
-			    void *pdata, unsigned int size, bool is_read)
-{
-	u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2);
-	u64 offset = gpa - aperture_gpa;
-
-	if (!vgpu_gpa_is_aperture(vgpu, gpa + size - 1)) {
-		gvt_vgpu_err("Aperture rw out of range, offset %llx, size %d\n",
-			     offset, size);
-		return -EINVAL;
-	}
-
-	if (!vgpu->gm.aperture_va) {
-		gvt_vgpu_err("BAR is not enabled\n");
-		return -ENXIO;
-	}
-
-	if (is_read)
-		memcpy(pdata, vgpu->gm.aperture_va + offset, size);
-	else
-		memcpy(vgpu->gm.aperture_va + offset, pdata, size);
-	return 0;
-}
-
 static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa,
 		void *p_data, unsigned int bytes, bool read)
 {
@@ -117,25 +85,6 @@ static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa,
 		else
 			memcpy(pt, p_data, bytes);
 
-	} else if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
-		struct intel_vgpu_guest_page *gp;
-
-		/* Since we enter the failsafe mode early during guest boot,
-		 * guest may not have chance to set up its ppgtt table, so
-		 * there should not be any wp pages for guest. Keep the wp
-		 * related code here in case we need to handle it in furture.
-		 */
-		gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
-		if (gp) {
-			/* remove write protection to prevent furture traps */
-			intel_vgpu_clean_guest_page(vgpu, gp);
-			if (read)
-				intel_gvt_hypervisor_read_gpa(vgpu, pa,
-						p_data, bytes);
-			else
-				intel_gvt_hypervisor_write_gpa(vgpu, pa,
-						p_data, bytes);
-		}
 	}
 	mutex_unlock(&gvt->lock);
 }
@@ -157,37 +106,12 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
 	unsigned int offset = 0;
 	int ret = -EINVAL;
 
-
 	if (vgpu->failsafe) {
 		failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, true);
 		return 0;
 	}
 	mutex_lock(&gvt->lock);
 
-	if (vgpu_gpa_is_aperture(vgpu, pa)) {
-		ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, true);
-		mutex_unlock(&gvt->lock);
-		return ret;
-	}
-
-	if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
-		struct intel_vgpu_guest_page *gp;
-
-		gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
-		if (gp) {
-			ret = intel_gvt_hypervisor_read_gpa(vgpu, pa,
-					p_data, bytes);
-			if (ret) {
-				gvt_vgpu_err("guest page read error %d, "
-					"gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n",
-					ret, gp->gfn, pa, *(u32 *)p_data,
-					bytes);
-			}
-			mutex_unlock(&gvt->lock);
-			return ret;
-		}
-	}
-
 	offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa);
 
 	if (WARN_ON(bytes > 8))
@@ -205,14 +129,12 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
 				p_data, bytes);
 		if (ret)
 			goto err;
-		mutex_unlock(&gvt->lock);
-		return ret;
+		goto out;
 	}
 
 	if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) {
 		ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, p_data, bytes);
-		mutex_unlock(&gvt->lock);
-		return ret;
+		goto out;
 	}
 
 	if (WARN_ON(!reg_is_mmio(gvt, offset + bytes - 1)))
@@ -228,11 +150,13 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
 		goto err;
 
 	intel_gvt_mmio_set_accessed(gvt, offset);
-	mutex_unlock(&gvt->lock);
-	return 0;
+	ret = 0;
+	goto out;
+
 err:
 	gvt_vgpu_err("fail to emulate MMIO read %08x len %d\n",
 			offset, bytes);
+out:
 	mutex_unlock(&gvt->lock);
 	return ret;
 }
@@ -261,30 +185,6 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
 
 	mutex_lock(&gvt->lock);
 
-	if (vgpu_gpa_is_aperture(vgpu, pa)) {
-		ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, false);
-		mutex_unlock(&gvt->lock);
-		return ret;
-	}
-
-	if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
-		struct intel_vgpu_guest_page *gp;
-
-		gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
-		if (gp) {
-			ret = gp->handler(gp, pa, p_data, bytes);
-			if (ret) {
-				gvt_err("guest page write error %d, "
-					"gfn 0x%lx, pa 0x%llx, "
-					"var 0x%x, len %d\n",
-					ret, gp->gfn, pa,
-					*(u32 *)p_data, bytes);
-			}
-			mutex_unlock(&gvt->lock);
-			return ret;
-		}
-	}
-
 	offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa);
 
 	if (WARN_ON(bytes > 8))
@@ -302,14 +202,12 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
 				p_data, bytes);
 		if (ret)
 			goto err;
-		mutex_unlock(&gvt->lock);
-		return ret;
+		goto out;
 	}
 
 	if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) {
 		ret = intel_gvt_hypervisor_write_gpa(vgpu, pa, p_data, bytes);
-		mutex_unlock(&gvt->lock);
-		return ret;
+		goto out;
 	}
 
 	ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, false);
@@ -317,11 +215,12 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
 		goto err;
 
 	intel_gvt_mmio_set_accessed(gvt, offset);
-	mutex_unlock(&gvt->lock);
-	return 0;
+	ret = 0;
+	goto out;
 err:
 	gvt_vgpu_err("fail to emulate MMIO write %08x len %d\n", offset,
 		     bytes);
+out:
 	mutex_unlock(&gvt->lock);
 	return ret;
 }
@@ -342,10 +241,10 @@ void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr)
 		memcpy(vgpu->mmio.vreg, mmio, info->mmio_size);
 		memcpy(vgpu->mmio.sreg, mmio, info->mmio_size);
 
-		vgpu_vreg(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0;
+		vgpu_vreg_t(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0;
 
 		/* set the bit 0:2(Core C-State ) to C0 */
-		vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0;
+		vgpu_vreg_t(vgpu, GEN6_GT_CORE_STATUS) = 0;
 
 		vgpu->mmio.disable_warn_untrack = false;
 	} else {
diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
index dbc04ad2c7a1..71b620875943 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.h
+++ b/drivers/gpu/drm/i915/gvt/mmio.h
@@ -72,12 +72,9 @@ bool intel_gvt_match_device(struct intel_gvt *gvt, unsigned long device);
 
 int intel_gvt_setup_mmio_info(struct intel_gvt *gvt);
 void intel_gvt_clean_mmio_info(struct intel_gvt *gvt);
-
-#define INTEL_GVT_MMIO_OFFSET(reg) ({ \
-	typeof(reg) __reg = reg; \
-	u32 *offset = (u32 *)&__reg; \
-	*offset; \
-})
+int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt,
+	int (*handler)(struct intel_gvt *gvt, u32 offset, void *data),
+	void *data);
 
 int intel_vgpu_init_mmio(struct intel_vgpu *vgpu);
 void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr);
diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c
new file mode 100644
index 000000000000..152df3d0291e
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.c
@@ -0,0 +1,415 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Authors:
+ *    Eddie Dong <eddie.dong@intel.com>
+ *    Kevin Tian <kevin.tian@intel.com>
+ *
+ * Contributors:
+ *    Zhi Wang <zhi.a.wang@intel.com>
+ *    Changbin Du <changbin.du@intel.com>
+ *    Zhenyu Wang <zhenyuw@linux.intel.com>
+ *    Tina Zhang <tina.zhang@intel.com>
+ *    Bing Niu <bing.niu@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+#include "gvt.h"
+#include "trace.h"
+
+/**
+ * Defined in Intel Open Source PRM.
+ * Ref: https://01.org/linuxgraphics/documentation/hardware-specification-prms
+ */
+#define TRVATTL3PTRDW(i)	_MMIO(0x4de0 + (i)*4)
+#define TRNULLDETCT		_MMIO(0x4de8)
+#define TRINVTILEDETCT		_MMIO(0x4dec)
+#define TRVADR			_MMIO(0x4df0)
+#define TRTTE			_MMIO(0x4df4)
+#define RING_EXCC(base)		_MMIO((base) + 0x28)
+#define RING_GFX_MODE(base)	_MMIO((base) + 0x29c)
+#define VF_GUARDBAND		_MMIO(0x83a4)
+
+/* Raw offset is appened to each line for convenience. */
+static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = {
+	{RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
+	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+	{RCS, HWSTAM, 0x0, false}, /* 0x2098 */
+	{RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
+	{RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
+	{RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
+	{RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
+	{RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
+	{RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
+	{RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
+
+	{BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
+	{BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
+	{BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
+	{BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
+	{BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
+	{RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
+};
+
+static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = {
+	{RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
+	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+	{RCS, HWSTAM, 0x0, false}, /* 0x2098 */
+	{RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
+	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
+	{RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
+	{RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
+	{RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
+	{RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
+	{RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
+	{RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
+
+	{RCS, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */
+	{RCS, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */
+	{RCS, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */
+	{RCS, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */
+	{RCS, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */
+	{RCS, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */
+	{RCS, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */
+	{RCS, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */
+	{RCS, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */
+	{RCS, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */
+	{RCS, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */
+	{RCS, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */
+	{RCS, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */
+	{RCS, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */
+	{RCS, TRNULLDETCT, 0, false}, /* 0x4de8 */
+	{RCS, TRINVTILEDETCT, 0, false}, /* 0x4dec */
+	{RCS, TRVADR, 0, false}, /* 0x4df0 */
+	{RCS, TRTTE, 0, false}, /* 0x4df4 */
+
+	{BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
+	{BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
+	{BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
+	{BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
+	{BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
+
+	{VCS2, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */
+
+	{VECS, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */
+
+	{RCS, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */
+	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+	{RCS, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */
+	{RCS, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */
+
+	{RCS, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */
+	{RCS, GEN9_CSFE_CHICKEN1_RCS, 0x0, false}, /* 0x20d4 */
+
+	{RCS, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */
+	{RCS, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */
+	{RCS, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */
+	{RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
+};
+
+static struct {
+	bool initialized;
+	u32 control_table[I915_NUM_ENGINES][64];
+	u32 l3cc_table[32];
+} gen9_render_mocs;
+
+static void load_render_mocs(struct drm_i915_private *dev_priv)
+{
+	i915_reg_t offset;
+	u32 regs[] = {
+		[RCS] = 0xc800,
+		[VCS] = 0xc900,
+		[VCS2] = 0xca00,
+		[BCS] = 0xcc00,
+		[VECS] = 0xcb00,
+	};
+	int ring_id, i;
+
+	for (ring_id = 0; ring_id < ARRAY_SIZE(regs); ring_id++) {
+		offset.reg = regs[ring_id];
+		for (i = 0; i < 64; i++) {
+			gen9_render_mocs.control_table[ring_id][i] =
+				I915_READ_FW(offset);
+			offset.reg += 4;
+		}
+	}
+
+	offset.reg = 0xb020;
+	for (i = 0; i < 32; i++) {
+		gen9_render_mocs.l3cc_table[i] =
+			I915_READ_FW(offset);
+		offset.reg += 4;
+	}
+	gen9_render_mocs.initialized = true;
+}
+
+static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	enum forcewake_domains fw;
+	i915_reg_t reg;
+	u32 regs[] = {
+		[RCS] = 0x4260,
+		[VCS] = 0x4264,
+		[VCS2] = 0x4268,
+		[BCS] = 0x426c,
+		[VECS] = 0x4270,
+	};
+
+	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+		return;
+
+	if (!test_and_clear_bit(ring_id, (void *)s->tlb_handle_pending))
+		return;
+
+	reg = _MMIO(regs[ring_id]);
+
+	/* WaForceWakeRenderDuringMmioTLBInvalidate:skl
+	 * we need to put a forcewake when invalidating RCS TLB caches,
+	 * otherwise device can go to RC6 state and interrupt invalidation
+	 * process
+	 */
+	fw = intel_uncore_forcewake_for_reg(dev_priv, reg,
+					    FW_REG_READ | FW_REG_WRITE);
+	if (ring_id == RCS && (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)))
+		fw |= FORCEWAKE_RENDER;
+
+	intel_uncore_forcewake_get(dev_priv, fw);
+
+	I915_WRITE_FW(reg, 0x1);
+
+	if (wait_for_atomic((I915_READ_FW(reg) == 0), 50))
+		gvt_vgpu_err("timeout in invalidate ring (%d) tlb\n", ring_id);
+	else
+		vgpu_vreg_t(vgpu, reg) = 0;
+
+	intel_uncore_forcewake_put(dev_priv, fw);
+
+	gvt_dbg_core("invalidate TLB for ring %d\n", ring_id);
+}
+
+static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
+			int ring_id)
+{
+	struct drm_i915_private *dev_priv;
+	i915_reg_t offset, l3_offset;
+	u32 old_v, new_v;
+
+	u32 regs[] = {
+		[RCS] = 0xc800,
+		[VCS] = 0xc900,
+		[VCS2] = 0xca00,
+		[BCS] = 0xcc00,
+		[VECS] = 0xcb00,
+	};
+	int i;
+
+	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;
+	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
+		return;
+
+	if (!pre && !gen9_render_mocs.initialized)
+		load_render_mocs(dev_priv);
+
+	offset.reg = regs[ring_id];
+	for (i = 0; i < 64; i++) {
+		if (pre)
+			old_v = vgpu_vreg_t(pre, offset);
+		else
+			old_v = gen9_render_mocs.control_table[ring_id][i];
+		if (next)
+			new_v = vgpu_vreg_t(next, offset);
+		else
+			new_v = gen9_render_mocs.control_table[ring_id][i];
+
+		if (old_v != new_v)
+			I915_WRITE_FW(offset, new_v);
+
+		offset.reg += 4;
+	}
+
+	if (ring_id == RCS) {
+		l3_offset.reg = 0xb020;
+		for (i = 0; i < 32; i++) {
+			if (pre)
+				old_v = vgpu_vreg_t(pre, l3_offset);
+			else
+				old_v = gen9_render_mocs.l3cc_table[i];
+			if (next)
+				new_v = vgpu_vreg_t(next, l3_offset);
+			else
+				new_v = gen9_render_mocs.l3cc_table[i];
+
+			if (old_v != new_v)
+				I915_WRITE_FW(l3_offset, new_v);
+
+			l3_offset.reg += 4;
+		}
+	}
+}
+
+#define CTX_CONTEXT_CONTROL_VAL	0x03
+
+/* Switch ring mmio values (context). */
+static void switch_mmio(struct intel_vgpu *pre,
+			struct intel_vgpu *next,
+			int ring_id)
+{
+	struct drm_i915_private *dev_priv;
+	struct intel_vgpu_submission *s;
+	u32 *reg_state, ctx_ctrl;
+	u32 inhibit_mask =
+		_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
+	struct engine_mmio *mmio;
+	u32 old_v, new_v;
+
+	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;
+	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+		switch_mocs(pre, next, ring_id);
+
+	for (mmio = dev_priv->gvt->engine_mmio_list;
+	     i915_mmio_reg_valid(mmio->reg); mmio++) {
+		if (mmio->ring_id != ring_id)
+			continue;
+		// save
+		if (pre) {
+			vgpu_vreg_t(pre, mmio->reg) = I915_READ_FW(mmio->reg);
+			if (mmio->mask)
+				vgpu_vreg_t(pre, mmio->reg) &=
+						~(mmio->mask << 16);
+			old_v = vgpu_vreg_t(pre, mmio->reg);
+		} else
+			old_v = mmio->value = I915_READ_FW(mmio->reg);
+
+		// restore
+		if (next) {
+			s = &next->submission;
+			reg_state =
+				s->shadow_ctx->engine[ring_id].lrc_reg_state;
+			ctx_ctrl = reg_state[CTX_CONTEXT_CONTROL_VAL];
+			/*
+			 * if it is an inhibit context, load in_context mmio
+			 * into HW by mmio write. If it is not, skip this mmio
+			 * write.
+			 */
+			if (mmio->in_context &&
+			    (ctx_ctrl & inhibit_mask) != inhibit_mask)
+				continue;
+
+			if (mmio->mask)
+				new_v = vgpu_vreg_t(next, mmio->reg) |
+							(mmio->mask << 16);
+			else
+				new_v = vgpu_vreg_t(next, mmio->reg);
+		} else {
+			if (mmio->in_context)
+				continue;
+			if (mmio->mask)
+				new_v = mmio->value | (mmio->mask << 16);
+			else
+				new_v = mmio->value;
+		}
+
+		I915_WRITE_FW(mmio->reg, new_v);
+
+		trace_render_mmio(pre ? pre->id : 0,
+				  next ? next->id : 0,
+				  "switch",
+				  i915_mmio_reg_offset(mmio->reg),
+				  old_v, new_v);
+	}
+
+	if (next)
+		handle_tlb_pending_event(next, ring_id);
+}
+
+/**
+ * intel_gvt_switch_render_mmio - switch mmio context of specific engine
+ * @pre: the last vGPU that own the engine
+ * @next: the vGPU to switch to
+ * @ring_id: specify the engine
+ *
+ * If pre is null indicates that host own the engine. If next is null
+ * indicates that we are switching to host workload.
+ */
+void intel_gvt_switch_mmio(struct intel_vgpu *pre,
+			   struct intel_vgpu *next, int ring_id)
+{
+	struct drm_i915_private *dev_priv;
+
+	if (WARN_ON(!pre && !next))
+		return;
+
+	gvt_dbg_render("switch ring %d from %s to %s\n", ring_id,
+		       pre ? "vGPU" : "host", next ? "vGPU" : "HOST");
+
+	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;
+
+	/**
+	 * We are using raw mmio access wrapper to improve the
+	 * performace for batch mmio read/write, so we need
+	 * handle forcewake mannually.
+	 */
+	intel_runtime_pm_get(dev_priv);
+	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	switch_mmio(pre, next, ring_id);
+	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_runtime_pm_put(dev_priv);
+}
+
+/**
+ * intel_gvt_init_engine_mmio_context - Initiate the engine mmio list
+ * @gvt: GVT device
+ *
+ */
+void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt)
+{
+	if (IS_SKYLAKE(gvt->dev_priv) || IS_KABYLAKE(gvt->dev_priv))
+		gvt->engine_mmio_list = gen9_engine_mmio_list;
+	else
+		gvt->engine_mmio_list = gen8_engine_mmio_list;
+}
diff --git a/drivers/gpu/drm/i915/gvt/render.h b/drivers/gpu/drm/i915/gvt/mmio_context.h
similarity index 91%
rename from drivers/gpu/drm/i915/gvt/render.h
rename to drivers/gpu/drm/i915/gvt/mmio_context.h
index 91db1d39d28f..ca2c6a745673 100644
--- a/drivers/gpu/drm/i915/gvt/render.h
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.h
@@ -36,8 +36,17 @@
 #ifndef __GVT_RENDER_H__
 #define __GVT_RENDER_H__
 
+struct engine_mmio {
+	int ring_id;
+	i915_reg_t reg;
+	u32 mask;
+	bool in_context;
+	u32 value;
+};
+
 void intel_gvt_switch_mmio(struct intel_vgpu *pre,
 			   struct intel_vgpu *next, int ring_id);
 
+void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt);
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index f0e5487e6688..81aff4eacbfe 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -154,51 +154,53 @@ static inline unsigned long intel_gvt_hypervisor_virt_to_mfn(void *p)
 }
 
 /**
- * intel_gvt_hypervisor_set_wp_page - set a guest page to write-protected
+ * intel_gvt_hypervisor_enable - set a guest page to write-protected
  * @vgpu: a vGPU
- * @p: intel_vgpu_guest_page
+ * @t: page track data structure
  *
  * Returns:
  * Zero on success, negative error code if failed.
  */
-static inline int intel_gvt_hypervisor_set_wp_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *p)
+static inline int intel_gvt_hypervisor_enable_page_track(
+		struct intel_vgpu *vgpu,
+		struct intel_vgpu_page_track *t)
 {
 	int ret;
 
-	if (p->writeprotection)
+	if (t->tracked)
 		return 0;
 
-	ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, p->gfn);
+	ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, t->gfn);
 	if (ret)
 		return ret;
-	p->writeprotection = true;
-	atomic_inc(&vgpu->gtt.n_write_protected_guest_page);
+	t->tracked = true;
+	atomic_inc(&vgpu->gtt.n_tracked_guest_page);
 	return 0;
 }
 
 /**
- * intel_gvt_hypervisor_unset_wp_page - remove the write-protection of a
+ * intel_gvt_hypervisor_disable_page_track - remove the write-protection of a
  * guest page
  * @vgpu: a vGPU
- * @p: intel_vgpu_guest_page
+ * @t: page track data structure
  *
  * Returns:
  * Zero on success, negative error code if failed.
  */
-static inline int intel_gvt_hypervisor_unset_wp_page(struct intel_vgpu *vgpu,
-		struct intel_vgpu_guest_page *p)
+static inline int intel_gvt_hypervisor_disable_page_track(
+		struct intel_vgpu *vgpu,
+		struct intel_vgpu_page_track *t)
 {
 	int ret;
 
-	if (!p->writeprotection)
+	if (!t->tracked)
 		return 0;
 
-	ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, p->gfn);
+	ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, t->gfn);
 	if (ret)
 		return ret;
-	p->writeprotection = false;
-	atomic_dec(&vgpu->gtt.n_write_protected_guest_page);
+	t->tracked = false;
+	atomic_dec(&vgpu->gtt.n_tracked_guest_page);
 	return 0;
 }
 
@@ -292,4 +294,66 @@ static inline int intel_gvt_hypervisor_set_trap_area(
 	return intel_gvt_host.mpt->set_trap_area(vgpu->handle, start, end, map);
 }
 
+/**
+ * intel_gvt_hypervisor_set_opregion - Set opregion for guest
+ * @vgpu: a vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_set_opregion(struct intel_vgpu *vgpu)
+{
+	if (!intel_gvt_host.mpt->set_opregion)
+		return 0;
+
+	return intel_gvt_host.mpt->set_opregion(vgpu);
+}
+
+/**
+ * intel_gvt_hypervisor_get_vfio_device - increase vfio device ref count
+ * @vgpu: a vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline int intel_gvt_hypervisor_get_vfio_device(struct intel_vgpu *vgpu)
+{
+	if (!intel_gvt_host.mpt->get_vfio_device)
+		return 0;
+
+	return intel_gvt_host.mpt->get_vfio_device(vgpu);
+}
+
+/**
+ * intel_gvt_hypervisor_put_vfio_device - decrease vfio device ref count
+ * @vgpu: a vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+static inline void intel_gvt_hypervisor_put_vfio_device(struct intel_vgpu *vgpu)
+{
+	if (!intel_gvt_host.mpt->put_vfio_device)
+		return;
+
+	intel_gvt_host.mpt->put_vfio_device(vgpu);
+}
+
+/**
+ * intel_gvt_hypervisor_is_valid_gfn - check if a visible gfn
+ * @vgpu: a vGPU
+ * @gfn: guest PFN
+ *
+ * Returns:
+ * true on valid gfn, false on not.
+ */
+static inline bool intel_gvt_hypervisor_is_valid_gfn(
+		struct intel_vgpu *vgpu, unsigned long gfn)
+{
+	if (!intel_gvt_host.mpt->is_valid_gfn)
+		return true;
+
+	return intel_gvt_host.mpt->is_valid_gfn(vgpu->handle, gfn);
+}
+
 #endif /* _GVT_MPT_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c
index 311799136d7f..fa75a2eead90 100644
--- a/drivers/gpu/drm/i915/gvt/opregion.c
+++ b/drivers/gpu/drm/i915/gvt/opregion.c
@@ -25,36 +25,237 @@
 #include "i915_drv.h"
 #include "gvt.h"
 
-static int init_vgpu_opregion(struct intel_vgpu *vgpu, u32 gpa)
+/*
+ * Note: Only for GVT-g virtual VBT generation, other usage must
+ * not do like this.
+ */
+#define _INTEL_BIOS_PRIVATE
+#include "intel_vbt_defs.h"
+
+#define OPREGION_SIGNATURE "IntelGraphicsMem"
+#define MBOX_VBT      (1<<3)
+
+/* device handle */
+#define DEVICE_TYPE_CRT    0x01
+#define DEVICE_TYPE_EFP1   0x04
+#define DEVICE_TYPE_EFP2   0x40
+#define DEVICE_TYPE_EFP3   0x20
+#define DEVICE_TYPE_EFP4   0x10
+
+#define DEV_SIZE	38
+
+struct opregion_header {
+	u8 signature[16];
+	u32 size;
+	u32 opregion_ver;
+	u8 bios_ver[32];
+	u8 vbios_ver[16];
+	u8 driver_ver[16];
+	u32 mboxes;
+	u32 driver_model;
+	u32 pcon;
+	u8 dver[32];
+	u8 rsvd[124];
+} __packed;
+
+struct bdb_data_header {
+	u8 id;
+	u16 size; /* data size */
+} __packed;
+
+struct efp_child_device_config {
+	u16 handle;
+	u16 device_type;
+	u16 device_class;
+	u8 i2c_speed;
+	u8 dp_onboard_redriver; /* 158 */
+	u8 dp_ondock_redriver; /* 158 */
+	u8 hdmi_level_shifter_value:4; /* 169 */
+	u8 hdmi_max_data_rate:4; /* 204 */
+	u16 dtd_buf_ptr; /* 161 */
+	u8 edidless_efp:1; /* 161 */
+	u8 compression_enable:1; /* 198 */
+	u8 compression_method:1; /* 198 */
+	u8 ganged_edp:1; /* 202 */
+	u8 skip0:4;
+	u8 compression_structure_index:4; /* 198 */
+	u8 skip1:4;
+	u8 slave_port; /*  202 */
+	u8 skip2;
+	u8 dvo_port;
+	u8 i2c_pin; /* for add-in card */
+	u8 slave_addr; /* for add-in card */
+	u8 ddc_pin;
+	u16 edid_ptr;
+	u8 dvo_config;
+	u8 efp_docked_port:1; /* 158 */
+	u8 lane_reversal:1; /* 184 */
+	u8 onboard_lspcon:1; /* 192 */
+	u8 iboost_enable:1; /* 196 */
+	u8 hpd_invert:1; /* BXT 196 */
+	u8 slip3:3;
+	u8 hdmi_compat:1;
+	u8 dp_compat:1;
+	u8 tmds_compat:1;
+	u8 skip4:5;
+	u8 aux_channel;
+	u8 dongle_detect;
+	u8 pipe_cap:2;
+	u8 sdvo_stall:1; /* 158 */
+	u8 hpd_status:2;
+	u8 integrated_encoder:1;
+	u8 skip5:2;
+	u8 dvo_wiring;
+	u8 mipi_bridge_type; /* 171 */
+	u16 device_class_ext;
+	u8 dvo_function;
+	u8 dp_usb_type_c:1; /* 195 */
+	u8 skip6:7;
+	u8 dp_usb_type_c_2x_gpio_index; /* 195 */
+	u16 dp_usb_type_c_2x_gpio_pin; /* 195 */
+	u8 iboost_dp:4; /* 196 */
+	u8 iboost_hdmi:4; /* 196 */
+} __packed;
+
+struct vbt {
+	/* header->bdb_offset point to bdb_header offset */
+	struct vbt_header header;
+	struct bdb_header bdb_header;
+
+	struct bdb_data_header general_features_header;
+	struct bdb_general_features general_features;
+
+	struct bdb_data_header general_definitions_header;
+	struct bdb_general_definitions general_definitions;
+
+	struct efp_child_device_config child0;
+	struct efp_child_device_config child1;
+	struct efp_child_device_config child2;
+	struct efp_child_device_config child3;
+
+	struct bdb_data_header driver_features_header;
+	struct bdb_driver_features driver_features;
+};
+
+static void virt_vbt_generation(struct vbt *v)
 {
-	u8 *buf;
-	int i;
+	int num_child;
+
+	memset(v, 0, sizeof(struct vbt));
+
+	v->header.signature[0] = '$';
+	v->header.signature[1] = 'V';
+	v->header.signature[2] = 'B';
+	v->header.signature[3] = 'T';
+
+	/* there's features depending on version! */
+	v->header.version = 155;
+	v->header.header_size = sizeof(v->header);
+	v->header.vbt_size = sizeof(struct vbt) - sizeof(v->header);
+	v->header.bdb_offset = offsetof(struct vbt, bdb_header);
+
+	strcpy(&v->bdb_header.signature[0], "BIOS_DATA_BLOCK");
+	v->bdb_header.version = 186; /* child_dev_size = 38 */
+	v->bdb_header.header_size = sizeof(v->bdb_header);
+
+	v->bdb_header.bdb_size = sizeof(struct vbt) - sizeof(struct vbt_header)
+		- sizeof(struct bdb_header);
+
+	/* general features */
+	v->general_features_header.id = BDB_GENERAL_FEATURES;
+	v->general_features_header.size = sizeof(struct bdb_general_features);
+	v->general_features.int_crt_support = 0;
+	v->general_features.int_tv_support = 0;
+
+	/* child device */
+	num_child = 4; /* each port has one child */
+	v->general_definitions_header.id = BDB_GENERAL_DEFINITIONS;
+	/* size will include child devices */
+	v->general_definitions_header.size =
+		sizeof(struct bdb_general_definitions) + num_child * DEV_SIZE;
+	v->general_definitions.child_dev_size = DEV_SIZE;
+
+	/* portA */
+	v->child0.handle = DEVICE_TYPE_EFP1;
+	v->child0.device_type = DEVICE_TYPE_DP;
+	v->child0.dvo_port = DVO_PORT_DPA;
+	v->child0.aux_channel = DP_AUX_A;
+	v->child0.dp_compat = true;
+	v->child0.integrated_encoder = true;
+
+	/* portB */
+	v->child1.handle = DEVICE_TYPE_EFP2;
+	v->child1.device_type = DEVICE_TYPE_DP;
+	v->child1.dvo_port = DVO_PORT_DPB;
+	v->child1.aux_channel = DP_AUX_B;
+	v->child1.dp_compat = true;
+	v->child1.integrated_encoder = true;
+
+	/* portC */
+	v->child2.handle = DEVICE_TYPE_EFP3;
+	v->child2.device_type = DEVICE_TYPE_DP;
+	v->child2.dvo_port = DVO_PORT_DPC;
+	v->child2.aux_channel = DP_AUX_C;
+	v->child2.dp_compat = true;
+	v->child2.integrated_encoder = true;
+
+	/* portD */
+	v->child3.handle = DEVICE_TYPE_EFP4;
+	v->child3.device_type = DEVICE_TYPE_DP;
+	v->child3.dvo_port = DVO_PORT_DPD;
+	v->child3.aux_channel = DP_AUX_D;
+	v->child3.dp_compat = true;
+	v->child3.integrated_encoder = true;
+
+	/* driver features */
+	v->driver_features_header.id = BDB_DRIVER_FEATURES;
+	v->driver_features_header.size = sizeof(struct bdb_driver_features);
+	v->driver_features.lvds_config = BDB_DRIVER_FEATURE_NO_LVDS;
+}
 
-	if (WARN((vgpu_opregion(vgpu)->va),
-			"vgpu%d: opregion has been initialized already.\n",
-			vgpu->id))
-		return -EINVAL;
+/**
+ * intel_vgpu_init_opregion - initialize the stuff used to emulate opregion
+ * @vgpu: a vGPU
+ * @gpa: guest physical address of opregion
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_vgpu_init_opregion(struct intel_vgpu *vgpu)
+{
+	u8 *buf;
+	struct opregion_header *header;
+	struct vbt v;
+	const char opregion_signature[16] = OPREGION_SIGNATURE;
 
+	gvt_dbg_core("init vgpu%d opregion\n", vgpu->id);
 	vgpu_opregion(vgpu)->va = (void *)__get_free_pages(GFP_KERNEL |
 			__GFP_ZERO,
 			get_order(INTEL_GVT_OPREGION_SIZE));
-
-	if (!vgpu_opregion(vgpu)->va)
+	if (!vgpu_opregion(vgpu)->va) {
+		gvt_err("fail to get memory for vgpu virt opregion\n");
 		return -ENOMEM;
+	}
 
-	memcpy(vgpu_opregion(vgpu)->va, vgpu->gvt->opregion.opregion_va,
-	       INTEL_GVT_OPREGION_SIZE);
-
-	for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
-		vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
+	/* emulated opregion with VBT mailbox only */
+	buf = (u8 *)vgpu_opregion(vgpu)->va;
+	header = (struct opregion_header *)buf;
+	memcpy(header->signature, opregion_signature,
+	       sizeof(opregion_signature));
+	header->size = 0x8;
+	header->opregion_ver = 0x02000000;
+	header->mboxes = MBOX_VBT;
 
 	/* for unknown reason, the value in LID field is incorrect
 	 * which block the windows guest, so workaround it by force
 	 * setting it to "OPEN"
 	 */
-	buf = (u8 *)vgpu_opregion(vgpu)->va;
 	buf[INTEL_GVT_OPREGION_CLID] = 0x3;
 
+	/* emulated vbt from virt vbt generation */
+	virt_vbt_generation(&v);
+	memcpy(buf + INTEL_GVT_OPREGION_VBT_OFFSET, &v, sizeof(struct vbt));
+
 	return 0;
 }
 
@@ -79,93 +280,80 @@ static int map_vgpu_opregion(struct intel_vgpu *vgpu, bool map)
 			return ret;
 		}
 	}
-	return 0;
-}
 
-/**
- * intel_vgpu_clean_opregion - clean the stuff used to emulate opregion
- * @vgpu: a vGPU
- *
- */
-void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu)
-{
-	gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id);
-
-	if (!vgpu_opregion(vgpu)->va)
-		return;
-
-	if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) {
-		map_vgpu_opregion(vgpu, false);
-		free_pages((unsigned long)vgpu_opregion(vgpu)->va,
-				get_order(INTEL_GVT_OPREGION_SIZE));
+	vgpu_opregion(vgpu)->mapped = map;
 
-		vgpu_opregion(vgpu)->va = NULL;
-	}
+	return 0;
 }
 
 /**
- * intel_vgpu_init_opregion - initialize the stuff used to emulate opregion
+ * intel_vgpu_opregion_base_write_handler - Opregion base register write handler
+ *
  * @vgpu: a vGPU
  * @gpa: guest physical address of opregion
  *
  * Returns:
  * Zero on success, negative error code if failed.
  */
-int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa)
+int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa)
 {
-	int ret;
 
-	gvt_dbg_core("vgpu%d: init vgpu opregion\n", vgpu->id);
+	int i, ret = 0;
 
-	if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) {
-		gvt_dbg_core("emulate opregion from kernel\n");
+	gvt_dbg_core("emulate opregion from kernel\n");
 
-		ret = init_vgpu_opregion(vgpu, gpa);
-		if (ret)
-			return ret;
+	switch (intel_gvt_host.hypervisor_type) {
+	case INTEL_GVT_HYPERVISOR_KVM:
+		for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
+			vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
+		break;
+	case INTEL_GVT_HYPERVISOR_XEN:
+		/**
+		 * Wins guest on Xengt will write this register twice: xen
+		 * hvmloader and windows graphic driver.
+		 */
+		if (vgpu_opregion(vgpu)->mapped)
+			map_vgpu_opregion(vgpu, false);
+
+		for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
+			vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
 
 		ret = map_vgpu_opregion(vgpu, true);
-		if (ret)
-			return ret;
+		break;
+	default:
+		ret = -EINVAL;
+		gvt_vgpu_err("not supported hypervisor\n");
 	}
 
-	return 0;
-}
-
-/**
- * intel_gvt_clean_opregion - clean host opergion related stuffs
- * @gvt: a GVT device
- *
- */
-void intel_gvt_clean_opregion(struct intel_gvt *gvt)
-{
-	memunmap(gvt->opregion.opregion_va);
-	gvt->opregion.opregion_va = NULL;
+	return ret;
 }
 
 /**
- * intel_gvt_init_opregion - initialize host opergion related stuffs
- * @gvt: a GVT device
+ * intel_vgpu_clean_opregion - clean the stuff used to emulate opregion
+ * @vgpu: a vGPU
  *
- * Returns:
- * Zero on success, negative error code if failed.
  */
-int intel_gvt_init_opregion(struct intel_gvt *gvt)
+void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu)
 {
-	gvt_dbg_core("init host opregion\n");
+	gvt_dbg_core("vgpu%d: clean vgpu opregion\n", vgpu->id);
 
-	pci_read_config_dword(gvt->dev_priv->drm.pdev, INTEL_GVT_PCI_OPREGION,
-			&gvt->opregion.opregion_pa);
+	if (!vgpu_opregion(vgpu)->va)
+		return;
 
-	gvt->opregion.opregion_va = memremap(gvt->opregion.opregion_pa,
-					     INTEL_GVT_OPREGION_SIZE, MEMREMAP_WB);
-	if (!gvt->opregion.opregion_va) {
-		gvt_err("fail to map host opregion\n");
-		return -EFAULT;
+	if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_XEN) {
+		if (vgpu_opregion(vgpu)->mapped)
+			map_vgpu_opregion(vgpu, false);
+	} else if (intel_gvt_host.hypervisor_type == INTEL_GVT_HYPERVISOR_KVM) {
+		/* Guest opregion is released by VFIO */
 	}
-	return 0;
+	free_pages((unsigned long)vgpu_opregion(vgpu)->va,
+		   get_order(INTEL_GVT_OPREGION_SIZE));
+
+	vgpu_opregion(vgpu)->va = NULL;
+
 }
 
+
 #define GVT_OPREGION_FUNC(scic)					\
 	({							\
 	 u32 __ret;						\
@@ -281,11 +469,45 @@ static bool querying_capabilities(u32 scic)
  */
 int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci)
 {
-	u32 *scic, *parm;
+	u32 scic, parm;
 	u32 func, subfunc;
+	u64 scic_pa = 0, parm_pa = 0;
+	int ret;
+
+	switch (intel_gvt_host.hypervisor_type) {
+	case INTEL_GVT_HYPERVISOR_XEN:
+		scic = *((u32 *)vgpu_opregion(vgpu)->va +
+					INTEL_GVT_OPREGION_SCIC);
+		parm = *((u32 *)vgpu_opregion(vgpu)->va +
+					INTEL_GVT_OPREGION_PARM);
+		break;
+	case INTEL_GVT_HYPERVISOR_KVM:
+		scic_pa = (vgpu_opregion(vgpu)->gfn[0] << PAGE_SHIFT) +
+					INTEL_GVT_OPREGION_SCIC;
+		parm_pa = (vgpu_opregion(vgpu)->gfn[0] << PAGE_SHIFT) +
+					INTEL_GVT_OPREGION_PARM;
+
+		ret = intel_gvt_hypervisor_read_gpa(vgpu, scic_pa,
+						    &scic, sizeof(scic));
+		if (ret) {
+			gvt_vgpu_err("guest opregion read error %d, gpa 0x%llx, len %lu\n",
+				ret, scic_pa, sizeof(scic));
+			return ret;
+		}
 
-	scic = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_SCIC;
-	parm = vgpu_opregion(vgpu)->va + INTEL_GVT_OPREGION_PARM;
+		ret = intel_gvt_hypervisor_read_gpa(vgpu, parm_pa,
+						    &parm, sizeof(parm));
+		if (ret) {
+			gvt_vgpu_err("guest opregion read error %d, gpa 0x%llx, len %lu\n",
+				ret, scic_pa, sizeof(scic));
+			return ret;
+		}
+
+		break;
+	default:
+		gvt_vgpu_err("not supported hypervisor\n");
+		return -EINVAL;
+	}
 
 	if (!(swsci & SWSCI_SCI_SELECT)) {
 		gvt_vgpu_err("requesting SMI service\n");
@@ -298,9 +520,9 @@ int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci)
 		return 0;
 	}
 
-	func = GVT_OPREGION_FUNC(*scic);
-	subfunc = GVT_OPREGION_SUBFUNC(*scic);
-	if (!querying_capabilities(*scic)) {
+	func = GVT_OPREGION_FUNC(scic);
+	subfunc = GVT_OPREGION_SUBFUNC(scic);
+	if (!querying_capabilities(scic)) {
 		gvt_vgpu_err("requesting runtime service: func \"%s\","
 				" subfunc \"%s\"\n",
 				opregion_func_name(func),
@@ -309,11 +531,43 @@ int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci)
 		 * emulate exit status of function call, '0' means
 		 * "failure, generic, unsupported or unknown cause"
 		 */
-		*scic &= ~OPREGION_SCIC_EXIT_MASK;
-		return 0;
+		scic &= ~OPREGION_SCIC_EXIT_MASK;
+		goto out;
+	}
+
+	scic = 0;
+	parm = 0;
+
+out:
+	switch (intel_gvt_host.hypervisor_type) {
+	case INTEL_GVT_HYPERVISOR_XEN:
+		*((u32 *)vgpu_opregion(vgpu)->va +
+					INTEL_GVT_OPREGION_SCIC) = scic;
+		*((u32 *)vgpu_opregion(vgpu)->va +
+					INTEL_GVT_OPREGION_PARM) = parm;
+		break;
+	case INTEL_GVT_HYPERVISOR_KVM:
+		ret = intel_gvt_hypervisor_write_gpa(vgpu, scic_pa,
+						    &scic, sizeof(scic));
+		if (ret) {
+			gvt_vgpu_err("guest opregion write error %d, gpa 0x%llx, len %lu\n",
+				ret, scic_pa, sizeof(scic));
+			return ret;
+		}
+
+		ret = intel_gvt_hypervisor_write_gpa(vgpu, parm_pa,
+						    &parm, sizeof(parm));
+		if (ret) {
+			gvt_vgpu_err("guest opregion write error %d, gpa 0x%llx, len %lu\n",
+				ret, scic_pa, sizeof(scic));
+			return ret;
+		}
+
+		break;
+	default:
+		gvt_vgpu_err("not supported hypervisor\n");
+		return -EINVAL;
 	}
 
-	*scic = 0;
-	*parm = 0;
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
index 7d01c77a0f7a..d4f7ce6dc1d7 100644
--- a/drivers/gpu/drm/i915/gvt/reg.h
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -51,6 +51,9 @@
 
 #define INTEL_GVT_OPREGION_PAGES	2
 #define INTEL_GVT_OPREGION_SIZE		(INTEL_GVT_OPREGION_PAGES * PAGE_SIZE)
+#define INTEL_GVT_OPREGION_VBT_OFFSET	0x400
+#define INTEL_GVT_OPREGION_VBT_SIZE	\
+		(INTEL_GVT_OPREGION_SIZE - INTEL_GVT_OPREGION_VBT_OFFSET)
 
 #define VGT_SPRSTRIDE(pipe)	_PIPE(pipe, _SPRA_STRIDE, _PLANE_STRIDE_2_B)
 
@@ -71,6 +74,7 @@
 #define RB_HEAD_OFF_MASK	((1U << 21) - (1U << 2))
 #define RB_TAIL_OFF_MASK	((1U << 21) - (1U << 3))
 #define RB_TAIL_SIZE_MASK	((1U << 21) - (1U << 12))
-#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + GTT_PAGE_SIZE)
+#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + \
+		I915_GTT_PAGE_SIZE)
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c
deleted file mode 100644
index 6d066cf35478..000000000000
--- a/drivers/gpu/drm/i915/gvt/render.c
+++ /dev/null
@@ -1,405 +0,0 @@
-/*
- * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * Authors:
- *    Eddie Dong <eddie.dong@intel.com>
- *    Kevin Tian <kevin.tian@intel.com>
- *
- * Contributors:
- *    Zhi Wang <zhi.a.wang@intel.com>
- *    Changbin Du <changbin.du@intel.com>
- *    Zhenyu Wang <zhenyuw@linux.intel.com>
- *    Tina Zhang <tina.zhang@intel.com>
- *    Bing Niu <bing.niu@intel.com>
- *
- */
-
-#include "i915_drv.h"
-#include "gvt.h"
-#include "trace.h"
-
-struct render_mmio {
-	int ring_id;
-	i915_reg_t reg;
-	u32 mask;
-	bool in_context;
-	u32 value;
-};
-
-static struct render_mmio gen8_render_mmio_list[] __cacheline_aligned = {
-	{RCS, _MMIO(0x229c), 0xffff, false},
-	{RCS, _MMIO(0x2248), 0x0, false},
-	{RCS, _MMIO(0x2098), 0x0, false},
-	{RCS, _MMIO(0x20c0), 0xffff, true},
-	{RCS, _MMIO(0x24d0), 0, false},
-	{RCS, _MMIO(0x24d4), 0, false},
-	{RCS, _MMIO(0x24d8), 0, false},
-	{RCS, _MMIO(0x24dc), 0, false},
-	{RCS, _MMIO(0x24e0), 0, false},
-	{RCS, _MMIO(0x24e4), 0, false},
-	{RCS, _MMIO(0x24e8), 0, false},
-	{RCS, _MMIO(0x24ec), 0, false},
-	{RCS, _MMIO(0x24f0), 0, false},
-	{RCS, _MMIO(0x24f4), 0, false},
-	{RCS, _MMIO(0x24f8), 0, false},
-	{RCS, _MMIO(0x24fc), 0, false},
-	{RCS, _MMIO(0x7004), 0xffff, true},
-	{RCS, _MMIO(0x7008), 0xffff, true},
-	{RCS, _MMIO(0x7000), 0xffff, true},
-	{RCS, _MMIO(0x7010), 0xffff, true},
-	{RCS, _MMIO(0x7300), 0xffff, true},
-	{RCS, _MMIO(0x83a4), 0xffff, true},
-
-	{BCS, _MMIO(0x2229c), 0xffff, false},
-	{BCS, _MMIO(0x2209c), 0xffff, false},
-	{BCS, _MMIO(0x220c0), 0xffff, false},
-	{BCS, _MMIO(0x22098), 0x0, false},
-	{BCS, _MMIO(0x22028), 0x0, false},
-};
-
-static struct render_mmio gen9_render_mmio_list[] __cacheline_aligned = {
-	{RCS, _MMIO(0x229c), 0xffff, false},
-	{RCS, _MMIO(0x2248), 0x0, false},
-	{RCS, _MMIO(0x2098), 0x0, false},
-	{RCS, _MMIO(0x20c0), 0xffff, true},
-	{RCS, _MMIO(0x24d0), 0, false},
-	{RCS, _MMIO(0x24d4), 0, false},
-	{RCS, _MMIO(0x24d8), 0, false},
-	{RCS, _MMIO(0x24dc), 0, false},
-	{RCS, _MMIO(0x24e0), 0, false},
-	{RCS, _MMIO(0x24e4), 0, false},
-	{RCS, _MMIO(0x24e8), 0, false},
-	{RCS, _MMIO(0x24ec), 0, false},
-	{RCS, _MMIO(0x24f0), 0, false},
-	{RCS, _MMIO(0x24f4), 0, false},
-	{RCS, _MMIO(0x24f8), 0, false},
-	{RCS, _MMIO(0x24fc), 0, false},
-	{RCS, _MMIO(0x7004), 0xffff, true},
-	{RCS, _MMIO(0x7008), 0xffff, true},
-	{RCS, _MMIO(0x7000), 0xffff, true},
-	{RCS, _MMIO(0x7010), 0xffff, true},
-	{RCS, _MMIO(0x7300), 0xffff, true},
-	{RCS, _MMIO(0x83a4), 0xffff, true},
-
-	{RCS, _MMIO(0x40e0), 0, false},
-	{RCS, _MMIO(0x40e4), 0, false},
-	{RCS, _MMIO(0x2580), 0xffff, true},
-	{RCS, _MMIO(0x7014), 0xffff, true},
-	{RCS, _MMIO(0x20ec), 0xffff, false},
-	{RCS, _MMIO(0xb118), 0, false},
-	{RCS, _MMIO(0xe100), 0xffff, true},
-	{RCS, _MMIO(0xe180), 0xffff, true},
-	{RCS, _MMIO(0xe184), 0xffff, true},
-	{RCS, _MMIO(0xe188), 0xffff, true},
-	{RCS, _MMIO(0xe194), 0xffff, true},
-	{RCS, _MMIO(0x4de0), 0, false},
-	{RCS, _MMIO(0x4de4), 0, false},
-	{RCS, _MMIO(0x4de8), 0, false},
-	{RCS, _MMIO(0x4dec), 0, false},
-	{RCS, _MMIO(0x4df0), 0, false},
-	{RCS, _MMIO(0x4df4), 0, false},
-
-	{BCS, _MMIO(0x2229c), 0xffff, false},
-	{BCS, _MMIO(0x2209c), 0xffff, false},
-	{BCS, _MMIO(0x220c0), 0xffff, false},
-	{BCS, _MMIO(0x22098), 0x0, false},
-	{BCS, _MMIO(0x22028), 0x0, false},
-
-	{VCS2, _MMIO(0x1c028), 0xffff, false},
-
-	{VECS, _MMIO(0x1a028), 0xffff, false},
-
-	{RCS, _MMIO(0x7304), 0xffff, true},
-	{RCS, _MMIO(0x2248), 0x0, false},
-	{RCS, _MMIO(0x940c), 0x0, false},
-	{RCS, _MMIO(0x4ab8), 0x0, false},
-
-	{RCS, _MMIO(0x4ab0), 0x0, false},
-	{RCS, _MMIO(0x20d4), 0x0, false},
-
-	{RCS, _MMIO(0xb004), 0x0, false},
-	{RCS, _MMIO(0x20a0), 0x0, false},
-	{RCS, _MMIO(0x20e4), 0xffff, false},
-};
-
-static u32 gen9_render_mocs[I915_NUM_ENGINES][64];
-static u32 gen9_render_mocs_L3[32];
-
-static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
-{
-	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	enum forcewake_domains fw;
-	i915_reg_t reg;
-	u32 regs[] = {
-		[RCS] = 0x4260,
-		[VCS] = 0x4264,
-		[VCS2] = 0x4268,
-		[BCS] = 0x426c,
-		[VECS] = 0x4270,
-	};
-
-	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
-		return;
-
-	if (!test_and_clear_bit(ring_id, (void *)vgpu->tlb_handle_pending))
-		return;
-
-	reg = _MMIO(regs[ring_id]);
-
-	/* WaForceWakeRenderDuringMmioTLBInvalidate:skl
-	 * we need to put a forcewake when invalidating RCS TLB caches,
-	 * otherwise device can go to RC6 state and interrupt invalidation
-	 * process
-	 */
-	fw = intel_uncore_forcewake_for_reg(dev_priv, reg,
-					    FW_REG_READ | FW_REG_WRITE);
-	if (ring_id == RCS && (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)))
-		fw |= FORCEWAKE_RENDER;
-
-	intel_uncore_forcewake_get(dev_priv, fw);
-
-	I915_WRITE_FW(reg, 0x1);
-
-	if (wait_for_atomic((I915_READ_FW(reg) == 0), 50))
-		gvt_vgpu_err("timeout in invalidate ring (%d) tlb\n", ring_id);
-	else
-		vgpu_vreg(vgpu, regs[ring_id]) = 0;
-
-	intel_uncore_forcewake_put(dev_priv, fw);
-
-	gvt_dbg_core("invalidate TLB for ring %d\n", ring_id);
-}
-
-static void load_mocs(struct intel_vgpu *vgpu, int ring_id)
-{
-	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	i915_reg_t offset, l3_offset;
-	u32 regs[] = {
-		[RCS] = 0xc800,
-		[VCS] = 0xc900,
-		[VCS2] = 0xca00,
-		[BCS] = 0xcc00,
-		[VECS] = 0xcb00,
-	};
-	int i;
-
-	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
-		return;
-
-	offset.reg = regs[ring_id];
-	for (i = 0; i < 64; i++) {
-		gen9_render_mocs[ring_id][i] = I915_READ_FW(offset);
-		I915_WRITE(offset, vgpu_vreg(vgpu, offset));
-		offset.reg += 4;
-	}
-
-	if (ring_id == RCS) {
-		l3_offset.reg = 0xb020;
-		for (i = 0; i < 32; i++) {
-			gen9_render_mocs_L3[i] = I915_READ_FW(l3_offset);
-			I915_WRITE_FW(l3_offset, vgpu_vreg(vgpu, l3_offset));
-			l3_offset.reg += 4;
-		}
-	}
-}
-
-static void restore_mocs(struct intel_vgpu *vgpu, int ring_id)
-{
-	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	i915_reg_t offset, l3_offset;
-	u32 regs[] = {
-		[RCS] = 0xc800,
-		[VCS] = 0xc900,
-		[VCS2] = 0xca00,
-		[BCS] = 0xcc00,
-		[VECS] = 0xcb00,
-	};
-	int i;
-
-	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
-		return;
-
-	offset.reg = regs[ring_id];
-	for (i = 0; i < 64; i++) {
-		vgpu_vreg(vgpu, offset) = I915_READ_FW(offset);
-		I915_WRITE_FW(offset, gen9_render_mocs[ring_id][i]);
-		offset.reg += 4;
-	}
-
-	if (ring_id == RCS) {
-		l3_offset.reg = 0xb020;
-		for (i = 0; i < 32; i++) {
-			vgpu_vreg(vgpu, l3_offset) = I915_READ_FW(l3_offset);
-			I915_WRITE_FW(l3_offset, gen9_render_mocs_L3[i]);
-			l3_offset.reg += 4;
-		}
-	}
-}
-
-#define CTX_CONTEXT_CONTROL_VAL	0x03
-
-/* Switch ring mmio values (context) from host to a vgpu. */
-static void switch_mmio_to_vgpu(struct intel_vgpu *vgpu, int ring_id)
-{
-	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	struct render_mmio *mmio;
-	u32 v;
-	int i, array_size;
-	u32 *reg_state = vgpu->shadow_ctx->engine[ring_id].lrc_reg_state;
-	u32 ctx_ctrl = reg_state[CTX_CONTEXT_CONTROL_VAL];
-	u32 inhibit_mask =
-		_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
-	i915_reg_t last_reg = _MMIO(0);
-
-	if (IS_SKYLAKE(vgpu->gvt->dev_priv)
-		|| IS_KABYLAKE(vgpu->gvt->dev_priv)) {
-		mmio = gen9_render_mmio_list;
-		array_size = ARRAY_SIZE(gen9_render_mmio_list);
-		load_mocs(vgpu, ring_id);
-	} else {
-		mmio = gen8_render_mmio_list;
-		array_size = ARRAY_SIZE(gen8_render_mmio_list);
-	}
-
-	for (i = 0; i < array_size; i++, mmio++) {
-		if (mmio->ring_id != ring_id)
-			continue;
-
-		mmio->value = I915_READ_FW(mmio->reg);
-
-		/*
-		 * if it is an inhibit context, load in_context mmio
-		 * into HW by mmio write. If it is not, skip this mmio
-		 * write.
-		 */
-		if (mmio->in_context &&
-				((ctx_ctrl & inhibit_mask) != inhibit_mask) &&
-				i915_modparams.enable_execlists)
-			continue;
-
-		if (mmio->mask)
-			v = vgpu_vreg(vgpu, mmio->reg) | (mmio->mask << 16);
-		else
-			v = vgpu_vreg(vgpu, mmio->reg);
-
-		I915_WRITE_FW(mmio->reg, v);
-		last_reg = mmio->reg;
-
-		trace_render_mmio(vgpu->id, "load",
-				  i915_mmio_reg_offset(mmio->reg),
-				  mmio->value, v);
-	}
-
-	/* Make sure the swiched MMIOs has taken effect. */
-	if (likely(INTEL_GVT_MMIO_OFFSET(last_reg)))
-		I915_READ_FW(last_reg);
-
-	handle_tlb_pending_event(vgpu, ring_id);
-}
-
-/* Switch ring mmio values (context) from vgpu to host. */
-static void switch_mmio_to_host(struct intel_vgpu *vgpu, int ring_id)
-{
-	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	struct render_mmio *mmio;
-	i915_reg_t last_reg = _MMIO(0);
-	u32 v;
-	int i, array_size;
-
-	if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
-		mmio = gen9_render_mmio_list;
-		array_size = ARRAY_SIZE(gen9_render_mmio_list);
-		restore_mocs(vgpu, ring_id);
-	} else {
-		mmio = gen8_render_mmio_list;
-		array_size = ARRAY_SIZE(gen8_render_mmio_list);
-	}
-
-	for (i = 0; i < array_size; i++, mmio++) {
-		if (mmio->ring_id != ring_id)
-			continue;
-
-		vgpu_vreg(vgpu, mmio->reg) = I915_READ_FW(mmio->reg);
-
-		if (mmio->mask) {
-			vgpu_vreg(vgpu, mmio->reg) &= ~(mmio->mask << 16);
-			v = mmio->value | (mmio->mask << 16);
-		} else
-			v = mmio->value;
-
-		if (mmio->in_context)
-			continue;
-
-		I915_WRITE_FW(mmio->reg, v);
-		last_reg = mmio->reg;
-
-		trace_render_mmio(vgpu->id, "restore",
-				  i915_mmio_reg_offset(mmio->reg),
-				  mmio->value, v);
-	}
-
-	/* Make sure the swiched MMIOs has taken effect. */
-	if (likely(INTEL_GVT_MMIO_OFFSET(last_reg)))
-		I915_READ_FW(last_reg);
-}
-
-/**
- * intel_gvt_switch_render_mmio - switch mmio context of specific engine
- * @pre: the last vGPU that own the engine
- * @next: the vGPU to switch to
- * @ring_id: specify the engine
- *
- * If pre is null indicates that host own the engine. If next is null
- * indicates that we are switching to host workload.
- */
-void intel_gvt_switch_mmio(struct intel_vgpu *pre,
-			   struct intel_vgpu *next, int ring_id)
-{
-	struct drm_i915_private *dev_priv;
-
-	if (WARN_ON(!pre && !next))
-		return;
-
-	gvt_dbg_render("switch ring %d from %s to %s\n", ring_id,
-		       pre ? "vGPU" : "host", next ? "vGPU" : "HOST");
-
-	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;
-
-	/**
-	 * We are using raw mmio access wrapper to improve the
-	 * performace for batch mmio read/write, so we need
-	 * handle forcewake mannually.
-	 */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
-
-	/**
-	 * TODO: Optimize for vGPU to vGPU switch by merging
-	 * switch_mmio_to_host() and switch_mmio_to_vgpu().
-	 */
-	if (pre)
-		switch_mmio_to_host(pre, ring_id);
-
-	if (next)
-		switch_mmio_to_vgpu(next, ring_id);
-
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
-}
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.c b/drivers/gpu/drm/i915/gvt/sched_policy.c
index 03532dfc0cd5..cc1ce361cd76 100644
--- a/drivers/gpu/drm/i915/gvt/sched_policy.c
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.c
@@ -50,6 +50,7 @@ static bool vgpu_has_pending_workload(struct intel_vgpu *vgpu)
 struct vgpu_sched_data {
 	struct list_head lru_list;
 	struct intel_vgpu *vgpu;
+	bool active;
 
 	ktime_t sched_in_time;
 	ktime_t sched_out_time;
@@ -308,8 +309,15 @@ static int tbs_sched_init_vgpu(struct intel_vgpu *vgpu)
 
 static void tbs_sched_clean_vgpu(struct intel_vgpu *vgpu)
 {
+	struct intel_gvt *gvt = vgpu->gvt;
+	struct gvt_sched_data *sched_data = gvt->scheduler.sched_data;
+
 	kfree(vgpu->sched_data);
 	vgpu->sched_data = NULL;
+
+	/* this vgpu id has been removed */
+	if (idr_is_empty(&gvt->vgpu_idr))
+		hrtimer_cancel(&sched_data->timer);
 }
 
 static void tbs_sched_start_schedule(struct intel_vgpu *vgpu)
@@ -325,6 +333,7 @@ static void tbs_sched_start_schedule(struct intel_vgpu *vgpu)
 	if (!hrtimer_active(&sched_data->timer))
 		hrtimer_start(&sched_data->timer, ktime_add_ns(ktime_get(),
 			sched_data->period), HRTIMER_MODE_ABS);
+	vgpu_data->active = true;
 }
 
 static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu)
@@ -332,6 +341,7 @@ static void tbs_sched_stop_schedule(struct intel_vgpu *vgpu)
 	struct vgpu_sched_data *vgpu_data = vgpu->sched_data;
 
 	list_del_init(&vgpu_data->lru_list);
+	vgpu_data->active = false;
 }
 
 static struct intel_gvt_sched_policy_ops tbs_schedule_ops = {
@@ -367,9 +377,17 @@ void intel_vgpu_clean_sched_policy(struct intel_vgpu *vgpu)
 
 void intel_vgpu_start_schedule(struct intel_vgpu *vgpu)
 {
-	gvt_dbg_core("vgpu%d: start schedule\n", vgpu->id);
+	struct vgpu_sched_data *vgpu_data = vgpu->sched_data;
 
-	vgpu->gvt->scheduler.sched_ops->start_schedule(vgpu);
+	if (!vgpu_data->active) {
+		gvt_dbg_core("vgpu%d: start schedule\n", vgpu->id);
+		vgpu->gvt->scheduler.sched_ops->start_schedule(vgpu);
+	}
+}
+
+void intel_gvt_kick_schedule(struct intel_gvt *gvt)
+{
+	intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED);
 }
 
 void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu)
@@ -377,6 +395,10 @@ void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu)
 	struct intel_gvt_workload_scheduler *scheduler =
 		&vgpu->gvt->scheduler;
 	int ring_id;
+	struct vgpu_sched_data *vgpu_data = vgpu->sched_data;
+
+	if (!vgpu_data->active)
+		return;
 
 	gvt_dbg_core("vgpu%d: stop schedule\n", vgpu->id);
 
diff --git a/drivers/gpu/drm/i915/gvt/sched_policy.h b/drivers/gpu/drm/i915/gvt/sched_policy.h
index ba00a5f7455f..7b59e3e88b8b 100644
--- a/drivers/gpu/drm/i915/gvt/sched_policy.h
+++ b/drivers/gpu/drm/i915/gvt/sched_policy.h
@@ -57,4 +57,6 @@ void intel_vgpu_start_schedule(struct intel_vgpu *vgpu);
 
 void intel_vgpu_stop_schedule(struct intel_vgpu *vgpu);
 
+void intel_gvt_kick_schedule(struct intel_gvt *gvt);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 69f8f0d155b9..d74d6f05c62c 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -52,12 +52,60 @@ static void set_context_pdp_root_pointer(
 		pdp_pair[i].val = pdp[7 - i];
 }
 
+/*
+ * when populating shadow ctx from guest, we should not overrride oa related
+ * registers, so that they will not be overlapped by guest oa configs. Thus
+ * made it possible to capture oa data from host for both host and guests.
+ */
+static void sr_oa_regs(struct intel_vgpu_workload *workload,
+		u32 *reg_state, bool save)
+{
+	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
+	u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset;
+	u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset;
+	int i = 0;
+	u32 flex_mmio[] = {
+		i915_mmio_reg_offset(EU_PERF_CNTL0),
+		i915_mmio_reg_offset(EU_PERF_CNTL1),
+		i915_mmio_reg_offset(EU_PERF_CNTL2),
+		i915_mmio_reg_offset(EU_PERF_CNTL3),
+		i915_mmio_reg_offset(EU_PERF_CNTL4),
+		i915_mmio_reg_offset(EU_PERF_CNTL5),
+		i915_mmio_reg_offset(EU_PERF_CNTL6),
+	};
+
+	if (!workload || !reg_state || workload->ring_id != RCS)
+		return;
+
+	if (save) {
+		workload->oactxctrl = reg_state[ctx_oactxctrl + 1];
+
+		for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
+			u32 state_offset = ctx_flexeu0 + i * 2;
+
+			workload->flex_mmio[i] = reg_state[state_offset + 1];
+		}
+	} else {
+		reg_state[ctx_oactxctrl] =
+			i915_mmio_reg_offset(GEN8_OACTXCONTROL);
+		reg_state[ctx_oactxctrl + 1] = workload->oactxctrl;
+
+		for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
+			u32 state_offset = ctx_flexeu0 + i * 2;
+			u32 mmio = flex_mmio[i];
+
+			reg_state[state_offset] = mmio;
+			reg_state[state_offset + 1] = workload->flex_mmio[i];
+		}
+	}
+}
+
 static int populate_shadow_context(struct intel_vgpu_workload *workload)
 {
 	struct intel_vgpu *vgpu = workload->vgpu;
 	struct intel_gvt *gvt = vgpu->gvt;
 	int ring_id = workload->ring_id;
-	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+	struct i915_gem_context *shadow_ctx = vgpu->submission.shadow_ctx;
 	struct drm_i915_gem_object *ctx_obj =
 		shadow_ctx->engine[ring_id].state->obj;
 	struct execlist_ring_context *shadow_ring_context;
@@ -81,16 +129,16 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 	while (i < context_page_num) {
 		context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
 				(u32)((workload->ctx_desc.lrca + i) <<
-				GTT_PAGE_SHIFT));
+				I915_GTT_PAGE_SHIFT));
 		if (context_gpa == INTEL_GVT_INVALID_ADDR) {
 			gvt_vgpu_err("Invalid guest context descriptor\n");
-			return -EINVAL;
+			return -EFAULT;
 		}
 
 		page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
 		dst = kmap(page);
 		intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
-				GTT_PAGE_SIZE);
+				I915_GTT_PAGE_SIZE);
 		kunmap(page);
 		i++;
 	}
@@ -98,6 +146,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
 	shadow_ring_context = kmap(page);
 
+	sr_oa_regs(workload, (u32 *)shadow_ring_context, true);
 #define COPY_REG(name) \
 	intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
 		+ RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
@@ -120,8 +169,9 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 			sizeof(*shadow_ring_context),
 			(void *)shadow_ring_context +
 			sizeof(*shadow_ring_context),
-			GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+			I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 
+	sr_oa_regs(workload, (u32 *)shadow_ring_context, false);
 	kunmap(page);
 	return 0;
 }
@@ -248,7 +298,7 @@ static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)
 	return 0;
 }
 
-void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
 {
 	if (!wa_ctx->indirect_ctx.obj)
 		return;
@@ -267,11 +317,12 @@ void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
  */
 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 {
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct i915_gem_context *shadow_ctx = s->shadow_ctx;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	int ring_id = workload->ring_id;
-	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
-	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine = dev_priv->engine[ring_id];
-	struct intel_vgpu *vgpu = workload->vgpu;
 	struct intel_ring *ring;
 	int ret;
 
@@ -284,7 +335,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 	shadow_ctx->desc_template |= workload->ctx_desc.addressing_mode <<
 				    GEN8_CTX_ADDRESSING_MODE_SHIFT;
 
-	if (!test_and_set_bit(ring_id, vgpu->shadow_ctx_desc_updated))
+	if (!test_and_set_bit(ring_id, s->shadow_ctx_desc_updated))
 		shadow_context_descriptor_update(shadow_ctx,
 					dev_priv->engine[ring_id]);
 
@@ -327,14 +378,15 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 	return ret;
 }
 
-int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
+static int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
 {
 	int ring_id = workload->ring_id;
 	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine = dev_priv->engine[ring_id];
 	struct drm_i915_gem_request *rq;
 	struct intel_vgpu *vgpu = workload->vgpu;
-	struct i915_gem_context *shadow_ctx = vgpu->shadow_ctx;
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct i915_gem_context *shadow_ctx = s->shadow_ctx;
 	int ret;
 
 	rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
@@ -358,11 +410,214 @@ int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
 	return ret;
 }
 
+static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload);
+
+static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+	struct intel_gvt *gvt = workload->vgpu->gvt;
+	const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
+	struct intel_vgpu_shadow_bb *bb;
+	int ret;
+
+	list_for_each_entry(bb, &workload->shadow_bb, list) {
+		bb->vma = i915_gem_object_ggtt_pin(bb->obj, NULL, 0, 0, 0);
+		if (IS_ERR(bb->vma)) {
+			ret = PTR_ERR(bb->vma);
+			goto err;
+		}
+
+		/* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va
+		 * is only updated into ring_scan_buffer, not real ring address
+		 * allocated in later copy_workload_to_ring_buffer. pls be noted
+		 * shadow_ring_buffer_va is now pointed to real ring buffer va
+		 * in copy_workload_to_ring_buffer.
+		 */
+
+		if (bb->bb_offset)
+			bb->bb_start_cmd_va = workload->shadow_ring_buffer_va
+				+ bb->bb_offset;
+
+		/* relocate shadow batch buffer */
+		bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
+		if (gmadr_bytes == 8)
+			bb->bb_start_cmd_va[2] = 0;
+
+		/* No one is going to touch shadow bb from now on. */
+		if (bb->clflush & CLFLUSH_AFTER) {
+			drm_clflush_virt_range(bb->va, bb->obj->base.size);
+			bb->clflush &= ~CLFLUSH_AFTER;
+		}
+
+		ret = i915_gem_object_set_to_gtt_domain(bb->obj, false);
+		if (ret)
+			goto err;
+
+		i915_gem_obj_finish_shmem_access(bb->obj);
+		bb->accessing = false;
+
+		i915_vma_move_to_active(bb->vma, workload->req, 0);
+	}
+	return 0;
+err:
+	release_shadow_batch_buffer(workload);
+	return ret;
+}
+
+static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	struct intel_vgpu_workload *workload = container_of(wa_ctx,
+					struct intel_vgpu_workload,
+					wa_ctx);
+	int ring_id = workload->ring_id;
+	struct intel_vgpu_submission *s = &workload->vgpu->submission;
+	struct i915_gem_context *shadow_ctx = s->shadow_ctx;
+	struct drm_i915_gem_object *ctx_obj =
+		shadow_ctx->engine[ring_id].state->obj;
+	struct execlist_ring_context *shadow_ring_context;
+	struct page *page;
+
+	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+	shadow_ring_context = kmap_atomic(page);
+
+	shadow_ring_context->bb_per_ctx_ptr.val =
+		(shadow_ring_context->bb_per_ctx_ptr.val &
+		(~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
+	shadow_ring_context->rcs_indirect_ctx.val =
+		(shadow_ring_context->rcs_indirect_ctx.val &
+		(~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
+
+	kunmap_atomic(shadow_ring_context);
+	return 0;
+}
+
+static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+	struct i915_vma *vma;
+	unsigned char *per_ctx_va =
+		(unsigned char *)wa_ctx->indirect_ctx.shadow_va +
+		wa_ctx->indirect_ctx.size;
+
+	if (wa_ctx->indirect_ctx.size == 0)
+		return 0;
+
+	vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
+				       0, CACHELINE_BYTES, 0);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
+
+	/* FIXME: we are not tracking our pinned VMA leaving it
+	 * up to the core to fix up the stray pin_count upon
+	 * free.
+	 */
+
+	wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
+
+	wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
+	memset(per_ctx_va, 0, CACHELINE_BYTES);
+
+	update_wa_ctx_2_shadow_ctx(wa_ctx);
+	return 0;
+}
+
+static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct intel_vgpu_shadow_bb *bb, *pos;
+
+	if (list_empty(&workload->shadow_bb))
+		return;
+
+	bb = list_first_entry(&workload->shadow_bb,
+			struct intel_vgpu_shadow_bb, list);
+
+	mutex_lock(&dev_priv->drm.struct_mutex);
+
+	list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) {
+		if (bb->obj) {
+			if (bb->accessing)
+				i915_gem_obj_finish_shmem_access(bb->obj);
+
+			if (bb->va && !IS_ERR(bb->va))
+				i915_gem_object_unpin_map(bb->obj);
+
+			if (bb->vma && !IS_ERR(bb->vma)) {
+				i915_vma_unpin(bb->vma);
+				i915_vma_close(bb->vma);
+			}
+			__i915_gem_object_release_unless_active(bb->obj);
+		}
+		list_del(&bb->list);
+		kfree(bb);
+	}
+
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+static int prepare_workload(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu *vgpu = workload->vgpu;
+	int ret = 0;
+
+	ret = intel_vgpu_pin_mm(workload->shadow_mm);
+	if (ret) {
+		gvt_vgpu_err("fail to vgpu pin mm\n");
+		return ret;
+	}
+
+	ret = intel_vgpu_sync_oos_pages(workload->vgpu);
+	if (ret) {
+		gvt_vgpu_err("fail to vgpu sync oos pages\n");
+		goto err_unpin_mm;
+	}
+
+	ret = intel_vgpu_flush_post_shadow(workload->vgpu);
+	if (ret) {
+		gvt_vgpu_err("fail to flush post shadow\n");
+		goto err_unpin_mm;
+	}
+
+	ret = intel_gvt_generate_request(workload);
+	if (ret) {
+		gvt_vgpu_err("fail to generate request\n");
+		goto err_unpin_mm;
+	}
+
+	ret = prepare_shadow_batch_buffer(workload);
+	if (ret) {
+		gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
+		goto err_unpin_mm;
+	}
+
+	ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
+	if (ret) {
+		gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
+		goto err_shadow_batch;
+	}
+
+	if (workload->prepare) {
+		ret = workload->prepare(workload);
+		if (ret)
+			goto err_shadow_wa_ctx;
+	}
+
+	return 0;
+err_shadow_wa_ctx:
+	release_shadow_wa_ctx(&workload->wa_ctx);
+err_shadow_batch:
+	release_shadow_batch_buffer(workload);
+err_unpin_mm:
+	intel_vgpu_unpin_mm(workload->shadow_mm);
+	return ret;
+}
+
 static int dispatch_workload(struct intel_vgpu_workload *workload)
 {
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct i915_gem_context *shadow_ctx = s->shadow_ctx;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	int ring_id = workload->ring_id;
-	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
-	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine = dev_priv->engine[ring_id];
 	int ret = 0;
 
@@ -375,12 +630,10 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
 	if (ret)
 		goto out;
 
-	if (workload->prepare) {
-		ret = workload->prepare(workload);
-		if (ret) {
-			engine->context_unpin(engine, shadow_ctx);
-			goto out;
-		}
+	ret = prepare_workload(workload);
+	if (ret) {
+		engine->context_unpin(engine, shadow_ctx);
+		goto out;
 	}
 
 out:
@@ -448,7 +701,7 @@ static struct intel_vgpu_workload *pick_next_workload(
 
 	gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload);
 
-	atomic_inc(&workload->vgpu->running_workload_num);
+	atomic_inc(&workload->vgpu->submission.running_workload_num);
 out:
 	mutex_unlock(&gvt->lock);
 	return workload;
@@ -458,8 +711,9 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 {
 	struct intel_vgpu *vgpu = workload->vgpu;
 	struct intel_gvt *gvt = vgpu->gvt;
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct i915_gem_context *shadow_ctx = s->shadow_ctx;
 	int ring_id = workload->ring_id;
-	struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
 	struct drm_i915_gem_object *ctx_obj =
 		shadow_ctx->engine[ring_id].state->obj;
 	struct execlist_ring_context *shadow_ring_context;
@@ -483,7 +737,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 	while (i < context_page_num) {
 		context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
 				(u32)((workload->ctx_desc.lrca + i) <<
-					GTT_PAGE_SHIFT));
+					I915_GTT_PAGE_SHIFT));
 		if (context_gpa == INTEL_GVT_INVALID_ADDR) {
 			gvt_vgpu_err("invalid guest context descriptor\n");
 			return;
@@ -492,7 +746,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 		page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
 		src = kmap(page);
 		intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
-				GTT_PAGE_SIZE);
+				I915_GTT_PAGE_SIZE);
 		kunmap(page);
 		i++;
 	}
@@ -517,23 +771,41 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 			sizeof(*shadow_ring_context),
 			(void *)shadow_ring_context +
 			sizeof(*shadow_ring_context),
-			GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+			I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 
 	kunmap(page);
 }
 
+static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask)
+{
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct intel_engine_cs *engine;
+	struct intel_vgpu_workload *pos, *n;
+	unsigned int tmp;
+
+	/* free the unsubmited workloads in the queues. */
+	for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
+		list_for_each_entry_safe(pos, n,
+			&s->workload_q_head[engine->id], list) {
+			list_del_init(&pos->list);
+			intel_vgpu_destroy_workload(pos);
+		}
+		clear_bit(engine->id, s->shadow_ctx_desc_updated);
+	}
+}
+
 static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 {
 	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
-	struct intel_vgpu_workload *workload;
-	struct intel_vgpu *vgpu;
+	struct intel_vgpu_workload *workload =
+		scheduler->current_workload[ring_id];
+	struct intel_vgpu *vgpu = workload->vgpu;
+	struct intel_vgpu_submission *s = &vgpu->submission;
 	int event;
 
 	mutex_lock(&gvt->lock);
 
-	workload = scheduler->current_workload[ring_id];
-	vgpu = workload->vgpu;
-
 	/* For the workload w/ request, needs to wait for the context
 	 * switch to make sure request is completed.
 	 * For the workload w/o request, directly complete the workload.
@@ -570,7 +842,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 		}
 		mutex_lock(&dev_priv->drm.struct_mutex);
 		/* unpin shadow ctx as the shadow_ctx update is done */
-		engine->context_unpin(engine, workload->vgpu->shadow_ctx);
+		engine->context_unpin(engine, s->shadow_ctx);
 		mutex_unlock(&dev_priv->drm.struct_mutex);
 	}
 
@@ -580,9 +852,32 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 	scheduler->current_workload[ring_id] = NULL;
 
 	list_del_init(&workload->list);
+
+	if (!workload->status) {
+		release_shadow_batch_buffer(workload);
+		release_shadow_wa_ctx(&workload->wa_ctx);
+	}
+
+	if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
+		/* if workload->status is not successful means HW GPU
+		 * has occurred GPU hang or something wrong with i915/GVT,
+		 * and GVT won't inject context switch interrupt to guest.
+		 * So this error is a vGPU hang actually to the guest.
+		 * According to this we should emunlate a vGPU hang. If
+		 * there are pending workloads which are already submitted
+		 * from guest, we should clean them up like HW GPU does.
+		 *
+		 * if it is in middle of engine resetting, the pending
+		 * workloads won't be submitted to HW GPU and will be
+		 * cleaned up during the resetting process later, so doing
+		 * the workload clean up here doesn't have any impact.
+		 **/
+		clean_workloads(vgpu, ENGINE_MASK(ring_id));
+	}
+
 	workload->complete(workload);
 
-	atomic_dec(&vgpu->running_workload_num);
+	atomic_dec(&s->running_workload_num);
 	wake_up(&scheduler->workload_complete_wq);
 
 	if (gvt->scheduler.need_reschedule)
@@ -665,20 +960,23 @@ static int workload_thread(void *priv)
 					FORCEWAKE_ALL);
 
 		intel_runtime_pm_put(gvt->dev_priv);
+		if (ret && (vgpu_is_vm_unhealthy(ret)))
+			enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
 	}
 	return 0;
 }
 
 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
 {
+	struct intel_vgpu_submission *s = &vgpu->submission;
 	struct intel_gvt *gvt = vgpu->gvt;
 	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 
-	if (atomic_read(&vgpu->running_workload_num)) {
+	if (atomic_read(&s->running_workload_num)) {
 		gvt_dbg_sched("wait vgpu idle\n");
 
 		wait_event(scheduler->workload_complete_wq,
-				!atomic_read(&vgpu->running_workload_num));
+				!atomic_read(&s->running_workload_num));
 	}
 }
 
@@ -743,26 +1041,372 @@ int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
 	return ret;
 }
 
-void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu)
+/**
+ * intel_vgpu_clean_submission - free submission-related resource for vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is called when a vGPU is being destroyed.
+ *
+ */
+void intel_vgpu_clean_submission(struct intel_vgpu *vgpu)
 {
-	i915_gem_context_put(vgpu->shadow_ctx);
+	struct intel_vgpu_submission *s = &vgpu->submission;
+
+	intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0);
+	i915_gem_context_put(s->shadow_ctx);
+	kmem_cache_destroy(s->workloads);
 }
 
-int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu)
+
+/**
+ * intel_vgpu_reset_submission - reset submission-related resource for vGPU
+ * @vgpu: a vGPU
+ * @engine_mask: engines expected to be reset
+ *
+ * This function is called when a vGPU is being destroyed.
+ *
+ */
+void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
+		unsigned long engine_mask)
+{
+	struct intel_vgpu_submission *s = &vgpu->submission;
+
+	if (!s->active)
+		return;
+
+	clean_workloads(vgpu, engine_mask);
+	s->ops->reset(vgpu, engine_mask);
+}
+
+/**
+ * intel_vgpu_setup_submission - setup submission-related resource for vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is called when a vGPU is being created.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
 {
-	atomic_set(&vgpu->running_workload_num, 0);
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	enum intel_engine_id i;
+	struct intel_engine_cs *engine;
+	int ret;
 
-	vgpu->shadow_ctx = i915_gem_context_create_gvt(
+	s->shadow_ctx = i915_gem_context_create_gvt(
 			&vgpu->gvt->dev_priv->drm);
-	if (IS_ERR(vgpu->shadow_ctx))
-		return PTR_ERR(vgpu->shadow_ctx);
+	if (IS_ERR(s->shadow_ctx))
+		return PTR_ERR(s->shadow_ctx);
+
+	if (HAS_LOGICAL_RING_PREEMPTION(vgpu->gvt->dev_priv))
+		s->shadow_ctx->priority = INT_MAX;
+
+	bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
+
+	s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload",
+						  sizeof(struct intel_vgpu_workload), 0,
+						  SLAB_HWCACHE_ALIGN,
+						  offsetof(struct intel_vgpu_workload, rb_tail),
+						  sizeof_field(struct intel_vgpu_workload, rb_tail),
+						  NULL);
+
+	if (!s->workloads) {
+		ret = -ENOMEM;
+		goto out_shadow_ctx;
+	}
+
+	for_each_engine(engine, vgpu->gvt->dev_priv, i)
+		INIT_LIST_HEAD(&s->workload_q_head[i]);
+
+	atomic_set(&s->running_workload_num, 0);
+	bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES);
+
+	return 0;
+
+out_shadow_ctx:
+	i915_gem_context_put(s->shadow_ctx);
+	return ret;
+}
+
+/**
+ * intel_vgpu_select_submission_ops - select virtual submission interface
+ * @vgpu: a vGPU
+ * @interface: expected vGPU virtual submission interface
+ *
+ * This function is called when guest configures submission interface.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
+				     unsigned long engine_mask,
+				     unsigned int interface)
+{
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	const struct intel_vgpu_submission_ops *ops[] = {
+		[INTEL_VGPU_EXECLIST_SUBMISSION] =
+			&intel_vgpu_execlist_submission_ops,
+	};
+	int ret;
+
+	if (WARN_ON(interface >= ARRAY_SIZE(ops)))
+		return -EINVAL;
+
+	if (WARN_ON(interface == 0 && engine_mask != ALL_ENGINES))
+		return -EINVAL;
+
+	if (s->active)
+		s->ops->clean(vgpu, engine_mask);
+
+	if (interface == 0) {
+		s->ops = NULL;
+		s->virtual_submission_interface = 0;
+		s->active = false;
+		gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id);
+		return 0;
+	}
+
+	ret = ops[interface]->init(vgpu, engine_mask);
+	if (ret)
+		return ret;
+
+	s->ops = ops[interface];
+	s->virtual_submission_interface = interface;
+	s->active = true;
+
+	gvt_dbg_core("vgpu%d: activate ops [ %s ]\n",
+			vgpu->id, s->ops->name);
+
+	return 0;
+}
+
+/**
+ * intel_vgpu_destroy_workload - destroy a vGPU workload
+ * @vgpu: a vGPU
+ *
+ * This function is called when destroy a vGPU workload.
+ *
+ */
+void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)
+{
+	struct intel_vgpu_submission *s = &workload->vgpu->submission;
+
+	if (workload->shadow_mm)
+		intel_gvt_mm_unreference(workload->shadow_mm);
+
+	kmem_cache_free(s->workloads, workload);
+}
+
+static struct intel_vgpu_workload *
+alloc_workload(struct intel_vgpu *vgpu)
+{
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct intel_vgpu_workload *workload;
+
+	workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL);
+	if (!workload)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&workload->list);
+	INIT_LIST_HEAD(&workload->shadow_bb);
+
+	init_waitqueue_head(&workload->shadow_ctx_status_wq);
+	atomic_set(&workload->shadow_ctx_active, 0);
+
+	workload->status = -EINPROGRESS;
+	workload->shadowed = false;
+	workload->vgpu = vgpu;
+
+	return workload;
+}
+
+#define RING_CTX_OFF(x) \
+	offsetof(struct execlist_ring_context, x)
+
+static void read_guest_pdps(struct intel_vgpu *vgpu,
+		u64 ring_context_gpa, u32 pdp[8])
+{
+	u64 gpa;
+	int i;
+
+	gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val);
+
+	for (i = 0; i < 8; i++)
+		intel_gvt_hypervisor_read_gpa(vgpu,
+				gpa + i * 8, &pdp[7 - i], 4);
+}
 
-	if (INTEL_INFO(vgpu->gvt->dev_priv)->has_logical_ring_preemption)
-		vgpu->shadow_ctx->priority = INT_MAX;
+static int prepare_mm(struct intel_vgpu_workload *workload)
+{
+	struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
+	struct intel_vgpu_mm *mm;
+	struct intel_vgpu *vgpu = workload->vgpu;
+	int page_table_level;
+	u32 pdp[8];
+
+	if (desc->addressing_mode == 1) { /* legacy 32-bit */
+		page_table_level = 3;
+	} else if (desc->addressing_mode == 3) { /* legacy 64 bit */
+		page_table_level = 4;
+	} else {
+		gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
+		return -EINVAL;
+	}
 
-	vgpu->shadow_ctx->engine[RCS].initialised = true;
+	read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp);
 
-	bitmap_zero(vgpu->shadow_ctx_desc_updated, I915_NUM_ENGINES);
+	mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp);
+	if (mm) {
+		intel_gvt_mm_reference(mm);
+	} else {
 
+		mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT,
+				pdp, page_table_level, 0);
+		if (IS_ERR(mm)) {
+			gvt_vgpu_err("fail to create mm object.\n");
+			return PTR_ERR(mm);
+		}
+	}
+	workload->shadow_mm = mm;
 	return 0;
 }
+
+#define same_context(a, b) (((a)->context_id == (b)->context_id) && \
+		((a)->lrca == (b)->lrca))
+
+#define get_last_workload(q) \
+	(list_empty(q) ? NULL : container_of(q->prev, \
+	struct intel_vgpu_workload, list))
+/**
+ * intel_vgpu_create_workload - create a vGPU workload
+ * @vgpu: a vGPU
+ * @desc: a guest context descriptor
+ *
+ * This function is called when creating a vGPU workload.
+ *
+ * Returns:
+ * struct intel_vgpu_workload * on success, negative error code in
+ * pointer if failed.
+ *
+ */
+struct intel_vgpu_workload *
+intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
+			   struct execlist_ctx_descriptor_format *desc)
+{
+	struct intel_vgpu_submission *s = &vgpu->submission;
+	struct list_head *q = workload_q_head(vgpu, ring_id);
+	struct intel_vgpu_workload *last_workload = get_last_workload(q);
+	struct intel_vgpu_workload *workload = NULL;
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	u64 ring_context_gpa;
+	u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
+	int ret;
+
+	ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+			(u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT));
+	if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
+		gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
+		return ERR_PTR(-EINVAL);
+	}
+
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(ring_header.val), &head, 4);
+
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(ring_tail.val), &tail, 4);
+
+	head &= RB_HEAD_OFF_MASK;
+	tail &= RB_TAIL_OFF_MASK;
+
+	if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
+		gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
+		gvt_dbg_el("ctx head %x real head %lx\n", head,
+				last_workload->rb_tail);
+		/*
+		 * cannot use guest context head pointer here,
+		 * as it might not be updated at this time
+		 */
+		head = last_workload->rb_tail;
+	}
+
+	gvt_dbg_el("ring id %d begin a new workload\n", ring_id);
+
+	/* record some ring buffer register values for scan and shadow */
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(rb_start.val), &start, 4);
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
+	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
+
+	workload = alloc_workload(vgpu);
+	if (IS_ERR(workload))
+		return workload;
+
+	workload->ring_id = ring_id;
+	workload->ctx_desc = *desc;
+	workload->ring_context_gpa = ring_context_gpa;
+	workload->rb_head = head;
+	workload->rb_tail = tail;
+	workload->rb_start = start;
+	workload->rb_ctl = ctl;
+
+	if (ring_id == RCS) {
+		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
+		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+			RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
+
+		workload->wa_ctx.indirect_ctx.guest_gma =
+			indirect_ctx & INDIRECT_CTX_ADDR_MASK;
+		workload->wa_ctx.indirect_ctx.size =
+			(indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
+			CACHELINE_BYTES;
+		workload->wa_ctx.per_ctx.guest_gma =
+			per_ctx & PER_CTX_ADDR_MASK;
+		workload->wa_ctx.per_ctx.valid = per_ctx & 1;
+	}
+
+	gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
+			workload, ring_id, head, tail, start, ctl);
+
+	ret = prepare_mm(workload);
+	if (ret) {
+		kmem_cache_free(s->workloads, workload);
+		return ERR_PTR(ret);
+	}
+
+	/* Only scan and shadow the first workload in the queue
+	 * as there is only one pre-allocated buf-obj for shadow.
+	 */
+	if (list_empty(workload_q_head(vgpu, ring_id))) {
+		intel_runtime_pm_get(dev_priv);
+		mutex_lock(&dev_priv->drm.struct_mutex);
+		ret = intel_gvt_scan_and_shadow_workload(workload);
+		mutex_unlock(&dev_priv->drm.struct_mutex);
+		intel_runtime_pm_put(dev_priv);
+	}
+
+	if (ret && (vgpu_is_vm_unhealthy(ret))) {
+		enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
+		intel_vgpu_destroy_workload(workload);
+		return ERR_PTR(ret);
+	}
+
+	return workload;
+}
+
+/**
+ * intel_vgpu_queue_workload - Qeue a vGPU workload
+ * @workload: the workload to queue in
+ */
+void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload)
+{
+	list_add_tail(&workload->list,
+		workload_q_head(workload->vgpu, workload->ring_id));
+	intel_gvt_kick_schedule(workload->vgpu->gvt);
+	wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->ring_id]);
+}
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index b9f872204d7e..a79a4f60637e 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -110,26 +110,27 @@ struct intel_vgpu_workload {
 	/* shadow batch buffer */
 	struct list_head shadow_bb;
 	struct intel_shadow_wa_ctx wa_ctx;
+
+	/* oa registers */
+	u32 oactxctrl;
+	u32 flex_mmio[7];
 };
 
-/* Intel shadow batch buffer is a i915 gem object */
-struct intel_shadow_bb_entry {
+struct intel_vgpu_shadow_bb {
 	struct list_head list;
 	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
 	void *va;
-	unsigned long len;
 	u32 *bb_start_cmd_va;
+	unsigned int clflush;
+	bool accessing;
+	unsigned long bb_offset;
 };
 
 #define workload_q_head(vgpu, ring_id) \
-	(&(vgpu->workload_q_head[ring_id]))
+	(&(vgpu->submission.workload_q_head[ring_id]))
 
-#define queue_workload(workload) do { \
-	list_add_tail(&workload->list, \
-	workload_q_head(workload->vgpu, workload->ring_id)); \
-	wake_up(&workload->vgpu->gvt-> \
-	scheduler.waitq[workload->ring_id]); \
-} while (0)
+void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload);
 
 int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt);
 
@@ -137,12 +138,24 @@ void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt);
 
 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu);
 
-int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu);
+int intel_vgpu_setup_submission(struct intel_vgpu *vgpu);
+
+void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
+				 unsigned long engine_mask);
+
+void intel_vgpu_clean_submission(struct intel_vgpu *vgpu);
+
+int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
+				     unsigned long engine_mask,
+				     unsigned int interface);
 
-void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu);
+extern const struct intel_vgpu_submission_ops
+intel_vgpu_execlist_submission_ops;
 
-void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx);
+struct intel_vgpu_workload *
+intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
+			   struct execlist_ctx_descriptor_format *desc);
 
-int intel_gvt_generate_request(struct intel_vgpu_workload *workload);
+void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload);
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/trace.h b/drivers/gpu/drm/i915/gvt/trace.h
index 8c150381d9a4..736bd2bc5127 100644
--- a/drivers/gpu/drm/i915/gvt/trace.h
+++ b/drivers/gpu/drm/i915/gvt/trace.h
@@ -330,13 +330,14 @@ TRACE_EVENT(inject_msi,
 );
 
 TRACE_EVENT(render_mmio,
-	TP_PROTO(int id, char *action, unsigned int reg,
+	TP_PROTO(int old_id, int new_id, char *action, unsigned int reg,
 		 unsigned int old_val, unsigned int new_val),
 
-	TP_ARGS(id, action, reg, new_val, old_val),
+	TP_ARGS(old_id, new_id, action, reg, old_val, new_val),
 
 	TP_STRUCT__entry(
-		__field(int, id)
+		__field(int, old_id)
+		__field(int, new_id)
 		__array(char, buf, GVT_TEMP_STR_LEN)
 		__field(unsigned int, reg)
 		__field(unsigned int, old_val)
@@ -344,15 +345,17 @@ TRACE_EVENT(render_mmio,
 	),
 
 	TP_fast_assign(
-		__entry->id = id;
+		__entry->old_id = old_id;
+		__entry->new_id = new_id;
 		snprintf(__entry->buf, GVT_TEMP_STR_LEN, "%s", action);
 		__entry->reg = reg;
 		__entry->old_val = old_val;
 		__entry->new_val = new_val;
 	),
 
-	TP_printk("VM%u %s reg %x, old %08x new %08x\n",
-		  __entry->id, __entry->buf, __entry->reg,
+	TP_printk("VM%u -> VM%u %s reg %x, old %08x new %08x\n",
+		  __entry->old_id, __entry->new_id,
+		  __entry->buf, __entry->reg,
 		  __entry->old_val, __entry->new_val)
 );
 
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 02c61a1ad56a..b87b19d8443c 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -38,22 +38,25 @@
 void populate_pvinfo_page(struct intel_vgpu *vgpu)
 {
 	/* setup the ballooning information */
-	vgpu_vreg64(vgpu, vgtif_reg(magic)) = VGT_MAGIC;
-	vgpu_vreg(vgpu, vgtif_reg(version_major)) = 1;
-	vgpu_vreg(vgpu, vgtif_reg(version_minor)) = 0;
-	vgpu_vreg(vgpu, vgtif_reg(display_ready)) = 0;
-	vgpu_vreg(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
-	vgpu_vreg(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT;
-	vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
+	vgpu_vreg64_t(vgpu, vgtif_reg(magic)) = VGT_MAGIC;
+	vgpu_vreg_t(vgpu, vgtif_reg(version_major)) = 1;
+	vgpu_vreg_t(vgpu, vgtif_reg(version_minor)) = 0;
+	vgpu_vreg_t(vgpu, vgtif_reg(display_ready)) = 0;
+	vgpu_vreg_t(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
+
+	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT;
+	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HWSP_EMULATION;
+
+	vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
 		vgpu_aperture_gmadr_base(vgpu);
-	vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) =
+	vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) =
 		vgpu_aperture_sz(vgpu);
-	vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) =
+	vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.base)) =
 		vgpu_hidden_gmadr_base(vgpu);
-	vgpu_vreg(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) =
+	vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.nonmappable_gmadr.size)) =
 		vgpu_hidden_sz(vgpu);
 
-	vgpu_vreg(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu);
+	vgpu_vreg_t(vgpu, vgtif_reg(avail_rs.fence_num)) = vgpu_fence_sz(vgpu);
 
 	gvt_dbg_core("Populate PVINFO PAGE for vGPU %d\n", vgpu->id);
 	gvt_dbg_core("aperture base [GMADR] 0x%llx size 0x%llx\n",
@@ -226,13 +229,14 @@ void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu)
 
 	vgpu->active = false;
 
-	if (atomic_read(&vgpu->running_workload_num)) {
+	if (atomic_read(&vgpu->submission.running_workload_num)) {
 		mutex_unlock(&gvt->lock);
 		intel_gvt_wait_vgpu_idle(vgpu);
 		mutex_lock(&gvt->lock);
 	}
 
 	intel_vgpu_stop_schedule(vgpu);
+	intel_vgpu_dmabuf_cleanup(vgpu);
 
 	mutex_unlock(&gvt->lock);
 }
@@ -252,16 +256,19 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu)
 
 	WARN(vgpu->active, "vGPU is still active!\n");
 
+	intel_gvt_debugfs_remove_vgpu(vgpu);
 	idr_remove(&gvt->vgpu_idr, vgpu->id);
+	if (idr_is_empty(&gvt->vgpu_idr))
+		intel_gvt_clean_irq(gvt);
 	intel_vgpu_clean_sched_policy(vgpu);
-	intel_vgpu_clean_gvt_context(vgpu);
-	intel_vgpu_clean_execlist(vgpu);
+	intel_vgpu_clean_submission(vgpu);
 	intel_vgpu_clean_display(vgpu);
 	intel_vgpu_clean_opregion(vgpu);
 	intel_vgpu_clean_gtt(vgpu);
 	intel_gvt_hypervisor_detach_vgpu(vgpu);
 	intel_vgpu_free_resource(vgpu);
 	intel_vgpu_clean_mmio(vgpu);
+	intel_vgpu_dmabuf_cleanup(vgpu);
 	vfree(vgpu);
 
 	intel_gvt_update_vgpu_types(gvt);
@@ -293,7 +300,7 @@ struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt)
 	vgpu->gvt = gvt;
 
 	for (i = 0; i < I915_NUM_ENGINES; i++)
-		INIT_LIST_HEAD(&vgpu->workload_q_head[i]);
+		INIT_LIST_HEAD(&vgpu->submission.workload_q_head[i]);
 
 	ret = intel_vgpu_init_sched_policy(vgpu);
 	if (ret)
@@ -346,8 +353,8 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
 	vgpu->handle = param->handle;
 	vgpu->gvt = gvt;
 	vgpu->sched_ctl.weight = param->weight;
-	bitmap_zero(vgpu->tlb_handle_pending, I915_NUM_ENGINES);
-
+	INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head);
+	idr_init(&vgpu->object_idr);
 	intel_vgpu_init_cfg_space(vgpu, param->primary);
 
 	ret = intel_vgpu_init_mmio(vgpu);
@@ -368,32 +375,42 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
 	if (ret)
 		goto out_detach_hypervisor_vgpu;
 
-	ret = intel_vgpu_init_display(vgpu, param->resolution);
+	ret = intel_vgpu_init_opregion(vgpu);
 	if (ret)
 		goto out_clean_gtt;
 
-	ret = intel_vgpu_init_execlist(vgpu);
+	ret = intel_vgpu_init_display(vgpu, param->resolution);
 	if (ret)
-		goto out_clean_display;
+		goto out_clean_opregion;
 
-	ret = intel_vgpu_init_gvt_context(vgpu);
+	ret = intel_vgpu_setup_submission(vgpu);
 	if (ret)
-		goto out_clean_execlist;
+		goto out_clean_display;
 
 	ret = intel_vgpu_init_sched_policy(vgpu);
 	if (ret)
-		goto out_clean_shadow_ctx;
+		goto out_clean_submission;
+
+	ret = intel_gvt_debugfs_add_vgpu(vgpu);
+	if (ret)
+		goto out_clean_sched_policy;
+
+	ret = intel_gvt_hypervisor_set_opregion(vgpu);
+	if (ret)
+		goto out_clean_sched_policy;
 
 	mutex_unlock(&gvt->lock);
 
 	return vgpu;
 
-out_clean_shadow_ctx:
-	intel_vgpu_clean_gvt_context(vgpu);
-out_clean_execlist:
-	intel_vgpu_clean_execlist(vgpu);
+out_clean_sched_policy:
+	intel_vgpu_clean_sched_policy(vgpu);
+out_clean_submission:
+	intel_vgpu_clean_submission(vgpu);
 out_clean_display:
 	intel_vgpu_clean_display(vgpu);
+out_clean_opregion:
+	intel_vgpu_clean_opregion(vgpu);
 out_clean_gtt:
 	intel_vgpu_clean_gtt(vgpu);
 out_detach_hypervisor_vgpu:
@@ -500,11 +517,10 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
 		mutex_lock(&gvt->lock);
 	}
 
-	intel_vgpu_reset_execlist(vgpu, resetting_eng);
-
+	intel_vgpu_reset_submission(vgpu, resetting_eng);
 	/* full GPU reset or device model level reset */
 	if (engine_mask == ALL_ENGINES || dmlr) {
-
+		intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0);
 		/*fence will not be reset during virtual reset */
 		if (dmlr) {
 			intel_vgpu_reset_gtt(vgpu);
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 8ba932b22f7c..95478db9998b 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -26,6 +26,7 @@
  */
 
 #include "i915_drv.h"
+#include "intel_ringbuffer.h"
 
 /**
  * DOC: batch buffer command parser
@@ -798,22 +799,15 @@ struct cmd_node {
  */
 static inline u32 cmd_header_key(u32 x)
 {
-	u32 shift;
-
 	switch (x >> INSTR_CLIENT_SHIFT) {
 	default:
 	case INSTR_MI_CLIENT:
-		shift = STD_MI_OPCODE_SHIFT;
-		break;
+		return x >> STD_MI_OPCODE_SHIFT;
 	case INSTR_RC_CLIENT:
-		shift = STD_3D_OPCODE_SHIFT;
-		break;
+		return x >> STD_3D_OPCODE_SHIFT;
 	case INSTR_BC_CLIENT:
-		shift = STD_2D_OPCODE_SHIFT;
-		break;
+		return x >> STD_2D_OPCODE_SHIFT;
 	}
-
-	return x >> shift;
 }
 
 static int init_hash_table(struct intel_engine_cs *engine,
@@ -947,7 +941,7 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
 		return;
 	}
 
-	engine->needs_cmd_parser = true;
+	engine->flags |= I915_ENGINE_NEEDS_CMD_PARSER;
 }
 
 /**
@@ -959,7 +953,7 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
  */
 void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine)
 {
-	if (!engine->needs_cmd_parser)
+	if (!intel_engine_needs_cmd_parser(engine))
 		return;
 
 	fini_hash_table(engine);
@@ -1038,7 +1032,7 @@ find_reg(const struct intel_engine_cs *engine, bool is_master, u32 addr)
 	const struct drm_i915_reg_table *table = engine->reg_tables;
 	int count = engine->reg_table_count;
 
-	do {
+	for (; count > 0; ++table, --count) {
 		if (!table->master || is_master) {
 			const struct drm_i915_reg_descriptor *reg;
 
@@ -1046,7 +1040,7 @@ find_reg(const struct intel_engine_cs *engine, bool is_master, u32 addr)
 			if (reg != NULL)
 				return reg;
 		}
-	} while (table++, --count);
+	}
 
 	return NULL;
 }
@@ -1218,6 +1212,12 @@ static bool check_cmd(const struct intel_engine_cs *engine,
 					continue;
 			}
 
+			if (desc->bits[i].offset >= length) {
+				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X, too short to check bitmask (%s)\n",
+						 *cmd, engine->name);
+				return false;
+			}
+
 			dword = cmd[desc->bits[i].offset] &
 				desc->bits[i].mask;
 
@@ -1357,7 +1357,7 @@ int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
 
 	/* If the command parser is not enabled, report 0 - unsupported */
 	for_each_engine(engine, dev_priv, id) {
-		if (engine->needs_cmd_parser) {
+		if (intel_engine_needs_cmd_parser(engine)) {
 			active = true;
 			break;
 		}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index c65e381b85f3..e968aeae1d84 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -30,47 +30,28 @@
 #include <linux/sort.h>
 #include <linux/sched/mm.h>
 #include "intel_drv.h"
-#include "i915_guc_submission.h"
+#include "intel_guc_submission.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
 {
 	return to_i915(node->minor->dev);
 }
 
-static __always_inline void seq_print_param(struct seq_file *m,
-					    const char *name,
-					    const char *type,
-					    const void *x)
-{
-	if (!__builtin_strcmp(type, "bool"))
-		seq_printf(m, "i915.%s=%s\n", name, yesno(*(const bool *)x));
-	else if (!__builtin_strcmp(type, "int"))
-		seq_printf(m, "i915.%s=%d\n", name, *(const int *)x);
-	else if (!__builtin_strcmp(type, "unsigned int"))
-		seq_printf(m, "i915.%s=%u\n", name, *(const unsigned int *)x);
-	else if (!__builtin_strcmp(type, "char *"))
-		seq_printf(m, "i915.%s=%s\n", name, *(const char **)x);
-	else
-		BUILD_BUG();
-}
-
 static int i915_capabilities(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	const struct intel_device_info *info = INTEL_INFO(dev_priv);
+	struct drm_printer p = drm_seq_file_printer(m);
 
 	seq_printf(m, "gen: %d\n", INTEL_GEN(dev_priv));
 	seq_printf(m, "platform: %s\n", intel_platform_name(info->platform));
 	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(dev_priv));
 
-#define PRINT_FLAG(x)  seq_printf(m, #x ": %s\n", yesno(info->x))
-	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
-#undef PRINT_FLAG
+	intel_device_info_dump_flags(info, &p);
+	intel_device_info_dump_runtime(info, &p);
 
 	kernel_param_lock(THIS_MODULE);
-#define PRINT_PARAM(T, x, ...) seq_print_param(m, #x, #T, &i915_modparams.x);
-	I915_PARAMS_FOR_EACH(PRINT_PARAM);
-#undef PRINT_PARAM
+	i915_params_dump(&i915_modparams, &p);
 	kernel_param_unlock(THIS_MODULE);
 
 	return 0;
@@ -111,8 +92,8 @@ static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
 	u64 size = 0;
 	struct i915_vma *vma;
 
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (i915_vma_is_ggtt(vma) && drm_mm_node_allocated(&vma->node))
+	for_each_ggtt_vma(vma, obj) {
+		if (drm_mm_node_allocated(&vma->node))
 			size += vma->node.size;
 	}
 
@@ -522,8 +503,8 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
 	seq_printf(m, "%u display objects (globally pinned), %llu bytes\n",
 		   dpy_count, dpy_size);
 
-	seq_printf(m, "%llu [%llu] gtt total\n",
-		   ggtt->base.total, ggtt->mappable_end);
+	seq_printf(m, "%llu [%pa] gtt total\n",
+		   ggtt->base.total, &ggtt->mappable_end);
 	seq_printf(m, "Supported page sizes: %s\n",
 		   stringify_page_sizes(INTEL_INFO(dev_priv)->page_sizes,
 					buf, sizeof(buf)));
@@ -664,38 +645,6 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static void i915_ring_seqno_info(struct seq_file *m,
-				 struct intel_engine_cs *engine)
-{
-	struct intel_breadcrumbs *b = &engine->breadcrumbs;
-	struct rb_node *rb;
-
-	seq_printf(m, "Current sequence (%s): %x\n",
-		   engine->name, intel_engine_get_seqno(engine));
-
-	spin_lock_irq(&b->rb_lock);
-	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
-		struct intel_wait *w = rb_entry(rb, typeof(*w), node);
-
-		seq_printf(m, "Waiting (%s): %s [%d] on %x\n",
-			   engine->name, w->tsk->comm, w->tsk->pid, w->seqno);
-	}
-	spin_unlock_irq(&b->rb_lock);
-}
-
-static int i915_gem_seqno_info(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
-
-	for_each_engine(engine, dev_priv, id)
-		i915_ring_seqno_info(m, engine);
-
-	return 0;
-}
-
-
 static int i915_interrupt_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -896,13 +845,12 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 		seq_printf(m, "Graphics Interrupt mask:		%08x\n",
 			   I915_READ(GTIMR));
 	}
-	for_each_engine(engine, dev_priv, id) {
-		if (INTEL_GEN(dev_priv) >= 6) {
+	if (INTEL_GEN(dev_priv) >= 6) {
+		for_each_engine(engine, dev_priv, id) {
 			seq_printf(m,
 				   "Graphics Interrupt mask (%s):	%08x\n",
 				   engine->name, I915_READ_IMR(engine));
 		}
-		i915_ring_seqno_info(m, engine);
 	}
 	intel_runtime_pm_put(dev_priv);
 
@@ -1151,13 +1099,8 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		rpdownei = I915_READ(GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
 		rpcurdown = I915_READ(GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
 		rpprevdown = I915_READ(GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
-		if (INTEL_GEN(dev_priv) >= 9)
-			cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
-		else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
-			cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
-		else
-			cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
-		cagf = intel_gpu_freq(dev_priv, cagf);
+		cagf = intel_gpu_freq(dev_priv,
+				      intel_get_cagf(dev_priv, rpstat));
 
 		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
@@ -1639,20 +1582,23 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused)
 static int i915_fbc_status(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
+	struct intel_fbc *fbc = &dev_priv->fbc;
 
-	if (!HAS_FBC(dev_priv)) {
-		seq_puts(m, "FBC unsupported on this chipset\n");
-		return 0;
-	}
+	if (!HAS_FBC(dev_priv))
+		return -ENODEV;
 
 	intel_runtime_pm_get(dev_priv);
-	mutex_lock(&dev_priv->fbc.lock);
+	mutex_lock(&fbc->lock);
 
 	if (intel_fbc_is_active(dev_priv))
 		seq_puts(m, "FBC enabled\n");
 	else
-		seq_printf(m, "FBC disabled: %s\n",
-			   dev_priv->fbc.no_fbc_reason);
+		seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
+
+	if (fbc->work.scheduled)
+		seq_printf(m, "FBC worker scheduled on vblank %u, now %llu\n",
+			   fbc->work.scheduled_vblank,
+			   drm_crtc_vblank_count(&fbc->crtc->base));
 
 	if (intel_fbc_is_active(dev_priv)) {
 		u32 mask;
@@ -1672,7 +1618,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 		seq_printf(m, "Compressing: %s\n", yesno(mask));
 	}
 
-	mutex_unlock(&dev_priv->fbc.lock);
+	mutex_unlock(&fbc->lock);
 	intel_runtime_pm_put(dev_priv);
 
 	return 0;
@@ -1719,10 +1665,8 @@ static int i915_ips_status(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 
-	if (!HAS_IPS(dev_priv)) {
-		seq_puts(m, "not supported\n");
-		return 0;
-	}
+	if (!HAS_IPS(dev_priv))
+		return -ENODEV;
 
 	intel_runtime_pm_get(dev_priv);
 
@@ -1808,10 +1752,8 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 	int gpu_freq, ia_freq;
 	unsigned int max_gpu_freq, min_gpu_freq;
 
-	if (!HAS_LLC(dev_priv)) {
-		seq_puts(m, "unsupported on this chipset\n");
-		return 0;
-	}
+	if (!HAS_LLC(dev_priv))
+		return -ENODEV;
 
 	intel_runtime_pm_get(dev_priv);
 
@@ -1974,7 +1916,6 @@ static int i915_context_status(struct seq_file *m, void *unused)
 			struct intel_context *ce = &ctx->engine[engine->id];
 
 			seq_printf(m, "%s: ", engine->name);
-			seq_putc(m, ce->initialised ? 'I' : 'i');
 			if (ce->state)
 				describe_obj(m, ce->state->obj);
 			if (ce->ring)
@@ -1990,75 +1931,6 @@ static int i915_context_status(struct seq_file *m, void *unused)
 	return 0;
 }
 
-static void i915_dump_lrc_obj(struct seq_file *m,
-			      struct i915_gem_context *ctx,
-			      struct intel_engine_cs *engine)
-{
-	struct i915_vma *vma = ctx->engine[engine->id].state;
-	struct page *page;
-	int j;
-
-	seq_printf(m, "CONTEXT: %s %u\n", engine->name, ctx->hw_id);
-
-	if (!vma) {
-		seq_puts(m, "\tFake context\n");
-		return;
-	}
-
-	if (vma->flags & I915_VMA_GLOBAL_BIND)
-		seq_printf(m, "\tBound in GGTT at 0x%08x\n",
-			   i915_ggtt_offset(vma));
-
-	if (i915_gem_object_pin_pages(vma->obj)) {
-		seq_puts(m, "\tFailed to get pages for context object\n\n");
-		return;
-	}
-
-	page = i915_gem_object_get_page(vma->obj, LRC_STATE_PN);
-	if (page) {
-		u32 *reg_state = kmap_atomic(page);
-
-		for (j = 0; j < 0x600 / sizeof(u32) / 4; j += 4) {
-			seq_printf(m,
-				   "\t[0x%04x] 0x%08x 0x%08x 0x%08x 0x%08x\n",
-				   j * 4,
-				   reg_state[j], reg_state[j + 1],
-				   reg_state[j + 2], reg_state[j + 3]);
-		}
-		kunmap_atomic(reg_state);
-	}
-
-	i915_gem_object_unpin_pages(vma->obj);
-	seq_putc(m, '\n');
-}
-
-static int i915_dump_lrc(struct seq_file *m, void *unused)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_engine_cs *engine;
-	struct i915_gem_context *ctx;
-	enum intel_engine_id id;
-	int ret;
-
-	if (!i915_modparams.enable_execlists) {
-		seq_printf(m, "Logical Ring Contexts are disabled\n");
-		return 0;
-	}
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	list_for_each_entry(ctx, &dev_priv->contexts.list, link)
-		for_each_engine(engine, dev_priv, id)
-			i915_dump_lrc_obj(m, ctx, engine);
-
-	mutex_unlock(&dev->struct_mutex);
-
-	return 0;
-}
-
 static const char *swizzle_string(unsigned swizzle)
 {
 	switch (swizzle) {
@@ -2361,8 +2233,8 @@ static int i915_huc_load_status_info(struct seq_file *m, void *data)
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct drm_printer p;
 
-	if (!HAS_HUC_UCODE(dev_priv))
-		return 0;
+	if (!HAS_HUC(dev_priv))
+		return -ENODEV;
 
 	p = drm_seq_file_printer(m);
 	intel_uc_fw_dump(&dev_priv->huc.fw, &p);
@@ -2380,8 +2252,8 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data)
 	struct drm_printer p;
 	u32 tmp, i;
 
-	if (!HAS_GUC_UCODE(dev_priv))
-		return 0;
+	if (!HAS_GUC(dev_priv))
+		return -ENODEV;
 
 	p = drm_seq_file_printer(m);
 	intel_uc_fw_dump(&dev_priv->guc.fw, &p);
@@ -2434,7 +2306,7 @@ static void i915_guc_log_info(struct seq_file *m,
 
 static void i915_guc_client_info(struct seq_file *m,
 				 struct drm_i915_private *dev_priv,
-				 struct i915_guc_client *client)
+				 struct intel_guc_client *client)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
@@ -2454,29 +2326,16 @@ static void i915_guc_client_info(struct seq_file *m,
 	seq_printf(m, "\tTotal: %llu\n", tot);
 }
 
-static bool check_guc_submission(struct seq_file *m)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	const struct intel_guc *guc = &dev_priv->guc;
-
-	if (!guc->execbuf_client) {
-		seq_printf(m, "GuC submission %s\n",
-			   HAS_GUC_SCHED(dev_priv) ?
-			   "disabled" :
-			   "not supported");
-		return false;
-	}
-
-	return true;
-}
-
 static int i915_guc_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	const struct intel_guc *guc = &dev_priv->guc;
 
-	if (!check_guc_submission(m))
-		return 0;
+	if (!USES_GUC_SUBMISSION(dev_priv))
+		return -ENODEV;
+
+	GEM_BUG_ON(!guc->execbuf_client);
+	GEM_BUG_ON(!guc->preempt_client);
 
 	seq_printf(m, "Doorbell map:\n");
 	seq_printf(m, "\t%*pb\n", GUC_NUM_DOORBELLS, guc->doorbell_bitmap);
@@ -2484,6 +2343,8 @@ static int i915_guc_info(struct seq_file *m, void *data)
 
 	seq_printf(m, "\nGuC execbuf client @ %p:\n", guc->execbuf_client);
 	i915_guc_client_info(m, dev_priv, guc->execbuf_client);
+	seq_printf(m, "\nGuC preempt client @ %p:\n", guc->preempt_client);
+	i915_guc_client_info(m, dev_priv, guc->preempt_client);
 
 	i915_guc_log_info(m, dev_priv);
 
@@ -2497,12 +2358,12 @@ static int i915_guc_stage_pool(struct seq_file *m, void *data)
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	const struct intel_guc *guc = &dev_priv->guc;
 	struct guc_stage_desc *desc = guc->stage_desc_pool_vaddr;
-	struct i915_guc_client *client = guc->execbuf_client;
+	struct intel_guc_client *client = guc->execbuf_client;
 	unsigned int tmp;
 	int index;
 
-	if (!check_guc_submission(m))
-		return 0;
+	if (!USES_GUC_SUBMISSION(dev_priv))
+		return -ENODEV;
 
 	for (index = 0; index < GUC_MAX_STAGE_DESCRIPTORS; index++, desc++) {
 		struct intel_engine_cs *engine;
@@ -2555,6 +2416,9 @@ static int i915_guc_log_dump(struct seq_file *m, void *data)
 	u32 *log;
 	int i = 0;
 
+	if (!HAS_GUC(dev_priv))
+		return -ENODEV;
+
 	if (dump_load_err)
 		obj = dev_priv->guc.load_err_log;
 	else if (dev_priv->guc.log.vma)
@@ -2586,6 +2450,9 @@ static int i915_guc_log_control_get(void *data, u64 *val)
 {
 	struct drm_i915_private *dev_priv = data;
 
+	if (!HAS_GUC(dev_priv))
+		return -ENODEV;
+
 	if (!dev_priv->guc.log.vma)
 		return -EINVAL;
 
@@ -2599,6 +2466,9 @@ static int i915_guc_log_control_set(void *data, u64 val)
 	struct drm_i915_private *dev_priv = data;
 	int ret;
 
+	if (!HAS_GUC(dev_priv))
+		return -ENODEV;
+
 	if (!dev_priv->guc.log.vma)
 		return -EINVAL;
 
@@ -2649,10 +2519,8 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 	enum pipe pipe;
 	bool enabled = false;
 
-	if (!HAS_PSR(dev_priv)) {
-		seq_puts(m, "PSR not supported\n");
-		return 0;
-	}
+	if (!HAS_PSR(dev_priv))
+		return -ENODEV;
 
 	intel_runtime_pm_get(dev_priv);
 
@@ -2734,39 +2602,76 @@ static int i915_sink_crc(struct seq_file *m, void *data)
 	struct intel_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	struct intel_dp *intel_dp = NULL;
+	struct drm_modeset_acquire_ctx ctx;
 	int ret;
 	u8 crc[6];
 
-	drm_modeset_lock_all(dev);
+	drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
+
 	drm_connector_list_iter_begin(dev, &conn_iter);
+
 	for_each_intel_connector_iter(connector, &conn_iter) {
 		struct drm_crtc *crtc;
+		struct drm_connector_state *state;
+		struct intel_crtc_state *crtc_state;
 
-		if (!connector->base.state->best_encoder)
+		if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
 			continue;
 
-		crtc = connector->base.state->crtc;
-		if (!crtc->state->active)
+retry:
+		ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx);
+		if (ret)
+			goto err;
+
+		state = connector->base.state;
+		if (!state->best_encoder)
 			continue;
 
-		if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
+		crtc = state->crtc;
+		ret = drm_modeset_lock(&crtc->mutex, &ctx);
+		if (ret)
+			goto err;
+
+		crtc_state = to_intel_crtc_state(crtc->state);
+		if (!crtc_state->base.active)
 			continue;
 
-		intel_dp = enc_to_intel_dp(connector->base.state->best_encoder);
+		/*
+		 * We need to wait for all crtc updates to complete, to make
+		 * sure any pending modesets and plane updates are completed.
+		 */
+		if (crtc_state->base.commit) {
+			ret = wait_for_completion_interruptible(&crtc_state->base.commit->hw_done);
+
+			if (ret)
+				goto err;
+		}
 
-		ret = intel_dp_sink_crc(intel_dp, crc);
+		intel_dp = enc_to_intel_dp(state->best_encoder);
+
+		ret = intel_dp_sink_crc(intel_dp, crtc_state, crc);
 		if (ret)
-			goto out;
+			goto err;
 
 		seq_printf(m, "%02x%02x%02x%02x%02x%02x\n",
 			   crc[0], crc[1], crc[2],
 			   crc[3], crc[4], crc[5]);
 		goto out;
+
+err:
+		if (ret == -EDEADLK) {
+			ret = drm_modeset_backoff(&ctx);
+			if (!ret)
+				goto retry;
+		}
+		goto out;
 	}
 	ret = -ENODEV;
 out:
 	drm_connector_list_iter_end(&conn_iter);
-	drm_modeset_unlock_all(dev);
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
 	return ret;
 }
 
@@ -2854,10 +2759,8 @@ static int i915_dmc_info(struct seq_file *m, void *unused)
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct intel_csr *csr;
 
-	if (!HAS_CSR(dev_priv)) {
-		seq_puts(m, "not supported\n");
-		return 0;
-	}
+	if (!HAS_CSR(dev_priv))
+		return -ENODEV;
 
 	csr = &dev_priv->csr;
 
@@ -3049,7 +2952,7 @@ static void intel_connector_info(struct seq_file *m,
 		break;
 	case DRM_MODE_CONNECTOR_HDMIA:
 		if (intel_encoder->type == INTEL_OUTPUT_HDMI ||
-		    intel_encoder->type == INTEL_OUTPUT_UNKNOWN)
+		    intel_encoder->type == INTEL_OUTPUT_DDI)
 			intel_hdmi_info(m, intel_connector);
 		break;
 	default:
@@ -3244,10 +3147,12 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 		   yesno(dev_priv->gt.awake));
 	seq_printf(m, "Global active requests: %d\n",
 		   dev_priv->gt.active_requests);
+	seq_printf(m, "CS timestamp frequency: %u kHz\n",
+		   dev_priv->info.cs_timestamp_frequency_khz);
 
 	p = drm_seq_file_printer(m);
 	for_each_engine(engine, dev_priv, id)
-		intel_engine_dump(engine, &p);
+		intel_engine_dump(engine, &p, "%s\n", engine->name);
 
 	intel_runtime_pm_put(dev_priv);
 
@@ -3264,69 +3169,6 @@ static int i915_shrinker_info(struct seq_file *m, void *unused)
 	return 0;
 }
 
-static int i915_semaphore_status(struct seq_file *m, void *unused)
-{
-	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_engine_cs *engine;
-	int num_rings = INTEL_INFO(dev_priv)->num_rings;
-	enum intel_engine_id id;
-	int j, ret;
-
-	if (!i915_modparams.semaphores) {
-		seq_puts(m, "Semaphores are disabled\n");
-		return 0;
-	}
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-	intel_runtime_pm_get(dev_priv);
-
-	if (IS_BROADWELL(dev_priv)) {
-		struct page *page;
-		uint64_t *seqno;
-
-		page = i915_gem_object_get_page(dev_priv->semaphore->obj, 0);
-
-		seqno = (uint64_t *)kmap_atomic(page);
-		for_each_engine(engine, dev_priv, id) {
-			uint64_t offset;
-
-			seq_printf(m, "%s\n", engine->name);
-
-			seq_puts(m, "  Last signal:");
-			for (j = 0; j < num_rings; j++) {
-				offset = id * I915_NUM_ENGINES + j;
-				seq_printf(m, "0x%08llx (0x%02llx) ",
-					   seqno[offset], offset * 8);
-			}
-			seq_putc(m, '\n');
-
-			seq_puts(m, "  Last wait:  ");
-			for (j = 0; j < num_rings; j++) {
-				offset = id + (j * I915_NUM_ENGINES);
-				seq_printf(m, "0x%08llx (0x%02llx) ",
-					   seqno[offset], offset * 8);
-			}
-			seq_putc(m, '\n');
-
-		}
-		kunmap_atomic(seqno);
-	} else {
-		seq_puts(m, "  Last signal:");
-		for_each_engine(engine, dev_priv, id)
-			for (j = 0; j < num_rings; j++)
-				seq_printf(m, "0x%08x\n",
-					   I915_READ(engine->semaphore.mbox.signal[j]));
-		seq_putc(m, '\n');
-	}
-
-	intel_runtime_pm_put(dev_priv);
-	mutex_unlock(&dev->struct_mutex);
-	return 0;
-}
-
 static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -3454,7 +3296,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused)
 	int plane;
 
 	if (INTEL_GEN(dev_priv) < 9)
-		return 0;
+		return -ENODEV;
 
 	drm_modeset_lock_all(dev);
 
@@ -3601,7 +3443,7 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused)
 			continue;
 
 		seq_printf(m, "MST Source Port %c\n",
-			   port_name(intel_dig_port->port));
+			   port_name(intel_dig_port->base.port));
 		drm_dp_mst_dump_topology(m, &intel_dig_port->dp.mst_mgr);
 	}
 	drm_connector_list_iter_end(&conn_iter);
@@ -4448,6 +4290,61 @@ static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
 	}
 }
 
+static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
+				     struct sseu_dev_info *sseu)
+{
+	const struct intel_device_info *info = INTEL_INFO(dev_priv);
+	int s_max = 6, ss_max = 4;
+	int s, ss;
+	u32 s_reg[s_max], eu_reg[2 * s_max], eu_mask[2];
+
+	for (s = 0; s < s_max; s++) {
+		/*
+		 * FIXME: Valid SS Mask respects the spec and read
+		 * only valid bits for those registers, excluding reserverd
+		 * although this seems wrong because it would leave many
+		 * subslices without ACK.
+		 */
+		s_reg[s] = I915_READ(GEN10_SLICE_PGCTL_ACK(s)) &
+			GEN10_PGCTL_VALID_SS_MASK(s);
+		eu_reg[2 * s] = I915_READ(GEN10_SS01_EU_PGCTL_ACK(s));
+		eu_reg[2 * s + 1] = I915_READ(GEN10_SS23_EU_PGCTL_ACK(s));
+	}
+
+	eu_mask[0] = GEN9_PGCTL_SSA_EU08_ACK |
+		     GEN9_PGCTL_SSA_EU19_ACK |
+		     GEN9_PGCTL_SSA_EU210_ACK |
+		     GEN9_PGCTL_SSA_EU311_ACK;
+	eu_mask[1] = GEN9_PGCTL_SSB_EU08_ACK |
+		     GEN9_PGCTL_SSB_EU19_ACK |
+		     GEN9_PGCTL_SSB_EU210_ACK |
+		     GEN9_PGCTL_SSB_EU311_ACK;
+
+	for (s = 0; s < s_max; s++) {
+		if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0)
+			/* skip disabled slice */
+			continue;
+
+		sseu->slice_mask |= BIT(s);
+		sseu->subslice_mask = info->sseu.subslice_mask;
+
+		for (ss = 0; ss < ss_max; ss++) {
+			unsigned int eu_cnt;
+
+			if (!(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss))))
+				/* skip disabled subslice */
+				continue;
+
+			eu_cnt = 2 * hweight32(eu_reg[2 * s + ss / 2] &
+					       eu_mask[ss % 2]);
+			sseu->eu_total += eu_cnt;
+			sseu->eu_per_subslice = max_t(unsigned int,
+						      sseu->eu_per_subslice,
+						      eu_cnt);
+		}
+	}
+}
+
 static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
 				    struct sseu_dev_info *sseu)
 {
@@ -4483,7 +4380,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
 
 		sseu->slice_mask |= BIT(s);
 
-		if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv))
+		if (IS_GEN9_BC(dev_priv))
 			sseu->subslice_mask =
 				INTEL_INFO(dev_priv)->sseu.subslice_mask;
 
@@ -4589,8 +4486,10 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
 		cherryview_sseu_device_status(dev_priv, &sseu);
 	} else if (IS_BROADWELL(dev_priv)) {
 		broadwell_sseu_device_status(dev_priv, &sseu);
-	} else if (INTEL_GEN(dev_priv) >= 9) {
+	} else if (IS_GEN9(dev_priv)) {
 		gen9_sseu_device_status(dev_priv, &sseu);
+	} else if (INTEL_GEN(dev_priv) >= 10) {
+		gen10_sseu_device_status(dev_priv, &sseu);
 	}
 
 	intel_runtime_pm_put(dev_priv);
@@ -4712,7 +4611,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_gem_objects", i915_gem_object_info, 0},
 	{"i915_gem_gtt", i915_gem_gtt_info, 0},
 	{"i915_gem_stolen", i915_gem_stolen_list_info },
-	{"i915_gem_seqno", i915_gem_seqno_info, 0},
 	{"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
 	{"i915_gem_interrupt", i915_interrupt_info, 0},
 	{"i915_gem_batch_pool", i915_gem_batch_pool_info, 0},
@@ -4736,7 +4634,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_vbt", i915_vbt, 0},
 	{"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
 	{"i915_context_status", i915_context_status, 0},
-	{"i915_dump_lrc", i915_dump_lrc, 0},
 	{"i915_forcewake_domains", i915_forcewake_domains, 0},
 	{"i915_swizzle_info", i915_swizzle_info, 0},
 	{"i915_ppgtt_info", i915_ppgtt_info, 0},
@@ -4750,7 +4647,6 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_display_info", i915_display_info, 0},
 	{"i915_engine_info", i915_engine_info, 0},
 	{"i915_shrinker_info", i915_shrinker_info, 0},
-	{"i915_semaphore_status", i915_semaphore_status, 0},
 	{"i915_shared_dplls_info", i915_shared_dplls_info, 0},
 	{"i915_dp_mst_info", i915_dp_mst_info, 0},
 	{"i915_wa_registers", i915_wa_registers, 0},
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 2cf10d17acfb..2f5209de0391 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -48,6 +48,7 @@
 
 #include "i915_drv.h"
 #include "i915_trace.h"
+#include "i915_pmu.h"
 #include "i915_vgpu.h"
 #include "intel_drv.h"
 #include "intel_uc.h"
@@ -321,7 +322,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		value = USES_PPGTT(dev_priv);
 		break;
 	case I915_PARAM_HAS_SEMAPHORES:
-		value = i915_modparams.semaphores;
+		value = HAS_LEGACY_SEMAPHORES(dev_priv);
 		break;
 	case I915_PARAM_HAS_SECURE_BATCHES:
 		value = capable(CAP_SYS_ADMIN);
@@ -371,10 +372,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		if (dev_priv->engine[RCS] && dev_priv->engine[RCS]->schedule) {
 			value |= I915_SCHEDULER_CAP_ENABLED;
 			value |= I915_SCHEDULER_CAP_PRIORITY;
-
-			if (INTEL_INFO(dev_priv)->has_logical_ring_preemption &&
-			    i915_modparams.enable_execlists &&
-			    !i915_modparams.enable_guc_submission)
+			if (HAS_LOGICAL_RING_PREEMPTION(dev_priv))
 				value |= I915_SCHEDULER_CAP_PREEMPTION;
 		}
 		break;
@@ -407,6 +405,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		 */
 		value = 1;
 		break;
+	case I915_PARAM_HAS_CONTEXT_ISOLATION:
+		value = intel_engines_has_context_isolation(dev_priv);
+		break;
 	case I915_PARAM_SLICE_MASK:
 		value = INTEL_INFO(dev_priv)->sseu.slice_mask;
 		if (!value)
@@ -417,6 +418,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		if (!value)
 			return -ENODEV;
 		break;
+	case I915_PARAM_CS_TIMESTAMP_FREQUENCY:
+		value = 1000 * INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz;
+		break;
 	default:
 		DRM_DEBUG("Unknown parameter %d\n", param->param);
 		return -EINVAL;
@@ -613,10 +617,12 @@ static void i915_gem_fini(struct drm_i915_private *dev_priv)
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
 	intel_uc_fini_hw(dev_priv);
+	intel_uc_fini(dev_priv);
 	i915_gem_cleanup_engines(dev_priv);
 	i915_gem_contexts_fini(dev_priv);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
+	intel_uc_fini_wq(dev_priv);
 	i915_gem_cleanup_userptr(dev_priv);
 
 	i915_gem_drain_freed_objects(dev_priv);
@@ -677,7 +683,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	if (ret)
 		goto cleanup_uc;
 
-	intel_modeset_gem_init(dev);
+	intel_setup_overlay(dev_priv);
 
 	if (INTEL_INFO(dev_priv)->num_pipes == 0)
 		return 0;
@@ -689,8 +695,6 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	/* Only enable hotplug handling once the fbdev is fully set up. */
 	intel_hpd_init(dev_priv);
 
-	drm_kms_helper_poll_init(dev);
-
 	return 0;
 
 cleanup_gem:
@@ -724,7 +728,7 @@ static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
 	if (!ap)
 		return -ENOMEM;
 
-	ap->ranges[0].base = ggtt->mappable_base;
+	ap->ranges[0].base = ggtt->gmadr.start;
 	ap->ranges[0].size = ggtt->mappable_end;
 
 	primary =
@@ -838,6 +842,11 @@ static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
  * We don't keep the workarounds for pre-production hardware, so we expect our
  * driver to fail on these machines in one way or another. A little warning on
  * dmesg may help both the user and the bug triagers.
+ *
+ * Our policy for removing pre-production workarounds is to keep the
+ * current gen workarounds as a guide to the bring-up of the next gen
+ * (workarounds have a habit of persisting!). Anything older than that
+ * should be removed along with the complications they introduce.
  */
 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
 {
@@ -892,7 +901,6 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 	mutex_init(&dev_priv->backlight_lock);
 	spin_lock_init(&dev_priv->uncore.lock);
 
-	spin_lock_init(&dev_priv->mm.object_stat_lock);
 	mutex_init(&dev_priv->sb_lock);
 	mutex_init(&dev_priv->modeset_restore_lock);
 	mutex_init(&dev_priv->av_mutex);
@@ -923,12 +931,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
 
 	intel_display_crc_init(dev_priv);
 
-	intel_device_info_dump(dev_priv);
-
 	intel_detect_preproduction_hw(dev_priv);
 
-	i915_perf_init(dev_priv);
-
 	return 0;
 
 err_irq:
@@ -945,7 +949,6 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
  */
 static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
 {
-	i915_perf_fini(dev_priv);
 	i915_gem_load_cleanup(dev_priv);
 	intel_irq_fini(dev_priv);
 	i915_workqueues_cleanup(dev_priv);
@@ -1048,10 +1051,6 @@ static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
 
 static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 {
-	i915_modparams.enable_execlists =
-		intel_sanitize_enable_execlists(dev_priv,
-						i915_modparams.enable_execlists);
-
 	/*
 	 * i915.enable_ppgtt is read-only, so do an early pass to validate the
 	 * user's requested state against the hardware/driver capabilities.  We
@@ -1063,11 +1062,6 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 					    i915_modparams.enable_ppgtt);
 	DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915_modparams.enable_ppgtt);
 
-	i915_modparams.semaphores =
-		intel_sanitize_semaphores(dev_priv, i915_modparams.semaphores);
-	DRM_DEBUG_DRIVER("use GPU semaphores? %s\n",
-			 yesno(i915_modparams.semaphores));
-
 	intel_uc_sanitize_options(dev_priv);
 
 	intel_gvt_sanitize_options(dev_priv);
@@ -1088,10 +1082,12 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 	if (i915_inject_load_failure())
 		return -ENODEV;
 
-	intel_device_info_runtime_init(dev_priv);
+	intel_device_info_runtime_init(mkwrite_device_info(dev_priv));
 
 	intel_sanitize_options(dev_priv);
 
+	i915_perf_init(dev_priv);
+
 	ret = i915_ggtt_probe_hw(dev_priv);
 	if (ret)
 		return ret;
@@ -1197,6 +1193,8 @@ static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
 {
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 
+	i915_perf_fini(dev_priv);
+
 	if (pdev->msi_enabled)
 		pci_disable_msi(pdev);
 
@@ -1215,7 +1213,8 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
 
-	i915_gem_shrinker_init(dev_priv);
+	i915_gem_shrinker_register(dev_priv);
+	i915_pmu_register(dev_priv);
 
 	/*
 	 * Notify a valid surface after modesetting,
@@ -1254,6 +1253,13 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
 	 * cannot run before the connectors are registered.
 	 */
 	intel_fbdev_initial_config_async(dev);
+
+	/*
+	 * We need to coordinate the hotplugs with the asynchronous fbdev
+	 * configuration, for which we use the fbdev->async_cookie.
+	 */
+	if (INTEL_INFO(dev_priv)->num_pipes)
+		drm_kms_helper_poll_init(dev);
 }
 
 /**
@@ -1265,17 +1271,40 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 	intel_fbdev_unregister(dev_priv);
 	intel_audio_deinit(dev_priv);
 
+	/*
+	 * After flushing the fbdev (incl. a late async config which will
+	 * have delayed queuing of a hotplug event), then flush the hotplug
+	 * events.
+	 */
+	drm_kms_helper_poll_fini(&dev_priv->drm);
+
 	intel_gpu_ips_teardown();
 	acpi_video_unregister();
 	intel_opregion_unregister(dev_priv);
 
 	i915_perf_unregister(dev_priv);
+	i915_pmu_unregister(dev_priv);
 
 	i915_teardown_sysfs(dev_priv);
 	i915_guc_log_unregister(dev_priv);
 	drm_dev_unregister(&dev_priv->drm);
 
-	i915_gem_shrinker_cleanup(dev_priv);
+	i915_gem_shrinker_unregister(dev_priv);
+}
+
+static void i915_welcome_messages(struct drm_i915_private *dev_priv)
+{
+	if (drm_debug & DRM_UT_DRIVER) {
+		struct drm_printer p = drm_debug_printer("i915 device info:");
+
+		intel_device_info_dump(&dev_priv->info, &p);
+		intel_device_info_dump_runtime(&dev_priv->info, &p);
+	}
+
+	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+		DRM_INFO("DRM_I915_DEBUG enabled\n");
+	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
+		DRM_INFO("DRM_I915_DEBUG_GEM enabled\n");
 }
 
 /**
@@ -1363,13 +1392,10 @@ int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	intel_init_ipc(dev_priv);
 
-	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
-		DRM_INFO("DRM_I915_DEBUG enabled\n");
-	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
-		DRM_INFO("DRM_I915_DEBUG_GEM enabled\n");
-
 	intel_runtime_pm_put(dev_priv);
 
+	i915_welcome_messages(dev_priv);
+
 	return 0;
 
 out_cleanup_hw:
@@ -1407,19 +1433,7 @@ void i915_driver_unload(struct drm_device *dev)
 
 	intel_modeset_cleanup(dev);
 
-	/*
-	 * free the memory space allocated for the child device
-	 * config parsed from VBT
-	 */
-	if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
-		kfree(dev_priv->vbt.child_dev);
-		dev_priv->vbt.child_dev = NULL;
-		dev_priv->vbt.child_dev_num = 0;
-	}
-	kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
-	dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
-	kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
-	dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
+	intel_bios_cleanup(dev_priv);
 
 	vga_switcheroo_unregister_client(pdev);
 	vga_client_register(pdev, NULL, NULL, NULL);
@@ -1682,8 +1696,6 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	intel_csr_ucode_resume(dev_priv);
 
-	i915_gem_resume(dev_priv);
-
 	i915_restore_state(dev_priv);
 	intel_pps_unlock_regs_wa(dev_priv);
 	intel_opregion_setup(dev_priv);
@@ -1704,14 +1716,7 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	drm_mode_config_reset(dev);
 
-	mutex_lock(&dev->struct_mutex);
-	if (i915_gem_init_hw(dev_priv)) {
-		DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
-		i915_gem_set_wedged(dev_priv);
-	}
-	mutex_unlock(&dev->struct_mutex);
-
-	intel_guc_resume(dev_priv);
+	i915_gem_resume(dev_priv);
 
 	intel_modeset_init_hw(dev);
 	intel_init_clock_gating(dev_priv);
@@ -1745,8 +1750,6 @@ static int i915_drm_resume(struct drm_device *dev)
 
 	intel_opregion_notify_adapter(dev_priv, PCI_D0);
 
-	intel_autoenable_gt_powersave(dev_priv);
-
 	enable_rpm_wakeref_asserts(dev_priv);
 
 	return 0;
@@ -1827,6 +1830,8 @@ static int i915_drm_resume_early(struct drm_device *dev)
 	if (IS_GEN9_LP(dev_priv) ||
 	    !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
 		intel_power_domains_init_hw(dev_priv, true);
+	else
+		intel_display_set_init_power(dev_priv, true);
 
 	i915_gem_sanitize(dev_priv);
 
@@ -1874,7 +1879,9 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 {
 	struct i915_gpu_error *error = &i915->gpu_error;
 	int ret;
+	int i;
 
+	might_sleep();
 	lockdep_assert_held(&i915->drm.struct_mutex);
 	GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags));
 
@@ -1892,22 +1899,30 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 	disable_irq(i915->drm.irq);
 	ret = i915_gem_reset_prepare(i915);
 	if (ret) {
-		DRM_ERROR("GPU recovery failed\n");
+		dev_err(i915->drm.dev, "GPU recovery failed\n");
 		intel_gpu_reset(i915, ALL_ENGINES);
-		goto error;
+		goto taint;
 	}
 
-	ret = intel_gpu_reset(i915, ALL_ENGINES);
-	if (ret) {
-		if (ret != -ENODEV)
-			DRM_ERROR("Failed to reset chip: %i\n", ret);
+	if (!intel_has_gpu_reset(i915)) {
+		if (i915_modparams.reset)
+			dev_err(i915->drm.dev, "GPU reset not supported\n");
 		else
 			DRM_DEBUG_DRIVER("GPU reset disabled\n");
 		goto error;
 	}
 
-	i915_gem_reset(i915);
-	intel_overlay_reset(i915);
+	for (i = 0; i < 3; i++) {
+		ret = intel_gpu_reset(i915, ALL_ENGINES);
+		if (ret == 0)
+			break;
+
+		msleep(100);
+	}
+	if (ret) {
+		dev_err(i915->drm.dev, "Failed to reset chip\n");
+		goto taint;
+	}
 
 	/* Ok, now get things going again... */
 
@@ -1921,6 +1936,9 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 		goto error;
 	}
 
+	i915_gem_reset(i915);
+	intel_overlay_reset(i915);
+
 	/*
 	 * Next we need to restore the context, but we don't use those
 	 * yet either...
@@ -1946,12 +1964,32 @@ void i915_reset(struct drm_i915_private *i915, unsigned int flags)
 	wake_up_bit(&error->flags, I915_RESET_HANDOFF);
 	return;
 
+taint:
+	/*
+	 * History tells us that if we cannot reset the GPU now, we
+	 * never will. This then impacts everything that is run
+	 * subsequently. On failing the reset, we mark the driver
+	 * as wedged, preventing further execution on the GPU.
+	 * We also want to go one step further and add a taint to the
+	 * kernel so that any subsequent faults can be traced back to
+	 * this failure. This is important for CI, where if the
+	 * GPU/driver fails we would like to reboot and restart testing
+	 * rather than continue on into oblivion. For everyone else,
+	 * the system should still plod along, but they have been warned!
+	 */
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
 error:
 	i915_gem_set_wedged(i915);
 	i915_gem_retire_requests(i915);
 	goto finish;
 }
 
+static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv,
+					struct intel_engine_cs *engine)
+{
+	return intel_gpu_reset(dev_priv, intel_engine_flag(engine));
+}
+
 /**
  * i915_reset_engine - reset GPU engine to recover from a hang
  * @engine: engine to reset
@@ -1973,23 +2011,27 @@ int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
 
 	GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &error->flags));
 
+	active_request = i915_gem_reset_prepare_engine(engine);
+	if (IS_ERR_OR_NULL(active_request)) {
+		/* Either the previous reset failed, or we pardon the reset. */
+		ret = PTR_ERR(active_request);
+		goto out;
+	}
+
 	if (!(flags & I915_RESET_QUIET)) {
 		dev_notice(engine->i915->drm.dev,
 			   "Resetting %s after gpu hang\n", engine->name);
 	}
 	error->reset_engine_count[engine->id]++;
 
-	active_request = i915_gem_reset_prepare_engine(engine);
-	if (IS_ERR(active_request)) {
-		DRM_DEBUG_DRIVER("Previous reset failed, promote to full reset\n");
-		ret = PTR_ERR(active_request);
-		goto out;
-	}
-
-	ret = intel_gpu_reset(engine->i915, intel_engine_flag(engine));
+	if (!engine->i915->guc.execbuf_client)
+		ret = intel_gt_reset_engine(engine->i915, engine);
+	else
+		ret = intel_guc_reset_engine(&engine->i915->guc, engine);
 	if (ret) {
 		/* If we fail here, we expect to fallback to a global reset */
-		DRM_DEBUG_DRIVER("Failed to reset %s, ret=%d\n",
+		DRM_DEBUG_DRIVER("%sFailed to reset %s, ret=%d\n",
+				 engine->i915->guc.execbuf_client ? "GuC " : "",
 				 engine->name, ret);
 		goto out;
 	}
@@ -2504,7 +2546,7 @@ static int intel_runtime_suspend(struct device *kdev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int ret;
 
-	if (WARN_ON_ONCE(!(dev_priv->gt_pm.rc6.enabled && intel_rc6_enabled())))
+	if (WARN_ON_ONCE(!(dev_priv->gt_pm.rc6.enabled && HAS_RC6(dev_priv))))
 		return -ENODEV;
 
 	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev_priv)))
@@ -2524,6 +2566,8 @@ static int intel_runtime_suspend(struct device *kdev)
 
 	intel_runtime_pm_disable_interrupts(dev_priv);
 
+	intel_uncore_suspend(dev_priv);
+
 	ret = 0;
 	if (IS_GEN9_LP(dev_priv)) {
 		bxt_display_core_uninit(dev_priv);
@@ -2536,6 +2580,8 @@ static int intel_runtime_suspend(struct device *kdev)
 
 	if (ret) {
 		DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
+		intel_uncore_runtime_resume(dev_priv);
+
 		intel_runtime_pm_enable_interrupts(dev_priv);
 
 		enable_rpm_wakeref_asserts(dev_priv);
@@ -2543,8 +2589,6 @@ static int intel_runtime_suspend(struct device *kdev)
 		return ret;
 	}
 
-	intel_uncore_suspend(dev_priv);
-
 	enable_rpm_wakeref_asserts(dev_priv);
 	WARN_ON_ONCE(atomic_read(&dev_priv->runtime_pm.wakeref_count));
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index e143004e66d5..d307429a5ae0 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -40,6 +40,7 @@
 #include <linux/hash.h>
 #include <linux/intel-iommu.h>
 #include <linux/kref.h>
+#include <linux/perf_event.h>
 #include <linux/pm_qos.h>
 #include <linux/reservation.h>
 #include <linux/shmem_fs.h>
@@ -55,19 +56,21 @@
 #include "i915_reg.h"
 #include "i915_utils.h"
 
-#include "intel_uncore.h"
 #include "intel_bios.h"
+#include "intel_device_info.h"
+#include "intel_display.h"
 #include "intel_dpll_mgr.h"
-#include "intel_uc.h"
 #include "intel_lrc.h"
+#include "intel_opregion.h"
 #include "intel_ringbuffer.h"
+#include "intel_uncore.h"
+#include "intel_uc.h"
 
 #include "i915_gem.h"
 #include "i915_gem_context.h"
 #include "i915_gem_fence_reg.h"
 #include "i915_gem_object.h"
 #include "i915_gem_gtt.h"
-#include "i915_gem_render_state.h"
 #include "i915_gem_request.h"
 #include "i915_gem_timeline.h"
 
@@ -80,8 +83,8 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20171023"
-#define DRIVER_TIMESTAMP	1508748913
+#define DRIVER_DATE		"20171222"
+#define DRIVER_TIMESTAMP	1513971710
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -243,173 +246,6 @@ static inline uint_fixed_16_16_t add_fixed16_u32(uint_fixed_16_16_t add1,
 	return clamp_u64_to_fixed16(interm_sum);
 }
 
-static inline const char *yesno(bool v)
-{
-	return v ? "yes" : "no";
-}
-
-static inline const char *onoff(bool v)
-{
-	return v ? "on" : "off";
-}
-
-static inline const char *enableddisabled(bool v)
-{
-	return v ? "enabled" : "disabled";
-}
-
-enum pipe {
-	INVALID_PIPE = -1,
-	PIPE_A = 0,
-	PIPE_B,
-	PIPE_C,
-	_PIPE_EDP,
-	I915_MAX_PIPES = _PIPE_EDP
-};
-#define pipe_name(p) ((p) + 'A')
-
-enum transcoder {
-	TRANSCODER_A = 0,
-	TRANSCODER_B,
-	TRANSCODER_C,
-	TRANSCODER_EDP,
-	TRANSCODER_DSI_A,
-	TRANSCODER_DSI_C,
-	I915_MAX_TRANSCODERS
-};
-
-static inline const char *transcoder_name(enum transcoder transcoder)
-{
-	switch (transcoder) {
-	case TRANSCODER_A:
-		return "A";
-	case TRANSCODER_B:
-		return "B";
-	case TRANSCODER_C:
-		return "C";
-	case TRANSCODER_EDP:
-		return "EDP";
-	case TRANSCODER_DSI_A:
-		return "DSI A";
-	case TRANSCODER_DSI_C:
-		return "DSI C";
-	default:
-		return "<invalid>";
-	}
-}
-
-static inline bool transcoder_is_dsi(enum transcoder transcoder)
-{
-	return transcoder == TRANSCODER_DSI_A || transcoder == TRANSCODER_DSI_C;
-}
-
-/*
- * Global legacy plane identifier. Valid only for primary/sprite
- * planes on pre-g4x, and only for primary planes on g4x+.
- */
-enum plane {
-	PLANE_A,
-	PLANE_B,
-	PLANE_C,
-};
-#define plane_name(p) ((p) + 'A')
-
-#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A')
-
-/*
- * Per-pipe plane identifier.
- * I915_MAX_PLANES in the enum below is the maximum (across all platforms)
- * number of planes per CRTC.  Not all platforms really have this many planes,
- * which means some arrays of size I915_MAX_PLANES may have unused entries
- * between the topmost sprite plane and the cursor plane.
- *
- * This is expected to be passed to various register macros
- * (eg. PLANE_CTL(), PS_PLANE_SEL(), etc.) so adjust with care.
- */
-enum plane_id {
-	PLANE_PRIMARY,
-	PLANE_SPRITE0,
-	PLANE_SPRITE1,
-	PLANE_SPRITE2,
-	PLANE_CURSOR,
-	I915_MAX_PLANES,
-};
-
-#define for_each_plane_id_on_crtc(__crtc, __p) \
-	for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \
-		for_each_if ((__crtc)->plane_ids_mask & BIT(__p))
-
-enum port {
-	PORT_NONE = -1,
-	PORT_A = 0,
-	PORT_B,
-	PORT_C,
-	PORT_D,
-	PORT_E,
-	I915_MAX_PORTS
-};
-#define port_name(p) ((p) + 'A')
-
-#define I915_NUM_PHYS_VLV 2
-
-enum dpio_channel {
-	DPIO_CH0,
-	DPIO_CH1
-};
-
-enum dpio_phy {
-	DPIO_PHY0,
-	DPIO_PHY1,
-	DPIO_PHY2,
-};
-
-enum intel_display_power_domain {
-	POWER_DOMAIN_PIPE_A,
-	POWER_DOMAIN_PIPE_B,
-	POWER_DOMAIN_PIPE_C,
-	POWER_DOMAIN_PIPE_A_PANEL_FITTER,
-	POWER_DOMAIN_PIPE_B_PANEL_FITTER,
-	POWER_DOMAIN_PIPE_C_PANEL_FITTER,
-	POWER_DOMAIN_TRANSCODER_A,
-	POWER_DOMAIN_TRANSCODER_B,
-	POWER_DOMAIN_TRANSCODER_C,
-	POWER_DOMAIN_TRANSCODER_EDP,
-	POWER_DOMAIN_TRANSCODER_DSI_A,
-	POWER_DOMAIN_TRANSCODER_DSI_C,
-	POWER_DOMAIN_PORT_DDI_A_LANES,
-	POWER_DOMAIN_PORT_DDI_B_LANES,
-	POWER_DOMAIN_PORT_DDI_C_LANES,
-	POWER_DOMAIN_PORT_DDI_D_LANES,
-	POWER_DOMAIN_PORT_DDI_E_LANES,
-	POWER_DOMAIN_PORT_DDI_A_IO,
-	POWER_DOMAIN_PORT_DDI_B_IO,
-	POWER_DOMAIN_PORT_DDI_C_IO,
-	POWER_DOMAIN_PORT_DDI_D_IO,
-	POWER_DOMAIN_PORT_DDI_E_IO,
-	POWER_DOMAIN_PORT_DSI,
-	POWER_DOMAIN_PORT_CRT,
-	POWER_DOMAIN_PORT_OTHER,
-	POWER_DOMAIN_VGA,
-	POWER_DOMAIN_AUDIO,
-	POWER_DOMAIN_PLLS,
-	POWER_DOMAIN_AUX_A,
-	POWER_DOMAIN_AUX_B,
-	POWER_DOMAIN_AUX_C,
-	POWER_DOMAIN_AUX_D,
-	POWER_DOMAIN_GMBUS,
-	POWER_DOMAIN_MODESET,
-	POWER_DOMAIN_INIT,
-
-	POWER_DOMAIN_NUM,
-};
-
-#define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A)
-#define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \
-		((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER)
-#define POWER_DOMAIN_TRANSCODER(tran) \
-	((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \
-	 (tran) + POWER_DOMAIN_TRANSCODER_A)
-
 enum hpd_pin {
 	HPD_NONE = 0,
 	HPD_TV = HPD_NONE,     /* TV is known to be unreliable */
@@ -471,122 +307,6 @@ struct i915_hotplug {
 	 I915_GEM_DOMAIN_INSTRUCTION | \
 	 I915_GEM_DOMAIN_VERTEX)
 
-#define for_each_pipe(__dev_priv, __p) \
-	for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++)
-#define for_each_pipe_masked(__dev_priv, __p, __mask) \
-	for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \
-		for_each_if ((__mask) & (1 << (__p)))
-#define for_each_universal_plane(__dev_priv, __pipe, __p)		\
-	for ((__p) = 0;							\
-	     (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1;	\
-	     (__p)++)
-#define for_each_sprite(__dev_priv, __p, __s)				\
-	for ((__s) = 0;							\
-	     (__s) < INTEL_INFO(__dev_priv)->num_sprites[(__p)];	\
-	     (__s)++)
-
-#define for_each_port_masked(__port, __ports_mask) \
-	for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++)	\
-		for_each_if ((__ports_mask) & (1 << (__port)))
-
-#define for_each_crtc(dev, crtc) \
-	list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head)
-
-#define for_each_intel_plane(dev, intel_plane) \
-	list_for_each_entry(intel_plane,			\
-			    &(dev)->mode_config.plane_list,	\
-			    base.head)
-
-#define for_each_intel_plane_mask(dev, intel_plane, plane_mask)		\
-	list_for_each_entry(intel_plane,				\
-			    &(dev)->mode_config.plane_list,		\
-			    base.head)					\
-		for_each_if ((plane_mask) &				\
-			     (1 << drm_plane_index(&intel_plane->base)))
-
-#define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane)	\
-	list_for_each_entry(intel_plane,				\
-			    &(dev)->mode_config.plane_list,		\
-			    base.head)					\
-		for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe)
-
-#define for_each_intel_crtc(dev, intel_crtc)				\
-	list_for_each_entry(intel_crtc,					\
-			    &(dev)->mode_config.crtc_list,		\
-			    base.head)
-
-#define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask)		\
-	list_for_each_entry(intel_crtc,					\
-			    &(dev)->mode_config.crtc_list,		\
-			    base.head)					\
-		for_each_if ((crtc_mask) & (1 << drm_crtc_index(&intel_crtc->base)))
-
-#define for_each_intel_encoder(dev, intel_encoder)		\
-	list_for_each_entry(intel_encoder,			\
-			    &(dev)->mode_config.encoder_list,	\
-			    base.head)
-
-#define for_each_intel_connector_iter(intel_connector, iter) \
-	while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter))))
-
-#define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
-	list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
-		for_each_if ((intel_encoder)->base.crtc == (__crtc))
-
-#define for_each_connector_on_encoder(dev, __encoder, intel_connector) \
-	list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \
-		for_each_if ((intel_connector)->base.encoder == (__encoder))
-
-#define for_each_power_domain(domain, mask)				\
-	for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)	\
-		for_each_if (BIT_ULL(domain) & (mask))
-
-#define for_each_power_well(__dev_priv, __power_well)				\
-	for ((__power_well) = (__dev_priv)->power_domains.power_wells;	\
-	     (__power_well) - (__dev_priv)->power_domains.power_wells <	\
-		(__dev_priv)->power_domains.power_well_count;		\
-	     (__power_well)++)
-
-#define for_each_power_well_rev(__dev_priv, __power_well)			\
-	for ((__power_well) = (__dev_priv)->power_domains.power_wells +		\
-			      (__dev_priv)->power_domains.power_well_count - 1;	\
-	     (__power_well) - (__dev_priv)->power_domains.power_wells >= 0;	\
-	     (__power_well)--)
-
-#define for_each_power_domain_well(__dev_priv, __power_well, __domain_mask)	\
-	for_each_power_well(__dev_priv, __power_well)				\
-		for_each_if ((__power_well)->domains & (__domain_mask))
-
-#define for_each_power_domain_well_rev(__dev_priv, __power_well, __domain_mask) \
-	for_each_power_well_rev(__dev_priv, __power_well)		        \
-		for_each_if ((__power_well)->domains & (__domain_mask))
-
-#define for_each_intel_plane_in_state(__state, plane, plane_state, __i) \
-	for ((__i) = 0; \
-	     (__i) < (__state)->base.dev->mode_config.num_total_plane && \
-		     ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \
-		      (plane_state) = to_intel_plane_state((__state)->base.planes[__i].state), 1); \
-	     (__i)++) \
-		for_each_if (plane_state)
-
-#define for_each_new_intel_crtc_in_state(__state, crtc, new_crtc_state, __i) \
-	for ((__i) = 0; \
-	     (__i) < (__state)->base.dev->mode_config.num_crtc && \
-		     ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \
-		      (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \
-	     (__i)++) \
-		for_each_if (crtc)
-
-
-#define for_each_oldnew_intel_plane_in_state(__state, plane, old_plane_state, new_plane_state, __i) \
-	for ((__i) = 0; \
-	     (__i) < (__state)->base.dev->mode_config.num_total_plane && \
-		     ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \
-		      (old_plane_state) = to_intel_plane_state((__state)->base.planes[__i].old_state), \
-		      (new_plane_state) = to_intel_plane_state((__state)->base.planes[__i].new_state), 1); \
-	     (__i)++) \
-		for_each_if (plane)
-
 struct drm_i915_private;
 struct i915_mm_struct;
 struct i915_mmu_object;
@@ -623,20 +343,6 @@ struct drm_i915_file_private {
 	atomic_t context_bans;
 };
 
-/* Used by dp and fdi links */
-struct intel_link_m_n {
-	uint32_t	tu;
-	uint32_t	gmch_m;
-	uint32_t	gmch_n;
-	uint32_t	link_m;
-	uint32_t	link_n;
-};
-
-void intel_link_compute_m_n(int bpp, int nlanes,
-			    int pixel_clock, int link_clock,
-			    struct intel_link_m_n *m_n,
-			    bool reduce_m_n);
-
 /* Interface history:
  *
  * 1.1: Original.
@@ -651,27 +357,6 @@ void intel_link_compute_m_n(int bpp, int nlanes,
 #define DRIVER_MINOR		6
 #define DRIVER_PATCHLEVEL	0
 
-struct opregion_header;
-struct opregion_acpi;
-struct opregion_swsci;
-struct opregion_asle;
-
-struct intel_opregion {
-	struct opregion_header *header;
-	struct opregion_acpi *acpi;
-	struct opregion_swsci *swsci;
-	u32 swsci_gbda_sub_functions;
-	u32 swsci_sbcb_sub_functions;
-	struct opregion_asle *asle;
-	void *rvda;
-	void *vbt_firmware;
-	const void *vbt;
-	u32 vbt_size;
-	u32 *lid_state;
-	struct work_struct asle_work;
-};
-#define OPREGION_SIZE            (8*1024)
-
 struct intel_overlay;
 struct intel_overlay_error_state;
 
@@ -699,7 +384,8 @@ struct drm_i915_display_funcs {
 			  struct intel_cdclk_state *cdclk_state);
 	void (*set_cdclk)(struct drm_i915_private *dev_priv,
 			  const struct intel_cdclk_state *cdclk_state);
-	int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane);
+	int (*get_fifo_size)(struct drm_i915_private *dev_priv,
+			     enum i9xx_plane_id i9xx_plane);
 	int (*compute_pipe_wm)(struct intel_crtc_state *cstate);
 	int (*compute_intermediate_wm)(struct drm_device *dev,
 				       struct intel_crtc *intel_crtc,
@@ -726,10 +412,12 @@ struct drm_i915_display_funcs {
 	void (*crtc_disable)(struct intel_crtc_state *old_crtc_state,
 			     struct drm_atomic_state *old_state);
 	void (*update_crtcs)(struct drm_atomic_state *state);
-	void (*audio_codec_enable)(struct drm_connector *connector,
-				   struct intel_encoder *encoder,
-				   const struct drm_display_mode *adjusted_mode);
-	void (*audio_codec_disable)(struct intel_encoder *encoder);
+	void (*audio_codec_enable)(struct intel_encoder *encoder,
+				   const struct intel_crtc_state *crtc_state,
+				   const struct drm_connector_state *conn_state);
+	void (*audio_codec_disable)(struct intel_encoder *encoder,
+				    const struct intel_crtc_state *old_crtc_state,
+				    const struct drm_connector_state *old_conn_state);
 	void (*fdi_link_train)(struct intel_crtc *crtc,
 			       const struct intel_crtc_state *crtc_state);
 	void (*init_clock_gating)(struct drm_i915_private *dev_priv);
@@ -761,135 +449,6 @@ struct intel_csr {
 	uint32_t allowed_dc_mask;
 };
 
-#define DEV_INFO_FOR_EACH_FLAG(func) \
-	func(is_mobile); \
-	func(is_lp); \
-	func(is_alpha_support); \
-	/* Keep has_* in alphabetical order */ \
-	func(has_64bit_reloc); \
-	func(has_aliasing_ppgtt); \
-	func(has_csr); \
-	func(has_ddi); \
-	func(has_dp_mst); \
-	func(has_reset_engine); \
-	func(has_fbc); \
-	func(has_fpga_dbg); \
-	func(has_full_ppgtt); \
-	func(has_full_48bit_ppgtt); \
-	func(has_gmch_display); \
-	func(has_guc); \
-	func(has_guc_ct); \
-	func(has_hotplug); \
-	func(has_l3_dpf); \
-	func(has_llc); \
-	func(has_logical_ring_contexts); \
-	func(has_logical_ring_preemption); \
-	func(has_overlay); \
-	func(has_pooled_eu); \
-	func(has_psr); \
-	func(has_rc6); \
-	func(has_rc6p); \
-	func(has_resource_streamer); \
-	func(has_runtime_pm); \
-	func(has_snoop); \
-	func(unfenced_needs_alignment); \
-	func(cursor_needs_physical); \
-	func(hws_needs_physical); \
-	func(overlay_needs_physical); \
-	func(supports_tv); \
-	func(has_ipc);
-
-struct sseu_dev_info {
-	u8 slice_mask;
-	u8 subslice_mask;
-	u8 eu_total;
-	u8 eu_per_subslice;
-	u8 min_eu_in_pool;
-	/* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */
-	u8 subslice_7eu[3];
-	u8 has_slice_pg:1;
-	u8 has_subslice_pg:1;
-	u8 has_eu_pg:1;
-};
-
-static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu)
-{
-	return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask);
-}
-
-/* Keep in gen based order, and chronological order within a gen */
-enum intel_platform {
-	INTEL_PLATFORM_UNINITIALIZED = 0,
-	INTEL_I830,
-	INTEL_I845G,
-	INTEL_I85X,
-	INTEL_I865G,
-	INTEL_I915G,
-	INTEL_I915GM,
-	INTEL_I945G,
-	INTEL_I945GM,
-	INTEL_G33,
-	INTEL_PINEVIEW,
-	INTEL_I965G,
-	INTEL_I965GM,
-	INTEL_G45,
-	INTEL_GM45,
-	INTEL_IRONLAKE,
-	INTEL_SANDYBRIDGE,
-	INTEL_IVYBRIDGE,
-	INTEL_VALLEYVIEW,
-	INTEL_HASWELL,
-	INTEL_BROADWELL,
-	INTEL_CHERRYVIEW,
-	INTEL_SKYLAKE,
-	INTEL_BROXTON,
-	INTEL_KABYLAKE,
-	INTEL_GEMINILAKE,
-	INTEL_COFFEELAKE,
-	INTEL_CANNONLAKE,
-	INTEL_MAX_PLATFORMS
-};
-
-struct intel_device_info {
-	u16 device_id;
-	u16 gen_mask;
-
-	u8 gen;
-	u8 gt; /* GT number, 0 if undefined */
-	u8 num_rings;
-	u8 ring_mask; /* Rings supported by the HW */
-
-	enum intel_platform platform;
-	u32 platform_mask;
-
-	u32 display_mmio_offset;
-
-	u8 num_pipes;
-	u8 num_sprites[I915_MAX_PIPES];
-	u8 num_scalers[I915_MAX_PIPES];
-
-	unsigned int page_sizes; /* page sizes supported by the HW */
-
-#define DEFINE_FLAG(name) u8 name:1
-	DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
-#undef DEFINE_FLAG
-	u16 ddb_size; /* in blocks */
-
-	/* Register offsets for the various display pipes and transcoders */
-	int pipe_offsets[I915_MAX_TRANSCODERS];
-	int trans_offsets[I915_MAX_TRANSCODERS];
-	int palette_offsets[I915_MAX_PIPES];
-	int cursor_offsets[I915_MAX_PIPES];
-
-	/* Slice/subslice/EU info */
-	struct sseu_dev_info sseu;
-
-	struct color_luts {
-		u16 degamma_lut_size;
-		u16 gamma_lut_size;
-	} color;
-};
-
 struct intel_display_error_state;
 
 struct i915_gpu_state {
@@ -911,6 +470,12 @@ struct i915_gpu_state {
 	struct intel_device_info device_info;
 	struct i915_params params;
 
+	struct i915_error_uc {
+		struct intel_uc_fw guc_fw;
+		struct intel_uc_fw huc_fw;
+		struct drm_i915_error_object *guc_log;
+	} uc;
+
 	/* Generic register state */
 	u32 eir;
 	u32 pgtbl_er;
@@ -933,12 +498,11 @@ struct i915_gpu_state {
 	u64 fence[I915_MAX_NUM_FENCES];
 	struct intel_overlay_error_state *overlay;
 	struct intel_display_error_state *display;
-	struct drm_i915_error_object *semaphore;
-	struct drm_i915_error_object *guc_log;
 
 	struct drm_i915_error_engine {
 		int engine_id;
 		/* Software tracked state */
+		bool idle;
 		bool waiting;
 		int num_waiters;
 		unsigned long hangcheck_timestamp;
@@ -1001,6 +565,7 @@ struct i915_gpu_state {
 		long user_bo_count;
 
 		struct drm_i915_error_object *wa_ctx;
+		struct drm_i915_error_object *default_state;
 
 		struct drm_i915_error_request {
 			long jiffies;
@@ -1137,7 +702,7 @@ struct intel_fbc {
 
 		struct {
 			enum pipe pipe;
-			enum plane plane;
+			enum i9xx_plane_id i9xx_plane;
 			unsigned int fence_y_offset;
 		} crtc;
 
@@ -1386,7 +951,6 @@ struct intel_gen6_power_mgmt {
 	struct intel_rps rps;
 	struct intel_rc6 rc6;
 	struct intel_llc_pstate llc_pstate;
-	struct delayed_work autoenable_work;
 };
 
 /* defined intel_pm.c */
@@ -1529,9 +1093,6 @@ struct i915_gem_mm {
 	 */
 	struct pagevec wc_stash;
 
-	/** Usable portion of the GTT for GEM */
-	dma_addr_t stolen_base; /* limited to low memory (32-bit) */
-
 	/**
 	 * tmpfs instance used for shmem backed objects
 	 */
@@ -1580,6 +1141,8 @@ struct drm_i915_error_state_buf {
 	loff_t pos;
 };
 
+#define I915_IDLE_ENGINES_TIMEOUT (200) /* in ms */
+
 #define I915_RESET_TIMEOUT (10 * HZ) /* 10s */
 #define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */
 
@@ -1698,6 +1261,8 @@ enum modeset_restore {
 #define DDC_PIN_D  0x06
 
 struct ddi_vbt_port_info {
+	int max_tmds_clock;
+
 	/*
 	 * This is an index in the HDMI/DVI DDI buffer translation table.
 	 * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't
@@ -1784,6 +1349,7 @@ struct intel_vbt_data {
 		u32 size;
 		u8 *data;
 		const u8 *sequence[MIPI_SEQ_MAX];
+		u8 *deassert_seq; /* Used by fixup_mipi_sequences() */
 	} dsi;
 
 	int crt_ddc_pin;
@@ -2228,6 +1794,7 @@ struct i915_oa_ops {
 
 struct intel_cdclk_state {
 	unsigned int cdclk, vco, ref;
+	u8 voltage_level;
 };
 
 struct drm_i915_private {
@@ -2242,6 +1809,30 @@ struct drm_i915_private {
 
 	const struct intel_device_info info;
 
+	/**
+	 * Data Stolen Memory - aka "i915 stolen memory" gives us the start and
+	 * end of stolen which we can optionally use to create GEM objects
+	 * backed by stolen memory. Note that stolen_usable_size tells us
+	 * exactly how much of this we are actually allowed to use, given that
+	 * some portion of it is in fact reserved for use by hardware functions.
+	 */
+	struct resource dsm;
+	/**
+	 * Reseved portion of Data Stolen Memory
+	 */
+	struct resource dsm_reserved;
+
+	/*
+	 * Stolen memory is segmented in hardware with different portions
+	 * offlimits to certain functions.
+	 *
+	 * The drm_mm is initialised to the total accessible range, as found
+	 * from the PCI config. On Broadwell+, this is further restricted to
+	 * avoid the first page! The upper end of stolen memory is reserved for
+	 * hardware functions and similarly removed from the accessible range.
+	 */
+	resource_size_t stolen_usable_size;	/* Total size minus reserved ranges */
+
 	void __iomem *regs;
 
 	struct intel_uncore uncore;
@@ -2281,7 +1872,8 @@ struct drm_i915_private {
 	struct i915_gem_context *kernel_context;
 	/* Context only to be used for injecting preemption commands */
 	struct i915_gem_context *preempt_context;
-	struct i915_vma *semaphore;
+	struct intel_engine_cs *engine_class[MAX_ENGINE_CLASS + 1]
+					    [MAX_ENGINE_INSTANCE + 1];
 
 	struct drm_dma_handle *status_page_dmah;
 	struct resource mch_res;
@@ -2339,6 +1931,7 @@ struct drm_i915_private {
 	unsigned int max_dotclk_freq;
 	unsigned int rawclk_freq;
 	unsigned int hpll_freq;
+	unsigned int fdi_pll_freq;
 	unsigned int czclk_freq;
 
 	struct {
@@ -2418,6 +2011,8 @@ struct drm_i915_private {
 	unsigned int active_crtcs;
 	/* minimum acceptable cdclk for each pipe */
 	int min_cdclk[I915_MAX_PIPES];
+	/* minimum acceptable voltage level for each pipe */
+	u8 min_voltage_level[I915_MAX_PIPES];
 
 	int dpio_phy_iosf_port[I915_NUM_PHYS_VLV];
 
@@ -2609,7 +2204,6 @@ struct drm_i915_private {
 
 			bool periodic;
 			int period_exponent;
-			int timestamp_frequency;
 
 			struct i915_oa_config test_config;
 
@@ -2754,6 +2348,8 @@ struct drm_i915_private {
 		int	irq;
 	} lpe_audio;
 
+	struct i915_pmu pmu;
+
 	/*
 	 * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
 	 * will be rejected. Instead look for a better place.
@@ -3049,6 +2645,8 @@ intel_info(const struct drm_i915_private *dev_priv)
 				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0)
 #define IS_CFL_GT2(dev_priv)	(IS_COFFEELAKE(dev_priv) && \
 				 (dev_priv)->info.gt == 2)
+#define IS_CFL_GT3(dev_priv)	(IS_COFFEELAKE(dev_priv) && \
+				 (dev_priv)->info.gt == 3)
 
 #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
 
@@ -3130,6 +2728,8 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define HAS_BLT(dev_priv)	HAS_ENGINE(dev_priv, BCS)
 #define HAS_VEBOX(dev_priv)	HAS_ENGINE(dev_priv, VECS)
 
+#define HAS_LEGACY_SEMAPHORES(dev_priv) IS_GEN7(dev_priv)
+
 #define HAS_LLC(dev_priv)	((dev_priv)->info.has_llc)
 #define HAS_SNOOP(dev_priv)	((dev_priv)->info.has_snoop)
 #define HAS_EDRAM(dev_priv)	(!!((dev_priv)->edram_cap & EDRAM_ENABLED))
@@ -3140,6 +2740,11 @@ intel_info(const struct drm_i915_private *dev_priv)
 
 #define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
 		((dev_priv)->info.has_logical_ring_contexts)
+#define HAS_LOGICAL_RING_PREEMPTION(dev_priv) \
+		((dev_priv)->info.has_logical_ring_preemption)
+
+#define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv)
+
 #define USES_PPGTT(dev_priv)		(i915_modparams.enable_ppgtt)
 #define USES_FULL_PPGTT(dev_priv)	(i915_modparams.enable_ppgtt >= 2)
 #define USES_FULL_48BIT_PPGTT(dev_priv)	(i915_modparams.enable_ppgtt == 3)
@@ -3191,8 +2796,10 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define HAS_DDI(dev_priv)		 ((dev_priv)->info.has_ddi)
 #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg)
 #define HAS_PSR(dev_priv)		 ((dev_priv)->info.has_psr)
+
 #define HAS_RC6(dev_priv)		 ((dev_priv)->info.has_rc6)
 #define HAS_RC6p(dev_priv)		 ((dev_priv)->info.has_rc6p)
+#define HAS_RC6pp(dev_priv)		 (false) /* HW was never validated */
 
 #define HAS_CSR(dev_priv)	((dev_priv)->info.has_csr)
 
@@ -3210,8 +2817,16 @@ intel_info(const struct drm_i915_private *dev_priv)
 #define HAS_GUC_CT(dev_priv)	((dev_priv)->info.has_guc_ct)
 #define HAS_GUC_UCODE(dev_priv)	(HAS_GUC(dev_priv))
 #define HAS_GUC_SCHED(dev_priv)	(HAS_GUC(dev_priv))
+
+/* For now, anything with a GuC has also HuC */
+#define HAS_HUC(dev_priv)	(HAS_GUC(dev_priv))
 #define HAS_HUC_UCODE(dev_priv)	(HAS_GUC(dev_priv))
 
+/* Having a GuC is not the same as using a GuC */
+#define USES_GUC(dev_priv)		intel_uc_is_using_guc()
+#define USES_GUC_SUBMISSION(dev_priv)	intel_uc_is_using_guc_submission()
+#define USES_HUC(dev_priv)		intel_uc_is_using_huc()
+
 #define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer)
 
 #define HAS_POOLED_EU(dev_priv)	((dev_priv)->info.has_pooled_eu)
@@ -3288,8 +2903,6 @@ intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv)
 int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 				int enable_ppgtt);
 
-bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value);
-
 /* i915_drv.c */
 void __printf(3, 4)
 __i915_printk(struct drm_i915_private *dev_priv, const char *level,
@@ -3318,7 +2931,9 @@ extern int i915_reset_engine(struct intel_engine_cs *engine,
 			     unsigned int flags);
 
 extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv);
-extern int intel_guc_reset(struct drm_i915_private *dev_priv);
+extern int intel_reset_guc(struct drm_i915_private *dev_priv);
+extern int intel_guc_reset_engine(struct intel_guc *guc,
+				  struct intel_engine_cs *engine);
 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine);
 extern void intel_hangcheck_init(struct drm_i915_private *dev_priv);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
@@ -3855,6 +3470,8 @@ int __must_check i915_gem_evict_for_node(struct i915_address_space *vm,
 					 unsigned int flags);
 int i915_gem_evict_vm(struct i915_address_space *vm);
 
+void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv);
+
 /* belongs in i915_gem_gtt.h */
 static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv)
 {
@@ -3876,12 +3493,13 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
 int i915_gem_init_stolen(struct drm_i915_private *dev_priv);
 void i915_gem_cleanup_stolen(struct drm_device *dev);
 struct drm_i915_gem_object *
-i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, u32 size);
+i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
+			      resource_size_t size);
 struct drm_i915_gem_object *
 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv,
-					       u32 stolen_offset,
-					       u32 gtt_offset,
-					       u32 size);
+					       resource_size_t stolen_offset,
+					       resource_size_t gtt_offset,
+					       resource_size_t size);
 
 /* i915_gem_internal.c */
 struct drm_i915_gem_object *
@@ -3889,7 +3507,7 @@ i915_gem_object_create_internal(struct drm_i915_private *dev_priv,
 				phys_addr_t size);
 
 /* i915_gem_shrinker.c */
-unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
+unsigned long i915_gem_shrink(struct drm_i915_private *i915,
 			      unsigned long target,
 			      unsigned long *nr_scanned,
 			      unsigned flags);
@@ -3898,9 +3516,9 @@ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
 #define I915_SHRINK_BOUND 0x4
 #define I915_SHRINK_ACTIVE 0x8
 #define I915_SHRINK_VMAPS 0x10
-unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
-void i915_gem_shrinker_init(struct drm_i915_private *dev_priv);
-void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv);
+unsigned long i915_gem_shrink_all(struct drm_i915_private *i915);
+void i915_gem_shrinker_register(struct drm_i915_private *i915);
+void i915_gem_shrinker_unregister(struct drm_i915_private *i915);
 
 
 /* i915_gem_tiling.c */
@@ -4040,6 +3658,7 @@ extern void intel_i2c_reset(struct drm_i915_private *dev_priv);
 
 /* intel_bios.c */
 void intel_bios_init(struct drm_i915_private *dev_priv);
+void intel_bios_cleanup(struct drm_i915_private *dev_priv);
 bool intel_bios_is_valid_vbt(const void *buf, size_t size);
 bool intel_bios_is_tv_present(struct drm_i915_private *dev_priv);
 bool intel_bios_is_lvds_present(struct drm_i915_private *dev_priv, u8 *i2c_pin);
@@ -4052,41 +3671,6 @@ bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv,
 bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv,
 				enum port port);
 
-
-/* intel_opregion.c */
-#ifdef CONFIG_ACPI
-extern int intel_opregion_setup(struct drm_i915_private *dev_priv);
-extern void intel_opregion_register(struct drm_i915_private *dev_priv);
-extern void intel_opregion_unregister(struct drm_i915_private *dev_priv);
-extern void intel_opregion_asle_intr(struct drm_i915_private *dev_priv);
-extern int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
-					 bool enable);
-extern int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv,
-					 pci_power_t state);
-extern int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv);
-#else
-static inline int intel_opregion_setup(struct drm_i915_private *dev) { return 0; }
-static inline void intel_opregion_register(struct drm_i915_private *dev_priv) { }
-static inline void intel_opregion_unregister(struct drm_i915_private *dev_priv) { }
-static inline void intel_opregion_asle_intr(struct drm_i915_private *dev_priv)
-{
-}
-static inline int
-intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable)
-{
-	return 0;
-}
-static inline int
-intel_opregion_notify_adapter(struct drm_i915_private *dev, pci_power_t state)
-{
-	return 0;
-}
-static inline int intel_opregion_get_panel_type(struct drm_i915_private *dev)
-{
-	return -ENODEV;
-}
-#endif
-
 /* intel_acpi.c */
 #ifdef CONFIG_ACPI
 extern void intel_register_dsm_handler(void);
@@ -4103,14 +3687,9 @@ mkwrite_device_info(struct drm_i915_private *dev_priv)
 	return (struct intel_device_info *)&dev_priv->info;
 }
 
-const char *intel_platform_name(enum intel_platform platform);
-void intel_device_info_runtime_init(struct drm_i915_private *dev_priv);
-void intel_device_info_dump(struct drm_i915_private *dev_priv);
-
 /* modesetting */
 extern void intel_modeset_init_hw(struct drm_device *dev);
 extern int intel_modeset_init(struct drm_device *dev);
-extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_connector_register(struct drm_connector *);
 extern void intel_connector_unregister(struct drm_connector *);
@@ -4140,7 +3719,11 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
 					    struct intel_display_error_state *error);
 
 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val);
-int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val);
+int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv, u32 mbox,
+				    u32 val, int timeout_us);
+#define sandybridge_pcode_write(dev_priv, mbox, val)	\
+	sandybridge_pcode_write_timeout(dev_priv, mbox, val, 500)
+
 int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
 		      u32 reply_mask, u32 reply, int timeout_base_ms);
 
@@ -4177,8 +3760,7 @@ bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
 			    enum dpio_phy phy);
 bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
 			      enum dpio_phy phy);
-uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
-					     uint8_t lane_count);
+uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(uint8_t lane_count);
 void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
 				     uint8_t lane_lat_optim_mask);
 uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder);
@@ -4187,24 +3769,39 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 			      u32 deemph_reg_value, u32 margin_reg_value,
 			      bool uniq_trans_scale);
 void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+			      const struct intel_crtc_state *crtc_state,
 			      bool reset);
-void chv_phy_pre_pll_enable(struct intel_encoder *encoder);
-void chv_phy_pre_encoder_enable(struct intel_encoder *encoder);
+void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
+			    const struct intel_crtc_state *crtc_state);
+void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+				const struct intel_crtc_state *crtc_state);
 void chv_phy_release_cl2_override(struct intel_encoder *encoder);
-void chv_phy_post_pll_disable(struct intel_encoder *encoder);
+void chv_phy_post_pll_disable(struct intel_encoder *encoder,
+			      const struct intel_crtc_state *old_crtc_state);
 
 void vlv_set_phy_signal_level(struct intel_encoder *encoder,
 			      u32 demph_reg_value, u32 preemph_reg_value,
 			      u32 uniqtranscale_reg_value, u32 tx3_demph);
-void vlv_phy_pre_pll_enable(struct intel_encoder *encoder);
-void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder);
-void vlv_phy_reset_lanes(struct intel_encoder *encoder);
+void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
+			    const struct intel_crtc_state *crtc_state);
+void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+				const struct intel_crtc_state *crtc_state);
+void vlv_phy_reset_lanes(struct intel_encoder *encoder,
+			 const struct intel_crtc_state *old_crtc_state);
 
 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val);
 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
-u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
+u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
 			   const i915_reg_t reg);
 
+u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat1);
+
+static inline u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
+					 const i915_reg_t reg)
+{
+	return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(dev_priv, reg), 1000);
+}
+
 #define I915_READ8(reg)		dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true)
 #define I915_WRITE8(reg, val)	dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true)
 
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 5cfba89ed586..6ff5d655c202 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -434,20 +434,28 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
 			dma_fence_put(shared[i]);
 		kfree(shared);
 
+		/*
+		 * If both shared fences and an exclusive fence exist,
+		 * then by construction the shared fences must be later
+		 * than the exclusive fence. If we successfully wait for
+		 * all the shared fences, we know that the exclusive fence
+		 * must all be signaled. If all the shared fences are
+		 * signaled, we can prune the array and recover the
+		 * floating references on the fences/requests.
+		 */
 		prune_fences = count && timeout >= 0;
 	} else {
 		excl = reservation_object_get_excl_rcu(resv);
 	}
 
-	if (excl && timeout >= 0) {
+	if (excl && timeout >= 0)
 		timeout = i915_gem_object_wait_fence(excl, flags, timeout,
 						     rps_client);
-		prune_fences = timeout >= 0;
-	}
 
 	dma_fence_put(excl);
 
-	/* Oportunistically prune the fences iff we know they have *all* been
+	/*
+	 * Opportunistically prune the fences iff we know they have *all* been
 	 * signaled and that the reservation object has not been changed (i.e.
 	 * no new fences have been added).
 	 */
@@ -531,7 +539,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
  * @obj: i915 gem object
  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
  * @timeout: how long to wait
- * @rps: client (user process) to charge for any waitboosting
+ * @rps_client: client (user process) to charge for any waitboosting
  */
 int
 i915_gem_object_wait(struct drm_i915_gem_object *obj,
@@ -666,17 +674,13 @@ fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
 		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
 }
 
-static void
-flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
+void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv)
 {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-
-	if (!(obj->base.write_domain & flush_domains))
-		return;
-
-	/* No actual flushing is required for the GTT write domain.  Writes
-	 * to it "immediately" go to main memory as far as we know, so there's
-	 * no chipset flush.  It also doesn't land in render cache.
+	/*
+	 * No actual flushing is required for the GTT write domain for reads
+	 * from the GTT domain. Writes to it "immediately" go to main memory
+	 * as far as we know, so there's no chipset flush. It also doesn't
+	 * land in the GPU render cache.
 	 *
 	 * However, we do have to enforce the order so that all writes through
 	 * the GTT land before any writes to the device, such as updates to
@@ -687,22 +691,43 @@ flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
 	 * timing. This issue has only been observed when switching quickly
 	 * between GTT writes and CPU reads from inside the kernel on recent hw,
 	 * and it appears to only affect discrete GTT blocks (i.e. on LLC
-	 * system agents we cannot reproduce this behaviour).
+	 * system agents we cannot reproduce this behaviour, until Cannonlake
+	 * that was!).
 	 */
+
 	wmb();
 
+	intel_runtime_pm_get(dev_priv);
+	spin_lock_irq(&dev_priv->uncore.lock);
+
+	POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE));
+
+	spin_unlock_irq(&dev_priv->uncore.lock);
+	intel_runtime_pm_put(dev_priv);
+}
+
+static void
+flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
+{
+	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
+	struct i915_vma *vma;
+
+	if (!(obj->base.write_domain & flush_domains))
+		return;
+
 	switch (obj->base.write_domain) {
 	case I915_GEM_DOMAIN_GTT:
-		if (!HAS_LLC(dev_priv)) {
-			intel_runtime_pm_get(dev_priv);
-			spin_lock_irq(&dev_priv->uncore.lock);
-			POSTING_READ_FW(RING_HEAD(dev_priv->engine[RCS]->mmio_base));
-			spin_unlock_irq(&dev_priv->uncore.lock);
-			intel_runtime_pm_put(dev_priv);
-		}
+		i915_gem_flush_ggtt_writes(dev_priv);
 
 		intel_fb_obj_flush(obj,
 				   fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
+
+		for_each_ggtt_vma(vma, obj) {
+			if (vma->iomap)
+				continue;
+
+			i915_vma_unset_ggtt_write(vma);
+		}
 		break;
 
 	case I915_GEM_DOMAIN_CPU:
@@ -1099,7 +1124,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 			page_base += offset & PAGE_MASK;
 		}
 
-		if (gtt_user_read(&ggtt->mappable, page_base, page_offset,
+		if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 				  user_data, page_length)) {
 			ret = -EFAULT;
 			break;
@@ -1307,7 +1332,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 		 * If the object is non-shmem backed, we retry again with the
 		 * path that handles page fault.
 		 */
-		if (ggtt_write(&ggtt->mappable, page_base, page_offset,
+		if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 			       user_data, page_length)) {
 			ret = -EFAULT;
 			break;
@@ -1549,10 +1574,7 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
 
 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (!i915_vma_is_ggtt(vma))
-			break;
-
+	for_each_ggtt_vma(vma, obj) {
 		if (i915_vma_is_active(vma))
 			continue;
 
@@ -1612,7 +1634,19 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	if (err)
 		goto out;
 
-	/* Flush and acquire obj->pages so that we are coherent through
+	/*
+	 * Proxy objects do not control access to the backing storage, ergo
+	 * they cannot be used as a means to manipulate the cache domain
+	 * tracking for that backing storage. The proxy object is always
+	 * considered to be outside of any cache domain.
+	 */
+	if (i915_gem_object_is_proxy(obj)) {
+		err = -ENXIO;
+		goto out;
+	}
+
+	/*
+	 * Flush and acquire obj->pages so that we are coherent through
 	 * direct access in memory with previous cached writes through
 	 * shmemfs and that our cache domain tracking remains valid.
 	 * For example, if the obj->filp was moved to swap without us
@@ -1668,6 +1702,11 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 	if (!obj)
 		return -ENOENT;
 
+	/*
+	 * Proxy objects are barred from CPU access, so there is no
+	 * need to ban sw_finish as it is a nop.
+	 */
+
 	/* Pinned buffers may be scanout, so flush the cache */
 	i915_gem_object_flush_if_display(obj);
 	i915_gem_object_put(obj);
@@ -1718,7 +1757,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 	 */
 	if (!obj->base.filp) {
 		i915_gem_object_put(obj);
-		return -EINVAL;
+		return -ENXIO;
 	}
 
 	addr = vm_mmap(obj->base.filp, 0, args->size,
@@ -1936,9 +1975,9 @@ int i915_gem_fault(struct vm_fault *vmf)
 	/* Finally, remap it using the new GTT offset */
 	ret = remap_io_mapping(area,
 			       area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
-			       (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT,
+			       (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
 			       min_t(u64, vma->size, area->vm_end - area->vm_start),
-			       &ggtt->mappable);
+			       &ggtt->iomap);
 	if (ret)
 		goto err_fence;
 
@@ -1948,6 +1987,8 @@ int i915_gem_fault(struct vm_fault *vmf)
 		list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
 	GEM_BUG_ON(!obj->userfault_count);
 
+	i915_vma_set_ggtt_write(vma);
+
 err_fence:
 	i915_vma_unpin_fence(vma);
 err_unpin:
@@ -2012,12 +2053,8 @@ static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
 	drm_vma_node_unmap(&obj->base.vma_node,
 			   obj->base.dev->anon_inode->i_mapping);
 
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (!i915_vma_is_ggtt(vma))
-			break;
-
+	for_each_ggtt_vma(vma, obj)
 		i915_vma_unset_userfault(vma);
-	}
 }
 
 /**
@@ -2567,7 +2604,7 @@ static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 	}
 
 	err = obj->ops->get_pages(obj);
-	GEM_BUG_ON(!err && IS_ERR_OR_NULL(obj->mm.pages));
+	GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
 
 	return err;
 }
@@ -2662,7 +2699,8 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 	void *ptr;
 	int ret;
 
-	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
+	if (unlikely(!i915_gem_object_has_struct_page(obj)))
+		return ERR_PTR(-ENXIO);
 
 	ret = mutex_lock_interruptible(&obj->mm.lock);
 	if (ret)
@@ -2908,13 +2946,23 @@ i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
 	 * Prevent request submission to the hardware until we have
 	 * completed the reset in i915_gem_reset_finish(). If a request
 	 * is completed by one engine, it may then queue a request
-	 * to a second via its engine->irq_tasklet *just* as we are
+	 * to a second via its execlists->tasklet *just* as we are
 	 * calling engine->init_hw() and also writing the ELSP.
-	 * Turning off the engine->irq_tasklet until the reset is over
+	 * Turning off the execlists->tasklet until the reset is over
 	 * prevents the race.
 	 */
-	tasklet_kill(&engine->execlists.irq_tasklet);
-	tasklet_disable(&engine->execlists.irq_tasklet);
+	tasklet_kill(&engine->execlists.tasklet);
+	tasklet_disable(&engine->execlists.tasklet);
+
+	/*
+	 * We're using worker to queue preemption requests from the tasklet in
+	 * GuC submission mode.
+	 * Even though tasklet was disabled, we may still have a worker queued.
+	 * Let's make sure that all workers scheduled before disabling the
+	 * tasklet are completed before continuing with the reset.
+	 */
+	if (engine->i915->guc.preempt_wq)
+		flush_workqueue(engine->i915->guc.preempt_wq);
 
 	if (engine->irq_seqno_barrier)
 		engine->irq_seqno_barrier(engine);
@@ -3049,7 +3097,12 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
 void i915_gem_reset_engine(struct intel_engine_cs *engine,
 			   struct drm_i915_gem_request *request)
 {
-	engine->irq_posted = 0;
+	/*
+	 * Make sure this write is visible before we re-enable the interrupt
+	 * handlers on another CPU, as tasklet_enable() resolves to just
+	 * a compiler barrier which is insufficient for our purpose here.
+	 */
+	smp_store_mb(engine->irq_posted, 0);
 
 	if (request)
 		request = i915_gem_reset_request(engine, request);
@@ -3079,6 +3132,25 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 		ctx = fetch_and_zero(&engine->last_retired_context);
 		if (ctx)
 			engine->context_unpin(engine, ctx);
+
+		/*
+		 * Ostensibily, we always want a context loaded for powersaving,
+		 * so if the engine is idle after the reset, send a request
+		 * to load our scratch kernel_context.
+		 *
+		 * More mysteriously, if we leave the engine idle after a reset,
+		 * the next userspace batch may hang, with what appears to be
+		 * an incoherent read by the CS (presumably stale TLB). An
+		 * empty request appears sufficient to paper over the glitch.
+		 */
+		if (list_empty(&engine->timeline->requests)) {
+			struct drm_i915_gem_request *rq;
+
+			rq = i915_gem_request_alloc(engine,
+						    dev_priv->kernel_context);
+			if (!IS_ERR(rq))
+				__i915_add_request(rq, false);
+		}
 	}
 
 	i915_gem_restore_fences(dev_priv);
@@ -3093,7 +3165,7 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 
 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
 {
-	tasklet_enable(&engine->execlists.irq_tasklet);
+	tasklet_enable(&engine->execlists.tasklet);
 	kthread_unpark(engine->breadcrumbs.signaler);
 
 	intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
@@ -3141,8 +3213,10 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
 	 * rolling the global seqno forward (since this would complete requests
 	 * for which we haven't set the fence error to EIO yet).
 	 */
-	for_each_engine(engine, i915, id)
+	for_each_engine(engine, i915, id) {
+		i915_gem_reset_prepare_engine(engine);
 		engine->submit_request = nop_submit_request;
+	}
 
 	/*
 	 * Make sure no one is running the old callback before we proceed with
@@ -3180,6 +3254,8 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
 		intel_engine_init_global_seqno(engine,
 					       intel_engine_last_submit(engine));
 		spin_unlock_irqrestore(&engine->timeline->lock, flags);
+
+		i915_gem_reset_finish_engine(engine);
 	}
 
 	set_bit(I915_WEDGED, &i915->gpu_error.flags);
@@ -3259,16 +3335,22 @@ i915_gem_retire_work_handler(struct work_struct *work)
 		mutex_unlock(&dev->struct_mutex);
 	}
 
-	/* Keep the retire handler running until we are finally idle.
+	/*
+	 * Keep the retire handler running until we are finally idle.
 	 * We do not need to do this test under locking as in the worst-case
 	 * we queue the retire worker once too often.
 	 */
-	if (READ_ONCE(dev_priv->gt.awake)) {
-		i915_queue_hangcheck(dev_priv);
+	if (READ_ONCE(dev_priv->gt.awake))
 		queue_delayed_work(dev_priv->wq,
 				   &dev_priv->gt.retire_work,
 				   round_jiffies_up_relative(HZ));
-	}
+}
+
+static inline bool
+new_requests_since_last_retire(const struct drm_i915_private *i915)
+{
+	return (READ_ONCE(i915->gt.active_requests) ||
+		work_pending(&i915->gt.idle_work.work));
 }
 
 static void
@@ -3276,8 +3358,8 @@ i915_gem_idle_work_handler(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(work, typeof(*dev_priv), gt.idle_work.work);
-	struct drm_device *dev = &dev_priv->drm;
 	bool rearm_hangcheck;
+	ktime_t end;
 
 	if (!READ_ONCE(dev_priv->gt.awake))
 		return;
@@ -3286,14 +3368,21 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	 * Wait for last execlists context complete, but bail out in case a
 	 * new request is submitted.
 	 */
-	wait_for(intel_engines_are_idle(dev_priv), 10);
-	if (READ_ONCE(dev_priv->gt.active_requests))
-		return;
+	end = ktime_add_ms(ktime_get(), I915_IDLE_ENGINES_TIMEOUT);
+	do {
+		if (new_requests_since_last_retire(dev_priv))
+			return;
+
+		if (intel_engines_are_idle(dev_priv))
+			break;
+
+		usleep_range(100, 500);
+	} while (ktime_before(ktime_get(), end));
 
 	rearm_hangcheck =
 		cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
 
-	if (!mutex_trylock(&dev->struct_mutex)) {
+	if (!mutex_trylock(&dev_priv->drm.struct_mutex)) {
 		/* Currently busy, come back later */
 		mod_delayed_work(dev_priv->wq,
 				 &dev_priv->gt.idle_work,
@@ -3305,17 +3394,26 @@ i915_gem_idle_work_handler(struct work_struct *work)
 	 * New request retired after this work handler started, extend active
 	 * period until next instance of the work.
 	 */
-	if (work_pending(work))
+	if (new_requests_since_last_retire(dev_priv))
 		goto out_unlock;
 
-	if (dev_priv->gt.active_requests)
-		goto out_unlock;
+	/*
+	 * Be paranoid and flush a concurrent interrupt to make sure
+	 * we don't reactivate any irq tasklets after parking.
+	 *
+	 * FIXME: Note that even though we have waited for execlists to be idle,
+	 * there may still be an in-flight interrupt even though the CSB
+	 * is now empty. synchronize_irq() makes sure that a residual interrupt
+	 * is completed before we continue, but it doesn't prevent the HW from
+	 * raising a spurious interrupt later. To complete the shield we should
+	 * coordinate disabling the CS irq with flushing the interrupts.
+	 */
+	synchronize_irq(dev_priv->drm.irq);
 
-	if (wait_for(intel_engines_are_idle(dev_priv), 10))
-		DRM_ERROR("Timeout waiting for engines to idle\n");
+	intel_engines_park(dev_priv);
+	i915_gem_timelines_park(dev_priv);
 
-	intel_engines_mark_idle(dev_priv);
-	i915_gem_timelines_mark_idle(dev_priv);
+	i915_pmu_gt_parked(dev_priv);
 
 	GEM_BUG_ON(!dev_priv->gt.awake);
 	dev_priv->gt.awake = false;
@@ -3323,9 +3421,12 @@ i915_gem_idle_work_handler(struct work_struct *work)
 
 	if (INTEL_GEN(dev_priv) >= 6)
 		gen6_rps_idle(dev_priv);
+
+	intel_display_power_put(dev_priv, POWER_DOMAIN_GT_IRQ);
+
 	intel_runtime_pm_put(dev_priv);
 out_unlock:
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 out_rearm:
 	if (rearm_hangcheck) {
@@ -3467,8 +3568,19 @@ static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
 
 static int wait_for_engines(struct drm_i915_private *i915)
 {
-	if (wait_for(intel_engines_are_idle(i915), 50)) {
-		DRM_ERROR("Failed to idle engines, declaring wedged!\n");
+	if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) {
+		dev_err(i915->drm.dev,
+			"Failed to idle engines, declaring wedged!\n");
+		if (drm_debug & DRM_UT_DRIVER) {
+			struct drm_printer p = drm_debug_printer(__func__);
+			struct intel_engine_cs *engine;
+			enum intel_engine_id id;
+
+			for_each_engine(engine, i915, id)
+				intel_engine_dump(engine, &p,
+						  "%s", engine->name);
+		}
+
 		i915_gem_set_wedged(i915);
 		return -EIO;
 	}
@@ -3494,9 +3606,7 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
 			if (ret)
 				return ret;
 		}
-
 		i915_gem_retire_requests(i915);
-		GEM_BUG_ON(i915->gt.active_requests);
 
 		ret = wait_for_engines(i915);
 	} else {
@@ -3695,7 +3805,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 			return -EBUSY;
 		}
 
-		if (i915_gem_valid_gtt_space(vma, cache_level))
+		if (!i915_vma_is_closed(vma) &&
+		    i915_gem_valid_gtt_space(vma, cache_level))
 			continue;
 
 		ret = i915_vma_unbind(vma);
@@ -3748,7 +3859,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 			 * dropped the fence as all snoopable access is
 			 * supposed to be linear.
 			 */
-			list_for_each_entry(vma, &obj->vma_list, obj_link) {
+			for_each_ggtt_vma(vma, obj) {
 				ret = i915_vma_put_fence(vma);
 				if (ret)
 					return ret;
@@ -3850,6 +3961,15 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 	if (!obj)
 		return -ENOENT;
 
+	/*
+	 * The caching mode of proxy object is handled by its generator, and
+	 * not allowed to be changed by userspace.
+	 */
+	if (i915_gem_object_is_proxy(obj)) {
+		ret = -ENXIO;
+		goto out;
+	}
+
 	if (obj->cache_level == level)
 		goto out;
 
@@ -4655,14 +4775,16 @@ void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
 		i915_gem_object_put(obj);
 }
 
-static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv)
+static void assert_kernel_context_is_current(struct drm_i915_private *i915)
 {
+	struct i915_gem_context *kernel_context = i915->kernel_context;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
-	for_each_engine(engine, dev_priv, id)
-		GEM_BUG_ON(engine->last_retired_context &&
-			   !i915_gem_context_is_kernel(engine->last_retired_context));
+	for_each_engine(engine, i915, id) {
+		GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline->last_request));
+		GEM_BUG_ON(engine->last_retired_context != kernel_context);
+	}
 }
 
 void i915_gem_sanitize(struct drm_i915_private *i915)
@@ -4768,23 +4890,43 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
 	return ret;
 }
 
-void i915_gem_resume(struct drm_i915_private *dev_priv)
+void i915_gem_resume(struct drm_i915_private *i915)
 {
-	struct drm_device *dev = &dev_priv->drm;
+	WARN_ON(i915->gt.awake);
 
-	WARN_ON(dev_priv->gt.awake);
+	mutex_lock(&i915->drm.struct_mutex);
+	intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 
-	mutex_lock(&dev->struct_mutex);
-	i915_gem_restore_gtt_mappings(dev_priv);
-	i915_gem_restore_fences(dev_priv);
+	i915_gem_restore_gtt_mappings(i915);
+	i915_gem_restore_fences(i915);
 
-	/* As we didn't flush the kernel context before suspend, we cannot
+	/*
+	 * As we didn't flush the kernel context before suspend, we cannot
 	 * guarantee that the context image is complete. So let's just reset
 	 * it and start again.
 	 */
-	dev_priv->gt.resume(dev_priv);
+	i915->gt.resume(i915);
 
-	mutex_unlock(&dev->struct_mutex);
+	if (i915_gem_init_hw(i915))
+		goto err_wedged;
+
+	intel_guc_resume(i915);
+
+	/* Always reload a context for powersaving. */
+	if (i915_gem_switch_to_kernel_context(i915))
+		goto err_wedged;
+
+out_unlock:
+	intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
+	mutex_unlock(&i915->drm.struct_mutex);
+	return;
+
+err_wedged:
+	if (!i915_terminally_wedged(&i915->gpu_error)) {
+		DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
+		i915_gem_set_wedged(i915);
+	}
+	goto out_unlock;
 }
 
 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
@@ -4901,40 +5043,132 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
 		goto out;
 	}
 
-	/* Need to do basic initialisation of all rings first: */
-	ret = __i915_gem_restart_engines(dev_priv);
-	if (ret)
-		goto out;
-
-	intel_mocs_init_l3cc_table(dev_priv);
-
 	/* We can't enable contexts until all firmware is loaded */
 	ret = intel_uc_init_hw(dev_priv);
 	if (ret)
 		goto out;
 
+	intel_mocs_init_l3cc_table(dev_priv);
+
+	/* Only when the HW is re-initialised, can we replay the requests */
+	ret = __i915_gem_restart_engines(dev_priv);
 out:
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 	return ret;
 }
 
-bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
+static int __intel_engines_record_defaults(struct drm_i915_private *i915)
 {
-	if (INTEL_INFO(dev_priv)->gen < 6)
-		return false;
+	struct i915_gem_context *ctx;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	int err;
 
-	/* TODO: make semaphores and Execlists play nicely together */
-	if (i915_modparams.enable_execlists)
-		return false;
+	/*
+	 * As we reset the gpu during very early sanitisation, the current
+	 * register state on the GPU should reflect its defaults values.
+	 * We load a context onto the hw (with restore-inhibit), then switch
+	 * over to a second context to save that default register state. We
+	 * can then prime every new context with that state so they all start
+	 * from the same default HW values.
+	 */
 
-	if (value >= 0)
-		return value;
+	ctx = i915_gem_context_create_kernel(i915, 0);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
 
-	/* Enable semaphores on SNB when IO remapping is off */
-	if (IS_GEN6(dev_priv) && intel_vtd_active())
-		return false;
+	for_each_engine(engine, i915, id) {
+		struct drm_i915_gem_request *rq;
 
-	return true;
+		rq = i915_gem_request_alloc(engine, ctx);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto out_ctx;
+		}
+
+		err = 0;
+		if (engine->init_context)
+			err = engine->init_context(rq);
+
+		__i915_add_request(rq, true);
+		if (err)
+			goto err_active;
+	}
+
+	err = i915_gem_switch_to_kernel_context(i915);
+	if (err)
+		goto err_active;
+
+	err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+	if (err)
+		goto err_active;
+
+	assert_kernel_context_is_current(i915);
+
+	for_each_engine(engine, i915, id) {
+		struct i915_vma *state;
+
+		state = ctx->engine[id].state;
+		if (!state)
+			continue;
+
+		/*
+		 * As we will hold a reference to the logical state, it will
+		 * not be torn down with the context, and importantly the
+		 * object will hold onto its vma (making it possible for a
+		 * stray GTT write to corrupt our defaults). Unmap the vma
+		 * from the GTT to prevent such accidents and reclaim the
+		 * space.
+		 */
+		err = i915_vma_unbind(state);
+		if (err)
+			goto err_active;
+
+		err = i915_gem_object_set_to_cpu_domain(state->obj, false);
+		if (err)
+			goto err_active;
+
+		engine->default_state = i915_gem_object_get(state->obj);
+	}
+
+	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
+		unsigned int found = intel_engines_has_context_isolation(i915);
+
+		/*
+		 * Make sure that classes with multiple engine instances all
+		 * share the same basic configuration.
+		 */
+		for_each_engine(engine, i915, id) {
+			unsigned int bit = BIT(engine->uabi_class);
+			unsigned int expected = engine->default_state ? bit : 0;
+
+			if ((found & bit) != expected) {
+				DRM_ERROR("mismatching default context state for class %d on engine %s\n",
+					  engine->uabi_class, engine->name);
+			}
+		}
+	}
+
+out_ctx:
+	i915_gem_context_set_closed(ctx);
+	i915_gem_context_put(ctx);
+	return err;
+
+err_active:
+	/*
+	 * If we have to abandon now, we expect the engines to be idle
+	 * and ready to be torn-down. First try to flush any remaining
+	 * request, ensure we are pointing at the kernel context and
+	 * then remove it.
+	 */
+	if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
+		goto out_ctx;
+
+	if (WARN_ON(i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED)))
+		goto out_ctx;
+
+	i915_gem_contexts_lost(i915);
+	goto out_ctx;
 }
 
 int i915_gem_init(struct drm_i915_private *dev_priv)
@@ -4952,18 +5186,22 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 
 	dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
 
-	if (!i915_modparams.enable_execlists) {
-		dev_priv->gt.resume = intel_legacy_submission_resume;
-		dev_priv->gt.cleanup_engine = intel_engine_cleanup;
-	} else {
+	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
 		dev_priv->gt.resume = intel_lr_context_resume;
 		dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
+	} else {
+		dev_priv->gt.resume = intel_legacy_submission_resume;
+		dev_priv->gt.cleanup_engine = intel_engine_cleanup;
 	}
 
 	ret = i915_gem_init_userptr(dev_priv);
 	if (ret)
 		return ret;
 
+	ret = intel_uc_init_wq(dev_priv);
+	if (ret)
+		return ret;
+
 	/* This is just a security blanket to placate dragons.
 	 * On some systems, we very sporadically observe that the first TLBs
 	 * used by the CS may be stale, despite us poking the TLB reset. If
@@ -4974,20 +5212,96 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
 	ret = i915_gem_init_ggtt(dev_priv);
-	if (ret)
-		goto out_unlock;
+	if (ret) {
+		GEM_BUG_ON(ret == -EIO);
+		goto err_unlock;
+	}
 
 	ret = i915_gem_contexts_init(dev_priv);
-	if (ret)
-		goto out_unlock;
+	if (ret) {
+		GEM_BUG_ON(ret == -EIO);
+		goto err_ggtt;
+	}
 
 	ret = intel_engines_init(dev_priv);
+	if (ret) {
+		GEM_BUG_ON(ret == -EIO);
+		goto err_context;
+	}
+
+	intel_init_gt_powersave(dev_priv);
+
+	ret = intel_uc_init(dev_priv);
 	if (ret)
-		goto out_unlock;
+		goto err_pm;
 
 	ret = i915_gem_init_hw(dev_priv);
+	if (ret)
+		goto err_uc_init;
+
+	/*
+	 * Despite its name intel_init_clock_gating applies both display
+	 * clock gating workarounds; GT mmio workarounds and the occasional
+	 * GT power context workaround. Worse, sometimes it includes a context
+	 * register workaround which we need to apply before we record the
+	 * default HW state for all contexts.
+	 *
+	 * FIXME: break up the workarounds and apply them at the right time!
+	 */
+	intel_init_clock_gating(dev_priv);
+
+	ret = __intel_engines_record_defaults(dev_priv);
+	if (ret)
+		goto err_init_hw;
+
+	if (i915_inject_load_failure()) {
+		ret = -ENODEV;
+		goto err_init_hw;
+	}
+
+	if (i915_inject_load_failure()) {
+		ret = -EIO;
+		goto err_init_hw;
+	}
+
+	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
+	return 0;
+
+	/*
+	 * Unwinding is complicated by that we want to handle -EIO to mean
+	 * disable GPU submission but keep KMS alive. We want to mark the
+	 * HW as irrevisibly wedged, but keep enough state around that the
+	 * driver doesn't explode during runtime.
+	 */
+err_init_hw:
+	i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
+	i915_gem_contexts_lost(dev_priv);
+	intel_uc_fini_hw(dev_priv);
+err_uc_init:
+	intel_uc_fini(dev_priv);
+err_pm:
+	if (ret != -EIO) {
+		intel_cleanup_gt_powersave(dev_priv);
+		i915_gem_cleanup_engines(dev_priv);
+	}
+err_context:
+	if (ret != -EIO)
+		i915_gem_contexts_fini(dev_priv);
+err_ggtt:
+err_unlock:
+	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+
+	intel_uc_fini_wq(dev_priv);
+
+	if (ret != -EIO)
+		i915_gem_cleanup_userptr(dev_priv);
+
 	if (ret == -EIO) {
-		/* Allow engine initialisation to fail by marking the GPU as
+		/*
+		 * Allow engine initialisation to fail by marking the GPU as
 		 * wedged. But we only want to do this where the GPU is angry,
 		 * for all other failure, such as an allocation failure, bail.
 		 */
@@ -4998,10 +5312,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 		ret = 0;
 	}
 
-out_unlock:
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-
+	i915_gem_drain_freed_objects(dev_priv);
 	return ret;
 }
 
@@ -5052,6 +5363,22 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 	i915_gem_detect_bit_6_swizzle(dev_priv);
 }
 
+static void i915_gem_init__mm(struct drm_i915_private *i915)
+{
+	spin_lock_init(&i915->mm.object_stat_lock);
+	spin_lock_init(&i915->mm.obj_lock);
+	spin_lock_init(&i915->mm.free_lock);
+
+	init_llist_head(&i915->mm.free_list);
+
+	INIT_LIST_HEAD(&i915->mm.unbound_list);
+	INIT_LIST_HEAD(&i915->mm.bound_list);
+	INIT_LIST_HEAD(&i915->mm.fence_list);
+	INIT_LIST_HEAD(&i915->mm.userfault_list);
+
+	INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
+}
+
 int
 i915_gem_load_init(struct drm_i915_private *dev_priv)
 {
@@ -5093,15 +5420,7 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
 	if (err)
 		goto err_priorities;
 
-	INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
-
-	spin_lock_init(&dev_priv->mm.obj_lock);
-	spin_lock_init(&dev_priv->mm.free_lock);
-	init_llist_head(&dev_priv->mm.free_list);
-	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
-	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
-	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-	INIT_LIST_HEAD(&dev_priv->mm.userfault_list);
+	i915_gem_init__mm(dev_priv);
 
 	INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
 			  i915_gem_retire_work_handler);
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index ee54597465b6..e920dab7f1b8 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -28,7 +28,11 @@
 #include <linux/bug.h>
 
 #ifdef CONFIG_DRM_I915_DEBUG_GEM
-#define GEM_BUG_ON(expr) BUG_ON(expr)
+#define GEM_BUG_ON(condition) do { if (unlikely((condition))) {	\
+		printk(KERN_ERR "GEM_BUG_ON(%s)\n", __stringify(condition)); \
+		BUG(); \
+		} \
+	} while(0)
 #define GEM_WARN_ON(expr) WARN_ON(expr)
 
 #define GEM_DEBUG_DECL(var) var
@@ -44,6 +48,12 @@
 #define GEM_DEBUG_BUG_ON(expr)
 #endif
 
+#if IS_ENABLED(CONFIG_DRM_I915_TRACE_GEM)
+#define GEM_TRACE(...) trace_printk(__VA_ARGS__)
+#else
+#define GEM_TRACE(...) do { } while (0)
+#endif
+
 #define I915_NUM_ENGINES 5
 
 #endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_clflush.c b/drivers/gpu/drm/i915/i915_gem_clflush.c
index f663cd919795..b9b53ac14176 100644
--- a/drivers/gpu/drm/i915/i915_gem_clflush.c
+++ b/drivers/gpu/drm/i915/i915_gem_clflush.c
@@ -167,7 +167,7 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj,
 		i915_sw_fence_await_reservation(&clflush->wait,
 						obj->resv, NULL,
 						true, I915_FENCE_TIMEOUT,
-						GFP_KERNEL);
+						I915_FENCE_GFP);
 
 		reservation_object_lock(obj->resv, NULL);
 		reservation_object_add_excl_fence(obj->resv, &clflush->dma);
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index f782cf2069c1..0c963fcf31ff 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -316,7 +316,7 @@ __create_hw_context(struct drm_i915_private *dev_priv,
 	 * present or not in use we still need a small bias as ring wraparound
 	 * at offset 0 sometimes hangs. No idea why.
 	 */
-	if (HAS_GUC(dev_priv) && i915_modparams.enable_guc_loading)
+	if (USES_GUC(dev_priv))
 		ctx->ggtt_offset_bias = GUC_WOPCM_TOP;
 	else
 		ctx->ggtt_offset_bias = I915_GTT_PAGE_SIZE;
@@ -409,7 +409,7 @@ i915_gem_context_create_gvt(struct drm_device *dev)
 	i915_gem_context_set_closed(ctx); /* not user accessible */
 	i915_gem_context_clear_bannable(ctx);
 	i915_gem_context_set_force_single_submission(ctx);
-	if (!i915_modparams.enable_guc_submission)
+	if (!USES_GUC_SUBMISSION(to_i915(dev)))
 		ctx->ring_size = 512 * PAGE_SIZE; /* Max ring buffer size */
 
 	GEM_BUG_ON(i915_gem_context_is_kernel(ctx));
@@ -418,8 +418,8 @@ i915_gem_context_create_gvt(struct drm_device *dev)
 	return ctx;
 }
 
-static struct i915_gem_context *
-create_kernel_context(struct drm_i915_private *i915, int prio)
+struct i915_gem_context *
+i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio)
 {
 	struct i915_gem_context *ctx;
 
@@ -460,20 +460,12 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
 	INIT_WORK(&dev_priv->contexts.free_work, contexts_free_worker);
 	init_llist_head(&dev_priv->contexts.free_list);
 
-	if (intel_vgpu_active(dev_priv) &&
-	    HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
-		if (!i915_modparams.enable_execlists) {
-			DRM_INFO("Only EXECLIST mode is supported in vgpu.\n");
-			return -EINVAL;
-		}
-	}
-
 	/* Using the simple ida interface, the max is limited by sizeof(int) */
 	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX);
 	ida_init(&dev_priv->contexts.hw_ida);
 
 	/* lowest priority; idle task */
-	ctx = create_kernel_context(dev_priv, I915_PRIORITY_MIN);
+	ctx = i915_gem_context_create_kernel(dev_priv, I915_PRIORITY_MIN);
 	if (IS_ERR(ctx)) {
 		DRM_ERROR("Failed to create default global context\n");
 		err = PTR_ERR(ctx);
@@ -487,7 +479,7 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
 	dev_priv->kernel_context = ctx;
 
 	/* highest priority; preempting task */
-	ctx = create_kernel_context(dev_priv, INT_MAX);
+	ctx = i915_gem_context_create_kernel(dev_priv, INT_MAX);
 	if (IS_ERR(ctx)) {
 		DRM_ERROR("Failed to create default preempt context\n");
 		err = PTR_ERR(ctx);
@@ -515,6 +507,7 @@ void i915_gem_contexts_lost(struct drm_i915_private *dev_priv)
 
 	for_each_engine(engine, dev_priv, id) {
 		engine->legacy_active_context = NULL;
+		engine->legacy_active_ppgtt = NULL;
 
 		if (!engine->last_retired_context)
 			continue;
@@ -522,28 +515,6 @@ void i915_gem_contexts_lost(struct drm_i915_private *dev_priv)
 		engine->context_unpin(engine, engine->last_retired_context);
 		engine->last_retired_context = NULL;
 	}
-
-	/* Force the GPU state to be restored on enabling */
-	if (!i915_modparams.enable_execlists) {
-		struct i915_gem_context *ctx;
-
-		list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
-			if (!i915_gem_context_is_default(ctx))
-				continue;
-
-			for_each_engine(engine, dev_priv, id)
-				ctx->engine[engine->id].initialised = false;
-
-			ctx->remap_slice = ALL_L3_SLICES(dev_priv);
-		}
-
-		for_each_engine(engine, dev_priv, id) {
-			struct intel_context *kce =
-				&dev_priv->kernel_context->engine[engine->id];
-
-			kce->initialised = true;
-		}
-	}
 }
 
 void i915_gem_contexts_fini(struct drm_i915_private *i915)
@@ -596,310 +567,7 @@ void i915_gem_context_close(struct drm_file *file)
 	idr_destroy(&file_priv->context_idr);
 }
 
-static inline int
-mi_set_context(struct drm_i915_gem_request *req, u32 flags)
-{
-	struct drm_i915_private *dev_priv = req->i915;
-	struct intel_engine_cs *engine = req->engine;
-	enum intel_engine_id id;
-	const int num_rings =
-		/* Use an extended w/a on gen7 if signalling from other rings */
-		(i915_modparams.semaphores && INTEL_GEN(dev_priv) == 7) ?
-		INTEL_INFO(dev_priv)->num_rings - 1 :
-		0;
-	int len;
-	u32 *cs;
-
-	flags |= MI_MM_SPACE_GTT;
-	if (IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8)
-		/* These flags are for resource streamer on HSW+ */
-		flags |= HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN;
-	else
-		flags |= MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN;
-
-	len = 4;
-	if (INTEL_GEN(dev_priv) >= 7)
-		len += 2 + (num_rings ? 4*num_rings + 6 : 0);
-
-	cs = intel_ring_begin(req, len);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
-	if (INTEL_GEN(dev_priv) >= 7) {
-		*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
-		if (num_rings) {
-			struct intel_engine_cs *signaller;
-
-			*cs++ = MI_LOAD_REGISTER_IMM(num_rings);
-			for_each_engine(signaller, dev_priv, id) {
-				if (signaller == engine)
-					continue;
-
-				*cs++ = i915_mmio_reg_offset(
-					   RING_PSMI_CTL(signaller->mmio_base));
-				*cs++ = _MASKED_BIT_ENABLE(
-						GEN6_PSMI_SLEEP_MSG_DISABLE);
-			}
-		}
-	}
-
-	*cs++ = MI_NOOP;
-	*cs++ = MI_SET_CONTEXT;
-	*cs++ = i915_ggtt_offset(req->ctx->engine[RCS].state) | flags;
-	/*
-	 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
-	 * WaMiSetContext_Hang:snb,ivb,vlv
-	 */
-	*cs++ = MI_NOOP;
-
-	if (INTEL_GEN(dev_priv) >= 7) {
-		if (num_rings) {
-			struct intel_engine_cs *signaller;
-			i915_reg_t last_reg = {}; /* keep gcc quiet */
-
-			*cs++ = MI_LOAD_REGISTER_IMM(num_rings);
-			for_each_engine(signaller, dev_priv, id) {
-				if (signaller == engine)
-					continue;
-
-				last_reg = RING_PSMI_CTL(signaller->mmio_base);
-				*cs++ = i915_mmio_reg_offset(last_reg);
-				*cs++ = _MASKED_BIT_DISABLE(
-						GEN6_PSMI_SLEEP_MSG_DISABLE);
-			}
-
-			/* Insert a delay before the next switch! */
-			*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
-			*cs++ = i915_mmio_reg_offset(last_reg);
-			*cs++ = i915_ggtt_offset(engine->scratch);
-			*cs++ = MI_NOOP;
-		}
-		*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
-	}
-
-	intel_ring_advance(req, cs);
-
-	return 0;
-}
-
-static int remap_l3(struct drm_i915_gem_request *req, int slice)
-{
-	u32 *cs, *remap_info = req->i915->l3_parity.remap_info[slice];
-	int i;
-
-	if (!remap_info)
-		return 0;
-
-	cs = intel_ring_begin(req, GEN7_L3LOG_SIZE/4 * 2 + 2);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	/*
-	 * Note: We do not worry about the concurrent register cacheline hang
-	 * here because no other code should access these registers other than
-	 * at initialization time.
-	 */
-	*cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4);
-	for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
-		*cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
-		*cs++ = remap_info[i];
-	}
-	*cs++ = MI_NOOP;
-	intel_ring_advance(req, cs);
-
-	return 0;
-}
-
-static inline bool skip_rcs_switch(struct i915_hw_ppgtt *ppgtt,
-				   struct intel_engine_cs *engine,
-				   struct i915_gem_context *to)
-{
-	if (to->remap_slice)
-		return false;
-
-	if (!to->engine[RCS].initialised)
-		return false;
-
-	if (ppgtt && (intel_engine_flag(engine) & ppgtt->pd_dirty_rings))
-		return false;
-
-	return to == engine->legacy_active_context;
-}
-
-static bool
-needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt, struct intel_engine_cs *engine)
-{
-	struct i915_gem_context *from = engine->legacy_active_context;
-
-	if (!ppgtt)
-		return false;
-
-	/* Always load the ppgtt on first use */
-	if (!from)
-		return true;
-
-	/* Same context without new entries, skip */
-	if ((!from->ppgtt || from->ppgtt == ppgtt) &&
-	    !(intel_engine_flag(engine) & ppgtt->pd_dirty_rings))
-		return false;
-
-	if (engine->id != RCS)
-		return true;
-
-	if (INTEL_GEN(engine->i915) < 8)
-		return true;
-
-	return false;
-}
-
-static bool
-needs_pd_load_post(struct i915_hw_ppgtt *ppgtt,
-		   struct i915_gem_context *to,
-		   u32 hw_flags)
-{
-	if (!ppgtt)
-		return false;
-
-	if (!IS_GEN8(to->i915))
-		return false;
-
-	if (hw_flags & MI_RESTORE_INHIBIT)
-		return true;
-
-	return false;
-}
-
-static int do_rcs_switch(struct drm_i915_gem_request *req)
-{
-	struct i915_gem_context *to = req->ctx;
-	struct intel_engine_cs *engine = req->engine;
-	struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
-	struct i915_gem_context *from = engine->legacy_active_context;
-	u32 hw_flags;
-	int ret, i;
-
-	GEM_BUG_ON(engine->id != RCS);
-
-	if (skip_rcs_switch(ppgtt, engine, to))
-		return 0;
-
-	if (needs_pd_load_pre(ppgtt, engine)) {
-		/* Older GENs and non render rings still want the load first,
-		 * "PP_DCLV followed by PP_DIR_BASE register through Load
-		 * Register Immediate commands in Ring Buffer before submitting
-		 * a context."*/
-		trace_switch_mm(engine, to);
-		ret = ppgtt->switch_mm(ppgtt, req);
-		if (ret)
-			return ret;
-	}
-
-	if (!to->engine[RCS].initialised || i915_gem_context_is_default(to))
-		/* NB: If we inhibit the restore, the context is not allowed to
-		 * die because future work may end up depending on valid address
-		 * space. This means we must enforce that a page table load
-		 * occur when this occurs. */
-		hw_flags = MI_RESTORE_INHIBIT;
-	else if (ppgtt && intel_engine_flag(engine) & ppgtt->pd_dirty_rings)
-		hw_flags = MI_FORCE_RESTORE;
-	else
-		hw_flags = 0;
-
-	if (to != from || (hw_flags & MI_FORCE_RESTORE)) {
-		ret = mi_set_context(req, hw_flags);
-		if (ret)
-			return ret;
-
-		engine->legacy_active_context = to;
-	}
-
-	/* GEN8 does *not* require an explicit reload if the PDPs have been
-	 * setup, and we do not wish to move them.
-	 */
-	if (needs_pd_load_post(ppgtt, to, hw_flags)) {
-		trace_switch_mm(engine, to);
-		ret = ppgtt->switch_mm(ppgtt, req);
-		/* The hardware context switch is emitted, but we haven't
-		 * actually changed the state - so it's probably safe to bail
-		 * here. Still, let the user know something dangerous has
-		 * happened.
-		 */
-		if (ret)
-			return ret;
-	}
-
-	if (ppgtt)
-		ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
-
-	for (i = 0; i < MAX_L3_SLICES; i++) {
-		if (!(to->remap_slice & (1<<i)))
-			continue;
-
-		ret = remap_l3(req, i);
-		if (ret)
-			return ret;
-
-		to->remap_slice &= ~(1<<i);
-	}
-
-	if (!to->engine[RCS].initialised) {
-		if (engine->init_context) {
-			ret = engine->init_context(req);
-			if (ret)
-				return ret;
-		}
-		to->engine[RCS].initialised = true;
-	}
-
-	return 0;
-}
-
-/**
- * i915_switch_context() - perform a GPU context switch.
- * @req: request for which we'll execute the context switch
- *
- * The context life cycle is simple. The context refcount is incremented and
- * decremented by 1 and create and destroy. If the context is in use by the GPU,
- * it will have a refcount > 1. This allows us to destroy the context abstract
- * object while letting the normal object tracking destroy the backing BO.
- *
- * This function should not be used in execlists mode.  Instead the context is
- * switched by writing to the ELSP and requests keep a reference to their
- * context.
- */
-int i915_switch_context(struct drm_i915_gem_request *req)
-{
-	struct intel_engine_cs *engine = req->engine;
-
-	lockdep_assert_held(&req->i915->drm.struct_mutex);
-	if (i915_modparams.enable_execlists)
-		return 0;
-
-	if (!req->ctx->engine[engine->id].state) {
-		struct i915_gem_context *to = req->ctx;
-		struct i915_hw_ppgtt *ppgtt =
-			to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
-
-		if (needs_pd_load_pre(ppgtt, engine)) {
-			int ret;
-
-			trace_switch_mm(engine, to);
-			ret = ppgtt->switch_mm(ppgtt, req);
-			if (ret)
-				return ret;
-
-			ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
-		}
-
-		engine->legacy_active_context = to;
-		return 0;
-	}
-
-	return do_rcs_switch(req);
-}
-
-static bool engine_has_kernel_context(struct intel_engine_cs *engine)
+static bool engine_has_idle_kernel_context(struct intel_engine_cs *engine)
 {
 	struct i915_gem_timeline *timeline;
 
@@ -915,8 +583,7 @@ static bool engine_has_kernel_context(struct intel_engine_cs *engine)
 			return false;
 	}
 
-	return (!engine->last_retired_context ||
-		i915_gem_context_is_kernel(engine->last_retired_context));
+	return intel_engine_has_kernel_context(engine);
 }
 
 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
@@ -931,9 +598,8 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 
 	for_each_engine(engine, dev_priv, id) {
 		struct drm_i915_gem_request *req;
-		int ret;
 
-		if (engine_has_kernel_context(engine))
+		if (engine_has_idle_kernel_context(engine))
 			continue;
 
 		req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
@@ -951,13 +617,17 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
 			if (prev)
 				i915_sw_fence_await_sw_fence_gfp(&req->submit,
 								 &prev->submit,
-								 GFP_KERNEL);
+								 I915_FENCE_GFP);
 		}
 
-		ret = i915_switch_context(req);
-		i915_add_request(req);
-		if (ret)
-			return ret;
+		/*
+		 * Force a flush after the switch to ensure that all rendering
+		 * and operations prior to switching to the kernel context hits
+		 * memory. This should be guaranteed by the previous request,
+		 * but an extra layer of paranoia before we declare the system
+		 * idle (on suspend etc) is advisable!
+		 */
+		__i915_add_request(req, true);
 	}
 
 	return 0;
@@ -1133,7 +803,7 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 
 	case I915_CONTEXT_PARAM_PRIORITY:
 		{
-			int priority = args->value;
+			s64 priority = args->value;
 
 			if (args->size)
 				ret = -EINVAL;
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index 44688e22a5c2..4bfb72f8e1cb 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -157,7 +157,6 @@ struct i915_gem_context {
 		u32 *lrc_reg_state;
 		u64 lrc_desc;
 		int pin_count;
-		bool initialised;
 	} engine[I915_NUM_ENGINES];
 
 	/** ring_size: size for allocating the per-engine ring buffer */
@@ -292,6 +291,9 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
 				       struct drm_file *file);
 
+struct i915_gem_context *
+i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio);
+
 static inline struct i915_gem_context *
 i915_gem_context_get(struct i915_gem_context *ctx)
 {
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 8daa8a78cdc0..60ca4f05ae94 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -46,7 +46,7 @@ static bool ggtt_is_idle(struct drm_i915_private *i915)
 	       return false;
 
        for_each_engine(engine, i915, id) {
-	       if (engine->last_retired_context != i915->kernel_context)
+	       if (!intel_engine_has_kernel_context(engine))
 		       return false;
        }
 
@@ -73,6 +73,7 @@ static int ggtt_flush(struct drm_i915_private *i915)
 	if (err)
 		return err;
 
+	GEM_BUG_ON(!ggtt_is_idle(i915));
 	return 0;
 }
 
@@ -216,6 +217,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
 		if (ret)
 			return ret;
 
+		cond_resched();
 		goto search_again;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 435ed95df144..3ab1ace2a6bd 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -271,7 +271,7 @@ static inline u64 gen8_noncanonical_addr(u64 address)
 
 static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
 {
-	return eb->engine->needs_cmd_parser && eb->batch_len;
+	return intel_engine_needs_cmd_parser(eb->engine) && eb->batch_len;
 }
 
 static int eb_create(struct i915_execbuffer *eb)
@@ -505,6 +505,8 @@ eb_add_vma(struct i915_execbuffer *eb, unsigned int i, struct i915_vma *vma)
 		list_add_tail(&vma->exec_link, &eb->unbound);
 		if (drm_mm_node_allocated(&vma->node))
 			err = i915_vma_unbind(vma);
+		if (unlikely(err))
+			vma->exec_flags = NULL;
 	}
 	return err;
 }
@@ -1012,7 +1014,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 		offset += page << PAGE_SHIFT;
 	}
 
-	vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->mappable,
+	vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap,
 							 offset);
 	cache->page = page;
 	cache->vaddr = (unsigned long)vaddr;
@@ -1111,14 +1113,6 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
 	if (err)
 		goto err_request;
 
-	err = eb->engine->emit_flush(rq, EMIT_INVALIDATE);
-	if (err)
-		goto err_request;
-
-	err = i915_switch_context(rq);
-	if (err)
-		goto err_request;
-
 	err = eb->engine->emit_bb_start(rq,
 					batch->node.start, PAGE_SIZE,
 					cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
@@ -1818,8 +1812,7 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
 	/* Unconditionally flush any chipset caches (for streaming writes). */
 	i915_gem_chipset_flush(eb->i915);
 
-	/* Unconditionally invalidate GPU caches and TLBs. */
-	return eb->engine->emit_flush(eb->request, EMIT_INVALIDATE);
+	return 0;
 }
 
 static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
@@ -1965,10 +1958,6 @@ static int eb_submit(struct i915_execbuffer *eb)
 	if (err)
 		return err;
 
-	err = i915_switch_context(eb->request);
-	if (err)
-		return err;
-
 	if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
 		err = i915_reset_gen7_sol_offsets(eb->request);
 		if (err)
@@ -2074,23 +2063,27 @@ static struct drm_syncobj **
 get_fence_array(struct drm_i915_gem_execbuffer2 *args,
 		struct drm_file *file)
 {
-	const unsigned int nfences = args->num_cliprects;
+	const unsigned long nfences = args->num_cliprects;
 	struct drm_i915_gem_exec_fence __user *user;
 	struct drm_syncobj **fences;
-	unsigned int n;
+	unsigned long n;
 	int err;
 
 	if (!(args->flags & I915_EXEC_FENCE_ARRAY))
 		return NULL;
 
-	if (nfences > SIZE_MAX / sizeof(*fences))
+	/* Check multiplication overflow for access_ok() and kvmalloc_array() */
+	BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
+	if (nfences > min_t(unsigned long,
+			    ULONG_MAX / sizeof(*user),
+			    SIZE_MAX / sizeof(*fences)))
 		return ERR_PTR(-EINVAL);
 
 	user = u64_to_user_ptr(args->cliprects_ptr);
-	if (!access_ok(VERIFY_READ, user, nfences * 2 * sizeof(u32)))
+	if (!access_ok(VERIFY_READ, user, nfences * sizeof(*user)))
 		return ERR_PTR(-EFAULT);
 
-	fences = kvmalloc_array(args->num_cliprects, sizeof(*fences),
+	fences = kvmalloc_array(nfences, sizeof(*fences),
 				__GFP_NOWARN | GFP_KERNEL);
 	if (!fences)
 		return ERR_PTR(-ENOMEM);
@@ -2419,7 +2412,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 	if (out_fence) {
 		if (err == 0) {
 			fd_install(out_fence_fd, out_fence->file);
-			args->rsvd2 &= GENMASK_ULL(0, 31); /* keep in-fence */
+			args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */
 			args->rsvd2 |= (u64)out_fence_fd << 32;
 			out_fence_fd = -1;
 		} else {
@@ -2447,6 +2440,26 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 	return err;
 }
 
+static size_t eb_element_size(void)
+{
+	return (sizeof(struct drm_i915_gem_exec_object2) +
+		sizeof(struct i915_vma *) +
+		sizeof(unsigned int));
+}
+
+static bool check_buffer_count(size_t count)
+{
+	const size_t sz = eb_element_size();
+
+	/*
+	 * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup
+	 * array size (see eb_create()). Otherwise, we can accept an array as
+	 * large as can be addressed (though use large arrays at your peril)!
+	 */
+
+	return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
+}
+
 /*
  * Legacy execbuffer just creates an exec2 list from the original exec object
  * list array and passes it to the real function.
@@ -2455,18 +2468,16 @@ int
 i915_gem_execbuffer(struct drm_device *dev, void *data,
 		    struct drm_file *file)
 {
-	const size_t sz = (sizeof(struct drm_i915_gem_exec_object2) +
-			   sizeof(struct i915_vma *) +
-			   sizeof(unsigned int));
 	struct drm_i915_gem_execbuffer *args = data;
 	struct drm_i915_gem_execbuffer2 exec2;
 	struct drm_i915_gem_exec_object *exec_list = NULL;
 	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
+	const size_t count = args->buffer_count;
 	unsigned int i;
 	int err;
 
-	if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
-		DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
+	if (!check_buffer_count(count)) {
+		DRM_DEBUG("execbuf2 with %zd buffers\n", count);
 		return -EINVAL;
 	}
 
@@ -2485,9 +2496,9 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 		return -EINVAL;
 
 	/* Copy in the exec list from userland */
-	exec_list = kvmalloc_array(args->buffer_count, sizeof(*exec_list),
+	exec_list = kvmalloc_array(count, sizeof(*exec_list),
 				   __GFP_NOWARN | GFP_KERNEL);
-	exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+	exec2_list = kvmalloc_array(count + 1, eb_element_size(),
 				    __GFP_NOWARN | GFP_KERNEL);
 	if (exec_list == NULL || exec2_list == NULL) {
 		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
@@ -2498,7 +2509,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
 	}
 	err = copy_from_user(exec_list,
 			     u64_to_user_ptr(args->buffers_ptr),
-			     sizeof(*exec_list) * args->buffer_count);
+			     sizeof(*exec_list) * count);
 	if (err) {
 		DRM_DEBUG("copy %d exec entries failed %d\n",
 			  args->buffer_count, err);
@@ -2548,16 +2559,14 @@ int
 i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		     struct drm_file *file)
 {
-	const size_t sz = (sizeof(struct drm_i915_gem_exec_object2) +
-			   sizeof(struct i915_vma *) +
-			   sizeof(unsigned int));
 	struct drm_i915_gem_execbuffer2 *args = data;
 	struct drm_i915_gem_exec_object2 *exec2_list;
 	struct drm_syncobj **fences = NULL;
+	const size_t count = args->buffer_count;
 	int err;
 
-	if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
-		DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
+	if (!check_buffer_count(count)) {
+		DRM_DEBUG("execbuf2 with %zd buffers\n", count);
 		return -EINVAL;
 	}
 
@@ -2565,17 +2574,17 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		return -EINVAL;
 
 	/* Allocate an extra slot for use by the command parser */
-	exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+	exec2_list = kvmalloc_array(count + 1, eb_element_size(),
 				    __GFP_NOWARN | GFP_KERNEL);
 	if (exec2_list == NULL) {
-		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
-			  args->buffer_count);
+		DRM_DEBUG("Failed to allocate exec list for %zd buffers\n",
+			  count);
 		return -ENOMEM;
 	}
 	if (copy_from_user(exec2_list,
 			   u64_to_user_ptr(args->buffers_ptr),
-			   sizeof(*exec2_list) * args->buffer_count)) {
-		DRM_DEBUG("copy %d exec entries failed\n", args->buffer_count);
+			   sizeof(*exec2_list) * count)) {
+		DRM_DEBUG("copy %zd exec entries failed\n", count);
 		kvfree(exec2_list);
 		return -EFAULT;
 	}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 2af65ecf2df8..7e403eaa9e0f 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -178,7 +178,7 @@ int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
 		return 0;
 	}
 
-	if (INTEL_GEN(dev_priv) >= 8 && i915_modparams.enable_execlists) {
+	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
 		if (has_full_48bit_ppgtt)
 			return 3;
 
@@ -377,6 +377,7 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr,
 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 {
 	struct pagevec *pvec = &vm->free_pages;
+	struct pagevec stash;
 
 	if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
 		i915_gem_shrink_all(vm->i915);
@@ -395,7 +396,15 @@ static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 	if (likely(pvec->nr))
 		return pvec->pages[--pvec->nr];
 
-	/* Otherwise batch allocate pages to amoritize cost of set_pages_wc. */
+	/*
+	 * Otherwise batch allocate pages to amoritize cost of set_pages_wc.
+	 *
+	 * We have to be careful as page allocation may trigger the shrinker
+	 * (via direct reclaim) which will fill up the WC stash underneath us.
+	 * So we add our WB pages into a temporary pvec on the stack and merge
+	 * them into the WC stash after all the allocations are complete.
+	 */
+	pagevec_init(&stash);
 	do {
 		struct page *page;
 
@@ -403,15 +412,24 @@ static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 		if (unlikely(!page))
 			break;
 
-		pvec->pages[pvec->nr++] = page;
-	} while (pagevec_space(pvec));
+		stash.pages[stash.nr++] = page;
+	} while (stash.nr < pagevec_space(pvec));
+
+	if (stash.nr) {
+		int nr = min_t(int, stash.nr, pagevec_space(pvec));
+		struct page **pages = stash.pages + stash.nr - nr;
 
-	if (unlikely(!pvec->nr))
-		return NULL;
+		if (nr && !set_pages_array_wc(pages, nr)) {
+			memcpy(pvec->pages + pvec->nr,
+			       pages, sizeof(pages[0]) * nr);
+			pvec->nr += nr;
+			stash.nr -= nr;
+		}
 
-	set_pages_array_wc(pvec->pages, pvec->nr);
+		pagevec_release(&stash);
+	}
 
-	return pvec->pages[--pvec->nr];
+	return likely(pvec->nr) ? pvec->pages[--pvec->nr] : NULL;
 }
 
 static void vm_free_pages_release(struct i915_address_space *vm,
@@ -454,6 +472,14 @@ static void vm_free_pages_release(struct i915_address_space *vm,
 
 static void vm_free_page(struct i915_address_space *vm, struct page *page)
 {
+	/*
+	 * On !llc, we need to change the pages back to WB. We only do so
+	 * in bulk, so we rarely need to change the page attributes here,
+	 * but doing so requires a stop_machine() from deep inside arch/x86/mm.
+	 * To make detection of the possible sleep more likely, use an
+	 * unconditional might_sleep() for everybody.
+	 */
+	might_sleep();
 	if (!pagevec_add(&vm->free_pages, page))
 		vm_free_pages_release(vm, false);
 }
@@ -1333,15 +1359,18 @@ static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
 		int count = gen8_pte_count(start, length);
 
 		if (pt == vm->scratch_pt) {
+			pd->used_pdes++;
+
 			pt = alloc_pt(vm);
-			if (IS_ERR(pt))
+			if (IS_ERR(pt)) {
+				pd->used_pdes--;
 				goto unwind;
+			}
 
 			if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
 				gen8_initialize_pt(vm, pt);
 
 			gen8_ppgtt_set_pde(vm, pd, pt, pde);
-			pd->used_pdes++;
 			GEM_BUG_ON(pd->used_pdes > I915_PDES);
 		}
 
@@ -1365,13 +1394,16 @@ static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
 
 	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
 		if (pd == vm->scratch_pd) {
+			pdp->used_pdpes++;
+
 			pd = alloc_pd(vm);
-			if (IS_ERR(pd))
+			if (IS_ERR(pd)) {
+				pdp->used_pdpes--;
 				goto unwind;
+			}
 
 			gen8_initialize_pd(vm, pd);
 			gen8_ppgtt_set_pdpe(vm, pdp, pd, pdpe);
-			pdp->used_pdpes++;
 			GEM_BUG_ON(pdp->used_pdpes > i915_pdpes_per_pdp(vm));
 
 			mark_tlbs_dirty(i915_vm_to_ppgtt(vm));
@@ -2154,7 +2186,7 @@ int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
 	/* In the case of execlists, PPGTT is enabled by the context descriptor
 	 * and the PDPs are contained within the context itself.  We don't
 	 * need to do anything here. */
-	if (i915_modparams.enable_execlists)
+	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv))
 		return 0;
 
 	if (!USES_PPGTT(dev_priv))
@@ -2248,35 +2280,73 @@ static bool needs_idle_maps(struct drm_i915_private *dev_priv)
 	return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active();
 }
 
-void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
+static void gen6_check_and_clear_faults(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-
-	if (INTEL_INFO(dev_priv)->gen < 6)
-		return;
+	u32 fault;
 
 	for_each_engine(engine, dev_priv, id) {
-		u32 fault_reg;
-		fault_reg = I915_READ(RING_FAULT_REG(engine));
-		if (fault_reg & RING_FAULT_VALID) {
+		fault = I915_READ(RING_FAULT_REG(engine));
+		if (fault & RING_FAULT_VALID) {
 			DRM_DEBUG_DRIVER("Unexpected fault\n"
 					 "\tAddr: 0x%08lx\n"
 					 "\tAddress space: %s\n"
 					 "\tSource ID: %d\n"
 					 "\tType: %d\n",
-					 fault_reg & PAGE_MASK,
-					 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
-					 RING_FAULT_SRCID(fault_reg),
-					 RING_FAULT_FAULT_TYPE(fault_reg));
+					 fault & PAGE_MASK,
+					 fault & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
+					 RING_FAULT_SRCID(fault),
+					 RING_FAULT_FAULT_TYPE(fault));
 			I915_WRITE(RING_FAULT_REG(engine),
-				   fault_reg & ~RING_FAULT_VALID);
+				   fault & ~RING_FAULT_VALID);
 		}
 	}
 
-	/* Engine specific init may not have been done till this point. */
-	if (dev_priv->engine[RCS])
-		POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
+	POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
+}
+
+static void gen8_check_and_clear_faults(struct drm_i915_private *dev_priv)
+{
+	u32 fault = I915_READ(GEN8_RING_FAULT_REG);
+
+	if (fault & RING_FAULT_VALID) {
+		u32 fault_data0, fault_data1;
+		u64 fault_addr;
+
+		fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
+		fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
+		fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
+			     ((u64)fault_data0 << 12);
+
+		DRM_DEBUG_DRIVER("Unexpected fault\n"
+				 "\tAddr: 0x%08x_%08x\n"
+				 "\tAddress space: %s\n"
+				 "\tEngine ID: %d\n"
+				 "\tSource ID: %d\n"
+				 "\tType: %d\n",
+				 upper_32_bits(fault_addr),
+				 lower_32_bits(fault_addr),
+				 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
+				 GEN8_RING_FAULT_ENGINE_ID(fault),
+				 RING_FAULT_SRCID(fault),
+				 RING_FAULT_FAULT_TYPE(fault));
+		I915_WRITE(GEN8_RING_FAULT_REG,
+			   fault & ~RING_FAULT_VALID);
+	}
+
+	POSTING_READ(GEN8_RING_FAULT_REG);
+}
+
+void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
+{
+	/* From GEN8 onwards we only have one 'All Engine Fault Register' */
+	if (INTEL_GEN(dev_priv) >= 8)
+		gen8_check_and_clear_faults(dev_priv);
+	else if (INTEL_GEN(dev_priv) >= 6)
+		gen6_check_and_clear_faults(dev_priv);
+	else
+		return;
 }
 
 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
@@ -2877,7 +2947,7 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	arch_phys_wc_del(ggtt->mtrr);
-	io_mapping_fini(&ggtt->mappable);
+	io_mapping_fini(&ggtt->iomap);
 }
 
 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
@@ -2914,50 +2984,6 @@ static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
 	return 0;
 }
 
-static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
-{
-	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
-	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
-	return (size_t)snb_gmch_ctl << 25; /* 32 MB units */
-}
-
-static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
-{
-	bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
-	bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
-	return (size_t)bdw_gmch_ctl << 25; /* 32 MB units */
-}
-
-static size_t chv_get_stolen_size(u16 gmch_ctrl)
-{
-	gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
-	gmch_ctrl &= SNB_GMCH_GMS_MASK;
-
-	/*
-	 * 0x0  to 0x10: 32MB increments starting at 0MB
-	 * 0x11 to 0x16: 4MB increments starting at 8MB
-	 * 0x17 to 0x1d: 4MB increments start at 36MB
-	 */
-	if (gmch_ctrl < 0x11)
-		return (size_t)gmch_ctrl << 25;
-	else if (gmch_ctrl < 0x17)
-		return (size_t)(gmch_ctrl - 0x11 + 2) << 22;
-	else
-		return (size_t)(gmch_ctrl - 0x17 + 9) << 22;
-}
-
-static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
-{
-	gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
-	gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
-
-	if (gen9_gmch_ctl < 0xf0)
-		return (size_t)gen9_gmch_ctl << 25; /* 32 MB units */
-	else
-		/* 4MB increments starting at 0xf0 for 4MB */
-		return (size_t)(gen9_gmch_ctl - 0xf0 + 1) << 22;
-}
-
 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 {
 	struct drm_i915_private *dev_priv = ggtt->base.i915;
@@ -3041,7 +3067,7 @@ const struct intel_ppat_entry *
 intel_ppat_get(struct drm_i915_private *i915, u8 value)
 {
 	struct intel_ppat *ppat = &i915->ppat;
-	struct intel_ppat_entry *entry;
+	struct intel_ppat_entry *entry = NULL;
 	unsigned int scanned, best_score;
 	int i;
 
@@ -3064,7 +3090,7 @@ intel_ppat_get(struct drm_i915_private *i915, u8 value)
 	}
 
 	if (scanned == ppat->max_entries) {
-		if (!best_score)
+		if (!entry)
 			return ERR_PTR(-ENOSPC);
 
 		kref_get(&entry->ref);
@@ -3171,12 +3197,6 @@ static void cnl_setup_private_ppat(struct intel_ppat *ppat)
 	ppat->match = bdw_private_pat_match;
 	ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
 
-	/* XXX: spec is unclear if this is still needed for CNL+ */
-	if (!USES_PPGTT(ppat->i915)) {
-		__alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
-		return;
-	}
-
 	__alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
 	__alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
 	__alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
@@ -3303,8 +3323,10 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 	int err;
 
 	/* TODO: We're not aware of mappable constraints on gen8 yet */
-	ggtt->mappable_base = pci_resource_start(pdev, 2);
-	ggtt->mappable_end = pci_resource_len(pdev, 2);
+	ggtt->gmadr =
+		(struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
+						 pci_resource_len(pdev, 2));
+	ggtt->mappable_end = resource_size(&ggtt->gmadr);
 
 	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
 	if (!err)
@@ -3315,13 +3337,10 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
 	if (INTEL_GEN(dev_priv) >= 9) {
-		ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl);
 		size = gen8_get_total_gtt_size(snb_gmch_ctl);
 	} else if (IS_CHERRYVIEW(dev_priv)) {
-		ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl);
 		size = chv_get_total_gtt_size(snb_gmch_ctl);
 	} else {
-		ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl);
 		size = gen8_get_total_gtt_size(snb_gmch_ctl);
 	}
 
@@ -3361,14 +3380,16 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 	u16 snb_gmch_ctl;
 	int err;
 
-	ggtt->mappable_base = pci_resource_start(pdev, 2);
-	ggtt->mappable_end = pci_resource_len(pdev, 2);
+	ggtt->gmadr =
+		(struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
+						 pci_resource_len(pdev, 2));
+	ggtt->mappable_end = resource_size(&ggtt->gmadr);
 
 	/* 64/512MB is the current min/max we actually know of, but this is just
 	 * a coarse sanity check.
 	 */
 	if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
-		DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end);
+		DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end);
 		return -ENXIO;
 	}
 
@@ -3379,8 +3400,6 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 		DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
 	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 
-	ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl);
-
 	size = gen6_get_total_gtt_size(snb_gmch_ctl);
 	ggtt->base.total = (size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
 
@@ -3417,6 +3436,7 @@ static void i915_gmch_remove(struct i915_address_space *vm)
 static int i915_gmch_probe(struct i915_ggtt *ggtt)
 {
 	struct drm_i915_private *dev_priv = ggtt->base.i915;
+	phys_addr_t gmadr_base;
 	int ret;
 
 	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
@@ -3426,10 +3446,13 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt)
 	}
 
 	intel_gtt_get(&ggtt->base.total,
-		      &ggtt->stolen_size,
-		      &ggtt->mappable_base,
+		      &gmadr_base,
 		      &ggtt->mappable_end);
 
+	ggtt->gmadr =
+		(struct resource) DEFINE_RES_MEM(gmadr_base,
+						 ggtt->mappable_end);
+
 	ggtt->do_idle_maps = needs_idle_maps(dev_priv);
 	ggtt->base.insert_page = i915_ggtt_insert_page;
 	ggtt->base.insert_entries = i915_ggtt_insert_entries;
@@ -3474,9 +3497,9 @@ int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
 	 * currently don't have any bits spare to pass in this upper
 	 * restriction!
 	 */
-	if (HAS_GUC(dev_priv) && i915_modparams.enable_guc_loading) {
+	if (USES_GUC(dev_priv)) {
 		ggtt->base.total = min_t(u64, ggtt->base.total, GUC_GGTT_TOP);
-		ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
+		ggtt->mappable_end = min_t(u64, ggtt->mappable_end, ggtt->base.total);
 	}
 
 	if ((ggtt->base.total - 1) >> 32) {
@@ -3484,21 +3507,21 @@ int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
 			  " of address space! Found %lldM!\n",
 			  ggtt->base.total >> 20);
 		ggtt->base.total = 1ULL << 32;
-		ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total);
+		ggtt->mappable_end = min_t(u64, ggtt->mappable_end, ggtt->base.total);
 	}
 
 	if (ggtt->mappable_end > ggtt->base.total) {
 		DRM_ERROR("mappable aperture extends past end of GGTT,"
-			  " aperture=%llx, total=%llx\n",
-			  ggtt->mappable_end, ggtt->base.total);
+			  " aperture=%pa, total=%llx\n",
+			  &ggtt->mappable_end, ggtt->base.total);
 		ggtt->mappable_end = ggtt->base.total;
 	}
 
 	/* GMADR is the PCI mmio aperture into the global GTT. */
-	DRM_INFO("Memory usable by graphics device = %lluM\n",
-		 ggtt->base.total >> 20);
-	DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20);
-	DRM_DEBUG_DRIVER("GTT stolen size = %uM\n", ggtt->stolen_size >> 20);
+	DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->base.total >> 20);
+	DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20);
+	DRM_DEBUG_DRIVER("DSM size = %lluM\n",
+			 (u64)resource_size(&intel_graphics_stolen_res) >> 20);
 	if (intel_vtd_active())
 		DRM_INFO("VT-d active for gfx access\n");
 
@@ -3527,14 +3550,14 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
 		ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
-	if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
-				dev_priv->ggtt.mappable_base,
+	if (!io_mapping_init_wc(&dev_priv->ggtt.iomap,
+				dev_priv->ggtt.gmadr.start,
 				dev_priv->ggtt.mappable_end)) {
 		ret = -EIO;
 		goto out_gtt_cleanup;
 	}
 
-	ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base, ggtt->mappable_end);
+	ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, ggtt->mappable_end);
 
 	/*
 	 * Initialise stolen early so that we may reserve preallocated
@@ -3564,6 +3587,8 @@ void i915_ggtt_enable_guc(struct drm_i915_private *i915)
 	GEM_BUG_ON(i915->ggtt.invalidate != gen6_ggtt_invalidate);
 
 	i915->ggtt.invalidate = guc_ggtt_invalidate;
+
+	i915_ggtt_invalidate(i915);
 }
 
 void i915_ggtt_disable_guc(struct drm_i915_private *i915)
@@ -3572,6 +3597,8 @@ void i915_ggtt_disable_guc(struct drm_i915_private *i915)
 	GEM_BUG_ON(i915->ggtt.invalidate != guc_ggtt_invalidate);
 
 	i915->ggtt.invalidate = gen6_ggtt_invalidate;
+
+	i915_ggtt_invalidate(i915);
 }
 
 void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
@@ -3591,10 +3618,7 @@ void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
 		bool ggtt_bound = false;
 		struct i915_vma *vma;
 
-		list_for_each_entry(vma, &obj->vma_list, obj_link) {
-			if (vma->vm != &ggtt->base)
-				continue;
-
+		for_each_ggtt_vma(vma, obj) {
 			if (!i915_vma_unbind(vma))
 				continue;
 
@@ -3708,9 +3732,6 @@ intel_rotate_pages(struct intel_rotation_info *rot_info,
 				  rot_info->plane[i].stride, st, sg);
 	}
 
-	DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages)\n",
-		      obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
-
 	kvfree(page_addr_list);
 
 	return st;
@@ -3720,8 +3741,8 @@ intel_rotate_pages(struct intel_rotation_info *rot_info,
 err_st_alloc:
 	kvfree(page_addr_list);
 
-	DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
-		      obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
+	DRM_DEBUG_DRIVER("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
+			 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
 
 	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 93211a96fdad..a42890d9af38 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -368,23 +368,10 @@ i915_vm_has_scratch_64K(struct i915_address_space *vm)
  */
 struct i915_ggtt {
 	struct i915_address_space base;
-	struct io_mapping mappable;	/* Mapping to our CPU mappable region */
 
-	phys_addr_t mappable_base;	/* PA of our GMADR */
-	u64 mappable_end;		/* End offset that we can CPU map */
-
-	/* Stolen memory is segmented in hardware with different portions
-	 * offlimits to certain functions.
-	 *
-	 * The drm_mm is initialised to the total accessible range, as found
-	 * from the PCI config. On Broadwell+, this is further restricted to
-	 * avoid the first page! The upper end of stolen memory is reserved for
-	 * hardware functions and similarly removed from the accessible range.
-	 */
-	u32 stolen_size;		/* Total size of stolen memory */
-	u32 stolen_usable_size;	/* Total size minus reserved ranges */
-	u32 stolen_reserved_base;
-	u32 stolen_reserved_size;
+	struct io_mapping iomap;	/* Mapping to our CPU mappable region */
+	struct resource gmadr;          /* GMADR resource */
+	resource_size_t mappable_end;	/* End offset that we can CPU map */
 
 	/** "Graphics Stolen Memory" holds the global PTEs */
 	void __iomem *gsm;
diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c
index ee83ec838ee7..a1d6956734f7 100644
--- a/drivers/gpu/drm/i915/i915_gem_internal.c
+++ b/drivers/gpu/drm/i915/i915_gem_internal.c
@@ -27,6 +27,7 @@
 #include "i915_drv.h"
 
 #define QUIET (__GFP_NORETRY | __GFP_NOWARN)
+#define MAYFAIL (__GFP_RETRY_MAYFAIL | __GFP_NOWARN)
 
 /* convert swiotlb segment size into sensible units (pages)! */
 #define IO_TLB_SEGPAGES (IO_TLB_SEGSIZE << IO_TLB_SHIFT >> PAGE_SHIFT)
@@ -95,7 +96,8 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 		struct page *page;
 
 		do {
-			page = alloc_pages(gfp | (order ? QUIET : 0), order);
+			page = alloc_pages(gfp | (order ? QUIET : MAYFAIL),
+					   order);
 			if (page)
 				break;
 			if (!order--)
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index 63ce38c1cce9..05e89e1c0a08 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -53,8 +53,9 @@ struct i915_lut_handle {
 
 struct drm_i915_gem_object_ops {
 	unsigned int flags;
-#define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0)
-#define I915_GEM_OBJECT_IS_SHRINKABLE   BIT(1)
+#define I915_GEM_OBJECT_HAS_STRUCT_PAGE	BIT(0)
+#define I915_GEM_OBJECT_IS_SHRINKABLE	BIT(1)
+#define I915_GEM_OBJECT_IS_PROXY	BIT(2)
 
 	/* Interface between the GEM object and its backing storage.
 	 * get_pages() is called once prior to the use of the associated set
@@ -260,6 +261,8 @@ struct drm_i915_gem_object {
 		} userptr;
 
 		unsigned long scratch;
+
+		void *gvt_info;
 	};
 
 	/** for phys allocated objects */
@@ -361,6 +364,12 @@ i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
 	return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE;
 }
 
+static inline bool
+i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj)
+{
+	return obj->ops->flags & I915_GEM_OBJECT_IS_PROXY;
+}
+
 static inline bool
 i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
 {
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 3703dc91eeda..f7fc0df251ac 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -26,10 +26,12 @@
  */
 
 #include "i915_drv.h"
+#include "i915_gem_render_state.h"
 #include "intel_renderstate.h"
 
 struct intel_render_state {
 	const struct intel_renderstate_rodata *rodata;
+	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	u32 batch_offset;
 	u32 batch_size;
@@ -40,6 +42,9 @@ struct intel_render_state {
 static const struct intel_renderstate_rodata *
 render_state_get_rodata(const struct intel_engine_cs *engine)
 {
+	if (engine->id != RCS)
+		return NULL;
+
 	switch (INTEL_GEN(engine->i915)) {
 	case 6:
 		return &gen6_null_state;
@@ -74,17 +79,16 @@ static int render_state_setup(struct intel_render_state *so,
 			      struct drm_i915_private *i915)
 {
 	const struct intel_renderstate_rodata *rodata = so->rodata;
-	struct drm_i915_gem_object *obj = so->vma->obj;
 	unsigned int i = 0, reloc_index = 0;
 	unsigned int needs_clflush;
 	u32 *d;
 	int ret;
 
-	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
+	ret = i915_gem_obj_prepare_shmem_write(so->obj, &needs_clflush);
 	if (ret)
 		return ret;
 
-	d = kmap_atomic(i915_gem_object_get_dirty_page(obj, 0));
+	d = kmap_atomic(i915_gem_object_get_dirty_page(so->obj, 0));
 
 	while (i < rodata->batch_items) {
 		u32 s = rodata->batch[i];
@@ -112,7 +116,7 @@ static int render_state_setup(struct intel_render_state *so,
 		goto err;
 	}
 
-	so->batch_offset = so->vma->node.start;
+	so->batch_offset = i915_ggtt_offset(so->vma);
 	so->batch_size = rodata->batch_items * sizeof(u32);
 
 	while (i % CACHELINE_DWORDS)
@@ -160,9 +164,9 @@ static int render_state_setup(struct intel_render_state *so,
 		drm_clflush_virt_range(d, i * sizeof(u32));
 	kunmap_atomic(d);
 
-	ret = i915_gem_object_set_to_gtt_domain(obj, false);
+	ret = i915_gem_object_set_to_gtt_domain(so->obj, false);
 out:
-	i915_gem_obj_finish_shmem_access(obj);
+	i915_gem_obj_finish_shmem_access(so->obj);
 	return ret;
 
 err:
@@ -173,112 +177,57 @@ static int render_state_setup(struct intel_render_state *so,
 
 #undef OUT_BATCH
 
-int i915_gem_render_state_init(struct intel_engine_cs *engine)
+int i915_gem_render_state_emit(struct drm_i915_gem_request *rq)
 {
-	struct intel_render_state *so;
-	const struct intel_renderstate_rodata *rodata;
-	struct drm_i915_gem_object *obj;
-	int ret;
+	struct intel_engine_cs *engine = rq->engine;
+	struct intel_render_state so = {}; /* keep the compiler happy */
+	int err;
 
-	if (engine->id != RCS)
+	so.rodata = render_state_get_rodata(engine);
+	if (!so.rodata)
 		return 0;
 
-	rodata = render_state_get_rodata(engine);
-	if (!rodata)
-		return 0;
-
-	if (rodata->batch_items * 4 > PAGE_SIZE)
+	if (so.rodata->batch_items * 4 > PAGE_SIZE)
 		return -EINVAL;
 
-	so = kmalloc(sizeof(*so), GFP_KERNEL);
-	if (!so)
-		return -ENOMEM;
-
-	obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
-	if (IS_ERR(obj)) {
-		ret = PTR_ERR(obj);
-		goto err_free;
-	}
+	so.obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
+	if (IS_ERR(so.obj))
+		return PTR_ERR(so.obj);
 
-	so->vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
-	if (IS_ERR(so->vma)) {
-		ret = PTR_ERR(so->vma);
+	so.vma = i915_vma_instance(so.obj, &engine->i915->ggtt.base, NULL);
+	if (IS_ERR(so.vma)) {
+		err = PTR_ERR(so.vma);
 		goto err_obj;
 	}
 
-	so->rodata = rodata;
-	engine->render_state = so;
-	return 0;
-
-err_obj:
-	i915_gem_object_put(obj);
-err_free:
-	kfree(so);
-	return ret;
-}
-
-int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
-{
-	struct intel_render_state *so;
-	int ret;
-
-	lockdep_assert_held(&req->i915->drm.struct_mutex);
-
-	so = req->engine->render_state;
-	if (!so)
-		return 0;
-
-	/* Recreate the page after shrinking */
-	if (!i915_gem_object_has_pages(so->vma->obj))
-		so->batch_offset = -1;
-
-	ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
-	if (ret)
-		return ret;
+	err = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
+	if (err)
+		goto err_vma;
 
-	if (so->vma->node.start != so->batch_offset) {
-		ret = render_state_setup(so, req->i915);
-		if (ret)
-			goto err_unpin;
-	}
-
-	ret = req->engine->emit_flush(req, EMIT_INVALIDATE);
-	if (ret)
+	err = render_state_setup(&so, rq->i915);
+	if (err)
 		goto err_unpin;
 
-	ret = req->engine->emit_bb_start(req,
-					 so->batch_offset, so->batch_size,
-					 I915_DISPATCH_SECURE);
-	if (ret)
+	err = engine->emit_bb_start(rq,
+				    so.batch_offset, so.batch_size,
+				    I915_DISPATCH_SECURE);
+	if (err)
 		goto err_unpin;
 
-	if (so->aux_size > 8) {
-		ret = req->engine->emit_bb_start(req,
-						 so->aux_offset, so->aux_size,
-						 I915_DISPATCH_SECURE);
-		if (ret)
+	if (so.aux_size > 8) {
+		err = engine->emit_bb_start(rq,
+					    so.aux_offset, so.aux_size,
+					    I915_DISPATCH_SECURE);
+		if (err)
 			goto err_unpin;
 	}
 
-	i915_vma_move_to_active(so->vma, req, 0);
+	i915_vma_move_to_active(so.vma, rq, 0);
 err_unpin:
-	i915_vma_unpin(so->vma);
-	return ret;
-}
-
-void i915_gem_render_state_fini(struct intel_engine_cs *engine)
-{
-	struct intel_render_state *so;
-	struct drm_i915_gem_object *obj;
-
-	so = fetch_and_zero(&engine->render_state);
-	if (!so)
-		return;
-
-	obj = so->vma->obj;
-
-	i915_vma_close(so->vma);
-	__i915_gem_object_release_unless_active(obj);
-
-	kfree(so);
+	i915_vma_unpin(so.vma);
+err_vma:
+	i915_vma_close(so.vma);
+err_obj:
+	__i915_gem_object_release_unless_active(so.obj);
+	return err;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h
index 87481845799d..86369520482e 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.h
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.h
@@ -26,8 +26,6 @@
 
 struct drm_i915_gem_request;
 
-int i915_gem_render_state_init(struct intel_engine_cs *engine);
-int i915_gem_render_state_emit(struct drm_i915_gem_request *req);
-void i915_gem_render_state_fini(struct intel_engine_cs *engine);
+int i915_gem_render_state_emit(struct drm_i915_gem_request *rq);
 
 #endif /* _I915_GEM_RENDER_STATE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index d140fcf5c6a3..a3e93d46316a 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -252,12 +252,31 @@ static void mark_busy(struct drm_i915_private *i915)
 	GEM_BUG_ON(!i915->gt.active_requests);
 
 	intel_runtime_pm_get_noresume(i915);
+
+	/*
+	 * It seems that the DMC likes to transition between the DC states a lot
+	 * when there are no connected displays (no active power domains) during
+	 * command submission.
+	 *
+	 * This activity has negative impact on the performance of the chip with
+	 * huge latencies observed in the interrupt handler and elsewhere.
+	 *
+	 * Work around it by grabbing a GT IRQ power domain whilst there is any
+	 * GT activity, preventing any DC state transitions.
+	 */
+	intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
+
 	i915->gt.awake = true;
 
 	intel_enable_gt_powersave(i915);
 	i915_update_gfx_val(i915);
 	if (INTEL_GEN(i915) >= 6)
 		gen6_rps_busy(i915);
+	i915_pmu_gt_unparked(i915);
+
+	intel_engines_unpark(i915);
+
+	i915_queue_hangcheck(i915);
 
 	queue_delayed_work(i915->wq,
 			   &i915->gt.retire_work,
@@ -457,11 +476,10 @@ void __i915_gem_request_submit(struct drm_i915_gem_request *request)
 	GEM_BUG_ON(!irqs_disabled());
 	lockdep_assert_held(&engine->timeline->lock);
 
-	trace_i915_gem_request_execute(request);
-
 	/* Transfer from per-context onto the global per-engine timeline */
 	timeline = engine->timeline;
 	GEM_BUG_ON(timeline == request->timeline);
+	GEM_BUG_ON(request->global_seqno);
 
 	seqno = timeline_get_seqno(timeline);
 	GEM_BUG_ON(!seqno);
@@ -481,6 +499,8 @@ void __i915_gem_request_submit(struct drm_i915_gem_request *request)
 	list_move_tail(&request->link, &timeline->requests);
 	spin_unlock(&request->timeline->lock);
 
+	trace_i915_gem_request_execute(request);
+
 	wake_up_all(&request->execute);
 }
 
@@ -508,6 +528,7 @@ void __i915_gem_request_unsubmit(struct drm_i915_gem_request *request)
 	/* Only unwind in reverse order, required so that the per-context list
 	 * is kept in seqno/ring order.
 	 */
+	GEM_BUG_ON(!request->global_seqno);
 	GEM_BUG_ON(request->global_seqno != engine->timeline->seqno);
 	engine->timeline->seqno--;
 
@@ -622,6 +643,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	if (ret)
 		goto err_unpin;
 
+	ret = intel_ring_wait_for_space(ring, MIN_SPACE_FOR_ADD_REQUEST);
+	if (ret)
+		goto err_unreserve;
+
 	/* Move the oldest request to the slab-cache (if not in use!) */
 	req = list_first_entry_or_null(&engine->timeline->requests,
 				       typeof(*req), link);
@@ -656,10 +681,21 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	 *
 	 * Do not use kmem_cache_zalloc() here!
 	 */
-	req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL);
-	if (!req) {
-		ret = -ENOMEM;
-		goto err_unreserve;
+	req = kmem_cache_alloc(dev_priv->requests,
+			       GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
+	if (unlikely(!req)) {
+		/* Ratelimit ourselves to prevent oom from malicious clients */
+		ret = i915_gem_wait_for_idle(dev_priv,
+					     I915_WAIT_LOCKED |
+					     I915_WAIT_INTERRUPTIBLE);
+		if (ret)
+			goto err_unreserve;
+
+		req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL);
+		if (!req) {
+			ret = -ENOMEM;
+			goto err_unreserve;
+		}
 	}
 
 	req->timeline = i915_gem_context_lookup_timeline(ctx, engine);
@@ -701,22 +737,30 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
 	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
 	GEM_BUG_ON(req->reserved_space < engine->emit_breadcrumb_sz);
 
-	ret = engine->request_alloc(req);
-	if (ret)
-		goto err_ctx;
-
-	/* Record the position of the start of the request so that
+	/*
+	 * Record the position of the start of the request so that
 	 * should we detect the updated seqno part-way through the
 	 * GPU processing the request, we never over-estimate the
 	 * position of the head.
 	 */
 	req->head = req->ring->emit;
 
+	/* Unconditionally invalidate GPU caches and TLBs. */
+	ret = engine->emit_flush(req, EMIT_INVALIDATE);
+	if (ret)
+		goto err_unwind;
+
+	ret = engine->request_alloc(req);
+	if (ret)
+		goto err_unwind;
+
 	/* Check that we didn't interrupt ourselves with a new request */
 	GEM_BUG_ON(req->timeline->seqno != req->fence.seqno);
 	return req;
 
-err_ctx:
+err_unwind:
+	req->ring->emit = req->head;
+
 	/* Make sure we didn't add ourselves to external state before freeing */
 	GEM_BUG_ON(!list_empty(&req->active_list));
 	GEM_BUG_ON(!list_empty(&req->priotree.signalers_list));
@@ -753,7 +797,7 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
 	if (to->engine == from->engine) {
 		ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
 						       &from->submit,
-						       GFP_KERNEL);
+						       I915_FENCE_GFP);
 		return ret < 0 ? ret : 0;
 	}
 
@@ -781,7 +825,7 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
 await_dma_fence:
 	ret = i915_sw_fence_await_dma_fence(&to->submit,
 					    &from->fence, 0,
-					    GFP_KERNEL);
+					    I915_FENCE_GFP);
 	return ret < 0 ? ret : 0;
 }
 
@@ -832,7 +876,7 @@ i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
 		else
 			ret = i915_sw_fence_await_dma_fence(&req->submit, fence,
 							    I915_FENCE_TIMEOUT,
-							    GFP_KERNEL);
+							    I915_FENCE_GFP);
 		if (ret < 0)
 			return ret;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h
index 26249f39de67..0d6d39f19506 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.h
+++ b/drivers/gpu/drm/i915/i915_gem_request.h
@@ -204,6 +204,8 @@ struct drm_i915_gem_request {
 	struct list_head client_link;
 };
 
+#define I915_FENCE_GFP (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN)
+
 extern const struct dma_fence_ops i915_fence_ops;
 
 static inline bool dma_fence_is_i915(const struct dma_fence *fence)
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 3770e3323fc8..0e158f9287c4 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -35,9 +35,9 @@
 #include "i915_drv.h"
 #include "i915_trace.h"
 
-static bool shrinker_lock(struct drm_i915_private *dev_priv, bool *unlock)
+static bool shrinker_lock(struct drm_i915_private *i915, bool *unlock)
 {
-	switch (mutex_trylock_recursive(&dev_priv->drm.struct_mutex)) {
+	switch (mutex_trylock_recursive(&i915->drm.struct_mutex)) {
 	case MUTEX_TRYLOCK_RECURSIVE:
 		*unlock = false;
 		return true;
@@ -47,7 +47,7 @@ static bool shrinker_lock(struct drm_i915_private *dev_priv, bool *unlock)
 		preempt_disable();
 		do {
 			cpu_relax();
-			if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
+			if (mutex_trylock(&i915->drm.struct_mutex)) {
 				*unlock = true;
 				break;
 			}
@@ -63,12 +63,12 @@ static bool shrinker_lock(struct drm_i915_private *dev_priv, bool *unlock)
 	BUG();
 }
 
-static void shrinker_unlock(struct drm_i915_private *dev_priv, bool unlock)
+static void shrinker_unlock(struct drm_i915_private *i915, bool unlock)
 {
 	if (!unlock)
 		return;
 
-	mutex_unlock(&dev_priv->drm.struct_mutex);
+	mutex_unlock(&i915->drm.struct_mutex);
 }
 
 static bool swap_available(void)
@@ -118,7 +118,7 @@ static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
 
 /**
  * i915_gem_shrink - Shrink buffer object caches
- * @dev_priv: i915 device
+ * @i915: i915 device
  * @target: amount of memory to make available, in pages
  * @nr_scanned: optional output for number of pages scanned (incremental)
  * @flags: control flags for selecting cache types
@@ -142,7 +142,7 @@ static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
  * The number of pages of backing storage actually released.
  */
 unsigned long
-i915_gem_shrink(struct drm_i915_private *dev_priv,
+i915_gem_shrink(struct drm_i915_private *i915,
 		unsigned long target,
 		unsigned long *nr_scanned,
 		unsigned flags)
@@ -151,15 +151,15 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 		struct list_head *list;
 		unsigned int bit;
 	} phases[] = {
-		{ &dev_priv->mm.unbound_list, I915_SHRINK_UNBOUND },
-		{ &dev_priv->mm.bound_list, I915_SHRINK_BOUND },
+		{ &i915->mm.unbound_list, I915_SHRINK_UNBOUND },
+		{ &i915->mm.bound_list, I915_SHRINK_BOUND },
 		{ NULL, 0 },
 	}, *phase;
 	unsigned long count = 0;
 	unsigned long scanned = 0;
 	bool unlock;
 
-	if (!shrinker_lock(dev_priv, &unlock))
+	if (!shrinker_lock(i915, &unlock))
 		return 0;
 
 	/*
@@ -172,10 +172,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 	 * we will free as much as we can and hope to get a second chance.
 	 */
 	if (flags & I915_SHRINK_ACTIVE)
-		i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
+		i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
 
-	trace_i915_gem_shrink(dev_priv, target, flags);
-	i915_gem_retire_requests(dev_priv);
+	trace_i915_gem_shrink(i915, target, flags);
+	i915_gem_retire_requests(i915);
 
 	/*
 	 * Unbinding of objects will require HW access; Let us not wake the
@@ -183,7 +183,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 	 * we will force the wake during oom-notifier.
 	 */
 	if ((flags & I915_SHRINK_BOUND) &&
-	    !intel_runtime_pm_get_if_in_use(dev_priv))
+	    !intel_runtime_pm_get_if_in_use(i915))
 		flags &= ~I915_SHRINK_BOUND;
 
 	/*
@@ -221,7 +221,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 		 * to be able to shrink their pages, so they remain on
 		 * the unbound/bound list until actually freed.
 		 */
-		spin_lock(&dev_priv->mm.obj_lock);
+		spin_lock(&i915->mm.obj_lock);
 		while (count < target &&
 		       (obj = list_first_entry_or_null(phase->list,
 						       typeof(*obj),
@@ -244,7 +244,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 			if (!can_release_pages(obj))
 				continue;
 
-			spin_unlock(&dev_priv->mm.obj_lock);
+			spin_unlock(&i915->mm.obj_lock);
 
 			if (unsafe_drop_pages(obj)) {
 				/* May arrive from get_pages on another bo */
@@ -258,18 +258,18 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 			}
 			scanned += obj->base.size >> PAGE_SHIFT;
 
-			spin_lock(&dev_priv->mm.obj_lock);
+			spin_lock(&i915->mm.obj_lock);
 		}
 		list_splice_tail(&still_in_list, phase->list);
-		spin_unlock(&dev_priv->mm.obj_lock);
+		spin_unlock(&i915->mm.obj_lock);
 	}
 
 	if (flags & I915_SHRINK_BOUND)
-		intel_runtime_pm_put(dev_priv);
+		intel_runtime_pm_put(i915);
 
-	i915_gem_retire_requests(dev_priv);
+	i915_gem_retire_requests(i915);
 
-	shrinker_unlock(dev_priv, unlock);
+	shrinker_unlock(i915, unlock);
 
 	if (nr_scanned)
 		*nr_scanned += scanned;
@@ -278,7 +278,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
 /**
  * i915_gem_shrink_all - Shrink buffer object caches completely
- * @dev_priv: i915 device
+ * @i915: i915 device
  *
  * This is a simple wraper around i915_gem_shrink() to aggressively shrink all
  * caches completely. It also first waits for and retires all outstanding
@@ -290,16 +290,16 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  * Returns:
  * The number of pages of backing storage actually released.
  */
-unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv)
+unsigned long i915_gem_shrink_all(struct drm_i915_private *i915)
 {
 	unsigned long freed;
 
-	intel_runtime_pm_get(dev_priv);
-	freed = i915_gem_shrink(dev_priv, -1UL, NULL,
+	intel_runtime_pm_get(i915);
+	freed = i915_gem_shrink(i915, -1UL, NULL,
 				I915_SHRINK_BOUND |
 				I915_SHRINK_UNBOUND |
 				I915_SHRINK_ACTIVE);
-	intel_runtime_pm_put(dev_priv);
+	intel_runtime_pm_put(i915);
 
 	return freed;
 }
@@ -347,53 +347,53 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 static unsigned long
 i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 {
-	struct drm_i915_private *dev_priv =
+	struct drm_i915_private *i915 =
 		container_of(shrinker, struct drm_i915_private, mm.shrinker);
 	unsigned long freed;
 	bool unlock;
 
 	sc->nr_scanned = 0;
 
-	if (!shrinker_lock(dev_priv, &unlock))
+	if (!shrinker_lock(i915, &unlock))
 		return SHRINK_STOP;
 
-	freed = i915_gem_shrink(dev_priv,
+	freed = i915_gem_shrink(i915,
 				sc->nr_to_scan,
 				&sc->nr_scanned,
 				I915_SHRINK_BOUND |
 				I915_SHRINK_UNBOUND |
 				I915_SHRINK_PURGEABLE);
-	if (freed < sc->nr_to_scan)
-		freed += i915_gem_shrink(dev_priv,
+	if (sc->nr_scanned < sc->nr_to_scan)
+		freed += i915_gem_shrink(i915,
 					 sc->nr_to_scan - sc->nr_scanned,
 					 &sc->nr_scanned,
 					 I915_SHRINK_BOUND |
 					 I915_SHRINK_UNBOUND);
-	if (freed < sc->nr_to_scan && current_is_kswapd()) {
-		intel_runtime_pm_get(dev_priv);
-		freed += i915_gem_shrink(dev_priv,
+	if (sc->nr_scanned < sc->nr_to_scan && current_is_kswapd()) {
+		intel_runtime_pm_get(i915);
+		freed += i915_gem_shrink(i915,
 					 sc->nr_to_scan - sc->nr_scanned,
 					 &sc->nr_scanned,
 					 I915_SHRINK_ACTIVE |
 					 I915_SHRINK_BOUND |
 					 I915_SHRINK_UNBOUND);
-		intel_runtime_pm_put(dev_priv);
+		intel_runtime_pm_put(i915);
 	}
 
-	shrinker_unlock(dev_priv, unlock);
+	shrinker_unlock(i915, unlock);
 
 	return sc->nr_scanned ? freed : SHRINK_STOP;
 }
 
 static bool
-shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, bool *unlock,
+shrinker_lock_uninterruptible(struct drm_i915_private *i915, bool *unlock,
 			      int timeout_ms)
 {
 	unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
 
 	do {
-		if (i915_gem_wait_for_idle(dev_priv, 0) == 0 &&
-		    shrinker_lock(dev_priv, unlock))
+		if (i915_gem_wait_for_idle(i915, 0) == 0 &&
+		    shrinker_lock(i915, unlock))
 			break;
 
 		schedule_timeout_killable(1);
@@ -412,32 +412,32 @@ shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv, bool *unlock,
 static int
 i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
 {
-	struct drm_i915_private *dev_priv =
+	struct drm_i915_private *i915 =
 		container_of(nb, struct drm_i915_private, mm.oom_notifier);
 	struct drm_i915_gem_object *obj;
 	unsigned long unevictable, bound, unbound, freed_pages;
 
-	freed_pages = i915_gem_shrink_all(dev_priv);
+	freed_pages = i915_gem_shrink_all(i915);
 
 	/* Because we may be allocating inside our own driver, we cannot
 	 * assert that there are no objects with pinned pages that are not
 	 * being pointed to by hardware.
 	 */
 	unbound = bound = unevictable = 0;
-	spin_lock(&dev_priv->mm.obj_lock);
-	list_for_each_entry(obj, &dev_priv->mm.unbound_list, mm.link) {
+	spin_lock(&i915->mm.obj_lock);
+	list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) {
 		if (!can_release_pages(obj))
 			unevictable += obj->base.size >> PAGE_SHIFT;
 		else
 			unbound += obj->base.size >> PAGE_SHIFT;
 	}
-	list_for_each_entry(obj, &dev_priv->mm.bound_list, mm.link) {
+	list_for_each_entry(obj, &i915->mm.bound_list, mm.link) {
 		if (!can_release_pages(obj))
 			unevictable += obj->base.size >> PAGE_SHIFT;
 		else
 			bound += obj->base.size >> PAGE_SHIFT;
 	}
-	spin_unlock(&dev_priv->mm.obj_lock);
+	spin_unlock(&i915->mm.obj_lock);
 
 	if (freed_pages || unbound || bound)
 		pr_info("Purging GPU memory, %lu pages freed, "
@@ -455,74 +455,74 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
 static int
 i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
 {
-	struct drm_i915_private *dev_priv =
+	struct drm_i915_private *i915 =
 		container_of(nb, struct drm_i915_private, mm.vmap_notifier);
 	struct i915_vma *vma, *next;
 	unsigned long freed_pages = 0;
 	bool unlock;
 	int ret;
 
-	if (!shrinker_lock_uninterruptible(dev_priv, &unlock, 5000))
+	if (!shrinker_lock_uninterruptible(i915, &unlock, 5000))
 		return NOTIFY_DONE;
 
 	/* Force everything onto the inactive lists */
-	ret = i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
+	ret = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
 	if (ret)
 		goto out;
 
-	intel_runtime_pm_get(dev_priv);
-	freed_pages += i915_gem_shrink(dev_priv, -1UL, NULL,
+	intel_runtime_pm_get(i915);
+	freed_pages += i915_gem_shrink(i915, -1UL, NULL,
 				       I915_SHRINK_BOUND |
 				       I915_SHRINK_UNBOUND |
 				       I915_SHRINK_ACTIVE |
 				       I915_SHRINK_VMAPS);
-	intel_runtime_pm_put(dev_priv);
+	intel_runtime_pm_put(i915);
 
 	/* We also want to clear any cached iomaps as they wrap vmap */
 	list_for_each_entry_safe(vma, next,
-				 &dev_priv->ggtt.base.inactive_list, vm_link) {
+				 &i915->ggtt.base.inactive_list, vm_link) {
 		unsigned long count = vma->node.size >> PAGE_SHIFT;
 		if (vma->iomap && i915_vma_unbind(vma) == 0)
 			freed_pages += count;
 	}
 
 out:
-	shrinker_unlock(dev_priv, unlock);
+	shrinker_unlock(i915, unlock);
 
 	*(unsigned long *)ptr += freed_pages;
 	return NOTIFY_DONE;
 }
 
 /**
- * i915_gem_shrinker_init - Initialize i915 shrinker
- * @dev_priv: i915 device
+ * i915_gem_shrinker_register - Register the i915 shrinker
+ * @i915: i915 device
  *
  * This function registers and sets up the i915 shrinker and OOM handler.
  */
-void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
+void i915_gem_shrinker_register(struct drm_i915_private *i915)
 {
-	dev_priv->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
-	dev_priv->mm.shrinker.count_objects = i915_gem_shrinker_count;
-	dev_priv->mm.shrinker.seeks = DEFAULT_SEEKS;
-	dev_priv->mm.shrinker.batch = 4096;
-	WARN_ON(register_shrinker(&dev_priv->mm.shrinker));
+	i915->mm.shrinker.scan_objects = i915_gem_shrinker_scan;
+	i915->mm.shrinker.count_objects = i915_gem_shrinker_count;
+	i915->mm.shrinker.seeks = DEFAULT_SEEKS;
+	i915->mm.shrinker.batch = 4096;
+	WARN_ON(register_shrinker(&i915->mm.shrinker));
 
-	dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
-	WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
+	i915->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
+	WARN_ON(register_oom_notifier(&i915->mm.oom_notifier));
 
-	dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
-	WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
+	i915->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
+	WARN_ON(register_vmap_purge_notifier(&i915->mm.vmap_notifier));
 }
 
 /**
- * i915_gem_shrinker_cleanup - Clean up i915 shrinker
- * @dev_priv: i915 device
+ * i915_gem_shrinker_unregister - Unregisters the i915 shrinker
+ * @i915: i915 device
  *
  * This function unregisters the i915 shrinker and OOM handler.
  */
-void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
+void i915_gem_shrinker_unregister(struct drm_i915_private *i915)
 {
-	WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
-	WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
-	unregister_shrinker(&dev_priv->mm.shrinker);
+	WARN_ON(unregister_vmap_purge_notifier(&i915->mm.vmap_notifier));
+	WARN_ON(unregister_oom_notifier(&i915->mm.oom_notifier));
+	unregister_shrinker(&i915->mm.shrinker);
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 03e7abc7e043..d3f222fa6356 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -30,9 +30,6 @@
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 
-#define KB(x) ((x) * 1024)
-#define MB(x) (KB(x) * 1024)
-
 /*
  * The BIOS typically reserves some of the system's memory for the exclusive
  * use of the integrated graphics. This memory is no longer available for
@@ -79,129 +76,26 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv,
 	mutex_unlock(&dev_priv->mm.stolen_lock);
 }
 
-static dma_addr_t i915_stolen_to_dma(struct drm_i915_private *dev_priv)
+static int i915_adjust_stolen(struct drm_i915_private *dev_priv,
+			      struct resource *dsm)
 {
-	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct resource *r;
-	dma_addr_t base;
-
-	/* Almost universally we can find the Graphics Base of Stolen Memory
-	 * at register BSM (0x5c) in the igfx configuration space. On a few
-	 * (desktop) machines this is also mirrored in the bridge device at
-	 * different locations, or in the MCHBAR.
-	 *
-	 * On 865 we just check the TOUD register.
-	 *
-	 * On 830/845/85x the stolen memory base isn't available in any
-	 * register. We need to calculate it as TOM-TSEG_SIZE-stolen_size.
-	 *
-	 */
-	base = 0;
-	if (INTEL_GEN(dev_priv) >= 3) {
-		u32 bsm;
-
-		pci_read_config_dword(pdev, INTEL_BSM, &bsm);
-
-		base = bsm & INTEL_BSM_MASK;
-	} else if (IS_I865G(dev_priv)) {
-		u32 tseg_size = 0;
-		u16 toud = 0;
-		u8 tmp;
-
-		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
-					 I845_ESMRAMC, &tmp);
-
-		if (tmp & TSEG_ENABLE) {
-			switch (tmp & I845_TSEG_SIZE_MASK) {
-			case I845_TSEG_SIZE_512K:
-				tseg_size = KB(512);
-				break;
-			case I845_TSEG_SIZE_1M:
-				tseg_size = MB(1);
-				break;
-			}
-		}
-
-		pci_bus_read_config_word(pdev->bus, PCI_DEVFN(0, 0),
-					 I865_TOUD, &toud);
-
-		base = (toud << 16) + tseg_size;
-	} else if (IS_I85X(dev_priv)) {
-		u32 tseg_size = 0;
-		u32 tom;
-		u8 tmp;
-
-		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
-					 I85X_ESMRAMC, &tmp);
-
-		if (tmp & TSEG_ENABLE)
-			tseg_size = MB(1);
-
-		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 1),
-					 I85X_DRB3, &tmp);
-		tom = tmp * MB(32);
-
-		base = tom - tseg_size - ggtt->stolen_size;
-	} else if (IS_I845G(dev_priv)) {
-		u32 tseg_size = 0;
-		u32 tom;
-		u8 tmp;
-
-		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
-					 I845_ESMRAMC, &tmp);
-
-		if (tmp & TSEG_ENABLE) {
-			switch (tmp & I845_TSEG_SIZE_MASK) {
-			case I845_TSEG_SIZE_512K:
-				tseg_size = KB(512);
-				break;
-			case I845_TSEG_SIZE_1M:
-				tseg_size = MB(1);
-				break;
-			}
-		}
-
-		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
-					 I830_DRB3, &tmp);
-		tom = tmp * MB(32);
-
-		base = tom - tseg_size - ggtt->stolen_size;
-	} else if (IS_I830(dev_priv)) {
-		u32 tseg_size = 0;
-		u32 tom;
-		u8 tmp;
 
-		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
-					 I830_ESMRAMC, &tmp);
+	if (dsm->start == 0 || dsm->end <= dsm->start)
+		return -EINVAL;
 
-		if (tmp & TSEG_ENABLE) {
-			if (tmp & I830_TSEG_SIZE_1M)
-				tseg_size = MB(1);
-			else
-				tseg_size = KB(512);
-		}
-
-		pci_bus_read_config_byte(pdev->bus, PCI_DEVFN(0, 0),
-					 I830_DRB3, &tmp);
-		tom = tmp * MB(32);
-
-		base = tom - tseg_size - ggtt->stolen_size;
-	}
-
-	if (base == 0 || add_overflows(base, ggtt->stolen_size))
-		return 0;
+	/*
+	 * TODO: We have yet too encounter the case where the GTT wasn't at the
+	 * end of stolen. With that assumption we could simplify this.
+	 */
 
-	/* make sure we don't clobber the GTT if it's within stolen memory */
+	/* Make sure we don't clobber the GTT if it's within stolen memory */
 	if (INTEL_GEN(dev_priv) <= 4 &&
 	    !IS_G33(dev_priv) && !IS_PINEVIEW(dev_priv) && !IS_G4X(dev_priv)) {
-		struct {
-			dma_addr_t start, end;
-		} stolen[2] = {
-			{ .start = base, .end = base + ggtt->stolen_size, },
-			{ .start = base, .end = base + ggtt->stolen_size, },
-		};
-		u64 ggtt_start, ggtt_end;
+		struct resource stolen[2] = {*dsm, *dsm};
+		struct resource ggtt_res;
+		resource_size_t ggtt_start;
 
 		ggtt_start = I915_READ(PGTBL_CTL);
 		if (IS_GEN4(dev_priv))
@@ -209,70 +103,64 @@ static dma_addr_t i915_stolen_to_dma(struct drm_i915_private *dev_priv)
 				     (ggtt_start & PGTBL_ADDRESS_HI_MASK) << 28;
 		else
 			ggtt_start &= PGTBL_ADDRESS_LO_MASK;
-		ggtt_end = ggtt_start + ggtt_total_entries(ggtt) * 4;
-
-		if (ggtt_start >= stolen[0].start && ggtt_start < stolen[0].end)
-			stolen[0].end = ggtt_start;
-		if (ggtt_end > stolen[1].start && ggtt_end <= stolen[1].end)
-			stolen[1].start = ggtt_end;
-
-		/* pick the larger of the two chunks */
-		if (stolen[0].end - stolen[0].start >
-		    stolen[1].end - stolen[1].start) {
-			base = stolen[0].start;
-			ggtt->stolen_size = stolen[0].end - stolen[0].start;
-		} else {
-			base = stolen[1].start;
-			ggtt->stolen_size = stolen[1].end - stolen[1].start;
-		}
+
+		ggtt_res =
+			(struct resource) DEFINE_RES_MEM(ggtt_start,
+							 ggtt_total_entries(ggtt) * 4);
+
+		if (ggtt_res.start >= stolen[0].start && ggtt_res.start < stolen[0].end)
+			stolen[0].end = ggtt_res.start;
+		if (ggtt_res.end > stolen[1].start && ggtt_res.end <= stolen[1].end)
+			stolen[1].start = ggtt_res.end;
+
+		/* Pick the larger of the two chunks */
+		if (resource_size(&stolen[0]) > resource_size(&stolen[1]))
+			*dsm = stolen[0];
+		else
+			*dsm = stolen[1];
 
 		if (stolen[0].start != stolen[1].start ||
 		    stolen[0].end != stolen[1].end) {
-			dma_addr_t end = base + ggtt->stolen_size - 1;
-
-			DRM_DEBUG_KMS("GTT within stolen memory at 0x%llx-0x%llx\n",
-				      (unsigned long long)ggtt_start,
-				      (unsigned long long)ggtt_end - 1);
-			DRM_DEBUG_KMS("Stolen memory adjusted to %pad - %pad\n",
-				      &base, &end);
+			DRM_DEBUG_KMS("GTT within stolen memory at %pR\n", &ggtt_res);
+			DRM_DEBUG_KMS("Stolen memory adjusted to %pR\n", dsm);
 		}
 	}
 
-
-	/* Verify that nothing else uses this physical address. Stolen
+	/*
+	 * Verify that nothing else uses this physical address. Stolen
 	 * memory should be reserved by the BIOS and hidden from the
 	 * kernel. So if the region is already marked as busy, something
 	 * is seriously wrong.
 	 */
-	r = devm_request_mem_region(dev_priv->drm.dev, base, ggtt->stolen_size,
+	r = devm_request_mem_region(dev_priv->drm.dev, dsm->start,
+				    resource_size(dsm),
 				    "Graphics Stolen Memory");
 	if (r == NULL) {
 		/*
 		 * One more attempt but this time requesting region from
-		 * base + 1, as we have seen that this resolves the region
+		 * start + 1, as we have seen that this resolves the region
 		 * conflict with the PCI Bus.
 		 * This is a BIOS w/a: Some BIOS wrap stolen in the root
 		 * PCI bus, but have an off-by-one error. Hence retry the
 		 * reservation starting from 1 instead of 0.
 		 * There's also BIOS with off-by-one on the other end.
 		 */
-		r = devm_request_mem_region(dev_priv->drm.dev, base + 1,
-					    ggtt->stolen_size - 2,
+		r = devm_request_mem_region(dev_priv->drm.dev, dsm->start + 1,
+					    resource_size(dsm) - 2,
 					    "Graphics Stolen Memory");
 		/*
 		 * GEN3 firmware likes to smash pci bridges into the stolen
 		 * range. Apparently this works.
 		 */
 		if (r == NULL && !IS_GEN3(dev_priv)) {
-			dma_addr_t end = base + ggtt->stolen_size;
+			DRM_ERROR("conflict detected with stolen region: %pR\n",
+				  dsm);
 
-			DRM_ERROR("conflict detected with stolen region: [%pad - %pad]\n",
-				  &base, &end);
-			base = 0;
+			return -EBUSY;
 		}
 	}
 
-	return base;
+	return 0;
 }
 
 void i915_gem_cleanup_stolen(struct drm_device *dev)
@@ -286,13 +174,24 @@ void i915_gem_cleanup_stolen(struct drm_device *dev)
 }
 
 static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				    dma_addr_t *base, u32 *size)
+				    resource_size_t *base, resource_size_t *size)
 {
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	uint32_t reg_val = I915_READ(IS_GM45(dev_priv) ?
 				     CTG_STOLEN_RESERVED :
 				     ELK_STOLEN_RESERVED);
-	dma_addr_t stolen_top = dev_priv->mm.stolen_base + ggtt->stolen_size;
+	resource_size_t stolen_top = dev_priv->dsm.end + 1;
+
+	if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0) {
+		*base = 0;
+		*size = 0;
+		return;
+	}
+
+	/*
+	 * Whether ILK really reuses the ELK register for this is unclear.
+	 * Let's see if we catch anyone with this supposedly enabled on ILK.
+	 */
+	WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val);
 
 	*base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 
@@ -309,10 +208,16 @@ static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				     dma_addr_t *base, u32 *size)
+				     resource_size_t *base, resource_size_t *size)
 {
 	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
+	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+		*base = 0;
+		*size = 0;
+		return;
+	}
+
 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
 	switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
@@ -335,10 +240,16 @@ static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				     dma_addr_t *base, u32 *size)
+				     resource_size_t *base, resource_size_t *size)
 {
 	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
+	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+		*base = 0;
+		*size = 0;
+		return;
+	}
+
 	*base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 
 	switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
@@ -355,10 +266,16 @@ static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				    dma_addr_t *base, u32 *size)
+				    resource_size_t *base, resource_size_t *size)
 {
 	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
+	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+		*base = 0;
+		*size = 0;
+		return;
+	}
+
 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
 	switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
@@ -381,13 +298,18 @@ static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 }
 
 static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
-				    dma_addr_t *base, u32 *size)
+				    resource_size_t *base, resource_size_t *size)
 {
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
-	dma_addr_t stolen_top;
+	resource_size_t stolen_top;
 
-	stolen_top = dev_priv->mm.stolen_base + ggtt->stolen_size;
+	if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+		*base = 0;
+		*size = 0;
+		return;
+	}
+
+	stolen_top = dev_priv->dsm.end + 1;
 
 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
@@ -403,10 +325,9 @@ static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
 
 int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 {
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	dma_addr_t reserved_base, stolen_top;
-	u32 reserved_total, reserved_size;
-	u32 stolen_usable_start;
+	resource_size_t reserved_base, stolen_top;
+	resource_size_t reserved_total, reserved_size;
+	resource_size_t stolen_usable_start;
 
 	mutex_init(&dev_priv->mm.stolen_lock);
 
@@ -420,14 +341,18 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 		return 0;
 	}
 
-	if (ggtt->stolen_size == 0)
+	if (resource_size(&intel_graphics_stolen_res) == 0)
 		return 0;
 
-	dev_priv->mm.stolen_base = i915_stolen_to_dma(dev_priv);
-	if (dev_priv->mm.stolen_base == 0)
+	dev_priv->dsm = intel_graphics_stolen_res;
+
+	if (i915_adjust_stolen(dev_priv, &dev_priv->dsm))
 		return 0;
 
-	stolen_top = dev_priv->mm.stolen_base + ggtt->stolen_size;
+	GEM_BUG_ON(dev_priv->dsm.start == 0);
+	GEM_BUG_ON(dev_priv->dsm.end <= dev_priv->dsm.start);
+
+	stolen_top = dev_priv->dsm.end + 1;
 	reserved_base = 0;
 	reserved_size = 0;
 
@@ -436,14 +361,12 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 	case 3:
 		break;
 	case 4:
-		if (IS_G4X(dev_priv))
-			g4x_get_stolen_reserved(dev_priv,
-						&reserved_base, &reserved_size);
-		break;
+		if (!IS_G4X(dev_priv))
+			break;
+		/* fall through */
 	case 5:
-		/* Assume the gen6 maximum for the older platforms. */
-		reserved_size = 1024 * 1024;
-		reserved_base = stolen_top - reserved_size;
+		g4x_get_stolen_reserved(dev_priv,
+					&reserved_base, &reserved_size);
 		break;
 	case 6:
 		gen6_get_stolen_reserved(dev_priv,
@@ -470,50 +393,47 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
 		reserved_base = stolen_top;
 	}
 
-	if (reserved_base < dev_priv->mm.stolen_base ||
-	    reserved_base + reserved_size > stolen_top) {
-		dma_addr_t reserved_top = reserved_base + reserved_size;
-		DRM_DEBUG_KMS("Stolen reserved area [%pad - %pad] outside stolen memory [%pad - %pad]\n",
-			      &reserved_base, &reserved_top,
-			      &dev_priv->mm.stolen_base, &stolen_top);
+	dev_priv->dsm_reserved =
+		(struct resource) DEFINE_RES_MEM(reserved_base, reserved_size);
+
+	if (!resource_contains(&dev_priv->dsm, &dev_priv->dsm_reserved)) {
+		DRM_ERROR("Stolen reserved area %pR outside stolen memory %pR\n",
+			  &dev_priv->dsm_reserved, &dev_priv->dsm);
 		return 0;
 	}
 
-	ggtt->stolen_reserved_base = reserved_base;
-	ggtt->stolen_reserved_size = reserved_size;
-
 	/* It is possible for the reserved area to end before the end of stolen
 	 * memory, so just consider the start. */
 	reserved_total = stolen_top - reserved_base;
 
-	DRM_DEBUG_KMS("Memory reserved for graphics device: %uK, usable: %uK\n",
-		      ggtt->stolen_size >> 10,
-		      (ggtt->stolen_size - reserved_total) >> 10);
+	DRM_DEBUG_KMS("Memory reserved for graphics device: %lluK, usable: %lluK\n",
+		      (u64)resource_size(&dev_priv->dsm) >> 10,
+		      ((u64)resource_size(&dev_priv->dsm) - reserved_total) >> 10);
 
 	stolen_usable_start = 0;
 	/* WaSkipStolenMemoryFirstPage:bdw+ */
 	if (INTEL_GEN(dev_priv) >= 8)
 		stolen_usable_start = 4096;
 
-	ggtt->stolen_usable_size =
-		ggtt->stolen_size - reserved_total - stolen_usable_start;
+	dev_priv->stolen_usable_size =
+		resource_size(&dev_priv->dsm) - reserved_total - stolen_usable_start;
 
 	/* Basic memrange allocator for stolen space. */
 	drm_mm_init(&dev_priv->mm.stolen, stolen_usable_start,
-		    ggtt->stolen_usable_size);
+		    dev_priv->stolen_usable_size);
 
 	return 0;
 }
 
 static struct sg_table *
 i915_pages_create_for_stolen(struct drm_device *dev,
-			     u32 offset, u32 size)
+			     resource_size_t offset, resource_size_t size)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct sg_table *st;
 	struct scatterlist *sg;
 
-	GEM_BUG_ON(range_overflows(offset, size, dev_priv->ggtt.stolen_size));
+	GEM_BUG_ON(range_overflows(offset, size, resource_size(&dev_priv->dsm)));
 
 	/* We hide that we have no struct page backing our stolen object
 	 * by wrapping the contiguous physical allocation with a fake
@@ -533,7 +453,7 @@ i915_pages_create_for_stolen(struct drm_device *dev,
 	sg->offset = 0;
 	sg->length = size;
 
-	sg_dma_address(sg) = (dma_addr_t)dev_priv->mm.stolen_base + offset;
+	sg_dma_address(sg) = (dma_addr_t)dev_priv->dsm.start + offset;
 	sg_dma_len(sg) = size;
 
 	return st;
@@ -611,7 +531,8 @@ _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 }
 
 struct drm_i915_gem_object *
-i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, u32 size)
+i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
+			      resource_size_t size)
 {
 	struct drm_i915_gem_object *obj;
 	struct drm_mm_node *stolen;
@@ -644,9 +565,9 @@ i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, u32 size)
 
 struct drm_i915_gem_object *
 i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv,
-					       u32 stolen_offset,
-					       u32 gtt_offset,
-					       u32 size)
+					       resource_size_t stolen_offset,
+					       resource_size_t gtt_offset,
+					       resource_size_t size)
 {
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct drm_i915_gem_object *obj;
@@ -659,8 +580,8 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-	DRM_DEBUG_KMS("creating preallocated stolen object: stolen_offset=%x, gtt_offset=%x, size=%x\n",
-			stolen_offset, gtt_offset, size);
+	DRM_DEBUG_KMS("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n",
+			&stolen_offset, &gtt_offset, &size);
 
 	/* KISS and expect everything to be page-aligned */
 	if (WARN_ON(size == 0) ||
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 1294cf695df0..d9dc9df523b5 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -205,10 +205,7 @@ i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
 	if (tiling_mode == I915_TILING_NONE)
 		return 0;
 
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (!i915_vma_is_ggtt(vma))
-			break;
-
+	for_each_ggtt_vma(vma, obj) {
 		if (i915_vma_fence_prepare(vma, tiling_mode, stride))
 			continue;
 
@@ -285,10 +282,7 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
 	}
 	mutex_unlock(&obj->mm.lock);
 
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (!i915_vma_is_ggtt(vma))
-			break;
-
+	for_each_ggtt_vma(vma, obj) {
 		vma->fence_size =
 			i915_gem_fence_size(i915, vma->size, tiling, stride);
 		vma->fence_alignment =
@@ -345,6 +339,15 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
 	if (!obj)
 		return -ENOENT;
 
+	/*
+	 * The tiling mode of proxy objects is handled by its generator, and
+	 * not allowed to be changed by userspace.
+	 */
+	if (i915_gem_object_is_proxy(obj)) {
+		err = -ENXIO;
+		goto err;
+	}
+
 	if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
 		err = -EINVAL;
 		goto err;
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c
index c597ce277a04..e9fd87604067 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.c
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.c
@@ -33,11 +33,8 @@ static void __intel_timeline_init(struct intel_timeline *tl,
 {
 	tl->fence_context = context;
 	tl->common = parent;
-#ifdef CONFIG_DEBUG_SPINLOCK
-	__raw_spin_lock_init(&tl->lock.rlock, lockname, lockclass);
-#else
 	spin_lock_init(&tl->lock);
-#endif
+	lockdep_set_class_and_name(&tl->lock, lockclass, lockname);
 	init_request_active(&tl->last_request, NULL);
 	INIT_LIST_HEAD(&tl->requests);
 	i915_syncmap_init(&tl->sync);
@@ -107,8 +104,8 @@ int i915_gem_timeline_init__global(struct drm_i915_private *i915)
 }
 
 /**
- * i915_gem_timelines_mark_idle -- called when the driver idles
- * @i915 - the drm_i915_private device
+ * i915_gem_timelines_park - called when the driver idles
+ * @i915: the drm_i915_private device
  *
  * When the driver is completely idle, we know that all of our sync points
  * have been signaled and our tracking is then entirely redundant. Any request
@@ -116,7 +113,7 @@ int i915_gem_timeline_init__global(struct drm_i915_private *i915)
  * the fence is signaled and therefore we will not even look them up in the
  * sync point map.
  */
-void i915_gem_timelines_mark_idle(struct drm_i915_private *i915)
+void i915_gem_timelines_park(struct drm_i915_private *i915)
 {
 	struct i915_gem_timeline *timeline;
 	int i;
diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h
index bfb5eb94c64d..b5a22400a01f 100644
--- a/drivers/gpu/drm/i915/i915_gem_timeline.h
+++ b/drivers/gpu/drm/i915/i915_gem_timeline.h
@@ -93,7 +93,7 @@ int i915_gem_timeline_init(struct drm_i915_private *i915,
 			   struct i915_gem_timeline *tl,
 			   const char *name);
 int i915_gem_timeline_init__global(struct drm_i915_private *i915);
-void i915_gem_timelines_mark_idle(struct drm_i915_private *i915);
+void i915_gem_timelines_park(struct drm_i915_private *i915);
 void i915_gem_timeline_fini(struct i915_gem_timeline *tl);
 
 static inline int __intel_timeline_sync_set(struct intel_timeline *tl,
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 653fb69e7ecb..944059322daa 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -30,6 +30,8 @@
 #include <generated/utsrelease.h>
 #include <linux/stop_machine.h>
 #include <linux/zlib.h>
+#include <drm/drm_print.h>
+
 #include "i915_drv.h"
 
 static const char *engine_str(int engine)
@@ -175,6 +177,21 @@ static void i915_error_puts(struct drm_i915_error_state_buf *e,
 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
 #define err_puts(e, s) i915_error_puts(e, s)
 
+static void __i915_printfn_error(struct drm_printer *p, struct va_format *vaf)
+{
+	i915_error_vprintf(p->arg, vaf->fmt, *vaf->va);
+}
+
+static inline struct drm_printer
+i915_error_printer(struct drm_i915_error_state_buf *e)
+{
+	struct drm_printer p = {
+		.printfn = __i915_printfn_error,
+		.arg = e,
+	};
+	return p;
+}
+
 #ifdef CONFIG_DRM_I915_COMPRESS_ERROR
 
 struct compress {
@@ -399,6 +416,7 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 	int n;
 
 	err_printf(m, "%s command stream:\n", engine_str(ee->engine_id));
+	err_printf(m, "  IDLE?: %s\n", yesno(ee->idle));
 	err_printf(m, "  START: 0x%08x\n", ee->start);
 	err_printf(m, "  HEAD:  0x%08x [0x%08x]\n", ee->head, ee->rq_head);
 	err_printf(m, "  TAIL:  0x%08x [0x%08x, 0x%08x]\n",
@@ -547,34 +565,17 @@ static void print_error_obj(struct drm_i915_error_state_buf *m,
 static void err_print_capabilities(struct drm_i915_error_state_buf *m,
 				   const struct intel_device_info *info)
 {
-#define PRINT_FLAG(x)  err_printf(m, #x ": %s\n", yesno(info->x))
-	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
-#undef PRINT_FLAG
-}
+	struct drm_printer p = i915_error_printer(m);
 
-static __always_inline void err_print_param(struct drm_i915_error_state_buf *m,
-					    const char *name,
-					    const char *type,
-					    const void *x)
-{
-	if (!__builtin_strcmp(type, "bool"))
-		err_printf(m, "i915.%s=%s\n", name, yesno(*(const bool *)x));
-	else if (!__builtin_strcmp(type, "int"))
-		err_printf(m, "i915.%s=%d\n", name, *(const int *)x);
-	else if (!__builtin_strcmp(type, "unsigned int"))
-		err_printf(m, "i915.%s=%u\n", name, *(const unsigned int *)x);
-	else if (!__builtin_strcmp(type, "char *"))
-		err_printf(m, "i915.%s=%s\n", name, *(const char **)x);
-	else
-		BUILD_BUG();
+	intel_device_info_dump_flags(info, &p);
 }
 
 static void err_print_params(struct drm_i915_error_state_buf *m,
-			     const struct i915_params *p)
+			     const struct i915_params *params)
 {
-#define PRINT(T, x, ...) err_print_param(m, #x, #T, &p->x);
-	I915_PARAMS_FOR_EACH(PRINT);
-#undef PRINT
+	struct drm_printer p = i915_error_printer(m);
+
+	i915_params_dump(params, &p);
 }
 
 static void err_print_pciid(struct drm_i915_error_state_buf *m,
@@ -589,6 +590,21 @@ static void err_print_pciid(struct drm_i915_error_state_buf *m,
 		   pdev->subsystem_device);
 }
 
+static void err_print_uc(struct drm_i915_error_state_buf *m,
+			 const struct i915_error_uc *error_uc)
+{
+	struct drm_printer p = i915_error_printer(m);
+	const struct i915_gpu_state *error =
+		container_of(error_uc, typeof(*error), uc);
+
+	if (!error->device_info.has_guc)
+		return;
+
+	intel_uc_fw_dump(&error_uc->guc_fw, &p);
+	intel_uc_fw_dump(&error_uc->huc_fw, &p);
+	print_error_obj(m, NULL, "GuC log buffer", error_uc->guc_log);
+}
+
 int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			    const struct i915_gpu_state *error)
 {
@@ -759,11 +775,10 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
 		print_error_obj(m, dev_priv->engine[i],
 				"WA batchbuffer", ee->wa_batchbuffer);
-	}
-
-	print_error_obj(m, NULL, "Semaphores", error->semaphore);
 
-	print_error_obj(m, NULL, "GuC log buffer", error->guc_log);
+		print_error_obj(m, dev_priv->engine[i],
+				"NULL context", ee->default_state);
+	}
 
 	if (error->overlay)
 		intel_overlay_print_error_state(m, error->overlay);
@@ -773,6 +788,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
 	err_print_capabilities(m, &error->device_info);
 	err_print_params(m, &error->params);
+	err_print_uc(m, &error->uc);
 
 	if (m->bytes == 0 && m->err)
 		return m->err;
@@ -831,6 +847,22 @@ static __always_inline void free_param(const char *type, void *x)
 		kfree(*(void **)x);
 }
 
+static void cleanup_params(struct i915_gpu_state *error)
+{
+#define FREE(T, x, ...) free_param(#T, &error->params.x);
+	I915_PARAMS_FOR_EACH(FREE);
+#undef FREE
+}
+
+static void cleanup_uc_state(struct i915_gpu_state *error)
+{
+	struct i915_error_uc *error_uc = &error->uc;
+
+	kfree(error_uc->guc_fw.path);
+	kfree(error_uc->huc_fw.path);
+	i915_error_object_free(error_uc->guc_log);
+}
+
 void __i915_gpu_state_free(struct kref *error_ref)
 {
 	struct i915_gpu_state *error =
@@ -856,9 +888,6 @@ void __i915_gpu_state_free(struct kref *error_ref)
 			kfree(ee->waiters);
 	}
 
-	i915_error_object_free(error->semaphore);
-	i915_error_object_free(error->guc_log);
-
 	for (i = 0; i < ARRAY_SIZE(error->active_bo); i++)
 		kfree(error->active_bo[i]);
 	kfree(error->pinned_bo);
@@ -866,9 +895,8 @@ void __i915_gpu_state_free(struct kref *error_ref)
 	kfree(error->overlay);
 	kfree(error->display);
 
-#define FREE(T, x, ...) free_param(#T, &error->params.x);
-	I915_PARAMS_FOR_EACH(FREE);
-#undef FREE
+	cleanup_params(error);
+	cleanup_uc_state(error);
 
 	kfree(error);
 }
@@ -912,7 +940,7 @@ i915_error_object_create(struct drm_i915_private *i915,
 		ggtt->base.insert_page(&ggtt->base, dma, slot,
 				       I915_CACHE_NONE, 0);
 
-		s = io_mapping_map_atomic_wc(&ggtt->mappable, slot);
+		s = io_mapping_map_atomic_wc(&ggtt->iomap, slot);
 		ret = compress_page(&compress, (void  __force *)s, dst);
 		io_mapping_unmap_atomic(s);
 
@@ -1071,34 +1099,6 @@ gen8_engine_sync_index(struct intel_engine_cs *engine,
 	return idx;
 }
 
-static void gen8_record_semaphore_state(struct i915_gpu_state *error,
-					struct intel_engine_cs *engine,
-					struct drm_i915_error_engine *ee)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-	struct intel_engine_cs *to;
-	enum intel_engine_id id;
-
-	if (!error->semaphore)
-		return;
-
-	for_each_engine(to, dev_priv, id) {
-		int idx;
-		u16 signal_offset;
-		u32 *tmp;
-
-		if (engine == to)
-			continue;
-
-		signal_offset =
-			(GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4;
-		tmp = error->semaphore->pages[0];
-		idx = gen8_engine_sync_index(engine, to);
-
-		ee->semaphore_mboxes[idx] = tmp[signal_offset];
-	}
-}
-
 static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
 					struct drm_i915_error_engine *ee)
 {
@@ -1172,11 +1172,12 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 
 	if (INTEL_GEN(dev_priv) >= 6) {
 		ee->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
-		ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
-		if (INTEL_GEN(dev_priv) >= 8)
-			gen8_record_semaphore_state(error, engine, ee);
-		else
+		if (INTEL_GEN(dev_priv) >= 8) {
+			ee->fault_reg = I915_READ(GEN8_RING_FAULT_REG);
+		} else {
 			gen6_record_semaphore_state(engine, ee);
+			ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
+		}
 	}
 
 	if (INTEL_GEN(dev_priv) >= 4) {
@@ -1239,6 +1240,7 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 		ee->hws = I915_READ(mmio);
 	}
 
+	ee->idle = intel_engine_is_idle(engine);
 	ee->hangcheck_timestamp = engine->hangcheck.action_timestamp;
 	ee->hangcheck_action = engine->hangcheck.action;
 	ee->hangcheck_stalled = engine->hangcheck.stalled;
@@ -1400,15 +1402,30 @@ static void request_record_user_bo(struct drm_i915_gem_request *request,
 	ee->user_bo_count = count;
 }
 
+static struct drm_i915_error_object *
+capture_object(struct drm_i915_private *dev_priv,
+	       struct drm_i915_gem_object *obj)
+{
+	if (obj && i915_gem_object_has_pages(obj)) {
+		struct i915_vma fake = {
+			.node = { .start = U64_MAX, .size = obj->base.size },
+			.size = obj->base.size,
+			.pages = obj->mm.pages,
+			.obj = obj,
+		};
+
+		return i915_error_object_create(dev_priv, &fake);
+	} else {
+		return NULL;
+	}
+}
+
 static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 				  struct i915_gpu_state *error)
 {
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	int i;
 
-	error->semaphore =
-		i915_error_object_create(dev_priv, dev_priv->semaphore);
-
 	for (i = 0; i < I915_NUM_ENGINES; i++) {
 		struct intel_engine_cs *engine = dev_priv->engine[i];
 		struct drm_i915_error_engine *ee = &error->engine[i];
@@ -1474,6 +1491,9 @@ static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
 
 		ee->wa_ctx =
 			i915_error_object_create(dev_priv, engine->wa_ctx.vma);
+
+		ee->default_state =
+			capture_object(dev_priv, engine->default_state);
 	}
 }
 
@@ -1559,15 +1579,25 @@ static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv,
 	error->pinned_bo = bo;
 }
 
-static void i915_gem_capture_guc_log_buffer(struct drm_i915_private *dev_priv,
-					    struct i915_gpu_state *error)
+static void capture_uc_state(struct i915_gpu_state *error)
 {
-	/* Capturing log buf contents won't be useful if logging was disabled */
-	if (!dev_priv->guc.log.vma || (i915_modparams.guc_log_level < 0))
+	struct drm_i915_private *i915 = error->i915;
+	struct i915_error_uc *error_uc = &error->uc;
+
+	/* Capturing uC state won't be useful if there is no GuC */
+	if (!error->device_info.has_guc)
 		return;
 
-	error->guc_log = i915_error_object_create(dev_priv,
-						  dev_priv->guc.log.vma);
+	error_uc->guc_fw = i915->guc.fw;
+	error_uc->huc_fw = i915->huc.fw;
+
+	/* Non-default firmware paths will be specified by the modparam.
+	 * As modparams are generally accesible from the userspace make
+	 * explicit copies of the firmware paths.
+	 */
+	error_uc->guc_fw.path = kstrdup(i915->guc.fw.path, GFP_ATOMIC);
+	error_uc->huc_fw.path = kstrdup(i915->huc.fw.path, GFP_ATOMIC);
+	error_uc->guc_log = i915_error_object_create(i915, i915->guc.log.vma);
 }
 
 /* Capture all registers which don't fit into another category. */
@@ -1695,6 +1725,14 @@ static __always_inline void dup_param(const char *type, void *x)
 		*(void **)x = kstrdup(*(void **)x, GFP_ATOMIC);
 }
 
+static void capture_params(struct i915_gpu_state *error)
+{
+	error->params = i915_modparams;
+#define DUP(T, x, ...) dup_param(#T, &error->params.x);
+	I915_PARAMS_FOR_EACH(DUP);
+#undef DUP
+}
+
 static int capture(void *data)
 {
 	struct i915_gpu_state *error = data;
@@ -1705,10 +1743,8 @@ static int capture(void *data)
 		ktime_to_timeval(ktime_sub(ktime_get(),
 					   error->i915->gt.last_init_time));
 
-	error->params = i915_modparams;
-#define DUP(T, x, ...) dup_param(#T, &error->params.x);
-	I915_PARAMS_FOR_EACH(DUP);
-#undef DUP
+	capture_params(error);
+	capture_uc_state(error);
 
 	i915_capture_gen_state(error->i915, error);
 	i915_capture_reg_state(error->i915, error);
@@ -1716,7 +1752,6 @@ static int capture(void *data)
 	i915_gem_record_rings(error->i915, error);
 	i915_capture_active_buffers(error->i915, error);
 	i915_capture_pinned_buffers(error->i915, error);
-	i915_gem_capture_guc_log_buffer(error->i915, error);
 
 	error->overlay = intel_overlay_capture_error_state(error->i915);
 	error->display = intel_display_capture_error_state(error->i915);
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index f8205841868b..3517c6548e2c 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -1068,6 +1068,9 @@ static void notify_ring(struct intel_engine_cs *engine)
 	struct drm_i915_gem_request *rq = NULL;
 	struct intel_wait *wait;
 
+	if (!engine->breadcrumbs.irq_armed)
+		return;
+
 	atomic_inc(&engine->irq_count);
 	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
 
@@ -1101,7 +1104,8 @@ static void notify_ring(struct intel_engine_cs *engine)
 		if (wakeup)
 			wake_up_process(wait->tsk);
 	} else {
-		__intel_engine_disarm_breadcrumbs(engine);
+		if (engine->breadcrumbs.irq_armed)
+			__intel_engine_disarm_breadcrumbs(engine);
 	}
 	spin_unlock(&engine->breadcrumbs.irq_lock);
 
@@ -1396,11 +1400,11 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
 
 	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
 		notify_ring(engine);
-		tasklet |= i915_modparams.enable_guc_submission;
+		tasklet |= USES_GUC_SUBMISSION(engine->i915);
 	}
 
 	if (tasklet)
-		tasklet_hi_schedule(&execlists->irq_tasklet);
+		tasklet_hi_schedule(&execlists->tasklet);
 }
 
 static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
@@ -3064,7 +3068,7 @@ static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
 	i9xx_pipestat_irq_reset(dev_priv);
 
 	GEN3_IRQ_RESET(VLV_);
-	dev_priv->irq_mask = ~0;
+	dev_priv->irq_mask = ~0u;
 }
 
 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
@@ -3089,7 +3093,7 @@ static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
 		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
 			I915_LPE_PIPE_C_INTERRUPT;
 
-	WARN_ON(dev_priv->irq_mask != ~0);
+	WARN_ON(dev_priv->irq_mask != ~0u);
 
 	dev_priv->irq_mask = ~enable_mask;
 
diff --git a/drivers/gpu/drm/i915/i915_memcpy.c b/drivers/gpu/drm/i915/i915_memcpy.c
index 49a079494b68..79f8ec756362 100644
--- a/drivers/gpu/drm/i915/i915_memcpy.c
+++ b/drivers/gpu/drm/i915/i915_memcpy.c
@@ -96,6 +96,11 @@ bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
 
 void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
 {
-	if (static_cpu_has(X86_FEATURE_XMM4_1))
+	/*
+	 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions
+	 * emulation. So don't enable movntdqa in hypervisor guest.
+	 */
+	if (static_cpu_has(X86_FEATURE_XMM4_1) &&
+	    !boot_cpu_has(X86_FEATURE_HYPERVISOR))
 		static_branch_enable(&has_movntdqa);
 }
diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.c b/drivers/gpu/drm/i915/i915_oa_bdw.c
index abdf4d0abcce..4abd2e8b5083 100644
--- a/drivers/gpu/drm/i915/i915_oa_bdw.c
+++ b/drivers/gpu/drm/i915/i915_oa_bdw.c
@@ -85,9 +85,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"d6de6f55-e526-4f79-a6a6-d7315c09044e",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.c b/drivers/gpu/drm/i915/i915_oa_bxt.c
index b69b900de0fe..cb6f304ec16a 100644
--- a/drivers/gpu/drm/i915/i915_oa_bxt.c
+++ b/drivers/gpu/drm/i915/i915_oa_bxt.c
@@ -83,9 +83,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"5ee72f5c-092f-421e-8b70-225f7c3e9612",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt2.c b/drivers/gpu/drm/i915/i915_oa_cflgt2.c
index 368c87d7ee9a..8641ae30e343 100644
--- a/drivers/gpu/drm/i915/i915_oa_cflgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_cflgt2.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"74fb4902-d3d3-4237-9e90-cbdc68d0a446",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt3.c b/drivers/gpu/drm/i915/i915_oa_cflgt3.c
new file mode 100644
index 000000000000..792facdb6702
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cflgt3.c
@@ -0,0 +1,109 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_cflgt3.h"
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2744), 0x00800000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x00000000 },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x00000000 },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x00000000 },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0x9840), 0x00000080 },
+	{ _MMIO(0x9888), 0x11810000 },
+	{ _MMIO(0x9888), 0x07810013 },
+	{ _MMIO(0x9888), 0x1f810000 },
+	{ _MMIO(0x9888), 0x1d810000 },
+	{ _MMIO(0x9888), 0x1b930040 },
+	{ _MMIO(0x9888), 0x07e54000 },
+	{ _MMIO(0x9888), 0x1f908000 },
+	{ _MMIO(0x9888), 0x11900000 },
+	{ _MMIO(0x9888), 0x37900000 },
+	{ _MMIO(0x9888), 0x53900000 },
+	{ _MMIO(0x9888), 0x45900000 },
+	{ _MMIO(0x9888), 0x33900000 },
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "1\n");
+}
+
+void
+i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv)
+{
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
+		"577e8e2c-3fa0-4875-8743-3538d585e3b0",
+		sizeof(dev_priv->perf.oa.test_config.uuid));
+	dev_priv->perf.oa.test_config.id = 1;
+
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
+
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
+
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
+
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "577e8e2c-3fa0-4875-8743-3538d585e3b0";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
+
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt3.h b/drivers/gpu/drm/i915/i915_oa_cflgt3.h
new file mode 100644
index 000000000000..c13b5aac01b9
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cflgt3.h
@@ -0,0 +1,34 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_CFLGT3_H__
+#define __I915_OA_CFLGT3_H__
+
+extern void i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_chv.c b/drivers/gpu/drm/i915/i915_oa_chv.c
index 322a3f94cd16..556febb2c3c8 100644
--- a/drivers/gpu/drm/i915/i915_oa_chv.c
+++ b/drivers/gpu/drm/i915/i915_oa_chv.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"4a534b07-cba3-414d-8d60-874830e883aa",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_cnl.c b/drivers/gpu/drm/i915/i915_oa_cnl.c
new file mode 100644
index 000000000000..ba9140c87cc0
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cnl.c
@@ -0,0 +1,121 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_cnl.h"
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+	{ _MMIO(0x2740), 0x00000000 },
+	{ _MMIO(0x2710), 0x00000000 },
+	{ _MMIO(0x2714), 0xf0800000 },
+	{ _MMIO(0x2720), 0x00000000 },
+	{ _MMIO(0x2724), 0xf0800000 },
+	{ _MMIO(0x2770), 0x00000004 },
+	{ _MMIO(0x2774), 0x0000ffff },
+	{ _MMIO(0x2778), 0x00000003 },
+	{ _MMIO(0x277c), 0x0000ffff },
+	{ _MMIO(0x2780), 0x00000007 },
+	{ _MMIO(0x2784), 0x0000ffff },
+	{ _MMIO(0x2788), 0x00100002 },
+	{ _MMIO(0x278c), 0x0000fff7 },
+	{ _MMIO(0x2790), 0x00100002 },
+	{ _MMIO(0x2794), 0x0000ffcf },
+	{ _MMIO(0x2798), 0x00100082 },
+	{ _MMIO(0x279c), 0x0000ffef },
+	{ _MMIO(0x27a0), 0x001000c2 },
+	{ _MMIO(0x27a4), 0x0000ffe7 },
+	{ _MMIO(0x27a8), 0x00100001 },
+	{ _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+	{ _MMIO(0xd04), 0x00000200 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x17060000 },
+	{ _MMIO(0x9840), 0x00000000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x13034000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x07060066 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x05060000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x0f080040 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x07091000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x0f041000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x1d004000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x35000000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x49000000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x3d000000 },
+	{ _MMIO(0x9884), 0x00000007 },
+	{ _MMIO(0x9888), 0x31000000 },
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "1\n");
+}
+
+void
+i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv)
+{
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
+		"db41edd4-d8e7-4730-ad11-b9a2d6833503",
+		sizeof(dev_priv->perf.oa.test_config.uuid));
+	dev_priv->perf.oa.test_config.id = 1;
+
+	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+	dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
+
+	dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+	dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
+
+	dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+	dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
+
+	dev_priv->perf.oa.test_config.sysfs_metric.name = "db41edd4-d8e7-4730-ad11-b9a2d6833503";
+	dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+	dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
+
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+	dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+	dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_cnl.h b/drivers/gpu/drm/i915/i915_oa_cnl.h
new file mode 100644
index 000000000000..fb918b131105
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cnl.h
@@ -0,0 +1,34 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_CNL_H__
+#define __I915_OA_CNL_H__
+
+extern void i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_glk.c b/drivers/gpu/drm/i915/i915_oa_glk.c
index 4ee527e4c926..971db587957c 100644
--- a/drivers/gpu/drm/i915/i915_oa_glk.c
+++ b/drivers/gpu/drm/i915/i915_oa_glk.c
@@ -83,9 +83,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"dd3fd789-e783-4204-8cd0-b671bbccb0cf",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.c b/drivers/gpu/drm/i915/i915_oa_hsw.c
index 56b03773bb9d..434a9b96d7ab 100644
--- a/drivers/gpu/drm/i915/i915_oa_hsw.c
+++ b/drivers/gpu/drm/i915/i915_oa_hsw.c
@@ -113,9 +113,9 @@ show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *b
 void
 i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"403d8832-1a27-4aa6-a64e-f5389ce7b212",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_render_basic;
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.c b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
index b6e7cc774136..2fa98a40bbc8 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"baa3c7e4-52b6-4b85-801e-465a94b746dd",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.c b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
index 5576afdd9a7e..f3cb6679a1bc 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt3.c
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"f1792f32-6db2-4b50-b4b2-557128f1688d",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.c b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
index 890d55879946..bf8b8cd8a50d 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
@@ -83,9 +83,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"1651949f-0ac0-4cb1-a06f-dafd74a407d1",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.c b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
index 85e51addf86a..ae534c7c8135 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt3.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"2b985803-d3c9-4629-8a4f-634bfecba0e8",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.c b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
index bce031ee4445..817fba2d82df 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt4.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv)
 {
-	strncpy(dev_priv->perf.oa.test_config.uuid,
+	strlcpy(dev_priv->perf.oa.test_config.uuid,
 		"882fa433-1f4a-4a67-a962-c741888fe5f5",
-		UUID_STRING_LEN);
+		sizeof(dev_priv->perf.oa.test_config.uuid));
 	dev_priv->perf.oa.test_config.id = 1;
 
 	dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index b4faeb6aa2bd..b5f3eb4fa8a3 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -22,6 +22,8 @@
  * IN THE SOFTWARE.
  */
 
+#include <drm/drm_print.h>
+
 #include "i915_params.h"
 #include "i915_drv.h"
 
@@ -46,17 +48,6 @@ i915_param_named_unsafe(panel_ignore_lid, int, 0600,
 	"Override lid status (0=autodetect, 1=autodetect disabled [default], "
 	"-1=force lid closed, -2=force lid open)");
 
-i915_param_named_unsafe(semaphores, int, 0400,
-	"Use semaphores for inter-ring sync "
-	"(default: -1 (use per-chip defaults))");
-
-i915_param_named_unsafe(enable_rc6, int, 0400,
-	"Enable power-saving render C-state 6. "
-	"Different stages can be selected via bitmask values "
-	"(0 = disable; 1 = enable rc6; 2 = enable deep rc6; 4 = enable deepest rc6). "
-	"For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. "
-	"default: -1 (use per-chip default)");
-
 i915_param_named_unsafe(enable_dc, int, 0400,
 	"Enable power-saving display C-states. "
 	"(-1=auto [default]; 0=disable; 1=up to DC5; 2=up to DC6)");
@@ -99,10 +90,6 @@ i915_param_named_unsafe(enable_ppgtt, int, 0400,
 	"Override PPGTT usage. "
 	"(-1=auto [default], 0=disabled, 1=aliasing, 2=full, 3=full with extended address space)");
 
-i915_param_named_unsafe(enable_execlists, int, 0400,
-	"Override execlists usage. "
-	"(-1=auto [default], 0=disabled, 1=enabled)");
-
 i915_param_named_unsafe(enable_psr, int, 0600,
 	"Enable PSR "
 	"(0=disabled, 1=enabled - link mode chosen per-platform, 2=force link-standby mode, 3=force link-off mode) "
@@ -162,13 +149,10 @@ i915_param_named_unsafe(edp_vswing, int, 0400,
 	"(0=use value from vbt [default], 1=low power swing(200mV),"
 	"2=default swing(400mV))");
 
-i915_param_named_unsafe(enable_guc_loading, int, 0400,
-	"Enable GuC firmware loading "
-	"(-1=auto, 0=never [default], 1=if available, 2=required)");
-
-i915_param_named_unsafe(enable_guc_submission, int, 0400,
-	"Enable GuC submission "
-	"(-1=auto, 0=never [default], 1=if available, 2=required)");
+i915_param_named_unsafe(enable_guc, int, 0400,
+	"Enable GuC load for GuC submission and/or HuC load. "
+	"Required functionality can be selected using bitmask values. "
+	"(-1=auto, 0=disable [default], 1=GuC submission, 2=HuC load)");
 
 i915_param_named(guc_log_level, int, 0400,
 	"GuC firmware logging level (-1:disabled (default), 0-3:enabled)");
@@ -190,3 +174,34 @@ i915_param_named(enable_dpcd_backlight, bool, 0600,
 
 i915_param_named(enable_gvt, bool, 0400,
 	"Enable support for Intel GVT-g graphics virtualization host support(default:false)");
+
+static __always_inline void _print_param(struct drm_printer *p,
+					 const char *name,
+					 const char *type,
+					 const void *x)
+{
+	if (!__builtin_strcmp(type, "bool"))
+		drm_printf(p, "i915.%s=%s\n", name, yesno(*(const bool *)x));
+	else if (!__builtin_strcmp(type, "int"))
+		drm_printf(p, "i915.%s=%d\n", name, *(const int *)x);
+	else if (!__builtin_strcmp(type, "unsigned int"))
+		drm_printf(p, "i915.%s=%u\n", name, *(const unsigned int *)x);
+	else if (!__builtin_strcmp(type, "char *"))
+		drm_printf(p, "i915.%s=%s\n", name, *(const char **)x);
+	else
+		BUILD_BUG();
+}
+
+/**
+ * i915_params_dump - dump i915 modparams
+ * @params: i915 modparams
+ * @p: the &drm_printer
+ *
+ * Pretty printer for i915 modparams.
+ */
+void i915_params_dump(const struct i915_params *params, struct drm_printer *p)
+{
+#define PRINT(T, x, ...) _print_param(p, #x, #T, &params->x);
+	I915_PARAMS_FOR_EACH(PRINT);
+#undef PRINT
+}
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index c7292268ed43..c96360398072 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -25,27 +25,29 @@
 #ifndef _I915_PARAMS_H_
 #define _I915_PARAMS_H_
 
+#include <linux/bitops.h>
 #include <linux/cache.h> /* for __read_mostly */
 
+struct drm_printer;
+
+#define ENABLE_GUC_SUBMISSION		BIT(0)
+#define ENABLE_GUC_LOAD_HUC		BIT(1)
+
 #define I915_PARAMS_FOR_EACH(param) \
 	param(char *, vbt_firmware, NULL) \
 	param(int, modeset, -1) \
 	param(int, panel_ignore_lid, 1) \
-	param(int, semaphores, -1) \
 	param(int, lvds_channel_mode, 0) \
 	param(int, panel_use_ssc, -1) \
 	param(int, vbt_sdvo_panel_type, -1) \
-	param(int, enable_rc6, -1) \
 	param(int, enable_dc, -1) \
 	param(int, enable_fbc, -1) \
 	param(int, enable_ppgtt, -1) \
-	param(int, enable_execlists, -1) \
 	param(int, enable_psr, -1) \
 	param(int, disable_power_well, -1) \
 	param(int, enable_ips, 1) \
 	param(int, invert_brightness, 0) \
-	param(int, enable_guc_loading, 0) \
-	param(int, enable_guc_submission, 0) \
+	param(int, enable_guc, 0) \
 	param(int, guc_log_level, -1) \
 	param(char *, guc_firmware_path, NULL) \
 	param(char *, huc_firmware_path, NULL) \
@@ -77,5 +79,7 @@ struct i915_params {
 
 extern struct i915_params i915_modparams __read_mostly;
 
+void i915_params_dump(const struct i915_params *params, struct drm_printer *p);
+
 #endif
 
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 6458c309c039..1c30c688f23a 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -74,19 +74,19 @@
 	GEN_DEFAULT_PAGE_SIZES, \
 	CURSOR_OFFSETS
 
-static const struct intel_device_info intel_i830_info __initconst = {
+static const struct intel_device_info intel_i830_info = {
 	GEN2_FEATURES,
 	.platform = INTEL_I830,
 	.is_mobile = 1, .cursor_needs_physical = 1,
 	.num_pipes = 2, /* legal, last one wins */
 };
 
-static const struct intel_device_info intel_i845g_info __initconst = {
+static const struct intel_device_info intel_i845g_info = {
 	GEN2_FEATURES,
 	.platform = INTEL_I845G,
 };
 
-static const struct intel_device_info intel_i85x_info __initconst = {
+static const struct intel_device_info intel_i85x_info = {
 	GEN2_FEATURES,
 	.platform = INTEL_I85X, .is_mobile = 1,
 	.num_pipes = 2, /* legal, last one wins */
@@ -94,7 +94,7 @@ static const struct intel_device_info intel_i85x_info __initconst = {
 	.has_fbc = 1,
 };
 
-static const struct intel_device_info intel_i865g_info __initconst = {
+static const struct intel_device_info intel_i865g_info = {
 	GEN2_FEATURES,
 	.platform = INTEL_I865G,
 };
@@ -108,7 +108,7 @@ static const struct intel_device_info intel_i865g_info __initconst = {
 	GEN_DEFAULT_PAGE_SIZES, \
 	CURSOR_OFFSETS
 
-static const struct intel_device_info intel_i915g_info __initconst = {
+static const struct intel_device_info intel_i915g_info = {
 	GEN3_FEATURES,
 	.platform = INTEL_I915G, .cursor_needs_physical = 1,
 	.has_overlay = 1, .overlay_needs_physical = 1,
@@ -116,7 +116,7 @@ static const struct intel_device_info intel_i915g_info __initconst = {
 	.unfenced_needs_alignment = 1,
 };
 
-static const struct intel_device_info intel_i915gm_info __initconst = {
+static const struct intel_device_info intel_i915gm_info = {
 	GEN3_FEATURES,
 	.platform = INTEL_I915GM,
 	.is_mobile = 1,
@@ -128,7 +128,7 @@ static const struct intel_device_info intel_i915gm_info __initconst = {
 	.unfenced_needs_alignment = 1,
 };
 
-static const struct intel_device_info intel_i945g_info __initconst = {
+static const struct intel_device_info intel_i945g_info = {
 	GEN3_FEATURES,
 	.platform = INTEL_I945G,
 	.has_hotplug = 1, .cursor_needs_physical = 1,
@@ -137,7 +137,7 @@ static const struct intel_device_info intel_i945g_info __initconst = {
 	.unfenced_needs_alignment = 1,
 };
 
-static const struct intel_device_info intel_i945gm_info __initconst = {
+static const struct intel_device_info intel_i945gm_info = {
 	GEN3_FEATURES,
 	.platform = INTEL_I945GM, .is_mobile = 1,
 	.has_hotplug = 1, .cursor_needs_physical = 1,
@@ -148,14 +148,14 @@ static const struct intel_device_info intel_i945gm_info __initconst = {
 	.unfenced_needs_alignment = 1,
 };
 
-static const struct intel_device_info intel_g33_info __initconst = {
+static const struct intel_device_info intel_g33_info = {
 	GEN3_FEATURES,
 	.platform = INTEL_G33,
 	.has_hotplug = 1,
 	.has_overlay = 1,
 };
 
-static const struct intel_device_info intel_pineview_info __initconst = {
+static const struct intel_device_info intel_pineview_info = {
 	GEN3_FEATURES,
 	.platform = INTEL_PINEVIEW, .is_mobile = 1,
 	.has_hotplug = 1,
@@ -172,7 +172,7 @@ static const struct intel_device_info intel_pineview_info __initconst = {
 	GEN_DEFAULT_PAGE_SIZES, \
 	CURSOR_OFFSETS
 
-static const struct intel_device_info intel_i965g_info __initconst = {
+static const struct intel_device_info intel_i965g_info = {
 	GEN4_FEATURES,
 	.platform = INTEL_I965G,
 	.has_overlay = 1,
@@ -180,7 +180,7 @@ static const struct intel_device_info intel_i965g_info __initconst = {
 	.has_snoop = false,
 };
 
-static const struct intel_device_info intel_i965gm_info __initconst = {
+static const struct intel_device_info intel_i965gm_info = {
 	GEN4_FEATURES,
 	.platform = INTEL_I965GM,
 	.is_mobile = 1, .has_fbc = 1,
@@ -190,13 +190,13 @@ static const struct intel_device_info intel_i965gm_info __initconst = {
 	.has_snoop = false,
 };
 
-static const struct intel_device_info intel_g45_info __initconst = {
+static const struct intel_device_info intel_g45_info = {
 	GEN4_FEATURES,
 	.platform = INTEL_G45,
 	.ring_mask = RENDER_RING | BSD_RING,
 };
 
-static const struct intel_device_info intel_gm45_info __initconst = {
+static const struct intel_device_info intel_gm45_info = {
 	GEN4_FEATURES,
 	.platform = INTEL_GM45,
 	.is_mobile = 1, .has_fbc = 1,
@@ -209,16 +209,18 @@ static const struct intel_device_info intel_gm45_info __initconst = {
 	.has_hotplug = 1, \
 	.ring_mask = RENDER_RING | BSD_RING, \
 	.has_snoop = true, \
+	/* ilk does support rc6, but we do not implement [power] contexts */ \
+	.has_rc6 = 0, \
 	GEN_DEFAULT_PIPEOFFSETS, \
 	GEN_DEFAULT_PAGE_SIZES, \
 	CURSOR_OFFSETS
 
-static const struct intel_device_info intel_ironlake_d_info __initconst = {
+static const struct intel_device_info intel_ironlake_d_info = {
 	GEN5_FEATURES,
 	.platform = INTEL_IRONLAKE,
 };
 
-static const struct intel_device_info intel_ironlake_m_info __initconst = {
+static const struct intel_device_info intel_ironlake_m_info = {
 	GEN5_FEATURES,
 	.platform = INTEL_IRONLAKE,
 	.is_mobile = 1, .has_fbc = 1,
@@ -241,12 +243,12 @@ static const struct intel_device_info intel_ironlake_m_info __initconst = {
 	GEN6_FEATURES, \
 	.platform = INTEL_SANDYBRIDGE
 
-static const struct intel_device_info intel_sandybridge_d_gt1_info __initconst = {
+static const struct intel_device_info intel_sandybridge_d_gt1_info = {
 	SNB_D_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_sandybridge_d_gt2_info __initconst = {
+static const struct intel_device_info intel_sandybridge_d_gt2_info = {
 	SNB_D_PLATFORM,
 	.gt = 2,
 };
@@ -257,12 +259,12 @@ static const struct intel_device_info intel_sandybridge_d_gt2_info __initconst =
 	.is_mobile = 1
 
 
-static const struct intel_device_info intel_sandybridge_m_gt1_info __initconst = {
+static const struct intel_device_info intel_sandybridge_m_gt1_info = {
 	SNB_M_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_sandybridge_m_gt2_info __initconst = {
+static const struct intel_device_info intel_sandybridge_m_gt2_info = {
 	SNB_M_PLATFORM,
 	.gt = 2,
 };
@@ -286,12 +288,12 @@ static const struct intel_device_info intel_sandybridge_m_gt2_info __initconst =
 	.platform = INTEL_IVYBRIDGE, \
 	.has_l3_dpf = 1
 
-static const struct intel_device_info intel_ivybridge_d_gt1_info __initconst = {
+static const struct intel_device_info intel_ivybridge_d_gt1_info = {
 	IVB_D_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_ivybridge_d_gt2_info __initconst = {
+static const struct intel_device_info intel_ivybridge_d_gt2_info = {
 	IVB_D_PLATFORM,
 	.gt = 2,
 };
@@ -302,17 +304,17 @@ static const struct intel_device_info intel_ivybridge_d_gt2_info __initconst = {
 	.is_mobile = 1, \
 	.has_l3_dpf = 1
 
-static const struct intel_device_info intel_ivybridge_m_gt1_info __initconst = {
+static const struct intel_device_info intel_ivybridge_m_gt1_info = {
 	IVB_M_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_ivybridge_m_gt2_info __initconst = {
+static const struct intel_device_info intel_ivybridge_m_gt2_info = {
 	IVB_M_PLATFORM,
 	.gt = 2,
 };
 
-static const struct intel_device_info intel_ivybridge_q_info __initconst = {
+static const struct intel_device_info intel_ivybridge_q_info = {
 	GEN7_FEATURES,
 	.platform = INTEL_IVYBRIDGE,
 	.gt = 2,
@@ -320,7 +322,7 @@ static const struct intel_device_info intel_ivybridge_q_info __initconst = {
 	.has_l3_dpf = 1,
 };
 
-static const struct intel_device_info intel_valleyview_info __initconst = {
+static const struct intel_device_info intel_valleyview_info = {
 	.platform = INTEL_VALLEYVIEW,
 	.gen = 7,
 	.is_lp = 1,
@@ -356,17 +358,17 @@ static const struct intel_device_info intel_valleyview_info __initconst = {
 	.platform = INTEL_HASWELL, \
 	.has_l3_dpf = 1
 
-static const struct intel_device_info intel_haswell_gt1_info __initconst = {
+static const struct intel_device_info intel_haswell_gt1_info = {
 	HSW_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_haswell_gt2_info __initconst = {
+static const struct intel_device_info intel_haswell_gt2_info = {
 	HSW_PLATFORM,
 	.gt = 2,
 };
 
-static const struct intel_device_info intel_haswell_gt3_info __initconst = {
+static const struct intel_device_info intel_haswell_gt3_info = {
 	HSW_PLATFORM,
 	.gt = 3,
 };
@@ -386,17 +388,17 @@ static const struct intel_device_info intel_haswell_gt3_info __initconst = {
 	.gen = 8, \
 	.platform = INTEL_BROADWELL
 
-static const struct intel_device_info intel_broadwell_gt1_info __initconst = {
+static const struct intel_device_info intel_broadwell_gt1_info = {
 	BDW_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_broadwell_gt2_info __initconst = {
+static const struct intel_device_info intel_broadwell_gt2_info = {
 	BDW_PLATFORM,
 	.gt = 2,
 };
 
-static const struct intel_device_info intel_broadwell_rsvd_info __initconst = {
+static const struct intel_device_info intel_broadwell_rsvd_info = {
 	BDW_PLATFORM,
 	.gt = 3,
 	/* According to the device ID those devices are GT3, they were
@@ -404,13 +406,13 @@ static const struct intel_device_info intel_broadwell_rsvd_info __initconst = {
 	 */
 };
 
-static const struct intel_device_info intel_broadwell_gt3_info __initconst = {
+static const struct intel_device_info intel_broadwell_gt3_info = {
 	BDW_PLATFORM,
 	.gt = 3,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
 };
 
-static const struct intel_device_info intel_cherryview_info __initconst = {
+static const struct intel_device_info intel_cherryview_info = {
 	.gen = 8, .num_pipes = 3,
 	.has_hotplug = 1,
 	.is_lp = 1,
@@ -453,12 +455,12 @@ static const struct intel_device_info intel_cherryview_info __initconst = {
 	.gen = 9, \
 	.platform = INTEL_SKYLAKE
 
-static const struct intel_device_info intel_skylake_gt1_info __initconst = {
+static const struct intel_device_info intel_skylake_gt1_info = {
 	SKL_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_skylake_gt2_info __initconst = {
+static const struct intel_device_info intel_skylake_gt2_info = {
 	SKL_PLATFORM,
 	.gt = 2,
 };
@@ -468,12 +470,12 @@ static const struct intel_device_info intel_skylake_gt2_info __initconst = {
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING
 
 
-static const struct intel_device_info intel_skylake_gt3_info __initconst = {
+static const struct intel_device_info intel_skylake_gt3_info = {
 	SKL_GT3_PLUS_PLATFORM,
 	.gt = 3,
 };
 
-static const struct intel_device_info intel_skylake_gt4_info __initconst = {
+static const struct intel_device_info intel_skylake_gt4_info = {
 	SKL_GT3_PLUS_PLATFORM,
 	.gt = 4,
 };
@@ -509,13 +511,13 @@ static const struct intel_device_info intel_skylake_gt4_info __initconst = {
 	IVB_CURSOR_OFFSETS, \
 	BDW_COLORS
 
-static const struct intel_device_info intel_broxton_info __initconst = {
+static const struct intel_device_info intel_broxton_info = {
 	GEN9_LP_FEATURES,
 	.platform = INTEL_BROXTON,
 	.ddb_size = 512,
 };
 
-static const struct intel_device_info intel_geminilake_info __initconst = {
+static const struct intel_device_info intel_geminilake_info = {
 	GEN9_LP_FEATURES,
 	.platform = INTEL_GEMINILAKE,
 	.ddb_size = 1024,
@@ -527,17 +529,17 @@ static const struct intel_device_info intel_geminilake_info __initconst = {
 	.gen = 9, \
 	.platform = INTEL_KABYLAKE
 
-static const struct intel_device_info intel_kabylake_gt1_info __initconst = {
+static const struct intel_device_info intel_kabylake_gt1_info = {
 	KBL_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_kabylake_gt2_info __initconst = {
+static const struct intel_device_info intel_kabylake_gt2_info = {
 	KBL_PLATFORM,
 	.gt = 2,
 };
 
-static const struct intel_device_info intel_kabylake_gt3_info __initconst = {
+static const struct intel_device_info intel_kabylake_gt3_info = {
 	KBL_PLATFORM,
 	.gt = 3,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
@@ -548,17 +550,17 @@ static const struct intel_device_info intel_kabylake_gt3_info __initconst = {
 	.gen = 9, \
 	.platform = INTEL_COFFEELAKE
 
-static const struct intel_device_info intel_coffeelake_gt1_info __initconst = {
+static const struct intel_device_info intel_coffeelake_gt1_info = {
 	CFL_PLATFORM,
 	.gt = 1,
 };
 
-static const struct intel_device_info intel_coffeelake_gt2_info __initconst = {
+static const struct intel_device_info intel_coffeelake_gt2_info = {
 	CFL_PLATFORM,
 	.gt = 2,
 };
 
-static const struct intel_device_info intel_coffeelake_gt3_info __initconst = {
+static const struct intel_device_info intel_coffeelake_gt3_info = {
 	CFL_PLATFORM,
 	.gt = 3,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
@@ -569,7 +571,7 @@ static const struct intel_device_info intel_coffeelake_gt3_info __initconst = {
 	.ddb_size = 1024, \
 	GLK_COLORS
 
-static const struct intel_device_info intel_cannonlake_gt2_info __initconst = {
+static const struct intel_device_info intel_cannonlake_gt2_info = {
 	GEN10_FEATURES,
 	.is_alpha_support = 1,
 	.platform = INTEL_CANNONLAKE,
@@ -631,6 +633,8 @@ static const struct pci_device_id pciidlist[] = {
 	INTEL_CFL_S_GT1_IDS(&intel_coffeelake_gt1_info),
 	INTEL_CFL_S_GT2_IDS(&intel_coffeelake_gt2_info),
 	INTEL_CFL_H_GT2_IDS(&intel_coffeelake_gt2_info),
+	INTEL_CFL_U_GT1_IDS(&intel_coffeelake_gt1_info),
+	INTEL_CFL_U_GT2_IDS(&intel_coffeelake_gt2_info),
 	INTEL_CFL_U_GT3_IDS(&intel_coffeelake_gt3_info),
 	INTEL_CNL_U_GT2_IDS(&intel_cannonlake_gt2_info),
 	INTEL_CNL_Y_GT2_IDS(&intel_cannonlake_gt2_info),
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 59ee808f8fd9..f8fe5ffcdcff 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -207,6 +207,8 @@
 #include "i915_oa_kblgt3.h"
 #include "i915_oa_glk.h"
 #include "i915_oa_cflgt2.h"
+#include "i915_oa_cflgt3.h"
+#include "i915_oa_cnl.h"
 
 /* HW requires this to be a power of two, between 128k and 16M, though driver
  * is currently generally designed assuming the largest 16M size is used such
@@ -242,7 +244,7 @@
  * The two separate pointers let us decouple read()s from tail pointer aging.
  *
  * The tail pointers are checked and updated at a limited rate within a hrtimer
- * callback (the same callback that is used for delivering POLLIN events)
+ * callback (the same callback that is used for delivering EPOLLIN events)
  *
  * Initially the tails are marked invalid with %INVALID_TAIL_PTR which
  * indicates that an updated tail pointer is needed.
@@ -1214,9 +1216,9 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream)
 {
 	struct drm_i915_private *dev_priv = stream->dev_priv;
 
-	if (i915_modparams.enable_execlists)
+	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
 		dev_priv->perf.oa.specific_ctx_id = stream->ctx->hw_id;
-	else {
+	} else {
 		struct intel_engine_cs *engine = dev_priv->engine[RCS];
 		struct intel_ring *ring;
 		int ret;
@@ -1260,7 +1262,7 @@ static void oa_put_render_ctx_id(struct i915_perf_stream *stream)
 {
 	struct drm_i915_private *dev_priv = stream->dev_priv;
 
-	if (i915_modparams.enable_execlists) {
+	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
 		dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID;
 	} else {
 		struct intel_engine_cs *engine = dev_priv->engine[RCS];
@@ -1301,9 +1303,8 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 	 */
 	mutex_lock(&dev_priv->drm.struct_mutex);
 	dev_priv->perf.oa.exclusive_stream = NULL;
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-
 	dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	free_oa_buffer(dev_priv);
 
@@ -1724,10 +1725,9 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
 							 GFP_KERNEL);
 	}
 
-	ret = i915_switch_context(req);
 	i915_add_request(req);
 
-	return ret;
+	return 0;
 }
 
 /*
@@ -1755,22 +1755,13 @@ static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_pr
  * Note: it's only the RCS/Render context that has any OA state.
  */
 static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
-				       const struct i915_oa_config *oa_config,
-				       bool interruptible)
+				       const struct i915_oa_config *oa_config)
 {
 	struct i915_gem_context *ctx;
 	int ret;
 	unsigned int wait_flags = I915_WAIT_LOCKED;
 
-	if (interruptible) {
-		ret = i915_mutex_lock_interruptible(&dev_priv->drm);
-		if (ret)
-			return ret;
-
-		wait_flags |= I915_WAIT_INTERRUPTIBLE;
-	} else {
-		mutex_lock(&dev_priv->drm.struct_mutex);
-	}
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
 	/* Switch away from any user context. */
 	ret = gen8_switch_to_updated_kernel_context(dev_priv, oa_config);
@@ -1818,8 +1809,6 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 	}
 
  out:
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-
 	return ret;
 }
 
@@ -1851,7 +1840,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
 	 * be read back from automatically triggered reports, as part of the
 	 * RPT_ID field.
 	 */
-	if (IS_GEN9(dev_priv)) {
+	if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) {
 		I915_WRITE(GEN8_OA_DEBUG,
 			   _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
 					      GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
@@ -1862,7 +1851,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
 	 * to make sure all slices/subslices are ON before writing to NOA
 	 * registers.
 	 */
-	ret = gen8_configure_all_contexts(dev_priv, oa_config, true);
+	ret = gen8_configure_all_contexts(dev_priv, oa_config);
 	if (ret)
 		return ret;
 
@@ -1877,13 +1866,23 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
 static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
 {
 	/* Reset all contexts' slices/subslices configurations. */
-	gen8_configure_all_contexts(dev_priv, NULL, false);
+	gen8_configure_all_contexts(dev_priv, NULL);
 
 	I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) &
 				      ~GT_NOA_ENABLE));
 
 }
 
+static void gen10_disable_metric_set(struct drm_i915_private *dev_priv)
+{
+	/* Reset all contexts' slices/subslices configurations. */
+	gen8_configure_all_contexts(dev_priv, NULL);
+
+	/* Make sure we disable noa to save power. */
+	I915_WRITE(RPM_CONFIG1,
+		   I915_READ(RPM_CONFIG1) & ~GEN10_GT_NOA_ENABLE);
+}
+
 static void gen7_oa_enable(struct drm_i915_private *dev_priv)
 {
 	/*
@@ -2127,6 +2126,10 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	if (ret)
 		goto err_oa_buf_alloc;
 
+	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
+	if (ret)
+		goto err_lock;
+
 	ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv,
 						      stream->oa_config);
 	if (ret)
@@ -2134,23 +2137,17 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 
 	stream->ops = &i915_oa_stream_ops;
 
-	/* Lock device for exclusive_stream access late because
-	 * enable_metric_set() might lock as well on gen8+.
-	 */
-	ret = i915_mutex_lock_interruptible(&dev_priv->drm);
-	if (ret)
-		goto err_lock;
-
 	dev_priv->perf.oa.exclusive_stream = stream;
 
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	return 0;
 
-err_lock:
+err_enable:
 	dev_priv->perf.oa.ops.disable_metric_set(dev_priv);
+	mutex_unlock(&dev_priv->drm.struct_mutex);
 
-err_enable:
+err_lock:
 	free_oa_buffer(dev_priv);
 
 err_oa_buf_alloc:
@@ -2281,13 +2278,13 @@ static ssize_t i915_perf_read(struct file *file,
 		mutex_unlock(&dev_priv->perf.lock);
 	}
 
-	/* We allow the poll checking to sometimes report false positive POLLIN
+	/* We allow the poll checking to sometimes report false positive EPOLLIN
 	 * events where we might actually report EAGAIN on read() if there's
 	 * not really any data available. In this situation though we don't
-	 * want to enter a busy loop between poll() reporting a POLLIN event
+	 * want to enter a busy loop between poll() reporting a EPOLLIN event
 	 * and read() returning -EAGAIN. Clearing the oa.pollin state here
 	 * effectively ensures we back off until the next hrtimer callback
-	 * before reporting another POLLIN event.
+	 * before reporting another EPOLLIN event.
 	 */
 	if (ret >= 0 || ret == -EAGAIN) {
 		/* Maybe make ->pollin per-stream state if we support multiple
@@ -2331,12 +2328,12 @@ static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer)
  *
  * Returns: any poll events that are ready without sleeping
  */
-static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv,
+static __poll_t i915_perf_poll_locked(struct drm_i915_private *dev_priv,
 					  struct i915_perf_stream *stream,
 					  struct file *file,
 					  poll_table *wait)
 {
-	unsigned int events = 0;
+	__poll_t events = 0;
 
 	stream->ops->poll_wait(stream, file, wait);
 
@@ -2347,7 +2344,7 @@ static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv,
 	 * samples to read.
 	 */
 	if (dev_priv->perf.oa.pollin)
-		events |= POLLIN;
+		events |= EPOLLIN;
 
 	return events;
 }
@@ -2365,11 +2362,11 @@ static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv,
  *
  * Returns: any poll events that are ready without sleeping
  */
-static unsigned int i915_perf_poll(struct file *file, poll_table *wait)
+static __poll_t i915_perf_poll(struct file *file, poll_table *wait)
 {
 	struct i915_perf_stream *stream = file->private_data;
 	struct drm_i915_private *dev_priv = stream->dev_priv;
-	int ret;
+	__poll_t ret;
 
 	mutex_lock(&dev_priv->perf.lock);
 	ret = i915_perf_poll_locked(dev_priv, stream, file, wait);
@@ -2679,8 +2676,8 @@ i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv,
 
 static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent)
 {
-	return div_u64(1000000000ULL * (2ULL << exponent),
-		       dev_priv->perf.oa.timestamp_frequency);
+	return div64_u64(1000000000ULL * (2ULL << exponent),
+			 1000ULL * INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz);
 }
 
 /**
@@ -2934,6 +2931,10 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
 	} else if (IS_COFFEELAKE(dev_priv)) {
 		if (IS_CFL_GT2(dev_priv))
 			i915_perf_load_test_config_cflgt2(dev_priv);
+		if (IS_CFL_GT3(dev_priv))
+			i915_perf_load_test_config_cflgt3(dev_priv);
+	} else if (IS_CANNONLAKE(dev_priv)) {
+		i915_perf_load_test_config_cnl(dev_priv);
 	}
 
 	if (dev_priv->perf.oa.test_config.id == 0)
@@ -2991,7 +2992,7 @@ static bool gen8_is_valid_flex_addr(struct drm_i915_private *dev_priv, u32 addr)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
-		if (flex_eu_regs[i].reg == addr)
+		if (i915_mmio_reg_offset(flex_eu_regs[i]) == addr)
 			return true;
 	}
 	return false;
@@ -2999,31 +3000,47 @@ static bool gen8_is_valid_flex_addr(struct drm_i915_private *dev_priv, u32 addr)
 
 static bool gen7_is_valid_b_counter_addr(struct drm_i915_private *dev_priv, u32 addr)
 {
-	return (addr >= OASTARTTRIG1.reg && addr <= OASTARTTRIG8.reg) ||
-		(addr >= OAREPORTTRIG1.reg && addr <= OAREPORTTRIG8.reg) ||
-		(addr >= OACEC0_0.reg && addr <= OACEC7_1.reg);
+	return (addr >= i915_mmio_reg_offset(OASTARTTRIG1) &&
+		addr <= i915_mmio_reg_offset(OASTARTTRIG8)) ||
+		(addr >= i915_mmio_reg_offset(OAREPORTTRIG1) &&
+		 addr <= i915_mmio_reg_offset(OAREPORTTRIG8)) ||
+		(addr >= i915_mmio_reg_offset(OACEC0_0) &&
+		 addr <= i915_mmio_reg_offset(OACEC7_1));
 }
 
 static bool gen7_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
 {
-	return addr == HALF_SLICE_CHICKEN2.reg ||
-		(addr >= MICRO_BP0_0.reg && addr <= NOA_WRITE.reg) ||
-		(addr >= OA_PERFCNT1_LO.reg && addr <= OA_PERFCNT2_HI.reg) ||
-		(addr >= OA_PERFMATRIX_LO.reg && addr <= OA_PERFMATRIX_HI.reg);
+	return addr == i915_mmio_reg_offset(HALF_SLICE_CHICKEN2) ||
+		(addr >= i915_mmio_reg_offset(MICRO_BP0_0) &&
+		 addr <= i915_mmio_reg_offset(NOA_WRITE)) ||
+		(addr >= i915_mmio_reg_offset(OA_PERFCNT1_LO) &&
+		 addr <= i915_mmio_reg_offset(OA_PERFCNT2_HI)) ||
+		(addr >= i915_mmio_reg_offset(OA_PERFMATRIX_LO) &&
+		 addr <= i915_mmio_reg_offset(OA_PERFMATRIX_HI));
 }
 
 static bool gen8_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
 {
 	return gen7_is_valid_mux_addr(dev_priv, addr) ||
-		addr == WAIT_FOR_RC6_EXIT.reg ||
-		(addr >= RPM_CONFIG0.reg && addr <= NOA_CONFIG(8).reg);
+		addr == i915_mmio_reg_offset(WAIT_FOR_RC6_EXIT) ||
+		(addr >= i915_mmio_reg_offset(RPM_CONFIG0) &&
+		 addr <= i915_mmio_reg_offset(NOA_CONFIG(8)));
+}
+
+static bool gen10_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+	return gen8_is_valid_mux_addr(dev_priv, addr) ||
+		(addr >= i915_mmio_reg_offset(OA_PERFCNT3_LO) &&
+		 addr <= i915_mmio_reg_offset(OA_PERFCNT4_HI));
 }
 
 static bool hsw_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
 {
 	return gen7_is_valid_mux_addr(dev_priv, addr) ||
 		(addr >= 0x25100 && addr <= 0x2FF90) ||
-		addr == 0x9ec0;
+		(addr >= i915_mmio_reg_offset(HSW_MBVID2_NOA0) &&
+		 addr <= i915_mmio_reg_offset(HSW_MBVID2_NOA9)) ||
+		addr == i915_mmio_reg_offset(HSW_MBVID2_MISR0);
 }
 
 static bool chv_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
@@ -3038,14 +3055,14 @@ static uint32_t mask_reg_value(u32 reg, u32 val)
 	 * WaDisableSTUnitPowerOptimization workaround. Make sure the value
 	 * programmed by userspace doesn't change this.
 	 */
-	if (HALF_SLICE_CHICKEN2.reg == reg)
+	if (i915_mmio_reg_offset(HALF_SLICE_CHICKEN2) == reg)
 		val = val & ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE);
 
 	/* WAIT_FOR_RC6_EXIT has only one bit fullfilling the function
 	 * indicated by its name and a bunch of selection fields used by OA
 	 * configs.
 	 */
-	if (WAIT_FOR_RC6_EXIT.reg == reg)
+	if (i915_mmio_reg_offset(WAIT_FOR_RC6_EXIT) == reg)
 		val = val & ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE);
 
 	return val;
@@ -3392,8 +3409,6 @@ static struct ctl_table dev_root[] = {
  */
 void i915_perf_init(struct drm_i915_private *dev_priv)
 {
-	dev_priv->perf.oa.timestamp_frequency = 0;
-
 	if (IS_HASWELL(dev_priv)) {
 		dev_priv->perf.oa.ops.is_valid_b_counter_reg =
 			gen7_is_valid_b_counter_addr;
@@ -3409,70 +3424,68 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 		dev_priv->perf.oa.ops.oa_hw_tail_read =
 			gen7_oa_hw_tail_read;
 
-		dev_priv->perf.oa.timestamp_frequency = 12500000;
-
 		dev_priv->perf.oa.oa_formats = hsw_oa_formats;
-	} else if (i915_modparams.enable_execlists) {
+	} else if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
 		/* Note: that although we could theoretically also support the
 		 * legacy ringbuffer mode on BDW (and earlier iterations of
 		 * this driver, before upstreaming did this) it didn't seem
 		 * worth the complexity to maintain now that BDW+ enable
 		 * execlist mode by default.
 		 */
-		dev_priv->perf.oa.ops.is_valid_b_counter_reg =
-			gen7_is_valid_b_counter_addr;
-		dev_priv->perf.oa.ops.is_valid_mux_reg =
-			gen8_is_valid_mux_addr;
-		dev_priv->perf.oa.ops.is_valid_flex_reg =
-			gen8_is_valid_flex_addr;
+		dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats;
 
 		dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer;
-		dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set;
-		dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set;
 		dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable;
 		dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable;
 		dev_priv->perf.oa.ops.read = gen8_oa_read;
 		dev_priv->perf.oa.ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
 
-		dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats;
-
-		if (IS_GEN8(dev_priv)) {
-			dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120;
-			dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce;
+		if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv)) {
+			dev_priv->perf.oa.ops.is_valid_b_counter_reg =
+				gen7_is_valid_b_counter_addr;
+			dev_priv->perf.oa.ops.is_valid_mux_reg =
+				gen8_is_valid_mux_addr;
+			dev_priv->perf.oa.ops.is_valid_flex_reg =
+				gen8_is_valid_flex_addr;
 
-			dev_priv->perf.oa.timestamp_frequency = 12500000;
-
-			dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25);
 			if (IS_CHERRYVIEW(dev_priv)) {
 				dev_priv->perf.oa.ops.is_valid_mux_reg =
 					chv_is_valid_mux_addr;
 			}
-		} else if (IS_GEN9(dev_priv)) {
+
+			dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set;
+			dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set;
+
+			if (IS_GEN8(dev_priv)) {
+				dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120;
+				dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce;
+
+				dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25);
+			} else {
+				dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128;
+				dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de;
+
+				dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
+			}
+		} else if (IS_GEN10(dev_priv)) {
+			dev_priv->perf.oa.ops.is_valid_b_counter_reg =
+				gen7_is_valid_b_counter_addr;
+			dev_priv->perf.oa.ops.is_valid_mux_reg =
+				gen10_is_valid_mux_addr;
+			dev_priv->perf.oa.ops.is_valid_flex_reg =
+				gen8_is_valid_flex_addr;
+
+			dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set;
+			dev_priv->perf.oa.ops.disable_metric_set = gen10_disable_metric_set;
+
 			dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128;
 			dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de;
 
 			dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
-
-			switch (dev_priv->info.platform) {
-			case INTEL_BROXTON:
-			case INTEL_GEMINILAKE:
-				dev_priv->perf.oa.timestamp_frequency = 19200000;
-				break;
-			case INTEL_SKYLAKE:
-			case INTEL_KABYLAKE:
-			case INTEL_COFFEELAKE:
-				dev_priv->perf.oa.timestamp_frequency = 12000000;
-				break;
-			default:
-				/* Leave timestamp_frequency to 0 so we can
-				 * detect unsupported platforms.
-				 */
-				break;
-			}
 		}
 	}
 
-	if (dev_priv->perf.oa.timestamp_frequency) {
+	if (dev_priv->perf.oa.ops.enable_metric_set) {
 		hrtimer_init(&dev_priv->perf.oa.poll_check_timer,
 				CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 		dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb;
@@ -3482,8 +3495,8 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
 		mutex_init(&dev_priv->perf.lock);
 		spin_lock_init(&dev_priv->perf.oa.oa_buffer.ptr_lock);
 
-		oa_sample_rate_hard_limit =
-			dev_priv->perf.oa.timestamp_frequency / 2;
+		oa_sample_rate_hard_limit = 1000 *
+			(INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz / 2);
 		dev_priv->perf.sysctl_header = register_sysctl_table(dev_root);
 
 		mutex_init(&dev_priv->perf.metrics_lock);
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
new file mode 100644
index 000000000000..0e9b98c32b62
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_pmu.c
@@ -0,0 +1,920 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/perf_event.h>
+#include <linux/pm_runtime.h>
+
+#include "i915_drv.h"
+#include "i915_pmu.h"
+#include "intel_ringbuffer.h"
+
+/* Frequency for the sampling timer for events which need it. */
+#define FREQUENCY 200
+#define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
+
+#define ENGINE_SAMPLE_MASK \
+	(BIT(I915_SAMPLE_BUSY) | \
+	 BIT(I915_SAMPLE_WAIT) | \
+	 BIT(I915_SAMPLE_SEMA))
+
+#define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)
+
+static cpumask_t i915_pmu_cpumask;
+
+static u8 engine_config_sample(u64 config)
+{
+	return config & I915_PMU_SAMPLE_MASK;
+}
+
+static u8 engine_event_sample(struct perf_event *event)
+{
+	return engine_config_sample(event->attr.config);
+}
+
+static u8 engine_event_class(struct perf_event *event)
+{
+	return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
+}
+
+static u8 engine_event_instance(struct perf_event *event)
+{
+	return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
+}
+
+static bool is_engine_config(u64 config)
+{
+	return config < __I915_PMU_OTHER(0);
+}
+
+static unsigned int config_enabled_bit(u64 config)
+{
+	if (is_engine_config(config))
+		return engine_config_sample(config);
+	else
+		return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
+}
+
+static u64 config_enabled_mask(u64 config)
+{
+	return BIT_ULL(config_enabled_bit(config));
+}
+
+static bool is_engine_event(struct perf_event *event)
+{
+	return is_engine_config(event->attr.config);
+}
+
+static unsigned int event_enabled_bit(struct perf_event *event)
+{
+	return config_enabled_bit(event->attr.config);
+}
+
+static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active)
+{
+	u64 enable;
+
+	/*
+	 * Only some counters need the sampling timer.
+	 *
+	 * We start with a bitmask of all currently enabled events.
+	 */
+	enable = i915->pmu.enable;
+
+	/*
+	 * Mask out all the ones which do not need the timer, or in
+	 * other words keep all the ones that could need the timer.
+	 */
+	enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
+		  config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
+		  ENGINE_SAMPLE_MASK;
+
+	/*
+	 * When the GPU is idle per-engine counters do not need to be
+	 * running so clear those bits out.
+	 */
+	if (!gpu_active)
+		enable &= ~ENGINE_SAMPLE_MASK;
+	/*
+	 * Also there is software busyness tracking available we do not
+	 * need the timer for I915_SAMPLE_BUSY counter.
+	 *
+	 * Use RCS as proxy for all engines.
+	 */
+	else if (intel_engine_supports_stats(i915->engine[RCS]))
+		enable &= ~BIT(I915_SAMPLE_BUSY);
+
+	/*
+	 * If some bits remain it means we need the sampling timer running.
+	 */
+	return enable;
+}
+
+void i915_pmu_gt_parked(struct drm_i915_private *i915)
+{
+	if (!i915->pmu.base.event_init)
+		return;
+
+	spin_lock_irq(&i915->pmu.lock);
+	/*
+	 * Signal sampling timer to stop if only engine events are enabled and
+	 * GPU went idle.
+	 */
+	i915->pmu.timer_enabled = pmu_needs_timer(i915, false);
+	spin_unlock_irq(&i915->pmu.lock);
+}
+
+static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915)
+{
+	if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) {
+		i915->pmu.timer_enabled = true;
+		hrtimer_start_range_ns(&i915->pmu.timer,
+				       ns_to_ktime(PERIOD), 0,
+				       HRTIMER_MODE_REL_PINNED);
+	}
+}
+
+void i915_pmu_gt_unparked(struct drm_i915_private *i915)
+{
+	if (!i915->pmu.base.event_init)
+		return;
+
+	spin_lock_irq(&i915->pmu.lock);
+	/*
+	 * Re-enable sampling timer when GPU goes active.
+	 */
+	__i915_pmu_maybe_start_timer(i915);
+	spin_unlock_irq(&i915->pmu.lock);
+}
+
+static bool grab_forcewake(struct drm_i915_private *i915, bool fw)
+{
+	if (!fw)
+		intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
+
+	return true;
+}
+
+static void
+update_sample(struct i915_pmu_sample *sample, u32 unit, u32 val)
+{
+	sample->cur += mul_u32_u32(val, unit);
+}
+
+static void engines_sample(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	bool fw = false;
+
+	if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
+		return;
+
+	if (!dev_priv->gt.awake)
+		return;
+
+	if (!intel_runtime_pm_get_if_in_use(dev_priv))
+		return;
+
+	for_each_engine(engine, dev_priv, id) {
+		u32 current_seqno = intel_engine_get_seqno(engine);
+		u32 last_seqno = intel_engine_last_submit(engine);
+		u32 val;
+
+		val = !i915_seqno_passed(current_seqno, last_seqno);
+
+		update_sample(&engine->pmu.sample[I915_SAMPLE_BUSY],
+			      PERIOD, val);
+
+		if (val && (engine->pmu.enable &
+		    (BIT(I915_SAMPLE_WAIT) | BIT(I915_SAMPLE_SEMA)))) {
+			fw = grab_forcewake(dev_priv, fw);
+
+			val = I915_READ_FW(RING_CTL(engine->mmio_base));
+		} else {
+			val = 0;
+		}
+
+		update_sample(&engine->pmu.sample[I915_SAMPLE_WAIT],
+			      PERIOD, !!(val & RING_WAIT));
+
+		update_sample(&engine->pmu.sample[I915_SAMPLE_SEMA],
+			      PERIOD, !!(val & RING_WAIT_SEMAPHORE));
+	}
+
+	if (fw)
+		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+
+	intel_runtime_pm_put(dev_priv);
+}
+
+static void frequency_sample(struct drm_i915_private *dev_priv)
+{
+	if (dev_priv->pmu.enable &
+	    config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
+		u32 val;
+
+		val = dev_priv->gt_pm.rps.cur_freq;
+		if (dev_priv->gt.awake &&
+		    intel_runtime_pm_get_if_in_use(dev_priv)) {
+			val = intel_get_cagf(dev_priv,
+					     I915_READ_NOTRACE(GEN6_RPSTAT1));
+			intel_runtime_pm_put(dev_priv);
+		}
+
+		update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT],
+			      1, intel_gpu_freq(dev_priv, val));
+	}
+
+	if (dev_priv->pmu.enable &
+	    config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
+		update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ], 1,
+			      intel_gpu_freq(dev_priv,
+					     dev_priv->gt_pm.rps.cur_freq));
+	}
+}
+
+static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
+{
+	struct drm_i915_private *i915 =
+		container_of(hrtimer, struct drm_i915_private, pmu.timer);
+
+	if (!READ_ONCE(i915->pmu.timer_enabled))
+		return HRTIMER_NORESTART;
+
+	engines_sample(i915);
+	frequency_sample(i915);
+
+	hrtimer_forward_now(hrtimer, ns_to_ktime(PERIOD));
+	return HRTIMER_RESTART;
+}
+
+static u64 count_interrupts(struct drm_i915_private *i915)
+{
+	/* open-coded kstat_irqs() */
+	struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq);
+	u64 sum = 0;
+	int cpu;
+
+	if (!desc || !desc->kstat_irqs)
+		return 0;
+
+	for_each_possible_cpu(cpu)
+		sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
+
+	return sum;
+}
+
+static void engine_event_destroy(struct perf_event *event)
+{
+	struct drm_i915_private *i915 =
+		container_of(event->pmu, typeof(*i915), pmu.base);
+	struct intel_engine_cs *engine;
+
+	engine = intel_engine_lookup_user(i915,
+					  engine_event_class(event),
+					  engine_event_instance(event));
+	if (WARN_ON_ONCE(!engine))
+		return;
+
+	if (engine_event_sample(event) == I915_SAMPLE_BUSY &&
+	    intel_engine_supports_stats(engine))
+		intel_disable_engine_stats(engine);
+}
+
+static void i915_pmu_event_destroy(struct perf_event *event)
+{
+	WARN_ON(event->parent);
+
+	if (is_engine_event(event))
+		engine_event_destroy(event);
+}
+
+static int
+engine_event_status(struct intel_engine_cs *engine,
+		    enum drm_i915_pmu_engine_sample sample)
+{
+	switch (sample) {
+	case I915_SAMPLE_BUSY:
+	case I915_SAMPLE_WAIT:
+		break;
+	case I915_SAMPLE_SEMA:
+		if (INTEL_GEN(engine->i915) < 6)
+			return -ENODEV;
+		break;
+	default:
+		return -ENOENT;
+	}
+
+	return 0;
+}
+
+static int engine_event_init(struct perf_event *event)
+{
+	struct drm_i915_private *i915 =
+		container_of(event->pmu, typeof(*i915), pmu.base);
+	struct intel_engine_cs *engine;
+	u8 sample;
+	int ret;
+
+	engine = intel_engine_lookup_user(i915, engine_event_class(event),
+					  engine_event_instance(event));
+	if (!engine)
+		return -ENODEV;
+
+	sample = engine_event_sample(event);
+	ret = engine_event_status(engine, sample);
+	if (ret)
+		return ret;
+
+	if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine))
+		ret = intel_enable_engine_stats(engine);
+
+	return ret;
+}
+
+static int i915_pmu_event_init(struct perf_event *event)
+{
+	struct drm_i915_private *i915 =
+		container_of(event->pmu, typeof(*i915), pmu.base);
+	int ret;
+
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	/* unsupported modes and filters */
+	if (event->attr.sample_period) /* no sampling */
+		return -EINVAL;
+
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
+	if (event->cpu < 0)
+		return -EINVAL;
+
+	/* only allow running on one cpu at a time */
+	if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
+		return -EINVAL;
+
+	if (is_engine_event(event)) {
+		ret = engine_event_init(event);
+	} else {
+		ret = 0;
+		switch (event->attr.config) {
+		case I915_PMU_ACTUAL_FREQUENCY:
+			if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
+				 /* Requires a mutex for sampling! */
+				ret = -ENODEV;
+		case I915_PMU_REQUESTED_FREQUENCY:
+			if (INTEL_GEN(i915) < 6)
+				ret = -ENODEV;
+			break;
+		case I915_PMU_INTERRUPTS:
+			break;
+		case I915_PMU_RC6_RESIDENCY:
+			if (!HAS_RC6(i915))
+				ret = -ENODEV;
+			break;
+		default:
+			ret = -ENOENT;
+			break;
+		}
+	}
+	if (ret)
+		return ret;
+
+	if (!event->parent)
+		event->destroy = i915_pmu_event_destroy;
+
+	return 0;
+}
+
+static u64 __get_rc6(struct drm_i915_private *i915)
+{
+	u64 val;
+
+	val = intel_rc6_residency_ns(i915,
+				     IS_VALLEYVIEW(i915) ?
+				     VLV_GT_RENDER_RC6 :
+				     GEN6_GT_GFX_RC6);
+
+	if (HAS_RC6p(i915))
+		val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);
+
+	if (HAS_RC6pp(i915))
+		val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);
+
+	return val;
+}
+
+static u64 get_rc6(struct drm_i915_private *i915, bool locked)
+{
+#if IS_ENABLED(CONFIG_PM)
+	unsigned long flags;
+	u64 val;
+
+	if (intel_runtime_pm_get_if_in_use(i915)) {
+		val = __get_rc6(i915);
+		intel_runtime_pm_put(i915);
+
+		/*
+		 * If we are coming back from being runtime suspended we must
+		 * be careful not to report a larger value than returned
+		 * previously.
+		 */
+
+		if (!locked)
+			spin_lock_irqsave(&i915->pmu.lock, flags);
+
+		if (val >= i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
+			i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = 0;
+			i915->pmu.sample[__I915_SAMPLE_RC6].cur = val;
+		} else {
+			val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
+		}
+
+		if (!locked)
+			spin_unlock_irqrestore(&i915->pmu.lock, flags);
+	} else {
+		struct pci_dev *pdev = i915->drm.pdev;
+		struct device *kdev = &pdev->dev;
+		unsigned long flags2;
+
+		/*
+		 * We are runtime suspended.
+		 *
+		 * Report the delta from when the device was suspended to now,
+		 * on top of the last known real value, as the approximated RC6
+		 * counter value.
+		 */
+		if (!locked)
+			spin_lock_irqsave(&i915->pmu.lock, flags);
+
+		spin_lock_irqsave(&kdev->power.lock, flags2);
+
+		if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur)
+			i915->pmu.suspended_jiffies_last =
+						kdev->power.suspended_jiffies;
+
+		val = kdev->power.suspended_jiffies -
+		      i915->pmu.suspended_jiffies_last;
+		val += jiffies - kdev->power.accounting_timestamp;
+
+		spin_unlock_irqrestore(&kdev->power.lock, flags2);
+
+		val = jiffies_to_nsecs(val);
+		val += i915->pmu.sample[__I915_SAMPLE_RC6].cur;
+		i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val;
+
+		if (!locked)
+			spin_unlock_irqrestore(&i915->pmu.lock, flags);
+	}
+
+	return val;
+#else
+	return __get_rc6(i915);
+#endif
+}
+
+static u64 __i915_pmu_event_read(struct perf_event *event, bool locked)
+{
+	struct drm_i915_private *i915 =
+		container_of(event->pmu, typeof(*i915), pmu.base);
+	u64 val = 0;
+
+	if (is_engine_event(event)) {
+		u8 sample = engine_event_sample(event);
+		struct intel_engine_cs *engine;
+
+		engine = intel_engine_lookup_user(i915,
+						  engine_event_class(event),
+						  engine_event_instance(event));
+
+		if (WARN_ON_ONCE(!engine)) {
+			/* Do nothing */
+		} else if (sample == I915_SAMPLE_BUSY &&
+			   intel_engine_supports_stats(engine)) {
+			val = ktime_to_ns(intel_engine_get_busy_time(engine));
+		} else {
+			val = engine->pmu.sample[sample].cur;
+		}
+	} else {
+		switch (event->attr.config) {
+		case I915_PMU_ACTUAL_FREQUENCY:
+			val =
+			   div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur,
+				   FREQUENCY);
+			break;
+		case I915_PMU_REQUESTED_FREQUENCY:
+			val =
+			   div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur,
+				   FREQUENCY);
+			break;
+		case I915_PMU_INTERRUPTS:
+			val = count_interrupts(i915);
+			break;
+		case I915_PMU_RC6_RESIDENCY:
+			val = get_rc6(i915, locked);
+			break;
+		}
+	}
+
+	return val;
+}
+
+static void i915_pmu_event_read(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	u64 prev, new;
+
+again:
+	prev = local64_read(&hwc->prev_count);
+	new = __i915_pmu_event_read(event, false);
+
+	if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
+		goto again;
+
+	local64_add(new - prev, &event->count);
+}
+
+static void i915_pmu_enable(struct perf_event *event)
+{
+	struct drm_i915_private *i915 =
+		container_of(event->pmu, typeof(*i915), pmu.base);
+	unsigned int bit = event_enabled_bit(event);
+	unsigned long flags;
+
+	spin_lock_irqsave(&i915->pmu.lock, flags);
+
+	/*
+	 * Update the bitmask of enabled events and increment
+	 * the event reference counter.
+	 */
+	GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
+	GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0);
+	i915->pmu.enable |= BIT_ULL(bit);
+	i915->pmu.enable_count[bit]++;
+
+	/*
+	 * Start the sampling timer if needed and not already enabled.
+	 */
+	__i915_pmu_maybe_start_timer(i915);
+
+	/*
+	 * For per-engine events the bitmask and reference counting
+	 * is stored per engine.
+	 */
+	if (is_engine_event(event)) {
+		u8 sample = engine_event_sample(event);
+		struct intel_engine_cs *engine;
+
+		engine = intel_engine_lookup_user(i915,
+						  engine_event_class(event),
+						  engine_event_instance(event));
+		GEM_BUG_ON(!engine);
+		engine->pmu.enable |= BIT(sample);
+
+		GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
+		GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
+		engine->pmu.enable_count[sample]++;
+	}
+
+	/*
+	 * Store the current counter value so we can report the correct delta
+	 * for all listeners. Even when the event was already enabled and has
+	 * an existing non-zero value.
+	 */
+	local64_set(&event->hw.prev_count, __i915_pmu_event_read(event, true));
+
+	spin_unlock_irqrestore(&i915->pmu.lock, flags);
+}
+
+static void i915_pmu_disable(struct perf_event *event)
+{
+	struct drm_i915_private *i915 =
+		container_of(event->pmu, typeof(*i915), pmu.base);
+	unsigned int bit = event_enabled_bit(event);
+	unsigned long flags;
+
+	spin_lock_irqsave(&i915->pmu.lock, flags);
+
+	if (is_engine_event(event)) {
+		u8 sample = engine_event_sample(event);
+		struct intel_engine_cs *engine;
+
+		engine = intel_engine_lookup_user(i915,
+						  engine_event_class(event),
+						  engine_event_instance(event));
+		GEM_BUG_ON(!engine);
+		GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
+		GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
+		/*
+		 * Decrement the reference count and clear the enabled
+		 * bitmask when the last listener on an event goes away.
+		 */
+		if (--engine->pmu.enable_count[sample] == 0)
+			engine->pmu.enable &= ~BIT(sample);
+	}
+
+	GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
+	GEM_BUG_ON(i915->pmu.enable_count[bit] == 0);
+	/*
+	 * Decrement the reference count and clear the enabled
+	 * bitmask when the last listener on an event goes away.
+	 */
+	if (--i915->pmu.enable_count[bit] == 0) {
+		i915->pmu.enable &= ~BIT_ULL(bit);
+		i915->pmu.timer_enabled &= pmu_needs_timer(i915, true);
+	}
+
+	spin_unlock_irqrestore(&i915->pmu.lock, flags);
+}
+
+static void i915_pmu_event_start(struct perf_event *event, int flags)
+{
+	i915_pmu_enable(event);
+	event->hw.state = 0;
+}
+
+static void i915_pmu_event_stop(struct perf_event *event, int flags)
+{
+	if (flags & PERF_EF_UPDATE)
+		i915_pmu_event_read(event);
+	i915_pmu_disable(event);
+	event->hw.state = PERF_HES_STOPPED;
+}
+
+static int i915_pmu_event_add(struct perf_event *event, int flags)
+{
+	if (flags & PERF_EF_START)
+		i915_pmu_event_start(event, flags);
+
+	return 0;
+}
+
+static void i915_pmu_event_del(struct perf_event *event, int flags)
+{
+	i915_pmu_event_stop(event, PERF_EF_UPDATE);
+}
+
+static int i915_pmu_event_event_idx(struct perf_event *event)
+{
+	return 0;
+}
+
+struct i915_str_attribute {
+	struct device_attribute attr;
+	const char *str;
+};
+
+static ssize_t i915_pmu_format_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct i915_str_attribute *eattr;
+
+	eattr = container_of(attr, struct i915_str_attribute, attr);
+	return sprintf(buf, "%s\n", eattr->str);
+}
+
+#define I915_PMU_FORMAT_ATTR(_name, _config) \
+	(&((struct i915_str_attribute[]) { \
+		{ .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
+		  .str = _config, } \
+	})[0].attr.attr)
+
+static struct attribute *i915_pmu_format_attrs[] = {
+	I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
+	NULL,
+};
+
+static const struct attribute_group i915_pmu_format_attr_group = {
+	.name = "format",
+	.attrs = i915_pmu_format_attrs,
+};
+
+struct i915_ext_attribute {
+	struct device_attribute attr;
+	unsigned long val;
+};
+
+static ssize_t i915_pmu_event_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct i915_ext_attribute *eattr;
+
+	eattr = container_of(attr, struct i915_ext_attribute, attr);
+	return sprintf(buf, "config=0x%lx\n", eattr->val);
+}
+
+#define I915_EVENT_ATTR(_name, _config) \
+	(&((struct i915_ext_attribute[]) { \
+		{ .attr = __ATTR(_name, 0444, i915_pmu_event_show, NULL), \
+		  .val = _config, } \
+	})[0].attr.attr)
+
+#define I915_EVENT_STR(_name, _str) \
+	(&((struct perf_pmu_events_attr[]) { \
+		{ .attr	     = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
+		  .id	     = 0, \
+		  .event_str = _str, } \
+	})[0].attr.attr)
+
+#define I915_EVENT(_name, _config, _unit) \
+	I915_EVENT_ATTR(_name, _config), \
+	I915_EVENT_STR(_name.unit, _unit)
+
+#define I915_ENGINE_EVENT(_name, _class, _instance, _sample) \
+	I915_EVENT_ATTR(_name, __I915_PMU_ENGINE(_class, _instance, _sample)), \
+	I915_EVENT_STR(_name.unit, "ns")
+
+#define I915_ENGINE_EVENTS(_name, _class, _instance) \
+	I915_ENGINE_EVENT(_name##_instance-busy, _class, _instance, I915_SAMPLE_BUSY), \
+	I915_ENGINE_EVENT(_name##_instance-sema, _class, _instance, I915_SAMPLE_SEMA), \
+	I915_ENGINE_EVENT(_name##_instance-wait, _class, _instance, I915_SAMPLE_WAIT)
+
+static struct attribute *i915_pmu_events_attrs[] = {
+	I915_ENGINE_EVENTS(rcs, I915_ENGINE_CLASS_RENDER, 0),
+	I915_ENGINE_EVENTS(bcs, I915_ENGINE_CLASS_COPY, 0),
+	I915_ENGINE_EVENTS(vcs, I915_ENGINE_CLASS_VIDEO, 0),
+	I915_ENGINE_EVENTS(vcs, I915_ENGINE_CLASS_VIDEO, 1),
+	I915_ENGINE_EVENTS(vecs, I915_ENGINE_CLASS_VIDEO_ENHANCE, 0),
+
+	I915_EVENT(actual-frequency,    I915_PMU_ACTUAL_FREQUENCY,    "MHz"),
+	I915_EVENT(requested-frequency, I915_PMU_REQUESTED_FREQUENCY, "MHz"),
+
+	I915_EVENT_ATTR(interrupts, I915_PMU_INTERRUPTS),
+
+	I915_EVENT(rc6-residency,   I915_PMU_RC6_RESIDENCY,   "ns"),
+
+	NULL,
+};
+
+static const struct attribute_group i915_pmu_events_attr_group = {
+	.name = "events",
+	.attrs = i915_pmu_events_attrs,
+};
+
+static ssize_t
+i915_pmu_get_attr_cpumask(struct device *dev,
+			  struct device_attribute *attr,
+			  char *buf)
+{
+	return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
+}
+
+static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL);
+
+static struct attribute *i915_cpumask_attrs[] = {
+	&dev_attr_cpumask.attr,
+	NULL,
+};
+
+static struct attribute_group i915_pmu_cpumask_attr_group = {
+	.attrs = i915_cpumask_attrs,
+};
+
+static const struct attribute_group *i915_pmu_attr_groups[] = {
+	&i915_pmu_format_attr_group,
+	&i915_pmu_events_attr_group,
+	&i915_pmu_cpumask_attr_group,
+	NULL
+};
+
+static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
+{
+	struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
+
+	GEM_BUG_ON(!pmu->base.event_init);
+
+	/* Select the first online CPU as a designated reader. */
+	if (!cpumask_weight(&i915_pmu_cpumask))
+		cpumask_set_cpu(cpu, &i915_pmu_cpumask);
+
+	return 0;
+}
+
+static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
+{
+	struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
+	unsigned int target;
+
+	GEM_BUG_ON(!pmu->base.event_init);
+
+	if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
+		target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
+		/* Migrate events if there is a valid target */
+		if (target < nr_cpu_ids) {
+			cpumask_set_cpu(target, &i915_pmu_cpumask);
+			perf_pmu_migrate_context(&pmu->base, cpu, target);
+		}
+	}
+
+	return 0;
+}
+
+static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
+
+static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915)
+{
+	enum cpuhp_state slot;
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+				      "perf/x86/intel/i915:online",
+				      i915_pmu_cpu_online,
+				      i915_pmu_cpu_offline);
+	if (ret < 0)
+		return ret;
+
+	slot = ret;
+	ret = cpuhp_state_add_instance(slot, &i915->pmu.node);
+	if (ret) {
+		cpuhp_remove_multi_state(slot);
+		return ret;
+	}
+
+	cpuhp_slot = slot;
+	return 0;
+}
+
+static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915)
+{
+	WARN_ON(cpuhp_slot == CPUHP_INVALID);
+	WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node));
+	cpuhp_remove_multi_state(cpuhp_slot);
+}
+
+void i915_pmu_register(struct drm_i915_private *i915)
+{
+	int ret;
+
+	if (INTEL_GEN(i915) <= 2) {
+		DRM_INFO("PMU not supported for this GPU.");
+		return;
+	}
+
+	i915->pmu.base.attr_groups	= i915_pmu_attr_groups;
+	i915->pmu.base.task_ctx_nr	= perf_invalid_context;
+	i915->pmu.base.event_init	= i915_pmu_event_init;
+	i915->pmu.base.add		= i915_pmu_event_add;
+	i915->pmu.base.del		= i915_pmu_event_del;
+	i915->pmu.base.start		= i915_pmu_event_start;
+	i915->pmu.base.stop		= i915_pmu_event_stop;
+	i915->pmu.base.read		= i915_pmu_event_read;
+	i915->pmu.base.event_idx	= i915_pmu_event_event_idx;
+
+	spin_lock_init(&i915->pmu.lock);
+	hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	i915->pmu.timer.function = i915_sample;
+
+	ret = perf_pmu_register(&i915->pmu.base, "i915", -1);
+	if (ret)
+		goto err;
+
+	ret = i915_pmu_register_cpuhp_state(i915);
+	if (ret)
+		goto err_unreg;
+
+	return;
+
+err_unreg:
+	perf_pmu_unregister(&i915->pmu.base);
+err:
+	i915->pmu.base.event_init = NULL;
+	DRM_NOTE("Failed to register PMU! (err=%d)\n", ret);
+}
+
+void i915_pmu_unregister(struct drm_i915_private *i915)
+{
+	if (!i915->pmu.base.event_init)
+		return;
+
+	WARN_ON(i915->pmu.enable);
+
+	hrtimer_cancel(&i915->pmu.timer);
+
+	i915_pmu_unregister_cpuhp_state(i915);
+
+	perf_pmu_unregister(&i915->pmu.base);
+	i915->pmu.base.event_init = NULL;
+}
diff --git a/drivers/gpu/drm/i915/i915_pmu.h b/drivers/gpu/drm/i915/i915_pmu.h
new file mode 100644
index 000000000000..bb62df15afa4
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_pmu.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+#ifndef __I915_PMU_H__
+#define __I915_PMU_H__
+
+enum {
+	__I915_SAMPLE_FREQ_ACT = 0,
+	__I915_SAMPLE_FREQ_REQ,
+	__I915_SAMPLE_RC6,
+	__I915_SAMPLE_RC6_ESTIMATED,
+	__I915_NUM_PMU_SAMPLERS
+};
+
+/**
+ * How many different events we track in the global PMU mask.
+ *
+ * It is also used to know to needed number of event reference counters.
+ */
+#define I915_PMU_MASK_BITS \
+	((1 << I915_PMU_SAMPLE_BITS) + \
+	 (I915_PMU_LAST + 1 - __I915_PMU_OTHER(0)))
+
+struct i915_pmu_sample {
+	u64 cur;
+};
+
+struct i915_pmu {
+	/**
+	 * @node: List node for CPU hotplug handling.
+	 */
+	struct hlist_node node;
+	/**
+	 * @base: PMU base.
+	 */
+	struct pmu base;
+	/**
+	 * @lock: Lock protecting enable mask and ref count handling.
+	 */
+	spinlock_t lock;
+	/**
+	 * @timer: Timer for internal i915 PMU sampling.
+	 */
+	struct hrtimer timer;
+	/**
+	 * @enable: Bitmask of all currently enabled events.
+	 *
+	 * Bits are derived from uAPI event numbers in a way that low 16 bits
+	 * correspond to engine event _sample_ _type_ (I915_SAMPLE_QUEUED is
+	 * bit 0), and higher bits correspond to other events (for instance
+	 * I915_PMU_ACTUAL_FREQUENCY is bit 16 etc).
+	 *
+	 * In other words, low 16 bits are not per engine but per engine
+	 * sampler type, while the upper bits are directly mapped to other
+	 * event types.
+	 */
+	u64 enable;
+	/**
+	 * @enable_count: Reference counts for the enabled events.
+	 *
+	 * Array indices are mapped in the same way as bits in the @enable field
+	 * and they are used to control sampling on/off when multiple clients
+	 * are using the PMU API.
+	 */
+	unsigned int enable_count[I915_PMU_MASK_BITS];
+	/**
+	 * @timer_enabled: Should the internal sampling timer be running.
+	 */
+	bool timer_enabled;
+	/**
+	 * @sample: Current and previous (raw) counters for sampling events.
+	 *
+	 * These counters are updated from the i915 PMU sampling timer.
+	 *
+	 * Only global counters are held here, while the per-engine ones are in
+	 * struct intel_engine_cs.
+	 */
+	struct i915_pmu_sample sample[__I915_NUM_PMU_SAMPLERS];
+	/**
+	 * @suspended_jiffies_last: Cached suspend time from PM core.
+	 */
+	unsigned long suspended_jiffies_last;
+};
+
+#ifdef CONFIG_PERF_EVENTS
+void i915_pmu_register(struct drm_i915_private *i915);
+void i915_pmu_unregister(struct drm_i915_private *i915);
+void i915_pmu_gt_parked(struct drm_i915_private *i915);
+void i915_pmu_gt_unparked(struct drm_i915_private *i915);
+#else
+static inline void i915_pmu_register(struct drm_i915_private *i915) {}
+static inline void i915_pmu_unregister(struct drm_i915_private *i915) {}
+static inline void i915_pmu_gt_parked(struct drm_i915_private *i915) {}
+static inline void i915_pmu_gt_unparked(struct drm_i915_private *i915) {}
+#endif
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 7923dfd9963c..33eb0c5b1d32 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -186,6 +186,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define VIDEO_ENHANCEMENT_CLASS	2
 #define COPY_ENGINE_CLASS	3
 #define OTHER_CLASS		4
+#define MAX_ENGINE_CLASS	4
+
+#define MAX_ENGINE_INSTANCE    1
 
 /* PCI config space */
 
@@ -355,9 +358,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define   ECOCHK_PPGTT_WT_HSW		(0x2<<3)
 #define   ECOCHK_PPGTT_WB_HSW		(0x3<<3)
 
-#define GEN8_CONFIG0			_MMIO(0xD00)
-#define  GEN9_DEFAULT_FIXES		(1 << 3 | 1 << 2 | 1 << 1)
-
 #define GAC_ECO_BITS			_MMIO(0x14090)
 #define   ECOBITS_SNB_BIT		(1<<13)
 #define   ECOBITS_PPGTT_CACHE64B	(3<<8)
@@ -382,6 +382,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define GEN8_STOLEN_RESERVED_2M		(1 << 7)
 #define GEN8_STOLEN_RESERVED_4M		(2 << 7)
 #define GEN8_STOLEN_RESERVED_8M		(3 << 7)
+#define GEN6_STOLEN_RESERVED_ENABLE	(1 << 0)
 
 /* VGA stuff */
 
@@ -1109,16 +1110,50 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OA_PERFCNT1_HI      _MMIO(0x91BC)
 #define OA_PERFCNT2_LO      _MMIO(0x91C0)
 #define OA_PERFCNT2_HI      _MMIO(0x91C4)
+#define OA_PERFCNT3_LO      _MMIO(0x91C8)
+#define OA_PERFCNT3_HI      _MMIO(0x91CC)
+#define OA_PERFCNT4_LO      _MMIO(0x91D8)
+#define OA_PERFCNT4_HI      _MMIO(0x91DC)
 
 #define OA_PERFMATRIX_LO    _MMIO(0x91C8)
 #define OA_PERFMATRIX_HI    _MMIO(0x91CC)
 
 /* RPM unit config (Gen8+) */
 #define RPM_CONFIG0	    _MMIO(0x0D00)
-#define RPM_CONFIG1	    _MMIO(0x0D04)
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT	3
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK	(1 << GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT)
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ	0
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ	1
+#define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT	1
+#define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK	(0x3 << GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT)
 
-/* RPC unit config (Gen8+) */
-#define RPM_CONFIG	    _MMIO(0x0D08)
+#define RPM_CONFIG1	    _MMIO(0x0D04)
+#define  GEN10_GT_NOA_ENABLE  (1 << 9)
+
+/* GPM unit config (Gen9+) */
+#define CTC_MODE			_MMIO(0xA26C)
+#define  CTC_SOURCE_PARAMETER_MASK 1
+#define  CTC_SOURCE_CRYSTAL_CLOCK	0
+#define  CTC_SOURCE_DIVIDE_LOGIC	1
+#define  CTC_SHIFT_PARAMETER_SHIFT	1
+#define  CTC_SHIFT_PARAMETER_MASK	(0x3 << CTC_SHIFT_PARAMETER_SHIFT)
+
+/* RCP unit config (Gen8+) */
+#define RCP_CONFIG	    _MMIO(0x0D08)
+
+/* NOA (HSW) */
+#define HSW_MBVID2_NOA0		_MMIO(0x9E80)
+#define HSW_MBVID2_NOA1		_MMIO(0x9E84)
+#define HSW_MBVID2_NOA2		_MMIO(0x9E88)
+#define HSW_MBVID2_NOA3		_MMIO(0x9E8C)
+#define HSW_MBVID2_NOA4		_MMIO(0x9E90)
+#define HSW_MBVID2_NOA5		_MMIO(0x9E94)
+#define HSW_MBVID2_NOA6		_MMIO(0x9E98)
+#define HSW_MBVID2_NOA7		_MMIO(0x9E9C)
+#define HSW_MBVID2_NOA8		_MMIO(0x9EA0)
+#define HSW_MBVID2_NOA9		_MMIO(0x9EA4)
+
+#define HSW_MBVID2_MISR0	_MMIO(0x9EC0)
 
 /* NOA (Gen8+) */
 #define NOA_CONFIG(i)	    _MMIO(0x0D0C + (i) * 4)
@@ -1992,7 +2027,7 @@ enum i915_power_well_id {
 #define _CNL_PORT_TX_DW5_LN0_AE		0x162454
 #define _CNL_PORT_TX_DW5_LN0_B		0x162654
 #define _CNL_PORT_TX_DW5_LN0_C		0x162C54
-#define _CNL_PORT_TX_DW5_LN0_D		0x162ED4
+#define _CNL_PORT_TX_DW5_LN0_D		0x162E54
 #define _CNL_PORT_TX_DW5_LN0_F		0x162854
 #define CNL_PORT_TX_DW5_GRP(port)	_MMIO_PORT6(port, \
 						    _CNL_PORT_TX_DW5_GRP_AE, \
@@ -2023,7 +2058,7 @@ enum i915_power_well_id {
 #define _CNL_PORT_TX_DW7_LN0_AE		0x16245C
 #define _CNL_PORT_TX_DW7_LN0_B		0x16265C
 #define _CNL_PORT_TX_DW7_LN0_C		0x162C5C
-#define _CNL_PORT_TX_DW7_LN0_D		0x162EDC
+#define _CNL_PORT_TX_DW7_LN0_D		0x162E5C
 #define _CNL_PORT_TX_DW7_LN0_F		0x16285C
 #define CNL_PORT_TX_DW7_GRP(port)	_MMIO_PORT6(port, \
 						    _CNL_PORT_TX_DW7_GRP_AE, \
@@ -2329,6 +2364,8 @@ enum i915_power_well_id {
 #define   ARB_MODE_SWIZZLE_BDW	(1<<1)
 #define RENDER_HWS_PGA_GEN7	_MMIO(0x04080)
 #define RING_FAULT_REG(engine)	_MMIO(0x4094 + 0x100*(engine)->hw_id)
+#define GEN8_RING_FAULT_REG	_MMIO(0x4094)
+#define   GEN8_RING_FAULT_ENGINE_ID(x)	(((x) >> 12) & 0x7)
 #define   RING_FAULT_GTTSEL_MASK (1<<11)
 #define   RING_FAULT_SRCID(x)	(((x) >> 3) & 0xff)
 #define   RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3)
@@ -2452,6 +2489,8 @@ enum i915_power_well_id {
 
 #define GEN8_FAULT_TLB_DATA0		_MMIO(0x4b10)
 #define GEN8_FAULT_TLB_DATA1		_MMIO(0x4b14)
+#define   FAULT_VA_HIGH_BITS		(0xf << 0)
+#define   FAULT_GTT_SEL			(1 << 4)
 
 #define FPGA_DBG		_MMIO(0x42300)
 #define   FPGA_DBG_RM_NOCLAIM	(1<<31)
@@ -3241,6 +3280,7 @@ enum i915_power_well_id {
 # define AUDUNIT_CLOCK_GATE_DISABLE		(1 << 26) /* 965 */
 # define DPUNIT_A_CLOCK_GATE_DISABLE		(1 << 25) /* 965 */
 # define DPCUNIT_CLOCK_GATE_DISABLE		(1 << 24) /* 965 */
+# define PNV_GMBUSUNIT_CLOCK_GATE_DISABLE	(1 << 24) /* pnv */
 # define TVRUNIT_CLOCK_GATE_DISABLE		(1 << 23) /* 915-945 */
 # define TVCUNIT_CLOCK_GATE_DISABLE		(1 << 22) /* 915-945 */
 # define TVFUNIT_CLOCK_GATE_DISABLE		(1 << 21) /* 915-945 */
@@ -3395,6 +3435,7 @@ enum i915_power_well_id {
 #define ELK_STOLEN_RESERVED		_MMIO(MCHBAR_MIRROR_BASE + 0x48)
 #define G4X_STOLEN_RESERVED_ADDR1_MASK	(0xFFFF << 16)
 #define G4X_STOLEN_RESERVED_ADDR2_MASK	(0xFFF << 4)
+#define G4X_STOLEN_RESERVED_ENABLE	(1 << 0)
 
 /* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */
 #define DCLK _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04)
@@ -3816,9 +3857,13 @@ enum {
  * GEN9 clock gating regs
  */
 #define GEN9_CLKGATE_DIS_0		_MMIO(0x46530)
+#define   DARBF_GATING_DIS		(1 << 27)
 #define   PWM2_GATING_DIS		(1 << 14)
 #define   PWM1_GATING_DIS		(1 << 13)
 
+#define GEN9_CLKGATE_DIS_4		_MMIO(0x4653C)
+#define   BXT_GMBUS_GATING_DIS		(1 << 14)
+
 #define _CLKGATE_DIS_PSL_A		0x46520
 #define _CLKGATE_DIS_PSL_B		0x46524
 #define _CLKGATE_DIS_PSL_C		0x46528
@@ -3834,6 +3879,10 @@ enum {
  */
 #define SLICE_UNIT_LEVEL_CLKGATE	_MMIO(0x94d4)
 #define  SARBUNIT_CLKGATE_DIS		(1 << 5)
+#define  RCCUNIT_CLKGATE_DIS		(1 << 7)
+
+#define UNSLICE_UNIT_LEVEL_CLKGATE	_MMIO(0x9434)
+#define  VFUNIT_CLKGATE_DIS		(1 << 20)
 
 /*
  * Display engine regs
@@ -6260,7 +6309,7 @@ enum {
 #define _PLANE_CTL_2_A				0x70280
 #define _PLANE_CTL_3_A				0x70380
 #define   PLANE_CTL_ENABLE			(1 << 31)
-#define   PLANE_CTL_PIPE_GAMMA_ENABLE		(1 << 30)
+#define   PLANE_CTL_PIPE_GAMMA_ENABLE		(1 << 30)   /* Pre-GLK */
 #define   PLANE_CTL_FORMAT_MASK			(0xf << 24)
 #define   PLANE_CTL_FORMAT_YUV422		(  0 << 24)
 #define   PLANE_CTL_FORMAT_NV12			(  1 << 24)
@@ -6270,7 +6319,7 @@ enum {
 #define   PLANE_CTL_FORMAT_AYUV			(  8 << 24)
 #define   PLANE_CTL_FORMAT_INDEXED		( 12 << 24)
 #define   PLANE_CTL_FORMAT_RGB_565		( 14 << 24)
-#define   PLANE_CTL_PIPE_CSC_ENABLE		(1 << 23)
+#define   PLANE_CTL_PIPE_CSC_ENABLE		(1 << 23) /* Pre-GLK */
 #define   PLANE_CTL_KEY_ENABLE_MASK		(0x3 << 21)
 #define   PLANE_CTL_KEY_ENABLE_SOURCE		(  1 << 21)
 #define   PLANE_CTL_KEY_ENABLE_DESTINATION	(  2 << 21)
@@ -6283,13 +6332,14 @@ enum {
 #define   PLANE_CTL_YUV422_VYUY			(  3 << 16)
 #define   PLANE_CTL_DECOMPRESSION_ENABLE	(1 << 15)
 #define   PLANE_CTL_TRICKLE_FEED_DISABLE	(1 << 14)
-#define   PLANE_CTL_PLANE_GAMMA_DISABLE		(1 << 13)
+#define   PLANE_CTL_PLANE_GAMMA_DISABLE		(1 << 13) /* Pre-GLK */
 #define   PLANE_CTL_TILED_MASK			(0x7 << 10)
 #define   PLANE_CTL_TILED_LINEAR		(  0 << 10)
 #define   PLANE_CTL_TILED_X			(  1 << 10)
 #define   PLANE_CTL_TILED_Y			(  4 << 10)
 #define   PLANE_CTL_TILED_YF			(  5 << 10)
-#define   PLANE_CTL_ALPHA_MASK			(0x3 << 4)
+#define   PLANE_CTL_FLIP_HORIZONTAL		(  1 << 8)
+#define   PLANE_CTL_ALPHA_MASK			(0x3 << 4) /* Pre-GLK */
 #define   PLANE_CTL_ALPHA_DISABLE		(  0 << 4)
 #define   PLANE_CTL_ALPHA_SW_PREMULTIPLY	(  2 << 4)
 #define   PLANE_CTL_ALPHA_HW_PREMULTIPLY	(  3 << 4)
@@ -6329,6 +6379,10 @@ enum {
 #define   PLANE_COLOR_PIPE_GAMMA_ENABLE		(1 << 30)
 #define   PLANE_COLOR_PIPE_CSC_ENABLE		(1 << 23)
 #define   PLANE_COLOR_PLANE_GAMMA_DISABLE	(1 << 13)
+#define   PLANE_COLOR_ALPHA_MASK		(0x3 << 4)
+#define   PLANE_COLOR_ALPHA_DISABLE		(0 << 4)
+#define   PLANE_COLOR_ALPHA_SW_PREMULTIPLY	(2 << 4)
+#define   PLANE_COLOR_ALPHA_HW_PREMULTIPLY	(3 << 4)
 #define _PLANE_BUF_CFG_1_A			0x7027c
 #define _PLANE_BUF_CFG_2_A			0x7037c
 #define _PLANE_NV12_BUF_CFG_1_A		0x70278
@@ -7511,6 +7565,7 @@ enum {
 #define FDI_RX_CHICKEN(pipe)	_MMIO_PIPE(pipe, _FDI_RXA_CHICKEN, _FDI_RXB_CHICKEN)
 
 #define SOUTH_DSPCLK_GATE_D	_MMIO(0xc2020)
+#define  PCH_GMBUSUNIT_CLOCK_GATE_DISABLE (1<<31)
 #define  PCH_DPLUNIT_CLOCK_GATE_DISABLE (1<<30)
 #define  PCH_DPLSUNIT_CLOCK_GATE_DISABLE (1<<29)
 #define  PCH_CPUNIT_CLOCK_GATE_DISABLE (1<<14)
@@ -7774,8 +7829,9 @@ enum {
 #define  FORCEWAKE_ACK_MEDIA_GEN9		_MMIO(0x0D88)
 #define  FORCEWAKE_ACK_RENDER_GEN9		_MMIO(0x0D84)
 #define  FORCEWAKE_ACK_BLITTER_GEN9		_MMIO(0x130044)
-#define   FORCEWAKE_KERNEL			0x1
-#define   FORCEWAKE_USER			0x2
+#define   FORCEWAKE_KERNEL			BIT(0)
+#define   FORCEWAKE_USER			BIT(1)
+#define   FORCEWAKE_KERNEL_FALLBACK		BIT(15)
 #define  FORCEWAKE_MT_ACK			_MMIO(0x130040)
 #define  ECOBUS					_MMIO(0xa180)
 #define    FORCEWAKE_MT_ENABLE			(1<<5)
@@ -7905,6 +7961,7 @@ enum {
 #define GEN6_RC1_WAKE_RATE_LIMIT		_MMIO(0xA098)
 #define GEN6_RC6_WAKE_RATE_LIMIT		_MMIO(0xA09C)
 #define GEN6_RC6pp_WAKE_RATE_LIMIT		_MMIO(0xA0A0)
+#define GEN10_MEDIA_WAKE_RATE_LIMIT		_MMIO(0xA0A0)
 #define GEN6_RC_EVALUATION_INTERVAL		_MMIO(0xA0A8)
 #define GEN6_RC_IDLE_HYSTERSIS			_MMIO(0xA0AC)
 #define GEN6_RC_SLEEP				_MMIO(0xA0B0)
@@ -8036,11 +8093,18 @@ enum {
 #define   CHV_EU311_PG_ENABLE		(1<<1)
 
 #define GEN9_SLICE_PGCTL_ACK(slice)	_MMIO(0x804c + (slice)*0x4)
+#define GEN10_SLICE_PGCTL_ACK(slice)	_MMIO(0x804c + ((slice) / 3) * 0x34 + \
+					      ((slice) % 3) * 0x4)
 #define   GEN9_PGCTL_SLICE_ACK		(1 << 0)
 #define   GEN9_PGCTL_SS_ACK(subslice)	(1 << (2 + (subslice)*2))
+#define   GEN10_PGCTL_VALID_SS_MASK(slice) ((slice) == 0 ? 0x7F : 0x1F)
 
 #define GEN9_SS01_EU_PGCTL_ACK(slice)	_MMIO(0x805c + (slice)*0x8)
+#define GEN10_SS01_EU_PGCTL_ACK(slice)	_MMIO(0x805c + ((slice) / 3) * 0x30 + \
+					      ((slice) % 3) * 0x8)
 #define GEN9_SS23_EU_PGCTL_ACK(slice)	_MMIO(0x8060 + (slice)*0x8)
+#define GEN10_SS23_EU_PGCTL_ACK(slice)	_MMIO(0x8060 + ((slice) / 3) * 0x30 + \
+					      ((slice) % 3) * 0x8)
 #define   GEN9_PGCTL_SSA_EU08_ACK	(1 << 0)
 #define   GEN9_PGCTL_SSA_EU19_ACK	(1 << 2)
 #define   GEN9_PGCTL_SSA_EU210_ACK	(1 << 4)
@@ -8092,6 +8156,7 @@ enum {
 #define   PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE	(1<<8)
 #define   STALL_DOP_GATING_DISABLE		(1<<5)
 #define   THROTTLE_12_5				(7<<2)
+#define   DISABLE_EARLY_EOT			(1<<1)
 
 #define GEN7_ROW_CHICKEN2		_MMIO(0xe4f4)
 #define GEN7_ROW_CHICKEN2_GT2		_MMIO(0xf4f4)
@@ -8838,6 +8903,12 @@ enum skl_power_gate {
 #define ILK_TIMESTAMP_HI	_MMIO(0x70070)
 #define IVB_TIMESTAMP_CTR	_MMIO(0x44070)
 
+#define GEN9_TIMESTAMP_OVERRIDE				_MMIO(0x44074)
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_SHIFT	0
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK	0x3ff
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT	12
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK	(0xf << 12)
+
 #define _PIPE_FRMTMSTMP_A		0x70048
 #define PIPE_FRMTMSTMP(pipe)		\
 			_MMIO_PIPE2(pipe, _PIPE_FRMTMSTMP_A)
diff --git a/drivers/gpu/drm/i915/i915_selftest.h b/drivers/gpu/drm/i915/i915_selftest.h
index 78e1a1b168ff..9766e806dce6 100644
--- a/drivers/gpu/drm/i915/i915_selftest.h
+++ b/drivers/gpu/drm/i915/i915_selftest.h
@@ -99,6 +99,6 @@ __printf(2, 3)
 bool __igt_timeout(unsigned long timeout, const char *fmt, ...);
 
 #define igt_timeout(t, fmt, ...) \
-	__igt_timeout((t), KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__)
+	__igt_timeout((t), KERN_NOTICE pr_fmt(fmt), ##__VA_ARGS__)
 
 #endif /* !__I915_SELFTEST_H__ */
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c
index ac236b88c99c..3669f5eeb91e 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence.c
@@ -303,6 +303,7 @@ static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 	int pending;
 
 	debug_fence_assert(fence);
+	might_sleep_if(gfpflags_allow_blocking(gfp));
 
 	if (i915_sw_fence_done(signaler))
 		return 0;
@@ -419,6 +420,7 @@ int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 	int ret;
 
 	debug_fence_assert(fence);
+	might_sleep_if(gfpflags_allow_blocking(gfp));
 
 	if (dma_fence_is_signaled(dma))
 		return 0;
@@ -465,6 +467,7 @@ int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 	int ret = 0, pending;
 
 	debug_fence_assert(fence);
+	might_sleep_if(gfpflags_allow_blocking(gfp));
 
 	if (write) {
 		struct dma_fence **shared;
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 791759f632e1..e5e6f6bb2b05 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -42,14 +42,30 @@ static inline struct drm_i915_private *kdev_minor_to_i915(struct device *kdev)
 static u32 calc_residency(struct drm_i915_private *dev_priv,
 			  i915_reg_t reg)
 {
-	return DIV_ROUND_CLOSEST_ULL(intel_rc6_residency_us(dev_priv, reg),
-				     1000);
+	u64 res;
+
+	intel_runtime_pm_get(dev_priv);
+	res = intel_rc6_residency_us(dev_priv, reg);
+	intel_runtime_pm_put(dev_priv);
+
+	return DIV_ROUND_CLOSEST_ULL(res, 1000);
 }
 
 static ssize_t
 show_rc6_mask(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-	return snprintf(buf, PAGE_SIZE, "%x\n", intel_rc6_enabled());
+	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
+	unsigned int mask;
+
+	mask = 0;
+	if (HAS_RC6(dev_priv))
+		mask |= BIT(0);
+	if (HAS_RC6p(dev_priv))
+		mask |= BIT(1);
+	if (HAS_RC6pp(dev_priv))
+		mask |= BIT(2);
+
+	return snprintf(buf, PAGE_SIZE, "%x\n", mask);
 }
 
 static ssize_t
@@ -252,14 +268,9 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
 		freq = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
 		ret = intel_gpu_freq(dev_priv, (freq >> 8) & 0xff);
 	} else {
-		u32 rpstat = I915_READ(GEN6_RPSTAT1);
-		if (INTEL_GEN(dev_priv) >= 9)
-			ret = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
-		else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
-			ret = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
-		else
-			ret = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
-		ret = intel_gpu_freq(dev_priv, ret);
+		ret = intel_gpu_freq(dev_priv,
+				     intel_get_cagf(dev_priv,
+						    I915_READ(GEN6_RPSTAT1)));
 	}
 	mutex_unlock(&dev_priv->pcu_lock);
 
@@ -293,8 +304,9 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
 {
 	struct drm_i915_private *dev_priv = kdev_minor_to_i915(kdev);
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
-	u32 val;
+	bool boost = false;
 	ssize_t ret;
+	u32 val;
 
 	ret = kstrtou32(buf, 0, &val);
 	if (ret)
@@ -306,8 +318,13 @@ static ssize_t gt_boost_freq_mhz_store(struct device *kdev,
 		return -EINVAL;
 
 	mutex_lock(&dev_priv->pcu_lock);
-	rps->boost_freq = val;
+	if (val != rps->boost_freq) {
+		rps->boost_freq = val;
+		boost = atomic_read(&rps->num_waiters);
+	}
 	mutex_unlock(&dev_priv->pcu_lock);
+	if (boost)
+		schedule_work(&rps->work);
 
 	return count;
 }
@@ -434,13 +451,13 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
 	return ret ?: count;
 }
 
-static DEVICE_ATTR(gt_act_freq_mhz, S_IRUGO, gt_act_freq_mhz_show, NULL);
-static DEVICE_ATTR(gt_cur_freq_mhz, S_IRUGO, gt_cur_freq_mhz_show, NULL);
-static DEVICE_ATTR(gt_boost_freq_mhz, S_IRUGO | S_IWUSR, gt_boost_freq_mhz_show, gt_boost_freq_mhz_store);
-static DEVICE_ATTR(gt_max_freq_mhz, S_IRUGO | S_IWUSR, gt_max_freq_mhz_show, gt_max_freq_mhz_store);
-static DEVICE_ATTR(gt_min_freq_mhz, S_IRUGO | S_IWUSR, gt_min_freq_mhz_show, gt_min_freq_mhz_store);
+static DEVICE_ATTR_RO(gt_act_freq_mhz);
+static DEVICE_ATTR_RO(gt_cur_freq_mhz);
+static DEVICE_ATTR_RW(gt_boost_freq_mhz);
+static DEVICE_ATTR_RW(gt_max_freq_mhz);
+static DEVICE_ATTR_RW(gt_min_freq_mhz);
 
-static DEVICE_ATTR(vlv_rpe_freq_mhz, S_IRUGO, vlv_rpe_freq_mhz_show, NULL);
+static DEVICE_ATTR_RO(vlv_rpe_freq_mhz);
 
 static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf);
 static DEVICE_ATTR(gt_RP0_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 4e76768ffa95..e1169c02eb2b 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -616,6 +616,7 @@ TRACE_EVENT(i915_gem_request_queue,
 
 	    TP_STRUCT__entry(
 			     __field(u32, dev)
+			     __field(u32, hw_id)
 			     __field(u32, ring)
 			     __field(u32, ctx)
 			     __field(u32, seqno)
@@ -624,15 +625,16 @@ TRACE_EVENT(i915_gem_request_queue,
 
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
+			   __entry->hw_id = req->ctx->hw_id;
 			   __entry->ring = req->engine->id;
 			   __entry->ctx = req->fence.context;
 			   __entry->seqno = req->fence.seqno;
 			   __entry->flags = flags;
 			   ),
 
-	    TP_printk("dev=%u, ring=%u, ctx=%u, seqno=%u, flags=0x%x",
-		      __entry->dev, __entry->ring, __entry->ctx, __entry->seqno,
-		      __entry->flags)
+	    TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, flags=0x%x",
+		      __entry->dev, __entry->hw_id, __entry->ring, __entry->ctx,
+		      __entry->seqno, __entry->flags)
 );
 
 DECLARE_EVENT_CLASS(i915_gem_request,
@@ -641,23 +643,25 @@ DECLARE_EVENT_CLASS(i915_gem_request,
 
 	    TP_STRUCT__entry(
 			     __field(u32, dev)
-			     __field(u32, ctx)
+			     __field(u32, hw_id)
 			     __field(u32, ring)
+			     __field(u32, ctx)
 			     __field(u32, seqno)
 			     __field(u32, global)
 			     ),
 
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
+			   __entry->hw_id = req->ctx->hw_id;
 			   __entry->ring = req->engine->id;
 			   __entry->ctx = req->fence.context;
 			   __entry->seqno = req->fence.seqno;
 			   __entry->global = req->global_seqno;
 			   ),
 
-	    TP_printk("dev=%u, ring=%u, ctx=%u, seqno=%u, global=%u",
-		      __entry->dev, __entry->ring, __entry->ctx, __entry->seqno,
-		      __entry->global)
+	    TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u",
+		      __entry->dev, __entry->hw_id, __entry->ring, __entry->ctx,
+		      __entry->seqno, __entry->global)
 );
 
 DEFINE_EVENT(i915_gem_request, i915_gem_request_add,
@@ -683,15 +687,17 @@ DECLARE_EVENT_CLASS(i915_gem_request_hw,
 
 		    TP_STRUCT__entry(
 				     __field(u32, dev)
+				     __field(u32, hw_id)
 				     __field(u32, ring)
+				     __field(u32, ctx)
 				     __field(u32, seqno)
 				     __field(u32, global_seqno)
-				     __field(u32, ctx)
 				     __field(u32, port)
 				    ),
 
 		    TP_fast_assign(
 			           __entry->dev = req->i915->drm.primary->index;
+			           __entry->hw_id = req->ctx->hw_id;
 			           __entry->ring = req->engine->id;
 			           __entry->ctx = req->fence.context;
 			           __entry->seqno = req->fence.seqno;
@@ -699,10 +705,10 @@ DECLARE_EVENT_CLASS(i915_gem_request_hw,
 			           __entry->port = port;
 			          ),
 
-		    TP_printk("dev=%u, ring=%u, ctx=%u, seqno=%u, global=%u, port=%u",
-			      __entry->dev, __entry->ring, __entry->ctx,
-			      __entry->seqno, __entry->global_seqno,
-			      __entry->port)
+		    TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u, port=%u",
+			      __entry->dev, __entry->hw_id, __entry->ring,
+			      __entry->ctx, __entry->seqno,
+			      __entry->global_seqno, __entry->port)
 );
 
 DEFINE_EVENT(i915_gem_request_hw, i915_gem_request_in,
@@ -772,6 +778,7 @@ TRACE_EVENT(i915_gem_request_wait_begin,
 
 	    TP_STRUCT__entry(
 			     __field(u32, dev)
+			     __field(u32, hw_id)
 			     __field(u32, ring)
 			     __field(u32, ctx)
 			     __field(u32, seqno)
@@ -787,6 +794,7 @@ TRACE_EVENT(i915_gem_request_wait_begin,
 	     */
 	    TP_fast_assign(
 			   __entry->dev = req->i915->drm.primary->index;
+			   __entry->hw_id = req->ctx->hw_id;
 			   __entry->ring = req->engine->id;
 			   __entry->ctx = req->fence.context;
 			   __entry->seqno = req->fence.seqno;
@@ -794,10 +802,10 @@ TRACE_EVENT(i915_gem_request_wait_begin,
 			   __entry->flags = flags;
 			   ),
 
-	    TP_printk("dev=%u, ring=%u, ctx=%u, seqno=%u, global=%u, blocking=%u, flags=0x%x",
-		      __entry->dev, __entry->ring, __entry->ctx, __entry->seqno,
-		      __entry->global, !!(__entry->flags & I915_WAIT_LOCKED),
-		      __entry->flags)
+	    TP_printk("dev=%u, hw_id=%u, ring=%u, ctx=%u, seqno=%u, global=%u, blocking=%u, flags=0x%x",
+		      __entry->dev, __entry->hw_id, __entry->ring, __entry->ctx,
+		      __entry->seqno, __entry->global,
+		      !!(__entry->flags & I915_WAIT_LOCKED), __entry->flags)
 );
 
 DEFINE_EVENT(i915_gem_request, i915_gem_request_wait_end,
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index af3d7cc53fa1..51dbfe5bb418 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -83,8 +83,11 @@
 	(typeof(ptr))(__v & -BIT(n));					\
 })
 
-#define ptr_pack_bits(ptr, bits, n)					\
-	((typeof(ptr))((unsigned long)(ptr) | (bits)))
+#define ptr_pack_bits(ptr, bits, n) ({					\
+	unsigned long __bits = (bits);					\
+	GEM_BUG_ON(__bits & -BIT(n));					\
+	((typeof(ptr))((unsigned long)(ptr) | __bits));			\
+})
 
 #define page_mask_bits(ptr) ptr_mask_bits(ptr, PAGE_SHIFT)
 #define page_unmask_bits(ptr) ptr_unmask_bits(ptr, PAGE_SHIFT)
@@ -137,4 +140,19 @@ static inline void drain_delayed_work(struct delayed_work *dw)
 	} while (delayed_work_pending(dw));
 }
 
+static inline const char *yesno(bool v)
+{
+	return v ? "yes" : "no";
+}
+
+static inline const char *onoff(bool v)
+{
+	return v ? "on" : "off";
+}
+
+static inline const char *enableddisabled(bool v)
+{
+	return v ? "enabled" : "disabled";
+}
+
 #endif /* !__I915_UTILS_H */
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index fbfab2f33023..e0e7c48f45dc 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -142,6 +142,12 @@ vma_create(struct drm_i915_gem_object *obj,
 								i915_gem_object_get_stride(obj));
 		GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
 
+		/*
+		 * We put the GGTT vma at the start of the vma-list, followed
+		 * by the ppGGTT vma. This allows us to break early when
+		 * iterating over only the GGTT vma for an object, see
+		 * for_each_ggtt_vma()
+		 */
 		vma->flags |= I915_VMA_GGTT;
 		list_add(&vma->obj_link, &obj->vma_list);
 	} else {
@@ -305,7 +311,7 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 
 	ptr = vma->iomap;
 	if (ptr == NULL) {
-		ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable,
+		ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
 					vma->node.start,
 					vma->node.size);
 		if (ptr == NULL) {
@@ -322,6 +328,7 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 	if (err)
 		goto err_unpin;
 
+	i915_vma_set_ggtt_write(vma);
 	return ptr;
 
 err_unpin:
@@ -330,12 +337,24 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 	return IO_ERR_PTR(err);
 }
 
+void i915_vma_flush_writes(struct i915_vma *vma)
+{
+	if (!i915_vma_has_ggtt_write(vma))
+		return;
+
+	i915_gem_flush_ggtt_writes(vma->vm->i915);
+
+	i915_vma_unset_ggtt_write(vma);
+}
+
 void i915_vma_unpin_iomap(struct i915_vma *vma)
 {
 	lockdep_assert_held(&vma->obj->base.dev->struct_mutex);
 
 	GEM_BUG_ON(vma->iomap == NULL);
 
+	i915_vma_flush_writes(vma);
+
 	i915_vma_unpin_fence(vma);
 	i915_vma_unpin(vma);
 }
@@ -466,6 +485,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 	u64 start, end;
 	int ret;
 
+	GEM_BUG_ON(i915_vma_is_closed(vma));
 	GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
 
@@ -640,15 +660,17 @@ int __i915_vma_do_pin(struct i915_vma *vma,
 		if (ret)
 			goto err_unpin;
 	}
+	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
 	ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
 	if (ret)
 		goto err_remove;
 
+	GEM_BUG_ON((vma->flags & I915_VMA_BIND_MASK) == 0);
+
 	if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
 		__i915_vma_set_map_and_fenceable(vma);
 
-	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 	GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
 	return 0;
 
@@ -656,6 +678,7 @@ int __i915_vma_do_pin(struct i915_vma *vma,
 	if ((bound & I915_VMA_BIND_MASK) == 0) {
 		i915_vma_remove(vma);
 		GEM_BUG_ON(vma->pages);
+		GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK);
 	}
 err_unpin:
 	__i915_vma_unpin(vma);
@@ -675,7 +698,9 @@ static void i915_vma_destroy(struct i915_vma *vma)
 		GEM_BUG_ON(i915_gem_active_isset(&vma->last_read[i]));
 	GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence));
 
+	list_del(&vma->obj_link);
 	list_del(&vma->vm_link);
+
 	if (!i915_vma_is_ggtt(vma))
 		i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
 
@@ -687,7 +712,6 @@ void i915_vma_close(struct i915_vma *vma)
 	GEM_BUG_ON(i915_vma_is_closed(vma));
 	vma->flags |= I915_VMA_CLOSED;
 
-	list_del(&vma->obj_link);
 	rb_erase(&vma->obj_node, &vma->obj->vma_tree);
 
 	if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
@@ -740,6 +764,7 @@ int i915_vma_unbind(struct i915_vma *vma)
 	/* First wait upon any activity as retiring the request may
 	 * have side-effects such as unpinning or even unbinding this vma.
 	 */
+	might_sleep();
 	active = i915_vma_get_active(vma);
 	if (active) {
 		int idx;
@@ -786,6 +811,15 @@ int i915_vma_unbind(struct i915_vma *vma)
 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 
 	if (i915_vma_is_map_and_fenceable(vma)) {
+		/*
+		 * Check that we have flushed all writes through the GGTT
+		 * before the unbind, other due to non-strict nature of those
+		 * indirect writes they may end up referencing the GGTT PTE
+		 * after the unbind.
+		 */
+		i915_vma_flush_writes(vma);
+		GEM_BUG_ON(i915_vma_has_ggtt_write(vma));
+
 		/* release the fence reg _after_ flushing */
 		ret = i915_vma_put_fence(vma);
 		if (ret)
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 1e2bc9b3c3ac..fd5b84904f7c 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -90,6 +90,7 @@ struct i915_vma {
 #define I915_VMA_CLOSED		BIT(10)
 #define I915_VMA_USERFAULT_BIT	11
 #define I915_VMA_USERFAULT	BIT(I915_VMA_USERFAULT_BIT)
+#define I915_VMA_GGTT_WRITE	BIT(12)
 
 	unsigned int active;
 	struct i915_gem_active last_read[I915_NUM_ENGINES];
@@ -138,6 +139,24 @@ static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
 	return vma->flags & I915_VMA_GGTT;
 }
 
+static inline bool i915_vma_has_ggtt_write(const struct i915_vma *vma)
+{
+	return vma->flags & I915_VMA_GGTT_WRITE;
+}
+
+static inline void i915_vma_set_ggtt_write(struct i915_vma *vma)
+{
+	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+	vma->flags |= I915_VMA_GGTT_WRITE;
+}
+
+static inline void i915_vma_unset_ggtt_write(struct i915_vma *vma)
+{
+	vma->flags &= ~I915_VMA_GGTT_WRITE;
+}
+
+void i915_vma_flush_writes(struct i915_vma *vma);
+
 static inline bool i915_vma_is_map_and_fenceable(const struct i915_vma *vma)
 {
 	return vma->flags & I915_VMA_CAN_FENCE;
@@ -389,5 +408,19 @@ i915_vma_unpin_fence(struct i915_vma *vma)
 		__i915_vma_unpin_fence(vma);
 }
 
-#endif
+#define for_each_until(cond) if (cond) break; else
+
+/**
+ * for_each_ggtt_vma - Iterate over the GGTT VMA belonging to an object.
+ * @V: the #i915_vma iterator
+ * @OBJ: the #drm_i915_gem_object
+ *
+ * GGTT VMA are placed at the being of the object's vma_list, see
+ * vma_create(), so we can stop our walk as soon as we see a ppgtt VMA,
+ * or the list is empty ofc.
+ */
+#define for_each_ggtt_vma(V, OBJ) \
+	list_for_each_entry(V, &(OBJ)->vma_list, obj_link)		\
+		for_each_until(!i915_vma_is_ggtt(V))
 
+#endif
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 0ddba16fde1b..4a01f62a392d 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -102,13 +102,13 @@ static const struct dp_aud_n_m dp_aud_n_m[] = {
 };
 
 static const struct dp_aud_n_m *
-audio_config_dp_get_n_m(struct intel_crtc *intel_crtc, int rate)
+audio_config_dp_get_n_m(const struct intel_crtc_state *crtc_state, int rate)
 {
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) {
 		if (rate == dp_aud_n_m[i].sample_rate &&
-		    intel_crtc->config->port_clock == dp_aud_n_m[i].clock)
+		    crtc_state->port_clock == dp_aud_n_m[i].clock)
 			return &dp_aud_n_m[i];
 	}
 
@@ -157,8 +157,10 @@ static const struct {
 };
 
 /* get AUD_CONFIG_PIXEL_CLOCK_HDMI_* value for mode */
-static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted_mode)
+static u32 audio_config_hdmi_pixel_clock(const struct intel_crtc_state *crtc_state)
 {
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(hdmi_audio_clock); i++) {
@@ -179,9 +181,11 @@ static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted
 	return hdmi_audio_clock[i].config;
 }
 
-static int audio_config_hdmi_get_n(const struct drm_display_mode *adjusted_mode,
+static int audio_config_hdmi_get_n(const struct intel_crtc_state *crtc_state,
 				   int rate)
 {
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(hdmi_aud_ncts); i++) {
@@ -220,7 +224,9 @@ static bool intel_eld_uptodate(struct drm_connector *connector,
 	return true;
 }
 
-static void g4x_audio_codec_disable(struct intel_encoder *encoder)
+static void g4x_audio_codec_disable(struct intel_encoder *encoder,
+				    const struct intel_crtc_state *old_crtc_state,
+				    const struct drm_connector_state *old_conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	uint32_t eldv, tmp;
@@ -239,11 +245,12 @@ static void g4x_audio_codec_disable(struct intel_encoder *encoder)
 	I915_WRITE(G4X_AUD_CNTL_ST, tmp);
 }
 
-static void g4x_audio_codec_enable(struct drm_connector *connector,
-				   struct intel_encoder *encoder,
-				   const struct drm_display_mode *adjusted_mode)
+static void g4x_audio_codec_enable(struct intel_encoder *encoder,
+				   const struct intel_crtc_state *crtc_state,
+				   const struct drm_connector_state *conn_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(connector->dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_connector *connector = conn_state->connector;
 	uint8_t *eld = connector->eld;
 	uint32_t eldv;
 	uint32_t tmp;
@@ -279,16 +286,20 @@ static void g4x_audio_codec_enable(struct drm_connector *connector,
 }
 
 static void
-hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
-			   const struct drm_display_mode *adjusted_mode)
+hsw_dp_audio_config_update(struct intel_encoder *encoder,
+			   const struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	int rate = acomp ? acomp->aud_sample_rate[port] : 0;
-	const struct dp_aud_n_m *nm = audio_config_dp_get_n_m(intel_crtc, rate);
-	enum pipe pipe = intel_crtc->pipe;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	enum port port = encoder->port;
+	enum pipe pipe = crtc->pipe;
+	const struct dp_aud_n_m *nm;
+	int rate;
 	u32 tmp;
 
+	rate = acomp ? acomp->aud_sample_rate[port] : 0;
+	nm = audio_config_dp_get_n_m(crtc_state, rate);
 	if (nm)
 		DRM_DEBUG_KMS("using Maud %u, Naud %u\n", nm->m, nm->n);
 	else
@@ -323,23 +334,26 @@ hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
 }
 
 static void
-hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
-			     const struct drm_display_mode *adjusted_mode)
+hsw_hdmi_audio_config_update(struct intel_encoder *encoder,
+			     const struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	int rate = acomp ? acomp->aud_sample_rate[port] : 0;
-	enum pipe pipe = intel_crtc->pipe;
-	int n;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	enum port port = encoder->port;
+	enum pipe pipe = crtc->pipe;
+	int n, rate;
 	u32 tmp;
 
+	rate = acomp ? acomp->aud_sample_rate[port] : 0;
+
 	tmp = I915_READ(HSW_AUD_CFG(pipe));
 	tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
 	tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
 	tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-	tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
+	tmp |= audio_config_hdmi_pixel_clock(crtc_state);
 
-	n = audio_config_hdmi_get_n(adjusted_mode, rate);
+	n = audio_config_hdmi_get_n(crtc_state, rate);
 	if (n != 0) {
 		DRM_DEBUG_KMS("using N %d\n", n);
 
@@ -363,20 +377,22 @@ hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
 }
 
 static void
-hsw_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
-			const struct drm_display_mode *adjusted_mode)
+hsw_audio_config_update(struct intel_encoder *encoder,
+			const struct intel_crtc_state *crtc_state)
 {
-	if (intel_crtc_has_dp_encoder(intel_crtc->config))
-		hsw_dp_audio_config_update(intel_crtc, port, adjusted_mode);
+	if (intel_crtc_has_dp_encoder(crtc_state))
+		hsw_dp_audio_config_update(encoder, crtc_state);
 	else
-		hsw_hdmi_audio_config_update(intel_crtc, port, adjusted_mode);
+		hsw_hdmi_audio_config_update(encoder, crtc_state);
 }
 
-static void hsw_audio_codec_disable(struct intel_encoder *encoder)
+static void hsw_audio_codec_disable(struct intel_encoder *encoder,
+				    const struct intel_crtc_state *old_crtc_state,
+				    const struct drm_connector_state *old_conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-	enum pipe pipe = intel_crtc->pipe;
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+	enum pipe pipe = crtc->pipe;
 	uint32_t tmp;
 
 	DRM_DEBUG_KMS("Disable audio codec on pipe %c\n", pipe_name(pipe));
@@ -389,7 +405,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 	tmp |= AUD_CONFIG_N_PROG_ENABLE;
 	tmp &= ~AUD_CONFIG_UPPER_N_MASK;
 	tmp &= ~AUD_CONFIG_LOWER_N_MASK;
-	if (intel_crtc_has_dp_encoder(intel_crtc->config))
+	if (intel_crtc_has_dp_encoder(old_crtc_state))
 		tmp |= AUD_CONFIG_N_VALUE_INDEX;
 	I915_WRITE(HSW_AUD_CFG(pipe), tmp);
 
@@ -402,14 +418,14 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 	mutex_unlock(&dev_priv->av_mutex);
 }
 
-static void hsw_audio_codec_enable(struct drm_connector *connector,
-				   struct intel_encoder *intel_encoder,
-				   const struct drm_display_mode *adjusted_mode)
+static void hsw_audio_codec_enable(struct intel_encoder *encoder,
+				   const struct intel_crtc_state *crtc_state,
+				   const struct drm_connector_state *conn_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
-	enum pipe pipe = intel_crtc->pipe;
-	enum port port = intel_encoder->port;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_connector *connector = conn_state->connector;
+	enum pipe pipe = crtc->pipe;
 	const uint8_t *eld = connector->eld;
 	uint32_t tmp;
 	int len, i;
@@ -448,17 +464,19 @@ static void hsw_audio_codec_enable(struct drm_connector *connector,
 	I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
 
 	/* Enable timestamps */
-	hsw_audio_config_update(intel_crtc, port, adjusted_mode);
+	hsw_audio_config_update(encoder, crtc_state);
 
 	mutex_unlock(&dev_priv->av_mutex);
 }
 
-static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
+static void ilk_audio_codec_disable(struct intel_encoder *encoder,
+				    const struct intel_crtc_state *old_crtc_state,
+				    const struct drm_connector_state *old_conn_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
-	enum pipe pipe = intel_crtc->pipe;
-	enum port port = intel_encoder->port;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+	enum pipe pipe = crtc->pipe;
+	enum port port = encoder->port;
 	uint32_t tmp, eldv;
 	i915_reg_t aud_config, aud_cntrl_st2;
 
@@ -485,7 +503,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
 	tmp |= AUD_CONFIG_N_PROG_ENABLE;
 	tmp &= ~AUD_CONFIG_UPPER_N_MASK;
 	tmp &= ~AUD_CONFIG_LOWER_N_MASK;
-	if (intel_crtc_has_dp_encoder(intel_crtc->config))
+	if (intel_crtc_has_dp_encoder(old_crtc_state))
 		tmp |= AUD_CONFIG_N_VALUE_INDEX;
 	I915_WRITE(aud_config, tmp);
 
@@ -497,14 +515,15 @@ static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
 	I915_WRITE(aud_cntrl_st2, tmp);
 }
 
-static void ilk_audio_codec_enable(struct drm_connector *connector,
-				   struct intel_encoder *intel_encoder,
-				   const struct drm_display_mode *adjusted_mode)
+static void ilk_audio_codec_enable(struct intel_encoder *encoder,
+				   const struct intel_crtc_state *crtc_state,
+				   const struct drm_connector_state *conn_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(connector->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
-	enum pipe pipe = intel_crtc->pipe;
-	enum port port = intel_encoder->port;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_connector *connector = conn_state->connector;
+	enum pipe pipe = crtc->pipe;
+	enum port port = encoder->port;
 	uint8_t *eld = connector->eld;
 	uint32_t tmp, eldv;
 	int len, i;
@@ -568,36 +587,36 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
 	tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
 	tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
 	tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-	if (intel_crtc_has_dp_encoder(intel_crtc->config))
+	if (intel_crtc_has_dp_encoder(crtc_state))
 		tmp |= AUD_CONFIG_N_VALUE_INDEX;
 	else
-		tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
+		tmp |= audio_config_hdmi_pixel_clock(crtc_state);
 	I915_WRITE(aud_config, tmp);
 }
 
 /**
  * intel_audio_codec_enable - Enable the audio codec for HD audio
- * @intel_encoder: encoder on which to enable audio
+ * @encoder: encoder on which to enable audio
  * @crtc_state: pointer to the current crtc state.
  * @conn_state: pointer to the current connector state.
  *
  * The enable sequences may only be performed after enabling the transcoder and
  * port, and after completed link training.
  */
-void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
+void intel_audio_codec_enable(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *crtc_state,
 			      const struct drm_connector_state *conn_state)
 {
-	struct drm_encoder *encoder = &intel_encoder->base;
-	const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
-	struct drm_connector *connector;
-	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	enum port port = intel_encoder->port;
-	enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_connector *connector = conn_state->connector;
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
+	enum port port = encoder->port;
+	enum pipe pipe = crtc->pipe;
 
-	connector = conn_state->connector;
-	if (!connector || !connector->eld[0])
+	if (!connector->eld[0])
 		return;
 
 	DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
@@ -609,19 +628,20 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
 	connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
 
 	if (dev_priv->display.audio_codec_enable)
-		dev_priv->display.audio_codec_enable(connector, intel_encoder,
-						     adjusted_mode);
+		dev_priv->display.audio_codec_enable(encoder,
+						     crtc_state,
+						     conn_state);
 
 	mutex_lock(&dev_priv->av_mutex);
-	intel_encoder->audio_connector = connector;
+	encoder->audio_connector = connector;
 
 	/* referred in audio callbacks */
-	dev_priv->av_enc_map[pipe] = intel_encoder;
+	dev_priv->av_enc_map[pipe] = encoder;
 	mutex_unlock(&dev_priv->av_mutex);
 
 	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
 		/* audio drivers expect pipe = -1 to indicate Non-MST cases */
-		if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+		if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))
 			pipe = -1;
 		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
 						 (int) port, (int) pipe);
@@ -629,36 +649,41 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
 
 	intel_lpe_audio_notify(dev_priv, pipe, port, connector->eld,
 			       crtc_state->port_clock,
-			       intel_encoder->type == INTEL_OUTPUT_DP);
+			       intel_crtc_has_dp_encoder(crtc_state));
 }
 
 /**
  * intel_audio_codec_disable - Disable the audio codec for HD audio
- * @intel_encoder: encoder on which to disable audio
+ * @encoder: encoder on which to disable audio
+ * @old_crtc_state: pointer to the old crtc state.
+ * @old_conn_state: pointer to the old connector state.
  *
  * The disable sequences must be performed before disabling the transcoder or
  * port.
  */
-void intel_audio_codec_disable(struct intel_encoder *intel_encoder)
+void intel_audio_codec_disable(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *old_crtc_state,
+			       const struct drm_connector_state *old_conn_state)
 {
-	struct drm_encoder *encoder = &intel_encoder->base;
-	struct drm_i915_private *dev_priv = to_i915(encoder->dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct i915_audio_component *acomp = dev_priv->audio_component;
-	enum port port = intel_encoder->port;
-	struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+	enum port port = encoder->port;
 	enum pipe pipe = crtc->pipe;
 
 	if (dev_priv->display.audio_codec_disable)
-		dev_priv->display.audio_codec_disable(intel_encoder);
+		dev_priv->display.audio_codec_disable(encoder,
+						      old_crtc_state,
+						      old_conn_state);
 
 	mutex_lock(&dev_priv->av_mutex);
-	intel_encoder->audio_connector = NULL;
+	encoder->audio_connector = NULL;
 	dev_priv->av_enc_map[pipe] = NULL;
 	mutex_unlock(&dev_priv->av_mutex);
 
 	if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
 		/* audio drivers expect pipe = -1 to indicate Non-MST cases */
-		if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+		if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
 			pipe = -1;
 		acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
 						 (int) port, (int) pipe);
@@ -754,11 +779,11 @@ static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv,
 {
 	struct intel_encoder *encoder;
 
-	if (WARN_ON(pipe >= INTEL_INFO(dev_priv)->num_pipes))
-		return NULL;
-
 	/* MST */
 	if (pipe >= 0) {
+		if (WARN_ON(pipe >= ARRAY_SIZE(dev_priv->av_enc_map)))
+			return NULL;
+
 		encoder = dev_priv->av_enc_map[pipe];
 		/*
 		 * when bootup, audio driver may not know it is
@@ -793,10 +818,9 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 						int pipe, int rate)
 {
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
-	struct intel_encoder *intel_encoder;
-	struct intel_crtc *crtc;
-	struct drm_display_mode *adjusted_mode;
 	struct i915_audio_component *acomp = dev_priv->audio_component;
+	struct intel_encoder *encoder;
+	struct intel_crtc *crtc;
 	int err = 0;
 
 	if (!HAS_DDI(dev_priv))
@@ -806,23 +830,19 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 	mutex_lock(&dev_priv->av_mutex);
 
 	/* 1. get the pipe */
-	intel_encoder = get_saved_enc(dev_priv, port, pipe);
-	if (!intel_encoder || !intel_encoder->base.crtc) {
+	encoder = get_saved_enc(dev_priv, port, pipe);
+	if (!encoder || !encoder->base.crtc) {
 		DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port));
 		err = -ENODEV;
 		goto unlock;
 	}
 
-	/* pipe passed from the audio driver will be -1 for Non-MST case */
-	crtc = to_intel_crtc(intel_encoder->base.crtc);
-	pipe = crtc->pipe;
-
-	adjusted_mode = &crtc->config->base.adjusted_mode;
+	crtc = to_intel_crtc(encoder->base.crtc);
 
 	/* port must be valid now, otherwise the pipe will be invalid */
 	acomp->aud_sample_rate[port] = rate;
 
-	hsw_audio_config_update(crtc, port, adjusted_mode);
+	hsw_audio_config_update(encoder, crtc->config);
 
  unlock:
 	mutex_unlock(&dev_priv->av_mutex);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index fd23023df7c1..b49a2df44430 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -947,6 +947,86 @@ static int goto_next_sequence_v3(const u8 *data, int index, int total)
 	return 0;
 }
 
+/*
+ * Get len of pre-fixed deassert fragment from a v1 init OTP sequence,
+ * skip all delay + gpio operands and stop at the first DSI packet op.
+ */
+static int get_init_otp_deassert_fragment_len(struct drm_i915_private *dev_priv)
+{
+	const u8 *data = dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP];
+	int index, len;
+
+	if (WARN_ON(!data || dev_priv->vbt.dsi.seq_version != 1))
+		return 0;
+
+	/* index = 1 to skip sequence byte */
+	for (index = 1; data[index] != MIPI_SEQ_ELEM_END; index += len) {
+		switch (data[index]) {
+		case MIPI_SEQ_ELEM_SEND_PKT:
+			return index == 1 ? 0 : index;
+		case MIPI_SEQ_ELEM_DELAY:
+			len = 5; /* 1 byte for operand + uint32 */
+			break;
+		case MIPI_SEQ_ELEM_GPIO:
+			len = 3; /* 1 byte for op, 1 for gpio_nr, 1 for value */
+			break;
+		default:
+			return 0;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Some v1 VBT MIPI sequences do the deassert in the init OTP sequence.
+ * The deassert must be done before calling intel_dsi_device_ready, so for
+ * these devices we split the init OTP sequence into a deassert sequence and
+ * the actual init OTP part.
+ */
+static void fixup_mipi_sequences(struct drm_i915_private *dev_priv)
+{
+	u8 *init_otp;
+	int len;
+
+	/* Limit this to VLV for now. */
+	if (!IS_VALLEYVIEW(dev_priv))
+		return;
+
+	/* Limit this to v1 vid-mode sequences */
+	if (dev_priv->vbt.dsi.config->is_cmd_mode ||
+	    dev_priv->vbt.dsi.seq_version != 1)
+		return;
+
+	/* Only do this if there are otp and assert seqs and no deassert seq */
+	if (!dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP] ||
+	    !dev_priv->vbt.dsi.sequence[MIPI_SEQ_ASSERT_RESET] ||
+	    dev_priv->vbt.dsi.sequence[MIPI_SEQ_DEASSERT_RESET])
+		return;
+
+	/* The deassert-sequence ends at the first DSI packet */
+	len = get_init_otp_deassert_fragment_len(dev_priv);
+	if (!len)
+		return;
+
+	DRM_DEBUG_KMS("Using init OTP fragment to deassert reset\n");
+
+	/* Copy the fragment, update seq byte and terminate it */
+	init_otp = (u8 *)dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP];
+	dev_priv->vbt.dsi.deassert_seq = kmemdup(init_otp, len + 1, GFP_KERNEL);
+	if (!dev_priv->vbt.dsi.deassert_seq)
+		return;
+	dev_priv->vbt.dsi.deassert_seq[0] = MIPI_SEQ_DEASSERT_RESET;
+	dev_priv->vbt.dsi.deassert_seq[len] = MIPI_SEQ_ELEM_END;
+	/* Use the copy for deassert */
+	dev_priv->vbt.dsi.sequence[MIPI_SEQ_DEASSERT_RESET] =
+		dev_priv->vbt.dsi.deassert_seq;
+	/* Replace the last byte of the fragment with init OTP seq byte */
+	init_otp[len - 1] = MIPI_SEQ_INIT_OTP;
+	/* And make MIPI_MIPI_SEQ_INIT_OTP point to it */
+	dev_priv->vbt.dsi.sequence[MIPI_SEQ_INIT_OTP] = init_otp + len - 1;
+}
+
 static void
 parse_mipi_sequence(struct drm_i915_private *dev_priv,
 		    const struct bdb_header *bdb)
@@ -1016,6 +1096,8 @@ parse_mipi_sequence(struct drm_i915_private *dev_priv,
 	dev_priv->vbt.dsi.size = seq_size;
 	dev_priv->vbt.dsi.seq_version = sequence->version;
 
+	fixup_mipi_sequences(dev_priv);
+
 	DRM_DEBUG_DRIVER("MIPI related VBT parsing complete\n");
 	return;
 
@@ -1107,6 +1189,7 @@ static void sanitize_aux_ch(struct drm_i915_private *dev_priv,
 }
 
 static const u8 cnp_ddc_pin_map[] = {
+	[0] = 0, /* N/A */
 	[DDC_BUS_DDI_B] = GMBUS_PIN_1_BXT,
 	[DDC_BUS_DDI_C] = GMBUS_PIN_2_BXT,
 	[DDC_BUS_DDI_D] = GMBUS_PIN_4_CNP, /* sic */
@@ -1115,9 +1198,14 @@ static const u8 cnp_ddc_pin_map[] = {
 
 static u8 map_ddc_pin(struct drm_i915_private *dev_priv, u8 vbt_pin)
 {
-	if (HAS_PCH_CNP(dev_priv) &&
-	    vbt_pin > 0 && vbt_pin < ARRAY_SIZE(cnp_ddc_pin_map))
-		return cnp_ddc_pin_map[vbt_pin];
+	if (HAS_PCH_CNP(dev_priv)) {
+		if (vbt_pin < ARRAY_SIZE(cnp_ddc_pin_map)) {
+			return cnp_ddc_pin_map[vbt_pin];
+		} else {
+			DRM_DEBUG_KMS("Ignoring alternate pin: VBT claims DDC pin %d, which is not valid for this platform\n", vbt_pin);
+			return 0;
+		}
+	}
 
 	return vbt_pin;
 }
@@ -1234,6 +1322,30 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 		info->hdmi_level_shift = hdmi_level_shift;
 	}
 
+	if (bdb_version >= 204) {
+		int max_tmds_clock;
+
+		switch (child->hdmi_max_data_rate) {
+		default:
+			MISSING_CASE(child->hdmi_max_data_rate);
+			/* fall through */
+		case HDMI_MAX_DATA_RATE_PLATFORM:
+			max_tmds_clock = 0;
+			break;
+		case HDMI_MAX_DATA_RATE_297:
+			max_tmds_clock = 297000;
+			break;
+		case HDMI_MAX_DATA_RATE_165:
+			max_tmds_clock = 165000;
+			break;
+		}
+
+		if (max_tmds_clock)
+			DRM_DEBUG_KMS("VBT HDMI max TMDS clock for port %c: %d kHz\n",
+				      port_name(port), max_tmds_clock);
+		info->max_tmds_clock = max_tmds_clock;
+	}
+
 	/* Parse the I_boost config for SKL and above */
 	if (bdb_version >= 196 && child->iboost) {
 		info->dp_boost_level = translate_iboost(child->dp_iboost_level);
@@ -1299,11 +1411,13 @@ parse_general_definitions(struct drm_i915_private *dev_priv,
 		expected_size = LEGACY_CHILD_DEVICE_CONFIG_SIZE;
 	} else if (bdb->version == 195) {
 		expected_size = 37;
-	} else if (bdb->version <= 197) {
+	} else if (bdb->version <= 215) {
 		expected_size = 38;
+	} else if (bdb->version <= 216) {
+		expected_size = 39;
 	} else {
-		expected_size = 38;
-		BUILD_BUG_ON(sizeof(*child) < 38);
+		expected_size = sizeof(*child);
+		BUILD_BUG_ON(sizeof(*child) < 39);
 		DRM_DEBUG_DRIVER("Expected child device config size for VBT version %u not known; assuming %u\n",
 				 bdb->version, expected_size);
 	}
@@ -1556,6 +1670,29 @@ void intel_bios_init(struct drm_i915_private *dev_priv)
 		pci_unmap_rom(pdev, bios);
 }
 
+/**
+ * intel_bios_cleanup - Free any resources allocated by intel_bios_init()
+ * @dev_priv: i915 device instance
+ */
+void intel_bios_cleanup(struct drm_i915_private *dev_priv)
+{
+	kfree(dev_priv->vbt.child_dev);
+	dev_priv->vbt.child_dev = NULL;
+	dev_priv->vbt.child_dev_num = 0;
+	kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
+	dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
+	kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
+	dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
+	kfree(dev_priv->vbt.dsi.data);
+	dev_priv->vbt.dsi.data = NULL;
+	kfree(dev_priv->vbt.dsi.pps);
+	dev_priv->vbt.dsi.pps = NULL;
+	kfree(dev_priv->vbt.dsi.config);
+	dev_priv->vbt.dsi.config = NULL;
+	kfree(dev_priv->vbt.dsi.deassert_seq);
+	dev_priv->vbt.dsi.deassert_seq = NULL;
+}
+
 /**
  * intel_bios_is_tv_present - is integrated TV present in VBT
  * @dev_priv:	i915 device instance
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index bcbc7abe6693..f54ddda9fdad 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -27,6 +27,12 @@
 
 #include "i915_drv.h"
 
+#ifdef CONFIG_SMP
+#define task_asleep(tsk) ((tsk)->state & TASK_NORMAL && !(tsk)->on_cpu)
+#else
+#define task_asleep(tsk) ((tsk)->state & TASK_NORMAL)
+#endif
+
 static unsigned int __intel_breadcrumbs_wakeup(struct intel_breadcrumbs *b)
 {
 	struct intel_wait *wait;
@@ -36,8 +42,20 @@ static unsigned int __intel_breadcrumbs_wakeup(struct intel_breadcrumbs *b)
 
 	wait = b->irq_wait;
 	if (wait) {
+		/*
+		 * N.B. Since task_asleep() and ttwu are not atomic, the
+		 * waiter may actually go to sleep after the check, causing
+		 * us to suppress a valid wakeup. We prefer to reduce the
+		 * number of false positive missed_breadcrumb() warnings
+		 * at the expense of a few false negatives, as it it easy
+		 * to trigger a false positive under heavy load. Enough
+		 * signal should remain from genuine missed_breadcrumb()
+		 * for us to detect in CI.
+		 */
+		bool was_asleep = task_asleep(wait->tsk);
+
 		result = ENGINE_WAKEUP_WAITER;
-		if (wake_up_process(wait->tsk))
+		if (wake_up_process(wait->tsk) && was_asleep)
 			result |= ENGINE_WAKEUP_ASLEEP;
 	}
 
@@ -64,20 +82,21 @@ static unsigned long wait_timeout(void)
 
 static noinline void missed_breadcrumb(struct intel_engine_cs *engine)
 {
-	DRM_DEBUG_DRIVER("%s missed breadcrumb at %pS, irq posted? %s, current seqno=%x, last=%x\n",
-			 engine->name, __builtin_return_address(0),
-			 yesno(test_bit(ENGINE_IRQ_BREADCRUMB,
-					&engine->irq_posted)),
-			 intel_engine_get_seqno(engine),
-			 intel_engine_last_submit(engine));
+	if (drm_debug & DRM_UT_DRIVER) {
+		struct drm_printer p = drm_debug_printer(__func__);
+
+		intel_engine_dump(engine, &p,
+				  "%s missed breadcrumb at %pS\n",
+				  engine->name, __builtin_return_address(0));
+	}
 
 	set_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings);
 }
 
 static void intel_breadcrumbs_hangcheck(struct timer_list *t)
 {
-	struct intel_engine_cs *engine = from_timer(engine, t,
-						    breadcrumbs.hangcheck);
+	struct intel_engine_cs *engine =
+		from_timer(engine, t, breadcrumbs.hangcheck);
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 
 	if (!b->irq_armed)
@@ -103,7 +122,7 @@ static void intel_breadcrumbs_hangcheck(struct timer_list *t)
 	 */
 	if (intel_engine_wakeup(engine) & ENGINE_WAKEUP_ASLEEP) {
 		missed_breadcrumb(engine);
-		mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
+		mod_timer(&b->fake_irq, jiffies + 1);
 	} else {
 		mod_timer(&b->hangcheck, wait_timeout());
 	}
@@ -123,28 +142,25 @@ static void intel_breadcrumbs_fake_irq(struct timer_list *t)
 	 */
 
 	spin_lock_irq(&b->irq_lock);
-	if (!__intel_breadcrumbs_wakeup(b))
+	if (b->irq_armed && !__intel_breadcrumbs_wakeup(b))
 		__intel_engine_disarm_breadcrumbs(engine);
 	spin_unlock_irq(&b->irq_lock);
 	if (!b->irq_armed)
 		return;
 
 	mod_timer(&b->fake_irq, jiffies + 1);
-
-	/* Ensure that even if the GPU hangs, we get woken up.
-	 *
-	 * However, note that if no one is waiting, we never notice
-	 * a gpu hang. Eventually, we will have to wait for a resource
-	 * held by the GPU and so trigger a hangcheck. In the most
-	 * pathological case, this will be upon memory starvation! To
-	 * prevent this, we also queue the hangcheck from the retire
-	 * worker.
-	 */
-	i915_queue_hangcheck(engine->i915);
 }
 
 static void irq_enable(struct intel_engine_cs *engine)
 {
+	/*
+	 * FIXME: Ideally we want this on the API boundary, but for the
+	 * sake of testing with mock breadcrumbs (no HW so unable to
+	 * enable irqs) we place it deep within the bowels, at the point
+	 * of no return.
+	 */
+	GEM_BUG_ON(!intel_irqs_enabled(engine->i915));
+
 	/* Enabling the IRQ may miss the generation of the interrupt, but
 	 * we still need to force the barrier before reading the seqno,
 	 * just in case.
@@ -171,39 +187,64 @@ void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 
 	lockdep_assert_held(&b->irq_lock);
 	GEM_BUG_ON(b->irq_wait);
+	GEM_BUG_ON(!b->irq_armed);
 
-	if (b->irq_enabled) {
+	GEM_BUG_ON(!b->irq_enabled);
+	if (!--b->irq_enabled)
 		irq_disable(engine);
-		b->irq_enabled = false;
-	}
 
 	b->irq_armed = false;
 }
 
+void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine)
+{
+	struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+	spin_lock_irq(&b->irq_lock);
+	if (!b->irq_enabled++)
+		irq_enable(engine);
+	GEM_BUG_ON(!b->irq_enabled); /* no overflow! */
+	spin_unlock_irq(&b->irq_lock);
+}
+
+void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine)
+{
+	struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+	spin_lock_irq(&b->irq_lock);
+	GEM_BUG_ON(!b->irq_enabled); /* no underflow! */
+	if (!--b->irq_enabled)
+		irq_disable(engine);
+	spin_unlock_irq(&b->irq_lock);
+}
+
 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
-	struct intel_wait *wait, *n, *first;
+	struct intel_wait *wait, *n;
 
 	if (!b->irq_armed)
 		goto wakeup_signaler;
 
-	/* We only disarm the irq when we are idle (all requests completed),
+	/*
+	 * We only disarm the irq when we are idle (all requests completed),
 	 * so if the bottom-half remains asleep, it missed the request
 	 * completion.
 	 */
+	if (intel_engine_wakeup(engine) & ENGINE_WAKEUP_ASLEEP)
+		missed_breadcrumb(engine);
 
 	spin_lock_irq(&b->rb_lock);
 
 	spin_lock(&b->irq_lock);
-	first = fetch_and_zero(&b->irq_wait);
-	__intel_engine_disarm_breadcrumbs(engine);
+	b->irq_wait = NULL;
+	if (b->irq_armed)
+		__intel_engine_disarm_breadcrumbs(engine);
 	spin_unlock(&b->irq_lock);
 
 	rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) {
 		RB_CLEAR_NODE(&wait->node);
-		if (wake_up_process(wait->tsk) && wait == first)
-			missed_breadcrumb(engine);
+		wake_up_process(wait->tsk);
 	}
 	b->waiters = RB_ROOT;
 
@@ -249,6 +290,7 @@ static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 	struct intel_engine_cs *engine =
 		container_of(b, struct intel_engine_cs, breadcrumbs);
 	struct drm_i915_private *i915 = engine->i915;
+	bool enabled;
 
 	lockdep_assert_held(&b->irq_lock);
 	if (b->irq_armed)
@@ -260,7 +302,6 @@ static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 	 * the irq.
 	 */
 	b->irq_armed = true;
-	GEM_BUG_ON(b->irq_enabled);
 
 	if (I915_SELFTEST_ONLY(b->mock)) {
 		/* For our mock objects we want to avoid interaction
@@ -281,14 +322,15 @@ static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
 	 */
 
 	/* No interrupts? Kick the waiter every jiffie! */
-	if (intel_irqs_enabled(i915)) {
-		if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))
-			irq_enable(engine);
-		b->irq_enabled = true;
+	enabled = false;
+	if (!b->irq_enabled++ &&
+	    !test_bit(engine->id, &i915->gpu_error.test_irq_rings)) {
+		irq_enable(engine);
+		enabled = true;
 	}
 
 	enable_fake_irq(b);
-	return true;
+	return enabled;
 }
 
 static inline struct intel_wait *to_wait(struct rb_node *node)
@@ -552,29 +594,16 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine,
 	spin_unlock_irq(&b->rb_lock);
 }
 
-static bool signal_valid(const struct drm_i915_gem_request *request)
-{
-	return intel_wait_check_request(&request->signaling.wait, request);
-}
-
 static bool signal_complete(const struct drm_i915_gem_request *request)
 {
 	if (!request)
 		return false;
 
-	/* If another process served as the bottom-half it may have already
-	 * signalled that this wait is already completed.
-	 */
-	if (intel_wait_complete(&request->signaling.wait))
-		return signal_valid(request);
-
-	/* Carefully check if the request is complete, giving time for the
+	/*
+	 * Carefully check if the request is complete, giving time for the
 	 * seqno to be visible or if the GPU hung.
 	 */
-	if (__i915_request_irq_complete(request))
-		return true;
-
-	return false;
+	return __i915_request_irq_complete(request);
 }
 
 static struct drm_i915_gem_request *to_signaler(struct rb_node *rb)
@@ -617,9 +646,13 @@ static int intel_breadcrumbs_signaler(void *arg)
 			request = i915_gem_request_get_rcu(request);
 		rcu_read_unlock();
 		if (signal_complete(request)) {
-			local_bh_disable();
-			dma_fence_signal(&request->fence);
-			local_bh_enable(); /* kick start the tasklets */
+			if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+				      &request->fence.flags)) {
+				local_bh_disable();
+				dma_fence_signal(&request->fence);
+				GEM_BUG_ON(!i915_gem_request_completed(request));
+				local_bh_enable(); /* kick start the tasklets */
+			}
 
 			spin_lock_irq(&b->rb_lock);
 
diff --git a/drivers/gpu/drm/i915/intel_cdclk.c b/drivers/gpu/drm/i915/intel_cdclk.c
index 60cf4e58389a..1704c8897afd 100644
--- a/drivers/gpu/drm/i915/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/intel_cdclk.c
@@ -437,13 +437,45 @@ static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 		return 200000;
 }
 
+static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
+{
+	if (IS_VALLEYVIEW(dev_priv)) {
+		if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
+			return 2;
+		else if (cdclk >= 266667)
+			return 1;
+		else
+			return 0;
+	} else {
+		/*
+		 * Specs are full of misinformation, but testing on actual
+		 * hardware has shown that we just need to write the desired
+		 * CCK divider into the Punit register.
+		 */
+		return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
+	}
+}
+
 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 			  struct intel_cdclk_state *cdclk_state)
 {
+	u32 val;
+
 	cdclk_state->vco = vlv_get_hpll_vco(dev_priv);
 	cdclk_state->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 					       CCK_DISPLAY_CLOCK_CONTROL,
 					       cdclk_state->vco);
+
+	mutex_lock(&dev_priv->pcu_lock);
+	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+	mutex_unlock(&dev_priv->pcu_lock);
+
+	if (IS_VALLEYVIEW(dev_priv))
+		cdclk_state->voltage_level = (val & DSPFREQGUAR_MASK) >>
+			DSPFREQGUAR_SHIFT;
+	else
+		cdclk_state->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
+			DSPFREQGUAR_SHIFT_CHV;
 }
 
 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
@@ -486,7 +518,19 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 			  const struct intel_cdclk_state *cdclk_state)
 {
 	int cdclk = cdclk_state->cdclk;
-	u32 val, cmd;
+	u32 val, cmd = cdclk_state->voltage_level;
+
+	switch (cdclk) {
+	case 400000:
+	case 333333:
+	case 320000:
+	case 266667:
+	case 200000:
+		break;
+	default:
+		MISSING_CASE(cdclk);
+		return;
+	}
 
 	/* There are cases where we can end up here with power domains
 	 * off and a CDCLK frequency other than the minimum, like when
@@ -496,13 +540,6 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 	 */
 	intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
-	if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
-		cmd = 2;
-	else if (cdclk == 266667)
-		cmd = 1;
-	else
-		cmd = 0;
-
 	mutex_lock(&dev_priv->pcu_lock);
 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 	val &= ~DSPFREQGUAR_MASK;
@@ -562,7 +599,7 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 			  const struct intel_cdclk_state *cdclk_state)
 {
 	int cdclk = cdclk_state->cdclk;
-	u32 val, cmd;
+	u32 val, cmd = cdclk_state->voltage_level;
 
 	switch (cdclk) {
 	case 333333:
@@ -583,13 +620,6 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 	 */
 	intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
-	/*
-	 * Specs are full of misinformation, but testing on actual
-	 * hardware has shown that we just need to write the desired
-	 * CCK divider into the Punit register.
-	 */
-	cmd = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
-
 	mutex_lock(&dev_priv->pcu_lock);
 	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 	val &= ~DSPFREQGUAR_MASK_CHV;
@@ -621,6 +651,21 @@ static int bdw_calc_cdclk(int min_cdclk)
 		return 337500;
 }
 
+static u8 bdw_calc_voltage_level(int cdclk)
+{
+	switch (cdclk) {
+	default:
+	case 337500:
+		return 2;
+	case 450000:
+		return 0;
+	case 540000:
+		return 1;
+	case 675000:
+		return 3;
+	}
+}
+
 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 			  struct intel_cdclk_state *cdclk_state)
 {
@@ -639,13 +684,20 @@ static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 		cdclk_state->cdclk = 337500;
 	else
 		cdclk_state->cdclk = 675000;
+
+	/*
+	 * Can't read this out :( Let's assume it's
+	 * at least what the CDCLK frequency requires.
+	 */
+	cdclk_state->voltage_level =
+		bdw_calc_voltage_level(cdclk_state->cdclk);
 }
 
 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 			  const struct intel_cdclk_state *cdclk_state)
 {
 	int cdclk = cdclk_state->cdclk;
-	uint32_t val, data;
+	uint32_t val;
 	int ret;
 
 	if (WARN((I915_READ(LCPLL_CTL) &
@@ -681,25 +733,21 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 	val &= ~LCPLL_CLK_FREQ_MASK;
 
 	switch (cdclk) {
+	default:
+		MISSING_CASE(cdclk);
+		/* fall through */
+	case 337500:
+		val |= LCPLL_CLK_FREQ_337_5_BDW;
+		break;
 	case 450000:
 		val |= LCPLL_CLK_FREQ_450;
-		data = 0;
 		break;
 	case 540000:
 		val |= LCPLL_CLK_FREQ_54O_BDW;
-		data = 1;
-		break;
-	case 337500:
-		val |= LCPLL_CLK_FREQ_337_5_BDW;
-		data = 2;
 		break;
 	case 675000:
 		val |= LCPLL_CLK_FREQ_675_BDW;
-		data = 3;
 		break;
-	default:
-		WARN(1, "invalid cdclk frequency\n");
-		return;
 	}
 
 	I915_WRITE(LCPLL_CTL, val);
@@ -713,16 +761,13 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 		DRM_ERROR("Switching back to LCPLL failed\n");
 
 	mutex_lock(&dev_priv->pcu_lock);
-	sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data);
+	sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
+				cdclk_state->voltage_level);
 	mutex_unlock(&dev_priv->pcu_lock);
 
 	I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 
 	intel_update_cdclk(dev_priv);
-
-	WARN(cdclk != dev_priv->cdclk.hw.cdclk,
-	     "cdclk requested %d kHz but got %d kHz\n",
-	     cdclk, dev_priv->cdclk.hw.cdclk);
 }
 
 static int skl_calc_cdclk(int min_cdclk, int vco)
@@ -748,6 +793,24 @@ static int skl_calc_cdclk(int min_cdclk, int vco)
 	}
 }
 
+static u8 skl_calc_voltage_level(int cdclk)
+{
+	switch (cdclk) {
+	default:
+	case 308571:
+	case 337500:
+		return 0;
+	case 450000:
+	case 432000:
+		return 1;
+	case 540000:
+		return 2;
+	case 617143:
+	case 675000:
+		return 3;
+	}
+}
+
 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 			     struct intel_cdclk_state *cdclk_state)
 {
@@ -798,7 +861,7 @@ static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 	cdclk_state->cdclk = cdclk_state->ref;
 
 	if (cdclk_state->vco == 0)
-		return;
+		goto out;
 
 	cdctl = I915_READ(CDCLK_CTL);
 
@@ -839,6 +902,14 @@ static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 			break;
 		}
 	}
+
+ out:
+	/*
+	 * Can't read this out :( Let's assume it's
+	 * at least what the CDCLK frequency requires.
+	 */
+	cdclk_state->voltage_level =
+		skl_calc_voltage_level(cdclk_state->cdclk);
 }
 
 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
@@ -917,11 +988,9 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 {
 	int cdclk = cdclk_state->cdclk;
 	int vco = cdclk_state->vco;
-	u32 freq_select, pcu_ack, cdclk_ctl;
+	u32 freq_select, cdclk_ctl;
 	int ret;
 
-	WARN_ON((cdclk == 24000) != (vco == 0));
-
 	mutex_lock(&dev_priv->pcu_lock);
 	ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
 				SKL_CDCLK_PREPARE_FOR_CHANGE,
@@ -936,25 +1005,24 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 
 	/* Choose frequency for this cdclk */
 	switch (cdclk) {
+	default:
+		WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
+		WARN_ON(vco != 0);
+		/* fall through */
+	case 308571:
+	case 337500:
+		freq_select = CDCLK_FREQ_337_308;
+		break;
 	case 450000:
 	case 432000:
 		freq_select = CDCLK_FREQ_450_432;
-		pcu_ack = 1;
 		break;
 	case 540000:
 		freq_select = CDCLK_FREQ_540;
-		pcu_ack = 2;
-		break;
-	case 308571:
-	case 337500:
-	default:
-		freq_select = CDCLK_FREQ_337_308;
-		pcu_ack = 0;
 		break;
 	case 617143:
 	case 675000:
 		freq_select = CDCLK_FREQ_675_617;
-		pcu_ack = 3;
 		break;
 	}
 
@@ -993,7 +1061,8 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 
 	/* inform PCU of the change */
 	mutex_lock(&dev_priv->pcu_lock);
-	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
+	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
+				cdclk_state->voltage_level);
 	mutex_unlock(&dev_priv->pcu_lock);
 
 	intel_update_cdclk(dev_priv);
@@ -1012,6 +1081,8 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
 		goto sanitize;
 
 	intel_update_cdclk(dev_priv);
+	intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
+
 	/* Is PLL enabled and locked ? */
 	if (dev_priv->cdclk.hw.vco == 0 ||
 	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.ref)
@@ -1072,6 +1143,7 @@ void skl_init_cdclk(struct drm_i915_private *dev_priv)
 	if (cdclk_state.vco == 0)
 		cdclk_state.vco = 8100000;
 	cdclk_state.cdclk = skl_calc_cdclk(0, cdclk_state.vco);
+	cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk);
 
 	skl_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1089,6 +1161,7 @@ void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
 
 	cdclk_state.cdclk = cdclk_state.ref;
 	cdclk_state.vco = 0;
+	cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk);
 
 	skl_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1117,6 +1190,11 @@ static int glk_calc_cdclk(int min_cdclk)
 		return 79200;
 }
 
+static u8 bxt_calc_voltage_level(int cdclk)
+{
+	return DIV_ROUND_UP(cdclk, 25000);
+}
+
 static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 {
 	int ratio;
@@ -1127,6 +1205,7 @@ static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 	switch (cdclk) {
 	default:
 		MISSING_CASE(cdclk);
+		/* fall through */
 	case 144000:
 	case 288000:
 	case 384000:
@@ -1151,6 +1230,7 @@ static int glk_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 	switch (cdclk) {
 	default:
 		MISSING_CASE(cdclk);
+		/* fall through */
 	case  79200:
 	case 158400:
 	case 316800:
@@ -1191,7 +1271,7 @@ static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
 	cdclk_state->cdclk = cdclk_state->ref;
 
 	if (cdclk_state->vco == 0)
-		return;
+		goto out;
 
 	divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
 
@@ -1215,6 +1295,14 @@ static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
 	}
 
 	cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, div);
+
+ out:
+	/*
+	 * Can't read this out :( Let's assume it's
+	 * at least what the CDCLK frequency requires.
+	 */
+	cdclk_state->voltage_level =
+		bxt_calc_voltage_level(cdclk_state->cdclk);
 }
 
 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
@@ -1263,31 +1351,34 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 
 	/* cdclk = vco / 2 / div{1,1.5,2,4} */
 	switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
-	case 8:
-		divider = BXT_CDCLK_CD2X_DIV_SEL_4;
-		break;
-	case 4:
-		divider = BXT_CDCLK_CD2X_DIV_SEL_2;
+	default:
+		WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
+		WARN_ON(vco != 0);
+		/* fall through */
+	case 2:
+		divider = BXT_CDCLK_CD2X_DIV_SEL_1;
 		break;
 	case 3:
 		WARN(IS_GEMINILAKE(dev_priv), "Unsupported divider\n");
 		divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
 		break;
-	case 2:
-		divider = BXT_CDCLK_CD2X_DIV_SEL_1;
+	case 4:
+		divider = BXT_CDCLK_CD2X_DIV_SEL_2;
 		break;
-	default:
-		WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
-		WARN_ON(vco != 0);
-
-		divider = BXT_CDCLK_CD2X_DIV_SEL_1;
+	case 8:
+		divider = BXT_CDCLK_CD2X_DIV_SEL_4;
 		break;
 	}
 
-	/* Inform power controller of upcoming frequency change */
+	/*
+	 * Inform power controller of upcoming frequency change. BSpec
+	 * requires us to wait up to 150usec, but that leads to timeouts;
+	 * the 2ms used here is based on experiment.
+	 */
 	mutex_lock(&dev_priv->pcu_lock);
-	ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
-				      0x80000000);
+	ret = sandybridge_pcode_write_timeout(dev_priv,
+					      HSW_PCODE_DE_WRITE_FREQ_REQ,
+					      0x80000000, 2000);
 	mutex_unlock(&dev_priv->pcu_lock);
 
 	if (ret) {
@@ -1318,8 +1409,15 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 	I915_WRITE(CDCLK_CTL, val);
 
 	mutex_lock(&dev_priv->pcu_lock);
-	ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
-				      DIV_ROUND_UP(cdclk, 25000));
+	/*
+	 * The timeout isn't specified, the 2ms used here is based on
+	 * experiment.
+	 * FIXME: Waiting for the request completion could be delayed until
+	 * the next PCODE request based on BSpec.
+	 */
+	ret = sandybridge_pcode_write_timeout(dev_priv,
+					      HSW_PCODE_DE_WRITE_FREQ_REQ,
+					      cdclk_state->voltage_level, 2000);
 	mutex_unlock(&dev_priv->pcu_lock);
 
 	if (ret) {
@@ -1336,6 +1434,7 @@ static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
 	u32 cdctl, expected;
 
 	intel_update_cdclk(dev_priv);
+	intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
 
 	if (dev_priv->cdclk.hw.vco == 0 ||
 	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.ref)
@@ -1411,6 +1510,7 @@ void bxt_init_cdclk(struct drm_i915_private *dev_priv)
 		cdclk_state.cdclk = bxt_calc_cdclk(0);
 		cdclk_state.vco = bxt_de_pll_vco(dev_priv, cdclk_state.cdclk);
 	}
+	cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk);
 
 	bxt_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1428,6 +1528,7 @@ void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
 
 	cdclk_state.cdclk = cdclk_state.ref;
 	cdclk_state.vco = 0;
+	cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk);
 
 	bxt_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1442,6 +1543,19 @@ static int cnl_calc_cdclk(int min_cdclk)
 		return 168000;
 }
 
+static u8 cnl_calc_voltage_level(int cdclk)
+{
+	switch (cdclk) {
+	default:
+	case 168000:
+		return 0;
+	case 336000:
+		return 1;
+	case 528000:
+		return 2;
+	}
+}
+
 static void cnl_cdclk_pll_update(struct drm_i915_private *dev_priv,
 				 struct intel_cdclk_state *cdclk_state)
 {
@@ -1475,7 +1589,7 @@ static void cnl_get_cdclk(struct drm_i915_private *dev_priv,
 	cdclk_state->cdclk = cdclk_state->ref;
 
 	if (cdclk_state->vco == 0)
-		return;
+		goto out;
 
 	divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
 
@@ -1492,6 +1606,14 @@ static void cnl_get_cdclk(struct drm_i915_private *dev_priv,
 	}
 
 	cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, div);
+
+ out:
+	/*
+	 * Can't read this out :( Let's assume it's
+	 * at least what the CDCLK frequency requires.
+	 */
+	cdclk_state->voltage_level =
+		cnl_calc_voltage_level(cdclk_state->cdclk);
 }
 
 static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
@@ -1532,7 +1654,7 @@ static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
 {
 	int cdclk = cdclk_state->cdclk;
 	int vco = cdclk_state->vco;
-	u32 val, divider, pcu_ack;
+	u32 val, divider;
 	int ret;
 
 	mutex_lock(&dev_priv->pcu_lock);
@@ -1549,30 +1671,15 @@ static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
 
 	/* cdclk = vco / 2 / div{1,2} */
 	switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
-	case 4:
-		divider = BXT_CDCLK_CD2X_DIV_SEL_2;
-		break;
-	case 2:
-		divider = BXT_CDCLK_CD2X_DIV_SEL_1;
-		break;
 	default:
 		WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
 		WARN_ON(vco != 0);
-
+		/* fall through */
+	case 2:
 		divider = BXT_CDCLK_CD2X_DIV_SEL_1;
 		break;
-	}
-
-	switch (cdclk) {
-	case 528000:
-		pcu_ack = 2;
-		break;
-	case 336000:
-		pcu_ack = 1;
-		break;
-	case 168000:
-	default:
-		pcu_ack = 0;
+	case 4:
+		divider = BXT_CDCLK_CD2X_DIV_SEL_2;
 		break;
 	}
 
@@ -1593,10 +1700,17 @@ static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
 
 	/* inform PCU of the change */
 	mutex_lock(&dev_priv->pcu_lock);
-	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
+	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
+				cdclk_state->voltage_level);
 	mutex_unlock(&dev_priv->pcu_lock);
 
 	intel_update_cdclk(dev_priv);
+
+	/*
+	 * Can't read out the voltage level :(
+	 * Let's just assume everything is as expected.
+	 */
+	dev_priv->cdclk.hw.voltage_level = cdclk_state->voltage_level;
 }
 
 static int cnl_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
@@ -1609,6 +1723,7 @@ static int cnl_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 	switch (cdclk) {
 	default:
 		MISSING_CASE(cdclk);
+		/* fall through */
 	case 168000:
 	case 336000:
 		ratio = dev_priv->cdclk.hw.ref == 19200 ? 35 : 28;
@@ -1626,6 +1741,7 @@ static void cnl_sanitize_cdclk(struct drm_i915_private *dev_priv)
 	u32 cdctl, expected;
 
 	intel_update_cdclk(dev_priv);
+	intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
 
 	if (dev_priv->cdclk.hw.vco == 0 ||
 	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.ref)
@@ -1685,6 +1801,7 @@ void cnl_init_cdclk(struct drm_i915_private *dev_priv)
 
 	cdclk_state.cdclk = cnl_calc_cdclk(0);
 	cdclk_state.vco = cnl_cdclk_pll_vco(dev_priv, cdclk_state.cdclk);
+	cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk);
 
 	cnl_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1702,22 +1819,48 @@ void cnl_uninit_cdclk(struct drm_i915_private *dev_priv)
 
 	cdclk_state.cdclk = cdclk_state.ref;
 	cdclk_state.vco = 0;
+	cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk);
 
 	cnl_set_cdclk(dev_priv, &cdclk_state);
 }
 
 /**
- * intel_cdclk_state_compare - Determine if two CDCLK states differ
+ * intel_cdclk_needs_modeset - Determine if two CDCLK states require a modeset on all pipes
  * @a: first CDCLK state
  * @b: second CDCLK state
  *
  * Returns:
- * True if the CDCLK states are identical, false if they differ.
+ * True if the CDCLK states require pipes to be off during reprogramming, false if not.
  */
-bool intel_cdclk_state_compare(const struct intel_cdclk_state *a,
+bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a,
 			       const struct intel_cdclk_state *b)
 {
-	return memcmp(a, b, sizeof(*a)) == 0;
+	return a->cdclk != b->cdclk ||
+		a->vco != b->vco ||
+		a->ref != b->ref;
+}
+
+/**
+ * intel_cdclk_changed - Determine if two CDCLK states are different
+ * @a: first CDCLK state
+ * @b: second CDCLK state
+ *
+ * Returns:
+ * True if the CDCLK states don't match, false if they do.
+ */
+bool intel_cdclk_changed(const struct intel_cdclk_state *a,
+			 const struct intel_cdclk_state *b)
+{
+	return intel_cdclk_needs_modeset(a, b) ||
+		a->voltage_level != b->voltage_level;
+}
+
+void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
+			    const char *context)
+{
+	DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, voltage level %d\n",
+			 context, cdclk_state->cdclk, cdclk_state->vco,
+			 cdclk_state->ref, cdclk_state->voltage_level);
 }
 
 /**
@@ -1731,29 +1874,28 @@ bool intel_cdclk_state_compare(const struct intel_cdclk_state *a,
 void intel_set_cdclk(struct drm_i915_private *dev_priv,
 		     const struct intel_cdclk_state *cdclk_state)
 {
-	if (intel_cdclk_state_compare(&dev_priv->cdclk.hw, cdclk_state))
+	if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state))
 		return;
 
 	if (WARN_ON_ONCE(!dev_priv->display.set_cdclk))
 		return;
 
-	DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz, VCO %d kHz, ref %d kHz\n",
-			 cdclk_state->cdclk, cdclk_state->vco,
-			 cdclk_state->ref);
+	intel_dump_cdclk_state(cdclk_state, "Changing CDCLK to");
 
 	dev_priv->display.set_cdclk(dev_priv, cdclk_state);
+
+	if (WARN(intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state),
+		 "cdclk state doesn't match!\n")) {
+		intel_dump_cdclk_state(&dev_priv->cdclk.hw, "[hw state]");
+		intel_dump_cdclk_state(cdclk_state, "[sw state]");
+	}
 }
 
 static int intel_pixel_rate_to_cdclk(struct drm_i915_private *dev_priv,
 				     int pixel_rate)
 {
 	if (INTEL_GEN(dev_priv) >= 10)
-		/*
-		 * FIXME: Switch to DIV_ROUND_UP(pixel_rate, 2)
-		 * once DDI clock voltage requirements are
-		 * handled correctly.
-		 */
-		return pixel_rate;
+		return DIV_ROUND_UP(pixel_rate, 2);
 	else if (IS_GEMINILAKE(dev_priv))
 		/*
 		 * FIXME: Avoid using a pixel clock that is more than 99% of the cdclk
@@ -1783,7 +1925,7 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
 	min_cdclk = intel_pixel_rate_to_cdclk(dev_priv, crtc_state->pixel_rate);
 
 	/* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
-	if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled)
+	if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
 		min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
 
 	/* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
@@ -1810,6 +1952,14 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
 	if (crtc_state->has_audio && INTEL_GEN(dev_priv) >= 9)
 		min_cdclk = max(2 * 96000, min_cdclk);
 
+	/*
+	 * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
+	 * than 320000KHz.
+	 */
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
+	    IS_VALLEYVIEW(dev_priv))
+		min_cdclk = max(320000, min_cdclk);
+
 	if (min_cdclk > dev_priv->max_cdclk_freq) {
 		DRM_DEBUG_KMS("required cdclk (%d kHz) exceeds max (%d kHz)\n",
 			      min_cdclk, dev_priv->max_cdclk_freq);
@@ -1846,6 +1996,43 @@ static int intel_compute_min_cdclk(struct drm_atomic_state *state)
 	return min_cdclk;
 }
 
+/*
+ * Note that this functions assumes that 0 is
+ * the lowest voltage value, and higher values
+ * correspond to increasingly higher voltages.
+ *
+ * Should that relationship no longer hold on
+ * future platforms this code will need to be
+ * adjusted.
+ */
+static u8 cnl_compute_min_voltage_level(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *crtc_state;
+	u8 min_voltage_level;
+	int i;
+	enum pipe pipe;
+
+	memcpy(state->min_voltage_level, dev_priv->min_voltage_level,
+	       sizeof(state->min_voltage_level));
+
+	for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
+		if (crtc_state->base.enable)
+			state->min_voltage_level[i] =
+				crtc_state->min_voltage_level;
+		else
+			state->min_voltage_level[i] = 0;
+	}
+
+	min_voltage_level = 0;
+	for_each_pipe(dev_priv, pipe)
+		min_voltage_level = max(state->min_voltage_level[pipe],
+					min_voltage_level);
+
+	return min_voltage_level;
+}
+
 static int vlv_modeset_calc_cdclk(struct drm_atomic_state *state)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->dev);
@@ -1859,11 +2046,15 @@ static int vlv_modeset_calc_cdclk(struct drm_atomic_state *state)
 	cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
 
 	intel_state->cdclk.logical.cdclk = cdclk;
+	intel_state->cdclk.logical.voltage_level =
+		vlv_calc_voltage_level(dev_priv, cdclk);
 
 	if (!intel_state->active_crtcs) {
 		cdclk = vlv_calc_cdclk(dev_priv, 0);
 
 		intel_state->cdclk.actual.cdclk = cdclk;
+		intel_state->cdclk.actual.voltage_level =
+			vlv_calc_voltage_level(dev_priv, cdclk);
 	} else {
 		intel_state->cdclk.actual =
 			intel_state->cdclk.logical;
@@ -1888,11 +2079,15 @@ static int bdw_modeset_calc_cdclk(struct drm_atomic_state *state)
 	cdclk = bdw_calc_cdclk(min_cdclk);
 
 	intel_state->cdclk.logical.cdclk = cdclk;
+	intel_state->cdclk.logical.voltage_level =
+		bdw_calc_voltage_level(cdclk);
 
 	if (!intel_state->active_crtcs) {
 		cdclk = bdw_calc_cdclk(0);
 
 		intel_state->cdclk.actual.cdclk = cdclk;
+		intel_state->cdclk.actual.voltage_level =
+			bdw_calc_voltage_level(cdclk);
 	} else {
 		intel_state->cdclk.actual =
 			intel_state->cdclk.logical;
@@ -1923,12 +2118,16 @@ static int skl_modeset_calc_cdclk(struct drm_atomic_state *state)
 
 	intel_state->cdclk.logical.vco = vco;
 	intel_state->cdclk.logical.cdclk = cdclk;
+	intel_state->cdclk.logical.voltage_level =
+		skl_calc_voltage_level(cdclk);
 
 	if (!intel_state->active_crtcs) {
 		cdclk = skl_calc_cdclk(0, vco);
 
 		intel_state->cdclk.actual.vco = vco;
 		intel_state->cdclk.actual.cdclk = cdclk;
+		intel_state->cdclk.actual.voltage_level =
+			skl_calc_voltage_level(cdclk);
 	} else {
 		intel_state->cdclk.actual =
 			intel_state->cdclk.logical;
@@ -1957,6 +2156,8 @@ static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
 
 	intel_state->cdclk.logical.vco = vco;
 	intel_state->cdclk.logical.cdclk = cdclk;
+	intel_state->cdclk.logical.voltage_level =
+		bxt_calc_voltage_level(cdclk);
 
 	if (!intel_state->active_crtcs) {
 		if (IS_GEMINILAKE(dev_priv)) {
@@ -1969,6 +2170,8 @@ static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
 
 		intel_state->cdclk.actual.vco = vco;
 		intel_state->cdclk.actual.cdclk = cdclk;
+		intel_state->cdclk.actual.voltage_level =
+			bxt_calc_voltage_level(cdclk);
 	} else {
 		intel_state->cdclk.actual =
 			intel_state->cdclk.logical;
@@ -1992,6 +2195,9 @@ static int cnl_modeset_calc_cdclk(struct drm_atomic_state *state)
 
 	intel_state->cdclk.logical.vco = vco;
 	intel_state->cdclk.logical.cdclk = cdclk;
+	intel_state->cdclk.logical.voltage_level =
+		max(cnl_calc_voltage_level(cdclk),
+		    cnl_compute_min_voltage_level(intel_state));
 
 	if (!intel_state->active_crtcs) {
 		cdclk = cnl_calc_cdclk(0);
@@ -1999,6 +2205,8 @@ static int cnl_modeset_calc_cdclk(struct drm_atomic_state *state)
 
 		intel_state->cdclk.actual.vco = vco;
 		intel_state->cdclk.actual.cdclk = cdclk;
+		intel_state->cdclk.actual.voltage_level =
+			cnl_calc_voltage_level(cdclk);
 	} else {
 		intel_state->cdclk.actual =
 			intel_state->cdclk.logical;
@@ -2012,12 +2220,7 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
 	int max_cdclk_freq = dev_priv->max_cdclk_freq;
 
 	if (INTEL_GEN(dev_priv) >= 10)
-		/*
-		 * FIXME: Allow '2 * max_cdclk_freq'
-		 * once DDI clock voltage requirements are
-		 * handled correctly.
-		 */
-		return max_cdclk_freq;
+		return 2 * max_cdclk_freq;
 	else if (IS_GEMINILAKE(dev_priv))
 		/*
 		 * FIXME: Limiting to 99% as a temporary workaround. See
@@ -2116,10 +2319,6 @@ void intel_update_cdclk(struct drm_i915_private *dev_priv)
 {
 	dev_priv->display.get_cdclk(dev_priv, &dev_priv->cdclk.hw);
 
-	DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n",
-			 dev_priv->cdclk.hw.cdclk, dev_priv->cdclk.hw.vco,
-			 dev_priv->cdclk.hw.ref);
-
 	/*
 	 * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
 	 * Programmng [sic] note: bit[9:2] should be programmed to the number
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index b8315bca852b..aa66e952a95d 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -370,7 +370,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
 	 */
 	if (IS_HASWELL(dev_priv) && intel_crtc_state->ips_enabled &&
 	    (intel_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)) {
-		hsw_disable_ips(intel_crtc);
+		hsw_disable_ips(intel_crtc_state);
 		reenable_ips = true;
 	}
 
@@ -380,7 +380,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
 	i9xx_load_luts(crtc_state);
 
 	if (reenable_ips)
-		hsw_enable_ips(intel_crtc);
+		hsw_enable_ips(intel_crtc_state);
 }
 
 static void bdw_load_degamma_lut(struct drm_crtc_state *state)
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 437339f5d098..9f31aea51dff 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -119,6 +119,8 @@ static unsigned int intel_crt_get_flags(struct intel_encoder *encoder)
 static void intel_crt_get_config(struct intel_encoder *encoder,
 				 struct intel_crtc_state *pipe_config)
 {
+	pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG);
+
 	pipe_config->base.adjusted_mode.flags |= intel_crt_get_flags(encoder);
 
 	pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock;
@@ -217,11 +219,9 @@ static void hsw_disable_crt(struct intel_encoder *encoder,
 			    const struct intel_crtc_state *old_crtc_state,
 			    const struct drm_connector_state *old_conn_state)
 {
-	struct drm_crtc *crtc = old_crtc_state->base.crtc;
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	WARN_ON(!intel_crtc->config->has_pch_encoder);
+	WARN_ON(!old_crtc_state->has_pch_encoder);
 
 	intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
 }
@@ -245,46 +245,42 @@ static void hsw_post_disable_crt(struct intel_encoder *encoder,
 }
 
 static void hsw_pre_pll_enable_crt(struct intel_encoder *encoder,
-				   const struct intel_crtc_state *pipe_config,
+				   const struct intel_crtc_state *crtc_state,
 				   const struct drm_connector_state *conn_state)
 {
-	struct drm_crtc *crtc = pipe_config->base.crtc;
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	WARN_ON(!intel_crtc->config->has_pch_encoder);
+	WARN_ON(!crtc_state->has_pch_encoder);
 
 	intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
 }
 
 static void hsw_pre_enable_crt(struct intel_encoder *encoder,
-			       const struct intel_crtc_state *pipe_config,
+			       const struct intel_crtc_state *crtc_state,
 			       const struct drm_connector_state *conn_state)
 {
-	struct drm_crtc *crtc = pipe_config->base.crtc;
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int pipe = intel_crtc->pipe;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	enum pipe pipe = crtc->pipe;
 
-	WARN_ON(!intel_crtc->config->has_pch_encoder);
+	WARN_ON(!crtc_state->has_pch_encoder);
 
 	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 
-	dev_priv->display.fdi_link_train(intel_crtc, pipe_config);
+	dev_priv->display.fdi_link_train(crtc, crtc_state);
 }
 
 static void hsw_enable_crt(struct intel_encoder *encoder,
-			   const struct intel_crtc_state *pipe_config,
+			   const struct intel_crtc_state *crtc_state,
 			   const struct drm_connector_state *conn_state)
 {
-	struct drm_crtc *crtc = pipe_config->base.crtc;
-	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int pipe = intel_crtc->pipe;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	enum pipe pipe = crtc->pipe;
 
-	WARN_ON(!intel_crtc->config->has_pch_encoder);
+	WARN_ON(!crtc_state->has_pch_encoder);
 
-	intel_crt_set_dpms(encoder, pipe_config, DRM_MODE_DPMS_ON);
+	intel_crt_set_dpms(encoder, crtc_state, DRM_MODE_DPMS_ON);
 
 	intel_wait_for_vblank(dev_priv, pipe);
 	intel_wait_for_vblank(dev_priv, pipe);
@@ -293,10 +289,10 @@ static void hsw_enable_crt(struct intel_encoder *encoder,
 }
 
 static void intel_enable_crt(struct intel_encoder *encoder,
-			     const struct intel_crtc_state *pipe_config,
+			     const struct intel_crtc_state *crtc_state,
 			     const struct drm_connector_state *conn_state)
 {
-	intel_crt_set_dpms(encoder, pipe_config, DRM_MODE_DPMS_ON);
+	intel_crt_set_dpms(encoder, crtc_state, DRM_MODE_DPMS_ON);
 }
 
 static enum drm_mode_status
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index da9de47562b8..7fe4aac0facc 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -37,16 +37,16 @@
 #define I915_CSR_GLK "i915/glk_dmc_ver1_04.bin"
 #define GLK_CSR_VERSION_REQUIRED	CSR_VERSION(1, 4)
 
-#define I915_CSR_CNL "i915/cnl_dmc_ver1_04.bin"
-#define CNL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 4)
+#define I915_CSR_CNL "i915/cnl_dmc_ver1_06.bin"
+#define CNL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 6)
 
-#define I915_CSR_KBL "i915/kbl_dmc_ver1_01.bin"
+#define I915_CSR_KBL "i915/kbl_dmc_ver1_04.bin"
 MODULE_FIRMWARE(I915_CSR_KBL);
-#define KBL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 1)
+#define KBL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 4)
 
-#define I915_CSR_SKL "i915/skl_dmc_ver1_26.bin"
+#define I915_CSR_SKL "i915/skl_dmc_ver1_27.bin"
 MODULE_FIRMWARE(I915_CSR_SKL);
-#define SKL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 26)
+#define SKL_CSR_VERSION_REQUIRED	CSR_VERSION(1, 27)
 
 #define I915_CSR_BXT "i915/bxt_dmc_ver1_07.bin"
 MODULE_FIRMWARE(I915_CSR_BXT);
@@ -198,6 +198,7 @@ intel_get_stepping_info(struct drm_i915_private *dev_priv)
 		si = bxt_stepping_info;
 	} else {
 		size = 0;
+		si = NULL;
 	}
 
 	if (INTEL_REVID(dev_priv) < size)
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 58a3755544b2..6aff9d096e13 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -492,24 +492,6 @@ static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_1_05V[] = {
 	{ 0x2, 0x7F, 0x3F, 0x00, 0x00 },	/* 400   400      0.0   */
 };
 
-enum port intel_ddi_get_encoder_port(struct intel_encoder *encoder)
-{
-	switch (encoder->type) {
-	case INTEL_OUTPUT_DP_MST:
-		return enc_to_mst(&encoder->base)->primary->port;
-	case INTEL_OUTPUT_DP:
-	case INTEL_OUTPUT_EDP:
-	case INTEL_OUTPUT_HDMI:
-	case INTEL_OUTPUT_UNKNOWN:
-		return enc_to_dig_port(&encoder->base)->port;
-	case INTEL_OUTPUT_ANALOG:
-		return PORT_E;
-	default:
-		MISSING_CASE(encoder->type);
-		return PORT_A;
-	}
-}
-
 static const struct ddi_buf_trans *
 bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 {
@@ -811,31 +793,24 @@ static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port por
  * values in advance. This function programs the correct values for
  * DP/eDP/FDI use cases.
  */
-static void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder)
+static void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder,
+					 const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 iboost_bit = 0;
 	int i, n_entries;
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	const struct ddi_buf_trans *ddi_translations;
 
-	switch (encoder->type) {
-	case INTEL_OUTPUT_EDP:
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
+		ddi_translations = intel_ddi_get_buf_trans_fdi(dev_priv,
+							       &n_entries);
+	else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
 		ddi_translations = intel_ddi_get_buf_trans_edp(dev_priv, port,
 							       &n_entries);
-		break;
-	case INTEL_OUTPUT_DP:
+	else
 		ddi_translations = intel_ddi_get_buf_trans_dp(dev_priv, port,
 							      &n_entries);
-		break;
-	case INTEL_OUTPUT_ANALOG:
-		ddi_translations = intel_ddi_get_buf_trans_fdi(dev_priv,
-							       &n_entries);
-		break;
-	default:
-		MISSING_CASE(encoder->type);
-		return;
-	}
 
 	/* If we're boosting the current, set bit 31 of trans1 */
 	if (IS_GEN9_BC(dev_priv) &&
@@ -861,7 +836,7 @@ static void intel_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 iboost_bit = 0;
 	int n_entries;
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	const struct ddi_buf_trans *ddi_translations;
 
 	ddi_translations = intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries);
@@ -937,7 +912,7 @@ void hsw_fdi_link_train(struct intel_crtc *crtc,
 
 	for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
 		WARN_ON(encoder->type != INTEL_OUTPUT_ANALOG);
-		intel_prepare_dp_ddi_buffers(encoder);
+		intel_prepare_dp_ddi_buffers(encoder, crtc_state);
 	}
 
 	/* Set the FDI_RX_MISC pwrdn lanes and the 2 workarounds listed at the
@@ -1448,19 +1423,16 @@ static void hsw_ddi_clock_get(struct intel_encoder *encoder,
 	ddi_dotclock_get(pipe_config);
 }
 
-static int bxt_calc_pll_link(struct drm_i915_private *dev_priv,
-			     enum intel_dpll_id pll_id)
+static int bxt_calc_pll_link(struct intel_crtc_state *crtc_state)
 {
-	struct intel_shared_dpll *pll;
 	struct intel_dpll_hw_state *state;
 	struct dpll clock;
 
 	/* For DDI ports we always use a shared PLL. */
-	if (WARN_ON(pll_id == DPLL_ID_PRIVATE))
+	if (WARN_ON(!crtc_state->shared_dpll))
 		return 0;
 
-	pll = &dev_priv->shared_dplls[pll_id];
-	state = &pll->state.hw_state;
+	state = &crtc_state->dpll_hw_state;
 
 	clock.m1 = 2;
 	clock.m2 = (state->pll0 & PORT_PLL_M2_MASK) << 22;
@@ -1474,19 +1446,15 @@ static int bxt_calc_pll_link(struct drm_i915_private *dev_priv,
 }
 
 static void bxt_ddi_clock_get(struct intel_encoder *encoder,
-				struct intel_crtc_state *pipe_config)
+			      struct intel_crtc_state *pipe_config)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
-	enum intel_dpll_id pll_id = port;
-
-	pipe_config->port_clock = bxt_calc_pll_link(dev_priv, pll_id);
+	pipe_config->port_clock = bxt_calc_pll_link(pipe_config);
 
 	ddi_dotclock_get(pipe_config);
 }
 
-void intel_ddi_clock_get(struct intel_encoder *encoder,
-			 struct intel_crtc_state *pipe_config)
+static void intel_ddi_clock_get(struct intel_encoder *encoder,
+				struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
@@ -1504,33 +1472,34 @@ void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-	int type = encoder->type;
-	uint32_t temp;
+	u32 temp;
 
-	if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP_MST) {
-		WARN_ON(transcoder_is_dsi(cpu_transcoder));
+	if (!intel_crtc_has_dp_encoder(crtc_state))
+		return;
 
-		temp = TRANS_MSA_SYNC_CLK;
-		switch (crtc_state->pipe_bpp) {
-		case 18:
-			temp |= TRANS_MSA_6_BPC;
-			break;
-		case 24:
-			temp |= TRANS_MSA_8_BPC;
-			break;
-		case 30:
-			temp |= TRANS_MSA_10_BPC;
-			break;
-		case 36:
-			temp |= TRANS_MSA_12_BPC;
-			break;
-		default:
-			BUG();
-		}
-		I915_WRITE(TRANS_MSA_MISC(cpu_transcoder), temp);
+	WARN_ON(transcoder_is_dsi(cpu_transcoder));
+
+	temp = TRANS_MSA_SYNC_CLK;
+	switch (crtc_state->pipe_bpp) {
+	case 18:
+		temp |= TRANS_MSA_6_BPC;
+		break;
+	case 24:
+		temp |= TRANS_MSA_8_BPC;
+		break;
+	case 30:
+		temp |= TRANS_MSA_10_BPC;
+		break;
+	case 36:
+		temp |= TRANS_MSA_12_BPC;
+		break;
+	default:
+		MISSING_CASE(crtc_state->pipe_bpp);
+		break;
 	}
+
+	I915_WRITE(TRANS_MSA_MISC(cpu_transcoder), temp);
 }
 
 void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
@@ -1540,6 +1509,7 @@ void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	uint32_t temp;
+
 	temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
 	if (state == true)
 		temp |= TRANS_DDI_DP_VC_PAYLOAD_ALLOC;
@@ -1555,8 +1525,7 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-	enum port port = intel_ddi_get_encoder_port(encoder);
-	int type = encoder->type;
+	enum port port = encoder->port;
 	uint32_t temp;
 
 	/* Enable TRANS_DDI_FUNC_CTL for the pipe to work in HDMI mode */
@@ -1611,7 +1580,7 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state)
 		}
 	}
 
-	if (type == INTEL_OUTPUT_HDMI) {
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
 		if (crtc_state->has_hdmi_sink)
 			temp |= TRANS_DDI_MODE_SELECT_HDMI;
 		else
@@ -1621,19 +1590,15 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state)
 			temp |= TRANS_DDI_HDMI_SCRAMBLING_MASK;
 		if (crtc_state->hdmi_high_tmds_clock_ratio)
 			temp |= TRANS_DDI_HIGH_TMDS_CHAR_RATE;
-	} else if (type == INTEL_OUTPUT_ANALOG) {
+	} else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
 		temp |= TRANS_DDI_MODE_SELECT_FDI;
 		temp |= (crtc_state->fdi_lanes - 1) << 1;
-	} else if (type == INTEL_OUTPUT_DP ||
-		   type == INTEL_OUTPUT_EDP) {
-		temp |= TRANS_DDI_MODE_SELECT_DP_SST;
-		temp |= DDI_PORT_WIDTH(crtc_state->lane_count);
-	} else if (type == INTEL_OUTPUT_DP_MST) {
+	} else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) {
 		temp |= TRANS_DDI_MODE_SELECT_DP_MST;
 		temp |= DDI_PORT_WIDTH(crtc_state->lane_count);
 	} else {
-		WARN(1, "Invalid encoder type %d for pipe %c\n",
-		     encoder->type, pipe_name(pipe));
+		temp |= TRANS_DDI_MODE_SELECT_DP_SST;
+		temp |= DDI_PORT_WIDTH(crtc_state->lane_count);
 	}
 
 	I915_WRITE(TRANS_DDI_FUNC_CTL(cpu_transcoder), temp);
@@ -1656,7 +1621,7 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_encoder *encoder = intel_connector->encoder;
 	int type = intel_connector->base.connector_type;
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	enum pipe pipe = 0;
 	enum transcoder cpu_transcoder;
 	uint32_t tmp;
@@ -1715,9 +1680,9 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
 {
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
+	enum pipe p;
 	u32 tmp;
-	int i;
 	bool ret;
 
 	if (!intel_display_power_get_if_enabled(dev_priv,
@@ -1752,15 +1717,17 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
 		goto out;
 	}
 
-	for (i = TRANSCODER_A; i <= TRANSCODER_C; i++) {
-		tmp = I915_READ(TRANS_DDI_FUNC_CTL(i));
+	for_each_pipe(dev_priv, p) {
+		enum transcoder cpu_transcoder = (enum transcoder) p;
+
+		tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
 
 		if ((tmp & TRANS_DDI_PORT_MASK) == TRANS_DDI_SELECT_PORT(port)) {
 			if ((tmp & TRANS_DDI_MODE_SELECT_MASK) ==
 			    TRANS_DDI_MODE_SELECT_DP_MST)
 				goto out;
 
-			*pipe = i;
+			*pipe = p;
 			ret = true;
 
 			goto out;
@@ -1800,7 +1767,7 @@ void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state)
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
 	if (cpu_transcoder != TRANSCODER_EDP)
@@ -1836,8 +1803,8 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder,
 			       int level, enum intel_output_type type)
 {
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base);
-	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
-	enum port port = intel_dig_port->port;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
 	uint8_t iboost;
 
 	if (type == INTEL_OUTPUT_HDMI)
@@ -1939,8 +1906,8 @@ static void cnl_ddi_vswing_program(struct intel_encoder *encoder,
 				   int level, enum intel_output_type type)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
 	const struct cnl_ddi_buf_trans *ddi_translations;
+	enum port port = encoder->port;
 	int n_entries, ln;
 	u32 val;
 
@@ -2003,7 +1970,7 @@ static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder,
 				    int level, enum intel_output_type type)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	int width, rate, ln;
 	u32 val;
 
@@ -2122,13 +2089,13 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 				 const struct intel_shared_dpll *pll)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	uint32_t val;
 
 	if (WARN_ON(!pll))
 		return;
 
-	 mutex_lock(&dev_priv->dpll_lock);
+	mutex_lock(&dev_priv->dpll_lock);
 
 	if (IS_CANNONLAKE(dev_priv)) {
 		/* Configure DPCLKA_CFGCR0 to map the DPLL to the DDI. */
@@ -2150,7 +2117,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 		val = I915_READ(DPLL_CTRL2);
 
 		val &= ~(DPLL_CTRL2_DDI_CLK_OFF(port) |
-			DPLL_CTRL2_DDI_CLK_SEL_MASK(port));
+			 DPLL_CTRL2_DDI_CLK_SEL_MASK(port));
 		val |= (DPLL_CTRL2_DDI_CLK_SEL(pll->id, port) |
 			DPLL_CTRL2_DDI_SEL_OVERRIDE(port));
 
@@ -2166,7 +2133,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 static void intel_ddi_clk_disable(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 
 	if (IS_CANNONLAKE(dev_priv))
 		I915_WRITE(DPCLKA_CFGCR0, I915_READ(DPCLKA_CFGCR0) |
@@ -2184,7 +2151,7 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
 	bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST);
 	int level = intel_ddi_dp_level(intel_dp);
@@ -2205,11 +2172,10 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 	else if (IS_GEN9_LP(dev_priv))
 		bxt_ddi_vswing_sequence(encoder, level, encoder->type);
 	else
-		intel_prepare_dp_ddi_buffers(encoder);
+		intel_prepare_dp_ddi_buffers(encoder, crtc_state);
 
 	intel_ddi_init_dp_buf_reg(encoder);
-	if (!is_mst)
-		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
+	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
 	intel_dp_start_link_train(intel_dp);
 	if (port != PORT_A || INTEL_GEN(dev_priv) >= 9)
 		intel_dp_stop_link_train(intel_dp);
@@ -2222,7 +2188,7 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder,
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base);
 	struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	int level = intel_ddi_hdmi_level(dev_priv, port);
 	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
 
@@ -2254,6 +2220,19 @@ static void intel_ddi_pre_enable(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
+	/*
+	 * When called from DP MST code:
+	 * - conn_state will be NULL
+	 * - encoder will be the main encoder (ie. mst->primary)
+	 * - the main connector associated with this port
+	 *   won't be active or linked to a crtc
+	 * - crtc_state will be the state of the first stream to
+	 *   be activated on this port, and it may not be the same
+	 *   stream that will be deactivated last, but each stream
+	 *   should have a state that is identical when it comes to
+	 *   the DP link parameteres
+	 */
+
 	WARN_ON(crtc_state->has_pch_encoder);
 
 	intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
@@ -2267,7 +2246,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *encoder,
 static void intel_disable_ddi_buf(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	bool wait = false;
 	u32 val;
 
@@ -2294,19 +2273,12 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
 	struct intel_dp *intel_dp = &dig_port->dp;
-	/*
-	 * old_crtc_state and old_conn_state are NULL when called from
-	 * DP_MST. The main connector associated with this port is never
-	 * bound to a crtc for MST.
-	 */
-	bool is_mst = !old_crtc_state;
 
 	/*
 	 * Power down sink before disabling the port, otherwise we end
 	 * up getting interrupts from the sink on detecting link loss.
 	 */
-	if (!is_mst)
-		intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
+	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
 
 	intel_disable_ddi_buf(encoder);
 
@@ -2343,12 +2315,19 @@ static void intel_ddi_post_disable(struct intel_encoder *encoder,
 				   const struct drm_connector_state *old_conn_state)
 {
 	/*
-	 * old_crtc_state and old_conn_state are NULL when called from
-	 * DP_MST. The main connector associated with this port is never
-	 * bound to a crtc for MST.
+	 * When called from DP MST code:
+	 * - old_conn_state will be NULL
+	 * - encoder will be the main encoder (ie. mst->primary)
+	 * - the main connector associated with this port
+	 *   won't be active or linked to a crtc
+	 * - old_crtc_state will be the state of the last stream to
+	 *   be deactivated on this port, and it may not be the same
+	 *   stream that was activated last, but each stream
+	 *   should have a state that is identical when it comes to
+	 *   the DP link parameteres
 	 */
-	if (old_crtc_state &&
-	    intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI))
+
+	if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI))
 		intel_ddi_post_disable_hdmi(encoder,
 					    old_crtc_state, old_conn_state);
 	else
@@ -2396,7 +2375,7 @@ static void intel_enable_ddi_dp(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 
 	if (port == PORT_A && INTEL_GEN(dev_priv) < 9)
 		intel_dp_stop_link_train(intel_dp);
@@ -2415,7 +2394,7 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
-	enum port port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 
 	intel_hdmi_handle_sink_scrambling(encoder,
 					  conn_state->connector,
@@ -2450,7 +2429,8 @@ static void intel_disable_ddi_dp(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
 	if (old_crtc_state->has_audio)
-		intel_audio_codec_disable(encoder);
+		intel_audio_codec_disable(encoder,
+					  old_crtc_state, old_conn_state);
 
 	intel_edp_drrs_disable(intel_dp, old_crtc_state);
 	intel_psr_disable(intel_dp, old_crtc_state);
@@ -2462,7 +2442,8 @@ static void intel_disable_ddi_hdmi(struct intel_encoder *encoder,
 				   const struct drm_connector_state *old_conn_state)
 {
 	if (old_crtc_state->has_audio)
-		intel_audio_codec_disable(encoder);
+		intel_audio_codec_disable(encoder,
+					  old_crtc_state, old_conn_state);
 
 	intel_hdmi_handle_sink_scrambling(encoder,
 					  old_conn_state->connector,
@@ -2493,7 +2474,7 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv =
 		to_i915(intel_dig_port->base.base.dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 	uint32_t val;
 	bool wait = false;
 
@@ -2534,24 +2515,31 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
 	udelay(600);
 }
 
-bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
-				 struct intel_crtc *intel_crtc)
+static bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
+				       enum transcoder cpu_transcoder)
 {
-	u32 temp;
+	if (cpu_transcoder == TRANSCODER_EDP)
+		return false;
 
-	if (intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_AUDIO)) {
-		temp = I915_READ(HSW_AUD_PIN_ELD_CP_VLD);
-		if (temp & AUDIO_OUTPUT_ENABLE(intel_crtc->pipe))
-			return true;
-	}
-	return false;
+	if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_AUDIO))
+		return false;
+
+	return I915_READ(HSW_AUD_PIN_ELD_CP_VLD) &
+		AUDIO_OUTPUT_ENABLE(cpu_transcoder);
+}
+
+void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv,
+					 struct intel_crtc_state *crtc_state)
+{
+	if (IS_CANNONLAKE(dev_priv) && crtc_state->port_clock > 594000)
+		crtc_state->min_voltage_level = 2;
 }
 
 void intel_ddi_get_config(struct intel_encoder *encoder,
 			  struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
 	enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
 	struct intel_digital_port *intel_dig_port;
 	u32 temp, flags = 0;
@@ -2604,12 +2592,23 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 			pipe_config->hdmi_high_tmds_clock_ratio = true;
 		/* fall through */
 	case TRANS_DDI_MODE_SELECT_DVI:
+		pipe_config->output_types |= BIT(INTEL_OUTPUT_HDMI);
 		pipe_config->lane_count = 4;
 		break;
 	case TRANS_DDI_MODE_SELECT_FDI:
+		pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG);
 		break;
 	case TRANS_DDI_MODE_SELECT_DP_SST:
+		if (encoder->type == INTEL_OUTPUT_EDP)
+			pipe_config->output_types |= BIT(INTEL_OUTPUT_EDP);
+		else
+			pipe_config->output_types |= BIT(INTEL_OUTPUT_DP);
+		pipe_config->lane_count =
+			((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
+		intel_dp_get_m_n(intel_crtc, pipe_config);
+		break;
 	case TRANS_DDI_MODE_SELECT_DP_MST:
+		pipe_config->output_types |= BIT(INTEL_OUTPUT_DP_MST);
 		pipe_config->lane_count =
 			((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
 		intel_dp_get_m_n(intel_crtc, pipe_config);
@@ -2619,7 +2618,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 	}
 
 	pipe_config->has_audio =
-		intel_ddi_is_audio_enabled(dev_priv, intel_crtc);
+		intel_ddi_is_audio_enabled(dev_priv, cpu_transcoder);
 
 	if (encoder->type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.bpp &&
 	    pipe_config->pipe_bpp > dev_priv->vbt.edp.bpp) {
@@ -2646,6 +2645,26 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 	if (IS_GEN9_LP(dev_priv))
 		pipe_config->lane_lat_optim_mask =
 			bxt_ddi_phy_get_lane_lat_optim_mask(encoder);
+
+	intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
+}
+
+static enum intel_output_type
+intel_ddi_compute_output_type(struct intel_encoder *encoder,
+			      struct intel_crtc_state *crtc_state,
+			      struct drm_connector_state *conn_state)
+{
+	switch (conn_state->connector->connector_type) {
+	case DRM_MODE_CONNECTOR_HDMIA:
+		return INTEL_OUTPUT_HDMI;
+	case DRM_MODE_CONNECTOR_eDP:
+		return INTEL_OUTPUT_EDP;
+	case DRM_MODE_CONNECTOR_DisplayPort:
+		return INTEL_OUTPUT_DP;
+	default:
+		MISSING_CASE(conn_state->connector->connector_type);
+		return INTEL_OUTPUT_UNUSED;
+	}
 }
 
 static bool intel_ddi_compute_config(struct intel_encoder *encoder,
@@ -2653,24 +2672,22 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder,
 				     struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	int type = encoder->type;
-	int port = intel_ddi_get_encoder_port(encoder);
+	enum port port = encoder->port;
 	int ret;
 
-	WARN(type == INTEL_OUTPUT_UNKNOWN, "compute_config() on unknown output!\n");
-
 	if (port == PORT_A)
 		pipe_config->cpu_transcoder = TRANSCODER_EDP;
 
-	if (type == INTEL_OUTPUT_HDMI)
+	if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
 		ret = intel_hdmi_compute_config(encoder, pipe_config, conn_state);
 	else
 		ret = intel_dp_compute_config(encoder, pipe_config, conn_state);
 
 	if (IS_GEN9_LP(dev_priv) && ret)
 		pipe_config->lane_lat_optim_mask =
-			bxt_ddi_phy_calc_lane_lat_optim_mask(encoder,
-							     pipe_config->lane_count);
+			bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
+
+	intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
 
 	return ret;
 
@@ -2685,7 +2702,7 @@ static struct intel_connector *
 intel_ddi_init_dp_connector(struct intel_digital_port *intel_dig_port)
 {
 	struct intel_connector *connector;
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 
 	connector = intel_connector_alloc();
 	if (!connector)
@@ -2704,7 +2721,7 @@ static struct intel_connector *
 intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port)
 {
 	struct intel_connector *connector;
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 
 	connector = intel_connector_alloc();
 	if (!connector)
@@ -2716,6 +2733,34 @@ intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port)
 	return connector;
 }
 
+static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dport)
+{
+	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
+
+	if (dport->base.port != PORT_A)
+		return false;
+
+	if (dport->saved_port_bits & DDI_A_4_LANES)
+		return false;
+
+	/* Broxton/Geminilake: Bspec says that DDI_A_4_LANES is the only
+	 *                     supported configuration
+	 */
+	if (IS_GEN9_LP(dev_priv))
+		return true;
+
+	/* Cannonlake: Most of SKUs don't support DDI_E, and the only
+	 *             one who does also have a full A/E split called
+	 *             DDI_F what makes DDI_E useless. However for this
+	 *             case let's trust VBT info.
+	 */
+	if (IS_CANNONLAKE(dev_priv) &&
+	    !intel_bios_is_port_present(dev_priv, PORT_E))
+		return true;
+
+	return false;
+}
+
 void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 {
 	struct intel_digital_port *intel_dig_port;
@@ -2782,6 +2827,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 	drm_encoder_init(&dev_priv->drm, encoder, &intel_ddi_funcs,
 			 DRM_MODE_ENCODER_TMDS, "DDI %c", port_name(port));
 
+	intel_encoder->compute_output_type = intel_ddi_compute_output_type;
 	intel_encoder->compute_config = intel_ddi_compute_config;
 	intel_encoder->enable = intel_enable_ddi;
 	if (IS_GEN9_LP(dev_priv))
@@ -2794,7 +2840,6 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 	intel_encoder->suspend = intel_dp_encoder_suspend;
 	intel_encoder->get_power_domains = intel_ddi_get_power_domains;
 
-	intel_dig_port->port = port;
 	intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) &
 					  (DDI_BUF_PORT_REVERSAL |
 					   DDI_A_4_LANES);
@@ -2825,23 +2870,20 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 	}
 
 	/*
-	 * Bspec says that DDI_A_4_LANES is the only supported configuration
-	 * for Broxton.  Yet some BIOS fail to set this bit on port A if eDP
-	 * wasn't lit up at boot.  Force this bit on in our internal
-	 * configuration so that we use the proper lane count for our
-	 * calculations.
+	 * Some BIOS might fail to set this bit on port A if eDP
+	 * wasn't lit up at boot.  Force this bit set when needed
+	 * so we use the proper lane count for our calculations.
 	 */
-	if (IS_GEN9_LP(dev_priv) && port == PORT_A) {
-		if (!(intel_dig_port->saved_port_bits & DDI_A_4_LANES)) {
-			DRM_DEBUG_KMS("BXT BIOS forgot to set DDI_A_4_LANES for port A; fixing\n");
-			intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
-			max_lanes = 4;
-		}
+	if (intel_ddi_a_force_4_lanes(intel_dig_port)) {
+		DRM_DEBUG_KMS("Forcing DDI_A_4_LANES for port A\n");
+		intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
+		max_lanes = 4;
 	}
 
+	intel_dig_port->dp.output_reg = INVALID_MMIO_REG;
 	intel_dig_port->max_lanes = max_lanes;
 
-	intel_encoder->type = INTEL_OUTPUT_UNKNOWN;
+	intel_encoder->type = INTEL_OUTPUT_DDI;
 	intel_encoder->power_domain = intel_port_to_power_domain(port);
 	intel_encoder->port = port;
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 875d428ea75f..d28592e43512 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -22,6 +22,9 @@
  *
  */
 
+#include <drm/drm_print.h>
+
+#include "intel_device_info.h"
 #include "i915_drv.h"
 
 #define PLATFORM_NAME(x) [INTEL_##x] = #x
@@ -67,21 +70,55 @@ const char *intel_platform_name(enum intel_platform platform)
 	return platform_names[platform];
 }
 
-void intel_device_info_dump(struct drm_i915_private *dev_priv)
+void intel_device_info_dump_flags(const struct intel_device_info *info,
+				  struct drm_printer *p)
 {
-	const struct intel_device_info *info = &dev_priv->info;
-
-	DRM_DEBUG_DRIVER("i915 device info: platform=%s gen=%i pciid=0x%04x rev=0x%02x",
-			 intel_platform_name(info->platform),
-			 info->gen,
-			 dev_priv->drm.pdev->device,
-			 dev_priv->drm.pdev->revision);
-#define PRINT_FLAG(name) \
-	DRM_DEBUG_DRIVER("i915 device info: " #name ": %s", yesno(info->name))
+#define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->name));
 	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
 }
 
+static void sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p)
+{
+	drm_printf(p, "slice mask: %04x\n", sseu->slice_mask);
+	drm_printf(p, "slice total: %u\n", hweight8(sseu->slice_mask));
+	drm_printf(p, "subslice total: %u\n", sseu_subslice_total(sseu));
+	drm_printf(p, "subslice mask %04x\n", sseu->subslice_mask);
+	drm_printf(p, "subslice per slice: %u\n",
+		   hweight8(sseu->subslice_mask));
+	drm_printf(p, "EU total: %u\n", sseu->eu_total);
+	drm_printf(p, "EU per subslice: %u\n", sseu->eu_per_subslice);
+	drm_printf(p, "has slice power gating: %s\n",
+		   yesno(sseu->has_slice_pg));
+	drm_printf(p, "has subslice power gating: %s\n",
+		   yesno(sseu->has_subslice_pg));
+	drm_printf(p, "has EU power gating: %s\n", yesno(sseu->has_eu_pg));
+}
+
+void intel_device_info_dump_runtime(const struct intel_device_info *info,
+				    struct drm_printer *p)
+{
+	sseu_dump(&info->sseu, p);
+
+	drm_printf(p, "CS timestamp frequency: %u kHz\n",
+		   info->cs_timestamp_frequency_khz);
+}
+
+void intel_device_info_dump(const struct intel_device_info *info,
+			    struct drm_printer *p)
+{
+	struct drm_i915_private *dev_priv =
+		container_of(info, struct drm_i915_private, info);
+
+	drm_printf(p, "pciid=0x%04x rev=0x%02x platform=%s gen=%i\n",
+		   INTEL_DEVID(dev_priv),
+		   INTEL_REVID(dev_priv),
+		   intel_platform_name(info->platform),
+		   info->gen);
+
+	intel_device_info_dump_flags(info, p);
+}
+
 static void gen10_sseu_info_init(struct drm_i915_private *dev_priv)
 {
 	struct sseu_dev_info *sseu = &mkwrite_device_info(dev_priv)->sseu;
@@ -235,16 +272,6 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 #define IS_SS_DISABLED(ss)	(!(sseu->subslice_mask & BIT(ss)))
 		info->has_pooled_eu = hweight8(sseu->subslice_mask) == 3;
 
-		/*
-		 * There is a HW issue in 2x6 fused down parts that requires
-		 * Pooled EU to be enabled as a WA. The pool configuration
-		 * changes depending upon which subslice is fused down. This
-		 * doesn't affect if the device has all 3 subslices enabled.
-		 */
-		/* WaEnablePooledEuFor2x6:bxt */
-		info->has_pooled_eu |= (hweight8(sseu->subslice_mask) == 2 &&
-					IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST));
-
 		sseu->min_eu_in_pool = 0;
 		if (info->has_pooled_eu) {
 			if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0))
@@ -329,7 +356,111 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
 	sseu->has_eu_pg = 0;
 }
 
-/*
+static u32 read_reference_ts_freq(struct drm_i915_private *dev_priv)
+{
+	u32 ts_override = I915_READ(GEN9_TIMESTAMP_OVERRIDE);
+	u32 base_freq, frac_freq;
+
+	base_freq = ((ts_override & GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK) >>
+		     GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_SHIFT) + 1;
+	base_freq *= 1000;
+
+	frac_freq = ((ts_override &
+		      GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK) >>
+		     GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT);
+	frac_freq = 1000 / (frac_freq + 1);
+
+	return base_freq + frac_freq;
+}
+
+static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv)
+{
+	u32 f12_5_mhz = 12500;
+	u32 f19_2_mhz = 19200;
+	u32 f24_mhz = 24000;
+
+	if (INTEL_GEN(dev_priv) <= 4) {
+		/* PRMs say:
+		 *
+		 *     "The value in this register increments once every 16
+		 *      hclks." (through the “Clocking Configuration”
+		 *      (“CLKCFG”) MCHBAR register)
+		 */
+		return dev_priv->rawclk_freq / 16;
+	} else if (INTEL_GEN(dev_priv) <= 8) {
+		/* PRMs say:
+		 *
+		 *     "The PCU TSC counts 10ns increments; this timestamp
+		 *      reflects bits 38:3 of the TSC (i.e. 80ns granularity,
+		 *      rolling over every 1.5 hours).
+		 */
+		return f12_5_mhz;
+	} else if (INTEL_GEN(dev_priv) <= 9) {
+		u32 ctc_reg = I915_READ(CTC_MODE);
+		u32 freq = 0;
+
+		if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
+			freq = read_reference_ts_freq(dev_priv);
+		} else {
+			freq = IS_GEN9_LP(dev_priv) ? f19_2_mhz : f24_mhz;
+
+			/* Now figure out how the command stream's timestamp
+			 * register increments from this frequency (it might
+			 * increment only every few clock cycle).
+			 */
+			freq >>= 3 - ((ctc_reg & CTC_SHIFT_PARAMETER_MASK) >>
+				      CTC_SHIFT_PARAMETER_SHIFT);
+		}
+
+		return freq;
+	} else if (INTEL_GEN(dev_priv) <= 10) {
+		u32 ctc_reg = I915_READ(CTC_MODE);
+		u32 freq = 0;
+		u32 rpm_config_reg = 0;
+
+		/* First figure out the reference frequency. There are 2 ways
+		 * we can compute the frequency, either through the
+		 * TIMESTAMP_OVERRIDE register or through RPM_CONFIG. CTC_MODE
+		 * tells us which one we should use.
+		 */
+		if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
+			freq = read_reference_ts_freq(dev_priv);
+		} else {
+			u32 crystal_clock;
+
+			rpm_config_reg = I915_READ(RPM_CONFIG0);
+			crystal_clock = (rpm_config_reg &
+					 GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >>
+				GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT;
+			switch (crystal_clock) {
+			case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ:
+				freq = f19_2_mhz;
+				break;
+			case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ:
+				freq = f24_mhz;
+				break;
+			}
+
+			/* Now figure out how the command stream's timestamp
+			 * register increments from this frequency (it might
+			 * increment only every few clock cycle).
+			 */
+			freq >>= 3 - ((rpm_config_reg &
+				       GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >>
+				      GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT);
+		}
+
+		return freq;
+	}
+
+	MISSING_CASE("Unknown gen, unable to read command streamer timestamp frequency\n");
+	return 0;
+}
+
+/**
+ * intel_device_info_runtime_init - initialize runtime info
+ * @info: intel device info struct
+ *
  * Determine various intel_device_info fields at runtime.
  *
  * Use it when either:
@@ -342,12 +473,16 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
  *   - after the PCH has been detected,
  *   - before the first usage of the fields it can tweak.
  */
-void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
+void intel_device_info_runtime_init(struct intel_device_info *info)
 {
-	struct intel_device_info *info = mkwrite_device_info(dev_priv);
+	struct drm_i915_private *dev_priv =
+		container_of(info, struct drm_i915_private, info);
 	enum pipe pipe;
 
-	if (INTEL_GEN(dev_priv) >= 9) {
+	if (INTEL_GEN(dev_priv) >= 10) {
+		for_each_pipe(dev_priv, pipe)
+			info->num_scalers[pipe] = 2;
+	} else if (INTEL_GEN(dev_priv) == 9) {
 		info->num_scalers[PIPE_A] = 2;
 		info->num_scalers[PIPE_B] = 2;
 		info->num_scalers[PIPE_C] = 1;
@@ -447,19 +582,6 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 	else if (INTEL_GEN(dev_priv) >= 10)
 		gen10_sseu_info_init(dev_priv);
 
-	DRM_DEBUG_DRIVER("slice mask: %04x\n", info->sseu.slice_mask);
-	DRM_DEBUG_DRIVER("slice total: %u\n", hweight8(info->sseu.slice_mask));
-	DRM_DEBUG_DRIVER("subslice total: %u\n",
-			 sseu_subslice_total(&info->sseu));
-	DRM_DEBUG_DRIVER("subslice mask %04x\n", info->sseu.subslice_mask);
-	DRM_DEBUG_DRIVER("subslice per slice: %u\n",
-			 hweight8(info->sseu.subslice_mask));
-	DRM_DEBUG_DRIVER("EU total: %u\n", info->sseu.eu_total);
-	DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->sseu.eu_per_subslice);
-	DRM_DEBUG_DRIVER("has slice power gating: %s\n",
-			 info->sseu.has_slice_pg ? "y" : "n");
-	DRM_DEBUG_DRIVER("has subslice power gating: %s\n",
-			 info->sseu.has_subslice_pg ? "y" : "n");
-	DRM_DEBUG_DRIVER("has EU power gating: %s\n",
-			 info->sseu.has_eu_pg ? "y" : "n");
+	/* Initialize command stream timestamp frequency */
+	info->cs_timestamp_frequency_khz = read_timestamp_frequency(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
new file mode 100644
index 000000000000..49cb27bd04c1
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -0,0 +1,183 @@
+/*
+ * Copyright © 2014-2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _INTEL_DEVICE_INFO_H_
+#define _INTEL_DEVICE_INFO_H_
+
+#include "intel_display.h"
+
+struct drm_printer;
+struct drm_i915_private;
+
+/* Keep in gen based order, and chronological order within a gen */
+enum intel_platform {
+	INTEL_PLATFORM_UNINITIALIZED = 0,
+	/* gen2 */
+	INTEL_I830,
+	INTEL_I845G,
+	INTEL_I85X,
+	INTEL_I865G,
+	/* gen3 */
+	INTEL_I915G,
+	INTEL_I915GM,
+	INTEL_I945G,
+	INTEL_I945GM,
+	INTEL_G33,
+	INTEL_PINEVIEW,
+	/* gen4 */
+	INTEL_I965G,
+	INTEL_I965GM,
+	INTEL_G45,
+	INTEL_GM45,
+	/* gen5 */
+	INTEL_IRONLAKE,
+	/* gen6 */
+	INTEL_SANDYBRIDGE,
+	/* gen7 */
+	INTEL_IVYBRIDGE,
+	INTEL_VALLEYVIEW,
+	INTEL_HASWELL,
+	/* gen8 */
+	INTEL_BROADWELL,
+	INTEL_CHERRYVIEW,
+	/* gen9 */
+	INTEL_SKYLAKE,
+	INTEL_BROXTON,
+	INTEL_KABYLAKE,
+	INTEL_GEMINILAKE,
+	INTEL_COFFEELAKE,
+	/* gen10 */
+	INTEL_CANNONLAKE,
+	INTEL_MAX_PLATFORMS
+};
+
+#define DEV_INFO_FOR_EACH_FLAG(func) \
+	func(is_mobile); \
+	func(is_lp); \
+	func(is_alpha_support); \
+	/* Keep has_* in alphabetical order */ \
+	func(has_64bit_reloc); \
+	func(has_aliasing_ppgtt); \
+	func(has_csr); \
+	func(has_ddi); \
+	func(has_dp_mst); \
+	func(has_reset_engine); \
+	func(has_fbc); \
+	func(has_fpga_dbg); \
+	func(has_full_ppgtt); \
+	func(has_full_48bit_ppgtt); \
+	func(has_gmch_display); \
+	func(has_guc); \
+	func(has_guc_ct); \
+	func(has_hotplug); \
+	func(has_l3_dpf); \
+	func(has_llc); \
+	func(has_logical_ring_contexts); \
+	func(has_logical_ring_preemption); \
+	func(has_overlay); \
+	func(has_pooled_eu); \
+	func(has_psr); \
+	func(has_rc6); \
+	func(has_rc6p); \
+	func(has_resource_streamer); \
+	func(has_runtime_pm); \
+	func(has_snoop); \
+	func(unfenced_needs_alignment); \
+	func(cursor_needs_physical); \
+	func(hws_needs_physical); \
+	func(overlay_needs_physical); \
+	func(supports_tv); \
+	func(has_ipc);
+
+struct sseu_dev_info {
+	u8 slice_mask;
+	u8 subslice_mask;
+	u8 eu_total;
+	u8 eu_per_subslice;
+	u8 min_eu_in_pool;
+	/* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */
+	u8 subslice_7eu[3];
+	u8 has_slice_pg:1;
+	u8 has_subslice_pg:1;
+	u8 has_eu_pg:1;
+};
+
+struct intel_device_info {
+	u16 device_id;
+	u16 gen_mask;
+
+	u8 gen;
+	u8 gt; /* GT number, 0 if undefined */
+	u8 num_rings;
+	u8 ring_mask; /* Rings supported by the HW */
+
+	enum intel_platform platform;
+	u32 platform_mask;
+
+	u32 display_mmio_offset;
+
+	u8 num_pipes;
+	u8 num_sprites[I915_MAX_PIPES];
+	u8 num_scalers[I915_MAX_PIPES];
+
+	unsigned int page_sizes; /* page sizes supported by the HW */
+
+#define DEFINE_FLAG(name) u8 name:1
+	DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
+#undef DEFINE_FLAG
+	u16 ddb_size; /* in blocks */
+
+	/* Register offsets for the various display pipes and transcoders */
+	int pipe_offsets[I915_MAX_TRANSCODERS];
+	int trans_offsets[I915_MAX_TRANSCODERS];
+	int palette_offsets[I915_MAX_PIPES];
+	int cursor_offsets[I915_MAX_PIPES];
+
+	/* Slice/subslice/EU info */
+	struct sseu_dev_info sseu;
+
+	u32 cs_timestamp_frequency_khz;
+
+	struct color_luts {
+		u16 degamma_lut_size;
+		u16 gamma_lut_size;
+	} color;
+};
+
+static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu)
+{
+	return hweight8(sseu->slice_mask) * hweight8(sseu->subslice_mask);
+}
+
+const char *intel_platform_name(enum intel_platform platform);
+
+void intel_device_info_runtime_init(struct intel_device_info *info);
+void intel_device_info_dump(const struct intel_device_info *info,
+			    struct drm_printer *p);
+void intel_device_info_dump_flags(const struct intel_device_info *info,
+				  struct drm_printer *p);
+void intel_device_info_dump_runtime(const struct intel_device_info *info,
+				    struct drm_printer *p);
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 50f8443641b8..f288bcc7be22 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -219,10 +219,8 @@ intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 {
 	if (HAS_DDI(dev_priv))
 		return pipe_config->port_clock; /* SPLL */
-	else if (IS_GEN5(dev_priv))
-		return ((I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2) * 10000;
 	else
-		return 270000;
+		return dev_priv->fdi_pll_freq;
 }
 
 static const struct intel_limit intel_limits_i8xx_dac = {
@@ -491,7 +489,7 @@ static const struct intel_limit intel_limits_bxt = {
 };
 
 static bool
-needs_modeset(struct drm_crtc_state *state)
+needs_modeset(const struct drm_crtc_state *state)
 {
 	return drm_atomic_crtc_needs_modeset(state);
 }
@@ -1040,28 +1038,14 @@ static void intel_wait_for_pipe_scanline_moving(struct intel_crtc *crtc)
 	wait_for_pipe_scanline_moving(crtc, true);
 }
 
-/*
- * intel_wait_for_pipe_off - wait for pipe to turn off
- * @crtc: crtc whose pipe to wait for
- *
- * After disabling a pipe, we can't wait for vblank in the usual way,
- * spinning on the vblank interrupt status bit, since we won't actually
- * see an interrupt when the pipe is disabled.
- *
- * On Gen4 and above:
- *   wait for the pipe register state bit to turn off
- *
- * Otherwise:
- *   wait for the display line value to settle (it usually
- *   ends up stopping at the start of the next frame).
- *
- */
-static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
+static void
+intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
 {
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 
 	if (INTEL_GEN(dev_priv) >= 4) {
+		enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 		i915_reg_t reg = PIPECONF(cpu_transcoder);
 
 		/* Wait for the Pipe State to go off */
@@ -1653,7 +1637,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 	u32 port_mask;
 	i915_reg_t dpll_reg;
 
-	switch (dport->port) {
+	switch (dport->base.port) {
 	case PORT_B:
 		port_mask = DPLL_PORTB_READY_MASK;
 		dpll_reg = DPLL(0);
@@ -1675,7 +1659,8 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 				    dpll_reg, port_mask, expected_mask,
 				    1000))
 		WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
-		     port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask);
+		     port_name(dport->base.port),
+		     I915_READ(dpll_reg) & port_mask, expected_mask);
 }
 
 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
@@ -1823,27 +1808,18 @@ enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-	WARN_ON(!crtc->config->has_pch_encoder);
-
 	if (HAS_PCH_LPT(dev_priv))
 		return PIPE_A;
 	else
 		return crtc->pipe;
 }
 
-/**
- * intel_enable_pipe - enable a pipe, asserting requirements
- * @crtc: crtc responsible for the pipe
- *
- * Enable @crtc's pipe, making sure that various hardware specific requirements
- * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
- */
-static void intel_enable_pipe(struct intel_crtc *crtc)
+static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
 	enum pipe pipe = crtc->pipe;
-	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 	i915_reg_t reg;
 	u32 val;
 
@@ -1857,12 +1833,12 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	 * need the check.
 	 */
 	if (HAS_GMCH_DISPLAY(dev_priv)) {
-		if (intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DSI))
+		if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 			assert_dsi_pll_enabled(dev_priv);
 		else
 			assert_pll_enabled(dev_priv, pipe);
 	} else {
-		if (crtc->config->has_pch_encoder) {
+		if (new_crtc_state->has_pch_encoder) {
 			/* if driving the PCH, we need FDI enabled */
 			assert_fdi_rx_pll_enabled(dev_priv,
 						  intel_crtc_pch_transcoder(crtc));
@@ -1890,24 +1866,15 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 	 * when it's derived from the timestamps. So let's wait for the
 	 * pipe to start properly before we call drm_crtc_vblank_on()
 	 */
-	if (dev->max_vblank_count == 0)
+	if (dev_priv->drm.max_vblank_count == 0)
 		intel_wait_for_pipe_scanline_moving(crtc);
 }
 
-/**
- * intel_disable_pipe - disable a pipe, asserting requirements
- * @crtc: crtc whose pipes is to be disabled
- *
- * Disable the pipe of @crtc, making sure that various hardware
- * specific requirements are met, if applicable, e.g. plane
- * disabled, panel fitter off, etc.
- *
- * Will wait until the pipe has shut down before returning.
- */
-static void intel_disable_pipe(struct intel_crtc *crtc)
+static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
 {
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
+	enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 	enum pipe pipe = crtc->pipe;
 	i915_reg_t reg;
 	u32 val;
@@ -1929,7 +1896,7 @@ static void intel_disable_pipe(struct intel_crtc *crtc)
 	 * Double wide has implications for planes
 	 * so best keep it disabled when not needed.
 	 */
-	if (crtc->config->double_wide)
+	if (old_crtc_state->double_wide)
 		val &= ~PIPECONF_DOUBLE_WIDE;
 
 	/* Don't disable pipe or pipe PLLs if needed */
@@ -1938,7 +1905,7 @@ static void intel_disable_pipe(struct intel_crtc *crtc)
 
 	I915_WRITE(reg, val);
 	if ((val & PIPECONF_ENABLE) == 0)
-		intel_wait_for_pipe_off(crtc);
+		intel_wait_for_pipe_off(old_crtc_state);
 }
 
 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
@@ -2672,7 +2639,6 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct drm_i915_gem_object *obj = NULL;
 	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
 	struct drm_framebuffer *fb = &plane_config->fb->base;
@@ -2688,7 +2654,7 @@ intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
 	/* If the FB is too big, just don't use it since fbdev is not very
 	 * important and we should probably use that space with FBC or other
 	 * features. */
-	if (size_aligned * 2 > ggtt->stolen_usable_size)
+	if (size_aligned * 2 > dev_priv->stolen_usable_size)
 		return false;
 
 	mutex_lock(&dev->struct_mutex);
@@ -3107,6 +3073,12 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state)
 	unsigned int rotation = plane_state->base.rotation;
 	int ret;
 
+	if (rotation & DRM_MODE_REFLECT_X &&
+	    fb->modifier == DRM_FORMAT_MOD_LINEAR) {
+		DRM_DEBUG_KMS("horizontal flip is not supported with linear surface formats\n");
+		return -EINVAL;
+	}
+
 	if (!plane_state->base.visible)
 		return 0;
 
@@ -3241,16 +3213,16 @@ int i9xx_check_plane_surface(struct intel_plane_state *plane_state)
 	return 0;
 }
 
-static void i9xx_update_primary_plane(struct intel_plane *primary,
-				      const struct intel_crtc_state *crtc_state,
-				      const struct intel_plane_state *plane_state)
+static void i9xx_update_plane(struct intel_plane *plane,
+			      const struct intel_crtc_state *crtc_state,
+			      const struct intel_plane_state *plane_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(primary->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	const struct drm_framebuffer *fb = plane_state->base.fb;
-	enum plane plane = primary->plane;
+	enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
 	u32 linear_offset;
 	u32 dspcntr = plane_state->ctl;
-	i915_reg_t reg = DSPCNTR(plane);
+	i915_reg_t reg = DSPCNTR(i9xx_plane);
 	int x = plane_state->main.x;
 	int y = plane_state->main.y;
 	unsigned long irqflags;
@@ -3269,34 +3241,34 @@ static void i9xx_update_primary_plane(struct intel_plane *primary,
 		/* pipesrc and dspsize control the size that is scaled from,
 		 * which should always be the user's requested size.
 		 */
-		I915_WRITE_FW(DSPSIZE(plane),
+		I915_WRITE_FW(DSPSIZE(i9xx_plane),
 			      ((crtc_state->pipe_src_h - 1) << 16) |
 			      (crtc_state->pipe_src_w - 1));
-		I915_WRITE_FW(DSPPOS(plane), 0);
-	} else if (IS_CHERRYVIEW(dev_priv) && plane == PLANE_B) {
-		I915_WRITE_FW(PRIMSIZE(plane),
+		I915_WRITE_FW(DSPPOS(i9xx_plane), 0);
+	} else if (IS_CHERRYVIEW(dev_priv) && i9xx_plane == PLANE_B) {
+		I915_WRITE_FW(PRIMSIZE(i9xx_plane),
 			      ((crtc_state->pipe_src_h - 1) << 16) |
 			      (crtc_state->pipe_src_w - 1));
-		I915_WRITE_FW(PRIMPOS(plane), 0);
-		I915_WRITE_FW(PRIMCNSTALPHA(plane), 0);
+		I915_WRITE_FW(PRIMPOS(i9xx_plane), 0);
+		I915_WRITE_FW(PRIMCNSTALPHA(i9xx_plane), 0);
 	}
 
 	I915_WRITE_FW(reg, dspcntr);
 
-	I915_WRITE_FW(DSPSTRIDE(plane), fb->pitches[0]);
+	I915_WRITE_FW(DSPSTRIDE(i9xx_plane), fb->pitches[0]);
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
-		I915_WRITE_FW(DSPSURF(plane),
+		I915_WRITE_FW(DSPSURF(i9xx_plane),
 			      intel_plane_ggtt_offset(plane_state) +
 			      dspaddr_offset);
-		I915_WRITE_FW(DSPOFFSET(plane), (y << 16) | x);
+		I915_WRITE_FW(DSPOFFSET(i9xx_plane), (y << 16) | x);
 	} else if (INTEL_GEN(dev_priv) >= 4) {
-		I915_WRITE_FW(DSPSURF(plane),
+		I915_WRITE_FW(DSPSURF(i9xx_plane),
 			      intel_plane_ggtt_offset(plane_state) +
 			      dspaddr_offset);
-		I915_WRITE_FW(DSPTILEOFF(plane), (y << 16) | x);
-		I915_WRITE_FW(DSPLINOFF(plane), linear_offset);
+		I915_WRITE_FW(DSPTILEOFF(i9xx_plane), (y << 16) | x);
+		I915_WRITE_FW(DSPLINOFF(i9xx_plane), linear_offset);
 	} else {
-		I915_WRITE_FW(DSPADDR(plane),
+		I915_WRITE_FW(DSPADDR(i9xx_plane),
 			      intel_plane_ggtt_offset(plane_state) +
 			      dspaddr_offset);
 	}
@@ -3305,32 +3277,31 @@ static void i9xx_update_primary_plane(struct intel_plane *primary,
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-static void i9xx_disable_primary_plane(struct intel_plane *primary,
-				       struct intel_crtc *crtc)
+static void i9xx_disable_plane(struct intel_plane *plane,
+			       struct intel_crtc *crtc)
 {
-	struct drm_i915_private *dev_priv = to_i915(primary->base.dev);
-	enum plane plane = primary->plane;
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+	enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
 	unsigned long irqflags;
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-	I915_WRITE_FW(DSPCNTR(plane), 0);
-	if (INTEL_INFO(dev_priv)->gen >= 4)
-		I915_WRITE_FW(DSPSURF(plane), 0);
+	I915_WRITE_FW(DSPCNTR(i9xx_plane), 0);
+	if (INTEL_GEN(dev_priv) >= 4)
+		I915_WRITE_FW(DSPSURF(i9xx_plane), 0);
 	else
-		I915_WRITE_FW(DSPADDR(plane), 0);
-	POSTING_READ_FW(DSPCNTR(plane));
+		I915_WRITE_FW(DSPADDR(i9xx_plane), 0);
+	POSTING_READ_FW(DSPCNTR(i9xx_plane));
 
 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-static bool i9xx_plane_get_hw_state(struct intel_plane *primary)
+static bool i9xx_plane_get_hw_state(struct intel_plane *plane)
 {
-
-	struct drm_i915_private *dev_priv = to_i915(primary->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	enum intel_display_power_domain power_domain;
-	enum plane plane = primary->plane;
-	enum pipe pipe = primary->pipe;
+	enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
+	enum pipe pipe = plane->pipe;
 	bool ret;
 
 	/*
@@ -3342,7 +3313,7 @@ static bool i9xx_plane_get_hw_state(struct intel_plane *primary)
 	if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
 		return false;
 
-	ret = I915_READ(DSPCNTR(plane)) & DISPLAY_PLANE_ENABLE;
+	ret = I915_READ(DSPCNTR(i9xx_plane)) & DISPLAY_PLANE_ENABLE;
 
 	intel_display_power_put(dev_priv, power_domain);
 
@@ -3415,20 +3386,11 @@ static u32 skl_plane_ctl_format(uint32_t pixel_format)
 	case DRM_FORMAT_RGB565:
 		return PLANE_CTL_FORMAT_RGB_565;
 	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR8888:
 		return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
 	case DRM_FORMAT_XRGB8888:
-		return PLANE_CTL_FORMAT_XRGB_8888;
-	/*
-	 * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
-	 * to be already pre-multiplied. We need to add a knob (or a different
-	 * DRM_FORMAT) for user-space to configure that.
-	 */
-	case DRM_FORMAT_ABGR8888:
-		return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX |
-			PLANE_CTL_ALPHA_SW_PREMULTIPLY;
 	case DRM_FORMAT_ARGB8888:
-		return PLANE_CTL_FORMAT_XRGB_8888 |
-			PLANE_CTL_ALPHA_SW_PREMULTIPLY;
+		return PLANE_CTL_FORMAT_XRGB_8888;
 	case DRM_FORMAT_XRGB2101010:
 		return PLANE_CTL_FORMAT_XRGB_2101010;
 	case DRM_FORMAT_XBGR2101010:
@@ -3448,6 +3410,33 @@ static u32 skl_plane_ctl_format(uint32_t pixel_format)
 	return 0;
 }
 
+/*
+ * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
+ * to be already pre-multiplied. We need to add a knob (or a different
+ * DRM_FORMAT) for user-space to configure that.
+ */
+static u32 skl_plane_ctl_alpha(uint32_t pixel_format)
+{
+	switch (pixel_format) {
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_ARGB8888:
+		return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
+	default:
+		return PLANE_CTL_ALPHA_DISABLE;
+	}
+}
+
+static u32 glk_plane_color_ctl_alpha(uint32_t pixel_format)
+{
+	switch (pixel_format) {
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_ARGB8888:
+		return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
+	default:
+		return PLANE_COLOR_ALPHA_DISABLE;
+	}
+}
+
 static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
 {
 	switch (fb_modifier) {
@@ -3470,9 +3459,9 @@ static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
 	return 0;
 }
 
-static u32 skl_plane_ctl_rotation(unsigned int rotation)
+static u32 skl_plane_ctl_rotate(unsigned int rotate)
 {
-	switch (rotation) {
+	switch (rotate) {
 	case DRM_MODE_ROTATE_0:
 		break;
 	/*
@@ -3486,7 +3475,22 @@ static u32 skl_plane_ctl_rotation(unsigned int rotation)
 	case DRM_MODE_ROTATE_270:
 		return PLANE_CTL_ROTATE_90;
 	default:
-		MISSING_CASE(rotation);
+		MISSING_CASE(rotate);
+	}
+
+	return 0;
+}
+
+static u32 cnl_plane_ctl_flip(unsigned int reflect)
+{
+	switch (reflect) {
+	case 0:
+		break;
+	case DRM_MODE_REFLECT_X:
+		return PLANE_CTL_FLIP_HORIZONTAL;
+	case DRM_MODE_REFLECT_Y:
+	default:
+		MISSING_CASE(reflect);
 	}
 
 	return 0;
@@ -3504,7 +3508,8 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 
 	plane_ctl = PLANE_CTL_ENABLE;
 
-	if (!IS_GEMINILAKE(dev_priv) && !IS_CANNONLAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
+		plane_ctl |= skl_plane_ctl_alpha(fb->format->format);
 		plane_ctl |=
 			PLANE_CTL_PIPE_GAMMA_ENABLE |
 			PLANE_CTL_PIPE_CSC_ENABLE |
@@ -3513,7 +3518,11 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 
 	plane_ctl |= skl_plane_ctl_format(fb->format->format);
 	plane_ctl |= skl_plane_ctl_tiling(fb->modifier);
-	plane_ctl |= skl_plane_ctl_rotation(rotation);
+	plane_ctl |= skl_plane_ctl_rotate(rotation & DRM_MODE_ROTATE_MASK);
+
+	if (INTEL_GEN(dev_priv) >= 10)
+		plane_ctl |= cnl_plane_ctl_flip(rotation &
+						DRM_MODE_REFLECT_MASK);
 
 	if (key->flags & I915_SET_COLORKEY_DESTINATION)
 		plane_ctl |= PLANE_CTL_KEY_ENABLE_DESTINATION;
@@ -3523,6 +3532,20 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 	return plane_ctl;
 }
 
+u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
+			const struct intel_plane_state *plane_state)
+{
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	u32 plane_color_ctl = 0;
+
+	plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
+	plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
+	plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
+	plane_color_ctl |= glk_plane_color_ctl_alpha(fb->format->format);
+
+	return plane_color_ctl;
+}
+
 static int
 __intel_display_resume(struct drm_device *dev,
 		       struct drm_atomic_state *state,
@@ -4465,7 +4488,7 @@ intel_trans_dp_port_sel(struct intel_crtc *crtc)
 	for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
 		if (encoder->type == INTEL_OUTPUT_DP ||
 		    encoder->type == INTEL_OUTPUT_EDP)
-			return enc_to_dig_port(&encoder->base)->port;
+			return encoder->port;
 	}
 
 	return -1;
@@ -4816,12 +4839,13 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc)
 	}
 }
 
-void hsw_enable_ips(struct intel_crtc *crtc)
+void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
 {
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (!crtc->config->ips_enabled)
+	if (!crtc_state->ips_enabled)
 		return;
 
 	/*
@@ -4829,8 +4853,7 @@ void hsw_enable_ips(struct intel_crtc *crtc)
 	 * This function is called from post_plane_update, which is run after
 	 * a vblank wait.
 	 */
-
-	assert_plane_enabled(to_intel_plane(crtc->base.primary));
+	WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)));
 
 	if (IS_BROADWELL(dev_priv)) {
 		mutex_lock(&dev_priv->pcu_lock);
@@ -4856,16 +4879,15 @@ void hsw_enable_ips(struct intel_crtc *crtc)
 	}
 }
 
-void hsw_disable_ips(struct intel_crtc *crtc)
+void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
 {
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	if (!crtc->config->ips_enabled)
+	if (!crtc_state->ips_enabled)
 		return;
 
-	assert_plane_enabled(to_intel_plane(crtc->base.primary));
-
 	if (IS_BROADWELL(dev_priv)) {
 		mutex_lock(&dev_priv->pcu_lock);
 		WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
@@ -4910,21 +4932,14 @@ static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
  * completely hide the primary plane.
  */
 static void
-intel_post_enable_primary(struct drm_crtc *crtc)
+intel_post_enable_primary(struct drm_crtc *crtc,
+			  const struct intel_crtc_state *new_crtc_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
 
-	/*
-	 * FIXME IPS should be fine as long as one plane is
-	 * enabled, but in practice it seems to have problems
-	 * when going from primary only to sprite only and vice
-	 * versa.
-	 */
-	hsw_enable_ips(intel_crtc);
-
 	/*
 	 * Gen2 reports pipe underruns whenever all planes are disabled.
 	 * So don't enable underrun reporting before at least some planes
@@ -4940,9 +4955,9 @@ intel_post_enable_primary(struct drm_crtc *crtc)
 	intel_check_pch_fifo_underruns(dev_priv);
 }
 
-/* FIXME move all this to pre_plane_update() with proper state tracking */
+/* FIXME get rid of this and use pre_plane_update */
 static void
-intel_pre_disable_primary(struct drm_crtc *crtc)
+intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -4951,32 +4966,12 @@ intel_pre_disable_primary(struct drm_crtc *crtc)
 
 	/*
 	 * Gen2 reports pipe underruns whenever all planes are disabled.
-	 * So diasble underrun reporting before all the planes get disabled.
-	 * FIXME: Need to fix the logic to work when we turn off all planes
-	 * but leave the pipe running.
+	 * So disable underrun reporting before all the planes get disabled.
 	 */
 	if (IS_GEN2(dev_priv))
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 
-	/*
-	 * FIXME IPS should be fine as long as one plane is
-	 * enabled, but in practice it seems to have problems
-	 * when going from primary only to sprite only and vice
-	 * versa.
-	 */
-	hsw_disable_ips(intel_crtc);
-}
-
-/* FIXME get rid of this and use pre_plane_update */
-static void
-intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int pipe = intel_crtc->pipe;
-
-	intel_pre_disable_primary(crtc);
+	hsw_disable_ips(to_intel_crtc_state(crtc->state));
 
 	/*
 	 * Vblank time updates from the shadow to live plane control register
@@ -4992,6 +4987,38 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
 		intel_wait_for_vblank(dev_priv, pipe);
 }
 
+static bool hsw_pre_update_disable_ips(const struct intel_crtc_state *old_crtc_state,
+				       const struct intel_crtc_state *new_crtc_state)
+{
+	if (!old_crtc_state->ips_enabled)
+		return false;
+
+	if (needs_modeset(&new_crtc_state->base))
+		return true;
+
+	return !new_crtc_state->ips_enabled;
+}
+
+static bool hsw_post_update_enable_ips(const struct intel_crtc_state *old_crtc_state,
+				       const struct intel_crtc_state *new_crtc_state)
+{
+	if (!new_crtc_state->ips_enabled)
+		return false;
+
+	if (needs_modeset(&new_crtc_state->base))
+		return true;
+
+	/*
+	 * We can't read out IPS on broadwell, assume the worst and
+	 * forcibly enable IPS on the first fastset.
+	 */
+	if (new_crtc_state->update_pipe &&
+	    old_crtc_state->base.adjusted_mode.private_flags & I915_MODE_FLAG_INHERITED)
+		return true;
+
+	return !old_crtc_state->ips_enabled;
+}
+
 static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
@@ -5008,6 +5035,9 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 	if (pipe_config->update_wm_post && pipe_config->base.active)
 		intel_update_watermarks(crtc);
 
+	if (hsw_post_update_enable_ips(old_crtc_state, pipe_config))
+		hsw_enable_ips(pipe_config);
+
 	if (old_pri_state) {
 		struct intel_plane_state *primary_state =
 			intel_atomic_get_new_plane_state(to_intel_atomic_state(old_state),
@@ -5020,7 +5050,7 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 		if (primary_state->base.visible &&
 		    (needs_modeset(&pipe_config->base) ||
 		     !old_primary_state->base.visible))
-			intel_post_enable_primary(&crtc->base);
+			intel_post_enable_primary(&crtc->base, pipe_config);
 	}
 }
 
@@ -5038,6 +5068,9 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
 	struct intel_atomic_state *old_intel_state =
 		to_intel_atomic_state(old_state);
 
+	if (hsw_pre_update_disable_ips(old_crtc_state, pipe_config))
+		hsw_disable_ips(old_crtc_state);
+
 	if (old_pri_state) {
 		struct intel_plane_state *primary_state =
 			intel_atomic_get_new_plane_state(old_intel_state,
@@ -5046,10 +5079,13 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
 			to_intel_plane_state(old_pri_state);
 
 		intel_fbc_pre_update(crtc, pipe_config, primary_state);
-
-		if (old_primary_state->base.visible &&
+		/*
+		 * Gen2 reports pipe underruns whenever all planes are disabled.
+		 * So disable underrun reporting before all the planes get disabled.
+		 */
+		if (IS_GEN2(dev_priv) && old_primary_state->base.visible &&
 		    (modeset || !primary_state->base.visible))
-			intel_pre_disable_primary(&crtc->base);
+			intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
 	}
 
 	/*
@@ -5312,7 +5348,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	if (dev_priv->display.initial_watermarks != NULL)
 		dev_priv->display.initial_watermarks(old_intel_state, intel_crtc->config);
-	intel_enable_pipe(intel_crtc);
+	intel_enable_pipe(pipe_config);
 
 	if (intel_crtc->config->has_pch_encoder)
 		ironlake_pch_enable(pipe_config);
@@ -5431,7 +5467,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	/* XXX: Do the pipe assertions at the right place for BXT DSI. */
 	if (!transcoder_is_dsi(cpu_transcoder))
-		intel_enable_pipe(intel_crtc);
+		intel_enable_pipe(pipe_config);
 
 	if (intel_crtc->config->has_pch_encoder)
 		lpt_pch_enable(pipe_config);
@@ -5497,7 +5533,7 @@ static void ironlake_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	drm_crtc_vblank_off(crtc);
 	assert_vblank_disabled(crtc);
 
-	intel_disable_pipe(intel_crtc);
+	intel_disable_pipe(old_crtc_state);
 
 	ironlake_pfit_disable(intel_crtc, false);
 
@@ -5549,7 +5585,7 @@ static void haswell_crtc_disable(struct intel_crtc_state *old_crtc_state,
 
 	/* XXX: Do the pipe assertions at the right place for BXT DSI. */
 	if (!transcoder_is_dsi(cpu_transcoder))
-		intel_disable_pipe(intel_crtc);
+		intel_disable_pipe(old_crtc_state);
 
 	if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DP_MST))
 		intel_ddi_set_vc_payload_alloc(intel_crtc->config, false);
@@ -5625,8 +5661,8 @@ static u64 get_crtc_power_domains(struct drm_crtc *crtc,
 	if (!crtc_state->base.active)
 		return 0;
 
-	mask = BIT(POWER_DOMAIN_PIPE(pipe));
-	mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
+	mask = BIT_ULL(POWER_DOMAIN_PIPE(pipe));
+	mask |= BIT_ULL(POWER_DOMAIN_TRANSCODER(transcoder));
 	if (crtc_state->pch_pfit.enabled ||
 	    crtc_state->pch_pfit.force_thru)
 		mask |= BIT_ULL(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
@@ -5638,7 +5674,7 @@ static u64 get_crtc_power_domains(struct drm_crtc *crtc,
 	}
 
 	if (HAS_DDI(dev_priv) && crtc_state->has_audio)
-		mask |= BIT(POWER_DOMAIN_AUDIO);
+		mask |= BIT_ULL(POWER_DOMAIN_AUDIO);
 
 	if (crtc_state->shared_dpll)
 		mask |= BIT_ULL(POWER_DOMAIN_PLLS);
@@ -5727,7 +5763,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	dev_priv->display.initial_watermarks(old_intel_state,
 					     pipe_config);
-	intel_enable_pipe(intel_crtc);
+	intel_enable_pipe(pipe_config);
 
 	assert_vblank_disabled(crtc);
 	drm_crtc_vblank_on(crtc);
@@ -5786,7 +5822,7 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
 						     intel_crtc->config);
 	else
 		intel_update_watermarks(intel_crtc);
-	intel_enable_pipe(intel_crtc);
+	intel_enable_pipe(pipe_config);
 
 	assert_vblank_disabled(crtc);
 	drm_crtc_vblank_on(crtc);
@@ -5830,7 +5866,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state,
 	drm_crtc_vblank_off(crtc);
 	assert_vblank_disabled(crtc);
 
-	intel_disable_pipe(intel_crtc);
+	intel_disable_pipe(old_crtc_state);
 
 	i9xx_pfit_disable(intel_crtc);
 
@@ -5925,6 +5961,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
 
 	dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
 	dev_priv->min_cdclk[intel_crtc->pipe] = 0;
+	dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
 }
 
 /*
@@ -6179,18 +6216,20 @@ static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
 	return ret;
 }
 
-static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv,
-				     struct intel_crtc_state *pipe_config)
+bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
 {
-	if (pipe_config->ips_force_disable)
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+
+	/* IPS only exists on ULT machines and is tied to pipe A. */
+	if (!hsw_crtc_supports_ips(crtc))
 		return false;
 
-	if (pipe_config->pipe_bpp > 24)
+	if (!i915_modparams.enable_ips)
 		return false;
 
-	/* HSW can handle pixel rate up to cdclk? */
-	if (IS_HASWELL(dev_priv))
-		return true;
+	if (crtc_state->pipe_bpp > 24)
+		return false;
 
 	/*
 	 * We compare against max which means we must take
@@ -6199,19 +6238,36 @@ static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv,
 	 *
 	 * Should measure whether using a lower cdclk w/o IPS
 	 */
-	return pipe_config->pixel_rate <=
-		dev_priv->max_cdclk_freq * 95 / 100;
+	if (IS_BROADWELL(dev_priv) &&
+	    crtc_state->pixel_rate > dev_priv->max_cdclk_freq * 95 / 100)
+		return false;
+
+	return true;
 }
 
-static void hsw_compute_ips_config(struct intel_crtc *crtc,
-				   struct intel_crtc_state *pipe_config)
+static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv =
+		to_i915(crtc_state->base.crtc->dev);
+	struct intel_atomic_state *intel_state =
+		to_intel_atomic_state(crtc_state->base.state);
 
-	pipe_config->ips_enabled = i915_modparams.enable_ips &&
-		hsw_crtc_supports_ips(crtc) &&
-		pipe_config_supports_ips(dev_priv, pipe_config);
+	if (!hsw_crtc_state_ips_capable(crtc_state))
+		return false;
+
+	if (crtc_state->ips_force_disable)
+		return false;
+
+	/* IPS should be fine as long as at least one plane is enabled. */
+	if (!(crtc_state->active_planes & ~BIT(PLANE_CURSOR)))
+		return false;
+
+	/* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
+	if (IS_BROADWELL(dev_priv) &&
+	    crtc_state->pixel_rate > intel_state->cdclk.logical.cdclk * 95 / 100)
+		return false;
+
+	return true;
 }
 
 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
@@ -6329,9 +6385,6 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 
 	intel_crtc_compute_pixel_rate(pipe_config);
 
-	if (HAS_IPS(dev_priv))
-		hsw_compute_ips_config(crtc, pipe_config);
-
 	if (pipe_config->has_pch_encoder)
 		return ironlake_fdi_compute_config(crtc, pipe_config);
 
@@ -7388,15 +7441,16 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
+	enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
+	enum pipe pipe = crtc->pipe;
 	u32 val, base, offset;
-	int pipe = crtc->pipe, plane = crtc->plane;
 	int fourcc, pixel_format;
 	unsigned int aligned_height;
 	struct drm_framebuffer *fb;
 	struct intel_framebuffer *intel_fb;
 
-	val = I915_READ(DSPCNTR(plane));
-	if (!(val & DISPLAY_PLANE_ENABLE))
+	if (!plane->get_hw_state(plane))
 		return;
 
 	intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
@@ -7409,6 +7463,8 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 
 	fb->dev = dev;
 
+	val = I915_READ(DSPCNTR(i9xx_plane));
+
 	if (INTEL_GEN(dev_priv) >= 4) {
 		if (val & DISPPLANE_TILED) {
 			plane_config->tiling = I915_TILING_X;
@@ -7420,14 +7476,17 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 	fourcc = i9xx_format_to_fourcc(pixel_format);
 	fb->format = drm_format_info(fourcc);
 
-	if (INTEL_GEN(dev_priv) >= 4) {
+	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
+		offset = I915_READ(DSPOFFSET(i9xx_plane));
+		base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
+	} else if (INTEL_GEN(dev_priv) >= 4) {
 		if (plane_config->tiling)
-			offset = I915_READ(DSPTILEOFF(plane));
+			offset = I915_READ(DSPTILEOFF(i9xx_plane));
 		else
-			offset = I915_READ(DSPLINOFF(plane));
-		base = I915_READ(DSPSURF(plane)) & 0xfffff000;
+			offset = I915_READ(DSPLINOFF(i9xx_plane));
+		base = I915_READ(DSPSURF(i9xx_plane)) & 0xfffff000;
 	} else {
-		base = I915_READ(DSPADDR(plane));
+		base = I915_READ(DSPADDR(i9xx_plane));
 	}
 	plane_config->base = base;
 
@@ -7435,15 +7494,15 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 	fb->width = ((val >> 16) & 0xfff) + 1;
 	fb->height = ((val >> 0) & 0xfff) + 1;
 
-	val = I915_READ(DSPSTRIDE(pipe));
+	val = I915_READ(DSPSTRIDE(i9xx_plane));
 	fb->pitches[0] = val & 0xffffffc0;
 
 	aligned_height = intel_fb_align_height(fb, 0, fb->height);
 
 	plane_config->size = fb->pitches[0] * aligned_height;
 
-	DRM_DEBUG_KMS("pipe/plane %c/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
-		      pipe_name(pipe), plane, fb->width, fb->height,
+	DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
+		      crtc->base.name, plane->base.name, fb->width, fb->height,
 		      fb->format->cpp[0] * 8, base, fb->pitches[0],
 		      plane_config->size);
 
@@ -7619,7 +7678,7 @@ static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
 			break;
 		case INTEL_OUTPUT_EDP:
 			has_panel = true;
-			if (enc_to_dig_port(&encoder->base)->port == PORT_A)
+			if (encoder->port == PORT_A)
 				has_cpu_edp = true;
 			break;
 		default:
@@ -8412,13 +8471,18 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	u32 val, base, offset, stride_mult, tiling;
-	int pipe = crtc->pipe;
+	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
+	enum plane_id plane_id = plane->id;
+	enum pipe pipe = crtc->pipe;
+	u32 val, base, offset, stride_mult, tiling, alpha;
 	int fourcc, pixel_format;
 	unsigned int aligned_height;
 	struct drm_framebuffer *fb;
 	struct intel_framebuffer *intel_fb;
 
+	if (!plane->get_hw_state(plane))
+		return;
+
 	intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
 	if (!intel_fb) {
 		DRM_DEBUG_KMS("failed to alloc fb\n");
@@ -8429,14 +8493,19 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 
 	fb->dev = dev;
 
-	val = I915_READ(PLANE_CTL(pipe, 0));
-	if (!(val & PLANE_CTL_ENABLE))
-		goto error;
+	val = I915_READ(PLANE_CTL(pipe, plane_id));
 
 	pixel_format = val & PLANE_CTL_FORMAT_MASK;
+
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
+		alpha = I915_READ(PLANE_COLOR_CTL(pipe, plane_id));
+		alpha &= PLANE_COLOR_ALPHA_MASK;
+	} else {
+		alpha = val & PLANE_CTL_ALPHA_MASK;
+	}
+
 	fourcc = skl_format_to_fourcc(pixel_format,
-				      val & PLANE_CTL_ORDER_RGBX,
-				      val & PLANE_CTL_ALPHA_MASK);
+				      val & PLANE_CTL_ORDER_RGBX, alpha);
 	fb->format = drm_format_info(fourcc);
 
 	tiling = val & PLANE_CTL_TILED_MASK;
@@ -8465,16 +8534,16 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 		goto error;
 	}
 
-	base = I915_READ(PLANE_SURF(pipe, 0)) & 0xfffff000;
+	base = I915_READ(PLANE_SURF(pipe, plane_id)) & 0xfffff000;
 	plane_config->base = base;
 
-	offset = I915_READ(PLANE_OFFSET(pipe, 0));
+	offset = I915_READ(PLANE_OFFSET(pipe, plane_id));
 
-	val = I915_READ(PLANE_SIZE(pipe, 0));
+	val = I915_READ(PLANE_SIZE(pipe, plane_id));
 	fb->height = ((val >> 16) & 0xfff) + 1;
 	fb->width = ((val >> 0) & 0x1fff) + 1;
 
-	val = I915_READ(PLANE_STRIDE(pipe, 0));
+	val = I915_READ(PLANE_STRIDE(pipe, plane_id));
 	stride_mult = intel_fb_stride_alignment(fb, 0);
 	fb->pitches[0] = (val & 0x3ff) * stride_mult;
 
@@ -8482,8 +8551,8 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 
 	plane_config->size = fb->pitches[0] * aligned_height;
 
-	DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
-		      pipe_name(pipe), fb->width, fb->height,
+	DRM_DEBUG_KMS("%s/%s with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
+		      crtc->base.name, plane->base.name, fb->width, fb->height,
 		      fb->format->cpp[0] * 8, base, fb->pitches[0],
 		      plane_config->size);
 
@@ -8518,74 +8587,6 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc,
 	}
 }
 
-static void
-ironlake_get_initial_plane_config(struct intel_crtc *crtc,
-				  struct intel_initial_plane_config *plane_config)
-{
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	u32 val, base, offset;
-	int pipe = crtc->pipe;
-	int fourcc, pixel_format;
-	unsigned int aligned_height;
-	struct drm_framebuffer *fb;
-	struct intel_framebuffer *intel_fb;
-
-	val = I915_READ(DSPCNTR(pipe));
-	if (!(val & DISPLAY_PLANE_ENABLE))
-		return;
-
-	intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
-	if (!intel_fb) {
-		DRM_DEBUG_KMS("failed to alloc fb\n");
-		return;
-	}
-
-	fb = &intel_fb->base;
-
-	fb->dev = dev;
-
-	if (INTEL_GEN(dev_priv) >= 4) {
-		if (val & DISPPLANE_TILED) {
-			plane_config->tiling = I915_TILING_X;
-			fb->modifier = I915_FORMAT_MOD_X_TILED;
-		}
-	}
-
-	pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
-	fourcc = i9xx_format_to_fourcc(pixel_format);
-	fb->format = drm_format_info(fourcc);
-
-	base = I915_READ(DSPSURF(pipe)) & 0xfffff000;
-	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
-		offset = I915_READ(DSPOFFSET(pipe));
-	} else {
-		if (plane_config->tiling)
-			offset = I915_READ(DSPTILEOFF(pipe));
-		else
-			offset = I915_READ(DSPLINOFF(pipe));
-	}
-	plane_config->base = base;
-
-	val = I915_READ(PIPESRC(pipe));
-	fb->width = ((val >> 16) & 0xfff) + 1;
-	fb->height = ((val >> 0) & 0xfff) + 1;
-
-	val = I915_READ(DSPSTRIDE(pipe));
-	fb->pitches[0] = val & 0xffffffc0;
-
-	aligned_height = intel_fb_align_height(fb, 0, fb->height);
-
-	plane_config->size = fb->pitches[0] * aligned_height;
-
-	DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
-		      pipe_name(pipe), fb->width, fb->height,
-		      fb->format->cpp[0] * 8, base, fb->pitches[0],
-		      plane_config->size);
-
-	plane_config->fb = intel_fb;
-}
-
 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
 				     struct intel_crtc_state *pipe_config)
 {
@@ -8843,7 +8844,9 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 	}
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+
 	intel_update_cdclk(dev_priv);
+	intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
 }
 
 /*
@@ -9217,9 +9220,18 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 			ironlake_get_pfit_config(crtc, pipe_config);
 	}
 
-	if (IS_HASWELL(dev_priv))
-		pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
-			(I915_READ(IPS_CTL) & IPS_ENABLE);
+	if (hsw_crtc_supports_ips(crtc)) {
+		if (IS_HASWELL(dev_priv))
+			pipe_config->ips_enabled = I915_READ(IPS_CTL) & IPS_ENABLE;
+		else {
+			/*
+			 * We cannot readout IPS state on broadwell, set to
+			 * true so we can set it to a defined state on first
+			 * commit.
+			 */
+			pipe_config->ips_enabled = true;
+		}
+	}
 
 	if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
 	    !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
@@ -9300,11 +9312,12 @@ static int intel_check_cursor(struct intel_crtc_state *crtc_state,
 	u32 offset;
 	int ret;
 
-	ret = drm_plane_helper_check_state(&plane_state->base,
-					   &plane_state->clip,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   true, true);
+	ret = drm_atomic_helper_check_plane_state(&plane_state->base,
+						  &crtc_state->base,
+						  &plane_state->clip,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  true, true);
 	if (ret)
 		return ret;
 
@@ -9706,111 +9719,27 @@ intel_framebuffer_create(struct drm_i915_gem_object *obj,
 	return ERR_PTR(ret);
 }
 
-static u32
-intel_framebuffer_pitch_for_width(int width, int bpp)
-{
-	u32 pitch = DIV_ROUND_UP(width * bpp, 8);
-	return ALIGN(pitch, 64);
-}
-
-static u32
-intel_framebuffer_size_for_mode(const struct drm_display_mode *mode, int bpp)
-{
-	u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
-	return PAGE_ALIGN(pitch * mode->vdisplay);
-}
-
-static struct drm_framebuffer *
-intel_framebuffer_create_for_mode(struct drm_device *dev,
-				  const struct drm_display_mode *mode,
-				  int depth, int bpp)
-{
-	struct drm_framebuffer *fb;
-	struct drm_i915_gem_object *obj;
-	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
-
-	obj = i915_gem_object_create(to_i915(dev),
-				    intel_framebuffer_size_for_mode(mode, bpp));
-	if (IS_ERR(obj))
-		return ERR_CAST(obj);
-
-	mode_cmd.width = mode->hdisplay;
-	mode_cmd.height = mode->vdisplay;
-	mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width,
-								bpp);
-	mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
-
-	fb = intel_framebuffer_create(obj, &mode_cmd);
-	if (IS_ERR(fb))
-		i915_gem_object_put(obj);
-
-	return fb;
-}
-
-static struct drm_framebuffer *
-mode_fits_in_fbdev(struct drm_device *dev,
-		   const struct drm_display_mode *mode)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_i915_gem_object *obj;
-	struct drm_framebuffer *fb;
-
-	if (!dev_priv->fbdev)
-		return NULL;
-
-	if (!dev_priv->fbdev->fb)
-		return NULL;
-
-	obj = dev_priv->fbdev->fb->obj;
-	BUG_ON(!obj);
-
-	fb = &dev_priv->fbdev->fb->base;
-	if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
-							       fb->format->cpp[0] * 8))
-		return NULL;
-
-	if (obj->base.size < mode->vdisplay * fb->pitches[0])
-		return NULL;
-
-	drm_framebuffer_get(fb);
-	return fb;
-#else
-	return NULL;
-#endif
-}
-
-static int intel_modeset_setup_plane_state(struct drm_atomic_state *state,
-					   struct drm_crtc *crtc,
-					   const struct drm_display_mode *mode,
-					   struct drm_framebuffer *fb,
-					   int x, int y)
+static int intel_modeset_disable_planes(struct drm_atomic_state *state,
+					struct drm_crtc *crtc)
 {
+	struct drm_plane *plane;
 	struct drm_plane_state *plane_state;
-	int hdisplay, vdisplay;
-	int ret;
-
-	plane_state = drm_atomic_get_plane_state(state, crtc->primary);
-	if (IS_ERR(plane_state))
-		return PTR_ERR(plane_state);
-
-	if (mode)
-		drm_mode_get_hv_timing(mode, &hdisplay, &vdisplay);
-	else
-		hdisplay = vdisplay = 0;
+	int ret, i;
 
-	ret = drm_atomic_set_crtc_for_plane(plane_state, fb ? crtc : NULL);
+	ret = drm_atomic_add_affected_planes(state, crtc);
 	if (ret)
 		return ret;
-	drm_atomic_set_fb_for_plane(plane_state, fb);
-	plane_state->crtc_x = 0;
-	plane_state->crtc_y = 0;
-	plane_state->crtc_w = hdisplay;
-	plane_state->crtc_h = vdisplay;
-	plane_state->src_x = x << 16;
-	plane_state->src_y = y << 16;
-	plane_state->src_w = hdisplay << 16;
-	plane_state->src_h = vdisplay << 16;
+
+	for_each_new_plane_in_state(state, plane, plane_state, i) {
+		if (plane_state->crtc != crtc)
+			continue;
+
+		ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
+		if (ret)
+			return ret;
+
+		drm_atomic_set_fb_for_plane(plane_state, NULL);
+	}
 
 	return 0;
 }
@@ -9828,7 +9757,6 @@ int intel_get_load_detect_pipe(struct drm_connector *connector,
 	struct drm_crtc *crtc = NULL;
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_framebuffer *fb;
 	struct drm_mode_config *config = &dev->mode_config;
 	struct drm_atomic_state *state = NULL, *restore_state = NULL;
 	struct drm_connector_state *connector_state;
@@ -9896,10 +9824,6 @@ int intel_get_load_detect_pipe(struct drm_connector *connector,
 found:
 	intel_crtc = to_intel_crtc(crtc);
 
-	ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
-	if (ret)
-		goto fail;
-
 	state = drm_atomic_state_alloc(dev);
 	restore_state = drm_atomic_state_alloc(dev);
 	if (!state || !restore_state) {
@@ -9931,39 +9855,17 @@ int intel_get_load_detect_pipe(struct drm_connector *connector,
 	if (!mode)
 		mode = &load_detect_mode;
 
-	/* We need a framebuffer large enough to accommodate all accesses
-	 * that the plane may generate whilst we perform load detection.
-	 * We can not rely on the fbcon either being present (we get called
-	 * during its initialisation to detect all boot displays, or it may
-	 * not even exist) or that it is large enough to satisfy the
-	 * requested mode.
-	 */
-	fb = mode_fits_in_fbdev(dev, mode);
-	if (fb == NULL) {
-		DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
-		fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
-	} else
-		DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
-	if (IS_ERR(fb)) {
-		DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
-		ret = PTR_ERR(fb);
-		goto fail;
-	}
-
-	ret = intel_modeset_setup_plane_state(state, crtc, mode, fb, 0, 0);
-	drm_framebuffer_put(fb);
+	ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
 	if (ret)
 		goto fail;
 
-	ret = drm_atomic_set_mode_for_crtc(&crtc_state->base, mode);
+	ret = intel_modeset_disable_planes(state, crtc);
 	if (ret)
 		goto fail;
 
 	ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
 	if (!ret)
 		ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, crtc));
-	if (!ret)
-		ret = PTR_ERR_OR_ZERO(drm_atomic_get_plane_state(restore_state, crtc->primary));
 	if (ret) {
 		DRM_DEBUG_KMS("Failed to create a copy of old state to restore: %i\n", ret);
 		goto fail;
@@ -10472,6 +10374,9 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
 							 pipe_config);
 	}
 
+	if (HAS_IPS(dev_priv))
+		pipe_config->ips_enabled = hsw_compute_ips_config(pipe_config);
+
 	return ret;
 }
 
@@ -10601,7 +10506,7 @@ static const char * const output_type_str[] = {
 	OUTPUT_TYPE(DP),
 	OUTPUT_TYPE(EDP),
 	OUTPUT_TYPE(DSI),
-	OUTPUT_TYPE(UNKNOWN),
+	OUTPUT_TYPE(DDI),
 	OUTPUT_TYPE(DP_MST),
 };
 
@@ -10772,13 +10677,13 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 
 		switch (encoder->type) {
 			unsigned int port_mask;
-		case INTEL_OUTPUT_UNKNOWN:
+		case INTEL_OUTPUT_DDI:
 			if (WARN_ON(!HAS_DDI(to_i915(dev))))
 				break;
 		case INTEL_OUTPUT_DP:
 		case INTEL_OUTPUT_HDMI:
 		case INTEL_OUTPUT_EDP:
-			port_mask = 1 << enc_to_dig_port(&encoder->base)->port;
+			port_mask = 1 << encoder->port;
 
 			/* the same port mustn't appear more than once */
 			if (used_ports & port_mask)
@@ -10788,7 +10693,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 			break;
 		case INTEL_OUTPUT_DP_MST:
 			used_mst_ports |=
-				1 << enc_to_mst(&encoder->base)->primary->port;
+				1 << encoder->port;
 			break;
 		default:
 			break;
@@ -10905,7 +10810,12 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
 		 * Determine output_types before calling the .compute_config()
 		 * hooks so that the hooks can use this information safely.
 		 */
-		pipe_config->output_types |= 1 << encoder->type;
+		if (encoder->compute_output_type)
+			pipe_config->output_types |=
+				BIT(encoder->compute_output_type(encoder, pipe_config,
+								 connector_state));
+		else
+			pipe_config->output_types |= BIT(encoder->type);
 	}
 
 encoder_retry:
@@ -10969,31 +10879,6 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
 	return ret;
 }
 
-static void
-intel_modeset_update_crtc_state(struct drm_atomic_state *state)
-{
-	struct drm_crtc *crtc;
-	struct drm_crtc_state *new_crtc_state;
-	int i;
-
-	/* Double check state. */
-	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
-		to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state);
-
-		/*
-		 * Update legacy state to satisfy fbc code. This can
-		 * be removed when fbc uses the atomic state.
-		 */
-		if (drm_atomic_get_existing_plane_state(state, crtc->primary)) {
-			struct drm_plane_state *plane_state = crtc->primary->state;
-
-			crtc->primary->fb = plane_state->fb;
-			crtc->x = plane_state->src_x >> 16;
-			crtc->y = plane_state->src_y >> 16;
-		}
-	}
-}
-
 static bool intel_fuzzy_clock_check(int clock1, int clock2)
 {
 	int diff;
@@ -11094,6 +10979,9 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 			  bool adjust)
 {
 	bool ret = true;
+	bool fixup_inherited = adjust &&
+		(current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
+		!(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
 
 #define PIPE_CONF_CHECK_X(name)	\
 	if (current_config->name != pipe_config->name) { \
@@ -11113,6 +11001,31 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 		ret = false; \
 	}
 
+#define PIPE_CONF_CHECK_BOOL(name)	\
+	if (current_config->name != pipe_config->name) { \
+		pipe_config_err(adjust, __stringify(name), \
+			  "(expected %s, found %s)\n", \
+			  yesno(current_config->name), \
+			  yesno(pipe_config->name)); \
+		ret = false; \
+	}
+
+/*
+ * Checks state where we only read out the enabling, but not the entire
+ * state itself (like full infoframes or ELD for audio). These states
+ * require a full modeset on bootup to fix up.
+ */
+#define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) \
+	if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
+		PIPE_CONF_CHECK_BOOL(name); \
+	} else { \
+		pipe_config_err(adjust, __stringify(name), \
+			  "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
+			  yesno(current_config->name), \
+			  yesno(pipe_config->name)); \
+		ret = false; \
+	}
+
 #define PIPE_CONF_CHECK_P(name)	\
 	if (current_config->name != pipe_config->name) { \
 		pipe_config_err(adjust, __stringify(name), \
@@ -11198,7 +11111,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 
 	PIPE_CONF_CHECK_I(cpu_transcoder);
 
-	PIPE_CONF_CHECK_I(has_pch_encoder);
+	PIPE_CONF_CHECK_BOOL(has_pch_encoder);
 	PIPE_CONF_CHECK_I(fdi_lanes);
 	PIPE_CONF_CHECK_M_N(fdi_m_n);
 
@@ -11230,17 +11143,17 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 	PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
 
 	PIPE_CONF_CHECK_I(pixel_multiplier);
-	PIPE_CONF_CHECK_I(has_hdmi_sink);
+	PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
 	if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
 	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		PIPE_CONF_CHECK_I(limited_color_range);
+		PIPE_CONF_CHECK_BOOL(limited_color_range);
 
-	PIPE_CONF_CHECK_I(hdmi_scrambling);
-	PIPE_CONF_CHECK_I(hdmi_high_tmds_clock_ratio);
-	PIPE_CONF_CHECK_I(has_infoframe);
-	PIPE_CONF_CHECK_I(ycbcr420);
+	PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
+	PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
+	PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe);
+	PIPE_CONF_CHECK_BOOL(ycbcr420);
 
-	PIPE_CONF_CHECK_I(has_audio);
+	PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
 
 	PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
 			      DRM_MODE_FLAG_INTERLACE);
@@ -11266,7 +11179,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 		PIPE_CONF_CHECK_I(pipe_src_w);
 		PIPE_CONF_CHECK_I(pipe_src_h);
 
-		PIPE_CONF_CHECK_I(pch_pfit.enabled);
+		PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
 		if (current_config->pch_pfit.enabled) {
 			PIPE_CONF_CHECK_X(pch_pfit.pos);
 			PIPE_CONF_CHECK_X(pch_pfit.size);
@@ -11276,11 +11189,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 		PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
 	}
 
-	/* BDW+ don't expose a synchronous way to read the state */
-	if (IS_HASWELL(dev_priv))
-		PIPE_CONF_CHECK_I(ips_enabled);
-
-	PIPE_CONF_CHECK_I(double_wide);
+	PIPE_CONF_CHECK_BOOL(double_wide);
 
 	PIPE_CONF_CHECK_P(shared_dpll);
 	PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
@@ -11314,8 +11223,12 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 	PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
 	PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
 
+	PIPE_CONF_CHECK_I(min_voltage_level);
+
 #undef PIPE_CONF_CHECK_X
 #undef PIPE_CONF_CHECK_I
+#undef PIPE_CONF_CHECK_BOOL
+#undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
 #undef PIPE_CONF_CHECK_P
 #undef PIPE_CONF_CHECK_FLAGS
 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
@@ -11582,10 +11495,8 @@ verify_crtc_state(struct drm_crtc *crtc,
 				"Encoder connected to wrong pipe %c\n",
 				pipe_name(pipe));
 
-		if (active) {
-			pipe_config->output_types |= 1 << encoder->type;
+		if (active)
 			encoder->get_config(encoder, pipe_config);
-		}
 	}
 
 	intel_crtc_compute_pixel_rate(pipe_config);
@@ -11606,6 +11517,18 @@ verify_crtc_state(struct drm_crtc *crtc,
 	}
 }
 
+static void
+intel_verify_planes(struct intel_atomic_state *state)
+{
+	struct intel_plane *plane;
+	const struct intel_plane_state *plane_state;
+	int i;
+
+	for_each_new_intel_plane_in_state(state, plane,
+					  plane_state, i)
+		assert_plane(plane, plane_state->base.visible);
+}
+
 static void
 verify_single_dpll_state(struct drm_i915_private *dev_priv,
 			 struct intel_shared_dpll *pll,
@@ -11956,16 +11879,16 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 		 * holding all the crtc locks, even if we don't end up
 		 * touching the hardware
 		 */
-		if (!intel_cdclk_state_compare(&dev_priv->cdclk.logical,
-					       &intel_state->cdclk.logical)) {
+		if (intel_cdclk_changed(&dev_priv->cdclk.logical,
+					&intel_state->cdclk.logical)) {
 			ret = intel_lock_all_pipes(state);
 			if (ret < 0)
 				return ret;
 		}
 
 		/* All pipes must be switched off while we change the cdclk. */
-		if (!intel_cdclk_state_compare(&dev_priv->cdclk.actual,
-					       &intel_state->cdclk.actual)) {
+		if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
+					      &intel_state->cdclk.actual)) {
 			ret = intel_modeset_all_pipes(state);
 			if (ret < 0)
 				return ret;
@@ -11974,6 +11897,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 		DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
 			      intel_state->cdclk.logical.cdclk,
 			      intel_state->cdclk.actual.cdclk);
+		DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
+			      intel_state->cdclk.logical.voltage_level,
+			      intel_state->cdclk.actual.voltage_level);
 	} else {
 		to_intel_atomic_state(state)->cdclk.logical = dev_priv->cdclk.logical;
 	}
@@ -12085,7 +12011,7 @@ static int intel_atomic_check(struct drm_device *dev,
 	if (ret)
 		return ret;
 
-	intel_fbc_choose_crtc(dev_priv, state);
+	intel_fbc_choose_crtc(dev_priv, intel_state);
 	return calc_watermark_data(state);
 }
 
@@ -12325,9 +12251,9 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		}
 	}
 
-	/* Only after disabling all output pipelines that will be changed can we
-	 * update the the output configuration. */
-	intel_modeset_update_crtc_state(state);
+	/* FIXME: Eventually get rid of our intel_crtc->config pointer */
+	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i)
+		to_intel_crtc(crtc)->config = to_intel_crtc_state(new_crtc_state);
 
 	if (intel_state->modeset) {
 		drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
@@ -12396,6 +12322,9 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
 	}
 
+	if (intel_state->modeset)
+		intel_verify_planes(intel_state);
+
 	if (intel_state->modeset && intel_can_enable_sagv(state))
 		intel_enable_sagv(dev_priv);
 
@@ -12542,6 +12471,9 @@ static int intel_atomic_commit(struct drm_device *dev,
 	if (intel_state->modeset) {
 		memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
 		       sizeof(intel_state->min_cdclk));
+		memcpy(dev_priv->min_voltage_level,
+		       intel_state->min_voltage_level,
+		       sizeof(intel_state->min_voltage_level));
 		dev_priv->active_crtcs = intel_state->active_crtcs;
 		dev_priv->cdclk.logical = intel_state->cdclk.logical;
 		dev_priv->cdclk.actual = intel_state->cdclk.actual;
@@ -12783,7 +12715,7 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state
 	crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
 	max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
 
-	if (IS_GEMINILAKE(dev_priv))
+	if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
 		max_dotclk *= 2;
 
 	if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
@@ -12822,10 +12754,11 @@ intel_check_primary_plane(struct intel_plane *plane,
 		can_position = true;
 	}
 
-	ret = drm_plane_helper_check_state(&state->base,
-					   &state->clip,
-					   min_scale, max_scale,
-					   can_position, true);
+	ret = drm_atomic_helper_check_plane_state(&state->base,
+						  &crtc_state->base,
+						  &state->clip,
+						  min_scale, max_scale,
+						  can_position, true);
 	if (ret)
 		return ret;
 
@@ -12846,6 +12779,9 @@ intel_check_primary_plane(struct intel_plane *plane,
 		state->ctl = i9xx_plane_ctl(crtc_state, state);
 	}
 
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+		state->color_ctl = glk_plane_color_ctl(crtc_state, state);
+
 	return 0;
 }
 
@@ -12890,6 +12826,7 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
 				     struct drm_crtc_state *old_crtc_state)
 {
+	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct intel_atomic_state *old_intel_state =
 		to_intel_atomic_state(old_crtc_state->state);
@@ -12897,6 +12834,20 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
 		intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
 
 	intel_pipe_update_end(new_crtc_state);
+
+	if (new_crtc_state->update_pipe &&
+	    !needs_modeset(&new_crtc_state->base) &&
+	    old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED) {
+		if (!IS_GEN2(dev_priv))
+			intel_set_cpu_fifo_underrun_reporting(dev_priv, intel_crtc->pipe, true);
+
+		if (new_crtc_state->has_pch_encoder) {
+			enum pipe pch_transcoder =
+				intel_crtc_pch_transcoder(intel_crtc);
+
+			intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
+		}
+	}
 }
 
 /**
@@ -13198,9 +13149,9 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 	 * port is hooked to pipe B. Hence we want plane A feeding pipe B.
 	 */
 	if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4)
-		primary->plane = (enum plane) !pipe;
+		primary->i9xx_plane = (enum i9xx_plane_id) !pipe;
 	else
-		primary->plane = (enum plane) pipe;
+		primary->i9xx_plane = (enum i9xx_plane_id) pipe;
 	primary->id = PLANE_PRIMARY;
 	primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
 	primary->check_plane = intel_check_primary_plane;
@@ -13229,16 +13180,16 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 		num_formats = ARRAY_SIZE(i965_primary_formats);
 		modifiers = i9xx_format_modifiers;
 
-		primary->update_plane = i9xx_update_primary_plane;
-		primary->disable_plane = i9xx_disable_primary_plane;
+		primary->update_plane = i9xx_update_plane;
+		primary->disable_plane = i9xx_disable_plane;
 		primary->get_hw_state = i9xx_plane_get_hw_state;
 	} else {
 		intel_primary_formats = i8xx_primary_formats;
 		num_formats = ARRAY_SIZE(i8xx_primary_formats);
 		modifiers = i9xx_format_modifiers;
 
-		primary->update_plane = i9xx_update_primary_plane;
-		primary->disable_plane = i9xx_disable_primary_plane;
+		primary->update_plane = i9xx_update_plane;
+		primary->disable_plane = i9xx_disable_plane;
 		primary->get_hw_state = i9xx_plane_get_hw_state;
 	}
 
@@ -13262,11 +13213,17 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe)
 					       intel_primary_formats, num_formats,
 					       modifiers,
 					       DRM_PLANE_TYPE_PRIMARY,
-					       "plane %c", plane_name(primary->plane));
+					       "plane %c",
+					       plane_name(primary->i9xx_plane));
 	if (ret)
 		goto fail;
 
-	if (INTEL_GEN(dev_priv) >= 9) {
+	if (INTEL_GEN(dev_priv) >= 10) {
+		supported_rotations =
+			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
+			DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270 |
+			DRM_MODE_REFLECT_X;
+	} else if (INTEL_GEN(dev_priv) >= 9) {
 		supported_rotations =
 			DRM_MODE_ROTATE_0 | DRM_MODE_ROTATE_90 |
 			DRM_MODE_ROTATE_180 | DRM_MODE_ROTATE_270;
@@ -13322,7 +13279,7 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv,
 	cursor->can_scale = false;
 	cursor->max_downscale = 1;
 	cursor->pipe = pipe;
-	cursor->plane = pipe;
+	cursor->i9xx_plane = (enum i9xx_plane_id) pipe;
 	cursor->id = PLANE_CURSOR;
 	cursor->frontbuffer_bit = INTEL_FRONTBUFFER_CURSOR(pipe);
 
@@ -13450,14 +13407,13 @@ static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
 		goto fail;
 
 	intel_crtc->pipe = pipe;
-	intel_crtc->plane = primary->plane;
 
 	/* initialize shared scalers */
 	intel_crtc_init_scalers(intel_crtc, crtc_state);
 
 	BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
-	       dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
-	dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = intel_crtc;
+	       dev_priv->plane_to_crtc_mapping[primary->i9xx_plane] != NULL);
+	dev_priv->plane_to_crtc_mapping[primary->i9xx_plane] = intel_crtc;
 	dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = intel_crtc;
 
 	drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
@@ -14137,7 +14093,7 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
 	} else if (HAS_DDI(dev_priv)) {
 		dev_priv->display.get_pipe_config = haswell_get_pipe_config;
 		dev_priv->display.get_initial_plane_config =
-			ironlake_get_initial_plane_config;
+			i9xx_get_initial_plane_config;
 		dev_priv->display.crtc_compute_clock =
 			haswell_crtc_compute_clock;
 		dev_priv->display.crtc_enable = haswell_crtc_enable;
@@ -14145,7 +14101,7 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
 	} else if (HAS_PCH_SPLIT(dev_priv)) {
 		dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
 		dev_priv->display.get_initial_plane_config =
-			ironlake_get_initial_plane_config;
+			i9xx_get_initial_plane_config;
 		dev_priv->display.crtc_compute_clock =
 			ironlake_crtc_compute_clock;
 		dev_priv->display.crtc_enable = ironlake_crtc_enable;
@@ -14384,6 +14340,7 @@ void intel_modeset_init_hw(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
 	intel_update_cdclk(dev_priv);
+	intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
 	dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
 }
 
@@ -14463,6 +14420,8 @@ static void sanitize_watermarks(struct drm_device *dev)
 
 		cs->wm.need_postvbl_update = true;
 		dev_priv->display.optimize_watermarks(intel_state, cs);
+
+		to_intel_crtc_state(crtc->state)->wm = cs->wm;
 	}
 
 put_state:
@@ -14472,6 +14431,22 @@ static void sanitize_watermarks(struct drm_device *dev)
 	drm_modeset_acquire_fini(&ctx);
 }
 
+static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
+{
+	if (IS_GEN5(dev_priv)) {
+		u32 fdi_pll_clk =
+			I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
+
+		dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
+	} else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
+		dev_priv->fdi_pll_freq = 270000;
+	} else {
+		return;
+	}
+
+	DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
+}
+
 int intel_modeset_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -14544,7 +14519,7 @@ int intel_modeset_init(struct drm_device *dev)
 		dev->mode_config.cursor_height = MAX_CURSOR_HEIGHT;
 	}
 
-	dev->mode_config.fb_base = ggtt->mappable_base;
+	dev->mode_config.fb_base = ggtt->gmadr.start;
 
 	DRM_DEBUG_KMS("%d display pipe%s available.\n",
 		      INTEL_INFO(dev_priv)->num_pipes,
@@ -14561,6 +14536,7 @@ int intel_modeset_init(struct drm_device *dev)
 	}
 
 	intel_shared_dpll_init(dev);
+	intel_update_fdi_pll_freq(dev_priv);
 
 	intel_update_czclk(dev_priv);
 	intel_modeset_init_hw(dev);
@@ -14612,6 +14588,7 @@ int intel_modeset_init(struct drm_device *dev)
 
 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 {
+	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
 	/* 640x480@60Hz, ~25175 kHz */
 	struct dpll clock = {
 		.m1 = 18,
@@ -14675,6 +14652,8 @@ void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 
 	I915_WRITE(PIPECONF(pipe), PIPECONF_ENABLE | PIPECONF_PROGRESSIVE);
 	POSTING_READ(PIPECONF(pipe));
+
+	intel_wait_for_pipe_scanline_moving(crtc);
 }
 
 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
@@ -14700,11 +14679,11 @@ void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 }
 
 static bool intel_plane_mapping_ok(struct intel_crtc *crtc,
-				   struct intel_plane *primary)
+				   struct intel_plane *plane)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum plane plane = primary->plane;
-	u32 val = I915_READ(DSPCNTR(plane));
+	enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
+	u32 val = I915_READ(DSPCNTR(i9xx_plane));
 
 	return (val & DISPLAY_PLANE_ENABLE) == 0 ||
 		(val & DISPPLANE_SEL_PIPE_MASK) == DISPPLANE_SEL_PIPE(crtc->pipe);
@@ -14768,7 +14747,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc,
 	enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 
 	/* Clear any frame start delays used for debugging left by the BIOS */
-	if (!transcoder_is_dsi(cpu_transcoder)) {
+	if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
 		i915_reg_t reg = PIPECONF(cpu_transcoder);
 
 		I915_WRITE(reg,
@@ -14866,8 +14845,6 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 		connector->base.dpms = DRM_MODE_DPMS_OFF;
 		connector->base.encoder = NULL;
 	}
-	/* Enabled encoders without active connectors will be fixed in
-	 * the crtc fixup. */
 }
 
 void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv)
@@ -14980,7 +14957,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 			crtc_state = to_intel_crtc_state(crtc->base.state);
 
 			encoder->base.crtc = &crtc->base;
-			crtc_state->output_types |= 1 << encoder->type;
 			encoder->get_config(encoder, crtc_state);
 		} else {
 			encoder->base.crtc = NULL;
@@ -15059,6 +15035,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 		}
 
 		dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
+		dev_priv->min_voltage_level[crtc->pipe] =
+			crtc_state->min_voltage_level;
 
 		intel_pipe_config_sanity_check(dev_priv, crtc_state);
 	}
@@ -15082,6 +15060,23 @@ get_encoder_power_domains(struct drm_i915_private *dev_priv)
 	}
 }
 
+static void intel_early_display_was(struct drm_i915_private *dev_priv)
+{
+	/* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
+	if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
+		I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
+			   DARBF_GATING_DIS);
+
+	if (IS_HASWELL(dev_priv)) {
+		/*
+		 * WaRsPkgCStateDisplayPMReq:hsw
+		 * System hang if this isn't done before disabling all planes!
+		 */
+		I915_WRITE(CHICKEN_PAR1_1,
+			   I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
+	}
+}
+
 /* Scan out the current hw modeset state,
  * and sanitizes it to the current state
  */
@@ -15095,15 +15090,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev,
 	struct intel_encoder *encoder;
 	int i;
 
-	if (IS_HASWELL(dev_priv)) {
-		/*
-		 * WaRsPkgCStateDisplayPMReq:hsw
-		 * System hang if this isn't done before disabling all planes!
-		 */
-		I915_WRITE(CHICKEN_PAR1_1,
-			   I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
-	}
-
+	intel_early_display_was(dev_priv);
 	intel_modeset_readout_hw_state(dev);
 
 	/* HW state is read out, now we need to sanitize this mess. */
@@ -15197,17 +15184,6 @@ void intel_display_resume(struct drm_device *dev)
 		drm_atomic_state_put(state);
 }
 
-void intel_modeset_gem_init(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-
-	intel_init_gt_powersave(dev_priv);
-
-	intel_init_clock_gating(dev_priv);
-
-	intel_setup_overlay(dev_priv);
-}
-
 int intel_connector_register(struct drm_connector *connector)
 {
 	struct intel_connector *intel_connector = to_intel_connector(connector);
@@ -15236,10 +15212,7 @@ static void intel_hpd_poll_fini(struct drm_device *dev)
 	struct intel_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 
-	/* First disable polling... */
-	drm_kms_helper_poll_fini(dev);
-
-	/* Then kill the work that may have been queued by hpd. */
+	/* Kill all the work that may have been queued by hpd. */
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	for_each_intel_connector_iter(connector, &conn_iter) {
 		if (connector->modeset_retry_work.func)
diff --git a/drivers/gpu/drm/i915/intel_display.h b/drivers/gpu/drm/i915/intel_display.h
new file mode 100644
index 000000000000..a0d2b6169361
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_display.h
@@ -0,0 +1,321 @@
+/*
+ * Copyright © 2006-2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _INTEL_DISPLAY_H_
+#define _INTEL_DISPLAY_H_
+
+enum pipe {
+	INVALID_PIPE = -1,
+
+	PIPE_A = 0,
+	PIPE_B,
+	PIPE_C,
+	_PIPE_EDP,
+
+	I915_MAX_PIPES = _PIPE_EDP
+};
+
+#define pipe_name(p) ((p) + 'A')
+
+enum transcoder {
+	TRANSCODER_A = 0,
+	TRANSCODER_B,
+	TRANSCODER_C,
+	TRANSCODER_EDP,
+	TRANSCODER_DSI_A,
+	TRANSCODER_DSI_C,
+
+	I915_MAX_TRANSCODERS
+};
+
+static inline const char *transcoder_name(enum transcoder transcoder)
+{
+	switch (transcoder) {
+	case TRANSCODER_A:
+		return "A";
+	case TRANSCODER_B:
+		return "B";
+	case TRANSCODER_C:
+		return "C";
+	case TRANSCODER_EDP:
+		return "EDP";
+	case TRANSCODER_DSI_A:
+		return "DSI A";
+	case TRANSCODER_DSI_C:
+		return "DSI C";
+	default:
+		return "<invalid>";
+	}
+}
+
+static inline bool transcoder_is_dsi(enum transcoder transcoder)
+{
+	return transcoder == TRANSCODER_DSI_A || transcoder == TRANSCODER_DSI_C;
+}
+
+/*
+ * Global legacy plane identifier. Valid only for primary/sprite
+ * planes on pre-g4x, and only for primary planes on g4x-bdw.
+ */
+enum i9xx_plane_id {
+	PLANE_A,
+	PLANE_B,
+	PLANE_C,
+};
+
+#define plane_name(p) ((p) + 'A')
+#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A')
+
+/*
+ * Per-pipe plane identifier.
+ * I915_MAX_PLANES in the enum below is the maximum (across all platforms)
+ * number of planes per CRTC.  Not all platforms really have this many planes,
+ * which means some arrays of size I915_MAX_PLANES may have unused entries
+ * between the topmost sprite plane and the cursor plane.
+ *
+ * This is expected to be passed to various register macros
+ * (eg. PLANE_CTL(), PS_PLANE_SEL(), etc.) so adjust with care.
+ */
+enum plane_id {
+	PLANE_PRIMARY,
+	PLANE_SPRITE0,
+	PLANE_SPRITE1,
+	PLANE_SPRITE2,
+	PLANE_CURSOR,
+
+	I915_MAX_PLANES,
+};
+
+#define for_each_plane_id_on_crtc(__crtc, __p) \
+	for ((__p) = PLANE_PRIMARY; (__p) < I915_MAX_PLANES; (__p)++) \
+		for_each_if((__crtc)->plane_ids_mask & BIT(__p))
+
+enum port {
+	PORT_NONE = -1,
+
+	PORT_A = 0,
+	PORT_B,
+	PORT_C,
+	PORT_D,
+	PORT_E,
+
+	I915_MAX_PORTS
+};
+
+#define port_name(p) ((p) + 'A')
+
+enum dpio_channel {
+	DPIO_CH0,
+	DPIO_CH1
+};
+
+enum dpio_phy {
+	DPIO_PHY0,
+	DPIO_PHY1,
+	DPIO_PHY2,
+};
+
+#define I915_NUM_PHYS_VLV 2
+
+enum intel_display_power_domain {
+	POWER_DOMAIN_PIPE_A,
+	POWER_DOMAIN_PIPE_B,
+	POWER_DOMAIN_PIPE_C,
+	POWER_DOMAIN_PIPE_A_PANEL_FITTER,
+	POWER_DOMAIN_PIPE_B_PANEL_FITTER,
+	POWER_DOMAIN_PIPE_C_PANEL_FITTER,
+	POWER_DOMAIN_TRANSCODER_A,
+	POWER_DOMAIN_TRANSCODER_B,
+	POWER_DOMAIN_TRANSCODER_C,
+	POWER_DOMAIN_TRANSCODER_EDP,
+	POWER_DOMAIN_TRANSCODER_DSI_A,
+	POWER_DOMAIN_TRANSCODER_DSI_C,
+	POWER_DOMAIN_PORT_DDI_A_LANES,
+	POWER_DOMAIN_PORT_DDI_B_LANES,
+	POWER_DOMAIN_PORT_DDI_C_LANES,
+	POWER_DOMAIN_PORT_DDI_D_LANES,
+	POWER_DOMAIN_PORT_DDI_E_LANES,
+	POWER_DOMAIN_PORT_DDI_A_IO,
+	POWER_DOMAIN_PORT_DDI_B_IO,
+	POWER_DOMAIN_PORT_DDI_C_IO,
+	POWER_DOMAIN_PORT_DDI_D_IO,
+	POWER_DOMAIN_PORT_DDI_E_IO,
+	POWER_DOMAIN_PORT_DSI,
+	POWER_DOMAIN_PORT_CRT,
+	POWER_DOMAIN_PORT_OTHER,
+	POWER_DOMAIN_VGA,
+	POWER_DOMAIN_AUDIO,
+	POWER_DOMAIN_PLLS,
+	POWER_DOMAIN_AUX_A,
+	POWER_DOMAIN_AUX_B,
+	POWER_DOMAIN_AUX_C,
+	POWER_DOMAIN_AUX_D,
+	POWER_DOMAIN_GMBUS,
+	POWER_DOMAIN_MODESET,
+	POWER_DOMAIN_GT_IRQ,
+	POWER_DOMAIN_INIT,
+
+	POWER_DOMAIN_NUM,
+};
+
+#define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A)
+#define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \
+		((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER)
+#define POWER_DOMAIN_TRANSCODER(tran) \
+	((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \
+	 (tran) + POWER_DOMAIN_TRANSCODER_A)
+
+/* Used by dp and fdi links */
+struct intel_link_m_n {
+	u32 tu;
+	u32 gmch_m;
+	u32 gmch_n;
+	u32 link_m;
+	u32 link_n;
+};
+
+#define for_each_pipe(__dev_priv, __p) \
+	for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++)
+
+#define for_each_pipe_masked(__dev_priv, __p, __mask) \
+	for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \
+		for_each_if((__mask) & BIT(__p))
+
+#define for_each_universal_plane(__dev_priv, __pipe, __p)		\
+	for ((__p) = 0;							\
+	     (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1;	\
+	     (__p)++)
+
+#define for_each_sprite(__dev_priv, __p, __s)				\
+	for ((__s) = 0;							\
+	     (__s) < INTEL_INFO(__dev_priv)->num_sprites[(__p)];	\
+	     (__s)++)
+
+#define for_each_port_masked(__port, __ports_mask) \
+	for ((__port) = PORT_A; (__port) < I915_MAX_PORTS; (__port)++)	\
+		for_each_if((__ports_mask) & BIT(__port))
+
+#define for_each_crtc(dev, crtc) \
+	list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head)
+
+#define for_each_intel_plane(dev, intel_plane) \
+	list_for_each_entry(intel_plane,			\
+			    &(dev)->mode_config.plane_list,	\
+			    base.head)
+
+#define for_each_intel_plane_mask(dev, intel_plane, plane_mask)		\
+	list_for_each_entry(intel_plane,				\
+			    &(dev)->mode_config.plane_list,		\
+			    base.head)					\
+		for_each_if((plane_mask) &				\
+			    BIT(drm_plane_index(&intel_plane->base)))
+
+#define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane)	\
+	list_for_each_entry(intel_plane,				\
+			    &(dev)->mode_config.plane_list,		\
+			    base.head)					\
+		for_each_if((intel_plane)->pipe == (intel_crtc)->pipe)
+
+#define for_each_intel_crtc(dev, intel_crtc)				\
+	list_for_each_entry(intel_crtc,					\
+			    &(dev)->mode_config.crtc_list,		\
+			    base.head)
+
+#define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask)		\
+	list_for_each_entry(intel_crtc,					\
+			    &(dev)->mode_config.crtc_list,		\
+			    base.head)					\
+		for_each_if((crtc_mask) & BIT(drm_crtc_index(&intel_crtc->base)))
+
+#define for_each_intel_encoder(dev, intel_encoder)		\
+	list_for_each_entry(intel_encoder,			\
+			    &(dev)->mode_config.encoder_list,	\
+			    base.head)
+
+#define for_each_intel_connector_iter(intel_connector, iter) \
+	while ((intel_connector = to_intel_connector(drm_connector_list_iter_next(iter))))
+
+#define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
+	list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
+		for_each_if((intel_encoder)->base.crtc == (__crtc))
+
+#define for_each_connector_on_encoder(dev, __encoder, intel_connector) \
+	list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \
+		for_each_if((intel_connector)->base.encoder == (__encoder))
+
+#define for_each_power_domain(domain, mask)				\
+	for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)	\
+		for_each_if(BIT_ULL(domain) & (mask))
+
+#define for_each_power_well(__dev_priv, __power_well)				\
+	for ((__power_well) = (__dev_priv)->power_domains.power_wells;	\
+	     (__power_well) - (__dev_priv)->power_domains.power_wells <	\
+		(__dev_priv)->power_domains.power_well_count;		\
+	     (__power_well)++)
+
+#define for_each_power_well_rev(__dev_priv, __power_well)			\
+	for ((__power_well) = (__dev_priv)->power_domains.power_wells +		\
+			      (__dev_priv)->power_domains.power_well_count - 1;	\
+	     (__power_well) - (__dev_priv)->power_domains.power_wells >= 0;	\
+	     (__power_well)--)
+
+#define for_each_power_domain_well(__dev_priv, __power_well, __domain_mask)	\
+	for_each_power_well(__dev_priv, __power_well)				\
+		for_each_if((__power_well)->domains & (__domain_mask))
+
+#define for_each_power_domain_well_rev(__dev_priv, __power_well, __domain_mask) \
+	for_each_power_well_rev(__dev_priv, __power_well)		        \
+		for_each_if((__power_well)->domains & (__domain_mask))
+
+#define for_each_new_intel_plane_in_state(__state, plane, new_plane_state, __i) \
+	for ((__i) = 0; \
+	     (__i) < (__state)->base.dev->mode_config.num_total_plane && \
+		     ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \
+		      (new_plane_state) = to_intel_plane_state((__state)->base.planes[__i].new_state), 1); \
+	     (__i)++) \
+		for_each_if(plane)
+
+#define for_each_new_intel_crtc_in_state(__state, crtc, new_crtc_state, __i) \
+	for ((__i) = 0; \
+	     (__i) < (__state)->base.dev->mode_config.num_crtc && \
+		     ((crtc) = to_intel_crtc((__state)->base.crtcs[__i].ptr), \
+		      (new_crtc_state) = to_intel_crtc_state((__state)->base.crtcs[__i].new_state), 1); \
+	     (__i)++) \
+		for_each_if(crtc)
+
+#define for_each_oldnew_intel_plane_in_state(__state, plane, old_plane_state, new_plane_state, __i) \
+	for ((__i) = 0; \
+	     (__i) < (__state)->base.dev->mode_config.num_total_plane && \
+		     ((plane) = to_intel_plane((__state)->base.planes[__i].ptr), \
+		      (old_plane_state) = to_intel_plane_state((__state)->base.planes[__i].old_state), \
+		      (new_plane_state) = to_intel_plane_state((__state)->base.planes[__i].new_state), 1); \
+	     (__i)++) \
+		for_each_if(plane)
+
+void intel_link_compute_m_n(int bpp, int nlanes,
+			    int pixel_clock, int link_clock,
+			    struct intel_link_m_n *m_n,
+			    bool reduce_m_n);
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 158438bb0389..a29868cd30c7 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -129,11 +129,13 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector)
 	return enc_to_intel_dp(&intel_attached_encoder(connector)->base);
 }
 
-static void intel_dp_link_down(struct intel_dp *intel_dp);
+static void intel_dp_link_down(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *old_crtc_state);
 static bool edp_panel_vdd_on(struct intel_dp *intel_dp);
 static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync);
-static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp);
-static void vlv_steal_power_sequencer(struct drm_device *dev,
+static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder,
+					   const struct intel_crtc_state *crtc_state);
+static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
 				      enum pipe pipe);
 static void intel_dp_unset_edid(struct intel_dp *intel_dp);
 
@@ -221,7 +223,7 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	enum port port = dig_port->port;
+	enum port port = dig_port->base.port;
 	const int *source_rates;
 	int size;
 	u32 voltage;
@@ -427,24 +429,19 @@ static void intel_dp_unpack_aux(uint32_t src, uint8_t *dst, int dst_bytes)
 }
 
 static void
-intel_dp_init_panel_power_sequencer(struct drm_device *dev,
-				    struct intel_dp *intel_dp);
+intel_dp_init_panel_power_sequencer(struct intel_dp *intel_dp);
 static void
-intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
-					      struct intel_dp *intel_dp,
+intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp,
 					      bool force_disable_vdd);
 static void
-intel_dp_pps_init(struct drm_device *dev, struct intel_dp *intel_dp);
+intel_dp_pps_init(struct intel_dp *intel_dp);
 
 static void pps_lock(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_encoder *encoder = &intel_dig_port->base;
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	/*
-	 * See vlv_power_sequencer_reset() why we need
+	 * See intel_power_sequencer_reset() why we need
 	 * a power domain reference here.
 	 */
 	intel_display_power_get(dev_priv, intel_dp->aux_power_domain);
@@ -454,10 +451,7 @@ static void pps_lock(struct intel_dp *intel_dp)
 
 static void pps_unlock(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_encoder *encoder = &intel_dig_port->base;
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	mutex_unlock(&dev_priv->pps_mutex);
 
@@ -467,8 +461,8 @@ static void pps_unlock(struct intel_dp *intel_dp)
 static void
 vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
 	enum pipe pipe = intel_dp->pps_pipe;
 	bool pll_enabled, release_cl_override = false;
 	enum dpio_phy phy = DPIO_PHY(pipe);
@@ -477,11 +471,11 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 
 	if (WARN(I915_READ(intel_dp->output_reg) & DP_PORT_EN,
 		 "skipping pipe %c power seqeuncer kick due to port %c being active\n",
-		 pipe_name(pipe), port_name(intel_dig_port->port)))
+		 pipe_name(pipe), port_name(intel_dig_port->base.port)))
 		return;
 
 	DRM_DEBUG_KMS("kicking pipe %c power sequencer for port %c\n",
-		      pipe_name(pipe), port_name(intel_dig_port->port));
+		      pipe_name(pipe), port_name(intel_dig_port->base.port));
 
 	/* Preserve the BIOS-computed detected bit. This is
 	 * supposed to be read-only.
@@ -578,9 +572,8 @@ static enum pipe vlv_find_free_pps(struct drm_i915_private *dev_priv)
 static enum pipe
 vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum pipe pipe;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
@@ -603,16 +596,16 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 	if (WARN_ON(pipe == INVALID_PIPE))
 		pipe = PIPE_A;
 
-	vlv_steal_power_sequencer(dev, pipe);
+	vlv_steal_power_sequencer(dev_priv, pipe);
 	intel_dp->pps_pipe = pipe;
 
 	DRM_DEBUG_KMS("picked pipe %c power sequencer for port %c\n",
 		      pipe_name(intel_dp->pps_pipe),
-		      port_name(intel_dig_port->port));
+		      port_name(intel_dig_port->base.port));
 
 	/* init power sequencer on this pipe and port */
-	intel_dp_init_panel_power_sequencer(dev, intel_dp);
-	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, true);
+	intel_dp_init_panel_power_sequencer(intel_dp);
+	intel_dp_init_panel_power_sequencer_registers(intel_dp, true);
 
 	/*
 	 * Even vdd force doesn't work until we've made
@@ -626,22 +619,16 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 static int
 bxt_power_sequencer_idx(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
+	int backlight_controller = dev_priv->vbt.backlight.controller;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
 	/* We should never land here with regular DP ports */
 	WARN_ON(!intel_dp_is_edp(intel_dp));
 
-	/*
-	 * TODO: BXT has 2 PPS instances. The correct port->PPS instance
-	 * mapping needs to be retrieved from VBT, for now just hard-code to
-	 * use instance #0 always.
-	 */
 	if (!intel_dp->pps_reset)
-		return 0;
+		return backlight_controller;
 
 	intel_dp->pps_reset = false;
 
@@ -649,9 +636,9 @@ bxt_power_sequencer_idx(struct intel_dp *intel_dp)
 	 * Only the HW needs to be reprogrammed, the SW state is fixed and
 	 * has been setup during connector init.
 	 */
-	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false);
+	intel_dp_init_panel_power_sequencer_registers(intel_dp, false);
 
-	return 0;
+	return backlight_controller;
 }
 
 typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv,
@@ -701,10 +688,9 @@ vlv_initial_pps_pipe(struct drm_i915_private *dev_priv,
 static void
 vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -731,13 +717,12 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
 	DRM_DEBUG_KMS("initial power sequencer for port %c: pipe %c\n",
 		      port_name(port), pipe_name(intel_dp->pps_pipe));
 
-	intel_dp_init_panel_power_sequencer(dev, intel_dp);
-	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false);
+	intel_dp_init_panel_power_sequencer(intel_dp);
+	intel_dp_init_panel_power_sequencer_registers(intel_dp, false);
 }
 
 void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
 {
-	struct drm_device *dev = &dev_priv->drm;
 	struct intel_encoder *encoder;
 
 	if (WARN_ON(!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
@@ -754,15 +739,20 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
 	 * should use them always.
 	 */
 
-	for_each_intel_encoder(dev, encoder) {
+	for_each_intel_encoder(&dev_priv->drm, encoder) {
 		struct intel_dp *intel_dp;
 
 		if (encoder->type != INTEL_OUTPUT_DP &&
-		    encoder->type != INTEL_OUTPUT_EDP)
+		    encoder->type != INTEL_OUTPUT_EDP &&
+		    encoder->type != INTEL_OUTPUT_DDI)
 			continue;
 
 		intel_dp = enc_to_intel_dp(&encoder->base);
 
+		/* Skip pure DVI/HDMI DDI encoders */
+		if (!i915_mmio_reg_valid(intel_dp->output_reg))
+			continue;
+
 		WARN_ON(intel_dp->active_pipe != INVALID_PIPE);
 
 		if (encoder->type != INTEL_OUTPUT_EDP)
@@ -783,10 +773,10 @@ struct pps_registers {
 	i915_reg_t pp_div;
 };
 
-static void intel_pps_get_registers(struct drm_i915_private *dev_priv,
-				    struct intel_dp *intel_dp,
+static void intel_pps_get_registers(struct intel_dp *intel_dp,
 				    struct pps_registers *regs)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	int pps_idx = 0;
 
 	memset(regs, 0, sizeof(*regs));
@@ -809,8 +799,7 @@ _pp_ctrl_reg(struct intel_dp *intel_dp)
 {
 	struct pps_registers regs;
 
-	intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp,
-				&regs);
+	intel_pps_get_registers(intel_dp, &regs);
 
 	return regs.pp_ctrl;
 }
@@ -820,8 +809,7 @@ _pp_stat_reg(struct intel_dp *intel_dp)
 {
 	struct pps_registers regs;
 
-	intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp,
-				&regs);
+	intel_pps_get_registers(intel_dp, &regs);
 
 	return regs.pp_stat;
 }
@@ -833,8 +821,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 {
 	struct intel_dp *intel_dp = container_of(this, typeof(* intel_dp),
 						 edp_notifier);
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	if (!intel_dp_is_edp(intel_dp) || code != SYS_RESTART)
 		return 0;
@@ -864,8 +851,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 
 static bool edp_have_panel_power(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -878,8 +864,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
 
 static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -893,8 +878,7 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 static void
 intel_dp_check_edp(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	if (!intel_dp_is_edp(intel_dp))
 		return;
@@ -910,9 +894,7 @@ intel_dp_check_edp(struct intel_dp *intel_dp)
 static uint32_t
 intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
 	uint32_t status;
 	bool done;
@@ -959,7 +941,7 @@ static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 	 * like to run at 2MHz.  So, take the cdclk or PCH rawclk value and
 	 * divide by 2000 and use that
 	 */
-	if (intel_dig_port->port == PORT_A)
+	if (intel_dig_port->base.port == PORT_A)
 		return DIV_ROUND_CLOSEST(dev_priv->cdclk.hw.cdclk, 2000);
 	else
 		return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000);
@@ -970,7 +952,7 @@ static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
 
-	if (intel_dig_port->port != PORT_A && HAS_PCH_LPT_H(dev_priv)) {
+	if (intel_dig_port->base.port != PORT_A && HAS_PCH_LPT_H(dev_priv)) {
 		/* Workaround for non-ULT HSW */
 		switch (index) {
 		case 0: return 63;
@@ -1440,7 +1422,7 @@ static void intel_aux_reg_init(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	enum port port = intel_aux_port(dev_priv,
-					dp_to_dig_port(intel_dp)->port);
+					dp_to_dig_port(intel_dp)->base.port);
 	int i;
 
 	intel_dp->aux_ch_ctl_reg = intel_aux_ctl_reg(dev_priv, port);
@@ -1458,7 +1440,7 @@ static void
 intel_dp_aux_init(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 
 	intel_aux_reg_init(intel_dp);
 	drm_dp_aux_init(&intel_dp->aux);
@@ -1479,8 +1461,7 @@ static void
 intel_dp_set_clock(struct intel_encoder *encoder,
 		   struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	const struct dp_link_dpll *divisor = NULL;
 	int i, count = 0;
 
@@ -1628,7 +1609,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum port port = encoder->port;
 	struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
 	struct intel_connector *intel_connector = intel_dp->attached_connector;
 	struct intel_digital_connector_state *intel_conn_state =
@@ -1658,7 +1639,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		pipe_config->has_pch_encoder = true;
 
 	pipe_config->has_drrs = false;
-	if (port == PORT_A)
+	if (IS_G4X(dev_priv) || port == PORT_A)
 		pipe_config->has_audio = false;
 	else if (intel_conn_state->force_audio == HDMI_AUDIO_AUTO)
 		pipe_config->has_audio = intel_dp->has_audio;
@@ -1692,6 +1673,10 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 						conn_state->scaling_mode);
 	}
 
+	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
+	    adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE)
+		return false;
+
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK)
 		return false;
 
@@ -1849,11 +1834,10 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp,
 static void intel_dp_prepare(struct intel_encoder *encoder,
 			     const struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	enum port port = dp_to_dig_port(intel_dp)->port;
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+	enum port port = encoder->port;
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
 	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 
 	intel_dp_set_link_params(intel_dp, pipe_config->port_clock,
@@ -1940,20 +1924,18 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 #define IDLE_CYCLE_MASK		(PP_ON | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK)
 #define IDLE_CYCLE_VALUE	(0     | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_OFF_IDLE)
 
-static void intel_pps_verify_state(struct drm_i915_private *dev_priv,
-				   struct intel_dp *intel_dp);
+static void intel_pps_verify_state(struct intel_dp *intel_dp);
 
 static void wait_panel_status(struct intel_dp *intel_dp,
 				       u32 mask,
 				       u32 value)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	intel_pps_verify_state(dev_priv, intel_dp);
+	intel_pps_verify_state(intel_dp);
 
 	pp_stat_reg = _pp_stat_reg(intel_dp);
 	pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
@@ -2024,8 +2006,7 @@ static void edp_wait_backlight_off(struct intel_dp *intel_dp)
 
 static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u32 control;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
@@ -2046,9 +2027,8 @@ static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
  */
 static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 pp;
 	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 	bool need_to_disable = !intel_dp->want_panel_vdd;
@@ -2067,7 +2047,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
 	intel_display_power_get(dev_priv, intel_dp->aux_power_domain);
 
 	DRM_DEBUG_KMS("Turning eDP port %c VDD on\n",
-		      port_name(intel_dig_port->port));
+		      port_name(intel_dig_port->base.port));
 
 	if (!edp_have_panel_power(intel_dp))
 		wait_panel_power_cycle(intel_dp);
@@ -2087,7 +2067,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
 	 */
 	if (!edp_have_panel_power(intel_dp)) {
 		DRM_DEBUG_KMS("eDP port %c panel power wasn't enabled\n",
-			      port_name(intel_dig_port->port));
+			      port_name(intel_dig_port->base.port));
 		msleep(intel_dp->panel_power_up_delay);
 	}
 
@@ -2113,13 +2093,12 @@ void intel_edp_panel_vdd_on(struct intel_dp *intel_dp)
 	pps_unlock(intel_dp);
 
 	I915_STATE_WARN(!vdd, "eDP port %c VDD already requested on\n",
-	     port_name(dp_to_dig_port(intel_dp)->port));
+	     port_name(dp_to_dig_port(intel_dp)->base.port));
 }
 
 static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port =
 		dp_to_dig_port(intel_dp);
 	u32 pp;
@@ -2133,7 +2112,7 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 		return;
 
 	DRM_DEBUG_KMS("Turning eDP port %c VDD off\n",
-		      port_name(intel_dig_port->port));
+		      port_name(intel_dig_port->base.port));
 
 	pp = ironlake_get_pp_control(intel_dp);
 	pp &= ~EDP_FORCE_VDD;
@@ -2193,7 +2172,7 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
 		return;
 
 	I915_STATE_WARN(!intel_dp->want_panel_vdd, "eDP port %c VDD not forced on",
-	     port_name(dp_to_dig_port(intel_dp)->port));
+	     port_name(dp_to_dig_port(intel_dp)->base.port));
 
 	intel_dp->want_panel_vdd = false;
 
@@ -2205,8 +2184,7 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
 
 static void edp_panel_on(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u32 pp;
 	i915_reg_t pp_ctrl_reg;
 
@@ -2216,11 +2194,11 @@ static void edp_panel_on(struct intel_dp *intel_dp)
 		return;
 
 	DRM_DEBUG_KMS("Turn eDP port %c panel power on\n",
-		      port_name(dp_to_dig_port(intel_dp)->port));
+		      port_name(dp_to_dig_port(intel_dp)->base.port));
 
 	if (WARN(edp_have_panel_power(intel_dp),
 		 "eDP port %c panel power already on\n",
-		 port_name(dp_to_dig_port(intel_dp)->port)))
+		 port_name(dp_to_dig_port(intel_dp)->base.port)))
 		return;
 
 	wait_panel_power_cycle(intel_dp);
@@ -2264,8 +2242,7 @@ void intel_edp_panel_on(struct intel_dp *intel_dp)
 
 static void edp_panel_off(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u32 pp;
 	i915_reg_t pp_ctrl_reg;
 
@@ -2275,10 +2252,10 @@ static void edp_panel_off(struct intel_dp *intel_dp)
 		return;
 
 	DRM_DEBUG_KMS("Turn eDP port %c panel power off\n",
-		      port_name(dp_to_dig_port(intel_dp)->port));
+		      port_name(dp_to_dig_port(intel_dp)->base.port));
 
 	WARN(!intel_dp->want_panel_vdd, "Need eDP port %c VDD to turn off panel\n",
-	     port_name(dp_to_dig_port(intel_dp)->port));
+	     port_name(dp_to_dig_port(intel_dp)->base.port));
 
 	pp = ironlake_get_pp_control(intel_dp);
 	/* We need to switch off panel power _and_ force vdd, for otherwise some
@@ -2313,9 +2290,7 @@ void intel_edp_panel_off(struct intel_dp *intel_dp)
 /* Enable backlight in the panel power control. */
 static void _intel_edp_backlight_on(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u32 pp;
 	i915_reg_t pp_ctrl_reg;
 
@@ -2358,8 +2333,7 @@ void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state,
 /* Disable backlight in the panel power control. */
 static void _intel_edp_backlight_off(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u32 pp;
 	i915_reg_t pp_ctrl_reg;
 
@@ -2430,7 +2404,7 @@ static void assert_dp_port(struct intel_dp *intel_dp, bool state)
 
 	I915_STATE_WARN(cur_state != state,
 			"DP port %c state assertion failure (expected %s, current %s)\n",
-			port_name(dig_port->port),
+			port_name(dig_port->base.port),
 			onoff(state), onoff(cur_state));
 }
 #define assert_dp_port_disabled(d) assert_dp_port((d), false)
@@ -2486,10 +2460,10 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp,
 	udelay(200);
 }
 
-static void ironlake_edp_pll_off(struct intel_dp *intel_dp)
+static void ironlake_edp_pll_off(struct intel_dp *intel_dp,
+				 const struct intel_crtc_state *old_crtc_state)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
 	assert_pipe_disabled(dev_priv, crtc->pipe);
@@ -2505,6 +2479,21 @@ static void ironlake_edp_pll_off(struct intel_dp *intel_dp)
 	udelay(200);
 }
 
+static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp)
+{
+	/*
+	 * DPCD 1.2+ should support BRANCH_DEVICE_CTRL, and thus
+	 * be capable of signalling downstream hpd with a long pulse.
+	 * Whether or not that means D3 is safe to use is not clear,
+	 * but let's assume so until proven otherwise.
+	 *
+	 * FIXME should really check all downstream ports...
+	 */
+	return intel_dp->dpcd[DP_DPCD_REV] == 0x11 &&
+		intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT &&
+		intel_dp->downstream_ports[0] & DP_DS_PORT_HPD;
+}
+
 /* If the sink supports it, try to set the power state appropriately */
 void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
 {
@@ -2515,6 +2504,9 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
 		return;
 
 	if (mode != DRM_MODE_DPMS_ON) {
+		if (downstream_hpd_needs_d0(intel_dp))
+			return;
+
 		ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER,
 					 DP_SET_POWER_D3);
 	} else {
@@ -2544,10 +2536,9 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
 static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
 				  enum pipe *pipe)
 {
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	enum port port = dp_to_dig_port(intel_dp)->port;
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	enum port port = encoder->port;
 	u32 tmp;
 	bool ret;
 
@@ -2596,12 +2587,16 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
 static void intel_dp_get_config(struct intel_encoder *encoder,
 				struct intel_crtc_state *pipe_config)
 {
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	u32 tmp, flags = 0;
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum port port = dp_to_dig_port(intel_dp)->port;
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+	enum port port = encoder->port;
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
+
+	if (encoder->type == INTEL_OUTPUT_EDP)
+		pipe_config->output_types |= BIT(INTEL_OUTPUT_EDP);
+	else
+		pipe_config->output_types |= BIT(INTEL_OUTPUT_DP);
 
 	tmp = I915_READ(intel_dp->output_reg);
 
@@ -2680,7 +2675,8 @@ static void intel_disable_dp(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
 	if (old_crtc_state->has_audio)
-		intel_audio_codec_disable(encoder);
+		intel_audio_codec_disable(encoder,
+					  old_crtc_state, old_conn_state);
 
 	/* Make sure the panel is off before trying to change the mode. But also
 	 * ensure that we have vdd while we switch off the panel. */
@@ -2694,12 +2690,10 @@ static void g4x_disable_dp(struct intel_encoder *encoder,
 			   const struct intel_crtc_state *old_crtc_state,
 			   const struct drm_connector_state *old_conn_state)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-
 	intel_disable_dp(encoder, old_crtc_state, old_conn_state);
 
 	/* disable the port before the pipe on g4x */
-	intel_dp_link_down(intel_dp);
+	intel_dp_link_down(encoder, old_crtc_state);
 }
 
 static void ilk_disable_dp(struct intel_encoder *encoder,
@@ -2725,38 +2719,34 @@ static void ilk_post_disable_dp(struct intel_encoder *encoder,
 				const struct drm_connector_state *old_conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum port port = encoder->port;
 
-	intel_dp_link_down(intel_dp);
+	intel_dp_link_down(encoder, old_crtc_state);
 
 	/* Only ilk+ has port A */
 	if (port == PORT_A)
-		ironlake_edp_pll_off(intel_dp);
+		ironlake_edp_pll_off(intel_dp, old_crtc_state);
 }
 
 static void vlv_post_disable_dp(struct intel_encoder *encoder,
 				const struct intel_crtc_state *old_crtc_state,
 				const struct drm_connector_state *old_conn_state)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-
-	intel_dp_link_down(intel_dp);
+	intel_dp_link_down(encoder, old_crtc_state);
 }
 
 static void chv_post_disable_dp(struct intel_encoder *encoder,
 				const struct intel_crtc_state *old_crtc_state,
 				const struct drm_connector_state *old_conn_state)
 {
-	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	intel_dp_link_down(intel_dp);
+	intel_dp_link_down(encoder, old_crtc_state);
 
 	mutex_lock(&dev_priv->sb_lock);
 
 	/* Assert data lane reset */
-	chv_data_lane_soft_reset(encoder, true);
+	chv_data_lane_soft_reset(encoder, old_crtc_state, true);
 
 	mutex_unlock(&dev_priv->sb_lock);
 }
@@ -2766,10 +2756,9 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 			 uint32_t *DP,
 			 uint8_t dp_train_pat)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 
 	if (dp_train_pat & DP_TRAINING_PATTERN_MASK)
 		DRM_DEBUG_KMS("Using DP training pattern TPS%d\n",
@@ -2852,8 +2841,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 static void intel_dp_enable_port(struct intel_dp *intel_dp,
 				 const struct intel_crtc_state *old_crtc_state)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	/* enable with pattern 1 (as per spec) */
 
@@ -2877,10 +2865,9 @@ static void intel_enable_dp(struct intel_encoder *encoder,
 			    const struct intel_crtc_state *pipe_config,
 			    const struct drm_connector_state *conn_state)
 {
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
 	uint32_t dp_reg = I915_READ(intel_dp->output_reg);
 	enum pipe pipe = crtc->pipe;
 
@@ -2890,7 +2877,7 @@ static void intel_enable_dp(struct intel_encoder *encoder,
 	pps_lock(intel_dp);
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		vlv_init_panel_power_sequencer(intel_dp);
+		vlv_init_panel_power_sequencer(encoder, pipe_config);
 
 	intel_dp_enable_port(intel_dp, pipe_config);
 
@@ -2944,7 +2931,7 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder,
 			      const struct drm_connector_state *conn_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum port port = encoder->port;
 
 	intel_dp_prepare(encoder, pipe_config);
 
@@ -2977,22 +2964,21 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
 	 * from a port.
 	 */
 	DRM_DEBUG_KMS("detaching pipe %c power sequencer from port %c\n",
-		      pipe_name(pipe), port_name(intel_dig_port->port));
+		      pipe_name(pipe), port_name(intel_dig_port->base.port));
 	I915_WRITE(pp_on_reg, 0);
 	POSTING_READ(pp_on_reg);
 
 	intel_dp->pps_pipe = INVALID_PIPE;
 }
 
-static void vlv_steal_power_sequencer(struct drm_device *dev,
+static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
 				      enum pipe pipe)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_encoder *encoder;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	for_each_intel_encoder(dev, encoder) {
+	for_each_intel_encoder(&dev_priv->drm, encoder) {
 		struct intel_dp *intel_dp;
 		enum port port;
 
@@ -3001,7 +2987,7 @@ static void vlv_steal_power_sequencer(struct drm_device *dev,
 			continue;
 
 		intel_dp = enc_to_intel_dp(&encoder->base);
-		port = dp_to_dig_port(intel_dp)->port;
+		port = dp_to_dig_port(intel_dp)->base.port;
 
 		WARN(intel_dp->active_pipe == pipe,
 		     "stealing pipe %c power sequencer from active (e)DP port %c\n",
@@ -3018,13 +3004,12 @@ static void vlv_steal_power_sequencer(struct drm_device *dev,
 	}
 }
 
-static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
+static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder,
+					   const struct intel_crtc_state *crtc_state)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_encoder *encoder = &intel_dig_port->base;
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -3044,7 +3029,7 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
 	 * We may be stealing the power
 	 * sequencer from another port.
 	 */
-	vlv_steal_power_sequencer(dev, crtc->pipe);
+	vlv_steal_power_sequencer(dev_priv, crtc->pipe);
 
 	intel_dp->active_pipe = crtc->pipe;
 
@@ -3055,18 +3040,18 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
 	intel_dp->pps_pipe = crtc->pipe;
 
 	DRM_DEBUG_KMS("initializing pipe %c power sequencer for port %c\n",
-		      pipe_name(intel_dp->pps_pipe), port_name(intel_dig_port->port));
+		      pipe_name(intel_dp->pps_pipe), port_name(encoder->port));
 
 	/* init power sequencer on this pipe and port */
-	intel_dp_init_panel_power_sequencer(dev, intel_dp);
-	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, true);
+	intel_dp_init_panel_power_sequencer(intel_dp);
+	intel_dp_init_panel_power_sequencer_registers(intel_dp, true);
 }
 
 static void vlv_pre_enable_dp(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *pipe_config,
 			      const struct drm_connector_state *conn_state)
 {
-	vlv_phy_pre_encoder_enable(encoder);
+	vlv_phy_pre_encoder_enable(encoder, pipe_config);
 
 	intel_enable_dp(encoder, pipe_config, conn_state);
 }
@@ -3077,14 +3062,14 @@ static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder,
 {
 	intel_dp_prepare(encoder, pipe_config);
 
-	vlv_phy_pre_pll_enable(encoder);
+	vlv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_pre_enable_dp(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *pipe_config,
 			      const struct drm_connector_state *conn_state)
 {
-	chv_phy_pre_encoder_enable(encoder);
+	chv_phy_pre_encoder_enable(encoder, pipe_config);
 
 	intel_enable_dp(encoder, pipe_config, conn_state);
 
@@ -3098,14 +3083,14 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder,
 {
 	intel_dp_prepare(encoder, pipe_config);
 
-	chv_phy_pre_pll_enable(encoder);
+	chv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_dp_post_pll_disable(struct intel_encoder *encoder,
-				    const struct intel_crtc_state *pipe_config,
-				    const struct drm_connector_state *conn_state)
+				    const struct intel_crtc_state *old_crtc_state,
+				    const struct drm_connector_state *old_conn_state)
 {
-	chv_phy_post_pll_disable(encoder);
+	chv_phy_post_pll_disable(encoder, old_crtc_state);
 }
 
 /*
@@ -3153,7 +3138,7 @@ uint8_t
 intel_dp_voltage_max(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
-	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum port port = dp_to_dig_port(intel_dp)->base.port;
 
 	if (INTEL_GEN(dev_priv) >= 9) {
 		struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
@@ -3172,7 +3157,7 @@ uint8_t
 intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 {
 	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
-	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum port port = dp_to_dig_port(intel_dp)->base.port;
 
 	if (INTEL_GEN(dev_priv) >= 9) {
 		switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
@@ -3505,10 +3490,9 @@ gen7_edp_signal_levels(uint8_t train_set)
 void
 intel_dp_set_signal_levels(struct intel_dp *intel_dp)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	enum port port = intel_dig_port->port;
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	enum port port = intel_dig_port->base.port;
 	uint32_t signal_levels, mask = 0;
 	uint8_t train_set = intel_dp->train_set[0];
 
@@ -3563,10 +3547,9 @@ intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
 
 void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 	uint32_t val;
 
 	if (!HAS_DDI(dev_priv))
@@ -3595,13 +3578,13 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 }
 
 static void
-intel_dp_link_down(struct intel_dp *intel_dp)
+intel_dp_link_down(struct intel_encoder *encoder,
+		   const struct intel_crtc_state *old_crtc_state)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
-	enum port port = intel_dig_port->port;
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+	enum port port = encoder->port;
 	uint32_t DP = intel_dp->DP;
 
 	if (WARN_ON(HAS_DDI(dev_priv)))
@@ -3747,11 +3730,11 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
 	if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV,
 			     intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) ==
 			     sizeof(intel_dp->edp_dpcd))
-		DRM_DEBUG_KMS("EDP DPCD : %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
+		DRM_DEBUG_KMS("eDP DPCD: %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
 			      intel_dp->edp_dpcd);
 
-	/* Intermediate frequency support */
-	if (intel_dp->edp_dpcd[0] >= 0x03) { /* eDp v1.4 or higher */
+	/* Read the eDP 1.4+ supported link rates. */
+	if (intel_dp->edp_dpcd[0] >= DP_EDP_14) {
 		__le16 sink_rates[DP_MAX_SUPPORTED_RATES];
 		int i;
 
@@ -3775,6 +3758,10 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
 		intel_dp->num_sink_rates = i;
 	}
 
+	/*
+	 * Use DP_LINK_RATE_SET if DP_SUPPORTED_LINK_RATES are available,
+	 * default to DP_MAX_LINK_RATE and DP_LINK_BW_SET otherwise.
+	 */
 	if (intel_dp->num_sink_rates)
 		intel_dp->use_rate_select = true;
 	else
@@ -3874,11 +3861,12 @@ intel_dp_configure_mst(struct intel_dp *intel_dp)
 					intel_dp->is_mst);
 }
 
-static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
+static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp,
+				  struct intel_crtc_state *crtc_state, bool disable_wa)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	u8 buf;
 	int ret = 0;
 	int count = 0;
@@ -3914,15 +3902,17 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
 	}
 
  out:
-	hsw_enable_ips(intel_crtc);
+	if (disable_wa)
+		hsw_enable_ips(crtc_state);
 	return ret;
 }
 
-static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
+static int intel_dp_sink_crc_start(struct intel_dp *intel_dp,
+				   struct intel_crtc_state *crtc_state)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	u8 buf;
 	int ret;
 
@@ -3936,16 +3926,16 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
 		return -EIO;
 
 	if (buf & DP_TEST_SINK_START) {
-		ret = intel_dp_sink_crc_stop(intel_dp);
+		ret = intel_dp_sink_crc_stop(intel_dp, crtc_state, false);
 		if (ret)
 			return ret;
 	}
 
-	hsw_disable_ips(intel_crtc);
+	hsw_disable_ips(crtc_state);
 
 	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
 			       buf | DP_TEST_SINK_START) < 0) {
-		hsw_enable_ips(intel_crtc);
+		hsw_enable_ips(crtc_state);
 		return -EIO;
 	}
 
@@ -3953,16 +3943,16 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
 	return 0;
 }
 
-int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
+int intel_dp_sink_crc(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state, u8 *crc)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
 	u8 buf;
 	int count, ret;
 	int attempts = 6;
 
-	ret = intel_dp_sink_crc_start(intel_dp);
+	ret = intel_dp_sink_crc_start(intel_dp, crtc_state);
 	if (ret)
 		return ret;
 
@@ -3990,7 +3980,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
 	}
 
 stop:
-	intel_dp_sink_crc_stop(intel_dp);
+	intel_dp_sink_crc_stop(intel_dp, crtc_state, true);
 	return ret;
 }
 
@@ -4285,21 +4275,29 @@ intel_dp_retrain_link(struct intel_dp *intel_dp)
 static void
 intel_dp_check_link_status(struct intel_dp *intel_dp)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
+	struct drm_connector_state *conn_state =
+		intel_dp->attached_connector->base.state;
 	u8 link_status[DP_LINK_STATUS_SIZE];
 
-	WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
+	WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
 
 	if (!intel_dp_get_link_status(intel_dp, link_status)) {
 		DRM_ERROR("Failed to get link status\n");
 		return;
 	}
 
-	if (!intel_encoder->base.crtc)
+	if (!conn_state->crtc)
+		return;
+
+	WARN_ON(!drm_modeset_is_locked(&conn_state->crtc->mutex));
+
+	if (!conn_state->crtc->state->active)
 		return;
 
-	if (!to_intel_crtc(intel_encoder->base.crtc)->active)
+	if (conn_state->commit &&
+	    !try_wait_for_completion(&conn_state->commit->hw_done))
 		return;
 
 	/*
@@ -4335,8 +4333,7 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 static bool
 intel_dp_short_pulse(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u8 sink_irq_vector = 0;
 	u8 old_sink_count = intel_dp->sink_count;
 	bool ret;
@@ -4375,13 +4372,12 @@ intel_dp_short_pulse(struct intel_dp *intel_dp)
 			DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n");
 	}
 
-	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
 	intel_dp_check_link_status(intel_dp);
-	drm_modeset_unlock(&dev->mode_config.connection_mutex);
+
 	if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
 		DRM_DEBUG_KMS("Link Training Compliance Test requested\n");
 		/* Send a Hotplug Uevent to userspace to start modeset */
-		drm_kms_helper_hotplug_event(intel_encoder->base.dev);
+		drm_kms_helper_hotplug_event(&dev_priv->drm);
 	}
 
 	return true;
@@ -4445,8 +4441,7 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 static enum drm_connector_status
 edp_detect(struct intel_dp *intel_dp)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	enum drm_connector_status status;
 
 	status = intel_panel_detect(dev_priv);
@@ -4461,7 +4456,7 @@ static bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
 {
 	u32 bit;
 
-	switch (port->port) {
+	switch (port->base.port) {
 	case PORT_B:
 		bit = SDE_PORTB_HOTPLUG;
 		break;
@@ -4472,7 +4467,7 @@ static bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
 		bit = SDE_PORTD_HOTPLUG;
 		break;
 	default:
-		MISSING_CASE(port->port);
+		MISSING_CASE(port->base.port);
 		return false;
 	}
 
@@ -4484,7 +4479,7 @@ static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv,
 {
 	u32 bit;
 
-	switch (port->port) {
+	switch (port->base.port) {
 	case PORT_B:
 		bit = SDE_PORTB_HOTPLUG_CPT;
 		break;
@@ -4495,7 +4490,7 @@ static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv,
 		bit = SDE_PORTD_HOTPLUG_CPT;
 		break;
 	default:
-		MISSING_CASE(port->port);
+		MISSING_CASE(port->base.port);
 		return false;
 	}
 
@@ -4507,7 +4502,7 @@ static bool spt_digital_port_connected(struct drm_i915_private *dev_priv,
 {
 	u32 bit;
 
-	switch (port->port) {
+	switch (port->base.port) {
 	case PORT_A:
 		bit = SDE_PORTA_HOTPLUG_SPT;
 		break;
@@ -4526,7 +4521,7 @@ static bool g4x_digital_port_connected(struct drm_i915_private *dev_priv,
 {
 	u32 bit;
 
-	switch (port->port) {
+	switch (port->base.port) {
 	case PORT_B:
 		bit = PORTB_HOTPLUG_LIVE_STATUS_G4X;
 		break;
@@ -4537,7 +4532,7 @@ static bool g4x_digital_port_connected(struct drm_i915_private *dev_priv,
 		bit = PORTD_HOTPLUG_LIVE_STATUS_G4X;
 		break;
 	default:
-		MISSING_CASE(port->port);
+		MISSING_CASE(port->base.port);
 		return false;
 	}
 
@@ -4549,7 +4544,7 @@ static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv,
 {
 	u32 bit;
 
-	switch (port->port) {
+	switch (port->base.port) {
 	case PORT_B:
 		bit = PORTB_HOTPLUG_LIVE_STATUS_GM45;
 		break;
@@ -4560,7 +4555,7 @@ static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv,
 		bit = PORTD_HOTPLUG_LIVE_STATUS_GM45;
 		break;
 	default:
-		MISSING_CASE(port->port);
+		MISSING_CASE(port->base.port);
 		return false;
 	}
 
@@ -4570,7 +4565,7 @@ static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool ilk_digital_port_connected(struct drm_i915_private *dev_priv,
 				       struct intel_digital_port *port)
 {
-	if (port->port == PORT_A)
+	if (port->base.port == PORT_A)
 		return I915_READ(DEISR) & DE_DP_A_HOTPLUG;
 	else
 		return ibx_digital_port_connected(dev_priv, port);
@@ -4579,7 +4574,7 @@ static bool ilk_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool snb_digital_port_connected(struct drm_i915_private *dev_priv,
 				       struct intel_digital_port *port)
 {
-	if (port->port == PORT_A)
+	if (port->base.port == PORT_A)
 		return I915_READ(DEISR) & DE_DP_A_HOTPLUG;
 	else
 		return cpt_digital_port_connected(dev_priv, port);
@@ -4588,7 +4583,7 @@ static bool snb_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool ivb_digital_port_connected(struct drm_i915_private *dev_priv,
 				       struct intel_digital_port *port)
 {
-	if (port->port == PORT_A)
+	if (port->base.port == PORT_A)
 		return I915_READ(DEISR) & DE_DP_A_HOTPLUG_IVB;
 	else
 		return cpt_digital_port_connected(dev_priv, port);
@@ -4597,7 +4592,7 @@ static bool ivb_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool bdw_digital_port_connected(struct drm_i915_private *dev_priv,
 				       struct intel_digital_port *port)
 {
-	if (port->port == PORT_A)
+	if (port->base.port == PORT_A)
 		return I915_READ(GEN8_DE_PORT_ISR) & GEN8_PORT_DP_A_HOTPLUG;
 	else
 		return cpt_digital_port_connected(dev_priv, port);
@@ -4702,24 +4697,21 @@ intel_dp_unset_edid(struct intel_dp *intel_dp)
 }
 
 static int
-intel_dp_long_pulse(struct intel_connector *intel_connector)
+intel_dp_long_pulse(struct intel_connector *connector)
 {
-	struct drm_connector *connector = &intel_connector->base;
-	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_encoder *intel_encoder = &intel_dig_port->base;
-	struct drm_device *dev = connector->dev;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_dp *intel_dp = intel_attached_dp(&connector->base);
 	enum drm_connector_status status;
 	u8 sink_irq_vector = 0;
 
-	WARN_ON(!drm_modeset_is_locked(&connector->dev->mode_config.connection_mutex));
+	WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
 
-	intel_display_power_get(to_i915(dev), intel_dp->aux_power_domain);
+	intel_display_power_get(dev_priv, intel_dp->aux_power_domain);
 
 	/* Can't disconnect eDP, but you can close the lid... */
 	if (intel_dp_is_edp(intel_dp))
 		status = edp_detect(intel_dp);
-	else if (intel_digital_port_connected(to_i915(dev),
+	else if (intel_digital_port_connected(dev_priv,
 					      dp_to_dig_port(intel_dp)))
 		status = intel_dp_detect_dpcd(intel_dp);
 	else
@@ -4740,9 +4732,6 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 		goto out;
 	}
 
-	if (intel_encoder->type != INTEL_OUTPUT_EDP)
-		intel_encoder->type = INTEL_OUTPUT_DP;
-
 	if (intel_dp->reset_link_params) {
 		/* Initial max link lane count */
 		intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp);
@@ -4793,7 +4782,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 	intel_dp->aux.i2c_defer_count = 0;
 
 	intel_dp_set_edid(intel_dp);
-	if (intel_dp_is_edp(intel_dp) || intel_connector->detect_edid)
+	if (intel_dp_is_edp(intel_dp) || connector->detect_edid)
 		status = connector_status_connected;
 	intel_dp->detect_done = true;
 
@@ -4816,7 +4805,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
 	if (status != connector_status_connected && !intel_dp->is_mst)
 		intel_dp_unset_edid(intel_dp);
 
-	intel_display_power_put(to_i915(dev), intel_dp->aux_power_domain);
+	intel_display_power_put(dev_priv, intel_dp->aux_power_domain);
 	return status;
 }
 
@@ -4832,8 +4821,19 @@ intel_dp_detect(struct drm_connector *connector,
 		      connector->base.id, connector->name);
 
 	/* If full detect is not performed yet, do a full detect */
-	if (!intel_dp->detect_done)
+	if (!intel_dp->detect_done) {
+		struct drm_crtc *crtc;
+		int ret;
+
+		crtc = connector->state->crtc;
+		if (crtc) {
+			ret = drm_modeset_lock(&crtc->mutex, ctx);
+			if (ret)
+				return ret;
+		}
+
 		status = intel_dp_long_pulse(intel_dp->attached_connector);
+	}
 
 	intel_dp->detect_done = false;
 
@@ -4859,9 +4859,6 @@ intel_dp_force(struct drm_connector *connector)
 	intel_dp_set_edid(intel_dp);
 
 	intel_display_power_put(dev_priv, intel_dp->aux_power_domain);
-
-	if (intel_encoder->type != INTEL_OUTPUT_EDP)
-		intel_encoder->type = INTEL_OUTPUT_DP;
 }
 
 static int intel_dp_get_modes(struct drm_connector *connector)
@@ -4986,9 +4983,7 @@ void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder)
 
 static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp)
 {
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -5041,7 +5036,7 @@ void intel_dp_encoder_reset(struct drm_encoder *encoder)
 
 	if (intel_dp_is_edp(intel_dp)) {
 		/* Reinit the power sequencer, in case BIOS did something with it. */
-		intel_dp_pps_init(encoder->dev, intel_dp);
+		intel_dp_pps_init(intel_dp);
 		intel_edp_panel_vdd_sanitize(intel_dp);
 	}
 
@@ -5076,14 +5071,9 @@ enum irqreturn
 intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
 {
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
-	struct drm_device *dev = intel_dig_port->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	enum irqreturn ret = IRQ_NONE;
 
-	if (intel_dig_port->base.type != INTEL_OUTPUT_EDP &&
-	    intel_dig_port->base.type != INTEL_OUTPUT_HDMI)
-		intel_dig_port->base.type = INTEL_OUTPUT_DP;
-
 	if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) {
 		/*
 		 * vdd off can generate a long pulse on eDP which
@@ -5092,12 +5082,12 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
 		 * "vdd off -> long hpd -> vdd on -> detect -> vdd off -> ..."
 		 */
 		DRM_DEBUG_KMS("ignoring long hpd on eDP port %c\n",
-			      port_name(intel_dig_port->port));
+			      port_name(intel_dig_port->base.port));
 		return IRQ_HANDLED;
 	}
 
 	DRM_DEBUG_KMS("got hpd irq on port %c - %s\n",
-		      port_name(intel_dig_port->port),
+		      port_name(intel_dig_port->base.port),
 		      long_hpd ? "long" : "short");
 
 	if (long_hpd) {
@@ -5125,7 +5115,38 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
 	}
 
 	if (!intel_dp->is_mst) {
-		if (!intel_dp_short_pulse(intel_dp)) {
+		struct drm_modeset_acquire_ctx ctx;
+		struct drm_connector *connector = &intel_dp->attached_connector->base;
+		struct drm_crtc *crtc;
+		int iret;
+		bool handled = false;
+
+		drm_modeset_acquire_init(&ctx, 0);
+retry:
+		iret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, &ctx);
+		if (iret)
+			goto err;
+
+		crtc = connector->state->crtc;
+		if (crtc) {
+			iret = drm_modeset_lock(&crtc->mutex, &ctx);
+			if (iret)
+				goto err;
+		}
+
+		handled = intel_dp_short_pulse(intel_dp);
+
+err:
+		if (iret == -EDEADLK) {
+			drm_modeset_backoff(&ctx);
+			goto retry;
+		}
+
+		drm_modeset_drop_locks(&ctx);
+		drm_modeset_acquire_fini(&ctx);
+		WARN(iret, "Acquiring modeset locks failed with %i\n", iret);
+
+		if (!handled) {
 			intel_dp->detect_done = false;
 			goto put_power;
 		}
@@ -5159,8 +5180,11 @@ static void
 intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connector)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
+	enum port port = dp_to_dig_port(intel_dp)->base.port;
+
+	if (!IS_G4X(dev_priv) && port != PORT_A)
+		intel_attach_force_audio_property(connector);
 
-	intel_attach_force_audio_property(connector);
 	intel_attach_broadcast_rgb_property(connector);
 
 	if (intel_dp_is_edp(intel_dp)) {
@@ -5185,13 +5209,13 @@ static void intel_dp_init_panel_power_timestamps(struct intel_dp *intel_dp)
 }
 
 static void
-intel_pps_readout_hw_state(struct drm_i915_private *dev_priv,
-			   struct intel_dp *intel_dp, struct edp_power_seq *seq)
+intel_pps_readout_hw_state(struct intel_dp *intel_dp, struct edp_power_seq *seq)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u32 pp_on, pp_off, pp_div = 0, pp_ctl = 0;
 	struct pps_registers regs;
 
-	intel_pps_get_registers(dev_priv, intel_dp, &regs);
+	intel_pps_get_registers(intel_dp, &regs);
 
 	/* Workaround: Need to write PP_CONTROL with the unlock key as
 	 * the very first thing. */
@@ -5235,13 +5259,12 @@ intel_pps_dump_state(const char *state_name, const struct edp_power_seq *seq)
 }
 
 static void
-intel_pps_verify_state(struct drm_i915_private *dev_priv,
-		       struct intel_dp *intel_dp)
+intel_pps_verify_state(struct intel_dp *intel_dp)
 {
 	struct edp_power_seq hw;
 	struct edp_power_seq *sw = &intel_dp->pps_delays;
 
-	intel_pps_readout_hw_state(dev_priv, intel_dp, &hw);
+	intel_pps_readout_hw_state(intel_dp, &hw);
 
 	if (hw.t1_t3 != sw->t1_t3 || hw.t8 != sw->t8 || hw.t9 != sw->t9 ||
 	    hw.t10 != sw->t10 || hw.t11_t12 != sw->t11_t12) {
@@ -5252,10 +5275,9 @@ intel_pps_verify_state(struct drm_i915_private *dev_priv,
 }
 
 static void
-intel_dp_init_panel_power_sequencer(struct drm_device *dev,
-				    struct intel_dp *intel_dp)
+intel_dp_init_panel_power_sequencer(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	struct edp_power_seq cur, vbt, spec,
 		*final = &intel_dp->pps_delays;
 
@@ -5265,7 +5287,7 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
 	if (final->t11_t12 != 0)
 		return;
 
-	intel_pps_readout_hw_state(dev_priv, intel_dp, &cur);
+	intel_pps_readout_hw_state(intel_dp, &cur);
 
 	intel_pps_dump_state("cur", &cur);
 
@@ -5336,23 +5358,28 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
 	 */
 	final->t8 = 1;
 	final->t9 = 1;
+
+	/*
+	 * HW has only a 100msec granularity for t11_t12 so round it up
+	 * accordingly.
+	 */
+	final->t11_t12 = roundup(final->t11_t12, 100 * 10);
 }
 
 static void
-intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
-					      struct intel_dp *intel_dp,
+intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp,
 					      bool force_disable_vdd)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 	u32 pp_on, pp_off, pp_div, port_sel = 0;
 	int div = dev_priv->rawclk_freq / 1000;
 	struct pps_registers regs;
-	enum port port = dp_to_dig_port(intel_dp)->port;
+	enum port port = dp_to_dig_port(intel_dp)->base.port;
 	const struct edp_power_seq *seq = &intel_dp->pps_delays;
 
 	lockdep_assert_held(&dev_priv->pps_mutex);
 
-	intel_pps_get_registers(dev_priv, intel_dp, &regs);
+	intel_pps_get_registers(intel_dp, &regs);
 
 	/*
 	 * On some VLV machines the BIOS can leave the VDD
@@ -5424,16 +5451,15 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 		      I915_READ(regs.pp_div));
 }
 
-static void intel_dp_pps_init(struct drm_device *dev,
-			      struct intel_dp *intel_dp)
+static void intel_dp_pps_init(struct intel_dp *intel_dp)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		vlv_initial_power_sequencer_setup(intel_dp);
 	} else {
-		intel_dp_init_panel_power_sequencer(dev, intel_dp);
-		intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false);
+		intel_dp_init_panel_power_sequencer(intel_dp);
+		intel_dp_init_panel_power_sequencer_registers(intel_dp, false);
 	}
 }
 
@@ -5472,7 +5498,6 @@ static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv,
 
 	dig_port = dp_to_dig_port(intel_dp);
 	encoder = &dig_port->base;
-	intel_crtc = to_intel_crtc(encoder->base.crtc);
 
 	if (!intel_crtc) {
 		DRM_DEBUG_KMS("DRRS: intel_crtc not initialized\n");
@@ -5545,8 +5570,7 @@ static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv,
 void intel_edp_drrs_enable(struct intel_dp *intel_dp,
 			   const struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	if (!crtc_state->has_drrs) {
 		DRM_DEBUG_KMS("Panel doesn't support DRRS\n");
@@ -5581,8 +5605,7 @@ void intel_edp_drrs_enable(struct intel_dp *intel_dp,
 void intel_edp_drrs_disable(struct intel_dp *intel_dp,
 			    const struct intel_crtc_state *old_crtc_state)
 {
-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
 	if (!old_crtc_state->has_drrs)
 		return;
@@ -5765,7 +5788,7 @@ void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
 
 /**
  * intel_dp_drrs_init - Init basic DRRS work and mutex.
- * @intel_connector: eDP connector
+ * @connector: eDP connector
  * @fixed_mode: preferred mode of panel
  *
  * This function is  called only once at driver load to initialize basic
@@ -5777,12 +5800,10 @@ void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
  * from VBT setting).
  */
 static struct drm_display_mode *
-intel_dp_drrs_init(struct intel_connector *intel_connector,
-		struct drm_display_mode *fixed_mode)
+intel_dp_drrs_init(struct intel_connector *connector,
+		   struct drm_display_mode *fixed_mode)
 {
-	struct drm_connector *connector = &intel_connector->base;
-	struct drm_device *dev = connector->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct drm_display_mode *downclock_mode = NULL;
 
 	INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work);
@@ -5798,8 +5819,8 @@ intel_dp_drrs_init(struct intel_connector *intel_connector,
 		return NULL;
 	}
 
-	downclock_mode = intel_find_panel_downclock
-					(dev_priv, fixed_mode, connector);
+	downclock_mode = intel_find_panel_downclock(dev_priv, fixed_mode,
+						    &connector->base);
 
 	if (!downclock_mode) {
 		DRM_DEBUG_KMS("Downclock mode is not found. DRRS not supported\n");
@@ -5816,11 +5837,9 @@ intel_dp_drrs_init(struct intel_connector *intel_connector,
 static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 				     struct intel_connector *intel_connector)
 {
-	struct drm_connector *connector = &intel_connector->base;
-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-	struct intel_encoder *intel_encoder = &intel_dig_port->base;
-	struct drm_device *dev = intel_encoder->base.dev;
+	struct drm_device *dev = intel_dp_to_dev(intel_dp);
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_connector *connector = &intel_connector->base;
 	struct drm_display_mode *fixed_mode = NULL;
 	struct drm_display_mode *alt_fixed_mode = NULL;
 	struct drm_display_mode *downclock_mode = NULL;
@@ -5838,7 +5857,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	 * eDP and LVDS bail out early in this case to prevent interfering
 	 * with an already powered-on LVDS power sequencer.
 	 */
-	if (intel_get_lvds_encoder(dev)) {
+	if (intel_get_lvds_encoder(&dev_priv->drm)) {
 		WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)));
 		DRM_INFO("LVDS was detected, not registering eDP\n");
 
@@ -5848,7 +5867,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	pps_lock(intel_dp);
 
 	intel_dp_init_panel_power_timestamps(intel_dp);
-	intel_dp_pps_init(dev, intel_dp);
+	intel_dp_pps_init(intel_dp);
 	intel_edp_panel_vdd_sanitize(intel_dp);
 
 	pps_unlock(intel_dp);
@@ -5868,7 +5887,6 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 		if (drm_add_edid_modes(connector, edid)) {
 			drm_mode_connector_update_edid_property(connector,
 								edid);
-			drm_edid_to_eld(connector, edid);
 		} else {
 			kfree(edid);
 			edid = ERR_PTR(-EINVAL);
@@ -5949,9 +5967,9 @@ intel_dp_init_connector_port_info(struct intel_digital_port *intel_dig_port)
 	struct intel_encoder *encoder = &intel_dig_port->base;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 
-	encoder->hpd_pin = intel_hpd_pin(intel_dig_port->port);
+	encoder->hpd_pin = intel_hpd_pin(encoder->port);
 
-	switch (intel_dig_port->port) {
+	switch (encoder->port) {
 	case PORT_A:
 		intel_dp->aux_power_domain = POWER_DOMAIN_AUX_A;
 		break;
@@ -5969,7 +5987,7 @@ intel_dp_init_connector_port_info(struct intel_digital_port *intel_dig_port)
 		intel_dp->aux_power_domain = POWER_DOMAIN_AUX_D;
 		break;
 	default:
-		MISSING_CASE(intel_dig_port->port);
+		MISSING_CASE(encoder->port);
 	}
 }
 
@@ -6005,7 +6023,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	struct intel_encoder *intel_encoder = &intel_dig_port->base;
 	struct drm_device *dev = intel_encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_encoder->port;
 	int type;
 
 	/* Initialize the work for modeset in case of link train failure */
@@ -6074,7 +6092,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
 	drm_connector_init(dev, connector, &intel_dp_connector_funcs, type);
 	drm_connector_helper_add(connector, &intel_dp_connector_helper_funcs);
 
-	connector->interlace_allowed = true;
+	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
+		connector->interlace_allowed = true;
 	connector->doublescan_allowed = 0;
 
 	intel_dp_init_connector_port_info(intel_dig_port);
@@ -6175,7 +6194,6 @@ bool intel_dp_init(struct drm_i915_private *dev_priv,
 		intel_encoder->disable = g4x_disable_dp;
 	}
 
-	intel_dig_port->port = port;
 	intel_dig_port->dp.output_reg = output_reg;
 	intel_dig_port->max_lanes = 4;
 
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index 05907fa8a553..cf8fef8b6f58 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -328,14 +328,22 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
 	return;
 
  failure_handling:
-	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Link Training failed at link rate = %d, lane count = %d",
-		      intel_connector->base.base.id,
-		      intel_connector->base.name,
-		      intel_dp->link_rate, intel_dp->lane_count);
-	if (!intel_dp_get_link_train_fallback_values(intel_dp,
-						     intel_dp->link_rate,
-						     intel_dp->lane_count))
-		/* Schedule a Hotplug Uevent to userspace to start modeset */
-		schedule_work(&intel_connector->modeset_retry_work);
+	/* Dont fallback and prune modes if its eDP */
+	if (!intel_dp_is_edp(intel_dp)) {
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Link Training failed at link rate = %d, lane count = %d",
+			      intel_connector->base.base.id,
+			      intel_connector->base.name,
+			      intel_dp->link_rate, intel_dp->lane_count);
+		if (!intel_dp_get_link_train_fallback_values(intel_dp,
+							     intel_dp->link_rate,
+							     intel_dp->lane_count))
+			/* Schedule a Hotplug Uevent to userspace to start modeset */
+			schedule_work(&intel_connector->modeset_retry_work);
+	} else {
+		DRM_ERROR("[CONNECTOR:%d:%s] Link Training failed at link rate = %d, lane count = %d",
+			  intel_connector->base.base.id,
+			  intel_connector->base.name,
+			  intel_dp->link_rate, intel_dp->lane_count);
+	}
 	return;
 }
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 772521440a9f..c3de0918ee13 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -34,6 +34,7 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 					struct intel_crtc_state *pipe_config,
 					struct drm_connector_state *conn_state)
 {
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
@@ -87,6 +88,12 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
 	pipe_config->dp_m_n.tu = slots;
 
+	if (IS_GEN9_LP(dev_priv))
+		pipe_config->lane_lat_optim_mask =
+			bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
+
+	intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
+
 	return true;
 }
 
@@ -142,7 +149,8 @@ static void intel_mst_disable_dp(struct intel_encoder *encoder,
 		DRM_ERROR("failed to update payload %d\n", ret);
 	}
 	if (old_crtc_state->has_audio)
-		intel_audio_codec_disable(encoder);
+		intel_audio_codec_disable(encoder,
+					  old_crtc_state, old_conn_state);
 }
 
 static void intel_mst_post_disable_dp(struct intel_encoder *encoder,
@@ -172,13 +180,27 @@ static void intel_mst_post_disable_dp(struct intel_encoder *encoder,
 	intel_dp->active_mst_links--;
 
 	intel_mst->connector = NULL;
-	if (intel_dp->active_mst_links == 0) {
+	if (intel_dp->active_mst_links == 0)
 		intel_dig_port->base.post_disable(&intel_dig_port->base,
-						  NULL, NULL);
-	}
+						  old_crtc_state, NULL);
+
 	DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
 }
 
+static void intel_mst_pre_pll_enable_dp(struct intel_encoder *encoder,
+					const struct intel_crtc_state *pipe_config,
+					const struct drm_connector_state *conn_state)
+{
+	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
+	struct intel_digital_port *intel_dig_port = intel_mst->primary;
+	struct intel_dp *intel_dp = &intel_dig_port->dp;
+
+	if (intel_dp->active_mst_links == 0 &&
+	    intel_dig_port->base.pre_pll_enable)
+		intel_dig_port->base.pre_pll_enable(&intel_dig_port->base,
+						    pipe_config, NULL);
+}
+
 static void intel_mst_pre_enable_dp(struct intel_encoder *encoder,
 				    const struct intel_crtc_state *pipe_config,
 				    const struct drm_connector_state *conn_state)
@@ -187,7 +209,7 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder,
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
 	int ret;
@@ -231,7 +253,7 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder,
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &intel_dig_port->dp;
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_dig_port->base.port;
 	int ret;
 
 	DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
@@ -265,48 +287,8 @@ static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder,
 {
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
 	struct intel_digital_port *intel_dig_port = intel_mst->primary;
-	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
-	u32 temp, flags = 0;
-
-	pipe_config->has_audio =
-		intel_ddi_is_audio_enabled(dev_priv, crtc);
-
-	temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
-	if (temp & TRANS_DDI_PHSYNC)
-		flags |= DRM_MODE_FLAG_PHSYNC;
-	else
-		flags |= DRM_MODE_FLAG_NHSYNC;
-	if (temp & TRANS_DDI_PVSYNC)
-		flags |= DRM_MODE_FLAG_PVSYNC;
-	else
-		flags |= DRM_MODE_FLAG_NVSYNC;
-
-	switch (temp & TRANS_DDI_BPC_MASK) {
-	case TRANS_DDI_BPC_6:
-		pipe_config->pipe_bpp = 18;
-		break;
-	case TRANS_DDI_BPC_8:
-		pipe_config->pipe_bpp = 24;
-		break;
-	case TRANS_DDI_BPC_10:
-		pipe_config->pipe_bpp = 30;
-		break;
-	case TRANS_DDI_BPC_12:
-		pipe_config->pipe_bpp = 36;
-		break;
-	default:
-		break;
-	}
-	pipe_config->base.adjusted_mode.flags |= flags;
-
-	pipe_config->lane_count =
-		((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
-
-	intel_dp_get_m_n(crtc, pipe_config);
 
-	intel_ddi_clock_get(&intel_dig_port->base, pipe_config);
+	intel_ddi_get_config(&intel_dig_port->base, pipe_config);
 }
 
 static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector)
@@ -570,13 +552,14 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum
 
 	intel_encoder->type = INTEL_OUTPUT_DP_MST;
 	intel_encoder->power_domain = intel_dig_port->base.power_domain;
-	intel_encoder->port = intel_dig_port->port;
+	intel_encoder->port = intel_dig_port->base.port;
 	intel_encoder->crtc_mask = 0x7;
 	intel_encoder->cloneable = 0;
 
 	intel_encoder->compute_config = intel_dp_mst_compute_config;
 	intel_encoder->disable = intel_mst_disable_dp;
 	intel_encoder->post_disable = intel_mst_post_disable_dp;
+	intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp;
 	intel_encoder->pre_enable = intel_mst_pre_enable_dp;
 	intel_encoder->enable = intel_mst_enable_dp;
 	intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state;
diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c
index de38d014ed39..76473e9836c6 100644
--- a/drivers/gpu/drm/i915/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/intel_dpio_phy.c
@@ -466,21 +466,21 @@ void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 
 	lockdep_assert_held(&dev_priv->power_domains.lock);
 
-	if (rcomp_phy != -1) {
+	was_enabled = true;
+	if (rcomp_phy != -1)
 		was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy);
 
-		/*
-		 * We need to copy the GRC calibration value from rcomp_phy,
-		 * so make sure it's powered up.
-		 */
-		if (!was_enabled)
-			_bxt_ddi_phy_init(dev_priv, rcomp_phy);
-	}
+	/*
+	 * We need to copy the GRC calibration value from rcomp_phy,
+	 * so make sure it's powered up.
+	 */
+	if (!was_enabled)
+		_bxt_ddi_phy_init(dev_priv, rcomp_phy);
 
 	_bxt_ddi_phy_init(dev_priv, phy);
 
-	if (rcomp_phy != -1 && !was_enabled)
-		bxt_ddi_phy_uninit(dev_priv, phy_info->rcomp_phy);
+	if (!was_enabled)
+		bxt_ddi_phy_uninit(dev_priv, rcomp_phy);
 }
 
 static bool __printf(6, 7)
@@ -567,8 +567,7 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
 }
 
 uint8_t
-bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
-				     uint8_t lane_count)
+bxt_ddi_phy_calc_lane_lat_optim_mask(uint8_t lane_count)
 {
 	switch (lane_count) {
 	case 1:
@@ -587,9 +586,8 @@ bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
 void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
 				     uint8_t lane_lat_optim_mask)
 {
-	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
-	enum port port = dport->port;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
 	enum dpio_phy phy;
 	enum dpio_channel ch;
 	int lane;
@@ -614,9 +612,8 @@ void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
 uint8_t
 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
 {
-	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
-	enum port port = dport->port;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum port port = encoder->port;
 	enum dpio_phy phy;
 	enum dpio_channel ch;
 	int lane;
@@ -642,7 +639,7 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct intel_crtc *intel_crtc = to_intel_crtc(dport->base.base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
 	enum dpio_channel ch = vlv_dport_to_channel(dport);
 	enum pipe pipe = intel_crtc->pipe;
 	u32 val;
@@ -734,11 +731,12 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 }
 
 void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+			      const struct intel_crtc_state *crtc_state,
 			      bool reset)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	enum pipe pipe = crtc->pipe;
 	uint32_t val;
 
@@ -777,17 +775,16 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder,
 	}
 }
 
-void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
+void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
+			    const struct intel_crtc_state *crtc_state)
 {
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc =
-		to_intel_crtc(encoder->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	enum dpio_channel ch = vlv_dport_to_channel(dport);
-	enum pipe pipe = intel_crtc->pipe;
+	enum pipe pipe = crtc->pipe;
 	unsigned int lane_mask =
-		intel_dp_unused_lane_mask(intel_crtc->config->lane_count);
+		intel_dp_unused_lane_mask(crtc_state->lane_count);
 	u32 val;
 
 	/*
@@ -803,7 +800,7 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
 	mutex_lock(&dev_priv->sb_lock);
 
 	/* Assert data lane reset */
-	chv_data_lane_soft_reset(encoder, true);
+	chv_data_lane_soft_reset(encoder, crtc_state, true);
 
 	/* program left/right clock distribution */
 	if (pipe != PIPE_B) {
@@ -833,7 +830,7 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
 		val |= CHV_PCS_USEDCLKCHANNEL;
 	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val);
 
-	if (intel_crtc->config->lane_count > 2) {
+	if (crtc_state->lane_count > 2) {
 		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
 		val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
 		if (pipe != PIPE_B)
@@ -858,16 +855,15 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
 	mutex_unlock(&dev_priv->sb_lock);
 }
 
-void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
+void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+				const struct intel_crtc_state *crtc_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc =
-		to_intel_crtc(encoder->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	enum dpio_channel ch = vlv_dport_to_channel(dport);
-	int pipe = intel_crtc->pipe;
+	enum pipe pipe = crtc->pipe;
 	int data, i, stagger;
 	u32 val;
 
@@ -878,16 +874,16 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
 	val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
 	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
 
-	if (intel_crtc->config->lane_count > 2) {
+	if (crtc_state->lane_count > 2) {
 		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
 		val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
 		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
 	}
 
 	/* Program Tx lane latency optimal setting*/
-	for (i = 0; i < intel_crtc->config->lane_count; i++) {
+	for (i = 0; i < crtc_state->lane_count; i++) {
 		/* Set the upar bit */
-		if (intel_crtc->config->lane_count == 1)
+		if (crtc_state->lane_count == 1)
 			data = 0x0;
 		else
 			data = (i == 1) ? 0x0 : 0x1;
@@ -896,13 +892,13 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
 	}
 
 	/* Data lane stagger programming */
-	if (intel_crtc->config->port_clock > 270000)
+	if (crtc_state->port_clock > 270000)
 		stagger = 0x18;
-	else if (intel_crtc->config->port_clock > 135000)
+	else if (crtc_state->port_clock > 135000)
 		stagger = 0xd;
-	else if (intel_crtc->config->port_clock > 67500)
+	else if (crtc_state->port_clock > 67500)
 		stagger = 0x7;
-	else if (intel_crtc->config->port_clock > 33750)
+	else if (crtc_state->port_clock > 33750)
 		stagger = 0x4;
 	else
 		stagger = 0x2;
@@ -911,7 +907,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
 	val |= DPIO_TX2_STAGGER_MASK(0x1f);
 	vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
 
-	if (intel_crtc->config->lane_count > 2) {
+	if (crtc_state->lane_count > 2) {
 		val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
 		val |= DPIO_TX2_STAGGER_MASK(0x1f);
 		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
@@ -924,7 +920,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
 		       DPIO_TX1_STAGGER_MULT(6) |
 		       DPIO_TX2_STAGGER_MULT(0));
 
-	if (intel_crtc->config->lane_count > 2) {
+	if (crtc_state->lane_count > 2) {
 		vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
 			       DPIO_LANESTAGGER_STRAP(stagger) |
 			       DPIO_LANESTAGGER_STRAP_OVRD |
@@ -934,7 +930,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
 	}
 
 	/* Deassert data lane reset */
-	chv_data_lane_soft_reset(encoder, false);
+	chv_data_lane_soft_reset(encoder, crtc_state, false);
 
 	mutex_unlock(&dev_priv->sb_lock);
 }
@@ -950,10 +946,11 @@ void chv_phy_release_cl2_override(struct intel_encoder *encoder)
 	}
 }
 
-void chv_phy_post_pll_disable(struct intel_encoder *encoder)
+void chv_phy_post_pll_disable(struct intel_encoder *encoder,
+			      const struct intel_crtc_state *old_crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
+	enum pipe pipe = to_intel_crtc(old_crtc_state->base.crtc)->pipe;
 	u32 val;
 
 	mutex_lock(&dev_priv->sb_lock);
@@ -991,7 +988,7 @@ void vlv_set_phy_signal_level(struct intel_encoder *encoder,
 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
 	enum dpio_channel port = vlv_dport_to_channel(dport);
-	int pipe = intel_crtc->pipe;
+	enum pipe pipe = intel_crtc->pipe;
 
 	mutex_lock(&dev_priv->sb_lock);
 	vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000);
@@ -1009,15 +1006,14 @@ void vlv_set_phy_signal_level(struct intel_encoder *encoder,
 	mutex_unlock(&dev_priv->sb_lock);
 }
 
-void vlv_phy_pre_pll_enable(struct intel_encoder *encoder)
+void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
+			    const struct intel_crtc_state *crtc_state)
 {
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc =
-		to_intel_crtc(encoder->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	enum dpio_channel port = vlv_dport_to_channel(dport);
-	int pipe = intel_crtc->pipe;
+	enum pipe pipe = crtc->pipe;
 
 	/* Program Tx lane resets to default */
 	mutex_lock(&dev_priv->sb_lock);
@@ -1037,15 +1033,15 @@ void vlv_phy_pre_pll_enable(struct intel_encoder *encoder)
 	mutex_unlock(&dev_priv->sb_lock);
 }
 
-void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder)
+void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+				const struct intel_crtc_state *crtc_state)
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 	struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	enum dpio_channel port = vlv_dport_to_channel(dport);
-	int pipe = intel_crtc->pipe;
+	enum pipe pipe = crtc->pipe;
 	u32 val;
 
 	mutex_lock(&dev_priv->sb_lock);
@@ -1067,14 +1063,14 @@ void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder)
 	mutex_unlock(&dev_priv->sb_lock);
 }
 
-void vlv_phy_reset_lanes(struct intel_encoder *encoder)
+void vlv_phy_reset_lanes(struct intel_encoder *encoder,
+			 const struct intel_crtc_state *old_crtc_state)
 {
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *intel_crtc =
-		to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
 	enum dpio_channel port = vlv_dport_to_channel(dport);
-	int pipe = intel_crtc->pipe;
+	enum pipe pipe = crtc->pipe;
 
 	mutex_lock(&dev_priv->sb_lock);
 	vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000);
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index df808a94c511..51c5ae4e9116 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -813,15 +813,11 @@ hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	memset(&crtc_state->dpll_hw_state, 0,
 	       sizeof(crtc_state->dpll_hw_state));
 
-	if (encoder->type == INTEL_OUTPUT_HDMI) {
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
 		pll = hsw_ddi_hdmi_get_dpll(clock, crtc, crtc_state);
-
-	} else if (encoder->type == INTEL_OUTPUT_DP ||
-		   encoder->type == INTEL_OUTPUT_DP_MST ||
-		   encoder->type == INTEL_OUTPUT_EDP) {
+	} else if (intel_crtc_has_dp_encoder(crtc_state)) {
 		pll = hsw_ddi_dp_get_dpll(encoder, clock);
-
-	} else if (encoder->type == INTEL_OUTPUT_ANALOG) {
+	} else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
 		if (WARN_ON(crtc_state->port_clock / 2 != 135000))
 			return NULL;
 
@@ -1369,15 +1365,13 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 
 	memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
-	if (encoder->type == INTEL_OUTPUT_HDMI) {
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
 		bret = skl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
 			return NULL;
 		}
-	} else if (encoder->type == INTEL_OUTPUT_DP ||
-		   encoder->type == INTEL_OUTPUT_DP_MST ||
-		   encoder->type == INTEL_OUTPUT_EDP) {
+	} else if (intel_crtc_has_dp_encoder(crtc_state)) {
 		bret = skl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
@@ -1388,7 +1382,7 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 		return NULL;
 	}
 
-	if (encoder->type == INTEL_OUTPUT_EDP)
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
 		pll = intel_find_shared_dpll(crtc, crtc_state,
 					     DPLL_ID_SKL_DPLL0,
 					     DPLL_ID_SKL_DPLL0);
@@ -1808,18 +1802,15 @@ bxt_get_dpll(struct intel_crtc *crtc,
 {
 	struct intel_dpll_hw_state dpll_hw_state = { };
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_digital_port *intel_dig_port;
 	struct intel_shared_dpll *pll;
 	int i, clock = crtc_state->port_clock;
 
-	if (encoder->type == INTEL_OUTPUT_HDMI &&
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) &&
 	    !bxt_ddi_hdmi_set_dpll_hw_state(crtc, crtc_state, clock,
 					    &dpll_hw_state))
 		return NULL;
 
-	if ((encoder->type == INTEL_OUTPUT_DP ||
-	     encoder->type == INTEL_OUTPUT_EDP ||
-	     encoder->type == INTEL_OUTPUT_DP_MST) &&
+	if (intel_crtc_has_dp_encoder(crtc_state) &&
 	    !bxt_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state))
 		return NULL;
 
@@ -1828,15 +1819,8 @@ bxt_get_dpll(struct intel_crtc *crtc,
 
 	crtc_state->dpll_hw_state = dpll_hw_state;
 
-	if (encoder->type == INTEL_OUTPUT_DP_MST) {
-		struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
-
-		intel_dig_port = intel_mst->primary;
-	} else
-		intel_dig_port = enc_to_dig_port(&encoder->base);
-
 	/* 1:1 mapping between ports and PLLs */
-	i = (enum intel_dpll_id) intel_dig_port->port;
+	i = (enum intel_dpll_id) encoder->port;
 	pll = intel_get_shared_dpll_by_id(dev_priv, i);
 
 	DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n",
@@ -2008,8 +1992,8 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	 * requirement, follow the Display Voltage Frequency Switching
 	 * Sequence Before Frequency Change
 	 *
-	 * FIXME: (DVFS) is used to adjust the display voltage to match the
-	 * display clock frequencies
+	 * Note: DVFS is actually handled via the cdclk code paths,
+	 * hence we do nothing here.
 	 */
 
 	/* 6. Enable DPLL in DPLL_ENABLE. */
@@ -2030,8 +2014,8 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	 * requirement, follow the Display Voltage Frequency Switching
 	 * Sequence After Frequency Change
 	 *
-	 * FIXME: (DVFS) is used to adjust the display voltage to match the
-	 * display clock frequencies
+	 * Note: DVFS is actually handled via the cdclk code paths,
+	 * hence we do nothing here.
 	 */
 
 	/*
@@ -2055,8 +2039,8 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 	 * requirement, follow the Display Voltage Frequency Switching
 	 * Sequence Before Frequency Change
 	 *
-	 * FIXME: (DVFS) is used to adjust the display voltage to match the
-	 * display clock frequencies
+	 * Note: DVFS is actually handled via the cdclk code paths,
+	 * hence we do nothing here.
 	 */
 
 	/* 3. Disable DPLL through DPLL_ENABLE. */
@@ -2077,8 +2061,8 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 	 * requirement, follow the Display Voltage Frequency Switching
 	 * Sequence After Frequency Change
 	 *
-	 * FIXME: (DVFS) is used to adjust the display voltage to match the
-	 * display clock frequencies
+	 * Note: DVFS is actually handled via the cdclk code paths,
+	 * hence we do nothing here.
 	 */
 
 	/* 6. Disable DPLL power in DPLL_ENABLE. */
@@ -2126,10 +2110,8 @@ static bool cnl_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv,
 	return ret;
 }
 
-static void cnl_wrpll_get_multipliers(unsigned int bestdiv,
-				      unsigned int *pdiv,
-				      unsigned int *qdiv,
-				      unsigned int *kdiv)
+static void cnl_wrpll_get_multipliers(int bestdiv, int *pdiv,
+				      int *qdiv, int *kdiv)
 {
 	/* even dividers */
 	if (bestdiv % 2 == 0) {
@@ -2167,10 +2149,12 @@ static void cnl_wrpll_get_multipliers(unsigned int bestdiv,
 	}
 }
 
-static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, uint32_t dco_freq,
-				      uint32_t ref_freq, uint32_t pdiv, uint32_t qdiv,
-				      uint32_t kdiv)
+static void cnl_wrpll_params_populate(struct skl_wrpll_params *params,
+				      u32 dco_freq, u32 ref_freq,
+				      int pdiv, int qdiv, int kdiv)
 {
+	u32 dco;
+
 	switch (kdiv) {
 	case 1:
 		params->kdiv = 1;
@@ -2202,39 +2186,35 @@ static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, uint32_t
 		WARN(1, "Incorrect PDiv\n");
 	}
 
-	if (kdiv != 2)
-		qdiv = 1;
+	WARN_ON(kdiv != 2 && qdiv != 1);
 
 	params->qdiv_ratio = qdiv;
 	params->qdiv_mode = (qdiv == 1) ? 0 : 1;
 
-	params->dco_integer = div_u64(dco_freq, ref_freq);
-	params->dco_fraction = div_u64((div_u64((uint64_t)dco_freq<<15, (uint64_t)ref_freq) -
-					((uint64_t)params->dco_integer<<15)) * 0x8000, 0x8000);
+	dco = div_u64((u64)dco_freq << 15, ref_freq);
+
+	params->dco_integer = dco >> 15;
+	params->dco_fraction = dco & 0x7fff;
 }
 
 static bool
-cnl_ddi_calculate_wrpll(int clock /* in Hz */,
+cnl_ddi_calculate_wrpll(int clock,
 			struct drm_i915_private *dev_priv,
 			struct skl_wrpll_params *wrpll_params)
 {
-	uint64_t afe_clock = clock * 5 / KHz(1); /* clocks in kHz */
-	unsigned int dco_min = 7998 * KHz(1);
-	unsigned int dco_max = 10000 * KHz(1);
-	unsigned int dco_mid = (dco_min + dco_max) / 2;
-
+	u32 afe_clock = clock * 5;
+	u32 dco_min = 7998000;
+	u32 dco_max = 10000000;
+	u32 dco_mid = (dco_min + dco_max) / 2;
 	static const int dividers[] = {  2,  4,  6,  8, 10, 12,  14,  16,
 					 18, 20, 24, 28, 30, 32,  36,  40,
 					 42, 44, 48, 50, 52, 54,  56,  60,
 					 64, 66, 68, 70, 72, 76,  78,  80,
 					 84, 88, 90, 92, 96, 98, 100, 102,
 					  3,  5,  7,  9, 15, 21 };
-	unsigned int d, dco;
-	unsigned int dco_centrality = 0;
-	unsigned int best_dco_centrality = 999999;
-	unsigned int best_div = 0;
-	unsigned int best_dco = 0;
-	unsigned int pdiv = 0, qdiv = 0, kdiv = 0;
+	u32 dco, best_dco = 0, dco_centrality = 0;
+	u32 best_dco_centrality = U32_MAX; /* Spec meaning of 999999 MHz */
+	int d, best_div = 0, pdiv = 0, qdiv = 0, kdiv = 0;
 
 	for (d = 0; d < ARRAY_SIZE(dividers); d++) {
 		dco = afe_clock * dividers[d];
@@ -2271,7 +2251,7 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
 
 	cfgcr0 = DPLL_CFGCR0_HDMI_MODE;
 
-	if (!cnl_ddi_calculate_wrpll(clock * 1000, dev_priv, &wrpll_params))
+	if (!cnl_ddi_calculate_wrpll(clock, dev_priv, &wrpll_params))
 		return false;
 
 	cfgcr0 |= DPLL_CFGCR0_DCO_FRACTION(wrpll_params.dco_fraction) |
@@ -2281,7 +2261,6 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
 		DPLL_CFGCR1_QDIV_MODE(wrpll_params.qdiv_mode) |
 		DPLL_CFGCR1_KDIV(wrpll_params.kdiv) |
 		DPLL_CFGCR1_PDIV(wrpll_params.pdiv) |
-		wrpll_params.central_freq |
 		DPLL_CFGCR1_CENTRAL_FREQ;
 
 	memset(&crtc_state->dpll_hw_state, 0,
@@ -2345,15 +2324,13 @@ cnl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 
 	memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
-	if (encoder->type == INTEL_OUTPUT_HDMI) {
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
 		bret = cnl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
 			return NULL;
 		}
-	} else if (encoder->type == INTEL_OUTPUT_DP ||
-		   encoder->type == INTEL_OUTPUT_DP_MST ||
-		   encoder->type == INTEL_OUTPUT_EDP) {
+	} else if (intel_crtc_has_dp_encoder(crtc_state)) {
 		bret = cnl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
@@ -2361,8 +2338,8 @@ cnl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 		}
 		crtc_state->dpll_hw_state = dpll_hw_state;
 	} else {
-		DRM_DEBUG_KMS("Skip DPLL setup for encoder %d\n",
-			      encoder->type);
+		DRM_DEBUG_KMS("Skip DPLL setup for output_types 0x%x\n",
+			      crtc_state->output_types);
 		return NULL;
 	}
 
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 5d77f75a9f9c..30f791f89d64 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -47,14 +47,12 @@
  * contexts. Note that it's important that we check the condition again after
  * having timed out, since the timeout could be due to preemption or similar and
  * we've never had a chance to check the condition before the timeout.
- *
- * TODO: When modesetting has fully transitioned to atomic, the below
- * drm_can_sleep() can be removed and in_atomic()/!in_atomic() asserts
- * added.
  */
-#define _wait_for(COND, US, W) ({ \
+#define _wait_for(COND, US, Wmin, Wmax) ({ \
 	unsigned long timeout__ = jiffies + usecs_to_jiffies(US) + 1;	\
+	long wait__ = (Wmin); /* recommended min for usleep is 10 us */	\
 	int ret__;							\
+	might_sleep();							\
 	for (;;) {							\
 		bool expired__ = time_after(jiffies, timeout__);	\
 		if (COND) {						\
@@ -65,16 +63,14 @@
 			ret__ = -ETIMEDOUT;				\
 			break;						\
 		}							\
-		if ((W) && drm_can_sleep()) {				\
-			usleep_range((W), (W)*2);			\
-		} else {						\
-			cpu_relax();					\
-		}							\
+		usleep_range(wait__, wait__ * 2);			\
+		if (wait__ < (Wmax))					\
+			wait__ <<= 1;					\
 	}								\
 	ret__;								\
 })
 
-#define wait_for(COND, MS)	  	_wait_for((COND), (MS) * 1000, 1000)
+#define wait_for(COND, MS)	_wait_for((COND), (MS) * 1000, 10, 1000)
 
 /* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */
 #if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT)
@@ -123,7 +119,7 @@
 	int ret__; \
 	BUILD_BUG_ON(!__builtin_constant_p(US)); \
 	if ((US) > 10) \
-		ret__ = _wait_for((COND), (US), 10); \
+		ret__ = _wait_for((COND), (US), 10, 10); \
 	else \
 		ret__ = _wait_for_atomic((COND), (US), 0); \
 	ret__; \
@@ -173,7 +169,7 @@ enum intel_output_type {
 	INTEL_OUTPUT_DP = 7,
 	INTEL_OUTPUT_EDP = 8,
 	INTEL_OUTPUT_DSI = 9,
-	INTEL_OUTPUT_UNKNOWN = 10,
+	INTEL_OUTPUT_DDI = 10,
 	INTEL_OUTPUT_DP_MST = 11,
 };
 
@@ -216,6 +212,9 @@ struct intel_encoder {
 	enum port port;
 	unsigned int cloneable;
 	void (*hot_plug)(struct intel_encoder *);
+	enum intel_output_type (*compute_output_type)(struct intel_encoder *,
+						      struct intel_crtc_state *,
+						      struct drm_connector_state *);
 	bool (*compute_config)(struct intel_encoder *,
 			       struct intel_crtc_state *,
 			       struct drm_connector_state *);
@@ -386,6 +385,8 @@ struct intel_atomic_state {
 	unsigned int active_crtcs;
 	/* minimum acceptable cdclk for each pipe */
 	int min_cdclk[I915_MAX_PIPES];
+	/* minimum acceptable voltage level for each pipe */
+	u8 min_voltage_level[I915_MAX_PIPES];
 
 	struct intel_shared_dpll_state shared_dpll[I915_NUM_PLLS];
 
@@ -420,6 +421,9 @@ struct intel_plane_state {
 	/* plane control register */
 	u32 ctl;
 
+	/* plane color control register */
+	u32 color_ctl;
+
 	/*
 	 * scaler_id
 	 *    = -1 : not using a scaler
@@ -738,6 +742,9 @@ struct intel_crtc_state {
 	 */
 	uint8_t lane_lat_optim_mask;
 
+	/* minimum acceptable voltage level */
+	u8 min_voltage_level;
+
 	/* Panel fitter controls for gen2-gen4 + VLV */
 	struct {
 		u32 control;
@@ -795,7 +802,6 @@ struct intel_crtc_state {
 struct intel_crtc {
 	struct drm_crtc base;
 	enum pipe pipe;
-	enum plane plane;
 	/*
 	 * Whether the crtc and the connected output pipeline is active. Implies
 	 * that crtc->enabled is set, i.e. the current mode configuration has
@@ -840,7 +846,7 @@ struct intel_crtc {
 
 struct intel_plane {
 	struct drm_plane base;
-	u8 plane;
+	enum i9xx_plane_id i9xx_plane;
 	enum plane_id id;
 	enum pipe pipe;
 	bool can_scale;
@@ -1049,7 +1055,6 @@ struct intel_lspcon {
 
 struct intel_digital_port {
 	struct intel_encoder base;
-	enum port port;
 	u32 saved_port_bits;
 	struct intel_dp dp;
 	struct intel_hdmi hdmi;
@@ -1081,7 +1086,7 @@ struct intel_dp_mst_encoder {
 static inline enum dpio_channel
 vlv_dport_to_channel(struct intel_digital_port *dport)
 {
-	switch (dport->port) {
+	switch (dport->base.port) {
 	case PORT_B:
 	case PORT_D:
 		return DPIO_CH0;
@@ -1095,7 +1100,7 @@ vlv_dport_to_channel(struct intel_digital_port *dport)
 static inline enum dpio_phy
 vlv_dport_to_phy(struct intel_digital_port *dport)
 {
-	switch (dport->port) {
+	switch (dport->base.port) {
 	case PORT_B:
 	case PORT_C:
 		return DPIO_PHY0;
@@ -1127,7 +1132,7 @@ intel_get_crtc_for_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 }
 
 static inline struct intel_crtc *
-intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum plane plane)
+intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum i9xx_plane_id plane)
 {
 	return dev_priv->plane_to_crtc_mapping[plane];
 }
@@ -1148,7 +1153,7 @@ enc_to_dig_port(struct drm_encoder *encoder)
 	struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
 
 	switch (intel_encoder->type) {
-	case INTEL_OUTPUT_UNKNOWN:
+	case INTEL_OUTPUT_DDI:
 		WARN_ON(!HAS_DDI(to_i915(encoder->dev)));
 	case INTEL_OUTPUT_DP:
 	case INTEL_OUTPUT_EDP:
@@ -1272,7 +1277,6 @@ void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder,
 void hsw_fdi_link_train(struct intel_crtc *crtc,
 			const struct intel_crtc_state *crtc_state);
 void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port);
-enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder);
 bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe);
 void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state);
 void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv,
@@ -1284,15 +1288,13 @@ intel_ddi_get_crtc_new_encoder(struct intel_crtc_state *crtc_state);
 void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state);
 void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp);
 bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
-bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
-				 struct intel_crtc *intel_crtc);
 void intel_ddi_get_config(struct intel_encoder *encoder,
 			  struct intel_crtc_state *pipe_config);
 
-void intel_ddi_clock_get(struct intel_encoder *encoder,
-			 struct intel_crtc_state *pipe_config);
 void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
 				    bool state);
+void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv,
+					 struct intel_crtc_state *crtc_state);
 u32 bxt_signal_levels(struct intel_dp *intel_dp);
 uint32_t ddi_signal_levels(struct intel_dp *intel_dp);
 u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder);
@@ -1305,7 +1307,9 @@ void intel_init_audio_hooks(struct drm_i915_private *dev_priv);
 void intel_audio_codec_enable(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *crtc_state,
 			      const struct drm_connector_state *conn_state);
-void intel_audio_codec_disable(struct intel_encoder *encoder);
+void intel_audio_codec_disable(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *old_crtc_state,
+			       const struct drm_connector_state *old_conn_state);
 void i915_audio_component_init(struct drm_i915_private *dev_priv);
 void i915_audio_component_cleanup(struct drm_i915_private *dev_priv);
 void intel_audio_init(struct drm_i915_private *dev_priv);
@@ -1323,10 +1327,14 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv);
 void intel_update_max_cdclk(struct drm_i915_private *dev_priv);
 void intel_update_cdclk(struct drm_i915_private *dev_priv);
 void intel_update_rawclk(struct drm_i915_private *dev_priv);
-bool intel_cdclk_state_compare(const struct intel_cdclk_state *a,
+bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a,
 			       const struct intel_cdclk_state *b);
+bool intel_cdclk_changed(const struct intel_cdclk_state *a,
+			 const struct intel_cdclk_state *b);
 void intel_set_cdclk(struct drm_i915_private *dev_priv,
 		     const struct intel_cdclk_state *cdclk_state);
+void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
+			    const char *context);
 
 /* intel_display.c */
 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
@@ -1478,8 +1486,9 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
 int chv_calc_dpll_params(int refclk, struct dpll *pll_clock);
 
 bool intel_crtc_active(struct intel_crtc *crtc);
-void hsw_enable_ips(struct intel_crtc *crtc);
-void hsw_disable_ips(struct intel_crtc *crtc);
+bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state);
+void hsw_enable_ips(const struct intel_crtc_state *crtc_state);
+void hsw_disable_ips(const struct intel_crtc_state *crtc_state);
 enum intel_display_power_domain intel_port_to_power_domain(enum port port);
 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
 				 struct intel_crtc_state *pipe_config);
@@ -1492,6 +1501,8 @@ static inline u32 intel_plane_ggtt_offset(const struct intel_plane_state *state)
 	return i915_ggtt_offset(state->vma);
 }
 
+u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
+			const struct intel_plane_state *plane_state);
 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 		  const struct intel_plane_state *plane_state);
 u32 skl_plane_stride(const struct drm_framebuffer *fb, int plane,
@@ -1522,7 +1533,8 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
 void intel_dp_encoder_reset(struct drm_encoder *encoder);
 void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_destroy(struct drm_encoder *encoder);
-int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc);
+int intel_dp_sink_crc(struct intel_dp *intel_dp,
+		      struct intel_crtc_state *crtc_state, u8 *crc);
 bool intel_dp_compute_config(struct intel_encoder *encoder,
 			     struct intel_crtc_state *pipe_config,
 			     struct drm_connector_state *conn_state);
@@ -1641,7 +1653,7 @@ static inline void intel_fbdev_restore_mode(struct drm_device *dev)
 
 /* intel_fbc.c */
 void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
-			   struct drm_atomic_state *state);
+			   struct intel_atomic_state *state);
 bool intel_fbc_is_active(struct drm_i915_private *dev_priv);
 void intel_fbc_pre_update(struct intel_crtc *crtc,
 			  struct intel_crtc_state *crtc_state,
@@ -1869,7 +1881,6 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
 void gen6_rps_busy(struct drm_i915_private *dev_priv);
@@ -1897,15 +1908,10 @@ bool skl_ddb_allocation_overlaps(struct drm_i915_private *dev_priv,
 				 const struct skl_ddb_entry *ddb,
 				 int ignore);
 bool ilk_disable_lp_wm(struct drm_device *dev);
-int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6);
 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
 				  struct intel_crtc_state *cstate);
 void intel_init_ipc(struct drm_i915_private *dev_priv);
 void intel_enable_ipc(struct drm_i915_private *dev_priv);
-static inline int intel_rc6_enabled(void)
-{
-	return i915_modparams.enable_rc6;
-}
 
 /* intel_sdvo.c */
 bool intel_sdvo_init(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index 83f15848098a..f67d321376e4 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -662,11 +662,11 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 	}
 }
 
-static void intel_dsi_port_enable(struct intel_encoder *encoder)
+static void intel_dsi_port_enable(struct intel_encoder *encoder,
+				  const struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	enum port port;
 
@@ -705,7 +705,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder)
 			if (IS_BROXTON(dev_priv))
 				temp |= LANE_CONFIGURATION_DUAL_LINK_A;
 			else
-				temp |= intel_crtc->pipe ?
+				temp |= crtc->pipe ?
 					LANE_CONFIGURATION_DUAL_LINK_B :
 					LANE_CONFIGURATION_DUAL_LINK_A;
 		}
@@ -875,7 +875,7 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
 
 		intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DISPLAY_ON);
 
-		intel_dsi_port_enable(encoder);
+		intel_dsi_port_enable(encoder, pipe_config);
 	}
 
 	intel_panel_enable_backlight(pipe_config, conn_state);
@@ -1082,7 +1082,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 	struct drm_display_mode *adjusted_mode =
 					&pipe_config->base.adjusted_mode;
 	struct drm_display_mode *adjusted_mode_sw;
-	struct intel_crtc *intel_crtc;
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 	unsigned int lane_count = intel_dsi->lane_count;
 	unsigned int bpp, fmt;
@@ -1093,8 +1093,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
 				crtc_hblank_start_sw, crtc_hblank_end_sw;
 
 	/* FIXME: hw readout should not depend on SW state */
-	intel_crtc = to_intel_crtc(encoder->base.crtc);
-	adjusted_mode_sw = &intel_crtc->config->base.adjusted_mode;
+	adjusted_mode_sw = &crtc->config->base.adjusted_mode;
 
 	/*
 	 * Atleast one port is active as encoder->get_config called only if
@@ -1243,6 +1242,8 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
 	u32 pclk;
 	DRM_DEBUG_KMS("\n");
 
+	pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI);
+
 	if (IS_GEN9_LP(dev_priv))
 		bxt_dsi_get_pipe_config(encoder, pipe_config);
 
@@ -1665,6 +1666,27 @@ static const struct drm_connector_funcs intel_dsi_connector_funcs = {
 	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
 };
 
+static int intel_dsi_get_panel_orientation(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	int orientation = DRM_MODE_PANEL_ORIENTATION_NORMAL;
+	enum i9xx_plane_id plane;
+	u32 val;
+
+	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+		if (connector->encoder->crtc_mask == BIT(PIPE_B))
+			plane = PLANE_B;
+		else
+			plane = PLANE_A;
+
+		val = I915_READ(DSPCNTR(plane));
+		if (val & DISPPLANE_ROTATE_180)
+			orientation = DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP;
+	}
+
+	return orientation;
+}
+
 static void intel_dsi_add_properties(struct intel_connector *connector)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
@@ -1680,6 +1702,13 @@ static void intel_dsi_add_properties(struct intel_connector *connector)
 								allowed_scalers);
 
 		connector->base.state->scaling_mode = DRM_MODE_SCALE_ASPECT;
+
+		connector->base.display_info.panel_orientation =
+			intel_dsi_get_panel_orientation(connector);
+		drm_connector_init_panel_orientation_property(
+				&connector->base,
+				connector->panel.fixed_mode->hdisplay,
+				connector->panel.fixed_mode->vdisplay);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 53c9b763f4ce..754baa00bea9 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -159,6 +159,8 @@ static void intel_dvo_get_config(struct intel_encoder *encoder,
 	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
 	u32 tmp, flags = 0;
 
+	pipe_config->output_types |= BIT(INTEL_OUTPUT_DVO);
+
 	tmp = I915_READ(intel_dvo->dev.dvo_reg);
 	if (tmp & DVO_HSYNC_ACTIVE_HIGH)
 		flags |= DRM_MODE_FLAG_PHSYNC;
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 6074e04dc99f..fa960cfd2764 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -37,8 +37,6 @@
  * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
  */
 #define HSW_CXT_TOTAL_SIZE		(17 * PAGE_SIZE)
-/* Same as Haswell, but 72064 bytes now. */
-#define GEN8_CXT_TOTAL_SIZE		(18 * PAGE_SIZE)
 
 #define GEN8_LR_CONTEXT_RENDER_SIZE	(20 * PAGE_SIZE)
 #define GEN9_LR_CONTEXT_RENDER_SIZE	(22 * PAGE_SIZE)
@@ -50,6 +48,8 @@ struct engine_class_info {
 	const char *name;
 	int (*init_legacy)(struct intel_engine_cs *engine);
 	int (*init_execlists)(struct intel_engine_cs *engine);
+
+	u8 uabi_class;
 };
 
 static const struct engine_class_info intel_engine_classes[] = {
@@ -57,21 +57,25 @@ static const struct engine_class_info intel_engine_classes[] = {
 		.name = "rcs",
 		.init_execlists = logical_render_ring_init,
 		.init_legacy = intel_init_render_ring_buffer,
+		.uabi_class = I915_ENGINE_CLASS_RENDER,
 	},
 	[COPY_ENGINE_CLASS] = {
 		.name = "bcs",
 		.init_execlists = logical_xcs_ring_init,
 		.init_legacy = intel_init_blt_ring_buffer,
+		.uabi_class = I915_ENGINE_CLASS_COPY,
 	},
 	[VIDEO_DECODE_CLASS] = {
 		.name = "vcs",
 		.init_execlists = logical_xcs_ring_init,
 		.init_legacy = intel_init_bsd_ring_buffer,
+		.uabi_class = I915_ENGINE_CLASS_VIDEO,
 	},
 	[VIDEO_ENHANCEMENT_CLASS] = {
 		.name = "vecs",
 		.init_execlists = logical_xcs_ring_init,
 		.init_legacy = intel_init_vebox_ring_buffer,
+		.uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
 	},
 };
 
@@ -158,9 +162,7 @@ __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
 		case 9:
 			return GEN9_LR_CONTEXT_RENDER_SIZE;
 		case 8:
-			return i915_modparams.enable_execlists ?
-			       GEN8_LR_CONTEXT_RENDER_SIZE :
-			       GEN8_CXT_TOTAL_SIZE;
+			return GEN8_LR_CONTEXT_RENDER_SIZE;
 		case 7:
 			if (IS_HASWELL(dev_priv))
 				return HSW_CXT_TOTAL_SIZE;
@@ -203,6 +205,15 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 	GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
 	class_info = &intel_engine_classes[info->class];
 
+	if (GEM_WARN_ON(info->class > MAX_ENGINE_CLASS))
+		return -EINVAL;
+
+	if (GEM_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
+		return -EINVAL;
+
+	if (GEM_WARN_ON(dev_priv->engine_class[info->class][info->instance]))
+		return -EINVAL;
+
 	GEM_BUG_ON(dev_priv->engine[id]);
 	engine = kzalloc(sizeof(*engine), GFP_KERNEL);
 	if (!engine)
@@ -213,13 +224,15 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 	WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
 			 class_info->name, info->instance) >=
 		sizeof(engine->name));
-	engine->uabi_id = info->uabi_id;
 	engine->hw_id = engine->guc_id = info->hw_id;
 	engine->mmio_base = info->mmio_base;
 	engine->irq_shift = info->irq_shift;
 	engine->class = info->class;
 	engine->instance = info->instance;
 
+	engine->uabi_id = info->uabi_id;
+	engine->uabi_class = class_info->uabi_class;
+
 	engine->context_size = __intel_engine_context_size(dev_priv,
 							   engine->class);
 	if (WARN_ON(engine->context_size > BIT(20)))
@@ -228,8 +241,11 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 	/* Nothing to do here, execute in order of dependencies */
 	engine->schedule = NULL;
 
+	spin_lock_init(&engine->stats.lock);
+
 	ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
 
+	dev_priv->engine_class[info->class][info->instance] = engine;
 	dev_priv->engine[id] = engine;
 	return 0;
 }
@@ -281,6 +297,8 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 
 	device_info->num_rings = hweight32(mask);
 
+	i915_check_and_clear_faults(dev_priv);
+
 	return 0;
 
 cleanup:
@@ -306,7 +324,7 @@ int intel_engines_init(struct drm_i915_private *dev_priv)
 			&intel_engine_classes[engine->class];
 		int (*init)(struct intel_engine_cs *engine);
 
-		if (i915_modparams.enable_execlists)
+		if (HAS_EXECLISTS(dev_priv))
 			init = class_info->init_execlists;
 		else
 			init = class_info->init_legacy;
@@ -356,18 +374,6 @@ void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
 		if (HAS_VEBOX(dev_priv))
 			I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
 	}
-	if (dev_priv->semaphore) {
-		struct page *page = i915_vma_first_page(dev_priv->semaphore);
-		void *semaphores;
-
-		/* Semaphores are in noncoherent memory, flush to be safe */
-		semaphores = kmap_atomic(page);
-		memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
-		       0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
-		drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
-				       I915_NUM_ENGINES * gen8_semaphore_seqno_size);
-		kunmap_atomic(semaphores);
-	}
 
 	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
 	clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
@@ -620,7 +626,7 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	 * Similarly the preempt context must always be available so that
 	 * we can interrupt the engine at any time.
 	 */
-	if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) {
+	if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) {
 		ring = engine->context_pin(engine,
 					   engine->i915->preempt_context);
 		if (IS_ERR(ring)) {
@@ -633,25 +639,19 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	if (ret)
 		goto err_unpin_preempt;
 
-	ret = i915_gem_render_state_init(engine);
-	if (ret)
-		goto err_breadcrumbs;
-
 	if (HWS_NEEDS_PHYSICAL(engine->i915))
 		ret = init_phys_status_page(engine);
 	else
 		ret = init_status_page(engine);
 	if (ret)
-		goto err_rs_fini;
+		goto err_breadcrumbs;
 
 	return 0;
 
-err_rs_fini:
-	i915_gem_render_state_fini(engine);
 err_breadcrumbs:
 	intel_engine_fini_breadcrumbs(engine);
 err_unpin_preempt:
-	if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
+	if (HAS_LOGICAL_RING_PREEMPTION(engine->i915))
 		engine->context_unpin(engine, engine->i915->preempt_context);
 err_unpin_kernel:
 	engine->context_unpin(engine, engine->i915->kernel_context);
@@ -674,12 +674,14 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 	else
 		cleanup_status_page(engine);
 
-	i915_gem_render_state_fini(engine);
 	intel_engine_fini_breadcrumbs(engine);
 	intel_engine_cleanup_cmd_parser(engine);
 	i915_gem_batch_pool_fini(&engine->batch_pool);
 
-	if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
+	if (engine->default_state)
+		i915_gem_object_put(engine->default_state);
+
+	if (HAS_LOGICAL_RING_PREEMPTION(engine->i915))
 		engine->context_unpin(engine, engine->i915->preempt_context);
 	engine->context_unpin(engine, engine->i915->kernel_context);
 }
@@ -1014,22 +1016,6 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 				  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 
-	/* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
-		WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
-				  GEN9_DG_MIRROR_FIX_ENABLE);
-
-	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-		WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
-				  GEN9_RHWO_OPTIMIZATION_DISABLE);
-		/*
-		 * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
-		 * but we do that in per ctx batchbuffer as there is an issue
-		 * with this register not getting restored on ctx restore
-		 */
-	}
-
 	/* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
 	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
 	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
@@ -1045,11 +1031,6 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 			  GEN9_CCS_TLB_PREFETCH_ENABLE);
 
-	/* WaDisableMaskBasedCammingInRCC:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
-		WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
-				  PIXEL_MASK_CAMMING_DISABLE);
-
 	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
 	WA_SET_BIT_MASKED(HDC_CHICKEN0,
 			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
@@ -1079,14 +1060,22 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
 	if (IS_SKYLAKE(dev_priv) ||
 	    IS_KABYLAKE(dev_priv) ||
-	    IS_COFFEELAKE(dev_priv) ||
-	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
+	    IS_COFFEELAKE(dev_priv))
 		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 				  GEN8_SAMPLER_POWER_BYPASS_DIS);
 
 	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
 	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 
+	/* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */
+	if (IS_GEN9_LP(dev_priv)) {
+		u32 val = I915_READ(GEN8_L3SQCREG1);
+
+		val &= ~L3_PRIO_CREDITS_MASK;
+		val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
+		I915_WRITE(GEN8_L3SQCREG1, val);
+	}
+
 	/* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
 	I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
 				    GEN8_LQSC_FLUSH_COHERENT_LINES));
@@ -1210,66 +1199,22 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
 	if (ret)
 		return ret;
 
-	/* WaStoreMultiplePTEenable:bxt */
-	/* This is a requirement according to Hardware specification */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
-		I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
-
-	/* WaSetClckGatingDisableMedia:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-		I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
-					    ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
-	}
-
 	/* WaDisableThreadStallDopClockGating:bxt */
 	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 			  STALL_DOP_GATING_DISABLE);
 
 	/* WaDisablePooledEuLoadBalancingFix:bxt */
-	if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
-		I915_WRITE(FF_SLICE_CS_CHICKEN2,
-			   _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
-	}
-
-	/* WaDisableSbeCacheDispatchPortSharing:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
-		WA_SET_BIT_MASKED(
-			GEN7_HALF_SLICE_CHICKEN1,
-			GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
-	}
-
-	/* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
-	/* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
-	/* WaDisableObjectLevelPreemtionForInstanceId:bxt */
-	/* WaDisableLSQCROPERFforOCL:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-		ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
-		if (ret)
-			return ret;
-
-		ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
-		if (ret)
-			return ret;
-	}
-
-	/* WaProgramL3SqcReg1DefaultForPerf:bxt */
-	if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
-		u32 val = I915_READ(GEN8_L3SQCREG1);
-		val &= ~L3_PRIO_CREDITS_MASK;
-		val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
-		I915_WRITE(GEN8_L3SQCREG1, val);
-	}
+	I915_WRITE(FF_SLICE_CS_CHICKEN2,
+		   _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
 
 	/* WaToEnableHwFixForPushConstHWBug:bxt */
-	if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
-		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-				  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+	WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+			  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 
 	/* WaInPlaceDecompressionHang:bxt */
-	if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
-		I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
-			   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
-			    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
+	I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+		   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+		    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
 
 	return 0;
 }
@@ -1327,6 +1272,9 @@ static int cnl_init_workarounds(struct intel_engine_cs *engine)
 	if (ret)
 		return ret;
 
+	/* WaDisableEarlyEOT:cnl */
+	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT);
+
 	return 0;
 }
 
@@ -1510,7 +1458,9 @@ static bool ring_is_idle(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 	bool idle = true;
 
-	intel_runtime_pm_get(dev_priv);
+	/* If the whole device is asleep, the engine must be idle */
+	if (!intel_runtime_pm_get_if_in_use(dev_priv))
+		return true;
 
 	/* First check that no commands are left in the ring */
 	if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
@@ -1573,10 +1523,8 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
-	if (READ_ONCE(dev_priv->gt.active_requests))
-		return false;
-
-	/* If the driver is wedged, HW state may be very inconsistent and
+	/*
+	 * If the driver is wedged, HW state may be very inconsistent and
 	 * report that it is still busy, even though we have stopped using it.
 	 */
 	if (i915_terminally_wedged(&dev_priv->gpu_error))
@@ -1590,6 +1538,34 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
 	return true;
 }
 
+/**
+ * intel_engine_has_kernel_context:
+ * @engine: the engine
+ *
+ * Returns true if the last context to be executed on this engine, or has been
+ * executed if the engine is already idle, is the kernel context
+ * (#i915.kernel_context).
+ */
+bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
+{
+	const struct i915_gem_context * const kernel_context =
+		engine->i915->kernel_context;
+	struct drm_i915_gem_request *rq;
+
+	lockdep_assert_held(&engine->i915->drm.struct_mutex);
+
+	/*
+	 * Check the last context seen by the engine. If active, it will be
+	 * the last request that remains in the timeline. When idle, it is
+	 * the last executed context as tracked by retirement.
+	 */
+	rq = __i915_gem_active_peek(&engine->timeline->last_request);
+	if (rq)
+		return rq->ctx == kernel_context;
+	else
+		return engine->last_retired_context == kernel_context;
+}
+
 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
 {
 	struct intel_engine_cs *engine;
@@ -1599,19 +1575,63 @@ void intel_engines_reset_default_submission(struct drm_i915_private *i915)
 		engine->set_default_submission(engine);
 }
 
-void intel_engines_mark_idle(struct drm_i915_private *i915)
+/**
+ * intel_engines_park: called when the GT is transitioning from busy->idle
+ * @i915: the i915 device
+ *
+ * The GT is now idle and about to go to sleep (maybe never to wake again?).
+ * Time for us to tidy and put away our toys (release resources back to the
+ * system).
+ */
+void intel_engines_park(struct drm_i915_private *i915)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
 	for_each_engine(engine, i915, id) {
+		/* Flush the residual irq tasklets first. */
 		intel_engine_disarm_breadcrumbs(engine);
+		tasklet_kill(&engine->execlists.tasklet);
+
+		/*
+		 * We are committed now to parking the engines, make sure there
+		 * will be no more interrupts arriving later and the engines
+		 * are truly idle.
+		 */
+		if (wait_for(intel_engine_is_idle(engine), 10)) {
+			struct drm_printer p = drm_debug_printer(__func__);
+
+			dev_err(i915->drm.dev,
+				"%s is not idle before parking\n",
+				engine->name);
+			intel_engine_dump(engine, &p, NULL);
+		}
+
+		if (engine->park)
+			engine->park(engine);
+
 		i915_gem_batch_pool_fini(&engine->batch_pool);
-		tasklet_kill(&engine->execlists.irq_tasklet);
 		engine->execlists.no_priolist = false;
 	}
 }
 
+/**
+ * intel_engines_unpark: called when the GT is transitioning from idle->busy
+ * @i915: the i915 device
+ *
+ * The GT was idle and now about to fire up with some new user requests.
+ */
+void intel_engines_unpark(struct drm_i915_private *i915)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, i915, id) {
+		if (engine->unpark)
+			engine->unpark(engine);
+	}
+}
+
 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
 {
 	switch (INTEL_GEN(engine->i915)) {
@@ -1627,6 +1647,20 @@ bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
 	}
 }
 
+unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	unsigned int which;
+
+	which = 0;
+	for_each_engine(engine, i915, id)
+		if (engine->default_state)
+			which |= BIT(engine->uabi_class);
+
+	return which;
+}
+
 static void print_request(struct drm_printer *m,
 			  struct drm_i915_gem_request *rq,
 			  const char *prefix)
@@ -1640,7 +1674,38 @@ static void print_request(struct drm_printer *m,
 		   rq->timeline->common->name);
 }
 
-void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
+static void hexdump(struct drm_printer *m, const void *buf, size_t len)
+{
+	const size_t rowsize = 8 * sizeof(u32);
+	const void *prev = NULL;
+	bool skip = false;
+	size_t pos;
+
+	for (pos = 0; pos < len; pos += rowsize) {
+		char line[128];
+
+		if (prev && !memcmp(prev, buf + pos, rowsize)) {
+			if (!skip) {
+				drm_printf(m, "*\n");
+				skip = true;
+			}
+			continue;
+		}
+
+		WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
+						rowsize, sizeof(u32),
+						line, sizeof(line),
+						false) >= sizeof(line));
+		drm_printf(m, "%08zx %s\n", pos, line);
+
+		prev = buf + pos;
+		skip = false;
+	}
+}
+
+void intel_engine_dump(struct intel_engine_cs *engine,
+		       struct drm_printer *m,
+		       const char *header, ...)
 {
 	struct intel_breadcrumbs * const b = &engine->breadcrumbs;
 	const struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -1648,17 +1713,29 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
 	struct drm_i915_private *dev_priv = engine->i915;
 	struct drm_i915_gem_request *rq;
 	struct rb_node *rb;
+	char hdr[80];
 	u64 addr;
 
-	drm_printf(m, "%s\n", engine->name);
+	if (header) {
+		va_list ap;
+
+		va_start(ap, header);
+		drm_vprintf(m, header, &ap);
+		va_end(ap);
+	}
+
+	if (i915_terminally_wedged(&engine->i915->gpu_error))
+		drm_printf(m, "*** WEDGED ***\n");
+
 	drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms], inflight %d\n",
 		   intel_engine_get_seqno(engine),
 		   intel_engine_last_submit(engine),
 		   engine->hangcheck.seqno,
 		   jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp),
 		   engine->timeline->inflight_seqnos);
-	drm_printf(m, "\tReset count: %d\n",
-		   i915_reset_engine_count(error, engine));
+	drm_printf(m, "\tReset count: %d (global %d)\n",
+		   i915_reset_engine_count(error, engine),
+		   i915_reset_count(error));
 
 	rcu_read_lock();
 
@@ -1693,9 +1770,23 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
 	drm_printf(m, "\tRING_TAIL:  0x%08x [0x%08x]\n",
 		   I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
 		   rq ? rq->ring->tail : 0);
-	drm_printf(m, "\tRING_CTL:   0x%08x [%s]\n",
+	drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
 		   I915_READ(RING_CTL(engine->mmio_base)),
-		   I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : "");
+		   I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
+	if (INTEL_GEN(engine->i915) > 2) {
+		drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
+			   I915_READ(RING_MI_MODE(engine->mmio_base)),
+			   I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : "");
+	}
+	if (HAS_LEGACY_SEMAPHORES(dev_priv)) {
+		drm_printf(m, "\tSYNC_0: 0x%08x\n",
+			   I915_READ(RING_SYNC_0(engine->mmio_base)));
+		drm_printf(m, "\tSYNC_1: 0x%08x\n",
+			   I915_READ(RING_SYNC_1(engine->mmio_base)));
+		if (HAS_VEBOX(dev_priv))
+			drm_printf(m, "\tSYNC_2: 0x%08x\n",
+				   I915_READ(RING_SYNC_2(engine->mmio_base)));
+	}
 
 	rcu_read_unlock();
 
@@ -1705,8 +1796,26 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
 	addr = intel_engine_get_last_batch_head(engine);
 	drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
 		   upper_32_bits(addr), lower_32_bits(addr));
+	if (INTEL_GEN(dev_priv) >= 8)
+		addr = I915_READ64_2x32(RING_DMA_FADD(engine->mmio_base),
+					RING_DMA_FADD_UDW(engine->mmio_base));
+	else if (INTEL_GEN(dev_priv) >= 4)
+		addr = I915_READ(RING_DMA_FADD(engine->mmio_base));
+	else
+		addr = I915_READ(DMA_FADD_I8XX);
+	drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
+		   upper_32_bits(addr), lower_32_bits(addr));
+	if (INTEL_GEN(dev_priv) >= 4) {
+		drm_printf(m, "\tIPEIR: 0x%08x\n",
+			   I915_READ(RING_IPEIR(engine->mmio_base)));
+		drm_printf(m, "\tIPEHR: 0x%08x\n",
+			   I915_READ(RING_IPEHR(engine->mmio_base)));
+	} else {
+		drm_printf(m, "\tIPEIR: 0x%08x\n", I915_READ(IPEIR));
+		drm_printf(m, "\tIPEHR: 0x%08x\n", I915_READ(IPEHR));
+	}
 
-	if (i915_modparams.enable_execlists) {
+	if (HAS_EXECLISTS(dev_priv)) {
 		const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
 		u32 ptr, read, write;
 		unsigned int idx;
@@ -1746,12 +1855,12 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
 
 			rq = port_unpack(&execlists->port[idx], &count);
 			if (rq) {
-				drm_printf(m, "\t\tELSP[%d] count=%d, ",
-					   idx, count);
-				print_request(m, rq, "rq: ");
+				snprintf(hdr, sizeof(hdr),
+					 "\t\tELSP[%d] count=%d, rq: ",
+					 idx, count);
+				print_request(m, rq, hdr);
 			} else {
-				drm_printf(m, "\t\tELSP[%d] idle\n",
-					   idx);
+				drm_printf(m, "\t\tELSP[%d] idle\n", idx);
 			}
 		}
 		drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
@@ -1786,7 +1895,147 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
 	}
 	spin_unlock_irq(&b->rb_lock);
 
-	drm_printf(m, "\n");
+	if (INTEL_GEN(dev_priv) >= 6) {
+		drm_printf(m, "\tRING_IMR: %08x\n", I915_READ_IMR(engine));
+	}
+
+	drm_printf(m, "IRQ? 0x%lx (breadcrumbs? %s) (execlists? %s)\n",
+		   engine->irq_posted,
+		   yesno(test_bit(ENGINE_IRQ_BREADCRUMB,
+				  &engine->irq_posted)),
+		   yesno(test_bit(ENGINE_IRQ_EXECLIST,
+				  &engine->irq_posted)));
+
+	drm_printf(m, "HWSP:\n");
+	hexdump(m, engine->status_page.page_addr, PAGE_SIZE);
+
+	drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
+}
+
+static u8 user_class_map[] = {
+	[I915_ENGINE_CLASS_RENDER] = RENDER_CLASS,
+	[I915_ENGINE_CLASS_COPY] = COPY_ENGINE_CLASS,
+	[I915_ENGINE_CLASS_VIDEO] = VIDEO_DECODE_CLASS,
+	[I915_ENGINE_CLASS_VIDEO_ENHANCE] = VIDEO_ENHANCEMENT_CLASS,
+};
+
+struct intel_engine_cs *
+intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance)
+{
+	if (class >= ARRAY_SIZE(user_class_map))
+		return NULL;
+
+	class = user_class_map[class];
+
+	GEM_BUG_ON(class > MAX_ENGINE_CLASS);
+
+	if (instance > MAX_ENGINE_INSTANCE)
+		return NULL;
+
+	return i915->engine_class[class][instance];
+}
+
+/**
+ * intel_enable_engine_stats() - Enable engine busy tracking on engine
+ * @engine: engine to enable stats collection
+ *
+ * Start collecting the engine busyness data for @engine.
+ *
+ * Returns 0 on success or a negative error code.
+ */
+int intel_enable_engine_stats(struct intel_engine_cs *engine)
+{
+	struct intel_engine_execlists *execlists = &engine->execlists;
+	unsigned long flags;
+	int err = 0;
+
+	if (!intel_engine_supports_stats(engine))
+		return -ENODEV;
+
+	tasklet_disable(&execlists->tasklet);
+	spin_lock_irqsave(&engine->stats.lock, flags);
+
+	if (unlikely(engine->stats.enabled == ~0)) {
+		err = -EBUSY;
+		goto unlock;
+	}
+
+	if (engine->stats.enabled++ == 0) {
+		const struct execlist_port *port = execlists->port;
+		unsigned int num_ports = execlists_num_ports(execlists);
+
+		engine->stats.enabled_at = ktime_get();
+
+		/* XXX submission method oblivious? */
+		while (num_ports-- && port_isset(port)) {
+			engine->stats.active++;
+			port++;
+		}
+
+		if (engine->stats.active)
+			engine->stats.start = engine->stats.enabled_at;
+	}
+
+unlock:
+	spin_unlock_irqrestore(&engine->stats.lock, flags);
+	tasklet_enable(&execlists->tasklet);
+
+	return err;
+}
+
+static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine)
+{
+	ktime_t total = engine->stats.total;
+
+	/*
+	 * If the engine is executing something at the moment
+	 * add it to the total.
+	 */
+	if (engine->stats.active)
+		total = ktime_add(total,
+				  ktime_sub(ktime_get(), engine->stats.start));
+
+	return total;
+}
+
+/**
+ * intel_engine_get_busy_time() - Return current accumulated engine busyness
+ * @engine: engine to report on
+ *
+ * Returns accumulated time @engine was busy since engine stats were enabled.
+ */
+ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine)
+{
+	ktime_t total;
+	unsigned long flags;
+
+	spin_lock_irqsave(&engine->stats.lock, flags);
+	total = __intel_engine_get_busy_time(engine);
+	spin_unlock_irqrestore(&engine->stats.lock, flags);
+
+	return total;
+}
+
+/**
+ * intel_disable_engine_stats() - Disable engine busy tracking on engine
+ * @engine: engine to disable stats collection
+ *
+ * Stops collecting the engine busyness data for @engine.
+ */
+void intel_disable_engine_stats(struct intel_engine_cs *engine)
+{
+	unsigned long flags;
+
+	if (!intel_engine_supports_stats(engine))
+		return;
+
+	spin_lock_irqsave(&engine->stats.lock, flags);
+	WARN_ON_ONCE(engine->stats.enabled == 0);
+	if (--engine->stats.enabled == 0) {
+		engine->stats.total = __intel_engine_get_busy_time(engine);
+		engine->stats.active = 0;
+	}
+	spin_unlock_irqrestore(&engine->stats.lock, flags);
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index 1a0f5e0c8d10..f88c1b5dae4c 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -151,7 +151,7 @@ static void i8xx_fbc_activate(struct drm_i915_private *dev_priv)
 
 		/* Set it up... */
 		fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
-		fbc_ctl2 |= FBC_CTL_PLANE(params->crtc.plane);
+		fbc_ctl2 |= FBC_CTL_PLANE(params->crtc.i9xx_plane);
 		I915_WRITE(FBC_CONTROL2, fbc_ctl2);
 		I915_WRITE(FBC_FENCE_OFF, params->crtc.fence_y_offset);
 	}
@@ -177,7 +177,7 @@ static void g4x_fbc_activate(struct drm_i915_private *dev_priv)
 	struct intel_fbc_reg_params *params = &dev_priv->fbc.params;
 	u32 dpfc_ctl;
 
-	dpfc_ctl = DPFC_CTL_PLANE(params->crtc.plane) | DPFC_SR_EN;
+	dpfc_ctl = DPFC_CTL_PLANE(params->crtc.i9xx_plane) | DPFC_SR_EN;
 	if (params->fb.format->cpp[0] == 2)
 		dpfc_ctl |= DPFC_CTL_LIMIT_2X;
 	else
@@ -224,7 +224,7 @@ static void ilk_fbc_activate(struct drm_i915_private *dev_priv)
 	u32 dpfc_ctl;
 	int threshold = dev_priv->fbc.threshold;
 
-	dpfc_ctl = DPFC_CTL_PLANE(params->crtc.plane);
+	dpfc_ctl = DPFC_CTL_PLANE(params->crtc.i9xx_plane);
 	if (params->fb.format->cpp[0] == 2)
 		threshold++;
 
@@ -306,7 +306,7 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv)
 
 	dpfc_ctl = 0;
 	if (IS_IVYBRIDGE(dev_priv))
-		dpfc_ctl |= IVB_DPFC_CTL_PLANE(params->crtc.plane);
+		dpfc_ctl |= IVB_DPFC_CTL_PLANE(params->crtc.i9xx_plane);
 
 	if (params->fb.format->cpp[0] == 2)
 		threshold++;
@@ -531,7 +531,6 @@ static int find_compression_threshold(struct drm_i915_private *dev_priv,
 				      int size,
 				      int fb_cpp)
 {
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	int compression_threshold = 1;
 	int ret;
 	u64 end;
@@ -541,7 +540,7 @@ static int find_compression_threshold(struct drm_i915_private *dev_priv,
 	 * If we enable FBC using a CFB on that memory range we'll get FIFO
 	 * underruns, even if that range is not reserved by the BIOS. */
 	if (IS_BROADWELL(dev_priv) || IS_GEN9_BC(dev_priv))
-		end = ggtt->stolen_size - 8 * 1024 * 1024;
+		end = resource_size(&dev_priv->dsm) - 8 * 1024 * 1024;
 	else
 		end = U64_MAX;
 
@@ -615,10 +614,16 @@ static int intel_fbc_alloc_cfb(struct intel_crtc *crtc)
 
 		fbc->compressed_llb = compressed_llb;
 
+		GEM_BUG_ON(range_overflows_t(u64, dev_priv->dsm.start,
+					     fbc->compressed_fb.start,
+					     U32_MAX));
+		GEM_BUG_ON(range_overflows_t(u64, dev_priv->dsm.start,
+					     fbc->compressed_llb->start,
+					     U32_MAX));
 		I915_WRITE(FBC_CFB_BASE,
-			   dev_priv->mm.stolen_base + fbc->compressed_fb.start);
+			   dev_priv->dsm.start + fbc->compressed_fb.start);
 		I915_WRITE(FBC_LL_BASE,
-			   dev_priv->mm.stolen_base + compressed_llb->start);
+			   dev_priv->dsm.start + compressed_llb->start);
 	}
 
 	DRM_DEBUG_KMS("reserved %llu bytes of contiguous stolen space for FBC, threshold: %d\n",
@@ -890,7 +895,7 @@ static void intel_fbc_get_reg_params(struct intel_crtc *crtc,
 	params->vma = cache->vma;
 
 	params->crtc.pipe = crtc->pipe;
-	params->crtc.plane = crtc->plane;
+	params->crtc.i9xx_plane = to_intel_plane(crtc->base.primary)->i9xx_plane;
 	params->crtc.fence_y_offset = get_crtc_fence_y_offset(fbc);
 
 	params->fb.format = cache->fb.format;
@@ -1054,11 +1059,11 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv,
  * enable FBC for the chosen CRTC. If it does, it will set dev_priv->fbc.crtc.
  */
 void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
-			   struct drm_atomic_state *state)
+			   struct intel_atomic_state *state)
 {
 	struct intel_fbc *fbc = &dev_priv->fbc;
-	struct drm_plane *plane;
-	struct drm_plane_state *plane_state;
+	struct intel_plane *plane;
+	struct intel_plane_state *plane_state;
 	bool crtc_chosen = false;
 	int i;
 
@@ -1066,7 +1071,7 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
 
 	/* Does this atomic commit involve the CRTC currently tied to FBC? */
 	if (fbc->crtc &&
-	    !drm_atomic_get_existing_crtc_state(state, &fbc->crtc->base))
+	    !intel_atomic_get_new_crtc_state(state, fbc->crtc))
 		goto out;
 
 	if (!intel_fbc_can_enable(dev_priv))
@@ -1076,25 +1081,22 @@ void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
 	 * plane. We could go for fancier schemes such as checking the plane
 	 * size, but this would just affect the few platforms that don't tie FBC
 	 * to pipe or plane A. */
-	for_each_new_plane_in_state(state, plane, plane_state, i) {
-		struct intel_plane_state *intel_plane_state =
-			to_intel_plane_state(plane_state);
-		struct intel_crtc_state *intel_crtc_state;
-		struct intel_crtc *crtc = to_intel_crtc(plane_state->crtc);
+	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
+		struct intel_crtc_state *crtc_state;
+		struct intel_crtc *crtc = to_intel_crtc(plane_state->base.crtc);
 
-		if (!intel_plane_state->base.visible)
+		if (!plane_state->base.visible)
 			continue;
 
 		if (fbc_on_pipe_a_only(dev_priv) && crtc->pipe != PIPE_A)
 			continue;
 
-		if (fbc_on_plane_a_only(dev_priv) && crtc->plane != PLANE_A)
+		if (fbc_on_plane_a_only(dev_priv) && plane->i9xx_plane != PLANE_A)
 			continue;
 
-		intel_crtc_state = to_intel_crtc_state(
-			drm_atomic_get_existing_crtc_state(state, &crtc->base));
+		crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
 
-		intel_crtc_state->enable_fbc = true;
+		crtc_state->enable_fbc = true;
 		crtc_chosen = true;
 		break;
 	}
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index ea96682568e8..da48af11eb6b 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -115,7 +115,6 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
 	struct drm_framebuffer *fb;
 	struct drm_device *dev = helper->dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	struct drm_mode_fb_cmd2 mode_cmd = {};
 	struct drm_i915_gem_object *obj;
 	int size, ret;
@@ -139,7 +138,7 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
 	 * important and we should probably use that space with FBC or other
 	 * features. */
 	obj = NULL;
-	if (size * 2 < ggtt->stolen_usable_size)
+	if (size * 2 < dev_priv->stolen_usable_size)
 		obj = i915_gem_object_create_stolen(dev_priv, size);
 	if (obj == NULL)
 		obj = i915_gem_object_create(dev_priv, size);
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 10037c0fdf95..3c6bf5a34c3c 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -23,6 +23,7 @@
  */
 
 #include "intel_guc.h"
+#include "intel_guc_submission.h"
 #include "i915_drv.h"
 
 static void gen8_guc_raise_irq(struct intel_guc *guc)
@@ -60,6 +61,7 @@ void intel_guc_init_send_regs(struct intel_guc *guc)
 
 void intel_guc_init_early(struct intel_guc *guc)
 {
+	intel_guc_fw_init_early(guc);
 	intel_guc_ct_init_early(&guc->ct);
 
 	mutex_init(&guc->send_mutex);
@@ -67,6 +69,114 @@ void intel_guc_init_early(struct intel_guc *guc)
 	guc->notify = gen8_guc_raise_irq;
 }
 
+int intel_guc_init_wq(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	/*
+	 * GuC log buffer flush work item has to do register access to
+	 * send the ack to GuC and this work item, if not synced before
+	 * suspend, can potentially get executed after the GFX device is
+	 * suspended.
+	 * By marking the WQ as freezable, we don't have to bother about
+	 * flushing of this work item from the suspend hooks, the pending
+	 * work item if any will be either executed before the suspend
+	 * or scheduled later on resume. This way the handling of work
+	 * item can be kept same between system suspend & rpm suspend.
+	 */
+	guc->log.runtime.flush_wq = alloc_ordered_workqueue("i915-guc_log",
+						WQ_HIGHPRI | WQ_FREEZABLE);
+	if (!guc->log.runtime.flush_wq)
+		return -ENOMEM;
+
+	/*
+	 * Even though both sending GuC action, and adding a new workitem to
+	 * GuC workqueue are serialized (each with its own locking), since
+	 * we're using mutliple engines, it's possible that we're going to
+	 * issue a preempt request with two (or more - each for different
+	 * engine) workitems in GuC queue. In this situation, GuC may submit
+	 * all of them, which will make us very confused.
+	 * Our preemption contexts may even already be complete - before we
+	 * even had the chance to sent the preempt action to GuC!. Rather
+	 * than introducing yet another lock, we can just use ordered workqueue
+	 * to make sure we're always sending a single preemption request with a
+	 * single workitem.
+	 */
+	if (HAS_LOGICAL_RING_PREEMPTION(dev_priv) &&
+	    USES_GUC_SUBMISSION(dev_priv)) {
+		guc->preempt_wq = alloc_ordered_workqueue("i915-guc_preempt",
+							  WQ_HIGHPRI);
+		if (!guc->preempt_wq) {
+			destroy_workqueue(guc->log.runtime.flush_wq);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+void intel_guc_fini_wq(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	if (HAS_LOGICAL_RING_PREEMPTION(dev_priv) &&
+	    USES_GUC_SUBMISSION(dev_priv))
+		destroy_workqueue(guc->preempt_wq);
+
+	destroy_workqueue(guc->log.runtime.flush_wq);
+}
+
+static int guc_shared_data_create(struct intel_guc *guc)
+{
+	struct i915_vma *vma;
+	void *vaddr;
+
+	vma = intel_guc_allocate_vma(guc, PAGE_SIZE);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
+
+	vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		i915_vma_unpin_and_release(&vma);
+		return PTR_ERR(vaddr);
+	}
+
+	guc->shared_data = vma;
+	guc->shared_data_vaddr = vaddr;
+
+	return 0;
+}
+
+static void guc_shared_data_destroy(struct intel_guc *guc)
+{
+	i915_gem_object_unpin_map(guc->shared_data->obj);
+	i915_vma_unpin_and_release(&guc->shared_data);
+}
+
+int intel_guc_init(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	int ret;
+
+	ret = guc_shared_data_create(guc);
+	if (ret)
+		return ret;
+	GEM_BUG_ON(!guc->shared_data);
+
+	/* We need to notify the guc whenever we change the GGTT */
+	i915_ggtt_enable_guc(dev_priv);
+
+	return 0;
+}
+
+void intel_guc_fini(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	i915_ggtt_disable_guc(dev_priv);
+	guc_shared_data_destroy(guc);
+}
+
 static u32 get_gt_type(struct drm_i915_private *dev_priv)
 {
 	/* XXX: GT type based on PCI device ID? field seems unused by fw */
@@ -127,7 +237,7 @@ void intel_guc_init_params(struct intel_guc *guc)
 	}
 
 	/* If GuC submission is enabled, set up additional parameters here */
-	if (i915_modparams.enable_guc_submission) {
+	if (USES_GUC_SUBMISSION(dev_priv)) {
 		u32 ads = guc_ggtt_offset(guc->ads_vma) >> PAGE_SHIFT;
 		u32 pgs = guc_ggtt_offset(dev_priv->guc.stage_desc_pool);
 		u32 ctx_in_16 = GUC_MAX_STAGE_DESCRIPTORS / 16;
@@ -230,8 +340,7 @@ int intel_guc_sample_forcewake(struct intel_guc *guc)
 
 	action[0] = INTEL_GUC_ACTION_SAMPLE_FORCEWAKE;
 	/* WaRsDisableCoarsePowerGating:skl,bxt */
-	if (!intel_rc6_enabled() ||
-	    NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
+	if (!HAS_RC6(dev_priv) || NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
 		action[1] = 0;
 	else
 		/* bit 0 and 1 are for Render and Media domain separately */
@@ -268,7 +377,6 @@ int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset)
 int intel_guc_suspend(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
-	struct i915_gem_context *ctx;
 	u32 data[3];
 
 	if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
@@ -276,14 +384,33 @@ int intel_guc_suspend(struct drm_i915_private *dev_priv)
 
 	gen9_disable_guc_interrupts(dev_priv);
 
-	ctx = dev_priv->kernel_context;
-
 	data[0] = INTEL_GUC_ACTION_ENTER_S_STATE;
 	/* any value greater than GUC_POWER_D0 */
 	data[1] = GUC_POWER_D1;
-	/* first page is shared data with GuC */
-	data[2] = guc_ggtt_offset(ctx->engine[RCS].state) +
-		  LRC_GUCSHR_PN * PAGE_SIZE;
+	data[2] = guc_ggtt_offset(guc->shared_data);
+
+	return intel_guc_send(guc, data, ARRAY_SIZE(data));
+}
+
+/**
+ * intel_guc_reset_engine() - ask GuC to reset an engine
+ * @guc:	intel_guc structure
+ * @engine:	engine to be reset
+ */
+int intel_guc_reset_engine(struct intel_guc *guc,
+			   struct intel_engine_cs *engine)
+{
+	u32 data[7];
+
+	GEM_BUG_ON(!guc->execbuf_client);
+
+	data[0] = INTEL_GUC_ACTION_REQUEST_ENGINE_RESET;
+	data[1] = engine->guc_id;
+	data[2] = 0;
+	data[3] = 0;
+	data[4] = 0;
+	data[5] = guc->execbuf_client->stage_id;
+	data[6] = guc_ggtt_offset(guc->shared_data);
 
 	return intel_guc_send(guc, data, ARRAY_SIZE(data));
 }
@@ -295,7 +422,6 @@ int intel_guc_suspend(struct drm_i915_private *dev_priv)
 int intel_guc_resume(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
-	struct i915_gem_context *ctx;
 	u32 data[3];
 
 	if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
@@ -304,13 +430,9 @@ int intel_guc_resume(struct drm_i915_private *dev_priv)
 	if (i915_modparams.guc_log_level >= 0)
 		gen9_enable_guc_interrupts(dev_priv);
 
-	ctx = dev_priv->kernel_context;
-
 	data[0] = INTEL_GUC_ACTION_EXIT_S_STATE;
 	data[1] = GUC_POWER_D0;
-	/* first page is shared data with GuC */
-	data[2] = guc_ggtt_offset(ctx->engine[RCS].state) +
-		  LRC_GUCSHR_PN * PAGE_SIZE;
+	data[2] = guc_ggtt_offset(guc->shared_data);
 
 	return intel_guc_send(guc, data, ARRAY_SIZE(data));
 }
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 418450b1ae27..52856a97477d 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -30,13 +30,18 @@
 #include "intel_guc_fwif.h"
 #include "intel_guc_ct.h"
 #include "intel_guc_log.h"
+#include "intel_guc_reg.h"
 #include "intel_uc_fw.h"
-#include "i915_guc_reg.h"
 #include "i915_vma.h"
 
+struct guc_preempt_work {
+	struct work_struct work;
+	struct intel_engine_cs *engine;
+};
+
 /*
  * Top level structure of GuC. It handles firmware loading and manages client
- * pool and doorbells. intel_guc owns a i915_guc_client to replace the legacy
+ * pool and doorbells. intel_guc owns a intel_guc_client to replace the legacy
  * ExecList submission.
  */
 struct intel_guc {
@@ -54,8 +59,14 @@ struct intel_guc {
 	struct i915_vma *stage_desc_pool;
 	void *stage_desc_pool_vaddr;
 	struct ida stage_ids;
+	struct i915_vma *shared_data;
+	void *shared_data_vaddr;
+
+	struct intel_guc_client *execbuf_client;
+	struct intel_guc_client *preempt_client;
 
-	struct i915_guc_client *execbuf_client;
+	struct guc_preempt_work preempt_work[I915_NUM_ENGINES];
+	struct workqueue_struct *preempt_wq;
 
 	DECLARE_BITMAP(doorbell_bitmap, GUC_NUM_DOORBELLS);
 	/* Cyclic counter mod pagesize	*/
@@ -108,6 +119,10 @@ static inline u32 guc_ggtt_offset(struct i915_vma *vma)
 void intel_guc_init_early(struct intel_guc *guc);
 void intel_guc_init_send_regs(struct intel_guc *guc);
 void intel_guc_init_params(struct intel_guc *guc);
+int intel_guc_init_wq(struct intel_guc *guc);
+void intel_guc_fini_wq(struct intel_guc *guc);
+int intel_guc_init(struct intel_guc *guc);
+void intel_guc_fini(struct intel_guc *guc);
 int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len);
 int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len);
 int intel_guc_sample_forcewake(struct intel_guc *guc);
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.c b/drivers/gpu/drm/i915/intel_guc_ct.c
index c4cbec140101..24ad55752396 100644
--- a/drivers/gpu/drm/i915/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/intel_guc_ct.c
@@ -198,6 +198,7 @@ static int ctch_open(struct intel_guc *guc,
 		err = ctch_init(guc, ctch);
 		if (unlikely(err))
 			goto err_out;
+		GEM_BUG_ON(!ctch->vma);
 	}
 
 	/* vma should be already allocated and map'ed */
diff --git a/drivers/gpu/drm/i915/intel_guc_fw.c b/drivers/gpu/drm/i915/intel_guc_fw.c
index ef67a36354c5..3b0932942857 100644
--- a/drivers/gpu/drm/i915/intel_guc_fw.c
+++ b/drivers/gpu/drm/i915/intel_guc_fw.c
@@ -30,17 +30,14 @@
 #include "intel_guc_fw.h"
 #include "i915_drv.h"
 
-#define SKL_FW_MAJOR 6
-#define SKL_FW_MINOR 1
+#define SKL_FW_MAJOR 9
+#define SKL_FW_MINOR 33
 
-#define BXT_FW_MAJOR 8
-#define BXT_FW_MINOR 7
+#define BXT_FW_MAJOR 9
+#define BXT_FW_MINOR 29
 
 #define KBL_FW_MAJOR 9
-#define KBL_FW_MINOR 14
-
-#define GLK_FW_MAJOR 10
-#define GLK_FW_MINOR 56
+#define KBL_FW_MINOR 39
 
 #define GUC_FW_PATH(platform, major, minor) \
        "i915/" __stringify(platform) "_guc_ver" __stringify(major) "_" __stringify(minor) ".bin"
@@ -54,66 +51,96 @@ MODULE_FIRMWARE(I915_BXT_GUC_UCODE);
 #define I915_KBL_GUC_UCODE GUC_FW_PATH(kbl, KBL_FW_MAJOR, KBL_FW_MINOR)
 MODULE_FIRMWARE(I915_KBL_GUC_UCODE);
 
-#define I915_GLK_GUC_UCODE GUC_FW_PATH(glk, GLK_FW_MAJOR, GLK_FW_MINOR)
-
-/**
- * intel_guc_fw_select() - selects GuC firmware for uploading
- *
- * @guc:	intel_guc struct
- *
- * Return: zero when we know firmware, non-zero in other case
- */
-int intel_guc_fw_select(struct intel_guc *guc)
+static void guc_fw_select(struct intel_uc_fw *guc_fw)
 {
+	struct intel_guc *guc = container_of(guc_fw, struct intel_guc, fw);
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
-	intel_uc_fw_init(&guc->fw, INTEL_UC_FW_TYPE_GUC);
+	GEM_BUG_ON(guc_fw->type != INTEL_UC_FW_TYPE_GUC);
+
+	if (!HAS_GUC(dev_priv))
+		return;
 
 	if (i915_modparams.guc_firmware_path) {
-		guc->fw.path = i915_modparams.guc_firmware_path;
-		guc->fw.major_ver_wanted = 0;
-		guc->fw.minor_ver_wanted = 0;
+		guc_fw->path = i915_modparams.guc_firmware_path;
+		guc_fw->major_ver_wanted = 0;
+		guc_fw->minor_ver_wanted = 0;
 	} else if (IS_SKYLAKE(dev_priv)) {
-		guc->fw.path = I915_SKL_GUC_UCODE;
-		guc->fw.major_ver_wanted = SKL_FW_MAJOR;
-		guc->fw.minor_ver_wanted = SKL_FW_MINOR;
+		guc_fw->path = I915_SKL_GUC_UCODE;
+		guc_fw->major_ver_wanted = SKL_FW_MAJOR;
+		guc_fw->minor_ver_wanted = SKL_FW_MINOR;
 	} else if (IS_BROXTON(dev_priv)) {
-		guc->fw.path = I915_BXT_GUC_UCODE;
-		guc->fw.major_ver_wanted = BXT_FW_MAJOR;
-		guc->fw.minor_ver_wanted = BXT_FW_MINOR;
+		guc_fw->path = I915_BXT_GUC_UCODE;
+		guc_fw->major_ver_wanted = BXT_FW_MAJOR;
+		guc_fw->minor_ver_wanted = BXT_FW_MINOR;
 	} else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
-		guc->fw.path = I915_KBL_GUC_UCODE;
-		guc->fw.major_ver_wanted = KBL_FW_MAJOR;
-		guc->fw.minor_ver_wanted = KBL_FW_MINOR;
-	} else if (IS_GEMINILAKE(dev_priv)) {
-		guc->fw.path = I915_GLK_GUC_UCODE;
-		guc->fw.major_ver_wanted = GLK_FW_MAJOR;
-		guc->fw.minor_ver_wanted = GLK_FW_MINOR;
+		guc_fw->path = I915_KBL_GUC_UCODE;
+		guc_fw->major_ver_wanted = KBL_FW_MAJOR;
+		guc_fw->minor_ver_wanted = KBL_FW_MINOR;
 	} else {
-		DRM_ERROR("No GuC firmware known for platform with GuC!\n");
-		return -ENOENT;
+		DRM_WARN("%s: No firmware known for this platform!\n",
+			 intel_uc_fw_type_repr(guc_fw->type));
 	}
-
-	return 0;
 }
 
-/*
- * Read the GuC status register (GUC_STATUS) and store it in the
- * specified location; then return a boolean indicating whether
- * the value matches either of two values representing completion
- * of the GuC boot process.
+/**
+ * intel_guc_fw_init_early() - initializes GuC firmware struct
+ * @guc: intel_guc struct
  *
- * This is used for polling the GuC status in a wait_for()
- * loop below.
+ * On platforms with GuC selects firmware for uploading
  */
-static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
-				      u32 *status)
+void intel_guc_fw_init_early(struct intel_guc *guc)
 {
-	u32 val = I915_READ(GUC_STATUS);
-	u32 uk_val = val & GS_UKERNEL_MASK;
-	*status = val;
-	return (uk_val == GS_UKERNEL_READY ||
-		((val & GS_MIA_CORE_STATE) && uk_val == GS_UKERNEL_LAPIC_DONE));
+	struct intel_uc_fw *guc_fw = &guc->fw;
+
+	intel_uc_fw_init(guc_fw, INTEL_UC_FW_TYPE_GUC);
+	guc_fw_select(guc_fw);
+}
+
+static void guc_prepare_xfer(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	/* Must program this register before loading the ucode with DMA */
+	I915_WRITE(GUC_SHIM_CONTROL, GUC_DISABLE_SRAM_INIT_TO_ZEROES |
+				     GUC_ENABLE_READ_CACHE_LOGIC |
+				     GUC_ENABLE_MIA_CACHING |
+				     GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA |
+				     GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
+				     GUC_ENABLE_MIA_CLOCK_GATING);
+
+	if (IS_GEN9_LP(dev_priv))
+		I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
+	else
+		I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
+
+	if (IS_GEN9(dev_priv)) {
+		/* DOP Clock Gating Enable for GuC clocks */
+		I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
+					    I915_READ(GEN7_MISCCPCTL)));
+
+		/* allows for 5us (in 10ns units) before GT can go to RC6 */
+		I915_WRITE(GUC_ARAT_C6DIS, 0x1FF);
+	}
+}
+
+/* Copy RSA signature from the fw image to HW for verification */
+static int guc_xfer_rsa(struct intel_guc *guc, struct i915_vma *vma)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_uc_fw *guc_fw = &guc->fw;
+	struct sg_table *sg = vma->pages;
+	u32 rsa[UOS_RSA_SCRATCH_COUNT];
+	int i;
+
+	if (sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, sizeof(rsa),
+			       guc_fw->rsa_offset) != sizeof(rsa))
+		return -EINVAL;
+
+	for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++)
+		I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]);
+
+	return 0;
 }
 
 /*
@@ -122,29 +149,19 @@ static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
  * Architecturally, the DMA engine is bidirectional, and can potentially even
  * transfer between GTT locations. This functionality is left out of the API
  * for now as there is no need for it.
- *
- * Note that GuC needs the CSS header plus uKernel code to be copied by the
- * DMA engine in one operation, whereas the RSA signature is loaded via MMIO.
  */
-static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv,
-			      struct i915_vma *vma)
+static int guc_xfer_ucode(struct intel_guc *guc, struct i915_vma *vma)
 {
-	struct intel_uc_fw *guc_fw = &dev_priv->guc.fw;
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_uc_fw *guc_fw = &guc->fw;
 	unsigned long offset;
-	struct sg_table *sg = vma->pages;
-	u32 status, rsa[UOS_RSA_SCRATCH_MAX_COUNT];
-	int i, ret = 0;
-
-	/* where RSA signature starts */
-	offset = guc_fw->rsa_offset;
-
-	/* Copy RSA signature from the fw image to HW for verification */
-	sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, sizeof(rsa), offset);
-	for (i = 0; i < UOS_RSA_SCRATCH_MAX_COUNT; i++)
-		I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]);
+	u32 status;
+	int ret;
 
-	/* The header plus uCode will be copied to WOPCM via DMA, excluding any
-	 * other components */
+	/*
+	 * The header plus uCode will be copied to WOPCM via DMA, excluding any
+	 * other components
+	 */
 	I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size);
 
 	/* Set the source address for the new blob */
@@ -162,33 +179,62 @@ static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv,
 	/* Finally start the DMA */
 	I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(UOS_MOVE | START_DMA));
 
+	/* Wait for DMA to finish */
+	ret = __intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0,
+					   2, 100, &status);
+	DRM_DEBUG_DRIVER("GuC DMA status %#x\n", status);
+
+	return ret;
+}
+
+/*
+ * Read the GuC status register (GUC_STATUS) and store it in the
+ * specified location; then return a boolean indicating whether
+ * the value matches either of two values representing completion
+ * of the GuC boot process.
+ *
+ * This is used for polling the GuC status in a wait_for()
+ * loop below.
+ */
+static inline bool guc_ready(struct intel_guc *guc, u32 *status)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	u32 val = I915_READ(GUC_STATUS);
+	u32 uk_val = val & GS_UKERNEL_MASK;
+
+	*status = val;
+	return (uk_val == GS_UKERNEL_READY) ||
+		((val & GS_MIA_CORE_STATE) && (uk_val == GS_UKERNEL_LAPIC_DONE));
+}
+
+static int guc_wait_ucode(struct intel_guc *guc)
+{
+	u32 status;
+	int ret;
+
 	/*
-	 * Wait for the DMA to complete & the GuC to start up.
+	 * Wait for the GuC to start up.
 	 * NB: Docs recommend not using the interrupt for completion.
 	 * Measurements indicate this should take no more than 20ms, so a
 	 * timeout here indicates that the GuC has failed and is unusable.
 	 * (Higher levels of the driver will attempt to fall back to
 	 * execlist mode if this happens.)
 	 */
-	ret = wait_for(guc_ucode_response(dev_priv, &status), 100);
-
-	DRM_DEBUG_DRIVER("DMA status 0x%x, GuC status 0x%x\n",
-			I915_READ(DMA_CTRL), status);
+	ret = wait_for(guc_ready(guc, &status), 100);
+	DRM_DEBUG_DRIVER("GuC status %#x\n", status);
 
 	if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) {
 		DRM_ERROR("GuC firmware signature verification failed\n");
 		ret = -ENOEXEC;
 	}
 
-	DRM_DEBUG_DRIVER("returning %d\n", ret);
-
 	return ret;
 }
 
 /*
  * Load the GuC firmware blob into the MinuteIA.
  */
-static int guc_ucode_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
+static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
 {
 	struct intel_guc *guc = container_of(guc_fw, struct intel_guc, fw);
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
@@ -198,34 +244,24 @@ static int guc_ucode_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
 
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	/* Enable MIA caching. GuC clock gating is disabled. */
-	I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE);
-
-	/* WaDisableMinuteIaClockGating:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-		I915_WRITE(GUC_SHIM_CONTROL, (I915_READ(GUC_SHIM_CONTROL) &
-					      ~GUC_ENABLE_MIA_CLOCK_GATING));
-	}
-
-	/* WaC6DisallowByGfxPause:bxt */
-	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
-		I915_WRITE(GEN6_GFXPAUSE, 0x30FFF);
-
-	if (IS_GEN9_LP(dev_priv))
-		I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
-	else
-		I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
+	guc_prepare_xfer(guc);
 
-	if (IS_GEN9(dev_priv)) {
-		/* DOP Clock Gating Enable for GuC clocks */
-		I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
-					    I915_READ(GEN7_MISCCPCTL)));
+	/*
+	 * Note that GuC needs the CSS header plus uKernel code to be copied
+	 * by the DMA engine in one operation, whereas the RSA signature is
+	 * loaded via MMIO.
+	 */
+	ret = guc_xfer_rsa(guc, vma);
+	if (ret)
+		DRM_WARN("GuC firmware signature xfer error %d\n", ret);
 
-		/* allows for 5us (in 10ns units) before GT can go to RC6 */
-		I915_WRITE(GUC_ARAT_C6DIS, 0x1FF);
-	}
+	ret = guc_xfer_ucode(guc, vma);
+	if (ret)
+		DRM_WARN("GuC firmware code xfer error %d\n", ret);
 
-	ret = guc_ucode_xfer_dma(dev_priv, vma);
+	ret = guc_wait_ucode(guc);
+	if (ret)
+		DRM_ERROR("GuC firmware xfer error %d\n", ret);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
@@ -247,5 +283,5 @@ static int guc_ucode_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
  */
 int intel_guc_fw_upload(struct intel_guc *guc)
 {
-	return intel_uc_fw_upload(&guc->fw, guc_ucode_xfer);
+	return intel_uc_fw_upload(&guc->fw, guc_fw_xfer);
 }
diff --git a/drivers/gpu/drm/i915/intel_guc_fw.h b/drivers/gpu/drm/i915/intel_guc_fw.h
index 023f5baa9dd6..4ec5d3d9e2b0 100644
--- a/drivers/gpu/drm/i915/intel_guc_fw.h
+++ b/drivers/gpu/drm/i915/intel_guc_fw.h
@@ -27,7 +27,7 @@
 
 struct intel_guc;
 
-int intel_guc_fw_select(struct intel_guc *guc);
+void intel_guc_fw_init_early(struct intel_guc *guc);
 int intel_guc_fw_upload(struct intel_guc *guc);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index 80c507435458..6a10aa6f04d3 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -544,9 +544,37 @@ union guc_log_control {
 	u32 value;
 } __packed;
 
+struct guc_ctx_report {
+	u32 report_return_status;
+	u32 reserved1[64];
+	u32 affected_count;
+	u32 reserved2[2];
+} __packed;
+
+/* GuC Shared Context Data Struct */
+struct guc_shared_ctx_data {
+	u32 addr_of_last_preempted_data_low;
+	u32 addr_of_last_preempted_data_high;
+	u32 addr_of_last_preempted_data_high_tmp;
+	u32 padding;
+	u32 is_mapped_to_proxy;
+	u32 proxy_ctx_id;
+	u32 engine_reset_ctx_id;
+	u32 media_reset_count;
+	u32 reserved1[8];
+	u32 uk_last_ctx_switch_reason;
+	u32 was_reset;
+	u32 lrca_gpu_addr;
+	u64 execlist_ctx;
+	u32 reserved2[66];
+	struct guc_ctx_report preempt_ctx_report[GUC_MAX_ENGINES_NUM];
+} __packed;
+
 /* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */
 enum intel_guc_action {
 	INTEL_GUC_ACTION_DEFAULT = 0x0,
+	INTEL_GUC_ACTION_REQUEST_PREEMPTION = 0x2,
+	INTEL_GUC_ACTION_REQUEST_ENGINE_RESET = 0x3,
 	INTEL_GUC_ACTION_SAMPLE_FORCEWAKE = 0x6,
 	INTEL_GUC_ACTION_ALLOCATE_DOORBELL = 0x10,
 	INTEL_GUC_ACTION_DEALLOCATE_DOORBELL = 0x20,
@@ -562,6 +590,18 @@ enum intel_guc_action {
 	INTEL_GUC_ACTION_LIMIT
 };
 
+enum intel_guc_preempt_options {
+	INTEL_GUC_PREEMPT_OPTION_DROP_WORK_Q = 0x4,
+	INTEL_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q = 0x8,
+};
+
+enum intel_guc_report_status {
+	INTEL_GUC_REPORT_STATUS_UNKNOWN = 0x0,
+	INTEL_GUC_REPORT_STATUS_ACKED = 0x1,
+	INTEL_GUC_REPORT_STATUS_ERROR = 0x2,
+	INTEL_GUC_REPORT_STATUS_COMPLETE = 0x4,
+};
+
 /*
  * The GuC sends its response to a command by overwriting the
  * command in SS0. The response is distinguishable from a command
diff --git a/drivers/gpu/drm/i915/intel_guc_log.c b/drivers/gpu/drm/i915/intel_guc_log.c
index 76d3eb1e4614..eaedd63e3819 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/intel_guc_log.c
@@ -411,30 +411,8 @@ static int guc_log_runtime_create(struct intel_guc *guc)
 	guc->log.runtime.relay_chan = guc_log_relay_chan;
 
 	INIT_WORK(&guc->log.runtime.flush_work, capture_logs_work);
-
-	/*
-	 * GuC log buffer flush work item has to do register access to
-	 * send the ack to GuC and this work item, if not synced before
-	 * suspend, can potentially get executed after the GFX device is
-	 * suspended.
-	 * By marking the WQ as freezable, we don't have to bother about
-	 * flushing of this work item from the suspend hooks, the pending
-	 * work item if any will be either executed before the suspend
-	 * or scheduled later on resume. This way the handling of work
-	 * item can be kept same between system suspend & rpm suspend.
-	 */
-	guc->log.runtime.flush_wq = alloc_ordered_workqueue("i915-guc_log",
-						WQ_HIGHPRI | WQ_FREEZABLE);
-	if (!guc->log.runtime.flush_wq) {
-		DRM_ERROR("Couldn't allocate the wq for GuC logging\n");
-		ret = -ENOMEM;
-		goto err_relaychan;
-	}
-
 	return 0;
 
-err_relaychan:
-	relay_close(guc->log.runtime.relay_chan);
 err_vaddr:
 	i915_gem_object_unpin_map(guc->log.vma->obj);
 	guc->log.runtime.buf_addr = NULL;
@@ -450,7 +428,6 @@ static void guc_log_runtime_destroy(struct intel_guc *guc)
 	if (!guc_log_has_runtime(guc))
 		return;
 
-	destroy_workqueue(guc->log.runtime.flush_wq);
 	relay_close(guc->log.runtime.relay_chan);
 	i915_gem_object_unpin_map(guc->log.vma->obj);
 	guc->log.runtime.buf_addr = NULL;
@@ -505,7 +482,7 @@ static void guc_flush_logs(struct intel_guc *guc)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
-	if (!i915_modparams.enable_guc_submission ||
+	if (!USES_GUC_SUBMISSION(dev_priv) ||
 	    (i915_modparams.guc_log_level < 0))
 		return;
 
@@ -646,7 +623,7 @@ int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val)
 
 void i915_guc_log_register(struct drm_i915_private *dev_priv)
 {
-	if (!i915_modparams.enable_guc_submission ||
+	if (!USES_GUC_SUBMISSION(dev_priv) ||
 	    (i915_modparams.guc_log_level < 0))
 		return;
 
@@ -657,7 +634,7 @@ void i915_guc_log_register(struct drm_i915_private *dev_priv)
 
 void i915_guc_log_unregister(struct drm_i915_private *dev_priv)
 {
-	if (!i915_modparams.enable_guc_submission)
+	if (!USES_GUC_SUBMISSION(dev_priv))
 		return;
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/i915_guc_reg.h b/drivers/gpu/drm/i915/intel_guc_reg.h
similarity index 92%
rename from drivers/gpu/drm/i915/i915_guc_reg.h
rename to drivers/gpu/drm/i915/intel_guc_reg.h
index 35cf9918d09a..19a9247c5664 100644
--- a/drivers/gpu/drm/i915/i915_guc_reg.h
+++ b/drivers/gpu/drm/i915/intel_guc_reg.h
@@ -21,8 +21,8 @@
  * IN THE SOFTWARE.
  *
  */
-#ifndef _I915_GUC_REG_H_
-#define _I915_GUC_REG_H_
+#ifndef _INTEL_GUC_REG_H_
+#define _INTEL_GUC_REG_H_
 
 /* Definitions of GuC H/W registers, bits, etc */
 
@@ -52,7 +52,8 @@
 #define SOFT_SCRATCH_COUNT		16
 
 #define UOS_RSA_SCRATCH(i)		_MMIO(0xc200 + (i) * 4)
-#define   UOS_RSA_SCRATCH_MAX_COUNT	  64
+#define UOS_RSA_SCRATCH_COUNT		64
+
 #define DMA_ADDR_0_LOW			_MMIO(0xc300)
 #define DMA_ADDR_0_HIGH			_MMIO(0xc304)
 #define DMA_ADDR_1_LOW			_MMIO(0xc308)
@@ -102,13 +103,6 @@
 #define   GUC_ENABLE_MIA_CLOCK_GATING		(1<<15)
 #define   GUC_GEN10_SHIM_WC_ENABLE		(1<<21)
 
-#define GUC_SHIM_CONTROL_VALUE	(GUC_DISABLE_SRAM_INIT_TO_ZEROES	| \
-				 GUC_ENABLE_READ_CACHE_LOGIC		| \
-				 GUC_ENABLE_MIA_CACHING			| \
-				 GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA	| \
-				 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA	| \
-				 GUC_ENABLE_MIA_CLOCK_GATING)
-
 #define GUC_SEND_INTERRUPT		_MMIO(0xc4c8)
 #define   GUC_SEND_TRIGGER		  (1<<0)
 
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c
similarity index 63%
rename from drivers/gpu/drm/i915/i915_guc_submission.c
rename to drivers/gpu/drm/i915/intel_guc_submission.c
index f84c267728fd..4d2409466a3a 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/intel_guc_submission.c
@@ -25,23 +25,24 @@
 #include <linux/circ_buf.h>
 #include <trace/events/dma_fence.h>
 
-#include "i915_guc_submission.h"
+#include "intel_guc_submission.h"
 #include "i915_drv.h"
 
 /**
  * DOC: GuC-based command submission
  *
  * GuC client:
- * A i915_guc_client refers to a submission path through GuC. Currently, there
- * is only one of these (the execbuf_client) and this one is charged with all
- * submissions to the GuC. This struct is the owner of a doorbell, a process
- * descriptor and a workqueue (all of them inside a single gem object that
- * contains all required pages for these elements).
+ * A intel_guc_client refers to a submission path through GuC. Currently, there
+ * are two clients. One of them (the execbuf_client) is charged with all
+ * submissions to the GuC, the other one (preempt_client) is responsible for
+ * preempting the execbuf_client. This struct is the owner of a doorbell, a
+ * process descriptor and a workqueue (all of them inside a single gem object
+ * that contains all required pages for these elements).
  *
  * GuC stage descriptor:
  * During initialization, the driver allocates a static pool of 1024 such
  * descriptors, and shares them with the GuC.
- * Currently, there exists a 1:1 mapping between a i915_guc_client and a
+ * Currently, there exists a 1:1 mapping between a intel_guc_client and a
  * guc_stage_desc (via the client's stage_id), so effectively only one
  * gets used. This stage descriptor lets the GuC know about the doorbell,
  * workqueue and process descriptor. Theoretically, it also lets the GuC
@@ -70,7 +71,7 @@
  * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which
  * represents in-order queue. The kernel driver packs ring tail pointer and an
  * ELSP context descriptor dword into Work Item.
- * See guc_wq_item_append()
+ * See guc_add_request()
  *
  * ADS:
  * The Additional Data Struct (ADS) has pointers for different buffers used by
@@ -81,12 +82,13 @@
  *
  */
 
-static inline bool is_high_priority(struct i915_guc_client* client)
+static inline bool is_high_priority(struct intel_guc_client *client)
 {
-	return client->priority <= GUC_CLIENT_PRIORITY_HIGH;
+	return (client->priority == GUC_CLIENT_PRIORITY_KMD_HIGH ||
+		client->priority == GUC_CLIENT_PRIORITY_HIGH);
 }
 
-static int __reserve_doorbell(struct i915_guc_client *client)
+static int reserve_doorbell(struct intel_guc_client *client)
 {
 	unsigned long offset;
 	unsigned long end;
@@ -100,7 +102,7 @@ static int __reserve_doorbell(struct i915_guc_client *client)
 	 * priority contexts, the second half for high-priority ones.
 	 */
 	offset = 0;
-	end = GUC_NUM_DOORBELLS/2;
+	end = GUC_NUM_DOORBELLS / 2;
 	if (is_high_priority(client)) {
 		offset = end;
 		end += offset;
@@ -118,7 +120,7 @@ static int __reserve_doorbell(struct i915_guc_client *client)
 	return 0;
 }
 
-static void __unreserve_doorbell(struct i915_guc_client *client)
+static void unreserve_doorbell(struct intel_guc_client *client)
 {
 	GEM_BUG_ON(client->doorbell_id == GUC_DOORBELL_INVALID);
 
@@ -150,7 +152,7 @@ static int __guc_deallocate_doorbell(struct intel_guc *guc, u32 stage_id)
 	return intel_guc_send(guc, action, ARRAY_SIZE(action));
 }
 
-static struct guc_stage_desc *__get_stage_desc(struct i915_guc_client *client)
+static struct guc_stage_desc *__get_stage_desc(struct intel_guc_client *client)
 {
 	struct guc_stage_desc *base = client->guc->stage_desc_pool_vaddr;
 
@@ -164,7 +166,7 @@ static struct guc_stage_desc *__get_stage_desc(struct i915_guc_client *client)
  * client object which contains the page being used for the doorbell
  */
 
-static void __update_doorbell_desc(struct i915_guc_client *client, u16 new_id)
+static void __update_doorbell_desc(struct intel_guc_client *client, u16 new_id)
 {
 	struct guc_stage_desc *desc;
 
@@ -173,12 +175,12 @@ static void __update_doorbell_desc(struct i915_guc_client *client, u16 new_id)
 	desc->db_id = new_id;
 }
 
-static struct guc_doorbell_info *__get_doorbell(struct i915_guc_client *client)
+static struct guc_doorbell_info *__get_doorbell(struct intel_guc_client *client)
 {
 	return client->vaddr + client->doorbell_offset;
 }
 
-static bool has_doorbell(struct i915_guc_client *client)
+static bool has_doorbell(struct intel_guc_client *client)
 {
 	if (client->doorbell_id == GUC_DOORBELL_INVALID)
 		return false;
@@ -186,29 +188,21 @@ static bool has_doorbell(struct i915_guc_client *client)
 	return test_bit(client->doorbell_id, client->guc->doorbell_bitmap);
 }
 
-static int __create_doorbell(struct i915_guc_client *client)
+static void __create_doorbell(struct intel_guc_client *client)
 {
 	struct guc_doorbell_info *doorbell;
-	int err;
 
 	doorbell = __get_doorbell(client);
 	doorbell->db_status = GUC_DOORBELL_ENABLED;
 	doorbell->cookie = 0;
-
-	err = __guc_allocate_doorbell(client->guc, client->stage_id);
-	if (err)
-		doorbell->db_status = GUC_DOORBELL_DISABLED;
-
-	return err;
 }
 
-static int __destroy_doorbell(struct i915_guc_client *client)
+static void __destroy_doorbell(struct intel_guc_client *client)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(client->guc);
 	struct guc_doorbell_info *doorbell;
 	u16 db_id = client->doorbell_id;
 
-	GEM_BUG_ON(db_id >= GUC_DOORBELL_INVALID);
 
 	doorbell = __get_doorbell(client);
 	doorbell->db_status = GUC_DOORBELL_DISABLED;
@@ -216,56 +210,49 @@ static int __destroy_doorbell(struct i915_guc_client *client)
 
 	/* Doorbell release flow requires that we wait for GEN8_DRB_VALID bit
 	 * to go to zero after updating db_status before we call the GuC to
-	 * release the doorbell */
+	 * release the doorbell
+	 */
 	if (wait_for_us(!(I915_READ(GEN8_DRBREGL(db_id)) & GEN8_DRB_VALID), 10))
 		WARN_ONCE(true, "Doorbell never became invalid after disable\n");
-
-	return __guc_deallocate_doorbell(client->guc, client->stage_id);
 }
 
-static int create_doorbell(struct i915_guc_client *client)
+static int create_doorbell(struct intel_guc_client *client)
 {
 	int ret;
 
-	ret = __reserve_doorbell(client);
-	if (ret)
-		return ret;
-
 	__update_doorbell_desc(client, client->doorbell_id);
-
-	ret = __create_doorbell(client);
-	if (ret)
-		goto err;
+	__create_doorbell(client);
+
+	ret = __guc_allocate_doorbell(client->guc, client->stage_id);
+	if (ret) {
+		__destroy_doorbell(client);
+		__update_doorbell_desc(client, GUC_DOORBELL_INVALID);
+		DRM_ERROR("Couldn't create client %u doorbell: %d\n",
+			  client->stage_id, ret);
+		return ret;
+	}
 
 	return 0;
-
-err:
-	__update_doorbell_desc(client, GUC_DOORBELL_INVALID);
-	__unreserve_doorbell(client);
-	return ret;
 }
 
-static int destroy_doorbell(struct i915_guc_client *client)
+static int destroy_doorbell(struct intel_guc_client *client)
 {
-	int err;
+	int ret;
 
 	GEM_BUG_ON(!has_doorbell(client));
 
-	/* XXX: wait for any interrupts */
-	/* XXX: wait for workqueue to drain */
-
-	err = __destroy_doorbell(client);
-	if (err)
-		return err;
+	__destroy_doorbell(client);
+	ret = __guc_deallocate_doorbell(client->guc, client->stage_id);
+	if (ret)
+		DRM_ERROR("Couldn't destroy client %u doorbell: %d\n",
+			  client->stage_id, ret);
 
 	__update_doorbell_desc(client, GUC_DOORBELL_INVALID);
 
-	__unreserve_doorbell(client);
-
-	return 0;
+	return ret;
 }
 
-static unsigned long __select_cacheline(struct intel_guc* guc)
+static unsigned long __select_cacheline(struct intel_guc *guc)
 {
 	unsigned long offset;
 
@@ -276,12 +263,12 @@ static unsigned long __select_cacheline(struct intel_guc* guc)
 	guc->db_cacheline += cache_line_size();
 
 	DRM_DEBUG_DRIVER("reserved cacheline 0x%lx, next 0x%x, linesize %u\n",
-			offset, guc->db_cacheline, cache_line_size());
+			 offset, guc->db_cacheline, cache_line_size());
 	return offset;
 }
 
 static inline struct guc_process_desc *
-__get_process_desc(struct i915_guc_client *client)
+__get_process_desc(struct intel_guc_client *client)
 {
 	return client->vaddr + client->proc_desc_offset;
 }
@@ -290,7 +277,7 @@ __get_process_desc(struct i915_guc_client *client)
  * Initialise the process descriptor shared with the GuC firmware.
  */
 static void guc_proc_desc_init(struct intel_guc *guc,
-			       struct i915_guc_client *client)
+			       struct intel_guc_client *client)
 {
 	struct guc_process_desc *desc;
 
@@ -311,6 +298,37 @@ static void guc_proc_desc_init(struct intel_guc *guc,
 	desc->priority = client->priority;
 }
 
+static int guc_stage_desc_pool_create(struct intel_guc *guc)
+{
+	struct i915_vma *vma;
+	void *vaddr;
+
+	vma = intel_guc_allocate_vma(guc,
+				     PAGE_ALIGN(sizeof(struct guc_stage_desc) *
+				     GUC_MAX_STAGE_DESCRIPTORS));
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
+
+	vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		i915_vma_unpin_and_release(&vma);
+		return PTR_ERR(vaddr);
+	}
+
+	guc->stage_desc_pool = vma;
+	guc->stage_desc_pool_vaddr = vaddr;
+	ida_init(&guc->stage_ids);
+
+	return 0;
+}
+
+static void guc_stage_desc_pool_destroy(struct intel_guc *guc)
+{
+	ida_destroy(&guc->stage_ids);
+	i915_gem_object_unpin_map(guc->stage_desc_pool->obj);
+	i915_vma_unpin_and_release(&guc->stage_desc_pool);
+}
+
 /*
  * Initialise/clear the stage descriptor shared with the GuC firmware.
  *
@@ -319,7 +337,7 @@ static void guc_proc_desc_init(struct intel_guc *guc,
  * write queue, etc).
  */
 static void guc_stage_desc_init(struct intel_guc *guc,
-				struct i915_guc_client *client)
+				struct intel_guc_client *client)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	struct intel_engine_cs *engine;
@@ -331,7 +349,10 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 	desc = __get_stage_desc(client);
 	memset(desc, 0, sizeof(*desc));
 
-	desc->attribute = GUC_STAGE_DESC_ATTR_ACTIVE | GUC_STAGE_DESC_ATTR_KERNEL;
+	desc->attribute = GUC_STAGE_DESC_ATTR_ACTIVE |
+			  GUC_STAGE_DESC_ATTR_KERNEL;
+	if (is_high_priority(client))
+		desc->attribute |= GUC_STAGE_DESC_ATTR_PREEMPT;
 	desc->stage_id = client->stage_id;
 	desc->priority = client->priority;
 	desc->db_id = client->doorbell_id;
@@ -356,7 +377,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 		 * submission or, in other words, not using a direct submission
 		 * model) the KMD's LRCA is not used for any work submission.
 		 * Instead, the GuC uses the LRCA of the user mode context (see
-		 * guc_wq_item_append below).
+		 * guc_add_request below).
 		 */
 		lrc->context_desc = lower_32_bits(ce->lrc_desc);
 
@@ -365,7 +386,8 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 			guc_ggtt_offset(ce->state) + LRC_STATE_PN * PAGE_SIZE;
 
 		/* XXX: In direct submission, the GuC wants the HW context id
-		 * here. In proxy submission, it wants the stage id */
+		 * here. In proxy submission, it wants the stage id
+		 */
 		lrc->context_id = (client->stage_id << GUC_ELC_CTXID_OFFSET) |
 				(guc_engine_id << GUC_ELC_ENGINE_OFFSET);
 
@@ -378,7 +400,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 	}
 
 	DRM_DEBUG_DRIVER("Host engines 0x%x => GuC engines used 0x%x\n",
-			client->engines, desc->engines_used);
+			 client->engines, desc->engines_used);
 	WARN_ON(desc->engines_used == 0);
 
 	/*
@@ -398,7 +420,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 }
 
 static void guc_stage_desc_fini(struct intel_guc *guc,
-				struct i915_guc_client *client)
+				struct intel_guc_client *client)
 {
 	struct guc_stage_desc *desc;
 
@@ -407,23 +429,19 @@ static void guc_stage_desc_fini(struct intel_guc *guc,
 }
 
 /* Construct a Work Item and append it to the GuC's Work Queue */
-static void guc_wq_item_append(struct i915_guc_client *client,
-			       struct drm_i915_gem_request *rq)
+static void guc_wq_item_append(struct intel_guc_client *client,
+			       u32 target_engine, u32 context_desc,
+			       u32 ring_tail, u32 fence_id)
 {
 	/* wqi_len is in DWords, and does not include the one-word header */
 	const size_t wqi_size = sizeof(struct guc_wq_item);
 	const u32 wqi_len = wqi_size / sizeof(u32) - 1;
-	struct intel_engine_cs *engine = rq->engine;
-	struct i915_gem_context *ctx = rq->ctx;
 	struct guc_process_desc *desc = __get_process_desc(client);
 	struct guc_wq_item *wqi;
-	u32 ring_tail, wq_off;
+	u32 wq_off;
 
 	lockdep_assert_held(&client->wq_lock);
 
-	ring_tail = intel_ring_set_tail(rq->ring, rq->tail) / sizeof(u64);
-	GEM_BUG_ON(ring_tail > WQ_RING_TAIL_MAX);
-
 	/* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
 	 * should not have the case where structure wqi is across page, neither
 	 * wrapped to the beginning. This simplifies the implementation below.
@@ -445,19 +463,18 @@ static void guc_wq_item_append(struct i915_guc_client *client,
 	/* Now fill in the 4-word work queue item */
 	wqi->header = WQ_TYPE_INORDER |
 		      (wqi_len << WQ_LEN_SHIFT) |
-		      (engine->guc_id << WQ_TARGET_SHIFT) |
+		      (target_engine << WQ_TARGET_SHIFT) |
 		      WQ_NO_WCFLUSH_WAIT;
-
-	wqi->context_desc = lower_32_bits(intel_lr_context_descriptor(ctx, engine));
-
+	wqi->context_desc = context_desc;
 	wqi->submit_element_info = ring_tail << WQ_RING_TAIL_SHIFT;
-	wqi->fence_id = rq->global_seqno;
+	GEM_BUG_ON(ring_tail > WQ_RING_TAIL_MAX);
+	wqi->fence_id = fence_id;
 
-	/* Postincrement WQ tail for next time. */
+	/* Make the update visible to GuC */
 	WRITE_ONCE(desc->tail, (wq_off + wqi_size) & (GUC_WQ_SIZE - 1));
 }
 
-static void guc_reset_wq(struct i915_guc_client *client)
+static void guc_reset_wq(struct intel_guc_client *client)
 {
 	struct guc_process_desc *desc = __get_process_desc(client);
 
@@ -465,7 +482,7 @@ static void guc_reset_wq(struct i915_guc_client *client)
 	desc->tail = 0;
 }
 
-static void guc_ring_doorbell(struct i915_guc_client *client)
+static void guc_ring_doorbell(struct intel_guc_client *client)
 {
 	struct guc_doorbell_info *db;
 	u32 cookie;
@@ -475,29 +492,166 @@ static void guc_ring_doorbell(struct i915_guc_client *client)
 	/* pointer of current doorbell cacheline */
 	db = __get_doorbell(client);
 
-	/* we're not expecting the doorbell cookie to change behind our back */
+	/*
+	 * We're not expecting the doorbell cookie to change behind our back,
+	 * we also need to treat 0 as a reserved value.
+	 */
 	cookie = READ_ONCE(db->cookie);
-	WARN_ON_ONCE(xchg(&db->cookie, cookie + 1) != cookie);
+	WARN_ON_ONCE(xchg(&db->cookie, cookie + 1 ?: cookie + 2) != cookie);
 
 	/* XXX: doorbell was lost and need to acquire it again */
 	GEM_BUG_ON(db->db_status != GUC_DOORBELL_ENABLED);
 }
 
+static void guc_add_request(struct intel_guc *guc,
+			    struct drm_i915_gem_request *rq)
+{
+	struct intel_guc_client *client = guc->execbuf_client;
+	struct intel_engine_cs *engine = rq->engine;
+	u32 ctx_desc = lower_32_bits(intel_lr_context_descriptor(rq->ctx,
+								 engine));
+	u32 ring_tail = intel_ring_set_tail(rq->ring, rq->tail) / sizeof(u64);
+
+	spin_lock(&client->wq_lock);
+
+	guc_wq_item_append(client, engine->guc_id, ctx_desc,
+			   ring_tail, rq->global_seqno);
+	guc_ring_doorbell(client);
+
+	client->submissions[engine->id] += 1;
+
+	spin_unlock(&client->wq_lock);
+}
+
+/*
+ * When we're doing submissions using regular execlists backend, writing to
+ * ELSP from CPU side is enough to make sure that writes to ringbuffer pages
+ * pinned in mappable aperture portion of GGTT are visible to command streamer.
+ * Writes done by GuC on our behalf are not guaranteeing such ordering,
+ * therefore, to ensure the flush, we're issuing a POSTING READ.
+ */
+static void flush_ggtt_writes(struct i915_vma *vma)
+{
+	struct drm_i915_private *dev_priv = to_i915(vma->obj->base.dev);
+
+	if (i915_vma_is_map_and_fenceable(vma))
+		POSTING_READ_FW(GUC_STATUS);
+}
+
+#define GUC_PREEMPT_FINISHED 0x1
+#define GUC_PREEMPT_BREADCRUMB_DWORDS 0x8
+static void inject_preempt_context(struct work_struct *work)
+{
+	struct guc_preempt_work *preempt_work =
+		container_of(work, typeof(*preempt_work), work);
+	struct intel_engine_cs *engine = preempt_work->engine;
+	struct intel_guc *guc = container_of(preempt_work, typeof(*guc),
+					     preempt_work[engine->id]);
+	struct intel_guc_client *client = guc->preempt_client;
+	struct guc_stage_desc *stage_desc = __get_stage_desc(client);
+	struct intel_ring *ring = client->owner->engine[engine->id].ring;
+	u32 ctx_desc = lower_32_bits(intel_lr_context_descriptor(client->owner,
+								 engine));
+	u32 *cs = ring->vaddr + ring->tail;
+	u32 data[7];
+
+	if (engine->id == RCS) {
+		cs = gen8_emit_ggtt_write_rcs(cs, GUC_PREEMPT_FINISHED,
+				intel_hws_preempt_done_address(engine));
+	} else {
+		cs = gen8_emit_ggtt_write(cs, GUC_PREEMPT_FINISHED,
+				intel_hws_preempt_done_address(engine));
+		*cs++ = MI_NOOP;
+		*cs++ = MI_NOOP;
+	}
+	*cs++ = MI_USER_INTERRUPT;
+	*cs++ = MI_NOOP;
+
+	GEM_BUG_ON(!IS_ALIGNED(ring->size,
+			       GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32)));
+	GEM_BUG_ON((void *)cs - (ring->vaddr + ring->tail) !=
+		   GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32));
+
+	ring->tail += GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32);
+	ring->tail &= (ring->size - 1);
+
+	flush_ggtt_writes(ring->vma);
+
+	spin_lock_irq(&client->wq_lock);
+	guc_wq_item_append(client, engine->guc_id, ctx_desc,
+			   ring->tail / sizeof(u64), 0);
+	spin_unlock_irq(&client->wq_lock);
+
+	/*
+	 * If GuC firmware performs an engine reset while that engine had
+	 * a preemption pending, it will set the terminated attribute bit
+	 * on our preemption stage descriptor. GuC firmware retains all
+	 * pending work items for a high-priority GuC client, unlike the
+	 * normal-priority GuC client where work items are dropped. It
+	 * wants to make sure the preempt-to-idle work doesn't run when
+	 * scheduling resumes, and uses this bit to inform its scheduler
+	 * and presumably us as well. Our job is to clear it for the next
+	 * preemption after reset, otherwise that and future preemptions
+	 * will never complete. We'll just clear it every time.
+	 */
+	stage_desc->attribute &= ~GUC_STAGE_DESC_ATTR_TERMINATED;
+
+	data[0] = INTEL_GUC_ACTION_REQUEST_PREEMPTION;
+	data[1] = client->stage_id;
+	data[2] = INTEL_GUC_PREEMPT_OPTION_DROP_WORK_Q |
+		  INTEL_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q;
+	data[3] = engine->guc_id;
+	data[4] = guc->execbuf_client->priority;
+	data[5] = guc->execbuf_client->stage_id;
+	data[6] = guc_ggtt_offset(guc->shared_data);
+
+	if (WARN_ON(intel_guc_send(guc, data, ARRAY_SIZE(data)))) {
+		execlists_clear_active(&engine->execlists,
+				       EXECLISTS_ACTIVE_PREEMPT);
+		tasklet_schedule(&engine->execlists.tasklet);
+	}
+}
+
+/*
+ * We're using user interrupt and HWSP value to mark that preemption has
+ * finished and GPU is idle. Normally, we could unwind and continue similar to
+ * execlists submission path. Unfortunately, with GuC we also need to wait for
+ * it to finish its own postprocessing, before attempting to submit. Otherwise
+ * GuC may silently ignore our submissions, and thus we risk losing request at
+ * best, executing out-of-order and causing kernel panic at worst.
+ */
+#define GUC_PREEMPT_POSTPROCESS_DELAY_MS 10
+static void wait_for_guc_preempt_report(struct intel_engine_cs *engine)
+{
+	struct intel_guc *guc = &engine->i915->guc;
+	struct guc_shared_ctx_data *data = guc->shared_data_vaddr;
+	struct guc_ctx_report *report =
+		&data->preempt_ctx_report[engine->guc_id];
+
+	WARN_ON(wait_for_atomic(report->report_return_status ==
+				INTEL_GUC_REPORT_STATUS_COMPLETE,
+				GUC_PREEMPT_POSTPROCESS_DELAY_MS));
+	/*
+	 * GuC is expecting that we're also going to clear the affected context
+	 * counter, let's also reset the return status to not depend on GuC
+	 * resetting it after recieving another preempt action
+	 */
+	report->affected_count = 0;
+	report->report_return_status = INTEL_GUC_REPORT_STATUS_UNKNOWN;
+}
+
 /**
- * i915_guc_submit() - Submit commands through GuC
+ * guc_submit() - Submit commands through GuC
  * @engine: engine associated with the commands
  *
  * The only error here arises if the doorbell hardware isn't functioning
  * as expected, which really shouln't happen.
  */
-static void i915_guc_submit(struct intel_engine_cs *engine)
+static void guc_submit(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	struct intel_guc *guc = &dev_priv->guc;
-	struct i915_guc_client *client = guc->execbuf_client;
+	struct intel_guc *guc = &engine->i915->guc;
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
-	const unsigned int engine_id = engine->id;
 	unsigned int n;
 
 	for (n = 0; n < execlists_num_ports(execlists); n++) {
@@ -508,57 +662,22 @@ static void i915_guc_submit(struct intel_engine_cs *engine)
 		if (rq && count == 0) {
 			port_set(&port[n], port_pack(rq, ++count));
 
-			if (i915_vma_is_map_and_fenceable(rq->ring->vma))
-				POSTING_READ_FW(GUC_STATUS);
+			flush_ggtt_writes(rq->ring->vma);
 
-			spin_lock(&client->wq_lock);
-
-			guc_wq_item_append(client, rq);
-			guc_ring_doorbell(client);
-
-			client->submissions[engine_id] += 1;
-
-			spin_unlock(&client->wq_lock);
+			guc_add_request(guc, rq);
 		}
 	}
 }
 
-static void nested_enable_signaling(struct drm_i915_gem_request *rq)
-{
-	/* If we use dma_fence_enable_sw_signaling() directly, lockdep
-	 * detects an ordering issue between the fence lockclass and the
-	 * global_timeline. This circular dependency can only occur via 2
-	 * different fences (but same fence lockclass), so we use the nesting
-	 * annotation here to prevent the warn, equivalent to the nesting
-	 * inside i915_gem_request_submit() for when we also enable the
-	 * signaler.
-	 */
-
-	if (test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
-			     &rq->fence.flags))
-		return;
-
-	GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags));
-	trace_dma_fence_enable_signal(&rq->fence);
-
-	spin_lock_nested(&rq->lock, SINGLE_DEPTH_NESTING);
-	intel_engine_enable_signaling(rq, true);
-	spin_unlock(&rq->lock);
-}
-
 static void port_assign(struct execlist_port *port,
 			struct drm_i915_gem_request *rq)
 {
-	GEM_BUG_ON(rq == port_request(port));
-
-	if (port_isset(port))
-		i915_gem_request_put(port_request(port));
+	GEM_BUG_ON(port_isset(port));
 
-	port_set(port, port_pack(i915_gem_request_get(rq), port_count(port)));
-	nested_enable_signaling(rq);
+	port_set(port, i915_gem_request_get(rq));
 }
 
-static void i915_guc_dequeue(struct intel_engine_cs *engine)
+static void guc_dequeue(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
@@ -568,13 +687,35 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine)
 	bool submit = false;
 	struct rb_node *rb;
 
-	if (port_isset(port))
-		port++;
-
 	spin_lock_irq(&engine->timeline->lock);
 	rb = execlists->first;
 	GEM_BUG_ON(rb_first(&execlists->queue) != rb);
-	while (rb) {
+
+	if (!rb)
+		goto unlock;
+
+	if (port_isset(port)) {
+		if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) {
+			struct guc_preempt_work *preempt_work =
+				&engine->i915->guc.preempt_work[engine->id];
+
+			if (rb_entry(rb, struct i915_priolist, node)->priority >
+			    max(port_request(port)->priotree.priority, 0)) {
+				execlists_set_active(execlists,
+						     EXECLISTS_ACTIVE_PREEMPT);
+				queue_work(engine->i915->guc.preempt_wq,
+					   &preempt_work->work);
+				goto unlock;
+			}
+		}
+
+		port++;
+		if (port_isset(port))
+			goto unlock;
+	}
+	GEM_BUG_ON(port_isset(port));
+
+	do {
 		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
 		struct drm_i915_gem_request *rq, *rn;
 
@@ -592,10 +733,10 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine)
 			}
 
 			INIT_LIST_HEAD(&rq->priotree.link);
-			rq->priotree.priority = INT_MAX;
 
 			__i915_gem_request_submit(rq);
-			trace_i915_gem_request_in(rq, port_index(port, execlists));
+			trace_i915_gem_request_in(rq,
+						  port_index(port, execlists));
 			last = rq;
 			submit = true;
 		}
@@ -605,24 +746,23 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine)
 		INIT_LIST_HEAD(&p->requests);
 		if (p->priority != I915_PRIORITY_NORMAL)
 			kmem_cache_free(engine->i915->priorities, p);
-	}
+	} while (rb);
 done:
 	execlists->first = rb;
 	if (submit) {
 		port_assign(port, last);
 		execlists_set_active(execlists, EXECLISTS_ACTIVE_USER);
-		i915_guc_submit(engine);
+		guc_submit(engine);
 	}
+unlock:
 	spin_unlock_irq(&engine->timeline->lock);
 }
 
-static void i915_guc_irq_handler(unsigned long data)
+static void guc_submission_tasklet(unsigned long data)
 {
 	struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
-	const struct execlist_port * const last_port =
-		&execlists->port[execlists->port_mask];
 	struct drm_i915_gem_request *rq;
 
 	rq = port_request(&port[0]);
@@ -637,14 +777,26 @@ static void i915_guc_irq_handler(unsigned long data)
 	if (!rq)
 		execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
 
-	if (!port_isset(last_port))
-		i915_guc_dequeue(engine);
+	if (execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT) &&
+	    intel_read_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX) ==
+	    GUC_PREEMPT_FINISHED) {
+		execlists_cancel_port_requests(&engine->execlists);
+		execlists_unwind_incomplete_requests(execlists);
+
+		wait_for_guc_preempt_report(engine);
+
+		execlists_clear_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
+		intel_write_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX, 0);
+	}
+
+	if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT))
+		guc_dequeue(engine);
 }
 
 /*
  * Everything below here is concerned with setup & teardown, and is
  * therefore not part of the somewhat time-critical batch-submission
- * path of i915_guc_submit() above.
+ * path of guc_submit() above.
  */
 
 /* Check that a doorbell register is in the expected state */
@@ -668,101 +820,67 @@ static bool doorbell_ok(struct intel_guc *guc, u16 db_id)
 	return false;
 }
 
-/*
- * If the GuC thinks that the doorbell is unassigned (e.g. because we reset and
- * reloaded the GuC FW) we can use this function to tell the GuC to reassign the
- * doorbell to the rightful owner.
- */
-static int __reset_doorbell(struct i915_guc_client* client, u16 db_id)
+static bool guc_verify_doorbells(struct intel_guc *guc)
 {
-	int err;
+	u16 db_id;
 
-	__update_doorbell_desc(client, db_id);
-	err = __create_doorbell(client);
-	if (!err)
-		err = __destroy_doorbell(client);
+	for (db_id = 0; db_id < GUC_NUM_DOORBELLS; ++db_id)
+		if (!doorbell_ok(guc, db_id))
+			return false;
 
-	return err;
+	return true;
 }
 
-/*
- * Set up & tear down each unused doorbell in turn, to ensure that all doorbell
- * HW is (re)initialised. For that end, we might have to borrow the first
- * client. Also, tell GuC about all the doorbells in use by all clients.
- * We do this because the KMD, the GuC and the doorbell HW can easily go out of
- * sync (e.g. we can reset the GuC, but not the doorbel HW).
- */
-static int guc_init_doorbell_hw(struct intel_guc *guc)
+static int guc_clients_doorbell_init(struct intel_guc *guc)
 {
-	struct i915_guc_client *client = guc->execbuf_client;
-	bool recreate_first_client = false;
-	u16 db_id;
 	int ret;
 
-	/* For unused doorbells, make sure they are disabled */
-	for_each_clear_bit(db_id, guc->doorbell_bitmap, GUC_NUM_DOORBELLS) {
-		if (doorbell_ok(guc, db_id))
-			continue;
-
-		if (has_doorbell(client)) {
-			/* Borrow execbuf_client (we will recreate it later) */
-			destroy_doorbell(client);
-			recreate_first_client = true;
-		}
-
-		ret = __reset_doorbell(client, db_id);
-		WARN(ret, "Doorbell %u reset failed, err %d\n", db_id, ret);
-	}
-
-	if (recreate_first_client) {
-		ret = __reserve_doorbell(client);
-		if (unlikely(ret)) {
-			DRM_ERROR("Couldn't re-reserve first client db: %d\n", ret);
-			return ret;
-		}
-
-		__update_doorbell_desc(client, client->doorbell_id);
-	}
+	ret = create_doorbell(guc->execbuf_client);
+	if (ret)
+		return ret;
 
-	/* Now for every client (and not only execbuf_client) make sure their
-	 * doorbells are known by the GuC */
-	//for (client = client_list; client != NULL; client = client->next)
-	{
-		ret = __create_doorbell(client);
-		if (ret) {
-			DRM_ERROR("Couldn't recreate client %u doorbell: %d\n",
-				client->stage_id, ret);
-			return ret;
-		}
+	ret = create_doorbell(guc->preempt_client);
+	if (ret) {
+		destroy_doorbell(guc->execbuf_client);
+		return ret;
 	}
 
-	/* Read back & verify all (used & unused) doorbell registers */
-	for (db_id = 0; db_id < GUC_NUM_DOORBELLS; ++db_id)
-		WARN_ON(!doorbell_ok(guc, db_id));
-
 	return 0;
 }
 
+static void guc_clients_doorbell_fini(struct intel_guc *guc)
+{
+	/*
+	 * By the time we're here, GuC has already been reset.
+	 * Instead of trying (in vain) to communicate with it, let's just
+	 * cleanup the doorbell HW and our internal state.
+	 */
+	__destroy_doorbell(guc->preempt_client);
+	__update_doorbell_desc(guc->preempt_client, GUC_DOORBELL_INVALID);
+	__destroy_doorbell(guc->execbuf_client);
+	__update_doorbell_desc(guc->execbuf_client, GUC_DOORBELL_INVALID);
+}
+
 /**
- * guc_client_alloc() - Allocate an i915_guc_client
+ * guc_client_alloc() - Allocate an intel_guc_client
  * @dev_priv:	driver private data structure
  * @engines:	The set of engines to enable for this client
  * @priority:	four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW
- * 		The kernel client to replace ExecList submission is created with
- * 		NORMAL priority. Priority of a client for scheduler can be HIGH,
- * 		while a preemption context can use CRITICAL.
+ *		The kernel client to replace ExecList submission is created with
+ *		NORMAL priority. Priority of a client for scheduler can be HIGH,
+ *		while a preemption context can use CRITICAL.
  * @ctx:	the context that owns the client (we use the default render
- * 		context)
+ *		context)
  *
- * Return:	An i915_guc_client object if success, else NULL.
+ * Return:	An intel_guc_client object if success, else NULL.
  */
-static struct i915_guc_client *
+static struct intel_guc_client *
 guc_client_alloc(struct drm_i915_private *dev_priv,
 		 u32 engines,
 		 u32 priority,
 		 struct i915_gem_context *ctx)
 {
-	struct i915_guc_client *client;
+	struct intel_guc_client *client;
 	struct intel_guc *guc = &dev_priv->guc;
 	struct i915_vma *vma;
 	void *vaddr;
@@ -780,7 +898,7 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
 	spin_lock_init(&client->wq_lock);
 
 	ret = ida_simple_get(&guc->stage_ids, 0, GUC_MAX_STAGE_DESCRIPTORS,
-				GFP_KERNEL);
+			     GFP_KERNEL);
 	if (ret < 0)
 		goto err_client;
 
@@ -818,7 +936,7 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
 	guc_proc_desc_init(guc, client);
 	guc_stage_desc_init(guc, client);
 
-	ret = create_doorbell(client);
+	ret = reserve_doorbell(client);
 	if (ret)
 		goto err_vaddr;
 
@@ -840,17 +958,9 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
 	return ERR_PTR(ret);
 }
 
-static void guc_client_free(struct i915_guc_client *client)
+static void guc_client_free(struct intel_guc_client *client)
 {
-	/*
-	 * XXX: wait for any outstanding submissions before freeing memory.
-	 * Be sure to drop any locks
-	 */
-
-	/* FIXME: in many cases, by the time we get here the GuC has been
-	 * reset, so we cannot destroy the doorbell properly. Ignore the
-	 * error message for now */
-	destroy_doorbell(client);
+	unreserve_doorbell(client);
 	guc_stage_desc_fini(client->guc, client);
 	i915_gem_object_unpin_map(client->vma->obj);
 	i915_vma_unpin_and_release(&client->vma);
@@ -858,6 +968,50 @@ static void guc_client_free(struct i915_guc_client *client)
 	kfree(client);
 }
 
+static int guc_clients_create(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_guc_client *client;
+
+	GEM_BUG_ON(guc->execbuf_client);
+	GEM_BUG_ON(guc->preempt_client);
+
+	client = guc_client_alloc(dev_priv,
+				  INTEL_INFO(dev_priv)->ring_mask,
+				  GUC_CLIENT_PRIORITY_KMD_NORMAL,
+				  dev_priv->kernel_context);
+	if (IS_ERR(client)) {
+		DRM_ERROR("Failed to create GuC client for submission!\n");
+		return PTR_ERR(client);
+	}
+	guc->execbuf_client = client;
+
+	client = guc_client_alloc(dev_priv,
+				  INTEL_INFO(dev_priv)->ring_mask,
+				  GUC_CLIENT_PRIORITY_KMD_HIGH,
+				  dev_priv->preempt_context);
+	if (IS_ERR(client)) {
+		DRM_ERROR("Failed to create GuC client for preemption!\n");
+		guc_client_free(guc->execbuf_client);
+		guc->execbuf_client = NULL;
+		return PTR_ERR(client);
+	}
+	guc->preempt_client = client;
+
+	return 0;
+}
+
+static void guc_clients_destroy(struct intel_guc *guc)
+{
+	struct intel_guc_client *client;
+
+	client = fetch_and_zero(&guc->execbuf_client);
+	guc_client_free(client);
+
+	client = fetch_and_zero(&guc->preempt_client);
+	guc_client_free(client);
+}
+
 static void guc_policy_init(struct guc_policy *policy)
 {
 	policy->execution_quantum = POLICY_DEFAULT_EXECUTION_QUANTUM_US;
@@ -941,7 +1095,8 @@ static int guc_ads_create(struct intel_guc *guc)
 	 * because our GuC shared data is there.
 	 */
 	blob->ads.golden_context_lrca =
-		guc_ggtt_offset(dev_priv->kernel_context->engine[RCS].state) + skipped_offset;
+		guc_ggtt_offset(dev_priv->kernel_context->engine[RCS].state) +
+		skipped_offset;
 
 	/*
 	 * The GuC expects us to exclude the portion of the context image that
@@ -950,7 +1105,8 @@ static int guc_ads_create(struct intel_guc *guc)
 	 * dwords). Weird guc is weird.
 	 */
 	for_each_engine(engine, dev_priv, id)
-		blob->ads.eng_state_size[engine->guc_id] = engine->context_size - skipped_size;
+		blob->ads.eng_state_size[engine->guc_id] =
+			engine->context_size - skipped_size;
 
 	base = guc_ggtt_offset(vma);
 	blob->ads.scheduler_policies = base + ptr_offset(blob, policies);
@@ -971,62 +1127,68 @@ static void guc_ads_destroy(struct intel_guc *guc)
  * Set up the memory resources to be shared with the GuC (via the GGTT)
  * at firmware loading time.
  */
-int i915_guc_submission_init(struct drm_i915_private *dev_priv)
+int intel_guc_submission_init(struct intel_guc *guc)
 {
-	struct intel_guc *guc = &dev_priv->guc;
-	struct i915_vma *vma;
-	void *vaddr;
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int ret;
 
 	if (guc->stage_desc_pool)
 		return 0;
 
-	vma = intel_guc_allocate_vma(guc,
-				PAGE_ALIGN(sizeof(struct guc_stage_desc) *
-					GUC_MAX_STAGE_DESCRIPTORS));
-	if (IS_ERR(vma))
-		return PTR_ERR(vma);
-
-	guc->stage_desc_pool = vma;
-
-	vaddr = i915_gem_object_pin_map(guc->stage_desc_pool->obj, I915_MAP_WB);
-	if (IS_ERR(vaddr)) {
-		ret = PTR_ERR(vaddr);
-		goto err_vma;
-	}
-
-	guc->stage_desc_pool_vaddr = vaddr;
+	ret = guc_stage_desc_pool_create(guc);
+	if (ret)
+		return ret;
+	/*
+	 * Keep static analysers happy, let them know that we allocated the
+	 * vma after testing that it didn't exist earlier.
+	 */
+	GEM_BUG_ON(!guc->stage_desc_pool);
 
 	ret = intel_guc_log_create(guc);
 	if (ret < 0)
-		goto err_vaddr;
+		goto err_stage_desc_pool;
 
 	ret = guc_ads_create(guc);
 	if (ret < 0)
 		goto err_log;
+	GEM_BUG_ON(!guc->ads_vma);
 
-	ida_init(&guc->stage_ids);
+	WARN_ON(!guc_verify_doorbells(guc));
+	ret = guc_clients_create(guc);
+	if (ret)
+		return ret;
+
+	for_each_engine(engine, dev_priv, id) {
+		guc->preempt_work[id].engine = engine;
+		INIT_WORK(&guc->preempt_work[id].work, inject_preempt_context);
+	}
 
 	return 0;
 
 err_log:
 	intel_guc_log_destroy(guc);
-err_vaddr:
-	i915_gem_object_unpin_map(guc->stage_desc_pool->obj);
-err_vma:
-	i915_vma_unpin_and_release(&guc->stage_desc_pool);
+err_stage_desc_pool:
+	guc_stage_desc_pool_destroy(guc);
 	return ret;
 }
 
-void i915_guc_submission_fini(struct drm_i915_private *dev_priv)
+void intel_guc_submission_fini(struct intel_guc *guc)
 {
-	struct intel_guc *guc = &dev_priv->guc;
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	for_each_engine(engine, dev_priv, id)
+		cancel_work_sync(&guc->preempt_work[id].work);
+
+	guc_clients_destroy(guc);
+	WARN_ON(!guc_verify_doorbells(guc));
 
-	ida_destroy(&guc->stage_ids);
 	guc_ads_destroy(guc);
 	intel_guc_log_destroy(guc);
-	i915_gem_object_unpin_map(guc->stage_desc_pool->obj);
-	i915_vma_unpin_and_release(&guc->stage_desc_pool);
+	guc_stage_desc_pool_destroy(guc);
 }
 
 static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
@@ -1036,7 +1198,9 @@ static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
 	enum intel_engine_id id;
 	int irqs;
 
-	/* tell all command streamers to forward interrupts (but not vblank) to GuC */
+	/* tell all command streamers to forward interrupts (but not vblank)
+	 * to GuC
+	 */
 	irqs = _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
 	for_each_engine(engine, dev_priv, id)
 		I915_WRITE(RING_MODE_GEN7(engine), irqs);
@@ -1097,10 +1261,19 @@ static void guc_interrupts_release(struct drm_i915_private *dev_priv)
 	rps->pm_intrmsk_mbz &= ~ARAT_EXPIRED_INTRMSK;
 }
 
-int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
+static void guc_submission_park(struct intel_engine_cs *engine)
 {
-	struct intel_guc *guc = &dev_priv->guc;
-	struct i915_guc_client *client = guc->execbuf_client;
+	intel_engine_unpin_breadcrumbs_irq(engine);
+}
+
+static void guc_submission_unpark(struct intel_engine_cs *engine)
+{
+	intel_engine_pin_breadcrumbs_irq(engine);
+}
+
+int intel_guc_submission_enable(struct intel_guc *guc)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 	int err;
@@ -1118,61 +1291,49 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
 		     sizeof(struct guc_wq_item) *
 		     I915_NUM_ENGINES > GUC_WQ_SIZE);
 
-	if (!client) {
-		client = guc_client_alloc(dev_priv,
-					  INTEL_INFO(dev_priv)->ring_mask,
-					  GUC_CLIENT_PRIORITY_KMD_NORMAL,
-					  dev_priv->kernel_context);
-		if (IS_ERR(client)) {
-			DRM_ERROR("Failed to create GuC client for execbuf!\n");
-			return PTR_ERR(client);
-		}
+	GEM_BUG_ON(!guc->execbuf_client);
 
-		guc->execbuf_client = client;
-	}
+	guc_reset_wq(guc->execbuf_client);
+	guc_reset_wq(guc->preempt_client);
 
 	err = intel_guc_sample_forcewake(guc);
 	if (err)
-		goto err_execbuf_client;
-
-	guc_reset_wq(client);
+		return err;
 
-	err = guc_init_doorbell_hw(guc);
+	err = guc_clients_doorbell_init(guc);
 	if (err)
-		goto err_execbuf_client;
+		return err;
 
 	/* Take over from manual control of ELSP (execlists) */
 	guc_interrupts_capture(dev_priv);
 
 	for_each_engine(engine, dev_priv, id) {
-		struct intel_engine_execlists * const execlists = &engine->execlists;
-		/* The tasklet was initialised by execlists, and may be in
-		 * a state of flux (across a reset) and so we just want to
-		 * take over the callback without changing any other state
-		 * in the tasklet.
-		 */
-		execlists->irq_tasklet.func = i915_guc_irq_handler;
-		clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-		tasklet_schedule(&execlists->irq_tasklet);
+		struct intel_engine_execlists * const execlists =
+			&engine->execlists;
+
+		execlists->tasklet.func = guc_submission_tasklet;
+		engine->park = guc_submission_park;
+		engine->unpark = guc_submission_unpark;
+
+		engine->flags &= ~I915_ENGINE_SUPPORTS_STATS;
 	}
 
 	return 0;
-
-err_execbuf_client:
-	guc_client_free(guc->execbuf_client);
-	guc->execbuf_client = NULL;
-	return err;
 }
 
-void i915_guc_submission_disable(struct drm_i915_private *dev_priv)
+void intel_guc_submission_disable(struct intel_guc *guc)
 {
-	struct intel_guc *guc = &dev_priv->guc;
+	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+	GEM_BUG_ON(dev_priv->gt.awake); /* GT should be parked first */
 
 	guc_interrupts_release(dev_priv);
+	guc_clients_doorbell_fini(guc);
 
 	/* Revert back to manual ELSP submission */
 	intel_engines_reset_default_submission(dev_priv);
-
-	guc_client_free(guc->execbuf_client);
-	guc->execbuf_client = NULL;
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/intel_guc.c"
+#endif
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.h b/drivers/gpu/drm/i915/intel_guc_submission.h
similarity index 85%
rename from drivers/gpu/drm/i915/i915_guc_submission.h
rename to drivers/gpu/drm/i915/intel_guc_submission.h
index cb4353b59059..fb081cefef93 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.h
+++ b/drivers/gpu/drm/i915/intel_guc_submission.h
@@ -22,8 +22,8 @@
  *
  */
 
-#ifndef _I915_GUC_SUBMISSION_H_
-#define _I915_GUC_SUBMISSION_H_
+#ifndef _INTEL_GUC_SUBMISSION_H_
+#define _INTEL_GUC_SUBMISSION_H_
 
 #include <linux/spinlock.h>
 
@@ -52,7 +52,7 @@ struct drm_i915_private;
  * queue (a circular array of work items), again described in the process
  * descriptor. Work queue pages are mapped momentarily as required.
  */
-struct i915_guc_client {
+struct intel_guc_client {
 	struct i915_vma *vma;
 	void *vaddr;
 	struct i915_gem_context *owner;
@@ -67,14 +67,17 @@ struct i915_guc_client {
 	u16 doorbell_id;
 	unsigned long doorbell_offset;
 
+	/* Protects GuC client's WQ access */
 	spinlock_t wq_lock;
 	/* Per-engine counts of GuC submissions */
 	u64 submissions[I915_NUM_ENGINES];
 };
 
-int i915_guc_submission_init(struct drm_i915_private *dev_priv);
-int i915_guc_submission_enable(struct drm_i915_private *dev_priv);
-void i915_guc_submission_disable(struct drm_i915_private *dev_priv);
-void i915_guc_submission_fini(struct drm_i915_private *dev_priv);
+int intel_guc_submission_init(struct intel_guc *guc);
+int intel_guc_submission_enable(struct intel_guc *guc);
+void intel_guc_submission_disable(struct intel_guc *guc);
+void intel_guc_submission_fini(struct intel_guc *guc);
+int intel_guc_preempt_work_create(struct intel_guc *guc);
+void intel_guc_preempt_work_destroy(struct intel_guc *guc);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
index b4a7f31f0214..a2fe7c8d4477 100644
--- a/drivers/gpu/drm/i915/intel_gvt.c
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -95,12 +95,7 @@ int intel_gvt_init(struct drm_i915_private *dev_priv)
 		return 0;
 	}
 
-	if (!i915_modparams.enable_execlists) {
-		DRM_ERROR("i915 GVT-g loading failed due to disabled execlists mode\n");
-		return -EIO;
-	}
-
-	if (i915_modparams.enable_guc_submission) {
+	if (USES_GUC_SUBMISSION(dev_priv)) {
 		DRM_ERROR("i915 GVT-g loading failed due to Graphics virtualization is not yet supported with GuC submission\n");
 		return -EIO;
 	}
diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c
index 12ac270a5f93..53747318f4a7 100644
--- a/drivers/gpu/drm/i915/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/intel_hangcheck.c
@@ -27,13 +27,9 @@
 static bool
 ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr)
 {
-	if (INTEL_GEN(engine->i915) >= 8) {
-		return (ipehr >> 23) == 0x1c;
-	} else {
-		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
-		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
-				 MI_SEMAPHORE_REGISTER);
-	}
+	ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
+	return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
+			 MI_SEMAPHORE_REGISTER);
 }
 
 static struct intel_engine_cs *
@@ -41,31 +37,20 @@ semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
 				 u64 offset)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
+	u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
 	struct intel_engine_cs *signaller;
 	enum intel_engine_id id;
 
-	if (INTEL_GEN(dev_priv) >= 8) {
-		for_each_engine(signaller, dev_priv, id) {
-			if (engine == signaller)
-				continue;
-
-			if (offset == signaller->semaphore.signal_ggtt[engine->hw_id])
-				return signaller;
-		}
-	} else {
-		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
+	for_each_engine(signaller, dev_priv, id) {
+		if (engine == signaller)
+			continue;
 
-		for_each_engine(signaller, dev_priv, id) {
-			if(engine == signaller)
-				continue;
-
-			if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id])
-				return signaller;
-		}
+		if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id])
+			return signaller;
 	}
 
-	DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n",
-			 engine->name, ipehr, offset);
+	DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x\n",
+			 engine->name, ipehr);
 
 	return ERR_PTR(-ENODEV);
 }
@@ -135,11 +120,6 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
 		return NULL;
 
 	*seqno = ioread32(vaddr + head + 4) + 1;
-	if (INTEL_GEN(dev_priv) >= 8) {
-		offset = ioread32(vaddr + head + 12);
-		offset <<= 32;
-		offset |= ioread32(vaddr + head + 8);
-	}
 	return semaphore_wait_to_signaller_ring(engine, ipehr, offset);
 }
 
@@ -266,19 +246,19 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd)
 	 */
 	tmp = I915_READ_CTL(engine);
 	if (tmp & RING_WAIT) {
-		i915_handle_error(dev_priv, 0,
+		i915_handle_error(dev_priv, BIT(engine->id),
 				  "Kicking stuck wait on %s",
 				  engine->name);
 		I915_WRITE_CTL(engine, tmp);
 		return ENGINE_WAIT_KICK;
 	}
 
-	if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) {
+	if (IS_GEN(dev_priv, 6, 7) && tmp & RING_WAIT_SEMAPHORE) {
 		switch (semaphore_passed(engine)) {
 		default:
 			return ENGINE_DEAD;
 		case 1:
-			i915_handle_error(dev_priv, 0,
+			i915_handle_error(dev_priv, ALL_ENGINES,
 					  "Kicking stuck semaphore on %s",
 					  engine->name);
 			I915_WRITE_CTL(engine, tmp);
@@ -369,13 +349,18 @@ static void hangcheck_accumulate_sample(struct intel_engine_cs *engine,
 
 	case ENGINE_ACTIVE_HEAD:
 	case ENGINE_ACTIVE_SUBUNITS:
-		/* Seqno stuck with still active engine gets leeway,
+		/*
+		 * Seqno stuck with still active engine gets leeway,
 		 * in hopes that it is just a long shader.
 		 */
 		timeout = I915_SEQNO_DEAD_TIMEOUT;
 		break;
 
 	case ENGINE_DEAD:
+		if (drm_debug & DRM_UT_DRIVER) {
+			struct drm_printer p = drm_debug_printer("hangcheck");
+			intel_engine_dump(engine, &p, "%s", engine->name);
+		}
 		break;
 
 	default:
@@ -426,7 +411,6 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 	unsigned int hung = 0, stuck = 0;
-	int busy_count = 0;
 
 	if (!i915_modparams.enable_hangcheck)
 		return;
@@ -444,30 +428,26 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
 
 	for_each_engine(engine, dev_priv, id) {
-		struct intel_engine_hangcheck cur_state, *hc = &cur_state;
-		const bool busy = intel_engine_has_waiter(engine);
+		struct intel_engine_hangcheck hc;
 
 		semaphore_clear_deadlocks(dev_priv);
 
-		hangcheck_load_sample(engine, hc);
-		hangcheck_accumulate_sample(engine, hc);
-		hangcheck_store_sample(engine, hc);
+		hangcheck_load_sample(engine, &hc);
+		hangcheck_accumulate_sample(engine, &hc);
+		hangcheck_store_sample(engine, &hc);
 
 		if (engine->hangcheck.stalled) {
 			hung |= intel_engine_flag(engine);
-			if (hc->action != ENGINE_DEAD)
+			if (hc.action != ENGINE_DEAD)
 				stuck |= intel_engine_flag(engine);
 		}
-
-		busy_count += busy;
 	}
 
 	if (hung)
 		hangcheck_declare_hang(dev_priv, hung, stuck);
 
 	/* Reset timer in case GPU hangs without another request being added */
-	if (busy_count)
-		i915_queue_hangcheck(dev_priv);
+	i915_queue_hangcheck(dev_priv);
 }
 
 void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 4dea833f9d1b..179d0ad3889d 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -186,7 +186,7 @@ static bool g4x_infoframe_enabled(struct drm_encoder *encoder,
 	if ((val & VIDEO_DIP_ENABLE) == 0)
 		return false;
 
-	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port))
+	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port))
 		return false;
 
 	return val & (VIDEO_DIP_ENABLE_AVI |
@@ -245,7 +245,7 @@ static bool ibx_infoframe_enabled(struct drm_encoder *encoder,
 	if ((val & VIDEO_DIP_ENABLE) == 0)
 		return false;
 
-	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port))
+	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port))
 		return false;
 
 	return val & (VIDEO_DIP_ENABLE_AVI |
@@ -362,7 +362,7 @@ static bool vlv_infoframe_enabled(struct drm_encoder *encoder,
 	if ((val & VIDEO_DIP_ENABLE) == 0)
 		return false;
 
-	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port))
+	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port))
 		return false;
 
 	return val & (VIDEO_DIP_ENABLE_AVI |
@@ -513,12 +513,14 @@ static void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder,
 
 static void
 intel_hdmi_set_hdmi_infoframe(struct drm_encoder *encoder,
-			      const struct intel_crtc_state *crtc_state)
+			      const struct intel_crtc_state *crtc_state,
+			      const struct drm_connector_state *conn_state)
 {
 	union hdmi_infoframe frame;
 	int ret;
 
 	ret = drm_hdmi_vendor_infoframe_from_display_mode(&frame.vendor.hdmi,
+							  conn_state->connector,
 							  &crtc_state->base.adjusted_mode);
 	if (ret < 0)
 		return;
@@ -536,7 +538,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
 	struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
 	i915_reg_t reg = VIDEO_DIP_CTL;
 	u32 val = I915_READ(reg);
-	u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
+	u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port);
 
 	assert_hdmi_port_disabled(intel_hdmi);
 
@@ -585,7 +587,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
 
 	intel_hdmi_set_avi_infoframe(encoder, crtc_state);
 	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state);
+	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
 }
 
 static bool hdmi_sink_is_deep_color(const struct drm_connector_state *conn_state)
@@ -687,7 +689,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
 	struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
 	i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
-	u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
+	u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port);
 
 	assert_hdmi_port_disabled(intel_hdmi);
 
@@ -726,7 +728,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
 
 	intel_hdmi_set_avi_infoframe(encoder, crtc_state);
 	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state);
+	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
 }
 
 static void cpt_set_infoframes(struct drm_encoder *encoder,
@@ -769,7 +771,7 @@ static void cpt_set_infoframes(struct drm_encoder *encoder,
 
 	intel_hdmi_set_avi_infoframe(encoder, crtc_state);
 	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state);
+	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
 }
 
 static void vlv_set_infoframes(struct drm_encoder *encoder,
@@ -783,7 +785,7 @@ static void vlv_set_infoframes(struct drm_encoder *encoder,
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
 	i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
 	u32 val = I915_READ(reg);
-	u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
+	u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port);
 
 	assert_hdmi_port_disabled(intel_hdmi);
 
@@ -822,7 +824,7 @@ static void vlv_set_infoframes(struct drm_encoder *encoder,
 
 	intel_hdmi_set_avi_infoframe(encoder, crtc_state);
 	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state);
+	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
 }
 
 static void hsw_set_infoframes(struct drm_encoder *encoder,
@@ -855,7 +857,7 @@ static void hsw_set_infoframes(struct drm_encoder *encoder,
 
 	intel_hdmi_set_avi_infoframe(encoder, crtc_state);
 	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state);
+	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
 }
 
 void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable)
@@ -958,6 +960,8 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 	u32 tmp, flags = 0;
 	int dotclock;
 
+	pipe_config->output_types |= BIT(INTEL_OUTPUT_HDMI);
+
 	tmp = I915_READ(intel_hdmi->hdmi_reg);
 
 	if (tmp & SDVO_HSYNC_ACTIVE_HIGH)
@@ -1205,7 +1209,8 @@ static void g4x_disable_hdmi(struct intel_encoder *encoder,
 			     const struct drm_connector_state *old_conn_state)
 {
 	if (old_crtc_state->has_audio)
-		intel_audio_codec_disable(encoder);
+		intel_audio_codec_disable(encoder,
+					  old_crtc_state, old_conn_state);
 
 	intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
 }
@@ -1215,7 +1220,8 @@ static void pch_disable_hdmi(struct intel_encoder *encoder,
 			     const struct drm_connector_state *old_conn_state)
 {
 	if (old_crtc_state->has_audio)
-		intel_audio_codec_disable(encoder);
+		intel_audio_codec_disable(encoder,
+					  old_crtc_state, old_conn_state);
 }
 
 static void pch_post_disable_hdmi(struct intel_encoder *encoder,
@@ -1225,24 +1231,34 @@ static void pch_post_disable_hdmi(struct intel_encoder *encoder,
 	intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
 }
 
-static int intel_hdmi_source_max_tmds_clock(struct drm_i915_private *dev_priv)
+static int intel_hdmi_source_max_tmds_clock(struct intel_encoder *encoder)
 {
-	if (IS_G4X(dev_priv))
-		return 165000;
-	else if (IS_GEMINILAKE(dev_priv))
-		return 594000;
-	else if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8)
-		return 300000;
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	const struct ddi_vbt_port_info *info =
+		&dev_priv->vbt.ddi_port_info[encoder->port];
+	int max_tmds_clock;
+
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+		max_tmds_clock = 594000;
+	else if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv))
+		max_tmds_clock = 300000;
+	else if (INTEL_GEN(dev_priv) >= 5)
+		max_tmds_clock = 225000;
 	else
-		return 225000;
+		max_tmds_clock = 165000;
+
+	if (info->max_tmds_clock)
+		max_tmds_clock = min(max_tmds_clock, info->max_tmds_clock);
+
+	return max_tmds_clock;
 }
 
 static int hdmi_port_clock_limit(struct intel_hdmi *hdmi,
 				 bool respect_downstream_limits,
 				 bool force_dvi)
 {
-	struct drm_device *dev = intel_hdmi_to_dev(hdmi);
-	int max_tmds_clock = intel_hdmi_source_max_tmds_clock(to_i915(dev));
+	struct intel_encoder *encoder = &hdmi_to_dig_port(hdmi)->base;
+	int max_tmds_clock = intel_hdmi_source_max_tmds_clock(encoder);
 
 	if (respect_downstream_limits) {
 		struct intel_connector *connector = hdmi->attached_connector;
@@ -1337,6 +1353,12 @@ static bool hdmi_12bpc_possible(const struct intel_crtc_state *crtc_state)
 	if (HAS_GMCH_DISPLAY(dev_priv))
 		return false;
 
+	if (crtc_state->pipe_bpp <= 8*3)
+		return false;
+
+	if (!crtc_state->has_hdmi_sink)
+		return false;
+
 	/*
 	 * HDMI 12bpc affects the clocks, so it's only possible
 	 * when not cloning with other encoder types.
@@ -1361,7 +1383,7 @@ static bool hdmi_12bpc_possible(const struct intel_crtc_state *crtc_state)
 		}
 	}
 
-	/* Display Wa #1139 */
+	/* Display WA #1139: glk */
 	if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1) &&
 	    crtc_state->base.adjusted_mode.htotal > 5460)
 		return false;
@@ -1462,9 +1484,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 	 * outputs. We also need to check that the higher clock still fits
 	 * within limits.
 	 */
-	if (pipe_config->pipe_bpp > 8*3 && pipe_config->has_hdmi_sink && !force_dvi &&
-	    hdmi_port_clock_valid(intel_hdmi, clock_12bpc, true, force_dvi) == MODE_OK &&
-	    hdmi_12bpc_possible(pipe_config)) {
+	if (hdmi_12bpc_possible(pipe_config) &&
+	    hdmi_port_clock_valid(intel_hdmi, clock_12bpc, true, force_dvi) == MODE_OK) {
 		DRM_DEBUG_KMS("picking bpc to 12 for HDMI output\n");
 		desired_bpp = 12*3;
 
@@ -1493,7 +1514,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 
 	pipe_config->lane_count = 4;
 
-	if (scdc->scrambling.supported && IS_GEMINILAKE(dev_priv)) {
+	if (scdc->scrambling.supported && (INTEL_GEN(dev_priv) >= 10 ||
+					   IS_GEMINILAKE(dev_priv))) {
 		if (scdc->scrambling.low_rates)
 			pipe_config->hdmi_scrambling = true;
 
@@ -1527,7 +1549,7 @@ intel_hdmi_dp_dual_mode_detect(struct drm_connector *connector, bool has_edid)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
 	struct intel_hdmi *hdmi = intel_attached_hdmi(connector);
-	enum port port = hdmi_to_dig_port(hdmi)->port;
+	enum port port = hdmi_to_dig_port(hdmi)->base.port;
 	struct i2c_adapter *adapter =
 		intel_gmbus_get_adapter(dev_priv, hdmi->ddc_bus);
 	enum drm_dp_dual_mode_type type = drm_dp_dual_mode_detect(adapter);
@@ -1573,12 +1595,20 @@ intel_hdmi_set_edid(struct drm_connector *connector)
 	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
 	struct edid *edid;
 	bool connected = false;
+	struct i2c_adapter *i2c;
 
 	intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
 
-	edid = drm_get_edid(connector,
-			    intel_gmbus_get_adapter(dev_priv,
-			    intel_hdmi->ddc_bus));
+	i2c = intel_gmbus_get_adapter(dev_priv, intel_hdmi->ddc_bus);
+
+	edid = drm_get_edid(connector, i2c);
+
+	if (!edid && !intel_gmbus_is_forced_bit(i2c)) {
+		DRM_DEBUG_KMS("HDMI GMBUS EDID read failed, retry using GPIO bit-banging\n");
+		intel_gmbus_force_bit(i2c, true);
+		edid = drm_get_edid(connector, i2c);
+		intel_gmbus_force_bit(i2c, false);
+	}
 
 	intel_hdmi_dp_dual_mode_detect(connector, edid != NULL);
 
@@ -1611,12 +1641,9 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 
 	intel_hdmi_unset_edid(connector);
 
-	if (intel_hdmi_set_edid(connector)) {
-		struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
-
-		hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
+	if (intel_hdmi_set_edid(connector))
 		status = connector_status_connected;
-	} else
+	else
 		status = connector_status_disconnected;
 
 	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
@@ -1627,8 +1654,6 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 static void
 intel_hdmi_force(struct drm_connector *connector)
 {
-	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
-
 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
 		      connector->base.id, connector->name);
 
@@ -1638,7 +1663,6 @@ intel_hdmi_force(struct drm_connector *connector)
 		return;
 
 	intel_hdmi_set_edid(connector);
-	hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
 }
 
 static int intel_hdmi_get_modes(struct drm_connector *connector)
@@ -1671,10 +1695,9 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder,
 				const struct drm_connector_state *conn_state)
 {
 	struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-	struct drm_device *dev = encoder->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	vlv_phy_pre_encoder_enable(encoder);
+	vlv_phy_pre_encoder_enable(encoder, pipe_config);
 
 	/* HDMI 1.0V-2dB */
 	vlv_set_phy_signal_level(encoder, 0x2b245f5f, 0x00002000, 0x5578b83a,
@@ -1695,7 +1718,7 @@ static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
 {
 	intel_hdmi_prepare(encoder, pipe_config);
 
-	vlv_phy_pre_pll_enable(encoder);
+	vlv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
@@ -1704,14 +1727,14 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
 {
 	intel_hdmi_prepare(encoder, pipe_config);
 
-	chv_phy_pre_pll_enable(encoder);
+	chv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder,
 				      const struct intel_crtc_state *old_crtc_state,
 				      const struct drm_connector_state *old_conn_state)
 {
-	chv_phy_post_pll_disable(encoder);
+	chv_phy_post_pll_disable(encoder, old_crtc_state);
 }
 
 static void vlv_hdmi_post_disable(struct intel_encoder *encoder,
@@ -1719,7 +1742,7 @@ static void vlv_hdmi_post_disable(struct intel_encoder *encoder,
 				  const struct drm_connector_state *old_conn_state)
 {
 	/* Reset lanes to avoid HDMI flicker (VLV w/a) */
-	vlv_phy_reset_lanes(encoder);
+	vlv_phy_reset_lanes(encoder, old_crtc_state);
 }
 
 static void chv_hdmi_post_disable(struct intel_encoder *encoder,
@@ -1732,7 +1755,7 @@ static void chv_hdmi_post_disable(struct intel_encoder *encoder,
 	mutex_lock(&dev_priv->sb_lock);
 
 	/* Assert data lane reset */
-	chv_data_lane_soft_reset(encoder, true);
+	chv_data_lane_soft_reset(encoder, old_crtc_state, true);
 
 	mutex_unlock(&dev_priv->sb_lock);
 }
@@ -1745,7 +1768,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder,
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 
-	chv_phy_pre_encoder_enable(encoder);
+	chv_phy_pre_encoder_enable(encoder, pipe_config);
 
 	/* FIXME: Program the support xxx V-dB */
 	/* Use 800mV-0dB */
@@ -2004,7 +2027,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 	struct intel_encoder *intel_encoder = &intel_dig_port->base;
 	struct drm_device *dev = intel_encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	enum port port = intel_dig_port->port;
+	enum port port = intel_encoder->port;
 
 	DRM_DEBUG_KMS("Adding HDMI connector on port %c\n",
 		      port_name(port));
@@ -2022,7 +2045,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 	connector->doublescan_allowed = 0;
 	connector->stereo_allowed = 1;
 
-	if (IS_GEMINILAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
 		connector->ycbcr_420_allowed = true;
 
 	intel_hdmi->ddc_bus = intel_hdmi_ddc_pin(dev_priv, port);
@@ -2124,7 +2147,6 @@ void intel_hdmi_init(struct drm_i915_private *dev_priv,
 	if (IS_G4X(dev_priv))
 		intel_encoder->cloneable |= 1 << INTEL_OUTPUT_HDMI;
 
-	intel_dig_port->port = port;
 	intel_dig_port->hdmi.hdmi_reg = hdmi_reg;
 	intel_dig_port->dp.output_reg = INVALID_MMIO_REG;
 	intel_dig_port->max_lanes = 4;
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index c8a48cbc2b7d..8ed05182f944 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -54,10 +54,6 @@
 #define KBL_HUC_FW_MINOR 00
 #define KBL_BLD_NUM 1810
 
-#define GLK_HUC_FW_MAJOR 02
-#define GLK_HUC_FW_MINOR 00
-#define GLK_BLD_NUM 1748
-
 #define HUC_FW_PATH(platform, major, minor, bld_num) \
 	"i915/" __stringify(platform) "_huc_ver" __stringify(major) "_" \
 	__stringify(minor) "_" __stringify(bld_num) ".bin"
@@ -74,45 +70,52 @@ MODULE_FIRMWARE(I915_BXT_HUC_UCODE);
 	KBL_HUC_FW_MINOR, KBL_BLD_NUM)
 MODULE_FIRMWARE(I915_KBL_HUC_UCODE);
 
-#define I915_GLK_HUC_UCODE HUC_FW_PATH(glk, GLK_HUC_FW_MAJOR, \
-	GLK_HUC_FW_MINOR, GLK_BLD_NUM)
-
-/**
- * intel_huc_select_fw() - selects HuC firmware for loading
- * @huc:	intel_huc struct
- */
-void intel_huc_select_fw(struct intel_huc *huc)
+static void huc_fw_select(struct intel_uc_fw *huc_fw)
 {
+	struct intel_huc *huc = container_of(huc_fw, struct intel_huc, fw);
 	struct drm_i915_private *dev_priv = huc_to_i915(huc);
 
-	intel_uc_fw_init(&huc->fw, INTEL_UC_FW_TYPE_HUC);
+	GEM_BUG_ON(huc_fw->type != INTEL_UC_FW_TYPE_HUC);
+
+	if (!HAS_HUC(dev_priv))
+		return;
 
 	if (i915_modparams.huc_firmware_path) {
-		huc->fw.path = i915_modparams.huc_firmware_path;
-		huc->fw.major_ver_wanted = 0;
-		huc->fw.minor_ver_wanted = 0;
+		huc_fw->path = i915_modparams.huc_firmware_path;
+		huc_fw->major_ver_wanted = 0;
+		huc_fw->minor_ver_wanted = 0;
 	} else if (IS_SKYLAKE(dev_priv)) {
-		huc->fw.path = I915_SKL_HUC_UCODE;
-		huc->fw.major_ver_wanted = SKL_HUC_FW_MAJOR;
-		huc->fw.minor_ver_wanted = SKL_HUC_FW_MINOR;
+		huc_fw->path = I915_SKL_HUC_UCODE;
+		huc_fw->major_ver_wanted = SKL_HUC_FW_MAJOR;
+		huc_fw->minor_ver_wanted = SKL_HUC_FW_MINOR;
 	} else if (IS_BROXTON(dev_priv)) {
-		huc->fw.path = I915_BXT_HUC_UCODE;
-		huc->fw.major_ver_wanted = BXT_HUC_FW_MAJOR;
-		huc->fw.minor_ver_wanted = BXT_HUC_FW_MINOR;
+		huc_fw->path = I915_BXT_HUC_UCODE;
+		huc_fw->major_ver_wanted = BXT_HUC_FW_MAJOR;
+		huc_fw->minor_ver_wanted = BXT_HUC_FW_MINOR;
 	} else if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) {
-		huc->fw.path = I915_KBL_HUC_UCODE;
-		huc->fw.major_ver_wanted = KBL_HUC_FW_MAJOR;
-		huc->fw.minor_ver_wanted = KBL_HUC_FW_MINOR;
-	} else if (IS_GEMINILAKE(dev_priv)) {
-		huc->fw.path = I915_GLK_HUC_UCODE;
-		huc->fw.major_ver_wanted = GLK_HUC_FW_MAJOR;
-		huc->fw.minor_ver_wanted = GLK_HUC_FW_MINOR;
+		huc_fw->path = I915_KBL_HUC_UCODE;
+		huc_fw->major_ver_wanted = KBL_HUC_FW_MAJOR;
+		huc_fw->minor_ver_wanted = KBL_HUC_FW_MINOR;
 	} else {
-		DRM_ERROR("No HuC firmware known for platform with HuC!\n");
-		return;
+		DRM_WARN("%s: No firmware known for this platform!\n",
+			 intel_uc_fw_type_repr(huc_fw->type));
 	}
 }
 
+/**
+ * intel_huc_init_early() - initializes HuC struct
+ * @huc: intel_huc struct
+ *
+ * On platforms with HuC selects firmware for uploading
+ */
+void intel_huc_init_early(struct intel_huc *huc)
+{
+	struct intel_uc_fw *huc_fw = &huc->fw;
+
+	intel_uc_fw_init(huc_fw, INTEL_UC_FW_TYPE_HUC);
+	huc_fw_select(huc_fw);
+}
+
 /**
  * huc_ucode_xfer() - DMA's the firmware
  * @dev_priv: the drm_i915_private device
@@ -151,7 +154,7 @@ static int huc_ucode_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
 	I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(HUC_UKERNEL | START_DMA));
 
 	/* Wait for DMA to finish */
-	ret = wait_for((I915_READ(DMA_CTRL) & START_DMA) == 0, 100);
+	ret = intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0, 100);
 
 	DRM_DEBUG_DRIVER("HuC DMA transfer wait over with ret %d\n", ret);
 
@@ -167,17 +170,17 @@ static int huc_ucode_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
  * intel_huc_init_hw() - load HuC uCode to device
  * @huc: intel_huc structure
  *
- * Called from guc_setup() during driver loading and also after a GPU reset.
- * Be note that HuC loading must be done before GuC loading.
+ * Called from intel_uc_init_hw() during driver loading and also after a GPU
+ * reset. Be note that HuC loading must be done before GuC loading.
  *
  * The firmware image should have already been fetched into memory by the
- * earlier call to intel_huc_init(), so here we need only check that
+ * earlier call to intel_uc_init_fw(), so here we need only check that
  * is succeeded, and then transfer the image to the h/w.
  *
  */
-void intel_huc_init_hw(struct intel_huc *huc)
+int intel_huc_init_hw(struct intel_huc *huc)
 {
-	intel_uc_fw_upload(&huc->fw, huc_ucode_xfer);
+	return intel_uc_fw_upload(&huc->fw, huc_ucode_xfer);
 }
 
 /**
@@ -191,7 +194,7 @@ void intel_huc_init_hw(struct intel_huc *huc)
  * signature through intel_guc_auth_huc(). It then waits for 50ms for
  * firmware verification ACK and unpins the object.
  */
-void intel_huc_auth(struct intel_huc *huc)
+int intel_huc_auth(struct intel_huc *huc)
 {
 	struct drm_i915_private *i915 = huc_to_i915(huc);
 	struct intel_guc *guc = &i915->guc;
@@ -199,14 +202,14 @@ void intel_huc_auth(struct intel_huc *huc)
 	int ret;
 
 	if (huc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
-		return;
+		return -ENOEXEC;
 
 	vma = i915_gem_object_ggtt_pin(huc->fw.obj, NULL, 0, 0,
 				PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
 	if (IS_ERR(vma)) {
-		DRM_ERROR("failed to pin huc fw object %d\n",
-				(int)PTR_ERR(vma));
-		return;
+		ret = PTR_ERR(vma);
+		DRM_ERROR("HuC: Failed to pin huc fw object %d\n", ret);
+		return ret;
 	}
 
 	ret = intel_guc_auth_huc(guc,
@@ -229,4 +232,5 @@ void intel_huc_auth(struct intel_huc *huc)
 
 out:
 	i915_vma_unpin(vma);
+	return ret;
 }
diff --git a/drivers/gpu/drm/i915/intel_huc.h b/drivers/gpu/drm/i915/intel_huc.h
index aaa38b9e5817..40039db59e04 100644
--- a/drivers/gpu/drm/i915/intel_huc.h
+++ b/drivers/gpu/drm/i915/intel_huc.h
@@ -34,8 +34,8 @@ struct intel_huc {
 	/* HuC-specific additions */
 };
 
-void intel_huc_select_fw(struct intel_huc *huc);
-void intel_huc_init_hw(struct intel_huc *huc);
-void intel_huc_auth(struct intel_huc *huc);
+void intel_huc_init_early(struct intel_huc *huc);
+int intel_huc_init_hw(struct intel_huc *huc);
+int intel_huc_auth(struct intel_huc *huc);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 49fdf09f9919..ef9f91a0b0c9 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -128,22 +128,46 @@ intel_i2c_reset(struct drm_i915_private *dev_priv)
 	I915_WRITE(GMBUS4, 0);
 }
 
-static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable)
+static void pnv_gmbus_clock_gating(struct drm_i915_private *dev_priv,
+				   bool enable)
 {
 	u32 val;
 
 	/* When using bit bashing for I2C, this bit needs to be set to 1 */
-	if (!IS_PINEVIEW(dev_priv))
-		return;
-
 	val = I915_READ(DSPCLK_GATE_D);
-	if (enable)
-		val |= DPCUNIT_CLOCK_GATE_DISABLE;
+	if (!enable)
+		val |= PNV_GMBUSUNIT_CLOCK_GATE_DISABLE;
 	else
-		val &= ~DPCUNIT_CLOCK_GATE_DISABLE;
+		val &= ~PNV_GMBUSUNIT_CLOCK_GATE_DISABLE;
 	I915_WRITE(DSPCLK_GATE_D, val);
 }
 
+static void pch_gmbus_clock_gating(struct drm_i915_private *dev_priv,
+				   bool enable)
+{
+	u32 val;
+
+	val = I915_READ(SOUTH_DSPCLK_GATE_D);
+	if (!enable)
+		val |= PCH_GMBUSUNIT_CLOCK_GATE_DISABLE;
+	else
+		val &= ~PCH_GMBUSUNIT_CLOCK_GATE_DISABLE;
+	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
+}
+
+static void bxt_gmbus_clock_gating(struct drm_i915_private *dev_priv,
+				   bool enable)
+{
+	u32 val;
+
+	val = I915_READ(GEN9_CLKGATE_DIS_4);
+	if (!enable)
+		val |= BXT_GMBUS_GATING_DIS;
+	else
+		val &= ~BXT_GMBUS_GATING_DIS;
+	I915_WRITE(GEN9_CLKGATE_DIS_4, val);
+}
+
 static u32 get_reserved(struct intel_gmbus *bus)
 {
 	struct drm_i915_private *dev_priv = bus->dev_priv;
@@ -221,7 +245,10 @@ intel_gpio_pre_xfer(struct i2c_adapter *adapter)
 	struct drm_i915_private *dev_priv = bus->dev_priv;
 
 	intel_i2c_reset(dev_priv);
-	intel_i2c_quirk_set(dev_priv, true);
+
+	if (IS_PINEVIEW(dev_priv))
+		pnv_gmbus_clock_gating(dev_priv, false);
+
 	set_data(bus, 1);
 	set_clock(bus, 1);
 	udelay(I2C_RISEFALL_TIME);
@@ -238,7 +265,9 @@ intel_gpio_post_xfer(struct i2c_adapter *adapter)
 
 	set_data(bus, 1);
 	set_clock(bus, 1);
-	intel_i2c_quirk_set(dev_priv, false);
+
+	if (IS_PINEVIEW(dev_priv))
+		pnv_gmbus_clock_gating(dev_priv, true);
 }
 
 static void
@@ -481,6 +510,13 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 	int i = 0, inc, try = 0;
 	int ret = 0;
 
+	/* Display WA #0868: skl,bxt,kbl,cfl,glk,cnl */
+	if (IS_GEN9_LP(dev_priv))
+		bxt_gmbus_clock_gating(dev_priv, false);
+	else if (HAS_PCH_SPT(dev_priv) ||
+		 HAS_PCH_KBP(dev_priv) || HAS_PCH_CNP(dev_priv))
+		pch_gmbus_clock_gating(dev_priv, false);
+
 retry:
 	I915_WRITE_FW(GMBUS0, bus->reg0);
 
@@ -582,6 +618,13 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 	ret = -EAGAIN;
 
 out:
+	/* Display WA #0868: skl,bxt,kbl,cfl,glk,cnl */
+	if (IS_GEN9_LP(dev_priv))
+		bxt_gmbus_clock_gating(dev_priv, true);
+	else if (HAS_PCH_SPT(dev_priv) ||
+		 HAS_PCH_KBP(dev_priv) || HAS_PCH_CNP(dev_priv))
+		pch_gmbus_clock_gating(dev_priv, true);
+
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index e71a8cd50498..e0fca035ff78 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -136,6 +136,7 @@
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
+#include "i915_gem_render_state.h"
 #include "intel_mocs.h"
 
 #define RING_EXECLIST_QFULL		(1 << 0x2)
@@ -153,9 +154,7 @@
 #define GEN8_CTX_STATUS_LITE_RESTORE	(1 << 15)
 
 #define GEN8_CTX_STATUS_COMPLETED_MASK \
-	 (GEN8_CTX_STATUS_ACTIVE_IDLE | \
-	  GEN8_CTX_STATUS_PREEMPTED | \
-	  GEN8_CTX_STATUS_ELEMENT_SWITCH)
+	 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
 
 #define CTX_LRI_HEADER_0		0x01
 #define CTX_CONTEXT_CONTROL		0x02
@@ -219,37 +218,6 @@ static void execlists_init_reg_state(u32 *reg_state,
 				     struct intel_engine_cs *engine,
 				     struct intel_ring *ring);
 
-/**
- * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
- * @dev_priv: i915 device private
- * @enable_execlists: value of i915.enable_execlists module parameter.
- *
- * Only certain platforms support Execlists (the prerequisites being
- * support for Logical Ring Contexts and Aliasing PPGTT or better).
- *
- * Return: 1 if Execlists is supported and has to be enabled.
- */
-int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists)
-{
-	/* On platforms with execlist available, vGPU will only
-	 * support execlist mode, no ring buffer mode.
-	 */
-	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv))
-		return 1;
-
-	if (INTEL_GEN(dev_priv) >= 9)
-		return 1;
-
-	if (enable_execlists == 0)
-		return 0;
-
-	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) &&
-	    USES_PPGTT(dev_priv))
-		return 1;
-
-	return 0;
-}
-
 /**
  * intel_lr_context_descriptor_update() - calculate & cache the descriptor
  * 					  descriptor for a pinned context
@@ -354,7 +322,7 @@ static void unwind_wa_tail(struct drm_i915_gem_request *rq)
 	assert_ring_tail_valid(rq->ring, rq->tail);
 }
 
-static void unwind_incomplete_requests(struct intel_engine_cs *engine)
+static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_request *rq, *rn;
 	struct i915_priolist *uninitialized_var(p);
@@ -385,6 +353,17 @@ static void unwind_incomplete_requests(struct intel_engine_cs *engine)
 	}
 }
 
+void
+execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
+{
+	struct intel_engine_cs *engine =
+		container_of(execlists, typeof(*engine), execlists);
+
+	spin_lock_irq(&engine->timeline->lock);
+	__unwind_incomplete_requests(engine);
+	spin_unlock_irq(&engine->timeline->lock);
+}
+
 static inline void
 execlists_context_status_change(struct drm_i915_gem_request *rq,
 				unsigned long status)
@@ -400,6 +379,20 @@ execlists_context_status_change(struct drm_i915_gem_request *rq,
 				   status, rq);
 }
 
+static inline void
+execlists_context_schedule_in(struct drm_i915_gem_request *rq)
+{
+	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
+	intel_engine_context_in(rq->engine);
+}
+
+static inline void
+execlists_context_schedule_out(struct drm_i915_gem_request *rq)
+{
+	intel_engine_context_out(rq->engine);
+	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
+}
+
 static void
 execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
 {
@@ -438,8 +431,6 @@ static inline void elsp_write(u64 desc, u32 __iomem *elsp)
 static void execlists_submit_ports(struct intel_engine_cs *engine)
 {
 	struct execlist_port *port = engine->execlists.port;
-	u32 __iomem *elsp =
-		engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine));
 	unsigned int n;
 
 	for (n = execlists_num_ports(&engine->execlists); n--; ) {
@@ -451,17 +442,23 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
 		if (rq) {
 			GEM_BUG_ON(count > !n);
 			if (!count++)
-				execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
+				execlists_context_schedule_in(rq);
 			port_set(&port[n], port_pack(rq, count));
 			desc = execlists_update_context(rq);
 			GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
+
+			GEM_TRACE("%s in[%d]:  ctx=%d.%d, seqno=%x\n",
+				  engine->name, n,
+				  port[n].context_id, count,
+				  rq->global_seqno);
 		} else {
 			GEM_BUG_ON(!n);
 			desc = 0;
 		}
 
-		elsp_write(desc, elsp);
+		elsp_write(desc, engine->execlists.elsp);
 	}
+	execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK);
 }
 
 static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
@@ -497,8 +494,6 @@ static void inject_preempt_context(struct intel_engine_cs *engine)
 {
 	struct intel_context *ce =
 		&engine->i915->preempt_context->engine[engine->id];
-	u32 __iomem *elsp =
-		engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine));
 	unsigned int n;
 
 	GEM_BUG_ON(engine->i915->preempt_context->hw_id != PREEMPT_ID);
@@ -509,15 +504,12 @@ static void inject_preempt_context(struct intel_engine_cs *engine)
 	ce->ring->tail &= (ce->ring->size - 1);
 	ce->lrc_reg_state[CTX_RING_TAIL+1] = ce->ring->tail;
 
+	GEM_TRACE("%s\n", engine->name);
 	for (n = execlists_num_ports(&engine->execlists); --n; )
-		elsp_write(0, elsp);
+		elsp_write(0, engine->execlists.elsp);
 
-	elsp_write(ce->lrc_desc, elsp);
-}
-
-static bool can_preempt(struct intel_engine_cs *engine)
-{
-	return INTEL_INFO(engine->i915)->has_logical_ring_preemption;
+	elsp_write(ce->lrc_desc, engine->execlists.elsp);
+	execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK);
 }
 
 static void execlists_dequeue(struct intel_engine_cs *engine)
@@ -564,10 +556,21 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		 * know the next preemption status we see corresponds
 		 * to this ELSP update.
 		 */
+		GEM_BUG_ON(!port_count(&port[0]));
 		if (port_count(&port[0]) > 1)
 			goto unlock;
 
-		if (can_preempt(engine) &&
+		/*
+		 * If we write to ELSP a second time before the HW has had
+		 * a chance to respond to the previous write, we can confuse
+		 * the HW and hit "undefined behaviour". After writing to ELSP,
+		 * we must then wait until we see a context-switch event from
+		 * the HW to indicate that it has had a chance to respond.
+		 */
+		if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
+			goto unlock;
+
+		if (HAS_LOGICAL_RING_PREEMPTION(engine->i915) &&
 		    rb_entry(rb, struct i915_priolist, node)->priority >
 		    max(last->priotree.priority, 0)) {
 			/*
@@ -690,8 +693,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 	}
 }
 
-static void
-execlist_cancel_port_requests(struct intel_engine_execlists *execlists)
+void
+execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
 {
 	struct execlist_port *port = execlists->port;
 	unsigned int num_ports = execlists_num_ports(execlists);
@@ -700,6 +703,7 @@ execlist_cancel_port_requests(struct intel_engine_execlists *execlists)
 		struct drm_i915_gem_request *rq = port_request(port);
 
 		GEM_BUG_ON(!execlists->active);
+		intel_engine_context_out(rq->engine);
 		execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_PREEMPTED);
 		i915_gem_request_put(rq);
 
@@ -715,10 +719,12 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	struct rb_node *rb;
 	unsigned long flags;
 
+	GEM_TRACE("%s\n", engine->name);
+
 	spin_lock_irqsave(&engine->timeline->lock, flags);
 
 	/* Cancel the requests on the HW and clear the ELSP tracker. */
-	execlist_cancel_port_requests(execlists);
+	execlists_cancel_port_requests(execlists);
 
 	/* Mark all executing requests as skipped. */
 	list_for_each_entry(rq, &engine->timeline->requests, link) {
@@ -761,6 +767,9 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
 	 */
 	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
 
+	/* Mark all CS interrupts as complete */
+	execlists->active = 0;
+
 	spin_unlock_irqrestore(&engine->timeline->lock, flags);
 }
 
@@ -768,7 +777,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
  * Check the unread Context Status Buffers and manage the submission of new
  * contexts to the ELSP accordingly.
  */
-static void intel_lrc_irq_handler(unsigned long data)
+static void execlists_submission_tasklet(unsigned long data)
 {
 	struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
 	struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -826,6 +835,10 @@ static void intel_lrc_irq_handler(unsigned long data)
 			head = execlists->csb_head;
 			tail = READ_ONCE(buf[write_idx]);
 		}
+		GEM_TRACE("%s cs-irq head=%d [%d], tail=%d [%d]\n",
+			  engine->name,
+			  head, GEN8_CSB_READ_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))),
+			  tail, GEN8_CSB_WRITE_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))));
 
 		while (head != tail) {
 			struct drm_i915_gem_request *rq;
@@ -853,16 +866,31 @@ static void intel_lrc_irq_handler(unsigned long data)
 			 */
 
 			status = READ_ONCE(buf[2 * head]); /* maybe mmio! */
+			GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
+				  engine->name, head,
+				  status, buf[2*head + 1],
+				  execlists->active);
+
+			if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
+				      GEN8_CTX_STATUS_PREEMPTED))
+				execlists_set_active(execlists,
+						     EXECLISTS_ACTIVE_HWACK);
+			if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
+				execlists_clear_active(execlists,
+						       EXECLISTS_ACTIVE_HWACK);
+
 			if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
 				continue;
 
-			if (status & GEN8_CTX_STATUS_ACTIVE_IDLE &&
+			/* We should never get a COMPLETED | IDLE_ACTIVE! */
+			GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);
+
+			if (status & GEN8_CTX_STATUS_COMPLETE &&
 			    buf[2*head + 1] == PREEMPT_ID) {
-				execlist_cancel_port_requests(execlists);
+				GEM_TRACE("%s preempt-idle\n", engine->name);
 
-				spin_lock_irq(&engine->timeline->lock);
-				unwind_incomplete_requests(engine);
-				spin_unlock_irq(&engine->timeline->lock);
+				execlists_cancel_port_requests(execlists);
+				execlists_unwind_incomplete_requests(execlists);
 
 				GEM_BUG_ON(!execlists_is_active(execlists,
 								EXECLISTS_ACTIVE_PREEMPT));
@@ -883,12 +911,17 @@ static void intel_lrc_irq_handler(unsigned long data)
 			GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
 
 			rq = port_unpack(port, &count);
+			GEM_TRACE("%s out[0]: ctx=%d.%d, seqno=%x\n",
+				  engine->name,
+				  port->context_id, count,
+				  rq ? rq->global_seqno : 0);
 			GEM_BUG_ON(count == 0);
 			if (--count == 0) {
 				GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
+				GEM_BUG_ON(port_isset(&port[1]) &&
+					   !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
 				GEM_BUG_ON(!i915_gem_request_completed(rq));
-				execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
-
+				execlists_context_schedule_out(rq);
 				trace_i915_gem_request_out(rq);
 				i915_gem_request_put(rq);
 
@@ -926,7 +959,7 @@ static void insert_request(struct intel_engine_cs *engine,
 
 	list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests);
 	if (ptr_unmask_bits(p, 1))
-		tasklet_hi_schedule(&engine->execlists.irq_tasklet);
+		tasklet_hi_schedule(&engine->execlists.tasklet);
 }
 
 static void execlists_submit_request(struct drm_i915_gem_request *request)
@@ -1060,12 +1093,34 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
 	spin_unlock_irq(&engine->timeline->lock);
 }
 
+static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma)
+{
+	unsigned int flags;
+	int err;
+
+	/*
+	 * Clear this page out of any CPU caches for coherent swap-in/out.
+	 * We only want to do this on the first bind so that we do not stall
+	 * on an active context (which by nature is already on the GPU).
+	 */
+	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
+		err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
+		if (err)
+			return err;
+	}
+
+	flags = PIN_GLOBAL | PIN_HIGH;
+	if (ctx->ggtt_offset_bias)
+		flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;
+
+	return i915_vma_pin(vma, 0, GEN8_LR_CONTEXT_ALIGN, flags);
+}
+
 static struct intel_ring *
 execlists_context_pin(struct intel_engine_cs *engine,
 		      struct i915_gem_context *ctx)
 {
 	struct intel_context *ce = &ctx->engine[engine->id];
-	unsigned int flags;
 	void *vaddr;
 	int ret;
 
@@ -1082,11 +1137,7 @@ execlists_context_pin(struct intel_engine_cs *engine,
 	}
 	GEM_BUG_ON(!ce->state);
 
-	flags = PIN_GLOBAL | PIN_HIGH;
-	if (ctx->ggtt_offset_bias)
-		flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;
-
-	ret = i915_vma_pin(ce->state, 0, GEN8_LR_CONTEXT_ALIGN, flags);
+	ret = __context_pin(ctx, ce->state);
 	if (ret)
 		goto err;
 
@@ -1106,9 +1157,7 @@ execlists_context_pin(struct intel_engine_cs *engine,
 	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
 		i915_ggtt_offset(ce->ring->vma);
 
-	ce->state->obj->mm.dirty = true;
 	ce->state->obj->pin_global++;
-
 	i915_gem_context_get(ctx);
 out:
 	return ce->ring;
@@ -1146,7 +1195,6 @@ static int execlists_request_alloc(struct drm_i915_gem_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
 	struct intel_context *ce = &request->ctx->engine[engine->id];
-	u32 *cs;
 	int ret;
 
 	GEM_BUG_ON(!ce->pin_count);
@@ -1157,17 +1205,9 @@ static int execlists_request_alloc(struct drm_i915_gem_request *request)
 	 */
 	request->reserved_space += EXECLISTS_REQUEST_SIZE;
 
-	cs = intel_ring_begin(request, 0);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	if (!ce->initialised) {
-		ret = engine->init_context(request);
-		if (ret)
-			return ret;
-
-		ce->initialised = true;
-	}
+	ret = intel_ring_wait_for_space(request->ring, request->reserved_space);
+	if (ret)
+		return ret;
 
 	/* Note that after this point, we have committed to using
 	 * this request as it is being used to both track the
@@ -1476,9 +1516,12 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 	execlists->csb_head = -1;
 	execlists->active = 0;
 
+	execlists->elsp =
+		dev_priv->regs + i915_mmio_reg_offset(RING_ELSP(engine));
+
 	/* After a GPU reset, we may have requests to replay */
-	if (!i915_modparams.enable_guc_submission && execlists->first)
-		tasklet_schedule(&execlists->irq_tasklet);
+	if (execlists->first)
+		tasklet_schedule(&execlists->tasklet);
 
 	return 0;
 }
@@ -1523,6 +1566,8 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 	struct intel_context *ce;
 	unsigned long flags;
 
+	GEM_TRACE("%s seqno=%x\n",
+		  engine->name, request ? request->global_seqno : 0);
 	spin_lock_irqsave(&engine->timeline->lock, flags);
 
 	/*
@@ -1534,10 +1579,10 @@ static void reset_common_ring(struct intel_engine_cs *engine,
 	 * guessing the missed context-switch events by looking at what
 	 * requests were completed.
 	 */
-	execlist_cancel_port_requests(execlists);
+	execlists_cancel_port_requests(execlists);
 
 	/* Push back any incomplete requests for replay after the reset. */
-	unwind_incomplete_requests(engine);
+	__unwind_incomplete_requests(engine);
 
 	spin_unlock_irqrestore(&engine->timeline->lock, flags);
 
@@ -1797,10 +1842,8 @@ static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs)
 	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
 	BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
 
-	*cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
-	*cs++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT;
-	*cs++ = 0;
-	*cs++ = request->global_seqno;
+	cs = gen8_emit_ggtt_write(cs, request->global_seqno,
+				  intel_hws_seqno_address(request->engine));
 	*cs++ = MI_USER_INTERRUPT;
 	*cs++ = MI_NOOP;
 	request->tail = intel_ring_offset(request, cs);
@@ -1810,24 +1853,14 @@ static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs)
 }
 static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
 
-static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
+static void gen8_emit_breadcrumb_rcs(struct drm_i915_gem_request *request,
 					u32 *cs)
 {
 	/* We're using qword write, seqno should be aligned to 8 bytes. */
 	BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
 
-	/* w/a for post sync ops following a GPGPU operation we
-	 * need a prior CS_STALL, which is emitted by the flush
-	 * following the batch.
-	 */
-	*cs++ = GFX_OP_PIPE_CONTROL(6);
-	*cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
-		PIPE_CONTROL_QW_WRITE;
-	*cs++ = intel_hws_seqno_address(request->engine);
-	*cs++ = 0;
-	*cs++ = request->global_seqno;
-	/* We're thrashing one dword of HWS. */
-	*cs++ = 0;
+	cs = gen8_emit_ggtt_write_rcs(cs, request->global_seqno,
+				      intel_hws_seqno_address(request->engine));
 	*cs++ = MI_USER_INTERRUPT;
 	*cs++ = MI_NOOP;
 	request->tail = intel_ring_offset(request, cs);
@@ -1835,7 +1868,7 @@ static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
 
 	gen8_emit_wa_tail(request, cs);
 }
-static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS;
+static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS;
 
 static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
 {
@@ -1868,8 +1901,9 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 	 * Tasklet cannot be active at this point due intel_mark_active/idle
 	 * so this is just for documentation.
 	 */
-	if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->execlists.irq_tasklet.state)))
-		tasklet_kill(&engine->execlists.irq_tasklet);
+	if (WARN_ON(test_bit(TASKLET_STATE_SCHED,
+			     &engine->execlists.tasklet.state)))
+		tasklet_kill(&engine->execlists.tasklet);
 
 	dev_priv = engine->i915;
 
@@ -1893,7 +1927,12 @@ static void execlists_set_default_submission(struct intel_engine_cs *engine)
 	engine->submit_request = execlists_submit_request;
 	engine->cancel_requests = execlists_cancel_requests;
 	engine->schedule = execlists_schedule;
-	engine->execlists.irq_tasklet.func = intel_lrc_irq_handler;
+	engine->execlists.tasklet.func = execlists_submission_tasklet;
+
+	engine->park = NULL;
+	engine->unpark = NULL;
+
+	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
 }
 
 static void
@@ -1952,8 +1991,8 @@ logical_ring_setup(struct intel_engine_cs *engine)
 
 	engine->execlists.fw_domains = fw_domains;
 
-	tasklet_init(&engine->execlists.irq_tasklet,
-		     intel_lrc_irq_handler, (unsigned long)engine);
+	tasklet_init(&engine->execlists.tasklet,
+		     execlists_submission_tasklet, (unsigned long)engine);
 
 	logical_ring_default_vfuncs(engine);
 	logical_ring_default_irqs(engine);
@@ -1991,8 +2030,8 @@ int logical_render_ring_init(struct intel_engine_cs *engine)
 		engine->init_hw = gen8_init_render_ring;
 	engine->init_context = gen8_init_rcs_context;
 	engine->emit_flush = gen8_emit_flush_render;
-	engine->emit_breadcrumb = gen8_emit_breadcrumb_render;
-	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz;
+	engine->emit_breadcrumb = gen8_emit_breadcrumb_rcs;
+	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_rcs_sz;
 
 	ret = intel_engine_create_scratch(engine, PAGE_SIZE);
 	if (ret)
@@ -2109,7 +2148,6 @@ static void execlists_init_reg_state(u32 *regs,
 
 	CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine),
 		_MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
-				   CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
 				   (HAS_RESOURCE_STREAMER(dev_priv) ?
 				   CTX_CTRL_RS_CTX_ENABLE : 0)));
 	CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0);
@@ -2186,6 +2224,7 @@ populate_lr_context(struct i915_gem_context *ctx,
 		    struct intel_ring *ring)
 {
 	void *vaddr;
+	u32 *regs;
 	int ret;
 
 	ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
@@ -2202,11 +2241,31 @@ populate_lr_context(struct i915_gem_context *ctx,
 	}
 	ctx_obj->mm.dirty = true;
 
+	if (engine->default_state) {
+		/*
+		 * We only want to copy over the template context state;
+		 * skipping over the headers reserved for GuC communication,
+		 * leaving those as zero.
+		 */
+		const unsigned long start = LRC_HEADER_PAGES * PAGE_SIZE;
+		void *defaults;
+
+		defaults = i915_gem_object_pin_map(engine->default_state,
+						   I915_MAP_WB);
+		if (IS_ERR(defaults))
+			return PTR_ERR(defaults);
+
+		memcpy(vaddr + start, defaults + start, engine->context_size);
+		i915_gem_object_unpin_map(engine->default_state);
+	}
+
 	/* The second page of the context object contains some fields which must
 	 * be set up prior to the first execution. */
-
-	execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE,
-				 ctx, engine, ring);
+	regs = vaddr + LRC_STATE_PN * PAGE_SIZE;
+	execlists_init_reg_state(regs, ctx, engine, ring);
+	if (!engine->default_state)
+		regs[CTX_CONTEXT_CONTROL + 1] |=
+			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
 
 	i915_gem_object_unpin_map(ctx_obj);
 
@@ -2259,7 +2318,6 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 
 	ce->ring = ring;
 	ce->state = vma;
-	ce->initialised |= engine->init_context == NULL;
 
 	return 0;
 
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 689fde1a63a9..6d4f9b995a11 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -107,9 +107,4 @@ intel_lr_context_descriptor(struct i915_gem_context *ctx,
 	return ctx->engine[engine->id].lrc_desc;
 }
 
-
-/* Execlists */
-int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv,
-				    int enable_execlists);
-
 #endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 38572d65e46e..ef80499113ee 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -125,6 +125,8 @@ static void intel_lvds_get_config(struct intel_encoder *encoder,
 	struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
 	u32 tmp, flags = 0;
 
+	pipe_config->output_types |= BIT(INTEL_OUTPUT_LVDS);
+
 	tmp = I915_READ(lvds_encoder->reg);
 	if (tmp & LVDS_HSYNC_POLARITY)
 		flags |= DRM_MODE_FLAG_NHSYNC;
diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_modes.c
index 28a778b785ac..4e43f873c889 100644
--- a/drivers/gpu/drm/i915/intel_modes.c
+++ b/drivers/gpu/drm/i915/intel_modes.c
@@ -57,7 +57,6 @@ int intel_connector_update_modes(struct drm_connector *connector,
 
 	drm_mode_connector_update_edid_property(connector, edid);
 	ret = drm_add_edid_modes(connector, edid);
-	drm_edid_to_eld(connector, edid);
 
 	intel_connector_update_eld_conn_type(connector);
 
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 1d946240e55f..c58e5f53bab0 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -32,6 +32,8 @@
 
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
+
+#include "intel_opregion.h"
 #include "i915_drv.h"
 #include "intel_drv.h"
 
@@ -367,7 +369,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
 	if (intel_encoder->type == INTEL_OUTPUT_DSI)
 		port = 0;
 	else
-		port = intel_ddi_get_encoder_port(intel_encoder);
+		port = intel_encoder->port;
 
 	if (port == PORT_E)  {
 		port = 0;
@@ -383,7 +385,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
 	case INTEL_OUTPUT_ANALOG:
 		type = DISPLAY_TYPE_CRT;
 		break;
-	case INTEL_OUTPUT_UNKNOWN:
+	case INTEL_OUTPUT_DDI:
 	case INTEL_OUTPUT_DP:
 	case INTEL_OUTPUT_HDMI:
 	case INTEL_OUTPUT_DP_MST:
diff --git a/drivers/gpu/drm/i915/intel_opregion.h b/drivers/gpu/drm/i915/intel_opregion.h
new file mode 100644
index 000000000000..e0e437ba9e51
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_opregion.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright © 2008-2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _INTEL_OPREGION_H_
+#define _INTEL_OPREGION_H_
+
+#include <linux/workqueue.h>
+#include <linux/pci.h>
+
+struct drm_i915_private;
+struct intel_encoder;
+
+struct opregion_header;
+struct opregion_acpi;
+struct opregion_swsci;
+struct opregion_asle;
+
+struct intel_opregion {
+	struct opregion_header *header;
+	struct opregion_acpi *acpi;
+	struct opregion_swsci *swsci;
+	u32 swsci_gbda_sub_functions;
+	u32 swsci_sbcb_sub_functions;
+	struct opregion_asle *asle;
+	void *rvda;
+	void *vbt_firmware;
+	const void *vbt;
+	u32 vbt_size;
+	u32 *lid_state;
+	struct work_struct asle_work;
+};
+
+#define OPREGION_SIZE            (8 * 1024)
+
+#ifdef CONFIG_ACPI
+
+int intel_opregion_setup(struct drm_i915_private *dev_priv);
+void intel_opregion_register(struct drm_i915_private *dev_priv);
+void intel_opregion_unregister(struct drm_i915_private *dev_priv);
+void intel_opregion_asle_intr(struct drm_i915_private *dev_priv);
+int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
+				  bool enable);
+int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv,
+				  pci_power_t state);
+int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv);
+
+#else /* CONFIG_ACPI*/
+
+static inline int intel_opregion_setup(struct drm_i915_private *dev_priv)
+{
+	return 0;
+}
+
+static inline void intel_opregion_register(struct drm_i915_private *dev_priv)
+{
+}
+
+static inline void intel_opregion_unregister(struct drm_i915_private *dev_priv)
+{
+}
+
+static inline void intel_opregion_asle_intr(struct drm_i915_private *dev_priv)
+{
+}
+
+static inline int
+intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable)
+{
+	return 0;
+}
+
+static inline int
+intel_opregion_notify_adapter(struct drm_i915_private *dev, pci_power_t state)
+{
+	return 0;
+}
+
+static inline int intel_opregion_get_panel_type(struct drm_i915_private *dev)
+{
+	return -ENODEV;
+}
+
+#endif /* CONFIG_ACPI */
+
+#endif
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index 1b397b41cb4f..41e9465d44a8 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -219,7 +219,7 @@ intel_overlay_map_regs(struct intel_overlay *overlay)
 	if (OVERLAY_NEEDS_PHYSICAL(dev_priv))
 		regs = (struct overlay_registers __iomem *)overlay->reg_bo->phys_handle->vaddr;
 	else
-		regs = io_mapping_map_wc(&dev_priv->ggtt.mappable,
+		regs = io_mapping_map_wc(&dev_priv->ggtt.iomap,
 					 overlay->flip_addr,
 					 PAGE_SIZE);
 
@@ -1508,7 +1508,7 @@ intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
 		regs = (struct overlay_registers __iomem *)
 			overlay->reg_bo->phys_handle->vaddr;
 	else
-		regs = io_mapping_map_atomic_wc(&dev_priv->ggtt.mappable,
+		regs = io_mapping_map_atomic_wc(&dev_priv->ggtt.iomap,
 						overlay->flip_addr);
 
 	return regs;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index adc51e452e3e..fa6831f8c004 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -543,25 +543,6 @@ static u32 pwm_get_backlight(struct intel_connector *connector)
 	return DIV_ROUND_UP(duty_ns * 100, CRC_PMIC_PWM_PERIOD_NS);
 }
 
-static u32 intel_panel_get_backlight(struct intel_connector *connector)
-{
-	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct intel_panel *panel = &connector->panel;
-	u32 val = 0;
-
-	mutex_lock(&dev_priv->backlight_lock);
-
-	if (panel->backlight.enabled) {
-		val = panel->backlight.get(connector);
-		val = intel_panel_compute_brightness(connector, val);
-	}
-
-	mutex_unlock(&dev_priv->backlight_lock);
-
-	DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val);
-	return val;
-}
-
 static void lpt_set_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
 	struct intel_connector *connector = to_intel_connector(conn_state->connector);
@@ -649,31 +630,6 @@ intel_panel_actually_set_backlight(const struct drm_connector_state *conn_state,
 	panel->backlight.set(conn_state, level);
 }
 
-/* set backlight brightness to level in range [0..max], scaling wrt hw min */
-static void intel_panel_set_backlight(const struct drm_connector_state *conn_state,
-				      u32 user_level, u32 user_max)
-{
-	struct intel_connector *connector = to_intel_connector(conn_state->connector);
-	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
-	struct intel_panel *panel = &connector->panel;
-	u32 hw_level;
-
-	if (!panel->backlight.present)
-		return;
-
-	mutex_lock(&dev_priv->backlight_lock);
-
-	WARN_ON(panel->backlight.max == 0);
-
-	hw_level = scale_user_to_hw(connector, user_level, user_max);
-	panel->backlight.level = hw_level;
-
-	if (panel->backlight.enabled)
-		intel_panel_actually_set_backlight(conn_state, hw_level);
-
-	mutex_unlock(&dev_priv->backlight_lock);
-}
-
 /* set backlight brightness to level in range [0..max], assuming hw min is
  * respected.
  */
@@ -1182,6 +1138,50 @@ void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state,
 }
 
 #if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE)
+static u32 intel_panel_get_backlight(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_panel *panel = &connector->panel;
+	u32 val = 0;
+
+	mutex_lock(&dev_priv->backlight_lock);
+
+	if (panel->backlight.enabled) {
+		val = panel->backlight.get(connector);
+		val = intel_panel_compute_brightness(connector, val);
+	}
+
+	mutex_unlock(&dev_priv->backlight_lock);
+
+	DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val);
+	return val;
+}
+
+/* set backlight brightness to level in range [0..max], scaling wrt hw min */
+static void intel_panel_set_backlight(const struct drm_connector_state *conn_state,
+				      u32 user_level, u32 user_max)
+{
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_panel *panel = &connector->panel;
+	u32 hw_level;
+
+	if (!panel->backlight.present)
+		return;
+
+	mutex_lock(&dev_priv->backlight_lock);
+
+	WARN_ON(panel->backlight.max == 0);
+
+	hw_level = scale_user_to_hw(connector, user_level, user_max);
+	panel->backlight.level = hw_level;
+
+	if (panel->backlight.enabled)
+		intel_panel_actually_set_backlight(conn_state, hw_level);
+
+	mutex_unlock(&dev_priv->backlight_lock);
+}
+
 static int intel_backlight_device_update_status(struct backlight_device *bd)
 {
 	struct intel_connector *connector = bl_get_data(bd);
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c
index 899839f2f7c6..1f5cd572a7ff 100644
--- a/drivers/gpu/drm/i915/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.c
@@ -269,7 +269,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_i915_private *dev_priv,
 		case INTEL_OUTPUT_DP:
 		case INTEL_OUTPUT_EDP:
 			dig_port = enc_to_dig_port(&encoder->base);
-			switch (dig_port->port) {
+			switch (dig_port->base.port) {
 			case PORT_B:
 				*source = INTEL_PIPE_CRC_SOURCE_DP_B;
 				break;
@@ -281,7 +281,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_i915_private *dev_priv,
 				break;
 			default:
 				WARN(1, "nonexisting DP port %c\n",
-				     port_name(dig_port->port));
+				     port_name(dig_port->base.port));
 				break;
 			}
 			break;
@@ -541,8 +541,6 @@ static void hsw_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
 		 * completely disable it.
 		 */
 		pipe_config->ips_force_disable = enable;
-		if (pipe_config->ips_enabled == enable)
-			pipe_config->base.connectors_changed = true;
 	}
 
 	if (IS_HASWELL(dev_priv)) {
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index f0d0dbab4150..1a6e699e19e0 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -52,16 +52,13 @@
  * which brings the most power savings; deeper states save more power, but
  * require higher latency to switch to and wake up.
  */
-#define INTEL_RC6_ENABLE			(1<<0)
-#define INTEL_RC6p_ENABLE			(1<<1)
-#define INTEL_RC6pp_ENABLE			(1<<2)
 
 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
 {
 	if (HAS_LLC(dev_priv)) {
 		/*
 		 * WaCompressedResourceDisplayNewHashMode:skl,kbl
-		 * Display WA#0390: skl,kbl
+		 * Display WA #0390: skl,kbl
 		 *
 		 * Must match Sampler, Pixel Back End, and Media. See
 		 * WaCompressedResourceSamplerPbeMediaNewHashMode.
@@ -75,9 +72,6 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
 	I915_WRITE(CHICKEN_PAR1_1,
 		   I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
 
-	I915_WRITE(GEN8_CONFIG0,
-		   I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES);
-
 	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
 	I915_WRITE(GEN8_CHICKEN_DCPR_1,
 		   I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
@@ -515,38 +509,41 @@ static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 	fifo_state->plane[PLANE_CURSOR] = 63;
 }
 
-static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
+static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
+			      enum i9xx_plane_id i9xx_plane)
 {
 	uint32_t dsparb = I915_READ(DSPARB);
 	int size;
 
 	size = dsparb & 0x7f;
-	if (plane)
+	if (i9xx_plane == PLANE_B)
 		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 
-	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-		      plane ? "B" : "A", size);
+	DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
+		      dsparb, plane_name(i9xx_plane), size);
 
 	return size;
 }
 
-static int i830_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
+static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
+			      enum i9xx_plane_id i9xx_plane)
 {
 	uint32_t dsparb = I915_READ(DSPARB);
 	int size;
 
 	size = dsparb & 0x1ff;
-	if (plane)
+	if (i9xx_plane == PLANE_B)
 		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 	size >>= 1; /* Convert to cachelines */
 
-	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-		      plane ? "B" : "A", size);
+	DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
+		      dsparb, plane_name(i9xx_plane), size);
 
 	return size;
 }
 
-static int i845_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
+static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
+			      enum i9xx_plane_id i9xx_plane)
 {
 	uint32_t dsparb = I915_READ(DSPARB);
 	int size;
@@ -554,9 +551,8 @@ static int i845_get_fifo_size(struct drm_i915_private *dev_priv, int plane)
 	size = dsparb & 0x7f;
 	size >>= 2; /* Convert to cachelines */
 
-	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-		      plane ? "B" : "A",
-		      size);
+	DRM_DEBUG_KMS("FIFO size - (0x%08x) %c: %d\n",
+		      dsparb, plane_name(i9xx_plane), size);
 
 	return size;
 }
@@ -922,7 +918,7 @@ static void pineview_update_wm(struct intel_crtc *unused_crtc)
  *  and the size of 8 whole lines. This adjustment is always performed
  *  in the actual pixel depth regardless of whether FBC is enabled or not."
  */
-static int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
+static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 {
 	int tlb_miss = fifo_size * 64 - width * cpp * 8;
 
@@ -1099,8 +1095,8 @@ static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->base.adjusted_mode;
-	int clock, htotal, cpp, width, wm;
-	int latency = dev_priv->wm.pri_latency[level] * 10;
+	unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
+	unsigned int clock, htotal, cpp, width, wm;
 
 	if (latency == 0)
 		return USHRT_MAX;
@@ -1139,7 +1135,7 @@ static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 		   level == G4X_WM_LEVEL_NORMAL) {
 		wm = intel_wm_method1(clock, cpp, latency);
 	} else {
-		int small, large;
+		unsigned int small, large;
 
 		small = intel_wm_method1(clock, cpp, latency);
 		large = intel_wm_method2(clock, htotal, width, cpp, latency);
@@ -1152,7 +1148,7 @@ static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 
 	wm = DIV_ROUND_UP(wm, 64) + 2;
 
-	return min_t(int, wm, USHRT_MAX);
+	return min_t(unsigned int, wm, USHRT_MAX);
 }
 
 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
@@ -1403,17 +1399,29 @@ static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
 
 static int g4x_compute_intermediate_wm(struct drm_device *dev,
 				       struct intel_crtc *crtc,
-				       struct intel_crtc_state *crtc_state)
+				       struct intel_crtc_state *new_crtc_state)
 {
-	struct g4x_wm_state *intermediate = &crtc_state->wm.g4x.intermediate;
-	const struct g4x_wm_state *optimal = &crtc_state->wm.g4x.optimal;
-	const struct g4x_wm_state *active = &crtc->wm.active.g4x;
+	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
+	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
+	struct intel_atomic_state *intel_state =
+		to_intel_atomic_state(new_crtc_state->base.state);
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(intel_state, crtc);
+	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
 	enum plane_id plane_id;
 
+	if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
+		*intermediate = *optimal;
+
+		intermediate->cxsr = false;
+		intermediate->hpll_en = false;
+		goto out;
+	}
+
 	intermediate->cxsr = optimal->cxsr && active->cxsr &&
-		!crtc_state->disable_cxsr;
+		!new_crtc_state->disable_cxsr;
 	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
-		!crtc_state->disable_cxsr;
+		!new_crtc_state->disable_cxsr;
 	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
 
 	for_each_plane_id_on_crtc(crtc, plane_id) {
@@ -1455,12 +1463,13 @@ static int g4x_compute_intermediate_wm(struct drm_device *dev,
 	WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
 		intermediate->fbc_en && intermediate->hpll_en);
 
+out:
 	/*
 	 * If our intermediate WM are identical to the final WM, then we can
 	 * omit the post-vblank programming; only update if it's different.
 	 */
 	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
-		crtc_state->wm.need_postvbl_update = true;
+		new_crtc_state->wm.need_postvbl_update = true;
 
 	return 0;
 }
@@ -1596,7 +1605,7 @@ static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->base.adjusted_mode;
-	int clock, htotal, cpp, width, wm;
+	unsigned int clock, htotal, cpp, width, wm;
 
 	if (dev_priv->wm.pri_latency[level] == 0)
 		return USHRT_MAX;
@@ -1622,7 +1631,7 @@ static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
 				    dev_priv->wm.pri_latency[level] * 10);
 	}
 
-	return min_t(int, wm, USHRT_MAX);
+	return min_t(unsigned int, wm, USHRT_MAX);
 }
 
 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
@@ -2023,16 +2032,27 @@ static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
 
 static int vlv_compute_intermediate_wm(struct drm_device *dev,
 				       struct intel_crtc *crtc,
-				       struct intel_crtc_state *crtc_state)
+				       struct intel_crtc_state *new_crtc_state)
 {
-	struct vlv_wm_state *intermediate = &crtc_state->wm.vlv.intermediate;
-	const struct vlv_wm_state *optimal = &crtc_state->wm.vlv.optimal;
-	const struct vlv_wm_state *active = &crtc->wm.active.vlv;
+	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
+	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
+	struct intel_atomic_state *intel_state =
+		to_intel_atomic_state(new_crtc_state->base.state);
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(intel_state, crtc);
+	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
 	int level;
 
+	if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
+		*intermediate = *optimal;
+
+		intermediate->cxsr = false;
+		goto out;
+	}
+
 	intermediate->num_levels = min(optimal->num_levels, active->num_levels);
 	intermediate->cxsr = optimal->cxsr && active->cxsr &&
-		!crtc_state->disable_cxsr;
+		!new_crtc_state->disable_cxsr;
 
 	for (level = 0; level < intermediate->num_levels; level++) {
 		enum plane_id plane_id;
@@ -2051,12 +2071,13 @@ static int vlv_compute_intermediate_wm(struct drm_device *dev,
 
 	vlv_invalidate_wms(crtc, intermediate, level);
 
+out:
 	/*
 	 * If our intermediate WM are identical to the final WM, then we can
 	 * omit the post-vblank programming; only update if it's different.
 	 */
 	if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
-		crtc_state->wm.need_postvbl_update = true;
+		new_crtc_state->wm.need_postvbl_update = true;
 
 	return 0;
 }
@@ -2255,8 +2276,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc)
 	else
 		wm_info = &i830_a_wm_info;
 
-	fifo_size = dev_priv->display.get_fifo_size(dev_priv, 0);
-	crtc = intel_get_crtc_for_plane(dev_priv, 0);
+	fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_A);
+	crtc = intel_get_crtc_for_plane(dev_priv, PLANE_A);
 	if (intel_crtc_active(crtc)) {
 		const struct drm_display_mode *adjusted_mode =
 			&crtc->config->base.adjusted_mode;
@@ -2282,8 +2303,8 @@ static void i9xx_update_wm(struct intel_crtc *unused_crtc)
 	if (IS_GEN2(dev_priv))
 		wm_info = &i830_bc_wm_info;
 
-	fifo_size = dev_priv->display.get_fifo_size(dev_priv, 1);
-	crtc = intel_get_crtc_for_plane(dev_priv, 1);
+	fifo_size = dev_priv->display.get_fifo_size(dev_priv, PLANE_B);
+	crtc = intel_get_crtc_for_plane(dev_priv, PLANE_B);
 	if (intel_crtc_active(crtc)) {
 		const struct drm_display_mode *adjusted_mode =
 			&crtc->config->base.adjusted_mode;
@@ -2395,7 +2416,7 @@ static void i845_update_wm(struct intel_crtc *unused_crtc)
 	adjusted_mode = &crtc->config->base.adjusted_mode;
 	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
 				       &i845_wm_info,
-				       dev_priv->display.get_fifo_size(dev_priv, 0),
+				       dev_priv->display.get_fifo_size(dev_priv, PLANE_A),
 				       4, pessimal_latency_ns);
 	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
 	fwater_lo |= (3<<8) | planea_wm;
@@ -3924,6 +3945,7 @@ skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
 				  struct intel_crtc_state *cstate)
 {
+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
 	struct drm_crtc_state *crtc_state = &cstate->base;
 	struct drm_atomic_state *state = crtc_state->state;
 	struct drm_plane *plane;
@@ -3966,7 +3988,7 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
 	crtc_clock = crtc_state->adjusted_mode.crtc_clock;
 	dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
 
-	if (IS_GEMINILAKE(to_i915(intel_crtc->base.dev)))
+	if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
 		dotclk *= 2;
 
 	pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
@@ -6392,29 +6414,8 @@ static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RP_CONTROL, 0);
 }
 
-static void intel_print_rc6_info(struct drm_i915_private *dev_priv, u32 mode)
-{
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-		if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
-			mode = GEN6_RC_CTL_RC6_ENABLE;
-		else
-			mode = 0;
-	}
-	if (HAS_RC6p(dev_priv))
-		DRM_DEBUG_DRIVER("Enabling RC6 states: "
-				 "RC6 %s RC6p %s RC6pp %s\n",
-				 onoff(mode & GEN6_RC_CTL_RC6_ENABLE),
-				 onoff(mode & GEN6_RC_CTL_RC6p_ENABLE),
-				 onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE));
-
-	else
-		DRM_DEBUG_DRIVER("Enabling RC6 states: RC6 %s\n",
-				 onoff(mode & GEN6_RC_CTL_RC6_ENABLE));
-}
-
 static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
 {
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	bool enable_rc6 = true;
 	unsigned long rc6_ctx_base;
 	u32 rc_ctl;
@@ -6439,9 +6440,8 @@ static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
 	 * for this check.
 	 */
 	rc6_ctx_base = I915_READ(RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
-	if (!((rc6_ctx_base >= ggtt->stolen_reserved_base) &&
-	      (rc6_ctx_base + PAGE_SIZE <= ggtt->stolen_reserved_base +
-					ggtt->stolen_reserved_size))) {
+	if (!((rc6_ctx_base >= dev_priv->dsm_reserved.start) &&
+	      (rc6_ctx_base + PAGE_SIZE < dev_priv->dsm_reserved.end))) {
 		DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
 		enable_rc6 = false;
 	}
@@ -6474,42 +6474,30 @@ static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
 	return enable_rc6;
 }
 
-int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6)
+static bool sanitize_rc6(struct drm_i915_private *i915)
 {
-	/* No RC6 before Ironlake and code is gone for ilk. */
-	if (INTEL_INFO(dev_priv)->gen < 6)
-		return 0;
+	struct intel_device_info *info = mkwrite_device_info(i915);
 
-	if (!enable_rc6)
-		return 0;
+	/* Powersaving is controlled by the host when inside a VM */
+	if (intel_vgpu_active(i915))
+		info->has_rc6 = 0;
 
-	if (IS_GEN9_LP(dev_priv) && !bxt_check_bios_rc6_setup(dev_priv)) {
+	if (info->has_rc6 &&
+	    IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(i915)) {
 		DRM_INFO("RC6 disabled by BIOS\n");
-		return 0;
-	}
-
-	/* Respect the kernel parameter if it is set */
-	if (enable_rc6 >= 0) {
-		int mask;
-
-		if (HAS_RC6p(dev_priv))
-			mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
-			       INTEL_RC6pp_ENABLE;
-		else
-			mask = INTEL_RC6_ENABLE;
-
-		if ((enable_rc6 & mask) != enable_rc6)
-			DRM_DEBUG_DRIVER("Adjusting RC6 mask to %d "
-					 "(requested %d, valid %d)\n",
-					 enable_rc6 & mask, enable_rc6, mask);
-
-		return enable_rc6 & mask;
+		info->has_rc6 = 0;
 	}
 
-	if (IS_IVYBRIDGE(dev_priv))
-		return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
+	/*
+	 * We assume that we do not have any deep rc6 levels if we don't have
+	 * have the previous rc6 level supported, i.e. we use HAS_RC6()
+	 * as the initial coarse check for rc6 in general, moving on to
+	 * progressively finer/deeper levels.
+	 */
+	if (!info->has_rc6 && info->has_rc6p)
+		info->has_rc6p = 0;
 
-	return INTEL_RC6_ENABLE;
+	return info->has_rc6;
 }
 
 static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
@@ -6579,9 +6567,10 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 {
 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-	/* Program defaults and thresholds for RPS*/
-	I915_WRITE(GEN6_RC_VIDEO_FREQ,
-		GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
+	/* Program defaults and thresholds for RPS */
+	if (IS_GEN9(dev_priv))
+		I915_WRITE(GEN6_RC_VIDEO_FREQ,
+			GEN9_FREQUENCY(dev_priv->gt_pm.rps.rp1_freq));
 
 	/* 1 second timeout*/
 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
@@ -6601,7 +6590,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-	u32 rc6_mode, rc6_mask = 0;
+	u32 rc6_mode;
 
 	/* 1a: Software RC state - RC0 */
 	I915_WRITE(GEN6_RC_STATE, 0);
@@ -6614,12 +6603,19 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC_CONTROL, 0);
 
 	/* 2b: Program RC6 thresholds.*/
-
-	/* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
-	if (IS_SKYLAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 10) {
+		I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
+		I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
+	} else if (IS_SKYLAKE(dev_priv)) {
+		/*
+		 * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
+		 * when CPG is enabled
+		 */
 		I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
-	else
+	} else {
 		I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
+	}
+
 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
 	for_each_engine(engine, dev_priv, id)
@@ -6635,9 +6631,6 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
 
 	/* 3a: Enable RC6 */
-	if (intel_rc6_enabled() & INTEL_RC6_ENABLE)
-		rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
-	DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
 	I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
 
 	/* WaRsUseTimeoutMode:cnl (pre-prod) */
@@ -6647,7 +6640,9 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 		rc6_mode = GEN6_RC_CTL_EI_MODE(1);
 
 	I915_WRITE(GEN6_RC_CONTROL,
-		   GEN6_RC_CTL_HW_ENABLE | rc6_mode | rc6_mask);
+		   GEN6_RC_CTL_HW_ENABLE |
+		   GEN6_RC_CTL_RC6_ENABLE |
+		   rc6_mode);
 
 	/*
 	 * 3b: Enable Coarse Power Gating only when RC6 is enabled.
@@ -6656,8 +6651,8 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 	if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
 		I915_WRITE(GEN9_PG_ENABLE, 0);
 	else
-		I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
-				(GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0);
+		I915_WRITE(GEN9_PG_ENABLE,
+			   GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -6666,7 +6661,6 @@ static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-	uint32_t rc6_mask = 0;
 
 	/* 1a: Software RC state - RC0 */
 	I915_WRITE(GEN6_RC_STATE, 0);
@@ -6688,13 +6682,11 @@ static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
 
 	/* 3: Enable RC6 */
-	if (intel_rc6_enabled() & INTEL_RC6_ENABLE)
-		rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
-	intel_print_rc6_info(dev_priv, rc6_mask);
 
-	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
-			GEN7_RC_CTL_TO_MODE |
-			rc6_mask);
+	I915_WRITE(GEN6_RC_CONTROL,
+		   GEN6_RC_CTL_HW_ENABLE |
+		   GEN7_RC_CTL_TO_MODE |
+		   GEN6_RC_CTL_RC6_ENABLE);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -6743,9 +6735,8 @@ static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-	u32 rc6vids, rc6_mask = 0;
+	u32 rc6vids, rc6_mask;
 	u32 gtfifodbg;
-	int rc6_mode;
 	int ret;
 
 	I915_WRITE(GEN6_RC_STATE, 0);
@@ -6780,22 +6771,12 @@ static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
 	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
 
-	/* Check if we are enabling RC6 */
-	rc6_mode = intel_rc6_enabled();
-	if (rc6_mode & INTEL_RC6_ENABLE)
-		rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
-
 	/* We don't use those on Haswell */
-	if (!IS_HASWELL(dev_priv)) {
-		if (rc6_mode & INTEL_RC6p_ENABLE)
-			rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
-
-		if (rc6_mode & INTEL_RC6pp_ENABLE)
-			rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
-	}
-
-	intel_print_rc6_info(dev_priv, rc6_mask);
-
+	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
+	if (HAS_RC6p(dev_priv))
+		rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
+	if (HAS_RC6pp(dev_priv))
+		rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
 	I915_WRITE(GEN6_RC_CONTROL,
 		   rc6_mask |
 		   GEN6_RC_CTL_EI_MODE(1) |
@@ -7037,7 +7018,7 @@ static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
 {
 	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
 
-	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
+	WARN_ON(pctx_addr != dev_priv->dsm.start +
 			     dev_priv->vlv_pctx->stolen->start);
 }
 
@@ -7052,16 +7033,15 @@ static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
 
 static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
 {
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	unsigned long pctx_paddr, paddr;
+	resource_size_t pctx_paddr, paddr;
+	resource_size_t pctx_size = 32*1024;
 	u32 pcbr;
-	int pctx_size = 32*1024;
 
 	pcbr = I915_READ(VLV_PCBR);
 	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
 		DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
-		paddr = (dev_priv->mm.stolen_base +
-			 (ggtt->stolen_size - pctx_size));
+		paddr = dev_priv->dsm.end + 1 - pctx_size;
+		GEM_BUG_ON(paddr > U32_MAX);
 
 		pctx_paddr = (paddr & (~4095));
 		I915_WRITE(VLV_PCBR, pctx_paddr);
@@ -7073,16 +7053,16 @@ static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
 static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
 {
 	struct drm_i915_gem_object *pctx;
-	unsigned long pctx_paddr;
+	resource_size_t pctx_paddr;
+	resource_size_t pctx_size = 24*1024;
 	u32 pcbr;
-	int pctx_size = 24*1024;
 
 	pcbr = I915_READ(VLV_PCBR);
 	if (pcbr) {
 		/* BIOS set it up already, grab the pre-alloc'd space */
-		int pcbr_offset;
+		resource_size_t pcbr_offset;
 
-		pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
+		pcbr_offset = (pcbr & (~4095)) - dev_priv->dsm.start;
 		pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv,
 								      pcbr_offset,
 								      I915_GTT_OFFSET_NONE,
@@ -7106,7 +7086,11 @@ static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
 		goto out;
 	}
 
-	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
+	GEM_BUG_ON(range_overflows_t(u64,
+				     dev_priv->dsm.start,
+				     pctx->stolen->start,
+				     U32_MAX));
+	pctx_paddr = dev_priv->dsm.start + pctx->stolen->start;
 	I915_WRITE(VLV_PCBR, pctx_paddr);
 
 out:
@@ -7238,7 +7222,7 @@ static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-	u32 gtfifodbg, rc6_mode = 0, pcbr;
+	u32 gtfifodbg, rc6_mode, pcbr;
 
 	gtfifodbg = I915_READ(GTFIFODBG) & ~(GT_FIFO_SBDEDICATE_FREE_ENTRY_CHV |
 					     GT_FIFO_FREE_ENTRIES_CHV);
@@ -7279,10 +7263,9 @@ static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
 	pcbr = I915_READ(VLV_PCBR);
 
 	/* 3: Enable RC6 */
-	if ((intel_rc6_enabled() & INTEL_RC6_ENABLE) &&
-	    (pcbr >> VLV_PCBR_ADDR_SHIFT))
+	rc6_mode = 0;
+	if (pcbr >> VLV_PCBR_ADDR_SHIFT)
 		rc6_mode = GEN7_RC_CTL_TO_MODE;
-
 	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
@@ -7334,7 +7317,7 @@ static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-	u32 gtfifodbg, rc6_mode = 0;
+	u32 gtfifodbg;
 
 	valleyview_check_pctx(dev_priv);
 
@@ -7367,12 +7350,8 @@ static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
 				      VLV_MEDIA_RC6_COUNT_EN |
 				      VLV_RENDER_RC6_COUNT_EN));
 
-	if (intel_rc6_enabled() & INTEL_RC6_ENABLE)
-		rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
-
-	intel_print_rc6_info(dev_priv, rc6_mode);
-
-	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
+	I915_WRITE(GEN6_RC_CONTROL,
+		   GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
 
 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -7899,12 +7878,11 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
 	 * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
 	 * requirement.
 	 */
-	if (!i915_modparams.enable_rc6) {
+	if (!sanitize_rc6(dev_priv)) {
 		DRM_INFO("RC6 disabled, disabling runtime PM support\n");
 		intel_runtime_pm_get(dev_priv);
 	}
 
-	mutex_lock(&dev_priv->drm.struct_mutex);
 	mutex_lock(&dev_priv->pcu_lock);
 
 	/* Initialize RPS limits (for userspace) */
@@ -7946,9 +7924,6 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
 	rps->boost_freq = rps->max_freq;
 
 	mutex_unlock(&dev_priv->pcu_lock);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-
-	intel_autoenable_gt_powersave(dev_priv);
 }
 
 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
@@ -7956,7 +7931,7 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
 	if (IS_VALLEYVIEW(dev_priv))
 		valleyview_cleanup_gt_powersave(dev_priv);
 
-	if (!i915_modparams.enable_rc6)
+	if (!HAS_RC6(dev_priv))
 		intel_runtime_pm_put(dev_priv);
 }
 
@@ -7973,9 +7948,6 @@ void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
 	if (INTEL_GEN(dev_priv) < 6)
 		return;
 
-	if (cancel_delayed_work_sync(&dev_priv->gt_pm.autoenable_work))
-		intel_runtime_pm_put(dev_priv);
-
 	/* gen6_rps_idle() will be called later to disable interrupts */
 }
 
@@ -8126,7 +8098,8 @@ void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
 
 	mutex_lock(&dev_priv->pcu_lock);
 
-	intel_enable_rc6(dev_priv);
+	if (HAS_RC6(dev_priv))
+		intel_enable_rc6(dev_priv);
 	intel_enable_rps(dev_priv);
 	if (HAS_LLC(dev_priv))
 		intel_enable_llc_pstate(dev_priv);
@@ -8134,65 +8107,6 @@ void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
 	mutex_unlock(&dev_priv->pcu_lock);
 }
 
-static void __intel_autoenable_gt_powersave(struct work_struct *work)
-{
-	struct drm_i915_private *dev_priv =
-		container_of(work,
-			     typeof(*dev_priv),
-			     gt_pm.autoenable_work.work);
-	struct intel_engine_cs *rcs;
-	struct drm_i915_gem_request *req;
-
-	rcs = dev_priv->engine[RCS];
-	if (rcs->last_retired_context)
-		goto out;
-
-	if (!rcs->init_context)
-		goto out;
-
-	mutex_lock(&dev_priv->drm.struct_mutex);
-
-	req = i915_gem_request_alloc(rcs, dev_priv->kernel_context);
-	if (IS_ERR(req))
-		goto unlock;
-
-	if (!i915_modparams.enable_execlists && i915_switch_context(req) == 0)
-		rcs->init_context(req);
-
-	/* Mark the device busy, calling intel_enable_gt_powersave() */
-	i915_add_request(req);
-
-unlock:
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-out:
-	intel_runtime_pm_put(dev_priv);
-}
-
-void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv)
-{
-	if (IS_IRONLAKE_M(dev_priv)) {
-		ironlake_enable_drps(dev_priv);
-		intel_init_emon(dev_priv);
-	} else if (INTEL_INFO(dev_priv)->gen >= 6) {
-		/*
-		 * PCU communication is slow and this doesn't need to be
-		 * done at any specific time, so do this out of our fast path
-		 * to make resume and init faster.
-		 *
-		 * We depend on the HW RC6 power context save/restore
-		 * mechanism when entering D3 through runtime PM suspend. So
-		 * disable RPM until RPS/RC6 is properly setup. We can only
-		 * get here via the driver load/system resume/runtime resume
-		 * paths, so the _noresume version is enough (and in case of
-		 * runtime resume it's necessary).
-		 */
-		if (queue_delayed_work(dev_priv->wq,
-				       &dev_priv->gt_pm.autoenable_work,
-				       round_jiffies_up_relative(HZ)))
-			intel_runtime_pm_get_noresume(dev_priv);
-	}
-}
-
 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
 {
 	/*
@@ -8504,13 +8418,14 @@ static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
 	if (!HAS_PCH_CNP(dev_priv))
 		return;
 
-	/* Wa #1181 */
+	/* Display WA #1181: cnp */
 	I915_WRITE(SOUTH_DSPCLK_GATE_D, I915_READ(SOUTH_DSPCLK_GATE_D) |
 		   CNP_PWM_CGE_GATING_DISABLE);
 }
 
 static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
 {
+	u32 val;
 	cnp_init_clock_gating(dev_priv);
 
 	/* This is not an Wa. Enable for better image quality */
@@ -8525,11 +8440,18 @@ static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
 	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
 		   DISP_FBC_MEMORY_WAKE);
 
+	val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
+	/* ReadHitWriteOnlyDisable:cnl */
+	val |= RCCUNIT_CLKGATE_DIS;
 	/* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
 	if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
-		I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE,
-			   I915_READ(SLICE_UNIT_LEVEL_CLKGATE) |
-			   SARBUNIT_CLKGATE_DIS);
+		val |= SARBUNIT_CLKGATE_DIS;
+	I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
+
+	/* WaDisableVFclkgate:cnl */
+	val = I915_READ(UNSLICE_UNIT_LEVEL_CLKGATE);
+	val |= VFUNIT_CLKGATE_DIS;
+	I915_WRITE(UNSLICE_UNIT_LEVEL_CLKGATE, val);
 }
 
 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -9227,8 +9149,8 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 	return 0;
 }
 
-int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
-			    u32 mbox, u32 val)
+int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv,
+				    u32 mbox, u32 val, int timeout_us)
 {
 	int status;
 
@@ -9251,7 +9173,7 @@ int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
 
 	if (__intel_wait_for_register_fw(dev_priv,
 					 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
-					 500, 0, NULL)) {
+					 timeout_us, 0, NULL)) {
 		DRM_ERROR("timeout waiting for pcode write of 0x%08x to mbox %x to finish for %ps\n",
 			  val, mbox, __builtin_return_address(0));
 		return -ETIMEDOUT;
@@ -9324,7 +9246,7 @@ int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request,
 		ret = 0;
 		goto out;
 	}
-	ret = _wait_for(COND, timeout_base_ms * 1000, 10);
+	ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10);
 	if (!ret)
 		goto out;
 
@@ -9416,8 +9338,6 @@ void intel_pm_setup(struct drm_i915_private *dev_priv)
 {
 	mutex_init(&dev_priv->pcu_lock);
 
-	INIT_DELAYED_WORK(&dev_priv->gt_pm.autoenable_work,
-			  __intel_autoenable_gt_powersave);
 	atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
 
 	dev_priv->runtime_pm.suspended = false;
@@ -9428,12 +9348,13 @@ static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
 			     const i915_reg_t reg)
 {
 	u32 lower, upper, tmp;
+	unsigned long flags;
 	int loop = 2;
 
 	/* The register accessed do not need forcewake. We borrow
 	 * uncore lock to prevent concurrent access to range reg.
 	 */
-	spin_lock_irq(&dev_priv->uncore.lock);
+	spin_lock_irqsave(&dev_priv->uncore.lock, flags);
 
 	/* vlv and chv residency counters are 40 bits in width.
 	 * With a control bit, we can choose between upper or lower
@@ -9464,39 +9385,51 @@ static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
 	 * now.
 	 */
 
-	spin_unlock_irq(&dev_priv->uncore.lock);
+	spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
 
 	return lower | (u64)upper << 8;
 }
 
-u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
+u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
 			   const i915_reg_t reg)
 {
-	u64 time_hw, units, div;
+	u64 time_hw;
+	u32 mul, div;
 
-	if (!intel_rc6_enabled())
+	if (!HAS_RC6(dev_priv))
 		return 0;
 
-	intel_runtime_pm_get(dev_priv);
-
 	/* On VLV and CHV, residency time is in CZ units rather than 1.28us */
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-		units = 1000;
+		mul = 1000000;
 		div = dev_priv->czclk_freq;
-
 		time_hw = vlv_residency_raw(dev_priv, reg);
-	} else if (IS_GEN9_LP(dev_priv)) {
-		units = 1000;
-		div = 1200;		/* 833.33ns */
-
-		time_hw = I915_READ(reg);
 	} else {
-		units = 128000; /* 1.28us */
-		div = 100000;
+		/* 833.33ns units on Gen9LP, 1.28us elsewhere. */
+		if (IS_GEN9_LP(dev_priv)) {
+			mul = 10000;
+			div = 12;
+		} else {
+			mul = 1280;
+			div = 1;
+		}
 
 		time_hw = I915_READ(reg);
 	}
 
-	intel_runtime_pm_put(dev_priv);
-	return DIV_ROUND_UP_ULL(time_hw * units, div);
+	return DIV_ROUND_UP_ULL(time_hw * mul, div);
+}
+
+u32 intel_get_cagf(struct drm_i915_private *dev_priv, u32 rpstat)
+{
+	u32 cagf;
+
+	if (INTEL_GEN(dev_priv) >= 9)
+		cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT;
+	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
+		cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT;
+	else
+		cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT;
+
+	return  cagf;
 }
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 55ea5eb3b7df..2e32615eeada 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -163,7 +163,7 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
 		[3] = 1 - 1,
 		[4] = DP_SET_POWER_D0,
 	};
-	enum port port = dig_port->port;
+	enum port port = dig_port->base.port;
 	u32 aux_ctl;
 	int i;
 
@@ -376,7 +376,7 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
 	 * ones. Since by Display design transcoder EDP is tied to port A
 	 * we can safely escape based on the port A.
 	 */
-	if (HAS_DDI(dev_priv) && dig_port->port != PORT_A) {
+	if (HAS_DDI(dev_priv) && dig_port->base.port != PORT_A) {
 		DRM_DEBUG_KMS("PSR condition failed: Port not supported\n");
 		return;
 	}
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 8da1bde442dd..e2085820b586 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -28,9 +28,12 @@
  */
 
 #include <linux/log2.h>
+
 #include <drm/drmP.h>
-#include "i915_drv.h"
 #include <drm/i915_drm.h>
+
+#include "i915_drv.h"
+#include "i915_gem_render_state.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
 
@@ -337,50 +340,6 @@ gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 	return 0;
 }
 
-static int
-gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
-{
-	u32 flags;
-	u32 *cs;
-
-	cs = intel_ring_begin(req, mode & EMIT_INVALIDATE ? 12 : 6);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	flags = PIPE_CONTROL_CS_STALL;
-
-	if (mode & EMIT_FLUSH) {
-		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
-		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
-		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
-		flags |= PIPE_CONTROL_FLUSH_ENABLE;
-	}
-	if (mode & EMIT_INVALIDATE) {
-		flags |= PIPE_CONTROL_TLB_INVALIDATE;
-		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
-		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
-		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
-		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
-		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
-		flags |= PIPE_CONTROL_QW_WRITE;
-		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
-
-		/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
-		cs = gen8_emit_pipe_control(cs,
-					    PIPE_CONTROL_CS_STALL |
-					    PIPE_CONTROL_STALL_AT_SCOREBOARD,
-					    0);
-	}
-
-	cs = gen8_emit_pipe_control(cs, flags,
-				    i915_ggtt_offset(req->engine->scratch) +
-				    2 * CACHELINE_BYTES);
-
-	intel_ring_advance(req, cs);
-
-	return 0;
-}
-
 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
@@ -424,7 +383,6 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 	} else if (IS_GEN6(dev_priv)) {
 		mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
 	} else {
-		/* XXX: gen8 returns to sanity */
 		mmio = RING_HWS_PGA(engine->mmio_base);
 	}
 
@@ -434,13 +392,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 	I915_WRITE(mmio, engine->status_page.ggtt_offset);
 	POSTING_READ(mmio);
 
-	/*
-	 * Flush the TLB for this page
-	 *
-	 * FIXME: These two bits have disappeared on gen8, so a question
-	 * arises: do we still need this and if so how should we go about
-	 * invalidating the TLB?
-	 */
+	/* Flush the TLB for this page */
 	if (IS_GEN(dev_priv, 6, 7)) {
 		i915_reg_t reg = RING_INSTPM(engine->mmio_base);
 
@@ -480,10 +432,14 @@ static bool stop_ring(struct intel_engine_cs *engine)
 		}
 	}
 
-	I915_WRITE_CTL(engine, 0);
+	I915_WRITE_HEAD(engine, I915_READ_TAIL(engine));
+
 	I915_WRITE_HEAD(engine, 0);
 	I915_WRITE_TAIL(engine, 0);
 
+	/* The ring must be empty before it is disabled */
+	I915_WRITE_CTL(engine, 0);
+
 	return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
 }
 
@@ -604,8 +560,6 @@ static void reset_ring_common(struct intel_engine_cs *engine,
 		struct intel_context *ce = &request->ctx->engine[engine->id];
 		struct i915_hw_ppgtt *ppgtt;
 
-		/* FIXME consider gen8 reset */
-
 		if (ce->state) {
 			I915_WRITE(CCID,
 				   i915_ggtt_offset(ce->state) |
@@ -637,6 +591,7 @@ static void reset_ring_common(struct intel_engine_cs *engine,
 			request->ring->head = request->postfix;
 	} else {
 		engine->legacy_active_context = NULL;
+		engine->legacy_active_ppgtt = NULL;
 	}
 }
 
@@ -706,62 +661,6 @@ static int init_render_ring(struct intel_engine_cs *engine)
 	return init_workarounds_ring(engine);
 }
 
-static void render_ring_cleanup(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	i915_vma_unpin_and_release(&dev_priv->semaphore);
-}
-
-static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *cs)
-{
-	struct drm_i915_private *dev_priv = req->i915;
-	struct intel_engine_cs *waiter;
-	enum intel_engine_id id;
-
-	for_each_engine(waiter, dev_priv, id) {
-		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
-		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
-			continue;
-
-		*cs++ = GFX_OP_PIPE_CONTROL(6);
-		*cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE |
-			PIPE_CONTROL_CS_STALL;
-		*cs++ = lower_32_bits(gtt_offset);
-		*cs++ = upper_32_bits(gtt_offset);
-		*cs++ = req->global_seqno;
-		*cs++ = 0;
-		*cs++ = MI_SEMAPHORE_SIGNAL |
-			MI_SEMAPHORE_TARGET(waiter->hw_id);
-		*cs++ = 0;
-	}
-
-	return cs;
-}
-
-static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *cs)
-{
-	struct drm_i915_private *dev_priv = req->i915;
-	struct intel_engine_cs *waiter;
-	enum intel_engine_id id;
-
-	for_each_engine(waiter, dev_priv, id) {
-		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
-		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
-			continue;
-
-		*cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
-		*cs++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT;
-		*cs++ = upper_32_bits(gtt_offset);
-		*cs++ = req->global_seqno;
-		*cs++ = MI_SEMAPHORE_SIGNAL |
-			MI_SEMAPHORE_TARGET(waiter->hw_id);
-		*cs++ = 0;
-	}
-
-	return cs;
-}
-
 static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *cs)
 {
 	struct drm_i915_private *dev_priv = req->i915;
@@ -844,70 +743,6 @@ static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req, u32 *cs)
 				    req->engine->semaphore.signal(req, cs));
 }
 
-static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req,
-					u32 *cs)
-{
-	struct intel_engine_cs *engine = req->engine;
-
-	if (engine->semaphore.signal)
-		cs = engine->semaphore.signal(req, cs);
-
-	*cs++ = GFX_OP_PIPE_CONTROL(6);
-	*cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
-		PIPE_CONTROL_QW_WRITE;
-	*cs++ = intel_hws_seqno_address(engine);
-	*cs++ = 0;
-	*cs++ = req->global_seqno;
-	/* We're thrashing one dword of HWS. */
-	*cs++ = 0;
-	*cs++ = MI_USER_INTERRUPT;
-	*cs++ = MI_NOOP;
-
-	req->tail = intel_ring_offset(req, cs);
-	assert_ring_tail_valid(req->ring, req->tail);
-}
-
-static const int gen8_render_emit_breadcrumb_sz = 8;
-
-/**
- * intel_ring_sync - sync the waiter to the signaller on seqno
- *
- * @waiter - ring that is waiting
- * @signaller - ring which has, or will signal
- * @seqno - seqno which the waiter will block on
- */
-
-static int
-gen8_ring_sync_to(struct drm_i915_gem_request *req,
-		  struct drm_i915_gem_request *signal)
-{
-	struct drm_i915_private *dev_priv = req->i915;
-	u64 offset = GEN8_WAIT_OFFSET(req->engine, signal->engine->id);
-	struct i915_hw_ppgtt *ppgtt;
-	u32 *cs;
-
-	cs = intel_ring_begin(req, 4);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	*cs++ = MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT |
-		MI_SEMAPHORE_SAD_GTE_SDD;
-	*cs++ = signal->global_seqno;
-	*cs++ = lower_32_bits(offset);
-	*cs++ = upper_32_bits(offset);
-	intel_ring_advance(req, cs);
-
-	/* When the !RCS engines idle waiting upon a semaphore, they lose their
-	 * pagetables and we must reload them before executing the batch.
-	 * We do this on the i915_switch_context() following the wait and
-	 * before the dispatch.
-	 */
-	ppgtt = req->ctx->ppgtt;
-	if (ppgtt && req->engine->id != RCS)
-		ppgtt->pd_dirty_rings |= intel_engine_flag(req->engine);
-	return 0;
-}
-
 static int
 gen6_ring_sync_to(struct drm_i915_gem_request *req,
 		  struct drm_i915_gem_request *signal)
@@ -1083,25 +918,6 @@ hsw_vebox_irq_disable(struct intel_engine_cs *engine)
 	gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask);
 }
 
-static void
-gen8_irq_enable(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	I915_WRITE_IMR(engine,
-		       ~(engine->irq_enable_mask |
-			 engine->irq_keep_mask));
-	POSTING_READ_FW(RING_IMR(engine->mmio_base));
-}
-
-static void
-gen8_irq_disable(struct intel_engine_cs *engine)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
-}
-
 static int
 i965_emit_bb_start(struct drm_i915_gem_request *req,
 		   u64 offset, u32 length,
@@ -1359,12 +1175,13 @@ static int context_pin(struct i915_gem_context *ctx)
 	struct i915_vma *vma = ctx->engine[RCS].state;
 	int ret;
 
-	/* Clear this page out of any CPU caches for coherent swap-in/out.
+	/*
+	 * Clear this page out of any CPU caches for coherent swap-in/out.
 	 * We only want to do this on the first bind so that we do not stall
 	 * on an active context (which by nature is already on the GPU).
 	 */
 	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
-		ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
+		ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
 		if (ret)
 			return ret;
 	}
@@ -1379,11 +1196,34 @@ alloc_context_vma(struct intel_engine_cs *engine)
 	struct drm_i915_private *i915 = engine->i915;
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
+	int err;
 
 	obj = i915_gem_object_create(i915, engine->context_size);
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
+	if (engine->default_state) {
+		void *defaults, *vaddr;
+
+		vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+		if (IS_ERR(vaddr)) {
+			err = PTR_ERR(vaddr);
+			goto err_obj;
+		}
+
+		defaults = i915_gem_object_pin_map(engine->default_state,
+						   I915_MAP_WB);
+		if (IS_ERR(defaults)) {
+			err = PTR_ERR(defaults);
+			goto err_map;
+		}
+
+		memcpy(vaddr, defaults, engine->context_size);
+
+		i915_gem_object_unpin_map(engine->default_state);
+		i915_gem_object_unpin_map(obj);
+	}
+
 	/*
 	 * Try to make the context utilize L3 as well as LLC.
 	 *
@@ -1405,10 +1245,18 @@ alloc_context_vma(struct intel_engine_cs *engine)
 	}
 
 	vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
-	if (IS_ERR(vma))
-		i915_gem_object_put(obj);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_obj;
+	}
 
 	return vma;
+
+err_map:
+	i915_gem_object_unpin_map(obj);
+err_obj:
+	i915_gem_object_put(obj);
+	return ERR_PTR(err);
 }
 
 static struct intel_ring *
@@ -1441,20 +1289,9 @@ intel_ring_context_pin(struct intel_engine_cs *engine,
 		if (ret)
 			goto err;
 
-		ce->state->obj->mm.dirty = true;
 		ce->state->obj->pin_global++;
 	}
 
-	/* The kernel context is only used as a placeholder for flushing the
-	 * active context. It is never used for submitting user rendering and
-	 * as such never requires the golden render context, and so we can skip
-	 * emitting it when we switch to the kernel context. This is required
-	 * as during eviction we cannot allocate and pin the renderstate in
-	 * order to initialise the context.
-	 */
-	if (i915_gem_context_is_kernel(ctx))
-		ce->initialised = true;
-
 	i915_gem_context_get(ctx);
 
 out:
@@ -1548,10 +1385,194 @@ void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
 		intel_ring_reset(engine->buffer, 0);
 }
 
-static int ring_request_alloc(struct drm_i915_gem_request *request)
+static inline int mi_set_context(struct drm_i915_gem_request *rq, u32 flags)
 {
+	struct drm_i915_private *i915 = rq->i915;
+	struct intel_engine_cs *engine = rq->engine;
+	enum intel_engine_id id;
+	const int num_rings =
+		/* Use an extended w/a on gen7 if signalling from other rings */
+		(HAS_LEGACY_SEMAPHORES(i915) && IS_GEN7(i915)) ?
+		INTEL_INFO(i915)->num_rings - 1 :
+		0;
+	int len;
 	u32 *cs;
 
+	flags |= MI_MM_SPACE_GTT;
+	if (IS_HASWELL(i915))
+		/* These flags are for resource streamer on HSW+ */
+		flags |= HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN;
+	else
+		flags |= MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN;
+
+	len = 4;
+	if (IS_GEN7(i915))
+		len += 2 + (num_rings ? 4*num_rings + 6 : 0);
+
+	cs = intel_ring_begin(rq, len);
+	if (IS_ERR(cs))
+		return PTR_ERR(cs);
+
+	/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
+	if (IS_GEN7(i915)) {
+		*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
+		if (num_rings) {
+			struct intel_engine_cs *signaller;
+
+			*cs++ = MI_LOAD_REGISTER_IMM(num_rings);
+			for_each_engine(signaller, i915, id) {
+				if (signaller == engine)
+					continue;
+
+				*cs++ = i915_mmio_reg_offset(
+					   RING_PSMI_CTL(signaller->mmio_base));
+				*cs++ = _MASKED_BIT_ENABLE(
+						GEN6_PSMI_SLEEP_MSG_DISABLE);
+			}
+		}
+	}
+
+	*cs++ = MI_NOOP;
+	*cs++ = MI_SET_CONTEXT;
+	*cs++ = i915_ggtt_offset(rq->ctx->engine[RCS].state) | flags;
+	/*
+	 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
+	 * WaMiSetContext_Hang:snb,ivb,vlv
+	 */
+	*cs++ = MI_NOOP;
+
+	if (IS_GEN7(i915)) {
+		if (num_rings) {
+			struct intel_engine_cs *signaller;
+			i915_reg_t last_reg = {}; /* keep gcc quiet */
+
+			*cs++ = MI_LOAD_REGISTER_IMM(num_rings);
+			for_each_engine(signaller, i915, id) {
+				if (signaller == engine)
+					continue;
+
+				last_reg = RING_PSMI_CTL(signaller->mmio_base);
+				*cs++ = i915_mmio_reg_offset(last_reg);
+				*cs++ = _MASKED_BIT_DISABLE(
+						GEN6_PSMI_SLEEP_MSG_DISABLE);
+			}
+
+			/* Insert a delay before the next switch! */
+			*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
+			*cs++ = i915_mmio_reg_offset(last_reg);
+			*cs++ = i915_ggtt_offset(engine->scratch);
+			*cs++ = MI_NOOP;
+		}
+		*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
+	}
+
+	intel_ring_advance(rq, cs);
+
+	return 0;
+}
+
+static int remap_l3(struct drm_i915_gem_request *rq, int slice)
+{
+	u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice];
+	int i;
+
+	if (!remap_info)
+		return 0;
+
+	cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2);
+	if (IS_ERR(cs))
+		return PTR_ERR(cs);
+
+	/*
+	 * Note: We do not worry about the concurrent register cacheline hang
+	 * here because no other code should access these registers other than
+	 * at initialization time.
+	 */
+	*cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4);
+	for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
+		*cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
+		*cs++ = remap_info[i];
+	}
+	*cs++ = MI_NOOP;
+	intel_ring_advance(rq, cs);
+
+	return 0;
+}
+
+static int switch_context(struct drm_i915_gem_request *rq)
+{
+	struct intel_engine_cs *engine = rq->engine;
+	struct i915_gem_context *to_ctx = rq->ctx;
+	struct i915_hw_ppgtt *to_mm =
+		to_ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt;
+	struct i915_gem_context *from_ctx = engine->legacy_active_context;
+	struct i915_hw_ppgtt *from_mm = engine->legacy_active_ppgtt;
+	u32 hw_flags = 0;
+	int ret, i;
+
+	lockdep_assert_held(&rq->i915->drm.struct_mutex);
+	GEM_BUG_ON(HAS_EXECLISTS(rq->i915));
+
+	if (to_mm != from_mm ||
+	    (to_mm && intel_engine_flag(engine) & to_mm->pd_dirty_rings)) {
+		trace_switch_mm(engine, to_ctx);
+		ret = to_mm->switch_mm(to_mm, rq);
+		if (ret)
+			goto err;
+
+		to_mm->pd_dirty_rings &= ~intel_engine_flag(engine);
+		engine->legacy_active_ppgtt = to_mm;
+		hw_flags = MI_FORCE_RESTORE;
+	}
+
+	if (to_ctx->engine[engine->id].state &&
+	    (to_ctx != from_ctx || hw_flags & MI_FORCE_RESTORE)) {
+		GEM_BUG_ON(engine->id != RCS);
+
+		/*
+		 * The kernel context(s) is treated as pure scratch and is not
+		 * expected to retain any state (as we sacrifice it during
+		 * suspend and on resume it may be corrupted). This is ok,
+		 * as nothing actually executes using the kernel context; it
+		 * is purely used for flushing user contexts.
+		 */
+		if (i915_gem_context_is_kernel(to_ctx))
+			hw_flags = MI_RESTORE_INHIBIT;
+
+		ret = mi_set_context(rq, hw_flags);
+		if (ret)
+			goto err_mm;
+
+		engine->legacy_active_context = to_ctx;
+	}
+
+	if (to_ctx->remap_slice) {
+		for (i = 0; i < MAX_L3_SLICES; i++) {
+			if (!(to_ctx->remap_slice & BIT(i)))
+				continue;
+
+			ret = remap_l3(rq, i);
+			if (ret)
+				goto err_ctx;
+		}
+
+		to_ctx->remap_slice = 0;
+	}
+
+	return 0;
+
+err_ctx:
+	engine->legacy_active_context = from_ctx;
+err_mm:
+	engine->legacy_active_ppgtt = from_mm;
+err:
+	return ret;
+}
+
+static int ring_request_alloc(struct drm_i915_gem_request *request)
+{
+	int ret;
+
 	GEM_BUG_ON(!request->ctx->engine[request->engine->id].pin_count);
 
 	/* Flush enough space to reduce the likelihood of waiting after
@@ -1560,37 +1581,28 @@ static int ring_request_alloc(struct drm_i915_gem_request *request)
 	 */
 	request->reserved_space += LEGACY_REQUEST_SIZE;
 
-	cs = intel_ring_begin(request, 0);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
+	ret = intel_ring_wait_for_space(request->ring, request->reserved_space);
+	if (ret)
+		return ret;
+
+	ret = switch_context(request);
+	if (ret)
+		return ret;
 
 	request->reserved_space -= LEGACY_REQUEST_SIZE;
 	return 0;
 }
 
-static noinline int wait_for_space(struct drm_i915_gem_request *req,
-				   unsigned int bytes)
+static noinline int wait_for_space(struct intel_ring *ring, unsigned int bytes)
 {
-	struct intel_ring *ring = req->ring;
 	struct drm_i915_gem_request *target;
 	long timeout;
 
-	lockdep_assert_held(&req->i915->drm.struct_mutex);
+	lockdep_assert_held(&ring->vma->vm->i915->drm.struct_mutex);
 
 	if (intel_ring_update_space(ring) >= bytes)
 		return 0;
 
-	/*
-	 * Space is reserved in the ringbuffer for finalising the request,
-	 * as that cannot be allowed to fail. During request finalisation,
-	 * reserved_space is set to 0 to stop the overallocation and the
-	 * assumption is that then we never need to wait (which has the
-	 * risk of failing with EINTR).
-	 *
-	 * See also i915_gem_request_alloc() and i915_add_request().
-	 */
-	GEM_BUG_ON(!req->reserved_space);
-
 	list_for_each_entry(target, &ring->request_list, ring_link) {
 		/* Would completion of this request free enough space? */
 		if (bytes <= __intel_ring_space(target->postfix,
@@ -1614,6 +1626,22 @@ static noinline int wait_for_space(struct drm_i915_gem_request *req,
 	return 0;
 }
 
+int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes)
+{
+	GEM_BUG_ON(bytes > ring->effective_size);
+	if (unlikely(bytes > ring->effective_size - ring->emit))
+		bytes += ring->size - ring->emit;
+
+	if (unlikely(bytes > ring->space)) {
+		int ret = wait_for_space(ring, bytes);
+		if (unlikely(ret))
+			return ret;
+	}
+
+	GEM_BUG_ON(ring->space < bytes);
+	return 0;
+}
+
 u32 *intel_ring_begin(struct drm_i915_gem_request *req,
 		      unsigned int num_dwords)
 {
@@ -1653,7 +1681,20 @@ u32 *intel_ring_begin(struct drm_i915_gem_request *req,
 	}
 
 	if (unlikely(total_bytes > ring->space)) {
-		int ret = wait_for_space(req, total_bytes);
+		int ret;
+
+		/*
+		 * Space is reserved in the ringbuffer for finalising the
+		 * request, as that cannot be allowed to fail. During request
+		 * finalisation, reserved_space is set to 0 to stop the
+		 * overallocation and the assumption is that then we never need
+		 * to wait (which has the risk of failing with EINTR).
+		 *
+		 * See also i915_gem_request_alloc() and i915_add_request().
+		 */
+		GEM_BUG_ON(!req->reserved_space);
+
+		ret = wait_for_space(ring, total_bytes);
 		if (unlikely(ret))
 			return ERR_PTR(ret);
 	}
@@ -1748,8 +1789,6 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 		return PTR_ERR(cs);
 
 	cmd = MI_FLUSH_DW;
-	if (INTEL_GEN(req->i915) >= 8)
-		cmd += 1;
 
 	/* We always require a command barrier so that subsequent
 	 * commands, such as breadcrumb interrupts, are strictly ordered
@@ -1769,38 +1808,9 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 
 	*cs++ = cmd;
 	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
-	if (INTEL_GEN(req->i915) >= 8) {
-		*cs++ = 0; /* upper addr */
-		*cs++ = 0; /* value */
-	} else  {
-		*cs++ = 0;
-		*cs++ = MI_NOOP;
-	}
-	intel_ring_advance(req, cs);
-	return 0;
-}
-
-static int
-gen8_emit_bb_start(struct drm_i915_gem_request *req,
-		   u64 offset, u32 len,
-		   unsigned int dispatch_flags)
-{
-	bool ppgtt = USES_PPGTT(req->i915) &&
-			!(dispatch_flags & I915_DISPATCH_SECURE);
-	u32 *cs;
-
-	cs = intel_ring_begin(req, 4);
-	if (IS_ERR(cs))
-		return PTR_ERR(cs);
-
-	/* FIXME(BDW): Address space and security selectors. */
-	*cs++ = MI_BATCH_BUFFER_START_GEN8 | (ppgtt << 8) | (dispatch_flags &
-		I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
-	*cs++ = lower_32_bits(offset);
-	*cs++ = upper_32_bits(offset);
+	*cs++ = 0;
 	*cs++ = MI_NOOP;
 	intel_ring_advance(req, cs);
-
 	return 0;
 }
 
@@ -1857,8 +1867,6 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 		return PTR_ERR(cs);
 
 	cmd = MI_FLUSH_DW;
-	if (INTEL_GEN(req->i915) >= 8)
-		cmd += 1;
 
 	/* We always require a command barrier so that subsequent
 	 * commands, such as breadcrumb interrupts, are strictly ordered
@@ -1877,13 +1885,8 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 		cmd |= MI_INVALIDATE_TLB;
 	*cs++ = cmd;
 	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
-	if (INTEL_GEN(req->i915) >= 8) {
-		*cs++ = 0; /* upper addr */
-		*cs++ = 0; /* value */
-	} else  {
-		*cs++ = 0;
-		*cs++ = MI_NOOP;
-	}
+	*cs++ = 0;
+	*cs++ = MI_NOOP;
 	intel_ring_advance(req, cs);
 
 	return 0;
@@ -1892,110 +1895,61 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
 				       struct intel_engine_cs *engine)
 {
-	struct drm_i915_gem_object *obj;
-	int ret, i;
+	int i;
 
-	if (!i915_modparams.semaphores)
+	if (!HAS_LEGACY_SEMAPHORES(dev_priv))
 		return;
 
-	if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore) {
-		struct i915_vma *vma;
-
-		obj = i915_gem_object_create(dev_priv, PAGE_SIZE);
-		if (IS_ERR(obj))
-			goto err;
-
-		vma = i915_vma_instance(obj, &dev_priv->ggtt.base, NULL);
-		if (IS_ERR(vma))
-			goto err_obj;
-
-		ret = i915_gem_object_set_to_gtt_domain(obj, false);
-		if (ret)
-			goto err_obj;
-
-		ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
-		if (ret)
-			goto err_obj;
-
-		dev_priv->semaphore = vma;
-	}
-
-	if (INTEL_GEN(dev_priv) >= 8) {
-		u32 offset = i915_ggtt_offset(dev_priv->semaphore);
-
-		engine->semaphore.sync_to = gen8_ring_sync_to;
-		engine->semaphore.signal = gen8_xcs_signal;
-
-		for (i = 0; i < I915_NUM_ENGINES; i++) {
-			u32 ring_offset;
-
-			if (i != engine->id)
-				ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
-			else
-				ring_offset = MI_SEMAPHORE_SYNC_INVALID;
-
-			engine->semaphore.signal_ggtt[i] = ring_offset;
-		}
-	} else if (INTEL_GEN(dev_priv) >= 6) {
-		engine->semaphore.sync_to = gen6_ring_sync_to;
-		engine->semaphore.signal = gen6_signal;
+	GEM_BUG_ON(INTEL_GEN(dev_priv) < 6);
+	engine->semaphore.sync_to = gen6_ring_sync_to;
+	engine->semaphore.signal = gen6_signal;
 
-		/*
-		 * The current semaphore is only applied on pre-gen8
-		 * platform.  And there is no VCS2 ring on the pre-gen8
-		 * platform. So the semaphore between RCS and VCS2 is
-		 * initialized as INVALID.  Gen8 will initialize the
-		 * sema between VCS2 and RCS later.
-		 */
-		for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) {
-			static const struct {
-				u32 wait_mbox;
-				i915_reg_t mbox_reg;
-			} sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = {
-				[RCS_HW] = {
-					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
-					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
-					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
-				},
-				[VCS_HW] = {
-					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
-					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
-					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
-				},
-				[BCS_HW] = {
-					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
-					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
-					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
-				},
-				[VECS_HW] = {
-					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
-					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
-					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
-				},
-			};
+	/*
+	 * The current semaphore is only applied on pre-gen8
+	 * platform.  And there is no VCS2 ring on the pre-gen8
+	 * platform. So the semaphore between RCS and VCS2 is
+	 * initialized as INVALID.
+	 */
+	for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) {
+		static const struct {
 			u32 wait_mbox;
 			i915_reg_t mbox_reg;
+		} sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = {
+			[RCS_HW] = {
+				[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
+				[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
+				[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
+			},
+			[VCS_HW] = {
+				[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
+				[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
+				[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
+			},
+			[BCS_HW] = {
+				[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
+				[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
+				[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
+			},
+			[VECS_HW] = {
+				[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
+				[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
+				[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
+			},
+		};
+		u32 wait_mbox;
+		i915_reg_t mbox_reg;
 
-			if (i == engine->hw_id) {
-				wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
-				mbox_reg = GEN6_NOSYNC;
-			} else {
-				wait_mbox = sem_data[engine->hw_id][i].wait_mbox;
-				mbox_reg = sem_data[engine->hw_id][i].mbox_reg;
-			}
-
-			engine->semaphore.mbox.wait[i] = wait_mbox;
-			engine->semaphore.mbox.signal[i] = mbox_reg;
+		if (i == engine->hw_id) {
+			wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
+			mbox_reg = GEN6_NOSYNC;
+		} else {
+			wait_mbox = sem_data[engine->hw_id][i].wait_mbox;
+			mbox_reg = sem_data[engine->hw_id][i].mbox_reg;
 		}
-	}
-
-	return;
 
-err_obj:
-	i915_gem_object_put(obj);
-err:
-	DRM_DEBUG_DRIVER("Failed to allocate space for semaphores, disabling\n");
-	i915_modparams.semaphores = 0;
+		engine->semaphore.mbox.wait[i] = wait_mbox;
+		engine->semaphore.mbox.signal[i] = mbox_reg;
+	}
 }
 
 static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
@@ -2003,11 +1957,7 @@ static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
 {
 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << engine->irq_shift;
 
-	if (INTEL_GEN(dev_priv) >= 8) {
-		engine->irq_enable = gen8_irq_enable;
-		engine->irq_disable = gen8_irq_disable;
-		engine->irq_seqno_barrier = gen6_seqno_barrier;
-	} else if (INTEL_GEN(dev_priv) >= 6) {
+	if (INTEL_GEN(dev_priv) >= 6) {
 		engine->irq_enable = gen6_irq_enable;
 		engine->irq_disable = gen6_irq_disable;
 		engine->irq_seqno_barrier = gen6_seqno_barrier;
@@ -2028,17 +1978,23 @@ static void i9xx_set_default_submission(struct intel_engine_cs *engine)
 {
 	engine->submit_request = i9xx_submit_request;
 	engine->cancel_requests = cancel_requests;
+
+	engine->park = NULL;
+	engine->unpark = NULL;
 }
 
 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
 {
+	i9xx_set_default_submission(engine);
 	engine->submit_request = gen6_bsd_submit_request;
-	engine->cancel_requests = cancel_requests;
 }
 
 static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
 				      struct intel_engine_cs *engine)
 {
+	/* gen8+ are only supported with execlists */
+	GEM_BUG_ON(INTEL_GEN(dev_priv) >= 8);
+
 	intel_ring_init_irq(dev_priv, engine);
 	intel_ring_init_semaphores(dev_priv, engine);
 
@@ -2052,26 +2008,20 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
 
 	engine->emit_breadcrumb = i9xx_emit_breadcrumb;
 	engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz;
-	if (i915_modparams.semaphores) {
+	if (HAS_LEGACY_SEMAPHORES(dev_priv)) {
 		int num_rings;
 
 		engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
 
 		num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
-		if (INTEL_GEN(dev_priv) >= 8) {
-			engine->emit_breadcrumb_sz += num_rings * 6;
-		} else {
-			engine->emit_breadcrumb_sz += num_rings * 3;
-			if (num_rings & 1)
-				engine->emit_breadcrumb_sz++;
-		}
+		engine->emit_breadcrumb_sz += num_rings * 3;
+		if (num_rings & 1)
+			engine->emit_breadcrumb_sz++;
 	}
 
 	engine->set_default_submission = i9xx_set_default_submission;
 
-	if (INTEL_GEN(dev_priv) >= 8)
-		engine->emit_bb_start = gen8_emit_bb_start;
-	else if (INTEL_GEN(dev_priv) >= 6)
+	if (INTEL_GEN(dev_priv) >= 6)
 		engine->emit_bb_start = gen6_emit_bb_start;
 	else if (INTEL_GEN(dev_priv) >= 4)
 		engine->emit_bb_start = i965_emit_bb_start;
@@ -2091,20 +2041,7 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
 	if (HAS_L3_DPF(dev_priv))
 		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 
-	if (INTEL_GEN(dev_priv) >= 8) {
-		engine->init_context = intel_rcs_ctx_init;
-		engine->emit_breadcrumb = gen8_render_emit_breadcrumb;
-		engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz;
-		engine->emit_flush = gen8_render_ring_flush;
-		if (i915_modparams.semaphores) {
-			int num_rings;
-
-			engine->semaphore.signal = gen8_rcs_signal;
-
-			num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
-			engine->emit_breadcrumb_sz += num_rings * 8;
-		}
-	} else if (INTEL_GEN(dev_priv) >= 6) {
+	if (INTEL_GEN(dev_priv) >= 6) {
 		engine->init_context = intel_rcs_ctx_init;
 		engine->emit_flush = gen7_render_ring_flush;
 		if (IS_GEN6(dev_priv))
@@ -2123,7 +2060,6 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
 		engine->emit_bb_start = hsw_emit_bb_start;
 
 	engine->init_hw = init_render_ring;
-	engine->cleanup = render_ring_cleanup;
 
 	ret = intel_init_ring_buffer(engine);
 	if (ret)
@@ -2153,8 +2089,7 @@ int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
 		if (IS_GEN6(dev_priv))
 			engine->set_default_submission = gen6_bsd_set_default_submission;
 		engine->emit_flush = gen6_bsd_ring_flush;
-		if (INTEL_GEN(dev_priv) < 8)
-			engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
+		engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
 	} else {
 		engine->mmio_base = BSD_RING_BASE;
 		engine->emit_flush = bsd_ring_flush;
@@ -2174,8 +2109,7 @@ int intel_init_blt_ring_buffer(struct intel_engine_cs *engine)
 	intel_ring_default_vfuncs(dev_priv, engine);
 
 	engine->emit_flush = gen6_ring_flush;
-	if (INTEL_GEN(dev_priv) < 8)
-		engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
+	engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
 
 	return intel_init_ring_buffer(engine);
 }
@@ -2187,12 +2121,9 @@ int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine)
 	intel_ring_default_vfuncs(dev_priv, engine);
 
 	engine->emit_flush = gen6_ring_flush;
-
-	if (INTEL_GEN(dev_priv) < 8) {
-		engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
-		engine->irq_enable = hsw_vebox_irq_enable;
-		engine->irq_disable = hsw_vebox_irq_disable;
-	}
+	engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
+	engine->irq_enable = hsw_vebox_irq_enable;
+	engine->irq_disable = hsw_vebox_irq_disable;
 
 	return intel_init_ring_buffer(engine);
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 2863d5a65187..a0e7a6c2a57c 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -6,6 +6,7 @@
 #include "i915_gem_batch_pool.h"
 #include "i915_gem_request.h"
 #include "i915_gem_timeline.h"
+#include "i915_pmu.h"
 #include "i915_selftest.h"
 
 struct drm_printer;
@@ -47,16 +48,6 @@ struct intel_hw_status_page {
 /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
  * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
  */
-#define gen8_semaphore_seqno_size sizeof(uint64_t)
-#define GEN8_SEMAPHORE_OFFSET(__from, __to)			     \
-	(((__from) * I915_NUM_ENGINES  + (__to)) * gen8_semaphore_seqno_size)
-#define GEN8_SIGNAL_OFFSET(__ring, to)			     \
-	(dev_priv->semaphore->node.start + \
-	 GEN8_SEMAPHORE_OFFSET((__ring)->id, (to)))
-#define GEN8_WAIT_OFFSET(__ring, from)			     \
-	(dev_priv->semaphore->node.start + \
-	 GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
-
 enum intel_engine_hangcheck_action {
 	ENGINE_IDLE = 0,
 	ENGINE_WAIT,
@@ -166,7 +157,6 @@ struct i915_ctx_workarounds {
 };
 
 struct drm_i915_gem_request;
-struct intel_render_state;
 
 /*
  * Engine IDs definitions.
@@ -195,9 +185,9 @@ struct i915_priolist {
  */
 struct intel_engine_execlists {
 	/**
-	 * @irq_tasklet: softirq tasklet for bottom handler
+	 * @tasklet: softirq tasklet for bottom handler
 	 */
-	struct tasklet_struct irq_tasklet;
+	struct tasklet_struct tasklet;
 
 	/**
 	 * @default_priolist: priority list for I915_PRIORITY_NORMAL
@@ -209,6 +199,11 @@ struct intel_engine_execlists {
 	 */
 	bool no_priolist;
 
+	/**
+	 * @elsp: the ExecList Submission Port register
+	 */
+	u32 __iomem *elsp;
+
 	/**
 	 * @port: execlist port states
 	 *
@@ -253,6 +248,7 @@ struct intel_engine_execlists {
 	unsigned int active;
 #define EXECLISTS_ACTIVE_USER 0
 #define EXECLISTS_ACTIVE_PREEMPT 1
+#define EXECLISTS_ACTIVE_HWACK 2
 
 	/**
 	 * @port_mask: number of execlist ports - 1
@@ -290,11 +286,14 @@ struct intel_engine_execlists {
 struct intel_engine_cs {
 	struct drm_i915_private *i915;
 	char name[INTEL_ENGINE_CS_MAX_NAME];
+
 	enum intel_engine_id id;
-	unsigned int uabi_id;
 	unsigned int hw_id;
 	unsigned int guc_id;
 
+	u8 uabi_id;
+	u8 uabi_class;
+
 	u8 class;
 	u8 instance;
 	u32 context_size;
@@ -304,7 +303,7 @@ struct intel_engine_cs {
 	struct intel_ring *buffer;
 	struct intel_timeline *timeline;
 
-	struct intel_render_state *render_state;
+	struct drm_i915_gem_object *default_state;
 
 	atomic_t irq_count;
 	unsigned long irq_posted;
@@ -340,12 +339,35 @@ struct intel_engine_cs {
 		struct timer_list hangcheck; /* detect missed interrupts */
 
 		unsigned int hangcheck_interrupts;
+		unsigned int irq_enabled;
 
 		bool irq_armed : 1;
-		bool irq_enabled : 1;
 		I915_SELFTEST_DECLARE(bool mock : 1);
 	} breadcrumbs;
 
+	struct {
+		/**
+		 * @enable: Bitmask of enable sample events on this engine.
+		 *
+		 * Bits correspond to sample event types, for instance
+		 * I915_SAMPLE_QUEUED is bit 0 etc.
+		 */
+		u32 enable;
+		/**
+		 * @enable_count: Reference count for the enabled samplers.
+		 *
+		 * Index number corresponds to the bit number from @enable.
+		 */
+		unsigned int enable_count[I915_PMU_SAMPLE_BITS];
+		/**
+		 * @sample: Counter values for sampling events.
+		 *
+		 * Our internal timer stores the current counters in this field.
+		 */
+#define I915_ENGINE_SAMPLE_MAX (I915_SAMPLE_SEMA + 1)
+		struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_MAX];
+	} pmu;
+
 	/*
 	 * A pool of objects to use as shadow copies of client batch buffers
 	 * when the command parser is enabled. Prevents the client from
@@ -366,6 +388,9 @@ struct intel_engine_cs {
 	void		(*reset_hw)(struct intel_engine_cs *engine,
 				    struct drm_i915_gem_request *req);
 
+	void		(*park)(struct intel_engine_cs *engine);
+	void		(*unpark)(struct intel_engine_cs *engine);
+
 	void		(*set_default_submission)(struct intel_engine_cs *engine);
 
 	struct intel_ring *(*context_pin)(struct intel_engine_cs *engine,
@@ -462,18 +487,15 @@ struct intel_engine_cs {
 	 *  ie. transpose of f(x, y)
 	 */
 	struct {
-		union {
 #define GEN6_SEMAPHORE_LAST	VECS_HW
 #define GEN6_NUM_SEMAPHORES	(GEN6_SEMAPHORE_LAST + 1)
 #define GEN6_SEMAPHORES_MASK	GENMASK(GEN6_SEMAPHORE_LAST, 0)
-			struct {
-				/* our mbox written by others */
-				u32		wait[GEN6_NUM_SEMAPHORES];
-				/* mboxes this ring signals to */
-				i915_reg_t	signal[GEN6_NUM_SEMAPHORES];
-			} mbox;
-			u64		signal_ggtt[I915_NUM_ENGINES];
-		};
+		struct {
+			/* our mbox written by others */
+			u32		wait[GEN6_NUM_SEMAPHORES];
+			/* mboxes this ring signals to */
+			i915_reg_t	signal[GEN6_NUM_SEMAPHORES];
+		} mbox;
 
 		/* AKA wait() */
 		int	(*sync_to)(struct drm_i915_gem_request *req,
@@ -501,13 +523,16 @@ struct intel_engine_cs {
 	 * stream (ring).
 	 */
 	struct i915_gem_context *legacy_active_context;
+	struct i915_hw_ppgtt *legacy_active_ppgtt;
 
 	/* status_notifier: list of callbacks for context-switch changes */
 	struct atomic_notifier_head context_status_notifier;
 
 	struct intel_engine_hangcheck hangcheck;
 
-	bool needs_cmd_parser;
+#define I915_ENGINE_NEEDS_CMD_PARSER BIT(0)
+#define I915_ENGINE_SUPPORTS_STATS   BIT(1)
+	unsigned int flags;
 
 	/*
 	 * Table of commands the command parser needs to know about
@@ -532,8 +557,50 @@ struct intel_engine_cs {
 	 * certain bits to encode the command length in the header).
 	 */
 	u32 (*get_cmd_length_mask)(u32 cmd_header);
+
+	struct {
+		/**
+		 * @lock: Lock protecting the below fields.
+		 */
+		spinlock_t lock;
+		/**
+		 * @enabled: Reference count indicating number of listeners.
+		 */
+		unsigned int enabled;
+		/**
+		 * @active: Number of contexts currently scheduled in.
+		 */
+		unsigned int active;
+		/**
+		 * @enabled_at: Timestamp when busy stats were enabled.
+		 */
+		ktime_t enabled_at;
+		/**
+		 * @start: Timestamp of the last idle to active transition.
+		 *
+		 * Idle is defined as active == 0, active is active > 0.
+		 */
+		ktime_t start;
+		/**
+		 * @total: Total time this engine was busy.
+		 *
+		 * Accumulated time not counting the most recent block in cases
+		 * where engine is currently busy (active > 0).
+		 */
+		ktime_t total;
+	} stats;
 };
 
+static inline bool intel_engine_needs_cmd_parser(struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_NEEDS_CMD_PARSER;
+}
+
+static inline bool intel_engine_supports_stats(struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_SUPPORTS_STATS;
+}
+
 static inline void
 execlists_set_active(struct intel_engine_execlists *execlists,
 		     unsigned int bit)
@@ -555,6 +622,12 @@ execlists_is_active(const struct intel_engine_execlists *execlists,
 	return test_bit(bit, (unsigned long *)&execlists->active);
 }
 
+void
+execlists_cancel_port_requests(struct intel_engine_execlists * const execlists);
+
+void
+execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists);
+
 static inline unsigned int
 execlists_num_ports(const struct intel_engine_execlists * const execlists)
 {
@@ -624,6 +697,8 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
  */
 #define I915_GEM_HWS_INDEX		0x30
 #define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
+#define I915_GEM_HWS_PREEMPT_INDEX	0x32
+#define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
 #define I915_GEM_HWS_SCRATCH_INDEX	0x40
 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
 
@@ -648,6 +723,7 @@ void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
 
 int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
 
+int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes);
 u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req,
 				   unsigned int n);
 
@@ -776,6 +852,11 @@ static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
 	return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
 }
 
+static inline u32 intel_hws_preempt_done_address(struct intel_engine_cs *engine)
+{
+	return engine->status_page.ggtt_offset + I915_GEM_HWS_PREEMPT_ADDR;
+}
+
 /* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
 
@@ -846,6 +927,9 @@ unsigned int intel_engine_wakeup(struct intel_engine_cs *engine);
 #define ENGINE_WAKEUP_WAITER BIT(0)
 #define ENGINE_WAKEUP_ASLEEP BIT(1)
 
+void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine);
+void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine);
+
 void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
 
@@ -864,14 +948,123 @@ static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
 	return batch + 6;
 }
 
+static inline u32 *
+gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset)
+{
+	/* We're using qword write, offset should be aligned to 8 bytes. */
+	GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8));
+
+	/* w/a for post sync ops following a GPGPU operation we
+	 * need a prior CS_STALL, which is emitted by the flush
+	 * following the batch.
+	 */
+	*cs++ = GFX_OP_PIPE_CONTROL(6);
+	*cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
+		PIPE_CONTROL_QW_WRITE;
+	*cs++ = gtt_offset;
+	*cs++ = 0;
+	*cs++ = value;
+	/* We're thrashing one dword of HWS. */
+	*cs++ = 0;
+
+	return cs;
+}
+
+static inline u32 *
+gen8_emit_ggtt_write(u32 *cs, u32 value, u32 gtt_offset)
+{
+	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
+	GEM_BUG_ON(gtt_offset & (1 << 5));
+	/* Offset should be aligned to 8 bytes for both (QW/DW) write types */
+	GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8));
+
+	*cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
+	*cs++ = gtt_offset | MI_FLUSH_DW_USE_GTT;
+	*cs++ = 0;
+	*cs++ = value;
+
+	return cs;
+}
+
 bool intel_engine_is_idle(struct intel_engine_cs *engine);
 bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
 
-void intel_engines_mark_idle(struct drm_i915_private *i915);
+bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine);
+
+void intel_engines_park(struct drm_i915_private *i915);
+void intel_engines_unpark(struct drm_i915_private *i915);
+
 void intel_engines_reset_default_submission(struct drm_i915_private *i915);
+unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915);
 
 bool intel_engine_can_store_dword(struct intel_engine_cs *engine);
 
-void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *p);
+__printf(3, 4)
+void intel_engine_dump(struct intel_engine_cs *engine,
+		       struct drm_printer *m,
+		       const char *header, ...);
+
+struct intel_engine_cs *
+intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance);
+
+static inline void intel_engine_context_in(struct intel_engine_cs *engine)
+{
+	unsigned long flags;
+
+	if (READ_ONCE(engine->stats.enabled) == 0)
+		return;
+
+	spin_lock_irqsave(&engine->stats.lock, flags);
+
+	if (engine->stats.enabled > 0) {
+		if (engine->stats.active++ == 0)
+			engine->stats.start = ktime_get();
+		GEM_BUG_ON(engine->stats.active == 0);
+	}
+
+	spin_unlock_irqrestore(&engine->stats.lock, flags);
+}
+
+static inline void intel_engine_context_out(struct intel_engine_cs *engine)
+{
+	unsigned long flags;
+
+	if (READ_ONCE(engine->stats.enabled) == 0)
+		return;
+
+	spin_lock_irqsave(&engine->stats.lock, flags);
+
+	if (engine->stats.enabled > 0) {
+		ktime_t last;
+
+		if (engine->stats.active && --engine->stats.active == 0) {
+			/*
+			 * Decrement the active context count and in case GPU
+			 * is now idle add up to the running total.
+			 */
+			last = ktime_sub(ktime_get(), engine->stats.start);
+
+			engine->stats.total = ktime_add(engine->stats.total,
+							last);
+		} else if (engine->stats.active == 0) {
+			/*
+			 * After turning on engine stats, context out might be
+			 * the first event in which case we account from the
+			 * time stats gathering was turned on.
+			 */
+			last = ktime_sub(ktime_get(), engine->stats.enabled_at);
+
+			engine->stats.total = ktime_add(engine->stats.total,
+							last);
+		}
+	}
+
+	spin_unlock_irqrestore(&engine->stats.lock, flags);
+}
+
+int intel_enable_engine_stats(struct intel_engine_cs *engine);
+void intel_disable_engine_stats(struct intel_engine_cs *engine);
+
+ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine);
 
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 7e115f3927f6..d758da6156a8 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -130,6 +130,8 @@ intel_display_power_domain_str(enum intel_display_power_domain domain)
 		return "INIT";
 	case POWER_DOMAIN_MODESET:
 		return "MODESET";
+	case POWER_DOMAIN_GT_IRQ:
+		return "GT_IRQ";
 	default:
 		MISSING_CASE(domain);
 		return "?";
@@ -715,7 +717,8 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 
 	dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
-	WARN_ON(!intel_cdclk_state_compare(&dev_priv->cdclk.hw, &cdclk_state));
+	/* Can't read out voltage_level so can't use intel_cdclk_changed() */
+	WARN_ON(intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, &cdclk_state));
 
 	gen9_assert_dbuf_enabled(dev_priv);
 
@@ -1714,6 +1717,7 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_INIT))
 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (		\
 	SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_GT_IRQ) |			\
 	BIT_ULL(POWER_DOMAIN_MODESET) |			\
 	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
 	BIT_ULL(POWER_DOMAIN_INIT))
@@ -1732,12 +1736,13 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_AUX_C) |			\
 	BIT_ULL(POWER_DOMAIN_AUDIO) |			\
 	BIT_ULL(POWER_DOMAIN_VGA) |				\
-	BIT_ULL(POWER_DOMAIN_GMBUS) |			\
 	BIT_ULL(POWER_DOMAIN_INIT))
 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (		\
 	BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_GT_IRQ) |			\
 	BIT_ULL(POWER_DOMAIN_MODESET) |			\
 	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
+	BIT_ULL(POWER_DOMAIN_GMBUS) |			\
 	BIT_ULL(POWER_DOMAIN_INIT))
 #define BXT_DPIO_CMN_A_POWER_DOMAINS (			\
 	BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |		\
@@ -1794,6 +1799,7 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
 	BIT_ULL(POWER_DOMAIN_INIT))
 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (		\
 	GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |		\
+	BIT_ULL(POWER_DOMAIN_GT_IRQ) |			\
 	BIT_ULL(POWER_DOMAIN_MODESET) |			\
 	BIT_ULL(POWER_DOMAIN_AUX_A) |			\
 	BIT_ULL(POWER_DOMAIN_GMBUS) |			\
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 7437944b388f..2b8764897d68 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1429,6 +1429,8 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
 	u8 val;
 	bool ret;
 
+	pipe_config->output_types |= BIT(INTEL_OUTPUT_SDVO);
+
 	sdvox = I915_READ(intel_sdvo->sdvo_reg);
 
 	ret = intel_sdvo_get_input_timing(intel_sdvo, &dtd);
@@ -1510,7 +1512,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
-	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
 	u32 temp;
 
 	intel_sdvo_set_active_outputs(intel_sdvo, 0);
@@ -1569,7 +1571,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder,
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
 	u32 temp;
 	bool input1, input2;
 	int i;
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 4a8a5d918a83..dd485f59eb1d 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -263,13 +263,9 @@ skl_update_plane(struct intel_plane *plane,
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-	if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
 		I915_WRITE_FW(PLANE_COLOR_CTL(pipe, plane_id),
-			      PLANE_COLOR_PIPE_GAMMA_ENABLE |
-			      PLANE_COLOR_PIPE_CSC_ENABLE |
-			      PLANE_COLOR_PLANE_GAMMA_DISABLE);
-	}
-
+			      plane_state->color_ctl);
 	if (key->flags) {
 		I915_WRITE_FW(PLANE_KEYVAL(pipe, plane_id), key->min_value);
 		I915_WRITE_FW(PLANE_KEYMAX(pipe, plane_id), key->max_value);
@@ -1056,6 +1052,9 @@ intel_check_sprite_plane(struct intel_plane *plane,
 		state->ctl = g4x_sprite_ctl(crtc_state, state);
 	}
 
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+		state->color_ctl = glk_plane_color_ctl(crtc_state, state);
+
 	return 0;
 }
 
@@ -1385,7 +1384,7 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv,
 	}
 
 	intel_plane->pipe = pipe;
-	intel_plane->plane = plane;
+	intel_plane->i9xx_plane = plane;
 	intel_plane->id = PLANE_SPRITE0 + plane;
 	intel_plane->frontbuffer_bit = INTEL_FRONTBUFFER_SPRITE(pipe, plane);
 	intel_plane->check_plane = intel_check_sprite_plane;
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index a79a7591b2cf..b3dabc219e6a 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -822,7 +822,7 @@ intel_enable_tv(struct intel_encoder *encoder,
 
 	/* Prevents vblank waits from timing out in intel_tv_detect_type() */
 	intel_wait_for_vblank(dev_priv,
-			      to_intel_crtc(encoder->base.crtc)->pipe);
+			      to_intel_crtc(pipe_config->base.crtc)->pipe);
 
 	I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE);
 }
@@ -868,6 +868,8 @@ static void
 intel_tv_get_config(struct intel_encoder *encoder,
 		    struct intel_crtc_state *pipe_config)
 {
+	pipe_config->output_types |= BIT(INTEL_OUTPUT_TVOUT);
+
 	pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock;
 }
 
@@ -980,7 +982,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
 				const struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
 	struct intel_tv *intel_tv = enc_to_tv(encoder);
 	const struct tv_mode *tv_mode = intel_tv_mode_find(conn_state);
 	u32 tv_ctl;
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index 25bd162f38d2..d82ca0f438f5 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -23,8 +23,9 @@
  */
 
 #include "intel_uc.h"
+#include "intel_guc_submission.h"
+#include "intel_guc.h"
 #include "i915_drv.h"
-#include "i915_guc_submission.h"
 
 /* Reset GuC providing us with fresh state for both GuC and HuC.
  */
@@ -33,9 +34,9 @@ static int __intel_uc_reset_hw(struct drm_i915_private *dev_priv)
 	int ret;
 	u32 guc_status;
 
-	ret = intel_guc_reset(dev_priv);
+	ret = intel_reset_guc(dev_priv);
 	if (ret) {
-		DRM_ERROR("GuC reset failed, ret = %d\n", ret);
+		DRM_ERROR("Failed to reset GuC, ret = %d\n", ret);
 		return ret;
 	}
 
@@ -47,55 +48,93 @@ static int __intel_uc_reset_hw(struct drm_i915_private *dev_priv)
 	return ret;
 }
 
-void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
+static int __get_platform_enable_guc(struct drm_i915_private *dev_priv)
 {
-	if (!HAS_GUC(dev_priv)) {
-		if (i915_modparams.enable_guc_loading > 0 ||
-		    i915_modparams.enable_guc_submission > 0)
-			DRM_INFO("Ignoring GuC options, no hardware\n");
+	struct intel_uc_fw *guc_fw = &dev_priv->guc.fw;
+	struct intel_uc_fw *huc_fw = &dev_priv->huc.fw;
+	int enable_guc = 0;
 
-		i915_modparams.enable_guc_loading = 0;
-		i915_modparams.enable_guc_submission = 0;
-		return;
-	}
+	/* Default is to enable GuC/HuC if we know their firmwares */
+	if (intel_uc_fw_is_selected(guc_fw))
+		enable_guc |= ENABLE_GUC_SUBMISSION;
+	if (intel_uc_fw_is_selected(huc_fw))
+		enable_guc |= ENABLE_GUC_LOAD_HUC;
 
-	/* A negative value means "use platform default" */
-	if (i915_modparams.enable_guc_loading < 0)
-		i915_modparams.enable_guc_loading = HAS_GUC_UCODE(dev_priv);
+	/* Any platform specific fine-tuning can be done here */
+
+	return enable_guc;
+}
 
-	/* Verify firmware version */
-	if (i915_modparams.enable_guc_loading) {
-		if (HAS_HUC_UCODE(dev_priv))
-			intel_huc_select_fw(&dev_priv->huc);
+/**
+ * intel_uc_sanitize_options - sanitize uC related modparam options
+ * @dev_priv: device private
+ *
+ * In case of "enable_guc" option this function will attempt to modify
+ * it only if it was initially set to "auto(-1)". Default value for this
+ * modparam varies between platforms and it is hardcoded in driver code.
+ * Any other modparam value is only monitored against availability of the
+ * related hardware or firmware definitions.
+ */
+void intel_uc_sanitize_options(struct drm_i915_private *dev_priv)
+{
+	struct intel_uc_fw *guc_fw = &dev_priv->guc.fw;
+	struct intel_uc_fw *huc_fw = &dev_priv->huc.fw;
 
-		if (intel_guc_fw_select(&dev_priv->guc))
-			i915_modparams.enable_guc_loading = 0;
+	/* A negative value means "use platform default" */
+	if (i915_modparams.enable_guc < 0)
+		i915_modparams.enable_guc = __get_platform_enable_guc(dev_priv);
+
+	DRM_DEBUG_DRIVER("enable_guc=%d (submission:%s huc:%s)\n",
+			 i915_modparams.enable_guc,
+			 yesno(intel_uc_is_using_guc_submission()),
+			 yesno(intel_uc_is_using_huc()));
+
+	/* Verify GuC firmware availability */
+	if (intel_uc_is_using_guc() && !intel_uc_fw_is_selected(guc_fw)) {
+		DRM_WARN("Incompatible option detected: enable_guc=%d, %s!\n",
+			 i915_modparams.enable_guc,
+			 !HAS_GUC(dev_priv) ? "no GuC hardware" :
+					      "no GuC firmware");
 	}
 
-	/* Can't enable guc submission without guc loaded */
-	if (!i915_modparams.enable_guc_loading)
-		i915_modparams.enable_guc_submission = 0;
+	/* Verify HuC firmware availability */
+	if (intel_uc_is_using_huc() && !intel_uc_fw_is_selected(huc_fw)) {
+		DRM_WARN("Incompatible option detected: enable_guc=%d, %s!\n",
+			 i915_modparams.enable_guc,
+			 !HAS_HUC(dev_priv) ? "no HuC hardware" :
+					      "no HuC firmware");
+	}
 
-	/* A negative value means "use platform default" */
-	if (i915_modparams.enable_guc_submission < 0)
-		i915_modparams.enable_guc_submission = HAS_GUC_SCHED(dev_priv);
+	/* Make sure that sanitization was done */
+	GEM_BUG_ON(i915_modparams.enable_guc < 0);
 }
 
 void intel_uc_init_early(struct drm_i915_private *dev_priv)
 {
 	intel_guc_init_early(&dev_priv->guc);
+	intel_huc_init_early(&dev_priv->huc);
 }
 
 void intel_uc_init_fw(struct drm_i915_private *dev_priv)
 {
-	intel_uc_fw_fetch(dev_priv, &dev_priv->huc.fw);
+	if (!USES_GUC(dev_priv))
+		return;
+
+	if (USES_HUC(dev_priv))
+		intel_uc_fw_fetch(dev_priv, &dev_priv->huc.fw);
+
 	intel_uc_fw_fetch(dev_priv, &dev_priv->guc.fw);
 }
 
 void intel_uc_fini_fw(struct drm_i915_private *dev_priv)
 {
+	if (!USES_GUC(dev_priv))
+		return;
+
 	intel_uc_fw_fini(&dev_priv->guc.fw);
-	intel_uc_fw_fini(&dev_priv->huc.fw);
+
+	if (USES_HUC(dev_priv))
+		intel_uc_fw_fini(&dev_priv->huc.fw);
 }
 
 /**
@@ -149,30 +188,89 @@ static void guc_disable_communication(struct intel_guc *guc)
 	guc->send = intel_guc_send_nop;
 }
 
-int intel_uc_init_hw(struct drm_i915_private *dev_priv)
+int intel_uc_init_wq(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	if (!USES_GUC(dev_priv))
+		return 0;
+
+	ret = intel_guc_init_wq(&dev_priv->guc);
+	if (ret) {
+		DRM_ERROR("Couldn't allocate workqueues for GuC\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+void intel_uc_fini_wq(struct drm_i915_private *dev_priv)
+{
+	if (!USES_GUC(dev_priv))
+		return;
+
+	intel_guc_fini_wq(&dev_priv->guc);
+}
+
+int intel_uc_init(struct drm_i915_private *dev_priv)
 {
 	struct intel_guc *guc = &dev_priv->guc;
-	int ret, attempts;
+	int ret;
 
-	if (!i915_modparams.enable_guc_loading)
+	if (!USES_GUC(dev_priv))
 		return 0;
 
-	guc_disable_communication(guc);
-	gen9_reset_guc_interrupts(dev_priv);
+	if (!HAS_GUC(dev_priv))
+		return -ENODEV;
 
-	/* We need to notify the guc whenever we change the GGTT */
-	i915_ggtt_enable_guc(dev_priv);
+	ret = intel_guc_init(guc);
+	if (ret)
+		return ret;
 
-	if (i915_modparams.enable_guc_submission) {
+	if (USES_GUC_SUBMISSION(dev_priv)) {
 		/*
 		 * This is stuff we need to have available at fw load time
 		 * if we are planning to enable submission later
 		 */
-		ret = i915_guc_submission_init(dev_priv);
-		if (ret)
-			goto err_guc;
+		ret = intel_guc_submission_init(guc);
+		if (ret) {
+			intel_guc_fini(guc);
+			return ret;
+		}
 	}
 
+	return 0;
+}
+
+void intel_uc_fini(struct drm_i915_private *dev_priv)
+{
+	struct intel_guc *guc = &dev_priv->guc;
+
+	if (!USES_GUC(dev_priv))
+		return;
+
+	GEM_BUG_ON(!HAS_GUC(dev_priv));
+
+	if (USES_GUC_SUBMISSION(dev_priv))
+		intel_guc_submission_fini(guc);
+
+	intel_guc_fini(guc);
+}
+
+int intel_uc_init_hw(struct drm_i915_private *dev_priv)
+{
+	struct intel_guc *guc = &dev_priv->guc;
+	struct intel_huc *huc = &dev_priv->huc;
+	int ret, attempts;
+
+	if (!USES_GUC(dev_priv))
+		return 0;
+
+	GEM_BUG_ON(!HAS_GUC(dev_priv));
+
+	guc_disable_communication(guc);
+	gen9_reset_guc_interrupts(dev_priv);
+
 	/* init WOPCM */
 	I915_WRITE(GUC_WOPCM_SIZE, intel_guc_wopcm_size(dev_priv));
 	I915_WRITE(DMA_GUC_WOPCM_OFFSET,
@@ -192,9 +290,14 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 		 */
 		ret = __intel_uc_reset_hw(dev_priv);
 		if (ret)
-			goto err_submission;
+			goto err_out;
+
+		if (USES_HUC(dev_priv)) {
+			ret = intel_huc_init_hw(huc);
+			if (ret)
+				goto err_out;
+		}
 
-		intel_huc_init_hw(&dev_priv->huc);
 		intel_guc_init_params(guc);
 		ret = intel_guc_fw_upload(guc);
 		if (ret == 0 || ret != -EAGAIN)
@@ -212,79 +315,67 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
 	if (ret)
 		goto err_log_capture;
 
-	intel_huc_auth(&dev_priv->huc);
-	if (i915_modparams.enable_guc_submission) {
+	if (USES_HUC(dev_priv)) {
+		ret = intel_huc_auth(huc);
+		if (ret)
+			goto err_communication;
+	}
+
+	if (USES_GUC_SUBMISSION(dev_priv)) {
 		if (i915_modparams.guc_log_level >= 0)
 			gen9_enable_guc_interrupts(dev_priv);
 
-		ret = i915_guc_submission_enable(dev_priv);
+		ret = intel_guc_submission_enable(guc);
 		if (ret)
 			goto err_interrupts;
 	}
 
-	dev_info(dev_priv->drm.dev, "GuC %s (firmware %s [version %u.%u])\n",
-		 i915_modparams.enable_guc_submission ? "submission enabled" :
-							"loaded",
-		 guc->fw.path,
+	dev_info(dev_priv->drm.dev, "GuC firmware version %u.%u\n",
 		 guc->fw.major_ver_found, guc->fw.minor_ver_found);
+	dev_info(dev_priv->drm.dev, "GuC submission %s\n",
+		 enableddisabled(USES_GUC_SUBMISSION(dev_priv)));
+	dev_info(dev_priv->drm.dev, "HuC %s\n",
+		 enableddisabled(USES_HUC(dev_priv)));
 
 	return 0;
 
 	/*
 	 * We've failed to load the firmware :(
-	 *
-	 * Decide whether to disable GuC submission and fall back to
-	 * execlist mode, and whether to hide the error by returning
-	 * zero or to return -EIO, which the caller will treat as a
-	 * nonfatal error (i.e. it doesn't prevent driver load, but
-	 * marks the GPU as wedged until reset).
 	 */
 err_interrupts:
-	guc_disable_communication(guc);
 	gen9_disable_guc_interrupts(dev_priv);
+err_communication:
+	guc_disable_communication(guc);
 err_log_capture:
 	guc_capture_load_err_log(guc);
-err_submission:
-	if (i915_modparams.enable_guc_submission)
-		i915_guc_submission_fini(dev_priv);
-err_guc:
-	i915_ggtt_disable_guc(dev_priv);
-
-	if (i915_modparams.enable_guc_loading > 1 ||
-	    i915_modparams.enable_guc_submission > 1) {
-		DRM_ERROR("GuC init failed. Firmware loading disabled.\n");
-		ret = -EIO;
-	} else {
-		DRM_NOTE("GuC init failed. Firmware loading disabled.\n");
-		ret = 0;
-	}
-
-	if (i915_modparams.enable_guc_submission) {
-		i915_modparams.enable_guc_submission = 0;
-		DRM_NOTE("Falling back from GuC submission to execlist mode\n");
-	}
-
-	i915_modparams.enable_guc_loading = 0;
+err_out:
+	/*
+	 * Note that there is no fallback as either user explicitly asked for
+	 * the GuC or driver default option was to run with the GuC enabled.
+	 */
+	if (GEM_WARN_ON(ret == -EIO))
+		ret = -EINVAL;
 
+	dev_err(dev_priv->drm.dev, "GuC initialization failed %d\n", ret);
 	return ret;
 }
 
 void intel_uc_fini_hw(struct drm_i915_private *dev_priv)
 {
-	guc_free_load_err_log(&dev_priv->guc);
+	struct intel_guc *guc = &dev_priv->guc;
 
-	if (!i915_modparams.enable_guc_loading)
+	guc_free_load_err_log(guc);
+
+	if (!USES_GUC(dev_priv))
 		return;
 
-	if (i915_modparams.enable_guc_submission)
-		i915_guc_submission_disable(dev_priv);
+	GEM_BUG_ON(!HAS_GUC(dev_priv));
 
-	guc_disable_communication(&dev_priv->guc);
+	if (USES_GUC_SUBMISSION(dev_priv))
+		intel_guc_submission_disable(guc);
 
-	if (i915_modparams.enable_guc_submission) {
-		gen9_disable_guc_interrupts(dev_priv);
-		i915_guc_submission_fini(dev_priv);
-	}
+	guc_disable_communication(guc);
 
-	i915_ggtt_disable_guc(dev_priv);
+	if (USES_GUC_SUBMISSION(dev_priv))
+		gen9_disable_guc_interrupts(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/intel_uc.h b/drivers/gpu/drm/i915/intel_uc.h
index e18d3bb02088..8a7249722ef1 100644
--- a/drivers/gpu/drm/i915/intel_uc.h
+++ b/drivers/gpu/drm/i915/intel_uc.h
@@ -26,13 +26,36 @@
 
 #include "intel_guc.h"
 #include "intel_huc.h"
+#include "i915_params.h"
 
 void intel_uc_sanitize_options(struct drm_i915_private *dev_priv);
 void intel_uc_init_early(struct drm_i915_private *dev_priv);
 void intel_uc_init_mmio(struct drm_i915_private *dev_priv);
 void intel_uc_init_fw(struct drm_i915_private *dev_priv);
 void intel_uc_fini_fw(struct drm_i915_private *dev_priv);
+int intel_uc_init_wq(struct drm_i915_private *dev_priv);
+void intel_uc_fini_wq(struct drm_i915_private *dev_priv);
 int intel_uc_init_hw(struct drm_i915_private *dev_priv);
 void intel_uc_fini_hw(struct drm_i915_private *dev_priv);
+int intel_uc_init(struct drm_i915_private *dev_priv);
+void intel_uc_fini(struct drm_i915_private *dev_priv);
+
+static inline bool intel_uc_is_using_guc(void)
+{
+	GEM_BUG_ON(i915_modparams.enable_guc < 0);
+	return i915_modparams.enable_guc > 0;
+}
+
+static inline bool intel_uc_is_using_guc_submission(void)
+{
+	GEM_BUG_ON(i915_modparams.enable_guc < 0);
+	return i915_modparams.enable_guc & ENABLE_GUC_SUBMISSION;
+}
+
+static inline bool intel_uc_is_using_huc(void)
+{
+	GEM_BUG_ON(i915_modparams.enable_guc < 0);
+	return i915_modparams.enable_guc & ENABLE_GUC_LOAD_HUC;
+}
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_uc_fw.c b/drivers/gpu/drm/i915/intel_uc_fw.c
index 973888e94cba..784eff9cdfc8 100644
--- a/drivers/gpu/drm/i915/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/intel_uc_fw.c
@@ -105,7 +105,7 @@ void intel_uc_fw_fetch(struct drm_i915_private *dev_priv,
 	}
 
 	/* now RSA */
-	if (css->key_size_dw != UOS_RSA_SCRATCH_MAX_COUNT) {
+	if (css->key_size_dw != UOS_RSA_SCRATCH_COUNT) {
 		DRM_WARN("%s: Mismatched firmware RSA key size (%u)\n",
 			 intel_uc_fw_type_repr(uc_fw->type), css->key_size_dw);
 		err = -ENOEXEC;
@@ -214,7 +214,7 @@ int intel_uc_fw_upload(struct intel_uc_fw *uc_fw,
 			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path);
 
 	if (uc_fw->fetch_status != INTEL_UC_FIRMWARE_SUCCESS)
-		return -EIO;
+		return -ENOEXEC;
 
 	uc_fw->load_status = INTEL_UC_FIRMWARE_PENDING;
 	DRM_DEBUG_DRIVER("%s fw load %s\n",
@@ -299,7 +299,7 @@ void intel_uc_fw_fini(struct intel_uc_fw *uc_fw)
  *
  * Pretty printer for uC firmware.
  */
-void intel_uc_fw_dump(struct intel_uc_fw *uc_fw, struct drm_printer *p)
+void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p)
 {
 	drm_printf(p, "%s firmware: %s\n",
 		   intel_uc_fw_type_repr(uc_fw->type), uc_fw->path);
diff --git a/drivers/gpu/drm/i915/intel_uc_fw.h b/drivers/gpu/drm/i915/intel_uc_fw.h
index 132903669391..d5fd4609c785 100644
--- a/drivers/gpu/drm/i915/intel_uc_fw.h
+++ b/drivers/gpu/drm/i915/intel_uc_fw.h
@@ -110,12 +110,17 @@ void intel_uc_fw_init(struct intel_uc_fw *uc_fw, enum intel_uc_fw_type type)
 	uc_fw->type = type;
 }
 
+static inline bool intel_uc_fw_is_selected(struct intel_uc_fw *uc_fw)
+{
+	return uc_fw->path != NULL;
+}
+
 void intel_uc_fw_fetch(struct drm_i915_private *dev_priv,
 		       struct intel_uc_fw *uc_fw);
 int intel_uc_fw_upload(struct intel_uc_fw *uc_fw,
 		       int (*xfer)(struct intel_uc_fw *uc_fw,
 				   struct i915_vma *vma));
 void intel_uc_fw_fini(struct intel_uc_fw *uc_fw);
-void intel_uc_fw_dump(struct intel_uc_fw *uc_fw, struct drm_printer *p);
+void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 8c2ce81f01c2..89547b614aa6 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -69,17 +69,104 @@ fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
 			       HRTIMER_MODE_REL);
 }
 
+static inline int
+__wait_for_ack(const struct drm_i915_private *i915,
+	       const struct intel_uncore_forcewake_domain *d,
+	       const u32 ack,
+	       const u32 value)
+{
+	return wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) & ack) == value,
+			       FORCEWAKE_ACK_TIMEOUT_MS);
+}
+
+static inline int
+wait_ack_clear(const struct drm_i915_private *i915,
+	       const struct intel_uncore_forcewake_domain *d,
+	       const u32 ack)
+{
+	return __wait_for_ack(i915, d, ack, 0);
+}
+
+static inline int
+wait_ack_set(const struct drm_i915_private *i915,
+	     const struct intel_uncore_forcewake_domain *d,
+	     const u32 ack)
+{
+	return __wait_for_ack(i915, d, ack, ack);
+}
+
 static inline void
 fw_domain_wait_ack_clear(const struct drm_i915_private *i915,
 			 const struct intel_uncore_forcewake_domain *d)
 {
-	if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
-			     FORCEWAKE_KERNEL) == 0,
-			    FORCEWAKE_ACK_TIMEOUT_MS))
+	if (wait_ack_clear(i915, d, FORCEWAKE_KERNEL))
 		DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
 			  intel_uncore_forcewake_domain_to_str(d->id));
 }
 
+enum ack_type {
+	ACK_CLEAR = 0,
+	ACK_SET
+};
+
+static int
+fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915,
+				 const struct intel_uncore_forcewake_domain *d,
+				 const enum ack_type type)
+{
+	const u32 ack_bit = FORCEWAKE_KERNEL;
+	const u32 value = type == ACK_SET ? ack_bit : 0;
+	unsigned int pass;
+	bool ack_detected;
+
+	/*
+	 * There is a possibility of driver's wake request colliding
+	 * with hardware's own wake requests and that can cause
+	 * hardware to not deliver the driver's ack message.
+	 *
+	 * Use a fallback bit toggle to kick the gpu state machine
+	 * in the hope that the original ack will be delivered along with
+	 * the fallback ack.
+	 *
+	 * This workaround is described in HSDES #1604254524
+	 */
+
+	pass = 1;
+	do {
+		wait_ack_clear(i915, d, FORCEWAKE_KERNEL_FALLBACK);
+
+		__raw_i915_write32(i915, d->reg_set,
+				   _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK));
+		/* Give gt some time to relax before the polling frenzy */
+		udelay(10 * pass);
+		wait_ack_set(i915, d, FORCEWAKE_KERNEL_FALLBACK);
+
+		ack_detected = (__raw_i915_read32(i915, d->reg_ack) & ack_bit) == value;
+
+		__raw_i915_write32(i915, d->reg_set,
+				   _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK));
+	} while (!ack_detected && pass++ < 10);
+
+	DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",
+			 intel_uncore_forcewake_domain_to_str(d->id),
+			 type == ACK_SET ? "set" : "clear",
+			 __raw_i915_read32(i915, d->reg_ack),
+			 pass);
+
+	return ack_detected ? 0 : -ETIMEDOUT;
+}
+
+static inline void
+fw_domain_wait_ack_clear_fallback(const struct drm_i915_private *i915,
+				  const struct intel_uncore_forcewake_domain *d)
+{
+	if (likely(!wait_ack_clear(i915, d, FORCEWAKE_KERNEL)))
+		return;
+
+	if (fw_domain_wait_ack_with_fallback(i915, d, ACK_CLEAR))
+		fw_domain_wait_ack_clear(i915, d);
+}
+
 static inline void
 fw_domain_get(struct drm_i915_private *i915,
 	      const struct intel_uncore_forcewake_domain *d)
@@ -88,16 +175,25 @@ fw_domain_get(struct drm_i915_private *i915,
 }
 
 static inline void
-fw_domain_wait_ack(const struct drm_i915_private *i915,
-		   const struct intel_uncore_forcewake_domain *d)
+fw_domain_wait_ack_set(const struct drm_i915_private *i915,
+		       const struct intel_uncore_forcewake_domain *d)
 {
-	if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
-			     FORCEWAKE_KERNEL),
-			    FORCEWAKE_ACK_TIMEOUT_MS))
+	if (wait_ack_set(i915, d, FORCEWAKE_KERNEL))
 		DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
 			  intel_uncore_forcewake_domain_to_str(d->id));
 }
 
+static inline void
+fw_domain_wait_ack_set_fallback(const struct drm_i915_private *i915,
+				const struct intel_uncore_forcewake_domain *d)
+{
+	if (likely(!wait_ack_set(i915, d, FORCEWAKE_KERNEL)))
+		return;
+
+	if (fw_domain_wait_ack_with_fallback(i915, d, ACK_SET))
+		fw_domain_wait_ack_set(i915, d);
+}
+
 static inline void
 fw_domain_put(const struct drm_i915_private *i915,
 	      const struct intel_uncore_forcewake_domain *d)
@@ -119,7 +215,27 @@ fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
 	}
 
 	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
-		fw_domain_wait_ack(i915, d);
+		fw_domain_wait_ack_set(i915, d);
+
+	i915->uncore.fw_domains_active |= fw_domains;
+}
+
+static void
+fw_domains_get_with_fallback(struct drm_i915_private *i915,
+			     enum forcewake_domains fw_domains)
+{
+	struct intel_uncore_forcewake_domain *d;
+	unsigned int tmp;
+
+	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
+
+	for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
+		fw_domain_wait_ack_clear_fallback(i915, d);
+		fw_domain_get(i915, d);
+	}
+
+	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
+		fw_domain_wait_ack_set_fallback(i915, d);
 
 	i915->uncore.fw_domains_active |= fw_domains;
 }
@@ -229,6 +345,7 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
 	return HRTIMER_NORESTART;
 }
 
+/* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 					 bool restore)
 {
@@ -237,6 +354,8 @@ static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 	int retry_count = 100;
 	enum forcewake_domains fw, active_domains;
 
+	iosf_mbi_assert_punit_acquired();
+
 	/* Hold uncore.lock across reset to prevent any register access
 	 * with forcewake not set correctly. Wait until all pending
 	 * timers are run before holding.
@@ -416,14 +535,18 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 				   GT_FIFO_CTL_RC6_POLICY_STALL);
 	}
 
+	iosf_mbi_punit_acquire();
 	intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
+	iosf_mbi_punit_release();
 }
 
 void intel_uncore_suspend(struct drm_i915_private *dev_priv)
 {
-	iosf_mbi_unregister_pmic_bus_access_notifier(
+	iosf_mbi_punit_acquire();
+	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 		&dev_priv->uncore.pmic_bus_access_nb);
 	intel_uncore_forcewake_reset(dev_priv, false);
+	iosf_mbi_punit_release();
 }
 
 void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
@@ -442,9 +565,6 @@ void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv)
 
 void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
 {
-	i915_modparams.enable_rc6 =
-		sanitize_rc6_option(dev_priv, i915_modparams.enable_rc6);
-
 	/* BIOS often leaves RC6 enabled, but disable it for hw init */
 	intel_sanitize_gt_powersave(dev_priv);
 }
@@ -1148,7 +1268,8 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 	}
 
 	if (INTEL_GEN(dev_priv) >= 9) {
-		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
+		dev_priv->uncore.funcs.force_wake_get =
+			fw_domains_get_with_fallback;
 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_RENDER_GEN9,
@@ -1309,18 +1430,18 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 
 	iosf_mbi_register_pmic_bus_access_notifier(
 		&dev_priv->uncore.pmic_bus_access_nb);
-
-	i915_check_and_clear_faults(dev_priv);
 }
 
 void intel_uncore_fini(struct drm_i915_private *dev_priv)
 {
-	iosf_mbi_unregister_pmic_bus_access_notifier(
-		&dev_priv->uncore.pmic_bus_access_nb);
-
 	/* Paranoia: make sure we have disabled everything before we exit. */
 	intel_uncore_sanitize(dev_priv);
+
+	iosf_mbi_punit_acquire();
+	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
+		&dev_priv->uncore.pmic_bus_access_nb);
 	intel_uncore_forcewake_reset(dev_priv, false);
+	iosf_mbi_punit_release();
 }
 
 static const struct reg_whitelist {
@@ -1400,10 +1521,14 @@ static void gen3_stop_engine(struct intel_engine_cs *engine)
 		DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
 				 engine->name);
 
-	I915_WRITE_FW(RING_CTL(base), 0);
+	I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base)));
+
 	I915_WRITE_FW(RING_HEAD(base), 0);
 	I915_WRITE_FW(RING_TAIL(base), 0);
 
+	/* The ring must be empty before it is disabled */
+	I915_WRITE_FW(RING_CTL(base), 0);
+
 	/* Check acts as a post */
 	if (I915_READ_FW(RING_HEAD(base)) != 0)
 		DRM_DEBUG_DRIVER("%s: ring head not parked\n",
@@ -1801,18 +1926,13 @@ bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
 	return intel_get_gpu_reset(dev_priv) != NULL;
 }
 
-/*
- * When GuC submission is enabled, GuC manages ELSP and can initiate the
- * engine reset too. For now, fall back to full GPU reset if it is enabled.
- */
 bool intel_has_reset_engine(struct drm_i915_private *dev_priv)
 {
 	return (dev_priv->info.has_reset_engine &&
-		!dev_priv->guc.execbuf_client &&
 		i915_modparams.reset >= 2);
 }
 
-int intel_guc_reset(struct drm_i915_private *dev_priv)
+int intel_reset_guc(struct drm_i915_private *dev_priv)
 {
 	int ret;
 
diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h
index f225c288a121..98dff6058d3c 100644
--- a/drivers/gpu/drm/i915/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/intel_vbt_defs.h
@@ -304,6 +304,10 @@ struct bdb_general_features {
 #define DVO_PORT_MIPIC		23				/* 171 */
 #define DVO_PORT_MIPID		24				/* 171 */
 
+#define HDMI_MAX_DATA_RATE_PLATFORM	0			/* 204 */
+#define HDMI_MAX_DATA_RATE_297		1			/* 204 */
+#define HDMI_MAX_DATA_RATE_165		2			/* 204 */
+
 #define LEGACY_CHILD_DEVICE_CONFIG_SIZE		33
 
 /* DDC Bus DDI Type 155+ */
@@ -342,8 +346,8 @@ struct child_device_config {
 			u8 i2c_speed;
 			u8 dp_onboard_redriver;			/* 158 */
 			u8 dp_ondock_redriver;			/* 158 */
-			u8 hdmi_level_shifter_value:4;		/* 169 */
-			u8 hdmi_max_data_rate:4;		/* 204 */
+			u8 hdmi_level_shifter_value:5;		/* 169 */
+			u8 hdmi_max_data_rate:3;		/* 204 */
 			u16 dtd_buf_ptr;			/* 161 */
 			u8 edidless_efp:1;			/* 161 */
 			u8 compression_enable:1;		/* 198 */
@@ -408,6 +412,8 @@ struct child_device_config {
 	u16 dp_gpio_pin_num;					/* 195 */
 	u8 dp_iboost_level:4;					/* 196 */
 	u8 hdmi_iboost_level:4;					/* 196 */
+	u8 dp_max_link_rate:2;					/* 216 CNL+ */
+	u8 dp_max_link_rate_reserved:6;				/* 216 */
 } __packed;
 
 struct bdb_general_definitions {
diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c
index 5cc8101bb2b1..2ea69394f428 100644
--- a/drivers/gpu/drm/i915/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/selftests/huge_pages.c
@@ -27,6 +27,7 @@
 #include <linux/prime_numbers.h>
 
 #include "mock_drm.h"
+#include "i915_random.h"
 
 static const unsigned int page_sizes[] = {
 	I915_GTT_PAGE_SIZE_2M,
@@ -989,17 +990,9 @@ static int gpu_write(struct i915_vma *vma,
 	i915_vma_unpin(batch);
 	i915_vma_close(batch);
 
-	err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
-	if (err)
-		goto err_request;
-
-	err = i915_switch_context(rq);
-	if (err)
-		goto err_request;
-
-	err = rq->engine->emit_bb_start(rq,
-					batch->node.start, batch->node.size,
-					flags);
+	err = engine->emit_bb_start(rq,
+				    batch->node.start, batch->node.size,
+				    flags);
 	if (err)
 		goto err_request;
 
@@ -1047,19 +1040,78 @@ static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val)
 	return err;
 }
 
+static int __igt_write_huge(struct i915_gem_context *ctx,
+			    struct intel_engine_cs *engine,
+			    struct drm_i915_gem_object *obj,
+			    u64 size, u64 offset,
+			    u32 dword, u32 val)
+{
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	struct i915_address_space *vm = ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
+	unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
+	struct i915_vma *vma;
+	int err;
+
+	vma = i915_vma_instance(obj, vm, NULL);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
+
+	err = i915_vma_unbind(vma);
+	if (err)
+		goto out_vma_close;
+
+	err = i915_vma_pin(vma, size, 0, flags | offset);
+	if (err) {
+		/*
+		 * The ggtt may have some pages reserved so
+		 * refrain from erroring out.
+		 */
+		if (err == -ENOSPC && i915_is_ggtt(vm))
+			err = 0;
+
+		goto out_vma_close;
+	}
+
+	err = igt_check_page_sizes(vma);
+	if (err)
+		goto out_vma_unpin;
+
+	err = gpu_write(vma, ctx, engine, dword, val);
+	if (err) {
+		pr_err("gpu-write failed at offset=%llx\n", offset);
+		goto out_vma_unpin;
+	}
+
+	err = cpu_check(obj, dword, val);
+	if (err) {
+		pr_err("cpu-check failed at offset=%llx\n", offset);
+		goto out_vma_unpin;
+	}
+
+out_vma_unpin:
+	i915_vma_unpin(vma);
+out_vma_close:
+	i915_vma_close(vma);
+
+	return err;
+}
+
 static int igt_write_huge(struct i915_gem_context *ctx,
 			  struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 	struct i915_address_space *vm = ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
+	static struct intel_engine_cs *engines[I915_NUM_ENGINES];
 	struct intel_engine_cs *engine;
-	struct i915_vma *vma;
-	unsigned int flags = PIN_USER | PIN_OFFSET_FIXED;
+	I915_RND_STATE(prng);
+	IGT_TIMEOUT(end_time);
 	unsigned int max_page_size;
 	unsigned int id;
 	u64 max;
 	u64 num;
 	u64 size;
+	int *order;
+	int i, n;
 	int err = 0;
 
 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
@@ -1071,78 +1123,56 @@ static int igt_write_huge(struct i915_gem_context *ctx,
 	max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg);
 	max = div_u64((vm->total - size), max_page_size);
 
-	vma = i915_vma_instance(obj, vm, NULL);
-	if (IS_ERR(vma))
-		return PTR_ERR(vma);
-
+	n = 0;
 	for_each_engine(engine, i915, id) {
-		IGT_TIMEOUT(end_time);
-
 		if (!intel_engine_can_store_dword(engine)) {
-			pr_info("store-dword-imm not supported on engine=%u\n",
-				id);
+			pr_info("store-dword-imm not supported on engine=%u\n", id);
 			continue;
 		}
+		engines[n++] = engine;
+	}
 
-		/*
-		 * Try various offsets until we timeout -- we want to avoid
-		 * issues hidden by effectively always using offset = 0.
-		 */
-		for_each_prime_number_from(num, 0, max) {
-			u64 offset = num * max_page_size;
-			u32 dword;
-
-			err = i915_vma_unbind(vma);
-			if (err)
-				goto out_vma_close;
-
-			err = i915_vma_pin(vma, size, max_page_size, flags | offset);
-			if (err) {
-				/*
-				 * The ggtt may have some pages reserved so
-				 * refrain from erroring out.
-				 */
-				if (err == -ENOSPC && i915_is_ggtt(vm)) {
-					err = 0;
-					continue;
-				}
-
-				goto out_vma_close;
-			}
+	if (!n)
+		return 0;
 
-			err = igt_check_page_sizes(vma);
-			if (err)
-				goto out_vma_unpin;
+	/*
+	 * To keep things interesting when alternating between engines in our
+	 * randomized order, lets also make feeding to the same engine a few
+	 * times in succession a possibility by enlarging the permutation array.
+	 */
+	order = i915_random_order(n * I915_NUM_ENGINES, &prng);
+	if (!order)
+		return -ENOMEM;
 
-			dword = offset_in_page(num) / 4;
+	/*
+	 * Try various offsets in an ascending/descending fashion until we
+	 * timeout -- we want to avoid issues hidden by effectively always using
+	 * offset = 0.
+	 */
+	i = 0;
+	for_each_prime_number_from(num, 0, max) {
+		u64 offset_low = num * max_page_size;
+		u64 offset_high = (max - num) * max_page_size;
+		u32 dword = offset_in_page(num) / 4;
 
-			err = gpu_write(vma, ctx, engine, dword, num + 1);
-			if (err) {
-				pr_err("gpu-write failed at offset=%llx", offset);
-				goto out_vma_unpin;
-			}
+		engine = engines[order[i] % n];
+		i = (i + 1) % (n * I915_NUM_ENGINES);
 
-			err = cpu_check(obj, dword, num + 1);
-			if (err) {
-				pr_err("cpu-check failed at offset=%llx", offset);
-				goto out_vma_unpin;
-			}
+		err = __igt_write_huge(ctx, engine, obj, size, offset_low, dword, num + 1);
+		if (err)
+			break;
 
-			i915_vma_unpin(vma);
+		err = __igt_write_huge(ctx, engine, obj, size, offset_high, dword, num + 1);
+		if (err)
+			break;
 
-			if (num > 0 &&
-			    igt_timeout(end_time,
-					"%s timed out on engine=%u at offset=%llx, max_page_size=%x\n",
-					__func__, id, offset, max_page_size))
-				break;
-		}
+		if (igt_timeout(end_time,
+				"%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n",
+				__func__, engine->id, offset_low, offset_high, max_page_size))
+			break;
 	}
 
-out_vma_unpin:
-	if (i915_vma_is_pinned(vma))
-		i915_vma_unpin(vma);
-out_vma_close:
-	i915_vma_close(vma);
+	kfree(order);
 
 	return err;
 }
@@ -1159,6 +1189,9 @@ static int igt_ppgtt_exhaust_huge(void *arg)
 	int n, i;
 	int err = -ENODEV;
 
+	if (supported == I915_GTT_PAGE_SIZE_4K)
+		return 0;
+
 	/*
 	 * Sanity check creating objects with a varying mix of page sizes --
 	 * ensuring that our writes lands in the right place.
@@ -1604,7 +1637,7 @@ static int igt_shrink_thp(void *arg)
 	 * shmem to truncate our pages.
 	 */
 	i915_gem_shrink_all(i915);
-	if (!IS_ERR_OR_NULL(obj->mm.pages)) {
+	if (i915_gem_object_has_pages(obj)) {
 		pr_err("shrink-all didn't truncate the pages\n");
 		err = -EINVAL;
 		goto out_close;
@@ -1716,6 +1749,7 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
 		return PTR_ERR(file);
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
+	intel_runtime_pm_get(dev_priv);
 
 	ctx = live_context(dev_priv, file);
 	if (IS_ERR(ctx)) {
@@ -1726,6 +1760,7 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
 	err = i915_subtests(tests, ctx);
 
 out_unlock:
+	intel_runtime_pm_put(dev_priv);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	mock_file_free(dev_priv, file);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index 35d778d70626..7a0d1e17c1ad 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -33,7 +33,7 @@ static int cpu_set(struct drm_i915_gem_object *obj,
 {
 	unsigned int needs_clflush;
 	struct page *page;
-	typeof(v) *map;
+	u32 *map;
 	int err;
 
 	err = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
@@ -59,7 +59,7 @@ static int cpu_get(struct drm_i915_gem_object *obj,
 {
 	unsigned int needs_clflush;
 	struct page *page;
-	typeof(v) map;
+	u32 *map;
 	int err;
 
 	err = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
@@ -82,7 +82,7 @@ static int gtt_set(struct drm_i915_gem_object *obj,
 		   u32 v)
 {
 	struct i915_vma *vma;
-	typeof(v) *map;
+	u32 __iomem *map;
 	int err;
 
 	err = i915_gem_object_set_to_gtt_domain(obj, true);
@@ -98,7 +98,7 @@ static int gtt_set(struct drm_i915_gem_object *obj,
 	if (IS_ERR(map))
 		return PTR_ERR(map);
 
-	map[offset / sizeof(*map)] = v;
+	iowrite32(v, &map[offset / sizeof(*map)]);
 	i915_vma_unpin_iomap(vma);
 
 	return 0;
@@ -109,7 +109,7 @@ static int gtt_get(struct drm_i915_gem_object *obj,
 		   u32 *v)
 {
 	struct i915_vma *vma;
-	typeof(v) map;
+	u32 __iomem *map;
 	int err;
 
 	err = i915_gem_object_set_to_gtt_domain(obj, false);
@@ -125,7 +125,7 @@ static int gtt_get(struct drm_i915_gem_object *obj,
 	if (IS_ERR(map))
 		return PTR_ERR(map);
 
-	*v = map[offset / sizeof(*map)];
+	*v = ioread32(&map[offset / sizeof(*map)]);
 	i915_vma_unpin_iomap(vma);
 
 	return 0;
@@ -135,7 +135,7 @@ static int wc_set(struct drm_i915_gem_object *obj,
 		  unsigned long offset,
 		  u32 v)
 {
-	typeof(v) *map;
+	u32 *map;
 	int err;
 
 	err = i915_gem_object_set_to_wc_domain(obj, true);
@@ -156,7 +156,7 @@ static int wc_get(struct drm_i915_gem_object *obj,
 		  unsigned long offset,
 		  u32 *v)
 {
-	typeof(v) map;
+	u32 *map;
 	int err;
 
 	err = i915_gem_object_set_to_wc_domain(obj, false);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index def5052862ae..56a803d11916 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -158,14 +158,6 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
 		goto err_batch;
 	}
 
-	err = engine->emit_flush(rq, EMIT_INVALIDATE);
-	if (err)
-		goto err_request;
-
-	err = i915_switch_context(rq);
-	if (err)
-		goto err_request;
-
 	flags = 0;
 	if (INTEL_GEN(vm->i915) <= 5)
 		flags |= I915_DISPATCH_SECURE;
@@ -272,6 +264,23 @@ static int cpu_check(struct drm_i915_gem_object *obj, unsigned int max)
 	return err;
 }
 
+static int file_add_object(struct drm_file *file,
+			    struct drm_i915_gem_object *obj)
+{
+	int err;
+
+	GEM_BUG_ON(obj->base.handle_count);
+
+	/* tie the object to the drm_file for easy reaping */
+	err = idr_alloc(&file->object_idr, &obj->base, 1, 0, GFP_KERNEL);
+	if (err < 0)
+		return  err;
+
+	i915_gem_object_get(obj);
+	obj->base.handle_count++;
+	return 0;
+}
+
 static struct drm_i915_gem_object *
 create_test_object(struct i915_gem_context *ctx,
 		   struct drm_file *file,
@@ -281,7 +290,6 @@ create_test_object(struct i915_gem_context *ctx,
 	struct i915_address_space *vm =
 		ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base;
 	u64 size;
-	u32 handle;
 	int err;
 
 	size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
@@ -291,8 +299,7 @@ create_test_object(struct i915_gem_context *ctx,
 	if (IS_ERR(obj))
 		return obj;
 
-	/* tie the handle to the drm_file for easy reaping */
-	err = drm_gem_handle_create(file, &obj->base, &handle);
+	err = file_add_object(file, obj);
 	i915_gem_object_put(obj);
 	if (err)
 		return ERR_PTR(err);
@@ -325,7 +332,7 @@ static int igt_ctx_exec(void *arg)
 	LIST_HEAD(objects);
 	unsigned long ncontexts, ndwords, dw;
 	bool first_shared_gtt = true;
-	int err;
+	int err = -ENODEV;
 
 	/* Create a few different contexts (with different mm) and write
 	 * through each ctx/mm using the GPU making sure those writes end
@@ -369,7 +376,9 @@ static int igt_ctx_exec(void *arg)
 				}
 			}
 
+			intel_runtime_pm_get(i915);
 			err = gpu_fill(obj, ctx, engine, dw);
+			intel_runtime_pm_put(i915);
 			if (err) {
 				pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
 				       ndwords, dw, max_dwords(obj),
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index f463105ff48d..e1ddad635d73 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -355,6 +355,7 @@ static int igt_evict_contexts(void *arg)
 		return 0;
 
 	mutex_lock(&i915->drm.struct_mutex);
+	intel_runtime_pm_get(i915);
 
 	/* Reserve a block so that we know we have enough to fit a few rq */
 	memset(&hole, 0, sizeof(hole));
@@ -463,6 +464,7 @@ static int igt_evict_contexts(void *arg)
 	}
 	if (drm_mm_node_allocated(&hole))
 		drm_mm_remove_node(&hole);
+	intel_runtime_pm_put(i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	return err;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index 9da0c9f99916..4a28d713a7d8 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -216,13 +216,21 @@ static int lowlevel_hole(struct drm_i915_private *i915,
 		hole_size = (hole_end - hole_start) >> size;
 		if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 			hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
-		count = hole_size;
+		count = hole_size >> 1;
+		if (!count) {
+			pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
+				 __func__, hole_start, hole_end, size, hole_size);
+			break;
+		}
+
 		do {
-			count >>= 1;
 			order = i915_random_order(count, &prng);
-		} while (!order && count);
-		if (!order)
-			break;
+			if (order)
+				break;
+		} while (count >>= 1);
+		if (!count)
+			return -ENOMEM;
+		GEM_BUG_ON(!order);
 
 		GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
 		GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
@@ -267,7 +275,9 @@ static int lowlevel_hole(struct drm_i915_private *i915,
 			mock_vma.node.size = BIT_ULL(size);
 			mock_vma.node.start = addr;
 
+			intel_runtime_pm_get(i915);
 			vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0);
+			intel_runtime_pm_put(i915);
 		}
 		count = n;
 
@@ -697,18 +707,26 @@ static int drunk_hole(struct drm_i915_private *i915,
 		unsigned int *order, count, n;
 		struct i915_vma *vma;
 		u64 hole_size;
-		int err;
+		int err = -ENODEV;
 
 		hole_size = (hole_end - hole_start) >> size;
 		if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
 			hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
-		count = hole_size;
+		count = hole_size >> 1;
+		if (!count) {
+			pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
+				 __func__, hole_start, hole_end, size, hole_size);
+			break;
+		}
+
 		do {
-			count >>= 1;
 			order = i915_random_order(count, &prng);
-		} while (!order && count);
-		if (!order)
-			break;
+			if (order)
+				break;
+		} while (count >>= 1);
+		if (!count)
+			return -ENOMEM;
+		GEM_BUG_ON(!order);
 
 		/* Ignore allocation failures (i.e. don't report them as
 		 * a test failure) as we are purposefully allocating very
@@ -956,7 +974,7 @@ static int exercise_ggtt(struct drm_i915_private *i915,
 	u64 hole_start, hole_end, last = 0;
 	struct drm_mm_node *node;
 	IGT_TIMEOUT(end_time);
-	int err;
+	int err = 0;
 
 	mutex_lock(&i915->drm.struct_mutex);
 restart:
@@ -1047,6 +1065,7 @@ static int igt_ggtt_page(void *arg)
 		goto out_remove;
 	}
 
+	intel_runtime_pm_get(i915);
 	for (n = 0; n < count; n++) {
 		u64 offset = tmp.start + order[n] * PAGE_SIZE;
 		u32 __iomem *vaddr;
@@ -1055,7 +1074,7 @@ static int igt_ggtt_page(void *arg)
 				       i915_gem_object_get_dma_address(obj, 0),
 				       offset, I915_CACHE_NONE, 0);
 
-		vaddr = io_mapping_map_atomic_wc(&ggtt->mappable, offset);
+		vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
 		iowrite32(n, vaddr + n);
 		io_mapping_unmap_atomic(vaddr);
 
@@ -1073,7 +1092,7 @@ static int igt_ggtt_page(void *arg)
 				       i915_gem_object_get_dma_address(obj, 0),
 				       offset, I915_CACHE_NONE, 0);
 
-		vaddr = io_mapping_map_atomic_wc(&ggtt->mappable, offset);
+		vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset);
 		val = ioread32(vaddr + n);
 		io_mapping_unmap_atomic(vaddr);
 
@@ -1086,6 +1105,7 @@ static int igt_ggtt_page(void *arg)
 			break;
 		}
 	}
+	intel_runtime_pm_put(i915);
 
 	kfree(order);
 out_remove:
@@ -1160,7 +1180,7 @@ static int igt_gtt_reserve(void *arg)
 	struct drm_i915_gem_object *obj, *on;
 	LIST_HEAD(objects);
 	u64 total;
-	int err;
+	int err = -ENODEV;
 
 	/* i915_gem_gtt_reserve() tries to reserve the precise range
 	 * for the node, and evicts if it has to. So our test checks that
@@ -1351,7 +1371,7 @@ static int igt_gtt_insert(void *arg)
 	}, *ii;
 	LIST_HEAD(objects);
 	u64 total;
-	int err;
+	int err = -ENODEV;
 
 	/* i915_gem_gtt_insert() tries to allocate some free space in the GTT
 	 * to the node, evicting if required.
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 1b8774a42e48..f32aa6bb79e2 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -317,6 +317,7 @@ static int igt_partial_tiling(void *arg)
 	}
 
 	mutex_lock(&i915->drm.struct_mutex);
+	intel_runtime_pm_get(i915);
 
 	if (1) {
 		IGT_TIMEOUT(end);
@@ -418,6 +419,7 @@ next_tiling: ;
 	}
 
 out_unlock:
+	intel_runtime_pm_put(i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 	i915_gem_object_unpin_pages(obj);
 out:
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_request.c b/drivers/gpu/drm/i915/selftests/i915_gem_request.c
index a999161e8db1..647bf2bbd799 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_request.c
@@ -332,7 +332,7 @@ static int live_nop_request(void *arg)
 	struct intel_engine_cs *engine;
 	struct live_test t;
 	unsigned int id;
-	int err;
+	int err = -ENODEV;
 
 	/* Submit various sized batches of empty requests, to each engine
 	 * (individually), and wait for the batch to complete. We can check
@@ -459,14 +459,6 @@ empty_request(struct intel_engine_cs *engine,
 	if (IS_ERR(request))
 		return request;
 
-	err = engine->emit_flush(request, EMIT_INVALIDATE);
-	if (err)
-		goto out_request;
-
-	err = i915_switch_context(request);
-	if (err)
-		goto out_request;
-
 	err = engine->emit_bb_start(request,
 				    batch->node.start,
 				    batch->node.size,
@@ -675,12 +667,6 @@ static int live_all_engines(void *arg)
 			goto out_request;
 		}
 
-		err = engine->emit_flush(request[id], EMIT_INVALIDATE);
-		GEM_BUG_ON(err);
-
-		err = i915_switch_context(request[id]);
-		GEM_BUG_ON(err);
-
 		err = engine->emit_bb_start(request[id],
 					    batch->node.start,
 					    batch->node.size,
@@ -797,12 +783,6 @@ static int live_sequential_engines(void *arg)
 			}
 		}
 
-		err = engine->emit_flush(request[id], EMIT_INVALIDATE);
-		GEM_BUG_ON(err);
-
-		err = i915_switch_context(request[id]);
-		GEM_BUG_ON(err);
-
 		err = engine->emit_bb_start(request[id],
 					    batch->node.start,
 					    batch->node.size,
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c b/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
index 4795877abe56..3000e6a7d82d 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
@@ -79,7 +79,7 @@ static int igt_sync(void *arg)
 	}, *p;
 	struct intel_timeline *tl;
 	int order, offset;
-	int ret;
+	int ret = -ENODEV;
 
 	tl = mock_timeline(0);
 	if (!tl)
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index d7dd98a6acad..088f45bc6199 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -20,3 +20,4 @@ selftest(evict, i915_gem_evict_live_selftests)
 selftest(hugepages, i915_gem_huge_page_live_selftests)
 selftest(contexts, i915_gem_context_live_selftests)
 selftest(hangcheck, intel_hangcheck_live_selftests)
+selftest(guc, intel_guc_live_selftest)
diff --git a/drivers/gpu/drm/i915/selftests/i915_random.c b/drivers/gpu/drm/i915/selftests/i915_random.c
index b85872cc7fbe..2088ae57aa89 100644
--- a/drivers/gpu/drm/i915/selftests/i915_random.c
+++ b/drivers/gpu/drm/i915/selftests/i915_random.c
@@ -57,7 +57,7 @@ unsigned int *i915_random_order(unsigned int count, struct rnd_state *state)
 {
 	unsigned int *order, i;
 
-	order = kmalloc_array(count, sizeof(*order), GFP_KERNEL);
+	order = kmalloc_array(count, sizeof(*order), GFP_KERNEL | __GFP_NOWARN);
 	if (!order)
 		return order;
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_syncmap.c b/drivers/gpu/drm/i915/selftests/i915_syncmap.c
index bcab3d00a785..47f4ae18a1ef 100644
--- a/drivers/gpu/drm/i915/selftests/i915_syncmap.c
+++ b/drivers/gpu/drm/i915/selftests/i915_syncmap.c
@@ -333,7 +333,7 @@ static int igt_syncmap_join_below(void *arg)
 {
 	struct i915_syncmap *sync;
 	unsigned int step, order, idx;
-	int err;
+	int err = -ENODEV;
 
 	i915_syncmap_init(&sync);
 
@@ -402,7 +402,7 @@ static int igt_syncmap_neighbours(void *arg)
 	I915_RND_STATE(prng);
 	IGT_TIMEOUT(end_time);
 	struct i915_syncmap *sync;
-	int err;
+	int err = -ENODEV;
 
 	/*
 	 * Each leaf holds KSYNCMAP seqno. Check that when we create KSYNCMAP
@@ -447,7 +447,7 @@ static int igt_syncmap_compact(void *arg)
 {
 	struct i915_syncmap *sync;
 	unsigned int idx, order;
-	int err;
+	int err = -ENODEV;
 
 	i915_syncmap_init(&sync);
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index 2e86ec136b35..eb89e301b602 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -150,7 +150,7 @@ static int igt_vma_create(void *arg)
 	IGT_TIMEOUT(end_time);
 	LIST_HEAD(contexts);
 	LIST_HEAD(objects);
-	int err;
+	int err = -ENOMEM;
 
 	/* Exercise creating many vma amonst many objections, checking the
 	 * vma creation and lookup routines.
diff --git a/drivers/gpu/drm/i915/selftests/intel_guc.c b/drivers/gpu/drm/i915/selftests/intel_guc.c
new file mode 100644
index 000000000000..3f9016466dea
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/intel_guc.c
@@ -0,0 +1,353 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "../i915_selftest.h"
+
+/* max doorbell number + negative test for each client type */
+#define ATTEMPTS (GUC_NUM_DOORBELLS + GUC_CLIENT_PRIORITY_NUM)
+
+static struct intel_guc_client *clients[ATTEMPTS];
+
+static bool available_dbs(struct intel_guc *guc, u32 priority)
+{
+	unsigned long offset;
+	unsigned long end;
+	u16 id;
+
+	/* first half is used for normal priority, second half for high */
+	offset = 0;
+	end = GUC_NUM_DOORBELLS / 2;
+	if (priority <= GUC_CLIENT_PRIORITY_HIGH) {
+		offset = end;
+		end += offset;
+	}
+
+	id = find_next_zero_bit(guc->doorbell_bitmap, end, offset);
+	if (id < end)
+		return true;
+
+	return false;
+}
+
+static int check_all_doorbells(struct intel_guc *guc)
+{
+	u16 db_id;
+
+	pr_info_once("Max number of doorbells: %d", GUC_NUM_DOORBELLS);
+	for (db_id = 0; db_id < GUC_NUM_DOORBELLS; ++db_id) {
+		if (!doorbell_ok(guc, db_id)) {
+			pr_err("doorbell %d, not ok\n", db_id);
+			return -EIO;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Basic client sanity check, handy to validate create_clients.
+ */
+static int validate_client(struct intel_guc_client *client,
+			   int client_priority,
+			   bool is_preempt_client)
+{
+	struct drm_i915_private *dev_priv = guc_to_i915(client->guc);
+	struct i915_gem_context *ctx_owner = is_preempt_client ?
+			dev_priv->preempt_context : dev_priv->kernel_context;
+
+	if (client->owner != ctx_owner ||
+	    client->engines != INTEL_INFO(dev_priv)->ring_mask ||
+	    client->priority != client_priority ||
+	    client->doorbell_id == GUC_DOORBELL_INVALID)
+		return -EINVAL;
+	else
+		return 0;
+}
+
+static bool client_doorbell_in_sync(struct intel_guc_client *client)
+{
+	return doorbell_ok(client->guc, client->doorbell_id);
+}
+
+/*
+ * Check that we're able to synchronize guc_clients with their doorbells
+ *
+ * We're creating clients and reserving doorbells once, at module load. During
+ * module lifetime, GuC, doorbell HW, and i915 state may go out of sync due to
+ * GuC being reset. In other words - GuC clients are still around, but the
+ * status of their doorbells may be incorrect. This is the reason behind
+ * validating that the doorbells status expected by the driver matches what the
+ * GuC/HW have.
+ */
+static int igt_guc_clients(void *args)
+{
+	struct drm_i915_private *dev_priv = args;
+	struct intel_guc *guc;
+	int err = 0;
+
+	GEM_BUG_ON(!HAS_GUC(dev_priv));
+	mutex_lock(&dev_priv->drm.struct_mutex);
+
+	guc = &dev_priv->guc;
+	if (!guc) {
+		pr_err("No guc object!\n");
+		err = -EINVAL;
+		goto unlock;
+	}
+
+	err = check_all_doorbells(guc);
+	if (err)
+		goto unlock;
+
+	/*
+	 * Get rid of clients created during driver load because the test will
+	 * recreate them.
+	 */
+	guc_clients_destroy(guc);
+	if (guc->execbuf_client || guc->preempt_client) {
+		pr_err("guc_clients_destroy lied!\n");
+		err = -EINVAL;
+		goto unlock;
+	}
+
+	err = guc_clients_create(guc);
+	if (err) {
+		pr_err("Failed to create clients\n");
+		goto unlock;
+	}
+	GEM_BUG_ON(!guc->execbuf_client);
+	GEM_BUG_ON(!guc->preempt_client);
+
+	err = validate_client(guc->execbuf_client,
+			      GUC_CLIENT_PRIORITY_KMD_NORMAL, false);
+	if (err) {
+		pr_err("execbug client validation failed\n");
+		goto out;
+	}
+
+	err = validate_client(guc->preempt_client,
+			      GUC_CLIENT_PRIORITY_KMD_HIGH, true);
+	if (err) {
+		pr_err("preempt client validation failed\n");
+		goto out;
+	}
+
+	/* each client should now have reserved a doorbell */
+	if (!has_doorbell(guc->execbuf_client) ||
+	    !has_doorbell(guc->preempt_client)) {
+		pr_err("guc_clients_create didn't reserve doorbells\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	/* Now create the doorbells */
+	guc_clients_doorbell_init(guc);
+
+	/* each client should now have received a doorbell */
+	if (!client_doorbell_in_sync(guc->execbuf_client) ||
+	    !client_doorbell_in_sync(guc->preempt_client)) {
+		pr_err("failed to initialize the doorbells\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	/*
+	 * Basic test - an attempt to reallocate a valid doorbell to the
+	 * client it is currently assigned should not cause a failure.
+	 */
+	err = guc_clients_doorbell_init(guc);
+	if (err)
+		goto out;
+
+	/*
+	 * Negative test - a client with no doorbell (invalid db id).
+	 * After destroying the doorbell, the db id is changed to
+	 * GUC_DOORBELL_INVALID and the firmware will reject any attempt to
+	 * allocate a doorbell with an invalid id (db has to be reserved before
+	 * allocation).
+	 */
+	destroy_doorbell(guc->execbuf_client);
+	if (client_doorbell_in_sync(guc->execbuf_client)) {
+		pr_err("destroy db did not work\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	unreserve_doorbell(guc->execbuf_client);
+	err = guc_clients_doorbell_init(guc);
+	if (err != -EIO) {
+		pr_err("unexpected (err = %d)", err);
+		goto out;
+	}
+
+	if (!available_dbs(guc, guc->execbuf_client->priority)) {
+		pr_err("doorbell not available when it should\n");
+		err = -EIO;
+		goto out;
+	}
+
+	/* clean after test */
+	err = reserve_doorbell(guc->execbuf_client);
+	if (err) {
+		pr_err("failed to reserve back the doorbell back\n");
+	}
+	err = create_doorbell(guc->execbuf_client);
+	if (err) {
+		pr_err("recreate doorbell failed\n");
+		goto out;
+	}
+
+out:
+	/*
+	 * Leave clean state for other test, plus the driver always destroy the
+	 * clients during unload.
+	 */
+	destroy_doorbell(guc->execbuf_client);
+	destroy_doorbell(guc->preempt_client);
+	guc_clients_destroy(guc);
+	guc_clients_create(guc);
+	guc_clients_doorbell_init(guc);
+unlock:
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	return err;
+}
+
+/*
+ * Create as many clients as number of doorbells. Note that there's already
+ * client(s)/doorbell(s) created during driver load, but this test creates
+ * its own and do not interact with the existing ones.
+ */
+static int igt_guc_doorbells(void *arg)
+{
+	struct drm_i915_private *dev_priv = arg;
+	struct intel_guc *guc;
+	int i, err = 0;
+	u16 db_id;
+
+	GEM_BUG_ON(!HAS_GUC(dev_priv));
+	mutex_lock(&dev_priv->drm.struct_mutex);
+
+	guc = &dev_priv->guc;
+	if (!guc) {
+		pr_err("No guc object!\n");
+		err = -EINVAL;
+		goto unlock;
+	}
+
+	err = check_all_doorbells(guc);
+	if (err)
+		goto unlock;
+
+	for (i = 0; i < ATTEMPTS; i++) {
+		clients[i] = guc_client_alloc(dev_priv,
+					      INTEL_INFO(dev_priv)->ring_mask,
+					      i % GUC_CLIENT_PRIORITY_NUM,
+					      dev_priv->kernel_context);
+
+		if (!clients[i]) {
+			pr_err("[%d] No guc client\n", i);
+			err = -EINVAL;
+			goto out;
+		}
+
+		if (IS_ERR(clients[i])) {
+			if (PTR_ERR(clients[i]) != -ENOSPC) {
+				pr_err("[%d] unexpected error\n", i);
+				err = PTR_ERR(clients[i]);
+				goto out;
+			}
+
+			if (available_dbs(guc, i % GUC_CLIENT_PRIORITY_NUM)) {
+				pr_err("[%d] non-db related alloc fail\n", i);
+				err = -EINVAL;
+				goto out;
+			}
+
+			/* expected, ran out of dbs for this client type */
+			continue;
+		}
+
+		/*
+		 * The check below is only valid because we keep a doorbell
+		 * assigned during the whole life of the client.
+		 */
+		if (clients[i]->stage_id >= GUC_NUM_DOORBELLS) {
+			pr_err("[%d] more clients than doorbells (%d >= %d)\n",
+			       i, clients[i]->stage_id, GUC_NUM_DOORBELLS);
+			err = -EINVAL;
+			goto out;
+		}
+
+		err = validate_client(clients[i],
+				      i % GUC_CLIENT_PRIORITY_NUM, false);
+		if (err) {
+			pr_err("[%d] client_alloc sanity check failed!\n", i);
+			err = -EINVAL;
+			goto out;
+		}
+
+		db_id = clients[i]->doorbell_id;
+
+		err = create_doorbell(clients[i]);
+		if (err) {
+			pr_err("[%d] Failed to create a doorbell\n", i);
+			goto out;
+		}
+
+		/* doorbell id shouldn't change, we are holding the mutex */
+		if (db_id != clients[i]->doorbell_id) {
+			pr_err("[%d] doorbell id changed (%d != %d)\n",
+			       i, db_id, clients[i]->doorbell_id);
+			err = -EINVAL;
+			goto out;
+		}
+
+		err = check_all_doorbells(guc);
+		if (err)
+			goto out;
+	}
+
+out:
+	for (i = 0; i < ATTEMPTS; i++)
+		if (!IS_ERR_OR_NULL(clients[i])) {
+			destroy_doorbell(clients[i]);
+			guc_client_free(clients[i]);
+		}
+unlock:
+	mutex_unlock(&dev_priv->drm.struct_mutex);
+	return err;
+}
+
+int intel_guc_live_selftest(struct drm_i915_private *dev_priv)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(igt_guc_clients),
+		SUBTEST(igt_guc_doorbells),
+	};
+
+	if (!USES_GUC_SUBMISSION(dev_priv))
+		return 0;
+
+	return i915_subtests(tests, dev_priv);
+}
diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
index 71ce06680d66..d1f91a533afa 100644
--- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
@@ -114,14 +114,6 @@ static int emit_recurse_batch(struct hang *h,
 	if (err)
 		goto unpin_vma;
 
-	err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
-	if (err)
-		goto unpin_hws;
-
-	err = i915_switch_context(rq);
-	if (err)
-		goto unpin_hws;
-
 	i915_vma_move_to_active(vma, rq, 0);
 	if (!i915_gem_object_has_active_reference(vma->obj)) {
 		i915_gem_object_get(vma->obj);
@@ -140,6 +132,12 @@ static int emit_recurse_batch(struct hang *h,
 		*batch++ = lower_32_bits(hws_address(hws, rq));
 		*batch++ = upper_32_bits(hws_address(hws, rq));
 		*batch++ = rq->fence.seqno;
+		*batch++ = MI_ARB_CHECK;
+
+		memset(batch, 0, 1024);
+		batch += 1024 / sizeof(*batch);
+
+		*batch++ = MI_ARB_CHECK;
 		*batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
 		*batch++ = lower_32_bits(vma->node.start);
 		*batch++ = upper_32_bits(vma->node.start);
@@ -148,6 +146,12 @@ static int emit_recurse_batch(struct hang *h,
 		*batch++ = 0;
 		*batch++ = lower_32_bits(hws_address(hws, rq));
 		*batch++ = rq->fence.seqno;
+		*batch++ = MI_ARB_CHECK;
+
+		memset(batch, 0, 1024);
+		batch += 1024 / sizeof(*batch);
+
+		*batch++ = MI_ARB_CHECK;
 		*batch++ = MI_BATCH_BUFFER_START | 1 << 8;
 		*batch++ = lower_32_bits(vma->node.start);
 	} else if (INTEL_GEN(i915) >= 4) {
@@ -155,12 +159,24 @@ static int emit_recurse_batch(struct hang *h,
 		*batch++ = 0;
 		*batch++ = lower_32_bits(hws_address(hws, rq));
 		*batch++ = rq->fence.seqno;
+		*batch++ = MI_ARB_CHECK;
+
+		memset(batch, 0, 1024);
+		batch += 1024 / sizeof(*batch);
+
+		*batch++ = MI_ARB_CHECK;
 		*batch++ = MI_BATCH_BUFFER_START | 2 << 6;
 		*batch++ = lower_32_bits(vma->node.start);
 	} else {
 		*batch++ = MI_STORE_DWORD_IMM;
 		*batch++ = lower_32_bits(hws_address(hws, rq));
 		*batch++ = rq->fence.seqno;
+		*batch++ = MI_ARB_CHECK;
+
+		memset(batch, 0, 1024);
+		batch += 1024 / sizeof(*batch);
+
+		*batch++ = MI_ARB_CHECK;
 		*batch++ = MI_BATCH_BUFFER_START | 2 << 6 | 1;
 		*batch++ = lower_32_bits(vma->node.start);
 	}
@@ -173,7 +189,6 @@ static int emit_recurse_batch(struct hang *h,
 
 	err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
 
-unpin_hws:
 	i915_vma_unpin(hws);
 unpin_vma:
 	i915_vma_unpin(vma);
@@ -243,6 +258,16 @@ static void hang_fini(struct hang *h)
 	i915_gem_wait_for_idle(h->i915, I915_WAIT_LOCKED);
 }
 
+static bool wait_for_hang(struct hang *h, struct drm_i915_gem_request *rq)
+{
+	return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
+					       rq->fence.seqno),
+			     10) &&
+		 wait_for(i915_seqno_passed(hws_seqno(h, rq),
+					    rq->fence.seqno),
+			  1000));
+}
+
 static int igt_hang_sanitycheck(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -305,6 +330,9 @@ static void global_reset_lock(struct drm_i915_private *i915)
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
+	pr_debug("%s: current gpu_error=%08lx\n",
+		 __func__, i915->gpu_error.flags);
+
 	while (test_and_set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags))
 		wait_event(i915->gpu_error.reset_queue,
 			   !test_bit(I915_RESET_BACKOFF,
@@ -362,54 +390,128 @@ static int igt_global_reset(void *arg)
 	return err;
 }
 
-static int igt_reset_engine(void *arg)
+static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 {
-	struct drm_i915_private *i915 = arg;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-	unsigned int reset_count, reset_engine_count;
+	struct hang h;
 	int err = 0;
 
-	/* Check that we can issue a global GPU and engine reset */
+	/* Check that we can issue an engine reset on an idle engine (no-op) */
 
 	if (!intel_has_reset_engine(i915))
 		return 0;
 
+	if (active) {
+		mutex_lock(&i915->drm.struct_mutex);
+		err = hang_init(&h, i915);
+		mutex_unlock(&i915->drm.struct_mutex);
+		if (err)
+			return err;
+	}
+
 	for_each_engine(engine, i915, id) {
-		set_bit(I915_RESET_ENGINE + engine->id, &i915->gpu_error.flags);
+		unsigned int reset_count, reset_engine_count;
+		IGT_TIMEOUT(end_time);
+
+		if (active && !intel_engine_can_store_dword(engine))
+			continue;
+
 		reset_count = i915_reset_count(&i915->gpu_error);
 		reset_engine_count = i915_reset_engine_count(&i915->gpu_error,
 							     engine);
 
-		err = i915_reset_engine(engine, I915_RESET_QUIET);
-		if (err) {
-			pr_err("i915_reset_engine failed\n");
-			break;
-		}
+		set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
+		do {
+			if (active) {
+				struct drm_i915_gem_request *rq;
+
+				mutex_lock(&i915->drm.struct_mutex);
+				rq = hang_create_request(&h, engine,
+							 i915->kernel_context);
+				if (IS_ERR(rq)) {
+					err = PTR_ERR(rq);
+					mutex_unlock(&i915->drm.struct_mutex);
+					break;
+				}
+
+				i915_gem_request_get(rq);
+				__i915_add_request(rq, true);
+				mutex_unlock(&i915->drm.struct_mutex);
+
+				if (!wait_for_hang(&h, rq)) {
+					struct drm_printer p = drm_info_printer(i915->drm.dev);
+
+					pr_err("%s: Failed to start request %x, at %x\n",
+					       __func__, rq->fence.seqno, hws_seqno(&h, rq));
+					intel_engine_dump(engine, &p,
+							  "%s\n", engine->name);
+
+					i915_gem_request_put(rq);
+					err = -EIO;
+					break;
+				}
 
-		if (i915_reset_count(&i915->gpu_error) != reset_count) {
-			pr_err("Full GPU reset recorded! (engine reset expected)\n");
-			err = -EINVAL;
-			break;
-		}
+				i915_gem_request_put(rq);
+			}
+
+			engine->hangcheck.stalled = true;
+			engine->hangcheck.seqno =
+				intel_engine_get_seqno(engine);
+
+			err = i915_reset_engine(engine, I915_RESET_QUIET);
+			if (err) {
+				pr_err("i915_reset_engine failed\n");
+				break;
+			}
+
+			if (i915_reset_count(&i915->gpu_error) != reset_count) {
+				pr_err("Full GPU reset recorded! (engine reset expected)\n");
+				err = -EINVAL;
+				break;
+			}
+
+			reset_engine_count += active;
+			if (i915_reset_engine_count(&i915->gpu_error, engine) !=
+			    reset_engine_count) {
+				pr_err("%s engine reset %srecorded!\n",
+				       engine->name, active ? "not " : "");
+				err = -EINVAL;
+				break;
+			}
+
+			engine->hangcheck.stalled = false;
+		} while (time_before(jiffies, end_time));
+		clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 
-		if (i915_reset_engine_count(&i915->gpu_error, engine) ==
-		    reset_engine_count) {
-			pr_err("No %s engine reset recorded!\n", engine->name);
-			err = -EINVAL;
+		if (err)
 			break;
-		}
 
-		clear_bit(I915_RESET_ENGINE + engine->id,
-			  &i915->gpu_error.flags);
+		cond_resched();
 	}
 
 	if (i915_terminally_wedged(&i915->gpu_error))
 		err = -EIO;
 
+	if (active) {
+		mutex_lock(&i915->drm.struct_mutex);
+		hang_fini(&h);
+		mutex_unlock(&i915->drm.struct_mutex);
+	}
+
 	return err;
 }
 
+static int igt_reset_idle_engine(void *arg)
+{
+	return __igt_reset_engine(arg, false);
+}
+
+static int igt_reset_active_engine(void *arg)
+{
+	return __igt_reset_engine(arg, true);
+}
+
 static int active_engine(void *data)
 {
 	struct intel_engine_cs *engine = data;
@@ -471,11 +573,12 @@ static int active_engine(void *data)
 	return err;
 }
 
-static int igt_reset_active_engines(void *arg)
+static int __igt_reset_engine_others(struct drm_i915_private *i915,
+				     bool active)
 {
-	struct drm_i915_private *i915 = arg;
-	struct intel_engine_cs *engine, *active;
+	struct intel_engine_cs *engine, *other;
 	enum intel_engine_id id, tmp;
+	struct hang h;
 	int err = 0;
 
 	/* Check that issuing a reset on one engine does not interfere
@@ -485,24 +588,36 @@ static int igt_reset_active_engines(void *arg)
 	if (!intel_has_reset_engine(i915))
 		return 0;
 
+	if (active) {
+		mutex_lock(&i915->drm.struct_mutex);
+		err = hang_init(&h, i915);
+		mutex_unlock(&i915->drm.struct_mutex);
+		if (err)
+			return err;
+	}
+
 	for_each_engine(engine, i915, id) {
-		struct task_struct *threads[I915_NUM_ENGINES];
+		struct task_struct *threads[I915_NUM_ENGINES] = {};
 		unsigned long resets[I915_NUM_ENGINES];
 		unsigned long global = i915_reset_count(&i915->gpu_error);
+		unsigned long count = 0;
 		IGT_TIMEOUT(end_time);
 
+		if (active && !intel_engine_can_store_dword(engine))
+			continue;
+
 		memset(threads, 0, sizeof(threads));
-		for_each_engine(active, i915, tmp) {
+		for_each_engine(other, i915, tmp) {
 			struct task_struct *tsk;
 
-			if (active == engine)
-				continue;
-
 			resets[tmp] = i915_reset_engine_count(&i915->gpu_error,
-							      active);
+							      other);
+
+			if (other == engine)
+				continue;
 
-			tsk = kthread_run(active_engine, active,
-					  "igt/%s", active->name);
+			tsk = kthread_run(active_engine, other,
+					  "igt/%s", other->name);
 			if (IS_ERR(tsk)) {
 				err = PTR_ERR(tsk);
 				goto unwind;
@@ -512,20 +627,70 @@ static int igt_reset_active_engines(void *arg)
 			get_task_struct(tsk);
 		}
 
-		set_bit(I915_RESET_ENGINE + engine->id, &i915->gpu_error.flags);
+		set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
 		do {
+			if (active) {
+				struct drm_i915_gem_request *rq;
+
+				mutex_lock(&i915->drm.struct_mutex);
+				rq = hang_create_request(&h, engine,
+							 i915->kernel_context);
+				if (IS_ERR(rq)) {
+					err = PTR_ERR(rq);
+					mutex_unlock(&i915->drm.struct_mutex);
+					break;
+				}
+
+				i915_gem_request_get(rq);
+				__i915_add_request(rq, true);
+				mutex_unlock(&i915->drm.struct_mutex);
+
+				if (!wait_for_hang(&h, rq)) {
+					struct drm_printer p = drm_info_printer(i915->drm.dev);
+
+					pr_err("%s: Failed to start request %x, at %x\n",
+					       __func__, rq->fence.seqno, hws_seqno(&h, rq));
+					intel_engine_dump(engine, &p,
+							  "%s\n", engine->name);
+
+					i915_gem_request_put(rq);
+					err = -EIO;
+					break;
+				}
+
+				i915_gem_request_put(rq);
+			}
+
+			engine->hangcheck.stalled = true;
+			engine->hangcheck.seqno =
+				intel_engine_get_seqno(engine);
+
 			err = i915_reset_engine(engine, I915_RESET_QUIET);
 			if (err) {
-				pr_err("i915_reset_engine(%s) failed, err=%d\n",
-				       engine->name, err);
+				pr_err("i915_reset_engine(%s:%s) failed, err=%d\n",
+				       engine->name, active ? "active" : "idle", err);
 				break;
 			}
+
+			engine->hangcheck.stalled = false;
+			count++;
 		} while (time_before(jiffies, end_time));
-		clear_bit(I915_RESET_ENGINE + engine->id,
-			  &i915->gpu_error.flags);
+		clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
+		pr_info("i915_reset_engine(%s:%s): %lu resets\n",
+			engine->name, active ? "active" : "idle", count);
+
+		if (i915_reset_engine_count(&i915->gpu_error, engine) -
+		    resets[engine->id] != (active ? count : 0)) {
+			pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n",
+			       engine->name, active ? "active" : "idle", count,
+			       i915_reset_engine_count(&i915->gpu_error,
+						       engine) - resets[engine->id]);
+			if (!err)
+				err = -EINVAL;
+		}
 
 unwind:
-		for_each_engine(active, i915, tmp) {
+		for_each_engine(other, i915, tmp) {
 			int ret;
 
 			if (!threads[tmp])
@@ -533,27 +698,29 @@ static int igt_reset_active_engines(void *arg)
 
 			ret = kthread_stop(threads[tmp]);
 			if (ret) {
-				pr_err("kthread for active engine %s failed, err=%d\n",
-				       active->name, ret);
+				pr_err("kthread for other engine %s failed, err=%d\n",
+				       other->name, ret);
 				if (!err)
 					err = ret;
 			}
 			put_task_struct(threads[tmp]);
 
 			if (resets[tmp] != i915_reset_engine_count(&i915->gpu_error,
-								   active)) {
+								   other)) {
 				pr_err("Innocent engine %s was reset (count=%ld)\n",
-				       active->name,
+				       other->name,
 				       i915_reset_engine_count(&i915->gpu_error,
-							       active) - resets[tmp]);
-				err = -EIO;
+							       other) - resets[tmp]);
+				if (!err)
+					err = -EINVAL;
 			}
 		}
 
 		if (global != i915_reset_count(&i915->gpu_error)) {
 			pr_err("Global reset (count=%ld)!\n",
 			       i915_reset_count(&i915->gpu_error) - global);
-			err = -EIO;
+			if (!err)
+				err = -EINVAL;
 		}
 
 		if (err)
@@ -565,9 +732,25 @@ static int igt_reset_active_engines(void *arg)
 	if (i915_terminally_wedged(&i915->gpu_error))
 		err = -EIO;
 
+	if (active) {
+		mutex_lock(&i915->drm.struct_mutex);
+		hang_fini(&h);
+		mutex_unlock(&i915->drm.struct_mutex);
+	}
+
 	return err;
 }
 
+static int igt_reset_idle_engine_others(void *arg)
+{
+	return __igt_reset_engine_others(arg, false);
+}
+
+static int igt_reset_active_engine_others(void *arg)
+{
+	return __igt_reset_engine_others(arg, true);
+}
+
 static u32 fake_hangcheck(struct drm_i915_gem_request *rq)
 {
 	u32 reset_count;
@@ -583,16 +766,6 @@ static u32 fake_hangcheck(struct drm_i915_gem_request *rq)
 	return reset_count;
 }
 
-static bool wait_for_hang(struct hang *h, struct drm_i915_gem_request *rq)
-{
-	return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq),
-					       rq->fence.seqno),
-			     10) &&
-		 wait_for(i915_seqno_passed(hws_seqno(h, rq),
-					    rq->fence.seqno),
-			  1000));
-}
-
 static int igt_wait_reset(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -626,9 +799,9 @@ static int igt_wait_reset(void *arg)
 	if (!wait_for_hang(&h, rq)) {
 		struct drm_printer p = drm_info_printer(i915->drm.dev);
 
-		pr_err("Failed to start request %x, at %x\n",
-		       rq->fence.seqno, hws_seqno(&h, rq));
-		intel_engine_dump(rq->engine, &p);
+		pr_err("%s: Failed to start request %x, at %x\n",
+		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
+		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
 
 		i915_reset(i915, 0);
 		i915_gem_set_wedged(i915);
@@ -721,9 +894,10 @@ static int igt_reset_queue(void *arg)
 			if (!wait_for_hang(&h, prev)) {
 				struct drm_printer p = drm_info_printer(i915->drm.dev);
 
-				pr_err("Failed to start request %x, at %x\n",
-				       prev->fence.seqno, hws_seqno(&h, prev));
-				intel_engine_dump(rq->engine, &p);
+				pr_err("%s: Failed to start request %x, at %x\n",
+				       __func__, prev->fence.seqno, hws_seqno(&h, prev));
+				intel_engine_dump(prev->engine, &p,
+						  "%s\n", prev->engine->name);
 
 				i915_gem_request_put(rq);
 				i915_gem_request_put(prev);
@@ -827,9 +1001,9 @@ static int igt_handle_error(void *arg)
 	if (!wait_for_hang(&h, rq)) {
 		struct drm_printer p = drm_info_printer(i915->drm.dev);
 
-		pr_err("Failed to start request %x, at %x\n",
-		       rq->fence.seqno, hws_seqno(&h, rq));
-		intel_engine_dump(rq->engine, &p);
+		pr_err("%s: Failed to start request %x, at %x\n",
+		       __func__, rq->fence.seqno, hws_seqno(&h, rq));
+		intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name);
 
 		i915_reset(i915, 0);
 		i915_gem_set_wedged(i915);
@@ -872,21 +1046,26 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 	static const struct i915_subtest tests[] = {
 		SUBTEST(igt_global_reset), /* attempt to recover GPU first */
 		SUBTEST(igt_hang_sanitycheck),
-		SUBTEST(igt_reset_engine),
-		SUBTEST(igt_reset_active_engines),
+		SUBTEST(igt_reset_idle_engine),
+		SUBTEST(igt_reset_active_engine),
+		SUBTEST(igt_reset_idle_engine_others),
+		SUBTEST(igt_reset_active_engine_others),
 		SUBTEST(igt_wait_reset),
 		SUBTEST(igt_reset_queue),
 		SUBTEST(igt_handle_error),
 	};
+	bool saved_hangcheck;
 	int err;
 
 	if (!intel_has_gpu_reset(i915))
 		return 0;
 
 	intel_runtime_pm_get(i915);
+	saved_hangcheck = fetch_and_zero(&i915_modparams.enable_hangcheck);
 
 	err = i915_subtests(tests, i915);
 
+	i915_modparams.enable_hangcheck = saved_hangcheck;
 	intel_runtime_pm_put(i915);
 
 	return err;
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 3cac22eb47ce..2f6367643171 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -120,10 +120,10 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri
 	    !IS_CHERRYVIEW(dev_priv))
 		return 0;
 
-	if (IS_VALLEYVIEW(dev_priv)) /* XXX system lockup! */
-		return 0;
-
-	if (IS_BROADWELL(dev_priv)) /* XXX random GPU hang afterwards! */
+	/*
+	 * This test may lockup the machine or cause GPU hangs afterwards.
+	 */
+	if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
 		return 0;
 
 	valid = kzalloc(BITS_TO_LONGS(FW_RANGE) * sizeof(*valid),
@@ -148,7 +148,10 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri
 	for_each_set_bit(offset, valid, FW_RANGE) {
 		i915_reg_t reg = { offset };
 
+		iosf_mbi_punit_acquire();
 		intel_uncore_forcewake_reset(dev_priv, false);
+		iosf_mbi_punit_release();
+
 		check_for_unclaimed_mmio(dev_priv);
 
 		(void)I915_READ(reg);
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 331c2b09869e..55c0e2c15782 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -32,6 +32,13 @@ static struct mock_request *first_request(struct mock_engine *engine)
 					link);
 }
 
+static void advance(struct mock_engine *engine,
+		    struct mock_request *request)
+{
+	list_del_init(&request->link);
+	mock_seqno_advance(&engine->base, request->base.global_seqno);
+}
+
 static void hw_delay_complete(struct timer_list *t)
 {
 	struct mock_engine *engine = from_timer(engine, t, hw_delay);
@@ -39,15 +46,23 @@ static void hw_delay_complete(struct timer_list *t)
 
 	spin_lock(&engine->hw_lock);
 
-	request = first_request(engine);
-	if (request) {
-		list_del_init(&request->link);
-		mock_seqno_advance(&engine->base, request->base.global_seqno);
-	}
-
+	/* Timer fired, first request is complete */
 	request = first_request(engine);
 	if (request)
-		mod_timer(&engine->hw_delay, jiffies + request->delay);
+		advance(engine, request);
+
+	/*
+	 * Also immediately signal any subsequent 0-delay requests, but
+	 * requeue the timer for the next delayed request.
+	 */
+	while ((request = first_request(engine))) {
+		if (request->delay) {
+			mod_timer(&engine->hw_delay, jiffies + request->delay);
+			break;
+		}
+
+		advance(engine, request);
+	}
 
 	spin_unlock(&engine->hw_lock);
 }
@@ -98,16 +113,22 @@ static void mock_submit_request(struct drm_i915_gem_request *request)
 
 	spin_lock_irq(&engine->hw_lock);
 	list_add_tail(&mock->link, &engine->hw_queue);
-	if (mock->link.prev == &engine->hw_queue)
-		mod_timer(&engine->hw_delay, jiffies + mock->delay);
+	if (mock->link.prev == &engine->hw_queue) {
+		if (mock->delay)
+			mod_timer(&engine->hw_delay, jiffies + mock->delay);
+		else
+			advance(engine, mock);
+	}
 	spin_unlock_irq(&engine->hw_lock);
 }
 
 static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
 {
-	const unsigned long sz = roundup_pow_of_two(sizeof(struct intel_ring));
+	const unsigned long sz = PAGE_SIZE / 2;
 	struct intel_ring *ring;
 
+	BUILD_BUG_ON(MIN_SPACE_FOR_ADD_REQUEST > sz);
+
 	ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL);
 	if (!ring)
 		return NULL;
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 04eb9362f4f8..1bc61f3f76fc 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -85,6 +85,8 @@ static void mock_device_release(struct drm_device *dev)
 
 	i915_gemfs_fini(i915);
 
+	drm_mode_config_cleanup(&i915->drm);
+
 	drm_dev_fini(&i915->drm);
 	put_device(&i915->drm.pdev->dev);
 }
@@ -179,20 +181,15 @@ struct drm_i915_private *mock_gem_device(void)
 		I915_GTT_PAGE_SIZE_64K |
 		I915_GTT_PAGE_SIZE_2M;
 
-	spin_lock_init(&i915->mm.object_stat_lock);
 	mock_uncore_init(i915);
+	i915_gem_init__mm(i915);
 
 	init_waitqueue_head(&i915->gpu_error.wait_queue);
 	init_waitqueue_head(&i915->gpu_error.reset_queue);
 
 	i915->wq = alloc_ordered_workqueue("mock", 0);
 	if (!i915->wq)
-		goto put_device;
-
-	INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
-	init_llist_head(&i915->mm.free_list);
-	INIT_LIST_HEAD(&i915->mm.unbound_list);
-	INIT_LIST_HEAD(&i915->mm.bound_list);
+		goto err_drv;
 
 	mock_init_contexts(i915);
 
@@ -271,6 +268,9 @@ struct drm_i915_private *mock_gem_device(void)
 	kmem_cache_destroy(i915->objects);
 err_wq:
 	destroy_workqueue(i915->wq);
+err_drv:
+	drm_mode_config_cleanup(&i915->drm);
+	drm_dev_fini(&i915->drm);
 put_device:
 	put_device(&pdev->dev);
 err:
diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c
index 336e1afb250f..e96873f96116 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c
@@ -110,8 +110,8 @@ void mock_init_ggtt(struct drm_i915_private *i915)
 
 	ggtt->base.i915 = i915;
 
-	ggtt->mappable_base = 0;
-	ggtt->mappable_end = 2048 * PAGE_SIZE;
+	ggtt->gmadr = (struct resource) DEFINE_RES_MEM(0, 2048 * PAGE_SIZE);
+	ggtt->mappable_end = resource_size(&ggtt->gmadr);
 	ggtt->base.total = 4096 * PAGE_SIZE;
 
 	ggtt->base.clear_range = nop_clear_range;
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index 17d2f3a1c562..1d053bbefc02 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -38,7 +38,6 @@
 struct imx_drm_device {
 	struct drm_device			*drm;
 	unsigned int				pipes;
-	struct drm_fbdev_cma			*fbhelper;
 	struct drm_atomic_state			*state;
 };
 
@@ -47,13 +46,6 @@ static int legacyfb_depth = 16;
 module_param(legacyfb_depth, int, 0444);
 #endif
 
-static void imx_drm_driver_lastclose(struct drm_device *drm)
-{
-	struct imx_drm_device *imxdrm = drm->dev_private;
-
-	drm_fbdev_cma_restore_mode(imxdrm->fbhelper);
-}
-
 DEFINE_DRM_GEM_CMA_FOPS(imx_drm_driver_fops);
 
 void imx_drm_connector_destroy(struct drm_connector *connector)
@@ -69,13 +61,6 @@ void imx_drm_encoder_destroy(struct drm_encoder *encoder)
 }
 EXPORT_SYMBOL_GPL(imx_drm_encoder_destroy);
 
-static void imx_drm_output_poll_changed(struct drm_device *drm)
-{
-	struct imx_drm_device *imxdrm = drm->dev_private;
-
-	drm_fbdev_cma_hotplug_event(imxdrm->fbhelper);
-}
-
 static int imx_drm_atomic_check(struct drm_device *dev,
 				struct drm_atomic_state *state)
 {
@@ -107,7 +92,7 @@ static int imx_drm_atomic_check(struct drm_device *dev,
 
 static const struct drm_mode_config_funcs imx_drm_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
-	.output_poll_changed = imx_drm_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = imx_drm_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -186,7 +171,7 @@ static const struct drm_ioctl_desc imx_drm_ioctls[] = {
 static struct drm_driver imx_drm_driver = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
 				  DRIVER_ATOMIC,
-	.lastclose		= imx_drm_driver_lastclose,
+	.lastclose		= drm_fb_helper_lastclose,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 	.dumb_create		= drm_gem_cma_dumb_create,
@@ -272,6 +257,7 @@ static int imx_drm_bind(struct device *dev)
 	drm->mode_config.max_height = 4096;
 	drm->mode_config.funcs = &imx_drm_mode_config_funcs;
 	drm->mode_config.helper_private = &imx_drm_mode_config_helpers;
+	drm->mode_config.allow_fb_modifiers = true;
 
 	drm_mode_config_init(drm);
 
@@ -298,12 +284,9 @@ static int imx_drm_bind(struct device *dev)
 		dev_warn(dev, "Invalid legacyfb_depth.  Defaulting to 16bpp\n");
 		legacyfb_depth = 16;
 	}
-	imxdrm->fbhelper = drm_fbdev_cma_init(drm, legacyfb_depth, MAX_CRTC);
-	if (IS_ERR(imxdrm->fbhelper)) {
-		ret = PTR_ERR(imxdrm->fbhelper);
-		imxdrm->fbhelper = NULL;
+	ret = drm_fb_cma_fbdev_init(drm, legacyfb_depth, MAX_CRTC);
+	if (ret)
 		goto err_unbind;
-	}
 #endif
 
 	drm_kms_helper_poll_init(drm);
@@ -317,8 +300,7 @@ static int imx_drm_bind(struct device *dev)
 err_fbhelper:
 	drm_kms_helper_poll_fini(drm);
 #if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
-	if (imxdrm->fbhelper)
-		drm_fbdev_cma_fini(imxdrm->fbhelper);
+	drm_fb_cma_fbdev_fini(drm);
 err_unbind:
 #endif
 	component_unbind_all(drm->dev, drm);
@@ -333,14 +315,12 @@ static int imx_drm_bind(struct device *dev)
 static void imx_drm_unbind(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
-	struct imx_drm_device *imxdrm = drm->dev_private;
 
 	drm_dev_unregister(drm);
 
 	drm_kms_helper_poll_fini(drm);
 
-	if (imxdrm->fbhelper)
-		drm_fbdev_cma_fini(imxdrm->fbhelper);
+	drm_fb_cma_fbdev_fini(drm);
 
 	drm_mode_config_cleanup(drm);
 
diff --git a/drivers/gpu/drm/imx/imx-drm.h b/drivers/gpu/drm/imx/imx-drm.h
index f0b7556c0857..15c2bec47a04 100644
--- a/drivers/gpu/drm/imx/imx-drm.h
+++ b/drivers/gpu/drm/imx/imx-drm.h
@@ -8,7 +8,6 @@ struct drm_connector;
 struct drm_device;
 struct drm_display_mode;
 struct drm_encoder;
-struct drm_fbdev_cma;
 struct drm_framebuffer;
 struct drm_plane;
 struct imx_drm_crtc;
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index 9a9961802f5c..e83af0f2be86 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -225,7 +225,11 @@ static void ipu_crtc_atomic_begin(struct drm_crtc *crtc,
 				  struct drm_crtc_state *old_crtc_state)
 {
 	drm_crtc_vblank_on(crtc);
+}
 
+static void ipu_crtc_atomic_flush(struct drm_crtc *crtc,
+				  struct drm_crtc_state *old_crtc_state)
+{
 	spin_lock_irq(&crtc->dev->event_lock);
 	if (crtc->state->event) {
 		WARN_ON(drm_crtc_vblank_get(crtc));
@@ -293,6 +297,7 @@ static const struct drm_crtc_helper_funcs ipu_helper_funcs = {
 	.mode_set_nofb = ipu_crtc_mode_set_nofb,
 	.atomic_check = ipu_crtc_atomic_check,
 	.atomic_begin = ipu_crtc_atomic_begin,
+	.atomic_flush = ipu_crtc_atomic_flush,
 	.atomic_disable = ipu_crtc_atomic_disable,
 	.atomic_enable = ipu_crtc_atomic_enable,
 };
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c
index 247c60e6bed2..d9113faaa62f 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3-plane.c
@@ -22,6 +22,7 @@
 #include <drm/drm_plane_helper.h>
 
 #include "video/imx-ipu-v3.h"
+#include "imx-drm.h"
 #include "ipuv3-plane.h"
 
 struct ipu_plane_state {
@@ -77,6 +78,18 @@ static const uint32_t ipu_plane_formats[] = {
 	DRM_FORMAT_BGRX8888_A8,
 };
 
+static const uint64_t ipu_format_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
+static const uint64_t pre_format_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_VIVANTE_TILED,
+	DRM_FORMAT_MOD_VIVANTE_SUPER_TILED,
+	DRM_FORMAT_MOD_INVALID
+};
+
 int ipu_plane_irq(struct ipu_plane *ipu_plane)
 {
 	return ipu_idmac_channel_irq(ipu_plane->ipu, ipu_plane->ipu_ch,
@@ -260,7 +273,7 @@ static void ipu_plane_destroy(struct drm_plane *plane)
 	kfree(ipu_plane);
 }
 
-void ipu_plane_state_reset(struct drm_plane *plane)
+static void ipu_plane_state_reset(struct drm_plane *plane)
 {
 	struct ipu_plane_state *ipu_state;
 
@@ -280,7 +293,8 @@ void ipu_plane_state_reset(struct drm_plane *plane)
 	plane->state = &ipu_state->base;
 }
 
-struct drm_plane_state *ipu_plane_duplicate_state(struct drm_plane *plane)
+static struct drm_plane_state *
+ipu_plane_duplicate_state(struct drm_plane *plane)
 {
 	struct ipu_plane_state *state;
 
@@ -294,8 +308,8 @@ struct drm_plane_state *ipu_plane_duplicate_state(struct drm_plane *plane)
 	return &state->base;
 }
 
-void ipu_plane_destroy_state(struct drm_plane *plane,
-			     struct drm_plane_state *state)
+static void ipu_plane_destroy_state(struct drm_plane *plane,
+				    struct drm_plane_state *state)
 {
 	struct ipu_plane_state *ipu_state = to_ipu_plane_state(state);
 
@@ -303,6 +317,22 @@ void ipu_plane_destroy_state(struct drm_plane *plane,
 	kfree(ipu_state);
 }
 
+static bool ipu_plane_format_mod_supported(struct drm_plane *plane,
+					   uint32_t format, uint64_t modifier)
+{
+	struct ipu_soc *ipu = to_ipu_plane(plane)->ipu;
+
+	/* linear is supported for all planes and formats */
+	if (modifier == DRM_FORMAT_MOD_LINEAR)
+		return true;
+
+	/* without a PRG there are no supported modifiers */
+	if (!ipu_prg_present(ipu))
+		return false;
+
+	return ipu_prg_format_supported(ipu, format, modifier);
+}
+
 static const struct drm_plane_funcs ipu_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
@@ -310,6 +340,7 @@ static const struct drm_plane_funcs ipu_plane_funcs = {
 	.reset		= ipu_plane_state_reset,
 	.atomic_duplicate_state	= ipu_plane_duplicate_state,
 	.atomic_destroy_state	= ipu_plane_destroy_state,
+	.format_mod_supported = ipu_plane_format_mod_supported,
 };
 
 static int ipu_plane_atomic_check(struct drm_plane *plane,
@@ -342,10 +373,10 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 	clip.y1 = 0;
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
-	ret = drm_plane_helper_check_state(state, &clip,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   can_position, true);
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  can_position, true);
 	if (ret)
 		return ret;
 
@@ -550,8 +581,8 @@ static void ipu_plane_atomic_update(struct drm_plane *plane,
 		ipu_prg_channel_configure(ipu_plane->ipu_ch, axi_id,
 					  drm_rect_width(&state->src) >> 16,
 					  drm_rect_height(&state->src) >> 16,
-					  fb->pitches[0],
-					  fb->format->format, &eba);
+					  fb->pitches[0], fb->format->format,
+					  fb->modifier, &eba);
 	}
 
 	if (old_state->fb && !drm_atomic_crtc_needs_modeset(crtc_state)) {
@@ -700,18 +731,71 @@ static const struct drm_plane_helper_funcs ipu_plane_helper_funcs = {
 int ipu_planes_assign_pre(struct drm_device *dev,
 			  struct drm_atomic_state *state)
 {
+	struct drm_crtc_state *old_crtc_state, *crtc_state;
 	struct drm_plane_state *plane_state;
+	struct ipu_plane_state *ipu_state;
+	struct ipu_plane *ipu_plane;
 	struct drm_plane *plane;
+	struct drm_crtc *crtc;
 	int available_pres = ipu_prg_max_active_channels();
-	int i;
+	int ret, i;
 
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, crtc_state, i) {
+		ret = drm_atomic_add_affected_planes(state, crtc);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * We are going over the planes in 2 passes: first we assign PREs to
+	 * planes with a tiling modifier, which need the PREs to resolve into
+	 * linear. Any failure to assign a PRE there is fatal. In the second
+	 * pass we try to assign PREs to linear FBs, to improve memory access
+	 * patterns for them. Failure at this point is non-fatal, as we can
+	 * scan out linear FBs without a PRE.
+	 */
 	for_each_new_plane_in_state(state, plane, plane_state, i) {
-		struct ipu_plane_state *ipu_state =
-				to_ipu_plane_state(plane_state);
-		struct ipu_plane *ipu_plane = to_ipu_plane(plane);
+		ipu_state = to_ipu_plane_state(plane_state);
+		ipu_plane = to_ipu_plane(plane);
+
+		if (!plane_state->fb) {
+			ipu_state->use_pre = false;
+			continue;
+		}
+
+		if (!(plane_state->fb->flags & DRM_MODE_FB_MODIFIERS) ||
+		    plane_state->fb->modifier == DRM_FORMAT_MOD_LINEAR)
+			continue;
+
+		if (!ipu_prg_present(ipu_plane->ipu) || !available_pres)
+			return -EINVAL;
+
+		if (!ipu_prg_format_supported(ipu_plane->ipu,
+					      plane_state->fb->format->format,
+					      plane_state->fb->modifier))
+			return -EINVAL;
+
+		ipu_state->use_pre = true;
+		available_pres--;
+	}
+
+	for_each_new_plane_in_state(state, plane, plane_state, i) {
+		ipu_state = to_ipu_plane_state(plane_state);
+		ipu_plane = to_ipu_plane(plane);
+
+		if (!plane_state->fb) {
+			ipu_state->use_pre = false;
+			continue;
+		}
+
+		if ((plane_state->fb->flags & DRM_MODE_FB_MODIFIERS) &&
+		    plane_state->fb->modifier != DRM_FORMAT_MOD_LINEAR)
+			continue;
+
+		/* make sure that modifier is initialized */
+		plane_state->fb->modifier = DRM_FORMAT_MOD_LINEAR;
 
 		if (ipu_prg_present(ipu_plane->ipu) && available_pres &&
-		    plane_state->fb &&
 		    ipu_prg_format_supported(ipu_plane->ipu,
 					     plane_state->fb->format->format,
 					     plane_state->fb->modifier)) {
@@ -731,6 +815,7 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 				 enum drm_plane_type type)
 {
 	struct ipu_plane *ipu_plane;
+	const uint64_t *modifiers = ipu_format_modifiers;
 	int ret;
 
 	DRM_DEBUG_KMS("channel %d, dp flow %d, possible_crtcs=0x%x\n",
@@ -746,10 +831,13 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
 	ipu_plane->dma = dma;
 	ipu_plane->dp_flow = dp;
 
+	if (ipu_prg_present(ipu))
+		modifiers = pre_format_modifiers;
+
 	ret = drm_universal_plane_init(dev, &ipu_plane->base, possible_crtcs,
 				       &ipu_plane_funcs, ipu_plane_formats,
 				       ARRAY_SIZE(ipu_plane_formats),
-				       NULL, type, NULL);
+				       modifiers, type, NULL);
 	if (ret) {
 		DRM_ERROR("failed to initialize plane\n");
 		kfree(ipu_plane);
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
index 6f121891430f..5ef898b93d8d 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
@@ -111,10 +111,10 @@ static int mtk_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->mode.hdisplay;
 	clip.y2 = crtc_state->mode.vdisplay;
 
-	return drm_plane_helper_check_state(state, &clip,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    true, true);
+	return drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   true, true);
 }
 
 static void mtk_plane_atomic_update(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index 3ff502771ba2..59a11026dceb 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -1054,7 +1054,8 @@ static int mtk_hdmi_setup_vendor_specific_infoframe(struct mtk_hdmi *hdmi,
 	u8 buffer[10];
 	ssize_t err;
 
-	err = drm_hdmi_vendor_infoframe_from_display_mode(&frame, mode);
+	err = drm_hdmi_vendor_infoframe_from_display_mode(&frame,
+							  &hdmi->conn, mode);
 	if (err) {
 		dev_err(hdmi->dev,
 			"Failed to get vendor infoframe from mode: %zd\n", err);
@@ -1222,7 +1223,6 @@ static int mtk_hdmi_conn_get_modes(struct drm_connector *conn)
 	drm_mode_connector_update_edid_property(conn, edid);
 
 	ret = drm_add_edid_modes(conn, edid);
-	drm_edid_to_eld(conn, edid);
 	kfree(edid);
 	return ret;
 }
diff --git a/drivers/gpu/drm/meson/meson_crtc.c b/drivers/gpu/drm/meson/meson_crtc.c
index 5155f0179b61..05520202c967 100644
--- a/drivers/gpu/drm/meson/meson_crtc.c
+++ b/drivers/gpu/drm/meson/meson_crtc.c
@@ -36,6 +36,7 @@
 #include "meson_venc.h"
 #include "meson_vpp.h"
 #include "meson_viu.h"
+#include "meson_canvas.h"
 #include "meson_registers.h"
 
 /* CRTC definition */
@@ -192,6 +193,11 @@ void meson_crtc_irq(struct meson_drm *priv)
 		} else
 			meson_vpp_disable_interlace_vscaler_osd1(priv);
 
+		meson_canvas_setup(priv, MESON_CANVAS_ID_OSD1,
+			   priv->viu.osd1_addr, priv->viu.osd1_stride,
+			   priv->viu.osd1_height, MESON_CANVAS_WRAP_NONE,
+			   MESON_CANVAS_BLKMODE_LINEAR);
+
 		/* Enable OSD1 */
 		writel_bits_relaxed(VPP_OSD1_POSTBLEND, VPP_OSD1_POSTBLEND,
 				    priv->io_base + _REG(VPP_MISC));
diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
index 3b804fdaf7a0..f9ad0e960263 100644
--- a/drivers/gpu/drm/meson/meson_drv.c
+++ b/drivers/gpu/drm/meson/meson_drv.c
@@ -151,6 +151,14 @@ static struct regmap_config meson_regmap_config = {
 	.max_register   = 0x1000,
 };
 
+static void meson_vpu_init(struct meson_drm *priv)
+{
+	writel_relaxed(0x210000, priv->io_base + _REG(VPU_RDARB_MODE_L1C1));
+	writel_relaxed(0x10000, priv->io_base + _REG(VPU_RDARB_MODE_L1C2));
+	writel_relaxed(0x900000, priv->io_base + _REG(VPU_RDARB_MODE_L2C1));
+	writel_relaxed(0x20000, priv->io_base + _REG(VPU_WRARB_MODE_L2C1));
+}
+
 static int meson_drv_bind_master(struct device *dev, bool has_components)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -222,6 +230,7 @@ static int meson_drv_bind_master(struct device *dev, bool has_components)
 
 	/* Hardware Initialization */
 
+	meson_vpu_init(priv);
 	meson_venc_init(priv);
 	meson_vpp_init(priv);
 	meson_viu_init(priv);
diff --git a/drivers/gpu/drm/meson/meson_drv.h b/drivers/gpu/drm/meson/meson_drv.h
index 5e8b392b9d1f..8450d6ac8c9b 100644
--- a/drivers/gpu/drm/meson/meson_drv.h
+++ b/drivers/gpu/drm/meson/meson_drv.h
@@ -43,6 +43,9 @@ struct meson_drm {
 		bool osd1_commit;
 		uint32_t osd1_ctrl_stat;
 		uint32_t osd1_blk0_cfg[5];
+		uint32_t osd1_addr;
+		uint32_t osd1_stride;
+		uint32_t osd1_height;
 	} viu;
 
 	struct {
diff --git a/drivers/gpu/drm/meson/meson_dw_hdmi.c b/drivers/gpu/drm/meson/meson_dw_hdmi.c
index cef414466f9f..17de3afd98f6 100644
--- a/drivers/gpu/drm/meson/meson_dw_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_dw_hdmi.c
@@ -23,6 +23,7 @@
 #include <linux/of_graph.h>
 #include <linux/reset.h>
 #include <linux/clk.h>
+#include <linux/regulator/consumer.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_edid.h>
@@ -137,6 +138,7 @@ struct meson_dw_hdmi {
 	struct reset_control *hdmitx_phy;
 	struct clk *hdmi_pclk;
 	struct clk *venci_clk;
+	struct regulator *hdmi_supply;
 	u32 irq_stat;
 };
 #define encoder_to_meson_dw_hdmi(x) \
@@ -751,6 +753,17 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
 	dw_plat_data = &meson_dw_hdmi->dw_plat_data;
 	encoder = &meson_dw_hdmi->encoder;
 
+	meson_dw_hdmi->hdmi_supply = devm_regulator_get_optional(dev, "hdmi");
+	if (IS_ERR(meson_dw_hdmi->hdmi_supply)) {
+		if (PTR_ERR(meson_dw_hdmi->hdmi_supply) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		meson_dw_hdmi->hdmi_supply = NULL;
+	} else {
+		ret = regulator_enable(meson_dw_hdmi->hdmi_supply);
+		if (ret)
+			return ret;
+	}
+
 	meson_dw_hdmi->hdmitx_apb = devm_reset_control_get_exclusive(dev,
 						"hdmitx_apb");
 	if (IS_ERR(meson_dw_hdmi->hdmitx_apb)) {
diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c
index 17e96fa47868..27bd3503e1e4 100644
--- a/drivers/gpu/drm/meson/meson_plane.c
+++ b/drivers/gpu/drm/meson/meson_plane.c
@@ -61,10 +61,10 @@ static int meson_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->mode.hdisplay;
 	clip.y2 = crtc_state->mode.vdisplay;
 
-	return drm_plane_helper_check_state(state, &clip,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    true, true);
+	return drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   true, true);
 }
 
 /* Takes a fixed 16.16 number and converts it to integer. */
@@ -164,10 +164,9 @@ static void meson_plane_atomic_update(struct drm_plane *plane,
 	/* Update Canvas with buffer address */
 	gem = drm_fb_cma_get_gem_obj(fb, 0);
 
-	meson_canvas_setup(priv, MESON_CANVAS_ID_OSD1,
-			   gem->paddr, fb->pitches[0],
-			   fb->height, MESON_CANVAS_WRAP_NONE,
-			   MESON_CANVAS_BLKMODE_LINEAR);
+	priv->viu.osd1_addr = gem->paddr;
+	priv->viu.osd1_stride = fb->pitches[0];
+	priv->viu.osd1_height = fb->height;
 
 	spin_unlock_irqrestore(&priv->drm->event_lock, flags);
 }
diff --git a/drivers/gpu/drm/meson/meson_registers.h b/drivers/gpu/drm/meson/meson_registers.h
index 284738196af9..bca87143e548 100644
--- a/drivers/gpu/drm/meson/meson_registers.h
+++ b/drivers/gpu/drm/meson/meson_registers.h
@@ -1363,6 +1363,10 @@
 #define VPU_PROT3_STAT_1 0x277a
 #define VPU_PROT3_STAT_2 0x277b
 #define VPU_PROT3_REQ_ONOFF 0x277c
+#define VPU_RDARB_MODE_L1C1 0x2790
+#define VPU_RDARB_MODE_L1C2 0x2799
+#define VPU_RDARB_MODE_L2C1 0x279d
+#define VPU_WRARB_MODE_L2C1 0x27a2
 
 /* osd super scale */
 #define OSDSR_HV_SIZEIN 0x3130
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index 3e7e1cd31395..c97009bb77dd 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -216,9 +216,10 @@ static struct ttm_tt *mgag200_ttm_tt_create(struct ttm_bo_device *bdev,
 	return tt;
 }
 
-static int mgag200_ttm_tt_populate(struct ttm_tt *ttm)
+static int mgag200_ttm_tt_populate(struct ttm_tt *ttm,
+			struct ttm_operation_ctx *ctx)
 {
-	return ttm_pool_populate(ttm);
+	return ttm_pool_populate(ttm, ctx);
 }
 
 static void mgag200_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -237,7 +238,6 @@ struct ttm_bo_driver mgag200_bo_driver = {
 	.verify_access = mgag200_bo_verify_access,
 	.io_mem_reserve = &mgag200_ttm_io_mem_reserve,
 	.io_mem_free = &mgag200_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 };
 
 int mgag200_mm_init(struct mga_device *mdev)
@@ -354,6 +354,7 @@ static inline u64 mgag200_bo_gpu_offset(struct mgag200_bo *bo)
 
 int mgag200_bo_pin(struct mgag200_bo *bo, u32 pl_flag, u64 *gpu_addr)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 
 	if (bo->pin_count) {
@@ -366,7 +367,7 @@ int mgag200_bo_pin(struct mgag200_bo *bo, u32 pl_flag, u64 *gpu_addr)
 	mgag200_ttm_placement(bo, pl_flag);
 	for (i = 0; i < bo->placement.num_placement; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret)
 		return ret;
 
@@ -378,6 +379,7 @@ int mgag200_bo_pin(struct mgag200_bo *bo, u32 pl_flag, u64 *gpu_addr)
 
 int mgag200_bo_unpin(struct mgag200_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i;
 	if (!bo->pin_count) {
 		DRM_ERROR("unpin bad %p\n", bo);
@@ -389,11 +391,12 @@ int mgag200_bo_unpin(struct mgag200_bo *bo)
 
 	for (i = 0; i < bo->placement.num_placement ; i++)
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
-	return ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	return ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 }
 
 int mgag200_bo_push_sysram(struct mgag200_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int i, ret;
 	if (!bo->pin_count) {
 		DRM_ERROR("unpin bad %p\n", bo);
@@ -410,7 +413,7 @@ int mgag200_bo_push_sysram(struct mgag200_bo *bo)
 	for (i = 0; i < bo->placement.num_placement ; i++)
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 
-	ret = ttm_bo_validate(&bo->bo, &bo->placement, false, false);
+	ret = ttm_bo_validate(&bo->bo, &bo->placement, &ctx);
 	if (ret) {
 		DRM_ERROR("pushing to VRAM failed\n");
 		return ret;
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index a1f4eeeb73e2..7e09d44e4a15 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -17,6 +17,8 @@
 #include <linux/dma-mapping.h>
 #include <linux/of_address.h>
 #include <linux/soc/qcom/mdt_loader.h>
+#include <linux/pm_opp.h>
+#include <linux/nvmem-consumer.h>
 #include "msm_gem.h"
 #include "msm_mmu.h"
 #include "a5xx_gpu.h"
@@ -595,6 +597,12 @@ static int a5xx_hw_init(struct msm_gpu *gpu)
 	/* Turn on performance counters */
 	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
 
+	/* Select CP0 to always count cycles */
+	gpu_write(gpu, REG_A5XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT);
+
+	/* Select RBBM0 to countable 6 to get the busy status for devfreq */
+	gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);
+
 	/* Increase VFD cache access so LRZ and other data gets evicted less */
 	gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
 
@@ -1165,6 +1173,14 @@ static struct msm_ringbuffer *a5xx_active_ring(struct msm_gpu *gpu)
 	return a5xx_gpu->cur_ring;
 }
 
+static int a5xx_gpu_busy(struct msm_gpu *gpu, uint64_t *value)
+{
+	*value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_RBBM_0_LO,
+		REG_A5XX_RBBM_PERFCTR_RBBM_0_HI);
+
+	return 0;
+}
+
 static const struct adreno_gpu_funcs funcs = {
 	.base = {
 		.get_param = adreno_get_param,
@@ -1180,10 +1196,30 @@ static const struct adreno_gpu_funcs funcs = {
 #ifdef CONFIG_DEBUG_FS
 		.show = a5xx_show,
 #endif
+		.gpu_busy = a5xx_gpu_busy,
 	},
 	.get_timestamp = a5xx_get_timestamp,
 };
 
+static void check_speed_bin(struct device *dev)
+{
+	struct nvmem_cell *cell;
+	u32 bin, val;
+
+	cell = nvmem_cell_get(dev, "speed_bin");
+
+	/* If a nvmem cell isn't defined, nothing to do */
+	if (IS_ERR(cell))
+		return;
+
+	bin = *((u32 *) nvmem_cell_read(cell, NULL));
+	nvmem_cell_put(cell);
+
+	val = (1 << bin);
+
+	dev_pm_opp_set_supported_hw(dev, &val, 1);
+}
+
 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 {
 	struct msm_drm_private *priv = dev->dev_private;
@@ -1210,6 +1246,8 @@ struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
 
 	a5xx_gpu->lm_leakage = 0x4E001A;
 
+	check_speed_bin(&pdev->dev);
+
 	ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 4);
 	if (ret) {
 		a5xx_destroy(&(a5xx_gpu->base.base));
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_power.c b/drivers/gpu/drm/msm/adreno/a5xx_power.c
index e5700bbf09dd..4e4d965fd9ab 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_power.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_power.c
@@ -103,10 +103,16 @@ static inline uint32_t _get_mvolts(struct msm_gpu *gpu, uint32_t freq)
 	struct msm_drm_private *priv = dev->dev_private;
 	struct platform_device *pdev = priv->gpu_pdev;
 	struct dev_pm_opp *opp;
+	u32 ret = 0;
 
 	opp = dev_pm_opp_find_freq_exact(&pdev->dev, freq, true);
 
-	return (!IS_ERR(opp)) ? dev_pm_opp_get_voltage(opp) / 1000 : 0;
+	if (!IS_ERR(opp)) {
+		ret = dev_pm_opp_get_voltage(opp) / 1000;
+		dev_pm_opp_put(opp);
+	}
+
+	return ret;
 }
 
 /* Setup thermal limit management */
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 05022ea2a007..62bdb7316da1 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -17,7 +17,6 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <linux/pm_opp.h>
 #include "adreno_gpu.h"
 
 #define ANY_ID 0xff
@@ -90,14 +89,19 @@ static const struct adreno_info gpulist[] = {
 	},
 };
 
-MODULE_FIRMWARE("a300_pm4.fw");
-MODULE_FIRMWARE("a300_pfp.fw");
-MODULE_FIRMWARE("a330_pm4.fw");
-MODULE_FIRMWARE("a330_pfp.fw");
-MODULE_FIRMWARE("a420_pm4.fw");
-MODULE_FIRMWARE("a420_pfp.fw");
-MODULE_FIRMWARE("a530_fm4.fw");
-MODULE_FIRMWARE("a530_pfp.fw");
+MODULE_FIRMWARE("qcom/a300_pm4.fw");
+MODULE_FIRMWARE("qcom/a300_pfp.fw");
+MODULE_FIRMWARE("qcom/a330_pm4.fw");
+MODULE_FIRMWARE("qcom/a330_pfp.fw");
+MODULE_FIRMWARE("qcom/a420_pm4.fw");
+MODULE_FIRMWARE("qcom/a420_pfp.fw");
+MODULE_FIRMWARE("qcom/a530_pm4.fw");
+MODULE_FIRMWARE("qcom/a530_pfp.fw");
+MODULE_FIRMWARE("qcom/a530v3_gpmu.fw2");
+MODULE_FIRMWARE("qcom/a530_zap.mdt");
+MODULE_FIRMWARE("qcom/a530_zap.b00");
+MODULE_FIRMWARE("qcom/a530_zap.b01");
+MODULE_FIRMWARE("qcom/a530_zap.b02");
 
 static inline bool _rev_match(uint8_t entry, uint8_t id)
 {
@@ -125,11 +129,14 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 {
 	struct msm_drm_private *priv = dev->dev_private;
 	struct platform_device *pdev = priv->gpu_pdev;
-	struct msm_gpu *gpu = platform_get_drvdata(priv->gpu_pdev);
+	struct msm_gpu *gpu = NULL;
 	int ret;
 
+	if (pdev)
+		gpu = platform_get_drvdata(pdev);
+
 	if (!gpu) {
-		dev_err(dev->dev, "no adreno device\n");
+		dev_err_once(dev->dev, "no GPU device was found\n");
 		return NULL;
 	}
 
@@ -153,101 +160,45 @@ static void set_gpu_pdev(struct drm_device *dev,
 	priv->gpu_pdev = pdev;
 }
 
-static int find_chipid(struct device *dev, u32 *chipid)
+static int find_chipid(struct device *dev, struct adreno_rev *rev)
 {
 	struct device_node *node = dev->of_node;
 	const char *compat;
 	int ret;
+	u32 chipid;
 
 	/* first search the compat strings for qcom,adreno-XYZ.W: */
 	ret = of_property_read_string_index(node, "compatible", 0, &compat);
 	if (ret == 0) {
-		unsigned rev, patch;
+		unsigned int r, patch;
 
-		if (sscanf(compat, "qcom,adreno-%u.%u", &rev, &patch) == 2) {
-			*chipid = 0;
-			*chipid |= (rev / 100) << 24;  /* core */
-			rev %= 100;
-			*chipid |= (rev / 10) << 16;   /* major */
-			rev %= 10;
-			*chipid |= rev << 8;           /* minor */
-			*chipid |= patch;
+		if (sscanf(compat, "qcom,adreno-%u.%u", &r, &patch) == 2) {
+			rev->core = r / 100;
+			r %= 100;
+			rev->major = r / 10;
+			r %= 10;
+			rev->minor = r;
+			rev->patchid = patch;
 
 			return 0;
 		}
 	}
 
 	/* and if that fails, fall back to legacy "qcom,chipid" property: */
-	ret = of_property_read_u32(node, "qcom,chipid", chipid);
-	if (ret)
+	ret = of_property_read_u32(node, "qcom,chipid", &chipid);
+	if (ret) {
+		dev_err(dev, "could not parse qcom,chipid: %d\n", ret);
 		return ret;
-
-	dev_warn(dev, "Using legacy qcom,chipid binding!\n");
-	dev_warn(dev, "Use compatible qcom,adreno-%u%u%u.%u instead.\n",
-			(*chipid >> 24) & 0xff, (*chipid >> 16) & 0xff,
-			(*chipid >> 8) & 0xff, *chipid & 0xff);
-
-	return 0;
-}
-
-/* Get legacy powerlevels from qcom,gpu-pwrlevels and populate the opp table */
-static int adreno_get_legacy_pwrlevels(struct device *dev)
-{
-	struct device_node *child, *node;
-	int ret;
-
-	node = of_find_compatible_node(dev->of_node, NULL,
-		"qcom,gpu-pwrlevels");
-	if (!node) {
-		dev_err(dev, "Could not find the GPU powerlevels\n");
-		return -ENXIO;
 	}
 
-	for_each_child_of_node(node, child) {
-		unsigned int val;
-
-		ret = of_property_read_u32(child, "qcom,gpu-freq", &val);
-		if (ret)
-			continue;
+	rev->core = (chipid >> 24) & 0xff;
+	rev->major = (chipid >> 16) & 0xff;
+	rev->minor = (chipid >> 8) & 0xff;
+	rev->patchid = (chipid & 0xff);
 
-		/*
-		 * Skip the intentionally bogus clock value found at the bottom
-		 * of most legacy frequency tables
-		 */
-		if (val != 27000000)
-			dev_pm_opp_add(dev, val, 0);
-	}
-
-	return 0;
-}
-
-static int adreno_get_pwrlevels(struct device *dev,
-		struct adreno_platform_config *config)
-{
-	unsigned long freq = ULONG_MAX;
-	struct dev_pm_opp *opp;
-	int ret;
-
-	/* You down with OPP? */
-	if (!of_find_property(dev->of_node, "operating-points-v2", NULL))
-		ret = adreno_get_legacy_pwrlevels(dev);
-	else
-		ret = dev_pm_opp_of_add_table(dev);
-
-	if (ret)
-		return ret;
-
-	/* Find the fastest defined rate */
-	opp = dev_pm_opp_find_freq_floor(dev, &freq);
-	if (!IS_ERR(opp))
-		config->fast_rate = dev_pm_opp_get_freq(opp);
-
-	if (!config->fast_rate) {
-		DRM_DEV_INFO(dev,
-			"Could not find clock rate. Using default\n");
-		/* Pick a suitably safe clock speed for any target */
-		config->fast_rate = 200000000;
-	}
+	dev_warn(dev, "Using legacy qcom,chipid binding!\n");
+	dev_warn(dev, "Use compatible qcom,adreno-%u%u%u.%u instead.\n",
+		rev->core, rev->major, rev->minor, rev->patchid);
 
 	return 0;
 }
@@ -258,22 +209,9 @@ static int adreno_bind(struct device *dev, struct device *master, void *data)
 	const struct adreno_info *info;
 	struct drm_device *drm = dev_get_drvdata(master);
 	struct msm_gpu *gpu;
-	u32 val;
 	int ret;
 
-	ret = find_chipid(dev, &val);
-	if (ret) {
-		dev_err(dev, "could not find chipid: %d\n", ret);
-		return ret;
-	}
-
-	config.rev = ADRENO_REV((val >> 24) & 0xff,
-			(val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff);
-
-	/* find clock rates: */
-	config.fast_rate = 0;
-
-	ret = adreno_get_pwrlevels(dev, &config);
+	ret = find_chipid(dev, &config.rev);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index e2ffecce59a3..de63ff26a062 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -17,11 +17,11 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/pm_opp.h>
 #include "adreno_gpu.h"
 #include "msm_gem.h"
 #include "msm_mmu.h"
 
-
 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -461,10 +461,80 @@ void adreno_wait_ring(struct msm_ringbuffer *ring, uint32_t ndwords)
 {
 	if (spin_until(ring_freewords(ring) >= ndwords))
 		DRM_DEV_ERROR(ring->gpu->dev->dev,
-			"timeout waiting for space in ringubffer %d\n",
+			"timeout waiting for space in ringbuffer %d\n",
 			ring->id);
 }
 
+/* Get legacy powerlevels from qcom,gpu-pwrlevels and populate the opp table */
+static int adreno_get_legacy_pwrlevels(struct device *dev)
+{
+	struct device_node *child, *node;
+	int ret;
+
+	node = of_find_compatible_node(dev->of_node, NULL,
+		"qcom,gpu-pwrlevels");
+	if (!node) {
+		dev_err(dev, "Could not find the GPU powerlevels\n");
+		return -ENXIO;
+	}
+
+	for_each_child_of_node(node, child) {
+		unsigned int val;
+
+		ret = of_property_read_u32(child, "qcom,gpu-freq", &val);
+		if (ret)
+			continue;
+
+		/*
+		 * Skip the intentionally bogus clock value found at the bottom
+		 * of most legacy frequency tables
+		 */
+		if (val != 27000000)
+			dev_pm_opp_add(dev, val, 0);
+	}
+
+	return 0;
+}
+
+static int adreno_get_pwrlevels(struct device *dev,
+		struct msm_gpu *gpu)
+{
+	unsigned long freq = ULONG_MAX;
+	struct dev_pm_opp *opp;
+	int ret;
+
+	gpu->fast_rate = 0;
+
+	/* You down with OPP? */
+	if (!of_find_property(dev->of_node, "operating-points-v2", NULL))
+		ret = adreno_get_legacy_pwrlevels(dev);
+	else {
+		ret = dev_pm_opp_of_add_table(dev);
+		if (ret)
+			dev_err(dev, "Unable to set the OPP table\n");
+	}
+
+	if (!ret) {
+		/* Find the fastest defined rate */
+		opp = dev_pm_opp_find_freq_floor(dev, &freq);
+		if (!IS_ERR(opp)) {
+			gpu->fast_rate = freq;
+			dev_pm_opp_put(opp);
+		}
+	}
+
+	if (!gpu->fast_rate) {
+		dev_warn(dev,
+			"Could not find a clock rate. Using a reasonable default\n");
+		/* Pick a suitably safe clock speed for any target */
+		gpu->fast_rate = 200000000;
+	}
+
+	DBG("fast_rate=%u, slow_rate=27000000", gpu->fast_rate);
+
+	return 0;
+}
+
 int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 		struct adreno_gpu *adreno_gpu,
 		const struct adreno_gpu_funcs *funcs, int nr_rings)
@@ -479,15 +549,6 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	adreno_gpu->revn = adreno_gpu->info->revn;
 	adreno_gpu->rev = config->rev;
 
-	gpu->fast_rate = config->fast_rate;
-	gpu->bus_freq  = config->bus_freq;
-#ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
-	gpu->bus_scale_table = config->bus_scale_table;
-#endif
-
-	DBG("fast_rate=%u, slow_rate=27000000, bus_freq=%u",
-			gpu->fast_rate, gpu->bus_freq);
-
 	adreno_gpu_config.ioname = "kgsl_3d0_reg_memory";
 	adreno_gpu_config.irqname = "kgsl_3d0_irq";
 
@@ -496,6 +557,8 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 
 	adreno_gpu_config.nr_rings = nr_rings;
 
+	adreno_get_pwrlevels(&pdev->dev, gpu);
+
 	pm_runtime_set_autosuspend_delay(&pdev->dev, DRM_MSM_INACTIVE_PERIOD);
 	pm_runtime_use_autosuspend(&pdev->dev);
 	pm_runtime_enable(&pdev->dev);
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index 28e3de6e5f94..8d3d0a924908 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -129,10 +129,6 @@ struct adreno_gpu {
 /* platform config data (ie. from DT, or pdata) */
 struct adreno_platform_config {
 	struct adreno_rev rev;
-	uint32_t fast_rate, bus_freq;
-#ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
-	struct msm_bus_scale_pdata *bus_scale_table;
-#endif
 };
 
 #define ADRENO_IDLE_TIMEOUT msecs_to_jiffies(1000)
diff --git a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_14nm.c b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_14nm.c
index fe15aa64086f..71fe60e5f01f 100644
--- a/drivers/gpu/drm/msm/dsi/pll/dsi_pll_14nm.c
+++ b/drivers/gpu/drm/msm/dsi/pll/dsi_pll_14nm.c
@@ -698,7 +698,7 @@ static unsigned long dsi_pll_14nm_postdiv_recalc_rate(struct clk_hw *hw,
 	val &= div_mask(width);
 
 	return divider_recalc_rate(hw, parent_rate, val, NULL,
-				   postdiv->flags);
+				   postdiv->flags, width);
 }
 
 static long dsi_pll_14nm_postdiv_round_rate(struct clk_hw *hw,
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
index 940de51ac5cd..a1b3e31e959e 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
@@ -234,10 +234,6 @@ static inline struct clk *mpd4_lvds_pll_init(struct drm_device *dev)
 #endif
 
 #ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
-static inline int match_dev_name(struct device *dev, void *data)
-{
-	return !strcmp(dev_name(dev), data);
-}
 /* bus scaling data is associated with extra pointless platform devices,
  * "dtv", etc.. this is a bit of a hack, but we need a way for encoders
  * to find their pdata to make the bus-scaling stuff work.
@@ -245,8 +241,7 @@ static inline int match_dev_name(struct device *dev, void *data)
 static inline void *mdp4_find_pdata(const char *devname)
 {
 	struct device *dev;
-	dev = bus_find_device(&platform_bus_type, NULL,
-			(void *)devname, match_dev_name);
+	dev = bus_find_device_by_name(&platform_bus_type, NULL, devname);
 	return dev ? dev->platform_data : NULL;
 }
 #endif
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
index 3e9bba4d6624..6d8e3a9a6fc0 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
@@ -680,7 +680,7 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 	} else {
 		dev_info(&pdev->dev,
 			 "no iommu, fallback to phys contig buffers for scanout\n");
-		aspace = NULL;;
+		aspace = NULL;
 	}
 
 	pm_runtime_put_sync(&pdev->dev);
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
index be50445f9901..29678876fc09 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
@@ -327,8 +327,9 @@ static int mdp5_plane_atomic_check_with_state(struct drm_crtc_state *crtc_state,
 	min_scale = FRAC_16_16(1, 8);
 	max_scale = FRAC_16_16(8, 1);
 
-	ret = drm_plane_helper_check_state(state, &clip, min_scale,
-					   max_scale, true, true);
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  min_scale, max_scale,
+						  true, true);
 	if (ret)
 		return ret;
 
@@ -505,8 +506,9 @@ static int mdp5_plane_atomic_async_check(struct drm_plane *plane,
 	min_scale = FRAC_16_16(1, 8);
 	max_scale = FRAC_16_16(8, 1);
 
-	ret = drm_plane_helper_check_state(state, &clip, min_scale,
-					   max_scale, true, true);
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  min_scale, max_scale,
+						  true, true);
 	if (ret)
 		return ret;
 
@@ -964,8 +966,6 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	uint32_t src_x, src_y;
 	uint32_t src_w, src_h;
 	uint32_t src_img_w, src_img_h;
-	uint32_t src_x_r;
-	int crtc_x_r;
 	int ret;
 
 	nplanes = fb->format->num_planes;
@@ -1010,9 +1010,6 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 		crtc_w /= 2;
 		src_w /= 2;
 		src_img_w /= 2;
-
-		crtc_x_r = crtc_x + crtc_w;
-		src_x_r = src_x + src_w;
 	}
 
 	ret = calc_scalex_steps(plane, pix_format, src_w, crtc_w, step.x);
@@ -1052,9 +1049,9 @@ static int mdp5_plane_mode_set(struct drm_plane *plane,
 	if (right_hwpipe)
 		mdp5_hwpipe_mode_set(mdp5_kms, right_hwpipe, fb, &step, &pe,
 				     config, hdecm, vdecm, hflip, vflip,
-				     crtc_x_r, crtc_y, crtc_w, crtc_h,
+				     crtc_x + crtc_w, crtc_y, crtc_w, crtc_h,
 				     src_img_w, src_img_h,
-				     src_x_r, src_y, src_w, src_h);
+				     src_x + src_w, src_y, src_w, src_h);
 
 	plane->fb = fb;
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 0a3ea3034e39..d90ef1d78a1b 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -37,16 +37,9 @@
 #define MSM_VERSION_MINOR	3
 #define MSM_VERSION_PATCHLEVEL	0
 
-static void msm_fb_output_poll_changed(struct drm_device *dev)
-{
-	struct msm_drm_private *priv = dev->dev_private;
-	if (priv->fbdev)
-		drm_fb_helper_hotplug_event(priv->fbdev);
-}
-
 static const struct drm_mode_config_funcs mode_config_funcs = {
 	.fb_create = msm_framebuffer_create,
-	.output_poll_changed = msm_fb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = msm_atomic_commit,
 	.atomic_state_alloc = msm_atomic_state_alloc,
@@ -551,13 +544,6 @@ static void msm_postclose(struct drm_device *dev, struct drm_file *file)
 	context_close(ctx);
 }
 
-static void msm_lastclose(struct drm_device *dev)
-{
-	struct msm_drm_private *priv = dev->dev_private;
-	if (priv->fbdev)
-		drm_fb_helper_restore_fbdev_mode_unlocked(priv->fbdev);
-}
-
 static irqreturn_t msm_irq(int irq, void *arg)
 {
 	struct drm_device *dev = arg;
@@ -866,7 +852,7 @@ static struct drm_driver msm_driver = {
 				DRIVER_MODESET,
 	.open               = msm_open,
 	.postclose           = msm_postclose,
-	.lastclose          = msm_lastclose,
+	.lastclose          = drm_fb_helper_lastclose,
 	.irq_handler        = msm_irq,
 	.irq_preinstall     = msm_irq_preinstall,
 	.irq_postinstall    = msm_irq_postinstall,
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index c646843d8822..0a653dd2e618 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -303,7 +303,8 @@ int msm_perf_debugfs_init(struct drm_minor *minor);
 void msm_perf_debugfs_cleanup(struct msm_drm_private *priv);
 #else
 static inline int msm_debugfs_late_init(struct drm_device *dev) { return 0; }
-static inline void msm_rd_dump_submit(struct msm_gem_submit *submit) {}
+static inline void msm_rd_dump_submit(struct msm_rd_state *rd, struct msm_gem_submit *submit,
+		const char *fmt, ...) {}
 static inline void msm_rd_debugfs_cleanup(struct msm_drm_private *priv) {}
 static inline void msm_perf_debugfs_cleanup(struct msm_drm_private *priv) {}
 #endif
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 81fe6d6740ce..07376de9ff4c 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -93,14 +93,17 @@ static struct page **get_pages(struct drm_gem_object *obj)
 			return p;
 		}
 
+		msm_obj->pages = p;
+
 		msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
 		if (IS_ERR(msm_obj->sgt)) {
+			void *ptr = ERR_CAST(msm_obj->sgt);
+
 			dev_err(dev->dev, "failed to allocate sgt\n");
-			return ERR_CAST(msm_obj->sgt);
+			msm_obj->sgt = NULL;
+			return ptr;
 		}
 
-		msm_obj->pages = p;
-
 		/* For non-cached buffers, ensure the new pages are clean
 		 * because display controller, GPU, etc. are not coherent:
 		 */
@@ -135,7 +138,10 @@ static void put_pages(struct drm_gem_object *obj)
 		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))
 			dma_unmap_sg(obj->dev->dev, msm_obj->sgt->sgl,
 					msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
-		sg_free_table(msm_obj->sgt);
+
+		if (msm_obj->sgt)
+			sg_free_table(msm_obj->sgt);
+
 		kfree(msm_obj->sgt);
 
 		if (use_pages(obj))
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 232201403439..bd376f9e18a7 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -21,42 +21,90 @@
 #include "msm_fence.h"
 
 #include <linux/string_helpers.h>
+#include <linux/pm_opp.h>
+#include <linux/devfreq.h>
 
 
 /*
  * Power Management:
  */
 
-#ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
-#include <mach/board.h>
-static void bs_init(struct msm_gpu *gpu)
+static int msm_devfreq_target(struct device *dev, unsigned long *freq,
+		u32 flags)
 {
-	if (gpu->bus_scale_table) {
-		gpu->bsc = msm_bus_scale_register_client(gpu->bus_scale_table);
-		DBG("bus scale client: %08x", gpu->bsc);
-	}
+	struct msm_gpu *gpu = platform_get_drvdata(to_platform_device(dev));
+	struct dev_pm_opp *opp;
+
+	opp = devfreq_recommended_opp(dev, freq, flags);
+
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+
+	clk_set_rate(gpu->core_clk, *freq);
+	dev_pm_opp_put(opp);
+
+	return 0;
 }
 
-static void bs_fini(struct msm_gpu *gpu)
+static int msm_devfreq_get_dev_status(struct device *dev,
+		struct devfreq_dev_status *status)
 {
-	if (gpu->bsc) {
-		msm_bus_scale_unregister_client(gpu->bsc);
-		gpu->bsc = 0;
-	}
+	struct msm_gpu *gpu = platform_get_drvdata(to_platform_device(dev));
+	u64 cycles;
+	u32 freq = ((u32) status->current_frequency) / 1000000;
+	ktime_t time;
+
+	status->current_frequency = (unsigned long) clk_get_rate(gpu->core_clk);
+	gpu->funcs->gpu_busy(gpu, &cycles);
+
+	status->busy_time = ((u32) (cycles - gpu->devfreq.busy_cycles)) / freq;
+
+	gpu->devfreq.busy_cycles = cycles;
+
+	time = ktime_get();
+	status->total_time = ktime_us_delta(time, gpu->devfreq.time);
+	gpu->devfreq.time = time;
+
+	return 0;
+}
+
+static int msm_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
+{
+	struct msm_gpu *gpu = platform_get_drvdata(to_platform_device(dev));
+
+	*freq = (unsigned long) clk_get_rate(gpu->core_clk);
+
+	return 0;
 }
 
-static void bs_set(struct msm_gpu *gpu, int idx)
+static struct devfreq_dev_profile msm_devfreq_profile = {
+	.polling_ms = 10,
+	.target = msm_devfreq_target,
+	.get_dev_status = msm_devfreq_get_dev_status,
+	.get_cur_freq = msm_devfreq_get_cur_freq,
+};
+
+static void msm_devfreq_init(struct msm_gpu *gpu)
 {
-	if (gpu->bsc) {
-		DBG("set bus scaling: %d", idx);
-		msm_bus_scale_client_update_request(gpu->bsc, idx);
+	/* We need target support to do devfreq */
+	if (!gpu->funcs->gpu_busy)
+		return;
+
+	msm_devfreq_profile.initial_freq = gpu->fast_rate;
+
+	/*
+	 * Don't set the freq_table or max_state and let devfreq build the table
+	 * from OPP
+	 */
+
+	gpu->devfreq.devfreq = devm_devfreq_add_device(&gpu->pdev->dev,
+			&msm_devfreq_profile, "simple_ondemand", NULL);
+
+	if (IS_ERR(gpu->devfreq.devfreq)) {
+		dev_err(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n");
+		gpu->devfreq.devfreq = NULL;
 	}
 }
-#else
-static void bs_init(struct msm_gpu *gpu) {}
-static void bs_fini(struct msm_gpu *gpu) {}
-static void bs_set(struct msm_gpu *gpu, int idx) {}
-#endif
 
 static int enable_pwrrail(struct msm_gpu *gpu)
 {
@@ -143,8 +191,6 @@ static int enable_axi(struct msm_gpu *gpu)
 {
 	if (gpu->ebi1_clk)
 		clk_prepare_enable(gpu->ebi1_clk);
-	if (gpu->bus_freq)
-		bs_set(gpu, gpu->bus_freq);
 	return 0;
 }
 
@@ -152,8 +198,6 @@ static int disable_axi(struct msm_gpu *gpu)
 {
 	if (gpu->ebi1_clk)
 		clk_disable_unprepare(gpu->ebi1_clk);
-	if (gpu->bus_freq)
-		bs_set(gpu, 0);
 	return 0;
 }
 
@@ -175,6 +219,13 @@ int msm_gpu_pm_resume(struct msm_gpu *gpu)
 	if (ret)
 		return ret;
 
+	if (gpu->devfreq.devfreq) {
+		gpu->devfreq.busy_cycles = 0;
+		gpu->devfreq.time = ktime_get();
+
+		devfreq_resume_device(gpu->devfreq.devfreq);
+	}
+
 	gpu->needs_hw_init = true;
 
 	return 0;
@@ -186,6 +237,9 @@ int msm_gpu_pm_suspend(struct msm_gpu *gpu)
 
 	DBG("%s", gpu->name);
 
+	if (gpu->devfreq.devfreq)
+		devfreq_suspend_device(gpu->devfreq.devfreq);
+
 	ret = disable_axi(gpu);
 	if (ret)
 		return ret;
@@ -294,6 +348,8 @@ static void recover_worker(struct work_struct *work)
 
 			msm_rd_dump_submit(priv->hangrd, submit,
 				"offending task: %s (%s)", task->comm, cmd);
+
+			kfree(cmd);
 		} else {
 			msm_rd_dump_submit(priv->hangrd, submit, NULL);
 		}
@@ -306,7 +362,7 @@ static void recover_worker(struct work_struct *work)
 	 * needs to happen after msm_rd_dump_submit() to ensure that the
 	 * bo's referenced by the offending submit are still around.
 	 */
-	for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {
+	for (i = 0; i < gpu->nr_rings; i++) {
 		struct msm_ringbuffer *ring = gpu->rb[i];
 
 		uint32_t fence = ring->memptrs->fence;
@@ -753,7 +809,7 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	gpu->pdev = pdev;
 	platform_set_drvdata(pdev, gpu);
 
-	bs_init(gpu);
+	msm_devfreq_init(gpu);
 
 	gpu->aspace = msm_gpu_create_address_space(gpu, pdev,
 		config->va_start, config->va_end);
@@ -824,8 +880,6 @@ void msm_gpu_cleanup(struct msm_gpu *gpu)
 
 	WARN_ON(!list_empty(&gpu->active_list));
 
-	bs_fini(gpu);
-
 	for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {
 		msm_ringbuffer_destroy(gpu->rb[i]);
 		gpu->rb[i] = NULL;
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index e113d64574d3..fccfccd303af 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -66,6 +66,7 @@ struct msm_gpu_funcs {
 	/* show GPU status in debugfs: */
 	void (*show)(struct msm_gpu *gpu, struct seq_file *m);
 #endif
+	int (*gpu_busy)(struct msm_gpu *gpu, uint64_t *value);
 };
 
 struct msm_gpu {
@@ -108,12 +109,7 @@ struct msm_gpu {
 	struct clk **grp_clks;
 	int nr_clocks;
 	struct clk *ebi1_clk, *core_clk, *rbbmtimer_clk;
-	uint32_t fast_rate, bus_freq;
-
-#ifdef DOWNSTREAM_CONFIG_MSM_BUS_SCALING
-	struct msm_bus_scale_pdata *bus_scale_table;
-	uint32_t bsc;
-#endif
+	uint32_t fast_rate;
 
 	/* Hang and Inactivity Detection:
 	 */
@@ -125,6 +121,12 @@ struct msm_gpu {
 	struct work_struct recover_work;
 
 	struct drm_gem_object *memptrs_bo;
+
+	struct {
+		struct devfreq *devfreq;
+		u64 busy_cycles;
+		ktime_t time;
+	} devfreq;
 };
 
 /* It turns out that all targets use the same ringbuffer size */
diff --git a/drivers/gpu/drm/nouveau/dispnv04/arb.c b/drivers/gpu/drm/nouveau/dispnv04/arb.c
index 90075b676256..c79160c37f84 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/arb.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/arb.c
@@ -213,8 +213,10 @@ nv04_update_arb(struct drm_device *dev, int VClk, int bpp,
 	if ((dev->pdev->device & 0xffff) == 0x01a0 /*CHIPSET_NFORCE*/ ||
 	    (dev->pdev->device & 0xffff) == 0x01f0 /*CHIPSET_NFORCE2*/) {
 		uint32_t type;
+		int domain = pci_domain_nr(dev->pdev->bus);
 
-		pci_read_config_dword(pci_get_bus_and_slot(0, 1), 0x7c, &type);
+		pci_read_config_dword(pci_get_domain_bus_and_slot(domain, 0, 1),
+				      0x7c, &type);
 
 		sim_data.memory_type = (type >> 12) & 1;
 		sim_data.memory_width = 64;
diff --git a/drivers/gpu/drm/nouveau/dispnv04/hw.c b/drivers/gpu/drm/nouveau/dispnv04/hw.c
index b98599002831..0c9bdf023f5b 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/hw.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/hw.c
@@ -216,12 +216,15 @@ nouveau_hw_get_clock(struct drm_device *dev, enum nvbios_pll_type plltype)
 {
 	struct nvkm_pll_vals pllvals;
 	int ret;
+	int domain;
+
+	domain = pci_domain_nr(dev->pdev->bus);
 
 	if (plltype == PLL_MEMORY &&
 	    (dev->pdev->device & 0x0ff0) == CHIPSET_NFORCE) {
 		uint32_t mpllP;
-
-		pci_read_config_dword(pci_get_bus_and_slot(0, 3), 0x6c, &mpllP);
+		pci_read_config_dword(pci_get_domain_bus_and_slot(domain, 0, 3),
+				      0x6c, &mpllP);
 		mpllP = (mpllP >> 8) & 0xf;
 		if (!mpllP)
 			mpllP = 4;
@@ -232,7 +235,8 @@ nouveau_hw_get_clock(struct drm_device *dev, enum nvbios_pll_type plltype)
 	    (dev->pdev->device & 0xff0) == CHIPSET_NFORCE2) {
 		uint32_t clock;
 
-		pci_read_config_dword(pci_get_bus_and_slot(0, 5), 0x4c, &clock);
+		pci_read_config_dword(pci_get_domain_bus_and_slot(domain, 0, 5),
+				      0x4c, &clock);
 		return clock / 1000;
 	}
 
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
index adb78f7d083a..92be0e5269c6 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
@@ -75,6 +75,7 @@ int mcp89_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gf100_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gf108_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gk104_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
+int gk110_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gk20a_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gm107_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gm200_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h
index 59f3ba551681..b57fe4ae93ba 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h
@@ -60,6 +60,7 @@ int nvkm_secboot_reset(struct nvkm_secboot *, unsigned long);
 int gm200_secboot_new(struct nvkm_device *, int, struct nvkm_secboot **);
 int gm20b_secboot_new(struct nvkm_device *, int, struct nvkm_secboot **);
 int gp102_secboot_new(struct nvkm_device *, int, struct nvkm_secboot **);
+int gp108_secboot_new(struct nvkm_device *, int, struct nvkm_secboot **);
 int gp10b_secboot_new(struct nvkm_device *, int, struct nvkm_secboot **);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
index b1ac47eb786e..9398d9f09339 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/therm.h
@@ -46,6 +46,16 @@ enum nvkm_therm_attr_type {
 	NVKM_THERM_ATTR_THRS_SHUTDOWN_HYST = 17,
 };
 
+struct nvkm_therm_clkgate_init {
+	u32 addr;
+	u8  count;
+	u32 data;
+};
+
+struct nvkm_therm_clkgate_pack {
+	const struct nvkm_therm_clkgate_init *init;
+};
+
 struct nvkm_therm {
 	const struct nvkm_therm_func *func;
 	struct nvkm_subdev subdev;
@@ -85,17 +95,24 @@ struct nvkm_therm {
 
 	int (*attr_get)(struct nvkm_therm *, enum nvkm_therm_attr_type);
 	int (*attr_set)(struct nvkm_therm *, enum nvkm_therm_attr_type, int);
+
+	bool clkgating_enabled;
 };
 
 int nvkm_therm_temp_get(struct nvkm_therm *);
 int nvkm_therm_fan_sense(struct nvkm_therm *);
 int nvkm_therm_cstate(struct nvkm_therm *, int, int);
+void nvkm_therm_clkgate_init(struct nvkm_therm *,
+			     const struct nvkm_therm_clkgate_pack *);
+void nvkm_therm_clkgate_enable(struct nvkm_therm *);
+void nvkm_therm_clkgate_fini(struct nvkm_therm *, bool);
 
 int nv40_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int nv50_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int g84_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gt215_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gf119_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
+int gk104_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gm107_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gm200_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
 int gp100_therm_new(struct nvkm_device *, int, struct nvkm_therm **);
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index 380f340204e8..debbbf0fd4bd 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -134,7 +134,7 @@ nv50_get_intensity(struct backlight_device *bd)
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
 	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
 	struct nvif_object *device = &drm->client.device.object;
-	int or = nv_encoder->or;
+	int or = ffs(nv_encoder->dcb->or) - 1;
 	u32 div = 1025;
 	u32 val;
 
@@ -149,7 +149,7 @@ nv50_set_intensity(struct backlight_device *bd)
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
 	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
 	struct nvif_object *device = &drm->client.device.object;
-	int or = nv_encoder->or;
+	int or = ffs(nv_encoder->dcb->or) - 1;
 	u32 div = 1025;
 	u32 val = (bd->props.brightness * div) / 100;
 
@@ -170,7 +170,7 @@ nva3_get_intensity(struct backlight_device *bd)
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
 	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
 	struct nvif_object *device = &drm->client.device.object;
-	int or = nv_encoder->or;
+	int or = ffs(nv_encoder->dcb->or) - 1;
 	u32 div, val;
 
 	div  = nvif_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
@@ -188,7 +188,7 @@ nva3_set_intensity(struct backlight_device *bd)
 	struct nouveau_encoder *nv_encoder = bl_get_data(bd);
 	struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev);
 	struct nvif_object *device = &drm->client.device.object;
-	int or = nv_encoder->or;
+	int or = ffs(nv_encoder->dcb->or) - 1;
 	u32 div, val;
 
 	div = nvif_rd32(device, NV50_PDISP_SOR_PWM_DIV(or));
@@ -228,7 +228,7 @@ nv50_backlight_init(struct drm_connector *connector)
 			return -ENODEV;
 	}
 
-	if (!nvif_rd32(device, NV50_PDISP_SOR_PWM_CTL(nv_encoder->or)))
+	if (!nvif_rd32(device, NV50_PDISP_SOR_PWM_CTL(ffs(nv_encoder->dcb->or) - 1)))
 		return 0;
 
 	if (drm->client.device.info.chipset <= 0xa0 ||
@@ -268,13 +268,13 @@ nouveau_backlight_init(struct drm_device *dev)
 	struct nvif_device *device = &drm->client.device;
 	struct drm_connector *connector;
 
+	INIT_LIST_HEAD(&drm->bl_connectors);
+
 	if (apple_gmux_present()) {
 		NV_INFO(drm, "Apple GMUX detected: not registering Nouveau backlight interface\n");
 		return 0;
 	}
 
-	INIT_LIST_HEAD(&drm->bl_connectors);
-
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		if (connector->connector_type != DRM_MODE_CONNECTOR_LVDS &&
 		    connector->connector_type != DRM_MODE_CONNECTOR_eDP)
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index ef687414969e..80fa68d54bd3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -549,10 +549,10 @@ int
 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
 		    bool no_wait_gpu)
 {
+	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
 	int ret;
 
-	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
-			      interruptible, no_wait_gpu);
+	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx);
 	if (ret)
 		return ret;
 
@@ -1200,6 +1200,7 @@ static int
 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
 		      bool no_wait_gpu, struct ttm_mem_reg *new_reg)
 {
+	struct ttm_operation_ctx ctx = { intr, no_wait_gpu };
 	struct ttm_place placement_memtype = {
 		.fpfn = 0,
 		.lpfn = 0,
@@ -1214,11 +1215,11 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
 
 	tmp_reg = *new_reg;
 	tmp_reg.mm_node = NULL;
-	ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, intr, no_wait_gpu);
+	ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, &ctx);
 	if (ret)
 		return ret;
 
-	ret = ttm_tt_bind(bo->ttm, &tmp_reg);
+	ret = ttm_tt_bind(bo->ttm, &tmp_reg, &ctx);
 	if (ret)
 		goto out;
 
@@ -1226,7 +1227,7 @@ nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
 	if (ret)
 		goto out;
 
-	ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, new_reg);
+	ret = ttm_bo_move_ttm(bo, &ctx, new_reg);
 out:
 	ttm_bo_mem_put(bo, &tmp_reg);
 	return ret;
@@ -1236,6 +1237,7 @@ static int
 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
 		      bool no_wait_gpu, struct ttm_mem_reg *new_reg)
 {
+	struct ttm_operation_ctx ctx = { intr, no_wait_gpu };
 	struct ttm_place placement_memtype = {
 		.fpfn = 0,
 		.lpfn = 0,
@@ -1250,11 +1252,11 @@ nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
 
 	tmp_reg = *new_reg;
 	tmp_reg.mm_node = NULL;
-	ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, intr, no_wait_gpu);
+	ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, &ctx);
 	if (ret)
 		return ret;
 
-	ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, &tmp_reg);
+	ret = ttm_bo_move_ttm(bo, &ctx, &tmp_reg);
 	if (ret)
 		goto out;
 
@@ -1327,8 +1329,9 @@ nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
 }
 
 static int
-nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
-		bool no_wait_gpu, struct ttm_mem_reg *new_reg)
+nouveau_bo_move(struct ttm_buffer_object *bo, bool evict,
+		struct ttm_operation_ctx *ctx,
+		struct ttm_mem_reg *new_reg)
 {
 	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
 	struct nouveau_bo *nvbo = nouveau_bo(bo);
@@ -1336,7 +1339,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 	struct nouveau_drm_tile *new_tile = NULL;
 	int ret = 0;
 
-	ret = ttm_bo_wait(bo, intr, no_wait_gpu);
+	ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 	if (ret)
 		return ret;
 
@@ -1360,22 +1363,25 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
 	/* Hardware assisted copy. */
 	if (drm->ttm.move) {
 		if (new_reg->mem_type == TTM_PL_SYSTEM)
-			ret = nouveau_bo_move_flipd(bo, evict, intr,
-						    no_wait_gpu, new_reg);
+			ret = nouveau_bo_move_flipd(bo, evict,
+						    ctx->interruptible,
+						    ctx->no_wait_gpu, new_reg);
 		else if (old_reg->mem_type == TTM_PL_SYSTEM)
-			ret = nouveau_bo_move_flips(bo, evict, intr,
-						    no_wait_gpu, new_reg);
+			ret = nouveau_bo_move_flips(bo, evict,
+						    ctx->interruptible,
+						    ctx->no_wait_gpu, new_reg);
 		else
-			ret = nouveau_bo_move_m2mf(bo, evict, intr,
-						   no_wait_gpu, new_reg);
+			ret = nouveau_bo_move_m2mf(bo, evict,
+						   ctx->interruptible,
+						   ctx->no_wait_gpu, new_reg);
 		if (!ret)
 			goto out;
 	}
 
 	/* Fallback to software copy. */
-	ret = ttm_bo_wait(bo, intr, no_wait_gpu);
+	ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 	if (ret == 0)
-		ret = ttm_bo_move_memcpy(bo, intr, no_wait_gpu, new_reg);
+		ret = ttm_bo_move_memcpy(bo, ctx, new_reg);
 
 out:
 	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
@@ -1544,7 +1550,7 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
 }
 
 static int
-nouveau_ttm_tt_populate(struct ttm_tt *ttm)
+nouveau_ttm_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
 {
 	struct ttm_dma_tt *ttm_dma = (void *)ttm;
 	struct nouveau_drm *drm;
@@ -1569,17 +1575,17 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
 
 #if IS_ENABLED(CONFIG_AGP)
 	if (drm->agp.bridge) {
-		return ttm_agp_tt_populate(ttm);
+		return ttm_agp_tt_populate(ttm, ctx);
 	}
 #endif
 
 #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
 	if (swiotlb_nr_tbl()) {
-		return ttm_dma_populate((void *)ttm, dev);
+		return ttm_dma_populate((void *)ttm, dev, ctx);
 	}
 #endif
 
-	r = ttm_pool_populate(ttm);
+	r = ttm_pool_populate(ttm, ctx);
 	if (r) {
 		return r;
 	}
@@ -1669,5 +1675,4 @@ struct ttm_bo_driver nouveau_bo_driver = {
 	.fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
 	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
 	.io_mem_free = &nouveau_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.h b/drivers/gpu/drm/nouveau/nouveau_bo.h
index 7b5cc5c73d20..be8e00b49cde 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.h
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.h
@@ -105,4 +105,32 @@ nvbo_kmap_obj_iovirtual(struct nouveau_bo *nvbo)
 	return ioptr;
 }
 
+static inline void
+nouveau_bo_unmap_unpin_unref(struct nouveau_bo **pnvbo)
+{
+	if (*pnvbo) {
+		nouveau_bo_unmap(*pnvbo);
+		nouveau_bo_unpin(*pnvbo);
+		nouveau_bo_ref(NULL, pnvbo);
+	}
+}
+
+static inline int
+nouveau_bo_new_pin_map(struct nouveau_cli *cli, u64 size, int align, u32 flags,
+		       struct nouveau_bo **pnvbo)
+{
+	int ret = nouveau_bo_new(cli, size, align, flags,
+				 0, 0, NULL, NULL, pnvbo);
+	if (ret == 0) {
+		ret = nouveau_bo_pin(*pnvbo, flags, true);
+		if (ret == 0) {
+			ret = nouveau_bo_map(*pnvbo);
+			if (ret == 0)
+				return ret;
+			nouveau_bo_unpin(*pnvbo);
+		}
+		nouveau_bo_ref(NULL, pnvbo);
+	}
+	return ret;
+}
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 69d6e61a01ec..6ed9cb053dfa 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -570,9 +570,15 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 		nv_connector->edid = NULL;
 	}
 
-	ret = pm_runtime_get_sync(connector->dev->dev);
-	if (ret < 0 && ret != -EACCES)
-		return conn_status;
+	/* Outputs are only polled while runtime active, so acquiring a
+	 * runtime PM ref here is unnecessary (and would deadlock upon
+	 * runtime suspend because it waits for polling to finish).
+	 */
+	if (!drm_kms_helper_is_poll_worker()) {
+		ret = pm_runtime_get_sync(connector->dev->dev);
+		if (ret < 0 && ret != -EACCES)
+			return conn_status;
+	}
 
 	nv_encoder = nouveau_connector_ddc_detect(connector);
 	if (nv_encoder && (i2c = nv_encoder->i2c) != NULL) {
@@ -647,8 +653,10 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
 
  out:
 
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
 
 	return conn_status;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h
index b7a18fbee6dc..366acb928f57 100644
--- a/drivers/gpu/drm/nouveau/nouveau_crtc.h
+++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h
@@ -60,7 +60,6 @@ struct nouveau_crtc {
 	} cursor;
 
 	struct {
-		struct nouveau_bo *nvbo;
 		int depth;
 	} lut;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 2e7785f49e6d..009713404cc4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -29,6 +29,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 
 #include <nvif/class.h>
 
@@ -292,7 +293,7 @@ nouveau_user_framebuffer_create(struct drm_device *dev,
 
 static const struct drm_mode_config_funcs nouveau_mode_config_funcs = {
 	.fb_create = nouveau_user_framebuffer_create,
-	.output_poll_changed = nouveau_fbcon_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 };
 
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 56fe261b6268..3e293029e3a6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -524,7 +524,8 @@ nouveau_get_hdmi_dev(struct nouveau_drm *drm)
 	}
 
 	/* subfunction one is a hdmi audio device? */
-	drm->hdmi_device = pci_get_bus_and_slot((unsigned int)pdev->bus->number,
+	drm->hdmi_device = pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
+						(unsigned int)pdev->bus->number,
 						PCI_DEVFN(PCI_SLOT(pdev->devfn), 1));
 
 	if (!drm->hdmi_device) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index be7357bf2246..85c1f10bc2b6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -56,6 +56,10 @@ MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration");
 int nouveau_nofbaccel = 0;
 module_param_named(nofbaccel, nouveau_nofbaccel, int, 0400);
 
+MODULE_PARM_DESC(fbcon_bpp, "fbcon bits-per-pixel (default: auto)");
+static int nouveau_fbcon_bpp;
+module_param_named(fbcon_bpp, nouveau_fbcon_bpp, int, 0400);
+
 static void
 nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
@@ -413,14 +417,6 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 	return ret;
 }
 
-void
-nouveau_fbcon_output_poll_changed(struct drm_device *dev)
-{
-	struct nouveau_drm *drm = nouveau_drm(dev);
-	if (drm->fbcon)
-		drm_fb_helper_hotplug_event(&drm->fbcon->helper);
-}
-
 static int
 nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon)
 {
@@ -496,7 +492,7 @@ nouveau_fbcon_init(struct drm_device *dev)
 {
 	struct nouveau_drm *drm = nouveau_drm(dev);
 	struct nouveau_fbdev *fbcon;
-	int preferred_bpp;
+	int preferred_bpp = nouveau_fbcon_bpp;
 	int ret;
 
 	if (!dev->mode_config.num_crtc ||
@@ -520,13 +516,15 @@ nouveau_fbcon_init(struct drm_device *dev)
 	if (ret)
 		goto fini;
 
-	if (drm->client.device.info.ram_size <= 32 * 1024 * 1024)
-		preferred_bpp = 8;
-	else
-	if (drm->client.device.info.ram_size <= 64 * 1024 * 1024)
-		preferred_bpp = 16;
-	else
-		preferred_bpp = 32;
+	if (preferred_bpp != 8 && preferred_bpp != 16 && preferred_bpp != 32) {
+		if (drm->client.device.info.ram_size <= 32 * 1024 * 1024)
+			preferred_bpp = 8;
+		else
+		if (drm->client.device.info.ram_size <= 64 * 1024 * 1024)
+			preferred_bpp = 16;
+		else
+			preferred_bpp = 32;
+	}
 
 	/* disable all the possible outputs/crtcs before entering KMS mode */
 	if (!drm_drv_uses_atomic_modeset(dev))
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index e2bca729721e..a6f192ea3fa6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -68,8 +68,6 @@ void nouveau_fbcon_set_suspend(struct drm_device *dev, int state);
 void nouveau_fbcon_accel_save_disable(struct drm_device *dev);
 void nouveau_fbcon_accel_restore(struct drm_device *dev);
 
-void nouveau_fbcon_output_poll_changed(struct drm_device *dev);
-
 extern int nouveau_nofbaccel;
 
 #endif /* __NV50_FBCON_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index efc89aaef66a..e72a7e37eb0a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -354,7 +354,7 @@ validate_fini_no_ticket(struct validate_op *op, struct nouveau_fence *fence,
 
 		list_del(&nvbo->entry);
 		nvbo->reserved_by = NULL;
-		ttm_bo_unreserve_ticket(&nvbo->bo, &op->ticket);
+		ttm_bo_unreserve(&nvbo->bo);
 		drm_gem_object_unreference_unlocked(&nvbo->gem);
 	}
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.c b/drivers/gpu/drm/nouveau/nouveau_vga.c
index 52e52a360fb1..3da5a4305aa4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vga.c
+++ b/drivers/gpu/drm/nouveau/nouveau_vga.c
@@ -4,6 +4,7 @@
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 
 #include "nouveau_drv.h"
 #include "nouveau_acpi.h"
@@ -61,7 +62,7 @@ static void
 nouveau_switcheroo_reprobe(struct pci_dev *pdev)
 {
 	struct drm_device *dev = pci_get_drvdata(pdev);
-	nouveau_fbcon_output_poll_changed(dev);
+	drm_fb_helper_output_poll_changed(dev);
 }
 
 static bool
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 584466ef688f..caddce88d2d8 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -137,8 +137,10 @@ struct nv50_head_atom {
 	} mode;
 
 	struct {
+		bool visible;
 		u32 handle;
 		u64 offset:40;
+		u8  mode:4;
 	} lut;
 
 	struct {
@@ -192,6 +194,7 @@ struct nv50_head_atom {
 
 	union {
 		struct {
+			bool ilut:1;
 			bool core:1;
 			bool curs:1;
 		};
@@ -200,6 +203,7 @@ struct nv50_head_atom {
 
 	union {
 		struct {
+			bool ilut:1;
 			bool core:1;
 			bool curs:1;
 			bool view:1;
@@ -660,6 +664,10 @@ nv50_ovly_create(struct nvif_device *device, struct nvif_object *disp,
 
 struct nv50_head {
 	struct nouveau_crtc base;
+	struct {
+		struct nouveau_bo *nvbo[2];
+		int next;
+	} lut;
 	struct nv50_ovly ovly;
 	struct nv50_oimm oimm;
 };
@@ -1143,10 +1151,11 @@ nv50_curs_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
 {
 	int ret;
 
-	ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   true, true);
+	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
+						  &asyw->clip,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  true, true);
 	asyh->curs.visible = asyw->state.visible;
 	if (ret || !asyh->curs.visible)
 		return ret;
@@ -1432,10 +1441,11 @@ nv50_base_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
 	if (!fb->format->depth)
 		return -EINVAL;
 
-	ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   DRM_PLANE_HELPER_NO_SCALING,
-					   false, true);
+	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
+						  &asyw->clip,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, true);
 	if (ret)
 		return ret;
 
@@ -1792,6 +1802,54 @@ nv50_head_lut_clr(struct nv50_head *head)
 	}
 }
 
+static void
+nv50_head_lut_load(struct drm_property_blob *blob, int mode,
+		   struct nouveau_bo *nvbo)
+{
+	struct drm_color_lut *in = (struct drm_color_lut *)blob->data;
+	void __iomem *lut = (u8 *)nvbo_kmap_obj_iovirtual(nvbo);
+	const int size = blob->length / sizeof(*in);
+	int bits, shift, i;
+	u16 zero, r, g, b;
+
+	/* This can't happen.. But it shuts the compiler up. */
+	if (WARN_ON(size != 256))
+		return;
+
+	switch (mode) {
+	case 0: /* LORES. */
+	case 1: /* HIRES. */
+		bits = 11;
+		shift = 3;
+		zero = 0x0000;
+		break;
+	case 7: /* INTERPOLATE_257_UNITY_RANGE. */
+		bits = 14;
+		shift = 0;
+		zero = 0x6000;
+		break;
+	default:
+		WARN_ON(1);
+		return;
+	}
+
+	for (i = 0; i < size; i++) {
+		r = (drm_color_lut_extract(in[i].  red, bits) + zero) << shift;
+		g = (drm_color_lut_extract(in[i].green, bits) + zero) << shift;
+		b = (drm_color_lut_extract(in[i]. blue, bits) + zero) << shift;
+		writew(r, lut + (i * 0x08) + 0);
+		writew(g, lut + (i * 0x08) + 2);
+		writew(b, lut + (i * 0x08) + 4);
+	}
+
+	/* INTERPOLATE modes require a "next" entry to interpolate with,
+	 * so we replicate the last entry to deal with this for now.
+	 */
+	writew(r, lut + (i * 0x08) + 0);
+	writew(g, lut + (i * 0x08) + 2);
+	writew(b, lut + (i * 0x08) + 4);
+}
+
 static void
 nv50_head_lut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
@@ -1800,18 +1858,18 @@ nv50_head_lut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
 	if ((push = evo_wait(core, 7))) {
 		if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
 			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
-			evo_data(push, 0xc0000000);
+			evo_data(push, 0x80000000 | asyh->lut.mode << 30);
 			evo_data(push, asyh->lut.offset >> 8);
 		} else
 		if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
 			evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
-			evo_data(push, 0xc0000000);
+			evo_data(push, 0x80000000 | asyh->lut.mode << 30);
 			evo_data(push, asyh->lut.offset >> 8);
 			evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
 			evo_data(push, asyh->lut.handle);
 		} else {
 			evo_mthd(push, 0x0440 + (head->base.index * 0x300), 4);
-			evo_data(push, 0x83000000);
+			evo_data(push, 0x80000000 | asyh->lut.mode << 24);
 			evo_data(push, asyh->lut.offset >> 8);
 			evo_data(push, 0x00000000);
 			evo_data(push, 0x00000000);
@@ -1894,7 +1952,7 @@ nv50_head_view(struct nv50_head *head, struct nv50_head_atom *asyh)
 static void
 nv50_head_flush_clr(struct nv50_head *head, struct nv50_head_atom *asyh, bool y)
 {
-	if (asyh->clr.core && (!asyh->set.core || y))
+	if (asyh->clr.ilut && (!asyh->set.ilut || y))
 		nv50_head_lut_clr(head);
 	if (asyh->clr.core && (!asyh->set.core || y))
 		nv50_head_core_clr(head);
@@ -1907,7 +1965,15 @@ nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
 	if (asyh->set.view   ) nv50_head_view    (head, asyh);
 	if (asyh->set.mode   ) nv50_head_mode    (head, asyh);
-	if (asyh->set.core   ) nv50_head_lut_set (head, asyh);
+	if (asyh->set.ilut   ) {
+		struct nouveau_bo *nvbo = head->lut.nvbo[head->lut.next];
+		struct drm_property_blob *blob = asyh->state.gamma_lut;
+		if (blob)
+			nv50_head_lut_load(blob, asyh->lut.mode, nvbo);
+		asyh->lut.offset = nvbo->bo.offset;
+		head->lut.next ^= 1;
+		nv50_head_lut_set(head, asyh);
+	}
 	if (asyh->set.core   ) nv50_head_core_set(head, asyh);
 	if (asyh->set.curs   ) nv50_head_curs_set(head, asyh);
 	if (asyh->set.base   ) nv50_head_base    (head, asyh);
@@ -2041,6 +2107,37 @@ nv50_head_atomic_check_view(struct nv50_head_atom *armh,
 	asyh->set.view = true;
 }
 
+static void
+nv50_head_atomic_check_lut(struct nv50_head *head,
+			   struct nv50_head_atom *armh,
+			   struct nv50_head_atom *asyh)
+{
+	struct nv50_disp *disp = nv50_disp(head->base.base.dev);
+
+	/* An I8 surface without an input LUT makes no sense, and
+	 * EVO will throw an error if you try.
+	 *
+	 * Legacy clients actually cause this due to the order in
+	 * which they call ioctls, so we will enable the LUT with
+	 * whatever contents the buffer already contains to avoid
+	 * triggering the error check.
+	 */
+	if (!asyh->state.gamma_lut && asyh->base.cpp != 1) {
+		asyh->lut.handle = 0;
+		asyh->clr.ilut = armh->lut.visible;
+		return;
+	}
+
+	if (disp->disp->oclass < GF110_DISP) {
+		asyh->lut.mode = (asyh->base.cpp == 1) ? 0 : 1;
+		asyh->set.ilut = true;
+	} else {
+		asyh->lut.mode = 7;
+		asyh->set.ilut = asyh->state.color_mgmt_changed;
+	}
+	asyh->lut.handle = disp->mast.base.vram.handle;
+}
+
 static void
 nv50_head_atomic_check_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
 {
@@ -2126,6 +2223,11 @@ nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
 		if (asyh->state.mode_changed)
 			nv50_head_atomic_check_mode(head, asyh);
 
+		if (asyh->state.color_mgmt_changed ||
+		    asyh->base.cpp != armh->base.cpp)
+			nv50_head_atomic_check_lut(head, armh, asyh);
+		asyh->lut.visible = asyh->lut.handle != 0;
+
 		if (asyc) {
 			if (asyc->set.scaler)
 				nv50_head_atomic_check_view(armh, asyh, asyc);
@@ -2141,7 +2243,8 @@ nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
 			asyh->core.w = asyh->base.w;
 			asyh->core.h = asyh->base.h;
 		} else
-		if ((asyh->core.visible = asyh->curs.visible)) {
+		if ((asyh->core.visible = asyh->curs.visible) ||
+		    (asyh->core.visible = asyh->lut.visible)) {
 			/*XXX: We need to either find some way of having the
 			 *     primary base layer appear black, while still
 			 *     being able to display the other layers, or we
@@ -2159,11 +2262,10 @@ nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
 		asyh->core.layout = 1;
 		asyh->core.block = 0;
 		asyh->core.pitch = ALIGN(asyh->core.w, 64) * 4;
-		asyh->lut.handle = disp->mast.base.vram.handle;
-		asyh->lut.offset = head->base.lut.nvbo->bo.offset;
 		asyh->set.base = armh->base.cpp != asyh->base.cpp;
 		asyh->set.ovly = armh->ovly.cpp != asyh->ovly.cpp;
 	} else {
+		asyh->lut.visible = false;
 		asyh->core.visible = false;
 		asyh->curs.visible = false;
 		asyh->base.cpp = 0;
@@ -2187,8 +2289,10 @@ nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
 			asyh->clr.curs = true;
 		}
 	} else {
+		asyh->clr.ilut = armh->lut.visible;
 		asyh->clr.core = armh->core.visible;
 		asyh->clr.curs = armh->curs.visible;
+		asyh->set.ilut = asyh->lut.visible;
 		asyh->set.core = asyh->core.visible;
 		asyh->set.curs = asyh->curs.visible;
 	}
@@ -2198,47 +2302,11 @@ nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
 	return 0;
 }
 
-static void
-nv50_head_lut_load(struct drm_crtc *crtc)
-{
-	struct nv50_disp *disp = nv50_disp(crtc->dev);
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-	void __iomem *lut = nvbo_kmap_obj_iovirtual(nv_crtc->lut.nvbo);
-	u16 *r, *g, *b;
-	int i;
-
-	r = crtc->gamma_store;
-	g = r + crtc->gamma_size;
-	b = g + crtc->gamma_size;
-
-	for (i = 0; i < 256; i++) {
-		if (disp->disp->oclass < GF110_DISP) {
-			writew((*r++ >> 2) + 0x0000, lut + (i * 0x08) + 0);
-			writew((*g++ >> 2) + 0x0000, lut + (i * 0x08) + 2);
-			writew((*b++ >> 2) + 0x0000, lut + (i * 0x08) + 4);
-		} else {
-			/* 0x6000 interferes with the 14-bit color??? */
-			writew((*r++ >> 2) + 0x6000, lut + (i * 0x20) + 0);
-			writew((*g++ >> 2) + 0x6000, lut + (i * 0x20) + 2);
-			writew((*b++ >> 2) + 0x6000, lut + (i * 0x20) + 4);
-		}
-	}
-}
-
 static const struct drm_crtc_helper_funcs
 nv50_head_help = {
 	.atomic_check = nv50_head_atomic_check,
 };
 
-static int
-nv50_head_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
-		    uint32_t size,
-		    struct drm_modeset_acquire_ctx *ctx)
-{
-	nv50_head_lut_load(crtc);
-	return 0;
-}
-
 static void
 nv50_head_atomic_destroy_state(struct drm_crtc *crtc,
 			       struct drm_crtc_state *state)
@@ -2294,17 +2362,15 @@ nv50_head_reset(struct drm_crtc *crtc)
 static void
 nv50_head_destroy(struct drm_crtc *crtc)
 {
-	struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
 	struct nv50_disp *disp = nv50_disp(crtc->dev);
 	struct nv50_head *head = nv50_head(crtc);
+	int i;
 
 	nv50_dmac_destroy(&head->ovly.base, disp->disp);
 	nv50_pioc_destroy(&head->oimm.base);
 
-	nouveau_bo_unmap(nv_crtc->lut.nvbo);
-	if (nv_crtc->lut.nvbo)
-		nouveau_bo_unpin(nv_crtc->lut.nvbo);
-	nouveau_bo_ref(NULL, &nv_crtc->lut.nvbo);
+	for (i = 0; i < ARRAY_SIZE(head->lut.nvbo); i++)
+		nouveau_bo_unmap_unpin_unref(&head->lut.nvbo[i]);
 
 	drm_crtc_cleanup(crtc);
 	kfree(crtc);
@@ -2313,7 +2379,7 @@ nv50_head_destroy(struct drm_crtc *crtc)
 static const struct drm_crtc_funcs
 nv50_head_func = {
 	.reset = nv50_head_reset,
-	.gamma_set = nv50_head_gamma_set,
+	.gamma_set = drm_atomic_helper_legacy_gamma_set,
 	.destroy = nv50_head_destroy,
 	.set_config = drm_atomic_helper_set_config,
 	.page_flip = drm_atomic_helper_page_flip,
@@ -2331,7 +2397,7 @@ nv50_head_create(struct drm_device *dev, int index)
 	struct nv50_base *base;
 	struct nv50_curs *curs;
 	struct drm_crtc *crtc;
-	int ret;
+	int ret, i;
 
 	head = kzalloc(sizeof(*head), GFP_KERNEL);
 	if (!head)
@@ -2353,22 +2419,14 @@ nv50_head_create(struct drm_device *dev, int index)
 	drm_crtc_helper_add(crtc, &nv50_head_help);
 	drm_mode_crtc_set_gamma_size(crtc, 256);
 
-	ret = nouveau_bo_new(&drm->client, 8192, 0x100, TTM_PL_FLAG_VRAM,
-			     0, 0x0000, NULL, NULL, &head->base.lut.nvbo);
-	if (!ret) {
-		ret = nouveau_bo_pin(head->base.lut.nvbo, TTM_PL_FLAG_VRAM, true);
-		if (!ret) {
-			ret = nouveau_bo_map(head->base.lut.nvbo);
-			if (ret)
-				nouveau_bo_unpin(head->base.lut.nvbo);
-		}
+	for (i = 0; i < ARRAY_SIZE(head->lut.nvbo); i++) {
+		ret = nouveau_bo_new_pin_map(&drm->client, 1025 * 8, 0x100,
+					     TTM_PL_FLAG_VRAM,
+					     &head->lut.nvbo[i]);
 		if (ret)
-			nouveau_bo_ref(NULL, &head->base.lut.nvbo);
+			goto out;
 	}
 
-	if (ret)
-		goto out;
-
 	/* allocate overlay resources */
 	ret = nv50_oimm_create(device, disp->disp, index, &head->oimm);
 	if (ret)
@@ -2688,7 +2746,6 @@ nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
 	if (!drm_detect_monitor_audio(nv_connector->edid))
 		return;
 
-	drm_edid_to_eld(&nv_connector->base, nv_connector->edid);
 	memcpy(args.data, nv_connector->base.eld, sizeof(args.data));
 
 	nvif_mthd(disp->disp, 0, &args,
@@ -2755,7 +2812,8 @@ nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
 			= hdmi_infoframe_pack(&avi_frame, args.infoframes, 17);
 	}
 
-	ret = drm_hdmi_vendor_infoframe_from_display_mode(&vendor_frame.vendor.hdmi, mode);
+	ret = drm_hdmi_vendor_infoframe_from_display_mode(&vendor_frame.vendor.hdmi,
+							  &nv_connector->base, mode);
 	if (!ret) {
 		/* We have a Vendor InfoFrame, populate it to the display */
 		args.pwr.vendor_infoframe_length
@@ -3064,10 +3122,8 @@ nv50_mstc_get_modes(struct drm_connector *connector)
 
 	mstc->edid = drm_dp_mst_get_edid(&mstc->connector, mstc->port->mgr, mstc->port);
 	drm_mode_connector_update_edid_property(&mstc->connector, mstc->edid);
-	if (mstc->edid) {
+	if (mstc->edid)
 		ret = drm_add_edid_modes(&mstc->connector, mstc->edid);
-		drm_edid_to_eld(&mstc->connector, mstc->edid);
-	}
 
 	if (!mstc->connector.display_info.bpc)
 		mstc->connector.display_info.bpc = 8;
@@ -4311,7 +4367,7 @@ nv50_disp_atomic_state_alloc(struct drm_device *dev)
 static const struct drm_mode_config_funcs
 nv50_disp_func = {
 	.fb_create = nouveau_user_framebuffer_create,
-	.output_poll_changed = nouveau_fbcon_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = nv50_disp_atomic_check,
 	.atomic_commit = nv50_disp_atomic_commit,
 	.atomic_state_alloc = nv50_disp_atomic_state_alloc,
@@ -4350,7 +4406,6 @@ nv50_display_init(struct drm_device *dev)
 {
 	struct drm_encoder *encoder;
 	struct drm_plane *plane;
-	struct drm_crtc *crtc;
 	u32 *push;
 
 	push = evo_wait(nv50_mast(dev), 32);
@@ -4369,10 +4424,6 @@ nv50_display_init(struct drm_device *dev)
 		}
 	}
 
-	drm_for_each_crtc(crtc, dev) {
-		nv50_head_lut_load(crtc);
-	}
-
 	drm_for_each_plane(plane, dev) {
 		struct nv50_wndw *wndw = nv50_wndw(plane);
 		if (plane->funcs != &nv50_wndw)
@@ -4426,6 +4477,7 @@ nv50_display_create(struct drm_device *dev)
 	nouveau_display(dev)->fini = nv50_display_fini;
 	disp->disp = &nouveau_display(dev)->disp;
 	dev->mode_config.funcs = &nv50_disp_func;
+	dev->driver->driver_features |= DRIVER_PREFER_XBGR_30BPP;
 	if (nouveau_atomic)
 		dev->driver->driver_features |= DRIVER_ATOMIC;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 08e77cd55e6e..05cd674326a6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -28,6 +28,7 @@
 #include <core/option.h>
 
 #include <subdev/bios.h>
+#include <subdev/therm.h>
 
 static DEFINE_MUTEX(nv_devices_mutex);
 static LIST_HEAD(nv_devices);
@@ -1682,7 +1683,7 @@ nve4_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gk104_pmu_new,
-	.therm = gf119_therm_new,
+	.therm = gk104_therm_new,
 	.timer = nv41_timer_new,
 	.top = gk104_top_new,
 	.volt = gk104_volt_new,
@@ -1721,7 +1722,7 @@ nve6_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gk104_pmu_new,
-	.therm = gf119_therm_new,
+	.therm = gk104_therm_new,
 	.timer = nv41_timer_new,
 	.top = gk104_top_new,
 	.volt = gk104_volt_new,
@@ -1760,7 +1761,7 @@ nve7_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gk104_pmu_new,
-	.therm = gf119_therm_new,
+	.therm = gk104_therm_new,
 	.timer = nv41_timer_new,
 	.top = gk104_top_new,
 	.volt = gk104_volt_new,
@@ -1811,7 +1812,7 @@ nvf0_chipset = {
 	.bus = gf100_bus_new,
 	.clk = gk104_clk_new,
 	.devinit = gf100_devinit_new,
-	.fb = gk104_fb_new,
+	.fb = gk110_fb_new,
 	.fuse = gf100_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gk104_i2c_new,
@@ -1824,7 +1825,7 @@ nvf0_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gk110_pmu_new,
-	.therm = gf119_therm_new,
+	.therm = gk104_therm_new,
 	.timer = nv41_timer_new,
 	.top = gk104_top_new,
 	.volt = gk104_volt_new,
@@ -1849,7 +1850,7 @@ nvf1_chipset = {
 	.bus = gf100_bus_new,
 	.clk = gk104_clk_new,
 	.devinit = gf100_devinit_new,
-	.fb = gk104_fb_new,
+	.fb = gk110_fb_new,
 	.fuse = gf100_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gk104_i2c_new,
@@ -1862,7 +1863,7 @@ nvf1_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gk110_pmu_new,
-	.therm = gf119_therm_new,
+	.therm = gk104_therm_new,
 	.timer = nv41_timer_new,
 	.top = gk104_top_new,
 	.volt = gk104_volt_new,
@@ -1887,7 +1888,7 @@ nv106_chipset = {
 	.bus = gf100_bus_new,
 	.clk = gk104_clk_new,
 	.devinit = gf100_devinit_new,
-	.fb = gk104_fb_new,
+	.fb = gk110_fb_new,
 	.fuse = gf100_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gk104_i2c_new,
@@ -1900,7 +1901,7 @@ nv106_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gk208_pmu_new,
-	.therm = gf119_therm_new,
+	.therm = gk104_therm_new,
 	.timer = nv41_timer_new,
 	.top = gk104_top_new,
 	.volt = gk104_volt_new,
@@ -1925,7 +1926,7 @@ nv108_chipset = {
 	.bus = gf100_bus_new,
 	.clk = gk104_clk_new,
 	.devinit = gf100_devinit_new,
-	.fb = gk104_fb_new,
+	.fb = gk110_fb_new,
 	.fuse = gf100_fuse_new,
 	.gpio = gk104_gpio_new,
 	.i2c = gk104_i2c_new,
@@ -1938,7 +1939,7 @@ nv108_chipset = {
 	.mxm = nv50_mxm_new,
 	.pci = gk104_pci_new,
 	.pmu = gk208_pmu_new,
-	.therm = gf119_therm_new,
+	.therm = gk104_therm_new,
 	.timer = nv41_timer_new,
 	.top = gk104_top_new,
 	.volt = gk104_volt_new,
@@ -2345,6 +2346,7 @@ nv138_chipset = {
 	.mc = gp100_mc_new,
 	.mmu = gp100_mmu_new,
 	.therm = gp100_therm_new,
+	.secboot = gp108_secboot_new,
 	.pci = gp100_pci_new,
 	.pmu = gp102_pmu_new,
 	.timer = gk20a_timer_new,
@@ -2356,6 +2358,10 @@ nv138_chipset = {
 	.disp = gp102_disp_new,
 	.dma = gf119_dma_new,
 	.fifo = gp100_fifo_new,
+	.gr = gp107_gr_new,
+	.nvdec = gp102_nvdec_new,
+	.sec2 = gp102_sec2_new,
+	.sw = gf100_sw_new,
 };
 
 static const struct nvkm_device_chip
@@ -2508,6 +2514,7 @@ nvkm_device_fini(struct nvkm_device *device, bool suspend)
 		}
 	}
 
+	nvkm_therm_clkgate_fini(device->therm, suspend);
 
 	if (device->func->fini)
 		device->func->fini(device, suspend);
@@ -2597,6 +2604,7 @@ nvkm_device_init(struct nvkm_device *device)
 	}
 
 	nvkm_acpi_init(device);
+	nvkm_therm_clkgate_enable(device->therm);
 
 	time = ktime_to_us(ktime_get()) - time;
 	nvdev_trace(device, "init completed in %lldus\n", time);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
index d7c2adb9b543..c8ec3fd97155 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
@@ -137,6 +137,7 @@ struct gf100_gr_func {
 	int (*rops)(struct gf100_gr *);
 	int ppc_nr;
 	const struct gf100_grctx_func *grctx;
+	const struct nvkm_therm_clkgate_pack *clkgate_pack;
 	struct nvkm_sclass sclass[];
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
index 5e82f94c2245..1b52fcb2c49a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c
@@ -22,6 +22,7 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 #include "gf100.h"
+#include "gk104.h"
 #include "ctxgf100.h"
 
 #include <nvif/class.h>
@@ -173,6 +174,208 @@ gk104_gr_pack_mmio[] = {
 	{}
 };
 
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_main_0[] = {
+	{ 0x4041f0, 1, 0x00004046 },
+	{ 0x409890, 1, 0x00000045 },
+	{ 0x4098b0, 1, 0x0000007f },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_rstr2d_0[] = {
+	{ 0x4078c0, 1, 0x00000042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_unk_0[] = {
+	{ 0x406000, 1, 0x00004044 },
+	{ 0x405860, 1, 0x00004042 },
+	{ 0x40590c, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gcc_0[] = {
+	{ 0x408040, 1, 0x00004044 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_sked_0[] = {
+	{ 0x407000, 1, 0x00004044 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_unk_1[] = {
+	{ 0x405bf0, 1, 0x00004044 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_ctxctl_0[] = {
+	{ 0x41a890, 1, 0x00000042 },
+	{ 0x41a8b0, 1, 0x0000007f },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_unk_0[] = {
+	{ 0x418500, 1, 0x00004042 },
+	{ 0x418608, 1, 0x00004042 },
+	{ 0x418688, 1, 0x00004042 },
+	{ 0x418718, 1, 0x00000042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_esetup_0[] = {
+	{ 0x418828, 1, 0x00000044 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_tpbus_0[] = {
+	{ 0x418bbc, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_zcull_0[] = {
+	{ 0x418970, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_tpconf_0[] = {
+	{ 0x418c70, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_unk_1[] = {
+	{ 0x418cf0, 1, 0x00004042 },
+	{ 0x418d70, 1, 0x00004042 },
+	{ 0x418f0c, 1, 0x00004042 },
+	{ 0x418e0c, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_gcc_0[] = {
+	{ 0x419020, 1, 0x00004042 },
+	{ 0x419038, 1, 0x00000042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_ffb_0[] = {
+	{ 0x418898, 1, 0x00000042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_tex_0[] = {
+	{ 0x419a40, 9, 0x00004042 },
+	{ 0x419acc, 1, 0x00004047 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_poly_0[] = {
+	{ 0x419868, 1, 0x00000042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_l1c_0[] = {
+	{ 0x419ccc, 3, 0x00000042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_unk_2[] = {
+	{ 0x419c70, 1, 0x00004045 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_mp_0[] = {
+	{ 0x419fd0, 1, 0x00004043 },
+	{ 0x419fd8, 1, 0x00004049 },
+	{ 0x419fe0, 2, 0x00004042 },
+	{ 0x419ff0, 1, 0x00004046 },
+	{ 0x419ff8, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_gpc_ppc_0[] = {
+	{ 0x41be28, 1, 0x00000042 },
+	{ 0x41bfe8, 1, 0x00004042 },
+	{ 0x41bed0, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_rop_zrop_0[] = {
+	{ 0x408810, 2, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_rop_0[] = {
+	{ 0x408a80, 6, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_rop_crop_0[] = {
+	{ 0x4089a8, 1, 0x00004042 },
+	{ 0x4089b0, 1, 0x00000042 },
+	{ 0x4089b8, 1, 0x00004042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_clkgate_blcg_init_pxbar_0[] = {
+	{ 0x13c820, 1, 0x0001007f },
+	{ 0x13cbe0, 1, 0x00000042 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_pack
+gk104_clkgate_pack[] = {
+	{ gk104_clkgate_blcg_init_main_0 },
+	{ gk104_clkgate_blcg_init_rstr2d_0 },
+	{ gk104_clkgate_blcg_init_unk_0 },
+	{ gk104_clkgate_blcg_init_gcc_0 },
+	{ gk104_clkgate_blcg_init_sked_0 },
+	{ gk104_clkgate_blcg_init_unk_1 },
+	{ gk104_clkgate_blcg_init_gpc_ctxctl_0 },
+	{ gk104_clkgate_blcg_init_gpc_unk_0 },
+	{ gk104_clkgate_blcg_init_gpc_esetup_0 },
+	{ gk104_clkgate_blcg_init_gpc_tpbus_0 },
+	{ gk104_clkgate_blcg_init_gpc_zcull_0 },
+	{ gk104_clkgate_blcg_init_gpc_tpconf_0 },
+	{ gk104_clkgate_blcg_init_gpc_unk_1 },
+	{ gk104_clkgate_blcg_init_gpc_gcc_0 },
+	{ gk104_clkgate_blcg_init_gpc_ffb_0 },
+	{ gk104_clkgate_blcg_init_gpc_tex_0 },
+	{ gk104_clkgate_blcg_init_gpc_poly_0 },
+	{ gk104_clkgate_blcg_init_gpc_l1c_0 },
+	{ gk104_clkgate_blcg_init_gpc_unk_2 },
+	{ gk104_clkgate_blcg_init_gpc_mp_0 },
+	{ gk104_clkgate_blcg_init_gpc_ppc_0 },
+	{ gk104_clkgate_blcg_init_rop_zrop_0 },
+	{ gk104_clkgate_blcg_init_rop_0 },
+	{ gk104_clkgate_blcg_init_rop_crop_0 },
+	{ gk104_clkgate_blcg_init_pxbar_0 },
+	{}
+};
+
 /*******************************************************************************
  * PGRAPH engine/subdev functions
  ******************************************************************************/
@@ -214,6 +417,9 @@ gk104_gr_init(struct gf100_gr *gr)
 	gr->func->init_gpc_mmu(gr);
 
 	gf100_gr_mmio(gr, gr->func->mmio);
+	if (gr->func->clkgate_pack)
+		nvkm_therm_clkgate_init(gr->base.engine.subdev.device->therm,
+					gr->func->clkgate_pack);
 
 	nvkm_wr32(device, GPC_UNIT(0, 0x3018), 0x00000001);
 
@@ -338,6 +544,7 @@ gk104_gr = {
 	.rops = gf100_gr_rops,
 	.ppc_nr = 1,
 	.grctx = &gk104_grctx,
+	.clkgate_pack = gk104_clkgate_pack,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_A },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h
new file mode 100644
index 000000000000..a24c177365d1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Lyude Paul <lyude@redhat.com>
+ */
+#ifndef __GK104_GR_H__
+#define __GK104_GR_H__
+
+#include <subdev/therm.h>
+
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_main_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_rstr2d_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_unk_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gcc_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_sked_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_unk_1[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_ctxctl_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_unk_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_esetup_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_tpbus_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_zcull_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_tpconf_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_unk_1[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_gcc_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_ffb_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_tex_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_poly_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_l1c_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_unk_2[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_mp_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_gpc_ppc_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_rop_zrop_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_rop_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_rop_crop_0[];
+extern const struct nvkm_therm_clkgate_init gk104_clkgate_blcg_init_pxbar_0[];
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
index a38e19b61c1d..4da916a9fc73 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c
@@ -22,6 +22,7 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 #include "gf100.h"
+#include "gk104.h"
 #include "ctxgf100.h"
 
 #include <subdev/timer.h>
@@ -156,6 +157,159 @@ gk110_gr_pack_mmio[] = {
 	{}
 };
 
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_blcg_init_sked_0[] = {
+	{ 0x407000, 1, 0x00004041 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_blcg_init_gpc_gcc_0[] = {
+	{ 0x419020, 1, 0x00000042 },
+	{ 0x419038, 1, 0x00000042 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_blcg_init_gpc_l1c_0[] = {
+	{ 0x419cd4, 2, 0x00004042 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_blcg_init_gpc_mp_0[] = {
+	{ 0x419fd0, 1, 0x00004043 },
+	{ 0x419fd8, 1, 0x00004049 },
+	{ 0x419fe0, 2, 0x00004042 },
+	{ 0x419ff0, 1, 0x00000046 },
+	{ 0x419ff8, 1, 0x00004042 },
+	{ 0x419f90, 1, 0x00004042 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_main_0[] = {
+	{ 0x4041f4, 1, 0x00000000 },
+	{ 0x409894, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_unk_0[] = {
+	{ 0x406004, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_sked_0[] = {
+	{ 0x407004, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_ctxctl_0[] = {
+	{ 0x41a894, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_unk_0[] = {
+	{ 0x418504, 1, 0x00000000 },
+	{ 0x41860c, 1, 0x00000000 },
+	{ 0x41868c, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_esetup_0[] = {
+	{ 0x41882c, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_zcull_0[] = {
+	{ 0x418974, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_l1c_0[] = {
+	{ 0x419cd8, 2, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_unk_1[] = {
+	{ 0x419c74, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_mp_0[] = {
+	{ 0x419fd4, 1, 0x00004a4a },
+	{ 0x419fdc, 1, 0x00000014 },
+	{ 0x419fe4, 1, 0x00000000 },
+	{ 0x419ff4, 1, 0x00001724 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_gpc_ppc_0[] = {
+	{ 0x41be2c, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_init
+gk110_clkgate_slcg_init_pcounter_0[] = {
+	{ 0x1be018, 1, 0x000001ff },
+	{ 0x1bc018, 1, 0x000001ff },
+	{ 0x1b8018, 1, 0x000001ff },
+	{ 0x1b4124, 1, 0x00000000 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_pack
+gk110_clkgate_pack[] = {
+	{ gk104_clkgate_blcg_init_main_0 },
+	{ gk104_clkgate_blcg_init_rstr2d_0 },
+	{ gk104_clkgate_blcg_init_unk_0 },
+	{ gk104_clkgate_blcg_init_gcc_0 },
+	{ gk110_clkgate_blcg_init_sked_0 },
+	{ gk104_clkgate_blcg_init_unk_1 },
+	{ gk104_clkgate_blcg_init_gpc_ctxctl_0 },
+	{ gk104_clkgate_blcg_init_gpc_unk_0 },
+	{ gk104_clkgate_blcg_init_gpc_esetup_0 },
+	{ gk104_clkgate_blcg_init_gpc_tpbus_0 },
+	{ gk104_clkgate_blcg_init_gpc_zcull_0 },
+	{ gk104_clkgate_blcg_init_gpc_tpconf_0 },
+	{ gk104_clkgate_blcg_init_gpc_unk_1 },
+	{ gk110_clkgate_blcg_init_gpc_gcc_0 },
+	{ gk104_clkgate_blcg_init_gpc_ffb_0 },
+	{ gk104_clkgate_blcg_init_gpc_tex_0 },
+	{ gk104_clkgate_blcg_init_gpc_poly_0 },
+	{ gk110_clkgate_blcg_init_gpc_l1c_0 },
+	{ gk104_clkgate_blcg_init_gpc_unk_2 },
+	{ gk110_clkgate_blcg_init_gpc_mp_0 },
+	{ gk104_clkgate_blcg_init_gpc_ppc_0 },
+	{ gk104_clkgate_blcg_init_rop_zrop_0 },
+	{ gk104_clkgate_blcg_init_rop_0 },
+	{ gk104_clkgate_blcg_init_rop_crop_0 },
+	{ gk104_clkgate_blcg_init_pxbar_0 },
+	{ gk110_clkgate_slcg_init_main_0 },
+	{ gk110_clkgate_slcg_init_unk_0 },
+	{ gk110_clkgate_slcg_init_sked_0 },
+	{ gk110_clkgate_slcg_init_gpc_ctxctl_0 },
+	{ gk110_clkgate_slcg_init_gpc_unk_0 },
+	{ gk110_clkgate_slcg_init_gpc_esetup_0 },
+	{ gk110_clkgate_slcg_init_gpc_zcull_0 },
+	{ gk110_clkgate_slcg_init_gpc_l1c_0 },
+	{ gk110_clkgate_slcg_init_gpc_unk_1 },
+	{ gk110_clkgate_slcg_init_gpc_mp_0 },
+	{ gk110_clkgate_slcg_init_gpc_ppc_0 },
+	{ gk110_clkgate_slcg_init_pcounter_0 },
+	{}
+};
+
 /*******************************************************************************
  * PGRAPH engine/subdev functions
  ******************************************************************************/
@@ -192,6 +346,7 @@ gk110_gr = {
 	.rops = gf100_gr_rops,
 	.ppc_nr = 2,
 	.grctx = &gk110_grctx,
+	.clkgate_pack = gk110_clkgate_pack,
 	.sclass = {
 		{ -1, -1, FERMI_TWOD_A },
 		{ -1, -1, KEPLER_INLINE_TO_MEMORY_B },
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
index dde89a4a0f5b..53859b6254d6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c
@@ -462,7 +462,7 @@ nvkm_perfmon_mthd_query_domain(struct nvkm_perfmon *perfmon,
 
 		args->v0.id         = di;
 		args->v0.signal_nr  = nvkm_perfdom_count_perfsig(dom);
-		strncpy(args->v0.name, dom->name, sizeof(args->v0.name));
+		strncpy(args->v0.name, dom->name, sizeof(args->v0.name) - 1);
 
 		/* Currently only global counters (PCOUNTER) are implemented
 		 * but this will be different for local counters (MP). */
@@ -514,7 +514,7 @@ nvkm_perfmon_mthd_query_signal(struct nvkm_perfmon *perfmon,
 				 "/%s/%02x", dom->name, si);
 		} else {
 			strncpy(args->v0.name, sig->name,
-				sizeof(args->v0.name));
+				sizeof(args->v0.name) - 1);
 		}
 
 		args->v0.signal = si;
@@ -572,7 +572,7 @@ nvkm_perfmon_mthd_query_source(struct nvkm_perfmon *perfmon,
 
 		args->v0.source = sig->source[si];
 		args->v0.mask   = src->mask;
-		strncpy(args->v0.name, src->name, sizeof(args->v0.name));
+		strncpy(args->v0.name, src->name, sizeof(args->v0.name) - 1);
 	}
 
 	if (++si < source_nr) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
index 77273b53672c..58a59b7db2e5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
+++ b/drivers/gpu/drm/nouveau/nvkm/falcon/msgqueue.c
@@ -505,6 +505,7 @@ nvkm_msgqueue_new(u32 version, struct nvkm_falcon *falcon,
 		ret = msgqueue_0137bca5_new(falcon, sb, queue);
 		break;
 	case 0x0148cdec:
+	case 0x015ccf3e:
 		ret = msgqueue_0148cdec_new(falcon, sb, queue);
 		break;
 	default:
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
index 96e0941c8edd..f0a26881d9b9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c
@@ -110,6 +110,7 @@ read_pll(struct gt215_clk *clk, int idx, u32 pll)
 	struct nvkm_device *device = clk->base.subdev.device;
 	u32 ctrl = nvkm_rd32(device, pll + 0);
 	u32 sclk = 0, P = 1, N = 1, M = 1;
+	u32 MP;
 
 	if (!(ctrl & 0x00000008)) {
 		if (ctrl & 0x00000001) {
@@ -130,10 +131,12 @@ read_pll(struct gt215_clk *clk, int idx, u32 pll)
 		sclk = read_clk(clk, 0x10 + idx, false);
 	}
 
-	if (M * P)
-		return sclk * N / (M * P);
+	MP = M * P;
 
-	return 0;
+	if (!MP)
+		return 0;
+
+	return sclk * N / MP;
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
index 2571530e82f1..b4f22cce5d43 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
@@ -22,6 +22,7 @@ nvkm-y += nvkm/subdev/fb/mcp89.o
 nvkm-y += nvkm/subdev/fb/gf100.o
 nvkm-y += nvkm/subdev/fb/gf108.o
 nvkm-y += nvkm/subdev/fb/gk104.o
+nvkm-y += nvkm/subdev/fb/gk110.o
 nvkm-y += nvkm/subdev/fb/gk20a.o
 nvkm-y += nvkm/subdev/fb/gm107.o
 nvkm-y += nvkm/subdev/fb/gm200.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
index 47d28c279707..cdc4e0a2cc6b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
@@ -26,6 +26,7 @@
 
 #include <core/memory.h>
 #include <core/option.h>
+#include <subdev/therm.h>
 
 void
 gf100_fb_intr(struct nvkm_fb *base)
@@ -92,6 +93,11 @@ gf100_fb_init(struct nvkm_fb *base)
 
 	if (fb->r100c10_page)
 		nvkm_wr32(device, 0x100c10, fb->r100c10 >> 8);
+
+	if (base->func->clkgate_pack) {
+		nvkm_therm_clkgate_init(device->therm,
+					base->func->clkgate_pack);
+	}
 }
 
 void *
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
index 0a6e8eaad42c..48fd98e08baa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
@@ -20,10 +20,56 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  *
  * Authors: Ben Skeggs
+ *          Lyude Paul
  */
+#include "gk104.h"
 #include "gf100.h"
 #include "ram.h"
 
+/*
+ *******************************************************************************
+ * PGRAPH registers for clockgating
+ *******************************************************************************
+ */
+const struct nvkm_therm_clkgate_init
+gk104_fb_clkgate_blcg_init_unk_0[] = {
+	{ 0x100d10, 1, 0x0000c244 },
+	{ 0x100d30, 1, 0x0000c242 },
+	{ 0x100d3c, 1, 0x00000242 },
+	{ 0x100d48, 1, 0x00000242 },
+	{ 0x100d1c, 1, 0x00000042 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_fb_clkgate_blcg_init_vm_0[] = {
+	{ 0x100c98, 1, 0x00000242 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_fb_clkgate_blcg_init_main_0[] = {
+	{ 0x10f000, 1, 0x00000042 },
+	{ 0x17e030, 1, 0x00000044 },
+	{ 0x17e040, 1, 0x00000044 },
+	{}
+};
+
+const struct nvkm_therm_clkgate_init
+gk104_fb_clkgate_blcg_init_bcast_0[] = {
+	{ 0x17ea60, 4, 0x00000044 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_pack
+gk104_fb_clkgate_pack[] = {
+	{ gk104_fb_clkgate_blcg_init_unk_0 },
+	{ gk104_fb_clkgate_blcg_init_vm_0 },
+	{ gk104_fb_clkgate_blcg_init_main_0 },
+	{ gk104_fb_clkgate_blcg_init_bcast_0 },
+	{}
+};
+
 static const struct nvkm_fb_func
 gk104_fb = {
 	.dtor = gf100_fb_dtor,
@@ -33,6 +79,7 @@ gk104_fb = {
 	.intr = gf100_fb_intr,
 	.ram_new = gk104_ram_new,
 	.default_bigpage = 17,
+	.clkgate_pack = gk104_fb_clkgate_pack,
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h
new file mode 100644
index 000000000000..b3c78e4ff706
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Lyude Paul
+ */
+
+#ifndef __GK104_FB_H__
+#define __GK104_FB_H__
+
+#include <subdev/therm.h>
+
+extern const struct nvkm_therm_clkgate_init gk104_fb_clkgate_blcg_init_unk_0[];
+extern const struct nvkm_therm_clkgate_init gk104_fb_clkgate_blcg_init_vm_0[];
+extern const struct nvkm_therm_clkgate_init gk104_fb_clkgate_blcg_init_main_0[];
+extern const struct nvkm_therm_clkgate_init gk104_fb_clkgate_blcg_init_bcast_0[];
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk110.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk110.c
new file mode 100644
index 000000000000..0695e5dd360e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk110.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Lyude Paul
+ */
+#include "gf100.h"
+#include "gk104.h"
+#include "ram.h"
+#include <subdev/therm.h>
+#include <subdev/fb.h>
+
+/*
+ *******************************************************************************
+ * PGRAPH registers for clockgating
+ *******************************************************************************
+ */
+
+static const struct nvkm_therm_clkgate_init
+gk110_fb_clkgate_blcg_init_unk_0[] = {
+	{ 0x100d10, 1, 0x0000c242 },
+	{ 0x100d30, 1, 0x0000c242 },
+	{ 0x100d3c, 1, 0x00000242 },
+	{ 0x100d48, 1, 0x0000c242 },
+	{ 0x100d1c, 1, 0x00000042 },
+	{}
+};
+
+static const struct nvkm_therm_clkgate_pack
+gk110_fb_clkgate_pack[] = {
+	{ gk110_fb_clkgate_blcg_init_unk_0 },
+	{ gk104_fb_clkgate_blcg_init_vm_0 },
+	{ gk104_fb_clkgate_blcg_init_main_0 },
+	{ gk104_fb_clkgate_blcg_init_bcast_0 },
+	{}
+};
+
+static const struct nvkm_fb_func
+gk110_fb = {
+	.dtor = gf100_fb_dtor,
+	.oneinit = gf100_fb_oneinit,
+	.init = gf100_fb_init,
+	.init_page = gf100_fb_init_page,
+	.intr = gf100_fb_intr,
+	.ram_new = gk104_ram_new,
+	.default_bigpage = 17,
+	.clkgate_pack = gk110_fb_clkgate_pack,
+};
+
+int
+gk110_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
+{
+	return gf100_fb_new_(&gk110_fb, device, index, pfb);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
index 9351188d5d76..414a423e0e55 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
@@ -3,6 +3,7 @@
 #define __NVKM_FB_PRIV_H__
 #define nvkm_fb(p) container_of((p), struct nvkm_fb, subdev)
 #include <subdev/fb.h>
+#include <subdev/therm.h>
 struct nvkm_bios;
 
 struct nvkm_fb_func {
@@ -27,6 +28,7 @@ struct nvkm_fb_func {
 	int (*ram_new)(struct nvkm_fb *, struct nvkm_ram **);
 
 	u8 default_bigpage;
+	const struct nvkm_therm_clkgate_pack *clkgate_pack;
 };
 
 void nvkm_fb_ctor(const struct nvkm_fb_func *, struct nvkm_device *device,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
index 4c07d10bb976..18241c6ba5fa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramnv1a.c
@@ -28,8 +28,16 @@ nv1a_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
 {
 	struct pci_dev *bridge;
 	u32 mem, mib;
+	int domain = 0;
+	struct pci_dev *pdev = NULL;
 
-	bridge = pci_get_bus_and_slot(0, PCI_DEVFN(0, 1));
+	if (dev_is_pci(fb->subdev.device->dev))
+		pdev = to_pci_dev(fb->subdev.device->dev);
+
+	if (pdev)
+		domain = pci_domain_nr(pdev->bus);
+
+	bridge = pci_get_domain_bus_and_slot(domain, 0, PCI_DEVFN(0, 1));
 	if (!bridge) {
 		nvkm_error(&fb->subdev, "no bridge device\n");
 		return -ENODEV;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
index fa81d0c1ba41..37b201b95f15 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c
@@ -106,7 +106,8 @@ nvkm_uvmm_mthd_map(struct nvkm_uvmm *uvmm, void *argv, u32 argc)
 	} else
 		return ret;
 
-	if (IS_ERR((memory = nvkm_umem_search(client, handle)))) {
+	memory = nvkm_umem_search(client, handle);
+	if (IS_ERR(memory)) {
 		VMM_DEBUG(vmm, "memory %016llx %ld\n", handle, PTR_ERR(memory));
 		return PTR_ERR(memory);
 	}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
index e35d3e17cd7c..1c12e58f44c2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
@@ -642,7 +642,7 @@ nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
 			else
 				block = (size >> page[i].shift) << page[i].shift;
 		} else {
-			block = (size >> page[i].shift) << page[i].shift;;
+			block = (size >> page[i].shift) << page[i].shift;
 		}
 
 		/* Perform operation. */
@@ -1354,7 +1354,7 @@ nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
 
 		tail = this->addr + this->size;
 		if (vmm->func->page_block && next && next->page != p)
-			tail = ALIGN_DOWN(addr, vmm->func->page_block);
+			tail = ALIGN_DOWN(tail, vmm->func->page_block);
 
 		if (addr <= tail && tail - addr >= size) {
 			rb_erase(&this->tree, &vmm->free);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
index 53d01fb00a8b..1dbe593e5960 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h
@@ -47,8 +47,8 @@ static uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x584d454d,
-	0x00000756,
-	0x00000748,
+	0x00000754,
+	0x00000746,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -69,8 +69,8 @@ static uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x46524550,
-	0x0000075a,
 	0x00000758,
+	0x00000756,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -91,8 +91,8 @@ static uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x5f433249,
-	0x00000b8a,
-	0x00000a2d,
+	0x00000b88,
+	0x00000a2b,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -113,8 +113,8 @@ static uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x54534554,
-	0x00000bb3,
-	0x00000b8c,
+	0x00000bb1,
+	0x00000b8a,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -135,8 +135,8 @@ static uint32_t gf100_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x454c4449,
-	0x00000bbf,
 	0x00000bbd,
+	0x00000bbb,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -237,19 +237,19 @@ static uint32_t gf100_pmu_data[] = {
 	0x000005d3,
 	0x00000003,
 	0x00000002,
-	0x0000069d,
+	0x0000069b,
 	0x00040004,
 	0x00000000,
-	0x000006b9,
+	0x000006b7,
 	0x00010005,
 	0x00000000,
-	0x000006d6,
+	0x000006d4,
 	0x00010006,
 	0x00000000,
 	0x0000065b,
 	0x00000007,
 	0x00000000,
-	0x000006e1,
+	0x000006df,
 /* 0x03c4: memx_func_tail */
 /* 0x03c4: memx_ts_start */
 	0x00000000,
@@ -1373,432 +1373,432 @@ static uint32_t gf100_pmu_code[] = {
 /* 0x065b: memx_func_wait_vblank */
 	0x9800f840,
 	0x66b00016,
-	0x130bf400,
+	0x120bf400,
 	0xf40166b0,
 	0x0ef4060b,
 /* 0x066d: memx_func_wait_vblank_head1 */
-	0x2077f12e,
-	0x070ef400,
-/* 0x0674: memx_func_wait_vblank_head0 */
-	0x000877f1,
-/* 0x0678: memx_func_wait_vblank_0 */
-	0x07c467f1,
-	0xcf0664b6,
-	0x67fd0066,
-	0xf31bf404,
-/* 0x0688: memx_func_wait_vblank_1 */
-	0x07c467f1,
-	0xcf0664b6,
-	0x67fd0066,
-	0xf30bf404,
-/* 0x0698: memx_func_wait_vblank_fini */
-	0xf80410b6,
-/* 0x069d: memx_func_wr32 */
-	0x00169800,
-	0xb6011598,
-	0x60f90810,
-	0xd0fc50f9,
-	0x21f4e0fc,
-	0x0242b640,
-	0xf8e91bf4,
-/* 0x06b9: memx_func_wait */
-	0x2c87f000,
-	0xcf0684b6,
-	0x1e980088,
-	0x011d9800,
-	0x98021c98,
-	0x10b6031b,
-	0xa321f410,
-/* 0x06d6: memx_func_delay */
-	0x1e9800f8,
-	0x0410b600,
-	0xf87e21f4,
-/* 0x06e1: memx_func_train */
-/* 0x06e3: memx_exec */
-	0xf900f800,
-	0xb9d0f9e0,
-	0xb2b902c1,
-/* 0x06ed: memx_exec_next */
-	0x00139802,
-	0xe70410b6,
-	0xe701f034,
-	0xb601e033,
-	0x30f00132,
-	0xde35980c,
-	0x12b855f9,
-	0xe41ef406,
-	0x98f10b98,
-	0xcbbbf20c,
-	0xc4b7f102,
-	0x06b4b607,
-	0xfc00bbcf,
-	0xf5e0fcd0,
-	0xf8033621,
-/* 0x0729: memx_info */
-	0x01c67000,
-/* 0x072f: memx_info_data */
-	0xf10e0bf4,
-	0xf103ccc7,
-	0xf40800b7,
-/* 0x073a: memx_info_train */
-	0xc7f10b0e,
-	0xb7f10bcc,
-/* 0x0742: memx_info_send */
-	0x21f50100,
-	0x00f80336,
-/* 0x0748: memx_recv */
-	0xf401d6b0,
-	0xd6b0980b,
-	0xd80bf400,
-/* 0x0756: memx_init */
-	0x00f800f8,
-/* 0x0758: perf_recv */
-/* 0x075a: perf_init */
+	0x2077f02c,
+/* 0x0673: memx_func_wait_vblank_head0 */
+	0xf0060ef4,
+/* 0x0676: memx_func_wait_vblank_0 */
+	0x67f10877,
+	0x64b607c4,
+	0x0066cf06,
+	0xf40467fd,
+/* 0x0686: memx_func_wait_vblank_1 */
+	0x67f1f31b,
+	0x64b607c4,
+	0x0066cf06,
+	0xf40467fd,
+/* 0x0696: memx_func_wait_vblank_fini */
+	0x10b6f30b,
+/* 0x069b: memx_func_wr32 */
+	0x9800f804,
+	0x15980016,
+	0x0810b601,
+	0x50f960f9,
+	0xe0fcd0fc,
+	0xb64021f4,
+	0x1bf40242,
+/* 0x06b7: memx_func_wait */
+	0xf000f8e9,
+	0x84b62c87,
+	0x0088cf06,
+	0x98001e98,
+	0x1c98011d,
+	0x031b9802,
+	0xf41010b6,
+	0x00f8a321,
+/* 0x06d4: memx_func_delay */
+	0xb6001e98,
+	0x21f40410,
+/* 0x06df: memx_func_train */
+	0xf800f87e,
+/* 0x06e1: memx_exec */
+	0xf9e0f900,
+	0x02c1b9d0,
+/* 0x06eb: memx_exec_next */
+	0x9802b2b9,
+	0x10b60013,
+	0xf034e704,
+	0xe033e701,
+	0x0132b601,
+	0x980c30f0,
+	0x55f9de35,
+	0xf40612b8,
+	0x0b98e41e,
+	0xf20c98f1,
+	0xf102cbbb,
+	0xb607c4b7,
+	0xbbcf06b4,
+	0xfcd0fc00,
+	0x3621f5e0,
+/* 0x0727: memx_info */
+	0x7000f803,
+	0x0bf401c6,
+/* 0x072d: memx_info_data */
+	0xccc7f10e,
+	0x00b7f103,
+	0x0b0ef408,
+/* 0x0738: memx_info_train */
+	0x0bccc7f1,
+	0x0100b7f1,
+/* 0x0740: memx_info_send */
+	0x033621f5,
+/* 0x0746: memx_recv */
+	0xd6b000f8,
+	0x980bf401,
+	0xf400d6b0,
+	0x00f8d80b,
+/* 0x0754: memx_init */
+/* 0x0756: perf_recv */
 	0x00f800f8,
-/* 0x075c: i2c_drive_scl */
-	0xf40036b0,
-	0x07f1110b,
-	0x04b607e0,
-	0x0001d006,
-	0x00f804bd,
-/* 0x0770: i2c_drive_scl_lo */
-	0x07e407f1,
-	0xd00604b6,
-	0x04bd0001,
-/* 0x077e: i2c_drive_sda */
+/* 0x0758: perf_init */
+/* 0x075a: i2c_drive_scl */
 	0x36b000f8,
 	0x110bf400,
 	0x07e007f1,
 	0xd00604b6,
-	0x04bd0002,
-/* 0x0792: i2c_drive_sda_lo */
+	0x04bd0001,
+/* 0x076e: i2c_drive_scl_lo */
 	0x07f100f8,
 	0x04b607e4,
+	0x0001d006,
+	0x00f804bd,
+/* 0x077c: i2c_drive_sda */
+	0xf40036b0,
+	0x07f1110b,
+	0x04b607e0,
 	0x0002d006,
 	0x00f804bd,
-/* 0x07a0: i2c_sense_scl */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0431fd00,
-	0xf4060bf4,
-/* 0x07b6: i2c_sense_scl_done */
-	0x00f80131,
-/* 0x07b8: i2c_sense_sda */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0432fd00,
-	0xf4060bf4,
-/* 0x07ce: i2c_sense_sda_done */
-	0x00f80131,
-/* 0x07d0: i2c_raise_scl */
-	0x47f140f9,
-	0x37f00898,
-	0x5c21f501,
-/* 0x07dd: i2c_raise_scl_wait */
-	0xe8e7f107,
-	0x7e21f403,
-	0x07a021f5,
-	0xb60901f4,
-	0x1bf40142,
-/* 0x07f1: i2c_raise_scl_done */
-	0xf840fcef,
-/* 0x07f5: i2c_start */
-	0xa021f500,
-	0x0d11f407,
-	0x07b821f5,
-	0xf40611f4,
-/* 0x0806: i2c_start_rep */
-	0x37f0300e,
-	0x5c21f500,
-	0x0137f007,
-	0x077e21f5,
-	0xb60076bb,
-	0x50f90465,
-	0xbb046594,
-	0x50bd0256,
-	0xfc0475fd,
-	0xd021f550,
-	0x0464b607,
-/* 0x0833: i2c_start_send */
-	0xf01f11f4,
+/* 0x0790: i2c_drive_sda_lo */
+	0x07e407f1,
+	0xd00604b6,
+	0x04bd0002,
+/* 0x079e: i2c_sense_scl */
+	0x32f400f8,
+	0xc437f101,
+	0x0634b607,
+	0xfd0033cf,
+	0x0bf40431,
+	0x0131f406,
+/* 0x07b4: i2c_sense_scl_done */
+/* 0x07b6: i2c_sense_sda */
+	0x32f400f8,
+	0xc437f101,
+	0x0634b607,
+	0xfd0033cf,
+	0x0bf40432,
+	0x0131f406,
+/* 0x07cc: i2c_sense_sda_done */
+/* 0x07ce: i2c_raise_scl */
+	0x40f900f8,
+	0x089847f1,
+	0xf50137f0,
+/* 0x07db: i2c_raise_scl_wait */
+	0xf1075a21,
+	0xf403e8e7,
+	0x21f57e21,
+	0x01f4079e,
+	0x0142b609,
+/* 0x07ef: i2c_raise_scl_done */
+	0xfcef1bf4,
+/* 0x07f3: i2c_start */
+	0xf500f840,
+	0xf4079e21,
+	0x21f50d11,
+	0x11f407b6,
+	0x300ef406,
+/* 0x0804: i2c_start_rep */
+	0xf50037f0,
+	0xf0075a21,
+	0x21f50137,
+	0x76bb077c,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb607ce21,
+	0x11f40464,
+/* 0x0831: i2c_start_send */
+	0x0037f01f,
+	0x077c21f5,
+	0x1388e7f1,
+	0xf07e21f4,
 	0x21f50037,
-	0xe7f1077e,
+	0xe7f1075a,
 	0x21f41388,
-	0x0037f07e,
-	0x075c21f5,
-	0x1388e7f1,
-/* 0x084f: i2c_start_out */
-	0xf87e21f4,
-/* 0x0851: i2c_stop */
-	0x0037f000,
-	0x075c21f5,
-	0xf50037f0,
-	0xf1077e21,
-	0xf403e8e7,
-	0x37f07e21,
-	0x5c21f501,
-	0x88e7f107,
-	0x7e21f413,
+/* 0x084d: i2c_start_out */
+/* 0x084f: i2c_stop */
+	0xf000f87e,
+	0x21f50037,
+	0x37f0075a,
+	0x7c21f500,
+	0xe8e7f107,
+	0x7e21f403,
 	0xf50137f0,
-	0xf1077e21,
+	0xf1075a21,
 	0xf41388e7,
-	0x00f87e21,
-/* 0x0884: i2c_bitw */
-	0x077e21f5,
-	0x03e8e7f1,
-	0xbb7e21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x07d021f5,
-	0xf40464b6,
-	0xe7f11811,
-	0x21f41388,
-	0x0037f07e,
-	0x075c21f5,
-	0x1388e7f1,
-/* 0x08c3: i2c_bitw_out */
-	0xf87e21f4,
-/* 0x08c5: i2c_bitr */
-	0x0137f000,
-	0x077e21f5,
-	0x03e8e7f1,
-	0xbb7e21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x07d021f5,
-	0xf40464b6,
-	0x21f51b11,
-	0x37f007b8,
-	0x5c21f500,
+	0x37f07e21,
+	0x7c21f501,
 	0x88e7f107,
 	0x7e21f413,
-	0xf4013cf0,
-/* 0x090a: i2c_bitr_done */
-	0x00f80131,
-/* 0x090c: i2c_get_byte */
-	0xf00057f0,
-/* 0x0912: i2c_get_byte_next */
-	0x54b60847,
-	0x0076bb01,
+/* 0x0882: i2c_bitw */
+	0x21f500f8,
+	0xe7f1077c,
+	0x21f403e8,
+	0x0076bb7e,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b608c5,
-	0x2b11f404,
-	0xb60553fd,
-	0x1bf40142,
-	0x0137f0d8,
-	0xb60076bb,
-	0x50f90465,
-	0xbb046594,
-	0x50bd0256,
-	0xfc0475fd,
-	0x8421f550,
-	0x0464b608,
-/* 0x095c: i2c_get_byte_done */
-/* 0x095e: i2c_put_byte */
-	0x47f000f8,
-/* 0x0961: i2c_put_byte_next */
-	0x0142b608,
-	0xbb3854ff,
+	0x64b607ce,
+	0x1811f404,
+	0x1388e7f1,
+	0xf07e21f4,
+	0x21f50037,
+	0xe7f1075a,
+	0x21f41388,
+/* 0x08c1: i2c_bitw_out */
+/* 0x08c3: i2c_bitr */
+	0xf000f87e,
+	0x21f50137,
+	0xe7f1077c,
+	0x21f403e8,
+	0x0076bb7e,
+	0xf90465b6,
+	0x04659450,
+	0xbd0256bb,
+	0x0475fd50,
+	0x21f550fc,
+	0x64b607ce,
+	0x1b11f404,
+	0x07b621f5,
+	0xf50037f0,
+	0xf1075a21,
+	0xf41388e7,
+	0x3cf07e21,
+	0x0131f401,
+/* 0x0908: i2c_bitr_done */
+/* 0x090a: i2c_get_byte */
+	0x57f000f8,
+	0x0847f000,
+/* 0x0910: i2c_get_byte_next */
+	0xbb0154b6,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x088421f5,
+	0x08c321f5,
 	0xf40464b6,
-	0x46b03411,
-	0xd81bf400,
-	0xb60076bb,
-	0x50f90465,
-	0xbb046594,
-	0x50bd0256,
-	0xfc0475fd,
-	0xc521f550,
-	0x0464b608,
-	0xbb0f11f4,
-	0x36b00076,
-	0x061bf401,
-/* 0x09b7: i2c_put_byte_done */
-	0xf80132f4,
-/* 0x09b9: i2c_addr */
-	0x0076bb00,
+	0x53fd2b11,
+	0x0142b605,
+	0xf0d81bf4,
+	0x76bb0137,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb6088221,
+/* 0x095a: i2c_get_byte_done */
+	0x00f80464,
+/* 0x095c: i2c_put_byte */
+/* 0x095f: i2c_put_byte_next */
+	0xb60847f0,
+	0x54ff0142,
+	0x0076bb38,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b607f5,
-	0x2911f404,
-	0x012ec3e7,
-	0xfd0134b6,
-	0x76bb0553,
+	0x64b60882,
+	0x3411f404,
+	0xf40046b0,
+	0x76bbd81b,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb6095e21,
-/* 0x09fe: i2c_addr_done */
-	0x00f80464,
-/* 0x0a00: i2c_acquire_addr */
-	0xb6f8cec7,
-	0xe0b702e4,
-	0xee980d1c,
-/* 0x0a0f: i2c_acquire */
-	0xf500f800,
-	0xf40a0021,
-	0xd9f00421,
-	0x4021f403,
-/* 0x0a1e: i2c_release */
-	0x21f500f8,
-	0x21f40a00,
-	0x03daf004,
-	0xf84021f4,
-/* 0x0a2d: i2c_recv */
-	0x0132f400,
-	0xb6f8c1c7,
-	0x16b00214,
-	0x3a1ff528,
-	0xf413a001,
-	0x0032980c,
-	0x0ccc13a0,
-	0xf4003198,
-	0xd0f90231,
-	0xd0f9e0f9,
-	0x000067f1,
-	0x100063f1,
-	0xbb016792,
+	0xb608c321,
+	0x11f40464,
+	0x0076bb0f,
+	0xf40136b0,
+	0x32f4061b,
+/* 0x09b5: i2c_put_byte_done */
+/* 0x09b7: i2c_addr */
+	0xbb00f801,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0a0f21f5,
-	0xfc0464b6,
-	0x00d6b0d0,
-	0x00b31bf5,
-	0xbb0057f0,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x09b921f5,
-	0xf50464b6,
-	0xc700d011,
-	0x76bbe0c5,
-	0x0465b600,
-	0x659450f9,
-	0x0256bb04,
-	0x75fd50bd,
-	0xf550fc04,
-	0xb6095e21,
-	0x11f50464,
-	0x57f000ad,
+	0x07f321f5,
+	0xf40464b6,
+	0xc3e72911,
+	0x34b6012e,
+	0x0553fd01,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x5c21f550,
+	0x0464b609,
+/* 0x09fc: i2c_addr_done */
+/* 0x09fe: i2c_acquire_addr */
+	0xcec700f8,
+	0x02e4b6f8,
+	0x0d1ce0b7,
+	0xf800ee98,
+/* 0x0a0d: i2c_acquire */
+	0xfe21f500,
+	0x0421f409,
+	0xf403d9f0,
+	0x00f84021,
+/* 0x0a1c: i2c_release */
+	0x09fe21f5,
+	0xf00421f4,
+	0x21f403da,
+/* 0x0a2b: i2c_recv */
+	0xf400f840,
+	0xc1c70132,
+	0x0214b6f8,
+	0xf52816b0,
+	0xa0013a1f,
+	0x980cf413,
+	0x13a00032,
+	0x31980ccc,
+	0x0231f400,
+	0xe0f9d0f9,
+	0x67f1d0f9,
+	0x63f10000,
+	0x67921000,
 	0x0076bb01,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b609b9,
-	0x8a11f504,
+	0x64b60a0d,
+	0xb0d0fc04,
+	0x1bf500d6,
+	0x57f000b3,
 	0x0076bb00,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b6090c,
-	0x6a11f404,
-	0xbbe05bcb,
+	0x64b609b7,
+	0xd011f504,
+	0xe0c5c700,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x5c21f550,
+	0x0464b609,
+	0x00ad11f5,
+	0xbb0157f0,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x085121f5,
-	0xb90464b6,
-	0x74bd025b,
-/* 0x0b33: i2c_recv_not_rd08 */
-	0xb0430ef4,
-	0x1bf401d6,
-	0x0057f03d,
-	0x09b921f5,
-	0xc73311f4,
-	0x21f5e0c5,
-	0x11f4095e,
-	0x0057f029,
-	0x09b921f5,
-	0xc71f11f4,
-	0x21f5e0b5,
-	0x11f4095e,
-	0x5121f515,
-	0xc774bd08,
-	0x1bf408c5,
-	0x0232f409,
-/* 0x0b73: i2c_recv_not_wr08 */
-/* 0x0b73: i2c_recv_done */
-	0xc7030ef4,
-	0x21f5f8ce,
-	0xe0fc0a1e,
-	0x12f4d0fc,
-	0x027cb90a,
-	0x033621f5,
-/* 0x0b88: i2c_recv_exit */
-/* 0x0b8a: i2c_init */
-	0x00f800f8,
-/* 0x0b8c: test_recv */
-	0x05d817f1,
+	0x09b721f5,
+	0xf50464b6,
+	0xbb008a11,
+	0x65b60076,
+	0x9450f904,
+	0x56bb0465,
+	0xfd50bd02,
+	0x50fc0475,
+	0x090a21f5,
+	0xf40464b6,
+	0x5bcb6a11,
+	0x0076bbe0,
+	0xf90465b6,
+	0x04659450,
+	0xbd0256bb,
+	0x0475fd50,
+	0x21f550fc,
+	0x64b6084f,
+	0x025bb904,
+	0x0ef474bd,
+/* 0x0b31: i2c_recv_not_rd08 */
+	0x01d6b043,
+	0xf03d1bf4,
+	0x21f50057,
+	0x11f409b7,
+	0xe0c5c733,
+	0x095c21f5,
+	0xf02911f4,
+	0x21f50057,
+	0x11f409b7,
+	0xe0b5c71f,
+	0x095c21f5,
+	0xf51511f4,
+	0xbd084f21,
+	0x08c5c774,
+	0xf4091bf4,
+	0x0ef40232,
+/* 0x0b71: i2c_recv_not_wr08 */
+/* 0x0b71: i2c_recv_done */
+	0xf8cec703,
+	0x0a1c21f5,
+	0xd0fce0fc,
+	0xb90a12f4,
+	0x21f5027c,
+/* 0x0b86: i2c_recv_exit */
+	0x00f80336,
+/* 0x0b88: i2c_init */
+/* 0x0b8a: test_recv */
+	0x17f100f8,
+	0x14b605d8,
+	0x0011cf06,
+	0xf10110b6,
+	0xb605d807,
+	0x01d00604,
+	0xf104bd00,
+	0xf1d900e7,
+	0xf5134fe3,
+	0xf8025621,
+/* 0x0bb1: test_init */
+	0x00e7f100,
+	0x5621f508,
+/* 0x0bbb: idle_recv */
+	0xf800f802,
+/* 0x0bbd: idle */
+	0x0031f400,
+	0x05d417f1,
 	0xcf0614b6,
 	0x10b60011,
-	0xd807f101,
+	0xd407f101,
 	0x0604b605,
 	0xbd0001d0,
-	0x00e7f104,
-	0x4fe3f1d9,
-	0x5621f513,
-/* 0x0bb3: test_init */
-	0xf100f802,
-	0xf50800e7,
-	0xf8025621,
-/* 0x0bbd: idle_recv */
-/* 0x0bbf: idle */
-	0xf400f800,
-	0x17f10031,
-	0x14b605d4,
-	0x0011cf06,
-	0xf10110b6,
-	0xb605d407,
-	0x01d00604,
-/* 0x0bdb: idle_loop */
-	0xf004bd00,
-	0x32f45817,
-/* 0x0be1: idle_proc */
-/* 0x0be1: idle_proc_exec */
-	0xb910f902,
-	0x21f5021e,
-	0x10fc033f,
-	0xf40911f4,
-	0x0ef40231,
-/* 0x0bf5: idle_proc_next */
-	0x5810b6ef,
-	0xf4061fb8,
-	0x02f4e61b,
-	0x0028f4dd,
-	0x00bb0ef4,
+/* 0x0bd9: idle_loop */
+	0x5817f004,
+/* 0x0bdf: idle_proc */
+/* 0x0bdf: idle_proc_exec */
+	0xf90232f4,
+	0x021eb910,
+	0x033f21f5,
+	0x11f410fc,
+	0x0231f409,
+/* 0x0bf3: idle_proc_next */
+	0xb6ef0ef4,
+	0x1fb85810,
+	0xe61bf406,
+	0xf4dd02f4,
+	0x0ef40028,
+	0x000000bb,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
index c4edbc79e41a..e0222cb832fb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h
@@ -47,8 +47,8 @@ static uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x584d454d,
-	0x000005f3,
-	0x000005e5,
+	0x000005ee,
+	0x000005e0,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -69,8 +69,8 @@ static uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x46524550,
-	0x000005f7,
-	0x000005f5,
+	0x000005f2,
+	0x000005f0,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -91,8 +91,8 @@ static uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x5f433249,
-	0x000009f8,
-	0x000008a2,
+	0x000009f3,
+	0x0000089d,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -113,8 +113,8 @@ static uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x54534554,
-	0x00000a16,
-	0x000009fa,
+	0x00000a11,
+	0x000009f5,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -135,8 +135,8 @@ static uint32_t gk208_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x454c4449,
-	0x00000a21,
-	0x00000a1f,
+	0x00000a1c,
+	0x00000a1a,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -234,22 +234,22 @@ static uint32_t gk208_pmu_data[] = {
 /* 0x037c: memx_func_next */
 	0x00000002,
 	0x00000000,
-	0x000004cf,
+	0x000004cc,
 	0x00000003,
 	0x00000002,
-	0x00000546,
+	0x00000541,
 	0x00040004,
 	0x00000000,
-	0x00000563,
+	0x0000055e,
 	0x00010005,
 	0x00000000,
-	0x0000057d,
+	0x00000578,
 	0x00010006,
 	0x00000000,
-	0x00000541,
+	0x0000053c,
 	0x00000007,
 	0x00000000,
-	0x00000589,
+	0x00000584,
 /* 0x03c4: memx_func_tail */
 /* 0x03c4: memx_ts_start */
 	0x00000000,
@@ -1239,454 +1239,454 @@ static uint32_t gk208_pmu_code[] = {
 	0x0001f604,
 	0x00f804bd,
 /* 0x045c: memx_func_enter */
-	0x162067f1,
-	0xf55d77f1,
-	0x047e6eb2,
-	0xd8b20000,
-	0xf90487fd,
-	0xfc80f960,
-	0x7ee0fcd0,
-	0x0700002d,
-	0x7e6eb2fe,
+	0x47162046,
+	0x6eb2f55d,
+	0x0000047e,
+	0x87fdd8b2,
+	0xf960f904,
+	0xfcd0fc80,
+	0x002d7ee0,
+	0xb2fe0700,
+	0x00047e6e,
+	0xfdd8b200,
+	0x60f90487,
+	0xd0fc80f9,
+	0x2d7ee0fc,
+	0xf0460000,
+	0x7e6eb226,
 	0xb2000004,
 	0x0487fdd8,
 	0x80f960f9,
 	0xe0fcd0fc,
 	0x00002d7e,
-	0x26f067f1,
-	0x047e6eb2,
-	0xd8b20000,
-	0xf90487fd,
-	0xfc80f960,
-	0x7ee0fcd0,
-	0x0600002d,
-	0x07e04004,
-	0xbd0006f6,
-/* 0x04b9: memx_func_enter_wait */
-	0x07c04604,
-	0xf00066cf,
-	0x0bf40464,
-	0xcf2c06f7,
-	0x06b50066,
-/* 0x04cf: memx_func_leave */
-	0x0600f8f1,
-	0x0066cf2c,
-	0x06f206b5,
-	0x07e44004,
-	0xbd0006f6,
-/* 0x04e1: memx_func_leave_wait */
-	0x07c04604,
-	0xf00066cf,
-	0x1bf40464,
-	0xf067f1f7,
+	0xe0400406,
+	0x0006f607,
+/* 0x04b6: memx_func_enter_wait */
+	0xc04604bd,
+	0x0066cf07,
+	0xf40464f0,
+	0x2c06f70b,
+	0xb50066cf,
+	0x00f8f106,
+/* 0x04cc: memx_func_leave */
+	0x66cf2c06,
+	0xf206b500,
+	0xe4400406,
+	0x0006f607,
+/* 0x04de: memx_func_leave_wait */
+	0xc04604bd,
+	0x0066cf07,
+	0xf40464f0,
+	0xf046f71b,
 	0xb2010726,
 	0x00047e6e,
 	0xfdd8b200,
 	0x60f90587,
 	0xd0fc80f9,
 	0x2d7ee0fc,
-	0x67f10000,
-	0x6eb21620,
-	0x0000047e,
-	0x87fdd8b2,
-	0xf960f905,
-	0xfcd0fc80,
-	0x002d7ee0,
-	0x0aa24700,
-	0x047e6eb2,
-	0xd8b20000,
-	0xf90587fd,
-	0xfc80f960,
-	0x7ee0fcd0,
-	0xf800002d,
-/* 0x0541: memx_func_wait_vblank */
+	0x20460000,
+	0x7e6eb216,
+	0xb2000004,
+	0x0587fdd8,
+	0x80f960f9,
+	0xe0fcd0fc,
+	0x00002d7e,
+	0xb20aa247,
+	0x00047e6e,
+	0xfdd8b200,
+	0x60f90587,
+	0xd0fc80f9,
+	0x2d7ee0fc,
+	0x00f80000,
+/* 0x053c: memx_func_wait_vblank */
+	0xf80410b6,
+/* 0x0541: memx_func_wr32 */
+	0x00169800,
+	0xb6011598,
+	0x60f90810,
+	0xd0fc50f9,
+	0x2d7ee0fc,
+	0x42b60000,
+	0xe81bf402,
+/* 0x055e: memx_func_wait */
+	0x2c0800f8,
+	0x980088cf,
+	0x1d98001e,
+	0x021c9801,
+	0xb6031b98,
+	0x747e1010,
+	0x00f80000,
+/* 0x0578: memx_func_delay */
+	0xb6001e98,
+	0x587e0410,
+	0x00f80000,
+/* 0x0584: memx_func_train */
+/* 0x0586: memx_exec */
+	0xe0f900f8,
+	0xc1b2d0f9,
+/* 0x058e: memx_exec_next */
+	0x1398b2b2,
 	0x0410b600,
-/* 0x0546: memx_func_wr32 */
-	0x169800f8,
-	0x01159800,
-	0xf90810b6,
-	0xfc50f960,
+	0x01f034e7,
+	0x01e033e7,
+	0xf00132b6,
+	0x35980c30,
+	0xa655f9de,
+	0xe51ef412,
+	0x98f10b98,
+	0xcbbbf20c,
+	0x07c44b02,
+	0xfc00bbcf,
 	0x7ee0fcd0,
-	0xb600002d,
-	0x1bf40242,
-/* 0x0563: memx_func_wait */
-	0x0800f8e8,
-	0x0088cf2c,
-	0x98001e98,
-	0x1c98011d,
-	0x031b9802,
-	0x7e1010b6,
-	0xf8000074,
-/* 0x057d: memx_func_delay */
-	0x001e9800,
-	0x7e0410b6,
-	0xf8000058,
-/* 0x0589: memx_func_train */
-/* 0x058b: memx_exec */
-	0xf900f800,
-	0xb2d0f9e0,
-/* 0x0593: memx_exec_next */
-	0x98b2b2c1,
-	0x10b60013,
-	0xf034e704,
-	0xe033e701,
-	0x0132b601,
-	0x980c30f0,
-	0x55f9de35,
-	0x1ef412a6,
-	0xf10b98e5,
-	0xbbf20c98,
-	0xc44b02cb,
-	0x00bbcf07,
-	0xe0fcd0fc,
-	0x00029f7e,
-/* 0x05ca: memx_info */
-	0xc67000f8,
-	0x0c0bf401,
-/* 0x05d0: memx_info_data */
-	0x4b03cc4c,
-	0x0ef40800,
-/* 0x05d9: memx_info_train */
-	0x0bcc4c09,
-/* 0x05df: memx_info_send */
-	0x7e01004b,
 	0xf800029f,
-/* 0x05e5: memx_recv */
-	0x01d6b000,
-	0xb0a30bf4,
-	0x0bf400d6,
-/* 0x05f3: memx_init */
-	0xf800f8dc,
-/* 0x05f5: perf_recv */
-/* 0x05f7: perf_init */
-	0xf800f800,
-/* 0x05f9: i2c_drive_scl */
-	0x0036b000,
-	0x400d0bf4,
-	0x01f607e0,
-	0xf804bd00,
-/* 0x0609: i2c_drive_scl_lo */
-	0x07e44000,
-	0xbd0001f6,
-/* 0x0613: i2c_drive_sda */
-	0xb000f804,
-	0x0bf40036,
-	0x07e0400d,
-	0xbd0002f6,
-/* 0x0623: i2c_drive_sda_lo */
-	0x4000f804,
-	0x02f607e4,
-	0xf804bd00,
-/* 0x062d: i2c_sense_scl */
-	0x0132f400,
-	0xcf07c443,
-	0x31fd0033,
-	0x060bf404,
-/* 0x063f: i2c_sense_scl_done */
-	0xf80131f4,
-/* 0x0641: i2c_sense_sda */
-	0x0132f400,
-	0xcf07c443,
-	0x32fd0033,
-	0x060bf404,
-/* 0x0653: i2c_sense_sda_done */
-	0xf80131f4,
-/* 0x0655: i2c_raise_scl */
-	0x4440f900,
-	0x01030898,
-	0x0005f97e,
-/* 0x0660: i2c_raise_scl_wait */
-	0x7e03e84e,
-	0x7e000058,
-	0xf400062d,
-	0x42b60901,
-	0xef1bf401,
-/* 0x0674: i2c_raise_scl_done */
-	0x00f840fc,
-/* 0x0678: i2c_start */
-	0x00062d7e,
-	0x7e0d11f4,
-	0xf4000641,
-	0x0ef40611,
-/* 0x0689: i2c_start_rep */
-	0x7e00032e,
-	0x030005f9,
-	0x06137e01,
+/* 0x05c5: memx_info */
+	0x01c67000,
+/* 0x05cb: memx_info_data */
+	0x4c0c0bf4,
+	0x004b03cc,
+	0x090ef408,
+/* 0x05d4: memx_info_train */
+	0x4b0bcc4c,
+/* 0x05da: memx_info_send */
+	0x9f7e0100,
+	0x00f80002,
+/* 0x05e0: memx_recv */
+	0xf401d6b0,
+	0xd6b0a30b,
+	0xdc0bf400,
+/* 0x05ee: memx_init */
+	0x00f800f8,
+/* 0x05f0: perf_recv */
+/* 0x05f2: perf_init */
+	0x00f800f8,
+/* 0x05f4: i2c_drive_scl */
+	0xf40036b0,
+	0xe0400d0b,
+	0x0001f607,
+	0x00f804bd,
+/* 0x0604: i2c_drive_scl_lo */
+	0xf607e440,
+	0x04bd0001,
+/* 0x060e: i2c_drive_sda */
+	0x36b000f8,
+	0x0d0bf400,
+	0xf607e040,
+	0x04bd0002,
+/* 0x061e: i2c_drive_sda_lo */
+	0xe44000f8,
+	0x0002f607,
+	0x00f804bd,
+/* 0x0628: i2c_sense_scl */
+	0x430132f4,
+	0x33cf07c4,
+	0x0431fd00,
+	0xf4060bf4,
+/* 0x063a: i2c_sense_scl_done */
+	0x00f80131,
+/* 0x063c: i2c_sense_sda */
+	0x430132f4,
+	0x33cf07c4,
+	0x0432fd00,
+	0xf4060bf4,
+/* 0x064e: i2c_sense_sda_done */
+	0x00f80131,
+/* 0x0650: i2c_raise_scl */
+	0x984440f9,
+	0x7e010308,
+/* 0x065b: i2c_raise_scl_wait */
+	0x4e0005f4,
+	0x587e03e8,
+	0x287e0000,
+	0x01f40006,
+	0x0142b609,
+/* 0x066f: i2c_raise_scl_done */
+	0xfcef1bf4,
+/* 0x0673: i2c_start */
+	0x7e00f840,
+	0xf4000628,
+	0x3c7e0d11,
+	0x11f40006,
+	0x2e0ef406,
+/* 0x0684: i2c_start_rep */
+	0xf47e0003,
+	0x01030005,
+	0x00060e7e,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x06507e50,
+	0x0464b600,
+/* 0x06af: i2c_start_send */
+	0x031d11f4,
+	0x060e7e00,
+	0x13884e00,
+	0x0000587e,
+	0xf47e0003,
+	0x884e0005,
+	0x00587e13,
+/* 0x06c9: i2c_start_out */
+/* 0x06cb: i2c_stop */
+	0x0300f800,
+	0x05f47e00,
+	0x7e000300,
+	0x4e00060e,
+	0x587e03e8,
+	0x01030000,
+	0x0005f47e,
+	0x7e13884e,
+	0x03000058,
+	0x060e7e01,
+	0x13884e00,
+	0x0000587e,
+/* 0x06fa: i2c_bitw */
+	0x0e7e00f8,
+	0xe84e0006,
+	0x00587e03,
 	0x0076bb00,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
-	0x557e50fc,
+	0x507e50fc,
 	0x64b60006,
-	0x1d11f404,
-/* 0x06b4: i2c_start_send */
-	0x137e0003,
-	0x884e0006,
-	0x00587e13,
-	0x7e000300,
-	0x4e0005f9,
-	0x587e1388,
-/* 0x06ce: i2c_start_out */
-	0x00f80000,
-/* 0x06d0: i2c_stop */
-	0xf97e0003,
-	0x00030005,
-	0x0006137e,
-	0x7e03e84e,
+	0x1711f404,
+	0x7e13884e,
 	0x03000058,
-	0x05f97e01,
+	0x05f47e00,
 	0x13884e00,
 	0x0000587e,
-	0x137e0103,
-	0x884e0006,
-	0x00587e13,
-/* 0x06ff: i2c_bitw */
-	0x7e00f800,
-	0x4e000613,
-	0x587e03e8,
-	0x76bb0000,
+/* 0x0738: i2c_bitw_out */
+/* 0x073a: i2c_bitr */
+	0x010300f8,
+	0x00060e7e,
+	0x7e03e84e,
+	0xbb000058,
+	0x65b60076,
+	0x9450f904,
+	0x56bb0465,
+	0xfd50bd02,
+	0x50fc0475,
+	0x0006507e,
+	0xf40464b6,
+	0x3c7e1a11,
+	0x00030006,
+	0x0005f47e,
+	0x7e13884e,
+	0xf0000058,
+	0x31f4013c,
+/* 0x077d: i2c_bitr_done */
+/* 0x077f: i2c_get_byte */
+	0x0500f801,
+/* 0x0783: i2c_get_byte_next */
+	0xb6080400,
+	0x76bb0154,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0x7e50fc04,
-	0xb6000655,
+	0xb600073a,
 	0x11f40464,
-	0x13884e17,
-	0x0000587e,
-	0xf97e0003,
-	0x884e0005,
-	0x00587e13,
-/* 0x073d: i2c_bitw_out */
-/* 0x073f: i2c_bitr */
-	0x0300f800,
-	0x06137e01,
-	0x03e84e00,
-	0x0000587e,
+	0x0553fd2a,
+	0xf40142b6,
+	0x0103d81b,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x06557e50,
+	0x06fa7e50,
 	0x0464b600,
-	0x7e1a11f4,
-	0x03000641,
-	0x05f97e00,
-	0x13884e00,
-	0x0000587e,
-	0xf4013cf0,
-/* 0x0782: i2c_bitr_done */
-	0x00f80131,
-/* 0x0784: i2c_get_byte */
-	0x08040005,
-/* 0x0788: i2c_get_byte_next */
-	0xbb0154b6,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x00073f7e,
-	0xf40464b6,
-	0x53fd2a11,
-	0x0142b605,
-	0x03d81bf4,
-	0x0076bb01,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0xff7e50fc,
-	0x64b60006,
-/* 0x07d1: i2c_get_byte_done */
-/* 0x07d3: i2c_put_byte */
-	0x0400f804,
-/* 0x07d5: i2c_put_byte_next */
-	0x0142b608,
-	0xbb3854ff,
+/* 0x07cc: i2c_get_byte_done */
+/* 0x07ce: i2c_put_byte */
+	0x080400f8,
+/* 0x07d0: i2c_put_byte_next */
+	0xff0142b6,
+	0x76bb3854,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0x7e50fc04,
+	0xb60006fa,
+	0x11f40464,
+	0x0046b034,
+	0xbbd81bf4,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0006ff7e,
+	0x00073a7e,
 	0xf40464b6,
-	0x46b03411,
-	0xd81bf400,
+	0x76bb0f11,
+	0x0136b000,
+	0xf4061bf4,
+/* 0x0826: i2c_put_byte_done */
+	0x00f80132,
+/* 0x0828: i2c_addr */
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0x073f7e50,
+	0x06737e50,
 	0x0464b600,
-	0xbb0f11f4,
-	0x36b00076,
-	0x061bf401,
-/* 0x082b: i2c_put_byte_done */
-	0xf80132f4,
-/* 0x082d: i2c_addr */
-	0x0076bb00,
+	0xe72911f4,
+	0xb6012ec3,
+	0x53fd0134,
+	0x0076bb05,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
-	0x787e50fc,
-	0x64b60006,
-	0x2911f404,
-	0x012ec3e7,
-	0xfd0134b6,
-	0x76bb0553,
-	0x0465b600,
-	0x659450f9,
-	0x0256bb04,
-	0x75fd50bd,
-	0x7e50fc04,
-	0xb60007d3,
-/* 0x0872: i2c_addr_done */
-	0x00f80464,
-/* 0x0874: i2c_acquire_addr */
-	0xb6f8cec7,
-	0xe0b705e4,
-	0x00f8d014,
-/* 0x0880: i2c_acquire */
-	0x0008747e,
+	0xce7e50fc,
+	0x64b60007,
+/* 0x086d: i2c_addr_done */
+/* 0x086f: i2c_acquire_addr */
+	0xc700f804,
+	0xe4b6f8ce,
+	0x14e0b705,
+/* 0x087b: i2c_acquire */
+	0x7e00f8d0,
+	0x7e00086f,
+	0xf0000004,
+	0x2d7e03d9,
+	0x00f80000,
+/* 0x088c: i2c_release */
+	0x00086f7e,
 	0x0000047e,
-	0x7e03d9f0,
+	0x7e03daf0,
 	0xf800002d,
-/* 0x0891: i2c_release */
-	0x08747e00,
-	0x00047e00,
-	0x03daf000,
-	0x00002d7e,
-/* 0x08a2: i2c_recv */
-	0x32f400f8,
-	0xf8c1c701,
-	0xb00214b6,
-	0x1ff52816,
-	0x13b80134,
-	0x98000cf4,
-	0x13b80032,
-	0x98000ccc,
-	0x31f40031,
-	0xf9d0f902,
-	0xd6d0f9e0,
-	0x10000000,
-	0xbb016792,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x0008807e,
-	0xfc0464b6,
-	0x00d6b0d0,
-	0x00b01bf5,
-	0x76bb0005,
+/* 0x089d: i2c_recv */
+	0x0132f400,
+	0xb6f8c1c7,
+	0x16b00214,
+	0x341ff528,
+	0xf413b801,
+	0x3298000c,
+	0xcc13b800,
+	0x3198000c,
+	0x0231f400,
+	0xe0f9d0f9,
+	0x00d6d0f9,
+	0x92100000,
+	0x76bb0167,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0x7e50fc04,
-	0xb600082d,
-	0x11f50464,
-	0xc5c700cc,
-	0x0076bbe0,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0xd37e50fc,
-	0x64b60007,
-	0xa911f504,
-	0xbb010500,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x00082d7e,
-	0xf50464b6,
-	0xbb008711,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x0007847e,
-	0xf40464b6,
-	0x5bcb6711,
-	0x0076bbe0,
+	0xb600087b,
+	0xd0fc0464,
+	0xf500d6b0,
+	0x0500b01b,
+	0x0076bb00,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
-	0xd07e50fc,
-	0x64b60006,
-	0xbd5bb204,
-	0x410ef474,
-/* 0x09a4: i2c_recv_not_rd08 */
-	0xf401d6b0,
-	0x00053b1b,
-	0x00082d7e,
-	0xc73211f4,
-	0xd37ee0c5,
-	0x11f40007,
-	0x7e000528,
-	0xf400082d,
-	0xb5c71f11,
-	0x07d37ee0,
-	0x1511f400,
-	0x0006d07e,
-	0xc5c774bd,
-	0x091bf408,
-	0xf40232f4,
-/* 0x09e2: i2c_recv_not_wr08 */
-/* 0x09e2: i2c_recv_done */
-	0xcec7030e,
-	0x08917ef8,
-	0xfce0fc00,
-	0x0912f4d0,
-	0x9f7e7cb2,
-/* 0x09f6: i2c_recv_exit */
-	0x00f80002,
-/* 0x09f8: i2c_init */
-/* 0x09fa: test_recv */
-	0x584100f8,
-	0x0011cf04,
-	0x400110b6,
-	0x01f60458,
-	0xde04bd00,
-	0x134fd900,
-	0x0001de7e,
-/* 0x0a16: test_init */
-	0x004e00f8,
-	0x01de7e08,
-/* 0x0a1f: idle_recv */
+	0x287e50fc,
+	0x64b60008,
+	0xcc11f504,
+	0xe0c5c700,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x07ce7e50,
+	0x0464b600,
+	0x00a911f5,
+	0x76bb0105,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0x7e50fc04,
+	0xb6000828,
+	0x11f50464,
+	0x76bb0087,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0x7e50fc04,
+	0xb600077f,
+	0x11f40464,
+	0xe05bcb67,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x06cb7e50,
+	0x0464b600,
+	0x74bd5bb2,
+/* 0x099f: i2c_recv_not_rd08 */
+	0xb0410ef4,
+	0x1bf401d6,
+	0x7e00053b,
+	0xf4000828,
+	0xc5c73211,
+	0x07ce7ee0,
+	0x2811f400,
+	0x287e0005,
+	0x11f40008,
+	0xe0b5c71f,
+	0x0007ce7e,
+	0x7e1511f4,
+	0xbd0006cb,
+	0x08c5c774,
+	0xf4091bf4,
+	0x0ef40232,
+/* 0x09dd: i2c_recv_not_wr08 */
+/* 0x09dd: i2c_recv_done */
+	0xf8cec703,
+	0x00088c7e,
+	0xd0fce0fc,
+	0xb20912f4,
+	0x029f7e7c,
+/* 0x09f1: i2c_recv_exit */
+/* 0x09f3: i2c_init */
 	0xf800f800,
-/* 0x0a21: idle */
-	0x0031f400,
-	0xcf045441,
-	0x10b60011,
-	0x04544001,
-	0xbd0001f6,
-/* 0x0a35: idle_loop */
-	0xf4580104,
-/* 0x0a3a: idle_proc */
-/* 0x0a3a: idle_proc_exec */
-	0x10f90232,
-	0xa87e1eb2,
-	0x10fc0002,
-	0xf40911f4,
-	0x0ef40231,
-/* 0x0a4d: idle_proc_next */
-	0x5810b6f0,
-	0x1bf41fa6,
-	0xe002f4e8,
-	0xf40028f4,
-	0x0000c60e,
+/* 0x09f5: test_recv */
+	0x04584100,
+	0xb60011cf,
+	0x58400110,
+	0x0001f604,
+	0x00de04bd,
+	0x7e134fd9,
+	0xf80001de,
+/* 0x0a11: test_init */
+	0x08004e00,
+	0x0001de7e,
+/* 0x0a1a: idle_recv */
+	0x00f800f8,
+/* 0x0a1c: idle */
+	0x410031f4,
+	0x11cf0454,
+	0x0110b600,
+	0xf6045440,
+	0x04bd0001,
+/* 0x0a30: idle_loop */
+	0x32f45801,
+/* 0x0a35: idle_proc */
+/* 0x0a35: idle_proc_exec */
+	0xb210f902,
+	0x02a87e1e,
+	0xf410fc00,
+	0x31f40911,
+	0xf00ef402,
+/* 0x0a48: idle_proc_next */
+	0xa65810b6,
+	0xe81bf41f,
+	0xf4e002f4,
+	0x0ef40028,
+	0x000000c6,
+	0x00000000,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
index 6a2572e8945a..defddf5957ee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h
@@ -47,8 +47,8 @@ static uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x584d454d,
-	0x0000083a,
-	0x0000082c,
+	0x00000833,
+	0x00000825,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -69,8 +69,8 @@ static uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x46524550,
-	0x0000083e,
-	0x0000083c,
+	0x00000837,
+	0x00000835,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -91,8 +91,8 @@ static uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x5f433249,
-	0x00000c6e,
-	0x00000b11,
+	0x00000c67,
+	0x00000b0a,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -113,8 +113,8 @@ static uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x54534554,
-	0x00000c97,
-	0x00000c70,
+	0x00000c90,
+	0x00000c69,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -135,8 +135,8 @@ static uint32_t gt215_pmu_data[] = {
 	0x00000000,
 	0x00000000,
 	0x454c4449,
-	0x00000ca3,
-	0x00000ca1,
+	0x00000c9c,
+	0x00000c9a,
 	0x00000000,
 	0x00000000,
 	0x00000000,
@@ -234,22 +234,22 @@ static uint32_t gt215_pmu_data[] = {
 /* 0x037c: memx_func_next */
 	0x00000002,
 	0x00000000,
-	0x000005a0,
+	0x0000059f,
 	0x00000003,
 	0x00000002,
-	0x00000632,
+	0x0000062f,
 	0x00040004,
 	0x00000000,
-	0x0000064e,
+	0x0000064b,
 	0x00010005,
 	0x00000000,
-	0x0000066b,
+	0x00000668,
 	0x00010006,
 	0x00000000,
-	0x000005f0,
+	0x000005ef,
 	0x00000007,
 	0x00000000,
-	0x00000676,
+	0x00000673,
 /* 0x03c4: memx_func_tail */
 /* 0x03c4: memx_ts_start */
 	0x00000000,
@@ -1305,560 +1305,560 @@ static uint32_t gt215_pmu_code[] = {
 	0x67f102d7,
 	0x63f1fffc,
 	0x76fdffff,
-	0x0267f104,
-	0x0576fd00,
-	0x70f980f9,
-	0xe0fcd0fc,
-	0xf04021f4,
+	0x0267f004,
+	0xf90576fd,
+	0xfc70f980,
+	0xf4e0fcd0,
+	0x67f04021,
+	0xe007f104,
+	0x0604b607,
+	0xbd0006d0,
+/* 0x0581: memx_func_enter_wait */
+	0xc067f104,
+	0x0664b607,
+	0xf00066cf,
+	0x0bf40464,
+	0x2c67f0f3,
+	0xcf0664b6,
+	0x06800066,
+/* 0x059f: memx_func_leave */
+	0xf000f8f1,
+	0x64b62c67,
+	0x0066cf06,
+	0xf0f20680,
 	0x07f10467,
-	0x04b607e0,
+	0x04b607e4,
 	0x0006d006,
-/* 0x0582: memx_func_enter_wait */
+/* 0x05ba: memx_func_leave_wait */
 	0x67f104bd,
 	0x64b607c0,
 	0x0066cf06,
 	0xf40464f0,
-	0x67f0f30b,
-	0x0664b62c,
-	0x800066cf,
-	0x00f8f106,
-/* 0x05a0: memx_func_leave */
-	0xb62c67f0,
-	0x66cf0664,
-	0xf2068000,
-	0xf10467f0,
-	0xb607e407,
-	0x06d00604,
-/* 0x05bb: memx_func_leave_wait */
-	0xf104bd00,
-	0xb607c067,
-	0x66cf0664,
-	0x0464f000,
-	0xf1f31bf4,
-	0xb9161087,
-	0x21f4028e,
-	0x02d7b904,
-	0xffcc67f1,
-	0xffff63f1,
-	0xf90476fd,
-	0xfc70f980,
-	0xf4e0fcd0,
-	0x00f84021,
-/* 0x05f0: memx_func_wait_vblank */
-	0xb0001698,
-	0x0bf40066,
-	0x0166b013,
-	0xf4060bf4,
-/* 0x0602: memx_func_wait_vblank_head1 */
-	0x77f12e0e,
-	0x0ef40020,
-/* 0x0609: memx_func_wait_vblank_head0 */
-	0x0877f107,
-/* 0x060d: memx_func_wait_vblank_0 */
-	0xc467f100,
-	0x0664b607,
-	0xfd0066cf,
-	0x1bf40467,
-/* 0x061d: memx_func_wait_vblank_1 */
-	0xc467f1f3,
-	0x0664b607,
-	0xfd0066cf,
-	0x0bf40467,
-/* 0x062d: memx_func_wait_vblank_fini */
-	0x0410b6f3,
-/* 0x0632: memx_func_wr32 */
-	0x169800f8,
-	0x01159800,
-	0xf90810b6,
-	0xfc50f960,
-	0xf4e0fcd0,
-	0x42b64021,
-	0xe91bf402,
-/* 0x064e: memx_func_wait */
-	0x87f000f8,
-	0x0684b62c,
-	0x980088cf,
-	0x1d98001e,
-	0x021c9801,
-	0xb6031b98,
-	0x21f41010,
-/* 0x066b: memx_func_delay */
-	0x9800f8a3,
-	0x10b6001e,
-	0x7e21f404,
-/* 0x0676: memx_func_train */
-	0x57f100f8,
-	0x77f10003,
-	0x97f10000,
-	0x93f00000,
-	0x029eb970,
-	0xb90421f4,
-	0xe7f102d8,
-	0x21f42710,
-/* 0x0695: memx_func_train_loop_outer */
-	0x0158e07e,
-	0x0083f101,
-	0xe097f102,
-	0x1193f011,
-	0x80f990f9,
+	0x87f1f31b,
+	0x8eb91610,
+	0x0421f402,
+	0xf102d7b9,
+	0xf1ffcc67,
+	0xfdffff63,
+	0x80f90476,
+	0xd0fc70f9,
+	0x21f4e0fc,
+/* 0x05ef: memx_func_wait_vblank */
+	0x9800f840,
+	0x66b00016,
+	0x120bf400,
+	0xf40166b0,
+	0x0ef4060b,
+/* 0x0601: memx_func_wait_vblank_head1 */
+	0x2077f02c,
+/* 0x0607: memx_func_wait_vblank_head0 */
+	0xf0060ef4,
+/* 0x060a: memx_func_wait_vblank_0 */
+	0x67f10877,
+	0x64b607c4,
+	0x0066cf06,
+	0xf40467fd,
+/* 0x061a: memx_func_wait_vblank_1 */
+	0x67f1f31b,
+	0x64b607c4,
+	0x0066cf06,
+	0xf40467fd,
+/* 0x062a: memx_func_wait_vblank_fini */
+	0x10b6f30b,
+/* 0x062f: memx_func_wr32 */
+	0x9800f804,
+	0x15980016,
+	0x0810b601,
+	0x50f960f9,
 	0xe0fcd0fc,
-	0xf94021f4,
-	0x0067f150,
-/* 0x06b5: memx_func_train_loop_inner */
-	0x1187f100,
-	0x9068ff11,
-	0xfd109894,
-	0x97f10589,
-	0x93f00720,
-	0xf990f910,
-	0xfcd0fc80,
-	0x4021f4e0,
-	0x008097f1,
-	0xb91093f0,
-	0x21f4029e,
-	0x02d8b904,
-	0xf92088c5,
+	0xb64021f4,
+	0x1bf40242,
+/* 0x064b: memx_func_wait */
+	0xf000f8e9,
+	0x84b62c87,
+	0x0088cf06,
+	0x98001e98,
+	0x1c98011d,
+	0x031b9802,
+	0xf41010b6,
+	0x00f8a321,
+/* 0x0668: memx_func_delay */
+	0xb6001e98,
+	0x21f40410,
+/* 0x0673: memx_func_train */
+	0xf000f87e,
+	0x77f00357,
+	0x0097f100,
+	0x7093f000,
+	0xf4029eb9,
+	0xd8b90421,
+	0x10e7f102,
+	0x7e21f427,
+/* 0x0690: memx_func_train_loop_outer */
+	0x010158e0,
+	0x020083f1,
+	0x11e097f1,
+	0xf91193f0,
+	0xfc80f990,
+	0xf4e0fcd0,
+	0x50f94021,
+/* 0x06af: memx_func_train_loop_inner */
+	0xf10067f0,
+	0xff111187,
+	0x98949068,
+	0x0589fd10,
+	0x072097f1,
+	0xf91093f0,
 	0xfc80f990,
 	0xf4e0fcd0,
 	0x97f14021,
-	0x93f0053c,
-	0x0287f110,
-	0x0083f130,
-	0xf990f980,
+	0x93f00080,
+	0x029eb910,
+	0xb90421f4,
+	0x88c502d8,
+	0xf990f920,
 	0xfcd0fc80,
 	0x4021f4e0,
-	0x0560e7f1,
-	0xf110e3f0,
-	0xf10000d7,
-	0x908000d3,
-	0xb7f100dc,
-	0xb3f08480,
-	0xa321f41e,
-	0x000057f1,
-	0xffff97f1,
-	0x830093f1,
-/* 0x0734: memx_func_train_loop_4x */
-	0x0080a7f1,
-	0xb910a3f0,
-	0x21f402ae,
-	0x02d8b904,
-	0xffdfb7f1,
-	0xffffb3f1,
-	0xf9048bfd,
-	0xfc80f9a0,
+	0x053c97f1,
+	0xf11093f0,
+	0xf1300287,
+	0xf9800083,
+	0xfc80f990,
 	0xf4e0fcd0,
-	0xa7f14021,
-	0xa3f0053c,
-	0x0287f110,
-	0x0083f130,
-	0xf9a0f980,
-	0xfcd0fc80,
-	0x4021f4e0,
-	0x0560e7f1,
-	0xf110e3f0,
-	0xf10000d7,
-	0xb98000d3,
-	0xb7f102dc,
-	0xb3f02710,
-	0xa321f400,
-	0xf402eeb9,
-	0xddb90421,
-	0x949dff02,
+	0xe7f14021,
+	0xe3f00560,
+	0x00d7f110,
+	0x00d3f100,
+	0x00dc9080,
+	0x8480b7f1,
+	0xf41eb3f0,
+	0x57f0a321,
+	0xff97f100,
+	0x0093f1ff,
+/* 0x072d: memx_func_train_loop_4x */
+	0x80a7f183,
+	0x10a3f000,
+	0xf402aeb9,
+	0xd8b90421,
+	0xdfb7f102,
+	0xffb3f1ff,
+	0x048bfdff,
+	0x80f9a0f9,
+	0xe0fcd0fc,
+	0xf14021f4,
+	0xf0053ca7,
+	0x87f110a3,
+	0x83f13002,
+	0xa0f98000,
+	0xd0fc80f9,
+	0x21f4e0fc,
+	0x60e7f140,
+	0x10e3f005,
+	0x0000d7f1,
+	0x8000d3f1,
+	0xf102dcb9,
+	0xf02710b7,
+	0x21f400b3,
+	0x02eeb9a3,
+	0xb90421f4,
+	0x9dff02dd,
+	0x0150b694,
+	0xf4045670,
+	0x7aa0921e,
+	0xa9800bcc,
+	0x0160b600,
+	0x700470b6,
+	0x1ef51066,
+	0x50fcff01,
 	0x700150b6,
-	0x1ef40456,
-	0xcc7aa092,
-	0x00a9800b,
-	0xb60160b6,
-	0x66700470,
-	0x001ef510,
-	0xb650fcff,
-	0x56700150,
-	0xd41ef507,
-/* 0x07c7: memx_exec */
-	0xf900f8fe,
-	0xb9d0f9e0,
-	0xb2b902c1,
-/* 0x07d1: memx_exec_next */
-	0x00139802,
-	0xe70410b6,
-	0xe701f034,
-	0xb601e033,
-	0x30f00132,
-	0xde35980c,
-	0x12b855f9,
-	0xe41ef406,
-	0x98f10b98,
-	0xcbbbf20c,
-	0xc4b7f102,
-	0x06b4b607,
-	0xfc00bbcf,
-	0xf5e0fcd0,
+	0x1ef50756,
+	0x00f8fed6,
+/* 0x07c0: memx_exec */
+	0xd0f9e0f9,
+	0xb902c1b9,
+/* 0x07ca: memx_exec_next */
+	0x139802b2,
+	0x0410b600,
+	0x01f034e7,
+	0x01e033e7,
+	0xf00132b6,
+	0x35980c30,
+	0xb855f9de,
+	0x1ef40612,
+	0xf10b98e4,
+	0xbbf20c98,
+	0xb7f102cb,
+	0xb4b607c4,
+	0x00bbcf06,
+	0xe0fcd0fc,
+	0x033621f5,
+/* 0x0806: memx_info */
+	0xc67000f8,
+	0x0e0bf401,
+/* 0x080c: memx_info_data */
+	0x03ccc7f1,
+	0x0800b7f1,
+/* 0x0817: memx_info_train */
+	0xf10b0ef4,
+	0xf10bccc7,
+/* 0x081f: memx_info_send */
+	0xf50100b7,
 	0xf8033621,
-/* 0x080d: memx_info */
-	0x01c67000,
-/* 0x0813: memx_info_data */
-	0xf10e0bf4,
-	0xf103ccc7,
-	0xf40800b7,
-/* 0x081e: memx_info_train */
-	0xc7f10b0e,
-	0xb7f10bcc,
-/* 0x0826: memx_info_send */
-	0x21f50100,
-	0x00f80336,
-/* 0x082c: memx_recv */
-	0xf401d6b0,
-	0xd6b0980b,
-	0xd80bf400,
-/* 0x083a: memx_init */
-	0x00f800f8,
-/* 0x083c: perf_recv */
-/* 0x083e: perf_init */
-	0x00f800f8,
-/* 0x0840: i2c_drive_scl */
-	0xf40036b0,
-	0x07f1110b,
-	0x04b607e0,
-	0x0001d006,
-	0x00f804bd,
-/* 0x0854: i2c_drive_scl_lo */
-	0x07e407f1,
-	0xd00604b6,
-	0x04bd0001,
-/* 0x0862: i2c_drive_sda */
-	0x36b000f8,
-	0x110bf400,
-	0x07e007f1,
-	0xd00604b6,
-	0x04bd0002,
-/* 0x0876: i2c_drive_sda_lo */
-	0x07f100f8,
-	0x04b607e4,
-	0x0002d006,
-	0x00f804bd,
-/* 0x0884: i2c_sense_scl */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0431fd00,
-	0xf4060bf4,
-/* 0x089a: i2c_sense_scl_done */
-	0x00f80131,
-/* 0x089c: i2c_sense_sda */
-	0xf10132f4,
-	0xb607c437,
-	0x33cf0634,
-	0x0432fd00,
-	0xf4060bf4,
-/* 0x08b2: i2c_sense_sda_done */
-	0x00f80131,
-/* 0x08b4: i2c_raise_scl */
-	0x47f140f9,
-	0x37f00898,
-	0x4021f501,
-/* 0x08c1: i2c_raise_scl_wait */
+/* 0x0825: memx_recv */
+	0x01d6b000,
+	0xb0980bf4,
+	0x0bf400d6,
+/* 0x0833: memx_init */
+	0xf800f8d8,
+/* 0x0835: perf_recv */
+/* 0x0837: perf_init */
+	0xf800f800,
+/* 0x0839: i2c_drive_scl */
+	0x0036b000,
+	0xf1110bf4,
+	0xb607e007,
+	0x01d00604,
+	0xf804bd00,
+/* 0x084d: i2c_drive_scl_lo */
+	0xe407f100,
+	0x0604b607,
+	0xbd0001d0,
+/* 0x085b: i2c_drive_sda */
+	0xb000f804,
+	0x0bf40036,
+	0xe007f111,
+	0x0604b607,
+	0xbd0002d0,
+/* 0x086f: i2c_drive_sda_lo */
+	0xf100f804,
+	0xb607e407,
+	0x02d00604,
+	0xf804bd00,
+/* 0x087d: i2c_sense_scl */
+	0x0132f400,
+	0x07c437f1,
+	0xcf0634b6,
+	0x31fd0033,
+	0x060bf404,
+/* 0x0893: i2c_sense_scl_done */
+	0xf80131f4,
+/* 0x0895: i2c_sense_sda */
+	0x0132f400,
+	0x07c437f1,
+	0xcf0634b6,
+	0x32fd0033,
+	0x060bf404,
+/* 0x08ab: i2c_sense_sda_done */
+	0xf80131f4,
+/* 0x08ad: i2c_raise_scl */
+	0xf140f900,
+	0xf0089847,
+	0x21f50137,
+/* 0x08ba: i2c_raise_scl_wait */
+	0xe7f10839,
+	0x21f403e8,
+	0x7d21f57e,
+	0x0901f408,
+	0xf40142b6,
+/* 0x08ce: i2c_raise_scl_done */
+	0x40fcef1b,
+/* 0x08d2: i2c_start */
+	0x21f500f8,
+	0x11f4087d,
+	0x9521f50d,
+	0x0611f408,
+/* 0x08e3: i2c_start_rep */
+	0xf0300ef4,
+	0x21f50037,
+	0x37f00839,
+	0x5b21f501,
+	0x0076bb08,
+	0xf90465b6,
+	0x04659450,
+	0xbd0256bb,
+	0x0475fd50,
+	0x21f550fc,
+	0x64b608ad,
+	0x1f11f404,
+/* 0x0910: i2c_start_send */
+	0xf50037f0,
+	0xf1085b21,
+	0xf41388e7,
+	0x37f07e21,
+	0x3921f500,
+	0x88e7f108,
+	0x7e21f413,
+/* 0x092c: i2c_start_out */
+/* 0x092e: i2c_stop */
+	0x37f000f8,
+	0x3921f500,
+	0x0037f008,
+	0x085b21f5,
+	0x03e8e7f1,
+	0xf07e21f4,
+	0x21f50137,
+	0xe7f10839,
+	0x21f41388,
+	0x0137f07e,
+	0x085b21f5,
+	0x1388e7f1,
+	0xf87e21f4,
+/* 0x0961: i2c_bitw */
+	0x5b21f500,
 	0xe8e7f108,
 	0x7e21f403,
-	0x088421f5,
-	0xb60901f4,
-	0x1bf40142,
-/* 0x08d5: i2c_raise_scl_done */
-	0xf840fcef,
-/* 0x08d9: i2c_start */
-	0x8421f500,
-	0x0d11f408,
-	0x089c21f5,
-	0xf40611f4,
-/* 0x08ea: i2c_start_rep */
-	0x37f0300e,
-	0x4021f500,
-	0x0137f008,
-	0x086221f5,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0xb421f550,
+	0xad21f550,
 	0x0464b608,
-/* 0x0917: i2c_start_send */
-	0xf01f11f4,
-	0x21f50037,
-	0xe7f10862,
-	0x21f41388,
-	0x0037f07e,
-	0x084021f5,
-	0x1388e7f1,
-/* 0x0933: i2c_start_out */
-	0xf87e21f4,
-/* 0x0935: i2c_stop */
-	0x0037f000,
-	0x084021f5,
-	0xf50037f0,
-	0xf1086221,
-	0xf403e8e7,
+	0xf11811f4,
+	0xf41388e7,
 	0x37f07e21,
-	0x4021f501,
+	0x3921f500,
 	0x88e7f108,
 	0x7e21f413,
-	0xf50137f0,
-	0xf1086221,
-	0xf41388e7,
-	0x00f87e21,
-/* 0x0968: i2c_bitw */
-	0x086221f5,
-	0x03e8e7f1,
-	0xbb7e21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x08b421f5,
-	0xf40464b6,
-	0xe7f11811,
+/* 0x09a0: i2c_bitw_out */
+/* 0x09a2: i2c_bitr */
+	0x37f000f8,
+	0x5b21f501,
+	0xe8e7f108,
+	0x7e21f403,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xad21f550,
+	0x0464b608,
+	0xf51b11f4,
+	0xf0089521,
+	0x21f50037,
+	0xe7f10839,
 	0x21f41388,
-	0x0037f07e,
-	0x084021f5,
-	0x1388e7f1,
-/* 0x09a7: i2c_bitw_out */
-	0xf87e21f4,
-/* 0x09a9: i2c_bitr */
-	0x0137f000,
-	0x086221f5,
-	0x03e8e7f1,
-	0xbb7e21f4,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x08b421f5,
-	0xf40464b6,
-	0x21f51b11,
-	0x37f0089c,
-	0x4021f500,
-	0x88e7f108,
-	0x7e21f413,
-	0xf4013cf0,
-/* 0x09ee: i2c_bitr_done */
-	0x00f80131,
-/* 0x09f0: i2c_get_byte */
-	0xf00057f0,
-/* 0x09f6: i2c_get_byte_next */
-	0x54b60847,
+	0x013cf07e,
+/* 0x09e7: i2c_bitr_done */
+	0xf80131f4,
+/* 0x09e9: i2c_get_byte */
+	0x0057f000,
+/* 0x09ef: i2c_get_byte_next */
+	0xb60847f0,
+	0x76bb0154,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb609a221,
+	0x11f40464,
+	0x0553fd2b,
+	0xf40142b6,
+	0x37f0d81b,
 	0x0076bb01,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b609a9,
-	0x2b11f404,
-	0xb60553fd,
-	0x1bf40142,
-	0x0137f0d8,
-	0xb60076bb,
-	0x50f90465,
-	0xbb046594,
-	0x50bd0256,
-	0xfc0475fd,
-	0x6821f550,
-	0x0464b609,
-/* 0x0a40: i2c_get_byte_done */
-/* 0x0a42: i2c_put_byte */
-	0x47f000f8,
-/* 0x0a45: i2c_put_byte_next */
-	0x0142b608,
-	0xbb3854ff,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x096821f5,
-	0xf40464b6,
-	0x46b03411,
-	0xd81bf400,
+	0x64b60961,
+/* 0x0a39: i2c_get_byte_done */
+/* 0x0a3b: i2c_put_byte */
+	0xf000f804,
+/* 0x0a3e: i2c_put_byte_next */
+	0x42b60847,
+	0x3854ff01,
 	0xb60076bb,
 	0x50f90465,
 	0xbb046594,
 	0x50bd0256,
 	0xfc0475fd,
-	0xa921f550,
+	0x6121f550,
 	0x0464b609,
-	0xbb0f11f4,
-	0x36b00076,
-	0x061bf401,
-/* 0x0a9b: i2c_put_byte_done */
-	0xf80132f4,
-/* 0x0a9d: i2c_addr */
-	0x0076bb00,
+	0xb03411f4,
+	0x1bf40046,
+	0x0076bbd8,
 	0xf90465b6,
 	0x04659450,
 	0xbd0256bb,
 	0x0475fd50,
 	0x21f550fc,
-	0x64b608d9,
-	0x2911f404,
-	0x012ec3e7,
-	0xfd0134b6,
-	0x76bb0553,
+	0x64b609a2,
+	0x0f11f404,
+	0xb00076bb,
+	0x1bf40136,
+	0x0132f406,
+/* 0x0a94: i2c_put_byte_done */
+/* 0x0a96: i2c_addr */
+	0x76bb00f8,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb60a4221,
-/* 0x0ae2: i2c_addr_done */
-	0x00f80464,
-/* 0x0ae4: i2c_acquire_addr */
-	0xb6f8cec7,
-	0xe0b702e4,
-	0xee980d1c,
-/* 0x0af3: i2c_acquire */
-	0xf500f800,
-	0xf40ae421,
-	0xd9f00421,
-	0x4021f403,
-/* 0x0b02: i2c_release */
-	0x21f500f8,
-	0x21f40ae4,
-	0x03daf004,
-	0xf84021f4,
-/* 0x0b11: i2c_recv */
-	0x0132f400,
-	0xb6f8c1c7,
-	0x16b00214,
-	0x3a1ff528,
-	0xf413a001,
-	0x0032980c,
-	0x0ccc13a0,
-	0xf4003198,
-	0xd0f90231,
-	0xd0f9e0f9,
-	0x000067f1,
-	0x100063f1,
-	0xbb016792,
+	0xb608d221,
+	0x11f40464,
+	0x2ec3e729,
+	0x0134b601,
+	0xbb0553fd,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0af321f5,
-	0xfc0464b6,
-	0x00d6b0d0,
-	0x00b31bf5,
-	0xbb0057f0,
+	0x0a3b21f5,
+/* 0x0adb: i2c_addr_done */
+	0xf80464b6,
+/* 0x0add: i2c_acquire_addr */
+	0xf8cec700,
+	0xb702e4b6,
+	0x980d1ce0,
+	0x00f800ee,
+/* 0x0aec: i2c_acquire */
+	0x0add21f5,
+	0xf00421f4,
+	0x21f403d9,
+/* 0x0afb: i2c_release */
+	0xf500f840,
+	0xf40add21,
+	0xdaf00421,
+	0x4021f403,
+/* 0x0b0a: i2c_recv */
+	0x32f400f8,
+	0xf8c1c701,
+	0xb00214b6,
+	0x1ff52816,
+	0x13a0013a,
+	0x32980cf4,
+	0xcc13a000,
+	0x0031980c,
+	0xf90231f4,
+	0xf9e0f9d0,
+	0x0067f1d0,
+	0x0063f100,
+	0x01679210,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0xec21f550,
+	0x0464b60a,
+	0xd6b0d0fc,
+	0xb31bf500,
+	0x0057f000,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x9621f550,
+	0x0464b60a,
+	0x00d011f5,
+	0xbbe0c5c7,
 	0x65b60076,
 	0x9450f904,
 	0x56bb0465,
 	0xfd50bd02,
 	0x50fc0475,
-	0x0a9d21f5,
+	0x0a3b21f5,
 	0xf50464b6,
-	0xc700d011,
-	0x76bbe0c5,
+	0xf000ad11,
+	0x76bb0157,
 	0x0465b600,
 	0x659450f9,
 	0x0256bb04,
 	0x75fd50bd,
 	0xf550fc04,
-	0xb60a4221,
+	0xb60a9621,
 	0x11f50464,
-	0x57f000ad,
-	0x0076bb01,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b60a9d,
-	0x8a11f504,
-	0x0076bb00,
-	0xf90465b6,
-	0x04659450,
-	0xbd0256bb,
-	0x0475fd50,
-	0x21f550fc,
-	0x64b609f0,
-	0x6a11f404,
-	0xbbe05bcb,
-	0x65b60076,
-	0x9450f904,
-	0x56bb0465,
-	0xfd50bd02,
-	0x50fc0475,
-	0x093521f5,
-	0xb90464b6,
-	0x74bd025b,
-/* 0x0c17: i2c_recv_not_rd08 */
-	0xb0430ef4,
-	0x1bf401d6,
-	0x0057f03d,
-	0x0a9d21f5,
-	0xc73311f4,
-	0x21f5e0c5,
-	0x11f40a42,
-	0x0057f029,
-	0x0a9d21f5,
-	0xc71f11f4,
-	0x21f5e0b5,
-	0x11f40a42,
-	0x3521f515,
-	0xc774bd09,
-	0x1bf408c5,
-	0x0232f409,
-/* 0x0c57: i2c_recv_not_wr08 */
-/* 0x0c57: i2c_recv_done */
-	0xc7030ef4,
-	0x21f5f8ce,
-	0xe0fc0b02,
-	0x12f4d0fc,
-	0x027cb90a,
-	0x033621f5,
-/* 0x0c6c: i2c_recv_exit */
-/* 0x0c6e: i2c_init */
+	0x76bb008a,
+	0x0465b600,
+	0x659450f9,
+	0x0256bb04,
+	0x75fd50bd,
+	0xf550fc04,
+	0xb609e921,
+	0x11f40464,
+	0xe05bcb6a,
+	0xb60076bb,
+	0x50f90465,
+	0xbb046594,
+	0x50bd0256,
+	0xfc0475fd,
+	0x2e21f550,
+	0x0464b609,
+	0xbd025bb9,
+	0x430ef474,
+/* 0x0c10: i2c_recv_not_rd08 */
+	0xf401d6b0,
+	0x57f03d1b,
+	0x9621f500,
+	0x3311f40a,
+	0xf5e0c5c7,
+	0xf40a3b21,
+	0x57f02911,
+	0x9621f500,
+	0x1f11f40a,
+	0xf5e0b5c7,
+	0xf40a3b21,
+	0x21f51511,
+	0x74bd092e,
+	0xf408c5c7,
+	0x32f4091b,
+	0x030ef402,
+/* 0x0c50: i2c_recv_not_wr08 */
+/* 0x0c50: i2c_recv_done */
+	0xf5f8cec7,
+	0xfc0afb21,
+	0xf4d0fce0,
+	0x7cb90a12,
+	0x3621f502,
+/* 0x0c65: i2c_recv_exit */
+/* 0x0c67: i2c_init */
+	0xf800f803,
+/* 0x0c69: test_recv */
+	0xd817f100,
+	0x0614b605,
+	0xb60011cf,
+	0x07f10110,
+	0x04b605d8,
+	0x0001d006,
+	0xe7f104bd,
+	0xe3f1d900,
+	0x21f5134f,
+	0x00f80256,
+/* 0x0c90: test_init */
+	0x0800e7f1,
+	0x025621f5,
+/* 0x0c9a: idle_recv */
 	0x00f800f8,
-/* 0x0c70: test_recv */
-	0x05d817f1,
-	0xcf0614b6,
-	0x10b60011,
-	0xd807f101,
-	0x0604b605,
-	0xbd0001d0,
-	0x00e7f104,
-	0x4fe3f1d9,
-	0x5621f513,
-/* 0x0c97: test_init */
-	0xf100f802,
-	0xf50800e7,
-	0xf8025621,
-/* 0x0ca1: idle_recv */
-/* 0x0ca3: idle */
-	0xf400f800,
-	0x17f10031,
-	0x14b605d4,
-	0x0011cf06,
-	0xf10110b6,
-	0xb605d407,
-	0x01d00604,
-/* 0x0cbf: idle_loop */
-	0xf004bd00,
-	0x32f45817,
-/* 0x0cc5: idle_proc */
-/* 0x0cc5: idle_proc_exec */
-	0xb910f902,
-	0x21f5021e,
-	0x10fc033f,
-	0xf40911f4,
-	0x0ef40231,
-/* 0x0cd9: idle_proc_next */
-	0x5810b6ef,
-	0xf4061fb8,
-	0x02f4e61b,
-	0x0028f4dd,
-	0x00bb0ef4,
+/* 0x0c9c: idle */
+	0xf10031f4,
+	0xb605d417,
+	0x11cf0614,
+	0x0110b600,
+	0x05d407f1,
+	0xd00604b6,
+	0x04bd0001,
+/* 0x0cb8: idle_loop */
+	0xf45817f0,
+/* 0x0cbe: idle_proc */
+/* 0x0cbe: idle_proc_exec */
+	0x10f90232,
+	0xf5021eb9,
+	0xfc033f21,
+	0x0911f410,
+	0xf40231f4,
+/* 0x0cd2: idle_proc_next */
+	0x10b6ef0e,
+	0x061fb858,
+	0xf4e61bf4,
+	0x28f4dd02,
+	0xbb0ef400,
+	0x00000000,
+	0x00000000,
 	0x00000000,
 	0x00000000,
 	0x00000000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/memx.fuc b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/memx.fuc
index ec03f9a4290b..1663bf943d77 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/memx.fuc
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/memx.fuc
@@ -82,15 +82,15 @@ memx_train_tail:
 // $r0  - zero
 memx_func_enter:
 #if NVKM_PPWR_CHIPSET == GT215
-	movw $r8 0x1610
+	mov $r8 0x1610
 	nv_rd32($r7, $r8)
 	imm32($r6, 0xfffffffc)
 	and $r7 $r6
-	movw $r6 0x2
+	mov $r6 0x2
 	or $r7 $r6
 	nv_wr32($r8, $r7)
 #else
-	movw $r6 0x001620
+	mov $r6 0x001620
 	imm32($r7, ~0x00000aa2);
 	nv_rd32($r8, $r6)
 	and $r8 $r7
@@ -101,7 +101,7 @@ memx_func_enter:
 	and $r8 $r7
 	nv_wr32($r6, $r8)
 
-	movw $r6 0x0026f0
+	mov $r6 0x0026f0
 	nv_rd32($r8, $r6)
 	and $r8 $r7
 	nv_wr32($r6, $r8)
@@ -136,19 +136,19 @@ memx_func_leave:
 		bra nz #memx_func_leave_wait
 
 #if NVKM_PPWR_CHIPSET == GT215
-	movw $r8 0x1610
+	mov $r8 0x1610
 	nv_rd32($r7, $r8)
 	imm32($r6, 0xffffffcc)
 	and $r7 $r6
 	nv_wr32($r8, $r7)
 #else
-	movw $r6 0x0026f0
+	mov $r6 0x0026f0
 	imm32($r7, 0x00000001)
 	nv_rd32($r8, $r6)
 	or $r8 $r7
 	nv_wr32($r6, $r8)
 
-	movw $r6 0x001620
+	mov $r6 0x001620
 	nv_rd32($r8, $r6)
 	or $r8 $r7
 	nv_wr32($r6, $r8)
@@ -177,11 +177,11 @@ memx_func_wait_vblank:
 	bra #memx_func_wait_vblank_fini
 
 	memx_func_wait_vblank_head1:
-	movw $r7 0x20
+	mov $r7 0x20
 	bra #memx_func_wait_vblank_0
 
 	memx_func_wait_vblank_head0:
-	movw $r7 0x8
+	mov $r7 0x8
 
 	memx_func_wait_vblank_0:
 		nv_iord($r6, NV_PPWR_INPUT)
@@ -273,13 +273,13 @@ memx_func_train:
 // $r5 - outer loop counter
 // $r6 - inner loop counter
 // $r7 - entry counter (#memx_train_head + $r7)
-	movw $r5 0x3
-	movw $r7 0x0
+	mov $r5 0x3
+	mov $r7 0x0
 
 // Read random memory to wake up... things
 	imm32($r9, 0x700000)
 	nv_rd32($r8,$r9)
-	movw $r14 0x2710
+	mov $r14 0x2710
 	call(nsec)
 
 	memx_func_train_loop_outer:
@@ -289,9 +289,9 @@ memx_func_train:
 		nv_wr32($r9, $r8)
 		push $r5
 
-		movw $r6 0x0
+		mov $r6 0x0
 		memx_func_train_loop_inner:
-			movw $r8 0x1111
+			mov $r8 0x1111
 			mulu $r9 $r6 $r8
 			shl b32 $r8 $r9 0x10
 			or $r8 $r9
@@ -315,7 +315,7 @@ memx_func_train:
 
 			// $r5 - inner inner loop counter
 			// $r9 - result
-			movw $r5 0
+			mov $r5 0
 			imm32($r9, 0x8300ffff)
 			memx_func_train_loop_4x:
 				imm32($r10, 0x100080)
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/Kbuild
index e698f4836521..ed08120eefe0 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/Kbuild
@@ -7,8 +7,10 @@ nvkm-y += nvkm/subdev/secboot/acr_r352.o
 nvkm-y += nvkm/subdev/secboot/acr_r361.o
 nvkm-y += nvkm/subdev/secboot/acr_r364.o
 nvkm-y += nvkm/subdev/secboot/acr_r367.o
+nvkm-y += nvkm/subdev/secboot/acr_r370.o
 nvkm-y += nvkm/subdev/secboot/acr_r375.o
 nvkm-y += nvkm/subdev/secboot/gm200.o
 nvkm-y += nvkm/subdev/secboot/gm20b.o
 nvkm-y += nvkm/subdev/secboot/gp102.o
+nvkm-y += nvkm/subdev/secboot/gp108.o
 nvkm-y += nvkm/subdev/secboot/gp10b.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr.h
index b615fc81aca4..73a2ac81ac69 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr.h
@@ -64,6 +64,7 @@ struct nvkm_acr *acr_r352_new(unsigned long);
 struct nvkm_acr *acr_r361_new(unsigned long);
 struct nvkm_acr *acr_r364_new(unsigned long);
 struct nvkm_acr *acr_r367_new(enum nvkm_secboot_falcon, unsigned long);
+struct nvkm_acr *acr_r370_new(enum nvkm_secboot_falcon, unsigned long);
 struct nvkm_acr *acr_r375_new(enum nvkm_secboot_falcon, unsigned long);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.c
new file mode 100644
index 000000000000..2f890dfae7fc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "acr_r370.h"
+#include "acr_r367.h"
+
+#include <core/msgqueue.h>
+#include <engine/falcon.h>
+#include <engine/sec2.h>
+
+static void
+acr_r370_generate_flcn_bl_desc(const struct nvkm_acr *acr,
+			       const struct ls_ucode_img *img, u64 wpr_addr,
+			       void *_desc)
+{
+	struct acr_r370_flcn_bl_desc *desc = _desc;
+	const struct ls_ucode_img_desc *pdesc = &img->ucode_desc;
+	u64 base, addr_code, addr_data;
+
+	base = wpr_addr + img->ucode_off + pdesc->app_start_offset;
+	addr_code = base + pdesc->app_resident_code_offset;
+	addr_data = base + pdesc->app_resident_data_offset;
+
+	desc->ctx_dma = FALCON_DMAIDX_UCODE;
+	desc->code_dma_base = u64_to_flcn64(addr_code);
+	desc->non_sec_code_off = pdesc->app_resident_code_offset;
+	desc->non_sec_code_size = pdesc->app_resident_code_size;
+	desc->code_entry_point = pdesc->app_imem_entry;
+	desc->data_dma_base = u64_to_flcn64(addr_data);
+	desc->data_size = pdesc->app_resident_data_size;
+}
+
+const struct acr_r352_ls_func
+acr_r370_ls_fecs_func = {
+	.load = acr_ls_ucode_load_fecs,
+	.generate_bl_desc = acr_r370_generate_flcn_bl_desc,
+	.bl_desc_size = sizeof(struct acr_r370_flcn_bl_desc),
+};
+
+const struct acr_r352_ls_func
+acr_r370_ls_gpccs_func = {
+	.load = acr_ls_ucode_load_gpccs,
+	.generate_bl_desc = acr_r370_generate_flcn_bl_desc,
+	.bl_desc_size = sizeof(struct acr_r370_flcn_bl_desc),
+	/* GPCCS will be loaded using PRI */
+	.lhdr_flags = LSF_FLAG_FORCE_PRIV_LOAD,
+};
+
+static void
+acr_r370_generate_sec2_bl_desc(const struct nvkm_acr *acr,
+			       const struct ls_ucode_img *img, u64 wpr_addr,
+			       void *_desc)
+{
+	const struct ls_ucode_img_desc *pdesc = &img->ucode_desc;
+	const struct nvkm_sec2 *sec = acr->subdev->device->sec2;
+	struct acr_r370_flcn_bl_desc *desc = _desc;
+	u64 base, addr_code, addr_data;
+	u32 addr_args;
+
+	base = wpr_addr + img->ucode_off + pdesc->app_start_offset;
+	/* For some reason we should not add app_resident_code_offset here */
+	addr_code = base;
+	addr_data = base + pdesc->app_resident_data_offset;
+	addr_args = sec->falcon->data.limit;
+	addr_args -= NVKM_MSGQUEUE_CMDLINE_SIZE;
+
+	desc->ctx_dma = FALCON_SEC2_DMAIDX_UCODE;
+	desc->code_dma_base = u64_to_flcn64(addr_code);
+	desc->non_sec_code_off = pdesc->app_resident_code_offset;
+	desc->non_sec_code_size = pdesc->app_resident_code_size;
+	desc->code_entry_point = pdesc->app_imem_entry;
+	desc->data_dma_base = u64_to_flcn64(addr_data);
+	desc->data_size = pdesc->app_resident_data_size;
+	desc->argc = 1;
+	/* args are stored at the beginning of EMEM */
+	desc->argv = 0x01000000;
+}
+
+const struct acr_r352_ls_func
+acr_r370_ls_sec2_func = {
+	.load = acr_ls_ucode_load_sec2,
+	.generate_bl_desc = acr_r370_generate_sec2_bl_desc,
+	.bl_desc_size = sizeof(struct acr_r370_flcn_bl_desc),
+	.post_run = acr_ls_sec2_post_run,
+};
+
+void
+acr_r370_generate_hs_bl_desc(const struct hsf_load_header *hdr, void *_bl_desc,
+			     u64 offset)
+{
+	struct acr_r370_flcn_bl_desc *bl_desc = _bl_desc;
+
+	bl_desc->ctx_dma = FALCON_DMAIDX_VIRT;
+	bl_desc->non_sec_code_off = hdr->non_sec_code_off;
+	bl_desc->non_sec_code_size = hdr->non_sec_code_size;
+	bl_desc->sec_code_off = hsf_load_header_app_off(hdr, 0);
+	bl_desc->sec_code_size = hsf_load_header_app_size(hdr, 0);
+	bl_desc->code_entry_point = 0;
+	bl_desc->code_dma_base = u64_to_flcn64(offset);
+	bl_desc->data_dma_base = u64_to_flcn64(offset + hdr->data_dma_base);
+	bl_desc->data_size = hdr->data_size;
+}
+
+const struct acr_r352_func
+acr_r370_func = {
+	.fixup_hs_desc = acr_r367_fixup_hs_desc,
+	.generate_hs_bl_desc = acr_r370_generate_hs_bl_desc,
+	.hs_bl_desc_size = sizeof(struct acr_r370_flcn_bl_desc),
+	.shadow_blob = true,
+	.ls_ucode_img_load = acr_r367_ls_ucode_img_load,
+	.ls_fill_headers = acr_r367_ls_fill_headers,
+	.ls_write_wpr = acr_r367_ls_write_wpr,
+	.ls_func = {
+		[NVKM_SECBOOT_FALCON_SEC2] = &acr_r370_ls_sec2_func,
+		[NVKM_SECBOOT_FALCON_FECS] = &acr_r370_ls_fecs_func,
+		[NVKM_SECBOOT_FALCON_GPCCS] = &acr_r370_ls_gpccs_func,
+	},
+};
+
+struct nvkm_acr *
+acr_r370_new(enum nvkm_secboot_falcon boot_falcon,
+	     unsigned long managed_falcons)
+{
+	return acr_r352_new_(&acr_r370_func, boot_falcon, managed_falcons);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.h
new file mode 100644
index 000000000000..3426f86a15e4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r370.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __NVKM_SECBOOT_ACR_R370_H__
+#define __NVKM_SECBOOT_ACR_R370_H__
+
+#include "priv.h"
+struct hsf_load_header;
+
+/* Same as acr_r361_flcn_bl_desc, plus argc/argv */
+struct acr_r370_flcn_bl_desc {
+	u32 reserved[4];
+	u32 signature[4];
+	u32 ctx_dma;
+	struct flcn_u64 code_dma_base;
+	u32 non_sec_code_off;
+	u32 non_sec_code_size;
+	u32 sec_code_off;
+	u32 sec_code_size;
+	u32 code_entry_point;
+	struct flcn_u64 data_dma_base;
+	u32 data_size;
+	u32 argc;
+	u32 argv;
+};
+
+void acr_r370_generate_hs_bl_desc(const struct hsf_load_header *, void *, u64);
+extern const struct acr_r352_ls_func acr_r370_ls_fecs_func;
+extern const struct acr_r352_ls_func acr_r370_ls_gpccs_func;
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r375.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r375.c
index ddb795bb007b..7bdef93cb7ae 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r375.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/acr_r375.c
@@ -20,90 +20,12 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include "acr_r370.h"
 #include "acr_r367.h"
 
-#include <engine/falcon.h>
 #include <core/msgqueue.h>
 #include <subdev/pmu.h>
 
-/*
- * r375 ACR: similar to r367, but with a unified bootloader descriptor
- * structure for GR and PMU falcons.
- */
-
-/* Same as acr_r361_flcn_bl_desc, plus argc/argv */
-struct acr_r375_flcn_bl_desc {
-	u32 reserved[4];
-	u32 signature[4];
-	u32 ctx_dma;
-	struct flcn_u64 code_dma_base;
-	u32 non_sec_code_off;
-	u32 non_sec_code_size;
-	u32 sec_code_off;
-	u32 sec_code_size;
-	u32 code_entry_point;
-	struct flcn_u64 data_dma_base;
-	u32 data_size;
-	u32 argc;
-	u32 argv;
-};
-
-static void
-acr_r375_generate_flcn_bl_desc(const struct nvkm_acr *acr,
-			       const struct ls_ucode_img *img, u64 wpr_addr,
-			       void *_desc)
-{
-	struct acr_r375_flcn_bl_desc *desc = _desc;
-	const struct ls_ucode_img_desc *pdesc = &img->ucode_desc;
-	u64 base, addr_code, addr_data;
-
-	base = wpr_addr + img->ucode_off + pdesc->app_start_offset;
-	addr_code = base + pdesc->app_resident_code_offset;
-	addr_data = base + pdesc->app_resident_data_offset;
-
-	desc->ctx_dma = FALCON_DMAIDX_UCODE;
-	desc->code_dma_base = u64_to_flcn64(addr_code);
-	desc->non_sec_code_off = pdesc->app_resident_code_offset;
-	desc->non_sec_code_size = pdesc->app_resident_code_size;
-	desc->code_entry_point = pdesc->app_imem_entry;
-	desc->data_dma_base = u64_to_flcn64(addr_data);
-	desc->data_size = pdesc->app_resident_data_size;
-}
-
-static void
-acr_r375_generate_hs_bl_desc(const struct hsf_load_header *hdr, void *_bl_desc,
-			     u64 offset)
-{
-	struct acr_r375_flcn_bl_desc *bl_desc = _bl_desc;
-
-	bl_desc->ctx_dma = FALCON_DMAIDX_VIRT;
-	bl_desc->non_sec_code_off = hdr->non_sec_code_off;
-	bl_desc->non_sec_code_size = hdr->non_sec_code_size;
-	bl_desc->sec_code_off = hsf_load_header_app_off(hdr, 0);
-	bl_desc->sec_code_size = hsf_load_header_app_size(hdr, 0);
-	bl_desc->code_entry_point = 0;
-	bl_desc->code_dma_base = u64_to_flcn64(offset);
-	bl_desc->data_dma_base = u64_to_flcn64(offset + hdr->data_dma_base);
-	bl_desc->data_size = hdr->data_size;
-}
-
-const struct acr_r352_ls_func
-acr_r375_ls_fecs_func = {
-	.load = acr_ls_ucode_load_fecs,
-	.generate_bl_desc = acr_r375_generate_flcn_bl_desc,
-	.bl_desc_size = sizeof(struct acr_r375_flcn_bl_desc),
-};
-
-const struct acr_r352_ls_func
-acr_r375_ls_gpccs_func = {
-	.load = acr_ls_ucode_load_gpccs,
-	.generate_bl_desc = acr_r375_generate_flcn_bl_desc,
-	.bl_desc_size = sizeof(struct acr_r375_flcn_bl_desc),
-	/* GPCCS will be loaded using PRI */
-	.lhdr_flags = LSF_FLAG_FORCE_PRIV_LOAD,
-};
-
-
 static void
 acr_r375_generate_pmu_bl_desc(const struct nvkm_acr *acr,
 			      const struct ls_ucode_img *img, u64 wpr_addr,
@@ -111,7 +33,7 @@ acr_r375_generate_pmu_bl_desc(const struct nvkm_acr *acr,
 {
 	const struct ls_ucode_img_desc *pdesc = &img->ucode_desc;
 	const struct nvkm_pmu *pmu = acr->subdev->device->pmu;
-	struct acr_r375_flcn_bl_desc *desc = _desc;
+	struct acr_r370_flcn_bl_desc *desc = _desc;
 	u64 base, addr_code, addr_data;
 	u32 addr_args;
 
@@ -136,23 +58,22 @@ const struct acr_r352_ls_func
 acr_r375_ls_pmu_func = {
 	.load = acr_ls_ucode_load_pmu,
 	.generate_bl_desc = acr_r375_generate_pmu_bl_desc,
-	.bl_desc_size = sizeof(struct acr_r375_flcn_bl_desc),
+	.bl_desc_size = sizeof(struct acr_r370_flcn_bl_desc),
 	.post_run = acr_ls_pmu_post_run,
 };
 
-
 const struct acr_r352_func
 acr_r375_func = {
 	.fixup_hs_desc = acr_r367_fixup_hs_desc,
-	.generate_hs_bl_desc = acr_r375_generate_hs_bl_desc,
-	.hs_bl_desc_size = sizeof(struct acr_r375_flcn_bl_desc),
+	.generate_hs_bl_desc = acr_r370_generate_hs_bl_desc,
+	.hs_bl_desc_size = sizeof(struct acr_r370_flcn_bl_desc),
 	.shadow_blob = true,
 	.ls_ucode_img_load = acr_r367_ls_ucode_img_load,
 	.ls_fill_headers = acr_r367_ls_fill_headers,
 	.ls_write_wpr = acr_r367_ls_write_wpr,
 	.ls_func = {
-		[NVKM_SECBOOT_FALCON_FECS] = &acr_r375_ls_fecs_func,
-		[NVKM_SECBOOT_FALCON_GPCCS] = &acr_r375_ls_gpccs_func,
+		[NVKM_SECBOOT_FALCON_FECS] = &acr_r370_ls_fecs_func,
+		[NVKM_SECBOOT_FALCON_GPCCS] = &acr_r370_ls_gpccs_func,
 		[NVKM_SECBOOT_FALCON_PMU] = &acr_r375_ls_pmu_func,
 	},
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c
index f3b3c66349d2..1f7a3c1a7f50 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp102.c
@@ -133,7 +133,7 @@ gp102_secboot_run_blob(struct nvkm_secboot *sb, struct nvkm_gpuobj *blob,
 	return gm200_secboot_run_blob(sb, blob, falcon);
 }
 
-static const struct nvkm_secboot_func
+const struct nvkm_secboot_func
 gp102_secboot = {
 	.dtor = gm200_secboot_dtor,
 	.oneinit = gm200_secboot_oneinit,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c
new file mode 100644
index 000000000000..e8c27ec700de
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gp108.c
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2017 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "gm200.h"
+#include "acr.h"
+
+int
+gp108_secboot_new(struct nvkm_device *device, int index,
+		  struct nvkm_secboot **psb)
+{
+	struct gm200_secboot *gsb;
+	struct nvkm_acr *acr;
+
+	acr = acr_r370_new(NVKM_SECBOOT_FALCON_SEC2,
+			   BIT(NVKM_SECBOOT_FALCON_FECS) |
+			   BIT(NVKM_SECBOOT_FALCON_GPCCS) |
+			   BIT(NVKM_SECBOOT_FALCON_SEC2));
+	if (IS_ERR(acr))
+		return PTR_ERR(acr);
+
+	if (!(gsb = kzalloc(sizeof(*gsb), GFP_KERNEL))) {
+		acr->func->dtor(acr);
+		return -ENOMEM;
+	}
+	*psb = &gsb->base;
+
+	return nvkm_secboot_ctor(&gp102_secboot, acr, device, index, &gsb->base);
+}
+
+MODULE_FIRMWARE("nvidia/gp108/acr/bl.bin");
+MODULE_FIRMWARE("nvidia/gp108/acr/unload_bl.bin");
+MODULE_FIRMWARE("nvidia/gp108/acr/ucode_load.bin");
+MODULE_FIRMWARE("nvidia/gp108/acr/ucode_unload.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/fecs_bl.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/fecs_inst.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/fecs_data.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/fecs_sig.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/gpccs_bl.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/gpccs_inst.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/gpccs_data.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/gpccs_sig.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/sw_ctx.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/sw_nonctx.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/sw_bundle_init.bin");
+MODULE_FIRMWARE("nvidia/gp108/gr/sw_method_init.bin");
+MODULE_FIRMWARE("nvidia/gp108/nvdec/scrubber.bin");
+MODULE_FIRMWARE("nvidia/gp108/sec2/desc.bin");
+MODULE_FIRMWARE("nvidia/gp108/sec2/image.bin");
+MODULE_FIRMWARE("nvidia/gp108/sec2/sig.bin");
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
index d9091f029506..959a7b2dbdc9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
@@ -40,6 +40,8 @@ int nvkm_secboot_ctor(const struct nvkm_secboot_func *, struct nvkm_acr *,
 int nvkm_secboot_falcon_reset(struct nvkm_secboot *);
 int nvkm_secboot_falcon_run(struct nvkm_secboot *);
 
+extern const struct nvkm_secboot_func gp102_secboot;
+
 struct flcn_u64 {
 	u32 lo;
 	u32 hi;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
index 7ba56b12badd..550702eab0b1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/Kbuild
@@ -9,7 +9,9 @@ nvkm-y += nvkm/subdev/therm/nv40.o
 nvkm-y += nvkm/subdev/therm/nv50.o
 nvkm-y += nvkm/subdev/therm/g84.o
 nvkm-y += nvkm/subdev/therm/gt215.o
+nvkm-y += nvkm/subdev/therm/gf100.o
 nvkm-y += nvkm/subdev/therm/gf119.o
+nvkm-y += nvkm/subdev/therm/gk104.o
 nvkm-y += nvkm/subdev/therm/gm107.o
 nvkm-y += nvkm/subdev/therm/gm200.o
 nvkm-y += nvkm/subdev/therm/gp100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c
index f27fc6d0d4c6..3695cde669f8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/base.c
@@ -21,6 +21,7 @@
  *
  * Authors: Martin Peres
  */
+#include <nvkm/core/option.h>
 #include "priv.h"
 
 int
@@ -297,6 +298,38 @@ nvkm_therm_attr_set(struct nvkm_therm *therm,
 	return -EINVAL;
 }
 
+void
+nvkm_therm_clkgate_enable(struct nvkm_therm *therm)
+{
+	if (!therm || !therm->func->clkgate_enable || !therm->clkgating_enabled)
+		return;
+
+	nvkm_debug(&therm->subdev,
+		   "Enabling clockgating\n");
+	therm->func->clkgate_enable(therm);
+}
+
+void
+nvkm_therm_clkgate_fini(struct nvkm_therm *therm, bool suspend)
+{
+	if (!therm || !therm->func->clkgate_fini || !therm->clkgating_enabled)
+		return;
+
+	nvkm_debug(&therm->subdev,
+		   "Preparing clockgating for %s\n",
+		   suspend ? "suspend" : "fini");
+	therm->func->clkgate_fini(therm, suspend);
+}
+
+static void
+nvkm_therm_clkgate_oneinit(struct nvkm_therm *therm)
+{
+	if (!therm->func->clkgate_enable || !therm->clkgating_enabled)
+		return;
+
+	nvkm_info(&therm->subdev, "Clockgating enabled\n");
+}
+
 static void
 nvkm_therm_intr(struct nvkm_subdev *subdev)
 {
@@ -333,6 +366,7 @@ nvkm_therm_oneinit(struct nvkm_subdev *subdev)
 	nvkm_therm_fan_ctor(therm);
 	nvkm_therm_fan_mode(therm, NVKM_THERM_CTRL_AUTO);
 	nvkm_therm_sensor_preinit(therm);
+	nvkm_therm_clkgate_oneinit(therm);
 	return 0;
 }
 
@@ -357,6 +391,16 @@ nvkm_therm_init(struct nvkm_subdev *subdev)
 	return 0;
 }
 
+void
+nvkm_therm_clkgate_init(struct nvkm_therm *therm,
+			const struct nvkm_therm_clkgate_pack *p)
+{
+	if (!therm || !therm->func->clkgate_init || !therm->clkgating_enabled)
+		return;
+
+	therm->func->clkgate_init(therm, p);
+}
+
 static void *
 nvkm_therm_dtor(struct nvkm_subdev *subdev)
 {
@@ -374,15 +418,10 @@ nvkm_therm = {
 	.intr = nvkm_therm_intr,
 };
 
-int
-nvkm_therm_new_(const struct nvkm_therm_func *func, struct nvkm_device *device,
-		int index, struct nvkm_therm **ptherm)
+void
+nvkm_therm_ctor(struct nvkm_therm *therm, struct nvkm_device *device,
+		int index, const struct nvkm_therm_func *func)
 {
-	struct nvkm_therm *therm;
-
-	if (!(therm = *ptherm = kzalloc(sizeof(*therm), GFP_KERNEL)))
-		return -ENOMEM;
-
 	nvkm_subdev_ctor(&nvkm_therm, device, index, &therm->subdev);
 	therm->func = func;
 
@@ -395,5 +434,20 @@ nvkm_therm_new_(const struct nvkm_therm_func *func, struct nvkm_device *device,
 	therm->attr_get = nvkm_therm_attr_get;
 	therm->attr_set = nvkm_therm_attr_set;
 	therm->mode = therm->suspend = -1; /* undefined */
+
+	therm->clkgating_enabled = nvkm_boolopt(device->cfgopt,
+						"NvPmEnableGating", false);
+}
+
+int
+nvkm_therm_new_(const struct nvkm_therm_func *func, struct nvkm_device *device,
+		int index, struct nvkm_therm **ptherm)
+{
+	struct nvkm_therm *therm;
+
+	if (!(therm = *ptherm = kzalloc(sizeof(*therm), GFP_KERNEL)))
+		return -ENOMEM;
+
+	nvkm_therm_ctor(therm, device, index, func);
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/basics/grph_object_id.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.c
similarity index 50%
rename from drivers/gpu/drm/amd/display/dc/basics/grph_object_id.c
rename to drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.c
index 147822545252..5ae6913320e8 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/grph_object_id.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-15 Advanced Micro Devices, Inc.
+ * Copyright 2018 Red Hat Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -19,57 +19,40 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  *
- * Authors: AMD
- *
+ * Authors: Lyude Paul
  */
+#include <core/device.h>
 
-#include "dm_services.h"
-#include "include/grph_object_id.h"
-
-static bool dal_graphics_object_id_is_valid(struct graphics_object_id id)
-{
-	bool rc = true;
-
-	switch (id.type) {
-	case OBJECT_TYPE_UNKNOWN:
-		rc = false;
-		break;
-	case OBJECT_TYPE_GPU:
-	case OBJECT_TYPE_ENGINE:
-		/* do NOT check for id.id == 0 */
-		if (id.enum_id == ENUM_ID_UNKNOWN)
-			rc = false;
-		break;
-	default:
-		if (id.id == 0 || id.enum_id == ENUM_ID_UNKNOWN)
-			rc = false;
-		break;
-	}
-
-	return rc;
-}
+#include "priv.h"
 
-bool dal_graphics_object_id_is_equal(
-	struct graphics_object_id id1,
-	struct graphics_object_id id2)
+#define pack_for_each_init(init, pack, head)                          \
+	for (pack = head; pack && pack->init; pack++)                 \
+		  for (init = pack->init; init && init->count; init++)
+void
+gf100_clkgate_init(struct nvkm_therm *therm,
+		   const struct nvkm_therm_clkgate_pack *p)
 {
-	if (false == dal_graphics_object_id_is_valid(id1)) {
-		dm_output_to_console(
-		"%s: Warning: comparing invalid object 'id1'!\n", __func__);
-		return false;
+	struct nvkm_device *device = therm->subdev.device;
+	const struct nvkm_therm_clkgate_pack *pack;
+	const struct nvkm_therm_clkgate_init *init;
+	u32 next, addr;
+
+	pack_for_each_init(init, pack, p) {
+		next = init->addr + init->count * 8;
+		addr = init->addr;
+
+		nvkm_trace(&therm->subdev, "{ 0x%06x, %d, 0x%08x }\n",
+			   init->addr, init->count, init->data);
+		while (addr < next) {
+			nvkm_trace(&therm->subdev, "\t0x%06x = 0x%08x\n",
+				   addr, init->data);
+			nvkm_wr32(device, addr, init->data);
+			addr += 8;
+		}
 	}
-
-	if (false == dal_graphics_object_id_is_valid(id2)) {
-		dm_output_to_console(
-		"%s: Warning: comparing invalid object 'id2'!\n", __func__);
-		return false;
-	}
-
-	if (id1.id == id2.id && id1.enum_id == id2.enum_id
-		&& id1.type == id2.type)
-		return true;
-
-	return false;
 }
 
-
+/*
+ * TODO: Fermi clockgating isn't understood fully yet, so we don't specify any
+ * clockgate functions to use
+ */
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h
new file mode 100644
index 000000000000..cfb25af77c60
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf100.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Lyude Paul
+ */
+
+#ifndef __GF100_THERM_H__
+#define __GF100_THERM_H__
+
+#include <core/device.h>
+
+struct gf100_idle_filter {
+	u32 fecs;
+	u32 hubmmu;
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c
index 06dcfd6ee966..0981b02790e2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c
@@ -49,7 +49,7 @@ pwm_info(struct nvkm_therm *therm, int line)
 	return -ENODEV;
 }
 
-static int
+int
 gf119_fan_pwm_ctrl(struct nvkm_therm *therm, int line, bool enable)
 {
 	struct nvkm_device *device = therm->subdev.device;
@@ -63,7 +63,7 @@ gf119_fan_pwm_ctrl(struct nvkm_therm *therm, int line, bool enable)
 	return 0;
 }
 
-static int
+int
 gf119_fan_pwm_get(struct nvkm_therm *therm, int line, u32 *divs, u32 *duty)
 {
 	struct nvkm_device *device = therm->subdev.device;
@@ -85,7 +85,7 @@ gf119_fan_pwm_get(struct nvkm_therm *therm, int line, u32 *divs, u32 *duty)
 	return -EINVAL;
 }
 
-static int
+int
 gf119_fan_pwm_set(struct nvkm_therm *therm, int line, u32 divs, u32 duty)
 {
 	struct nvkm_device *device = therm->subdev.device;
@@ -102,7 +102,7 @@ gf119_fan_pwm_set(struct nvkm_therm *therm, int line, u32 divs, u32 duty)
 	return 0;
 }
 
-static int
+int
 gf119_fan_pwm_clock(struct nvkm_therm *therm, int line)
 {
 	struct nvkm_device *device = therm->subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c
new file mode 100644
index 000000000000..4e03971d2e3d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.c
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Lyude Paul
+ */
+#include <core/device.h>
+
+#include "priv.h"
+#include "gk104.h"
+
+void
+gk104_clkgate_enable(struct nvkm_therm *base)
+{
+	struct gk104_therm *therm = gk104_therm(base);
+	struct nvkm_device *dev = therm->base.subdev.device;
+	const struct gk104_clkgate_engine_info *order = therm->clkgate_order;
+	int i;
+
+	/* Program ENG_MANT, ENG_FILTER */
+	for (i = 0; order[i].engine != NVKM_SUBDEV_NR; i++) {
+		if (!nvkm_device_subdev(dev, order[i].engine))
+			continue;
+
+		nvkm_mask(dev, 0x20200 + order[i].offset, 0xff00, 0x4500);
+	}
+
+	/* magic */
+	nvkm_wr32(dev, 0x020288, therm->idle_filter->fecs);
+	nvkm_wr32(dev, 0x02028c, therm->idle_filter->hubmmu);
+
+	/* Enable clockgating (ENG_CLK = RUN->AUTO) */
+	for (i = 0; order[i].engine != NVKM_SUBDEV_NR; i++) {
+		if (!nvkm_device_subdev(dev, order[i].engine))
+			continue;
+
+		nvkm_mask(dev, 0x20200 + order[i].offset, 0x00ff, 0x0045);
+	}
+}
+
+void
+gk104_clkgate_fini(struct nvkm_therm *base, bool suspend)
+{
+	struct gk104_therm *therm = gk104_therm(base);
+	struct nvkm_device *dev = therm->base.subdev.device;
+	const struct gk104_clkgate_engine_info *order = therm->clkgate_order;
+	int i;
+
+	/* ENG_CLK = AUTO->RUN, ENG_PWR = RUN->AUTO */
+	for (i = 0; order[i].engine != NVKM_SUBDEV_NR; i++) {
+		if (!nvkm_device_subdev(dev, order[i].engine))
+			continue;
+
+		nvkm_mask(dev, 0x20200 + order[i].offset, 0xff, 0x54);
+	}
+}
+
+const struct gk104_clkgate_engine_info gk104_clkgate_engine_info[] = {
+	{ NVKM_ENGINE_GR,     0x00 },
+	{ NVKM_ENGINE_MSPDEC, 0x04 },
+	{ NVKM_ENGINE_MSPPP,  0x08 },
+	{ NVKM_ENGINE_MSVLD,  0x0c },
+	{ NVKM_ENGINE_CE0,    0x10 },
+	{ NVKM_ENGINE_CE1,    0x14 },
+	{ NVKM_ENGINE_MSENC,  0x18 },
+	{ NVKM_ENGINE_CE2,    0x1c },
+	{ NVKM_SUBDEV_NR, 0 },
+};
+
+const struct gf100_idle_filter gk104_idle_filter = {
+	.fecs = 0x00001000,
+	.hubmmu = 0x00001000,
+};
+
+static const struct nvkm_therm_func
+gk104_therm_func = {
+	.init = gf119_therm_init,
+	.fini = g84_therm_fini,
+	.pwm_ctrl = gf119_fan_pwm_ctrl,
+	.pwm_get = gf119_fan_pwm_get,
+	.pwm_set = gf119_fan_pwm_set,
+	.pwm_clock = gf119_fan_pwm_clock,
+	.temp_get = g84_temp_get,
+	.fan_sense = gt215_therm_fan_sense,
+	.program_alarms = nvkm_therm_program_alarms_polling,
+	.clkgate_init = gf100_clkgate_init,
+	.clkgate_enable = gk104_clkgate_enable,
+	.clkgate_fini = gk104_clkgate_fini,
+};
+
+static int
+gk104_therm_new_(const struct nvkm_therm_func *func,
+		 struct nvkm_device *device,
+		 int index,
+		 const struct gk104_clkgate_engine_info *clkgate_order,
+		 const struct gf100_idle_filter *idle_filter,
+		 struct nvkm_therm **ptherm)
+{
+	struct gk104_therm *therm = kzalloc(sizeof(*therm), GFP_KERNEL);
+
+	if (!therm)
+		return -ENOMEM;
+
+	nvkm_therm_ctor(&therm->base, device, index, func);
+	*ptherm = &therm->base;
+	therm->clkgate_order = clkgate_order;
+	therm->idle_filter = idle_filter;
+
+	return 0;
+}
+
+int
+gk104_therm_new(struct nvkm_device *device,
+		int index, struct nvkm_therm **ptherm)
+{
+	return gk104_therm_new_(&gk104_therm_func, device, index,
+				gk104_clkgate_engine_info, &gk104_idle_filter,
+				ptherm);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h
new file mode 100644
index 000000000000..293e7743b19b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gk104.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Lyude Paul
+ */
+
+#ifndef __GK104_THERM_H__
+#define __GK104_THERM_H__
+#define gk104_therm(p) (container_of((p), struct gk104_therm, base))
+
+#include <subdev/therm.h>
+#include "priv.h"
+#include "gf100.h"
+
+struct gk104_clkgate_engine_info {
+	enum nvkm_devidx engine;
+	u8 offset;
+};
+
+struct gk104_therm {
+	struct nvkm_therm base;
+
+	const struct gk104_clkgate_engine_info *clkgate_order;
+	const struct gf100_idle_filter *idle_filter;
+};
+
+extern const struct gk104_clkgate_engine_info gk104_clkgate_engine_info[];
+extern const struct gf100_idle_filter gk104_idle_filter;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gt215.c
index c08097f2aff5..4caf401d001a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gt215.c
@@ -36,7 +36,7 @@ gt215_therm_fan_sense(struct nvkm_therm *therm)
 	return -ENODEV;
 }
 
-static void
+void
 gt215_therm_init(struct nvkm_therm *therm)
 {
 	struct nvkm_device *device = therm->subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h
index 1f46e371d7c4..21659daf1864 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/priv.h
@@ -32,6 +32,8 @@
 
 int nvkm_therm_new_(const struct nvkm_therm_func *, struct nvkm_device *,
 		    int index, struct nvkm_therm **);
+void nvkm_therm_ctor(struct nvkm_therm *therm, struct nvkm_device *device,
+		     int index, const struct nvkm_therm_func *func);
 
 struct nvkm_fan {
 	struct nvkm_therm *parent;
@@ -66,8 +68,6 @@ int nvkm_therm_fan_set(struct nvkm_therm *, bool now, int percent);
 int nvkm_therm_fan_user_get(struct nvkm_therm *);
 int nvkm_therm_fan_user_set(struct nvkm_therm *, int percent);
 
-int nvkm_therm_preinit(struct nvkm_therm *);
-
 int  nvkm_therm_sensor_init(struct nvkm_therm *);
 int  nvkm_therm_sensor_fini(struct nvkm_therm *, bool suspend);
 void nvkm_therm_sensor_preinit(struct nvkm_therm *);
@@ -96,6 +96,11 @@ struct nvkm_therm_func {
 	int (*fan_sense)(struct nvkm_therm *);
 
 	void (*program_alarms)(struct nvkm_therm *);
+
+	void (*clkgate_init)(struct nvkm_therm *,
+			     const struct nvkm_therm_clkgate_pack *);
+	void (*clkgate_enable)(struct nvkm_therm *);
+	void (*clkgate_fini)(struct nvkm_therm *, bool);
 };
 
 void nv40_therm_intr(struct nvkm_therm *);
@@ -111,9 +116,21 @@ void g84_therm_fini(struct nvkm_therm *);
 
 int gt215_therm_fan_sense(struct nvkm_therm *);
 
+void gf100_clkgate_init(struct nvkm_therm *,
+			const struct nvkm_therm_clkgate_pack *);
+
 void g84_therm_init(struct nvkm_therm *);
+
+int gf119_fan_pwm_ctrl(struct nvkm_therm *, int, bool);
+int gf119_fan_pwm_get(struct nvkm_therm *, int, u32 *, u32 *);
+int gf119_fan_pwm_set(struct nvkm_therm *, int, u32, u32);
+int gf119_fan_pwm_clock(struct nvkm_therm *, int);
 void gf119_therm_init(struct nvkm_therm *);
 
+void gk104_therm_init(struct nvkm_therm *);
+void gk104_clkgate_enable(struct nvkm_therm *);
+void gk104_clkgate_fini(struct nvkm_therm *, bool);
+
 int nvkm_fanpwm_create(struct nvkm_therm *, struct dcb_gpio_func *);
 int nvkm_fantog_create(struct nvkm_therm *, struct dcb_gpio_func *);
 int nvkm_fannil_create(struct nvkm_therm *);
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
index 542a76503fbd..95ea6abae914 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
@@ -1,7 +1,7 @@
 /*
  * Analog TV Connector driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
index 05fa24a518c8..10b4b97d3595 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
@@ -1,7 +1,7 @@
 /*
  * Generic DVI Connector driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
index 4600d3841c25..2867476419dc 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
@@ -1,7 +1,7 @@
 /*
  * HDMI Connector driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
index b1f6aa09f699..d523c67a3ae3 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
@@ -6,7 +6,7 @@
  *
  * based on encoder-tfp410
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
index 947295f9e30f..e01ab3db6d86 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
@@ -1,7 +1,7 @@
 /*
  * TFP410 DPI-to-DVI encoder driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -173,7 +173,8 @@ static int tfp410_probe_of(struct platform_device *pdev)
 	if (gpio_is_valid(gpio) || gpio == -ENOENT) {
 		ddata->pd_gpio = gpio;
 	} else {
-		dev_err(&pdev->dev, "failed to parse PD gpio\n");
+		if (gpio != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "failed to parse PD gpio\n");
 		return gpio;
 	}
 
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
index e3d98d78fc40..1fd493e5fa3d 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
@@ -1,7 +1,7 @@
 /*
  * TPD12S015 HDMI ESD protection & level shifter chip driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
index e065f7e10cca..efff6dbbb86f 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
@@ -1,7 +1,7 @@
 /*
  * Generic MIPI DPI Panel Driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
index 92c556ac22c7..15399a1a666b 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
@@ -1,7 +1,7 @@
 /*
  * Generic DSI Command Mode panel driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -22,9 +22,10 @@
 #include <linux/slab.h>
 #include <linux/workqueue.h>
 #include <linux/of_device.h>
-#include <linux/of_gpio.h>
+#include <linux/regulator/consumer.h>
 
 #include <video/mipi_display.h>
+#include <video/of_display_timing.h>
 
 #include "../dss/omapdss.h"
 
@@ -49,6 +50,7 @@ struct panel_drv_data {
 	struct mutex lock;
 
 	struct backlight_device *bldev;
+	struct backlight_device *extbldev;
 
 	unsigned long	hw_guard_end;	/* next value of jiffies when we can
 					 * issue the next sleep in/out command
@@ -56,11 +58,17 @@ struct panel_drv_data {
 	unsigned long	hw_guard_wait;	/* max guard time in jiffies */
 
 	/* panel HW configuration from DT or platform data */
-	int reset_gpio;
-	int ext_te_gpio;
+	struct gpio_desc *reset_gpio;
+	struct gpio_desc *ext_te_gpio;
+
+	struct regulator *vpnl;
+	struct regulator *vddi;
 
 	bool use_dsi_backlight;
 
+	int width_mm;
+	int height_mm;
+
 	struct omap_dsi_pin_config pin_config;
 
 	/* runtime variables */
@@ -92,6 +100,30 @@ static int dsicm_panel_reset(struct panel_drv_data *ddata);
 
 static void dsicm_ulps_work(struct work_struct *work);
 
+static void dsicm_bl_power(struct panel_drv_data *ddata, bool enable)
+{
+	struct backlight_device *backlight;
+
+	if (ddata->bldev)
+		backlight = ddata->bldev;
+	else if (ddata->extbldev)
+		backlight = ddata->extbldev;
+	else
+		return;
+
+	if (enable) {
+		backlight->props.fb_blank = FB_BLANK_UNBLANK;
+		backlight->props.state = ~(BL_CORE_FBBLANK | BL_CORE_SUSPENDED);
+		backlight->props.power = FB_BLANK_UNBLANK;
+	} else {
+		backlight->props.fb_blank = FB_BLANK_NORMAL;
+		backlight->props.power = FB_BLANK_POWERDOWN;
+		backlight->props.state |= BL_CORE_FBBLANK | BL_CORE_SUSPENDED;
+	}
+
+	backlight_update_status(backlight);
+}
+
 static void hw_guard_start(struct panel_drv_data *ddata, int guard_msec)
 {
 	ddata->hw_guard_wait = msecs_to_jiffies(guard_msec);
@@ -255,8 +287,8 @@ static int dsicm_enter_ulps(struct panel_drv_data *ddata)
 	if (r)
 		goto err;
 
-	if (gpio_is_valid(ddata->ext_te_gpio))
-		disable_irq(gpio_to_irq(ddata->ext_te_gpio));
+	if (ddata->ext_te_gpio)
+		disable_irq(gpiod_to_irq(ddata->ext_te_gpio));
 
 	in->ops.dsi->disable(in, false, true);
 
@@ -297,8 +329,8 @@ static int dsicm_exit_ulps(struct panel_drv_data *ddata)
 		goto err2;
 	}
 
-	if (gpio_is_valid(ddata->ext_te_gpio))
-		enable_irq(gpio_to_irq(ddata->ext_te_gpio));
+	if (ddata->ext_te_gpio)
+		enable_irq(gpiod_to_irq(ddata->ext_te_gpio));
 
 	dsicm_queue_ulps_work(ddata);
 
@@ -311,8 +343,8 @@ static int dsicm_exit_ulps(struct panel_drv_data *ddata)
 
 	r = dsicm_panel_reset(ddata);
 	if (!r) {
-		if (gpio_is_valid(ddata->ext_te_gpio))
-			enable_irq(gpio_to_irq(ddata->ext_te_gpio));
+		if (ddata->ext_te_gpio)
+			enable_irq(gpiod_to_irq(ddata->ext_te_gpio));
 		ddata->ulps_enabled = false;
 	}
 err1:
@@ -335,7 +367,7 @@ static int dsicm_bl_update_status(struct backlight_device *dev)
 {
 	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
 	struct omap_dss_device *in = ddata->in;
-	int r;
+	int r = 0;
 	int level;
 
 	if (dev->props.fb_blank == FB_BLANK_UNBLANK &&
@@ -356,8 +388,6 @@ static int dsicm_bl_update_status(struct backlight_device *dev)
 			r = dsicm_dcs_write_1(ddata, DCS_BRIGHTNESS, level);
 
 		in->ops.dsi->bus_unlock(in);
-	} else {
-		r = 0;
 	}
 
 	mutex_unlock(&ddata->lock);
@@ -560,16 +590,13 @@ static const struct attribute_group dsicm_attr_group = {
 
 static void dsicm_hw_reset(struct panel_drv_data *ddata)
 {
-	if (!gpio_is_valid(ddata->reset_gpio))
-		return;
-
-	gpio_set_value(ddata->reset_gpio, 1);
+	gpiod_set_value(ddata->reset_gpio, 1);
 	udelay(10);
 	/* reset the panel */
-	gpio_set_value(ddata->reset_gpio, 0);
+	gpiod_set_value(ddata->reset_gpio, 0);
 	/* assert reset */
 	udelay(10);
-	gpio_set_value(ddata->reset_gpio, 1);
+	gpiod_set_value(ddata->reset_gpio, 1);
 	/* wait after releasing reset */
 	usleep_range(5000, 10000);
 }
@@ -589,25 +616,43 @@ static int dsicm_power_on(struct panel_drv_data *ddata)
 		.lp_clk_max = 10000000,
 	};
 
+	if (ddata->vpnl) {
+		r = regulator_enable(ddata->vpnl);
+		if (r) {
+			dev_err(&ddata->pdev->dev,
+				"failed to enable VPNL: %d\n", r);
+			return r;
+		}
+	}
+
+	if (ddata->vddi) {
+		r = regulator_enable(ddata->vddi);
+		if (r) {
+			dev_err(&ddata->pdev->dev,
+				"failed to enable VDDI: %d\n", r);
+			goto err_vpnl;
+		}
+	}
+
 	if (ddata->pin_config.num_pins > 0) {
 		r = in->ops.dsi->configure_pins(in, &ddata->pin_config);
 		if (r) {
 			dev_err(&ddata->pdev->dev,
 				"failed to configure DSI pins\n");
-			goto err0;
+			goto err_vddi;
 		}
 	}
 
 	r = in->ops.dsi->set_config(in, &dsi_config);
 	if (r) {
 		dev_err(&ddata->pdev->dev, "failed to configure DSI\n");
-		goto err0;
+		goto err_vddi;
 	}
 
 	r = in->ops.dsi->enable(in);
 	if (r) {
 		dev_err(&ddata->pdev->dev, "failed to enable DSI\n");
-		goto err0;
+		goto err_vddi;
 	}
 
 	dsicm_hw_reset(ddata);
@@ -665,7 +710,13 @@ static int dsicm_power_on(struct panel_drv_data *ddata)
 	dsicm_hw_reset(ddata);
 
 	in->ops.dsi->disable(in, true, false);
-err0:
+err_vddi:
+	if (ddata->vddi)
+		regulator_disable(ddata->vddi);
+err_vpnl:
+	if (ddata->vpnl)
+		regulator_disable(ddata->vpnl);
+
 	return r;
 }
 
@@ -688,6 +739,11 @@ static void dsicm_power_off(struct panel_drv_data *ddata)
 
 	in->ops.dsi->disable(in, true, false);
 
+	if (ddata->vddi)
+		regulator_disable(ddata->vddi);
+	if (ddata->vpnl)
+		regulator_disable(ddata->vpnl);
+
 	ddata->enabled = 0;
 }
 
@@ -782,6 +838,8 @@ static int dsicm_enable(struct omap_dss_device *dssdev)
 
 	mutex_unlock(&ddata->lock);
 
+	dsicm_bl_power(ddata, true);
+
 	return 0;
 err:
 	dev_dbg(&ddata->pdev->dev, "enable failed\n");
@@ -797,6 +855,8 @@ static void dsicm_disable(struct omap_dss_device *dssdev)
 
 	dev_dbg(&ddata->pdev->dev, "disable\n");
 
+	dsicm_bl_power(ddata, false);
+
 	mutex_lock(&ddata->lock);
 
 	dsicm_cancel_ulps_work(ddata);
@@ -890,7 +950,7 @@ static int dsicm_update(struct omap_dss_device *dssdev,
 	if (r)
 		goto err;
 
-	if (ddata->te_enabled && gpio_is_valid(ddata->ext_te_gpio)) {
+	if (ddata->te_enabled && ddata->ext_te_gpio) {
 		schedule_delayed_work(&ddata->te_timeout_work,
 				msecs_to_jiffies(250));
 		atomic_set(&ddata->do_update, 1);
@@ -937,7 +997,7 @@ static int _dsicm_enable_te(struct panel_drv_data *ddata, bool enable)
 	else
 		r = dsicm_dcs_write_0(ddata, MIPI_DCS_SET_TEAR_OFF);
 
-	if (!gpio_is_valid(ddata->ext_te_gpio))
+	if (!ddata->ext_te_gpio)
 		in->ops.dsi->enable_te(in, enable);
 
 	/* possible panel bug */
@@ -1099,6 +1159,45 @@ static void dsicm_ulps_work(struct work_struct *work)
 	mutex_unlock(&ddata->lock);
 }
 
+static void dsicm_get_timings(struct omap_dss_device *dssdev,
+			      struct videomode *vm)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*vm = ddata->vm;
+}
+
+static int dsicm_check_timings(struct omap_dss_device *dssdev,
+			       struct videomode *vm)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+	int ret = 0;
+
+	if (vm->hactive != ddata->vm.hactive)
+		ret = -EINVAL;
+
+	if (vm->vactive != ddata->vm.vactive)
+		ret = -EINVAL;
+
+	if (ret) {
+		dev_warn(dssdev->dev, "wrong resolution: %d x %d",
+			 vm->hactive, vm->vactive);
+		dev_warn(dssdev->dev, "panel resolution: %d x %d",
+			 ddata->vm.hactive, ddata->vm.vactive);
+	}
+
+	return ret;
+}
+
+static void dsicm_get_size(struct omap_dss_device *dssdev,
+			  unsigned int *width, unsigned int *height)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
+
+	*width = ddata->width_mm;
+	*height = ddata->height_mm;
+}
+
 static struct omap_dss_driver dsicm_ops = {
 	.connect	= dsicm_connect,
 	.disconnect	= dsicm_disconnect,
@@ -1109,6 +1208,10 @@ static struct omap_dss_driver dsicm_ops = {
 	.update		= dsicm_update,
 	.sync		= dsicm_sync,
 
+	.get_timings	= dsicm_get_timings,
+	.check_timings	= dsicm_check_timings,
+	.get_size	= dsicm_get_size,
+
 	.enable_te	= dsicm_enable_te,
 	.get_te		= dsicm_get_te,
 
@@ -1118,41 +1221,87 @@ static struct omap_dss_driver dsicm_ops = {
 static int dsicm_probe_of(struct platform_device *pdev)
 {
 	struct device_node *node = pdev->dev.of_node;
+	struct device_node *backlight;
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
 	struct omap_dss_device *in;
-	int gpio;
+	struct display_timing timing;
+	int err;
+
+	ddata->reset_gpio = devm_gpiod_get(&pdev->dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ddata->reset_gpio)) {
+		err = PTR_ERR(ddata->reset_gpio);
+		dev_err(&pdev->dev, "reset gpio request failed: %d", err);
+		return err;
+	}
 
-	gpio = of_get_named_gpio(node, "reset-gpios", 0);
-	if (!gpio_is_valid(gpio)) {
-		dev_err(&pdev->dev, "failed to parse reset gpio\n");
-		return gpio;
+	ddata->ext_te_gpio = devm_gpiod_get_optional(&pdev->dev, "te",
+						     GPIOD_IN);
+	if (IS_ERR(ddata->ext_te_gpio)) {
+		err = PTR_ERR(ddata->ext_te_gpio);
+		dev_err(&pdev->dev, "TE gpio request failed: %d", err);
+		return err;
 	}
-	ddata->reset_gpio = gpio;
 
-	gpio = of_get_named_gpio(node, "te-gpios", 0);
-	if (gpio_is_valid(gpio) || gpio == -ENOENT) {
-		ddata->ext_te_gpio = gpio;
+	err = of_get_display_timing(node, "panel-timing", &timing);
+	if (!err) {
+		videomode_from_timing(&timing, &ddata->vm);
+		if (!ddata->vm.pixelclock)
+			ddata->vm.pixelclock =
+				ddata->vm.hactive * ddata->vm.vactive * 60;
 	} else {
-		dev_err(&pdev->dev, "failed to parse TE gpio\n");
-		return gpio;
+		dev_warn(&pdev->dev,
+			 "failed to get video timing, using defaults\n");
 	}
 
+	ddata->width_mm = 0;
+	of_property_read_u32(node, "width-mm", &ddata->width_mm);
+
+	ddata->height_mm = 0;
+	of_property_read_u32(node, "height-mm", &ddata->height_mm);
+
 	in = omapdss_of_find_source_for_first_ep(node);
 	if (IS_ERR(in)) {
 		dev_err(&pdev->dev, "failed to find video source\n");
 		return PTR_ERR(in);
 	}
 
+	ddata->vpnl = devm_regulator_get_optional(&pdev->dev, "vpnl");
+	if (IS_ERR(ddata->vpnl)) {
+		err = PTR_ERR(ddata->vpnl);
+		if (err == -EPROBE_DEFER)
+			return err;
+		ddata->vpnl = NULL;
+	}
+
+	ddata->vddi = devm_regulator_get_optional(&pdev->dev, "vddi");
+	if (IS_ERR(ddata->vddi)) {
+		err = PTR_ERR(ddata->vddi);
+		if (err == -EPROBE_DEFER)
+			return err;
+		ddata->vddi = NULL;
+	}
+
 	ddata->in = in;
 
-	/* TODO: ulps, backlight */
+	backlight = of_parse_phandle(node, "backlight", 0);
+	if (backlight) {
+		ddata->extbldev = of_find_backlight_by_node(backlight);
+		of_node_put(backlight);
+
+		if (!ddata->extbldev)
+			return -EPROBE_DEFER;
+	} else {
+		/* assume native backlight support */
+		ddata->use_dsi_backlight = true;
+	}
+
+	/* TODO: ulps */
 
 	return 0;
 }
 
 static int dsicm_probe(struct platform_device *pdev)
 {
-	struct backlight_properties props;
 	struct panel_drv_data *ddata;
 	struct backlight_device *bldev = NULL;
 	struct device *dev = &pdev->dev;
@@ -1171,14 +1320,14 @@ static int dsicm_probe(struct platform_device *pdev)
 	if (!pdev->dev.of_node)
 		return -ENODEV;
 
-	r = dsicm_probe_of(pdev);
-	if (r)
-		return r;
-
 	ddata->vm.hactive = 864;
 	ddata->vm.vactive = 480;
 	ddata->vm.pixelclock = 864 * 480 * 60;
 
+	r = dsicm_probe_of(pdev);
+	if (r)
+		return r;
+
 	dssdev = &ddata->dssdev;
 	dssdev->dev = dev;
 	dssdev->driver = &dsicm_ops;
@@ -1200,31 +1349,15 @@ static int dsicm_probe(struct platform_device *pdev)
 
 	atomic_set(&ddata->do_update, 0);
 
-	if (gpio_is_valid(ddata->reset_gpio)) {
-		r = devm_gpio_request_one(dev, ddata->reset_gpio,
-				GPIOF_OUT_INIT_LOW, "taal rst");
-		if (r) {
-			dev_err(dev, "failed to request reset gpio\n");
-			return r;
-		}
-	}
-
-	if (gpio_is_valid(ddata->ext_te_gpio)) {
-		r = devm_gpio_request_one(dev, ddata->ext_te_gpio,
-				GPIOF_IN, "taal irq");
-		if (r) {
-			dev_err(dev, "GPIO request failed\n");
-			return r;
-		}
-
-		r = devm_request_irq(dev, gpio_to_irq(ddata->ext_te_gpio),
+	if (ddata->ext_te_gpio) {
+		r = devm_request_irq(dev, gpiod_to_irq(ddata->ext_te_gpio),
 				dsicm_te_isr,
 				IRQF_TRIGGER_RISING,
 				"taal vsync", ddata);
 
 		if (r) {
 			dev_err(dev, "IRQ request failed\n");
-			return r;
+			goto err_reg;
 		}
 
 		INIT_DEFERRABLE_WORK(&ddata->te_timeout_work,
@@ -1234,48 +1367,43 @@ static int dsicm_probe(struct platform_device *pdev)
 	}
 
 	ddata->workqueue = create_singlethread_workqueue("dsicm_wq");
-	if (ddata->workqueue == NULL) {
-		dev_err(dev, "can't create workqueue\n");
-		return -ENOMEM;
+	if (!ddata->workqueue) {
+		r = -ENOMEM;
+		goto err_reg;
 	}
 	INIT_DELAYED_WORK(&ddata->ulps_work, dsicm_ulps_work);
 
 	dsicm_hw_reset(ddata);
 
 	if (ddata->use_dsi_backlight) {
-		memset(&props, 0, sizeof(props));
+		struct backlight_properties props = { 0 };
 		props.max_brightness = 255;
-
 		props.type = BACKLIGHT_RAW;
-		bldev = backlight_device_register(dev_name(dev),
-				dev, ddata, &dsicm_bl_ops, &props);
+
+		bldev = devm_backlight_device_register(dev, dev_name(dev),
+			dev, ddata, &dsicm_bl_ops, &props);
 		if (IS_ERR(bldev)) {
 			r = PTR_ERR(bldev);
 			goto err_bl;
 		}
 
 		ddata->bldev = bldev;
-
-		bldev->props.fb_blank = FB_BLANK_UNBLANK;
-		bldev->props.power = FB_BLANK_UNBLANK;
-		bldev->props.brightness = 255;
-
-		dsicm_bl_update_status(bldev);
 	}
 
 	r = sysfs_create_group(&dev->kobj, &dsicm_attr_group);
 	if (r) {
 		dev_err(dev, "failed to create sysfs files\n");
-		goto err_sysfs_create;
+		goto err_bl;
 	}
 
 	return 0;
 
-err_sysfs_create:
-	backlight_device_unregister(bldev);
 err_bl:
 	destroy_workqueue(ddata->workqueue);
 err_reg:
+	if (ddata->extbldev)
+		put_device(&ddata->extbldev->dev);
+
 	return r;
 }
 
@@ -1283,7 +1411,6 @@ static int __exit dsicm_remove(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
 	struct omap_dss_device *dssdev = &ddata->dssdev;
-	struct backlight_device *bldev;
 
 	dev_dbg(&pdev->dev, "remove\n");
 
@@ -1294,12 +1421,8 @@ static int __exit dsicm_remove(struct platform_device *pdev)
 
 	sysfs_remove_group(&pdev->dev.kobj, &dsicm_attr_group);
 
-	bldev = ddata->bldev;
-	if (bldev != NULL) {
-		bldev->props.power = FB_BLANK_POWERDOWN;
-		dsicm_bl_update_status(bldev);
-		backlight_device_unregister(bldev);
-	}
+	if (ddata->extbldev)
+		put_device(&ddata->extbldev->dev);
 
 	omap_dss_put_device(ddata->in);
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
index 74d13969b9ca..57af22ce87c5 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
@@ -1,7 +1,7 @@
 /*
  * LG.Philips LB035Q02 LCD Panel driver
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  * Based on a driver by: Steve Sakoman <steve@sakoman.com>
  *
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
index df8132d3b9c6..bf53676263ad 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
@@ -1,7 +1,7 @@
 /*
  * NEC NL8048HL11 Panel driver
  *
- * Copyright (C) 2010 Texas Instruments Inc.
+ * Copyright (C) 2010 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Erik Gilling <konkers@android.com>
  * Converted to new DSS device model: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
index 98d170aecaba..34555801fa4c 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
@@ -1,7 +1,7 @@
 /*
  * LCD panel driver for Sharp LS037V7DW01
  *
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
index 0a38a0e8c925..2721a86ac5e7 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
@@ -452,15 +452,27 @@ static int td028ttec1_panel_remove(struct spi_device *spi)
 }
 
 static const struct of_device_id td028ttec1_of_match[] = {
+	{ .compatible = "omapdss,tpo,td028ttec1", },
+	/* keep to not break older DTB */
 	{ .compatible = "omapdss,toppoly,td028ttec1", },
 	{},
 };
 
 MODULE_DEVICE_TABLE(of, td028ttec1_of_match);
 
+static const struct spi_device_id td028ttec1_ids[] = {
+	{ "toppoly,td028ttec1", 0 },
+	{ "tpo,td028ttec1", 0},
+	{ /* sentinel */ }
+};
+
+MODULE_DEVICE_TABLE(spi, td028ttec1_ids);
+
+
 static struct spi_driver td028ttec1_spi_driver = {
 	.probe		= td028ttec1_panel_probe,
 	.remove		= td028ttec1_panel_remove,
+	.id_table	= td028ttec1_ids,
 
 	.driver         = {
 		.name   = "panel-tpo-td028ttec1",
@@ -471,7 +483,6 @@ static struct spi_driver td028ttec1_spi_driver = {
 
 module_spi_driver(td028ttec1_spi_driver);
 
-MODULE_ALIAS("spi:toppoly,td028ttec1");
 MODULE_AUTHOR("H. Nikolaus Schaller <hns@goldelico.com>");
 MODULE_DESCRIPTION("Toppoly TD028TTEC1 panel driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/omapdrm/dss/base.c b/drivers/gpu/drm/omapdrm/dss/base.c
index 13e91faaf7a6..67cc87a4f1f6 100644
--- a/drivers/gpu/drm/omapdrm/dss/base.c
+++ b/drivers/gpu/drm/omapdrm/dss/base.c
@@ -1,3 +1,18 @@
+/*
+ * OMAP Display Subsystem Base
+ *
+ * Copyright (C) 2015-2017 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
diff --git a/drivers/gpu/drm/omapdrm/dss/core.c b/drivers/gpu/drm/omapdrm/dss/core.c
index 197ddbc1512b..acef7ece5783 100644
--- a/drivers/gpu/drm/omapdrm/dss/core.c
+++ b/drivers/gpu/drm/omapdrm/dss/core.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/core.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -30,38 +28,21 @@
 #include "dss.h"
 
 /* INIT */
-static int (*dss_output_drv_reg_funcs[])(void) __initdata = {
-	dss_init_platform_driver,
-	dispc_init_platform_driver,
+static struct platform_driver * const omap_dss_drivers[] = {
+	&omap_dsshw_driver,
+	&omap_dispchw_driver,
 #ifdef CONFIG_OMAP2_DSS_DSI
-	dsi_init_platform_driver,
+	&omap_dsihw_driver,
 #endif
 #ifdef CONFIG_OMAP2_DSS_VENC
-	venc_init_platform_driver,
+	&omap_venchw_driver,
 #endif
 #ifdef CONFIG_OMAP4_DSS_HDMI
-	hdmi4_init_platform_driver,
+	&omapdss_hdmi4hw_driver,
 #endif
 #ifdef CONFIG_OMAP5_DSS_HDMI
-	hdmi5_init_platform_driver,
-#endif
-};
-
-static void (*dss_output_drv_unreg_funcs[])(void) = {
-#ifdef CONFIG_OMAP5_DSS_HDMI
-	hdmi5_uninit_platform_driver,
-#endif
-#ifdef CONFIG_OMAP4_DSS_HDMI
-	hdmi4_uninit_platform_driver,
+	&omapdss_hdmi5hw_driver,
 #endif
-#ifdef CONFIG_OMAP2_DSS_VENC
-	venc_uninit_platform_driver,
-#endif
-#ifdef CONFIG_OMAP2_DSS_DSI
-	dsi_uninit_platform_driver,
-#endif
-	dispc_uninit_platform_driver,
-	dss_uninit_platform_driver,
 };
 
 static struct platform_device *omap_drm_device;
@@ -69,13 +50,11 @@ static struct platform_device *omap_drm_device;
 static int __init omap_dss_init(void)
 {
 	int r;
-	int i;
 
-	for (i = 0; i < ARRAY_SIZE(dss_output_drv_reg_funcs); ++i) {
-		r = dss_output_drv_reg_funcs[i]();
-		if (r)
-			goto err_reg;
-	}
+	r = platform_register_drivers(omap_dss_drivers,
+				      ARRAY_SIZE(omap_dss_drivers));
+	if (r)
+		goto err_reg;
 
 	omap_drm_device = platform_device_register_simple("omapdrm", 0, NULL, 0);
 	if (IS_ERR(omap_drm_device)) {
@@ -86,22 +65,18 @@ static int __init omap_dss_init(void)
 	return 0;
 
 err_reg:
-	for (i = ARRAY_SIZE(dss_output_drv_reg_funcs) - i;
-			i < ARRAY_SIZE(dss_output_drv_reg_funcs);
-			++i)
-		dss_output_drv_unreg_funcs[i]();
+	platform_unregister_drivers(omap_dss_drivers,
+				    ARRAY_SIZE(omap_dss_drivers));
 
 	return r;
 }
 
 static void __exit omap_dss_exit(void)
 {
-	int i;
-
 	platform_device_unregister(omap_drm_device);
 
-	for (i = 0; i < ARRAY_SIZE(dss_output_drv_unreg_funcs); ++i)
-		dss_output_drv_unreg_funcs[i]();
+	platform_unregister_drivers(omap_dss_drivers,
+				    ARRAY_SIZE(omap_dss_drivers));
 }
 
 module_init(omap_dss_init);
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c
index 0f4fdb221498..4e8f68efd169 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/dispc.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -4325,6 +4323,17 @@ static void dispc_free_irq(void *dev_id)
 	dispc.user_data = NULL;
 }
 
+static u32 dispc_get_memory_bandwidth_limit(void)
+{
+	u32 limit = 0;
+
+	/* Optional maximum memory bandwidth */
+	of_property_read_u32(dispc.pdev->dev.of_node, "max-memory-bandwidth",
+			     &limit);
+
+	return limit;
+}
+
 /*
  * Workaround for errata i734 in DSS dispc
  *  - LCD1 Gamma Correction Is Not Working When GFX Pipe Is Disabled
@@ -4497,6 +4506,8 @@ static const struct dispc_ops dispc_ops = {
 	.get_num_ovls = dispc_get_num_ovls,
 	.get_num_mgrs = dispc_get_num_mgrs,
 
+	.get_memory_bandwidth_limit = dispc_get_memory_bandwidth_limit,
+
 	.mgr_enable = dispc_mgr_enable,
 	.mgr_is_enabled = dispc_mgr_is_enabled,
 	.mgr_get_vsync_irq = dispc_mgr_get_vsync_irq,
@@ -4685,7 +4696,7 @@ static const struct dev_pm_ops dispc_pm_ops = {
 	.runtime_resume = dispc_runtime_resume,
 };
 
-static struct platform_driver omap_dispchw_driver = {
+struct platform_driver omap_dispchw_driver = {
 	.probe		= dispc_probe,
 	.remove         = dispc_remove,
 	.driver         = {
@@ -4695,13 +4706,3 @@ static struct platform_driver omap_dispchw_driver = {
 		.suppress_bind_attrs = true,
 	},
 };
-
-int __init dispc_init_platform_driver(void)
-{
-	return platform_driver_register(&omap_dispchw_driver);
-}
-
-void dispc_uninit_platform_driver(void)
-{
-	platform_driver_unregister(&omap_dispchw_driver);
-}
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.h b/drivers/gpu/drm/omapdrm/dss/dispc.h
index 003adce532f4..e901dd1e4365 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc.h
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.h
@@ -1,10 +1,7 @@
 /*
- * linux/drivers/video/omap2/dss/dispc.h
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Archit Taneja <archit@ti.com>
  *
- *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
  * the Free Software Foundation.
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c b/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
index 34fad2376f8d..44804c8c8777 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
@@ -1,7 +1,5 @@
 /*
- * linux/drivers/video/omap2/dss/dispc_coefs.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Chandrabhanu Mahapatra <cmahapatra@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/dss/display.c b/drivers/gpu/drm/omapdrm/dss/display.c
index 42279933790e..0c9480ba85c0 100644
--- a/drivers/gpu/drm/omapdrm/dss/display.c
+++ b/drivers/gpu/drm/omapdrm/dss/display.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/display.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -175,17 +173,3 @@ struct omap_dss_device *omap_dss_get_next_device(struct omap_dss_device *from)
 	return dssdev;
 }
 EXPORT_SYMBOL(omap_dss_get_next_device);
-
-struct omap_dss_device *omap_dss_find_device(void *data,
-		int (*match)(struct omap_dss_device *dssdev, void *data))
-{
-	struct omap_dss_device *dssdev = NULL;
-
-	while ((dssdev = omap_dss_get_next_device(dssdev)) != NULL) {
-		if (match(dssdev, data))
-			return dssdev;
-	}
-
-	return NULL;
-}
-EXPORT_SYMBOL(omap_dss_find_device);
diff --git a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
index ca1e3b489540..ea44137ed08c 100644
--- a/drivers/gpu/drm/omapdrm/dss/dpi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/dpi.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -52,8 +50,6 @@ struct dpi_data {
 	int data_lines;
 
 	struct omap_dss_device output;
-
-	bool port_initialized;
 };
 
 static struct dpi_data *dpi_get_data_from_dssdev(struct omap_dss_device *dssdev)
@@ -786,8 +782,6 @@ int dpi_init_port(struct platform_device *pdev, struct device_node *port,
 
 	dpi_init_output_port(dpi, port);
 
-	dpi->port_initialized = true;
-
 	return 0;
 
 err_datalines:
@@ -800,7 +794,7 @@ void dpi_uninit_port(struct device_node *port)
 {
 	struct dpi_data *dpi = port->data;
 
-	if (!dpi->port_initialized)
+	if (!dpi)
 		return;
 
 	dpi_uninit_output_port(port);
diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index c2cf6d98e577..80f1f3679a3c 100644
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/dsi.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -5660,7 +5658,7 @@ static const struct dev_pm_ops dsi_pm_ops = {
 	.runtime_resume = dsi_runtime_resume,
 };
 
-static struct platform_driver omap_dsihw_driver = {
+struct platform_driver omap_dsihw_driver = {
 	.probe		= dsi_probe,
 	.remove		= dsi_remove,
 	.driver         = {
@@ -5670,13 +5668,3 @@ static struct platform_driver omap_dsihw_driver = {
 		.suppress_bind_attrs = true,
 	},
 };
-
-int __init dsi_init_platform_driver(void)
-{
-	return platform_driver_register(&omap_dsihw_driver);
-}
-
-void dsi_uninit_platform_driver(void)
-{
-	platform_driver_unregister(&omap_dsihw_driver);
-}
diff --git a/drivers/gpu/drm/omapdrm/dss/dss-of.c b/drivers/gpu/drm/omapdrm/dss/dss-of.c
index c6b86f348a5c..967d9e1b34e5 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss-of.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss-of.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Texas Instruments
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.c b/drivers/gpu/drm/omapdrm/dss/dss.c
index d1755f12236b..04300b2da1b1 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/dss.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -23,6 +21,7 @@
 #define DSS_SUBSYS_NAME "DSS"
 
 #include <linux/debugfs.h>
+#include <linux/dma-mapping.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/io.h>
@@ -367,7 +366,8 @@ const char *dss_get_clk_source_name(enum dss_clk_source clk_src)
 	return dss_generic_clk_source_names[clk_src];
 }
 
-void dss_dump_clocks(struct seq_file *s)
+#if defined(CONFIG_OMAP2_DSS_DEBUGFS)
+static void dss_dump_clocks(struct seq_file *s)
 {
 	const char *fclk_name;
 	unsigned long fclk_rate;
@@ -386,6 +386,7 @@ void dss_dump_clocks(struct seq_file *s)
 
 	dss_runtime_put();
 }
+#endif
 
 static void dss_dump_regs(struct seq_file *s)
 {
@@ -1441,6 +1442,12 @@ static int dss_probe(struct platform_device *pdev)
 
 	dss.pdev = pdev;
 
+	r = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
+	if (r) {
+		dev_err(&pdev->dev, "Failed to set the DMA mask\n");
+		return r;
+	}
+
 	/*
 	 * The various OMAP3-based SoCs can't be told apart using the compatible
 	 * string, use SoC device matching.
@@ -1527,7 +1534,7 @@ static const struct dev_pm_ops dss_pm_ops = {
 	.runtime_resume = dss_runtime_resume,
 };
 
-static struct platform_driver omap_dsshw_driver = {
+struct platform_driver omap_dsshw_driver = {
 	.probe		= dss_probe,
 	.remove		= dss_remove,
 	.shutdown	= dss_shutdown,
@@ -1538,13 +1545,3 @@ static struct platform_driver omap_dsshw_driver = {
 		.suppress_bind_attrs = true,
 	},
 };
-
-int __init dss_init_platform_driver(void)
-{
-	return platform_driver_register(&omap_dsshw_driver);
-}
-
-void dss_uninit_platform_driver(void)
-{
-	platform_driver_unregister(&omap_dsshw_driver);
-}
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.h b/drivers/gpu/drm/omapdrm/dss/dss.h
index ed465572491e..6374e57ed9da 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.h
+++ b/drivers/gpu/drm/omapdrm/dss/dss.h
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/dss.h
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -264,9 +262,6 @@ static inline int dss_debugfs_create_file(const char *name,
 }
 #endif /* CONFIG_OMAP2_DSS_DEBUGFS */
 
-int dss_init_platform_driver(void) __init;
-void dss_uninit_platform_driver(void);
-
 int dss_runtime_get(void);
 void dss_runtime_put(void);
 
@@ -277,7 +272,6 @@ int dss_dpi_select_source(int port, enum omap_channel channel);
 void dss_select_hdmi_venc_clk_source(enum dss_hdmi_venc_clk_source_select);
 enum dss_hdmi_venc_clk_source_select dss_get_hdmi_venc_clk_source(void);
 const char *dss_get_clk_source_name(enum dss_clk_source clk_src);
-void dss_dump_clocks(struct seq_file *s);
 
 /* DSS VIDEO PLL */
 struct dss_pll *dss_video_pll_init(struct platform_device *pdev, int id,
@@ -329,9 +323,6 @@ static inline void sdi_uninit_port(struct device_node *port)
 struct dentry;
 struct file_operations;
 
-int dsi_init_platform_driver(void) __init;
-void dsi_uninit_platform_driver(void);
-
 void dsi_dump_clocks(struct seq_file *s);
 
 void dsi_irq_handler(void);
@@ -355,8 +346,6 @@ static inline void dpi_uninit_port(struct device_node *port)
 #endif
 
 /* DISPC */
-int dispc_init_platform_driver(void) __init;
-void dispc_uninit_platform_driver(void);
 void dispc_dump_clocks(struct seq_file *s);
 
 int dispc_runtime_get(void);
@@ -400,18 +389,6 @@ void dispc_wb_set_channel_in(enum dss_writeback_channel channel);
 int dispc_wb_setup(const struct omap_dss_writeback_info *wi,
 		bool mem_to_mem, const struct videomode *vm);
 
-/* VENC */
-int venc_init_platform_driver(void) __init;
-void venc_uninit_platform_driver(void);
-
-/* HDMI */
-int hdmi4_init_platform_driver(void) __init;
-void hdmi4_uninit_platform_driver(void);
-
-int hdmi5_init_platform_driver(void) __init;
-void hdmi5_uninit_platform_driver(void);
-
-
 #ifdef CONFIG_OMAP2_DSS_COLLECT_IRQ_STATS
 static inline void dss_collect_irq_stats(u32 irqstatus, unsigned *irq_arr)
 {
@@ -455,4 +432,19 @@ int dss_pll_write_config_type_b(struct dss_pll *pll,
 		const struct dss_pll_clock_info *cinfo);
 int dss_pll_wait_reset_done(struct dss_pll *pll);
 
+extern struct platform_driver omap_dsshw_driver;
+extern struct platform_driver omap_dispchw_driver;
+#ifdef CONFIG_OMAP2_DSS_DSI
+extern struct platform_driver omap_dsihw_driver;
+#endif
+#ifdef CONFIG_OMAP2_DSS_VENC
+extern struct platform_driver omap_venchw_driver;
+#endif
+#ifdef CONFIG_OMAP4_DSS_HDMI
+extern struct platform_driver omapdss_hdmi4hw_driver;
+#endif
+#ifdef CONFIG_OMAP5_DSS_HDMI
+extern struct platform_driver omapdss_hdmi5hw_driver;
+#endif
+
 #endif
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index a598dfdeb585..bf914f2ac99e 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -1,5 +1,6 @@
 /*
  * HDMI interface DSS driver for TI's OMAP4 family of SoCs.
+ *
  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com/
  * Authors: Yong Zhi
  *	Mythri pk <mythripk@ti.com>
@@ -844,7 +845,7 @@ static const struct of_device_id hdmi_of_match[] = {
 	{},
 };
 
-static struct platform_driver omapdss_hdmihw_driver = {
+struct platform_driver omapdss_hdmi4hw_driver = {
 	.probe		= hdmi4_probe,
 	.remove		= hdmi4_remove,
 	.driver         = {
@@ -854,13 +855,3 @@ static struct platform_driver omapdss_hdmihw_driver = {
 		.suppress_bind_attrs = true,
 	},
 };
-
-int __init hdmi4_init_platform_driver(void)
-{
-	return platform_driver_register(&omapdss_hdmihw_driver);
-}
-
-void hdmi4_uninit_platform_driver(void)
-{
-	platform_driver_unregister(&omapdss_hdmihw_driver);
-}
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
index b06f9956e733..35ed2add6189 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c
@@ -1,7 +1,6 @@
 /*
- * ti_hdmi_4xxx_ip.c
- *
  * HDMI TI81xx, TI38xx, TI OMAP4 etc IP driver Library
+ *
  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com/
  * Authors: Yong Zhi
  *	Mythri pk <mythripk@ti.com>
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index b3221ca5bcd8..689cda41858b 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -1,7 +1,7 @@
 /*
  * HDMI driver for OMAP5
  *
- * Copyright (C) 2014 Texas Instruments Incorporated
+ * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com/
  *
  * Authors:
  *	Yong Zhi
@@ -841,7 +841,7 @@ static const struct of_device_id hdmi_of_match[] = {
 	{},
 };
 
-static struct platform_driver omapdss_hdmihw_driver = {
+struct platform_driver omapdss_hdmi5hw_driver = {
 	.probe		= hdmi5_probe,
 	.remove		= hdmi5_remove,
 	.driver         = {
@@ -851,13 +851,3 @@ static struct platform_driver omapdss_hdmihw_driver = {
 		.suppress_bind_attrs = true,
 	},
 };
-
-int __init hdmi5_init_platform_driver(void)
-{
-	return platform_driver_register(&omapdss_hdmihw_driver);
-}
-
-void hdmi5_uninit_platform_driver(void)
-{
-	platform_driver_unregister(&omapdss_hdmihw_driver);
-}
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
index ab179ec133c0..09759f8ea7bc 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c
@@ -1,8 +1,7 @@
 /*
  * OMAP5 HDMI CORE IP driver library
  *
- * Copyright (C) 2014 Texas Instruments Incorporated
- *
+ * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com/
  * Authors:
  *	Yong Zhi
  *	Mythri pk
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
index a156292b1820..5c14ed851609 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
@@ -1,7 +1,7 @@
 /*
  * HDMI PHY
  *
- * Copyright (C) 2013 Texas Instruments Incorporated
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
index 55bee81f4dd5..08885d7de1e8 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
@@ -1,7 +1,7 @@
 /*
  * HDMI PLL
  *
- * Copyright (C) 2013 Texas Instruments Incorporated
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
index 88034fbe0e9f..806e5fdcfe52 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
@@ -1,7 +1,7 @@
 /*
  * HDMI wrapper
  *
- * Copyright (C) 2013 Texas Instruments Incorporated
+ * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
index bf626acae271..3bfb95d230e0 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2014 Texas Instruments
+ * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h
index 990422b35784..f8f83e826a56 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss.h
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Texas Instruments
+ * Copyright (C) 2016 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -563,6 +563,8 @@ struct omap_dss_driver {
 			    struct videomode *vm);
 	void (*get_timings)(struct omap_dss_device *dssdev,
 			    struct videomode *vm);
+	void (*get_size)(struct omap_dss_device *dssdev,
+			 unsigned int *width, unsigned int *height);
 
 	int (*set_wss)(struct omap_dss_device *dssdev, u32 wss);
 	u32 (*get_wss)(struct omap_dss_device *dssdev);
@@ -585,9 +587,6 @@ struct omap_dss_driver {
 
 bool omapdss_is_initialized(void);
 
-int omap_dss_register_driver(struct omap_dss_driver *);
-void omap_dss_unregister_driver(struct omap_dss_driver *);
-
 int omapdss_register_display(struct omap_dss_device *dssdev);
 void omapdss_unregister_display(struct omap_dss_device *dssdev);
 
@@ -595,9 +594,6 @@ struct omap_dss_device *omap_dss_get_device(struct omap_dss_device *dssdev);
 void omap_dss_put_device(struct omap_dss_device *dssdev);
 #define for_each_dss_dev(d) while ((d = omap_dss_get_next_device(d)) != NULL)
 struct omap_dss_device *omap_dss_get_next_device(struct omap_dss_device *from);
-struct omap_dss_device *omap_dss_find_device(void *data,
-		int (*match)(struct omap_dss_device *dssdev, void *data));
-
 
 int omap_dss_get_num_overlay_managers(void);
 
@@ -695,6 +691,8 @@ struct dispc_ops {
 	int (*get_num_ovls)(void);
 	int (*get_num_mgrs)(void);
 
+	u32 (*get_memory_bandwidth_limit)(void);
+
 	void (*mgr_enable)(enum omap_channel channel, bool enable);
 	bool (*mgr_is_enabled)(enum omap_channel channel);
 	u32 (*mgr_get_vsync_irq)(enum omap_channel channel);
diff --git a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
index 3c572b699ed3..b9afd80ae385 100644
--- a/drivers/gpu/drm/omapdrm/dss/output.c
+++ b/drivers/gpu/drm/omapdrm/dss/output.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2012 Texas Instruments Ltd
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Archit Taneja <archit@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/dss/pll.c b/drivers/gpu/drm/omapdrm/dss/pll.c
index 9d9d9d42009b..058714b1eb56 100644
--- a/drivers/gpu/drm/omapdrm/dss/pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/pll.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2014 Texas Instruments Incorporated
+ * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com/
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published by
diff --git a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
index d18ad58c5a19..d8ab31f3a813 100644
--- a/drivers/gpu/drm/omapdrm/dss/sdi.c
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/sdi.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
index d58da6f32693..6de9d734ddb9 100644
--- a/drivers/gpu/drm/omapdrm/dss/venc.c
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c
@@ -1,6 +1,4 @@
 /*
- * linux/drivers/video/omap2/dss/venc.c
- *
  * Copyright (C) 2009 Nokia Corporation
  * Author: Tomi Valkeinen <tomi.valkeinen@nokia.com>
  *
@@ -857,10 +855,10 @@ static int venc_probe_of(struct platform_device *pdev)
 	of_node_put(ep);
 
 	return 0;
+
 err:
 	of_node_put(ep);
-
-	return 0;
+	return r;
 }
 
 /* VENC HW IP initialisation */
@@ -986,7 +984,7 @@ static const struct of_device_id venc_of_match[] = {
 	{},
 };
 
-static struct platform_driver omap_venchw_driver = {
+struct platform_driver omap_venchw_driver = {
 	.probe		= venc_probe,
 	.remove		= venc_remove,
 	.driver         = {
@@ -996,13 +994,3 @@ static struct platform_driver omap_venchw_driver = {
 		.suppress_bind_attrs = true,
 	},
 };
-
-int __init venc_init_platform_driver(void)
-{
-	return platform_driver_register(&omap_venchw_driver);
-}
-
-void venc_uninit_platform_driver(void)
-{
-	platform_driver_unregister(&omap_venchw_driver);
-}
diff --git a/drivers/gpu/drm/omapdrm/dss/video-pll.c b/drivers/gpu/drm/omapdrm/dss/video-pll.c
index 38a239cc5e04..bbedac797927 100644
--- a/drivers/gpu/drm/omapdrm/dss/video-pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/video-pll.c
@@ -1,13 +1,15 @@
 /*
-* Copyright (C) 2014 Texas Instruments Ltd
-*
-* This program is free software; you can redistribute it and/or modify it
-* under the terms of the GNU General Public License version 2 as published by
-* the Free Software Foundation.
-*
-* You should have received a copy of the GNU General Public License along with
-* this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
+ * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com/
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
 
 #include <linux/clk.h>
 #include <linux/delay.h>
diff --git a/drivers/gpu/drm/omapdrm/omap_connector.c b/drivers/gpu/drm/omapdrm/omap_connector.c
index aa5ba9ae2191..a0d7b1d905e8 100644
--- a/drivers/gpu/drm/omapdrm/omap_connector.c
+++ b/drivers/gpu/drm/omapdrm/omap_connector.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_connector.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -149,6 +147,12 @@ static int omap_connector_get_modes(struct drm_connector *connector)
 		drm_mode_set_name(mode);
 		drm_mode_probed_add(connector, mode);
 
+		if (dssdrv->get_size) {
+			dssdrv->get_size(dssdev,
+					 &connector->display_info.width_mm,
+					 &connector->display_info.height_mm);
+		}
+
 		n = 1;
 	}
 
diff --git a/drivers/gpu/drm/omapdrm/omap_connector.h b/drivers/gpu/drm/omapdrm/omap_connector.h
new file mode 100644
index 000000000000..98bbc779b302
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_connector.h
@@ -0,0 +1,37 @@
+/*
+ * omap_connector.h -- OMAP DRM Connector
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_CONNECTOR_H__
+#define __OMAPDRM_CONNECTOR_H__
+
+#include <linux/types.h>
+
+struct drm_connector;
+struct drm_device;
+struct drm_encoder;
+struct omap_dss_device;
+
+struct drm_connector *omap_connector_init(struct drm_device *dev,
+		int connector_type, struct omap_dss_device *dssdev,
+		struct drm_encoder *encoder);
+struct drm_encoder *omap_connector_attached_encoder(
+		struct drm_connector *connector);
+bool omap_connector_get_hdmi_mode(struct drm_connector *connector);
+
+#endif /* __OMAPDRM_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index cc85c16cbc2a..1b8154e58d18 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_crtc.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -23,6 +21,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_mode.h>
 #include <drm/drm_plane_helper.h>
+#include <linux/math64.h>
 
 #include "omap_drv.h"
 
@@ -400,6 +399,41 @@ static void omap_crtc_atomic_disable(struct drm_crtc *crtc,
 	drm_crtc_vblank_off(crtc);
 }
 
+static enum drm_mode_status omap_crtc_mode_valid(struct drm_crtc *crtc,
+					const struct drm_display_mode *mode)
+{
+	struct omap_drm_private *priv = crtc->dev->dev_private;
+
+	/* Check for bandwidth limit */
+	if (priv->max_bandwidth) {
+		/*
+		 * Estimation for the bandwidth need of a given mode with one
+		 * full screen plane:
+		 * bandwidth = resolution * 32bpp * (pclk / (vtotal * htotal))
+		 *					^^ Refresh rate ^^
+		 *
+		 * The interlaced mode is taken into account by using the
+		 * pixelclock in the calculation.
+		 *
+		 * The equation is rearranged for 64bit arithmetic.
+		 */
+		uint64_t bandwidth = mode->clock * 1000;
+		unsigned int bpp = 4;
+
+		bandwidth = bandwidth * mode->hdisplay * mode->vdisplay * bpp;
+		bandwidth = div_u64(bandwidth, mode->htotal * mode->vtotal);
+
+		/*
+		 * Reject modes which would need more bandwidth if used with one
+		 * full resolution plane (most common use case).
+		 */
+		if (priv->max_bandwidth < bandwidth)
+			return MODE_BAD;
+	}
+
+	return MODE_OK;
+}
+
 static void omap_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
@@ -621,6 +655,7 @@ static const struct drm_crtc_helper_funcs omap_crtc_helper_funcs = {
 	.atomic_flush = omap_crtc_atomic_flush,
 	.atomic_enable = omap_crtc_atomic_enable,
 	.atomic_disable = omap_crtc_atomic_disable,
+	.mode_valid = omap_crtc_mode_valid,
 };
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.h b/drivers/gpu/drm/omapdrm/omap_crtc.h
new file mode 100644
index 000000000000..ad7b007c6174
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.h
@@ -0,0 +1,43 @@
+/*
+ * omap_crtc.h -- OMAP DRM CRTC
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_CRTC_H__
+#define __OMAPDRM_CRTC_H__
+
+#include <linux/types.h>
+
+enum omap_channel;
+
+struct drm_crtc;
+struct drm_device;
+struct drm_plane;
+struct omap_dss_device;
+struct videomode;
+
+struct videomode *omap_crtc_timings(struct drm_crtc *crtc);
+enum omap_channel omap_crtc_channel(struct drm_crtc *crtc);
+void omap_crtc_pre_init(void);
+void omap_crtc_pre_uninit(void);
+struct drm_crtc *omap_crtc_init(struct drm_device *dev,
+		struct drm_plane *plane, struct omap_dss_device *dssdev);
+int omap_crtc_wait_pending(struct drm_crtc *crtc);
+void omap_crtc_error_irq(struct drm_crtc *crtc, uint32_t irqstatus);
+void omap_crtc_vblank_irq(struct drm_crtc *crtc);
+
+#endif /* __OMAPDRM_CRTC_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_debugfs.c b/drivers/gpu/drm/omapdrm/omap_debugfs.c
index 19b716745623..b42e286616b0 100644
--- a/drivers/gpu/drm/omapdrm/omap_debugfs.c
+++ b/drivers/gpu/drm/omapdrm/omap_debugfs.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_debugfs.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob.clark@linaro.org>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/omap_dmm_priv.h b/drivers/gpu/drm/omapdrm/omap_dmm_priv.h
index 9f32a83ca507..600064d5c25b 100644
--- a/drivers/gpu/drm/omapdrm/omap_dmm_priv.h
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_priv.h
@@ -1,5 +1,4 @@
 /*
- *
  * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *         Andy Gross <andy.gross@ti.com>
@@ -13,6 +12,7 @@
  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  */
+
 #ifndef OMAP_DMM_PRIV_H
 #define OMAP_DMM_PRIV_H
 
@@ -59,12 +59,12 @@
 #define DMM_IRQSTAT_ERR_UPD_DATA	(1<<6)
 #define DMM_IRQSTAT_ERR_LUT_MISS	(1<<7)
 
-#define DMM_IRQSTAT_ERR_MASK	(DMM_IRQ_STAT_ERR_INV_DSC | \
-				DMM_IRQ_STAT_ERR_INV_DATA | \
-				DMM_IRQ_STAT_ERR_UPD_AREA | \
-				DMM_IRQ_STAT_ERR_UPD_CTRL | \
-				DMM_IRQ_STAT_ERR_UPD_DATA | \
-				DMM_IRQ_STAT_ERR_LUT_MISS)
+#define DMM_IRQSTAT_ERR_MASK	(DMM_IRQSTAT_ERR_INV_DSC | \
+				DMM_IRQSTAT_ERR_INV_DATA | \
+				DMM_IRQSTAT_ERR_UPD_AREA | \
+				DMM_IRQSTAT_ERR_UPD_CTRL | \
+				DMM_IRQSTAT_ERR_UPD_DATA | \
+				DMM_IRQSTAT_ERR_LUT_MISS)
 
 #define DMM_PATSTATUS_READY		(1<<0)
 #define DMM_PATSTATUS_VALID		(1<<1)
diff --git a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
index c60a85e82c6d..4be0c94673f5 100644
--- a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
@@ -1,11 +1,10 @@
 /*
  * DMM IOMMU driver support functions for TI OMAP processors.
  *
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *         Andy Gross <andy.gross@ti.com>
  *
- * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
- *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation version 2.
@@ -121,14 +120,22 @@ static int wait_status(struct refill_engine *engine, uint32_t wait_mask)
 	while (true) {
 		r = dmm_read(dmm, reg[PAT_STATUS][engine->id]);
 		err = r & DMM_PATSTATUS_ERR;
-		if (err)
+		if (err) {
+			dev_err(dmm->dev,
+				"%s: error (engine%d). PAT_STATUS: 0x%08x\n",
+				__func__, engine->id, r);
 			return -EFAULT;
+		}
 
 		if ((r & wait_mask) == wait_mask)
 			break;
 
-		if (--i == 0)
+		if (--i == 0) {
+			dev_err(dmm->dev,
+				"%s: timeout (engine%d). PAT_STATUS: 0x%08x\n",
+				__func__, engine->id, r);
 			return -ETIMEDOUT;
+		}
 
 		udelay(1);
 	}
@@ -158,6 +165,11 @@ static irqreturn_t omap_dmm_irq_handler(int irq, void *arg)
 	dmm_write(dmm, status, DMM_PAT_IRQSTATUS);
 
 	for (i = 0; i < dmm->num_engines; i++) {
+		if (status & DMM_IRQSTAT_ERR_MASK)
+			dev_err(dmm->dev,
+				"irq error(engine%d): IRQSTAT 0x%02x\n",
+				i, status & 0xff);
+
 		if (status & DMM_IRQSTAT_LST) {
 			if (dmm->engines[i].async)
 				release_engine(&dmm->engines[i]);
@@ -298,7 +310,12 @@ static int dmm_txn_commit(struct dmm_txn *txn, bool wait)
 				msecs_to_jiffies(100))) {
 			dev_err(dmm->dev, "timed out waiting for done\n");
 			ret = -ETIMEDOUT;
+			goto cleanup;
 		}
+
+		/* Check the engine status before continue */
+		ret = wait_status(engine, DMM_PATSTATUS_READY |
+				  DMM_PATSTATUS_VALID | DMM_PATSTATUS_DONE);
 	}
 
 cleanup:
diff --git a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h
index e83c78372db8..cc78ba4fe6ab 100644
--- a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.h
@@ -1,5 +1,4 @@
 /*
- *
  * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *         Andy Gross <andy.gross@ti.com>
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index cdf5b0601eba..dd68b2556f5b 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_drv.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -46,14 +44,6 @@
  *                 devices
  */
 
-static void omap_fb_output_poll_changed(struct drm_device *dev)
-{
-	struct omap_drm_private *priv = dev->dev_private;
-	DBG("dev=%p", dev);
-	if (priv->fbdev)
-		drm_fb_helper_hotplug_event(priv->fbdev);
-}
-
 static void omap_atomic_wait_for_completion(struct drm_device *dev,
 					    struct drm_atomic_state *old_state)
 {
@@ -132,7 +122,7 @@ static const struct drm_mode_config_helper_funcs omap_mode_config_helper_funcs =
 
 static const struct drm_mode_config_funcs omap_mode_config_funcs = {
 	.fb_create = omap_framebuffer_create,
-	.output_poll_changed = omap_fb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -467,28 +457,6 @@ static int dev_open(struct drm_device *dev, struct drm_file *file)
 	return 0;
 }
 
-/**
- * lastclose - clean up after all DRM clients have exited
- * @dev: DRM device
- *
- * Take care of cleaning up after all DRM clients have exited.  In the
- * mode setting case, we want to restore the kernel's initial mode (just
- * in case the last client left us in a bad state).
- */
-static void dev_lastclose(struct drm_device *dev)
-{
-	struct omap_drm_private *priv = dev->dev_private;
-	int ret;
-
-	DBG("lastclose: dev=%p", dev);
-
-	if (priv->fbdev) {
-		ret = drm_fb_helper_restore_fbdev_mode_unlocked(priv->fbdev);
-		if (ret)
-			DBG("failed to restore crtc mode");
-	}
-}
-
 static const struct vm_operations_struct omap_gem_vm_ops = {
 	.fault = omap_gem_fault,
 	.open = drm_gem_vm_open,
@@ -511,7 +479,7 @@ static struct drm_driver omap_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM  | DRIVER_PRIME |
 		DRIVER_ATOMIC | DRIVER_RENDER,
 	.open = dev_open,
-	.lastclose = dev_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = omap_debugfs_init,
 #endif
@@ -593,6 +561,11 @@ static int pdev_probe(struct platform_device *pdev)
 	ddev->dev_private = priv;
 	platform_set_drvdata(pdev, ddev);
 
+	/* Get memory bandwidth limits */
+	if (priv->dispc_ops->get_memory_bandwidth_limit)
+		priv->max_bandwidth =
+				priv->dispc_ops->get_memory_bandwidth_limit();
+
 	omap_gem_init(ddev);
 
 	ret = omap_modeset_init(ddev);
@@ -740,7 +713,7 @@ static int omap_drm_resume(struct device *dev)
 
 	drm_kms_helper_poll_enable(drm_dev);
 
-	return omap_gem_resume(dev);
+	return omap_gem_resume(drm_dev);
 }
 #endif
 
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
index 4bd1e9070b31..0ac97fe09f9b 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.h
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_drv.h
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -17,8 +15,8 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef __OMAP_DRV_H__
-#define __OMAP_DRV_H__
+#ifndef __OMAPDRM_DRV_H__
+#define __OMAPDRM_DRV_H__
 
 #include <linux/module.h>
 #include <linux/types.h>
@@ -31,6 +29,15 @@
 
 #include "dss/omapdss.h"
 
+#include "omap_connector.h"
+#include "omap_crtc.h"
+#include "omap_encoder.h"
+#include "omap_fb.h"
+#include "omap_fbdev.h"
+#include "omap_gem.h"
+#include "omap_irq.h"
+#include "omap_plane.h"
+
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 #define VERB(fmt, ...) if (0) DRM_DEBUG(fmt, ##__VA_ARGS__) /* verbose debug */
 
@@ -38,14 +45,6 @@
 
 struct omap_drm_usergart;
 
-/* For KMS code that needs to wait for a certain # of IRQs:
- */
-struct omap_irq_wait;
-struct omap_irq_wait * omap_irq_wait_init(struct drm_device *dev,
-		uint32_t irqmask, int count);
-int omap_irq_wait(struct drm_device *dev, struct omap_irq_wait *wait,
-		unsigned long timeout);
-
 struct omap_drm_private {
 	uint32_t omaprev;
 
@@ -83,117 +82,12 @@ struct omap_drm_private {
 	spinlock_t wait_lock;		/* protects the wait_list */
 	struct list_head wait_list;	/* list of omap_irq_wait */
 	uint32_t irq_mask;		/* enabled irqs in addition to wait_list */
+
+	/* memory bandwidth limit if it is needed on the platform */
+	unsigned int max_bandwidth;
 };
 
 
-#ifdef CONFIG_DEBUG_FS
 int omap_debugfs_init(struct drm_minor *minor);
-void omap_framebuffer_describe(struct drm_framebuffer *fb, struct seq_file *m);
-void omap_gem_describe(struct drm_gem_object *obj, struct seq_file *m);
-void omap_gem_describe_objects(struct list_head *list, struct seq_file *m);
-#endif
-
-#ifdef CONFIG_PM
-int omap_gem_resume(struct device *dev);
-#endif
-
-int omap_irq_enable_vblank(struct drm_crtc *crtc);
-void omap_irq_disable_vblank(struct drm_crtc *crtc);
-void omap_drm_irq_uninstall(struct drm_device *dev);
-int omap_drm_irq_install(struct drm_device *dev);
-
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-struct drm_fb_helper *omap_fbdev_init(struct drm_device *dev);
-void omap_fbdev_free(struct drm_device *dev);
-#else
-static inline struct drm_fb_helper *omap_fbdev_init(struct drm_device *dev)
-{
-	return NULL;
-}
-static inline void omap_fbdev_free(struct drm_device *dev)
-{
-}
-#endif
-
-struct videomode *omap_crtc_timings(struct drm_crtc *crtc);
-enum omap_channel omap_crtc_channel(struct drm_crtc *crtc);
-void omap_crtc_pre_init(void);
-void omap_crtc_pre_uninit(void);
-struct drm_crtc *omap_crtc_init(struct drm_device *dev,
-		struct drm_plane *plane, struct omap_dss_device *dssdev);
-int omap_crtc_wait_pending(struct drm_crtc *crtc);
-void omap_crtc_error_irq(struct drm_crtc *crtc, uint32_t irqstatus);
-void omap_crtc_vblank_irq(struct drm_crtc *crtc);
-
-struct drm_plane *omap_plane_init(struct drm_device *dev,
-		int idx, enum drm_plane_type type,
-		u32 possible_crtcs);
-void omap_plane_install_properties(struct drm_plane *plane,
-		struct drm_mode_object *obj);
-
-struct drm_encoder *omap_encoder_init(struct drm_device *dev,
-		struct omap_dss_device *dssdev);
-
-struct drm_connector *omap_connector_init(struct drm_device *dev,
-		int connector_type, struct omap_dss_device *dssdev,
-		struct drm_encoder *encoder);
-struct drm_encoder *omap_connector_attached_encoder(
-		struct drm_connector *connector);
-bool omap_connector_get_hdmi_mode(struct drm_connector *connector);
-
-struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
-		struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd);
-struct drm_framebuffer *omap_framebuffer_init(struct drm_device *dev,
-		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
-int omap_framebuffer_pin(struct drm_framebuffer *fb);
-void omap_framebuffer_unpin(struct drm_framebuffer *fb);
-void omap_framebuffer_update_scanout(struct drm_framebuffer *fb,
-		struct drm_plane_state *state, struct omap_overlay_info *info);
-struct drm_connector *omap_framebuffer_get_next_connector(
-		struct drm_framebuffer *fb, struct drm_connector *from);
-bool omap_framebuffer_supports_rotation(struct drm_framebuffer *fb);
-
-void omap_gem_init(struct drm_device *dev);
-void omap_gem_deinit(struct drm_device *dev);
-
-struct drm_gem_object *omap_gem_new(struct drm_device *dev,
-		union omap_gem_size gsize, uint32_t flags);
-struct drm_gem_object *omap_gem_new_dmabuf(struct drm_device *dev, size_t size,
-		struct sg_table *sgt);
-int omap_gem_new_handle(struct drm_device *dev, struct drm_file *file,
-		union omap_gem_size gsize, uint32_t flags, uint32_t *handle);
-void omap_gem_free_object(struct drm_gem_object *obj);
-void *omap_gem_vaddr(struct drm_gem_object *obj);
-int omap_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
-		uint32_t handle, uint64_t *offset);
-int omap_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
-		struct drm_mode_create_dumb *args);
-int omap_gem_mmap(struct file *filp, struct vm_area_struct *vma);
-int omap_gem_mmap_obj(struct drm_gem_object *obj,
-		struct vm_area_struct *vma);
-int omap_gem_fault(struct vm_fault *vmf);
-int omap_gem_roll(struct drm_gem_object *obj, uint32_t roll);
-void omap_gem_cpu_sync_page(struct drm_gem_object *obj, int pgoff);
-void omap_gem_dma_sync_buffer(struct drm_gem_object *obj,
-		enum dma_data_direction dir);
-int omap_gem_pin(struct drm_gem_object *obj, dma_addr_t *dma_addr);
-void omap_gem_unpin(struct drm_gem_object *obj);
-int omap_gem_get_pages(struct drm_gem_object *obj, struct page ***pages,
-		bool remap);
-int omap_gem_put_pages(struct drm_gem_object *obj);
-uint32_t omap_gem_flags(struct drm_gem_object *obj);
-int omap_gem_rotated_dma_addr(struct drm_gem_object *obj, uint32_t orient,
-		int x, int y, dma_addr_t *dma_addr);
-uint64_t omap_gem_mmap_offset(struct drm_gem_object *obj);
-size_t omap_gem_mmap_size(struct drm_gem_object *obj);
-int omap_gem_tiled_stride(struct drm_gem_object *obj, uint32_t orient);
-
-struct dma_buf *omap_gem_prime_export(struct drm_device *dev,
-		struct drm_gem_object *obj, int flags);
-struct drm_gem_object *omap_gem_prime_import(struct drm_device *dev,
-		struct dma_buf *buffer);
-
-/* map crtc to vblank mask */
-struct omap_dss_device *omap_encoder_get_dssdev(struct drm_encoder *encoder);
-
-#endif /* __OMAP_DRV_H__ */
+
+#endif /* __OMAPDRM_DRV_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.c b/drivers/gpu/drm/omapdrm/omap_encoder.c
index 624f5b50b755..fcdf4b0a8eec 100644
--- a/drivers/gpu/drm/omapdrm/omap_encoder.c
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_encoder.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.h b/drivers/gpu/drm/omapdrm/omap_encoder.h
new file mode 100644
index 000000000000..d2f308bec494
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.h
@@ -0,0 +1,33 @@
+/*
+ * omap_encoder.h -- OMAP DRM Encoder
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_ENCODER_H__
+#define __OMAPDRM_ENCODER_H__
+
+struct drm_device;
+struct drm_encoder;
+struct omap_dss_device;
+
+struct drm_encoder *omap_encoder_init(struct drm_device *dev,
+		struct omap_dss_device *dssdev);
+
+/* map crtc to vblank mask */
+struct omap_dss_device *omap_encoder_get_dssdev(struct drm_encoder *encoder);
+
+#endif /* __OMAPDRM_ENCODER_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
index b1a762b70cbf..b2539a90e1a4 100644
--- a/drivers/gpu/drm/omapdrm/omap_fb.c
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_fb.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.h b/drivers/gpu/drm/omapdrm/omap_fb.h
new file mode 100644
index 000000000000..94ad5f9e4404
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_fb.h
@@ -0,0 +1,46 @@
+/*
+ * omap_fb.h -- OMAP DRM Framebuffer
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_FB_H__
+#define __OMAPDRM_FB_H__
+
+struct drm_connector;
+struct drm_device;
+struct drm_file;
+struct drm_framebuffer;
+struct drm_gem_object;
+struct drm_mode_fb_cmd2;
+struct drm_plane_state;
+struct omap_overlay_info;
+struct seq_file;
+
+struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
+		struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd);
+struct drm_framebuffer *omap_framebuffer_init(struct drm_device *dev,
+		const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
+int omap_framebuffer_pin(struct drm_framebuffer *fb);
+void omap_framebuffer_unpin(struct drm_framebuffer *fb);
+void omap_framebuffer_update_scanout(struct drm_framebuffer *fb,
+		struct drm_plane_state *state, struct omap_overlay_info *info);
+struct drm_connector *omap_framebuffer_get_next_connector(
+		struct drm_framebuffer *fb, struct drm_connector *from);
+bool omap_framebuffer_supports_rotation(struct drm_framebuffer *fb);
+void omap_framebuffer_describe(struct drm_framebuffer *fb, struct seq_file *m);
+
+#endif /* __OMAPDRM_FB_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index 9273118040b7..fb309d19ca1b 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_fbdev.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob@ti.com>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -84,9 +82,6 @@ static struct fb_ops omap_fb_ops = {
 	.owner = THIS_MODULE,
 	DRM_FB_HELPER_DEFAULT_OPS,
 
-	/* Note: to properly handle manual update displays, we wrap the
-	 * basic fbdev ops which write to the framebuffer
-	 */
 	.fb_read = drm_fb_helper_sys_read,
 	.fb_write = drm_fb_helper_sys_write,
 	.fb_fillrect = drm_fb_helper_sys_fillrect,
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.h b/drivers/gpu/drm/omapdrm/omap_fbdev.h
new file mode 100644
index 000000000000..1f5ba0996a1a
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.h
@@ -0,0 +1,39 @@
+/*
+ * omap_fbdev.h -- OMAP DRM FBDEV Compatibility
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_FBDEV_H__
+#define __OMAPDRM_FBDEV_H__
+
+struct drm_device;
+struct drm_fb_helper;
+
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+struct drm_fb_helper *omap_fbdev_init(struct drm_device *dev);
+void omap_fbdev_free(struct drm_device *dev);
+#else
+static inline struct drm_fb_helper *omap_fbdev_init(struct drm_device *dev)
+{
+	return NULL;
+}
+static inline void omap_fbdev_free(struct drm_device *dev)
+{
+}
+#endif
+
+#endif /* __OMAPDRM_FBDEV_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
index 5c5c86ddd6f4..443469d4fa46 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_gem.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob.clark@linaro.org>
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -996,10 +994,9 @@ void *omap_gem_vaddr(struct drm_gem_object *obj)
 
 #ifdef CONFIG_PM
 /* re-pin objects in DMM in resume path: */
-int omap_gem_resume(struct device *dev)
+int omap_gem_resume(struct drm_device *dev)
 {
-	struct drm_device *drm_dev = dev_get_drvdata(dev);
-	struct omap_drm_private *priv = drm_dev->dev_private;
+	struct omap_drm_private *priv = dev->dev_private;
 	struct omap_gem_object *omap_obj;
 	int ret = 0;
 
@@ -1012,7 +1009,7 @@ int omap_gem_resume(struct device *dev)
 					omap_obj->pages, npages,
 					omap_obj->roll, true);
 			if (ret) {
-				dev_err(dev, "could not repin: %d\n", ret);
+				dev_err(dev->dev, "could not repin: %d\n", ret);
 				return ret;
 			}
 		}
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.h b/drivers/gpu/drm/omapdrm/omap_gem.h
new file mode 100644
index 000000000000..35fa690b3d90
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_gem.h
@@ -0,0 +1,99 @@
+/*
+ * omap_gem.h -- OMAP DRM GEM Object Management
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_GEM_H__
+#define __OMAPDRM_GEM_H__
+
+#include <linux/types.h>
+
+enum dma_data_direction;
+
+struct dma_buf;
+struct drm_device;
+struct drm_file;
+struct drm_gem_object;
+struct drm_mode_create_dumb;
+struct file;
+struct list_head;
+struct page;
+struct seq_file;
+struct vm_area_struct;
+struct vm_fault;
+
+union omap_gem_size;
+
+/* Initialization and Cleanup */
+void omap_gem_init(struct drm_device *dev);
+void omap_gem_deinit(struct drm_device *dev);
+
+#ifdef CONFIG_PM
+int omap_gem_resume(struct drm_device *dev);
+#endif
+
+#ifdef CONFIG_DEBUG_FS
+void omap_gem_describe(struct drm_gem_object *obj, struct seq_file *m);
+void omap_gem_describe_objects(struct list_head *list, struct seq_file *m);
+#endif
+
+/* GEM Object Creation and Deletion */
+struct drm_gem_object *omap_gem_new(struct drm_device *dev,
+		union omap_gem_size gsize, uint32_t flags);
+struct drm_gem_object *omap_gem_new_dmabuf(struct drm_device *dev, size_t size,
+		struct sg_table *sgt);
+int omap_gem_new_handle(struct drm_device *dev, struct drm_file *file,
+		union omap_gem_size gsize, uint32_t flags, uint32_t *handle);
+void omap_gem_free_object(struct drm_gem_object *obj);
+void *omap_gem_vaddr(struct drm_gem_object *obj);
+
+/* Dumb Buffers Interface */
+int omap_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
+		uint32_t handle, uint64_t *offset);
+int omap_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
+		struct drm_mode_create_dumb *args);
+
+/* mmap() Interface */
+int omap_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+int omap_gem_mmap_obj(struct drm_gem_object *obj,
+		struct vm_area_struct *vma);
+uint64_t omap_gem_mmap_offset(struct drm_gem_object *obj);
+size_t omap_gem_mmap_size(struct drm_gem_object *obj);
+
+/* PRIME Interface */
+struct dma_buf *omap_gem_prime_export(struct drm_device *dev,
+		struct drm_gem_object *obj, int flags);
+struct drm_gem_object *omap_gem_prime_import(struct drm_device *dev,
+		struct dma_buf *buffer);
+
+int omap_gem_fault(struct vm_fault *vmf);
+int omap_gem_roll(struct drm_gem_object *obj, uint32_t roll);
+void omap_gem_cpu_sync_page(struct drm_gem_object *obj, int pgoff);
+void omap_gem_dma_sync_buffer(struct drm_gem_object *obj,
+		enum dma_data_direction dir);
+int omap_gem_pin(struct drm_gem_object *obj, dma_addr_t *dma_addr);
+void omap_gem_unpin(struct drm_gem_object *obj);
+int omap_gem_get_pages(struct drm_gem_object *obj, struct page ***pages,
+		bool remap);
+int omap_gem_put_pages(struct drm_gem_object *obj);
+
+uint32_t omap_gem_flags(struct drm_gem_object *obj);
+int omap_gem_rotated_dma_addr(struct drm_gem_object *obj, uint32_t orient,
+		int x, int y, dma_addr_t *dma_addr);
+int omap_gem_tiled_stride(struct drm_gem_object *obj, uint32_t orient);
+
+#endif /* __OMAPDRM_GEM_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
index afdbad5c866a..8e41d649e248 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_gem_dmabuf.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob.clark@linaro.org>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/omap_irq.c b/drivers/gpu/drm/omapdrm/omap_irq.c
index 013b0bba712f..53ba424823b2 100644
--- a/drivers/gpu/drm/omapdrm/omap_irq.c
+++ b/drivers/gpu/drm/omapdrm/omap_irq.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_irq.c
- *
- * Copyright (C) 2012 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob.clark@linaro.org>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/omap_irq.h b/drivers/gpu/drm/omapdrm/omap_irq.h
new file mode 100644
index 000000000000..606c09932bc0
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_irq.h
@@ -0,0 +1,39 @@
+/*
+ * omap_irq.h -- OMAP DRM IRQ Handling
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_IRQ_H__
+#define __OMAPDRM_IRQ_H__
+
+#include <linux/types.h>
+
+struct drm_crtc;
+struct drm_device;
+struct omap_irq_wait;
+
+int omap_irq_enable_vblank(struct drm_crtc *crtc);
+void omap_irq_disable_vblank(struct drm_crtc *crtc);
+void omap_drm_irq_uninstall(struct drm_device *dev);
+int omap_drm_irq_install(struct drm_device *dev);
+
+struct omap_irq_wait *omap_irq_wait_init(struct drm_device *dev,
+		uint32_t irqmask, int count);
+int omap_irq_wait(struct drm_device *dev, struct omap_irq_wait *wait,
+		unsigned long timeout);
+
+#endif /* __OMAPDRM_IRQ_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index 15e5d5d325c6..7d789d1551a1 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -1,7 +1,5 @@
 /*
- * drivers/gpu/drm/omapdrm/omap_plane.c
- *
- * Copyright (C) 2011 Texas Instruments
+ * Copyright (C) 2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Rob Clark <rob.clark@linaro.org>
  *
  * This program is free software; you can redistribute it and/or modify it
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.h b/drivers/gpu/drm/omapdrm/omap_plane.h
new file mode 100644
index 000000000000..dc5e82ad061d
--- /dev/null
+++ b/drivers/gpu/drm/omapdrm/omap_plane.h
@@ -0,0 +1,37 @@
+/*
+ * omap_plane.h -- OMAP DRM Plane
+ *
+ * Copyright (C) 2011 Texas Instruments
+ * Author: Rob Clark <rob@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef __OMAPDRM_PLANE_H__
+#define __OMAPDRM_PLANE_H__
+
+#include <linux/types.h>
+
+enum drm_plane_type;
+
+struct drm_device;
+struct drm_mode_object;
+struct drm_plane;
+
+struct drm_plane *omap_plane_init(struct drm_device *dev,
+		int idx, enum drm_plane_type type,
+		u32 possible_crtcs);
+void omap_plane_install_properties(struct drm_plane *plane,
+		struct drm_mode_object *obj);
+
+#endif /* __OMAPDRM_PLANE_H__ */
diff --git a/drivers/gpu/drm/omapdrm/tcm-sita.c b/drivers/gpu/drm/omapdrm/tcm-sita.c
index c10fdfc0930f..661362d072f7 100644
--- a/drivers/gpu/drm/omapdrm/tcm-sita.c
+++ b/drivers/gpu/drm/omapdrm/tcm-sita.c
@@ -1,13 +1,11 @@
 /*
- * tcm-sita.c
- *
  * SImple Tiler Allocator (SiTA): 2D and 1D allocation(reservation) algorithm
  *
  * Authors: Ravi Ramachandra <r.ramachandra@ti.com>,
  *          Lajos Molnar <molnar@ti.com>
  *          Andy Gross <andy.gross@ti.com>
  *
- * Copyright (C) 2012 Texas Instruments, Inc.
+ * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/
  *
  * This package is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
diff --git a/drivers/gpu/drm/omapdrm/tcm-sita.h b/drivers/gpu/drm/omapdrm/tcm-sita.h
index 0444f868671c..460e63dbf825 100644
--- a/drivers/gpu/drm/omapdrm/tcm-sita.h
+++ b/drivers/gpu/drm/omapdrm/tcm-sita.h
@@ -1,11 +1,9 @@
 /*
- * tcm_sita.h
- *
  * SImple Tiler Allocator (SiTA) private structures.
  *
+ * Copyright (C) 2009-2011 Texas Instruments Incorporated - http://www.ti.com/
  * Author: Ravi Ramachandra <r.ramachandra@ti.com>
  *
- * Copyright (C) 2009-2011 Texas Instruments, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
diff --git a/drivers/gpu/drm/omapdrm/tcm.h b/drivers/gpu/drm/omapdrm/tcm.h
index ef7df7d6fc84..d8a369a4f269 100644
--- a/drivers/gpu/drm/omapdrm/tcm.h
+++ b/drivers/gpu/drm/omapdrm/tcm.h
@@ -1,6 +1,4 @@
 /*
- * tcm.h
- *
  * TILER container manager specification and support functions for TI
  * TILER driver.
  *
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 726f3fb3312d..6ba4031f3919 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -28,6 +28,14 @@ config DRM_PANEL_SIMPLE
 	  that it can be automatically turned off when the panel goes into a
 	  low power state.
 
+config DRM_PANEL_ILITEK_IL9322
+	tristate "Ilitek ILI9322 320x240 QVGA panels"
+	depends on OF && SPI
+	select REGMAP
+	help
+	  Say Y here if you want to enable support for Ilitek IL9322
+	  QVGA (320x240) RGB, YUV and ITU-T BT.656 panels.
+
 config DRM_PANEL_INNOLUX_P079ZCA
 	tristate "Innolux P079ZCA panel"
 	depends on OF
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index 2c4e1a93e05f..6d251ebc568c 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_DRM_PANEL_LVDS) += panel-lvds.o
 obj-$(CONFIG_DRM_PANEL_SIMPLE) += panel-simple.o
+obj-$(CONFIG_DRM_PANEL_ILITEK_IL9322) += panel-ilitek-ili9322.o
 obj-$(CONFIG_DRM_PANEL_INNOLUX_P079ZCA) += panel-innolux-p079zca.o
 obj-$(CONFIG_DRM_PANEL_JDI_LT070ME05000) += panel-jdi-lt070me05000.o
 obj-$(CONFIG_DRM_PANEL_LG_LG4573) += panel-lg-lg4573.o
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9322.c b/drivers/gpu/drm/panel/panel-ilitek-ili9322.c
new file mode 100644
index 000000000000..b4ec0ecff807
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9322.c
@@ -0,0 +1,962 @@
+/*
+ * Ilitek ILI9322 TFT LCD drm_panel driver.
+ *
+ * This panel can be configured to support:
+ * - 8-bit serial RGB interface
+ * - 24-bit parallel RGB interface
+ * - 8-bit ITU-R BT.601 interface
+ * - 8-bit ITU-R BT.656 interface
+ * - Up to 320RGBx240 dots resolution TFT LCD displays
+ * - Scaling, brightness and contrast
+ *
+ * The scaling means that the display accepts a 640x480 or 720x480
+ * input and rescales it to fit to the 320x240 display. So what we
+ * present to the system is something else than what comes out on the
+ * actual display.
+ *
+ * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org>
+ * Derived from drivers/drm/gpu/panel/panel-samsung-ld9040.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_panel.h>
+
+#include <linux/of_device.h>
+#include <linux/bitops.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spi/spi.h>
+
+#include <video/mipi_display.h>
+#include <video/of_videomode.h>
+#include <video/videomode.h>
+
+#define ILI9322_CHIP_ID			0x00
+#define ILI9322_CHIP_ID_MAGIC		0x96
+
+/*
+ * Voltage on the communication interface, from 0.7 (0x00)
+ * to 1.32 (0x1f) times the VREG1OUT voltage in 2% increments.
+ * 1.00 (0x0f) is the default.
+ */
+#define ILI9322_VCOM_AMP		0x01
+
+/*
+ * High voltage on the communication signals, from 0.37 (0x00) to
+ * 1.0 (0x3f) times the VREGOUT1 voltage in 1% increments.
+ * 0.83 (0x2e) is the default.
+ */
+#define ILI9322_VCOM_HIGH		0x02
+
+/*
+ * VREG1 voltage regulator from 3.6V (0x00) to 6.0V (0x18) in 0.1V
+ * increments. 5.4V (0x12) is the default. This is the reference
+ * voltage for the VCOM levels and the greyscale level.
+ */
+#define ILI9322_VREG1_VOLTAGE		0x03
+
+/* Describes the incoming signal */
+#define ILI9322_ENTRY			0x06
+/* 0 = right-to-left, 1 = left-to-right (default), horizontal flip */
+#define ILI9322_ENTRY_HDIR		BIT(0)
+/* 0 = down-to-up, 1 = up-to-down (default), vertical flip  */
+#define ILI9322_ENTRY_VDIR		BIT(1)
+/* NTSC, PAL or autodetect */
+#define ILI9322_ENTRY_NTSC		(0 << 2)
+#define ILI9322_ENTRY_PAL		(1 << 2)
+#define ILI9322_ENTRY_AUTODETECT	(3 << 2)
+/* Input format */
+#define ILI9322_ENTRY_SERIAL_RGB_THROUGH (0 << 4)
+#define ILI9322_ENTRY_SERIAL_RGB_ALIGNED (1 << 4)
+#define ILI9322_ENTRY_SERIAL_RGB_DUMMY_320X240 (2 << 4)
+#define ILI9322_ENTRY_SERIAL_RGB_DUMMY_360X240 (3 << 4)
+#define ILI9322_ENTRY_DISABLE_1		(4 << 4)
+#define ILI9322_ENTRY_PARALLEL_RGB_THROUGH (5 << 4)
+#define ILI9322_ENTRY_PARALLEL_RGB_ALIGNED (6 << 4)
+#define ILI9322_ENTRY_YUV_640Y_320CBCR_25_54_MHZ (7 << 4)
+#define ILI9322_ENTRY_YUV_720Y_360CBCR_27_MHZ (8 << 4)
+#define ILI9322_ENTRY_DISABLE_2		(9 << 4)
+#define ILI9322_ENTRY_ITU_R_BT_656_720X360 (10 << 4)
+#define ILI9322_ENTRY_ITU_R_BT_656_640X320 (11 << 4)
+
+/* Power control */
+#define ILI9322_POW_CTRL		0x07
+#define ILI9322_POW_CTRL_STB		BIT(0) /* 0 = standby, 1 = normal */
+#define ILI9322_POW_CTRL_VGL		BIT(1) /* 0 = off, 1 = on  */
+#define ILI9322_POW_CTRL_VGH		BIT(2) /* 0 = off, 1 = on  */
+#define ILI9322_POW_CTRL_DDVDH		BIT(3) /* 0 = off, 1 = on  */
+#define ILI9322_POW_CTRL_VCOM		BIT(4) /* 0 = off, 1 = on  */
+#define ILI9322_POW_CTRL_VCL		BIT(5) /* 0 = off, 1 = on  */
+#define ILI9322_POW_CTRL_AUTO		BIT(6) /* 0 = interactive, 1 = auto */
+#define ILI9322_POW_CTRL_STANDBY	(ILI9322_POW_CTRL_VGL | \
+					 ILI9322_POW_CTRL_VGH | \
+					 ILI9322_POW_CTRL_DDVDH | \
+					 ILI9322_POW_CTRL_VCL | \
+					 ILI9322_POW_CTRL_AUTO | \
+					 BIT(7))
+#define ILI9322_POW_CTRL_DEFAULT	(ILI9322_POW_CTRL_STANDBY | \
+					 ILI9322_POW_CTRL_STB)
+
+/* Vertical back porch bits 0..5 */
+#define ILI9322_VBP			0x08
+
+/* Horizontal back porch, 8 bits */
+#define ILI9322_HBP			0x09
+
+/*
+ * Polarity settings:
+ * 1 = positive polarity
+ * 0 = negative polarity
+ */
+#define ILI9322_POL			0x0a
+#define ILI9322_POL_DCLK		BIT(0) /* 1 default */
+#define ILI9322_POL_HSYNC		BIT(1) /* 0 default */
+#define ILI9322_POL_VSYNC		BIT(2) /* 0 default */
+#define ILI9322_POL_DE			BIT(3) /* 1 default */
+/*
+ * 0 means YCBCR are ordered Cb0,Y0,Cr0,Y1,Cb2,Y2,Cr2,Y3 (default)
+ *   in RGB mode this means RGB comes in RGBRGB
+ * 1 means YCBCR are ordered Cr0,Y0,Cb0,Y1,Cr2,Y2,Cb2,Y3
+ *   in RGB mode this means RGB comes in BGRBGR
+ */
+#define ILI9322_POL_YCBCR_MODE		BIT(4)
+/* Formula A for YCbCR->RGB = 0, Formula B = 1 */
+#define ILI9322_POL_FORMULA		BIT(5)
+/* Reverse polarity: 0 = 0..255, 1 = 255..0 */
+#define ILI9322_POL_REV			BIT(6)
+
+#define ILI9322_IF_CTRL			0x0b
+#define ILI9322_IF_CTRL_HSYNC_VSYNC	0x00
+#define ILI9322_IF_CTRL_HSYNC_VSYNC_DE	BIT(2)
+#define ILI9322_IF_CTRL_DE_ONLY		BIT(3)
+#define ILI9322_IF_CTRL_SYNC_DISABLED	(BIT(2) | BIT(3))
+#define ILI9322_IF_CTRL_LINE_INVERSION	BIT(0) /* Not set means frame inv */
+
+#define ILI9322_GLOBAL_RESET		0x04
+#define ILI9322_GLOBAL_RESET_ASSERT	0x00 /* bit 0 = 0 -> reset */
+
+/*
+ * 4+4 bits of negative and positive gamma correction
+ * Upper nybble, bits 4-7 are negative gamma
+ * Lower nybble, bits 0-3 are positive gamma
+ */
+#define ILI9322_GAMMA_1			0x10
+#define ILI9322_GAMMA_2			0x11
+#define ILI9322_GAMMA_3			0x12
+#define ILI9322_GAMMA_4			0x13
+#define ILI9322_GAMMA_5			0x14
+#define ILI9322_GAMMA_6			0x15
+#define ILI9322_GAMMA_7			0x16
+#define ILI9322_GAMMA_8			0x17
+
+/**
+ * enum ili9322_input - the format of the incoming signal to the panel
+ *
+ * The panel can be connected to various input streams and four of them can
+ * be selected by electronic straps on the display. However it is possible
+ * to select another mode or override the electronic default with this
+ * setting.
+ */
+enum ili9322_input {
+	ILI9322_INPUT_SRGB_THROUGH = 0x0,
+	ILI9322_INPUT_SRGB_ALIGNED = 0x1,
+	ILI9322_INPUT_SRGB_DUMMY_320X240 = 0x2,
+	ILI9322_INPUT_SRGB_DUMMY_360X240 = 0x3,
+	ILI9322_INPUT_DISABLED_1 = 0x4,
+	ILI9322_INPUT_PRGB_THROUGH = 0x5,
+	ILI9322_INPUT_PRGB_ALIGNED = 0x6,
+	ILI9322_INPUT_YUV_640X320_YCBCR = 0x7,
+	ILI9322_INPUT_YUV_720X360_YCBCR = 0x8,
+	ILI9322_INPUT_DISABLED_2 = 0x9,
+	ILI9322_INPUT_ITU_R_BT656_720X360_YCBCR = 0xa,
+	ILI9322_INPUT_ITU_R_BT656_640X320_YCBCR = 0xb,
+	ILI9322_INPUT_UNKNOWN = 0xc,
+};
+
+const char *ili9322_inputs[] = {
+	"8 bit serial RGB through",
+	"8 bit serial RGB aligned",
+	"8 bit serial RGB dummy 320x240",
+	"8 bit serial RGB dummy 360x240",
+	"disabled 1",
+	"24 bit parallel RGB through",
+	"24 bit parallel RGB aligned",
+	"24 bit YUV 640Y 320CbCr",
+	"24 bit YUV 720Y 360CbCr",
+	"disabled 2",
+	"8 bit ITU-R BT.656 720Y 360CbCr",
+	"8 bit ITU-R BT.656 640Y 320CbCr",
+};
+
+/**
+ * struct ili9322_config - the system specific ILI9322 configuration
+ * @width_mm: physical panel width [mm]
+ * @height_mm: physical panel height [mm]
+ * @flip_horizontal: flip the image horizontally (right-to-left scan)
+ * (only in RGB and YUV modes)
+ * @flip_vertical: flip the image vertically (down-to-up scan)
+ * (only in RGB and YUV modes)
+ * @input: the input/entry type used in this system, if this is set to
+ * ILI9322_INPUT_UNKNOWN the driver will try to figure it out by probing
+ * the hardware
+ * @vreg1out_mv: the output in microvolts for the VREGOUT1 regulator used
+ * to drive the physical display. Valid ranges are 3600 thru 6000 in 100
+ * microvolt increments. If not specified, hardware defaults will be
+ * used (4.5V).
+ * @vcom_high_percent: the percentage of VREGOUT1 used for the peak
+ * voltage on the communications link. Valid ranges are 37 thru 100
+ * percent. If not specified, hardware defaults will be used (91%).
+ * @vcom_amplitude_percent: the percentage of VREGOUT1 used for the
+ * peak-to-peak amplitude of the communcation signals to the physical
+ * display. Valid ranges are 70 thru 132 percent in increments if two
+ * percent. Odd percentages will be truncated. If not specified, hardware
+ * defaults will be used (114%).
+ * @dclk_active_high: data/pixel clock active high, data will be clocked
+ * in on the rising edge of the DCLK (this is usually the case).
+ * @syncmode: The synchronization mode, what sync signals are emitted.
+ * See the enum for details.
+ * @de_active_high: DE (data entry) is active high
+ * @hsync_active_high: HSYNC is active high
+ * @vsync_active_high: VSYNC is active high
+ * @gamma_corr_pos: a set of 8 nybbles describing positive
+ * gamma correction for voltages V1 thru V8. Valid range 0..15
+ * @gamma_corr_neg: a set of 8 nybbles describing negative
+ * gamma correction for voltages V1 thru V8. Valid range 0..15
+ *
+ * These adjust what grayscale voltage will be output for input data V1 = 0,
+ * V2 = 16, V3 = 48, V4 = 96, V5 = 160, V6 = 208, V7 = 240 and V8 = 255.
+ * The curve is shaped like this:
+ *
+ *  ^
+ *  |                                                        V8
+ *  |                                                   V7
+ *  |                                          V6
+ *  |                               V5
+ *  |                    V4
+ *  |            V3
+ *  |     V2
+ *  | V1
+ *  +----------------------------------------------------------->
+ *    0   16     48      96         160        208      240  255
+ *
+ * The negative and postive gamma values adjust the V1 thru V8 up/down
+ * according to the datasheet specifications. This is a property of the
+ * physical display connected to the display controller and may vary.
+ * If defined, both arrays must be supplied in full. If the properties
+ * are not supplied, hardware defaults will be used.
+ */
+struct ili9322_config {
+	u32 width_mm;
+	u32 height_mm;
+	bool flip_horizontal;
+	bool flip_vertical;
+	enum ili9322_input input;
+	u32 vreg1out_mv;
+	u32 vcom_high_percent;
+	u32 vcom_amplitude_percent;
+	bool dclk_active_high;
+	bool de_active_high;
+	bool hsync_active_high;
+	bool vsync_active_high;
+	u8 syncmode;
+	u8 gamma_corr_pos[8];
+	u8 gamma_corr_neg[8];
+};
+
+struct ili9322 {
+	struct device *dev;
+	const struct ili9322_config *conf;
+	struct drm_panel panel;
+	struct regmap *regmap;
+	struct regulator_bulk_data supplies[3];
+	struct gpio_desc *reset_gpio;
+	enum ili9322_input input;
+	struct videomode vm;
+	u8 gamma[8];
+	u8 vreg1out;
+	u8 vcom_high;
+	u8 vcom_amplitude;
+};
+
+static inline struct ili9322 *panel_to_ili9322(struct drm_panel *panel)
+{
+	return container_of(panel, struct ili9322, panel);
+}
+
+static int ili9322_regmap_spi_write(void *context, const void *data,
+				    size_t count)
+{
+	struct device *dev = context;
+	struct spi_device *spi = to_spi_device(dev);
+	u8 buf[2];
+
+	/* Clear bit 7 to write */
+	memcpy(buf, data, 2);
+	buf[0] &= ~0x80;
+
+	dev_dbg(dev, "WRITE: %02x %02x\n", buf[0], buf[1]);
+	return spi_write_then_read(spi, buf, 2, NULL, 0);
+}
+
+static int ili9322_regmap_spi_read(void *context, const void *reg,
+				   size_t reg_size, void *val, size_t val_size)
+{
+	struct device *dev = context;
+	struct spi_device *spi = to_spi_device(dev);
+	u8 buf[1];
+
+	/* Set bit 7 to 1 to read */
+	memcpy(buf, reg, 1);
+	dev_dbg(dev, "READ: %02x reg size = %zu, val size = %zu\n",
+		buf[0], reg_size, val_size);
+	buf[0] |= 0x80;
+
+	return spi_write_then_read(spi, buf, 1, val, 1);
+}
+
+static struct regmap_bus ili9322_regmap_bus = {
+	.write = ili9322_regmap_spi_write,
+	.read = ili9322_regmap_spi_read,
+	.reg_format_endian_default = REGMAP_ENDIAN_BIG,
+	.val_format_endian_default = REGMAP_ENDIAN_BIG,
+};
+
+static bool ili9322_volatile_reg(struct device *dev, unsigned int reg)
+{
+	return false;
+}
+
+static bool ili9322_writeable_reg(struct device *dev, unsigned int reg)
+{
+	/* Just register 0 is read-only */
+	if (reg == 0x00)
+		return false;
+	return true;
+}
+
+const struct regmap_config ili9322_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = 0x44,
+	.cache_type = REGCACHE_RBTREE,
+	.volatile_reg = ili9322_volatile_reg,
+	.writeable_reg = ili9322_writeable_reg,
+};
+
+static int ili9322_init(struct drm_panel *panel, struct ili9322 *ili)
+{
+	struct drm_connector *connector = panel->connector;
+	u8 reg;
+	int ret;
+	int i;
+
+	/* Reset display */
+	ret = regmap_write(ili->regmap, ILI9322_GLOBAL_RESET,
+			   ILI9322_GLOBAL_RESET_ASSERT);
+	if (ret) {
+		dev_err(ili->dev, "can't issue GRESET (%d)\n", ret);
+		return ret;
+	}
+
+	/* Set up the main voltage regulator */
+	if (ili->vreg1out != U8_MAX) {
+		ret = regmap_write(ili->regmap, ILI9322_VREG1_VOLTAGE,
+				   ili->vreg1out);
+		if (ret) {
+			dev_err(ili->dev, "can't set up VREG1OUT (%d)\n", ret);
+			return ret;
+		}
+	}
+
+	if (ili->vcom_amplitude != U8_MAX) {
+		ret = regmap_write(ili->regmap, ILI9322_VCOM_AMP,
+				   ili->vcom_amplitude);
+		if (ret) {
+			dev_err(ili->dev,
+				"can't set up VCOM amplitude (%d)\n", ret);
+			return ret;
+		}
+	};
+
+	if (ili->vcom_high != U8_MAX) {
+		ret = regmap_write(ili->regmap, ILI9322_VCOM_HIGH,
+				   ili->vcom_high);
+		if (ret) {
+			dev_err(ili->dev, "can't set up VCOM high (%d)\n", ret);
+			return ret;
+		}
+	};
+
+	/* Set up gamma correction */
+	for (i = 0; i < ARRAY_SIZE(ili->gamma); i++) {
+		ret = regmap_write(ili->regmap, ILI9322_GAMMA_1 + i,
+				   ili->gamma[i]);
+		if (ret) {
+			dev_err(ili->dev,
+				"can't write gamma V%d to 0x%02x (%d)\n",
+				i + 1, ILI9322_GAMMA_1 + i, ret);
+			return ret;
+		}
+	}
+
+	/*
+	 * Polarity and inverted color order for RGB input.
+	 * None of this applies in the BT.656 mode.
+	 */
+	if (ili->conf->dclk_active_high) {
+		reg = ILI9322_POL_DCLK;
+		connector->display_info.bus_flags |=
+			DRM_BUS_FLAG_PIXDATA_POSEDGE;
+	} else {
+		reg = 0;
+		connector->display_info.bus_flags |=
+			DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+	}
+	if (ili->conf->de_active_high) {
+		reg |= ILI9322_POL_DE;
+		connector->display_info.bus_flags |=
+			DRM_BUS_FLAG_DE_HIGH;
+	} else {
+		connector->display_info.bus_flags |=
+			DRM_BUS_FLAG_DE_LOW;
+	}
+	if (ili->conf->hsync_active_high)
+		reg |= ILI9322_POL_HSYNC;
+	if (ili->conf->vsync_active_high)
+		reg |= ILI9322_POL_VSYNC;
+	ret = regmap_write(ili->regmap, ILI9322_POL, reg);
+	if (ret) {
+		dev_err(ili->dev, "can't write POL register (%d)\n", ret);
+		return ret;
+	}
+
+	/*
+	 * Set up interface control.
+	 * This is not used in the BT.656 mode (no H/Vsync or DE signals).
+	 */
+	reg = ili->conf->syncmode;
+	reg |= ILI9322_IF_CTRL_LINE_INVERSION;
+	ret = regmap_write(ili->regmap, ILI9322_IF_CTRL, reg);
+	if (ret) {
+		dev_err(ili->dev, "can't write IF CTRL register (%d)\n", ret);
+		return ret;
+	}
+
+	/* Set up the input mode */
+	reg = (ili->input << 4);
+	/* These are inverted, setting to 1 is the default, clearing flips */
+	if (!ili->conf->flip_horizontal)
+		reg |= ILI9322_ENTRY_HDIR;
+	if (!ili->conf->flip_vertical)
+		reg |= ILI9322_ENTRY_VDIR;
+	reg |= ILI9322_ENTRY_AUTODETECT;
+	ret = regmap_write(ili->regmap, ILI9322_ENTRY, reg);
+	if (ret) {
+		dev_err(ili->dev, "can't write ENTRY reg (%d)\n", ret);
+		return ret;
+	}
+	dev_info(ili->dev, "display is in %s mode, syncmode %02x\n",
+		 ili9322_inputs[ili->input],
+		 ili->conf->syncmode);
+
+	dev_info(ili->dev, "initialized display\n");
+
+	return 0;
+}
+
+/*
+ * This power-on sequence if from the datasheet, page 57.
+ */
+static int ili9322_power_on(struct ili9322 *ili)
+{
+	int ret;
+
+	/* Assert RESET */
+	gpiod_set_value(ili->reset_gpio, 1);
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ili->supplies), ili->supplies);
+	if (ret < 0) {
+		dev_err(ili->dev, "unable to enable regulators\n");
+		return ret;
+	}
+	msleep(20);
+
+	/* De-assert RESET */
+	gpiod_set_value(ili->reset_gpio, 0);
+
+	msleep(10);
+
+	return 0;
+}
+
+static int ili9322_power_off(struct ili9322 *ili)
+{
+	return regulator_bulk_disable(ARRAY_SIZE(ili->supplies), ili->supplies);
+}
+
+static int ili9322_disable(struct drm_panel *panel)
+{
+	struct ili9322 *ili = panel_to_ili9322(panel);
+	int ret;
+
+	ret = regmap_write(ili->regmap, ILI9322_POW_CTRL,
+			   ILI9322_POW_CTRL_STANDBY);
+	if (ret) {
+		dev_err(ili->dev, "unable to go to standby mode\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ili9322_unprepare(struct drm_panel *panel)
+{
+	struct ili9322 *ili = panel_to_ili9322(panel);
+
+	return ili9322_power_off(ili);
+}
+
+static int ili9322_prepare(struct drm_panel *panel)
+{
+	struct ili9322 *ili = panel_to_ili9322(panel);
+	int ret;
+
+	ret = ili9322_power_on(ili);
+	if (ret < 0)
+		return ret;
+
+	ret = ili9322_init(panel, ili);
+	if (ret < 0)
+		ili9322_unprepare(panel);
+
+	return ret;
+}
+
+static int ili9322_enable(struct drm_panel *panel)
+{
+	struct ili9322 *ili = panel_to_ili9322(panel);
+	int ret;
+
+	ret = regmap_write(ili->regmap, ILI9322_POW_CTRL,
+			   ILI9322_POW_CTRL_DEFAULT);
+	if (ret) {
+		dev_err(ili->dev, "unable to enable panel\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+/* Serial RGB modes */
+static const struct drm_display_mode srgb_320x240_mode = {
+	.clock = 2453500,
+	.hdisplay = 320,
+	.hsync_start = 320 + 359,
+	.hsync_end = 320 + 359 + 1,
+	.htotal = 320 + 359 + 1 + 241,
+	.vdisplay = 240,
+	.vsync_start = 240 + 4,
+	.vsync_end = 240 + 4 + 1,
+	.vtotal = 262,
+	.vrefresh = 60,
+	.flags = 0,
+};
+
+static const struct drm_display_mode srgb_360x240_mode = {
+	.clock = 2700000,
+	.hdisplay = 360,
+	.hsync_start = 360 + 35,
+	.hsync_end = 360 + 35 + 1,
+	.htotal = 360 + 35 + 1 + 241,
+	.vdisplay = 240,
+	.vsync_start = 240 + 21,
+	.vsync_end = 240 + 21 + 1,
+	.vtotal = 262,
+	.vrefresh = 60,
+	.flags = 0,
+};
+
+/* This is the only mode listed for parallel RGB in the datasheet */
+static const struct drm_display_mode prgb_320x240_mode = {
+	.clock = 6400000,
+	.hdisplay = 320,
+	.hsync_start = 320 + 38,
+	.hsync_end = 320 + 38 + 1,
+	.htotal = 320 + 38 + 1 + 50,
+	.vdisplay = 240,
+	.vsync_start = 240 + 4,
+	.vsync_end = 240 + 4 + 1,
+	.vtotal = 262,
+	.vrefresh = 60,
+	.flags = 0,
+};
+
+/* YUV modes */
+static const struct drm_display_mode yuv_640x320_mode = {
+	.clock = 2454000,
+	.hdisplay = 640,
+	.hsync_start = 640 + 252,
+	.hsync_end = 640 + 252 + 1,
+	.htotal = 640 + 252 + 1 + 28,
+	.vdisplay = 320,
+	.vsync_start = 320 + 4,
+	.vsync_end = 320 + 4 + 1,
+	.vtotal = 320 + 4 + 1 + 18,
+	.vrefresh = 60,
+	.flags = 0,
+};
+
+static const struct drm_display_mode yuv_720x360_mode = {
+	.clock = 2700000,
+	.hdisplay = 720,
+	.hsync_start = 720 + 252,
+	.hsync_end = 720 + 252 + 1,
+	.htotal = 720 + 252 + 1 + 24,
+	.vdisplay = 360,
+	.vsync_start = 360 + 4,
+	.vsync_end = 360 + 4 + 1,
+	.vtotal = 360 + 4 + 1 + 18,
+	.vrefresh = 60,
+	.flags = 0,
+};
+
+/* BT.656 VGA mode, 640x480 */
+static const struct drm_display_mode itu_r_bt_656_640_mode = {
+	.clock = 2454000,
+	.hdisplay = 640,
+	.hsync_start = 640 + 3,
+	.hsync_end = 640 + 3 + 1,
+	.htotal = 640 + 3 + 1 + 272,
+	.vdisplay = 480,
+	.vsync_start = 480 + 4,
+	.vsync_end = 480 + 4 + 1,
+	.vtotal = 500,
+	.vrefresh = 60,
+	.flags = 0,
+};
+
+/* BT.656 D1 mode 720x480 */
+static const struct drm_display_mode itu_r_bt_656_720_mode = {
+	.clock = 2700000,
+	.hdisplay = 720,
+	.hsync_start = 720 + 3,
+	.hsync_end = 720 + 3 + 1,
+	.htotal = 720 + 3 + 1 + 272,
+	.vdisplay = 480,
+	.vsync_start = 480 + 4,
+	.vsync_end = 480 + 4 + 1,
+	.vtotal = 500,
+	.vrefresh = 60,
+	.flags = 0,
+};
+
+static int ili9322_get_modes(struct drm_panel *panel)
+{
+	struct drm_connector *connector = panel->connector;
+	struct ili9322 *ili = panel_to_ili9322(panel);
+	struct drm_display_mode *mode;
+
+	strncpy(connector->display_info.name, "ILI9322 TFT LCD driver\0",
+		DRM_DISPLAY_INFO_LEN);
+	connector->display_info.width_mm = ili->conf->width_mm;
+	connector->display_info.height_mm = ili->conf->height_mm;
+
+	switch (ili->input) {
+	case ILI9322_INPUT_SRGB_DUMMY_320X240:
+		mode = drm_mode_duplicate(panel->drm, &srgb_320x240_mode);
+		break;
+	case ILI9322_INPUT_SRGB_DUMMY_360X240:
+		mode = drm_mode_duplicate(panel->drm, &srgb_360x240_mode);
+		break;
+	case ILI9322_INPUT_PRGB_THROUGH:
+	case ILI9322_INPUT_PRGB_ALIGNED:
+		mode = drm_mode_duplicate(panel->drm, &prgb_320x240_mode);
+		break;
+	case ILI9322_INPUT_YUV_640X320_YCBCR:
+		mode = drm_mode_duplicate(panel->drm, &yuv_640x320_mode);
+		break;
+	case ILI9322_INPUT_YUV_720X360_YCBCR:
+		mode = drm_mode_duplicate(panel->drm, &yuv_720x360_mode);
+		break;
+	case ILI9322_INPUT_ITU_R_BT656_720X360_YCBCR:
+		mode = drm_mode_duplicate(panel->drm, &itu_r_bt_656_720_mode);
+		break;
+	case ILI9322_INPUT_ITU_R_BT656_640X320_YCBCR:
+		mode = drm_mode_duplicate(panel->drm, &itu_r_bt_656_640_mode);
+		break;
+	default:
+		mode = NULL;
+		break;
+	}
+	if (!mode) {
+		DRM_ERROR("bad mode or failed to add mode\n");
+		return -EINVAL;
+	}
+	drm_mode_set_name(mode);
+	/*
+	 * This is the preferred mode because most people are going
+	 * to want to use the display with VGA type graphics.
+	 */
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+
+	/* Set up the polarity */
+	if (ili->conf->hsync_active_high)
+		mode->flags |= DRM_MODE_FLAG_PHSYNC;
+	else
+		mode->flags |= DRM_MODE_FLAG_NHSYNC;
+	if (ili->conf->vsync_active_high)
+		mode->flags |= DRM_MODE_FLAG_PVSYNC;
+	else
+		mode->flags |= DRM_MODE_FLAG_NVSYNC;
+
+	mode->width_mm = ili->conf->width_mm;
+	mode->height_mm = ili->conf->height_mm;
+	drm_mode_probed_add(connector, mode);
+
+	return 1; /* Number of modes */
+}
+
+static const struct drm_panel_funcs ili9322_drm_funcs = {
+	.disable = ili9322_disable,
+	.unprepare = ili9322_unprepare,
+	.prepare = ili9322_prepare,
+	.enable = ili9322_enable,
+	.get_modes = ili9322_get_modes,
+};
+
+static int ili9322_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct ili9322 *ili;
+	const struct regmap_config *regmap_config;
+	u8 gamma;
+	u32 val;
+	int ret;
+	int i;
+
+	ili = devm_kzalloc(dev, sizeof(struct ili9322), GFP_KERNEL);
+	if (!ili)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, ili);
+
+	ili->dev = dev;
+
+	/*
+	 * Every new incarnation of this display must have a unique
+	 * data entry for the system in this driver.
+	 */
+	ili->conf = of_device_get_match_data(dev);
+	if (!ili->conf) {
+		dev_err(dev, "missing device configuration\n");
+		return -ENODEV;
+	}
+
+	val = ili->conf->vreg1out_mv;
+	if (!val) {
+		/* Default HW value, do not touch (should be 4.5V) */
+		ili->vreg1out = U8_MAX;
+	} else {
+		if (val < 3600) {
+			dev_err(dev, "too low VREG1OUT\n");
+			return -EINVAL;
+		}
+		if (val > 6000) {
+			dev_err(dev, "too high VREG1OUT\n");
+			return -EINVAL;
+		}
+		if ((val % 100) != 0) {
+			dev_err(dev, "VREG1OUT is no even 100 microvolt\n");
+			return -EINVAL;
+		}
+		val -= 3600;
+		val /= 100;
+		dev_dbg(dev, "VREG1OUT = 0x%02x\n", val);
+		ili->vreg1out = val;
+	}
+
+	val = ili->conf->vcom_high_percent;
+	if (!val) {
+		/* Default HW value, do not touch (should be 91%) */
+		ili->vcom_high = U8_MAX;
+	} else {
+		if (val < 37) {
+			dev_err(dev, "too low VCOM high\n");
+			return -EINVAL;
+		}
+		if (val > 100) {
+			dev_err(dev, "too high VCOM high\n");
+			return -EINVAL;
+		}
+		val -= 37;
+		dev_dbg(dev, "VCOM high = 0x%02x\n", val);
+		ili->vcom_high = val;
+	}
+
+	val = ili->conf->vcom_amplitude_percent;
+	if (!val) {
+		/* Default HW value, do not touch (should be 114%) */
+		ili->vcom_high = U8_MAX;
+	} else {
+		if (val < 70) {
+			dev_err(dev, "too low VCOM amplitude\n");
+			return -EINVAL;
+		}
+		if (val > 132) {
+			dev_err(dev, "too high VCOM amplitude\n");
+			return -EINVAL;
+		}
+		val -= 70;
+		val >>= 1; /* Increments of 2% */
+		dev_dbg(dev, "VCOM amplitude = 0x%02x\n", val);
+		ili->vcom_amplitude = val;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(ili->gamma); i++) {
+		val = ili->conf->gamma_corr_neg[i];
+		if (val > 15) {
+			dev_err(dev, "negative gamma %u > 15, capping\n", val);
+			val = 15;
+		}
+		gamma = val << 4;
+		val = ili->conf->gamma_corr_pos[i];
+		if (val > 15) {
+			dev_err(dev, "positive gamma %u > 15, capping\n", val);
+			val = 15;
+		}
+		gamma |= val;
+		ili->gamma[i] = gamma;
+		dev_dbg(dev, "gamma V%d: 0x%02x\n", i + 1, gamma);
+	}
+
+	ili->supplies[0].supply = "vcc"; /* 2.7-3.6 V */
+	ili->supplies[1].supply = "iovcc"; /* 1.65-3.6V */
+	ili->supplies[2].supply = "vci"; /* 2.7-3.6V */
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ili->supplies),
+				      ili->supplies);
+	if (ret < 0)
+		return ret;
+	ret = regulator_set_voltage(ili->supplies[0].consumer,
+				    2700000, 3600000);
+	if (ret)
+		return ret;
+	ret = regulator_set_voltage(ili->supplies[1].consumer,
+				    1650000, 3600000);
+	if (ret)
+		return ret;
+	ret = regulator_set_voltage(ili->supplies[2].consumer,
+				    2700000, 3600000);
+	if (ret)
+		return ret;
+
+	ili->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ili->reset_gpio)) {
+		dev_err(dev, "failed to get RESET GPIO\n");
+		return PTR_ERR(ili->reset_gpio);
+	}
+
+	spi->bits_per_word = 8;
+	ret = spi_setup(spi);
+	if (ret < 0) {
+		dev_err(dev, "spi setup failed.\n");
+		return ret;
+	}
+	regmap_config = &ili9322_regmap_config;
+	ili->regmap = devm_regmap_init(dev, &ili9322_regmap_bus, dev,
+				       regmap_config);
+	if (IS_ERR(ili->regmap)) {
+		dev_err(dev, "failed to allocate register map\n");
+		return PTR_ERR(ili->regmap);
+	}
+
+	ret = regmap_read(ili->regmap, ILI9322_CHIP_ID, &val);
+	if (ret) {
+		dev_err(dev, "can't get chip ID (%d)\n", ret);
+		return ret;
+	}
+	if (val != ILI9322_CHIP_ID_MAGIC) {
+		dev_err(dev, "chip ID 0x%0x2, expected 0x%02x\n", val,
+			ILI9322_CHIP_ID_MAGIC);
+		return -ENODEV;
+	}
+
+	/* Probe the system to find the display setting */
+	if (ili->conf->input == ILI9322_INPUT_UNKNOWN) {
+		ret = regmap_read(ili->regmap, ILI9322_ENTRY, &val);
+		if (ret) {
+			dev_err(dev, "can't get entry setting (%d)\n", ret);
+			return ret;
+		}
+		/* Input enum corresponds to HW setting */
+		ili->input = (val >> 4) & 0x0f;
+		if (ili->input >= ILI9322_INPUT_UNKNOWN)
+			ili->input = ILI9322_INPUT_UNKNOWN;
+	} else {
+		ili->input = ili->conf->input;
+	}
+
+	drm_panel_init(&ili->panel);
+	ili->panel.dev = dev;
+	ili->panel.funcs = &ili9322_drm_funcs;
+
+	return drm_panel_add(&ili->panel);
+}
+
+static int ili9322_remove(struct spi_device *spi)
+{
+	struct ili9322 *ili = spi_get_drvdata(spi);
+
+	ili9322_power_off(ili);
+	drm_panel_remove(&ili->panel);
+
+	return 0;
+}
+
+/*
+ * The D-Link DIR-685 panel is marked LM918A01-1A SY-B4-091116-E0199
+ */
+static const struct ili9322_config ili9322_dir_685 = {
+	.width_mm = 65,
+	.height_mm = 50,
+	.input = ILI9322_INPUT_ITU_R_BT656_640X320_YCBCR,
+	.vreg1out_mv = 4600,
+	.vcom_high_percent = 91,
+	.vcom_amplitude_percent = 114,
+	.syncmode = ILI9322_IF_CTRL_SYNC_DISABLED,
+	.dclk_active_high = true,
+	.gamma_corr_neg = { 0xa, 0x5, 0x7, 0x7, 0x7, 0x5, 0x1, 0x6 },
+	.gamma_corr_pos = { 0x7, 0x7, 0x3, 0x2, 0x3, 0x5, 0x7, 0x2 },
+};
+
+static const struct of_device_id ili9322_of_match[] = {
+	{
+		.compatible = "dlink,dir-685-panel",
+		.data = &ili9322_dir_685,
+	},
+	{
+		.compatible = "ilitek,ili9322",
+		.data = NULL,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ili9322_of_match);
+
+static struct spi_driver ili9322_driver = {
+	.probe = ili9322_probe,
+	.remove = ili9322_remove,
+	.driver = {
+		.name = "panel-ilitek-ili9322",
+		.of_match_table = ili9322_of_match,
+	},
+};
+module_spi_driver(ili9322_driver);
+
+MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
+MODULE_DESCRIPTION("ILI9322 LCD panel driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-lvds.c b/drivers/gpu/drm/panel/panel-lvds.c
index e2d57c01200b..b5e3994f0aa8 100644
--- a/drivers/gpu/drm/panel/panel-lvds.c
+++ b/drivers/gpu/drm/panel/panel-lvds.c
@@ -17,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 
 #include <drm/drmP.h>
@@ -39,6 +40,7 @@ struct panel_lvds {
 	bool data_mirror;
 
 	struct backlight_device *backlight;
+	struct regulator *supply;
 
 	struct gpio_desc *enable_gpio;
 	struct gpio_desc *reset_gpio;
@@ -69,6 +71,9 @@ static int panel_lvds_unprepare(struct drm_panel *panel)
 	if (lvds->enable_gpio)
 		gpiod_set_value_cansleep(lvds->enable_gpio, 0);
 
+	if (lvds->supply)
+		regulator_disable(lvds->supply);
+
 	return 0;
 }
 
@@ -76,6 +81,17 @@ static int panel_lvds_prepare(struct drm_panel *panel)
 {
 	struct panel_lvds *lvds = to_panel_lvds(panel);
 
+	if (lvds->supply) {
+		int err;
+
+		err = regulator_enable(lvds->supply);
+		if (err < 0) {
+			dev_err(lvds->dev, "failed to enable supply: %d\n",
+				err);
+			return err;
+		}
+	}
+
 	if (lvds->enable_gpio)
 		gpiod_set_value_cansleep(lvds->enable_gpio, 1);
 
@@ -196,6 +212,20 @@ static int panel_lvds_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
+	lvds->supply = devm_regulator_get_optional(lvds->dev, "power");
+	if (IS_ERR(lvds->supply)) {
+		ret = PTR_ERR(lvds->supply);
+
+		if (ret != -ENODEV) {
+			if (ret != -EPROBE_DEFER)
+				dev_err(lvds->dev, "failed to request regulator: %d\n",
+					ret);
+			return ret;
+		}
+
+		lvds->supply = NULL;
+	}
+
 	/* Get GPIOs and backlight controller. */
 	lvds->enable_gpio = devm_gpiod_get_optional(lvds->dev, "enable",
 						     GPIOD_OUT_LOW);
diff --git a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
index 890fd6ff397c..d964d454e4ae 100644
--- a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
+++ b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
@@ -221,7 +221,7 @@ static struct rpi_touchscreen *panel_to_ts(struct drm_panel *panel)
 	return container_of(panel, struct rpi_touchscreen, base);
 }
 
-static u8 rpi_touchscreen_i2c_read(struct rpi_touchscreen *ts, u8 reg)
+static int rpi_touchscreen_i2c_read(struct rpi_touchscreen *ts, u8 reg)
 {
 	return i2c_smbus_read_byte_data(ts->i2c, reg);
 }
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index b7c4709f7b34..5591984a392b 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -1356,6 +1356,38 @@ static const struct panel_desc lg_lp129qe = {
 	},
 };
 
+static const struct drm_display_mode mitsubishi_aa070mc01_mode = {
+	.clock = 30400,
+	.hdisplay = 800,
+	.hsync_start = 800 + 0,
+	.hsync_end = 800 + 1,
+	.htotal = 800 + 0 + 1 + 160,
+	.vdisplay = 480,
+	.vsync_start = 480 + 0,
+	.vsync_end = 480 + 48 + 1,
+	.vtotal = 480 + 48 + 1 + 0,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+};
+
+static const struct panel_desc mitsubishi_aa070mc01 = {
+	.modes = &mitsubishi_aa070mc01_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 152,
+		.height = 91,
+	},
+
+	.delay = {
+		.enable = 200,
+		.unprepare = 200,
+		.disable = 400,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
+};
+
 static const struct display_timing nec_nl12880bc20_05_timing = {
 	.pixelclock = { 67000000, 71000000, 75000000 },
 	.hactive = { 1280, 1280, 1280 },
@@ -1837,6 +1869,30 @@ static const struct panel_desc tianma_tm070jdhg30 = {
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
 };
 
+static const struct display_timing tianma_tm070rvhg71_timing = {
+	.pixelclock = { 27700000, 29200000, 39600000 },
+	.hactive = { 800, 800, 800 },
+	.hfront_porch = { 12, 40, 212 },
+	.hback_porch = { 88, 88, 88 },
+	.hsync_len = { 1, 1, 40 },
+	.vactive = { 480, 480, 480 },
+	.vfront_porch = { 1, 13, 88 },
+	.vback_porch = { 32, 32, 32 },
+	.vsync_len = { 1, 1, 3 },
+	.flags = DISPLAY_FLAGS_DE_HIGH,
+};
+
+static const struct panel_desc tianma_tm070rvhg71 = {
+	.timings = &tianma_tm070rvhg71_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 154,
+		.height = 86,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+};
+
 static const struct drm_display_mode toshiba_lt089ac29000_mode = {
 	.clock = 79500,
 	.hdisplay = 1280,
@@ -2085,6 +2141,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "lg,lp129qe",
 		.data = &lg_lp129qe,
+	}, {
+		.compatible = "mitsubishi,aa070mc01-ca1",
+		.data = &mitsubishi_aa070mc01,
 	}, {
 		.compatible = "nec,nl12880bc20-05",
 		.data = &nec_nl12880bc20_05,
@@ -2142,6 +2201,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "tianma,tm070jdhg30",
 		.data = &tianma_tm070jdhg30,
+	}, {
+		.compatible = "tianma,tm070rvhg71",
+		.data = &tianma_tm070rvhg71,
 	}, {
 		.compatible = "toshiba,lt089ac29000",
 		.data = &toshiba_lt089ac29000,
diff --git a/drivers/gpu/drm/pl111/pl111_drm.h b/drivers/gpu/drm/pl111/pl111_drm.h
index 440f53ebee8c..07fa2cdb364a 100644
--- a/drivers/gpu/drm/pl111/pl111_drm.h
+++ b/drivers/gpu/drm/pl111/pl111_drm.h
@@ -53,7 +53,6 @@ struct pl111_drm_dev_private {
 	struct drm_panel *panel;
 	struct drm_bridge *bridge;
 	struct drm_simple_display_pipe pipe;
-	struct drm_fbdev_cma *fbdev;
 
 	void *regs;
 	u32 ienb;
diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c
index 201d57d5cb54..acb738c69873 100644
--- a/drivers/gpu/drm/pl111/pl111_drv.c
+++ b/drivers/gpu/drm/pl111/pl111_drv.c
@@ -64,6 +64,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_bridge.h>
@@ -137,8 +138,7 @@ static int pl111_modeset_init(struct drm_device *dev)
 
 	drm_mode_config_reset(dev);
 
-	priv->fbdev = drm_fbdev_cma_init(dev, 32,
-					 dev->mode_config.num_connector);
+	drm_fb_cma_fbdev_init(dev, 32, 0);
 
 	drm_kms_helper_poll_init(dev);
 
@@ -155,17 +155,10 @@ static int pl111_modeset_init(struct drm_device *dev)
 
 DEFINE_DRM_GEM_CMA_FOPS(drm_fops);
 
-static void pl111_lastclose(struct drm_device *dev)
-{
-	struct pl111_drm_dev_private *priv = dev->dev_private;
-
-	drm_fbdev_cma_restore_mode(priv->fbdev);
-}
-
 static struct drm_driver pl111_drm_driver = {
 	.driver_features =
 		DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_ATOMIC,
-	.lastclose = pl111_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.ioctls = NULL,
 	.fops = &drm_fops,
 	.name = "pl111",
@@ -281,8 +274,7 @@ static int pl111_amba_remove(struct amba_device *amba_dev)
 	struct pl111_drm_dev_private *priv = drm->dev_private;
 
 	drm_dev_unregister(drm);
-	if (priv->fbdev)
-		drm_fbdev_cma_fini(priv->fbdev);
+	drm_fb_cma_fbdev_fini(drm);
 	if (priv->panel)
 		drm_panel_bridge_remove(priv->bridge);
 	drm_mode_config_cleanup(drm);
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 4756b3c9bf2c..9a9214ae0fb5 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -289,6 +289,7 @@ static void qxl_crtc_destroy(struct drm_crtc *crtc)
 {
 	struct qxl_crtc *qxl_crtc = to_qxl_crtc(crtc);
 
+	qxl_bo_unref(&qxl_crtc->cursor_bo);
 	drm_crtc_cleanup(crtc);
 	kfree(qxl_crtc);
 }
@@ -495,6 +496,53 @@ static int qxl_primary_atomic_check(struct drm_plane *plane,
 	return 0;
 }
 
+static int qxl_primary_apply_cursor(struct drm_plane *plane)
+{
+	struct drm_device *dev = plane->dev;
+	struct qxl_device *qdev = dev->dev_private;
+	struct drm_framebuffer *fb = plane->state->fb;
+	struct qxl_crtc *qcrtc = to_qxl_crtc(plane->state->crtc);
+	struct qxl_cursor_cmd *cmd;
+	struct qxl_release *release;
+	int ret = 0;
+
+	if (!qcrtc->cursor_bo)
+		return 0;
+
+	ret = qxl_alloc_release_reserved(qdev, sizeof(*cmd),
+					 QXL_RELEASE_CURSOR_CMD,
+					 &release, NULL);
+	if (ret)
+		return ret;
+
+	ret = qxl_release_list_add(release, qcrtc->cursor_bo);
+	if (ret)
+		goto out_free_release;
+
+	ret = qxl_release_reserve_list(release, false);
+	if (ret)
+		goto out_free_release;
+
+	cmd = (struct qxl_cursor_cmd *)qxl_release_map(qdev, release);
+	cmd->type = QXL_CURSOR_SET;
+	cmd->u.set.position.x = plane->state->crtc_x + fb->hot_x;
+	cmd->u.set.position.y = plane->state->crtc_y + fb->hot_y;
+
+	cmd->u.set.shape = qxl_bo_physical_address(qdev, qcrtc->cursor_bo, 0);
+
+	cmd->u.set.visible = 1;
+	qxl_release_unmap(qdev, release, &cmd->release_info);
+
+	qxl_push_cursor_ring_release(qdev, release, QXL_CMD_CURSOR, false);
+	qxl_release_fence_buffer_objects(release);
+
+	return ret;
+
+out_free_release:
+	qxl_release_free(qdev, release);
+	return ret;
+}
+
 static void qxl_primary_atomic_update(struct drm_plane *plane,
 				      struct drm_plane_state *old_state)
 {
@@ -510,6 +558,7 @@ static void qxl_primary_atomic_update(struct drm_plane *plane,
 	    .x2 = qfb->base.width,
 	    .y2 = qfb->base.height
 	};
+	int ret;
 	bool same_shadow = false;
 
 	if (old_state->fb) {
@@ -531,6 +580,11 @@ static void qxl_primary_atomic_update(struct drm_plane *plane,
 		if (!same_shadow)
 			qxl_io_destroy_primary(qdev);
 		bo_old->is_primary = false;
+
+		ret = qxl_primary_apply_cursor(plane);
+		if (ret)
+			DRM_ERROR(
+			"could not set cursor after creating primary");
 	}
 
 	if (!bo->is_primary) {
@@ -571,11 +625,12 @@ static void qxl_cursor_atomic_update(struct drm_plane *plane,
 	struct drm_device *dev = plane->dev;
 	struct qxl_device *qdev = dev->dev_private;
 	struct drm_framebuffer *fb = plane->state->fb;
+	struct qxl_crtc *qcrtc = to_qxl_crtc(plane->state->crtc);
 	struct qxl_release *release;
 	struct qxl_cursor_cmd *cmd;
 	struct qxl_cursor *cursor;
 	struct drm_gem_object *obj;
-	struct qxl_bo *cursor_bo, *user_bo = NULL;
+	struct qxl_bo *cursor_bo = NULL, *user_bo = NULL;
 	int ret;
 	void *user_ptr;
 	int size = 64*64*4;
@@ -628,6 +683,10 @@ static void qxl_cursor_atomic_update(struct drm_plane *plane,
 		cmd->u.set.shape = qxl_bo_physical_address(qdev,
 							   cursor_bo, 0);
 		cmd->type = QXL_CURSOR_SET;
+
+		qxl_bo_unref(&qcrtc->cursor_bo);
+		qcrtc->cursor_bo = cursor_bo;
+		cursor_bo = NULL;
 	} else {
 
 		ret = qxl_release_reserve_list(release, true);
@@ -645,6 +704,8 @@ static void qxl_cursor_atomic_update(struct drm_plane *plane,
 	qxl_push_cursor_ring_release(qdev, release, QXL_CMD_CURSOR, false);
 	qxl_release_fence_buffer_objects(release);
 
+	qxl_bo_unref(&cursor_bo);
+
 	return;
 
 out_backoff:
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 08752c0ffb35..00a1a66b052a 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -111,6 +111,8 @@ struct qxl_bo_list {
 struct qxl_crtc {
 	struct drm_crtc base;
 	int index;
+
+	struct qxl_bo *cursor_bo;
 };
 
 struct qxl_output {
diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c
index 31effed4a3c8..e8c0b1037230 100644
--- a/drivers/gpu/drm/qxl/qxl_ioctl.c
+++ b/drivers/gpu/drm/qxl/qxl_ioctl.c
@@ -309,6 +309,7 @@ static int qxl_update_area_ioctl(struct drm_device *dev, void *data,
 	int ret;
 	struct drm_gem_object *gobj = NULL;
 	struct qxl_bo *qobj = NULL;
+	struct ttm_operation_ctx ctx = { true, false };
 
 	if (update_area->left >= update_area->right ||
 	    update_area->top >= update_area->bottom)
@@ -326,8 +327,7 @@ static int qxl_update_area_ioctl(struct drm_device *dev, void *data,
 
 	if (!qobj->pin_count) {
 		qxl_ttm_placement_from_domain(qobj, qobj->type, false);
-		ret = ttm_bo_validate(&qobj->tbo, &qobj->placement,
-				      true, false);
+		ret = ttm_bo_validate(&qobj->tbo, &qobj->placement, &ctx);
 		if (unlikely(ret))
 			goto out;
 	}
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index 0a67ddf19c3d..f6b80fe47d1f 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -223,6 +223,7 @@ struct qxl_bo *qxl_bo_ref(struct qxl_bo *bo)
 
 static int __qxl_bo_pin(struct qxl_bo *bo, u32 domain, u64 *gpu_addr)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	struct drm_device *ddev = bo->gem_base.dev;
 	int r;
 
@@ -233,7 +234,7 @@ static int __qxl_bo_pin(struct qxl_bo *bo, u32 domain, u64 *gpu_addr)
 		return 0;
 	}
 	qxl_ttm_placement_from_domain(bo, domain, true);
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (likely(r == 0)) {
 		bo->pin_count = 1;
 		if (gpu_addr != NULL)
@@ -246,6 +247,7 @@ static int __qxl_bo_pin(struct qxl_bo *bo, u32 domain, u64 *gpu_addr)
 
 static int __qxl_bo_unpin(struct qxl_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	struct drm_device *ddev = bo->gem_base.dev;
 	int r, i;
 
@@ -258,7 +260,7 @@ static int __qxl_bo_unpin(struct qxl_bo *bo)
 		return 0;
 	for (i = 0; i < bo->placement.num_placement; i++)
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (unlikely(r != 0))
 		dev_err(ddev->dev, "%p validate failed for unpin\n", bo);
 	return r;
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index a6da6fa6ad58..b223c8d0a491 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -230,12 +230,12 @@ int qxl_release_list_add(struct qxl_release *release, struct qxl_bo *bo)
 
 static int qxl_release_validate_bo(struct qxl_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	int ret;
 
 	if (!bo->pin_count) {
 		qxl_ttm_placement_from_domain(bo, bo->type, false);
-		ret = ttm_bo_validate(&bo->tbo, &bo->placement,
-				      true, false);
+		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		if (ret)
 			return ret;
 	}
@@ -468,7 +468,7 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release)
 
 		reservation_object_add_shared_fence(bo->resv, &release->base);
 		ttm_bo_add_to_lru(bo);
-		__ttm_bo_unreserve(bo);
+		reservation_object_unlock(bo->resv);
 	}
 	spin_unlock(&glob->lru_lock);
 	ww_acquire_fini(&release->ticket);
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index ab4823875311..59cd74c3f3af 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -123,11 +123,8 @@ int qxl_mmap(struct file *filp, struct vm_area_struct *vma)
 	struct qxl_device *qdev;
 	int r;
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) {
-		pr_info("%s: vma->vm_pgoff (%ld) < DRM_FILE_PAGE_OFFSET\n",
-			__func__, vma->vm_pgoff);
+	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
 		return -EINVAL;
-	}
 
 	file_priv = filp->private_data;
 	qdev = file_priv->minor->dev->dev_private;
@@ -294,14 +291,15 @@ static struct ttm_backend_func qxl_backend_func = {
 	.destroy = &qxl_ttm_backend_destroy,
 };
 
-static int qxl_ttm_tt_populate(struct ttm_tt *ttm)
+static int qxl_ttm_tt_populate(struct ttm_tt *ttm,
+			struct ttm_operation_ctx *ctx)
 {
 	int r;
 
 	if (ttm->state != tt_unpopulated)
 		return 0;
 
-	r = ttm_pool_populate(ttm);
+	r = ttm_pool_populate(ttm, ctx);
 	if (r)
 		return r;
 
@@ -344,15 +342,14 @@ static void qxl_move_null(struct ttm_buffer_object *bo,
 	new_mem->mm_node = NULL;
 }
 
-static int qxl_bo_move(struct ttm_buffer_object *bo,
-		       bool evict, bool interruptible,
-		       bool no_wait_gpu,
+static int qxl_bo_move(struct ttm_buffer_object *bo, bool evict,
+		       struct ttm_operation_ctx *ctx,
 		       struct ttm_mem_reg *new_mem)
 {
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	int ret;
 
-	ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+	ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 	if (ret)
 		return ret;
 
@@ -361,8 +358,7 @@ static int qxl_bo_move(struct ttm_buffer_object *bo,
 		qxl_move_null(bo, new_mem);
 		return 0;
 	}
-	return ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu,
-				  new_mem);
+	return ttm_bo_move_memcpy(bo, ctx, new_mem);
 }
 
 static void qxl_bo_move_notify(struct ttm_buffer_object *bo,
@@ -393,7 +389,6 @@ static struct ttm_bo_driver qxl_bo_driver = {
 	.verify_access = &qxl_verify_access,
 	.io_mem_reserve = &qxl_ttm_io_mem_reserve,
 	.io_mem_free = &qxl_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 	.move_notify = &qxl_bo_move_notify,
 };
 
diff --git a/drivers/gpu/drm/r128/r128_state.c b/drivers/gpu/drm/r128/r128_state.c
index 8fdc56c1c953..b9bfa806d346 100644
--- a/drivers/gpu/drm/r128/r128_state.c
+++ b/drivers/gpu/drm/r128/r128_state.c
@@ -982,25 +982,14 @@ static int r128_cce_dispatch_write_pixels(struct drm_device *dev,
 
 	xbuf_size = count * sizeof(*x);
 	ybuf_size = count * sizeof(*y);
-	x = kmalloc(xbuf_size, GFP_KERNEL);
-	if (x == NULL)
-		return -ENOMEM;
-	y = kmalloc(ybuf_size, GFP_KERNEL);
-	if (y == NULL) {
-		kfree(x);
-		return -ENOMEM;
-	}
-	if (copy_from_user(x, depth->x, xbuf_size)) {
-		kfree(x);
-		kfree(y);
-		return -EFAULT;
-	}
-	if (copy_from_user(y, depth->y, xbuf_size)) {
+	x = memdup_user(depth->x, xbuf_size);
+	if (IS_ERR(x))
+		return PTR_ERR(x);
+	y = memdup_user(depth->y, ybuf_size);
+	if (IS_ERR(y)) {
 		kfree(x);
-		kfree(y);
-		return -EFAULT;
+		return PTR_ERR(y);
 	}
-
 	buffer_size = depth->n * sizeof(u32);
 	buffer = memdup_user(depth->buffer, buffer_size);
 	if (IS_ERR(buffer)) {
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index a6511918f632..7c73bc7e2f85 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -1627,8 +1627,6 @@ static const u32 godavari_golden_registers[] =
 
 static void cik_init_golden_registers(struct radeon_device *rdev)
 {
-	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
-	mutex_lock(&rdev->grbm_idx_mutex);
 	switch (rdev->family) {
 	case CHIP_BONAIRE:
 		radeon_program_register_sequence(rdev,
@@ -1703,7 +1701,6 @@ static void cik_init_golden_registers(struct radeon_device *rdev)
 	default:
 		break;
 	}
-	mutex_unlock(&rdev->grbm_idx_mutex);
 }
 
 /**
@@ -3120,7 +3117,6 @@ static void cik_setup_rb(struct radeon_device *rdev,
 	u32 disabled_rbs = 0;
 	u32 enabled_rbs = 0;
 
-	mutex_lock(&rdev->grbm_idx_mutex);
 	for (i = 0; i < se_num; i++) {
 		for (j = 0; j < sh_per_se; j++) {
 			cik_select_se_sh(rdev, i, j);
@@ -3132,7 +3128,6 @@ static void cik_setup_rb(struct radeon_device *rdev,
 		}
 	}
 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
-	mutex_unlock(&rdev->grbm_idx_mutex);
 
 	mask = 1;
 	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
@@ -3143,7 +3138,6 @@ static void cik_setup_rb(struct radeon_device *rdev,
 
 	rdev->config.cik.backend_enable_mask = enabled_rbs;
 
-	mutex_lock(&rdev->grbm_idx_mutex);
 	for (i = 0; i < se_num; i++) {
 		cik_select_se_sh(rdev, i, 0xffffffff);
 		data = 0;
@@ -3171,7 +3165,6 @@ static void cik_setup_rb(struct radeon_device *rdev,
 		WREG32(PA_SC_RASTER_CONFIG, data);
 	}
 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
-	mutex_unlock(&rdev->grbm_idx_mutex);
 }
 
 /**
@@ -3228,35 +3221,8 @@ static void cik_gpu_init(struct radeon_device *rdev)
 	case CHIP_KAVERI:
 		rdev->config.cik.max_shader_engines = 1;
 		rdev->config.cik.max_tile_pipes = 4;
-		if ((rdev->pdev->device == 0x1304) ||
-		    (rdev->pdev->device == 0x1305) ||
-		    (rdev->pdev->device == 0x130C) ||
-		    (rdev->pdev->device == 0x130F) ||
-		    (rdev->pdev->device == 0x1310) ||
-		    (rdev->pdev->device == 0x1311) ||
-		    (rdev->pdev->device == 0x131C)) {
-			rdev->config.cik.max_cu_per_sh = 8;
-			rdev->config.cik.max_backends_per_se = 2;
-		} else if ((rdev->pdev->device == 0x1309) ||
-			   (rdev->pdev->device == 0x130A) ||
-			   (rdev->pdev->device == 0x130D) ||
-			   (rdev->pdev->device == 0x1313) ||
-			   (rdev->pdev->device == 0x131D)) {
-			rdev->config.cik.max_cu_per_sh = 6;
-			rdev->config.cik.max_backends_per_se = 2;
-		} else if ((rdev->pdev->device == 0x1306) ||
-			   (rdev->pdev->device == 0x1307) ||
-			   (rdev->pdev->device == 0x130B) ||
-			   (rdev->pdev->device == 0x130E) ||
-			   (rdev->pdev->device == 0x1315) ||
-			   (rdev->pdev->device == 0x1318) ||
-			   (rdev->pdev->device == 0x131B)) {
-			rdev->config.cik.max_cu_per_sh = 4;
-			rdev->config.cik.max_backends_per_se = 1;
-		} else {
-			rdev->config.cik.max_cu_per_sh = 3;
-			rdev->config.cik.max_backends_per_se = 1;
-		}
+		rdev->config.cik.max_cu_per_sh = 8;
+		rdev->config.cik.max_backends_per_se = 2;
 		rdev->config.cik.max_sh_per_se = 1;
 		rdev->config.cik.max_texture_channel_caches = 4;
 		rdev->config.cik.max_gprs = 256;
@@ -3391,12 +3357,6 @@ static void cik_gpu_init(struct radeon_device *rdev)
 	/* set HW defaults for 3D engine */
 	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
 
-	mutex_lock(&rdev->grbm_idx_mutex);
-	/*
-	 * making sure that the following register writes will be broadcasted
-	 * to all the shaders
-	 */
-	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
 	WREG32(SX_DEBUG_1, 0x20);
 
 	WREG32(TA_CNTL_AUX, 0x00010000);
@@ -3452,7 +3412,6 @@ static void cik_gpu_init(struct radeon_device *rdev)
 
 	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
 	WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
-	mutex_unlock(&rdev->grbm_idx_mutex);
 
 	udelay(50);
 }
@@ -4432,11 +4391,12 @@ static int cik_mec_init(struct radeon_device *rdev)
 	/*
 	 * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
 	 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
-	 * Nonetheless, we assign only 1 pipe because all other pipes will
-	 * be handled by KFD
 	 */
-	rdev->mec.num_mec = 1;
-	rdev->mec.num_pipe = 1;
+	if (rdev->family == CHIP_KAVERI)
+		rdev->mec.num_mec = 2;
+	else
+		rdev->mec.num_mec = 1;
+	rdev->mec.num_pipe = 4;
 	rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
 
 	if (rdev->mec.hpd_eop_obj == NULL) {
@@ -4579,8 +4539,11 @@ static int cik_cp_compute_resume(struct radeon_device *rdev)
 	/* init the pipes */
 	mutex_lock(&rdev->srbm_mutex);
 
-	for (i = 0; i < rdev->mec.num_pipe; ++i) {
-		cik_srbm_select(rdev, 0, i, 0, 0);
+	for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) {
+		int me = (i < 4) ? 1 : 2;
+		int pipe = (i < 4) ? i : (i - 4);
+
+		cik_srbm_select(rdev, me, pipe, 0, 0);
 
 		eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
 		/* write the EOP addr */
@@ -4597,6 +4560,7 @@ static int cik_cp_compute_resume(struct radeon_device *rdev)
 		WREG32(CP_HPD_EOP_CONTROL, tmp);
 
 	}
+	cik_srbm_select(rdev, 0, 0, 0, 0);
 	mutex_unlock(&rdev->srbm_mutex);
 
 	/* init the queues.  Just two for now. */
@@ -5830,7 +5794,6 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
 	u32 i, j, k;
 	u32 mask;
 
-	mutex_lock(&rdev->grbm_idx_mutex);
 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
 			cik_select_se_sh(rdev, i, j);
@@ -5842,7 +5805,6 @@ static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
 		}
 	}
 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
-	mutex_unlock(&rdev->grbm_idx_mutex);
 
 	mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
 	for (k = 0; k < rdev->usec_timeout; k++) {
@@ -5977,12 +5939,10 @@ static int cik_rlc_resume(struct radeon_device *rdev)
 	WREG32(RLC_LB_CNTR_INIT, 0);
 	WREG32(RLC_LB_CNTR_MAX, 0x00008000);
 
-	mutex_lock(&rdev->grbm_idx_mutex);
 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
 	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
 	WREG32(RLC_LB_PARAMS, 0x00600408);
 	WREG32(RLC_LB_CNTL, 0x80000004);
-	mutex_unlock(&rdev->grbm_idx_mutex);
 
 	WREG32(RLC_MC_CNTL, 0);
 	WREG32(RLC_UCODE_CNTL, 0);
@@ -6049,13 +6009,11 @@ static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
 
 		tmp = cik_halt_rlc(rdev);
 
-		mutex_lock(&rdev->grbm_idx_mutex);
 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
 		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
 		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
 		tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
 		WREG32(RLC_SERDES_WR_CTRL, tmp2);
-		mutex_unlock(&rdev->grbm_idx_mutex);
 
 		cik_update_rlc(rdev, tmp);
 
@@ -6098,13 +6056,11 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
 
 		tmp = cik_halt_rlc(rdev);
 
-		mutex_lock(&rdev->grbm_idx_mutex);
 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
 		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
 		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
 		data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
 		WREG32(RLC_SERDES_WR_CTRL, data);
-		mutex_unlock(&rdev->grbm_idx_mutex);
 
 		cik_update_rlc(rdev, tmp);
 
@@ -6148,13 +6104,11 @@ static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
 
 		tmp = cik_halt_rlc(rdev);
 
-		mutex_lock(&rdev->grbm_idx_mutex);
 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
 		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
 		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
 		data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
 		WREG32(RLC_SERDES_WR_CTRL, data);
-		mutex_unlock(&rdev->grbm_idx_mutex);
 
 		cik_update_rlc(rdev, tmp);
 	}
@@ -6583,12 +6537,10 @@ static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
 	u32 mask = 0, tmp, tmp1;
 	int i;
 
-	mutex_lock(&rdev->grbm_idx_mutex);
 	cik_select_se_sh(rdev, se, sh);
 	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
 	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
-	mutex_unlock(&rdev->grbm_idx_mutex);
 
 	tmp &= 0xffff0000;
 
@@ -7074,7 +7026,8 @@ static int cik_irq_init(struct radeon_device *rdev)
 int cik_irq_set(struct radeon_device *rdev)
 {
 	u32 cp_int_cntl;
-	u32 cp_m1p0;
+	u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
+	u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
 	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
 	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
 	u32 grbm_int_cntl = 0;
@@ -7107,6 +7060,13 @@ int cik_irq_set(struct radeon_device *rdev)
 	dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
 
 	cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+	cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+	cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+	cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+	cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+	cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+	cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
+	cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
 
 	/* enable CP interrupts on all rings */
 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
@@ -7121,6 +7081,33 @@ int cik_irq_set(struct radeon_device *rdev)
 			case 0:
 				cp_m1p0 |= TIME_STAMP_INT_ENABLE;
 				break;
+			case 1:
+				cp_m1p1 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 2:
+				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 3:
+				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+				break;
+			default:
+				DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
+				break;
+			}
+		} else if (ring->me == 2) {
+			switch (ring->pipe) {
+			case 0:
+				cp_m2p0 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 1:
+				cp_m2p1 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 2:
+				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 3:
+				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+				break;
 			default:
 				DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
 				break;
@@ -7137,6 +7124,33 @@ int cik_irq_set(struct radeon_device *rdev)
 			case 0:
 				cp_m1p0 |= TIME_STAMP_INT_ENABLE;
 				break;
+			case 1:
+				cp_m1p1 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 2:
+				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 3:
+				cp_m1p2 |= TIME_STAMP_INT_ENABLE;
+				break;
+			default:
+				DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
+				break;
+			}
+		} else if (ring->me == 2) {
+			switch (ring->pipe) {
+			case 0:
+				cp_m2p0 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 1:
+				cp_m2p1 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 2:
+				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+				break;
+			case 3:
+				cp_m2p2 |= TIME_STAMP_INT_ENABLE;
+				break;
 			default:
 				DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
 				break;
@@ -7217,6 +7231,13 @@ int cik_irq_set(struct radeon_device *rdev)
 	WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
 
 	WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
+	WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
+	WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
+	WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
+	WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
+	WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
+	WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
+	WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
 
 	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
 
diff --git a/drivers/gpu/drm/radeon/cik_reg.h b/drivers/gpu/drm/radeon/cik_reg.h
index 4e883fdc59d8..318377df09ef 100644
--- a/drivers/gpu/drm/radeon/cik_reg.h
+++ b/drivers/gpu/drm/radeon/cik_reg.h
@@ -147,8 +147,6 @@
 
 #define CIK_LB_DESKTOP_HEIGHT                     0x6b0c
 
-#define KFD_CIK_SDMA_QUEUE_OFFSET		0x200
-
 #define SQ_IND_INDEX					0x8DE0
 #define SQ_CMD						0x8DEC
 #define SQ_IND_DATA					0x8DE4
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 24fe66c89dfb..5712d63dca20 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -3513,6 +3513,7 @@ static void evergreen_gpu_init(struct radeon_device *rdev)
 		tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
 						EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
 	}
+	rdev->config.evergreen.backend_map = tmp;
 	WREG32(GB_BACKEND_MAP, tmp);
 
 	WREG32(CGTS_SYS_TCC_DISABLE, 0);
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index 9eccd0c81d88..381b0255ff02 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -1148,6 +1148,7 @@ static void cayman_gpu_init(struct radeon_device *rdev)
 						rdev->config.cayman.max_shader_engines,
 						CAYMAN_MAX_BACKENDS, disabled_rb_mask);
 	}
+	rdev->config.cayman.backend_map = tmp;
 	WREG32(GB_BACKEND_MAP, tmp);
 
 	cgts_tcc_disable = 0xffff0000;
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index a8e546569858..d34887873dea 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -731,10 +731,6 @@ struct radeon_doorbell {
 
 int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
 void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
-void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
-				  phys_addr_t *aperture_base,
-				  size_t *aperture_size,
-				  size_t *start_offset);
 
 /*
  * IRQS.
@@ -2442,8 +2438,6 @@ struct radeon_device {
 	struct radeon_atcs		atcs;
 	/* srbm instance registers */
 	struct mutex			srbm_mutex;
-	/* GRBM index mutex. Protects concurrents access to GRBM index */
-	struct mutex			grbm_idx_mutex;
 	/* clock, powergating flags */
 	u32 cg_flags;
 	u32 pg_flags;
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 59dcefb2df3b..df9469a8fdb1 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -90,25 +90,18 @@ void radeon_connector_hotplug(struct drm_connector *connector)
 		/* don't do anything if sink is not display port, i.e.,
 		 * passive dp->(dvi|hdmi) adaptor
 		 */
-		if (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) {
-			int saved_dpms = connector->dpms;
-			/* Only turn off the display if it's physically disconnected */
-			if (!radeon_hpd_sense(rdev, radeon_connector->hpd.hpd)) {
-				drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
-			} else if (radeon_dp_needs_link_train(radeon_connector)) {
-				/* Don't try to start link training before we
-				 * have the dpcd */
-				if (!radeon_dp_getdpcd(radeon_connector))
-					return;
-
-				/* set it to OFF so that drm_helper_connector_dpms()
-				 * won't return immediately since the current state
-				 * is ON at this point.
-				 */
-				connector->dpms = DRM_MODE_DPMS_OFF;
-				drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
-			}
-			connector->dpms = saved_dpms;
+		if (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT &&
+		    radeon_hpd_sense(rdev, radeon_connector->hpd.hpd) &&
+		    radeon_dp_needs_link_train(radeon_connector)) {
+			/* Don't start link training before we have the DPCD */
+			if (!radeon_dp_getdpcd(radeon_connector))
+				return;
+
+			/* Turn the connector off and back on immediately, which
+			 * will trigger link training
+			 */
+			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
+			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
 		}
 	}
 }
@@ -392,7 +385,6 @@ static int radeon_ddc_get_modes(struct drm_connector *connector)
 	if (radeon_connector->edid) {
 		drm_mode_connector_update_edid_property(connector, radeon_connector->edid);
 		ret = drm_add_edid_modes(connector, radeon_connector->edid);
-		drm_edid_to_eld(connector, radeon_connector->edid);
 		return ret;
 	}
 	drm_mode_connector_update_edid_property(connector, NULL);
@@ -900,9 +892,11 @@ radeon_lvds_detect(struct drm_connector *connector, bool force)
 	enum drm_connector_status ret = connector_status_disconnected;
 	int r;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	if (encoder) {
 		struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
@@ -925,8 +919,12 @@ radeon_lvds_detect(struct drm_connector *connector, bool force)
 	/* check acpi lid status ??? */
 
 	radeon_connector_update_scratch_regs(connector, ret);
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
+
 	return ret;
 }
 
@@ -1040,9 +1038,11 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
 	enum drm_connector_status ret = connector_status_disconnected;
 	int r;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	encoder = radeon_best_single_encoder(connector);
 	if (!encoder)
@@ -1109,8 +1109,10 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
 	radeon_connector_update_scratch_regs(connector, ret);
 
 out:
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
 
 	return ret;
 }
@@ -1174,9 +1176,11 @@ radeon_tv_detect(struct drm_connector *connector, bool force)
 	if (!radeon_connector->dac_load_detect)
 		return ret;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	encoder = radeon_best_single_encoder(connector);
 	if (!encoder)
@@ -1188,8 +1192,12 @@ radeon_tv_detect(struct drm_connector *connector, bool force)
 	if (ret == connector_status_connected)
 		ret = radeon_connector_analog_encoder_conflict_solve(connector, encoder, ret, false);
 	radeon_connector_update_scratch_regs(connector, ret);
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
+
 	return ret;
 }
 
@@ -1252,9 +1260,11 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
 	enum drm_connector_status ret = connector_status_disconnected;
 	bool dret = false, broken_edid = false;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	if (radeon_connector->detected_hpd_without_ddc) {
 		force = true;
@@ -1437,8 +1447,10 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
 	}
 
 exit:
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
 
 	return ret;
 }
@@ -1689,9 +1701,11 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
 	if (radeon_dig_connector->is_mst)
 		return connector_status_disconnected;
 
-	r = pm_runtime_get_sync(connector->dev->dev);
-	if (r < 0)
-		return connector_status_disconnected;
+	if (!drm_kms_helper_is_poll_worker()) {
+		r = pm_runtime_get_sync(connector->dev->dev);
+		if (r < 0)
+			return connector_status_disconnected;
+	}
 
 	if (!force && radeon_check_hpd_status_unchanged(connector)) {
 		ret = connector->status;
@@ -1778,8 +1792,10 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
 	}
 
 out:
-	pm_runtime_mark_last_busy(connector->dev->dev);
-	pm_runtime_put_autosuspend(connector->dev->dev);
+	if (!drm_kms_helper_is_poll_worker()) {
+		pm_runtime_mark_last_busy(connector->dev->dev);
+		pm_runtime_put_autosuspend(connector->dev->dev);
+	}
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index ffc10cadcf34..7828a5e10629 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -392,37 +392,6 @@ void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell)
 		__clear_bit(doorbell, rdev->doorbell.used);
 }
 
-/**
- * radeon_doorbell_get_kfd_info - Report doorbell configuration required to
- *                                setup KFD
- *
- * @rdev: radeon_device pointer
- * @aperture_base: output returning doorbell aperture base physical address
- * @aperture_size: output returning doorbell aperture size in bytes
- * @start_offset: output returning # of doorbell bytes reserved for radeon.
- *
- * Radeon and the KFD share the doorbell aperture. Radeon sets it up,
- * takes doorbells required for its own rings and reports the setup to KFD.
- * Radeon reserved doorbells are at the start of the doorbell aperture.
- */
-void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
-				  phys_addr_t *aperture_base,
-				  size_t *aperture_size,
-				  size_t *start_offset)
-{
-	/* The first num_doorbells are used by radeon.
-	 * KFD takes whatever's left in the aperture. */
-	if (rdev->doorbell.size > rdev->doorbell.num_doorbells * sizeof(u32)) {
-		*aperture_base = rdev->doorbell.base;
-		*aperture_size = rdev->doorbell.size;
-		*start_offset = rdev->doorbell.num_doorbells * sizeof(u32);
-	} else {
-		*aperture_base = 0;
-		*aperture_size = 0;
-		*start_offset = 0;
-	}
-}
-
 /*
  * radeon_wb_*()
  * Writeback is the the method by which the the GPU updates special pages
@@ -1341,7 +1310,6 @@ int radeon_device_init(struct radeon_device *rdev,
 	mutex_init(&rdev->pm.mutex);
 	mutex_init(&rdev->gpu_clock_mutex);
 	mutex_init(&rdev->srbm_mutex);
-	mutex_init(&rdev->grbm_idx_mutex);
 	init_rwsem(&rdev->pm.mclk_lock);
 	init_rwsem(&rdev->exclusive_lock);
 	init_waitqueue_head(&rdev->irq.vblank_queue);
@@ -1397,6 +1365,10 @@ int radeon_device_init(struct radeon_device *rdev,
 	if ((rdev->flags & RADEON_IS_PCI) &&
 	    (rdev->family <= CHIP_RS740))
 		rdev->need_dma32 = true;
+#ifdef CONFIG_PPC64
+	if (rdev->family == CHIP_CEDAR)
+		rdev->need_dma32 = true;
+#endif
 
 	dma_bits = rdev->need_dma32 ? 32 : 40;
 	r = pci_set_dma_mask(rdev->pdev, DMA_BIT_MASK(dma_bits));
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index ddfe91efa61e..dfda5e0ed166 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -32,6 +32,7 @@
 
 #include <linux/pm_runtime.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_edid.h>
 
@@ -1362,15 +1363,9 @@ radeon_user_framebuffer_create(struct drm_device *dev,
 	return &radeon_fb->base;
 }
 
-static void radeon_output_poll_changed(struct drm_device *dev)
-{
-	struct radeon_device *rdev = dev->dev_private;
-	radeon_fb_output_poll_changed(rdev);
-}
-
 static const struct drm_mode_config_funcs radeon_mode_funcs = {
 	.fb_create = radeon_user_framebuffer_create,
-	.output_poll_changed = radeon_output_poll_changed
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 };
 
 static const struct drm_prop_enum_list radeon_tmds_pll_enum_list[] =
diff --git a/drivers/gpu/drm/radeon/radeon_dp_mst.c b/drivers/gpu/drm/radeon/radeon_dp_mst.c
index 2917ea1b667e..238e6eb842ea 100644
--- a/drivers/gpu/drm/radeon/radeon_dp_mst.c
+++ b/drivers/gpu/drm/radeon/radeon_dp_mst.c
@@ -197,7 +197,6 @@ static int radeon_dp_mst_get_ddc_modes(struct drm_connector *connector)
 	if (radeon_connector->edid) {
 		drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid);
 		ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid);
-		drm_edid_to_eld(&radeon_connector->base, radeon_connector->edid);
 		return ret;
 	}
 	drm_mode_connector_update_edid_property(&radeon_connector->base, NULL);
@@ -329,7 +328,7 @@ static void radeon_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
 	drm_kms_helper_hotplug_event(dev);
 }
 
-const struct drm_dp_mst_topology_cbs mst_cbs = {
+static const struct drm_dp_mst_topology_cbs mst_cbs = {
 	.add_connector = radeon_dp_add_mst_connector,
 	.register_connector = radeon_dp_register_mst_connector,
 	.destroy_connector = radeon_dp_destroy_mst_connector,
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 33b821d6d018..57c5404a1654 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -306,12 +306,6 @@ static int radeonfb_create(struct drm_fb_helper *helper,
 	return ret;
 }
 
-void radeon_fb_output_poll_changed(struct radeon_device *rdev)
-{
-	if (rdev->mode_info.rfbdev)
-		drm_fb_helper_hotplug_event(&rdev->mode_info.rfbdev->helper);
-}
-
 static int radeon_fbdev_destroy(struct drm_device *dev, struct radeon_fbdev *rfbdev)
 {
 	struct radeon_framebuffer *rfb = &rfbdev->rfb;
@@ -422,19 +416,3 @@ void radeon_fb_remove_connector(struct radeon_device *rdev, struct drm_connector
 	if (rdev->mode_info.rfbdev)
 		drm_fb_helper_remove_one_connector(&rdev->mode_info.rfbdev->helper, connector);
 }
-
-void radeon_fbdev_restore_mode(struct radeon_device *rdev)
-{
-	struct radeon_fbdev *rfbdev = rdev->mode_info.rfbdev;
-	struct drm_fb_helper *fb_helper;
-	int ret;
-
-	if (!rfbdev)
-		return;
-
-	fb_helper = &rfbdev->helper;
-
-	ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper);
-	if (ret)
-		DRM_DEBUG("failed to restore crtc mode\n");
-}
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index cf3deb283da5..27d8e7dd2d06 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -34,8 +34,6 @@ void radeon_gem_object_free(struct drm_gem_object *gobj)
 	struct radeon_bo *robj = gem_to_radeon_bo(gobj);
 
 	if (robj) {
-		if (robj->gem_base.import_attach)
-			drm_prime_gem_destroy(&robj->gem_base, robj->tbo.sg);
 		radeon_mn_unregister(robj);
 		radeon_bo_unref(&robj);
 	}
@@ -285,6 +283,7 @@ int radeon_gem_create_ioctl(struct drm_device *dev, void *data,
 int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *filp)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct radeon_device *rdev = dev->dev_private;
 	struct drm_radeon_gem_userptr *args = data;
 	struct drm_gem_object *gobj;
@@ -343,7 +342,7 @@ int radeon_gem_userptr_ioctl(struct drm_device *dev, void *data,
 		}
 
 		radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_GTT);
-		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		radeon_bo_unreserve(bo);
 		up_read(&current->mm->mmap_sem);
 		if (r)
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index cde037f213d7..dec1e081f529 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -26,6 +26,7 @@
  *          Jerome Glisse
  */
 #include <drm/drmP.h>
+#include <drm/drm_fb_helper.h>
 #include "radeon.h"
 #include <drm/radeon_drm.h>
 #include "radeon_asic.h"
@@ -629,9 +630,7 @@ static int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file
  */
 void radeon_driver_lastclose_kms(struct drm_device *dev)
 {
-	struct radeon_device *rdev = dev->dev_private;
-
-	radeon_fbdev_restore_mode(rdev);
+	drm_fb_helper_lastclose(dev);
 	vga_switcheroo_process_delayed_switch();
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_mn.c b/drivers/gpu/drm/radeon/radeon_mn.c
index 1d62288b7ee3..abd24975c9b1 100644
--- a/drivers/gpu/drm/radeon/radeon_mn.c
+++ b/drivers/gpu/drm/radeon/radeon_mn.c
@@ -124,6 +124,7 @@ static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
 					     unsigned long end)
 {
 	struct radeon_mn *rmn = container_of(mn, struct radeon_mn, mn);
+	struct ttm_operation_ctx ctx = { false, false };
 	struct interval_tree_node *it;
 
 	/* notification is exclusive, but interval is inclusive */
@@ -157,7 +158,7 @@ static void radeon_mn_invalidate_range_start(struct mmu_notifier *mn,
 				DRM_ERROR("(%ld) failed to wait for user bo\n", r);
 
 			radeon_ttm_placement_from_domain(bo, RADEON_GEM_DOMAIN_CPU);
-			r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 			if (r)
 				DRM_ERROR("(%ld) failed to validate user bo\n", r);
 
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index ca0a7ed28c9b..3243e5e01432 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -984,9 +984,6 @@ int radeon_fbdev_init(struct radeon_device *rdev);
 void radeon_fbdev_fini(struct radeon_device *rdev);
 void radeon_fbdev_set_suspend(struct radeon_device *rdev, int state);
 bool radeon_fbdev_robj_is_fb(struct radeon_device *rdev, struct radeon_bo *robj);
-void radeon_fbdev_restore_mode(struct radeon_device *rdev);
-
-void radeon_fb_output_poll_changed(struct radeon_device *rdev);
 
 void radeon_crtc_handle_vblank(struct radeon_device *rdev, int crtc_id);
 
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 093594976126..31f5ad605e59 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -82,6 +82,8 @@ static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 	mutex_unlock(&bo->rdev->gem.mutex);
 	radeon_bo_clear_surface_reg(bo);
 	WARN_ON_ONCE(!list_empty(&bo->va));
+	if (bo->gem_base.import_attach)
+		drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
 	drm_gem_object_release(&bo->gem_base);
 	kfree(bo);
 }
@@ -329,6 +331,7 @@ void radeon_bo_unref(struct radeon_bo **bo)
 int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,
 			     u64 *gpu_addr)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int r, i;
 
 	if (radeon_ttm_tt_has_userptr(bo->tbo.ttm))
@@ -371,7 +374,7 @@ int radeon_bo_pin_restricted(struct radeon_bo *bo, u32 domain, u64 max_offset,
 		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 	}
 
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (likely(r == 0)) {
 		bo->pin_count = 1;
 		if (gpu_addr != NULL)
@@ -393,6 +396,7 @@ int radeon_bo_pin(struct radeon_bo *bo, u32 domain, u64 *gpu_addr)
 
 int radeon_bo_unpin(struct radeon_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	int r, i;
 
 	if (!bo->pin_count) {
@@ -406,7 +410,7 @@ int radeon_bo_unpin(struct radeon_bo *bo)
 		bo->placements[i].lpfn = 0;
 		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
 	}
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (likely(r == 0)) {
 		if (bo->tbo.mem.mem_type == TTM_PL_VRAM)
 			bo->rdev->vram_pin_size -= radeon_bo_size(bo);
@@ -531,6 +535,7 @@ int radeon_bo_list_validate(struct radeon_device *rdev,
 			    struct ww_acquire_ctx *ticket,
 			    struct list_head *head, int ring)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct radeon_bo_list *lobj;
 	struct list_head duplicates;
 	int r;
@@ -572,7 +577,7 @@ int radeon_bo_list_validate(struct radeon_device *rdev,
 				radeon_uvd_force_into_uvd_segment(bo, allowed);
 
 			initial_bytes_moved = atomic64_read(&rdev->num_bytes_moved);
-			r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 			bytes_moved += atomic64_read(&rdev->num_bytes_moved) -
 				       initial_bytes_moved;
 
@@ -792,6 +797,7 @@ void radeon_bo_move_notify(struct ttm_buffer_object *bo,
 
 int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	struct radeon_device *rdev;
 	struct radeon_bo *rbo;
 	unsigned long offset, size, lpfn;
@@ -823,10 +829,10 @@ int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 		    (!rbo->placements[i].lpfn || rbo->placements[i].lpfn > lpfn))
 			rbo->placements[i].lpfn = lpfn;
 	}
-	r = ttm_bo_validate(bo, &rbo->placement, false, false);
+	r = ttm_bo_validate(bo, &rbo->placement, &ctx);
 	if (unlikely(r == -ENOMEM)) {
 		radeon_ttm_placement_from_domain(rbo, RADEON_GEM_DOMAIN_GTT);
-		return ttm_bo_validate(bo, &rbo->placement, false, false);
+		return ttm_bo_validate(bo, &rbo->placement, &ctx);
 	} else if (unlikely(r != 0)) {
 		return r;
 	}
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 326ad068c15a..4b6542538ff9 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -47,7 +47,6 @@ static bool radeon_pm_in_vbl(struct radeon_device *rdev);
 static bool radeon_pm_debug_check_in_vbl(struct radeon_device *rdev, bool finish);
 static void radeon_pm_update_profile(struct radeon_device *rdev);
 static void radeon_pm_set_clocks(struct radeon_device *rdev);
-static void radeon_pm_compute_clocks_dpm(struct radeon_device *rdev);
 
 int radeon_pm_get_type_index(struct radeon_device *rdev,
 			     enum radeon_pm_state_type ps_type,
@@ -80,8 +79,6 @@ void radeon_pm_acpi_event_handler(struct radeon_device *rdev)
 				radeon_dpm_enable_bapm(rdev, rdev->pm.dpm.ac_power);
 		}
 		mutex_unlock(&rdev->pm.mutex);
-		/* allow new DPM state to be picked */
-		radeon_pm_compute_clocks_dpm(rdev);
 	} else if (rdev->pm.pm_method == PM_METHOD_PROFILE) {
 		if (rdev->pm.profile == PM_PROFILE_AUTO) {
 			mutex_lock(&rdev->pm.mutex);
@@ -885,8 +882,7 @@ static struct radeon_ps *radeon_dpm_pick_power_state(struct radeon_device *rdev,
 		dpm_state = POWER_STATE_TYPE_INTERNAL_3DPERF;
 	/* balanced states don't exist at the moment */
 	if (dpm_state == POWER_STATE_TYPE_BALANCED)
-		dpm_state = rdev->pm.dpm.ac_power ?
-			POWER_STATE_TYPE_PERFORMANCE : POWER_STATE_TYPE_BATTERY;
+		dpm_state = POWER_STATE_TYPE_PERFORMANCE;
 
 restart_search:
 	/* Pick the best power state based on current conditions */
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 6ada64db00e9..a0a839bc39bf 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -311,6 +311,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
 				bool no_wait_gpu,
 				struct ttm_mem_reg *new_mem)
 {
+	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
 	struct radeon_device *rdev;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	struct ttm_mem_reg tmp_mem;
@@ -328,8 +329,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
 	placements.fpfn = 0;
 	placements.lpfn = 0;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
-	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
-			     interruptible, no_wait_gpu);
+	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx);
 	if (unlikely(r)) {
 		return r;
 	}
@@ -339,7 +339,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
 		goto out_cleanup;
 	}
 
-	r = ttm_tt_bind(bo->ttm, &tmp_mem);
+	r = ttm_tt_bind(bo->ttm, &tmp_mem, &ctx);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
@@ -347,7 +347,7 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
-	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem);
+	r = ttm_bo_move_ttm(bo, &ctx, new_mem);
 out_cleanup:
 	ttm_bo_mem_put(bo, &tmp_mem);
 	return r;
@@ -358,6 +358,7 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
 				bool no_wait_gpu,
 				struct ttm_mem_reg *new_mem)
 {
+	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
 	struct radeon_device *rdev;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	struct ttm_mem_reg tmp_mem;
@@ -375,12 +376,11 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
 	placements.fpfn = 0;
 	placements.lpfn = 0;
 	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
-	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
-			     interruptible, no_wait_gpu);
+	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, &ctx);
 	if (unlikely(r)) {
 		return r;
 	}
-	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem);
+	r = ttm_bo_move_ttm(bo, &ctx, &tmp_mem);
 	if (unlikely(r)) {
 		goto out_cleanup;
 	}
@@ -393,17 +393,16 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
 	return r;
 }
 
-static int radeon_bo_move(struct ttm_buffer_object *bo,
-			bool evict, bool interruptible,
-			bool no_wait_gpu,
-			struct ttm_mem_reg *new_mem)
+static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict,
+			  struct ttm_operation_ctx *ctx,
+			  struct ttm_mem_reg *new_mem)
 {
 	struct radeon_device *rdev;
 	struct radeon_bo *rbo;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	int r;
 
-	r = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+	r = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 	if (r)
 		return r;
 
@@ -433,19 +432,20 @@ static int radeon_bo_move(struct ttm_buffer_object *bo,
 
 	if (old_mem->mem_type == TTM_PL_VRAM &&
 	    new_mem->mem_type == TTM_PL_SYSTEM) {
-		r = radeon_move_vram_ram(bo, evict, interruptible,
-					no_wait_gpu, new_mem);
+		r = radeon_move_vram_ram(bo, evict, ctx->interruptible,
+					ctx->no_wait_gpu, new_mem);
 	} else if (old_mem->mem_type == TTM_PL_SYSTEM &&
 		   new_mem->mem_type == TTM_PL_VRAM) {
-		r = radeon_move_ram_vram(bo, evict, interruptible,
-					    no_wait_gpu, new_mem);
+		r = radeon_move_ram_vram(bo, evict, ctx->interruptible,
+					    ctx->no_wait_gpu, new_mem);
 	} else {
-		r = radeon_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem);
+		r = radeon_move_blit(bo, evict, ctx->no_wait_gpu,
+				     new_mem, old_mem);
 	}
 
 	if (r) {
 memcpy:
-		r = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, new_mem);
+		r = ttm_bo_move_memcpy(bo, ctx, new_mem);
 		if (r) {
 			return r;
 		}
@@ -721,7 +721,8 @@ static struct radeon_ttm_tt *radeon_ttm_tt_to_gtt(struct ttm_tt *ttm)
 	return (struct radeon_ttm_tt *)ttm;
 }
 
-static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
+static int radeon_ttm_tt_populate(struct ttm_tt *ttm,
+			struct ttm_operation_ctx *ctx)
 {
 	struct radeon_ttm_tt *gtt = radeon_ttm_tt_to_gtt(ttm);
 	struct radeon_device *rdev;
@@ -750,17 +751,17 @@ static int radeon_ttm_tt_populate(struct ttm_tt *ttm)
 	rdev = radeon_get_rdev(ttm->bdev);
 #if IS_ENABLED(CONFIG_AGP)
 	if (rdev->flags & RADEON_IS_AGP) {
-		return ttm_agp_tt_populate(ttm);
+		return ttm_agp_tt_populate(ttm, ctx);
 	}
 #endif
 
 #ifdef CONFIG_SWIOTLB
 	if (swiotlb_nr_tbl()) {
-		return ttm_dma_populate(&gtt->ttm, rdev->dev);
+		return ttm_dma_populate(&gtt->ttm, rdev->dev, ctx);
 	}
 #endif
 
-	return ttm_populate_and_map_pages(rdev->dev, &gtt->ttm);
+	return ttm_populate_and_map_pages(rdev->dev, &gtt->ttm, ctx);
 }
 
 static void radeon_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -844,7 +845,6 @@ static struct ttm_bo_driver radeon_bo_driver = {
 	.fault_reserve_notify = &radeon_bo_fault_reserve_notify,
 	.io_mem_reserve = &radeon_ttm_io_mem_reserve,
 	.io_mem_free = &radeon_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 };
 
 int radeon_ttm_init(struct radeon_device *rdev)
diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c
index d34d1cf33895..95f4db70dd22 100644
--- a/drivers/gpu/drm/radeon/radeon_uvd.c
+++ b/drivers/gpu/drm/radeon/radeon_uvd.c
@@ -995,7 +995,7 @@ int radeon_uvd_calc_upll_dividers(struct radeon_device *rdev,
 		/* calc dclk divider with current vco freq */
 		dclk_div = radeon_uvd_calc_upll_post_div(vco_freq, dclk,
 							 pd_min, pd_even);
-		if (vclk_div > pd_max)
+		if (dclk_div > pd_max)
 			break; /* vco is too big, it has to stop */
 
 		/* calc score with current vco freq */
diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c
index e5c0e635e371..7f1a9c787bd1 100644
--- a/drivers/gpu/drm/radeon/radeon_vm.c
+++ b/drivers/gpu/drm/radeon/radeon_vm.c
@@ -387,6 +387,7 @@ static void radeon_vm_set_pages(struct radeon_device *rdev,
 static int radeon_vm_clear_bo(struct radeon_device *rdev,
 			      struct radeon_bo *bo)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct radeon_ib ib;
 	unsigned entries;
 	uint64_t addr;
@@ -396,7 +397,7 @@ static int radeon_vm_clear_bo(struct radeon_device *rdev,
 	if (r)
 		return r;
 
-	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (r)
 		goto error_unreserve;
 
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index ee3e74266a13..97a0a639dad9 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -2984,6 +2984,11 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev,
 		    (rdev->pdev->device == 0x6667)) {
 			max_sclk = 75000;
 		}
+		if ((rdev->pdev->revision == 0xC3) ||
+		    (rdev->pdev->device == 0x6665)) {
+			max_sclk = 60000;
+			max_mclk = 80000;
+		}
 	} else if (rdev->family == CHIP_OLAND) {
 		if ((rdev->pdev->revision == 0xC7) ||
 		    (rdev->pdev->revision == 0x80) ||
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 301ea1a8018e..5685d5af6998 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -319,7 +319,8 @@ static void rcar_du_crtc_update_planes(struct rcar_du_crtc *rcrtc)
 		struct rcar_du_plane *plane = &rcrtc->group->planes[i];
 		unsigned int j;
 
-		if (plane->plane.state->crtc != &rcrtc->crtc)
+		if (plane->plane.state->crtc != &rcrtc->crtc ||
+		    !plane->plane.state->visible)
 			continue;
 
 		/* Insert the plane in the sorted planes array. */
@@ -557,41 +558,6 @@ static void rcar_du_crtc_stop(struct rcar_du_crtc *rcrtc)
 	rcar_du_group_start_stop(rcrtc->group, false);
 }
 
-void rcar_du_crtc_suspend(struct rcar_du_crtc *rcrtc)
-{
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
-		rcar_du_vsp_disable(rcrtc);
-
-	rcar_du_crtc_stop(rcrtc);
-	rcar_du_crtc_put(rcrtc);
-}
-
-void rcar_du_crtc_resume(struct rcar_du_crtc *rcrtc)
-{
-	unsigned int i;
-
-	if (!rcrtc->crtc.state->active)
-		return;
-
-	rcar_du_crtc_get(rcrtc);
-	rcar_du_crtc_setup(rcrtc);
-
-	/* Commit the planes state. */
-	if (!rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE)) {
-		for (i = 0; i < rcrtc->group->num_planes; ++i) {
-			struct rcar_du_plane *plane = &rcrtc->group->planes[i];
-
-			if (plane->plane.state->crtc != &rcrtc->crtc)
-				continue;
-
-			rcar_du_plane_setup(plane);
-		}
-	}
-
-	rcar_du_crtc_update_planes(rcrtc);
-	rcar_du_crtc_start(rcrtc);
-}
-
 /* -----------------------------------------------------------------------------
  * CRTC Functions
  */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
index d2f29e6b1112..6e02c762a557 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -22,6 +22,7 @@
 #include <linux/wait.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
@@ -34,6 +35,48 @@
  * Device Information
  */
 
+static const struct rcar_du_device_info rzg1_du_r8a7743_info = {
+	.gen = 2,
+	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
+		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
+	.num_crtcs = 2,
+	.routes = {
+		/*
+		 * R8A7743 has one RGB output and one LVDS output
+		 */
+		[RCAR_DU_OUTPUT_DPAD0] = {
+			.possible_crtcs = BIT(1) | BIT(0),
+			.port = 0,
+		},
+		[RCAR_DU_OUTPUT_LVDS0] = {
+			.possible_crtcs = BIT(0),
+			.port = 1,
+		},
+	},
+	.num_lvds = 1,
+};
+
+static const struct rcar_du_device_info rzg1_du_r8a7745_info = {
+	.gen = 2,
+	.features = RCAR_DU_FEATURE_CRTC_IRQ_CLOCK
+		  | RCAR_DU_FEATURE_EXT_CTRL_REGS,
+	.num_crtcs = 2,
+	.routes = {
+		/*
+		 * R8A7745 has two RGB outputs
+		 */
+		[RCAR_DU_OUTPUT_DPAD0] = {
+			.possible_crtcs = BIT(0),
+			.port = 0,
+		},
+		[RCAR_DU_OUTPUT_DPAD1] = {
+			.possible_crtcs = BIT(1),
+			.port = 1,
+		},
+	},
+	.num_lvds = 0,
+};
+
 static const struct rcar_du_device_info rcar_du_r8a7779_info = {
 	.gen = 2,
 	.features = 0,
@@ -207,6 +250,8 @@ static const struct rcar_du_device_info rcar_du_r8a7796_info = {
 };
 
 static const struct of_device_id rcar_du_of_table[] = {
+	{ .compatible = "renesas,du-r8a7743", .data = &rzg1_du_r8a7743_info },
+	{ .compatible = "renesas,du-r8a7745", .data = &rzg1_du_r8a7745_info },
 	{ .compatible = "renesas,du-r8a7779", .data = &rcar_du_r8a7779_info },
 	{ .compatible = "renesas,du-r8a7790", .data = &rcar_du_r8a7790_info },
 	{ .compatible = "renesas,du-r8a7791", .data = &rcar_du_r8a7791_info },
@@ -265,9 +310,19 @@ static struct drm_driver rcar_du_driver = {
 static int rcar_du_pm_suspend(struct device *dev)
 {
 	struct rcar_du_device *rcdu = dev_get_drvdata(dev);
+	struct drm_atomic_state *state;
 
 	drm_kms_helper_poll_disable(rcdu->ddev);
-	/* TODO Suspend the CRTC */
+	drm_fbdev_cma_set_suspend_unlocked(rcdu->fbdev, true);
+
+	state = drm_atomic_helper_suspend(rcdu->ddev);
+	if (IS_ERR(state)) {
+		drm_fbdev_cma_set_suspend_unlocked(rcdu->fbdev, false);
+		drm_kms_helper_poll_enable(rcdu->ddev);
+		return PTR_ERR(state);
+	}
+
+	rcdu->suspend_state = state;
 
 	return 0;
 }
@@ -276,9 +331,10 @@ static int rcar_du_pm_resume(struct device *dev)
 {
 	struct rcar_du_device *rcdu = dev_get_drvdata(dev);
 
-	/* TODO Resume the CRTC */
-
+	drm_atomic_helper_resume(rcdu->ddev, rcdu->suspend_state);
+	drm_fbdev_cma_set_suspend_unlocked(rcdu->fbdev, false);
 	drm_kms_helper_poll_enable(rcdu->ddev);
+
 	return 0;
 }
 #endif
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.h b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
index f8cd79488ece..f400fde65a0c 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
@@ -81,6 +81,7 @@ struct rcar_du_device {
 
 	struct drm_device *ddev;
 	struct drm_fbdev_cma *fbdev;
+	struct drm_atomic_state *suspend_state;
 
 	struct rcar_du_crtc crtcs[RCAR_DU_MAX_CRTCS];
 	unsigned int num_crtcs;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
index b373ad48ef5f..e96f2df0c305 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
@@ -79,10 +79,6 @@ int rcar_du_lvds_connector_init(struct rcar_du_device *rcdu,
 
 	drm_connector_helper_add(connector, &connector_helper_funcs);
 
-	connector->dpms = DRM_MODE_DPMS_OFF;
-	drm_object_property_set_value(&connector->base,
-		rcdu->ddev->mode_config.dpms_property, DRM_MODE_DPMS_OFF);
-
 	ret = drm_mode_connector_attach_encoder(connector, encoder);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
index 61833cc1c699..4a3d16cf3ed6 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
@@ -332,8 +332,8 @@ static void rcar_du_plane_write(struct rcar_du_group *rgrp,
 static void rcar_du_plane_setup_scanout(struct rcar_du_group *rgrp,
 					const struct rcar_du_plane_state *state)
 {
-	unsigned int src_x = state->state.src_x >> 16;
-	unsigned int src_y = state->state.src_y >> 16;
+	unsigned int src_x = state->state.src.x1 >> 16;
+	unsigned int src_y = state->state.src.y1 >> 16;
 	unsigned int index = state->hwindex;
 	unsigned int pitch;
 	bool interlaced;
@@ -357,7 +357,7 @@ static void rcar_du_plane_setup_scanout(struct rcar_du_group *rgrp,
 			dma[i] = gem->paddr + fb->offsets[i];
 		}
 	} else {
-		pitch = state->state.src_w >> 16;
+		pitch = drm_rect_width(&state->state.src) >> 16;
 		dma[0] = 0;
 		dma[1] = 0;
 	}
@@ -521,6 +521,7 @@ static void rcar_du_plane_setup_format(struct rcar_du_group *rgrp,
 				       const struct rcar_du_plane_state *state)
 {
 	struct rcar_du_device *rcdu = rgrp->dev;
+	const struct drm_rect *dst = &state->state.dst;
 
 	if (rcdu->info->gen < 3)
 		rcar_du_plane_setup_format_gen2(rgrp, index, state);
@@ -528,10 +529,10 @@ static void rcar_du_plane_setup_format(struct rcar_du_group *rgrp,
 		rcar_du_plane_setup_format_gen3(rgrp, index, state);
 
 	/* Destination position and size */
-	rcar_du_plane_write(rgrp, index, PnDSXR, state->state.crtc_w);
-	rcar_du_plane_write(rgrp, index, PnDSYR, state->state.crtc_h);
-	rcar_du_plane_write(rgrp, index, PnDPXR, state->state.crtc_x);
-	rcar_du_plane_write(rgrp, index, PnDPYR, state->state.crtc_y);
+	rcar_du_plane_write(rgrp, index, PnDSXR, drm_rect_width(dst));
+	rcar_du_plane_write(rgrp, index, PnDSYR, drm_rect_height(dst));
+	rcar_du_plane_write(rgrp, index, PnDPXR, dst->x1);
+	rcar_du_plane_write(rgrp, index, PnDPYR, dst->y1);
 
 	if (rcdu->info->gen < 3) {
 		/* Wrap-around and blinking, disabled */
@@ -565,27 +566,49 @@ void __rcar_du_plane_setup(struct rcar_du_group *rgrp,
 	}
 }
 
-static int rcar_du_plane_atomic_check(struct drm_plane *plane,
-				      struct drm_plane_state *state)
+int __rcar_du_plane_atomic_check(struct drm_plane *plane,
+				 struct drm_plane_state *state,
+				 const struct rcar_du_format_info **format)
 {
-	struct rcar_du_plane_state *rstate = to_rcar_plane_state(state);
-	struct rcar_du_plane *rplane = to_rcar_plane(plane);
-	struct rcar_du_device *rcdu = rplane->group->dev;
+	struct drm_device *dev = plane->dev;
+	struct drm_crtc_state *crtc_state;
+	struct drm_rect clip;
+	int ret;
 
-	if (!state->fb || !state->crtc) {
-		rstate->format = NULL;
+	if (!state->crtc) {
+		/*
+		 * The visible field is not reset by the DRM core but only
+		 * updated by drm_plane_helper_check_state(), set it manually.
+		 */
+		state->visible = false;
+		*format = NULL;
 		return 0;
 	}
 
-	if (state->src_w >> 16 != state->crtc_w ||
-	    state->src_h >> 16 != state->crtc_h) {
-		dev_dbg(rcdu->dev, "%s: scaling not supported\n", __func__);
-		return -EINVAL;
+	crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	clip.x1 = 0;
+	clip.y1 = 0;
+	clip.x2 = crtc_state->mode.hdisplay;
+	clip.y2 = crtc_state->mode.vdisplay;
+
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  true, true);
+	if (ret < 0)
+		return ret;
+
+	if (!state->visible) {
+		*format = NULL;
+		return 0;
 	}
 
-	rstate->format = rcar_du_format_info(state->fb->format->format);
-	if (rstate->format == NULL) {
-		dev_dbg(rcdu->dev, "%s: unsupported format %08x\n", __func__,
+	*format = rcar_du_format_info(state->fb->format->format);
+	if (*format == NULL) {
+		dev_dbg(dev->dev, "%s: unsupported format %08x\n", __func__,
 			state->fb->format->format);
 		return -EINVAL;
 	}
@@ -593,6 +616,14 @@ static int rcar_du_plane_atomic_check(struct drm_plane *plane,
 	return 0;
 }
 
+static int rcar_du_plane_atomic_check(struct drm_plane *plane,
+				      struct drm_plane_state *state)
+{
+	struct rcar_du_plane_state *rstate = to_rcar_plane_state(state);
+
+	return __rcar_du_plane_atomic_check(plane, state, &rstate->format);
+}
+
 static void rcar_du_plane_atomic_update(struct drm_plane *plane,
 					struct drm_plane_state *old_state)
 {
@@ -600,7 +631,7 @@ static void rcar_du_plane_atomic_update(struct drm_plane *plane,
 	struct rcar_du_plane_state *old_rstate;
 	struct rcar_du_plane_state *new_rstate;
 
-	if (!plane->state->crtc)
+	if (!plane->state->visible)
 		return;
 
 	rcar_du_plane_setup(rplane);
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.h b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
index f62e09f195de..890321b4665d 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.h
@@ -73,6 +73,10 @@ to_rcar_plane_state(struct drm_plane_state *state)
 int rcar_du_atomic_check_planes(struct drm_device *dev,
 				struct drm_atomic_state *state);
 
+int __rcar_du_plane_atomic_check(struct drm_plane *plane,
+				 struct drm_plane_state *state,
+				 const struct rcar_du_format_info **format);
+
 int rcar_du_planes_init(struct rcar_du_group *rgrp);
 
 void __rcar_du_plane_setup(struct rcar_du_group *rgrp,
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
index 2c96147bc444..2c260c33840b 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
@@ -55,14 +55,14 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 	struct rcar_du_plane_state state = {
 		.state = {
 			.crtc = &crtc->crtc,
-			.crtc_x = 0,
-			.crtc_y = 0,
-			.crtc_w = mode->hdisplay,
-			.crtc_h = mode->vdisplay,
-			.src_x = 0,
-			.src_y = 0,
-			.src_w = mode->hdisplay << 16,
-			.src_h = mode->vdisplay << 16,
+			.dst.x1 = 0,
+			.dst.y1 = 0,
+			.dst.x2 = mode->hdisplay,
+			.dst.y2 = mode->vdisplay,
+			.src.x1 = 0,
+			.src.y1 = 0,
+			.src.x2 = mode->hdisplay << 16,
+			.src.y2 = mode->vdisplay << 16,
 			.zpos = 0,
 		},
 		.format = rcar_du_format_info(DRM_FORMAT_ARGB8888),
@@ -178,15 +178,15 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 	};
 	unsigned int i;
 
-	cfg.src.left = state->state.src_x >> 16;
-	cfg.src.top = state->state.src_y >> 16;
-	cfg.src.width = state->state.src_w >> 16;
-	cfg.src.height = state->state.src_h >> 16;
+	cfg.src.left = state->state.src.x1 >> 16;
+	cfg.src.top = state->state.src.y1 >> 16;
+	cfg.src.width = drm_rect_width(&state->state.src) >> 16;
+	cfg.src.height = drm_rect_height(&state->state.src) >> 16;
 
-	cfg.dst.left = state->state.crtc_x;
-	cfg.dst.top = state->state.crtc_y;
-	cfg.dst.width = state->state.crtc_w;
-	cfg.dst.height = state->state.crtc_h;
+	cfg.dst.left = state->state.dst.x1;
+	cfg.dst.top = state->state.dst.y1;
+	cfg.dst.width = drm_rect_width(&state->state.dst);
+	cfg.dst.height = drm_rect_height(&state->state.dst);
 
 	for (i = 0; i < state->format->planes; ++i)
 		cfg.mem[i] = sg_dma_address(state->sg_tables[i].sgl)
@@ -212,7 +212,11 @@ static int rcar_du_vsp_plane_prepare_fb(struct drm_plane *plane,
 	unsigned int i;
 	int ret;
 
-	if (!state->fb)
+	/*
+	 * There's no need to prepare (and unprepare) the framebuffer when the
+	 * plane is not visible, as it will not be displayed.
+	 */
+	if (!state->visible)
 		return 0;
 
 	for (i = 0; i < rstate->format->planes; ++i) {
@@ -253,7 +257,7 @@ static void rcar_du_vsp_plane_cleanup_fb(struct drm_plane *plane,
 	struct rcar_du_vsp *vsp = to_rcar_vsp_plane(plane)->vsp;
 	unsigned int i;
 
-	if (!state->fb)
+	if (!state->visible)
 		return;
 
 	for (i = 0; i < rstate->format->planes; ++i) {
@@ -268,28 +272,8 @@ static int rcar_du_vsp_plane_atomic_check(struct drm_plane *plane,
 					  struct drm_plane_state *state)
 {
 	struct rcar_du_vsp_plane_state *rstate = to_rcar_vsp_plane_state(state);
-	struct rcar_du_vsp_plane *rplane = to_rcar_vsp_plane(plane);
-	struct rcar_du_device *rcdu = rplane->vsp->dev;
 
-	if (!state->fb || !state->crtc) {
-		rstate->format = NULL;
-		return 0;
-	}
-
-	if (state->src_w >> 16 != state->crtc_w ||
-	    state->src_h >> 16 != state->crtc_h) {
-		dev_dbg(rcdu->dev, "%s: scaling not supported\n", __func__);
-		return -EINVAL;
-	}
-
-	rstate->format = rcar_du_format_info(state->fb->format->format);
-	if (rstate->format == NULL) {
-		dev_dbg(rcdu->dev, "%s: unsupported format %08x\n", __func__,
-			state->fb->format->format);
-		return -EINVAL;
-	}
-
-	return 0;
+	return __rcar_du_plane_atomic_check(plane, state, &rstate->format);
 }
 
 static void rcar_du_vsp_plane_atomic_update(struct drm_plane *plane,
@@ -298,7 +282,7 @@ static void rcar_du_vsp_plane_atomic_update(struct drm_plane *plane,
 	struct rcar_du_vsp_plane *rplane = to_rcar_vsp_plane(plane);
 	struct rcar_du_crtc *crtc = to_rcar_crtc(old_state->crtc);
 
-	if (plane->state->crtc)
+	if (plane->state->visible)
 		rcar_du_vsp_plane_setup(rplane);
 	else
 		vsp1_du_atomic_update(rplane->vsp->vsp, crtc->vsp_pipe,
diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
index 93b7102dd008..1262120a3834 100644
--- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
@@ -267,11 +267,10 @@ static struct drm_encoder_funcs rockchip_dp_encoder_funcs = {
 	.destroy = rockchip_dp_drm_encoder_destroy,
 };
 
-static int rockchip_dp_init(struct rockchip_dp_device *dp)
+static int rockchip_dp_of_probe(struct rockchip_dp_device *dp)
 {
 	struct device *dev = dp->dev;
 	struct device_node *np = dev->of_node;
-	int ret;
 
 	dp->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf");
 	if (IS_ERR(dp->grf)) {
@@ -301,19 +300,6 @@ static int rockchip_dp_init(struct rockchip_dp_device *dp)
 		return PTR_ERR(dp->rst);
 	}
 
-	ret = clk_prepare_enable(dp->pclk);
-	if (ret < 0) {
-		DRM_DEV_ERROR(dp->dev, "failed to enable pclk %d\n", ret);
-		return ret;
-	}
-
-	ret = rockchip_dp_pre_init(dp);
-	if (ret < 0) {
-		DRM_DEV_ERROR(dp->dev, "failed to pre init %d\n", ret);
-		clk_disable_unprepare(dp->pclk);
-		return ret;
-	}
-
 	return 0;
 }
 
@@ -359,10 +345,6 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
 	if (!dp_data)
 		return -ENODEV;
 
-	ret = rockchip_dp_init(dp);
-	if (ret < 0)
-		return ret;
-
 	dp->data = dp_data;
 	dp->drm_dev = drm_dev;
 
@@ -396,7 +378,6 @@ static void rockchip_dp_unbind(struct device *dev, struct device *master,
 	rockchip_drm_psr_unregister(&dp->encoder);
 
 	analogix_dp_unbind(dev, master, data);
-	clk_disable_unprepare(dp->pclk);
 }
 
 static const struct component_ops rockchip_dp_component_ops = {
@@ -412,7 +393,7 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 	int ret;
 
 	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, 0, &panel, NULL);
-	if (ret)
+	if (ret < 0)
 		return ret;
 
 	dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL);
@@ -420,9 +401,12 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	dp->dev = dev;
-
 	dp->plat_data.panel = panel;
 
+	ret = rockchip_dp_of_probe(dp);
+	if (ret < 0)
+		return ret;
+
 	/*
 	 * We just use the drvdata until driver run into component
 	 * add function, and then we would set drvdata to null, so
diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.c b/drivers/gpu/drm/rockchip/cdn-dp-core.c
index 275844d0d0ec..ec999d9f15f6 100644
--- a/drivers/gpu/drm/rockchip/cdn-dp-core.c
+++ b/drivers/gpu/drm/rockchip/cdn-dp-core.c
@@ -276,11 +276,9 @@ static int cdn_dp_connector_get_modes(struct drm_connector *connector)
 
 		dp->sink_has_audio = drm_detect_monitor_audio(edid);
 		ret = drm_add_edid_modes(connector, edid);
-		if (ret) {
+		if (ret)
 			drm_mode_connector_update_edid_property(connector,
 								edid);
-			drm_edid_to_eld(connector, edid);
-		}
 	}
 	mutex_unlock(&dp->lock);
 
diff --git a/drivers/gpu/drm/rockchip/inno_hdmi.c b/drivers/gpu/drm/rockchip/inno_hdmi.c
index ee584d87111f..fab30927a889 100644
--- a/drivers/gpu/drm/rockchip/inno_hdmi.c
+++ b/drivers/gpu/drm/rockchip/inno_hdmi.c
@@ -282,6 +282,7 @@ static int inno_hdmi_config_video_vsi(struct inno_hdmi *hdmi,
 	int rc;
 
 	rc = drm_hdmi_vendor_infoframe_from_display_mode(&frame.vendor.hdmi,
+							 &hdmi->connector,
 							 mode);
 
 	return inno_hdmi_upload_frame(hdmi, rc, &frame, INFOFRAME_VSI,
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index 76d63de5921d..d85431400a0d 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -207,13 +207,6 @@ static void rockchip_drm_unbind(struct device *dev)
 	drm_dev_unref(drm_dev);
 }
 
-static void rockchip_drm_lastclose(struct drm_device *dev)
-{
-	struct rockchip_drm_private *priv = dev->dev_private;
-
-	drm_fb_helper_restore_fbdev_mode_unlocked(&priv->fbdev_helper);
-}
-
 static const struct file_operations rockchip_drm_driver_fops = {
 	.owner = THIS_MODULE,
 	.open = drm_open,
@@ -228,7 +221,7 @@ static const struct file_operations rockchip_drm_driver_fops = {
 static struct drm_driver rockchip_drm_driver = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM |
 				  DRIVER_PRIME | DRIVER_ATOMIC,
-	.lastclose		= rockchip_drm_lastclose,
+	.lastclose		= drm_fb_helper_lastclose,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 	.gem_free_object_unlocked = rockchip_gem_free_object,
 	.dumb_create		= rockchip_gem_dumb_create,
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
index cd2ace0c3caa..e266539e04e5 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
@@ -167,20 +167,13 @@ rockchip_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 	return ERR_PTR(ret);
 }
 
-static void rockchip_drm_output_poll_changed(struct drm_device *dev)
-{
-	struct rockchip_drm_private *private = dev->dev_private;
-
-	drm_fb_helper_hotplug_event(&private->fbdev_helper);
-}
-
 static const struct drm_mode_config_helper_funcs rockchip_mode_config_helpers = {
 	.atomic_commit_tail = drm_atomic_helper_commit_tail_rpm,
 };
 
 static const struct drm_mode_config_funcs rockchip_drm_mode_config_funcs = {
 	.fb_create = rockchip_user_fb_create,
-	.output_poll_changed = rockchip_drm_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 19128b4dea54..ba7505292b78 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -659,9 +659,9 @@ static int vop_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
 
-	ret = drm_plane_helper_check_state(state, &clip,
-					   min_scale, max_scale,
-					   true, true);
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  min_scale, max_scale,
+						  true, true);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_lvds.c b/drivers/gpu/drm/rockchip/rockchip_lvds.c
index 84911bdc27d1..e67f4ea28c0e 100644
--- a/drivers/gpu/drm/rockchip/rockchip_lvds.c
+++ b/drivers/gpu/drm/rockchip/rockchip_lvds.c
@@ -25,6 +25,7 @@
 #include <linux/clk.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of_graph.h>
+#include <linux/pinctrl/devinfo.h>
 #include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/reset.h>
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index 4a39049e901a..2e4eea3459fe 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -149,6 +149,34 @@ static const struct vop_data rk3036_vop = {
 	.win_size = ARRAY_SIZE(rk3036_vop_win_data),
 };
 
+static const struct vop_win_phy rk3126_win1_data = {
+	.data_formats = formats_win_lite,
+	.nformats = ARRAY_SIZE(formats_win_lite),
+	.enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 1),
+	.format = VOP_REG(RK3036_SYS_CTRL, 0x7, 6),
+	.rb_swap = VOP_REG(RK3036_SYS_CTRL, 0x1, 19),
+	.dsp_info = VOP_REG(RK3126_WIN1_DSP_INFO, 0x0fff0fff, 0),
+	.dsp_st = VOP_REG(RK3126_WIN1_DSP_ST, 0x1fff1fff, 0),
+	.yrgb_mst = VOP_REG(RK3126_WIN1_MST, 0xffffffff, 0),
+	.yrgb_vir = VOP_REG(RK3036_WIN1_VIR, 0xffff, 0),
+};
+
+static const struct vop_win_data rk3126_vop_win_data[] = {
+	{ .base = 0x00, .phy = &rk3036_win0_data,
+	  .type = DRM_PLANE_TYPE_PRIMARY },
+	{ .base = 0x00, .phy = &rk3126_win1_data,
+	  .type = DRM_PLANE_TYPE_CURSOR },
+};
+
+static const struct vop_data rk3126_vop = {
+	.intr = &rk3036_intr,
+	.common = &rk3036_common,
+	.modeset = &rk3036_modeset,
+	.output = &rk3036_output,
+	.win = rk3126_vop_win_data,
+	.win_size = ARRAY_SIZE(rk3126_vop_win_data),
+};
+
 static const struct vop_scl_extension rk3288_win_full_scl_ext = {
 	.cbcr_vsd_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 31),
 	.cbcr_vsu_mode = VOP_REG(RK3288_WIN0_CTRL1, 0x1, 30),
@@ -510,6 +538,8 @@ static const struct vop_data rk3328_vop = {
 static const struct of_device_id vop_driver_dt_match[] = {
 	{ .compatible = "rockchip,rk3036-vop",
 	  .data = &rk3036_vop },
+	{ .compatible = "rockchip,rk3126-vop",
+	  .data = &rk3126_vop },
 	{ .compatible = "rockchip,rk3288-vop",
 	  .data = &rk3288_vop },
 	{ .compatible = "rockchip,rk3368-vop",
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
index 4a4799ff65de..f81b510ea99c 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
@@ -878,4 +878,10 @@
 #define RK3036_HWC_LUT_ADDR		0x800
 /* rk3036 register definition end */
 
+/* rk3126 register definition */
+#define RK3126_WIN1_MST			0x4c
+#define RK3126_WIN1_DSP_INFO		0x50
+#define RK3126_WIN1_DSP_ST		0x54
+/* rk3126 register definition end */
+
 #endif /* _ROCKCHIP_VOP_REG_H */
diff --git a/drivers/gpu/drm/scheduler/Makefile b/drivers/gpu/drm/scheduler/Makefile
new file mode 100644
index 000000000000..bd0377c0d2ee
--- /dev/null
+++ b/drivers/gpu/drm/scheduler/Makefile
@@ -0,0 +1,26 @@
+#
+# Copyright 2017 Advanced Micro Devices, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+#
+ccflags-y := -Iinclude/drm
+gpu-sched-y := gpu_scheduler.o sched_fence.o
+
+obj-$(CONFIG_DRM_SCHED) += gpu-sched.o
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/scheduler/gpu_scheduler.c
similarity index 55%
rename from drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
rename to drivers/gpu/drm/scheduler/gpu_scheduler.c
index 92ec663fdada..0d95888ccc3e 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/scheduler/gpu_scheduler.c
@@ -19,32 +19,36 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  *
- *
  */
+
 #include <linux/kthread.h>
 #include <linux/wait.h>
 #include <linux/sched.h>
 #include <uapi/linux/sched/types.h>
 #include <drm/drmP.h>
-#include "gpu_scheduler.h"
+#include <drm/gpu_scheduler.h>
+#include <drm/spsc_queue.h>
 
 #define CREATE_TRACE_POINTS
-#include "gpu_sched_trace.h"
+#include <drm/gpu_scheduler_trace.h>
+
+#define to_drm_sched_job(sched_job)		\
+		container_of((sched_job), struct drm_sched_job, queue_node)
 
-static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
-static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
-static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
+static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity);
+static void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
+static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
 
 /* Initialize a given run queue struct */
-static void amd_sched_rq_init(struct amd_sched_rq *rq)
+static void drm_sched_rq_init(struct drm_sched_rq *rq)
 {
 	spin_lock_init(&rq->lock);
 	INIT_LIST_HEAD(&rq->entities);
 	rq->current_entity = NULL;
 }
 
-static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
-				    struct amd_sched_entity *entity)
+static void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
+				    struct drm_sched_entity *entity)
 {
 	if (!list_empty(&entity->list))
 		return;
@@ -53,8 +57,8 @@ static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
 	spin_unlock(&rq->lock);
 }
 
-static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
-				       struct amd_sched_entity *entity)
+static void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
+				       struct drm_sched_entity *entity)
 {
 	if (list_empty(&entity->list))
 		return;
@@ -72,17 +76,17 @@ static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
  *
  * Try to find a ready entity, returns NULL if none found.
  */
-static struct amd_sched_entity *
-amd_sched_rq_select_entity(struct amd_sched_rq *rq)
+static struct drm_sched_entity *
+drm_sched_rq_select_entity(struct drm_sched_rq *rq)
 {
-	struct amd_sched_entity *entity;
+	struct drm_sched_entity *entity;
 
 	spin_lock(&rq->lock);
 
 	entity = rq->current_entity;
 	if (entity) {
 		list_for_each_entry_continue(entity, &rq->entities, list) {
-			if (amd_sched_entity_is_ready(entity)) {
+			if (drm_sched_entity_is_ready(entity)) {
 				rq->current_entity = entity;
 				spin_unlock(&rq->lock);
 				return entity;
@@ -92,7 +96,7 @@ amd_sched_rq_select_entity(struct amd_sched_rq *rq)
 
 	list_for_each_entry(entity, &rq->entities, list) {
 
-		if (amd_sched_entity_is_ready(entity)) {
+		if (drm_sched_entity_is_ready(entity)) {
 			rq->current_entity = entity;
 			spin_unlock(&rq->lock);
 			return entity;
@@ -111,39 +115,37 @@ amd_sched_rq_select_entity(struct amd_sched_rq *rq)
  * Init a context entity used by scheduler when submit to HW ring.
  *
  * @sched	The pointer to the scheduler
- * @entity	The pointer to a valid amd_sched_entity
+ * @entity	The pointer to a valid drm_sched_entity
  * @rq		The run queue this entity belongs
  * @kernel	If this is an entity for the kernel
  * @jobs	The max number of jobs in the job queue
  *
  * return 0 if succeed. negative error code on failure
 */
-int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
-			  struct amd_sched_entity *entity,
-			  struct amd_sched_rq *rq,
-			  uint32_t jobs)
+int drm_sched_entity_init(struct drm_gpu_scheduler *sched,
+			  struct drm_sched_entity *entity,
+			  struct drm_sched_rq *rq,
+			  uint32_t jobs, atomic_t *guilty)
 {
-	int r;
-
 	if (!(sched && entity && rq))
 		return -EINVAL;
 
-	memset(entity, 0, sizeof(struct amd_sched_entity));
+	memset(entity, 0, sizeof(struct drm_sched_entity));
 	INIT_LIST_HEAD(&entity->list);
 	entity->rq = rq;
 	entity->sched = sched;
+	entity->guilty = guilty;
 
 	spin_lock_init(&entity->rq_lock);
 	spin_lock_init(&entity->queue_lock);
-	r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL);
-	if (r)
-		return r;
+	spsc_queue_init(&entity->job_queue);
 
 	atomic_set(&entity->fence_seq, 0);
 	entity->fence_context = dma_fence_context_alloc(2);
 
 	return 0;
 }
+EXPORT_SYMBOL(drm_sched_entity_init);
 
 /**
  * Query if entity is initialized
@@ -153,8 +155,8 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
  *
  * return true if entity is initialized, false otherwise
 */
-static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
-					    struct amd_sched_entity *entity)
+static bool drm_sched_entity_is_initialized(struct drm_gpu_scheduler *sched,
+					    struct drm_sched_entity *entity)
 {
 	return entity->sched == sched &&
 		entity->rq != NULL;
@@ -167,10 +169,10 @@ static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
  *
  * Return true if entity don't has any unscheduled jobs.
  */
-static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
+static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity)
 {
 	rmb();
-	if (kfifo_is_empty(&entity->job_queue))
+	if (spsc_queue_peek(&entity->job_queue) == NULL)
 		return true;
 
 	return false;
@@ -183,9 +185,9 @@ static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
  *
  * Return true if entity could provide a job.
  */
-static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
+static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
 {
-	if (kfifo_is_empty(&entity->job_queue))
+	if (spsc_queue_peek(&entity->job_queue) == NULL)
 		return false;
 
 	if (READ_ONCE(entity->dependency))
@@ -202,12 +204,12 @@ static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity)
  *
  * Cleanup and free the allocated resources.
  */
-void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
-			   struct amd_sched_entity *entity)
+void drm_sched_entity_fini(struct drm_gpu_scheduler *sched,
+			   struct drm_sched_entity *entity)
 {
 	int r;
 
-	if (!amd_sched_entity_is_initialized(sched, entity))
+	if (!drm_sched_entity_is_initialized(sched, entity))
 		return;
 	/**
 	 * The client will not queue more IBs during this fini, consume existing
@@ -217,48 +219,55 @@ void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 		r = -ERESTARTSYS;
 	else
 		r = wait_event_killable(sched->job_scheduled,
-					amd_sched_entity_is_idle(entity));
-	amd_sched_entity_set_rq(entity, NULL);
+					drm_sched_entity_is_idle(entity));
+	drm_sched_entity_set_rq(entity, NULL);
 	if (r) {
-		struct amd_sched_job *job;
+		struct drm_sched_job *job;
 
 		/* Park the kernel for a moment to make sure it isn't processing
 		 * our enity.
 		 */
 		kthread_park(sched->thread);
 		kthread_unpark(sched->thread);
-		while (kfifo_out(&entity->job_queue, &job, sizeof(job))) {
-			struct amd_sched_fence *s_fence = job->s_fence;
-			amd_sched_fence_scheduled(s_fence);
+		if (entity->dependency) {
+			dma_fence_remove_callback(entity->dependency,
+						  &entity->cb);
+			dma_fence_put(entity->dependency);
+			entity->dependency = NULL;
+		}
+
+		while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) {
+			struct drm_sched_fence *s_fence = job->s_fence;
+			drm_sched_fence_scheduled(s_fence);
 			dma_fence_set_error(&s_fence->finished, -ESRCH);
-			amd_sched_fence_finished(s_fence);
+			drm_sched_fence_finished(s_fence);
+			WARN_ON(s_fence->parent);
 			dma_fence_put(&s_fence->finished);
 			sched->ops->free_job(job);
 		}
-
 	}
-	kfifo_free(&entity->job_queue);
 }
+EXPORT_SYMBOL(drm_sched_entity_fini);
 
-static void amd_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
+static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
 {
-	struct amd_sched_entity *entity =
-		container_of(cb, struct amd_sched_entity, cb);
+	struct drm_sched_entity *entity =
+		container_of(cb, struct drm_sched_entity, cb);
 	entity->dependency = NULL;
 	dma_fence_put(f);
-	amd_sched_wakeup(entity->sched);
+	drm_sched_wakeup(entity->sched);
 }
 
-static void amd_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb)
+static void drm_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb)
 {
-	struct amd_sched_entity *entity =
-		container_of(cb, struct amd_sched_entity, cb);
+	struct drm_sched_entity *entity =
+		container_of(cb, struct drm_sched_entity, cb);
 	entity->dependency = NULL;
 	dma_fence_put(f);
 }
 
-void amd_sched_entity_set_rq(struct amd_sched_entity *entity,
-			     struct amd_sched_rq *rq)
+void drm_sched_entity_set_rq(struct drm_sched_entity *entity,
+			     struct drm_sched_rq *rq)
 {
 	if (entity->rq == rq)
 		return;
@@ -266,37 +275,39 @@ void amd_sched_entity_set_rq(struct amd_sched_entity *entity,
 	spin_lock(&entity->rq_lock);
 
 	if (entity->rq)
-		amd_sched_rq_remove_entity(entity->rq, entity);
+		drm_sched_rq_remove_entity(entity->rq, entity);
 
 	entity->rq = rq;
 	if (rq)
-		amd_sched_rq_add_entity(rq, entity);
+		drm_sched_rq_add_entity(rq, entity);
 
 	spin_unlock(&entity->rq_lock);
 }
+EXPORT_SYMBOL(drm_sched_entity_set_rq);
 
-bool amd_sched_dependency_optimized(struct dma_fence* fence,
-				    struct amd_sched_entity *entity)
+bool drm_sched_dependency_optimized(struct dma_fence* fence,
+				    struct drm_sched_entity *entity)
 {
-	struct amd_gpu_scheduler *sched = entity->sched;
-	struct amd_sched_fence *s_fence;
+	struct drm_gpu_scheduler *sched = entity->sched;
+	struct drm_sched_fence *s_fence;
 
 	if (!fence || dma_fence_is_signaled(fence))
 		return false;
 	if (fence->context == entity->fence_context)
 		return true;
-	s_fence = to_amd_sched_fence(fence);
+	s_fence = to_drm_sched_fence(fence);
 	if (s_fence && s_fence->sched == sched)
 		return true;
 
 	return false;
 }
+EXPORT_SYMBOL(drm_sched_dependency_optimized);
 
-static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
+static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity)
 {
-	struct amd_gpu_scheduler *sched = entity->sched;
+	struct drm_gpu_scheduler *sched = entity->sched;
 	struct dma_fence * fence = entity->dependency;
-	struct amd_sched_fence *s_fence;
+	struct drm_sched_fence *s_fence;
 
 	if (fence->context == entity->fence_context) {
 		/* We can ignore fences from ourself */
@@ -304,7 +315,7 @@ static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 		return false;
 	}
 
-	s_fence = to_amd_sched_fence(fence);
+	s_fence = to_drm_sched_fence(fence);
 	if (s_fence && s_fence->sched == sched) {
 
 		/*
@@ -315,7 +326,7 @@ static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 		dma_fence_put(entity->dependency);
 		entity->dependency = fence;
 		if (!dma_fence_add_callback(fence, &entity->cb,
-					    amd_sched_entity_clear_dep))
+					    drm_sched_entity_clear_dep))
 			return true;
 
 		/* Ignore it when it is already scheduled */
@@ -324,48 +335,52 @@ static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 	}
 
 	if (!dma_fence_add_callback(entity->dependency, &entity->cb,
-				    amd_sched_entity_wakeup))
+				    drm_sched_entity_wakeup))
 		return true;
 
 	dma_fence_put(entity->dependency);
 	return false;
 }
 
-static struct amd_sched_job *
-amd_sched_entity_peek_job(struct amd_sched_entity *entity)
+static struct drm_sched_job *
+drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 {
-	struct amd_gpu_scheduler *sched = entity->sched;
-	struct amd_sched_job *sched_job;
+	struct drm_gpu_scheduler *sched = entity->sched;
+	struct drm_sched_job *sched_job = to_drm_sched_job(
+						spsc_queue_peek(&entity->job_queue));
 
-	if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job)))
+	if (!sched_job)
 		return NULL;
 
-	while ((entity->dependency = sched->ops->dependency(sched_job)))
-		if (amd_sched_entity_add_dependency_cb(entity))
+	while ((entity->dependency = sched->ops->dependency(sched_job, entity)))
+		if (drm_sched_entity_add_dependency_cb(entity))
 			return NULL;
 
+	/* skip jobs from entity that marked guilty */
+	if (entity->guilty && atomic_read(entity->guilty))
+		dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED);
+
+	spsc_queue_pop(&entity->job_queue);
 	return sched_job;
 }
 
 /**
- * Helper to submit a job to the job queue
+ * Submit a job to the job queue
  *
  * @sched_job		The pointer to job required to submit
  *
- * Returns true if we could submit the job.
+ * Returns 0 for success, negative error code otherwise.
  */
-static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
+void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
+			       struct drm_sched_entity *entity)
 {
-	struct amd_gpu_scheduler *sched = sched_job->sched;
-	struct amd_sched_entity *entity = sched_job->s_entity;
-	bool added, first = false;
+	struct drm_gpu_scheduler *sched = sched_job->sched;
+	bool first = false;
 
-	spin_lock(&entity->queue_lock);
-	added = kfifo_in(&entity->job_queue, &sched_job,
-			sizeof(sched_job)) == sizeof(sched_job);
+	trace_drm_sched_job(sched_job, entity);
 
-	if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job))
-		first = true;
+	spin_lock(&entity->queue_lock);
+	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
 	spin_unlock(&entity->queue_lock);
 
@@ -373,26 +388,26 @@ static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
 	if (first) {
 		/* Add the entity to the run queue */
 		spin_lock(&entity->rq_lock);
-		amd_sched_rq_add_entity(entity->rq, entity);
+		drm_sched_rq_add_entity(entity->rq, entity);
 		spin_unlock(&entity->rq_lock);
-		amd_sched_wakeup(sched);
+		drm_sched_wakeup(sched);
 	}
-	return added;
 }
+EXPORT_SYMBOL(drm_sched_entity_push_job);
 
 /* job_finish is called after hw fence signaled
  */
-static void amd_sched_job_finish(struct work_struct *work)
+static void drm_sched_job_finish(struct work_struct *work)
 {
-	struct amd_sched_job *s_job = container_of(work, struct amd_sched_job,
+	struct drm_sched_job *s_job = container_of(work, struct drm_sched_job,
 						   finish_work);
-	struct amd_gpu_scheduler *sched = s_job->sched;
+	struct drm_gpu_scheduler *sched = s_job->sched;
 
 	/* remove job from ring_mirror_list */
 	spin_lock(&sched->job_list_lock);
 	list_del_init(&s_job->node);
 	if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
-		struct amd_sched_job *next;
+		struct drm_sched_job *next;
 
 		spin_unlock(&sched->job_list_lock);
 		cancel_delayed_work_sync(&s_job->work_tdr);
@@ -400,7 +415,7 @@ static void amd_sched_job_finish(struct work_struct *work)
 
 		/* queue TDR for next job */
 		next = list_first_entry_or_null(&sched->ring_mirror_list,
-						struct amd_sched_job, node);
+						struct drm_sched_job, node);
 
 		if (next)
 			schedule_delayed_work(&next->work_tdr, sched->timeout);
@@ -410,41 +425,43 @@ static void amd_sched_job_finish(struct work_struct *work)
 	sched->ops->free_job(s_job);
 }
 
-static void amd_sched_job_finish_cb(struct dma_fence *f,
+static void drm_sched_job_finish_cb(struct dma_fence *f,
 				    struct dma_fence_cb *cb)
 {
-	struct amd_sched_job *job = container_of(cb, struct amd_sched_job,
+	struct drm_sched_job *job = container_of(cb, struct drm_sched_job,
 						 finish_cb);
 	schedule_work(&job->finish_work);
 }
 
-static void amd_sched_job_begin(struct amd_sched_job *s_job)
+static void drm_sched_job_begin(struct drm_sched_job *s_job)
 {
-	struct amd_gpu_scheduler *sched = s_job->sched;
+	struct drm_gpu_scheduler *sched = s_job->sched;
 
 	dma_fence_add_callback(&s_job->s_fence->finished, &s_job->finish_cb,
-			       amd_sched_job_finish_cb);
+			       drm_sched_job_finish_cb);
 
 	spin_lock(&sched->job_list_lock);
 	list_add_tail(&s_job->node, &sched->ring_mirror_list);
 	if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
 	    list_first_entry_or_null(&sched->ring_mirror_list,
-				     struct amd_sched_job, node) == s_job)
+				     struct drm_sched_job, node) == s_job)
 		schedule_delayed_work(&s_job->work_tdr, sched->timeout);
 	spin_unlock(&sched->job_list_lock);
 }
 
-static void amd_sched_job_timedout(struct work_struct *work)
+static void drm_sched_job_timedout(struct work_struct *work)
 {
-	struct amd_sched_job *job = container_of(work, struct amd_sched_job,
+	struct drm_sched_job *job = container_of(work, struct drm_sched_job,
 						 work_tdr.work);
 
 	job->sched->ops->timedout_job(job);
 }
 
-void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched)
+void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad)
 {
-	struct amd_sched_job *s_job;
+	struct drm_sched_job *s_job;
+	struct drm_sched_entity *entity, *tmp;
+	int i;
 
 	spin_lock(&sched->job_list_lock);
 	list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
@@ -457,31 +474,57 @@ void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched)
 		}
 	}
 	spin_unlock(&sched->job_list_lock);
-}
 
-void amd_sched_job_kickout(struct amd_sched_job *s_job)
-{
-	struct amd_gpu_scheduler *sched = s_job->sched;
-
-	spin_lock(&sched->job_list_lock);
-	list_del_init(&s_job->node);
-	spin_unlock(&sched->job_list_lock);
+	if (bad && bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
+		atomic_inc(&bad->karma);
+		/* don't increase @bad's karma if it's from KERNEL RQ,
+		 * becuase sometimes GPU hang would cause kernel jobs (like VM updating jobs)
+		 * corrupt but keep in mind that kernel jobs always considered good.
+		 */
+		for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; i++ ) {
+			struct drm_sched_rq *rq = &sched->sched_rq[i];
+
+			spin_lock(&rq->lock);
+			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
+				if (bad->s_fence->scheduled.context == entity->fence_context) {
+				    if (atomic_read(&bad->karma) > bad->sched->hang_limit)
+						if (entity->guilty)
+							atomic_set(entity->guilty, 1);
+					break;
+				}
+			}
+			spin_unlock(&rq->lock);
+			if (&entity->list != &rq->entities)
+				break;
+		}
+	}
 }
+EXPORT_SYMBOL(drm_sched_hw_job_reset);
 
-void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
+void drm_sched_job_recovery(struct drm_gpu_scheduler *sched)
 {
-	struct amd_sched_job *s_job, *tmp;
+	struct drm_sched_job *s_job, *tmp;
+	bool found_guilty = false;
 	int r;
 
 	spin_lock(&sched->job_list_lock);
 	s_job = list_first_entry_or_null(&sched->ring_mirror_list,
-					 struct amd_sched_job, node);
+					 struct drm_sched_job, node);
 	if (s_job && sched->timeout != MAX_SCHEDULE_TIMEOUT)
 		schedule_delayed_work(&s_job->work_tdr, sched->timeout);
 
 	list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
-		struct amd_sched_fence *s_fence = s_job->s_fence;
+		struct drm_sched_fence *s_fence = s_job->s_fence;
 		struct dma_fence *fence;
+		uint64_t guilty_context;
+
+		if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
+			found_guilty = true;
+			guilty_context = s_job->s_fence->scheduled.context;
+		}
+
+		if (found_guilty && s_job->s_fence->scheduled.context == guilty_context)
+			dma_fence_set_error(&s_fence->finished, -ECANCELED);
 
 		spin_unlock(&sched->job_list_lock);
 		fence = sched->ops->run_job(s_job);
@@ -489,62 +532,47 @@ void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
 		if (fence) {
 			s_fence->parent = dma_fence_get(fence);
 			r = dma_fence_add_callback(fence, &s_fence->cb,
-						   amd_sched_process_job);
+						   drm_sched_process_job);
 			if (r == -ENOENT)
-				amd_sched_process_job(fence, &s_fence->cb);
+				drm_sched_process_job(fence, &s_fence->cb);
 			else if (r)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
 			dma_fence_put(fence);
 		} else {
-			DRM_ERROR("Failed to run job!\n");
-			amd_sched_process_job(NULL, &s_fence->cb);
+			drm_sched_process_job(NULL, &s_fence->cb);
 		}
 		spin_lock(&sched->job_list_lock);
 	}
 	spin_unlock(&sched->job_list_lock);
 }
-
-/**
- * Submit a job to the job queue
- *
- * @sched_job		The pointer to job required to submit
- *
- * Returns 0 for success, negative error code otherwise.
- */
-void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
-{
-	struct amd_sched_entity *entity = sched_job->s_entity;
-
-	trace_amd_sched_job(sched_job);
-	wait_event(entity->sched->job_scheduled,
-		   amd_sched_entity_in(sched_job));
-}
+EXPORT_SYMBOL(drm_sched_job_recovery);
 
 /* init a sched_job with basic field */
-int amd_sched_job_init(struct amd_sched_job *job,
-		       struct amd_gpu_scheduler *sched,
-		       struct amd_sched_entity *entity,
+int drm_sched_job_init(struct drm_sched_job *job,
+		       struct drm_gpu_scheduler *sched,
+		       struct drm_sched_entity *entity,
 		       void *owner)
 {
 	job->sched = sched;
-	job->s_entity = entity;
-	job->s_fence = amd_sched_fence_create(entity, owner);
+	job->s_priority = entity->rq - sched->sched_rq;
+	job->s_fence = drm_sched_fence_create(entity, owner);
 	if (!job->s_fence)
 		return -ENOMEM;
 	job->id = atomic64_inc_return(&sched->job_id_count);
 
-	INIT_WORK(&job->finish_work, amd_sched_job_finish);
+	INIT_WORK(&job->finish_work, drm_sched_job_finish);
 	INIT_LIST_HEAD(&job->node);
-	INIT_DELAYED_WORK(&job->work_tdr, amd_sched_job_timedout);
+	INIT_DELAYED_WORK(&job->work_tdr, drm_sched_job_timedout);
 
 	return 0;
 }
+EXPORT_SYMBOL(drm_sched_job_init);
 
 /**
  * Return ture if we can push more jobs to the hw.
  */
-static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
+static bool drm_sched_ready(struct drm_gpu_scheduler *sched)
 {
 	return atomic_read(&sched->hw_rq_count) <
 		sched->hw_submission_limit;
@@ -553,27 +581,27 @@ static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
 /**
  * Wake up the scheduler when it is ready
  */
-static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
+static void drm_sched_wakeup(struct drm_gpu_scheduler *sched)
 {
-	if (amd_sched_ready(sched))
+	if (drm_sched_ready(sched))
 		wake_up_interruptible(&sched->wake_up_worker);
 }
 
 /**
  * Select next entity to process
 */
-static struct amd_sched_entity *
-amd_sched_select_entity(struct amd_gpu_scheduler *sched)
+static struct drm_sched_entity *
+drm_sched_select_entity(struct drm_gpu_scheduler *sched)
 {
-	struct amd_sched_entity *entity;
+	struct drm_sched_entity *entity;
 	int i;
 
-	if (!amd_sched_ready(sched))
+	if (!drm_sched_ready(sched))
 		return NULL;
 
 	/* Kernel run queue has higher priority than normal run queue*/
-	for (i = AMD_SCHED_PRIORITY_MAX - 1; i >= AMD_SCHED_PRIORITY_MIN; i--) {
-		entity = amd_sched_rq_select_entity(&sched->sched_rq[i]);
+	for (i = DRM_SCHED_PRIORITY_MAX - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) {
+		entity = drm_sched_rq_select_entity(&sched->sched_rq[i]);
 		if (entity)
 			break;
 	}
@@ -581,22 +609,22 @@ amd_sched_select_entity(struct amd_gpu_scheduler *sched)
 	return entity;
 }
 
-static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
+static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb)
 {
-	struct amd_sched_fence *s_fence =
-		container_of(cb, struct amd_sched_fence, cb);
-	struct amd_gpu_scheduler *sched = s_fence->sched;
+	struct drm_sched_fence *s_fence =
+		container_of(cb, struct drm_sched_fence, cb);
+	struct drm_gpu_scheduler *sched = s_fence->sched;
 
 	dma_fence_get(&s_fence->finished);
 	atomic_dec(&sched->hw_rq_count);
-	amd_sched_fence_finished(s_fence);
+	drm_sched_fence_finished(s_fence);
 
-	trace_amd_sched_process_job(s_fence);
+	trace_drm_sched_process_job(s_fence);
 	dma_fence_put(&s_fence->finished);
 	wake_up_interruptible(&sched->wake_up_worker);
 }
 
-static bool amd_sched_blocked(struct amd_gpu_scheduler *sched)
+static bool drm_sched_blocked(struct drm_gpu_scheduler *sched)
 {
 	if (kthread_should_park()) {
 		kthread_parkme();
@@ -606,58 +634,54 @@ static bool amd_sched_blocked(struct amd_gpu_scheduler *sched)
 	return false;
 }
 
-static int amd_sched_main(void *param)
+static int drm_sched_main(void *param)
 {
 	struct sched_param sparam = {.sched_priority = 1};
-	struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
-	int r, count;
+	struct drm_gpu_scheduler *sched = (struct drm_gpu_scheduler *)param;
+	int r;
 
 	sched_setscheduler(current, SCHED_FIFO, &sparam);
 
 	while (!kthread_should_stop()) {
-		struct amd_sched_entity *entity = NULL;
-		struct amd_sched_fence *s_fence;
-		struct amd_sched_job *sched_job;
+		struct drm_sched_entity *entity = NULL;
+		struct drm_sched_fence *s_fence;
+		struct drm_sched_job *sched_job;
 		struct dma_fence *fence;
 
 		wait_event_interruptible(sched->wake_up_worker,
-					 (!amd_sched_blocked(sched) &&
-					  (entity = amd_sched_select_entity(sched))) ||
+					 (!drm_sched_blocked(sched) &&
+					  (entity = drm_sched_select_entity(sched))) ||
 					 kthread_should_stop());
 
 		if (!entity)
 			continue;
 
-		sched_job = amd_sched_entity_peek_job(entity);
+		sched_job = drm_sched_entity_pop_job(entity);
 		if (!sched_job)
 			continue;
 
 		s_fence = sched_job->s_fence;
 
 		atomic_inc(&sched->hw_rq_count);
-		amd_sched_job_begin(sched_job);
+		drm_sched_job_begin(sched_job);
 
 		fence = sched->ops->run_job(sched_job);
-		amd_sched_fence_scheduled(s_fence);
+		drm_sched_fence_scheduled(s_fence);
 
 		if (fence) {
 			s_fence->parent = dma_fence_get(fence);
 			r = dma_fence_add_callback(fence, &s_fence->cb,
-						   amd_sched_process_job);
+						   drm_sched_process_job);
 			if (r == -ENOENT)
-				amd_sched_process_job(fence, &s_fence->cb);
+				drm_sched_process_job(fence, &s_fence->cb);
 			else if (r)
 				DRM_ERROR("fence add callback failed (%d)\n",
 					  r);
 			dma_fence_put(fence);
 		} else {
-			DRM_ERROR("Failed to run job!\n");
-			amd_sched_process_job(NULL, &s_fence->cb);
+			drm_sched_process_job(NULL, &s_fence->cb);
 		}
 
-		count = kfifo_out(&entity->job_queue, &sched_job,
-				sizeof(sched_job));
-		WARN_ON(count != sizeof(sched_job));
 		wake_up(&sched->job_scheduled);
 	}
 	return 0;
@@ -673,17 +697,21 @@ static int amd_sched_main(void *param)
  *
  * Return 0 on success, otherwise error code.
 */
-int amd_sched_init(struct amd_gpu_scheduler *sched,
-		   const struct amd_sched_backend_ops *ops,
-		   unsigned hw_submission, long timeout, const char *name)
+int drm_sched_init(struct drm_gpu_scheduler *sched,
+		   const struct drm_sched_backend_ops *ops,
+		   unsigned hw_submission,
+		   unsigned hang_limit,
+		   long timeout,
+		   const char *name)
 {
 	int i;
 	sched->ops = ops;
 	sched->hw_submission_limit = hw_submission;
 	sched->name = name;
 	sched->timeout = timeout;
-	for (i = AMD_SCHED_PRIORITY_MIN; i < AMD_SCHED_PRIORITY_MAX; i++)
-		amd_sched_rq_init(&sched->sched_rq[i]);
+	sched->hang_limit = hang_limit;
+	for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_MAX; i++)
+		drm_sched_rq_init(&sched->sched_rq[i]);
 
 	init_waitqueue_head(&sched->wake_up_worker);
 	init_waitqueue_head(&sched->job_scheduled);
@@ -693,7 +721,7 @@ int amd_sched_init(struct amd_gpu_scheduler *sched,
 	atomic64_set(&sched->job_id_count, 0);
 
 	/* Each scheduler will run on a seperate kernel thread */
-	sched->thread = kthread_run(amd_sched_main, sched, sched->name);
+	sched->thread = kthread_run(drm_sched_main, sched, sched->name);
 	if (IS_ERR(sched->thread)) {
 		DRM_ERROR("Failed to create scheduler for %s.\n", name);
 		return PTR_ERR(sched->thread);
@@ -701,14 +729,16 @@ int amd_sched_init(struct amd_gpu_scheduler *sched,
 
 	return 0;
 }
+EXPORT_SYMBOL(drm_sched_init);
 
 /**
  * Destroy a gpu scheduler
  *
  * @sched	The pointer to the scheduler
  */
-void amd_sched_fini(struct amd_gpu_scheduler *sched)
+void drm_sched_fini(struct drm_gpu_scheduler *sched)
 {
 	if (sched->thread)
 		kthread_stop(sched->thread);
 }
+EXPORT_SYMBOL(drm_sched_fini);
diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
similarity index 58%
rename from drivers/gpu/drm/amd/scheduler/sched_fence.c
rename to drivers/gpu/drm/scheduler/sched_fence.c
index 33f54d0a5c4f..69aab086b913 100644
--- a/drivers/gpu/drm/amd/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -19,20 +19,20 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  *
- *
  */
+
 #include <linux/kthread.h>
 #include <linux/wait.h>
 #include <linux/sched.h>
 #include <drm/drmP.h>
-#include "gpu_scheduler.h"
+#include <drm/gpu_scheduler.h>
 
 static struct kmem_cache *sched_fence_slab;
 
-int amd_sched_fence_slab_init(void)
+static int __init drm_sched_fence_slab_init(void)
 {
 	sched_fence_slab = kmem_cache_create(
-		"amd_sched_fence", sizeof(struct amd_sched_fence), 0,
+		"drm_sched_fence", sizeof(struct drm_sched_fence), 0,
 		SLAB_HWCACHE_ALIGN, NULL);
 	if (!sched_fence_slab)
 		return -ENOMEM;
@@ -40,36 +40,13 @@ int amd_sched_fence_slab_init(void)
 	return 0;
 }
 
-void amd_sched_fence_slab_fini(void)
+static void __exit drm_sched_fence_slab_fini(void)
 {
 	rcu_barrier();
 	kmem_cache_destroy(sched_fence_slab);
 }
 
-struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *entity,
-					       void *owner)
-{
-	struct amd_sched_fence *fence = NULL;
-	unsigned seq;
-
-	fence = kmem_cache_zalloc(sched_fence_slab, GFP_KERNEL);
-	if (fence == NULL)
-		return NULL;
-
-	fence->owner = owner;
-	fence->sched = entity->sched;
-	spin_lock_init(&fence->lock);
-
-	seq = atomic_inc_return(&entity->fence_seq);
-	dma_fence_init(&fence->scheduled, &amd_sched_fence_ops_scheduled,
-		       &fence->lock, entity->fence_context, seq);
-	dma_fence_init(&fence->finished, &amd_sched_fence_ops_finished,
-		       &fence->lock, entity->fence_context + 1, seq);
-
-	return fence;
-}
-
-void amd_sched_fence_scheduled(struct amd_sched_fence *fence)
+void drm_sched_fence_scheduled(struct drm_sched_fence *fence)
 {
 	int ret = dma_fence_signal(&fence->scheduled);
 
@@ -81,7 +58,7 @@ void amd_sched_fence_scheduled(struct amd_sched_fence *fence)
 				"was already signaled\n");
 }
 
-void amd_sched_fence_finished(struct amd_sched_fence *fence)
+void drm_sched_fence_finished(struct drm_sched_fence *fence)
 {
 	int ret = dma_fence_signal(&fence->finished);
 
@@ -93,18 +70,18 @@ void amd_sched_fence_finished(struct amd_sched_fence *fence)
 				"was already signaled\n");
 }
 
-static const char *amd_sched_fence_get_driver_name(struct dma_fence *fence)
+static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence)
 {
-	return "amd_sched";
+	return "drm_sched";
 }
 
-static const char *amd_sched_fence_get_timeline_name(struct dma_fence *f)
+static const char *drm_sched_fence_get_timeline_name(struct dma_fence *f)
 {
-	struct amd_sched_fence *fence = to_amd_sched_fence(f);
+	struct drm_sched_fence *fence = to_drm_sched_fence(f);
 	return (const char *)fence->sched->name;
 }
 
-static bool amd_sched_fence_enable_signaling(struct dma_fence *f)
+static bool drm_sched_fence_enable_signaling(struct dma_fence *f)
 {
 	return true;
 }
@@ -116,10 +93,10 @@ static bool amd_sched_fence_enable_signaling(struct dma_fence *f)
  *
  * Free up the fence memory after the RCU grace period.
  */
-static void amd_sched_fence_free(struct rcu_head *rcu)
+static void drm_sched_fence_free(struct rcu_head *rcu)
 {
 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
-	struct amd_sched_fence *fence = to_amd_sched_fence(f);
+	struct drm_sched_fence *fence = to_drm_sched_fence(f);
 
 	dma_fence_put(fence->parent);
 	kmem_cache_free(sched_fence_slab, fence);
@@ -133,11 +110,11 @@ static void amd_sched_fence_free(struct rcu_head *rcu)
  * This function is called when the reference count becomes zero.
  * It just RCU schedules freeing up the fence.
  */
-static void amd_sched_fence_release_scheduled(struct dma_fence *f)
+static void drm_sched_fence_release_scheduled(struct dma_fence *f)
 {
-	struct amd_sched_fence *fence = to_amd_sched_fence(f);
+	struct drm_sched_fence *fence = to_drm_sched_fence(f);
 
-	call_rcu(&fence->finished.rcu, amd_sched_fence_free);
+	call_rcu(&fence->finished.rcu, drm_sched_fence_free);
 }
 
 /**
@@ -147,27 +124,68 @@ static void amd_sched_fence_release_scheduled(struct dma_fence *f)
  *
  * Drop the extra reference from the scheduled fence to the base fence.
  */
-static void amd_sched_fence_release_finished(struct dma_fence *f)
+static void drm_sched_fence_release_finished(struct dma_fence *f)
 {
-	struct amd_sched_fence *fence = to_amd_sched_fence(f);
+	struct drm_sched_fence *fence = to_drm_sched_fence(f);
 
 	dma_fence_put(&fence->scheduled);
 }
 
-const struct dma_fence_ops amd_sched_fence_ops_scheduled = {
-	.get_driver_name = amd_sched_fence_get_driver_name,
-	.get_timeline_name = amd_sched_fence_get_timeline_name,
-	.enable_signaling = amd_sched_fence_enable_signaling,
+const struct dma_fence_ops drm_sched_fence_ops_scheduled = {
+	.get_driver_name = drm_sched_fence_get_driver_name,
+	.get_timeline_name = drm_sched_fence_get_timeline_name,
+	.enable_signaling = drm_sched_fence_enable_signaling,
 	.signaled = NULL,
 	.wait = dma_fence_default_wait,
-	.release = amd_sched_fence_release_scheduled,
+	.release = drm_sched_fence_release_scheduled,
 };
 
-const struct dma_fence_ops amd_sched_fence_ops_finished = {
-	.get_driver_name = amd_sched_fence_get_driver_name,
-	.get_timeline_name = amd_sched_fence_get_timeline_name,
-	.enable_signaling = amd_sched_fence_enable_signaling,
+const struct dma_fence_ops drm_sched_fence_ops_finished = {
+	.get_driver_name = drm_sched_fence_get_driver_name,
+	.get_timeline_name = drm_sched_fence_get_timeline_name,
+	.enable_signaling = drm_sched_fence_enable_signaling,
 	.signaled = NULL,
 	.wait = dma_fence_default_wait,
-	.release = amd_sched_fence_release_finished,
+	.release = drm_sched_fence_release_finished,
 };
+
+struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f)
+{
+	if (f->ops == &drm_sched_fence_ops_scheduled)
+		return container_of(f, struct drm_sched_fence, scheduled);
+
+	if (f->ops == &drm_sched_fence_ops_finished)
+		return container_of(f, struct drm_sched_fence, finished);
+
+	return NULL;
+}
+EXPORT_SYMBOL(to_drm_sched_fence);
+
+struct drm_sched_fence *drm_sched_fence_create(struct drm_sched_entity *entity,
+					       void *owner)
+{
+	struct drm_sched_fence *fence = NULL;
+	unsigned seq;
+
+	fence = kmem_cache_zalloc(sched_fence_slab, GFP_KERNEL);
+	if (fence == NULL)
+		return NULL;
+
+	fence->owner = owner;
+	fence->sched = entity->sched;
+	spin_lock_init(&fence->lock);
+
+	seq = atomic_inc_return(&entity->fence_seq);
+	dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled,
+		       &fence->lock, entity->fence_context, seq);
+	dma_fence_init(&fence->finished, &drm_sched_fence_ops_finished,
+		       &fence->lock, entity->fence_context + 1, seq);
+
+	return fence;
+}
+
+module_init(drm_sched_fence_slab_init);
+module_exit(drm_sched_fence_slab_fini);
+
+MODULE_DESCRIPTION("DRM GPU scheduler");
+MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/selftests/test-drm_mm.c b/drivers/gpu/drm/selftests/test-drm_mm.c
index 86eb4c185a28..7cc935d7b7aa 100644
--- a/drivers/gpu/drm/selftests/test-drm_mm.c
+++ b/drivers/gpu/drm/selftests/test-drm_mm.c
@@ -682,6 +682,8 @@ static int __igt_insert(unsigned int count, u64 size, bool replace)
 		drm_mm_for_each_node_safe(node, next, &mm)
 			drm_mm_remove_node(node);
 		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
+
+		cond_resched();
 	}
 
 	ret = 0;
@@ -944,6 +946,8 @@ static int __igt_insert_range(unsigned int count, u64 size, u64 start, u64 end)
 		drm_mm_for_each_node_safe(node, next, &mm)
 			drm_mm_remove_node(node);
 		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
+
+		cond_resched();
 	}
 
 	ret = 0;
@@ -1068,6 +1072,7 @@ static int igt_align(void *ignored)
 		drm_mm_for_each_node_safe(node, next, &mm)
 			drm_mm_remove_node(node);
 		DRM_MM_BUG_ON(!drm_mm_clean(&mm));
+
 		cond_resched();
 	}
 
diff --git a/drivers/gpu/drm/sti/Kconfig b/drivers/gpu/drm/sti/Kconfig
index acd72865feac..cca4b3c9aeb5 100644
--- a/drivers/gpu/drm/sti/Kconfig
+++ b/drivers/gpu/drm/sti/Kconfig
@@ -8,5 +8,6 @@ config DRM_STI
 	select DRM_PANEL
 	select FW_LOADER
 	select SND_SOC_HDMI_CODEC if SND_SOC
+	select OF
 	help
 	  Choose this option to enable DRM on STM stiH4xx chipset
diff --git a/drivers/gpu/drm/sti/sti_awg_utils.c b/drivers/gpu/drm/sti/sti_awg_utils.c
index 2da7d6866d5d..7c5a7830b6e8 100644
--- a/drivers/gpu/drm/sti/sti_awg_utils.c
+++ b/drivers/gpu/drm/sti/sti_awg_utils.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include "sti_awg_utils.h"
diff --git a/drivers/gpu/drm/sti/sti_awg_utils.h b/drivers/gpu/drm/sti/sti_awg_utils.h
index 45d599bd570a..258a568f050b 100644
--- a/drivers/gpu/drm/sti/sti_awg_utils.h
+++ b/drivers/gpu/drm/sti/sti_awg_utils.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_AWG_UTILS_H_
diff --git a/drivers/gpu/drm/sti/sti_compositor.c b/drivers/gpu/drm/sti/sti_compositor.c
index 6e4bf68262db..021b8fcaa0b9 100644
--- a/drivers/gpu/drm/sti/sti_compositor.c
+++ b/drivers/gpu/drm/sti/sti_compositor.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/component.h>
diff --git a/drivers/gpu/drm/sti/sti_compositor.h b/drivers/gpu/drm/sti/sti_compositor.h
index 2952a2d25a52..ac4bb3834810 100644
--- a/drivers/gpu/drm/sti/sti_compositor.h
+++ b/drivers/gpu/drm/sti/sti_compositor.h
@@ -1,9 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_COMPOSITOR_H_
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index e8a4d48e985a..21e50d7b1f86 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
diff --git a/drivers/gpu/drm/sti/sti_crtc.h b/drivers/gpu/drm/sti/sti_crtc.h
index 3f2d89a3634d..d87c488212d6 100644
--- a/drivers/gpu/drm/sti/sti_crtc.h
+++ b/drivers/gpu/drm/sti/sti_crtc.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_CRTC_H_
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index b709ebbec095..df0a282b9615 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Vincent Abriou <vincent.abriou@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/seq_file.h>
diff --git a/drivers/gpu/drm/sti/sti_cursor.h b/drivers/gpu/drm/sti/sti_cursor.h
index 2ee5c10e8b33..067feda5226c 100644
--- a/drivers/gpu/drm/sti/sti_cursor.h
+++ b/drivers/gpu/drm/sti/sti_cursor.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2013
  * Authors: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_CURSOR_H_
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index 9e9343101738..55b6967d27e1 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <drm/drmP.h>
@@ -17,6 +17,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_of.h>
 
@@ -138,16 +139,9 @@ static int sti_atomic_check(struct drm_device *dev,
 	return ret;
 }
 
-static void sti_output_poll_changed(struct drm_device *ddev)
-{
-	struct sti_private *private = ddev->dev_private;
-
-	drm_fbdev_cma_hotplug_event(private->fbdev);
-}
-
 static const struct drm_mode_config_funcs sti_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
-	.output_poll_changed = sti_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = sti_atomic_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
@@ -230,11 +224,7 @@ static void sti_cleanup(struct drm_device *ddev)
 {
 	struct sti_private *private = ddev->dev_private;
 
-	if (private->fbdev) {
-		drm_fbdev_cma_fini(private->fbdev);
-		private->fbdev = NULL;
-	}
-
+	drm_fb_cma_fbdev_fini(ddev);
 	drm_kms_helper_poll_fini(ddev);
 	component_unbind_all(ddev->dev, ddev);
 	kfree(private);
@@ -244,8 +234,6 @@ static void sti_cleanup(struct drm_device *ddev)
 static int sti_bind(struct device *dev)
 {
 	struct drm_device *ddev;
-	struct sti_private *private;
-	struct drm_fbdev_cma *fbdev;
 	int ret;
 
 	ddev = drm_dev_alloc(&sti_driver, dev);
@@ -266,15 +254,10 @@ static int sti_bind(struct device *dev)
 
 	drm_mode_config_reset(ddev);
 
-	private = ddev->dev_private;
 	if (ddev->mode_config.num_connector) {
-		fbdev = drm_fbdev_cma_init(ddev, 32,
-					   ddev->mode_config.num_connector);
-		if (IS_ERR(fbdev)) {
+		ret = drm_fb_cma_fbdev_init(ddev, 32, 0);
+		if (ret)
 			DRM_DEBUG_DRIVER("Warning: fails to create fbdev\n");
-			fbdev = NULL;
-		}
-		private->fbdev = fbdev;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/sti/sti_drv.h b/drivers/gpu/drm/sti/sti_drv.h
index 6502ed2d3351..4b41142a22e4 100644
--- a/drivers/gpu/drm/sti/sti_drv.h
+++ b/drivers/gpu/drm/sti/sti_drv.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_DRV_H_
@@ -24,7 +24,6 @@ struct sti_private {
 	struct sti_compositor *compo;
 	struct drm_property *plane_zorder_property;
 	struct drm_device *drm_dev;
-	struct drm_fbdev_cma *fbdev;
 };
 
 extern struct platform_driver sti_tvout_driver;
diff --git a/drivers/gpu/drm/sti/sti_dvo.c b/drivers/gpu/drm/sti/sti_dvo.c
index 83314aee65cb..a5979cd25cc7 100644
--- a/drivers/gpu/drm/sti/sti_dvo.c
+++ b/drivers/gpu/drm/sti/sti_dvo.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index b65eea4f2c97..9b2c47051b51 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 #include <linux/seq_file.h>
 
diff --git a/drivers/gpu/drm/sti/sti_gdp.h b/drivers/gpu/drm/sti/sti_gdp.h
index 73947a4a8004..d3e8ebfe2e66 100644
--- a/drivers/gpu/drm/sti/sti_gdp.h
+++ b/drivers/gpu/drm/sti/sti_gdp.h
@@ -1,9 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_GDP_H_
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index cf65e32b5090..67bbdb49fffc 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
diff --git a/drivers/gpu/drm/sti/sti_hdmi.c b/drivers/gpu/drm/sti/sti_hdmi.c
index 30f02d2fdd03..58f431102512 100644
--- a/drivers/gpu/drm/sti/sti_hdmi.c
+++ b/drivers/gpu/drm/sti/sti_hdmi.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
@@ -515,7 +515,9 @@ static int hdmi_vendor_infoframe_config(struct sti_hdmi *hdmi)
 
 	DRM_DEBUG_DRIVER("\n");
 
-	ret = drm_hdmi_vendor_infoframe_from_display_mode(&infoframe, mode);
+	ret = drm_hdmi_vendor_infoframe_from_display_mode(&infoframe,
+							  hdmi->drm_connector,
+							  mode);
 	if (ret < 0) {
 		/*
 		 * Going into that statement does not means vendor infoframe
@@ -976,7 +978,6 @@ static int sti_hdmi_connector_get_modes(struct drm_connector *connector)
 
 	count = drm_add_edid_modes(connector, edid);
 	drm_mode_connector_update_edid_property(connector, edid);
-	drm_edid_to_eld(connector, edid);
 
 	kfree(edid);
 	return count;
@@ -1414,6 +1415,11 @@ static int sti_hdmi_probe(struct platform_device *pdev)
 	init_waitqueue_head(&hdmi->wait_event);
 
 	hdmi->irq = platform_get_irq_byname(pdev, "irq");
+	if (hdmi->irq < 0) {
+		DRM_ERROR("Cannot get HDMI irq\n");
+		ret = hdmi->irq;
+		goto release_adapter;
+	}
 
 	ret = devm_request_threaded_irq(dev, hdmi->irq, hdmi_irq,
 			hdmi_irq_thread, IRQF_ONESHOT, dev_name(dev), hdmi);
diff --git a/drivers/gpu/drm/sti/sti_hdmi.h b/drivers/gpu/drm/sti/sti_hdmi.h
index c6469b56ce7e..63a24941db3b 100644
--- a/drivers/gpu/drm/sti/sti_hdmi.h
+++ b/drivers/gpu/drm/sti/sti_hdmi.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_HDMI_H_
diff --git a/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.c b/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.c
index 8e0ceb0ced33..01699af6a768 100644
--- a/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.c
+++ b/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Vincent Abriou <vincent.abriou@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include "sti_hdmi_tx3g4c28phy.h"
diff --git a/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.h b/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.h
index f99a7ff281ef..d261947ef30a 100644
--- a/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.h
+++ b/drivers/gpu/drm/sti/sti_hdmi_tx3g4c28phy.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_HDMI_TX3G4C28PHY_H_
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index b19b3430b296..106be8c4e58b 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/component.h>
diff --git a/drivers/gpu/drm/sti/sti_hqvdp_lut.h b/drivers/gpu/drm/sti/sti_hqvdp_lut.h
index 619af7f4384e..57cccd9546a3 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp_lut.h
+++ b/drivers/gpu/drm/sti/sti_hqvdp_lut.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_HQVDP_LUT_H_
diff --git a/drivers/gpu/drm/sti/sti_mixer.c b/drivers/gpu/drm/sti/sti_mixer.c
index 2bd1d46fe1cd..a4f45c74d678 100644
--- a/drivers/gpu/drm/sti/sti_mixer.c
+++ b/drivers/gpu/drm/sti/sti_mixer.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 #include <linux/seq_file.h>
 
diff --git a/drivers/gpu/drm/sti/sti_mixer.h b/drivers/gpu/drm/sti/sti_mixer.h
index e64a00e61049..4cb3cfddc03a 100644
--- a/drivers/gpu/drm/sti/sti_mixer.h
+++ b/drivers/gpu/drm/sti/sti_mixer.h
@@ -1,9 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_MIXER_H_
diff --git a/drivers/gpu/drm/sti/sti_plane.c b/drivers/gpu/drm/sti/sti_plane.c
index 427d8f58c6b1..b074609c960a 100644
--- a/drivers/gpu/drm/sti/sti_plane.c
+++ b/drivers/gpu/drm/sti/sti_plane.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <drm/drmP.h>
diff --git a/drivers/gpu/drm/sti/sti_plane.h b/drivers/gpu/drm/sti/sti_plane.h
index c36c13faaa18..b8d7fae2a014 100644
--- a/drivers/gpu/drm/sti/sti_plane.h
+++ b/drivers/gpu/drm/sti/sti_plane.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_PLANE_H_
diff --git a/drivers/gpu/drm/sti/sti_tvout.c b/drivers/gpu/drm/sti/sti_tvout.c
index 8959fcc743a8..ea4a3b87fa55 100644
--- a/drivers/gpu/drm/sti/sti_tvout.c
+++ b/drivers/gpu/drm/sti/sti_tvout.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Vincent Abriou <vincent.abriou@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
diff --git a/drivers/gpu/drm/sti/sti_vid.c b/drivers/gpu/drm/sti/sti_vid.c
index 577a3341d3c1..2aac36c95835 100644
--- a/drivers/gpu/drm/sti/sti_vid.c
+++ b/drivers/gpu/drm/sti/sti_vid.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 #include <linux/seq_file.h>
 
diff --git a/drivers/gpu/drm/sti/sti_vid.h b/drivers/gpu/drm/sti/sti_vid.h
index fdc90f922a05..9dbd78461de1 100644
--- a/drivers/gpu/drm/sti/sti_vid.h
+++ b/drivers/gpu/drm/sti/sti_vid.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_VID_H_
diff --git a/drivers/gpu/drm/sti/sti_vtg.c b/drivers/gpu/drm/sti/sti_vtg.c
index 2dcba1d3a122..6c421644de18 100644
--- a/drivers/gpu/drm/sti/sti_vtg.c
+++ b/drivers/gpu/drm/sti/sti_vtg.c
@@ -1,14 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Benjamin Gaignard <benjamin.gaignard@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          Vincent Abriou <vincent.abriou@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/module.h>
 #include <linux/notifier.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
 
 #include <drm/drmP.h>
@@ -72,8 +73,6 @@
 #define AWG_DELAY_ED        (-8)
 #define AWG_DELAY_SD        (-7)
 
-static LIST_HEAD(vtg_lookup);
-
 /*
  * STI VTG register offset structure
  *
@@ -123,42 +122,31 @@ struct sti_vtg_sync_params {
 /**
  * STI VTG structure
  *
- * @dev: pointer to device driver
- * @np: device node
  * @regs: register mapping
  * @sync_params: synchronisation parameters used to generate timings
  * @irq: VTG irq
  * @irq_status: store the IRQ status value
  * @notifier_list: notifier callback
  * @crtc: the CRTC for vblank event
- * @link: List node to link the structure in lookup list
  */
 struct sti_vtg {
-	struct device *dev;
-	struct device_node *np;
 	void __iomem *regs;
 	struct sti_vtg_sync_params sync_params[VTG_MAX_SYNC_OUTPUT];
 	int irq;
 	u32 irq_status;
 	struct raw_notifier_head notifier_list;
 	struct drm_crtc *crtc;
-	struct list_head link;
 };
 
-static void vtg_register(struct sti_vtg *vtg)
-{
-	list_add_tail(&vtg->link, &vtg_lookup);
-}
-
 struct sti_vtg *of_vtg_find(struct device_node *np)
 {
-	struct sti_vtg *vtg;
+	struct platform_device *pdev;
 
-	list_for_each_entry(vtg, &vtg_lookup, link) {
-		if (vtg->np == np)
-			return vtg;
-	}
-	return NULL;
+	pdev = of_find_device_by_node(np);
+	if (!pdev)
+		return NULL;
+
+	return (struct sti_vtg *)platform_get_drvdata(pdev);
 }
 
 static void vtg_reset(struct sti_vtg *vtg)
@@ -397,9 +385,6 @@ static int vtg_probe(struct platform_device *pdev)
 	if (!vtg)
 		return -ENOMEM;
 
-	vtg->dev = dev;
-	vtg->np = pdev->dev.of_node;
-
 	/* Get Memory ressources */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
@@ -428,16 +413,10 @@ static int vtg_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	vtg_register(vtg);
 	platform_set_drvdata(pdev, vtg);
 
-	DRM_INFO("%s %s\n", __func__, dev_name(vtg->dev));
-
-	return 0;
-}
+	DRM_INFO("%s %s\n", __func__, dev_name(dev));
 
-static int vtg_remove(struct platform_device *pdev)
-{
 	return 0;
 }
 
@@ -454,7 +433,6 @@ struct platform_driver sti_vtg_driver = {
 		.of_match_table = vtg_of_match,
 	},
 	.probe	= vtg_probe,
-	.remove = vtg_remove,
 };
 
 MODULE_AUTHOR("Benjamin Gaignard <benjamin.gaignard@st.com>");
diff --git a/drivers/gpu/drm/sti/sti_vtg.h b/drivers/gpu/drm/sti/sti_vtg.h
index f1dcdf9c2342..d177129e5bcb 100644
--- a/drivers/gpu/drm/sti/sti_vtg.h
+++ b/drivers/gpu/drm/sti/sti_vtg.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _STI_VTG_H_
diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
index c857663eafc2..8fe954c27fba 100644
--- a/drivers/gpu/drm/stm/drv.c
+++ b/drivers/gpu/drm/stm/drv.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2017
  *
@@ -5,8 +6,6 @@
  *          Yannick Fertre <yannick.fertre@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          Mickael Reulier <mickael.reulier@st.com>
- *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/component.h>
@@ -15,6 +14,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -24,35 +24,19 @@
 #define STM_MAX_FB_WIDTH	2048
 #define STM_MAX_FB_HEIGHT	2048 /* same as width to handle orientation */
 
-static void drv_output_poll_changed(struct drm_device *ddev)
-{
-	struct ltdc_device *ldev = ddev->dev_private;
-
-	drm_fbdev_cma_hotplug_event(ldev->fbdev);
-}
-
 static const struct drm_mode_config_funcs drv_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
-	.output_poll_changed = drv_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-static void drv_lastclose(struct drm_device *ddev)
-{
-	struct ltdc_device *ldev = ddev->dev_private;
-
-	DRM_DEBUG("%s\n", __func__);
-
-	drm_fbdev_cma_restore_mode(ldev->fbdev);
-}
-
 DEFINE_DRM_GEM_CMA_FOPS(drv_driver_fops);
 
 static struct drm_driver drv_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
-	.lastclose = drv_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.name = "stm",
 	.desc = "STMicroelectronics SoC DRM",
 	.date = "20170330",
@@ -79,7 +63,6 @@ static struct drm_driver drv_driver = {
 static int drv_load(struct drm_device *ddev)
 {
 	struct platform_device *pdev = to_platform_device(ddev->dev);
-	struct drm_fbdev_cma *fbdev;
 	struct ltdc_device *ldev;
 	int ret;
 
@@ -112,14 +95,9 @@ static int drv_load(struct drm_device *ddev)
 	drm_kms_helper_poll_init(ddev);
 
 	if (ddev->mode_config.num_connector) {
-		ldev = ddev->dev_private;
-		fbdev = drm_fbdev_cma_init(ddev, 16,
-					   ddev->mode_config.num_connector);
-		if (IS_ERR(fbdev)) {
+		ret = drm_fb_cma_fbdev_init(ddev, 16, 0);
+		if (ret)
 			DRM_DEBUG("Warning: fails to create fbdev\n");
-			fbdev = NULL;
-		}
-		ldev->fbdev = fbdev;
 	}
 
 	platform_set_drvdata(pdev, ddev);
@@ -132,14 +110,9 @@ static int drv_load(struct drm_device *ddev)
 
 static void drv_unload(struct drm_device *ddev)
 {
-	struct ltdc_device *ldev = ddev->dev_private;
-
 	DRM_DEBUG("%s\n", __func__);
 
-	if (ldev->fbdev) {
-		drm_fbdev_cma_fini(ldev->fbdev);
-		ldev->fbdev = NULL;
-	}
+	drm_fb_cma_fbdev_fini(ddev);
 	drm_kms_helper_poll_fini(ddev);
 	ltdc_unload(ddev);
 	drm_mode_config_cleanup(ddev);
diff --git a/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c b/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
index e5b6310240fe..fd02506274da 100644
--- a/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
+++ b/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
@@ -1,10 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2017
  *
  * Authors: Philippe Cornu <philippe.cornu@st.com>
  *          Yannick Fertre <yannick.fertre@st.com>
- *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
@@ -129,7 +128,7 @@ static int dsi_pll_get_params(int clkin_khz, int clkout_khz,
 	int fvco_min, fvco_max, delta, best_delta; /* all in khz */
 
 	/* Early checks preventing division by 0 & odd results */
-	if ((clkin_khz <= 0) || (clkout_khz <= 0))
+	if (clkin_khz <= 0 || clkout_khz <= 0)
 		return -EINVAL;
 
 	fvco_min = LANE_MIN_KBPS * 2 * ODF_MAX;
@@ -155,7 +154,7 @@ static int dsi_pll_get_params(int clkin_khz, int clkout_khz,
 		for (o = ODF_MIN; o <= ODF_MAX; o *= 2) {
 			n = DIV_ROUND_CLOSEST(i * o * clkout_khz, clkin_khz);
 			/* Check ndiv according to vco range */
-			if ((n < n_min) || (n > n_max))
+			if (n < n_min || n > n_max)
 				continue;
 			/* Check if new delta is better & saves parameters */
 			delta = dsi_pll_get_clkout_khz(clkin_khz, i, n, o) -
@@ -291,11 +290,6 @@ static int dw_mipi_dsi_stm_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		DRM_ERROR("Unable to get resource\n");
-		return -ENODEV;
-	}
-
 	dsi->base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(dsi->base)) {
 		DRM_ERROR("Unable to get dsi registers\n");
@@ -342,7 +336,7 @@ static struct platform_driver dw_mipi_dsi_stm_driver = {
 	.remove		= dw_mipi_dsi_stm_remove,
 	.driver		= {
 		.of_match_table = dw_mipi_dsi_stm_dt_ids,
-		.name	= "dw_mipi_dsi-stm",
+		.name	= "stm32-display-dsi",
 	},
 };
 
diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c
index 735c9081202a..6dc5d4ec4e17 100644
--- a/drivers/gpu/drm/stm/ltdc.c
+++ b/drivers/gpu/drm/stm/ltdc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2017
  *
@@ -5,8 +6,6 @@
  *          Yannick Fertre <yannick.fertre@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          Mickael Reulier <mickael.reulier@st.com>
- *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
@@ -33,6 +32,8 @@
 
 #define MAX_IRQ 4
 
+#define MAX_ENDPOINTS 2
+
 #define HWVER_10200 0x010200
 #define HWVER_10300 0x010300
 #define HWVER_20101 0x020101
@@ -556,7 +557,7 @@ static int ltdc_plane_atomic_check(struct drm_plane *plane,
 	src_h = state->src_h >> 16;
 
 	/* Reject scaling */
-	if ((src_w != state->crtc_w) || (src_h != state->crtc_h)) {
+	if (src_w != state->crtc_w || src_h != state->crtc_h) {
 		DRM_ERROR("Scaling is not supported");
 		return -EINVAL;
 	}
@@ -856,18 +857,33 @@ int ltdc_load(struct drm_device *ddev)
 	struct ltdc_device *ldev = ddev->dev_private;
 	struct device *dev = ddev->dev;
 	struct device_node *np = dev->of_node;
-	struct drm_bridge *bridge;
-	struct drm_panel *panel;
+	struct drm_bridge *bridge[MAX_ENDPOINTS] = {NULL};
+	struct drm_panel *panel[MAX_ENDPOINTS] = {NULL};
 	struct drm_crtc *crtc;
 	struct reset_control *rstc;
 	struct resource *res;
-	int irq, ret, i;
+	int irq, ret, i, endpoint_not_ready = -ENODEV;
 
 	DRM_DEBUG_DRIVER("\n");
 
-	ret = drm_of_find_panel_or_bridge(np, 0, 0, &panel, &bridge);
-	if (ret)
-		return ret;
+	/* Get endpoints if any */
+	for (i = 0; i < MAX_ENDPOINTS; i++) {
+		ret = drm_of_find_panel_or_bridge(np, 0, i, &panel[i],
+						  &bridge[i]);
+
+		/*
+		 * If at least one endpoint is ready, continue probing,
+		 * else if at least one endpoint is -EPROBE_DEFER and
+		 * there is no previous ready endpoints, defer probing.
+		 */
+		if (!ret)
+			endpoint_not_ready = 0;
+		else if (ret == -EPROBE_DEFER && endpoint_not_ready)
+			endpoint_not_ready = -EPROBE_DEFER;
+	}
+
+	if (endpoint_not_ready)
+		return endpoint_not_ready;
 
 	rstc = devm_reset_control_get_exclusive(dev, NULL);
 
@@ -885,12 +901,6 @@ int ltdc_load(struct drm_device *ddev)
 	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		DRM_ERROR("Unable to get resource\n");
-		ret = -ENODEV;
-		goto err;
-	}
-
 	ldev->regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(ldev->regs)) {
 		DRM_ERROR("Unable to get ltdc registers\n");
@@ -928,19 +938,25 @@ int ltdc_load(struct drm_device *ddev)
 
 	DRM_INFO("ltdc hw version 0x%08x - ready\n", ldev->caps.hw_version);
 
-	if (panel) {
-		bridge = drm_panel_bridge_add(panel, DRM_MODE_CONNECTOR_DPI);
-		if (IS_ERR(bridge)) {
-			DRM_ERROR("Failed to create panel-bridge\n");
-			ret = PTR_ERR(bridge);
-			goto err;
+	/* Add endpoints panels or bridges if any */
+	for (i = 0; i < MAX_ENDPOINTS; i++) {
+		if (panel[i]) {
+			bridge[i] = drm_panel_bridge_add(panel[i],
+							DRM_MODE_CONNECTOR_DPI);
+			if (IS_ERR(bridge[i])) {
+				DRM_ERROR("panel-bridge endpoint %d\n", i);
+				ret = PTR_ERR(bridge[i]);
+				goto err;
+			}
 		}
-	}
 
-	ret = ltdc_encoder_init(ddev, bridge);
-	if (ret) {
-		DRM_ERROR("Failed to init encoder\n");
-		goto err;
+		if (bridge[i]) {
+			ret = ltdc_encoder_init(ddev, bridge[i]);
+			if (ret) {
+				DRM_ERROR("init encoder endpoint %d\n", i);
+				goto err;
+			}
+		}
 	}
 
 	crtc = devm_kzalloc(dev, sizeof(*crtc), GFP_KERNEL);
@@ -968,7 +984,8 @@ int ltdc_load(struct drm_device *ddev)
 	return 0;
 
 err:
-	drm_panel_bridge_remove(bridge);
+	for (i = 0; i < MAX_ENDPOINTS; i++)
+		drm_panel_bridge_remove(bridge[i]);
 
 	clk_disable_unprepare(ldev->pixel_clk);
 
@@ -978,10 +995,12 @@ int ltdc_load(struct drm_device *ddev)
 void ltdc_unload(struct drm_device *ddev)
 {
 	struct ltdc_device *ldev = ddev->dev_private;
+	int i;
 
 	DRM_DEBUG_DRIVER("\n");
 
-	drm_of_panel_bridge_remove(ddev->dev->of_node, 0, 0);
+	for (i = 0; i < MAX_ENDPOINTS; i++)
+		drm_of_panel_bridge_remove(ddev->dev->of_node, 0, i);
 
 	clk_disable_unprepare(ldev->pixel_clk);
 }
diff --git a/drivers/gpu/drm/stm/ltdc.h b/drivers/gpu/drm/stm/ltdc.h
index ae437557d715..edd1c0a446d1 100644
--- a/drivers/gpu/drm/stm/ltdc.h
+++ b/drivers/gpu/drm/stm/ltdc.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2017
  *
@@ -5,8 +6,6 @@
  *          Yannick Fertre <yannick.fertre@st.com>
  *          Fabien Dessenne <fabien.dessenne@st.com>
  *          Mickael Reulier <mickael.reulier@st.com>
- *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef _LTDC_H_
@@ -21,7 +20,6 @@ struct ltdc_caps {
 };
 
 struct ltdc_device {
-	struct drm_fbdev_cma *fbdev;
 	void __iomem *regs;
 	struct clk *pixel_clk;	/* lcd pixel clock */
 	struct mutex err_lock;	/* protecting error_status */
diff --git a/drivers/gpu/drm/sun4i/Makefile b/drivers/gpu/drm/sun4i/Makefile
index 0c2f8c7facae..2b37a6abbb1d 100644
--- a/drivers/gpu/drm/sun4i/Makefile
+++ b/drivers/gpu/drm/sun4i/Makefile
@@ -9,10 +9,13 @@ sun4i-drm-hdmi-y		+= sun4i_hdmi_enc.o
 sun4i-drm-hdmi-y		+= sun4i_hdmi_i2c.o
 sun4i-drm-hdmi-y		+= sun4i_hdmi_tmds_clk.o
 
-sun8i-mixer-y			+= sun8i_mixer.o sun8i_layer.o
+sun8i-mixer-y			+= sun8i_mixer.o sun8i_ui_layer.o \
+				   sun8i_vi_layer.o sun8i_ui_scaler.o \
+				   sun8i_vi_scaler.o sun8i_csc.o
 
 sun4i-tcon-y			+= sun4i_crtc.o
 sun4i-tcon-y			+= sun4i_dotclock.o
+sun4i-tcon-y			+= sun4i_lvds.o
 sun4i-tcon-y			+= sun4i_tcon.o
 sun4i-tcon-y			+= sun4i_rgb.o
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c
index 5decae0069d0..78cbc3145e44 100644
--- a/drivers/gpu/drm/sun4i/sun4i_crtc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c
@@ -93,6 +93,8 @@ static void sun4i_crtc_atomic_disable(struct drm_crtc *crtc,
 
 	DRM_DEBUG_DRIVER("Disabling the CRTC\n");
 
+	drm_crtc_vblank_off(crtc);
+
 	sun4i_tcon_set_status(scrtc->tcon, encoder, false);
 
 	if (crtc->state->event && !crtc->state->active) {
@@ -113,6 +115,8 @@ static void sun4i_crtc_atomic_enable(struct drm_crtc *crtc,
 	DRM_DEBUG_DRIVER("Enabling the CRTC\n");
 
 	sun4i_tcon_set_status(scrtc->tcon, encoder, true);
+
+	drm_crtc_vblank_on(crtc);
 }
 
 static void sun4i_crtc_mode_set_nofb(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/sun4i/sun4i_dotclock.c b/drivers/gpu/drm/sun4i/sun4i_dotclock.c
index d401156490f3..e36004fbe453 100644
--- a/drivers/gpu/drm/sun4i/sun4i_dotclock.c
+++ b/drivers/gpu/drm/sun4i/sun4i_dotclock.c
@@ -17,8 +17,9 @@
 #include "sun4i_dotclock.h"
 
 struct sun4i_dclk {
-	struct clk_hw	hw;
-	struct regmap	*regmap;
+	struct clk_hw		hw;
+	struct regmap		*regmap;
+	struct sun4i_tcon	*tcon;
 };
 
 static inline struct sun4i_dclk *hw_to_dclk(struct clk_hw *hw)
@@ -73,11 +74,13 @@ static unsigned long sun4i_dclk_recalc_rate(struct clk_hw *hw,
 static long sun4i_dclk_round_rate(struct clk_hw *hw, unsigned long rate,
 				  unsigned long *parent_rate)
 {
+	struct sun4i_dclk *dclk = hw_to_dclk(hw);
+	struct sun4i_tcon *tcon = dclk->tcon;
 	unsigned long best_parent = 0;
 	u8 best_div = 1;
 	int i;
 
-	for (i = 6; i <= 127; i++) {
+	for (i = tcon->dclk_min_div; i <= tcon->dclk_max_div; i++) {
 		unsigned long ideal = rate * i;
 		unsigned long rounded;
 
@@ -129,10 +132,13 @@ static int sun4i_dclk_get_phase(struct clk_hw *hw)
 static int sun4i_dclk_set_phase(struct clk_hw *hw, int degrees)
 {
 	struct sun4i_dclk *dclk = hw_to_dclk(hw);
+	u32 val = degrees / 120;
+
+	val <<= 28;
 
 	regmap_update_bits(dclk->regmap, SUN4I_TCON0_IO_POL_REG,
 			   GENMASK(29, 28),
-			   degrees / 120);
+			   val);
 
 	return 0;
 }
@@ -167,6 +173,7 @@ int sun4i_dclk_create(struct device *dev, struct sun4i_tcon *tcon)
 	dclk = devm_kzalloc(dev, sizeof(*dclk), GFP_KERNEL);
 	if (!dclk)
 		return -ENOMEM;
+	dclk->tcon = tcon;
 
 	init.name = clk_name;
 	init.ops = &sun4i_dclk_ops;
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
index 75c76cdd82bc..d9a71f361b14 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
@@ -26,20 +26,13 @@
 #include "sun4i_framebuffer.h"
 #include "sun4i_tcon.h"
 
-static void sun4i_drv_lastclose(struct drm_device *dev)
-{
-	struct sun4i_drv *drv = dev->dev_private;
-
-	drm_fbdev_cma_restore_mode(drv->fbdev);
-}
-
 DEFINE_DRM_GEM_CMA_FOPS(sun4i_drv_fops);
 
 static struct drm_driver sun4i_drv_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_ATOMIC,
 
 	/* Generic Operations */
-	.lastclose		= sun4i_drv_lastclose,
+	.lastclose		= drm_fb_helper_lastclose,
 	.fops			= &sun4i_drv_fops,
 	.name			= "sun4i-drm",
 	.desc			= "Allwinner sun4i Display Engine",
@@ -118,7 +111,7 @@ static int sun4i_drv_bind(struct device *dev)
 	/* drm_vblank_init calls kcalloc, which can fail */
 	ret = drm_vblank_init(drm, drm->mode_config.num_crtc);
 	if (ret)
-		goto free_mem_region;
+		goto cleanup_mode_config;
 
 	drm->irq_enabled = true;
 
@@ -126,10 +119,9 @@ static int sun4i_drv_bind(struct device *dev)
 	sun4i_remove_framebuffers();
 
 	/* Create our framebuffer */
-	drv->fbdev = sun4i_framebuffer_init(drm);
-	if (IS_ERR(drv->fbdev)) {
+	ret = sun4i_framebuffer_init(drm);
+	if (ret) {
 		dev_err(drm->dev, "Couldn't create our framebuffer\n");
-		ret = PTR_ERR(drv->fbdev);
 		goto cleanup_mode_config;
 	}
 
@@ -147,7 +139,6 @@ static int sun4i_drv_bind(struct device *dev)
 	sun4i_framebuffer_free(drm);
 cleanup_mode_config:
 	drm_mode_config_cleanup(drm);
-free_mem_region:
 	of_reserved_mem_device_release(dev);
 free_drm:
 	drm_dev_unref(drm);
@@ -187,13 +178,7 @@ static bool sun4i_drv_node_is_frontend(struct device_node *node)
 
 static bool sun4i_drv_node_is_tcon(struct device_node *node)
 {
-	return of_device_is_compatible(node, "allwinner,sun4i-a10-tcon") ||
-		of_device_is_compatible(node, "allwinner,sun5i-a13-tcon") ||
-		of_device_is_compatible(node, "allwinner,sun6i-a31-tcon") ||
-		of_device_is_compatible(node, "allwinner,sun6i-a31s-tcon") ||
-		of_device_is_compatible(node, "allwinner,sun7i-a20-tcon") ||
-		of_device_is_compatible(node, "allwinner,sun8i-a33-tcon") ||
-		of_device_is_compatible(node, "allwinner,sun8i-v3s-tcon");
+	return !!of_match_node(sun4i_tcon_of_table, node);
 }
 
 static int compare_of(struct device *dev, void *data)
@@ -353,6 +338,7 @@ static const struct of_device_id sun4i_drv_of_table[] = {
 	{ .compatible = "allwinner,sun6i-a31s-display-engine" },
 	{ .compatible = "allwinner,sun7i-a20-display-engine" },
 	{ .compatible = "allwinner,sun8i-a33-display-engine" },
+	{ .compatible = "allwinner,sun8i-a83t-display-engine" },
 	{ .compatible = "allwinner,sun8i-v3s-display-engine" },
 	{ }
 };
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.h b/drivers/gpu/drm/sun4i/sun4i_drv.h
index a960c89270cc..2825f140da54 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.h
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.h
@@ -20,8 +20,6 @@
 struct sun4i_drv {
 	struct list_head	engine_list;
 	struct list_head	tcon_list;
-
-	struct drm_fbdev_cma	*fbdev;
 };
 
 #endif /* _SUN4I_DRV_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
index 2992f0a6b349..38a36c0dfa2f 100644
--- a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
@@ -11,6 +11,7 @@
  */
 
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drmP.h>
@@ -18,21 +19,14 @@
 #include "sun4i_drv.h"
 #include "sun4i_framebuffer.h"
 
-static void sun4i_de_output_poll_changed(struct drm_device *drm)
-{
-	struct sun4i_drv *drv = drm->dev_private;
-
-	drm_fbdev_cma_hotplug_event(drv->fbdev);
-}
-
 static const struct drm_mode_config_funcs sun4i_de_mode_config_funcs = {
-	.output_poll_changed	= sun4i_de_output_poll_changed,
+	.output_poll_changed	= drm_fb_helper_output_poll_changed,
 	.atomic_check		= drm_atomic_helper_check,
 	.atomic_commit		= drm_atomic_helper_commit,
 	.fb_create		= drm_gem_fb_create,
 };
 
-struct drm_fbdev_cma *sun4i_framebuffer_init(struct drm_device *drm)
+int sun4i_framebuffer_init(struct drm_device *drm)
 {
 	drm_mode_config_reset(drm);
 
@@ -41,12 +35,10 @@ struct drm_fbdev_cma *sun4i_framebuffer_init(struct drm_device *drm)
 
 	drm->mode_config.funcs = &sun4i_de_mode_config_funcs;
 
-	return drm_fbdev_cma_init(drm, 32, drm->mode_config.num_connector);
+	return drm_fb_cma_fbdev_init(drm, 32, 0);
 }
 
 void sun4i_framebuffer_free(struct drm_device *drm)
 {
-	struct sun4i_drv *drv = drm->dev_private;
-
-	drm_fbdev_cma_fini(drv->fbdev);
+	drm_fb_cma_fbdev_fini(drm);
 }
diff --git a/drivers/gpu/drm/sun4i/sun4i_framebuffer.h b/drivers/gpu/drm/sun4i/sun4i_framebuffer.h
index 3afd65252ee0..7ef0aed8384c 100644
--- a/drivers/gpu/drm/sun4i/sun4i_framebuffer.h
+++ b/drivers/gpu/drm/sun4i/sun4i_framebuffer.h
@@ -13,7 +13,7 @@
 #ifndef _SUN4I_FRAMEBUFFER_H_
 #define _SUN4I_FRAMEBUFFER_H_
 
-struct drm_fbdev_cma *sun4i_framebuffer_init(struct drm_device *drm);
+int sun4i_framebuffer_init(struct drm_device *drm);
 void sun4i_framebuffer_free(struct drm_device *drm);
 
 #endif /* _SUN4I_FRAMEBUFFER_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
index 500b6fb3e028..fa4bcd092eaf 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
@@ -538,7 +538,8 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 					     &sun4i_hdmi_regmap_config);
 	if (IS_ERR(hdmi->regmap)) {
 		dev_err(dev, "Couldn't create HDMI encoder regmap\n");
-		return PTR_ERR(hdmi->regmap);
+		ret = PTR_ERR(hdmi->regmap);
+		goto err_disable_mod_clk;
 	}
 
 	ret = sun4i_tmds_create(hdmi);
@@ -551,7 +552,8 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 		hdmi->ddc_parent_clk = devm_clk_get(dev, "ddc");
 		if (IS_ERR(hdmi->ddc_parent_clk)) {
 			dev_err(dev, "Couldn't get the HDMI DDC clock\n");
-			return PTR_ERR(hdmi->ddc_parent_clk);
+			ret = PTR_ERR(hdmi->ddc_parent_clk);
+			goto err_disable_mod_clk;
 		}
 	} else {
 		hdmi->ddc_parent_clk = hdmi->tmds_clk;
diff --git a/drivers/gpu/drm/sun4i/sun4i_lvds.c b/drivers/gpu/drm/sun4i/sun4i_lvds.c
new file mode 100644
index 000000000000..be3f14d7746d
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun4i_lvds.c
@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2017 Free Electrons
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ */
+
+#include <linux/clk.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+
+#include "sun4i_crtc.h"
+#include "sun4i_tcon.h"
+#include "sun4i_lvds.h"
+
+struct sun4i_lvds {
+	struct drm_connector	connector;
+	struct drm_encoder	encoder;
+
+	struct sun4i_tcon	*tcon;
+};
+
+static inline struct sun4i_lvds *
+drm_connector_to_sun4i_lvds(struct drm_connector *connector)
+{
+	return container_of(connector, struct sun4i_lvds,
+			    connector);
+}
+
+static inline struct sun4i_lvds *
+drm_encoder_to_sun4i_lvds(struct drm_encoder *encoder)
+{
+	return container_of(encoder, struct sun4i_lvds,
+			    encoder);
+}
+
+static int sun4i_lvds_get_modes(struct drm_connector *connector)
+{
+	struct sun4i_lvds *lvds =
+		drm_connector_to_sun4i_lvds(connector);
+	struct sun4i_tcon *tcon = lvds->tcon;
+
+	return drm_panel_get_modes(tcon->panel);
+}
+
+static struct drm_connector_helper_funcs sun4i_lvds_con_helper_funcs = {
+	.get_modes	= sun4i_lvds_get_modes,
+};
+
+static void
+sun4i_lvds_connector_destroy(struct drm_connector *connector)
+{
+	struct sun4i_lvds *lvds = drm_connector_to_sun4i_lvds(connector);
+	struct sun4i_tcon *tcon = lvds->tcon;
+
+	drm_panel_detach(tcon->panel);
+	drm_connector_cleanup(connector);
+}
+
+static const struct drm_connector_funcs sun4i_lvds_con_funcs = {
+	.fill_modes		= drm_helper_probe_single_connector_modes,
+	.destroy		= sun4i_lvds_connector_destroy,
+	.reset			= drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+};
+
+static void sun4i_lvds_encoder_enable(struct drm_encoder *encoder)
+{
+	struct sun4i_lvds *lvds = drm_encoder_to_sun4i_lvds(encoder);
+	struct sun4i_tcon *tcon = lvds->tcon;
+
+	DRM_DEBUG_DRIVER("Enabling LVDS output\n");
+
+	if (!IS_ERR(tcon->panel)) {
+		drm_panel_prepare(tcon->panel);
+		drm_panel_enable(tcon->panel);
+	}
+}
+
+static void sun4i_lvds_encoder_disable(struct drm_encoder *encoder)
+{
+	struct sun4i_lvds *lvds = drm_encoder_to_sun4i_lvds(encoder);
+	struct sun4i_tcon *tcon = lvds->tcon;
+
+	DRM_DEBUG_DRIVER("Disabling LVDS output\n");
+
+	if (!IS_ERR(tcon->panel)) {
+		drm_panel_disable(tcon->panel);
+		drm_panel_unprepare(tcon->panel);
+	}
+}
+
+static const struct drm_encoder_helper_funcs sun4i_lvds_enc_helper_funcs = {
+	.disable	= sun4i_lvds_encoder_disable,
+	.enable		= sun4i_lvds_encoder_enable,
+};
+
+static const struct drm_encoder_funcs sun4i_lvds_enc_funcs = {
+	.destroy	= drm_encoder_cleanup,
+};
+
+int sun4i_lvds_init(struct drm_device *drm, struct sun4i_tcon *tcon)
+{
+	struct drm_encoder *encoder;
+	struct drm_bridge *bridge;
+	struct sun4i_lvds *lvds;
+	int ret;
+
+	lvds = devm_kzalloc(drm->dev, sizeof(*lvds), GFP_KERNEL);
+	if (!lvds)
+		return -ENOMEM;
+	lvds->tcon = tcon;
+	encoder = &lvds->encoder;
+
+	ret = drm_of_find_panel_or_bridge(tcon->dev->of_node, 1, 0,
+					  &tcon->panel, &bridge);
+	if (ret) {
+		dev_info(drm->dev, "No panel or bridge found... LVDS output disabled\n");
+		return 0;
+	}
+
+	drm_encoder_helper_add(&lvds->encoder,
+			       &sun4i_lvds_enc_helper_funcs);
+	ret = drm_encoder_init(drm,
+			       &lvds->encoder,
+			       &sun4i_lvds_enc_funcs,
+			       DRM_MODE_ENCODER_LVDS,
+			       NULL);
+	if (ret) {
+		dev_err(drm->dev, "Couldn't initialise the lvds encoder\n");
+		goto err_out;
+	}
+
+	/* The LVDS encoder can only work with the TCON channel 0 */
+	lvds->encoder.possible_crtcs = BIT(drm_crtc_index(&tcon->crtc->crtc));
+
+	if (tcon->panel) {
+		drm_connector_helper_add(&lvds->connector,
+					 &sun4i_lvds_con_helper_funcs);
+		ret = drm_connector_init(drm, &lvds->connector,
+					 &sun4i_lvds_con_funcs,
+					 DRM_MODE_CONNECTOR_LVDS);
+		if (ret) {
+			dev_err(drm->dev, "Couldn't initialise the lvds connector\n");
+			goto err_cleanup_connector;
+		}
+
+		drm_mode_connector_attach_encoder(&lvds->connector,
+						  &lvds->encoder);
+
+		ret = drm_panel_attach(tcon->panel, &lvds->connector);
+		if (ret) {
+			dev_err(drm->dev, "Couldn't attach our panel\n");
+			goto err_cleanup_connector;
+		}
+	}
+
+	if (bridge) {
+		ret = drm_bridge_attach(encoder, bridge, NULL);
+		if (ret) {
+			dev_err(drm->dev, "Couldn't attach our bridge\n");
+			goto err_cleanup_connector;
+		}
+	}
+
+	return 0;
+
+err_cleanup_connector:
+	drm_encoder_cleanup(&lvds->encoder);
+err_out:
+	return ret;
+}
+EXPORT_SYMBOL(sun4i_lvds_init);
diff --git a/drivers/gpu/drm/sun4i/sun4i_lvds.h b/drivers/gpu/drm/sun4i/sun4i_lvds.h
new file mode 100644
index 000000000000..f3e90faa3082
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun4i_lvds.h
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2017 Free Electrons
+ * Maxime Ripard <maxime.ripard@free-electrons.com>
+ */
+
+#ifndef _SUN4I_LVDS_H_
+#define _SUN4I_LVDS_H_
+
+int sun4i_lvds_init(struct drm_device *drm, struct sun4i_tcon *tcon);
+
+#endif /* _SUN4I_LVDS_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index 832f8f9bc47f..b8da5a50a61d 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -92,6 +92,8 @@ static int sun4i_rgb_mode_valid(struct drm_connector *connector,
 
 	DRM_DEBUG_DRIVER("Vertical parameters OK\n");
 
+	tcon->dclk_min_div = 6;
+	tcon->dclk_max_div = 127;
 	rounded_rate = clk_round_rate(tcon->dclk, rate);
 	if (rounded_rate < rate)
 		return MODE_CLOCK_LOW;
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c
index f4284b51bdca..a818ca491605 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c
@@ -31,10 +31,52 @@
 #include "sun4i_crtc.h"
 #include "sun4i_dotclock.h"
 #include "sun4i_drv.h"
+#include "sun4i_lvds.h"
 #include "sun4i_rgb.h"
 #include "sun4i_tcon.h"
 #include "sunxi_engine.h"
 
+static struct drm_connector *sun4i_tcon_get_connector(const struct drm_encoder *encoder)
+{
+	struct drm_connector *connector;
+	struct drm_connector_list_iter iter;
+
+	drm_connector_list_iter_begin(encoder->dev, &iter);
+	drm_for_each_connector_iter(connector, &iter)
+		if (connector->encoder == encoder) {
+			drm_connector_list_iter_end(&iter);
+			return connector;
+		}
+	drm_connector_list_iter_end(&iter);
+
+	return NULL;
+}
+
+static int sun4i_tcon_get_pixel_depth(const struct drm_encoder *encoder)
+{
+	struct drm_connector *connector;
+	struct drm_display_info *info;
+
+	connector = sun4i_tcon_get_connector(encoder);
+	if (!connector)
+		return -EINVAL;
+
+	info = &connector->display_info;
+	if (info->num_bus_formats != 1)
+		return -EINVAL;
+
+	switch (info->bus_formats[0]) {
+	case MEDIA_BUS_FMT_RGB666_1X7X3_SPWG:
+		return 18;
+
+	case MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA:
+	case MEDIA_BUS_FMT_RGB888_1X7X4_SPWG:
+		return 24;
+	}
+
+	return -EINVAL;
+}
+
 static void sun4i_tcon_channel_set_status(struct sun4i_tcon *tcon, int channel,
 					  bool enabled)
 {
@@ -59,19 +101,72 @@ static void sun4i_tcon_channel_set_status(struct sun4i_tcon *tcon, int channel,
 		return;
 	}
 
-	if (enabled)
+	if (enabled) {
 		clk_prepare_enable(clk);
-	else
+		clk_rate_exclusive_get(clk);
+	} else {
+		clk_rate_exclusive_put(clk);
 		clk_disable_unprepare(clk);
+	}
+}
+
+static void sun4i_tcon_lvds_set_status(struct sun4i_tcon *tcon,
+				       const struct drm_encoder *encoder,
+				       bool enabled)
+{
+	if (enabled) {
+		u8 val;
+
+		regmap_update_bits(tcon->regs, SUN4I_TCON0_LVDS_IF_REG,
+				   SUN4I_TCON0_LVDS_IF_EN,
+				   SUN4I_TCON0_LVDS_IF_EN);
+
+		/*
+		 * As their name suggest, these values only apply to the A31
+		 * and later SoCs. We'll have to rework this when merging
+		 * support for the older SoCs.
+		 */
+		regmap_write(tcon->regs, SUN4I_TCON0_LVDS_ANA0_REG,
+			     SUN6I_TCON0_LVDS_ANA0_C(2) |
+			     SUN6I_TCON0_LVDS_ANA0_V(3) |
+			     SUN6I_TCON0_LVDS_ANA0_PD(2) |
+			     SUN6I_TCON0_LVDS_ANA0_EN_LDO);
+		udelay(2);
+
+		regmap_update_bits(tcon->regs, SUN4I_TCON0_LVDS_ANA0_REG,
+				   SUN6I_TCON0_LVDS_ANA0_EN_MB,
+				   SUN6I_TCON0_LVDS_ANA0_EN_MB);
+		udelay(2);
+
+		regmap_update_bits(tcon->regs, SUN4I_TCON0_LVDS_ANA0_REG,
+				   SUN6I_TCON0_LVDS_ANA0_EN_DRVC,
+				   SUN6I_TCON0_LVDS_ANA0_EN_DRVC);
+
+		if (sun4i_tcon_get_pixel_depth(encoder) == 18)
+			val = 7;
+		else
+			val = 0xf;
+
+		regmap_write_bits(tcon->regs, SUN4I_TCON0_LVDS_ANA0_REG,
+				  SUN6I_TCON0_LVDS_ANA0_EN_DRVD(0xf),
+				  SUN6I_TCON0_LVDS_ANA0_EN_DRVD(val));
+	} else {
+		regmap_update_bits(tcon->regs, SUN4I_TCON0_LVDS_IF_REG,
+				   SUN4I_TCON0_LVDS_IF_EN, 0);
+	}
 }
 
 void sun4i_tcon_set_status(struct sun4i_tcon *tcon,
 			   const struct drm_encoder *encoder,
 			   bool enabled)
 {
+	bool is_lvds = false;
 	int channel;
 
 	switch (encoder->encoder_type) {
+	case DRM_MODE_ENCODER_LVDS:
+		is_lvds = true;
+		/* Fallthrough */
 	case DRM_MODE_ENCODER_NONE:
 		channel = 0;
 		break;
@@ -84,10 +179,16 @@ void sun4i_tcon_set_status(struct sun4i_tcon *tcon,
 		return;
 	}
 
+	if (is_lvds && !enabled)
+		sun4i_tcon_lvds_set_status(tcon, encoder, false);
+
 	regmap_update_bits(tcon->regs, SUN4I_TCON_GCTL_REG,
 			   SUN4I_TCON_GCTL_TCON_ENABLE,
 			   enabled ? SUN4I_TCON_GCTL_TCON_ENABLE : 0);
 
+	if (is_lvds && enabled)
+		sun4i_tcon_lvds_set_status(tcon, encoder, true);
+
 	sun4i_tcon_channel_set_status(tcon, channel, enabled);
 }
 
@@ -170,6 +271,78 @@ static void sun4i_tcon0_mode_set_common(struct sun4i_tcon *tcon,
 		     SUN4I_TCON0_BASIC0_Y(mode->crtc_vdisplay));
 }
 
+static void sun4i_tcon0_mode_set_lvds(struct sun4i_tcon *tcon,
+				      const struct drm_encoder *encoder,
+				      const struct drm_display_mode *mode)
+{
+	unsigned int bp;
+	u8 clk_delay;
+	u32 reg, val = 0;
+
+	tcon->dclk_min_div = 7;
+	tcon->dclk_max_div = 7;
+	sun4i_tcon0_mode_set_common(tcon, mode);
+
+	/* Adjust clock delay */
+	clk_delay = sun4i_tcon_get_clk_delay(mode, 0);
+	regmap_update_bits(tcon->regs, SUN4I_TCON0_CTL_REG,
+			   SUN4I_TCON0_CTL_CLK_DELAY_MASK,
+			   SUN4I_TCON0_CTL_CLK_DELAY(clk_delay));
+
+	/*
+	 * This is called a backporch in the register documentation,
+	 * but it really is the back porch + hsync
+	 */
+	bp = mode->crtc_htotal - mode->crtc_hsync_start;
+	DRM_DEBUG_DRIVER("Setting horizontal total %d, backporch %d\n",
+			 mode->crtc_htotal, bp);
+
+	/* Set horizontal display timings */
+	regmap_write(tcon->regs, SUN4I_TCON0_BASIC1_REG,
+		     SUN4I_TCON0_BASIC1_H_TOTAL(mode->htotal) |
+		     SUN4I_TCON0_BASIC1_H_BACKPORCH(bp));
+
+	/*
+	 * This is called a backporch in the register documentation,
+	 * but it really is the back porch + hsync
+	 */
+	bp = mode->crtc_vtotal - mode->crtc_vsync_start;
+	DRM_DEBUG_DRIVER("Setting vertical total %d, backporch %d\n",
+			 mode->crtc_vtotal, bp);
+
+	/* Set vertical display timings */
+	regmap_write(tcon->regs, SUN4I_TCON0_BASIC2_REG,
+		     SUN4I_TCON0_BASIC2_V_TOTAL(mode->crtc_vtotal * 2) |
+		     SUN4I_TCON0_BASIC2_V_BACKPORCH(bp));
+
+	reg = SUN4I_TCON0_LVDS_IF_CLK_SEL_TCON0 |
+		SUN4I_TCON0_LVDS_IF_DATA_POL_NORMAL |
+		SUN4I_TCON0_LVDS_IF_CLK_POL_NORMAL;
+	if (sun4i_tcon_get_pixel_depth(encoder) == 24)
+		reg |= SUN4I_TCON0_LVDS_IF_BITWIDTH_24BITS;
+	else
+		reg |= SUN4I_TCON0_LVDS_IF_BITWIDTH_18BITS;
+
+	regmap_write(tcon->regs, SUN4I_TCON0_LVDS_IF_REG, reg);
+
+	/* Setup the polarity of the various signals */
+	if (!(mode->flags & DRM_MODE_FLAG_PHSYNC))
+		val |= SUN4I_TCON0_IO_POL_HSYNC_POSITIVE;
+
+	if (!(mode->flags & DRM_MODE_FLAG_PVSYNC))
+		val |= SUN4I_TCON0_IO_POL_VSYNC_POSITIVE;
+
+	regmap_write(tcon->regs, SUN4I_TCON0_IO_POL_REG, val);
+
+	/* Map output pins to channel 0 */
+	regmap_update_bits(tcon->regs, SUN4I_TCON_GCTL_REG,
+			   SUN4I_TCON_GCTL_IOMAP_MASK,
+			   SUN4I_TCON_GCTL_IOMAP_TCON0);
+
+	/* Enable the output on the pins */
+	regmap_write(tcon->regs, SUN4I_TCON0_IO_TRI_REG, 0xe0000000);
+}
+
 static void sun4i_tcon0_mode_set_rgb(struct sun4i_tcon *tcon,
 				     const struct drm_display_mode *mode)
 {
@@ -177,6 +350,8 @@ static void sun4i_tcon0_mode_set_rgb(struct sun4i_tcon *tcon,
 	u8 clk_delay;
 	u32 val = 0;
 
+	tcon->dclk_min_div = 6;
+	tcon->dclk_max_div = 127;
 	sun4i_tcon0_mode_set_common(tcon, mode);
 
 	/* Adjust clock delay */
@@ -334,6 +509,9 @@ void sun4i_tcon_mode_set(struct sun4i_tcon *tcon,
 			 const struct drm_display_mode *mode)
 {
 	switch (encoder->encoder_type) {
+	case DRM_MODE_ENCODER_LVDS:
+		sun4i_tcon0_mode_set_lvds(tcon, encoder, mode);
+		break;
 	case DRM_MODE_ENCODER_NONE:
 		sun4i_tcon0_mode_set_rgb(tcon, mode);
 		sun4i_tcon_set_mux(tcon, 0, encoder);
@@ -665,7 +843,9 @@ static int sun4i_tcon_bind(struct device *dev, struct device *master,
 	struct drm_device *drm = data;
 	struct sun4i_drv *drv = drm->dev_private;
 	struct sunxi_engine *engine;
+	struct device_node *remote;
 	struct sun4i_tcon *tcon;
+	bool has_lvds_rst, has_lvds_alt, can_lvds;
 	int ret;
 
 	engine = sun4i_tcon_find_engine(drv, dev->of_node);
@@ -696,6 +876,58 @@ static int sun4i_tcon_bind(struct device *dev, struct device *master,
 		return ret;
 	}
 
+	if (tcon->quirks->supports_lvds) {
+		/*
+		 * This can only be made optional since we've had DT
+		 * nodes without the LVDS reset properties.
+		 *
+		 * If the property is missing, just disable LVDS, and
+		 * print a warning.
+		 */
+		tcon->lvds_rst = devm_reset_control_get_optional(dev, "lvds");
+		if (IS_ERR(tcon->lvds_rst)) {
+			dev_err(dev, "Couldn't get our reset line\n");
+			return PTR_ERR(tcon->lvds_rst);
+		} else if (tcon->lvds_rst) {
+			has_lvds_rst = true;
+			reset_control_reset(tcon->lvds_rst);
+		} else {
+			has_lvds_rst = false;
+		}
+
+		/*
+		 * This can only be made optional since we've had DT
+		 * nodes without the LVDS reset properties.
+		 *
+		 * If the property is missing, just disable LVDS, and
+		 * print a warning.
+		 */
+		if (tcon->quirks->has_lvds_alt) {
+			tcon->lvds_pll = devm_clk_get(dev, "lvds-alt");
+			if (IS_ERR(tcon->lvds_pll)) {
+				if (PTR_ERR(tcon->lvds_pll) == -ENOENT) {
+					has_lvds_alt = false;
+				} else {
+					dev_err(dev, "Couldn't get the LVDS PLL\n");
+					return PTR_ERR(tcon->lvds_pll);
+				}
+			} else {
+				has_lvds_alt = true;
+			}
+		}
+
+		if (!has_lvds_rst ||
+		    (tcon->quirks->has_lvds_alt && !has_lvds_alt)) {
+			dev_warn(dev, "Missing LVDS properties, Please upgrade your DT\n");
+			dev_warn(dev, "LVDS output disabled\n");
+			can_lvds = false;
+		} else {
+			can_lvds = true;
+		}
+	} else {
+		can_lvds = false;
+	}
+
 	ret = sun4i_tcon_init_clocks(dev, tcon);
 	if (ret) {
 		dev_err(dev, "Couldn't init our TCON clocks\n");
@@ -727,7 +959,21 @@ static int sun4i_tcon_bind(struct device *dev, struct device *master,
 		goto err_free_dotclock;
 	}
 
-	ret = sun4i_rgb_init(drm, tcon);
+	/*
+	 * If we have an LVDS panel connected to the TCON, we should
+	 * just probe the LVDS connector. Otherwise, just probe RGB as
+	 * we used to.
+	 */
+	remote = of_graph_get_remote_node(dev->of_node, 1, 0);
+	if (of_device_is_compatible(remote, "panel-lvds"))
+		if (can_lvds)
+			ret = sun4i_lvds_init(drm, tcon);
+		else
+			ret = -EINVAL;
+	else
+		ret = sun4i_rgb_init(drm, tcon);
+	of_node_put(remote);
+
 	if (ret < 0)
 		goto err_free_dotclock;
 
@@ -877,6 +1123,7 @@ static const struct sun4i_tcon_quirks sun5i_a13_quirks = {
 
 static const struct sun4i_tcon_quirks sun6i_a31_quirks = {
 	.has_channel_1		= true,
+	.has_lvds_alt		= true,
 	.needs_de_be_mux	= true,
 	.set_mux		= sun6i_tcon_set_mux,
 };
@@ -893,24 +1140,31 @@ static const struct sun4i_tcon_quirks sun7i_a20_quirks = {
 };
 
 static const struct sun4i_tcon_quirks sun8i_a33_quirks = {
-	/* nothing is supported */
+	.has_lvds_alt		= true,
+};
+
+static const struct sun4i_tcon_quirks sun8i_a83t_lcd_quirks = {
+	.supports_lvds		= true,
 };
 
 static const struct sun4i_tcon_quirks sun8i_v3s_quirks = {
 	/* nothing is supported */
 };
 
-static const struct of_device_id sun4i_tcon_of_table[] = {
+/* sun4i_drv uses this list to check if a device node is a TCON */
+const struct of_device_id sun4i_tcon_of_table[] = {
 	{ .compatible = "allwinner,sun4i-a10-tcon", .data = &sun4i_a10_quirks },
 	{ .compatible = "allwinner,sun5i-a13-tcon", .data = &sun5i_a13_quirks },
 	{ .compatible = "allwinner,sun6i-a31-tcon", .data = &sun6i_a31_quirks },
 	{ .compatible = "allwinner,sun6i-a31s-tcon", .data = &sun6i_a31s_quirks },
 	{ .compatible = "allwinner,sun7i-a20-tcon", .data = &sun7i_a20_quirks },
 	{ .compatible = "allwinner,sun8i-a33-tcon", .data = &sun8i_a33_quirks },
+	{ .compatible = "allwinner,sun8i-a83t-tcon-lcd", .data = &sun8i_a83t_lcd_quirks },
 	{ .compatible = "allwinner,sun8i-v3s-tcon", .data = &sun8i_v3s_quirks },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun4i_tcon_of_table);
+EXPORT_SYMBOL(sun4i_tcon_of_table);
 
 static struct platform_driver sun4i_tcon_platform_driver = {
 	.probe		= sun4i_tcon_probe,
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.h b/drivers/gpu/drm/sun4i/sun4i_tcon.h
index f61bf6d83b4a..278700c7bf9f 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.h
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.h
@@ -70,7 +70,21 @@
 #define SUN4I_TCON0_TTL2_REG			0x78
 #define SUN4I_TCON0_TTL3_REG			0x7c
 #define SUN4I_TCON0_TTL4_REG			0x80
+
 #define SUN4I_TCON0_LVDS_IF_REG			0x84
+#define SUN4I_TCON0_LVDS_IF_EN				BIT(31)
+#define SUN4I_TCON0_LVDS_IF_BITWIDTH_MASK		BIT(26)
+#define SUN4I_TCON0_LVDS_IF_BITWIDTH_18BITS		(1 << 26)
+#define SUN4I_TCON0_LVDS_IF_BITWIDTH_24BITS		(0 << 26)
+#define SUN4I_TCON0_LVDS_IF_CLK_SEL_MASK		BIT(20)
+#define SUN4I_TCON0_LVDS_IF_CLK_SEL_TCON0		(1 << 20)
+#define SUN4I_TCON0_LVDS_IF_CLK_POL_MASK		BIT(4)
+#define SUN4I_TCON0_LVDS_IF_CLK_POL_NORMAL		(1 << 4)
+#define SUN4I_TCON0_LVDS_IF_CLK_POL_INV			(0 << 4)
+#define SUN4I_TCON0_LVDS_IF_DATA_POL_MASK		GENMASK(3, 0)
+#define SUN4I_TCON0_LVDS_IF_DATA_POL_NORMAL		(0xf)
+#define SUN4I_TCON0_LVDS_IF_DATA_POL_INV		(0)
+
 #define SUN4I_TCON0_IO_POL_REG			0x88
 #define SUN4I_TCON0_IO_POL_DCLK_PHASE(phase)		((phase & 3) << 28)
 #define SUN4I_TCON0_IO_POL_HSYNC_POSITIVE		BIT(25)
@@ -131,6 +145,16 @@
 #define SUN4I_TCON_CEU_RANGE_G_REG		0x144
 #define SUN4I_TCON_CEU_RANGE_B_REG		0x148
 #define SUN4I_TCON_MUX_CTRL_REG			0x200
+
+#define SUN4I_TCON0_LVDS_ANA0_REG		0x220
+#define SUN6I_TCON0_LVDS_ANA0_EN_MB			BIT(31)
+#define SUN6I_TCON0_LVDS_ANA0_EN_LDO			BIT(30)
+#define SUN6I_TCON0_LVDS_ANA0_EN_DRVC			BIT(24)
+#define SUN6I_TCON0_LVDS_ANA0_EN_DRVD(x)		(((x) & 0xf) << 20)
+#define SUN6I_TCON0_LVDS_ANA0_C(x)			(((x) & 3) << 17)
+#define SUN6I_TCON0_LVDS_ANA0_V(x)			(((x) & 3) << 8)
+#define SUN6I_TCON0_LVDS_ANA0_PD(x)			(((x) & 3) << 4)
+
 #define SUN4I_TCON1_FILL_CTL_REG		0x300
 #define SUN4I_TCON1_FILL_BEG0_REG		0x304
 #define SUN4I_TCON1_FILL_END0_REG		0x308
@@ -149,7 +173,9 @@ struct sun4i_tcon;
 
 struct sun4i_tcon_quirks {
 	bool	has_channel_1;	/* a33 does not have channel 1 */
+	bool	has_lvds_alt;	/* Does the LVDS clock have a parent other than the TCON clock? */
 	bool	needs_de_be_mux; /* sun6i needs mux to select backend */
+	bool	supports_lvds;   /* Does the TCON support an LVDS output? */
 
 	/* callback to handle tcon muxing options */
 	int	(*set_mux)(struct sun4i_tcon *, const struct drm_encoder *);
@@ -167,11 +193,17 @@ struct sun4i_tcon {
 	struct clk			*sclk0;
 	struct clk			*sclk1;
 
+	/* Possible mux for the LVDS clock */
+	struct clk			*lvds_pll;
+
 	/* Pixel clock */
 	struct clk			*dclk;
+	u8				dclk_max_div;
+	u8				dclk_min_div;
 
 	/* Reset control */
 	struct reset_control		*lcd_rst;
+	struct reset_control		*lvds_rst;
 
 	struct drm_panel		*panel;
 
@@ -197,4 +229,6 @@ void sun4i_tcon_mode_set(struct sun4i_tcon *tcon,
 void sun4i_tcon_set_status(struct sun4i_tcon *crtc,
 			   const struct drm_encoder *encoder, bool enable);
 
+extern const struct of_device_id sun4i_tcon_of_table[];
+
 #endif /* __SUN4I_TCON_H__ */
diff --git a/drivers/gpu/drm/sun4i/sun8i_csc.c b/drivers/gpu/drm/sun4i/sun8i_csc.c
new file mode 100644
index 000000000000..b14925b40ccf
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_csc.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+
+#include "sun8i_csc.h"
+#include "sun8i_mixer.h"
+
+static const u32 ccsc_base[2][2] = {
+	{CCSC00_OFFSET, CCSC01_OFFSET},
+	{CCSC10_OFFSET, CCSC11_OFFSET},
+};
+
+/*
+ * Factors are in two's complement format, 10 bits for fractinal part.
+ * First tree values in each line are multiplication factor and last
+ * value is constant, which is added at the end.
+ */
+static const u32 yuv2rgb[] = {
+	0x000004A8, 0x00000000, 0x00000662, 0xFFFC845A,
+	0x000004A8, 0xFFFFFE6F, 0xFFFFFCBF, 0x00021DF4,
+	0x000004A8, 0x00000813, 0x00000000, 0xFFFBAC4A,
+};
+
+static const u32 yvu2rgb[] = {
+	0x000004A8, 0x00000662, 0x00000000, 0xFFFC845A,
+	0x000004A8, 0xFFFFFCBF, 0xFFFFFE6F, 0x00021DF4,
+	0x000004A8, 0x00000000, 0x00000813, 0xFFFBAC4A,
+};
+
+static void sun8i_csc_set_coefficients(struct regmap *map, u32 base,
+				       enum sun8i_csc_mode mode)
+{
+	const u32 *table;
+	int i, data;
+
+	switch (mode) {
+	case SUN8I_CSC_MODE_YUV2RGB:
+		table = yuv2rgb;
+		break;
+	case SUN8I_CSC_MODE_YVU2RGB:
+		table = yvu2rgb;
+		break;
+	default:
+		DRM_WARN("Wrong CSC mode specified.\n");
+		return;
+	}
+
+	for (i = 0; i < 12; i++) {
+		data = table[i];
+		/* For some reason, 0x200 must be added to constant parts */
+		if (((i + 1) & 3) == 0)
+			data += 0x200;
+		regmap_write(map, SUN8I_CSC_COEFF(base, i), data);
+	}
+}
+
+static void sun8i_csc_enable(struct regmap *map, u32 base, bool enable)
+{
+	u32 val;
+
+	if (enable)
+		val = SUN8I_CSC_CTRL_EN;
+	else
+		val = 0;
+
+	regmap_update_bits(map, SUN8I_CSC_CTRL(base), SUN8I_CSC_CTRL_EN, val);
+}
+
+void sun8i_csc_set_ccsc_coefficients(struct sun8i_mixer *mixer, int layer,
+				     enum sun8i_csc_mode mode)
+{
+	u32 base;
+
+	base = ccsc_base[mixer->cfg->ccsc][layer];
+
+	sun8i_csc_set_coefficients(mixer->engine.regs, base, mode);
+}
+
+void sun8i_csc_enable_ccsc(struct sun8i_mixer *mixer, int layer, bool enable)
+{
+	u32 base;
+
+	base = ccsc_base[mixer->cfg->ccsc][layer];
+
+	sun8i_csc_enable(mixer->engine.regs, base, enable);
+}
diff --git a/drivers/gpu/drm/sun4i/sun8i_csc.h b/drivers/gpu/drm/sun4i/sun8i_csc.h
new file mode 100644
index 000000000000..880e8fbb0855
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_csc.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#ifndef _SUN8I_CSC_H_
+#define _SUN8I_CSC_H_
+
+struct sun8i_mixer;
+
+/* VI channel CSC units offsets */
+#define CCSC00_OFFSET 0xAA050
+#define CCSC01_OFFSET 0xFA000
+#define CCSC10_OFFSET 0xA0000
+#define CCSC11_OFFSET 0xF0000
+
+#define SUN8I_CSC_CTRL(base)		(base + 0x0)
+#define SUN8I_CSC_COEFF(base, i)	(base + 0x10 + 4 * i)
+
+#define SUN8I_CSC_CTRL_EN		BIT(0)
+
+enum sun8i_csc_mode {
+	SUN8I_CSC_MODE_OFF,
+	SUN8I_CSC_MODE_YUV2RGB,
+	SUN8I_CSC_MODE_YVU2RGB,
+};
+
+void sun8i_csc_set_ccsc_coefficients(struct sun8i_mixer *mixer, int layer,
+				     enum sun8i_csc_mode mode);
+void sun8i_csc_enable_ccsc(struct sun8i_mixer *mixer, int layer, bool enable);
+
+#endif
diff --git a/drivers/gpu/drm/sun4i/sun8i_layer.c b/drivers/gpu/drm/sun4i/sun8i_layer.c
deleted file mode 100644
index 23810ff72684..000000000000
--- a/drivers/gpu/drm/sun4i/sun8i_layer.c
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright (C) Icenowy Zheng <icenowy@aosc.io>
- *
- * Based on sun4i_layer.h, which is:
- *   Copyright (C) 2015 Free Electrons
- *   Copyright (C) 2015 NextThing Co
- *
- *   Maxime Ripard <maxime.ripard@free-electrons.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- */
-
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
-#include <drm/drmP.h>
-
-#include "sun8i_layer.h"
-#include "sun8i_mixer.h"
-
-struct sun8i_plane_desc {
-	       enum drm_plane_type     type;
-	       const uint32_t          *formats;
-	       uint32_t                nformats;
-};
-
-static void sun8i_mixer_layer_atomic_disable(struct drm_plane *plane,
-					       struct drm_plane_state *old_state)
-{
-	struct sun8i_layer *layer = plane_to_sun8i_layer(plane);
-	struct sun8i_mixer *mixer = layer->mixer;
-
-	sun8i_mixer_layer_enable(mixer, layer->id, false);
-}
-
-static void sun8i_mixer_layer_atomic_update(struct drm_plane *plane,
-					      struct drm_plane_state *old_state)
-{
-	struct sun8i_layer *layer = plane_to_sun8i_layer(plane);
-	struct sun8i_mixer *mixer = layer->mixer;
-
-	sun8i_mixer_update_layer_coord(mixer, layer->id, plane);
-	sun8i_mixer_update_layer_formats(mixer, layer->id, plane);
-	sun8i_mixer_update_layer_buffer(mixer, layer->id, plane);
-	sun8i_mixer_layer_enable(mixer, layer->id, true);
-}
-
-static struct drm_plane_helper_funcs sun8i_mixer_layer_helper_funcs = {
-	.atomic_disable	= sun8i_mixer_layer_atomic_disable,
-	.atomic_update	= sun8i_mixer_layer_atomic_update,
-};
-
-static const struct drm_plane_funcs sun8i_mixer_layer_funcs = {
-	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
-	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
-	.destroy		= drm_plane_cleanup,
-	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
-	.update_plane		= drm_atomic_helper_update_plane,
-};
-
-static const uint32_t sun8i_mixer_layer_formats[] = {
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_XRGB8888,
-};
-
-static const struct sun8i_plane_desc sun8i_mixer_planes[] = {
-	{
-		.type = DRM_PLANE_TYPE_PRIMARY,
-		.formats = sun8i_mixer_layer_formats,
-		.nformats = ARRAY_SIZE(sun8i_mixer_layer_formats),
-	},
-};
-
-static struct sun8i_layer *sun8i_layer_init_one(struct drm_device *drm,
-						struct sun8i_mixer *mixer,
-						const struct sun8i_plane_desc *plane)
-{
-	struct sun8i_layer *layer;
-	int ret;
-
-	layer = devm_kzalloc(drm->dev, sizeof(*layer), GFP_KERNEL);
-	if (!layer)
-		return ERR_PTR(-ENOMEM);
-
-	/* possible crtcs are set later */
-	ret = drm_universal_plane_init(drm, &layer->plane, 0,
-				       &sun8i_mixer_layer_funcs,
-				       plane->formats, plane->nformats,
-				       NULL, plane->type, NULL);
-	if (ret) {
-		dev_err(drm->dev, "Couldn't initialize layer\n");
-		return ERR_PTR(ret);
-	}
-
-	drm_plane_helper_add(&layer->plane,
-			     &sun8i_mixer_layer_helper_funcs);
-	layer->mixer = mixer;
-
-	return layer;
-}
-
-struct drm_plane **sun8i_layers_init(struct drm_device *drm,
-				     struct sunxi_engine *engine)
-{
-	struct drm_plane **planes;
-	struct sun8i_mixer *mixer = engine_to_sun8i_mixer(engine);
-	int i;
-
-	planes = devm_kcalloc(drm->dev, ARRAY_SIZE(sun8i_mixer_planes) + 1,
-			      sizeof(*planes), GFP_KERNEL);
-	if (!planes)
-		return ERR_PTR(-ENOMEM);
-
-	for (i = 0; i < ARRAY_SIZE(sun8i_mixer_planes); i++) {
-		const struct sun8i_plane_desc *plane = &sun8i_mixer_planes[i];
-		struct sun8i_layer *layer;
-
-		layer = sun8i_layer_init_one(drm, mixer, plane);
-		if (IS_ERR(layer)) {
-			dev_err(drm->dev, "Couldn't initialize %s plane\n",
-				i ? "overlay" : "primary");
-			return ERR_CAST(layer);
-		};
-
-		layer->id = i;
-		planes[i] = &layer->plane;
-	};
-
-	return planes;
-}
diff --git a/drivers/gpu/drm/sun4i/sun8i_layer.h b/drivers/gpu/drm/sun4i/sun8i_layer.h
deleted file mode 100644
index e5eccd27cff0..000000000000
--- a/drivers/gpu/drm/sun4i/sun8i_layer.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (C) Icenowy Zheng <icenowy@aosc.io>
- *
- * Based on sun4i_layer.h, which is:
- *   Copyright (C) 2015 Free Electrons
- *   Copyright (C) 2015 NextThing Co
- *
- *   Maxime Ripard <maxime.ripard@free-electrons.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- */
-
-#ifndef _SUN8I_LAYER_H_
-#define _SUN8I_LAYER_H_
-
-struct sunxi_engine;
-
-struct sun8i_layer {
-	struct drm_plane	plane;
-	struct sun4i_drv	*drv;
-	struct sun8i_mixer	*mixer;
-	int			id;
-};
-
-static inline struct sun8i_layer *
-plane_to_sun8i_layer(struct drm_plane *plane)
-{
-	return container_of(plane, struct sun8i_layer, plane);
-}
-
-struct drm_plane **sun8i_layers_init(struct drm_device *drm,
-				     struct sunxi_engine *engine);
-#endif /* _SUN8I_LAYER_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c
index cb193c5f1686..2cbb2de6d39c 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c
@@ -26,204 +26,288 @@
 
 #include "sun4i_drv.h"
 #include "sun8i_mixer.h"
-#include "sun8i_layer.h"
+#include "sun8i_ui_layer.h"
+#include "sun8i_vi_layer.h"
 #include "sunxi_engine.h"
 
-static void sun8i_mixer_commit(struct sunxi_engine *engine)
-{
-	DRM_DEBUG_DRIVER("Committing changes\n");
-
-	regmap_write(engine->regs, SUN8I_MIXER_GLOBAL_DBUFF,
-		     SUN8I_MIXER_GLOBAL_DBUFF_ENABLE);
-}
-
-void sun8i_mixer_layer_enable(struct sun8i_mixer *mixer,
-				int layer, bool enable)
-{
-	u32 val;
-	/* Currently the first UI channel is used */
-	int chan = mixer->cfg->vi_num;
-
-	DRM_DEBUG_DRIVER("Enabling layer %d in channel %d\n", layer, chan);
-
-	if (enable)
-		val = SUN8I_MIXER_CHAN_UI_LAYER_ATTR_EN;
-	else
-		val = 0;
-
-	regmap_update_bits(mixer->engine.regs,
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR(chan, layer),
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_EN, val);
-
-	/* Set the alpha configuration */
-	regmap_update_bits(mixer->engine.regs,
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR(chan, layer),
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MODE_MASK,
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MODE_DEF);
-	regmap_update_bits(mixer->engine.regs,
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR(chan, layer),
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MASK,
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_DEF);
-}
-
-static int sun8i_mixer_drm_format_to_layer(struct drm_plane *plane,
-					     u32 format, u32 *mode)
-{
-	switch (format) {
-	case DRM_FORMAT_ARGB8888:
-		*mode = SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_ARGB8888;
-		break;
-
-	case DRM_FORMAT_XRGB8888:
-		*mode = SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_XRGB8888;
-		break;
-
-	case DRM_FORMAT_RGB888:
-		*mode = SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_RGB888;
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
+static const struct de2_fmt_info de2_formats[] = {
+	{
+		.drm_fmt = DRM_FORMAT_ARGB8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_ARGB8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_ABGR8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_ABGR8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_RGBA8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGBA8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_BGRA8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_BGRA8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_XRGB8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_XRGB8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_XBGR8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_XBGR8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_RGBX8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGBX8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_BGRX8888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_BGRX8888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_RGB888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGB888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_BGR888,
+		.de2_fmt = SUN8I_MIXER_FBFMT_BGR888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_RGB565,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGB565,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_BGR565,
+		.de2_fmt = SUN8I_MIXER_FBFMT_BGR565,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_ARGB4444,
+		.de2_fmt = SUN8I_MIXER_FBFMT_ARGB4444,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_ABGR4444,
+		.de2_fmt = SUN8I_MIXER_FBFMT_ABGR4444,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_RGBA4444,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGBA4444,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_BGRA4444,
+		.de2_fmt = SUN8I_MIXER_FBFMT_BGRA4444,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_ARGB1555,
+		.de2_fmt = SUN8I_MIXER_FBFMT_ARGB1555,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_ABGR1555,
+		.de2_fmt = SUN8I_MIXER_FBFMT_ABGR1555,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_RGBA5551,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGBA5551,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_BGRA5551,
+		.de2_fmt = SUN8I_MIXER_FBFMT_BGRA5551,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_OFF,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_UYVY,
+		.de2_fmt = SUN8I_MIXER_FBFMT_UYVY,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_VYUY,
+		.de2_fmt = SUN8I_MIXER_FBFMT_VYUY,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YUYV,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YUYV,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YVYU,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YVYU,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_NV16,
+		.de2_fmt = SUN8I_MIXER_FBFMT_NV16,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_NV61,
+		.de2_fmt = SUN8I_MIXER_FBFMT_NV61,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_NV12,
+		.de2_fmt = SUN8I_MIXER_FBFMT_NV12,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_NV21,
+		.de2_fmt = SUN8I_MIXER_FBFMT_NV21,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YUV444,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGB888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YUV422,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YUV422,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YUV420,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YUV420,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YUV411,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YUV411,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YUV2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YVU444,
+		.de2_fmt = SUN8I_MIXER_FBFMT_RGB888,
+		.rgb = true,
+		.csc = SUN8I_CSC_MODE_YVU2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YVU422,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YUV422,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YVU2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YVU420,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YUV420,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YVU2RGB,
+	},
+	{
+		.drm_fmt = DRM_FORMAT_YVU411,
+		.de2_fmt = SUN8I_MIXER_FBFMT_YUV411,
+		.rgb = false,
+		.csc = SUN8I_CSC_MODE_YVU2RGB,
+	},
+};
 
-int sun8i_mixer_update_layer_coord(struct sun8i_mixer *mixer,
-				     int layer, struct drm_plane *plane)
+const struct de2_fmt_info *sun8i_mixer_format_info(u32 format)
 {
-	struct drm_plane_state *state = plane->state;
-	struct drm_framebuffer *fb = state->fb;
-	/* Currently the first UI channel is used */
-	int chan = mixer->cfg->vi_num;
-
-	DRM_DEBUG_DRIVER("Updating layer %d\n", layer);
-
-	if (plane->type == DRM_PLANE_TYPE_PRIMARY) {
-		DRM_DEBUG_DRIVER("Primary layer, updating global size W: %u H: %u\n",
-				 state->crtc_w, state->crtc_h);
-		regmap_write(mixer->engine.regs, SUN8I_MIXER_GLOBAL_SIZE,
-			     SUN8I_MIXER_SIZE(state->crtc_w,
-					      state->crtc_h));
-		DRM_DEBUG_DRIVER("Updating blender size\n");
-		regmap_write(mixer->engine.regs,
-			     SUN8I_MIXER_BLEND_ATTR_INSIZE(0),
-			     SUN8I_MIXER_SIZE(state->crtc_w,
-					      state->crtc_h));
-		regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_OUTSIZE,
-			     SUN8I_MIXER_SIZE(state->crtc_w,
-					      state->crtc_h));
-		DRM_DEBUG_DRIVER("Updating channel size\n");
-		regmap_write(mixer->engine.regs,
-			     SUN8I_MIXER_CHAN_UI_OVL_SIZE(chan),
-			     SUN8I_MIXER_SIZE(state->crtc_w,
-					      state->crtc_h));
-	}
+	unsigned int i;
 
-	/* Set the line width */
-	DRM_DEBUG_DRIVER("Layer line width: %d bytes\n", fb->pitches[0]);
-	regmap_write(mixer->engine.regs,
-		     SUN8I_MIXER_CHAN_UI_LAYER_PITCH(chan, layer),
-		     fb->pitches[0]);
-
-	/* Set height and width */
-	DRM_DEBUG_DRIVER("Layer size W: %u H: %u\n",
-			 state->crtc_w, state->crtc_h);
-	regmap_write(mixer->engine.regs,
-		     SUN8I_MIXER_CHAN_UI_LAYER_SIZE(chan, layer),
-		     SUN8I_MIXER_SIZE(state->crtc_w, state->crtc_h));
-
-	/* Set base coordinates */
-	DRM_DEBUG_DRIVER("Layer coordinates X: %d Y: %d\n",
-			 state->crtc_x, state->crtc_y);
-	regmap_write(mixer->engine.regs,
-		     SUN8I_MIXER_CHAN_UI_LAYER_COORD(chan, layer),
-		     SUN8I_MIXER_COORD(state->crtc_x, state->crtc_y));
+	for (i = 0; i < ARRAY_SIZE(de2_formats); ++i)
+		if (de2_formats[i].drm_fmt == format)
+			return &de2_formats[i];
 
-	return 0;
+	return NULL;
 }
 
-int sun8i_mixer_update_layer_formats(struct sun8i_mixer *mixer,
-				       int layer, struct drm_plane *plane)
+static void sun8i_mixer_commit(struct sunxi_engine *engine)
 {
-	struct drm_plane_state *state = plane->state;
-	struct drm_framebuffer *fb = state->fb;
-	bool interlaced = false;
-	u32 val;
-	/* Currently the first UI channel is used */
-	int chan = mixer->cfg->vi_num;
-	int ret;
-
-	if (plane->state->crtc)
-		interlaced = plane->state->crtc->state->adjusted_mode.flags
-			& DRM_MODE_FLAG_INTERLACE;
-
-	regmap_update_bits(mixer->engine.regs, SUN8I_MIXER_BLEND_OUTCTL,
-			   SUN8I_MIXER_BLEND_OUTCTL_INTERLACED,
-			   interlaced ?
-			   SUN8I_MIXER_BLEND_OUTCTL_INTERLACED : 0);
-
-	DRM_DEBUG_DRIVER("Switching display mixer interlaced mode %s\n",
-			 interlaced ? "on" : "off");
-
-	ret = sun8i_mixer_drm_format_to_layer(plane, fb->format->format,
-						&val);
-	if (ret) {
-		DRM_DEBUG_DRIVER("Invalid format\n");
-		return ret;
-	}
-
-	regmap_update_bits(mixer->engine.regs,
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR(chan, layer),
-			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_MASK, val);
+	DRM_DEBUG_DRIVER("Committing changes\n");
 
-	return 0;
+	regmap_write(engine->regs, SUN8I_MIXER_GLOBAL_DBUFF,
+		     SUN8I_MIXER_GLOBAL_DBUFF_ENABLE);
 }
 
-int sun8i_mixer_update_layer_buffer(struct sun8i_mixer *mixer,
-				      int layer, struct drm_plane *plane)
+static struct drm_plane **sun8i_layers_init(struct drm_device *drm,
+					    struct sunxi_engine *engine)
 {
-	struct drm_plane_state *state = plane->state;
-	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *gem;
-	dma_addr_t paddr;
-	/* Currently the first UI channel is used */
-	int chan = mixer->cfg->vi_num;
-	int bpp;
-
-	/* Get the physical address of the buffer in memory */
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
-
-	DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->paddr);
-
-	/* Compute the start of the displayed memory */
-	bpp = fb->format->cpp[0];
-	paddr = gem->paddr + fb->offsets[0];
-
-	/* Fixup framebuffer address for src coordinates */
-	paddr += (state->src_x >> 16) * bpp;
-	paddr += (state->src_y >> 16) * fb->pitches[0];
-
-	/*
-	 * The hardware cannot correctly deal with negative crtc
-	 * coordinates, the display is cropped to the requested size,
-	 * but the display content is not moved.
-	 * Manually move the display content by fixup the framebuffer
-	 * address when crtc_x or crtc_y is negative, like what we
-	 * have did for src_x and src_y.
-	 */
-	if (state->crtc_x < 0)
-		paddr += -state->crtc_x * bpp;
-	if (state->crtc_y < 0)
-		paddr += -state->crtc_y * fb->pitches[0];
-
-	DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr);
-
-	regmap_write(mixer->engine.regs,
-		     SUN8I_MIXER_CHAN_UI_LAYER_TOP_LADDR(chan, layer),
-		     lower_32_bits(paddr));
-
-	return 0;
+	struct drm_plane **planes;
+	struct sun8i_mixer *mixer = engine_to_sun8i_mixer(engine);
+	int i;
+
+	planes = devm_kcalloc(drm->dev,
+			      mixer->cfg->vi_num + mixer->cfg->ui_num + 1,
+			      sizeof(*planes), GFP_KERNEL);
+	if (!planes)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < mixer->cfg->vi_num; i++) {
+		struct sun8i_vi_layer *layer;
+
+		layer = sun8i_vi_layer_init_one(drm, mixer, i);
+		if (IS_ERR(layer)) {
+			dev_err(drm->dev,
+				"Couldn't initialize overlay plane\n");
+			return ERR_CAST(layer);
+		};
+
+		planes[i] = &layer->plane;
+	};
+
+	for (i = 0; i < mixer->cfg->ui_num; i++) {
+		struct sun8i_ui_layer *layer;
+
+		layer = sun8i_ui_layer_init_one(drm, mixer, i);
+		if (IS_ERR(layer)) {
+			dev_err(drm->dev, "Couldn't initialize %s plane\n",
+				i ? "overlay" : "primary");
+			return ERR_CAST(layer);
+		};
+
+		planes[mixer->cfg->vi_num + i] = &layer->plane;
+	};
+
+	return planes;
 }
 
 static const struct sunxi_engine_ops sun8i_engine_ops = {
@@ -247,6 +331,7 @@ static int sun8i_mixer_bind(struct device *dev, struct device *master,
 	struct sun8i_mixer *mixer;
 	struct resource *res;
 	void __iomem *regs;
+	int plane_cnt;
 	int i, ret;
 
 	/*
@@ -313,6 +398,15 @@ static int sun8i_mixer_bind(struct device *dev, struct device *master,
 		ret = PTR_ERR(mixer->mod_clk);
 		goto err_disable_bus_clk;
 	}
+
+	/*
+	 * It seems that we need to enforce that rate for whatever
+	 * reason for the mixer to be functional. Make sure it's the
+	 * case.
+	 */
+	if (mixer->cfg->mod_rate)
+		clk_set_rate(mixer->mod_clk, mixer->cfg->mod_rate);
+
 	clk_prepare_enable(mixer->mod_clk);
 
 	list_add_tail(&mixer->engine.list, &drv->engine_list);
@@ -325,27 +419,26 @@ static int sun8i_mixer_bind(struct device *dev, struct device *master,
 	regmap_write(mixer->engine.regs, SUN8I_MIXER_GLOBAL_CTL,
 		     SUN8I_MIXER_GLOBAL_CTL_RT_EN);
 
-	/* Initialize blender */
-	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_FCOLOR_CTL,
-		     SUN8I_MIXER_BLEND_FCOLOR_CTL_DEF);
-	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_PREMULTIPLY,
-		     SUN8I_MIXER_BLEND_PREMULTIPLY_DEF);
+	/* Set background color to black */
 	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_BKCOLOR,
-		     SUN8I_MIXER_BLEND_BKCOLOR_DEF);
-	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_MODE(0),
-		     SUN8I_MIXER_BLEND_MODE_DEF);
-	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_CK_CTL,
-		     SUN8I_MIXER_BLEND_CK_CTL_DEF);
-
-	regmap_write(mixer->engine.regs,
-		     SUN8I_MIXER_BLEND_ATTR_FCOLOR(0),
-		     SUN8I_MIXER_BLEND_ATTR_FCOLOR_DEF);
-
-	/* Select the first UI channel */
-	DRM_DEBUG_DRIVER("Selecting channel %d (first UI channel)\n",
-			 mixer->cfg->vi_num);
-	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_ROUTE,
-		     mixer->cfg->vi_num);
+		     SUN8I_MIXER_BLEND_COLOR_BLACK);
+
+	/*
+	 * Set fill color of bottom plane to black. Generally not needed
+	 * except when VI plane is at bottom (zpos = 0) and enabled.
+	 */
+	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_PIPE_CTL,
+		     SUN8I_MIXER_BLEND_PIPE_CTL_FC_EN(0));
+	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_ATTR_FCOLOR(0),
+		     SUN8I_MIXER_BLEND_COLOR_BLACK);
+
+	/* Fixed zpos for now */
+	regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_ROUTE, 0x43210);
+
+	plane_cnt = mixer->cfg->vi_num + mixer->cfg->ui_num;
+	for (i = 0; i < plane_cnt; i++)
+		regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_MODE(i),
+			     SUN8I_MIXER_BLEND_MODE_DEF);
 
 	return 0;
 
@@ -385,12 +478,26 @@ static int sun8i_mixer_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct sun8i_mixer_cfg sun8i_a83t_mixer0_cfg = {
+	.ccsc		= 0,
+	.scaler_mask	= 0xf,
+	.ui_num		= 3,
+	.vi_num		= 1,
+};
+
 static const struct sun8i_mixer_cfg sun8i_v3s_mixer_cfg = {
 	.vi_num = 2,
 	.ui_num = 1,
+	.scaler_mask = 0x3,
+	.ccsc = 0,
+	.mod_rate = 150000000,
 };
 
 static const struct of_device_id sun8i_mixer_of_table[] = {
+	{
+		.compatible = "allwinner,sun8i-a83t-de2-mixer-0",
+		.data = &sun8i_a83t_mixer0_cfg,
+	},
 	{
 		.compatible = "allwinner,sun8i-v3s-de2-mixer",
 		.data = &sun8i_v3s_mixer_cfg,
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.h b/drivers/gpu/drm/sun4i/sun8i_mixer.h
index 4785ac090b8c..f34e70c42adf 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.h
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.h
@@ -14,10 +14,9 @@
 #include <linux/regmap.h>
 #include <linux/reset.h>
 
+#include "sun8i_csc.h"
 #include "sunxi_engine.h"
 
-#define SUN8I_MIXER_MAX_CHAN_COUNT		4
-
 #define SUN8I_MIXER_SIZE(w, h)			(((h) - 1) << 16 | ((w) - 1))
 #define SUN8I_MIXER_COORD(x, y)			((y) << 16 | (x))
 
@@ -26,14 +25,14 @@
 #define SUN8I_MIXER_GLOBAL_DBUFF		0x8
 #define SUN8I_MIXER_GLOBAL_SIZE			0xc
 
-#define SUN8I_MIXER_GLOBAL_CTL_RT_EN		0x1
+#define SUN8I_MIXER_GLOBAL_CTL_RT_EN		BIT(0)
 
-#define SUN8I_MIXER_GLOBAL_DBUFF_ENABLE		0x1
+#define SUN8I_MIXER_GLOBAL_DBUFF_ENABLE		BIT(0)
 
-#define SUN8I_MIXER_BLEND_FCOLOR_CTL		0x1000
+#define SUN8I_MIXER_BLEND_PIPE_CTL		0x1000
 #define SUN8I_MIXER_BLEND_ATTR_FCOLOR(x)	(0x1004 + 0x10 * (x) + 0x0)
 #define SUN8I_MIXER_BLEND_ATTR_INSIZE(x)	(0x1004 + 0x10 * (x) + 0x4)
-#define SUN8I_MIXER_BLEND_ATTR_OFFSET(x)	(0x1004 + 0x10 * (x) + 0x8)
+#define SUN8I_MIXER_BLEND_ATTR_COORD(x)		(0x1004 + 0x10 * (x) + 0x8)
 #define SUN8I_MIXER_BLEND_ROUTE			0x1080
 #define SUN8I_MIXER_BLEND_PREMULTIPLY		0x1084
 #define SUN8I_MIXER_BLEND_BKCOLOR		0x1088
@@ -45,57 +44,56 @@
 #define SUN8I_MIXER_BLEND_CK_MIN(x)		(0x10e0 + 0x04 * (x))
 #define SUN8I_MIXER_BLEND_OUTCTL		0x10fc
 
+#define SUN8I_MIXER_BLEND_PIPE_CTL_EN(pipe)	BIT(8 + pipe)
+#define SUN8I_MIXER_BLEND_PIPE_CTL_FC_EN(pipe)	BIT(pipe)
+/* colors are always in AARRGGBB format */
+#define SUN8I_MIXER_BLEND_COLOR_BLACK		0xff000000
 /* The following numbers are some still unknown magic numbers */
-#define SUN8I_MIXER_BLEND_ATTR_FCOLOR_DEF	0xff000000
-#define SUN8I_MIXER_BLEND_FCOLOR_CTL_DEF	0x00000101
-#define SUN8I_MIXER_BLEND_PREMULTIPLY_DEF	0x0
-#define SUN8I_MIXER_BLEND_BKCOLOR_DEF		0xff000000
 #define SUN8I_MIXER_BLEND_MODE_DEF		0x03010301
-#define SUN8I_MIXER_BLEND_CK_CTL_DEF		0x0
 
 #define SUN8I_MIXER_BLEND_OUTCTL_INTERLACED	BIT(1)
 
-/*
- * VI channels are not used now, but the support of them may be introduced in
- * the future.
- */
-
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR(ch, layer) \
-			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x0)
-#define SUN8I_MIXER_CHAN_UI_LAYER_SIZE(ch, layer) \
-			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x4)
-#define SUN8I_MIXER_CHAN_UI_LAYER_COORD(ch, layer) \
-			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x8)
-#define SUN8I_MIXER_CHAN_UI_LAYER_PITCH(ch, layer) \
-			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0xc)
-#define SUN8I_MIXER_CHAN_UI_LAYER_TOP_LADDR(ch, layer) \
-			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x10)
-#define SUN8I_MIXER_CHAN_UI_LAYER_BOT_LADDR(ch, layer) \
-			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x14)
-#define SUN8I_MIXER_CHAN_UI_LAYER_FCOLOR(ch, layer) \
-			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x18)
-#define SUN8I_MIXER_CHAN_UI_TOP_HADDR(ch)	(0x2000 + 0x1000 * (ch) + 0x80)
-#define SUN8I_MIXER_CHAN_UI_BOT_HADDR(ch)	(0x2000 + 0x1000 * (ch) + 0x84)
-#define SUN8I_MIXER_CHAN_UI_OVL_SIZE(ch)	(0x2000 + 0x1000 * (ch) + 0x88)
-
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_EN		BIT(0)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MODE_MASK	GENMASK(2, 1)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_MASK	GENMASK(11, 8)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MASK	GENMASK(31, 24)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MODE_DEF	(1 << 1)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_ARGB8888	(0 << 8)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_XRGB8888	(4 << 8)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_RGB888	(8 << 8)
-#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_DEF	(0xff << 24)
+#define SUN8I_MIXER_FBFMT_ARGB8888	0
+#define SUN8I_MIXER_FBFMT_ABGR8888	1
+#define SUN8I_MIXER_FBFMT_RGBA8888	2
+#define SUN8I_MIXER_FBFMT_BGRA8888	3
+#define SUN8I_MIXER_FBFMT_XRGB8888	4
+#define SUN8I_MIXER_FBFMT_XBGR8888	5
+#define SUN8I_MIXER_FBFMT_RGBX8888	6
+#define SUN8I_MIXER_FBFMT_BGRX8888	7
+#define SUN8I_MIXER_FBFMT_RGB888	8
+#define SUN8I_MIXER_FBFMT_BGR888	9
+#define SUN8I_MIXER_FBFMT_RGB565	10
+#define SUN8I_MIXER_FBFMT_BGR565	11
+#define SUN8I_MIXER_FBFMT_ARGB4444	12
+#define SUN8I_MIXER_FBFMT_ABGR4444	13
+#define SUN8I_MIXER_FBFMT_RGBA4444	14
+#define SUN8I_MIXER_FBFMT_BGRA4444	15
+#define SUN8I_MIXER_FBFMT_ARGB1555	16
+#define SUN8I_MIXER_FBFMT_ABGR1555	17
+#define SUN8I_MIXER_FBFMT_RGBA5551	18
+#define SUN8I_MIXER_FBFMT_BGRA5551	19
+
+#define SUN8I_MIXER_FBFMT_YUYV		0
+#define SUN8I_MIXER_FBFMT_UYVY		1
+#define SUN8I_MIXER_FBFMT_YVYU		2
+#define SUN8I_MIXER_FBFMT_VYUY		3
+#define SUN8I_MIXER_FBFMT_NV16		4
+#define SUN8I_MIXER_FBFMT_NV61		5
+#define SUN8I_MIXER_FBFMT_YUV422	6
+/* format 7 doesn't exist */
+#define SUN8I_MIXER_FBFMT_NV12		8
+#define SUN8I_MIXER_FBFMT_NV21		9
+#define SUN8I_MIXER_FBFMT_YUV420	10
+/* format 11 doesn't exist */
+/* format 12 is semi-planar YUV411 UVUV */
+/* format 13 is semi-planar YUV411 VUVU */
+#define SUN8I_MIXER_FBFMT_YUV411	14
 
 /*
  * These sub-engines are still unknown now, the EN registers are here only to
  * be used to disable these sub-engines.
  */
-#define SUN8I_MIXER_VSU_EN			0x20000
-#define SUN8I_MIXER_GSU1_EN			0x30000
-#define SUN8I_MIXER_GSU2_EN			0x40000
-#define SUN8I_MIXER_GSU3_EN			0x50000
 #define SUN8I_MIXER_FCE_EN			0xa0000
 #define SUN8I_MIXER_BWS_EN			0xa2000
 #define SUN8I_MIXER_LTI_EN			0xa4000
@@ -104,9 +102,34 @@
 #define SUN8I_MIXER_FCC_EN			0xaa000
 #define SUN8I_MIXER_DCSC_EN			0xb0000
 
+struct de2_fmt_info {
+	u32			drm_fmt;
+	u32			de2_fmt;
+	bool			rgb;
+	enum sun8i_csc_mode	csc;
+};
+
+/**
+ * struct sun8i_mixer_cfg - mixer HW configuration
+ * @vi_num: number of VI channels
+ * @ui_num: number of UI channels
+ * @scaler_mask: bitmask which tells which channel supports scaling
+ *	First, scaler supports for VI channels is defined and after that, scaler
+ *	support for UI channels. For example, if mixer has 2 VI channels without
+ *	scaler and 2 UI channels with scaler, bitmask would be 0xC.
+ * @ccsc: select set of CCSC base addresses
+ *	Set value to 0 if this is first mixer or second mixer with VEP support.
+ *	Set value to 1 if this is second mixer without VEP support. Other values
+ *	are invalid.
+ * @mod_rate: module clock rate that needs to be set in order to have
+ *	a functional block.
+ */
 struct sun8i_mixer_cfg {
 	int		vi_num;
 	int		ui_num;
+	int		scaler_mask;
+	int		ccsc;
+	unsigned long	mod_rate;
 };
 
 struct sun8i_mixer {
@@ -126,12 +149,5 @@ engine_to_sun8i_mixer(struct sunxi_engine *engine)
 	return container_of(engine, struct sun8i_mixer, engine);
 }
 
-void sun8i_mixer_layer_enable(struct sun8i_mixer *mixer,
-				int layer, bool enable);
-int sun8i_mixer_update_layer_coord(struct sun8i_mixer *mixer,
-				     int layer, struct drm_plane *plane);
-int sun8i_mixer_update_layer_formats(struct sun8i_mixer *mixer,
-				       int layer, struct drm_plane *plane);
-int sun8i_mixer_update_layer_buffer(struct sun8i_mixer *mixer,
-				      int layer, struct drm_plane *plane);
+const struct de2_fmt_info *sun8i_mixer_format_info(u32 format);
 #endif /* _SUN8I_MIXER_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
new file mode 100644
index 000000000000..28d7c48d50fe
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
@@ -0,0 +1,349 @@
+/*
+ * Copyright (C) Icenowy Zheng <icenowy@aosc.io>
+ *
+ * Based on sun4i_layer.h, which is:
+ *   Copyright (C) 2015 Free Electrons
+ *   Copyright (C) 2015 NextThing Co
+ *
+ *   Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drmP.h>
+
+#include "sun8i_ui_layer.h"
+#include "sun8i_mixer.h"
+#include "sun8i_ui_scaler.h"
+
+static void sun8i_ui_layer_enable(struct sun8i_mixer *mixer, int channel,
+				  int overlay, bool enable)
+{
+	u32 val;
+
+	DRM_DEBUG_DRIVER("%sabling channel %d overlay %d\n",
+			 enable ? "En" : "Dis", channel, overlay);
+
+	if (enable)
+		val = SUN8I_MIXER_CHAN_UI_LAYER_ATTR_EN;
+	else
+		val = 0;
+
+	regmap_update_bits(mixer->engine.regs,
+			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR(channel, overlay),
+			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_EN, val);
+
+	if (enable)
+		val = SUN8I_MIXER_BLEND_PIPE_CTL_EN(channel);
+	else
+		val = 0;
+
+	regmap_update_bits(mixer->engine.regs,
+			   SUN8I_MIXER_BLEND_PIPE_CTL,
+			   SUN8I_MIXER_BLEND_PIPE_CTL_EN(channel), val);
+}
+
+static int sun8i_ui_layer_update_coord(struct sun8i_mixer *mixer, int channel,
+				       int overlay, struct drm_plane *plane)
+{
+	struct drm_plane_state *state = plane->state;
+	u32 src_w, src_h, dst_w, dst_h;
+	u32 outsize, insize;
+	u32 hphase, vphase;
+
+	DRM_DEBUG_DRIVER("Updating UI channel %d overlay %d\n",
+			 channel, overlay);
+
+	src_w = drm_rect_width(&state->src) >> 16;
+	src_h = drm_rect_height(&state->src) >> 16;
+	dst_w = drm_rect_width(&state->dst);
+	dst_h = drm_rect_height(&state->dst);
+
+	hphase = state->src.x1 & 0xffff;
+	vphase = state->src.y1 & 0xffff;
+
+	insize = SUN8I_MIXER_SIZE(src_w, src_h);
+	outsize = SUN8I_MIXER_SIZE(dst_w, dst_h);
+
+	if (plane->type == DRM_PLANE_TYPE_PRIMARY) {
+		bool interlaced = false;
+		u32 val;
+
+		DRM_DEBUG_DRIVER("Primary layer, updating global size W: %u H: %u\n",
+				 dst_w, dst_h);
+		regmap_write(mixer->engine.regs,
+			     SUN8I_MIXER_GLOBAL_SIZE,
+			     outsize);
+		regmap_write(mixer->engine.regs, SUN8I_MIXER_BLEND_OUTSIZE,
+			     outsize);
+
+		if (state->crtc)
+			interlaced = state->crtc->state->adjusted_mode.flags
+				& DRM_MODE_FLAG_INTERLACE;
+
+		if (interlaced)
+			val = SUN8I_MIXER_BLEND_OUTCTL_INTERLACED;
+		else
+			val = 0;
+
+		regmap_update_bits(mixer->engine.regs,
+				   SUN8I_MIXER_BLEND_OUTCTL,
+				   SUN8I_MIXER_BLEND_OUTCTL_INTERLACED,
+				   val);
+
+		DRM_DEBUG_DRIVER("Switching display mixer interlaced mode %s\n",
+				 interlaced ? "on" : "off");
+	}
+
+	/* Set height and width */
+	DRM_DEBUG_DRIVER("Layer source offset X: %d Y: %d\n",
+			 state->src.x1 >> 16, state->src.y1 >> 16);
+	DRM_DEBUG_DRIVER("Layer source size W: %d H: %d\n", src_w, src_h);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_UI_LAYER_SIZE(channel, overlay),
+		     insize);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_UI_OVL_SIZE(channel),
+		     insize);
+
+	if (insize != outsize || hphase || vphase) {
+		u32 hscale, vscale;
+
+		DRM_DEBUG_DRIVER("HW scaling is enabled\n");
+
+		hscale = state->src_w / state->crtc_w;
+		vscale = state->src_h / state->crtc_h;
+
+		sun8i_ui_scaler_setup(mixer, channel, src_w, src_h, dst_w,
+				      dst_h, hscale, vscale, hphase, vphase);
+		sun8i_ui_scaler_enable(mixer, channel, true);
+	} else {
+		DRM_DEBUG_DRIVER("HW scaling is not needed\n");
+		sun8i_ui_scaler_enable(mixer, channel, false);
+	}
+
+	/* Set base coordinates */
+	DRM_DEBUG_DRIVER("Layer destination coordinates X: %d Y: %d\n",
+			 state->dst.x1, state->dst.y1);
+	DRM_DEBUG_DRIVER("Layer destination size W: %d H: %d\n", dst_w, dst_h);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_BLEND_ATTR_COORD(channel),
+		     SUN8I_MIXER_COORD(state->dst.x1, state->dst.y1));
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_BLEND_ATTR_INSIZE(channel),
+		     outsize);
+
+	return 0;
+}
+
+static int sun8i_ui_layer_update_formats(struct sun8i_mixer *mixer, int channel,
+					 int overlay, struct drm_plane *plane)
+{
+	struct drm_plane_state *state = plane->state;
+	const struct de2_fmt_info *fmt_info;
+	u32 val;
+
+	fmt_info = sun8i_mixer_format_info(state->fb->format->format);
+	if (!fmt_info || !fmt_info->rgb) {
+		DRM_DEBUG_DRIVER("Invalid format\n");
+		return -EINVAL;
+	}
+
+	val = fmt_info->de2_fmt << SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_OFFSET;
+	regmap_update_bits(mixer->engine.regs,
+			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR(channel, overlay),
+			   SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_MASK, val);
+
+	return 0;
+}
+
+static int sun8i_ui_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
+					int overlay, struct drm_plane *plane)
+{
+	struct drm_plane_state *state = plane->state;
+	struct drm_framebuffer *fb = state->fb;
+	struct drm_gem_cma_object *gem;
+	dma_addr_t paddr;
+	int bpp;
+
+	/* Get the physical address of the buffer in memory */
+	gem = drm_fb_cma_get_gem_obj(fb, 0);
+
+	DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->paddr);
+
+	/* Compute the start of the displayed memory */
+	bpp = fb->format->cpp[0];
+	paddr = gem->paddr + fb->offsets[0];
+
+	/* Fixup framebuffer address for src coordinates */
+	paddr += (state->src.x1 >> 16) * bpp;
+	paddr += (state->src.y1 >> 16) * fb->pitches[0];
+
+	/* Set the line width */
+	DRM_DEBUG_DRIVER("Layer line width: %d bytes\n", fb->pitches[0]);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_UI_LAYER_PITCH(channel, overlay),
+		     fb->pitches[0]);
+
+	DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr);
+
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_UI_LAYER_TOP_LADDR(channel, overlay),
+		     lower_32_bits(paddr));
+
+	return 0;
+}
+
+static int sun8i_ui_layer_atomic_check(struct drm_plane *plane,
+				       struct drm_plane_state *state)
+{
+	struct sun8i_ui_layer *layer = plane_to_sun8i_ui_layer(plane);
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_crtc_state *crtc_state;
+	int min_scale, max_scale;
+	struct drm_rect clip;
+
+	if (!crtc)
+		return 0;
+
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc);
+	if (WARN_ON(!crtc_state))
+		return -EINVAL;
+
+	clip.x1 = 0;
+	clip.y1 = 0;
+	clip.x2 = crtc_state->adjusted_mode.hdisplay;
+	clip.y2 = crtc_state->adjusted_mode.vdisplay;
+
+	min_scale = DRM_PLANE_HELPER_NO_SCALING;
+	max_scale = DRM_PLANE_HELPER_NO_SCALING;
+
+	if (layer->mixer->cfg->scaler_mask & BIT(layer->channel)) {
+		min_scale = SUN8I_UI_SCALER_SCALE_MIN;
+		max_scale = SUN8I_UI_SCALER_SCALE_MAX;
+	}
+
+	return drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						   min_scale, max_scale,
+						   true, true);
+}
+
+static void sun8i_ui_layer_atomic_disable(struct drm_plane *plane,
+					  struct drm_plane_state *old_state)
+{
+	struct sun8i_ui_layer *layer = plane_to_sun8i_ui_layer(plane);
+	struct sun8i_mixer *mixer = layer->mixer;
+
+	sun8i_ui_layer_enable(mixer, layer->channel, layer->overlay, false);
+}
+
+static void sun8i_ui_layer_atomic_update(struct drm_plane *plane,
+					 struct drm_plane_state *old_state)
+{
+	struct sun8i_ui_layer *layer = plane_to_sun8i_ui_layer(plane);
+	struct sun8i_mixer *mixer = layer->mixer;
+
+	if (!plane->state->visible) {
+		sun8i_ui_layer_enable(mixer, layer->channel,
+				      layer->overlay, false);
+		return;
+	}
+
+	sun8i_ui_layer_update_coord(mixer, layer->channel,
+				    layer->overlay, plane);
+	sun8i_ui_layer_update_formats(mixer, layer->channel,
+				      layer->overlay, plane);
+	sun8i_ui_layer_update_buffer(mixer, layer->channel,
+				     layer->overlay, plane);
+	sun8i_ui_layer_enable(mixer, layer->channel, layer->overlay, true);
+}
+
+static struct drm_plane_helper_funcs sun8i_ui_layer_helper_funcs = {
+	.atomic_check	= sun8i_ui_layer_atomic_check,
+	.atomic_disable	= sun8i_ui_layer_atomic_disable,
+	.atomic_update	= sun8i_ui_layer_atomic_update,
+};
+
+static const struct drm_plane_funcs sun8i_ui_layer_funcs = {
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.destroy		= drm_plane_cleanup,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.reset			= drm_atomic_helper_plane_reset,
+	.update_plane		= drm_atomic_helper_update_plane,
+};
+
+static const u32 sun8i_ui_layer_formats[] = {
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_BGRA4444,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_RGBA4444,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_XRGB8888,
+};
+
+struct sun8i_ui_layer *sun8i_ui_layer_init_one(struct drm_device *drm,
+					       struct sun8i_mixer *mixer,
+					       int index)
+{
+	enum drm_plane_type type = DRM_PLANE_TYPE_OVERLAY;
+	int channel = mixer->cfg->vi_num + index;
+	struct sun8i_ui_layer *layer;
+	int ret;
+
+	layer = devm_kzalloc(drm->dev, sizeof(*layer), GFP_KERNEL);
+	if (!layer)
+		return ERR_PTR(-ENOMEM);
+
+	if (index == 0)
+		type = DRM_PLANE_TYPE_PRIMARY;
+
+	/* possible crtcs are set later */
+	ret = drm_universal_plane_init(drm, &layer->plane, 0,
+				       &sun8i_ui_layer_funcs,
+				       sun8i_ui_layer_formats,
+				       ARRAY_SIZE(sun8i_ui_layer_formats),
+				       NULL, type, NULL);
+	if (ret) {
+		dev_err(drm->dev, "Couldn't initialize layer\n");
+		return ERR_PTR(ret);
+	}
+
+	/* fixed zpos for now */
+	ret = drm_plane_create_zpos_immutable_property(&layer->plane, channel);
+	if (ret) {
+		dev_err(drm->dev, "Couldn't add zpos property\n");
+		return ERR_PTR(ret);
+	}
+
+	drm_plane_helper_add(&layer->plane, &sun8i_ui_layer_helper_funcs);
+	layer->mixer = mixer;
+	layer->channel = channel;
+	layer->overlay = 0;
+
+	return layer;
+}
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_layer.h b/drivers/gpu/drm/sun4i/sun8i_ui_layer.h
new file mode 100644
index 000000000000..123b15ea9918
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_layer.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) Icenowy Zheng <icenowy@aosc.io>
+ *
+ * Based on sun4i_layer.h, which is:
+ *   Copyright (C) 2015 Free Electrons
+ *   Copyright (C) 2015 NextThing Co
+ *
+ *   Maxime Ripard <maxime.ripard@free-electrons.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#ifndef _SUN8I_UI_LAYER_H_
+#define _SUN8I_UI_LAYER_H_
+
+#include <drm/drm_plane.h>
+
+#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR(ch, layer) \
+			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x0)
+#define SUN8I_MIXER_CHAN_UI_LAYER_SIZE(ch, layer) \
+			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x4)
+#define SUN8I_MIXER_CHAN_UI_LAYER_COORD(ch, layer) \
+			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x8)
+#define SUN8I_MIXER_CHAN_UI_LAYER_PITCH(ch, layer) \
+			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0xc)
+#define SUN8I_MIXER_CHAN_UI_LAYER_TOP_LADDR(ch, layer) \
+			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x10)
+#define SUN8I_MIXER_CHAN_UI_LAYER_BOT_LADDR(ch, layer) \
+			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x14)
+#define SUN8I_MIXER_CHAN_UI_LAYER_FCOLOR(ch, layer) \
+			(0x2000 + 0x1000 * (ch) + 0x20 * (layer) + 0x18)
+#define SUN8I_MIXER_CHAN_UI_TOP_HADDR(ch)	(0x2000 + 0x1000 * (ch) + 0x80)
+#define SUN8I_MIXER_CHAN_UI_BOT_HADDR(ch)	(0x2000 + 0x1000 * (ch) + 0x84)
+#define SUN8I_MIXER_CHAN_UI_OVL_SIZE(ch)	(0x2000 + 0x1000 * (ch) + 0x88)
+
+#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_EN		BIT(0)
+#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MODE_MASK	GENMASK(2, 1)
+#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_MASK	GENMASK(12, 8)
+#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_FBFMT_OFFSET	8
+#define SUN8I_MIXER_CHAN_UI_LAYER_ATTR_ALPHA_MASK	GENMASK(31, 24)
+
+struct sun8i_mixer;
+
+struct sun8i_ui_layer {
+	struct drm_plane	plane;
+	struct sun8i_mixer	*mixer;
+	int			channel;
+	int			overlay;
+};
+
+static inline struct sun8i_ui_layer *
+plane_to_sun8i_ui_layer(struct drm_plane *plane)
+{
+	return container_of(plane, struct sun8i_ui_layer, plane);
+}
+
+struct sun8i_ui_layer *sun8i_ui_layer_init_one(struct drm_device *drm,
+					       struct sun8i_mixer *mixer,
+					       int index);
+#endif /* _SUN8I_UI_LAYER_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_scaler.c b/drivers/gpu/drm/sun4i/sun8i_ui_scaler.c
new file mode 100644
index 000000000000..6bb2aa164c8e
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_scaler.c
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2017 Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * Coefficients are taken from BSP driver, which is:
+ * Copyright (C) 2014-2015 Allwinner
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include "sun8i_ui_scaler.h"
+
+static const u32 lan2coefftab16[240] = {
+	0x00004000, 0x00033ffe, 0x00063efc, 0x000a3bfb,
+	0xff0f37fb, 0xfe1433fb, 0xfd192ffb, 0xfd1f29fb,
+	0xfc2424fc, 0xfb291ffd, 0xfb2f19fd, 0xfb3314fe,
+	0xfb370fff, 0xfb3b0a00, 0xfc3e0600, 0xfe3f0300,
+
+	0xff053804, 0xff083801, 0xff0a3700, 0xff0e34ff,
+	0xff1232fd, 0xfe162ffd, 0xfd1b2cfc, 0xfd1f28fc,
+	0xfd2323fd, 0xfc281ffd, 0xfc2c1bfd, 0xfd2f16fe,
+	0xfd3212ff, 0xff340eff, 0x00360a00, 0x02370700,
+
+	0xff083207, 0xff0a3205, 0xff0d3103, 0xfe113001,
+	0xfe142e00, 0xfe182bff, 0xfe1b29fe, 0xfe1f25fe,
+	0xfe2222fe, 0xfe251ffe, 0xfe291bfe, 0xff2b18fe,
+	0x002e14fe, 0x013010ff, 0x03310dff, 0x05310a00,
+
+	0xff0a2e09, 0xff0c2e07, 0xff0f2d05, 0xff122c03,
+	0xfe152b02, 0xfe182901, 0xfe1b2700, 0xff1e24ff,
+	0xff2121ff, 0xff241eff, 0x00261bff, 0x012818ff,
+	0x022a15ff, 0x032c12ff, 0x052d0fff, 0x072d0c00,
+
+	0xff0c2a0b, 0xff0e2a09, 0xff102a07, 0xff132905,
+	0xff162803, 0xff182702, 0xff1b2501, 0xff1e2300,
+	0x00202000, 0x01221d00, 0x01251bff, 0x032618ff,
+	0x042815ff, 0x052913ff, 0x072a10ff, 0x092a0d00,
+
+	0xff0d280c, 0xff0f280a, 0xff112808, 0xff142706,
+	0xff162605, 0xff192503, 0x001b2302, 0x001d2201,
+	0x011f1f01, 0x01221d00, 0x02231b00, 0x04241800,
+	0x052616ff, 0x072713ff, 0x08271100, 0x0a280e00,
+
+	0xff0e260d, 0xff10260b, 0xff122609, 0xff142508,
+	0x00152506, 0x00182305, 0x001b2203, 0x011d2002,
+	0x011f1f01, 0x02201d01, 0x03221b00, 0x04231801,
+	0x06241600, 0x08251300, 0x09261100, 0x0b260f00,
+
+	0xff0e250e, 0xff10250c, 0x0011250a, 0x00142408,
+	0x00162307, 0x00182206, 0x011a2104, 0x011c2003,
+	0x021e1e02, 0x03201c01, 0x04211a01, 0x05221801,
+	0x07231600, 0x08241400, 0x0a241200, 0x0c241000,
+
+	0x000e240e, 0x0010240c, 0x0013230a, 0x00142309,
+	0x00162208, 0x01182106, 0x011a2005, 0x021b1f04,
+	0x031d1d03, 0x041e1c02, 0x05201a01, 0x06211801,
+	0x07221601, 0x09231400, 0x0a231300, 0x0c231100,
+
+	0x000f220f, 0x0011220d, 0x0013220b, 0x0015210a,
+	0x01162108, 0x01182007, 0x02191f06, 0x031a1e05,
+	0x041c1c04, 0x051d1b03, 0x061f1902, 0x07201801,
+	0x08211601, 0x0a211500, 0x0b221300, 0x0d221100,
+
+	0x0010210f, 0x0011210e, 0x0013210c, 0x0114200b,
+	0x01161f0a, 0x02171f08, 0x03181e07, 0x031a1d06,
+	0x041c1c04, 0x051d1a04, 0x071d1903, 0x081e1802,
+	0x091f1602, 0x0b1f1501, 0x0c211300, 0x0e201200,
+
+	0x00102010, 0x0012200e, 0x0013200d, 0x01151f0b,
+	0x01161f0a, 0x02171e09, 0x03191d07, 0x041a1c06,
+	0x051b1b05, 0x061c1a04, 0x071d1903, 0x081e1703,
+	0x0a1f1601, 0x0b1f1501, 0x0d201300, 0x0e201200,
+
+	0x00102010, 0x00121f0f, 0x00141f0d, 0x01141f0c,
+	0x02161e0a, 0x03171d09, 0x03181d08, 0x041a1c06,
+	0x051b1b05, 0x061c1a04, 0x081c1903, 0x091d1703,
+	0x0a1e1602, 0x0c1e1501, 0x0d1f1400, 0x0e1f1201,
+
+	0x00111e11, 0x00131e0f, 0x01131e0e, 0x02151d0c,
+	0x02161d0b, 0x03171c0a, 0x04181b09, 0x05191b07,
+	0x061a1a06, 0x071b1905, 0x091b1804, 0x0a1c1703,
+	0x0b1d1602, 0x0c1d1502, 0x0e1d1401, 0x0f1e1300,
+
+	0x00111e11, 0x00131d10, 0x01141d0e, 0x02151c0d,
+	0x03161c0b, 0x04171b0a, 0x05171b09, 0x06181a08,
+	0x07191907, 0x081a1806, 0x091a1805, 0x0a1b1704,
+	0x0b1c1603, 0x0d1c1502, 0x0e1d1401, 0x0f1d1301,
+};
+
+static int sun8i_ui_scaler_coef_index(unsigned int step)
+{
+	unsigned int scale, int_part, float_part;
+
+	scale = step >> (SUN8I_UI_SCALER_SCALE_FRAC - 3);
+	int_part = scale >> 3;
+	float_part = scale & 0x7;
+
+	switch (int_part) {
+	case 0:
+		return 0;
+	case 1:
+		return float_part;
+	case 2:
+		return 8 + (float_part >> 1);
+	case 3:
+		return 12;
+	case 4:
+		return 13;
+	default:
+		return 14;
+	}
+}
+
+void sun8i_ui_scaler_enable(struct sun8i_mixer *mixer, int layer, bool enable)
+{
+	int vi_cnt = mixer->cfg->vi_num;
+	u32 val;
+
+	if (WARN_ON(layer < vi_cnt))
+		return;
+
+	if (enable)
+		val = SUN8I_SCALER_GSU_CTRL_EN |
+		      SUN8I_SCALER_GSU_CTRL_COEFF_RDY;
+	else
+		val = 0;
+
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_GSU_CTRL(vi_cnt, layer - vi_cnt), val);
+}
+
+void sun8i_ui_scaler_setup(struct sun8i_mixer *mixer, int layer,
+			   u32 src_w, u32 src_h, u32 dst_w, u32 dst_h,
+			   u32 hscale, u32 vscale, u32 hphase, u32 vphase)
+{
+	int vi_cnt = mixer->cfg->vi_num;
+	u32 insize, outsize;
+	int i, offset;
+
+	if (WARN_ON(layer < vi_cnt))
+		return;
+
+	hphase <<= SUN8I_UI_SCALER_PHASE_FRAC - 16;
+	vphase <<= SUN8I_UI_SCALER_PHASE_FRAC - 16;
+	hscale <<= SUN8I_UI_SCALER_SCALE_FRAC - 16;
+	vscale <<= SUN8I_UI_SCALER_SCALE_FRAC - 16;
+
+	insize = SUN8I_UI_SCALER_SIZE(src_w, src_h);
+	outsize = SUN8I_UI_SCALER_SIZE(dst_w, dst_h);
+
+	layer -= vi_cnt;
+
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_GSU_OUTSIZE(vi_cnt, layer), outsize);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_GSU_INSIZE(vi_cnt, layer), insize);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_GSU_HSTEP(vi_cnt, layer), hscale);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_GSU_VSTEP(vi_cnt, layer), vscale);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_GSU_HPHASE(vi_cnt, layer), hphase);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_GSU_VPHASE(vi_cnt, layer), vphase);
+	offset = sun8i_ui_scaler_coef_index(hscale) *
+			SUN8I_UI_SCALER_COEFF_COUNT;
+	for (i = 0; i < SUN8I_UI_SCALER_COEFF_COUNT; i++)
+		regmap_write(mixer->engine.regs,
+			     SUN8I_SCALER_GSU_HCOEFF(vi_cnt, layer, i),
+			     lan2coefftab16[offset + i]);
+}
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_scaler.h b/drivers/gpu/drm/sun4i/sun8i_ui_scaler.h
new file mode 100644
index 000000000000..86295be8be78
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_scaler.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2017 Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#ifndef _SUN8I_UI_SCALER_H_
+#define _SUN8I_UI_SCALER_H_
+
+#include "sun8i_mixer.h"
+
+/* this two macros assumes 16 fractional bits which is standard in DRM */
+#define SUN8I_UI_SCALER_SCALE_MIN		1
+#define SUN8I_UI_SCALER_SCALE_MAX		((1UL << 20) - 1)
+
+#define SUN8I_UI_SCALER_SCALE_FRAC		20
+#define SUN8I_UI_SCALER_PHASE_FRAC		20
+#define SUN8I_UI_SCALER_COEFF_COUNT		16
+#define SUN8I_UI_SCALER_SIZE(w, h)		(((h) - 1) << 16 | ((w) - 1))
+
+#define SUN8I_SCALER_GSU_CTRL(vi_cnt, ui_idx) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x0)
+#define SUN8I_SCALER_GSU_OUTSIZE(vi_cnt, ui_idx) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x40)
+#define SUN8I_SCALER_GSU_INSIZE(vi_cnt, ui_idx) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x80)
+#define SUN8I_SCALER_GSU_HSTEP(vi_cnt, ui_idx) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x88)
+#define SUN8I_SCALER_GSU_VSTEP(vi_cnt, ui_idx) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x8c)
+#define SUN8I_SCALER_GSU_HPHASE(vi_cnt, ui_idx) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x90)
+#define SUN8I_SCALER_GSU_VPHASE(vi_cnt, ui_idx) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x98)
+#define SUN8I_SCALER_GSU_HCOEFF(vi_cnt, ui_idx, index) \
+	(0x20000 + 0x20000 * (vi_cnt) + 0x10000 * (ui_idx) + 0x200 + \
+	0x4 * (index))
+
+#define SUN8I_SCALER_GSU_CTRL_EN		BIT(0)
+#define SUN8I_SCALER_GSU_CTRL_COEFF_RDY		BIT(4)
+
+void sun8i_ui_scaler_enable(struct sun8i_mixer *mixer, int layer, bool enable);
+void sun8i_ui_scaler_setup(struct sun8i_mixer *mixer, int layer,
+			   u32 src_w, u32 src_h, u32 dst_w, u32 dst_h,
+			   u32 hscale, u32 vscale, u32 hphase, u32 vphase);
+
+#endif
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
new file mode 100644
index 000000000000..40c3b303068a
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
@@ -0,0 +1,390 @@
+/*
+ * Copyright (C) Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drmP.h>
+
+#include "sun8i_vi_layer.h"
+#include "sun8i_mixer.h"
+#include "sun8i_vi_scaler.h"
+
+static void sun8i_vi_layer_enable(struct sun8i_mixer *mixer, int channel,
+				  int overlay, bool enable)
+{
+	u32 val;
+
+	DRM_DEBUG_DRIVER("%sabling VI channel %d overlay %d\n",
+			 enable ? "En" : "Dis", channel, overlay);
+
+	if (enable)
+		val = SUN8I_MIXER_CHAN_VI_LAYER_ATTR_EN;
+	else
+		val = 0;
+
+	regmap_update_bits(mixer->engine.regs,
+			   SUN8I_MIXER_CHAN_VI_LAYER_ATTR(channel, overlay),
+			   SUN8I_MIXER_CHAN_VI_LAYER_ATTR_EN, val);
+
+	if (enable)
+		val = SUN8I_MIXER_BLEND_PIPE_CTL_EN(channel);
+	else
+		val = 0;
+
+	regmap_update_bits(mixer->engine.regs,
+			   SUN8I_MIXER_BLEND_PIPE_CTL,
+			   SUN8I_MIXER_BLEND_PIPE_CTL_EN(channel), val);
+}
+
+static int sun8i_vi_layer_update_coord(struct sun8i_mixer *mixer, int channel,
+				       int overlay, struct drm_plane *plane)
+{
+	struct drm_plane_state *state = plane->state;
+	const struct drm_format_info *format = state->fb->format;
+	u32 src_w, src_h, dst_w, dst_h;
+	u32 outsize, insize;
+	u32 hphase, vphase;
+	bool subsampled;
+
+	DRM_DEBUG_DRIVER("Updating VI channel %d overlay %d\n",
+			 channel, overlay);
+
+	src_w = drm_rect_width(&state->src) >> 16;
+	src_h = drm_rect_height(&state->src) >> 16;
+	dst_w = drm_rect_width(&state->dst);
+	dst_h = drm_rect_height(&state->dst);
+
+	hphase = state->src.x1 & 0xffff;
+	vphase = state->src.y1 & 0xffff;
+
+	/* make coordinates dividable by subsampling factor */
+	if (format->hsub > 1) {
+		int mask, remainder;
+
+		mask = format->hsub - 1;
+		remainder = (state->src.x1 >> 16) & mask;
+		src_w = (src_w + remainder) & ~mask;
+		hphase += remainder << 16;
+	}
+
+	if (format->vsub > 1) {
+		int mask, remainder;
+
+		mask = format->vsub - 1;
+		remainder = (state->src.y1 >> 16) & mask;
+		src_h = (src_h + remainder) & ~mask;
+		vphase += remainder << 16;
+	}
+
+	insize = SUN8I_MIXER_SIZE(src_w, src_h);
+	outsize = SUN8I_MIXER_SIZE(dst_w, dst_h);
+
+	/* Set height and width */
+	DRM_DEBUG_DRIVER("Layer source offset X: %d Y: %d\n",
+			 (state->src.x1 >> 16) & ~(format->hsub - 1),
+			 (state->src.y1 >> 16) & ~(format->vsub - 1));
+	DRM_DEBUG_DRIVER("Layer source size W: %d H: %d\n", src_w, src_h);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_VI_LAYER_SIZE(channel, overlay),
+		     insize);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_VI_OVL_SIZE(channel),
+		     insize);
+
+	/*
+	 * Scaler must be enabled for subsampled formats, so it scales
+	 * chroma to same size as luma.
+	 */
+	subsampled = format->hsub > 1 || format->vsub > 1;
+
+	if (insize != outsize || subsampled || hphase || vphase) {
+		u32 hscale, vscale;
+
+		DRM_DEBUG_DRIVER("HW scaling is enabled\n");
+
+		hscale = state->src_w / state->crtc_w;
+		vscale = state->src_h / state->crtc_h;
+
+		sun8i_vi_scaler_setup(mixer, channel, src_w, src_h, dst_w,
+				      dst_h, hscale, vscale, hphase, vphase,
+				      format);
+		sun8i_vi_scaler_enable(mixer, channel, true);
+	} else {
+		DRM_DEBUG_DRIVER("HW scaling is not needed\n");
+		sun8i_vi_scaler_enable(mixer, channel, false);
+	}
+
+	/* Set base coordinates */
+	DRM_DEBUG_DRIVER("Layer destination coordinates X: %d Y: %d\n",
+			 state->dst.x1, state->dst.y1);
+	DRM_DEBUG_DRIVER("Layer destination size W: %d H: %d\n", dst_w, dst_h);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_BLEND_ATTR_COORD(channel),
+		     SUN8I_MIXER_COORD(state->dst.x1, state->dst.y1));
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_BLEND_ATTR_INSIZE(channel),
+		     outsize);
+
+	return 0;
+}
+
+static int sun8i_vi_layer_update_formats(struct sun8i_mixer *mixer, int channel,
+					 int overlay, struct drm_plane *plane)
+{
+	struct drm_plane_state *state = plane->state;
+	const struct de2_fmt_info *fmt_info;
+	u32 val;
+
+	fmt_info = sun8i_mixer_format_info(state->fb->format->format);
+	if (!fmt_info) {
+		DRM_DEBUG_DRIVER("Invalid format\n");
+		return -EINVAL;
+	}
+
+	val = fmt_info->de2_fmt << SUN8I_MIXER_CHAN_VI_LAYER_ATTR_FBFMT_OFFSET;
+	regmap_update_bits(mixer->engine.regs,
+			   SUN8I_MIXER_CHAN_VI_LAYER_ATTR(channel, overlay),
+			   SUN8I_MIXER_CHAN_VI_LAYER_ATTR_FBFMT_MASK, val);
+
+	if (fmt_info->csc != SUN8I_CSC_MODE_OFF) {
+		sun8i_csc_set_ccsc_coefficients(mixer, channel, fmt_info->csc);
+		sun8i_csc_enable_ccsc(mixer, channel, true);
+	} else {
+		sun8i_csc_enable_ccsc(mixer, channel, false);
+	}
+
+	if (fmt_info->rgb)
+		val = SUN8I_MIXER_CHAN_VI_LAYER_ATTR_RGB_MODE;
+	else
+		val = 0;
+
+	regmap_update_bits(mixer->engine.regs,
+			   SUN8I_MIXER_CHAN_VI_LAYER_ATTR(channel, overlay),
+			   SUN8I_MIXER_CHAN_VI_LAYER_ATTR_RGB_MODE, val);
+
+	return 0;
+}
+
+static int sun8i_vi_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
+					int overlay, struct drm_plane *plane)
+{
+	struct drm_plane_state *state = plane->state;
+	struct drm_framebuffer *fb = state->fb;
+	const struct drm_format_info *format = fb->format;
+	struct drm_gem_cma_object *gem;
+	u32 dx, dy, src_x, src_y;
+	dma_addr_t paddr;
+	int i;
+
+	/* Adjust x and y to be dividable by subsampling factor */
+	src_x = (state->src.x1 >> 16) & ~(format->hsub - 1);
+	src_y = (state->src.y1 >> 16) & ~(format->vsub - 1);
+
+	for (i = 0; i < format->num_planes; i++) {
+		/* Get the physical address of the buffer in memory */
+		gem = drm_fb_cma_get_gem_obj(fb, i);
+
+		DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->paddr);
+
+		/* Compute the start of the displayed memory */
+		paddr = gem->paddr + fb->offsets[i];
+
+		dx = src_x;
+		dy = src_y;
+
+		if (i > 0) {
+			dx /= format->hsub;
+			dy /= format->vsub;
+		}
+
+		/* Fixup framebuffer address for src coordinates */
+		paddr += dx * format->cpp[i];
+		paddr += dy * fb->pitches[i];
+
+		/* Set the line width */
+		DRM_DEBUG_DRIVER("Layer %d. line width: %d bytes\n",
+				 i + 1, fb->pitches[i]);
+		regmap_write(mixer->engine.regs,
+			     SUN8I_MIXER_CHAN_VI_LAYER_PITCH(channel,
+							     overlay, i),
+	       fb->pitches[i]);
+
+		DRM_DEBUG_DRIVER("Setting %d. buffer address to %pad\n",
+				 i + 1, &paddr);
+
+		regmap_write(mixer->engine.regs,
+			     SUN8I_MIXER_CHAN_VI_LAYER_TOP_LADDR(channel,
+								 overlay, i),
+	       lower_32_bits(paddr));
+	}
+
+	return 0;
+}
+
+static int sun8i_vi_layer_atomic_check(struct drm_plane *plane,
+				       struct drm_plane_state *state)
+{
+	struct sun8i_vi_layer *layer = plane_to_sun8i_vi_layer(plane);
+	struct drm_crtc *crtc = state->crtc;
+	struct drm_crtc_state *crtc_state;
+	int min_scale, max_scale;
+	struct drm_rect clip;
+
+	if (!crtc)
+		return 0;
+
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state, crtc);
+	if (WARN_ON(!crtc_state))
+		return -EINVAL;
+
+	clip.x1 = 0;
+	clip.y1 = 0;
+	clip.x2 = crtc_state->adjusted_mode.hdisplay;
+	clip.y2 = crtc_state->adjusted_mode.vdisplay;
+
+	min_scale = DRM_PLANE_HELPER_NO_SCALING;
+	max_scale = DRM_PLANE_HELPER_NO_SCALING;
+
+	if (layer->mixer->cfg->scaler_mask & BIT(layer->channel)) {
+		min_scale = SUN8I_VI_SCALER_SCALE_MIN;
+		max_scale = SUN8I_VI_SCALER_SCALE_MAX;
+	}
+
+	return drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						   min_scale, max_scale,
+						   true, true);
+}
+
+static void sun8i_vi_layer_atomic_disable(struct drm_plane *plane,
+					  struct drm_plane_state *old_state)
+{
+	struct sun8i_vi_layer *layer = plane_to_sun8i_vi_layer(plane);
+	struct sun8i_mixer *mixer = layer->mixer;
+
+	sun8i_vi_layer_enable(mixer, layer->channel, layer->overlay, false);
+}
+
+static void sun8i_vi_layer_atomic_update(struct drm_plane *plane,
+					 struct drm_plane_state *old_state)
+{
+	struct sun8i_vi_layer *layer = plane_to_sun8i_vi_layer(plane);
+	struct sun8i_mixer *mixer = layer->mixer;
+
+	if (!plane->state->visible) {
+		sun8i_vi_layer_enable(mixer, layer->channel,
+				      layer->overlay, false);
+		return;
+	}
+
+	sun8i_vi_layer_update_coord(mixer, layer->channel,
+				    layer->overlay, plane);
+	sun8i_vi_layer_update_formats(mixer, layer->channel,
+				      layer->overlay, plane);
+	sun8i_vi_layer_update_buffer(mixer, layer->channel,
+				     layer->overlay, plane);
+	sun8i_vi_layer_enable(mixer, layer->channel, layer->overlay, true);
+}
+
+static struct drm_plane_helper_funcs sun8i_vi_layer_helper_funcs = {
+	.atomic_check	= sun8i_vi_layer_atomic_check,
+	.atomic_disable	= sun8i_vi_layer_atomic_disable,
+	.atomic_update	= sun8i_vi_layer_atomic_update,
+};
+
+static const struct drm_plane_funcs sun8i_vi_layer_funcs = {
+	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
+	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.destroy		= drm_plane_cleanup,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.reset			= drm_atomic_helper_plane_reset,
+	.update_plane		= drm_atomic_helper_update_plane,
+};
+
+/*
+ * While all RGB formats are supported, VI planes don't support
+ * alpha blending, so there is no point having formats with alpha
+ * channel if their opaque analog exist.
+ */
+static const u32 sun8i_vi_layer_formats[] = {
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_BGRA4444,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_RGBA4444,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_XRGB8888,
+
+	DRM_FORMAT_NV16,
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_NV21,
+	DRM_FORMAT_NV61,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_YUV411,
+	DRM_FORMAT_YUV420,
+	DRM_FORMAT_YUV422,
+	DRM_FORMAT_YUV444,
+	DRM_FORMAT_YVU411,
+	DRM_FORMAT_YVU420,
+	DRM_FORMAT_YVU422,
+	DRM_FORMAT_YVU444,
+};
+
+struct sun8i_vi_layer *sun8i_vi_layer_init_one(struct drm_device *drm,
+					       struct sun8i_mixer *mixer,
+					       int index)
+{
+	struct sun8i_vi_layer *layer;
+	int ret;
+
+	layer = devm_kzalloc(drm->dev, sizeof(*layer), GFP_KERNEL);
+	if (!layer)
+		return ERR_PTR(-ENOMEM);
+
+	/* possible crtcs are set later */
+	ret = drm_universal_plane_init(drm, &layer->plane, 0,
+				       &sun8i_vi_layer_funcs,
+				       sun8i_vi_layer_formats,
+				       ARRAY_SIZE(sun8i_vi_layer_formats),
+				       NULL, DRM_PLANE_TYPE_OVERLAY, NULL);
+	if (ret) {
+		dev_err(drm->dev, "Couldn't initialize layer\n");
+		return ERR_PTR(ret);
+	}
+
+	/* fixed zpos for now */
+	ret = drm_plane_create_zpos_immutable_property(&layer->plane, index);
+	if (ret) {
+		dev_err(drm->dev, "Couldn't add zpos property\n");
+		return ERR_PTR(ret);
+	}
+
+	drm_plane_helper_add(&layer->plane, &sun8i_vi_layer_helper_funcs);
+	layer->mixer = mixer;
+	layer->channel = index;
+	layer->overlay = 0;
+
+	return layer;
+}
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.h b/drivers/gpu/drm/sun4i/sun8i_vi_layer.h
new file mode 100644
index 000000000000..6996627a0a76
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ */
+
+#ifndef _SUN8I_VI_LAYER_H_
+#define _SUN8I_VI_LAYER_H_
+
+#include <drm/drm_plane.h>
+
+#define SUN8I_MIXER_CHAN_VI_LAYER_ATTR(ch, layer) \
+		(0x2000 + 0x1000 * (ch) + 0x30 * (layer) + 0x0)
+#define SUN8I_MIXER_CHAN_VI_LAYER_SIZE(ch, layer) \
+		(0x2000 + 0x1000 * (ch) + 0x30 * (layer) + 0x4)
+#define SUN8I_MIXER_CHAN_VI_LAYER_COORD(ch, layer) \
+		(0x2000 + 0x1000 * (ch) + 0x30 * (layer) + 0x8)
+#define SUN8I_MIXER_CHAN_VI_LAYER_PITCH(ch, layer, plane) \
+		(0x2000 + 0x1000 * (ch) + 0x30 * (layer) + 0xc + 4 * (plane))
+#define SUN8I_MIXER_CHAN_VI_LAYER_TOP_LADDR(ch, layer, plane) \
+		(0x2000 + 0x1000 * (ch) + 0x30 * (layer) + 0x18 + 4 * (plane))
+#define SUN8I_MIXER_CHAN_VI_OVL_SIZE(ch)	(0x2000 + 0x1000 * (ch) + 0xe8)
+
+#define SUN8I_MIXER_CHAN_VI_LAYER_ATTR_EN		BIT(0)
+/* RGB mode should be set for RGB formats and cleared for YCbCr */
+#define SUN8I_MIXER_CHAN_VI_LAYER_ATTR_RGB_MODE		BIT(15)
+#define SUN8I_MIXER_CHAN_VI_LAYER_ATTR_FBFMT_OFFSET	8
+#define SUN8I_MIXER_CHAN_VI_LAYER_ATTR_FBFMT_MASK	GENMASK(12, 8)
+
+struct sun8i_mixer;
+
+struct sun8i_vi_layer {
+	struct drm_plane	plane;
+	struct sun8i_mixer	*mixer;
+	int			channel;
+	int			overlay;
+};
+
+static inline struct sun8i_vi_layer *
+plane_to_sun8i_vi_layer(struct drm_plane *plane)
+{
+	return container_of(plane, struct sun8i_vi_layer, plane);
+}
+
+struct sun8i_vi_layer *sun8i_vi_layer_init_one(struct drm_device *drm,
+					       struct sun8i_mixer *mixer,
+					       int index);
+#endif /* _SUN8I_VI_LAYER_H_ */
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c
new file mode 100644
index 000000000000..d3f1acb234b7
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c
@@ -0,0 +1,971 @@
+/*
+ * Copyright (C) 2017 Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * Coefficients are taken from BSP driver, which is:
+ * Copyright (C) 2014-2015 Allwinner
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#include "sun8i_vi_scaler.h"
+
+static const u32 lan3coefftab32_left[480] = {
+	0x40000000, 0x40fe0000, 0x3ffd0100, 0x3efc0100,
+	0x3efb0100, 0x3dfa0200, 0x3cf90200, 0x3bf80200,
+	0x39f70200, 0x37f70200, 0x35f70200, 0x33f70200,
+	0x31f70200, 0x2ef70200, 0x2cf70200, 0x2af70200,
+	0x27f70200, 0x24f80100, 0x22f80100, 0x1ef90100,
+	0x1cf90100, 0x19fa0100, 0x17fa0100, 0x14fb0100,
+	0x11fc0000, 0x0ffc0000, 0x0cfd0000, 0x0afd0000,
+	0x08fe0000, 0x05ff0000, 0x03ff0000, 0x02000000,
+
+	0x3806fc02, 0x3805fc02, 0x3803fd01, 0x3801fe01,
+	0x3700fe01, 0x35ffff01, 0x35fdff01, 0x34fc0001,
+	0x34fb0000, 0x33fa0000, 0x31fa0100, 0x2ff90100,
+	0x2df80200, 0x2bf80200, 0x2af70200, 0x28f70200,
+	0x27f70200, 0x24f70300, 0x22f70300, 0x1ff70300,
+	0x1ef70300, 0x1cf70300, 0x1af70300, 0x18f70300,
+	0x16f80300, 0x13f80300, 0x11f90300, 0x0ef90300,
+	0x0efa0200, 0x0cfa0200, 0x0afb0200, 0x08fb0200,
+
+	0x320bfa02, 0x3309fa02, 0x3208fb02, 0x3206fb02,
+	0x3205fb02, 0x3104fc02, 0x3102fc01, 0x3001fd01,
+	0x3000fd01, 0x2ffffd01, 0x2efefe01, 0x2dfdfe01,
+	0x2bfcff01, 0x29fcff01, 0x28fbff01, 0x27fa0001,
+	0x26fa0000, 0x24f90000, 0x22f90100, 0x20f90100,
+	0x1ff80100, 0x1ef80100, 0x1cf80100, 0x1af80200,
+	0x18f80200, 0x17f80200, 0x15f80200, 0x12f80200,
+	0x11f90200, 0x0ff90200, 0x0df90200, 0x0cfa0200,
+
+	0x2e0efa01, 0x2f0dfa01, 0x2f0bfa01, 0x2e0afa01,
+	0x2e09fa01, 0x2e07fb01, 0x2d06fb01, 0x2d05fb01,
+	0x2c04fb01, 0x2b03fc01, 0x2a02fc01, 0x2a01fc01,
+	0x2800fd01, 0x28fffd01, 0x26fefd01, 0x25fefe01,
+	0x24fdfe01, 0x23fcfe01, 0x21fcff01, 0x20fbff01,
+	0x1efbff01, 0x1efbff00, 0x1cfa0000, 0x1bfa0000,
+	0x19fa0000, 0x18fa0000, 0x17f90000, 0x15f90100,
+	0x14f90100, 0x12f90100, 0x11f90100, 0x0ff90100,
+
+	0x2b10fa00, 0x2b0ffa00, 0x2b0efa00, 0x2b0cfa00,
+	0x2b0bfa00, 0x2a0afb01, 0x2a09fb01, 0x2908fb01,
+	0x2807fb01, 0x2806fb01, 0x2805fb01, 0x2604fc01,
+	0x2503fc01, 0x2502fc01, 0x2401fc01, 0x2301fc01,
+	0x2100fd01, 0x21fffd01, 0x21fffd01, 0x20fefd01,
+	0x1dfefe01, 0x1cfdfe01, 0x1cfdfe00, 0x1bfcfe00,
+	0x19fcff00, 0x19fbff00, 0x17fbff00, 0x16fbff00,
+	0x15fbff00, 0x14fb0000, 0x13fa0000, 0x11fa0000,
+
+	0x2811fcff, 0x2810fcff, 0x280ffbff, 0x280efbff,
+	0x270dfb00, 0x270cfb00, 0x270bfb00, 0x260afb00,
+	0x2609fb00, 0x2508fb00, 0x2507fb00, 0x2407fb00,
+	0x2406fc00, 0x2305fc00, 0x2204fc00, 0x2203fc00,
+	0x2103fc00, 0x2002fc00, 0x1f01fd00, 0x1e01fd00,
+	0x1d00fd00, 0x1dfffd00, 0x1cfffd00, 0x1bfefd00,
+	0x1afefe00, 0x19fefe00, 0x18fdfe00, 0x17fdfe00,
+	0x16fdfe00, 0x15fcff00, 0x13fcff00, 0x12fcff00,
+
+	0x2512fdfe, 0x2511fdff, 0x2410fdff, 0x240ffdff,
+	0x240efcff, 0x240dfcff, 0x240dfcff, 0x240cfcff,
+	0x230bfcff, 0x230afc00, 0x2209fc00, 0x2108fc00,
+	0x2108fc00, 0x2007fc00, 0x2006fc00, 0x2005fc00,
+	0x1f05fc00, 0x1e04fc00, 0x1e03fc00, 0x1c03fd00,
+	0x1c02fd00, 0x1b02fd00, 0x1b01fd00, 0x1a00fd00,
+	0x1900fd00, 0x1800fd00, 0x17fffe00, 0x16fffe00,
+	0x16fefe00, 0x14fefe00, 0x13fefe00, 0x13fdfe00,
+
+	0x2212fffe, 0x2211fefe, 0x2211fefe, 0x2110fefe,
+	0x210ffeff, 0x220efdff, 0x210dfdff, 0x210dfdff,
+	0x210cfdff, 0x210bfdff, 0x200afdff, 0x200afdff,
+	0x1f09fdff, 0x1f08fdff, 0x1d08fd00, 0x1c07fd00,
+	0x1d06fd00, 0x1b06fd00, 0x1b05fd00, 0x1c04fd00,
+	0x1b04fd00, 0x1a03fd00, 0x1a03fd00, 0x1902fd00,
+	0x1802fd00, 0x1801fd00, 0x1701fd00, 0x1600fd00,
+	0x1400fe00, 0x1400fe00, 0x14fffe00, 0x13fffe00,
+
+	0x201200fe, 0x201100fe, 0x1f11fffe, 0x2010fffe,
+	0x1f0ffffe, 0x1e0ffffe, 0x1f0efeff, 0x1f0dfeff,
+	0x1f0dfeff, 0x1e0cfeff, 0x1e0bfeff, 0x1d0bfeff,
+	0x1d0afeff, 0x1d09fdff, 0x1d09fdff, 0x1c08fdff,
+	0x1c07fdff, 0x1b07fd00, 0x1b06fd00, 0x1a06fd00,
+	0x1a05fd00, 0x1805fd00, 0x1904fd00, 0x1804fd00,
+	0x1703fd00, 0x1703fd00, 0x1602fe00, 0x1502fe00,
+	0x1501fe00, 0x1401fe00, 0x1301fe00, 0x1300fe00,
+
+	0x1c1202fe, 0x1c1102fe, 0x1b1102fe, 0x1c1001fe,
+	0x1b1001fe, 0x1b0f01ff, 0x1b0e00ff, 0x1b0e00ff,
+	0x1b0d00ff, 0x1a0d00ff, 0x1a0c00ff, 0x1a0cffff,
+	0x1a0bffff, 0x1a0bffff, 0x1a0affff, 0x180affff,
+	0x1909ffff, 0x1809ffff, 0x1808ffff, 0x1808feff,
+	0x1807feff, 0x1707fe00, 0x1606fe00, 0x1506fe00,
+	0x1605fe00, 0x1505fe00, 0x1504fe00, 0x1304fe00,
+	0x1304fe00, 0x1303fe00, 0x1203fe00, 0x1203fe00,
+
+	0x181104ff, 0x191103ff, 0x191003ff, 0x181003ff,
+	0x180f03ff, 0x190f02ff, 0x190e02ff, 0x180e02ff,
+	0x180d02ff, 0x180d01ff, 0x180d01ff, 0x180c01ff,
+	0x180c01ff, 0x180b00ff, 0x170b00ff, 0x170a00ff,
+	0x170a00ff, 0x170900ff, 0x160900ff, 0x160900ff,
+	0x1608ffff, 0x1508ffff, 0x1507ff00, 0x1507ff00,
+	0x1407ff00, 0x1306ff00, 0x1306ff00, 0x1305ff00,
+	0x1205ff00, 0x1105ff00, 0x1204ff00, 0x1104ff00,
+
+	0x171005ff, 0x171005ff, 0x171004ff, 0x170f04ff,
+	0x160f04ff, 0x170f03ff, 0x170e03ff, 0x160e03ff,
+	0x160d03ff, 0x160d02ff, 0x160d02ff, 0x160c02ff,
+	0x160c02ff, 0x160c02ff, 0x160b01ff, 0x150b01ff,
+	0x150a01ff, 0x150a01ff, 0x150a01ff, 0x140901ff,
+	0x14090000, 0x14090000, 0x14080000, 0x13080000,
+	0x13070000, 0x12070000, 0x12070000, 0x12060000,
+	0x11060000, 0x11060000, 0x11050000, 0x1105ff00,
+
+	0x14100600, 0x15100500, 0x150f0500, 0x150f0500,
+	0x140f0500, 0x150e0400, 0x140e0400, 0x130e0400,
+	0x140d0400, 0x150d0300, 0x130d0300, 0x140c0300,
+	0x140c0300, 0x140c0200, 0x140b0200, 0x130b0200,
+	0x120b0200, 0x130a0200, 0x130a0200, 0x130a0100,
+	0x13090100, 0x12090100, 0x11090100, 0x12080100,
+	0x11080100, 0x10080100, 0x11070100, 0x11070000,
+	0x10070000, 0x11060000, 0x10060000, 0x10060000,
+
+	0x140f0600, 0x140f0600, 0x130f0600, 0x140f0500,
+	0x140e0500, 0x130e0500, 0x130e0500, 0x140d0400,
+	0x140d0400, 0x130d0400, 0x120d0400, 0x130c0400,
+	0x130c0300, 0x130c0300, 0x130b0300, 0x130b0300,
+	0x110b0300, 0x130a0200, 0x120a0200, 0x120a0200,
+	0x120a0200, 0x12090200, 0x10090200, 0x11090100,
+	0x11080100, 0x11080100, 0x10080100, 0x10080100,
+	0x10070100, 0x10070100, 0x0f070100, 0x10060100,
+
+	0x120f0701, 0x130f0601, 0x130e0601, 0x130e0601,
+	0x120e0601, 0x130e0501, 0x130e0500, 0x130d0500,
+	0x120d0500, 0x120d0500, 0x130c0400, 0x130c0400,
+	0x120c0400, 0x110c0400, 0x120b0400, 0x120b0300,
+	0x120b0300, 0x120b0300, 0x120a0300, 0x110a0300,
+	0x110a0200, 0x11090200, 0x11090200, 0x10090200,
+	0x10090200, 0x10080200, 0x10080200, 0x10080100,
+	0x0f080100, 0x10070100, 0x0f070100, 0x0f070100
+};
+
+static const u32 lan3coefftab32_right[480] = {
+	0x00000000, 0x00000002, 0x0000ff04, 0x0000ff06,
+	0x0000fe08, 0x0000fd0a, 0x0000fd0c, 0x0000fc0f,
+	0x0000fc12, 0x0001fb14, 0x0001fa17, 0x0001fa19,
+	0x0001f91c, 0x0001f91f, 0x0001f822, 0x0001f824,
+	0x0002f727, 0x0002f72a, 0x0002f72c, 0x0002f72f,
+	0x0002f731, 0x0002f733, 0x0002f735, 0x0002f737,
+	0x0002f73a, 0x0002f83b, 0x0002f93c, 0x0002fa3d,
+	0x0001fb3e, 0x0001fc3f, 0x0001fd40, 0x0000fe40,
+
+	0x0002fc06, 0x0002fb08, 0x0002fb0a, 0x0002fa0c,
+	0x0002fa0e, 0x0003f910, 0x0003f912, 0x0003f814,
+	0x0003f816, 0x0003f719, 0x0003f71a, 0x0003f71d,
+	0x0003f71f, 0x0003f721, 0x0003f723, 0x0003f725,
+	0x0002f727, 0x0002f729, 0x0002f72b, 0x0002f82d,
+	0x0002f82e, 0x0001f930, 0x0001fa31, 0x0000fa34,
+	0x0000fb34, 0x0100fc35, 0x01fffd36, 0x01ffff37,
+	0x01fe0037, 0x01fe0138, 0x01fd0338, 0x02fc0538,
+
+	0x0002fa0b, 0x0002fa0c, 0x0002f90e, 0x0002f910,
+	0x0002f911, 0x0002f813, 0x0002f816, 0x0002f817,
+	0x0002f818, 0x0002f81a, 0x0001f81c, 0x0001f81e,
+	0x0001f820, 0x0001f921, 0x0001f923, 0x0000f925,
+	0x0000fa26, 0x0100fa28, 0x01fffb29, 0x01fffc2a,
+	0x01fffc2c, 0x01fefd2d, 0x01fefe2e, 0x01fdff2f,
+	0x01fd0030, 0x01fd0130, 0x01fc0232, 0x02fc0432,
+	0x02fb0532, 0x02fb0633, 0x02fb0833, 0x02fa0933,
+
+	0x0001fa0e, 0x0001f90f, 0x0001f911, 0x0001f913,
+	0x0001f914, 0x0001f915, 0x0000f918, 0x0000fa18,
+	0x0000fa1a, 0x0000fa1b, 0x0000fa1d, 0x00fffb1e,
+	0x01fffb1f, 0x01fffb20, 0x01fffc22, 0x01fefc23,
+	0x01fefd24, 0x01fefe25, 0x01fdfe27, 0x01fdff28,
+	0x01fd0029, 0x01fc012a, 0x01fc022b, 0x01fc032b,
+	0x01fb042d, 0x01fb052d, 0x01fb062e, 0x01fb072e,
+	0x01fa092e, 0x01fa0a2f, 0x01fa0b2f, 0x01fa0d2f,
+
+	0x0000fa11, 0x0000fa12, 0x0000fa13, 0x0000fb14,
+	0x00fffb16, 0x00fffb16, 0x00fffb17, 0x00fffb19,
+	0x00fffc1a, 0x00fefc1c, 0x00fefd1c, 0x01fefd1d,
+	0x01fefe1e, 0x01fdfe20, 0x01fdff21, 0x01fdff22,
+	0x01fd0023, 0x01fc0124, 0x01fc0124, 0x01fc0225,
+	0x01fc0326, 0x01fc0427, 0x01fb0528, 0x01fb0629,
+	0x01fb0729, 0x01fb0829, 0x01fb092a, 0x01fb0a2a,
+	0x00fa0b2c, 0x00fa0c2b, 0x00fa0e2b, 0x00fa0f2c,
+
+	0x00fffc11, 0x00fffc12, 0x00fffc14, 0x00fffc15,
+	0x00fefd16, 0x00fefd17, 0x00fefd18, 0x00fefe19,
+	0x00fefe1a, 0x00fdfe1d, 0x00fdff1d, 0x00fdff1e,
+	0x00fd001d, 0x00fd011e, 0x00fd0120, 0x00fc0221,
+	0x00fc0321, 0x00fc0323, 0x00fc0423, 0x00fc0523,
+	0x00fc0624, 0x00fb0725, 0x00fb0726, 0x00fb0827,
+	0x00fb0926, 0x00fb0a26, 0x00fb0b27, 0x00fb0c27,
+	0x00fb0d27, 0xfffb0e28, 0xfffb0f29, 0xfffc1028,
+
+	0x00fefd13, 0x00fefd13, 0x00fefe14, 0x00fefe15,
+	0x00fefe17, 0x00feff17, 0x00feff17, 0x00fd0018,
+	0x00fd001a, 0x00fd001a, 0x00fd011b, 0x00fd021c,
+	0x00fd021c, 0x00fd031d, 0x00fc031f, 0x00fc041f,
+	0x00fc051f, 0x00fc0521, 0x00fc0621, 0x00fc0721,
+	0x00fc0821, 0x00fc0822, 0x00fc0922, 0x00fc0a23,
+	0xfffc0b24, 0xfffc0c24, 0xfffc0d24, 0xfffc0d25,
+	0xfffc0e25, 0xfffd0f25, 0xfffd1025, 0xfffd1125,
+
+	0x00feff12, 0x00feff14, 0x00feff14, 0x00fe0015,
+	0x00fe0015, 0x00fd0017, 0x00fd0118, 0x00fd0118,
+	0x00fd0218, 0x00fd0219, 0x00fd031a, 0x00fd031a,
+	0x00fd041b, 0x00fd041c, 0x00fd051c, 0x00fd061d,
+	0x00fd061d, 0x00fd071e, 0x00fd081e, 0xfffd081f,
+	0xfffd091f, 0xfffd0a20, 0xfffd0a20, 0xfffd0b21,
+	0xfffd0c21, 0xfffd0d21, 0xfffd0d22, 0xfffd0e23,
+	0xfffe0f22, 0xfefe1022, 0xfefe1122, 0xfefe1123,
+
+	0x00fe0012, 0x00fe0013, 0x00fe0114, 0x00fe0114,
+	0x00fe0116, 0x00fe0216, 0x00fe0216, 0x00fd0317,
+	0x00fd0317, 0x00fd0418, 0x00fd0419, 0x00fd0519,
+	0x00fd051a, 0x00fd061b, 0x00fd061b, 0x00fd071c,
+	0xfffd071e, 0xfffd081d, 0xfffd091d, 0xfffd091e,
+	0xfffe0a1d, 0xfffe0b1e, 0xfffe0b1e, 0xfffe0c1e,
+	0xfffe0d1f, 0xfffe0d1f, 0xfffe0e1f, 0xfeff0f1f,
+	0xfeff0f20, 0xfeff1020, 0xfeff1120, 0xfe001120,
+
+	0x00fe0212, 0x00fe0312, 0x00fe0313, 0x00fe0314,
+	0x00fe0414, 0x00fe0414, 0x00fe0416, 0x00fe0515,
+	0x00fe0516, 0x00fe0616, 0x00fe0617, 0x00fe0717,
+	0xfffe0719, 0xfffe0818, 0xffff0818, 0xffff0919,
+	0xffff0919, 0xffff0a19, 0xffff0a1a, 0xffff0b1a,
+	0xffff0b1b, 0xffff0c1a, 0xff000c1b, 0xff000d1b,
+	0xff000d1b, 0xff000e1b, 0xff000e1c, 0xff010f1c,
+	0xfe01101c, 0xfe01101d, 0xfe02111c, 0xfe02111c,
+
+	0x00ff0411, 0x00ff0411, 0x00ff0412, 0x00ff0512,
+	0x00ff0513, 0x00ff0513, 0x00ff0613, 0x00ff0614,
+	0x00ff0714, 0x00ff0715, 0x00ff0715, 0xffff0816,
+	0xffff0816, 0xff000916, 0xff000917, 0xff000918,
+	0xff000a17, 0xff000a18, 0xff000b18, 0xff000b18,
+	0xff010c18, 0xff010c19, 0xff010d18, 0xff010d18,
+	0xff020d18, 0xff020e19, 0xff020e19, 0xff020f19,
+	0xff030f19, 0xff031019, 0xff031019, 0xff031119,
+
+	0x00ff0511, 0x00ff0511, 0x00000511, 0x00000611,
+	0x00000612, 0x00000612, 0x00000712, 0x00000713,
+	0x00000714, 0x00000814, 0x00000814, 0x00000914,
+	0x00000914, 0xff010914, 0xff010a15, 0xff010a16,
+	0xff010a17, 0xff010b16, 0xff010b16, 0xff020c16,
+	0xff020c16, 0xff020c16, 0xff020d16, 0xff020d17,
+	0xff030d17, 0xff030e17, 0xff030e17, 0xff030f17,
+	0xff040f17, 0xff040f17, 0xff041017, 0xff051017,
+
+	0x00000610, 0x00000610, 0x00000611, 0x00000611,
+	0x00000711, 0x00000712, 0x00010712, 0x00010812,
+	0x00010812, 0x00010812, 0x00010913, 0x00010913,
+	0x00010913, 0x00010a13, 0x00020a13, 0x00020a14,
+	0x00020b14, 0x00020b14, 0x00020b14, 0x00020c14,
+	0x00030c14, 0x00030c15, 0x00030d15, 0x00030d15,
+	0x00040d15, 0x00040e15, 0x00040e15, 0x00040e16,
+	0x00050f15, 0x00050f15, 0x00050f16, 0x00051015,
+
+	0x00000611, 0x00010610, 0x00010710, 0x00010710,
+	0x00010711, 0x00010811, 0x00010811, 0x00010812,
+	0x00010812, 0x00010912, 0x00020912, 0x00020912,
+	0x00020a12, 0x00020a12, 0x00020a13, 0x00020a13,
+	0x00030b13, 0x00030b13, 0x00030b14, 0x00030c13,
+	0x00030c13, 0x00040c13, 0x00040d14, 0x00040d14,
+	0x00040d15, 0x00040d15, 0x00050e14, 0x00050e14,
+	0x00050e15, 0x00050f14, 0x00060f14, 0x00060f14,
+
+	0x0001070f, 0x0001070f, 0x00010710, 0x00010710,
+	0x00010810, 0x00010810, 0x00020810, 0x00020811,
+	0x00020911, 0x00020911, 0x00020912, 0x00020912,
+	0x00020a12, 0x00030a12, 0x00030a12, 0x00030b12,
+	0x00030b12, 0x00030b12, 0x00040b12, 0x00040c12,
+	0x00040c13, 0x00040c14, 0x00040c14, 0x00050d13,
+	0x00050d13, 0x00050d14, 0x00050e13, 0x01050e13,
+	0x01060e13, 0x01060e13, 0x01060e14, 0x01060f13
+};
+
+static const u32 lan2coefftab32[480] = {
+	0x00004000, 0x000140ff, 0x00033ffe, 0x00043ffd,
+	0x00063efc, 0xff083dfc, 0x000a3bfb, 0xff0d39fb,
+	0xff0f37fb, 0xff1136fa, 0xfe1433fb, 0xfe1631fb,
+	0xfd192ffb, 0xfd1c2cfb, 0xfd1f29fb, 0xfc2127fc,
+	0xfc2424fc, 0xfc2721fc, 0xfb291ffd, 0xfb2c1cfd,
+	0xfb2f19fd, 0xfb3116fe, 0xfb3314fe, 0xfa3611ff,
+	0xfb370fff, 0xfb390dff, 0xfb3b0a00, 0xfc3d08ff,
+	0xfc3e0600, 0xfd3f0400, 0xfe3f0300, 0xff400100,
+
+	0xff053804, 0xff063803, 0xff083801, 0xff093701,
+	0xff0a3700, 0xff0c3500, 0xff0e34ff, 0xff1033fe,
+	0xff1232fd, 0xfe1431fd, 0xfe162ffd, 0xfe182dfd,
+	0xfd1b2cfc, 0xfd1d2afc, 0xfd1f28fc, 0xfd2126fc,
+	0xfd2323fd, 0xfc2621fd, 0xfc281ffd, 0xfc2a1dfd,
+	0xfc2c1bfd, 0xfd2d18fe, 0xfd2f16fe, 0xfd3114fe,
+	0xfd3212ff, 0xfe3310ff, 0xff340eff, 0x00350cff,
+	0x00360a00, 0x01360900, 0x02370700, 0x03370600,
+
+	0xff083207, 0xff093206, 0xff0a3205, 0xff0c3203,
+	0xff0d3103, 0xff0e3102, 0xfe113001, 0xfe132f00,
+	0xfe142e00, 0xfe162dff, 0xfe182bff, 0xfe192aff,
+	0xfe1b29fe, 0xfe1d27fe, 0xfe1f25fe, 0xfd2124fe,
+	0xfe2222fe, 0xfe2421fd, 0xfe251ffe, 0xfe271dfe,
+	0xfe291bfe, 0xff2a19fe, 0xff2b18fe, 0xff2d16fe,
+	0x002e14fe, 0x002f12ff, 0x013010ff, 0x02300fff,
+	0x03310dff, 0x04310cff, 0x05310a00, 0x06310900,
+
+	0xff0a2e09, 0xff0b2e08, 0xff0c2e07, 0xff0e2d06,
+	0xff0f2d05, 0xff102d04, 0xff122c03, 0xfe142c02,
+	0xfe152b02, 0xfe172a01, 0xfe182901, 0xfe1a2800,
+	0xfe1b2700, 0xfe1d2500, 0xff1e24ff, 0xfe2023ff,
+	0xff2121ff, 0xff2320fe, 0xff241eff, 0x00251dfe,
+	0x00261bff, 0x00281afe, 0x012818ff, 0x012a16ff,
+	0x022a15ff, 0x032b13ff, 0x032c12ff, 0x052c10ff,
+	0x052d0fff, 0x062d0d00, 0x072d0c00, 0x082d0b00,
+
+	0xff0c2a0b, 0xff0d2a0a, 0xff0e2a09, 0xff0f2a08,
+	0xff102a07, 0xff112a06, 0xff132905, 0xff142904,
+	0xff162803, 0xff172703, 0xff182702, 0xff1a2601,
+	0xff1b2501, 0xff1c2401, 0xff1e2300, 0xff1f2200,
+	0x00202000, 0x00211f00, 0x01221d00, 0x01231c00,
+	0x01251bff, 0x02251aff, 0x032618ff, 0x032717ff,
+	0x042815ff, 0x052814ff, 0x052913ff, 0x06291100,
+	0x072a10ff, 0x082a0e00, 0x092a0d00, 0x0a2a0c00,
+
+	0xff0d280c, 0xff0e280b, 0xff0f280a, 0xff102809,
+	0xff112808, 0xff122708, 0xff142706, 0xff152705,
+	0xff162605, 0xff172604, 0xff192503, 0xff1a2403,
+	0x001b2302, 0x001c2202, 0x001d2201, 0x001e2101,
+	0x011f1f01, 0x01211e00, 0x01221d00, 0x02221c00,
+	0x02231b00, 0x03241900, 0x04241800, 0x04251700,
+	0x052616ff, 0x06261400, 0x072713ff, 0x08271100,
+	0x08271100, 0x09271000, 0x0a280e00, 0x0b280d00,
+
+	0xff0e260d, 0xff0f260c, 0xff10260b, 0xff11260a,
+	0xff122609, 0xff132608, 0xff142508, 0xff152507,
+	0x00152506, 0x00172405, 0x00182305, 0x00192304,
+	0x001b2203, 0x001c2103, 0x011d2002, 0x011d2002,
+	0x011f1f01, 0x021f1e01, 0x02201d01, 0x03211c00,
+	0x03221b00, 0x04221a00, 0x04231801, 0x05241700,
+	0x06241600, 0x07241500, 0x08251300, 0x09251200,
+	0x09261100, 0x0a261000, 0x0b260f00, 0x0c260e00,
+
+	0xff0e250e, 0xff0f250d, 0xff10250c, 0xff11250b,
+	0x0011250a, 0x00132409, 0x00142408, 0x00152407,
+	0x00162307, 0x00172306, 0x00182206, 0x00192205,
+	0x011a2104, 0x011b2004, 0x011c2003, 0x021c1f03,
+	0x021e1e02, 0x031e1d02, 0x03201c01, 0x04201b01,
+	0x04211a01, 0x05221900, 0x05221801, 0x06231700,
+	0x07231600, 0x07241500, 0x08241400, 0x09241300,
+	0x0a241200, 0x0b241100, 0x0c241000, 0x0d240f00,
+
+	0x000e240e, 0x000f240d, 0x0010240c, 0x0011240b,
+	0x0013230a, 0x0013230a, 0x00142309, 0x00152308,
+	0x00162208, 0x00172207, 0x01182106, 0x01192105,
+	0x011a2005, 0x021b1f04, 0x021b1f04, 0x021d1e03,
+	0x031d1d03, 0x031e1d02, 0x041e1c02, 0x041f1b02,
+	0x05201a01, 0x05211901, 0x06211801, 0x07221700,
+	0x07221601, 0x08231500, 0x09231400, 0x0a231300,
+	0x0a231300, 0x0b231200, 0x0c231100, 0x0d231000,
+
+	0x000f220f, 0x0010220e, 0x0011220d, 0x0012220c,
+	0x0013220b, 0x0013220b, 0x0015210a, 0x0015210a,
+	0x01162108, 0x01172008, 0x01182007, 0x02191f06,
+	0x02191f06, 0x021a1e06, 0x031a1e05, 0x031c1d04,
+	0x041c1c04, 0x041d1c03, 0x051d1b03, 0x051e1a03,
+	0x061f1902, 0x061f1902, 0x07201801, 0x08201701,
+	0x08211601, 0x09211501, 0x0a211500, 0x0b211400,
+	0x0b221300, 0x0c221200, 0x0d221100, 0x0e221000,
+
+	0x0010210f, 0x0011210e, 0x0011210e, 0x0012210d,
+	0x0013210c, 0x0014200c, 0x0114200b, 0x0115200a,
+	0x01161f0a, 0x01171f09, 0x02171f08, 0x02181e08,
+	0x03181e07, 0x031a1d06, 0x031a1d06, 0x041b1c05,
+	0x041c1c04, 0x051c1b04, 0x051d1a04, 0x061d1a03,
+	0x071d1903, 0x071e1803, 0x081e1802, 0x081f1702,
+	0x091f1602, 0x0a201501, 0x0b1f1501, 0x0b201401,
+	0x0c211300, 0x0d211200, 0x0e201200, 0x0e211100,
+
+	0x00102010, 0x0011200f, 0x0012200e, 0x0013200d,
+	0x0013200d, 0x01141f0c, 0x01151f0b, 0x01151f0b,
+	0x01161f0a, 0x02171e09, 0x02171e09, 0x03181d08,
+	0x03191d07, 0x03191d07, 0x041a1c06, 0x041b1c05,
+	0x051b1b05, 0x051c1b04, 0x061c1a04, 0x071d1903,
+	0x071d1903, 0x081d1803, 0x081e1703, 0x091e1702,
+	0x0a1f1601, 0x0a1f1502, 0x0b1f1501, 0x0c1f1401,
+	0x0d201300, 0x0d201300, 0x0e201200, 0x0f201100,
+
+	0x00102010, 0x0011200f, 0x00121f0f, 0x00131f0e,
+	0x00141f0d, 0x01141f0c, 0x01141f0c, 0x01151e0c,
+	0x02161e0a, 0x02171e09, 0x03171d09, 0x03181d08,
+	0x03181d08, 0x04191c07, 0x041a1c06, 0x051a1b06,
+	0x051b1b05, 0x061b1a05, 0x061c1a04, 0x071c1904,
+	0x081c1903, 0x081d1803, 0x091d1703, 0x091e1702,
+	0x0a1e1602, 0x0b1e1502, 0x0c1e1501, 0x0c1f1401,
+	0x0d1f1400, 0x0e1f1300, 0x0e1f1201, 0x0f1f1200,
+
+	0x00111e11, 0x00121e10, 0x00131e0f, 0x00131e0f,
+	0x01131e0e, 0x01141d0e, 0x02151d0c, 0x02151d0c,
+	0x02161d0b, 0x03161c0b, 0x03171c0a, 0x04171c09,
+	0x04181b09, 0x05181b08, 0x05191b07, 0x06191a07,
+	0x061a1a06, 0x071a1906, 0x071b1905, 0x081b1805,
+	0x091b1804, 0x091c1704, 0x0a1c1703, 0x0a1c1604,
+	0x0b1d1602, 0x0c1d1502, 0x0c1d1502, 0x0d1d1402,
+	0x0e1d1401, 0x0e1e1301, 0x0f1e1300, 0x101e1200,
+
+	0x00111e11, 0x00121e10, 0x00131d10, 0x01131d0f,
+	0x01141d0e, 0x01141d0e, 0x02151c0d, 0x02151c0d,
+	0x03161c0b, 0x03161c0b, 0x04171b0a, 0x04171b0a,
+	0x05171b09, 0x05181a09, 0x06181a08, 0x06191a07,
+	0x07191907, 0x071a1906, 0x081a1806, 0x081a1806,
+	0x091a1805, 0x0a1b1704, 0x0a1b1704, 0x0b1c1603,
+	0x0b1c1603, 0x0c1c1503, 0x0d1c1502, 0x0d1d1402,
+	0x0e1d1401, 0x0f1d1301, 0x0f1d1301, 0x101e1200,
+};
+
+static const u32 bicubic8coefftab32_left[480] = {
+	0x40000000, 0x40ff0000, 0x3ffe0000, 0x3efe0000,
+	0x3dfd0000, 0x3cfc0000, 0x3bfc0000, 0x39fc0000,
+	0x36fc0000, 0x35fb0000, 0x33fb0000, 0x31fb0000,
+	0x2ffb0000, 0x2cfb0000, 0x29fc0000, 0x27fc0000,
+	0x24fc0000, 0x21fc0000, 0x1efd0000, 0x1cfd0000,
+	0x19fd0000, 0x16fe0000, 0x14fe0000, 0x11fe0000,
+	0x0dff0000, 0x0cff0000, 0x0aff0000, 0x08ff0000,
+	0x05000000, 0x03000000, 0x02000000, 0x01000000,
+
+	0x3904ff00, 0x3903ff00, 0x3902ff00, 0x38010000,
+	0x37000000, 0x36ff0000, 0x35ff0000, 0x34fe0000,
+	0x32fe0000, 0x31fd0000, 0x30fd0000, 0x2efc0000,
+	0x2cfc0000, 0x2afc0000, 0x28fc0000, 0x26fc0000,
+	0x24fc0000, 0x22fc0000, 0x20fc0000, 0x1efc0000,
+	0x1cfc0000, 0x19fc0000, 0x17fc0000, 0x15fd0000,
+	0x12fd0000, 0x11fd0000, 0x0ffd0000, 0x0dfe0000,
+	0x0bfe0000, 0x09fe0000, 0x08fe0000, 0x06ff0000,
+
+	0x3209fe00, 0x3407fe00, 0x3306fe00, 0x3305fe00,
+	0x3204fe00, 0x3102ff00, 0x3102ff00, 0x3001ff00,
+	0x2f00ff00, 0x2effff00, 0x2cff0000, 0x2bfe0000,
+	0x29fe0000, 0x28fe0000, 0x26fd0000, 0x24fd0000,
+	0x23fd0000, 0x21fd0000, 0x20fc0000, 0x1efc0000,
+	0x1dfc0000, 0x1bfc0000, 0x19fc0000, 0x17fc0000,
+	0x16fc0000, 0x14fc0000, 0x12fc0000, 0x10fd0000,
+	0x0ffd0000, 0x0dfd0000, 0x0cfd0000, 0x0afd0000,
+
+	0x2e0cfd00, 0x2e0bfd00, 0x2e09fd00, 0x2e08fd00,
+	0x2e07fd00, 0x2c06fe00, 0x2c05fe00, 0x2b04fe00,
+	0x2b03fe00, 0x2a02fe00, 0x2901fe00, 0x2701ff00,
+	0x2700ff00, 0x26ffff00, 0x24ffff00, 0x23ffff00,
+	0x22feff00, 0x20fe0000, 0x1ffe0000, 0x1efd0000,
+	0x1dfd0000, 0x1bfd0000, 0x1afd0000, 0x19fd0000,
+	0x17fd0000, 0x15fd0000, 0x13fd0000, 0x12fd0000,
+	0x11fd0000, 0x10fd0000, 0x0ffd0000, 0x0cfd0000,
+
+	0x2a0efd00, 0x2a0dfd00, 0x2a0cfd00, 0x290bfd00,
+	0x290afd00, 0x2909fd00, 0x2908fd00, 0x2807fd00,
+	0x2706fd00, 0x2705fd00, 0x2604fe00, 0x2603fe00,
+	0x2502fe00, 0x2402fe00, 0x2401fe00, 0x2200fe00,
+	0x2200fe00, 0x2000ff00, 0x1fffff00, 0x1effff00,
+	0x1dfeff00, 0x1cfeff00, 0x1afeff00, 0x19feff00,
+	0x17fe0000, 0x16fd0000, 0x15fd0000, 0x14fd0000,
+	0x12fd0000, 0x11fd0000, 0x10fd0000, 0x0ffd0000,
+
+	0x2610fd00, 0x260ffd00, 0x260efd00, 0x260dfd00,
+	0x260cfd00, 0x260bfd00, 0x260afd00, 0x2609fd00,
+	0x2508fd00, 0x2507fd00, 0x2406fd00, 0x2406fd00,
+	0x2305fd00, 0x2304fd00, 0x2203fe00, 0x2103fe00,
+	0x2002fe00, 0x1f01fe00, 0x1e01fe00, 0x1e00fe00,
+	0x1c00fe00, 0x1b00fe00, 0x1afffe00, 0x19ffff00,
+	0x18ffff00, 0x17feff00, 0x16feff00, 0x15feff00,
+	0x14feff00, 0x13feff00, 0x11feff00, 0x10fd0000,
+
+	0x2411feff, 0x2410feff, 0x240ffeff, 0x230efeff,
+	0x240dfeff, 0x240cfeff, 0x230cfd00, 0x230bfd00,
+	0x230afd00, 0x2309fd00, 0x2208fd00, 0x2108fd00,
+	0x2007fd00, 0x2106fd00, 0x2005fd00, 0x1f05fd00,
+	0x1f04fd00, 0x1e03fd00, 0x1d03fe00, 0x1c02fe00,
+	0x1b02fe00, 0x1a01fe00, 0x1a01fe00, 0x1900fe00,
+	0x1800fe00, 0x1700fe00, 0x16fffe00, 0x15fffe00,
+	0x13ffff00, 0x12ffff00, 0x12feff00, 0x11feff00,
+
+	0x2212fffe, 0x2211fffe, 0x2210ffff, 0x220ffeff,
+	0x220efeff, 0x210efeff, 0x210dfeff, 0x210cfeff,
+	0x210bfeff, 0x200bfeff, 0x200afeff, 0x1f09feff,
+	0x1f08feff, 0x1d08fe00, 0x1e07fd00, 0x1e06fd00,
+	0x1d06fd00, 0x1c05fd00, 0x1b04fe00, 0x1a04fe00,
+	0x1a03fe00, 0x1903fe00, 0x1802fe00, 0x1802fe00,
+	0x1701fe00, 0x1601fe00, 0x1501fe00, 0x1500fe00,
+	0x1400fe00, 0x1400fe00, 0x13fffe00, 0x12fffe00,
+
+	0x201200fe, 0x201100fe, 0x1f1100fe, 0x2010fffe,
+	0x200ffffe, 0x1f0ffffe, 0x1f0efffe, 0x1e0dffff,
+	0x1f0cfeff, 0x1e0cfeff, 0x1e0bfeff, 0x1e0afeff,
+	0x1d0afeff, 0x1d09feff, 0x1c08feff, 0x1b08feff,
+	0x1b07feff, 0x1a07feff, 0x1a06feff, 0x1a05feff,
+	0x1805fe00, 0x1904fe00, 0x1704fe00, 0x1703fe00,
+	0x1603fe00, 0x1602fe00, 0x1402fe00, 0x1402fe00,
+	0x1401fe00, 0x1301fe00, 0x1201fe00, 0x1200fe00,
+
+	0x1c1202fe, 0x1c1102fe, 0x1b1102fe, 0x1c1001fe,
+	0x1b1001fe, 0x1c0f01fe, 0x1b0f00fe, 0x1b0e00fe,
+	0x1b0e00fe, 0x1b0d00fe, 0x1b0c00fe, 0x1a0cfffe,
+	0x1a0bfffe, 0x1a0bfffe, 0x190afffe, 0x190afffe,
+	0x1909fffe, 0x1709ffff, 0x1808ffff, 0x1708feff,
+	0x1707feff, 0x1707feff, 0x1606feff, 0x1506feff,
+	0x1505feff, 0x1505feff, 0x1404feff, 0x1404feff,
+	0x1404feff, 0x1303feff, 0x1203feff, 0x1202feff,
+
+	0x191104fe, 0x191104fe, 0x191003fe, 0x191003fe,
+	0x171003fe, 0x180f03fe, 0x180f02fe, 0x180e02fe,
+	0x180e02fe, 0x180d01fe, 0x180d01fe, 0x180d01fe,
+	0x170c01fe, 0x160c01fe, 0x170b00fe, 0x170b00fe,
+	0x160a00fe, 0x160a00fe, 0x160a00fe, 0x150900fe,
+	0x1509fffe, 0x1508fffe, 0x1508fffe, 0x1408fffe,
+	0x1407fffe, 0x1307ffff, 0x1306ffff, 0x1206ffff,
+	0x1206ffff, 0x1205ffff, 0x1205ffff, 0x1104feff,
+
+	0x161006ff, 0x161005ff, 0x161005ff, 0x160f05ff,
+	0x160f04ff, 0x150f04ff, 0x150e04ff, 0x150e04ff,
+	0x150e03ff, 0x150d03ff, 0x150d03ff, 0x150d02ff,
+	0x140c02ff, 0x150c02fe, 0x150c02fe, 0x150b02fe,
+	0x140b01fe, 0x140b01fe, 0x140a01fe, 0x140a01fe,
+	0x140a01fe, 0x130900fe, 0x130900fe, 0x130900fe,
+	0x130800fe, 0x120800fe, 0x120800fe, 0x120700fe,
+	0x120700fe, 0x1107fffe, 0x1106fffe, 0x1106fffe,
+
+	0x140f0700, 0x140f0600, 0x140f0600, 0x140f0600,
+	0x140e0600, 0x130e0500, 0x140e05ff, 0x130e05ff,
+	0x140d05ff, 0x130d04ff, 0x130d04ff, 0x120d04ff,
+	0x130c04ff, 0x130c03ff, 0x130c03ff, 0x120c03ff,
+	0x120b03ff, 0x120b02ff, 0x120b02ff, 0x120a02ff,
+	0x120a02ff, 0x110a02ff, 0x110a01ff, 0x120901ff,
+	0x100901ff, 0x100901ff, 0x110801ff, 0x110801ff,
+	0x100800ff, 0x100800ff, 0x100700ff, 0x100700fe,
+
+	0x120f0701, 0x120e0701, 0x120e0701, 0x120e0701,
+	0x120e0600, 0x110e0600, 0x120d0600, 0x120d0600,
+	0x120d0500, 0x120d0500, 0x110d0500, 0x110c0500,
+	0x110c0500, 0x110c0400, 0x110c0400, 0x110b04ff,
+	0x110b04ff, 0x110b04ff, 0x110b03ff, 0x110b03ff,
+	0x110a03ff, 0x110a03ff, 0x100a03ff, 0x110a02ff,
+	0x100902ff, 0x100902ff, 0x100902ff, 0x0f0902ff,
+	0x0e0902ff, 0x100801ff, 0x0f0801ff, 0x0f0801ff,
+
+	0x100e0802, 0x100e0802, 0x110e0702, 0x110d0701,
+	0x110d0701, 0x100d0701, 0x100d0701, 0x110d0601,
+	0x110d0601, 0x110c0601, 0x110c0601, 0x100c0600,
+	0x100c0500, 0x100c0500, 0x100c0500, 0x100b0500,
+	0x100b0500, 0x100b0400, 0x100b0400, 0x0f0b0400,
+	0x100a0400, 0x0f0a0400, 0x0f0a0400, 0x0f0a0300,
+	0x0f0a03ff, 0x0f0903ff, 0x0f0903ff, 0x0f0903ff,
+	0x0f0903ff, 0x0f0902ff, 0x0f0902ff, 0x0f0802ff
+};
+
+static const u32 bicubic8coefftab32_right[480] = {
+	0x00000000, 0x00000001, 0x00000003, 0x00000004,
+	0x00000006, 0x0000ff09, 0x0000ff0a, 0x0000ff0c,
+	0x0000ff0f, 0x0000fe12, 0x0000fe14, 0x0000fe16,
+	0x0000fd19, 0x0000fd1c, 0x0000fd1e, 0x0000fc21,
+	0x0000fc24, 0x0000fc27, 0x0000fc29, 0x0000fb2c,
+	0x0000fb2f, 0x0000fb31, 0x0000fb33, 0x0000fb36,
+	0x0000fc38, 0x0000fc39, 0x0000fc3b, 0x0000fc3d,
+	0x0000fd3e, 0x0000fe3f, 0x0000fe40, 0x0000ff40,
+
+	0x0000ff05, 0x0000ff06, 0x0000fe08, 0x0000fe09,
+	0x0000fe0b, 0x0000fe0d, 0x0000fd0f, 0x0000fd11,
+	0x0000fd13, 0x0000fd15, 0x0000fc17, 0x0000fc1a,
+	0x0000fc1c, 0x0000fc1e, 0x0000fc20, 0x0000fc22,
+	0x0000fc24, 0x0000fc26, 0x0000fc28, 0x0000fc2a,
+	0x0000fc2c, 0x0000fc2f, 0x0000fd30, 0x0000fd31,
+	0x0000fe33, 0x0000fe34, 0x0000ff35, 0x0000ff36,
+	0x00000037, 0x00000138, 0x00ff0239, 0x00ff0339,
+
+	0x0000fe09, 0x0000fd0a, 0x0000fd0c, 0x0000fd0d,
+	0x0000fd0f, 0x0000fd11, 0x0000fc12, 0x0000fc14,
+	0x0000fc16, 0x0000fc18, 0x0000fc19, 0x0000fc1b,
+	0x0000fc1d, 0x0000fc1e, 0x0000fc21, 0x0000fd22,
+	0x0000fd23, 0x0000fd25, 0x0000fd27, 0x0000fe28,
+	0x0000fe29, 0x0000fe2b, 0x0000ff2c, 0x00ffff2f,
+	0x00ff002f, 0x00ff0130, 0x00ff0231, 0x00ff0232,
+	0x00fe0432, 0x00fe0533, 0x00fe0633, 0x00fe0734,
+
+	0x0000fd0c, 0x0000fd0d, 0x0000fd0f, 0x0000fd10,
+	0x0000fd11, 0x0000fd13, 0x0000fd14, 0x0000fd16,
+	0x0000fd17, 0x0000fd19, 0x0000fd1b, 0x0000fd1c,
+	0x0000fd1d, 0x0000fd1f, 0x0000fe20, 0x0000fe21,
+	0x00fffe24, 0x00ffff24, 0x00ffff25, 0x00ffff27,
+	0x00ff0027, 0x00ff0128, 0x00fe012a, 0x00fe022a,
+	0x00fe032b, 0x00fe042c, 0x00fe052d, 0x00fe062d,
+	0x00fd072e, 0x00fd082e, 0x00fd092e, 0x00fd0b2f,
+
+	0x0000fd0e, 0x0000fd0f, 0x0000fd10, 0x0000fd12,
+	0x0000fd13, 0x0000fd14, 0x0000fd15, 0x0000fd17,
+	0x0000fe18, 0x00fffe1a, 0x00fffe1b, 0x00fffe1c,
+	0x00fffe1e, 0x00ffff1e, 0x00ffff1f, 0x00ff0021,
+	0x00fe0022, 0x00fe0023, 0x00fe0124, 0x00fe0224,
+	0x00fe0226, 0x00fe0326, 0x00fe0427, 0x00fd0528,
+	0x00fd0628, 0x00fd0729, 0x00fd0829, 0x00fd0929,
+	0x00fd0a2a, 0x00fd0b2a, 0x00fd0c2a, 0x00fd0d2a,
+
+	0x0000fd10, 0x0000fd11, 0x00fffe12, 0x00fffe13,
+	0x00fffe14, 0x00fffe15, 0x00fffe16, 0x00fffe17,
+	0x00ffff18, 0x00ffff19, 0x00feff1c, 0x00fe001b,
+	0x00fe001d, 0x00fe001e, 0x00fe011e, 0x00fe011f,
+	0x00fe0220, 0x00fe0321, 0x00fe0322, 0x00fd0423,
+	0x00fd0524, 0x00fd0624, 0x00fd0626, 0x00fd0725,
+	0x00fd0825, 0x00fd0926, 0x00fd0a26, 0x00fd0b26,
+	0x00fd0c26, 0x00fd0d26, 0x00fd0e27, 0x00fd0f27,
+
+	0x00fffe11, 0x00fffe12, 0x00fffe13, 0x00ffff14,
+	0x00ffff14, 0x00feff16, 0x00feff17, 0x00fe0017,
+	0x00fe0018, 0x00fe0019, 0x00fe011a, 0x00fe011b,
+	0x00fe021c, 0x00fe021c, 0x00fe031d, 0x00fd031f,
+	0x00fd041f, 0x00fd0520, 0x00fd0520, 0x00fd0621,
+	0x00fd0721, 0x00fd0822, 0x00fd0822, 0x00fd0923,
+	0x00fd0a23, 0x00fd0b23, 0x00fd0b25, 0x00fe0c24,
+	0x00fe0d24, 0x00fe0e24, 0x00fe0f24, 0x00fe1024,
+
+	0x00feff12, 0x00feff13, 0x00feff13, 0x00fe0014,
+	0x00fe0015, 0x00fe0016, 0x00fe0116, 0x00fe0117,
+	0x00fe0118, 0x00fe0218, 0x00fe0219, 0x00fe031a,
+	0x00fe031b, 0x00fe041b, 0x00fd041d, 0x00fd051d,
+	0x00fd061d, 0x00fd061f, 0x00fe071e, 0x00fe081e,
+	0x00fe081f, 0x00fe091f, 0x00fe0a20, 0x00fe0a20,
+	0x00fe0b21, 0x00fe0c21, 0x00fe0d21, 0x00fe0d22,
+	0x00fe0e22, 0x00fe0f21, 0x00ff1021, 0x00ff1022,
+
+	0x00fe0012, 0x00fe0013, 0x00fe0113, 0x00fe0114,
+	0x00fe0115, 0x00fe0215, 0x00fe0216, 0x00fe0217,
+	0x00fe0317, 0x00fe0318, 0x00fe0418, 0x00fe0419,
+	0x00fe0519, 0x00fe051a, 0x00fe061b, 0x00fe071b,
+	0x00fe071c, 0x00fe081c, 0x00fe081d, 0x00fe091d,
+	0x00fe0a1d, 0x00fe0a1d, 0x00fe0b1e, 0x00fe0c1e,
+	0x00ff0c1e, 0x00ff0d1e, 0x00ff0e1f, 0x00ff0e1f,
+	0x00ff0f1f, 0x00ff0f20, 0x0000101f, 0x0000111f,
+
+	0x00fe0212, 0x00fe0312, 0x00fe0313, 0x00fe0314,
+	0x00fe0414, 0x00fe0414, 0x00fe0515, 0x00fe0516,
+	0x00fe0516, 0x00fe0616, 0x00fe0617, 0x00fe0718,
+	0x00fe0719, 0x00fe0818, 0x00ff0819, 0x00ff0918,
+	0x00ff0919, 0x00ff0a19, 0x00ff0a19, 0x00ff0b1a,
+	0x00ff0b1b, 0x00ff0c1a, 0x00000c1b, 0x00000d1b,
+	0x00000d1c, 0x00000e1b, 0x00000e1d, 0x00010f1b,
+	0x00010f1b, 0x0001101c, 0x0001101d, 0x0002111c,
+
+	0x00fe0412, 0x00fe0412, 0x00ff0512, 0x00ff0512,
+	0x00ff0613, 0x00ff0613, 0x00ff0614, 0x00ff0714,
+	0x00ff0714, 0x00ff0815, 0x00ff0815, 0x00ff0815,
+	0x00ff0916, 0x00000916, 0x00000a16, 0x00000a16,
+	0x00000a18, 0x00000b17, 0x00000b17, 0x00010c17,
+	0x00010c18, 0x00010d18, 0x00010d18, 0x00010d19,
+	0x00020e18, 0x00020e18, 0x00020f18, 0x00030f18,
+	0x00030f18, 0x00031018, 0x00031018, 0x00041119,
+
+	0x00ff0610, 0x00ff0611, 0x00ff0611, 0x00ff0711,
+	0x00000711, 0x00000712, 0x00000812, 0x00000812,
+	0x00000813, 0x00000913, 0x00000913, 0x00000914,
+	0x00010a14, 0x00010a14, 0x00010a14, 0x00010b14,
+	0x00010b16, 0x00020b15, 0x00020c15, 0x00020c15,
+	0x00020c15, 0x00020d17, 0x00030d16, 0x00030d16,
+	0x00030e16, 0x00040e16, 0x00040e16, 0x00040f16,
+	0x00040f16, 0x00050f17, 0x00051017, 0x00051017,
+
+	0x0000070f, 0x00000710, 0x00000710, 0x00000710,
+	0x00000810, 0x00010811, 0x00010811, 0x00010911,
+	0x00010911, 0x00010913, 0x00010913, 0x00020a12,
+	0x00020a12, 0x00020a13, 0x00020b12, 0x00020b13,
+	0x00030b13, 0x00030c13, 0x00030c13, 0x00030c14,
+	0x00040c13, 0x00040d13, 0x00040d14, 0x00040d14,
+	0x00050e14, 0x00050e14, 0x00050e14, 0x00050e14,
+	0x00060f14, 0x00060f14, 0x00060f15, 0x00061015,
+
+	0x0001070f, 0x0001080f, 0x0001080f, 0x0001080f,
+	0x00010811, 0x00020910, 0x00020910, 0x00020910,
+	0x00020911, 0x00020a10, 0x00030a10, 0x00030a11,
+	0x00030a11, 0x00030b11, 0x00030b11, 0x00040b12,
+	0x00040b12, 0x00040c11, 0x00040c12, 0x00040c12,
+	0x00050c12, 0x00050c12, 0x00050d12, 0x00050d12,
+	0x00060d13, 0x00060d13, 0x00060e12, 0x00060e13,
+	0x00070e13, 0x00070e13, 0x00070f13, 0x00070f13,
+
+	0x0002080e, 0x0002080e, 0x0002080e, 0x00020810,
+	0x0002090f, 0x0003090f, 0x0003090f, 0x0003090f,
+	0x0003090f, 0x00030a0f, 0x00030a0f, 0x00040a10,
+	0x00040a11, 0x00040b10, 0x00040b10, 0x00040b11,
+	0x00050b10, 0x00050b11, 0x00050c10, 0x00050c11,
+	0x00050c11, 0x00060c11, 0x00060c11, 0x00060d11,
+	0x00060d12, 0x00070d12, 0x00070d12, 0x00070e11,
+	0x00070e11, 0x00070e12, 0x00080e11, 0x00080e12
+};
+
+static const u32 bicubic4coefftab32[480] = {
+	0x00004000, 0x000140ff, 0x00033ffe, 0x00043ffd,
+	0x00063dfd, 0xff083dfc, 0xff0a3bfc, 0xff0c39fc,
+	0xff0e37fc, 0xfe1136fb, 0xfe1433fb, 0xfe1631fb,
+	0xfd192ffb, 0xfd1c2cfb, 0xfd1e29fc, 0xfc2127fc,
+	0xfc2424fc, 0xfc2721fc, 0xfc291efd, 0xfb2c1cfd,
+	0xfb2f19fd, 0xfb3116fe, 0xfb3314fe, 0xfb3611fe,
+	0xfc370eff, 0xfc390cff, 0xfc3b0aff, 0xfc3d08ff,
+	0xfd3d0600, 0xfd3f0400, 0xfe3f0300, 0xff400100,
+
+	0xfe053904, 0xfe063903, 0xfe083901, 0xfe0a3800,
+	0xfd0b3800, 0xfe0d36ff, 0xfd0f35ff, 0xfd1134fe,
+	0xfd1332fe, 0xfd1531fd, 0xfc1730fd, 0xfc1a2efc,
+	0xfc1c2cfc, 0xfc1e2afc, 0xfc2028fc, 0xfc2226fc,
+	0xfc2424fc, 0xfc2622fc, 0xfc2820fc, 0xfc2a1efc,
+	0xfc2c1cfc, 0xfc2e1afc, 0xfd3017fc, 0xfd3115fd,
+	0xfe3213fd, 0xfe3411fd, 0xff350ffd, 0xff360dfe,
+	0x00370bfe, 0x013809fe, 0x023808fe, 0x033806ff,
+
+	0xfd093208, 0xfd0a3207, 0xfd0c3205, 0xfd0d3204,
+	0xfc0f3203, 0xfc113102, 0xfc123002, 0xfc143000,
+	0xfc152f00, 0xfc172d00, 0xfc192cff, 0xfc1b2bfe,
+	0xfc1d29fe, 0xfc1e28fe, 0xfc2027fd, 0xfd2125fd,
+	0xfd2323fd, 0xfd2521fd, 0xfd2720fc, 0xfe281efc,
+	0xfe291dfc, 0xfe2b1bfc, 0xff2c19fc, 0x002d17fc,
+	0x002e16fc, 0x012f14fc, 0x022f12fd, 0x023110fd,
+	0x03310ffd, 0x05310dfd, 0x06320bfd, 0x07320afd,
+
+	0xfc0c2d0b, 0xfc0d2d0a, 0xfc0e2d09, 0xfc102d07,
+	0xfc112c07, 0xfc132c05, 0xfc142c04, 0xfc162b03,
+	0xfc172a03, 0xfc192a01, 0xfc1a2901, 0xfd1b2800,
+	0xfd1c2700, 0xfd1e2500, 0xfe1f24ff, 0xfe2023ff,
+	0xfe2222fe, 0xff2320fe, 0xff241ffe, 0x00251efd,
+	0x00271cfd, 0x01271bfd, 0x01281afd, 0x022918fd,
+	0x032a16fd, 0x032b15fd, 0x042b14fd, 0x052c12fd,
+	0x072c10fd, 0x082c0ffd, 0x092c0efd, 0x0a2c0dfd,
+
+	0xfd0d290d, 0xfd0e290c, 0xfd0f290b, 0xfd11280a,
+	0xfd122809, 0xfd132808, 0xfd142807, 0xfd162706,
+	0xfd172705, 0xfd192604, 0xfe1a2503, 0xfe1b2502,
+	0xfe1c2402, 0xfe1d2302, 0xff1e2201, 0xff1f2101,
+	0x00202000, 0x00211f00, 0x01221eff, 0x02221dff,
+	0x02241cfe, 0x03241bfe, 0x042519fe, 0x042618fe,
+	0x052617fe, 0x062716fd, 0x072714fe, 0x082713fe,
+	0x092812fd, 0x0a2811fd, 0x0b2810fd, 0x0c280ffd,
+
+	0xfd0f250f, 0xfd10250e, 0xfd11250d, 0xfd12250c,
+	0xfd13250b, 0xfe13250a, 0xfe152409, 0xfe162408,
+	0xfe172308, 0xff182306, 0xff192305, 0xff1a2205,
+	0x001b2104, 0x001c2103, 0x001d2003, 0x011e1f02,
+	0x011f1f01, 0x021f1e01, 0x03201d00, 0x03211c00,
+	0x04211b00, 0x05221aff, 0x062219ff, 0x062318ff,
+	0x082316ff, 0x082316ff, 0x092415fe, 0x0a2414fe,
+	0x0b2413fe, 0x0c2412fe, 0x0d2411fe, 0x0e2410fe,
+
+	0xfe10230f, 0xfe11230e, 0xfe12220e, 0xfe13220d,
+	0xfe14220c, 0xff14220b, 0xff15220a, 0xff16210a,
+	0x00162109, 0x00172108, 0x00182008, 0x01192006,
+	0x011a1f06, 0x021a1f05, 0x021b1e05, 0x031c1d04,
+	0x031d1d03, 0x041d1c03, 0x041e1b03, 0x051e1b02,
+	0x061f1a01, 0x06201901, 0x07201801, 0x08201800,
+	0x09201700, 0x0a211500, 0x0b2115ff, 0x0c2114ff,
+	0x0c2213ff, 0x0d2212ff, 0x0e2211ff, 0x0f2211fe,
+
+	0xff112010, 0xff12200f, 0xff12200f, 0xff13200e,
+	0x0013200d, 0x0014200c, 0x00151f0c, 0x00161f0b,
+	0x01161f0a, 0x01171e0a, 0x02171e09, 0x02181e08,
+	0x03191d07, 0x03191d07, 0x041a1c06, 0x041b1c05,
+	0x051b1b05, 0x051c1b04, 0x061c1a04, 0x071c1a03,
+	0x071d1903, 0x081e1802, 0x091d1802, 0x091e1702,
+	0x0a1f1601, 0x0b1f1600, 0x0b1f1501, 0x0c201400,
+	0x0d1f1400, 0x0e2013ff, 0x0f1f1200, 0x102011ff,
+
+	0x00111f10, 0x00121e10, 0x00131e0f, 0x00131e0f,
+	0x01131e0e, 0x01141e0d, 0x01151d0d, 0x02151d0c,
+	0x02161d0b, 0x03161d0a, 0x03171c0a, 0x04171c09,
+	0x04181c08, 0x05181b08, 0x05191b07, 0x06191a07,
+	0x061a1a06, 0x071a1906, 0x071b1905, 0x081b1805,
+	0x081c1804, 0x091c1704, 0x0a1c1703, 0x0a1d1603,
+	0x0b1d1602, 0x0c1d1502, 0x0c1d1502, 0x0d1e1401,
+	0x0e1d1401, 0x0e1e1301, 0x0f1e1300, 0x101e1200,
+
+	0x02111c11, 0x02121c10, 0x02131b10, 0x03131b0f,
+	0x03131b0f, 0x03141b0e, 0x04141b0d, 0x04151a0d,
+	0x05151a0c, 0x05151a0c, 0x05161a0b, 0x0616190b,
+	0x0616190b, 0x0716190a, 0x0717180a, 0x08171809,
+	0x08181808, 0x09181708, 0x09181708, 0x0a181707,
+	0x0a191607, 0x0b191606, 0x0b1a1605, 0x0c1a1505,
+	0x0c1a1505, 0x0d1a1504, 0x0d1b1404, 0x0e1b1403,
+	0x0f1b1303, 0x0f1b1303, 0x101b1302, 0x101c1202,
+
+	0x04111a11, 0x04121911, 0x04131910, 0x0513190f,
+	0x0513190f, 0x0513190f, 0x0613190e, 0x0614180e,
+	0x0714180d, 0x0714180d, 0x0715180c, 0x0814180c,
+	0x0815170c, 0x0816170b, 0x0916170a, 0x0916170a,
+	0x0a16160a, 0x0a171609, 0x0a171609, 0x0b171608,
+	0x0b171509, 0x0c171508, 0x0c181507, 0x0d171507,
+	0x0d181407, 0x0e181406, 0x0e181406, 0x0e191306,
+	0x0f191305, 0x0f191305, 0x10191304, 0x10191205,
+
+	0x05121811, 0x06121810, 0x06121810, 0x06131710,
+	0x0713170f, 0x0713170f, 0x0713170f, 0x0813170e,
+	0x0813170e, 0x0814170d, 0x0914160d, 0x0914160d,
+	0x0914160d, 0x0a14160c, 0x0a15160b, 0x0a15150c,
+	0x0b15150b, 0x0b15150b, 0x0b16150a, 0x0c15150a,
+	0x0c16140a, 0x0d161409, 0x0d161409, 0x0d171408,
+	0x0e161408, 0x0e171308, 0x0e171308, 0x0f171307,
+	0x0f171307, 0x10171306, 0x10181206, 0x10181206,
+
+	0x07111711, 0x07121710, 0x07121611, 0x08121610,
+	0x08121610, 0x0813160f, 0x0912160f, 0x0913160e,
+	0x0913160e, 0x0913160e, 0x0a14150d, 0x0a14150d,
+	0x0a14150d, 0x0b14150c, 0x0b14150c, 0x0b14150c,
+	0x0c14140c, 0x0c15140b, 0x0c15140b, 0x0c15140b,
+	0x0d15140a, 0x0d15140a, 0x0d15140a, 0x0e161309,
+	0x0e161309, 0x0e161309, 0x0f151309, 0x0f161308,
+	0x0f161209, 0x10161208, 0x10161208, 0x10171207,
+
+	0x0a111411, 0x0b111410, 0x0b111410, 0x0b111410,
+	0x0b111410, 0x0b12140f, 0x0b12140f, 0x0c12130f,
+	0x0c12130f, 0x0c12130f, 0x0c12130f, 0x0c12130f,
+	0x0d12130e, 0x0d12130e, 0x0d12130e, 0x0d13130d,
+	0x0d13130d, 0x0d13130d, 0x0e12130d, 0x0e13120d,
+	0x0e13120d, 0x0e13120d, 0x0e13120d, 0x0f13120c,
+	0x0f13120c, 0x0f13120c, 0x0f14120b, 0x0f14120b,
+	0x1013120b, 0x1013120b, 0x1013120b, 0x1014110b,
+
+	0x0c111310, 0x0c111310, 0x0c111310, 0x0d101310,
+	0x0d101310, 0x0d111210, 0x0d111210, 0x0d111210,
+	0x0d12120f, 0x0d12120f, 0x0d12120f, 0x0d12120f,
+	0x0e11120f, 0x0e12120e, 0x0e12120e, 0x0e12120e,
+	0x0e12120e, 0x0e12120e, 0x0e12120e, 0x0e12120e,
+	0x0f11120e, 0x0f12120d, 0x0f12120d, 0x0f12120d,
+	0x0f12120d, 0x0f12110e, 0x0f12110e, 0x0f12110e,
+	0x1012110d, 0x1012110d, 0x1013110c, 0x1013110c,
+};
+
+static int sun8i_vi_scaler_coef_index(unsigned int step)
+{
+	unsigned int scale, int_part, float_part;
+
+	scale = step >> (SUN8I_VI_SCALER_SCALE_FRAC - 3);
+	int_part = scale >> 3;
+	float_part = scale & 0x7;
+
+	switch (int_part) {
+	case 0:
+		return 0;
+	case 1:
+		return float_part;
+	case 2:
+		return 8 + (float_part >> 1);
+	case 3:
+		return 12;
+	case 4:
+		return 13;
+	default:
+		return 14;
+	}
+}
+
+static void sun8i_vi_scaler_set_coeff(struct regmap *map, int layer,
+				      u32 hstep, u32 vstep,
+				      const struct drm_format_info *format)
+{
+	const u32 *ch_left, *ch_right, *cy;
+	int offset, i;
+
+	if (format->hsub == 1 && format->vsub == 1) {
+		ch_left = lan3coefftab32_left;
+		ch_right = lan3coefftab32_right;
+		cy = lan2coefftab32;
+	} else {
+		ch_left = bicubic8coefftab32_left;
+		ch_right = bicubic8coefftab32_right;
+		cy = bicubic4coefftab32;
+	}
+
+	offset = sun8i_vi_scaler_coef_index(hstep) *
+			SUN8I_VI_SCALER_COEFF_COUNT;
+	for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) {
+		regmap_write(map, SUN8I_SCALER_VSU_YHCOEFF0(layer, i),
+			     lan3coefftab32_left[offset + i]);
+		regmap_write(map, SUN8I_SCALER_VSU_YHCOEFF1(layer, i),
+			     lan3coefftab32_right[offset + i]);
+		regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF0(layer, i),
+			     ch_left[offset + i]);
+		regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF1(layer, i),
+			     ch_right[offset + i]);
+	}
+
+	offset = sun8i_vi_scaler_coef_index(hstep) *
+			SUN8I_VI_SCALER_COEFF_COUNT;
+	for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) {
+		regmap_write(map, SUN8I_SCALER_VSU_YVCOEFF(layer, i),
+			     lan2coefftab32[offset + i]);
+		regmap_write(map, SUN8I_SCALER_VSU_CVCOEFF(layer, i),
+			     cy[offset + i]);
+	}
+}
+
+void sun8i_vi_scaler_enable(struct sun8i_mixer *mixer, int layer, bool enable)
+{
+	u32 val;
+
+	if (enable)
+		val = SUN8I_SCALER_VSU_CTRL_EN |
+		      SUN8I_SCALER_VSU_CTRL_COEFF_RDY;
+	else
+		val = 0;
+
+	regmap_write(mixer->engine.regs, SUN8I_SCALER_VSU_CTRL(layer), val);
+}
+
+void sun8i_vi_scaler_setup(struct sun8i_mixer *mixer, int layer,
+			   u32 src_w, u32 src_h, u32 dst_w, u32 dst_h,
+			   u32 hscale, u32 vscale, u32 hphase, u32 vphase,
+			   const struct drm_format_info *format)
+{
+	u32 chphase, cvphase;
+	u32 insize, outsize;
+
+	hphase <<= SUN8I_VI_SCALER_PHASE_FRAC - 16;
+	vphase <<= SUN8I_VI_SCALER_PHASE_FRAC - 16;
+	hscale <<= SUN8I_VI_SCALER_SCALE_FRAC - 16;
+	vscale <<= SUN8I_VI_SCALER_SCALE_FRAC - 16;
+
+	insize = SUN8I_VI_SCALER_SIZE(src_w, src_h);
+	outsize = SUN8I_VI_SCALER_SIZE(dst_w, dst_h);
+
+	/*
+	 * This is chroma V/H phase calculation as it appears in
+	 * BSP driver. There is no detailed explanation. YUV 420
+	 * chroma is threated specialy for some reason.
+	 */
+	if (format->hsub == 2 && format->vsub == 2) {
+		chphase = hphase >> 1;
+		cvphase = (vphase >> 1) -
+			(1UL << (SUN8I_VI_SCALER_SCALE_FRAC - 2));
+	} else {
+		chphase = hphase;
+		cvphase = vphase;
+	}
+
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_OUTSIZE(layer), outsize);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_YINSIZE(layer), insize);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_YHSTEP(layer), hscale);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_YVSTEP(layer), vscale);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_YHPHASE(layer), hphase);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_YVPHASE(layer), vphase);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_CINSIZE(layer),
+		     SUN8I_VI_SCALER_SIZE(src_w / format->hsub,
+					  src_h / format->vsub));
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_CHSTEP(layer),
+		     hscale / format->hsub);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_CVSTEP(layer),
+		     vscale / format->vsub);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_CHPHASE(layer), chphase);
+	regmap_write(mixer->engine.regs,
+		     SUN8I_SCALER_VSU_CVPHASE(layer), cvphase);
+	sun8i_vi_scaler_set_coeff(mixer->engine.regs, layer,
+				  hscale, vscale, format);
+}
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h
new file mode 100644
index 000000000000..a595ab643a5a
--- /dev/null
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2017 Jernej Skrabec <jernej.skrabec@siol.net>
+ *
+ * This file is licensed under the terms of the GNU General Public
+ * License version 2.  This program is licensed "as is" without any
+ * warranty of any kind, whether express or implied.
+ */
+
+#ifndef _SUN8I_VI_SCALER_H_
+#define _SUN8I_VI_SCALER_H_
+
+#include <drm/drm_fourcc.h>
+#include "sun8i_mixer.h"
+
+/* this two macros assumes 16 fractional bits which is standard in DRM */
+#define SUN8I_VI_SCALER_SCALE_MIN		1
+#define SUN8I_VI_SCALER_SCALE_MAX		((1UL << 20) - 1)
+
+#define SUN8I_VI_SCALER_SCALE_FRAC		20
+#define SUN8I_VI_SCALER_PHASE_FRAC		20
+#define SUN8I_VI_SCALER_COEFF_COUNT		32
+#define SUN8I_VI_SCALER_SIZE(w, h)		(((h) - 1) << 16 | ((w) - 1))
+
+#define SUN8I_SCALER_VSU_CTRL(ch)	(0x20000 + 0x20000 * (ch) + 0x0)
+#define SUN8I_SCALER_VSU_OUTSIZE(ch)	(0x20000 + 0x20000 * (ch) + 0x40)
+#define SUN8I_SCALER_VSU_YINSIZE(ch)	(0x20000 + 0x20000 * (ch) + 0x80)
+#define SUN8I_SCALER_VSU_YHSTEP(ch)	(0x20000 + 0x20000 * (ch) + 0x88)
+#define SUN8I_SCALER_VSU_YVSTEP(ch)	(0x20000 + 0x20000 * (ch) + 0x8c)
+#define SUN8I_SCALER_VSU_YHPHASE(ch)	(0x20000 + 0x20000 * (ch) + 0x90)
+#define SUN8I_SCALER_VSU_YVPHASE(ch)	(0x20000 + 0x20000 * (ch) + 0x98)
+#define SUN8I_SCALER_VSU_CINSIZE(ch)	(0x20000 + 0x20000 * (ch) + 0xc0)
+#define SUN8I_SCALER_VSU_CHSTEP(ch)	(0x20000 + 0x20000 * (ch) + 0xc8)
+#define SUN8I_SCALER_VSU_CVSTEP(ch)	(0x20000 + 0x20000 * (ch) + 0xcc)
+#define SUN8I_SCALER_VSU_CHPHASE(ch)	(0x20000 + 0x20000 * (ch) + 0xd0)
+#define SUN8I_SCALER_VSU_CVPHASE(ch)	(0x20000 + 0x20000 * (ch) + 0xd8)
+#define SUN8I_SCALER_VSU_YHCOEFF0(ch, i) \
+	(0x20000 + 0x20000 * (ch) + 0x200 + 0x4 * (i))
+#define SUN8I_SCALER_VSU_YHCOEFF1(ch, i) \
+	(0x20000 + 0x20000 * (ch) + 0x300 + 0x4 * (i))
+#define SUN8I_SCALER_VSU_YVCOEFF(ch, i) \
+	(0x20000 + 0x20000 * (ch) + 0x400 + 0x4 * (i))
+#define SUN8I_SCALER_VSU_CHCOEFF0(ch, i) \
+	(0x20000 + 0x20000 * (ch) + 0x600 + 0x4 * (i))
+#define SUN8I_SCALER_VSU_CHCOEFF1(ch, i) \
+	(0x20000 + 0x20000 * (ch) + 0x700 + 0x4 * (i))
+#define SUN8I_SCALER_VSU_CVCOEFF(ch, i) \
+	(0x20000 + 0x20000 * (ch) + 0x800 + 0x4 * (i))
+
+#define SUN8I_SCALER_VSU_CTRL_EN		BIT(0)
+#define SUN8I_SCALER_VSU_CTRL_COEFF_RDY		BIT(4)
+
+void sun8i_vi_scaler_enable(struct sun8i_mixer *mixer, int layer, bool enable);
+void sun8i_vi_scaler_setup(struct sun8i_mixer *mixer, int layer,
+			   u32 src_w, u32 src_h, u32 dst_w, u32 dst_h,
+			   u32 hscale, u32 vscale, u32 hphase, u32 vphase,
+			   const struct drm_format_info *format);
+
+#endif
diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile
index 46d65d39214d..2e0d6213f6bc 100644
--- a/drivers/gpu/drm/tegra/Makefile
+++ b/drivers/gpu/drm/tegra/Makefile
@@ -5,6 +5,8 @@ tegra-drm-y := \
 	drm.o \
 	gem.o \
 	fb.o \
+	hub.o \
+	plane.o \
 	dc.o \
 	output.o \
 	rgb.o \
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index 24a5ef4f5bb8..90b25ce363ca 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -19,82 +19,79 @@
 #include "dc.h"
 #include "drm.h"
 #include "gem.h"
+#include "hub.h"
+#include "plane.h"
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_plane_helper.h>
 
-struct tegra_plane {
-	struct drm_plane base;
-	unsigned int index;
-};
-
-static inline struct tegra_plane *to_tegra_plane(struct drm_plane *plane)
+static void tegra_dc_stats_reset(struct tegra_dc_stats *stats)
 {
-	return container_of(plane, struct tegra_plane, base);
+	stats->frames = 0;
+	stats->vblank = 0;
+	stats->underflow = 0;
+	stats->overflow = 0;
 }
 
-struct tegra_dc_state {
-	struct drm_crtc_state base;
+/* Reads the active copy of a register. */
+static u32 tegra_dc_readl_active(struct tegra_dc *dc, unsigned long offset)
+{
+	u32 value;
 
-	struct clk *clk;
-	unsigned long pclk;
-	unsigned int div;
+	tegra_dc_writel(dc, READ_MUX, DC_CMD_STATE_ACCESS);
+	value = tegra_dc_readl(dc, offset);
+	tegra_dc_writel(dc, 0, DC_CMD_STATE_ACCESS);
 
-	u32 planes;
-};
+	return value;
+}
 
-static inline struct tegra_dc_state *to_dc_state(struct drm_crtc_state *state)
+static inline unsigned int tegra_plane_offset(struct tegra_plane *plane,
+					      unsigned int offset)
 {
-	if (state)
-		return container_of(state, struct tegra_dc_state, base);
-
-	return NULL;
-}
+	if (offset >= 0x500 && offset <= 0x638) {
+		offset = 0x000 + (offset - 0x500);
+		return plane->offset + offset;
+	}
 
-struct tegra_plane_state {
-	struct drm_plane_state base;
+	if (offset >= 0x700 && offset <= 0x719) {
+		offset = 0x180 + (offset - 0x700);
+		return plane->offset + offset;
+	}
 
-	struct tegra_bo_tiling tiling;
-	u32 format;
-	u32 swap;
-};
+	if (offset >= 0x800 && offset <= 0x839) {
+		offset = 0x1c0 + (offset - 0x800);
+		return plane->offset + offset;
+	}
 
-static inline struct tegra_plane_state *
-to_tegra_plane_state(struct drm_plane_state *state)
-{
-	if (state)
-		return container_of(state, struct tegra_plane_state, base);
+	dev_WARN(plane->dc->dev, "invalid offset: %x\n", offset);
 
-	return NULL;
+	return plane->offset + offset;
 }
 
-static void tegra_dc_stats_reset(struct tegra_dc_stats *stats)
+static inline u32 tegra_plane_readl(struct tegra_plane *plane,
+				    unsigned int offset)
 {
-	stats->frames = 0;
-	stats->vblank = 0;
-	stats->underflow = 0;
-	stats->overflow = 0;
+	return tegra_dc_readl(plane->dc, tegra_plane_offset(plane, offset));
 }
 
-/*
- * Reads the active copy of a register. This takes the dc->lock spinlock to
- * prevent races with the VBLANK processing which also needs access to the
- * active copy of some registers.
- */
-static u32 tegra_dc_readl_active(struct tegra_dc *dc, unsigned long offset)
+static inline void tegra_plane_writel(struct tegra_plane *plane, u32 value,
+				      unsigned int offset)
 {
-	unsigned long flags;
-	u32 value;
+	tegra_dc_writel(plane->dc, value, tegra_plane_offset(plane, offset));
+}
 
-	spin_lock_irqsave(&dc->lock, flags);
+bool tegra_dc_has_output(struct tegra_dc *dc, struct device *dev)
+{
+	struct device_node *np = dc->dev->of_node;
+	struct of_phandle_iterator it;
+	int err;
 
-	tegra_dc_writel(dc, READ_MUX, DC_CMD_STATE_ACCESS);
-	value = tegra_dc_readl(dc, offset);
-	tegra_dc_writel(dc, 0, DC_CMD_STATE_ACCESS);
+	of_for_each_phandle(&it, err, np, "nvidia,outputs", NULL, 0)
+		if (it.node == dev->of_node)
+			return true;
 
-	spin_unlock_irqrestore(&dc->lock, flags);
-	return value;
+	return false;
 }
 
 /*
@@ -115,81 +112,6 @@ void tegra_dc_commit(struct tegra_dc *dc)
 	tegra_dc_writel(dc, GENERAL_ACT_REQ, DC_CMD_STATE_CONTROL);
 }
 
-static int tegra_dc_format(u32 fourcc, u32 *format, u32 *swap)
-{
-	/* assume no swapping of fetched data */
-	if (swap)
-		*swap = BYTE_SWAP_NOSWAP;
-
-	switch (fourcc) {
-	case DRM_FORMAT_XBGR8888:
-		*format = WIN_COLOR_DEPTH_R8G8B8A8;
-		break;
-
-	case DRM_FORMAT_XRGB8888:
-		*format = WIN_COLOR_DEPTH_B8G8R8A8;
-		break;
-
-	case DRM_FORMAT_RGB565:
-		*format = WIN_COLOR_DEPTH_B5G6R5;
-		break;
-
-	case DRM_FORMAT_UYVY:
-		*format = WIN_COLOR_DEPTH_YCbCr422;
-		break;
-
-	case DRM_FORMAT_YUYV:
-		if (swap)
-			*swap = BYTE_SWAP_SWAP2;
-
-		*format = WIN_COLOR_DEPTH_YCbCr422;
-		break;
-
-	case DRM_FORMAT_YUV420:
-		*format = WIN_COLOR_DEPTH_YCbCr420P;
-		break;
-
-	case DRM_FORMAT_YUV422:
-		*format = WIN_COLOR_DEPTH_YCbCr422P;
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static bool tegra_dc_format_is_yuv(unsigned int format, bool *planar)
-{
-	switch (format) {
-	case WIN_COLOR_DEPTH_YCbCr422:
-	case WIN_COLOR_DEPTH_YUV422:
-		if (planar)
-			*planar = false;
-
-		return true;
-
-	case WIN_COLOR_DEPTH_YCbCr420P:
-	case WIN_COLOR_DEPTH_YUV420P:
-	case WIN_COLOR_DEPTH_YCbCr422P:
-	case WIN_COLOR_DEPTH_YUV422P:
-	case WIN_COLOR_DEPTH_YCbCr422R:
-	case WIN_COLOR_DEPTH_YUV422R:
-	case WIN_COLOR_DEPTH_YCbCr422RA:
-	case WIN_COLOR_DEPTH_YUV422RA:
-		if (planar)
-			*planar = true;
-
-		return true;
-	}
-
-	if (planar)
-		*planar = false;
-
-	return false;
-}
-
 static inline u32 compute_dda_inc(unsigned int in, unsigned int out, bool v,
 				  unsigned int bpp)
 {
@@ -230,36 +152,104 @@ static inline u32 compute_initial_dda(unsigned int in)
 	return dfixed_frac(inf);
 }
 
-static void tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
+static void tegra_plane_setup_blending_legacy(struct tegra_plane *plane)
+{
+	u32 background[3] = {
+		BLEND_WEIGHT1(0) | BLEND_WEIGHT0(0) | BLEND_COLOR_KEY_NONE,
+		BLEND_WEIGHT1(0) | BLEND_WEIGHT0(0) | BLEND_COLOR_KEY_NONE,
+		BLEND_WEIGHT1(0) | BLEND_WEIGHT0(0) | BLEND_COLOR_KEY_NONE,
+	};
+	u32 foreground = BLEND_WEIGHT1(255) | BLEND_WEIGHT0(255) |
+			 BLEND_COLOR_KEY_NONE;
+	u32 blendnokey = BLEND_WEIGHT1(255) | BLEND_WEIGHT0(255);
+	struct tegra_plane_state *state;
+	unsigned int i;
+
+	state = to_tegra_plane_state(plane->base.state);
+
+	/* alpha contribution is 1 minus sum of overlapping windows */
+	for (i = 0; i < 3; i++) {
+		if (state->dependent[i])
+			background[i] |= BLEND_CONTROL_DEPENDENT;
+	}
+
+	/* enable alpha blending if pixel format has an alpha component */
+	if (!state->opaque)
+		foreground |= BLEND_CONTROL_ALPHA;
+
+	/*
+	 * Disable blending and assume Window A is the bottom-most window,
+	 * Window C is the top-most window and Window B is in the middle.
+	 */
+	tegra_plane_writel(plane, blendnokey, DC_WIN_BLEND_NOKEY);
+	tegra_plane_writel(plane, foreground, DC_WIN_BLEND_1WIN);
+
+	switch (plane->index) {
+	case 0:
+		tegra_plane_writel(plane, background[0], DC_WIN_BLEND_2WIN_X);
+		tegra_plane_writel(plane, background[1], DC_WIN_BLEND_2WIN_Y);
+		tegra_plane_writel(plane, background[2], DC_WIN_BLEND_3WIN_XY);
+		break;
+
+	case 1:
+		tegra_plane_writel(plane, foreground, DC_WIN_BLEND_2WIN_X);
+		tegra_plane_writel(plane, background[1], DC_WIN_BLEND_2WIN_Y);
+		tegra_plane_writel(plane, background[2], DC_WIN_BLEND_3WIN_XY);
+		break;
+
+	case 2:
+		tegra_plane_writel(plane, foreground, DC_WIN_BLEND_2WIN_X);
+		tegra_plane_writel(plane, foreground, DC_WIN_BLEND_2WIN_Y);
+		tegra_plane_writel(plane, foreground, DC_WIN_BLEND_3WIN_XY);
+		break;
+	}
+}
+
+static void tegra_plane_setup_blending(struct tegra_plane *plane,
+				       const struct tegra_dc_window *window)
+{
+	u32 value;
+
+	value = BLEND_FACTOR_DST_ALPHA_ZERO | BLEND_FACTOR_SRC_ALPHA_K2 |
+		BLEND_FACTOR_DST_COLOR_NEG_K1_TIMES_SRC |
+		BLEND_FACTOR_SRC_COLOR_K1_TIMES_SRC;
+	tegra_plane_writel(plane, value, DC_WIN_BLEND_MATCH_SELECT);
+
+	value = BLEND_FACTOR_DST_ALPHA_ZERO | BLEND_FACTOR_SRC_ALPHA_K2 |
+		BLEND_FACTOR_DST_COLOR_NEG_K1_TIMES_SRC |
+		BLEND_FACTOR_SRC_COLOR_K1_TIMES_SRC;
+	tegra_plane_writel(plane, value, DC_WIN_BLEND_NOMATCH_SELECT);
+
+	value = K2(255) | K1(255) | WINDOW_LAYER_DEPTH(255 - window->zpos);
+	tegra_plane_writel(plane, value, DC_WIN_BLEND_LAYER_CONTROL);
+}
+
+static void tegra_dc_setup_window(struct tegra_plane *plane,
 				  const struct tegra_dc_window *window)
 {
 	unsigned h_offset, v_offset, h_size, v_size, h_dda, v_dda, bpp;
-	unsigned long value, flags;
+	struct tegra_dc *dc = plane->dc;
 	bool yuv, planar;
+	u32 value;
 
 	/*
 	 * For YUV planar modes, the number of bytes per pixel takes into
 	 * account only the luma component and therefore is 1.
 	 */
-	yuv = tegra_dc_format_is_yuv(window->format, &planar);
+	yuv = tegra_plane_format_is_yuv(window->format, &planar);
 	if (!yuv)
 		bpp = window->bits_per_pixel / 8;
 	else
 		bpp = planar ? 1 : 2;
 
-	spin_lock_irqsave(&dc->lock, flags);
-
-	value = WINDOW_A_SELECT << index;
-	tegra_dc_writel(dc, value, DC_CMD_DISPLAY_WINDOW_HEADER);
-
-	tegra_dc_writel(dc, window->format, DC_WIN_COLOR_DEPTH);
-	tegra_dc_writel(dc, window->swap, DC_WIN_BYTE_SWAP);
+	tegra_plane_writel(plane, window->format, DC_WIN_COLOR_DEPTH);
+	tegra_plane_writel(plane, window->swap, DC_WIN_BYTE_SWAP);
 
 	value = V_POSITION(window->dst.y) | H_POSITION(window->dst.x);
-	tegra_dc_writel(dc, value, DC_WIN_POSITION);
+	tegra_plane_writel(plane, value, DC_WIN_POSITION);
 
 	value = V_SIZE(window->dst.h) | H_SIZE(window->dst.w);
-	tegra_dc_writel(dc, value, DC_WIN_SIZE);
+	tegra_plane_writel(plane, value, DC_WIN_SIZE);
 
 	h_offset = window->src.x * bpp;
 	v_offset = window->src.y;
@@ -267,7 +257,7 @@ static void tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
 	v_size = window->src.h;
 
 	value = V_PRESCALED_SIZE(v_size) | H_PRESCALED_SIZE(h_size);
-	tegra_dc_writel(dc, value, DC_WIN_PRESCALED_SIZE);
+	tegra_plane_writel(plane, value, DC_WIN_PRESCALED_SIZE);
 
 	/*
 	 * For DDA computations the number of bytes per pixel for YUV planar
@@ -280,33 +270,33 @@ static void tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
 	v_dda = compute_dda_inc(window->src.h, window->dst.h, true, bpp);
 
 	value = V_DDA_INC(v_dda) | H_DDA_INC(h_dda);
-	tegra_dc_writel(dc, value, DC_WIN_DDA_INC);
+	tegra_plane_writel(plane, value, DC_WIN_DDA_INC);
 
 	h_dda = compute_initial_dda(window->src.x);
 	v_dda = compute_initial_dda(window->src.y);
 
-	tegra_dc_writel(dc, h_dda, DC_WIN_H_INITIAL_DDA);
-	tegra_dc_writel(dc, v_dda, DC_WIN_V_INITIAL_DDA);
+	tegra_plane_writel(plane, h_dda, DC_WIN_H_INITIAL_DDA);
+	tegra_plane_writel(plane, v_dda, DC_WIN_V_INITIAL_DDA);
 
-	tegra_dc_writel(dc, 0, DC_WIN_UV_BUF_STRIDE);
-	tegra_dc_writel(dc, 0, DC_WIN_BUF_STRIDE);
+	tegra_plane_writel(plane, 0, DC_WIN_UV_BUF_STRIDE);
+	tegra_plane_writel(plane, 0, DC_WIN_BUF_STRIDE);
 
-	tegra_dc_writel(dc, window->base[0], DC_WINBUF_START_ADDR);
+	tegra_plane_writel(plane, window->base[0], DC_WINBUF_START_ADDR);
 
 	if (yuv && planar) {
-		tegra_dc_writel(dc, window->base[1], DC_WINBUF_START_ADDR_U);
-		tegra_dc_writel(dc, window->base[2], DC_WINBUF_START_ADDR_V);
+		tegra_plane_writel(plane, window->base[1], DC_WINBUF_START_ADDR_U);
+		tegra_plane_writel(plane, window->base[2], DC_WINBUF_START_ADDR_V);
 		value = window->stride[1] << 16 | window->stride[0];
-		tegra_dc_writel(dc, value, DC_WIN_LINE_STRIDE);
+		tegra_plane_writel(plane, value, DC_WIN_LINE_STRIDE);
 	} else {
-		tegra_dc_writel(dc, window->stride[0], DC_WIN_LINE_STRIDE);
+		tegra_plane_writel(plane, window->stride[0], DC_WIN_LINE_STRIDE);
 	}
 
 	if (window->bottom_up)
 		v_offset += window->src.h - 1;
 
-	tegra_dc_writel(dc, h_offset, DC_WINBUF_ADDR_H_OFFSET);
-	tegra_dc_writel(dc, v_offset, DC_WINBUF_ADDR_V_OFFSET);
+	tegra_plane_writel(plane, h_offset, DC_WINBUF_ADDR_H_OFFSET);
+	tegra_plane_writel(plane, v_offset, DC_WINBUF_ADDR_V_OFFSET);
 
 	if (dc->soc->supports_block_linear) {
 		unsigned long height = window->tiling.value;
@@ -326,7 +316,7 @@ static void tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
 			break;
 		}
 
-		tegra_dc_writel(dc, value, DC_WINBUF_SURFACE_KIND);
+		tegra_plane_writel(plane, value, DC_WINBUF_SURFACE_KIND);
 	} else {
 		switch (window->tiling.mode) {
 		case TEGRA_BO_TILING_MODE_PITCH:
@@ -347,21 +337,21 @@ static void tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
 			break;
 		}
 
-		tegra_dc_writel(dc, value, DC_WIN_BUFFER_ADDR_MODE);
+		tegra_plane_writel(plane, value, DC_WIN_BUFFER_ADDR_MODE);
 	}
 
 	value = WIN_ENABLE;
 
 	if (yuv) {
 		/* setup default colorspace conversion coefficients */
-		tegra_dc_writel(dc, 0x00f0, DC_WIN_CSC_YOF);
-		tegra_dc_writel(dc, 0x012a, DC_WIN_CSC_KYRGB);
-		tegra_dc_writel(dc, 0x0000, DC_WIN_CSC_KUR);
-		tegra_dc_writel(dc, 0x0198, DC_WIN_CSC_KVR);
-		tegra_dc_writel(dc, 0x039b, DC_WIN_CSC_KUG);
-		tegra_dc_writel(dc, 0x032f, DC_WIN_CSC_KVG);
-		tegra_dc_writel(dc, 0x0204, DC_WIN_CSC_KUB);
-		tegra_dc_writel(dc, 0x0000, DC_WIN_CSC_KVB);
+		tegra_plane_writel(plane, 0x00f0, DC_WIN_CSC_YOF);
+		tegra_plane_writel(plane, 0x012a, DC_WIN_CSC_KYRGB);
+		tegra_plane_writel(plane, 0x0000, DC_WIN_CSC_KUR);
+		tegra_plane_writel(plane, 0x0198, DC_WIN_CSC_KVR);
+		tegra_plane_writel(plane, 0x039b, DC_WIN_CSC_KUG);
+		tegra_plane_writel(plane, 0x032f, DC_WIN_CSC_KVG);
+		tegra_plane_writel(plane, 0x0204, DC_WIN_CSC_KUB);
+		tegra_plane_writel(plane, 0x0000, DC_WIN_CSC_KVB);
 
 		value |= CSC_ENABLE;
 	} else if (window->bits_per_pixel < 24) {
@@ -371,137 +361,74 @@ static void tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
 	if (window->bottom_up)
 		value |= V_DIRECTION;
 
-	tegra_dc_writel(dc, value, DC_WIN_WIN_OPTIONS);
+	tegra_plane_writel(plane, value, DC_WIN_WIN_OPTIONS);
 
-	/*
-	 * Disable blending and assume Window A is the bottom-most window,
-	 * Window C is the top-most window and Window B is in the middle.
-	 */
-	tegra_dc_writel(dc, 0xffff00, DC_WIN_BLEND_NOKEY);
-	tegra_dc_writel(dc, 0xffff00, DC_WIN_BLEND_1WIN);
-
-	switch (index) {
-	case 0:
-		tegra_dc_writel(dc, 0x000000, DC_WIN_BLEND_2WIN_X);
-		tegra_dc_writel(dc, 0x000000, DC_WIN_BLEND_2WIN_Y);
-		tegra_dc_writel(dc, 0x000000, DC_WIN_BLEND_3WIN_XY);
-		break;
-
-	case 1:
-		tegra_dc_writel(dc, 0xffff00, DC_WIN_BLEND_2WIN_X);
-		tegra_dc_writel(dc, 0x000000, DC_WIN_BLEND_2WIN_Y);
-		tegra_dc_writel(dc, 0x000000, DC_WIN_BLEND_3WIN_XY);
-		break;
-
-	case 2:
-		tegra_dc_writel(dc, 0xffff00, DC_WIN_BLEND_2WIN_X);
-		tegra_dc_writel(dc, 0xffff00, DC_WIN_BLEND_2WIN_Y);
-		tegra_dc_writel(dc, 0xffff00, DC_WIN_BLEND_3WIN_XY);
-		break;
-	}
-
-	spin_unlock_irqrestore(&dc->lock, flags);
-}
-
-static void tegra_plane_destroy(struct drm_plane *plane)
-{
-	struct tegra_plane *p = to_tegra_plane(plane);
-
-	drm_plane_cleanup(plane);
-	kfree(p);
+	if (dc->soc->supports_blending)
+		tegra_plane_setup_blending(plane, window);
+	else
+		tegra_plane_setup_blending_legacy(plane);
 }
 
-static const u32 tegra_primary_plane_formats[] = {
+static const u32 tegra20_primary_formats[] = {
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB8888,
+	/* non-native formats */
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGBX5551,
 	DRM_FORMAT_XBGR8888,
 	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB565,
 };
 
-static void tegra_primary_plane_destroy(struct drm_plane *plane)
-{
-	tegra_plane_destroy(plane);
-}
-
-static void tegra_plane_reset(struct drm_plane *plane)
-{
-	struct tegra_plane_state *state;
-
-	if (plane->state)
-		__drm_atomic_helper_plane_destroy_state(plane->state);
-
-	kfree(plane->state);
-	plane->state = NULL;
-
-	state = kzalloc(sizeof(*state), GFP_KERNEL);
-	if (state) {
-		plane->state = &state->base;
-		plane->state->plane = plane;
-	}
-}
-
-static struct drm_plane_state *tegra_plane_atomic_duplicate_state(struct drm_plane *plane)
-{
-	struct tegra_plane_state *state = to_tegra_plane_state(plane->state);
-	struct tegra_plane_state *copy;
-
-	copy = kmalloc(sizeof(*copy), GFP_KERNEL);
-	if (!copy)
-		return NULL;
-
-	__drm_atomic_helper_plane_duplicate_state(plane, &copy->base);
-	copy->tiling = state->tiling;
-	copy->format = state->format;
-	copy->swap = state->swap;
-
-	return &copy->base;
-}
-
-static void tegra_plane_atomic_destroy_state(struct drm_plane *plane,
-					     struct drm_plane_state *state)
-{
-	__drm_atomic_helper_plane_destroy_state(state);
-	kfree(state);
-}
-
-static const struct drm_plane_funcs tegra_primary_plane_funcs = {
-	.update_plane = drm_atomic_helper_update_plane,
-	.disable_plane = drm_atomic_helper_disable_plane,
-	.destroy = tegra_primary_plane_destroy,
-	.reset = tegra_plane_reset,
-	.atomic_duplicate_state = tegra_plane_atomic_duplicate_state,
-	.atomic_destroy_state = tegra_plane_atomic_destroy_state,
+static const u32 tegra114_primary_formats[] = {
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB8888,
+	/* new on Tegra114 */
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_XBGR1555,
+	DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
 };
 
-static int tegra_plane_state_add(struct tegra_plane *plane,
-				 struct drm_plane_state *state)
-{
-	struct drm_crtc_state *crtc_state;
-	struct tegra_dc_state *tegra;
-	struct drm_rect clip;
-	int err;
-
-	/* Propagate errors from allocation or locking failures. */
-	crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc);
-	if (IS_ERR(crtc_state))
-		return PTR_ERR(crtc_state);
-
-	clip.x1 = 0;
-	clip.y1 = 0;
-	clip.x2 = crtc_state->mode.hdisplay;
-	clip.y2 = crtc_state->mode.vdisplay;
-
-	/* Check plane state for visibility and calculate clipping bounds */
-	err = drm_plane_helper_check_state(state, &clip, 0, INT_MAX,
-					   true, true);
-	if (err < 0)
-		return err;
-
-	tegra = to_dc_state(crtc_state);
-
-	tegra->planes |= WIN_A_ACT_REQ << plane->index;
-
-	return 0;
-}
+static const u32 tegra124_primary_formats[] = {
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB8888,
+	/* new on Tegra114 */
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_XBGR1555,
+	DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	/* new on Tegra124 */
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+};
 
 static int tegra_plane_atomic_check(struct drm_plane *plane,
 				    struct drm_plane_state *state)
@@ -510,17 +437,40 @@ static int tegra_plane_atomic_check(struct drm_plane *plane,
 	struct tegra_bo_tiling *tiling = &plane_state->tiling;
 	struct tegra_plane *tegra = to_tegra_plane(plane);
 	struct tegra_dc *dc = to_tegra_dc(state->crtc);
+	unsigned int format;
 	int err;
 
 	/* no need for further checks if the plane is being disabled */
 	if (!state->crtc)
 		return 0;
 
-	err = tegra_dc_format(state->fb->format->format, &plane_state->format,
-			      &plane_state->swap);
+	err = tegra_plane_format(state->fb->format->format, &format,
+				 &plane_state->swap);
 	if (err < 0)
 		return err;
 
+	/*
+	 * Tegra20 and Tegra30 are special cases here because they support
+	 * only variants of specific formats with an alpha component, but not
+	 * the corresponding opaque formats. However, the opaque formats can
+	 * be emulated by disabling alpha blending for the plane.
+	 */
+	if (!dc->soc->supports_blending) {
+		if (!tegra_plane_format_has_alpha(format)) {
+			err = tegra_plane_format_get_alpha(format, &format);
+			if (err < 0)
+				return err;
+
+			plane_state->opaque = true;
+		} else {
+			plane_state->opaque = false;
+		}
+
+		tegra_plane_check_dependent(tegra, plane_state);
+	}
+
+	plane_state->format = format;
+
 	err = tegra_fb_get_tiling(state->fb, tiling);
 	if (err < 0)
 		return err;
@@ -553,32 +503,22 @@ static int tegra_plane_atomic_check(struct drm_plane *plane,
 static void tegra_plane_atomic_disable(struct drm_plane *plane,
 				       struct drm_plane_state *old_state)
 {
-	struct tegra_dc *dc = to_tegra_dc(old_state->crtc);
 	struct tegra_plane *p = to_tegra_plane(plane);
-	unsigned long flags;
 	u32 value;
 
 	/* rien ne va plus */
 	if (!old_state || !old_state->crtc)
 		return;
 
-	spin_lock_irqsave(&dc->lock, flags);
-
-	value = WINDOW_A_SELECT << p->index;
-	tegra_dc_writel(dc, value, DC_CMD_DISPLAY_WINDOW_HEADER);
-
-	value = tegra_dc_readl(dc, DC_WIN_WIN_OPTIONS);
+	value = tegra_plane_readl(p, DC_WIN_WIN_OPTIONS);
 	value &= ~WIN_ENABLE;
-	tegra_dc_writel(dc, value, DC_WIN_WIN_OPTIONS);
-
-	spin_unlock_irqrestore(&dc->lock, flags);
+	tegra_plane_writel(p, value, DC_WIN_WIN_OPTIONS);
 }
 
 static void tegra_plane_atomic_update(struct drm_plane *plane,
 				      struct drm_plane_state *old_state)
 {
 	struct tegra_plane_state *state = to_tegra_plane_state(plane->state);
-	struct tegra_dc *dc = to_tegra_dc(plane->state->crtc);
 	struct drm_framebuffer *fb = plane->state->fb;
 	struct tegra_plane *p = to_tegra_plane(plane);
 	struct tegra_dc_window window;
@@ -604,6 +544,7 @@ static void tegra_plane_atomic_update(struct drm_plane *plane,
 	window.bottom_up = tegra_fb_is_bottom_up(fb);
 
 	/* copy from state */
+	window.zpos = plane->state->normalized_zpos;
 	window.tiling = state->tiling;
 	window.format = state->format;
 	window.swap = state->swap;
@@ -622,7 +563,7 @@ static void tegra_plane_atomic_update(struct drm_plane *plane,
 			window.stride[i] = fb->pitches[i];
 	}
 
-	tegra_dc_setup_window(dc, p->index, &window);
+	tegra_dc_setup_window(p, &window);
 }
 
 static const struct drm_plane_helper_funcs tegra_plane_helper_funcs = {
@@ -631,8 +572,7 @@ static const struct drm_plane_helper_funcs tegra_plane_helper_funcs = {
 	.atomic_update = tegra_plane_atomic_update,
 };
 
-static struct drm_plane *tegra_dc_primary_plane_create(struct drm_device *drm,
-						       struct tegra_dc *dc)
+static unsigned long tegra_plane_get_possible_crtcs(struct drm_device *drm)
 {
 	/*
 	 * Ideally this would use drm_crtc_mask(), but that would require the
@@ -646,7 +586,14 @@ static struct drm_plane *tegra_dc_primary_plane_create(struct drm_device *drm,
 	 * of CRTCs that have been registered, and should therefore always be
 	 * the same as drm_crtc_index() after registration.
 	 */
-	unsigned long possible_crtcs = 1 << drm->mode_config.num_crtc;
+	return 1 << drm->mode_config.num_crtc;
+}
+
+static struct drm_plane *tegra_primary_plane_create(struct drm_device *drm,
+						    struct tegra_dc *dc)
+{
+	unsigned long possible_crtcs = tegra_plane_get_possible_crtcs(drm);
+	enum drm_plane_type type = DRM_PLANE_TYPE_PRIMARY;
 	struct tegra_plane *plane;
 	unsigned int num_formats;
 	const u32 *formats;
@@ -656,13 +603,17 @@ static struct drm_plane *tegra_dc_primary_plane_create(struct drm_device *drm,
 	if (!plane)
 		return ERR_PTR(-ENOMEM);
 
-	num_formats = ARRAY_SIZE(tegra_primary_plane_formats);
-	formats = tegra_primary_plane_formats;
+	/* Always use window A as primary window */
+	plane->offset = 0xa00;
+	plane->index = 0;
+	plane->dc = dc;
+
+	num_formats = dc->soc->num_primary_formats;
+	formats = dc->soc->primary_formats;
 
 	err = drm_universal_plane_init(drm, &plane->base, possible_crtcs,
-				       &tegra_primary_plane_funcs, formats,
-				       num_formats, NULL,
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
+				       &tegra_plane_funcs, formats,
+				       num_formats, NULL, type, NULL);
 	if (err < 0) {
 		kfree(plane);
 		return ERR_PTR(err);
@@ -670,6 +621,9 @@ static struct drm_plane *tegra_dc_primary_plane_create(struct drm_device *drm,
 
 	drm_plane_helper_add(&plane->base, &tegra_plane_helper_funcs);
 
+	if (dc->soc->supports_blending)
+		drm_plane_create_zpos_property(&plane->base, 0, 0, 255);
+
 	return &plane->base;
 }
 
@@ -786,15 +740,6 @@ static void tegra_cursor_atomic_disable(struct drm_plane *plane,
 	tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
 }
 
-static const struct drm_plane_funcs tegra_cursor_plane_funcs = {
-	.update_plane = drm_atomic_helper_update_plane,
-	.disable_plane = drm_atomic_helper_disable_plane,
-	.destroy = tegra_plane_destroy,
-	.reset = tegra_plane_reset,
-	.atomic_duplicate_state = tegra_plane_atomic_duplicate_state,
-	.atomic_destroy_state = tegra_plane_atomic_destroy_state,
-};
-
 static const struct drm_plane_helper_funcs tegra_cursor_plane_helper_funcs = {
 	.atomic_check = tegra_cursor_atomic_check,
 	.atomic_update = tegra_cursor_atomic_update,
@@ -804,6 +749,7 @@ static const struct drm_plane_helper_funcs tegra_cursor_plane_helper_funcs = {
 static struct drm_plane *tegra_dc_cursor_plane_create(struct drm_device *drm,
 						      struct tegra_dc *dc)
 {
+	unsigned long possible_crtcs = tegra_plane_get_possible_crtcs(drm);
 	struct tegra_plane *plane;
 	unsigned int num_formats;
 	const u32 *formats;
@@ -821,12 +767,13 @@ static struct drm_plane *tegra_dc_cursor_plane_create(struct drm_device *drm,
 	 * need to special-casing the cursor plane.
 	 */
 	plane->index = 6;
+	plane->dc = dc;
 
 	num_formats = ARRAY_SIZE(tegra_cursor_plane_formats);
 	formats = tegra_cursor_plane_formats;
 
-	err = drm_universal_plane_init(drm, &plane->base, 1 << dc->pipe,
-				       &tegra_cursor_plane_funcs, formats,
+	err = drm_universal_plane_init(drm, &plane->base, possible_crtcs,
+				       &tegra_plane_funcs, formats,
 				       num_formats, NULL,
 				       DRM_PLANE_TYPE_CURSOR, NULL);
 	if (err < 0) {
@@ -839,24 +786,76 @@ static struct drm_plane *tegra_dc_cursor_plane_create(struct drm_device *drm,
 	return &plane->base;
 }
 
-static void tegra_overlay_plane_destroy(struct drm_plane *plane)
-{
-	tegra_plane_destroy(plane);
-}
-
-static const struct drm_plane_funcs tegra_overlay_plane_funcs = {
-	.update_plane = drm_atomic_helper_update_plane,
-	.disable_plane = drm_atomic_helper_disable_plane,
-	.destroy = tegra_overlay_plane_destroy,
-	.reset = tegra_plane_reset,
-	.atomic_duplicate_state = tegra_plane_atomic_duplicate_state,
-	.atomic_destroy_state = tegra_plane_atomic_destroy_state,
+static const u32 tegra20_overlay_formats[] = {
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB8888,
+	/* non-native formats */
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_XRGB8888,
+	/* planar formats */
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YUV420,
+	DRM_FORMAT_YUV422,
 };
 
-static const uint32_t tegra_overlay_plane_formats[] = {
-	DRM_FORMAT_XBGR8888,
+static const u32 tegra114_overlay_formats[] = {
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB8888,
+	/* new on Tegra114 */
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_XBGR1555,
+	DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_RGBA8888,
 	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	/* planar formats */
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YUV420,
+	DRM_FORMAT_YUV422,
+};
+
+static const u32 tegra124_overlay_formats[] = {
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_ARGB1555,
 	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_ARGB8888,
+	/* new on Tegra114 */
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_XBGR1555,
+	DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	/* new on Tegra124 */
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+	/* planar formats */
 	DRM_FORMAT_UYVY,
 	DRM_FORMAT_YUYV,
 	DRM_FORMAT_YUV420,
@@ -867,6 +866,7 @@ static struct drm_plane *tegra_dc_overlay_plane_create(struct drm_device *drm,
 						       struct tegra_dc *dc,
 						       unsigned int index)
 {
+	unsigned long possible_crtcs = tegra_plane_get_possible_crtcs(drm);
 	struct tegra_plane *plane;
 	unsigned int num_formats;
 	const u32 *formats;
@@ -876,13 +876,15 @@ static struct drm_plane *tegra_dc_overlay_plane_create(struct drm_device *drm,
 	if (!plane)
 		return ERR_PTR(-ENOMEM);
 
+	plane->offset = 0xa00 + 0x200 * index;
 	plane->index = index;
+	plane->dc = dc;
 
-	num_formats = ARRAY_SIZE(tegra_overlay_plane_formats);
-	formats = tegra_overlay_plane_formats;
+	num_formats = dc->soc->num_overlay_formats;
+	formats = dc->soc->overlay_formats;
 
-	err = drm_universal_plane_init(drm, &plane->base, 1 << dc->pipe,
-				       &tegra_overlay_plane_funcs, formats,
+	err = drm_universal_plane_init(drm, &plane->base, possible_crtcs,
+				       &tegra_plane_funcs, formats,
 				       num_formats, NULL,
 				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (err < 0) {
@@ -892,147 +894,494 @@ static struct drm_plane *tegra_dc_overlay_plane_create(struct drm_device *drm,
 
 	drm_plane_helper_add(&plane->base, &tegra_plane_helper_funcs);
 
+	if (dc->soc->supports_blending)
+		drm_plane_create_zpos_property(&plane->base, 0, 0, 255);
+
 	return &plane->base;
 }
 
-static int tegra_dc_add_planes(struct drm_device *drm, struct tegra_dc *dc)
+static struct drm_plane *tegra_dc_add_shared_planes(struct drm_device *drm,
+						    struct tegra_dc *dc)
 {
-	struct drm_plane *plane;
+	struct drm_plane *plane, *primary = NULL;
+	unsigned int i, j;
+
+	for (i = 0; i < dc->soc->num_wgrps; i++) {
+		const struct tegra_windowgroup_soc *wgrp = &dc->soc->wgrps[i];
+
+		if (wgrp->dc == dc->pipe) {
+			for (j = 0; j < wgrp->num_windows; j++) {
+				unsigned int index = wgrp->windows[j];
+
+				plane = tegra_shared_plane_create(drm, dc,
+								  wgrp->index,
+								  index);
+				if (IS_ERR(plane))
+					return plane;
+
+				/*
+				 * Choose the first shared plane owned by this
+				 * head as the primary plane.
+				 */
+				if (!primary) {
+					plane->type = DRM_PLANE_TYPE_PRIMARY;
+					primary = plane;
+				}
+			}
+		}
+	}
+
+	return primary;
+}
+
+static struct drm_plane *tegra_dc_add_planes(struct drm_device *drm,
+					     struct tegra_dc *dc)
+{
+	struct drm_plane *planes[2], *primary;
 	unsigned int i;
+	int err;
+
+	primary = tegra_primary_plane_create(drm, dc);
+	if (IS_ERR(primary))
+		return primary;
 
 	for (i = 0; i < 2; i++) {
-		plane = tegra_dc_overlay_plane_create(drm, dc, 1 + i);
-		if (IS_ERR(plane))
-			return PTR_ERR(plane);
+		planes[i] = tegra_dc_overlay_plane_create(drm, dc, 1 + i);
+		if (IS_ERR(planes[i])) {
+			err = PTR_ERR(planes[i]);
+
+			while (i--)
+				tegra_plane_funcs.destroy(planes[i]);
+
+			tegra_plane_funcs.destroy(primary);
+			return ERR_PTR(err);
+		}
 	}
 
-	return 0;
+	return primary;
 }
 
-static u32 tegra_dc_get_vblank_counter(struct drm_crtc *crtc)
+static void tegra_dc_destroy(struct drm_crtc *crtc)
 {
-	struct tegra_dc *dc = to_tegra_dc(crtc);
-
-	if (dc->syncpt)
-		return host1x_syncpt_read(dc->syncpt);
-
-	/* fallback to software emulated VBLANK counter */
-	return drm_crtc_vblank_count(&dc->base);
+	drm_crtc_cleanup(crtc);
 }
 
-static int tegra_dc_enable_vblank(struct drm_crtc *crtc)
+static void tegra_crtc_reset(struct drm_crtc *crtc)
 {
-	struct tegra_dc *dc = to_tegra_dc(crtc);
-	unsigned long value, flags;
+	struct tegra_dc_state *state;
 
-	spin_lock_irqsave(&dc->lock, flags);
+	if (crtc->state)
+		__drm_atomic_helper_crtc_destroy_state(crtc->state);
 
-	value = tegra_dc_readl(dc, DC_CMD_INT_MASK);
-	value |= VBLANK_INT;
-	tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
+	kfree(crtc->state);
+	crtc->state = NULL;
 
-	spin_unlock_irqrestore(&dc->lock, flags);
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state) {
+		crtc->state = &state->base;
+		crtc->state->crtc = crtc;
+	}
 
-	return 0;
+	drm_crtc_vblank_reset(crtc);
 }
 
-static void tegra_dc_disable_vblank(struct drm_crtc *crtc)
+static struct drm_crtc_state *
+tegra_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
 {
-	struct tegra_dc *dc = to_tegra_dc(crtc);
-	unsigned long value, flags;
+	struct tegra_dc_state *state = to_dc_state(crtc->state);
+	struct tegra_dc_state *copy;
 
-	spin_lock_irqsave(&dc->lock, flags);
+	copy = kmalloc(sizeof(*copy), GFP_KERNEL);
+	if (!copy)
+		return NULL;
 
-	value = tegra_dc_readl(dc, DC_CMD_INT_MASK);
-	value &= ~VBLANK_INT;
-	tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &copy->base);
+	copy->clk = state->clk;
+	copy->pclk = state->pclk;
+	copy->div = state->div;
+	copy->planes = state->planes;
 
-	spin_unlock_irqrestore(&dc->lock, flags);
+	return &copy->base;
 }
 
-static void tegra_dc_finish_page_flip(struct tegra_dc *dc)
+static void tegra_crtc_atomic_destroy_state(struct drm_crtc *crtc,
+					    struct drm_crtc_state *state)
 {
-	struct drm_device *drm = dc->base.dev;
-	struct drm_crtc *crtc = &dc->base;
-	unsigned long flags, base;
-	struct tegra_bo *bo;
+	__drm_atomic_helper_crtc_destroy_state(state);
+	kfree(state);
+}
 
-	spin_lock_irqsave(&drm->event_lock, flags);
+#define DEBUGFS_REG32(_name) { .name = #_name, .offset = _name }
+
+static const struct debugfs_reg32 tegra_dc_regs[] = {
+	DEBUGFS_REG32(DC_CMD_GENERAL_INCR_SYNCPT),
+	DEBUGFS_REG32(DC_CMD_GENERAL_INCR_SYNCPT_CNTRL),
+	DEBUGFS_REG32(DC_CMD_GENERAL_INCR_SYNCPT_ERROR),
+	DEBUGFS_REG32(DC_CMD_WIN_A_INCR_SYNCPT),
+	DEBUGFS_REG32(DC_CMD_WIN_A_INCR_SYNCPT_CNTRL),
+	DEBUGFS_REG32(DC_CMD_WIN_A_INCR_SYNCPT_ERROR),
+	DEBUGFS_REG32(DC_CMD_WIN_B_INCR_SYNCPT),
+	DEBUGFS_REG32(DC_CMD_WIN_B_INCR_SYNCPT_CNTRL),
+	DEBUGFS_REG32(DC_CMD_WIN_B_INCR_SYNCPT_ERROR),
+	DEBUGFS_REG32(DC_CMD_WIN_C_INCR_SYNCPT),
+	DEBUGFS_REG32(DC_CMD_WIN_C_INCR_SYNCPT_CNTRL),
+	DEBUGFS_REG32(DC_CMD_WIN_C_INCR_SYNCPT_ERROR),
+	DEBUGFS_REG32(DC_CMD_CONT_SYNCPT_VSYNC),
+	DEBUGFS_REG32(DC_CMD_DISPLAY_COMMAND_OPTION0),
+	DEBUGFS_REG32(DC_CMD_DISPLAY_COMMAND),
+	DEBUGFS_REG32(DC_CMD_SIGNAL_RAISE),
+	DEBUGFS_REG32(DC_CMD_DISPLAY_POWER_CONTROL),
+	DEBUGFS_REG32(DC_CMD_INT_STATUS),
+	DEBUGFS_REG32(DC_CMD_INT_MASK),
+	DEBUGFS_REG32(DC_CMD_INT_ENABLE),
+	DEBUGFS_REG32(DC_CMD_INT_TYPE),
+	DEBUGFS_REG32(DC_CMD_INT_POLARITY),
+	DEBUGFS_REG32(DC_CMD_SIGNAL_RAISE1),
+	DEBUGFS_REG32(DC_CMD_SIGNAL_RAISE2),
+	DEBUGFS_REG32(DC_CMD_SIGNAL_RAISE3),
+	DEBUGFS_REG32(DC_CMD_STATE_ACCESS),
+	DEBUGFS_REG32(DC_CMD_STATE_CONTROL),
+	DEBUGFS_REG32(DC_CMD_DISPLAY_WINDOW_HEADER),
+	DEBUGFS_REG32(DC_CMD_REG_ACT_CONTROL),
+	DEBUGFS_REG32(DC_COM_CRC_CONTROL),
+	DEBUGFS_REG32(DC_COM_CRC_CHECKSUM),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_ENABLE(0)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_ENABLE(1)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_ENABLE(2)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_ENABLE(3)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_POLARITY(0)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_POLARITY(1)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_POLARITY(2)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_POLARITY(3)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_DATA(0)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_DATA(1)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_DATA(2)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_DATA(3)),
+	DEBUGFS_REG32(DC_COM_PIN_INPUT_ENABLE(0)),
+	DEBUGFS_REG32(DC_COM_PIN_INPUT_ENABLE(1)),
+	DEBUGFS_REG32(DC_COM_PIN_INPUT_ENABLE(2)),
+	DEBUGFS_REG32(DC_COM_PIN_INPUT_ENABLE(3)),
+	DEBUGFS_REG32(DC_COM_PIN_INPUT_DATA(0)),
+	DEBUGFS_REG32(DC_COM_PIN_INPUT_DATA(1)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_SELECT(0)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_SELECT(1)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_SELECT(2)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_SELECT(3)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_SELECT(4)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_SELECT(5)),
+	DEBUGFS_REG32(DC_COM_PIN_OUTPUT_SELECT(6)),
+	DEBUGFS_REG32(DC_COM_PIN_MISC_CONTROL),
+	DEBUGFS_REG32(DC_COM_PIN_PM0_CONTROL),
+	DEBUGFS_REG32(DC_COM_PIN_PM0_DUTY_CYCLE),
+	DEBUGFS_REG32(DC_COM_PIN_PM1_CONTROL),
+	DEBUGFS_REG32(DC_COM_PIN_PM1_DUTY_CYCLE),
+	DEBUGFS_REG32(DC_COM_SPI_CONTROL),
+	DEBUGFS_REG32(DC_COM_SPI_START_BYTE),
+	DEBUGFS_REG32(DC_COM_HSPI_WRITE_DATA_AB),
+	DEBUGFS_REG32(DC_COM_HSPI_WRITE_DATA_CD),
+	DEBUGFS_REG32(DC_COM_HSPI_CS_DC),
+	DEBUGFS_REG32(DC_COM_SCRATCH_REGISTER_A),
+	DEBUGFS_REG32(DC_COM_SCRATCH_REGISTER_B),
+	DEBUGFS_REG32(DC_COM_GPIO_CTRL),
+	DEBUGFS_REG32(DC_COM_GPIO_DEBOUNCE_COUNTER),
+	DEBUGFS_REG32(DC_COM_CRC_CHECKSUM_LATCHED),
+	DEBUGFS_REG32(DC_DISP_DISP_SIGNAL_OPTIONS0),
+	DEBUGFS_REG32(DC_DISP_DISP_SIGNAL_OPTIONS1),
+	DEBUGFS_REG32(DC_DISP_DISP_WIN_OPTIONS),
+	DEBUGFS_REG32(DC_DISP_DISP_MEM_HIGH_PRIORITY),
+	DEBUGFS_REG32(DC_DISP_DISP_MEM_HIGH_PRIORITY_TIMER),
+	DEBUGFS_REG32(DC_DISP_DISP_TIMING_OPTIONS),
+	DEBUGFS_REG32(DC_DISP_REF_TO_SYNC),
+	DEBUGFS_REG32(DC_DISP_SYNC_WIDTH),
+	DEBUGFS_REG32(DC_DISP_BACK_PORCH),
+	DEBUGFS_REG32(DC_DISP_ACTIVE),
+	DEBUGFS_REG32(DC_DISP_FRONT_PORCH),
+	DEBUGFS_REG32(DC_DISP_H_PULSE0_CONTROL),
+	DEBUGFS_REG32(DC_DISP_H_PULSE0_POSITION_A),
+	DEBUGFS_REG32(DC_DISP_H_PULSE0_POSITION_B),
+	DEBUGFS_REG32(DC_DISP_H_PULSE0_POSITION_C),
+	DEBUGFS_REG32(DC_DISP_H_PULSE0_POSITION_D),
+	DEBUGFS_REG32(DC_DISP_H_PULSE1_CONTROL),
+	DEBUGFS_REG32(DC_DISP_H_PULSE1_POSITION_A),
+	DEBUGFS_REG32(DC_DISP_H_PULSE1_POSITION_B),
+	DEBUGFS_REG32(DC_DISP_H_PULSE1_POSITION_C),
+	DEBUGFS_REG32(DC_DISP_H_PULSE1_POSITION_D),
+	DEBUGFS_REG32(DC_DISP_H_PULSE2_CONTROL),
+	DEBUGFS_REG32(DC_DISP_H_PULSE2_POSITION_A),
+	DEBUGFS_REG32(DC_DISP_H_PULSE2_POSITION_B),
+	DEBUGFS_REG32(DC_DISP_H_PULSE2_POSITION_C),
+	DEBUGFS_REG32(DC_DISP_H_PULSE2_POSITION_D),
+	DEBUGFS_REG32(DC_DISP_V_PULSE0_CONTROL),
+	DEBUGFS_REG32(DC_DISP_V_PULSE0_POSITION_A),
+	DEBUGFS_REG32(DC_DISP_V_PULSE0_POSITION_B),
+	DEBUGFS_REG32(DC_DISP_V_PULSE0_POSITION_C),
+	DEBUGFS_REG32(DC_DISP_V_PULSE1_CONTROL),
+	DEBUGFS_REG32(DC_DISP_V_PULSE1_POSITION_A),
+	DEBUGFS_REG32(DC_DISP_V_PULSE1_POSITION_B),
+	DEBUGFS_REG32(DC_DISP_V_PULSE1_POSITION_C),
+	DEBUGFS_REG32(DC_DISP_V_PULSE2_CONTROL),
+	DEBUGFS_REG32(DC_DISP_V_PULSE2_POSITION_A),
+	DEBUGFS_REG32(DC_DISP_V_PULSE3_CONTROL),
+	DEBUGFS_REG32(DC_DISP_V_PULSE3_POSITION_A),
+	DEBUGFS_REG32(DC_DISP_M0_CONTROL),
+	DEBUGFS_REG32(DC_DISP_M1_CONTROL),
+	DEBUGFS_REG32(DC_DISP_DI_CONTROL),
+	DEBUGFS_REG32(DC_DISP_PP_CONTROL),
+	DEBUGFS_REG32(DC_DISP_PP_SELECT_A),
+	DEBUGFS_REG32(DC_DISP_PP_SELECT_B),
+	DEBUGFS_REG32(DC_DISP_PP_SELECT_C),
+	DEBUGFS_REG32(DC_DISP_PP_SELECT_D),
+	DEBUGFS_REG32(DC_DISP_DISP_CLOCK_CONTROL),
+	DEBUGFS_REG32(DC_DISP_DISP_INTERFACE_CONTROL),
+	DEBUGFS_REG32(DC_DISP_DISP_COLOR_CONTROL),
+	DEBUGFS_REG32(DC_DISP_SHIFT_CLOCK_OPTIONS),
+	DEBUGFS_REG32(DC_DISP_DATA_ENABLE_OPTIONS),
+	DEBUGFS_REG32(DC_DISP_SERIAL_INTERFACE_OPTIONS),
+	DEBUGFS_REG32(DC_DISP_LCD_SPI_OPTIONS),
+	DEBUGFS_REG32(DC_DISP_BORDER_COLOR),
+	DEBUGFS_REG32(DC_DISP_COLOR_KEY0_LOWER),
+	DEBUGFS_REG32(DC_DISP_COLOR_KEY0_UPPER),
+	DEBUGFS_REG32(DC_DISP_COLOR_KEY1_LOWER),
+	DEBUGFS_REG32(DC_DISP_COLOR_KEY1_UPPER),
+	DEBUGFS_REG32(DC_DISP_CURSOR_FOREGROUND),
+	DEBUGFS_REG32(DC_DISP_CURSOR_BACKGROUND),
+	DEBUGFS_REG32(DC_DISP_CURSOR_START_ADDR),
+	DEBUGFS_REG32(DC_DISP_CURSOR_START_ADDR_NS),
+	DEBUGFS_REG32(DC_DISP_CURSOR_POSITION),
+	DEBUGFS_REG32(DC_DISP_CURSOR_POSITION_NS),
+	DEBUGFS_REG32(DC_DISP_INIT_SEQ_CONTROL),
+	DEBUGFS_REG32(DC_DISP_SPI_INIT_SEQ_DATA_A),
+	DEBUGFS_REG32(DC_DISP_SPI_INIT_SEQ_DATA_B),
+	DEBUGFS_REG32(DC_DISP_SPI_INIT_SEQ_DATA_C),
+	DEBUGFS_REG32(DC_DISP_SPI_INIT_SEQ_DATA_D),
+	DEBUGFS_REG32(DC_DISP_DC_MCCIF_FIFOCTRL),
+	DEBUGFS_REG32(DC_DISP_MCCIF_DISPLAY0A_HYST),
+	DEBUGFS_REG32(DC_DISP_MCCIF_DISPLAY0B_HYST),
+	DEBUGFS_REG32(DC_DISP_MCCIF_DISPLAY1A_HYST),
+	DEBUGFS_REG32(DC_DISP_MCCIF_DISPLAY1B_HYST),
+	DEBUGFS_REG32(DC_DISP_DAC_CRT_CTRL),
+	DEBUGFS_REG32(DC_DISP_DISP_MISC_CONTROL),
+	DEBUGFS_REG32(DC_DISP_SD_CONTROL),
+	DEBUGFS_REG32(DC_DISP_SD_CSC_COEFF),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(0)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(1)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(2)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(3)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(4)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(5)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(6)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(7)),
+	DEBUGFS_REG32(DC_DISP_SD_LUT(8)),
+	DEBUGFS_REG32(DC_DISP_SD_FLICKER_CONTROL),
+	DEBUGFS_REG32(DC_DISP_DC_PIXEL_COUNT),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(0)),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(1)),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(2)),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(3)),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(4)),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(5)),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(6)),
+	DEBUGFS_REG32(DC_DISP_SD_HISTOGRAM(7)),
+	DEBUGFS_REG32(DC_DISP_SD_BL_TF(0)),
+	DEBUGFS_REG32(DC_DISP_SD_BL_TF(1)),
+	DEBUGFS_REG32(DC_DISP_SD_BL_TF(2)),
+	DEBUGFS_REG32(DC_DISP_SD_BL_TF(3)),
+	DEBUGFS_REG32(DC_DISP_SD_BL_CONTROL),
+	DEBUGFS_REG32(DC_DISP_SD_HW_K_VALUES),
+	DEBUGFS_REG32(DC_DISP_SD_MAN_K_VALUES),
+	DEBUGFS_REG32(DC_DISP_CURSOR_START_ADDR_HI),
+	DEBUGFS_REG32(DC_DISP_BLEND_CURSOR_CONTROL),
+	DEBUGFS_REG32(DC_WIN_WIN_OPTIONS),
+	DEBUGFS_REG32(DC_WIN_BYTE_SWAP),
+	DEBUGFS_REG32(DC_WIN_BUFFER_CONTROL),
+	DEBUGFS_REG32(DC_WIN_COLOR_DEPTH),
+	DEBUGFS_REG32(DC_WIN_POSITION),
+	DEBUGFS_REG32(DC_WIN_SIZE),
+	DEBUGFS_REG32(DC_WIN_PRESCALED_SIZE),
+	DEBUGFS_REG32(DC_WIN_H_INITIAL_DDA),
+	DEBUGFS_REG32(DC_WIN_V_INITIAL_DDA),
+	DEBUGFS_REG32(DC_WIN_DDA_INC),
+	DEBUGFS_REG32(DC_WIN_LINE_STRIDE),
+	DEBUGFS_REG32(DC_WIN_BUF_STRIDE),
+	DEBUGFS_REG32(DC_WIN_UV_BUF_STRIDE),
+	DEBUGFS_REG32(DC_WIN_BUFFER_ADDR_MODE),
+	DEBUGFS_REG32(DC_WIN_DV_CONTROL),
+	DEBUGFS_REG32(DC_WIN_BLEND_NOKEY),
+	DEBUGFS_REG32(DC_WIN_BLEND_1WIN),
+	DEBUGFS_REG32(DC_WIN_BLEND_2WIN_X),
+	DEBUGFS_REG32(DC_WIN_BLEND_2WIN_Y),
+	DEBUGFS_REG32(DC_WIN_BLEND_3WIN_XY),
+	DEBUGFS_REG32(DC_WIN_HP_FETCH_CONTROL),
+	DEBUGFS_REG32(DC_WINBUF_START_ADDR),
+	DEBUGFS_REG32(DC_WINBUF_START_ADDR_NS),
+	DEBUGFS_REG32(DC_WINBUF_START_ADDR_U),
+	DEBUGFS_REG32(DC_WINBUF_START_ADDR_U_NS),
+	DEBUGFS_REG32(DC_WINBUF_START_ADDR_V),
+	DEBUGFS_REG32(DC_WINBUF_START_ADDR_V_NS),
+	DEBUGFS_REG32(DC_WINBUF_ADDR_H_OFFSET),
+	DEBUGFS_REG32(DC_WINBUF_ADDR_H_OFFSET_NS),
+	DEBUGFS_REG32(DC_WINBUF_ADDR_V_OFFSET),
+	DEBUGFS_REG32(DC_WINBUF_ADDR_V_OFFSET_NS),
+	DEBUGFS_REG32(DC_WINBUF_UFLOW_STATUS),
+	DEBUGFS_REG32(DC_WINBUF_AD_UFLOW_STATUS),
+	DEBUGFS_REG32(DC_WINBUF_BD_UFLOW_STATUS),
+	DEBUGFS_REG32(DC_WINBUF_CD_UFLOW_STATUS),
+};
 
-	if (!dc->event) {
-		spin_unlock_irqrestore(&drm->event_lock, flags);
-		return;
+static int tegra_dc_show_regs(struct seq_file *s, void *data)
+{
+	struct drm_info_node *node = s->private;
+	struct tegra_dc *dc = node->info_ent->data;
+	unsigned int i;
+	int err = 0;
+
+	drm_modeset_lock(&dc->base.mutex, NULL);
+
+	if (!dc->base.state->active) {
+		err = -EBUSY;
+		goto unlock;
 	}
 
-	bo = tegra_fb_get_plane(crtc->primary->fb, 0);
+	for (i = 0; i < ARRAY_SIZE(tegra_dc_regs); i++) {
+		unsigned int offset = tegra_dc_regs[i].offset;
 
-	spin_lock(&dc->lock);
+		seq_printf(s, "%-40s %#05x %08x\n", tegra_dc_regs[i].name,
+			   offset, tegra_dc_readl(dc, offset));
+	}
 
-	/* check if new start address has been latched */
-	tegra_dc_writel(dc, WINDOW_A_SELECT, DC_CMD_DISPLAY_WINDOW_HEADER);
-	tegra_dc_writel(dc, READ_MUX, DC_CMD_STATE_ACCESS);
-	base = tegra_dc_readl(dc, DC_WINBUF_START_ADDR);
-	tegra_dc_writel(dc, 0, DC_CMD_STATE_ACCESS);
+unlock:
+	drm_modeset_unlock(&dc->base.mutex);
+	return err;
+}
+
+static int tegra_dc_show_crc(struct seq_file *s, void *data)
+{
+	struct drm_info_node *node = s->private;
+	struct tegra_dc *dc = node->info_ent->data;
+	int err = 0;
+	u32 value;
 
-	spin_unlock(&dc->lock);
+	drm_modeset_lock(&dc->base.mutex, NULL);
 
-	if (base == bo->paddr + crtc->primary->fb->offsets[0]) {
-		drm_crtc_send_vblank_event(crtc, dc->event);
-		drm_crtc_vblank_put(crtc);
-		dc->event = NULL;
+	if (!dc->base.state->active) {
+		err = -EBUSY;
+		goto unlock;
 	}
 
-	spin_unlock_irqrestore(&drm->event_lock, flags);
+	value = DC_COM_CRC_CONTROL_ACTIVE_DATA | DC_COM_CRC_CONTROL_ENABLE;
+	tegra_dc_writel(dc, value, DC_COM_CRC_CONTROL);
+	tegra_dc_commit(dc);
+
+	drm_crtc_wait_one_vblank(&dc->base);
+	drm_crtc_wait_one_vblank(&dc->base);
+
+	value = tegra_dc_readl(dc, DC_COM_CRC_CHECKSUM);
+	seq_printf(s, "%08x\n", value);
+
+	tegra_dc_writel(dc, 0, DC_COM_CRC_CONTROL);
+
+unlock:
+	drm_modeset_unlock(&dc->base.mutex);
+	return err;
 }
 
-static void tegra_dc_destroy(struct drm_crtc *crtc)
+static int tegra_dc_show_stats(struct seq_file *s, void *data)
 {
-	drm_crtc_cleanup(crtc);
+	struct drm_info_node *node = s->private;
+	struct tegra_dc *dc = node->info_ent->data;
+
+	seq_printf(s, "frames: %lu\n", dc->stats.frames);
+	seq_printf(s, "vblank: %lu\n", dc->stats.vblank);
+	seq_printf(s, "underflow: %lu\n", dc->stats.underflow);
+	seq_printf(s, "overflow: %lu\n", dc->stats.overflow);
+
+	return 0;
 }
 
-static void tegra_crtc_reset(struct drm_crtc *crtc)
+static struct drm_info_list debugfs_files[] = {
+	{ "regs", tegra_dc_show_regs, 0, NULL },
+	{ "crc", tegra_dc_show_crc, 0, NULL },
+	{ "stats", tegra_dc_show_stats, 0, NULL },
+};
+
+static int tegra_dc_late_register(struct drm_crtc *crtc)
 {
-	struct tegra_dc_state *state;
+	unsigned int i, count = ARRAY_SIZE(debugfs_files);
+	struct drm_minor *minor = crtc->dev->primary;
+	struct dentry *root;
+	struct tegra_dc *dc = to_tegra_dc(crtc);
+	int err;
 
-	if (crtc->state)
-		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+#ifdef CONFIG_DEBUG_FS
+	root = crtc->debugfs_entry;
+#else
+	root = NULL;
+#endif
 
-	kfree(crtc->state);
-	crtc->state = NULL;
+	dc->debugfs_files = kmemdup(debugfs_files, sizeof(debugfs_files),
+				    GFP_KERNEL);
+	if (!dc->debugfs_files)
+		return -ENOMEM;
 
-	state = kzalloc(sizeof(*state), GFP_KERNEL);
-	if (state) {
-		crtc->state = &state->base;
-		crtc->state->crtc = crtc;
-	}
+	for (i = 0; i < count; i++)
+		dc->debugfs_files[i].data = dc;
+
+	err = drm_debugfs_create_files(dc->debugfs_files, count, root, minor);
+	if (err < 0)
+		goto free;
+
+	return 0;
+
+free:
+	kfree(dc->debugfs_files);
+	dc->debugfs_files = NULL;
+
+	return err;
+}
+
+static void tegra_dc_early_unregister(struct drm_crtc *crtc)
+{
+	unsigned int count = ARRAY_SIZE(debugfs_files);
+	struct drm_minor *minor = crtc->dev->primary;
+	struct tegra_dc *dc = to_tegra_dc(crtc);
+
+	drm_debugfs_remove_files(dc->debugfs_files, count, minor);
+	kfree(dc->debugfs_files);
+	dc->debugfs_files = NULL;
+}
+
+static u32 tegra_dc_get_vblank_counter(struct drm_crtc *crtc)
+{
+	struct tegra_dc *dc = to_tegra_dc(crtc);
 
-	drm_crtc_vblank_reset(crtc);
+	/* XXX vblank syncpoints don't work with nvdisplay yet */
+	if (dc->syncpt && !dc->soc->has_nvdisplay)
+		return host1x_syncpt_read(dc->syncpt);
+
+	/* fallback to software emulated VBLANK counter */
+	return drm_crtc_vblank_count(&dc->base);
 }
 
-static struct drm_crtc_state *
-tegra_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
+static int tegra_dc_enable_vblank(struct drm_crtc *crtc)
 {
-	struct tegra_dc_state *state = to_dc_state(crtc->state);
-	struct tegra_dc_state *copy;
-
-	copy = kmalloc(sizeof(*copy), GFP_KERNEL);
-	if (!copy)
-		return NULL;
+	struct tegra_dc *dc = to_tegra_dc(crtc);
+	u32 value;
 
-	__drm_atomic_helper_crtc_duplicate_state(crtc, &copy->base);
-	copy->clk = state->clk;
-	copy->pclk = state->pclk;
-	copy->div = state->div;
-	copy->planes = state->planes;
+	value = tegra_dc_readl(dc, DC_CMD_INT_MASK);
+	value |= VBLANK_INT;
+	tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
 
-	return &copy->base;
+	return 0;
 }
 
-static void tegra_crtc_atomic_destroy_state(struct drm_crtc *crtc,
-					    struct drm_crtc_state *state)
+static void tegra_dc_disable_vblank(struct drm_crtc *crtc)
 {
-	__drm_atomic_helper_crtc_destroy_state(state);
-	kfree(state);
+	struct tegra_dc *dc = to_tegra_dc(crtc);
+	u32 value;
+
+	value = tegra_dc_readl(dc, DC_CMD_INT_MASK);
+	value &= ~VBLANK_INT;
+	tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
 }
 
 static const struct drm_crtc_funcs tegra_crtc_funcs = {
@@ -1042,6 +1391,8 @@ static const struct drm_crtc_funcs tegra_crtc_funcs = {
 	.reset = tegra_crtc_reset,
 	.atomic_duplicate_state = tegra_crtc_atomic_duplicate_state,
 	.atomic_destroy_state = tegra_crtc_atomic_destroy_state,
+	.late_register = tegra_dc_late_register,
+	.early_unregister = tegra_dc_early_unregister,
 	.get_vblank_counter = tegra_dc_get_vblank_counter,
 	.enable_vblank = tegra_dc_enable_vblank,
 	.disable_vblank = tegra_dc_disable_vblank,
@@ -1054,10 +1405,12 @@ static int tegra_dc_set_timings(struct tegra_dc *dc,
 	unsigned int v_ref_to_sync = 1;
 	unsigned long value;
 
-	tegra_dc_writel(dc, 0x0, DC_DISP_DISP_TIMING_OPTIONS);
+	if (!dc->soc->has_nvdisplay) {
+		tegra_dc_writel(dc, 0x0, DC_DISP_DISP_TIMING_OPTIONS);
 
-	value = (v_ref_to_sync << 16) | h_ref_to_sync;
-	tegra_dc_writel(dc, value, DC_DISP_REF_TO_SYNC);
+		value = (v_ref_to_sync << 16) | h_ref_to_sync;
+		tegra_dc_writel(dc, value, DC_DISP_REF_TO_SYNC);
+	}
 
 	value = ((mode->vsync_end - mode->vsync_start) << 16) |
 		((mode->hsync_end - mode->hsync_start) <<  0);
@@ -1136,8 +1489,10 @@ static void tegra_dc_commit_state(struct tegra_dc *dc,
 		      state->div);
 	DRM_DEBUG_KMS("pclk: %lu\n", state->pclk);
 
-	value = SHIFT_CLK_DIVIDER(state->div) | PIXEL_CLK_DIVIDER_PCD1;
-	tegra_dc_writel(dc, value, DC_DISP_DISP_CLOCK_CONTROL);
+	if (!dc->soc->has_nvdisplay) {
+		value = SHIFT_CLK_DIVIDER(state->div) | PIXEL_CLK_DIVIDER_PCD1;
+		tegra_dc_writel(dc, value, DC_DISP_DISP_CLOCK_CONTROL);
+	}
 
 	err = clk_set_rate(dc->clk, state->pclk);
 	if (err < 0)
@@ -1223,6 +1578,15 @@ static void tegra_crtc_atomic_disable(struct drm_crtc *crtc,
 	tegra_dc_stats_reset(&dc->stats);
 	drm_crtc_vblank_off(crtc);
 
+	spin_lock_irq(&crtc->dev->event_lock);
+
+	if (crtc->state->event) {
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+	}
+
+	spin_unlock_irq(&crtc->dev->event_lock);
+
 	pm_runtime_put_sync(dc->dev);
 }
 
@@ -1238,41 +1602,70 @@ static void tegra_crtc_atomic_enable(struct drm_crtc *crtc,
 
 	/* initialize display controller */
 	if (dc->syncpt) {
-		u32 syncpt = host1x_syncpt_id(dc->syncpt);
+		u32 syncpt = host1x_syncpt_id(dc->syncpt), enable;
+
+		if (dc->soc->has_nvdisplay)
+			enable = 1 << 31;
+		else
+			enable = 1 << 8;
 
 		value = SYNCPT_CNTRL_NO_STALL;
 		tegra_dc_writel(dc, value, DC_CMD_GENERAL_INCR_SYNCPT_CNTRL);
 
-		value = SYNCPT_VSYNC_ENABLE | syncpt;
+		value = enable | syncpt;
 		tegra_dc_writel(dc, value, DC_CMD_CONT_SYNCPT_VSYNC);
 	}
 
-	value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-		WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-	tegra_dc_writel(dc, value, DC_CMD_INT_TYPE);
+	if (dc->soc->has_nvdisplay) {
+		value = DSC_TO_UF_INT | DSC_BBUF_UF_INT | DSC_RBUF_UF_INT |
+			DSC_OBUF_UF_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_TYPE);
 
-	value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-		WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-	tegra_dc_writel(dc, value, DC_CMD_INT_POLARITY);
+		value = DSC_TO_UF_INT | DSC_BBUF_UF_INT | DSC_RBUF_UF_INT |
+			DSC_OBUF_UF_INT | SD3_BUCKET_WALK_DONE_INT |
+			HEAD_UF_INT | MSF_INT | REG_TMOUT_INT |
+			REGION_CRC_INT | V_PULSE2_INT | V_PULSE3_INT |
+			VBLANK_INT | FRAME_END_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_POLARITY);
 
-	/* initialize timer */
-	value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(0x20) |
-		WINDOW_B_THRESHOLD(0x20) | WINDOW_C_THRESHOLD(0x20);
-	tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY);
+		value = SD3_BUCKET_WALK_DONE_INT | HEAD_UF_INT | VBLANK_INT |
+			FRAME_END_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_ENABLE);
 
-	value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(1) |
-		WINDOW_B_THRESHOLD(1) | WINDOW_C_THRESHOLD(1);
-	tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY_TIMER);
+		value = HEAD_UF_INT | REG_TMOUT_INT | FRAME_END_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
 
-	value = VBLANK_INT | WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-		WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-	tegra_dc_writel(dc, value, DC_CMD_INT_ENABLE);
-
-	value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-		WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-	tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
+		tegra_dc_writel(dc, READ_MUX, DC_CMD_STATE_ACCESS);
+	} else {
+		value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+			WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_TYPE);
+
+		value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+			WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_POLARITY);
+
+		/* initialize timer */
+		value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(0x20) |
+			WINDOW_B_THRESHOLD(0x20) | WINDOW_C_THRESHOLD(0x20);
+		tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY);
+
+		value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(1) |
+			WINDOW_B_THRESHOLD(1) | WINDOW_C_THRESHOLD(1);
+		tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY_TIMER);
+
+		value = VBLANK_INT | WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+			WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_ENABLE);
+
+		value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+			WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+		tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
+	}
 
-	if (dc->soc->supports_border_color)
+	if (dc->soc->supports_background_color)
+		tegra_dc_writel(dc, 0, DC_DISP_BLEND_BACKGROUND_COLOR);
+	else
 		tegra_dc_writel(dc, 0, DC_DISP_BORDER_COLOR);
 
 	/* apply PLL and pixel clock changes */
@@ -1293,10 +1686,18 @@ static void tegra_crtc_atomic_enable(struct drm_crtc *crtc,
 	value |= DISP_CTRL_MODE_C_DISPLAY;
 	tegra_dc_writel(dc, value, DC_CMD_DISPLAY_COMMAND);
 
-	value = tegra_dc_readl(dc, DC_CMD_DISPLAY_POWER_CONTROL);
-	value |= PW0_ENABLE | PW1_ENABLE | PW2_ENABLE | PW3_ENABLE |
-		 PW4_ENABLE | PM0_ENABLE | PM1_ENABLE;
-	tegra_dc_writel(dc, value, DC_CMD_DISPLAY_POWER_CONTROL);
+	if (!dc->soc->has_nvdisplay) {
+		value = tegra_dc_readl(dc, DC_CMD_DISPLAY_POWER_CONTROL);
+		value |= PW0_ENABLE | PW1_ENABLE | PW2_ENABLE | PW3_ENABLE |
+			 PW4_ENABLE | PM0_ENABLE | PM1_ENABLE;
+		tegra_dc_writel(dc, value, DC_CMD_DISPLAY_POWER_CONTROL);
+	}
+
+	/* enable underflow reporting and display red for missing pixels */
+	if (dc->soc->has_nvdisplay) {
+		value = UNDERFLOW_MODE_RED | UNDERFLOW_REPORT_ENABLE;
+		tegra_dc_writel(dc, value, DC_COM_RG_UNDERFLOW);
+	}
 
 	tegra_dc_commit(dc);
 
@@ -1306,20 +1707,43 @@ static void tegra_crtc_atomic_enable(struct drm_crtc *crtc,
 static int tegra_crtc_atomic_check(struct drm_crtc *crtc,
 				   struct drm_crtc_state *state)
 {
+	struct tegra_atomic_state *s = to_tegra_atomic_state(state->state);
+	struct tegra_dc_state *tegra = to_dc_state(state);
+
+	/*
+	 * The display hub display clock needs to be fed by the display clock
+	 * with the highest frequency to ensure proper functioning of all the
+	 * displays.
+	 *
+	 * Note that this isn't used before Tegra186, but it doesn't hurt and
+	 * conditionalizing it would make the code less clean.
+	 */
+	if (state->active) {
+		if (!s->clk_disp || tegra->pclk > s->rate) {
+			s->dc = to_tegra_dc(crtc);
+			s->clk_disp = s->dc->clk;
+			s->rate = tegra->pclk;
+		}
+	}
+
 	return 0;
 }
 
 static void tegra_crtc_atomic_begin(struct drm_crtc *crtc,
 				    struct drm_crtc_state *old_crtc_state)
 {
-	struct tegra_dc *dc = to_tegra_dc(crtc);
+	unsigned long flags;
 
 	if (crtc->state->event) {
-		crtc->state->event->pipe = drm_crtc_index(crtc);
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+
+		if (drm_crtc_vblank_get(crtc) != 0)
+			drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		else
+			drm_crtc_arm_vblank_event(crtc, crtc->state->event);
 
-		WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 
-		dc->event = crtc->state->event;
 		crtc->state->event = NULL;
 	}
 }
@@ -1329,9 +1753,15 @@ static void tegra_crtc_atomic_flush(struct drm_crtc *crtc,
 {
 	struct tegra_dc_state *state = to_dc_state(crtc->state);
 	struct tegra_dc *dc = to_tegra_dc(crtc);
+	u32 value;
 
-	tegra_dc_writel(dc, state->planes << 8, DC_CMD_STATE_CONTROL);
-	tegra_dc_writel(dc, state->planes, DC_CMD_STATE_CONTROL);
+	value = state->planes << 8 | GENERAL_UPDATE;
+	tegra_dc_writel(dc, value, DC_CMD_STATE_CONTROL);
+	value = tegra_dc_readl(dc, DC_CMD_STATE_CONTROL);
+
+	value = state->planes | GENERAL_ACT_REQ;
+	tegra_dc_writel(dc, value, DC_CMD_STATE_CONTROL);
+	value = tegra_dc_readl(dc, DC_CMD_STATE_CONTROL);
 }
 
 static const struct drm_crtc_helper_funcs tegra_crtc_helper_funcs = {
@@ -1362,7 +1792,6 @@ static irqreturn_t tegra_dc_irq(int irq, void *data)
 		dev_dbg(dc->dev, "%s(): vertical blank\n", __func__);
 		*/
 		drm_crtc_handle_vblank(&dc->base);
-		tegra_dc_finish_page_flip(dc);
 		dc->stats.vblank++;
 	}
 
@@ -1380,357 +1809,18 @@ static irqreturn_t tegra_dc_irq(int irq, void *data)
 		dc->stats.overflow++;
 	}
 
-	return IRQ_HANDLED;
-}
-
-static int tegra_dc_show_regs(struct seq_file *s, void *data)
-{
-	struct drm_info_node *node = s->private;
-	struct tegra_dc *dc = node->info_ent->data;
-	int err = 0;
-
-	drm_modeset_lock(&dc->base.mutex, NULL);
-
-	if (!dc->base.state->active) {
-		err = -EBUSY;
-		goto unlock;
-	}
-
-#define DUMP_REG(name)						\
-	seq_printf(s, "%-40s %#05x %08x\n", #name, name,	\
-		   tegra_dc_readl(dc, name))
-
-	DUMP_REG(DC_CMD_GENERAL_INCR_SYNCPT);
-	DUMP_REG(DC_CMD_GENERAL_INCR_SYNCPT_CNTRL);
-	DUMP_REG(DC_CMD_GENERAL_INCR_SYNCPT_ERROR);
-	DUMP_REG(DC_CMD_WIN_A_INCR_SYNCPT);
-	DUMP_REG(DC_CMD_WIN_A_INCR_SYNCPT_CNTRL);
-	DUMP_REG(DC_CMD_WIN_A_INCR_SYNCPT_ERROR);
-	DUMP_REG(DC_CMD_WIN_B_INCR_SYNCPT);
-	DUMP_REG(DC_CMD_WIN_B_INCR_SYNCPT_CNTRL);
-	DUMP_REG(DC_CMD_WIN_B_INCR_SYNCPT_ERROR);
-	DUMP_REG(DC_CMD_WIN_C_INCR_SYNCPT);
-	DUMP_REG(DC_CMD_WIN_C_INCR_SYNCPT_CNTRL);
-	DUMP_REG(DC_CMD_WIN_C_INCR_SYNCPT_ERROR);
-	DUMP_REG(DC_CMD_CONT_SYNCPT_VSYNC);
-	DUMP_REG(DC_CMD_DISPLAY_COMMAND_OPTION0);
-	DUMP_REG(DC_CMD_DISPLAY_COMMAND);
-	DUMP_REG(DC_CMD_SIGNAL_RAISE);
-	DUMP_REG(DC_CMD_DISPLAY_POWER_CONTROL);
-	DUMP_REG(DC_CMD_INT_STATUS);
-	DUMP_REG(DC_CMD_INT_MASK);
-	DUMP_REG(DC_CMD_INT_ENABLE);
-	DUMP_REG(DC_CMD_INT_TYPE);
-	DUMP_REG(DC_CMD_INT_POLARITY);
-	DUMP_REG(DC_CMD_SIGNAL_RAISE1);
-	DUMP_REG(DC_CMD_SIGNAL_RAISE2);
-	DUMP_REG(DC_CMD_SIGNAL_RAISE3);
-	DUMP_REG(DC_CMD_STATE_ACCESS);
-	DUMP_REG(DC_CMD_STATE_CONTROL);
-	DUMP_REG(DC_CMD_DISPLAY_WINDOW_HEADER);
-	DUMP_REG(DC_CMD_REG_ACT_CONTROL);
-	DUMP_REG(DC_COM_CRC_CONTROL);
-	DUMP_REG(DC_COM_CRC_CHECKSUM);
-	DUMP_REG(DC_COM_PIN_OUTPUT_ENABLE(0));
-	DUMP_REG(DC_COM_PIN_OUTPUT_ENABLE(1));
-	DUMP_REG(DC_COM_PIN_OUTPUT_ENABLE(2));
-	DUMP_REG(DC_COM_PIN_OUTPUT_ENABLE(3));
-	DUMP_REG(DC_COM_PIN_OUTPUT_POLARITY(0));
-	DUMP_REG(DC_COM_PIN_OUTPUT_POLARITY(1));
-	DUMP_REG(DC_COM_PIN_OUTPUT_POLARITY(2));
-	DUMP_REG(DC_COM_PIN_OUTPUT_POLARITY(3));
-	DUMP_REG(DC_COM_PIN_OUTPUT_DATA(0));
-	DUMP_REG(DC_COM_PIN_OUTPUT_DATA(1));
-	DUMP_REG(DC_COM_PIN_OUTPUT_DATA(2));
-	DUMP_REG(DC_COM_PIN_OUTPUT_DATA(3));
-	DUMP_REG(DC_COM_PIN_INPUT_ENABLE(0));
-	DUMP_REG(DC_COM_PIN_INPUT_ENABLE(1));
-	DUMP_REG(DC_COM_PIN_INPUT_ENABLE(2));
-	DUMP_REG(DC_COM_PIN_INPUT_ENABLE(3));
-	DUMP_REG(DC_COM_PIN_INPUT_DATA(0));
-	DUMP_REG(DC_COM_PIN_INPUT_DATA(1));
-	DUMP_REG(DC_COM_PIN_OUTPUT_SELECT(0));
-	DUMP_REG(DC_COM_PIN_OUTPUT_SELECT(1));
-	DUMP_REG(DC_COM_PIN_OUTPUT_SELECT(2));
-	DUMP_REG(DC_COM_PIN_OUTPUT_SELECT(3));
-	DUMP_REG(DC_COM_PIN_OUTPUT_SELECT(4));
-	DUMP_REG(DC_COM_PIN_OUTPUT_SELECT(5));
-	DUMP_REG(DC_COM_PIN_OUTPUT_SELECT(6));
-	DUMP_REG(DC_COM_PIN_MISC_CONTROL);
-	DUMP_REG(DC_COM_PIN_PM0_CONTROL);
-	DUMP_REG(DC_COM_PIN_PM0_DUTY_CYCLE);
-	DUMP_REG(DC_COM_PIN_PM1_CONTROL);
-	DUMP_REG(DC_COM_PIN_PM1_DUTY_CYCLE);
-	DUMP_REG(DC_COM_SPI_CONTROL);
-	DUMP_REG(DC_COM_SPI_START_BYTE);
-	DUMP_REG(DC_COM_HSPI_WRITE_DATA_AB);
-	DUMP_REG(DC_COM_HSPI_WRITE_DATA_CD);
-	DUMP_REG(DC_COM_HSPI_CS_DC);
-	DUMP_REG(DC_COM_SCRATCH_REGISTER_A);
-	DUMP_REG(DC_COM_SCRATCH_REGISTER_B);
-	DUMP_REG(DC_COM_GPIO_CTRL);
-	DUMP_REG(DC_COM_GPIO_DEBOUNCE_COUNTER);
-	DUMP_REG(DC_COM_CRC_CHECKSUM_LATCHED);
-	DUMP_REG(DC_DISP_DISP_SIGNAL_OPTIONS0);
-	DUMP_REG(DC_DISP_DISP_SIGNAL_OPTIONS1);
-	DUMP_REG(DC_DISP_DISP_WIN_OPTIONS);
-	DUMP_REG(DC_DISP_DISP_MEM_HIGH_PRIORITY);
-	DUMP_REG(DC_DISP_DISP_MEM_HIGH_PRIORITY_TIMER);
-	DUMP_REG(DC_DISP_DISP_TIMING_OPTIONS);
-	DUMP_REG(DC_DISP_REF_TO_SYNC);
-	DUMP_REG(DC_DISP_SYNC_WIDTH);
-	DUMP_REG(DC_DISP_BACK_PORCH);
-	DUMP_REG(DC_DISP_ACTIVE);
-	DUMP_REG(DC_DISP_FRONT_PORCH);
-	DUMP_REG(DC_DISP_H_PULSE0_CONTROL);
-	DUMP_REG(DC_DISP_H_PULSE0_POSITION_A);
-	DUMP_REG(DC_DISP_H_PULSE0_POSITION_B);
-	DUMP_REG(DC_DISP_H_PULSE0_POSITION_C);
-	DUMP_REG(DC_DISP_H_PULSE0_POSITION_D);
-	DUMP_REG(DC_DISP_H_PULSE1_CONTROL);
-	DUMP_REG(DC_DISP_H_PULSE1_POSITION_A);
-	DUMP_REG(DC_DISP_H_PULSE1_POSITION_B);
-	DUMP_REG(DC_DISP_H_PULSE1_POSITION_C);
-	DUMP_REG(DC_DISP_H_PULSE1_POSITION_D);
-	DUMP_REG(DC_DISP_H_PULSE2_CONTROL);
-	DUMP_REG(DC_DISP_H_PULSE2_POSITION_A);
-	DUMP_REG(DC_DISP_H_PULSE2_POSITION_B);
-	DUMP_REG(DC_DISP_H_PULSE2_POSITION_C);
-	DUMP_REG(DC_DISP_H_PULSE2_POSITION_D);
-	DUMP_REG(DC_DISP_V_PULSE0_CONTROL);
-	DUMP_REG(DC_DISP_V_PULSE0_POSITION_A);
-	DUMP_REG(DC_DISP_V_PULSE0_POSITION_B);
-	DUMP_REG(DC_DISP_V_PULSE0_POSITION_C);
-	DUMP_REG(DC_DISP_V_PULSE1_CONTROL);
-	DUMP_REG(DC_DISP_V_PULSE1_POSITION_A);
-	DUMP_REG(DC_DISP_V_PULSE1_POSITION_B);
-	DUMP_REG(DC_DISP_V_PULSE1_POSITION_C);
-	DUMP_REG(DC_DISP_V_PULSE2_CONTROL);
-	DUMP_REG(DC_DISP_V_PULSE2_POSITION_A);
-	DUMP_REG(DC_DISP_V_PULSE3_CONTROL);
-	DUMP_REG(DC_DISP_V_PULSE3_POSITION_A);
-	DUMP_REG(DC_DISP_M0_CONTROL);
-	DUMP_REG(DC_DISP_M1_CONTROL);
-	DUMP_REG(DC_DISP_DI_CONTROL);
-	DUMP_REG(DC_DISP_PP_CONTROL);
-	DUMP_REG(DC_DISP_PP_SELECT_A);
-	DUMP_REG(DC_DISP_PP_SELECT_B);
-	DUMP_REG(DC_DISP_PP_SELECT_C);
-	DUMP_REG(DC_DISP_PP_SELECT_D);
-	DUMP_REG(DC_DISP_DISP_CLOCK_CONTROL);
-	DUMP_REG(DC_DISP_DISP_INTERFACE_CONTROL);
-	DUMP_REG(DC_DISP_DISP_COLOR_CONTROL);
-	DUMP_REG(DC_DISP_SHIFT_CLOCK_OPTIONS);
-	DUMP_REG(DC_DISP_DATA_ENABLE_OPTIONS);
-	DUMP_REG(DC_DISP_SERIAL_INTERFACE_OPTIONS);
-	DUMP_REG(DC_DISP_LCD_SPI_OPTIONS);
-	DUMP_REG(DC_DISP_BORDER_COLOR);
-	DUMP_REG(DC_DISP_COLOR_KEY0_LOWER);
-	DUMP_REG(DC_DISP_COLOR_KEY0_UPPER);
-	DUMP_REG(DC_DISP_COLOR_KEY1_LOWER);
-	DUMP_REG(DC_DISP_COLOR_KEY1_UPPER);
-	DUMP_REG(DC_DISP_CURSOR_FOREGROUND);
-	DUMP_REG(DC_DISP_CURSOR_BACKGROUND);
-	DUMP_REG(DC_DISP_CURSOR_START_ADDR);
-	DUMP_REG(DC_DISP_CURSOR_START_ADDR_NS);
-	DUMP_REG(DC_DISP_CURSOR_POSITION);
-	DUMP_REG(DC_DISP_CURSOR_POSITION_NS);
-	DUMP_REG(DC_DISP_INIT_SEQ_CONTROL);
-	DUMP_REG(DC_DISP_SPI_INIT_SEQ_DATA_A);
-	DUMP_REG(DC_DISP_SPI_INIT_SEQ_DATA_B);
-	DUMP_REG(DC_DISP_SPI_INIT_SEQ_DATA_C);
-	DUMP_REG(DC_DISP_SPI_INIT_SEQ_DATA_D);
-	DUMP_REG(DC_DISP_DC_MCCIF_FIFOCTRL);
-	DUMP_REG(DC_DISP_MCCIF_DISPLAY0A_HYST);
-	DUMP_REG(DC_DISP_MCCIF_DISPLAY0B_HYST);
-	DUMP_REG(DC_DISP_MCCIF_DISPLAY1A_HYST);
-	DUMP_REG(DC_DISP_MCCIF_DISPLAY1B_HYST);
-	DUMP_REG(DC_DISP_DAC_CRT_CTRL);
-	DUMP_REG(DC_DISP_DISP_MISC_CONTROL);
-	DUMP_REG(DC_DISP_SD_CONTROL);
-	DUMP_REG(DC_DISP_SD_CSC_COEFF);
-	DUMP_REG(DC_DISP_SD_LUT(0));
-	DUMP_REG(DC_DISP_SD_LUT(1));
-	DUMP_REG(DC_DISP_SD_LUT(2));
-	DUMP_REG(DC_DISP_SD_LUT(3));
-	DUMP_REG(DC_DISP_SD_LUT(4));
-	DUMP_REG(DC_DISP_SD_LUT(5));
-	DUMP_REG(DC_DISP_SD_LUT(6));
-	DUMP_REG(DC_DISP_SD_LUT(7));
-	DUMP_REG(DC_DISP_SD_LUT(8));
-	DUMP_REG(DC_DISP_SD_FLICKER_CONTROL);
-	DUMP_REG(DC_DISP_DC_PIXEL_COUNT);
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(0));
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(1));
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(2));
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(3));
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(4));
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(5));
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(6));
-	DUMP_REG(DC_DISP_SD_HISTOGRAM(7));
-	DUMP_REG(DC_DISP_SD_BL_TF(0));
-	DUMP_REG(DC_DISP_SD_BL_TF(1));
-	DUMP_REG(DC_DISP_SD_BL_TF(2));
-	DUMP_REG(DC_DISP_SD_BL_TF(3));
-	DUMP_REG(DC_DISP_SD_BL_CONTROL);
-	DUMP_REG(DC_DISP_SD_HW_K_VALUES);
-	DUMP_REG(DC_DISP_SD_MAN_K_VALUES);
-	DUMP_REG(DC_DISP_CURSOR_START_ADDR_HI);
-	DUMP_REG(DC_DISP_BLEND_CURSOR_CONTROL);
-	DUMP_REG(DC_WIN_WIN_OPTIONS);
-	DUMP_REG(DC_WIN_BYTE_SWAP);
-	DUMP_REG(DC_WIN_BUFFER_CONTROL);
-	DUMP_REG(DC_WIN_COLOR_DEPTH);
-	DUMP_REG(DC_WIN_POSITION);
-	DUMP_REG(DC_WIN_SIZE);
-	DUMP_REG(DC_WIN_PRESCALED_SIZE);
-	DUMP_REG(DC_WIN_H_INITIAL_DDA);
-	DUMP_REG(DC_WIN_V_INITIAL_DDA);
-	DUMP_REG(DC_WIN_DDA_INC);
-	DUMP_REG(DC_WIN_LINE_STRIDE);
-	DUMP_REG(DC_WIN_BUF_STRIDE);
-	DUMP_REG(DC_WIN_UV_BUF_STRIDE);
-	DUMP_REG(DC_WIN_BUFFER_ADDR_MODE);
-	DUMP_REG(DC_WIN_DV_CONTROL);
-	DUMP_REG(DC_WIN_BLEND_NOKEY);
-	DUMP_REG(DC_WIN_BLEND_1WIN);
-	DUMP_REG(DC_WIN_BLEND_2WIN_X);
-	DUMP_REG(DC_WIN_BLEND_2WIN_Y);
-	DUMP_REG(DC_WIN_BLEND_3WIN_XY);
-	DUMP_REG(DC_WIN_HP_FETCH_CONTROL);
-	DUMP_REG(DC_WINBUF_START_ADDR);
-	DUMP_REG(DC_WINBUF_START_ADDR_NS);
-	DUMP_REG(DC_WINBUF_START_ADDR_U);
-	DUMP_REG(DC_WINBUF_START_ADDR_U_NS);
-	DUMP_REG(DC_WINBUF_START_ADDR_V);
-	DUMP_REG(DC_WINBUF_START_ADDR_V_NS);
-	DUMP_REG(DC_WINBUF_ADDR_H_OFFSET);
-	DUMP_REG(DC_WINBUF_ADDR_H_OFFSET_NS);
-	DUMP_REG(DC_WINBUF_ADDR_V_OFFSET);
-	DUMP_REG(DC_WINBUF_ADDR_V_OFFSET_NS);
-	DUMP_REG(DC_WINBUF_UFLOW_STATUS);
-	DUMP_REG(DC_WINBUF_AD_UFLOW_STATUS);
-	DUMP_REG(DC_WINBUF_BD_UFLOW_STATUS);
-	DUMP_REG(DC_WINBUF_CD_UFLOW_STATUS);
-
-#undef DUMP_REG
-
-unlock:
-	drm_modeset_unlock(&dc->base.mutex);
-	return err;
-}
-
-static int tegra_dc_show_crc(struct seq_file *s, void *data)
-{
-	struct drm_info_node *node = s->private;
-	struct tegra_dc *dc = node->info_ent->data;
-	int err = 0;
-	u32 value;
-
-	drm_modeset_lock(&dc->base.mutex, NULL);
-
-	if (!dc->base.state->active) {
-		err = -EBUSY;
-		goto unlock;
-	}
-
-	value = DC_COM_CRC_CONTROL_ACTIVE_DATA | DC_COM_CRC_CONTROL_ENABLE;
-	tegra_dc_writel(dc, value, DC_COM_CRC_CONTROL);
-	tegra_dc_commit(dc);
-
-	drm_crtc_wait_one_vblank(&dc->base);
-	drm_crtc_wait_one_vblank(&dc->base);
-
-	value = tegra_dc_readl(dc, DC_COM_CRC_CHECKSUM);
-	seq_printf(s, "%08x\n", value);
-
-	tegra_dc_writel(dc, 0, DC_COM_CRC_CONTROL);
-
-unlock:
-	drm_modeset_unlock(&dc->base.mutex);
-	return err;
-}
-
-static int tegra_dc_show_stats(struct seq_file *s, void *data)
-{
-	struct drm_info_node *node = s->private;
-	struct tegra_dc *dc = node->info_ent->data;
-
-	seq_printf(s, "frames: %lu\n", dc->stats.frames);
-	seq_printf(s, "vblank: %lu\n", dc->stats.vblank);
-	seq_printf(s, "underflow: %lu\n", dc->stats.underflow);
-	seq_printf(s, "overflow: %lu\n", dc->stats.overflow);
-
-	return 0;
-}
-
-static struct drm_info_list debugfs_files[] = {
-	{ "regs", tegra_dc_show_regs, 0, NULL },
-	{ "crc", tegra_dc_show_crc, 0, NULL },
-	{ "stats", tegra_dc_show_stats, 0, NULL },
-};
-
-static int tegra_dc_debugfs_init(struct tegra_dc *dc, struct drm_minor *minor)
-{
-	unsigned int i;
-	char *name;
-	int err;
-
-	name = kasprintf(GFP_KERNEL, "dc.%d", dc->pipe);
-	dc->debugfs = debugfs_create_dir(name, minor->debugfs_root);
-	kfree(name);
-
-	if (!dc->debugfs)
-		return -ENOMEM;
-
-	dc->debugfs_files = kmemdup(debugfs_files, sizeof(debugfs_files),
-				    GFP_KERNEL);
-	if (!dc->debugfs_files) {
-		err = -ENOMEM;
-		goto remove;
+	if (status & HEAD_UF_INT) {
+		dev_dbg_ratelimited(dc->dev, "%s(): head underflow\n", __func__);
+		dc->stats.underflow++;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
-		dc->debugfs_files[i].data = dc;
-
-	err = drm_debugfs_create_files(dc->debugfs_files,
-				       ARRAY_SIZE(debugfs_files),
-				       dc->debugfs, minor);
-	if (err < 0)
-		goto free;
-
-	dc->minor = minor;
-
-	return 0;
-
-free:
-	kfree(dc->debugfs_files);
-	dc->debugfs_files = NULL;
-remove:
-	debugfs_remove(dc->debugfs);
-	dc->debugfs = NULL;
-
-	return err;
-}
-
-static int tegra_dc_debugfs_exit(struct tegra_dc *dc)
-{
-	drm_debugfs_remove_files(dc->debugfs_files, ARRAY_SIZE(debugfs_files),
-				 dc->minor);
-	dc->minor = NULL;
-
-	kfree(dc->debugfs_files);
-	dc->debugfs_files = NULL;
-
-	debugfs_remove(dc->debugfs);
-	dc->debugfs = NULL;
-
-	return 0;
+	return IRQ_HANDLED;
 }
 
 static int tegra_dc_init(struct host1x_client *client)
 {
 	struct drm_device *drm = dev_get_drvdata(client->parent);
+	struct iommu_group *group = iommu_group_get(client->dev);
 	unsigned long flags = HOST1X_SYNCPT_CLIENT_MANAGED;
 	struct tegra_dc *dc = host1x_client_to_dc(client);
 	struct tegra_drm *tegra = drm->dev_private;
@@ -1742,18 +1832,27 @@ static int tegra_dc_init(struct host1x_client *client)
 	if (!dc->syncpt)
 		dev_warn(dc->dev, "failed to allocate syncpoint\n");
 
-	if (tegra->domain) {
-		err = iommu_attach_device(tegra->domain, dc->dev);
-		if (err < 0) {
-			dev_err(dc->dev, "failed to attach to domain: %d\n",
-				err);
-			return err;
+	if (group && tegra->domain) {
+		if (group != tegra->group) {
+			err = iommu_attach_group(tegra->domain, group);
+			if (err < 0) {
+				dev_err(dc->dev,
+					"failed to attach to domain: %d\n",
+					err);
+				return err;
+			}
+
+			tegra->group = group;
 		}
 
 		dc->domain = tegra->domain;
 	}
 
-	primary = tegra_dc_primary_plane_create(drm, dc);
+	if (dc->soc->wgrps)
+		primary = tegra_dc_add_shared_planes(drm, dc);
+	else
+		primary = tegra_dc_add_planes(drm, dc);
+
 	if (IS_ERR(primary)) {
 		err = PTR_ERR(primary);
 		goto cleanup;
@@ -1787,16 +1886,6 @@ static int tegra_dc_init(struct host1x_client *client)
 		goto cleanup;
 	}
 
-	err = tegra_dc_add_planes(drm, dc);
-	if (err < 0)
-		goto cleanup;
-
-	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
-		err = tegra_dc_debugfs_init(dc, drm->primary);
-		if (err < 0)
-			dev_err(dc->dev, "debugfs setup failed: %d\n", err);
-	}
-
 	err = devm_request_irq(dc->dev, dc->irq, tegra_dc_irq, 0,
 			       dev_name(dc->dev), dc);
 	if (err < 0) {
@@ -1808,14 +1897,18 @@ static int tegra_dc_init(struct host1x_client *client)
 	return 0;
 
 cleanup:
-	if (cursor)
+	if (!IS_ERR_OR_NULL(cursor))
 		drm_plane_cleanup(cursor);
 
-	if (primary)
+	if (!IS_ERR(primary))
 		drm_plane_cleanup(primary);
 
-	if (tegra->domain) {
-		iommu_detach_device(tegra->domain, dc->dev);
+	if (group && dc->domain) {
+		if (group == tegra->group) {
+			iommu_detach_group(dc->domain, group);
+			tegra->group = NULL;
+		}
+
 		dc->domain = NULL;
 	}
 
@@ -1824,25 +1917,26 @@ static int tegra_dc_init(struct host1x_client *client)
 
 static int tegra_dc_exit(struct host1x_client *client)
 {
+	struct drm_device *drm = dev_get_drvdata(client->parent);
+	struct iommu_group *group = iommu_group_get(client->dev);
 	struct tegra_dc *dc = host1x_client_to_dc(client);
+	struct tegra_drm *tegra = drm->dev_private;
 	int err;
 
 	devm_free_irq(dc->dev, dc->irq, dc);
 
-	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
-		err = tegra_dc_debugfs_exit(dc);
-		if (err < 0)
-			dev_err(dc->dev, "debugfs cleanup failed: %d\n", err);
-	}
-
 	err = tegra_dc_rgb_exit(dc);
 	if (err) {
 		dev_err(dc->dev, "failed to shutdown RGB output: %d\n", err);
 		return err;
 	}
 
-	if (dc->domain) {
-		iommu_detach_device(dc->domain, dc->dev);
+	if (group && dc->domain) {
+		if (group == tegra->group) {
+			iommu_detach_group(dc->domain, group);
+			tegra->group = NULL;
+		}
+
 		dc->domain = NULL;
 	}
 
@@ -1857,57 +1951,138 @@ static const struct host1x_client_ops dc_client_ops = {
 };
 
 static const struct tegra_dc_soc_info tegra20_dc_soc_info = {
-	.supports_border_color = true,
+	.supports_background_color = false,
 	.supports_interlacing = false,
 	.supports_cursor = false,
 	.supports_block_linear = false,
+	.supports_blending = false,
 	.pitch_align = 8,
 	.has_powergate = false,
-	.broken_reset = true,
+	.coupled_pm = true,
+	.has_nvdisplay = false,
+	.num_primary_formats = ARRAY_SIZE(tegra20_primary_formats),
+	.primary_formats = tegra20_primary_formats,
+	.num_overlay_formats = ARRAY_SIZE(tegra20_overlay_formats),
+	.overlay_formats = tegra20_overlay_formats,
 };
 
 static const struct tegra_dc_soc_info tegra30_dc_soc_info = {
-	.supports_border_color = true,
+	.supports_background_color = false,
 	.supports_interlacing = false,
 	.supports_cursor = false,
 	.supports_block_linear = false,
+	.supports_blending = false,
 	.pitch_align = 8,
 	.has_powergate = false,
-	.broken_reset = false,
+	.coupled_pm = false,
+	.has_nvdisplay = false,
+	.num_primary_formats = ARRAY_SIZE(tegra20_primary_formats),
+	.primary_formats = tegra20_primary_formats,
+	.num_overlay_formats = ARRAY_SIZE(tegra20_overlay_formats),
+	.overlay_formats = tegra20_overlay_formats,
 };
 
 static const struct tegra_dc_soc_info tegra114_dc_soc_info = {
-	.supports_border_color = true,
+	.supports_background_color = false,
 	.supports_interlacing = false,
 	.supports_cursor = false,
 	.supports_block_linear = false,
+	.supports_blending = false,
 	.pitch_align = 64,
 	.has_powergate = true,
-	.broken_reset = false,
+	.coupled_pm = false,
+	.has_nvdisplay = false,
+	.num_primary_formats = ARRAY_SIZE(tegra114_primary_formats),
+	.primary_formats = tegra114_primary_formats,
+	.num_overlay_formats = ARRAY_SIZE(tegra114_overlay_formats),
+	.overlay_formats = tegra114_overlay_formats,
 };
 
 static const struct tegra_dc_soc_info tegra124_dc_soc_info = {
-	.supports_border_color = false,
+	.supports_background_color = true,
 	.supports_interlacing = true,
 	.supports_cursor = true,
 	.supports_block_linear = true,
+	.supports_blending = true,
 	.pitch_align = 64,
 	.has_powergate = true,
-	.broken_reset = false,
+	.coupled_pm = false,
+	.has_nvdisplay = false,
+	.num_primary_formats = ARRAY_SIZE(tegra124_primary_formats),
+	.primary_formats = tegra124_primary_formats,
+	.num_overlay_formats = ARRAY_SIZE(tegra124_overlay_formats),
+	.overlay_formats = tegra124_overlay_formats,
 };
 
 static const struct tegra_dc_soc_info tegra210_dc_soc_info = {
-	.supports_border_color = false,
+	.supports_background_color = true,
 	.supports_interlacing = true,
 	.supports_cursor = true,
 	.supports_block_linear = true,
+	.supports_blending = true,
 	.pitch_align = 64,
 	.has_powergate = true,
-	.broken_reset = false,
+	.coupled_pm = false,
+	.has_nvdisplay = false,
+	.num_primary_formats = ARRAY_SIZE(tegra114_primary_formats),
+	.primary_formats = tegra114_primary_formats,
+	.num_overlay_formats = ARRAY_SIZE(tegra114_overlay_formats),
+	.overlay_formats = tegra114_overlay_formats,
+};
+
+static const struct tegra_windowgroup_soc tegra186_dc_wgrps[] = {
+	{
+		.index = 0,
+		.dc = 0,
+		.windows = (const unsigned int[]) { 0 },
+		.num_windows = 1,
+	}, {
+		.index = 1,
+		.dc = 1,
+		.windows = (const unsigned int[]) { 1 },
+		.num_windows = 1,
+	}, {
+		.index = 2,
+		.dc = 1,
+		.windows = (const unsigned int[]) { 2 },
+		.num_windows = 1,
+	}, {
+		.index = 3,
+		.dc = 2,
+		.windows = (const unsigned int[]) { 3 },
+		.num_windows = 1,
+	}, {
+		.index = 4,
+		.dc = 2,
+		.windows = (const unsigned int[]) { 4 },
+		.num_windows = 1,
+	}, {
+		.index = 5,
+		.dc = 2,
+		.windows = (const unsigned int[]) { 5 },
+		.num_windows = 1,
+	},
+};
+
+static const struct tegra_dc_soc_info tegra186_dc_soc_info = {
+	.supports_background_color = true,
+	.supports_interlacing = true,
+	.supports_cursor = true,
+	.supports_block_linear = true,
+	.supports_blending = true,
+	.pitch_align = 64,
+	.has_powergate = false,
+	.coupled_pm = false,
+	.has_nvdisplay = true,
+	.wgrps = tegra186_dc_wgrps,
+	.num_wgrps = ARRAY_SIZE(tegra186_dc_wgrps),
 };
 
 static const struct of_device_id tegra_dc_of_match[] = {
 	{
+		.compatible = "nvidia,tegra186-dc",
+		.data = &tegra186_dc_soc_info,
+	}, {
 		.compatible = "nvidia,tegra210-dc",
 		.data = &tegra210_dc_soc_info,
 	}, {
@@ -1965,6 +2140,43 @@ static int tegra_dc_parse_dt(struct tegra_dc *dc)
 	return 0;
 }
 
+static int tegra_dc_match_by_pipe(struct device *dev, void *data)
+{
+	struct tegra_dc *dc = dev_get_drvdata(dev);
+	unsigned int pipe = (unsigned long)data;
+
+	return dc->pipe == pipe;
+}
+
+static int tegra_dc_couple(struct tegra_dc *dc)
+{
+	/*
+	 * On Tegra20, DC1 requires DC0 to be taken out of reset in order to
+	 * be enabled, otherwise CPU hangs on writing to CMD_DISPLAY_COMMAND /
+	 * POWER_CONTROL registers during CRTC enabling.
+	 */
+	if (dc->soc->coupled_pm && dc->pipe == 1) {
+		u32 flags = DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE;
+		struct device_link *link;
+		struct device *partner;
+
+		partner = driver_find_device(dc->dev->driver, NULL, NULL,
+					     tegra_dc_match_by_pipe);
+		if (!partner)
+			return -EPROBE_DEFER;
+
+		link = device_link_add(dc->dev, partner, flags);
+		if (!link) {
+			dev_err(dc->dev, "failed to link controllers\n");
+			return -EINVAL;
+		}
+
+		dev_dbg(dc->dev, "coupled to %s\n", dev_name(partner));
+	}
+
+	return 0;
+}
+
 static int tegra_dc_probe(struct platform_device *pdev)
 {
 	struct resource *regs;
@@ -1977,7 +2189,6 @@ static int tegra_dc_probe(struct platform_device *pdev)
 
 	dc->soc = of_device_get_match_data(&pdev->dev);
 
-	spin_lock_init(&dc->lock);
 	INIT_LIST_HEAD(&dc->list);
 	dc->dev = &pdev->dev;
 
@@ -1985,6 +2196,10 @@ static int tegra_dc_probe(struct platform_device *pdev)
 	if (err < 0)
 		return err;
 
+	err = tegra_dc_couple(dc);
+	if (err < 0)
+		return err;
+
 	dc->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(dc->clk)) {
 		dev_err(&pdev->dev, "failed to get clock\n");
@@ -1998,21 +2213,19 @@ static int tegra_dc_probe(struct platform_device *pdev)
 	}
 
 	/* assert reset and disable clock */
-	if (!dc->soc->broken_reset) {
-		err = clk_prepare_enable(dc->clk);
-		if (err < 0)
-			return err;
+	err = clk_prepare_enable(dc->clk);
+	if (err < 0)
+		return err;
 
-		usleep_range(2000, 4000);
+	usleep_range(2000, 4000);
 
-		err = reset_control_assert(dc->rst);
-		if (err < 0)
-			return err;
+	err = reset_control_assert(dc->rst);
+	if (err < 0)
+		return err;
 
-		usleep_range(2000, 4000);
+	usleep_range(2000, 4000);
 
-		clk_disable_unprepare(dc->clk);
-	}
+	clk_disable_unprepare(dc->clk);
 
 	if (dc->soc->has_powergate) {
 		if (dc->pipe == 0)
@@ -2086,12 +2299,10 @@ static int tegra_dc_suspend(struct device *dev)
 	struct tegra_dc *dc = dev_get_drvdata(dev);
 	int err;
 
-	if (!dc->soc->broken_reset) {
-		err = reset_control_assert(dc->rst);
-		if (err < 0) {
-			dev_err(dev, "failed to assert reset: %d\n", err);
-			return err;
-		}
+	err = reset_control_assert(dc->rst);
+	if (err < 0) {
+		dev_err(dev, "failed to assert reset: %d\n", err);
+		return err;
 	}
 
 	if (dc->soc->has_powergate)
@@ -2121,13 +2332,10 @@ static int tegra_dc_resume(struct device *dev)
 			return err;
 		}
 
-		if (!dc->soc->broken_reset) {
-			err = reset_control_deassert(dc->rst);
-			if (err < 0) {
-				dev_err(dev,
-					"failed to deassert reset: %d\n", err);
-				return err;
-			}
+		err = reset_control_deassert(dc->rst);
+		if (err < 0) {
+			dev_err(dev, "failed to deassert reset: %d\n", err);
+			return err;
 		}
 	}
 
diff --git a/drivers/gpu/drm/tegra/dc.h b/drivers/gpu/drm/tegra/dc.h
index cb100b6e3282..096a81ad6d8d 100644
--- a/drivers/gpu/drm/tegra/dc.h
+++ b/drivers/gpu/drm/tegra/dc.h
@@ -18,6 +18,24 @@
 
 struct tegra_output;
 
+struct tegra_dc_state {
+	struct drm_crtc_state base;
+
+	struct clk *clk;
+	unsigned long pclk;
+	unsigned int div;
+
+	u32 planes;
+};
+
+static inline struct tegra_dc_state *to_dc_state(struct drm_crtc_state *state)
+{
+	if (state)
+		return container_of(state, struct tegra_dc_state, base);
+
+	return NULL;
+}
+
 struct tegra_dc_stats {
 	unsigned long frames;
 	unsigned long vblank;
@@ -25,21 +43,35 @@ struct tegra_dc_stats {
 	unsigned long overflow;
 };
 
+struct tegra_windowgroup_soc {
+	unsigned int index;
+	unsigned int dc;
+	const unsigned int *windows;
+	unsigned int num_windows;
+};
+
 struct tegra_dc_soc_info {
-	bool supports_border_color;
+	bool supports_background_color;
 	bool supports_interlacing;
 	bool supports_cursor;
 	bool supports_block_linear;
+	bool supports_blending;
 	unsigned int pitch_align;
 	bool has_powergate;
-	bool broken_reset;
+	bool coupled_pm;
+	bool has_nvdisplay;
+	const struct tegra_windowgroup_soc *wgrps;
+	unsigned int num_wgrps;
+	const u32 *primary_formats;
+	unsigned int num_primary_formats;
+	const u32 *overlay_formats;
+	unsigned int num_overlay_formats;
 };
 
 struct tegra_dc {
 	struct host1x_client client;
 	struct host1x_syncpt *syncpt;
 	struct device *dev;
-	spinlock_t lock;
 
 	struct drm_crtc base;
 	unsigned int powergate;
@@ -56,11 +88,6 @@ struct tegra_dc {
 	struct list_head list;
 
 	struct drm_info_list *debugfs_files;
-	struct drm_minor *minor;
-	struct dentry *debugfs;
-
-	/* page-flip handling */
-	struct drm_pending_vblank_event *event;
 
 	const struct tegra_dc_soc_info *soc;
 
@@ -110,6 +137,7 @@ struct tegra_dc_window {
 	unsigned int bits_per_pixel;
 	unsigned int stride[2];
 	unsigned long base[3];
+	unsigned int zpos;
 	bool bottom_up;
 
 	struct tegra_bo_tiling tiling;
@@ -118,6 +146,7 @@ struct tegra_dc_window {
 };
 
 /* from dc.c */
+bool tegra_dc_has_output(struct tegra_dc *dc, struct device *dev);
 void tegra_dc_commit(struct tegra_dc *dc);
 int tegra_dc_state_setup_clock(struct tegra_dc *dc,
 			       struct drm_crtc_state *crtc_state,
@@ -167,15 +196,26 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define DC_CMD_INT_ENABLE			0x039
 #define DC_CMD_INT_TYPE				0x03a
 #define DC_CMD_INT_POLARITY			0x03b
-#define CTXSW_INT     (1 << 0)
-#define FRAME_END_INT (1 << 1)
-#define VBLANK_INT    (1 << 2)
-#define WIN_A_UF_INT  (1 << 8)
-#define WIN_B_UF_INT  (1 << 9)
-#define WIN_C_UF_INT  (1 << 10)
-#define WIN_A_OF_INT  (1 << 14)
-#define WIN_B_OF_INT  (1 << 15)
-#define WIN_C_OF_INT  (1 << 16)
+#define CTXSW_INT                (1 << 0)
+#define FRAME_END_INT            (1 << 1)
+#define VBLANK_INT               (1 << 2)
+#define V_PULSE3_INT             (1 << 4)
+#define V_PULSE2_INT             (1 << 5)
+#define REGION_CRC_INT           (1 << 6)
+#define REG_TMOUT_INT            (1 << 7)
+#define WIN_A_UF_INT             (1 << 8)
+#define WIN_B_UF_INT             (1 << 9)
+#define WIN_C_UF_INT             (1 << 10)
+#define MSF_INT                  (1 << 12)
+#define WIN_A_OF_INT             (1 << 14)
+#define WIN_B_OF_INT             (1 << 15)
+#define WIN_C_OF_INT             (1 << 16)
+#define HEAD_UF_INT              (1 << 23)
+#define SD3_BUCKET_WALK_DONE_INT (1 << 24)
+#define DSC_OBUF_UF_INT          (1 << 26)
+#define DSC_RBUF_UF_INT          (1 << 27)
+#define DSC_BBUF_UF_INT          (1 << 28)
+#define DSC_TO_UF_INT            (1 << 29)
 
 #define DC_CMD_SIGNAL_RAISE1			0x03c
 #define DC_CMD_SIGNAL_RAISE2			0x03d
@@ -196,6 +236,8 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define WIN_B_UPDATE    (1 << 10)
 #define WIN_C_UPDATE    (1 << 11)
 #define CURSOR_UPDATE   (1 << 15)
+#define COMMON_ACTREQ   (1 << 16)
+#define COMMON_UPDATE   (1 << 17)
 #define NC_HOST_TRIG    (1 << 24)
 
 #define DC_CMD_DISPLAY_WINDOW_HEADER		0x042
@@ -238,6 +280,10 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define DC_COM_GPIO_DEBOUNCE_COUNTER		0x328
 #define DC_COM_CRC_CHECKSUM_LATCHED		0x329
 
+#define DC_COM_RG_UNDERFLOW			0x365
+#define  UNDERFLOW_MODE_RED      (1 << 8)
+#define  UNDERFLOW_REPORT_ENABLE (1 << 0)
+
 #define DC_DISP_DISP_SIGNAL_OPTIONS0		0x400
 #define H_PULSE0_ENABLE (1 <<  8)
 #define H_PULSE1_ENABLE (1 << 10)
@@ -249,10 +295,10 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define HDMI_ENABLE	(1 << 30)
 #define DSI_ENABLE	(1 << 29)
 #define SOR1_TIMING_CYA	(1 << 27)
-#define SOR1_ENABLE	(1 << 26)
-#define SOR_ENABLE	(1 << 25)
 #define CURSOR_ENABLE	(1 << 16)
 
+#define SOR_ENABLE(x)	(1 << (25 + (x)))
+
 #define DC_DISP_DISP_MEM_HIGH_PRIORITY		0x403
 #define CURSOR_THRESHOLD(x)   (((x) & 0x03) << 24)
 #define WINDOW_A_THRESHOLD(x) (((x) & 0x7f) << 16)
@@ -360,29 +406,33 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define DISP_ORDER_BLUE_RED        (1 << 9)
 
 #define DC_DISP_DISP_COLOR_CONTROL		0x430
-#define BASE_COLOR_SIZE666     (0 << 0)
-#define BASE_COLOR_SIZE111     (1 << 0)
-#define BASE_COLOR_SIZE222     (2 << 0)
-#define BASE_COLOR_SIZE333     (3 << 0)
-#define BASE_COLOR_SIZE444     (4 << 0)
-#define BASE_COLOR_SIZE555     (5 << 0)
-#define BASE_COLOR_SIZE565     (6 << 0)
-#define BASE_COLOR_SIZE332     (7 << 0)
-#define BASE_COLOR_SIZE888     (8 << 0)
+#define BASE_COLOR_SIZE666     ( 0 << 0)
+#define BASE_COLOR_SIZE111     ( 1 << 0)
+#define BASE_COLOR_SIZE222     ( 2 << 0)
+#define BASE_COLOR_SIZE333     ( 3 << 0)
+#define BASE_COLOR_SIZE444     ( 4 << 0)
+#define BASE_COLOR_SIZE555     ( 5 << 0)
+#define BASE_COLOR_SIZE565     ( 6 << 0)
+#define BASE_COLOR_SIZE332     ( 7 << 0)
+#define BASE_COLOR_SIZE888     ( 8 << 0)
+#define BASE_COLOR_SIZE101010  (10 << 0)
+#define BASE_COLOR_SIZE121212  (12 << 0)
 #define DITHER_CONTROL_MASK    (3 << 8)
 #define DITHER_CONTROL_DISABLE (0 << 8)
 #define DITHER_CONTROL_ORDERED (2 << 8)
 #define DITHER_CONTROL_ERRDIFF (3 << 8)
 #define BASE_COLOR_SIZE_MASK   (0xf << 0)
-#define BASE_COLOR_SIZE_666    (0 << 0)
-#define BASE_COLOR_SIZE_111    (1 << 0)
-#define BASE_COLOR_SIZE_222    (2 << 0)
-#define BASE_COLOR_SIZE_333    (3 << 0)
-#define BASE_COLOR_SIZE_444    (4 << 0)
-#define BASE_COLOR_SIZE_555    (5 << 0)
-#define BASE_COLOR_SIZE_565    (6 << 0)
-#define BASE_COLOR_SIZE_332    (7 << 0)
-#define BASE_COLOR_SIZE_888    (8 << 0)
+#define BASE_COLOR_SIZE_666    (  0 << 0)
+#define BASE_COLOR_SIZE_111    (  1 << 0)
+#define BASE_COLOR_SIZE_222    (  2 << 0)
+#define BASE_COLOR_SIZE_333    (  3 << 0)
+#define BASE_COLOR_SIZE_444    (  4 << 0)
+#define BASE_COLOR_SIZE_555    (  5 << 0)
+#define BASE_COLOR_SIZE_565    (  6 << 0)
+#define BASE_COLOR_SIZE_332    (  7 << 0)
+#define BASE_COLOR_SIZE_888    (  8 << 0)
+#define BASE_COLOR_SIZE_101010 ( 10 << 0)
+#define BASE_COLOR_SIZE_121212 ( 12 << 0)
 
 #define DC_DISP_SHIFT_CLOCK_OPTIONS		0x431
 #define  SC1_H_QUALIFIER_NONE	(1 << 16)
@@ -449,6 +499,12 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define DC_DISP_SD_HW_K_VALUES			0x4dd
 #define DC_DISP_SD_MAN_K_VALUES			0x4de
 
+#define DC_DISP_BLEND_BACKGROUND_COLOR		0x4e4
+#define  BACKGROUND_COLOR_ALPHA(x) (((x) & 0xff) << 24)
+#define  BACKGROUND_COLOR_BLUE(x)  (((x) & 0xff) << 16)
+#define  BACKGROUND_COLOR_GREEN(x) (((x) & 0xff) << 8)
+#define  BACKGROUND_COLOR_RED(x)   (((x) & 0xff) << 0)
+
 #define DC_DISP_INTERLACE_CONTROL		0x4e5
 #define  INTERLACE_STATUS (1 << 2)
 #define  INTERLACE_START  (1 << 1)
@@ -467,6 +523,35 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define CURSOR_SRC_BLEND_MASK			(3 << 8)
 #define CURSOR_ALPHA				0xff
 
+#define DC_WIN_CORE_ACT_CONTROL 0x50e
+#define  VCOUNTER (0 << 0)
+#define  HCOUNTER (1 << 0)
+
+#define DC_WIN_CORE_IHUB_WGRP_LATENCY_CTLA 0x543
+#define  LATENCY_CTL_MODE_ENABLE (1 << 2)
+
+#define DC_WIN_CORE_IHUB_WGRP_LATENCY_CTLB 0x544
+#define  WATERMARK_MASK 0x1fffffff
+
+#define DC_WIN_CORE_PRECOMP_WGRP_PIPE_METER 0x560
+#define  PIPE_METER_INT(x)  (((x) & 0xff) << 8)
+#define  PIPE_METER_FRAC(x) (((x) & 0xff) << 0)
+
+#define DC_WIN_CORE_IHUB_WGRP_POOL_CONFIG 0x561
+#define  MEMPOOL_ENTRIES(x) (((x) & 0xffff) << 0)
+
+#define DC_WIN_CORE_IHUB_WGRP_FETCH_METER 0x562
+#define  SLOTS(x) (((x) & 0xff) << 0)
+
+#define DC_WIN_CORE_IHUB_LINEBUF_CONFIG 0x563
+#define  MODE_TWO_LINES  (0 << 14)
+#define  MODE_FOUR_LINES (1 << 14)
+
+#define DC_WIN_CORE_IHUB_THREAD_GROUP 0x568
+#define  THREAD_NUM_MASK (0x1f << 1)
+#define  THREAD_NUM(x) (((x) & 0x1f) << 1)
+#define  THREAD_GROUP_ENABLE (1 << 0)
+
 #define DC_WIN_CSC_YOF				0x611
 #define DC_WIN_CSC_KYRGB			0x612
 #define DC_WIN_CSC_KUR				0x613
@@ -502,9 +587,9 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define WIN_COLOR_DEPTH_P4              2
 #define WIN_COLOR_DEPTH_P8              3
 #define WIN_COLOR_DEPTH_B4G4R4A4        4
-#define WIN_COLOR_DEPTH_B5G5R5A         5
+#define WIN_COLOR_DEPTH_B5G5R5A1        5
 #define WIN_COLOR_DEPTH_B5G6R5          6
-#define WIN_COLOR_DEPTH_AB5G5R5         7
+#define WIN_COLOR_DEPTH_A1B5G5R5        7
 #define WIN_COLOR_DEPTH_B8G8R8A8       12
 #define WIN_COLOR_DEPTH_R8G8B8A8       13
 #define WIN_COLOR_DEPTH_B6x2G6x2R6x2A8 14
@@ -519,18 +604,32 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define WIN_COLOR_DEPTH_YUV422R        23
 #define WIN_COLOR_DEPTH_YCbCr422RA     24
 #define WIN_COLOR_DEPTH_YUV422RA       25
+#define WIN_COLOR_DEPTH_R4G4B4A4       27
+#define WIN_COLOR_DEPTH_R5G5B5A        28
+#define WIN_COLOR_DEPTH_AR5G5B5        29
+#define WIN_COLOR_DEPTH_B5G5R5X1       30
+#define WIN_COLOR_DEPTH_X1B5G5R5       31
+#define WIN_COLOR_DEPTH_R5G5B5X1       32
+#define WIN_COLOR_DEPTH_X1R5G5B5       33
+#define WIN_COLOR_DEPTH_R5G6B5         34
+#define WIN_COLOR_DEPTH_A8R8G8B8       35
+#define WIN_COLOR_DEPTH_A8B8G8R8       36
+#define WIN_COLOR_DEPTH_B8G8R8X8       37
+#define WIN_COLOR_DEPTH_R8G8B8X8       38
+#define WIN_COLOR_DEPTH_X8B8G8R8       65
+#define WIN_COLOR_DEPTH_X8R8G8B8       66
 
 #define DC_WIN_POSITION				0x704
-#define H_POSITION(x) (((x) & 0x1fff) <<  0)
-#define V_POSITION(x) (((x) & 0x1fff) << 16)
+#define H_POSITION(x) (((x) & 0x1fff) <<  0) /* XXX 0x7fff on Tegra186 */
+#define V_POSITION(x) (((x) & 0x1fff) << 16) /* XXX 0x7fff on Tegra186 */
 
 #define DC_WIN_SIZE				0x705
-#define H_SIZE(x) (((x) & 0x1fff) <<  0)
-#define V_SIZE(x) (((x) & 0x1fff) << 16)
+#define H_SIZE(x) (((x) & 0x1fff) <<  0) /* XXX 0x7fff on Tegra186 */
+#define V_SIZE(x) (((x) & 0x1fff) << 16) /* XXX 0x7fff on Tegra186 */
 
 #define DC_WIN_PRESCALED_SIZE			0x706
 #define H_PRESCALED_SIZE(x) (((x) & 0x7fff) <<  0)
-#define V_PRESCALED_SIZE(x) (((x) & 0x1fff) << 16)
+#define V_PRESCALED_SIZE(x) (((x) & 0x1fff) << 16) /* XXX 0x7fff on Tegra186 */
 
 #define DC_WIN_H_INITIAL_DDA			0x707
 #define DC_WIN_V_INITIAL_DDA			0x708
@@ -546,11 +645,24 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define DC_WIN_BUFFER_ADDR_MODE_TILE		(1 <<  0)
 #define DC_WIN_BUFFER_ADDR_MODE_LINEAR_UV	(0 << 16)
 #define DC_WIN_BUFFER_ADDR_MODE_TILE_UV		(1 << 16)
+
 #define DC_WIN_DV_CONTROL			0x70e
 
 #define DC_WIN_BLEND_NOKEY			0x70f
+#define  BLEND_WEIGHT1(x) (((x) & 0xff) << 16)
+#define  BLEND_WEIGHT0(x) (((x) & 0xff) <<  8)
+
 #define DC_WIN_BLEND_1WIN			0x710
+#define  BLEND_CONTROL_FIX    (0 << 2)
+#define  BLEND_CONTROL_ALPHA  (1 << 2)
+#define  BLEND_COLOR_KEY_NONE (0 << 0)
+#define  BLEND_COLOR_KEY_0    (1 << 0)
+#define  BLEND_COLOR_KEY_1    (2 << 0)
+#define  BLEND_COLOR_KEY_BOTH (3 << 0)
+
 #define DC_WIN_BLEND_2WIN_X			0x711
+#define  BLEND_CONTROL_DEPENDENT (2 << 2)
+
 #define DC_WIN_BLEND_2WIN_Y			0x712
 #define DC_WIN_BLEND_3WIN_XY			0x713
 
@@ -575,8 +687,97 @@ int tegra_dc_rgb_exit(struct tegra_dc *dc);
 #define DC_WINBUF_SURFACE_KIND_BLOCK	(2 << 0)
 #define DC_WINBUF_SURFACE_KIND_BLOCK_HEIGHT(x) (((x) & 0x7) << 4)
 
+#define DC_WINBUF_START_ADDR_HI			0x80d
+
+#define DC_WINBUF_CDE_CONTROL			0x82f
+#define  ENABLE_SURFACE (1 << 0)
+
 #define DC_WINBUF_AD_UFLOW_STATUS		0xbca
 #define DC_WINBUF_BD_UFLOW_STATUS		0xdca
 #define DC_WINBUF_CD_UFLOW_STATUS		0xfca
 
+/* Tegra186 and later */
+#define DC_DISP_CORE_SOR_SET_CONTROL(x)		(0x403 + (x))
+#define PROTOCOL_MASK (0xf << 8)
+#define PROTOCOL_SINGLE_TMDS_A (0x1 << 8)
+
+#define DC_WIN_CORE_WINDOWGROUP_SET_CONTROL	0x702
+#define OWNER_MASK (0xf << 0)
+#define OWNER(x) (((x) & 0xf) << 0)
+
+#define DC_WIN_CROPPED_SIZE			0x706
+
+#define DC_WIN_PLANAR_STORAGE			0x709
+#define PITCH(x) (((x) >> 6) & 0x1fff)
+
+#define DC_WIN_SET_PARAMS			0x70d
+#define  CLAMP_BEFORE_BLEND (1 << 15)
+#define  DEGAMMA_NONE (0 << 13)
+#define  DEGAMMA_SRGB (1 << 13)
+#define  DEGAMMA_YUV8_10 (2 << 13)
+#define  DEGAMMA_YUV12 (3 << 13)
+#define  INPUT_RANGE_BYPASS (0 << 10)
+#define  INPUT_RANGE_LIMITED (1 << 10)
+#define  INPUT_RANGE_FULL (2 << 10)
+#define  COLOR_SPACE_RGB (0 << 8)
+#define  COLOR_SPACE_YUV_601 (1 << 8)
+#define  COLOR_SPACE_YUV_709 (2 << 8)
+#define  COLOR_SPACE_YUV_2020 (3 << 8)
+
+#define DC_WIN_WINDOWGROUP_SET_CONTROL_INPUT_SCALER	0x70e
+#define  HORIZONTAL_TAPS_2 (1 << 3)
+#define  HORIZONTAL_TAPS_5 (4 << 3)
+#define  VERTICAL_TAPS_2 (1 << 0)
+#define  VERTICAL_TAPS_5 (4 << 0)
+
+#define DC_WIN_WINDOWGROUP_SET_INPUT_SCALER_USAGE	0x711
+#define  INPUT_SCALER_USE422  (1 << 2)
+#define  INPUT_SCALER_VBYPASS (1 << 1)
+#define  INPUT_SCALER_HBYPASS (1 << 0)
+
+#define DC_WIN_BLEND_LAYER_CONTROL		0x716
+#define  COLOR_KEY_NONE (0 << 25)
+#define  COLOR_KEY_SRC (1 << 25)
+#define  COLOR_KEY_DST (2 << 25)
+#define  BLEND_BYPASS (1 << 24)
+#define  K2(x) (((x) & 0xff) << 16)
+#define  K1(x) (((x) & 0xff) << 8)
+#define  WINDOW_LAYER_DEPTH(x) (((x) & 0xff) << 0)
+
+#define DC_WIN_BLEND_MATCH_SELECT		0x717
+#define  BLEND_FACTOR_DST_ALPHA_ZERO			(0 << 12)
+#define  BLEND_FACTOR_DST_ALPHA_ONE			(1 << 12)
+#define  BLEND_FACTOR_DST_ALPHA_NEG_K1_TIMES_SRC	(2 << 12)
+#define  BLEND_FACTOR_DST_ALPHA_K2			(3 << 12)
+#define  BLEND_FACTOR_SRC_ALPHA_ZERO			(0 << 8)
+#define  BLEND_FACTOR_SRC_ALPHA_K1			(1 << 8)
+#define  BLEND_FACTOR_SRC_ALPHA_K2			(2 << 8)
+#define  BLEND_FACTOR_SRC_ALPHA_NEG_K1_TIMES_DST	(3 << 8)
+#define  BLEND_FACTOR_DST_COLOR_ZERO			(0 << 4)
+#define  BLEND_FACTOR_DST_COLOR_ONE			(1 << 4)
+#define  BLEND_FACTOR_DST_COLOR_K1			(2 << 4)
+#define  BLEND_FACTOR_DST_COLOR_K2			(3 << 4)
+#define  BLEND_FACTOR_DST_COLOR_K1_TIMES_DST		(4 << 4)
+#define  BLEND_FACTOR_DST_COLOR_NEG_K1_TIMES_DST	(5 << 4)
+#define  BLEND_FACTOR_DST_COLOR_NEG_K1_TIMES_SRC	(6 << 4)
+#define  BLEND_FACTOR_DST_COLOR_NEG_K1			(7 << 4)
+#define  BLEND_FACTOR_SRC_COLOR_ZERO			(0 << 0)
+#define  BLEND_FACTOR_SRC_COLOR_ONE			(1 << 0)
+#define  BLEND_FACTOR_SRC_COLOR_K1			(2 << 0)
+#define  BLEND_FACTOR_SRC_COLOR_K1_TIMES_DST		(3 << 0)
+#define  BLEND_FACTOR_SRC_COLOR_NEG_K1_TIMES_DST	(4 << 0)
+#define  BLEND_FACTOR_SRC_COLOR_K1_TIMES_SRC		(5 << 0)
+
+#define DC_WIN_BLEND_NOMATCH_SELECT		0x718
+
+#define DC_WIN_PRECOMP_WGRP_PARAMS		0x724
+#define  SWAP_UV (1 << 0)
+
+#define DC_WIN_WINDOW_SET_CONTROL		0x730
+#define  CONTROL_CSC_ENABLE (1 << 5)
+
+#define DC_WINBUF_CROPPED_POINT			0x806
+#define OFFSET_Y(x) (((x) & 0xffff) << 16)
+#define OFFSET_X(x) (((x) & 0xffff) << 0)
+
 #endif /* TEGRA_DC_H */
diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c
index e4da041ba89b..d84e81ff36ad 100644
--- a/drivers/gpu/drm/tegra/dpaux.c
+++ b/drivers/gpu/drm/tegra/dpaux.c
@@ -15,6 +15,7 @@
 #include <linux/pinctrl/pinconf-generic.h>
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/pinctrl/pinmux.h>
+#include <linux/pm_runtime.h>
 #include <linux/platform_device.h>
 #include <linux/reset.h>
 #include <linux/regulator/consumer.h>
@@ -321,6 +322,9 @@ static int tegra_dpaux_pad_config(struct tegra_dpaux *dpaux, unsigned function)
 	case DPAUX_PADCTL_FUNC_I2C:
 		value = DPAUX_HYBRID_PADCTL_I2C_SDA_INPUT_RCV |
 			DPAUX_HYBRID_PADCTL_I2C_SCL_INPUT_RCV |
+			DPAUX_HYBRID_PADCTL_AUX_CMH(2) |
+			DPAUX_HYBRID_PADCTL_AUX_DRVZ(4) |
+			DPAUX_HYBRID_PADCTL_AUX_DRVI(0x18) |
 			DPAUX_HYBRID_PADCTL_MODE_I2C;
 		break;
 
@@ -467,52 +471,37 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
 		return PTR_ERR(dpaux->clk);
 	}
 
-	err = clk_prepare_enable(dpaux->clk);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to enable module clock: %d\n",
-			err);
-		return err;
-	}
-
-	if (dpaux->rst)
-		reset_control_deassert(dpaux->rst);
-
 	dpaux->clk_parent = devm_clk_get(&pdev->dev, "parent");
 	if (IS_ERR(dpaux->clk_parent)) {
 		dev_err(&pdev->dev, "failed to get parent clock: %ld\n",
 			PTR_ERR(dpaux->clk_parent));
-		err = PTR_ERR(dpaux->clk_parent);
-		goto assert_reset;
-	}
-
-	err = clk_prepare_enable(dpaux->clk_parent);
-	if (err < 0) {
-		dev_err(&pdev->dev, "failed to enable parent clock: %d\n",
-			err);
-		goto assert_reset;
+		return PTR_ERR(dpaux->clk_parent);
 	}
 
 	err = clk_set_rate(dpaux->clk_parent, 270000000);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to set clock to 270 MHz: %d\n",
 			err);
-		goto disable_parent_clk;
+		return err;
 	}
 
 	dpaux->vdd = devm_regulator_get(&pdev->dev, "vdd");
 	if (IS_ERR(dpaux->vdd)) {
 		dev_err(&pdev->dev, "failed to get VDD supply: %ld\n",
 			PTR_ERR(dpaux->vdd));
-		err = PTR_ERR(dpaux->vdd);
-		goto disable_parent_clk;
+		return PTR_ERR(dpaux->vdd);
 	}
 
+	platform_set_drvdata(pdev, dpaux);
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_get_sync(&pdev->dev);
+
 	err = devm_request_irq(dpaux->dev, dpaux->irq, tegra_dpaux_irq, 0,
 			       dev_name(dpaux->dev), dpaux);
 	if (err < 0) {
 		dev_err(dpaux->dev, "failed to request IRQ#%u: %d\n",
 			dpaux->irq, err);
-		goto disable_parent_clk;
+		return err;
 	}
 
 	disable_irq(dpaux->irq);
@@ -522,7 +511,7 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
 
 	err = drm_dp_aux_register(&dpaux->aux);
 	if (err < 0)
-		goto disable_parent_clk;
+		return err;
 
 	/*
 	 * Assume that by default the DPAUX/I2C pads will be used for HDMI,
@@ -560,47 +549,97 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
 	list_add_tail(&dpaux->list, &dpaux_list);
 	mutex_unlock(&dpaux_lock);
 
-	platform_set_drvdata(pdev, dpaux);
-
 	return 0;
-
-disable_parent_clk:
-	clk_disable_unprepare(dpaux->clk_parent);
-assert_reset:
-	if (dpaux->rst)
-		reset_control_assert(dpaux->rst);
-
-	clk_disable_unprepare(dpaux->clk);
-
-	return err;
 }
 
 static int tegra_dpaux_remove(struct platform_device *pdev)
 {
 	struct tegra_dpaux *dpaux = platform_get_drvdata(pdev);
 
+	cancel_work_sync(&dpaux->work);
+
 	/* make sure pads are powered down when not in use */
 	tegra_dpaux_pad_power_down(dpaux);
 
+	pm_runtime_put(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
 	drm_dp_aux_unregister(&dpaux->aux);
 
 	mutex_lock(&dpaux_lock);
 	list_del(&dpaux->list);
 	mutex_unlock(&dpaux_lock);
 
-	cancel_work_sync(&dpaux->work);
+	return 0;
+}
 
-	clk_disable_unprepare(dpaux->clk_parent);
+#ifdef CONFIG_PM
+static int tegra_dpaux_suspend(struct device *dev)
+{
+	struct tegra_dpaux *dpaux = dev_get_drvdata(dev);
+	int err = 0;
+
+	if (dpaux->rst) {
+		err = reset_control_assert(dpaux->rst);
+		if (err < 0) {
+			dev_err(dev, "failed to assert reset: %d\n", err);
+			return err;
+		}
+	}
 
-	if (dpaux->rst)
-		reset_control_assert(dpaux->rst);
+	usleep_range(1000, 2000);
 
+	clk_disable_unprepare(dpaux->clk_parent);
 	clk_disable_unprepare(dpaux->clk);
 
+	return err;
+}
+
+static int tegra_dpaux_resume(struct device *dev)
+{
+	struct tegra_dpaux *dpaux = dev_get_drvdata(dev);
+	int err;
+
+	err = clk_prepare_enable(dpaux->clk);
+	if (err < 0) {
+		dev_err(dev, "failed to enable clock: %d\n", err);
+		return err;
+	}
+
+	err = clk_prepare_enable(dpaux->clk_parent);
+	if (err < 0) {
+		dev_err(dev, "failed to enable parent clock: %d\n", err);
+		goto disable_clk;
+	}
+
+	usleep_range(1000, 2000);
+
+	if (dpaux->rst) {
+		err = reset_control_deassert(dpaux->rst);
+		if (err < 0) {
+			dev_err(dev, "failed to deassert reset: %d\n", err);
+			goto disable_parent;
+		}
+
+		usleep_range(1000, 2000);
+	}
+
 	return 0;
+
+disable_parent:
+	clk_disable_unprepare(dpaux->clk_parent);
+disable_clk:
+	clk_disable_unprepare(dpaux->clk);
+	return err;
 }
+#endif
+
+static const struct dev_pm_ops tegra_dpaux_pm_ops = {
+	SET_RUNTIME_PM_OPS(tegra_dpaux_suspend, tegra_dpaux_resume, NULL)
+};
 
 static const struct of_device_id tegra_dpaux_of_match[] = {
+	{ .compatible = "nvidia,tegra186-dpaux", },
 	{ .compatible = "nvidia,tegra210-dpaux", },
 	{ .compatible = "nvidia,tegra124-dpaux", },
 	{ },
@@ -611,6 +650,7 @@ struct platform_driver tegra_dpaux_driver = {
 	.driver = {
 		.name = "tegra-dpaux",
 		.of_match_table = tegra_dpaux_of_match,
+		.pm = &tegra_dpaux_pm_ops,
 	},
 	.probe = tegra_dpaux_probe,
 	.remove = tegra_dpaux_remove,
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 52552b9b89ef..7fcf4a242840 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -33,97 +33,91 @@ struct tegra_drm_file {
 	struct mutex lock;
 };
 
-static void tegra_atomic_schedule(struct tegra_drm *tegra,
-				  struct drm_atomic_state *state)
+static int tegra_atomic_check(struct drm_device *drm,
+			      struct drm_atomic_state *state)
 {
-	tegra->commit.state = state;
-	schedule_work(&tegra->commit.work);
-}
+	int err;
 
-static void tegra_atomic_complete(struct tegra_drm *tegra,
-				  struct drm_atomic_state *state)
-{
-	struct drm_device *drm = tegra->drm;
+	err = drm_atomic_helper_check_modeset(drm, state);
+	if (err < 0)
+		return err;
 
-	/*
-	 * Everything below can be run asynchronously without the need to grab
-	 * any modeset locks at all under one condition: It must be guaranteed
-	 * that the asynchronous work has either been cancelled (if the driver
-	 * supports it, which at least requires that the framebuffers get
-	 * cleaned up with drm_atomic_helper_cleanup_planes()) or completed
-	 * before the new state gets committed on the software side with
-	 * drm_atomic_helper_swap_state().
-	 *
-	 * This scheme allows new atomic state updates to be prepared and
-	 * checked in parallel to the asynchronous completion of the previous
-	 * update. Which is important since compositors need to figure out the
-	 * composition of the next frame right after having submitted the
-	 * current layout.
-	 */
+	err = drm_atomic_normalize_zpos(drm, state);
+	if (err < 0)
+		return err;
 
-	drm_atomic_helper_commit_modeset_disables(drm, state);
-	drm_atomic_helper_commit_modeset_enables(drm, state);
-	drm_atomic_helper_commit_planes(drm, state,
-					DRM_PLANE_COMMIT_ACTIVE_ONLY);
+	err = drm_atomic_helper_check_planes(drm, state);
+	if (err < 0)
+		return err;
 
-	drm_atomic_helper_wait_for_vblanks(drm, state);
+	if (state->legacy_cursor_update)
+		state->async_update = !drm_atomic_helper_async_check(drm, state);
 
-	drm_atomic_helper_cleanup_planes(drm, state);
-	drm_atomic_state_put(state);
+	return 0;
 }
 
-static void tegra_atomic_work(struct work_struct *work)
+static struct drm_atomic_state *
+tegra_atomic_state_alloc(struct drm_device *drm)
 {
-	struct tegra_drm *tegra = container_of(work, struct tegra_drm,
-					       commit.work);
+	struct tegra_atomic_state *state = kzalloc(sizeof(*state), GFP_KERNEL);
+
+	if (!state || drm_atomic_state_init(drm, &state->base) < 0) {
+		kfree(state);
+		return NULL;
+	}
 
-	tegra_atomic_complete(tegra, tegra->commit.state);
+	return &state->base;
 }
 
-static int tegra_atomic_commit(struct drm_device *drm,
-			       struct drm_atomic_state *state, bool nonblock)
+static void tegra_atomic_state_clear(struct drm_atomic_state *state)
 {
-	struct tegra_drm *tegra = drm->dev_private;
-	int err;
-
-	err = drm_atomic_helper_prepare_planes(drm, state);
-	if (err)
-		return err;
+	struct tegra_atomic_state *tegra = to_tegra_atomic_state(state);
 
-	/* serialize outstanding nonblocking commits */
-	mutex_lock(&tegra->commit.lock);
-	flush_work(&tegra->commit.work);
-
-	/*
-	 * This is the point of no return - everything below never fails except
-	 * when the hw goes bonghits. Which means we can commit the new state on
-	 * the software side now.
-	 */
-
-	err = drm_atomic_helper_swap_state(state, true);
-	if (err) {
-		mutex_unlock(&tegra->commit.lock);
-		drm_atomic_helper_cleanup_planes(drm, state);
-		return err;
-	}
-
-	drm_atomic_state_get(state);
-	if (nonblock)
-		tegra_atomic_schedule(tegra, state);
-	else
-		tegra_atomic_complete(tegra, state);
+	drm_atomic_state_default_clear(state);
+	tegra->clk_disp = NULL;
+	tegra->dc = NULL;
+	tegra->rate = 0;
+}
 
-	mutex_unlock(&tegra->commit.lock);
-	return 0;
+static void tegra_atomic_state_free(struct drm_atomic_state *state)
+{
+	drm_atomic_state_default_release(state);
+	kfree(state);
 }
 
-static const struct drm_mode_config_funcs tegra_drm_mode_funcs = {
+static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = {
 	.fb_create = tegra_fb_create,
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-	.output_poll_changed = tegra_fb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 #endif
-	.atomic_check = drm_atomic_helper_check,
-	.atomic_commit = tegra_atomic_commit,
+	.atomic_check = tegra_atomic_check,
+	.atomic_commit = drm_atomic_helper_commit,
+	.atomic_state_alloc = tegra_atomic_state_alloc,
+	.atomic_state_clear = tegra_atomic_state_clear,
+	.atomic_state_free = tegra_atomic_state_free,
+};
+
+static void tegra_atomic_commit_tail(struct drm_atomic_state *old_state)
+{
+	struct drm_device *drm = old_state->dev;
+	struct tegra_drm *tegra = drm->dev_private;
+
+	if (tegra->hub) {
+		drm_atomic_helper_commit_modeset_disables(drm, old_state);
+		tegra_display_hub_atomic_commit(drm, old_state);
+		drm_atomic_helper_commit_planes(drm, old_state, 0);
+		drm_atomic_helper_commit_modeset_enables(drm, old_state);
+		drm_atomic_helper_commit_hw_done(old_state);
+		drm_atomic_helper_wait_for_vblanks(drm, old_state);
+		drm_atomic_helper_cleanup_planes(drm, old_state);
+	} else {
+		drm_atomic_helper_commit_tail_rpm(old_state);
+	}
+}
+
+static const struct drm_mode_config_helper_funcs
+tegra_drm_mode_config_helpers = {
+	.atomic_commit_tail = tegra_atomic_commit_tail,
 };
 
 static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
@@ -172,9 +166,6 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 	mutex_init(&tegra->clients_lock);
 	INIT_LIST_HEAD(&tegra->clients);
 
-	mutex_init(&tegra->commit.lock);
-	INIT_WORK(&tegra->commit.work, tegra_atomic_work);
-
 	drm->dev_private = tegra;
 	tegra->drm = drm;
 
@@ -188,7 +179,8 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 
 	drm->mode_config.allow_fb_modifiers = true;
 
-	drm->mode_config.funcs = &tegra_drm_mode_funcs;
+	drm->mode_config.funcs = &tegra_drm_mode_config_funcs;
+	drm->mode_config.helper_private = &tegra_drm_mode_config_helpers;
 
 	err = tegra_drm_fb_prepare(drm);
 	if (err < 0)
@@ -200,6 +192,12 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 	if (err < 0)
 		goto fbdev;
 
+	if (tegra->hub) {
+		err = tegra_display_hub_prepare(tegra->hub);
+		if (err < 0)
+			goto device;
+	}
+
 	/*
 	 * We don't use the drm_irq_install() helpers provided by the DRM
 	 * core, so we need to set this manually in order to allow the
@@ -212,16 +210,19 @@ static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 
 	err = drm_vblank_init(drm, drm->mode_config.num_crtc);
 	if (err < 0)
-		goto device;
+		goto hub;
 
 	drm_mode_config_reset(drm);
 
 	err = tegra_drm_fb_init(drm);
 	if (err < 0)
-		goto device;
+		goto hub;
 
 	return 0;
 
+hub:
+	if (tegra->hub)
+		tegra_display_hub_cleanup(tegra->hub);
 device:
 	host1x_device_exit(device);
 fbdev:
@@ -249,6 +250,7 @@ static void tegra_drm_unload(struct drm_device *drm)
 
 	drm_kms_helper_poll_fini(drm);
 	tegra_drm_fb_exit(drm);
+	drm_atomic_helper_shutdown(drm);
 	drm_mode_config_cleanup(drm);
 
 	err = host1x_device_exit(device);
@@ -286,15 +288,6 @@ static void tegra_drm_context_free(struct tegra_drm_context *context)
 	kfree(context);
 }
 
-static void tegra_drm_lastclose(struct drm_device *drm)
-{
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	struct tegra_drm *tegra = drm->dev_private;
-
-	tegra_fbdev_restore_mode(tegra->fbdev);
-#endif
-}
-
 static struct host1x_bo *
 host1x_bo_lookup(struct drm_file *file, u32 handle)
 {
@@ -660,7 +653,8 @@ static int tegra_syncpt_wait(struct drm_device *drm, void *data,
 	if (!sp)
 		return -EINVAL;
 
-	return host1x_syncpt_wait(sp, args->thresh, args->timeout,
+	return host1x_syncpt_wait(sp, args->thresh,
+				  msecs_to_jiffies(args->timeout),
 				  &args->value);
 }
 
@@ -1100,7 +1094,7 @@ static struct drm_driver tegra_drm_driver = {
 	.unload = tegra_drm_unload,
 	.open = tegra_drm_open,
 	.postclose = tegra_drm_postclose,
-	.lastclose = tegra_drm_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 
 #if defined(CONFIG_DEBUG_FS)
 	.debugfs_init = tegra_debugfs_init,
@@ -1148,8 +1142,7 @@ int tegra_drm_unregister_client(struct tegra_drm *tegra,
 	return 0;
 }
 
-void *tegra_drm_alloc(struct tegra_drm *tegra, size_t size,
-			      dma_addr_t *dma)
+void *tegra_drm_alloc(struct tegra_drm *tegra, size_t size, dma_addr_t *dma)
 {
 	struct iova *alloc;
 	void *virt;
@@ -1317,6 +1310,10 @@ static const struct of_device_id host1x_drm_subdevs[] = {
 	{ .compatible = "nvidia,tegra210-sor", },
 	{ .compatible = "nvidia,tegra210-sor1", },
 	{ .compatible = "nvidia,tegra210-vic", },
+	{ .compatible = "nvidia,tegra186-display", },
+	{ .compatible = "nvidia,tegra186-dc", },
+	{ .compatible = "nvidia,tegra186-sor", },
+	{ .compatible = "nvidia,tegra186-sor1", },
 	{ .compatible = "nvidia,tegra186-vic", },
 	{ /* sentinel */ }
 };
@@ -1332,6 +1329,7 @@ static struct host1x_driver host1x_drm_driver = {
 };
 
 static struct platform_driver * const drivers[] = {
+	&tegra_display_hub_driver,
 	&tegra_dc_driver,
 	&tegra_hdmi_driver,
 	&tegra_dsi_driver,
diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h
index ddae331ad8b6..73b661ce7086 100644
--- a/drivers/gpu/drm/tegra/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -16,6 +16,7 @@
 #include <linux/of_gpio.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_encoder.h>
@@ -23,6 +24,7 @@
 #include <drm/drm_fixed.h>
 
 #include "gem.h"
+#include "hub.h"
 #include "trace.h"
 
 struct reset_control;
@@ -40,10 +42,25 @@ struct tegra_fbdev {
 };
 #endif
 
+struct tegra_atomic_state {
+	struct drm_atomic_state base;
+
+	struct clk *clk_disp;
+	struct tegra_dc *dc;
+	unsigned long rate;
+};
+
+static inline struct tegra_atomic_state *
+to_tegra_atomic_state(struct drm_atomic_state *state)
+{
+	return container_of(state, struct tegra_atomic_state, base);
+}
+
 struct tegra_drm {
 	struct drm_device *drm;
 
 	struct iommu_domain *domain;
+	struct iommu_group *group;
 	struct mutex mm_lock;
 	struct drm_mm mm;
 
@@ -62,11 +79,7 @@ struct tegra_drm {
 
 	unsigned int pitch_align;
 
-	struct {
-		struct drm_atomic_state *state;
-		struct work_struct work;
-		struct mutex lock;
-	} commit;
+	struct tegra_display_hub *hub;
 
 	struct drm_atomic_state *state;
 };
@@ -152,6 +165,8 @@ int tegra_output_probe(struct tegra_output *output);
 void tegra_output_remove(struct tegra_output *output);
 int tegra_output_init(struct drm_device *drm, struct tegra_output *output);
 void tegra_output_exit(struct tegra_output *output);
+void tegra_output_find_possible_crtcs(struct tegra_output *output,
+				      struct drm_device *drm);
 
 int tegra_output_connector_get_modes(struct drm_connector *connector);
 enum drm_connector_status
@@ -188,11 +203,8 @@ int tegra_drm_fb_init(struct drm_device *drm);
 void tegra_drm_fb_exit(struct drm_device *drm);
 void tegra_drm_fb_suspend(struct drm_device *drm);
 void tegra_drm_fb_resume(struct drm_device *drm);
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-void tegra_fbdev_restore_mode(struct tegra_fbdev *fbdev);
-void tegra_fb_output_poll_changed(struct drm_device *drm);
-#endif
 
+extern struct platform_driver tegra_display_hub_driver;
 extern struct platform_driver tegra_dc_driver;
 extern struct platform_driver tegra_hdmi_driver;
 extern struct platform_driver tegra_dsi_driver;
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index 046649ec9441..87c5d89bc9ba 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -65,8 +65,6 @@ struct tegra_dsi {
 	struct clk *clk;
 
 	struct drm_info_list *debugfs_files;
-	struct drm_minor *minor;
-	struct dentry *debugfs;
 
 	unsigned long flags;
 	enum mipi_dsi_pixel_format format;
@@ -122,12 +120,89 @@ static inline void tegra_dsi_writel(struct tegra_dsi *dsi, u32 value,
 	writel(value, dsi->regs + (offset << 2));
 }
 
+#define DEBUGFS_REG32(_name) { .name = #_name, .offset = _name }
+
+static const struct debugfs_reg32 tegra_dsi_regs[] = {
+	DEBUGFS_REG32(DSI_INCR_SYNCPT),
+	DEBUGFS_REG32(DSI_INCR_SYNCPT_CONTROL),
+	DEBUGFS_REG32(DSI_INCR_SYNCPT_ERROR),
+	DEBUGFS_REG32(DSI_CTXSW),
+	DEBUGFS_REG32(DSI_RD_DATA),
+	DEBUGFS_REG32(DSI_WR_DATA),
+	DEBUGFS_REG32(DSI_POWER_CONTROL),
+	DEBUGFS_REG32(DSI_INT_ENABLE),
+	DEBUGFS_REG32(DSI_INT_STATUS),
+	DEBUGFS_REG32(DSI_INT_MASK),
+	DEBUGFS_REG32(DSI_HOST_CONTROL),
+	DEBUGFS_REG32(DSI_CONTROL),
+	DEBUGFS_REG32(DSI_SOL_DELAY),
+	DEBUGFS_REG32(DSI_MAX_THRESHOLD),
+	DEBUGFS_REG32(DSI_TRIGGER),
+	DEBUGFS_REG32(DSI_TX_CRC),
+	DEBUGFS_REG32(DSI_STATUS),
+	DEBUGFS_REG32(DSI_INIT_SEQ_CONTROL),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_0),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_1),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_2),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_3),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_4),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_5),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_6),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_7),
+	DEBUGFS_REG32(DSI_PKT_SEQ_0_LO),
+	DEBUGFS_REG32(DSI_PKT_SEQ_0_HI),
+	DEBUGFS_REG32(DSI_PKT_SEQ_1_LO),
+	DEBUGFS_REG32(DSI_PKT_SEQ_1_HI),
+	DEBUGFS_REG32(DSI_PKT_SEQ_2_LO),
+	DEBUGFS_REG32(DSI_PKT_SEQ_2_HI),
+	DEBUGFS_REG32(DSI_PKT_SEQ_3_LO),
+	DEBUGFS_REG32(DSI_PKT_SEQ_3_HI),
+	DEBUGFS_REG32(DSI_PKT_SEQ_4_LO),
+	DEBUGFS_REG32(DSI_PKT_SEQ_4_HI),
+	DEBUGFS_REG32(DSI_PKT_SEQ_5_LO),
+	DEBUGFS_REG32(DSI_PKT_SEQ_5_HI),
+	DEBUGFS_REG32(DSI_DCS_CMDS),
+	DEBUGFS_REG32(DSI_PKT_LEN_0_1),
+	DEBUGFS_REG32(DSI_PKT_LEN_2_3),
+	DEBUGFS_REG32(DSI_PKT_LEN_4_5),
+	DEBUGFS_REG32(DSI_PKT_LEN_6_7),
+	DEBUGFS_REG32(DSI_PHY_TIMING_0),
+	DEBUGFS_REG32(DSI_PHY_TIMING_1),
+	DEBUGFS_REG32(DSI_PHY_TIMING_2),
+	DEBUGFS_REG32(DSI_BTA_TIMING),
+	DEBUGFS_REG32(DSI_TIMEOUT_0),
+	DEBUGFS_REG32(DSI_TIMEOUT_1),
+	DEBUGFS_REG32(DSI_TO_TALLY),
+	DEBUGFS_REG32(DSI_PAD_CONTROL_0),
+	DEBUGFS_REG32(DSI_PAD_CONTROL_CD),
+	DEBUGFS_REG32(DSI_PAD_CD_STATUS),
+	DEBUGFS_REG32(DSI_VIDEO_MODE_CONTROL),
+	DEBUGFS_REG32(DSI_PAD_CONTROL_1),
+	DEBUGFS_REG32(DSI_PAD_CONTROL_2),
+	DEBUGFS_REG32(DSI_PAD_CONTROL_3),
+	DEBUGFS_REG32(DSI_PAD_CONTROL_4),
+	DEBUGFS_REG32(DSI_GANGED_MODE_CONTROL),
+	DEBUGFS_REG32(DSI_GANGED_MODE_START),
+	DEBUGFS_REG32(DSI_GANGED_MODE_SIZE),
+	DEBUGFS_REG32(DSI_RAW_DATA_BYTE_COUNT),
+	DEBUGFS_REG32(DSI_ULTRA_LOW_POWER_CONTROL),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_8),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_9),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_10),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_11),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_12),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_13),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_14),
+	DEBUGFS_REG32(DSI_INIT_SEQ_DATA_15),
+};
+
 static int tegra_dsi_show_regs(struct seq_file *s, void *data)
 {
 	struct drm_info_node *node = s->private;
 	struct tegra_dsi *dsi = node->info_ent->data;
 	struct drm_crtc *crtc = dsi->output.encoder.crtc;
 	struct drm_device *drm = node->minor->dev;
+	unsigned int i;
 	int err = 0;
 
 	drm_modeset_lock_all(drm);
@@ -137,93 +212,12 @@ static int tegra_dsi_show_regs(struct seq_file *s, void *data)
 		goto unlock;
 	}
 
-#define DUMP_REG(name)						\
-	seq_printf(s, "%-32s %#05x %08x\n", #name, name,	\
-		   tegra_dsi_readl(dsi, name))
-
-	DUMP_REG(DSI_INCR_SYNCPT);
-	DUMP_REG(DSI_INCR_SYNCPT_CONTROL);
-	DUMP_REG(DSI_INCR_SYNCPT_ERROR);
-	DUMP_REG(DSI_CTXSW);
-	DUMP_REG(DSI_RD_DATA);
-	DUMP_REG(DSI_WR_DATA);
-	DUMP_REG(DSI_POWER_CONTROL);
-	DUMP_REG(DSI_INT_ENABLE);
-	DUMP_REG(DSI_INT_STATUS);
-	DUMP_REG(DSI_INT_MASK);
-	DUMP_REG(DSI_HOST_CONTROL);
-	DUMP_REG(DSI_CONTROL);
-	DUMP_REG(DSI_SOL_DELAY);
-	DUMP_REG(DSI_MAX_THRESHOLD);
-	DUMP_REG(DSI_TRIGGER);
-	DUMP_REG(DSI_TX_CRC);
-	DUMP_REG(DSI_STATUS);
-
-	DUMP_REG(DSI_INIT_SEQ_CONTROL);
-	DUMP_REG(DSI_INIT_SEQ_DATA_0);
-	DUMP_REG(DSI_INIT_SEQ_DATA_1);
-	DUMP_REG(DSI_INIT_SEQ_DATA_2);
-	DUMP_REG(DSI_INIT_SEQ_DATA_3);
-	DUMP_REG(DSI_INIT_SEQ_DATA_4);
-	DUMP_REG(DSI_INIT_SEQ_DATA_5);
-	DUMP_REG(DSI_INIT_SEQ_DATA_6);
-	DUMP_REG(DSI_INIT_SEQ_DATA_7);
-
-	DUMP_REG(DSI_PKT_SEQ_0_LO);
-	DUMP_REG(DSI_PKT_SEQ_0_HI);
-	DUMP_REG(DSI_PKT_SEQ_1_LO);
-	DUMP_REG(DSI_PKT_SEQ_1_HI);
-	DUMP_REG(DSI_PKT_SEQ_2_LO);
-	DUMP_REG(DSI_PKT_SEQ_2_HI);
-	DUMP_REG(DSI_PKT_SEQ_3_LO);
-	DUMP_REG(DSI_PKT_SEQ_3_HI);
-	DUMP_REG(DSI_PKT_SEQ_4_LO);
-	DUMP_REG(DSI_PKT_SEQ_4_HI);
-	DUMP_REG(DSI_PKT_SEQ_5_LO);
-	DUMP_REG(DSI_PKT_SEQ_5_HI);
-
-	DUMP_REG(DSI_DCS_CMDS);
-
-	DUMP_REG(DSI_PKT_LEN_0_1);
-	DUMP_REG(DSI_PKT_LEN_2_3);
-	DUMP_REG(DSI_PKT_LEN_4_5);
-	DUMP_REG(DSI_PKT_LEN_6_7);
-
-	DUMP_REG(DSI_PHY_TIMING_0);
-	DUMP_REG(DSI_PHY_TIMING_1);
-	DUMP_REG(DSI_PHY_TIMING_2);
-	DUMP_REG(DSI_BTA_TIMING);
-
-	DUMP_REG(DSI_TIMEOUT_0);
-	DUMP_REG(DSI_TIMEOUT_1);
-	DUMP_REG(DSI_TO_TALLY);
-
-	DUMP_REG(DSI_PAD_CONTROL_0);
-	DUMP_REG(DSI_PAD_CONTROL_CD);
-	DUMP_REG(DSI_PAD_CD_STATUS);
-	DUMP_REG(DSI_VIDEO_MODE_CONTROL);
-	DUMP_REG(DSI_PAD_CONTROL_1);
-	DUMP_REG(DSI_PAD_CONTROL_2);
-	DUMP_REG(DSI_PAD_CONTROL_3);
-	DUMP_REG(DSI_PAD_CONTROL_4);
-
-	DUMP_REG(DSI_GANGED_MODE_CONTROL);
-	DUMP_REG(DSI_GANGED_MODE_START);
-	DUMP_REG(DSI_GANGED_MODE_SIZE);
-
-	DUMP_REG(DSI_RAW_DATA_BYTE_COUNT);
-	DUMP_REG(DSI_ULTRA_LOW_POWER_CONTROL);
-
-	DUMP_REG(DSI_INIT_SEQ_DATA_8);
-	DUMP_REG(DSI_INIT_SEQ_DATA_9);
-	DUMP_REG(DSI_INIT_SEQ_DATA_10);
-	DUMP_REG(DSI_INIT_SEQ_DATA_11);
-	DUMP_REG(DSI_INIT_SEQ_DATA_12);
-	DUMP_REG(DSI_INIT_SEQ_DATA_13);
-	DUMP_REG(DSI_INIT_SEQ_DATA_14);
-	DUMP_REG(DSI_INIT_SEQ_DATA_15);
-
-#undef DUMP_REG
+	for (i = 0; i < ARRAY_SIZE(tegra_dsi_regs); i++) {
+		unsigned int offset = tegra_dsi_regs[i].offset;
+
+		seq_printf(s, "%-32s %#05x %08x\n", tegra_dsi_regs[i].name,
+			   offset, tegra_dsi_readl(dsi, offset));
+	}
 
 unlock:
 	drm_modeset_unlock_all(drm);
@@ -234,58 +228,46 @@ static struct drm_info_list debugfs_files[] = {
 	{ "regs", tegra_dsi_show_regs, 0, NULL },
 };
 
-static int tegra_dsi_debugfs_init(struct tegra_dsi *dsi,
-				  struct drm_minor *minor)
+static int tegra_dsi_late_register(struct drm_connector *connector)
 {
-	const char *name = dev_name(dsi->dev);
-	unsigned int i;
+	struct tegra_output *output = connector_to_output(connector);
+	unsigned int i, count = ARRAY_SIZE(debugfs_files);
+	struct drm_minor *minor = connector->dev->primary;
+	struct dentry *root = connector->debugfs_entry;
+	struct tegra_dsi *dsi = to_dsi(output);
 	int err;
 
-	dsi->debugfs = debugfs_create_dir(name, minor->debugfs_root);
-	if (!dsi->debugfs)
-		return -ENOMEM;
-
 	dsi->debugfs_files = kmemdup(debugfs_files, sizeof(debugfs_files),
 				     GFP_KERNEL);
-	if (!dsi->debugfs_files) {
-		err = -ENOMEM;
-		goto remove;
-	}
+	if (!dsi->debugfs_files)
+		return -ENOMEM;
 
-	for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
+	for (i = 0; i < count; i++)
 		dsi->debugfs_files[i].data = dsi;
 
-	err = drm_debugfs_create_files(dsi->debugfs_files,
-				       ARRAY_SIZE(debugfs_files),
-				       dsi->debugfs, minor);
+	err = drm_debugfs_create_files(dsi->debugfs_files, count, root, minor);
 	if (err < 0)
 		goto free;
 
-	dsi->minor = minor;
-
 	return 0;
 
 free:
 	kfree(dsi->debugfs_files);
 	dsi->debugfs_files = NULL;
-remove:
-	debugfs_remove(dsi->debugfs);
-	dsi->debugfs = NULL;
 
 	return err;
 }
 
-static void tegra_dsi_debugfs_exit(struct tegra_dsi *dsi)
+static void tegra_dsi_early_unregister(struct drm_connector *connector)
 {
-	drm_debugfs_remove_files(dsi->debugfs_files, ARRAY_SIZE(debugfs_files),
-				 dsi->minor);
-	dsi->minor = NULL;
+	struct tegra_output *output = connector_to_output(connector);
+	unsigned int count = ARRAY_SIZE(debugfs_files);
+	struct tegra_dsi *dsi = to_dsi(output);
 
+	drm_debugfs_remove_files(dsi->debugfs_files, count,
+				 connector->dev->primary);
 	kfree(dsi->debugfs_files);
 	dsi->debugfs_files = NULL;
-
-	debugfs_remove(dsi->debugfs);
-	dsi->debugfs = NULL;
 }
 
 #define PKT_ID0(id)	((((id) & 0x3f) <<  3) | (1 <<  9))
@@ -827,6 +809,8 @@ static const struct drm_connector_funcs tegra_dsi_connector_funcs = {
 	.destroy = tegra_output_connector_destroy,
 	.atomic_duplicate_state = tegra_dsi_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.late_register = tegra_dsi_late_register,
+	.early_unregister = tegra_dsi_early_unregister,
 };
 
 static enum drm_mode_status
@@ -1080,12 +1064,6 @@ static int tegra_dsi_init(struct host1x_client *client)
 		dsi->output.encoder.possible_crtcs = 0x3;
 	}
 
-	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
-		err = tegra_dsi_debugfs_init(dsi, drm->primary);
-		if (err < 0)
-			dev_err(dsi->dev, "debugfs setup failed: %d\n", err);
-	}
-
 	return 0;
 }
 
@@ -1095,11 +1073,6 @@ static int tegra_dsi_exit(struct host1x_client *client)
 
 	tegra_output_exit(&dsi->output);
 
-	if (IS_ENABLED(CONFIG_DEBUG_FS))
-		tegra_dsi_debugfs_exit(dsi);
-
-	regulator_disable(dsi->vdd);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index 80540c1c66dc..001cb77e2f59 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -54,17 +54,40 @@ int tegra_fb_get_tiling(struct drm_framebuffer *framebuffer,
 	struct tegra_fb *fb = to_tegra_fb(framebuffer);
 	uint64_t modifier = fb->base.modifier;
 
-	switch (fourcc_mod_tegra_mod(modifier)) {
-	case NV_FORMAT_MOD_TEGRA_TILED:
+	switch (modifier) {
+	case DRM_FORMAT_MOD_NVIDIA_TEGRA_TILED:
 		tiling->mode = TEGRA_BO_TILING_MODE_TILED;
 		tiling->value = 0;
 		break;
 
-	case NV_FORMAT_MOD_TEGRA_16BX2_BLOCK(0):
+	case DRM_FORMAT_MOD_NVIDIA_16BX2_BLOCK(0):
 		tiling->mode = TEGRA_BO_TILING_MODE_BLOCK;
-		tiling->value = fourcc_mod_tegra_param(modifier);
-		if (tiling->value > 5)
-			return -EINVAL;
+		tiling->value = 0;
+		break;
+
+	case DRM_FORMAT_MOD_NVIDIA_16BX2_BLOCK(1):
+		tiling->mode = TEGRA_BO_TILING_MODE_BLOCK;
+		tiling->value = 1;
+		break;
+
+	case DRM_FORMAT_MOD_NVIDIA_16BX2_BLOCK(2):
+		tiling->mode = TEGRA_BO_TILING_MODE_BLOCK;
+		tiling->value = 2;
+		break;
+
+	case DRM_FORMAT_MOD_NVIDIA_16BX2_BLOCK(3):
+		tiling->mode = TEGRA_BO_TILING_MODE_BLOCK;
+		tiling->value = 3;
+		break;
+
+	case DRM_FORMAT_MOD_NVIDIA_16BX2_BLOCK(4):
+		tiling->mode = TEGRA_BO_TILING_MODE_BLOCK;
+		tiling->value = 4;
+		break;
+
+	case DRM_FORMAT_MOD_NVIDIA_16BX2_BLOCK(5):
+		tiling->mode = TEGRA_BO_TILING_MODE_BLOCK;
+		tiling->value = 5;
 		break;
 
 	default:
@@ -230,6 +253,7 @@ static int tegra_fbdev_probe(struct drm_fb_helper *helper,
 	cmd.height = sizes->surface_height;
 	cmd.pitches[0] = round_up(sizes->surface_width * bytes_per_pixel,
 				  tegra->pitch_align);
+
 	cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
 						     sizes->surface_depth);
 
@@ -361,20 +385,6 @@ static void tegra_fbdev_exit(struct tegra_fbdev *fbdev)
 	drm_fb_helper_fini(&fbdev->base);
 	tegra_fbdev_free(fbdev);
 }
-
-void tegra_fbdev_restore_mode(struct tegra_fbdev *fbdev)
-{
-	if (fbdev)
-		drm_fb_helper_restore_fbdev_mode_unlocked(&fbdev->base);
-}
-
-void tegra_fb_output_poll_changed(struct drm_device *drm)
-{
-	struct tegra_drm *tegra = drm->dev_private;
-
-	if (tegra->fbdev)
-		drm_fb_helper_hotplug_event(&tegra->fbdev->base);
-}
 #endif
 
 int tegra_drm_fb_prepare(struct drm_device *drm)
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index ab1e53d434e8..49b9bf28f872 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -114,7 +114,7 @@ static const struct host1x_bo_ops tegra_bo_ops = {
 static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo)
 {
 	int prot = IOMMU_READ | IOMMU_WRITE;
-	ssize_t err;
+	int err;
 
 	if (bo->mm)
 		return -EBUSY;
@@ -128,22 +128,21 @@ static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo)
 	err = drm_mm_insert_node_generic(&tegra->mm,
 					 bo->mm, bo->gem.size, PAGE_SIZE, 0, 0);
 	if (err < 0) {
-		dev_err(tegra->drm->dev, "out of I/O virtual memory: %zd\n",
+		dev_err(tegra->drm->dev, "out of I/O virtual memory: %d\n",
 			err);
 		goto unlock;
 	}
 
 	bo->paddr = bo->mm->start;
 
-	err = iommu_map_sg(tegra->domain, bo->paddr, bo->sgt->sgl,
-			   bo->sgt->nents, prot);
-	if (err < 0) {
-		dev_err(tegra->drm->dev, "failed to map buffer: %zd\n", err);
+	bo->size = iommu_map_sg(tegra->domain, bo->paddr, bo->sgt->sgl,
+				bo->sgt->nents, prot);
+	if (!bo->size) {
+		dev_err(tegra->drm->dev, "failed to map buffer\n");
+		err = -ENOMEM;
 		goto remove;
 	}
 
-	bo->size = err;
-
 	mutex_unlock(&tegra->mm_lock);
 
 	return 0;
diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index 6434b3d3d1ba..784739a9f497 100644
--- a/drivers/gpu/drm/tegra/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -79,8 +79,6 @@ struct tegra_hdmi {
 	bool dvi;
 
 	struct drm_info_list *debugfs_files;
-	struct drm_minor *minor;
-	struct dentry *debugfs;
 };
 
 static inline struct tegra_hdmi *
@@ -910,6 +908,249 @@ tegra_hdmi_connector_detect(struct drm_connector *connector, bool force)
 	return status;
 }
 
+#define DEBUGFS_REG32(_name) { .name = #_name, .offset = _name }
+
+static const struct debugfs_reg32 tegra_hdmi_regs[] = {
+	DEBUGFS_REG32(HDMI_CTXSW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_STATE0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_STATE1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_STATE2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_AN_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_AN_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CN_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CN_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_AKSV_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_AKSV_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_BKSV_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_BKSV_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CKSV_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CKSV_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_DKSV_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_DKSV_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CMODE),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_MPRIME_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_MPRIME_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_SPRIME_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_SPRIME_LSB2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_SPRIME_LSB1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_RI),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CS_MSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_RG_HDCP_CS_LSB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_EMU0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_EMU_RDATA0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_EMU1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_EMU2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_STATUS),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_HEADER),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_SUBPACK0_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_SUBPACK0_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_STATUS),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_HEADER),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK0_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK0_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK1_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK1_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_STATUS),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_HEADER),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK0_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK0_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK1_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK1_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK2_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK2_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK3_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK3_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0320_SUBPACK_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0320_SUBPACK_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0441_SUBPACK_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0441_SUBPACK_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0882_SUBPACK_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0882_SUBPACK_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_1764_SUBPACK_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_1764_SUBPACK_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0480_SUBPACK_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0480_SUBPACK_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0960_SUBPACK_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_0960_SUBPACK_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_1920_SUBPACK_LOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_ACR_1920_SUBPACK_HIGH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_VSYNC_KEEPOUT),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_VSYNC_WINDOW),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GCP_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GCP_STATUS),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_GCP_SUBPACK),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_CHANNEL_STATUS1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_CHANNEL_STATUS2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_EMU0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_EMU1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_EMU1_RDATA),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_SPARE),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_SPDIF_CHN_STATUS1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_SPDIF_CHN_STATUS2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDMI_HDCPRIF_ROM_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_CAP),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_PWR),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_TEST),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_PLL0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_PLL1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_PLL2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_CSTM),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_LVDS),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_CRCA),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_CRCB),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_BLANK),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_CTL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(0)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(1)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(2)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(3)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(4)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(5)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(6)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(7)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(8)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(9)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(10)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(11)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(12)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(13)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(14)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_SEQ_INST(15)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_VCRCA0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_VCRCA1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_CCRCA0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_CCRCA1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_EDATAA0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_EDATAA1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_COUNTA0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_COUNTA1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_DEBUGA0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_DEBUGA1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_TRIG),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_MSCHECK),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_DEBUG0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_DEBUG1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_DEBUG2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_FS(0)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_FS(1)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_FS(2)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_FS(3)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_FS(4)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_FS(5)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_FS(6)),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_PULSE_WIDTH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_THRESHOLD),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_CNTRL0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_AUDIO_N),
+	DEBUGFS_REG32(HDMI_NV_PDISP_HDCPRIF_ROM_TIMING),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_REFCLK),
+	DEBUGFS_REG32(HDMI_NV_PDISP_CRC_CONTROL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_INPUT_CONTROL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SCRATCH),
+	DEBUGFS_REG32(HDMI_NV_PDISP_PE_CURRENT),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_CTRL),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_DEBUG0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_DEBUG1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_DEBUG2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_HDCP_KEY_0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_HDCP_KEY_1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_HDCP_KEY_2),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_HDCP_KEY_3),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_HDCP_KEY_TRIG),
+	DEBUGFS_REG32(HDMI_NV_PDISP_KEY_SKEY_INDEX),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_AUDIO_CNTRL0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_AUDIO_SPARE0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH0),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH1),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_AUDIO_HDA_ELD_BUFWR),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_AUDIO_HDA_PRESENSE),
+	DEBUGFS_REG32(HDMI_NV_PDISP_INT_STATUS),
+	DEBUGFS_REG32(HDMI_NV_PDISP_INT_MASK),
+	DEBUGFS_REG32(HDMI_NV_PDISP_INT_ENABLE),
+	DEBUGFS_REG32(HDMI_NV_PDISP_SOR_IO_PEAK_CURRENT),
+};
+
+static int tegra_hdmi_show_regs(struct seq_file *s, void *data)
+{
+	struct drm_info_node *node = s->private;
+	struct tegra_hdmi *hdmi = node->info_ent->data;
+	struct drm_crtc *crtc = hdmi->output.encoder.crtc;
+	struct drm_device *drm = node->minor->dev;
+	unsigned int i;
+	int err = 0;
+
+	drm_modeset_lock_all(drm);
+
+	if (!crtc || !crtc->state->active) {
+		err = -EBUSY;
+		goto unlock;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(tegra_hdmi_regs); i++) {
+		unsigned int offset = tegra_hdmi_regs[i].offset;
+
+		seq_printf(s, "%-56s %#05x %08x\n", tegra_hdmi_regs[i].name,
+			   offset, tegra_hdmi_readl(hdmi, offset));
+	}
+
+unlock:
+	drm_modeset_unlock_all(drm);
+	return err;
+}
+
+static struct drm_info_list debugfs_files[] = {
+	{ "regs", tegra_hdmi_show_regs, 0, NULL },
+};
+
+static int tegra_hdmi_late_register(struct drm_connector *connector)
+{
+	struct tegra_output *output = connector_to_output(connector);
+	unsigned int i, count = ARRAY_SIZE(debugfs_files);
+	struct drm_minor *minor = connector->dev->primary;
+	struct dentry *root = connector->debugfs_entry;
+	struct tegra_hdmi *hdmi = to_hdmi(output);
+	int err;
+
+	hdmi->debugfs_files = kmemdup(debugfs_files, sizeof(debugfs_files),
+				      GFP_KERNEL);
+	if (!hdmi->debugfs_files)
+		return -ENOMEM;
+
+	for (i = 0; i < count; i++)
+		hdmi->debugfs_files[i].data = hdmi;
+
+	err = drm_debugfs_create_files(hdmi->debugfs_files, count, root, minor);
+	if (err < 0)
+		goto free;
+
+	return 0;
+
+free:
+	kfree(hdmi->debugfs_files);
+	hdmi->debugfs_files = NULL;
+
+	return err;
+}
+
+static void tegra_hdmi_early_unregister(struct drm_connector *connector)
+{
+	struct tegra_output *output = connector_to_output(connector);
+	struct drm_minor *minor = connector->dev->primary;
+	unsigned int count = ARRAY_SIZE(debugfs_files);
+	struct tegra_hdmi *hdmi = to_hdmi(output);
+
+	drm_debugfs_remove_files(hdmi->debugfs_files, count, minor);
+	kfree(hdmi->debugfs_files);
+	hdmi->debugfs_files = NULL;
+}
+
 static const struct drm_connector_funcs tegra_hdmi_connector_funcs = {
 	.reset = drm_atomic_helper_connector_reset,
 	.detect = tegra_hdmi_connector_detect,
@@ -917,6 +1158,8 @@ static const struct drm_connector_funcs tegra_hdmi_connector_funcs = {
 	.destroy = tegra_output_connector_destroy,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.late_register = tegra_hdmi_late_register,
+	.early_unregister = tegra_hdmi_early_unregister,
 };
 
 static enum drm_mode_status
@@ -1225,254 +1468,6 @@ static const struct drm_encoder_helper_funcs tegra_hdmi_encoder_helper_funcs = {
 	.atomic_check = tegra_hdmi_encoder_atomic_check,
 };
 
-static int tegra_hdmi_show_regs(struct seq_file *s, void *data)
-{
-	struct drm_info_node *node = s->private;
-	struct tegra_hdmi *hdmi = node->info_ent->data;
-	struct drm_crtc *crtc = hdmi->output.encoder.crtc;
-	struct drm_device *drm = node->minor->dev;
-	int err = 0;
-
-	drm_modeset_lock_all(drm);
-
-	if (!crtc || !crtc->state->active) {
-		err = -EBUSY;
-		goto unlock;
-	}
-
-#define DUMP_REG(name)						\
-	seq_printf(s, "%-56s %#05x %08x\n", #name, name,	\
-		   tegra_hdmi_readl(hdmi, name))
-
-	DUMP_REG(HDMI_CTXSW);
-	DUMP_REG(HDMI_NV_PDISP_SOR_STATE0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_STATE1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_STATE2);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_AN_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_AN_LSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CN_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CN_LSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_AKSV_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_AKSV_LSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_BKSV_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_BKSV_LSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CKSV_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CKSV_LSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_DKSV_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_DKSV_LSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CMODE);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_MPRIME_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_MPRIME_LSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_SPRIME_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_SPRIME_LSB2);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_SPRIME_LSB1);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_RI);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CS_MSB);
-	DUMP_REG(HDMI_NV_PDISP_RG_HDCP_CS_LSB);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_EMU0);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_EMU_RDATA0);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_EMU1);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_EMU2);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_STATUS);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_HEADER);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_SUBPACK0_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_SUBPACK0_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_STATUS);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_HEADER);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK0_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK0_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK1_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_SUBPACK1_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_STATUS);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_HEADER);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK0_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK0_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK1_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK1_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK2_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK2_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK3_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GENERIC_SUBPACK3_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0320_SUBPACK_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0320_SUBPACK_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0441_SUBPACK_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0441_SUBPACK_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0882_SUBPACK_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0882_SUBPACK_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_1764_SUBPACK_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_1764_SUBPACK_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0480_SUBPACK_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0480_SUBPACK_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0960_SUBPACK_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_0960_SUBPACK_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_1920_SUBPACK_LOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_ACR_1920_SUBPACK_HIGH);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_VSYNC_KEEPOUT);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_VSYNC_WINDOW);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GCP_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GCP_STATUS);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_GCP_SUBPACK);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_CHANNEL_STATUS1);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_CHANNEL_STATUS2);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_EMU0);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_EMU1);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_EMU1_RDATA);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_SPARE);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_SPDIF_CHN_STATUS1);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_SPDIF_CHN_STATUS2);
-	DUMP_REG(HDMI_NV_PDISP_HDMI_HDCPRIF_ROM_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_SOR_CAP);
-	DUMP_REG(HDMI_NV_PDISP_SOR_PWR);
-	DUMP_REG(HDMI_NV_PDISP_SOR_TEST);
-	DUMP_REG(HDMI_NV_PDISP_SOR_PLL0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_PLL1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_PLL2);
-	DUMP_REG(HDMI_NV_PDISP_SOR_CSTM);
-	DUMP_REG(HDMI_NV_PDISP_SOR_LVDS);
-	DUMP_REG(HDMI_NV_PDISP_SOR_CRCA);
-	DUMP_REG(HDMI_NV_PDISP_SOR_CRCB);
-	DUMP_REG(HDMI_NV_PDISP_SOR_BLANK);
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_CTL);
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(0));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(1));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(2));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(3));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(4));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(5));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(6));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(7));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(8));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(9));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(10));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(11));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(12));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(13));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(14));
-	DUMP_REG(HDMI_NV_PDISP_SOR_SEQ_INST(15));
-	DUMP_REG(HDMI_NV_PDISP_SOR_VCRCA0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_VCRCA1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_CCRCA0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_CCRCA1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_EDATAA0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_EDATAA1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_COUNTA0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_COUNTA1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_DEBUGA0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_DEBUGA1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_TRIG);
-	DUMP_REG(HDMI_NV_PDISP_SOR_MSCHECK);
-	DUMP_REG(HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT);
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_DEBUG0);
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_DEBUG1);
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_DEBUG2);
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_FS(0));
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_FS(1));
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_FS(2));
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_FS(3));
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_FS(4));
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_FS(5));
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_FS(6));
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_PULSE_WIDTH);
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_THRESHOLD);
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_CNTRL0);
-	DUMP_REG(HDMI_NV_PDISP_AUDIO_N);
-	DUMP_REG(HDMI_NV_PDISP_HDCPRIF_ROM_TIMING);
-	DUMP_REG(HDMI_NV_PDISP_SOR_REFCLK);
-	DUMP_REG(HDMI_NV_PDISP_CRC_CONTROL);
-	DUMP_REG(HDMI_NV_PDISP_INPUT_CONTROL);
-	DUMP_REG(HDMI_NV_PDISP_SCRATCH);
-	DUMP_REG(HDMI_NV_PDISP_PE_CURRENT);
-	DUMP_REG(HDMI_NV_PDISP_KEY_CTRL);
-	DUMP_REG(HDMI_NV_PDISP_KEY_DEBUG0);
-	DUMP_REG(HDMI_NV_PDISP_KEY_DEBUG1);
-	DUMP_REG(HDMI_NV_PDISP_KEY_DEBUG2);
-	DUMP_REG(HDMI_NV_PDISP_KEY_HDCP_KEY_0);
-	DUMP_REG(HDMI_NV_PDISP_KEY_HDCP_KEY_1);
-	DUMP_REG(HDMI_NV_PDISP_KEY_HDCP_KEY_2);
-	DUMP_REG(HDMI_NV_PDISP_KEY_HDCP_KEY_3);
-	DUMP_REG(HDMI_NV_PDISP_KEY_HDCP_KEY_TRIG);
-	DUMP_REG(HDMI_NV_PDISP_KEY_SKEY_INDEX);
-	DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_CNTRL0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_SPARE0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH0);
-	DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH1);
-	DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_ELD_BUFWR);
-	DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_PRESENSE);
-	DUMP_REG(HDMI_NV_PDISP_INT_STATUS);
-	DUMP_REG(HDMI_NV_PDISP_INT_MASK);
-	DUMP_REG(HDMI_NV_PDISP_INT_ENABLE);
-	DUMP_REG(HDMI_NV_PDISP_SOR_IO_PEAK_CURRENT);
-
-#undef DUMP_REG
-
-unlock:
-	drm_modeset_unlock_all(drm);
-	return err;
-}
-
-static struct drm_info_list debugfs_files[] = {
-	{ "regs", tegra_hdmi_show_regs, 0, NULL },
-};
-
-static int tegra_hdmi_debugfs_init(struct tegra_hdmi *hdmi,
-				   struct drm_minor *minor)
-{
-	unsigned int i;
-	int err;
-
-	hdmi->debugfs = debugfs_create_dir("hdmi", minor->debugfs_root);
-	if (!hdmi->debugfs)
-		return -ENOMEM;
-
-	hdmi->debugfs_files = kmemdup(debugfs_files, sizeof(debugfs_files),
-				      GFP_KERNEL);
-	if (!hdmi->debugfs_files) {
-		err = -ENOMEM;
-		goto remove;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
-		hdmi->debugfs_files[i].data = hdmi;
-
-	err = drm_debugfs_create_files(hdmi->debugfs_files,
-				       ARRAY_SIZE(debugfs_files),
-				       hdmi->debugfs, minor);
-	if (err < 0)
-		goto free;
-
-	hdmi->minor = minor;
-
-	return 0;
-
-free:
-	kfree(hdmi->debugfs_files);
-	hdmi->debugfs_files = NULL;
-remove:
-	debugfs_remove(hdmi->debugfs);
-	hdmi->debugfs = NULL;
-
-	return err;
-}
-
-static void tegra_hdmi_debugfs_exit(struct tegra_hdmi *hdmi)
-{
-	drm_debugfs_remove_files(hdmi->debugfs_files, ARRAY_SIZE(debugfs_files),
-				 hdmi->minor);
-	hdmi->minor = NULL;
-
-	kfree(hdmi->debugfs_files);
-	hdmi->debugfs_files = NULL;
-
-	debugfs_remove(hdmi->debugfs);
-	hdmi->debugfs = NULL;
-}
-
 static int tegra_hdmi_init(struct host1x_client *client)
 {
 	struct drm_device *drm = dev_get_drvdata(client->parent);
@@ -1505,12 +1500,6 @@ static int tegra_hdmi_init(struct host1x_client *client)
 
 	hdmi->output.encoder.possible_crtcs = 0x3;
 
-	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
-		err = tegra_hdmi_debugfs_init(hdmi, drm->primary);
-		if (err < 0)
-			dev_err(client->dev, "debugfs setup failed: %d\n", err);
-	}
-
 	err = regulator_enable(hdmi->hdmi);
 	if (err < 0) {
 		dev_err(client->dev, "failed to enable HDMI regulator: %d\n",
@@ -1543,9 +1532,6 @@ static int tegra_hdmi_exit(struct host1x_client *client)
 	regulator_disable(hdmi->pll);
 	regulator_disable(hdmi->hdmi);
 
-	if (IS_ENABLED(CONFIG_DEBUG_FS))
-		tegra_hdmi_debugfs_exit(hdmi);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/tegra/hub.c b/drivers/gpu/drm/tegra/hub.c
new file mode 100644
index 000000000000..e10a47d57313
--- /dev/null
+++ b/drivers/gpu/drm/tegra/hub.c
@@ -0,0 +1,806 @@
+/*
+ * Copyright (C) 2017 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/host1x.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+
+#include "drm.h"
+#include "dc.h"
+#include "plane.h"
+
+static const u32 tegra_shared_plane_formats[] = {
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGBA5551,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	/* new on Tegra114 */
+	DRM_FORMAT_ABGR4444,
+	DRM_FORMAT_ABGR1555,
+	DRM_FORMAT_BGRA5551,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGBX5551,
+	DRM_FORMAT_XBGR1555,
+	DRM_FORMAT_BGRX5551,
+	DRM_FORMAT_BGR565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	/* planar formats */
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YUV420,
+	DRM_FORMAT_YUV422,
+};
+
+static inline unsigned int tegra_plane_offset(struct tegra_plane *plane,
+					      unsigned int offset)
+{
+	if (offset >= 0x500 && offset <= 0x581) {
+		offset = 0x000 + (offset - 0x500);
+		return plane->offset + offset;
+	}
+
+	if (offset >= 0x700 && offset <= 0x73c) {
+		offset = 0x180 + (offset - 0x700);
+		return plane->offset + offset;
+	}
+
+	if (offset >= 0x800 && offset <= 0x83e) {
+		offset = 0x1c0 + (offset - 0x800);
+		return plane->offset + offset;
+	}
+
+	dev_WARN(plane->dc->dev, "invalid offset: %x\n", offset);
+
+	return plane->offset + offset;
+}
+
+static inline u32 tegra_plane_readl(struct tegra_plane *plane,
+				    unsigned int offset)
+{
+	return tegra_dc_readl(plane->dc, tegra_plane_offset(plane, offset));
+}
+
+static inline void tegra_plane_writel(struct tegra_plane *plane, u32 value,
+				      unsigned int offset)
+{
+	tegra_dc_writel(plane->dc, value, tegra_plane_offset(plane, offset));
+}
+
+static int tegra_windowgroup_enable(struct tegra_windowgroup *wgrp)
+{
+	mutex_lock(&wgrp->lock);
+
+	if (wgrp->usecount == 0) {
+		pm_runtime_get_sync(wgrp->parent);
+		reset_control_deassert(wgrp->rst);
+	}
+
+	wgrp->usecount++;
+	mutex_unlock(&wgrp->lock);
+
+	return 0;
+}
+
+static void tegra_windowgroup_disable(struct tegra_windowgroup *wgrp)
+{
+	int err;
+
+	mutex_lock(&wgrp->lock);
+
+	if (wgrp->usecount == 1) {
+		err = reset_control_assert(wgrp->rst);
+		if (err < 0) {
+			pr_err("failed to assert reset for window group %u\n",
+			       wgrp->index);
+		}
+
+		pm_runtime_put(wgrp->parent);
+	}
+
+	wgrp->usecount--;
+	mutex_unlock(&wgrp->lock);
+}
+
+int tegra_display_hub_prepare(struct tegra_display_hub *hub)
+{
+	unsigned int i;
+
+	/*
+	 * XXX Enabling/disabling windowgroups needs to happen when the owner
+	 * display controller is disabled. There's currently no good point at
+	 * which this could be executed, so unconditionally enable all window
+	 * groups for now.
+	 */
+	for (i = 0; i < hub->soc->num_wgrps; i++) {
+		struct tegra_windowgroup *wgrp = &hub->wgrps[i];
+
+		tegra_windowgroup_enable(wgrp);
+	}
+
+	return 0;
+}
+
+void tegra_display_hub_cleanup(struct tegra_display_hub *hub)
+{
+	unsigned int i;
+
+	/*
+	 * XXX Remove this once window groups can be more fine-grainedly
+	 * enabled and disabled.
+	 */
+	for (i = 0; i < hub->soc->num_wgrps; i++) {
+		struct tegra_windowgroup *wgrp = &hub->wgrps[i];
+
+		tegra_windowgroup_disable(wgrp);
+	}
+}
+
+static void tegra_shared_plane_update(struct tegra_plane *plane)
+{
+	struct tegra_dc *dc = plane->dc;
+	unsigned long timeout;
+	u32 mask, value;
+
+	mask = COMMON_UPDATE | WIN_A_UPDATE << plane->base.index;
+	tegra_dc_writel(dc, mask, DC_CMD_STATE_CONTROL);
+
+	timeout = jiffies + msecs_to_jiffies(1000);
+
+	while (time_before(jiffies, timeout)) {
+		value = tegra_dc_readl(dc, DC_CMD_STATE_CONTROL);
+		if ((value & mask) == 0)
+			break;
+
+		usleep_range(100, 400);
+	}
+}
+
+static void tegra_shared_plane_activate(struct tegra_plane *plane)
+{
+	struct tegra_dc *dc = plane->dc;
+	unsigned long timeout;
+	u32 mask, value;
+
+	mask = COMMON_ACTREQ | WIN_A_ACT_REQ << plane->base.index;
+	tegra_dc_writel(dc, mask, DC_CMD_STATE_CONTROL);
+
+	timeout = jiffies + msecs_to_jiffies(1000);
+
+	while (time_before(jiffies, timeout)) {
+		value = tegra_dc_readl(dc, DC_CMD_STATE_CONTROL);
+		if ((value & mask) == 0)
+			break;
+
+		usleep_range(100, 400);
+	}
+}
+
+static unsigned int
+tegra_shared_plane_get_owner(struct tegra_plane *plane, struct tegra_dc *dc)
+{
+	unsigned int offset =
+		tegra_plane_offset(plane, DC_WIN_CORE_WINDOWGROUP_SET_CONTROL);
+
+	return tegra_dc_readl(dc, offset) & OWNER_MASK;
+}
+
+static bool tegra_dc_owns_shared_plane(struct tegra_dc *dc,
+				       struct tegra_plane *plane)
+{
+	struct device *dev = dc->dev;
+
+	if (tegra_shared_plane_get_owner(plane, dc) == dc->pipe) {
+		if (plane->dc == dc)
+			return true;
+
+		dev_WARN(dev, "head %u owns window %u but is not attached\n",
+			 dc->pipe, plane->index);
+	}
+
+	return false;
+}
+
+static int tegra_shared_plane_set_owner(struct tegra_plane *plane,
+					struct tegra_dc *new)
+{
+	unsigned int offset =
+		tegra_plane_offset(plane, DC_WIN_CORE_WINDOWGROUP_SET_CONTROL);
+	struct tegra_dc *old = plane->dc, *dc = new ? new : old;
+	struct device *dev = new ? new->dev : old->dev;
+	unsigned int owner, index = plane->index;
+	u32 value;
+
+	value = tegra_dc_readl(dc, offset);
+	owner = value & OWNER_MASK;
+
+	if (new && (owner != OWNER_MASK && owner != new->pipe)) {
+		dev_WARN(dev, "window %u owned by head %u\n", index, owner);
+		return -EBUSY;
+	}
+
+	/*
+	 * This seems to happen whenever the head has been disabled with one
+	 * or more windows being active. This is harmless because we'll just
+	 * reassign the window to the new head anyway.
+	 */
+	if (old && owner == OWNER_MASK)
+		dev_dbg(dev, "window %u not owned by head %u but %u\n", index,
+			old->pipe, owner);
+
+	value &= ~OWNER_MASK;
+
+	if (new)
+		value |= OWNER(new->pipe);
+	else
+		value |= OWNER_MASK;
+
+	tegra_dc_writel(dc, value, offset);
+
+	plane->dc = new;
+
+	return 0;
+}
+
+static void tegra_dc_assign_shared_plane(struct tegra_dc *dc,
+					 struct tegra_plane *plane)
+{
+	u32 value;
+	int err;
+
+	if (!tegra_dc_owns_shared_plane(dc, plane)) {
+		err = tegra_shared_plane_set_owner(plane, dc);
+		if (err < 0)
+			return;
+	}
+
+	value = tegra_plane_readl(plane, DC_WIN_CORE_IHUB_LINEBUF_CONFIG);
+	value |= MODE_FOUR_LINES;
+	tegra_plane_writel(plane, value, DC_WIN_CORE_IHUB_LINEBUF_CONFIG);
+
+	value = tegra_plane_readl(plane, DC_WIN_CORE_IHUB_WGRP_FETCH_METER);
+	value = SLOTS(1);
+	tegra_plane_writel(plane, value, DC_WIN_CORE_IHUB_WGRP_FETCH_METER);
+
+	/* disable watermark */
+	value = tegra_plane_readl(plane, DC_WIN_CORE_IHUB_WGRP_LATENCY_CTLA);
+	value &= ~LATENCY_CTL_MODE_ENABLE;
+	tegra_plane_writel(plane, value, DC_WIN_CORE_IHUB_WGRP_LATENCY_CTLA);
+
+	value = tegra_plane_readl(plane, DC_WIN_CORE_IHUB_WGRP_LATENCY_CTLB);
+	value |= WATERMARK_MASK;
+	tegra_plane_writel(plane, value, DC_WIN_CORE_IHUB_WGRP_LATENCY_CTLB);
+
+	/* pipe meter */
+	value = tegra_plane_readl(plane, DC_WIN_CORE_PRECOMP_WGRP_PIPE_METER);
+	value = PIPE_METER_INT(0) | PIPE_METER_FRAC(0);
+	tegra_plane_writel(plane, value, DC_WIN_CORE_PRECOMP_WGRP_PIPE_METER);
+
+	/* mempool entries */
+	value = tegra_plane_readl(plane, DC_WIN_CORE_IHUB_WGRP_POOL_CONFIG);
+	value = MEMPOOL_ENTRIES(0x331);
+	tegra_plane_writel(plane, value, DC_WIN_CORE_IHUB_WGRP_POOL_CONFIG);
+
+	value = tegra_plane_readl(plane, DC_WIN_CORE_IHUB_THREAD_GROUP);
+	value &= ~THREAD_NUM_MASK;
+	value |= THREAD_NUM(plane->base.index);
+	value |= THREAD_GROUP_ENABLE;
+	tegra_plane_writel(plane, value, DC_WIN_CORE_IHUB_THREAD_GROUP);
+
+	tegra_shared_plane_update(plane);
+	tegra_shared_plane_activate(plane);
+}
+
+static void tegra_dc_remove_shared_plane(struct tegra_dc *dc,
+					 struct tegra_plane *plane)
+{
+	tegra_shared_plane_set_owner(plane, NULL);
+}
+
+static int tegra_shared_plane_atomic_check(struct drm_plane *plane,
+					   struct drm_plane_state *state)
+{
+	struct tegra_plane_state *plane_state = to_tegra_plane_state(state);
+	struct tegra_shared_plane *tegra = to_tegra_shared_plane(plane);
+	struct tegra_bo_tiling *tiling = &plane_state->tiling;
+	struct tegra_dc *dc = to_tegra_dc(state->crtc);
+	int err;
+
+	/* no need for further checks if the plane is being disabled */
+	if (!state->crtc || !state->fb)
+		return 0;
+
+	err = tegra_plane_format(state->fb->format->format,
+				 &plane_state->format,
+				 &plane_state->swap);
+	if (err < 0)
+		return err;
+
+	err = tegra_fb_get_tiling(state->fb, tiling);
+	if (err < 0)
+		return err;
+
+	if (tiling->mode == TEGRA_BO_TILING_MODE_BLOCK &&
+	    !dc->soc->supports_block_linear) {
+		DRM_ERROR("hardware doesn't support block linear mode\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Tegra doesn't support different strides for U and V planes so we
+	 * error out if the user tries to display a framebuffer with such a
+	 * configuration.
+	 */
+	if (state->fb->format->num_planes > 2) {
+		if (state->fb->pitches[2] != state->fb->pitches[1]) {
+			DRM_ERROR("unsupported UV-plane configuration\n");
+			return -EINVAL;
+		}
+	}
+
+	/* XXX scaling is not yet supported, add a check here */
+
+	err = tegra_plane_state_add(&tegra->base, state);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static void tegra_shared_plane_atomic_disable(struct drm_plane *plane,
+					      struct drm_plane_state *old_state)
+{
+	struct tegra_dc *dc = to_tegra_dc(old_state->crtc);
+	struct tegra_plane *p = to_tegra_plane(plane);
+	u32 value;
+
+	/* rien ne va plus */
+	if (!old_state || !old_state->crtc)
+		return;
+
+	/*
+	 * XXX Legacy helpers seem to sometimes call ->atomic_disable() even
+	 * on planes that are already disabled. Make sure we fallback to the
+	 * head for this particular state instead of crashing.
+	 */
+	if (WARN_ON(p->dc == NULL))
+		p->dc = dc;
+
+	pm_runtime_get_sync(dc->dev);
+
+	value = tegra_plane_readl(p, DC_WIN_WIN_OPTIONS);
+	value &= ~WIN_ENABLE;
+	tegra_plane_writel(p, value, DC_WIN_WIN_OPTIONS);
+
+	tegra_dc_remove_shared_plane(dc, p);
+
+	pm_runtime_put(dc->dev);
+}
+
+static void tegra_shared_plane_atomic_update(struct drm_plane *plane,
+					     struct drm_plane_state *old_state)
+{
+	struct tegra_plane_state *state = to_tegra_plane_state(plane->state);
+	struct tegra_dc *dc = to_tegra_dc(plane->state->crtc);
+	unsigned int zpos = plane->state->normalized_zpos;
+	struct drm_framebuffer *fb = plane->state->fb;
+	struct tegra_plane *p = to_tegra_plane(plane);
+	struct tegra_bo *bo;
+	dma_addr_t base;
+	u32 value;
+
+	/* rien ne va plus */
+	if (!plane->state->crtc || !plane->state->fb)
+		return;
+
+	if (!plane->state->visible) {
+		tegra_shared_plane_atomic_disable(plane, old_state);
+		return;
+	}
+
+	pm_runtime_get_sync(dc->dev);
+
+	tegra_dc_assign_shared_plane(dc, p);
+
+	tegra_plane_writel(p, VCOUNTER, DC_WIN_CORE_ACT_CONTROL);
+
+	/* blending */
+	value = BLEND_FACTOR_DST_ALPHA_ZERO | BLEND_FACTOR_SRC_ALPHA_K2 |
+		BLEND_FACTOR_DST_COLOR_NEG_K1_TIMES_SRC |
+		BLEND_FACTOR_SRC_COLOR_K1_TIMES_SRC;
+	tegra_plane_writel(p, value, DC_WIN_BLEND_MATCH_SELECT);
+
+	value = BLEND_FACTOR_DST_ALPHA_ZERO | BLEND_FACTOR_SRC_ALPHA_K2 |
+		BLEND_FACTOR_DST_COLOR_NEG_K1_TIMES_SRC |
+		BLEND_FACTOR_SRC_COLOR_K1_TIMES_SRC;
+	tegra_plane_writel(p, value, DC_WIN_BLEND_NOMATCH_SELECT);
+
+	value = K2(255) | K1(255) | WINDOW_LAYER_DEPTH(255 - zpos);
+	tegra_plane_writel(p, value, DC_WIN_BLEND_LAYER_CONTROL);
+
+	/* bypass scaling */
+	value = HORIZONTAL_TAPS_5 | VERTICAL_TAPS_5;
+	tegra_plane_writel(p, value, DC_WIN_WINDOWGROUP_SET_CONTROL_INPUT_SCALER);
+
+	value = INPUT_SCALER_VBYPASS | INPUT_SCALER_HBYPASS;
+	tegra_plane_writel(p, value, DC_WIN_WINDOWGROUP_SET_INPUT_SCALER_USAGE);
+
+	/* disable compression */
+	tegra_plane_writel(p, 0, DC_WINBUF_CDE_CONTROL);
+
+	bo = tegra_fb_get_plane(fb, 0);
+	base = bo->paddr;
+
+	tegra_plane_writel(p, state->format, DC_WIN_COLOR_DEPTH);
+	tegra_plane_writel(p, 0, DC_WIN_PRECOMP_WGRP_PARAMS);
+
+	value = V_POSITION(plane->state->crtc_y) |
+		H_POSITION(plane->state->crtc_x);
+	tegra_plane_writel(p, value, DC_WIN_POSITION);
+
+	value = V_SIZE(plane->state->crtc_h) | H_SIZE(plane->state->crtc_w);
+	tegra_plane_writel(p, value, DC_WIN_SIZE);
+
+	value = WIN_ENABLE | COLOR_EXPAND;
+	tegra_plane_writel(p, value, DC_WIN_WIN_OPTIONS);
+
+	value = V_SIZE(plane->state->crtc_h) | H_SIZE(plane->state->crtc_w);
+	tegra_plane_writel(p, value, DC_WIN_CROPPED_SIZE);
+
+	tegra_plane_writel(p, upper_32_bits(base), DC_WINBUF_START_ADDR_HI);
+	tegra_plane_writel(p, lower_32_bits(base), DC_WINBUF_START_ADDR);
+
+	value = PITCH(fb->pitches[0]);
+	tegra_plane_writel(p, value, DC_WIN_PLANAR_STORAGE);
+
+	value = CLAMP_BEFORE_BLEND | DEGAMMA_SRGB | INPUT_RANGE_FULL;
+	tegra_plane_writel(p, value, DC_WIN_SET_PARAMS);
+
+	value = OFFSET_X(plane->state->src_y >> 16) |
+		OFFSET_Y(plane->state->src_x >> 16);
+	tegra_plane_writel(p, value, DC_WINBUF_CROPPED_POINT);
+
+	if (dc->soc->supports_block_linear) {
+		unsigned long height = state->tiling.value;
+
+		/* XXX */
+		switch (state->tiling.mode) {
+		case TEGRA_BO_TILING_MODE_PITCH:
+			value = DC_WINBUF_SURFACE_KIND_BLOCK_HEIGHT(0) |
+				DC_WINBUF_SURFACE_KIND_PITCH;
+			break;
+
+		/* XXX not supported on Tegra186 and later */
+		case TEGRA_BO_TILING_MODE_TILED:
+			value = DC_WINBUF_SURFACE_KIND_TILED;
+			break;
+
+		case TEGRA_BO_TILING_MODE_BLOCK:
+			value = DC_WINBUF_SURFACE_KIND_BLOCK_HEIGHT(height) |
+				DC_WINBUF_SURFACE_KIND_BLOCK;
+			break;
+		}
+
+		tegra_plane_writel(p, value, DC_WINBUF_SURFACE_KIND);
+	}
+
+	/* disable gamut CSC */
+	value = tegra_plane_readl(p, DC_WIN_WINDOW_SET_CONTROL);
+	value &= ~CONTROL_CSC_ENABLE;
+	tegra_plane_writel(p, value, DC_WIN_WINDOW_SET_CONTROL);
+
+	pm_runtime_put(dc->dev);
+}
+
+static const struct drm_plane_helper_funcs tegra_shared_plane_helper_funcs = {
+	.atomic_check = tegra_shared_plane_atomic_check,
+	.atomic_update = tegra_shared_plane_atomic_update,
+	.atomic_disable = tegra_shared_plane_atomic_disable,
+};
+
+struct drm_plane *tegra_shared_plane_create(struct drm_device *drm,
+					    struct tegra_dc *dc,
+					    unsigned int wgrp,
+					    unsigned int index)
+{
+	enum drm_plane_type type = DRM_PLANE_TYPE_OVERLAY;
+	struct tegra_drm *tegra = drm->dev_private;
+	struct tegra_display_hub *hub = tegra->hub;
+	/* planes can be assigned to arbitrary CRTCs */
+	unsigned int possible_crtcs = 0x7;
+	struct tegra_shared_plane *plane;
+	unsigned int num_formats;
+	struct drm_plane *p;
+	const u32 *formats;
+	int err;
+
+	plane = kzalloc(sizeof(*plane), GFP_KERNEL);
+	if (!plane)
+		return ERR_PTR(-ENOMEM);
+
+	plane->base.offset = 0x0a00 + 0x0300 * index;
+	plane->base.index = index;
+
+	plane->wgrp = &hub->wgrps[wgrp];
+	plane->wgrp->parent = dc->dev;
+
+	p = &plane->base.base;
+
+	num_formats = ARRAY_SIZE(tegra_shared_plane_formats);
+	formats = tegra_shared_plane_formats;
+
+	err = drm_universal_plane_init(drm, p, possible_crtcs,
+				       &tegra_plane_funcs, formats,
+				       num_formats, NULL, type, NULL);
+	if (err < 0) {
+		kfree(plane);
+		return ERR_PTR(err);
+	}
+
+	drm_plane_helper_add(p, &tegra_shared_plane_helper_funcs);
+	drm_plane_create_zpos_property(p, 0, 0, 255);
+
+	return p;
+}
+
+static void tegra_display_hub_update(struct tegra_dc *dc)
+{
+	u32 value;
+
+	pm_runtime_get_sync(dc->dev);
+
+	value = tegra_dc_readl(dc, DC_CMD_IHUB_COMMON_MISC_CTL);
+	value &= ~LATENCY_EVENT;
+	tegra_dc_writel(dc, value, DC_CMD_IHUB_COMMON_MISC_CTL);
+
+	value = tegra_dc_readl(dc, DC_DISP_IHUB_COMMON_DISPLAY_FETCH_METER);
+	value = CURS_SLOTS(1) | WGRP_SLOTS(1);
+	tegra_dc_writel(dc, value, DC_DISP_IHUB_COMMON_DISPLAY_FETCH_METER);
+
+	tegra_dc_writel(dc, COMMON_UPDATE, DC_CMD_STATE_CONTROL);
+	tegra_dc_readl(dc, DC_CMD_STATE_CONTROL);
+	tegra_dc_writel(dc, COMMON_ACTREQ, DC_CMD_STATE_CONTROL);
+	tegra_dc_readl(dc, DC_CMD_STATE_CONTROL);
+
+	pm_runtime_put(dc->dev);
+}
+
+void tegra_display_hub_atomic_commit(struct drm_device *drm,
+				     struct drm_atomic_state *state)
+{
+	struct tegra_atomic_state *s = to_tegra_atomic_state(state);
+	struct tegra_drm *tegra = drm->dev_private;
+	struct tegra_display_hub *hub = tegra->hub;
+	struct device *dev = hub->client.dev;
+	int err;
+
+	if (s->clk_disp) {
+		err = clk_set_rate(s->clk_disp, s->rate);
+		if (err < 0)
+			dev_err(dev, "failed to set rate of %pC to %lu Hz\n",
+				s->clk_disp, s->rate);
+
+		err = clk_set_parent(hub->clk_disp, s->clk_disp);
+		if (err < 0)
+			dev_err(dev, "failed to set parent of %pC to %pC: %d\n",
+				hub->clk_disp, s->clk_disp, err);
+	}
+
+	if (s->dc)
+		tegra_display_hub_update(s->dc);
+}
+
+static int tegra_display_hub_init(struct host1x_client *client)
+{
+	struct tegra_display_hub *hub = to_tegra_display_hub(client);
+	struct drm_device *drm = dev_get_drvdata(client->parent);
+	struct tegra_drm *tegra = drm->dev_private;
+
+	tegra->hub = hub;
+
+	return 0;
+}
+
+static int tegra_display_hub_exit(struct host1x_client *client)
+{
+	struct drm_device *drm = dev_get_drvdata(client->parent);
+	struct tegra_drm *tegra = drm->dev_private;
+
+	tegra->hub = NULL;
+
+	return 0;
+}
+
+static const struct host1x_client_ops tegra_display_hub_ops = {
+	.init = tegra_display_hub_init,
+	.exit = tegra_display_hub_exit,
+};
+
+static int tegra_display_hub_probe(struct platform_device *pdev)
+{
+	struct tegra_display_hub *hub;
+	unsigned int i;
+	int err;
+
+	hub = devm_kzalloc(&pdev->dev, sizeof(*hub), GFP_KERNEL);
+	if (!hub)
+		return -ENOMEM;
+
+	hub->soc = of_device_get_match_data(&pdev->dev);
+
+	hub->clk_disp = devm_clk_get(&pdev->dev, "disp");
+	if (IS_ERR(hub->clk_disp)) {
+		err = PTR_ERR(hub->clk_disp);
+		return err;
+	}
+
+	hub->clk_dsc = devm_clk_get(&pdev->dev, "dsc");
+	if (IS_ERR(hub->clk_dsc)) {
+		err = PTR_ERR(hub->clk_dsc);
+		return err;
+	}
+
+	hub->clk_hub = devm_clk_get(&pdev->dev, "hub");
+	if (IS_ERR(hub->clk_hub)) {
+		err = PTR_ERR(hub->clk_hub);
+		return err;
+	}
+
+	hub->rst = devm_reset_control_get(&pdev->dev, "misc");
+	if (IS_ERR(hub->rst)) {
+		err = PTR_ERR(hub->rst);
+		return err;
+	}
+
+	hub->wgrps = devm_kcalloc(&pdev->dev, hub->soc->num_wgrps,
+				  sizeof(*hub->wgrps), GFP_KERNEL);
+	if (!hub->wgrps)
+		return -ENOMEM;
+
+	for (i = 0; i < hub->soc->num_wgrps; i++) {
+		struct tegra_windowgroup *wgrp = &hub->wgrps[i];
+		char id[8];
+
+		snprintf(id, sizeof(id), "wgrp%u", i);
+		mutex_init(&wgrp->lock);
+		wgrp->usecount = 0;
+		wgrp->index = i;
+
+		wgrp->rst = devm_reset_control_get(&pdev->dev, id);
+		if (IS_ERR(wgrp->rst))
+			return PTR_ERR(wgrp->rst);
+
+		err = reset_control_assert(wgrp->rst);
+		if (err < 0)
+			return err;
+	}
+
+	/* XXX: enable clock across reset? */
+	err = reset_control_assert(hub->rst);
+	if (err < 0)
+		return err;
+
+	platform_set_drvdata(pdev, hub);
+	pm_runtime_enable(&pdev->dev);
+
+	INIT_LIST_HEAD(&hub->client.list);
+	hub->client.ops = &tegra_display_hub_ops;
+	hub->client.dev = &pdev->dev;
+
+	err = host1x_client_register(&hub->client);
+	if (err < 0)
+		dev_err(&pdev->dev, "failed to register host1x client: %d\n",
+			err);
+
+	return err;
+}
+
+static int tegra_display_hub_remove(struct platform_device *pdev)
+{
+	struct tegra_display_hub *hub = platform_get_drvdata(pdev);
+	int err;
+
+	err = host1x_client_unregister(&hub->client);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
+			err);
+	}
+
+	pm_runtime_disable(&pdev->dev);
+
+	return err;
+}
+
+static int __maybe_unused tegra_display_hub_suspend(struct device *dev)
+{
+	struct tegra_display_hub *hub = dev_get_drvdata(dev);
+	int err;
+
+	err = reset_control_assert(hub->rst);
+	if (err < 0)
+		return err;
+
+	clk_disable_unprepare(hub->clk_hub);
+	clk_disable_unprepare(hub->clk_dsc);
+	clk_disable_unprepare(hub->clk_disp);
+
+	return 0;
+}
+
+static int __maybe_unused tegra_display_hub_resume(struct device *dev)
+{
+	struct tegra_display_hub *hub = dev_get_drvdata(dev);
+	int err;
+
+	err = clk_prepare_enable(hub->clk_disp);
+	if (err < 0)
+		return err;
+
+	err = clk_prepare_enable(hub->clk_dsc);
+	if (err < 0)
+		goto disable_disp;
+
+	err = clk_prepare_enable(hub->clk_hub);
+	if (err < 0)
+		goto disable_dsc;
+
+	err = reset_control_deassert(hub->rst);
+	if (err < 0)
+		goto disable_hub;
+
+	return 0;
+
+disable_hub:
+	clk_disable_unprepare(hub->clk_hub);
+disable_dsc:
+	clk_disable_unprepare(hub->clk_dsc);
+disable_disp:
+	clk_disable_unprepare(hub->clk_disp);
+	return err;
+}
+
+static const struct dev_pm_ops tegra_display_hub_pm_ops = {
+	SET_RUNTIME_PM_OPS(tegra_display_hub_suspend,
+			   tegra_display_hub_resume, NULL)
+};
+
+static const struct tegra_display_hub_soc tegra186_display_hub = {
+	.num_wgrps = 6,
+};
+
+static const struct of_device_id tegra_display_hub_of_match[] = {
+	{
+		.compatible = "nvidia,tegra186-display",
+		.data = &tegra186_display_hub
+	}, {
+		/* sentinel */
+	}
+};
+MODULE_DEVICE_TABLE(of, tegra_display_hub_of_match);
+
+struct platform_driver tegra_display_hub_driver = {
+	.driver = {
+		.name = "tegra-display-hub",
+		.of_match_table = tegra_display_hub_of_match,
+		.pm = &tegra_display_hub_pm_ops,
+	},
+	.probe = tegra_display_hub_probe,
+	.remove = tegra_display_hub_remove,
+};
diff --git a/drivers/gpu/drm/tegra/hub.h b/drivers/gpu/drm/tegra/hub.h
new file mode 100644
index 000000000000..890a47cd05c3
--- /dev/null
+++ b/drivers/gpu/drm/tegra/hub.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2017 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef TEGRA_HUB_H
+#define TEGRA_HUB_H 1
+
+#include <drm/drmP.h>
+#include <drm/drm_plane.h>
+
+#include "plane.h"
+
+struct tegra_dc;
+
+struct tegra_windowgroup {
+	unsigned int usecount;
+	struct mutex lock;
+
+	unsigned int index;
+	struct device *parent;
+	struct reset_control *rst;
+};
+
+struct tegra_shared_plane {
+	struct tegra_plane base;
+	struct tegra_windowgroup *wgrp;
+};
+
+static inline struct tegra_shared_plane *
+to_tegra_shared_plane(struct drm_plane *plane)
+{
+	return container_of(plane, struct tegra_shared_plane, base.base);
+}
+
+struct tegra_display_hub_soc {
+	unsigned int num_wgrps;
+};
+
+struct tegra_display_hub {
+	struct host1x_client client;
+	struct clk *clk_disp;
+	struct clk *clk_dsc;
+	struct clk *clk_hub;
+	struct reset_control *rst;
+
+	const struct tegra_display_hub_soc *soc;
+	struct tegra_windowgroup *wgrps;
+};
+
+static inline struct tegra_display_hub *
+to_tegra_display_hub(struct host1x_client *client)
+{
+	return container_of(client, struct tegra_display_hub, client);
+}
+
+struct tegra_dc;
+struct tegra_plane;
+
+int tegra_display_hub_prepare(struct tegra_display_hub *hub);
+void tegra_display_hub_cleanup(struct tegra_display_hub *hub);
+
+struct drm_plane *tegra_shared_plane_create(struct drm_device *drm,
+					    struct tegra_dc *dc,
+					    unsigned int wgrp,
+					    unsigned int index);
+
+void tegra_display_hub_atomic_commit(struct drm_device *drm,
+				     struct drm_atomic_state *state);
+
+#define DC_CMD_IHUB_COMMON_MISC_CTL 0x068
+#define  LATENCY_EVENT (1 << 3)
+
+#define DC_DISP_IHUB_COMMON_DISPLAY_FETCH_METER 0x451
+#define  CURS_SLOTS(x) (((x) & 0xff) << 8)
+#define  WGRP_SLOTS(x) (((x) & 0xff) << 0)
+
+#endif /* TEGRA_HUB_H */
diff --git a/drivers/gpu/drm/tegra/output.c b/drivers/gpu/drm/tegra/output.c
index 1cfbacea8113..ffe34bd0bb9d 100644
--- a/drivers/gpu/drm/tegra/output.c
+++ b/drivers/gpu/drm/tegra/output.c
@@ -9,7 +9,9 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_panel.h>
+
 #include "drm.h"
+#include "dc.h"
 
 #include <media/cec-notifier.h>
 
@@ -39,7 +41,6 @@ int tegra_output_connector_get_modes(struct drm_connector *connector)
 
 	if (edid) {
 		err = drm_add_edid_modes(connector, edid);
-		drm_edid_to_eld(connector, edid);
 		kfree(edid);
 	}
 
@@ -219,3 +220,25 @@ void tegra_output_exit(struct tegra_output *output)
 	if (output->panel)
 		drm_panel_detach(output->panel);
 }
+
+void tegra_output_find_possible_crtcs(struct tegra_output *output,
+				      struct drm_device *drm)
+{
+	struct device *dev = output->dev;
+	struct drm_crtc *crtc;
+	unsigned int mask = 0;
+
+	drm_for_each_crtc(crtc, drm) {
+		struct tegra_dc *dc = to_tegra_dc(crtc);
+
+		if (tegra_dc_has_output(dc, dev))
+			mask |= drm_crtc_mask(crtc);
+	}
+
+	if (mask == 0) {
+		dev_warn(dev, "missing output definition for heads in DT\n");
+		mask = 0x3;
+	}
+
+	output->encoder.possible_crtcs = mask;
+}
diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c
new file mode 100644
index 000000000000..94dac79ac3c9
--- /dev/null
+++ b/drivers/gpu/drm/tegra/plane.c
@@ -0,0 +1,386 @@
+/*
+ * Copyright (C) 2017 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include "dc.h"
+#include "plane.h"
+
+static void tegra_plane_destroy(struct drm_plane *plane)
+{
+	struct tegra_plane *p = to_tegra_plane(plane);
+
+	drm_plane_cleanup(plane);
+	kfree(p);
+}
+
+static void tegra_plane_reset(struct drm_plane *plane)
+{
+	struct tegra_plane_state *state;
+
+	if (plane->state)
+		__drm_atomic_helper_plane_destroy_state(plane->state);
+
+	kfree(plane->state);
+	plane->state = NULL;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state) {
+		plane->state = &state->base;
+		plane->state->plane = plane;
+	}
+}
+
+static struct drm_plane_state *
+tegra_plane_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct tegra_plane_state *state = to_tegra_plane_state(plane->state);
+	struct tegra_plane_state *copy;
+	unsigned int i;
+
+	copy = kmalloc(sizeof(*copy), GFP_KERNEL);
+	if (!copy)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &copy->base);
+	copy->tiling = state->tiling;
+	copy->format = state->format;
+	copy->swap = state->swap;
+	copy->opaque = state->opaque;
+
+	for (i = 0; i < 3; i++)
+		copy->dependent[i] = state->dependent[i];
+
+	return &copy->base;
+}
+
+static void tegra_plane_atomic_destroy_state(struct drm_plane *plane,
+					     struct drm_plane_state *state)
+{
+	__drm_atomic_helper_plane_destroy_state(state);
+	kfree(state);
+}
+
+const struct drm_plane_funcs tegra_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = tegra_plane_destroy,
+	.reset = tegra_plane_reset,
+	.atomic_duplicate_state = tegra_plane_atomic_duplicate_state,
+	.atomic_destroy_state = tegra_plane_atomic_destroy_state,
+};
+
+int tegra_plane_state_add(struct tegra_plane *plane,
+			  struct drm_plane_state *state)
+{
+	struct drm_crtc_state *crtc_state;
+	struct tegra_dc_state *tegra;
+	struct drm_rect clip;
+	int err;
+
+	/* Propagate errors from allocation or locking failures. */
+	crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	clip.x1 = 0;
+	clip.y1 = 0;
+	clip.x2 = crtc_state->mode.hdisplay;
+	clip.y2 = crtc_state->mode.vdisplay;
+
+	/* Check plane state for visibility and calculate clipping bounds */
+	err = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  0, INT_MAX, true, true);
+	if (err < 0)
+		return err;
+
+	tegra = to_dc_state(crtc_state);
+
+	tegra->planes |= WIN_A_ACT_REQ << plane->index;
+
+	return 0;
+}
+
+int tegra_plane_format(u32 fourcc, u32 *format, u32 *swap)
+{
+	/* assume no swapping of fetched data */
+	if (swap)
+		*swap = BYTE_SWAP_NOSWAP;
+
+	switch (fourcc) {
+	case DRM_FORMAT_ARGB4444:
+		*format = WIN_COLOR_DEPTH_B4G4R4A4;
+		break;
+
+	case DRM_FORMAT_ARGB1555:
+		*format = WIN_COLOR_DEPTH_B5G5R5A1;
+		break;
+
+	case DRM_FORMAT_RGB565:
+		*format = WIN_COLOR_DEPTH_B5G6R5;
+		break;
+
+	case DRM_FORMAT_RGBA5551:
+		*format = WIN_COLOR_DEPTH_A1B5G5R5;
+		break;
+
+	case DRM_FORMAT_ARGB8888:
+		*format = WIN_COLOR_DEPTH_B8G8R8A8;
+		break;
+
+	case DRM_FORMAT_ABGR8888:
+		*format = WIN_COLOR_DEPTH_R8G8B8A8;
+		break;
+
+	case DRM_FORMAT_ABGR4444:
+		*format = WIN_COLOR_DEPTH_R4G4B4A4;
+		break;
+
+	case DRM_FORMAT_ABGR1555:
+		*format = WIN_COLOR_DEPTH_R5G5B5A;
+		break;
+
+	case DRM_FORMAT_BGRA5551:
+		*format = WIN_COLOR_DEPTH_AR5G5B5;
+		break;
+
+	case DRM_FORMAT_XRGB1555:
+		*format = WIN_COLOR_DEPTH_B5G5R5X1;
+		break;
+
+	case DRM_FORMAT_RGBX5551:
+		*format = WIN_COLOR_DEPTH_X1B5G5R5;
+		break;
+
+	case DRM_FORMAT_XBGR1555:
+		*format = WIN_COLOR_DEPTH_R5G5B5X1;
+		break;
+
+	case DRM_FORMAT_BGRX5551:
+		*format = WIN_COLOR_DEPTH_X1R5G5B5;
+		break;
+
+	case DRM_FORMAT_BGR565:
+		*format = WIN_COLOR_DEPTH_R5G6B5;
+		break;
+
+	case DRM_FORMAT_BGRA8888:
+		*format = WIN_COLOR_DEPTH_A8R8G8B8;
+		break;
+
+	case DRM_FORMAT_RGBA8888:
+		*format = WIN_COLOR_DEPTH_A8B8G8R8;
+		break;
+
+	case DRM_FORMAT_XRGB8888:
+		*format = WIN_COLOR_DEPTH_B8G8R8X8;
+		break;
+
+	case DRM_FORMAT_XBGR8888:
+		*format = WIN_COLOR_DEPTH_R8G8B8X8;
+		break;
+
+	case DRM_FORMAT_UYVY:
+		*format = WIN_COLOR_DEPTH_YCbCr422;
+		break;
+
+	case DRM_FORMAT_YUYV:
+		if (!swap)
+			return -EINVAL;
+
+		*format = WIN_COLOR_DEPTH_YCbCr422;
+		*swap = BYTE_SWAP_SWAP2;
+		break;
+
+	case DRM_FORMAT_YUV420:
+		*format = WIN_COLOR_DEPTH_YCbCr420P;
+		break;
+
+	case DRM_FORMAT_YUV422:
+		*format = WIN_COLOR_DEPTH_YCbCr422P;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+bool tegra_plane_format_is_yuv(unsigned int format, bool *planar)
+{
+	switch (format) {
+	case WIN_COLOR_DEPTH_YCbCr422:
+	case WIN_COLOR_DEPTH_YUV422:
+		if (planar)
+			*planar = false;
+
+		return true;
+
+	case WIN_COLOR_DEPTH_YCbCr420P:
+	case WIN_COLOR_DEPTH_YUV420P:
+	case WIN_COLOR_DEPTH_YCbCr422P:
+	case WIN_COLOR_DEPTH_YUV422P:
+	case WIN_COLOR_DEPTH_YCbCr422R:
+	case WIN_COLOR_DEPTH_YUV422R:
+	case WIN_COLOR_DEPTH_YCbCr422RA:
+	case WIN_COLOR_DEPTH_YUV422RA:
+		if (planar)
+			*planar = true;
+
+		return true;
+	}
+
+	if (planar)
+		*planar = false;
+
+	return false;
+}
+
+static bool __drm_format_has_alpha(u32 format)
+{
+	switch (format) {
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_ABGR8888:
+	case DRM_FORMAT_ARGB8888:
+		return true;
+	}
+
+	return false;
+}
+
+/*
+ * This is applicable to Tegra20 and Tegra30 only where the opaque formats can
+ * be emulated using the alpha formats and alpha blending disabled.
+ */
+bool tegra_plane_format_has_alpha(unsigned int format)
+{
+	switch (format) {
+	case WIN_COLOR_DEPTH_B5G5R5A1:
+	case WIN_COLOR_DEPTH_A1B5G5R5:
+	case WIN_COLOR_DEPTH_R8G8B8A8:
+	case WIN_COLOR_DEPTH_B8G8R8A8:
+		return true;
+	}
+
+	return false;
+}
+
+int tegra_plane_format_get_alpha(unsigned int opaque, unsigned int *alpha)
+{
+	if (tegra_plane_format_is_yuv(opaque, NULL)) {
+		*alpha = opaque;
+		return 0;
+	}
+
+	switch (opaque) {
+	case WIN_COLOR_DEPTH_B5G5R5X1:
+		*alpha = WIN_COLOR_DEPTH_B5G5R5A1;
+		return 0;
+
+	case WIN_COLOR_DEPTH_X1B5G5R5:
+		*alpha = WIN_COLOR_DEPTH_A1B5G5R5;
+		return 0;
+
+	case WIN_COLOR_DEPTH_R8G8B8X8:
+		*alpha = WIN_COLOR_DEPTH_R8G8B8A8;
+		return 0;
+
+	case WIN_COLOR_DEPTH_B8G8R8X8:
+		*alpha = WIN_COLOR_DEPTH_B8G8R8A8;
+		return 0;
+
+	case WIN_COLOR_DEPTH_B5G6R5:
+		*alpha = opaque;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+unsigned int tegra_plane_get_overlap_index(struct tegra_plane *plane,
+					   struct tegra_plane *other)
+{
+	unsigned int index = 0, i;
+
+	WARN_ON(plane == other);
+
+	for (i = 0; i < 3; i++) {
+		if (i == plane->index)
+			continue;
+
+		if (i == other->index)
+			break;
+
+		index++;
+	}
+
+	return index;
+}
+
+void tegra_plane_check_dependent(struct tegra_plane *tegra,
+				 struct tegra_plane_state *state)
+{
+	struct drm_plane_state *old, *new;
+	struct drm_plane *plane;
+	unsigned int zpos[2];
+	unsigned int i;
+
+	for (i = 0; i < 2; i++)
+		zpos[i] = 0;
+
+	for_each_oldnew_plane_in_state(state->base.state, plane, old, new, i) {
+		struct tegra_plane *p = to_tegra_plane(plane);
+		unsigned index;
+
+		/* skip this plane and planes on different CRTCs */
+		if (p == tegra || new->crtc != state->base.crtc)
+			continue;
+
+		index = tegra_plane_get_overlap_index(tegra, p);
+
+		state->dependent[index] = false;
+
+		/*
+		 * If any of the other planes is on top of this plane and uses
+		 * a format with an alpha component, mark this plane as being
+		 * dependent, meaning it's alpha value will be 1 minus the sum
+		 * of alpha components of the overlapping planes.
+		 */
+		if (p->index > tegra->index) {
+			if (__drm_format_has_alpha(new->fb->format->format))
+				state->dependent[index] = true;
+
+			/* keep track of the Z position */
+			zpos[index] = p->index;
+		}
+	}
+
+	/*
+	 * The region where three windows overlap is the intersection of the
+	 * two regions where two windows overlap. It contributes to the area
+	 * if any of the windows on top of it have an alpha component.
+	 */
+	for (i = 0; i < 2; i++)
+		state->dependent[2] = state->dependent[2] ||
+				      state->dependent[i];
+
+	/*
+	 * However, if any of the windows on top of this window is opaque, it
+	 * will completely conceal this window within that area, so avoid the
+	 * window from contributing to the area.
+	 */
+	for (i = 0; i < 2; i++) {
+		if (zpos[i] > tegra->index)
+			state->dependent[2] = state->dependent[2] &&
+					      state->dependent[i];
+	}
+}
diff --git a/drivers/gpu/drm/tegra/plane.h b/drivers/gpu/drm/tegra/plane.h
new file mode 100644
index 000000000000..6938719e7e5d
--- /dev/null
+++ b/drivers/gpu/drm/tegra/plane.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2017 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef TEGRA_PLANE_H
+#define TEGRA_PLANE_H 1
+
+#include <drm/drm_plane.h>
+
+struct tegra_bo;
+struct tegra_dc;
+
+struct tegra_plane {
+	struct drm_plane base;
+	struct tegra_dc *dc;
+	unsigned int offset;
+	unsigned int index;
+};
+
+struct tegra_cursor {
+	struct tegra_plane base;
+
+	struct tegra_bo *bo;
+	unsigned int width;
+	unsigned int height;
+};
+
+static inline struct tegra_plane *to_tegra_plane(struct drm_plane *plane)
+{
+	return container_of(plane, struct tegra_plane, base);
+}
+
+struct tegra_plane_state {
+	struct drm_plane_state base;
+
+	struct tegra_bo_tiling tiling;
+	u32 format;
+	u32 swap;
+
+	/* used for legacy blending support only */
+	bool opaque;
+	bool dependent[3];
+};
+
+static inline struct tegra_plane_state *
+to_tegra_plane_state(struct drm_plane_state *state)
+{
+	if (state)
+		return container_of(state, struct tegra_plane_state, base);
+
+	return NULL;
+}
+
+extern const struct drm_plane_funcs tegra_plane_funcs;
+
+int tegra_plane_state_add(struct tegra_plane *plane,
+			  struct drm_plane_state *state);
+
+int tegra_plane_format(u32 fourcc, u32 *format, u32 *swap);
+bool tegra_plane_format_is_yuv(unsigned int format, bool *planar);
+bool tegra_plane_format_has_alpha(unsigned int format);
+int tegra_plane_format_get_alpha(unsigned int opaque, unsigned int *alpha);
+void tegra_plane_check_dependent(struct tegra_plane *tegra,
+				 struct tegra_plane_state *state);
+
+#endif /* TEGRA_PLANE_H */
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index 476079f1255f..7d2a955fc515 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -22,23 +22,37 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_dp_helper.h>
 #include <drm/drm_panel.h>
+#include <drm/drm_scdc_helper.h>
 
 #include "dc.h"
 #include "drm.h"
 #include "sor.h"
 #include "trace.h"
 
+/*
+ * XXX Remove this after the commit adding it to soc/tegra/pmc.h has been
+ * merged. Having this around after the commit is merged should be safe since
+ * the preprocessor will effectively replace all occurrences and therefore no
+ * duplicate will be defined.
+ */
+#define TEGRA_IO_PAD_HDMI_DP0 26
+
 #define SOR_REKEY 0x38
 
 struct tegra_sor_hdmi_settings {
 	unsigned long frequency;
 
 	u8 vcocap;
+	u8 filter;
 	u8 ichpmp;
 	u8 loadadj;
-	u8 termadj;
-	u8 tx_pu;
-	u8 bg_vref;
+	u8 tmds_termadj;
+	u8 tx_pu_value;
+	u8 bg_temp_coef;
+	u8 bg_vref_level;
+	u8 avdd10_level;
+	u8 avdd14_level;
+	u8 sparepll;
 
 	u8 drive_current[4];
 	u8 preemphasis[4];
@@ -49,51 +63,76 @@ static const struct tegra_sor_hdmi_settings tegra210_sor_hdmi_defaults[] = {
 	{
 		.frequency = 54000000,
 		.vcocap = 0x0,
+		.filter = 0x0,
 		.ichpmp = 0x1,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x10,
-		.bg_vref = 0x8,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x10,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0x8,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x33, 0x3a, 0x3a, 0x3a },
 		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
 	}, {
 		.frequency = 75000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0x1,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x40,
-		.bg_vref = 0x8,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x40,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0x8,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x33, 0x3a, 0x3a, 0x3a },
 		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
 	}, {
 		.frequency = 150000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0x1,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x66,
-		.bg_vref = 0x8,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x66,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0x8,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x33, 0x3a, 0x3a, 0x3a },
 		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
 	}, {
 		.frequency = 300000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0x1,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x66,
-		.bg_vref = 0xa,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x66,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0xa,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x33, 0x3f, 0x3f, 0x3f },
 		.preemphasis = { 0x00, 0x17, 0x17, 0x17 },
 	}, {
 		.frequency = 600000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0x1,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x66,
-		.bg_vref = 0x8,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x66,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0x8,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x33, 0x3f, 0x3f, 0x3f },
 		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
 	},
@@ -103,53 +142,170 @@ static const struct tegra_sor_hdmi_settings tegra210_sor_hdmi_defaults[] = {
 	{
 		.frequency = 75000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0x1,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x40,
-		.bg_vref = 0x8,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x40,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0x8,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x29, 0x29, 0x29, 0x29 },
 		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
 	}, {
 		.frequency = 150000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0x1,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x66,
-		.bg_vref = 0x8,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x66,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0x8,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x30, 0x37, 0x37, 0x37 },
 		.preemphasis = { 0x01, 0x02, 0x02, 0x02 },
 	}, {
 		.frequency = 300000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0x6,
 		.loadadj = 0x3,
-		.termadj = 0x9,
-		.tx_pu = 0x66,
-		.bg_vref = 0xf,
+		.tmds_termadj = 0x9,
+		.tx_pu_value = 0x66,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0xf,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x30, 0x37, 0x37, 0x37 },
 		.preemphasis = { 0x10, 0x3e, 0x3e, 0x3e },
 	}, {
 		.frequency = 600000000,
 		.vcocap = 0x3,
+		.filter = 0x0,
 		.ichpmp = 0xa,
 		.loadadj = 0x3,
-		.termadj = 0xb,
-		.tx_pu = 0x66,
-		.bg_vref = 0xe,
+		.tmds_termadj = 0xb,
+		.tx_pu_value = 0x66,
+		.bg_temp_coef = 0x3,
+		.bg_vref_level = 0xe,
+		.avdd10_level = 0x4,
+		.avdd14_level = 0x4,
+		.sparepll = 0x0,
 		.drive_current = { 0x35, 0x3e, 0x3e, 0x3e },
 		.preemphasis = { 0x02, 0x3f, 0x3f, 0x3f },
 	},
 };
 #endif
 
+static const struct tegra_sor_hdmi_settings tegra186_sor_hdmi_defaults[] = {
+	{
+		.frequency = 54000000,
+		.vcocap = 0,
+		.filter = 5,
+		.ichpmp = 5,
+		.loadadj = 3,
+		.tmds_termadj = 0xf,
+		.tx_pu_value = 0,
+		.bg_temp_coef = 3,
+		.bg_vref_level = 8,
+		.avdd10_level = 4,
+		.avdd14_level = 4,
+		.sparepll = 0x54,
+		.drive_current = { 0x3a, 0x3a, 0x3a, 0x33 },
+		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
+	}, {
+		.frequency = 75000000,
+		.vcocap = 1,
+		.filter = 5,
+		.ichpmp = 5,
+		.loadadj = 3,
+		.tmds_termadj = 0xf,
+		.tx_pu_value = 0,
+		.bg_temp_coef = 3,
+		.bg_vref_level = 8,
+		.avdd10_level = 4,
+		.avdd14_level = 4,
+		.sparepll = 0x44,
+		.drive_current = { 0x3a, 0x3a, 0x3a, 0x33 },
+		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
+	}, {
+		.frequency = 150000000,
+		.vcocap = 3,
+		.filter = 5,
+		.ichpmp = 5,
+		.loadadj = 3,
+		.tmds_termadj = 15,
+		.tx_pu_value = 0x66 /* 0 */,
+		.bg_temp_coef = 3,
+		.bg_vref_level = 8,
+		.avdd10_level = 4,
+		.avdd14_level = 4,
+		.sparepll = 0x00, /* 0x34 */
+		.drive_current = { 0x3a, 0x3a, 0x3a, 0x37 },
+		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
+	}, {
+		.frequency = 300000000,
+		.vcocap = 3,
+		.filter = 5,
+		.ichpmp = 5,
+		.loadadj = 3,
+		.tmds_termadj = 15,
+		.tx_pu_value = 64,
+		.bg_temp_coef = 3,
+		.bg_vref_level = 8,
+		.avdd10_level = 4,
+		.avdd14_level = 4,
+		.sparepll = 0x34,
+		.drive_current = { 0x3d, 0x3d, 0x3d, 0x33 },
+		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
+	}, {
+		.frequency = 600000000,
+		.vcocap = 3,
+		.filter = 5,
+		.ichpmp = 5,
+		.loadadj = 3,
+		.tmds_termadj = 12,
+		.tx_pu_value = 96,
+		.bg_temp_coef = 3,
+		.bg_vref_level = 8,
+		.avdd10_level = 4,
+		.avdd14_level = 4,
+		.sparepll = 0x34,
+		.drive_current = { 0x3d, 0x3d, 0x3d, 0x33 },
+		.preemphasis = { 0x00, 0x00, 0x00, 0x00 },
+	}
+};
+
+struct tegra_sor_regs {
+	unsigned int head_state0;
+	unsigned int head_state1;
+	unsigned int head_state2;
+	unsigned int head_state3;
+	unsigned int head_state4;
+	unsigned int head_state5;
+	unsigned int pll0;
+	unsigned int pll1;
+	unsigned int pll2;
+	unsigned int pll3;
+	unsigned int dp_padctl0;
+	unsigned int dp_padctl2;
+};
+
 struct tegra_sor_soc {
 	bool supports_edp;
 	bool supports_lvds;
 	bool supports_hdmi;
 	bool supports_dp;
 
+	const struct tegra_sor_regs *regs;
+	bool has_nvdisplay;
+
 	const struct tegra_sor_hdmi_settings *settings;
 	unsigned int num_settings;
 
@@ -171,6 +327,7 @@ struct tegra_sor {
 
 	const struct tegra_sor_soc *soc;
 	void __iomem *regs;
+	unsigned int index;
 
 	struct reset_control *rst;
 	struct clk *clk_parent;
@@ -183,10 +340,9 @@ struct tegra_sor {
 	struct drm_dp_aux *aux;
 
 	struct drm_info_list *debugfs_files;
-	struct drm_minor *minor;
-	struct dentry *debugfs;
 
 	const struct tegra_sor_ops *ops;
+	enum tegra_io_pad pad;
 
 	/* for HDMI 2.0 */
 	struct tegra_sor_hdmi_settings *settings;
@@ -195,11 +351,16 @@ struct tegra_sor {
 	struct regulator *avdd_io_supply;
 	struct regulator *vdd_pll_supply;
 	struct regulator *hdmi_supply;
+
+	struct delayed_work scdc;
+	bool scdc_enabled;
 };
 
 struct tegra_sor_state {
 	struct drm_connector_state base;
 
+	unsigned int link_speed;
+	unsigned long pclk;
 	unsigned int bpc;
 };
 
@@ -389,23 +550,23 @@ static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
 	/* disable LVDS mode */
 	tegra_sor_writel(sor, 0, SOR_LVDS);
 
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value |= SOR_DP_PADCTL_TX_PU_ENABLE;
 	value &= ~SOR_DP_PADCTL_TX_PU_MASK;
 	value |= SOR_DP_PADCTL_TX_PU(2); /* XXX: don't hardcode? */
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value |= SOR_DP_PADCTL_CM_TXD_3 | SOR_DP_PADCTL_CM_TXD_2 |
 		 SOR_DP_PADCTL_CM_TXD_1 | SOR_DP_PADCTL_CM_TXD_0;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	usleep_range(10, 100);
 
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value &= ~(SOR_DP_PADCTL_CM_TXD_3 | SOR_DP_PADCTL_CM_TXD_2 |
 		   SOR_DP_PADCTL_CM_TXD_1 | SOR_DP_PADCTL_CM_TXD_0);
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	err = drm_dp_aux_prepare(sor->aux, DP_SET_ANSI_8B10B);
 	if (err < 0)
@@ -465,47 +626,6 @@ static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
 	return 0;
 }
 
-static void tegra_sor_dp_term_calibrate(struct tegra_sor *sor)
-{
-	u32 mask = 0x08, adj = 0, value;
-
-	/* enable pad calibration logic */
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
-	value &= ~SOR_DP_PADCTL_PAD_CAL_PD;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
-
-	value = tegra_sor_readl(sor, SOR_PLL1);
-	value |= SOR_PLL1_TMDS_TERM;
-	tegra_sor_writel(sor, value, SOR_PLL1);
-
-	while (mask) {
-		adj |= mask;
-
-		value = tegra_sor_readl(sor, SOR_PLL1);
-		value &= ~SOR_PLL1_TMDS_TERMADJ_MASK;
-		value |= SOR_PLL1_TMDS_TERMADJ(adj);
-		tegra_sor_writel(sor, value, SOR_PLL1);
-
-		usleep_range(100, 200);
-
-		value = tegra_sor_readl(sor, SOR_PLL1);
-		if (value & SOR_PLL1_TERM_COMPOUT)
-			adj &= ~mask;
-
-		mask >>= 1;
-	}
-
-	value = tegra_sor_readl(sor, SOR_PLL1);
-	value &= ~SOR_PLL1_TMDS_TERMADJ_MASK;
-	value |= SOR_PLL1_TMDS_TERMADJ(adj);
-	tegra_sor_writel(sor, value, SOR_PLL1);
-
-	/* disable pad calibration logic */
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
-	value |= SOR_DP_PADCTL_PAD_CAL_PD;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
-}
-
 static void tegra_sor_super_update(struct tegra_sor *sor)
 {
 	tegra_sor_writel(sor, 0, SOR_SUPER_STATE0);
@@ -897,31 +1017,31 @@ static void tegra_sor_mode_set(struct tegra_sor *sor,
 	 */
 
 	value = ((mode->vtotal & 0x7fff) << 16) | (mode->htotal & 0x7fff);
-	tegra_sor_writel(sor, value, SOR_HEAD_STATE1(dc->pipe));
+	tegra_sor_writel(sor, value, sor->soc->regs->head_state1 + dc->pipe);
 
 	/* sync end = sync width - 1 */
 	vse = mode->vsync_end - mode->vsync_start - 1;
 	hse = mode->hsync_end - mode->hsync_start - 1;
 
 	value = ((vse & 0x7fff) << 16) | (hse & 0x7fff);
-	tegra_sor_writel(sor, value, SOR_HEAD_STATE2(dc->pipe));
+	tegra_sor_writel(sor, value, sor->soc->regs->head_state2 + dc->pipe);
 
 	/* blank end = sync end + back porch */
 	vbe = vse + (mode->vtotal - mode->vsync_end);
 	hbe = hse + (mode->htotal - mode->hsync_end);
 
 	value = ((vbe & 0x7fff) << 16) | (hbe & 0x7fff);
-	tegra_sor_writel(sor, value, SOR_HEAD_STATE3(dc->pipe));
+	tegra_sor_writel(sor, value, sor->soc->regs->head_state3 + dc->pipe);
 
 	/* blank start = blank end + active */
 	vbs = vbe + mode->vdisplay;
 	hbs = hbe + mode->hdisplay;
 
 	value = ((vbs & 0x7fff) << 16) | (hbs & 0x7fff);
-	tegra_sor_writel(sor, value, SOR_HEAD_STATE4(dc->pipe));
+	tegra_sor_writel(sor, value, sor->soc->regs->head_state4 + dc->pipe);
 
 	/* XXX interlacing support */
-	tegra_sor_writel(sor, 0x001, SOR_HEAD_STATE5(dc->pipe));
+	tegra_sor_writel(sor, 0x001, sor->soc->regs->head_state5 + dc->pipe);
 }
 
 static int tegra_sor_detach(struct tegra_sor *sor)
@@ -1003,10 +1123,10 @@ static int tegra_sor_power_down(struct tegra_sor *sor)
 		return err;
 	}
 
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value &= ~(SOR_DP_PADCTL_PD_TXD_3 | SOR_DP_PADCTL_PD_TXD_0 |
 		   SOR_DP_PADCTL_PD_TXD_1 | SOR_DP_PADCTL_PD_TXD_2);
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	/* stop lane sequencer */
 	value = SOR_LANE_SEQ_CTL_TRIGGER | SOR_LANE_SEQ_CTL_SEQUENCE_UP |
@@ -1026,20 +1146,20 @@ static int tegra_sor_power_down(struct tegra_sor *sor)
 	if ((value & SOR_LANE_SEQ_CTL_TRIGGER) != 0)
 		return -ETIMEDOUT;
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value |= SOR_PLL2_PORT_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	usleep_range(20, 100);
 
-	value = tegra_sor_readl(sor, SOR_PLL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll0);
 	value |= SOR_PLL0_VCOPD | SOR_PLL0_PWR;
-	tegra_sor_writel(sor, value, SOR_PLL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll0);
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value |= SOR_PLL2_SEQ_PLLCAPPD;
 	value |= SOR_PLL2_SEQ_PLLCAPPD_ENFORCE;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	usleep_range(20, 100);
 
@@ -1105,12 +1225,133 @@ static int tegra_sor_show_crc(struct seq_file *s, void *data)
 	return err;
 }
 
+#define DEBUGFS_REG32(_name) { .name = #_name, .offset = _name }
+
+static const struct debugfs_reg32 tegra_sor_regs[] = {
+	DEBUGFS_REG32(SOR_CTXSW),
+	DEBUGFS_REG32(SOR_SUPER_STATE0),
+	DEBUGFS_REG32(SOR_SUPER_STATE1),
+	DEBUGFS_REG32(SOR_STATE0),
+	DEBUGFS_REG32(SOR_STATE1),
+	DEBUGFS_REG32(SOR_HEAD_STATE0(0)),
+	DEBUGFS_REG32(SOR_HEAD_STATE0(1)),
+	DEBUGFS_REG32(SOR_HEAD_STATE1(0)),
+	DEBUGFS_REG32(SOR_HEAD_STATE1(1)),
+	DEBUGFS_REG32(SOR_HEAD_STATE2(0)),
+	DEBUGFS_REG32(SOR_HEAD_STATE2(1)),
+	DEBUGFS_REG32(SOR_HEAD_STATE3(0)),
+	DEBUGFS_REG32(SOR_HEAD_STATE3(1)),
+	DEBUGFS_REG32(SOR_HEAD_STATE4(0)),
+	DEBUGFS_REG32(SOR_HEAD_STATE4(1)),
+	DEBUGFS_REG32(SOR_HEAD_STATE5(0)),
+	DEBUGFS_REG32(SOR_HEAD_STATE5(1)),
+	DEBUGFS_REG32(SOR_CRC_CNTRL),
+	DEBUGFS_REG32(SOR_DP_DEBUG_MVID),
+	DEBUGFS_REG32(SOR_CLK_CNTRL),
+	DEBUGFS_REG32(SOR_CAP),
+	DEBUGFS_REG32(SOR_PWR),
+	DEBUGFS_REG32(SOR_TEST),
+	DEBUGFS_REG32(SOR_PLL0),
+	DEBUGFS_REG32(SOR_PLL1),
+	DEBUGFS_REG32(SOR_PLL2),
+	DEBUGFS_REG32(SOR_PLL3),
+	DEBUGFS_REG32(SOR_CSTM),
+	DEBUGFS_REG32(SOR_LVDS),
+	DEBUGFS_REG32(SOR_CRCA),
+	DEBUGFS_REG32(SOR_CRCB),
+	DEBUGFS_REG32(SOR_BLANK),
+	DEBUGFS_REG32(SOR_SEQ_CTL),
+	DEBUGFS_REG32(SOR_LANE_SEQ_CTL),
+	DEBUGFS_REG32(SOR_SEQ_INST(0)),
+	DEBUGFS_REG32(SOR_SEQ_INST(1)),
+	DEBUGFS_REG32(SOR_SEQ_INST(2)),
+	DEBUGFS_REG32(SOR_SEQ_INST(3)),
+	DEBUGFS_REG32(SOR_SEQ_INST(4)),
+	DEBUGFS_REG32(SOR_SEQ_INST(5)),
+	DEBUGFS_REG32(SOR_SEQ_INST(6)),
+	DEBUGFS_REG32(SOR_SEQ_INST(7)),
+	DEBUGFS_REG32(SOR_SEQ_INST(8)),
+	DEBUGFS_REG32(SOR_SEQ_INST(9)),
+	DEBUGFS_REG32(SOR_SEQ_INST(10)),
+	DEBUGFS_REG32(SOR_SEQ_INST(11)),
+	DEBUGFS_REG32(SOR_SEQ_INST(12)),
+	DEBUGFS_REG32(SOR_SEQ_INST(13)),
+	DEBUGFS_REG32(SOR_SEQ_INST(14)),
+	DEBUGFS_REG32(SOR_SEQ_INST(15)),
+	DEBUGFS_REG32(SOR_PWM_DIV),
+	DEBUGFS_REG32(SOR_PWM_CTL),
+	DEBUGFS_REG32(SOR_VCRC_A0),
+	DEBUGFS_REG32(SOR_VCRC_A1),
+	DEBUGFS_REG32(SOR_VCRC_B0),
+	DEBUGFS_REG32(SOR_VCRC_B1),
+	DEBUGFS_REG32(SOR_CCRC_A0),
+	DEBUGFS_REG32(SOR_CCRC_A1),
+	DEBUGFS_REG32(SOR_CCRC_B0),
+	DEBUGFS_REG32(SOR_CCRC_B1),
+	DEBUGFS_REG32(SOR_EDATA_A0),
+	DEBUGFS_REG32(SOR_EDATA_A1),
+	DEBUGFS_REG32(SOR_EDATA_B0),
+	DEBUGFS_REG32(SOR_EDATA_B1),
+	DEBUGFS_REG32(SOR_COUNT_A0),
+	DEBUGFS_REG32(SOR_COUNT_A1),
+	DEBUGFS_REG32(SOR_COUNT_B0),
+	DEBUGFS_REG32(SOR_COUNT_B1),
+	DEBUGFS_REG32(SOR_DEBUG_A0),
+	DEBUGFS_REG32(SOR_DEBUG_A1),
+	DEBUGFS_REG32(SOR_DEBUG_B0),
+	DEBUGFS_REG32(SOR_DEBUG_B1),
+	DEBUGFS_REG32(SOR_TRIG),
+	DEBUGFS_REG32(SOR_MSCHECK),
+	DEBUGFS_REG32(SOR_XBAR_CTRL),
+	DEBUGFS_REG32(SOR_XBAR_POL),
+	DEBUGFS_REG32(SOR_DP_LINKCTL0),
+	DEBUGFS_REG32(SOR_DP_LINKCTL1),
+	DEBUGFS_REG32(SOR_LANE_DRIVE_CURRENT0),
+	DEBUGFS_REG32(SOR_LANE_DRIVE_CURRENT1),
+	DEBUGFS_REG32(SOR_LANE4_DRIVE_CURRENT0),
+	DEBUGFS_REG32(SOR_LANE4_DRIVE_CURRENT1),
+	DEBUGFS_REG32(SOR_LANE_PREEMPHASIS0),
+	DEBUGFS_REG32(SOR_LANE_PREEMPHASIS1),
+	DEBUGFS_REG32(SOR_LANE4_PREEMPHASIS0),
+	DEBUGFS_REG32(SOR_LANE4_PREEMPHASIS1),
+	DEBUGFS_REG32(SOR_LANE_POSTCURSOR0),
+	DEBUGFS_REG32(SOR_LANE_POSTCURSOR1),
+	DEBUGFS_REG32(SOR_DP_CONFIG0),
+	DEBUGFS_REG32(SOR_DP_CONFIG1),
+	DEBUGFS_REG32(SOR_DP_MN0),
+	DEBUGFS_REG32(SOR_DP_MN1),
+	DEBUGFS_REG32(SOR_DP_PADCTL0),
+	DEBUGFS_REG32(SOR_DP_PADCTL1),
+	DEBUGFS_REG32(SOR_DP_PADCTL2),
+	DEBUGFS_REG32(SOR_DP_DEBUG0),
+	DEBUGFS_REG32(SOR_DP_DEBUG1),
+	DEBUGFS_REG32(SOR_DP_SPARE0),
+	DEBUGFS_REG32(SOR_DP_SPARE1),
+	DEBUGFS_REG32(SOR_DP_AUDIO_CTRL),
+	DEBUGFS_REG32(SOR_DP_AUDIO_HBLANK_SYMBOLS),
+	DEBUGFS_REG32(SOR_DP_AUDIO_VBLANK_SYMBOLS),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_HEADER),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_SUBPACK0),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_SUBPACK1),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_SUBPACK2),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_SUBPACK3),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_SUBPACK4),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_SUBPACK5),
+	DEBUGFS_REG32(SOR_DP_GENERIC_INFOFRAME_SUBPACK6),
+	DEBUGFS_REG32(SOR_DP_TPG),
+	DEBUGFS_REG32(SOR_DP_TPG_CONFIG),
+	DEBUGFS_REG32(SOR_DP_LQ_CSTM0),
+	DEBUGFS_REG32(SOR_DP_LQ_CSTM1),
+	DEBUGFS_REG32(SOR_DP_LQ_CSTM2),
+};
+
 static int tegra_sor_show_regs(struct seq_file *s, void *data)
 {
 	struct drm_info_node *node = s->private;
 	struct tegra_sor *sor = node->info_ent->data;
 	struct drm_crtc *crtc = sor->output.encoder.crtc;
 	struct drm_device *drm = node->minor->dev;
+	unsigned int i;
 	int err = 0;
 
 	drm_modeset_lock_all(drm);
@@ -1120,126 +1361,12 @@ static int tegra_sor_show_regs(struct seq_file *s, void *data)
 		goto unlock;
 	}
 
-#define DUMP_REG(name)						\
-	seq_printf(s, "%-38s %#05x %08x\n", #name, name,	\
-		   tegra_sor_readl(sor, name))
-
-	DUMP_REG(SOR_CTXSW);
-	DUMP_REG(SOR_SUPER_STATE0);
-	DUMP_REG(SOR_SUPER_STATE1);
-	DUMP_REG(SOR_STATE0);
-	DUMP_REG(SOR_STATE1);
-	DUMP_REG(SOR_HEAD_STATE0(0));
-	DUMP_REG(SOR_HEAD_STATE0(1));
-	DUMP_REG(SOR_HEAD_STATE1(0));
-	DUMP_REG(SOR_HEAD_STATE1(1));
-	DUMP_REG(SOR_HEAD_STATE2(0));
-	DUMP_REG(SOR_HEAD_STATE2(1));
-	DUMP_REG(SOR_HEAD_STATE3(0));
-	DUMP_REG(SOR_HEAD_STATE3(1));
-	DUMP_REG(SOR_HEAD_STATE4(0));
-	DUMP_REG(SOR_HEAD_STATE4(1));
-	DUMP_REG(SOR_HEAD_STATE5(0));
-	DUMP_REG(SOR_HEAD_STATE5(1));
-	DUMP_REG(SOR_CRC_CNTRL);
-	DUMP_REG(SOR_DP_DEBUG_MVID);
-	DUMP_REG(SOR_CLK_CNTRL);
-	DUMP_REG(SOR_CAP);
-	DUMP_REG(SOR_PWR);
-	DUMP_REG(SOR_TEST);
-	DUMP_REG(SOR_PLL0);
-	DUMP_REG(SOR_PLL1);
-	DUMP_REG(SOR_PLL2);
-	DUMP_REG(SOR_PLL3);
-	DUMP_REG(SOR_CSTM);
-	DUMP_REG(SOR_LVDS);
-	DUMP_REG(SOR_CRCA);
-	DUMP_REG(SOR_CRCB);
-	DUMP_REG(SOR_BLANK);
-	DUMP_REG(SOR_SEQ_CTL);
-	DUMP_REG(SOR_LANE_SEQ_CTL);
-	DUMP_REG(SOR_SEQ_INST(0));
-	DUMP_REG(SOR_SEQ_INST(1));
-	DUMP_REG(SOR_SEQ_INST(2));
-	DUMP_REG(SOR_SEQ_INST(3));
-	DUMP_REG(SOR_SEQ_INST(4));
-	DUMP_REG(SOR_SEQ_INST(5));
-	DUMP_REG(SOR_SEQ_INST(6));
-	DUMP_REG(SOR_SEQ_INST(7));
-	DUMP_REG(SOR_SEQ_INST(8));
-	DUMP_REG(SOR_SEQ_INST(9));
-	DUMP_REG(SOR_SEQ_INST(10));
-	DUMP_REG(SOR_SEQ_INST(11));
-	DUMP_REG(SOR_SEQ_INST(12));
-	DUMP_REG(SOR_SEQ_INST(13));
-	DUMP_REG(SOR_SEQ_INST(14));
-	DUMP_REG(SOR_SEQ_INST(15));
-	DUMP_REG(SOR_PWM_DIV);
-	DUMP_REG(SOR_PWM_CTL);
-	DUMP_REG(SOR_VCRC_A0);
-	DUMP_REG(SOR_VCRC_A1);
-	DUMP_REG(SOR_VCRC_B0);
-	DUMP_REG(SOR_VCRC_B1);
-	DUMP_REG(SOR_CCRC_A0);
-	DUMP_REG(SOR_CCRC_A1);
-	DUMP_REG(SOR_CCRC_B0);
-	DUMP_REG(SOR_CCRC_B1);
-	DUMP_REG(SOR_EDATA_A0);
-	DUMP_REG(SOR_EDATA_A1);
-	DUMP_REG(SOR_EDATA_B0);
-	DUMP_REG(SOR_EDATA_B1);
-	DUMP_REG(SOR_COUNT_A0);
-	DUMP_REG(SOR_COUNT_A1);
-	DUMP_REG(SOR_COUNT_B0);
-	DUMP_REG(SOR_COUNT_B1);
-	DUMP_REG(SOR_DEBUG_A0);
-	DUMP_REG(SOR_DEBUG_A1);
-	DUMP_REG(SOR_DEBUG_B0);
-	DUMP_REG(SOR_DEBUG_B1);
-	DUMP_REG(SOR_TRIG);
-	DUMP_REG(SOR_MSCHECK);
-	DUMP_REG(SOR_XBAR_CTRL);
-	DUMP_REG(SOR_XBAR_POL);
-	DUMP_REG(SOR_DP_LINKCTL0);
-	DUMP_REG(SOR_DP_LINKCTL1);
-	DUMP_REG(SOR_LANE_DRIVE_CURRENT0);
-	DUMP_REG(SOR_LANE_DRIVE_CURRENT1);
-	DUMP_REG(SOR_LANE4_DRIVE_CURRENT0);
-	DUMP_REG(SOR_LANE4_DRIVE_CURRENT1);
-	DUMP_REG(SOR_LANE_PREEMPHASIS0);
-	DUMP_REG(SOR_LANE_PREEMPHASIS1);
-	DUMP_REG(SOR_LANE4_PREEMPHASIS0);
-	DUMP_REG(SOR_LANE4_PREEMPHASIS1);
-	DUMP_REG(SOR_LANE_POSTCURSOR0);
-	DUMP_REG(SOR_LANE_POSTCURSOR1);
-	DUMP_REG(SOR_DP_CONFIG0);
-	DUMP_REG(SOR_DP_CONFIG1);
-	DUMP_REG(SOR_DP_MN0);
-	DUMP_REG(SOR_DP_MN1);
-	DUMP_REG(SOR_DP_PADCTL0);
-	DUMP_REG(SOR_DP_PADCTL1);
-	DUMP_REG(SOR_DP_DEBUG0);
-	DUMP_REG(SOR_DP_DEBUG1);
-	DUMP_REG(SOR_DP_SPARE0);
-	DUMP_REG(SOR_DP_SPARE1);
-	DUMP_REG(SOR_DP_AUDIO_CTRL);
-	DUMP_REG(SOR_DP_AUDIO_HBLANK_SYMBOLS);
-	DUMP_REG(SOR_DP_AUDIO_VBLANK_SYMBOLS);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_HEADER);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_SUBPACK0);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_SUBPACK1);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_SUBPACK2);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_SUBPACK3);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_SUBPACK4);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_SUBPACK5);
-	DUMP_REG(SOR_DP_GENERIC_INFOFRAME_SUBPACK6);
-	DUMP_REG(SOR_DP_TPG);
-	DUMP_REG(SOR_DP_TPG_CONFIG);
-	DUMP_REG(SOR_DP_LQ_CSTM0);
-	DUMP_REG(SOR_DP_LQ_CSTM1);
-	DUMP_REG(SOR_DP_LQ_CSTM2);
-
-#undef DUMP_REG
+	for (i = 0; i < ARRAY_SIZE(tegra_sor_regs); i++) {
+		unsigned int offset = tegra_sor_regs[i].offset;
+
+		seq_printf(s, "%-38s %#05x %08x\n", tegra_sor_regs[i].name,
+			   offset, tegra_sor_readl(sor, offset));
+	}
 
 unlock:
 	drm_modeset_unlock_all(drm);
@@ -1251,57 +1378,46 @@ static const struct drm_info_list debugfs_files[] = {
 	{ "regs", tegra_sor_show_regs, 0, NULL },
 };
 
-static int tegra_sor_debugfs_init(struct tegra_sor *sor,
-				  struct drm_minor *minor)
+static int tegra_sor_late_register(struct drm_connector *connector)
 {
-	const char *name = sor->soc->supports_dp ? "sor1" : "sor";
-	unsigned int i;
+	struct tegra_output *output = connector_to_output(connector);
+	unsigned int i, count = ARRAY_SIZE(debugfs_files);
+	struct drm_minor *minor = connector->dev->primary;
+	struct dentry *root = connector->debugfs_entry;
+	struct tegra_sor *sor = to_sor(output);
 	int err;
 
-	sor->debugfs = debugfs_create_dir(name, minor->debugfs_root);
-	if (!sor->debugfs)
-		return -ENOMEM;
-
 	sor->debugfs_files = kmemdup(debugfs_files, sizeof(debugfs_files),
 				     GFP_KERNEL);
-	if (!sor->debugfs_files) {
-		err = -ENOMEM;
-		goto remove;
-	}
+	if (!sor->debugfs_files)
+		return -ENOMEM;
 
-	for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
+	for (i = 0; i < count; i++)
 		sor->debugfs_files[i].data = sor;
 
-	err = drm_debugfs_create_files(sor->debugfs_files,
-				       ARRAY_SIZE(debugfs_files),
-				       sor->debugfs, minor);
+	err = drm_debugfs_create_files(sor->debugfs_files, count, root, minor);
 	if (err < 0)
 		goto free;
 
-	sor->minor = minor;
-
 	return 0;
 
 free:
 	kfree(sor->debugfs_files);
 	sor->debugfs_files = NULL;
-remove:
-	debugfs_remove_recursive(sor->debugfs);
-	sor->debugfs = NULL;
+
 	return err;
 }
 
-static void tegra_sor_debugfs_exit(struct tegra_sor *sor)
+static void tegra_sor_early_unregister(struct drm_connector *connector)
 {
-	drm_debugfs_remove_files(sor->debugfs_files, ARRAY_SIZE(debugfs_files),
-				 sor->minor);
-	sor->minor = NULL;
+	struct tegra_output *output = connector_to_output(connector);
+	unsigned int count = ARRAY_SIZE(debugfs_files);
+	struct tegra_sor *sor = to_sor(output);
 
+	drm_debugfs_remove_files(sor->debugfs_files, count,
+				 connector->dev->primary);
 	kfree(sor->debugfs_files);
 	sor->debugfs_files = NULL;
-
-	debugfs_remove_recursive(sor->debugfs);
-	sor->debugfs = NULL;
 }
 
 static void tegra_sor_connector_reset(struct drm_connector *connector)
@@ -1354,6 +1470,8 @@ static const struct drm_connector_funcs tegra_sor_connector_funcs = {
 	.destroy = tegra_output_connector_destroy,
 	.atomic_duplicate_state = tegra_sor_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+	.late_register = tegra_sor_late_register,
+	.early_unregister = tegra_sor_early_unregister,
 };
 
 static int tegra_sor_connector_get_modes(struct drm_connector *connector)
@@ -1377,10 +1495,6 @@ static enum drm_mode_status
 tegra_sor_connector_mode_valid(struct drm_connector *connector,
 			       struct drm_display_mode *mode)
 {
-	/* HDMI 2.0 modes are not yet supported */
-	if (mode->clock > 340000)
-		return MODE_NOCLOCK;
-
 	return MODE_OK;
 }
 
@@ -1417,7 +1531,7 @@ static void tegra_sor_edp_disable(struct drm_encoder *encoder)
 	 */
 	if (dc) {
 		value = tegra_dc_readl(dc, DC_DISP_DISP_WIN_OPTIONS);
-		value &= ~SOR_ENABLE;
+		value &= ~SOR_ENABLE(0);
 		tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
 
 		tegra_dc_commit(dc);
@@ -1433,9 +1547,9 @@ static void tegra_sor_edp_disable(struct drm_encoder *encoder)
 			dev_err(sor->dev, "failed to disable DP: %d\n", err);
 	}
 
-	err = tegra_io_rail_power_off(TEGRA_IO_RAIL_LVDS);
+	err = tegra_io_pad_power_disable(sor->pad);
 	if (err < 0)
-		dev_err(sor->dev, "failed to power off I/O rail: %d\n", err);
+		dev_err(sor->dev, "failed to power off I/O pad: %d\n", err);
 
 	if (output->panel)
 		drm_panel_unprepare(output->panel);
@@ -1533,40 +1647,40 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	value |= SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_DPCLK;
 	tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_BANDGAP_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 	usleep_range(20, 100);
 
-	value = tegra_sor_readl(sor, SOR_PLL3);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll3);
 	value |= SOR_PLL3_PLL_VDD_MODE_3V3;
-	tegra_sor_writel(sor, value, SOR_PLL3);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll3);
 
 	value = SOR_PLL0_ICHPMP(0xf) | SOR_PLL0_VCOCAP_RST |
 		SOR_PLL0_PLLREG_LEVEL_V45 | SOR_PLL0_RESISTOR_EXT;
-	tegra_sor_writel(sor, value, SOR_PLL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll0);
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value |= SOR_PLL2_SEQ_PLLCAPPD;
 	value &= ~SOR_PLL2_SEQ_PLLCAPPD_ENFORCE;
 	value |= SOR_PLL2_LVDS_ENABLE;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	value = SOR_PLL1_TERM_COMPOUT | SOR_PLL1_TMDS_TERM;
-	tegra_sor_writel(sor, value, SOR_PLL1);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll1);
 
 	while (true) {
-		value = tegra_sor_readl(sor, SOR_PLL2);
+		value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 		if ((value & SOR_PLL2_SEQ_PLLCAPPD_ENFORCE) == 0)
 			break;
 
 		usleep_range(250, 1000);
 	}
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_POWERDOWN_OVERRIDE;
 	value &= ~SOR_PLL2_PORT_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	/*
 	 * power up
@@ -1579,49 +1693,49 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
 
 	/* step 1 */
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value |= SOR_PLL2_SEQ_PLLCAPPD_ENFORCE | SOR_PLL2_PORT_POWERDOWN |
 		 SOR_PLL2_BANDGAP_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
-	value = tegra_sor_readl(sor, SOR_PLL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll0);
 	value |= SOR_PLL0_VCOPD | SOR_PLL0_PWR;
-	tegra_sor_writel(sor, value, SOR_PLL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll0);
 
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value &= ~SOR_DP_PADCTL_PAD_CAL_PD;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	/* step 2 */
-	err = tegra_io_rail_power_on(TEGRA_IO_RAIL_LVDS);
+	err = tegra_io_pad_power_enable(sor->pad);
 	if (err < 0)
-		dev_err(sor->dev, "failed to power on I/O rail: %d\n", err);
+		dev_err(sor->dev, "failed to power on I/O pad: %d\n", err);
 
 	usleep_range(5, 100);
 
 	/* step 3 */
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_BANDGAP_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	usleep_range(20, 100);
 
 	/* step 4 */
-	value = tegra_sor_readl(sor, SOR_PLL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll0);
 	value &= ~SOR_PLL0_VCOPD;
 	value &= ~SOR_PLL0_PWR;
-	tegra_sor_writel(sor, value, SOR_PLL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll0);
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_SEQ_PLLCAPPD_ENFORCE;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	usleep_range(200, 1000);
 
 	/* step 5 */
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_PORT_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	/* XXX not in TRM */
 	for (value = 0, i = 0; i < 5; i++)
@@ -1637,7 +1751,7 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 		dev_err(sor->dev, "failed to set parent clock: %d\n", err);
 
 	/* power DP lanes */
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 
 	if (link.num_lanes <= 2)
 		value &= ~(SOR_DP_PADCTL_PD_TXD_3 | SOR_DP_PADCTL_PD_TXD_2);
@@ -1654,7 +1768,7 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	else
 		value |= SOR_DP_PADCTL_PD_TXD_0;
 
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	value = tegra_sor_readl(sor, SOR_DP_LINKCTL0);
 	value &= ~SOR_DP_LINKCTL_LANE_COUNT_MASK;
@@ -1698,9 +1812,9 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	tegra_sor_writel(sor, value, SOR_DP_TPG);
 
 	/* enable pad calibration logic */
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value |= SOR_DP_PADCTL_PAD_CAL_PD;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	err = drm_dp_link_probe(sor->aux, &link);
 	if (err < 0)
@@ -1773,7 +1887,7 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 	tegra_sor_update(sor);
 
 	value = tegra_dc_readl(dc, DC_DISP_DISP_WIN_OPTIONS);
-	value |= SOR_ENABLE;
+	value |= SOR_ENABLE(0);
 	tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
 
 	tegra_dc_commit(dc);
@@ -1805,6 +1919,18 @@ tegra_sor_encoder_atomic_check(struct drm_encoder *encoder,
 
 	info = &output->connector.display_info;
 
+	/*
+	 * For HBR2 modes, the SOR brick needs to use the x20 multiplier, so
+	 * the pixel clock must be corrected accordingly.
+	 */
+	if (pclk >= 340000000) {
+		state->link_speed = 20;
+		state->pclk = pclk / 2;
+	} else {
+		state->link_speed = 10;
+		state->pclk = pclk;
+	}
+
 	err = tegra_dc_state_setup_clock(dc, crtc_state, sor->clk_parent,
 					 pclk, 0);
 	if (err < 0) {
@@ -1955,6 +2081,81 @@ tegra_sor_hdmi_find_settings(struct tegra_sor *sor, unsigned long frequency)
 	return NULL;
 }
 
+static void tegra_sor_hdmi_disable_scrambling(struct tegra_sor *sor)
+{
+	u32 value;
+
+	value = tegra_sor_readl(sor, SOR_HDMI2_CTRL);
+	value &= ~SOR_HDMI2_CTRL_CLOCK_MODE_DIV_BY_4;
+	value &= ~SOR_HDMI2_CTRL_SCRAMBLE;
+	tegra_sor_writel(sor, value, SOR_HDMI2_CTRL);
+}
+
+static void tegra_sor_hdmi_scdc_disable(struct tegra_sor *sor)
+{
+	struct i2c_adapter *ddc = sor->output.ddc;
+
+	drm_scdc_set_high_tmds_clock_ratio(ddc, false);
+	drm_scdc_set_scrambling(ddc, false);
+
+	tegra_sor_hdmi_disable_scrambling(sor);
+}
+
+static void tegra_sor_hdmi_scdc_stop(struct tegra_sor *sor)
+{
+	if (sor->scdc_enabled) {
+		cancel_delayed_work_sync(&sor->scdc);
+		tegra_sor_hdmi_scdc_disable(sor);
+	}
+}
+
+static void tegra_sor_hdmi_enable_scrambling(struct tegra_sor *sor)
+{
+	u32 value;
+
+	value = tegra_sor_readl(sor, SOR_HDMI2_CTRL);
+	value |= SOR_HDMI2_CTRL_CLOCK_MODE_DIV_BY_4;
+	value |= SOR_HDMI2_CTRL_SCRAMBLE;
+	tegra_sor_writel(sor, value, SOR_HDMI2_CTRL);
+}
+
+static void tegra_sor_hdmi_scdc_enable(struct tegra_sor *sor)
+{
+	struct i2c_adapter *ddc = sor->output.ddc;
+
+	drm_scdc_set_high_tmds_clock_ratio(ddc, true);
+	drm_scdc_set_scrambling(ddc, true);
+
+	tegra_sor_hdmi_enable_scrambling(sor);
+}
+
+static void tegra_sor_hdmi_scdc_work(struct work_struct *work)
+{
+	struct tegra_sor *sor = container_of(work, struct tegra_sor, scdc.work);
+	struct i2c_adapter *ddc = sor->output.ddc;
+
+	if (!drm_scdc_get_scrambling_status(ddc)) {
+		DRM_DEBUG_KMS("SCDC not scrambled\n");
+		tegra_sor_hdmi_scdc_enable(sor);
+	}
+
+	schedule_delayed_work(&sor->scdc, msecs_to_jiffies(5000));
+}
+
+static void tegra_sor_hdmi_scdc_start(struct tegra_sor *sor)
+{
+	struct drm_scdc *scdc = &sor->output.connector.display_info.hdmi.scdc;
+	struct drm_display_mode *mode;
+
+	mode = &sor->output.encoder.crtc->state->adjusted_mode;
+
+	if (mode->clock >= 340000 && scdc->supported) {
+		schedule_delayed_work(&sor->scdc, msecs_to_jiffies(5000));
+		tegra_sor_hdmi_scdc_enable(sor);
+		sor->scdc_enabled = true;
+	}
+}
+
 static void tegra_sor_hdmi_disable(struct drm_encoder *encoder)
 {
 	struct tegra_output *output = encoder_to_output(encoder);
@@ -1963,6 +2164,8 @@ static void tegra_sor_hdmi_disable(struct drm_encoder *encoder)
 	u32 value;
 	int err;
 
+	tegra_sor_hdmi_scdc_stop(sor);
+
 	err = tegra_sor_detach(sor);
 	if (err < 0)
 		dev_err(sor->dev, "failed to detach SOR: %d\n", err);
@@ -1972,8 +2175,12 @@ static void tegra_sor_hdmi_disable(struct drm_encoder *encoder)
 
 	/* disable display to SOR clock */
 	value = tegra_dc_readl(dc, DC_DISP_DISP_WIN_OPTIONS);
-	value &= ~SOR1_TIMING_CYA;
-	value &= ~SOR1_ENABLE;
+
+	if (!sor->soc->has_nvdisplay)
+		value &= ~(SOR1_TIMING_CYA | SOR_ENABLE(1));
+	else
+		value &= ~SOR_ENABLE(sor->index);
+
 	tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
 
 	tegra_dc_commit(dc);
@@ -1982,9 +2189,9 @@ static void tegra_sor_hdmi_disable(struct drm_encoder *encoder)
 	if (err < 0)
 		dev_err(sor->dev, "failed to power down SOR: %d\n", err);
 
-	err = tegra_io_rail_power_off(TEGRA_IO_RAIL_HDMI);
+	err = tegra_io_pad_power_disable(sor->pad);
 	if (err < 0)
-		dev_err(sor->dev, "failed to power off HDMI rail: %d\n", err);
+		dev_err(sor->dev, "failed to power off I/O pad: %d\n", err);
 
 	pm_runtime_put(sor->dev);
 }
@@ -1998,12 +2205,14 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 	struct tegra_sor *sor = to_sor(output);
 	struct tegra_sor_state *state;
 	struct drm_display_mode *mode;
+	unsigned long rate, pclk;
 	unsigned int div, i;
 	u32 value;
 	int err;
 
 	state = to_sor_state(output->connector.state);
 	mode = &encoder->crtc->state->adjusted_mode;
+	pclk = mode->clock * 1000;
 
 	pm_runtime_get_sync(sor->dev);
 
@@ -2016,44 +2225,44 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 
 	div = clk_get_rate(sor->clk) / 1000000 * 4;
 
-	err = tegra_io_rail_power_on(TEGRA_IO_RAIL_HDMI);
+	err = tegra_io_pad_power_enable(sor->pad);
 	if (err < 0)
-		dev_err(sor->dev, "failed to power on HDMI rail: %d\n", err);
+		dev_err(sor->dev, "failed to power on I/O pad: %d\n", err);
 
 	usleep_range(20, 100);
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_BANDGAP_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	usleep_range(20, 100);
 
-	value = tegra_sor_readl(sor, SOR_PLL3);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll3);
 	value &= ~SOR_PLL3_PLL_VDD_MODE_3V3;
-	tegra_sor_writel(sor, value, SOR_PLL3);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll3);
 
-	value = tegra_sor_readl(sor, SOR_PLL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll0);
 	value &= ~SOR_PLL0_VCOPD;
 	value &= ~SOR_PLL0_PWR;
-	tegra_sor_writel(sor, value, SOR_PLL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll0);
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_SEQ_PLLCAPPD_ENFORCE;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	usleep_range(200, 400);
 
-	value = tegra_sor_readl(sor, SOR_PLL2);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll2);
 	value &= ~SOR_PLL2_POWERDOWN_OVERRIDE;
 	value &= ~SOR_PLL2_PORT_POWERDOWN;
-	tegra_sor_writel(sor, value, SOR_PLL2);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll2);
 
 	usleep_range(20, 100);
 
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value |= SOR_DP_PADCTL_PD_TXD_3 | SOR_DP_PADCTL_PD_TXD_0 |
 		 SOR_DP_PADCTL_PD_TXD_1 | SOR_DP_PADCTL_PD_TXD_2;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	while (true) {
 		value = tegra_sor_readl(sor, SOR_LANE_SEQ_CTL);
@@ -2079,18 +2288,30 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 	value &= ~SOR_CLK_CNTRL_DP_LINK_SPEED_MASK;
 	value &= ~SOR_CLK_CNTRL_DP_CLK_SEL_MASK;
 
-	if (mode->clock < 340000)
+	if (mode->clock < 340000) {
+		DRM_DEBUG_KMS("setting 2.7 GHz link speed\n");
 		value |= SOR_CLK_CNTRL_DP_LINK_SPEED_G2_70;
-	else
+	} else {
+		DRM_DEBUG_KMS("setting 5.4 GHz link speed\n");
 		value |= SOR_CLK_CNTRL_DP_LINK_SPEED_G5_40;
+	}
 
 	value |= SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_PCLK;
 	tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
 
+	/* SOR pad PLL stabilization time */
+	usleep_range(250, 1000);
+
+	value = tegra_sor_readl(sor, SOR_DP_LINKCTL0);
+	value &= ~SOR_DP_LINKCTL_LANE_COUNT_MASK;
+	value |= SOR_DP_LINKCTL_LANE_COUNT(4);
+	tegra_sor_writel(sor, value, SOR_DP_LINKCTL0);
+
 	value = tegra_sor_readl(sor, SOR_DP_SPARE0);
-	value |= SOR_DP_SPARE_DISP_VIDEO_PREAMBLE;
+	value &= ~SOR_DP_SPARE_DISP_VIDEO_PREAMBLE;
 	value &= ~SOR_DP_SPARE_PANEL_INTERNAL;
-	value |= SOR_DP_SPARE_SEQ_ENABLE;
+	value &= ~SOR_DP_SPARE_SEQ_ENABLE;
+	value &= ~SOR_DP_SPARE_MACRO_SOR_CLK;
 	tegra_sor_writel(sor, value, SOR_DP_SPARE0);
 
 	value = SOR_SEQ_CTL_PU_PC(0) | SOR_SEQ_CTL_PU_PC_ALT(0) |
@@ -2102,9 +2323,11 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 	tegra_sor_writel(sor, value, SOR_SEQ_INST(0));
 	tegra_sor_writel(sor, value, SOR_SEQ_INST(8));
 
-	/* program the reference clock */
-	value = SOR_REFCLK_DIV_INT(div) | SOR_REFCLK_DIV_FRAC(div);
-	tegra_sor_writel(sor, value, SOR_REFCLK);
+	if (!sor->soc->has_nvdisplay) {
+		/* program the reference clock */
+		value = SOR_REFCLK_DIV_INT(div) | SOR_REFCLK_DIV_FRAC(div);
+		tegra_sor_writel(sor, value, SOR_REFCLK);
+	}
 
 	/* XXX not in TRM */
 	for (value = 0, i = 0; i < 5; i++)
@@ -2127,13 +2350,25 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 		return;
 	}
 
-	value = SOR_INPUT_CONTROL_HDMI_SRC_SELECT(dc->pipe);
+	/* adjust clock rate for HDMI 2.0 modes */
+	rate = clk_get_rate(sor->clk_parent);
 
-	/* XXX is this the proper check? */
-	if (mode->clock < 75000)
-		value |= SOR_INPUT_CONTROL_ARM_VIDEO_RANGE_LIMITED;
+	if (mode->clock >= 340000)
+		rate /= 2;
 
-	tegra_sor_writel(sor, value, SOR_INPUT_CONTROL);
+	DRM_DEBUG_KMS("setting clock to %lu Hz, mode: %lu Hz\n", rate, pclk);
+
+	clk_set_rate(sor->clk, rate);
+
+	if (!sor->soc->has_nvdisplay) {
+		value = SOR_INPUT_CONTROL_HDMI_SRC_SELECT(dc->pipe);
+
+		/* XXX is this the proper check? */
+		if (mode->clock < 75000)
+			value |= SOR_INPUT_CONTROL_ARM_VIDEO_RANGE_LIMITED;
+
+		tegra_sor_writel(sor, value, SOR_INPUT_CONTROL);
+	}
 
 	max_ac = ((mode->htotal - mode->hdisplay) - SOR_REKEY - 18) / 32;
 
@@ -2141,20 +2376,23 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 		SOR_HDMI_CTRL_AUDIO_LAYOUT | SOR_HDMI_CTRL_REKEY(SOR_REKEY);
 	tegra_sor_writel(sor, value, SOR_HDMI_CTRL);
 
-	/* H_PULSE2 setup */
-	pulse_start = h_ref_to_sync + (mode->hsync_end - mode->hsync_start) +
-		      (mode->htotal - mode->hsync_end) - 10;
+	if (!dc->soc->has_nvdisplay) {
+		/* H_PULSE2 setup */
+		pulse_start = h_ref_to_sync +
+			      (mode->hsync_end - mode->hsync_start) +
+			      (mode->htotal - mode->hsync_end) - 10;
 
-	value = PULSE_LAST_END_A | PULSE_QUAL_VACTIVE |
-		PULSE_POLARITY_HIGH | PULSE_MODE_NORMAL;
-	tegra_dc_writel(dc, value, DC_DISP_H_PULSE2_CONTROL);
+		value = PULSE_LAST_END_A | PULSE_QUAL_VACTIVE |
+			PULSE_POLARITY_HIGH | PULSE_MODE_NORMAL;
+		tegra_dc_writel(dc, value, DC_DISP_H_PULSE2_CONTROL);
 
-	value = PULSE_END(pulse_start + 8) | PULSE_START(pulse_start);
-	tegra_dc_writel(dc, value, DC_DISP_H_PULSE2_POSITION_A);
+		value = PULSE_END(pulse_start + 8) | PULSE_START(pulse_start);
+		tegra_dc_writel(dc, value, DC_DISP_H_PULSE2_POSITION_A);
 
-	value = tegra_dc_readl(dc, DC_DISP_DISP_SIGNAL_OPTIONS0);
-	value |= H_PULSE2_ENABLE;
-	tegra_dc_writel(dc, value, DC_DISP_DISP_SIGNAL_OPTIONS0);
+		value = tegra_dc_readl(dc, DC_DISP_DISP_SIGNAL_OPTIONS0);
+		value |= H_PULSE2_ENABLE;
+		tegra_dc_writel(dc, value, DC_DISP_DISP_SIGNAL_OPTIONS0);
+	}
 
 	/* infoframe setup */
 	err = tegra_sor_hdmi_setup_avi_infoframe(sor, mode);
@@ -2171,9 +2409,9 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 	tegra_sor_writel(sor, value, SOR_STATE1);
 
 	/* power up pad calibration */
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value &= ~SOR_DP_PADCTL_PAD_CAL_PD;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
 	/* production settings */
 	settings = tegra_sor_hdmi_find_settings(sor, mode->clock * 1000);
@@ -2183,51 +2421,68 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 		return;
 	}
 
-	value = tegra_sor_readl(sor, SOR_PLL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll0);
 	value &= ~SOR_PLL0_ICHPMP_MASK;
+	value &= ~SOR_PLL0_FILTER_MASK;
 	value &= ~SOR_PLL0_VCOCAP_MASK;
 	value |= SOR_PLL0_ICHPMP(settings->ichpmp);
+	value |= SOR_PLL0_FILTER(settings->filter);
 	value |= SOR_PLL0_VCOCAP(settings->vcocap);
-	tegra_sor_writel(sor, value, SOR_PLL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll0);
 
-	tegra_sor_dp_term_calibrate(sor);
-
-	value = tegra_sor_readl(sor, SOR_PLL1);
+	/* XXX not in TRM */
+	value = tegra_sor_readl(sor, sor->soc->regs->pll1);
 	value &= ~SOR_PLL1_LOADADJ_MASK;
+	value &= ~SOR_PLL1_TMDS_TERMADJ_MASK;
 	value |= SOR_PLL1_LOADADJ(settings->loadadj);
-	tegra_sor_writel(sor, value, SOR_PLL1);
+	value |= SOR_PLL1_TMDS_TERMADJ(settings->tmds_termadj);
+	value |= SOR_PLL1_TMDS_TERM;
+	tegra_sor_writel(sor, value, sor->soc->regs->pll1);
 
-	value = tegra_sor_readl(sor, SOR_PLL3);
+	value = tegra_sor_readl(sor, sor->soc->regs->pll3);
+	value &= ~SOR_PLL3_BG_TEMP_COEF_MASK;
 	value &= ~SOR_PLL3_BG_VREF_LEVEL_MASK;
-	value |= SOR_PLL3_BG_VREF_LEVEL(settings->bg_vref);
-	tegra_sor_writel(sor, value, SOR_PLL3);
-
-	value = settings->drive_current[0] << 24 |
-		settings->drive_current[1] << 16 |
-		settings->drive_current[2] <<  8 |
-		settings->drive_current[3] <<  0;
+	value &= ~SOR_PLL3_AVDD10_LEVEL_MASK;
+	value &= ~SOR_PLL3_AVDD14_LEVEL_MASK;
+	value |= SOR_PLL3_BG_TEMP_COEF(settings->bg_temp_coef);
+	value |= SOR_PLL3_BG_VREF_LEVEL(settings->bg_vref_level);
+	value |= SOR_PLL3_AVDD10_LEVEL(settings->avdd10_level);
+	value |= SOR_PLL3_AVDD14_LEVEL(settings->avdd14_level);
+	tegra_sor_writel(sor, value, sor->soc->regs->pll3);
+
+	value = settings->drive_current[3] << 24 |
+		settings->drive_current[2] << 16 |
+		settings->drive_current[1] <<  8 |
+		settings->drive_current[0] <<  0;
 	tegra_sor_writel(sor, value, SOR_LANE_DRIVE_CURRENT0);
 
-	value = settings->preemphasis[0] << 24 |
-		settings->preemphasis[1] << 16 |
-		settings->preemphasis[2] <<  8 |
-		settings->preemphasis[3] <<  0;
+	value = settings->preemphasis[3] << 24 |
+		settings->preemphasis[2] << 16 |
+		settings->preemphasis[1] <<  8 |
+		settings->preemphasis[0] <<  0;
 	tegra_sor_writel(sor, value, SOR_LANE_PREEMPHASIS0);
 
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value &= ~SOR_DP_PADCTL_TX_PU_MASK;
 	value |= SOR_DP_PADCTL_TX_PU_ENABLE;
-	value |= SOR_DP_PADCTL_TX_PU(settings->tx_pu);
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	value |= SOR_DP_PADCTL_TX_PU(settings->tx_pu_value);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
+
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl2);
+	value &= ~SOR_DP_PADCTL_SPAREPLL_MASK;
+	value |= SOR_DP_PADCTL_SPAREPLL(settings->sparepll);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl2);
 
 	/* power down pad calibration */
-	value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
+	value = tegra_sor_readl(sor, sor->soc->regs->dp_padctl0);
 	value |= SOR_DP_PADCTL_PAD_CAL_PD;
-	tegra_sor_writel(sor, value, SOR_DP_PADCTL0);
+	tegra_sor_writel(sor, value, sor->soc->regs->dp_padctl0);
 
-	/* miscellaneous display controller settings */
-	value = VSYNC_H_POSITION(1);
-	tegra_dc_writel(dc, value, DC_DISP_DISP_TIMING_OPTIONS);
+	if (!dc->soc->has_nvdisplay) {
+		/* miscellaneous display controller settings */
+		value = VSYNC_H_POSITION(1);
+		tegra_dc_writel(dc, value, DC_DISP_DISP_TIMING_OPTIONS);
+	}
 
 	value = tegra_dc_readl(dc, DC_DISP_DISP_COLOR_CONTROL);
 	value &= ~DITHER_CONTROL_MASK;
@@ -2242,6 +2497,14 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 		value |= BASE_COLOR_SIZE_888;
 		break;
 
+	case 10:
+		value |= BASE_COLOR_SIZE_101010;
+		break;
+
+	case 12:
+		value |= BASE_COLOR_SIZE_121212;
+		break;
+
 	default:
 		WARN(1, "%u bits-per-color not supported\n", state->bpc);
 		value |= BASE_COLOR_SIZE_888;
@@ -2250,40 +2513,65 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
 
 	tegra_dc_writel(dc, value, DC_DISP_DISP_COLOR_CONTROL);
 
+	/* XXX set display head owner */
+	value = tegra_sor_readl(sor, SOR_STATE1);
+	value &= ~SOR_STATE_ASY_OWNER_MASK;
+	value |= SOR_STATE_ASY_OWNER(1 + dc->pipe);
+	tegra_sor_writel(sor, value, SOR_STATE1);
+
 	err = tegra_sor_power_up(sor, 250);
 	if (err < 0)
 		dev_err(sor->dev, "failed to power up SOR: %d\n", err);
 
 	/* configure dynamic range of output */
-	value = tegra_sor_readl(sor, SOR_HEAD_STATE0(dc->pipe));
+	value = tegra_sor_readl(sor, sor->soc->regs->head_state0 + dc->pipe);
 	value &= ~SOR_HEAD_STATE_RANGECOMPRESS_MASK;
 	value &= ~SOR_HEAD_STATE_DYNRANGE_MASK;
-	tegra_sor_writel(sor, value, SOR_HEAD_STATE0(dc->pipe));
+	tegra_sor_writel(sor, value, sor->soc->regs->head_state0 + dc->pipe);
 
 	/* configure colorspace */
-	value = tegra_sor_readl(sor, SOR_HEAD_STATE0(dc->pipe));
+	value = tegra_sor_readl(sor, sor->soc->regs->head_state0 + dc->pipe);
 	value &= ~SOR_HEAD_STATE_COLORSPACE_MASK;
 	value |= SOR_HEAD_STATE_COLORSPACE_RGB;
-	tegra_sor_writel(sor, value, SOR_HEAD_STATE0(dc->pipe));
+	tegra_sor_writel(sor, value, sor->soc->regs->head_state0 + dc->pipe);
 
 	tegra_sor_mode_set(sor, mode, state);
 
 	tegra_sor_update(sor);
 
+	/* program preamble timing in SOR (XXX) */
+	value = tegra_sor_readl(sor, SOR_DP_SPARE0);
+	value &= ~SOR_DP_SPARE_DISP_VIDEO_PREAMBLE;
+	tegra_sor_writel(sor, value, SOR_DP_SPARE0);
+
 	err = tegra_sor_attach(sor);
 	if (err < 0)
 		dev_err(sor->dev, "failed to attach SOR: %d\n", err);
 
 	/* enable display to SOR clock and generate HDMI preamble */
 	value = tegra_dc_readl(dc, DC_DISP_DISP_WIN_OPTIONS);
-	value |= SOR1_ENABLE | SOR1_TIMING_CYA;
+
+	if (!sor->soc->has_nvdisplay)
+		value |= SOR_ENABLE(1) | SOR1_TIMING_CYA;
+	else
+		value |= SOR_ENABLE(sor->index);
+
 	tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
 
+	if (dc->soc->has_nvdisplay) {
+		value = tegra_dc_readl(dc, DC_DISP_CORE_SOR_SET_CONTROL(sor->index));
+		value &= ~PROTOCOL_MASK;
+		value |= PROTOCOL_SINGLE_TMDS_A;
+		tegra_dc_writel(dc, value, DC_DISP_CORE_SOR_SET_CONTROL(sor->index));
+	}
+
 	tegra_dc_commit(dc);
 
 	err = tegra_sor_wakeup(sor);
 	if (err < 0)
 		dev_err(sor->dev, "failed to wakeup SOR: %d\n", err);
+
+	tegra_sor_hdmi_scdc_start(sor);
 }
 
 static const struct drm_encoder_helper_funcs tegra_sor_hdmi_helpers = {
@@ -2344,13 +2632,7 @@ static int tegra_sor_init(struct host1x_client *client)
 		return err;
 	}
 
-	sor->output.encoder.possible_crtcs = 0x3;
-
-	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
-		err = tegra_sor_debugfs_init(sor, drm->primary);
-		if (err < 0)
-			dev_err(sor->dev, "debugfs setup failed: %d\n", err);
-	}
+	tegra_output_find_possible_crtcs(&sor->output, drm);
 
 	if (sor->aux) {
 		err = drm_dp_aux_attach(sor->aux, &sor->output);
@@ -2420,9 +2702,6 @@ static int tegra_sor_exit(struct host1x_client *client)
 	clk_disable_unprepare(sor->clk_dp);
 	clk_disable_unprepare(sor->clk);
 
-	if (IS_ENABLED(CONFIG_DEBUG_FS))
-		tegra_sor_debugfs_exit(sor);
-
 	return 0;
 }
 
@@ -2480,6 +2759,8 @@ static int tegra_sor_hdmi_probe(struct tegra_sor *sor)
 		return err;
 	}
 
+	INIT_DELAYED_WORK(&sor->scdc, tegra_sor_hdmi_scdc_work);
+
 	return 0;
 }
 
@@ -2502,19 +2783,53 @@ static const u8 tegra124_sor_xbar_cfg[5] = {
 	0, 1, 2, 3, 4
 };
 
+static const struct tegra_sor_regs tegra124_sor_regs = {
+	.head_state0 = 0x05,
+	.head_state1 = 0x07,
+	.head_state2 = 0x09,
+	.head_state3 = 0x0b,
+	.head_state4 = 0x0d,
+	.head_state5 = 0x0f,
+	.pll0 = 0x17,
+	.pll1 = 0x18,
+	.pll2 = 0x19,
+	.pll3 = 0x1a,
+	.dp_padctl0 = 0x5c,
+	.dp_padctl2 = 0x73,
+};
+
 static const struct tegra_sor_soc tegra124_sor = {
 	.supports_edp = true,
 	.supports_lvds = true,
 	.supports_hdmi = false,
 	.supports_dp = false,
+	.regs = &tegra124_sor_regs,
+	.has_nvdisplay = false,
 	.xbar_cfg = tegra124_sor_xbar_cfg,
 };
 
+static const struct tegra_sor_regs tegra210_sor_regs = {
+	.head_state0 = 0x05,
+	.head_state1 = 0x07,
+	.head_state2 = 0x09,
+	.head_state3 = 0x0b,
+	.head_state4 = 0x0d,
+	.head_state5 = 0x0f,
+	.pll0 = 0x17,
+	.pll1 = 0x18,
+	.pll2 = 0x19,
+	.pll3 = 0x1a,
+	.dp_padctl0 = 0x5c,
+	.dp_padctl2 = 0x73,
+};
+
 static const struct tegra_sor_soc tegra210_sor = {
 	.supports_edp = true,
 	.supports_lvds = false,
 	.supports_hdmi = false,
 	.supports_dp = false,
+	.regs = &tegra210_sor_regs,
+	.has_nvdisplay = false,
 	.xbar_cfg = tegra124_sor_xbar_cfg,
 };
 
@@ -2528,13 +2843,60 @@ static const struct tegra_sor_soc tegra210_sor1 = {
 	.supports_hdmi = true,
 	.supports_dp = true,
 
+	.regs = &tegra210_sor_regs,
+	.has_nvdisplay = false,
+
 	.num_settings = ARRAY_SIZE(tegra210_sor_hdmi_defaults),
 	.settings = tegra210_sor_hdmi_defaults,
 
 	.xbar_cfg = tegra210_sor_xbar_cfg,
 };
 
+static const struct tegra_sor_regs tegra186_sor_regs = {
+	.head_state0 = 0x151,
+	.head_state1 = 0x154,
+	.head_state2 = 0x157,
+	.head_state3 = 0x15a,
+	.head_state4 = 0x15d,
+	.head_state5 = 0x160,
+	.pll0 = 0x163,
+	.pll1 = 0x164,
+	.pll2 = 0x165,
+	.pll3 = 0x166,
+	.dp_padctl0 = 0x168,
+	.dp_padctl2 = 0x16a,
+};
+
+static const struct tegra_sor_soc tegra186_sor = {
+	.supports_edp = false,
+	.supports_lvds = false,
+	.supports_hdmi = false,
+	.supports_dp = true,
+
+	.regs = &tegra186_sor_regs,
+	.has_nvdisplay = true,
+
+	.xbar_cfg = tegra124_sor_xbar_cfg,
+};
+
+static const struct tegra_sor_soc tegra186_sor1 = {
+	.supports_edp = false,
+	.supports_lvds = false,
+	.supports_hdmi = true,
+	.supports_dp = true,
+
+	.regs = &tegra186_sor_regs,
+	.has_nvdisplay = true,
+
+	.num_settings = ARRAY_SIZE(tegra186_sor_hdmi_defaults),
+	.settings = tegra186_sor_hdmi_defaults,
+
+	.xbar_cfg = tegra124_sor_xbar_cfg,
+};
+
 static const struct of_device_id tegra_sor_of_match[] = {
+	{ .compatible = "nvidia,tegra186-sor1", .data = &tegra186_sor1 },
+	{ .compatible = "nvidia,tegra186-sor", .data = &tegra186_sor },
 	{ .compatible = "nvidia,tegra210-sor1", .data = &tegra210_sor1 },
 	{ .compatible = "nvidia,tegra210-sor", .data = &tegra210_sor },
 	{ .compatible = "nvidia,tegra124-sor", .data = &tegra124_sor },
@@ -2542,6 +2904,29 @@ static const struct of_device_id tegra_sor_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, tegra_sor_of_match);
 
+static int tegra_sor_parse_dt(struct tegra_sor *sor)
+{
+	struct device_node *np = sor->dev->of_node;
+	u32 value;
+	int err;
+
+	if (sor->soc->has_nvdisplay) {
+		err = of_property_read_u32(np, "nvidia,interface", &value);
+		if (err < 0)
+			return err;
+
+		sor->index = value;
+
+		/*
+		 * override the default that we already set for Tegra210 and
+		 * earlier
+		 */
+		sor->pad = TEGRA_IO_PAD_HDMI_DP0 + sor->index;
+	}
+
+	return 0;
+}
+
 static int tegra_sor_probe(struct platform_device *pdev)
 {
 	struct device_node *np;
@@ -2577,6 +2962,7 @@ static int tegra_sor_probe(struct platform_device *pdev)
 	if (!sor->aux) {
 		if (sor->soc->supports_hdmi) {
 			sor->ops = &tegra_sor_hdmi_ops;
+			sor->pad = TEGRA_IO_PAD_HDMI;
 		} else if (sor->soc->supports_lvds) {
 			dev_err(&pdev->dev, "LVDS not supported yet\n");
 			return -ENODEV;
@@ -2587,6 +2973,7 @@ static int tegra_sor_probe(struct platform_device *pdev)
 	} else {
 		if (sor->soc->supports_edp) {
 			sor->ops = &tegra_sor_edp_ops;
+			sor->pad = TEGRA_IO_PAD_LVDS;
 		} else if (sor->soc->supports_dp) {
 			dev_err(&pdev->dev, "DisplayPort not supported yet\n");
 			return -ENODEV;
@@ -2596,6 +2983,10 @@ static int tegra_sor_probe(struct platform_device *pdev)
 		}
 	}
 
+	err = tegra_sor_parse_dt(sor);
+	if (err < 0)
+		return err;
+
 	err = tegra_output_probe(&sor->output);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to probe output: %d\n", err);
diff --git a/drivers/gpu/drm/tegra/sor.h b/drivers/gpu/drm/tegra/sor.h
index 865c73b48968..fb0854d92a27 100644
--- a/drivers/gpu/drm/tegra/sor.h
+++ b/drivers/gpu/drm/tegra/sor.h
@@ -89,6 +89,8 @@
 #define SOR_PLL0 0x17
 #define  SOR_PLL0_ICHPMP_MASK			(0xf << 24)
 #define  SOR_PLL0_ICHPMP(x)			(((x) & 0xf) << 24)
+#define  SOR_PLL0_FILTER_MASK			(0xf << 16)
+#define  SOR_PLL0_FILTER(x)			(((x) & 0xf) << 16)
 #define  SOR_PLL0_VCOCAP_MASK			(0xf << 8)
 #define  SOR_PLL0_VCOCAP(x)			(((x) & 0xf) << 8)
 #define  SOR_PLL0_VCOCAP_RST			SOR_PLL0_VCOCAP(3)
@@ -122,10 +124,16 @@
 #define  SOR_PLL2_SEQ_PLL_PULLDOWN		(1 << 16)
 
 #define SOR_PLL3 0x1a
+#define  SOR_PLL3_BG_TEMP_COEF_MASK		(0xf << 28)
+#define  SOR_PLL3_BG_TEMP_COEF(x)		(((x) & 0xf) << 28)
 #define  SOR_PLL3_BG_VREF_LEVEL_MASK		(0xf << 24)
 #define  SOR_PLL3_BG_VREF_LEVEL(x)		(((x) & 0xf) << 24)
 #define  SOR_PLL3_PLL_VDD_MODE_1V8		(0 << 13)
 #define  SOR_PLL3_PLL_VDD_MODE_3V3		(1 << 13)
+#define  SOR_PLL3_AVDD10_LEVEL_MASK		(0xf << 8)
+#define  SOR_PLL3_AVDD10_LEVEL(x)		(((x) & 0xf) << 8)
+#define  SOR_PLL3_AVDD14_LEVEL_MASK		(0xf << 4)
+#define  SOR_PLL3_AVDD14_LEVEL(x)		(((x) & 0xf) << 4)
 
 #define SOR_CSTM 0x1b
 #define  SOR_CSTM_ROTCLK_MASK			(0xf << 24)
@@ -334,6 +342,10 @@
 #define SOR_DP_LQ_CSTM1 0x70
 #define SOR_DP_LQ_CSTM2 0x71
 
+#define SOR_DP_PADCTL2 0x73
+#define  SOR_DP_PADCTL_SPAREPLL_MASK (0xff << 24)
+#define  SOR_DP_PADCTL_SPAREPLL(x) (((x) & 0xff) << 24)
+
 #define SOR_HDMI_AUDIO_INFOFRAME_CTRL 0x9a
 #define SOR_HDMI_AUDIO_INFOFRAME_STATUS 0x9b
 #define SOR_HDMI_AUDIO_INFOFRAME_HEADER 0x9c
@@ -370,4 +382,8 @@
 #define SOR_HDMI_VSI_INFOFRAME_STATUS 0x124
 #define SOR_HDMI_VSI_INFOFRAME_HEADER 0x125
 
+#define SOR_HDMI2_CTRL 0x13e
+#define  SOR_HDMI2_CTRL_CLOCK_MODE_DIV_BY_4 (1 << 1)
+#define  SOR_HDMI2_CTRL_SCRAMBLE (1 << 0)
+
 #endif
diff --git a/drivers/gpu/drm/tegra/vic.c b/drivers/gpu/drm/tegra/vic.c
index 18024183aa2b..f5794dd49f3b 100644
--- a/drivers/gpu/drm/tegra/vic.c
+++ b/drivers/gpu/drm/tegra/vic.c
@@ -115,7 +115,7 @@ static int vic_boot(struct vic *vic)
 }
 
 static void *vic_falcon_alloc(struct falcon *falcon, size_t size,
-			       dma_addr_t *iova)
+			      dma_addr_t *iova)
 {
 	struct tegra_drm *tegra = falcon->data;
 
@@ -138,13 +138,14 @@ static const struct falcon_ops vic_falcon_ops = {
 static int vic_init(struct host1x_client *client)
 {
 	struct tegra_drm_client *drm = host1x_to_drm_client(client);
+	struct iommu_group *group = iommu_group_get(client->dev);
 	struct drm_device *dev = dev_get_drvdata(client->parent);
 	struct tegra_drm *tegra = dev->dev_private;
 	struct vic *vic = to_vic(drm);
 	int err;
 
-	if (tegra->domain) {
-		err = iommu_attach_device(tegra->domain, vic->dev);
+	if (group && tegra->domain) {
+		err = iommu_attach_group(tegra->domain, group);
 		if (err < 0) {
 			dev_err(vic->dev, "failed to attach to domain: %d\n",
 				err);
@@ -158,13 +159,13 @@ static int vic_init(struct host1x_client *client)
 		vic->falcon.data = tegra;
 		err = falcon_load_firmware(&vic->falcon);
 		if (err < 0)
-			goto detach_device;
+			goto detach;
 	}
 
 	vic->channel = host1x_channel_request(client->dev);
 	if (!vic->channel) {
 		err = -ENOMEM;
-		goto detach_device;
+		goto detach;
 	}
 
 	client->syncpts[0] = host1x_syncpt_request(client, 0);
@@ -183,9 +184,9 @@ static int vic_init(struct host1x_client *client)
 	host1x_syncpt_free(client->syncpts[0]);
 free_channel:
 	host1x_channel_put(vic->channel);
-detach_device:
-	if (tegra->domain)
-		iommu_detach_device(tegra->domain, vic->dev);
+detach:
+	if (group && tegra->domain)
+		iommu_detach_group(tegra->domain, group);
 
 	return err;
 }
@@ -193,6 +194,7 @@ static int vic_init(struct host1x_client *client)
 static int vic_exit(struct host1x_client *client)
 {
 	struct tegra_drm_client *drm = host1x_to_drm_client(client);
+	struct iommu_group *group = iommu_group_get(client->dev);
 	struct drm_device *dev = dev_get_drvdata(client->parent);
 	struct tegra_drm *tegra = dev->dev_private;
 	struct vic *vic = to_vic(drm);
@@ -206,7 +208,7 @@ static int vic_exit(struct host1x_client *client)
 	host1x_channel_put(vic->channel);
 
 	if (vic->domain) {
-		iommu_detach_device(vic->domain, vic->dev);
+		iommu_detach_group(vic->domain, group);
 		vic->domain = NULL;
 	}
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 6ef4d1a1e3a9..8bf6bb93dc79 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -51,12 +51,8 @@ struct tilcdc_crtc {
 	ktime_t last_vblank;
 	unsigned int hvtotal_us;
 
-	struct drm_framebuffer *curr_fb;
 	struct drm_framebuffer *next_fb;
 
-	/* for deferred fb unref's: */
-	struct drm_flip_work unref_work;
-
 	/* Only set if an external encoder is connected */
 	bool simulate_vesa_sync;
 
@@ -70,20 +66,8 @@ struct tilcdc_crtc {
 };
 #define to_tilcdc_crtc(x) container_of(x, struct tilcdc_crtc, base)
 
-static void unref_worker(struct drm_flip_work *work, void *val)
-{
-	struct tilcdc_crtc *tilcdc_crtc =
-		container_of(work, struct tilcdc_crtc, unref_work);
-	struct drm_device *dev = tilcdc_crtc->base.dev;
-
-	mutex_lock(&dev->mode_config.mutex);
-	drm_framebuffer_put(val);
-	mutex_unlock(&dev->mode_config.mutex);
-}
-
 static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 {
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	struct drm_gem_cma_object *gem;
@@ -108,12 +92,6 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 
 	dma_base_and_ceiling = (u64)end << 32 | start;
 	tilcdc_write64(dev, LCDC_DMA_FB_BASE_ADDR_0_REG, dma_base_and_ceiling);
-
-	if (tilcdc_crtc->curr_fb)
-		drm_flip_work_queue(&tilcdc_crtc->unref_work,
-			tilcdc_crtc->curr_fb);
-
-	tilcdc_crtc->curr_fb = fb;
 }
 
 /*
@@ -294,7 +272,7 @@ static void tilcdc_crtc_set_clk(struct drm_crtc *crtc)
 				LCDC_V2_CORE_CLK_EN);
 }
 
-uint tilcdc_mode_hvtotal(const struct drm_display_mode *mode)
+static uint tilcdc_mode_hvtotal(const struct drm_display_mode *mode)
 {
 	return (uint) div_u64(1000llu * mode->htotal * mode->vtotal,
 			      mode->clock);
@@ -464,8 +442,6 @@ static void tilcdc_crtc_set_mode(struct drm_crtc *crtc)
 
 	set_scanout(crtc, fb);
 
-	drm_framebuffer_get(fb);
-
 	crtc->hwmode = crtc->state->adjusted_mode;
 
 	tilcdc_crtc->hvtotal_us =
@@ -524,7 +500,6 @@ static void tilcdc_crtc_off(struct drm_crtc *crtc, bool shutdown)
 {
 	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
-	struct tilcdc_drm_private *priv = dev->dev_private;
 	int ret;
 
 	mutex_lock(&tilcdc_crtc->enable_lock);
@@ -554,20 +529,6 @@ static void tilcdc_crtc_off(struct drm_crtc *crtc, bool shutdown)
 
 	pm_runtime_put_sync(dev->dev);
 
-	if (tilcdc_crtc->next_fb) {
-		drm_flip_work_queue(&tilcdc_crtc->unref_work,
-				    tilcdc_crtc->next_fb);
-		tilcdc_crtc->next_fb = NULL;
-	}
-
-	if (tilcdc_crtc->curr_fb) {
-		drm_flip_work_queue(&tilcdc_crtc->unref_work,
-				    tilcdc_crtc->curr_fb);
-		tilcdc_crtc->curr_fb = NULL;
-	}
-
-	drm_flip_work_commit(&tilcdc_crtc->unref_work, priv->wq);
-
 	tilcdc_crtc->enabled = false;
 	mutex_unlock(&tilcdc_crtc->enable_lock);
 }
@@ -614,7 +575,6 @@ static void tilcdc_crtc_recover_work(struct work_struct *work)
 
 static void tilcdc_crtc_destroy(struct drm_crtc *crtc)
 {
-	struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
 	struct tilcdc_drm_private *priv = crtc->dev->dev_private;
 
 	tilcdc_crtc_shutdown(crtc);
@@ -623,7 +583,6 @@ static void tilcdc_crtc_destroy(struct drm_crtc *crtc)
 
 	of_node_put(crtc->port);
 	drm_crtc_cleanup(crtc);
-	drm_flip_work_cleanup(&tilcdc_crtc->unref_work);
 }
 
 int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
@@ -638,9 +597,6 @@ int tilcdc_crtc_update_fb(struct drm_crtc *crtc,
 		return -EBUSY;
 	}
 
-	drm_framebuffer_get(fb);
-
-	crtc->primary->fb = fb;
 	tilcdc_crtc->event = event;
 
 	mutex_lock(&tilcdc_crtc->enable_lock);
@@ -936,8 +892,6 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 
 		now = ktime_get();
 
-		drm_flip_work_commit(&tilcdc_crtc->unref_work, priv->wq);
-
 		spin_lock_irqsave(&tilcdc_crtc->irq_lock, flags);
 
 		tilcdc_crtc->last_vblank = now;
@@ -1064,9 +1018,6 @@ int tilcdc_crtc_create(struct drm_device *dev)
 
 	init_waitqueue_head(&tilcdc_crtc->frame_done_wq);
 
-	drm_flip_work_init(&tilcdc_crtc->unref_work,
-			"unref", unref_worker);
-
 	spin_lock_init(&tilcdc_crtc->irq_lock);
 	INIT_WORK(&tilcdc_crtc->recover_work, tilcdc_crtc_recover_work);
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index 72ce063aa0d8..1afde61f1247 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -69,12 +69,6 @@ static struct drm_framebuffer *tilcdc_fb_create(struct drm_device *dev,
 	return drm_gem_fb_create(dev, file_priv, mode_cmd);
 }
 
-static void tilcdc_fb_output_poll_changed(struct drm_device *dev)
-{
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	drm_fbdev_cma_hotplug_event(priv->fbdev);
-}
-
 static int tilcdc_atomic_check(struct drm_device *dev,
 			       struct drm_atomic_state *state)
 {
@@ -146,7 +140,7 @@ static int tilcdc_commit(struct drm_device *dev,
 
 static const struct drm_mode_config_funcs mode_config_funcs = {
 	.fb_create = tilcdc_fb_create,
-	.output_poll_changed = tilcdc_fb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = tilcdc_atomic_check,
 	.atomic_commit = tilcdc_commit,
 };
@@ -198,8 +192,7 @@ static void tilcdc_fini(struct drm_device *dev)
 
 	drm_kms_helper_poll_fini(dev);
 
-	if (priv->fbdev)
-		drm_fbdev_cma_fini(priv->fbdev);
+	drm_fb_cma_fbdev_fini(dev);
 
 	drm_irq_uninstall(dev);
 	drm_mode_config_cleanup(dev);
@@ -405,12 +398,9 @@ static int tilcdc_init(struct drm_driver *ddrv, struct device *dev)
 
 	drm_mode_config_reset(ddev);
 
-	priv->fbdev = drm_fbdev_cma_init(ddev, bpp,
-					 ddev->mode_config.num_connector);
-	if (IS_ERR(priv->fbdev)) {
-		ret = PTR_ERR(priv->fbdev);
+	ret = drm_fb_cma_fbdev_init(ddev, bpp, 0);
+	if (ret)
 		goto init_failed;
-	}
 
 	drm_kms_helper_poll_init(ddev);
 
@@ -427,12 +417,6 @@ static int tilcdc_init(struct drm_driver *ddrv, struct device *dev)
 	return ret;
 }
 
-static void tilcdc_lastclose(struct drm_device *dev)
-{
-	struct tilcdc_drm_private *priv = dev->dev_private;
-	drm_fbdev_cma_restore_mode(priv->fbdev);
-}
-
 static irqreturn_t tilcdc_irq(int irq, void *arg)
 {
 	struct drm_device *dev = arg;
@@ -507,7 +491,6 @@ static int tilcdc_mm_show(struct seq_file *m, void *arg)
 static struct drm_info_list tilcdc_debugfs_list[] = {
 		{ "regs", tilcdc_regs_show, 0 },
 		{ "mm",   tilcdc_mm_show,   0 },
-		{ "fb",   drm_fb_cma_debugfs_show, 0 },
 };
 
 static int tilcdc_debugfs_init(struct drm_minor *minor)
@@ -538,9 +521,10 @@ DEFINE_DRM_GEM_CMA_FOPS(fops);
 static struct drm_driver tilcdc_driver = {
 	.driver_features    = (DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET |
 			       DRIVER_PRIME | DRIVER_ATOMIC),
-	.lastclose          = tilcdc_lastclose,
+	.lastclose          = drm_fb_helper_lastclose,
 	.irq_handler        = tilcdc_irq,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
+	.gem_print_info     = drm_gem_cma_print_info,
 	.gem_vm_ops         = &drm_gem_cma_vm_ops,
 	.dumb_create        = drm_gem_cma_dumb_create,
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.h b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
index 8caa11bc7aec..ead512216669 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.h
@@ -79,8 +79,6 @@ struct tilcdc_drm_private {
 
 	struct workqueue_struct *wq;
 
-	struct drm_fbdev_cma *fbdev;
-
 	struct drm_crtc *crtc;
 
 	unsigned int num_encoders;
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_regs.h b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
index 9d528c0a67a4..5048ebb86835 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_regs.h
+++ b/drivers/gpu/drm/tilcdc/tilcdc_regs.h
@@ -133,7 +133,7 @@ static inline void tilcdc_write64(struct drm_device *dev, u32 reg, u64 data)
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	volatile void __iomem *addr = priv->mmio + reg;
 
-#ifdef iowrite64
+#if defined(iowrite64) && !defined(iowrite64_is_nonatomic)
 	iowrite64(data, addr);
 #else
 	__iowmb();
diff --git a/drivers/gpu/drm/tinydrm/Kconfig b/drivers/gpu/drm/tinydrm/Kconfig
index 2e790e7dced5..b0e567d416b3 100644
--- a/drivers/gpu/drm/tinydrm/Kconfig
+++ b/drivers/gpu/drm/tinydrm/Kconfig
@@ -12,6 +12,16 @@ menuconfig DRM_TINYDRM
 config TINYDRM_MIPI_DBI
 	tristate
 
+config TINYDRM_ILI9225
+	tristate "DRM support for ILI9225 display panels"
+	depends on DRM_TINYDRM && SPI
+	select TINYDRM_MIPI_DBI
+	help
+	  DRM driver for the following Ilitek ILI9225 panels:
+	  * No-name 2.2" color screen module
+
+	  If M is selected the module will be called ili9225.
+
 config TINYDRM_MI0283QT
 	tristate "DRM support for MI0283QT"
 	depends on DRM_TINYDRM && SPI
@@ -42,3 +52,13 @@ config TINYDRM_ST7586
 	  * LEGO MINDSTORMS EV3
 
 	  If M is selected the module will be called st7586.
+
+config TINYDRM_ST7735R
+	tristate "DRM support for Sitronix ST7735R display panels"
+	depends on DRM_TINYDRM && SPI
+	select TINYDRM_MIPI_DBI
+	help
+	  DRM driver Sitronix ST7735R with one of the following LCDs:
+	  * JD-T18003-T01 1.8" 128x160 TFT
+
+	  If M is selected the module will be called st7735r.
diff --git a/drivers/gpu/drm/tinydrm/Makefile b/drivers/gpu/drm/tinydrm/Makefile
index 0c184bd1bb59..49a111929724 100644
--- a/drivers/gpu/drm/tinydrm/Makefile
+++ b/drivers/gpu/drm/tinydrm/Makefile
@@ -4,6 +4,8 @@ obj-$(CONFIG_DRM_TINYDRM)		+= core/
 obj-$(CONFIG_TINYDRM_MIPI_DBI)		+= mipi-dbi.o
 
 # Displays
+obj-$(CONFIG_TINYDRM_ILI9225)		+= ili9225.o
 obj-$(CONFIG_TINYDRM_MI0283QT)		+= mi0283qt.o
 obj-$(CONFIG_TINYDRM_REPAPER)		+= repaper.o
 obj-$(CONFIG_TINYDRM_ST7586)		+= st7586.o
+obj-$(CONFIG_TINYDRM_ST7735R)		+= st7735r.o
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-core.c b/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
index 1a8a57cad431..4c6616278c48 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
@@ -10,6 +10,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/tinydrm/tinydrm.h>
 #include <linux/device.h>
@@ -35,23 +36,6 @@
  * and registers the DRM device using devm_tinydrm_register().
  */
 
-/**
- * tinydrm_lastclose - DRM lastclose helper
- * @drm: DRM device
- *
- * This function ensures that fbdev is restored when drm_lastclose() is called
- * on the last drm_release(). Drivers can use this as their
- * &drm_driver->lastclose callback.
- */
-void tinydrm_lastclose(struct drm_device *drm)
-{
-	struct tinydrm_device *tdev = drm->dev_private;
-
-	DRM_DEBUG_KMS("\n");
-	drm_fbdev_cma_restore_mode(tdev->fbdev_cma);
-}
-EXPORT_SYMBOL(tinydrm_lastclose);
-
 /**
  * tinydrm_gem_cma_prime_import_sg_table - Produce a CMA GEM object from
  *     another driver's scatter/gather table of pinned pages
@@ -214,35 +198,24 @@ EXPORT_SYMBOL(devm_tinydrm_init);
 static int tinydrm_register(struct tinydrm_device *tdev)
 {
 	struct drm_device *drm = tdev->drm;
-	int bpp = drm->mode_config.preferred_depth;
-	struct drm_fbdev_cma *fbdev;
 	int ret;
 
 	ret = drm_dev_register(tdev->drm, 0);
 	if (ret)
 		return ret;
 
-	fbdev = drm_fbdev_cma_init_with_funcs(drm, bpp ? bpp : 32,
-					      drm->mode_config.num_connector,
-					      tdev->fb_funcs);
-	if (IS_ERR(fbdev))
-		DRM_ERROR("Failed to initialize fbdev: %ld\n", PTR_ERR(fbdev));
-	else
-		tdev->fbdev_cma = fbdev;
+	ret = drm_fb_cma_fbdev_init_with_funcs(drm, 0, 0, tdev->fb_funcs);
+	if (ret)
+		DRM_ERROR("Failed to initialize fbdev: %d\n", ret);
 
 	return 0;
 }
 
 static void tinydrm_unregister(struct tinydrm_device *tdev)
 {
-	struct drm_fbdev_cma *fbdev_cma = tdev->fbdev_cma;
-
 	drm_atomic_helper_shutdown(tdev->drm);
-	/* don't restore fbdev in lastclose, keep pipeline disabled */
-	tdev->fbdev_cma = NULL;
+	drm_fb_cma_fbdev_fini(tdev->drm);
 	drm_dev_unregister(tdev->drm);
-	if (fbdev_cma)
-		drm_fbdev_cma_fini(fbdev_cma);
 }
 
 static void devm_tinydrm_register_release(void *data)
@@ -292,71 +265,4 @@ void tinydrm_shutdown(struct tinydrm_device *tdev)
 }
 EXPORT_SYMBOL(tinydrm_shutdown);
 
-/**
- * tinydrm_suspend - Suspend tinydrm
- * @tdev: tinydrm device
- *
- * Used in driver PM operations to suspend tinydrm.
- * Suspends fbdev and DRM.
- * Resume with tinydrm_resume().
- *
- * Returns:
- * Zero on success, negative error code on failure.
- */
-int tinydrm_suspend(struct tinydrm_device *tdev)
-{
-	struct drm_atomic_state *state;
-
-	if (tdev->suspend_state) {
-		DRM_ERROR("Failed to suspend: state already set\n");
-		return -EINVAL;
-	}
-
-	drm_fbdev_cma_set_suspend_unlocked(tdev->fbdev_cma, 1);
-	state = drm_atomic_helper_suspend(tdev->drm);
-	if (IS_ERR(state)) {
-		drm_fbdev_cma_set_suspend_unlocked(tdev->fbdev_cma, 0);
-		return PTR_ERR(state);
-	}
-
-	tdev->suspend_state = state;
-
-	return 0;
-}
-EXPORT_SYMBOL(tinydrm_suspend);
-
-/**
- * tinydrm_resume - Resume tinydrm
- * @tdev: tinydrm device
- *
- * Used in driver PM operations to resume tinydrm.
- * Suspend with tinydrm_suspend().
- *
- * Returns:
- * Zero on success, negative error code on failure.
- */
-int tinydrm_resume(struct tinydrm_device *tdev)
-{
-	struct drm_atomic_state *state = tdev->suspend_state;
-	int ret;
-
-	if (!state) {
-		DRM_ERROR("Failed to resume: state is not set\n");
-		return -EINVAL;
-	}
-
-	tdev->suspend_state = NULL;
-
-	ret = drm_atomic_helper_resume(tdev->drm, state);
-	if (ret) {
-		DRM_ERROR("Error resuming state: %d\n", ret);
-		return ret;
-	}
-
-	drm_fbdev_cma_set_suspend_unlocked(tdev->fbdev_cma, 0);
-
-	return 0;
-}
-EXPORT_SYMBOL(tinydrm_resume);
-
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
index bd6cce093a85..bf96072d1b97 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
@@ -414,11 +414,9 @@ tinydrm_dbg_spi_print(struct spi_device *spi, struct spi_transfer *tr,
 void _tinydrm_dbg_spi_message(struct spi_device *spi, struct spi_message *m)
 {
 	struct spi_transfer *tmp;
-	struct list_head *pos;
 	int i = 0;
 
-	list_for_each(pos, &m->transfers) {
-		tmp = list_entry(pos, struct spi_transfer, transfer_list);
+	list_for_each_entry(tmp, &m->transfers, transfer_list) {
 
 		if (tmp->tx_buf)
 			tinydrm_dbg_spi_print(spi, tmp, tmp->tx_buf, i, true);
diff --git a/drivers/gpu/drm/tinydrm/ili9225.c b/drivers/gpu/drm/tinydrm/ili9225.c
new file mode 100644
index 000000000000..c0cf49849302
--- /dev/null
+++ b/drivers/gpu/drm/tinydrm/ili9225.c
@@ -0,0 +1,469 @@
+/*
+ * DRM driver for Ilitek ILI9225 panels
+ *
+ * Copyright 2017 David Lechner <david@lechnology.com>
+ *
+ * Some code copied from mipi-dbi.c
+ * Copyright 2016 Noralf Trønnes
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-buf.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/spi/spi.h>
+#include <video/mipi_display.h>
+
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/tinydrm/mipi-dbi.h>
+#include <drm/tinydrm/tinydrm-helpers.h>
+
+#define ILI9225_DRIVER_READ_CODE	0x00
+#define ILI9225_DRIVER_OUTPUT_CONTROL	0x01
+#define ILI9225_LCD_AC_DRIVING_CONTROL	0x02
+#define ILI9225_ENTRY_MODE		0x03
+#define ILI9225_DISPLAY_CONTROL_1	0x07
+#define ILI9225_BLANK_PERIOD_CONTROL_1	0x08
+#define ILI9225_FRAME_CYCLE_CONTROL	0x0b
+#define ILI9225_INTERFACE_CONTROL	0x0c
+#define ILI9225_OSCILLATION_CONTROL	0x0f
+#define ILI9225_POWER_CONTROL_1		0x10
+#define ILI9225_POWER_CONTROL_2		0x11
+#define ILI9225_POWER_CONTROL_3		0x12
+#define ILI9225_POWER_CONTROL_4		0x13
+#define ILI9225_POWER_CONTROL_5		0x14
+#define ILI9225_VCI_RECYCLING		0x15
+#define ILI9225_RAM_ADDRESS_SET_1	0x20
+#define ILI9225_RAM_ADDRESS_SET_2	0x21
+#define ILI9225_WRITE_DATA_TO_GRAM	0x22
+#define ILI9225_SOFTWARE_RESET		0x28
+#define ILI9225_GATE_SCAN_CONTROL	0x30
+#define ILI9225_VERTICAL_SCROLL_1	0x31
+#define ILI9225_VERTICAL_SCROLL_2	0x32
+#define ILI9225_VERTICAL_SCROLL_3	0x33
+#define ILI9225_PARTIAL_DRIVING_POS_1	0x34
+#define ILI9225_PARTIAL_DRIVING_POS_2	0x35
+#define ILI9225_HORIZ_WINDOW_ADDR_1	0x36
+#define ILI9225_HORIZ_WINDOW_ADDR_2	0x37
+#define ILI9225_VERT_WINDOW_ADDR_1	0x38
+#define ILI9225_VERT_WINDOW_ADDR_2	0x39
+#define ILI9225_GAMMA_CONTROL_1		0x50
+#define ILI9225_GAMMA_CONTROL_2		0x51
+#define ILI9225_GAMMA_CONTROL_3		0x52
+#define ILI9225_GAMMA_CONTROL_4		0x53
+#define ILI9225_GAMMA_CONTROL_5		0x54
+#define ILI9225_GAMMA_CONTROL_6		0x55
+#define ILI9225_GAMMA_CONTROL_7		0x56
+#define ILI9225_GAMMA_CONTROL_8		0x57
+#define ILI9225_GAMMA_CONTROL_9		0x58
+#define ILI9225_GAMMA_CONTROL_10	0x59
+
+static inline int ili9225_command(struct mipi_dbi *mipi, u8 cmd, u16 data)
+{
+	u8 par[2] = { data >> 8, data & 0xff };
+
+	return mipi_dbi_command_buf(mipi, cmd, par, 2);
+}
+
+static int ili9225_fb_dirty(struct drm_framebuffer *fb,
+			    struct drm_file *file_priv, unsigned int flags,
+			    unsigned int color, struct drm_clip_rect *clips,
+			    unsigned int num_clips)
+{
+	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	struct tinydrm_device *tdev = fb->dev->dev_private;
+	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	bool swap = mipi->swap_bytes;
+	struct drm_clip_rect clip;
+	u16 x_start, y_start;
+	u16 x1, x2, y1, y2;
+	int ret = 0;
+	bool full;
+	void *tr;
+
+	mutex_lock(&tdev->dirty_lock);
+
+	if (!mipi->enabled)
+		goto out_unlock;
+
+	/* fbdev can flush even when we're not interested */
+	if (tdev->pipe.plane.fb != fb)
+		goto out_unlock;
+
+	full = tinydrm_merge_clips(&clip, clips, num_clips, flags,
+				   fb->width, fb->height);
+
+	DRM_DEBUG("Flushing [FB:%d] x1=%u, x2=%u, y1=%u, y2=%u\n", fb->base.id,
+		  clip.x1, clip.x2, clip.y1, clip.y2);
+
+	if (!mipi->dc || !full || swap ||
+	    fb->format->format == DRM_FORMAT_XRGB8888) {
+		tr = mipi->tx_buf;
+		ret = mipi_dbi_buf_copy(mipi->tx_buf, fb, &clip, swap);
+		if (ret)
+			goto out_unlock;
+	} else {
+		tr = cma_obj->vaddr;
+	}
+
+	switch (mipi->rotation) {
+	default:
+		x1 = clip.x1;
+		x2 = clip.x2 - 1;
+		y1 = clip.y1;
+		y2 = clip.y2 - 1;
+		x_start = x1;
+		y_start = y1;
+		break;
+	case 90:
+		x1 = clip.y1;
+		x2 = clip.y2 - 1;
+		y1 = fb->width - clip.x2;
+		y2 = fb->width - clip.x1 - 1;
+		x_start = x1;
+		y_start = y2;
+		break;
+	case 180:
+		x1 = fb->width - clip.x2;
+		x2 = fb->width - clip.x1 - 1;
+		y1 = fb->height - clip.y2;
+		y2 = fb->height - clip.y1 - 1;
+		x_start = x2;
+		y_start = y2;
+		break;
+	case 270:
+		x1 = fb->height - clip.y2;
+		x2 = fb->height - clip.y1 - 1;
+		y1 = clip.x1;
+		y2 = clip.x2 - 1;
+		x_start = x2;
+		y_start = y1;
+		break;
+	}
+
+	ili9225_command(mipi, ILI9225_HORIZ_WINDOW_ADDR_1, x2);
+	ili9225_command(mipi, ILI9225_HORIZ_WINDOW_ADDR_2, x1);
+	ili9225_command(mipi, ILI9225_VERT_WINDOW_ADDR_1, y2);
+	ili9225_command(mipi, ILI9225_VERT_WINDOW_ADDR_2, y1);
+
+	ili9225_command(mipi, ILI9225_RAM_ADDRESS_SET_1, x_start);
+	ili9225_command(mipi, ILI9225_RAM_ADDRESS_SET_2, y_start);
+
+	ret = mipi_dbi_command_buf(mipi, ILI9225_WRITE_DATA_TO_GRAM, tr,
+				(clip.x2 - clip.x1) * (clip.y2 - clip.y1) * 2);
+
+out_unlock:
+	mutex_unlock(&tdev->dirty_lock);
+
+	if (ret)
+		dev_err_once(fb->dev->dev, "Failed to update display %d\n",
+			     ret);
+
+	return ret;
+}
+
+static const struct drm_framebuffer_funcs ili9225_fb_funcs = {
+	.destroy	= drm_gem_fb_destroy,
+	.create_handle	= drm_gem_fb_create_handle,
+	.dirty		= ili9225_fb_dirty,
+};
+
+static void ili9225_pipe_enable(struct drm_simple_display_pipe *pipe,
+				struct drm_crtc_state *crtc_state)
+{
+	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
+	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct drm_framebuffer *fb = pipe->plane.fb;
+	struct device *dev = tdev->drm->dev;
+	int ret;
+	u8 am_id;
+
+	DRM_DEBUG_KMS("\n");
+
+	mipi_dbi_hw_reset(mipi);
+
+	/*
+	 * There don't seem to be two example init sequences that match, so
+	 * using the one from the popular Arduino library for this display.
+	 * https://github.com/Nkawu/TFT_22_ILI9225/blob/master/src/TFT_22_ILI9225.cpp
+	 */
+
+	ret = ili9225_command(mipi, ILI9225_POWER_CONTROL_1, 0x0000);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "Error sending command %d\n", ret);
+		return;
+	}
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_2, 0x0000);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_3, 0x0000);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_4, 0x0000);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_5, 0x0000);
+
+	msleep(40);
+
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_2, 0x0018);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_3, 0x6121);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_4, 0x006f);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_5, 0x495f);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_1, 0x0800);
+
+	msleep(10);
+
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_2, 0x103b);
+
+	msleep(50);
+
+	switch (mipi->rotation) {
+	default:
+		am_id = 0x30;
+		break;
+	case 90:
+		am_id = 0x18;
+		break;
+	case 180:
+		am_id = 0x00;
+		break;
+	case 270:
+		am_id = 0x28;
+		break;
+	}
+	ili9225_command(mipi, ILI9225_DRIVER_OUTPUT_CONTROL, 0x011c);
+	ili9225_command(mipi, ILI9225_LCD_AC_DRIVING_CONTROL, 0x0100);
+	ili9225_command(mipi, ILI9225_ENTRY_MODE, 0x1000 | am_id);
+	ili9225_command(mipi, ILI9225_DISPLAY_CONTROL_1, 0x0000);
+	ili9225_command(mipi, ILI9225_BLANK_PERIOD_CONTROL_1, 0x0808);
+	ili9225_command(mipi, ILI9225_FRAME_CYCLE_CONTROL, 0x1100);
+	ili9225_command(mipi, ILI9225_INTERFACE_CONTROL, 0x0000);
+	ili9225_command(mipi, ILI9225_OSCILLATION_CONTROL, 0x0d01);
+	ili9225_command(mipi, ILI9225_VCI_RECYCLING, 0x0020);
+	ili9225_command(mipi, ILI9225_RAM_ADDRESS_SET_1, 0x0000);
+	ili9225_command(mipi, ILI9225_RAM_ADDRESS_SET_2, 0x0000);
+
+	ili9225_command(mipi, ILI9225_GATE_SCAN_CONTROL, 0x0000);
+	ili9225_command(mipi, ILI9225_VERTICAL_SCROLL_1, 0x00db);
+	ili9225_command(mipi, ILI9225_VERTICAL_SCROLL_2, 0x0000);
+	ili9225_command(mipi, ILI9225_VERTICAL_SCROLL_3, 0x0000);
+	ili9225_command(mipi, ILI9225_PARTIAL_DRIVING_POS_1, 0x00db);
+	ili9225_command(mipi, ILI9225_PARTIAL_DRIVING_POS_2, 0x0000);
+
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_1, 0x0000);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_2, 0x0808);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_3, 0x080a);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_4, 0x000a);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_5, 0x0a08);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_6, 0x0808);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_7, 0x0000);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_8, 0x0a00);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_9, 0x0710);
+	ili9225_command(mipi, ILI9225_GAMMA_CONTROL_10, 0x0710);
+
+	ili9225_command(mipi, ILI9225_DISPLAY_CONTROL_1, 0x0012);
+
+	msleep(50);
+
+	ili9225_command(mipi, ILI9225_DISPLAY_CONTROL_1, 0x1017);
+
+	mipi->enabled = true;
+
+	if (fb)
+		fb->funcs->dirty(fb, NULL, 0, 0, NULL, 0);
+}
+
+static void ili9225_pipe_disable(struct drm_simple_display_pipe *pipe)
+{
+	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
+	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+
+	DRM_DEBUG_KMS("\n");
+
+	if (!mipi->enabled)
+		return;
+
+	ili9225_command(mipi, ILI9225_DISPLAY_CONTROL_1, 0x0000);
+	msleep(50);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_2, 0x0007);
+	msleep(50);
+	ili9225_command(mipi, ILI9225_POWER_CONTROL_1, 0x0a02);
+
+	mipi->enabled = false;
+}
+
+static int ili9225_dbi_command(struct mipi_dbi *mipi, u8 cmd, u8 *par,
+			       size_t num)
+{
+	struct spi_device *spi = mipi->spi;
+	unsigned int bpw = 8;
+	u32 speed_hz;
+	int ret;
+
+	gpiod_set_value_cansleep(mipi->dc, 0);
+	speed_hz = mipi_dbi_spi_cmd_max_speed(spi, 1);
+	ret = tinydrm_spi_transfer(spi, speed_hz, NULL, 8, &cmd, 1);
+	if (ret || !num)
+		return ret;
+
+	if (cmd == ILI9225_WRITE_DATA_TO_GRAM && !mipi->swap_bytes)
+		bpw = 16;
+
+	gpiod_set_value_cansleep(mipi->dc, 1);
+	speed_hz = mipi_dbi_spi_cmd_max_speed(spi, num);
+
+	return tinydrm_spi_transfer(spi, speed_hz, NULL, bpw, par, num);
+}
+
+static const u32 ili9225_formats[] = {
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888,
+};
+
+static int ili9225_init(struct device *dev, struct mipi_dbi *mipi,
+			const struct drm_simple_display_pipe_funcs *pipe_funcs,
+			struct drm_driver *driver,
+			const struct drm_display_mode *mode,
+			unsigned int rotation)
+{
+	size_t bufsize = mode->vdisplay * mode->hdisplay * sizeof(u16);
+	struct tinydrm_device *tdev = &mipi->tinydrm;
+	int ret;
+
+	if (!mipi->command)
+		return -EINVAL;
+
+	mutex_init(&mipi->cmdlock);
+
+	mipi->tx_buf = devm_kmalloc(dev, bufsize, GFP_KERNEL);
+	if (!mipi->tx_buf)
+		return -ENOMEM;
+
+	ret = devm_tinydrm_init(dev, tdev, &ili9225_fb_funcs, driver);
+	if (ret)
+		return ret;
+
+	ret = tinydrm_display_pipe_init(tdev, pipe_funcs,
+					DRM_MODE_CONNECTOR_VIRTUAL,
+					ili9225_formats,
+					ARRAY_SIZE(ili9225_formats), mode,
+					rotation);
+	if (ret)
+		return ret;
+
+	tdev->drm->mode_config.preferred_depth = 16;
+	mipi->rotation = rotation;
+
+	drm_mode_config_reset(tdev->drm);
+
+	DRM_DEBUG_KMS("preferred_depth=%u, rotation = %u\n",
+		      tdev->drm->mode_config.preferred_depth, rotation);
+
+	return 0;
+}
+
+static const struct drm_simple_display_pipe_funcs ili9225_pipe_funcs = {
+	.enable		= ili9225_pipe_enable,
+	.disable	= ili9225_pipe_disable,
+	.update		= tinydrm_display_pipe_update,
+	.prepare_fb	= tinydrm_display_pipe_prepare_fb,
+};
+
+static const struct drm_display_mode ili9225_mode = {
+	TINYDRM_MODE(176, 220, 35, 44),
+};
+
+DEFINE_DRM_GEM_CMA_FOPS(ili9225_fops);
+
+static struct drm_driver ili9225_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
+				  DRIVER_ATOMIC,
+	.fops			= &ili9225_fops,
+	TINYDRM_GEM_DRIVER_OPS,
+	.lastclose		= drm_fb_helper_lastclose,
+	.name			= "ili9225",
+	.desc			= "Ilitek ILI9225",
+	.date			= "20171106",
+	.major			= 1,
+	.minor			= 0,
+};
+
+static const struct of_device_id ili9225_of_match[] = {
+	{ .compatible = "vot,v220hf01a-t" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ili9225_of_match);
+
+static const struct spi_device_id ili9225_id[] = {
+	{ "v220hf01a-t", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(spi, ili9225_id);
+
+static int ili9225_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct mipi_dbi *mipi;
+	struct gpio_desc *rs;
+	u32 rotation = 0;
+	int ret;
+
+	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	if (!mipi)
+		return -ENOMEM;
+
+	mipi->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(mipi->reset)) {
+		DRM_DEV_ERROR(dev, "Failed to get gpio 'reset'\n");
+		return PTR_ERR(mipi->reset);
+	}
+
+	rs = devm_gpiod_get(dev, "rs", GPIOD_OUT_LOW);
+	if (IS_ERR(rs)) {
+		DRM_DEV_ERROR(dev, "Failed to get gpio 'rs'\n");
+		return PTR_ERR(rs);
+	}
+
+	device_property_read_u32(dev, "rotation", &rotation);
+
+	ret = mipi_dbi_spi_init(spi, mipi, rs);
+	if (ret)
+		return ret;
+
+	/* override the command function set in  mipi_dbi_spi_init() */
+	mipi->command = ili9225_dbi_command;
+
+	ret = ili9225_init(&spi->dev, mipi, &ili9225_pipe_funcs,
+			   &ili9225_driver, &ili9225_mode, rotation);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, mipi);
+
+	return devm_tinydrm_register(&mipi->tinydrm);
+}
+
+static void ili9225_shutdown(struct spi_device *spi)
+{
+	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+
+	tinydrm_shutdown(&mipi->tinydrm);
+}
+
+static struct spi_driver ili9225_spi_driver = {
+	.driver = {
+		.name = "ili9225",
+		.owner = THIS_MODULE,
+		.of_match_table = ili9225_of_match,
+	},
+	.id_table = ili9225_id,
+	.probe = ili9225_probe,
+	.shutdown = ili9225_shutdown,
+};
+module_spi_driver(ili9225_spi_driver);
+
+MODULE_DESCRIPTION("Ilitek ILI9225 DRM driver");
+MODULE_AUTHOR("David Lechner <david@lechnology.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tinydrm/mi0283qt.c b/drivers/gpu/drm/tinydrm/mi0283qt.c
index 6a83b3093254..674d407640be 100644
--- a/drivers/gpu/drm/tinydrm/mi0283qt.c
+++ b/drivers/gpu/drm/tinydrm/mi0283qt.c
@@ -9,6 +9,8 @@
  * (at your option) any later version.
  */
 
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/tinydrm/ili9341.h>
 #include <drm/tinydrm/mipi-dbi.h>
 #include <drm/tinydrm/tinydrm-helpers.h>
@@ -139,7 +141,7 @@ static struct drm_driver mi0283qt_driver = {
 				  DRIVER_ATOMIC,
 	.fops			= &mi0283qt_fops,
 	TINYDRM_GEM_DRIVER_OPS,
-	.lastclose		= tinydrm_lastclose,
+	.lastclose		= drm_fb_helper_lastclose,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "mi0283qt",
 	.desc			= "Multi-Inno MI0283QT",
@@ -231,7 +233,7 @@ static int __maybe_unused mi0283qt_pm_suspend(struct device *dev)
 	struct mipi_dbi *mipi = dev_get_drvdata(dev);
 	int ret;
 
-	ret = tinydrm_suspend(&mipi->tinydrm);
+	ret = drm_mode_config_helper_suspend(mipi->tinydrm.drm);
 	if (ret)
 		return ret;
 
@@ -249,7 +251,9 @@ static int __maybe_unused mi0283qt_pm_resume(struct device *dev)
 	if (ret)
 		return ret;
 
-	return tinydrm_resume(&mipi->tinydrm);
+	drm_mode_config_helper_resume(mipi->tinydrm.drm);
+
+	return 0;
 }
 
 static const struct dev_pm_ops mi0283qt_pm_ops = {
diff --git a/drivers/gpu/drm/tinydrm/mipi-dbi.c b/drivers/gpu/drm/tinydrm/mipi-dbi.c
index d43e992ab432..aa6b6ce56891 100644
--- a/drivers/gpu/drm/tinydrm/mipi-dbi.c
+++ b/drivers/gpu/drm/tinydrm/mipi-dbi.c
@@ -154,8 +154,18 @@ int mipi_dbi_command_buf(struct mipi_dbi *mipi, u8 cmd, u8 *data, size_t len)
 }
 EXPORT_SYMBOL(mipi_dbi_command_buf);
 
-static int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
-				struct drm_clip_rect *clip, bool swap)
+/**
+ * mipi_dbi_buf_copy - Copy a framebuffer, transforming it if necessary
+ * @dst: The destination buffer
+ * @fb: The source framebuffer
+ * @clip: Clipping rectangle of the area to be copied
+ * @swap: When true, swap MSB/LSB of 16-bit values
+ *
+ * Returns:
+ * Zero on success, negative error code on failure.
+ */
+int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
+		      struct drm_clip_rect *clip, bool swap)
 {
 	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
 	struct dma_buf_attachment *import_attach = cma_obj->base.import_attach;
@@ -192,6 +202,7 @@ static int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
 					     DMA_FROM_DEVICE);
 	return ret;
 }
+EXPORT_SYMBOL(mipi_dbi_buf_copy);
 
 static int mipi_dbi_fb_dirty(struct drm_framebuffer *fb,
 			     struct drm_file *file_priv,
@@ -444,18 +455,23 @@ EXPORT_SYMBOL(mipi_dbi_display_is_on);
 
 #if IS_ENABLED(CONFIG_SPI)
 
-/*
+/**
+ * mipi_dbi_spi_cmd_max_speed - get the maximum SPI bus speed
+ * @spi: SPI device
+ * @len: The transfer buffer length.
+ *
  * Many controllers have a max speed of 10MHz, but can be pushed way beyond
  * that. Increase reliability by running pixel data at max speed and the rest
  * at 10MHz, preventing transfer glitches from messing up the init settings.
  */
-static u32 mipi_dbi_spi_cmd_max_speed(struct spi_device *spi, size_t len)
+u32 mipi_dbi_spi_cmd_max_speed(struct spi_device *spi, size_t len)
 {
 	if (len > 64)
 		return 0; /* use default */
 
 	return min_t(u32, 10000000, spi->max_speed_hz);
 }
+EXPORT_SYMBOL(mipi_dbi_spi_cmd_max_speed);
 
 /*
  * MIPI DBI Type C Option 1
@@ -961,10 +977,6 @@ static const struct file_operations mipi_dbi_debugfs_command_fops = {
 	.write = mipi_dbi_debugfs_command_write,
 };
 
-static const struct drm_info_list mipi_dbi_debugfs_list[] = {
-	{ "fb",   drm_fb_cma_debugfs_show, 0 },
-};
-
 /**
  * mipi_dbi_debugfs_init - Create debugfs entries
  * @minor: DRM minor
@@ -987,9 +999,7 @@ int mipi_dbi_debugfs_init(struct drm_minor *minor)
 	debugfs_create_file("command", mode, minor->debugfs_root, mipi,
 			    &mipi_dbi_debugfs_command_fops);
 
-	return drm_debugfs_create_files(mipi_dbi_debugfs_list,
-					ARRAY_SIZE(mipi_dbi_debugfs_list),
-					minor->debugfs_root, minor);
+	return 0;
 }
 EXPORT_SYMBOL(mipi_dbi_debugfs_init);
 
diff --git a/drivers/gpu/drm/tinydrm/st7586.c b/drivers/gpu/drm/tinydrm/st7586.c
index 0a2c60da5c0e..5aebfceb740e 100644
--- a/drivers/gpu/drm/tinydrm/st7586.c
+++ b/drivers/gpu/drm/tinydrm/st7586.c
@@ -17,6 +17,7 @@
 #include <linux/spi/spi.h>
 #include <video/mipi_display.h>
 
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/tinydrm/mipi-dbi.h>
 #include <drm/tinydrm/tinydrm-helpers.h>
@@ -320,7 +321,7 @@ static struct drm_driver st7586_driver = {
 				  DRIVER_ATOMIC,
 	.fops			= &st7586_fops,
 	TINYDRM_GEM_DRIVER_OPS,
-	.lastclose		= tinydrm_lastclose,
+	.lastclose		= drm_fb_helper_lastclose,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "st7586",
 	.desc			= "Sitronix ST7586",
diff --git a/drivers/gpu/drm/tinydrm/st7735r.c b/drivers/gpu/drm/tinydrm/st7735r.c
new file mode 100644
index 000000000000..98ff447f40b4
--- /dev/null
+++ b/drivers/gpu/drm/tinydrm/st7735r.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * DRM driver for Sitronix ST7735R panels
+ *
+ * Copyright 2017 David Lechner <david@lechnology.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-buf.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/spi/spi.h>
+#include <video/mipi_display.h>
+
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/tinydrm/mipi-dbi.h>
+#include <drm/tinydrm/tinydrm-helpers.h>
+
+#define ST7735R_FRMCTR1		0xb1
+#define ST7735R_FRMCTR2		0xb2
+#define ST7735R_FRMCTR3		0xb3
+#define ST7735R_INVCTR		0xb4
+#define ST7735R_PWCTR1		0xc0
+#define ST7735R_PWCTR2		0xc1
+#define ST7735R_PWCTR3		0xc2
+#define ST7735R_PWCTR4		0xc3
+#define ST7735R_PWCTR5		0xc4
+#define ST7735R_VMCTR1		0xc5
+#define ST7735R_GAMCTRP1	0xe0
+#define ST7735R_GAMCTRN1	0xe1
+
+#define ST7735R_MY	BIT(7)
+#define ST7735R_MX	BIT(6)
+#define ST7735R_MV	BIT(5)
+
+static void jd_t18003_t01_pipe_enable(struct drm_simple_display_pipe *pipe,
+				      struct drm_crtc_state *crtc_state)
+{
+	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
+	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct device *dev = tdev->drm->dev;
+	int ret;
+	u8 addr_mode;
+
+	DRM_DEBUG_KMS("\n");
+
+	mipi_dbi_hw_reset(mipi);
+
+	ret = mipi_dbi_command(mipi, MIPI_DCS_SOFT_RESET);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "Error sending command %d\n", ret);
+		return;
+	}
+
+	msleep(150);
+
+	mipi_dbi_command(mipi, MIPI_DCS_EXIT_SLEEP_MODE);
+	msleep(500);
+
+	mipi_dbi_command(mipi, ST7735R_FRMCTR1, 0x01, 0x2c, 0x2d);
+	mipi_dbi_command(mipi, ST7735R_FRMCTR2, 0x01, 0x2c, 0x2d);
+	mipi_dbi_command(mipi, ST7735R_FRMCTR3, 0x01, 0x2c, 0x2d, 0x01, 0x2c,
+			 0x2d);
+	mipi_dbi_command(mipi, ST7735R_INVCTR, 0x07);
+	mipi_dbi_command(mipi, ST7735R_PWCTR1, 0xa2, 0x02, 0x84);
+	mipi_dbi_command(mipi, ST7735R_PWCTR2, 0xc5);
+	mipi_dbi_command(mipi, ST7735R_PWCTR3, 0x0a, 0x00);
+	mipi_dbi_command(mipi, ST7735R_PWCTR4, 0x8a, 0x2a);
+	mipi_dbi_command(mipi, ST7735R_PWCTR5, 0x8a, 0xee);
+	mipi_dbi_command(mipi, ST7735R_VMCTR1, 0x0e);
+	mipi_dbi_command(mipi, MIPI_DCS_EXIT_INVERT_MODE);
+	switch (mipi->rotation) {
+	default:
+		addr_mode = ST7735R_MX | ST7735R_MY;
+		break;
+	case 90:
+		addr_mode = ST7735R_MX | ST7735R_MV;
+		break;
+	case 180:
+		addr_mode = 0;
+		break;
+	case 270:
+		addr_mode = ST7735R_MY | ST7735R_MV;
+		break;
+	}
+	mipi_dbi_command(mipi, MIPI_DCS_SET_ADDRESS_MODE, addr_mode);
+	mipi_dbi_command(mipi, MIPI_DCS_SET_PIXEL_FORMAT,
+			 MIPI_DCS_PIXEL_FMT_16BIT);
+	mipi_dbi_command(mipi, ST7735R_GAMCTRP1, 0x02, 0x1c, 0x07, 0x12, 0x37,
+			 0x32, 0x29, 0x2d, 0x29, 0x25, 0x2b, 0x39, 0x00, 0x01,
+			 0x03, 0x10);
+	mipi_dbi_command(mipi, ST7735R_GAMCTRN1, 0x03, 0x1d, 0x07, 0x06, 0x2e,
+			 0x2c, 0x29, 0x2d, 0x2e, 0x2e, 0x37, 0x3f, 0x00, 0x00,
+			 0x02, 0x10);
+	mipi_dbi_command(mipi, MIPI_DCS_SET_DISPLAY_ON);
+
+	msleep(100);
+
+	mipi_dbi_command(mipi, MIPI_DCS_ENTER_NORMAL_MODE);
+
+	msleep(20);
+
+	mipi_dbi_pipe_enable(pipe, crtc_state);
+}
+
+static const struct drm_simple_display_pipe_funcs jd_t18003_t01_pipe_funcs = {
+	.enable		= jd_t18003_t01_pipe_enable,
+	.disable	= mipi_dbi_pipe_disable,
+	.update		= tinydrm_display_pipe_update,
+	.prepare_fb	= tinydrm_display_pipe_prepare_fb,
+};
+
+static const struct drm_display_mode jd_t18003_t01_mode = {
+	TINYDRM_MODE(128, 160, 28, 35),
+};
+
+DEFINE_DRM_GEM_CMA_FOPS(st7735r_fops);
+
+static struct drm_driver st7735r_driver = {
+	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
+				  DRIVER_ATOMIC,
+	.fops			= &st7735r_fops,
+	TINYDRM_GEM_DRIVER_OPS,
+	.lastclose		= drm_fb_helper_lastclose,
+	.debugfs_init		= mipi_dbi_debugfs_init,
+	.name			= "st7735r",
+	.desc			= "Sitronix ST7735R",
+	.date			= "20171128",
+	.major			= 1,
+	.minor			= 0,
+};
+
+static const struct of_device_id st7735r_of_match[] = {
+	{ .compatible = "jianda,jd-t18003-t01" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, st7735r_of_match);
+
+static const struct spi_device_id st7735r_id[] = {
+	{ "jd-t18003-t01", 0 },
+	{ },
+};
+MODULE_DEVICE_TABLE(spi, st7735r_id);
+
+static int st7735r_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct mipi_dbi *mipi;
+	struct gpio_desc *dc;
+	u32 rotation = 0;
+	int ret;
+
+	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	if (!mipi)
+		return -ENOMEM;
+
+	mipi->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(mipi->reset)) {
+		DRM_DEV_ERROR(dev, "Failed to get gpio 'reset'\n");
+		return PTR_ERR(mipi->reset);
+	}
+
+	dc = devm_gpiod_get(dev, "dc", GPIOD_OUT_LOW);
+	if (IS_ERR(dc)) {
+		DRM_DEV_ERROR(dev, "Failed to get gpio 'dc'\n");
+		return PTR_ERR(dc);
+	}
+
+	mipi->backlight = tinydrm_of_find_backlight(dev);
+	if (IS_ERR(mipi->backlight))
+		return PTR_ERR(mipi->backlight);
+
+	device_property_read_u32(dev, "rotation", &rotation);
+
+	ret = mipi_dbi_spi_init(spi, mipi, dc);
+	if (ret)
+		return ret;
+
+	/* Cannot read from Adafruit 1.8" display via SPI */
+	mipi->read_commands = NULL;
+
+	ret = mipi_dbi_init(&spi->dev, mipi, &jd_t18003_t01_pipe_funcs,
+			    &st7735r_driver, &jd_t18003_t01_mode, rotation);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, mipi);
+
+	return devm_tinydrm_register(&mipi->tinydrm);
+}
+
+static void st7735r_shutdown(struct spi_device *spi)
+{
+	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+
+	tinydrm_shutdown(&mipi->tinydrm);
+}
+
+static struct spi_driver st7735r_spi_driver = {
+	.driver = {
+		.name = "st7735r",
+		.owner = THIS_MODULE,
+		.of_match_table = st7735r_of_match,
+	},
+	.id_table = st7735r_id,
+	.probe = st7735r_probe,
+	.shutdown = st7735r_shutdown,
+};
+module_spi_driver(st7735r_spi_driver);
+
+MODULE_DESCRIPTION("Sitronix ST7735R DRM driver");
+MODULE_AUTHOR("David Lechner <david@lechnology.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c
index 028ab6007873..3e795a099d06 100644
--- a/drivers/gpu/drm/ttm/ttm_agp_backend.c
+++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c
@@ -133,12 +133,12 @@ struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev,
 }
 EXPORT_SYMBOL(ttm_agp_tt_create);
 
-int ttm_agp_tt_populate(struct ttm_tt *ttm)
+int ttm_agp_tt_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
 {
 	if (ttm->state != tt_unpopulated)
 		return 0;
 
-	return ttm_pool_populate(ttm);
+	return ttm_pool_populate(ttm, ctx);
 }
 EXPORT_SYMBOL(ttm_agp_tt_populate);
 
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index c088703777e2..2fef09a56d16 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -42,11 +42,6 @@
 #include <linux/atomic.h>
 #include <linux/reservation.h>
 
-#define TTM_ASSERT_LOCKED(param)
-#define TTM_DEBUG(fmt, arg...)
-#define TTM_BO_HASH_ORDER 13
-
-static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
 static void ttm_bo_global_kobj_release(struct kobject *kobj);
 
 static struct attribute ttm_bo_count = {
@@ -165,7 +160,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_mem_type_manager *man;
 
-	lockdep_assert_held(&bo->resv->lock.base);
+	reservation_object_assert_held(bo->resv);
 
 	if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 
@@ -175,7 +170,8 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 		list_add_tail(&bo->lru, &man->lru[bo->priority]);
 		kref_get(&bo->list_kref);
 
-		if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {
+		if (bo->ttm && !(bo->ttm->page_flags &
+				 (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) {
 			list_add_tail(&bo->swap,
 				      &bo->glob->swap_lru[bo->priority]);
 			kref_get(&bo->list_kref);
@@ -216,7 +212,7 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
 
 void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
 {
-	lockdep_assert_held(&bo->resv->lock.base);
+	reservation_object_assert_held(bo->resv);
 
 	ttm_bo_del_from_lru(bo);
 	ttm_bo_add_to_lru(bo);
@@ -233,7 +229,7 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
 	int ret = 0;
 	uint32_t page_flags = 0;
 
-	TTM_ASSERT_LOCKED(&bo->mutex);
+	reservation_object_assert_held(bo->resv);
 	bo->ttm = NULL;
 
 	if (bdev->need_dma32)
@@ -269,9 +265,8 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
 }
 
 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
-				  struct ttm_mem_reg *mem,
-				  bool evict, bool interruptible,
-				  bool no_wait_gpu)
+				  struct ttm_mem_reg *mem, bool evict,
+				  struct ttm_operation_ctx *ctx)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
 	bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
@@ -306,7 +301,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 			goto out_err;
 
 		if (mem->mem_type != TTM_PL_SYSTEM) {
-			ret = ttm_tt_bind(bo->ttm, mem);
+			ret = ttm_tt_bind(bo->ttm, mem, ctx);
 			if (ret)
 				goto out_err;
 		}
@@ -325,12 +320,11 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 
 	if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 	    !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
-		ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem);
+		ret = ttm_bo_move_ttm(bo, ctx, mem);
 	else if (bdev->driver->move)
-		ret = bdev->driver->move(bo, evict, interruptible,
-					 no_wait_gpu, mem);
+		ret = bdev->driver->move(bo, evict, ctx, mem);
 	else
-		ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem);
+		ret = ttm_bo_move_memcpy(bo, ctx, mem);
 
 	if (ret) {
 		if (bdev->driver->move_notify) {
@@ -355,13 +349,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 		bo->evicted = false;
 	}
 
-	if (bo->mem.mm_node) {
+	if (bo->mem.mm_node)
 		bo->offset = (bo->mem.start << PAGE_SHIFT) +
 		    bdev->man[bo->mem.mem_type].gpu_offset;
-		bo->cur_placement = bo->mem.placement;
-	} else
+	else
 		bo->offset = 0;
 
+	ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
 	return 0;
 
 out_err:
@@ -390,8 +384,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
 	ttm_tt_destroy(bo->ttm);
 	bo->ttm = NULL;
 	ttm_bo_mem_put(bo, &bo->mem);
-
-	ww_mutex_unlock (&bo->resv->lock);
 }
 
 static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
@@ -448,7 +440,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 	}
 
 	spin_lock(&glob->lru_lock);
-	ret = __ttm_bo_reserve(bo, false, true, NULL);
+	ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
 	if (!ret) {
 		if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
 			ttm_bo_del_from_lru(bo);
@@ -457,6 +449,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 				reservation_object_unlock(&bo->ttm_resv);
 
 			ttm_bo_cleanup_memtype_use(bo);
+			reservation_object_unlock(bo->resv);
 			return;
 		}
 
@@ -472,7 +465,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 			ttm_bo_add_to_lru(bo);
 		}
 
-		__ttm_bo_unreserve(bo);
+		reservation_object_unlock(bo->resv);
 	}
 	if (bo->resv != &bo->ttm_resv)
 		reservation_object_unlock(&bo->ttm_resv);
@@ -487,20 +480,21 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
 }
 
 /**
- * function ttm_bo_cleanup_refs_and_unlock
+ * function ttm_bo_cleanup_refs
  * If bo idle, remove from delayed- and lru lists, and unref.
  * If not idle, do nothing.
  *
  * Must be called with lru_lock and reservation held, this function
- * will drop both before returning.
+ * will drop the lru lock and optionally the reservation lock before returning.
  *
  * @interruptible         Any sleeps should occur interruptibly.
  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
+ * @unlock_resv           Unlock the reservation lock as well.
  */
 
-static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
-					  bool interruptible,
-					  bool no_wait_gpu)
+static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
+			       bool interruptible, bool no_wait_gpu,
+			       bool unlock_resv)
 {
 	struct ttm_bo_global *glob = bo->glob;
 	struct reservation_object *resv;
@@ -518,7 +512,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
 
 	if (ret && !no_wait_gpu) {
 		long lret;
-		ww_mutex_unlock(&bo->resv->lock);
+
+		if (unlock_resv)
+			reservation_object_unlock(bo->resv);
 		spin_unlock(&glob->lru_lock);
 
 		lret = reservation_object_wait_timeout_rcu(resv, true,
@@ -531,24 +527,24 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
 			return -EBUSY;
 
 		spin_lock(&glob->lru_lock);
-		ret = __ttm_bo_reserve(bo, false, true, NULL);
-
-		/*
-		 * We raced, and lost, someone else holds the reservation now,
-		 * and is probably busy in ttm_bo_cleanup_memtype_use.
-		 *
-		 * Even if it's not the case, because we finished waiting any
-		 * delayed destruction would succeed, so just return success
-		 * here.
-		 */
-		if (ret) {
+		if (unlock_resv && !reservation_object_trylock(bo->resv)) {
+			/*
+			 * We raced, and lost, someone else holds the reservation now,
+			 * and is probably busy in ttm_bo_cleanup_memtype_use.
+			 *
+			 * Even if it's not the case, because we finished waiting any
+			 * delayed destruction would succeed, so just return success
+			 * here.
+			 */
 			spin_unlock(&glob->lru_lock);
 			return 0;
 		}
+		ret = 0;
 	}
 
 	if (ret || unlikely(list_empty(&bo->ddestroy))) {
-		__ttm_bo_unreserve(bo);
+		if (unlock_resv)
+			reservation_object_unlock(bo->resv);
 		spin_unlock(&glob->lru_lock);
 		return ret;
 	}
@@ -560,6 +556,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
 	spin_unlock(&glob->lru_lock);
 	ttm_bo_cleanup_memtype_use(bo);
 
+	if (unlock_resv)
+		reservation_object_unlock(bo->resv);
+
 	return 0;
 }
 
@@ -567,60 +566,44 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
  * encountered buffers.
  */
-
-static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
+static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 {
 	struct ttm_bo_global *glob = bdev->glob;
-	struct ttm_buffer_object *entry = NULL;
-	int ret = 0;
+	struct list_head removed;
+	bool empty;
 
-	spin_lock(&glob->lru_lock);
-	if (list_empty(&bdev->ddestroy))
-		goto out_unlock;
+	INIT_LIST_HEAD(&removed);
 
-	entry = list_first_entry(&bdev->ddestroy,
-		struct ttm_buffer_object, ddestroy);
-	kref_get(&entry->list_kref);
-
-	for (;;) {
-		struct ttm_buffer_object *nentry = NULL;
+	spin_lock(&glob->lru_lock);
+	while (!list_empty(&bdev->ddestroy)) {
+		struct ttm_buffer_object *bo;
 
-		if (entry->ddestroy.next != &bdev->ddestroy) {
-			nentry = list_first_entry(&entry->ddestroy,
-				struct ttm_buffer_object, ddestroy);
-			kref_get(&nentry->list_kref);
-		}
+		bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
+				      ddestroy);
+		kref_get(&bo->list_kref);
+		list_move_tail(&bo->ddestroy, &removed);
 
-		ret = __ttm_bo_reserve(entry, false, true, NULL);
-		if (remove_all && ret) {
+		if (remove_all || bo->resv != &bo->ttm_resv) {
 			spin_unlock(&glob->lru_lock);
-			ret = __ttm_bo_reserve(entry, false, false, NULL);
+			reservation_object_lock(bo->resv, NULL);
+
 			spin_lock(&glob->lru_lock);
-		}
+			ttm_bo_cleanup_refs(bo, false, !remove_all, true);
 
-		if (!ret)
-			ret = ttm_bo_cleanup_refs_and_unlock(entry, false,
-							     !remove_all);
-		else
+		} else if (reservation_object_trylock(bo->resv)) {
+			ttm_bo_cleanup_refs(bo, false, !remove_all, true);
+		} else {
 			spin_unlock(&glob->lru_lock);
+		}
 
-		kref_put(&entry->list_kref, ttm_bo_release_list);
-		entry = nentry;
-
-		if (ret || !entry)
-			goto out;
-
+		kref_put(&bo->list_kref, ttm_bo_release_list);
 		spin_lock(&glob->lru_lock);
-		if (list_empty(&entry->ddestroy))
-			break;
 	}
-
-out_unlock:
+	list_splice_tail(&removed, &bdev->ddestroy);
+	empty = list_empty(&bdev->ddestroy);
 	spin_unlock(&glob->lru_lock);
-out:
-	if (entry)
-		kref_put(&entry->list_kref, ttm_bo_release_list);
-	return ret;
+
+	return empty;
 }
 
 static void ttm_bo_delayed_workqueue(struct work_struct *work)
@@ -628,7 +611,7 @@ static void ttm_bo_delayed_workqueue(struct work_struct *work)
 	struct ttm_bo_device *bdev =
 	    container_of(work, struct ttm_bo_device, wq.work);
 
-	if (ttm_bo_delayed_delete(bdev, false)) {
+	if (!ttm_bo_delayed_delete(bdev, false)) {
 		schedule_delayed_work(&bdev->wq,
 				      ((HZ / 100) < 1) ? 1 : HZ / 100);
 	}
@@ -672,15 +655,15 @@ void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 }
 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 
-static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
-			bool no_wait_gpu)
+static int ttm_bo_evict(struct ttm_buffer_object *bo,
+			struct ttm_operation_ctx *ctx)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_mem_reg evict_mem;
 	struct ttm_placement placement;
 	int ret = 0;
 
-	lockdep_assert_held(&bo->resv->lock.base);
+	reservation_object_assert_held(bo->resv);
 
 	evict_mem = bo->mem;
 	evict_mem.mm_node = NULL;
@@ -690,8 +673,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
 	placement.num_placement = 0;
 	placement.num_busy_placement = 0;
 	bdev->driver->evict_flags(bo, &placement);
-	ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
-				no_wait_gpu);
+	ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
 	if (ret) {
 		if (ret != -ERESTARTSYS) {
 			pr_err("Failed to find memory space for buffer 0x%p eviction\n",
@@ -701,8 +683,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
 		goto out;
 	}
 
-	ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
-				     no_wait_gpu);
+	ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
 	if (unlikely(ret)) {
 		if (ret != -ERESTARTSYS)
 			pr_err("Buffer eviction failed\n");
@@ -728,49 +709,78 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 }
 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 
+/**
+ * Check the target bo is allowable to be evicted or swapout, including cases:
+ *
+ * a. if share same reservation object with ctx->resv, have assumption
+ * reservation objects should already be locked, so not lock again and
+ * return true directly when either the opreation allow_reserved_eviction
+ * or the target bo already is in delayed free list;
+ *
+ * b. Otherwise, trylock it.
+ */
+static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo,
+			struct ttm_operation_ctx *ctx, bool *locked)
+{
+	bool ret = false;
+
+	*locked = false;
+	if (bo->resv == ctx->resv) {
+		reservation_object_assert_held(bo->resv);
+		if (ctx->allow_reserved_eviction || !list_empty(&bo->ddestroy))
+			ret = true;
+	} else {
+		*locked = reservation_object_trylock(bo->resv);
+		ret = *locked;
+	}
+
+	return ret;
+}
+
 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
-				uint32_t mem_type,
-				const struct ttm_place *place,
-				bool interruptible,
-				bool no_wait_gpu)
+			       uint32_t mem_type,
+			       const struct ttm_place *place,
+			       struct ttm_operation_ctx *ctx)
 {
 	struct ttm_bo_global *glob = bdev->glob;
 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
-	struct ttm_buffer_object *bo;
-	int ret = -EBUSY;
+	struct ttm_buffer_object *bo = NULL;
+	bool locked = false;
 	unsigned i;
+	int ret;
 
 	spin_lock(&glob->lru_lock);
 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 		list_for_each_entry(bo, &man->lru[i], lru) {
-			ret = __ttm_bo_reserve(bo, false, true, NULL);
-			if (ret)
+			if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked))
 				continue;
 
 			if (place && !bdev->driver->eviction_valuable(bo,
 								      place)) {
-				__ttm_bo_unreserve(bo);
-				ret = -EBUSY;
+				if (locked)
+					reservation_object_unlock(bo->resv);
 				continue;
 			}
-
 			break;
 		}
 
-		if (!ret)
+		/* If the inner loop terminated early, we have our candidate */
+		if (&bo->lru != &man->lru[i])
 			break;
+
+		bo = NULL;
 	}
 
-	if (ret) {
+	if (!bo) {
 		spin_unlock(&glob->lru_lock);
-		return ret;
+		return -EBUSY;
 	}
 
 	kref_get(&bo->list_kref);
 
 	if (!list_empty(&bo->ddestroy)) {
-		ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible,
-						     no_wait_gpu);
+		ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
+					  ctx->no_wait_gpu, locked);
 		kref_put(&bo->list_kref, ttm_bo_release_list);
 		return ret;
 	}
@@ -778,10 +788,14 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 	ttm_bo_del_from_lru(bo);
 	spin_unlock(&glob->lru_lock);
 
-	BUG_ON(ret != 0);
-
-	ret = ttm_bo_evict(bo, interruptible, no_wait_gpu);
-	ttm_bo_unreserve(bo);
+	ret = ttm_bo_evict(bo, ctx);
+	if (locked) {
+		ttm_bo_unreserve(bo);
+	} else {
+		spin_lock(&glob->lru_lock);
+		ttm_bo_add_to_lru(bo);
+		spin_unlock(&glob->lru_lock);
+	}
 
 	kref_put(&bo->list_kref, ttm_bo_release_list);
 	return ret;
@@ -832,8 +846,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 					uint32_t mem_type,
 					const struct ttm_place *place,
 					struct ttm_mem_reg *mem,
-					bool interruptible,
-					bool no_wait_gpu)
+					struct ttm_operation_ctx *ctx)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
@@ -845,8 +858,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 			return ret;
 		if (mem->mm_node)
 			break;
-		ret = ttm_mem_evict_first(bdev, mem_type, place,
-					  interruptible, no_wait_gpu);
+		ret = ttm_mem_evict_first(bdev, mem_type, place, ctx);
 		if (unlikely(ret != 0))
 			return ret;
 	} while (1);
@@ -909,8 +921,7 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 			struct ttm_placement *placement,
 			struct ttm_mem_reg *mem,
-			bool interruptible,
-			bool no_wait_gpu)
+			struct ttm_operation_ctx *ctx)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_mem_type_manager *man;
@@ -1004,8 +1015,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 			return 0;
 		}
 
-		ret = ttm_bo_mem_force_space(bo, mem_type, place, mem,
-						interruptible, no_wait_gpu);
+		ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx);
 		if (ret == 0 && mem->mm_node) {
 			mem->placement = cur_flags;
 			return 0;
@@ -1024,14 +1034,13 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 EXPORT_SYMBOL(ttm_bo_mem_space);
 
 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
-			struct ttm_placement *placement,
-			bool interruptible,
-			bool no_wait_gpu)
+			      struct ttm_placement *placement,
+			      struct ttm_operation_ctx *ctx)
 {
 	int ret = 0;
 	struct ttm_mem_reg mem;
 
-	lockdep_assert_held(&bo->resv->lock.base);
+	reservation_object_assert_held(bo->resv);
 
 	mem.num_pages = bo->num_pages;
 	mem.size = mem.num_pages << PAGE_SHIFT;
@@ -1041,12 +1050,10 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 	/*
 	 * Determine where to move the buffer.
 	 */
-	ret = ttm_bo_mem_space(bo, placement, &mem,
-			       interruptible, no_wait_gpu);
+	ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
 	if (ret)
 		goto out_unlock;
-	ret = ttm_bo_handle_move_mem(bo, &mem, false,
-				     interruptible, no_wait_gpu);
+	ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
 out_unlock:
 	if (ret && mem.mm_node)
 		ttm_bo_mem_put(bo, &mem);
@@ -1097,20 +1104,18 @@ bool ttm_bo_mem_compat(struct ttm_placement *placement,
 EXPORT_SYMBOL(ttm_bo_mem_compat);
 
 int ttm_bo_validate(struct ttm_buffer_object *bo,
-			struct ttm_placement *placement,
-			bool interruptible,
-			bool no_wait_gpu)
+		    struct ttm_placement *placement,
+		    struct ttm_operation_ctx *ctx)
 {
 	int ret;
 	uint32_t new_flags;
 
-	lockdep_assert_held(&bo->resv->lock.base);
+	reservation_object_assert_held(bo->resv);
 	/*
 	 * Check whether we need to move buffer.
 	 */
 	if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
-		ret = ttm_bo_move_buffer(bo, placement, interruptible,
-					 no_wait_gpu);
+		ret = ttm_bo_move_buffer(bo, placement, ctx);
 		if (ret)
 			return ret;
 	} else {
@@ -1139,7 +1144,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
 			 enum ttm_bo_type type,
 			 struct ttm_placement *placement,
 			 uint32_t page_alignment,
-			 bool interruptible,
+			 struct ttm_operation_ctx *ctx,
 			 struct file *persistent_swap_storage,
 			 size_t acc_size,
 			 struct sg_table *sg,
@@ -1151,7 +1156,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
 	struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
 	bool locked;
 
-	ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
+	ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx);
 	if (ret) {
 		pr_err("Out of kernel memory\n");
 		if (destroy)
@@ -1199,7 +1204,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
 	bo->sg = sg;
 	if (resv) {
 		bo->resv = resv;
-		lockdep_assert_held(&bo->resv->lock.base);
+		reservation_object_assert_held(bo->resv);
 	} else {
 		bo->resv = &bo->ttm_resv;
 	}
@@ -1221,12 +1226,12 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
 	 * since otherwise lockdep will be angered in radeon.
 	 */
 	if (!resv) {
-		locked = ww_mutex_trylock(&bo->resv->lock);
+		locked = reservation_object_trylock(bo->resv);
 		WARN_ON(!locked);
 	}
 
 	if (likely(!ret))
-		ret = ttm_bo_validate(bo, placement, interruptible, false);
+		ret = ttm_bo_validate(bo, placement, ctx);
 
 	if (unlikely(ret)) {
 		if (!resv)
@@ -1259,10 +1264,11 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
 		struct reservation_object *resv,
 		void (*destroy) (struct ttm_buffer_object *))
 {
+	struct ttm_operation_ctx ctx = { interruptible, false };
 	int ret;
 
 	ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
-				   page_alignment, interruptible,
+				   page_alignment, &ctx,
 				   persistent_swap_storage, acc_size,
 				   sg, resv, destroy);
 	if (ret)
@@ -1334,6 +1340,7 @@ EXPORT_SYMBOL(ttm_bo_create);
 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
 				   unsigned mem_type)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 	struct ttm_bo_global *glob = bdev->glob;
 	struct dma_fence *fence;
@@ -1348,7 +1355,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 		while (!list_empty(&man->lru[i])) {
 			spin_unlock(&glob->lru_lock);
-			ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false);
+			ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
 			if (ret)
 				return ret;
 			spin_lock(&glob->lru_lock);
@@ -1469,7 +1476,6 @@ static void ttm_bo_global_kobj_release(struct kobject *kobj)
 	struct ttm_bo_global *glob =
 		container_of(kobj, struct ttm_bo_global, kobj);
 
-	ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink);
 	__free_page(glob->dummy_read_page);
 	kfree(glob);
 }
@@ -1494,6 +1500,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref)
 	mutex_init(&glob->device_list_mutex);
 	spin_lock_init(&glob->lru_lock);
 	glob->mem_glob = bo_ref->mem_glob;
+	glob->mem_glob->bo_glob = glob;
 	glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
 
 	if (unlikely(glob->dummy_read_page == NULL)) {
@@ -1504,14 +1511,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref)
 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
 		INIT_LIST_HEAD(&glob->swap_lru[i]);
 	INIT_LIST_HEAD(&glob->device_list);
-
-	ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
-	ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink);
-	if (unlikely(ret != 0)) {
-		pr_err("Could not register buffer object swapout\n");
-		goto out_no_shrink;
-	}
-
 	atomic_set(&glob->bo_count, 0);
 
 	ret = kobject_init_and_add(
@@ -1519,8 +1518,6 @@ int ttm_bo_global_init(struct drm_global_reference *ref)
 	if (unlikely(ret != 0))
 		kobject_put(&glob->kobj);
 	return ret;
-out_no_shrink:
-	__free_page(glob->dummy_read_page);
 out_no_drp:
 	kfree(glob);
 	return ret;
@@ -1554,16 +1551,13 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
 
 	cancel_delayed_work_sync(&bdev->wq);
 
-	while (ttm_bo_delayed_delete(bdev, true))
-		;
+	if (ttm_bo_delayed_delete(bdev, true))
+		pr_debug("Delayed destroy list was clean\n");
 
 	spin_lock(&glob->lru_lock);
-	if (list_empty(&bdev->ddestroy))
-		TTM_DEBUG("Delayed destroy list was clean\n");
-
 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
 		if (list_empty(&bdev->man[0].lru[0]))
-			TTM_DEBUG("Swap list %d was clean\n", i);
+			pr_debug("Swap list %d was clean\n", i);
 	spin_unlock(&glob->lru_lock);
 
 	drm_vma_offset_manager_destroy(&bdev->vma_manager);
@@ -1706,21 +1700,20 @@ EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
  * A buffer object shrink method that tries to swap out the first
  * buffer object on the bo_global::swap_lru list.
  */
-
-static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
+int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx)
 {
-	struct ttm_bo_global *glob =
-	    container_of(shrink, struct ttm_bo_global, shrink);
 	struct ttm_buffer_object *bo;
 	int ret = -EBUSY;
+	bool locked;
 	unsigned i;
 
 	spin_lock(&glob->lru_lock);
 	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
 		list_for_each_entry(bo, &glob->swap_lru[i], swap) {
-			ret = __ttm_bo_reserve(bo, false, true, NULL);
-			if (!ret)
+			if (ttm_bo_evict_swapout_allowable(bo, ctx, &locked)) {
+				ret = 0;
 				break;
+			}
 		}
 		if (!ret)
 			break;
@@ -1734,7 +1727,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
 	kref_get(&bo->list_kref);
 
 	if (!list_empty(&bo->ddestroy)) {
-		ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false);
+		ret = ttm_bo_cleanup_refs(bo, false, false, locked);
 		kref_put(&bo->list_kref, ttm_bo_release_list);
 		return ret;
 	}
@@ -1748,6 +1741,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
 
 	if (bo->mem.mem_type != TTM_PL_SYSTEM ||
 	    bo->ttm->caching_state != tt_cached) {
+		struct ttm_operation_ctx ctx = { false, false };
 		struct ttm_mem_reg evict_mem;
 
 		evict_mem = bo->mem;
@@ -1755,8 +1749,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
 		evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
 		evict_mem.mem_type = TTM_PL_SYSTEM;
 
-		ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
-					     false, false);
+		ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
 		if (unlikely(ret != 0))
 			goto out;
 	}
@@ -1787,15 +1780,21 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
 	 * Unreserve without putting on LRU to avoid swapping out an
 	 * already swapped buffer.
 	 */
-
-	__ttm_bo_unreserve(bo);
+	if (locked)
+		reservation_object_unlock(bo->resv);
 	kref_put(&bo->list_kref, ttm_bo_release_list);
 	return ret;
 }
+EXPORT_SYMBOL(ttm_bo_swapout);
 
 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
 {
-	while (ttm_bo_swapout(&bdev->glob->shrink) == 0)
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
+
+	while (ttm_bo_swapout(bdev->glob, &ctx) == 0)
 		;
 }
 EXPORT_SYMBOL(ttm_bo_swapout_all);
@@ -1822,10 +1821,12 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
 		return -ERESTARTSYS;
 	if (!ww_mutex_is_locked(&bo->resv->lock))
 		goto out_unlock;
-	ret = __ttm_bo_reserve(bo, true, false, NULL);
+	ret = reservation_object_lock_interruptible(bo->resv, NULL);
+	if (ret == -EINTR)
+		ret = -ERESTARTSYS;
 	if (unlikely(ret != 0))
 		goto out_unlock;
-	__ttm_bo_unreserve(bo);
+	reservation_object_unlock(bo->resv);
 
 out_unlock:
 	mutex_unlock(&bo->wu_mutex);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index e7a519f1849b..153de1bf0232 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -45,7 +45,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
 }
 
 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
-		    bool interruptible, bool no_wait_gpu,
+		   struct ttm_operation_ctx *ctx,
 		    struct ttm_mem_reg *new_mem)
 {
 	struct ttm_tt *ttm = bo->ttm;
@@ -53,7 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
 	int ret;
 
 	if (old_mem->mem_type != TTM_PL_SYSTEM) {
-		ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+		ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 
 		if (unlikely(ret != 0)) {
 			if (ret != -ERESTARTSYS)
@@ -73,7 +73,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
 		return ret;
 
 	if (new_mem->mem_type != TTM_PL_SYSTEM) {
-		ret = ttm_tt_bind(ttm, new_mem);
+		ret = ttm_tt_bind(ttm, new_mem, ctx);
 		if (unlikely(ret != 0))
 			return ret;
 	}
@@ -329,7 +329,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 }
 
 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
-		       bool interruptible, bool no_wait_gpu,
+		       struct ttm_operation_ctx *ctx,
 		       struct ttm_mem_reg *new_mem)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
@@ -345,7 +345,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 	unsigned long add = 0;
 	int dir;
 
-	ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+	ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 	if (ret)
 		return ret;
 
@@ -376,7 +376,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 	 * TTM might be null for moves within the same region.
 	 */
 	if (ttm && ttm->state == tt_unpopulated) {
-		ret = ttm->bdev->driver->ttm_tt_populate(ttm);
+		ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx);
 		if (ret)
 			goto out1;
 	}
@@ -485,7 +485,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 	fbo->acc_size = 0;
 	fbo->resv = &fbo->ttm_resv;
 	reservation_object_init(fbo->resv);
-	ret = ww_mutex_trylock(&fbo->resv->lock);
+	ret = reservation_object_trylock(fbo->resv);
 	WARN_ON(!ret);
 
 	*new_obj = fbo;
@@ -545,14 +545,19 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 			   unsigned long num_pages,
 			   struct ttm_bo_kmap_obj *map)
 {
-	struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
+	struct ttm_mem_reg *mem = &bo->mem;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
 	struct ttm_tt *ttm = bo->ttm;
+	pgprot_t prot;
 	int ret;
 
 	BUG_ON(!ttm);
 
 	if (ttm->state == tt_unpopulated) {
-		ret = ttm->bdev->driver->ttm_tt_populate(ttm);
+		ret = ttm->bdev->driver->ttm_tt_populate(ttm, &ctx);
 		if (ret)
 			return ret;
 	}
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index c8ebb757e36b..60fcef1593dd 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -92,6 +92,18 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 	return ret;
 }
 
+static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo,
+				       unsigned long page_offset)
+{
+	struct ttm_bo_device *bdev = bo->bdev;
+
+	if (bdev->driver->io_mem_pfn)
+		return bdev->driver->io_mem_pfn(bo, page_offset);
+
+	return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT)
+		+ page_offset;
+}
+
 static int ttm_bo_vm_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
@@ -215,12 +227,17 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf)
 		cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,
 						cvma.vm_page_prot);
 	} else {
+		struct ttm_operation_ctx ctx = {
+			.interruptible = false,
+			.no_wait_gpu = false
+		};
+
 		ttm = bo->ttm;
 		cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,
 						cvma.vm_page_prot);
 
 		/* Allocate all page at once, most common usage */
-		if (ttm->bdev->driver->ttm_tt_populate(ttm)) {
+		if (ttm->bdev->driver->ttm_tt_populate(ttm, &ctx)) {
 			retval = VM_FAULT_OOM;
 			goto out_io_unlock;
 		}
@@ -234,7 +251,7 @@ static int ttm_bo_vm_fault(struct vm_fault *vmf)
 		if (bo->mem.bus.is_iomem) {
 			/* Iomem should not be marked encrypted */
 			cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot);
-			pfn = bdev->driver->io_mem_pfn(bo, page_offset);
+			pfn = ttm_bo_io_mem_pfn(bo, page_offset);
 		} else {
 			page = ttm->pages[page_offset];
 			if (unlikely(!page && i == 0)) {
@@ -299,7 +316,7 @@ static void ttm_bo_vm_close(struct vm_area_struct *vma)
 
 static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
 				 unsigned long offset,
-				 void *buf, int len, int write)
+				 uint8_t *buf, int len, int write)
 {
 	unsigned long page = offset >> PAGE_SHIFT;
 	unsigned long bytes_left = len;
@@ -328,6 +345,7 @@ static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
 		ttm_bo_kunmap(&map);
 
 		page++;
+		buf += bytes;
 		bytes_left -= bytes;
 		offset = 0;
 	} while (bytes_left);
@@ -404,14 +422,6 @@ static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
 	return bo;
 }
 
-unsigned long ttm_bo_default_io_mem_pfn(struct ttm_buffer_object *bo,
-					unsigned long page_offset)
-{
-	return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT)
-		+ page_offset;
-}
-EXPORT_SYMBOL(ttm_bo_default_io_mem_pfn);
-
 int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
 		struct ttm_bo_device *bdev)
 {
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index 5e1bcabffef5..373ced0b2fc2 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -38,7 +38,7 @@ static void ttm_eu_backoff_reservation_reverse(struct list_head *list,
 	list_for_each_entry_continue_reverse(entry, list, head) {
 		struct ttm_buffer_object *bo = entry->bo;
 
-		__ttm_bo_unreserve(bo);
+		reservation_object_unlock(bo->resv);
 	}
 }
 
@@ -69,7 +69,7 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket,
 		struct ttm_buffer_object *bo = entry->bo;
 
 		ttm_bo_add_to_lru(bo);
-		__ttm_bo_unreserve(bo);
+		reservation_object_unlock(bo->resv);
 	}
 	spin_unlock(&glob->lru_lock);
 
@@ -112,7 +112,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket,
 
 		ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), ticket);
 		if (!ret && unlikely(atomic_read(&bo->cpu_writers) > 0)) {
-			__ttm_bo_unreserve(bo);
+			reservation_object_unlock(bo->resv);
 
 			ret = -EBUSY;
 
@@ -203,7 +203,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
 		else
 			reservation_object_add_excl_fence(bo->resv, fence);
 		ttm_bo_add_to_lru(bo);
-		__ttm_bo_unreserve(bo);
+		reservation_object_unlock(bo->resv);
 	}
 	spin_unlock(&glob->lru_lock);
 	if (ticket)
diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index e96374990398..aa0c38136958 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -211,35 +211,33 @@ static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,
  */
 
 static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,
-		       uint64_t extra)
+			uint64_t extra, struct ttm_operation_ctx *ctx)
 {
 	int ret;
-	struct ttm_mem_shrink *shrink;
 
 	spin_lock(&glob->lock);
-	if (glob->shrink == NULL)
-		goto out;
 
 	while (ttm_zones_above_swap_target(glob, from_wq, extra)) {
-		shrink = glob->shrink;
 		spin_unlock(&glob->lock);
-		ret = shrink->do_shrink(shrink);
+		ret = ttm_bo_swapout(glob->bo_glob, ctx);
 		spin_lock(&glob->lock);
 		if (unlikely(ret != 0))
-			goto out;
+			break;
 	}
-out:
+
 	spin_unlock(&glob->lock);
 }
 
-
-
 static void ttm_shrink_work(struct work_struct *work)
 {
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
 	struct ttm_mem_global *glob =
 	    container_of(work, struct ttm_mem_global, work);
 
-	ttm_shrink(glob, true, 0ULL);
+	ttm_shrink(glob, true, 0ULL, &ctx);
 }
 
 static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,
@@ -514,7 +512,7 @@ static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
 static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,
 				     struct ttm_mem_zone *single_zone,
 				     uint64_t memory,
-				     bool no_wait, bool interruptible)
+				     struct ttm_operation_ctx *ctx)
 {
 	int count = TTM_MEMORY_ALLOC_RETRIES;
 
@@ -522,33 +520,32 @@ static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,
 					       single_zone,
 					       memory, true)
 			!= 0)) {
-		if (no_wait)
+		if (ctx->no_wait_gpu)
 			return -ENOMEM;
 		if (unlikely(count-- == 0))
 			return -ENOMEM;
-		ttm_shrink(glob, false, memory + (memory >> 2) + 16);
+		ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx);
 	}
 
 	return 0;
 }
 
 int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
-			 bool no_wait, bool interruptible)
+			 struct ttm_operation_ctx *ctx)
 {
 	/**
 	 * Normal allocations of kernel memory are registered in
 	 * all zones.
 	 */
 
-	return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait,
-					 interruptible);
+	return ttm_mem_global_alloc_zone(glob, NULL, memory, ctx);
 }
 EXPORT_SYMBOL(ttm_mem_global_alloc);
 
 int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
-			      struct page *page, uint64_t size)
+			      struct page *page, uint64_t size,
+			      struct ttm_operation_ctx *ctx)
 {
-
 	struct ttm_mem_zone *zone = NULL;
 
 	/**
@@ -563,7 +560,7 @@ int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
 	if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
 		zone = glob->zone_kernel;
 #endif
-	return ttm_mem_global_alloc_zone(glob, zone, size, false, false);
+	return ttm_mem_global_alloc_zone(glob, zone, size, ctx);
 }
 
 void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page,
diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c
index 26a7ad0f4789..1aa2baa83959 100644
--- a/drivers/gpu/drm/ttm/ttm_object.c
+++ b/drivers/gpu/drm/ttm/ttm_object.c
@@ -325,6 +325,10 @@ int ttm_ref_object_add(struct ttm_object_file *tfile,
 	struct ttm_ref_object *ref;
 	struct drm_hash_item *hash;
 	struct ttm_mem_global *mem_glob = tfile->tdev->mem_glob;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
 	int ret = -EINVAL;
 
 	if (base->tfile != tfile && !base->shareable)
@@ -350,7 +354,7 @@ int ttm_ref_object_add(struct ttm_object_file *tfile,
 			return -EPERM;
 
 		ret = ttm_mem_global_alloc(mem_glob, sizeof(*ref),
-					   false, false);
+					   &ctx);
 		if (unlikely(ret != 0))
 			return ret;
 		ref = kmalloc(sizeof(*ref), GFP_KERNEL);
@@ -686,7 +690,10 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile,
 	dma_buf = prime->dma_buf;
 	if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) {
 		DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
-
+		struct ttm_operation_ctx ctx = {
+			.interruptible = true,
+			.no_wait_gpu = false
+		};
 		exp_info.ops = &tdev->ops;
 		exp_info.size = prime->size;
 		exp_info.flags = flags;
@@ -696,7 +703,7 @@ int ttm_prime_handle_to_fd(struct ttm_object_file *tfile,
 		 * Need to create a new dma_buf, with memory accounting.
 		 */
 		ret = ttm_mem_global_alloc(tdev->mem_glob, tdev->dma_buf_size,
-					   false, true);
+					   &ctx);
 		if (unlikely(ret != 0)) {
 			mutex_unlock(&prime->mutex);
 			goto out_unref;
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index 5d252fb27a82..2b12c55a3bff 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -477,12 +477,12 @@ ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 	return count;
 }
 
-static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
+static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
 {
 	manager->mm_shrink.count_objects = ttm_pool_shrink_count;
 	manager->mm_shrink.scan_objects = ttm_pool_shrink_scan;
 	manager->mm_shrink.seeks = 1;
-	register_shrinker(&manager->mm_shrink);
+	return register_shrinker(&manager->mm_shrink);
 }
 
 static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
@@ -1034,15 +1034,18 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 
 	ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
 				   &glob->kobj, "pool");
-	if (unlikely(ret != 0)) {
-		kobject_put(&_manager->kobj);
-		_manager = NULL;
-		return ret;
-	}
-
-	ttm_pool_mm_shrink_init(_manager);
+	if (unlikely(ret != 0))
+		goto error;
 
+	ret = ttm_pool_mm_shrink_init(_manager);
+	if (unlikely(ret != 0))
+		goto error;
 	return 0;
+
+error:
+	kobject_put(&_manager->kobj);
+	_manager = NULL;
+	return ret;
 }
 
 void ttm_page_alloc_fini(void)
@@ -1060,7 +1063,7 @@ void ttm_page_alloc_fini(void)
 	_manager = NULL;
 }
 
-int ttm_pool_populate(struct ttm_tt *ttm)
+int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
 {
 	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
 	unsigned i;
@@ -1072,15 +1075,17 @@ int ttm_pool_populate(struct ttm_tt *ttm)
 	ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
 			    ttm->caching_state);
 	if (unlikely(ret != 0)) {
-		ttm_pool_unpopulate(ttm);
+		ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+			      ttm->caching_state);
 		return ret;
 	}
 
 	for (i = 0; i < ttm->num_pages; ++i) {
 		ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
-						PAGE_SIZE);
+						PAGE_SIZE, ctx);
 		if (unlikely(ret != 0)) {
-			ttm_pool_unpopulate(ttm);
+			ttm_put_pages(ttm->pages, ttm->num_pages,
+				      ttm->page_flags, ttm->caching_state);
 			return -ENOMEM;
 		}
 	}
@@ -1115,12 +1120,13 @@ void ttm_pool_unpopulate(struct ttm_tt *ttm)
 }
 EXPORT_SYMBOL(ttm_pool_unpopulate);
 
-int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt)
+int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt,
+					struct ttm_operation_ctx *ctx)
 {
 	unsigned i, j;
 	int r;
 
-	r = ttm_pool_populate(&tt->ttm);
+	r = ttm_pool_populate(&tt->ttm, ctx);
 	if (r)
 		return r;
 
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
index 6b2627fe9bc1..a88051552ace 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
@@ -61,6 +61,7 @@
 #define SMALL_ALLOCATION		4
 #define FREE_ALL_PAGES			(~0U)
 #define VADDR_FLAG_HUGE_POOL		1UL
+#define VADDR_FLAG_UPDATED_COUNT	2UL
 
 enum pool_type {
 	IS_UNDEFINED	= 0,
@@ -333,14 +334,18 @@ static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page)
 static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool)
 {
 	struct dma_page *d_page;
+	unsigned long attrs = 0;
 	void *vaddr;
 
 	d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL);
 	if (!d_page)
 		return NULL;
 
-	vaddr = dma_alloc_coherent(pool->dev, pool->size, &d_page->dma,
-				   pool->gfp_flags);
+	if (pool->type & IS_HUGE)
+		attrs = DMA_ATTR_NO_WARN;
+
+	vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma,
+				pool->gfp_flags, attrs);
 	if (vaddr) {
 		if (is_vmalloc_addr(vaddr))
 			d_page->p = vmalloc_to_page(vaddr);
@@ -870,18 +875,18 @@ static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool,
 }
 
 /*
- * @return count of pages still required to fulfill the request.
  * The populate list is actually a stack (not that is matters as TTM
  * allocates one page at a time.
+ * return dma_page pointer if success, otherwise NULL.
  */
-static int ttm_dma_pool_get_pages(struct dma_pool *pool,
+static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool,
 				  struct ttm_dma_tt *ttm_dma,
 				  unsigned index)
 {
-	struct dma_page *d_page;
+	struct dma_page *d_page = NULL;
 	struct ttm_tt *ttm = &ttm_dma->ttm;
 	unsigned long irq_flags;
-	int count, r = -ENOMEM;
+	int count;
 
 	spin_lock_irqsave(&pool->lock, irq_flags);
 	count = ttm_dma_page_pool_fill_locked(pool, &irq_flags);
@@ -890,12 +895,11 @@ static int ttm_dma_pool_get_pages(struct dma_pool *pool,
 		ttm->pages[index] = d_page->p;
 		ttm_dma->dma_address[index] = d_page->dma;
 		list_move_tail(&d_page->page_list, &ttm_dma->pages_list);
-		r = 0;
 		pool->npages_in_use += 1;
 		pool->npages_free -= 1;
 	}
 	spin_unlock_irqrestore(&pool->lock, irq_flags);
-	return r;
+	return d_page;
 }
 
 static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge)
@@ -923,12 +927,14 @@ static gfp_t ttm_dma_pool_gfp_flags(struct ttm_dma_tt *ttm_dma, bool huge)
  * On success pages list will hold count number of correctly
  * cached pages. On failure will hold the negative return value (-ENOMEM, etc).
  */
-int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
+int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev,
+			struct ttm_operation_ctx *ctx)
 {
 	struct ttm_tt *ttm = &ttm_dma->ttm;
 	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
 	unsigned long num_pages = ttm->num_pages;
 	struct dma_pool *pool;
+	struct dma_page *d_page;
 	enum pool_type type;
 	unsigned i;
 	int ret;
@@ -957,17 +963,18 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 	while (num_pages >= HPAGE_PMD_NR) {
 		unsigned j;
 
-		ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
-		if (ret != 0)
+		d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i);
+		if (!d_page)
 			break;
 
 		ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
-						pool->size);
+						pool->size, ctx);
 		if (unlikely(ret != 0)) {
 			ttm_dma_unpopulate(ttm_dma, dev);
 			return -ENOMEM;
 		}
 
+		d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT;
 		for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) {
 			ttm->pages[j] = ttm->pages[j - 1] + 1;
 			ttm_dma->dma_address[j] = ttm_dma->dma_address[j - 1] +
@@ -991,19 +998,20 @@ int ttm_dma_populate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 	}
 
 	while (num_pages) {
-		ret = ttm_dma_pool_get_pages(pool, ttm_dma, i);
-		if (ret != 0) {
+		d_page = ttm_dma_pool_get_pages(pool, ttm_dma, i);
+		if (!d_page) {
 			ttm_dma_unpopulate(ttm_dma, dev);
 			return -ENOMEM;
 		}
 
 		ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
-						pool->size);
+						pool->size, ctx);
 		if (unlikely(ret != 0)) {
 			ttm_dma_unpopulate(ttm_dma, dev);
 			return -ENOMEM;
 		}
 
+		d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT;
 		++i;
 		--num_pages;
 	}
@@ -1044,8 +1052,11 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 				continue;
 
 			count++;
-			ttm_mem_global_free_page(ttm->glob->mem_glob,
-						 d_page->p, pool->size);
+			if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) {
+				ttm_mem_global_free_page(ttm->glob->mem_glob,
+							 d_page->p, pool->size);
+				d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT;
+			}
 			ttm_dma_page_put(pool, d_page);
 		}
 
@@ -1065,9 +1076,19 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 
 	/* make sure pages array match list and count number of pages */
 	count = 0;
-	list_for_each_entry(d_page, &ttm_dma->pages_list, page_list) {
+	list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list,
+				 page_list) {
 		ttm->pages[count] = d_page->p;
 		count++;
+
+		if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) {
+			ttm_mem_global_free_page(ttm->glob->mem_glob,
+						 d_page->p, pool->size);
+			d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT;
+		}
+
+		if (is_cached)
+			ttm_dma_page_put(pool, d_page);
 	}
 
 	spin_lock_irqsave(&pool->lock, irq_flags);
@@ -1087,19 +1108,6 @@ void ttm_dma_unpopulate(struct ttm_dma_tt *ttm_dma, struct device *dev)
 	}
 	spin_unlock_irqrestore(&pool->lock, irq_flags);
 
-	if (is_cached) {
-		list_for_each_entry_safe(d_page, next, &ttm_dma->pages_list, page_list) {
-			ttm_mem_global_free_page(ttm->glob->mem_glob,
-						 d_page->p, pool->size);
-			ttm_dma_page_put(pool, d_page);
-		}
-	} else {
-		for (i = 0; i < count; i++) {
-			ttm_mem_global_free_page(ttm->glob->mem_glob,
-						 ttm->pages[i], pool->size);
-		}
-	}
-
 	INIT_LIST_HEAD(&ttm_dma->pages_list);
 	for (i = 0; i < ttm->num_pages; i++) {
 		ttm->pages[i] = NULL;
@@ -1177,12 +1185,12 @@ ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 	return count;
 }
 
-static void ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager)
+static int ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager)
 {
 	manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count;
 	manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan;
 	manager->mm_shrink.seeks = 1;
-	register_shrinker(&manager->mm_shrink);
+	return register_shrinker(&manager->mm_shrink);
 }
 
 static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
@@ -1192,7 +1200,7 @@ static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
 
 int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 {
-	int ret = -ENOMEM;
+	int ret;
 
 	WARN_ON(_manager);
 
@@ -1200,7 +1208,7 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 
 	_manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
 	if (!_manager)
-		goto err;
+		return -ENOMEM;
 
 	mutex_init(&_manager->lock);
 	INIT_LIST_HEAD(&_manager->pools);
@@ -1212,13 +1220,17 @@ int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
 	/* This takes care of auto-freeing the _manager */
 	ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
 				   &glob->kobj, "dma_pool");
-	if (unlikely(ret != 0)) {
-		kobject_put(&_manager->kobj);
-		goto err;
-	}
-	ttm_dma_pool_mm_shrink_init(_manager);
+	if (unlikely(ret != 0))
+		goto error;
+
+	ret = ttm_dma_pool_mm_shrink_init(_manager);
+	if (unlikely(ret != 0))
+		goto error;
 	return 0;
-err:
+
+error:
+	kobject_put(&_manager->kobj);
+	_manager = NULL;
 	return ret;
 }
 
@@ -1244,15 +1256,12 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data)
 {
 	struct device_pools *p;
 	struct dma_pool *pool = NULL;
-	char *h[] = {"pool", "refills", "pages freed", "inuse", "available",
-		     "name", "virt", "busaddr"};
 
 	if (!_manager) {
 		seq_printf(m, "No pool allocator running.\n");
 		return 0;
 	}
-	seq_printf(m, "%13s %12s %13s %8s %8s %8s\n",
-		   h[0], h[1], h[2], h[3], h[4], h[5]);
+	seq_printf(m, "         pool      refills   pages freed    inuse available     name\n");
 	mutex_lock(&_manager->lock);
 	list_for_each_entry(p, &_manager->pools, pools) {
 		struct device *dev = p->dev;
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 8ebc8d3560c3..5a046a3c543a 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -261,7 +261,8 @@ void ttm_tt_unbind(struct ttm_tt *ttm)
 	}
 }
 
-int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
+int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem,
+		struct ttm_operation_ctx *ctx)
 {
 	int ret = 0;
 
@@ -271,7 +272,7 @@ int ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
 	if (ttm->state == tt_bound)
 		return 0;
 
-	ret = ttm->bdev->driver->ttm_tt_populate(ttm);
+	ret = ttm->bdev->driver->ttm_tt_populate(ttm, ctx);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/tve200/tve200_drm.h b/drivers/gpu/drm/tve200/tve200_drm.h
index 628b79324c48..5c270055bd58 100644
--- a/drivers/gpu/drm/tve200/tve200_drm.h
+++ b/drivers/gpu/drm/tve200/tve200_drm.h
@@ -103,7 +103,6 @@ struct tve200_drm_dev_private {
 	struct drm_panel *panel;
 	struct drm_bridge *bridge;
 	struct drm_simple_display_pipe pipe;
-	struct drm_fbdev_cma *fbdev;
 
 	void *regs;
 	struct clk *pclk;
diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c
index bd6c9454d767..44911d921864 100644
--- a/drivers/gpu/drm/tve200/tve200_drv.c
+++ b/drivers/gpu/drm/tve200/tve200_drv.c
@@ -46,6 +46,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_of.h>
@@ -130,8 +131,7 @@ static int tve200_modeset_init(struct drm_device *dev)
 	 * Passing in 16 here will make the RGB656 mode the default
 	 * Passing in 32 will use XRGB8888 mode
 	 */
-	priv->fbdev = drm_fbdev_cma_init(dev, 16,
-					 dev->mode_config.num_connector);
+	drm_fb_cma_fbdev_init(dev, 16, 0);
 	drm_kms_helper_poll_init(dev);
 
 	goto finish;
@@ -146,17 +146,10 @@ static int tve200_modeset_init(struct drm_device *dev)
 
 DEFINE_DRM_GEM_CMA_FOPS(drm_fops);
 
-static void tve200_lastclose(struct drm_device *dev)
-{
-	struct tve200_drm_dev_private *priv = dev->dev_private;
-
-	drm_fbdev_cma_restore_mode(priv->fbdev);
-}
-
 static struct drm_driver tve200_drm_driver = {
 	.driver_features =
 		DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_ATOMIC,
-	.lastclose = tve200_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.ioctls = NULL,
 	.fops = &drm_fops,
 	.name = "tve200",
@@ -270,8 +263,7 @@ static int tve200_remove(struct platform_device *pdev)
 	struct tve200_drm_dev_private *priv = drm->dev_private;
 
 	drm_dev_unregister(drm);
-	if (priv->fbdev)
-		drm_fbdev_cma_fini(priv->fbdev);
+	drm_fb_cma_fbdev_fini(drm);
 	if (priv->panel)
 		drm_panel_bridge_remove(priv->bridge);
 	drm_mode_config_cleanup(drm);
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index b5b335c9b2bb..2ebdc6d5a76e 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -159,10 +159,15 @@ static int udl_fb_mmap(struct fb_info *info, struct vm_area_struct *vma)
 {
 	unsigned long start = vma->vm_start;
 	unsigned long size = vma->vm_end - vma->vm_start;
-	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long offset;
 	unsigned long page, pos;
 
-	if (offset + size > info->fix.smem_len)
+	if (vma->vm_pgoff > (~0UL >> PAGE_SHIFT))
+		return -EINVAL;
+
+	offset = vma->vm_pgoff << PAGE_SHIFT;
+
+	if (offset > info->fix.smem_len || size > info->fix.smem_len - offset)
 		return -EINVAL;
 
 	pos = (unsigned long)info->fix.smem_start + offset;
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index e3c29729da2e..ceb385fd69c5 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -111,13 +111,6 @@ static int vc4_get_param_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
-static void vc4_lastclose(struct drm_device *dev)
-{
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-
-	drm_fbdev_cma_restore_mode(vc4->fbdev);
-}
-
 static const struct vm_operations_struct vc4_vm_ops = {
 	.fault = vc4_fault,
 	.open = drm_gem_vm_open,
@@ -159,7 +152,7 @@ static struct drm_driver vc4_drm_driver = {
 			    DRIVER_HAVE_IRQ |
 			    DRIVER_RENDER |
 			    DRIVER_PRIME),
-	.lastclose = vc4_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.irq_handler = vc4_irq,
 	.irq_preinstall = vc4_irq_preinstall,
 	.irq_postinstall = vc4_irq_postinstall,
@@ -301,12 +294,10 @@ static void vc4_drm_unbind(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct drm_device *drm = platform_get_drvdata(pdev);
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
 
 	drm_dev_unregister(drm);
 
-	if (vc4->fbdev)
-		drm_fbdev_cma_fini(vc4->fbdev);
+	drm_fb_cma_fbdev_fini(drm);
 
 	drm_mode_config_cleanup(drm);
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 9c0d380c96f2..3af22936d9b3 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -39,8 +39,6 @@ struct vc4_dev {
 	struct vc4_dsi *dsi1;
 	struct vc4_vec *vec;
 
-	struct drm_fbdev_cma *fbdev;
-
 	struct vc4_hang_state *hang_state;
 
 	/* The kernel-space BO cache.  Tracks buffers that have been
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index 0b2088264039..984501e3f0b0 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -287,7 +287,6 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 
 	drm_mode_connector_update_edid_property(connector, edid);
 	ret = drm_add_edid_modes(connector, edid);
-	drm_edid_to_eld(connector, edid);
 	kfree(edid);
 
 	return ret;
@@ -695,7 +694,22 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 	}
 }
 
+static enum drm_mode_status
+vc4_hdmi_encoder_mode_valid(struct drm_encoder *crtc,
+			    const struct drm_display_mode *mode)
+{
+	/* HSM clock must be 108% of the pixel clock.  Additionally,
+	 * the AXI clock needs to be at least 25% of pixel clock, but
+	 * HSM ends up being the limiting factor.
+	 */
+	if (mode->clock > HSM_CLOCK_FREQ / (1000 * 108 / 100))
+		return MODE_CLOCK_HIGH;
+
+	return MODE_OK;
+}
+
 static const struct drm_encoder_helper_funcs vc4_hdmi_encoder_helper_funcs = {
+	.mode_valid = vc4_hdmi_encoder_mode_valid,
 	.disable = vc4_hdmi_encoder_disable,
 	.enable = vc4_hdmi_encoder_enable,
 };
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index 50c4959b5bd3..4256f294c346 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -19,17 +19,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_plane_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include "vc4_drv.h"
 
-static void vc4_output_poll_changed(struct drm_device *dev)
-{
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-
-	drm_fbdev_cma_hotplug_event(vc4->fbdev);
-}
-
 static void
 vc4_atomic_complete_commit(struct drm_atomic_state *state)
 {
@@ -194,7 +188,7 @@ static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
 }
 
 static const struct drm_mode_config_funcs vc4_mode_funcs = {
-	.output_poll_changed = vc4_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = vc4_atomic_commit,
 	.fb_create = vc4_fb_create,
@@ -224,12 +218,8 @@ int vc4_kms_load(struct drm_device *dev)
 
 	drm_mode_config_reset(dev);
 
-	if (dev->mode_config.num_connector) {
-		vc4->fbdev = drm_fbdev_cma_init(dev, 32,
-						dev->mode_config.num_connector);
-		if (IS_ERR(vc4->fbdev))
-			vc4->fbdev = NULL;
-	}
+	if (dev->mode_config.num_connector)
+		drm_fb_cma_fbdev_init(dev, 32, 0);
 
 	drm_kms_helper_poll_init(dev);
 
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 423a23ed8fc2..515f97997624 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -86,7 +86,6 @@ static const struct hvs_format {
 	u32 hvs; /* HVS_FORMAT_* */
 	u32 pixel_order;
 	bool has_alpha;
-	bool flip_cbcr;
 } hvs_formats[] = {
 	{
 		.drm = DRM_FORMAT_XRGB8888, .hvs = HVS_PIXEL_FORMAT_RGBA8888,
@@ -120,31 +119,53 @@ static const struct hvs_format {
 		.drm = DRM_FORMAT_XRGB1555, .hvs = HVS_PIXEL_FORMAT_RGBA5551,
 		.pixel_order = HVS_PIXEL_ORDER_ABGR, .has_alpha = false,
 	},
+	{
+		.drm = DRM_FORMAT_RGB888, .hvs = HVS_PIXEL_FORMAT_RGB888,
+		.pixel_order = HVS_PIXEL_ORDER_XRGB, .has_alpha = false,
+	},
+	{
+		.drm = DRM_FORMAT_BGR888, .hvs = HVS_PIXEL_FORMAT_RGB888,
+		.pixel_order = HVS_PIXEL_ORDER_XBGR, .has_alpha = false,
+	},
 	{
 		.drm = DRM_FORMAT_YUV422,
 		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV422_3PLANE,
+		.pixel_order = HVS_PIXEL_ORDER_XYCBCR,
 	},
 	{
 		.drm = DRM_FORMAT_YVU422,
 		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV422_3PLANE,
-		.flip_cbcr = true,
+		.pixel_order = HVS_PIXEL_ORDER_XYCRCB,
 	},
 	{
 		.drm = DRM_FORMAT_YUV420,
 		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV420_3PLANE,
+		.pixel_order = HVS_PIXEL_ORDER_XYCBCR,
 	},
 	{
 		.drm = DRM_FORMAT_YVU420,
 		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV420_3PLANE,
-		.flip_cbcr = true,
+		.pixel_order = HVS_PIXEL_ORDER_XYCRCB,
 	},
 	{
 		.drm = DRM_FORMAT_NV12,
 		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV420_2PLANE,
+		.pixel_order = HVS_PIXEL_ORDER_XYCBCR,
+	},
+	{
+		.drm = DRM_FORMAT_NV21,
+		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV420_2PLANE,
+		.pixel_order = HVS_PIXEL_ORDER_XYCRCB,
 	},
 	{
 		.drm = DRM_FORMAT_NV16,
 		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV422_2PLANE,
+		.pixel_order = HVS_PIXEL_ORDER_XYCBCR,
+	},
+	{
+		.drm = DRM_FORMAT_NV61,
+		.hvs = HVS_PIXEL_FORMAT_YCBCR_YUV422_2PLANE,
+		.pixel_order = HVS_PIXEL_ORDER_XYCRCB,
 	},
 };
 
@@ -617,15 +638,8 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 	 * The pointers may be any byte address.
 	 */
 	vc4_state->ptr0_offset = vc4_state->dlist_count;
-	if (!format->flip_cbcr) {
-		for (i = 0; i < num_planes; i++)
-			vc4_dlist_write(vc4_state, vc4_state->offsets[i]);
-	} else {
-		WARN_ON_ONCE(num_planes != 3);
-		vc4_dlist_write(vc4_state, vc4_state->offsets[0]);
-		vc4_dlist_write(vc4_state, vc4_state->offsets[2]);
-		vc4_dlist_write(vc4_state, vc4_state->offsets[1]);
-	}
+	for (i = 0; i < num_planes; i++)
+		vc4_dlist_write(vc4_state, vc4_state->offsets[i]);
 
 	/* Pointer Context Word 0/1/2: Written by the HVS */
 	for (i = 0; i < num_planes; i++)
diff --git a/drivers/gpu/drm/via/via_drv.h b/drivers/gpu/drm/via/via_drv.h
index 9873942ca8f4..6d1ae834484c 100644
--- a/drivers/gpu/drm/via/via_drv.h
+++ b/drivers/gpu/drm/via/via_drv.h
@@ -74,9 +74,9 @@ typedef struct drm_via_private {
 	volatile uint32_t *last_pause_ptr;
 	volatile uint32_t *hw_addr_ptr;
 	drm_via_ring_buffer_t ring;
-	struct timeval last_vblank;
+	ktime_t last_vblank;
 	int last_vblank_valid;
-	unsigned usec_per_vblank;
+	ktime_t nsec_per_vblank;
 	atomic_t vbl_received;
 	drm_via_state_t hc_state;
 	char pci_buf[VIA_PCI_BUF_SIZE];
diff --git a/drivers/gpu/drm/via/via_irq.c b/drivers/gpu/drm/via/via_irq.c
index ea8172c747a2..c96830ccc0ec 100644
--- a/drivers/gpu/drm/via/via_irq.c
+++ b/drivers/gpu/drm/via/via_irq.c
@@ -88,13 +88,6 @@ static int via_num_unichrome = ARRAY_SIZE(via_unichrome_irqs);
 static int via_irqmap_unichrome[] = {-1, -1, -1, 0, -1, 1};
 
 
-static unsigned time_diff(struct timeval *now, struct timeval *then)
-{
-	return (now->tv_usec >= then->tv_usec) ?
-		now->tv_usec - then->tv_usec :
-		1000000 - (then->tv_usec - now->tv_usec);
-}
-
 u32 via_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 {
 	drm_via_private_t *dev_priv = dev->dev_private;
@@ -111,7 +104,7 @@ irqreturn_t via_driver_irq_handler(int irq, void *arg)
 	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
 	u32 status;
 	int handled = 0;
-	struct timeval cur_vblank;
+	ktime_t cur_vblank;
 	drm_via_irq_t *cur_irq = dev_priv->via_irqs;
 	int i;
 
@@ -119,18 +112,18 @@ irqreturn_t via_driver_irq_handler(int irq, void *arg)
 	if (status & VIA_IRQ_VBLANK_PENDING) {
 		atomic_inc(&dev_priv->vbl_received);
 		if (!(atomic_read(&dev_priv->vbl_received) & 0x0F)) {
-			do_gettimeofday(&cur_vblank);
+			cur_vblank = ktime_get();
 			if (dev_priv->last_vblank_valid) {
-				dev_priv->usec_per_vblank =
-					time_diff(&cur_vblank,
-						  &dev_priv->last_vblank) >> 4;
+				dev_priv->nsec_per_vblank =
+					ktime_sub(cur_vblank,
+						dev_priv->last_vblank) >> 4;
 			}
 			dev_priv->last_vblank = cur_vblank;
 			dev_priv->last_vblank_valid = 1;
 		}
 		if (!(atomic_read(&dev_priv->vbl_received) & 0xFF)) {
-			DRM_DEBUG("US per vblank is: %u\n",
-				  dev_priv->usec_per_vblank);
+			DRM_DEBUG("nsec per vblank is: %llu\n",
+				  ktime_to_ns(dev_priv->nsec_per_vblank));
 		}
 		drm_handle_vblank(dev, 0);
 		handled = 1;
@@ -350,7 +343,7 @@ void via_driver_irq_uninstall(struct drm_device *dev)
 int via_wait_irq(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	drm_via_irqwait_t *irqwait = data;
-	struct timeval now;
+	struct timespec64 now;
 	int ret = 0;
 	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
 	drm_via_irq_t *cur_irq = dev_priv->via_irqs;
@@ -384,9 +377,9 @@ int via_wait_irq(struct drm_device *dev, void *data, struct drm_file *file_priv)
 
 	ret = via_driver_irq_wait(dev, irqwait->request.irq, force_sequence,
 				  &irqwait->request.sequence);
-	do_gettimeofday(&now);
+	ktime_get_ts64(&now);
 	irqwait->reply.tval_sec = now.tv_sec;
-	irqwait->reply.tval_usec = now.tv_usec;
+	irqwait->reply.tval_usec = now.tv_nsec / NSEC_PER_USEC;
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 41b0930f7968..19114a3c5ee4 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -71,7 +71,19 @@ virtio_gpu_framebuffer_surface_dirty(struct drm_framebuffer *fb,
 	return virtio_gpu_surface_dirty(virtio_gpu_fb, clips, num_clips);
 }
 
+static int
+virtio_gpu_framebuffer_create_handle(struct drm_framebuffer *fb,
+				     struct drm_file *file_priv,
+				     unsigned int *handle)
+{
+	struct virtio_gpu_framebuffer *virtio_gpu_fb =
+		to_virtio_gpu_framebuffer(fb);
+
+	return drm_gem_handle_create(file_priv, virtio_gpu_fb->obj, handle);
+}
+
 static const struct drm_framebuffer_funcs virtio_gpu_fb_funcs = {
+	.create_handle = virtio_gpu_framebuffer_create_handle,
 	.destroy = virtio_gpu_user_framebuffer_destroy,
 	.dirty = virtio_gpu_framebuffer_surface_dirty,
 };
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index 0528edb4a2bf..677ac16c8a6d 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -56,6 +56,7 @@ static int virtio_gpu_map_ioctl(struct drm_device *dev, void *data,
 static int virtio_gpu_object_list_validate(struct ww_acquire_ctx *ticket,
 					   struct list_head *head)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	struct ttm_validate_buffer *buf;
 	struct ttm_buffer_object *bo;
 	struct virtio_gpu_object *qobj;
@@ -68,7 +69,7 @@ static int virtio_gpu_object_list_validate(struct ww_acquire_ctx *ticket,
 	list_for_each_entry(buf, head, head) {
 		bo = buf->bo;
 		qobj = container_of(bo, struct virtio_gpu_object, tbo);
-		ret = ttm_bo_validate(bo, &qobj->placement, false, false);
+		ret = ttm_bo_validate(bo, &qobj->placement, &ctx);
 		if (ret) {
 			ttm_eu_backoff_reservation(ticket, head);
 			return ret;
@@ -196,6 +197,9 @@ static int virtio_gpu_getparam_ioctl(struct drm_device *dev, void *data,
 	case VIRTGPU_PARAM_3D_FEATURES:
 		value = vgdev->has_virgl_3d == true ? 1 : 0;
 		break;
+	case VIRTGPU_PARAM_CAPSET_QUERY_FIX:
+		value = 1;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -261,7 +265,7 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data,
 		ret = virtio_gpu_object_attach(vgdev, qobj, res_id, NULL);
 	} else {
 		/* use a gem reference since unref list undoes them */
-		drm_gem_object_reference(&qobj->gem_base);
+		drm_gem_object_get(&qobj->gem_base);
 		mainbuf.bo = &qobj->tbo;
 		list_add(&mainbuf.head, &validate_list);
 
@@ -352,6 +356,7 @@ static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev,
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
 	struct drm_virtgpu_3d_transfer_from_host *args = data;
+	struct ttm_operation_ctx ctx = { true, false };
 	struct drm_gem_object *gobj = NULL;
 	struct virtio_gpu_object *qobj = NULL;
 	struct virtio_gpu_fence *fence;
@@ -372,8 +377,7 @@ static int virtio_gpu_transfer_from_host_ioctl(struct drm_device *dev,
 	if (ret)
 		goto out;
 
-	ret = ttm_bo_validate(&qobj->tbo, &qobj->placement,
-			      true, false);
+	ret = ttm_bo_validate(&qobj->tbo, &qobj->placement, &ctx);
 	if (unlikely(ret))
 		goto out_unres;
 
@@ -399,6 +403,7 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data,
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct virtio_gpu_fpriv *vfpriv = file->driver_priv;
 	struct drm_virtgpu_3d_transfer_to_host *args = data;
+	struct ttm_operation_ctx ctx = { true, false };
 	struct drm_gem_object *gobj = NULL;
 	struct virtio_gpu_object *qobj = NULL;
 	struct virtio_gpu_fence *fence;
@@ -416,8 +421,7 @@ static int virtio_gpu_transfer_to_host_ioctl(struct drm_device *dev, void *data,
 	if (ret)
 		goto out;
 
-	ret = ttm_bo_validate(&qobj->tbo, &qobj->placement,
-			      true, false);
+	ret = ttm_bo_validate(&qobj->tbo, &qobj->placement, &ctx);
 	if (unlikely(ret))
 		goto out_unres;
 
@@ -471,7 +475,7 @@ static int virtio_gpu_get_caps_ioctl(struct drm_device *dev,
 {
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct drm_virtgpu_get_caps *args = data;
-	int size;
+	unsigned size, host_caps_size;
 	int i;
 	int found_valid = -1;
 	int ret;
@@ -480,6 +484,10 @@ static int virtio_gpu_get_caps_ioctl(struct drm_device *dev,
 	if (vgdev->num_capsets == 0)
 		return -ENOSYS;
 
+	/* don't allow userspace to pass 0 */
+	if (args->size == 0)
+		return -EINVAL;
+
 	spin_lock(&vgdev->display_info_lock);
 	for (i = 0; i < vgdev->num_capsets; i++) {
 		if (vgdev->capsets[i].id == args->cap_set_id) {
@@ -495,11 +503,9 @@ static int virtio_gpu_get_caps_ioctl(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	size = vgdev->capsets[found_valid].max_size;
-	if (args->size > size) {
-		spin_unlock(&vgdev->display_info_lock);
-		return -EINVAL;
-	}
+	host_caps_size = vgdev->capsets[found_valid].max_size;
+	/* only copy to user the minimum of the host caps size or the guest caps size */
+	size = min(args->size, host_caps_size);
 
 	list_for_each_entry(cache_ent, &vgdev->cap_cache, head) {
 		if (cache_ent->id == args->cap_set_id &&
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 6400506a06b0..65060c08522d 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -272,20 +272,18 @@ int virtio_gpu_driver_open(struct drm_device *dev, struct drm_file *file)
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct virtio_gpu_fpriv *vfpriv;
 	uint32_t id;
-	char dbgname[64], tmpname[TASK_COMM_LEN];
+	char dbgname[TASK_COMM_LEN];
 
 	/* can't create contexts without 3d renderer */
 	if (!vgdev->has_virgl_3d)
 		return 0;
 
-	get_task_comm(tmpname, current);
-	snprintf(dbgname, sizeof(dbgname), "%s", tmpname);
-	dbgname[63] = 0;
 	/* allocate a virt GPU context for this opener */
 	vfpriv = kzalloc(sizeof(*vfpriv), GFP_KERNEL);
 	if (!vfpriv)
 		return -ENOMEM;
 
+	get_task_comm(dbgname, current);
 	virtio_gpu_context_create(vgdev, strlen(dbgname), dbgname, &id);
 
 	vfpriv->ctx_id = id;
diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c
index 6f66b7347cd0..0b90cdb3d9fe 100644
--- a/drivers/gpu/drm/virtio/virtgpu_object.c
+++ b/drivers/gpu/drm/virtio/virtgpu_object.c
@@ -124,13 +124,17 @@ int virtio_gpu_object_get_sg_table(struct virtio_gpu_device *qdev,
 	int ret;
 	struct page **pages = bo->tbo.ttm->pages;
 	int nr_pages = bo->tbo.num_pages;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
 
 	/* wtf swapping */
 	if (bo->pages)
 		return 0;
 
 	if (bo->tbo.ttm->state == tt_unpopulated)
-		bo->tbo.ttm->bdev->driver->ttm_tt_populate(bo->tbo.ttm);
+		bo->tbo.ttm->bdev->driver->ttm_tt_populate(bo->tbo.ttm, &ctx);
 	bo->pages = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
 	if (!bo->pages)
 		goto out;
diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index cd389c5eaef5..36655b709eb2 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -324,12 +324,13 @@ static struct ttm_backend_func virtio_gpu_backend_func = {
 	.destroy = &virtio_gpu_ttm_backend_destroy,
 };
 
-static int virtio_gpu_ttm_tt_populate(struct ttm_tt *ttm)
+static int virtio_gpu_ttm_tt_populate(struct ttm_tt *ttm,
+		struct ttm_operation_ctx *ctx)
 {
 	if (ttm->state != tt_unpopulated)
 		return 0;
 
-	return ttm_pool_populate(ttm);
+	return ttm_pool_populate(ttm, ctx);
 }
 
 static void virtio_gpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
@@ -369,14 +370,13 @@ static void virtio_gpu_move_null(struct ttm_buffer_object *bo,
 	new_mem->mm_node = NULL;
 }
 
-static int virtio_gpu_bo_move(struct ttm_buffer_object *bo,
-			      bool evict, bool interruptible,
-			      bool no_wait_gpu,
+static int virtio_gpu_bo_move(struct ttm_buffer_object *bo, bool evict,
+			      struct ttm_operation_ctx *ctx,
 			      struct ttm_mem_reg *new_mem)
 {
 	int ret;
 
-	ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+	ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
 	if (ret)
 		return ret;
 
@@ -431,7 +431,6 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = {
 	.verify_access = &virtio_gpu_verify_access,
 	.io_mem_reserve = &virtio_gpu_ttm_io_mem_reserve,
 	.io_mem_free = &virtio_gpu_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 	.move_notify = &virtio_gpu_bo_move_notify,
 	.swap_notify = &virtio_gpu_bo_swap_notify,
 };
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
index 9c42e96da510..55d32ae43aa4 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
@@ -1202,10 +1202,14 @@ struct vmw_ctx_binding_state *
 vmw_binding_state_alloc(struct vmw_private *dev_priv)
 {
 	struct vmw_ctx_binding_state *cbs;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
 	int ret;
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv), sizeof(*cbs),
-				   false, false);
+				&ctx);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index c7056322211c..22231bc9e845 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -394,6 +394,10 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
 	struct vmw_private *dev_priv = vmw_tt->dev_priv;
 	struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
 	struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	struct vmw_piter iter;
 	dma_addr_t old;
 	int ret = 0;
@@ -417,8 +421,7 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
 			sgt_size = ttm_round_pot(sizeof(struct sg_table));
 		}
 		vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages;
-		ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, false,
-					   true);
+		ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, &ctx);
 		if (unlikely(ret != 0))
 			return ret;
 
@@ -632,7 +635,7 @@ static void vmw_ttm_destroy(struct ttm_tt *ttm)
 }
 
 
-static int vmw_ttm_populate(struct ttm_tt *ttm)
+static int vmw_ttm_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
 {
 	struct vmw_ttm_tt *vmw_tt =
 		container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
@@ -646,15 +649,16 @@ static int vmw_ttm_populate(struct ttm_tt *ttm)
 	if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
 		size_t size =
 			ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
-		ret = ttm_mem_global_alloc(glob, size, false, true);
+		ret = ttm_mem_global_alloc(glob, size, ctx);
 		if (unlikely(ret != 0))
 			return ret;
 
-		ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
+		ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev,
+					ctx);
 		if (unlikely(ret != 0))
 			ttm_mem_global_free(glob, size);
 	} else
-		ret = ttm_pool_populate(ttm);
+		ret = ttm_pool_populate(ttm, ctx);
 
 	return ret;
 }
@@ -859,5 +863,4 @@ struct ttm_bo_driver vmw_bo_driver = {
 	.fault_reserve_notify = &vmw_ttm_fault_reserve_notify,
 	.io_mem_reserve = &vmw_ttm_io_mem_reserve,
 	.io_mem_free = &vmw_ttm_io_mem_free,
-	.io_mem_pfn = ttm_bo_default_io_mem_pfn,
 };
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
index 4212b3e673bc..3767ac335aca 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
@@ -746,6 +746,10 @@ static int vmw_context_define(struct drm_device *dev, void *data,
 	struct vmw_resource *tmp;
 	struct drm_vmw_context_arg *arg = (struct drm_vmw_context_arg *)data;
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	struct ttm_operation_ctx ttm_opt_ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	int ret;
 
 	if (!dev_priv->has_dx && dx) {
@@ -768,7 +772,7 @@ static int vmw_context_define(struct drm_device *dev, void *data,
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv),
 				   vmw_user_context_size,
-				   false, true);
+				   &ttm_opt_ctx);
 	if (unlikely(ret != 0)) {
 		if (ret != -ERESTARTSYS)
 			DRM_ERROR("Out of graphics memory for context"
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
index d87861bbe971..cbf54ea7b4c0 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
@@ -387,6 +387,7 @@ static int vmw_cotable_readback(struct vmw_resource *res)
  */
 static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 {
+	struct ttm_operation_ctx ctx = { false, false };
 	struct vmw_private *dev_priv = res->dev_priv;
 	struct vmw_cotable *vcotbl = vmw_cotable(res);
 	struct vmw_dma_buffer *buf, *old_buf = res->backup;
@@ -455,7 +456,7 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size)
 	}
 
 	/* Unpin new buffer, and switch backup buffers. */
-	ret = ttm_bo_validate(bo, &vmw_mob_placement, false, false);
+	ret = ttm_bo_validate(bo, &vmw_mob_placement, &ctx);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Failed validating new COTable backup buffer.\n");
 		goto out_wait;
@@ -572,6 +573,10 @@ struct vmw_resource *vmw_cotable_alloc(struct vmw_private *dev_priv,
 				       u32 type)
 {
 	struct vmw_cotable *vcotbl;
+	struct ttm_operation_ctx ttm_opt_ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	int ret;
 	u32 num_entries;
 
@@ -579,7 +584,7 @@ struct vmw_resource *vmw_cotable_alloc(struct vmw_private *dev_priv,
 		cotable_acc_size = ttm_round_pot(sizeof(struct vmw_cotable));
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv),
-				   cotable_acc_size, false, true);
+				   cotable_acc_size, &ttm_opt_ctx);
 	if (unlikely(ret))
 		return ERR_PTR(ret);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
index 0cd889015dc5..d45d2caffa5a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
@@ -47,6 +47,7 @@ int vmw_dmabuf_pin_in_placement(struct vmw_private *dev_priv,
 				struct ttm_placement *placement,
 				bool interruptible)
 {
+	struct ttm_operation_ctx ctx = {interruptible, false };
 	struct ttm_buffer_object *bo = &buf->base;
 	int ret;
 	uint32_t new_flags;
@@ -65,7 +66,7 @@ int vmw_dmabuf_pin_in_placement(struct vmw_private *dev_priv,
 		ret = ttm_bo_mem_compat(placement, &bo->mem,
 					&new_flags) == true ? 0 : -EINVAL;
 	else
-		ret = ttm_bo_validate(bo, placement, interruptible, false);
+		ret = ttm_bo_validate(bo, placement, &ctx);
 
 	if (!ret)
 		vmw_bo_pin_reserved(buf, true);
@@ -95,6 +96,7 @@ int vmw_dmabuf_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
 				  struct vmw_dma_buffer *buf,
 				  bool interruptible)
 {
+	struct ttm_operation_ctx ctx = {interruptible, false };
 	struct ttm_buffer_object *bo = &buf->base;
 	int ret;
 	uint32_t new_flags;
@@ -115,12 +117,11 @@ int vmw_dmabuf_pin_in_vram_or_gmr(struct vmw_private *dev_priv,
 		goto out_unreserve;
 	}
 
-	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, interruptible,
-			      false);
+	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, &ctx);
 	if (likely(ret == 0) || ret == -ERESTARTSYS)
 		goto out_unreserve;
 
-	ret = ttm_bo_validate(bo, &vmw_vram_placement, interruptible, false);
+	ret = ttm_bo_validate(bo, &vmw_vram_placement, &ctx);
 
 out_unreserve:
 	if (!ret)
@@ -170,6 +171,7 @@ int vmw_dmabuf_pin_in_start_of_vram(struct vmw_private *dev_priv,
 				    struct vmw_dma_buffer *buf,
 				    bool interruptible)
 {
+	struct ttm_operation_ctx ctx = {interruptible, false };
 	struct ttm_buffer_object *bo = &buf->base;
 	struct ttm_placement placement;
 	struct ttm_place place;
@@ -200,14 +202,16 @@ int vmw_dmabuf_pin_in_start_of_vram(struct vmw_private *dev_priv,
 	if (bo->mem.mem_type == TTM_PL_VRAM &&
 	    bo->mem.start < bo->num_pages &&
 	    bo->mem.start > 0 &&
-	    buf->pin_count == 0)
-		(void) ttm_bo_validate(bo, &vmw_sys_placement, false, false);
+	    buf->pin_count == 0) {
+		ctx.interruptible = false;
+		(void) ttm_bo_validate(bo, &vmw_sys_placement, &ctx);
+	}
 
 	if (buf->pin_count > 0)
 		ret = ttm_bo_mem_compat(&placement, &bo->mem,
 					&new_flags) == true ? 0 : -EINVAL;
 	else
-		ret = ttm_bo_validate(bo, &placement, interruptible, false);
+		ret = ttm_bo_validate(bo, &placement, &ctx);
 
 	/* For some reason we didn't end up at the start of vram */
 	WARN_ON(ret == 0 && bo->offset != 0);
@@ -286,6 +290,7 @@ void vmw_bo_get_guest_ptr(const struct ttm_buffer_object *bo,
  */
 void vmw_bo_pin_reserved(struct vmw_dma_buffer *vbo, bool pin)
 {
+	struct ttm_operation_ctx ctx = { false, true };
 	struct ttm_place pl;
 	struct ttm_placement placement;
 	struct ttm_buffer_object *bo = &vbo->base;
@@ -314,7 +319,7 @@ void vmw_bo_pin_reserved(struct vmw_dma_buffer *vbo, bool pin)
 	placement.num_placement = 1;
 	placement.placement = &pl;
 
-	ret = ttm_bo_validate(bo, &placement, false, true);
+	ret = ttm_bo_validate(bo, &placement, &ctx);
 
 	BUG_ON(ret != 0 || bo->mem.mem_type != old_mem_type);
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 184340d486c3..86d25f18aa99 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -1337,6 +1337,19 @@ static void __vmw_svga_disable(struct vmw_private *dev_priv)
  */
 void vmw_svga_disable(struct vmw_private *dev_priv)
 {
+	/*
+	 * Disabling SVGA will turn off device modesetting capabilities, so
+	 * notify KMS about that so that it doesn't cache atomic state that
+	 * isn't valid anymore, for example crtcs turned on.
+	 * Strictly we'd want to do this under the SVGA lock (or an SVGA mutex),
+	 * but vmw_kms_lost_device() takes the reservation sem and thus we'll
+	 * end up with lock order reversal. Thus, a master may actually perform
+	 * a new modeset just after we call vmw_kms_lost_device() and race with
+	 * vmw_svga_disable(), but that should at worst cause atomic KMS state
+	 * to be inconsistent with the device, causing modesetting problems.
+	 *
+	 */
+	vmw_kms_lost_device(dev_priv->dev);
 	ttm_write_lock(&dev_priv->reservation_sem, false);
 	spin_lock(&dev_priv->svga_lock);
 	if (dev_priv->bdev.man[TTM_PL_VRAM].use_type) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 7e5f30e234b1..9116fe8baebc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -713,7 +713,7 @@ extern int vmw_present_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_priv);
 extern int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
 				      struct drm_file *file_priv);
-extern unsigned int vmw_fops_poll(struct file *filp,
+extern __poll_t vmw_fops_poll(struct file *filp,
 				  struct poll_table_struct *wait);
 extern ssize_t vmw_fops_read(struct file *filp, char __user *buffer,
 			     size_t count, loff_t *offset);
@@ -938,6 +938,7 @@ int vmw_kms_present(struct vmw_private *dev_priv,
 int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file_priv);
 void vmw_kms_legacy_hotspot_clear(struct vmw_private *dev_priv);
+void vmw_kms_lost_device(struct drm_device *dev);
 
 int vmw_dumb_create(struct drm_file *file_priv,
 		    struct drm_device *dev,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 87e8af5776a3..c9d5cc237124 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -3703,14 +3703,14 @@ int vmw_validate_single_buffer(struct vmw_private *dev_priv,
 {
 	struct vmw_dma_buffer *vbo = container_of(bo, struct vmw_dma_buffer,
 						  base);
+	struct ttm_operation_ctx ctx = { interruptible, true };
 	int ret;
 
 	if (vbo->pin_count > 0)
 		return 0;
 
 	if (validate_as_mob)
-		return ttm_bo_validate(bo, &vmw_mob_placement, interruptible,
-				       false);
+		return ttm_bo_validate(bo, &vmw_mob_placement, &ctx);
 
 	/**
 	 * Put BO in VRAM if there is space, otherwise as a GMR.
@@ -3719,8 +3719,7 @@ int vmw_validate_single_buffer(struct vmw_private *dev_priv,
 	 * used as a GMR, this will return -ENOMEM.
 	 */
 
-	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, interruptible,
-			      false);
+	ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, &ctx);
 	if (likely(ret == 0 || ret == -ERESTARTSYS))
 		return ret;
 
@@ -3729,7 +3728,7 @@ int vmw_validate_single_buffer(struct vmw_private *dev_priv,
 	 * previous contents.
 	 */
 
-	ret = ttm_bo_validate(bo, &vmw_vram_placement, interruptible, false);
+	ret = ttm_bo_validate(bo, &vmw_vram_placement, &ctx);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index d6b1c509ae01..6c5c75cf5e6c 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -588,6 +588,10 @@ int vmw_user_fence_create(struct drm_file *file_priv,
 	struct vmw_user_fence *ufence;
 	struct vmw_fence_obj *tmp;
 	struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
 	int ret;
 
 	/*
@@ -596,7 +600,7 @@ int vmw_user_fence_create(struct drm_file *file_priv,
 	 */
 
 	ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size,
-				   false, false);
+				   &ctx);
 	if (unlikely(ret != 0))
 		return ret;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index 01be355525e4..67f844678ac8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -412,7 +412,7 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
  * Wrapper around the drm_poll function that makes sure the device is
  * processing the fifo if drm_poll decides to wait.
  */
-unsigned int vmw_fops_poll(struct file *filp, struct poll_table_struct *wait)
+__poll_t vmw_fops_poll(struct file *filp, struct poll_table_struct *wait)
 {
 	struct drm_file *file_priv = filp->private_data;
 	struct vmw_private *dev_priv =
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index fcd58145d0da..3c824fd7cbf3 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -31,7 +31,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_rect.h>
 
-
 /* Might need a hrtimer here? */
 #define VMWGFX_PRESENT_RATE ((HZ / 60 > 0) ? HZ / 60 : 1)
 
@@ -441,31 +440,23 @@ vmw_du_cursor_plane_atomic_update(struct drm_plane *plane,
 int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
 				      struct drm_plane_state *state)
 {
+	struct drm_crtc_state *crtc_state = NULL;
 	struct drm_framebuffer *new_fb = state->fb;
-	bool visible;
-
-	struct drm_rect src = {
-		.x1 = state->src_x,
-		.y1 = state->src_y,
-		.x2 = state->src_x + state->src_w,
-		.y2 = state->src_y + state->src_h,
-	};
-	struct drm_rect dest = {
-		.x1 = state->crtc_x,
-		.y1 = state->crtc_y,
-		.x2 = state->crtc_x + state->crtc_w,
-		.y2 = state->crtc_y + state->crtc_h,
-	};
-	struct drm_rect clip = dest;
+	struct drm_rect clip = {};
 	int ret;
 
-	ret = drm_plane_helper_check_update(plane, state->crtc, new_fb,
-					    &src, &dest, &clip,
-					    DRM_MODE_ROTATE_0,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    false, true, &visible);
+	if (state->crtc)
+		crtc_state = drm_atomic_get_new_crtc_state(state->state, state->crtc);
+
+	if (crtc_state && crtc_state->enable) {
+		clip.x2 = crtc_state->adjusted_mode.hdisplay;
+		clip.y2 = crtc_state->adjusted_mode.vdisplay;
+	}
 
+	ret = drm_atomic_helper_check_plane_state(state, crtc_state, &clip,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_HELPER_NO_SCALING,
+						  false, true);
 
 	if (!ret && new_fb) {
 		struct drm_crtc *crtc = state->crtc;
@@ -476,12 +467,6 @@ int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
 
 		vcs = vmw_connector_state_to_vcs(du->connector.state);
 
-		if ((dest.x2 > new_fb->width ||
-		     dest.y2 > new_fb->height)) {
-			DRM_ERROR("CRTC area outside of framebuffer\n");
-			return -EINVAL;
-		}
-
 		/* Only one active implicit framebuffer at a time. */
 		mutex_lock(&dev_priv->global_kms_state_mutex);
 		if (vcs->is_implicit && dev_priv->implicit_fb &&
@@ -2531,9 +2516,12 @@ void vmw_kms_helper_buffer_finish(struct vmw_private *dev_priv,
  * Helper to be used if an error forces the caller to undo the actions of
  * vmw_kms_helper_resource_prepare.
  */
-void vmw_kms_helper_resource_revert(struct vmw_resource *res)
+void vmw_kms_helper_resource_revert(struct vmw_validation_ctx *ctx)
 {
-	vmw_kms_helper_buffer_revert(res->backup);
+	struct vmw_resource *res = ctx->res;
+
+	vmw_kms_helper_buffer_revert(ctx->buf);
+	vmw_dmabuf_unreference(&ctx->buf);
 	vmw_resource_unreserve(res, false, NULL, 0);
 	mutex_unlock(&res->dev_priv->cmdbuf_mutex);
 }
@@ -2550,10 +2538,14 @@ void vmw_kms_helper_resource_revert(struct vmw_resource *res)
  * interrupted by a signal.
  */
 int vmw_kms_helper_resource_prepare(struct vmw_resource *res,
-				    bool interruptible)
+				    bool interruptible,
+				    struct vmw_validation_ctx *ctx)
 {
 	int ret = 0;
 
+	ctx->buf = NULL;
+	ctx->res = res;
+
 	if (interruptible)
 		ret = mutex_lock_interruptible(&res->dev_priv->cmdbuf_mutex);
 	else
@@ -2572,6 +2564,8 @@ int vmw_kms_helper_resource_prepare(struct vmw_resource *res,
 						    res->dev_priv->has_mob);
 		if (ret)
 			goto out_unreserve;
+
+		ctx->buf = vmw_dmabuf_reference(res->backup);
 	}
 	ret = vmw_resource_validate(res);
 	if (ret)
@@ -2579,7 +2573,7 @@ int vmw_kms_helper_resource_prepare(struct vmw_resource *res,
 	return 0;
 
 out_revert:
-	vmw_kms_helper_buffer_revert(res->backup);
+	vmw_kms_helper_buffer_revert(ctx->buf);
 out_unreserve:
 	vmw_resource_unreserve(res, false, NULL, 0);
 out_unlock:
@@ -2595,11 +2589,13 @@ int vmw_kms_helper_resource_prepare(struct vmw_resource *res,
  * @out_fence: Optional pointer to a fence pointer. If non-NULL, a
  * ref-counted fence pointer is returned here.
  */
-void vmw_kms_helper_resource_finish(struct vmw_resource *res,
-			     struct vmw_fence_obj **out_fence)
+void vmw_kms_helper_resource_finish(struct vmw_validation_ctx *ctx,
+				    struct vmw_fence_obj **out_fence)
 {
-	if (res->backup || out_fence)
-		vmw_kms_helper_buffer_finish(res->dev_priv, NULL, res->backup,
+	struct vmw_resource *res = ctx->res;
+
+	if (ctx->buf || out_fence)
+		vmw_kms_helper_buffer_finish(res->dev_priv, NULL, ctx->buf,
 					     out_fence, NULL);
 
 	vmw_resource_unreserve(res, false, NULL, 0);
@@ -2865,3 +2861,14 @@ int vmw_kms_set_config(struct drm_mode_set *set,
 
 	return drm_atomic_helper_set_config(set, ctx);
 }
+
+
+/**
+ * vmw_kms_lost_device - Notify kms that modesetting capabilities will be lost
+ *
+ * @dev: Pointer to the drm device
+ */
+void vmw_kms_lost_device(struct drm_device *dev)
+{
+	drm_atomic_helper_shutdown(dev);
+}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index cd9da2dd79af..3d2ca280eaa7 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -240,6 +240,11 @@ struct vmw_display_unit {
 	int set_gui_y;
 };
 
+struct vmw_validation_ctx {
+	struct vmw_resource *res;
+	struct vmw_dma_buffer *buf;
+};
+
 #define vmw_crtc_to_du(x) \
 	container_of(x, struct vmw_display_unit, crtc)
 #define vmw_connector_to_du(x) \
@@ -296,9 +301,10 @@ void vmw_kms_helper_buffer_finish(struct vmw_private *dev_priv,
 				  struct drm_vmw_fence_rep __user *
 				  user_fence_rep);
 int vmw_kms_helper_resource_prepare(struct vmw_resource *res,
-				    bool interruptible);
-void vmw_kms_helper_resource_revert(struct vmw_resource *res);
-void vmw_kms_helper_resource_finish(struct vmw_resource *res,
+				    bool interruptible,
+				    struct vmw_validation_ctx *ctx);
+void vmw_kms_helper_resource_revert(struct vmw_validation_ctx *ctx);
+void vmw_kms_helper_resource_finish(struct vmw_validation_ctx *ctx,
 				    struct vmw_fence_obj **out_fence);
 int vmw_kms_readback(struct vmw_private *dev_priv,
 		     struct drm_file *file_priv,
@@ -439,5 +445,4 @@ int vmw_kms_stdu_dma(struct vmw_private *dev_priv,
 
 int vmw_kms_set_config(struct drm_mode_set *set,
 		       struct drm_modeset_acquire_ctx *ctx);
-
 #endif
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
index b17f08fc50d3..736ca47e28ea 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
@@ -240,6 +240,10 @@ static int vmw_otable_batch_setup(struct vmw_private *dev_priv,
 	unsigned long offset;
 	unsigned long bo_size;
 	struct vmw_otable *otables = batch->otables;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
 	SVGAOTableType i;
 	int ret;
 
@@ -264,7 +268,7 @@ static int vmw_otable_batch_setup(struct vmw_private *dev_priv,
 
 	ret = ttm_bo_reserve(batch->otable_bo, false, true, NULL);
 	BUG_ON(ret != 0);
-	ret = vmw_bo_driver.ttm_tt_populate(batch->otable_bo->ttm);
+	ret = vmw_bo_driver.ttm_tt_populate(batch->otable_bo->ttm, &ctx);
 	if (unlikely(ret != 0))
 		goto out_unreserve;
 	ret = vmw_bo_map_dma(batch->otable_bo);
@@ -430,6 +434,11 @@ static int vmw_mob_pt_populate(struct vmw_private *dev_priv,
 			       struct vmw_mob *mob)
 {
 	int ret;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = false,
+		.no_wait_gpu = false
+	};
+
 	BUG_ON(mob->pt_bo != NULL);
 
 	ret = ttm_bo_create(&dev_priv->bdev, mob->num_pages * PAGE_SIZE,
@@ -442,7 +451,7 @@ static int vmw_mob_pt_populate(struct vmw_private *dev_priv,
 	ret = ttm_bo_reserve(mob->pt_bo, false, true, NULL);
 
 	BUG_ON(ret != 0);
-	ret = vmw_bo_driver.ttm_tt_populate(mob->pt_bo->ttm);
+	ret = vmw_bo_driver.ttm_tt_populate(mob->pt_bo->ttm, &ctx);
 	if (unlikely(ret != 0))
 		goto out_unreserve;
 	ret = vmw_bo_map_dma(mob->pt_bo);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index a96f90f017d1..200904ff9a22 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -968,6 +968,7 @@ vmw_resource_check_buffer(struct vmw_resource *res,
 			  bool interruptible,
 			  struct ttm_validate_buffer *val_buf)
 {
+	struct ttm_operation_ctx ctx = { true, false };
 	struct list_head val_list;
 	bool backup_dirty = false;
 	int ret;
@@ -992,7 +993,7 @@ vmw_resource_check_buffer(struct vmw_resource *res,
 	backup_dirty = res->backup_dirty;
 	ret = ttm_bo_validate(&res->backup->base,
 			      res->func->backup_placement,
-			      true, false);
+			      &ctx);
 
 	if (unlikely(ret != 0))
 		goto out_no_validate;
@@ -1446,6 +1447,7 @@ void vmw_resource_evict_all(struct vmw_private *dev_priv)
  */
 int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 {
+	struct ttm_operation_ctx ctx = { interruptible, false };
 	struct vmw_private *dev_priv = res->dev_priv;
 	int ret;
 
@@ -1466,7 +1468,7 @@ int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 				ret = ttm_bo_validate
 					(&vbo->base,
 					 res->func->backup_placement,
-					 interruptible, false);
+					 &ctx);
 				if (ret) {
 					ttm_bo_unreserve(&vbo->base);
 					goto out_no_validate;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index 63a4cd794b73..3ec9eae831b8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -909,12 +909,13 @@ int vmw_kms_sou_do_surface_dirty(struct vmw_private *dev_priv,
 	struct vmw_framebuffer_surface *vfbs =
 		container_of(framebuffer, typeof(*vfbs), base);
 	struct vmw_kms_sou_surface_dirty sdirty;
+	struct vmw_validation_ctx ctx;
 	int ret;
 
 	if (!srf)
 		srf = &vfbs->surface->res;
 
-	ret = vmw_kms_helper_resource_prepare(srf, true);
+	ret = vmw_kms_helper_resource_prepare(srf, true, &ctx);
 	if (ret)
 		return ret;
 
@@ -933,7 +934,7 @@ int vmw_kms_sou_do_surface_dirty(struct vmw_private *dev_priv,
 	ret = vmw_kms_helper_dirty(dev_priv, framebuffer, clips, vclips,
 				   dest_x, dest_y, num_clips, inc,
 				   &sdirty.base);
-	vmw_kms_helper_resource_finish(srf, out_fence);
+	vmw_kms_helper_resource_finish(&ctx, out_fence);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
index 9b832f136813..73b8e9a16368 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
@@ -607,6 +607,10 @@ int vmw_dx_shader_add(struct vmw_cmdbuf_res_manager *man,
 	struct vmw_dx_shader *shader;
 	struct vmw_resource *res;
 	struct vmw_private *dev_priv = ctx->dev_priv;
+	struct ttm_operation_ctx ttm_opt_ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	int ret;
 
 	if (!vmw_shader_dx_size)
@@ -616,7 +620,7 @@ int vmw_dx_shader_add(struct vmw_cmdbuf_res_manager *man,
 		return -EINVAL;
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv), vmw_shader_dx_size,
-				   false, true);
+				   &ttm_opt_ctx);
 	if (ret) {
 		if (ret != -ERESTARTSYS)
 			DRM_ERROR("Out of graphics memory for shader "
@@ -730,6 +734,10 @@ static int vmw_user_shader_alloc(struct vmw_private *dev_priv,
 {
 	struct vmw_user_shader *ushader;
 	struct vmw_resource *res, *tmp;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	int ret;
 
 	/*
@@ -742,7 +750,7 @@ static int vmw_user_shader_alloc(struct vmw_private *dev_priv,
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv),
 				   vmw_user_shader_size,
-				   false, true);
+				   &ctx);
 	if (unlikely(ret != 0)) {
 		if (ret != -ERESTARTSYS)
 			DRM_ERROR("Out of graphics memory for shader "
@@ -800,6 +808,10 @@ static struct vmw_resource *vmw_shader_alloc(struct vmw_private *dev_priv,
 {
 	struct vmw_shader *shader;
 	struct vmw_resource *res;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	int ret;
 
 	/*
@@ -812,7 +824,7 @@ static struct vmw_resource *vmw_shader_alloc(struct vmw_private *dev_priv,
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv),
 				   vmw_shader_size,
-				   false, true);
+				   &ctx);
 	if (unlikely(ret != 0)) {
 		if (ret != -ERESTARTSYS)
 			DRM_ERROR("Out of graphics memory for shader "
@@ -970,6 +982,7 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv,
 			  size_t size,
 			  struct list_head *list)
 {
+	struct ttm_operation_ctx ctx = { false, true };
 	struct vmw_dma_buffer *buf;
 	struct ttm_bo_kmap_obj map;
 	bool is_iomem;
@@ -1005,7 +1018,7 @@ int vmw_compat_shader_add(struct vmw_private *dev_priv,
 	WARN_ON(is_iomem);
 
 	ttm_bo_kunmap(&map);
-	ret = ttm_bo_validate(&buf->base, &vmw_sys_placement, false, true);
+	ret = ttm_bo_validate(&buf->base, &vmw_sys_placement, &ctx);
 	WARN_ON(ret != 0);
 	ttm_bo_unreserve(&buf->base);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
index 051d3b39b0ea..a0cb310665cc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
@@ -149,6 +149,10 @@ vmw_simple_resource_create_ioctl(struct drm_device *dev, void *data,
 	struct vmw_resource *res;
 	struct vmw_resource *tmp;
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	size_t alloc_size;
 	size_t account_size;
 	int ret;
@@ -162,7 +166,7 @@ vmw_simple_resource_create_ioctl(struct drm_device *dev, void *data,
 		return ret;
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv), account_size,
-				   false, true);
+				   &ctx);
 	ttm_read_unlock(&dev_priv->reservation_sem);
 	if (ret) {
 		if (ret != -ERESTARTSYS)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_so.c b/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
index 5a73eebd0f35..d3573c37c436 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
@@ -329,6 +329,10 @@ int vmw_view_add(struct vmw_cmdbuf_res_manager *man,
 	struct vmw_private *dev_priv = ctx->dev_priv;
 	struct vmw_resource *res;
 	struct vmw_view *view;
+	struct ttm_operation_ctx ttm_opt_ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	size_t size;
 	int ret;
 
@@ -345,7 +349,7 @@ int vmw_view_add(struct vmw_cmdbuf_res_manager *man,
 
 	size = offsetof(struct vmw_view, cmd) + cmd_size;
 
-	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv), size, false, true);
+	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv), size, &ttm_opt_ctx);
 	if (ret) {
 		if (ret != -ERESTARTSYS)
 			DRM_ERROR("Out of graphics memory for view"
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index b68d74888ab1..6b969e5dea2a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -980,12 +980,13 @@ int vmw_kms_stdu_surface_dirty(struct vmw_private *dev_priv,
 	struct vmw_framebuffer_surface *vfbs =
 		container_of(framebuffer, typeof(*vfbs), base);
 	struct vmw_stdu_dirty sdirty;
+	struct vmw_validation_ctx ctx;
 	int ret;
 
 	if (!srf)
 		srf = &vfbs->surface->res;
 
-	ret = vmw_kms_helper_resource_prepare(srf, true);
+	ret = vmw_kms_helper_resource_prepare(srf, true, &ctx);
 	if (ret)
 		return ret;
 
@@ -1008,7 +1009,7 @@ int vmw_kms_stdu_surface_dirty(struct vmw_private *dev_priv,
 				   dest_x, dest_y, num_clips, inc,
 				   &sdirty.base);
 out_finish:
-	vmw_kms_helper_resource_finish(srf, out_fence);
+	vmw_kms_helper_resource_finish(&ctx, out_fence);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index 6ac094ee8983..db1bb166845e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -700,6 +700,10 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 	struct drm_vmw_surface_create_req *req = &arg->req;
 	struct drm_vmw_surface_arg *rep = &arg->rep;
 	struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	int ret;
 	int i, j;
 	uint32_t cur_bo_offset;
@@ -741,7 +745,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 		return ret;
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv),
-				   size, false, true);
+				   size, &ctx);
 	if (unlikely(ret != 0)) {
 		if (ret != -ERESTARTSYS)
 			DRM_ERROR("Out of graphics memory for surface"
@@ -1479,6 +1483,10 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 {
 	struct vmw_private *dev_priv = vmw_priv(dev);
 	struct vmw_user_surface *user_srf;
+	struct ttm_operation_ctx ctx = {
+		.interruptible = true,
+		.no_wait_gpu = false
+	};
 	struct vmw_surface *srf;
 	int ret;
 	u32 num_layers;
@@ -1525,7 +1533,7 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 		return ret;
 
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv),
-				   user_accounting_size, false, true);
+				   user_accounting_size, &ctx);
 	if (unlikely(ret != 0)) {
 		if (ret != -ERESTARTSYS)
 			DRM_ERROR("Out of graphics memory for surface"
diff --git a/drivers/gpu/drm/zte/zx_drm_drv.c b/drivers/gpu/drm/zte/zx_drm_drv.c
index e8b8266c0cde..6f4205e80378 100644
--- a/drivers/gpu/drm/zte/zx_drm_drv.c
+++ b/drivers/gpu/drm/zte/zx_drm_drv.c
@@ -29,37 +29,19 @@
 #include "zx_drm_drv.h"
 #include "zx_vou.h"
 
-struct zx_drm_private {
-	struct drm_fbdev_cma *fbdev;
-};
-
-static void zx_drm_fb_output_poll_changed(struct drm_device *drm)
-{
-	struct zx_drm_private *priv = drm->dev_private;
-
-	drm_fbdev_cma_hotplug_event(priv->fbdev);
-}
-
 static const struct drm_mode_config_funcs zx_drm_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
-	.output_poll_changed = zx_drm_fb_output_poll_changed,
+	.output_poll_changed = drm_fb_helper_output_poll_changed,
 	.atomic_check = drm_atomic_helper_check,
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-static void zx_drm_lastclose(struct drm_device *drm)
-{
-	struct zx_drm_private *priv = drm->dev_private;
-
-	drm_fbdev_cma_restore_mode(priv->fbdev);
-}
-
 DEFINE_DRM_GEM_CMA_FOPS(zx_drm_fops);
 
 static struct drm_driver zx_drm_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
 			   DRIVER_ATOMIC,
-	.lastclose = zx_drm_lastclose,
+	.lastclose = drm_fb_helper_lastclose,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops = &drm_gem_cma_vm_ops,
 	.dumb_create = drm_gem_cma_dumb_create,
@@ -83,18 +65,12 @@ static struct drm_driver zx_drm_driver = {
 static int zx_drm_bind(struct device *dev)
 {
 	struct drm_device *drm;
-	struct zx_drm_private *priv;
 	int ret;
 
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
 	drm = drm_dev_alloc(&zx_drm_driver, dev);
 	if (IS_ERR(drm))
 		return PTR_ERR(drm);
 
-	drm->dev_private = priv;
 	dev_set_drvdata(dev, drm);
 
 	drm_mode_config_init(drm);
@@ -125,12 +101,9 @@ static int zx_drm_bind(struct device *dev)
 	drm_mode_config_reset(drm);
 	drm_kms_helper_poll_init(drm);
 
-	priv->fbdev = drm_fbdev_cma_init(drm, 32,
-					 drm->mode_config.num_connector);
-	if (IS_ERR(priv->fbdev)) {
-		ret = PTR_ERR(priv->fbdev);
+	ret = drm_fb_cma_fbdev_init(drm, 32, 0);
+	if (ret) {
 		DRM_DEV_ERROR(dev, "failed to init cma fbdev: %d\n", ret);
-		priv->fbdev = NULL;
 		goto out_poll_fini;
 	}
 
@@ -141,10 +114,7 @@ static int zx_drm_bind(struct device *dev)
 	return 0;
 
 out_fbdev_fini:
-	if (priv->fbdev) {
-		drm_fbdev_cma_fini(priv->fbdev);
-		priv->fbdev = NULL;
-	}
+	drm_fb_cma_fbdev_fini(drm);
 out_poll_fini:
 	drm_kms_helper_poll_fini(drm);
 	drm_mode_config_cleanup(drm);
@@ -152,7 +122,6 @@ static int zx_drm_bind(struct device *dev)
 	component_unbind_all(dev, drm);
 out_unregister:
 	dev_set_drvdata(dev, NULL);
-	drm->dev_private = NULL;
 	drm_dev_unref(drm);
 	return ret;
 }
@@ -160,18 +129,13 @@ static int zx_drm_bind(struct device *dev)
 static void zx_drm_unbind(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
-	struct zx_drm_private *priv = drm->dev_private;
 
 	drm_dev_unregister(drm);
-	if (priv->fbdev) {
-		drm_fbdev_cma_fini(priv->fbdev);
-		priv->fbdev = NULL;
-	}
+	drm_fb_cma_fbdev_fini(drm);
 	drm_kms_helper_poll_fini(drm);
 	drm_mode_config_cleanup(drm);
 	component_unbind_all(dev, drm);
 	dev_set_drvdata(dev, NULL);
-	drm->dev_private = NULL;
 	drm_dev_unref(drm);
 }
 
diff --git a/drivers/gpu/drm/zte/zx_hdmi.c b/drivers/gpu/drm/zte/zx_hdmi.c
index b8abb1b496ff..13ea90f7a185 100644
--- a/drivers/gpu/drm/zte/zx_hdmi.c
+++ b/drivers/gpu/drm/zte/zx_hdmi.c
@@ -108,6 +108,7 @@ static int zx_hdmi_config_video_vsi(struct zx_hdmi *hdmi,
 	int ret;
 
 	ret = drm_hdmi_vendor_infoframe_from_display_mode(&frame.vendor.hdmi,
+							  &hdmi->connector,
 							  mode);
 	if (ret) {
 		DRM_DEV_ERROR(hdmi->dev, "failed to get vendor infoframe: %d\n",
diff --git a/drivers/gpu/drm/zte/zx_plane.c b/drivers/gpu/drm/zte/zx_plane.c
index 18e763493264..68fd2e2dc78a 100644
--- a/drivers/gpu/drm/zte/zx_plane.c
+++ b/drivers/gpu/drm/zte/zx_plane.c
@@ -80,9 +80,9 @@ static int zx_vl_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
 
-	return drm_plane_helper_check_state(plane_state, &clip,
-					    min_scale, max_scale,
-					    true, true);
+	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						   &clip, min_scale, max_scale,
+						   true, true);
 }
 
 static int zx_vl_get_fmt(uint32_t format)
@@ -315,10 +315,11 @@ static int zx_gl_plane_atomic_check(struct drm_plane *plane,
 	clip.x2 = crtc_state->adjusted_mode.hdisplay;
 	clip.y2 = crtc_state->adjusted_mode.vdisplay;
 
-	return drm_plane_helper_check_state(plane_state, &clip,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    false, true);
+	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
+						   &clip,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_HELPER_NO_SCALING,
+						   false, true);
 }
 
 static int zx_gl_get_fmt(uint32_t format)
diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
index 2e57c9cea696..88a3558b7916 100644
--- a/drivers/gpu/host1x/bus.c
+++ b/drivers/gpu/host1x/bus.c
@@ -211,8 +211,7 @@ int host1x_device_init(struct host1x_device *device)
 				dev_err(&device->dev,
 					"failed to initialize %s: %d\n",
 					dev_name(client->dev), err);
-				mutex_unlock(&device->clients_lock);
-				return err;
+				goto teardown;
 			}
 		}
 	}
@@ -220,6 +219,14 @@ int host1x_device_init(struct host1x_device *device)
 	mutex_unlock(&device->clients_lock);
 
 	return 0;
+
+teardown:
+	list_for_each_entry_continue_reverse(client, &device->clients, list)
+		if (client->ops->exit)
+			client->ops->exit(client);
+
+	mutex_unlock(&device->clients_lock);
+	return err;
 }
 EXPORT_SYMBOL(host1x_device_init);
 
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index bf67c3aeb634..03db71173f5d 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -218,20 +218,27 @@ static int host1x_probe(struct platform_device *pdev)
 		return err;
 	}
 
-	if (iommu_present(&platform_bus_type)) {
+	host->group = iommu_group_get(&pdev->dev);
+	if (host->group) {
 		struct iommu_domain_geometry *geometry;
 		unsigned long order;
 
 		host->domain = iommu_domain_alloc(&platform_bus_type);
-		if (!host->domain)
-			return -ENOMEM;
-
-		err = iommu_attach_device(host->domain, &pdev->dev);
-		if (err == -ENODEV) {
-			iommu_domain_free(host->domain);
-			host->domain = NULL;
-			goto skip_iommu;
-		} else if (err) {
+		if (!host->domain) {
+			err = -ENOMEM;
+			goto put_group;
+		}
+
+		err = iommu_attach_group(host->domain, host->group);
+		if (err) {
+			if (err == -ENODEV) {
+				iommu_domain_free(host->domain);
+				host->domain = NULL;
+				iommu_group_put(host->group);
+				host->group = NULL;
+				goto skip_iommu;
+			}
+
 			goto fail_free_domain;
 		}
 
@@ -294,13 +301,15 @@ static int host1x_probe(struct platform_device *pdev)
 fail_free_channels:
 	host1x_channel_list_free(&host->channel_list);
 fail_detach_device:
-	if (host->domain) {
+	if (host->group && host->domain) {
 		put_iova_domain(&host->iova);
-		iommu_detach_device(host->domain, &pdev->dev);
+		iommu_detach_group(host->domain, host->group);
 	}
 fail_free_domain:
 	if (host->domain)
 		iommu_domain_free(host->domain);
+put_group:
+	iommu_group_put(host->group);
 
 	return err;
 }
@@ -317,8 +326,9 @@ static int host1x_remove(struct platform_device *pdev)
 
 	if (host->domain) {
 		put_iova_domain(&host->iova);
-		iommu_detach_device(host->domain, &pdev->dev);
+		iommu_detach_group(host->domain, host->group);
 		iommu_domain_free(host->domain);
+		iommu_group_put(host->group);
 	}
 
 	return 0;
diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h
index 502769726480..43e9fabb43a1 100644
--- a/drivers/gpu/host1x/dev.h
+++ b/drivers/gpu/host1x/dev.h
@@ -117,6 +117,7 @@ struct host1x {
 	struct clk *clk;
 	struct reset_control *rst;
 
+	struct iommu_group *group;
 	struct iommu_domain *domain;
 	struct iova_domain iova;
 	dma_addr_t iova_end;
diff --git a/drivers/gpu/ipu-v3/Kconfig b/drivers/gpu/ipu-v3/Kconfig
index 87a20b3dcf7a..fe6f8c5b4445 100644
--- a/drivers/gpu/ipu-v3/Kconfig
+++ b/drivers/gpu/ipu-v3/Kconfig
@@ -1,7 +1,9 @@
 config IMX_IPUV3_CORE
 	tristate "IPUv3 core support"
-	depends on SOC_IMX5 || SOC_IMX6Q || ARCH_MULTIPLATFORM
+	depends on SOC_IMX5 || SOC_IMX6Q || ARCH_MULTIPLATFORM || COMPILE_TEST
 	depends on DRM || !DRM # if DRM=m, this can't be 'y'
+	select BITREVERSE
+	select GENERIC_ALLOCATOR if DRM
 	select GENERIC_IRQ_CHIP
 	help
 	  Choose this if you have a i.MX5/6 system and want to use the Image
diff --git a/drivers/gpu/ipu-v3/ipu-common.c b/drivers/gpu/ipu-v3/ipu-common.c
index 658fa2d3e40c..48685cddbad1 100644
--- a/drivers/gpu/ipu-v3/ipu-common.c
+++ b/drivers/gpu/ipu-v3/ipu-common.c
@@ -1089,7 +1089,7 @@ static void ipu_irq_handler(struct irq_desc *desc)
 {
 	struct ipu_soc *ipu = irq_desc_get_handler_data(desc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
-	const int int_reg[] = { 0, 1, 2, 3, 10, 11, 12, 13, 14};
+	static const int int_reg[] = { 0, 1, 2, 3, 10, 11, 12, 13, 14};
 
 	chained_irq_enter(chip, desc);
 
@@ -1102,7 +1102,7 @@ static void ipu_err_irq_handler(struct irq_desc *desc)
 {
 	struct ipu_soc *ipu = irq_desc_get_handler_data(desc);
 	struct irq_chip *chip = irq_desc_get_chip(desc);
-	const int int_reg[] = { 4, 5, 8, 9};
+	static const int int_reg[] = { 4, 5, 8, 9};
 
 	chained_irq_enter(chip, desc);
 
diff --git a/drivers/gpu/ipu-v3/ipu-cpmem.c b/drivers/gpu/ipu-v3/ipu-cpmem.c
index 1cb82f445f91..9f2d9ec42add 100644
--- a/drivers/gpu/ipu-v3/ipu-cpmem.c
+++ b/drivers/gpu/ipu-v3/ipu-cpmem.c
@@ -12,6 +12,7 @@
 #include <linux/types.h>
 #include <linux/bitrev.h>
 #include <linux/io.h>
+#include <linux/sizes.h>
 #include <drm/drm_fourcc.h>
 #include "ipu-prv.h"
 
@@ -787,12 +788,14 @@ int ipu_cpmem_set_image(struct ipuv3_channel *ch, struct ipu_image *image)
 	case V4L2_PIX_FMT_SGBRG8:
 	case V4L2_PIX_FMT_SGRBG8:
 	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_GREY:
 		offset = image->rect.left + image->rect.top * pix->bytesperline;
 		break;
 	case V4L2_PIX_FMT_SBGGR16:
 	case V4L2_PIX_FMT_SGBRG16:
 	case V4L2_PIX_FMT_SGRBG16:
 	case V4L2_PIX_FMT_SRGGB16:
+	case V4L2_PIX_FMT_Y16:
 		offset = image->rect.left * 2 +
 			 image->rect.top * pix->bytesperline;
 		break;
diff --git a/drivers/gpu/ipu-v3/ipu-csi.c b/drivers/gpu/ipu-v3/ipu-csi.c
index 24e12b87a0cb..caa05b0702e1 100644
--- a/drivers/gpu/ipu-v3/ipu-csi.c
+++ b/drivers/gpu/ipu-v3/ipu-csi.c
@@ -288,6 +288,7 @@ static int mbus_code_to_bus_cfg(struct ipu_csi_bus_config *cfg, u32 mbus_code)
 	case MEDIA_BUS_FMT_SGBRG10_1X10:
 	case MEDIA_BUS_FMT_SGRBG10_1X10:
 	case MEDIA_BUS_FMT_SRGGB10_1X10:
+	case MEDIA_BUS_FMT_Y10_1X10:
 		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_BAYER;
 		cfg->mipi_dt = MIPI_DT_RAW10;
 		cfg->data_width = IPU_CSI_DATA_WIDTH_10;
@@ -296,6 +297,7 @@ static int mbus_code_to_bus_cfg(struct ipu_csi_bus_config *cfg, u32 mbus_code)
 	case MEDIA_BUS_FMT_SGBRG12_1X12:
 	case MEDIA_BUS_FMT_SGRBG12_1X12:
 	case MEDIA_BUS_FMT_SRGGB12_1X12:
+	case MEDIA_BUS_FMT_Y12_1X12:
 		cfg->data_fmt = CSI_SENS_CONF_DATA_FMT_BAYER;
 		cfg->mipi_dt = MIPI_DT_RAW12;
 		cfg->data_width = IPU_CSI_DATA_WIDTH_12;
diff --git a/drivers/gpu/ipu-v3/ipu-ic.c b/drivers/gpu/ipu-v3/ipu-ic.c
index 321eb983c2f5..67cc820253a9 100644
--- a/drivers/gpu/ipu-v3/ipu-ic.c
+++ b/drivers/gpu/ipu-v3/ipu-ic.c
@@ -17,6 +17,7 @@
 #include <linux/bitrev.h>
 #include <linux/io.h>
 #include <linux/err.h>
+#include <linux/sizes.h>
 #include "ipu-prv.h"
 
 /* IC Register Offsets */
diff --git a/drivers/gpu/ipu-v3/ipu-pre.c b/drivers/gpu/ipu-v3/ipu-pre.c
index c860a7997cb5..0f70e8847540 100644
--- a/drivers/gpu/ipu-v3/ipu-pre.c
+++ b/drivers/gpu/ipu-v3/ipu-pre.c
@@ -49,6 +49,10 @@
 #define IPU_PRE_TPR_CTRL				0x070
 #define  IPU_PRE_TPR_CTRL_TILE_FORMAT(v)		((v & 0xff) << 0)
 #define  IPU_PRE_TPR_CTRL_TILE_FORMAT_MASK		0xff
+#define  IPU_PRE_TPR_CTRL_TILE_FORMAT_16_BIT		(1 << 0)
+#define  IPU_PRE_TPR_CTRL_TILE_FORMAT_SPLIT_BUF		(1 << 4)
+#define  IPU_PRE_TPR_CTRL_TILE_FORMAT_SINGLE_BUF	(1 << 5)
+#define  IPU_PRE_TPR_CTRL_TILE_FORMAT_SUPER_TILED	(1 << 6)
 
 #define IPU_PRE_PREFETCH_ENG_CTRL			0x080
 #define  IPU_PRE_PREF_ENG_CTRL_PREFETCH_EN		(1 << 0)
@@ -125,11 +129,14 @@ ipu_pre_lookup_by_phandle(struct device *dev, const char *name, int index)
 		if (pre_node == pre->dev->of_node) {
 			mutex_unlock(&ipu_pre_list_mutex);
 			device_link_add(dev, pre->dev, DL_FLAG_AUTOREMOVE);
+			of_node_put(pre_node);
 			return pre;
 		}
 	}
 	mutex_unlock(&ipu_pre_list_mutex);
 
+	of_node_put(pre_node);
+
 	return NULL;
 }
 
@@ -147,7 +154,7 @@ int ipu_pre_get(struct ipu_pre *pre)
 	val = IPU_PRE_CTRL_HANDSHAKE_ABORT_SKIP_EN |
 	      IPU_PRE_CTRL_HANDSHAKE_EN |
 	      IPU_PRE_CTRL_TPR_REST_SEL |
-	      IPU_PRE_CTRL_BLOCK_16 | IPU_PRE_CTRL_SDW_UPDATE;
+	      IPU_PRE_CTRL_SDW_UPDATE;
 	writel(val, pre->regs + IPU_PRE_CTRL);
 
 	pre->in_use = true;
@@ -163,14 +170,17 @@ void ipu_pre_put(struct ipu_pre *pre)
 
 void ipu_pre_configure(struct ipu_pre *pre, unsigned int width,
 		       unsigned int height, unsigned int stride, u32 format,
-		       unsigned int bufaddr)
+		       uint64_t modifier, unsigned int bufaddr)
 {
 	const struct drm_format_info *info = drm_format_info(format);
 	u32 active_bpp = info->cpp[0] >> 1;
 	u32 val;
 
 	/* calculate safe window for ctrl register updates */
-	pre->safe_window_end = height - 2;
+	if (modifier == DRM_FORMAT_MOD_LINEAR)
+		pre->safe_window_end = height - 2;
+	else
+		pre->safe_window_end = DIV_ROUND_UP(height, 4) - 1;
 
 	writel(bufaddr, pre->regs + IPU_PRE_CUR_BUF);
 	writel(bufaddr, pre->regs + IPU_PRE_NEXT_BUF);
@@ -203,9 +213,25 @@ void ipu_pre_configure(struct ipu_pre *pre, unsigned int width,
 
 	writel(pre->buffer_paddr, pre->regs + IPU_PRE_STORE_ENG_ADDR);
 
+	val = readl(pre->regs + IPU_PRE_TPR_CTRL);
+	val &= ~IPU_PRE_TPR_CTRL_TILE_FORMAT_MASK;
+	if (modifier != DRM_FORMAT_MOD_LINEAR) {
+		/* only support single buffer formats for now */
+		val |= IPU_PRE_TPR_CTRL_TILE_FORMAT_SINGLE_BUF;
+		if (modifier == DRM_FORMAT_MOD_VIVANTE_SUPER_TILED)
+			val |= IPU_PRE_TPR_CTRL_TILE_FORMAT_SUPER_TILED;
+		if (info->cpp[0] == 2)
+			val |= IPU_PRE_TPR_CTRL_TILE_FORMAT_16_BIT;
+	}
+	writel(val, pre->regs + IPU_PRE_TPR_CTRL);
+
 	val = readl(pre->regs + IPU_PRE_CTRL);
 	val |= IPU_PRE_CTRL_EN_REPEAT | IPU_PRE_CTRL_ENABLE |
 	       IPU_PRE_CTRL_SDW_UPDATE;
+	if (modifier == DRM_FORMAT_MOD_LINEAR)
+		val &= ~IPU_PRE_CTRL_BLOCK_EN;
+	else
+		val |= IPU_PRE_CTRL_BLOCK_EN;
 	writel(val, pre->regs + IPU_PRE_CTRL);
 }
 
diff --git a/drivers/gpu/ipu-v3/ipu-prg.c b/drivers/gpu/ipu-v3/ipu-prg.c
index 0013ca9f72c8..83f9dd934a5d 100644
--- a/drivers/gpu/ipu-v3/ipu-prg.c
+++ b/drivers/gpu/ipu-v3/ipu-prg.c
@@ -20,6 +20,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <video/imx-ipu-v3.h>
 
@@ -101,11 +102,14 @@ ipu_prg_lookup_by_phandle(struct device *dev, const char *name, int ipu_id)
 			mutex_unlock(&ipu_prg_list_mutex);
 			device_link_add(dev, prg->dev, DL_FLAG_AUTOREMOVE);
 			prg->id = ipu_id;
+			of_node_put(prg_node);
 			return prg;
 		}
 	}
 	mutex_unlock(&ipu_prg_list_mutex);
 
+	of_node_put(prg_node);
+
 	return NULL;
 }
 
@@ -132,28 +136,25 @@ bool ipu_prg_format_supported(struct ipu_soc *ipu, uint32_t format,
 	if (info->num_planes != 1)
 		return false;
 
-	return true;
+	switch (modifier) {
+	case DRM_FORMAT_MOD_LINEAR:
+	case DRM_FORMAT_MOD_VIVANTE_TILED:
+	case DRM_FORMAT_MOD_VIVANTE_SUPER_TILED:
+		return true;
+	default:
+		return false;
+	}
 }
 EXPORT_SYMBOL_GPL(ipu_prg_format_supported);
 
 int ipu_prg_enable(struct ipu_soc *ipu)
 {
 	struct ipu_prg *prg = ipu->prg_priv;
-	int ret;
 
 	if (!prg)
 		return 0;
 
-	ret = clk_prepare_enable(prg->clk_axi);
-	if (ret)
-		goto fail_disable_ipg;
-
-	return 0;
-
-fail_disable_ipg:
-	clk_disable_unprepare(prg->clk_ipg);
-
-	return ret;
+	return pm_runtime_get_sync(prg->dev);
 }
 EXPORT_SYMBOL_GPL(ipu_prg_enable);
 
@@ -164,7 +165,7 @@ void ipu_prg_disable(struct ipu_soc *ipu)
 	if (!prg)
 		return;
 
-	clk_disable_unprepare(prg->clk_axi);
+	pm_runtime_put(prg->dev);
 }
 EXPORT_SYMBOL_GPL(ipu_prg_disable);
 
@@ -249,13 +250,17 @@ void ipu_prg_channel_disable(struct ipuv3_channel *ipu_chan)
 {
 	int prg_chan = ipu_prg_ipu_to_prg_chan(ipu_chan->num);
 	struct ipu_prg *prg = ipu_chan->ipu->prg_priv;
-	struct ipu_prg_channel *chan = &prg->chan[prg_chan];
+	struct ipu_prg_channel *chan;
 	u32 val;
 
-	if (!chan->enabled || prg_chan < 0)
+	if (prg_chan < 0)
+		return;
+
+	chan = &prg->chan[prg_chan];
+	if (!chan->enabled)
 		return;
 
-	clk_prepare_enable(prg->clk_ipg);
+	pm_runtime_get_sync(prg->dev);
 
 	val = readl(prg->regs + IPU_PRG_CTL);
 	val |= IPU_PRG_CTL_BYPASS(prg_chan);
@@ -264,7 +269,7 @@ void ipu_prg_channel_disable(struct ipuv3_channel *ipu_chan)
 	val = IPU_PRG_REG_UPDATE_REG_UPDATE;
 	writel(val, prg->regs + IPU_PRG_REG_UPDATE);
 
-	clk_disable_unprepare(prg->clk_ipg);
+	pm_runtime_put(prg->dev);
 
 	ipu_prg_put_pre(prg, prg_chan);
 
@@ -275,17 +280,19 @@ EXPORT_SYMBOL_GPL(ipu_prg_channel_disable);
 int ipu_prg_channel_configure(struct ipuv3_channel *ipu_chan,
 			      unsigned int axi_id, unsigned int width,
 			      unsigned int height, unsigned int stride,
-			      u32 format, unsigned long *eba)
+			      u32 format, uint64_t modifier, unsigned long *eba)
 {
 	int prg_chan = ipu_prg_ipu_to_prg_chan(ipu_chan->num);
 	struct ipu_prg *prg = ipu_chan->ipu->prg_priv;
-	struct ipu_prg_channel *chan = &prg->chan[prg_chan];
+	struct ipu_prg_channel *chan;
 	u32 val;
 	int ret;
 
 	if (prg_chan < 0)
 		return prg_chan;
 
+	chan = &prg->chan[prg_chan];
+
 	if (chan->enabled) {
 		ipu_pre_update(prg->pres[chan->used_pre], *eba);
 		return 0;
@@ -296,14 +303,10 @@ int ipu_prg_channel_configure(struct ipuv3_channel *ipu_chan,
 		return ret;
 
 	ipu_pre_configure(prg->pres[chan->used_pre],
-			  width, height, stride, format, *eba);
+			  width, height, stride, format, modifier, *eba);
 
 
-	ret = clk_prepare_enable(prg->clk_ipg);
-	if (ret) {
-		ipu_prg_put_pre(prg, prg_chan);
-		return ret;
-	}
+	pm_runtime_get_sync(prg->dev);
 
 	val = (stride - 1) & IPU_PRG_STRIDE_STRIDE_MASK;
 	writel(val, prg->regs + IPU_PRG_STRIDE(prg_chan));
@@ -336,7 +339,7 @@ int ipu_prg_channel_configure(struct ipuv3_channel *ipu_chan,
 			   (val & IPU_PRG_STATUS_BUFFER1_READY(prg_chan)),
 			   5, 1000);
 
-	clk_disable_unprepare(prg->clk_ipg);
+	pm_runtime_put(prg->dev);
 
 	chan->enabled = true;
 	return 0;
@@ -384,6 +387,12 @@ static int ipu_prg_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	ret = clk_prepare_enable(prg->clk_axi);
+	if (ret) {
+		clk_disable_unprepare(prg->clk_ipg);
+		return ret;
+	}
+
 	/* init to free running mode */
 	val = readl(prg->regs + IPU_PRG_CTL);
 	val |= IPU_PRG_CTL_SHADOW_EN;
@@ -392,7 +401,8 @@ static int ipu_prg_probe(struct platform_device *pdev)
 	/* disable address threshold */
 	writel(0xffffffff, prg->regs + IPU_PRG_THD);
 
-	clk_disable_unprepare(prg->clk_ipg);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
 
 	prg->dev = dev;
 	platform_set_drvdata(pdev, prg);
@@ -414,6 +424,40 @@ static int ipu_prg_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int prg_suspend(struct device *dev)
+{
+	struct ipu_prg *prg = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(prg->clk_axi);
+	clk_disable_unprepare(prg->clk_ipg);
+
+	return 0;
+}
+
+static int prg_resume(struct device *dev)
+{
+	struct ipu_prg *prg = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(prg->clk_ipg);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(prg->clk_axi);
+	if (ret) {
+		clk_disable_unprepare(prg->clk_ipg);
+		return ret;
+	}
+
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops prg_pm_ops = {
+	SET_RUNTIME_PM_OPS(prg_suspend, prg_resume, NULL)
+};
+
 static const struct of_device_id ipu_prg_dt_ids[] = {
 	{ .compatible = "fsl,imx6qp-prg", },
 	{ /* sentinel */ },
@@ -424,6 +468,7 @@ struct platform_driver ipu_prg_drv = {
 	.remove		= ipu_prg_remove,
 	.driver		= {
 		.name	= "imx-ipu-prg",
+		.pm	= &prg_pm_ops,
 		.of_match_table = ipu_prg_dt_ids,
 	},
 };
diff --git a/drivers/gpu/ipu-v3/ipu-prv.h b/drivers/gpu/ipu-v3/ipu-prv.h
index ac4b8d658500..d6beee99b6b8 100644
--- a/drivers/gpu/ipu-v3/ipu-prv.h
+++ b/drivers/gpu/ipu-v3/ipu-prv.h
@@ -269,8 +269,8 @@ int ipu_pre_get(struct ipu_pre *pre);
 void ipu_pre_put(struct ipu_pre *pre);
 u32 ipu_pre_get_baddr(struct ipu_pre *pre);
 void ipu_pre_configure(struct ipu_pre *pre, unsigned int width,
-		       unsigned int height,
-		       unsigned int stride, u32 format, unsigned int bufaddr);
+		       unsigned int height, unsigned int stride, u32 format,
+		       uint64_t modifier, unsigned int bufaddr);
 void ipu_pre_update(struct ipu_pre *pre, unsigned int bufaddr);
 
 struct ipu_prg *ipu_prg_lookup_by_phandle(struct device *dev, const char *name,
diff --git a/drivers/gpu/vga/vgaarb.c b/drivers/gpu/vga/vgaarb.c
index d35d6d271f3f..1c5e74cb9279 100644
--- a/drivers/gpu/vga/vgaarb.c
+++ b/drivers/gpu/vga/vgaarb.c
@@ -1266,12 +1266,12 @@ static ssize_t vga_arb_write(struct file *file, const char __user *buf,
 	return ret_val;
 }
 
-static unsigned int vga_arb_fpoll(struct file *file, poll_table *wait)
+static __poll_t vga_arb_fpoll(struct file *file, poll_table *wait)
 {
 	pr_debug("%s\n", __func__);
 
 	poll_wait(file, &vga_wait_queue, wait);
-	return POLLIN;
+	return EPOLLIN;
 }
 
 static int vga_arb_open(struct inode *inode, struct file *file)
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index 779c5ae47f36..19c499f5623d 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -280,6 +280,7 @@ config HID_ELECOM
 	---help---
 	Support for ELECOM devices:
 	  - BM084 Bluetooth Mouse
+	  - EX-G Trackball (Wired and wireless)
 	  - DEFT Trackball (Wired and wireless)
 	  - HUGE Trackball (Wired and wireless)
 
@@ -396,6 +397,17 @@ config HID_ITE
 	---help---
 	Support for ITE devices not fully compliant with HID standard.
 
+config HID_JABRA
+	tristate "Jabra USB HID Driver"
+	depends on HID
+	---help---
+	Support for Jabra USB HID devices.
+
+	Prevents mapping of vendor defined HID usages to input events. Without
+	this driver HID	reports from Jabra devices may incorrectly be seen as
+	mouse button events.
+	Say M here if you may ever plug in a Jabra USB device.
+
 config HID_TWINHAN
 	tristate "Twinhan IR remote control"
 	depends on HID
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index 235bd2a7b333..eb13b9e92d85 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for the HID driver
 #
-hid-y			:= hid-core.o hid-input.o
+hid-y			:= hid-core.o hid-input.o hid-quirks.o
 hid-$(CONFIG_DEBUG_FS)		+= hid-debug.o
 
 obj-$(CONFIG_HID)		+= hid.o
@@ -52,6 +52,7 @@ obj-$(CONFIG_HID_HOLTEK)	+= hid-holtekff.o
 obj-$(CONFIG_HID_HYPERV_MOUSE)	+= hid-hyperv.o
 obj-$(CONFIG_HID_ICADE)		+= hid-icade.o
 obj-$(CONFIG_HID_ITE)		+= hid-ite.o
+obj-$(CONFIG_HID_JABRA)		+= hid-jabra.o
 obj-$(CONFIG_HID_KENSINGTON)	+= hid-kensington.o
 obj-$(CONFIG_HID_KEYTOUCH)	+= hid-keytouch.o
 obj-$(CONFIG_HID_KYE)		+= hid-kye.o
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index 1bb7b63b3150..88b9703318e4 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -26,6 +26,7 @@
  * any later version.
  */
 
+#include <linux/dmi.h>
 #include <linux/hid.h>
 #include <linux/module.h>
 #include <linux/input/mt.h>
@@ -119,6 +120,24 @@ static const struct asus_touchpad_info asus_t100ta_tp = {
 	.max_contacts = 5,
 };
 
+static const struct asus_touchpad_info asus_t100ha_tp = {
+	.max_x = 2640,
+	.max_y = 1320,
+	.res_x = 30, /* units/mm */
+	.res_y = 29, /* units/mm */
+	.contact_size = 5,
+	.max_contacts = 5,
+};
+
+static const struct asus_touchpad_info asus_t200ta_tp = {
+	.max_x = 3120,
+	.max_y = 1716,
+	.res_x = 30, /* units/mm */
+	.res_y = 28, /* units/mm */
+	.contact_size = 5,
+	.max_contacts = 5,
+};
+
 static const struct asus_touchpad_info asus_t100chi_tp = {
 	.max_x = 2640,
 	.max_y = 1320,
@@ -606,7 +625,17 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id)
 
 		if (intf->altsetting->desc.bInterfaceNumber == T100_TPAD_INTF) {
 			drvdata->quirks = QUIRK_SKIP_INPUT_MAPPING;
-			drvdata->tp = &asus_t100ta_tp;
+			/*
+			 * The T100HA uses the same USB-ids as the T100TAF and
+			 * the T200TA uses the same USB-ids as the T100TA, while
+			 * both have different max x/y values as the T100TA[F].
+			 */
+			if (dmi_match(DMI_PRODUCT_NAME, "T100HAN"))
+				drvdata->tp = &asus_t100ha_tp;
+			else if (dmi_match(DMI_PRODUCT_NAME, "T200TA"))
+				drvdata->tp = &asus_t200ta_tp;
+			else
+				drvdata->tp = &asus_t100ta_tp;
 		}
 	}
 
@@ -686,9 +715,10 @@ static __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		hid_info(hdev, "Fixing up Asus notebook report descriptor\n");
 		rdesc[55] = 0xdd;
 	}
-	/* For the T100TA keyboard dock */
+	/* For the T100TA/T200TA keyboard dock */
 	if (drvdata->quirks & QUIRK_T100_KEYBOARD &&
-		 *rsize == 76 && rdesc[73] == 0x81 && rdesc[74] == 0x01) {
+		 (*rsize == 76 || *rsize == 101) &&
+		 rdesc[73] == 0x81 && rdesc[74] == 0x01) {
 		hid_info(hdev, "Fixing up Asus T100 keyb report descriptor\n");
 		rdesc[74] &= ~HID_MAIN_ITEM_CONSTANT;
 	}
@@ -751,7 +781,10 @@ static const struct hid_device_id asus_devices[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
 		USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD3), QUIRK_G752_KEYBOARD },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
-		USB_DEVICE_ID_ASUSTEK_T100_KEYBOARD),
+		USB_DEVICE_ID_ASUSTEK_T100TA_KEYBOARD),
+	  QUIRK_T100_KEYBOARD | QUIRK_NO_CONSUMER_USAGES },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK,
+		USB_DEVICE_ID_ASUSTEK_T100TAF_KEYBOARD),
 	  QUIRK_T100_KEYBOARD | QUIRK_NO_CONSUMER_USAGES },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_ASUS_AK1D) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_ASUS_MD_5110) },
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index 0c3f608131cf..c2560aae5542 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -830,31 +830,6 @@ static int hid_scan_report(struct hid_device *hid)
 		break;
 	}
 
-	/* fall back to generic driver in case specific driver doesn't exist */
-	switch (hid->group) {
-	case HID_GROUP_MULTITOUCH_WIN_8:
-		/* fall-through */
-	case HID_GROUP_MULTITOUCH:
-		if (!IS_ENABLED(CONFIG_HID_MULTITOUCH))
-			hid->group = HID_GROUP_GENERIC;
-		break;
-	case HID_GROUP_SENSOR_HUB:
-		if (!IS_ENABLED(CONFIG_HID_SENSOR_HUB))
-			hid->group = HID_GROUP_GENERIC;
-		break;
-	case HID_GROUP_RMI:
-		if (!IS_ENABLED(CONFIG_HID_RMI))
-			hid->group = HID_GROUP_GENERIC;
-		break;
-	case HID_GROUP_WACOM:
-		if (!IS_ENABLED(CONFIG_HID_WACOM))
-			hid->group = HID_GROUP_GENERIC;
-		break;
-	case HID_GROUP_LOGITECH_DJ_DEVICE:
-		if (!IS_ENABLED(CONFIG_HID_LOGITECH_DJ))
-			hid->group = HID_GROUP_GENERIC;
-		break;
-	}
 	vfree(parser);
 	return 0;
 }
@@ -1597,8 +1572,8 @@ int hid_input_report(struct hid_device *hid, int type, u8 *data, int size, int i
 }
 EXPORT_SYMBOL_GPL(hid_input_report);
 
-static bool hid_match_one_id(struct hid_device *hdev,
-		const struct hid_device_id *id)
+bool hid_match_one_id(const struct hid_device *hdev,
+		      const struct hid_device_id *id)
 {
 	return (id->bus == HID_BUS_ANY || id->bus == hdev->bus) &&
 		(id->group == HID_GROUP_ANY || id->group == hdev->group) &&
@@ -1606,7 +1581,7 @@ static bool hid_match_one_id(struct hid_device *hdev,
 		(id->product == HID_ANY_ID || id->product == hdev->product);
 }
 
-const struct hid_device_id *hid_match_id(struct hid_device *hdev,
+const struct hid_device_id *hid_match_id(const struct hid_device *hdev,
 		const struct hid_device_id *id)
 {
 	for (; id->bus; id++)
@@ -1862,541 +1837,6 @@ void hid_hw_close(struct hid_device *hdev)
 }
 EXPORT_SYMBOL_GPL(hid_hw_close);
 
-/*
- * A list of devices for which there is a specialized driver on HID bus.
- *
- * Please note that for multitouch devices (driven by hid-multitouch driver),
- * there is a proper autodetection and autoloading in place (based on presence
- * of HID_DG_CONTACTID), so those devices don't need to be added to this list,
- * as we are doing the right thing in hid_scan_usage().
- *
- * Autodetection for (USB) HID sensor hubs exists too. If a collection of type
- * physical is found inside a usage page of type sensor, hid-sensor-hub will be
- * used as a driver. See hid_scan_report().
- */
-static const struct hid_device_id hid_have_special_driver[] = {
-#if IS_ENABLED(CONFIG_HID_A4TECH)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_WCP32PU) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_X5_005D) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_RP_649) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ACCUTOUCH)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELO, USB_DEVICE_ID_ELO_ACCUTOUCH_2216) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ACRUX)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ACRUX, 0x0802) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ACRUX, 0xf705) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ALPS)
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_U1_DUAL) },
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_U1_DUAL) },
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_U1) },
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_T4_BTNLESS) },
-#endif
-#if IS_ENABLED(CONFIG_HID_APPLE)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MIGHTYMOUSE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_MINI_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_MINI_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_MINI_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_JIS) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_ISO) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_REVB_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_REVB_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_REVB_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ANSI) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ISO) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_JIS) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_ANSI) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_ISO) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) },
-#endif
-#if IS_ENABLED(CONFIG_HID_APPLEIR)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL3) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL4) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL5) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ASUS)
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_I2C_KEYBOARD) },
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_I2C_TOUCHPAD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD1) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD3) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T100_KEYBOARD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_JESS, USB_DEVICE_ID_ASUS_MD_5112) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_ASUS_MD_5110) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD) },
-#endif
-#if IS_ENABLED(CONFIG_HID_AUREAL)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AUREAL, USB_DEVICE_ID_AUREAL_W01RN) },
-#endif
-#if IS_ENABLED(CONFIG_HID_BELKIN)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BELKIN, USB_DEVICE_ID_FLIP_KVM) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LABTEC, USB_DEVICE_ID_LABTEC_WIRELESS_KEYBOARD) },
-#endif
-#if IS_ENABLED(CONFIG_HID_BETOP_FF)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185BFM, 0x2208) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185PC, 0x5506) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185V2PC, 0x1850) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185V2BFM, 0x5500) },
-#endif
-#if IS_ENABLED(CONFIG_HID_CHERRY)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CHERRY, USB_DEVICE_ID_CHERRY_CYMOTION) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CHERRY, USB_DEVICE_ID_CHERRY_CYMOTION_SOLAR) },
-#endif
-#if IS_ENABLED(CONFIG_HID_CHICONY)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_TACTICAL_PAD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_WIRELESS2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_ASUS_AK1D) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_ACER_SWITCH12) },
-#endif
-#if IS_ENABLED(CONFIG_HID_CMEDIA)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CMEDIA, USB_DEVICE_ID_CM6533) },
-#endif
-#if IS_ENABLED(CONFIG_HID_CORSAIR)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K90) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_SCIMITAR_PRO_RGB) },
-#endif
-#if IS_ENABLED(CONFIG_HID_CP2112)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_CP2112) },
-#endif
-#if IS_ENABLED(CONFIG_HID_CYPRESS)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_1) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_3) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_4) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_MOUSE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_DRAGONRISE)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, 0x0006) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, 0x0011) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ELECOM)
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_DEFT_WIRED) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_DEFT_WIRELESS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_HUGE_WIRED) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_HUGE_WIRELESS) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ELO)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELO, 0x0009) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ELO, 0x0030) },
-#endif
-#if IS_ENABLED(CONFIG_HID_EMS_FF)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_EMS, USB_DEVICE_ID_EMS_TRIO_LINKER_PLUS_II) },
-#endif
-#if IS_ENABLED(CONFIG_HID_EZKEY)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_EZKEY, USB_DEVICE_ID_BTC_8193) },
-#endif
-#if IS_ENABLED(CONFIG_HID_GEMBIRD)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GEMBIRD, USB_DEVICE_ID_GEMBIRD_JPD_DUALFORCE2) },
-#endif
-#if IS_ENABLED(CONFIG_HID_GFRM)
-        { HID_BLUETOOTH_DEVICE(0x58, 0x2000) },
-        { HID_BLUETOOTH_DEVICE(0x471, 0x2210) },
-#endif
-#if IS_ENABLED(CONFIG_HID_GREENASIA)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GREENASIA, 0x0012) },
-#endif
-#if IS_ENABLED(CONFIG_HID_GT683R)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MSI, USB_DEVICE_ID_MSI_GT683R_LED_PANEL) },
-#endif
-#if IS_ENABLED(CONFIG_HID_GYRATION)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GYRATION, USB_DEVICE_ID_GYRATION_REMOTE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GYRATION, USB_DEVICE_ID_GYRATION_REMOTE_2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GYRATION, USB_DEVICE_ID_GYRATION_REMOTE_3) },
-#endif
-#if IS_ENABLED(CONFIG_HID_HOLTEK)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK, USB_DEVICE_ID_HOLTEK_ON_LINE_GRIP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_KEYBOARD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A04A) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A067) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A070) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A072) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A081) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A0C2) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ICADE)
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ION, USB_DEVICE_ID_ICADE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ITE)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ITE, USB_DEVICE_ID_ITE8595) },
-#endif
-#if IS_ENABLED(CONFIG_HID_KENSINGTON)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KENSINGTON, USB_DEVICE_ID_KS_SLIMBLADE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_KEYTOUCH)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KEYTOUCH, USB_DEVICE_ID_KEYTOUCH_IEC) },
-#endif
-#if IS_ENABLED(CONFIG_HID_KYE)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_GENIUS_GILA_GAMING_MOUSE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_GENIUS_MANTICORE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_GENIUS_GX_IMPERATOR) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_ERGO_525V) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_I405X) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_M912) },
-#endif
-#if IS_ENABLED(CONFIG_HID_LCPOWER)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LCPOWER, USB_DEVICE_ID_LCPOWER_LC1000 ) },
-#endif
-#if IS_ENABLED(CONFIG_HID_LED)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DELCOM, USB_DEVICE_ID_DELCOM_VISUAL_IND) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_WN) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_FA) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_LUXAFOR) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_RISO_KAGAKU, USB_DEVICE_ID_RI_KA_WEBMAIL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THINGM, USB_DEVICE_ID_BLINK1) },
-#endif
-#if IS_ENABLED(CONFIG_HID_LENOVO)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_TPKBD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_CUSBKBD) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_CBTKBD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_TPPRODOCK) },
-#endif
-#if IS_ENABLED(CONFIG_HID_LOGITECH)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_MX3000_RECEIVER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_S510_RECEIVER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_S510_RECEIVER_2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RECEIVER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_DINOVO_DESKTOP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_DINOVO_EDGE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_DINOVO_MINI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_ELITE_KBD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_CORDLESS_DESKTOP_LX500) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_EXTREME_3D) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DUAL_ACTION) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD_CORD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD2_2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G29_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_F3D) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_FFG ) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_FORCE3D_PRO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_FLIGHT_SYSTEM_G940) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOMO_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_VIBRATION_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DFP_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DFGT_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G25_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G27_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WII_WHEEL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_SPACETRAVELLER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_SPACENAVIGATOR) },
-#endif
-#if IS_ENABLED(CONFIG_HID_LOGITECH_HIDPP)
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_T651) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G920_WHEEL) },
-#endif
-#if IS_ENABLED(CONFIG_HID_LOGITECH_DJ)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_UNIFYING_RECEIVER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_UNIFYING_RECEIVER_2) },
-#endif
-#if IS_ENABLED(CONFIG_HID_MAGICMOUSE)
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICMOUSE) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD) },
-#endif
-#if IS_ENABLED(CONFIG_HID_MAYFLASH)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_DOLPHINBAR) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_GAMECUBE1) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_GAMECUBE2) },
-#endif
-#if IS_ENABLED(CONFIG_HID_MICROSOFT)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_COMFORT_MOUSE_4500) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_COMFORT_KEYBOARD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_SIDEWINDER_GV) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_NE4K) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_NE4K_JP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_NE7K) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_LK6K) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_PRESENTER_8K_USB) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_3K) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_WIRELESS_OPTICAL_DESKTOP_3_0) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_OFFICE_KB) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_7K) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_600) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_3KV1) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_PRESENTER_8K_BT) },
-#endif
-#if IS_ENABLED(CONFIG_HID_MONTEREY)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MONTEREY, USB_DEVICE_ID_GENIUS_KB29E) },
-#endif
-#if IS_ENABLED(CONFIG_HID_MULTITOUCH)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LG, USB_DEVICE_ID_LG_MELFAS_MT) },
-#endif
-#if IS_ENABLED(CONFIG_HID_WIIMOTE)
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_NINTENDO, USB_DEVICE_ID_NINTENDO_WIIMOTE) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_NINTENDO, USB_DEVICE_ID_NINTENDO_WIIMOTE2) },
-#endif
-#if IS_ENABLED(CONFIG_HID_NTI)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTI, USB_DEVICE_ID_USB_SUN) },
-#endif
-#if IS_ENABLED(CONFIG_HID_NTRIG)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_1) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_3) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_4) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_5) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_6) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_7) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_8) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_9) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_10) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_11) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_12) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_13) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_14) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_15) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_16) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_17) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_18) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ORTEK)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ORTEK, USB_DEVICE_ID_ORTEK_PKB1700) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ORTEK, USB_DEVICE_ID_ORTEK_WKB2000) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ORTEK, USB_DEVICE_ID_ORTEK_IHOME_IMAC_A210S) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SKYCABLE, USB_DEVICE_ID_SKYCABLE_WIRELESS_PRESENTER) },
-#endif
-#if IS_ENABLED(CONFIG_HID_PANTHERLORD)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GAMERON, USB_DEVICE_ID_GAMERON_DUAL_PSX_ADAPTOR) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GAMERON, USB_DEVICE_ID_GAMERON_DUAL_PCS_ADAPTOR) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GREENASIA, 0x0003) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_JESS2, USB_DEVICE_ID_JESS2_COLOR_RUMBLE_PAD) },
-#endif
-#if IS_ENABLED(CONFIG_HID_PENMOUNT)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PENMOUNT, USB_DEVICE_ID_PENMOUNT_6000) },
-#endif
-#if IS_ENABLED(CONFIG_HID_PETALYNX)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PETALYNX, USB_DEVICE_ID_PETALYNX_MAXTER_REMOTE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_PICOLCD)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICOLCD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICOLCD_BOOTLOADER) },
-#endif
-#if IS_ENABLED(CONFIG_HID_PLANTRONICS)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PLANTRONICS, HID_ANY_ID) },
-#endif
-#if IS_ENABLED(CONFIG_HID_PRIMAX)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_KEYBOARD) },
-#endif
-#if IS_ENABLED(CONFIG_HID_PRODIKEYS)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CREATIVELABS, USB_DEVICE_ID_PRODIKEYS_PCMIDI) },
-#endif
-#if IS_ENABLED(CONFIG_HID_RETRODE)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_FUTURE_TECHNOLOGY, USB_DEVICE_ID_RETRODE2) },
-#endif
-#if IS_ENABLED(CONFIG_HID_RMI)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X1_COVER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_RAZER, USB_DEVICE_ID_RAZER_BLADE_14) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ROCCAT)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_ARVO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_ISKU) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_ISKUFX) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEPLUS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEPURE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEPURE_OPTICAL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEXTD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KOVAPLUS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_LUA) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_PYRA_WIRED) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_PYRA_WIRELESS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_RYOS_MK) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_RYOS_MK_GLOW) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_RYOS_MK_PRO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_SAVU) },
-#endif
-#if IS_ENABLED(CONFIG_HID_SAITEK)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_PS1000) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT7_OLD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT7) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT9) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_MMO7) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_RAT5) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_RAT9) },
-#endif
-#if IS_ENABLED(CONFIG_HID_SAMSUNG)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SAMSUNG, USB_DEVICE_ID_SAMSUNG_IR_REMOTE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SAMSUNG, USB_DEVICE_ID_SAMSUNG_WIRELESS_KBD_MOUSE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_SMARTJOYPLUS)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PLAYDOTCOM, USB_DEVICE_ID_PLAYDOTCOM_EMS_USBII) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_SMARTJOY_PLUS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_SUPER_JOY_BOX_3) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_DUAL_USB_JOYPAD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP_LTD, USB_DEVICE_ID_SUPER_JOY_BOX_3_PRO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP_LTD, USB_DEVICE_ID_SUPER_DUAL_BOX_PRO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP_LTD, USB_DEVICE_ID_SUPER_JOY_BOX_5_PRO) },
-#endif
-#if IS_ENABLED(CONFIG_HID_SONY)
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_HARMONY_PS3) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SMK, USB_DEVICE_ID_SMK_PS3_BDREMOTE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_BUZZ_CONTROLLER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_WIRELESS_BUZZ_CONTROLLER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_MOTION_CONTROLLER) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_MOTION_CONTROLLER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_NAVIGATION_CONTROLLER) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_NAVIGATION_CONTROLLER) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_BDREMOTE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_CONTROLLER) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_CONTROLLER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_DONGLE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_VAIO_VGX_MOUSE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_VAIO_VGP_MOUSE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SINO_LITE, USB_DEVICE_ID_SINO_LITE_CONTROLLER) },
-#endif
-#if IS_ENABLED(CONFIG_HID_SPEEDLINK)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_X_TENSIONS, USB_DEVICE_ID_SPEEDLINK_VAD_CEZANNE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_STEELSERIES)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_SRWS1) },
-#endif
-#if IS_ENABLED(CONFIG_HID_SUNPLUS)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SUNPLUS, USB_DEVICE_ID_SUNPLUS_WDESKTOP) },
-#endif
-#if IS_ENABLED(CONFIG_HID_THRUSTMASTER)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb300) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb304) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb323) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb324) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb605) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb651) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb653) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb654) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb65a) },
-#endif
-#if IS_ENABLED(CONFIG_HID_TIVO)
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_TIVO, USB_DEVICE_ID_TIVO_SLIDE_BT) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_TIVO, USB_DEVICE_ID_TIVO_SLIDE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_TIVO, USB_DEVICE_ID_TIVO_SLIDE_PRO) },
-#endif
-#if IS_ENABLED(CONFIG_HID_TOPSEED)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BTC, USB_DEVICE_ID_BTC_EMPREX_REMOTE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BTC, USB_DEVICE_ID_BTC_EMPREX_REMOTE_2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_WIRELESS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_TOPSEED, USB_DEVICE_ID_TOPSEED_CYBERLINK) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_TOPSEED2, USB_DEVICE_ID_TOPSEED2_RF_COMBO) },
-#endif
-#if IS_ENABLED(CONFIG_HID_TWINHAN)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_TWINHAN, USB_DEVICE_ID_TWINHAN_IR_REMOTE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_UCLOGIC)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_HUION, USB_DEVICE_ID_HUION_TABLET) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_HUION_TABLET) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_PF1209) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP4030U) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP5540U) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP8060U) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP1062) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_WIRELESS_TABLET_TWHL850) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWHA60) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_YIYNOVA_TABLET) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_81) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_45) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_DRAWIMAGE_G3) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_TABLET_EX07S) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_UGTIZER, USB_DEVICE_ID_UGTIZER_TABLET_GP0610) },
-#endif
-#if IS_ENABLED(CONFIG_HID_UDRAW_PS3)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_THQ, USB_DEVICE_ID_THQ_PS3_UDRAW) },
-#endif
-#if IS_ENABLED(CONFIG_HID_WALTOP)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SLIM_TABLET_5_8_INCH) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SLIM_TABLET_12_1_INCH) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_Q_PAD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_PID_0038) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_10_6_INCH) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_14_1_INCH) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SIRIUS_BATTERY_FREE_TABLET) },
-#endif
-#if IS_ENABLED(CONFIG_HID_XINMO)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_XIN_MO, USB_DEVICE_ID_XIN_MO_DUAL_ARCADE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_XIN_MO, USB_DEVICE_ID_THT_2P_ARCADE) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ZEROPLUS)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ZEROPLUS, 0x0005) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ZEROPLUS, 0x0030) },
-#endif
-#if IS_ENABLED(CONFIG_HID_ZYDACRON)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ZYDACRON, USB_DEVICE_ID_ZYDACRON_REMOTE_CONTROL) },
-#endif
-	{ }
-};
-
 struct hid_dynid {
 	struct list_head list;
 	struct hid_device_id id;
@@ -2463,8 +1903,8 @@ static void hid_free_dynids(struct hid_driver *hdrv)
 	spin_unlock(&hdrv->dyn_lock);
 }
 
-static const struct hid_device_id *hid_match_device(struct hid_device *hdev,
-		struct hid_driver *hdrv)
+const struct hid_device_id *hid_match_device(struct hid_device *hdev,
+					     struct hid_driver *hdrv)
 {
 	struct hid_dynid *dynid;
 
@@ -2479,6 +1919,7 @@ static const struct hid_device_id *hid_match_device(struct hid_device *hdev,
 
 	return hid_match_id(hdev, hdrv->id_table);
 }
+EXPORT_SYMBOL_GPL(hid_match_device);
 
 static int hid_bus_match(struct device *dev, struct device_driver *drv)
 {
@@ -2508,6 +1949,23 @@ static int hid_device_probe(struct device *dev)
 			goto unlock;
 		}
 
+		if (hdrv->match) {
+			if (!hdrv->match(hdev, hid_ignore_special_drivers)) {
+				ret = -ENODEV;
+				goto unlock;
+			}
+		} else {
+			/*
+			 * hid-generic implements .match(), so if
+			 * hid_ignore_special_drivers is set, we can safely
+			 * return.
+			 */
+			if (hid_ignore_special_drivers) {
+				ret = -ENODEV;
+				goto unlock;
+			}
+		}
+
 		hdev->driver = hdrv;
 		if (hdrv->probe) {
 			ret = hdrv->probe(hdev, id);
@@ -2604,7 +2062,7 @@ static int hid_uevent(struct device *dev, struct kobj_uevent_env *env)
 	return 0;
 }
 
-static struct bus_type hid_bus_type = {
+struct bus_type hid_bus_type = {
 	.name		= "hid",
 	.dev_groups	= hid_dev_groups,
 	.drv_groups	= hid_drv_groups,
@@ -2613,315 +2071,7 @@ static struct bus_type hid_bus_type = {
 	.remove		= hid_device_remove,
 	.uevent		= hid_uevent,
 };
-
-/* a list of devices that shouldn't be handled by HID core at all */
-static const struct hid_device_id hid_ignore_list[] = {
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ACECAD, USB_DEVICE_ID_ACECAD_FLAIR) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ACECAD, USB_DEVICE_ID_ACECAD_302) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ADS_TECH, USB_DEVICE_ID_ADS_TECH_RADIO_SI470X) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_01) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_10) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_20) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_21) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_22) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_23) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_24) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AIRCABLE, USB_DEVICE_ID_AIRCABLE1) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ALCOR, USB_DEVICE_ID_ALCOR_USBRS232) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_LCM)},
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_LCM2)},
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AVERMEDIA, USB_DEVICE_ID_AVER_FM_MR800) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_AXENTIA, USB_DEVICE_ID_AXENTIA_FM_RADIO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_BERKSHIRE, USB_DEVICE_ID_BERKSHIRE_PCWD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CIDC, 0x0103) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI470X) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI4713) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CMEDIA, USB_DEVICE_ID_CM109) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_HIDCOM) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_ULTRAMOUSE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DEALEXTREAME, USB_DEVICE_ID_DEALEXTREAME_RADIO_SI4701) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DELORME, USB_DEVICE_ID_DELORME_EARTHMATE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_DELORME, USB_DEVICE_ID_DELORME_EM_LT20) },
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, 0x0400) },
-	{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, 0x0401) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ESSENTIAL_REALITY, USB_DEVICE_ID_ESSENTIAL_REALITY_P5) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ETT, USB_DEVICE_ID_TC5UH) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ETT, USB_DEVICE_ID_TC4UM) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GENERAL_TOUCH, 0x0001) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GENERAL_TOUCH, 0x0002) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GENERAL_TOUCH, 0x0004) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GOTOP, USB_DEVICE_ID_SUPER_Q2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GOTOP, USB_DEVICE_ID_GOGOPEN) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GOTOP, USB_DEVICE_ID_PENPOWER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GRETAGMACBETH, USB_DEVICE_ID_GRETAGMACBETH_HUEY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GRIFFIN, USB_DEVICE_ID_POWERMATE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GRIFFIN, USB_DEVICE_ID_SOUNDKNOB) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GRIFFIN, USB_DEVICE_ID_RADIOSHARK) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_90) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_100) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_101) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_103) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_104) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_105) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_106) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_107) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_108) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_200) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_201) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_202) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_203) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_204) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_205) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_206) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_207) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_300) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_301) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_302) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_303) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_304) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_305) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_306) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_307) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_308) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_309) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_400) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_401) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_402) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_403) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_404) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_405) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_500) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_501) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_502) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_503) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_504) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1000) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1001) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1002) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1003) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1004) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1005) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1006) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1007) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_IMATION, USB_DEVICE_ID_DISC_STAKKA) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_JABRA, USB_DEVICE_ID_JABRA_SPEAK_410) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_JABRA, USB_DEVICE_ID_JABRA_SPEAK_510) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_JABRA, USB_DEVICE_ID_JABRA_GN9350E) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KBGEAR, USB_DEVICE_ID_KBGEAR_JAMSTUDIO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KWORLD, USB_DEVICE_ID_KWORLD_RADIO_FM700) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_GPEN_560) },
-	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_KYE, 0x0058) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_CASSY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_CASSY2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_POCKETCASSY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_POCKETCASSY2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOBILECASSY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOBILECASSY2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYVOLTAGE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYCURRENT) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYTIME) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYTEMPERATURE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYPH) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_JWM) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_DMMP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_UMIP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_UMIC) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_UMIB) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_XRAY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_XRAY2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_VIDEOCOM) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOTOR) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_COM3LAB) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_TELEPORT) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_NETWORKANALYSER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_POWERCONTROL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MACHINETEST) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOSTANALYSER) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOSTANALYSER2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_ABSESP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_AUTODATABUS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MCT) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_HYBRID) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_HEATCONTROL) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_BEATPAD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MCC, USB_DEVICE_ID_MCC_PMD1024LS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MCC, USB_DEVICE_ID_MCC_PMD1208LS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICKIT1) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICKIT2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICK16F1454) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICK16F1454_V2) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_NATIONAL_SEMICONDUCTOR, USB_DEVICE_ID_N_S_HARMONY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 20) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 30) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 100) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 108) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 118) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 200) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 300) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 400) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 500) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0001) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0002) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0003) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0004) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PETZL, USB_DEVICE_ID_PETZL_HEADLAMP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_PHILIPS, USB_DEVICE_ID_PHILIPS_IEEE802154_DONGLE) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_POWERCOM, USB_DEVICE_ID_POWERCOM_UPS) },
-#if IS_ENABLED(CONFIG_MOUSE_SYNAPTICS_USB)
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_TP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_INT_TP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_CPAD) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_STICK) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_WP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_COMP_TP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_WTP) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_DPAD) },
-#endif
-	{ HID_USB_DEVICE(USB_VENDOR_ID_YEALINK, USB_DEVICE_ID_YEALINK_P1K_P4K_B2K) },
-	{ }
-};
-
-/**
- * hid_mouse_ignore_list - mouse devices which should not be handled by the hid layer
- *
- * There are composite devices for which we want to ignore only a certain
- * interface. This is a list of devices for which only the mouse interface will
- * be ignored. This allows a dedicated driver to take care of the interface.
- */
-static const struct hid_device_id hid_mouse_ignore_list[] = {
-	/* appletouch driver */
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ISO) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY) },
-	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) },
-	{ }
-};
-
-bool hid_ignore(struct hid_device *hdev)
-{
-	if (hdev->quirks & HID_QUIRK_NO_IGNORE)
-		return false;
-	if (hdev->quirks & HID_QUIRK_IGNORE)
-		return true;
-
-	switch (hdev->vendor) {
-	case USB_VENDOR_ID_CODEMERCS:
-		/* ignore all Code Mercenaries IOWarrior devices */
-		if (hdev->product >= USB_DEVICE_ID_CODEMERCS_IOW_FIRST &&
-				hdev->product <= USB_DEVICE_ID_CODEMERCS_IOW_LAST)
-			return true;
-		break;
-	case USB_VENDOR_ID_LOGITECH:
-		if (hdev->product >= USB_DEVICE_ID_LOGITECH_HARMONY_FIRST &&
-				hdev->product <= USB_DEVICE_ID_LOGITECH_HARMONY_LAST)
-			return true;
-		/*
-		 * The Keene FM transmitter USB device has the same USB ID as
-		 * the Logitech AudioHub Speaker, but it should ignore the hid.
-		 * Check if the name is that of the Keene device.
-		 * For reference: the name of the AudioHub is
-		 * "HOLTEK  AudioHub Speaker".
-		 */
-		if (hdev->product == USB_DEVICE_ID_LOGITECH_AUDIOHUB &&
-			!strcmp(hdev->name, "HOLTEK  B-LINK USB Audio  "))
-				return true;
-		break;
-	case USB_VENDOR_ID_SOUNDGRAPH:
-		if (hdev->product >= USB_DEVICE_ID_SOUNDGRAPH_IMON_FIRST &&
-		    hdev->product <= USB_DEVICE_ID_SOUNDGRAPH_IMON_LAST)
-			return true;
-		break;
-	case USB_VENDOR_ID_HANWANG:
-		if (hdev->product >= USB_DEVICE_ID_HANWANG_TABLET_FIRST &&
-		    hdev->product <= USB_DEVICE_ID_HANWANG_TABLET_LAST)
-			return true;
-		break;
-	case USB_VENDOR_ID_JESS:
-		if (hdev->product == USB_DEVICE_ID_JESS_YUREX &&
-				hdev->type == HID_TYPE_USBNONE)
-			return true;
-		break;
-	case USB_VENDOR_ID_VELLEMAN:
-		/* These are not HID devices.  They are handled by comedi. */
-		if ((hdev->product >= USB_DEVICE_ID_VELLEMAN_K8055_FIRST &&
-		     hdev->product <= USB_DEVICE_ID_VELLEMAN_K8055_LAST) ||
-		    (hdev->product >= USB_DEVICE_ID_VELLEMAN_K8061_FIRST &&
-		     hdev->product <= USB_DEVICE_ID_VELLEMAN_K8061_LAST))
-			return true;
-		break;
-	case USB_VENDOR_ID_ATMEL_V_USB:
-		/* Masterkit MA901 usb radio based on Atmel tiny85 chip and
-		 * it has the same USB ID as many Atmel V-USB devices. This
-		 * usb radio is handled by radio-ma901.c driver so we want
-		 * ignore the hid. Check the name, bus, product and ignore
-		 * if we have MA901 usb radio.
-		 */
-		if (hdev->product == USB_DEVICE_ID_ATMEL_V_USB &&
-			hdev->bus == BUS_USB &&
-			strncmp(hdev->name, "www.masterkit.ru MA901", 22) == 0)
-			return true;
-		break;
-	}
-
-	if (hdev->type == HID_TYPE_USBMOUSE &&
-			hid_match_id(hdev, hid_mouse_ignore_list))
-		return true;
-
-	return !!hid_match_id(hdev, hid_ignore_list);
-}
-EXPORT_SYMBOL_GPL(hid_ignore);
+EXPORT_SYMBOL(hid_bus_type);
 
 int hid_add_device(struct hid_device *hdev)
 {
@@ -2931,6 +2081,8 @@ int hid_add_device(struct hid_device *hdev)
 	if (WARN_ON(hdev->status & HID_STAT_ADDED))
 		return -EBUSY;
 
+	hdev->quirks = hid_lookup_quirk(hdev);
+
 	/* we need to kill them here, otherwise they will stay allocated to
 	 * wait for coming driver */
 	if (hid_ignore(hdev))
@@ -2960,7 +2112,7 @@ int hid_add_device(struct hid_device *hdev)
 	if (hid_ignore_special_drivers) {
 		hdev->group = HID_GROUP_GENERIC;
 	} else if (!hdev->group &&
-		   !hid_match_id(hdev, hid_have_special_driver)) {
+		   !(hdev->quirks & HID_QUIRK_HAVE_SPECIAL_DRIVER)) {
 		ret = hid_scan_report(hdev);
 		if (ret)
 			hid_warn(hdev, "bad device descriptor (%d)\n", ret);
@@ -3044,6 +2196,29 @@ void hid_destroy_device(struct hid_device *hdev)
 }
 EXPORT_SYMBOL_GPL(hid_destroy_device);
 
+
+static int __bus_add_driver(struct device_driver *drv, void *data)
+{
+	struct hid_driver *added_hdrv = data;
+	struct hid_driver *hdrv = to_hid_driver(drv);
+
+	if (hdrv->bus_add_driver)
+		hdrv->bus_add_driver(added_hdrv);
+
+	return 0;
+}
+
+static int __bus_removed_driver(struct device_driver *drv, void *data)
+{
+	struct hid_driver *removed_hdrv = data;
+	struct hid_driver *hdrv = to_hid_driver(drv);
+
+	if (hdrv->bus_removed_driver)
+		hdrv->bus_removed_driver(removed_hdrv);
+
+	return 0;
+}
+
 int __hid_register_driver(struct hid_driver *hdrv, struct module *owner,
 		const char *mod_name)
 {
@@ -3055,6 +2230,8 @@ int __hid_register_driver(struct hid_driver *hdrv, struct module *owner,
 	INIT_LIST_HEAD(&hdrv->dyn_list);
 	spin_lock_init(&hdrv->dyn_lock);
 
+	bus_for_each_drv(&hid_bus_type, NULL, hdrv, __bus_add_driver);
+
 	return driver_register(&hdrv->driver);
 }
 EXPORT_SYMBOL_GPL(__hid_register_driver);
@@ -3063,6 +2240,8 @@ void hid_unregister_driver(struct hid_driver *hdrv)
 {
 	driver_unregister(&hdrv->driver);
 	hid_free_dynids(hdrv);
+
+	bus_for_each_drv(&hid_bus_type, NULL, hdrv, __bus_removed_driver);
 }
 EXPORT_SYMBOL_GPL(hid_unregister_driver);
 
@@ -3117,6 +2296,7 @@ static void __exit hid_exit(void)
 	hid_debug_exit();
 	hidraw_exit();
 	bus_unregister(&hid_bus_type);
+	hid_quirks_exit(HID_BUS_ANY);
 }
 
 module_init(hid_init);
diff --git a/drivers/hid/hid-debug.c b/drivers/hid/hid-debug.c
index 5271db593478..4f4e7a08a07b 100644
--- a/drivers/hid/hid-debug.c
+++ b/drivers/hid/hid-debug.c
@@ -1179,15 +1179,15 @@ static ssize_t hid_debug_events_read(struct file *file, char __user *buffer,
 	return ret;
 }
 
-static unsigned int hid_debug_events_poll(struct file *file, poll_table *wait)
+static __poll_t hid_debug_events_poll(struct file *file, poll_table *wait)
 {
 	struct hid_debug_list *list = file->private_data;
 
 	poll_wait(file, &list->hdev->debug_wait, wait);
 	if (list->head != list->tail)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	if (!list->hdev->debug)
-		return POLLERR | POLLHUP;
+		return EPOLLERR | EPOLLHUP;
 	return 0;
 }
 
diff --git a/drivers/hid/hid-elecom.c b/drivers/hid/hid-elecom.c
index 54aeea57d209..1a1ecc491c02 100644
--- a/drivers/hid/hid-elecom.c
+++ b/drivers/hid/hid-elecom.c
@@ -1,9 +1,15 @@
 /*
- *  HID driver for ELECOM devices.
+ *  HID driver for ELECOM devices:
+ *  - BM084 Bluetooth Mouse
+ *  - EX-G Trackball (Wired and wireless)
+ *  - DEFT Trackball (Wired and wireless)
+ *  - HUGE Trackball (Wired and wireless)
+ *
  *  Copyright (c) 2010 Richard Nauber <Richard.Nauber@gmail.com>
  *  Copyright (c) 2016 Yuxuan Shui <yshuiv7@gmail.com>
  *  Copyright (c) 2017 Diego Elio Pettenò <flameeyes@flameeyes.eu>
  *  Copyright (c) 2017 Alex Manoussakis <amanou@gnu.org>
+ *  Copyright (c) 2017 Tomasz Kramkowski <tk@the-tk.com>
  */
 
 /*
@@ -19,6 +25,34 @@
 
 #include "hid-ids.h"
 
+/*
+ * Certain ELECOM mice misreport their button count meaning that they only work
+ * correctly with the ELECOM mouse assistant software which is unavailable for
+ * Linux. A four extra INPUT reports and a FEATURE report are described by the
+ * report descriptor but it does not appear that these enable software to
+ * control what the extra buttons map to. The only simple and straightforward
+ * solution seems to involve fixing up the report descriptor.
+ *
+ * Report descriptor format:
+ * Positions 13, 15, 21 and 31 store the button bit count, button usage minimum,
+ * button usage maximum and padding bit count respectively.
+ */
+#define MOUSE_BUTTONS_MAX 8
+static void mouse_button_fixup(struct hid_device *hdev,
+			       __u8 *rdesc, unsigned int rsize,
+			       int nbuttons)
+{
+	if (rsize < 32 || rdesc[12] != 0x95 ||
+	    rdesc[14] != 0x75 || rdesc[15] != 0x01 ||
+	    rdesc[20] != 0x29 || rdesc[30] != 0x75)
+		return;
+	hid_info(hdev, "Fixing up Elecom mouse button count\n");
+	nbuttons = clamp(nbuttons, 0, MOUSE_BUTTONS_MAX);
+	rdesc[13] = nbuttons;
+	rdesc[21] = nbuttons;
+	rdesc[31] = MOUSE_BUTTONS_MAX - nbuttons;
+}
+
 static __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 		unsigned int *rsize)
 {
@@ -31,45 +65,15 @@ static __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 			rdesc[47] = 0x00;
 		}
 		break;
+	case USB_DEVICE_ID_ELECOM_EX_G_WIRED:
+	case USB_DEVICE_ID_ELECOM_EX_G_WIRELESS:
+		mouse_button_fixup(hdev, rdesc, *rsize, 6);
+		break;
 	case USB_DEVICE_ID_ELECOM_DEFT_WIRED:
 	case USB_DEVICE_ID_ELECOM_DEFT_WIRELESS:
 	case USB_DEVICE_ID_ELECOM_HUGE_WIRED:
 	case USB_DEVICE_ID_ELECOM_HUGE_WIRELESS:
-		/* The DEFT/HUGE trackball has eight buttons, but its descriptor
-		 * only reports five, disabling the three Fn buttons on the top
-		 * of the mouse.
-		 *
-		 * Apply the following diff to the descriptor:
-		 *
-		 * Collection (Physical),              Collection (Physical),
-		 *     Report ID (1),                      Report ID (1),
-		 *     Report Count (5),           ->      Report Count (8),
-		 *     Report Size (1),                    Report Size (1),
-		 *     Usage Page (Button),                Usage Page (Button),
-		 *     Usage Minimum (01h),                Usage Minimum (01h),
-		 *     Usage Maximum (05h),        ->      Usage Maximum (08h),
-		 *     Logical Minimum (0),                Logical Minimum (0),
-		 *     Logical Maximum (1),                Logical Maximum (1),
-		 *     Input (Variable),                   Input (Variable),
-		 *     Report Count (1),           ->      Report Count (0),
-		 *     Report Size (3),                    Report Size (3),
-		 *     Input (Constant),                   Input (Constant),
-		 *     Report Size (16),                   Report Size (16),
-		 *     Report Count (2),                   Report Count (2),
-		 *     Usage Page (Desktop),               Usage Page (Desktop),
-		 *     Usage (X),                          Usage (X),
-		 *     Usage (Y),                          Usage (Y),
-		 *     Logical Minimum (-32768),           Logical Minimum (-32768),
-		 *     Logical Maximum (32767),            Logical Maximum (32767),
-		 *     Input (Variable, Relative),         Input (Variable, Relative),
-		 * End Collection,                     End Collection,
-		 */
-		if (*rsize == 213 && rdesc[13] == 5 && rdesc[21] == 5) {
-			hid_info(hdev, "Fixing up Elecom DEFT/HUGE Fn buttons\n");
-			rdesc[13] = 8; /* Button/Variable Report Count */
-			rdesc[21] = 8; /* Button/Variable Usage Maximum */
-			rdesc[29] = 0; /* Button/Constant Report Count */
-		}
+		mouse_button_fixup(hdev, rdesc, *rsize, 8);
 		break;
 	}
 	return rdesc;
@@ -77,6 +81,8 @@ static __u8 *elecom_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 
 static const struct hid_device_id elecom_devices[] = {
 	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_EX_G_WIRED) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_EX_G_WIRELESS) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_DEFT_WIRED) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_DEFT_WIRELESS) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_HUGE_WIRED) },
diff --git a/drivers/hid/hid-elo.c b/drivers/hid/hid-elo.c
index 0cd4f7216239..5eea6fe0d7bd 100644
--- a/drivers/hid/hid-elo.c
+++ b/drivers/hid/hid-elo.c
@@ -42,6 +42,12 @@ static int elo_input_configured(struct hid_device *hdev,
 {
 	struct input_dev *input = hidinput->input;
 
+	/*
+	 * ELO devices have one Button usage in GenDesk field, which makes
+	 * hid-input map it to BTN_LEFT; that confuses userspace, which then
+	 * considers the device to be a mouse/touchpad instead of touchscreen.
+	 */
+	clear_bit(BTN_LEFT, input->keybit);
 	set_bit(BTN_TOUCH, input->keybit);
 	set_bit(ABS_PRESSURE, input->absbit);
 	input_set_abs_params(input, ABS_PRESSURE, 0, 256, 0, 0);
diff --git a/drivers/hid/hid-generic.c b/drivers/hid/hid-generic.c
index e288a4a06fe8..3c0a1bf433d7 100644
--- a/drivers/hid/hid-generic.c
+++ b/drivers/hid/hid-generic.c
@@ -24,8 +24,71 @@
 
 #include <linux/hid.h>
 
+static struct hid_driver hid_generic;
+
+static int __unmap_hid_generic(struct device *dev, void *data)
+{
+	struct hid_driver *hdrv = data;
+	struct hid_device *hdev = to_hid_device(dev);
+
+	/* only unbind matching devices already bound to hid-generic */
+	if (hdev->driver != &hid_generic ||
+	    hid_match_device(hdev, hdrv) == NULL)
+		return 0;
+
+	if (dev->parent)	/* Needed for USB */
+		device_lock(dev->parent);
+	device_release_driver(dev);
+	if (dev->parent)
+		device_unlock(dev->parent);
+
+	return 0;
+}
+
+static void hid_generic_add_driver(struct hid_driver *hdrv)
+{
+	bus_for_each_dev(&hid_bus_type, NULL, hdrv, __unmap_hid_generic);
+}
+
+static void hid_generic_removed_driver(struct hid_driver *hdrv)
+{
+	int ret;
+
+	ret = driver_attach(&hid_generic.driver);
+}
+
+static int __check_hid_generic(struct device_driver *drv, void *data)
+{
+	struct hid_driver *hdrv = to_hid_driver(drv);
+	struct hid_device *hdev = data;
+
+	if (hdrv == &hid_generic)
+		return 0;
+
+	return hid_match_device(hdev, hdrv) != NULL;
+}
+
+static bool hid_generic_match(struct hid_device *hdev,
+			      bool ignore_special_driver)
+{
+	if (ignore_special_driver)
+		return true;
+
+	if (hdev->quirks & HID_QUIRK_HAVE_SPECIAL_DRIVER)
+		return false;
+
+	/*
+	 * If any other driver wants the device, leave the device to this other
+	 * driver.
+	 */
+	if (bus_for_each_drv(&hid_bus_type, NULL, hdev, __check_hid_generic))
+		return false;
+
+	return true;
+}
+
 static const struct hid_device_id hid_table[] = {
-	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_GENERIC, HID_ANY_ID, HID_ANY_ID) },
+	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, HID_ANY_ID, HID_ANY_ID) },
 	{ }
 };
 MODULE_DEVICE_TABLE(hid, hid_table);
@@ -33,6 +96,9 @@ MODULE_DEVICE_TABLE(hid, hid_table);
 static struct hid_driver hid_generic = {
 	.name = "hid-generic",
 	.id_table = hid_table,
+	.match = hid_generic_match,
+	.bus_add_driver = hid_generic_add_driver,
+	.bus_removed_driver = hid_generic_removed_driver,
 };
 module_hid_driver(hid_generic);
 
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 5da3d6256d25..9454ac134ce2 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -178,7 +178,8 @@
 #define USB_VENDOR_ID_ASUSTEK		0x0b05
 #define USB_DEVICE_ID_ASUSTEK_LCM	0x1726
 #define USB_DEVICE_ID_ASUSTEK_LCM2	0x175b
-#define USB_DEVICE_ID_ASUSTEK_T100_KEYBOARD	0x17e0
+#define USB_DEVICE_ID_ASUSTEK_T100TA_KEYBOARD	0x17e0
+#define USB_DEVICE_ID_ASUSTEK_T100TAF_KEYBOARD	0x1807
 #define USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD	0x8502
 #define USB_DEVICE_ID_ASUSTEK_T304_KEYBOARD	0x184a
 #define USB_DEVICE_ID_ASUSTEK_I2C_KEYBOARD	0x8585
@@ -370,6 +371,8 @@
 
 #define USB_VENDOR_ID_ELECOM		0x056e
 #define USB_DEVICE_ID_ELECOM_BM084	0x0061
+#define USB_DEVICE_ID_ELECOM_EX_G_WIRED	0x00fb
+#define USB_DEVICE_ID_ELECOM_EX_G_WIRELESS	0x00fc
 #define USB_DEVICE_ID_ELECOM_DEFT_WIRED	0x00fe
 #define USB_DEVICE_ID_ELECOM_DEFT_WIRELESS	0x00ff
 #define USB_DEVICE_ID_ELECOM_HUGE_WIRED	0x010c
@@ -535,6 +538,7 @@
 #define USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0A4A	0x0a4a
 #define USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0B4A	0x0b4a
 #define USB_PRODUCT_ID_HP_PIXART_OEM_USB_OPTICAL_MOUSE		0x134a
+#define USB_PRODUCT_ID_HP_PIXART_OEM_USB_OPTICAL_MOUSE_094A	0x094a
 
 #define USB_VENDOR_ID_HUION		0x256c
 #define USB_DEVICE_ID_HUION_TABLET	0x006e
@@ -641,6 +645,9 @@
 #define USB_DEVICE_ID_LD_MICROCASSYTIME		0x1033
 #define USB_DEVICE_ID_LD_MICROCASSYTEMPERATURE	0x1035
 #define USB_DEVICE_ID_LD_MICROCASSYPH		0x1038
+#define USB_DEVICE_ID_LD_POWERANALYSERCASSY	0x1040
+#define USB_DEVICE_ID_LD_CONVERTERCONTROLLERCASSY	0x1042
+#define USB_DEVICE_ID_LD_MACHINETESTCASSY	0x1043
 #define USB_DEVICE_ID_LD_JWM		0x1080
 #define USB_DEVICE_ID_LD_DMMP		0x1081
 #define USB_DEVICE_ID_LD_UMIP		0x1090
@@ -1156,6 +1163,7 @@
 #define USB_VENDOR_ID_PRIMAX	0x0461
 #define USB_DEVICE_ID_PRIMAX_MOUSE_4D22	0x4d22
 #define USB_DEVICE_ID_PRIMAX_KEYBOARD	0x4e05
+#define USB_DEVICE_ID_PRIMAX_REZEL	0x4e72
 
 
 #define USB_VENDOR_ID_RISO_KAGAKU	0x1294	/* Riso Kagaku Corp. */
diff --git a/drivers/hid/hid-jabra.c b/drivers/hid/hid-jabra.c
new file mode 100644
index 000000000000..1f52daf14426
--- /dev/null
+++ b/drivers/hid/hid-jabra.c
@@ -0,0 +1,58 @@
+/*
+ *  Jabra USB HID Driver
+ *
+ *  Copyright (c) 2017 Niels Skou Olsen <nolsen@jabra.com>
+ */
+
+/*
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <linux/hid.h>
+#include <linux/module.h>
+
+#include "hid-ids.h"
+
+#define HID_UP_VENDOR_DEFINED_MIN	0xff000000
+#define HID_UP_VENDOR_DEFINED_MAX	0xffff0000
+
+static int jabra_input_mapping(struct hid_device *hdev,
+			       struct hid_input *hi,
+			       struct hid_field *field,
+			       struct hid_usage *usage,
+			       unsigned long **bit, int *max)
+{
+	int is_vendor_defined =
+		((usage->hid & HID_USAGE_PAGE) >= HID_UP_VENDOR_DEFINED_MIN &&
+		 (usage->hid & HID_USAGE_PAGE) <= HID_UP_VENDOR_DEFINED_MAX);
+
+	dbg_hid("hid=0x%08x appl=0x%08x coll_idx=0x%02x usage_idx=0x%02x: %s\n",
+		usage->hid,
+		field->application,
+		usage->collection_index,
+		usage->usage_index,
+		is_vendor_defined ? "ignored" : "defaulted");
+
+	/* Ignore vendor defined usages, default map standard usages */
+	return is_vendor_defined ? -1 : 0;
+}
+
+static const struct hid_device_id jabra_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_JABRA, HID_ANY_ID) },
+	{ }
+};
+MODULE_DEVICE_TABLE(hid, jabra_devices);
+
+static struct hid_driver jabra_driver = {
+	.name = "jabra",
+	.id_table = jabra_devices,
+	.input_mapping = jabra_input_mapping,
+};
+module_hid_driver(jabra_driver);
+
+MODULE_AUTHOR("Niels Skou Olsen <nolsen@jabra.com>");
+MODULE_DESCRIPTION("Jabra USB HID Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index 65ea23be9677..3b4739bde05d 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -85,11 +85,12 @@ MODULE_LICENSE("GPL");
 #define MT_IO_FLAGS_PENDING_SLOTS	2
 
 struct mt_slot {
-	__s32 x, y, cx, cy, p, w, h;
+	__s32 x, y, cx, cy, p, w, h, a;
 	__s32 contactid;	/* the device ContactID assigned to this slot */
 	bool touch_state;	/* is the touch valid? */
 	bool inrange_state;	/* is the finger in proximity of the sensor? */
 	bool confidence_state;  /* is the touch made by a finger? */
+	bool has_azimuth;       /* the contact reports azimuth */
 };
 
 struct mt_class {
@@ -119,6 +120,10 @@ struct mt_device {
 	unsigned long mt_io_flags;	/* mt flags (MT_IO_FLAGS_*) */
 	int cc_index;	/* contact count field index in the report */
 	int cc_value_index;	/* contact count value index in the field */
+	int scantime_index;	/* scantime field index in the report */
+	int scantime_val_index;	/* scantime value index in the field */
+	int prev_scantime;	/* scantime reported in the previous packet */
+	int left_button_state;	/* left button state */
 	unsigned last_slot_field;	/* the last field of a slot */
 	unsigned mt_report_id;	/* the report ID of the multitouch device */
 	unsigned long initial_quirks;	/* initial quirks state */
@@ -582,8 +587,15 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 			if (!(cls->quirks & MT_QUIRK_NO_AREA)) {
 				set_abs(hi->input, ABS_MT_TOUCH_MINOR, field,
 					cls->sn_height);
-				input_set_abs_params(hi->input,
-					ABS_MT_ORIENTATION, 0, 1, 0, 0);
+
+				/*
+				 * Only set ABS_MT_ORIENTATION if it is not
+				 * already set by the HID_DG_AZIMUTH usage.
+				 */
+				if (!test_bit(ABS_MT_ORIENTATION,
+						hi->input->absbit))
+					input_set_abs_params(hi->input,
+						ABS_MT_ORIENTATION, 0, 1, 0, 0);
 			}
 			mt_store_field(usage, td, hi);
 			return 1;
@@ -599,6 +611,12 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 				EV_MSC, MSC_TIMESTAMP);
 			input_set_capability(hi->input, EV_MSC, MSC_TIMESTAMP);
 			mt_store_field(usage, td, hi);
+			/* Ignore if indexes are out of bounds. */
+			if (field->index >= field->report->maxfield ||
+			    usage->usage_index >= field->report_count)
+				return 1;
+			td->scantime_index = field->index;
+			td->scantime_val_index = usage->usage_index;
 			return 1;
 		case HID_DG_CONTACTCOUNT:
 			/* Ignore if indexes are out of bounds. */
@@ -608,6 +626,21 @@ static int mt_touch_input_mapping(struct hid_device *hdev, struct hid_input *hi,
 			td->cc_index = field->index;
 			td->cc_value_index = usage->usage_index;
 			return 1;
+		case HID_DG_AZIMUTH:
+			hid_map_usage(hi, usage, bit, max,
+				EV_ABS, ABS_MT_ORIENTATION);
+			/*
+			 * Azimuth has the range of [0, MAX) representing a full
+			 * revolution. Set ABS_MT_ORIENTATION to a quarter of
+			 * MAX according the definition of ABS_MT_ORIENTATION
+			 */
+			input_set_abs_params(hi->input, ABS_MT_ORIENTATION,
+				-field->logical_maximum / 4,
+				field->logical_maximum / 4,
+				cls->sn_move ?
+				field->logical_maximum / cls->sn_move : 0, 0);
+			mt_store_field(usage, td, hi);
+			return 1;
 		case HID_DG_CONTACTMAX:
 			/* we don't set td->last_slot_field as contactcount and
 			 * contact max are global to the report */
@@ -700,6 +733,10 @@ static void mt_complete_slot(struct mt_device *td, struct input_dev *input)
 			int wide = (s->w > s->h);
 			int major = max(s->w, s->h);
 			int minor = min(s->w, s->h);
+			int orientation = wide;
+
+			if (s->has_azimuth)
+				orientation = s->a;
 
 			/*
 			 * divided by two to match visual scale of touch
@@ -716,7 +753,8 @@ static void mt_complete_slot(struct mt_device *td, struct input_dev *input)
 			input_event(input, EV_ABS, ABS_MT_TOOL_Y, s->cy);
 			input_event(input, EV_ABS, ABS_MT_DISTANCE,
 				!s->touch_state);
-			input_event(input, EV_ABS, ABS_MT_ORIENTATION, wide);
+			input_event(input, EV_ABS, ABS_MT_ORIENTATION,
+				orientation);
 			input_event(input, EV_ABS, ABS_MT_PRESSURE, s->p);
 			input_event(input, EV_ABS, ABS_MT_TOUCH_MAJOR, major);
 			input_event(input, EV_ABS, ABS_MT_TOUCH_MINOR, minor);
@@ -734,10 +772,16 @@ static void mt_complete_slot(struct mt_device *td, struct input_dev *input)
  */
 static void mt_sync_frame(struct mt_device *td, struct input_dev *input)
 {
+	__s32 cls = td->mtclass.name;
+
+	if (cls == MT_CLS_WIN_8 || cls == MT_CLS_WIN_8_DUAL)
+		input_event(input, EV_KEY, BTN_LEFT, td->left_button_state);
+
 	input_mt_sync_frame(input);
 	input_event(input, EV_MSC, MSC_TIMESTAMP, td->timestamp);
 	input_sync(input);
 	td->num_received = 0;
+	td->left_button_state = 0;
 	if (test_bit(MT_IO_FLAGS_ACTIVE_SLOTS, &td->mt_io_flags))
 		set_bit(MT_IO_FLAGS_PENDING_SLOTS, &td->mt_io_flags);
 	else
@@ -778,9 +822,11 @@ static int mt_touch_event(struct hid_device *hid, struct hid_field *field,
 }
 
 static void mt_process_mt_event(struct hid_device *hid, struct hid_field *field,
-				struct hid_usage *usage, __s32 value)
+				struct hid_usage *usage, __s32 value,
+				bool first_packet)
 {
 	struct mt_device *td = hid_get_drvdata(hid);
+	__s32 cls = td->mtclass.name;
 	__s32 quirks = td->mtclass.quirks;
 	struct input_dev *input = field->hidinput->input;
 
@@ -832,11 +878,49 @@ static void mt_process_mt_event(struct hid_device *hid, struct hid_field *field,
 			break;
 		case HID_DG_CONTACTCOUNT:
 			break;
+		case HID_DG_AZIMUTH:
+			/*
+			 * Azimuth is counter-clockwise and ranges from [0, MAX)
+			 * (a full revolution). Convert it to clockwise ranging
+			 * [-MAX/2, MAX/2].
+			 *
+			 * Note that ABS_MT_ORIENTATION require us to report
+			 * the limit of [-MAX/4, MAX/4], but the value can go
+			 * out of range to [-MAX/2, MAX/2] to report an upside
+			 * down ellipsis.
+			 */
+			if (value > field->logical_maximum / 2)
+				value -= field->logical_maximum;
+			td->curdata.a = -value;
+			td->curdata.has_azimuth = true;
+			break;
 		case HID_DG_TOUCH:
 			/* do nothing */
 			break;
 
 		default:
+			/*
+			 * For Win8 PTP touchpads we should only look at
+			 * non finger/touch events in the first_packet of
+			 * a (possible) multi-packet frame.
+			 */
+			if ((cls == MT_CLS_WIN_8 || cls == MT_CLS_WIN_8_DUAL) &&
+			    !first_packet)
+				return;
+
+			/*
+			 * For Win8 PTP touchpads we map both the clickpad click
+			 * and any "external" left buttons to BTN_LEFT if a
+			 * device claims to have both we need to report 1 for
+			 * BTN_LEFT if either is pressed, so we or all values
+			 * together and report the result in mt_sync_frame().
+			 */
+			if ((cls == MT_CLS_WIN_8 || cls == MT_CLS_WIN_8_DUAL) &&
+			    usage->type == EV_KEY && usage->code == BTN_LEFT) {
+				td->left_button_state |= value;
+				return;
+			}
+
 			if (usage->type)
 				input_event(input, usage->type, usage->code,
 						value);
@@ -855,9 +939,11 @@ static void mt_process_mt_event(struct hid_device *hid, struct hid_field *field,
 static void mt_touch_report(struct hid_device *hid, struct hid_report *report)
 {
 	struct mt_device *td = hid_get_drvdata(hid);
+	__s32 cls = td->mtclass.name;
 	struct hid_field *field;
+	bool first_packet;
 	unsigned count;
-	int r, n;
+	int r, n, scantime = 0;
 
 	/* sticky fingers release in progress, abort */
 	if (test_and_set_bit(MT_IO_FLAGS_RUNNING, &td->mt_io_flags))
@@ -867,13 +953,31 @@ static void mt_touch_report(struct hid_device *hid, struct hid_report *report)
 	 * Includes multi-packet support where subsequent
 	 * packets are sent with zero contactcount.
 	 */
+	if (td->scantime_index >= 0) {
+		field = report->field[td->scantime_index];
+		scantime = field->value[td->scantime_val_index];
+	}
 	if (td->cc_index >= 0) {
 		struct hid_field *field = report->field[td->cc_index];
 		int value = field->value[td->cc_value_index];
-		if (value)
+
+		/*
+		 * For Win8 PTPs the first packet (td->num_received == 0) may
+		 * have a contactcount of 0 if there only is a button event.
+		 * We double check that this is not a continuation packet
+		 * of a possible multi-packet frame be checking that the
+		 * timestamp has changed.
+		 */
+		if ((cls == MT_CLS_WIN_8 || cls == MT_CLS_WIN_8_DUAL) &&
+		    td->num_received == 0 && td->prev_scantime != scantime)
+			td->num_expected = value;
+		/* A non 0 contact count always indicates a first packet */
+		else if (value)
 			td->num_expected = value;
 	}
+	td->prev_scantime = scantime;
 
+	first_packet = td->num_received == 0;
 	for (r = 0; r < report->maxfield; r++) {
 		field = report->field[r];
 		count = field->report_count;
@@ -883,7 +987,7 @@ static void mt_touch_report(struct hid_device *hid, struct hid_report *report)
 
 		for (n = 0; n < count; n++)
 			mt_process_mt_event(hid, field, &field->usage[n],
-					field->value[n]);
+					    field->value[n], first_packet);
 	}
 
 	if (td->num_received >= td->num_expected)
@@ -1329,6 +1433,7 @@ static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id)
 	td->maxcontact_report_id = -1;
 	td->inputmode_value = MT_INPUTMODE_TOUCHSCREEN;
 	td->cc_index = -1;
+	td->scantime_index = -1;
 	td->mt_report_id = -1;
 	hid_set_drvdata(hdev, td);
 
@@ -1649,14 +1754,6 @@ static const struct hid_device_id mt_devices[] = {
 		MT_USB_DEVICE(USB_VENDOR_ID_TURBOX,
 			USB_DEVICE_ID_TURBOX_TOUCHSCREEN_MOSART) },
 
-	/* Panasonic panels */
-	{ .driver_data = MT_CLS_PANASONIC,
-		MT_USB_DEVICE(USB_VENDOR_ID_PANASONIC,
-			USB_DEVICE_ID_PANABOARD_UBT780) },
-	{ .driver_data = MT_CLS_PANASONIC,
-		MT_USB_DEVICE(USB_VENDOR_ID_PANASONIC,
-			USB_DEVICE_ID_PANABOARD_UBT880) },
-
 	/* Novatek Panel */
 	{ .driver_data = MT_CLS_NSMU,
 		MT_USB_DEVICE(USB_VENDOR_ID_NOVATEK,
@@ -1667,6 +1764,14 @@ static const struct hid_device_id mt_devices[] = {
 		HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8,
 			USB_VENDOR_ID_NTRIG, 0x1b05) },
 
+	/* Panasonic panels */
+	{ .driver_data = MT_CLS_PANASONIC,
+		MT_USB_DEVICE(USB_VENDOR_ID_PANASONIC,
+			USB_DEVICE_ID_PANABOARD_UBT780) },
+	{ .driver_data = MT_CLS_PANASONIC,
+		MT_USB_DEVICE(USB_VENDOR_ID_PANASONIC,
+			USB_DEVICE_ID_PANABOARD_UBT880) },
+
 	/* PixArt optical touch screen */
 	{ .driver_data = MT_CLS_INRANGE_CONTACTNUMBER,
 		MT_USB_DEVICE(USB_VENDOR_ID_PIXART,
diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c
new file mode 100644
index 000000000000..e92b77fa574a
--- /dev/null
+++ b/drivers/hid/hid-quirks.c
@@ -0,0 +1,1279 @@
+/*
+ *  HID quirks support for Linux
+ *
+ *  Copyright (c) 1999 Andreas Gal
+ *  Copyright (c) 2000-2005 Vojtech Pavlik <vojtech@suse.cz>
+ *  Copyright (c) 2005 Michael Haboustak <mike-@cinci.rr.com> for Concept2, Inc
+ *  Copyright (c) 2006-2007 Jiri Kosina
+ *  Copyright (c) 2007 Paul Walmsley
+ */
+
+/*
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ */
+
+#include <linux/hid.h>
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/mutex.h>
+
+#include "hid-ids.h"
+
+/*
+ * Alphabetically sorted by vendor then product.
+ */
+
+static const struct hid_device_id hid_quirks[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_GAMEPAD), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_PREDATOR), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AFATECH, USB_DEVICE_ID_AFATECH_AF9016), HID_QUIRK_FULLSPEED_INTERVAL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIREN, USB_DEVICE_ID_AIREN_SLIMPLUS), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AKAI_09E8, USB_DEVICE_ID_AKAI_09E8_MIDIMIX), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AKAI, USB_DEVICE_ID_AKAI_MPKMINI2), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ALPS, USB_DEVICE_ID_IBM_GAMEPAD), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AMI, USB_DEVICE_ID_AMI_VIRT_KEYBOARD_AND_MOUSE), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_2PORTKVM), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_4PORTKVMC), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_4PORTKVM), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS124U), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS1758), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS682), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS692), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_UC100KM), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_MULTI_TOUCH), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_PIXART_USB_OPTICAL_MOUSE), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_WIRELESS), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHIC, USB_DEVICE_ID_CHIC_GAMEPAD), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_3AXIS_5BUTTON_STICK), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_AXIS_295), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_COMBATSTICK), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FIGHTERSTICK), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FLIGHT_SIM_ECLIPSE_YOKE), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FLIGHT_SIM_YOKE), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_PEDALS), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_THROTTLE), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB_RAPIDFIRE), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70RGB), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70RGB_RAPIDFIRE), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70R), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K95RGB), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_M65RGB), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_SCIMITAR_PRO_RGB), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_STRAFE), HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CREATIVELABS, USB_DEVICE_ID_CREATIVE_SB_OMNI_SURROUND_51), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DELL, USB_DEVICE_ID_DELL_PIXART_USB_OPTICAL_MOUSE), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DMI, USB_DEVICE_ID_DMI_ENC), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRACAL_RAPHNET, USB_DEVICE_ID_RAPHNET_2NES2SNES), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRACAL_RAPHNET, USB_DEVICE_ID_RAPHNET_4NES4SNES), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_DOLPHINBAR), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_GAMECUBE1), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_WIIU), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DWAV, USB_DEVICE_ID_EGALAX_TOUCHCONTROLLER), HID_QUIRK_MULTI_INPUT | HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELAN, HID_ANY_ID), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELO, USB_DEVICE_ID_ELO_TS2700), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_EMS, USB_DEVICE_ID_EMS_TRIO_LINKER_PLUS_II), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ETURBOTOUCH, USB_DEVICE_ID_ETURBOTOUCH_2968), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ETURBOTOUCH, USB_DEVICE_ID_ETURBOTOUCH), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_FORMOSA, USB_DEVICE_ID_FORMOSA_IR_RECEIVER), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_FREESCALE, USB_DEVICE_ID_FREESCALE_MX28), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_FUTABA, USB_DEVICE_ID_LED_DISPLAY), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GREENASIA, USB_DEVICE_ID_GREENASIA_DUAL_USB_JOYPAD), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HAPP, USB_DEVICE_ID_UGCI_DRIVING), HID_QUIRK_BADPAD | HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HAPP, USB_DEVICE_ID_UGCI_FIGHTING), HID_QUIRK_BADPAD | HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HAPP, USB_DEVICE_ID_UGCI_FLYING), HID_QUIRK_BADPAD | HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_KEYBOARD_A096), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0A4A), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0B4A), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_PIXART_OEM_USB_OPTICAL_MOUSE), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_PIXART_OEM_USB_OPTICAL_MOUSE_094A), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_IDEACOM, USB_DEVICE_ID_IDEACOM_IDC6680), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_INNOMEDIA, USB_DEVICE_ID_INNEX_GENESIS_ATARI), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_M912), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_PIXART_USB_OPTICAL_MOUSE_ID2), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_C007), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_C077), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_KEYBOARD_G710_PLUS), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C01A), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C05A), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C06A), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MCS, USB_DEVICE_ID_MCS_GAMEPADBLOCK), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MGE, USB_DEVICE_ID_MGE_UPS), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_SURFACE_PRO_2), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TOUCH_COVER_2), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_2), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MOJO, USB_DEVICE_ID_RETRO_ADAPTER), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MSI, USB_DEVICE_ID_MSI_GT683R_LED_PANEL), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MULTIPLE_1781, USB_DEVICE_ID_RAPHNET_4NES4SNES_OLD), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NATSU, USB_DEVICE_ID_NATSU_GAMEPAD), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NEC, USB_DEVICE_ID_NEC_USB_GAME_PAD), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NEXIO, USB_DEVICE_ID_NEXIO_MULTITOUCH_PTI0750), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NEXTWINDOW, USB_DEVICE_ID_NEXTWINDOW_TOUCHSCREEN), HID_QUIRK_MULTI_INPUT},
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NOVATEK, USB_DEVICE_ID_NOVATEK_MOUSE), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_DUOSENSE), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PANTHERLORD, USB_DEVICE_ID_PANTHERLORD_TWIN_USB_JOYSTICK), HID_QUIRK_MULTI_INPUT | HID_QUIRK_SKIP_OUTPUT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PENMOUNT, USB_DEVICE_ID_PENMOUNT_1610), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PENMOUNT, USB_DEVICE_ID_PENMOUNT_1640), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PI_ENGINEERING, USB_DEVICE_ID_PI_ENGINEERING_VEC_USB_FOOTPEDAL), HID_QUIRK_HIDINPUT_FORCE },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_OPTICAL_TOUCH_SCREEN1), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_OPTICAL_TOUCH_SCREEN2), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_OPTICAL_TOUCH_SCREEN), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_USB_OPTICAL_MOUSE), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_MOUSE_4D22), HID_QUIRK_ALWAYS_POLL },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PRODIGE, USB_DEVICE_ID_PRODIGE_CORDLESS), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_QUANTA, USB_DEVICE_ID_QUANTA_OPTICAL_TOUCH_3001), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_QUANTA, USB_DEVICE_ID_QUANTA_OPTICAL_TOUCH_3003), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_QUANTA, USB_DEVICE_ID_QUANTA_OPTICAL_TOUCH_3008), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_REALTEK, USB_DEVICE_ID_REALTEK_READER), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RUMBLEPAD), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD2), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SENNHEISER, USB_DEVICE_ID_SENNHEISER_BTD500USB), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SIGMA_MICRO, USB_DEVICE_ID_SIGMA_MICRO_KEYBOARD), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SIGMATEL, USB_DEVICE_ID_SIGMATEL_STMP3780), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS1030_TOUCH), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS817_TOUCH), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS9200_TOUCH), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS_TS), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SUN, USB_DEVICE_ID_RARITAN_KVM_DONGLE), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYMBOL, USB_DEVICE_ID_SYMBOL_SCANNER_1), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYMBOL, USB_DEVICE_ID_SYMBOL_SCANNER_2), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_HD), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_LTS1), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_LTS2), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_QUAD_HD), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_TP_V103), HID_QUIRK_NO_INIT_REPORTS },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TOPMAX, USB_DEVICE_ID_TOPMAX_COBRAPAD), HID_QUIRK_BADPAD },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TOUCHPACK, USB_DEVICE_ID_TOUCHPACK_RTS), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TPV, USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8882), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TPV, USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8883), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_TURBOX_KEYBOARD), HID_QUIRK_NOGET },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_KNA5), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWA60), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_10_6_INCH), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_14_1_INCH), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SIRIUS_BATTERY_FREE_TABLET), HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP_LTD2, USB_DEVICE_ID_SMARTJOY_DUAL_PLUS), HID_QUIRK_NOGET | HID_QUIRK_MULTI_INPUT },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_QUAD_USB_JOYPAD), HID_QUIRK_NOGET | HID_QUIRK_MULTI_INPUT },
+
+	{ 0 }
+};
+
+/*
+ * A list of devices for which there is a specialized driver on HID bus.
+ *
+ * Please note that for multitouch devices (driven by hid-multitouch driver),
+ * there is a proper autodetection and autoloading in place (based on presence
+ * of HID_DG_CONTACTID), so those devices don't need to be added to this list,
+ * as we are doing the right thing in hid_scan_usage().
+ *
+ * Autodetection for (USB) HID sensor hubs exists too. If a collection of type
+ * physical is found inside a usage page of type sensor, hid-sensor-hub will be
+ * used as a driver. See hid_scan_report().
+ */
+static const struct hid_device_id hid_have_special_driver[] = {
+#if IS_ENABLED(CONFIG_HID_A4TECH)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_WCP32PU) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_X5_005D) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_A4TECH, USB_DEVICE_ID_A4TECH_RP_649) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ACCUTOUCH)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELO, USB_DEVICE_ID_ELO_ACCUTOUCH_2216) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ACRUX)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ACRUX, 0x0802) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ACRUX, 0xf705) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ALPS)
+	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_ANY, USB_VENDOR_ID_ALPS_JP, HID_DEVICE_ID_ALPS_U1_DUAL) },
+#endif
+#if IS_ENABLED(CONFIG_HID_APPLE)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MIGHTYMOUSE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_MINI_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_MINI_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_MINI_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_JIS) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_ISO) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_REVB_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_REVB_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_REVB_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ANSI) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_ISO) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2009_JIS) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_ANSI) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_ISO) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_ALU_WIRELESS_2011_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) },
+#endif
+#if IS_ENABLED(CONFIG_HID_APPLEIR)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL4) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL5) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ASUS)
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_I2C_KEYBOARD) },
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_I2C_TOUCHPAD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_ROG_KEYBOARD3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_JESS, USB_DEVICE_ID_ASUS_MD_5112) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_ASUS_MD_5110) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_T100CHI_KEYBOARD) },
+#endif
+#if IS_ENABLED(CONFIG_HID_AUREAL)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AUREAL, USB_DEVICE_ID_AUREAL_W01RN) },
+#endif
+#if IS_ENABLED(CONFIG_HID_BELKIN)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BELKIN, USB_DEVICE_ID_FLIP_KVM) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LABTEC, USB_DEVICE_ID_LABTEC_WIRELESS_KEYBOARD) },
+#endif
+#if IS_ENABLED(CONFIG_HID_BETOP_FF)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185BFM, 0x2208) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185PC, 0x5506) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185V2PC, 0x1850) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BETOP_2185V2BFM, 0x5500) },
+#endif
+#if IS_ENABLED(CONFIG_HID_CHERRY)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHERRY, USB_DEVICE_ID_CHERRY_CYMOTION) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHERRY, USB_DEVICE_ID_CHERRY_CYMOTION_SOLAR) },
+#endif
+#if IS_ENABLED(CONFIG_HID_CHICONY)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_TACTICAL_PAD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_WIRELESS2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_ASUS_AK1D) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_ACER_SWITCH12) },
+#endif
+#if IS_ENABLED(CONFIG_HID_CMEDIA)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CMEDIA, USB_DEVICE_ID_CM6533) },
+#endif
+#if IS_ENABLED(CONFIG_HID_CORSAIR)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K90) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_SCIMITAR_PRO_RGB) },
+#endif
+#if IS_ENABLED(CONFIG_HID_CP2112)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_CP2112) },
+#endif
+#if IS_ENABLED(CONFIG_HID_CYPRESS)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_BARCODE_4) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_MOUSE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_DRAGONRISE)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, 0x0006) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, 0x0011) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ELECOM)
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_EX_G_WIRED) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_EX_G_WIRELESS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_DEFT_WIRED) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_DEFT_WIRELESS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_HUGE_WIRED) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_HUGE_WIRELESS) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ELO)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELO, 0x0009) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ELO, 0x0030) },
+#endif
+#if IS_ENABLED(CONFIG_HID_EMS_FF)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_EMS, USB_DEVICE_ID_EMS_TRIO_LINKER_PLUS_II) },
+#endif
+#if IS_ENABLED(CONFIG_HID_EZKEY)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_EZKEY, USB_DEVICE_ID_BTC_8193) },
+#endif
+#if IS_ENABLED(CONFIG_HID_GEMBIRD)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GEMBIRD, USB_DEVICE_ID_GEMBIRD_JPD_DUALFORCE2) },
+#endif
+#if IS_ENABLED(CONFIG_HID_GFRM)
+	{ HID_BLUETOOTH_DEVICE(0x58, 0x2000) },
+	{ HID_BLUETOOTH_DEVICE(0x471, 0x2210) },
+#endif
+#if IS_ENABLED(CONFIG_HID_GREENASIA)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GREENASIA, 0x0012) },
+#endif
+#if IS_ENABLED(CONFIG_HID_GT683R)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MSI, USB_DEVICE_ID_MSI_GT683R_LED_PANEL) },
+#endif
+#if IS_ENABLED(CONFIG_HID_GYRATION)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GYRATION, USB_DEVICE_ID_GYRATION_REMOTE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GYRATION, USB_DEVICE_ID_GYRATION_REMOTE_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GYRATION, USB_DEVICE_ID_GYRATION_REMOTE_3) },
+#endif
+#if IS_ENABLED(CONFIG_HID_HOLTEK)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK, USB_DEVICE_ID_HOLTEK_ON_LINE_GRIP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_KEYBOARD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A04A) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A067) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A070) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A072) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A081) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_MOUSE_A0C2) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ITE)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ITE, USB_DEVICE_ID_ITE8595) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ICADE)
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ION, USB_DEVICE_ID_ICADE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_JABRA)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_JABRA, HID_ANY_ID) },
+#endif
+#if IS_ENABLED(CONFIG_HID_KENSINGTON)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KENSINGTON, USB_DEVICE_ID_KS_SLIMBLADE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_KEYTOUCH)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KEYTOUCH, USB_DEVICE_ID_KEYTOUCH_IEC) },
+#endif
+#if IS_ENABLED(CONFIG_HID_KYE)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_GENIUS_GILA_GAMING_MOUSE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_GENIUS_MANTICORE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_GENIUS_GX_IMPERATOR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_ERGO_525V) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_I405X) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_M912) },
+#endif
+#if IS_ENABLED(CONFIG_HID_LCPOWER)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LCPOWER, USB_DEVICE_ID_LCPOWER_LC1000) },
+#endif
+#if IS_ENABLED(CONFIG_HID_LED)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DELCOM, USB_DEVICE_ID_DELCOM_VISUAL_IND) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_WN) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DREAM_CHEEKY, USB_DEVICE_ID_DREAM_CHEEKY_FA) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_LUXAFOR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_RISO_KAGAKU, USB_DEVICE_ID_RI_KA_WEBMAIL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THINGM, USB_DEVICE_ID_BLINK1) },
+#endif
+#if IS_ENABLED(CONFIG_HID_LENOVO)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_TPKBD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_CUSBKBD) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_CBTKBD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_TPPRODOCK) },
+#endif
+#if IS_ENABLED(CONFIG_HID_LOGITECH)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_MX3000_RECEIVER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_S510_RECEIVER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_S510_RECEIVER_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RECEIVER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_DINOVO_DESKTOP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_DINOVO_EDGE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_DINOVO_MINI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_ELITE_KBD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_CORDLESS_DESKTOP_LX500) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_EXTREME_3D) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DUAL_ACTION) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD_CORD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD2_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G29_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_F3D) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WINGMAN_FFG) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_FORCE3D_PRO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_FLIGHT_SYSTEM_G940) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOMO_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_VIBRATION_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DFP_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DFGT_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G25_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G27_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_WII_WHEEL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_RUMBLEPAD2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_SPACETRAVELLER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_SPACENAVIGATOR) },
+#endif
+#if IS_ENABLED(CONFIG_HID_LOGITECH_HIDPP)
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_T651) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_G920_WHEEL) },
+#endif
+#if IS_ENABLED(CONFIG_HID_LOGITECH_DJ)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_UNIFYING_RECEIVER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_UNIFYING_RECEIVER_2) },
+#endif
+#if IS_ENABLED(CONFIG_HID_MAGICMOUSE)
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICMOUSE) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD) },
+#endif
+#if IS_ENABLED(CONFIG_HID_MAYFLASH)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_DOLPHINBAR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_GAMECUBE1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_GAMECUBE2) },
+#endif
+#if IS_ENABLED(CONFIG_HID_MICROSOFT)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_COMFORT_MOUSE_4500) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_COMFORT_KEYBOARD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_SIDEWINDER_GV) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_NE4K) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_NE4K_JP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_NE7K) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_LK6K) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_PRESENTER_8K_USB) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_3K) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_WIRELESS_OPTICAL_DESKTOP_3_0) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_OFFICE_KB) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_7K) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_600) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_DIGITAL_MEDIA_3KV1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_PRESENTER_8K_BT) },
+#endif
+#if IS_ENABLED(CONFIG_HID_MONTEREY)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MONTEREY, USB_DEVICE_ID_GENIUS_KB29E) },
+#endif
+#if IS_ENABLED(CONFIG_HID_MULTITOUCH)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LG, USB_DEVICE_ID_LG_MELFAS_MT) },
+#endif
+#if IS_ENABLED(CONFIG_HID_WIIMOTE)
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_NINTENDO, USB_DEVICE_ID_NINTENDO_WIIMOTE) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_NINTENDO, USB_DEVICE_ID_NINTENDO_WIIMOTE2) },
+#endif
+#if IS_ENABLED(CONFIG_HID_NTI)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTI, USB_DEVICE_ID_USB_SUN) },
+#endif
+#if IS_ENABLED(CONFIG_HID_NTRIG)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_4) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_5) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_6) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_7) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_8) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_9) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_10) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_11) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_12) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_13) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_14) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_15) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_16) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_17) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_TOUCH_SCREEN_18) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ORTEK)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ORTEK, USB_DEVICE_ID_ORTEK_PKB1700) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ORTEK, USB_DEVICE_ID_ORTEK_WKB2000) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ORTEK, USB_DEVICE_ID_ORTEK_IHOME_IMAC_A210S) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SKYCABLE, USB_DEVICE_ID_SKYCABLE_WIRELESS_PRESENTER) },
+#endif
+#if IS_ENABLED(CONFIG_HID_PANTHERLORD)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GAMERON, USB_DEVICE_ID_GAMERON_DUAL_PSX_ADAPTOR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GAMERON, USB_DEVICE_ID_GAMERON_DUAL_PCS_ADAPTOR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GREENASIA, 0x0003) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_JESS2, USB_DEVICE_ID_JESS2_COLOR_RUMBLE_PAD) },
+#endif
+#if IS_ENABLED(CONFIG_HID_PENMOUNT)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PENMOUNT, USB_DEVICE_ID_PENMOUNT_6000) },
+#endif
+#if IS_ENABLED(CONFIG_HID_PETALYNX)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PETALYNX, USB_DEVICE_ID_PETALYNX_MAXTER_REMOTE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_PICOLCD)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICOLCD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICOLCD_BOOTLOADER) },
+#endif
+#if IS_ENABLED(CONFIG_HID_PLANTRONICS)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PLANTRONICS, HID_ANY_ID) },
+#endif
+#if IS_ENABLED(CONFIG_HID_PRIMAX)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_KEYBOARD) },
+#endif
+#if IS_ENABLED(CONFIG_HID_PRODIKEYS)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CREATIVELABS, USB_DEVICE_ID_PRODIKEYS_PCMIDI) },
+#endif
+#if IS_ENABLED(CONFIG_HID_RETRODE)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_FUTURE_TECHNOLOGY, USB_DEVICE_ID_RETRODE2) },
+#endif
+#if IS_ENABLED(CONFIG_HID_RMI)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X1_COVER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_RAZER, USB_DEVICE_ID_RAZER_BLADE_14) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_REZEL) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ROCCAT)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_ARVO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_ISKU) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_ISKUFX) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEPLUS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEPURE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEPURE_OPTICAL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KONEXTD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_KOVAPLUS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_LUA) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_PYRA_WIRED) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_PYRA_WIRELESS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_RYOS_MK) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_RYOS_MK_GLOW) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_RYOS_MK_PRO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ROCCAT, USB_DEVICE_ID_ROCCAT_SAVU) },
+#endif
+#if IS_ENABLED(CONFIG_HID_SAITEK)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_PS1000) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT7_OLD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT7) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RAT9) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_MMO7) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_RAT5) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_RAT9) },
+#endif
+#if IS_ENABLED(CONFIG_HID_SAMSUNG)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAMSUNG, USB_DEVICE_ID_SAMSUNG_IR_REMOTE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SAMSUNG, USB_DEVICE_ID_SAMSUNG_WIRELESS_KBD_MOUSE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_SMARTJOYPLUS)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PLAYDOTCOM, USB_DEVICE_ID_PLAYDOTCOM_EMS_USBII) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_SMARTJOY_PLUS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_SUPER_JOY_BOX_3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_DUAL_USB_JOYPAD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP_LTD, USB_DEVICE_ID_SUPER_JOY_BOX_3_PRO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP_LTD, USB_DEVICE_ID_SUPER_DUAL_BOX_PRO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WISEGROUP_LTD, USB_DEVICE_ID_SUPER_JOY_BOX_5_PRO) },
+#endif
+#if IS_ENABLED(CONFIG_HID_SONY)
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_HARMONY_PS3) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SMK, USB_DEVICE_ID_SMK_PS3_BDREMOTE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_BUZZ_CONTROLLER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_WIRELESS_BUZZ_CONTROLLER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_MOTION_CONTROLLER) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_MOTION_CONTROLLER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_NAVIGATION_CONTROLLER) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_NAVIGATION_CONTROLLER) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_BDREMOTE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_CONTROLLER) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS3_CONTROLLER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_PS4_CONTROLLER_DONGLE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_VAIO_VGX_MOUSE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SONY, USB_DEVICE_ID_SONY_VAIO_VGP_MOUSE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SINO_LITE, USB_DEVICE_ID_SINO_LITE_CONTROLLER) },
+#endif
+#if IS_ENABLED(CONFIG_HID_SPEEDLINK)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_X_TENSIONS, USB_DEVICE_ID_SPEEDLINK_VAD_CEZANNE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_STEELSERIES)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_STEELSERIES, USB_DEVICE_ID_STEELSERIES_SRWS1) },
+#endif
+#if IS_ENABLED(CONFIG_HID_SUNPLUS)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SUNPLUS, USB_DEVICE_ID_SUNPLUS_WDESKTOP) },
+#endif
+#if IS_ENABLED(CONFIG_HID_THRUSTMASTER)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb300) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb304) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb323) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb324) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb605) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb651) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb653) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb654) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THRUSTMASTER, 0xb65a) },
+#endif
+#if IS_ENABLED(CONFIG_HID_TIVO)
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_TIVO, USB_DEVICE_ID_TIVO_SLIDE_BT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TIVO, USB_DEVICE_ID_TIVO_SLIDE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TIVO, USB_DEVICE_ID_TIVO_SLIDE_PRO) },
+#endif
+#if IS_ENABLED(CONFIG_HID_TOPSEED)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BTC, USB_DEVICE_ID_BTC_EMPREX_REMOTE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BTC, USB_DEVICE_ID_BTC_EMPREX_REMOTE_2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_WIRELESS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TOPSEED, USB_DEVICE_ID_TOPSEED_CYBERLINK) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TOPSEED2, USB_DEVICE_ID_TOPSEED2_RF_COMBO) },
+#endif
+#if IS_ENABLED(CONFIG_HID_TWINHAN)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TWINHAN, USB_DEVICE_ID_TWINHAN_IR_REMOTE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_UCLOGIC)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_HUION, USB_DEVICE_ID_HUION_TABLET) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_HUION_TABLET) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_PF1209) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP4030U) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP5540U) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP8060U) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_WP1062) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_WIRELESS_TABLET_TWHL850) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWHA60) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_YIYNOVA_TABLET) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_81) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UGEE_TABLET_45) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_DRAWIMAGE_G3) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_TABLET_EX07S) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_UGTIZER, USB_DEVICE_ID_UGTIZER_TABLET_GP0610) },
+#endif
+#if IS_ENABLED(CONFIG_HID_UDRAW_PS3)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_THQ, USB_DEVICE_ID_THQ_PS3_UDRAW) },
+#endif
+#if IS_ENABLED(CONFIG_HID_WALTOP)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SLIM_TABLET_5_8_INCH) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SLIM_TABLET_12_1_INCH) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_Q_PAD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_PID_0038) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_10_6_INCH) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_14_1_INCH) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SIRIUS_BATTERY_FREE_TABLET) },
+#endif
+#if IS_ENABLED(CONFIG_HID_XINMO)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_XIN_MO, USB_DEVICE_ID_XIN_MO_DUAL_ARCADE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_XIN_MO, USB_DEVICE_ID_THT_2P_ARCADE) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ZEROPLUS)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ZEROPLUS, 0x0005) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ZEROPLUS, 0x0030) },
+#endif
+#if IS_ENABLED(CONFIG_HID_ZYDACRON)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ZYDACRON, USB_DEVICE_ID_ZYDACRON_REMOTE_CONTROL) },
+#endif
+	{ }
+};
+
+/* a list of devices that shouldn't be handled by HID core at all */
+static const struct hid_device_id hid_ignore_list[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ACECAD, USB_DEVICE_ID_ACECAD_FLAIR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ACECAD, USB_DEVICE_ID_ACECAD_302) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ADS_TECH, USB_DEVICE_ID_ADS_TECH_RADIO_SI470X) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_01) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_10) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_20) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_21) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_22) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_23) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIPTEK, USB_DEVICE_ID_AIPTEK_24) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AIRCABLE, USB_DEVICE_ID_AIRCABLE1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ALCOR, USB_DEVICE_ID_ALCOR_USBRS232) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_LCM)},
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ASUSTEK, USB_DEVICE_ID_ASUSTEK_LCM2)},
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AVERMEDIA, USB_DEVICE_ID_AVER_FM_MR800) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AXENTIA, USB_DEVICE_ID_AXENTIA_FM_RADIO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_BERKSHIRE, USB_DEVICE_ID_BERKSHIRE_PCWD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CIDC, 0x0103) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI470X) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYGNAL, USB_DEVICE_ID_CYGNAL_RADIO_SI4713) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CMEDIA, USB_DEVICE_ID_CM109) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_HIDCOM) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CYPRESS, USB_DEVICE_ID_CYPRESS_ULTRAMOUSE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DEALEXTREAME, USB_DEVICE_ID_DEALEXTREAME_RADIO_SI4701) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DELORME, USB_DEVICE_ID_DELORME_EARTHMATE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_DELORME, USB_DEVICE_ID_DELORME_EM_LT20) },
+	{ HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, 0x0400) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ESSENTIAL_REALITY, USB_DEVICE_ID_ESSENTIAL_REALITY_P5) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ETT, USB_DEVICE_ID_TC5UH) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ETT, USB_DEVICE_ID_TC4UM) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GENERAL_TOUCH, 0x0001) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GENERAL_TOUCH, 0x0002) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GENERAL_TOUCH, 0x0004) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GOTOP, USB_DEVICE_ID_SUPER_Q2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GOTOP, USB_DEVICE_ID_GOGOPEN) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GOTOP, USB_DEVICE_ID_PENPOWER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GRETAGMACBETH, USB_DEVICE_ID_GRETAGMACBETH_HUEY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GRIFFIN, USB_DEVICE_ID_POWERMATE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GRIFFIN, USB_DEVICE_ID_SOUNDKNOB) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GRIFFIN, USB_DEVICE_ID_RADIOSHARK) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_90) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_100) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_101) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_103) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_104) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_105) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_106) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_107) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_108) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_200) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_201) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_202) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_203) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_204) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_205) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_206) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_207) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_300) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_301) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_302) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_303) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_304) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_305) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_306) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_307) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_308) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_309) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_400) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_401) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_402) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_403) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_404) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_405) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_500) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_501) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_502) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_503) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_504) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1000) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1001) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1002) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1003) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1004) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1005) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1006) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_GTCO, USB_DEVICE_ID_GTCO_1007) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_IMATION, USB_DEVICE_ID_DISC_STAKKA) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_JABRA, USB_DEVICE_ID_JABRA_GN9350E) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KBGEAR, USB_DEVICE_ID_KBGEAR_JAMSTUDIO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KWORLD, USB_DEVICE_ID_KWORLD_RADIO_FM700) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_GPEN_560) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_KYE, 0x0058) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_CASSY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_CASSY2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_POCKETCASSY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_POCKETCASSY2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOBILECASSY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOBILECASSY2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYVOLTAGE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYCURRENT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYTIME) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYTEMPERATURE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MICROCASSYPH) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_POWERANALYSERCASSY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_CONVERTERCONTROLLERCASSY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MACHINETESTCASSY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_JWM) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_DMMP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_UMIP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_UMIC) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_UMIB) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_XRAY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_XRAY2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_VIDEOCOM) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOTOR) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_COM3LAB) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_TELEPORT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_NETWORKANALYSER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_POWERCONTROL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MACHINETEST) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOSTANALYSER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MOSTANALYSER2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_ABSESP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_AUTODATABUS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_MCT) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_HYBRID) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_LD, USB_DEVICE_ID_LD_HEATCONTROL) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MADCATZ, USB_DEVICE_ID_MADCATZ_BEATPAD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MCC, USB_DEVICE_ID_MCC_PMD1024LS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MCC, USB_DEVICE_ID_MCC_PMD1208LS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICKIT1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICKIT2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICK16F1454) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_MICROCHIP, USB_DEVICE_ID_PICK16F1454_V2) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_NATIONAL_SEMICONDUCTOR, USB_DEVICE_ID_N_S_HARMONY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 20) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 30) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 100) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 108) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 118) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 200) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 300) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 400) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_ONTRAK, USB_DEVICE_ID_ONTRAK_ADU100 + 500) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0001) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0002) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0003) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PANJIT, 0x0004) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PETZL, USB_DEVICE_ID_PETZL_HEADLAMP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PHILIPS, USB_DEVICE_ID_PHILIPS_IEEE802154_DONGLE) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_POWERCOM, USB_DEVICE_ID_POWERCOM_UPS) },
+#if IS_ENABLED(CONFIG_MOUSE_SYNAPTICS_USB)
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_TP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_INT_TP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_CPAD) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_STICK) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_WP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_COMP_TP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_WTP) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_DPAD) },
+#endif
+	{ HID_USB_DEVICE(USB_VENDOR_ID_YEALINK, USB_DEVICE_ID_YEALINK_P1K_P4K_B2K) },
+	{ }
+};
+
+/**
+ * hid_mouse_ignore_list - mouse devices which should not be handled by the hid layer
+ *
+ * There are composite devices for which we want to ignore only a certain
+ * interface. This is a list of devices for which only the mouse interface will
+ * be ignored. This allows a dedicated driver to take care of the interface.
+ */
+static const struct hid_device_id hid_mouse_ignore_list[] = {
+	/* appletouch driver */
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER3_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER4_HF_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING2_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING3_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING4A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING5A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING6A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING7A_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING8_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ANSI) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_ISO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_WELLSPRING9_JIS) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_FOUNTAIN_TP_ONLY) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) },
+	{ }
+};
+
+bool hid_ignore(struct hid_device *hdev)
+{
+	if (hdev->quirks & HID_QUIRK_NO_IGNORE)
+		return false;
+	if (hdev->quirks & HID_QUIRK_IGNORE)
+		return true;
+
+	switch (hdev->vendor) {
+	case USB_VENDOR_ID_CODEMERCS:
+		/* ignore all Code Mercenaries IOWarrior devices */
+		if (hdev->product >= USB_DEVICE_ID_CODEMERCS_IOW_FIRST &&
+		    hdev->product <= USB_DEVICE_ID_CODEMERCS_IOW_LAST)
+			return true;
+		break;
+	case USB_VENDOR_ID_LOGITECH:
+		if (hdev->product >= USB_DEVICE_ID_LOGITECH_HARMONY_FIRST &&
+		    hdev->product <= USB_DEVICE_ID_LOGITECH_HARMONY_LAST)
+			return true;
+		/*
+		 * The Keene FM transmitter USB device has the same USB ID as
+		 * the Logitech AudioHub Speaker, but it should ignore the hid.
+		 * Check if the name is that of the Keene device.
+		 * For reference: the name of the AudioHub is
+		 * "HOLTEK  AudioHub Speaker".
+		 */
+		if (hdev->product == USB_DEVICE_ID_LOGITECH_AUDIOHUB &&
+		    !strcmp(hdev->name, "HOLTEK  B-LINK USB Audio  "))
+			return true;
+		break;
+	case USB_VENDOR_ID_SOUNDGRAPH:
+		if (hdev->product >= USB_DEVICE_ID_SOUNDGRAPH_IMON_FIRST &&
+		    hdev->product <= USB_DEVICE_ID_SOUNDGRAPH_IMON_LAST)
+			return true;
+		break;
+	case USB_VENDOR_ID_HANWANG:
+		if (hdev->product >= USB_DEVICE_ID_HANWANG_TABLET_FIRST &&
+		    hdev->product <= USB_DEVICE_ID_HANWANG_TABLET_LAST)
+			return true;
+		break;
+	case USB_VENDOR_ID_JESS:
+		if (hdev->product == USB_DEVICE_ID_JESS_YUREX &&
+		    hdev->type == HID_TYPE_USBNONE)
+			return true;
+		break;
+	case USB_VENDOR_ID_VELLEMAN:
+		/* These are not HID devices.  They are handled by comedi. */
+		if ((hdev->product >= USB_DEVICE_ID_VELLEMAN_K8055_FIRST &&
+		     hdev->product <= USB_DEVICE_ID_VELLEMAN_K8055_LAST) ||
+		    (hdev->product >= USB_DEVICE_ID_VELLEMAN_K8061_FIRST &&
+		     hdev->product <= USB_DEVICE_ID_VELLEMAN_K8061_LAST))
+			return true;
+		break;
+	case USB_VENDOR_ID_ATMEL_V_USB:
+		/* Masterkit MA901 usb radio based on Atmel tiny85 chip and
+		 * it has the same USB ID as many Atmel V-USB devices. This
+		 * usb radio is handled by radio-ma901.c driver so we want
+		 * ignore the hid. Check the name, bus, product and ignore
+		 * if we have MA901 usb radio.
+		 */
+		if (hdev->product == USB_DEVICE_ID_ATMEL_V_USB &&
+		    hdev->bus == BUS_USB &&
+		    strncmp(hdev->name, "www.masterkit.ru MA901", 22) == 0)
+			return true;
+		break;
+	case USB_VENDOR_ID_ELAN:
+		/*
+		 * Many Elan devices have a product id of 0x0401 and are handled
+		 * by the elan_i2c input driver. But the ACPI HID ELAN0800 dev
+		 * is not (and cannot be) handled by that driver ->
+		 * Ignore all 0x0401 devs except for the ELAN0800 dev.
+		 */
+		if (hdev->product == 0x0401 &&
+		    strncmp(hdev->name, "ELAN0800", 8) != 0)
+			return true;
+		break;
+	}
+
+	if (hdev->type == HID_TYPE_USBMOUSE &&
+	    hid_match_id(hdev, hid_mouse_ignore_list))
+		return true;
+
+	return !!hid_match_id(hdev, hid_ignore_list);
+}
+EXPORT_SYMBOL_GPL(hid_ignore);
+
+/* Dynamic HID quirks list - specified at runtime */
+struct quirks_list_struct {
+	struct hid_device_id hid_bl_item;
+	struct list_head node;
+};
+
+static LIST_HEAD(dquirks_list);
+static DEFINE_MUTEX(dquirks_lock);
+
+/* Runtime ("dynamic") quirks manipulation functions */
+
+/**
+ * hid_exists_dquirk: find any dynamic quirks for a HID device
+ * @hdev: the HID device to match
+ *
+ * Description:
+ *         Scans dquirks_list for a matching dynamic quirk and returns
+ *         the pointer to the relevant struct hid_device_id if found.
+ *         Must be called with a read lock held on dquirks_lock.
+ *
+ * Returns: NULL if no quirk found, struct hid_device_id * if found.
+ */
+static struct hid_device_id *hid_exists_dquirk(const struct hid_device *hdev)
+{
+	struct quirks_list_struct *q;
+	struct hid_device_id *bl_entry = NULL;
+
+	list_for_each_entry(q, &dquirks_list, node) {
+		if (hid_match_one_id(hdev, &q->hid_bl_item)) {
+			bl_entry = &q->hid_bl_item;
+			break;
+		}
+	}
+
+	if (bl_entry != NULL)
+		dbg_hid("Found dynamic quirk 0x%lx for HID device 0x%hx:0x%hx\n",
+			bl_entry->driver_data, bl_entry->vendor,
+			bl_entry->product);
+
+	return bl_entry;
+}
+
+
+/**
+ * hid_modify_dquirk: add/replace a HID quirk
+ * @id: the HID device to match
+ * @quirks: the unsigned long quirks value to add/replace
+ *
+ * Description:
+ *         If an dynamic quirk exists in memory for this device, replace its
+ *         quirks value with what was provided.  Otherwise, add the quirk
+ *         to the dynamic quirks list.
+ *
+ * Returns: 0 OK, -error on failure.
+ */
+static int hid_modify_dquirk(const struct hid_device_id *id,
+			     const unsigned long quirks)
+{
+	struct hid_device *hdev;
+	struct quirks_list_struct *q_new, *q;
+	int list_edited = 0;
+	int ret = 0;
+
+	hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
+	if (!hdev)
+		return -ENOMEM;
+
+	q_new = kmalloc(sizeof(struct quirks_list_struct), GFP_KERNEL);
+	if (!q_new) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	hdev->bus = q_new->hid_bl_item.bus = id->bus;
+	hdev->group = q_new->hid_bl_item.group = id->group;
+	hdev->vendor = q_new->hid_bl_item.vendor = id->vendor;
+	hdev->product = q_new->hid_bl_item.product = id->product;
+	q_new->hid_bl_item.driver_data = quirks;
+
+	mutex_lock(&dquirks_lock);
+
+	list_for_each_entry(q, &dquirks_list, node) {
+
+		if (hid_match_one_id(hdev, &q->hid_bl_item)) {
+
+			list_replace(&q->node, &q_new->node);
+			kfree(q);
+			list_edited = 1;
+			break;
+
+		}
+
+	}
+
+	if (!list_edited)
+		list_add_tail(&q_new->node, &dquirks_list);
+
+	mutex_unlock(&dquirks_lock);
+
+ out:
+	kfree(hdev);
+	return ret;
+}
+
+/**
+ * hid_remove_all_dquirks: remove all runtime HID quirks from memory
+ * @bus: bus to match against. Use HID_BUS_ANY if all need to be removed.
+ *
+ * Description:
+ *         Free all memory associated with dynamic quirks - called before
+ *         module unload.
+ *
+ */
+static void hid_remove_all_dquirks(__u16 bus)
+{
+	struct quirks_list_struct *q, *temp;
+
+	mutex_lock(&dquirks_lock);
+	list_for_each_entry_safe(q, temp, &dquirks_list, node) {
+		if (bus == HID_BUS_ANY || bus == q->hid_bl_item.bus) {
+			list_del(&q->node);
+			kfree(q);
+		}
+	}
+	mutex_unlock(&dquirks_lock);
+
+}
+
+/**
+ * hid_quirks_init: apply HID quirks specified at module load time
+ */
+int hid_quirks_init(char **quirks_param, __u16 bus, int count)
+{
+	struct hid_device_id id = { 0 };
+	int n = 0, m;
+	unsigned short int vendor, product;
+	u32 quirks;
+
+	id.bus = bus;
+
+	for (; n < count && quirks_param[n]; n++) {
+
+		m = sscanf(quirks_param[n], "0x%hx:0x%hx:0x%x",
+				&vendor, &product, &quirks);
+
+		id.vendor = (__u16)vendor;
+		id.product = (__u16)product;
+
+		if (m != 3 ||
+		    hid_modify_dquirk(&id, quirks) != 0) {
+			pr_warn("Could not parse HID quirk module param %s\n",
+				quirks_param[n]);
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(hid_quirks_init);
+
+/**
+ * hid_quirks_exit: release memory associated with dynamic_quirks
+ * @bus: a bus to match against
+ *
+ * Description:
+ *     Release all memory associated with dynamic quirks for a given bus.
+ *     Called upon module unload.
+ *     Use HID_BUS_ANY to remove all dynamic quirks.
+ *
+ * Returns: nothing
+ */
+void hid_quirks_exit(__u16 bus)
+{
+	hid_remove_all_dquirks(bus);
+}
+EXPORT_SYMBOL_GPL(hid_quirks_exit);
+
+/**
+ * hid_gets_squirk: return any static quirks for a HID device
+ * @hdev: the HID device to match
+ *
+ * Description:
+ *     Given a HID device, return a pointer to the quirked hid_device_id entry
+ *     associated with that device.
+ *
+ * Returns: the quirks.
+ */
+static unsigned long hid_gets_squirk(const struct hid_device *hdev)
+{
+	const struct hid_device_id *bl_entry;
+	unsigned long quirks = 0;
+
+	if (hid_match_id(hdev, hid_ignore_list))
+		quirks |= HID_QUIRK_IGNORE;
+
+	if (hid_match_id(hdev, hid_have_special_driver))
+		quirks |= HID_QUIRK_HAVE_SPECIAL_DRIVER;
+
+	bl_entry = hid_match_id(hdev, hid_quirks);
+	if (bl_entry != NULL)
+		quirks |= bl_entry->driver_data;
+
+	if (quirks)
+		dbg_hid("Found squirk 0x%lx for HID device 0x%hx:0x%hx\n",
+			quirks, hdev->vendor, hdev->product);
+	return quirks;
+}
+
+/**
+ * hid_lookup_quirk: return any quirks associated with a HID device
+ * @hdev: the HID device to look for
+ *
+ * Description:
+ *     Given a HID device, return any quirks associated with that device.
+ *
+ * Returns: an unsigned long quirks value.
+ */
+unsigned long hid_lookup_quirk(const struct hid_device *hdev)
+{
+	unsigned long quirks = 0;
+	const struct hid_device_id *quirk_entry = NULL;
+
+	/* NCR devices must not be queried for reports */
+	if (hdev->bus == BUS_USB &&
+	    hdev->vendor == USB_VENDOR_ID_NCR &&
+	    hdev->product >= USB_DEVICE_ID_NCR_FIRST &&
+	    hdev->product <= USB_DEVICE_ID_NCR_LAST)
+		return HID_QUIRK_NO_INIT_REPORTS;
+
+	/* These devices must be ignored if version (bcdDevice) is too old */
+	if (hdev->bus == BUS_USB && hdev->vendor == USB_VENDOR_ID_JABRA) {
+		switch (hdev->product) {
+		case USB_DEVICE_ID_JABRA_SPEAK_410:
+			if (hdev->version < 0x0111)
+				return HID_QUIRK_IGNORE;
+			break;
+		case USB_DEVICE_ID_JABRA_SPEAK_510:
+			if (hdev->version < 0x0214)
+				return HID_QUIRK_IGNORE;
+			break;
+		}
+	}
+
+	mutex_lock(&dquirks_lock);
+	quirk_entry = hid_exists_dquirk(hdev);
+	if (quirk_entry)
+		quirks = quirk_entry->driver_data;
+	else
+		quirks = hid_gets_squirk(hdev);
+	mutex_unlock(&dquirks_lock);
+
+	return quirks;
+}
+EXPORT_SYMBOL_GPL(hid_lookup_quirk);
diff --git a/drivers/hid/hid-rmi.c b/drivers/hid/hid-rmi.c
index 0f43c4292685..c6c05df3e8d2 100644
--- a/drivers/hid/hid-rmi.c
+++ b/drivers/hid/hid-rmi.c
@@ -731,6 +731,7 @@ static const struct hid_device_id rmi_id[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_RAZER, USB_DEVICE_ID_RAZER_BLADE_14),
 		.driver_data = RMI_DEVICE_HAS_PHYS_BUTTONS },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_LENOVO, USB_DEVICE_ID_LENOVO_X1_COVER) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_REZEL) },
 	{ HID_DEVICE(HID_BUS_ANY, HID_GROUP_RMI, HID_ANY_ID, HID_ANY_ID) },
 	{ }
 };
diff --git a/drivers/hid/hid-roccat-kovaplus.c b/drivers/hid/hid-roccat-kovaplus.c
index 43617fb28b87..317c9c2c0a7c 100644
--- a/drivers/hid/hid-roccat-kovaplus.c
+++ b/drivers/hid/hid-roccat-kovaplus.c
@@ -37,6 +37,8 @@ static uint kovaplus_convert_event_cpi(uint value)
 static void kovaplus_profile_activated(struct kovaplus_device *kovaplus,
 		uint new_profile_index)
 {
+	if (new_profile_index >= ARRAY_SIZE(kovaplus->profile_settings))
+		return;
 	kovaplus->actual_profile = new_profile_index;
 	kovaplus->actual_cpi = kovaplus->profile_settings[new_profile_index].cpi_startup_level;
 	kovaplus->actual_x_sensitivity = kovaplus->profile_settings[new_profile_index].sensitivity_x;
diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c
index fb77dec720a4..5be8de70c651 100644
--- a/drivers/hid/hid-roccat.c
+++ b/drivers/hid/hid-roccat.c
@@ -137,14 +137,14 @@ static ssize_t roccat_read(struct file *file, char __user *buffer,
 	return retval;
 }
 
-static unsigned int roccat_poll(struct file *file, poll_table *wait)
+static __poll_t roccat_poll(struct file *file, poll_table *wait)
 {
 	struct roccat_reader *reader = file->private_data;
 	poll_wait(file, &reader->device->wait, wait);
 	if (reader->cbuf_start != reader->device->cbuf_end)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	if (!reader->device->exist)
-		return POLLERR | POLLHUP;
+		return EPOLLERR | EPOLLHUP;
 	return 0;
 }
 
diff --git a/drivers/hid/hid-sensor-custom.c b/drivers/hid/hid-sensor-custom.c
index 0bcf041368c7..e8a114157f87 100644
--- a/drivers/hid/hid-sensor-custom.c
+++ b/drivers/hid/hid-sensor-custom.c
@@ -702,11 +702,11 @@ static int hid_sensor_custom_open(struct inode *inode, struct file *file)
 	return nonseekable_open(inode, file);
 }
 
-static unsigned int hid_sensor_custom_poll(struct file *file,
+static __poll_t hid_sensor_custom_poll(struct file *file,
 					   struct poll_table_struct *wait)
 {
 	struct hid_sensor_custom *sensor_inst;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	sensor_inst = container_of(file->private_data,
 				   struct hid_sensor_custom, custom_dev);
@@ -714,7 +714,7 @@ static unsigned int hid_sensor_custom_poll(struct file *file,
 	poll_wait(file, &sensor_inst->wait, wait);
 
 	if (!kfifo_is_empty(&sensor_inst->data_fifo))
-		mask = POLLIN | POLLRDNORM;
+		mask = EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index b9dc3ac4d4aa..ccdc5f2d01b1 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -473,6 +473,7 @@ struct motion_output_report_02 {
 #define DS4_FEATURE_REPORT_0x02_SIZE 37
 #define DS4_FEATURE_REPORT_0x05_SIZE 41
 #define DS4_FEATURE_REPORT_0x81_SIZE 7
+#define DS4_FEATURE_REPORT_0xA3_SIZE 49
 #define DS4_INPUT_REPORT_0x11_SIZE 78
 #define DS4_OUTPUT_REPORT_0x05_SIZE 32
 #define DS4_OUTPUT_REPORT_0x11_SIZE 78
@@ -544,6 +545,8 @@ struct sony_sc {
 	struct power_supply *battery;
 	struct power_supply_desc battery_desc;
 	int device_id;
+	unsigned fw_version;
+	unsigned hw_version;
 	u8 *output_report_dmabuf;
 
 #ifdef CONFIG_SONY_FF
@@ -627,6 +630,29 @@ static ssize_t ds4_store_poll_interval(struct device *dev,
 static DEVICE_ATTR(bt_poll_interval, 0644, ds4_show_poll_interval,
 		ds4_store_poll_interval);
 
+static ssize_t sony_show_firmware_version(struct device *dev,
+				struct device_attribute
+				*attr, char *buf)
+{
+	struct hid_device *hdev = to_hid_device(dev);
+	struct sony_sc *sc = hid_get_drvdata(hdev);
+
+	return snprintf(buf, PAGE_SIZE, "0x%04x\n", sc->fw_version);
+}
+
+static DEVICE_ATTR(firmware_version, 0444, sony_show_firmware_version, NULL);
+
+static ssize_t sony_show_hardware_version(struct device *dev,
+				struct device_attribute
+				*attr, char *buf)
+{
+	struct hid_device *hdev = to_hid_device(dev);
+	struct sony_sc *sc = hid_get_drvdata(hdev);
+
+	return snprintf(buf, PAGE_SIZE, "0x%04x\n", sc->hw_version);
+}
+
+static DEVICE_ATTR(hardware_version, 0444, sony_show_hardware_version, NULL);
 
 static u8 *motion_fixup(struct hid_device *hdev, u8 *rdesc,
 			     unsigned int *rsize)
@@ -1646,6 +1672,31 @@ static void dualshock4_calibration_work(struct work_struct *work)
 	spin_unlock_irqrestore(&sc->lock, flags);
 }
 
+static int dualshock4_get_version_info(struct sony_sc *sc)
+{
+	u8 *buf;
+	int ret;
+
+	buf = kmalloc(DS4_FEATURE_REPORT_0xA3_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	ret = hid_hw_raw_request(sc->hdev, 0xA3, buf,
+				 DS4_FEATURE_REPORT_0xA3_SIZE,
+				 HID_FEATURE_REPORT,
+				 HID_REQ_GET_REPORT);
+	if (ret < 0) {
+		kfree(buf);
+		return ret;
+	}
+
+	sc->hw_version = get_unaligned_le16(&buf[35]);
+	sc->fw_version = get_unaligned_le16(&buf[41]);
+
+	kfree(buf);
+	return 0;
+}
+
 static void sixaxis_set_leds_from_id(struct sony_sc *sc)
 {
 	static const u8 sixaxis_leds[10][4] = {
@@ -2399,10 +2450,7 @@ static int sony_check_add(struct sony_sc *sc)
 		memcpy(sc->mac_address, &buf[1], sizeof(sc->mac_address));
 
 		snprintf(sc->hdev->uniq, sizeof(sc->hdev->uniq),
-			"%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
-			sc->mac_address[5], sc->mac_address[4],
-			sc->mac_address[3], sc->mac_address[2],
-			sc->mac_address[1], sc->mac_address[0]);
+			 "%pMR", sc->mac_address);
 	} else if ((sc->quirks & SIXAXIS_CONTROLLER_USB) ||
 			(sc->quirks & NAVIGATION_CONTROLLER_USB)) {
 		buf = kmalloc(SIXAXIS_REPORT_0xF2_SIZE, GFP_KERNEL);
@@ -2432,10 +2480,7 @@ static int sony_check_add(struct sony_sc *sc)
 			sc->mac_address[5-n] = buf[4+n];
 
 		snprintf(sc->hdev->uniq, sizeof(sc->hdev->uniq),
-			"%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
-			sc->mac_address[5], sc->mac_address[4],
-			sc->mac_address[3], sc->mac_address[2],
-			sc->mac_address[1], sc->mac_address[0]);
+			 "%pMR", sc->mac_address);
 	} else {
 		return 0;
 	}
@@ -2619,6 +2664,28 @@ static int sony_input_configured(struct hid_device *hdev,
 			goto err_stop;
 		}
 
+		ret = dualshock4_get_version_info(sc);
+		if (ret < 0) {
+			hid_err(sc->hdev, "Failed to get version data from Dualshock 4\n");
+			goto err_stop;
+		}
+
+		ret = device_create_file(&sc->hdev->dev, &dev_attr_firmware_version);
+		if (ret) {
+			/* Make zero for cleanup reasons of sysfs entries. */
+			sc->fw_version = 0;
+			sc->hw_version = 0;
+			hid_err(sc->hdev, "can't create sysfs firmware_version attribute err: %d\n", ret);
+			goto err_stop;
+		}
+
+		ret = device_create_file(&sc->hdev->dev, &dev_attr_hardware_version);
+		if (ret) {
+			sc->hw_version = 0;
+			hid_err(sc->hdev, "can't create sysfs hardware_version attribute err: %d\n", ret);
+			goto err_stop;
+		}
+
 		/*
 		 * The Dualshock 4 touchpad supports 2 touches and has a
 		 * resolution of 1920x942 (44.86 dots/mm).
@@ -2695,6 +2762,10 @@ static int sony_input_configured(struct hid_device *hdev,
 	 */
 	if (sc->ds4_bt_poll_interval)
 		device_remove_file(&sc->hdev->dev, &dev_attr_bt_poll_interval);
+	if (sc->fw_version)
+		device_remove_file(&sc->hdev->dev, &dev_attr_firmware_version);
+	if (sc->hw_version)
+		device_remove_file(&sc->hdev->dev, &dev_attr_hardware_version);
 	if (sc->quirks & SONY_LED_SUPPORT)
 		sony_leds_remove(sc);
 	if (sc->quirks & SONY_BATTERY_SUPPORT)
@@ -2796,6 +2867,12 @@ static void sony_remove(struct hid_device *hdev)
 	if (sc->quirks & DUALSHOCK4_CONTROLLER_BT)
 		device_remove_file(&sc->hdev->dev, &dev_attr_bt_poll_interval);
 
+	if (sc->fw_version)
+		device_remove_file(&sc->hdev->dev, &dev_attr_firmware_version);
+
+	if (sc->hw_version)
+		device_remove_file(&sc->hdev->dev, &dev_attr_hardware_version);
+
 	sony_cancel_work_sync(sc);
 
 	kfree(sc->output_report_dmabuf);
diff --git a/drivers/hid/hidraw.c b/drivers/hid/hidraw.c
index 5fbe0f81ab2e..fbfcc8009432 100644
--- a/drivers/hid/hidraw.c
+++ b/drivers/hid/hidraw.c
@@ -249,15 +249,15 @@ static ssize_t hidraw_get_report(struct file *file, char __user *buffer, size_t
 	return ret;
 }
 
-static unsigned int hidraw_poll(struct file *file, poll_table *wait)
+static __poll_t hidraw_poll(struct file *file, poll_table *wait)
 {
 	struct hidraw_list *list = file->private_data;
 
 	poll_wait(file, &list->hidraw->wait, wait);
 	if (list->head != list->tail)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	if (!list->hidraw->exist)
-		return POLLERR | POLLHUP;
+		return EPOLLERR | EPOLLHUP;
 	return 0;
 }
 
diff --git a/drivers/hid/i2c-hid/i2c-hid.c b/drivers/hid/i2c-hid/i2c-hid.c
index e054ee43c1e2..7230243b94d3 100644
--- a/drivers/hid/i2c-hid/i2c-hid.c
+++ b/drivers/hid/i2c-hid/i2c-hid.c
@@ -934,11 +934,6 @@ static int i2c_hid_of_probe(struct i2c_client *client,
 	}
 	pdata->hid_descriptor_address = val;
 
-	ret = of_property_read_u32(dev->of_node, "post-power-on-delay-ms",
-				   &val);
-	if (!ret)
-		pdata->post_power_delay_ms = val;
-
 	return 0;
 }
 
@@ -955,6 +950,16 @@ static inline int i2c_hid_of_probe(struct i2c_client *client,
 }
 #endif
 
+static void i2c_hid_fwnode_probe(struct i2c_client *client,
+				 struct i2c_hid_platform_data *pdata)
+{
+	u32 val;
+
+	if (!device_property_read_u32(&client->dev, "post-power-on-delay-ms",
+				      &val))
+		pdata->post_power_delay_ms = val;
+}
+
 static int i2c_hid_probe(struct i2c_client *client,
 			 const struct i2c_device_id *dev_id)
 {
@@ -998,6 +1003,9 @@ static int i2c_hid_probe(struct i2c_client *client,
 		ihid->pdata = *platform_data;
 	}
 
+	/* Parse platform agnostic common properties from ACPI / device tree */
+	i2c_hid_fwnode_probe(client, &ihid->pdata);
+
 	ihid->pdata.supply = devm_regulator_get(&client->dev, "vdd");
 	if (IS_ERR(ihid->pdata.supply)) {
 		ret = PTR_ERR(ihid->pdata.supply);
diff --git a/drivers/hid/intel-ish-hid/ipc/hw-ish.h b/drivers/hid/intel-ish-hid/ipc/hw-ish.h
index 2aac097c3f70..97869b7410eb 100644
--- a/drivers/hid/intel-ish-hid/ipc/hw-ish.h
+++ b/drivers/hid/intel-ish-hid/ipc/hw-ish.h
@@ -28,6 +28,7 @@
 #define SPT_Ax_DEVICE_ID	0x9D35
 #define CNL_Ax_DEVICE_ID	0x9DFC
 #define GLK_Ax_DEVICE_ID	0x31A2
+#define CNL_H_DEVICE_ID		0xA37C
 
 #define	REVISION_ID_CHT_A0	0x6
 #define	REVISION_ID_CHT_Ax_SI	0x0
diff --git a/drivers/hid/intel-ish-hid/ipc/pci-ish.c b/drivers/hid/intel-ish-hid/ipc/pci-ish.c
index 20d824f74f99..582e449be9fe 100644
--- a/drivers/hid/intel-ish-hid/ipc/pci-ish.c
+++ b/drivers/hid/intel-ish-hid/ipc/pci-ish.c
@@ -37,6 +37,7 @@ static const struct pci_device_id ish_pci_tbl[] = {
 	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, SPT_Ax_DEVICE_ID)},
 	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, CNL_Ax_DEVICE_ID)},
 	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, GLK_Ax_DEVICE_ID)},
+	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, CNL_H_DEVICE_ID)},
 	{0, }
 };
 MODULE_DEVICE_TABLE(pci, ish_pci_tbl);
diff --git a/drivers/hid/uhid.c b/drivers/hid/uhid.c
index 6f819f144cb4..4e0e7baf8513 100644
--- a/drivers/hid/uhid.c
+++ b/drivers/hid/uhid.c
@@ -753,14 +753,14 @@ static ssize_t uhid_char_write(struct file *file, const char __user *buffer,
 	return ret ? ret : count;
 }
 
-static unsigned int uhid_char_poll(struct file *file, poll_table *wait)
+static __poll_t uhid_char_poll(struct file *file, poll_table *wait)
 {
 	struct uhid_device *uhid = file->private_data;
 
 	poll_wait(file, &uhid->waitq, wait);
 
 	if (uhid->head != uhid->tail)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/drivers/hid/usbhid/Makefile b/drivers/hid/usbhid/Makefile
index 0ff227d0c033..b6349e30bd93 100644
--- a/drivers/hid/usbhid/Makefile
+++ b/drivers/hid/usbhid/Makefile
@@ -3,7 +3,7 @@
 # Makefile for the USB input drivers
 #
 
-usbhid-y	:= hid-core.o hid-quirks.o
+usbhid-y	:= hid-core.o
 usbhid-$(CONFIG_USB_HIDDEV)	+= hiddev.o
 usbhid-$(CONFIG_HID_PID)	+= hid-pidff.o
 
diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c
index 640dfb937c69..77c50cdfff97 100644
--- a/drivers/hid/usbhid/hid-core.c
+++ b/drivers/hid/usbhid/hid-core.c
@@ -978,8 +978,7 @@ static int usbhid_parse(struct hid_device *hid)
 	int num_descriptors;
 	size_t offset = offsetof(struct hid_descriptor, desc);
 
-	quirks = usbhid_lookup_quirk(le16_to_cpu(dev->descriptor.idVendor),
-			le16_to_cpu(dev->descriptor.idProduct));
+	quirks = hid_lookup_quirk(hid);
 
 	if (quirks & HID_QUIRK_IGNORE)
 		return -ENODEV;
@@ -1328,8 +1327,8 @@ static int usbhid_probe(struct usb_interface *intf, const struct usb_device_id *
 	hid->bus = BUS_USB;
 	hid->vendor = le16_to_cpu(dev->descriptor.idVendor);
 	hid->product = le16_to_cpu(dev->descriptor.idProduct);
+	hid->version = le16_to_cpu(dev->descriptor.bcdDevice);
 	hid->name[0] = 0;
-	hid->quirks = usbhid_lookup_quirk(hid->vendor, hid->product);
 	if (intf->cur_altsetting->desc.bInterfaceProtocol ==
 			USB_INTERFACE_PROTOCOL_MOUSE)
 		hid->type = HID_TYPE_USBMOUSE;
@@ -1641,7 +1640,7 @@ static int __init hid_init(void)
 {
 	int retval = -ENOMEM;
 
-	retval = usbhid_quirks_init(quirks_param);
+	retval = hid_quirks_init(quirks_param, BUS_USB, MAX_USBHID_BOOT_QUIRKS);
 	if (retval)
 		goto usbhid_quirks_init_fail;
 	retval = usb_register(&hid_driver);
@@ -1651,7 +1650,7 @@ static int __init hid_init(void)
 
 	return 0;
 usb_register_fail:
-	usbhid_quirks_exit();
+	hid_quirks_exit(BUS_USB);
 usbhid_quirks_init_fail:
 	return retval;
 }
@@ -1659,7 +1658,7 @@ static int __init hid_init(void)
 static void __exit hid_exit(void)
 {
 	usb_deregister(&hid_driver);
-	usbhid_quirks_exit();
+	hid_quirks_exit(BUS_USB);
 }
 
 module_init(hid_init);
diff --git a/drivers/hid/usbhid/hid-quirks.c b/drivers/hid/usbhid/hid-quirks.c
deleted file mode 100644
index 331f7f34ec14..000000000000
--- a/drivers/hid/usbhid/hid-quirks.c
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- *  USB HID quirks support for Linux
- *
- *  Copyright (c) 1999 Andreas Gal
- *  Copyright (c) 2000-2005 Vojtech Pavlik <vojtech@suse.cz>
- *  Copyright (c) 2005 Michael Haboustak <mike-@cinci.rr.com> for Concept2, Inc
- *  Copyright (c) 2006-2007 Jiri Kosina
- *  Copyright (c) 2007 Paul Walmsley
- */
-
-/*
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- */
-
-#include <linux/hid.h>
-#include <linux/export.h>
-#include <linux/slab.h>
-
-#include "../hid-ids.h"
-
-/*
- * Alphabetically sorted blacklist by quirk type.
- */
-
-static const struct hid_blacklist {
-	__u16 idVendor;
-	__u16 idProduct;
-	__u32 quirks;
-} hid_blacklist[] = {
-	{ USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_GAMEPAD, HID_QUIRK_BADPAD },
-	{ USB_VENDOR_ID_AASHIMA, USB_DEVICE_ID_AASHIMA_PREDATOR, HID_QUIRK_BADPAD },
-	{ USB_VENDOR_ID_ALPS, USB_DEVICE_ID_IBM_GAMEPAD, HID_QUIRK_BADPAD },
-	{ USB_VENDOR_ID_CHIC, USB_DEVICE_ID_CHIC_GAMEPAD, HID_QUIRK_BADPAD },
-	{ USB_VENDOR_ID_DWAV, USB_DEVICE_ID_EGALAX_TOUCHCONTROLLER, HID_QUIRK_MULTI_INPUT | HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_MOJO, USB_DEVICE_ID_RETRO_ADAPTER, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_HAPP, USB_DEVICE_ID_UGCI_DRIVING, HID_QUIRK_BADPAD | HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_HAPP, USB_DEVICE_ID_UGCI_FLYING, HID_QUIRK_BADPAD | HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_HAPP, USB_DEVICE_ID_UGCI_FIGHTING, HID_QUIRK_BADPAD | HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_NATSU, USB_DEVICE_ID_NATSU_GAMEPAD, HID_QUIRK_BADPAD },
-	{ USB_VENDOR_ID_NEC, USB_DEVICE_ID_NEC_USB_GAME_PAD, HID_QUIRK_BADPAD },
-	{ USB_VENDOR_ID_NEXTWINDOW, USB_DEVICE_ID_NEXTWINDOW_TOUCHSCREEN, HID_QUIRK_MULTI_INPUT},
-	{ USB_VENDOR_ID_SAITEK, USB_DEVICE_ID_SAITEK_RUMBLEPAD, HID_QUIRK_BADPAD },
-	{ USB_VENDOR_ID_TOPMAX, USB_DEVICE_ID_TOPMAX_COBRAPAD, HID_QUIRK_BADPAD },
-
-	{ USB_VENDOR_ID_AFATECH, USB_DEVICE_ID_AFATECH_AF9016, HID_QUIRK_FULLSPEED_INTERVAL },
-
-	{ USB_VENDOR_ID_EMS, USB_DEVICE_ID_EMS_TRIO_LINKER_PLUS_II, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_ETURBOTOUCH, USB_DEVICE_ID_ETURBOTOUCH, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_ETURBOTOUCH, USB_DEVICE_ID_ETURBOTOUCH_2968, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_GREENASIA, USB_DEVICE_ID_GREENASIA_DUAL_USB_JOYPAD, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_PANTHERLORD, USB_DEVICE_ID_PANTHERLORD_TWIN_USB_JOYSTICK, HID_QUIRK_MULTI_INPUT | HID_QUIRK_SKIP_OUTPUT_REPORTS },
-	{ USB_VENDOR_ID_TOUCHPACK, USB_DEVICE_ID_TOUCHPACK_RTS, HID_QUIRK_MULTI_INPUT },
-
-	{ USB_VENDOR_ID_AIREN, USB_DEVICE_ID_AIREN_SLIMPLUS, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_AKAI, USB_DEVICE_ID_AKAI_MPKMINI2, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_AKAI_09E8, USB_DEVICE_ID_AKAI_09E8_MIDIMIX, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_AMI, USB_DEVICE_ID_AMI_VIRT_KEYBOARD_AND_MOUSE, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_UC100KM, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS124U, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_2PORTKVM, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_4PORTKVM, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_4PORTKVMC, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS682, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS692, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_ATEN, USB_DEVICE_ID_ATEN_CS1758, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FIGHTERSTICK, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_COMBATSTICK, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FLIGHT_SIM_ECLIPSE_YOKE, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_FLIGHT_SIM_YOKE, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_THROTTLE, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_PRO_PEDALS, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_3AXIS_5BUTTON_STICK, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CH, USB_DEVICE_ID_CH_AXIS_295, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_PIXART_USB_OPTICAL_MOUSE, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70R, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_M65RGB, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K95RGB, HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70RGB, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_STRAFE, HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K70RGB_RAPIDFIRE, HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_K65RGB_RAPIDFIRE, HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_CORSAIR, USB_DEVICE_ID_CORSAIR_SCIMITAR_PRO_RGB, HID_QUIRK_NO_INIT_REPORTS | HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_CREATIVELABS, USB_DEVICE_ID_CREATIVE_SB_OMNI_SURROUND_51, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_DELL, USB_DEVICE_ID_DELL_PIXART_USB_OPTICAL_MOUSE, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_DMI, USB_DEVICE_ID_DMI_ENC, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_WIIU, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_PS3, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_DOLPHINBAR, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_DRAGONRISE, USB_DEVICE_ID_DRAGONRISE_GAMECUBE1, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_ELAN, HID_ANY_ID, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_ELO, USB_DEVICE_ID_ELO_TS2700, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_FORMOSA, USB_DEVICE_ID_FORMOSA_IR_RECEIVER, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_FREESCALE, USB_DEVICE_ID_FREESCALE_MX28, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_FUTABA, USB_DEVICE_ID_LED_DISPLAY, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0A4A, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_LOGITECH_OEM_USB_OPTICAL_MOUSE_0B4A, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_HP, USB_PRODUCT_ID_HP_PIXART_OEM_USB_OPTICAL_MOUSE, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_IDEACOM, USB_DEVICE_ID_IDEACOM_IDC6680, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_C007, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_C077, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_KEYBOARD_G710_PLUS, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C01A, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C05A, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_MOUSE_C06A, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_MGE, USB_DEVICE_ID_MGE_UPS, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_SURFACE_PRO_2, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TYPE_COVER_2, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_TOUCH_COVER_2, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_MICROSOFT, USB_DEVICE_ID_MS_POWER_COVER, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_MSI, USB_DEVICE_ID_MSI_GT683R_LED_PANEL, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_NEXIO, USB_DEVICE_ID_NEXIO_MULTITOUCH_PTI0750, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_NOVATEK, USB_DEVICE_ID_NOVATEK_MOUSE, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_PENMOUNT, USB_DEVICE_ID_PENMOUNT_1610, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_PENMOUNT, USB_DEVICE_ID_PENMOUNT_1640, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_USB_OPTICAL_MOUSE, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_PIXART_USB_OPTICAL_MOUSE_ID2, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_OPTICAL_TOUCH_SCREEN, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_OPTICAL_TOUCH_SCREEN1, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_PIXART, USB_DEVICE_ID_PIXART_OPTICAL_TOUCH_SCREEN2, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_PRIMAX, USB_DEVICE_ID_PRIMAX_MOUSE_4D22, HID_QUIRK_ALWAYS_POLL },
-	{ USB_VENDOR_ID_PRODIGE, USB_DEVICE_ID_PRODIGE_CORDLESS, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_QUANTA, USB_DEVICE_ID_QUANTA_OPTICAL_TOUCH_3001, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_QUANTA, USB_DEVICE_ID_QUANTA_OPTICAL_TOUCH_3003, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_QUANTA, USB_DEVICE_ID_QUANTA_OPTICAL_TOUCH_3008, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_REALTEK, USB_DEVICE_ID_REALTEK_READER, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SENNHEISER, USB_DEVICE_ID_SENNHEISER_BTD500USB, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_SIGMATEL, USB_DEVICE_ID_SIGMATEL_STMP3780, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS9200_TOUCH, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS817_TOUCH, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS_TS, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SIS_TOUCH, USB_DEVICE_ID_SIS1030_TOUCH, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_SUN, USB_DEVICE_ID_RARITAN_KVM_DONGLE, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_SYMBOL, USB_DEVICE_ID_SYMBOL_SCANNER_1, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_SYMBOL, USB_DEVICE_ID_SYMBOL_SCANNER_2, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_TPV, USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8882, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_TPV, USB_DEVICE_ID_TPV_OPTICAL_TOUCHSCREEN_8883, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_TURBOX, USB_DEVICE_ID_TURBOX_KEYBOARD, HID_QUIRK_NOGET },
-	{ USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_KNA5, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_UCLOGIC, USB_DEVICE_ID_UCLOGIC_TABLET_TWA60, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_10_6_INCH, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_MEDIA_TABLET_14_1_INCH, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_WALTOP, USB_DEVICE_ID_WALTOP_SIRIUS_BATTERY_FREE_TABLET, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_WISEGROUP, USB_DEVICE_ID_QUAD_USB_JOYPAD, HID_QUIRK_NOGET | HID_QUIRK_MULTI_INPUT },
-
-	{ USB_VENDOR_ID_WISEGROUP_LTD2, USB_DEVICE_ID_SMARTJOY_DUAL_PLUS, HID_QUIRK_NOGET | HID_QUIRK_MULTI_INPUT },
-
-	{ USB_VENDOR_ID_PI_ENGINEERING, USB_DEVICE_ID_PI_ENGINEERING_VEC_USB_FOOTPEDAL, HID_QUIRK_HIDINPUT_FORCE },
-
-	{ USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_MULTI_TOUCH, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_CHICONY, USB_DEVICE_ID_CHICONY_WIRELESS, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_SIGMA_MICRO, USB_DEVICE_ID_SIGMA_MICRO_KEYBOARD, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_MOUSEPEN_I608X_V2, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_EASYPEN_M610X, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_KYE, USB_DEVICE_ID_KYE_PENSKETCH_M912, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_NTRIG, USB_DEVICE_ID_NTRIG_DUOSENSE, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SEMICO, USB_DEVICE_ID_SEMICO_USB_KEYKOARD2, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_LTS1, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_LTS2, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_HD, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_QUAD_HD, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_SYNAPTICS, USB_DEVICE_ID_SYNAPTICS_TP_V103, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_HOLTEK_ALT, USB_DEVICE_ID_HOLTEK_ALT_KEYBOARD_A096, HID_QUIRK_NO_INIT_REPORTS },
-	{ USB_VENDOR_ID_MULTIPLE_1781, USB_DEVICE_ID_RAPHNET_4NES4SNES_OLD, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_DRACAL_RAPHNET, USB_DEVICE_ID_RAPHNET_2NES2SNES, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_DRACAL_RAPHNET, USB_DEVICE_ID_RAPHNET_4NES4SNES, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_INNOMEDIA, USB_DEVICE_ID_INNEX_GENESIS_ATARI, HID_QUIRK_MULTI_INPUT },
-	{ USB_VENDOR_ID_MCS, USB_DEVICE_ID_MCS_GAMEPADBLOCK, HID_QUIRK_MULTI_INPUT },
-
-	{ 0, 0 }
-};
-
-/* Dynamic HID quirks list - specified at runtime */
-struct quirks_list_struct {
-	struct hid_blacklist hid_bl_item;
-	struct list_head node;
-};
-
-static LIST_HEAD(dquirks_list);
-static DECLARE_RWSEM(dquirks_rwsem);
-
-/* Runtime ("dynamic") quirks manipulation functions */
-
-/**
- * usbhid_exists_dquirk: find any dynamic quirks for a USB HID device
- * @idVendor: the 16-bit USB vendor ID, in native byteorder
- * @idProduct: the 16-bit USB product ID, in native byteorder
- *
- * Description:
- *         Scans dquirks_list for a matching dynamic quirk and returns
- *         the pointer to the relevant struct hid_blacklist if found.
- *         Must be called with a read lock held on dquirks_rwsem.
- *
- * Returns: NULL if no quirk found, struct hid_blacklist * if found.
- */
-static struct hid_blacklist *usbhid_exists_dquirk(const u16 idVendor,
-		const u16 idProduct)
-{
-	struct quirks_list_struct *q;
-	struct hid_blacklist *bl_entry = NULL;
-
-	list_for_each_entry(q, &dquirks_list, node) {
-		if (q->hid_bl_item.idVendor == idVendor &&
-				q->hid_bl_item.idProduct == idProduct) {
-			bl_entry = &q->hid_bl_item;
-			break;
-		}
-	}
-
-	if (bl_entry != NULL)
-		dbg_hid("Found dynamic quirk 0x%x for USB HID vendor 0x%hx prod 0x%hx\n",
-				bl_entry->quirks, bl_entry->idVendor,
-				bl_entry->idProduct);
-
-	return bl_entry;
-}
-
-
-/**
- * usbhid_modify_dquirk: add/replace a HID quirk
- * @idVendor: the 16-bit USB vendor ID, in native byteorder
- * @idProduct: the 16-bit USB product ID, in native byteorder
- * @quirks: the u32 quirks value to add/replace
- *
- * Description:
- *         If an dynamic quirk exists in memory for this (idVendor,
- *         idProduct) pair, replace its quirks value with what was
- *         provided.  Otherwise, add the quirk to the dynamic quirks list.
- *
- * Returns: 0 OK, -error on failure.
- */
-static int usbhid_modify_dquirk(const u16 idVendor, const u16 idProduct,
-				const u32 quirks)
-{
-	struct quirks_list_struct *q_new, *q;
-	int list_edited = 0;
-
-	if (!idVendor) {
-		dbg_hid("Cannot add a quirk with idVendor = 0\n");
-		return -EINVAL;
-	}
-
-	q_new = kmalloc(sizeof(struct quirks_list_struct), GFP_KERNEL);
-	if (!q_new)
-		return -ENOMEM;
-
-	q_new->hid_bl_item.idVendor = idVendor;
-	q_new->hid_bl_item.idProduct = idProduct;
-	q_new->hid_bl_item.quirks = quirks;
-
-	down_write(&dquirks_rwsem);
-
-	list_for_each_entry(q, &dquirks_list, node) {
-
-		if (q->hid_bl_item.idVendor == idVendor &&
-				q->hid_bl_item.idProduct == idProduct) {
-
-			list_replace(&q->node, &q_new->node);
-			kfree(q);
-			list_edited = 1;
-			break;
-
-		}
-
-	}
-
-	if (!list_edited)
-		list_add_tail(&q_new->node, &dquirks_list);
-
-	up_write(&dquirks_rwsem);
-
-	return 0;
-}
-
-/**
- * usbhid_remove_all_dquirks: remove all runtime HID quirks from memory
- *
- * Description:
- *         Free all memory associated with dynamic quirks - called before
- *         module unload.
- *
- */
-static void usbhid_remove_all_dquirks(void)
-{
-	struct quirks_list_struct *q, *temp;
-
-	down_write(&dquirks_rwsem);
-	list_for_each_entry_safe(q, temp, &dquirks_list, node) {
-		list_del(&q->node);
-		kfree(q);
-	}
-	up_write(&dquirks_rwsem);
-
-}
-
-/**
- * usbhid_quirks_init: apply USB HID quirks specified at module load time
- */
-int usbhid_quirks_init(char **quirks_param)
-{
-	u16 idVendor, idProduct;
-	u32 quirks;
-	int n = 0, m;
-
-	for (; n < MAX_USBHID_BOOT_QUIRKS && quirks_param[n]; n++) {
-
-		m = sscanf(quirks_param[n], "0x%hx:0x%hx:0x%x",
-				&idVendor, &idProduct, &quirks);
-
-		if (m != 3 ||
-		    usbhid_modify_dquirk(idVendor, idProduct, quirks) != 0) {
-			pr_warn("Could not parse HID quirk module param %s\n",
-				quirks_param[n]);
-		}
-	}
-
-	return 0;
-}
-
-/**
- * usbhid_quirks_exit: release memory associated with dynamic_quirks
- *
- * Description:
- *     Release all memory associated with dynamic quirks.  Called upon
- *     module unload.
- *
- * Returns: nothing
- */
-void usbhid_quirks_exit(void)
-{
-	usbhid_remove_all_dquirks();
-}
-
-/**
- * usbhid_exists_squirk: return any static quirks for a USB HID device
- * @idVendor: the 16-bit USB vendor ID, in native byteorder
- * @idProduct: the 16-bit USB product ID, in native byteorder
- *
- * Description:
- *     Given a USB vendor ID and product ID, return a pointer to
- *     the hid_blacklist entry associated with that device.
- *
- * Returns: pointer if quirk found, or NULL if no quirks found.
- */
-static const struct hid_blacklist *usbhid_exists_squirk(const u16 idVendor,
-		const u16 idProduct)
-{
-	const struct hid_blacklist *bl_entry = NULL;
-	int n = 0;
-
-	for (; hid_blacklist[n].idVendor; n++)
-		if (hid_blacklist[n].idVendor == idVendor &&
-			(hid_blacklist[n].idProduct == (__u16) HID_ANY_ID ||
-				hid_blacklist[n].idProduct == idProduct))
-			bl_entry = &hid_blacklist[n];
-
-	if (bl_entry != NULL)
-		dbg_hid("Found squirk 0x%x for USB HID vendor 0x%hx prod 0x%hx\n",
-				bl_entry->quirks, bl_entry->idVendor,
-				bl_entry->idProduct);
-	return bl_entry;
-}
-
-/**
- * usbhid_lookup_quirk: return any quirks associated with a USB HID device
- * @idVendor: the 16-bit USB vendor ID, in native byteorder
- * @idProduct: the 16-bit USB product ID, in native byteorder
- *
- * Description:
- *     Given a USB vendor ID and product ID, return any quirks associated
- *     with that device.
- *
- * Returns: a u32 quirks value.
- */
-u32 usbhid_lookup_quirk(const u16 idVendor, const u16 idProduct)
-{
-	u32 quirks = 0;
-	const struct hid_blacklist *bl_entry = NULL;
-
-	/* NCR devices must not be queried for reports */
-	if (idVendor == USB_VENDOR_ID_NCR &&
-			idProduct >= USB_DEVICE_ID_NCR_FIRST &&
-			idProduct <= USB_DEVICE_ID_NCR_LAST)
-			return HID_QUIRK_NO_INIT_REPORTS;
-
-	down_read(&dquirks_rwsem);
-	bl_entry = usbhid_exists_dquirk(idVendor, idProduct);
-	if (!bl_entry)
-		bl_entry = usbhid_exists_squirk(idVendor, idProduct);
-	if (bl_entry)
-		quirks = bl_entry->quirks;
-	up_read(&dquirks_rwsem);
-
-	return quirks;
-}
-
-EXPORT_SYMBOL_GPL(usbhid_lookup_quirk);
diff --git a/drivers/hid/usbhid/hiddev.c b/drivers/hid/usbhid/hiddev.c
index 7d749b19c27c..e3ce233f8bdc 100644
--- a/drivers/hid/usbhid/hiddev.c
+++ b/drivers/hid/usbhid/hiddev.c
@@ -422,15 +422,15 @@ static ssize_t hiddev_read(struct file * file, char __user * buffer, size_t coun
  * "poll" file op
  * No kernel lock - fine
  */
-static unsigned int hiddev_poll(struct file *file, poll_table *wait)
+static __poll_t hiddev_poll(struct file *file, poll_table *wait)
 {
 	struct hiddev_list *list = file->private_data;
 
 	poll_wait(file, &list->hiddev->wait, wait);
 	if (list->head != list->tail)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	if (!list->hiddev->exist)
-		return POLLERR | POLLHUP;
+		return EPOLLERR | EPOLLHUP;
 	return 0;
 }
 
diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c
index ee71ad9b6cc1..409543160af7 100644
--- a/drivers/hid/wacom_sys.c
+++ b/drivers/hid/wacom_sys.c
@@ -56,6 +56,107 @@ static int wacom_set_report(struct hid_device *hdev, u8 type, u8 *buf,
 	return retval;
 }
 
+static void wacom_wac_queue_insert(struct hid_device *hdev,
+				   struct kfifo_rec_ptr_2 *fifo,
+				   u8 *raw_data, int size)
+{
+	bool warned = false;
+
+	while (kfifo_avail(fifo) < size) {
+		if (!warned)
+			hid_warn(hdev, "%s: kfifo has filled, starting to drop events\n", __func__);
+		warned = true;
+
+		kfifo_skip(fifo);
+	}
+
+	kfifo_in(fifo, raw_data, size);
+}
+
+static void wacom_wac_queue_flush(struct hid_device *hdev,
+				  struct kfifo_rec_ptr_2 *fifo)
+{
+	while (!kfifo_is_empty(fifo)) {
+		u8 buf[WACOM_PKGLEN_MAX];
+		int size;
+		int err;
+
+		size = kfifo_out(fifo, buf, sizeof(buf));
+		err = hid_report_raw_event(hdev, HID_INPUT_REPORT, buf, size, false);
+		if (err) {
+			hid_warn(hdev, "%s: unable to flush event due to error %d\n",
+				 __func__, err);
+		}
+	}
+}
+
+static int wacom_wac_pen_serial_enforce(struct hid_device *hdev,
+		struct hid_report *report, u8 *raw_data, int size)
+{
+	struct wacom *wacom = hid_get_drvdata(hdev);
+	struct wacom_wac *wacom_wac = &wacom->wacom_wac;
+	struct wacom_features *features = &wacom_wac->features;
+	bool flush = false;
+	bool insert = false;
+	int i, j;
+
+	if (wacom_wac->serial[0] || !(features->quirks & WACOM_QUIRK_TOOLSERIAL))
+		return 0;
+
+	/* Queue events which have invalid tool type or serial number */
+	for (i = 0; i < report->maxfield; i++) {
+		for (j = 0; j < report->field[i]->maxusage; j++) {
+			struct hid_field *field = report->field[i];
+			struct hid_usage *usage = &field->usage[j];
+			unsigned int equivalent_usage = wacom_equivalent_usage(usage->hid);
+			unsigned int offset;
+			unsigned int size;
+			unsigned int value;
+
+			if (equivalent_usage != HID_DG_INRANGE &&
+			    equivalent_usage != HID_DG_TOOLSERIALNUMBER &&
+			    equivalent_usage != WACOM_HID_WD_SERIALHI &&
+			    equivalent_usage != WACOM_HID_WD_TOOLTYPE)
+				continue;
+
+			offset = field->report_offset;
+			size = field->report_size;
+			value = hid_field_extract(hdev, raw_data+1, offset + j * size, size);
+
+			/* If we go out of range, we need to flush the queue ASAP */
+			if (equivalent_usage == HID_DG_INRANGE)
+				value = !value;
+
+			if (value) {
+				flush = true;
+				switch (equivalent_usage) {
+				case HID_DG_TOOLSERIALNUMBER:
+					wacom_wac->serial[0] = value;
+					break;
+
+				case WACOM_HID_WD_SERIALHI:
+					wacom_wac->serial[0] |= ((__u64)value) << 32;
+					break;
+
+				case WACOM_HID_WD_TOOLTYPE:
+					wacom_wac->id[0] = value;
+					break;
+				}
+			}
+			else {
+				insert = true;
+			}
+		}
+	}
+
+	if (flush)
+		wacom_wac_queue_flush(hdev, &wacom_wac->pen_fifo);
+	else if (insert)
+		wacom_wac_queue_insert(hdev, &wacom_wac->pen_fifo, raw_data, size);
+
+	return insert && !flush;
+}
+
 static int wacom_raw_event(struct hid_device *hdev, struct hid_report *report,
 		u8 *raw_data, int size)
 {
@@ -64,6 +165,9 @@ static int wacom_raw_event(struct hid_device *hdev, struct hid_report *report,
 	if (size > WACOM_PKGLEN_MAX)
 		return 1;
 
+	if (wacom_wac_pen_serial_enforce(hdev, report, raw_data, size))
+		return -1;
+
 	memcpy(wacom->wacom_wac.data, raw_data, size);
 
 	wacom_wac_irq(&wacom->wacom_wac, size);
@@ -2347,23 +2451,23 @@ static void wacom_remote_destroy_one(struct wacom *wacom, unsigned int index)
 	int i;
 	unsigned long flags;
 
-	spin_lock_irqsave(&remote->remote_lock, flags);
-	remote->remotes[index].registered = false;
-	spin_unlock_irqrestore(&remote->remote_lock, flags);
+	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
+		if (remote->remotes[i].serial == serial) {
 
-	if (remote->remotes[index].battery.battery)
-		devres_release_group(&wacom->hdev->dev,
-				     &remote->remotes[index].battery.bat_desc);
+			spin_lock_irqsave(&remote->remote_lock, flags);
+			remote->remotes[i].registered = false;
+			spin_unlock_irqrestore(&remote->remote_lock, flags);
 
-	if (remote->remotes[index].group.name)
-		devres_release_group(&wacom->hdev->dev,
-				     &remote->remotes[index]);
+			if (remote->remotes[i].battery.battery)
+				devres_release_group(&wacom->hdev->dev,
+						     &remote->remotes[i].battery.bat_desc);
+
+			if (remote->remotes[i].group.name)
+				devres_release_group(&wacom->hdev->dev,
+						     &remote->remotes[i]);
 
-	for (i = 0; i < WACOM_MAX_REMOTES; i++) {
-		if (remote->remotes[i].serial == serial) {
 			remote->remotes[i].serial = 0;
 			remote->remotes[i].group.name = NULL;
-			remote->remotes[i].registered = false;
 			remote->remotes[i].battery.battery = NULL;
 			wacom->led.groups[i].select = WACOM_STATUS_UNKNOWN;
 		}
@@ -2580,6 +2684,10 @@ static int wacom_probe(struct hid_device *hdev,
 		goto fail;
 	}
 
+	error = kfifo_alloc(&wacom_wac->pen_fifo, WACOM_PKGLEN_MAX, GFP_KERNEL);
+	if (error)
+		goto fail;
+
 	wacom_wac->hid_data.inputmode = -1;
 	wacom_wac->mode_report = -1;
 
@@ -2643,6 +2751,8 @@ static void wacom_remove(struct hid_device *hdev)
 	if (wacom->wacom_wac.features.type != REMOTE)
 		wacom_release_resources(wacom);
 
+	kfifo_free(&wacom_wac->pen_fifo);
+
 	hid_set_drvdata(hdev, NULL);
 }
 
diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c
index 16af6886e828..90c38a0523e9 100644
--- a/drivers/hid/wacom_wac.c
+++ b/drivers/hid/wacom_wac.c
@@ -1924,7 +1924,6 @@ static void wacom_wac_pad_event(struct hid_device *hdev, struct hid_field *field
 	struct wacom_features *features = &wacom_wac->features;
 	unsigned equivalent_usage = wacom_equivalent_usage(usage->hid);
 	int i;
-	bool is_touch_on = value;
 	bool do_report = false;
 
 	/*
@@ -1969,16 +1968,17 @@ static void wacom_wac_pad_event(struct hid_device *hdev, struct hid_field *field
 		break;
 
 	case WACOM_HID_WD_MUTE_DEVICE:
-		if (wacom_wac->shared->touch_input && value) {
-			wacom_wac->shared->is_touch_on = !wacom_wac->shared->is_touch_on;
-			is_touch_on = wacom_wac->shared->is_touch_on;
-		}
-
-		/* fall through*/
 	case WACOM_HID_WD_TOUCHONOFF:
 		if (wacom_wac->shared->touch_input) {
+			bool *is_touch_on = &wacom_wac->shared->is_touch_on;
+
+			if (equivalent_usage == WACOM_HID_WD_MUTE_DEVICE && value)
+				*is_touch_on = !(*is_touch_on);
+			else if (equivalent_usage == WACOM_HID_WD_TOUCHONOFF)
+				*is_touch_on = value;
+
 			input_report_switch(wacom_wac->shared->touch_input,
-					    SW_MUTE_DEVICE, !is_touch_on);
+					    SW_MUTE_DEVICE, !(*is_touch_on));
 			input_sync(wacom_wac->shared->touch_input);
 		}
 		break;
@@ -2085,7 +2085,29 @@ static void wacom_wac_pen_usage_mapping(struct hid_device *hdev,
 		wacom_map_usage(input, usage, field, EV_KEY, BTN_STYLUS2, 0);
 		break;
 	case HID_DG_TOOLSERIALNUMBER:
+		features->quirks |= WACOM_QUIRK_TOOLSERIAL;
 		wacom_map_usage(input, usage, field, EV_MSC, MSC_SERIAL, 0);
+
+		/* Adjust AES usages to match modern convention */
+		if (usage->hid == WACOM_HID_WT_SERIALNUMBER && field->report_size == 16) {
+			if (field->index + 2 < field->report->maxfield) {
+				struct hid_field *a = field->report->field[field->index + 1];
+				struct hid_field *b = field->report->field[field->index + 2];
+
+				if (a->maxusage > 0 && a->usage[0].hid == HID_DG_TOOLSERIALNUMBER && a->report_size == 32 &&
+				    b->maxusage > 0 && b->usage[0].hid == 0xFF000000 && b->report_size == 8) {
+					features->quirks |= WACOM_QUIRK_AESPEN;
+					usage->hid = WACOM_HID_WD_TOOLTYPE;
+					field->logical_minimum = S16_MIN;
+					field->logical_maximum = S16_MAX;
+					a->logical_minimum = S32_MIN;
+					a->logical_maximum = S32_MAX;
+					b->usage[0].hid = WACOM_HID_WD_SERIALHI;
+					b->logical_minimum = 0;
+					b->logical_maximum = U8_MAX;
+				}
+			}
+		}
 		break;
 	case WACOM_HID_WD_SENSE:
 		features->quirks |= WACOM_QUIRK_SENSE;
@@ -2093,15 +2115,18 @@ static void wacom_wac_pen_usage_mapping(struct hid_device *hdev,
 		break;
 	case WACOM_HID_WD_SERIALHI:
 		wacom_map_usage(input, usage, field, EV_ABS, ABS_MISC, 0);
-		set_bit(EV_KEY, input->evbit);
-		input_set_capability(input, EV_KEY, BTN_TOOL_PEN);
-		input_set_capability(input, EV_KEY, BTN_TOOL_RUBBER);
-		input_set_capability(input, EV_KEY, BTN_TOOL_BRUSH);
-		input_set_capability(input, EV_KEY, BTN_TOOL_PENCIL);
-		input_set_capability(input, EV_KEY, BTN_TOOL_AIRBRUSH);
-		if (!(features->device_type & WACOM_DEVICETYPE_DIRECT)) {
-			input_set_capability(input, EV_KEY, BTN_TOOL_MOUSE);
-			input_set_capability(input, EV_KEY, BTN_TOOL_LENS);
+
+		if (!(features->quirks & WACOM_QUIRK_AESPEN)) {
+			set_bit(EV_KEY, input->evbit);
+			input_set_capability(input, EV_KEY, BTN_TOOL_PEN);
+			input_set_capability(input, EV_KEY, BTN_TOOL_RUBBER);
+			input_set_capability(input, EV_KEY, BTN_TOOL_BRUSH);
+			input_set_capability(input, EV_KEY, BTN_TOOL_PENCIL);
+			input_set_capability(input, EV_KEY, BTN_TOOL_AIRBRUSH);
+			if (!(features->device_type & WACOM_DEVICETYPE_DIRECT)) {
+				input_set_capability(input, EV_KEY, BTN_TOOL_MOUSE);
+				input_set_capability(input, EV_KEY, BTN_TOOL_LENS);
+			}
 		}
 		break;
 	case WACOM_HID_WD_FINGERWHEEL:
@@ -4390,6 +4415,12 @@ static const struct wacom_features wacom_features_0x360 =
 static const struct wacom_features wacom_features_0x361 =
 	{ "Wacom Intuos Pro L", 62200, 43200, 8191, 63,
 	  INTUOSP2_BT, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 9, .touch_max = 10 };
+static const struct wacom_features wacom_features_0x37A =
+	{ "Wacom One by Wacom S", 15200, 9500, 2047, 63,
+	  BAMBOO_PEN, WACOM_INTUOS_RES, WACOM_INTUOS_RES };
+static const struct wacom_features wacom_features_0x37B =
+	{ "Wacom One by Wacom M", 21600, 13500, 2047, 63,
+	  BAMBOO_PEN, WACOM_INTUOS_RES, WACOM_INTUOS_RES };
 
 static const struct wacom_features wacom_features_HID_ANY_ID =
 	{ "Wacom HID", .type = HID_GENERIC, .oVid = HID_ANY_ID, .oPid = HID_ANY_ID };
@@ -4558,6 +4589,8 @@ const struct hid_device_id wacom_ids[] = {
 	{ USB_DEVICE_WACOM(0x343) },
 	{ BT_DEVICE_WACOM(0x360) },
 	{ BT_DEVICE_WACOM(0x361) },
+	{ USB_DEVICE_WACOM(0x37A) },
+	{ USB_DEVICE_WACOM(0x37B) },
 	{ USB_DEVICE_WACOM(0x4001) },
 	{ USB_DEVICE_WACOM(0x4004) },
 	{ USB_DEVICE_WACOM(0x5000) },
diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h
index 64d8f014602e..15d9c14fbdf7 100644
--- a/drivers/hid/wacom_wac.h
+++ b/drivers/hid/wacom_wac.h
@@ -11,6 +11,7 @@
 
 #include <linux/types.h>
 #include <linux/hid.h>
+#include <linux/kfifo.h>
 
 /* maximum packet length for USB/BT devices */
 #define WACOM_PKGLEN_MAX	361
@@ -86,7 +87,9 @@
 /* device quirks */
 #define WACOM_QUIRK_BBTOUCH_LOWRES	0x0001
 #define WACOM_QUIRK_SENSE		0x0002
+#define WACOM_QUIRK_AESPEN		0x0004
 #define WACOM_QUIRK_BATTERY		0x0008
+#define WACOM_QUIRK_TOOLSERIAL		0x0010
 
 /* device types */
 #define WACOM_DEVICETYPE_NONE           0x0000
@@ -107,6 +110,7 @@
 #define WACOM_HID_WD_PEN                (WACOM_HID_UP_WACOMDIGITIZER | 0x02)
 #define WACOM_HID_WD_SENSE              (WACOM_HID_UP_WACOMDIGITIZER | 0x36)
 #define WACOM_HID_WD_DIGITIZERFNKEYS    (WACOM_HID_UP_WACOMDIGITIZER | 0x39)
+#define WACOM_HID_WD_SERIALNUMBER       (WACOM_HID_UP_WACOMDIGITIZER | 0x5b)
 #define WACOM_HID_WD_SERIALHI           (WACOM_HID_UP_WACOMDIGITIZER | 0x5c)
 #define WACOM_HID_WD_TOOLTYPE           (WACOM_HID_UP_WACOMDIGITIZER | 0x77)
 #define WACOM_HID_WD_DISTANCE           (WACOM_HID_UP_WACOMDIGITIZER | 0x0132)
@@ -150,6 +154,7 @@
 #define WACOM_HID_WT_TOUCHSCREEN        (WACOM_HID_UP_WACOMTOUCH | 0x04)
 #define WACOM_HID_WT_TOUCHPAD           (WACOM_HID_UP_WACOMTOUCH | 0x05)
 #define WACOM_HID_WT_CONTACTMAX         (WACOM_HID_UP_WACOMTOUCH | 0x55)
+#define WACOM_HID_WT_SERIALNUMBER       (WACOM_HID_UP_WACOMTOUCH | 0x5b)
 #define WACOM_HID_WT_X                  (WACOM_HID_UP_WACOMTOUCH | 0x130)
 #define WACOM_HID_WT_Y                  (WACOM_HID_UP_WACOMTOUCH | 0x131)
 
@@ -336,6 +341,7 @@ struct wacom_wac {
 	struct input_dev *pen_input;
 	struct input_dev *touch_input;
 	struct input_dev *pad_input;
+	struct kfifo_rec_ptr_2 pen_fifo;
 	int pid;
 	int num_contacts_left;
 	u8 bt_features;
diff --git a/drivers/hsi/clients/cmt_speech.c b/drivers/hsi/clients/cmt_speech.c
index 727f968ac1cb..324cb8ec9405 100644
--- a/drivers/hsi/clients/cmt_speech.c
+++ b/drivers/hsi/clients/cmt_speech.c
@@ -451,11 +451,11 @@ static void cs_hsi_read_on_control_complete(struct hsi_msg *msg)
 	dev_dbg(&hi->cl->device, "Read on control: %08X\n", cmd);
 	cs_release_cmd(msg);
 	if (hi->flags & CS_FEAT_TSTAMP_RX_CTRL) {
-		struct timespec tspec;
+		struct timespec64 tspec;
 		struct cs_timestamp *tstamp =
 			&hi->mmap_cfg->tstamp_rx_ctrl;
 
-		ktime_get_ts(&tspec);
+		ktime_get_ts64(&tspec);
 
 		tstamp->tv_sec = (__u32) tspec.tv_sec;
 		tstamp->tv_nsec = (__u32) tspec.tv_nsec;
@@ -1124,17 +1124,17 @@ static int cs_char_fasync(int fd, struct file *file, int on)
 	return 0;
 }
 
-static unsigned int cs_char_poll(struct file *file, poll_table *wait)
+static __poll_t cs_char_poll(struct file *file, poll_table *wait)
 {
 	struct cs_char *csdata = file->private_data;
-	unsigned int ret = 0;
+	__poll_t ret = 0;
 
 	poll_wait(file, &cs_char_data.wait, wait);
 	spin_lock_bh(&csdata->lock);
 	if (!list_empty(&csdata->chardev_queue))
-		ret = POLLIN | POLLRDNORM;
+		ret = EPOLLIN | EPOLLRDNORM;
 	else if (!list_empty(&csdata->dataind_queue))
-		ret = POLLIN | POLLRDNORM;
+		ret = EPOLLIN | EPOLLRDNORM;
 	spin_unlock_bh(&csdata->lock);
 
 	return ret;
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index 8267439dd1ee..fe96aab9e794 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -49,9 +49,6 @@ struct hv_context hv_context = {
  */
 int hv_init(void)
 {
-	if (!hv_is_hypercall_page_setup())
-		return -ENOTSUPP;
-
 	hv_context.cpu_context = alloc_percpu(struct hv_per_cpu_context);
 	if (!hv_context.cpu_context)
 		return -ENOMEM;
diff --git a/drivers/hv/hv_utils_transport.c b/drivers/hv/hv_utils_transport.c
index 4402a71e23f7..832777527936 100644
--- a/drivers/hv/hv_utils_transport.c
+++ b/drivers/hv/hv_utils_transport.c
@@ -104,7 +104,7 @@ static ssize_t hvt_op_write(struct file *file, const char __user *buf,
 	return ret ? ret : count;
 }
 
-static unsigned int hvt_op_poll(struct file *file, poll_table *wait)
+static __poll_t hvt_op_poll(struct file *file, poll_table *wait)
 {
 	struct hvutil_transport *hvt;
 
@@ -113,10 +113,10 @@ static unsigned int hvt_op_poll(struct file *file, poll_table *wait)
 	poll_wait(file, &hvt->outmsg_q, wait);
 
 	if (hvt->mode == HVUTIL_TRANSPORT_DESTROY)
-		return POLLERR | POLLHUP;
+		return EPOLLERR | EPOLLHUP;
 
 	if (hvt->outmsg_len > 0)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c
index 12eb8caa4263..8699bb969e7e 100644
--- a/drivers/hv/ring_buffer.c
+++ b/drivers/hv/ring_buffer.c
@@ -140,6 +140,29 @@ static u32 hv_copyto_ringbuffer(
 	return start_write_offset;
 }
 
+/*
+ *
+ * hv_get_ringbuffer_availbytes()
+ *
+ * Get number of bytes available to read and to write to
+ * for the specified ring buffer
+ */
+static void
+hv_get_ringbuffer_availbytes(const struct hv_ring_buffer_info *rbi,
+			     u32 *read, u32 *write)
+{
+	u32 read_loc, write_loc, dsize;
+
+	/* Capture the read/write indices before they changed */
+	read_loc = READ_ONCE(rbi->ring_buffer->read_index);
+	write_loc = READ_ONCE(rbi->ring_buffer->write_index);
+	dsize = rbi->ring_datasize;
+
+	*write = write_loc >= read_loc ? dsize - (write_loc - read_loc) :
+		read_loc - write_loc;
+	*read = dsize - *write;
+}
+
 /* Get various debug metrics for the specified ring buffer. */
 void hv_ringbuffer_get_debuginfo(const struct hv_ring_buffer_info *ring_info,
 				 struct hv_ring_buffer_debug_info *debug_info)
@@ -394,13 +417,24 @@ __hv_pkt_iter_next(struct vmbus_channel *channel,
 }
 EXPORT_SYMBOL_GPL(__hv_pkt_iter_next);
 
+/* How many bytes were read in this iterator cycle */
+static u32 hv_pkt_iter_bytes_read(const struct hv_ring_buffer_info *rbi,
+					u32 start_read_index)
+{
+	if (rbi->priv_read_index >= start_read_index)
+		return rbi->priv_read_index - start_read_index;
+	else
+		return rbi->ring_datasize - start_read_index +
+			rbi->priv_read_index;
+}
+
 /*
  * Update host ring buffer after iterating over packets.
  */
 void hv_pkt_iter_close(struct vmbus_channel *channel)
 {
 	struct hv_ring_buffer_info *rbi = &channel->inbound;
-	u32 orig_write_sz = hv_get_bytes_to_write(rbi);
+	u32 curr_write_sz, pending_sz, bytes_read, start_read_index;
 
 	/*
 	 * Make sure all reads are done before we update the read index since
@@ -408,8 +442,12 @@ void hv_pkt_iter_close(struct vmbus_channel *channel)
 	 * is updated.
 	 */
 	virt_rmb();
+	start_read_index = rbi->ring_buffer->read_index;
 	rbi->ring_buffer->read_index = rbi->priv_read_index;
 
+	if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz)
+		return;
+
 	/*
 	 * Issue a full memory barrier before making the signaling decision.
 	 * Here is the reason for having this barrier:
@@ -423,26 +461,29 @@ void hv_pkt_iter_close(struct vmbus_channel *channel)
 	 */
 	virt_mb();
 
-	/* If host has disabled notifications then skip */
-	if (rbi->ring_buffer->interrupt_mask)
+	pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);
+	if (!pending_sz)
 		return;
 
-	if (rbi->ring_buffer->feature_bits.feat_pending_send_sz) {
-		u32 pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);
+	/*
+	 * Ensure the read of write_index in hv_get_bytes_to_write()
+	 * happens after the read of pending_send_sz.
+	 */
+	virt_rmb();
+	curr_write_sz = hv_get_bytes_to_write(rbi);
+	bytes_read = hv_pkt_iter_bytes_read(rbi, start_read_index);
 
-		/*
-		 * If there was space before we began iteration,
-		 * then host was not blocked. Also handles case where
-		 * pending_sz is zero then host has nothing pending
-		 * and does not need to be signaled.
-		 */
-		if (orig_write_sz > pending_sz)
-			return;
+	/*
+	 * If there was space before we began iteration,
+	 * then host was not blocked.
+	 */
 
-		/* If pending write will not fit, don't give false hope. */
-		if (hv_get_bytes_to_write(rbi) < pending_sz)
-			return;
-	}
+	if (curr_write_sz - bytes_read > pending_sz)
+		return;
+
+	/* If pending write will not fit, don't give false hope. */
+	if (curr_write_sz <= pending_sz)
+		return;
 
 	vmbus_setevent(channel);
 }
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index 610223f0e945..bc65c4d79c1f 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -37,7 +37,6 @@
 #include <linux/sched/task_stack.h>
 
 #include <asm/hyperv.h>
-#include <asm/hypervisor.h>
 #include <asm/mshyperv.h>
 #include <linux/notifier.h>
 #include <linux/ptrace.h>
@@ -1053,7 +1052,7 @@ static int vmbus_bus_init(void)
 	 * Initialize the per-cpu interrupt state and
 	 * connect to the host.
 	 */
-	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv:online",
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hyperv/vmbus:online",
 				hv_synic_init, hv_synic_cleanup);
 	if (ret < 0)
 		goto err_alloc;
@@ -1193,7 +1192,7 @@ static ssize_t out_mask_show(const struct vmbus_channel *channel, char *buf)
 
 	return sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask);
 }
-VMBUS_CHAN_ATTR_RO(out_mask);
+static VMBUS_CHAN_ATTR_RO(out_mask);
 
 static ssize_t in_mask_show(const struct vmbus_channel *channel, char *buf)
 {
@@ -1201,7 +1200,7 @@ static ssize_t in_mask_show(const struct vmbus_channel *channel, char *buf)
 
 	return sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask);
 }
-VMBUS_CHAN_ATTR_RO(in_mask);
+static VMBUS_CHAN_ATTR_RO(in_mask);
 
 static ssize_t read_avail_show(const struct vmbus_channel *channel, char *buf)
 {
@@ -1209,7 +1208,7 @@ static ssize_t read_avail_show(const struct vmbus_channel *channel, char *buf)
 
 	return sprintf(buf, "%u\n", hv_get_bytes_to_read(rbi));
 }
-VMBUS_CHAN_ATTR_RO(read_avail);
+static VMBUS_CHAN_ATTR_RO(read_avail);
 
 static ssize_t write_avail_show(const struct vmbus_channel *channel, char *buf)
 {
@@ -1217,13 +1216,13 @@ static ssize_t write_avail_show(const struct vmbus_channel *channel, char *buf)
 
 	return sprintf(buf, "%u\n", hv_get_bytes_to_write(rbi));
 }
-VMBUS_CHAN_ATTR_RO(write_avail);
+static VMBUS_CHAN_ATTR_RO(write_avail);
 
 static ssize_t show_target_cpu(const struct vmbus_channel *channel, char *buf)
 {
 	return sprintf(buf, "%u\n", channel->target_cpu);
 }
-VMBUS_CHAN_ATTR(cpu, S_IRUGO, show_target_cpu, NULL);
+static VMBUS_CHAN_ATTR(cpu, S_IRUGO, show_target_cpu, NULL);
 
 static ssize_t channel_pending_show(const struct vmbus_channel *channel,
 				    char *buf)
@@ -1232,7 +1231,7 @@ static ssize_t channel_pending_show(const struct vmbus_channel *channel,
 		       channel_pending(channel,
 				       vmbus_connection.monitor_pages[1]));
 }
-VMBUS_CHAN_ATTR(pending, S_IRUGO, channel_pending_show, NULL);
+static VMBUS_CHAN_ATTR(pending, S_IRUGO, channel_pending_show, NULL);
 
 static ssize_t channel_latency_show(const struct vmbus_channel *channel,
 				    char *buf)
@@ -1241,19 +1240,34 @@ static ssize_t channel_latency_show(const struct vmbus_channel *channel,
 		       channel_latency(channel,
 				       vmbus_connection.monitor_pages[1]));
 }
-VMBUS_CHAN_ATTR(latency, S_IRUGO, channel_latency_show, NULL);
+static VMBUS_CHAN_ATTR(latency, S_IRUGO, channel_latency_show, NULL);
 
 static ssize_t channel_interrupts_show(const struct vmbus_channel *channel, char *buf)
 {
 	return sprintf(buf, "%llu\n", channel->interrupts);
 }
-VMBUS_CHAN_ATTR(interrupts, S_IRUGO, channel_interrupts_show, NULL);
+static VMBUS_CHAN_ATTR(interrupts, S_IRUGO, channel_interrupts_show, NULL);
 
 static ssize_t channel_events_show(const struct vmbus_channel *channel, char *buf)
 {
 	return sprintf(buf, "%llu\n", channel->sig_events);
 }
-VMBUS_CHAN_ATTR(events, S_IRUGO, channel_events_show, NULL);
+static VMBUS_CHAN_ATTR(events, S_IRUGO, channel_events_show, NULL);
+
+static ssize_t subchannel_monitor_id_show(const struct vmbus_channel *channel,
+					  char *buf)
+{
+	return sprintf(buf, "%u\n", channel->offermsg.monitorid);
+}
+static VMBUS_CHAN_ATTR(monitor_id, S_IRUGO, subchannel_monitor_id_show, NULL);
+
+static ssize_t subchannel_id_show(const struct vmbus_channel *channel,
+				  char *buf)
+{
+	return sprintf(buf, "%u\n",
+		       channel->offermsg.offer.sub_channel_index);
+}
+static VMBUS_CHAN_ATTR_RO(subchannel_id);
 
 static struct attribute *vmbus_chan_attrs[] = {
 	&chan_attr_out_mask.attr,
@@ -1265,6 +1279,8 @@ static struct attribute *vmbus_chan_attrs[] = {
 	&chan_attr_latency.attr,
 	&chan_attr_interrupts.attr,
 	&chan_attr_events.attr,
+	&chan_attr_monitor_id.attr,
+	&chan_attr_subchannel_id.attr,
 	NULL
 };
 
@@ -1717,7 +1733,7 @@ static int __init hv_acpi_init(void)
 {
 	int ret, t;
 
-	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
+	if (!hv_is_hyperv_initialized())
 		return -ENODEV;
 
 	init_completion(&probe_event);
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 7ad017690e3a..ef23553ff5cb 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -26,11 +26,9 @@ if HWMON
 
 config HWMON_VID
 	tristate
-	default n
 
 config HWMON_DEBUG_CHIP
 	bool "Hardware Monitoring Chip debugging messages"
-	default n
 	help
 	  Say Y here if you want the I2C chip drivers to produce a bunch of
 	  debug messages to the system log.  Select this if you are having
@@ -42,7 +40,6 @@ comment "Native drivers"
 config SENSORS_AB8500
 	tristate "AB8500 thermal monitoring"
 	depends on AB8500_GPADC && AB8500_BM
-	default n
 	help
 	  If you say yes here you get support for the thermal sensor part
 	  of the AB8500 chip. The driver includes thermal management for
@@ -302,7 +299,6 @@ config SENSORS_APPLESMC
 	select NEW_LEDS
 	select LEDS_CLASS
 	select INPUT_POLLDEV
-	default n
 	help
 	  This driver provides support for the Apple System Management
 	  Controller, which provides an accelerometer (Apple Sudden Motion
@@ -678,7 +674,6 @@ config SENSORS_JC42
 config SENSORS_POWR1220
 	tristate "Lattice POWR1220 Power Monitoring"
 	depends on I2C
-	default n
 	help
 	  If you say yes here you get access to the hardware monitoring
 	  functions of the Lattice POWR1220 isp Power Supply Monitoring,
@@ -702,7 +697,6 @@ config SENSORS_LTC2945
 	tristate "Linear Technology LTC2945"
 	depends on I2C
 	select REGMAP_I2C
-	default n
 	help
 	  If you say yes here you get support for Linear Technology LTC2945
 	  I2C System Monitor.
@@ -727,7 +721,6 @@ config SENSORS_LTC2990
 config SENSORS_LTC4151
 	tristate "Linear Technology LTC4151"
 	depends on I2C
-	default n
 	help
 	  If you say yes here you get support for Linear Technology LTC4151
 	  High Voltage I2C Current and Voltage Monitor interface.
@@ -738,7 +731,6 @@ config SENSORS_LTC4151
 config SENSORS_LTC4215
 	tristate "Linear Technology LTC4215"
 	depends on I2C
-	default n
 	help
 	  If you say yes here you get support for Linear Technology LTC4215
 	  Hot Swap Controller I2C interface.
@@ -750,7 +742,6 @@ config SENSORS_LTC4222
 	tristate "Linear Technology LTC4222"
 	depends on I2C
 	select REGMAP_I2C
-	default n
 	help
 	  If you say yes here you get support for Linear Technology LTC4222
 	  Dual Hot Swap Controller I2C interface.
@@ -761,7 +752,6 @@ config SENSORS_LTC4222
 config SENSORS_LTC4245
 	tristate "Linear Technology LTC4245"
 	depends on I2C
-	default n
 	help
 	  If you say yes here you get support for Linear Technology LTC4245
 	  Multiple Supply Hot Swap Controller I2C interface.
@@ -773,7 +763,6 @@ config SENSORS_LTC4260
 	tristate "Linear Technology LTC4260"
 	depends on I2C
 	select REGMAP_I2C
-	default n
 	help
 	  If you say yes here you get support for Linear Technology LTC4260
 	  Positive Voltage Hot Swap Controller I2C interface.
@@ -784,7 +773,6 @@ config SENSORS_LTC4260
 config SENSORS_LTC4261
 	tristate "Linear Technology LTC4261"
 	depends on I2C
-	default n
 	help
 	  If you say yes here you get support for Linear Technology LTC4261
 	  Negative Voltage Hot Swap Controller I2C interface.
@@ -1276,7 +1264,6 @@ config SENSORS_NSA320
 config SENSORS_PCF8591
 	tristate "Philips PCF8591 ADC/DAC"
 	depends on I2C
-	default n
 	help
 	  If you say yes here you get support for Philips PCF8591 4-channel
 	  ADC, 1-channel DAC chips.
@@ -1459,7 +1446,6 @@ config SENSORS_SMSC47B397
 
 config SENSORS_SCH56XX_COMMON
 	tristate
-	default n
 
 config SENSORS_SCH5627
 	tristate "SMSC SCH5627"
@@ -1505,7 +1491,6 @@ config SENSORS_STTS751
 config SENSORS_SMM665
 	tristate "Summit Microelectronics SMM665"
 	depends on I2C
-	default n
 	help
 	  If you say yes here you get support for the hardware monitoring
 	  features of the Summit Microelectronics SMM665/SMM665B Six-Channel
@@ -1725,6 +1710,16 @@ config SENSORS_VT8231
 	  This driver can also be built as a module.  If so, the module
 	  will be called vt8231.
 
+config SENSORS_W83773G
+	tristate "Nuvoton W83773G"
+	depends on I2C
+	help
+	  If you say yes here you get support for the Nuvoton W83773G hardware
+	  monitoring chip.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called w83773g.
+
 config SENSORS_W83781D
 	tristate "Winbond W83781D, W83782D, W83783S, Asus AS99127F"
 	depends on I2C
@@ -1782,7 +1777,6 @@ config SENSORS_W83795
 config SENSORS_W83795_FANCTRL
 	bool "Include automatic fan control support (DANGEROUS)"
 	depends on SENSORS_W83795
-	default n
 	help
 	  If you say yes here, support for automatic fan speed control
 	  will be included in the driver.
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 0fe489fab663..f814b4ace138 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_SENSORS_ATK0110)	+= asus_atk0110.o
 # asb100, then w83781d go first, as they can override other drivers' addresses.
 obj-$(CONFIG_SENSORS_ASB100)	+= asb100.o
 obj-$(CONFIG_SENSORS_W83627HF)	+= w83627hf.o
+obj-$(CONFIG_SENSORS_W83773G)	+= w83773g.o
 obj-$(CONFIG_SENSORS_W83792D)	+= w83792d.o
 obj-$(CONFIG_SENSORS_W83793)	+= w83793.o
 obj-$(CONFIG_SENSORS_W83795)	+= w83795.o
diff --git a/drivers/hwmon/aspeed-pwm-tacho.c b/drivers/hwmon/aspeed-pwm-tacho.c
index 63a95e23ca81..693a3d53cab5 100644
--- a/drivers/hwmon/aspeed-pwm-tacho.c
+++ b/drivers/hwmon/aspeed-pwm-tacho.c
@@ -19,6 +19,7 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/reset.h>
 #include <linux/sysfs.h>
 #include <linux/thermal.h>
 
@@ -181,6 +182,7 @@ struct aspeed_cooling_device {
 
 struct aspeed_pwm_tacho_data {
 	struct regmap *regmap;
+	struct reset_control *rst;
 	unsigned long clk_freq;
 	bool pwm_present[8];
 	bool fan_tach_present[16];
@@ -905,6 +907,13 @@ static int aspeed_create_fan(struct device *dev,
 	return 0;
 }
 
+static void aspeed_pwm_tacho_remove(void *data)
+{
+	struct aspeed_pwm_tacho_data *priv = data;
+
+	reset_control_assert(priv->rst);
+}
+
 static int aspeed_pwm_tacho_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -931,6 +940,19 @@ static int aspeed_pwm_tacho_probe(struct platform_device *pdev)
 			&aspeed_pwm_tacho_regmap_config);
 	if (IS_ERR(priv->regmap))
 		return PTR_ERR(priv->regmap);
+
+	priv->rst = devm_reset_control_get_exclusive(dev, NULL);
+	if (IS_ERR(priv->rst)) {
+		dev_err(dev,
+			"missing or invalid reset controller device tree entry");
+		return PTR_ERR(priv->rst);
+	}
+	reset_control_deassert(priv->rst);
+
+	ret = devm_add_action_or_reset(dev, aspeed_pwm_tacho_remove, priv);
+	if (ret)
+		return ret;
+
 	regmap_write(priv->regmap, ASPEED_PTCR_TACH_SOURCE, 0);
 	regmap_write(priv->regmap, ASPEED_PTCR_TACH_SOURCE_EXT, 0);
 
diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index c13a4fd86b3c..72c338eb5fae 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -246,7 +246,8 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev)
 	int err;
 	u32 eax, edx;
 	int i;
-	struct pci_dev *host_bridge = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
+	u16 devfn = PCI_DEVFN(0, 0);
+	struct pci_dev *host_bridge = pci_get_domain_bus_and_slot(0, 0, devfn);
 
 	/*
 	 * Explicit tjmax table entries override heuristics.
@@ -268,13 +269,13 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev)
 	for (i = 0; i < ARRAY_SIZE(tjmax_model_table); i++) {
 		const struct tjmax_model *tm = &tjmax_model_table[i];
 		if (c->x86_model == tm->model &&
-		    (tm->mask == ANY || c->x86_mask == tm->mask))
+		    (tm->mask == ANY || c->x86_stepping == tm->mask))
 			return tm->tjmax;
 	}
 
 	/* Early chips have no MSR for TjMax */
 
-	if (c->x86_model == 0xf && c->x86_mask < 4)
+	if (c->x86_model == 0xf && c->x86_stepping < 4)
 		usemsr_ee = 0;
 
 	if (c->x86_model > 0xe && usemsr_ee) {
@@ -425,7 +426,7 @@ static int chk_ucode_version(unsigned int cpu)
 	 * Readings might stop update when processor visited too deep sleep,
 	 * fixed for stepping D0 (6EC).
 	 */
-	if (c->x86_model == 0xe && c->x86_mask < 0xc && c->microcode < 0x39) {
+	if (c->x86_model == 0xe && c->x86_stepping < 0xc && c->microcode < 0x39) {
 		pr_err("Errata AE18 not fixed, update BIOS or microcode of the CPU!\n");
 		return -ENODEV;
 	}
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index c7c9e95e58a8..bf3bb7e1adab 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -76,6 +76,7 @@ static uint i8k_fan_mult = I8K_FAN_MULT;
 static uint i8k_pwm_mult;
 static uint i8k_fan_max = I8K_FAN_HIGH;
 static bool disallow_fan_type_call;
+static bool disallow_fan_support;
 
 #define I8K_HWMON_HAVE_TEMP1	(1 << 0)
 #define I8K_HWMON_HAVE_TEMP2	(1 << 1)
@@ -242,6 +243,9 @@ static int i8k_get_fan_status(int fan)
 {
 	struct smm_regs regs = { .eax = I8K_SMM_GET_FAN, };
 
+	if (disallow_fan_support)
+		return -EINVAL;
+
 	regs.ebx = fan & 0xff;
 	return i8k_smm(&regs) ? : regs.eax & 0xff;
 }
@@ -253,6 +257,9 @@ static int i8k_get_fan_speed(int fan)
 {
 	struct smm_regs regs = { .eax = I8K_SMM_GET_SPEED, };
 
+	if (disallow_fan_support)
+		return -EINVAL;
+
 	regs.ebx = fan & 0xff;
 	return i8k_smm(&regs) ? : (regs.eax & 0xffff) * i8k_fan_mult;
 }
@@ -264,7 +271,7 @@ static int _i8k_get_fan_type(int fan)
 {
 	struct smm_regs regs = { .eax = I8K_SMM_GET_FAN_TYPE, };
 
-	if (disallow_fan_type_call)
+	if (disallow_fan_support || disallow_fan_type_call)
 		return -EINVAL;
 
 	regs.ebx = fan & 0xff;
@@ -289,6 +296,9 @@ static int i8k_get_fan_nominal_speed(int fan, int speed)
 {
 	struct smm_regs regs = { .eax = I8K_SMM_GET_NOM_SPEED, };
 
+	if (disallow_fan_support)
+		return -EINVAL;
+
 	regs.ebx = (fan & 0xff) | (speed << 8);
 	return i8k_smm(&regs) ? : (regs.eax & 0xffff) * i8k_fan_mult;
 }
@@ -300,6 +310,9 @@ static int i8k_set_fan(int fan, int speed)
 {
 	struct smm_regs regs = { .eax = I8K_SMM_SET_FAN, };
 
+	if (disallow_fan_support)
+		return -EINVAL;
+
 	speed = (speed < 0) ? 0 : ((speed > i8k_fan_max) ? i8k_fan_max : speed);
 	regs.ebx = (fan & 0xff) | (speed << 8);
 
@@ -772,6 +785,8 @@ static struct attribute *i8k_attrs[] = {
 static umode_t i8k_is_visible(struct kobject *kobj, struct attribute *attr,
 			      int index)
 {
+	if (disallow_fan_support && index >= 8)
+		return 0;
 	if (disallow_fan_type_call &&
 	    (index == 9 || index == 12 || index == 15))
 		return 0;
@@ -1038,6 +1053,30 @@ static const struct dmi_system_id i8k_blacklist_fan_type_dmi_table[] __initconst
 	{ }
 };
 
+/*
+ * On some machines all fan related SMM functions implemented by Dell BIOS
+ * firmware freeze kernel for about 500ms. Until Dell fixes these problems fan
+ * support for affected blacklisted Dell machines stay disabled.
+ * See bug: https://bugzilla.kernel.org/show_bug.cgi?id=195751
+ */
+static struct dmi_system_id i8k_blacklist_fan_support_dmi_table[] __initdata = {
+	{
+		.ident = "Dell Inspiron 7720",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Inspiron 7720"),
+		},
+	},
+	{
+		.ident = "Dell Vostro 3360",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Vostro 3360"),
+		},
+	},
+	{ }
+};
+
 /*
  * Probe for the presence of a supported laptop.
  */
@@ -1060,8 +1099,17 @@ static int __init i8k_probe(void)
 			i8k_get_dmi_data(DMI_BIOS_VERSION));
 	}
 
-	if (dmi_check_system(i8k_blacklist_fan_type_dmi_table))
-		disallow_fan_type_call = true;
+	if (dmi_check_system(i8k_blacklist_fan_support_dmi_table)) {
+		pr_warn("broken Dell BIOS detected, disallow fan support\n");
+		if (!force)
+			disallow_fan_support = true;
+	}
+
+	if (dmi_check_system(i8k_blacklist_fan_type_dmi_table)) {
+		pr_warn("broken Dell BIOS detected, disallow fan type call\n");
+		if (!force)
+			disallow_fan_type_call = true;
+	}
 
 	strlcpy(bios_version, i8k_get_dmi_data(DMI_BIOS_VERSION),
 		sizeof(bios_version));
diff --git a/drivers/hwmon/hih6130.c b/drivers/hwmon/hih6130.c
index 7b73d2002d3e..0ae1ee1dbf76 100644
--- a/drivers/hwmon/hih6130.c
+++ b/drivers/hwmon/hih6130.c
@@ -37,7 +37,7 @@
 
 /**
  * struct hih6130 - HIH-6130 device specific data
- * @hwmon_dev: device registered with hwmon
+ * @client: pointer to I2C client device
  * @lock: mutex to protect measurement values
  * @valid: only false before first measurement is taken
  * @last_update: time of last update (jiffies)
diff --git a/drivers/hwmon/hwmon-vid.c b/drivers/hwmon/hwmon-vid.c
index ef91b8a67549..84e91286fc4f 100644
--- a/drivers/hwmon/hwmon-vid.c
+++ b/drivers/hwmon/hwmon-vid.c
@@ -293,7 +293,7 @@ u8 vid_which_vrm(void)
 	if (c->x86 < 6)		/* Any CPU with family lower than 6 */
 		return 0;	/* doesn't have VID */
 
-	vrm_ret = find_vrm(c->x86, c->x86_model, c->x86_mask, c->x86_vendor);
+	vrm_ret = find_vrm(c->x86, c->x86_model, c->x86_stepping, c->x86_vendor);
 	if (vrm_ret == 134)
 		vrm_ret = get_via_model_d_vrm();
 	if (vrm_ret == 0)
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index af5123042990..32083e452cde 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -678,7 +678,7 @@ EXPORT_SYMBOL_GPL(hwmon_device_register_with_groups);
  * @dev: the parent device
  * @name: hwmon name attribute
  * @drvdata: driver data to attach to created device
- * @info: pointer to hwmon chip information
+ * @chip: pointer to hwmon chip information
  * @extra_groups: pointer to list of additional non-standard attribute groups
  *
  * hwmon_device_unregister() must be called when the device is no
@@ -785,11 +785,11 @@ EXPORT_SYMBOL_GPL(devm_hwmon_device_register_with_groups);
 
 /**
  * devm_hwmon_device_register_with_info - register w/ hwmon
- * @dev: the parent device
- * @name: hwmon name attribute
- * @drvdata: driver data to attach to created device
- * @info: Pointer to hwmon chip information
- * @groups - pointer to list of driver specific attribute groups
+ * @dev:	the parent device
+ * @name:	hwmon name attribute
+ * @drvdata:	driver data to attach to created device
+ * @chip:	pointer to hwmon chip information
+ * @groups:	pointer to list of driver specific attribute groups
  *
  * Returns the pointer to the new device. The new device is automatically
  * unregistered with the parent device.
diff --git a/drivers/hwmon/iio_hwmon.c b/drivers/hwmon/iio_hwmon.c
index f6a76679c650..5e5b32a1ec4b 100644
--- a/drivers/hwmon/iio_hwmon.c
+++ b/drivers/hwmon/iio_hwmon.c
@@ -23,7 +23,8 @@
  * @channels:		filled with array of channels from iio
  * @num_channels:	number of channels in channels (saves counting twice)
  * @hwmon_dev:		associated hwmon device
- * @attr_group:	the group of attributes
+ * @attr_group:		the group of attributes
+ * @groups:		null terminated array of attribute groups
  * @attrs:		null terminated array of attribute pointers.
  */
 struct iio_hwmon_state {
diff --git a/drivers/hwmon/ina2xx.c b/drivers/hwmon/ina2xx.c
index 62e38fa8cda2..e9e6aeabbf84 100644
--- a/drivers/hwmon/ina2xx.c
+++ b/drivers/hwmon/ina2xx.c
@@ -95,18 +95,20 @@ enum ina2xx_ids { ina219, ina226 };
 
 struct ina2xx_config {
 	u16 config_default;
-	int calibration_factor;
+	int calibration_value;
 	int registers;
 	int shunt_div;
 	int bus_voltage_shift;
 	int bus_voltage_lsb;	/* uV */
-	int power_lsb;		/* uW */
+	int power_lsb_factor;
 };
 
 struct ina2xx_data {
 	const struct ina2xx_config *config;
 
 	long rshunt;
+	long current_lsb_uA;
+	long power_lsb_uW;
 	struct mutex config_lock;
 	struct regmap *regmap;
 
@@ -116,21 +118,21 @@ struct ina2xx_data {
 static const struct ina2xx_config ina2xx_config[] = {
 	[ina219] = {
 		.config_default = INA219_CONFIG_DEFAULT,
-		.calibration_factor = 40960000,
+		.calibration_value = 4096,
 		.registers = INA219_REGISTERS,
 		.shunt_div = 100,
 		.bus_voltage_shift = 3,
 		.bus_voltage_lsb = 4000,
-		.power_lsb = 20000,
+		.power_lsb_factor = 20,
 	},
 	[ina226] = {
 		.config_default = INA226_CONFIG_DEFAULT,
-		.calibration_factor = 5120000,
+		.calibration_value = 2048,
 		.registers = INA226_REGISTERS,
 		.shunt_div = 400,
 		.bus_voltage_shift = 0,
 		.bus_voltage_lsb = 1250,
-		.power_lsb = 25000,
+		.power_lsb_factor = 25,
 	},
 };
 
@@ -169,12 +171,16 @@ static u16 ina226_interval_to_reg(int interval)
 	return INA226_SHIFT_AVG(avg_bits);
 }
 
+/*
+ * Calibration register is set to the best value, which eliminates
+ * truncation errors on calculating current register in hardware.
+ * According to datasheet (eq. 3) the best values are 2048 for
+ * ina226 and 4096 for ina219. They are hardcoded as calibration_value.
+ */
 static int ina2xx_calibrate(struct ina2xx_data *data)
 {
-	u16 val = DIV_ROUND_CLOSEST(data->config->calibration_factor,
-				    data->rshunt);
-
-	return regmap_write(data->regmap, INA2XX_CALIBRATION, val);
+	return regmap_write(data->regmap, INA2XX_CALIBRATION,
+			    data->config->calibration_value);
 }
 
 /*
@@ -187,10 +193,6 @@ static int ina2xx_init(struct ina2xx_data *data)
 	if (ret < 0)
 		return ret;
 
-	/*
-	 * Set current LSB to 1mA, shunt is in uOhms
-	 * (equation 13 in datasheet).
-	 */
 	return ina2xx_calibrate(data);
 }
 
@@ -268,15 +270,15 @@ static int ina2xx_get_value(struct ina2xx_data *data, u8 reg,
 		val = DIV_ROUND_CLOSEST(val, 1000);
 		break;
 	case INA2XX_POWER:
-		val = regval * data->config->power_lsb;
+		val = regval * data->power_lsb_uW;
 		break;
 	case INA2XX_CURRENT:
-		/* signed register, LSB=1mA (selected), in mA */
-		val = (s16)regval;
+		/* signed register, result in mA */
+		val = regval * data->current_lsb_uA;
+		val = DIV_ROUND_CLOSEST(val, 1000);
 		break;
 	case INA2XX_CALIBRATION:
-		val = DIV_ROUND_CLOSEST(data->config->calibration_factor,
-					regval);
+		val = regval;
 		break;
 	default:
 		/* programmer goofed */
@@ -304,9 +306,32 @@ static ssize_t ina2xx_show_value(struct device *dev,
 			ina2xx_get_value(data, attr->index, regval));
 }
 
-static ssize_t ina2xx_set_shunt(struct device *dev,
-				struct device_attribute *da,
-				const char *buf, size_t count)
+/*
+ * In order to keep calibration register value fixed, the product
+ * of current_lsb and shunt_resistor should also be fixed and equal
+ * to shunt_voltage_lsb = 1 / shunt_div multiplied by 10^9 in order
+ * to keep the scale.
+ */
+static int ina2xx_set_shunt(struct ina2xx_data *data, long val)
+{
+	unsigned int dividend = DIV_ROUND_CLOSEST(1000000000,
+						  data->config->shunt_div);
+	if (val <= 0 || val > dividend)
+		return -EINVAL;
+
+	mutex_lock(&data->config_lock);
+	data->rshunt = val;
+	data->current_lsb_uA = DIV_ROUND_CLOSEST(dividend, val);
+	data->power_lsb_uW = data->config->power_lsb_factor *
+			     data->current_lsb_uA;
+	mutex_unlock(&data->config_lock);
+
+	return 0;
+}
+
+static ssize_t ina2xx_store_shunt(struct device *dev,
+				  struct device_attribute *da,
+				  const char *buf, size_t count)
 {
 	unsigned long val;
 	int status;
@@ -316,18 +341,9 @@ static ssize_t ina2xx_set_shunt(struct device *dev,
 	if (status < 0)
 		return status;
 
-	if (val == 0 ||
-	    /* Values greater than the calibration factor make no sense. */
-	    val > data->config->calibration_factor)
-		return -EINVAL;
-
-	mutex_lock(&data->config_lock);
-	data->rshunt = val;
-	status = ina2xx_calibrate(data);
-	mutex_unlock(&data->config_lock);
+	status = ina2xx_set_shunt(data, val);
 	if (status < 0)
 		return status;
-
 	return count;
 }
 
@@ -387,7 +403,7 @@ static SENSOR_DEVICE_ATTR(power1_input, S_IRUGO, ina2xx_show_value, NULL,
 
 /* shunt resistance */
 static SENSOR_DEVICE_ATTR(shunt_resistor, S_IRUGO | S_IWUSR,
-			  ina2xx_show_value, ina2xx_set_shunt,
+			  ina2xx_show_value, ina2xx_store_shunt,
 			  INA2XX_CALIBRATION);
 
 /* update interval (ina226 only) */
@@ -438,6 +454,7 @@ static int ina2xx_probe(struct i2c_client *client,
 
 	/* set the device type */
 	data->config = &ina2xx_config[chip];
+	mutex_init(&data->config_lock);
 
 	if (of_property_read_u32(dev->of_node, "shunt-resistor", &val) < 0) {
 		struct ina2xx_platform_data *pdata = dev_get_platdata(dev);
@@ -448,10 +465,7 @@ static int ina2xx_probe(struct i2c_client *client,
 			val = INA2XX_RSHUNT_DEFAULT;
 	}
 
-	if (val <= 0 || val > data->config->calibration_factor)
-		return -ENODEV;
-
-	data->rshunt = val;
+	ina2xx_set_shunt(data, val);
 
 	ina2xx_regmap_config.max_register = data->config->registers;
 
@@ -467,8 +481,6 @@ static int ina2xx_probe(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	mutex_init(&data->config_lock);
-
 	data->groups[group++] = &ina2xx_group;
 	if (id->driver_data == ina226)
 		data->groups[group++] = &ina226_group;
diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c
index 0721e175664a..051a72eecb24 100644
--- a/drivers/hwmon/k10temp.c
+++ b/drivers/hwmon/k10temp.c
@@ -86,6 +86,7 @@ static const struct tctl_offset tctl_offset_table[] = {
 	{ 0x17, "AMD Ryzen 7 1800X", 20000 },
 	{ 0x17, "AMD Ryzen Threadripper 1950X", 27000 },
 	{ 0x17, "AMD Ryzen Threadripper 1920X", 27000 },
+	{ 0x17, "AMD Ryzen Threadripper 1900X", 27000 },
 	{ 0x17, "AMD Ryzen Threadripper 1950", 10000 },
 	{ 0x17, "AMD Ryzen Threadripper 1920", 10000 },
 	{ 0x17, "AMD Ryzen Threadripper 1910", 10000 },
@@ -128,7 +129,10 @@ static ssize_t temp1_input_show(struct device *dev,
 
 	data->read_tempreg(data->pdev, &regval);
 	temp = (regval >> 21) * 125;
-	temp -= data->temp_offset;
+	if (temp > data->temp_offset)
+		temp -= data->temp_offset;
+	else
+		temp = 0;
 
 	return sprintf(buf, "%u\n", temp);
 }
@@ -226,7 +230,7 @@ static bool has_erratum_319(struct pci_dev *pdev)
 	 * and AM3 formats, but that's the best we can do.
 	 */
 	return boot_cpu_data.x86_model < 4 ||
-	       (boot_cpu_data.x86_model == 4 && boot_cpu_data.x86_mask <= 2);
+	       (boot_cpu_data.x86_model == 4 && boot_cpu_data.x86_stepping <= 2);
 }
 
 static int k10temp_probe(struct pci_dev *pdev,
diff --git a/drivers/hwmon/k8temp.c b/drivers/hwmon/k8temp.c
index 5a632bcf869b..e59f9113fb93 100644
--- a/drivers/hwmon/k8temp.c
+++ b/drivers/hwmon/k8temp.c
@@ -187,7 +187,7 @@ static int k8temp_probe(struct pci_dev *pdev,
 		return -ENOMEM;
 
 	model = boot_cpu_data.x86_model;
-	stepping = boot_cpu_data.x86_mask;
+	stepping = boot_cpu_data.x86_stepping;
 
 	/* feature available since SH-C0, exclude older revisions */
 	if ((model == 4 && stepping == 0) ||
diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c
index 005ffb5ffa92..49f4b33a5685 100644
--- a/drivers/hwmon/lm75.c
+++ b/drivers/hwmon/lm75.c
@@ -100,7 +100,7 @@ static int lm75_read(struct device *dev, enum hwmon_sensor_types type,
 		switch (attr) {
 		case hwmon_chip_update_interval:
 			*val = data->sample_time;
-			break;;
+			break;
 		default:
 			return -EINVAL;
 		}
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index 08479006c7f9..6e4298e99222 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -39,6 +39,7 @@ config SENSORS_ADM1275
 
 config SENSORS_IBM_CFFPS
 	tristate "IBM Common Form Factor Power Supply"
+	depends on LEDS_CLASS
 	help
 	  If you say yes here you get hardware monitoring support for the IBM
 	  Common Form Factor power supply.
diff --git a/drivers/hwmon/pmbus/ibm-cffps.c b/drivers/hwmon/pmbus/ibm-cffps.c
index cb56da6834e5..93d9a9ea112b 100644
--- a/drivers/hwmon/pmbus/ibm-cffps.c
+++ b/drivers/hwmon/pmbus/ibm-cffps.c
@@ -8,12 +8,29 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/debugfs.h>
 #include <linux/device.h>
+#include <linux/fs.h>
 #include <linux/i2c.h>
+#include <linux/jiffies.h>
+#include <linux/leds.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pmbus.h>
 
 #include "pmbus.h"
 
+#define CFFPS_FRU_CMD				0x9A
+#define CFFPS_PN_CMD				0x9B
+#define CFFPS_SN_CMD				0x9E
+#define CFFPS_CCIN_CMD				0xBD
+#define CFFPS_FW_CMD_START			0xFA
+#define CFFPS_FW_NUM_BYTES			4
+#define CFFPS_SYS_CONFIG_CMD			0xDA
+
+#define CFFPS_INPUT_HISTORY_CMD			0xD6
+#define CFFPS_INPUT_HISTORY_SIZE		100
+
 /* STATUS_MFR_SPECIFIC bits */
 #define CFFPS_MFR_FAN_FAULT			BIT(0)
 #define CFFPS_MFR_THERMAL_FAULT			BIT(1)
@@ -24,6 +41,153 @@
 #define CFFPS_MFR_VAUX_FAULT			BIT(6)
 #define CFFPS_MFR_CURRENT_SHARE_WARNING		BIT(7)
 
+#define CFFPS_LED_BLINK				BIT(0)
+#define CFFPS_LED_ON				BIT(1)
+#define CFFPS_LED_OFF				BIT(2)
+#define CFFPS_BLINK_RATE_MS			250
+
+enum {
+	CFFPS_DEBUGFS_INPUT_HISTORY = 0,
+	CFFPS_DEBUGFS_FRU,
+	CFFPS_DEBUGFS_PN,
+	CFFPS_DEBUGFS_SN,
+	CFFPS_DEBUGFS_CCIN,
+	CFFPS_DEBUGFS_FW,
+	CFFPS_DEBUGFS_NUM_ENTRIES
+};
+
+struct ibm_cffps_input_history {
+	struct mutex update_lock;
+	unsigned long last_update;
+
+	u8 byte_count;
+	u8 data[CFFPS_INPUT_HISTORY_SIZE];
+};
+
+struct ibm_cffps {
+	struct i2c_client *client;
+
+	struct ibm_cffps_input_history input_history;
+
+	int debugfs_entries[CFFPS_DEBUGFS_NUM_ENTRIES];
+
+	char led_name[32];
+	u8 led_state;
+	struct led_classdev led;
+};
+
+#define to_psu(x, y) container_of((x), struct ibm_cffps, debugfs_entries[(y)])
+
+static ssize_t ibm_cffps_read_input_history(struct ibm_cffps *psu,
+					    char __user *buf, size_t count,
+					    loff_t *ppos)
+{
+	int rc;
+	u8 msgbuf0[1] = { CFFPS_INPUT_HISTORY_CMD };
+	u8 msgbuf1[CFFPS_INPUT_HISTORY_SIZE + 1] = { 0 };
+	struct i2c_msg msg[2] = {
+		{
+			.addr = psu->client->addr,
+			.flags = psu->client->flags,
+			.len = 1,
+			.buf = msgbuf0,
+		}, {
+			.addr = psu->client->addr,
+			.flags = psu->client->flags | I2C_M_RD,
+			.len = CFFPS_INPUT_HISTORY_SIZE + 1,
+			.buf = msgbuf1,
+		},
+	};
+
+	if (!*ppos) {
+		mutex_lock(&psu->input_history.update_lock);
+		if (time_after(jiffies, psu->input_history.last_update + HZ)) {
+			/*
+			 * Use a raw i2c transfer, since we need more bytes
+			 * than Linux I2C supports through smbus xfr (only 32).
+			 */
+			rc = i2c_transfer(psu->client->adapter, msg, 2);
+			if (rc < 0) {
+				mutex_unlock(&psu->input_history.update_lock);
+				return rc;
+			}
+
+			psu->input_history.byte_count = msgbuf1[0];
+			memcpy(psu->input_history.data, &msgbuf1[1],
+			       CFFPS_INPUT_HISTORY_SIZE);
+			psu->input_history.last_update = jiffies;
+		}
+
+		mutex_unlock(&psu->input_history.update_lock);
+	}
+
+	return simple_read_from_buffer(buf, count, ppos,
+				       psu->input_history.data,
+				       psu->input_history.byte_count);
+}
+
+static ssize_t ibm_cffps_debugfs_op(struct file *file, char __user *buf,
+				    size_t count, loff_t *ppos)
+{
+	u8 cmd;
+	int i, rc;
+	int *idxp = file->private_data;
+	int idx = *idxp;
+	struct ibm_cffps *psu = to_psu(idxp, idx);
+	char data[I2C_SMBUS_BLOCK_MAX] = { 0 };
+
+	switch (idx) {
+	case CFFPS_DEBUGFS_INPUT_HISTORY:
+		return ibm_cffps_read_input_history(psu, buf, count, ppos);
+	case CFFPS_DEBUGFS_FRU:
+		cmd = CFFPS_FRU_CMD;
+		break;
+	case CFFPS_DEBUGFS_PN:
+		cmd = CFFPS_PN_CMD;
+		break;
+	case CFFPS_DEBUGFS_SN:
+		cmd = CFFPS_SN_CMD;
+		break;
+	case CFFPS_DEBUGFS_CCIN:
+		rc = i2c_smbus_read_word_swapped(psu->client, CFFPS_CCIN_CMD);
+		if (rc < 0)
+			return rc;
+
+		rc = snprintf(data, 5, "%04X", rc);
+		goto done;
+	case CFFPS_DEBUGFS_FW:
+		for (i = 0; i < CFFPS_FW_NUM_BYTES; ++i) {
+			rc = i2c_smbus_read_byte_data(psu->client,
+						      CFFPS_FW_CMD_START + i);
+			if (rc < 0)
+				return rc;
+
+			snprintf(&data[i * 2], 3, "%02X", rc);
+		}
+
+		rc = i * 2;
+		goto done;
+	default:
+		return -EINVAL;
+	}
+
+	rc = i2c_smbus_read_block_data(psu->client, cmd, data);
+	if (rc < 0)
+		return rc;
+
+done:
+	data[rc] = '\n';
+	rc += 2;
+
+	return simple_read_from_buffer(buf, count, ppos, data, rc);
+}
+
+static const struct file_operations ibm_cffps_fops = {
+	.llseek = noop_llseek,
+	.read = ibm_cffps_debugfs_op,
+	.open = simple_open,
+};
+
 static int ibm_cffps_read_byte_data(struct i2c_client *client, int page,
 				    int reg)
 {
@@ -105,6 +269,69 @@ static int ibm_cffps_read_word_data(struct i2c_client *client, int page,
 	return rc;
 }
 
+static void ibm_cffps_led_brightness_set(struct led_classdev *led_cdev,
+					 enum led_brightness brightness)
+{
+	int rc;
+	struct ibm_cffps *psu = container_of(led_cdev, struct ibm_cffps, led);
+
+	if (brightness == LED_OFF) {
+		psu->led_state = CFFPS_LED_OFF;
+	} else {
+		brightness = LED_FULL;
+		if (psu->led_state != CFFPS_LED_BLINK)
+			psu->led_state = CFFPS_LED_ON;
+	}
+
+	rc = i2c_smbus_write_byte_data(psu->client, CFFPS_SYS_CONFIG_CMD,
+				       psu->led_state);
+	if (rc < 0)
+		return;
+
+	led_cdev->brightness = brightness;
+}
+
+static int ibm_cffps_led_blink_set(struct led_classdev *led_cdev,
+				   unsigned long *delay_on,
+				   unsigned long *delay_off)
+{
+	int rc;
+	struct ibm_cffps *psu = container_of(led_cdev, struct ibm_cffps, led);
+
+	psu->led_state = CFFPS_LED_BLINK;
+
+	if (led_cdev->brightness == LED_OFF)
+		return 0;
+
+	rc = i2c_smbus_write_byte_data(psu->client, CFFPS_SYS_CONFIG_CMD,
+				       CFFPS_LED_BLINK);
+	if (rc < 0)
+		return rc;
+
+	*delay_on = CFFPS_BLINK_RATE_MS;
+	*delay_off = CFFPS_BLINK_RATE_MS;
+
+	return 0;
+}
+
+static void ibm_cffps_create_led_class(struct ibm_cffps *psu)
+{
+	int rc;
+	struct i2c_client *client = psu->client;
+	struct device *dev = &client->dev;
+
+	snprintf(psu->led_name, sizeof(psu->led_name), "%s-%02x", client->name,
+		 client->addr);
+	psu->led.name = psu->led_name;
+	psu->led.max_brightness = LED_FULL;
+	psu->led.brightness_set = ibm_cffps_led_brightness_set;
+	psu->led.blink_set = ibm_cffps_led_blink_set;
+
+	rc = devm_led_classdev_register(dev, &psu->led);
+	if (rc)
+		dev_warn(dev, "failed to register led class: %d\n", rc);
+}
+
 static struct pmbus_driver_info ibm_cffps_info = {
 	.pages = 1,
 	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT |
@@ -116,10 +343,69 @@ static struct pmbus_driver_info ibm_cffps_info = {
 	.read_word_data = ibm_cffps_read_word_data,
 };
 
+static struct pmbus_platform_data ibm_cffps_pdata = {
+	.flags = PMBUS_SKIP_STATUS_CHECK,
+};
+
 static int ibm_cffps_probe(struct i2c_client *client,
 			   const struct i2c_device_id *id)
 {
-	return pmbus_do_probe(client, id, &ibm_cffps_info);
+	int i, rc;
+	struct dentry *debugfs;
+	struct dentry *ibm_cffps_dir;
+	struct ibm_cffps *psu;
+
+	client->dev.platform_data = &ibm_cffps_pdata;
+	rc = pmbus_do_probe(client, id, &ibm_cffps_info);
+	if (rc)
+		return rc;
+
+	/*
+	 * Don't fail the probe if there isn't enough memory for leds and
+	 * debugfs.
+	 */
+	psu = devm_kzalloc(&client->dev, sizeof(*psu), GFP_KERNEL);
+	if (!psu)
+		return 0;
+
+	psu->client = client;
+	mutex_init(&psu->input_history.update_lock);
+	psu->input_history.last_update = jiffies - HZ;
+
+	ibm_cffps_create_led_class(psu);
+
+	/* Don't fail the probe if we can't create debugfs */
+	debugfs = pmbus_get_debugfs_dir(client);
+	if (!debugfs)
+		return 0;
+
+	ibm_cffps_dir = debugfs_create_dir(client->name, debugfs);
+	if (!ibm_cffps_dir)
+		return 0;
+
+	for (i = 0; i < CFFPS_DEBUGFS_NUM_ENTRIES; ++i)
+		psu->debugfs_entries[i] = i;
+
+	debugfs_create_file("input_history", 0444, ibm_cffps_dir,
+			    &psu->debugfs_entries[CFFPS_DEBUGFS_INPUT_HISTORY],
+			    &ibm_cffps_fops);
+	debugfs_create_file("fru", 0444, ibm_cffps_dir,
+			    &psu->debugfs_entries[CFFPS_DEBUGFS_FRU],
+			    &ibm_cffps_fops);
+	debugfs_create_file("part_number", 0444, ibm_cffps_dir,
+			    &psu->debugfs_entries[CFFPS_DEBUGFS_PN],
+			    &ibm_cffps_fops);
+	debugfs_create_file("serial_number", 0444, ibm_cffps_dir,
+			    &psu->debugfs_entries[CFFPS_DEBUGFS_SN],
+			    &ibm_cffps_fops);
+	debugfs_create_file("ccin", 0444, ibm_cffps_dir,
+			    &psu->debugfs_entries[CFFPS_DEBUGFS_CCIN],
+			    &ibm_cffps_fops);
+	debugfs_create_file("fw_version", 0444, ibm_cffps_dir,
+			    &psu->debugfs_entries[CFFPS_DEBUGFS_FW],
+			    &ibm_cffps_fops);
+
+	return 0;
 }
 
 static const struct i2c_device_id ibm_cffps_id[] = {
diff --git a/drivers/hwmon/pmbus/ir35221.c b/drivers/hwmon/pmbus/ir35221.c
index 8b906b44484b..977315b0fd90 100644
--- a/drivers/hwmon/pmbus/ir35221.c
+++ b/drivers/hwmon/pmbus/ir35221.c
@@ -25,168 +25,19 @@
 #define IR35221_MFR_IOUT_VALLEY		0xcb
 #define IR35221_MFR_TEMP_VALLEY		0xcc
 
-static long ir35221_reg2data(int data, enum pmbus_sensor_classes class)
-{
-	s16 exponent;
-	s32 mantissa;
-	long val;
-
-	/* We only modify LINEAR11 formats */
-	exponent = ((s16)data) >> 11;
-	mantissa = ((s16)((data & 0x7ff) << 5)) >> 5;
-
-	val = mantissa * 1000L;
-
-	/* scale result to micro-units for power sensors */
-	if (class == PSC_POWER)
-		val = val * 1000L;
-
-	if (exponent >= 0)
-		val <<= exponent;
-	else
-		val >>= -exponent;
-
-	return val;
-}
-
-#define MAX_MANTISSA	(1023 * 1000)
-#define MIN_MANTISSA	(511 * 1000)
-
-static u16 ir35221_data2reg(long val, enum pmbus_sensor_classes class)
-{
-	s16 exponent = 0, mantissa;
-	bool negative = false;
-
-	if (val == 0)
-		return 0;
-
-	if (val < 0) {
-		negative = true;
-		val = -val;
-	}
-
-	/* Power is in uW. Convert to mW before converting. */
-	if (class == PSC_POWER)
-		val = DIV_ROUND_CLOSEST(val, 1000L);
-
-	/* Reduce large mantissa until it fits into 10 bit */
-	while (val >= MAX_MANTISSA && exponent < 15) {
-		exponent++;
-		val >>= 1;
-	}
-	/* Increase small mantissa to improve precision */
-	while (val < MIN_MANTISSA && exponent > -15) {
-		exponent--;
-		val <<= 1;
-	}
-
-	/* Convert mantissa from milli-units to units */
-	mantissa = DIV_ROUND_CLOSEST(val, 1000);
-
-	/* Ensure that resulting number is within range */
-	if (mantissa > 0x3ff)
-		mantissa = 0x3ff;
-
-	/* restore sign */
-	if (negative)
-		mantissa = -mantissa;
-
-	/* Convert to 5 bit exponent, 11 bit mantissa */
-	return (mantissa & 0x7ff) | ((exponent << 11) & 0xf800);
-}
-
-static u16 ir35221_scale_result(s16 data, int shift,
-				enum pmbus_sensor_classes class)
-{
-	long val;
-
-	val = ir35221_reg2data(data, class);
-
-	if (shift < 0)
-		val >>= -shift;
-	else
-		val <<= shift;
-
-	return ir35221_data2reg(val, class);
-}
-
 static int ir35221_read_word_data(struct i2c_client *client, int page, int reg)
 {
 	int ret;
 
 	switch (reg) {
-	case PMBUS_IOUT_OC_FAULT_LIMIT:
-	case PMBUS_IOUT_OC_WARN_LIMIT:
-		ret = pmbus_read_word_data(client, page, reg);
-		if (ret < 0)
-			break;
-		ret = ir35221_scale_result(ret, 1, PSC_CURRENT_OUT);
-		break;
-	case PMBUS_VIN_OV_FAULT_LIMIT:
-	case PMBUS_VIN_OV_WARN_LIMIT:
-	case PMBUS_VIN_UV_WARN_LIMIT:
-		ret = pmbus_read_word_data(client, page, reg);
-		ret = ir35221_scale_result(ret, -4, PSC_VOLTAGE_IN);
-		break;
-	case PMBUS_IIN_OC_WARN_LIMIT:
-		ret = pmbus_read_word_data(client, page, reg);
-		if (ret < 0)
-			break;
-		ret = ir35221_scale_result(ret, -1, PSC_CURRENT_IN);
-		break;
-	case PMBUS_READ_VIN:
-		ret = pmbus_read_word_data(client, page, PMBUS_READ_VIN);
-		if (ret < 0)
-			break;
-		ret = ir35221_scale_result(ret, -5, PSC_VOLTAGE_IN);
-		break;
-	case PMBUS_READ_IIN:
-		ret = pmbus_read_word_data(client, page, PMBUS_READ_IIN);
-		if (ret < 0)
-			break;
-		if (page == 0)
-			ret = ir35221_scale_result(ret, -4, PSC_CURRENT_IN);
-		else
-			ret = ir35221_scale_result(ret, -5, PSC_CURRENT_IN);
-		break;
-	case PMBUS_READ_POUT:
-		ret = pmbus_read_word_data(client, page, PMBUS_READ_POUT);
-		if (ret < 0)
-			break;
-		ret = ir35221_scale_result(ret, -1, PSC_POWER);
-		break;
-	case PMBUS_READ_PIN:
-		ret = pmbus_read_word_data(client, page, PMBUS_READ_PIN);
-		if (ret < 0)
-			break;
-		ret = ir35221_scale_result(ret, -1, PSC_POWER);
-		break;
-	case PMBUS_READ_IOUT:
-		ret = pmbus_read_word_data(client, page, PMBUS_READ_IOUT);
-		if (ret < 0)
-			break;
-		if (page == 0)
-			ret = ir35221_scale_result(ret, -1, PSC_CURRENT_OUT);
-		else
-			ret = ir35221_scale_result(ret, -2, PSC_CURRENT_OUT);
-		break;
 	case PMBUS_VIRT_READ_VIN_MAX:
 		ret = pmbus_read_word_data(client, page, IR35221_MFR_VIN_PEAK);
-		if (ret < 0)
-			break;
-		ret = ir35221_scale_result(ret, -5, PSC_VOLTAGE_IN);
 		break;
 	case PMBUS_VIRT_READ_VOUT_MAX:
 		ret = pmbus_read_word_data(client, page, IR35221_MFR_VOUT_PEAK);
 		break;
 	case PMBUS_VIRT_READ_IOUT_MAX:
 		ret = pmbus_read_word_data(client, page, IR35221_MFR_IOUT_PEAK);
-		if (ret < 0)
-			break;
-		if (page == 0)
-			ret = ir35221_scale_result(ret, -1, PSC_CURRENT_IN);
-		else
-			ret = ir35221_scale_result(ret, -2, PSC_CURRENT_IN);
 		break;
 	case PMBUS_VIRT_READ_TEMP_MAX:
 		ret = pmbus_read_word_data(client, page, IR35221_MFR_TEMP_PEAK);
@@ -194,9 +45,6 @@ static int ir35221_read_word_data(struct i2c_client *client, int page, int reg)
 	case PMBUS_VIRT_READ_VIN_MIN:
 		ret = pmbus_read_word_data(client, page,
 					   IR35221_MFR_VIN_VALLEY);
-		if (ret < 0)
-			break;
-		ret = ir35221_scale_result(ret, -5, PSC_VOLTAGE_IN);
 		break;
 	case PMBUS_VIRT_READ_VOUT_MIN:
 		ret = pmbus_read_word_data(client, page,
@@ -205,12 +53,6 @@ static int ir35221_read_word_data(struct i2c_client *client, int page, int reg)
 	case PMBUS_VIRT_READ_IOUT_MIN:
 		ret = pmbus_read_word_data(client, page,
 					   IR35221_MFR_IOUT_VALLEY);
-		if (ret < 0)
-			break;
-		if (page == 0)
-			ret = ir35221_scale_result(ret, -1, PSC_CURRENT_IN);
-		else
-			ret = ir35221_scale_result(ret, -2, PSC_CURRENT_IN);
 		break;
 	case PMBUS_VIRT_READ_TEMP_MIN:
 		ret = pmbus_read_word_data(client, page,
@@ -224,36 +66,6 @@ static int ir35221_read_word_data(struct i2c_client *client, int page, int reg)
 	return ret;
 }
 
-static int ir35221_write_word_data(struct i2c_client *client, int page, int reg,
-				   u16 word)
-{
-	int ret;
-	u16 val;
-
-	switch (reg) {
-	case PMBUS_IOUT_OC_FAULT_LIMIT:
-	case PMBUS_IOUT_OC_WARN_LIMIT:
-		val = ir35221_scale_result(word, -1, PSC_CURRENT_OUT);
-		ret = pmbus_write_word_data(client, page, reg, val);
-		break;
-	case PMBUS_VIN_OV_FAULT_LIMIT:
-	case PMBUS_VIN_OV_WARN_LIMIT:
-	case PMBUS_VIN_UV_WARN_LIMIT:
-		val = ir35221_scale_result(word, 4, PSC_VOLTAGE_IN);
-		ret = pmbus_write_word_data(client, page, reg, val);
-		break;
-	case PMBUS_IIN_OC_WARN_LIMIT:
-		val = ir35221_scale_result(word, 1, PSC_CURRENT_IN);
-		ret = pmbus_write_word_data(client, page, reg, val);
-		break;
-	default:
-		ret = -ENODATA;
-		break;
-	}
-
-	return ret;
-}
-
 static int ir35221_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
@@ -292,7 +104,6 @@ static int ir35221_probe(struct i2c_client *client,
 	if (!info)
 		return -ENOMEM;
 
-	info->write_word_data = ir35221_write_word_data;
 	info->read_word_data = ir35221_read_word_data;
 
 	info->pages = 2;
diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c
index 10d17fb8f283..53db78753a0d 100644
--- a/drivers/hwmon/pmbus/lm25066.c
+++ b/drivers/hwmon/pmbus/lm25066.c
@@ -1,5 +1,5 @@
 /*
- * Hardware monitoring driver for LM25056 / LM25063 / LM25066 / LM5064 / LM5066
+ * Hardware monitoring driver for LM25056 / LM25066 / LM5064 / LM5066
  *
  * Copyright (c) 2011 Ericsson AB.
  * Copyright (c) 2013 Guenter Roeck
@@ -28,7 +28,7 @@
 #include <linux/i2c.h>
 #include "pmbus.h"
 
-enum chips { lm25056, lm25063, lm25066, lm5064, lm5066, lm5066i };
+enum chips { lm25056, lm25066, lm5064, lm5066, lm5066i };
 
 #define LM25066_READ_VAUX		0xd0
 #define LM25066_MFR_READ_IIN		0xd1
@@ -53,11 +53,6 @@ enum chips { lm25056, lm25063, lm25066, lm5064, lm5066, lm5066i };
 #define LM25056_MFR_STS_VAUX_OV_WARN	BIT(1)
 #define LM25056_MFR_STS_VAUX_UV_WARN	BIT(0)
 
-/* LM25063 only */
-
-#define LM25063_READ_VOUT_MAX		0xe5
-#define LM25063_READ_VOUT_MIN		0xe6
-
 struct __coeff {
 	short m, b, R;
 };
@@ -122,36 +117,6 @@ static struct __coeff lm25066_coeff[6][PSC_NUM_CLASSES + 2] = {
 			.m = 16,
 		},
 	},
-	[lm25063] = {
-		[PSC_VOLTAGE_IN] = {
-			.m = 16000,
-			.R = -2,
-		},
-		[PSC_VOLTAGE_OUT] = {
-			.m = 16000,
-			.R = -2,
-		},
-		[PSC_CURRENT_IN] = {
-			.m = 10000,
-			.R = -2,
-		},
-		[PSC_CURRENT_IN_L] = {
-			.m = 10000,
-			.R = -2,
-		},
-		[PSC_POWER] = {
-			.m = 5000,
-			.R = -3,
-		},
-		[PSC_POWER_L] = {
-			.m = 5000,
-			.R = -3,
-		},
-		[PSC_TEMPERATURE] = {
-			.m = 15596,
-			.R = -3,
-		},
-	},
 	[lm5064] = {
 		[PSC_VOLTAGE_IN] = {
 			.m = 4611,
@@ -272,10 +237,6 @@ static int lm25066_read_word_data(struct i2c_client *client, int page, int reg)
 			/* VIN: 6.14 mV VAUX: 293 uV LSB */
 			ret = DIV_ROUND_CLOSEST(ret * 293, 6140);
 			break;
-		case lm25063:
-			/* VIN: 6.25 mV VAUX: 200.0 uV LSB */
-			ret = DIV_ROUND_CLOSEST(ret * 20, 625);
-			break;
 		case lm25066:
 			/* VIN: 4.54 mV VAUX: 283.2 uV LSB */
 			ret = DIV_ROUND_CLOSEST(ret * 2832, 45400);
@@ -330,24 +291,6 @@ static int lm25066_read_word_data(struct i2c_client *client, int page, int reg)
 	return ret;
 }
 
-static int lm25063_read_word_data(struct i2c_client *client, int page, int reg)
-{
-	int ret;
-
-	switch (reg) {
-	case PMBUS_VIRT_READ_VOUT_MAX:
-		ret = pmbus_read_word_data(client, 0, LM25063_READ_VOUT_MAX);
-		break;
-	case PMBUS_VIRT_READ_VOUT_MIN:
-		ret = pmbus_read_word_data(client, 0, LM25063_READ_VOUT_MIN);
-		break;
-	default:
-		ret = lm25066_read_word_data(client, page, reg);
-		break;
-	}
-	return ret;
-}
-
 static int lm25056_read_word_data(struct i2c_client *client, int page, int reg)
 {
 	int ret;
@@ -502,11 +445,6 @@ static int lm25066_probe(struct i2c_client *client,
 		info->read_word_data = lm25056_read_word_data;
 		info->read_byte_data = lm25056_read_byte_data;
 		data->rlimit = 0x0fff;
-	} else if (data->id == lm25063) {
-		info->func[0] |= PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT
-		  | PMBUS_HAVE_POUT;
-		info->read_word_data = lm25063_read_word_data;
-		data->rlimit = 0xffff;
 	} else {
 		info->func[0] |= PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT;
 		info->read_word_data = lm25066_read_word_data;
@@ -543,7 +481,6 @@ static int lm25066_probe(struct i2c_client *client,
 
 static const struct i2c_device_id lm25066_id[] = {
 	{"lm25056", lm25056},
-	{"lm25063", lm25063},
 	{"lm25066", lm25066},
 	{"lm5064", lm5064},
 	{"lm5066", lm5066},
diff --git a/drivers/hwmon/pmbus/max31785.c b/drivers/hwmon/pmbus/max31785.c
index 9313849d5160..c9dc8799b5e1 100644
--- a/drivers/hwmon/pmbus/max31785.c
+++ b/drivers/hwmon/pmbus/max31785.c
@@ -16,12 +16,231 @@
 
 enum max31785_regs {
 	MFR_REVISION		= 0x9b,
+	MFR_FAN_CONFIG		= 0xf1,
 };
 
+#define MAX31785			0x3030
+#define MAX31785A			0x3040
+
+#define MFR_FAN_CONFIG_DUAL_TACH	BIT(12)
+
 #define MAX31785_NR_PAGES		23
+#define MAX31785_NR_FAN_PAGES		6
+
+static int max31785_read_byte_data(struct i2c_client *client, int page,
+				   int reg)
+{
+	if (page < MAX31785_NR_PAGES)
+		return -ENODATA;
+
+	switch (reg) {
+	case PMBUS_VOUT_MODE:
+		return -ENOTSUPP;
+	case PMBUS_FAN_CONFIG_12:
+		return pmbus_read_byte_data(client, page - MAX31785_NR_PAGES,
+					    reg);
+	}
+
+	return -ENODATA;
+}
+
+static int max31785_write_byte(struct i2c_client *client, int page, u8 value)
+{
+	if (page < MAX31785_NR_PAGES)
+		return -ENODATA;
+
+	return -ENOTSUPP;
+}
+
+static int max31785_read_long_data(struct i2c_client *client, int page,
+				   int reg, u32 *data)
+{
+	unsigned char cmdbuf[1];
+	unsigned char rspbuf[4];
+	int rc;
+
+	struct i2c_msg msg[2] = {
+		{
+			.addr = client->addr,
+			.flags = 0,
+			.len = sizeof(cmdbuf),
+			.buf = cmdbuf,
+		},
+		{
+			.addr = client->addr,
+			.flags = I2C_M_RD,
+			.len = sizeof(rspbuf),
+			.buf = rspbuf,
+		},
+	};
+
+	cmdbuf[0] = reg;
+
+	rc = pmbus_set_page(client, page);
+	if (rc < 0)
+		return rc;
+
+	rc = i2c_transfer(client->adapter, msg, ARRAY_SIZE(msg));
+	if (rc < 0)
+		return rc;
+
+	*data = (rspbuf[0] << (0 * 8)) | (rspbuf[1] << (1 * 8)) |
+		(rspbuf[2] << (2 * 8)) | (rspbuf[3] << (3 * 8));
+
+	return rc;
+}
+
+static int max31785_get_pwm(struct i2c_client *client, int page)
+{
+	int rv;
+
+	rv = pmbus_get_fan_rate_device(client, page, 0, percent);
+	if (rv < 0)
+		return rv;
+	else if (rv >= 0x8000)
+		return 0;
+	else if (rv >= 0x2711)
+		return 0x2710;
+
+	return rv;
+}
+
+static int max31785_get_pwm_mode(struct i2c_client *client, int page)
+{
+	int config;
+	int command;
+
+	config = pmbus_read_byte_data(client, page, PMBUS_FAN_CONFIG_12);
+	if (config < 0)
+		return config;
+
+	command = pmbus_read_word_data(client, page, PMBUS_FAN_COMMAND_1);
+	if (command < 0)
+		return command;
+
+	if (config & PB_FAN_1_RPM)
+		return (command >= 0x8000) ? 3 : 2;
+
+	if (command >= 0x8000)
+		return 3;
+	else if (command >= 0x2711)
+		return 0;
+
+	return 1;
+}
+
+static int max31785_read_word_data(struct i2c_client *client, int page,
+				   int reg)
+{
+	u32 val;
+	int rv;
+
+	switch (reg) {
+	case PMBUS_READ_FAN_SPEED_1:
+		if (page < MAX31785_NR_PAGES)
+			return -ENODATA;
+
+		rv = max31785_read_long_data(client, page - MAX31785_NR_PAGES,
+					     reg, &val);
+		if (rv < 0)
+			return rv;
+
+		rv = (val >> 16) & 0xffff;
+		break;
+	case PMBUS_FAN_COMMAND_1:
+		/*
+		 * PMBUS_FAN_COMMAND_x is probed to judge whether or not to
+		 * expose fan control registers.
+		 *
+		 * Don't expose fan_target attribute for virtual pages.
+		 */
+		rv = (page >= MAX31785_NR_PAGES) ? -ENOTSUPP : -ENODATA;
+		break;
+	case PMBUS_VIRT_PWM_1:
+		rv = max31785_get_pwm(client, page);
+		break;
+	case PMBUS_VIRT_PWM_ENABLE_1:
+		rv = max31785_get_pwm_mode(client, page);
+		break;
+	default:
+		rv = -ENODATA;
+		break;
+	}
+
+	return rv;
+}
+
+static inline u32 max31785_scale_pwm(u32 sensor_val)
+{
+	/*
+	 * The datasheet describes the accepted value range for manual PWM as
+	 * [0, 0x2710], while the hwmon pwmX sysfs interface accepts values in
+	 * [0, 255]. The MAX31785 uses DIRECT mode to scale the FAN_COMMAND
+	 * registers and in PWM mode the coefficients are m=1, b=0, R=2. The
+	 * important observation here is that 0x2710 == 10000 == 100 * 100.
+	 *
+	 * R=2 (== 10^2 == 100) accounts for scaling the value provided at the
+	 * sysfs interface into the required hardware resolution, but it does
+	 * not yet yield a value that we can write to the device (this initial
+	 * scaling is handled by pmbus_data2reg()). Multiplying by 100 below
+	 * translates the parameter value into the percentage units required by
+	 * PMBus, and then we scale back by 255 as required by the hwmon pwmX
+	 * interface to yield the percentage value at the appropriate
+	 * resolution for hardware.
+	 */
+	return (sensor_val * 100) / 255;
+}
+
+static int max31785_pwm_enable(struct i2c_client *client, int page,
+				    u16 word)
+{
+	int config = 0;
+	int rate;
+
+	switch (word) {
+	case 0:
+		rate = 0x7fff;
+		break;
+	case 1:
+		rate = pmbus_get_fan_rate_cached(client, page, 0, percent);
+		if (rate < 0)
+			return rate;
+		rate = max31785_scale_pwm(rate);
+		break;
+	case 2:
+		config = PB_FAN_1_RPM;
+		rate = pmbus_get_fan_rate_cached(client, page, 0, rpm);
+		if (rate < 0)
+			return rate;
+		break;
+	case 3:
+		rate = 0xffff;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return pmbus_update_fan(client, page, 0, config, PB_FAN_1_RPM, rate);
+}
+
+static int max31785_write_word_data(struct i2c_client *client, int page,
+				    int reg, u16 word)
+{
+	switch (reg) {
+	case PMBUS_VIRT_PWM_1:
+		return pmbus_update_fan(client, page, 0, 0, PB_FAN_1_RPM,
+					max31785_scale_pwm(word));
+	case PMBUS_VIRT_PWM_ENABLE_1:
+		return max31785_pwm_enable(client, page, word);
+	default:
+		break;
+	}
+
+	return -ENODATA;
+}
 
 #define MAX31785_FAN_FUNCS \
-	(PMBUS_HAVE_FAN12 | PMBUS_HAVE_STATUS_FAN12)
+	(PMBUS_HAVE_FAN12 | PMBUS_HAVE_STATUS_FAN12 | PMBUS_HAVE_PWM12)
 
 #define MAX31785_TEMP_FUNCS \
 	(PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP)
@@ -29,14 +248,26 @@ enum max31785_regs {
 #define MAX31785_VOUT_FUNCS \
 	(PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT)
 
+#define MAX37185_NUM_FAN_PAGES 6
+
 static const struct pmbus_driver_info max31785_info = {
 	.pages = MAX31785_NR_PAGES,
 
+	.write_word_data = max31785_write_word_data,
+	.read_byte_data = max31785_read_byte_data,
+	.read_word_data = max31785_read_word_data,
+	.write_byte = max31785_write_byte,
+
 	/* RPM */
 	.format[PSC_FAN] = direct,
 	.m[PSC_FAN] = 1,
 	.b[PSC_FAN] = 0,
 	.R[PSC_FAN] = 0,
+	/* PWM */
+	.format[PSC_PWM] = direct,
+	.m[PSC_PWM] = 1,
+	.b[PSC_PWM] = 0,
+	.R[PSC_PWM] = 2,
 	.func[0] = MAX31785_FAN_FUNCS,
 	.func[1] = MAX31785_FAN_FUNCS,
 	.func[2] = MAX31785_FAN_FUNCS,
@@ -72,13 +303,46 @@ static const struct pmbus_driver_info max31785_info = {
 	.func[22] = MAX31785_VOUT_FUNCS,
 };
 
+static int max31785_configure_dual_tach(struct i2c_client *client,
+					struct pmbus_driver_info *info)
+{
+	int ret;
+	int i;
+
+	for (i = 0; i < MAX31785_NR_FAN_PAGES; i++) {
+		ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, i);
+		if (ret < 0)
+			return ret;
+
+		ret = i2c_smbus_read_word_data(client, MFR_FAN_CONFIG);
+		if (ret < 0)
+			return ret;
+
+		if (ret & MFR_FAN_CONFIG_DUAL_TACH) {
+			int virtual = MAX31785_NR_PAGES + i;
+
+			info->pages = virtual + 1;
+			info->func[virtual] |= PMBUS_HAVE_FAN12;
+			info->func[virtual] |= PMBUS_PAGE_VIRTUAL;
+		}
+	}
+
+	return 0;
+}
+
 static int max31785_probe(struct i2c_client *client,
 			  const struct i2c_device_id *id)
 {
 	struct device *dev = &client->dev;
 	struct pmbus_driver_info *info;
+	bool dual_tach = false;
 	s64 ret;
 
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_BYTE_DATA |
+				     I2C_FUNC_SMBUS_WORD_DATA))
+		return -ENODEV;
+
 	info = devm_kzalloc(dev, sizeof(struct pmbus_driver_info), GFP_KERNEL);
 	if (!info)
 		return -ENOMEM;
@@ -89,6 +353,25 @@ static int max31785_probe(struct i2c_client *client,
 	if (ret < 0)
 		return ret;
 
+	ret = i2c_smbus_read_word_data(client, MFR_REVISION);
+	if (ret < 0)
+		return ret;
+
+	if (ret == MAX31785A) {
+		dual_tach = true;
+	} else if (ret == MAX31785) {
+		if (!strcmp("max31785a", id->name))
+			dev_warn(dev, "Expected max3175a, found max31785: cannot provide secondary tachometer readings\n");
+	} else {
+		return -ENODEV;
+	}
+
+	if (dual_tach) {
+		ret = max31785_configure_dual_tach(client, info);
+		if (ret < 0)
+			return ret;
+	}
+
 	return pmbus_do_probe(client, id, info);
 }
 
@@ -100,9 +383,18 @@ static const struct i2c_device_id max31785_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, max31785_id);
 
+static const struct of_device_id max31785_of_match[] = {
+	{ .compatible = "maxim,max31785" },
+	{ .compatible = "maxim,max31785a" },
+	{ },
+};
+
+MODULE_DEVICE_TABLE(of, max31785_of_match);
+
 static struct i2c_driver max31785_driver = {
 	.driver = {
 		.name = "max31785",
+		.of_match_table = max31785_of_match,
 	},
 	.probe = max31785_probe,
 	.remove = pmbus_do_remove,
diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h
index fa613bd209e3..1d24397d36ec 100644
--- a/drivers/hwmon/pmbus/pmbus.h
+++ b/drivers/hwmon/pmbus/pmbus.h
@@ -190,6 +190,33 @@ enum pmbus_regs {
 	PMBUS_VIRT_VMON_UV_FAULT_LIMIT,
 	PMBUS_VIRT_VMON_OV_FAULT_LIMIT,
 	PMBUS_VIRT_STATUS_VMON,
+
+	/*
+	 * RPM and PWM Fan control
+	 *
+	 * Drivers wanting to expose PWM control must define the behaviour of
+	 * PMBUS_VIRT_PWM_[1-4] and PMBUS_VIRT_PWM_ENABLE_[1-4] in the
+	 * {read,write}_word_data callback.
+	 *
+	 * pmbus core provides a default implementation for
+	 * PMBUS_VIRT_FAN_TARGET_[1-4].
+	 *
+	 * TARGET, PWM and PWM_ENABLE members must be defined sequentially;
+	 * pmbus core uses the difference between the provided register and
+	 * it's _1 counterpart to calculate the FAN/PWM ID.
+	 */
+	PMBUS_VIRT_FAN_TARGET_1,
+	PMBUS_VIRT_FAN_TARGET_2,
+	PMBUS_VIRT_FAN_TARGET_3,
+	PMBUS_VIRT_FAN_TARGET_4,
+	PMBUS_VIRT_PWM_1,
+	PMBUS_VIRT_PWM_2,
+	PMBUS_VIRT_PWM_3,
+	PMBUS_VIRT_PWM_4,
+	PMBUS_VIRT_PWM_ENABLE_1,
+	PMBUS_VIRT_PWM_ENABLE_2,
+	PMBUS_VIRT_PWM_ENABLE_3,
+	PMBUS_VIRT_PWM_ENABLE_4,
 };
 
 /*
@@ -223,6 +250,8 @@ enum pmbus_regs {
 #define PB_FAN_1_RPM			BIT(6)
 #define PB_FAN_1_INSTALLED		BIT(7)
 
+enum pmbus_fan_mode { percent = 0, rpm };
+
 /*
  * STATUS_BYTE, STATUS_WORD (lower)
  */
@@ -313,6 +342,7 @@ enum pmbus_sensor_classes {
 	PSC_POWER,
 	PSC_TEMPERATURE,
 	PSC_FAN,
+	PSC_PWM,
 	PSC_NUM_CLASSES		/* Number of power sensor classes */
 };
 
@@ -339,6 +369,10 @@ enum pmbus_sensor_classes {
 #define PMBUS_HAVE_STATUS_FAN34	BIT(17)
 #define PMBUS_HAVE_VMON		BIT(18)
 #define PMBUS_HAVE_STATUS_VMON	BIT(19)
+#define PMBUS_HAVE_PWM12	BIT(20)
+#define PMBUS_HAVE_PWM34	BIT(21)
+
+#define PMBUS_PAGE_VIRTUAL	BIT(31)
 
 enum pmbus_data_format { linear = 0, direct, vid };
 enum vrm_version { vr11 = 0, vr12, vr13 };
@@ -421,5 +455,12 @@ int pmbus_do_probe(struct i2c_client *client, const struct i2c_device_id *id,
 int pmbus_do_remove(struct i2c_client *client);
 const struct pmbus_driver_info *pmbus_get_driver_info(struct i2c_client
 						      *client);
+int pmbus_get_fan_rate_device(struct i2c_client *client, int page, int id,
+			      enum pmbus_fan_mode mode);
+int pmbus_get_fan_rate_cached(struct i2c_client *client, int page, int id,
+			      enum pmbus_fan_mode mode);
+int pmbus_update_fan(struct i2c_client *client, int page, int id,
+		     u8 config, u8 mask, u16 command);
+struct dentry *pmbus_get_debugfs_dir(struct i2c_client *client);
 
 #endif /* PMBUS_H */
diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
index a139940cd991..f7c47d7994e7 100644
--- a/drivers/hwmon/pmbus/pmbus_core.c
+++ b/drivers/hwmon/pmbus/pmbus_core.c
@@ -65,6 +65,7 @@ struct pmbus_sensor {
 	u16 reg;		/* register */
 	enum pmbus_sensor_classes class;	/* sensor class */
 	bool update;		/* runtime sensor update needed */
+	bool convert;		/* Whether or not to apply linear/vid/direct */
 	int data;		/* Sensor data.
 				   Negative if there was a read error */
 };
@@ -129,6 +130,27 @@ struct pmbus_debugfs_entry {
 	u8 reg;
 };
 
+static const int pmbus_fan_rpm_mask[] = {
+	PB_FAN_1_RPM,
+	PB_FAN_2_RPM,
+	PB_FAN_1_RPM,
+	PB_FAN_2_RPM,
+};
+
+static const int pmbus_fan_config_registers[] = {
+	PMBUS_FAN_CONFIG_12,
+	PMBUS_FAN_CONFIG_12,
+	PMBUS_FAN_CONFIG_34,
+	PMBUS_FAN_CONFIG_34
+};
+
+static const int pmbus_fan_command_registers[] = {
+	PMBUS_FAN_COMMAND_1,
+	PMBUS_FAN_COMMAND_2,
+	PMBUS_FAN_COMMAND_3,
+	PMBUS_FAN_COMMAND_4,
+};
+
 void pmbus_clear_cache(struct i2c_client *client)
 {
 	struct pmbus_data *data = i2c_get_clientdata(client);
@@ -140,18 +162,27 @@ EXPORT_SYMBOL_GPL(pmbus_clear_cache);
 int pmbus_set_page(struct i2c_client *client, int page)
 {
 	struct pmbus_data *data = i2c_get_clientdata(client);
-	int rv = 0;
-	int newpage;
+	int rv;
 
-	if (page >= 0 && page != data->currpage) {
+	if (page < 0 || page == data->currpage)
+		return 0;
+
+	if (!(data->info->func[page] & PMBUS_PAGE_VIRTUAL)) {
 		rv = i2c_smbus_write_byte_data(client, PMBUS_PAGE, page);
-		newpage = i2c_smbus_read_byte_data(client, PMBUS_PAGE);
-		if (newpage != page)
-			rv = -EIO;
-		else
-			data->currpage = page;
+		if (rv < 0)
+			return rv;
+
+		rv = i2c_smbus_read_byte_data(client, PMBUS_PAGE);
+		if (rv < 0)
+			return rv;
+
+		if (rv != page)
+			return -EIO;
 	}
-	return rv;
+
+	data->currpage = page;
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pmbus_set_page);
 
@@ -198,6 +229,28 @@ int pmbus_write_word_data(struct i2c_client *client, int page, u8 reg,
 }
 EXPORT_SYMBOL_GPL(pmbus_write_word_data);
 
+
+static int pmbus_write_virt_reg(struct i2c_client *client, int page, int reg,
+				u16 word)
+{
+	int bit;
+	int id;
+	int rv;
+
+	switch (reg) {
+	case PMBUS_VIRT_FAN_TARGET_1 ... PMBUS_VIRT_FAN_TARGET_4:
+		id = reg - PMBUS_VIRT_FAN_TARGET_1;
+		bit = pmbus_fan_rpm_mask[id];
+		rv = pmbus_update_fan(client, page, id, bit, bit, word);
+		break;
+	default:
+		rv = -ENXIO;
+		break;
+	}
+
+	return rv;
+}
+
 /*
  * _pmbus_write_word_data() is similar to pmbus_write_word_data(), but checks if
  * a device specific mapping function exists and calls it if necessary.
@@ -214,11 +267,38 @@ static int _pmbus_write_word_data(struct i2c_client *client, int page, int reg,
 		if (status != -ENODATA)
 			return status;
 	}
+
 	if (reg >= PMBUS_VIRT_BASE)
-		return -ENXIO;
+		return pmbus_write_virt_reg(client, page, reg, word);
+
 	return pmbus_write_word_data(client, page, reg, word);
 }
 
+int pmbus_update_fan(struct i2c_client *client, int page, int id,
+		     u8 config, u8 mask, u16 command)
+{
+	int from;
+	int rv;
+	u8 to;
+
+	from = pmbus_read_byte_data(client, page,
+				    pmbus_fan_config_registers[id]);
+	if (from < 0)
+		return from;
+
+	to = (from & ~mask) | (config & mask);
+	if (to != from) {
+		rv = pmbus_write_byte_data(client, page,
+					   pmbus_fan_config_registers[id], to);
+		if (rv < 0)
+			return rv;
+	}
+
+	return _pmbus_write_word_data(client, page,
+				      pmbus_fan_command_registers[id], command);
+}
+EXPORT_SYMBOL_GPL(pmbus_update_fan);
+
 int pmbus_read_word_data(struct i2c_client *client, int page, u8 reg)
 {
 	int rv;
@@ -231,6 +311,24 @@ int pmbus_read_word_data(struct i2c_client *client, int page, u8 reg)
 }
 EXPORT_SYMBOL_GPL(pmbus_read_word_data);
 
+static int pmbus_read_virt_reg(struct i2c_client *client, int page, int reg)
+{
+	int rv;
+	int id;
+
+	switch (reg) {
+	case PMBUS_VIRT_FAN_TARGET_1 ... PMBUS_VIRT_FAN_TARGET_4:
+		id = reg - PMBUS_VIRT_FAN_TARGET_1;
+		rv = pmbus_get_fan_rate_device(client, page, id, rpm);
+		break;
+	default:
+		rv = -ENXIO;
+		break;
+	}
+
+	return rv;
+}
+
 /*
  * _pmbus_read_word_data() is similar to pmbus_read_word_data(), but checks if
  * a device specific mapping function exists and calls it if necessary.
@@ -246,8 +344,10 @@ static int _pmbus_read_word_data(struct i2c_client *client, int page, int reg)
 		if (status != -ENODATA)
 			return status;
 	}
+
 	if (reg >= PMBUS_VIRT_BASE)
-		return -ENXIO;
+		return pmbus_read_virt_reg(client, page, reg);
+
 	return pmbus_read_word_data(client, page, reg);
 }
 
@@ -312,6 +412,68 @@ static int _pmbus_read_byte_data(struct i2c_client *client, int page, int reg)
 	return pmbus_read_byte_data(client, page, reg);
 }
 
+static struct pmbus_sensor *pmbus_find_sensor(struct pmbus_data *data, int page,
+					      int reg)
+{
+	struct pmbus_sensor *sensor;
+
+	for (sensor = data->sensors; sensor; sensor = sensor->next) {
+		if (sensor->page == page && sensor->reg == reg)
+			return sensor;
+	}
+
+	return ERR_PTR(-EINVAL);
+}
+
+static int pmbus_get_fan_rate(struct i2c_client *client, int page, int id,
+			      enum pmbus_fan_mode mode,
+			      bool from_cache)
+{
+	struct pmbus_data *data = i2c_get_clientdata(client);
+	bool want_rpm, have_rpm;
+	struct pmbus_sensor *s;
+	int config;
+	int reg;
+
+	want_rpm = (mode == rpm);
+
+	if (from_cache) {
+		reg = want_rpm ? PMBUS_VIRT_FAN_TARGET_1 : PMBUS_VIRT_PWM_1;
+		s = pmbus_find_sensor(data, page, reg + id);
+		if (IS_ERR(s))
+			return PTR_ERR(s);
+
+		return s->data;
+	}
+
+	config = pmbus_read_byte_data(client, page,
+				      pmbus_fan_config_registers[id]);
+	if (config < 0)
+		return config;
+
+	have_rpm = !!(config & pmbus_fan_rpm_mask[id]);
+	if (want_rpm == have_rpm)
+		return pmbus_read_word_data(client, page,
+					    pmbus_fan_command_registers[id]);
+
+	/* Can't sensibly map between RPM and PWM, just return zero */
+	return 0;
+}
+
+int pmbus_get_fan_rate_device(struct i2c_client *client, int page, int id,
+			      enum pmbus_fan_mode mode)
+{
+	return pmbus_get_fan_rate(client, page, id, mode, false);
+}
+EXPORT_SYMBOL_GPL(pmbus_get_fan_rate_device);
+
+int pmbus_get_fan_rate_cached(struct i2c_client *client, int page, int id,
+			      enum pmbus_fan_mode mode)
+{
+	return pmbus_get_fan_rate(client, page, id, mode, true);
+}
+EXPORT_SYMBOL_GPL(pmbus_get_fan_rate_cached);
+
 static void pmbus_clear_fault_page(struct i2c_client *client, int page)
 {
 	_pmbus_write_byte(client, page, PMBUS_CLEAR_FAULTS);
@@ -513,7 +675,7 @@ static long pmbus_reg2data_direct(struct pmbus_data *data,
 	/* X = 1/m * (Y * 10^-R - b) */
 	R = -R;
 	/* scale result to milli-units for everything but fans */
-	if (sensor->class != PSC_FAN) {
+	if (!(sensor->class == PSC_FAN || sensor->class == PSC_PWM)) {
 		R += 3;
 		b *= 1000;
 	}
@@ -568,6 +730,9 @@ static long pmbus_reg2data(struct pmbus_data *data, struct pmbus_sensor *sensor)
 {
 	long val;
 
+	if (!sensor->convert)
+		return sensor->data;
+
 	switch (data->info->format[sensor->class]) {
 	case direct:
 		val = pmbus_reg2data_direct(data, sensor);
@@ -672,7 +837,7 @@ static u16 pmbus_data2reg_direct(struct pmbus_data *data,
 	}
 
 	/* Calculate Y = (m * X + b) * 10^R */
-	if (sensor->class != PSC_FAN) {
+	if (!(sensor->class == PSC_FAN || sensor->class == PSC_PWM)) {
 		R -= 3;		/* Adjust R and b for data in milli-units */
 		b *= 1000;
 	}
@@ -703,6 +868,9 @@ static u16 pmbus_data2reg(struct pmbus_data *data,
 {
 	u16 regval;
 
+	if (!sensor->convert)
+		return val;
+
 	switch (data->info->format[sensor->class]) {
 	case direct:
 		regval = pmbus_data2reg_direct(data, sensor, val);
@@ -915,7 +1083,8 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data,
 					     const char *name, const char *type,
 					     int seq, int page, int reg,
 					     enum pmbus_sensor_classes class,
-					     bool update, bool readonly)
+					     bool update, bool readonly,
+					     bool convert)
 {
 	struct pmbus_sensor *sensor;
 	struct device_attribute *a;
@@ -925,12 +1094,18 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data,
 		return NULL;
 	a = &sensor->attribute;
 
-	snprintf(sensor->name, sizeof(sensor->name), "%s%d_%s",
-		 name, seq, type);
+	if (type)
+		snprintf(sensor->name, sizeof(sensor->name), "%s%d_%s",
+			 name, seq, type);
+	else
+		snprintf(sensor->name, sizeof(sensor->name), "%s%d",
+			 name, seq);
+
 	sensor->page = page;
 	sensor->reg = reg;
 	sensor->class = class;
 	sensor->update = update;
+	sensor->convert = convert;
 	pmbus_dev_attr_init(a, sensor->name,
 			    readonly ? S_IRUGO : S_IRUGO | S_IWUSR,
 			    pmbus_show_sensor, pmbus_set_sensor);
@@ -1029,7 +1204,7 @@ static int pmbus_add_limit_attrs(struct i2c_client *client,
 			curr = pmbus_add_sensor(data, name, l->attr, index,
 						page, l->reg, attr->class,
 						attr->update || l->update,
-						false);
+						false, true);
 			if (!curr)
 				return -ENOMEM;
 			if (l->sbit && (info->func[page] & attr->sfunc)) {
@@ -1068,7 +1243,7 @@ static int pmbus_add_sensor_attrs_one(struct i2c_client *client,
 			return ret;
 	}
 	base = pmbus_add_sensor(data, name, "input", index, page, attr->reg,
-				attr->class, true, true);
+				attr->class, true, true, true);
 	if (!base)
 		return -ENOMEM;
 	if (attr->sfunc) {
@@ -1592,13 +1767,6 @@ static const int pmbus_fan_registers[] = {
 	PMBUS_READ_FAN_SPEED_4
 };
 
-static const int pmbus_fan_config_registers[] = {
-	PMBUS_FAN_CONFIG_12,
-	PMBUS_FAN_CONFIG_12,
-	PMBUS_FAN_CONFIG_34,
-	PMBUS_FAN_CONFIG_34
-};
-
 static const int pmbus_fan_status_registers[] = {
 	PMBUS_STATUS_FAN_12,
 	PMBUS_STATUS_FAN_12,
@@ -1621,6 +1789,42 @@ static const u32 pmbus_fan_status_flags[] = {
 };
 
 /* Fans */
+
+/* Precondition: FAN_CONFIG_x_y and FAN_COMMAND_x must exist for the fan ID */
+static int pmbus_add_fan_ctrl(struct i2c_client *client,
+		struct pmbus_data *data, int index, int page, int id,
+		u8 config)
+{
+	struct pmbus_sensor *sensor;
+
+	sensor = pmbus_add_sensor(data, "fan", "target", index, page,
+				  PMBUS_VIRT_FAN_TARGET_1 + id, PSC_FAN,
+				  false, false, true);
+
+	if (!sensor)
+		return -ENOMEM;
+
+	if (!((data->info->func[page] & PMBUS_HAVE_PWM12) ||
+			(data->info->func[page] & PMBUS_HAVE_PWM34)))
+		return 0;
+
+	sensor = pmbus_add_sensor(data, "pwm", NULL, index, page,
+				  PMBUS_VIRT_PWM_1 + id, PSC_PWM,
+				  false, false, true);
+
+	if (!sensor)
+		return -ENOMEM;
+
+	sensor = pmbus_add_sensor(data, "pwm", "enable", index, page,
+				  PMBUS_VIRT_PWM_ENABLE_1 + id, PSC_PWM,
+				  true, false, false);
+
+	if (!sensor)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static int pmbus_add_fan_attributes(struct i2c_client *client,
 				    struct pmbus_data *data)
 {
@@ -1655,9 +1859,18 @@ static int pmbus_add_fan_attributes(struct i2c_client *client,
 
 			if (pmbus_add_sensor(data, "fan", "input", index,
 					     page, pmbus_fan_registers[f],
-					     PSC_FAN, true, true) == NULL)
+					     PSC_FAN, true, true, true) == NULL)
 				return -ENOMEM;
 
+			/* Fan control */
+			if (pmbus_check_word_register(client, page,
+					pmbus_fan_command_registers[f])) {
+				ret = pmbus_add_fan_ctrl(client, data, index,
+							 page, f, regval);
+				if (ret < 0)
+					return ret;
+			}
+
 			/*
 			 * Each fan status register covers multiple fans,
 			 * so we have to do some magic.
@@ -2168,6 +2381,14 @@ int pmbus_do_remove(struct i2c_client *client)
 }
 EXPORT_SYMBOL_GPL(pmbus_do_remove);
 
+struct dentry *pmbus_get_debugfs_dir(struct i2c_client *client)
+{
+	struct pmbus_data *data = i2c_get_clientdata(client);
+
+	return data->debugfs;
+}
+EXPORT_SYMBOL_GPL(pmbus_get_debugfs_dir);
+
 static int __init pmbus_core_init(void)
 {
 	pmbus_debugfs_dir = debugfs_create_dir("pmbus", NULL);
diff --git a/drivers/hwmon/sht15.c b/drivers/hwmon/sht15.c
index 25d28343ba93..2be77752cd56 100644
--- a/drivers/hwmon/sht15.c
+++ b/drivers/hwmon/sht15.c
@@ -179,6 +179,7 @@ struct sht15_data {
  * sht15_crc8() - compute crc8
  * @data:	sht15 specific data.
  * @value:	sht15 retrieved data.
+ * @len:	Length of retrieved data
  *
  * This implements section 2 of the CRC datasheet.
  */
diff --git a/drivers/hwmon/sht21.c b/drivers/hwmon/sht21.c
index 06706d288355..190e7b39ce32 100644
--- a/drivers/hwmon/sht21.c
+++ b/drivers/hwmon/sht21.c
@@ -41,7 +41,7 @@
 
 /**
  * struct sht21 - SHT21 device specific data
- * @hwmon_dev: device registered with hwmon
+ * @client: I2C client device
  * @lock: mutex to protect measurement values
  * @last_update: time of last update (jiffies)
  * @temperature: cached temperature measurement value
diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c
index 6ea99cd6ae79..370b57dafab7 100644
--- a/drivers/hwmon/sht3x.c
+++ b/drivers/hwmon/sht3x.c
@@ -732,6 +732,13 @@ static int sht3x_probe(struct i2c_client *client,
 	mutex_init(&data->i2c_lock);
 	mutex_init(&data->data_lock);
 
+	/*
+	 * An attempt to read limits register too early
+	 * causes a NACK response from the chip.
+	 * Waiting for an empirical delay of 500 us solves the issue.
+	 */
+	usleep_range(500, 600);
+
 	ret = limits_update(data);
 	if (ret)
 		return ret;
diff --git a/drivers/hwmon/w83773g.c b/drivers/hwmon/w83773g.c
new file mode 100644
index 000000000000..e858093ac806
--- /dev/null
+++ b/drivers/hwmon/w83773g.c
@@ -0,0 +1,329 @@
+/*
+ * Copyright (C) 2017 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Driver for the Nuvoton W83773G SMBus temperature sensor IC.
+ * Supported models: W83773G
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/i2c.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/err.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+
+/* W83773 has 3 channels */
+#define W83773_CHANNELS				3
+
+/* The W83773 registers */
+#define W83773_CONVERSION_RATE_REG_READ		0x04
+#define W83773_CONVERSION_RATE_REG_WRITE	0x0A
+#define W83773_MANUFACTURER_ID_REG		0xFE
+#define W83773_LOCAL_TEMP			0x00
+
+static const u8 W83773_STATUS[2] = { 0x02, 0x17 };
+
+static const u8 W83773_TEMP_LSB[2] = { 0x10, 0x25 };
+static const u8 W83773_TEMP_MSB[2] = { 0x01, 0x24 };
+
+static const u8 W83773_OFFSET_LSB[2] = { 0x12, 0x16 };
+static const u8 W83773_OFFSET_MSB[2] = { 0x11, 0x15 };
+
+/* this is the number of sensors in the device */
+static const struct i2c_device_id w83773_id[] = {
+	{ "w83773g" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(i2c, w83773_id);
+
+static const struct of_device_id w83773_of_match[] = {
+	{
+		.compatible = "nuvoton,w83773g"
+	},
+	{ },
+};
+MODULE_DEVICE_TABLE(of, w83773_of_match);
+
+static inline long temp_of_local(s8 reg)
+{
+	return reg * 1000;
+}
+
+static inline long temp_of_remote(s8 hb, u8 lb)
+{
+	return (hb << 3 | lb >> 5) * 125;
+}
+
+static int get_local_temp(struct regmap *regmap, long *val)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_read(regmap, W83773_LOCAL_TEMP, &regval);
+	if (ret < 0)
+		return ret;
+
+	*val = temp_of_local(regval);
+	return 0;
+}
+
+static int get_remote_temp(struct regmap *regmap, int index, long *val)
+{
+	unsigned int regval_high;
+	unsigned int regval_low;
+	int ret;
+
+	ret = regmap_read(regmap, W83773_TEMP_MSB[index], &regval_high);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, W83773_TEMP_LSB[index], &regval_low);
+	if (ret < 0)
+		return ret;
+
+	*val = temp_of_remote(regval_high, regval_low);
+	return 0;
+}
+
+static int get_fault(struct regmap *regmap, int index, long *val)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_read(regmap, W83773_STATUS[index], &regval);
+	if (ret < 0)
+		return ret;
+
+	*val = (regval & 0x04) >> 2;
+	return 0;
+}
+
+static int get_offset(struct regmap *regmap, int index, long *val)
+{
+	unsigned int regval_high;
+	unsigned int regval_low;
+	int ret;
+
+	ret = regmap_read(regmap, W83773_OFFSET_MSB[index], &regval_high);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_read(regmap, W83773_OFFSET_LSB[index], &regval_low);
+	if (ret < 0)
+		return ret;
+
+	*val = temp_of_remote(regval_high, regval_low);
+	return 0;
+}
+
+static int set_offset(struct regmap *regmap, int index, long val)
+{
+	int ret;
+	u8 high_byte;
+	u8 low_byte;
+
+	val = clamp_val(val, -127825, 127825);
+	/* offset value equals to (high_byte << 3 | low_byte >> 5) * 125 */
+	val /= 125;
+	high_byte = val >> 3;
+	low_byte = (val & 0x07) << 5;
+
+	ret = regmap_write(regmap, W83773_OFFSET_MSB[index], high_byte);
+	if (ret < 0)
+		return ret;
+
+	return regmap_write(regmap, W83773_OFFSET_LSB[index], low_byte);
+}
+
+static int get_update_interval(struct regmap *regmap, long *val)
+{
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_read(regmap, W83773_CONVERSION_RATE_REG_READ, &regval);
+	if (ret < 0)
+		return ret;
+
+	*val = 16000 >> regval;
+	return 0;
+}
+
+static int set_update_interval(struct regmap *regmap, long val)
+{
+	int rate;
+
+	/*
+	 * For valid rates, interval can be calculated as
+	 *	interval = (1 << (8 - rate)) * 62.5;
+	 * Rounded rate is therefore
+	 *	rate = 8 - __fls(interval * 8 / (62.5 * 7));
+	 * Use clamp_val() to avoid overflows, and to ensure valid input
+	 * for __fls.
+	 */
+	val = clamp_val(val, 62, 16000) * 10;
+	rate = 8 - __fls((val * 8 / (625 * 7)));
+	return regmap_write(regmap, W83773_CONVERSION_RATE_REG_WRITE, rate);
+}
+
+static int w83773_read(struct device *dev, enum hwmon_sensor_types type,
+		       u32 attr, int channel, long *val)
+{
+	struct regmap *regmap = dev_get_drvdata(dev);
+
+	if (type == hwmon_chip) {
+		if (attr == hwmon_chip_update_interval)
+			return get_update_interval(regmap, val);
+		return -EOPNOTSUPP;
+	}
+
+	switch (attr) {
+	case hwmon_temp_input:
+		if (channel == 0)
+			return get_local_temp(regmap, val);
+		return get_remote_temp(regmap, channel - 1, val);
+	case hwmon_temp_fault:
+		return get_fault(regmap, channel - 1, val);
+	case hwmon_temp_offset:
+		return get_offset(regmap, channel - 1, val);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int w83773_write(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long val)
+{
+	struct regmap *regmap = dev_get_drvdata(dev);
+
+	if (type == hwmon_chip && attr == hwmon_chip_update_interval)
+		return set_update_interval(regmap, val);
+
+	if (type == hwmon_temp && attr == hwmon_temp_offset)
+		return set_offset(regmap, channel - 1, val);
+
+	return -EOPNOTSUPP;
+}
+
+static umode_t w83773_is_visible(const void *data, enum hwmon_sensor_types type,
+				 u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_chip:
+		switch (attr) {
+		case hwmon_chip_update_interval:
+			return 0644;
+		}
+		break;
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+		case hwmon_temp_fault:
+			return 0444;
+		case hwmon_temp_offset:
+			return 0644;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static const u32 w83773_chip_config[] = {
+	HWMON_C_REGISTER_TZ | HWMON_C_UPDATE_INTERVAL,
+	0
+};
+
+static const struct hwmon_channel_info w83773_chip = {
+	.type = hwmon_chip,
+	.config = w83773_chip_config,
+};
+
+static const u32 w83773_temp_config[] = {
+	HWMON_T_INPUT,
+	HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_OFFSET,
+	HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_OFFSET,
+	0
+};
+
+static const struct hwmon_channel_info w83773_temp = {
+	.type = hwmon_temp,
+	.config = w83773_temp_config,
+};
+
+static const struct hwmon_channel_info *w83773_info[] = {
+	&w83773_chip,
+	&w83773_temp,
+	NULL
+};
+
+static const struct hwmon_ops w83773_ops = {
+	.is_visible = w83773_is_visible,
+	.read = w83773_read,
+	.write = w83773_write,
+};
+
+static const struct hwmon_chip_info w83773_chip_info = {
+	.ops = &w83773_ops,
+	.info = w83773_info,
+};
+
+static const struct regmap_config w83773_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+};
+
+static int w83773_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct device *dev = &client->dev;
+	struct device *hwmon_dev;
+	struct regmap *regmap;
+	int ret;
+
+	regmap = devm_regmap_init_i2c(client, &w83773_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "failed to allocate register map\n");
+		return PTR_ERR(regmap);
+	}
+
+	/* Set the conversion rate to 2 Hz */
+	ret = regmap_write(regmap, W83773_CONVERSION_RATE_REG_WRITE, 0x05);
+	if (ret < 0) {
+		dev_err(&client->dev, "error writing config rate register\n");
+		return ret;
+	}
+
+	i2c_set_clientdata(client, regmap);
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev,
+							 client->name,
+							 regmap,
+							 &w83773_chip_info,
+							 NULL);
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static struct i2c_driver w83773_driver = {
+	.class = I2C_CLASS_HWMON,
+	.driver = {
+		.name	= "w83773g",
+		.of_match_table = of_match_ptr(w83773_of_match),
+	},
+	.probe = w83773_probe,
+	.id_table = w83773_id,
+};
+
+module_i2c_driver(w83773_driver);
+
+MODULE_AUTHOR("Lei YU <mine260309@gmail.com>");
+MODULE_DESCRIPTION("W83773G temperature sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwtracing/coresight/coresight-dynamic-replicator.c b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
index 8f4357e2626c..043da86b0fe9 100644
--- a/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
+++ b/drivers/hwtracing/coresight/coresight-dynamic-replicator.c
@@ -163,10 +163,8 @@ static int replicator_probe(struct amba_device *adev, const struct amba_id *id)
 	desc.dev = &adev->dev;
 	desc.groups = replicator_groups;
 	drvdata->csdev = coresight_register(&desc);
-	if (IS_ERR(drvdata->csdev))
-		return PTR_ERR(drvdata->csdev);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(drvdata->csdev);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c
index e03e58933141..580cd381adf3 100644
--- a/drivers/hwtracing/coresight/coresight-etb10.c
+++ b/drivers/hwtracing/coresight/coresight-etb10.c
@@ -33,7 +33,6 @@
 #include <linux/mm.h>
 #include <linux/perf_event.h>
 
-#include <asm/local.h>
 
 #include "coresight-priv.h"
 
diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c
index fd3c396717f6..9f8ac0bef853 100644
--- a/drivers/hwtracing/coresight/coresight-funnel.c
+++ b/drivers/hwtracing/coresight/coresight-funnel.c
@@ -214,10 +214,8 @@ static int funnel_probe(struct amba_device *adev, const struct amba_id *id)
 	desc.dev = dev;
 	desc.groups = coresight_funnel_groups;
 	drvdata->csdev = coresight_register(&desc);
-	if (IS_ERR(drvdata->csdev))
-		return PTR_ERR(drvdata->csdev);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(drvdata->csdev);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c
index bef49a3a5ca7..805f7c2210fe 100644
--- a/drivers/hwtracing/coresight/coresight-tpiu.c
+++ b/drivers/hwtracing/coresight/coresight-tpiu.c
@@ -46,8 +46,11 @@
 #define TPIU_ITATBCTR0		0xef8
 
 /** register definition **/
+/* FFSR - 0x300 */
+#define FFSR_FT_STOPPED		BIT(1)
 /* FFCR - 0x304 */
 #define FFCR_FON_MAN		BIT(6)
+#define FFCR_STOP_FI		BIT(12)
 
 /**
  * @base:	memory mapped base address for this component.
@@ -85,10 +88,14 @@ static void tpiu_disable_hw(struct tpiu_drvdata *drvdata)
 {
 	CS_UNLOCK(drvdata->base);
 
-	/* Clear formatter controle reg. */
-	writel_relaxed(0x0, drvdata->base + TPIU_FFCR);
+	/* Clear formatter and stop on flush */
+	writel_relaxed(FFCR_STOP_FI, drvdata->base + TPIU_FFCR);
 	/* Generate manual flush */
-	writel_relaxed(FFCR_FON_MAN, drvdata->base + TPIU_FFCR);
+	writel_relaxed(FFCR_STOP_FI | FFCR_FON_MAN, drvdata->base + TPIU_FFCR);
+	/* Wait for flush to complete */
+	coresight_timeout(drvdata->base, TPIU_FFCR, FFCR_FON_MAN, 0);
+	/* Wait for formatter to stop */
+	coresight_timeout(drvdata->base, TPIU_FFSR, FFSR_FT_STOPPED, 1);
 
 	CS_LOCK(drvdata->base);
 }
@@ -160,10 +167,8 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id)
 	desc.pdata = pdata;
 	desc.dev = dev;
 	drvdata->csdev = coresight_register(&desc);
-	if (IS_ERR(drvdata->csdev))
-		return PTR_ERR(drvdata->csdev);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(drvdata->csdev);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/hwtracing/coresight/coresight.c b/drivers/hwtracing/coresight/coresight.c
index b8091bef21dc..389c4baeca9d 100644
--- a/drivers/hwtracing/coresight/coresight.c
+++ b/drivers/hwtracing/coresight/coresight.c
@@ -843,32 +843,17 @@ static void coresight_fixup_orphan_conns(struct coresight_device *csdev)
 }
 
 
-static int coresight_name_match(struct device *dev, void *data)
-{
-	char *to_match;
-	struct coresight_device *i_csdev;
-
-	to_match = data;
-	i_csdev = to_coresight_device(dev);
-
-	if (to_match && !strcmp(to_match, dev_name(&i_csdev->dev)))
-		return 1;
-
-	return 0;
-}
-
 static void coresight_fixup_device_conns(struct coresight_device *csdev)
 {
 	int i;
-	struct device *dev = NULL;
-	struct coresight_connection *conn;
 
 	for (i = 0; i < csdev->nr_outport; i++) {
-		conn = &csdev->conns[i];
-		dev = bus_find_device(&coresight_bustype, NULL,
-				      (void *)conn->child_name,
-				      coresight_name_match);
+		struct coresight_connection *conn = &csdev->conns[i];
+		struct device *dev = NULL;
 
+		if (conn->child_name)
+			dev = bus_find_device_by_name(&coresight_bustype, NULL,
+						      conn->child_name);
 		if (dev) {
 			conn->child_dev = to_coresight_device(dev);
 			/* and put reference from 'bus_find_device()' */
diff --git a/drivers/hwtracing/coresight/of_coresight.c b/drivers/hwtracing/coresight/of_coresight.c
index a18794128bf8..7c375443ede6 100644
--- a/drivers/hwtracing/coresight/of_coresight.c
+++ b/drivers/hwtracing/coresight/of_coresight.c
@@ -104,26 +104,17 @@ static int of_coresight_alloc_memory(struct device *dev,
 int of_coresight_get_cpu(const struct device_node *node)
 {
 	int cpu;
-	bool found;
-	struct device_node *dn, *np;
+	struct device_node *dn;
 
 	dn = of_parse_phandle(node, "cpu", 0);
-
 	/* Affinity defaults to CPU0 */
 	if (!dn)
 		return 0;
-
-	for_each_possible_cpu(cpu) {
-		np = of_cpu_device_node_get(cpu);
-		found = (dn == np);
-		of_node_put(np);
-		if (found)
-			break;
-	}
+	cpu = of_cpu_node_to_id(dn);
 	of_node_put(dn);
 
 	/* Affinity to CPU0 if no cpu nodes are found */
-	return found ? cpu : 0;
+	return (cpu < 0) ? 0 : cpu;
 }
 EXPORT_SYMBOL_GPL(of_coresight_get_cpu);
 
diff --git a/drivers/i2c/algos/i2c-algo-bit.c b/drivers/i2c/algos/i2c-algo-bit.c
index 1147bddb8b2c..3df0efd69ae3 100644
--- a/drivers/i2c/algos/i2c-algo-bit.c
+++ b/drivers/i2c/algos/i2c-algo-bit.c
@@ -649,6 +649,11 @@ static int __i2c_bit_add_bus(struct i2c_adapter *adap,
 	if (bit_adap->getscl == NULL)
 		adap->quirks = &i2c_bit_quirk_no_clk_stretch;
 
+	/* Bring bus to a known state. Looks like STOP if bus is not free yet */
+	setscl(bit_adap, 1);
+	udelay(bit_adap->udelay);
+	setsda(bit_adap, 1);
+
 	ret = add_adapter(adap);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 009345d8f49d..e2954fb86d65 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -123,8 +123,10 @@ config I2C_I801
 	    Wildcat Point (PCH)
 	    Wildcat Point-LP (PCH)
 	    BayTrail (SOC)
+	    Braswell (SOC)
 	    Sunrise Point-H (PCH)
 	    Sunrise Point-LP (PCH)
+	    Kaby Lake-H (PCH)
 	    DNV (SOC)
 	    Broxton (SOC)
 	    Lewisburg (PCH)
@@ -603,6 +605,14 @@ config I2C_GPIO
 	  This is a very simple bitbanging I2C driver utilizing the
 	  arch-neutral GPIO API to control the SCL and SDA lines.
 
+config I2C_GPIO_FAULT_INJECTOR
+	bool "GPIO-based fault injector"
+	depends on I2C_GPIO
+	help
+	  This adds some functionality to the i2c-gpio driver which can inject
+	  faults to an I2C bus, so another bus master can be stress-tested.
+	  This is for debugging. If unsure, say 'no'.
+
 config I2C_HIGHLANDER
 	tristate "Highlander FPGA SMBus interface"
 	depends on SH_HIGHLANDER
diff --git a/drivers/i2c/busses/i2c-acorn.c b/drivers/i2c/busses/i2c-acorn.c
index 9d7be5af2bf2..f4a5ae69bf6a 100644
--- a/drivers/i2c/busses/i2c-acorn.c
+++ b/drivers/i2c/busses/i2c-acorn.c
@@ -1,5 +1,5 @@
 /*
- *  linux/drivers/acorn/char/i2c.c
+ *  ARM IOC/IOMD i2c driver.
  *
  *  Copyright (C) 2000 Russell King
  *
@@ -7,8 +7,6 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
- *  ARM IOC/IOMD i2c driver.
- *
  *  On Acorn machines, the following i2c devices are on the bus:
  *	- PCF8583 real time clock & static RAM
  */
@@ -94,3 +92,7 @@ static int __init i2c_ioc_init(void)
 }
 
 module_init(i2c_ioc_init);
+
+MODULE_AUTHOR("Russell King <linux@armlinux.org.uk>");
+MODULE_DESCRIPTION("ARM IOC/IOMD i2c driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/i2c/busses/i2c-bcm2835.c b/drivers/i2c/busses/i2c-bcm2835.c
index cd07a69e2e93..44deae78913e 100644
--- a/drivers/i2c/busses/i2c-bcm2835.c
+++ b/drivers/i2c/busses/i2c-bcm2835.c
@@ -50,6 +50,9 @@
 #define BCM2835_I2C_S_CLKT	BIT(9)
 #define BCM2835_I2C_S_LEN	BIT(10) /* Fake bit for SW error reporting */
 
+#define BCM2835_I2C_FEDL_SHIFT	16
+#define BCM2835_I2C_REDL_SHIFT	0
+
 #define BCM2835_I2C_CDIV_MIN	0x0002
 #define BCM2835_I2C_CDIV_MAX	0xFFFE
 
@@ -81,7 +84,7 @@ static inline u32 bcm2835_i2c_readl(struct bcm2835_i2c_dev *i2c_dev, u32 reg)
 
 static int bcm2835_i2c_set_divider(struct bcm2835_i2c_dev *i2c_dev)
 {
-	u32 divider;
+	u32 divider, redl, fedl;
 
 	divider = DIV_ROUND_UP(clk_get_rate(i2c_dev->clk),
 			       i2c_dev->bus_clk_rate);
@@ -100,6 +103,22 @@ static int bcm2835_i2c_set_divider(struct bcm2835_i2c_dev *i2c_dev)
 
 	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_DIV, divider);
 
+	/*
+	 * Number of core clocks to wait after falling edge before
+	 * outputting the next data bit.  Note that both FEDL and REDL
+	 * can't be greater than CDIV/2.
+	 */
+	fedl = max(divider / 16, 1u);
+
+	/*
+	 * Number of core clocks to wait after rising edge before
+	 * sampling the next incoming data bit.
+	 */
+	redl = max(divider / 4, 1u);
+
+	bcm2835_i2c_writel(i2c_dev, BCM2835_I2C_DEL,
+			   (fedl << BCM2835_I2C_FEDL_SHIFT) |
+			   (redl << BCM2835_I2C_REDL_SHIFT));
 	return 0;
 }
 
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index 2ead9b9eebb7..75d6ab177055 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -33,7 +33,7 @@
 #include <linux/io.h>
 #include <linux/slab.h>
 #include <linux/cpufreq.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/of_device.h>
 #include <linux/platform_data/i2c-davinci.h>
 #include <linux/pm_runtime.h>
@@ -139,7 +139,6 @@ struct davinci_i2c_dev {
 	u8			terminate;
 	struct i2c_adapter	adapter;
 #ifdef CONFIG_CPU_FREQ
-	struct completion	xfr_complete;
 	struct notifier_block	freq_transition;
 #endif
 	struct davinci_i2c_platform_data *pdata;
@@ -294,7 +293,7 @@ static int i2c_davinci_init(struct davinci_i2c_dev *dev)
 }
 
 /*
- * This routine does i2c bus recovery by using i2c_generic_gpio_recovery
+ * This routine does i2c bus recovery by using i2c_generic_scl_recovery
  * which is provided by I2C Bus recovery infrastructure.
  */
 static void davinci_i2c_prepare_recovery(struct i2c_adapter *adap)
@@ -316,7 +315,7 @@ static void davinci_i2c_unprepare_recovery(struct i2c_adapter *adap)
 }
 
 static struct i2c_bus_recovery_info davinci_i2c_gpio_recovery_info = {
-	.recover_bus = i2c_generic_gpio_recovery,
+	.recover_bus = i2c_generic_scl_recovery,
 	.prepare_recovery = davinci_i2c_prepare_recovery,
 	.unprepare_recovery = davinci_i2c_unprepare_recovery,
 };
@@ -567,9 +566,6 @@ i2c_davinci_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 	}
 
 	ret = num;
-#ifdef CONFIG_CPU_FREQ
-	complete(&dev->xfr_complete);
-#endif
 
 out:
 	pm_runtime_mark_last_busy(dev->dev);
@@ -717,13 +713,15 @@ static int i2c_davinci_cpufreq_transition(struct notifier_block *nb,
 	struct davinci_i2c_dev *dev;
 
 	dev = container_of(nb, struct davinci_i2c_dev, freq_transition);
+
+	i2c_lock_adapter(&dev->adapter);
 	if (val == CPUFREQ_PRECHANGE) {
-		wait_for_completion(&dev->xfr_complete);
 		davinci_i2c_reset_ctrl(dev, 0);
 	} else if (val == CPUFREQ_POSTCHANGE) {
 		i2c_davinci_calc_clk_dividers(dev);
 		davinci_i2c_reset_ctrl(dev, 1);
 	}
+	i2c_unlock_adapter(&dev->adapter);
 
 	return 0;
 }
@@ -769,6 +767,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 	struct davinci_i2c_dev *dev;
 	struct i2c_adapter *adap;
 	struct resource *mem;
+	struct i2c_bus_recovery_info *rinfo;
 	int r, irq;
 
 	irq = platform_get_irq(pdev, 0);
@@ -789,9 +788,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 	}
 
 	init_completion(&dev->cmd_complete);
-#ifdef CONFIG_CPU_FREQ
-	init_completion(&dev->xfr_complete);
-#endif
+
 	dev->dev = &pdev->dev;
 	dev->irq = irq;
 	dev->pdata = dev_get_platdata(&pdev->dev);
@@ -868,10 +865,20 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 
 	if (dev->pdata->has_pfunc)
 		adap->bus_recovery_info = &davinci_i2c_scl_recovery_info;
-	else if (dev->pdata->scl_pin) {
-		adap->bus_recovery_info = &davinci_i2c_gpio_recovery_info;
-		adap->bus_recovery_info->scl_gpio = dev->pdata->scl_pin;
-		adap->bus_recovery_info->sda_gpio = dev->pdata->sda_pin;
+	else if (dev->pdata->gpio_recovery) {
+		rinfo =  &davinci_i2c_gpio_recovery_info;
+		adap->bus_recovery_info = rinfo;
+		rinfo->scl_gpiod = devm_gpiod_get(&pdev->dev, "scl",
+						  GPIOD_OUT_HIGH_OPEN_DRAIN);
+		if (IS_ERR(rinfo->scl_gpiod)) {
+			r = PTR_ERR(rinfo->scl_gpiod);
+			goto err_unuse_clocks;
+		}
+		rinfo->sda_gpiod = devm_gpiod_get(&pdev->dev, "sda", GPIOD_IN);
+		if (IS_ERR(rinfo->sda_gpiod)) {
+			r = PTR_ERR(rinfo->sda_gpiod);
+			goto err_unuse_clocks;
+		}
 	}
 
 	adap->nr = pdev->id;
diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c
index d1a69372432f..27ebd90de43b 100644
--- a/drivers/i2c/busses/i2c-designware-common.c
+++ b/drivers/i2c/busses/i2c-designware-common.c
@@ -21,6 +21,7 @@
  * ----------------------------------------------------------------------------
  *
  */
+#include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/export.h>
 #include <linux/errno.h>
@@ -185,6 +186,19 @@ unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev)
 	return dev->get_clk_rate_khz(dev);
 }
 
+int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare)
+{
+	if (IS_ERR(dev->clk))
+		return PTR_ERR(dev->clk);
+
+	if (prepare)
+		return clk_prepare_enable(dev->clk);
+
+	clk_disable_unprepare(dev->clk);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(i2c_dw_prepare_clk);
+
 int i2c_dw_acquire_lock(struct dw_i2c_dev *dev)
 {
 	int ret;
@@ -217,7 +231,11 @@ int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev)
 	while (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) {
 		if (timeout <= 0) {
 			dev_warn(dev->dev, "timeout waiting for bus ready\n");
-			return -ETIMEDOUT;
+			i2c_recover_bus(&dev->adapter);
+
+			if (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY)
+				return -ETIMEDOUT;
+			return 0;
 		}
 		timeout--;
 		usleep_range(1000, 1100);
diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h
index 21bf619a86c5..8707c76b2fee 100644
--- a/drivers/i2c/busses/i2c-designware-core.h
+++ b/drivers/i2c/busses/i2c-designware-core.h
@@ -280,12 +280,11 @@ struct dw_i2c_dev {
 	int			(*acquire_lock)(struct dw_i2c_dev *dev);
 	void			(*release_lock)(struct dw_i2c_dev *dev);
 	bool			pm_disabled;
-	bool			suspended;
-	bool			skip_resume;
 	void			(*disable)(struct dw_i2c_dev *dev);
 	void			(*disable_int)(struct dw_i2c_dev *dev);
 	int			(*init)(struct dw_i2c_dev *dev);
 	int			mode;
+	struct i2c_bus_recovery_info rinfo;
 };
 
 #define ACCESS_SWAP		0x00000001
@@ -301,6 +300,7 @@ u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset);
 void __i2c_dw_enable(struct dw_i2c_dev *dev, bool enable);
 void __i2c_dw_enable_and_wait(struct dw_i2c_dev *dev, bool enable);
 unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev);
+int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare);
 int i2c_dw_acquire_lock(struct dw_i2c_dev *dev);
 void i2c_dw_release_lock(struct dw_i2c_dev *dev);
 int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev);
diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c
index 418c233075d3..05732531829f 100644
--- a/drivers/i2c/busses/i2c-designware-master.c
+++ b/drivers/i2c/busses/i2c-designware-master.c
@@ -25,11 +25,13 @@
 #include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/export.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/reset.h>
 
 #include "i2c-designware-core.h"
 
@@ -207,7 +209,7 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev)
 	i2c_dw_disable_int(dev);
 
 	/* Enable the adapter */
-	__i2c_dw_enable(dev, true);
+	__i2c_dw_enable_and_wait(dev, true);
 
 	/* Clear and enable interrupts */
 	dw_readl(dev, DW_IC_CLR_INTR);
@@ -443,6 +445,7 @@ i2c_dw_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 	if (!wait_for_completion_timeout(&dev->cmd_complete, adap->timeout)) {
 		dev_err(dev->dev, "controller timed out\n");
 		/* i2c_dw_init implicitly disables the adapter */
+		i2c_recover_bus(&dev->adapter);
 		i2c_dw_init_master(dev);
 		ret = -ETIMEDOUT;
 		goto done;
@@ -613,6 +616,56 @@ static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static void i2c_dw_prepare_recovery(struct i2c_adapter *adap)
+{
+	struct dw_i2c_dev *dev = i2c_get_adapdata(adap);
+
+	i2c_dw_disable(dev);
+	reset_control_assert(dev->rst);
+	i2c_dw_prepare_clk(dev, false);
+}
+
+static void i2c_dw_unprepare_recovery(struct i2c_adapter *adap)
+{
+	struct dw_i2c_dev *dev = i2c_get_adapdata(adap);
+
+	i2c_dw_prepare_clk(dev, true);
+	reset_control_deassert(dev->rst);
+	i2c_dw_init_master(dev);
+}
+
+static int i2c_dw_init_recovery_info(struct dw_i2c_dev *dev)
+{
+	struct i2c_bus_recovery_info *rinfo = &dev->rinfo;
+	struct i2c_adapter *adap = &dev->adapter;
+	struct gpio_desc *gpio;
+	int r;
+
+	gpio = devm_gpiod_get(dev->dev, "scl", GPIOD_OUT_HIGH);
+	if (IS_ERR(gpio)) {
+		r = PTR_ERR(gpio);
+		if (r == -ENOENT || r == -ENOSYS)
+			return 0;
+		return r;
+	}
+	rinfo->scl_gpiod = gpio;
+
+	gpio = devm_gpiod_get_optional(dev->dev, "sda", GPIOD_IN);
+	if (IS_ERR(gpio))
+		return PTR_ERR(gpio);
+	rinfo->sda_gpiod = gpio;
+
+	rinfo->recover_bus = i2c_generic_scl_recovery;
+	rinfo->prepare_recovery = i2c_dw_prepare_recovery;
+	rinfo->unprepare_recovery = i2c_dw_unprepare_recovery;
+	adap->bus_recovery_info = rinfo;
+
+	dev_info(dev->dev, "running with gpio recovery mode! scl%s",
+		 rinfo->sda_gpiod ? ",sda" : "");
+
+	return 0;
+}
+
 int i2c_dw_probe(struct dw_i2c_dev *dev)
 {
 	struct i2c_adapter *adap = &dev->adapter;
@@ -652,6 +705,10 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 		return ret;
 	}
 
+	ret = i2c_dw_init_recovery_info(dev);
+	if (ret)
+		return ret;
+
 	/*
 	 * Increment PM usage count during adapter registration in order to
 	 * avoid possible spurious runtime suspend when adapter device is
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index 58add69a441c..5660daf6c92e 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -42,6 +42,7 @@
 #include <linux/reset.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
+#include <linux/suspend.h>
 
 #include "i2c-designware-core.h"
 
@@ -201,29 +202,6 @@ static void i2c_dw_configure_slave(struct dw_i2c_dev *dev)
 			 DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED;
 
 	dev->mode = DW_IC_SLAVE;
-
-	switch (dev->clk_freq) {
-	case 100000:
-		dev->slave_cfg |= DW_IC_CON_SPEED_STD;
-		break;
-	case 3400000:
-		dev->slave_cfg |= DW_IC_CON_SPEED_HIGH;
-		break;
-	default:
-		dev->slave_cfg |= DW_IC_CON_SPEED_FAST;
-	}
-}
-
-static int i2c_dw_plat_prepare_clk(struct dw_i2c_dev *i_dev, bool prepare)
-{
-	if (IS_ERR(i_dev->clk))
-		return PTR_ERR(i_dev->clk);
-
-	if (prepare)
-		return clk_prepare_enable(i_dev->clk);
-
-	clk_disable_unprepare(i_dev->clk);
-	return 0;
 }
 
 static void dw_i2c_set_fifo_size(struct dw_i2c_dev *dev, int id)
@@ -355,7 +333,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 		i2c_dw_configure_master(dev);
 
 	dev->clk = devm_clk_get(&pdev->dev, NULL);
-	if (!i2c_dw_plat_prepare_clk(dev, true)) {
+	if (!i2c_dw_prepare_clk(dev, true)) {
 		dev->get_clk_rate_khz = i2c_dw_get_clk_rate_khz;
 
 		if (!dev->sda_hold_time && ht)
@@ -372,6 +350,11 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	ACPI_COMPANION_SET(&adap->dev, ACPI_COMPANION(&pdev->dev));
 	adap->dev.of_node = pdev->dev.of_node;
 
+	dev_pm_set_driver_flags(&pdev->dev,
+				DPM_FLAG_SMART_PREPARE |
+				DPM_FLAG_SMART_SUSPEND |
+				DPM_FLAG_LEAVE_SUSPENDED);
+
 	/* The code below assumes runtime PM to be disabled. */
 	WARN_ON(pm_runtime_enabled(&pdev->dev));
 
@@ -435,12 +418,24 @@ MODULE_DEVICE_TABLE(of, dw_i2c_of_match);
 #ifdef CONFIG_PM_SLEEP
 static int dw_i2c_plat_prepare(struct device *dev)
 {
-	return pm_runtime_suspended(dev);
+	/*
+	 * If the ACPI companion device object is present for this device, it
+	 * may be accessed during suspend and resume of other devices via I2C
+	 * operation regions, so tell the PM core and middle layers to avoid
+	 * skipping system suspend/resume callbacks for it in that case.
+	 */
+	return !has_acpi_companion(dev);
 }
 
 static void dw_i2c_plat_complete(struct device *dev)
 {
-	if (dev->power.direct_complete)
+	/*
+	 * The device can only be in runtime suspend at this point if it has not
+	 * been resumed throughout the ending system suspend/resume cycle, so if
+	 * the platform firmware might mess up with it, request the runtime PM
+	 * framework to resume it.
+	 */
+	if (pm_runtime_suspended(dev) && pm_resume_via_firmware())
 		pm_request_resume(dev);
 }
 #else
@@ -453,15 +448,8 @@ static int dw_i2c_plat_suspend(struct device *dev)
 {
 	struct dw_i2c_dev *i_dev = dev_get_drvdata(dev);
 
-	if (i_dev->suspended) {
-		i_dev->skip_resume = true;
-		return 0;
-	}
-
 	i_dev->disable(i_dev);
-	i2c_dw_plat_prepare_clk(i_dev, false);
-
-	i_dev->suspended = true;
+	i2c_dw_prepare_clk(i_dev, false);
 
 	return 0;
 }
@@ -470,19 +458,9 @@ static int dw_i2c_plat_resume(struct device *dev)
 {
 	struct dw_i2c_dev *i_dev = dev_get_drvdata(dev);
 
-	if (!i_dev->suspended)
-		return 0;
-
-	if (i_dev->skip_resume) {
-		i_dev->skip_resume = false;
-		return 0;
-	}
-
-	i2c_dw_plat_prepare_clk(i_dev, true);
+	i2c_dw_prepare_clk(i_dev, true);
 	i_dev->init(i_dev);
 
-	i_dev->suspended = false;
-
 	return 0;
 }
 
diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c
index ea9578ab19a1..d42558d1b002 100644
--- a/drivers/i2c/busses/i2c-designware-slave.c
+++ b/drivers/i2c/busses/i2c-designware-slave.c
@@ -51,9 +51,7 @@ static void i2c_dw_configure_fifo_slave(struct dw_i2c_dev *dev)
  */
 static int i2c_dw_init_slave(struct dw_i2c_dev *dev)
 {
-	u32 sda_falling_time, scl_falling_time;
 	u32 reg, comp_param1;
-	u32 hcnt, lcnt;
 	int ret;
 
 	ret = i2c_dw_acquire_lock(dev);
@@ -79,68 +77,6 @@ static int i2c_dw_init_slave(struct dw_i2c_dev *dev)
 	/* Disable the adapter. */
 	__i2c_dw_enable_and_wait(dev, false);
 
-	/* Set standard and fast speed deviders for high/low periods. */
-	sda_falling_time = dev->sda_falling_time ?: 300; /* ns */
-	scl_falling_time = dev->scl_falling_time ?: 300; /* ns */
-
-	/* Set SCL timing parameters for standard-mode. */
-	if (dev->ss_hcnt && dev->ss_lcnt) {
-		hcnt = dev->ss_hcnt;
-		lcnt = dev->ss_lcnt;
-	} else {
-		hcnt = i2c_dw_scl_hcnt(i2c_dw_clk_rate(dev),
-				       4000,	/* tHD;STA = tHIGH = 4.0 us */
-				       sda_falling_time,
-				       0,	/* 0: DW default, 1: Ideal */
-				       0);	/* No offset */
-		lcnt = i2c_dw_scl_lcnt(i2c_dw_clk_rate(dev),
-				       4700,	/* tLOW = 4.7 us */
-				       scl_falling_time,
-				       0);	/* No offset */
-	}
-	dw_writel(dev, hcnt, DW_IC_SS_SCL_HCNT);
-	dw_writel(dev, lcnt, DW_IC_SS_SCL_LCNT);
-	dev_dbg(dev->dev, "Standard-mode HCNT:LCNT = %d:%d\n", hcnt, lcnt);
-
-	/* Set SCL timing parameters for fast-mode or fast-mode plus. */
-	if ((dev->clk_freq == 1000000) && dev->fp_hcnt && dev->fp_lcnt) {
-		hcnt = dev->fp_hcnt;
-		lcnt = dev->fp_lcnt;
-	} else if (dev->fs_hcnt && dev->fs_lcnt) {
-		hcnt = dev->fs_hcnt;
-		lcnt = dev->fs_lcnt;
-	} else {
-		hcnt = i2c_dw_scl_hcnt(i2c_dw_clk_rate(dev),
-				       600,	/* tHD;STA = tHIGH = 0.6 us */
-				       sda_falling_time,
-				       0,	/* 0: DW default, 1: Ideal */
-				       0);	/* No offset */
-		lcnt = i2c_dw_scl_lcnt(i2c_dw_clk_rate(dev),
-				       1300,	/* tLOW = 1.3 us */
-				       scl_falling_time,
-				       0);	/* No offset */
-	}
-	dw_writel(dev, hcnt, DW_IC_FS_SCL_HCNT);
-	dw_writel(dev, lcnt, DW_IC_FS_SCL_LCNT);
-	dev_dbg(dev->dev, "Fast-mode HCNT:LCNT = %d:%d\n", hcnt, lcnt);
-
-	if ((dev->slave_cfg & DW_IC_CON_SPEED_MASK) ==
-		DW_IC_CON_SPEED_HIGH) {
-		if ((comp_param1 & DW_IC_COMP_PARAM_1_SPEED_MODE_MASK)
-			!= DW_IC_COMP_PARAM_1_SPEED_MODE_HIGH) {
-			dev_err(dev->dev, "High Speed not supported!\n");
-			dev->slave_cfg &= ~DW_IC_CON_SPEED_MASK;
-			dev->slave_cfg |= DW_IC_CON_SPEED_FAST;
-		} else if (dev->hs_hcnt && dev->hs_lcnt) {
-			hcnt = dev->hs_hcnt;
-			lcnt = dev->hs_lcnt;
-			dw_writel(dev, hcnt, DW_IC_HS_SCL_HCNT);
-			dw_writel(dev, lcnt, DW_IC_HS_SCL_LCNT);
-			dev_dbg(dev->dev, "HighSpeed-mode HCNT:LCNT = %d:%d\n",
-				hcnt, lcnt);
-		}
-	}
-
 	/* Configure SDA Hold Time if required. */
 	reg = dw_readl(dev, DW_IC_COMP_VERSION);
 	if (reg >= DW_IC_SDA_HOLD_MIN_VERS) {
diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c
index 3855e0b11877..b02428498f6d 100644
--- a/drivers/i2c/busses/i2c-exynos5.c
+++ b/drivers/i2c/busses/i2c-exynos5.c
@@ -170,7 +170,7 @@
 #define HSI2C_HS_TX_CLOCK	1000000
 #define HSI2C_FS_TX_CLOCK	100000
 
-#define EXYNOS5_I2C_TIMEOUT (msecs_to_jiffies(1000))
+#define EXYNOS5_I2C_TIMEOUT (msecs_to_jiffies(100))
 
 #define HSI2C_EXYNOS7	BIT(0)
 
diff --git a/drivers/i2c/busses/i2c-gpio.c b/drivers/i2c/busses/i2c-gpio.c
index d80ea6ce91bb..58abb3eced58 100644
--- a/drivers/i2c/busses/i2c-gpio.c
+++ b/drivers/i2c/busses/i2c-gpio.c
@@ -7,6 +7,8 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
+#include <linux/debugfs.h>
+#include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <linux/i2c-gpio.h>
@@ -23,6 +25,9 @@ struct i2c_gpio_private_data {
 	struct i2c_adapter adap;
 	struct i2c_algo_bit_data bit_data;
 	struct i2c_gpio_platform_data pdata;
+#ifdef CONFIG_I2C_GPIO_FAULT_INJECTOR
+	struct dentry *debug_dir;
+#endif
 };
 
 /*
@@ -34,7 +39,7 @@ static void i2c_gpio_setsda_val(void *data, int state)
 {
 	struct i2c_gpio_private_data *priv = data;
 
-	gpiod_set_value(priv->sda, state);
+	gpiod_set_value_cansleep(priv->sda, state);
 }
 
 /*
@@ -47,23 +52,125 @@ static void i2c_gpio_setscl_val(void *data, int state)
 {
 	struct i2c_gpio_private_data *priv = data;
 
-	gpiod_set_value(priv->scl, state);
+	gpiod_set_value_cansleep(priv->scl, state);
 }
 
 static int i2c_gpio_getsda(void *data)
 {
 	struct i2c_gpio_private_data *priv = data;
 
-	return gpiod_get_value(priv->sda);
+	return gpiod_get_value_cansleep(priv->sda);
 }
 
 static int i2c_gpio_getscl(void *data)
 {
 	struct i2c_gpio_private_data *priv = data;
 
-	return gpiod_get_value(priv->scl);
+	return gpiod_get_value_cansleep(priv->scl);
+}
+
+#ifdef CONFIG_I2C_GPIO_FAULT_INJECTOR
+static struct dentry *i2c_gpio_debug_dir;
+
+#define setsda(bd, val)	((bd)->setsda((bd)->data, val))
+#define setscl(bd, val)	((bd)->setscl((bd)->data, val))
+#define getsda(bd)	((bd)->getsda((bd)->data))
+#define getscl(bd)	((bd)->getscl((bd)->data))
+
+#define WIRE_ATTRIBUTE(wire) \
+static int fops_##wire##_get(void *data, u64 *val)	\
+{							\
+	struct i2c_gpio_private_data *priv = data;	\
+							\
+	i2c_lock_adapter(&priv->adap);			\
+	*val = get##wire(&priv->bit_data);		\
+	i2c_unlock_adapter(&priv->adap);		\
+	return 0;					\
+}							\
+static int fops_##wire##_set(void *data, u64 val)	\
+{							\
+	struct i2c_gpio_private_data *priv = data;	\
+							\
+	i2c_lock_adapter(&priv->adap);			\
+	set##wire(&priv->bit_data, val);		\
+	i2c_unlock_adapter(&priv->adap);		\
+	return 0;					\
+}							\
+DEFINE_DEBUGFS_ATTRIBUTE(fops_##wire, fops_##wire##_get, fops_##wire##_set, "%llu\n")
+
+WIRE_ATTRIBUTE(scl);
+WIRE_ATTRIBUTE(sda);
+
+static int fops_incomplete_transfer_set(void *data, u64 addr)
+{
+	struct i2c_gpio_private_data *priv = data;
+	struct i2c_algo_bit_data *bit_data = &priv->bit_data;
+	int i, pattern;
+
+	if (addr > 0x7f)
+		return -EINVAL;
+
+	/* ADDR (7 bit) + RD (1 bit) + SDA hi (1 bit) */
+	pattern = (addr << 2) | 3;
+
+	i2c_lock_adapter(&priv->adap);
+
+	/* START condition */
+	setsda(bit_data, 0);
+	udelay(bit_data->udelay);
+
+	/* Send ADDR+RD, request ACK, don't send STOP */
+	for (i = 8; i >= 0; i--) {
+		setscl(bit_data, 0);
+		udelay(bit_data->udelay / 2);
+		setsda(bit_data, (pattern >> i) & 1);
+		udelay((bit_data->udelay + 1) / 2);
+		setscl(bit_data, 1);
+		udelay(bit_data->udelay);
+	}
+
+	i2c_unlock_adapter(&priv->adap);
+
+	return 0;
+}
+DEFINE_DEBUGFS_ATTRIBUTE(fops_incomplete_transfer, NULL, fops_incomplete_transfer_set, "%llu\n");
+
+static void i2c_gpio_fault_injector_init(struct platform_device *pdev)
+{
+	struct i2c_gpio_private_data *priv = platform_get_drvdata(pdev);
+
+	/*
+	 * If there will be a debugfs-dir per i2c adapter somewhen, put the
+	 * 'fault-injector' dir there. Until then, we have a global dir with
+	 * all adapters as subdirs.
+	 */
+	if (!i2c_gpio_debug_dir) {
+		i2c_gpio_debug_dir = debugfs_create_dir("i2c-fault-injector", NULL);
+		if (!i2c_gpio_debug_dir)
+			return;
+	}
+
+	priv->debug_dir = debugfs_create_dir(pdev->name, i2c_gpio_debug_dir);
+	if (!priv->debug_dir)
+		return;
+
+	debugfs_create_file_unsafe("scl", 0600, priv->debug_dir, priv, &fops_scl);
+	debugfs_create_file_unsafe("sda", 0600, priv->debug_dir, priv, &fops_sda);
+	debugfs_create_file_unsafe("incomplete_transfer", 0200, priv->debug_dir,
+				   priv, &fops_incomplete_transfer);
 }
 
+static void i2c_gpio_fault_injector_exit(struct platform_device *pdev)
+{
+	struct i2c_gpio_private_data *priv = platform_get_drvdata(pdev);
+
+	debugfs_remove_recursive(priv->debug_dir);
+}
+#else
+static inline void i2c_gpio_fault_injector_init(struct platform_device *pdev) {}
+static inline void i2c_gpio_fault_injector_exit(struct platform_device *pdev) {}
+#endif /* CONFIG_I2C_GPIO_FAULT_INJECTOR*/
+
 static void of_i2c_gpio_get_props(struct device_node *np,
 				  struct i2c_gpio_platform_data *pdata)
 {
@@ -179,6 +286,9 @@ static int i2c_gpio_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->scl))
 		return PTR_ERR(priv->scl);
 
+	if (gpiod_cansleep(priv->sda) || gpiod_cansleep(priv->scl))
+		dev_warn(dev, "Slow GPIO pins might wreak havoc into I2C/SMBus bus timing");
+
 	bit_data->setsda = i2c_gpio_setsda_val;
 	bit_data->setscl = i2c_gpio_setscl_val;
 
@@ -228,6 +338,8 @@ static int i2c_gpio_probe(struct platform_device *pdev)
 		 pdata->scl_is_output_only
 		 ? ", no clock stretching" : "");
 
+	i2c_gpio_fault_injector_init(pdev);
+
 	return 0;
 }
 
@@ -236,6 +348,8 @@ static int i2c_gpio_remove(struct platform_device *pdev)
 	struct i2c_gpio_private_data *priv;
 	struct i2c_adapter *adap;
 
+	i2c_gpio_fault_injector_exit(pdev);
+
 	priv = platform_get_drvdata(pdev);
 	adap = &priv->adap;
 
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index 8eac00efadc1..692b34125866 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -58,6 +58,7 @@
  * Wildcat Point (PCH)		0x8ca2	32	hard	yes	yes	yes
  * Wildcat Point-LP (PCH)	0x9ca2	32	hard	yes	yes	yes
  * BayTrail (SOC)		0x0f12	32	hard	yes	yes	yes
+ * Braswell (SOC)		0x2292	32	hard	yes	yes	yes
  * Sunrise Point-H (PCH) 	0xa123  32	hard	yes	yes	yes
  * Sunrise Point-LP (PCH)	0x9d23	32	hard	yes	yes	yes
  * DNV (SOC)			0x19df	32	hard	yes	yes	yes
diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c
index e86801a63120..e6da2c7a9a3e 100644
--- a/drivers/i2c/busses/i2c-imx-lpi2c.c
+++ b/drivers/i2c/busses/i2c-imx-lpi2c.c
@@ -30,6 +30,7 @@
 #include <linux/of_device.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 
@@ -90,6 +91,8 @@
 #define FAST_PLUS_MAX_BITRATE	3400000
 #define HIGHSPEED_MAX_BITRATE	5000000
 
+#define I2C_PM_TIMEOUT		10 /* ms */
+
 enum lpi2c_imx_mode {
 	STANDARD,	/* 100+Kbps */
 	FAST,		/* 400+Kbps */
@@ -274,8 +277,8 @@ static int lpi2c_imx_master_enable(struct lpi2c_imx_struct *lpi2c_imx)
 	unsigned int temp;
 	int ret;
 
-	ret = clk_enable(lpi2c_imx->clk);
-	if (ret)
+	ret = pm_runtime_get_sync(lpi2c_imx->adapter.dev.parent);
+	if (ret < 0)
 		return ret;
 
 	temp = MCR_RST;
@@ -284,7 +287,7 @@ static int lpi2c_imx_master_enable(struct lpi2c_imx_struct *lpi2c_imx)
 
 	ret = lpi2c_imx_config(lpi2c_imx);
 	if (ret)
-		goto clk_disable;
+		goto rpm_put;
 
 	temp = readl(lpi2c_imx->base + LPI2C_MCR);
 	temp |= MCR_MEN;
@@ -292,8 +295,9 @@ static int lpi2c_imx_master_enable(struct lpi2c_imx_struct *lpi2c_imx)
 
 	return 0;
 
-clk_disable:
-	clk_disable(lpi2c_imx->clk);
+rpm_put:
+	pm_runtime_mark_last_busy(lpi2c_imx->adapter.dev.parent);
+	pm_runtime_put_autosuspend(lpi2c_imx->adapter.dev.parent);
 
 	return ret;
 }
@@ -306,7 +310,8 @@ static int lpi2c_imx_master_disable(struct lpi2c_imx_struct *lpi2c_imx)
 	temp &= ~MCR_MEN;
 	writel(temp, lpi2c_imx->base + LPI2C_MCR);
 
-	clk_disable(lpi2c_imx->clk);
+	pm_runtime_mark_last_busy(lpi2c_imx->adapter.dev.parent);
+	pm_runtime_put_autosuspend(lpi2c_imx->adapter.dev.parent);
 
 	return 0;
 }
@@ -606,22 +611,31 @@ static int lpi2c_imx_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	pm_runtime_set_autosuspend_delay(&pdev->dev, I2C_PM_TIMEOUT);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_get_noresume(&pdev->dev);
+	pm_runtime_set_active(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+
 	temp = readl(lpi2c_imx->base + LPI2C_PARAM);
 	lpi2c_imx->txfifosize = 1 << (temp & 0x0f);
 	lpi2c_imx->rxfifosize = 1 << ((temp >> 8) & 0x0f);
 
-	clk_disable(lpi2c_imx->clk);
-
 	ret = i2c_add_adapter(&lpi2c_imx->adapter);
 	if (ret)
-		goto clk_unprepare;
+		goto rpm_disable;
+
+	pm_runtime_mark_last_busy(&pdev->dev);
+	pm_runtime_put_autosuspend(&pdev->dev);
 
 	dev_info(&lpi2c_imx->adapter.dev, "LPI2C adapter registered\n");
 
 	return 0;
 
-clk_unprepare:
-	clk_unprepare(lpi2c_imx->clk);
+rpm_disable:
+	pm_runtime_put(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 
 	return ret;
 }
@@ -632,28 +646,48 @@ static int lpi2c_imx_remove(struct platform_device *pdev)
 
 	i2c_del_adapter(&lpi2c_imx->adapter);
 
-	clk_unprepare(lpi2c_imx->clk);
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 
 	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
-static int lpi2c_imx_suspend(struct device *dev)
+static int lpi2c_runtime_suspend(struct device *dev)
 {
+	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(lpi2c_imx->clk);
 	pinctrl_pm_select_sleep_state(dev);
 
 	return 0;
 }
 
-static int lpi2c_imx_resume(struct device *dev)
+static int lpi2c_runtime_resume(struct device *dev)
 {
+	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
+	int ret;
+
 	pinctrl_pm_select_default_state(dev);
+	ret = clk_prepare_enable(lpi2c_imx->clk);
+	if (ret) {
+		dev_err(dev, "failed to enable I2C clock, ret=%d\n", ret);
+		return ret;
+	}
 
 	return 0;
 }
-#endif
 
-static SIMPLE_DEV_PM_OPS(imx_lpi2c_pm, lpi2c_imx_suspend, lpi2c_imx_resume);
+static const struct dev_pm_ops lpi2c_pm_ops = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				      pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(lpi2c_runtime_suspend,
+			   lpi2c_runtime_resume, NULL)
+};
+#define IMX_LPI2C_PM      (&lpi2c_pm_ops)
+#else
+#define IMX_LPI2C_PM      NULL
+#endif
 
 static struct platform_driver lpi2c_imx_driver = {
 	.probe = lpi2c_imx_probe,
@@ -661,7 +695,7 @@ static struct platform_driver lpi2c_imx_driver = {
 	.driver = {
 		.name = DRIVER_NAME,
 		.of_match_table = lpi2c_imx_of_match,
-		.pm = &imx_lpi2c_pm,
+		.pm = IMX_LPI2C_PM,
 	},
 };
 
diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c
index f96830ffd9f1..999557729ad2 100644
--- a/drivers/i2c/busses/i2c-imx.c
+++ b/drivers/i2c/busses/i2c-imx.c
@@ -37,6 +37,7 @@
 #include <linux/dmapool.h>
 #include <linux/err.h>
 #include <linux/errno.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -46,7 +47,6 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/of_dma.h>
-#include <linux/of_gpio.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_data/i2c-imx.h>
 #include <linux/platform_device.h>
@@ -1006,26 +1006,26 @@ static int i2c_imx_init_recovery_info(struct imx_i2c_struct *i2c_imx,
 			PINCTRL_STATE_DEFAULT);
 	i2c_imx->pinctrl_pins_gpio = pinctrl_lookup_state(i2c_imx->pinctrl,
 			"gpio");
-	rinfo->sda_gpio = of_get_named_gpio(pdev->dev.of_node, "sda-gpios", 0);
-	rinfo->scl_gpio = of_get_named_gpio(pdev->dev.of_node, "scl-gpios", 0);
+	rinfo->sda_gpiod = devm_gpiod_get(&pdev->dev, "sda", GPIOD_IN);
+	rinfo->scl_gpiod = devm_gpiod_get(&pdev->dev, "scl", GPIOD_OUT_HIGH);
 
-	if (rinfo->sda_gpio == -EPROBE_DEFER ||
-	    rinfo->scl_gpio == -EPROBE_DEFER) {
+	if (PTR_ERR(rinfo->sda_gpiod) == -EPROBE_DEFER ||
+	    PTR_ERR(rinfo->scl_gpiod) == -EPROBE_DEFER) {
 		return -EPROBE_DEFER;
-	} else if (!gpio_is_valid(rinfo->sda_gpio) ||
-		   !gpio_is_valid(rinfo->scl_gpio) ||
+	} else if (IS_ERR(rinfo->sda_gpiod) ||
+		   IS_ERR(rinfo->scl_gpiod) ||
 		   IS_ERR(i2c_imx->pinctrl_pins_default) ||
 		   IS_ERR(i2c_imx->pinctrl_pins_gpio)) {
 		dev_dbg(&pdev->dev, "recovery information incomplete\n");
 		return 0;
 	}
 
-	dev_dbg(&pdev->dev, "using scl-gpio %d and sda-gpio %d for recovery\n",
-			rinfo->scl_gpio, rinfo->sda_gpio);
+	dev_dbg(&pdev->dev, "using scl%s for recovery\n",
+		rinfo->sda_gpiod ? ",sda" : "");
 
 	rinfo->prepare_recovery = i2c_imx_prepare_recovery;
 	rinfo->unprepare_recovery = i2c_imx_unprepare_recovery;
-	rinfo->recover_bus = i2c_generic_gpio_recovery;
+	rinfo->recover_bus = i2c_generic_scl_recovery;
 	i2c_imx->adapter.bus_recovery_info = rinfo;
 
 	return 0;
diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c
index b51adffa4841..0d1c3ec8cb40 100644
--- a/drivers/i2c/busses/i2c-ismt.c
+++ b/drivers/i2c/busses/i2c-ismt.c
@@ -172,7 +172,7 @@ struct ismt_priv {
 	dma_addr_t io_rng_dma;			/* descriptor HW base addr */
 	u8 head;				/* ring buffer head pointer */
 	struct completion cmp;			/* interrupt completion */
-	u8 dma_buffer[I2C_SMBUS_BLOCK_MAX + 1];	/* temp R/W data buffer */
+	u8 buffer[I2C_SMBUS_BLOCK_MAX + 16];	/* temp R/W data buffer */
 };
 
 /**
@@ -320,10 +320,12 @@ static int ismt_process_desc(const struct ismt_desc *desc,
 			     struct ismt_priv *priv, int size,
 			     char read_write)
 {
-	u8 *dma_buffer = priv->dma_buffer;
+	u8 *dma_buffer = PTR_ALIGN(&priv->buffer[0], 16);
 
 	dev_dbg(&priv->pci_dev->dev, "Processing completed descriptor\n");
 	__ismt_desc_dump(&priv->pci_dev->dev, desc);
+	ismt_gen_reg_dump(priv);
+	ismt_mstr_reg_dump(priv);
 
 	if (desc->status & ISMT_DESC_SCS) {
 		if (read_write == I2C_SMBUS_WRITE &&
@@ -393,11 +395,12 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr,
 	struct ismt_desc *desc;
 	struct ismt_priv *priv = i2c_get_adapdata(adap);
 	struct device *dev = &priv->pci_dev->dev;
+	u8 *dma_buffer = PTR_ALIGN(&priv->buffer[0], 16);
 
 	desc = &priv->hw[priv->head];
 
 	/* Initialize the DMA buffer */
-	memset(priv->dma_buffer, 0, sizeof(priv->dma_buffer));
+	memset(priv->buffer, 0, sizeof(priv->buffer));
 
 	/* Initialize the descriptor */
 	memset(desc, 0, sizeof(struct ismt_desc));
@@ -446,8 +449,8 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr,
 			desc->wr_len_cmd = 2;
 			dma_size = 2;
 			dma_direction = DMA_TO_DEVICE;
-			priv->dma_buffer[0] = command;
-			priv->dma_buffer[1] = data->byte;
+			dma_buffer[0] = command;
+			dma_buffer[1] = data->byte;
 		} else {
 			/* Read Byte */
 			dev_dbg(dev, "I2C_SMBUS_BYTE_DATA:  READ\n");
@@ -466,9 +469,9 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr,
 			desc->wr_len_cmd = 3;
 			dma_size = 3;
 			dma_direction = DMA_TO_DEVICE;
-			priv->dma_buffer[0] = command;
-			priv->dma_buffer[1] = data->word & 0xff;
-			priv->dma_buffer[2] = data->word >> 8;
+			dma_buffer[0] = command;
+			dma_buffer[1] = data->word & 0xff;
+			dma_buffer[2] = data->word >> 8;
 		} else {
 			/* Read Word */
 			dev_dbg(dev, "I2C_SMBUS_WORD_DATA:  READ\n");
@@ -486,9 +489,9 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr,
 		desc->rd_len = 2;
 		dma_size = 3;
 		dma_direction = DMA_BIDIRECTIONAL;
-		priv->dma_buffer[0] = command;
-		priv->dma_buffer[1] = data->word & 0xff;
-		priv->dma_buffer[2] = data->word >> 8;
+		dma_buffer[0] = command;
+		dma_buffer[1] = data->word & 0xff;
+		dma_buffer[2] = data->word >> 8;
 		break;
 
 	case I2C_SMBUS_BLOCK_DATA:
@@ -499,8 +502,8 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr,
 			dma_direction = DMA_TO_DEVICE;
 			desc->wr_len_cmd = dma_size;
 			desc->control |= ISMT_DESC_BLK;
-			priv->dma_buffer[0] = command;
-			memcpy(&priv->dma_buffer[1], &data->block[1], dma_size - 1);
+			dma_buffer[0] = command;
+			memcpy(&dma_buffer[1], &data->block[1], dma_size - 1);
 		} else {
 			/* Block Read */
 			dev_dbg(dev, "I2C_SMBUS_BLOCK_DATA:  READ\n");
@@ -527,8 +530,8 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr,
 			dma_direction = DMA_TO_DEVICE;
 			desc->wr_len_cmd = dma_size;
 			desc->control |= ISMT_DESC_I2C;
-			priv->dma_buffer[0] = command;
-			memcpy(&priv->dma_buffer[1], &data->block[1], dma_size - 1);
+			dma_buffer[0] = command;
+			memcpy(&dma_buffer[1], &data->block[1], dma_size - 1);
 		} else {
 			/* i2c Block Read */
 			dev_dbg(dev, "I2C_SMBUS_I2C_BLOCK_DATA:  READ\n");
@@ -557,23 +560,22 @@ static int ismt_access(struct i2c_adapter *adap, u16 addr,
 	if (dma_size != 0) {
 		dev_dbg(dev, " dev=%p\n", dev);
 		dev_dbg(dev, " data=%p\n", data);
-		dev_dbg(dev, " dma_buffer=%p\n", priv->dma_buffer);
+		dev_dbg(dev, " dma_buffer=%p\n", dma_buffer);
 		dev_dbg(dev, " dma_size=%d\n", dma_size);
 		dev_dbg(dev, " dma_direction=%d\n", dma_direction);
 
 		dma_addr = dma_map_single(dev,
-				      priv->dma_buffer,
+				      dma_buffer,
 				      dma_size,
 				      dma_direction);
 
 		if (dma_mapping_error(dev, dma_addr)) {
 			dev_err(dev, "Error in mapping dma buffer %p\n",
-				priv->dma_buffer);
+				dma_buffer);
 			return -EIO;
 		}
 
-		dev_dbg(dev, " dma_addr = 0x%016llX\n",
-			(unsigned long long)dma_addr);
+		dev_dbg(dev, " dma_addr = %pad\n", &dma_addr);
 
 		desc->dptr_low = lower_32_bits(dma_addr);
 		desc->dptr_high = upper_32_bits(dma_addr);
diff --git a/drivers/i2c/busses/i2c-meson.c b/drivers/i2c/busses/i2c-meson.c
index 88d15b92ec35..90f5d0407d73 100644
--- a/drivers/i2c/busses/i2c-meson.c
+++ b/drivers/i2c/busses/i2c-meson.c
@@ -16,6 +16,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/types.h>
 
@@ -57,6 +58,10 @@ enum {
 	STATE_WRITE,
 };
 
+struct meson_i2c_data {
+	unsigned char div_factor;
+};
+
 /**
  * struct meson_i2c - Meson I2C device private data
  *
@@ -64,7 +69,6 @@ enum {
  * @dev:	Pointer to device structure
  * @regs:	Base address of the device memory mapped registers
  * @clk:	Pointer to clock structure
- * @irq:	IRQ number
  * @msg:	Pointer to the current I2C message
  * @state:	Current state in the driver state machine
  * @last:	Flag set for the last message in the transfer
@@ -75,6 +79,7 @@ enum {
  * @done:	Completion used to wait for transfer termination
  * @tokens:	Sequence of tokens to be written to the device
  * @num_tokens:	Number of tokens
+ * @data:	Pointer to the controlller's platform data
  */
 struct meson_i2c {
 	struct i2c_adapter	adap;
@@ -93,6 +98,8 @@ struct meson_i2c {
 	struct completion	done;
 	u32			tokens[2];
 	int			num_tokens;
+
+	const struct meson_i2c_data *data;
 };
 
 static void meson_i2c_set_mask(struct meson_i2c *i2c, int reg, u32 mask,
@@ -128,7 +135,7 @@ static void meson_i2c_set_clk_div(struct meson_i2c *i2c, unsigned int freq)
 	unsigned long clk_rate = clk_get_rate(i2c->clk);
 	unsigned int div;
 
-	div = DIV_ROUND_UP(clk_rate, freq * 4);
+	div = DIV_ROUND_UP(clk_rate, freq * i2c->data->div_factor);
 
 	/* clock divider has 12 bits */
 	if (div >= (1 << 12)) {
@@ -376,6 +383,9 @@ static int meson_i2c_probe(struct platform_device *pdev)
 	spin_lock_init(&i2c->lock);
 	init_completion(&i2c->done);
 
+	i2c->data = (const struct meson_i2c_data *)
+		of_device_get_match_data(&pdev->dev);
+
 	i2c->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(i2c->clk)) {
 		dev_err(&pdev->dev, "can't get device clock\n");
@@ -440,11 +450,25 @@ static int meson_i2c_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct meson_i2c_data i2c_meson6_data = {
+	.div_factor = 4,
+};
+
+static const struct meson_i2c_data i2c_gxbb_data = {
+	.div_factor = 4,
+};
+
+static const struct meson_i2c_data i2c_axg_data = {
+	.div_factor = 3,
+};
+
 static const struct of_device_id meson_i2c_match[] = {
-	{ .compatible = "amlogic,meson6-i2c" },
-	{ .compatible = "amlogic,meson-gxbb-i2c" },
-	{ },
+	{ .compatible = "amlogic,meson6-i2c", .data = &i2c_meson6_data },
+	{ .compatible = "amlogic,meson-gxbb-i2c", .data = &i2c_gxbb_data },
+	{ .compatible = "amlogic,meson-axg-i2c", .data = &i2c_axg_data },
+	{},
 };
+
 MODULE_DEVICE_TABLE(of, meson_i2c_match);
 
 static struct platform_driver meson_i2c_driver = {
diff --git a/drivers/i2c/busses/i2c-mpc.c b/drivers/i2c/busses/i2c-mpc.c
index 950a9d74f54d..d94f05c8b8b7 100644
--- a/drivers/i2c/busses/i2c-mpc.c
+++ b/drivers/i2c/busses/i2c-mpc.c
@@ -78,9 +78,7 @@ struct mpc_i2c_divider {
 };
 
 struct mpc_i2c_data {
-	void (*setup)(struct device_node *node, struct mpc_i2c *i2c,
-		      u32 clock, u32 prescaler);
-	u32 prescaler;
+	void (*setup)(struct device_node *node, struct mpc_i2c *i2c, u32 clock);
 };
 
 static inline void writeccr(struct mpc_i2c *i2c, u32 x)
@@ -201,7 +199,7 @@ static const struct mpc_i2c_divider mpc_i2c_dividers_52xx[] = {
 };
 
 static int mpc_i2c_get_fdr_52xx(struct device_node *node, u32 clock,
-					  int prescaler, u32 *real_clk)
+					  u32 *real_clk)
 {
 	const struct mpc_i2c_divider *div = NULL;
 	unsigned int pvr = mfspr(SPRN_PVR);
@@ -236,7 +234,7 @@ static int mpc_i2c_get_fdr_52xx(struct device_node *node, u32 clock,
 
 static void mpc_i2c_setup_52xx(struct device_node *node,
 					 struct mpc_i2c *i2c,
-					 u32 clock, u32 prescaler)
+					 u32 clock)
 {
 	int ret, fdr;
 
@@ -246,7 +244,7 @@ static void mpc_i2c_setup_52xx(struct device_node *node,
 		return;
 	}
 
-	ret = mpc_i2c_get_fdr_52xx(node, clock, prescaler, &i2c->real_clk);
+	ret = mpc_i2c_get_fdr_52xx(node, clock, &i2c->real_clk);
 	fdr = (ret >= 0) ? ret : 0x3f; /* backward compatibility */
 
 	writeb(fdr & 0xff, i2c->base + MPC_I2C_FDR);
@@ -258,7 +256,7 @@ static void mpc_i2c_setup_52xx(struct device_node *node,
 #else /* !(CONFIG_PPC_MPC52xx || CONFIG_PPC_MPC512x) */
 static void mpc_i2c_setup_52xx(struct device_node *node,
 					 struct mpc_i2c *i2c,
-					 u32 clock, u32 prescaler)
+					 u32 clock)
 {
 }
 #endif /* CONFIG_PPC_MPC52xx || CONFIG_PPC_MPC512x */
@@ -266,7 +264,7 @@ static void mpc_i2c_setup_52xx(struct device_node *node,
 #ifdef CONFIG_PPC_MPC512x
 static void mpc_i2c_setup_512x(struct device_node *node,
 					 struct mpc_i2c *i2c,
-					 u32 clock, u32 prescaler)
+					 u32 clock)
 {
 	struct device_node *node_ctrl;
 	void __iomem *ctrl;
@@ -289,12 +287,12 @@ static void mpc_i2c_setup_512x(struct device_node *node,
 	}
 
 	/* The clock setup for the 52xx works also fine for the 512x */
-	mpc_i2c_setup_52xx(node, i2c, clock, prescaler);
+	mpc_i2c_setup_52xx(node, i2c, clock);
 }
 #else /* CONFIG_PPC_MPC512x */
 static void mpc_i2c_setup_512x(struct device_node *node,
 					 struct mpc_i2c *i2c,
-					 u32 clock, u32 prescaler)
+					 u32 clock)
 {
 }
 #endif /* CONFIG_PPC_MPC512x */
@@ -332,14 +330,18 @@ static u32 mpc_i2c_get_sec_cfg_8xxx(void)
 		if (prop) {
 			/*
 			 * Map and check POR Device Status Register 2
-			 * (PORDEVSR2) at 0xE0014
+			 * (PORDEVSR2) at 0xE0014. Note than while MPC8533
+			 * and MPC8544 indicate SEC frequency ratio
+			 * configuration as bit 26 in PORDEVSR2, other MPC8xxx
+			 * parts may store it differently or may not have it
+			 * at all.
 			 */
 			reg = ioremap(get_immrbase() + *prop + 0x14, 0x4);
 			if (!reg)
 				printk(KERN_ERR
 				       "Error: couldn't map PORDEVSR2\n");
 			else
-				val = in_be32(reg) & 0x00000080; /* sec-cfg */
+				val = in_be32(reg) & 0x00000020; /* sec-cfg */
 			iounmap(reg);
 		}
 	}
@@ -350,7 +352,11 @@ static u32 mpc_i2c_get_sec_cfg_8xxx(void)
 
 static u32 mpc_i2c_get_prescaler_8xxx(void)
 {
-	/* mpc83xx and mpc82xx all have prescaler 1 */
+	/*
+	 * According to the AN2919 all MPC824x have prescaler 1, while MPC83xx
+	 * may have prescaler 1, 2, or 3, depending on the power-on
+	 * configuration.
+	 */
 	u32 prescaler = 1;
 
 	/* mpc85xx */
@@ -367,6 +373,10 @@ static u32 mpc_i2c_get_prescaler_8xxx(void)
 			|| (SVR_SOC_VER(svr) == SVR_8610))
 			/* the above 85xx SoCs have prescaler 1 */
 			prescaler = 1;
+		else if ((SVR_SOC_VER(svr) == SVR_8533)
+			|| (SVR_SOC_VER(svr) == SVR_8544))
+			/* the above 85xx SoCs have prescaler 3 or 2 */
+			prescaler = mpc_i2c_get_sec_cfg_8xxx() ? 3 : 2;
 		else
 			/* all the other 85xx have prescaler 2 */
 			prescaler = 2;
@@ -376,9 +386,10 @@ static u32 mpc_i2c_get_prescaler_8xxx(void)
 }
 
 static int mpc_i2c_get_fdr_8xxx(struct device_node *node, u32 clock,
-					  u32 prescaler, u32 *real_clk)
+					  u32 *real_clk)
 {
 	const struct mpc_i2c_divider *div = NULL;
+	u32 prescaler = mpc_i2c_get_prescaler_8xxx();
 	u32 divider;
 	int i;
 
@@ -388,12 +399,6 @@ static int mpc_i2c_get_fdr_8xxx(struct device_node *node, u32 clock,
 		return -EINVAL;
 	}
 
-	/* Determine proper divider value */
-	if (of_device_is_compatible(node, "fsl,mpc8544-i2c"))
-		prescaler = mpc_i2c_get_sec_cfg_8xxx() ? 3 : 2;
-	if (!prescaler)
-		prescaler = mpc_i2c_get_prescaler_8xxx();
-
 	divider = fsl_get_sys_freq() / clock / prescaler;
 
 	pr_debug("I2C: src_clock=%d clock=%d divider=%d\n",
@@ -415,7 +420,7 @@ static int mpc_i2c_get_fdr_8xxx(struct device_node *node, u32 clock,
 
 static void mpc_i2c_setup_8xxx(struct device_node *node,
 					 struct mpc_i2c *i2c,
-					 u32 clock, u32 prescaler)
+					 u32 clock)
 {
 	int ret, fdr;
 
@@ -426,7 +431,7 @@ static void mpc_i2c_setup_8xxx(struct device_node *node,
 		return;
 	}
 
-	ret = mpc_i2c_get_fdr_8xxx(node, clock, prescaler, &i2c->real_clk);
+	ret = mpc_i2c_get_fdr_8xxx(node, clock, &i2c->real_clk);
 	fdr = (ret >= 0) ? ret : 0x1031; /* backward compatibility */
 
 	writeb(fdr & 0xff, i2c->base + MPC_I2C_FDR);
@@ -440,7 +445,7 @@ static void mpc_i2c_setup_8xxx(struct device_node *node,
 #else /* !CONFIG_FSL_SOC */
 static void mpc_i2c_setup_8xxx(struct device_node *node,
 					 struct mpc_i2c *i2c,
-					 u32 clock, u32 prescaler)
+					 u32 clock)
 {
 }
 #endif /* CONFIG_FSL_SOC */
@@ -711,11 +716,11 @@ static int fsl_i2c_probe(struct platform_device *op)
 
 	if (match->data) {
 		const struct mpc_i2c_data *data = match->data;
-		data->setup(op->dev.of_node, i2c, clock, data->prescaler);
+		data->setup(op->dev.of_node, i2c, clock);
 	} else {
 		/* Backwards compatibility */
 		if (of_get_property(op->dev.of_node, "dfsrr", NULL))
-			mpc_i2c_setup_8xxx(op->dev.of_node, i2c, clock, 0);
+			mpc_i2c_setup_8xxx(op->dev.of_node, i2c, clock);
 	}
 
 	prop = of_get_property(op->dev.of_node, "fsl,timeout", &plen);
@@ -813,12 +818,10 @@ static const struct mpc_i2c_data mpc_i2c_data_8313 = {
 
 static const struct mpc_i2c_data mpc_i2c_data_8543 = {
 	.setup = mpc_i2c_setup_8xxx,
-	.prescaler = 2,
 };
 
 static const struct mpc_i2c_data mpc_i2c_data_8544 = {
 	.setup = mpc_i2c_setup_8xxx,
-	.prescaler = 3,
 };
 
 static const struct of_device_id mpc_i2c_of_match[] = {
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index 09d288ce0ddb..cf23a746cc17 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -61,6 +61,7 @@
 #define I2C_DMA_HARD_RST		0x0002
 #define I2C_DMA_4G_MODE			0x0001
 
+#define I2C_DEFAULT_CLK_DIV		5
 #define I2C_DEFAULT_SPEED		100000	/* hz */
 #define MAX_FS_MODE_SPEED		400000
 #define MAX_HS_MODE_SPEED		3400000
@@ -127,6 +128,7 @@ enum I2C_REGS_OFFSET {
 	OFFSET_DEBUGSTAT = 0x64,
 	OFFSET_DEBUGCTRL = 0x68,
 	OFFSET_TRANSFER_LEN_AUX = 0x6c,
+	OFFSET_CLOCK_DIV = 0x70,
 };
 
 struct mtk_i2c_compatible {
@@ -136,6 +138,7 @@ struct mtk_i2c_compatible {
 	unsigned char auto_restart: 1;
 	unsigned char aux_len_reg: 1;
 	unsigned char support_33bits: 1;
+	unsigned char timing_adjust: 1;
 };
 
 struct mtk_i2c {
@@ -176,6 +179,15 @@ static const struct i2c_adapter_quirks mt7622_i2c_quirks = {
 	.max_num_msgs = 255,
 };
 
+static const struct mtk_i2c_compatible mt2712_compat = {
+	.pmic_i2c = 0,
+	.dcm = 1,
+	.auto_restart = 1,
+	.aux_len_reg = 1,
+	.support_33bits = 1,
+	.timing_adjust = 1,
+};
+
 static const struct mtk_i2c_compatible mt6577_compat = {
 	.quirks = &mt6577_i2c_quirks,
 	.pmic_i2c = 0,
@@ -183,6 +195,7 @@ static const struct mtk_i2c_compatible mt6577_compat = {
 	.auto_restart = 0,
 	.aux_len_reg = 0,
 	.support_33bits = 0,
+	.timing_adjust = 0,
 };
 
 static const struct mtk_i2c_compatible mt6589_compat = {
@@ -192,6 +205,7 @@ static const struct mtk_i2c_compatible mt6589_compat = {
 	.auto_restart = 0,
 	.aux_len_reg = 0,
 	.support_33bits = 0,
+	.timing_adjust = 0,
 };
 
 static const struct mtk_i2c_compatible mt7622_compat = {
@@ -201,6 +215,7 @@ static const struct mtk_i2c_compatible mt7622_compat = {
 	.auto_restart = 1,
 	.aux_len_reg = 1,
 	.support_33bits = 0,
+	.timing_adjust = 0,
 };
 
 static const struct mtk_i2c_compatible mt8173_compat = {
@@ -209,9 +224,11 @@ static const struct mtk_i2c_compatible mt8173_compat = {
 	.auto_restart = 1,
 	.aux_len_reg = 1,
 	.support_33bits = 1,
+	.timing_adjust = 0,
 };
 
 static const struct of_device_id mtk_i2c_of_match[] = {
+	{ .compatible = "mediatek,mt2712-i2c", .data = &mt2712_compat },
 	{ .compatible = "mediatek,mt6577-i2c", .data = &mt6577_compat },
 	{ .compatible = "mediatek,mt6589-i2c", .data = &mt6589_compat },
 	{ .compatible = "mediatek,mt7622-i2c", .data = &mt7622_compat },
@@ -271,6 +288,9 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c)
 	if (i2c->dev_comp->dcm)
 		writew(I2C_DCM_DISABLE, i2c->base + OFFSET_DCM_EN);
 
+	if (i2c->dev_comp->timing_adjust)
+		writew(I2C_DEFAULT_CLK_DIV - 1, i2c->base + OFFSET_CLOCK_DIV);
+
 	writew(i2c->timing_reg, i2c->base + OFFSET_TIMING);
 	writew(i2c->high_speed_reg, i2c->base + OFFSET_HS);
 
@@ -725,10 +745,6 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 	if (!i2c)
 		return -ENOMEM;
 
-	ret = mtk_i2c_parse_dt(pdev->dev.of_node, i2c);
-	if (ret)
-		return -EINVAL;
-
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	i2c->base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(i2c->base))
@@ -759,6 +775,13 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 	i2c->adap.timeout = 2 * HZ;
 	i2c->adap.retries = 1;
 
+	ret = mtk_i2c_parse_dt(pdev->dev.of_node, i2c);
+	if (ret)
+		return -EINVAL;
+
+	if (i2c->dev_comp->timing_adjust)
+		i2c->clk_src_div *= I2C_DEFAULT_CLK_DIV;
+
 	if (i2c->have_pmic && !i2c->dev_comp->pmic_i2c)
 		return -EINVAL;
 
@@ -838,10 +861,19 @@ static int mtk_i2c_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int mtk_i2c_resume(struct device *dev)
 {
+	int ret;
 	struct mtk_i2c *i2c = dev_get_drvdata(dev);
 
+	ret = mtk_i2c_clock_enable(i2c);
+	if (ret) {
+		dev_err(dev, "clock enable failed!\n");
+		return ret;
+	}
+
 	mtk_i2c_init_hw(i2c);
 
+	mtk_i2c_clock_disable(i2c);
+
 	return 0;
 }
 #endif
diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c
index a832c45276a4..440fe4a96e68 100644
--- a/drivers/i2c/busses/i2c-mv64xxx.c
+++ b/drivers/i2c/busses/i2c-mv64xxx.c
@@ -135,6 +135,7 @@ struct mv64xxx_i2c_data {
 	u32			freq_m;
 	u32			freq_n;
 	struct clk              *clk;
+	struct clk              *reg_clk;
 	wait_queue_head_t	waitq;
 	spinlock_t		lock;
 	struct i2c_msg		*msg;
@@ -894,13 +895,20 @@ mv64xxx_i2c_probe(struct platform_device *pd)
 	init_waitqueue_head(&drv_data->waitq);
 	spin_lock_init(&drv_data->lock);
 
-	/* Not all platforms have a clk */
+	/* Not all platforms have clocks */
 	drv_data->clk = devm_clk_get(&pd->dev, NULL);
 	if (IS_ERR(drv_data->clk) && PTR_ERR(drv_data->clk) == -EPROBE_DEFER)
 		return -EPROBE_DEFER;
 	if (!IS_ERR(drv_data->clk))
 		clk_prepare_enable(drv_data->clk);
 
+	drv_data->reg_clk = devm_clk_get(&pd->dev, "reg");
+	if (IS_ERR(drv_data->reg_clk) &&
+	    PTR_ERR(drv_data->reg_clk) == -EPROBE_DEFER)
+		return -EPROBE_DEFER;
+	if (!IS_ERR(drv_data->reg_clk))
+		clk_prepare_enable(drv_data->reg_clk);
+
 	drv_data->irq = platform_get_irq(pd, 0);
 
 	if (pdata) {
@@ -950,9 +958,8 @@ mv64xxx_i2c_probe(struct platform_device *pd)
 exit_reset:
 	reset_control_assert(drv_data->rstc);
 exit_clk:
-	/* Not all platforms have a clk */
-	if (!IS_ERR(drv_data->clk))
-		clk_disable_unprepare(drv_data->clk);
+	clk_disable_unprepare(drv_data->reg_clk);
+	clk_disable_unprepare(drv_data->clk);
 
 	return rc;
 }
@@ -965,9 +972,8 @@ mv64xxx_i2c_remove(struct platform_device *dev)
 	i2c_del_adapter(&drv_data->adapter);
 	free_irq(drv_data->irq, drv_data);
 	reset_control_assert(drv_data->rstc);
-	/* Not all platforms have a clk */
-	if (!IS_ERR(drv_data->clk))
-		clk_disable_unprepare(drv_data->clk);
+	clk_disable_unprepare(drv_data->reg_clk);
+	clk_disable_unprepare(drv_data->clk);
 
 	return 0;
 }
diff --git a/drivers/i2c/busses/i2c-mxs.c b/drivers/i2c/busses/i2c-mxs.c
index d4e8f1954f23..e617bd600794 100644
--- a/drivers/i2c/busses/i2c-mxs.c
+++ b/drivers/i2c/busses/i2c-mxs.c
@@ -181,7 +181,7 @@ static int mxs_i2c_dma_setup_xfer(struct i2c_adapter *adap,
 	struct mxs_i2c_dev *i2c = i2c_get_adapdata(adap);
 
 	if (msg->flags & I2C_M_RD) {
-		i2c->dma_read = 1;
+		i2c->dma_read = true;
 		i2c->addr_data = (msg->addr << 1) | I2C_SMBUS_READ;
 
 		/*
@@ -239,7 +239,7 @@ static int mxs_i2c_dma_setup_xfer(struct i2c_adapter *adap,
 			goto read_init_dma_fail;
 		}
 	} else {
-		i2c->dma_read = 0;
+		i2c->dma_read = false;
 		i2c->addr_data = (msg->addr << 1) | I2C_SMBUS_WRITE;
 
 		/*
diff --git a/drivers/i2c/busses/i2c-octeon-core.c b/drivers/i2c/busses/i2c-octeon-core.c
index 1d8775799056..d9607905dc2f 100644
--- a/drivers/i2c/busses/i2c-octeon-core.c
+++ b/drivers/i2c/busses/i2c-octeon-core.c
@@ -233,6 +233,7 @@ static int octeon_i2c_check_status(struct octeon_i2c *i2c, int final_read)
 		return -EOPNOTSUPP;
 
 	case STAT_TXDATA_NAK:
+	case STAT_BUS_ERROR:
 		return -EIO;
 	case STAT_TXADDR_NAK:
 	case STAT_RXADDR_NAK:
diff --git a/drivers/i2c/busses/i2c-octeon-core.h b/drivers/i2c/busses/i2c-octeon-core.h
index a7ef19855bb8..9bb9f64fdda0 100644
--- a/drivers/i2c/busses/i2c-octeon-core.h
+++ b/drivers/i2c/busses/i2c-octeon-core.h
@@ -43,7 +43,7 @@
 #define TWSI_CTL_AAK		0x04	/* Assert ACK */
 
 /* Status values */
-#define STAT_ERROR		0x00
+#define STAT_BUS_ERROR		0x00
 #define STAT_START		0x08
 #define STAT_REP_START		0x10
 #define STAT_TXADDR_ACK		0x18
diff --git a/drivers/i2c/busses/i2c-pxa-pci.c b/drivers/i2c/busses/i2c-pxa-pci.c
index 004deb96afe3..72ea8f4c61aa 100644
--- a/drivers/i2c/busses/i2c-pxa-pci.c
+++ b/drivers/i2c/busses/i2c-pxa-pci.c
@@ -10,7 +10,7 @@
 #include <linux/init.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/of_address.h>
diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c
index 600d264e080c..fbf91d383b40 100644
--- a/drivers/i2c/busses/i2c-pxa.c
+++ b/drivers/i2c/busses/i2c-pxa.c
@@ -36,7 +36,7 @@
 #include <linux/clk.h>
 #include <linux/slab.h>
 #include <linux/io.h>
-#include <linux/i2c/pxa-i2c.h>
+#include <linux/platform_data/i2c-pxa.h>
 
 #include <asm/irq.h>
 
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index 15d764afec3b..4159ebcec2bb 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -62,7 +62,7 @@
 #define MIE	(1 << 3)	/* master if enable */
 #define TSBE	(1 << 2)
 #define FSB	(1 << 1)	/* force stop bit */
-#define ESG	(1 << 0)	/* en startbit gen */
+#define ESG	(1 << 0)	/* enable start bit gen */
 
 /* ICSSR (also for ICSIER) */
 #define GCAR	(1 << 6)	/* general call received */
@@ -132,6 +132,7 @@ struct rcar_i2c_priv {
 	int pos;
 	u32 icccr;
 	u32 flags;
+	u8 recovery_icmcr;	/* protected by adapter lock */
 	enum rcar_i2c_type devtype;
 	struct i2c_client *slave;
 
@@ -158,6 +159,46 @@ static u32 rcar_i2c_read(struct rcar_i2c_priv *priv, int reg)
 	return readl(priv->io + reg);
 }
 
+static int rcar_i2c_get_scl(struct i2c_adapter *adap)
+{
+	struct rcar_i2c_priv *priv = i2c_get_adapdata(adap);
+
+	return !!(rcar_i2c_read(priv, ICMCR) & FSCL);
+
+};
+
+static void rcar_i2c_set_scl(struct i2c_adapter *adap, int val)
+{
+	struct rcar_i2c_priv *priv = i2c_get_adapdata(adap);
+
+	if (val)
+		priv->recovery_icmcr |= FSCL;
+	else
+		priv->recovery_icmcr &= ~FSCL;
+
+	rcar_i2c_write(priv, ICMCR, priv->recovery_icmcr);
+};
+
+/* No get_sda, because the HW only reports its bus free logic, not SDA itself */
+
+static void rcar_i2c_set_sda(struct i2c_adapter *adap, int val)
+{
+	struct rcar_i2c_priv *priv = i2c_get_adapdata(adap);
+
+	if (val)
+		priv->recovery_icmcr |= FSDA;
+	else
+		priv->recovery_icmcr &= ~FSDA;
+
+	rcar_i2c_write(priv, ICMCR, priv->recovery_icmcr);
+};
+
+static struct i2c_bus_recovery_info rcar_i2c_bri = {
+	.get_scl = rcar_i2c_get_scl,
+	.set_scl = rcar_i2c_set_scl,
+	.set_sda = rcar_i2c_set_sda,
+	.recover_bus = i2c_generic_scl_recovery,
+};
 static void rcar_i2c_init(struct rcar_i2c_priv *priv)
 {
 	/* reset master mode */
@@ -170,7 +211,7 @@ static void rcar_i2c_init(struct rcar_i2c_priv *priv)
 
 static int rcar_i2c_bus_barrier(struct rcar_i2c_priv *priv)
 {
-	int i;
+	int i, ret;
 
 	for (i = 0; i < LOOP_TIMEOUT; i++) {
 		/* make sure that bus is not busy */
@@ -179,7 +220,15 @@ static int rcar_i2c_bus_barrier(struct rcar_i2c_priv *priv)
 		udelay(1);
 	}
 
-	return -EBUSY;
+	/* Waiting did not help, try to recover */
+	priv->recovery_icmcr = MDBS | OBPC | FSDA | FSCL;
+	ret = i2c_recover_bus(&priv->adap);
+
+	/* No failure when recovering, so check bus busy bit again */
+	if (ret == 0)
+		ret = (rcar_i2c_read(priv, ICMCR) & FSDA) ? -EBUSY : 0;
+
+	return ret;
 }
 
 static int rcar_i2c_clock_calculate(struct rcar_i2c_priv *priv, struct i2c_timings *t)
@@ -282,7 +331,7 @@ static void rcar_i2c_prepare_msg(struct rcar_i2c_priv *priv)
 
 	rcar_i2c_write(priv, ICMAR, (priv->msg->addr << 1) | read);
 	/*
-	 * We don't have a testcase but the HW engineers say that the write order
+	 * We don't have a test case but the HW engineers say that the write order
 	 * of ICMSR and ICMCR depends on whether we issue START or REP_START. Since
 	 * it didn't cause a drawback for me, let's rather be safe than sorry.
 	 */
@@ -359,7 +408,7 @@ static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
 	int len;
 
 	/* Do not use DMA if it's not available or for messages < 8 bytes */
-	if (IS_ERR(chan) || msg->len < 8)
+	if (IS_ERR(chan) || msg->len < 8 || !(msg->flags & I2C_M_DMA_SAFE))
 		return;
 
 	if (read) {
@@ -440,7 +489,7 @@ static void rcar_i2c_irq_send(struct rcar_i2c_priv *priv, u32 msr)
 
 		/*
 		 * Try to use DMA to transmit the rest of the data if
-		 * address transfer pashe just finished.
+		 * address transfer phase just finished.
 		 */
 		if (msr & MAT)
 			rcar_i2c_dma(priv);
@@ -851,6 +900,7 @@ static int rcar_i2c_probe(struct platform_device *pdev)
 	adap->retries = 3;
 	adap->dev.parent = dev;
 	adap->dev.of_node = dev->of_node;
+	adap->bus_recovery_info = &rcar_i2c_bri;
 	i2c_set_adapdata(adap, priv);
 	strlcpy(adap->name, pdev->name, sizeof(adap->name));
 
diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
index fe234578380a..e1a18d989f83 100644
--- a/drivers/i2c/busses/i2c-rk3x.c
+++ b/drivers/i2c/busses/i2c-rk3x.c
@@ -161,6 +161,7 @@ enum rk3x_i2c_state {
 };
 
 /**
+ * struct rk3x_i2c_soc_data:
  * @grf_offset: offset inside the grf regmap for setting the i2c type
  * @calc_timings: Callback function for i2c timing information calculated
  */
@@ -194,7 +195,7 @@ struct rk3x_i2c_soc_data {
 struct rk3x_i2c {
 	struct i2c_adapter adap;
 	struct device *dev;
-	struct rk3x_i2c_soc_data *soc_data;
+	const struct rk3x_i2c_soc_data *soc_data;
 
 	/* Hardware resources */
 	void __iomem *regs;
@@ -1164,27 +1165,27 @@ static const struct rk3x_i2c_soc_data rk3399_soc_data = {
 static const struct of_device_id rk3x_i2c_match[] = {
 	{
 		.compatible = "rockchip,rv1108-i2c",
-		.data = (void *)&rv1108_soc_data
+		.data = &rv1108_soc_data
 	},
 	{
 		.compatible = "rockchip,rk3066-i2c",
-		.data = (void *)&rk3066_soc_data
+		.data = &rk3066_soc_data
 	},
 	{
 		.compatible = "rockchip,rk3188-i2c",
-		.data = (void *)&rk3188_soc_data
+		.data = &rk3188_soc_data
 	},
 	{
 		.compatible = "rockchip,rk3228-i2c",
-		.data = (void *)&rk3228_soc_data
+		.data = &rk3228_soc_data
 	},
 	{
 		.compatible = "rockchip,rk3288-i2c",
-		.data = (void *)&rk3288_soc_data
+		.data = &rk3288_soc_data
 	},
 	{
 		.compatible = "rockchip,rk3399-i2c",
-		.data = (void *)&rk3399_soc_data
+		.data = &rk3399_soc_data
 	},
 	{},
 };
@@ -1207,7 +1208,7 @@ static int rk3x_i2c_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	match = of_match_node(rk3x_i2c_match, np);
-	i2c->soc_data = (struct rk3x_i2c_soc_data *)match->data;
+	i2c->soc_data = match->data;
 
 	/* use common interface to get I2C timing properties */
 	i2c_parse_fw_timings(&pdev->dev, &i2c->t, true);
diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c
index c03acdf71397..d856bc211715 100644
--- a/drivers/i2c/busses/i2c-sh_mobile.c
+++ b/drivers/i2c/busses/i2c-sh_mobile.c
@@ -40,21 +40,21 @@
 /* BUS:     S     A8     ACK   P(*)                                         */
 /* IRQ:       DTE   WAIT                                                    */
 /* ICIC:                                                                    */
-/* ICCR: 0x94 0x90                                                          */
+/* ICCR: 0x94       0x90                                                    */
 /* ICDR:      A8                                                            */
 /*                                                                          */
 /* 1 byte transmit                                                          */
 /* BUS:     S     A8     ACK   D8(1)   ACK   P(*)                           */
 /* IRQ:       DTE   WAIT         WAIT                                       */
 /* ICIC:      -DTE                                                          */
-/* ICCR: 0x94       0x90                                                    */
+/* ICCR: 0x94                    0x90                                       */
 /* ICDR:      A8    D8(1)                                                   */
 /*                                                                          */
 /* 2 byte transmit                                                          */
 /* BUS:     S     A8     ACK   D8(1)   ACK   D8(2)   ACK   P(*)             */
 /* IRQ:       DTE   WAIT         WAIT          WAIT                         */
 /* ICIC:      -DTE                                                          */
-/* ICCR: 0x94                    0x90                                       */
+/* ICCR: 0x94                                  0x90                         */
 /* ICDR:      A8    D8(1)        D8(2)                                      */
 /*                                                                          */
 /* 3 bytes or more, +---------+ gets repeated                               */
@@ -113,7 +113,6 @@ enum sh_mobile_i2c_op {
 	OP_TX_FIRST,
 	OP_TX,
 	OP_TX_STOP,
-	OP_TX_STOP_DATA,
 	OP_TX_TO_RX,
 	OP_RX,
 	OP_RX_STOP,
@@ -145,11 +144,12 @@ struct sh_mobile_i2c_data {
 	struct dma_chan *dma_rx;
 	struct scatterlist sg;
 	enum dma_data_direction dma_direction;
+	u8 *dma_buf;
 };
 
 struct sh_mobile_dt_config {
 	int clks_per_count;
-	void (*setup)(struct sh_mobile_i2c_data *pd);
+	int (*setup)(struct sh_mobile_i2c_data *pd);
 };
 
 #define IIC_FLAG_HAS_ICIC67	(1 << 0)
@@ -246,36 +246,10 @@ static u32 sh_mobile_i2c_icch(unsigned long count_khz, u32 tHIGH, u32 tf)
 	return (((count_khz * (tHIGH + tf)) + 5000) / 10000);
 }
 
-static int sh_mobile_i2c_init(struct sh_mobile_i2c_data *pd)
+static int sh_mobile_i2c_check_timing(struct sh_mobile_i2c_data *pd)
 {
-	unsigned long i2c_clk_khz;
-	u32 tHIGH, tLOW, tf;
-	uint16_t max_val;
-
-	/* Get clock rate after clock is enabled */
-	clk_prepare_enable(pd->clk);
-	i2c_clk_khz = clk_get_rate(pd->clk) / 1000;
-	clk_disable_unprepare(pd->clk);
-	i2c_clk_khz /= pd->clks_per_count;
-
-	if (pd->bus_speed == STANDARD_MODE) {
-		tLOW	= 47;	/* tLOW = 4.7 us */
-		tHIGH	= 40;	/* tHD;STA = tHIGH = 4.0 us */
-		tf	= 3;	/* tf = 0.3 us */
-	} else if (pd->bus_speed == FAST_MODE) {
-		tLOW	= 13;	/* tLOW = 1.3 us */
-		tHIGH	= 6;	/* tHD;STA = tHIGH = 0.6 us */
-		tf	= 3;	/* tf = 0.3 us */
-	} else {
-		dev_err(pd->dev, "unrecognized bus speed %lu Hz\n",
-			pd->bus_speed);
-		return -EINVAL;
-	}
-
-	pd->iccl = sh_mobile_i2c_iccl(i2c_clk_khz, tLOW, tf);
-	pd->icch = sh_mobile_i2c_icch(i2c_clk_khz, tHIGH, tf);
+	u16 max_val = pd->flags & IIC_FLAG_HAS_ICIC67 ? 0x1ff : 0xff;
 
-	max_val = pd->flags & IIC_FLAG_HAS_ICIC67 ? 0x1ff : 0xff;
 	if (pd->iccl > max_val || pd->icch > max_val) {
 		dev_err(pd->dev, "timing values out of range: L/H=0x%x/0x%x\n",
 			pd->iccl, pd->icch);
@@ -298,35 +272,43 @@ static int sh_mobile_i2c_init(struct sh_mobile_i2c_data *pd)
 	return 0;
 }
 
-static void activate_ch(struct sh_mobile_i2c_data *pd)
+static int sh_mobile_i2c_init(struct sh_mobile_i2c_data *pd)
 {
-	/* Wake up device and enable clock */
-	pm_runtime_get_sync(pd->dev);
-	clk_prepare_enable(pd->clk);
+	unsigned long i2c_clk_khz;
+	u32 tHIGH, tLOW, tf;
 
-	/* Enable channel and configure rx ack */
-	iic_set_clr(pd, ICCR, ICCR_ICE, 0);
+	i2c_clk_khz = clk_get_rate(pd->clk) / 1000 / pd->clks_per_count;
 
-	/* Mask all interrupts */
-	iic_wr(pd, ICIC, 0);
+	if (pd->bus_speed == STANDARD_MODE) {
+		tLOW	= 47;	/* tLOW = 4.7 us */
+		tHIGH	= 40;	/* tHD;STA = tHIGH = 4.0 us */
+		tf	= 3;	/* tf = 0.3 us */
+	} else if (pd->bus_speed == FAST_MODE) {
+		tLOW	= 13;	/* tLOW = 1.3 us */
+		tHIGH	= 6;	/* tHD;STA = tHIGH = 0.6 us */
+		tf	= 3;	/* tf = 0.3 us */
+	} else {
+		dev_err(pd->dev, "unrecognized bus speed %lu Hz\n",
+			pd->bus_speed);
+		return -EINVAL;
+	}
 
-	/* Set the clock */
-	iic_wr(pd, ICCL, pd->iccl & 0xff);
-	iic_wr(pd, ICCH, pd->icch & 0xff);
+	pd->iccl = sh_mobile_i2c_iccl(i2c_clk_khz, tLOW, tf);
+	pd->icch = sh_mobile_i2c_icch(i2c_clk_khz, tHIGH, tf);
+
+	return sh_mobile_i2c_check_timing(pd);
 }
 
-static void deactivate_ch(struct sh_mobile_i2c_data *pd)
+static int sh_mobile_i2c_v2_init(struct sh_mobile_i2c_data *pd)
 {
-	/* Clear/disable interrupts */
-	iic_wr(pd, ICSR, 0);
-	iic_wr(pd, ICIC, 0);
+	unsigned long clks_per_cycle;
 
-	/* Disable channel */
-	iic_set_clr(pd, ICCR, 0, ICCR_ICE);
+	/* L = 5, H = 4, L + H = 9 */
+	clks_per_cycle = clk_get_rate(pd->clk) / pd->bus_speed;
+	pd->iccl = DIV_ROUND_UP(clks_per_cycle * 5 / 9 - 1, pd->clks_per_count);
+	pd->icch = DIV_ROUND_UP(clks_per_cycle * 4 / 9 - 5, pd->clks_per_count);
 
-	/* Disable clock and mark device as idle */
-	clk_disable_unprepare(pd->clk);
-	pm_runtime_put_sync(pd->dev);
+	return sh_mobile_i2c_check_timing(pd);
 }
 
 static unsigned char i2c_op(struct sh_mobile_i2c_data *pd,
@@ -350,10 +332,7 @@ static unsigned char i2c_op(struct sh_mobile_i2c_data *pd,
 	case OP_TX: /* write data */
 		iic_wr(pd, ICDR, data);
 		break;
-	case OP_TX_STOP_DATA: /* write data and issue a stop afterwards */
-		iic_wr(pd, ICDR, data);
-		/* fallthrough */
-	case OP_TX_STOP: /* issue a stop */
+	case OP_TX_STOP: /* issue a stop (or rep_start) */
 		iic_wr(pd, ICCR, pd->send_stop ? ICCR_ICE | ICCR_TRS
 					       : ICCR_ICE | ICCR_TRS | ICCR_BBSY);
 		break;
@@ -387,11 +366,6 @@ static bool sh_mobile_i2c_is_first_byte(struct sh_mobile_i2c_data *pd)
 	return pd->pos == -1;
 }
 
-static bool sh_mobile_i2c_is_last_byte(struct sh_mobile_i2c_data *pd)
-{
-	return pd->pos == pd->msg->len - 1;
-}
-
 static void sh_mobile_i2c_get_data(struct sh_mobile_i2c_data *pd,
 				   unsigned char *buf)
 {
@@ -409,20 +383,12 @@ static int sh_mobile_i2c_isr_tx(struct sh_mobile_i2c_data *pd)
 	unsigned char data;
 
 	if (pd->pos == pd->msg->len) {
-		/* Send stop if we haven't yet (DMA case) */
-		if (pd->send_stop && pd->stop_after_dma)
-			i2c_op(pd, OP_TX_STOP, 0);
+		i2c_op(pd, OP_TX_STOP, 0);
 		return 1;
 	}
 
 	sh_mobile_i2c_get_data(pd, &data);
-
-	if (sh_mobile_i2c_is_last_byte(pd))
-		i2c_op(pd, OP_TX_STOP_DATA, data);
-	else if (sh_mobile_i2c_is_first_byte(pd))
-		i2c_op(pd, OP_TX_FIRST, data);
-	else
-		i2c_op(pd, OP_TX, data);
+	i2c_op(pd, sh_mobile_i2c_is_first_byte(pd) ? OP_TX_FIRST : OP_TX, data);
 
 	pd->pos++;
 	return 0;
@@ -464,8 +430,9 @@ static int sh_mobile_i2c_isr_rx(struct sh_mobile_i2c_data *pd)
 				break;
 			}
 			data = i2c_op(pd, OP_RX_STOP_DATA, 0);
-		} else
+		} else if (real_pos >= 0) {
 			data = i2c_op(pd, OP_RX, 0);
+		}
 
 		if (real_pos >= 0)
 			pd->msg->buf[real_pos] = data;
@@ -548,6 +515,8 @@ static void sh_mobile_i2c_dma_callback(void *data)
 	pd->pos = pd->msg->len;
 	pd->stop_after_dma = true;
 
+	i2c_release_dma_safe_msg_buf(pd->msg, pd->dma_buf);
+
 	iic_set_clr(pd, ICIC, 0, ICIC_TDMAE | ICIC_RDMAE);
 }
 
@@ -608,7 +577,7 @@ static void sh_mobile_i2c_xfer_dma(struct sh_mobile_i2c_data *pd)
 	if (IS_ERR(chan))
 		return;
 
-	dma_addr = dma_map_single(chan->device->dev, pd->msg->buf, pd->msg->len, dir);
+	dma_addr = dma_map_single(chan->device->dev, pd->dma_buf, pd->msg->len, dir);
 	if (dma_mapping_error(chan->device->dev, dma_addr)) {
 		dev_dbg(pd->dev, "dma map failed, using PIO\n");
 		return;
@@ -651,10 +620,10 @@ static int start_ch(struct sh_mobile_i2c_data *pd, struct i2c_msg *usr_msg,
 
 	if (do_init) {
 		/* Initialize channel registers */
-		iic_set_clr(pd, ICCR, 0, ICCR_ICE);
+		iic_wr(pd, ICCR, ICCR_SCP);
 
 		/* Enable channel and configure rx ack */
-		iic_set_clr(pd, ICCR, ICCR_ICE, 0);
+		iic_wr(pd, ICCR, ICCR_ICE | ICCR_SCP);
 
 		/* Set the clock */
 		iic_wr(pd, ICCL, pd->iccl & 0xff);
@@ -665,7 +634,8 @@ static int start_ch(struct sh_mobile_i2c_data *pd, struct i2c_msg *usr_msg,
 	pd->pos = -1;
 	pd->sr = 0;
 
-	if (pd->msg->len > 8)
+	pd->dma_buf = i2c_get_dma_safe_msg_buf(pd->msg, 8);
+	if (pd->dma_buf)
 		sh_mobile_i2c_xfer_dma(pd);
 
 	/* Enable all interrupts to begin with */
@@ -731,7 +701,8 @@ static int sh_mobile_i2c_xfer(struct i2c_adapter *adapter,
 	int i;
 	long timeout;
 
-	activate_ch(pd);
+	/* Wake up device and enable clock */
+	pm_runtime_get_sync(pd->dev);
 
 	/* Process all messages */
 	for (i = 0; i < num; i++) {
@@ -768,11 +739,13 @@ static int sh_mobile_i2c_xfer(struct i2c_adapter *adapter,
 			break;
 	}
 
-	deactivate_ch(pd);
+	/* Disable channel */
+	iic_wr(pd, ICCR, ICCR_SCP);
 
-	if (!err)
-		err = num;
-	return err;
+	/* Disable clock and mark device as idle */
+	pm_runtime_put_sync(pd->dev);
+
+	return err ?: num;
 }
 
 static u32 sh_mobile_i2c_func(struct i2c_adapter *adapter)
@@ -789,7 +762,7 @@ static const struct i2c_algorithm sh_mobile_i2c_algorithm = {
  * r8a7740 chip has lasting errata on I2C I/O pad reset.
  * this is work-around for it.
  */
-static void sh_mobile_i2c_r8a7740_workaround(struct sh_mobile_i2c_data *pd)
+static int sh_mobile_i2c_r8a7740_workaround(struct sh_mobile_i2c_data *pd)
 {
 	iic_set_clr(pd, ICCR, ICCR_ICE, 0);
 	iic_rd(pd, ICCR); /* dummy read */
@@ -810,14 +783,23 @@ static void sh_mobile_i2c_r8a7740_workaround(struct sh_mobile_i2c_data *pd)
 	udelay(10);
 	iic_wr(pd, ICCR, ICCR_TRS);
 	udelay(10);
+
+	return sh_mobile_i2c_init(pd);
 }
 
 static const struct sh_mobile_dt_config default_dt_config = {
 	.clks_per_count = 1,
+	.setup = sh_mobile_i2c_init,
 };
 
 static const struct sh_mobile_dt_config fast_clock_dt_config = {
 	.clks_per_count = 2,
+	.setup = sh_mobile_i2c_init,
+};
+
+static const struct sh_mobile_dt_config v2_freq_calc_dt_config = {
+	.clks_per_count = 2,
+	.setup = sh_mobile_i2c_v2_init,
 };
 
 static const struct sh_mobile_dt_config r8a7740_dt_config = {
@@ -828,7 +810,7 @@ static const struct sh_mobile_dt_config r8a7740_dt_config = {
 static const struct of_device_id sh_mobile_i2c_dt_ids[] = {
 	{ .compatible = "renesas,iic-r8a73a4", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-r8a7740", .data = &r8a7740_dt_config },
-	{ .compatible = "renesas,iic-r8a7790", .data = &fast_clock_dt_config },
+	{ .compatible = "renesas,iic-r8a7790", .data = &v2_freq_calc_dt_config },
 	{ .compatible = "renesas,iic-r8a7791", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-r8a7792", .data = &fast_clock_dt_config },
 	{ .compatible = "renesas,iic-r8a7793", .data = &fast_clock_dt_config },
@@ -910,32 +892,13 @@ static int sh_mobile_i2c_probe(struct platform_device *dev)
 		return PTR_ERR(pd->reg);
 
 	ret = of_property_read_u32(dev->dev.of_node, "clock-frequency", &bus_speed);
-	pd->bus_speed = ret ? STANDARD_MODE : bus_speed;
+	pd->bus_speed = (ret || !bus_speed) ? STANDARD_MODE : bus_speed;
 	pd->clks_per_count = 1;
 
-	config = of_device_get_match_data(&dev->dev);
-	if (config) {
-		pd->clks_per_count = config->clks_per_count;
-
-		if (config->setup)
-			config->setup(pd);
-	}
-
-	/* The IIC blocks on SH-Mobile ARM processors
-	 * come with two new bits in ICIC.
-	 */
+	/* Newer variants come with two new bits in ICIC */
 	if (resource_size(res) > 0x17)
 		pd->flags |= IIC_FLAG_HAS_ICIC67;
 
-	ret = sh_mobile_i2c_init(pd);
-	if (ret)
-		return ret;
-
-	/* Init DMA */
-	sg_init_table(&pd->sg, 1);
-	pd->dma_direction = DMA_NONE;
-	pd->dma_rx = pd->dma_tx = ERR_PTR(-EPROBE_DEFER);
-
 	/* Enable Runtime PM for this device.
 	 *
 	 * Also tell the Runtime PM core to ignore children
@@ -948,6 +911,24 @@ static int sh_mobile_i2c_probe(struct platform_device *dev)
 	 */
 	pm_suspend_ignore_children(&dev->dev, true);
 	pm_runtime_enable(&dev->dev);
+	pm_runtime_get_sync(&dev->dev);
+
+	config = of_device_get_match_data(&dev->dev);
+	if (config) {
+		pd->clks_per_count = config->clks_per_count;
+		ret = config->setup(pd);
+	} else {
+		ret = sh_mobile_i2c_init(pd);
+	}
+
+	pm_runtime_put_sync(&dev->dev);
+	if (ret)
+		return ret;
+
+	/* Init DMA */
+	sg_init_table(&pd->sg, 1);
+	pd->dma_direction = DMA_NONE;
+	pd->dma_rx = pd->dma_tx = ERR_PTR(-EPROBE_DEFER);
 
 	/* setup the private data */
 	adap = &pd->adap;
diff --git a/drivers/i2c/busses/i2c-sirf.c b/drivers/i2c/busses/i2c-sirf.c
index 2fd8b6d00391..87197ece0f90 100644
--- a/drivers/i2c/busses/i2c-sirf.c
+++ b/drivers/i2c/busses/i2c-sirf.c
@@ -341,7 +341,7 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, adap);
 	init_completion(&siic->done);
 
-	/* Controller Initalisation */
+	/* Controller initialisation */
 
 	writel(SIRFSOC_I2C_RESET, siic->base + SIRFSOC_I2C_CTRL);
 	while (readl(siic->base + SIRFSOC_I2C_CTRL) & SIRFSOC_I2C_RESET)
@@ -369,7 +369,7 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev)
 	 * but they start to affect the speed when clock is set to faster
 	 * frequencies.
 	 * Through the actual tests, use the different user_div value(which
-	 * in the divider formular 'Fio / (Fi2c * user_div)') to adapt
+	 * in the divider formula 'Fio / (Fi2c * user_div)') to adapt
 	 * the different ranges of i2c bus clock frequency, to make the SCL
 	 * more accurate.
 	 */
diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c
index b445b3bb0bb1..f273e28c39db 100644
--- a/drivers/i2c/busses/i2c-stm32f7.c
+++ b/drivers/i2c/busses/i2c-stm32f7.c
@@ -888,6 +888,11 @@ static int stm32f7_i2c_probe(struct platform_device *pdev)
 	}
 
 	setup = of_device_get_match_data(&pdev->dev);
+	if (!setup) {
+		dev_err(&pdev->dev, "Can't get device data\n");
+		ret = -ENODEV;
+		goto clk_free;
+	}
 	i2c_dev->setup = *setup;
 
 	ret = device_property_read_u32(i2c_dev->dev, "i2c-scl-rising-time-ns",
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index f7829a74140c..5a00bf443d06 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -27,7 +27,7 @@
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/errno.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/i2c-smbus.h>
 #include <linux/idr.h>
@@ -134,52 +134,22 @@ static int i2c_device_uevent(struct device *dev, struct kobj_uevent_env *env)
 /* i2c bus recovery routines */
 static int get_scl_gpio_value(struct i2c_adapter *adap)
 {
-	return gpio_get_value(adap->bus_recovery_info->scl_gpio);
+	return gpiod_get_value_cansleep(adap->bus_recovery_info->scl_gpiod);
 }
 
 static void set_scl_gpio_value(struct i2c_adapter *adap, int val)
 {
-	gpio_set_value(adap->bus_recovery_info->scl_gpio, val);
+	gpiod_set_value_cansleep(adap->bus_recovery_info->scl_gpiod, val);
 }
 
 static int get_sda_gpio_value(struct i2c_adapter *adap)
 {
-	return gpio_get_value(adap->bus_recovery_info->sda_gpio);
+	return gpiod_get_value_cansleep(adap->bus_recovery_info->sda_gpiod);
 }
 
-static int i2c_get_gpios_for_recovery(struct i2c_adapter *adap)
+static void set_sda_gpio_value(struct i2c_adapter *adap, int val)
 {
-	struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
-	struct device *dev = &adap->dev;
-	int ret = 0;
-
-	ret = gpio_request_one(bri->scl_gpio, GPIOF_OPEN_DRAIN |
-			GPIOF_OUT_INIT_HIGH, "i2c-scl");
-	if (ret) {
-		dev_warn(dev, "Can't get SCL gpio: %d\n", bri->scl_gpio);
-		return ret;
-	}
-
-	if (bri->get_sda) {
-		if (gpio_request_one(bri->sda_gpio, GPIOF_IN, "i2c-sda")) {
-			/* work without SDA polling */
-			dev_warn(dev, "Can't get SDA gpio: %d. Not using SDA polling\n",
-					bri->sda_gpio);
-			bri->get_sda = NULL;
-		}
-	}
-
-	return ret;
-}
-
-static void i2c_put_gpios_for_recovery(struct i2c_adapter *adap)
-{
-	struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
-
-	if (bri->get_sda)
-		gpio_free(bri->sda_gpio);
-
-	gpio_free(bri->scl_gpio);
+	gpiod_set_value_cansleep(adap->bus_recovery_info->sda_gpiod, val);
 }
 
 /*
@@ -190,7 +160,7 @@ static void i2c_put_gpios_for_recovery(struct i2c_adapter *adap)
 #define RECOVERY_NDELAY		5000
 #define RECOVERY_CLK_CNT	9
 
-static int i2c_generic_recovery(struct i2c_adapter *adap)
+int i2c_generic_scl_recovery(struct i2c_adapter *adap)
 {
 	struct i2c_bus_recovery_info *bri = adap->bus_recovery_info;
 	int i = 0, val = 1, ret = 0;
@@ -199,6 +169,8 @@ static int i2c_generic_recovery(struct i2c_adapter *adap)
 		bri->prepare_recovery(adap);
 
 	bri->set_scl(adap, val);
+	if (bri->set_sda)
+		bri->set_sda(adap, 1);
 	ndelay(RECOVERY_NDELAY);
 
 	/*
@@ -227,33 +199,25 @@ static int i2c_generic_recovery(struct i2c_adapter *adap)
 	if (bri->get_sda && !bri->get_sda(adap))
 		ret = -EBUSY;
 
+	/* If all went well, send STOP for a sane bus state. */
+	if (ret == 0 && bri->set_sda) {
+		bri->set_scl(adap, 0);
+		ndelay(RECOVERY_NDELAY / 2);
+		bri->set_sda(adap, 0);
+		ndelay(RECOVERY_NDELAY / 2);
+		bri->set_scl(adap, 1);
+		ndelay(RECOVERY_NDELAY / 2);
+		bri->set_sda(adap, 1);
+		ndelay(RECOVERY_NDELAY / 2);
+	}
+
 	if (bri->unprepare_recovery)
 		bri->unprepare_recovery(adap);
 
 	return ret;
 }
-
-int i2c_generic_scl_recovery(struct i2c_adapter *adap)
-{
-	return i2c_generic_recovery(adap);
-}
 EXPORT_SYMBOL_GPL(i2c_generic_scl_recovery);
 
-int i2c_generic_gpio_recovery(struct i2c_adapter *adap)
-{
-	int ret;
-
-	ret = i2c_get_gpios_for_recovery(adap);
-	if (ret)
-		return ret;
-
-	ret = i2c_generic_recovery(adap);
-	i2c_put_gpios_for_recovery(adap);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(i2c_generic_gpio_recovery);
-
 int i2c_recover_bus(struct i2c_adapter *adap)
 {
 	if (!adap->bus_recovery_info)
@@ -277,21 +241,19 @@ static void i2c_init_recovery(struct i2c_adapter *adap)
 		goto err;
 	}
 
-	/* Generic GPIO recovery */
-	if (bri->recover_bus == i2c_generic_gpio_recovery) {
-		if (!gpio_is_valid(bri->scl_gpio)) {
-			err_str = "invalid SCL gpio";
-			goto err;
-		}
-
-		if (gpio_is_valid(bri->sda_gpio))
-			bri->get_sda = get_sda_gpio_value;
-		else
-			bri->get_sda = NULL;
-
+	if (bri->scl_gpiod && bri->recover_bus == i2c_generic_scl_recovery) {
 		bri->get_scl = get_scl_gpio_value;
 		bri->set_scl = set_scl_gpio_value;
-	} else if (bri->recover_bus == i2c_generic_scl_recovery) {
+		if (bri->sda_gpiod) {
+			bri->get_sda = get_sda_gpio_value;
+			/* FIXME: add proper flag instead of '0' once available */
+			if (gpiod_get_direction(bri->sda_gpiod) == 0)
+				bri->set_sda = set_sda_gpio_value;
+		}
+		return;
+	}
+
+	if (bri->recover_bus == i2c_generic_scl_recovery) {
 		/* Generic SCL recovery */
 		if (!bri->set_scl || !bri->get_scl) {
 			err_str = "no {get|set}_scl() found";
@@ -1976,63 +1938,35 @@ int i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 EXPORT_SYMBOL(i2c_transfer);
 
 /**
- * i2c_master_send - issue a single I2C message in master transmit mode
- * @client: Handle to slave device
- * @buf: Data that will be written to the slave
- * @count: How many bytes to write, must be less than 64k since msg.len is u16
- *
- * Returns negative errno, or else the number of bytes written.
- */
-int i2c_master_send(const struct i2c_client *client, const char *buf, int count)
-{
-	int ret;
-	struct i2c_adapter *adap = client->adapter;
-	struct i2c_msg msg;
-
-	msg.addr = client->addr;
-	msg.flags = client->flags & I2C_M_TEN;
-	msg.len = count;
-	msg.buf = (char *)buf;
-
-	ret = i2c_transfer(adap, &msg, 1);
-
-	/*
-	 * If everything went ok (i.e. 1 msg transmitted), return #bytes
-	 * transmitted, else error code.
-	 */
-	return (ret == 1) ? count : ret;
-}
-EXPORT_SYMBOL(i2c_master_send);
-
-/**
- * i2c_master_recv - issue a single I2C message in master receive mode
+ * i2c_transfer_buffer_flags - issue a single I2C message transferring data
+ *			       to/from a buffer
  * @client: Handle to slave device
- * @buf: Where to store data read from slave
- * @count: How many bytes to read, must be less than 64k since msg.len is u16
+ * @buf: Where the data is stored
+ * @count: How many bytes to transfer, must be less than 64k since msg.len is u16
+ * @flags: The flags to be used for the message, e.g. I2C_M_RD for reads
  *
- * Returns negative errno, or else the number of bytes read.
+ * Returns negative errno, or else the number of bytes transferred.
  */
-int i2c_master_recv(const struct i2c_client *client, char *buf, int count)
+int i2c_transfer_buffer_flags(const struct i2c_client *client, char *buf,
+			      int count, u16 flags)
 {
-	struct i2c_adapter *adap = client->adapter;
-	struct i2c_msg msg;
 	int ret;
+	struct i2c_msg msg = {
+		.addr = client->addr,
+		.flags = flags | (client->flags & I2C_M_TEN),
+		.len = count,
+		.buf = buf,
+	};
 
-	msg.addr = client->addr;
-	msg.flags = client->flags & I2C_M_TEN;
-	msg.flags |= I2C_M_RD;
-	msg.len = count;
-	msg.buf = buf;
-
-	ret = i2c_transfer(adap, &msg, 1);
+	ret = i2c_transfer(client->adapter, &msg, 1);
 
 	/*
-	 * If everything went ok (i.e. 1 msg received), return #bytes received,
-	 * else error code.
+	 * If everything went ok (i.e. 1 msg transferred), return #bytes
+	 * transferred, else error code.
 	 */
 	return (ret == 1) ? count : ret;
 }
-EXPORT_SYMBOL(i2c_master_recv);
+EXPORT_SYMBOL(i2c_transfer_buffer_flags);
 
 /* ----------------------------------------------------
  * the i2c address scanning function
@@ -2265,6 +2199,52 @@ void i2c_put_adapter(struct i2c_adapter *adap)
 }
 EXPORT_SYMBOL(i2c_put_adapter);
 
+/**
+ * i2c_get_dma_safe_msg_buf() - get a DMA safe buffer for the given i2c_msg
+ * @msg: the message to be checked
+ * @threshold: the minimum number of bytes for which using DMA makes sense
+ *
+ * Return: NULL if a DMA safe buffer was not obtained. Use msg->buf with PIO.
+ *	   Or a valid pointer to be used with DMA. After use, release it by
+ *	   calling i2c_release_dma_safe_msg_buf().
+ *
+ * This function must only be called from process context!
+ */
+u8 *i2c_get_dma_safe_msg_buf(struct i2c_msg *msg, unsigned int threshold)
+{
+	if (msg->len < threshold)
+		return NULL;
+
+	if (msg->flags & I2C_M_DMA_SAFE)
+		return msg->buf;
+
+	pr_debug("using bounce buffer for addr=0x%02x, len=%d\n",
+		 msg->addr, msg->len);
+
+	if (msg->flags & I2C_M_RD)
+		return kzalloc(msg->len, GFP_KERNEL);
+	else
+		return kmemdup(msg->buf, msg->len, GFP_KERNEL);
+}
+EXPORT_SYMBOL_GPL(i2c_get_dma_safe_msg_buf);
+
+/**
+ * i2c_release_dma_safe_msg_buf - release DMA safe buffer and sync with i2c_msg
+ * @msg: the message to be synced with
+ * @buf: the buffer obtained from i2c_get_dma_safe_msg_buf(). May be NULL.
+ */
+void i2c_release_dma_safe_msg_buf(struct i2c_msg *msg, u8 *buf)
+{
+	if (!buf || buf == msg->buf)
+		return;
+
+	if (msg->flags & I2C_M_RD)
+		memcpy(msg->buf, buf, msg->len);
+
+	kfree(buf);
+}
+EXPORT_SYMBOL_GPL(i2c_release_dma_safe_msg_buf);
+
 MODULE_AUTHOR("Simon G. Vogl <simon@tk.uni-linz.ac.at>");
 MODULE_DESCRIPTION("I2C-Bus main module");
 MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c
index a1082c04ac5c..59d5cf376f6a 100644
--- a/drivers/i2c/i2c-core-smbus.c
+++ b/drivers/i2c/i2c-core-smbus.c
@@ -18,6 +18,7 @@
 #include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/i2c-smbus.h>
+#include <linux/slab.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/smbus.h>
@@ -291,6 +292,22 @@ s32 i2c_smbus_write_i2c_block_data(const struct i2c_client *client, u8 command,
 }
 EXPORT_SYMBOL(i2c_smbus_write_i2c_block_data);
 
+static void i2c_smbus_try_get_dmabuf(struct i2c_msg *msg, u8 init_val)
+{
+	bool is_read = msg->flags & I2C_M_RD;
+	unsigned char *dma_buf;
+
+	dma_buf = kzalloc(I2C_SMBUS_BLOCK_MAX + (is_read ? 2 : 3), GFP_KERNEL);
+	if (!dma_buf)
+		return;
+
+	msg->buf = dma_buf;
+	msg->flags |= I2C_M_DMA_SAFE;
+
+	if (init_val)
+		msg->buf[0] = init_val;
+}
+
 /* Simulate a SMBus command using the i2c protocol
    No checking of parameters is done!  */
 static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
@@ -368,6 +385,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
 			msg[1].flags |= I2C_M_RECV_LEN;
 			msg[1].len = 1; /* block length will be added by
 					   the underlying bus driver */
+			i2c_smbus_try_get_dmabuf(&msg[1], 0);
 		} else {
 			msg[0].len = data->block[0] + 2;
 			if (msg[0].len > I2C_SMBUS_BLOCK_MAX + 2) {
@@ -376,8 +394,10 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
 					data->block[0]);
 				return -EINVAL;
 			}
+
+			i2c_smbus_try_get_dmabuf(&msg[0], command);
 			for (i = 1; i < msg[0].len; i++)
-				msgbuf0[i] = data->block[i-1];
+				msg[0].buf[i] = data->block[i - 1];
 		}
 		break;
 	case I2C_SMBUS_BLOCK_PROC_CALL:
@@ -389,12 +409,16 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
 				data->block[0]);
 			return -EINVAL;
 		}
+
 		msg[0].len = data->block[0] + 2;
+		i2c_smbus_try_get_dmabuf(&msg[0], command);
 		for (i = 1; i < msg[0].len; i++)
-			msgbuf0[i] = data->block[i-1];
+			msg[0].buf[i] = data->block[i - 1];
+
 		msg[1].flags |= I2C_M_RECV_LEN;
 		msg[1].len = 1; /* block length will be added by
 				   the underlying bus driver */
+		i2c_smbus_try_get_dmabuf(&msg[1], 0);
 		break;
 	case I2C_SMBUS_I2C_BLOCK_DATA:
 		if (data->block[0] > I2C_SMBUS_BLOCK_MAX) {
@@ -406,10 +430,13 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
 
 		if (read_write == I2C_SMBUS_READ) {
 			msg[1].len = data->block[0];
+			i2c_smbus_try_get_dmabuf(&msg[1], 0);
 		} else {
 			msg[0].len = data->block[0] + 1;
+
+			i2c_smbus_try_get_dmabuf(&msg[0], command);
 			for (i = 1; i <= data->block[0]; i++)
-				msgbuf0[i] = data->block[i];
+				msg[0].buf[i] = data->block[i];
 		}
 		break;
 	default:
@@ -457,14 +484,20 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
 			break;
 		case I2C_SMBUS_I2C_BLOCK_DATA:
 			for (i = 0; i < data->block[0]; i++)
-				data->block[i+1] = msgbuf1[i];
+				data->block[i + 1] = msg[1].buf[i];
 			break;
 		case I2C_SMBUS_BLOCK_DATA:
 		case I2C_SMBUS_BLOCK_PROC_CALL:
-			for (i = 0; i < msgbuf1[0] + 1; i++)
-				data->block[i] = msgbuf1[i];
+			for (i = 0; i < msg[1].buf[0] + 1; i++)
+				data->block[i] = msg[1].buf[i];
 			break;
 		}
+
+	if (msg[0].flags & I2C_M_DMA_SAFE)
+		kfree(msg[0].buf);
+	if (msg[1].flags & I2C_M_DMA_SAFE)
+		kfree(msg[1].buf);
+
 	return 0;
 }
 
diff --git a/drivers/i2c/i2c-dev.c b/drivers/i2c/i2c-dev.c
index 2cab27a68479..036a03f0d0a6 100644
--- a/drivers/i2c/i2c-dev.c
+++ b/drivers/i2c/i2c-dev.c
@@ -264,6 +264,8 @@ static noinline int i2cdev_ioctl_rdwr(struct i2c_client *client,
 			res = PTR_ERR(msgs[i].buf);
 			break;
 		}
+		/* memdup_user allocates with GFP_KERNEL, so DMA is ok */
+		msgs[i].flags |= I2C_M_DMA_SAFE;
 
 		/*
 		 * If the message length is received from the slave (similar
diff --git a/drivers/i2c/muxes/Kconfig b/drivers/i2c/muxes/Kconfig
index 0f5c8fc36625..52a4a922e7e6 100644
--- a/drivers/i2c/muxes/Kconfig
+++ b/drivers/i2c/muxes/Kconfig
@@ -64,11 +64,11 @@ config I2C_MUX_PCA9541
 	  will be called i2c-mux-pca9541.
 
 config I2C_MUX_PCA954x
-	tristate "Philips PCA954x I2C Mux/switches"
+	tristate "NXP PCA954x and PCA984x I2C Mux/switches"
 	depends on GPIOLIB || COMPILE_TEST
 	help
-	  If you say yes here you get support for the Philips PCA954x
-	  I2C mux/switch devices.
+	  If you say yes here you get support for the NXP PCA954x
+	  and PCA984x I2C mux/switch devices.
 
 	  This driver can also be built as a module.  If so, the module
 	  will be called i2c-mux-pca954x.
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index 2ca068d8b92d..fbb84c7ef282 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -4,11 +4,11 @@
  * Copyright (c) 2008-2009 Rodolfo Giometti <giometti@linux.it>
  * Copyright (c) 2008-2009 Eurotech S.p.A. <info@eurotech.it>
  *
- * This module supports the PCA954x series of I2C multiplexer/switch chips
- * made by Philips Semiconductors.
+ * This module supports the PCA954x and PCA954x series of I2C multiplexer/switch
+ * chips made by NXP Semiconductors.
  * This includes the:
- *	 PCA9540, PCA9542, PCA9543, PCA9544, PCA9545, PCA9546, PCA9547
- *	 and PCA9548.
+ *	 PCA9540, PCA9542, PCA9543, PCA9544, PCA9545, PCA9546, PCA9547,
+ *	 PCA9548, PCA9846, PCA9847, PCA9848 and PCA9849.
  *
  * These chips are all controlled via the I2C bus itself, and all have a
  * single 8-bit register. The upstream "parent" bus fans out to two,
@@ -63,6 +63,10 @@ enum pca_type {
 	pca_9546,
 	pca_9547,
 	pca_9548,
+	pca_9846,
+	pca_9847,
+	pca_9848,
+	pca_9849,
 };
 
 struct chip_desc {
@@ -129,6 +133,24 @@ static const struct chip_desc chips[] = {
 		.nchans = 8,
 		.muxtype = pca954x_isswi,
 	},
+	[pca_9846] = {
+		.nchans = 4,
+		.muxtype = pca954x_isswi,
+	},
+	[pca_9847] = {
+		.nchans = 8,
+		.enable = 0x8,
+		.muxtype = pca954x_ismux,
+	},
+	[pca_9848] = {
+		.nchans = 8,
+		.muxtype = pca954x_isswi,
+	},
+	[pca_9849] = {
+		.nchans = 4,
+		.enable = 0x4,
+		.muxtype = pca954x_ismux,
+	},
 };
 
 static const struct i2c_device_id pca954x_id[] = {
@@ -140,6 +162,10 @@ static const struct i2c_device_id pca954x_id[] = {
 	{ "pca9546", pca_9546 },
 	{ "pca9547", pca_9547 },
 	{ "pca9548", pca_9548 },
+	{ "pca9846", pca_9846 },
+	{ "pca9847", pca_9847 },
+	{ "pca9848", pca_9848 },
+	{ "pca9849", pca_9849 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pca954x_id);
@@ -154,6 +180,10 @@ static const struct of_device_id pca954x_of_match[] = {
 	{ .compatible = "nxp,pca9546", .data = &chips[pca_9546] },
 	{ .compatible = "nxp,pca9547", .data = &chips[pca_9547] },
 	{ .compatible = "nxp,pca9548", .data = &chips[pca_9548] },
+	{ .compatible = "nxp,pca9846", .data = &chips[pca_9846] },
+	{ .compatible = "nxp,pca9847", .data = &chips[pca_9847] },
+	{ .compatible = "nxp,pca9848", .data = &chips[pca_9848] },
+	{ .compatible = "nxp,pca9849", .data = &chips[pca_9849] },
 	{}
 };
 MODULE_DEVICE_TABLE(of, pca954x_of_match);
diff --git a/drivers/i2c/muxes/i2c-mux-reg.c b/drivers/i2c/muxes/i2c-mux-reg.c
index f6c9c3dc6cad..c948e5a4cb04 100644
--- a/drivers/i2c/muxes/i2c-mux-reg.c
+++ b/drivers/i2c/muxes/i2c-mux-reg.c
@@ -177,6 +177,9 @@ static int i2c_mux_reg_probe(struct platform_device *pdev)
 			sizeof(mux->data));
 	} else {
 		ret = i2c_mux_reg_probe_dt(mux, pdev);
+		if (ret == -EPROBE_DEFER)
+			return ret;
+
 		if (ret < 0) {
 			dev_err(&pdev->dev, "Error parsing device tree");
 			return ret;
diff --git a/drivers/ide/ide-probe.c b/drivers/ide/ide-probe.c
index 17fd55af4d92..caa20eb5f26b 100644
--- a/drivers/ide/ide-probe.c
+++ b/drivers/ide/ide-probe.c
@@ -928,7 +928,7 @@ static int exact_lock(dev_t dev, void *data)
 {
 	struct gendisk *p = data;
 
-	if (!get_disk(p))
+	if (!get_disk_and_module(p))
 		return -1;
 	return 0;
 }
diff --git a/drivers/ide/ide-taskfile.c b/drivers/ide/ide-taskfile.c
index 4efe4c6e956c..abe0822dd429 100644
--- a/drivers/ide/ide-taskfile.c
+++ b/drivers/ide/ide-taskfile.c
@@ -232,7 +232,6 @@ void ide_pio_bytes(ide_drive_t *drive, struct ide_cmd *cmd,
 	unsigned int offset;
 	u8 *buf;
 
-	cursg = cmd->cursg;
 	if (cursg == NULL)
 		cursg = cmd->cursg = sg;
 
diff --git a/drivers/ide/sl82c105.c b/drivers/ide/sl82c105.c
index 8755df3330a0..3300dac56390 100644
--- a/drivers/ide/sl82c105.c
+++ b/drivers/ide/sl82c105.c
@@ -239,8 +239,9 @@ static u8 sl82c105_bridge_revision(struct pci_dev *dev)
 	/*
 	 * The bridge should be part of the same device, but function 0.
 	 */
-	bridge = pci_get_bus_and_slot(dev->bus->number,
-			       PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
+	bridge = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
+					dev->bus->number,
+					PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
 	if (!bridge)
 		return -1;
 
diff --git a/drivers/iio/accel/bmc150-accel-i2c.c b/drivers/iio/accel/bmc150-accel-i2c.c
index f85014fbaa12..8ffc308d5fd0 100644
--- a/drivers/iio/accel/bmc150-accel-i2c.c
+++ b/drivers/iio/accel/bmc150-accel-i2c.c
@@ -81,9 +81,21 @@ static const struct i2c_device_id bmc150_accel_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, bmc150_accel_id);
 
+static const struct of_device_id bmc150_accel_of_match[] = {
+	{ .compatible = "bosch,bmc150_accel" },
+	{ .compatible = "bosch,bmi055_accel" },
+	{ .compatible = "bosch,bma255" },
+	{ .compatible = "bosch,bma250e" },
+	{ .compatible = "bosch,bma222e" },
+	{ .compatible = "bosch,bma280" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, bmc150_accel_of_match);
+
 static struct i2c_driver bmc150_accel_driver = {
 	.driver = {
 		.name	= "bmc150_accel_i2c",
+		.of_match_table = bmc150_accel_of_match,
 		.acpi_match_table = ACPI_PTR(bmc150_accel_acpi_match),
 		.pm	= &bmc150_accel_pm_ops,
 	},
diff --git a/drivers/iio/accel/da280.c b/drivers/iio/accel/da280.c
index 6c214783241c..d4b555203427 100644
--- a/drivers/iio/accel/da280.c
+++ b/drivers/iio/accel/da280.c
@@ -11,6 +11,7 @@
 
 #include <linux/module.h>
 #include <linux/i2c.h>
+#include <linux/acpi.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/byteorder/generic.h>
@@ -25,7 +26,7 @@
 #define DA280_MODE_ENABLE		0x1e
 #define DA280_MODE_DISABLE		0x9e
 
-enum { da226, da280 };
+enum da280_chipset { da226, da280 };
 
 /*
  * a value of + or -4096 corresponds to + or - 1G
@@ -91,12 +92,24 @@ static const struct iio_info da280_info = {
 	.read_raw	= da280_read_raw,
 };
 
+static enum da280_chipset da280_match_acpi_device(struct device *dev)
+{
+	const struct acpi_device_id *id;
+
+	id = acpi_match_device(dev->driver->acpi_match_table, dev);
+	if (!id)
+		return -EINVAL;
+
+	return (enum da280_chipset) id->driver_data;
+}
+
 static int da280_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
 	int ret;
 	struct iio_dev *indio_dev;
 	struct da280_data *data;
+	enum da280_chipset chip;
 
 	ret = i2c_smbus_read_byte_data(client, DA280_REG_CHIP_ID);
 	if (ret != DA280_CHIP_ID)
@@ -114,7 +127,14 @@ static int da280_probe(struct i2c_client *client,
 	indio_dev->info = &da280_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = da280_channels;
-	if (id->driver_data == da226) {
+
+	if (ACPI_HANDLE(&client->dev)) {
+		chip = da280_match_acpi_device(&client->dev);
+	} else {
+		chip = id->driver_data;
+	}
+
+	if (chip == da226) {
 		indio_dev->name = "da226";
 		indio_dev->num_channels = 2;
 	} else {
@@ -158,6 +178,12 @@ static int da280_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(da280_pm_ops, da280_suspend, da280_resume);
 
+static const struct acpi_device_id da280_acpi_match[] = {
+	{"MIRAACC", da280},
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, da280_acpi_match);
+
 static const struct i2c_device_id da280_i2c_id[] = {
 	{ "da226", da226 },
 	{ "da280", da280 },
@@ -168,6 +194,7 @@ MODULE_DEVICE_TABLE(i2c, da280_i2c_id);
 static struct i2c_driver da280_driver = {
 	.driver = {
 		.name = "da280",
+		.acpi_match_table = ACPI_PTR(da280_acpi_match),
 		.pm = &da280_pm_ops,
 	},
 	.probe		= da280_probe,
diff --git a/drivers/iio/accel/kxsd9-i2c.c b/drivers/iio/accel/kxsd9-i2c.c
index 98fbb628d5bd..38411e1c155b 100644
--- a/drivers/iio/accel/kxsd9-i2c.c
+++ b/drivers/iio/accel/kxsd9-i2c.c
@@ -63,3 +63,6 @@ static struct i2c_driver kxsd9_i2c_driver = {
 	.id_table	= kxsd9_i2c_id,
 };
 module_i2c_driver(kxsd9_i2c_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("KXSD9 accelerometer I2C interface");
diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c
index bfd4bc806fc2..7a2da7f9d4dc 100644
--- a/drivers/iio/accel/mma8452.c
+++ b/drivers/iio/accel/mma8452.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * mma8452.c - Support for following Freescale / NXP 3-axis accelerometers:
  *
@@ -13,9 +14,6 @@
  * Copyright 2015 Martin Kepplinger <martink@posteo.de>
  * Copyright 2014 Peter Meerwald <pmeerw@pmeerw.net>
  *
- * This file is subject to the terms and conditions of version 2 of
- * the GNU General Public License.  See the file COPYING in the main
- * directory of this archive for more details.
  *
  * TODO: orientation events
  */
@@ -135,7 +133,7 @@ struct mma8452_event_regs {
 		u8 ev_count;
 };
 
-static const struct mma8452_event_regs ev_regs_accel_falling = {
+static const struct mma8452_event_regs ff_mt_ev_regs = {
 		.ev_cfg = MMA8452_FF_MT_CFG,
 		.ev_cfg_ele = MMA8452_FF_MT_CFG_ELE,
 		.ev_cfg_chan_shift = MMA8452_FF_MT_CHAN_SHIFT,
@@ -145,7 +143,7 @@ static const struct mma8452_event_regs ev_regs_accel_falling = {
 		.ev_count = MMA8452_FF_MT_COUNT
 };
 
-static const struct mma8452_event_regs ev_regs_accel_rising = {
+static const struct mma8452_event_regs trans_ev_regs = {
 		.ev_cfg = MMA8452_TRANSIENT_CFG,
 		.ev_cfg_ele = MMA8452_TRANSIENT_CFG_ELE,
 		.ev_cfg_chan_shift = MMA8452_TRANSIENT_CHAN_SHIFT,
@@ -284,7 +282,7 @@ static const int mma8452_samp_freq[8][2] = {
 };
 
 /* Datasheet table: step time "Relationship with the ODR" (sample frequency) */
-static const unsigned int mma8452_transient_time_step_us[4][8] = {
+static const unsigned int mma8452_time_step_us[4][8] = {
 	{ 1250, 2500, 5000, 10000, 20000, 20000, 20000, 20000 },  /* normal */
 	{ 1250, 2500, 5000, 10000, 20000, 80000, 80000, 80000 },  /* l p l n */
 	{ 1250, 2500, 2500, 2500, 2500, 2500, 2500, 2500 },	  /* high res*/
@@ -777,12 +775,12 @@ static int mma8452_get_event_regs(struct mma8452_data *data,
 					& MMA8452_INT_TRANS) &&
 				(data->chip_info->enabled_events
 					& MMA8452_INT_TRANS))
-				*ev_reg = &ev_regs_accel_rising;
+				*ev_reg = &trans_ev_regs;
 			else
-				*ev_reg = &ev_regs_accel_falling;
+				*ev_reg = &ff_mt_ev_regs;
 			return 0;
 		case IIO_EV_DIR_FALLING:
-			*ev_reg = &ev_regs_accel_falling;
+			*ev_reg = &ff_mt_ev_regs;
 			return 0;
 		default:
 			return -EINVAL;
@@ -826,7 +824,7 @@ static int mma8452_read_event_value(struct iio_dev *indio_dev,
 		if (power_mode < 0)
 			return power_mode;
 
-		us = ret * mma8452_transient_time_step_us[power_mode][
+		us = ret * mma8452_time_step_us[power_mode][
 				mma8452_get_odr_index(data)];
 		*val = us / USEC_PER_SEC;
 		*val2 = us % USEC_PER_SEC;
@@ -883,7 +881,7 @@ static int mma8452_write_event_value(struct iio_dev *indio_dev,
 			return ret;
 
 		steps = (val * USEC_PER_SEC + val2) /
-				mma8452_transient_time_step_us[ret][
+				mma8452_time_step_us[ret][
 					mma8452_get_odr_index(data)];
 
 		if (steps < 0 || steps > 0xff)
diff --git a/drivers/iio/accel/st_accel_core.c b/drivers/iio/accel/st_accel_core.c
index 460aa58e0159..3e6fd5a8ac5b 100644
--- a/drivers/iio/accel/st_accel_core.c
+++ b/drivers/iio/accel/st_accel_core.c
@@ -951,7 +951,7 @@ int st_accel_common_probe(struct iio_dev *indio_dev)
 	if (!pdata)
 		pdata = (struct st_sensors_platform_data *)&default_accel_pdata;
 
-	err = st_sensors_init_sensor(indio_dev, adata->dev->platform_data);
+	err = st_sensors_init_sensor(indio_dev, pdata);
 	if (err < 0)
 		goto st_accel_power_off;
 
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index ef86296b8b0d..72bc2b71765a 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -158,6 +158,7 @@ config AT91_SAMA5D2_ADC
 	tristate "Atmel AT91 SAMA5D2 ADC"
 	depends on ARCH_AT91 || COMPILE_TEST
 	depends on HAS_IOMEM
+	depends on HAS_DMA
 	select IIO_TRIGGERED_BUFFER
 	help
 	  Say yes here to build support for Atmel SAMA5D2 ADC which is
@@ -318,6 +319,8 @@ config HI8435
 config HX711
 	tristate "AVIA HX711 ADC for weight cells"
 	depends on GPIOLIB
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	help
 	  If you say yes here you get support for AVIA HX711 ADC which is used
 	  for weigh cells
@@ -629,6 +632,18 @@ config SPEAR_ADC
 	  To compile this driver as a module, choose M here: the
 	  module will be called spear_adc.
 
+config SD_ADC_MODULATOR
+	tristate "Generic sigma delta modulator"
+	depends on OF
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	help
+	  Select this option to enables sigma delta modulator. This driver can
+	  support generic sigma delta modulators.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called sd_adc_modulator.
+
 config STM32_ADC_CORE
 	tristate "STMicroelectronics STM32 adc core"
 	depends on ARCH_STM32 || COMPILE_TEST
@@ -656,6 +671,31 @@ config STM32_ADC
 	  This driver can also be built as a module.  If so, the module
 	  will be called stm32-adc.
 
+config STM32_DFSDM_CORE
+	tristate "STMicroelectronics STM32 DFSDM core"
+	depends on (ARCH_STM32 && OF) || COMPILE_TEST
+	select REGMAP
+	select REGMAP_MMIO
+	help
+	  Select this option to enable the  driver for STMicroelectronics
+	  STM32 digital filter for sigma delta converter.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called stm32-dfsdm-core.
+
+config STM32_DFSDM_ADC
+	tristate "STMicroelectronics STM32 dfsdm adc"
+	depends on (ARCH_STM32 && OF) || COMPILE_TEST
+	select STM32_DFSDM_CORE
+	select REGMAP_MMIO
+	select IIO_BUFFER_HW_CONSUMER
+	help
+	  Select this option to support ADCSigma delta modulator for
+	  STMicroelectronics STM32 digital filter for sigma delta converter.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called stm32-dfsdm-adc.
+
 config STX104
 	tristate "Apex Embedded Systems STX104 driver"
 	depends on PC104 && X86 && ISA_BUS_API
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index 9572c1090f35..28a9423997f3 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -64,6 +64,8 @@ obj-$(CONFIG_STX104) += stx104.o
 obj-$(CONFIG_SUN4I_GPADC) += sun4i-gpadc-iio.o
 obj-$(CONFIG_STM32_ADC_CORE) += stm32-adc-core.o
 obj-$(CONFIG_STM32_ADC) += stm32-adc.o
+obj-$(CONFIG_STM32_DFSDM_CORE) += stm32-dfsdm-core.o
+obj-$(CONFIG_STM32_DFSDM_ADC) += stm32-dfsdm-adc.o
 obj-$(CONFIG_TI_ADC081C) += ti-adc081c.o
 obj-$(CONFIG_TI_ADC0832) += ti-adc0832.o
 obj-$(CONFIG_TI_ADC084S021) += ti-adc084s021.o
@@ -82,3 +84,4 @@ obj-$(CONFIG_VF610_ADC) += vf610_adc.o
 obj-$(CONFIG_VIPERBOARD_ADC) += viperboard_adc.o
 xilinx-xadc-y := xilinx-xadc-core.o xilinx-xadc-events.o
 obj-$(CONFIG_XILINX_XADC) += xilinx-xadc.o
+obj-$(CONFIG_SD_ADC_MODULATOR) += sd_adc_modulator.o
diff --git a/drivers/iio/adc/aspeed_adc.c b/drivers/iio/adc/aspeed_adc.c
index 8a958d5f1905..9515ca165dfd 100644
--- a/drivers/iio/adc/aspeed_adc.c
+++ b/drivers/iio/adc/aspeed_adc.c
@@ -17,6 +17,7 @@
 #include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/reset.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
@@ -53,11 +54,12 @@ struct aspeed_adc_model_data {
 };
 
 struct aspeed_adc_data {
-	struct device	*dev;
-	void __iomem	*base;
-	spinlock_t	clk_lock;
-	struct clk_hw	*clk_prescaler;
-	struct clk_hw	*clk_scaler;
+	struct device		*dev;
+	void __iomem		*base;
+	spinlock_t		clk_lock;
+	struct clk_hw		*clk_prescaler;
+	struct clk_hw		*clk_scaler;
+	struct reset_control	*rst;
 };
 
 #define ASPEED_CHAN(_idx, _data_reg_addr) {			\
@@ -217,6 +219,15 @@ static int aspeed_adc_probe(struct platform_device *pdev)
 		goto scaler_error;
 	}
 
+	data->rst = devm_reset_control_get_exclusive(&pdev->dev, NULL);
+	if (IS_ERR(data->rst)) {
+		dev_err(&pdev->dev,
+			"invalid or missing reset controller device tree entry");
+		ret = PTR_ERR(data->rst);
+		goto reset_error;
+	}
+	reset_control_deassert(data->rst);
+
 	model_data = of_device_get_match_data(&pdev->dev);
 
 	if (model_data->wait_init_sequence) {
@@ -232,7 +243,7 @@ static int aspeed_adc_probe(struct platform_device *pdev)
 					 ASPEED_ADC_INIT_POLLING_TIME,
 					 ASPEED_ADC_INIT_TIMEOUT);
 		if (ret)
-			goto scaler_error;
+			goto poll_timeout_error;
 	}
 
 	/* Start all channels in normal mode. */
@@ -264,8 +275,10 @@ static int aspeed_adc_probe(struct platform_device *pdev)
 		data->base + ASPEED_REG_ENGINE_CONTROL);
 	clk_disable_unprepare(data->clk_scaler->clk);
 clk_enable_error:
+poll_timeout_error:
+	reset_control_assert(data->rst);
+reset_error:
 	clk_hw_unregister_divider(data->clk_scaler);
-
 scaler_error:
 	clk_hw_unregister_divider(data->clk_prescaler);
 	return ret;
@@ -280,6 +293,7 @@ static int aspeed_adc_remove(struct platform_device *pdev)
 	writel(ASPEED_OPERATION_MODE_POWER_DOWN,
 		data->base + ASPEED_REG_ENGINE_CONTROL);
 	clk_disable_unprepare(data->clk_scaler->clk);
+	reset_control_assert(data->rst);
 	clk_hw_unregister_divider(data->clk_scaler);
 	clk_hw_unregister_divider(data->clk_prescaler);
 
diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c
index 755a493c2a2c..4eff8351ce29 100644
--- a/drivers/iio/adc/at91-sama5d2_adc.c
+++ b/drivers/iio/adc/at91-sama5d2_adc.c
@@ -16,6 +16,8 @@
 
 #include <linux/bitops.h>
 #include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
@@ -100,6 +102,8 @@
 #define AT91_SAMA5D2_LCDR	0x20
 /* Interrupt Enable Register */
 #define AT91_SAMA5D2_IER	0x24
+/* Interrupt Enable Register - general overrun error */
+#define AT91_SAMA5D2_IER_GOVRE BIT(25)
 /* Interrupt Disable Register */
 #define AT91_SAMA5D2_IDR	0x28
 /* Interrupt Mask Register */
@@ -167,13 +171,19 @@
 
 /*
  * Maximum number of bytes to hold conversion from all channels
- * plus the timestamp
+ * without the timestamp.
  */
-#define AT91_BUFFER_MAX_BYTES ((AT91_SAMA5D2_SINGLE_CHAN_CNT +		\
-				AT91_SAMA5D2_DIFF_CHAN_CNT) * 2 + 8)
+#define AT91_BUFFER_MAX_CONVERSION_BYTES ((AT91_SAMA5D2_SINGLE_CHAN_CNT + \
+					 AT91_SAMA5D2_DIFF_CHAN_CNT) * 2)
+
+/* This total must also include the timestamp */
+#define AT91_BUFFER_MAX_BYTES (AT91_BUFFER_MAX_CONVERSION_BYTES + 8)
 
 #define AT91_BUFFER_MAX_HWORDS (AT91_BUFFER_MAX_BYTES / 2)
 
+#define AT91_HWFIFO_MAX_SIZE_STR	"128"
+#define AT91_HWFIFO_MAX_SIZE		128
+
 #define AT91_SAMA5D2_CHAN_SINGLE(num, addr)				\
 	{								\
 		.type = IIO_VOLTAGE,					\
@@ -228,6 +238,28 @@ struct at91_adc_trigger {
 	bool				hw_trig;
 };
 
+/**
+ * at91_adc_dma - at91-sama5d2 dma information struct
+ * @dma_chan:		the dma channel acquired
+ * @rx_buf:		dma coherent allocated area
+ * @rx_dma_buf:		dma handler for the buffer
+ * @phys_addr:		physical address of the ADC base register
+ * @buf_idx:		index inside the dma buffer where reading was last done
+ * @rx_buf_sz:		size of buffer used by DMA operation
+ * @watermark:		number of conversions to copy before DMA triggers irq
+ * @dma_ts:		hold the start timestamp of dma operation
+ */
+struct at91_adc_dma {
+	struct dma_chan			*dma_chan;
+	u8				*rx_buf;
+	dma_addr_t			rx_dma_buf;
+	phys_addr_t			phys_addr;
+	int				buf_idx;
+	int				rx_buf_sz;
+	int				watermark;
+	s64				dma_ts;
+};
+
 struct at91_adc_state {
 	void __iomem			*base;
 	int				irq;
@@ -242,6 +274,7 @@ struct at91_adc_state {
 	u32				conversion_value;
 	struct at91_adc_soc_info	soc_info;
 	wait_queue_head_t		wq_data_available;
+	struct at91_adc_dma		dma_st;
 	u16				buffer[AT91_BUFFER_MAX_HWORDS];
 	/*
 	 * lock to prevent concurrent 'single conversion' requests through
@@ -322,11 +355,17 @@ static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state)
 		if (state) {
 			at91_adc_writel(st, AT91_SAMA5D2_CHER,
 					BIT(chan->channel));
-			at91_adc_writel(st, AT91_SAMA5D2_IER,
-					BIT(chan->channel));
+			/* enable irq only if not using DMA */
+			if (!st->dma_st.dma_chan) {
+				at91_adc_writel(st, AT91_SAMA5D2_IER,
+						BIT(chan->channel));
+			}
 		} else {
-			at91_adc_writel(st, AT91_SAMA5D2_IDR,
-					BIT(chan->channel));
+			/* disable irq only if not using DMA */
+			if (!st->dma_st.dma_chan) {
+				at91_adc_writel(st, AT91_SAMA5D2_IDR,
+						BIT(chan->channel));
+			}
 			at91_adc_writel(st, AT91_SAMA5D2_CHDR,
 					BIT(chan->channel));
 		}
@@ -340,6 +379,10 @@ static int at91_adc_reenable_trigger(struct iio_trigger *trig)
 	struct iio_dev *indio = iio_trigger_get_drvdata(trig);
 	struct at91_adc_state *st = iio_priv(indio);
 
+	/* if we are using DMA, we must not reenable irq after each trigger */
+	if (st->dma_st.dma_chan)
+		return 0;
+
 	enable_irq(st->irq);
 
 	/* Needed to ACK the DRDY interruption */
@@ -350,6 +393,153 @@ static int at91_adc_reenable_trigger(struct iio_trigger *trig)
 static const struct iio_trigger_ops at91_adc_trigger_ops = {
 	.set_trigger_state = &at91_adc_configure_trigger,
 	.try_reenable = &at91_adc_reenable_trigger,
+	.validate_device = iio_trigger_validate_own_device,
+};
+
+static int at91_adc_dma_size_done(struct at91_adc_state *st)
+{
+	struct dma_tx_state state;
+	enum dma_status status;
+	int i, size;
+
+	status = dmaengine_tx_status(st->dma_st.dma_chan,
+				     st->dma_st.dma_chan->cookie,
+				     &state);
+	if (status != DMA_IN_PROGRESS)
+		return 0;
+
+	/* Transferred length is size in bytes from end of buffer */
+	i = st->dma_st.rx_buf_sz - state.residue;
+
+	/* Return available bytes */
+	if (i >= st->dma_st.buf_idx)
+		size = i - st->dma_st.buf_idx;
+	else
+		size = st->dma_st.rx_buf_sz + i - st->dma_st.buf_idx;
+	return size;
+}
+
+static void at91_dma_buffer_done(void *data)
+{
+	struct iio_dev *indio_dev = data;
+
+	iio_trigger_poll_chained(indio_dev->trig);
+}
+
+static int at91_adc_dma_start(struct iio_dev *indio_dev)
+{
+	struct at91_adc_state *st = iio_priv(indio_dev);
+	struct dma_async_tx_descriptor *desc;
+	dma_cookie_t cookie;
+	int ret;
+	u8 bit;
+
+	if (!st->dma_st.dma_chan)
+		return 0;
+
+	/* we start a new DMA, so set buffer index to start */
+	st->dma_st.buf_idx = 0;
+
+	/*
+	 * compute buffer size w.r.t. watermark and enabled channels.
+	 * scan_bytes is aligned so we need an exact size for DMA
+	 */
+	st->dma_st.rx_buf_sz = 0;
+
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->num_channels) {
+		struct iio_chan_spec const *chan = indio_dev->channels + bit;
+
+		st->dma_st.rx_buf_sz += chan->scan_type.storagebits / 8;
+	}
+	st->dma_st.rx_buf_sz *= st->dma_st.watermark;
+
+	/* Prepare a DMA cyclic transaction */
+	desc = dmaengine_prep_dma_cyclic(st->dma_st.dma_chan,
+					 st->dma_st.rx_dma_buf,
+					 st->dma_st.rx_buf_sz,
+					 st->dma_st.rx_buf_sz / 2,
+					 DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT);
+
+	if (!desc) {
+		dev_err(&indio_dev->dev, "cannot prepare DMA cyclic\n");
+		return -EBUSY;
+	}
+
+	desc->callback = at91_dma_buffer_done;
+	desc->callback_param = indio_dev;
+
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret) {
+		dev_err(&indio_dev->dev, "cannot submit DMA cyclic\n");
+		dmaengine_terminate_async(st->dma_st.dma_chan);
+		return ret;
+	}
+
+	/* enable general overrun error signaling */
+	at91_adc_writel(st, AT91_SAMA5D2_IER, AT91_SAMA5D2_IER_GOVRE);
+	/* Issue pending DMA requests */
+	dma_async_issue_pending(st->dma_st.dma_chan);
+
+	/* consider current time as DMA start time for timestamps */
+	st->dma_st.dma_ts = iio_get_time_ns(indio_dev);
+
+	dev_dbg(&indio_dev->dev, "DMA cyclic started\n");
+
+	return 0;
+}
+
+static int at91_adc_buffer_postenable(struct iio_dev *indio_dev)
+{
+	int ret;
+
+	ret = at91_adc_dma_start(indio_dev);
+	if (ret) {
+		dev_err(&indio_dev->dev, "buffer postenable failed\n");
+		return ret;
+	}
+
+	return iio_triggered_buffer_postenable(indio_dev);
+}
+
+static int at91_adc_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct at91_adc_state *st = iio_priv(indio_dev);
+	int ret;
+	u8 bit;
+
+	ret = iio_triggered_buffer_predisable(indio_dev);
+	if (ret < 0)
+		dev_err(&indio_dev->dev, "buffer predisable failed\n");
+
+	if (!st->dma_st.dma_chan)
+		return ret;
+
+	/* if we are using DMA we must clear registers and end DMA */
+	dmaengine_terminate_sync(st->dma_st.dma_chan);
+
+	/*
+	 * For each enabled channel we must read the last converted value
+	 * to clear EOC status and not get a possible interrupt later.
+	 * This value is being read by DMA from LCDR anyway
+	 */
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->num_channels) {
+		struct iio_chan_spec const *chan = indio_dev->channels + bit;
+
+		if (st->dma_st.dma_chan)
+			at91_adc_readl(st, chan->address);
+	}
+
+	/* read overflow register to clear possible overflow status */
+	at91_adc_readl(st, AT91_SAMA5D2_OVER);
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops at91_buffer_setup_ops = {
+	.postenable = &at91_adc_buffer_postenable,
+	.predisable = &at91_adc_buffer_predisable,
 };
 
 static struct iio_trigger *at91_adc_allocate_trigger(struct iio_dev *indio,
@@ -388,24 +578,77 @@ static int at91_adc_trigger_init(struct iio_dev *indio)
 	return 0;
 }
 
-static irqreturn_t at91_adc_trigger_handler(int irq, void *p)
+static void at91_adc_trigger_handler_nodma(struct iio_dev *indio_dev,
+					   struct iio_poll_func *pf)
 {
-	struct iio_poll_func *pf = p;
-	struct iio_dev *indio = pf->indio_dev;
-	struct at91_adc_state *st = iio_priv(indio);
+	struct at91_adc_state *st = iio_priv(indio_dev);
 	int i = 0;
 	u8 bit;
 
-	for_each_set_bit(bit, indio->active_scan_mask, indio->num_channels) {
-		struct iio_chan_spec const *chan = indio->channels + bit;
+	for_each_set_bit(bit, indio_dev->active_scan_mask,
+			 indio_dev->num_channels) {
+		struct iio_chan_spec const *chan = indio_dev->channels + bit;
 
 		st->buffer[i] = at91_adc_readl(st, chan->address);
 		i++;
 	}
+	iio_push_to_buffers_with_timestamp(indio_dev, st->buffer,
+					   pf->timestamp);
+}
 
-	iio_push_to_buffers_with_timestamp(indio, st->buffer, pf->timestamp);
+static void at91_adc_trigger_handler_dma(struct iio_dev *indio_dev)
+{
+	struct at91_adc_state *st = iio_priv(indio_dev);
+	int transferred_len = at91_adc_dma_size_done(st);
+	s64 ns = iio_get_time_ns(indio_dev);
+	s64 interval;
+	int sample_index = 0, sample_count, sample_size;
+
+	u32 status = at91_adc_readl(st, AT91_SAMA5D2_ISR);
+	/* if we reached this point, we cannot sample faster */
+	if (status & AT91_SAMA5D2_IER_GOVRE)
+		pr_info_ratelimited("%s: conversion overrun detected\n",
+				    indio_dev->name);
 
-	iio_trigger_notify_done(indio->trig);
+	sample_size = div_s64(st->dma_st.rx_buf_sz, st->dma_st.watermark);
+
+	sample_count = div_s64(transferred_len, sample_size);
+
+	/*
+	 * interval between samples is total time since last transfer handling
+	 * divided by the number of samples (total size divided by sample size)
+	 */
+	interval = div_s64((ns - st->dma_st.dma_ts), sample_count);
+
+	while (transferred_len >= sample_size) {
+		iio_push_to_buffers_with_timestamp(indio_dev,
+				(st->dma_st.rx_buf + st->dma_st.buf_idx),
+				(st->dma_st.dma_ts + interval * sample_index));
+		/* adjust remaining length */
+		transferred_len -= sample_size;
+		/* adjust buffer index */
+		st->dma_st.buf_idx += sample_size;
+		/* in case of reaching end of buffer, reset index */
+		if (st->dma_st.buf_idx >= st->dma_st.rx_buf_sz)
+			st->dma_st.buf_idx = 0;
+		sample_index++;
+	}
+	/* adjust saved time for next transfer handling */
+	st->dma_st.dma_ts = iio_get_time_ns(indio_dev);
+}
+
+static irqreturn_t at91_adc_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct at91_adc_state *st = iio_priv(indio_dev);
+
+	if (st->dma_st.dma_chan)
+		at91_adc_trigger_handler_dma(indio_dev);
+	else
+		at91_adc_trigger_handler_nodma(indio_dev, pf);
+
+	iio_trigger_notify_done(indio_dev->trig);
 
 	return IRQ_HANDLED;
 }
@@ -414,7 +657,7 @@ static int at91_adc_buffer_init(struct iio_dev *indio)
 {
 	return devm_iio_triggered_buffer_setup(&indio->dev, indio,
 			&iio_pollfunc_store_time,
-			&at91_adc_trigger_handler, NULL);
+			&at91_adc_trigger_handler, &at91_buffer_setup_ops);
 }
 
 static unsigned at91_adc_startup_time(unsigned startup_time_min,
@@ -485,10 +728,13 @@ static irqreturn_t at91_adc_interrupt(int irq, void *private)
 	if (!(status & imr))
 		return IRQ_NONE;
 
-	if (iio_buffer_enabled(indio)) {
+	if (iio_buffer_enabled(indio) && !st->dma_st.dma_chan) {
 		disable_irq_nosync(irq);
 		iio_trigger_poll(indio->trig);
-	} else {
+	} else if (iio_buffer_enabled(indio) && st->dma_st.dma_chan) {
+		disable_irq_nosync(irq);
+		WARN(true, "Unexpected irq occurred\n");
+	} else if (!iio_buffer_enabled(indio)) {
 		st->conversion_value = at91_adc_readl(st, st->chan->address);
 		st->conversion_done = true;
 		wake_up_interruptible(&st->wq_data_available);
@@ -510,7 +756,6 @@ static int at91_adc_read_raw(struct iio_dev *indio_dev,
 		ret = iio_device_claim_direct_mode(indio_dev);
 		if (ret)
 			return ret;
-
 		mutex_lock(&st->lock);
 
 		st->chan = chan;
@@ -541,6 +786,9 @@ static int at91_adc_read_raw(struct iio_dev *indio_dev,
 		at91_adc_writel(st, AT91_SAMA5D2_IDR, BIT(chan->channel));
 		at91_adc_writel(st, AT91_SAMA5D2_CHDR, BIT(chan->channel));
 
+		/* Needed to ACK the DRDY interruption */
+		at91_adc_readl(st, AT91_SAMA5D2_LCDR);
+
 		mutex_unlock(&st->lock);
 
 		iio_device_release_direct_mode(indio_dev);
@@ -580,9 +828,123 @@ static int at91_adc_write_raw(struct iio_dev *indio_dev,
 	return 0;
 }
 
+static void at91_adc_dma_init(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct at91_adc_state *st = iio_priv(indio_dev);
+	struct dma_slave_config config = {0};
+	/*
+	 * We make the buffer double the size of the fifo,
+	 * such that DMA uses one half of the buffer (full fifo size)
+	 * and the software uses the other half to read/write.
+	 */
+	unsigned int pages = DIV_ROUND_UP(AT91_HWFIFO_MAX_SIZE *
+					  AT91_BUFFER_MAX_CONVERSION_BYTES * 2,
+					  PAGE_SIZE);
+
+	if (st->dma_st.dma_chan)
+		return;
+
+	st->dma_st.dma_chan = dma_request_slave_channel(&pdev->dev, "rx");
+
+	if (!st->dma_st.dma_chan)  {
+		dev_info(&pdev->dev, "can't get DMA channel\n");
+		goto dma_exit;
+	}
+
+	st->dma_st.rx_buf = dma_alloc_coherent(st->dma_st.dma_chan->device->dev,
+					       pages * PAGE_SIZE,
+					       &st->dma_st.rx_dma_buf,
+					       GFP_KERNEL);
+	if (!st->dma_st.rx_buf) {
+		dev_info(&pdev->dev, "can't allocate coherent DMA area\n");
+		goto dma_chan_disable;
+	}
+
+	/* Configure DMA channel to read data register */
+	config.direction = DMA_DEV_TO_MEM;
+	config.src_addr = (phys_addr_t)(st->dma_st.phys_addr
+			  + AT91_SAMA5D2_LCDR);
+	config.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+	config.src_maxburst = 1;
+	config.dst_maxburst = 1;
+
+	if (dmaengine_slave_config(st->dma_st.dma_chan, &config)) {
+		dev_info(&pdev->dev, "can't configure DMA slave\n");
+		goto dma_free_area;
+	}
+
+	dev_info(&pdev->dev, "using %s for rx DMA transfers\n",
+		 dma_chan_name(st->dma_st.dma_chan));
+
+	return;
+
+dma_free_area:
+	dma_free_coherent(st->dma_st.dma_chan->device->dev, pages * PAGE_SIZE,
+			  st->dma_st.rx_buf, st->dma_st.rx_dma_buf);
+dma_chan_disable:
+	dma_release_channel(st->dma_st.dma_chan);
+	st->dma_st.dma_chan = 0;
+dma_exit:
+	dev_info(&pdev->dev, "continuing without DMA support\n");
+}
+
+static void at91_adc_dma_disable(struct platform_device *pdev)
+{
+	struct iio_dev *indio_dev = platform_get_drvdata(pdev);
+	struct at91_adc_state *st = iio_priv(indio_dev);
+	unsigned int pages = DIV_ROUND_UP(AT91_HWFIFO_MAX_SIZE *
+					  AT91_BUFFER_MAX_CONVERSION_BYTES * 2,
+					  PAGE_SIZE);
+
+	/* if we are not using DMA, just return */
+	if (!st->dma_st.dma_chan)
+		return;
+
+	/* wait for all transactions to be terminated first*/
+	dmaengine_terminate_sync(st->dma_st.dma_chan);
+
+	dma_free_coherent(st->dma_st.dma_chan->device->dev, pages * PAGE_SIZE,
+			  st->dma_st.rx_buf, st->dma_st.rx_dma_buf);
+	dma_release_channel(st->dma_st.dma_chan);
+	st->dma_st.dma_chan = 0;
+
+	dev_info(&pdev->dev, "continuing without DMA support\n");
+}
+
+static int at91_adc_set_watermark(struct iio_dev *indio_dev, unsigned int val)
+{
+	struct at91_adc_state *st = iio_priv(indio_dev);
+
+	if (val > AT91_HWFIFO_MAX_SIZE)
+		return -EINVAL;
+
+	if (!st->selected_trig->hw_trig) {
+		dev_dbg(&indio_dev->dev, "we need hw trigger for DMA\n");
+		return 0;
+	}
+
+	dev_dbg(&indio_dev->dev, "new watermark is %u\n", val);
+	st->dma_st.watermark = val;
+
+	/*
+	 * The logic here is: if we have watermark 1, it means we do
+	 * each conversion with it's own IRQ, thus we don't need DMA.
+	 * If the watermark is higher, we do DMA to do all the transfers in bulk
+	 */
+
+	if (val == 1)
+		at91_adc_dma_disable(to_platform_device(&indio_dev->dev));
+	else if (val > 1)
+		at91_adc_dma_init(to_platform_device(&indio_dev->dev));
+
+	return 0;
+}
+
 static const struct iio_info at91_adc_info = {
 	.read_raw = &at91_adc_read_raw,
 	.write_raw = &at91_adc_write_raw,
+	.hwfifo_set_watermark = &at91_adc_set_watermark,
 };
 
 static void at91_adc_hw_init(struct at91_adc_state *st)
@@ -599,6 +961,42 @@ static void at91_adc_hw_init(struct at91_adc_state *st)
 	at91_adc_setup_samp_freq(st, st->soc_info.min_sample_rate);
 }
 
+static ssize_t at91_adc_get_fifo_state(struct device *dev,
+				       struct device_attribute *attr, char *buf)
+{
+	struct iio_dev *indio_dev =
+			platform_get_drvdata(to_platform_device(dev));
+	struct at91_adc_state *st = iio_priv(indio_dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n", !!st->dma_st.dma_chan);
+}
+
+static ssize_t at91_adc_get_watermark(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct iio_dev *indio_dev =
+			platform_get_drvdata(to_platform_device(dev));
+	struct at91_adc_state *st = iio_priv(indio_dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n", st->dma_st.watermark);
+}
+
+static IIO_DEVICE_ATTR(hwfifo_enabled, 0444,
+		       at91_adc_get_fifo_state, NULL, 0);
+static IIO_DEVICE_ATTR(hwfifo_watermark, 0444,
+		       at91_adc_get_watermark, NULL, 0);
+
+static IIO_CONST_ATTR(hwfifo_watermark_min, "2");
+static IIO_CONST_ATTR(hwfifo_watermark_max, AT91_HWFIFO_MAX_SIZE_STR);
+
+static const struct attribute *at91_adc_fifo_attributes[] = {
+	&iio_const_attr_hwfifo_watermark_min.dev_attr.attr,
+	&iio_const_attr_hwfifo_watermark_max.dev_attr.attr,
+	&iio_dev_attr_hwfifo_watermark.dev_attr.attr,
+	&iio_dev_attr_hwfifo_enabled.dev_attr.attr,
+	NULL,
+};
+
 static int at91_adc_probe(struct platform_device *pdev)
 {
 	struct iio_dev *indio_dev;
@@ -674,6 +1072,9 @@ static int at91_adc_probe(struct platform_device *pdev)
 	if (!res)
 		return -EINVAL;
 
+	/* if we plan to use DMA, we need the physical address of the regs */
+	st->dma_st.phys_addr = res->start;
+
 	st->base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(st->base))
 		return PTR_ERR(st->base);
@@ -737,11 +1138,22 @@ static int at91_adc_probe(struct platform_device *pdev)
 			dev_err(&pdev->dev, "couldn't setup the triggers.\n");
 			goto per_clk_disable_unprepare;
 		}
+		/*
+		 * Initially the iio buffer has a length of 2 and
+		 * a watermark of 1
+		 */
+		st->dma_st.watermark = 1;
+
+		iio_buffer_set_attrs(indio_dev->buffer,
+				     at91_adc_fifo_attributes);
 	}
 
+	if (dma_coerce_mask_and_coherent(&indio_dev->dev, DMA_BIT_MASK(32)))
+		dev_info(&pdev->dev, "cannot set DMA mask to 32-bit\n");
+
 	ret = iio_device_register(indio_dev);
 	if (ret < 0)
-		goto per_clk_disable_unprepare;
+		goto dma_disable;
 
 	if (st->selected_trig->hw_trig)
 		dev_info(&pdev->dev, "setting up trigger as %s\n",
@@ -752,6 +1164,8 @@ static int at91_adc_probe(struct platform_device *pdev)
 
 	return 0;
 
+dma_disable:
+	at91_adc_dma_disable(pdev);
 per_clk_disable_unprepare:
 	clk_disable_unprepare(st->per_clk);
 vref_disable:
@@ -768,6 +1182,8 @@ static int at91_adc_remove(struct platform_device *pdev)
 
 	iio_device_unregister(indio_dev);
 
+	at91_adc_dma_disable(pdev);
+
 	clk_disable_unprepare(st->per_clk);
 
 	regulator_disable(st->vref);
diff --git a/drivers/iio/adc/at91_adc.c b/drivers/iio/adc/at91_adc.c
index 3836d4222a3e..71a5ee652b79 100644
--- a/drivers/iio/adc/at91_adc.c
+++ b/drivers/iio/adc/at91_adc.c
@@ -1177,9 +1177,9 @@ static int at91_adc_probe(struct platform_device *pdev)
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 
 	st->reg_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(st->reg_base)) {
+	if (IS_ERR(st->reg_base))
 		return PTR_ERR(st->reg_base);
-	}
+
 
 	/*
 	 * Disable all IRQs before setting up the handler
diff --git a/drivers/iio/adc/axp288_adc.c b/drivers/iio/adc/axp288_adc.c
index 60c9e853dd81..031d568b4972 100644
--- a/drivers/iio/adc/axp288_adc.c
+++ b/drivers/iio/adc/axp288_adc.c
@@ -92,22 +92,14 @@ static const struct iio_chan_spec axp288_adc_channels[] = {
 	},
 };
 
-#define AXP288_ADC_MAP(_adc_channel_label, _consumer_dev_name,	\
-		_consumer_channel)				\
-	{							\
-		.adc_channel_label = _adc_channel_label,	\
-		.consumer_dev_name = _consumer_dev_name,	\
-		.consumer_channel = _consumer_channel,		\
-	}
-
 /* for consumer drivers */
 static struct iio_map axp288_adc_default_maps[] = {
-	AXP288_ADC_MAP("TS_PIN", "axp288-batt", "axp288-batt-temp"),
-	AXP288_ADC_MAP("PMIC_TEMP", "axp288-pmic", "axp288-pmic-temp"),
-	AXP288_ADC_MAP("GPADC", "axp288-gpadc", "axp288-system-temp"),
-	AXP288_ADC_MAP("BATT_CHG_I", "axp288-chrg", "axp288-chrg-curr"),
-	AXP288_ADC_MAP("BATT_DISCHRG_I", "axp288-chrg", "axp288-chrg-d-curr"),
-	AXP288_ADC_MAP("BATT_V", "axp288-batt", "axp288-batt-volt"),
+	IIO_MAP("TS_PIN", "axp288-batt", "axp288-batt-temp"),
+	IIO_MAP("PMIC_TEMP", "axp288-pmic", "axp288-pmic-temp"),
+	IIO_MAP("GPADC", "axp288-gpadc", "axp288-system-temp"),
+	IIO_MAP("BATT_CHG_I", "axp288-chrg", "axp288-chrg-curr"),
+	IIO_MAP("BATT_DISCHRG_I", "axp288-chrg", "axp288-chrg-d-curr"),
+	IIO_MAP("BATT_V", "axp288-batt", "axp288-batt-volt"),
 	{},
 };
 
diff --git a/drivers/iio/adc/hx711.c b/drivers/iio/adc/hx711.c
index d10b9f13d557..9430b54121e0 100644
--- a/drivers/iio/adc/hx711.c
+++ b/drivers/iio/adc/hx711.c
@@ -24,6 +24,9 @@
 #include <linux/delay.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 #include <linux/gpio/consumer.h>
 #include <linux/regulator/consumer.h>
 
@@ -89,6 +92,11 @@ struct hx711_data {
 	int			gain_set;	/* gain set on device */
 	int			gain_chan_a;	/* gain for channel A */
 	struct mutex		lock;
+	/*
+	 * triggered buffer
+	 * 2x32-bit channel + 64-bit timestamp
+	 */
+	u32			buffer[4];
 };
 
 static int hx711_cycle(struct hx711_data *hx711_data)
@@ -145,15 +153,16 @@ static int hx711_wait_for_ready(struct hx711_data *hx711_data)
 	int i, val;
 
 	/*
-	 * a maximum reset cycle time of 56 ms was measured.
-	 * we round it up to 100 ms
+	 * in some rare cases the reset takes quite a long time
+	 * especially when the channel is changed.
+	 * Allow up to one second for it
 	 */
 	for (i = 0; i < 100; i++) {
 		val = gpiod_get_value(hx711_data->gpiod_dout);
 		if (!val)
 			break;
-		/* sleep at least 1 ms */
-		msleep(1);
+		/* sleep at least 10 ms */
+		msleep(10);
 	}
 	if (val)
 		return -EIO;
@@ -195,9 +204,7 @@ static int hx711_reset(struct hx711_data *hx711_data)
 		 * after a dummy read we need to wait vor readiness
 		 * for not mixing gain pulses with the clock
 		 */
-		ret = hx711_wait_for_ready(hx711_data);
-		if (ret)
-			return ret;
+		val = hx711_wait_for_ready(hx711_data);
 	}
 
 	return val;
@@ -236,34 +243,40 @@ static int hx711_set_gain_for_channel(struct hx711_data *hx711_data, int chan)
 	return 0;
 }
 
+static int hx711_reset_read(struct hx711_data *hx711_data, int chan)
+{
+	int ret;
+	int val;
+
+	/*
+	 * hx711_reset() must be called from here
+	 * because it could be calling hx711_read() by itself
+	 */
+	if (hx711_reset(hx711_data)) {
+		dev_err(hx711_data->dev, "reset failed!");
+		return -EIO;
+	}
+
+	ret = hx711_set_gain_for_channel(hx711_data, chan);
+	if (ret < 0)
+		return ret;
+
+	val = hx711_read(hx711_data);
+
+	return val;
+}
+
 static int hx711_read_raw(struct iio_dev *indio_dev,
 				const struct iio_chan_spec *chan,
 				int *val, int *val2, long mask)
 {
 	struct hx711_data *hx711_data = iio_priv(indio_dev);
-	int ret;
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
 		mutex_lock(&hx711_data->lock);
 
-		/*
-		 * hx711_reset() must be called from here
-		 * because it could be calling hx711_read() by itself
-		 */
-		if (hx711_reset(hx711_data)) {
-			mutex_unlock(&hx711_data->lock);
-			dev_err(hx711_data->dev, "reset failed!");
-			return -EIO;
-		}
-
-		ret = hx711_set_gain_for_channel(hx711_data, chan->channel);
-		if (ret < 0) {
-			mutex_unlock(&hx711_data->lock);
-			return ret;
-		}
-
-		*val = hx711_read(hx711_data);
+		*val = hx711_reset_read(hx711_data, chan->channel);
 
 		mutex_unlock(&hx711_data->lock);
 
@@ -339,6 +352,36 @@ static int hx711_write_raw_get_fmt(struct iio_dev *indio_dev,
 	return IIO_VAL_INT_PLUS_NANO;
 }
 
+static irqreturn_t hx711_trigger(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct hx711_data *hx711_data = iio_priv(indio_dev);
+	int i, j = 0;
+
+	mutex_lock(&hx711_data->lock);
+
+	memset(hx711_data->buffer, 0, sizeof(hx711_data->buffer));
+
+	for (i = 0; i < indio_dev->masklength; i++) {
+		if (!test_bit(i, indio_dev->active_scan_mask))
+			continue;
+
+		hx711_data->buffer[j] = hx711_reset_read(hx711_data,
+					indio_dev->channels[i].channel);
+		j++;
+	}
+
+	iio_push_to_buffers_with_timestamp(indio_dev, hx711_data->buffer,
+							pf->timestamp);
+
+	mutex_unlock(&hx711_data->lock);
+
+	iio_trigger_notify_done(indio_dev->trig);
+
+	return IRQ_HANDLED;
+}
+
 static ssize_t hx711_scale_available_show(struct device *dev,
 				struct device_attribute *attr,
 				char *buf)
@@ -387,6 +430,13 @@ static const struct iio_chan_spec hx711_chan_spec[] = {
 		.indexed = 1,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 24,
+			.storagebits = 32,
+			.endianness = IIO_CPU,
+		},
 	},
 	{
 		.type = IIO_VOLTAGE,
@@ -394,7 +444,15 @@ static const struct iio_chan_spec hx711_chan_spec[] = {
 		.indexed = 1,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = 1,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 24,
+			.storagebits = 32,
+			.endianness = IIO_CPU,
+		},
 	},
+	IIO_CHAN_SOFT_TIMESTAMP(2),
 };
 
 static int hx711_probe(struct platform_device *pdev)
@@ -459,10 +517,9 @@ static int hx711_probe(struct platform_device *pdev)
 	 * 1 LSB = (AVDD * 100) / GAIN / 1678 [10^-9 mV]
 	 */
 	ret = regulator_get_voltage(hx711_data->reg_avdd);
-	if (ret < 0) {
-		regulator_disable(hx711_data->reg_avdd);
-		return ret;
-	}
+	if (ret < 0)
+		goto error_regulator;
+
 	/* we need 10^-9 mV */
 	ret *= 100;
 
@@ -482,12 +539,27 @@ static int hx711_probe(struct platform_device *pdev)
 	indio_dev->channels = hx711_chan_spec;
 	indio_dev->num_channels = ARRAY_SIZE(hx711_chan_spec);
 
+	ret = iio_triggered_buffer_setup(indio_dev, iio_pollfunc_store_time,
+							hx711_trigger, NULL);
+	if (ret < 0) {
+		dev_err(dev, "setup of iio triggered buffer failed\n");
+		goto error_regulator;
+	}
+
 	ret = iio_device_register(indio_dev);
 	if (ret < 0) {
 		dev_err(dev, "Couldn't register the device\n");
-		regulator_disable(hx711_data->reg_avdd);
+		goto error_buffer;
 	}
 
+	return 0;
+
+error_buffer:
+	iio_triggered_buffer_cleanup(indio_dev);
+
+error_regulator:
+	regulator_disable(hx711_data->reg_avdd);
+
 	return ret;
 }
 
@@ -501,6 +573,8 @@ static int hx711_remove(struct platform_device *pdev)
 
 	iio_device_unregister(indio_dev);
 
+	iio_triggered_buffer_cleanup(indio_dev);
+
 	regulator_disable(hx711_data->reg_avdd);
 
 	return 0;
diff --git a/drivers/iio/adc/ina2xx-adc.c b/drivers/iio/adc/ina2xx-adc.c
index 84a43871f7dc..0635a79864bf 100644
--- a/drivers/iio/adc/ina2xx-adc.c
+++ b/drivers/iio/adc/ina2xx-adc.c
@@ -44,13 +44,14 @@
 
 #define INA226_MASK_ENABLE		0x06
 #define INA226_CVRF			BIT(3)
-#define INA219_CNVR			BIT(1)
 
 #define INA2XX_MAX_REGISTERS            8
 
 /* settings - depend on use case */
-#define INA219_CONFIG_DEFAULT           0x399F	/* PGA=8 */
+#define INA219_CONFIG_DEFAULT           0x399F	/* PGA=1/8, BRNG=32V */
 #define INA219_DEFAULT_IT		532
+#define INA219_DEFAULT_BRNG             1   /* 32V */
+#define INA219_DEFAULT_PGA              125 /* 1000/8 */
 #define INA226_CONFIG_DEFAULT           0x4327
 #define INA226_DEFAULT_AVG              4
 #define INA226_DEFAULT_IT		1110
@@ -63,6 +64,14 @@
  */
 #define INA2XX_MODE_MASK	GENMASK(3, 0)
 
+/* Gain for VShunt: 1/8 (default), 1/4, 1/2, 1 */
+#define INA219_PGA_MASK		GENMASK(12, 11)
+#define INA219_SHIFT_PGA(val)	((val) << 11)
+
+/* VBus range: 32V (default), 16V */
+#define INA219_BRNG_MASK	BIT(13)
+#define INA219_SHIFT_BRNG(val)	((val) << 13)
+
 /* Averaging for VBus/VShunt/Power */
 #define INA226_AVG_MASK		GENMASK(11, 9)
 #define INA226_SHIFT_AVG(val)	((val) << 9)
@@ -79,6 +88,11 @@
 #define INA226_ITS_MASK		GENMASK(5, 3)
 #define INA226_SHIFT_ITS(val)	((val) << 3)
 
+/* INA219 Bus voltage register, low bits are flags */
+#define INA219_OVF		BIT(0)
+#define INA219_CNVR		BIT(1)
+#define INA219_BUS_VOLTAGE_SHIFT	3
+
 /* Cosmetic macro giving the sampling period for a full P=UxI cycle */
 #define SAMPLING_PERIOD(c)	((c->int_time_vbus + c->int_time_vshunt) \
 				 * c->avg)
@@ -110,11 +124,12 @@ enum ina2xx_ids { ina219, ina226 };
 
 struct ina2xx_config {
 	u16 config_default;
-	int calibration_factor;
-	int shunt_div;
-	int bus_voltage_shift;
+	int calibration_value;
+	int shunt_voltage_lsb;	/* nV */
+	int bus_voltage_shift;	/* position of lsb */
 	int bus_voltage_lsb;	/* uV */
-	int power_lsb;		/* uW */
+	/* fixed relation between current and power lsb, uW/uA */
+	int power_lsb_factor;
 	enum ina2xx_ids chip_id;
 };
 
@@ -127,26 +142,28 @@ struct ina2xx_chip_info {
 	int avg;
 	int int_time_vbus; /* Bus voltage integration time uS */
 	int int_time_vshunt; /* Shunt voltage integration time uS */
+	int range_vbus; /* Bus voltage maximum in V */
+	int pga_gain_vshunt; /* Shunt voltage PGA gain */
 	bool allow_async_readout;
 };
 
 static const struct ina2xx_config ina2xx_config[] = {
 	[ina219] = {
 		.config_default = INA219_CONFIG_DEFAULT,
-		.calibration_factor = 40960000,
-		.shunt_div = 100,
-		.bus_voltage_shift = 3,
+		.calibration_value = 4096,
+		.shunt_voltage_lsb = 10000,
+		.bus_voltage_shift = INA219_BUS_VOLTAGE_SHIFT,
 		.bus_voltage_lsb = 4000,
-		.power_lsb = 20000,
+		.power_lsb_factor = 20,
 		.chip_id = ina219,
 	},
 	[ina226] = {
 		.config_default = INA226_CONFIG_DEFAULT,
-		.calibration_factor = 5120000,
-		.shunt_div = 400,
+		.calibration_value = 2048,
+		.shunt_voltage_lsb = 2500,
 		.bus_voltage_shift = 0,
 		.bus_voltage_lsb = 1250,
-		.power_lsb = 25000,
+		.power_lsb_factor = 25,
 		.chip_id = ina226,
 	},
 };
@@ -170,6 +187,9 @@ static int ina2xx_read_raw(struct iio_dev *indio_dev,
 		else
 			*val  = regval;
 
+		if (chan->address == INA2XX_BUS_VOLTAGE)
+			*val >>= chip->config->bus_voltage_shift;
+
 		return IIO_VAL_INT;
 
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
@@ -197,26 +217,48 @@ static int ina2xx_read_raw(struct iio_dev *indio_dev,
 	case IIO_CHAN_INFO_SCALE:
 		switch (chan->address) {
 		case INA2XX_SHUNT_VOLTAGE:
-			/* processed (mV) = raw/shunt_div */
-			*val2 = chip->config->shunt_div;
-			*val = 1;
+			/* processed (mV) = raw * lsb(nV) / 1000000 */
+			*val = chip->config->shunt_voltage_lsb;
+			*val2 = 1000000;
 			return IIO_VAL_FRACTIONAL;
 
 		case INA2XX_BUS_VOLTAGE:
-			/* processed (mV) = raw*lsb (uV) / (1000 << shift) */
+			/* processed (mV) = raw * lsb (uV) / 1000 */
 			*val = chip->config->bus_voltage_lsb;
-			*val2 = 1000 << chip->config->bus_voltage_shift;
+			*val2 = 1000;
+			return IIO_VAL_FRACTIONAL;
+
+		case INA2XX_CURRENT:
+			/*
+			 * processed (mA) = raw * current_lsb (mA)
+			 * current_lsb (mA) = shunt_voltage_lsb (nV) /
+			 *                    shunt_resistor (uOhm)
+			 */
+			*val = chip->config->shunt_voltage_lsb;
+			*val2 = chip->shunt_resistor_uohm;
 			return IIO_VAL_FRACTIONAL;
 
 		case INA2XX_POWER:
-			/* processed (mW) = raw*lsb (uW) / 1000 */
-			*val = chip->config->power_lsb;
+			/*
+			 * processed (mW) = raw * power_lsb (mW)
+			 * power_lsb (mW) = power_lsb_factor (mW/mA) *
+			 *                  current_lsb (mA)
+			 */
+			*val = chip->config->power_lsb_factor *
+			       chip->config->shunt_voltage_lsb;
+			*val2 = chip->shunt_resistor_uohm;
+			return IIO_VAL_FRACTIONAL;
+		}
+
+	case IIO_CHAN_INFO_HARDWAREGAIN:
+		switch (chan->address) {
+		case INA2XX_SHUNT_VOLTAGE:
+			*val = chip->pga_gain_vshunt;
 			*val2 = 1000;
 			return IIO_VAL_FRACTIONAL;
 
-		case INA2XX_CURRENT:
-			/* processed (mA) = raw (mA) */
-			*val = 1;
+		case INA2XX_BUS_VOLTAGE:
+			*val = chip->range_vbus == 32 ? 1 : 2;
 			return IIO_VAL_INT;
 		}
 	}
@@ -353,6 +395,74 @@ static int ina219_set_int_time_vshunt(struct ina2xx_chip_info *chip,
 	return 0;
 }
 
+static const int ina219_vbus_range_tab[] = { 1, 2 };
+static int ina219_set_vbus_range_denom(struct ina2xx_chip_info *chip,
+				       unsigned int range,
+				       unsigned int *config)
+{
+	if (range == 1)
+		chip->range_vbus = 32;
+	else if (range == 2)
+		chip->range_vbus = 16;
+	else
+		return -EINVAL;
+
+	*config &= ~INA219_BRNG_MASK;
+	*config |= INA219_SHIFT_BRNG(range == 1 ? 1 : 0) & INA219_BRNG_MASK;
+
+	return 0;
+}
+
+static const int ina219_vshunt_gain_tab[] = { 125, 250, 500, 1000 };
+static const int ina219_vshunt_gain_frac[] = {
+	125, 1000, 250, 1000, 500, 1000, 1000, 1000 };
+
+static int ina219_set_vshunt_pga_gain(struct ina2xx_chip_info *chip,
+				      unsigned int gain,
+				      unsigned int *config)
+{
+	int bits;
+
+	if (gain < 125 || gain > 1000)
+		return -EINVAL;
+
+	bits = find_closest(gain, ina219_vshunt_gain_tab,
+			    ARRAY_SIZE(ina219_vshunt_gain_tab));
+
+	chip->pga_gain_vshunt = ina219_vshunt_gain_tab[bits];
+	bits = 3 - bits;
+
+	*config &= ~INA219_PGA_MASK;
+	*config |= INA219_SHIFT_PGA(bits) & INA219_PGA_MASK;
+
+	return 0;
+}
+
+static int ina2xx_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type, int *length,
+			     long mask)
+{
+	switch (mask) {
+	case IIO_CHAN_INFO_HARDWAREGAIN:
+		switch (chan->address) {
+		case INA2XX_SHUNT_VOLTAGE:
+			*type = IIO_VAL_FRACTIONAL;
+			*length = sizeof(ina219_vshunt_gain_frac) / sizeof(int);
+			*vals = ina219_vshunt_gain_frac;
+			return IIO_AVAIL_LIST;
+
+		case INA2XX_BUS_VOLTAGE:
+			*type = IIO_VAL_INT;
+			*length = sizeof(ina219_vbus_range_tab) / sizeof(int);
+			*vals = ina219_vbus_range_tab;
+			return IIO_AVAIL_LIST;
+		}
+	}
+
+	return -EINVAL;
+}
+
 static int ina2xx_write_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan,
 			    int val, int val2, long mask)
@@ -395,6 +505,14 @@ static int ina2xx_write_raw(struct iio_dev *indio_dev,
 		}
 		break;
 
+	case IIO_CHAN_INFO_HARDWAREGAIN:
+		if (chan->address == INA2XX_SHUNT_VOLTAGE)
+			ret = ina219_set_vshunt_pga_gain(chip, val * 1000 +
+							 val2 / 1000, &tmp);
+		else
+			ret = ina219_set_vbus_range_denom(chip, val, &tmp);
+		break;
+
 	default:
 		ret = -EINVAL;
 	}
@@ -434,25 +552,21 @@ static ssize_t ina2xx_allow_async_readout_store(struct device *dev,
 }
 
 /*
- * Set current LSB to 1mA, shunt is in uOhms
- * (equation 13 in datasheet). We hardcode a Current_LSB
- * of 1.0 x10-3. The only remaining parameter is RShunt.
- * There is no need to expose the CALIBRATION register
- * to the user for now. But we need to reset this register
- * if the user updates RShunt after driver init, e.g upon
- * reading an EEPROM/Probe-type value.
+ * Calibration register is set to the best value, which eliminates
+ * truncation errors on calculating current register in hardware.
+ * According to datasheet (INA 226: eq. 3, INA219: eq. 4) the best values
+ * are 2048 for ina226 and 4096 for ina219. They are hardcoded as
+ * calibration_value.
  */
 static int ina2xx_set_calibration(struct ina2xx_chip_info *chip)
 {
-	u16 regval = DIV_ROUND_CLOSEST(chip->config->calibration_factor,
-				   chip->shunt_resistor_uohm);
-
-	return regmap_write(chip->regmap, INA2XX_CALIBRATION, regval);
+	return regmap_write(chip->regmap, INA2XX_CALIBRATION,
+			    chip->config->calibration_value);
 }
 
 static int set_shunt_resistor(struct ina2xx_chip_info *chip, unsigned int val)
 {
-	if (val <= 0 || val > chip->config->calibration_factor)
+	if (val == 0 || val > INT_MAX)
 		return -EINVAL;
 
 	chip->shunt_resistor_uohm = val;
@@ -485,11 +599,6 @@ static ssize_t ina2xx_shunt_resistor_store(struct device *dev,
 	if (ret)
 		return ret;
 
-	/* Update the Calibration register */
-	ret = ina2xx_set_calibration(chip);
-	if (ret)
-		return ret;
-
 	return len;
 }
 
@@ -532,19 +641,23 @@ static ssize_t ina2xx_shunt_resistor_store(struct device *dev,
  * Sampling Freq is a consequence of the integration times of
  * the Voltage channels.
  */
-#define INA219_CHAN_VOLTAGE(_index, _address) { \
+#define INA219_CHAN_VOLTAGE(_index, _address, _shift) { \
 	.type = IIO_VOLTAGE, \
 	.address = (_address), \
 	.indexed = 1, \
 	.channel = (_index), \
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \
 			      BIT(IIO_CHAN_INFO_SCALE) | \
-			      BIT(IIO_CHAN_INFO_INT_TIME), \
+			      BIT(IIO_CHAN_INFO_INT_TIME) | \
+			      BIT(IIO_CHAN_INFO_HARDWAREGAIN), \
+	.info_mask_separate_available = \
+			      BIT(IIO_CHAN_INFO_HARDWAREGAIN), \
 	.info_mask_shared_by_dir = BIT(IIO_CHAN_INFO_SAMP_FREQ), \
 	.scan_index = (_index), \
 	.scan_type = { \
 		.sign = 'u', \
-		.realbits = 16, \
+		.shift = _shift, \
+		.realbits = 16 - _shift, \
 		.storagebits = 16, \
 		.endianness = IIO_LE, \
 	} \
@@ -579,23 +692,18 @@ static const struct iio_chan_spec ina226_channels[] = {
 };
 
 static const struct iio_chan_spec ina219_channels[] = {
-	INA219_CHAN_VOLTAGE(0, INA2XX_SHUNT_VOLTAGE),
-	INA219_CHAN_VOLTAGE(1, INA2XX_BUS_VOLTAGE),
+	INA219_CHAN_VOLTAGE(0, INA2XX_SHUNT_VOLTAGE, 0),
+	INA219_CHAN_VOLTAGE(1, INA2XX_BUS_VOLTAGE, INA219_BUS_VOLTAGE_SHIFT),
 	INA219_CHAN(IIO_POWER, 2, INA2XX_POWER),
 	INA219_CHAN(IIO_CURRENT, 3, INA2XX_CURRENT),
 	IIO_CHAN_SOFT_TIMESTAMP(4),
 };
 
-static int ina2xx_work_buffer(struct iio_dev *indio_dev)
+static int ina2xx_conversion_ready(struct iio_dev *indio_dev)
 {
 	struct ina2xx_chip_info *chip = iio_priv(indio_dev);
-	unsigned short data[8];
-	int bit, ret, i = 0;
-	s64 time_a, time_b;
+	int ret;
 	unsigned int alert;
-	int cnvr_need_clear = 0;
-
-	time_a = iio_get_time_ns(indio_dev);
 
 	/*
 	 * Because the timer thread and the chip conversion clock
@@ -608,23 +716,31 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev)
 	 * For now, we do an extra read of the MASK_ENABLE register (INA226)
 	 * resp. the BUS_VOLTAGE register (INA219).
 	 */
-	if (!chip->allow_async_readout)
-		do {
-			if (chip->config->chip_id == ina226) {
-				ret = regmap_read(chip->regmap,
-						  INA226_MASK_ENABLE, &alert);
-				alert &= INA226_CVRF;
-			} else {
-				ret = regmap_read(chip->regmap,
-						  INA2XX_BUS_VOLTAGE, &alert);
-				alert &= INA219_CNVR;
-				cnvr_need_clear = alert;
-			}
+	if (chip->config->chip_id == ina226) {
+		ret = regmap_read(chip->regmap,
+				  INA226_MASK_ENABLE, &alert);
+		alert &= INA226_CVRF;
+	} else {
+		ret = regmap_read(chip->regmap,
+				  INA2XX_BUS_VOLTAGE, &alert);
+		alert &= INA219_CNVR;
+	}
 
-			if (ret < 0)
-				return ret;
+	if (ret < 0)
+		return ret;
+
+	return !!alert;
+}
+
+static int ina2xx_work_buffer(struct iio_dev *indio_dev)
+{
+	struct ina2xx_chip_info *chip = iio_priv(indio_dev);
+	/* data buffer needs space for channel data and timestap */
+	unsigned short data[4 + sizeof(s64)/sizeof(short)];
+	int bit, ret, i = 0;
+	s64 time;
 
-		} while (!alert);
+	time = iio_get_time_ns(indio_dev);
 
 	/*
 	 * Single register reads: bulk_read will not work with ina226/219
@@ -640,26 +756,11 @@ static int ina2xx_work_buffer(struct iio_dev *indio_dev)
 			return ret;
 
 		data[i++] = val;
-
-		if (INA2XX_SHUNT_VOLTAGE + bit == INA2XX_POWER)
-			cnvr_need_clear = 0;
-	}
-
-	/* Dummy read on INA219 power register to clear CNVR flag */
-	if (cnvr_need_clear && chip->config->chip_id == ina219) {
-		unsigned int val;
-
-		ret = regmap_read(chip->regmap, INA2XX_POWER, &val);
-		if (ret < 0)
-			return ret;
 	}
 
-	time_b = iio_get_time_ns(indio_dev);
+	iio_push_to_buffers_with_timestamp(indio_dev, data, time);
 
-	iio_push_to_buffers_with_timestamp(indio_dev,
-					   (unsigned int *)data, time_a);
-
-	return (unsigned long)(time_b - time_a) / 1000;
+	return 0;
 };
 
 static int ina2xx_capture_thread(void *data)
@@ -667,7 +768,9 @@ static int ina2xx_capture_thread(void *data)
 	struct iio_dev *indio_dev = data;
 	struct ina2xx_chip_info *chip = iio_priv(indio_dev);
 	int sampling_us = SAMPLING_PERIOD(chip);
-	int buffer_us;
+	int ret;
+	struct timespec64 next, now, delta;
+	s64 delay_us;
 
 	/*
 	 * Poll a bit faster than the chip internal Fs, in case
@@ -676,13 +779,43 @@ static int ina2xx_capture_thread(void *data)
 	if (!chip->allow_async_readout)
 		sampling_us -= 200;
 
+	ktime_get_ts64(&next);
+
 	do {
-		buffer_us = ina2xx_work_buffer(indio_dev);
-		if (buffer_us < 0)
-			return buffer_us;
+		while (!chip->allow_async_readout) {
+			ret = ina2xx_conversion_ready(indio_dev);
+			if (ret < 0)
+				return ret;
 
-		if (sampling_us > buffer_us)
-			udelay(sampling_us - buffer_us);
+			/*
+			 * If the conversion was not yet finished,
+			 * reset the reference timestamp.
+			 */
+			if (ret == 0)
+				ktime_get_ts64(&next);
+			else
+				break;
+		}
+
+		ret = ina2xx_work_buffer(indio_dev);
+		if (ret < 0)
+			return ret;
+
+		ktime_get_ts64(&now);
+
+		/*
+		 * Advance the timestamp for the next poll by one sampling
+		 * interval, and sleep for the remainder (next - now)
+		 * In case "next" has already passed, the interval is added
+		 * multiple times, i.e. samples are dropped.
+		 */
+		do {
+			timespec64_add_ns(&next, 1000 * sampling_us);
+			delta = timespec64_sub(next, now);
+			delay_us = div_s64(timespec64_to_ns(&delta), 1000);
+		} while (delay_us <= 0);
+
+		usleep_range(delay_us, (delay_us * 3) >> 1);
 
 	} while (!kthread_should_stop());
 
@@ -746,7 +879,6 @@ static IIO_CONST_ATTR_NAMED(ina226_integration_time_available,
 			    integration_time_available,
 			    "0.000140 0.000204 0.000332 0.000588 0.001100 0.002116 0.004156 0.008244");
 
-
 static IIO_DEVICE_ATTR(in_allow_async_readout, S_IRUGO | S_IWUSR,
 		       ina2xx_allow_async_readout_show,
 		       ina2xx_allow_async_readout_store, 0);
@@ -780,6 +912,7 @@ static const struct attribute_group ina226_attribute_group = {
 static const struct iio_info ina219_info = {
 	.attrs = &ina219_attribute_group,
 	.read_raw = ina2xx_read_raw,
+	.read_avail = ina2xx_read_avail,
 	.write_raw = ina2xx_write_raw,
 	.debugfs_reg_access = ina2xx_debug_reg,
 };
@@ -860,6 +993,8 @@ static int ina2xx_probe(struct i2c_client *client,
 		chip->avg = 1;
 		ina219_set_int_time_vbus(chip, INA219_DEFAULT_IT, &val);
 		ina219_set_int_time_vshunt(chip, INA219_DEFAULT_IT, &val);
+		ina219_set_vbus_range_denom(chip, INA219_DEFAULT_BRNG, &val);
+		ina219_set_vshunt_pga_gain(chip, INA219_DEFAULT_PGA, &val);
 	}
 
 	ret = ina2xx_init(chip, val);
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index 36047147ce7c..ede955d9b2a4 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -96,8 +96,8 @@
 	#define MESON_SAR_ADC_FIFO_RD_SAMPLE_VALUE_MASK		GENMASK(11, 0)
 
 #define MESON_SAR_ADC_AUX_SW					0x1c
-	#define MESON_SAR_ADC_AUX_SW_MUX_SEL_CHAN_MASK(_chan)	\
-					(GENMASK(10, 8) << (((_chan) - 2) * 2))
+	#define MESON_SAR_ADC_AUX_SW_MUX_SEL_CHAN_SHIFT(_chan)	\
+					(8 + (((_chan) - 2) * 3))
 	#define MESON_SAR_ADC_AUX_SW_VREF_P_MUX			BIT(6)
 	#define MESON_SAR_ADC_AUX_SW_VREF_N_MUX			BIT(5)
 	#define MESON_SAR_ADC_AUX_SW_MODE_SEL			BIT(4)
@@ -221,6 +221,7 @@ enum meson_sar_adc_chan7_mux_sel {
 
 struct meson_sar_adc_data {
 	bool					has_bl30_integration;
+	unsigned long				clock_rate;
 	u32					bandgap_reg;
 	unsigned int				resolution;
 	const char				*name;
@@ -233,7 +234,6 @@ struct meson_sar_adc_priv {
 	const struct meson_sar_adc_data		*data;
 	struct clk				*clkin;
 	struct clk				*core_clk;
-	struct clk				*sana_clk;
 	struct clk				*adc_sel_clk;
 	struct clk				*adc_clk;
 	struct clk_gate				clk_gate;
@@ -462,8 +462,10 @@ static int meson_sar_adc_lock(struct iio_dev *indio_dev)
 			regmap_read(priv->regmap, MESON_SAR_ADC_DELAY, &val);
 		} while (val & MESON_SAR_ADC_DELAY_BL30_BUSY && timeout--);
 
-		if (timeout < 0)
+		if (timeout < 0) {
+			mutex_unlock(&indio_dev->mlock);
 			return -ETIMEDOUT;
+		}
 	}
 
 	return 0;
@@ -622,7 +624,7 @@ static int meson_sar_adc_clk_init(struct iio_dev *indio_dev,
 static int meson_sar_adc_init(struct iio_dev *indio_dev)
 {
 	struct meson_sar_adc_priv *priv = iio_priv(indio_dev);
-	int regval, ret;
+	int regval, i, ret;
 
 	/*
 	 * make sure we start at CH7 input since the other muxes are only used
@@ -677,6 +679,32 @@ static int meson_sar_adc_init(struct iio_dev *indio_dev)
 			   FIELD_PREP(MESON_SAR_ADC_DELAY_INPUT_DLY_SEL_MASK,
 				      1));
 
+	/*
+	 * set up the input channel muxes in MESON_SAR_ADC_CHAN_10_SW
+	 * (0 = SAR_ADC_CH0, 1 = SAR_ADC_CH1)
+	 */
+	regval = FIELD_PREP(MESON_SAR_ADC_CHAN_10_SW_CHAN0_MUX_SEL_MASK, 0);
+	regmap_update_bits(priv->regmap, MESON_SAR_ADC_CHAN_10_SW,
+			   MESON_SAR_ADC_CHAN_10_SW_CHAN0_MUX_SEL_MASK,
+			   regval);
+	regval = FIELD_PREP(MESON_SAR_ADC_CHAN_10_SW_CHAN1_MUX_SEL_MASK, 1);
+	regmap_update_bits(priv->regmap, MESON_SAR_ADC_CHAN_10_SW,
+			   MESON_SAR_ADC_CHAN_10_SW_CHAN1_MUX_SEL_MASK,
+			   regval);
+
+	/*
+	 * set up the input channel muxes in MESON_SAR_ADC_AUX_SW
+	 * (2 = SAR_ADC_CH2, 3 = SAR_ADC_CH3, ...) and enable
+	 * MESON_SAR_ADC_AUX_SW_YP_DRIVE_SW and
+	 * MESON_SAR_ADC_AUX_SW_XP_DRIVE_SW like the vendor driver.
+	 */
+	regval = 0;
+	for (i = 2; i <= 7; i++)
+		regval |= i << MESON_SAR_ADC_AUX_SW_MUX_SEL_CHAN_SHIFT(i);
+	regval |= MESON_SAR_ADC_AUX_SW_YP_DRIVE_SW;
+	regval |= MESON_SAR_ADC_AUX_SW_XP_DRIVE_SW;
+	regmap_write(priv->regmap, MESON_SAR_ADC_AUX_SW, regval);
+
 	ret = clk_set_parent(priv->adc_sel_clk, priv->clkin);
 	if (ret) {
 		dev_err(indio_dev->dev.parent,
@@ -684,7 +712,7 @@ static int meson_sar_adc_init(struct iio_dev *indio_dev)
 		return ret;
 	}
 
-	ret = clk_set_rate(priv->adc_clk, 1200000);
+	ret = clk_set_rate(priv->adc_clk, priv->data->clock_rate);
 	if (ret) {
 		dev_err(indio_dev->dev.parent,
 			"failed to set adc clock rate\n");
@@ -731,12 +759,6 @@ static int meson_sar_adc_hw_enable(struct iio_dev *indio_dev)
 		goto err_core_clk;
 	}
 
-	ret = clk_prepare_enable(priv->sana_clk);
-	if (ret) {
-		dev_err(indio_dev->dev.parent, "failed to enable sana clk\n");
-		goto err_sana_clk;
-	}
-
 	regval = FIELD_PREP(MESON_SAR_ADC_REG0_FIFO_CNT_IRQ_MASK, 1);
 	regmap_update_bits(priv->regmap, MESON_SAR_ADC_REG0,
 			   MESON_SAR_ADC_REG0_FIFO_CNT_IRQ_MASK, regval);
@@ -763,8 +785,6 @@ static int meson_sar_adc_hw_enable(struct iio_dev *indio_dev)
 	regmap_update_bits(priv->regmap, MESON_SAR_ADC_REG3,
 			   MESON_SAR_ADC_REG3_ADC_EN, 0);
 	meson_sar_adc_set_bandgap(indio_dev, false);
-	clk_disable_unprepare(priv->sana_clk);
-err_sana_clk:
 	clk_disable_unprepare(priv->core_clk);
 err_core_clk:
 	regulator_disable(priv->vref);
@@ -790,7 +810,6 @@ static int meson_sar_adc_hw_disable(struct iio_dev *indio_dev)
 
 	meson_sar_adc_set_bandgap(indio_dev, false);
 
-	clk_disable_unprepare(priv->sana_clk);
 	clk_disable_unprepare(priv->core_clk);
 
 	regulator_disable(priv->vref);
@@ -866,6 +885,7 @@ static const struct iio_info meson_sar_adc_iio_info = {
 
 static const struct meson_sar_adc_data meson_sar_adc_meson8_data = {
 	.has_bl30_integration = false,
+	.clock_rate = 1150000,
 	.bandgap_reg = MESON_SAR_ADC_DELTA_10,
 	.regmap_config = &meson_sar_adc_regmap_config_meson8,
 	.resolution = 10,
@@ -874,6 +894,7 @@ static const struct meson_sar_adc_data meson_sar_adc_meson8_data = {
 
 static const struct meson_sar_adc_data meson_sar_adc_meson8b_data = {
 	.has_bl30_integration = false,
+	.clock_rate = 1150000,
 	.bandgap_reg = MESON_SAR_ADC_DELTA_10,
 	.regmap_config = &meson_sar_adc_regmap_config_meson8,
 	.resolution = 10,
@@ -882,6 +903,7 @@ static const struct meson_sar_adc_data meson_sar_adc_meson8b_data = {
 
 static const struct meson_sar_adc_data meson_sar_adc_gxbb_data = {
 	.has_bl30_integration = true,
+	.clock_rate = 1200000,
 	.bandgap_reg = MESON_SAR_ADC_REG11,
 	.regmap_config = &meson_sar_adc_regmap_config_gxbb,
 	.resolution = 10,
@@ -890,6 +912,7 @@ static const struct meson_sar_adc_data meson_sar_adc_gxbb_data = {
 
 static const struct meson_sar_adc_data meson_sar_adc_gxl_data = {
 	.has_bl30_integration = true,
+	.clock_rate = 1200000,
 	.bandgap_reg = MESON_SAR_ADC_REG11,
 	.regmap_config = &meson_sar_adc_regmap_config_gxbb,
 	.resolution = 12,
@@ -898,6 +921,7 @@ static const struct meson_sar_adc_data meson_sar_adc_gxl_data = {
 
 static const struct meson_sar_adc_data meson_sar_adc_gxm_data = {
 	.has_bl30_integration = true,
+	.clock_rate = 1200000,
 	.bandgap_reg = MESON_SAR_ADC_REG11,
 	.regmap_config = &meson_sar_adc_regmap_config_gxbb,
 	.resolution = 12,
@@ -993,16 +1017,6 @@ static int meson_sar_adc_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->core_clk);
 	}
 
-	priv->sana_clk = devm_clk_get(&pdev->dev, "sana");
-	if (IS_ERR(priv->sana_clk)) {
-		if (PTR_ERR(priv->sana_clk) == -ENOENT) {
-			priv->sana_clk = NULL;
-		} else {
-			dev_err(&pdev->dev, "failed to get sana clk\n");
-			return PTR_ERR(priv->sana_clk);
-		}
-	}
-
 	priv->adc_clk = devm_clk_get(&pdev->dev, "adc_clk");
 	if (IS_ERR(priv->adc_clk)) {
 		if (PTR_ERR(priv->adc_clk) == -ENOENT) {
diff --git a/drivers/iio/adc/qcom-vadc-common.c b/drivers/iio/adc/qcom-vadc-common.c
index 47d24ae5462f..fe3d7826783c 100644
--- a/drivers/iio/adc/qcom-vadc-common.c
+++ b/drivers/iio/adc/qcom-vadc-common.c
@@ -5,6 +5,7 @@
 #include <linux/math64.h>
 #include <linux/log2.h>
 #include <linux/err.h>
+#include <linux/module.h>
 
 #include "qcom-vadc-common.h"
 
@@ -229,3 +230,6 @@ int qcom_vadc_decimation_from_dt(u32 value)
 	return __ffs64(value / VADC_DECIMATION_MIN);
 }
 EXPORT_SYMBOL(qcom_vadc_decimation_from_dt);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Qualcomm ADC common functionality");
diff --git a/drivers/iio/adc/sd_adc_modulator.c b/drivers/iio/adc/sd_adc_modulator.c
new file mode 100644
index 000000000000..560d8c7d9d86
--- /dev/null
+++ b/drivers/iio/adc/sd_adc_modulator.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Generic sigma delta modulator driver
+ *
+ * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
+ * Author: Arnaud Pouliquen <arnaud.pouliquen@st.com>.
+ */
+
+#include <linux/iio/iio.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+
+static const struct iio_info iio_sd_mod_iio_info;
+
+static const struct iio_chan_spec iio_sd_mod_ch = {
+	.type = IIO_VOLTAGE,
+	.indexed = 1,
+	.scan_type = {
+		.sign = 'u',
+		.realbits = 1,
+		.shift = 0,
+	},
+};
+
+static int iio_sd_mod_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct iio_dev *iio;
+
+	iio = devm_iio_device_alloc(dev, 0);
+	if (!iio)
+		return -ENOMEM;
+
+	iio->dev.parent = dev;
+	iio->dev.of_node = dev->of_node;
+	iio->name = dev_name(dev);
+	iio->info = &iio_sd_mod_iio_info;
+	iio->modes = INDIO_BUFFER_HARDWARE;
+
+	iio->num_channels = 1;
+	iio->channels = &iio_sd_mod_ch;
+
+	platform_set_drvdata(pdev, iio);
+
+	return devm_iio_device_register(&pdev->dev, iio);
+}
+
+static const struct of_device_id sd_adc_of_match[] = {
+	{ .compatible = "sd-modulator" },
+	{ .compatible = "ads1201" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sd_adc_of_match);
+
+static struct platform_driver iio_sd_mod_adc = {
+	.driver = {
+		.name = "iio_sd_adc_mod",
+		.of_match_table = of_match_ptr(sd_adc_of_match),
+	},
+	.probe = iio_sd_mod_probe,
+};
+
+module_platform_driver(iio_sd_mod_adc);
+
+MODULE_DESCRIPTION("Basic sigma delta modulator");
+MODULE_AUTHOR("Arnaud Pouliquen <arnaud.pouliquen@st.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c
index 6aefef99f935..40be7d9fadbf 100644
--- a/drivers/iio/adc/stm32-adc-core.c
+++ b/drivers/iio/adc/stm32-adc-core.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * This file is part of STM32 ADC driver
  *
@@ -6,19 +7,6 @@
  *
  * Inspired from: fsl-imx25-tsadc
  *
- * License type: GPLv2
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/clk.h>
diff --git a/drivers/iio/adc/stm32-adc-core.h b/drivers/iio/adc/stm32-adc-core.h
index 250ee958a669..8af507b3f32d 100644
--- a/drivers/iio/adc/stm32-adc-core.h
+++ b/drivers/iio/adc/stm32-adc-core.h
@@ -1,22 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * This file is part of STM32 ADC driver
  *
  * Copyright (C) 2016, STMicroelectronics - All Rights Reserved
  * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
  *
- * License type: GPLv2
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef __STM32_ADC_H
diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c
index c9d96f935dba..9a2583caedaa 100644
--- a/drivers/iio/adc/stm32-adc.c
+++ b/drivers/iio/adc/stm32-adc.c
@@ -1,22 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * This file is part of STM32 ADC driver
  *
  * Copyright (C) 2016, STMicroelectronics - All Rights Reserved
  * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
- *
- * License type: GPLv2
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/clk.h>
@@ -92,6 +79,7 @@
 #define STM32H7_ADC_SQR3		0x38
 #define STM32H7_ADC_SQR4		0x3C
 #define STM32H7_ADC_DR			0x40
+#define STM32H7_ADC_DIFSEL		0xC0
 #define STM32H7_ADC_CALFACT		0xC4
 #define STM32H7_ADC_CALFACT2		0xC8
 
@@ -153,6 +141,8 @@ enum stm32h7_adc_dmngt {
 /* BOOST bit must be set on STM32H7 when ADC clock is above 20MHz */
 #define STM32H7_BOOST_CLKRATE		20000000UL
 
+#define STM32_ADC_CH_MAX		20	/* max number of channels */
+#define STM32_ADC_CH_SZ			10	/* max channel name size */
 #define STM32_ADC_MAX_SQ		16	/* SQ1..SQ16 */
 #define STM32_ADC_MAX_SMP		7	/* SMPx range is [0..7] */
 #define STM32_ADC_TIMEOUT_US		100000
@@ -297,9 +287,11 @@ struct stm32_adc_cfg {
  * @rx_buf:		dma rx buffer cpu address
  * @rx_dma_buf:		dma rx buffer bus address
  * @rx_buf_sz:		dma rx buffer size
+ * @difsel		bitmask to set single-ended/differential channel
  * @pcsel		bitmask to preselect channels on some devices
  * @smpr_val:		sampling time settings (e.g. smpr1 / smpr2)
  * @cal:		optional calibration data on some devices
+ * @chan_name:		channel name array
  */
 struct stm32_adc {
 	struct stm32_adc_common	*common;
@@ -318,72 +310,37 @@ struct stm32_adc {
 	u8			*rx_buf;
 	dma_addr_t		rx_dma_buf;
 	unsigned int		rx_buf_sz;
+	u32			difsel;
 	u32			pcsel;
 	u32			smpr_val[2];
 	struct stm32_adc_calib	cal;
+	char			chan_name[STM32_ADC_CH_MAX][STM32_ADC_CH_SZ];
 };
 
-/**
- * struct stm32_adc_chan_spec - specification of stm32 adc channel
- * @type:	IIO channel type
- * @channel:	channel number (single ended)
- * @name:	channel name (single ended)
- */
-struct stm32_adc_chan_spec {
-	enum iio_chan_type	type;
-	int			channel;
-	const char		*name;
+struct stm32_adc_diff_channel {
+	u32 vinp;
+	u32 vinn;
 };
 
 /**
  * struct stm32_adc_info - stm32 ADC, per instance config data
- * @channels:		Reference to stm32 channels spec
  * @max_channels:	Number of channels
  * @resolutions:	available resolutions
  * @num_res:		number of available resolutions
  */
 struct stm32_adc_info {
-	const struct stm32_adc_chan_spec *channels;
 	int max_channels;
 	const unsigned int *resolutions;
 	const unsigned int num_res;
 };
 
-/*
- * Input definitions common for all instances:
- * stm32f4 can have up to 16 channels
- * stm32h7 can have up to 20 channels
- */
-static const struct stm32_adc_chan_spec stm32_adc_channels[] = {
-	{ IIO_VOLTAGE, 0, "in0" },
-	{ IIO_VOLTAGE, 1, "in1" },
-	{ IIO_VOLTAGE, 2, "in2" },
-	{ IIO_VOLTAGE, 3, "in3" },
-	{ IIO_VOLTAGE, 4, "in4" },
-	{ IIO_VOLTAGE, 5, "in5" },
-	{ IIO_VOLTAGE, 6, "in6" },
-	{ IIO_VOLTAGE, 7, "in7" },
-	{ IIO_VOLTAGE, 8, "in8" },
-	{ IIO_VOLTAGE, 9, "in9" },
-	{ IIO_VOLTAGE, 10, "in10" },
-	{ IIO_VOLTAGE, 11, "in11" },
-	{ IIO_VOLTAGE, 12, "in12" },
-	{ IIO_VOLTAGE, 13, "in13" },
-	{ IIO_VOLTAGE, 14, "in14" },
-	{ IIO_VOLTAGE, 15, "in15" },
-	{ IIO_VOLTAGE, 16, "in16" },
-	{ IIO_VOLTAGE, 17, "in17" },
-	{ IIO_VOLTAGE, 18, "in18" },
-	{ IIO_VOLTAGE, 19, "in19" },
-};
-
 static const unsigned int stm32f4_adc_resolutions[] = {
 	/* sorted values so the index matches RES[1:0] in STM32F4_ADC_CR1 */
 	12, 10, 8, 6,
 };
 
+/* stm32f4 can have up to 16 channels */
 static const struct stm32_adc_info stm32f4_adc_info = {
-	.channels = stm32_adc_channels,
 	.max_channels = 16,
 	.resolutions = stm32f4_adc_resolutions,
 	.num_res = ARRAY_SIZE(stm32f4_adc_resolutions),
@@ -394,9 +351,9 @@ static const unsigned int stm32h7_adc_resolutions[] = {
 	16, 14, 12, 10, 8,
 };
 
+/* stm32h7 can have up to 20 channels */
 static const struct stm32_adc_info stm32h7_adc_info = {
-	.channels = stm32_adc_channels,
-	.max_channels = 20,
+	.max_channels = STM32_ADC_CH_MAX,
 	.resolutions = stm32h7_adc_resolutions,
 	.num_res = ARRAY_SIZE(stm32h7_adc_resolutions),
 };
@@ -765,8 +722,6 @@ static int stm32h7_adc_enable(struct stm32_adc *adc)
 	int ret;
 	u32 val;
 
-	/* Clear ADRDY by writing one, then enable ADC */
-	stm32_adc_set_bits(adc, STM32H7_ADC_ISR, STM32H7_ADRDY);
 	stm32_adc_set_bits(adc, STM32H7_ADC_CR, STM32H7_ADEN);
 
 	/* Poll for ADRDY to be set (after adc startup time) */
@@ -774,8 +729,11 @@ static int stm32h7_adc_enable(struct stm32_adc *adc)
 					   val & STM32H7_ADRDY,
 					   100, STM32_ADC_TIMEOUT_US);
 	if (ret) {
-		stm32_adc_clr_bits(adc, STM32H7_ADC_CR, STM32H7_ADEN);
+		stm32_adc_set_bits(adc, STM32H7_ADC_CR, STM32H7_ADDIS);
 		dev_err(&indio_dev->dev, "Failed to enable ADC\n");
+	} else {
+		/* Clear ADRDY by writing one */
+		stm32_adc_set_bits(adc, STM32H7_ADC_ISR, STM32H7_ADRDY);
 	}
 
 	return ret;
@@ -983,15 +941,19 @@ static int stm32h7_adc_selfcalib(struct stm32_adc *adc)
  * stm32h7_adc_prepare() - Leave power down mode to enable ADC.
  * @adc: stm32 adc instance
  * Leave power down mode.
+ * Configure channels as single ended or differential before enabling ADC.
  * Enable ADC.
  * Restore calibration data.
- * Pre-select channels that may be used in PCSEL (required by input MUX / IO).
+ * Pre-select channels that may be used in PCSEL (required by input MUX / IO):
+ * - Only one input is selected for single ended (e.g. 'vinp')
+ * - Two inputs are selected for differential channels (e.g. 'vinp' & 'vinn')
  */
 static int stm32h7_adc_prepare(struct stm32_adc *adc)
 {
 	int ret;
 
 	stm32h7_adc_exit_pwr_down(adc);
+	stm32_adc_writel(adc, STM32H7_ADC_DIFSEL, adc->difsel);
 
 	ret = stm32h7_adc_enable(adc);
 	if (ret)
@@ -1263,10 +1225,23 @@ static int stm32_adc_read_raw(struct iio_dev *indio_dev,
 		return ret;
 
 	case IIO_CHAN_INFO_SCALE:
-		*val = adc->common->vref_mv;
-		*val2 = chan->scan_type.realbits;
+		if (chan->differential) {
+			*val = adc->common->vref_mv * 2;
+			*val2 = chan->scan_type.realbits;
+		} else {
+			*val = adc->common->vref_mv;
+			*val2 = chan->scan_type.realbits;
+		}
 		return IIO_VAL_FRACTIONAL_LOG2;
 
+	case IIO_CHAN_INFO_OFFSET:
+		if (chan->differential)
+			/* ADC_full_scale / 2 */
+			*val = -((1 << chan->scan_type.realbits) / 2);
+		else
+			*val = 0;
+		return IIO_VAL_INT;
+
 	default:
 		return -EINVAL;
 	}
@@ -1315,6 +1290,7 @@ static int stm32_adc_set_watermark(struct iio_dev *indio_dev, unsigned int val)
 {
 	struct stm32_adc *adc = iio_priv(indio_dev);
 	unsigned int watermark = STM32_DMA_BUFFER_SIZE / 2;
+	unsigned int rx_buf_sz = STM32_DMA_BUFFER_SIZE;
 
 	/*
 	 * dma cyclic transfers are used, buffer is split into two periods.
@@ -1323,7 +1299,7 @@ static int stm32_adc_set_watermark(struct iio_dev *indio_dev, unsigned int val)
 	 * - one buffer (period) driver can push with iio_trigger_poll().
 	 */
 	watermark = min(watermark, val * (unsigned)(sizeof(u16)));
-	adc->rx_buf_sz = watermark * 2;
+	adc->rx_buf_sz = min(rx_buf_sz, watermark * 2 * adc->num_conv);
 
 	return 0;
 }
@@ -1628,29 +1604,40 @@ static void stm32_adc_smpr_init(struct stm32_adc *adc, int channel, u32 smp_ns)
 }
 
 static void stm32_adc_chan_init_one(struct iio_dev *indio_dev,
-				    struct iio_chan_spec *chan,
-				    const struct stm32_adc_chan_spec *channel,
-				    int scan_index, u32 smp)
+				    struct iio_chan_spec *chan, u32 vinp,
+				    u32 vinn, int scan_index, bool differential)
 {
 	struct stm32_adc *adc = iio_priv(indio_dev);
-
-	chan->type = channel->type;
-	chan->channel = channel->channel;
-	chan->datasheet_name = channel->name;
+	char *name = adc->chan_name[vinp];
+
+	chan->type = IIO_VOLTAGE;
+	chan->channel = vinp;
+	if (differential) {
+		chan->differential = 1;
+		chan->channel2 = vinn;
+		snprintf(name, STM32_ADC_CH_SZ, "in%d-in%d", vinp, vinn);
+	} else {
+		snprintf(name, STM32_ADC_CH_SZ, "in%d", vinp);
+	}
+	chan->datasheet_name = name;
 	chan->scan_index = scan_index;
 	chan->indexed = 1;
 	chan->info_mask_separate = BIT(IIO_CHAN_INFO_RAW);
-	chan->info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE);
+	chan->info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) |
+					 BIT(IIO_CHAN_INFO_OFFSET);
 	chan->scan_type.sign = 'u';
 	chan->scan_type.realbits = adc->cfg->adc_info->resolutions[adc->res];
 	chan->scan_type.storagebits = 16;
 	chan->ext_info = stm32_adc_ext_info;
 
-	/* Prepare sampling time settings */
-	stm32_adc_smpr_init(adc, chan->channel, smp);
-
 	/* pre-build selected channels mask */
 	adc->pcsel |= BIT(chan->channel);
+	if (differential) {
+		/* pre-build diff channels mask */
+		adc->difsel |= BIT(chan->channel);
+		/* Also add negative input to pre-selected channels */
+		adc->pcsel |= BIT(chan->channel2);
+	}
 }
 
 static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
@@ -1658,17 +1645,40 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
 	struct device_node *node = indio_dev->dev.of_node;
 	struct stm32_adc *adc = iio_priv(indio_dev);
 	const struct stm32_adc_info *adc_info = adc->cfg->adc_info;
+	struct stm32_adc_diff_channel diff[STM32_ADC_CH_MAX];
 	struct property *prop;
 	const __be32 *cur;
 	struct iio_chan_spec *channels;
-	int scan_index = 0, num_channels, ret;
+	int scan_index = 0, num_channels = 0, num_diff = 0, ret, i;
 	u32 val, smp = 0;
 
-	num_channels = of_property_count_u32_elems(node, "st,adc-channels");
-	if (num_channels < 0 ||
-	    num_channels > adc_info->max_channels) {
+	ret = of_property_count_u32_elems(node, "st,adc-channels");
+	if (ret > adc_info->max_channels) {
 		dev_err(&indio_dev->dev, "Bad st,adc-channels?\n");
-		return num_channels < 0 ? num_channels : -EINVAL;
+		return -EINVAL;
+	} else if (ret > 0) {
+		num_channels += ret;
+	}
+
+	ret = of_property_count_elems_of_size(node, "st,adc-diff-channels",
+					      sizeof(*diff));
+	if (ret > adc_info->max_channels) {
+		dev_err(&indio_dev->dev, "Bad st,adc-diff-channels?\n");
+		return -EINVAL;
+	} else if (ret > 0) {
+		int size = ret * sizeof(*diff) / sizeof(u32);
+
+		num_diff = ret;
+		num_channels += ret;
+		ret = of_property_read_u32_array(node, "st,adc-diff-channels",
+						 (u32 *)diff, size);
+		if (ret)
+			return ret;
+	}
+
+	if (!num_channels) {
+		dev_err(&indio_dev->dev, "No channels configured\n");
+		return -ENODATA;
 	}
 
 	/* Optional sample time is provided either for each, or all channels */
@@ -1689,6 +1699,33 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
 			return -EINVAL;
 		}
 
+		/* Channel can't be configured both as single-ended & diff */
+		for (i = 0; i < num_diff; i++) {
+			if (val == diff[i].vinp) {
+				dev_err(&indio_dev->dev,
+					"channel %d miss-configured\n",	val);
+				return -EINVAL;
+			}
+		}
+		stm32_adc_chan_init_one(indio_dev, &channels[scan_index], val,
+					0, scan_index, false);
+		scan_index++;
+	}
+
+	for (i = 0; i < num_diff; i++) {
+		if (diff[i].vinp >= adc_info->max_channels ||
+		    diff[i].vinn >= adc_info->max_channels) {
+			dev_err(&indio_dev->dev, "Invalid channel in%d-in%d\n",
+				diff[i].vinp, diff[i].vinn);
+			return -EINVAL;
+		}
+		stm32_adc_chan_init_one(indio_dev, &channels[scan_index],
+					diff[i].vinp, diff[i].vinn, scan_index,
+					true);
+		scan_index++;
+	}
+
+	for (i = 0; i < scan_index; i++) {
 		/*
 		 * Using of_property_read_u32_index(), smp value will only be
 		 * modified if valid u32 value can be decoded. This allows to
@@ -1696,12 +1733,9 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev)
 		 * value per channel.
 		 */
 		of_property_read_u32_index(node, "st,min-sample-time-nsecs",
-					   scan_index, &smp);
-
-		stm32_adc_chan_init_one(indio_dev, &channels[scan_index],
-					&adc_info->channels[val],
-					scan_index, smp);
-		scan_index++;
+					   i, &smp);
+		/* Prepare sampling time settings */
+		stm32_adc_smpr_init(adc, channels[i].channel, smp);
 	}
 
 	indio_dev->num_channels = scan_index;
diff --git a/drivers/iio/adc/stm32-dfsdm-adc.c b/drivers/iio/adc/stm32-dfsdm-adc.c
new file mode 100644
index 000000000000..01422d11753c
--- /dev/null
+++ b/drivers/iio/adc/stm32-dfsdm-adc.c
@@ -0,0 +1,1208 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is the ADC part of the STM32 DFSDM driver
+ *
+ * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
+ * Author: Arnaud Pouliquen <arnaud.pouliquen@st.com>.
+ */
+
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/iio/buffer.h>
+#include <linux/iio/hw-consumer.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include "stm32-dfsdm.h"
+
+#define DFSDM_DMA_BUFFER_SIZE (4 * PAGE_SIZE)
+
+/* Conversion timeout */
+#define DFSDM_TIMEOUT_US 100000
+#define DFSDM_TIMEOUT (msecs_to_jiffies(DFSDM_TIMEOUT_US / 1000))
+
+/* Oversampling attribute default */
+#define DFSDM_DEFAULT_OVERSAMPLING  100
+
+/* Oversampling max values */
+#define DFSDM_MAX_INT_OVERSAMPLING 256
+#define DFSDM_MAX_FL_OVERSAMPLING 1024
+
+/* Max sample resolutions */
+#define DFSDM_MAX_RES BIT(31)
+#define DFSDM_DATA_RES BIT(23)
+
+enum sd_converter_type {
+	DFSDM_AUDIO,
+	DFSDM_IIO,
+};
+
+struct stm32_dfsdm_dev_data {
+	int type;
+	int (*init)(struct iio_dev *indio_dev);
+	unsigned int num_channels;
+	const struct regmap_config *regmap_cfg;
+};
+
+struct stm32_dfsdm_adc {
+	struct stm32_dfsdm *dfsdm;
+	const struct stm32_dfsdm_dev_data *dev_data;
+	unsigned int fl_id;
+
+	/* ADC specific */
+	unsigned int oversamp;
+	struct iio_hw_consumer *hwc;
+	struct completion completion;
+	u32 *buffer;
+
+	/* Audio specific */
+	unsigned int spi_freq;  /* SPI bus clock frequency */
+	unsigned int sample_freq; /* Sample frequency after filter decimation */
+	int (*cb)(const void *data, size_t size, void *cb_priv);
+	void *cb_priv;
+
+	/* DMA */
+	u8 *rx_buf;
+	unsigned int bufi; /* Buffer current position */
+	unsigned int buf_sz; /* Buffer size */
+	struct dma_chan	*dma_chan;
+	dma_addr_t dma_buf;
+};
+
+struct stm32_dfsdm_str2field {
+	const char	*name;
+	unsigned int	val;
+};
+
+/* DFSDM channel serial interface type */
+static const struct stm32_dfsdm_str2field stm32_dfsdm_chan_type[] = {
+	{ "SPI_R", 0 }, /* SPI with data on rising edge */
+	{ "SPI_F", 1 }, /* SPI with data on falling edge */
+	{ "MANCH_R", 2 }, /* Manchester codec, rising edge = logic 0 */
+	{ "MANCH_F", 3 }, /* Manchester codec, falling edge = logic 1 */
+	{},
+};
+
+/* DFSDM channel clock source */
+static const struct stm32_dfsdm_str2field stm32_dfsdm_chan_src[] = {
+	/* External SPI clock (CLKIN x) */
+	{ "CLKIN", DFSDM_CHANNEL_SPI_CLOCK_EXTERNAL },
+	/* Internal SPI clock (CLKOUT) */
+	{ "CLKOUT", DFSDM_CHANNEL_SPI_CLOCK_INTERNAL },
+	/* Internal SPI clock divided by 2 (falling edge) */
+	{ "CLKOUT_F", DFSDM_CHANNEL_SPI_CLOCK_INTERNAL_DIV2_FALLING },
+	/* Internal SPI clock divided by 2 (falling edge) */
+	{ "CLKOUT_R", DFSDM_CHANNEL_SPI_CLOCK_INTERNAL_DIV2_RISING },
+	{},
+};
+
+static int stm32_dfsdm_str2val(const char *str,
+			       const struct stm32_dfsdm_str2field *list)
+{
+	const struct stm32_dfsdm_str2field *p = list;
+
+	for (p = list; p && p->name; p++)
+		if (!strcmp(p->name, str))
+			return p->val;
+
+	return -EINVAL;
+}
+
+static int stm32_dfsdm_set_osrs(struct stm32_dfsdm_filter *fl,
+				unsigned int fast, unsigned int oversamp)
+{
+	unsigned int i, d, fosr, iosr;
+	u64 res;
+	s64 delta;
+	unsigned int m = 1;	/* multiplication factor */
+	unsigned int p = fl->ford;	/* filter order (ford) */
+
+	pr_debug("%s: Requested oversampling: %d\n",  __func__, oversamp);
+	/*
+	 * This function tries to compute filter oversampling and integrator
+	 * oversampling, base on oversampling ratio requested by user.
+	 *
+	 * Decimation d depends on the filter order and the oversampling ratios.
+	 * ford: filter order
+	 * fosr: filter over sampling ratio
+	 * iosr: integrator over sampling ratio
+	 */
+	if (fl->ford == DFSDM_FASTSINC_ORDER) {
+		m = 2;
+		p = 2;
+	}
+
+	/*
+	 * Look for filter and integrator oversampling ratios which allows
+	 * to reach 24 bits data output resolution.
+	 * Leave as soon as if exact resolution if reached.
+	 * Otherwise the higher resolution below 32 bits is kept.
+	 */
+	for (fosr = 1; fosr <= DFSDM_MAX_FL_OVERSAMPLING; fosr++) {
+		for (iosr = 1; iosr <= DFSDM_MAX_INT_OVERSAMPLING; iosr++) {
+			if (fast)
+				d = fosr * iosr;
+			else if (fl->ford == DFSDM_FASTSINC_ORDER)
+				d = fosr * (iosr + 3) + 2;
+			else
+				d = fosr * (iosr - 1 + p) + p;
+
+			if (d > oversamp)
+				break;
+			else if (d != oversamp)
+				continue;
+			/*
+			 * Check resolution (limited to signed 32 bits)
+			 *   res <= 2^31
+			 * Sincx filters:
+			 *   res = m * fosr^p x iosr (with m=1, p=ford)
+			 * FastSinc filter
+			 *   res = m * fosr^p x iosr (with m=2, p=2)
+			 */
+			res = fosr;
+			for (i = p - 1; i > 0; i--) {
+				res = res * (u64)fosr;
+				if (res > DFSDM_MAX_RES)
+					break;
+			}
+			if (res > DFSDM_MAX_RES)
+				continue;
+			res = res * (u64)m * (u64)iosr;
+			if (res > DFSDM_MAX_RES)
+				continue;
+
+			delta = res - DFSDM_DATA_RES;
+
+			if (res >= fl->res) {
+				fl->res = res;
+				fl->fosr = fosr;
+				fl->iosr = iosr;
+				fl->fast = fast;
+				pr_debug("%s: fosr = %d, iosr = %d\n",
+					 __func__, fl->fosr, fl->iosr);
+			}
+
+			if (!delta)
+				return 0;
+		}
+	}
+
+	if (!fl->fosr)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int stm32_dfsdm_start_channel(struct stm32_dfsdm *dfsdm,
+				     unsigned int ch_id)
+{
+	return regmap_update_bits(dfsdm->regmap, DFSDM_CHCFGR1(ch_id),
+				  DFSDM_CHCFGR1_CHEN_MASK,
+				  DFSDM_CHCFGR1_CHEN(1));
+}
+
+static void stm32_dfsdm_stop_channel(struct stm32_dfsdm *dfsdm,
+				     unsigned int ch_id)
+{
+	regmap_update_bits(dfsdm->regmap, DFSDM_CHCFGR1(ch_id),
+			   DFSDM_CHCFGR1_CHEN_MASK, DFSDM_CHCFGR1_CHEN(0));
+}
+
+static int stm32_dfsdm_chan_configure(struct stm32_dfsdm *dfsdm,
+				      struct stm32_dfsdm_channel *ch)
+{
+	unsigned int id = ch->id;
+	struct regmap *regmap = dfsdm->regmap;
+	int ret;
+
+	ret = regmap_update_bits(regmap, DFSDM_CHCFGR1(id),
+				 DFSDM_CHCFGR1_SITP_MASK,
+				 DFSDM_CHCFGR1_SITP(ch->type));
+	if (ret < 0)
+		return ret;
+	ret = regmap_update_bits(regmap, DFSDM_CHCFGR1(id),
+				 DFSDM_CHCFGR1_SPICKSEL_MASK,
+				 DFSDM_CHCFGR1_SPICKSEL(ch->src));
+	if (ret < 0)
+		return ret;
+	return regmap_update_bits(regmap, DFSDM_CHCFGR1(id),
+				  DFSDM_CHCFGR1_CHINSEL_MASK,
+				  DFSDM_CHCFGR1_CHINSEL(ch->alt_si));
+}
+
+static int stm32_dfsdm_start_filter(struct stm32_dfsdm *dfsdm,
+				    unsigned int fl_id)
+{
+	int ret;
+
+	/* Enable filter */
+	ret = regmap_update_bits(dfsdm->regmap, DFSDM_CR1(fl_id),
+				 DFSDM_CR1_DFEN_MASK, DFSDM_CR1_DFEN(1));
+	if (ret < 0)
+		return ret;
+
+	/* Start conversion */
+	return regmap_update_bits(dfsdm->regmap, DFSDM_CR1(fl_id),
+				  DFSDM_CR1_RSWSTART_MASK,
+				  DFSDM_CR1_RSWSTART(1));
+}
+
+static void stm32_dfsdm_stop_filter(struct stm32_dfsdm *dfsdm, unsigned int fl_id)
+{
+	/* Disable conversion */
+	regmap_update_bits(dfsdm->regmap, DFSDM_CR1(fl_id),
+			   DFSDM_CR1_DFEN_MASK, DFSDM_CR1_DFEN(0));
+}
+
+static int stm32_dfsdm_filter_configure(struct stm32_dfsdm *dfsdm,
+					unsigned int fl_id, unsigned int ch_id)
+{
+	struct regmap *regmap = dfsdm->regmap;
+	struct stm32_dfsdm_filter *fl = &dfsdm->fl_list[fl_id];
+	int ret;
+
+	/* Average integrator oversampling */
+	ret = regmap_update_bits(regmap, DFSDM_FCR(fl_id), DFSDM_FCR_IOSR_MASK,
+				 DFSDM_FCR_IOSR(fl->iosr - 1));
+	if (ret)
+		return ret;
+
+	/* Filter order and Oversampling */
+	ret = regmap_update_bits(regmap, DFSDM_FCR(fl_id), DFSDM_FCR_FOSR_MASK,
+				 DFSDM_FCR_FOSR(fl->fosr - 1));
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(regmap, DFSDM_FCR(fl_id), DFSDM_FCR_FORD_MASK,
+				 DFSDM_FCR_FORD(fl->ford));
+	if (ret)
+		return ret;
+
+	/* No scan mode supported for the moment */
+	ret = regmap_update_bits(regmap, DFSDM_CR1(fl_id), DFSDM_CR1_RCH_MASK,
+				 DFSDM_CR1_RCH(ch_id));
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(regmap, DFSDM_CR1(fl_id),
+				  DFSDM_CR1_RSYNC_MASK,
+				  DFSDM_CR1_RSYNC(fl->sync_mode));
+}
+
+static int stm32_dfsdm_channel_parse_of(struct stm32_dfsdm *dfsdm,
+					struct iio_dev *indio_dev,
+					struct iio_chan_spec *ch)
+{
+	struct stm32_dfsdm_channel *df_ch;
+	const char *of_str;
+	int chan_idx = ch->scan_index;
+	int ret, val;
+
+	ret = of_property_read_u32_index(indio_dev->dev.of_node,
+					 "st,adc-channels", chan_idx,
+					 &ch->channel);
+	if (ret < 0) {
+		dev_err(&indio_dev->dev,
+			" Error parsing 'st,adc-channels' for idx %d\n",
+			chan_idx);
+		return ret;
+	}
+	if (ch->channel >= dfsdm->num_chs) {
+		dev_err(&indio_dev->dev,
+			" Error bad channel number %d (max = %d)\n",
+			ch->channel, dfsdm->num_chs);
+		return -EINVAL;
+	}
+
+	ret = of_property_read_string_index(indio_dev->dev.of_node,
+					    "st,adc-channel-names", chan_idx,
+					    &ch->datasheet_name);
+	if (ret < 0) {
+		dev_err(&indio_dev->dev,
+			" Error parsing 'st,adc-channel-names' for idx %d\n",
+			chan_idx);
+		return ret;
+	}
+
+	df_ch =  &dfsdm->ch_list[ch->channel];
+	df_ch->id = ch->channel;
+
+	ret = of_property_read_string_index(indio_dev->dev.of_node,
+					    "st,adc-channel-types", chan_idx,
+					    &of_str);
+	if (!ret) {
+		val  = stm32_dfsdm_str2val(of_str, stm32_dfsdm_chan_type);
+		if (val < 0)
+			return val;
+	} else {
+		val = 0;
+	}
+	df_ch->type = val;
+
+	ret = of_property_read_string_index(indio_dev->dev.of_node,
+					    "st,adc-channel-clk-src", chan_idx,
+					    &of_str);
+	if (!ret) {
+		val  = stm32_dfsdm_str2val(of_str, stm32_dfsdm_chan_src);
+		if (val < 0)
+			return val;
+	} else {
+		val = 0;
+	}
+	df_ch->src = val;
+
+	ret = of_property_read_u32_index(indio_dev->dev.of_node,
+					 "st,adc-alt-channel", chan_idx,
+					 &df_ch->alt_si);
+	if (ret < 0)
+		df_ch->alt_si = 0;
+
+	return 0;
+}
+
+static ssize_t dfsdm_adc_audio_get_spiclk(struct iio_dev *indio_dev,
+					  uintptr_t priv,
+					  const struct iio_chan_spec *chan,
+					  char *buf)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+
+	return snprintf(buf, PAGE_SIZE, "%d\n", adc->spi_freq);
+}
+
+static ssize_t dfsdm_adc_audio_set_spiclk(struct iio_dev *indio_dev,
+					  uintptr_t priv,
+					  const struct iio_chan_spec *chan,
+					  const char *buf, size_t len)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	struct stm32_dfsdm_filter *fl = &adc->dfsdm->fl_list[adc->fl_id];
+	struct stm32_dfsdm_channel *ch = &adc->dfsdm->ch_list[chan->channel];
+	unsigned int sample_freq = adc->sample_freq;
+	unsigned int spi_freq;
+	int ret;
+
+	dev_err(&indio_dev->dev, "enter %s\n", __func__);
+	/* If DFSDM is master on SPI, SPI freq can not be updated */
+	if (ch->src != DFSDM_CHANNEL_SPI_CLOCK_EXTERNAL)
+		return -EPERM;
+
+	ret = kstrtoint(buf, 0, &spi_freq);
+	if (ret)
+		return ret;
+
+	if (!spi_freq)
+		return -EINVAL;
+
+	if (sample_freq) {
+		if (spi_freq % sample_freq)
+			dev_warn(&indio_dev->dev,
+				 "Sampling rate not accurate (%d)\n",
+				 spi_freq / (spi_freq / sample_freq));
+
+		ret = stm32_dfsdm_set_osrs(fl, 0, (spi_freq / sample_freq));
+		if (ret < 0) {
+			dev_err(&indio_dev->dev,
+				"No filter parameters that match!\n");
+			return ret;
+		}
+	}
+	adc->spi_freq = spi_freq;
+
+	return len;
+}
+
+static int stm32_dfsdm_start_conv(struct stm32_dfsdm_adc *adc,
+				  const struct iio_chan_spec *chan,
+				  bool dma)
+{
+	struct regmap *regmap = adc->dfsdm->regmap;
+	int ret;
+	unsigned int dma_en = 0, cont_en = 0;
+
+	ret = stm32_dfsdm_start_channel(adc->dfsdm, chan->channel);
+	if (ret < 0)
+		return ret;
+
+	ret = stm32_dfsdm_filter_configure(adc->dfsdm, adc->fl_id,
+					   chan->channel);
+	if (ret < 0)
+		goto stop_channels;
+
+	if (dma) {
+		/* Enable DMA transfer*/
+		dma_en =  DFSDM_CR1_RDMAEN(1);
+		/* Enable conversion triggered by SPI clock*/
+		cont_en = DFSDM_CR1_RCONT(1);
+	}
+	/* Enable DMA transfer*/
+	ret = regmap_update_bits(regmap, DFSDM_CR1(adc->fl_id),
+				 DFSDM_CR1_RDMAEN_MASK, dma_en);
+	if (ret < 0)
+		goto stop_channels;
+
+	/* Enable conversion triggered by SPI clock*/
+	ret = regmap_update_bits(regmap, DFSDM_CR1(adc->fl_id),
+				 DFSDM_CR1_RCONT_MASK, cont_en);
+	if (ret < 0)
+		goto stop_channels;
+
+	ret = stm32_dfsdm_start_filter(adc->dfsdm, adc->fl_id);
+	if (ret < 0)
+		goto stop_channels;
+
+	return 0;
+
+stop_channels:
+	regmap_update_bits(regmap, DFSDM_CR1(adc->fl_id),
+			   DFSDM_CR1_RDMAEN_MASK, 0);
+
+	regmap_update_bits(regmap, DFSDM_CR1(adc->fl_id),
+			   DFSDM_CR1_RCONT_MASK, 0);
+	stm32_dfsdm_stop_channel(adc->dfsdm, chan->channel);
+
+	return ret;
+}
+
+static void stm32_dfsdm_stop_conv(struct stm32_dfsdm_adc *adc,
+				  const struct iio_chan_spec *chan)
+{
+	struct regmap *regmap = adc->dfsdm->regmap;
+
+	stm32_dfsdm_stop_filter(adc->dfsdm, adc->fl_id);
+
+	/* Clean conversion options */
+	regmap_update_bits(regmap, DFSDM_CR1(adc->fl_id),
+			   DFSDM_CR1_RDMAEN_MASK, 0);
+
+	regmap_update_bits(regmap, DFSDM_CR1(adc->fl_id),
+			   DFSDM_CR1_RCONT_MASK, 0);
+
+	stm32_dfsdm_stop_channel(adc->dfsdm, chan->channel);
+}
+
+static int stm32_dfsdm_set_watermark(struct iio_dev *indio_dev,
+				     unsigned int val)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	unsigned int watermark = DFSDM_DMA_BUFFER_SIZE / 2;
+
+	/*
+	 * DMA cyclic transfers are used, buffer is split into two periods.
+	 * There should be :
+	 * - always one buffer (period) DMA is working on
+	 * - one buffer (period) driver pushed to ASoC side.
+	 */
+	watermark = min(watermark, val * (unsigned int)(sizeof(u32)));
+	adc->buf_sz = watermark * 2;
+
+	return 0;
+}
+
+static unsigned int stm32_dfsdm_adc_dma_residue(struct stm32_dfsdm_adc *adc)
+{
+	struct dma_tx_state state;
+	enum dma_status status;
+
+	status = dmaengine_tx_status(adc->dma_chan,
+				     adc->dma_chan->cookie,
+				     &state);
+	if (status == DMA_IN_PROGRESS) {
+		/* Residue is size in bytes from end of buffer */
+		unsigned int i = adc->buf_sz - state.residue;
+		unsigned int size;
+
+		/* Return available bytes */
+		if (i >= adc->bufi)
+			size = i - adc->bufi;
+		else
+			size = adc->buf_sz + i - adc->bufi;
+
+		return size;
+	}
+
+	return 0;
+}
+
+static void stm32_dfsdm_audio_dma_buffer_done(void *data)
+{
+	struct iio_dev *indio_dev = data;
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	int available = stm32_dfsdm_adc_dma_residue(adc);
+	size_t old_pos;
+
+	/*
+	 * FIXME: In Kernel interface does not support cyclic DMA buffer,and
+	 * offers only an interface to push data samples per samples.
+	 * For this reason IIO buffer interface is not used and interface is
+	 * bypassed using a private callback registered by ASoC.
+	 * This should be a temporary solution waiting a cyclic DMA engine
+	 * support in IIO.
+	 */
+
+	dev_dbg(&indio_dev->dev, "%s: pos = %d, available = %d\n", __func__,
+		adc->bufi, available);
+	old_pos = adc->bufi;
+
+	while (available >= indio_dev->scan_bytes) {
+		u32 *buffer = (u32 *)&adc->rx_buf[adc->bufi];
+
+		/* Mask 8 LSB that contains the channel ID */
+		*buffer = (*buffer & 0xFFFFFF00) << 8;
+		available -= indio_dev->scan_bytes;
+		adc->bufi += indio_dev->scan_bytes;
+		if (adc->bufi >= adc->buf_sz) {
+			if (adc->cb)
+				adc->cb(&adc->rx_buf[old_pos],
+					 adc->buf_sz - old_pos, adc->cb_priv);
+			adc->bufi = 0;
+			old_pos = 0;
+		}
+	}
+	if (adc->cb)
+		adc->cb(&adc->rx_buf[old_pos], adc->bufi - old_pos,
+			adc->cb_priv);
+}
+
+static int stm32_dfsdm_adc_dma_start(struct iio_dev *indio_dev)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	struct dma_async_tx_descriptor *desc;
+	dma_cookie_t cookie;
+	int ret;
+
+	if (!adc->dma_chan)
+		return -EINVAL;
+
+	dev_dbg(&indio_dev->dev, "%s size=%d watermark=%d\n", __func__,
+		adc->buf_sz, adc->buf_sz / 2);
+
+	/* Prepare a DMA cyclic transaction */
+	desc = dmaengine_prep_dma_cyclic(adc->dma_chan,
+					 adc->dma_buf,
+					 adc->buf_sz, adc->buf_sz / 2,
+					 DMA_DEV_TO_MEM,
+					 DMA_PREP_INTERRUPT);
+	if (!desc)
+		return -EBUSY;
+
+	desc->callback = stm32_dfsdm_audio_dma_buffer_done;
+	desc->callback_param = indio_dev;
+
+	cookie = dmaengine_submit(desc);
+	ret = dma_submit_error(cookie);
+	if (ret) {
+		dmaengine_terminate_all(adc->dma_chan);
+		return ret;
+	}
+
+	/* Issue pending DMA requests */
+	dma_async_issue_pending(adc->dma_chan);
+
+	return 0;
+}
+
+static int stm32_dfsdm_postenable(struct iio_dev *indio_dev)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	const struct iio_chan_spec *chan = &indio_dev->channels[0];
+	int ret;
+
+	/* Reset adc buffer index */
+	adc->bufi = 0;
+
+	ret = stm32_dfsdm_start_dfsdm(adc->dfsdm);
+	if (ret < 0)
+		return ret;
+
+	ret = stm32_dfsdm_start_conv(adc, chan, true);
+	if (ret) {
+		dev_err(&indio_dev->dev, "Can't start conversion\n");
+		goto stop_dfsdm;
+	}
+
+	if (adc->dma_chan) {
+		ret = stm32_dfsdm_adc_dma_start(indio_dev);
+		if (ret) {
+			dev_err(&indio_dev->dev, "Can't start DMA\n");
+			goto err_stop_conv;
+		}
+	}
+
+	return 0;
+
+err_stop_conv:
+	stm32_dfsdm_stop_conv(adc, chan);
+stop_dfsdm:
+	stm32_dfsdm_stop_dfsdm(adc->dfsdm);
+
+	return ret;
+}
+
+static int stm32_dfsdm_predisable(struct iio_dev *indio_dev)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	const struct iio_chan_spec *chan = &indio_dev->channels[0];
+
+	if (adc->dma_chan)
+		dmaengine_terminate_all(adc->dma_chan);
+
+	stm32_dfsdm_stop_conv(adc, chan);
+
+	stm32_dfsdm_stop_dfsdm(adc->dfsdm);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops stm32_dfsdm_buffer_setup_ops = {
+	.postenable = &stm32_dfsdm_postenable,
+	.predisable = &stm32_dfsdm_predisable,
+};
+
+/**
+ * stm32_dfsdm_get_buff_cb() - register a callback that will be called when
+ *                             DMA transfer period is achieved.
+ *
+ * @iio_dev: Handle to IIO device.
+ * @cb: Pointer to callback function:
+ *      - data: pointer to data buffer
+ *      - size: size in byte of the data buffer
+ *      - private: pointer to consumer private structure.
+ * @private: Pointer to consumer private structure.
+ */
+int stm32_dfsdm_get_buff_cb(struct iio_dev *iio_dev,
+			    int (*cb)(const void *data, size_t size,
+				      void *private),
+			    void *private)
+{
+	struct stm32_dfsdm_adc *adc;
+
+	if (!iio_dev)
+		return -EINVAL;
+	adc = iio_priv(iio_dev);
+
+	adc->cb = cb;
+	adc->cb_priv = private;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(stm32_dfsdm_get_buff_cb);
+
+/**
+ * stm32_dfsdm_release_buff_cb - unregister buffer callback
+ *
+ * @iio_dev: Handle to IIO device.
+ */
+int stm32_dfsdm_release_buff_cb(struct iio_dev *iio_dev)
+{
+	struct stm32_dfsdm_adc *adc;
+
+	if (!iio_dev)
+		return -EINVAL;
+	adc = iio_priv(iio_dev);
+
+	adc->cb = NULL;
+	adc->cb_priv = NULL;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(stm32_dfsdm_release_buff_cb);
+
+static int stm32_dfsdm_single_conv(struct iio_dev *indio_dev,
+				   const struct iio_chan_spec *chan, int *res)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	long timeout;
+	int ret;
+
+	reinit_completion(&adc->completion);
+
+	adc->buffer = res;
+
+	ret = stm32_dfsdm_start_dfsdm(adc->dfsdm);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_update_bits(adc->dfsdm->regmap, DFSDM_CR2(adc->fl_id),
+				 DFSDM_CR2_REOCIE_MASK, DFSDM_CR2_REOCIE(1));
+	if (ret < 0)
+		goto stop_dfsdm;
+
+	ret = stm32_dfsdm_start_conv(adc, chan, false);
+	if (ret < 0) {
+		regmap_update_bits(adc->dfsdm->regmap, DFSDM_CR2(adc->fl_id),
+				   DFSDM_CR2_REOCIE_MASK, DFSDM_CR2_REOCIE(0));
+		goto stop_dfsdm;
+	}
+
+	timeout = wait_for_completion_interruptible_timeout(&adc->completion,
+							    DFSDM_TIMEOUT);
+
+	/* Mask IRQ for regular conversion achievement*/
+	regmap_update_bits(adc->dfsdm->regmap, DFSDM_CR2(adc->fl_id),
+			   DFSDM_CR2_REOCIE_MASK, DFSDM_CR2_REOCIE(0));
+
+	if (timeout == 0)
+		ret = -ETIMEDOUT;
+	else if (timeout < 0)
+		ret = timeout;
+	else
+		ret = IIO_VAL_INT;
+
+	stm32_dfsdm_stop_conv(adc, chan);
+
+stop_dfsdm:
+	stm32_dfsdm_stop_dfsdm(adc->dfsdm);
+
+	return ret;
+}
+
+static int stm32_dfsdm_write_raw(struct iio_dev *indio_dev,
+				 struct iio_chan_spec const *chan,
+				 int val, int val2, long mask)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	struct stm32_dfsdm_filter *fl = &adc->dfsdm->fl_list[adc->fl_id];
+	struct stm32_dfsdm_channel *ch = &adc->dfsdm->ch_list[chan->channel];
+	unsigned int spi_freq = adc->spi_freq;
+	int ret = -EINVAL;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		ret = stm32_dfsdm_set_osrs(fl, 0, val);
+		if (!ret)
+			adc->oversamp = val;
+
+		return ret;
+
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (!val)
+			return -EINVAL;
+		if (ch->src != DFSDM_CHANNEL_SPI_CLOCK_EXTERNAL)
+			spi_freq = adc->dfsdm->spi_master_freq;
+
+		if (spi_freq % val)
+			dev_warn(&indio_dev->dev,
+				 "Sampling rate not accurate (%d)\n",
+				 spi_freq / (spi_freq / val));
+
+		ret = stm32_dfsdm_set_osrs(fl, 0, (spi_freq / val));
+		if (ret < 0) {
+			dev_err(&indio_dev->dev,
+				"Not able to find parameter that match!\n");
+			return ret;
+		}
+		adc->sample_freq = val;
+
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int stm32_dfsdm_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan, int *val,
+				int *val2, long mask)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = iio_hw_consumer_enable(adc->hwc);
+		if (ret < 0) {
+			dev_err(&indio_dev->dev,
+				"%s: IIO enable failed (channel %d)\n",
+				__func__, chan->channel);
+			return ret;
+		}
+		ret = stm32_dfsdm_single_conv(indio_dev, chan, val);
+		iio_hw_consumer_disable(adc->hwc);
+		if (ret < 0) {
+			dev_err(&indio_dev->dev,
+				"%s: Conversion failed (channel %d)\n",
+				__func__, chan->channel);
+			return ret;
+		}
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*val = adc->oversamp;
+
+		return IIO_VAL_INT;
+
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = adc->sample_freq;
+
+		return IIO_VAL_INT;
+	}
+
+	return -EINVAL;
+}
+
+static const struct iio_info stm32_dfsdm_info_audio = {
+	.hwfifo_set_watermark = stm32_dfsdm_set_watermark,
+	.read_raw = stm32_dfsdm_read_raw,
+	.write_raw = stm32_dfsdm_write_raw,
+};
+
+static const struct iio_info stm32_dfsdm_info_adc = {
+	.read_raw = stm32_dfsdm_read_raw,
+	.write_raw = stm32_dfsdm_write_raw,
+};
+
+static irqreturn_t stm32_dfsdm_irq(int irq, void *arg)
+{
+	struct stm32_dfsdm_adc *adc = arg;
+	struct iio_dev *indio_dev = iio_priv_to_dev(adc);
+	struct regmap *regmap = adc->dfsdm->regmap;
+	unsigned int status, int_en;
+
+	regmap_read(regmap, DFSDM_ISR(adc->fl_id), &status);
+	regmap_read(regmap, DFSDM_CR2(adc->fl_id), &int_en);
+
+	if (status & DFSDM_ISR_REOCF_MASK) {
+		/* Read the data register clean the IRQ status */
+		regmap_read(regmap, DFSDM_RDATAR(adc->fl_id), adc->buffer);
+		complete(&adc->completion);
+	}
+
+	if (status & DFSDM_ISR_ROVRF_MASK) {
+		if (int_en & DFSDM_CR2_ROVRIE_MASK)
+			dev_warn(&indio_dev->dev, "Overrun detected\n");
+		regmap_update_bits(regmap, DFSDM_ICR(adc->fl_id),
+				   DFSDM_ICR_CLRROVRF_MASK,
+				   DFSDM_ICR_CLRROVRF_MASK);
+	}
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * Define external info for SPI Frequency and audio sampling rate that can be
+ * configured by ASoC driver through consumer.h API
+ */
+static const struct iio_chan_spec_ext_info dfsdm_adc_audio_ext_info[] = {
+	/* spi_clk_freq : clock freq on SPI/manchester bus used by channel */
+	{
+		.name = "spi_clk_freq",
+		.shared = IIO_SHARED_BY_TYPE,
+		.read = dfsdm_adc_audio_get_spiclk,
+		.write = dfsdm_adc_audio_set_spiclk,
+	},
+	{},
+};
+
+static void stm32_dfsdm_dma_release(struct iio_dev *indio_dev)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+
+	if (adc->dma_chan) {
+		dma_free_coherent(adc->dma_chan->device->dev,
+				  DFSDM_DMA_BUFFER_SIZE,
+				  adc->rx_buf, adc->dma_buf);
+		dma_release_channel(adc->dma_chan);
+	}
+}
+
+static int stm32_dfsdm_dma_request(struct iio_dev *indio_dev)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	struct dma_slave_config config = {
+		.src_addr = (dma_addr_t)adc->dfsdm->phys_base +
+			DFSDM_RDATAR(adc->fl_id),
+		.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES,
+	};
+	int ret;
+
+	adc->dma_chan = dma_request_slave_channel(&indio_dev->dev, "rx");
+	if (!adc->dma_chan)
+		return -EINVAL;
+
+	adc->rx_buf = dma_alloc_coherent(adc->dma_chan->device->dev,
+					 DFSDM_DMA_BUFFER_SIZE,
+					 &adc->dma_buf, GFP_KERNEL);
+	if (!adc->rx_buf) {
+		ret = -ENOMEM;
+		goto err_release;
+	}
+
+	ret = dmaengine_slave_config(adc->dma_chan, &config);
+	if (ret)
+		goto err_free;
+
+	return 0;
+
+err_free:
+	dma_free_coherent(adc->dma_chan->device->dev, DFSDM_DMA_BUFFER_SIZE,
+			  adc->rx_buf, adc->dma_buf);
+err_release:
+	dma_release_channel(adc->dma_chan);
+
+	return ret;
+}
+
+static int stm32_dfsdm_adc_chan_init_one(struct iio_dev *indio_dev,
+					 struct iio_chan_spec *ch)
+{
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	int ret;
+
+	ret = stm32_dfsdm_channel_parse_of(adc->dfsdm, indio_dev, ch);
+	if (ret < 0)
+		return ret;
+
+	ch->type = IIO_VOLTAGE;
+	ch->indexed = 1;
+
+	/*
+	 * IIO_CHAN_INFO_RAW: used to compute regular conversion
+	 * IIO_CHAN_INFO_OVERSAMPLING_RATIO: used to set oversampling
+	 */
+	ch->info_mask_separate = BIT(IIO_CHAN_INFO_RAW);
+	ch->info_mask_shared_by_all = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO);
+
+	if (adc->dev_data->type == DFSDM_AUDIO) {
+		ch->scan_type.sign = 's';
+		ch->ext_info = dfsdm_adc_audio_ext_info;
+	} else {
+		ch->scan_type.sign = 'u';
+	}
+	ch->scan_type.realbits = 24;
+	ch->scan_type.storagebits = 32;
+
+	return stm32_dfsdm_chan_configure(adc->dfsdm,
+					  &adc->dfsdm->ch_list[ch->channel]);
+}
+
+static int stm32_dfsdm_audio_init(struct iio_dev *indio_dev)
+{
+	struct iio_chan_spec *ch;
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	struct stm32_dfsdm_channel *d_ch;
+	int ret;
+
+	indio_dev->modes |= INDIO_BUFFER_SOFTWARE;
+	indio_dev->setup_ops = &stm32_dfsdm_buffer_setup_ops;
+
+	ch = devm_kzalloc(&indio_dev->dev, sizeof(*ch), GFP_KERNEL);
+	if (!ch)
+		return -ENOMEM;
+
+	ch->scan_index = 0;
+
+	ret = stm32_dfsdm_adc_chan_init_one(indio_dev, ch);
+	if (ret < 0) {
+		dev_err(&indio_dev->dev, "Channels init failed\n");
+		return ret;
+	}
+	ch->info_mask_separate = BIT(IIO_CHAN_INFO_SAMP_FREQ);
+
+	d_ch = &adc->dfsdm->ch_list[ch->channel];
+	if (d_ch->src != DFSDM_CHANNEL_SPI_CLOCK_EXTERNAL)
+		adc->spi_freq = adc->dfsdm->spi_master_freq;
+
+	indio_dev->num_channels = 1;
+	indio_dev->channels = ch;
+
+	return stm32_dfsdm_dma_request(indio_dev);
+}
+
+static int stm32_dfsdm_adc_init(struct iio_dev *indio_dev)
+{
+	struct iio_chan_spec *ch;
+	struct stm32_dfsdm_adc *adc = iio_priv(indio_dev);
+	int num_ch;
+	int ret, chan_idx;
+
+	adc->oversamp = DFSDM_DEFAULT_OVERSAMPLING;
+	ret = stm32_dfsdm_set_osrs(&adc->dfsdm->fl_list[adc->fl_id], 0,
+				   adc->oversamp);
+	if (ret < 0)
+		return ret;
+
+	num_ch = of_property_count_u32_elems(indio_dev->dev.of_node,
+					     "st,adc-channels");
+	if (num_ch < 0 || num_ch > adc->dfsdm->num_chs) {
+		dev_err(&indio_dev->dev, "Bad st,adc-channels\n");
+		return num_ch < 0 ? num_ch : -EINVAL;
+	}
+
+	/* Bind to SD modulator IIO device */
+	adc->hwc = devm_iio_hw_consumer_alloc(&indio_dev->dev);
+	if (IS_ERR(adc->hwc))
+		return -EPROBE_DEFER;
+
+	ch = devm_kcalloc(&indio_dev->dev, num_ch, sizeof(*ch),
+			  GFP_KERNEL);
+	if (!ch)
+		return -ENOMEM;
+
+	for (chan_idx = 0; chan_idx < num_ch; chan_idx++) {
+		ch[chan_idx].scan_index = chan_idx;
+		ret = stm32_dfsdm_adc_chan_init_one(indio_dev, &ch[chan_idx]);
+		if (ret < 0) {
+			dev_err(&indio_dev->dev, "Channels init failed\n");
+			return ret;
+		}
+	}
+
+	indio_dev->num_channels = num_ch;
+	indio_dev->channels = ch;
+
+	init_completion(&adc->completion);
+
+	return 0;
+}
+
+static const struct stm32_dfsdm_dev_data stm32h7_dfsdm_adc_data = {
+	.type = DFSDM_IIO,
+	.init = stm32_dfsdm_adc_init,
+};
+
+static const struct stm32_dfsdm_dev_data stm32h7_dfsdm_audio_data = {
+	.type = DFSDM_AUDIO,
+	.init = stm32_dfsdm_audio_init,
+};
+
+static const struct of_device_id stm32_dfsdm_adc_match[] = {
+	{
+		.compatible = "st,stm32-dfsdm-adc",
+		.data = &stm32h7_dfsdm_adc_data,
+	},
+	{
+		.compatible = "st,stm32-dfsdm-dmic",
+		.data = &stm32h7_dfsdm_audio_data,
+	},
+	{}
+};
+
+static int stm32_dfsdm_adc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct stm32_dfsdm_adc *adc;
+	struct device_node *np = dev->of_node;
+	const struct stm32_dfsdm_dev_data *dev_data;
+	struct iio_dev *iio;
+	char *name;
+	int ret, irq, val;
+
+
+	dev_data = of_device_get_match_data(dev);
+	iio = devm_iio_device_alloc(dev, sizeof(*adc));
+	if (!iio) {
+		dev_err(dev, "%s: Failed to allocate IIO\n", __func__);
+		return -ENOMEM;
+	}
+
+	adc = iio_priv(iio);
+	adc->dfsdm = dev_get_drvdata(dev->parent);
+
+	iio->dev.parent = dev;
+	iio->dev.of_node = np;
+	iio->modes = INDIO_DIRECT_MODE | INDIO_BUFFER_SOFTWARE;
+
+	platform_set_drvdata(pdev, adc);
+
+	ret = of_property_read_u32(dev->of_node, "reg", &adc->fl_id);
+	if (ret != 0) {
+		dev_err(dev, "Missing reg property\n");
+		return -EINVAL;
+	}
+
+	name = devm_kzalloc(dev, sizeof("dfsdm-adc0"), GFP_KERNEL);
+	if (!name)
+		return -ENOMEM;
+	if (dev_data->type == DFSDM_AUDIO) {
+		iio->info = &stm32_dfsdm_info_audio;
+		snprintf(name, sizeof("dfsdm-pdm0"), "dfsdm-pdm%d", adc->fl_id);
+	} else {
+		iio->info = &stm32_dfsdm_info_adc;
+		snprintf(name, sizeof("dfsdm-adc0"), "dfsdm-adc%d", adc->fl_id);
+	}
+	iio->name = name;
+
+	/*
+	 * In a first step IRQs generated for channels are not treated.
+	 * So IRQ associated to filter instance 0 is dedicated to the Filter 0.
+	 */
+	irq = platform_get_irq(pdev, 0);
+	ret = devm_request_irq(dev, irq, stm32_dfsdm_irq,
+			       0, pdev->name, adc);
+	if (ret < 0) {
+		dev_err(dev, "Failed to request IRQ\n");
+		return ret;
+	}
+
+	ret = of_property_read_u32(dev->of_node, "st,filter-order", &val);
+	if (ret < 0) {
+		dev_err(dev, "Failed to set filter order\n");
+		return ret;
+	}
+
+	adc->dfsdm->fl_list[adc->fl_id].ford = val;
+
+	ret = of_property_read_u32(dev->of_node, "st,filter0-sync", &val);
+	if (!ret)
+		adc->dfsdm->fl_list[adc->fl_id].sync_mode = val;
+
+	adc->dev_data = dev_data;
+	ret = dev_data->init(iio);
+	if (ret < 0)
+		return ret;
+
+	ret = iio_device_register(iio);
+	if (ret < 0)
+		goto err_cleanup;
+
+	dev_err(dev, "of_platform_populate\n");
+	if (dev_data->type == DFSDM_AUDIO) {
+		ret = of_platform_populate(np, NULL, NULL, dev);
+		if (ret < 0) {
+			dev_err(dev, "Failed to find an audio DAI\n");
+			goto err_unregister;
+		}
+	}
+
+	return 0;
+
+err_unregister:
+	iio_device_unregister(iio);
+err_cleanup:
+	stm32_dfsdm_dma_release(iio);
+
+	return ret;
+}
+
+static int stm32_dfsdm_adc_remove(struct platform_device *pdev)
+{
+	struct stm32_dfsdm_adc *adc = platform_get_drvdata(pdev);
+	struct iio_dev *indio_dev = iio_priv_to_dev(adc);
+
+	if (adc->dev_data->type == DFSDM_AUDIO)
+		of_platform_depopulate(&pdev->dev);
+	iio_device_unregister(indio_dev);
+	stm32_dfsdm_dma_release(indio_dev);
+
+	return 0;
+}
+
+static struct platform_driver stm32_dfsdm_adc_driver = {
+	.driver = {
+		.name = "stm32-dfsdm-adc",
+		.of_match_table = stm32_dfsdm_adc_match,
+	},
+	.probe = stm32_dfsdm_adc_probe,
+	.remove = stm32_dfsdm_adc_remove,
+};
+module_platform_driver(stm32_dfsdm_adc_driver);
+
+MODULE_DESCRIPTION("STM32 sigma delta ADC");
+MODULE_AUTHOR("Arnaud Pouliquen <arnaud.pouliquen@st.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/stm32-dfsdm-core.c b/drivers/iio/adc/stm32-dfsdm-core.c
new file mode 100644
index 000000000000..e50efdcc41ff
--- /dev/null
+++ b/drivers/iio/adc/stm32-dfsdm-core.c
@@ -0,0 +1,310 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file is part the core part STM32 DFSDM driver
+ *
+ * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
+ * Author(s): Arnaud Pouliquen <arnaud.pouliquen@st.com> for STMicroelectronics.
+ */
+
+#include <linux/clk.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include "stm32-dfsdm.h"
+
+struct stm32_dfsdm_dev_data {
+	unsigned int num_filters;
+	unsigned int num_channels;
+	const struct regmap_config *regmap_cfg;
+};
+
+#define STM32H7_DFSDM_NUM_FILTERS	4
+#define STM32H7_DFSDM_NUM_CHANNELS	8
+
+static bool stm32_dfsdm_volatile_reg(struct device *dev, unsigned int reg)
+{
+	if (reg < DFSDM_FILTER_BASE_ADR)
+		return false;
+
+	/*
+	 * Mask is done on register to avoid to list registers of all
+	 * filter instances.
+	 */
+	switch (reg & DFSDM_FILTER_REG_MASK) {
+	case DFSDM_CR1(0) & DFSDM_FILTER_REG_MASK:
+	case DFSDM_ISR(0) & DFSDM_FILTER_REG_MASK:
+	case DFSDM_JDATAR(0) & DFSDM_FILTER_REG_MASK:
+	case DFSDM_RDATAR(0) & DFSDM_FILTER_REG_MASK:
+		return true;
+	}
+
+	return false;
+}
+
+static const struct regmap_config stm32h7_dfsdm_regmap_cfg = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = sizeof(u32),
+	.max_register = 0x2B8,
+	.volatile_reg = stm32_dfsdm_volatile_reg,
+	.fast_io = true,
+};
+
+static const struct stm32_dfsdm_dev_data stm32h7_dfsdm_data = {
+	.num_filters = STM32H7_DFSDM_NUM_FILTERS,
+	.num_channels = STM32H7_DFSDM_NUM_CHANNELS,
+	.regmap_cfg = &stm32h7_dfsdm_regmap_cfg,
+};
+
+struct dfsdm_priv {
+	struct platform_device *pdev; /* platform device */
+
+	struct stm32_dfsdm dfsdm; /* common data exported for all instances */
+
+	unsigned int spi_clk_out_div; /* SPI clkout divider value */
+	atomic_t n_active_ch;	/* number of current active channels */
+
+	struct clk *clk; /* DFSDM clock */
+	struct clk *aclk; /* audio clock */
+};
+
+/**
+ * stm32_dfsdm_start_dfsdm - start global dfsdm interface.
+ *
+ * Enable interface if n_active_ch is not null.
+ * @dfsdm: Handle used to retrieve dfsdm context.
+ */
+int stm32_dfsdm_start_dfsdm(struct stm32_dfsdm *dfsdm)
+{
+	struct dfsdm_priv *priv = container_of(dfsdm, struct dfsdm_priv, dfsdm);
+	struct device *dev = &priv->pdev->dev;
+	unsigned int clk_div = priv->spi_clk_out_div, clk_src;
+	int ret;
+
+	if (atomic_inc_return(&priv->n_active_ch) == 1) {
+		ret = clk_prepare_enable(priv->clk);
+		if (ret < 0) {
+			dev_err(dev, "Failed to start clock\n");
+			goto error_ret;
+		}
+		if (priv->aclk) {
+			ret = clk_prepare_enable(priv->aclk);
+			if (ret < 0) {
+				dev_err(dev, "Failed to start audio clock\n");
+				goto disable_clk;
+			}
+		}
+
+		/* select clock source, e.g. 0 for "dfsdm" or 1 for "audio" */
+		clk_src = priv->aclk ? 1 : 0;
+		ret = regmap_update_bits(dfsdm->regmap, DFSDM_CHCFGR1(0),
+					 DFSDM_CHCFGR1_CKOUTSRC_MASK,
+					 DFSDM_CHCFGR1_CKOUTSRC(clk_src));
+		if (ret < 0)
+			goto disable_aclk;
+
+		/* Output the SPI CLKOUT (if clk_div == 0 clock if OFF) */
+		ret = regmap_update_bits(dfsdm->regmap, DFSDM_CHCFGR1(0),
+					 DFSDM_CHCFGR1_CKOUTDIV_MASK,
+					 DFSDM_CHCFGR1_CKOUTDIV(clk_div));
+		if (ret < 0)
+			goto disable_aclk;
+
+		/* Global enable of DFSDM interface */
+		ret = regmap_update_bits(dfsdm->regmap, DFSDM_CHCFGR1(0),
+					 DFSDM_CHCFGR1_DFSDMEN_MASK,
+					 DFSDM_CHCFGR1_DFSDMEN(1));
+		if (ret < 0)
+			goto disable_aclk;
+	}
+
+	dev_dbg(dev, "%s: n_active_ch %d\n", __func__,
+		atomic_read(&priv->n_active_ch));
+
+	return 0;
+
+disable_aclk:
+	clk_disable_unprepare(priv->aclk);
+disable_clk:
+	clk_disable_unprepare(priv->clk);
+
+error_ret:
+	atomic_dec(&priv->n_active_ch);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(stm32_dfsdm_start_dfsdm);
+
+/**
+ * stm32_dfsdm_stop_dfsdm - stop global DFSDM interface.
+ *
+ * Disable interface if n_active_ch is null
+ * @dfsdm: Handle used to retrieve dfsdm context.
+ */
+int stm32_dfsdm_stop_dfsdm(struct stm32_dfsdm *dfsdm)
+{
+	struct dfsdm_priv *priv = container_of(dfsdm, struct dfsdm_priv, dfsdm);
+	int ret;
+
+	if (atomic_dec_and_test(&priv->n_active_ch)) {
+		/* Global disable of DFSDM interface */
+		ret = regmap_update_bits(dfsdm->regmap, DFSDM_CHCFGR1(0),
+					 DFSDM_CHCFGR1_DFSDMEN_MASK,
+					 DFSDM_CHCFGR1_DFSDMEN(0));
+		if (ret < 0)
+			return ret;
+
+		/* Stop SPI CLKOUT */
+		ret = regmap_update_bits(dfsdm->regmap, DFSDM_CHCFGR1(0),
+					 DFSDM_CHCFGR1_CKOUTDIV_MASK,
+					 DFSDM_CHCFGR1_CKOUTDIV(0));
+		if (ret < 0)
+			return ret;
+
+		clk_disable_unprepare(priv->clk);
+		if (priv->aclk)
+			clk_disable_unprepare(priv->aclk);
+	}
+	dev_dbg(&priv->pdev->dev, "%s: n_active_ch %d\n", __func__,
+		atomic_read(&priv->n_active_ch));
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(stm32_dfsdm_stop_dfsdm);
+
+static int stm32_dfsdm_parse_of(struct platform_device *pdev,
+				struct dfsdm_priv *priv)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct resource *res;
+	unsigned long clk_freq;
+	unsigned int spi_freq, rem;
+	int ret;
+
+	if (!node)
+		return -EINVAL;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "Failed to get memory resource\n");
+		return -ENODEV;
+	}
+	priv->dfsdm.phys_base = res->start;
+	priv->dfsdm.base = devm_ioremap_resource(&pdev->dev, res);
+
+	/*
+	 * "dfsdm" clock is mandatory for DFSDM peripheral clocking.
+	 * "dfsdm" or "audio" clocks can be used as source clock for
+	 * the SPI clock out signal and internal processing, depending
+	 * on use case.
+	 */
+	priv->clk = devm_clk_get(&pdev->dev, "dfsdm");
+	if (IS_ERR(priv->clk)) {
+		dev_err(&pdev->dev, "No stm32_dfsdm_clk clock found\n");
+		return -EINVAL;
+	}
+
+	priv->aclk = devm_clk_get(&pdev->dev, "audio");
+	if (IS_ERR(priv->aclk))
+		priv->aclk = NULL;
+
+	if (priv->aclk)
+		clk_freq = clk_get_rate(priv->aclk);
+	else
+		clk_freq = clk_get_rate(priv->clk);
+
+	/* SPI clock out frequency */
+	ret = of_property_read_u32(pdev->dev.of_node, "spi-max-frequency",
+				   &spi_freq);
+	if (ret < 0) {
+		/* No SPI master mode */
+		return 0;
+	}
+
+	priv->spi_clk_out_div = div_u64_rem(clk_freq, spi_freq, &rem) - 1;
+	priv->dfsdm.spi_master_freq = spi_freq;
+
+	if (rem) {
+		dev_warn(&pdev->dev, "SPI clock not accurate\n");
+		dev_warn(&pdev->dev, "%ld = %d * %d + %d\n",
+			 clk_freq, spi_freq, priv->spi_clk_out_div + 1, rem);
+	}
+
+	return 0;
+};
+
+static const struct of_device_id stm32_dfsdm_of_match[] = {
+	{
+		.compatible = "st,stm32h7-dfsdm",
+		.data = &stm32h7_dfsdm_data,
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, stm32_dfsdm_of_match);
+
+static int stm32_dfsdm_probe(struct platform_device *pdev)
+{
+	struct dfsdm_priv *priv;
+	const struct stm32_dfsdm_dev_data *dev_data;
+	struct stm32_dfsdm *dfsdm;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->pdev = pdev;
+
+	dev_data = of_device_get_match_data(&pdev->dev);
+
+	dfsdm = &priv->dfsdm;
+	dfsdm->fl_list = devm_kcalloc(&pdev->dev, dev_data->num_filters,
+				      sizeof(*dfsdm->fl_list), GFP_KERNEL);
+	if (!dfsdm->fl_list)
+		return -ENOMEM;
+
+	dfsdm->num_fls = dev_data->num_filters;
+	dfsdm->ch_list = devm_kcalloc(&pdev->dev, dev_data->num_channels,
+				      sizeof(*dfsdm->ch_list),
+				      GFP_KERNEL);
+	if (!dfsdm->ch_list)
+		return -ENOMEM;
+	dfsdm->num_chs = dev_data->num_channels;
+
+	ret = stm32_dfsdm_parse_of(pdev, priv);
+	if (ret < 0)
+		return ret;
+
+	dfsdm->regmap = devm_regmap_init_mmio_clk(&pdev->dev, "dfsdm",
+						  dfsdm->base,
+						  dev_data->regmap_cfg);
+	if (IS_ERR(dfsdm->regmap)) {
+		ret = PTR_ERR(dfsdm->regmap);
+		dev_err(&pdev->dev, "%s: Failed to allocate regmap: %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, dfsdm);
+
+	return devm_of_platform_populate(&pdev->dev);
+}
+
+static struct platform_driver stm32_dfsdm_driver = {
+	.probe = stm32_dfsdm_probe,
+	.driver = {
+		.name = "stm32-dfsdm",
+		.of_match_table = stm32_dfsdm_of_match,
+	},
+};
+
+module_platform_driver(stm32_dfsdm_driver);
+
+MODULE_AUTHOR("Arnaud Pouliquen <arnaud.pouliquen@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics STM32 dfsdm driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/adc/stm32-dfsdm.h b/drivers/iio/adc/stm32-dfsdm.h
new file mode 100644
index 000000000000..8708394b0725
--- /dev/null
+++ b/drivers/iio/adc/stm32-dfsdm.h
@@ -0,0 +1,310 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This file is part of STM32 DFSDM driver
+ *
+ * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
+ * Author(s): Arnaud Pouliquen <arnaud.pouliquen@st.com>.
+ */
+
+#ifndef MDF_STM32_DFSDM__H
+#define MDF_STM32_DFSDM__H
+
+#include <linux/bitfield.h>
+
+/*
+ * STM32 DFSDM - global register map
+ * ________________________________________________________
+ * | Offset |                 Registers block             |
+ * --------------------------------------------------------
+ * | 0x000  |      CHANNEL 0 + COMMON CHANNEL FIELDS      |
+ * --------------------------------------------------------
+ * | 0x020  |                CHANNEL 1                    |
+ * --------------------------------------------------------
+ * | ...    |                .....                        |
+ * --------------------------------------------------------
+ * | 0x0E0  |                CHANNEL 7                    |
+ * --------------------------------------------------------
+ * | 0x100  |      FILTER  0 + COMMON  FILTER FIELDs      |
+ * --------------------------------------------------------
+ * | 0x200  |                FILTER  1                    |
+ * --------------------------------------------------------
+ * | 0x300  |                FILTER  2                    |
+ * --------------------------------------------------------
+ * | 0x400  |                FILTER  3                    |
+ * --------------------------------------------------------
+ */
+
+/*
+ * Channels register definitions
+ */
+#define DFSDM_CHCFGR1(y)  ((y) * 0x20 + 0x00)
+#define DFSDM_CHCFGR2(y)  ((y) * 0x20 + 0x04)
+#define DFSDM_AWSCDR(y)   ((y) * 0x20 + 0x08)
+#define DFSDM_CHWDATR(y)  ((y) * 0x20 + 0x0C)
+#define DFSDM_CHDATINR(y) ((y) * 0x20 + 0x10)
+
+/* CHCFGR1: Channel configuration register 1 */
+#define DFSDM_CHCFGR1_SITP_MASK     GENMASK(1, 0)
+#define DFSDM_CHCFGR1_SITP(v)       FIELD_PREP(DFSDM_CHCFGR1_SITP_MASK, v)
+#define DFSDM_CHCFGR1_SPICKSEL_MASK GENMASK(3, 2)
+#define DFSDM_CHCFGR1_SPICKSEL(v)   FIELD_PREP(DFSDM_CHCFGR1_SPICKSEL_MASK, v)
+#define DFSDM_CHCFGR1_SCDEN_MASK    BIT(5)
+#define DFSDM_CHCFGR1_SCDEN(v)      FIELD_PREP(DFSDM_CHCFGR1_SCDEN_MASK, v)
+#define DFSDM_CHCFGR1_CKABEN_MASK   BIT(6)
+#define DFSDM_CHCFGR1_CKABEN(v)     FIELD_PREP(DFSDM_CHCFGR1_CKABEN_MASK, v)
+#define DFSDM_CHCFGR1_CHEN_MASK     BIT(7)
+#define DFSDM_CHCFGR1_CHEN(v)       FIELD_PREP(DFSDM_CHCFGR1_CHEN_MASK, v)
+#define DFSDM_CHCFGR1_CHINSEL_MASK  BIT(8)
+#define DFSDM_CHCFGR1_CHINSEL(v)    FIELD_PREP(DFSDM_CHCFGR1_CHINSEL_MASK, v)
+#define DFSDM_CHCFGR1_DATMPX_MASK   GENMASK(13, 12)
+#define DFSDM_CHCFGR1_DATMPX(v)     FIELD_PREP(DFSDM_CHCFGR1_DATMPX_MASK, v)
+#define DFSDM_CHCFGR1_DATPACK_MASK  GENMASK(15, 14)
+#define DFSDM_CHCFGR1_DATPACK(v)    FIELD_PREP(DFSDM_CHCFGR1_DATPACK_MASK, v)
+#define DFSDM_CHCFGR1_CKOUTDIV_MASK GENMASK(23, 16)
+#define DFSDM_CHCFGR1_CKOUTDIV(v)   FIELD_PREP(DFSDM_CHCFGR1_CKOUTDIV_MASK, v)
+#define DFSDM_CHCFGR1_CKOUTSRC_MASK BIT(30)
+#define DFSDM_CHCFGR1_CKOUTSRC(v)   FIELD_PREP(DFSDM_CHCFGR1_CKOUTSRC_MASK, v)
+#define DFSDM_CHCFGR1_DFSDMEN_MASK  BIT(31)
+#define DFSDM_CHCFGR1_DFSDMEN(v)    FIELD_PREP(DFSDM_CHCFGR1_DFSDMEN_MASK, v)
+
+/* CHCFGR2: Channel configuration register 2 */
+#define DFSDM_CHCFGR2_DTRBS_MASK    GENMASK(7, 3)
+#define DFSDM_CHCFGR2_DTRBS(v)      FIELD_PREP(DFSDM_CHCFGR2_DTRBS_MASK, v)
+#define DFSDM_CHCFGR2_OFFSET_MASK   GENMASK(31, 8)
+#define DFSDM_CHCFGR2_OFFSET(v)     FIELD_PREP(DFSDM_CHCFGR2_OFFSET_MASK, v)
+
+/* AWSCDR: Channel analog watchdog and short circuit detector */
+#define DFSDM_AWSCDR_SCDT_MASK    GENMASK(7, 0)
+#define DFSDM_AWSCDR_SCDT(v)      FIELD_PREP(DFSDM_AWSCDR_SCDT_MASK, v)
+#define DFSDM_AWSCDR_BKSCD_MASK   GENMASK(15, 12)
+#define DFSDM_AWSCDR_BKSCD(v)	  FIELD_PREP(DFSDM_AWSCDR_BKSCD_MASK, v)
+#define DFSDM_AWSCDR_AWFOSR_MASK  GENMASK(20, 16)
+#define DFSDM_AWSCDR_AWFOSR(v)    FIELD_PREP(DFSDM_AWSCDR_AWFOSR_MASK, v)
+#define DFSDM_AWSCDR_AWFORD_MASK  GENMASK(23, 22)
+#define DFSDM_AWSCDR_AWFORD(v)    FIELD_PREP(DFSDM_AWSCDR_AWFORD_MASK, v)
+
+/*
+ * Filters register definitions
+ */
+#define DFSDM_FILTER_BASE_ADR		0x100
+#define DFSDM_FILTER_REG_MASK		0x7F
+#define DFSDM_FILTER_X_BASE_ADR(x)	((x) * 0x80 + DFSDM_FILTER_BASE_ADR)
+
+#define DFSDM_CR1(x)     (DFSDM_FILTER_X_BASE_ADR(x)  + 0x00)
+#define DFSDM_CR2(x)     (DFSDM_FILTER_X_BASE_ADR(x)  + 0x04)
+#define DFSDM_ISR(x)     (DFSDM_FILTER_X_BASE_ADR(x)  + 0x08)
+#define DFSDM_ICR(x)     (DFSDM_FILTER_X_BASE_ADR(x)  + 0x0C)
+#define DFSDM_JCHGR(x)   (DFSDM_FILTER_X_BASE_ADR(x)  + 0x10)
+#define DFSDM_FCR(x)     (DFSDM_FILTER_X_BASE_ADR(x)  + 0x14)
+#define DFSDM_JDATAR(x)  (DFSDM_FILTER_X_BASE_ADR(x)  + 0x18)
+#define DFSDM_RDATAR(x)  (DFSDM_FILTER_X_BASE_ADR(x)  + 0x1C)
+#define DFSDM_AWHTR(x)   (DFSDM_FILTER_X_BASE_ADR(x)  + 0x20)
+#define DFSDM_AWLTR(x)   (DFSDM_FILTER_X_BASE_ADR(x)  + 0x24)
+#define DFSDM_AWSR(x)    (DFSDM_FILTER_X_BASE_ADR(x)  + 0x28)
+#define DFSDM_AWCFR(x)   (DFSDM_FILTER_X_BASE_ADR(x)  + 0x2C)
+#define DFSDM_EXMAX(x)   (DFSDM_FILTER_X_BASE_ADR(x)  + 0x30)
+#define DFSDM_EXMIN(x)   (DFSDM_FILTER_X_BASE_ADR(x)  + 0x34)
+#define DFSDM_CNVTIMR(x) (DFSDM_FILTER_X_BASE_ADR(x)  + 0x38)
+
+/* CR1 Control register 1 */
+#define DFSDM_CR1_DFEN_MASK	BIT(0)
+#define DFSDM_CR1_DFEN(v)	FIELD_PREP(DFSDM_CR1_DFEN_MASK, v)
+#define DFSDM_CR1_JSWSTART_MASK	BIT(1)
+#define DFSDM_CR1_JSWSTART(v)	FIELD_PREP(DFSDM_CR1_JSWSTART_MASK, v)
+#define DFSDM_CR1_JSYNC_MASK	BIT(3)
+#define DFSDM_CR1_JSYNC(v)	FIELD_PREP(DFSDM_CR1_JSYNC_MASK, v)
+#define DFSDM_CR1_JSCAN_MASK	BIT(4)
+#define DFSDM_CR1_JSCAN(v)	FIELD_PREP(DFSDM_CR1_JSCAN_MASK, v)
+#define DFSDM_CR1_JDMAEN_MASK	BIT(5)
+#define DFSDM_CR1_JDMAEN(v)	FIELD_PREP(DFSDM_CR1_JDMAEN_MASK, v)
+#define DFSDM_CR1_JEXTSEL_MASK	GENMASK(12, 8)
+#define DFSDM_CR1_JEXTSEL(v)	FIELD_PREP(DFSDM_CR1_JEXTSEL_MASK, v)
+#define DFSDM_CR1_JEXTEN_MASK	GENMASK(14, 13)
+#define DFSDM_CR1_JEXTEN(v)	FIELD_PREP(DFSDM_CR1_JEXTEN_MASK, v)
+#define DFSDM_CR1_RSWSTART_MASK	BIT(17)
+#define DFSDM_CR1_RSWSTART(v)	FIELD_PREP(DFSDM_CR1_RSWSTART_MASK, v)
+#define DFSDM_CR1_RCONT_MASK	BIT(18)
+#define DFSDM_CR1_RCONT(v)	FIELD_PREP(DFSDM_CR1_RCONT_MASK, v)
+#define DFSDM_CR1_RSYNC_MASK	BIT(19)
+#define DFSDM_CR1_RSYNC(v)	FIELD_PREP(DFSDM_CR1_RSYNC_MASK, v)
+#define DFSDM_CR1_RDMAEN_MASK	BIT(21)
+#define DFSDM_CR1_RDMAEN(v)	FIELD_PREP(DFSDM_CR1_RDMAEN_MASK, v)
+#define DFSDM_CR1_RCH_MASK	GENMASK(26, 24)
+#define DFSDM_CR1_RCH(v)	FIELD_PREP(DFSDM_CR1_RCH_MASK, v)
+#define DFSDM_CR1_FAST_MASK	BIT(29)
+#define DFSDM_CR1_FAST(v)	FIELD_PREP(DFSDM_CR1_FAST_MASK, v)
+#define DFSDM_CR1_AWFSEL_MASK	BIT(30)
+#define DFSDM_CR1_AWFSEL(v)	FIELD_PREP(DFSDM_CR1_AWFSEL_MASK, v)
+
+/* CR2: Control register 2 */
+#define DFSDM_CR2_IE_MASK	GENMASK(6, 0)
+#define DFSDM_CR2_IE(v)		FIELD_PREP(DFSDM_CR2_IE_MASK, v)
+#define DFSDM_CR2_JEOCIE_MASK	BIT(0)
+#define DFSDM_CR2_JEOCIE(v)	FIELD_PREP(DFSDM_CR2_JEOCIE_MASK, v)
+#define DFSDM_CR2_REOCIE_MASK	BIT(1)
+#define DFSDM_CR2_REOCIE(v)	FIELD_PREP(DFSDM_CR2_REOCIE_MASK, v)
+#define DFSDM_CR2_JOVRIE_MASK	BIT(2)
+#define DFSDM_CR2_JOVRIE(v)	FIELD_PREP(DFSDM_CR2_JOVRIE_MASK, v)
+#define DFSDM_CR2_ROVRIE_MASK	BIT(3)
+#define DFSDM_CR2_ROVRIE(v)	FIELD_PREP(DFSDM_CR2_ROVRIE_MASK, v)
+#define DFSDM_CR2_AWDIE_MASK	BIT(4)
+#define DFSDM_CR2_AWDIE(v)	FIELD_PREP(DFSDM_CR2_AWDIE_MASK, v)
+#define DFSDM_CR2_SCDIE_MASK	BIT(5)
+#define DFSDM_CR2_SCDIE(v)	FIELD_PREP(DFSDM_CR2_SCDIE_MASK, v)
+#define DFSDM_CR2_CKABIE_MASK	BIT(6)
+#define DFSDM_CR2_CKABIE(v)	FIELD_PREP(DFSDM_CR2_CKABIE_MASK, v)
+#define DFSDM_CR2_EXCH_MASK	GENMASK(15, 8)
+#define DFSDM_CR2_EXCH(v)	FIELD_PREP(DFSDM_CR2_EXCH_MASK, v)
+#define DFSDM_CR2_AWDCH_MASK	GENMASK(23, 16)
+#define DFSDM_CR2_AWDCH(v)	FIELD_PREP(DFSDM_CR2_AWDCH_MASK, v)
+
+/* ISR: Interrupt status register */
+#define DFSDM_ISR_JEOCF_MASK	BIT(0)
+#define DFSDM_ISR_JEOCF(v)	FIELD_PREP(DFSDM_ISR_JEOCF_MASK, v)
+#define DFSDM_ISR_REOCF_MASK	BIT(1)
+#define DFSDM_ISR_REOCF(v)	FIELD_PREP(DFSDM_ISR_REOCF_MASK, v)
+#define DFSDM_ISR_JOVRF_MASK	BIT(2)
+#define DFSDM_ISR_JOVRF(v)	FIELD_PREP(DFSDM_ISR_JOVRF_MASK, v)
+#define DFSDM_ISR_ROVRF_MASK	BIT(3)
+#define DFSDM_ISR_ROVRF(v)	FIELD_PREP(DFSDM_ISR_ROVRF_MASK, v)
+#define DFSDM_ISR_AWDF_MASK	BIT(4)
+#define DFSDM_ISR_AWDF(v)	FIELD_PREP(DFSDM_ISR_AWDF_MASK, v)
+#define DFSDM_ISR_JCIP_MASK	BIT(13)
+#define DFSDM_ISR_JCIP(v)	FIELD_PREP(DFSDM_ISR_JCIP_MASK, v)
+#define DFSDM_ISR_RCIP_MASK	BIT(14)
+#define DFSDM_ISR_RCIP(v)	FIELD_PREP(DFSDM_ISR_RCIP, v)
+#define DFSDM_ISR_CKABF_MASK	GENMASK(23, 16)
+#define DFSDM_ISR_CKABF(v)	FIELD_PREP(DFSDM_ISR_CKABF_MASK, v)
+#define DFSDM_ISR_SCDF_MASK	GENMASK(31, 24)
+#define DFSDM_ISR_SCDF(v)	FIELD_PREP(DFSDM_ISR_SCDF_MASK, v)
+
+/* ICR: Interrupt flag clear register */
+#define DFSDM_ICR_CLRJOVRF_MASK	      BIT(2)
+#define DFSDM_ICR_CLRJOVRF(v)	      FIELD_PREP(DFSDM_ICR_CLRJOVRF_MASK, v)
+#define DFSDM_ICR_CLRROVRF_MASK	      BIT(3)
+#define DFSDM_ICR_CLRROVRF(v)	      FIELD_PREP(DFSDM_ICR_CLRROVRF_MASK, v)
+#define DFSDM_ICR_CLRCKABF_MASK	      GENMASK(23, 16)
+#define DFSDM_ICR_CLRCKABF(v)	      FIELD_PREP(DFSDM_ICR_CLRCKABF_MASK, v)
+#define DFSDM_ICR_CLRCKABF_CH_MASK(y) BIT(16 + (y))
+#define DFSDM_ICR_CLRCKABF_CH(v, y)   \
+			   (((v) << (16 + (y))) & DFSDM_ICR_CLRCKABF_CH_MASK(y))
+#define DFSDM_ICR_CLRSCDF_MASK	      GENMASK(31, 24)
+#define DFSDM_ICR_CLRSCDF(v)	      FIELD_PREP(DFSDM_ICR_CLRSCDF_MASK, v)
+#define DFSDM_ICR_CLRSCDF_CH_MASK(y)  BIT(24 + (y))
+#define DFSDM_ICR_CLRSCDF_CH(v, y)    \
+			       (((v) << (24 + (y))) & DFSDM_ICR_CLRSCDF_MASK(y))
+
+/* FCR: Filter control register */
+#define DFSDM_FCR_IOSR_MASK	GENMASK(7, 0)
+#define DFSDM_FCR_IOSR(v)	FIELD_PREP(DFSDM_FCR_IOSR_MASK, v)
+#define DFSDM_FCR_FOSR_MASK	GENMASK(25, 16)
+#define DFSDM_FCR_FOSR(v)	FIELD_PREP(DFSDM_FCR_FOSR_MASK, v)
+#define DFSDM_FCR_FORD_MASK	GENMASK(31, 29)
+#define DFSDM_FCR_FORD(v)	FIELD_PREP(DFSDM_FCR_FORD_MASK, v)
+
+/* RDATAR: Filter data register for regular channel */
+#define DFSDM_DATAR_CH_MASK	GENMASK(2, 0)
+#define DFSDM_DATAR_DATA_OFFSET 8
+#define DFSDM_DATAR_DATA_MASK	GENMASK(31, DFSDM_DATAR_DATA_OFFSET)
+
+/* AWLTR: Filter analog watchdog low threshold register */
+#define DFSDM_AWLTR_BKAWL_MASK	GENMASK(3, 0)
+#define DFSDM_AWLTR_BKAWL(v)	FIELD_PREP(DFSDM_AWLTR_BKAWL_MASK, v)
+#define DFSDM_AWLTR_AWLT_MASK	GENMASK(31, 8)
+#define DFSDM_AWLTR_AWLT(v)	FIELD_PREP(DFSDM_AWLTR_AWLT_MASK, v)
+
+/* AWHTR: Filter analog watchdog low threshold register */
+#define DFSDM_AWHTR_BKAWH_MASK	GENMASK(3, 0)
+#define DFSDM_AWHTR_BKAWH(v)	FIELD_PREP(DFSDM_AWHTR_BKAWH_MASK, v)
+#define DFSDM_AWHTR_AWHT_MASK	GENMASK(31, 8)
+#define DFSDM_AWHTR_AWHT(v)	FIELD_PREP(DFSDM_AWHTR_AWHT_MASK, v)
+
+/* AWSR: Filter watchdog status register */
+#define DFSDM_AWSR_AWLTF_MASK	GENMASK(7, 0)
+#define DFSDM_AWSR_AWLTF(v)	FIELD_PREP(DFSDM_AWSR_AWLTF_MASK, v)
+#define DFSDM_AWSR_AWHTF_MASK	GENMASK(15, 8)
+#define DFSDM_AWSR_AWHTF(v)	FIELD_PREP(DFSDM_AWSR_AWHTF_MASK, v)
+
+/* AWCFR: Filter watchdog status register */
+#define DFSDM_AWCFR_AWLTF_MASK	GENMASK(7, 0)
+#define DFSDM_AWCFR_AWLTF(v)	FIELD_PREP(DFSDM_AWCFR_AWLTF_MASK, v)
+#define DFSDM_AWCFR_AWHTF_MASK	GENMASK(15, 8)
+#define DFSDM_AWCFR_AWHTF(v)	FIELD_PREP(DFSDM_AWCFR_AWHTF_MASK, v)
+
+/* DFSDM filter order  */
+enum stm32_dfsdm_sinc_order {
+	DFSDM_FASTSINC_ORDER, /* FastSinc filter type */
+	DFSDM_SINC1_ORDER,    /* Sinc 1 filter type */
+	DFSDM_SINC2_ORDER,    /* Sinc 2 filter type */
+	DFSDM_SINC3_ORDER,    /* Sinc 3 filter type */
+	DFSDM_SINC4_ORDER,    /* Sinc 4 filter type (N.A. for watchdog) */
+	DFSDM_SINC5_ORDER,    /* Sinc 5 filter type (N.A. for watchdog) */
+	DFSDM_NB_SINC_ORDER,
+};
+
+/**
+ * struct stm32_dfsdm_filter - structure relative to stm32 FDSDM filter
+ * @iosr: integrator oversampling
+ * @fosr: filter oversampling
+ * @ford: filter order
+ * @res: output sample resolution
+ * @sync_mode: filter synchronized with filter 0
+ * @fast: filter fast mode
+ */
+struct stm32_dfsdm_filter {
+	unsigned int iosr;
+	unsigned int fosr;
+	enum stm32_dfsdm_sinc_order ford;
+	u64 res;
+	unsigned int sync_mode;
+	unsigned int fast;
+};
+
+/**
+ * struct stm32_dfsdm_channel - structure relative to stm32 FDSDM channel
+ * @id: id of the channel
+ * @type: interface type linked to stm32_dfsdm_chan_type
+ * @src: interface type linked to stm32_dfsdm_chan_src
+ * @alt_si: alternative serial input interface
+ */
+struct stm32_dfsdm_channel {
+	unsigned int id;
+	unsigned int type;
+	unsigned int src;
+	unsigned int alt_si;
+};
+
+/**
+ * struct stm32_dfsdm - stm32 FDSDM driver common data (for all instances)
+ * @base:	control registers base cpu addr
+ * @phys_base:	DFSDM IP register physical address
+ * @regmap:	regmap for register read/write
+ * @fl_list:	filter resources list
+ * @num_fls:	number of filter resources available
+ * @ch_list:	channel resources list
+ * @num_chs:	number of channel resources available
+ * @spi_master_freq: SPI clock out frequency
+ */
+struct stm32_dfsdm {
+	void __iomem	*base;
+	phys_addr_t	phys_base;
+	struct regmap *regmap;
+	struct stm32_dfsdm_filter *fl_list;
+	unsigned int num_fls;
+	struct stm32_dfsdm_channel *ch_list;
+	unsigned int num_chs;
+	unsigned int spi_master_freq;
+};
+
+/* DFSDM channel serial spi clock source */
+enum stm32_dfsdm_spi_clk_src {
+	DFSDM_CHANNEL_SPI_CLOCK_EXTERNAL,
+	DFSDM_CHANNEL_SPI_CLOCK_INTERNAL,
+	DFSDM_CHANNEL_SPI_CLOCK_INTERNAL_DIV2_FALLING,
+	DFSDM_CHANNEL_SPI_CLOCK_INTERNAL_DIV2_RISING
+};
+
+int stm32_dfsdm_start_dfsdm(struct stm32_dfsdm *dfsdm);
+int stm32_dfsdm_stop_dfsdm(struct stm32_dfsdm *dfsdm);
+
+#endif
diff --git a/drivers/iio/adc/ti_am335x_adc.c b/drivers/iio/adc/ti_am335x_adc.c
index b3e573cc6f5f..80df5a377d30 100644
--- a/drivers/iio/adc/ti_am335x_adc.c
+++ b/drivers/iio/adc/ti_am335x_adc.c
@@ -523,7 +523,7 @@ static int tiadc_read_raw(struct iio_dev *indio_dev,
 	}
 	am335x_tsc_se_adc_done(adc_dev->mfd_tscadc);
 
-	if (found == false)
+	if (!found)
 		ret =  -EBUSY;
 
 err_unlock:
diff --git a/drivers/iio/buffer/Kconfig b/drivers/iio/buffer/Kconfig
index 4ffd3db7817f..338774cba19b 100644
--- a/drivers/iio/buffer/Kconfig
+++ b/drivers/iio/buffer/Kconfig
@@ -29,6 +29,16 @@ config IIO_BUFFER_DMAENGINE
 
 	  Should be selected by drivers that want to use this functionality.
 
+config IIO_BUFFER_HW_CONSUMER
+	tristate "Industrial I/O HW buffering"
+	help
+	  Provides a way to bonding when an IIO device has a direct connection
+	  to another device in hardware. In this case buffers for data transfers
+	  are handled by hardware.
+
+	  Should be selected by drivers that want to use the generic Hw consumer
+	  interface.
+
 config IIO_KFIFO_BUF
 	tristate "Industrial I/O buffering based on kfifo"
 	help
diff --git a/drivers/iio/buffer/Makefile b/drivers/iio/buffer/Makefile
index 95f9f41c58b7..1403eb2f9409 100644
--- a/drivers/iio/buffer/Makefile
+++ b/drivers/iio/buffer/Makefile
@@ -7,5 +7,6 @@
 obj-$(CONFIG_IIO_BUFFER_CB) += industrialio-buffer-cb.o
 obj-$(CONFIG_IIO_BUFFER_DMA) += industrialio-buffer-dma.o
 obj-$(CONFIG_IIO_BUFFER_DMAENGINE) += industrialio-buffer-dmaengine.o
+obj-$(CONFIG_IIO_BUFFER_HW_CONSUMER) += industrialio-hw-consumer.o
 obj-$(CONFIG_IIO_TRIGGERED_BUFFER) += industrialio-triggered-buffer.o
 obj-$(CONFIG_IIO_KFIFO_BUF) += kfifo_buf.o
diff --git a/drivers/iio/buffer/industrialio-buffer-cb.c b/drivers/iio/buffer/industrialio-buffer-cb.c
index 4847534700e7..ea63c838eeae 100644
--- a/drivers/iio/buffer/industrialio-buffer-cb.c
+++ b/drivers/iio/buffer/industrialio-buffer-cb.c
@@ -104,6 +104,17 @@ struct iio_cb_buffer *iio_channel_get_all_cb(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(iio_channel_get_all_cb);
 
+int iio_channel_cb_set_buffer_watermark(struct iio_cb_buffer *cb_buff,
+					size_t watermark)
+{
+	if (!watermark)
+		return -EINVAL;
+	cb_buff->buffer.watermark = watermark;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(iio_channel_cb_set_buffer_watermark);
+
 int iio_channel_start_all_cb(struct iio_cb_buffer *cb_buff)
 {
 	return iio_update_buffers(cb_buff->indio_dev, &cb_buff->buffer,
diff --git a/drivers/iio/buffer/industrialio-buffer-dma.c b/drivers/iio/buffer/industrialio-buffer-dma.c
index ff03324dee13..05e0c353e089 100644
--- a/drivers/iio/buffer/industrialio-buffer-dma.c
+++ b/drivers/iio/buffer/industrialio-buffer-dma.c
@@ -222,7 +222,7 @@ void iio_dma_buffer_block_done(struct iio_dma_buffer_block *block)
 	spin_unlock_irqrestore(&queue->list_lock, flags);
 
 	iio_buffer_block_put_atomic(block);
-	wake_up_interruptible_poll(&queue->buffer.pollq, POLLIN | POLLRDNORM);
+	wake_up_interruptible_poll(&queue->buffer.pollq, EPOLLIN | EPOLLRDNORM);
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_block_done);
 
@@ -251,7 +251,7 @@ void iio_dma_buffer_block_list_abort(struct iio_dma_buffer_queue *queue,
 	}
 	spin_unlock_irqrestore(&queue->list_lock, flags);
 
-	wake_up_interruptible_poll(&queue->buffer.pollq, POLLIN | POLLRDNORM);
+	wake_up_interruptible_poll(&queue->buffer.pollq, EPOLLIN | EPOLLRDNORM);
 }
 EXPORT_SYMBOL_GPL(iio_dma_buffer_block_list_abort);
 
diff --git a/drivers/iio/buffer/industrialio-hw-consumer.c b/drivers/iio/buffer/industrialio-hw-consumer.c
new file mode 100644
index 000000000000..95165697d8ae
--- /dev/null
+++ b/drivers/iio/buffer/industrialio-hw-consumer.c
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2017 Analog Devices Inc.
+ *  Author: Lars-Peter Clausen <lars@metafoo.de>
+ */
+
+#include <linux/err.h>
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/consumer.h>
+#include <linux/iio/hw-consumer.h>
+#include <linux/iio/buffer_impl.h>
+
+/**
+ * struct iio_hw_consumer - IIO hw consumer block
+ * @buffers: hardware buffers list head.
+ * @channels: IIO provider channels.
+ */
+struct iio_hw_consumer {
+	struct list_head buffers;
+	struct iio_channel *channels;
+};
+
+struct hw_consumer_buffer {
+	struct list_head head;
+	struct iio_dev *indio_dev;
+	struct iio_buffer buffer;
+	long scan_mask[];
+};
+
+static struct hw_consumer_buffer *iio_buffer_to_hw_consumer_buffer(
+	struct iio_buffer *buffer)
+{
+	return container_of(buffer, struct hw_consumer_buffer, buffer);
+}
+
+static void iio_hw_buf_release(struct iio_buffer *buffer)
+{
+	struct hw_consumer_buffer *hw_buf =
+		iio_buffer_to_hw_consumer_buffer(buffer);
+	kfree(hw_buf);
+}
+
+static const struct iio_buffer_access_funcs iio_hw_buf_access = {
+	.release = &iio_hw_buf_release,
+	.modes = INDIO_BUFFER_HARDWARE,
+};
+
+static struct hw_consumer_buffer *iio_hw_consumer_get_buffer(
+	struct iio_hw_consumer *hwc, struct iio_dev *indio_dev)
+{
+	size_t mask_size = BITS_TO_LONGS(indio_dev->masklength) * sizeof(long);
+	struct hw_consumer_buffer *buf;
+
+	list_for_each_entry(buf, &hwc->buffers, head) {
+		if (buf->indio_dev == indio_dev)
+			return buf;
+	}
+
+	buf = kzalloc(sizeof(*buf) + mask_size, GFP_KERNEL);
+	if (!buf)
+		return NULL;
+
+	buf->buffer.access = &iio_hw_buf_access;
+	buf->indio_dev = indio_dev;
+	buf->buffer.scan_mask = buf->scan_mask;
+
+	iio_buffer_init(&buf->buffer);
+	list_add_tail(&buf->head, &hwc->buffers);
+
+	return buf;
+}
+
+/**
+ * iio_hw_consumer_alloc() - Allocate IIO hardware consumer
+ * @dev: Pointer to consumer device.
+ *
+ * Returns a valid iio_hw_consumer on success or a ERR_PTR() on failure.
+ */
+struct iio_hw_consumer *iio_hw_consumer_alloc(struct device *dev)
+{
+	struct hw_consumer_buffer *buf;
+	struct iio_hw_consumer *hwc;
+	struct iio_channel *chan;
+	int ret;
+
+	hwc = kzalloc(sizeof(*hwc), GFP_KERNEL);
+	if (!hwc)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&hwc->buffers);
+
+	hwc->channels = iio_channel_get_all(dev);
+	if (IS_ERR(hwc->channels)) {
+		ret = PTR_ERR(hwc->channels);
+		goto err_free_hwc;
+	}
+
+	chan = &hwc->channels[0];
+	while (chan->indio_dev) {
+		buf = iio_hw_consumer_get_buffer(hwc, chan->indio_dev);
+		if (!buf) {
+			ret = -ENOMEM;
+			goto err_put_buffers;
+		}
+		set_bit(chan->channel->scan_index, buf->buffer.scan_mask);
+		chan++;
+	}
+
+	return hwc;
+
+err_put_buffers:
+	list_for_each_entry(buf, &hwc->buffers, head)
+		iio_buffer_put(&buf->buffer);
+	iio_channel_release_all(hwc->channels);
+err_free_hwc:
+	kfree(hwc);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(iio_hw_consumer_alloc);
+
+/**
+ * iio_hw_consumer_free() - Free IIO hardware consumer
+ * @hwc: hw consumer to free.
+ */
+void iio_hw_consumer_free(struct iio_hw_consumer *hwc)
+{
+	struct hw_consumer_buffer *buf, *n;
+
+	iio_channel_release_all(hwc->channels);
+	list_for_each_entry_safe(buf, n, &hwc->buffers, head)
+		iio_buffer_put(&buf->buffer);
+	kfree(hwc);
+}
+EXPORT_SYMBOL_GPL(iio_hw_consumer_free);
+
+static void devm_iio_hw_consumer_release(struct device *dev, void *res)
+{
+	iio_hw_consumer_free(*(struct iio_hw_consumer **)res);
+}
+
+static int devm_iio_hw_consumer_match(struct device *dev, void *res, void *data)
+{
+	struct iio_hw_consumer **r = res;
+
+	if (!r || !*r) {
+		WARN_ON(!r || !*r);
+		return 0;
+	}
+	return *r == data;
+}
+
+/**
+ * devm_iio_hw_consumer_alloc - Resource-managed iio_hw_consumer_alloc()
+ * @dev: Pointer to consumer device.
+ *
+ * Managed iio_hw_consumer_alloc. iio_hw_consumer allocated with this function
+ * is automatically freed on driver detach.
+ *
+ * If an iio_hw_consumer allocated with this function needs to be freed
+ * separately, devm_iio_hw_consumer_free() must be used.
+ *
+ * returns pointer to allocated iio_hw_consumer on success, NULL on failure.
+ */
+struct iio_hw_consumer *devm_iio_hw_consumer_alloc(struct device *dev)
+{
+	struct iio_hw_consumer **ptr, *iio_hwc;
+
+	ptr = devres_alloc(devm_iio_hw_consumer_release, sizeof(*ptr),
+			   GFP_KERNEL);
+	if (!ptr)
+		return NULL;
+
+	iio_hwc = iio_hw_consumer_alloc(dev);
+	if (IS_ERR(iio_hwc)) {
+		devres_free(ptr);
+	} else {
+		*ptr = iio_hwc;
+		devres_add(dev, ptr);
+	}
+
+	return iio_hwc;
+}
+EXPORT_SYMBOL_GPL(devm_iio_hw_consumer_alloc);
+
+/**
+ * devm_iio_hw_consumer_free - Resource-managed iio_hw_consumer_free()
+ * @dev: Pointer to consumer device.
+ * @hwc: iio_hw_consumer to free.
+ *
+ * Free iio_hw_consumer allocated with devm_iio_hw_consumer_alloc().
+ */
+void devm_iio_hw_consumer_free(struct device *dev, struct iio_hw_consumer *hwc)
+{
+	int rc;
+
+	rc = devres_release(dev, devm_iio_hw_consumer_release,
+			    devm_iio_hw_consumer_match, hwc);
+	WARN_ON(rc);
+}
+EXPORT_SYMBOL_GPL(devm_iio_hw_consumer_free);
+
+/**
+ * iio_hw_consumer_enable() - Enable IIO hardware consumer
+ * @hwc: iio_hw_consumer to enable.
+ *
+ * Returns 0 on success.
+ */
+int iio_hw_consumer_enable(struct iio_hw_consumer *hwc)
+{
+	struct hw_consumer_buffer *buf;
+	int ret;
+
+	list_for_each_entry(buf, &hwc->buffers, head) {
+		ret = iio_update_buffers(buf->indio_dev, &buf->buffer, NULL);
+		if (ret)
+			goto err_disable_buffers;
+	}
+
+	return 0;
+
+err_disable_buffers:
+	list_for_each_entry_continue_reverse(buf, &hwc->buffers, head)
+		iio_update_buffers(buf->indio_dev, NULL, &buf->buffer);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(iio_hw_consumer_enable);
+
+/**
+ * iio_hw_consumer_disable() - Disable IIO hardware consumer
+ * @hwc: iio_hw_consumer to disable.
+ */
+void iio_hw_consumer_disable(struct iio_hw_consumer *hwc)
+{
+	struct hw_consumer_buffer *buf;
+
+	list_for_each_entry(buf, &hwc->buffers, head)
+		iio_update_buffers(buf->indio_dev, NULL, &buf->buffer);
+}
+EXPORT_SYMBOL_GPL(iio_hw_consumer_disable);
+
+MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
+MODULE_DESCRIPTION("Hardware consumer buffer the IIO framework");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/chemical/ccs811.c b/drivers/iio/chemical/ccs811.c
index 97bce8345c6a..1ea9f5513b02 100644
--- a/drivers/iio/chemical/ccs811.c
+++ b/drivers/iio/chemical/ccs811.c
@@ -96,7 +96,6 @@ static const struct iio_chan_spec ccs811_channels[] = {
 		.channel2 = IIO_MOD_CO2,
 		.modified = 1,
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
-				      BIT(IIO_CHAN_INFO_OFFSET) |
 				      BIT(IIO_CHAN_INFO_SCALE),
 		.scan_index = 0,
 		.scan_type = {
@@ -134,6 +133,9 @@ static int ccs811_start_sensor_application(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
+	if ((ret & CCS811_STATUS_FW_MODE_APPLICATION))
+		return 0;
+
 	if ((ret & CCS811_STATUS_APP_VALID_MASK) !=
 	    CCS811_STATUS_APP_VALID_LOADED)
 		return -EIO;
@@ -255,24 +257,18 @@ static int ccs811_read_raw(struct iio_dev *indio_dev,
 			switch (chan->channel2) {
 			case IIO_MOD_CO2:
 				*val = 0;
-				*val2 = 12834;
+				*val2 = 100;
 				return IIO_VAL_INT_PLUS_MICRO;
 			case IIO_MOD_VOC:
 				*val = 0;
-				*val2 = 84246;
-				return IIO_VAL_INT_PLUS_MICRO;
+				*val2 = 100;
+				return IIO_VAL_INT_PLUS_NANO;
 			default:
 				return -EINVAL;
 			}
 		default:
 			return -EINVAL;
 		}
-	case IIO_CHAN_INFO_OFFSET:
-		if (!(chan->type == IIO_CONCENTRATION &&
-		      chan->channel2 == IIO_MOD_CO2))
-			return -EINVAL;
-		*val = -400;
-		return IIO_VAL_INT;
 	default:
 		return -EINVAL;
 	}
diff --git a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
index ed8063f2da99..7d30c59da3e2 100644
--- a/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
+++ b/drivers/iio/common/cros_ec_sensors/cros_ec_sensors.c
@@ -191,7 +191,6 @@ static int cros_ec_sensors_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct cros_ec_dev *ec_dev = dev_get_drvdata(dev->parent);
-	struct cros_ec_device *ec_device;
 	struct iio_dev *indio_dev;
 	struct cros_ec_sensors_state *state;
 	struct iio_chan_spec *channel;
@@ -201,7 +200,6 @@ static int cros_ec_sensors_probe(struct platform_device *pdev)
 		dev_warn(&pdev->dev, "No CROS EC device found.\n");
 		return -EINVAL;
 	}
-	ec_device = ec_dev->ec_dev;
 
 	indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*state));
 	if (!indio_dev)
diff --git a/drivers/iio/common/ssp_sensors/ssp.h b/drivers/iio/common/ssp_sensors/ssp.h
index b910e91d7c0d..82a01addd919 100644
--- a/drivers/iio/common/ssp_sensors/ssp.h
+++ b/drivers/iio/common/ssp_sensors/ssp.h
@@ -188,7 +188,7 @@ struct ssp_sensorhub_info {
  */
 struct ssp_data {
 	struct spi_device *spi;
-	struct ssp_sensorhub_info *sensorhub_info;
+	const struct ssp_sensorhub_info *sensorhub_info;
 	struct timer_list wdt_timer;
 	struct work_struct work_wdt;
 	struct delayed_work work_refresh;
diff --git a/drivers/iio/common/ssp_sensors/ssp_dev.c b/drivers/iio/common/ssp_sensors/ssp_dev.c
index 2ba2ff5e59c4..af3aa38f67cd 100644
--- a/drivers/iio/common/ssp_sensors/ssp_dev.c
+++ b/drivers/iio/common/ssp_sensors/ssp_dev.c
@@ -486,7 +486,7 @@ static struct ssp_data *ssp_parse_dt(struct device *dev)
 	if (!match)
 		goto err_mcu_reset_gpio;
 
-	data->sensorhub_info = (struct ssp_sensorhub_info *)match->data;
+	data->sensorhub_info = match->data;
 
 	dev_set_drvdata(dev, data);
 
diff --git a/drivers/iio/common/ssp_sensors/ssp_spi.c b/drivers/iio/common/ssp_sensors/ssp_spi.c
index 704284a475ae..2ab106bb3e03 100644
--- a/drivers/iio/common/ssp_sensors/ssp_spi.c
+++ b/drivers/iio/common/ssp_sensors/ssp_spi.c
@@ -277,12 +277,9 @@ static int ssp_handle_big_data(struct ssp_data *data, char *dataframe, int *idx)
 static int ssp_parse_dataframe(struct ssp_data *data, char *dataframe, int len)
 {
 	int idx, sd;
-	struct timespec ts;
 	struct ssp_sensor_data *spd;
 	struct iio_dev **indio_devs = data->sensor_devs;
 
-	getnstimeofday(&ts);
-
 	for (idx = 0; idx < len;) {
 		switch (dataframe[idx++]) {
 		case SSP_MSG2AP_INST_BYPASS_DATA:
@@ -329,7 +326,7 @@ static int ssp_parse_dataframe(struct ssp_data *data, char *dataframe, int len)
 	}
 
 	if (data->time_syncing)
-		data->timestamp = ts.tv_sec * 1000000000ULL + ts.tv_nsec;
+		data->timestamp = ktime_get_real_ns();
 
 	return 0;
 }
diff --git a/drivers/iio/counter/stm32-lptimer-cnt.c b/drivers/iio/counter/stm32-lptimer-cnt.c
index 81ae5f74216d..42fb8ba67090 100644
--- a/drivers/iio/counter/stm32-lptimer-cnt.c
+++ b/drivers/iio/counter/stm32-lptimer-cnt.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * STM32 Low-Power Timer Encoder and Counter driver
  *
@@ -7,7 +8,6 @@
  *
  * Inspired by 104-quad-8 and stm32-timer-trigger drivers.
  *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/bitfield.h>
diff --git a/drivers/iio/dac/mcp4725.c b/drivers/iio/dac/mcp4725.c
index afa856d10c26..8b5aad4c32d9 100644
--- a/drivers/iio/dac/mcp4725.c
+++ b/drivers/iio/dac/mcp4725.c
@@ -476,7 +476,7 @@ static int mcp4725_probe(struct i2c_client *client,
 		goto err_disable_vref_reg;
 	}
 	pd = (inbuf[0] >> 1) & 0x3;
-	data->powerdown = pd > 0 ? true : false;
+	data->powerdown = pd > 0;
 	data->powerdown_mode = pd ? pd - 1 : 2; /* largest resistor to gnd */
 	data->dac_value = (inbuf[1] << 4) | (inbuf[2] >> 4);
 	if (data->id == MCP4726)
diff --git a/drivers/iio/dac/stm32-dac-core.c b/drivers/iio/dac/stm32-dac-core.c
index 55026fe1c610..d0fb3124de07 100644
--- a/drivers/iio/dac/stm32-dac-core.c
+++ b/drivers/iio/dac/stm32-dac-core.c
@@ -1,22 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * This file is part of STM32 DAC driver
  *
  * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
  * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
  *
- * License type: GPLv2
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/clk.h>
diff --git a/drivers/iio/dac/stm32-dac-core.h b/drivers/iio/dac/stm32-dac-core.h
index daf09931857c..d3b415fb9575 100644
--- a/drivers/iio/dac/stm32-dac-core.h
+++ b/drivers/iio/dac/stm32-dac-core.h
@@ -1,22 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * This file is part of STM32 DAC driver
  *
  * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
  * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
- *
- * License type: GPLv2
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef __STM32_DAC_CORE_H
diff --git a/drivers/iio/dac/stm32-dac.c b/drivers/iio/dac/stm32-dac.c
index 9ffab02bf9f9..cce26a3a6627 100644
--- a/drivers/iio/dac/stm32-dac.c
+++ b/drivers/iio/dac/stm32-dac.c
@@ -1,23 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * This file is part of STM32 DAC driver
  *
  * Copyright (C) 2017, STMicroelectronics - All Rights Reserved
  * Authors: Amelie Delaunay <amelie.delaunay@st.com>
  *	    Fabrice Gasnier <fabrice.gasnier@st.com>
- *
- * License type: GPLv2
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/bitfield.h>
diff --git a/drivers/iio/dummy/iio_dummy_evgen.c b/drivers/iio/dummy/iio_dummy_evgen.c
index fe8884543da0..efd0005f59b4 100644
--- a/drivers/iio/dummy/iio_dummy_evgen.c
+++ b/drivers/iio/dummy/iio_dummy_evgen.c
@@ -56,7 +56,7 @@ static int iio_dummy_evgen_create(void)
 		return -ENOMEM;
 
 	ret = irq_sim_init(&iio_evgen->irq_sim, IIO_EVENTGEN_NO);
-	if (ret) {
+	if (ret < 0) {
 		kfree(iio_evgen);
 		return ret;
 	}
diff --git a/drivers/iio/gyro/adis16136.c b/drivers/iio/gyro/adis16136.c
index 90ec4bed62b7..605eee23780c 100644
--- a/drivers/iio/gyro/adis16136.c
+++ b/drivers/iio/gyro/adis16136.c
@@ -124,7 +124,7 @@ static int adis16136_show_product_id(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16136_product_id_fops,
+DEFINE_DEBUGFS_ATTRIBUTE(adis16136_product_id_fops,
 	adis16136_show_product_id, NULL, "%llu\n");
 
 static int adis16136_show_flash_count(void *arg, u64 *val)
@@ -142,18 +142,21 @@ static int adis16136_show_flash_count(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16136_flash_count_fops,
+DEFINE_DEBUGFS_ATTRIBUTE(adis16136_flash_count_fops,
 	adis16136_show_flash_count, NULL, "%lld\n");
 
 static int adis16136_debugfs_init(struct iio_dev *indio_dev)
 {
 	struct adis16136 *adis16136 = iio_priv(indio_dev);
 
-	debugfs_create_file("serial_number", 0400, indio_dev->debugfs_dentry,
-		adis16136, &adis16136_serial_fops);
-	debugfs_create_file("product_id", 0400, indio_dev->debugfs_dentry,
+	debugfs_create_file_unsafe("serial_number", 0400,
+		indio_dev->debugfs_dentry, adis16136,
+		&adis16136_serial_fops);
+	debugfs_create_file_unsafe("product_id", 0400,
+		indio_dev->debugfs_dentry,
 		adis16136, &adis16136_product_id_fops);
-	debugfs_create_file("flash_count", 0400, indio_dev->debugfs_dentry,
+	debugfs_create_file_unsafe("flash_count", 0400,
+		indio_dev->debugfs_dentry,
 		adis16136, &adis16136_flash_count_fops);
 
 	return 0;
diff --git a/drivers/iio/gyro/bmg160_core.c b/drivers/iio/gyro/bmg160_core.c
index 15046172e437..63ca31628a93 100644
--- a/drivers/iio/gyro/bmg160_core.c
+++ b/drivers/iio/gyro/bmg160_core.c
@@ -27,7 +27,6 @@
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
 #include <linux/regmap.h>
-#include <linux/delay.h>
 #include "bmg160.h"
 
 #define BMG160_IRQ_NAME		"bmg160_event"
diff --git a/drivers/iio/health/max30102.c b/drivers/iio/health/max30102.c
index 147a8c14235f..15ccadc74891 100644
--- a/drivers/iio/health/max30102.c
+++ b/drivers/iio/health/max30102.c
@@ -3,6 +3,9 @@
  *
  * Copyright (C) 2017 Matt Ranostay <matt@ranostay.consulting>
  *
+ * Support for MAX30105 optical particle sensor
+ * Copyright (C) 2017 Peter Meerwald-Stadler <pmeerw@pmeerw.net>
+ *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
@@ -13,6 +16,7 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  *
+ * 7-bit I2C chip address: 0x57
  * TODO: proximity power saving feature
  */
 
@@ -32,6 +36,18 @@
 
 #define MAX30102_REGMAP_NAME	"max30102_regmap"
 #define MAX30102_DRV_NAME	"max30102"
+#define MAX30102_PART_NUMBER	0x15
+
+enum max30102_chip_id {
+	max30102,
+	max30105,
+};
+
+enum max3012_led_idx {
+	MAX30102_LED_RED,
+	MAX30102_LED_IR,
+	MAX30105_LED_GREEN,
+};
 
 #define MAX30102_REG_INT_STATUS			0x00
 #define MAX30102_REG_INT_STATUS_PWR_RDY		BIT(0)
@@ -52,7 +68,7 @@
 #define MAX30102_REG_FIFO_OVR_CTR		0x05
 #define MAX30102_REG_FIFO_RD_PTR		0x06
 #define MAX30102_REG_FIFO_DATA			0x07
-#define MAX30102_REG_FIFO_DATA_ENTRY_LEN	6
+#define MAX30102_REG_FIFO_DATA_BYTES		3
 
 #define MAX30102_REG_FIFO_CONFIG		0x08
 #define MAX30102_REG_FIFO_CONFIG_AVG_4SAMPLES	BIT(1)
@@ -60,11 +76,18 @@
 #define MAX30102_REG_FIFO_CONFIG_AFULL		BIT(0)
 
 #define MAX30102_REG_MODE_CONFIG		0x09
-#define MAX30102_REG_MODE_CONFIG_MODE_SPO2_EN	BIT(0)
-#define MAX30102_REG_MODE_CONFIG_MODE_HR_EN	BIT(1)
-#define MAX30102_REG_MODE_CONFIG_MODE_MASK	0x03
+#define MAX30102_REG_MODE_CONFIG_MODE_NONE	0x00
+#define MAX30102_REG_MODE_CONFIG_MODE_HR	0x02 /* red LED */
+#define MAX30102_REG_MODE_CONFIG_MODE_HR_SPO2	0x03 /* red + IR LED */
+#define MAX30102_REG_MODE_CONFIG_MODE_MULTI	0x07 /* multi-LED mode */
+#define MAX30102_REG_MODE_CONFIG_MODE_MASK	GENMASK(2, 0)
 #define MAX30102_REG_MODE_CONFIG_PWR		BIT(7)
 
+#define MAX30102_REG_MODE_CONTROL_SLOT21	0x11 /* multi-LED control */
+#define MAX30102_REG_MODE_CONTROL_SLOT43	0x12
+#define MAX30102_REG_MODE_CONTROL_SLOT_MASK	(GENMASK(6, 4) | GENMASK(2, 0))
+#define MAX30102_REG_MODE_CONTROL_SLOT_SHIFT	4
+
 #define MAX30102_REG_SPO2_CONFIG		0x0a
 #define MAX30102_REG_SPO2_CONFIG_PULSE_411_US	0x03
 #define MAX30102_REG_SPO2_CONFIG_SR_400HZ	0x03
@@ -75,6 +98,7 @@
 
 #define MAX30102_REG_RED_LED_CONFIG		0x0c
 #define MAX30102_REG_IR_LED_CONFIG		0x0d
+#define MAX30105_REG_GREEN_LED_CONFIG		0x0e
 
 #define MAX30102_REG_TEMP_CONFIG		0x21
 #define MAX30102_REG_TEMP_CONFIG_TEMP_EN	BIT(0)
@@ -82,14 +106,18 @@
 #define MAX30102_REG_TEMP_INTEGER		0x1f
 #define MAX30102_REG_TEMP_FRACTION		0x20
 
+#define MAX30102_REG_REV_ID			0xfe
+#define MAX30102_REG_PART_ID			0xff
+
 struct max30102_data {
 	struct i2c_client *client;
 	struct iio_dev *indio_dev;
 	struct mutex lock;
 	struct regmap *regmap;
+	enum max30102_chip_id chip_id;
 
-	u8 buffer[8];
-	__be32 processed_buffer[2]; /* 2 x 18-bit (padded to 32-bits) */
+	u8 buffer[12];
+	__be32 processed_buffer[3]; /* 3 x 18-bit (padded to 32-bits) */
 };
 
 static const struct regmap_config max30102_regmap_config = {
@@ -99,37 +127,47 @@ static const struct regmap_config max30102_regmap_config = {
 	.val_bits = 8,
 };
 
-static const unsigned long max30102_scan_masks[] = {0x3, 0};
+static const unsigned long max30102_scan_masks[] = {
+	BIT(MAX30102_LED_RED) | BIT(MAX30102_LED_IR),
+	0
+};
+
+static const unsigned long max30105_scan_masks[] = {
+	BIT(MAX30102_LED_RED) | BIT(MAX30102_LED_IR),
+	BIT(MAX30102_LED_RED) | BIT(MAX30102_LED_IR) |
+		BIT(MAX30105_LED_GREEN),
+	0
+};
+
+#define MAX30102_INTENSITY_CHANNEL(_si, _mod) { \
+		.type = IIO_INTENSITY, \
+		.channel2 = _mod, \
+		.modified = 1, \
+		.scan_index = _si, \
+		.scan_type = { \
+			.sign = 'u', \
+			.shift = 8, \
+			.realbits = 18, \
+			.storagebits = 32, \
+			.endianness = IIO_BE, \
+		}, \
+	}
 
 static const struct iio_chan_spec max30102_channels[] = {
+	MAX30102_INTENSITY_CHANNEL(MAX30102_LED_RED, IIO_MOD_LIGHT_RED),
+	MAX30102_INTENSITY_CHANNEL(MAX30102_LED_IR, IIO_MOD_LIGHT_IR),
 	{
-		.type = IIO_INTENSITY,
-		.channel2 = IIO_MOD_LIGHT_RED,
-		.modified = 1,
-
-		.scan_index = 0,
-		.scan_type = {
-			.sign = 'u',
-			.shift = 8,
-			.realbits = 18,
-			.storagebits = 32,
-			.endianness = IIO_BE,
-		},
-	},
-	{
-		.type = IIO_INTENSITY,
-		.channel2 = IIO_MOD_LIGHT_IR,
-		.modified = 1,
-
-		.scan_index = 1,
-		.scan_type = {
-			.sign = 'u',
-			.shift = 8,
-			.realbits = 18,
-			.storagebits = 32,
-			.endianness = IIO_BE,
-		},
+		.type = IIO_TEMP,
+		.info_mask_separate =
+			BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE),
+		.scan_index = -1,
 	},
+};
+
+static const struct iio_chan_spec max30105_channels[] = {
+	MAX30102_INTENSITY_CHANNEL(MAX30102_LED_RED, IIO_MOD_LIGHT_RED),
+	MAX30102_INTENSITY_CHANNEL(MAX30102_LED_IR, IIO_MOD_LIGHT_IR),
+	MAX30102_INTENSITY_CHANNEL(MAX30105_LED_GREEN, IIO_MOD_LIGHT_GREEN),
 	{
 		.type = IIO_TEMP,
 		.info_mask_separate =
@@ -138,25 +176,69 @@ static const struct iio_chan_spec max30102_channels[] = {
 	},
 };
 
-static int max30102_set_powermode(struct max30102_data *data, bool state)
+static int max30102_set_power(struct max30102_data *data, bool en)
 {
 	return regmap_update_bits(data->regmap, MAX30102_REG_MODE_CONFIG,
 				  MAX30102_REG_MODE_CONFIG_PWR,
-				  state ? 0 : MAX30102_REG_MODE_CONFIG_PWR);
+				  en ? 0 : MAX30102_REG_MODE_CONFIG_PWR);
 }
 
+static int max30102_set_powermode(struct max30102_data *data, u8 mode, bool en)
+{
+	u8 reg = mode;
+
+	if (!en)
+		reg |= MAX30102_REG_MODE_CONFIG_PWR;
+
+	return regmap_update_bits(data->regmap, MAX30102_REG_MODE_CONFIG,
+				  MAX30102_REG_MODE_CONFIG_PWR |
+				  MAX30102_REG_MODE_CONFIG_MODE_MASK, reg);
+}
+
+#define MAX30102_MODE_CONTROL_LED_SLOTS(slot2, slot1) \
+	((slot2 << MAX30102_REG_MODE_CONTROL_SLOT_SHIFT) | slot1)
+
 static int max30102_buffer_postenable(struct iio_dev *indio_dev)
 {
 	struct max30102_data *data = iio_priv(indio_dev);
+	int ret;
+	u8 reg;
 
-	return max30102_set_powermode(data, true);
+	switch (*indio_dev->active_scan_mask) {
+	case BIT(MAX30102_LED_RED) | BIT(MAX30102_LED_IR):
+		reg = MAX30102_REG_MODE_CONFIG_MODE_HR_SPO2;
+		break;
+	case BIT(MAX30102_LED_RED) | BIT(MAX30102_LED_IR) |
+	     BIT(MAX30105_LED_GREEN):
+		ret = regmap_update_bits(data->regmap,
+					 MAX30102_REG_MODE_CONTROL_SLOT21,
+					 MAX30102_REG_MODE_CONTROL_SLOT_MASK,
+					 MAX30102_MODE_CONTROL_LED_SLOTS(2, 1));
+		if (ret)
+			return ret;
+
+		ret = regmap_update_bits(data->regmap,
+					 MAX30102_REG_MODE_CONTROL_SLOT43,
+					 MAX30102_REG_MODE_CONTROL_SLOT_MASK,
+					 MAX30102_MODE_CONTROL_LED_SLOTS(0, 3));
+		if (ret)
+			return ret;
+
+		reg = MAX30102_REG_MODE_CONFIG_MODE_MULTI;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return max30102_set_powermode(data, reg, true);
 }
 
 static int max30102_buffer_predisable(struct iio_dev *indio_dev)
 {
 	struct max30102_data *data = iio_priv(indio_dev);
 
-	return max30102_set_powermode(data, false);
+	return max30102_set_powermode(data, MAX30102_REG_MODE_CONFIG_MODE_NONE,
+				      false);
 }
 
 static const struct iio_buffer_setup_ops max30102_buffer_setup_ops = {
@@ -180,32 +262,51 @@ static inline int max30102_fifo_count(struct max30102_data *data)
 	return 0;
 }
 
-static int max30102_read_measurement(struct max30102_data *data)
+#define MAX30102_COPY_DATA(i) \
+	memcpy(&data->processed_buffer[(i)], \
+	       &buffer[(i) * MAX30102_REG_FIFO_DATA_BYTES], \
+	       MAX30102_REG_FIFO_DATA_BYTES)
+
+static int max30102_read_measurement(struct max30102_data *data,
+				     unsigned int measurements)
 {
 	int ret;
 	u8 *buffer = (u8 *) &data->buffer;
 
 	ret = i2c_smbus_read_i2c_block_data(data->client,
 					    MAX30102_REG_FIFO_DATA,
-					    MAX30102_REG_FIFO_DATA_ENTRY_LEN,
+					    measurements *
+					    MAX30102_REG_FIFO_DATA_BYTES,
 					    buffer);
 
-	memcpy(&data->processed_buffer[0], &buffer[0], 3);
-	memcpy(&data->processed_buffer[1], &buffer[3], 3);
+	switch (measurements) {
+	case 3:
+		MAX30102_COPY_DATA(2);
+	case 2: /* fall-through */
+		MAX30102_COPY_DATA(1);
+	case 1: /* fall-through */
+		MAX30102_COPY_DATA(0);
+		break;
+	default:
+		return -EINVAL;
+	}
 
-	return (ret == MAX30102_REG_FIFO_DATA_ENTRY_LEN) ? 0 : -EINVAL;
+	return (ret == measurements * MAX30102_REG_FIFO_DATA_BYTES) ?
+	       0 : -EINVAL;
 }
 
 static irqreturn_t max30102_interrupt_handler(int irq, void *private)
 {
 	struct iio_dev *indio_dev = private;
 	struct max30102_data *data = iio_priv(indio_dev);
+	unsigned int measurements = bitmap_weight(indio_dev->active_scan_mask,
+						  indio_dev->masklength);
 	int ret, cnt = 0;
 
 	mutex_lock(&data->lock);
 
 	while (cnt || (cnt = max30102_fifo_count(data)) > 0) {
-		ret = max30102_read_measurement(data);
+		ret = max30102_read_measurement(data, measurements);
 		if (ret)
 			break;
 
@@ -251,6 +352,29 @@ static int max30102_led_init(struct max30102_data *data)
 	if (ret)
 		return ret;
 
+	if (data->chip_id == max30105) {
+		ret = of_property_read_u32(np,
+			"maxim,green-led-current-microamp", &val);
+		if (ret) {
+			dev_info(dev, "no green-led-current-microamp set\n");
+
+			/* Default to 7 mA green LED */
+			val = 7000;
+		}
+
+		ret = max30102_get_current_idx(val, &reg);
+		if (ret) {
+			dev_err(dev, "invalid green LED current setting %d\n",
+				val);
+			return ret;
+		}
+
+		ret = regmap_write(data->regmap, MAX30105_REG_GREEN_LED_CONFIG,
+				   reg);
+		if (ret)
+			return ret;
+	}
+
 	ret = of_property_read_u32(np, "maxim,ir-led-current-microamp", &val);
 	if (ret) {
 		dev_info(dev, "no ir-led-current-microamp set\n");
@@ -261,7 +385,7 @@ static int max30102_led_init(struct max30102_data *data)
 
 	ret = max30102_get_current_idx(val, &reg);
 	if (ret) {
-		dev_err(dev, "invalid IR LED current setting %d", val);
+		dev_err(dev, "invalid IR LED current setting %d\n", val);
 		return ret;
 	}
 
@@ -277,7 +401,7 @@ static int max30102_chip_init(struct max30102_data *data)
 	if (ret)
 		return ret;
 
-	/* enable 18-bit HR + SPO2 readings at 400Hz */
+	/* configure 18-bit HR + SpO2 readings at 400Hz */
 	ret = regmap_write(data->regmap, MAX30102_REG_SPO2_CONFIG,
 				(MAX30102_REG_SPO2_CONFIG_ADC_4096_STEPS
 				 << MAX30102_REG_SPO2_CONFIG_ADC_MASK_SHIFT) |
@@ -287,14 +411,6 @@ static int max30102_chip_init(struct max30102_data *data)
 	if (ret)
 		return ret;
 
-	/* enable SPO2 mode */
-	ret = regmap_update_bits(data->regmap, MAX30102_REG_MODE_CONFIG,
-				 MAX30102_REG_MODE_CONFIG_MODE_MASK,
-				 MAX30102_REG_MODE_CONFIG_MODE_HR_EN |
-				 MAX30102_REG_MODE_CONFIG_MODE_SPO2_EN);
-	if (ret)
-		return ret;
-
 	/* average 4 samples + generate FIFO interrupt */
 	ret = regmap_write(data->regmap, MAX30102_REG_FIFO_CONFIG,
 				(MAX30102_REG_FIFO_CONFIG_AVG_4SAMPLES
@@ -329,20 +445,31 @@ static int max30102_read_temp(struct max30102_data *data, int *val)
 	return 0;
 }
 
-static int max30102_get_temp(struct max30102_data *data, int *val)
+static int max30102_get_temp(struct max30102_data *data, int *val, bool en)
 {
 	int ret;
 
+	if (en) {
+		ret = max30102_set_power(data, true);
+		if (ret)
+			return ret;
+	}
+
 	/* start acquisition */
 	ret = regmap_update_bits(data->regmap, MAX30102_REG_TEMP_CONFIG,
 				 MAX30102_REG_TEMP_CONFIG_TEMP_EN,
 				 MAX30102_REG_TEMP_CONFIG_TEMP_EN);
 	if (ret)
-		return ret;
+		goto out;
 
 	msleep(35);
+	ret = max30102_read_temp(data, val);
 
-	return max30102_read_temp(data, val);
+out:
+	if (en)
+		max30102_set_power(data, false);
+
+	return ret;
 }
 
 static int max30102_read_raw(struct iio_dev *indio_dev,
@@ -355,20 +482,19 @@ static int max30102_read_raw(struct iio_dev *indio_dev,
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
 		/*
-		 * Temperature reading can only be acquired while engine
-		 * is running
+		 * Temperature reading can only be acquired when not in
+		 * shutdown; leave shutdown briefly when buffer not running
 		 */
 		mutex_lock(&indio_dev->mlock);
-
 		if (!iio_buffer_enabled(indio_dev))
-			ret = -EBUSY;
-		else {
-			ret = max30102_get_temp(data, val);
-			if (!ret)
-				ret = IIO_VAL_INT;
-		}
-
+			ret = max30102_get_temp(data, val, true);
+		else
+			ret = max30102_get_temp(data, val, false);
 		mutex_unlock(&indio_dev->mlock);
+		if (ret)
+			return ret;
+
+		ret = IIO_VAL_INT;
 		break;
 	case IIO_CHAN_INFO_SCALE:
 		*val = 1000;  /* 62.5 */
@@ -391,6 +517,7 @@ static int max30102_probe(struct i2c_client *client,
 	struct iio_buffer *buffer;
 	struct iio_dev *indio_dev;
 	int ret;
+	unsigned int reg;
 
 	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
 	if (!indio_dev)
@@ -403,10 +530,7 @@ static int max30102_probe(struct i2c_client *client,
 	iio_device_attach_buffer(indio_dev, buffer);
 
 	indio_dev->name = MAX30102_DRV_NAME;
-	indio_dev->channels = max30102_channels;
 	indio_dev->info = &max30102_info;
-	indio_dev->num_channels = ARRAY_SIZE(max30102_channels);
-	indio_dev->available_scan_masks = max30102_scan_masks;
 	indio_dev->modes = (INDIO_BUFFER_SOFTWARE | INDIO_DIRECT_MODE);
 	indio_dev->setup_ops = &max30102_buffer_setup_ops;
 	indio_dev->dev.parent = &client->dev;
@@ -414,16 +538,50 @@ static int max30102_probe(struct i2c_client *client,
 	data = iio_priv(indio_dev);
 	data->indio_dev = indio_dev;
 	data->client = client;
+	data->chip_id = id->driver_data;
 
 	mutex_init(&data->lock);
 	i2c_set_clientdata(client, indio_dev);
 
+	switch (data->chip_id) {
+	case max30105:
+		indio_dev->channels = max30105_channels;
+		indio_dev->num_channels = ARRAY_SIZE(max30105_channels);
+		indio_dev->available_scan_masks = max30105_scan_masks;
+		break;
+	case max30102:
+		indio_dev->channels = max30102_channels;
+		indio_dev->num_channels = ARRAY_SIZE(max30102_channels);
+		indio_dev->available_scan_masks = max30102_scan_masks;
+		break;
+	default:
+		return -ENODEV;
+	}
+
 	data->regmap = devm_regmap_init_i2c(client, &max30102_regmap_config);
 	if (IS_ERR(data->regmap)) {
-		dev_err(&client->dev, "regmap initialization failed.\n");
+		dev_err(&client->dev, "regmap initialization failed\n");
 		return PTR_ERR(data->regmap);
 	}
-	max30102_set_powermode(data, false);
+
+	/* check part ID */
+	ret = regmap_read(data->regmap, MAX30102_REG_PART_ID, &reg);
+	if (ret)
+		return ret;
+	if (reg != MAX30102_PART_NUMBER)
+		return -ENODEV;
+
+	/* show revision ID */
+	ret = regmap_read(data->regmap, MAX30102_REG_REV_ID, &reg);
+	if (ret)
+		return ret;
+	dev_dbg(&client->dev, "max3010x revision %02x\n", reg);
+
+	/* clear mode setting, chip shutdown */
+	ret = max30102_set_powermode(data, MAX30102_REG_MODE_CONFIG_MODE_NONE,
+				     false);
+	if (ret)
+		return ret;
 
 	ret = max30102_chip_init(data);
 	if (ret)
@@ -452,19 +610,21 @@ static int max30102_remove(struct i2c_client *client)
 	struct max30102_data *data = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
-	max30102_set_powermode(data, false);
+	max30102_set_power(data, false);
 
 	return 0;
 }
 
 static const struct i2c_device_id max30102_id[] = {
-	{ "max30102", 0 },
+	{ "max30102", max30102 },
+	{ "max30105", max30105 },
 	{}
 };
 MODULE_DEVICE_TABLE(i2c, max30102_id);
 
 static const struct of_device_id max30102_dt_ids[] = {
 	{ .compatible = "maxim,max30102" },
+	{ .compatible = "maxim,max30105" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, max30102_dt_ids);
@@ -481,5 +641,5 @@ static struct i2c_driver max30102_driver = {
 module_i2c_driver(max30102_driver);
 
 MODULE_AUTHOR("Matt Ranostay <matt@ranostay.consulting>");
-MODULE_DESCRIPTION("MAX30102 heart rate and pulse oximeter sensor");
+MODULE_DESCRIPTION("MAX30102 heart rate/pulse oximeter and MAX30105 particle sensor driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/iio/humidity/hts221.h b/drivers/iio/humidity/hts221.h
index 51d021966222..c581af8c0f5d 100644
--- a/drivers/iio/humidity/hts221.h
+++ b/drivers/iio/humidity/hts221.h
@@ -61,7 +61,8 @@ struct hts221_hw {
 extern const struct dev_pm_ops hts221_pm_ops;
 
 int hts221_write_with_mask(struct hts221_hw *hw, u8 addr, u8 mask, u8 val);
-int hts221_probe(struct iio_dev *iio_dev);
+int hts221_probe(struct device *dev, int irq, const char *name,
+		 const struct hts221_transfer_function *tf_ops);
 int hts221_set_enable(struct hts221_hw *hw, bool enable);
 int hts221_allocate_buffers(struct hts221_hw *hw);
 int hts221_allocate_trigger(struct hts221_hw *hw);
diff --git a/drivers/iio/humidity/hts221_core.c b/drivers/iio/humidity/hts221_core.c
index daef177219b6..d3f7904766bd 100644
--- a/drivers/iio/humidity/hts221_core.c
+++ b/drivers/iio/humidity/hts221_core.c
@@ -581,12 +581,26 @@ static const struct iio_info hts221_info = {
 
 static const unsigned long hts221_scan_masks[] = {0x3, 0x0};
 
-int hts221_probe(struct iio_dev *iio_dev)
+int hts221_probe(struct device *dev, int irq, const char *name,
+		 const struct hts221_transfer_function *tf_ops)
 {
-	struct hts221_hw *hw = iio_priv(iio_dev);
+	struct iio_dev *iio_dev;
+	struct hts221_hw *hw;
 	int err;
 	u8 data;
 
+	iio_dev = devm_iio_device_alloc(dev, sizeof(*hw));
+	if (!iio_dev)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, (void *)iio_dev);
+
+	hw = iio_priv(iio_dev);
+	hw->name = name;
+	hw->dev = dev;
+	hw->irq = irq;
+	hw->tf = tf_ops;
+
 	mutex_init(&hw->lock);
 
 	err = hts221_check_whoami(hw);
diff --git a/drivers/iio/humidity/hts221_i2c.c b/drivers/iio/humidity/hts221_i2c.c
index f38e4b7e0160..2c97350a0f76 100644
--- a/drivers/iio/humidity/hts221_i2c.c
+++ b/drivers/iio/humidity/hts221_i2c.c
@@ -66,22 +66,8 @@ static const struct hts221_transfer_function hts221_transfer_fn = {
 static int hts221_i2c_probe(struct i2c_client *client,
 			    const struct i2c_device_id *id)
 {
-	struct hts221_hw *hw;
-	struct iio_dev *iio_dev;
-
-	iio_dev = devm_iio_device_alloc(&client->dev, sizeof(*hw));
-	if (!iio_dev)
-		return -ENOMEM;
-
-	i2c_set_clientdata(client, iio_dev);
-
-	hw = iio_priv(iio_dev);
-	hw->name = client->name;
-	hw->dev = &client->dev;
-	hw->irq = client->irq;
-	hw->tf = &hts221_transfer_fn;
-
-	return hts221_probe(iio_dev);
+	return hts221_probe(&client->dev, client->irq,
+			    client->name, &hts221_transfer_fn);
 }
 
 static const struct acpi_device_id hts221_acpi_match[] = {
diff --git a/drivers/iio/humidity/hts221_spi.c b/drivers/iio/humidity/hts221_spi.c
index 57cbc256771b..55b29b53b9d1 100644
--- a/drivers/iio/humidity/hts221_spi.c
+++ b/drivers/iio/humidity/hts221_spi.c
@@ -80,22 +80,8 @@ static const struct hts221_transfer_function hts221_transfer_fn = {
 
 static int hts221_spi_probe(struct spi_device *spi)
 {
-	struct hts221_hw *hw;
-	struct iio_dev *iio_dev;
-
-	iio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*hw));
-	if (!iio_dev)
-		return -ENOMEM;
-
-	spi_set_drvdata(spi, iio_dev);
-
-	hw = iio_priv(iio_dev);
-	hw->name = spi->modalias;
-	hw->dev = &spi->dev;
-	hw->irq = spi->irq;
-	hw->tf = &hts221_transfer_fn;
-
-	return hts221_probe(iio_dev);
+	return hts221_probe(&spi->dev, spi->irq,
+			    spi->modalias, &hts221_transfer_fn);
 }
 
 static const struct of_device_id hts221_spi_of_match[] = {
diff --git a/drivers/iio/iio_core.h b/drivers/iio/iio_core.h
index 4c45488e3a7f..c775fedbcaf6 100644
--- a/drivers/iio/iio_core.h
+++ b/drivers/iio/iio_core.h
@@ -43,7 +43,7 @@ ssize_t iio_format_value(char *buf, unsigned int type, int size, int *vals);
 #ifdef CONFIG_IIO_BUFFER
 struct poll_table_struct;
 
-unsigned int iio_buffer_poll(struct file *filp,
+__poll_t iio_buffer_poll(struct file *filp,
 			     struct poll_table_struct *wait);
 ssize_t iio_buffer_read_first_n_outer(struct file *filp, char __user *buf,
 				      size_t n, loff_t *f_ps);
diff --git a/drivers/iio/imu/adis16480.c b/drivers/iio/imu/adis16480.c
index 7a33d6bd60e0..a27fe208f3ae 100644
--- a/drivers/iio/imu/adis16480.c
+++ b/drivers/iio/imu/adis16480.c
@@ -194,7 +194,7 @@ static int adis16480_show_serial_number(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16480_serial_number_fops,
+DEFINE_DEBUGFS_ATTRIBUTE(adis16480_serial_number_fops,
 	adis16480_show_serial_number, NULL, "0x%.4llx\n");
 
 static int adis16480_show_product_id(void *arg, u64 *val)
@@ -212,7 +212,7 @@ static int adis16480_show_product_id(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16480_product_id_fops,
+DEFINE_DEBUGFS_ATTRIBUTE(adis16480_product_id_fops,
 	adis16480_show_product_id, NULL, "%llu\n");
 
 static int adis16480_show_flash_count(void *arg, u64 *val)
@@ -230,24 +230,28 @@ static int adis16480_show_flash_count(void *arg, u64 *val)
 
 	return 0;
 }
-DEFINE_SIMPLE_ATTRIBUTE(adis16480_flash_count_fops,
+DEFINE_DEBUGFS_ATTRIBUTE(adis16480_flash_count_fops,
 	adis16480_show_flash_count, NULL, "%lld\n");
 
 static int adis16480_debugfs_init(struct iio_dev *indio_dev)
 {
 	struct adis16480 *adis16480 = iio_priv(indio_dev);
 
-	debugfs_create_file("firmware_revision", 0400,
+	debugfs_create_file_unsafe("firmware_revision", 0400,
 		indio_dev->debugfs_dentry, adis16480,
 		&adis16480_firmware_revision_fops);
-	debugfs_create_file("firmware_date", 0400, indio_dev->debugfs_dentry,
-		adis16480, &adis16480_firmware_date_fops);
-	debugfs_create_file("serial_number", 0400, indio_dev->debugfs_dentry,
-		adis16480, &adis16480_serial_number_fops);
-	debugfs_create_file("product_id", 0400, indio_dev->debugfs_dentry,
-		adis16480, &adis16480_product_id_fops);
-	debugfs_create_file("flash_count", 0400, indio_dev->debugfs_dentry,
-		adis16480, &adis16480_flash_count_fops);
+	debugfs_create_file_unsafe("firmware_date", 0400,
+		indio_dev->debugfs_dentry, adis16480,
+		&adis16480_firmware_date_fops);
+	debugfs_create_file_unsafe("serial_number", 0400,
+		indio_dev->debugfs_dentry, adis16480,
+		&adis16480_serial_number_fops);
+	debugfs_create_file_unsafe("product_id", 0400,
+		indio_dev->debugfs_dentry, adis16480,
+		&adis16480_product_id_fops);
+	debugfs_create_file_unsafe("flash_count", 0400,
+		indio_dev->debugfs_dentry, adis16480,
+		&adis16480_flash_count_fops);
 
 	return 0;
 }
diff --git a/drivers/iio/imu/adis_trigger.c b/drivers/iio/imu/adis_trigger.c
index 0dd5a381be64..457372f36791 100644
--- a/drivers/iio/imu/adis_trigger.c
+++ b/drivers/iio/imu/adis_trigger.c
@@ -46,6 +46,10 @@ int adis_probe_trigger(struct adis *adis, struct iio_dev *indio_dev)
 	if (adis->trig == NULL)
 		return -ENOMEM;
 
+	adis->trig->dev.parent = &adis->spi->dev;
+	adis->trig->ops = &adis_trigger_ops;
+	iio_trigger_set_drvdata(adis->trig, adis);
+
 	ret = request_irq(adis->spi->irq,
 			  &iio_trigger_generic_data_rdy_poll,
 			  IRQF_TRIGGER_RISING,
@@ -54,9 +58,6 @@ int adis_probe_trigger(struct adis *adis, struct iio_dev *indio_dev)
 	if (ret)
 		goto error_free_trig;
 
-	adis->trig->dev.parent = &adis->spi->dev;
-	adis->trig->ops = &adis_trigger_ops;
-	iio_trigger_set_drvdata(adis->trig, adis);
 	ret = iio_trigger_register(adis->trig);
 
 	indio_dev->trig = iio_trigger_get(adis->trig);
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c
index dd6fc6d21f9d..d78a10403bac 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c
@@ -196,8 +196,7 @@ void inv_mpu_acpi_delete_mux_client(struct i2c_client *client)
 {
 	struct inv_mpu6050_state *st = iio_priv(dev_get_drvdata(&client->dev));
 
-	if (st->mux_client)
-		i2c_unregister_device(st->mux_client);
+	i2c_unregister_device(st->mux_client);
 }
 #else
 
diff --git a/drivers/iio/imu/st_lsm6dsx/Kconfig b/drivers/iio/imu/st_lsm6dsx/Kconfig
index e57337159b57..14f2eb6e9fb7 100644
--- a/drivers/iio/imu/st_lsm6dsx/Kconfig
+++ b/drivers/iio/imu/st_lsm6dsx/Kconfig
@@ -16,7 +16,9 @@ config IIO_ST_LSM6DSX
 config IIO_ST_LSM6DSX_I2C
 	tristate
 	depends on IIO_ST_LSM6DSX
+	select REGMAP_I2C
 
 config IIO_ST_LSM6DSX_SPI
 	tristate
 	depends on IIO_ST_LSM6DSX
+	select REGMAP_SPI
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
index 4fdb7fcc3ea8..8fdd723afa05 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
@@ -27,23 +27,12 @@ enum st_lsm6dsx_hw_id {
 	ST_LSM6DSX_MAX_ID,
 };
 
+#define ST_LSM6DSX_BUFF_SIZE		256
 #define ST_LSM6DSX_CHAN_SIZE		2
 #define ST_LSM6DSX_SAMPLE_SIZE		6
-
-#if defined(CONFIG_SPI_MASTER)
-#define ST_LSM6DSX_RX_MAX_LENGTH	256
-#define ST_LSM6DSX_TX_MAX_LENGTH	8
-
-struct st_lsm6dsx_transfer_buffer {
-	u8 rx_buf[ST_LSM6DSX_RX_MAX_LENGTH];
-	u8 tx_buf[ST_LSM6DSX_TX_MAX_LENGTH] ____cacheline_aligned;
-};
-#endif /* CONFIG_SPI_MASTER */
-
-struct st_lsm6dsx_transfer_function {
-	int (*read)(struct device *dev, u8 addr, int len, u8 *data);
-	int (*write)(struct device *dev, u8 addr, int len, u8 *data);
-};
+#define ST_LSM6DSX_MAX_WORD_LEN		((32 / ST_LSM6DSX_SAMPLE_SIZE) * \
+					 ST_LSM6DSX_SAMPLE_SIZE)
+#define ST_LSM6DSX_SHIFT_VAL(val, mask)	(((val) << __ffs(mask)) & (mask))
 
 struct st_lsm6dsx_reg {
 	u8 addr;
@@ -127,47 +116,43 @@ struct st_lsm6dsx_sensor {
 /**
  * struct st_lsm6dsx_hw - ST IMU MEMS hw instance
  * @dev: Pointer to instance of struct device (I2C or SPI).
+ * @regmap: Register map of the device.
  * @irq: Device interrupt line (I2C or SPI).
- * @lock: Mutex to protect read and write operations.
  * @fifo_lock: Mutex to prevent concurrent access to the hw FIFO.
+ * @conf_lock: Mutex to prevent concurrent FIFO configuration update.
  * @fifo_mode: FIFO operating mode supported by the device.
  * @enable_mask: Enabled sensor bitmask.
  * @sip: Total number of samples (acc/gyro) in a given pattern.
+ * @buff: Device read buffer.
  * @iio_devs: Pointers to acc/gyro iio_dev instances.
  * @settings: Pointer to the specific sensor settings in use.
- * @tf: Transfer function structure used by I/O operations.
- * @tb: Transfer buffers used by SPI I/O operations.
  */
 struct st_lsm6dsx_hw {
 	struct device *dev;
+	struct regmap *regmap;
 	int irq;
 
-	struct mutex lock;
 	struct mutex fifo_lock;
+	struct mutex conf_lock;
 
 	enum st_lsm6dsx_fifo_mode fifo_mode;
 	u8 enable_mask;
 	u8 sip;
 
+	u8 *buff;
+
 	struct iio_dev *iio_devs[ST_LSM6DSX_ID_MAX];
 
 	const struct st_lsm6dsx_settings *settings;
-
-	const struct st_lsm6dsx_transfer_function *tf;
-#if defined(CONFIG_SPI_MASTER)
-	struct st_lsm6dsx_transfer_buffer tb;
-#endif /* CONFIG_SPI_MASTER */
 };
 
 extern const struct dev_pm_ops st_lsm6dsx_pm_ops;
 
 int st_lsm6dsx_probe(struct device *dev, int irq, int hw_id, const char *name,
-		     const struct st_lsm6dsx_transfer_function *tf_ops);
+		     struct regmap *regmap);
 int st_lsm6dsx_sensor_enable(struct st_lsm6dsx_sensor *sensor);
 int st_lsm6dsx_sensor_disable(struct st_lsm6dsx_sensor *sensor);
 int st_lsm6dsx_fifo_setup(struct st_lsm6dsx_hw *hw);
-int st_lsm6dsx_write_with_mask(struct st_lsm6dsx_hw *hw, u8 addr, u8 mask,
-			       u8 val);
 int st_lsm6dsx_update_watermark(struct st_lsm6dsx_sensor *sensor,
 				u16 watermark);
 int st_lsm6dsx_flush_fifo(struct st_lsm6dsx_hw *hw);
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 755c472e8a05..1d6aa9b1a4cf 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -30,6 +30,8 @@
 #include <linux/iio/kfifo_buf.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/buffer.h>
+#include <linux/regmap.h>
+#include <linux/bitfield.h>
 
 #include <linux/platform_data/st_sensors_pdata.h>
 
@@ -120,8 +122,10 @@ static int st_lsm6dsx_update_decimators(struct st_lsm6dsx_hw *hw)
 
 		dec_reg = &hw->settings->decimator[sensor->id];
 		if (dec_reg->addr) {
-			err = st_lsm6dsx_write_with_mask(hw, dec_reg->addr,
-							 dec_reg->mask, data);
+			int val = ST_LSM6DSX_SHIFT_VAL(data, dec_reg->mask);
+
+			err = regmap_update_bits(hw->regmap, dec_reg->addr,
+						 dec_reg->mask, val);
 			if (err < 0)
 				return err;
 		}
@@ -137,8 +141,10 @@ int st_lsm6dsx_set_fifo_mode(struct st_lsm6dsx_hw *hw,
 {
 	int err;
 
-	err = st_lsm6dsx_write_with_mask(hw, ST_LSM6DSX_REG_FIFO_MODE_ADDR,
-					 ST_LSM6DSX_FIFO_MODE_MASK, fifo_mode);
+	err = regmap_update_bits(hw->regmap, ST_LSM6DSX_REG_FIFO_MODE_ADDR,
+				 ST_LSM6DSX_FIFO_MODE_MASK,
+				 FIELD_PREP(ST_LSM6DSX_FIFO_MODE_MASK,
+					    fifo_mode));
 	if (err < 0)
 		return err;
 
@@ -154,8 +160,9 @@ static int st_lsm6dsx_set_fifo_odr(struct st_lsm6dsx_sensor *sensor,
 	u8 data;
 
 	data = hw->enable_mask ? ST_LSM6DSX_MAX_FIFO_ODR_VAL : 0;
-	return st_lsm6dsx_write_with_mask(hw, ST_LSM6DSX_REG_FIFO_MODE_ADDR,
-					  ST_LSM6DSX_FIFO_ODR_MASK, data);
+	return regmap_update_bits(hw->regmap, ST_LSM6DSX_REG_FIFO_MODE_ADDR,
+				 ST_LSM6DSX_FIFO_ODR_MASK,
+				 FIELD_PREP(ST_LSM6DSX_FIFO_ODR_MASK, data));
 }
 
 int st_lsm6dsx_update_watermark(struct st_lsm6dsx_sensor *sensor, u16 watermark)
@@ -163,9 +170,8 @@ int st_lsm6dsx_update_watermark(struct st_lsm6dsx_sensor *sensor, u16 watermark)
 	u16 fifo_watermark = ~0, cur_watermark, sip = 0, fifo_th_mask;
 	struct st_lsm6dsx_hw *hw = sensor->hw;
 	struct st_lsm6dsx_sensor *cur_sensor;
+	int i, err, data;
 	__le16 wdata;
-	int i, err;
-	u8 data;
 
 	for (i = 0; i < ST_LSM6DSX_ID_MAX; i++) {
 		cur_sensor = iio_priv(hw->iio_devs[i]);
@@ -187,24 +193,42 @@ int st_lsm6dsx_update_watermark(struct st_lsm6dsx_sensor *sensor, u16 watermark)
 	fifo_watermark = (fifo_watermark / sip) * sip;
 	fifo_watermark = fifo_watermark * hw->settings->fifo_ops.th_wl;
 
-	mutex_lock(&hw->lock);
-
-	err = hw->tf->read(hw->dev, hw->settings->fifo_ops.fifo_th.addr + 1,
-			   sizeof(data), &data);
+	err = regmap_read(hw->regmap, hw->settings->fifo_ops.fifo_th.addr + 1,
+			  &data);
 	if (err < 0)
-		goto out;
+		return err;
 
 	fifo_th_mask = hw->settings->fifo_ops.fifo_th.mask;
 	fifo_watermark = ((data << 8) & ~fifo_th_mask) |
 			 (fifo_watermark & fifo_th_mask);
 
 	wdata = cpu_to_le16(fifo_watermark);
-	err = hw->tf->write(hw->dev, hw->settings->fifo_ops.fifo_th.addr,
-			    sizeof(wdata), (u8 *)&wdata);
-out:
-	mutex_unlock(&hw->lock);
+	return regmap_bulk_write(hw->regmap,
+				 hw->settings->fifo_ops.fifo_th.addr,
+				 &wdata, sizeof(wdata));
+}
 
-	return err < 0 ? err : 0;
+/*
+ * Set max bulk read to ST_LSM6DSX_MAX_WORD_LEN in order to avoid
+ * a kmalloc for each bus access
+ */
+static inline int st_lsm6dsx_read_block(struct st_lsm6dsx_hw *hw, u8 *data,
+					unsigned int data_len)
+{
+	unsigned int word_len, read_len = 0;
+	int err;
+
+	while (read_len < data_len) {
+		word_len = min_t(unsigned int, data_len - read_len,
+				 ST_LSM6DSX_MAX_WORD_LEN);
+		err = regmap_bulk_read(hw->regmap,
+				       ST_LSM6DSX_REG_FIFO_OUTL_ADDR,
+				       data + read_len, word_len);
+		if (err < 0)
+			return err;
+		read_len += word_len;
+	}
+	return 0;
 }
 
 /**
@@ -223,11 +247,11 @@ static int st_lsm6dsx_read_fifo(struct st_lsm6dsx_hw *hw)
 	struct st_lsm6dsx_sensor *acc_sensor, *gyro_sensor;
 	s64 acc_ts, acc_delta_ts, gyro_ts, gyro_delta_ts;
 	u8 iio_buff[ALIGN(ST_LSM6DSX_SAMPLE_SIZE, sizeof(s64)) + sizeof(s64)];
-	u8 buff[pattern_len];
 	__le16 fifo_status;
 
-	err = hw->tf->read(hw->dev, hw->settings->fifo_ops.fifo_diff.addr,
-			   sizeof(fifo_status), (u8 *)&fifo_status);
+	err = regmap_bulk_read(hw->regmap,
+			       hw->settings->fifo_ops.fifo_diff.addr,
+			       &fifo_status, sizeof(fifo_status));
 	if (err < 0)
 		return err;
 
@@ -255,8 +279,7 @@ static int st_lsm6dsx_read_fifo(struct st_lsm6dsx_hw *hw)
 				samples);
 
 	for (read_len = 0; read_len < fifo_len; read_len += pattern_len) {
-		err = hw->tf->read(hw->dev, ST_LSM6DSX_REG_FIFO_OUTL_ADDR,
-				   sizeof(buff), buff);
+		err = st_lsm6dsx_read_block(hw, hw->buff, pattern_len);
 		if (err < 0)
 			return err;
 
@@ -281,7 +304,7 @@ static int st_lsm6dsx_read_fifo(struct st_lsm6dsx_hw *hw)
 
 		while (acc_sip > 0 || gyro_sip > 0) {
 			if (gyro_sip-- > 0) {
-				memcpy(iio_buff, &buff[offset],
+				memcpy(iio_buff, &hw->buff[offset],
 				       ST_LSM6DSX_SAMPLE_SIZE);
 				iio_push_to_buffers_with_timestamp(
 					hw->iio_devs[ST_LSM6DSX_ID_GYRO],
@@ -291,7 +314,7 @@ static int st_lsm6dsx_read_fifo(struct st_lsm6dsx_hw *hw)
 			}
 
 			if (acc_sip-- > 0) {
-				memcpy(iio_buff, &buff[offset],
+				memcpy(iio_buff, &hw->buff[offset],
 				       ST_LSM6DSX_SAMPLE_SIZE);
 				iio_push_to_buffers_with_timestamp(
 					hw->iio_devs[ST_LSM6DSX_ID_ACC],
@@ -325,38 +348,40 @@ static int st_lsm6dsx_update_fifo(struct iio_dev *iio_dev, bool enable)
 	struct st_lsm6dsx_hw *hw = sensor->hw;
 	int err;
 
+	mutex_lock(&hw->conf_lock);
+
 	if (hw->fifo_mode != ST_LSM6DSX_FIFO_BYPASS) {
 		err = st_lsm6dsx_flush_fifo(hw);
 		if (err < 0)
-			return err;
+			goto out;
 	}
 
 	if (enable) {
 		err = st_lsm6dsx_sensor_enable(sensor);
 		if (err < 0)
-			return err;
+			goto out;
 	} else {
 		err = st_lsm6dsx_sensor_disable(sensor);
 		if (err < 0)
-			return err;
+			goto out;
 	}
 
 	err = st_lsm6dsx_set_fifo_odr(sensor, enable);
 	if (err < 0)
-		return err;
+		goto out;
 
 	err = st_lsm6dsx_update_decimators(hw);
 	if (err < 0)
-		return err;
+		goto out;
 
 	err = st_lsm6dsx_update_watermark(sensor, sensor->watermark);
 	if (err < 0)
-		return err;
+		goto out;
 
 	if (hw->enable_mask) {
 		err = st_lsm6dsx_set_fifo_mode(hw, ST_LSM6DSX_FIFO_CONT);
 		if (err < 0)
-			return err;
+			goto out;
 
 		/*
 		 * store enable buffer timestamp as reference to compute
@@ -365,7 +390,10 @@ static int st_lsm6dsx_update_fifo(struct iio_dev *iio_dev, bool enable)
 		sensor->ts = iio_get_time_ns(iio_dev);
 	}
 
-	return 0;
+out:
+	mutex_unlock(&hw->conf_lock);
+
+	return err;
 }
 
 static irqreturn_t st_lsm6dsx_handler_irq(int irq, void *private)
@@ -444,17 +472,20 @@ int st_lsm6dsx_fifo_setup(struct st_lsm6dsx_hw *hw)
 		return -EINVAL;
 	}
 
-	err = st_lsm6dsx_write_with_mask(hw, ST_LSM6DSX_REG_HLACTIVE_ADDR,
-					 ST_LSM6DSX_REG_HLACTIVE_MASK,
-					 irq_active_low);
+	err = regmap_update_bits(hw->regmap, ST_LSM6DSX_REG_HLACTIVE_ADDR,
+				 ST_LSM6DSX_REG_HLACTIVE_MASK,
+				 FIELD_PREP(ST_LSM6DSX_REG_HLACTIVE_MASK,
+					    irq_active_low));
 	if (err < 0)
 		return err;
 
 	pdata = (struct st_sensors_platform_data *)hw->dev->platform_data;
 	if ((np && of_property_read_bool(np, "drive-open-drain")) ||
 	    (pdata && pdata->open_drain)) {
-		err = st_lsm6dsx_write_with_mask(hw, ST_LSM6DSX_REG_PP_OD_ADDR,
-						 ST_LSM6DSX_REG_PP_OD_MASK, 1);
+		err = regmap_update_bits(hw->regmap, ST_LSM6DSX_REG_PP_OD_ADDR,
+					 ST_LSM6DSX_REG_PP_OD_MASK,
+					 FIELD_PREP(ST_LSM6DSX_REG_PP_OD_MASK,
+						    1));
 		if (err < 0)
 			return err;
 
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
index 239c735242be..c2fa3239b9c6 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
@@ -37,6 +37,8 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 #include <linux/pm.h>
+#include <linux/regmap.h>
+#include <linux/bitfield.h>
 
 #include <linux/platform_data/st_sensors_pdata.h>
 
@@ -277,36 +279,9 @@ static const struct iio_chan_spec st_lsm6dsx_gyro_channels[] = {
 	IIO_CHAN_SOFT_TIMESTAMP(3),
 };
 
-int st_lsm6dsx_write_with_mask(struct st_lsm6dsx_hw *hw, u8 addr, u8 mask,
-			       u8 val)
-{
-	u8 data;
-	int err;
-
-	mutex_lock(&hw->lock);
-
-	err = hw->tf->read(hw->dev, addr, sizeof(data), &data);
-	if (err < 0) {
-		dev_err(hw->dev, "failed to read %02x register\n", addr);
-		goto out;
-	}
-
-	data = (data & ~mask) | ((val << __ffs(mask)) & mask);
-
-	err = hw->tf->write(hw->dev, addr, sizeof(data), &data);
-	if (err < 0)
-		dev_err(hw->dev, "failed to write %02x register\n", addr);
-
-out:
-	mutex_unlock(&hw->lock);
-
-	return err;
-}
-
 static int st_lsm6dsx_check_whoami(struct st_lsm6dsx_hw *hw, int id)
 {
-	int err, i, j;
-	u8 data;
+	int err, i, j, data;
 
 	for (i = 0; i < ARRAY_SIZE(st_lsm6dsx_sensor_settings); i++) {
 		for (j = 0; j < ST_LSM6DSX_MAX_ID; j++) {
@@ -322,8 +297,7 @@ static int st_lsm6dsx_check_whoami(struct st_lsm6dsx_hw *hw, int id)
 		return -ENODEV;
 	}
 
-	err = hw->tf->read(hw->dev, ST_LSM6DSX_REG_WHOAMI_ADDR, sizeof(data),
-			   &data);
+	err = regmap_read(hw->regmap, ST_LSM6DSX_REG_WHOAMI_ADDR, &data);
 	if (err < 0) {
 		dev_err(hw->dev, "failed to read whoami register\n");
 		return err;
@@ -342,22 +316,22 @@ static int st_lsm6dsx_check_whoami(struct st_lsm6dsx_hw *hw, int id)
 static int st_lsm6dsx_set_full_scale(struct st_lsm6dsx_sensor *sensor,
 				     u32 gain)
 {
-	enum st_lsm6dsx_sensor_id id = sensor->id;
+	struct st_lsm6dsx_hw *hw = sensor->hw;
+	const struct st_lsm6dsx_reg *reg;
 	int i, err;
 	u8 val;
 
 	for (i = 0; i < ST_LSM6DSX_FS_LIST_SIZE; i++)
-		if (st_lsm6dsx_fs_table[id].fs_avl[i].gain == gain)
+		if (st_lsm6dsx_fs_table[sensor->id].fs_avl[i].gain == gain)
 			break;
 
 	if (i == ST_LSM6DSX_FS_LIST_SIZE)
 		return -EINVAL;
 
-	val = st_lsm6dsx_fs_table[id].fs_avl[i].val;
-	err = st_lsm6dsx_write_with_mask(sensor->hw,
-					 st_lsm6dsx_fs_table[id].reg.addr,
-					 st_lsm6dsx_fs_table[id].reg.mask,
-					 val);
+	val = st_lsm6dsx_fs_table[sensor->id].fs_avl[i].val;
+	reg = &st_lsm6dsx_fs_table[sensor->id].reg;
+	err = regmap_update_bits(hw->regmap, reg->addr, reg->mask,
+				 ST_LSM6DSX_SHIFT_VAL(val, reg->mask));
 	if (err < 0)
 		return err;
 
@@ -385,7 +359,8 @@ static int st_lsm6dsx_check_odr(struct st_lsm6dsx_sensor *sensor, u16 odr,
 
 static int st_lsm6dsx_set_odr(struct st_lsm6dsx_sensor *sensor, u16 odr)
 {
-	enum st_lsm6dsx_sensor_id id = sensor->id;
+	struct st_lsm6dsx_hw *hw = sensor->hw;
+	const struct st_lsm6dsx_reg *reg;
 	int err;
 	u8 val;
 
@@ -393,10 +368,9 @@ static int st_lsm6dsx_set_odr(struct st_lsm6dsx_sensor *sensor, u16 odr)
 	if (err < 0)
 		return err;
 
-	return st_lsm6dsx_write_with_mask(sensor->hw,
-					  st_lsm6dsx_odr_table[id].reg.addr,
-					  st_lsm6dsx_odr_table[id].reg.mask,
-					  val);
+	reg = &st_lsm6dsx_odr_table[sensor->id].reg;
+	return regmap_update_bits(hw->regmap, reg->addr, reg->mask,
+				  ST_LSM6DSX_SHIFT_VAL(val, reg->mask));
 }
 
 int st_lsm6dsx_sensor_enable(struct st_lsm6dsx_sensor *sensor)
@@ -414,16 +388,17 @@ int st_lsm6dsx_sensor_enable(struct st_lsm6dsx_sensor *sensor)
 
 int st_lsm6dsx_sensor_disable(struct st_lsm6dsx_sensor *sensor)
 {
-	enum st_lsm6dsx_sensor_id id = sensor->id;
+	struct st_lsm6dsx_hw *hw = sensor->hw;
+	const struct st_lsm6dsx_reg *reg;
 	int err;
 
-	err = st_lsm6dsx_write_with_mask(sensor->hw,
-					 st_lsm6dsx_odr_table[id].reg.addr,
-					 st_lsm6dsx_odr_table[id].reg.mask, 0);
+	reg = &st_lsm6dsx_odr_table[sensor->id].reg;
+	err = regmap_update_bits(hw->regmap, reg->addr, reg->mask,
+				 ST_LSM6DSX_SHIFT_VAL(0, reg->mask));
 	if (err < 0)
 		return err;
 
-	sensor->hw->enable_mask &= ~BIT(id);
+	sensor->hw->enable_mask &= ~BIT(sensor->id);
 
 	return 0;
 }
@@ -431,6 +406,7 @@ int st_lsm6dsx_sensor_disable(struct st_lsm6dsx_sensor *sensor)
 static int st_lsm6dsx_read_oneshot(struct st_lsm6dsx_sensor *sensor,
 				   u8 addr, int *val)
 {
+	struct st_lsm6dsx_hw *hw = sensor->hw;
 	int err, delay;
 	__le16 data;
 
@@ -441,14 +417,13 @@ static int st_lsm6dsx_read_oneshot(struct st_lsm6dsx_sensor *sensor,
 	delay = 1000000 / sensor->odr;
 	usleep_range(delay, 2 * delay);
 
-	err = sensor->hw->tf->read(sensor->hw->dev, addr, sizeof(data),
-				   (u8 *)&data);
+	err = regmap_bulk_read(hw->regmap, addr, &data, sizeof(data));
 	if (err < 0)
 		return err;
 
 	st_lsm6dsx_sensor_disable(sensor);
 
-	*val = (s16)data;
+	*val = (s16)le16_to_cpu(data);
 
 	return IIO_VAL_INT;
 }
@@ -528,7 +503,12 @@ static int st_lsm6dsx_set_watermark(struct iio_dev *iio_dev, unsigned int val)
 	if (val < 1 || val > hw->settings->max_fifo_size)
 		return -EINVAL;
 
+	mutex_lock(&hw->conf_lock);
+
 	err = st_lsm6dsx_update_watermark(sensor, val);
+
+	mutex_unlock(&hw->conf_lock);
+
 	if (err < 0)
 		return err;
 
@@ -652,20 +632,20 @@ static int st_lsm6dsx_get_drdy_reg(struct st_lsm6dsx_hw *hw, u8 *drdy_reg)
 
 static int st_lsm6dsx_init_device(struct st_lsm6dsx_hw *hw)
 {
-	u8 data, drdy_int_reg;
+	u8 drdy_int_reg;
 	int err;
 
-	data = ST_LSM6DSX_REG_RESET_MASK;
-	err = hw->tf->write(hw->dev, ST_LSM6DSX_REG_RESET_ADDR, sizeof(data),
-			    &data);
+	err = regmap_write(hw->regmap, ST_LSM6DSX_REG_RESET_ADDR,
+			   ST_LSM6DSX_REG_RESET_MASK);
 	if (err < 0)
 		return err;
 
 	msleep(200);
 
 	/* enable Block Data Update */
-	err = st_lsm6dsx_write_with_mask(hw, ST_LSM6DSX_REG_BDU_ADDR,
-					 ST_LSM6DSX_REG_BDU_MASK, 1);
+	err = regmap_update_bits(hw->regmap, ST_LSM6DSX_REG_BDU_ADDR,
+				 ST_LSM6DSX_REG_BDU_MASK,
+				 FIELD_PREP(ST_LSM6DSX_REG_BDU_MASK, 1));
 	if (err < 0)
 		return err;
 
@@ -674,8 +654,10 @@ static int st_lsm6dsx_init_device(struct st_lsm6dsx_hw *hw)
 	if (err < 0)
 		return err;
 
-	return st_lsm6dsx_write_with_mask(hw, drdy_int_reg,
-					  ST_LSM6DSX_REG_FIFO_FTH_IRQ_MASK, 1);
+	return regmap_update_bits(hw->regmap, drdy_int_reg,
+				  ST_LSM6DSX_REG_FIFO_FTH_IRQ_MASK,
+				  FIELD_PREP(ST_LSM6DSX_REG_FIFO_FTH_IRQ_MASK,
+					     1));
 }
 
 static struct iio_dev *st_lsm6dsx_alloc_iiodev(struct st_lsm6dsx_hw *hw,
@@ -726,7 +708,7 @@ static struct iio_dev *st_lsm6dsx_alloc_iiodev(struct st_lsm6dsx_hw *hw,
 }
 
 int st_lsm6dsx_probe(struct device *dev, int irq, int hw_id, const char *name,
-		     const struct st_lsm6dsx_transfer_function *tf_ops)
+		     struct regmap *regmap)
 {
 	struct st_lsm6dsx_hw *hw;
 	int i, err;
@@ -737,12 +719,16 @@ int st_lsm6dsx_probe(struct device *dev, int irq, int hw_id, const char *name,
 
 	dev_set_drvdata(dev, (void *)hw);
 
-	mutex_init(&hw->lock);
 	mutex_init(&hw->fifo_lock);
+	mutex_init(&hw->conf_lock);
+
+	hw->buff = devm_kzalloc(dev, ST_LSM6DSX_BUFF_SIZE, GFP_KERNEL);
+	if (!hw->buff)
+		return -ENOMEM;
 
 	hw->dev = dev;
 	hw->irq = irq;
-	hw->tf = tf_ops;
+	hw->regmap = regmap;
 
 	err = st_lsm6dsx_check_whoami(hw, hw_id);
 	if (err < 0)
@@ -778,6 +764,7 @@ static int __maybe_unused st_lsm6dsx_suspend(struct device *dev)
 {
 	struct st_lsm6dsx_hw *hw = dev_get_drvdata(dev);
 	struct st_lsm6dsx_sensor *sensor;
+	const struct st_lsm6dsx_reg *reg;
 	int i, err = 0;
 
 	for (i = 0; i < ST_LSM6DSX_ID_MAX; i++) {
@@ -785,9 +772,9 @@ static int __maybe_unused st_lsm6dsx_suspend(struct device *dev)
 		if (!(hw->enable_mask & BIT(sensor->id)))
 			continue;
 
-		err = st_lsm6dsx_write_with_mask(hw,
-				st_lsm6dsx_odr_table[sensor->id].reg.addr,
-				st_lsm6dsx_odr_table[sensor->id].reg.mask, 0);
+		reg = &st_lsm6dsx_odr_table[sensor->id].reg;
+		err = regmap_update_bits(hw->regmap, reg->addr, reg->mask,
+					 ST_LSM6DSX_SHIFT_VAL(0, reg->mask));
 		if (err < 0)
 			return err;
 	}
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
index 305fec712ab0..41525dd2aab7 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c
@@ -14,55 +14,30 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/regmap.h>
 
 #include "st_lsm6dsx.h"
 
-static int st_lsm6dsx_i2c_read(struct device *dev, u8 addr, int len, u8 *data)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct i2c_msg msg[2];
-
-	msg[0].addr = client->addr;
-	msg[0].flags = client->flags;
-	msg[0].len = 1;
-	msg[0].buf = &addr;
-
-	msg[1].addr = client->addr;
-	msg[1].flags = client->flags | I2C_M_RD;
-	msg[1].len = len;
-	msg[1].buf = data;
-
-	return i2c_transfer(client->adapter, msg, 2);
-}
-
-static int st_lsm6dsx_i2c_write(struct device *dev, u8 addr, int len, u8 *data)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct i2c_msg msg;
-	u8 send[len + 1];
-
-	send[0] = addr;
-	memcpy(&send[1], data, len * sizeof(u8));
-
-	msg.addr = client->addr;
-	msg.flags = client->flags;
-	msg.len = len + 1;
-	msg.buf = send;
-
-	return i2c_transfer(client->adapter, &msg, 1);
-}
-
-static const struct st_lsm6dsx_transfer_function st_lsm6dsx_transfer_fn = {
-	.read = st_lsm6dsx_i2c_read,
-	.write = st_lsm6dsx_i2c_write,
+static const struct regmap_config st_lsm6dsx_i2c_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
 };
 
 static int st_lsm6dsx_i2c_probe(struct i2c_client *client,
 				const struct i2c_device_id *id)
 {
+	int hw_id = id->driver_data;
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_i2c(client, &st_lsm6dsx_i2c_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
+			(int)PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
 	return st_lsm6dsx_probe(&client->dev, client->irq,
-				(int)id->driver_data, id->name,
-				&st_lsm6dsx_transfer_fn);
+				hw_id, id->name, regmap);
 }
 
 static const struct of_device_id st_lsm6dsx_i2c_of_match[] = {
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
index 95472f153ad2..2c8135834479 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c
@@ -14,72 +14,30 @@
 #include <linux/spi/spi.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/regmap.h>
 
 #include "st_lsm6dsx.h"
 
-#define SENSORS_SPI_READ	BIT(7)
-
-static int st_lsm6dsx_spi_read(struct device *dev, u8 addr, int len,
-			       u8 *data)
-{
-	struct spi_device *spi = to_spi_device(dev);
-	struct st_lsm6dsx_hw *hw = spi_get_drvdata(spi);
-	int err;
-
-	struct spi_transfer xfers[] = {
-		{
-			.tx_buf = hw->tb.tx_buf,
-			.bits_per_word = 8,
-			.len = 1,
-		},
-		{
-			.rx_buf = hw->tb.rx_buf,
-			.bits_per_word = 8,
-			.len = len,
-		}
-	};
-
-	hw->tb.tx_buf[0] = addr | SENSORS_SPI_READ;
-
-	err = spi_sync_transfer(spi, xfers,  ARRAY_SIZE(xfers));
-	if (err < 0)
-		return err;
-
-	memcpy(data, hw->tb.rx_buf, len * sizeof(u8));
-
-	return len;
-}
-
-static int st_lsm6dsx_spi_write(struct device *dev, u8 addr, int len,
-				u8 *data)
-{
-	struct st_lsm6dsx_hw *hw;
-	struct spi_device *spi;
-
-	if (len >= ST_LSM6DSX_TX_MAX_LENGTH)
-		return -ENOMEM;
-
-	spi = to_spi_device(dev);
-	hw = spi_get_drvdata(spi);
-
-	hw->tb.tx_buf[0] = addr;
-	memcpy(&hw->tb.tx_buf[1], data, len);
-
-	return spi_write(spi, hw->tb.tx_buf, len + 1);
-}
-
-static const struct st_lsm6dsx_transfer_function st_lsm6dsx_transfer_fn = {
-	.read = st_lsm6dsx_spi_read,
-	.write = st_lsm6dsx_spi_write,
+static const struct regmap_config st_lsm6dsx_spi_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
 };
 
 static int st_lsm6dsx_spi_probe(struct spi_device *spi)
 {
 	const struct spi_device_id *id = spi_get_device_id(spi);
+	int hw_id = id->driver_data;
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_spi(spi, &st_lsm6dsx_spi_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
+			(int)PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
 
 	return st_lsm6dsx_probe(&spi->dev, spi->irq,
-				(int)id->driver_data, id->name,
-				&st_lsm6dsx_transfer_fn);
+				hw_id, id->name, regmap);
 }
 
 static const struct of_device_id st_lsm6dsx_spi_of_match[] = {
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index d2b465140a6b..cd5bfe39591b 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -166,21 +166,21 @@ ssize_t iio_buffer_read_first_n_outer(struct file *filp, char __user *buf,
  * @wait:	Poll table structure pointer for which the driver adds
  *		a wait queue
  *
- * Return: (POLLIN | POLLRDNORM) if data is available for reading
+ * Return: (EPOLLIN | EPOLLRDNORM) if data is available for reading
  *	   or 0 for other cases
  */
-unsigned int iio_buffer_poll(struct file *filp,
+__poll_t iio_buffer_poll(struct file *filp,
 			     struct poll_table_struct *wait)
 {
 	struct iio_dev *indio_dev = filp->private_data;
 	struct iio_buffer *rb = indio_dev->buffer;
 
-	if (!indio_dev->info)
+	if (!indio_dev->info || rb == NULL)
 		return 0;
 
 	poll_wait(filp, &rb->pollq, wait);
 	if (iio_buffer_ready(indio_dev, rb, rb->watermark, 0))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 
@@ -1198,6 +1198,18 @@ static ssize_t iio_buffer_store_watermark(struct device *dev,
 	return ret ? ret : len;
 }
 
+static ssize_t iio_dma_show_data_available(struct device *dev,
+						struct device_attribute *attr,
+						char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	size_t bytes;
+
+	bytes = iio_buffer_data_available(indio_dev->buffer);
+
+	return sprintf(buf, "%zu\n", bytes);
+}
+
 static DEVICE_ATTR(length, S_IRUGO | S_IWUSR, iio_buffer_read_length,
 		   iio_buffer_write_length);
 static struct device_attribute dev_attr_length_ro = __ATTR(length,
@@ -1208,11 +1220,14 @@ static DEVICE_ATTR(watermark, S_IRUGO | S_IWUSR,
 		   iio_buffer_show_watermark, iio_buffer_store_watermark);
 static struct device_attribute dev_attr_watermark_ro = __ATTR(watermark,
 	S_IRUGO, iio_buffer_show_watermark, NULL);
+static DEVICE_ATTR(data_available, S_IRUGO,
+		iio_dma_show_data_available, NULL);
 
 static struct attribute *iio_buffer_attrs[] = {
 	&dev_attr_length.attr,
 	&dev_attr_enable.attr,
 	&dev_attr_watermark.attr,
+	&dev_attr_data_available.attr,
 };
 
 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev *indio_dev)
@@ -1381,7 +1396,7 @@ static int iio_push_to_buffer(struct iio_buffer *buffer, const void *data)
 	 * We can't just test for watermark to decide if we wake the poll queue
 	 * because read may request less samples than the watermark.
 	 */
-	wake_up_interruptible_poll(&buffer->pollq, POLLIN | POLLRDNORM);
+	wake_up_interruptible_poll(&buffer->pollq, EPOLLIN | EPOLLRDNORM);
 	return 0;
 }
 
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index 2f0998ebeed2..19bdf3d2962a 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -588,6 +588,7 @@ static ssize_t __iio_format_value(char *buf, size_t len, unsigned int type,
 		return snprintf(buf, len, "%d", vals[0]);
 	case IIO_VAL_INT_PLUS_MICRO_DB:
 		scale_db = true;
+		/* fall through */
 	case IIO_VAL_INT_PLUS_MICRO:
 		if (vals[1] < 0)
 			return snprintf(buf, len, "-%d.%06u%s", abs(vals[0]),
diff --git a/drivers/iio/industrialio-event.c b/drivers/iio/industrialio-event.c
index 90fac8ec63c9..c6dfdf0aaac5 100644
--- a/drivers/iio/industrialio-event.c
+++ b/drivers/iio/industrialio-event.c
@@ -80,7 +80,7 @@ int iio_push_event(struct iio_dev *indio_dev, u64 ev_code, s64 timestamp)
 
 		copied = kfifo_put(&ev_int->det_events, ev);
 		if (copied != 0)
-			wake_up_poll(&ev_int->wait, POLLIN);
+			wake_up_poll(&ev_int->wait, EPOLLIN);
 	}
 
 	return 0;
@@ -92,15 +92,15 @@ EXPORT_SYMBOL(iio_push_event);
  * @filep:	File structure pointer to identify the device
  * @wait:	Poll table pointer to add the wait queue on
  *
- * Return: (POLLIN | POLLRDNORM) if data is available for reading
+ * Return: (EPOLLIN | EPOLLRDNORM) if data is available for reading
  *	   or a negative error code on failure
  */
-static unsigned int iio_event_poll(struct file *filep,
+static __poll_t iio_event_poll(struct file *filep,
 			     struct poll_table_struct *wait)
 {
 	struct iio_dev *indio_dev = filep->private_data;
 	struct iio_event_interface *ev_int = indio_dev->event_interface;
-	unsigned int events = 0;
+	__poll_t events = 0;
 
 	if (!indio_dev->info)
 		return events;
@@ -108,7 +108,7 @@ static unsigned int iio_event_poll(struct file *filep,
 	poll_wait(filep, &ev_int->wait, wait);
 
 	if (!kfifo_is_empty(&ev_int->det_events))
-		events = POLLIN | POLLRDNORM;
+		events = EPOLLIN | EPOLLRDNORM;
 
 	return events;
 }
diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
index 069defcc6d9b..ec98790e2a28 100644
--- a/drivers/iio/inkern.c
+++ b/drivers/iio/inkern.c
@@ -664,9 +664,8 @@ int iio_convert_raw_to_processed(struct iio_channel *chan, int raw,
 }
 EXPORT_SYMBOL_GPL(iio_convert_raw_to_processed);
 
-static int iio_read_channel_attribute(struct iio_channel *chan,
-				      int *val, int *val2,
-				      enum iio_chan_info_enum attribute)
+int iio_read_channel_attribute(struct iio_channel *chan, int *val, int *val2,
+			       enum iio_chan_info_enum attribute)
 {
 	int ret;
 
@@ -682,6 +681,7 @@ static int iio_read_channel_attribute(struct iio_channel *chan,
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(iio_read_channel_attribute);
 
 int iio_read_channel_offset(struct iio_channel *chan, int *val, int *val2)
 {
@@ -850,7 +850,8 @@ static int iio_channel_write(struct iio_channel *chan, int val, int val2,
 						chan->channel, val, val2, info);
 }
 
-int iio_write_channel_raw(struct iio_channel *chan, int val)
+int iio_write_channel_attribute(struct iio_channel *chan, int val, int val2,
+				enum iio_chan_info_enum attribute)
 {
 	int ret;
 
@@ -860,12 +861,18 @@ int iio_write_channel_raw(struct iio_channel *chan, int val)
 		goto err_unlock;
 	}
 
-	ret = iio_channel_write(chan, val, 0, IIO_CHAN_INFO_RAW);
+	ret = iio_channel_write(chan, val, val2, attribute);
 err_unlock:
 	mutex_unlock(&chan->indio_dev->info_exist_lock);
 
 	return ret;
 }
+EXPORT_SYMBOL_GPL(iio_write_channel_attribute);
+
+int iio_write_channel_raw(struct iio_channel *chan, int val)
+{
+	return iio_write_channel_attribute(chan, val, 0, IIO_CHAN_INFO_RAW);
+}
 EXPORT_SYMBOL_GPL(iio_write_channel_raw);
 
 unsigned int iio_get_channel_ext_info_count(struct iio_channel *chan)
diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig
index 2356ed9285df..93fd421b10d7 100644
--- a/drivers/iio/light/Kconfig
+++ b/drivers/iio/light/Kconfig
@@ -334,6 +334,30 @@ config STK3310
 	 Choosing M will build the driver as a module. If so, the module
 	 will be called stk3310.
 
+config ST_UVIS25
+	tristate "STMicroelectronics UVIS25 sensor driver"
+	depends on (I2C || SPI)
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
+	select ST_UVIS25_I2C if (I2C)
+	select ST_UVIS25_SPI if (SPI_MASTER)
+	help
+	  Say yes here to build support for STMicroelectronics UVIS25
+	  uv sensor
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called st_uvis25.
+
+config ST_UVIS25_I2C
+	tristate
+	depends on ST_UVIS25
+	select REGMAP_I2C
+
+config ST_UVIS25_SPI
+	tristate
+	depends on ST_UVIS25
+	select REGMAP_SPI
+
 config TCS3414
 	tristate "TAOS TCS3414 digital color sensor"
 	depends on I2C
@@ -425,4 +449,14 @@ config VL6180
 	 To compile this driver as a module, choose M here: the
 	 module will be called vl6180.
 
+config ZOPT2201
+	tristate "ZOPT2201 ALS and UV B sensor"
+	depends on I2C
+	help
+	 Say Y here if you want to build a driver for the IDT
+	 ZOPT2201 ambient light and UV B sensor.
+
+	 To compile this driver as a module, choose M here: the
+	 module will be called zopt2201.
+
 endmenu
diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile
index c263469b7ce9..f714067a7816 100644
--- a/drivers/iio/light/Makefile
+++ b/drivers/iio/light/Makefile
@@ -33,6 +33,9 @@ obj-$(CONFIG_RPR0521)		+= rpr0521.o
 obj-$(CONFIG_SENSORS_TSL2563)	+= tsl2563.o
 obj-$(CONFIG_SI1145)		+= si1145.o
 obj-$(CONFIG_STK3310)          += stk3310.o
+obj-$(CONFIG_ST_UVIS25)		+= st_uvis25_core.o
+obj-$(CONFIG_ST_UVIS25_I2C)	+= st_uvis25_i2c.o
+obj-$(CONFIG_ST_UVIS25_SPI)	+= st_uvis25_spi.o
 obj-$(CONFIG_TCS3414)		+= tcs3414.o
 obj-$(CONFIG_TCS3472)		+= tcs3472.o
 obj-$(CONFIG_TSL2583)		+= tsl2583.o
@@ -41,3 +44,4 @@ obj-$(CONFIG_US5182D)		+= us5182d.o
 obj-$(CONFIG_VCNL4000)		+= vcnl4000.o
 obj-$(CONFIG_VEML6070)		+= veml6070.o
 obj-$(CONFIG_VL6180)		+= vl6180.o
+obj-$(CONFIG_ZOPT2201)		+= zopt2201.o
diff --git a/drivers/iio/light/cros_ec_light_prox.c b/drivers/iio/light/cros_ec_light_prox.c
index b2a46b390d5c..acfad4aeb27a 100644
--- a/drivers/iio/light/cros_ec_light_prox.c
+++ b/drivers/iio/light/cros_ec_light_prox.c
@@ -181,7 +181,6 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct cros_ec_dev *ec_dev = dev_get_drvdata(dev->parent);
-	struct cros_ec_device *ec_device;
 	struct iio_dev *indio_dev;
 	struct cros_ec_light_prox_state *state;
 	struct iio_chan_spec *channel;
@@ -191,7 +190,6 @@ static int cros_ec_light_prox_probe(struct platform_device *pdev)
 		dev_warn(dev, "No CROS EC device found.\n");
 		return -EINVAL;
 	}
-	ec_device = ec_dev->ec_dev;
 
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*state));
 	if (!indio_dev)
diff --git a/drivers/iio/light/st_uvis25.h b/drivers/iio/light/st_uvis25.h
new file mode 100644
index 000000000000..5e970ab480cd
--- /dev/null
+++ b/drivers/iio/light/st_uvis25.h
@@ -0,0 +1,37 @@
+/*
+ * STMicroelectronics uvis25 sensor driver
+ *
+ * Copyright 2017 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#ifndef ST_UVIS25_H
+#define ST_UVIS25_H
+
+#define ST_UVIS25_DEV_NAME		"uvis25"
+
+#include <linux/iio/iio.h>
+
+/**
+ * struct st_uvis25_hw - ST UVIS25 sensor instance
+ * @regmap: Register map of the device.
+ * @trig: The trigger in use by the driver.
+ * @enabled: Status of the sensor (false->off, true->on).
+ * @irq: Device interrupt line (I2C or SPI).
+ */
+struct st_uvis25_hw {
+	struct regmap *regmap;
+
+	struct iio_trigger *trig;
+	bool enabled;
+	int irq;
+};
+
+extern const struct dev_pm_ops st_uvis25_pm_ops;
+
+int st_uvis25_probe(struct device *dev, int irq, struct regmap *regmap);
+
+#endif /* ST_UVIS25_H */
diff --git a/drivers/iio/light/st_uvis25_core.c b/drivers/iio/light/st_uvis25_core.c
new file mode 100644
index 000000000000..302635836e6b
--- /dev/null
+++ b/drivers/iio/light/st_uvis25_core.c
@@ -0,0 +1,359 @@
+/*
+ * STMicroelectronics uvis25 sensor driver
+ *
+ * Copyright 2017 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/iio/sysfs.h>
+#include <linux/delay.h>
+#include <linux/pm.h>
+#include <linux/interrupt.h>
+#include <linux/irqreturn.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/buffer.h>
+#include <linux/regmap.h>
+
+#include "st_uvis25.h"
+
+#define ST_UVIS25_REG_WHOAMI_ADDR	0x0f
+#define ST_UVIS25_REG_WHOAMI_VAL	0xca
+#define ST_UVIS25_REG_CTRL1_ADDR	0x20
+#define ST_UVIS25_REG_ODR_MASK		BIT(0)
+#define ST_UVIS25_REG_BDU_MASK		BIT(1)
+#define ST_UVIS25_REG_CTRL2_ADDR	0x21
+#define ST_UVIS25_REG_BOOT_MASK		BIT(7)
+#define ST_UVIS25_REG_CTRL3_ADDR	0x22
+#define ST_UVIS25_REG_HL_MASK		BIT(7)
+#define ST_UVIS25_REG_STATUS_ADDR	0x27
+#define ST_UVIS25_REG_UV_DA_MASK	BIT(0)
+#define ST_UVIS25_REG_OUT_ADDR		0x28
+
+static const struct iio_chan_spec st_uvis25_channels[] = {
+	{
+		.type = IIO_UVINDEX,
+		.address = ST_UVIS25_REG_OUT_ADDR,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+		.scan_index = 0,
+		.scan_type = {
+			.sign = 'u',
+			.realbits = 8,
+			.storagebits = 8,
+		},
+	},
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
+static int st_uvis25_check_whoami(struct st_uvis25_hw *hw)
+{
+	int err, data;
+
+	err = regmap_read(hw->regmap, ST_UVIS25_REG_WHOAMI_ADDR, &data);
+	if (err < 0) {
+		dev_err(regmap_get_device(hw->regmap),
+			"failed to read whoami register\n");
+		return err;
+	}
+
+	if (data != ST_UVIS25_REG_WHOAMI_VAL) {
+		dev_err(regmap_get_device(hw->regmap),
+			"wrong whoami {%02x vs %02x}\n",
+			data, ST_UVIS25_REG_WHOAMI_VAL);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int st_uvis25_set_enable(struct st_uvis25_hw *hw, bool enable)
+{
+	int err;
+
+	err = regmap_update_bits(hw->regmap, ST_UVIS25_REG_CTRL1_ADDR,
+				 ST_UVIS25_REG_ODR_MASK, enable);
+	if (err < 0)
+		return err;
+
+	hw->enabled = enable;
+
+	return 0;
+}
+
+static int st_uvis25_read_oneshot(struct st_uvis25_hw *hw, u8 addr, int *val)
+{
+	int err;
+
+	err = st_uvis25_set_enable(hw, true);
+	if (err < 0)
+		return err;
+
+	msleep(1500);
+
+	/*
+	 * in order to avoid possible race conditions with interrupt
+	 * generation, disable the sensor first and then poll output
+	 * register. That sequence guarantees the interrupt will be reset
+	 * when irq line is unmasked
+	 */
+	err = st_uvis25_set_enable(hw, false);
+	if (err < 0)
+		return err;
+
+	err = regmap_read(hw->regmap, addr, val);
+
+	return err < 0 ? err : IIO_VAL_INT;
+}
+
+static int st_uvis25_read_raw(struct iio_dev *iio_dev,
+			      struct iio_chan_spec const *ch,
+			      int *val, int *val2, long mask)
+{
+	int ret;
+
+	ret = iio_device_claim_direct_mode(iio_dev);
+	if (ret)
+		return ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_PROCESSED: {
+		struct st_uvis25_hw *hw = iio_priv(iio_dev);
+
+		/*
+		 * mask irq line during oneshot read since the sensor
+		 * does not export the capability to disable data-ready line
+		 * in the register map and it is enabled by default.
+		 * If the line is unmasked during read_raw() it will be set
+		 * active and never reset since the trigger is disabled
+		 */
+		if (hw->irq > 0)
+			disable_irq(hw->irq);
+		ret = st_uvis25_read_oneshot(hw, ch->address, val);
+		if (hw->irq > 0)
+			enable_irq(hw->irq);
+		break;
+	}
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	iio_device_release_direct_mode(iio_dev);
+
+	return ret;
+}
+
+static irqreturn_t st_uvis25_trigger_handler_thread(int irq, void *private)
+{
+	struct st_uvis25_hw *hw = private;
+	int err, status;
+
+	err = regmap_read(hw->regmap, ST_UVIS25_REG_STATUS_ADDR, &status);
+	if (err < 0)
+		return IRQ_HANDLED;
+
+	if (!(status & ST_UVIS25_REG_UV_DA_MASK))
+		return IRQ_NONE;
+
+	iio_trigger_poll_chained(hw->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int st_uvis25_allocate_trigger(struct iio_dev *iio_dev)
+{
+	struct st_uvis25_hw *hw = iio_priv(iio_dev);
+	struct device *dev = regmap_get_device(hw->regmap);
+	bool irq_active_low = false;
+	unsigned long irq_type;
+	int err;
+
+	irq_type = irqd_get_trigger_type(irq_get_irq_data(hw->irq));
+
+	switch (irq_type) {
+	case IRQF_TRIGGER_HIGH:
+	case IRQF_TRIGGER_RISING:
+		break;
+	case IRQF_TRIGGER_LOW:
+	case IRQF_TRIGGER_FALLING:
+		irq_active_low = true;
+		break;
+	default:
+		dev_info(dev, "mode %lx unsupported\n", irq_type);
+		return -EINVAL;
+	}
+
+	err = regmap_update_bits(hw->regmap, ST_UVIS25_REG_CTRL3_ADDR,
+				 ST_UVIS25_REG_HL_MASK, irq_active_low);
+	if (err < 0)
+		return err;
+
+	err = devm_request_threaded_irq(dev, hw->irq, NULL,
+					st_uvis25_trigger_handler_thread,
+					irq_type | IRQF_ONESHOT,
+					iio_dev->name, hw);
+	if (err) {
+		dev_err(dev, "failed to request trigger irq %d\n",
+			hw->irq);
+		return err;
+	}
+
+	hw->trig = devm_iio_trigger_alloc(dev, "%s-trigger",
+					  iio_dev->name);
+	if (!hw->trig)
+		return -ENOMEM;
+
+	iio_trigger_set_drvdata(hw->trig, iio_dev);
+	hw->trig->dev.parent = dev;
+
+	return devm_iio_trigger_register(dev, hw->trig);
+}
+
+static int st_uvis25_buffer_preenable(struct iio_dev *iio_dev)
+{
+	return st_uvis25_set_enable(iio_priv(iio_dev), true);
+}
+
+static int st_uvis25_buffer_postdisable(struct iio_dev *iio_dev)
+{
+	return st_uvis25_set_enable(iio_priv(iio_dev), false);
+}
+
+static const struct iio_buffer_setup_ops st_uvis25_buffer_ops = {
+	.preenable = st_uvis25_buffer_preenable,
+	.postenable = iio_triggered_buffer_postenable,
+	.predisable = iio_triggered_buffer_predisable,
+	.postdisable = st_uvis25_buffer_postdisable,
+};
+
+static irqreturn_t st_uvis25_buffer_handler_thread(int irq, void *p)
+{
+	u8 buffer[ALIGN(sizeof(u8), sizeof(s64)) + sizeof(s64)];
+	struct iio_poll_func *pf = p;
+	struct iio_dev *iio_dev = pf->indio_dev;
+	struct st_uvis25_hw *hw = iio_priv(iio_dev);
+	int err;
+
+	err = regmap_read(hw->regmap, ST_UVIS25_REG_OUT_ADDR, (int *)buffer);
+	if (err < 0)
+		goto out;
+
+	iio_push_to_buffers_with_timestamp(iio_dev, buffer,
+					   iio_get_time_ns(iio_dev));
+
+out:
+	iio_trigger_notify_done(hw->trig);
+
+	return IRQ_HANDLED;
+}
+
+static int st_uvis25_allocate_buffer(struct iio_dev *iio_dev)
+{
+	struct st_uvis25_hw *hw = iio_priv(iio_dev);
+
+	return devm_iio_triggered_buffer_setup(regmap_get_device(hw->regmap),
+					       iio_dev, NULL,
+					       st_uvis25_buffer_handler_thread,
+					       &st_uvis25_buffer_ops);
+}
+
+static const struct iio_info st_uvis25_info = {
+	.read_raw = st_uvis25_read_raw,
+};
+
+static int st_uvis25_init_sensor(struct st_uvis25_hw *hw)
+{
+	int err;
+
+	err = regmap_update_bits(hw->regmap, ST_UVIS25_REG_CTRL2_ADDR,
+				 ST_UVIS25_REG_BOOT_MASK, 1);
+	if (err < 0)
+		return err;
+
+	msleep(2000);
+
+	return regmap_update_bits(hw->regmap, ST_UVIS25_REG_CTRL1_ADDR,
+				  ST_UVIS25_REG_BDU_MASK, 1);
+}
+
+int st_uvis25_probe(struct device *dev, int irq, struct regmap *regmap)
+{
+	struct st_uvis25_hw *hw;
+	struct iio_dev *iio_dev;
+	int err;
+
+	iio_dev = devm_iio_device_alloc(dev, sizeof(*hw));
+	if (!iio_dev)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, (void *)iio_dev);
+
+	hw = iio_priv(iio_dev);
+	hw->irq = irq;
+	hw->regmap = regmap;
+
+	err = st_uvis25_check_whoami(hw);
+	if (err < 0)
+		return err;
+
+	iio_dev->modes = INDIO_DIRECT_MODE;
+	iio_dev->dev.parent = dev;
+	iio_dev->channels = st_uvis25_channels;
+	iio_dev->num_channels = ARRAY_SIZE(st_uvis25_channels);
+	iio_dev->name = ST_UVIS25_DEV_NAME;
+	iio_dev->info = &st_uvis25_info;
+
+	err = st_uvis25_init_sensor(hw);
+	if (err < 0)
+		return err;
+
+	if (hw->irq > 0) {
+		err = st_uvis25_allocate_buffer(iio_dev);
+		if (err < 0)
+			return err;
+
+		err = st_uvis25_allocate_trigger(iio_dev);
+		if (err)
+			return err;
+	}
+
+	return devm_iio_device_register(dev, iio_dev);
+}
+EXPORT_SYMBOL(st_uvis25_probe);
+
+static int __maybe_unused st_uvis25_suspend(struct device *dev)
+{
+	struct iio_dev *iio_dev = dev_get_drvdata(dev);
+	struct st_uvis25_hw *hw = iio_priv(iio_dev);
+
+	return regmap_update_bits(hw->regmap, ST_UVIS25_REG_CTRL1_ADDR,
+				  ST_UVIS25_REG_ODR_MASK, 0);
+}
+
+static int __maybe_unused st_uvis25_resume(struct device *dev)
+{
+	struct iio_dev *iio_dev = dev_get_drvdata(dev);
+	struct st_uvis25_hw *hw = iio_priv(iio_dev);
+
+	if (hw->enabled)
+		return regmap_update_bits(hw->regmap, ST_UVIS25_REG_CTRL1_ADDR,
+					  ST_UVIS25_REG_ODR_MASK, 1);
+
+	return 0;
+}
+
+const struct dev_pm_ops st_uvis25_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(st_uvis25_suspend, st_uvis25_resume)
+};
+EXPORT_SYMBOL(st_uvis25_pm_ops);
+
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>");
+MODULE_DESCRIPTION("STMicroelectronics uvis25 sensor driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/light/st_uvis25_i2c.c b/drivers/iio/light/st_uvis25_i2c.c
new file mode 100644
index 000000000000..afd6eb01a202
--- /dev/null
+++ b/drivers/iio/light/st_uvis25_i2c.c
@@ -0,0 +1,69 @@
+/*
+ * STMicroelectronics uvis25 i2c driver
+ *
+ * Copyright 2017 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/acpi.h>
+#include <linux/i2c.h>
+#include <linux/slab.h>
+#include <linux/regmap.h>
+
+#include "st_uvis25.h"
+
+#define UVIS25_I2C_AUTO_INCREMENT	BIT(7)
+
+static const struct regmap_config st_uvis25_i2c_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.write_flag_mask = UVIS25_I2C_AUTO_INCREMENT,
+	.read_flag_mask = UVIS25_I2C_AUTO_INCREMENT,
+};
+
+static int st_uvis25_i2c_probe(struct i2c_client *client,
+			       const struct i2c_device_id *id)
+{
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_i2c(client, &st_uvis25_i2c_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&client->dev, "Failed to register i2c regmap %d\n",
+			(int)PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	return st_uvis25_probe(&client->dev, client->irq, regmap);
+}
+
+static const struct of_device_id st_uvis25_i2c_of_match[] = {
+	{ .compatible = "st,uvis25", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, st_uvis25_i2c_of_match);
+
+static const struct i2c_device_id st_uvis25_i2c_id_table[] = {
+	{ ST_UVIS25_DEV_NAME },
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, st_uvis25_i2c_id_table);
+
+static struct i2c_driver st_uvis25_driver = {
+	.driver = {
+		.name = "st_uvis25_i2c",
+		.pm = &st_uvis25_pm_ops,
+		.of_match_table = of_match_ptr(st_uvis25_i2c_of_match),
+	},
+	.probe = st_uvis25_i2c_probe,
+	.id_table = st_uvis25_i2c_id_table,
+};
+module_i2c_driver(st_uvis25_driver);
+
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>");
+MODULE_DESCRIPTION("STMicroelectronics uvis25 i2c driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/light/st_uvis25_spi.c b/drivers/iio/light/st_uvis25_spi.c
new file mode 100644
index 000000000000..cdfee5e84d5e
--- /dev/null
+++ b/drivers/iio/light/st_uvis25_spi.c
@@ -0,0 +1,68 @@
+/*
+ * STMicroelectronics uvis25 spi driver
+ *
+ * Copyright 2017 STMicroelectronics Inc.
+ *
+ * Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
+ *
+ * Licensed under the GPL-2.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/spi/spi.h>
+#include <linux/slab.h>
+#include <linux/regmap.h>
+
+#include "st_uvis25.h"
+
+#define UVIS25_SENSORS_SPI_READ		BIT(7)
+#define UVIS25_SPI_AUTO_INCREMENT	BIT(6)
+
+static const struct regmap_config st_uvis25_spi_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.read_flag_mask = UVIS25_SENSORS_SPI_READ | UVIS25_SPI_AUTO_INCREMENT,
+	.write_flag_mask = UVIS25_SPI_AUTO_INCREMENT,
+};
+
+static int st_uvis25_spi_probe(struct spi_device *spi)
+{
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_spi(spi, &st_uvis25_spi_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(&spi->dev, "Failed to register spi regmap %d\n",
+			(int)PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	return st_uvis25_probe(&spi->dev, spi->irq, regmap);
+}
+
+static const struct of_device_id st_uvis25_spi_of_match[] = {
+	{ .compatible = "st,uvis25", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, st_uvis25_spi_of_match);
+
+static const struct spi_device_id st_uvis25_spi_id_table[] = {
+	{ ST_UVIS25_DEV_NAME },
+	{},
+};
+MODULE_DEVICE_TABLE(spi, st_uvis25_spi_id_table);
+
+static struct spi_driver st_uvis25_driver = {
+	.driver = {
+		.name = "st_uvis25_spi",
+		.pm = &st_uvis25_pm_ops,
+		.of_match_table = of_match_ptr(st_uvis25_spi_of_match),
+	},
+	.probe = st_uvis25_spi_probe,
+	.id_table = st_uvis25_spi_id_table,
+};
+module_spi_driver(st_uvis25_driver);
+
+MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>");
+MODULE_DESCRIPTION("STMicroelectronics uvis25 spi driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/iio/light/zopt2201.c b/drivers/iio/light/zopt2201.c
new file mode 100644
index 000000000000..041ac9effdb0
--- /dev/null
+++ b/drivers/iio/light/zopt2201.c
@@ -0,0 +1,568 @@
+/*
+ * zopt2201.c - Support for IDT ZOPT2201 ambient light and UV B sensor
+ *
+ * Copyright 2017 Peter Meerwald-Stadler <pmeerw@pmeerw.net>
+ *
+ * This file is subject to the terms and conditions of version 2 of
+ * the GNU General Public License.  See the file COPYING in the main
+ * directory of this archive for more details.
+ *
+ * Datasheet: https://www.idt.com/document/dst/zopt2201-datasheet
+ * 7-bit I2C slave addresses 0x53 (default) or 0x52 (programmed)
+ *
+ * TODO: interrupt support, ALS/UVB raw mode
+ */
+
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/mutex.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+
+#define ZOPT2201_DRV_NAME "zopt2201"
+
+/* Registers */
+#define ZOPT2201_MAIN_CTRL		0x00
+#define ZOPT2201_LS_MEAS_RATE		0x04
+#define ZOPT2201_LS_GAIN		0x05
+#define ZOPT2201_PART_ID		0x06
+#define ZOPT2201_MAIN_STATUS		0x07
+#define ZOPT2201_ALS_DATA		0x0d /* LSB first, 13 to 20 bits */
+#define ZOPT2201_UVB_DATA		0x10 /* LSB first, 13 to 20 bits */
+#define ZOPT2201_UV_COMP_DATA		0x13 /* LSB first, 13 to 20 bits */
+#define ZOPT2201_COMP_DATA		0x16 /* LSB first, 13 to 20 bits */
+#define ZOPT2201_INT_CFG		0x19
+#define ZOPT2201_INT_PST		0x1a
+
+#define ZOPT2201_MAIN_CTRL_LS_MODE	BIT(3) /* 0 .. ALS, 1 .. UV B */
+#define ZOPT2201_MAIN_CTRL_LS_EN	BIT(1)
+
+/* Values for ZOPT2201_LS_MEAS_RATE resolution / bit width */
+#define ZOPT2201_MEAS_RES_20BIT		0 /* takes 400 ms */
+#define ZOPT2201_MEAS_RES_19BIT		1 /* takes 200 ms */
+#define ZOPT2201_MEAS_RES_18BIT		2 /* takes 100 ms, default */
+#define ZOPT2201_MEAS_RES_17BIT		3 /* takes 50 ms */
+#define ZOPT2201_MEAS_RES_16BIT		4 /* takes 25 ms */
+#define ZOPT2201_MEAS_RES_13BIT		5 /* takes 3.125 ms */
+#define ZOPT2201_MEAS_RES_SHIFT		4
+
+/* Values for ZOPT2201_LS_MEAS_RATE measurement rate */
+#define ZOPT2201_MEAS_FREQ_25MS		0
+#define ZOPT2201_MEAS_FREQ_50MS		1
+#define ZOPT2201_MEAS_FREQ_100MS	2 /* default */
+#define ZOPT2201_MEAS_FREQ_200MS	3
+#define ZOPT2201_MEAS_FREQ_500MS	4
+#define ZOPT2201_MEAS_FREQ_1000MS	5
+#define ZOPT2201_MEAS_FREQ_2000MS	6
+
+/* Values for ZOPT2201_LS_GAIN */
+#define ZOPT2201_LS_GAIN_1		0
+#define ZOPT2201_LS_GAIN_3		1
+#define ZOPT2201_LS_GAIN_6		2
+#define ZOPT2201_LS_GAIN_9		3
+#define ZOPT2201_LS_GAIN_18		4
+
+/* Values for ZOPT2201_MAIN_STATUS */
+#define ZOPT2201_MAIN_STATUS_POWERON	BIT(5)
+#define ZOPT2201_MAIN_STATUS_INT	BIT(4)
+#define ZOPT2201_MAIN_STATUS_DRDY	BIT(3)
+
+#define ZOPT2201_PART_NUMBER		0xb2
+
+struct zopt2201_data {
+	struct i2c_client *client;
+	struct mutex lock;
+	u8 gain;
+	u8 res;
+	u8 rate;
+};
+
+static const struct {
+	unsigned int gain; /* gain factor */
+	unsigned int scale; /* micro lux per count */
+} zopt2201_gain_als[] = {
+	{  1, 19200000 },
+	{  3,  6400000 },
+	{  6,  3200000 },
+	{  9,  2133333 },
+	{ 18,  1066666 },
+};
+
+static const struct {
+	unsigned int gain; /* gain factor */
+	unsigned int scale; /* micro W/m2 per count */
+} zopt2201_gain_uvb[] = {
+	{  1, 460800 },
+	{  3, 153600 },
+	{  6,  76800 },
+	{  9,  51200 },
+	{ 18,  25600 },
+};
+
+static const struct {
+	unsigned int bits; /* sensor resolution in bits */
+	unsigned long us; /* measurement time in micro seconds */
+} zopt2201_resolution[] = {
+	{ 20, 400000 },
+	{ 19, 200000 },
+	{ 18, 100000 },
+	{ 17,  50000 },
+	{ 16,  25000 },
+	{ 13,   3125 },
+};
+
+static const struct {
+	unsigned int scale, uscale; /* scale factor as integer + micro */
+	u8 gain; /* gain register value */
+	u8 res; /* resolution register value */
+} zopt2201_scale_als[] = {
+	{ 19, 200000, 0, 5 },
+	{  6, 400000, 1, 5 },
+	{  3, 200000, 2, 5 },
+	{  2, 400000, 0, 4 },
+	{  2, 133333, 3, 5 },
+	{  1, 200000, 0, 3 },
+	{  1,  66666, 4, 5 },
+	{  0, 800000, 1, 4 },
+	{  0, 600000, 0, 2 },
+	{  0, 400000, 2, 4 },
+	{  0, 300000, 0, 1 },
+	{  0, 266666, 3, 4 },
+	{  0, 200000, 2, 3 },
+	{  0, 150000, 0, 0 },
+	{  0, 133333, 4, 4 },
+	{  0, 100000, 2, 2 },
+	{  0,  66666, 4, 3 },
+	{  0,  50000, 2, 1 },
+	{  0,  33333, 4, 2 },
+	{  0,  25000, 2, 0 },
+	{  0,  16666, 4, 1 },
+	{  0,   8333, 4, 0 },
+};
+
+static const struct {
+	unsigned int scale, uscale; /* scale factor as integer + micro */
+	u8 gain; /* gain register value */
+	u8 res; /* resolution register value */
+} zopt2201_scale_uvb[] = {
+	{ 0, 460800, 0, 5 },
+	{ 0, 153600, 1, 5 },
+	{ 0,  76800, 2, 5 },
+	{ 0,  57600, 0, 4 },
+	{ 0,  51200, 3, 5 },
+	{ 0,  28800, 0, 3 },
+	{ 0,  25600, 4, 5 },
+	{ 0,  19200, 1, 4 },
+	{ 0,  14400, 0, 2 },
+	{ 0,   9600, 2, 4 },
+	{ 0,   7200, 0, 1 },
+	{ 0,   6400, 3, 4 },
+	{ 0,   4800, 2, 3 },
+	{ 0,   3600, 0, 0 },
+	{ 0,   3200, 4, 4 },
+	{ 0,   2400, 2, 2 },
+	{ 0,   1600, 4, 3 },
+	{ 0,   1200, 2, 1 },
+	{ 0,    800, 4, 2 },
+	{ 0,    600, 2, 0 },
+	{ 0,    400, 4, 1 },
+	{ 0,    200, 4, 0 },
+};
+
+static int zopt2201_enable_mode(struct zopt2201_data *data, bool uvb_mode)
+{
+	u8 out = ZOPT2201_MAIN_CTRL_LS_EN;
+
+	if (uvb_mode)
+		out |= ZOPT2201_MAIN_CTRL_LS_MODE;
+
+	return i2c_smbus_write_byte_data(data->client, ZOPT2201_MAIN_CTRL, out);
+}
+
+static int zopt2201_read(struct zopt2201_data *data, u8 reg)
+{
+	struct i2c_client *client = data->client;
+	int tries = 10;
+	u8 buf[3];
+	int ret;
+
+	mutex_lock(&data->lock);
+	ret = zopt2201_enable_mode(data, reg == ZOPT2201_UVB_DATA);
+	if (ret < 0)
+		goto fail;
+
+	while (tries--) {
+		unsigned long t = zopt2201_resolution[data->res].us;
+
+		if (t <= 20000)
+			usleep_range(t, t + 1000);
+		else
+			msleep(t / 1000);
+		ret = i2c_smbus_read_byte_data(client, ZOPT2201_MAIN_STATUS);
+		if (ret < 0)
+			goto fail;
+		if (ret & ZOPT2201_MAIN_STATUS_DRDY)
+			break;
+	}
+
+	if (tries < 0) {
+		ret = -ETIMEDOUT;
+		goto fail;
+	}
+
+	ret = i2c_smbus_read_i2c_block_data(client, reg, sizeof(buf), buf);
+	if (ret < 0)
+		goto fail;
+
+	ret = i2c_smbus_write_byte_data(client, ZOPT2201_MAIN_CTRL, 0x00);
+	if (ret < 0)
+		goto fail;
+	mutex_unlock(&data->lock);
+
+	return (buf[2] << 16) | (buf[1] << 8) | buf[0];
+
+fail:
+	mutex_unlock(&data->lock);
+	return ret;
+}
+
+static const struct iio_chan_spec zopt2201_channels[] = {
+	{
+		.type = IIO_LIGHT,
+		.address = ZOPT2201_ALS_DATA,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_INT_TIME),
+	},
+	{
+		.type = IIO_INTENSITY,
+		.modified = 1,
+		.channel2 = IIO_MOD_LIGHT_UV,
+		.address = ZOPT2201_UVB_DATA,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |
+				      BIT(IIO_CHAN_INFO_SCALE),
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_INT_TIME),
+	},
+	{
+		.type = IIO_UVINDEX,
+		.info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED),
+	},
+};
+
+static int zopt2201_read_raw(struct iio_dev *indio_dev,
+				struct iio_chan_spec const *chan,
+				int *val, int *val2, long mask)
+{
+	struct zopt2201_data *data = iio_priv(indio_dev);
+	u64 tmp;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		ret = zopt2201_read(data, chan->address);
+		if (ret < 0)
+			return ret;
+		*val = ret;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_PROCESSED:
+		ret = zopt2201_read(data, ZOPT2201_UVB_DATA);
+		if (ret < 0)
+			return ret;
+		*val = ret * 18 *
+			(1 << (20 - zopt2201_resolution[data->res].bits)) /
+			zopt2201_gain_uvb[data->gain].gain;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->address) {
+		case ZOPT2201_ALS_DATA:
+			*val = zopt2201_gain_als[data->gain].scale;
+			break;
+		case ZOPT2201_UVB_DATA:
+			*val = zopt2201_gain_uvb[data->gain].scale;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		*val2 = 1000000;
+		*val2 *= (1 << (zopt2201_resolution[data->res].bits - 13));
+		tmp = div_s64(*val * 1000000ULL, *val2);
+		*val = div_s64_rem(tmp, 1000000, val2);
+
+		return IIO_VAL_INT_PLUS_MICRO;
+	case IIO_CHAN_INFO_INT_TIME:
+		*val = 0;
+		*val2 = zopt2201_resolution[data->res].us;
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int zopt2201_set_resolution(struct zopt2201_data *data, u8 res)
+{
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(data->client, ZOPT2201_LS_MEAS_RATE,
+					(res << ZOPT2201_MEAS_RES_SHIFT) |
+					data->rate);
+	if (ret < 0)
+		return ret;
+
+	data->res = res;
+
+	return 0;
+}
+
+static int zopt2201_write_resolution(struct zopt2201_data *data,
+				     int val, int val2)
+{
+	int i, ret;
+
+	if (val != 0)
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(zopt2201_resolution); i++)
+		if (val2 == zopt2201_resolution[i].us) {
+			mutex_lock(&data->lock);
+			ret = zopt2201_set_resolution(data, i);
+			mutex_unlock(&data->lock);
+			return ret;
+		}
+
+	return -EINVAL;
+}
+
+static int zopt2201_set_gain(struct zopt2201_data *data, u8 gain)
+{
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(data->client, ZOPT2201_LS_GAIN, gain);
+	if (ret < 0)
+		return ret;
+
+	data->gain = gain;
+
+	return 0;
+}
+
+static int zopt2201_write_scale_als_by_idx(struct zopt2201_data *data, int idx)
+{
+	int ret;
+
+	mutex_lock(&data->lock);
+	ret = zopt2201_set_resolution(data, zopt2201_scale_als[idx].res);
+	if (ret < 0)
+		goto unlock;
+
+	ret = zopt2201_set_gain(data, zopt2201_scale_als[idx].gain);
+
+unlock:
+	mutex_unlock(&data->lock);
+	return ret;
+}
+
+static int zopt2201_write_scale_als(struct zopt2201_data *data,
+				     int val, int val2)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(zopt2201_scale_als); i++)
+		if (val == zopt2201_scale_als[i].scale &&
+		    val2 == zopt2201_scale_als[i].uscale) {
+			return zopt2201_write_scale_als_by_idx(data, i);
+		}
+
+	return -EINVAL;
+}
+
+static int zopt2201_write_scale_uvb_by_idx(struct zopt2201_data *data, int idx)
+{
+	int ret;
+
+	mutex_lock(&data->lock);
+	ret = zopt2201_set_resolution(data, zopt2201_scale_als[idx].res);
+	if (ret < 0)
+		goto unlock;
+
+	ret = zopt2201_set_gain(data, zopt2201_scale_als[idx].gain);
+
+unlock:
+	mutex_unlock(&data->lock);
+	return ret;
+}
+
+static int zopt2201_write_scale_uvb(struct zopt2201_data *data,
+				     int val, int val2)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(zopt2201_scale_uvb); i++)
+		if (val == zopt2201_scale_uvb[i].scale &&
+		    val2 == zopt2201_scale_uvb[i].uscale)
+			return zopt2201_write_scale_uvb_by_idx(data, i);
+
+	return -EINVAL;
+}
+
+static int zopt2201_write_raw(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      int val, int val2, long mask)
+{
+	struct zopt2201_data *data = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_INT_TIME:
+		return zopt2201_write_resolution(data, val, val2);
+	case IIO_CHAN_INFO_SCALE:
+		switch (chan->address) {
+		case ZOPT2201_ALS_DATA:
+			return zopt2201_write_scale_als(data, val, val2);
+		case ZOPT2201_UVB_DATA:
+			return zopt2201_write_scale_uvb(data, val, val2);
+		default:
+			return -EINVAL;
+		}
+	}
+
+	return -EINVAL;
+}
+
+static ssize_t zopt2201_show_int_time_available(struct device *dev,
+						struct device_attribute *attr,
+						char *buf)
+{
+	size_t len = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(zopt2201_resolution); i++)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "0.%06lu ",
+				 zopt2201_resolution[i].us);
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+static IIO_DEV_ATTR_INT_TIME_AVAIL(zopt2201_show_int_time_available);
+
+static ssize_t zopt2201_show_als_scale_avail(struct device *dev,
+					     struct device_attribute *attr,
+					     char *buf)
+{
+	ssize_t len = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(zopt2201_scale_als); i++)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "%d.%06u ",
+				 zopt2201_scale_als[i].scale,
+				 zopt2201_scale_als[i].uscale);
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+static ssize_t zopt2201_show_uvb_scale_avail(struct device *dev,
+					     struct device_attribute *attr,
+					     char *buf)
+{
+	ssize_t len = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(zopt2201_scale_uvb); i++)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "%d.%06u ",
+				 zopt2201_scale_uvb[i].scale,
+				 zopt2201_scale_uvb[i].uscale);
+	buf[len - 1] = '\n';
+
+	return len;
+}
+
+static IIO_DEVICE_ATTR(in_illuminance_scale_available, 0444,
+		       zopt2201_show_als_scale_avail, NULL, 0);
+static IIO_DEVICE_ATTR(in_intensity_uv_scale_available, 0444,
+		       zopt2201_show_uvb_scale_avail, NULL, 0);
+
+static struct attribute *zopt2201_attributes[] = {
+	&iio_dev_attr_integration_time_available.dev_attr.attr,
+	&iio_dev_attr_in_illuminance_scale_available.dev_attr.attr,
+	&iio_dev_attr_in_intensity_uv_scale_available.dev_attr.attr,
+	NULL
+};
+
+static const struct attribute_group zopt2201_attribute_group = {
+	.attrs = zopt2201_attributes,
+};
+
+static const struct iio_info zopt2201_info = {
+	.read_raw = zopt2201_read_raw,
+	.write_raw = zopt2201_write_raw,
+	.attrs = &zopt2201_attribute_group,
+};
+
+static int zopt2201_probe(struct i2c_client *client,
+			  const struct i2c_device_id *id)
+{
+	struct zopt2201_data *data;
+	struct iio_dev *indio_dev;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_READ_I2C_BLOCK))
+		return -EOPNOTSUPP;
+
+	ret = i2c_smbus_read_byte_data(client, ZOPT2201_PART_ID);
+	if (ret < 0)
+		return ret;
+	if (ret != ZOPT2201_PART_NUMBER)
+		return -ENODEV;
+
+	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+	i2c_set_clientdata(client, indio_dev);
+	data->client = client;
+	mutex_init(&data->lock);
+
+	indio_dev->dev.parent = &client->dev;
+	indio_dev->info = &zopt2201_info;
+	indio_dev->channels = zopt2201_channels;
+	indio_dev->num_channels = ARRAY_SIZE(zopt2201_channels);
+	indio_dev->name = ZOPT2201_DRV_NAME;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	data->rate = ZOPT2201_MEAS_FREQ_100MS;
+	ret = zopt2201_set_resolution(data, ZOPT2201_MEAS_RES_18BIT);
+	if (ret < 0)
+		return ret;
+
+	ret = zopt2201_set_gain(data, ZOPT2201_LS_GAIN_3);
+	if (ret < 0)
+		return ret;
+
+	return devm_iio_device_register(&client->dev, indio_dev);
+}
+
+static const struct i2c_device_id zopt2201_id[] = {
+	{ "zopt2201", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, zopt2201_id);
+
+static struct i2c_driver zopt2201_driver = {
+	.driver = {
+		.name   = ZOPT2201_DRV_NAME,
+	},
+	.probe  = zopt2201_probe,
+	.id_table = zopt2201_id,
+};
+
+module_i2c_driver(zopt2201_driver);
+
+MODULE_AUTHOR("Peter Meerwald-Stadler <pmeerw@pmeerw.net>");
+MODULE_DESCRIPTION("IDT ZOPT2201 ambient light and UV B sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c
index c09329069d0a..42a827a66512 100644
--- a/drivers/iio/magnetometer/ak8975.c
+++ b/drivers/iio/magnetometer/ak8975.c
@@ -788,6 +788,7 @@ static const struct acpi_device_id ak_acpi_match[] = {
 	{"AK8975", AK8975},
 	{"AK8963", AK8963},
 	{"INVN6500", AK8963},
+	{"AK009911", AK09911},
 	{"AK09911", AK09911},
 	{"AK09912", AK09912},
 	{ },
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index fd1da26a62e4..5ec3e41b65f2 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -55,6 +55,28 @@ struct bmp180_calib {
 	s16 MD;
 };
 
+/* See datasheet Section 4.2.2. */
+struct bmp280_calib {
+	u16 T1;
+	s16 T2;
+	s16 T3;
+	u16 P1;
+	s16 P2;
+	s16 P3;
+	s16 P4;
+	s16 P5;
+	s16 P6;
+	s16 P7;
+	s16 P8;
+	s16 P9;
+	u8  H1;
+	s16 H2;
+	u8  H3;
+	s16 H4;
+	s16 H5;
+	s8  H6;
+};
+
 struct bmp280_data {
 	struct device *dev;
 	struct mutex lock;
@@ -62,7 +84,10 @@ struct bmp280_data {
 	struct completion done;
 	bool use_eoc;
 	const struct bmp280_chip_info *chip_info;
-	struct bmp180_calib calib;
+	union {
+		struct bmp180_calib bmp180;
+		struct bmp280_calib bmp280;
+	} calib;
 	struct regulator *vddd;
 	struct regulator *vdda;
 	unsigned int start_up_time; /* in microseconds */
@@ -120,67 +145,121 @@ static const struct iio_chan_spec bmp280_channels[] = {
 	},
 };
 
-/*
- * Returns humidity in percent, resolution is 0.01 percent. Output value of
- * "47445" represents 47445/1024 = 46.333 %RH.
- *
- * Taken from BME280 datasheet, Section 4.2.3, "Compensation formula".
- */
-
-static u32 bmp280_compensate_humidity(struct bmp280_data *data,
-				      s32 adc_humidity)
+static int bmp280_read_calib(struct bmp280_data *data,
+			     struct bmp280_calib *calib,
+			     unsigned int chip)
 {
+	int ret;
+	unsigned int tmp;
 	struct device *dev = data->dev;
-	unsigned int H1, H3, tmp;
-	int H2, H4, H5, H6, ret, var;
+	__le16 t_buf[BMP280_COMP_TEMP_REG_COUNT / 2];
+	__le16 p_buf[BMP280_COMP_PRESS_REG_COUNT / 2];
+
+	/* Read temperature calibration values. */
+	ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_TEMP_START,
+			       t_buf, BMP280_COMP_TEMP_REG_COUNT);
+	if (ret < 0) {
+		dev_err(data->dev,
+			"failed to read temperature calibration parameters\n");
+		return ret;
+	}
+
+	calib->T1 = le16_to_cpu(t_buf[T1]);
+	calib->T2 = le16_to_cpu(t_buf[T2]);
+	calib->T3 = le16_to_cpu(t_buf[T3]);
 
-	ret = regmap_read(data->regmap, BMP280_REG_COMP_H1, &H1);
+	/* Read pressure calibration values. */
+	ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_PRESS_START,
+			       p_buf, BMP280_COMP_PRESS_REG_COUNT);
+	if (ret < 0) {
+		dev_err(data->dev,
+			"failed to read pressure calibration parameters\n");
+		return ret;
+	}
+
+	calib->P1 = le16_to_cpu(p_buf[P1]);
+	calib->P2 = le16_to_cpu(p_buf[P2]);
+	calib->P3 = le16_to_cpu(p_buf[P3]);
+	calib->P4 = le16_to_cpu(p_buf[P4]);
+	calib->P5 = le16_to_cpu(p_buf[P5]);
+	calib->P6 = le16_to_cpu(p_buf[P6]);
+	calib->P7 = le16_to_cpu(p_buf[P7]);
+	calib->P8 = le16_to_cpu(p_buf[P8]);
+	calib->P9 = le16_to_cpu(p_buf[P9]);
+
+	/*
+	 * Read humidity calibration values.
+	 * Due to some odd register addressing we cannot just
+	 * do a big bulk read. Instead, we have to read each Hx
+	 * value separately and sometimes do some bit shifting...
+	 * Humidity data is only available on BME280.
+	 */
+	if (chip != BME280_CHIP_ID)
+		return 0;
+
+	ret = regmap_read(data->regmap, BMP280_REG_COMP_H1, &tmp);
 	if (ret < 0) {
 		dev_err(dev, "failed to read H1 comp value\n");
 		return ret;
 	}
+	calib->H1 = tmp;
 
 	ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H2, &tmp, 2);
 	if (ret < 0) {
 		dev_err(dev, "failed to read H2 comp value\n");
 		return ret;
 	}
-	H2 = sign_extend32(le16_to_cpu(tmp), 15);
+	calib->H2 = sign_extend32(le16_to_cpu(tmp), 15);
 
-	ret = regmap_read(data->regmap, BMP280_REG_COMP_H3, &H3);
+	ret = regmap_read(data->regmap, BMP280_REG_COMP_H3, &tmp);
 	if (ret < 0) {
 		dev_err(dev, "failed to read H3 comp value\n");
 		return ret;
 	}
+	calib->H3 = tmp;
 
 	ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H4, &tmp, 2);
 	if (ret < 0) {
 		dev_err(dev, "failed to read H4 comp value\n");
 		return ret;
 	}
-	H4 = sign_extend32(((be16_to_cpu(tmp) >> 4) & 0xff0) |
-			  (be16_to_cpu(tmp) & 0xf), 11);
+	calib->H4 = sign_extend32(((be16_to_cpu(tmp) >> 4) & 0xff0) |
+				  (be16_to_cpu(tmp) & 0xf), 11);
 
 	ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H5, &tmp, 2);
 	if (ret < 0) {
 		dev_err(dev, "failed to read H5 comp value\n");
 		return ret;
 	}
-	H5 = sign_extend32(((le16_to_cpu(tmp) >> 4) & 0xfff), 11);
+	calib->H5 = sign_extend32(((le16_to_cpu(tmp) >> 4) & 0xfff), 11);
 
 	ret = regmap_read(data->regmap, BMP280_REG_COMP_H6, &tmp);
 	if (ret < 0) {
 		dev_err(dev, "failed to read H6 comp value\n");
 		return ret;
 	}
-	H6 = sign_extend32(tmp, 7);
+	calib->H6 = sign_extend32(tmp, 7);
+
+	return 0;
+}
+/*
+ * Returns humidity in percent, resolution is 0.01 percent. Output value of
+ * "47445" represents 47445/1024 = 46.333 %RH.
+ *
+ * Taken from BME280 datasheet, Section 4.2.3, "Compensation formula".
+ */
+static u32 bmp280_compensate_humidity(struct bmp280_data *data,
+				      s32 adc_humidity)
+{
+	s32 var;
+	struct bmp280_calib *calib = &data->calib.bmp280;
 
 	var = ((s32)data->t_fine) - (s32)76800;
-	var = ((((adc_humidity << 14) - (H4 << 20) - (H5 * var))
-		+ (s32)16384) >> 15) * (((((((var * H6) >> 10)
-		* (((var * (s32)H3) >> 11) + (s32)32768)) >> 10)
-		+ (s32)2097152) * H2 + 8192) >> 14);
-	var -= ((((var >> 15) * (var >> 15)) >> 7) * (s32)H1) >> 4;
+	var = ((((adc_humidity << 14) - (calib->H4 << 20) - (calib->H5 * var))
+		+ (s32)16384) >> 15) * (((((((var * calib->H6) >> 10)
+		* (((var * (s32)calib->H3) >> 11) + (s32)32768)) >> 10)
+		+ (s32)2097152) * calib->H2 + 8192) >> 14);
+	var -= ((((var >> 15) * (var >> 15)) >> 7) * (s32)calib->H1) >> 4;
 
 	return var >> 12;
 };
@@ -195,31 +274,14 @@ static u32 bmp280_compensate_humidity(struct bmp280_data *data,
 static s32 bmp280_compensate_temp(struct bmp280_data *data,
 				  s32 adc_temp)
 {
-	int ret;
 	s32 var1, var2;
-	__le16 buf[BMP280_COMP_TEMP_REG_COUNT / 2];
+	struct bmp280_calib *calib = &data->calib.bmp280;
 
-	ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_TEMP_START,
-			       buf, BMP280_COMP_TEMP_REG_COUNT);
-	if (ret < 0) {
-		dev_err(data->dev,
-			"failed to read temperature calibration parameters\n");
-		return ret;
-	}
-
-	/*
-	 * The double casts are necessary because le16_to_cpu returns an
-	 * unsigned 16-bit value.  Casting that value directly to a
-	 * signed 32-bit will not do proper sign extension.
-	 *
-	 * Conversely, T1 and P1 are unsigned values, so they can be
-	 * cast straight to the larger type.
-	 */
-	var1 = (((adc_temp >> 3) - ((s32)le16_to_cpu(buf[T1]) << 1)) *
-		((s32)(s16)le16_to_cpu(buf[T2]))) >> 11;
-	var2 = (((((adc_temp >> 4) - ((s32)le16_to_cpu(buf[T1]))) *
-		  ((adc_temp >> 4) - ((s32)le16_to_cpu(buf[T1])))) >> 12) *
-		((s32)(s16)le16_to_cpu(buf[T3]))) >> 14;
+	var1 = (((adc_temp >> 3) - ((s32)calib->T1 << 1)) *
+		((s32)calib->T2)) >> 11;
+	var2 = (((((adc_temp >> 4) - ((s32)calib->T1)) *
+		  ((adc_temp >> 4) - ((s32)calib->T1))) >> 12) *
+		((s32)calib->T3)) >> 14;
 	data->t_fine = var1 + var2;
 
 	return (data->t_fine * 5 + 128) >> 8;
@@ -235,34 +297,25 @@ static s32 bmp280_compensate_temp(struct bmp280_data *data,
 static u32 bmp280_compensate_press(struct bmp280_data *data,
 				   s32 adc_press)
 {
-	int ret;
 	s64 var1, var2, p;
-	__le16 buf[BMP280_COMP_PRESS_REG_COUNT / 2];
-
-	ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_PRESS_START,
-			       buf, BMP280_COMP_PRESS_REG_COUNT);
-	if (ret < 0) {
-		dev_err(data->dev,
-			"failed to read pressure calibration parameters\n");
-		return ret;
-	}
+	struct bmp280_calib *calib = &data->calib.bmp280;
 
 	var1 = ((s64)data->t_fine) - 128000;
-	var2 = var1 * var1 * (s64)(s16)le16_to_cpu(buf[P6]);
-	var2 += (var1 * (s64)(s16)le16_to_cpu(buf[P5])) << 17;
-	var2 += ((s64)(s16)le16_to_cpu(buf[P4])) << 35;
-	var1 = ((var1 * var1 * (s64)(s16)le16_to_cpu(buf[P3])) >> 8) +
-		((var1 * (s64)(s16)le16_to_cpu(buf[P2])) << 12);
-	var1 = ((((s64)1) << 47) + var1) * ((s64)le16_to_cpu(buf[P1])) >> 33;
+	var2 = var1 * var1 * (s64)calib->P6;
+	var2 += (var1 * (s64)calib->P5) << 17;
+	var2 += ((s64)calib->P4) << 35;
+	var1 = ((var1 * var1 * (s64)calib->P3) >> 8) +
+		((var1 * (s64)calib->P2) << 12);
+	var1 = ((((s64)1) << 47) + var1) * ((s64)calib->P1) >> 33;
 
 	if (var1 == 0)
 		return 0;
 
 	p = ((((s64)1048576 - adc_press) << 31) - var2) * 3125;
 	p = div64_s64(p, var1);
-	var1 = (((s64)(s16)le16_to_cpu(buf[P9])) * (p >> 13) * (p >> 13)) >> 25;
-	var2 = (((s64)(s16)le16_to_cpu(buf[P8])) * p) >> 19;
-	p = ((p + var1 + var2) >> 8) + (((s64)(s16)le16_to_cpu(buf[P7])) << 4);
+	var1 = (((s64)calib->P9) * (p >> 13) * (p >> 13)) >> 25;
+	var2 = ((s64)(calib->P8) * p) >> 19;
+	p = ((p + var1 + var2) >> 8) + (((s64)calib->P7) << 4);
 
 	return (u32)p;
 }
@@ -752,7 +805,7 @@ static int bmp180_read_calib(struct bmp280_data *data,
 static s32 bmp180_compensate_temp(struct bmp280_data *data, s32 adc_temp)
 {
 	s32 x1, x2;
-	struct bmp180_calib *calib = &data->calib;
+	struct bmp180_calib *calib = &data->calib.bmp180;
 
 	x1 = ((adc_temp - calib->AC6) * calib->AC5) >> 15;
 	x2 = (calib->MC << 11) / (x1 + calib->MD);
@@ -814,7 +867,7 @@ static u32 bmp180_compensate_press(struct bmp280_data *data, s32 adc_press)
 	s32 b3, b6;
 	u32 b4, b7;
 	s32 oss = data->oversampling_press;
-	struct bmp180_calib *calib = &data->calib;
+	struct bmp180_calib *calib = &data->calib.bmp180;
 
 	b6 = data->t_fine - 4000;
 	x1 = (calib->B2 * (b6 * b6 >> 12)) >> 11;
@@ -1028,11 +1081,19 @@ int bmp280_common_probe(struct device *dev,
 	dev_set_drvdata(dev, indio_dev);
 
 	/*
-	 * The BMP085 and BMP180 has calibration in an E2PROM, read it out
-	 * at probe time. It will not change.
+	 * Some chips have calibration parameters "programmed into the devices'
+	 * non-volatile memory during production". Let's read them out at probe
+	 * time once. They will not change.
 	 */
 	if (chip_id  == BMP180_CHIP_ID) {
-		ret = bmp180_read_calib(data, &data->calib);
+		ret = bmp180_read_calib(data, &data->calib.bmp180);
+		if (ret < 0) {
+			dev_err(data->dev,
+				"failed to read calibration coefficients\n");
+			goto out_disable_vdda;
+		}
+	} else if (chip_id == BMP280_CHIP_ID || chip_id == BME280_CHIP_ID) {
+		ret = bmp280_read_calib(data, &data->calib.bmp280, chip_id);
 		if (ret < 0) {
 			dev_err(data->dev,
 				"failed to read calibration coefficients\n");
diff --git a/drivers/iio/pressure/st_pressure_core.c b/drivers/iio/pressure/st_pressure_core.c
index 349e5c713c03..4ddb6cf7d401 100644
--- a/drivers/iio/pressure/st_pressure_core.c
+++ b/drivers/iio/pressure/st_pressure_core.c
@@ -640,7 +640,7 @@ int st_press_common_probe(struct iio_dev *indio_dev)
 		       press_data->sensor_settings->drdy_irq.int2.addr))
 		pdata =	(struct st_sensors_platform_data *)&default_press_pdata;
 
-	err = st_sensors_init_sensor(indio_dev, press_data->dev->platform_data);
+	err = st_sensors_init_sensor(indio_dev, pdata);
 	if (err < 0)
 		goto st_press_power_off;
 
diff --git a/drivers/iio/proximity/Kconfig b/drivers/iio/proximity/Kconfig
index fcb1c4ba5e41..f726f9427602 100644
--- a/drivers/iio/proximity/Kconfig
+++ b/drivers/iio/proximity/Kconfig
@@ -68,6 +68,8 @@ config SX9500
 
 config SRF08
 	tristate "Devantech SRF02/SRF08/SRF10 ultrasonic ranger sensor"
+	select IIO_BUFFER
+	select IIO_TRIGGERED_BUFFER
 	depends on I2C
 	help
 	  Say Y here to build a driver for Devantech SRF02/SRF08/SRF10
diff --git a/drivers/iio/proximity/sx9500.c b/drivers/iio/proximity/sx9500.c
index df23dbcc030a..b8a2c2c8cac5 100644
--- a/drivers/iio/proximity/sx9500.c
+++ b/drivers/iio/proximity/sx9500.c
@@ -1031,6 +1031,7 @@ static const struct dev_pm_ops sx9500_pm_ops = {
 
 static const struct acpi_device_id sx9500_acpi_match[] = {
 	{"SSX9500", 0},
+	{"SASX9500", 0},
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, sx9500_acpi_match);
diff --git a/drivers/iio/trigger/stm32-lptimer-trigger.c b/drivers/iio/trigger/stm32-lptimer-trigger.c
index de361d879929..98cdc7e47f3d 100644
--- a/drivers/iio/trigger/stm32-lptimer-trigger.c
+++ b/drivers/iio/trigger/stm32-lptimer-trigger.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * STM32 Low-Power Timer Trigger driver
  *
@@ -5,8 +6,6 @@
  *
  * Author: Fabrice Gasnier <fabrice.gasnier@st.com>.
  *
- * License terms:  GNU General Public License (GPL), version 2
- *
  * Inspired by Benjamin Gaignard's stm32-timer-trigger driver
  */
 
diff --git a/drivers/iio/trigger/stm32-timer-trigger.c b/drivers/iio/trigger/stm32-timer-trigger.c
index b542dc484969..ccf1ce653b25 100644
--- a/drivers/iio/trigger/stm32-timer-trigger.c
+++ b/drivers/iio/trigger/stm32-timer-trigger.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics 2016
  *
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com>
  *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/iio/iio.h>
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index cbf186522016..8517d6ea91a6 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -4,6 +4,7 @@ menuconfig INFINIBAND
 	depends on NET
 	depends on INET
 	depends on m || IPV6 != m
+	depends on !ALPHA
 	select IRQ_POLL
 	---help---
 	  Core support for InfiniBand (IB).  Make sure to also select
@@ -19,7 +20,8 @@ config INFINIBAND_USER_MAD
 	  Userspace InfiniBand Management Datagram (MAD) support.  This
 	  is the kernel side of the userspace MAD support, which allows
 	  userspace processes to send and receive MADs. You will also
-	  need libibumad from <http://www.openfabrics.org/downloads/management/>.
+	  need libibumad from rdma-core
+	  <https://github.com/linux-rdma/rdma-core>.
 
 config INFINIBAND_USER_ACCESS
 	tristate "InfiniBand userspace access (verbs and CM)"
@@ -31,10 +33,10 @@ config INFINIBAND_USER_ACCESS
 	  to set up connections and directly access InfiniBand
 	  hardware for fast-path operations.  You will also need
 	  libibverbs, libibcm and a hardware driver library from
-	  <http://www.openfabrics.org/git/>.
+	  rdma-core <https://github.com/linux-rdma/rdma-core>.
 
 config INFINIBAND_EXP_USER_ACCESS
-	bool "Allow experimental support for Infiniband ABI"
+	bool "Enable the full uverbs ioctl interface (EXPERIMENTAL)"
 	depends on INFINIBAND_USER_ACCESS
 	---help---
 	  IOCTL based ABI support for Infiniband. This allows userspace
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index 504b926552c6..f69833db0a32 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -12,7 +12,7 @@ ib_core-y :=			packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
 				device.o fmr_pool.o cache.o netlink.o \
 				roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
 				multicast.o mad.o smi.o agent.o mad_rmpp.o \
-				security.o nldev.o
+				security.o nldev.o restrack.o
 
 ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
 ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index f4e8185bccd3..cb1d2ab13c66 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -207,6 +207,22 @@ int rdma_addr_size(struct sockaddr *addr)
 }
 EXPORT_SYMBOL(rdma_addr_size);
 
+int rdma_addr_size_in6(struct sockaddr_in6 *addr)
+{
+	int ret = rdma_addr_size((struct sockaddr *) addr);
+
+	return ret <= sizeof(*addr) ? ret : 0;
+}
+EXPORT_SYMBOL(rdma_addr_size_in6);
+
+int rdma_addr_size_kss(struct __kernel_sockaddr_storage *addr)
+{
+	int ret = rdma_addr_size((struct sockaddr *) addr);
+
+	return ret <= sizeof(*addr) ? ret : 0;
+}
+EXPORT_SYMBOL(rdma_addr_size_kss);
+
 static struct rdma_addr_client self;
 
 void rdma_addr_register_client(struct rdma_addr_client *client)
@@ -243,8 +259,7 @@ void rdma_copy_addr(struct rdma_dev_addr *dev_addr,
 EXPORT_SYMBOL(rdma_copy_addr);
 
 int rdma_translate_ip(const struct sockaddr *addr,
-		      struct rdma_dev_addr *dev_addr,
-		      u16 *vlan_id)
+		      struct rdma_dev_addr *dev_addr)
 {
 	struct net_device *dev;
 
@@ -266,9 +281,6 @@ int rdma_translate_ip(const struct sockaddr *addr,
 			return -EADDRNOTAVAIL;
 
 		rdma_copy_addr(dev_addr, dev, NULL);
-		dev_addr->bound_dev_if = dev->ifindex;
-		if (vlan_id)
-			*vlan_id = rdma_vlan_dev_vlan_id(dev);
 		dev_put(dev);
 		break;
 #if IS_ENABLED(CONFIG_IPV6)
@@ -279,9 +291,6 @@ int rdma_translate_ip(const struct sockaddr *addr,
 					  &((const struct sockaddr_in6 *)addr)->sin6_addr,
 					  dev, 1)) {
 				rdma_copy_addr(dev_addr, dev, NULL);
-				dev_addr->bound_dev_if = dev->ifindex;
-				if (vlan_id)
-					*vlan_id = rdma_vlan_dev_vlan_id(dev);
 				break;
 			}
 		}
@@ -481,7 +490,7 @@ static int addr_resolve_neigh(struct dst_entry *dst,
 	if (dst->dev->flags & IFF_LOOPBACK) {
 		int ret;
 
-		ret = rdma_translate_ip(dst_in, addr, NULL);
+		ret = rdma_translate_ip(dst_in, addr);
 		if (!ret)
 			memcpy(addr->dst_dev_addr, addr->src_dev_addr,
 			       MAX_ADDR_LEN);
@@ -557,18 +566,13 @@ static int addr_resolve(struct sockaddr *src_in,
 		dst_release(dst);
 	}
 
-	if (ndev->flags & IFF_LOOPBACK) {
-		ret = rdma_translate_ip(dst_in, addr, NULL);
-		/*
-		 * Put the loopback device and get the translated
-		 * device instead.
-		 */
+	if (ndev) {
+		if (ndev->flags & IFF_LOOPBACK)
+			ret = rdma_translate_ip(dst_in, addr);
+		else
+			addr->bound_dev_if = ndev->ifindex;
 		dev_put(ndev);
-		ndev = dev_get_by_index(addr->net, addr->bound_dev_if);
-	} else {
-		addr->bound_dev_if = ndev->ifindex;
 	}
-	dev_put(ndev);
 
 	return ret;
 }
@@ -598,6 +602,15 @@ static void process_one_req(struct work_struct *_work)
 	list_del(&req->list);
 	mutex_unlock(&lock);
 
+	/*
+	 * Although the work will normally have been canceled by the
+	 * workqueue, it can still be requeued as long as it is on the
+	 * req_list, so it could have been requeued before we grabbed &lock.
+	 * We need to cancel it after it is removed from req_list to really be
+	 * sure it is safe to free.
+	 */
+	cancel_delayed_work(&req->work);
+
 	req->callback(req->status, (struct sockaddr *)&req->src_addr,
 		req->addr, req->context);
 	put_client(req->client);
@@ -744,7 +757,6 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr)
 EXPORT_SYMBOL(rdma_addr_cancel);
 
 struct resolve_cb_context {
-	struct rdma_dev_addr *addr;
 	struct completion comp;
 	int status;
 };
@@ -752,39 +764,31 @@ struct resolve_cb_context {
 static void resolve_cb(int status, struct sockaddr *src_addr,
 	     struct rdma_dev_addr *addr, void *context)
 {
-	if (!status)
-		memcpy(((struct resolve_cb_context *)context)->addr,
-		       addr, sizeof(struct rdma_dev_addr));
 	((struct resolve_cb_context *)context)->status = status;
 	complete(&((struct resolve_cb_context *)context)->comp);
 }
 
 int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
 				 const union ib_gid *dgid,
-				 u8 *dmac, u16 *vlan_id, int *if_index,
+				 u8 *dmac, const struct net_device *ndev,
 				 int *hoplimit)
 {
-	int ret = 0;
 	struct rdma_dev_addr dev_addr;
 	struct resolve_cb_context ctx;
-	struct net_device *dev;
-
 	union {
 		struct sockaddr     _sockaddr;
 		struct sockaddr_in  _sockaddr_in;
 		struct sockaddr_in6 _sockaddr_in6;
 	} sgid_addr, dgid_addr;
-
+	int ret;
 
 	rdma_gid2ip(&sgid_addr._sockaddr, sgid);
 	rdma_gid2ip(&dgid_addr._sockaddr, dgid);
 
 	memset(&dev_addr, 0, sizeof(dev_addr));
-	if (if_index)
-		dev_addr.bound_dev_if = *if_index;
+	dev_addr.bound_dev_if = ndev->ifindex;
 	dev_addr.net = &init_net;
 
-	ctx.addr = &dev_addr;
 	init_completion(&ctx.comp);
 	ret = rdma_resolve_ip(&self, &sgid_addr._sockaddr, &dgid_addr._sockaddr,
 			&dev_addr, 1000, resolve_cb, &ctx);
@@ -798,42 +802,9 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
 		return ret;
 
 	memcpy(dmac, dev_addr.dst_dev_addr, ETH_ALEN);
-	dev = dev_get_by_index(&init_net, dev_addr.bound_dev_if);
-	if (!dev)
-		return -ENODEV;
-	if (if_index)
-		*if_index = dev_addr.bound_dev_if;
-	if (vlan_id)
-		*vlan_id = rdma_vlan_dev_vlan_id(dev);
-	if (hoplimit)
-		*hoplimit = dev_addr.hoplimit;
-	dev_put(dev);
-	return ret;
-}
-EXPORT_SYMBOL(rdma_addr_find_l2_eth_by_grh);
-
-int rdma_addr_find_smac_by_sgid(union ib_gid *sgid, u8 *smac, u16 *vlan_id)
-{
-	int ret = 0;
-	struct rdma_dev_addr dev_addr;
-	union {
-		struct sockaddr     _sockaddr;
-		struct sockaddr_in  _sockaddr_in;
-		struct sockaddr_in6 _sockaddr_in6;
-	} gid_addr;
-
-	rdma_gid2ip(&gid_addr._sockaddr, sgid);
-
-	memset(&dev_addr, 0, sizeof(dev_addr));
-	dev_addr.net = &init_net;
-	ret = rdma_translate_ip(&gid_addr._sockaddr, &dev_addr, vlan_id);
-	if (ret)
-		return ret;
-
-	memcpy(smac, dev_addr.src_dev_addr, ETH_ALEN);
-	return ret;
+	*hoplimit = dev_addr.hoplimit;
+	return 0;
 }
-EXPORT_SYMBOL(rdma_addr_find_smac_by_sgid);
 
 static int netevent_callback(struct notifier_block *self, unsigned long event,
 	void *ctx)
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index 77515638c55c..e9a409d7f4e2 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -573,27 +573,24 @@ static int ib_cache_gid_find_by_filter(struct ib_device *ib_dev,
 		struct ib_gid_attr attr;
 
 		if (table->data_vec[i].props & GID_TABLE_ENTRY_INVALID)
-			goto next;
+			continue;
 
 		if (memcmp(gid, &table->data_vec[i].gid, sizeof(*gid)))
-			goto next;
+			continue;
 
 		memcpy(&attr, &table->data_vec[i].attr, sizeof(attr));
 
-		if (filter(gid, &attr, context))
+		if (filter(gid, &attr, context)) {
 			found = true;
-
-next:
-		if (found)
+			if (index)
+				*index = i;
 			break;
+		}
 	}
 	read_unlock_irqrestore(&table->rwlock, flags);
 
 	if (!found)
 		return -ENOENT;
-
-	if (index)
-		*index = i;
 	return 0;
 }
 
@@ -824,12 +821,7 @@ static int gid_table_setup_one(struct ib_device *ib_dev)
 	if (err)
 		return err;
 
-	err = roce_rescan_device(ib_dev);
-
-	if (err) {
-		gid_table_cleanup_one(ib_dev);
-		gid_table_release_one(ib_dev);
-	}
+	rdma_roce_rescan_device(ib_dev);
 
 	return err;
 }
@@ -883,7 +875,6 @@ int ib_find_gid_by_filter(struct ib_device *device,
 					   port_num, filter,
 					   context, index);
 }
-EXPORT_SYMBOL(ib_find_gid_by_filter);
 
 int ib_get_cached_pkey(struct ib_device *device,
 		       u8                port_num,
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index f6b159d79977..e6749157fd86 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -452,13 +452,14 @@ static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 	cm_id_priv->private_data_len = private_data_len;
 }
 
-static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
-				    struct ib_grh *grh, struct cm_av *av)
+static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
+				   struct ib_grh *grh, struct cm_av *av)
 {
 	av->port = port;
 	av->pkey_index = wc->pkey_index;
-	ib_init_ah_from_wc(port->cm_dev->ib_device, port->port_num, wc,
-			   grh, &av->ah_attr);
+	return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
+				       port->port_num, wc,
+				       grh, &av->ah_attr);
 }
 
 static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av,
@@ -494,8 +495,11 @@ static int cm_init_av_by_path(struct sa_path_rec *path, struct cm_av *av,
 		return ret;
 
 	av->port = port;
-	ib_init_ah_from_path(cm_dev->ib_device, port->port_num, path,
-			     &av->ah_attr);
+	ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
+					&av->ah_attr);
+	if (ret)
+		return ret;
+
 	av->timeout = path->packet_life_time + 1;
 
 	spin_lock_irqsave(&cm.lock, flags);
@@ -1560,6 +1564,35 @@ static u16 cm_get_bth_pkey(struct cm_work *work)
 	return pkey;
 }
 
+/**
+ * Convert OPA SGID to IB SGID
+ * ULPs (such as IPoIB) do not understand OPA GIDs and will
+ * reject them as the local_gid will not match the sgid. Therefore,
+ * change the pathrec's SGID to an IB SGID.
+ *
+ * @work: Work completion
+ * @path: Path record
+ */
+static void cm_opa_to_ib_sgid(struct cm_work *work,
+			      struct sa_path_rec *path)
+{
+	struct ib_device *dev = work->port->cm_dev->ib_device;
+	u8 port_num = work->port->port_num;
+
+	if (rdma_cap_opa_ah(dev, port_num) &&
+	    (ib_is_opa_gid(&path->sgid))) {
+		union ib_gid sgid;
+
+		if (ib_get_cached_gid(dev, port_num, 0, &sgid, NULL)) {
+			dev_warn(&dev->dev,
+				 "Error updating sgid in CM request\n");
+			return;
+		}
+
+		path->sgid = sgid;
+	}
+}
+
 static void cm_format_req_event(struct cm_work *work,
 				struct cm_id_private *cm_id_priv,
 				struct ib_cm_id *listen_id)
@@ -1573,10 +1606,13 @@ static void cm_format_req_event(struct cm_work *work,
 	param->bth_pkey = cm_get_bth_pkey(work);
 	param->port = cm_id_priv->av.port->port_num;
 	param->primary_path = &work->path[0];
-	if (cm_req_has_alt_path(req_msg))
+	cm_opa_to_ib_sgid(work, param->primary_path);
+	if (cm_req_has_alt_path(req_msg)) {
 		param->alternate_path = &work->path[1];
-	else
+		cm_opa_to_ib_sgid(work, param->alternate_path);
+	} else {
 		param->alternate_path = NULL;
+	}
 	param->remote_ca_guid = req_msg->local_ca_guid;
 	param->remote_qkey = be32_to_cpu(req_msg->local_qkey);
 	param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));
@@ -1826,9 +1862,11 @@ static int cm_req_handler(struct cm_work *work)
 
 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
 	cm_id_priv->id.remote_id = req_msg->local_comm_id;
-	cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
-				work->mad_recv_wc->recv_buf.grh,
-				&cm_id_priv->av);
+	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
+				      work->mad_recv_wc->recv_buf.grh,
+				      &cm_id_priv->av);
+	if (ret)
+		goto destroy;
 	cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
 							    id.local_id);
 	if (IS_ERR(cm_id_priv->timewait_info)) {
@@ -1841,9 +1879,10 @@ static int cm_req_handler(struct cm_work *work)
 
 	listen_cm_id_priv = cm_match_req(work, cm_id_priv);
 	if (!listen_cm_id_priv) {
+		pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__,
+			 be32_to_cpu(cm_id->local_id));
 		ret = -EINVAL;
-		kfree(cm_id_priv->timewait_info);
-		goto destroy;
+		goto free_timeinfo;
 	}
 
 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
@@ -1861,56 +1900,50 @@ static int cm_req_handler(struct cm_work *work)
 				work->port->port_num,
 				grh->sgid_index,
 				&gid, &gid_attr);
-	if (!ret) {
-		if (gid_attr.ndev) {
-			work->path[0].rec_type =
-				sa_conv_gid_to_pathrec_type(gid_attr.gid_type);
-			sa_path_set_ifindex(&work->path[0],
-					    gid_attr.ndev->ifindex);
-			sa_path_set_ndev(&work->path[0],
-					 dev_net(gid_attr.ndev));
-			dev_put(gid_attr.ndev);
-		} else {
-			cm_path_set_rec_type(work->port->cm_dev->ib_device,
-					     work->port->port_num,
-					     &work->path[0],
-					     &req_msg->primary_local_gid);
-		}
-		if (cm_req_has_alt_path(req_msg))
-			work->path[1].rec_type = work->path[0].rec_type;
-		cm_format_paths_from_req(req_msg, &work->path[0],
-					 &work->path[1]);
-		if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
-			sa_path_set_dmac(&work->path[0],
-					 cm_id_priv->av.ah_attr.roce.dmac);
-		work->path[0].hop_limit = grh->hop_limit;
-		ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av,
-					 cm_id_priv);
+	if (ret) {
+		ib_send_cm_rej(cm_id, IB_CM_REJ_UNSUPPORTED, NULL, 0, NULL, 0);
+		goto rejected;
+	}
+
+	if (gid_attr.ndev) {
+		work->path[0].rec_type =
+			sa_conv_gid_to_pathrec_type(gid_attr.gid_type);
+		sa_path_set_ifindex(&work->path[0],
+				    gid_attr.ndev->ifindex);
+		sa_path_set_ndev(&work->path[0],
+				 dev_net(gid_attr.ndev));
+		dev_put(gid_attr.ndev);
+	} else {
+		cm_path_set_rec_type(work->port->cm_dev->ib_device,
+				     work->port->port_num,
+				     &work->path[0],
+				     &req_msg->primary_local_gid);
 	}
+	if (cm_req_has_alt_path(req_msg))
+		work->path[1].rec_type = work->path[0].rec_type;
+	cm_format_paths_from_req(req_msg, &work->path[0],
+				 &work->path[1]);
+	if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
+		sa_path_set_dmac(&work->path[0],
+				 cm_id_priv->av.ah_attr.roce.dmac);
+	work->path[0].hop_limit = grh->hop_limit;
+	ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av,
+				 cm_id_priv);
 	if (ret) {
-		int err = ib_get_cached_gid(work->port->cm_dev->ib_device,
-					    work->port->port_num, 0,
-					    &work->path[0].sgid,
-					    &gid_attr);
-		if (!err && gid_attr.ndev) {
-			work->path[0].rec_type =
-				sa_conv_gid_to_pathrec_type(gid_attr.gid_type);
-			sa_path_set_ifindex(&work->path[0],
-					    gid_attr.ndev->ifindex);
-			sa_path_set_ndev(&work->path[0],
-					 dev_net(gid_attr.ndev));
-			dev_put(gid_attr.ndev);
-		} else {
-			cm_path_set_rec_type(work->port->cm_dev->ib_device,
-					     work->port->port_num,
-					     &work->path[0],
-					     &req_msg->primary_local_gid);
-		}
-		if (cm_req_has_alt_path(req_msg))
-			work->path[1].rec_type = work->path[0].rec_type;
-		ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
-			       &work->path[0].sgid, sizeof work->path[0].sgid,
-			       NULL, 0);
+		int err;
+
+		err = ib_get_cached_gid(work->port->cm_dev->ib_device,
+					work->port->port_num, 0,
+					&work->path[0].sgid,
+					NULL);
+		if (err)
+			ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
+				       NULL, 0, NULL, 0);
+		else
+			ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
+				       &work->path[0].sgid,
+				       sizeof(work->path[0].sgid),
+				       NULL, 0);
 		goto rejected;
 	}
 	if (cm_req_has_alt_path(req_msg)) {
@@ -1919,7 +1952,7 @@ static int cm_req_handler(struct cm_work *work)
 		if (ret) {
 			ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
 				       &work->path[0].sgid,
-				       sizeof work->path[0].sgid, NULL, 0);
+				       sizeof(work->path[0].sgid), NULL, 0);
 			goto rejected;
 		}
 	}
@@ -1945,6 +1978,8 @@ static int cm_req_handler(struct cm_work *work)
 rejected:
 	atomic_dec(&cm_id_priv->refcount);
 	cm_deref_id(listen_cm_id_priv);
+free_timeinfo:
+	kfree(cm_id_priv->timewait_info);
 destroy:
 	ib_destroy_cm_id(cm_id);
 	return ret;
@@ -1997,6 +2032,8 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id,
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 	if (cm_id->state != IB_CM_REQ_RCVD &&
 	    cm_id->state != IB_CM_MRA_REQ_SENT) {
+		pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__,
+			 be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -2063,6 +2100,8 @@ int ib_send_cm_rtu(struct ib_cm_id *cm_id,
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 	if (cm_id->state != IB_CM_REP_RCVD &&
 	    cm_id->state != IB_CM_MRA_REP_SENT) {
+		pr_debug("%s: local_id %d, cm_id->state %d\n", __func__,
+			 be32_to_cpu(cm_id->local_id), cm_id->state);
 		ret = -EINVAL;
 		goto error;
 	}
@@ -2170,6 +2209,8 @@ static int cm_rep_handler(struct cm_work *work)
 	cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0);
 	if (!cm_id_priv) {
 		cm_dup_rep_handler(work);
+		pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__,
+			 be32_to_cpu(rep_msg->remote_comm_id));
 		return -EINVAL;
 	}
 
@@ -2183,6 +2224,10 @@ static int cm_rep_handler(struct cm_work *work)
 	default:
 		spin_unlock_irq(&cm_id_priv->lock);
 		ret = -EINVAL;
+		pr_debug("%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n",
+			 __func__, cm_id_priv->id.state,
+			 be32_to_cpu(rep_msg->local_comm_id),
+			 be32_to_cpu(rep_msg->remote_comm_id));
 		goto error;
 	}
 
@@ -2196,6 +2241,8 @@ static int cm_rep_handler(struct cm_work *work)
 		spin_unlock(&cm.lock);
 		spin_unlock_irq(&cm_id_priv->lock);
 		ret = -EINVAL;
+		pr_debug("%s: Failed to insert remote id %d\n", __func__,
+			 be32_to_cpu(rep_msg->remote_comm_id));
 		goto error;
 	}
 	/* Check for a stale connection. */
@@ -2213,6 +2260,10 @@ static int cm_rep_handler(struct cm_work *work)
 			     IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
 			     NULL, 0);
 		ret = -EINVAL;
+		pr_debug("%s: Stale connection. local_comm_id %d, remote_comm_id %d\n",
+			 __func__, be32_to_cpu(rep_msg->local_comm_id),
+			 be32_to_cpu(rep_msg->remote_comm_id));
+
 		if (cur_cm_id_priv) {
 			cm_id = &cur_cm_id_priv->id;
 			ib_send_cm_dreq(cm_id, NULL, 0);
@@ -2359,6 +2410,8 @@ int ib_send_cm_dreq(struct ib_cm_id *cm_id,
 	cm_id_priv = container_of(cm_id, struct cm_id_private, id);
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 	if (cm_id->state != IB_CM_ESTABLISHED) {
+		pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
+			 be32_to_cpu(cm_id->local_id), cm_id->state);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -2428,6 +2481,8 @@ int ib_send_cm_drep(struct ib_cm_id *cm_id,
 	if (cm_id->state != IB_CM_DREQ_RCVD) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		kfree(data);
+		pr_debug("%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n",
+			 __func__, be32_to_cpu(cm_id->local_id), cm_id->state);
 		return -EINVAL;
 	}
 
@@ -2493,6 +2548,9 @@ static int cm_dreq_handler(struct cm_work *work)
 		atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
 				counter[CM_DREQ_COUNTER]);
 		cm_issue_drep(work->port, work->mad_recv_wc);
+		pr_debug("%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n",
+			 __func__, be32_to_cpu(dreq_msg->local_comm_id),
+			 be32_to_cpu(dreq_msg->remote_comm_id));
 		return -EINVAL;
 	}
 
@@ -2535,6 +2593,9 @@ static int cm_dreq_handler(struct cm_work *work)
 				counter[CM_DREQ_COUNTER]);
 		goto unlock;
 	default:
+		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
+			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
+			 cm_id_priv->id.state);
 		goto unlock;
 	}
 	cm_id_priv->id.state = IB_CM_DREQ_RCVD;
@@ -2638,6 +2699,8 @@ int ib_send_cm_rej(struct ib_cm_id *cm_id,
 		cm_enter_timewait(cm_id_priv);
 		break;
 	default:
+		pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
+			 be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -2748,6 +2811,9 @@ static int cm_rej_handler(struct cm_work *work)
 		/* fall through */
 	default:
 		spin_unlock_irq(&cm_id_priv->lock);
+		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
+			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
+			 cm_id_priv->id.state);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -2811,6 +2877,9 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id,
 		}
 		/* fall through */
 	default:
+		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
+			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
+			 cm_id_priv->id.state);
 		ret = -EINVAL;
 		goto error1;
 	}
@@ -2912,6 +2981,9 @@ static int cm_mra_handler(struct cm_work *work)
 				counter[CM_MRA_COUNTER]);
 		/* fall through */
 	default:
+		pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n",
+			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
+			 cm_id_priv->id.state);
 		goto out;
 	}
 
@@ -3085,6 +3157,12 @@ static int cm_lap_handler(struct cm_work *work)
 	if (!cm_id_priv)
 		return -EINVAL;
 
+	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
+				      work->mad_recv_wc->recv_buf.grh,
+				      &cm_id_priv->av);
+	if (ret)
+		goto deref;
+
 	param = &work->cm_event.param.lap_rcvd;
 	memset(&work->path[0], 0, sizeof(work->path[1]));
 	cm_path_set_rec_type(work->port->cm_dev->ib_device,
@@ -3131,9 +3209,6 @@ static int cm_lap_handler(struct cm_work *work)
 
 	cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
 	cm_id_priv->tid = lap_msg->hdr.tid;
-	cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
-				work->mad_recv_wc->recv_buf.grh,
-				&cm_id_priv->av);
 	cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av,
 			   cm_id_priv);
 	ret = atomic_inc_and_test(&cm_id_priv->work_count);
@@ -3386,6 +3461,7 @@ static int cm_sidr_req_handler(struct cm_work *work)
 	struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
 	struct cm_sidr_req_msg *sidr_req_msg;
 	struct ib_wc *wc;
+	int ret;
 
 	cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
 	if (IS_ERR(cm_id))
@@ -3398,9 +3474,12 @@ static int cm_sidr_req_handler(struct cm_work *work)
 	wc = work->mad_recv_wc->wc;
 	cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
 	cm_id_priv->av.dgid.global.interface_id = 0;
-	cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
-				work->mad_recv_wc->recv_buf.grh,
-				&cm_id_priv->av);
+	ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
+				      work->mad_recv_wc->recv_buf.grh,
+				      &cm_id_priv->av);
+	if (ret)
+		goto out;
+
 	cm_id_priv->id.remote_id = sidr_req_msg->request_id;
 	cm_id_priv->tid = sidr_req_msg->hdr.tid;
 	atomic_inc(&cm_id_priv->work_count);
@@ -3692,6 +3771,7 @@ static void cm_work_handler(struct work_struct *_work)
 		ret = cm_timewait_handler(work);
 		break;
 	default:
+		pr_debug("cm_event.event: 0x%x\n", work->cm_event.event);
 		ret = -EINVAL;
 		break;
 	}
@@ -3727,6 +3807,8 @@ static int cm_establish(struct ib_cm_id *cm_id)
 		ret = -EISCONN;
 		break;
 	default:
+		pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
+			 be32_to_cpu(cm_id->local_id), cm_id->state);
 		ret = -EINVAL;
 		break;
 	}
@@ -3924,6 +4006,9 @@ static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
 		ret = 0;
 		break;
 	default:
+		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
+			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
+			 cm_id_priv->id.state);
 		ret = -EINVAL;
 		break;
 	}
@@ -3971,6 +4056,9 @@ static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
 		ret = 0;
 		break;
 	default:
+		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
+			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
+			 cm_id_priv->id.state);
 		ret = -EINVAL;
 		break;
 	}
@@ -4030,6 +4118,9 @@ static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
 		ret = 0;
 		break;
 	default:
+		pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
+			 __func__, be32_to_cpu(cm_id_priv->id.local_id),
+			 cm_id_priv->id.state);
 		ret = -EINVAL;
 		break;
 	}
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 6294a7001d33..a5367c5efbe7 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -601,7 +601,7 @@ static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_a
 	int ret;
 
 	if (addr->sa_family != AF_IB) {
-		ret = rdma_translate_ip(addr, dev_addr, NULL);
+		ret = rdma_translate_ip(addr, dev_addr);
 	} else {
 		cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
 		ret = 0;
@@ -612,11 +612,14 @@ static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_a
 
 static inline int cma_validate_port(struct ib_device *device, u8 port,
 				    enum ib_gid_type gid_type,
-				      union ib_gid *gid, int dev_type,
-				      int bound_if_index)
+				    union ib_gid *gid,
+				    struct rdma_id_private *id_priv)
 {
-	int ret = -ENODEV;
+	struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
+	int bound_if_index = dev_addr->bound_dev_if;
+	int dev_type = dev_addr->dev_type;
 	struct net_device *ndev = NULL;
+	int ret = -ENODEV;
 
 	if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
 		return ret;
@@ -624,11 +627,13 @@ static inline int cma_validate_port(struct ib_device *device, u8 port,
 	if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
 		return ret;
 
-	if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port))
-		ndev = dev_get_by_index(&init_net, bound_if_index);
-	else
+	if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
+		ndev = dev_get_by_index(dev_addr->net, bound_if_index);
+		if (!ndev)
+			return ret;
+	} else {
 		gid_type = IB_GID_TYPE_IB;
-
+	}
 
 	ret = ib_find_cached_gid_by_port(device, gid, gid_type, port,
 					 ndev, NULL);
@@ -669,8 +674,7 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv,
 					rdma_protocol_ib(cma_dev->device, port) ?
 					IB_GID_TYPE_IB :
 					listen_id_priv->gid_type, gidp,
-					dev_addr->dev_type,
-					dev_addr->bound_dev_if);
+					id_priv);
 		if (!ret) {
 			id_priv->id.port_num = port;
 			goto out;
@@ -691,8 +695,7 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv,
 						rdma_protocol_ib(cma_dev->device, port) ?
 						IB_GID_TYPE_IB :
 						cma_dev->default_gid_type[port - 1],
-						gidp, dev_addr->dev_type,
-						dev_addr->bound_dev_if);
+						gidp, id_priv);
 			if (!ret) {
 				id_priv->id.port_num = port;
 				goto out;
@@ -2036,6 +2039,33 @@ __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr)
 }
 EXPORT_SYMBOL(rdma_get_service_id);
 
+void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid,
+		    union ib_gid *dgid)
+{
+	struct rdma_addr *addr = &cm_id->route.addr;
+
+	if (!cm_id->device) {
+		if (sgid)
+			memset(sgid, 0, sizeof(*sgid));
+		if (dgid)
+			memset(dgid, 0, sizeof(*dgid));
+		return;
+	}
+
+	if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) {
+		if (sgid)
+			rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid);
+		if (dgid)
+			rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid);
+	} else {
+		if (sgid)
+			rdma_addr_get_sgid(&addr->dev_addr, sgid);
+		if (dgid)
+			rdma_addr_get_dgid(&addr->dev_addr, dgid);
+	}
+}
+EXPORT_SYMBOL(rdma_read_gids);
+
 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
 {
 	struct rdma_id_private *id_priv = iw_id->context;
@@ -2132,7 +2162,7 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
 	mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
 	conn_id->state = RDMA_CM_CONNECT;
 
-	ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr, NULL);
+	ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
 	if (ret) {
 		mutex_unlock(&conn_id->handler_mutex);
 		rdma_destroy_id(new_cm_id);
@@ -2414,6 +2444,26 @@ static void cma_ndev_work_handler(struct work_struct *_work)
 	kfree(work);
 }
 
+static void cma_init_resolve_route_work(struct cma_work *work,
+					struct rdma_id_private *id_priv)
+{
+	work->id = id_priv;
+	INIT_WORK(&work->work, cma_work_handler);
+	work->old_state = RDMA_CM_ROUTE_QUERY;
+	work->new_state = RDMA_CM_ROUTE_RESOLVED;
+	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+}
+
+static void cma_init_resolve_addr_work(struct cma_work *work,
+				       struct rdma_id_private *id_priv)
+{
+	work->id = id_priv;
+	INIT_WORK(&work->work, cma_work_handler);
+	work->old_state = RDMA_CM_ADDR_QUERY;
+	work->new_state = RDMA_CM_ADDR_RESOLVED;
+	work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
+}
+
 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
 {
 	struct rdma_route *route = &id_priv->id.route;
@@ -2424,11 +2474,7 @@ static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
 	if (!work)
 		return -ENOMEM;
 
-	work->id = id_priv;
-	INIT_WORK(&work->work, cma_work_handler);
-	work->old_state = RDMA_CM_ROUTE_QUERY;
-	work->new_state = RDMA_CM_ROUTE_RESOLVED;
-	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+	cma_init_resolve_route_work(work, id_priv);
 
 	route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
 	if (!route->path_rec) {
@@ -2449,10 +2495,63 @@ static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
 	return ret;
 }
 
-int rdma_set_ib_paths(struct rdma_cm_id *id,
-		      struct sa_path_rec *path_rec, int num_paths)
+static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
+					   unsigned long supported_gids,
+					   enum ib_gid_type default_gid)
+{
+	if ((network_type == RDMA_NETWORK_IPV4 ||
+	     network_type == RDMA_NETWORK_IPV6) &&
+	    test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
+		return IB_GID_TYPE_ROCE_UDP_ENCAP;
+
+	return default_gid;
+}
+
+/*
+ * cma_iboe_set_path_rec_l2_fields() is helper function which sets
+ * path record type based on GID type.
+ * It also sets up other L2 fields which includes destination mac address
+ * netdev ifindex, of the path record.
+ * It returns the netdev of the bound interface for this path record entry.
+ */
+static struct net_device *
+cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv)
+{
+	struct rdma_route *route = &id_priv->id.route;
+	enum ib_gid_type gid_type = IB_GID_TYPE_ROCE;
+	struct rdma_addr *addr = &route->addr;
+	unsigned long supported_gids;
+	struct net_device *ndev;
+
+	if (!addr->dev_addr.bound_dev_if)
+		return NULL;
+
+	ndev = dev_get_by_index(addr->dev_addr.net,
+				addr->dev_addr.bound_dev_if);
+	if (!ndev)
+		return NULL;
+
+	supported_gids = roce_gid_type_mask_support(id_priv->id.device,
+						    id_priv->id.port_num);
+	gid_type = cma_route_gid_type(addr->dev_addr.network,
+				      supported_gids,
+				      id_priv->gid_type);
+	/* Use the hint from IP Stack to select GID Type */
+	if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
+		gid_type = ib_network_to_gid_type(addr->dev_addr.network);
+	route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
+
+	sa_path_set_ndev(route->path_rec, addr->dev_addr.net);
+	sa_path_set_ifindex(route->path_rec, ndev->ifindex);
+	sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
+	return ndev;
+}
+
+int rdma_set_ib_path(struct rdma_cm_id *id,
+		     struct sa_path_rec *path_rec)
 {
 	struct rdma_id_private *id_priv;
+	struct net_device *ndev;
 	int ret;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
@@ -2460,20 +2559,33 @@ int rdma_set_ib_paths(struct rdma_cm_id *id,
 			   RDMA_CM_ROUTE_RESOLVED))
 		return -EINVAL;
 
-	id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
+	id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec),
 				     GFP_KERNEL);
 	if (!id->route.path_rec) {
 		ret = -ENOMEM;
 		goto err;
 	}
 
-	id->route.num_paths = num_paths;
+	if (rdma_protocol_roce(id->device, id->port_num)) {
+		ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
+		if (!ndev) {
+			ret = -ENODEV;
+			goto err_free;
+		}
+		dev_put(ndev);
+	}
+
+	id->route.num_paths = 1;
 	return 0;
+
+err_free:
+	kfree(id->route.path_rec);
+	id->route.path_rec = NULL;
 err:
 	cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
 	return ret;
 }
-EXPORT_SYMBOL(rdma_set_ib_paths);
+EXPORT_SYMBOL(rdma_set_ib_path);
 
 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
 {
@@ -2483,11 +2595,7 @@ static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
 	if (!work)
 		return -ENOMEM;
 
-	work->id = id_priv;
-	INIT_WORK(&work->work, cma_work_handler);
-	work->old_state = RDMA_CM_ROUTE_QUERY;
-	work->new_state = RDMA_CM_ROUTE_RESOLVED;
-	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+	cma_init_resolve_route_work(work, id_priv);
 	queue_work(cma_wq, &work->work);
 	return 0;
 }
@@ -2510,26 +2618,14 @@ static int iboe_tos_to_sl(struct net_device *ndev, int tos)
 	return 0;
 }
 
-static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
-					   unsigned long supported_gids,
-					   enum ib_gid_type default_gid)
-{
-	if ((network_type == RDMA_NETWORK_IPV4 ||
-	     network_type == RDMA_NETWORK_IPV6) &&
-	    test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
-		return IB_GID_TYPE_ROCE_UDP_ENCAP;
-
-	return default_gid;
-}
-
 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 {
 	struct rdma_route *route = &id_priv->id.route;
 	struct rdma_addr *addr = &route->addr;
 	struct cma_work *work;
 	int ret;
-	struct net_device *ndev = NULL;
-	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
+	struct net_device *ndev;
+
 	u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -
 					rdma_start_port(id_priv->cma_dev->device)];
 	u8 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos;
@@ -2539,9 +2635,6 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 	if (!work)
 		return -ENOMEM;
 
-	work->id = id_priv;
-	INIT_WORK(&work->work, cma_work_handler);
-
 	route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
 	if (!route->path_rec) {
 		ret = -ENOMEM;
@@ -2550,42 +2643,17 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 
 	route->num_paths = 1;
 
-	if (addr->dev_addr.bound_dev_if) {
-		unsigned long supported_gids;
-
-		ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
-		if (!ndev) {
-			ret = -ENODEV;
-			goto err2;
-		}
-
-		supported_gids = roce_gid_type_mask_support(id_priv->id.device,
-							    id_priv->id.port_num);
-		gid_type = cma_route_gid_type(addr->dev_addr.network,
-					      supported_gids,
-					      id_priv->gid_type);
-		route->path_rec->rec_type =
-			sa_conv_gid_to_pathrec_type(gid_type);
-		sa_path_set_ndev(route->path_rec, &init_net);
-		sa_path_set_ifindex(route->path_rec, ndev->ifindex);
-	}
+	ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
 	if (!ndev) {
 		ret = -ENODEV;
 		goto err2;
 	}
 
-	sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
-
 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 		    &route->path_rec->sgid);
 	rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
 		    &route->path_rec->dgid);
 
-	/* Use the hint from IP Stack to select GID Type */
-	if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
-		gid_type = ib_network_to_gid_type(addr->dev_addr.network);
-	route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
-
 	if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
 		/* TODO: get the hoplimit from the inet/inet6 device */
 		route->path_rec->hop_limit = addr->dev_addr.hoplimit;
@@ -2607,11 +2675,7 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
 		goto err2;
 	}
 
-	work->old_state = RDMA_CM_ROUTE_QUERY;
-	work->new_state = RDMA_CM_ROUTE_RESOLVED;
-	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
-	work->event.status = 0;
-
+	cma_init_resolve_route_work(work, id_priv);
 	queue_work(cma_wq, &work->work);
 
 	return 0;
@@ -2791,11 +2855,7 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv)
 	rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
 	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
 
-	work->id = id_priv;
-	INIT_WORK(&work->work, cma_work_handler);
-	work->old_state = RDMA_CM_ADDR_QUERY;
-	work->new_state = RDMA_CM_ADDR_RESOLVED;
-	work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
+	cma_init_resolve_addr_work(work, id_priv);
 	queue_work(cma_wq, &work->work);
 	return 0;
 err:
@@ -2821,11 +2881,7 @@ static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
 	rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
 		&(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
 
-	work->id = id_priv;
-	INIT_WORK(&work->work, cma_work_handler);
-	work->old_state = RDMA_CM_ADDR_QUERY;
-	work->new_state = RDMA_CM_ADDR_RESOLVED;
-	work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
+	cma_init_resolve_addr_work(work, id_priv);
 	queue_work(cma_wq, &work->work);
 	return 0;
 err:
@@ -3013,7 +3069,8 @@ static int cma_port_is_unique(struct rdma_bind_list *bind_list,
 			continue;
 
 		/* different dest port -> unique */
-		if (!cma_any_port(cur_daddr) &&
+		if (!cma_any_port(daddr) &&
+		    !cma_any_port(cur_daddr) &&
 		    (dport != cur_dport))
 			continue;
 
@@ -3024,7 +3081,8 @@ static int cma_port_is_unique(struct rdma_bind_list *bind_list,
 			continue;
 
 		/* different dst address -> unique */
-		if (!cma_any_addr(cur_daddr) &&
+		if (!cma_any_addr(daddr) &&
+		    !cma_any_addr(cur_daddr) &&
 		    cma_addr_cmp(daddr, cur_daddr))
 			continue;
 
@@ -3322,13 +3380,13 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
 		}
 #endif
 	}
+	daddr = cma_dst_addr(id_priv);
+	daddr->sa_family = addr->sa_family;
+
 	ret = cma_get_port(id_priv);
 	if (ret)
 		goto err2;
 
-	daddr = cma_dst_addr(id_priv);
-	daddr->sa_family = addr->sa_family;
-
 	return 0;
 err2:
 	if (id_priv->cma_dev)
@@ -3404,9 +3462,10 @@ static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
 			event.status = ret;
 			break;
 		}
-		ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
-				     id_priv->id.route.path_rec,
-				     &event.param.ud.ah_attr);
+		ib_init_ah_attr_from_path(id_priv->id.device,
+					  id_priv->id.port_num,
+					  id_priv->id.route.path_rec,
+					  &event.param.ud.ah_attr);
 		event.param.ud.qp_num = rep->qpn;
 		event.param.ud.qkey = rep->qkey;
 		event.event = RDMA_CM_EVENT_ESTABLISHED;
@@ -3873,7 +3932,7 @@ static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
 		struct rdma_dev_addr *dev_addr =
 			&id_priv->id.route.addr.dev_addr;
 		struct net_device *ndev =
-			dev_get_by_index(&init_net, dev_addr->bound_dev_if);
+			dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
 		enum ib_gid_type gid_type =
 			id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
 			rdma_start_port(id_priv->cma_dev->device)];
@@ -4010,8 +4069,10 @@ static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
 	} else if (addr->sa_family == AF_INET6) {
 		memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
 	} else {
-		mgid->raw[0] = (gid_type == IB_GID_TYPE_IB) ? 0xff : 0;
-		mgid->raw[1] = (gid_type == IB_GID_TYPE_IB) ? 0x0e : 0;
+		mgid->raw[0] =
+			(gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff;
+		mgid->raw[1] =
+			(gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e;
 		mgid->raw[2] = 0;
 		mgid->raw[3] = 0;
 		mgid->raw[4] = 0;
@@ -4061,7 +4122,7 @@ static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
 		mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
 
 	if (dev_addr->bound_dev_if)
-		ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
+		ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
 	if (!ndev) {
 		err = -ENODEV;
 		goto out2;
@@ -4114,6 +4175,9 @@ int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
 	struct cma_multicast *mc;
 	int ret;
 
+	if (!id->device)
+		return -EINVAL;
+
 	id_priv = container_of(id, struct rdma_id_private, id);
 	if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
 	    !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
@@ -4179,7 +4243,7 @@ void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
 					struct net_device *ndev = NULL;
 
 					if (dev_addr->bound_dev_if)
-						ndev = dev_get_by_index(&init_net,
+						ndev = dev_get_by_index(dev_addr->net,
 									dev_addr->bound_dev_if);
 					if (ndev) {
 						cma_igmp_send(ndev,
@@ -4235,7 +4299,7 @@ static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
 	if (event != NETDEV_BONDING_FAILOVER)
 		return NOTIFY_DONE;
 
-	if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
+	if (!netif_is_bond_master(ndev))
 		return NOTIFY_DONE;
 
 	mutex_lock(&lock);
@@ -4432,7 +4496,7 @@ static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
 					  RDMA_NL_RDMA_CM_ATTR_SRC_ADDR))
 				goto out;
 			if (ibnl_put_attr(skb, nlh,
-					  rdma_addr_size(cma_src_addr(id_priv)),
+					  rdma_addr_size(cma_dst_addr(id_priv)),
 					  cma_dst_addr(id_priv),
 					  RDMA_NL_RDMA_CM_ATTR_DST_ADDR))
 				goto out;
@@ -4444,6 +4508,7 @@ static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
 			id_stats->qp_type	= id->qp_type;
 
 			i_id++;
+			nlmsg_end(skb, nlh);
 		}
 
 		cb->args[1] = 0;
diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c
index 31dfee0c8295..eee38b40be99 100644
--- a/drivers/infiniband/core/cma_configfs.c
+++ b/drivers/infiniband/core/cma_configfs.c
@@ -295,7 +295,7 @@ static struct config_group *make_cma_dev(struct config_group *group,
 		goto fail;
 	}
 
-	strncpy(cma_dev_group->name, name, sizeof(cma_dev_group->name));
+	strlcpy(cma_dev_group->name, name, sizeof(cma_dev_group->name));
 
 	config_group_init_type_name(&cma_dev_group->ports_group, "ports",
 				    &cma_ports_group_type);
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index 66f0268f37a6..25bb178f6074 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -40,8 +40,12 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/opa_addr.h>
 #include <rdma/ib_mad.h>
+#include <rdma/restrack.h>
 #include "mad_priv.h"
 
+/* Total number of ports combined across all struct ib_devices's */
+#define RDMA_MAX_PORTS 1024
+
 struct pkey_index_qp_list {
 	struct list_head    pkey_index_list;
 	u16                 pkey_index;
@@ -137,7 +141,6 @@ int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
 int roce_gid_mgmt_init(void);
 void roce_gid_mgmt_cleanup(void);
 
-int roce_rescan_device(struct ib_device *ib_dev);
 unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port);
 
 int ib_cache_setup_one(struct ib_device *device);
@@ -191,13 +194,6 @@ void ib_sa_cleanup(void);
 int rdma_nl_init(void);
 void rdma_nl_exit(void);
 
-/**
- * Check if there are any listeners to the netlink group
- * @group: the netlink group ID
- * Returns 0 on success or a negative for no listeners.
- */
-int ibnl_chk_listeners(unsigned int group);
-
 int ib_nl_handle_resolve_resp(struct sk_buff *skb,
 			      struct nlmsghdr *nlh,
 			      struct netlink_ext_ack *extack);
@@ -213,11 +209,6 @@ int ib_get_cached_subnet_prefix(struct ib_device *device,
 				u64              *sn_pfx);
 
 #ifdef CONFIG_SECURITY_INFINIBAND
-int ib_security_pkey_access(struct ib_device *dev,
-			    u8 port_num,
-			    u16 pkey_index,
-			    void *sec);
-
 void ib_security_destroy_port_pkey_list(struct ib_device *device);
 
 void ib_security_cache_change(struct ib_device *device,
@@ -240,14 +231,6 @@ int ib_mad_agent_security_setup(struct ib_mad_agent *agent,
 void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent);
 int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index);
 #else
-static inline int ib_security_pkey_access(struct ib_device *dev,
-					  u8 port_num,
-					  u16 pkey_index,
-					  void *sec)
-{
-	return 0;
-}
-
 static inline void ib_security_destroy_port_pkey_list(struct ib_device *device)
 {
 }
@@ -318,4 +301,36 @@ struct ib_device *ib_device_get_by_index(u32 ifindex);
 /* RDMA device netlink */
 void nldev_init(void);
 void nldev_exit(void);
+
+static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
+					  struct ib_pd *pd,
+					  struct ib_qp_init_attr *attr,
+					  struct ib_udata *udata,
+					  struct ib_uobject *uobj)
+{
+	struct ib_qp *qp;
+
+	if (!dev->create_qp)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	qp = dev->create_qp(pd, attr, udata);
+	if (IS_ERR(qp))
+		return qp;
+
+	qp->device = dev;
+	qp->pd = pd;
+	qp->uobject = uobj;
+	/*
+	 * We don't track XRC QPs for now, because they don't have PD
+	 * and more importantly they are created internaly by driver,
+	 * see mlx5 create_dev_resources() as an example.
+	 */
+	if (attr->qp_type < IB_QPT_XRC_INI) {
+		qp->res.type = RDMA_RESTRACK_QP;
+		rdma_restrack_add(&qp->res);
+	} else
+		qp->res.valid = false;
+
+	return qp;
+}
 #endif /* _CORE_PRIV_H */
diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index f2ae75fa3128..af5ad6a56ae4 100644
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -17,6 +17,7 @@
 
 /* # of WCs to poll for with a single call to ib_poll_cq */
 #define IB_POLL_BATCH			16
+#define IB_POLL_BATCH_DIRECT		8
 
 /* # of WCs to iterate over before yielding */
 #define IB_POLL_BUDGET_IRQ		256
@@ -25,7 +26,8 @@
 #define IB_POLL_FLAGS \
 	(IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
 
-static int __ib_process_cq(struct ib_cq *cq, int budget)
+static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,
+			   int batch)
 {
 	int i, n, completed = 0;
 
@@ -34,10 +36,10 @@ static int __ib_process_cq(struct ib_cq *cq, int budget)
 	 * want to bound this call, thus we need unsigned
 	 * minimum here.
 	 */
-	while ((n = ib_poll_cq(cq, min_t(u32, IB_POLL_BATCH,
-			budget - completed), cq->wc)) > 0) {
+	while ((n = ib_poll_cq(cq, min_t(u32, batch,
+					 budget - completed), wcs)) > 0) {
 		for (i = 0; i < n; i++) {
-			struct ib_wc *wc = &cq->wc[i];
+			struct ib_wc *wc = &wcs[i];
 
 			if (wc->wr_cqe)
 				wc->wr_cqe->done(cq, wc);
@@ -47,8 +49,7 @@ static int __ib_process_cq(struct ib_cq *cq, int budget)
 
 		completed += n;
 
-		if (n != IB_POLL_BATCH ||
-		    (budget != -1 && completed >= budget))
+		if (n != batch || (budget != -1 && completed >= budget))
 			break;
 	}
 
@@ -60,18 +61,20 @@ static int __ib_process_cq(struct ib_cq *cq, int budget)
  * @cq:		CQ to process
  * @budget:	number of CQEs to poll for
  *
- * This function is used to process all outstanding CQ entries on a
- * %IB_POLL_DIRECT CQ.  It does not offload CQ processing to a different
- * context and does not ask for completion interrupts from the HCA.
+ * This function is used to process all outstanding CQ entries.
+ * It does not offload CQ processing to a different context and does
+ * not ask for completion interrupts from the HCA.
+ * Using direct processing on CQ with non IB_POLL_DIRECT type may trigger
+ * concurrent processing.
  *
  * Note: do not pass -1 as %budget unless it is guaranteed that the number
  * of completions that will be processed is small.
  */
 int ib_process_cq_direct(struct ib_cq *cq, int budget)
 {
-	WARN_ON_ONCE(cq->poll_ctx != IB_POLL_DIRECT);
+	struct ib_wc wcs[IB_POLL_BATCH_DIRECT];
 
-	return __ib_process_cq(cq, budget);
+	return __ib_process_cq(cq, budget, wcs, IB_POLL_BATCH_DIRECT);
 }
 EXPORT_SYMBOL(ib_process_cq_direct);
 
@@ -85,7 +88,7 @@ static int ib_poll_handler(struct irq_poll *iop, int budget)
 	struct ib_cq *cq = container_of(iop, struct ib_cq, iop);
 	int completed;
 
-	completed = __ib_process_cq(cq, budget);
+	completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
 	if (completed < budget) {
 		irq_poll_complete(&cq->iop);
 		if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
@@ -105,7 +108,8 @@ static void ib_cq_poll_work(struct work_struct *work)
 	struct ib_cq *cq = container_of(work, struct ib_cq, work);
 	int completed;
 
-	completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE);
+	completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, cq->wc,
+				    IB_POLL_BATCH);
 	if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
 	    ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
 		queue_work(ib_comp_wq, &cq->work);
@@ -117,20 +121,22 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
 }
 
 /**
- * ib_alloc_cq - allocate a completion queue
+ * __ib_alloc_cq - allocate a completion queue
  * @dev:		device to allocate the CQ for
  * @private:		driver private data, accessible from cq->cq_context
  * @nr_cqe:		number of CQEs to allocate
  * @comp_vector:	HCA completion vectors for this CQ
  * @poll_ctx:		context to poll the CQ from.
+ * @caller:		module owner name.
  *
  * This is the proper interface to allocate a CQ for in-kernel users. A
  * CQ allocated with this interface will automatically be polled from the
  * specified context. The ULP must use wr->wr_cqe instead of wr->wr_id
  * to use this CQ abstraction.
  */
-struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
-		int nr_cqe, int comp_vector, enum ib_poll_context poll_ctx)
+struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
+			    int nr_cqe, int comp_vector,
+			    enum ib_poll_context poll_ctx, const char *caller)
 {
 	struct ib_cq_init_attr cq_attr = {
 		.cqe		= nr_cqe,
@@ -154,6 +160,10 @@ struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
 	if (!cq->wc)
 		goto out_destroy_cq;
 
+	cq->res.type = RDMA_RESTRACK_CQ;
+	cq->res.kern_name = caller;
+	rdma_restrack_add(&cq->res);
+
 	switch (cq->poll_ctx) {
 	case IB_POLL_DIRECT:
 		cq->comp_handler = ib_cq_completion_direct;
@@ -178,11 +188,12 @@ struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
 
 out_free_wc:
 	kfree(cq->wc);
+	rdma_restrack_del(&cq->res);
 out_destroy_cq:
 	cq->device->destroy_cq(cq);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL(ib_alloc_cq);
+EXPORT_SYMBOL(__ib_alloc_cq);
 
 /**
  * ib_free_cq - free a completion queue
@@ -209,6 +220,7 @@ void ib_free_cq(struct ib_cq *cq)
 	}
 
 	kfree(cq->wc);
+	rdma_restrack_del(&cq->res);
 	ret = cq->device->destroy_cq(cq);
 	WARN_ON_ONCE(ret);
 }
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 465520627e4b..b7459cf524e4 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -263,6 +263,8 @@ struct ib_device *ib_alloc_device(size_t size)
 	if (!device)
 		return NULL;
 
+	rdma_restrack_init(&device->res);
+
 	device->dev.class = &ib_class;
 	device_initialize(&device->dev);
 
@@ -288,7 +290,8 @@ void ib_dealloc_device(struct ib_device *device)
 {
 	WARN_ON(device->reg_state != IB_DEV_UNREGISTERED &&
 		device->reg_state != IB_DEV_UNINITIALIZED);
-	kobject_put(&device->dev.kobj);
+	rdma_restrack_clean(&device->res);
+	put_device(&device->dev);
 }
 EXPORT_SYMBOL(ib_dealloc_device);
 
@@ -462,7 +465,6 @@ int ib_register_device(struct ib_device *device,
 	struct ib_udata uhw = {.outlen = 0, .inlen = 0};
 	struct device *parent = device->dev.parent;
 
-	WARN_ON_ONCE(!parent);
 	WARN_ON_ONCE(device->dma_device);
 	if (device->dev.dma_ops) {
 		/*
@@ -471,16 +473,25 @@ int ib_register_device(struct ib_device *device,
 		 * into device->dev.
 		 */
 		device->dma_device = &device->dev;
-		if (!device->dev.dma_mask)
-			device->dev.dma_mask = parent->dma_mask;
-		if (!device->dev.coherent_dma_mask)
-			device->dev.coherent_dma_mask =
-				parent->coherent_dma_mask;
+		if (!device->dev.dma_mask) {
+			if (parent)
+				device->dev.dma_mask = parent->dma_mask;
+			else
+				WARN_ON_ONCE(true);
+		}
+		if (!device->dev.coherent_dma_mask) {
+			if (parent)
+				device->dev.coherent_dma_mask =
+					parent->coherent_dma_mask;
+			else
+				WARN_ON_ONCE(true);
+		}
 	} else {
 		/*
 		 * The caller did not provide custom DMA operations. Use the
 		 * DMA mapping operations of the parent device.
 		 */
+		WARN_ON_ONCE(!parent);
 		device->dma_device = parent;
 	}
 
@@ -526,14 +537,14 @@ int ib_register_device(struct ib_device *device,
 	ret = device->query_device(device, &device->attrs, &uhw);
 	if (ret) {
 		pr_warn("Couldn't query the device attributes\n");
-		goto cache_cleanup;
+		goto cg_cleanup;
 	}
 
 	ret = ib_device_register_sysfs(device, port_callback);
 	if (ret) {
 		pr_warn("Couldn't register device %s with driver model\n",
 			device->name);
-		goto cache_cleanup;
+		goto cg_cleanup;
 	}
 
 	device->reg_state = IB_DEV_REGISTERED;
@@ -549,6 +560,8 @@ int ib_register_device(struct ib_device *device,
 	mutex_unlock(&device_mutex);
 	return 0;
 
+cg_cleanup:
+	ib_device_unregister_rdmacg(device);
 cache_cleanup:
 	ib_cache_cleanup_one(device);
 	ib_cache_release_one(device);
@@ -1033,32 +1046,22 @@ EXPORT_SYMBOL(ib_modify_port);
 
 /**
  * ib_find_gid - Returns the port number and GID table index where
- *   a specified GID value occurs.
+ *   a specified GID value occurs. Its searches only for IB link layer.
  * @device: The device to query.
  * @gid: The GID value to search for.
- * @gid_type: Type of GID.
  * @ndev: The ndev related to the GID to search for.
  * @port_num: The port number of the device where the GID value was found.
  * @index: The index into the GID table where the GID was found.  This
  *   parameter may be NULL.
  */
 int ib_find_gid(struct ib_device *device, union ib_gid *gid,
-		enum ib_gid_type gid_type, struct net_device *ndev,
-		u8 *port_num, u16 *index)
+		struct net_device *ndev, u8 *port_num, u16 *index)
 {
 	union ib_gid tmp_gid;
 	int ret, port, i;
 
 	for (port = rdma_start_port(device); port <= rdma_end_port(device); ++port) {
-		if (rdma_cap_roce_gid_table(device, port)) {
-			if (!ib_find_cached_gid_by_port(device, gid, gid_type, port,
-							ndev, index)) {
-				*port_num = port;
-				return 0;
-			}
-		}
-
-		if (gid_type != IB_GID_TYPE_IB)
+		if (rdma_cap_roce_gid_table(device, port))
 			continue;
 
 		for (i = 0; i < device->port_immutable[port].gid_tbl_len; ++i) {
diff --git a/drivers/infiniband/core/fmr_pool.c b/drivers/infiniband/core/fmr_pool.c
index 84d2615b5d4b..a0a9ed719031 100644
--- a/drivers/infiniband/core/fmr_pool.c
+++ b/drivers/infiniband/core/fmr_pool.c
@@ -388,13 +388,11 @@ int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
 EXPORT_SYMBOL(ib_flush_fmr_pool);
 
 /**
- * ib_fmr_pool_map_phys -
- * @pool:FMR pool to allocate FMR from
- * @page_list:List of pages to map
- * @list_len:Number of pages in @page_list
- * @io_virtual_address:I/O virtual address for new FMR
- *
- * Map an FMR from an FMR pool.
+ * ib_fmr_pool_map_phys - Map an FMR from an FMR pool.
+ * @pool_handle: FMR pool to allocate FMR from
+ * @page_list: List of pages to map
+ * @list_len: Number of pages in @page_list
+ * @io_virtual_address: I/O virtual address for new FMR
  */
 struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
 					 u64                *page_list,
diff --git a/drivers/infiniband/core/iwpm_util.c b/drivers/infiniband/core/iwpm_util.c
index 3c4faadb8cdd..81528f64061a 100644
--- a/drivers/infiniband/core/iwpm_util.c
+++ b/drivers/infiniband/core/iwpm_util.c
@@ -654,6 +654,7 @@ int iwpm_send_mapinfo(u8 nl_client, int iwpm_pid)
 	}
 	skb_num++;
 	spin_lock_irqsave(&iwpm_mapinfo_lock, flags);
+	ret = -EINVAL;
 	for (i = 0; i < IWPM_MAPINFO_HASH_SIZE; i++) {
 		hlist_for_each_entry(map_info, &iwpm_hash_bucket[i],
 				     hlist_node) {
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index cb91245e9163..c50596f7f98a 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -49,7 +49,6 @@
 #include "smi.h"
 #include "opa_smi.h"
 #include "agent.h"
-#include "core_priv.h"
 
 static int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
 static int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
diff --git a/drivers/infiniband/core/netlink.c b/drivers/infiniband/core/netlink.c
index 1fb72c356e36..3ccaae18ad75 100644
--- a/drivers/infiniband/core/netlink.c
+++ b/drivers/infiniband/core/netlink.c
@@ -41,8 +41,6 @@
 #include <linux/module.h>
 #include "core_priv.h"
 
-#include "core_priv.h"
-
 static DEFINE_MUTEX(rdma_nl_mutex);
 static struct sock *nls;
 static struct {
@@ -83,15 +81,13 @@ static bool is_nl_valid(unsigned int type, unsigned int op)
 	if (!is_nl_msg_valid(type, op))
 		return false;
 
-	cb_table = rdma_nl_types[type].cb_table;
-#ifdef CONFIG_MODULES
-	if (!cb_table) {
+	if (!rdma_nl_types[type].cb_table) {
 		mutex_unlock(&rdma_nl_mutex);
 		request_module("rdma-netlink-subsys-%d", type);
 		mutex_lock(&rdma_nl_mutex);
-		cb_table = rdma_nl_types[type].cb_table;
 	}
-#endif
+
+	cb_table = rdma_nl_types[type].cb_table;
 
 	if (!cb_table || (!cb_table[op].dump && !cb_table[op].doit))
 		return false;
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 0dcd1aa6f683..5326a684555f 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -31,6 +31,8 @@
  */
 
 #include <linux/module.h>
+#include <linux/pid.h>
+#include <linux/pid_namespace.h>
 #include <net/netlink.h>
 #include <rdma/rdma_netlink.h>
 
@@ -52,16 +54,42 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
 	[RDMA_NLDEV_ATTR_PORT_STATE]	= { .type = NLA_U8 },
 	[RDMA_NLDEV_ATTR_PORT_PHYS_STATE] = { .type = NLA_U8 },
 	[RDMA_NLDEV_ATTR_DEV_NODE_TYPE] = { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_SUMMARY]	= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY]	= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME] = { .type = NLA_NUL_STRING,
+					     .len = 16 },
+	[RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR] = { .type = NLA_U64 },
+	[RDMA_NLDEV_ATTR_RES_QP]		= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_QP_ENTRY]		= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_LQPN]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_RQPN]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_RQ_PSN]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_SQ_PSN]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE] = { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_TYPE]		= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_STATE]		= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_PID]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_KERN_NAME]		= { .type = NLA_NUL_STRING,
+						    .len = TASK_COMM_LEN },
 };
 
-static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
+static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
 {
-	char fw[IB_FW_VERSION_NAME_MAX];
-
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
 		return -EMSGSIZE;
 	if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME, device->name))
 		return -EMSGSIZE;
+
+	return 0;
+}
+
+static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
+{
+	char fw[IB_FW_VERSION_NAME_MAX];
+
+	if (fill_nldev_handle(msg, device))
+		return -EMSGSIZE;
+
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, rdma_end_port(device)))
 		return -EMSGSIZE;
 
@@ -92,10 +120,9 @@ static int fill_port_info(struct sk_buff *msg,
 	struct ib_port_attr attr;
 	int ret;
 
-	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_DEV_INDEX, device->index))
-		return -EMSGSIZE;
-	if (nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_NAME, device->name))
+	if (fill_nldev_handle(msg, device))
 		return -EMSGSIZE;
+
 	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, port))
 		return -EMSGSIZE;
 
@@ -126,6 +153,137 @@ static int fill_port_info(struct sk_buff *msg,
 	return 0;
 }
 
+static int fill_res_info_entry(struct sk_buff *msg,
+			       const char *name, u64 curr)
+{
+	struct nlattr *entry_attr;
+
+	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY);
+	if (!entry_attr)
+		return -EMSGSIZE;
+
+	if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_NAME, name))
+		goto err;
+	if (nla_put_u64_64bit(msg,
+			      RDMA_NLDEV_ATTR_RES_SUMMARY_ENTRY_CURR, curr, 0))
+		goto err;
+
+	nla_nest_end(msg, entry_attr);
+	return 0;
+
+err:
+	nla_nest_cancel(msg, entry_attr);
+	return -EMSGSIZE;
+}
+
+static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
+{
+	static const char * const names[RDMA_RESTRACK_MAX] = {
+		[RDMA_RESTRACK_PD] = "pd",
+		[RDMA_RESTRACK_CQ] = "cq",
+		[RDMA_RESTRACK_QP] = "qp",
+	};
+
+	struct rdma_restrack_root *res = &device->res;
+	struct nlattr *table_attr;
+	int ret, i, curr;
+
+	if (fill_nldev_handle(msg, device))
+		return -EMSGSIZE;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_SUMMARY);
+	if (!table_attr)
+		return -EMSGSIZE;
+
+	for (i = 0; i < RDMA_RESTRACK_MAX; i++) {
+		if (!names[i])
+			continue;
+		curr = rdma_restrack_count(res, i, task_active_pid_ns(current));
+		ret = fill_res_info_entry(msg, names[i], curr);
+		if (ret)
+			goto err;
+	}
+
+	nla_nest_end(msg, table_attr);
+	return 0;
+
+err:
+	nla_nest_cancel(msg, table_attr);
+	return ret;
+}
+
+static int fill_res_qp_entry(struct sk_buff *msg,
+			     struct ib_qp *qp, uint32_t port)
+{
+	struct rdma_restrack_entry *res = &qp->res;
+	struct ib_qp_init_attr qp_init_attr;
+	struct nlattr *entry_attr;
+	struct ib_qp_attr qp_attr;
+	int ret;
+
+	ret = ib_query_qp(qp, &qp_attr, 0, &qp_init_attr);
+	if (ret)
+		return ret;
+
+	if (port && port != qp_attr.port_num)
+		return 0;
+
+	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_QP_ENTRY);
+	if (!entry_attr)
+		goto out;
+
+	/* In create_qp() port is not set yet */
+	if (qp_attr.port_num &&
+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, qp_attr.port_num))
+		goto err;
+
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, qp->qp_num))
+		goto err;
+	if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC) {
+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQPN,
+				qp_attr.dest_qp_num))
+			goto err;
+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RQ_PSN,
+				qp_attr.rq_psn))
+			goto err;
+	}
+
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_SQ_PSN, qp_attr.sq_psn))
+		goto err;
+
+	if (qp->qp_type == IB_QPT_RC || qp->qp_type == IB_QPT_UC ||
+	    qp->qp_type == IB_QPT_XRC_INI || qp->qp_type == IB_QPT_XRC_TGT) {
+		if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_PATH_MIG_STATE,
+			       qp_attr.path_mig_state))
+			goto err;
+	}
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, qp->qp_type))
+		goto err;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, qp_attr.qp_state))
+		goto err;
+
+	/*
+	 * Existence of task means that it is user QP and netlink
+	 * user is invited to go and read /proc/PID/comm to get name
+	 * of the task file and res->task_com should be NULL.
+	 */
+	if (rdma_is_kernel_res(res)) {
+		if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME, res->kern_name))
+			goto err;
+	} else {
+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, task_pid_vnr(res->task)))
+			goto err;
+	}
+
+	nla_nest_end(msg, entry_attr);
+	return 0;
+
+err:
+	nla_nest_cancel(msg, entry_attr);
+out:
+	return -EMSGSIZE;
+}
+
 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 			  struct netlink_ext_ack *extack)
 {
@@ -321,6 +479,215 @@ static int nldev_port_get_dumpit(struct sk_buff *skb,
 	return skb->len;
 }
 
+static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+			      struct netlink_ext_ack *extack)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct ib_device *device;
+	struct sk_buff *msg;
+	u32 index;
+	int ret;
+
+	ret = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, extack);
+	if (ret || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+		return -EINVAL;
+
+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+	device = ib_device_get_by_index(index);
+	if (!device)
+		return -EINVAL;
+
+	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!msg) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	nlh = nlmsg_put(msg, NETLINK_CB(skb).portid, nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
+			0, 0);
+
+	ret = fill_res_info(msg, device);
+	if (ret)
+		goto err_free;
+
+	nlmsg_end(msg, nlh);
+	put_device(&device->dev);
+	return rdma_nl_unicast(msg, NETLINK_CB(skb).portid);
+
+err_free:
+	nlmsg_free(msg);
+err:
+	put_device(&device->dev);
+	return ret;
+}
+
+static int _nldev_res_get_dumpit(struct ib_device *device,
+				 struct sk_buff *skb,
+				 struct netlink_callback *cb,
+				 unsigned int idx)
+{
+	int start = cb->args[0];
+	struct nlmsghdr *nlh;
+
+	if (idx < start)
+		return 0;
+
+	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_GET),
+			0, NLM_F_MULTI);
+
+	if (fill_res_info(skb, device)) {
+		nlmsg_cancel(skb, nlh);
+		goto out;
+	}
+
+	nlmsg_end(skb, nlh);
+
+	idx++;
+
+out:
+	cb->args[0] = idx;
+	return skb->len;
+}
+
+static int nldev_res_get_dumpit(struct sk_buff *skb,
+				struct netlink_callback *cb)
+{
+	return ib_enum_all_devs(_nldev_res_get_dumpit, skb, cb);
+}
+
+static int nldev_res_get_qp_dumpit(struct sk_buff *skb,
+				   struct netlink_callback *cb)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct rdma_restrack_entry *res;
+	int err, ret = 0, idx = 0;
+	struct nlattr *table_attr;
+	struct ib_device *device;
+	int start = cb->args[0];
+	struct ib_qp *qp = NULL;
+	struct nlmsghdr *nlh;
+	u32 index, port = 0;
+
+	err = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, NULL);
+	/*
+	 * Right now, we are expecting the device index to get QP information,
+	 * but it is possible to extend this code to return all devices in
+	 * one shot by checking the existence of RDMA_NLDEV_ATTR_DEV_INDEX.
+	 * if it doesn't exist, we will iterate over all devices.
+	 *
+	 * But it is not needed for now.
+	 */
+	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+		return -EINVAL;
+
+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+	device = ib_device_get_by_index(index);
+	if (!device)
+		return -EINVAL;
+
+	/*
+	 * If no PORT_INDEX is supplied, we will return all QPs from that device
+	 */
+	if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
+		port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
+		if (!rdma_is_port_valid(device, port)) {
+			ret = -EINVAL;
+			goto err_index;
+		}
+	}
+
+	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV, RDMA_NLDEV_CMD_RES_QP_GET),
+			0, NLM_F_MULTI);
+
+	if (fill_nldev_handle(skb, device)) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	table_attr = nla_nest_start(skb, RDMA_NLDEV_ATTR_RES_QP);
+	if (!table_attr) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	down_read(&device->res.rwsem);
+	hash_for_each_possible(device->res.hash, res, node, RDMA_RESTRACK_QP) {
+		if (idx < start)
+			goto next;
+
+		if ((rdma_is_kernel_res(res) &&
+		     task_active_pid_ns(current) != &init_pid_ns) ||
+		    (!rdma_is_kernel_res(res) &&
+		     task_active_pid_ns(current) != task_active_pid_ns(res->task)))
+			/*
+			 * 1. Kernel QPs should be visible in init namspace only
+			 * 2. Present only QPs visible in the current namespace
+			 */
+			goto next;
+
+		if (!rdma_restrack_get(res))
+			/*
+			 * Resource is under release now, but we are not
+			 * relesing lock now, so it will be released in
+			 * our next pass, once we will get ->next pointer.
+			 */
+			goto next;
+
+		qp = container_of(res, struct ib_qp, res);
+
+		up_read(&device->res.rwsem);
+		ret = fill_res_qp_entry(skb, qp, port);
+		down_read(&device->res.rwsem);
+		/*
+		 * Return resource back, but it won't be released till
+		 * the &device->res.rwsem will be released for write.
+		 */
+		rdma_restrack_put(res);
+
+		if (ret == -EMSGSIZE)
+			/*
+			 * There is a chance to optimize here.
+			 * It can be done by using list_prepare_entry
+			 * and list_for_each_entry_continue afterwards.
+			 */
+			break;
+		if (ret)
+			goto res_err;
+next:		idx++;
+	}
+	up_read(&device->res.rwsem);
+
+	nla_nest_end(skb, table_attr);
+	nlmsg_end(skb, nlh);
+	cb->args[0] = idx;
+
+	/*
+	 * No more QPs to fill, cancel the message and
+	 * return 0 to mark end of dumpit.
+	 */
+	if (!qp)
+		goto err;
+
+	put_device(&device->dev);
+	return skb->len;
+
+res_err:
+	nla_nest_cancel(skb, table_attr);
+	up_read(&device->res.rwsem);
+
+err:
+	nlmsg_cancel(skb, nlh);
+
+err_index:
+	put_device(&device->dev);
+	return ret;
+}
+
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 	[RDMA_NLDEV_CMD_GET] = {
 		.doit = nldev_get_doit,
@@ -330,6 +697,23 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 		.doit = nldev_port_get_doit,
 		.dump = nldev_port_get_dumpit,
 	},
+	[RDMA_NLDEV_CMD_RES_GET] = {
+		.doit = nldev_res_get_doit,
+		.dump = nldev_res_get_dumpit,
+	},
+	[RDMA_NLDEV_CMD_RES_QP_GET] = {
+		.dump = nldev_res_get_qp_dumpit,
+		/*
+		 * .doit is not implemented yet for two reasons:
+		 * 1. It is not needed yet.
+		 * 2. There is a need to provide identifier, while it is easy
+		 * for the QPs (device index + port index + LQPN), it is not
+		 * the case for the rest of resources (PD and CQ). Because it
+		 * is better to provide similar interface for all resources,
+		 * let's wait till we will have other resources implemented
+		 * too.
+		 */
+	},
 };
 
 void __init nldev_init(void)
diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c
index 85b5ee4defa4..d8eead5d106d 100644
--- a/drivers/infiniband/core/rdma_core.c
+++ b/drivers/infiniband/core/rdma_core.c
@@ -141,7 +141,12 @@ static struct ib_uobject *alloc_uobj(struct ib_ucontext *context,
 	 */
 	uobj->context = context;
 	uobj->type = type;
-	atomic_set(&uobj->usecnt, 0);
+	/*
+	 * Allocated objects start out as write locked to deny any other
+	 * syscalls from accessing them until they are committed. See
+	 * rdma_alloc_commit_uobject
+	 */
+	atomic_set(&uobj->usecnt, -1);
 	kref_init(&uobj->ref);
 
 	return uobj;
@@ -196,7 +201,15 @@ static struct ib_uobject *lookup_get_idr_uobject(const struct uverbs_obj_type *t
 		goto free;
 	}
 
-	uverbs_uobject_get(uobj);
+	/*
+	 * The idr_find is guaranteed to return a pointer to something that
+	 * isn't freed yet, or NULL, as the free after idr_remove goes through
+	 * kfree_rcu(). However the object may still have been released and
+	 * kfree() could be called at any time.
+	 */
+	if (!kref_get_unless_zero(&uobj->ref))
+		uobj = ERR_PTR(-ENOENT);
+
 free:
 	rcu_read_unlock();
 	return uobj;
@@ -399,13 +412,13 @@ static int __must_check remove_commit_fd_uobject(struct ib_uobject *uobj,
 	return ret;
 }
 
-static void lockdep_check(struct ib_uobject *uobj, bool exclusive)
+static void assert_uverbs_usecnt(struct ib_uobject *uobj, bool exclusive)
 {
 #ifdef CONFIG_LOCKDEP
 	if (exclusive)
-		WARN_ON(atomic_read(&uobj->usecnt) > 0);
+		WARN_ON(atomic_read(&uobj->usecnt) != -1);
 	else
-		WARN_ON(atomic_read(&uobj->usecnt) == -1);
+		WARN_ON(atomic_read(&uobj->usecnt) <= 0);
 #endif
 }
 
@@ -444,7 +457,7 @@ int __must_check rdma_remove_commit_uobject(struct ib_uobject *uobj)
 		WARN(true, "ib_uverbs: Cleanup is running while removing an uobject\n");
 		return 0;
 	}
-	lockdep_check(uobj, true);
+	assert_uverbs_usecnt(uobj, true);
 	ret = _rdma_remove_commit_uobject(uobj, RDMA_REMOVE_DESTROY);
 
 	up_read(&ucontext->cleanup_rwsem);
@@ -474,16 +487,17 @@ int rdma_explicit_destroy(struct ib_uobject *uobject)
 		WARN(true, "ib_uverbs: Cleanup is running while removing an uobject\n");
 		return 0;
 	}
-	lockdep_check(uobject, true);
+	assert_uverbs_usecnt(uobject, true);
 	ret = uobject->type->type_class->remove_commit(uobject,
 						       RDMA_REMOVE_DESTROY);
 	if (ret)
-		return ret;
+		goto out;
 
 	uobject->type = &null_obj_type;
 
+out:
 	up_read(&ucontext->cleanup_rwsem);
-	return 0;
+	return ret;
 }
 
 static void alloc_commit_idr_uobject(struct ib_uobject *uobj)
@@ -527,6 +541,10 @@ int rdma_alloc_commit_uobject(struct ib_uobject *uobj)
 		return ret;
 	}
 
+	/* matches atomic_set(-1) in alloc_uobj */
+	assert_uverbs_usecnt(uobj, true);
+	atomic_set(&uobj->usecnt, 0);
+
 	uobj->type->type_class->alloc_commit(uobj);
 	up_read(&uobj->context->cleanup_rwsem);
 
@@ -561,7 +579,7 @@ static void lookup_put_fd_uobject(struct ib_uobject *uobj, bool exclusive)
 
 void rdma_lookup_put_uobject(struct ib_uobject *uobj, bool exclusive)
 {
-	lockdep_check(uobj, exclusive);
+	assert_uverbs_usecnt(uobj, exclusive);
 	uobj->type->type_class->lookup_put(uobj, exclusive);
 	/*
 	 * In order to unlock an object, either decrease its usecnt for
diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
new file mode 100644
index 000000000000..3dbc4e4cca41
--- /dev/null
+++ b/drivers/infiniband/core/restrack.c
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */
+/*
+ * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
+ */
+
+#include <rdma/ib_verbs.h>
+#include <rdma/restrack.h>
+#include <linux/mutex.h>
+#include <linux/sched/task.h>
+#include <linux/pid_namespace.h>
+
+void rdma_restrack_init(struct rdma_restrack_root *res)
+{
+	init_rwsem(&res->rwsem);
+}
+
+void rdma_restrack_clean(struct rdma_restrack_root *res)
+{
+	WARN_ON_ONCE(!hash_empty(res->hash));
+}
+
+int rdma_restrack_count(struct rdma_restrack_root *res,
+			enum rdma_restrack_type type,
+			struct pid_namespace *ns)
+{
+	struct rdma_restrack_entry *e;
+	u32 cnt = 0;
+
+	down_read(&res->rwsem);
+	hash_for_each_possible(res->hash, e, node, type) {
+		if (ns == &init_pid_ns ||
+		    (!rdma_is_kernel_res(e) &&
+		     ns == task_active_pid_ns(e->task)))
+			cnt++;
+	}
+	up_read(&res->rwsem);
+	return cnt;
+}
+EXPORT_SYMBOL(rdma_restrack_count);
+
+static void set_kern_name(struct rdma_restrack_entry *res)
+{
+	enum rdma_restrack_type type = res->type;
+	struct ib_qp *qp;
+
+	if (type != RDMA_RESTRACK_QP)
+		/* PD and CQ types already have this name embedded in */
+		return;
+
+	qp = container_of(res, struct ib_qp, res);
+	if (!qp->pd) {
+		WARN_ONCE(true, "XRC QPs are not supported\n");
+		/* Survive, despite the programmer's error */
+		res->kern_name = " ";
+		return;
+	}
+
+	res->kern_name = qp->pd->res.kern_name;
+}
+
+static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
+{
+	enum rdma_restrack_type type = res->type;
+	struct ib_device *dev;
+	struct ib_pd *pd;
+	struct ib_cq *cq;
+	struct ib_qp *qp;
+
+	switch (type) {
+	case RDMA_RESTRACK_PD:
+		pd = container_of(res, struct ib_pd, res);
+		dev = pd->device;
+		break;
+	case RDMA_RESTRACK_CQ:
+		cq = container_of(res, struct ib_cq, res);
+		dev = cq->device;
+		break;
+	case RDMA_RESTRACK_QP:
+		qp = container_of(res, struct ib_qp, res);
+		dev = qp->device;
+		break;
+	default:
+		WARN_ONCE(true, "Wrong resource tracking type %u\n", type);
+		return NULL;
+	}
+
+	return dev;
+}
+
+static bool res_is_user(struct rdma_restrack_entry *res)
+{
+	switch (res->type) {
+	case RDMA_RESTRACK_PD:
+		return container_of(res, struct ib_pd, res)->uobject;
+	case RDMA_RESTRACK_CQ:
+		return container_of(res, struct ib_cq, res)->uobject;
+	case RDMA_RESTRACK_QP:
+		return container_of(res, struct ib_qp, res)->uobject;
+	default:
+		WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
+		return false;
+	}
+}
+
+void rdma_restrack_add(struct rdma_restrack_entry *res)
+{
+	struct ib_device *dev = res_to_dev(res);
+
+	if (!dev)
+		return;
+
+	if (res_is_user(res)) {
+		get_task_struct(current);
+		res->task = current;
+		res->kern_name = NULL;
+	} else {
+		set_kern_name(res);
+		res->task = NULL;
+	}
+
+	kref_init(&res->kref);
+	init_completion(&res->comp);
+	res->valid = true;
+
+	down_write(&dev->res.rwsem);
+	hash_add(dev->res.hash, &res->node, res->type);
+	up_write(&dev->res.rwsem);
+}
+EXPORT_SYMBOL(rdma_restrack_add);
+
+int __must_check rdma_restrack_get(struct rdma_restrack_entry *res)
+{
+	return kref_get_unless_zero(&res->kref);
+}
+EXPORT_SYMBOL(rdma_restrack_get);
+
+static void restrack_release(struct kref *kref)
+{
+	struct rdma_restrack_entry *res;
+
+	res = container_of(kref, struct rdma_restrack_entry, kref);
+	complete(&res->comp);
+}
+
+int rdma_restrack_put(struct rdma_restrack_entry *res)
+{
+	return kref_put(&res->kref, restrack_release);
+}
+EXPORT_SYMBOL(rdma_restrack_put);
+
+void rdma_restrack_del(struct rdma_restrack_entry *res)
+{
+	struct ib_device *dev;
+
+	if (!res->valid)
+		return;
+
+	dev = res_to_dev(res);
+	if (!dev)
+		return;
+
+	rdma_restrack_put(res);
+
+	wait_for_completion(&res->comp);
+
+	down_write(&dev->res.rwsem);
+	hash_del(&res->node);
+	res->valid = false;
+	if (res->task)
+		put_task_struct(res->task);
+	up_write(&dev->res.rwsem);
+}
+EXPORT_SYMBOL(rdma_restrack_del);
diff --git a/drivers/infiniband/core/roce_gid_mgmt.c b/drivers/infiniband/core/roce_gid_mgmt.c
index 90e3889b7fbe..5a52ec77940a 100644
--- a/drivers/infiniband/core/roce_gid_mgmt.c
+++ b/drivers/infiniband/core/roce_gid_mgmt.c
@@ -410,15 +410,18 @@ static void enum_all_gids_of_dev_cb(struct ib_device *ib_dev,
 	rtnl_unlock();
 }
 
-/* This function will rescan all of the network devices in the system
- * and add their gids, as needed, to the relevant RoCE devices. */
-int roce_rescan_device(struct ib_device *ib_dev)
+/**
+ * rdma_roce_rescan_device - Rescan all of the network devices in the system
+ * and add their gids, as needed, to the relevant RoCE devices.
+ *
+ * @device:         the rdma device
+ */
+void rdma_roce_rescan_device(struct ib_device *ib_dev)
 {
 	ib_enum_roce_netdev(ib_dev, pass_all_filter, NULL,
 			    enum_all_gids_of_dev_cb, NULL);
-
-	return 0;
 }
+EXPORT_SYMBOL(rdma_roce_rescan_device);
 
 static void callback_for_addr_gid_device_scan(struct ib_device *device,
 					      u8 port,
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index ab5e1024fea9..9f029a1ca5ea 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -1227,9 +1227,9 @@ static u8 get_src_path_mask(struct ib_device *device, u8 port_num)
 	return src_path_mask;
 }
 
-int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
-			 struct sa_path_rec *rec,
-			 struct rdma_ah_attr *ah_attr)
+int ib_init_ah_attr_from_path(struct ib_device *device, u8 port_num,
+			      struct sa_path_rec *rec,
+			      struct rdma_ah_attr *ah_attr)
 {
 	int ret;
 	u16 gid_index;
@@ -1291,10 +1291,9 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 
 		resolved_dev = dev_get_by_index(dev_addr.net,
 						dev_addr.bound_dev_if);
-		if (resolved_dev->flags & IFF_LOOPBACK) {
-			dev_put(resolved_dev);
-			resolved_dev = idev;
-			dev_hold(resolved_dev);
+		if (!resolved_dev) {
+			dev_put(idev);
+			return -ENODEV;
 		}
 		ndev = ib_get_ndev_from_path(rec);
 		rcu_read_lock();
@@ -1341,10 +1340,11 @@ int ib_init_ah_from_path(struct ib_device *device, u8 port_num,
 
 	return 0;
 }
-EXPORT_SYMBOL(ib_init_ah_from_path);
+EXPORT_SYMBOL(ib_init_ah_attr_from_path);
 
 static int alloc_mad(struct ib_sa_query *query, gfp_t gfp_mask)
 {
+	struct rdma_ah_attr ah_attr;
 	unsigned long flags;
 
 	spin_lock_irqsave(&query->port->ah_lock, flags);
@@ -1356,6 +1356,15 @@ static int alloc_mad(struct ib_sa_query *query, gfp_t gfp_mask)
 	query->sm_ah = query->port->sm_ah;
 	spin_unlock_irqrestore(&query->port->ah_lock, flags);
 
+	/*
+	 * Always check if sm_ah has valid dlid assigned,
+	 * before querying for class port info
+	 */
+	if ((rdma_query_ah(query->sm_ah->ah, &ah_attr) < 0) ||
+	    !rdma_is_valid_unicast_lid(&ah_attr)) {
+		kref_put(&query->sm_ah->ref, free_sm_ah);
+		return -EAGAIN;
+	}
 	query->mad_buf = ib_create_send_mad(query->port->agent, 1,
 					    query->sm_ah->pkey_index,
 					    0, IB_MGMT_SA_HDR, IB_MGMT_SA_DATA,
diff --git a/drivers/infiniband/core/security.c b/drivers/infiniband/core/security.c
index 59b2f96d986a..b61dda6b04fc 100644
--- a/drivers/infiniband/core/security.c
+++ b/drivers/infiniband/core/security.c
@@ -653,12 +653,11 @@ int ib_security_modify_qp(struct ib_qp *qp,
 	}
 	return ret;
 }
-EXPORT_SYMBOL(ib_security_modify_qp);
 
-int ib_security_pkey_access(struct ib_device *dev,
-			    u8 port_num,
-			    u16 pkey_index,
-			    void *sec)
+static int ib_security_pkey_access(struct ib_device *dev,
+				   u8 port_num,
+				   u16 pkey_index,
+				   void *sec)
 {
 	u64 subnet_prefix;
 	u16 pkey;
@@ -678,7 +677,6 @@ int ib_security_pkey_access(struct ib_device *dev,
 
 	return security_ib_pkey_access(sec, subnet_prefix, pkey);
 }
-EXPORT_SYMBOL(ib_security_pkey_access);
 
 static int ib_mad_agent_security_change(struct notifier_block *nb,
 					unsigned long event,
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index e30d86fa1855..8ae1308eecc7 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -1276,7 +1276,6 @@ int ib_device_register_sysfs(struct ib_device *device,
 	int ret;
 	int i;
 
-	WARN_ON_ONCE(!device->dev.parent);
 	ret = dev_set_name(class_dev, "%s", device->name);
 	if (ret)
 		return ret;
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index f2a7f62c2834..01702265c1e1 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -53,6 +53,8 @@
 #include <rdma/ib_user_cm.h>
 #include <rdma/ib_marshall.h>
 
+#include "core_priv.h"
+
 MODULE_AUTHOR("Libor Michalek");
 MODULE_DESCRIPTION("InfiniBand userspace Connection Manager access");
 MODULE_LICENSE("Dual BSD/GPL");
@@ -104,10 +106,13 @@ struct ib_ucm_event {
 enum {
 	IB_UCM_MAJOR = 231,
 	IB_UCM_BASE_MINOR = 224,
-	IB_UCM_MAX_DEVICES = 32
+	IB_UCM_MAX_DEVICES = RDMA_MAX_PORTS,
+	IB_UCM_NUM_FIXED_MINOR = 32,
+	IB_UCM_NUM_DYNAMIC_MINOR = IB_UCM_MAX_DEVICES - IB_UCM_NUM_FIXED_MINOR,
 };
 
 #define IB_UCM_BASE_DEV MKDEV(IB_UCM_MAJOR, IB_UCM_BASE_MINOR)
+static dev_t dynamic_ucm_dev;
 
 static void ib_ucm_add_one(struct ib_device *device);
 static void ib_ucm_remove_one(struct ib_device *device, void *client_data);
@@ -1130,16 +1135,16 @@ static ssize_t ib_ucm_write(struct file *filp, const char __user *buf,
 	return result;
 }
 
-static unsigned int ib_ucm_poll(struct file *filp,
+static __poll_t ib_ucm_poll(struct file *filp,
 				struct poll_table_struct *wait)
 {
 	struct ib_ucm_file *file = filp->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(filp, &file->poll_wait, wait);
 
 	if (!list_empty(&file->events))
-		mask = POLLIN | POLLRDNORM;
+		mask = EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
@@ -1199,7 +1204,6 @@ static int ib_ucm_close(struct inode *inode, struct file *filp)
 	return 0;
 }
 
-static DECLARE_BITMAP(overflow_map, IB_UCM_MAX_DEVICES);
 static void ib_ucm_release_dev(struct device *dev)
 {
 	struct ib_ucm_device *ucm_dev;
@@ -1210,10 +1214,7 @@ static void ib_ucm_release_dev(struct device *dev)
 
 static void ib_ucm_free_dev(struct ib_ucm_device *ucm_dev)
 {
-	if (ucm_dev->devnum < IB_UCM_MAX_DEVICES)
-		clear_bit(ucm_dev->devnum, dev_map);
-	else
-		clear_bit(ucm_dev->devnum - IB_UCM_MAX_DEVICES, overflow_map);
+	clear_bit(ucm_dev->devnum, dev_map);
 }
 
 static const struct file_operations ucm_fops = {
@@ -1235,27 +1236,6 @@ static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
 }
 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
 
-static dev_t overflow_maj;
-static int find_overflow_devnum(void)
-{
-	int ret;
-
-	if (!overflow_maj) {
-		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UCM_MAX_DEVICES,
-					  "infiniband_cm");
-		if (ret) {
-			pr_err("ucm: couldn't register dynamic device number\n");
-			return ret;
-		}
-	}
-
-	ret = find_first_zero_bit(overflow_map, IB_UCM_MAX_DEVICES);
-	if (ret >= IB_UCM_MAX_DEVICES)
-		return -1;
-
-	return ret;
-}
-
 static void ib_ucm_add_one(struct ib_device *device)
 {
 	int devnum;
@@ -1274,19 +1254,14 @@ static void ib_ucm_add_one(struct ib_device *device)
 	ucm_dev->dev.release = ib_ucm_release_dev;
 
 	devnum = find_first_zero_bit(dev_map, IB_UCM_MAX_DEVICES);
-	if (devnum >= IB_UCM_MAX_DEVICES) {
-		devnum = find_overflow_devnum();
-		if (devnum < 0)
-			goto err;
-
-		ucm_dev->devnum = devnum + IB_UCM_MAX_DEVICES;
-		base = devnum + overflow_maj;
-		set_bit(devnum, overflow_map);
-	} else {
-		ucm_dev->devnum = devnum;
-		base = devnum + IB_UCM_BASE_DEV;
-		set_bit(devnum, dev_map);
-	}
+	if (devnum >= IB_UCM_MAX_DEVICES)
+		goto err;
+	ucm_dev->devnum = devnum;
+	set_bit(devnum, dev_map);
+	if (devnum >= IB_UCM_NUM_FIXED_MINOR)
+		base = dynamic_ucm_dev + devnum - IB_UCM_NUM_FIXED_MINOR;
+	else
+		base = IB_UCM_BASE_DEV + devnum;
 
 	cdev_init(&ucm_dev->cdev, &ucm_fops);
 	ucm_dev->cdev.owner = THIS_MODULE;
@@ -1334,13 +1309,20 @@ static int __init ib_ucm_init(void)
 {
 	int ret;
 
-	ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES,
+	ret = register_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR,
 				     "infiniband_cm");
 	if (ret) {
 		pr_err("ucm: couldn't register device number\n");
 		goto error1;
 	}
 
+	ret = alloc_chrdev_region(&dynamic_ucm_dev, 0, IB_UCM_NUM_DYNAMIC_MINOR,
+				  "infiniband_cm");
+	if (ret) {
+		pr_err("ucm: couldn't register dynamic device number\n");
+		goto err_alloc;
+	}
+
 	ret = class_create_file(&cm_class, &class_attr_abi_version.attr);
 	if (ret) {
 		pr_err("ucm: couldn't create abi_version attribute\n");
@@ -1357,7 +1339,9 @@ static int __init ib_ucm_init(void)
 error3:
 	class_remove_file(&cm_class, &class_attr_abi_version.attr);
 error2:
-	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES);
+	unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);
+err_alloc:
+	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);
 error1:
 	return ret;
 }
@@ -1366,9 +1350,8 @@ static void __exit ib_ucm_cleanup(void)
 {
 	ib_unregister_client(&ucm_client);
 	class_remove_file(&cm_class, &class_attr_abi_version.attr);
-	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_MAX_DEVICES);
-	if (overflow_maj)
-		unregister_chrdev_region(overflow_maj, IB_UCM_MAX_DEVICES);
+	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);
+	unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);
 	idr_destroy(&ctx_id_table);
 }
 
diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c
index eb85b546e223..d933336d7e01 100644
--- a/drivers/infiniband/core/ucma.c
+++ b/drivers/infiniband/core/ucma.c
@@ -132,7 +132,7 @@ static inline struct ucma_context *_ucma_find_context(int id,
 	ctx = idr_find(&ctx_idr, id);
 	if (!ctx)
 		ctx = ERR_PTR(-ENOENT);
-	else if (ctx->file != file)
+	else if (ctx->file != file || !ctx->cm_id)
 		ctx = ERR_PTR(-EINVAL);
 	return ctx;
 }
@@ -456,6 +456,7 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,
 	struct rdma_ucm_create_id cmd;
 	struct rdma_ucm_create_id_resp resp;
 	struct ucma_context *ctx;
+	struct rdma_cm_id *cm_id;
 	enum ib_qp_type qp_type;
 	int ret;
 
@@ -476,10 +477,10 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,
 		return -ENOMEM;
 
 	ctx->uid = cmd.uid;
-	ctx->cm_id = rdma_create_id(current->nsproxy->net_ns,
-				    ucma_event_handler, ctx, cmd.ps, qp_type);
-	if (IS_ERR(ctx->cm_id)) {
-		ret = PTR_ERR(ctx->cm_id);
+	cm_id = rdma_create_id(current->nsproxy->net_ns,
+			       ucma_event_handler, ctx, cmd.ps, qp_type);
+	if (IS_ERR(cm_id)) {
+		ret = PTR_ERR(cm_id);
 		goto err1;
 	}
 
@@ -489,14 +490,19 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,
 		ret = -EFAULT;
 		goto err2;
 	}
+
+	ctx->cm_id = cm_id;
 	return 0;
 
 err2:
-	rdma_destroy_id(ctx->cm_id);
+	rdma_destroy_id(cm_id);
 err1:
 	mutex_lock(&mut);
 	idr_remove(&ctx_idr, ctx->id);
 	mutex_unlock(&mut);
+	mutex_lock(&file->mut);
+	list_del(&ctx->list);
+	mutex_unlock(&file->mut);
 	kfree(ctx);
 	return ret;
 }
@@ -626,6 +632,9 @@ static ssize_t ucma_bind_ip(struct ucma_file *file, const char __user *inbuf,
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
+	if (!rdma_addr_size_in6(&cmd.addr))
+		return -EINVAL;
+
 	ctx = ucma_get_ctx(file, cmd.id);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
@@ -639,22 +648,21 @@ static ssize_t ucma_bind(struct ucma_file *file, const char __user *inbuf,
 			 int in_len, int out_len)
 {
 	struct rdma_ucm_bind cmd;
-	struct sockaddr *addr;
 	struct ucma_context *ctx;
 	int ret;
 
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
-	addr = (struct sockaddr *) &cmd.addr;
-	if (cmd.reserved || !cmd.addr_size || (cmd.addr_size != rdma_addr_size(addr)))
+	if (cmd.reserved || !cmd.addr_size ||
+	    cmd.addr_size != rdma_addr_size_kss(&cmd.addr))
 		return -EINVAL;
 
 	ctx = ucma_get_ctx(file, cmd.id);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
-	ret = rdma_bind_addr(ctx->cm_id, addr);
+	ret = rdma_bind_addr(ctx->cm_id, (struct sockaddr *) &cmd.addr);
 	ucma_put_ctx(ctx);
 	return ret;
 }
@@ -670,13 +678,16 @@ static ssize_t ucma_resolve_ip(struct ucma_file *file,
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
+	if (!rdma_addr_size_in6(&cmd.src_addr) ||
+	    !rdma_addr_size_in6(&cmd.dst_addr))
+		return -EINVAL;
+
 	ctx = ucma_get_ctx(file, cmd.id);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
 	ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr,
-				(struct sockaddr *) &cmd.dst_addr,
-				cmd.timeout_ms);
+				(struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms);
 	ucma_put_ctx(ctx);
 	return ret;
 }
@@ -686,24 +697,23 @@ static ssize_t ucma_resolve_addr(struct ucma_file *file,
 				 int in_len, int out_len)
 {
 	struct rdma_ucm_resolve_addr cmd;
-	struct sockaddr *src, *dst;
 	struct ucma_context *ctx;
 	int ret;
 
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
-	src = (struct sockaddr *) &cmd.src_addr;
-	dst = (struct sockaddr *) &cmd.dst_addr;
-	if (cmd.reserved || (cmd.src_size && (cmd.src_size != rdma_addr_size(src))) ||
-	    !cmd.dst_size || (cmd.dst_size != rdma_addr_size(dst)))
+	if (cmd.reserved ||
+	    (cmd.src_size && (cmd.src_size != rdma_addr_size_kss(&cmd.src_addr))) ||
+	    !cmd.dst_size || (cmd.dst_size != rdma_addr_size_kss(&cmd.dst_addr)))
 		return -EINVAL;
 
 	ctx = ucma_get_ctx(file, cmd.id);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
-	ret = rdma_resolve_addr(ctx->cm_id, src, dst, cmd.timeout_ms);
+	ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr,
+				(struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms);
 	ucma_put_ctx(ctx);
 	return ret;
 }
@@ -904,13 +914,14 @@ static ssize_t ucma_query_path(struct ucma_context *ctx,
 
 		resp->path_data[i].flags = IB_PATH_GMP | IB_PATH_PRIMARY |
 					   IB_PATH_BIDIRECTIONAL;
-		if (rec->rec_type == SA_PATH_REC_TYPE_IB) {
-			ib_sa_pack_path(rec, &resp->path_data[i].path_rec);
-		} else {
+		if (rec->rec_type == SA_PATH_REC_TYPE_OPA) {
 			struct sa_path_rec ib;
 
 			sa_convert_path_opa_to_ib(&ib, rec);
 			ib_sa_pack_path(&ib, &resp->path_data[i].path_rec);
+
+		} else {
+			ib_sa_pack_path(rec, &resp->path_data[i].path_rec);
 		}
 	}
 
@@ -943,8 +954,8 @@ static ssize_t ucma_query_gid(struct ucma_context *ctx,
 	} else {
 		addr->sib_family = AF_IB;
 		addr->sib_pkey = (__force __be16) resp.pkey;
-		rdma_addr_get_sgid(&ctx->cm_id->route.addr.dev_addr,
-				   (union ib_gid *) &addr->sib_addr);
+		rdma_read_gids(ctx->cm_id, (union ib_gid *)&addr->sib_addr,
+			       NULL);
 		addr->sib_sid = rdma_get_service_id(ctx->cm_id, (struct sockaddr *)
 						    &ctx->cm_id->route.addr.src_addr);
 	}
@@ -956,8 +967,8 @@ static ssize_t ucma_query_gid(struct ucma_context *ctx,
 	} else {
 		addr->sib_family = AF_IB;
 		addr->sib_pkey = (__force __be16) resp.pkey;
-		rdma_addr_get_dgid(&ctx->cm_id->route.addr.dev_addr,
-				   (union ib_gid *) &addr->sib_addr);
+		rdma_read_gids(ctx->cm_id, NULL,
+			       (union ib_gid *)&addr->sib_addr);
 		addr->sib_sid = rdma_get_service_id(ctx->cm_id, (struct sockaddr *)
 						    &ctx->cm_id->route.addr.dst_addr);
 	}
@@ -1148,10 +1159,18 @@ static ssize_t ucma_init_qp_attr(struct ucma_file *file,
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
+	if (cmd.qp_state > IB_QPS_ERR)
+		return -EINVAL;
+
 	ctx = ucma_get_ctx(file, cmd.id);
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
+	if (!ctx->cm_id->device) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	resp.qp_attr_mask = 0;
 	memset(&qp_attr, 0, sizeof qp_attr);
 	qp_attr.qp_state = cmd.qp_state;
@@ -1231,9 +1250,9 @@ static int ucma_set_ib_path(struct ucma_context *ctx,
 		struct sa_path_rec opa;
 
 		sa_convert_path_ib_to_opa(&opa, &sa_path);
-		ret = rdma_set_ib_paths(ctx->cm_id, &opa, 1);
+		ret = rdma_set_ib_path(ctx->cm_id, &opa);
 	} else {
-		ret = rdma_set_ib_paths(ctx->cm_id, &sa_path, 1);
+		ret = rdma_set_ib_path(ctx->cm_id, &sa_path);
 	}
 	if (ret)
 		return ret;
@@ -1293,6 +1312,9 @@ static ssize_t ucma_set_option(struct ucma_file *file, const char __user *inbuf,
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
+	if (unlikely(cmd.optlen > KMALLOC_MAX_SIZE))
+		return -EINVAL;
+
 	optval = memdup_user((void __user *) (unsigned long) cmd.optval,
 			     cmd.optlen);
 	if (IS_ERR(optval)) {
@@ -1314,7 +1336,7 @@ static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf,
 {
 	struct rdma_ucm_notify cmd;
 	struct ucma_context *ctx;
-	int ret;
+	int ret = -EINVAL;
 
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
@@ -1323,7 +1345,9 @@ static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf,
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
-	ret = rdma_notify(ctx->cm_id, (enum ib_event_type) cmd.event);
+	if (ctx->cm_id->device)
+		ret = rdma_notify(ctx->cm_id, (enum ib_event_type)cmd.event);
+
 	ucma_put_ctx(ctx);
 	return ret;
 }
@@ -1342,7 +1366,7 @@ static ssize_t ucma_process_join(struct ucma_file *file,
 		return -ENOSPC;
 
 	addr = (struct sockaddr *) &cmd->addr;
-	if (!cmd->addr_size || (cmd->addr_size != rdma_addr_size(addr)))
+	if (cmd->addr_size != rdma_addr_size(addr))
 		return -EINVAL;
 
 	if (cmd->join_flags == RDMA_MC_JOIN_FLAG_FULLMEMBER)
@@ -1409,7 +1433,10 @@ static ssize_t ucma_join_ip_multicast(struct ucma_file *file,
 	join_cmd.response = cmd.response;
 	join_cmd.uid = cmd.uid;
 	join_cmd.id = cmd.id;
-	join_cmd.addr_size = rdma_addr_size((struct sockaddr *) &cmd.addr);
+	join_cmd.addr_size = rdma_addr_size_in6(&cmd.addr);
+	if (!join_cmd.addr_size)
+		return -EINVAL;
+
 	join_cmd.join_flags = RDMA_MC_JOIN_FLAG_FULLMEMBER;
 	memcpy(&join_cmd.addr, &cmd.addr, join_cmd.addr_size);
 
@@ -1425,6 +1452,9 @@ static ssize_t ucma_join_multicast(struct ucma_file *file,
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
+	if (!rdma_addr_size_kss(&cmd.addr))
+		return -EINVAL;
+
 	return ucma_process_join(file, &cmd, out_len);
 }
 
@@ -1630,15 +1660,15 @@ static ssize_t ucma_write(struct file *filp, const char __user *buf,
 	return ret;
 }
 
-static unsigned int ucma_poll(struct file *filp, struct poll_table_struct *wait)
+static __poll_t ucma_poll(struct file *filp, struct poll_table_struct *wait)
 {
 	struct ucma_file *file = filp->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(filp, &file->poll_wait, wait);
 
 	if (!list_empty(&file->event_list))
-		mask = POLLIN | POLLRDNORM;
+		mask = EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index 130606c3b07c..9a4e899d94b3 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -352,7 +352,7 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
 		return -EINVAL;
 	}
 
-	ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->nmap, dst, length,
+	ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->npages, dst, length,
 				 offset + ib_umem_offset(umem));
 
 	if (ret < 0)
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 4b64dd02e090..bb98c9e4a7fd 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -55,16 +55,21 @@
 #include <rdma/ib_mad.h>
 #include <rdma/ib_user_mad.h>
 
+#include "core_priv.h"
+
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("InfiniBand userspace MAD packet access");
 MODULE_LICENSE("Dual BSD/GPL");
 
 enum {
-	IB_UMAD_MAX_PORTS  = 64,
+	IB_UMAD_MAX_PORTS  = RDMA_MAX_PORTS,
 	IB_UMAD_MAX_AGENTS = 32,
 
 	IB_UMAD_MAJOR      = 231,
-	IB_UMAD_MINOR_BASE = 0
+	IB_UMAD_MINOR_BASE = 0,
+	IB_UMAD_NUM_FIXED_MINOR = 64,
+	IB_UMAD_NUM_DYNAMIC_MINOR = IB_UMAD_MAX_PORTS - IB_UMAD_NUM_FIXED_MINOR,
+	IB_ISSM_MINOR_BASE        = IB_UMAD_NUM_FIXED_MINOR,
 };
 
 /*
@@ -127,9 +132,12 @@ struct ib_umad_packet {
 
 static struct class *umad_class;
 
-static const dev_t base_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE);
+static const dev_t base_umad_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE);
+static const dev_t base_issm_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE) +
+				   IB_UMAD_NUM_FIXED_MINOR;
+static dev_t dynamic_umad_dev;
+static dev_t dynamic_issm_dev;
 
-static DEFINE_SPINLOCK(port_lock);
 static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS);
 
 static void ib_umad_add_one(struct ib_device *device);
@@ -233,8 +241,7 @@ static void recv_handler(struct ib_mad_agent *agent,
 	 * On OPA devices it is okay to lose the upper 16 bits of LID as this
 	 * information is obtained elsewhere. Mask off the upper 16 bits.
 	 */
-	if (agent->device->port_immutable[agent->port_num].core_cap_flags &
-	    RDMA_CORE_PORT_INTEL_OPA)
+	if (rdma_cap_opa_mad(agent->device, agent->port_num))
 		packet->mad.hdr.lid = ib_lid_be16(0xFFFF &
 						  mad_recv_wc->wc->slid);
 	else
@@ -246,10 +253,14 @@ static void recv_handler(struct ib_mad_agent *agent,
 	if (packet->mad.hdr.grh_present) {
 		struct rdma_ah_attr ah_attr;
 		const struct ib_global_route *grh;
+		int ret;
 
-		ib_init_ah_from_wc(agent->device, agent->port_num,
-				   mad_recv_wc->wc, mad_recv_wc->recv_buf.grh,
-				   &ah_attr);
+		ret = ib_init_ah_attr_from_wc(agent->device, agent->port_num,
+					      mad_recv_wc->wc,
+					      mad_recv_wc->recv_buf.grh,
+					      &ah_attr);
+		if (ret)
+			goto err2;
 
 		grh = rdma_ah_read_grh(&ah_attr);
 		packet->mad.hdr.gid_index = grh->sgid_index;
@@ -500,7 +511,7 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 	}
 
 	memset(&ah_attr, 0, sizeof ah_attr);
-	ah_attr.type = rdma_ah_find_type(file->port->ib_dev,
+	ah_attr.type = rdma_ah_find_type(agent->device,
 					 file->port->port_num);
 	rdma_ah_set_dlid(&ah_attr, be16_to_cpu(packet->mad.hdr.lid));
 	rdma_ah_set_sl(&ah_attr, packet->mad.hdr.sl);
@@ -617,17 +628,17 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 	return ret;
 }
 
-static unsigned int ib_umad_poll(struct file *filp, struct poll_table_struct *wait)
+static __poll_t ib_umad_poll(struct file *filp, struct poll_table_struct *wait)
 {
 	struct ib_umad_file *file = filp->private_data;
 
 	/* we will always be able to post a MAD send */
-	unsigned int mask = POLLOUT | POLLWRNORM;
+	__poll_t mask = EPOLLOUT | EPOLLWRNORM;
 
 	poll_wait(filp, &file->recv_wait, wait);
 
 	if (!list_empty(&file->recv_list))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
@@ -1139,54 +1150,26 @@ static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
 static CLASS_ATTR_STRING(abi_version, S_IRUGO,
 			 __stringify(IB_USER_MAD_ABI_VERSION));
 
-static dev_t overflow_maj;
-static DECLARE_BITMAP(overflow_map, IB_UMAD_MAX_PORTS);
-static int find_overflow_devnum(struct ib_device *device)
-{
-	int ret;
-
-	if (!overflow_maj) {
-		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UMAD_MAX_PORTS * 2,
-					  "infiniband_mad");
-		if (ret) {
-			dev_err(&device->dev,
-				"couldn't register dynamic device number\n");
-			return ret;
-		}
-	}
-
-	ret = find_first_zero_bit(overflow_map, IB_UMAD_MAX_PORTS);
-	if (ret >= IB_UMAD_MAX_PORTS)
-		return -1;
-
-	return ret;
-}
-
 static int ib_umad_init_port(struct ib_device *device, int port_num,
 			     struct ib_umad_device *umad_dev,
 			     struct ib_umad_port *port)
 {
 	int devnum;
-	dev_t base;
+	dev_t base_umad;
+	dev_t base_issm;
 
-	spin_lock(&port_lock);
 	devnum = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
-	if (devnum >= IB_UMAD_MAX_PORTS) {
-		spin_unlock(&port_lock);
-		devnum = find_overflow_devnum(device);
-		if (devnum < 0)
-			return -1;
-
-		spin_lock(&port_lock);
-		port->dev_num = devnum + IB_UMAD_MAX_PORTS;
-		base = devnum + overflow_maj;
-		set_bit(devnum, overflow_map);
+	if (devnum >= IB_UMAD_MAX_PORTS)
+		return -1;
+	port->dev_num = devnum;
+	set_bit(devnum, dev_map);
+	if (devnum >= IB_UMAD_NUM_FIXED_MINOR) {
+		base_umad = dynamic_umad_dev + devnum - IB_UMAD_NUM_FIXED_MINOR;
+		base_issm = dynamic_issm_dev + devnum - IB_UMAD_NUM_FIXED_MINOR;
 	} else {
-		port->dev_num = devnum;
-		base = devnum + base_dev;
-		set_bit(devnum, dev_map);
+		base_umad = devnum + base_umad_dev;
+		base_issm = devnum + base_issm_dev;
 	}
-	spin_unlock(&port_lock);
 
 	port->ib_dev   = device;
 	port->port_num = port_num;
@@ -1198,7 +1181,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
 	port->cdev.owner = THIS_MODULE;
 	cdev_set_parent(&port->cdev, &umad_dev->kobj);
 	kobject_set_name(&port->cdev.kobj, "umad%d", port->dev_num);
-	if (cdev_add(&port->cdev, base, 1))
+	if (cdev_add(&port->cdev, base_umad, 1))
 		goto err_cdev;
 
 	port->dev = device_create(umad_class, device->dev.parent,
@@ -1212,12 +1195,11 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
 	if (device_create_file(port->dev, &dev_attr_port))
 		goto err_dev;
 
-	base += IB_UMAD_MAX_PORTS;
 	cdev_init(&port->sm_cdev, &umad_sm_fops);
 	port->sm_cdev.owner = THIS_MODULE;
 	cdev_set_parent(&port->sm_cdev, &umad_dev->kobj);
 	kobject_set_name(&port->sm_cdev.kobj, "issm%d", port->dev_num);
-	if (cdev_add(&port->sm_cdev, base, 1))
+	if (cdev_add(&port->sm_cdev, base_issm, 1))
 		goto err_sm_cdev;
 
 	port->sm_dev = device_create(umad_class, device->dev.parent,
@@ -1244,10 +1226,7 @@ static int ib_umad_init_port(struct ib_device *device, int port_num,
 
 err_cdev:
 	cdev_del(&port->cdev);
-	if (port->dev_num < IB_UMAD_MAX_PORTS)
-		clear_bit(devnum, dev_map);
-	else
-		clear_bit(devnum, overflow_map);
+	clear_bit(devnum, dev_map);
 
 	return -1;
 }
@@ -1281,11 +1260,7 @@ static void ib_umad_kill_port(struct ib_umad_port *port)
 	}
 
 	mutex_unlock(&port->file_mutex);
-
-	if (port->dev_num < IB_UMAD_MAX_PORTS)
-		clear_bit(port->dev_num, dev_map);
-	else
-		clear_bit(port->dev_num - IB_UMAD_MAX_PORTS, overflow_map);
+	clear_bit(port->dev_num, dev_map);
 }
 
 static void ib_umad_add_one(struct ib_device *device)
@@ -1361,13 +1336,23 @@ static int __init ib_umad_init(void)
 {
 	int ret;
 
-	ret = register_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2,
+	ret = register_chrdev_region(base_umad_dev,
+				     IB_UMAD_NUM_FIXED_MINOR * 2,
 				     "infiniband_mad");
 	if (ret) {
 		pr_err("couldn't register device number\n");
 		goto out;
 	}
 
+	ret = alloc_chrdev_region(&dynamic_umad_dev, 0,
+				  IB_UMAD_NUM_DYNAMIC_MINOR * 2,
+				  "infiniband_mad");
+	if (ret) {
+		pr_err("couldn't register dynamic device number\n");
+		goto out_alloc;
+	}
+	dynamic_issm_dev = dynamic_umad_dev + IB_UMAD_NUM_DYNAMIC_MINOR;
+
 	umad_class = class_create(THIS_MODULE, "infiniband_mad");
 	if (IS_ERR(umad_class)) {
 		ret = PTR_ERR(umad_class);
@@ -1395,7 +1380,12 @@ static int __init ib_umad_init(void)
 	class_destroy(umad_class);
 
 out_chrdev:
-	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);
+	unregister_chrdev_region(dynamic_umad_dev,
+				 IB_UMAD_NUM_DYNAMIC_MINOR * 2);
+
+out_alloc:
+	unregister_chrdev_region(base_umad_dev,
+				 IB_UMAD_NUM_FIXED_MINOR * 2);
 
 out:
 	return ret;
@@ -1405,9 +1395,10 @@ static void __exit ib_umad_cleanup(void)
 {
 	ib_unregister_client(&umad_client);
 	class_destroy(umad_class);
-	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);
-	if (overflow_maj)
-		unregister_chrdev_region(overflow_maj, IB_UMAD_MAX_PORTS * 2);
+	unregister_chrdev_region(base_umad_dev,
+				 IB_UMAD_NUM_FIXED_MINOR * 2);
+	unregister_chrdev_region(dynamic_umad_dev,
+				 IB_UMAD_NUM_DYNAMIC_MINOR * 2);
 }
 
 module_init(ib_umad_init);
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 840b24096690..a148de35df8d 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -340,6 +340,8 @@ ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file,
 	uobj->object = pd;
 	memset(&resp, 0, sizeof resp);
 	resp.pd_handle = uobj->id;
+	pd->res.type = RDMA_RESTRACK_PD;
+	rdma_restrack_add(&pd->res);
 
 	if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, sizeof resp)) {
 		ret = -EFAULT;
@@ -560,9 +562,10 @@ ssize_t ib_uverbs_open_xrcd(struct ib_uverbs_file *file,
 	if (f.file)
 		fdput(f);
 
+	mutex_unlock(&file->device->xrcd_tree_mutex);
+
 	uobj_alloc_commit(&obj->uobject);
 
-	mutex_unlock(&file->device->xrcd_tree_mutex);
 	return in_len;
 
 err_copy:
@@ -601,10 +604,8 @@ ssize_t ib_uverbs_close_xrcd(struct ib_uverbs_file *file,
 
 	uobj  = uobj_get_write(uobj_get_type(xrcd), cmd.xrcd_handle,
 			       file->ucontext);
-	if (IS_ERR(uobj)) {
-		mutex_unlock(&file->device->xrcd_tree_mutex);
+	if (IS_ERR(uobj))
 		return PTR_ERR(uobj);
-	}
 
 	ret = uobj_remove_commit(uobj);
 	return ret ?: in_len;
@@ -977,6 +978,9 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file,
 	struct ib_uverbs_ex_create_cq_resp resp;
 	struct ib_cq_init_attr attr = {};
 
+	if (!ib_dev->create_cq)
+		return ERR_PTR(-EOPNOTSUPP);
+
 	if (cmd->comp_vector >= file->device->num_comp_vectors)
 		return ERR_PTR(-EINVAL);
 
@@ -1028,12 +1032,14 @@ static struct ib_ucq_object *create_cq(struct ib_uverbs_file *file,
 	resp.response_length = offsetof(typeof(resp), response_length) +
 		sizeof(resp.response_length);
 
+	cq->res.type = RDMA_RESTRACK_CQ;
+	rdma_restrack_add(&cq->res);
+
 	ret = cb(file, obj, &resp, ucore, context);
 	if (ret)
 		goto err_cb;
 
 	uobj_alloc_commit(&obj->uobject);
-
 	return obj;
 
 err_cb:
@@ -1145,10 +1151,7 @@ int ib_uverbs_ex_create_cq(struct ib_uverbs_file *file,
 			min(ucore->inlen, sizeof(cmd)),
 			ib_uverbs_ex_create_cq_cb, NULL);
 
-	if (IS_ERR(obj))
-		return PTR_ERR(obj);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(obj);
 }
 
 ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
@@ -1199,7 +1202,7 @@ static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest,
 	tmp.opcode		= wc->opcode;
 	tmp.vendor_err		= wc->vendor_err;
 	tmp.byte_len		= wc->byte_len;
-	tmp.ex.imm_data		= (__u32 __force) wc->ex.imm_data;
+	tmp.ex.imm_data		= wc->ex.imm_data;
 	tmp.qp_num		= wc->qp->qp_num;
 	tmp.src_qp		= wc->src_qp;
 	tmp.wc_flags		= wc->wc_flags;
@@ -1517,7 +1520,8 @@ static int create_qp(struct ib_uverbs_file *file,
 	if (cmd->qp_type == IB_QPT_XRC_TGT)
 		qp = ib_create_qp(pd, &attr);
 	else
-		qp = device->create_qp(pd, &attr, uhw);
+		qp = _ib_create_qp(device, pd, &attr, uhw,
+				   &obj->uevent.uobject);
 
 	if (IS_ERR(qp)) {
 		ret = PTR_ERR(qp);
@@ -1530,7 +1534,6 @@ static int create_qp(struct ib_uverbs_file *file,
 			goto err_cb;
 
 		qp->real_qp	  = qp;
-		qp->device	  = device;
 		qp->pd		  = pd;
 		qp->send_cq	  = attr.send_cq;
 		qp->recv_cq	  = attr.recv_cq;
@@ -1550,8 +1553,10 @@ static int create_qp(struct ib_uverbs_file *file,
 			atomic_inc(&attr.srq->usecnt);
 		if (ind_tbl)
 			atomic_inc(&ind_tbl->usecnt);
+	} else {
+		/* It is done in _ib_create_qp for other QP types */
+		qp->uobject = &obj->uevent.uobject;
 	}
-	qp->uobject = &obj->uevent.uobject;
 
 	obj->uevent.uobject.object = qp;
 
@@ -1971,8 +1976,15 @@ static int modify_qp(struct ib_uverbs_file *file,
 		goto release_qp;
 	}
 
+	if ((cmd->base.attr_mask & IB_QP_AV) &&
+	    !rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) {
+		ret = -EINVAL;
+		goto release_qp;
+	}
+
 	if ((cmd->base.attr_mask & IB_QP_ALT_PATH) &&
-	    !rdma_is_port_valid(qp->device, cmd->base.alt_port_num)) {
+	    (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) ||
+	    !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num))) {
 		ret = -EINVAL;
 		goto release_qp;
 	}
@@ -2941,6 +2953,11 @@ int ib_uverbs_ex_create_wq(struct ib_uverbs_file *file,
 		wq_init_attr.create_flags = cmd.create_flags;
 	obj->uevent.events_reported = 0;
 	INIT_LIST_HEAD(&obj->uevent.event_list);
+
+	if (!pd->device->create_wq) {
+		err = -EOPNOTSUPP;
+		goto err_put_cq;
+	}
 	wq = pd->device->create_wq(pd, &wq_init_attr, uhw);
 	if (IS_ERR(wq)) {
 		err = PTR_ERR(wq);
@@ -3084,7 +3101,12 @@ int ib_uverbs_ex_modify_wq(struct ib_uverbs_file *file,
 		wq_attr.flags = cmd.flags;
 		wq_attr.flags_mask = cmd.flags_mask;
 	}
+	if (!wq->device->modify_wq) {
+		ret = -EOPNOTSUPP;
+		goto out;
+	}
 	ret = wq->device->modify_wq(wq, &wq_attr, cmd.attr_mask, uhw);
+out:
 	uobj_put_obj_read(wq);
 	return ret;
 }
@@ -3181,6 +3203,11 @@ int ib_uverbs_ex_create_rwq_ind_table(struct ib_uverbs_file *file,
 
 	init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size;
 	init_attr.ind_tbl = wqs;
+
+	if (!ib_dev->create_rwq_ind_table) {
+		err = -EOPNOTSUPP;
+		goto err_uobj;
+	}
 	rwq_ind_tbl = ib_dev->create_rwq_ind_table(ib_dev, &init_attr, uhw);
 
 	if (IS_ERR(rwq_ind_tbl)) {
@@ -3770,6 +3797,9 @@ int ib_uverbs_ex_query_device(struct ib_uverbs_file *file,
 	struct ib_device_attr attr = {0};
 	int err;
 
+	if (!ib_dev->query_device)
+		return -EOPNOTSUPP;
+
 	if (ucore->inlen < sizeof(cmd))
 		return -EINVAL;
 
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
index 71ff2644e053..339b85145044 100644
--- a/drivers/infiniband/core/uverbs_ioctl.c
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -59,6 +59,9 @@ static int uverbs_process_attr(struct ib_device *ibdev,
 			return 0;
 	}
 
+	if (test_bit(attr_id, attr_bundle_h->valid_bitmap))
+		return -EINVAL;
+
 	spec = &attr_spec_bucket->attrs[attr_id];
 	e = &elements[attr_id];
 	e->uattr = uattr_ptr;
@@ -243,16 +246,13 @@ static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
 	size_t ctx_size;
 	uintptr_t data[UVERBS_OPTIMIZE_USING_STACK_SZ / sizeof(uintptr_t)];
 
-	if (hdr->reserved)
-		return -EINVAL;
-
 	object_spec = uverbs_get_object(ib_dev, hdr->object_id);
 	if (!object_spec)
-		return -EOPNOTSUPP;
+		return -EPROTONOSUPPORT;
 
 	method_spec = uverbs_get_method(object_spec, hdr->method_id);
 	if (!method_spec)
-		return -EOPNOTSUPP;
+		return -EPROTONOSUPPORT;
 
 	if ((method_spec->flags & UVERBS_ACTION_FLAG_CREATE_ROOT) ^ !file->ucontext)
 		return -EINVAL;
@@ -305,6 +305,16 @@ static long ib_uverbs_cmd_verbs(struct ib_device *ib_dev,
 
 	err = uverbs_handle_method(buf, ctx->uattrs, hdr->num_attrs, ib_dev,
 				   file, method_spec, ctx->uverbs_attr_bundle);
+
+	/*
+	 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
+	 * not invoke the method because the request is not supported.  No
+	 * other cases should return this code.
+	*/
+	if (unlikely(err == -EPROTONOSUPPORT)) {
+		WARN_ON_ONCE(err == -EPROTONOSUPPORT);
+		err = -EINVAL;
+	}
 out:
 	if (ctx != (void *)data)
 		kfree(ctx);
@@ -341,7 +351,7 @@ long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		}
 
 		if (hdr.reserved) {
-			err = -EOPNOTSUPP;
+			err = -EPROTONOSUPPORT;
 			goto out;
 		}
 
diff --git a/drivers/infiniband/core/uverbs_ioctl_merge.c b/drivers/infiniband/core/uverbs_ioctl_merge.c
index 062485f9300d..62e1eb1d2a28 100644
--- a/drivers/infiniband/core/uverbs_ioctl_merge.c
+++ b/drivers/infiniband/core/uverbs_ioctl_merge.c
@@ -114,6 +114,7 @@ static size_t get_elements_above_id(const void **iters,
 	short min = SHRT_MAX;
 	const void *elem;
 	int i, j, last_stored = -1;
+	unsigned int equal_min = 0;
 
 	for_each_element(elem, i, j, elements, num_elements, num_offset,
 			 data_offset) {
@@ -136,6 +137,10 @@ static size_t get_elements_above_id(const void **iters,
 		 */
 		iters[last_stored == i ? num_iters - 1 : num_iters++] = elem;
 		last_stored = i;
+		if (min == GET_ID(id))
+			equal_min++;
+		else
+			equal_min = 1;
 		min = GET_ID(id);
 	}
 
@@ -146,15 +151,10 @@ static size_t get_elements_above_id(const void **iters,
 	 * Therefore, we need to clean the beginning of the array to make sure
 	 * all ids of final elements are equal to min.
 	 */
-	for (i = num_iters - 1; i >= 0 &&
-	     GET_ID(*(u16 *)(iters[i] + id_offset)) == min; i--)
-		;
-
-	num_iters -= i + 1;
-	memmove(iters, iters + i + 1, sizeof(*iters) * num_iters);
+	memmove(iters, iters + num_iters - equal_min, sizeof(*iters) * equal_min);
 
 	*min_id = min;
-	return num_iters;
+	return equal_min;
 }
 
 #define find_max_element_entry_id(num_elements, elements, num_objects_fld, \
@@ -322,7 +322,7 @@ static struct uverbs_method_spec *build_method_with_attrs(const struct uverbs_me
 		hash = kzalloc(sizeof(*hash) +
 			       ALIGN(sizeof(*hash->attrs) * (attr_max_bucket + 1),
 				     sizeof(long)) +
-			       BITS_TO_LONGS(attr_max_bucket) * sizeof(long),
+			       BITS_TO_LONGS(attr_max_bucket + 1) * sizeof(long),
 			       GFP_KERNEL);
 		if (!hash) {
 			res = -ENOMEM;
@@ -509,7 +509,7 @@ static struct uverbs_object_spec *build_object_with_methods(const struct uverbs_
 			 * first handler which != NULL. This also defines the
 			 * set of flags used for this handler.
 			 */
-			for (i = num_object_defs - 1;
+			for (i = num_method_defs - 1;
 			     i >= 0 && !method_defs[i]->handler; i--)
 				;
 			hash->methods[min_id++] = method;
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 381fd9c096ae..b1ca223aa380 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -62,14 +62,16 @@ MODULE_LICENSE("Dual BSD/GPL");
 enum {
 	IB_UVERBS_MAJOR       = 231,
 	IB_UVERBS_BASE_MINOR  = 192,
-	IB_UVERBS_MAX_DEVICES = 32
+	IB_UVERBS_MAX_DEVICES = RDMA_MAX_PORTS,
+	IB_UVERBS_NUM_FIXED_MINOR = 32,
+	IB_UVERBS_NUM_DYNAMIC_MINOR = IB_UVERBS_MAX_DEVICES - IB_UVERBS_NUM_FIXED_MINOR,
 };
 
 #define IB_UVERBS_BASE_DEV	MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)
 
+static dev_t dynamic_uverbs_dev;
 static struct class *uverbs_class;
 
-static DEFINE_SPINLOCK(map_lock);
 static DECLARE_BITMAP(dev_map, IB_UVERBS_MAX_DEVICES);
 
 static ssize_t (*uverbs_cmd_table[])(struct ib_uverbs_file *file,
@@ -339,29 +341,29 @@ static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf,
 				    sizeof(struct ib_uverbs_comp_event_desc));
 }
 
-static unsigned int ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,
+static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,
 					 struct file *filp,
 					 struct poll_table_struct *wait)
 {
-	unsigned int pollflags = 0;
+	__poll_t pollflags = 0;
 
 	poll_wait(filp, &ev_queue->poll_wait, wait);
 
 	spin_lock_irq(&ev_queue->lock);
 	if (!list_empty(&ev_queue->event_list))
-		pollflags = POLLIN | POLLRDNORM;
+		pollflags = EPOLLIN | EPOLLRDNORM;
 	spin_unlock_irq(&ev_queue->lock);
 
 	return pollflags;
 }
 
-static unsigned int ib_uverbs_async_event_poll(struct file *filp,
+static __poll_t ib_uverbs_async_event_poll(struct file *filp,
 					       struct poll_table_struct *wait)
 {
 	return ib_uverbs_event_poll(filp->private_data, filp, wait);
 }
 
-static unsigned int ib_uverbs_comp_event_poll(struct file *filp,
+static __poll_t ib_uverbs_comp_event_poll(struct file *filp,
 					      struct poll_table_struct *wait)
 {
 	struct ib_uverbs_completion_event_file *comp_ev_file =
@@ -648,12 +650,21 @@ static int verify_command_mask(struct ib_device *ib_dev, __u32 command)
 	return -1;
 }
 
+static bool verify_command_idx(u32 command, bool extended)
+{
+	if (extended)
+		return command < ARRAY_SIZE(uverbs_ex_cmd_table);
+
+	return command < ARRAY_SIZE(uverbs_cmd_table);
+}
+
 static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 			     size_t count, loff_t *pos)
 {
 	struct ib_uverbs_file *file = filp->private_data;
 	struct ib_device *ib_dev;
 	struct ib_uverbs_cmd_hdr hdr;
+	bool extended_command;
 	__u32 command;
 	__u32 flags;
 	int srcu_key;
@@ -686,6 +697,15 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 	}
 
 	command = hdr.command & IB_USER_VERBS_CMD_COMMAND_MASK;
+	flags = (hdr.command &
+		 IB_USER_VERBS_CMD_FLAGS_MASK) >> IB_USER_VERBS_CMD_FLAGS_SHIFT;
+
+	extended_command = flags & IB_USER_VERBS_CMD_FLAG_EXTENDED;
+	if (!verify_command_idx(command, extended_command)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	if (verify_command_mask(ib_dev, command)) {
 		ret = -EOPNOTSUPP;
 		goto out;
@@ -697,12 +717,8 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 		goto out;
 	}
 
-	flags = (hdr.command &
-		 IB_USER_VERBS_CMD_FLAGS_MASK) >> IB_USER_VERBS_CMD_FLAGS_SHIFT;
-
 	if (!flags) {
-		if (command >= ARRAY_SIZE(uverbs_cmd_table) ||
-		    !uverbs_cmd_table[command]) {
+		if (!uverbs_cmd_table[command]) {
 			ret = -EINVAL;
 			goto out;
 		}
@@ -723,8 +739,7 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 		struct ib_udata uhw;
 		size_t written_count = count;
 
-		if (command >= ARRAY_SIZE(uverbs_ex_cmd_table) ||
-		    !uverbs_ex_cmd_table[command]) {
+		if (!uverbs_ex_cmd_table[command]) {
 			ret = -ENOSYS;
 			goto out;
 		}
@@ -940,6 +955,7 @@ static const struct file_operations uverbs_fops = {
 	.llseek	 = no_llseek,
 #if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS)
 	.unlocked_ioctl = ib_uverbs_ioctl,
+	.compat_ioctl = ib_uverbs_ioctl,
 #endif
 };
 
@@ -952,6 +968,7 @@ static const struct file_operations uverbs_mmap_fops = {
 	.llseek	 = no_llseek,
 #if IS_ENABLED(CONFIG_INFINIBAND_EXP_USER_ACCESS)
 	.unlocked_ioctl = ib_uverbs_ioctl,
+	.compat_ioctl = ib_uverbs_ioctl,
 #endif
 };
 
@@ -1005,34 +1022,6 @@ static DEVICE_ATTR(abi_version, S_IRUGO, show_dev_abi_version, NULL);
 static CLASS_ATTR_STRING(abi_version, S_IRUGO,
 			 __stringify(IB_USER_VERBS_ABI_VERSION));
 
-static dev_t overflow_maj;
-static DECLARE_BITMAP(overflow_map, IB_UVERBS_MAX_DEVICES);
-
-/*
- * If we have more than IB_UVERBS_MAX_DEVICES, dynamically overflow by
- * requesting a new major number and doubling the number of max devices we
- * support. It's stupid, but simple.
- */
-static int find_overflow_devnum(void)
-{
-	int ret;
-
-	if (!overflow_maj) {
-		ret = alloc_chrdev_region(&overflow_maj, 0, IB_UVERBS_MAX_DEVICES,
-					  "infiniband_verbs");
-		if (ret) {
-			pr_err("user_verbs: couldn't register dynamic device number\n");
-			return ret;
-		}
-	}
-
-	ret = find_first_zero_bit(overflow_map, IB_UVERBS_MAX_DEVICES);
-	if (ret >= IB_UVERBS_MAX_DEVICES)
-		return -1;
-
-	return ret;
-}
-
 static void ib_uverbs_add_one(struct ib_device *device)
 {
 	int devnum;
@@ -1062,24 +1051,15 @@ static void ib_uverbs_add_one(struct ib_device *device)
 	INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list);
 	INIT_LIST_HEAD(&uverbs_dev->uverbs_events_file_list);
 
-	spin_lock(&map_lock);
 	devnum = find_first_zero_bit(dev_map, IB_UVERBS_MAX_DEVICES);
-	if (devnum >= IB_UVERBS_MAX_DEVICES) {
-		spin_unlock(&map_lock);
-		devnum = find_overflow_devnum();
-		if (devnum < 0)
-			goto err;
-
-		spin_lock(&map_lock);
-		uverbs_dev->devnum = devnum + IB_UVERBS_MAX_DEVICES;
-		base = devnum + overflow_maj;
-		set_bit(devnum, overflow_map);
-	} else {
-		uverbs_dev->devnum = devnum;
-		base = devnum + IB_UVERBS_BASE_DEV;
-		set_bit(devnum, dev_map);
-	}
-	spin_unlock(&map_lock);
+	if (devnum >= IB_UVERBS_MAX_DEVICES)
+		goto err;
+	uverbs_dev->devnum = devnum;
+	set_bit(devnum, dev_map);
+	if (devnum >= IB_UVERBS_NUM_FIXED_MINOR)
+		base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR;
+	else
+		base = IB_UVERBS_BASE_DEV + devnum;
 
 	rcu_assign_pointer(uverbs_dev->ib_dev, device);
 	uverbs_dev->num_comp_vectors = device->num_comp_vectors;
@@ -1124,10 +1104,7 @@ static void ib_uverbs_add_one(struct ib_device *device)
 
 err_cdev:
 	cdev_del(&uverbs_dev->cdev);
-	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
-		clear_bit(devnum, dev_map);
-	else
-		clear_bit(devnum, overflow_map);
+	clear_bit(devnum, dev_map);
 
 err:
 	if (atomic_dec_and_test(&uverbs_dev->refcount))
@@ -1219,11 +1196,7 @@ static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
 	dev_set_drvdata(uverbs_dev->dev, NULL);
 	device_destroy(uverbs_class, uverbs_dev->cdev.dev);
 	cdev_del(&uverbs_dev->cdev);
-
-	if (uverbs_dev->devnum < IB_UVERBS_MAX_DEVICES)
-		clear_bit(uverbs_dev->devnum, dev_map);
-	else
-		clear_bit(uverbs_dev->devnum - IB_UVERBS_MAX_DEVICES, overflow_map);
+	clear_bit(uverbs_dev->devnum, dev_map);
 
 	if (device->disassociate_ucontext) {
 		/* We disassociate HW resources and immediately return.
@@ -1265,13 +1238,22 @@ static int __init ib_uverbs_init(void)
 {
 	int ret;
 
-	ret = register_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES,
+	ret = register_chrdev_region(IB_UVERBS_BASE_DEV,
+				     IB_UVERBS_NUM_FIXED_MINOR,
 				     "infiniband_verbs");
 	if (ret) {
 		pr_err("user_verbs: couldn't register device number\n");
 		goto out;
 	}
 
+	ret = alloc_chrdev_region(&dynamic_uverbs_dev, 0,
+				  IB_UVERBS_NUM_DYNAMIC_MINOR,
+				  "infiniband_verbs");
+	if (ret) {
+		pr_err("couldn't register dynamic device number\n");
+		goto out_alloc;
+	}
+
 	uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
 	if (IS_ERR(uverbs_class)) {
 		ret = PTR_ERR(uverbs_class);
@@ -1299,7 +1281,12 @@ static int __init ib_uverbs_init(void)
 	class_destroy(uverbs_class);
 
 out_chrdev:
-	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
+	unregister_chrdev_region(dynamic_uverbs_dev,
+				 IB_UVERBS_NUM_DYNAMIC_MINOR);
+
+out_alloc:
+	unregister_chrdev_region(IB_UVERBS_BASE_DEV,
+				 IB_UVERBS_NUM_FIXED_MINOR);
 
 out:
 	return ret;
@@ -1309,9 +1296,10 @@ static void __exit ib_uverbs_cleanup(void)
 {
 	ib_unregister_client(&uverbs_client);
 	class_destroy(uverbs_class);
-	unregister_chrdev_region(IB_UVERBS_BASE_DEV, IB_UVERBS_MAX_DEVICES);
-	if (overflow_maj)
-		unregister_chrdev_region(overflow_maj, IB_UVERBS_MAX_DEVICES);
+	unregister_chrdev_region(IB_UVERBS_BASE_DEV,
+				 IB_UVERBS_NUM_FIXED_MINOR);
+	unregister_chrdev_region(dynamic_uverbs_dev,
+				 IB_UVERBS_NUM_DYNAMIC_MINOR);
 }
 
 module_init(ib_uverbs_init);
diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c
index c3ee5d9b336d..df1360e6774f 100644
--- a/drivers/infiniband/core/uverbs_std_types.c
+++ b/drivers/infiniband/core/uverbs_std_types.c
@@ -35,6 +35,7 @@
 #include <rdma/ib_verbs.h>
 #include <linux/bug.h>
 #include <linux/file.h>
+#include <rdma/restrack.h>
 #include "rdma_core.h"
 #include "uverbs.h"
 
@@ -233,15 +234,18 @@ static void create_udata(struct uverbs_attr_bundle *ctx,
 		uverbs_attr_get(ctx, UVERBS_UHW_OUT);
 
 	if (!IS_ERR(uhw_in)) {
-		udata->inbuf = uhw_in->ptr_attr.ptr;
 		udata->inlen = uhw_in->ptr_attr.len;
+		if (uverbs_attr_ptr_is_inline(uhw_in))
+			udata->inbuf = &uhw_in->uattr->data;
+		else
+			udata->inbuf = u64_to_user_ptr(uhw_in->ptr_attr.data);
 	} else {
 		udata->inbuf = NULL;
 		udata->inlen = 0;
 	}
 
 	if (!IS_ERR(uhw_out)) {
-		udata->outbuf = uhw_out->ptr_attr.ptr;
+		udata->outbuf = u64_to_user_ptr(uhw_out->ptr_attr.data);
 		udata->outlen = uhw_out->ptr_attr.len;
 	} else {
 		udata->outbuf = NULL;
@@ -315,12 +319,15 @@ static int uverbs_create_cq_handler(struct ib_device *ib_dev,
 	cq->uobject       = &obj->uobject;
 	cq->comp_handler  = ib_uverbs_comp_handler;
 	cq->event_handler = ib_uverbs_cq_event_handler;
-	cq->cq_context    = &ev_file->ev_queue;
+	cq->cq_context    = ev_file ? &ev_file->ev_queue : NULL;
 	obj->uobject.object = cq;
 	obj->uobject.user_handle = user_handle;
 	atomic_set(&cq->usecnt, 0);
+	cq->res.type = RDMA_RESTRACK_CQ;
+	rdma_restrack_add(&cq->res);
 
-	ret = uverbs_copy_to(attrs, CREATE_CQ_RESP_CQE, &cq->cqe);
+	ret = uverbs_copy_to(attrs, CREATE_CQ_RESP_CQE, &cq->cqe,
+			     sizeof(cq->cqe));
 	if (ret)
 		goto err_cq;
 
@@ -372,7 +379,7 @@ static int uverbs_destroy_cq_handler(struct ib_device *ib_dev,
 	resp.comp_events_reported  = obj->comp_events_reported;
 	resp.async_events_reported = obj->async_events_reported;
 
-	return uverbs_copy_to(attrs, DESTROY_CQ_RESP, &resp);
+	return uverbs_copy_to(attrs, DESTROY_CQ_RESP, &resp, sizeof(resp));
 }
 
 static DECLARE_UVERBS_METHOD(
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index e36d27ed4daa..93025d2009b8 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -124,16 +124,24 @@ EXPORT_SYMBOL(ib_wc_status_msg);
 __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
 {
 	switch (rate) {
-	case IB_RATE_2_5_GBPS: return  1;
-	case IB_RATE_5_GBPS:   return  2;
-	case IB_RATE_10_GBPS:  return  4;
-	case IB_RATE_20_GBPS:  return  8;
-	case IB_RATE_30_GBPS:  return 12;
-	case IB_RATE_40_GBPS:  return 16;
-	case IB_RATE_60_GBPS:  return 24;
-	case IB_RATE_80_GBPS:  return 32;
-	case IB_RATE_120_GBPS: return 48;
-	default:	       return -1;
+	case IB_RATE_2_5_GBPS: return   1;
+	case IB_RATE_5_GBPS:   return   2;
+	case IB_RATE_10_GBPS:  return   4;
+	case IB_RATE_20_GBPS:  return   8;
+	case IB_RATE_30_GBPS:  return  12;
+	case IB_RATE_40_GBPS:  return  16;
+	case IB_RATE_60_GBPS:  return  24;
+	case IB_RATE_80_GBPS:  return  32;
+	case IB_RATE_120_GBPS: return  48;
+	case IB_RATE_14_GBPS:  return   6;
+	case IB_RATE_56_GBPS:  return  22;
+	case IB_RATE_112_GBPS: return  45;
+	case IB_RATE_168_GBPS: return  67;
+	case IB_RATE_25_GBPS:  return  10;
+	case IB_RATE_100_GBPS: return  40;
+	case IB_RATE_200_GBPS: return  80;
+	case IB_RATE_300_GBPS: return 120;
+	default:	       return  -1;
 	}
 }
 EXPORT_SYMBOL(ib_rate_to_mult);
@@ -141,16 +149,24 @@ EXPORT_SYMBOL(ib_rate_to_mult);
 __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
 {
 	switch (mult) {
-	case 1:  return IB_RATE_2_5_GBPS;
-	case 2:  return IB_RATE_5_GBPS;
-	case 4:  return IB_RATE_10_GBPS;
-	case 8:  return IB_RATE_20_GBPS;
-	case 12: return IB_RATE_30_GBPS;
-	case 16: return IB_RATE_40_GBPS;
-	case 24: return IB_RATE_60_GBPS;
-	case 32: return IB_RATE_80_GBPS;
-	case 48: return IB_RATE_120_GBPS;
-	default: return IB_RATE_PORT_CURRENT;
+	case 1:   return IB_RATE_2_5_GBPS;
+	case 2:   return IB_RATE_5_GBPS;
+	case 4:   return IB_RATE_10_GBPS;
+	case 8:   return IB_RATE_20_GBPS;
+	case 12:  return IB_RATE_30_GBPS;
+	case 16:  return IB_RATE_40_GBPS;
+	case 24:  return IB_RATE_60_GBPS;
+	case 32:  return IB_RATE_80_GBPS;
+	case 48:  return IB_RATE_120_GBPS;
+	case 6:   return IB_RATE_14_GBPS;
+	case 22:  return IB_RATE_56_GBPS;
+	case 45:  return IB_RATE_112_GBPS;
+	case 67:  return IB_RATE_168_GBPS;
+	case 10:  return IB_RATE_25_GBPS;
+	case 40:  return IB_RATE_100_GBPS;
+	case 80:  return IB_RATE_200_GBPS;
+	case 120: return IB_RATE_300_GBPS;
+	default:  return IB_RATE_PORT_CURRENT;
 	}
 }
 EXPORT_SYMBOL(mult_to_ib_rate);
@@ -247,6 +263,10 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
 		mr_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE;
 	}
 
+	pd->res.type = RDMA_RESTRACK_PD;
+	pd->res.kern_name = caller;
+	rdma_restrack_add(&pd->res);
+
 	if (mr_access_flags) {
 		struct ib_mr *mr;
 
@@ -296,6 +316,7 @@ void ib_dealloc_pd(struct ib_pd *pd)
 	   requires the caller to guarantee we can't race here. */
 	WARN_ON(atomic_read(&pd->usecnt));
 
+	rdma_restrack_del(&pd->res);
 	/* Making delalloc_pd a void return is a WIP, no driver should return
 	   an error here. */
 	ret = pd->device->dealloc_pd(pd);
@@ -421,8 +442,7 @@ static bool find_gid_index(const union ib_gid *gid,
 			   const struct ib_gid_attr *gid_attr,
 			   void *context)
 {
-	struct find_gid_index_context *ctx =
-		(struct find_gid_index_context *)context;
+	struct find_gid_index_context *ctx = context;
 
 	if (ctx->gid_type != gid_attr->gid_type)
 		return false;
@@ -481,8 +501,53 @@ int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
 }
 EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
 
+/* Resolve destination mac address and hop limit for unicast destination
+ * GID entry, considering the source GID entry as well.
+ * ah_attribute must have have valid port_num, sgid_index.
+ */
+static int ib_resolve_unicast_gid_dmac(struct ib_device *device,
+				       struct rdma_ah_attr *ah_attr)
+{
+	struct ib_gid_attr sgid_attr;
+	struct ib_global_route *grh;
+	int hop_limit = 0xff;
+	union ib_gid sgid;
+	int ret;
+
+	grh = rdma_ah_retrieve_grh(ah_attr);
+
+	ret = ib_query_gid(device,
+			   rdma_ah_get_port_num(ah_attr),
+			   grh->sgid_index,
+			   &sgid, &sgid_attr);
+	if (ret || !sgid_attr.ndev) {
+		if (!ret)
+			ret = -ENXIO;
+		return ret;
+	}
+
+	/* If destination is link local and source GID is RoCEv1,
+	 * IP stack is not used.
+	 */
+	if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) &&
+	    sgid_attr.gid_type == IB_GID_TYPE_ROCE) {
+		rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
+				ah_attr->roce.dmac);
+		goto done;
+	}
+
+	ret = rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid,
+					   ah_attr->roce.dmac,
+					   sgid_attr.ndev, &hop_limit);
+done:
+	dev_put(sgid_attr.ndev);
+
+	grh->hop_limit = hop_limit;
+	return ret;
+}
+
 /*
- * This function creates ah from the incoming packet.
+ * This function initializes address handle attributes from the incoming packet.
  * Incoming packet has dgid of the receiver node on which this code is
  * getting executed and, sgid contains the GID of the sender.
  *
@@ -490,13 +555,10 @@ EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
  * as sgid and, sgid is used as dgid because sgid contains destinations
  * GID whom to respond to.
  *
- * This is why when calling rdma_addr_find_l2_eth_by_grh() function, the
- * position of arguments dgid and sgid do not match the order of the
- * parameters.
  */
-int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
-		       const struct ib_wc *wc, const struct ib_grh *grh,
-		       struct rdma_ah_attr *ah_attr)
+int ib_init_ah_attr_from_wc(struct ib_device *device, u8 port_num,
+			    const struct ib_wc *wc, const struct ib_grh *grh,
+			    struct rdma_ah_attr *ah_attr)
 {
 	u32 flow_class;
 	u16 gid_index;
@@ -523,57 +585,33 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 	if (ret)
 		return ret;
 
+	rdma_ah_set_sl(ah_attr, wc->sl);
+	rdma_ah_set_port_num(ah_attr, port_num);
+
 	if (rdma_protocol_roce(device, port_num)) {
-		int if_index = 0;
 		u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
 				wc->vlan_id : 0xffff;
-		struct net_device *idev;
-		struct net_device *resolved_dev;
 
 		if (!(wc->wc_flags & IB_WC_GRH))
 			return -EPROTOTYPE;
 
-		if (!device->get_netdev)
-			return -EOPNOTSUPP;
-
-		idev = device->get_netdev(device, port_num);
-		if (!idev)
-			return -ENODEV;
-
-		ret = rdma_addr_find_l2_eth_by_grh(&dgid, &sgid,
-						   ah_attr->roce.dmac,
-						   wc->wc_flags & IB_WC_WITH_VLAN ?
-						   NULL : &vlan_id,
-						   &if_index, &hoplimit);
-		if (ret) {
-			dev_put(idev);
-			return ret;
-		}
-
-		resolved_dev = dev_get_by_index(&init_net, if_index);
-		rcu_read_lock();
-		if (resolved_dev != idev && !rdma_is_upper_dev_rcu(idev,
-								   resolved_dev))
-			ret = -EHOSTUNREACH;
-		rcu_read_unlock();
-		dev_put(idev);
-		dev_put(resolved_dev);
+		ret = get_sgid_index_from_eth(device, port_num,
+					      vlan_id, &dgid,
+					      gid_type, &gid_index);
 		if (ret)
 			return ret;
 
-		ret = get_sgid_index_from_eth(device, port_num, vlan_id,
-					      &dgid, gid_type, &gid_index);
-		if (ret)
-			return ret;
-	}
-
-	rdma_ah_set_dlid(ah_attr, wc->slid);
-	rdma_ah_set_sl(ah_attr, wc->sl);
-	rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits);
-	rdma_ah_set_port_num(ah_attr, port_num);
+		flow_class = be32_to_cpu(grh->version_tclass_flow);
+		rdma_ah_set_grh(ah_attr, &sgid,
+				flow_class & 0xFFFFF,
+				(u8)gid_index, hoplimit,
+				(flow_class >> 20) & 0xFF);
+		return ib_resolve_unicast_gid_dmac(device, ah_attr);
+	} else {
+		rdma_ah_set_dlid(ah_attr, wc->slid);
+		rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits);
 
-	if (wc->wc_flags & IB_WC_GRH) {
-		if (!rdma_cap_eth_ah(device, port_num)) {
+		if (wc->wc_flags & IB_WC_GRH) {
 			if (dgid.global.interface_id != cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) {
 				ret = ib_find_cached_gid_by_port(device, &dgid,
 								 IB_GID_TYPE_IB,
@@ -584,18 +622,17 @@ int ib_init_ah_from_wc(struct ib_device *device, u8 port_num,
 			} else {
 				gid_index = 0;
 			}
-		}
-
-		flow_class = be32_to_cpu(grh->version_tclass_flow);
-		rdma_ah_set_grh(ah_attr, &sgid,
-				flow_class & 0xFFFFF,
-				(u8)gid_index, hoplimit,
-				(flow_class >> 20) & 0xFF);
 
+			flow_class = be32_to_cpu(grh->version_tclass_flow);
+			rdma_ah_set_grh(ah_attr, &sgid,
+					flow_class & 0xFFFFF,
+					(u8)gid_index, hoplimit,
+					(flow_class >> 20) & 0xFF);
+		}
+		return 0;
 	}
-	return 0;
 }
-EXPORT_SYMBOL(ib_init_ah_from_wc);
+EXPORT_SYMBOL(ib_init_ah_attr_from_wc);
 
 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
 				   const struct ib_grh *grh, u8 port_num)
@@ -603,7 +640,7 @@ struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
 	struct rdma_ah_attr ah_attr;
 	int ret;
 
-	ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
+	ret = ib_init_ah_attr_from_wc(pd->device, port_num, wc, grh, &ah_attr);
 	if (ret)
 		return ERR_PTR(ret);
 
@@ -850,7 +887,7 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
 	if (qp_init_attr->cap.max_rdma_ctxs)
 		rdma_rw_init_qp(device, qp_init_attr);
 
-	qp = device->create_qp(pd, qp_init_attr, NULL);
+	qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL);
 	if (IS_ERR(qp))
 		return qp;
 
@@ -860,9 +897,7 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
 		return ERR_PTR(ret);
 	}
 
-	qp->device     = device;
 	qp->real_qp    = qp;
-	qp->uobject    = NULL;
 	qp->qp_type    = qp_init_attr->qp_type;
 	qp->rwq_ind_tbl = qp_init_attr->rwq_ind_tbl;
 
@@ -890,7 +925,6 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
 			atomic_inc(&qp_init_attr->srq->usecnt);
 	}
 
-	qp->pd	    = pd;
 	qp->send_cq = qp_init_attr->send_cq;
 	qp->xrcd    = NULL;
 
@@ -1269,16 +1303,8 @@ static int ib_resolve_eth_dmac(struct ib_device *device,
 	if (!rdma_is_port_valid(device, rdma_ah_get_port_num(ah_attr)))
 		return -EINVAL;
 
-	if (ah_attr->type != RDMA_AH_ATTR_TYPE_ROCE)
-		return 0;
-
 	grh = rdma_ah_retrieve_grh(ah_attr);
 
-	if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw)) {
-		rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
-				ah_attr->roce.dmac);
-		return 0;
-	}
 	if (rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
 		if (ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw)) {
 			__be32 addr = 0;
@@ -1290,40 +1316,52 @@ static int ib_resolve_eth_dmac(struct ib_device *device,
 					(char *)ah_attr->roce.dmac);
 		}
 	} else {
-		union ib_gid		sgid;
-		struct ib_gid_attr	sgid_attr;
-		int			ifindex;
-		int			hop_limit;
-
-		ret = ib_query_gid(device,
-				   rdma_ah_get_port_num(ah_attr),
-				   grh->sgid_index,
-				   &sgid, &sgid_attr);
-
-		if (ret || !sgid_attr.ndev) {
-			if (!ret)
-				ret = -ENXIO;
-			goto out;
-		}
-
-		ifindex = sgid_attr.ndev->ifindex;
+		ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
+	}
+	return ret;
+}
 
-		ret =
-		rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid,
-					     ah_attr->roce.dmac,
-					     NULL, &ifindex, &hop_limit);
+/**
+ * IB core internal function to perform QP attributes modification.
+ */
+static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
+			 int attr_mask, struct ib_udata *udata)
+{
+	u8 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
+	int ret;
 
-		dev_put(sgid_attr.ndev);
+	if (rdma_ib_or_roce(qp->device, port)) {
+		if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
+			pr_warn("%s: %s rq_psn overflow, masking to 24 bits\n",
+				__func__, qp->device->name);
+			attr->rq_psn &= 0xffffff;
+		}
 
-		grh->hop_limit = hop_limit;
+		if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
+			pr_warn("%s: %s sq_psn overflow, masking to 24 bits\n",
+				__func__, qp->device->name);
+			attr->sq_psn &= 0xffffff;
+		}
 	}
-out:
+
+	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
+	if (!ret && (attr_mask & IB_QP_PORT))
+		qp->port = attr->port_num;
+
 	return ret;
 }
 
+static bool is_qp_type_connected(const struct ib_qp *qp)
+{
+	return (qp->qp_type == IB_QPT_UC ||
+		qp->qp_type == IB_QPT_RC ||
+		qp->qp_type == IB_QPT_XRC_INI ||
+		qp->qp_type == IB_QPT_XRC_TGT);
+}
+
 /**
  * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
- * @qp: The QP to modify.
+ * @ib_qp: The QP to modify.
  * @attr: On input, specifies the QP attributes to modify.  On output,
  *   the current values of selected QP attributes are returned.
  * @attr_mask: A bit-mask used to specify which attributes of the QP
@@ -1332,21 +1370,20 @@ static int ib_resolve_eth_dmac(struct ib_device *device,
  *   are being modified.
  * It returns 0 on success and returns appropriate error code on error.
  */
-int ib_modify_qp_with_udata(struct ib_qp *qp, struct ib_qp_attr *attr,
+int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,
 			    int attr_mask, struct ib_udata *udata)
 {
+	struct ib_qp *qp = ib_qp->real_qp;
 	int ret;
 
-	if (attr_mask & IB_QP_AV) {
+	if (attr_mask & IB_QP_AV &&
+	    attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
+	    is_qp_type_connected(qp)) {
 		ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr);
 		if (ret)
 			return ret;
 	}
-	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
-	if (!ret && (attr_mask & IB_QP_PORT))
-		qp->port = attr->port_num;
-
-	return ret;
+	return _ib_modify_qp(qp, attr, attr_mask, udata);
 }
 EXPORT_SYMBOL(ib_modify_qp_with_udata);
 
@@ -1409,7 +1446,7 @@ int ib_modify_qp(struct ib_qp *qp,
 		 struct ib_qp_attr *qp_attr,
 		 int qp_attr_mask)
 {
-	return ib_modify_qp_with_udata(qp, qp_attr, qp_attr_mask, NULL);
+	return _ib_modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
 }
 EXPORT_SYMBOL(ib_modify_qp);
 
@@ -1503,6 +1540,7 @@ int ib_destroy_qp(struct ib_qp *qp)
 	if (!qp->uobject)
 		rdma_rw_cleanup_mrs(qp);
 
+	rdma_restrack_del(&qp->res);
 	ret = qp->device->destroy_qp(qp);
 	if (!ret) {
 		if (pd)
@@ -1545,6 +1583,8 @@ struct ib_cq *ib_create_cq(struct ib_device *device,
 		cq->event_handler = event_handler;
 		cq->cq_context    = cq_context;
 		atomic_set(&cq->usecnt, 0);
+		cq->res.type = RDMA_RESTRACK_CQ;
+		rdma_restrack_add(&cq->res);
 	}
 
 	return cq;
@@ -1563,6 +1603,7 @@ int ib_destroy_cq(struct ib_cq *cq)
 	if (atomic_read(&cq->usecnt))
 		return -EBUSY;
 
+	rdma_restrack_del(&cq->res);
 	return cq->device->destroy_cq(cq);
 }
 EXPORT_SYMBOL(ib_destroy_cq);
@@ -1747,7 +1788,7 @@ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
 }
 EXPORT_SYMBOL(ib_detach_mcast);
 
-struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
+struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller)
 {
 	struct ib_xrcd *xrcd;
 
@@ -1765,7 +1806,7 @@ struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
 
 	return xrcd;
 }
-EXPORT_SYMBOL(ib_alloc_xrcd);
+EXPORT_SYMBOL(__ib_alloc_xrcd);
 
 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
 {
@@ -1790,11 +1831,11 @@ EXPORT_SYMBOL(ib_dealloc_xrcd);
  * ib_create_wq - Creates a WQ associated with the specified protection
  * domain.
  * @pd: The protection domain associated with the WQ.
- * @wq_init_attr: A list of initial attributes required to create the
+ * @wq_attr: A list of initial attributes required to create the
  * WQ. If WQ creation succeeds, then the attributes are updated to
  * the actual capabilities of the created WQ.
  *
- * wq_init_attr->max_wr and wq_init_attr->max_sge determine
+ * wq_attr->max_wr and wq_attr->max_sge determine
  * the requested size of the WQ, and set to the actual values allocated
  * on return.
  * If ib_create_wq() succeeds, then max_wr and max_sge will always be
@@ -2156,16 +2197,16 @@ static void __ib_drain_sq(struct ib_qp *qp)
 	struct ib_send_wr swr = {}, *bad_swr;
 	int ret;
 
-	swr.wr_cqe = &sdrain.cqe;
-	sdrain.cqe.done = ib_drain_qp_done;
-	init_completion(&sdrain.done);
-
 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
 	if (ret) {
 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
 		return;
 	}
 
+	swr.wr_cqe = &sdrain.cqe;
+	sdrain.cqe.done = ib_drain_qp_done;
+	init_completion(&sdrain.done);
+
 	ret = ib_post_send(qp, &swr, &bad_swr);
 	if (ret) {
 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
@@ -2190,16 +2231,16 @@ static void __ib_drain_rq(struct ib_qp *qp)
 	struct ib_recv_wr rwr = {}, *bad_rwr;
 	int ret;
 
-	rwr.wr_cqe = &rdrain.cqe;
-	rdrain.cqe.done = ib_drain_qp_done;
-	init_completion(&rdrain.done);
-
 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
 	if (ret) {
 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
 		return;
 	}
 
+	rwr.wr_cqe = &rdrain.cqe;
+	rdrain.cqe.done = ib_drain_qp_done;
+	init_completion(&rdrain.done);
+
 	ret = ib_post_recv(qp, &rwr, &bad_rwr);
 	if (ret) {
 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index ecbac91b2e14..96f76896488d 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -43,20 +43,41 @@
 #define ROCE_DRV_MODULE_VERSION		"1.0.0"
 
 #define BNXT_RE_DESC	"Broadcom NetXtreme-C/E RoCE Driver"
-
-#define BNXT_RE_PAGE_SIZE_4K		BIT(12)
-#define BNXT_RE_PAGE_SIZE_8K		BIT(13)
-#define BNXT_RE_PAGE_SIZE_64K		BIT(16)
-#define BNXT_RE_PAGE_SIZE_2M		BIT(21)
-#define BNXT_RE_PAGE_SIZE_8M		BIT(23)
-#define BNXT_RE_PAGE_SIZE_1G		BIT(30)
-
-#define BNXT_RE_MAX_MR_SIZE		BIT(30)
+#define BNXT_RE_PAGE_SHIFT_4K		(12)
+#define BNXT_RE_PAGE_SHIFT_8K		(13)
+#define BNXT_RE_PAGE_SHIFT_64K		(16)
+#define BNXT_RE_PAGE_SHIFT_2M		(21)
+#define BNXT_RE_PAGE_SHIFT_8M		(23)
+#define BNXT_RE_PAGE_SHIFT_1G		(30)
+
+#define BNXT_RE_PAGE_SIZE_4K		BIT(BNXT_RE_PAGE_SHIFT_4K)
+#define BNXT_RE_PAGE_SIZE_8K		BIT(BNXT_RE_PAGE_SHIFT_8K)
+#define BNXT_RE_PAGE_SIZE_64K		BIT(BNXT_RE_PAGE_SHIFT_64K)
+#define BNXT_RE_PAGE_SIZE_2M		BIT(BNXT_RE_PAGE_SHIFT_2M)
+#define BNXT_RE_PAGE_SIZE_8M		BIT(BNXT_RE_PAGE_SHIFT_8M)
+#define BNXT_RE_PAGE_SIZE_1G		BIT(BNXT_RE_PAGE_SHIFT_1G)
+
+#define BNXT_RE_MAX_MR_SIZE_LOW		BIT_ULL(BNXT_RE_PAGE_SHIFT_1G)
+#define BNXT_RE_MAX_MR_SIZE_HIGH	BIT_ULL(39)
+#define BNXT_RE_MAX_MR_SIZE		BNXT_RE_MAX_MR_SIZE_HIGH
 
 #define BNXT_RE_MAX_QPC_COUNT		(64 * 1024)
 #define BNXT_RE_MAX_MRW_COUNT		(64 * 1024)
 #define BNXT_RE_MAX_SRQC_COUNT		(64 * 1024)
 #define BNXT_RE_MAX_CQ_COUNT		(64 * 1024)
+#define BNXT_RE_MAX_MRW_COUNT_64K	(64 * 1024)
+#define BNXT_RE_MAX_MRW_COUNT_256K	(256 * 1024)
+
+/* Number of MRs to reserve for PF, leaving remainder for VFs */
+#define BNXT_RE_RESVD_MR_FOR_PF         (32 * 1024)
+#define BNXT_RE_MAX_GID_PER_VF          128
+
+/*
+ * Percentage of resources of each type reserved for PF.
+ * Remaining resources are divided equally among VFs.
+ * [0, 100]
+ */
+#define BNXT_RE_PCT_RSVD_FOR_PF         50
 
 #define BNXT_RE_UD_QP_HW_STALL		0x400000
 
@@ -99,7 +120,7 @@ struct bnxt_re_dev {
 #define BNXT_RE_FLAG_HAVE_L2_REF		3
 #define BNXT_RE_FLAG_RCFW_CHANNEL_EN		4
 #define BNXT_RE_FLAG_QOS_WORK_REG		5
-#define BNXT_RE_FLAG_TASK_IN_PROG		6
+#define BNXT_RE_FLAG_ISSUE_ROCE_STATS          29
 	struct net_device		*netdev;
 	unsigned int			version, major, minor;
 	struct bnxt_en_dev		*en_dev;
@@ -136,6 +157,7 @@ struct bnxt_re_dev {
 	atomic_t			srq_count;
 	atomic_t			mr_count;
 	atomic_t			mw_count;
+	atomic_t			sched_count;
 	/* Max of 2 lossless traffic class supported per port */
 	u16				cosq[2];
 
@@ -145,6 +167,9 @@ struct bnxt_re_dev {
 	struct bnxt_re_ah		*sqp_ah;
 	struct bnxt_re_sqp_entries sqp_tbl[1024];
 	atomic_t nq_alloc_cnt;
+	u32 is_virtfn;
+	u32 num_vfs;
+	struct bnxt_qplib_roce_stats	stats;
 };
 
 #define to_bnxt_re_dev(ptr, member)	\
diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c
index 7b28219eba46..77416bc61e6e 100644
--- a/drivers/infiniband/hw/bnxt_re/hw_counters.c
+++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c
@@ -58,16 +58,55 @@
 #include "hw_counters.h"
 
 static const char * const bnxt_re_stat_name[] = {
-	[BNXT_RE_ACTIVE_QP]           =  "active_qps",
-	[BNXT_RE_ACTIVE_SRQ]          =  "active_srqs",
-	[BNXT_RE_ACTIVE_CQ]           =  "active_cqs",
-	[BNXT_RE_ACTIVE_MR]           =  "active_mrs",
-	[BNXT_RE_ACTIVE_MW]           =  "active_mws",
-	[BNXT_RE_RX_PKTS]             =  "rx_pkts",
-	[BNXT_RE_RX_BYTES]            =  "rx_bytes",
-	[BNXT_RE_TX_PKTS]             =  "tx_pkts",
-	[BNXT_RE_TX_BYTES]            =  "tx_bytes",
-	[BNXT_RE_RECOVERABLE_ERRORS]  =  "recoverable_errors"
+	[BNXT_RE_ACTIVE_QP]		=  "active_qps",
+	[BNXT_RE_ACTIVE_SRQ]		=  "active_srqs",
+	[BNXT_RE_ACTIVE_CQ]		=  "active_cqs",
+	[BNXT_RE_ACTIVE_MR]		=  "active_mrs",
+	[BNXT_RE_ACTIVE_MW]		=  "active_mws",
+	[BNXT_RE_RX_PKTS]		=  "rx_pkts",
+	[BNXT_RE_RX_BYTES]		=  "rx_bytes",
+	[BNXT_RE_TX_PKTS]		=  "tx_pkts",
+	[BNXT_RE_TX_BYTES]		=  "tx_bytes",
+	[BNXT_RE_RECOVERABLE_ERRORS]	=  "recoverable_errors",
+	[BNXT_RE_TO_RETRANSMITS]        = "to_retransmits",
+	[BNXT_RE_SEQ_ERR_NAKS_RCVD]     = "seq_err_naks_rcvd",
+	[BNXT_RE_MAX_RETRY_EXCEEDED]    = "max_retry_exceeded",
+	[BNXT_RE_RNR_NAKS_RCVD]         = "rnr_naks_rcvd",
+	[BNXT_RE_MISSING_RESP]          = "missin_resp",
+	[BNXT_RE_UNRECOVERABLE_ERR]     = "unrecoverable_err",
+	[BNXT_RE_BAD_RESP_ERR]          = "bad_resp_err",
+	[BNXT_RE_LOCAL_QP_OP_ERR]       = "local_qp_op_err",
+	[BNXT_RE_LOCAL_PROTECTION_ERR]  = "local_protection_err",
+	[BNXT_RE_MEM_MGMT_OP_ERR]       = "mem_mgmt_op_err",
+	[BNXT_RE_REMOTE_INVALID_REQ_ERR] = "remote_invalid_req_err",
+	[BNXT_RE_REMOTE_ACCESS_ERR]     = "remote_access_err",
+	[BNXT_RE_REMOTE_OP_ERR]         = "remote_op_err",
+	[BNXT_RE_DUP_REQ]               = "dup_req",
+	[BNXT_RE_RES_EXCEED_MAX]        = "res_exceed_max",
+	[BNXT_RE_RES_LENGTH_MISMATCH]   = "res_length_mismatch",
+	[BNXT_RE_RES_EXCEEDS_WQE]       = "res_exceeds_wqe",
+	[BNXT_RE_RES_OPCODE_ERR]        = "res_opcode_err",
+	[BNXT_RE_RES_RX_INVALID_RKEY]   = "res_rx_invalid_rkey",
+	[BNXT_RE_RES_RX_DOMAIN_ERR]     = "res_rx_domain_err",
+	[BNXT_RE_RES_RX_NO_PERM]        = "res_rx_no_perm",
+	[BNXT_RE_RES_RX_RANGE_ERR]      = "res_rx_range_err",
+	[BNXT_RE_RES_TX_INVALID_RKEY]   = "res_tx_invalid_rkey",
+	[BNXT_RE_RES_TX_DOMAIN_ERR]     = "res_tx_domain_err",
+	[BNXT_RE_RES_TX_NO_PERM]        = "res_tx_no_perm",
+	[BNXT_RE_RES_TX_RANGE_ERR]      = "res_tx_range_err",
+	[BNXT_RE_RES_IRRQ_OFLOW]        = "res_irrq_oflow",
+	[BNXT_RE_RES_UNSUP_OPCODE]      = "res_unsup_opcode",
+	[BNXT_RE_RES_UNALIGNED_ATOMIC]  = "res_unaligned_atomic",
+	[BNXT_RE_RES_REM_INV_ERR]       = "res_rem_inv_err",
+	[BNXT_RE_RES_MEM_ERROR]         = "res_mem_err",
+	[BNXT_RE_RES_SRQ_ERR]           = "res_srq_err",
+	[BNXT_RE_RES_CMP_ERR]           = "res_cmp_err",
+	[BNXT_RE_RES_INVALID_DUP_RKEY]  = "res_invalid_dup_rkey",
+	[BNXT_RE_RES_WQE_FORMAT_ERR]    = "res_wqe_format_err",
+	[BNXT_RE_RES_CQ_LOAD_ERR]       = "res_cq_load_err",
+	[BNXT_RE_RES_SRQ_LOAD_ERR]      = "res_srq_load_err",
+	[BNXT_RE_RES_TX_PCI_ERR]        = "res_tx_pci_err",
+	[BNXT_RE_RES_RX_PCI_ERR]        = "res_rx_pci_err"
 };
 
 int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
@@ -76,6 +115,7 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
 {
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 	struct ctx_hw_stats *bnxt_re_stats = rdev->qplib_ctx.stats.dma;
+	int rc  = 0;
 
 	if (!port || !stats)
 		return -EINVAL;
@@ -97,6 +137,91 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
 		stats->value[BNXT_RE_TX_BYTES] =
 			le64_to_cpu(bnxt_re_stats->tx_ucast_bytes);
 	}
+	if (test_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags)) {
+		rc = bnxt_qplib_get_roce_stats(&rdev->rcfw, &rdev->stats);
+		if (rc)
+			clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS,
+				  &rdev->flags);
+		stats->value[BNXT_RE_TO_RETRANSMITS] =
+					rdev->stats.to_retransmits;
+		stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] =
+					rdev->stats.seq_err_naks_rcvd;
+		stats->value[BNXT_RE_MAX_RETRY_EXCEEDED] =
+					rdev->stats.max_retry_exceeded;
+		stats->value[BNXT_RE_RNR_NAKS_RCVD] =
+					rdev->stats.rnr_naks_rcvd;
+		stats->value[BNXT_RE_MISSING_RESP] =
+					rdev->stats.missing_resp;
+		stats->value[BNXT_RE_UNRECOVERABLE_ERR] =
+					rdev->stats.unrecoverable_err;
+		stats->value[BNXT_RE_BAD_RESP_ERR] =
+					rdev->stats.bad_resp_err;
+		stats->value[BNXT_RE_LOCAL_QP_OP_ERR]	=
+				rdev->stats.local_qp_op_err;
+		stats->value[BNXT_RE_LOCAL_PROTECTION_ERR] =
+				rdev->stats.local_protection_err;
+		stats->value[BNXT_RE_MEM_MGMT_OP_ERR] =
+				rdev->stats.mem_mgmt_op_err;
+		stats->value[BNXT_RE_REMOTE_INVALID_REQ_ERR] =
+				rdev->stats.remote_invalid_req_err;
+		stats->value[BNXT_RE_REMOTE_ACCESS_ERR] =
+				rdev->stats.remote_access_err;
+		stats->value[BNXT_RE_REMOTE_OP_ERR] =
+				rdev->stats.remote_op_err;
+		stats->value[BNXT_RE_DUP_REQ] =
+				rdev->stats.dup_req;
+		stats->value[BNXT_RE_RES_EXCEED_MAX] =
+				rdev->stats.res_exceed_max;
+		stats->value[BNXT_RE_RES_LENGTH_MISMATCH] =
+				rdev->stats.res_length_mismatch;
+		stats->value[BNXT_RE_RES_EXCEEDS_WQE] =
+				rdev->stats.res_exceeds_wqe;
+		stats->value[BNXT_RE_RES_OPCODE_ERR] =
+				rdev->stats.res_opcode_err;
+		stats->value[BNXT_RE_RES_RX_INVALID_RKEY] =
+				rdev->stats.res_rx_invalid_rkey;
+		stats->value[BNXT_RE_RES_RX_DOMAIN_ERR] =
+				rdev->stats.res_rx_domain_err;
+		stats->value[BNXT_RE_RES_RX_NO_PERM] =
+				rdev->stats.res_rx_no_perm;
+		stats->value[BNXT_RE_RES_RX_RANGE_ERR]  =
+				rdev->stats.res_rx_range_err;
+		stats->value[BNXT_RE_RES_TX_INVALID_RKEY] =
+				rdev->stats.res_tx_invalid_rkey;
+		stats->value[BNXT_RE_RES_TX_DOMAIN_ERR] =
+				rdev->stats.res_tx_domain_err;
+		stats->value[BNXT_RE_RES_TX_NO_PERM] =
+				rdev->stats.res_tx_no_perm;
+		stats->value[BNXT_RE_RES_TX_RANGE_ERR]  =
+				rdev->stats.res_tx_range_err;
+		stats->value[BNXT_RE_RES_IRRQ_OFLOW] =
+				rdev->stats.res_irrq_oflow;
+		stats->value[BNXT_RE_RES_UNSUP_OPCODE]  =
+				rdev->stats.res_unsup_opcode;
+		stats->value[BNXT_RE_RES_UNALIGNED_ATOMIC] =
+				rdev->stats.res_unaligned_atomic;
+		stats->value[BNXT_RE_RES_REM_INV_ERR]   =
+				rdev->stats.res_rem_inv_err;
+		stats->value[BNXT_RE_RES_MEM_ERROR] =
+				rdev->stats.res_mem_error;
+		stats->value[BNXT_RE_RES_SRQ_ERR] =
+				rdev->stats.res_srq_err;
+		stats->value[BNXT_RE_RES_CMP_ERR] =
+				rdev->stats.res_cmp_err;
+		stats->value[BNXT_RE_RES_INVALID_DUP_RKEY] =
+				rdev->stats.res_invalid_dup_rkey;
+		stats->value[BNXT_RE_RES_WQE_FORMAT_ERR] =
+				rdev->stats.res_wqe_format_err;
+		stats->value[BNXT_RE_RES_CQ_LOAD_ERR]   =
+				rdev->stats.res_cq_load_err;
+		stats->value[BNXT_RE_RES_SRQ_LOAD_ERR]  =
+				rdev->stats.res_srq_load_err;
+		stats->value[BNXT_RE_RES_TX_PCI_ERR]    =
+				rdev->stats.res_tx_pci_err;
+		stats->value[BNXT_RE_RES_RX_PCI_ERR]    =
+				rdev->stats.res_rx_pci_err;
+	}
+
 	return ARRAY_SIZE(bnxt_re_stat_name);
 }
 
diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.h b/drivers/infiniband/hw/bnxt_re/hw_counters.h
index be0dc0093b58..a01a922717d5 100644
--- a/drivers/infiniband/hw/bnxt_re/hw_counters.h
+++ b/drivers/infiniband/hw/bnxt_re/hw_counters.h
@@ -51,6 +51,45 @@ enum bnxt_re_hw_stats {
 	BNXT_RE_TX_PKTS,
 	BNXT_RE_TX_BYTES,
 	BNXT_RE_RECOVERABLE_ERRORS,
+	BNXT_RE_TO_RETRANSMITS,
+	BNXT_RE_SEQ_ERR_NAKS_RCVD,
+	BNXT_RE_MAX_RETRY_EXCEEDED,
+	BNXT_RE_RNR_NAKS_RCVD,
+	BNXT_RE_MISSING_RESP,
+	BNXT_RE_UNRECOVERABLE_ERR,
+	BNXT_RE_BAD_RESP_ERR,
+	BNXT_RE_LOCAL_QP_OP_ERR,
+	BNXT_RE_LOCAL_PROTECTION_ERR,
+	BNXT_RE_MEM_MGMT_OP_ERR,
+	BNXT_RE_REMOTE_INVALID_REQ_ERR,
+	BNXT_RE_REMOTE_ACCESS_ERR,
+	BNXT_RE_REMOTE_OP_ERR,
+	BNXT_RE_DUP_REQ,
+	BNXT_RE_RES_EXCEED_MAX,
+	BNXT_RE_RES_LENGTH_MISMATCH,
+	BNXT_RE_RES_EXCEEDS_WQE,
+	BNXT_RE_RES_OPCODE_ERR,
+	BNXT_RE_RES_RX_INVALID_RKEY,
+	BNXT_RE_RES_RX_DOMAIN_ERR,
+	BNXT_RE_RES_RX_NO_PERM,
+	BNXT_RE_RES_RX_RANGE_ERR,
+	BNXT_RE_RES_TX_INVALID_RKEY,
+	BNXT_RE_RES_TX_DOMAIN_ERR,
+	BNXT_RE_RES_TX_NO_PERM,
+	BNXT_RE_RES_TX_RANGE_ERR,
+	BNXT_RE_RES_IRRQ_OFLOW,
+	BNXT_RE_RES_UNSUP_OPCODE,
+	BNXT_RE_RES_UNALIGNED_ATOMIC,
+	BNXT_RE_RES_REM_INV_ERR,
+	BNXT_RE_RES_MEM_ERROR,
+	BNXT_RE_RES_SRQ_ERR,
+	BNXT_RE_RES_CMP_ERR,
+	BNXT_RE_RES_INVALID_DUP_RKEY,
+	BNXT_RE_RES_WQE_FORMAT_ERR,
+	BNXT_RE_RES_CQ_LOAD_ERR,
+	BNXT_RE_RES_SRQ_LOAD_ERR,
+	BNXT_RE_RES_TX_PCI_ERR,
+	BNXT_RE_RES_RX_PCI_ERR,
 	BNXT_RE_NUM_COUNTERS
 };
 
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 2032db7db766..8301d7e5fa8c 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -141,12 +141,13 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 
 	memset(ib_attr, 0, sizeof(*ib_attr));
-
-	ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver);
+	memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
+	       min(sizeof(dev_attr->fw_ver),
+		   sizeof(ib_attr->fw_ver)));
 	bnxt_qplib_get_guid(rdev->netdev->dev_addr,
 			    (u8 *)&ib_attr->sys_image_guid);
 	ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
-	ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K;
+	ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M;
 
 	ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
 	ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
@@ -173,10 +174,8 @@ int bnxt_re_query_device(struct ib_device *ibdev,
 	ib_attr->max_pd = dev_attr->max_pd;
 	ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
 	ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
-	if (dev_attr->is_atomic) {
-		ib_attr->atomic_cap = IB_ATOMIC_HCA;
-		ib_attr->masked_atomic_cap = IB_ATOMIC_HCA;
-	}
+	ib_attr->atomic_cap = IB_ATOMIC_NONE;
+	ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
 
 	ib_attr->max_ee_rd_atom = 0;
 	ib_attr->max_res_rd_atom = 0;
@@ -247,8 +246,7 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
 				    IB_PORT_VENDOR_CLASS_SUP |
 				    IB_PORT_IP_BASED_GIDS;
 
-	/* Max MSG size set to 2G for now */
-	port_attr->max_msg_sz = 0x80000000;
+	port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
 	port_attr->bad_pkey_cntr = 0;
 	port_attr->qkey_viol_cntr = 0;
 	port_attr->pkey_tbl_len = dev_attr->max_pkey;
@@ -281,6 +279,15 @@ int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
+void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+
+	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
+		 rdev->dev_attr.fw_ver[0], rdev->dev_attr.fw_ver[1],
+		 rdev->dev_attr.fw_ver[2], rdev->dev_attr.fw_ver[3]);
+}
+
 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
 		       u16 index, u16 *pkey)
 {
@@ -532,7 +539,7 @@ static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 	mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
 	pbl_tbl = dma_addr;
 	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl_tbl,
-			       BNXT_RE_FENCE_PBL_SIZE, false);
+			       BNXT_RE_FENCE_PBL_SIZE, false, PAGE_SIZE);
 	if (rc) {
 		dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n");
 		goto fail;
@@ -778,20 +785,51 @@ int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 	return 0;
 }
 
+unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
+	__acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&qp->scq->cq_lock, flags);
+	if (qp->rcq != qp->scq)
+		spin_lock(&qp->rcq->cq_lock);
+	else
+		__acquire(&qp->rcq->cq_lock);
+
+	return flags;
+}
+
+void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
+			unsigned long flags)
+	__releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
+{
+	if (qp->rcq != qp->scq)
+		spin_unlock(&qp->rcq->cq_lock);
+	else
+		__release(&qp->rcq->cq_lock);
+	spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
+}
+
 /* Queue Pairs */
 int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
 {
 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 	struct bnxt_re_dev *rdev = qp->rdev;
 	int rc;
+	unsigned int flags;
 
 	bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
-	bnxt_qplib_del_flush_qp(&qp->qplib_qp);
 	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 	if (rc) {
 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");
 		return rc;
 	}
+
+	flags = bnxt_re_lock_cqs(qp);
+	bnxt_qplib_clean_qp(&qp->qplib_qp);
+	bnxt_re_unlock_cqs(qp, flags);
+	bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
+
 	if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) {
 		rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
 					   &rdev->sqp_ah->qplib_ah);
@@ -801,7 +839,7 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
 			return rc;
 		}
 
-		bnxt_qplib_del_flush_qp(&qp->qplib_qp);
+		bnxt_qplib_clean_qp(&qp->qplib_qp);
 		rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
 					   &rdev->qp1_sqp->qplib_qp);
 		if (rc) {
@@ -1018,6 +1056,7 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
 	struct bnxt_re_qp *qp;
 	struct bnxt_re_cq *cq;
+	struct bnxt_re_srq *srq;
 	int rc, entries;
 
 	if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
@@ -1059,6 +1098,7 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 			goto fail;
 		}
 		qp->qplib_qp.scq = &cq->qplib_cq;
+		qp->scq = cq;
 	}
 
 	if (qp_init_attr->recv_cq) {
@@ -1070,12 +1110,19 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 			goto fail;
 		}
 		qp->qplib_qp.rcq = &cq->qplib_cq;
+		qp->rcq = cq;
 	}
 
 	if (qp_init_attr->srq) {
-		dev_err(rdev_to_dev(rdev), "SRQ not supported");
-		rc = -ENOTSUPP;
-		goto fail;
+		srq = container_of(qp_init_attr->srq, struct bnxt_re_srq,
+				   ib_srq);
+		if (!srq) {
+			dev_err(rdev_to_dev(rdev), "SRQ not found");
+			rc = -EINVAL;
+			goto fail;
+		}
+		qp->qplib_qp.srq = &srq->qplib_srq;
+		qp->qplib_qp.rq.max_wqe = 0;
 	} else {
 		/* Allocate 1 more than what's provided so posting max doesn't
 		 * mean empty
@@ -1169,7 +1216,7 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 		rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
 		if (rc) {
 			dev_err(rdev_to_dev(rdev), "Failed to create HW QP");
-			goto fail;
+			goto free_umem;
 		}
 	}
 
@@ -1197,6 +1244,13 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
 	return &qp->ib_qp;
 qp_destroy:
 	bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
+free_umem:
+	if (udata) {
+		if (qp->rumem)
+			ib_umem_release(qp->rumem);
+		if (qp->sumem)
+			ib_umem_release(qp->sumem);
+	}
 fail:
 	kfree(qp);
 	return ERR_PTR(rc);
@@ -1280,6 +1334,234 @@ static enum ib_mtu __to_ib_mtu(u32 mtu)
 	}
 }
 
+/* Shared Receive Queues */
+int bnxt_re_destroy_srq(struct ib_srq *ib_srq)
+{
+	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
+					       ib_srq);
+	struct bnxt_re_dev *rdev = srq->rdev;
+	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
+	struct bnxt_qplib_nq *nq = NULL;
+	int rc;
+
+	if (qplib_srq->cq)
+		nq = qplib_srq->cq->nq;
+	rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Destroy HW SRQ failed!");
+		return rc;
+	}
+
+	if (srq->umem)
+		ib_umem_release(srq->umem);
+	kfree(srq);
+	atomic_dec(&rdev->srq_count);
+	if (nq)
+		nq->budget--;
+	return 0;
+}
+
+static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
+				 struct bnxt_re_pd *pd,
+				 struct bnxt_re_srq *srq,
+				 struct ib_udata *udata)
+{
+	struct bnxt_re_srq_req ureq;
+	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
+	struct ib_umem *umem;
+	int bytes = 0;
+	struct ib_ucontext *context = pd->ib_pd.uobject->context;
+	struct bnxt_re_ucontext *cntx = container_of(context,
+						     struct bnxt_re_ucontext,
+						     ib_uctx);
+	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
+		return -EFAULT;
+
+	bytes = (qplib_srq->max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
+	bytes = PAGE_ALIGN(bytes);
+	umem = ib_umem_get(context, ureq.srqva, bytes,
+			   IB_ACCESS_LOCAL_WRITE, 1);
+	if (IS_ERR(umem))
+		return PTR_ERR(umem);
+
+	srq->umem = umem;
+	qplib_srq->nmap = umem->nmap;
+	qplib_srq->sglist = umem->sg_head.sgl;
+	qplib_srq->srq_handle = ureq.srq_handle;
+	qplib_srq->dpi = &cntx->dpi;
+
+	return 0;
+}
+
+struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd,
+				  struct ib_srq_init_attr *srq_init_attr,
+				  struct ib_udata *udata)
+{
+	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
+	struct bnxt_re_srq *srq;
+	struct bnxt_qplib_nq *nq = NULL;
+	int rc, entries;
+
+	if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
+		dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded");
+		rc = -EINVAL;
+		goto exit;
+	}
+
+	if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
+		rc = -ENOTSUPP;
+		goto exit;
+	}
+
+	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
+	if (!srq) {
+		rc = -ENOMEM;
+		goto exit;
+	}
+	srq->rdev = rdev;
+	srq->qplib_srq.pd = &pd->qplib_pd;
+	srq->qplib_srq.dpi = &rdev->dpi_privileged;
+	/* Allocate 1 more than what's provided so posting max doesn't
+	 * mean empty
+	 */
+	entries = roundup_pow_of_two(srq_init_attr->attr.max_wr + 1);
+	if (entries > dev_attr->max_srq_wqes + 1)
+		entries = dev_attr->max_srq_wqes + 1;
+
+	srq->qplib_srq.max_wqe = entries;
+	srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
+	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
+	srq->srq_limit = srq_init_attr->attr.srq_limit;
+	srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
+	nq = &rdev->nq[0];
+
+	if (udata) {
+		rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
+		if (rc)
+			goto fail;
+	}
+
+	rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!");
+		goto fail;
+	}
+
+	if (udata) {
+		struct bnxt_re_srq_resp resp;
+
+		resp.srqid = srq->qplib_srq.id;
+		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "SRQ copy to udata failed!");
+			bnxt_qplib_destroy_srq(&rdev->qplib_res,
+					       &srq->qplib_srq);
+			goto exit;
+		}
+	}
+	if (nq)
+		nq->budget++;
+	atomic_inc(&rdev->srq_count);
+
+	return &srq->ib_srq;
+
+fail:
+	if (srq->umem)
+		ib_umem_release(srq->umem);
+	kfree(srq);
+exit:
+	return ERR_PTR(rc);
+}
+
+int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
+		       enum ib_srq_attr_mask srq_attr_mask,
+		       struct ib_udata *udata)
+{
+	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
+					       ib_srq);
+	struct bnxt_re_dev *rdev = srq->rdev;
+	int rc;
+
+	switch (srq_attr_mask) {
+	case IB_SRQ_MAX_WR:
+		/* SRQ resize is not supported */
+		break;
+	case IB_SRQ_LIMIT:
+		/* Change the SRQ threshold */
+		if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
+			return -EINVAL;
+
+		srq->qplib_srq.threshold = srq_attr->srq_limit;
+		rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!");
+			return rc;
+		}
+		/* On success, update the shadow */
+		srq->srq_limit = srq_attr->srq_limit;
+		/* No need to Build and send response back to udata */
+		break;
+	default:
+		dev_err(rdev_to_dev(rdev),
+			"Unsupported srq_attr_mask 0x%x", srq_attr_mask);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
+{
+	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
+					       ib_srq);
+	struct bnxt_re_srq tsrq;
+	struct bnxt_re_dev *rdev = srq->rdev;
+	int rc;
+
+	/* Get live SRQ attr */
+	tsrq.qplib_srq.id = srq->qplib_srq.id;
+	rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Query HW SRQ failed!");
+		return rc;
+	}
+	srq_attr->max_wr = srq->qplib_srq.max_wqe;
+	srq_attr->max_sge = srq->qplib_srq.max_sge;
+	srq_attr->srq_limit = tsrq.qplib_srq.threshold;
+
+	return 0;
+}
+
+int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, struct ib_recv_wr *wr,
+			  struct ib_recv_wr **bad_wr)
+{
+	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
+					       ib_srq);
+	struct bnxt_qplib_swqe wqe;
+	unsigned long flags;
+	int rc = 0, payload_sz = 0;
+
+	spin_lock_irqsave(&srq->lock, flags);
+	while (wr) {
+		/* Transcribe each ib_recv_wr to qplib_swqe */
+		wqe.num_sge = wr->num_sge;
+		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
+					       wr->num_sge);
+		wqe.wr_id = wr->wr_id;
+		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
+
+		rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
+		if (rc) {
+			*bad_wr = wr;
+			break;
+		}
+		wr = wr->next;
+	}
+	spin_unlock_irqrestore(&srq->lock, flags);
+
+	return rc;
+}
 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
 				    struct bnxt_re_qp *qp1_qp,
 				    int qp_attr_mask)
@@ -1324,6 +1606,7 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 	int status;
 	union ib_gid sgid;
 	struct ib_gid_attr sgid_attr;
+	unsigned int flags;
 	u8 nw_type;
 
 	qp->qplib_qp.modify_flags = 0;
@@ -1352,14 +1635,18 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 			dev_dbg(rdev_to_dev(rdev),
 				"Move QP = %p to flush list\n",
 				qp);
+			flags = bnxt_re_lock_cqs(qp);
 			bnxt_qplib_add_flush_qp(&qp->qplib_qp);
+			bnxt_re_unlock_cqs(qp, flags);
 		}
 		if (!qp->sumem &&
 		    qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
 			dev_dbg(rdev_to_dev(rdev),
 				"Move QP = %p out of flush list\n",
 				qp);
-			bnxt_qplib_del_flush_qp(&qp->qplib_qp);
+			flags = bnxt_re_lock_cqs(qp);
+			bnxt_qplib_clean_qp(&qp->qplib_qp);
+			bnxt_re_unlock_cqs(qp, flags);
 		}
 	}
 	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
@@ -1945,10 +2232,13 @@ static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr,
 	wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
 	wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
 
+	/* Need unconditional fence for local invalidate
+	 * opcode to work as expected.
+	 */
+	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+
 	if (wr->send_flags & IB_SEND_SIGNALED)
 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
-	if (wr->send_flags & IB_SEND_FENCE)
-		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
 	if (wr->send_flags & IB_SEND_SOLICITED)
 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
 
@@ -1969,8 +2259,12 @@ static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr,
 	wqe->frmr.levels = qplib_frpl->hwq.level + 1;
 	wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
 
-	if (wr->wr.send_flags & IB_SEND_FENCE)
-		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+	/* Need unconditional fence for reg_mr
+	 * opcode to function as expected.
+	 */
+
+	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+
 	if (wr->wr.send_flags & IB_SEND_SIGNALED)
 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
 
@@ -2295,10 +2589,14 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr,
 /* Completion Queues */
 int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
 {
-	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
-	struct bnxt_re_dev *rdev = cq->rdev;
 	int rc;
-	struct bnxt_qplib_nq *nq = cq->qplib_cq.nq;
+	struct bnxt_re_cq *cq;
+	struct bnxt_qplib_nq *nq;
+	struct bnxt_re_dev *rdev;
+
+	cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
+	rdev = cq->rdev;
+	nq = cq->qplib_cq.nq;
 
 	rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
 	if (rc) {
@@ -2308,12 +2606,11 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
 	if (!IS_ERR_OR_NULL(cq->umem))
 		ib_umem_release(cq->umem);
 
-	if (cq) {
-		kfree(cq->cql);
-		kfree(cq);
-	}
 	atomic_dec(&rdev->cq_count);
 	nq->budget--;
+	kfree(cq->cql);
+	kfree(cq);
+
 	return 0;
 }
 
@@ -3078,7 +3375,8 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
 
 	mr->qplib_mr.hwq.level = PBL_LVL_MAX;
 	mr->qplib_mr.total_size = -1; /* Infinte length */
-	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false);
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false,
+			       PAGE_SIZE);
 	if (rc)
 		goto fail_mr;
 
@@ -3104,10 +3402,8 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr)
 	int rc;
 
 	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
-	if (rc) {
+	if (rc)
 		dev_err(rdev_to_dev(rdev), "Dereg MR failed: %#x\n", rc);
-		return rc;
-	}
 
 	if (mr->pages) {
 		rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
@@ -3170,7 +3466,7 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
 
 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
 	if (rc)
-		goto fail;
+		goto bail;
 
 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
 	mr->ib_mr.rkey = mr->ib_mr.lkey;
@@ -3192,9 +3488,10 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
 	return &mr->ib_mr;
 
 fail_mr:
-	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
-fail:
 	kfree(mr->pages);
+fail:
+	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+bail:
 	kfree(mr);
 	return ERR_PTR(rc);
 }
@@ -3248,6 +3545,46 @@ int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
 	return rc;
 }
 
+static int bnxt_re_page_size_ok(int page_shift)
+{
+	switch (page_shift) {
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G:
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig,
+			     int page_shift)
+{
+	u64 *pbl_tbl = pbl_tbl_orig;
+	u64 paddr;
+	u64 page_mask = (1ULL << page_shift) - 1;
+	int i, pages;
+	struct scatterlist *sg;
+	int entry;
+
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
+		pages = sg_dma_len(sg) >> PAGE_SHIFT;
+		for (i = 0; i < pages; i++) {
+			paddr = sg_dma_address(sg) + (i << PAGE_SHIFT);
+			if (pbl_tbl == pbl_tbl_orig)
+				*pbl_tbl++ = paddr & ~page_mask;
+			else if ((paddr & page_mask) == 0)
+				*pbl_tbl++ = paddr;
+		}
+	}
+	return pbl_tbl - pbl_tbl_orig;
+}
+
 /* uverbs */
 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 				  u64 virt_addr, int mr_access_flags,
@@ -3257,13 +3594,11 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 	struct bnxt_re_dev *rdev = pd->rdev;
 	struct bnxt_re_mr *mr;
 	struct ib_umem *umem;
-	u64 *pbl_tbl, *pbl_tbl_orig;
-	int i, umem_pgs, pages, rc;
-	struct scatterlist *sg;
-	int entry;
+	u64 *pbl_tbl = NULL;
+	int umem_pgs, page_shift, rc;
 
 	if (length > BNXT_RE_MAX_MR_SIZE) {
-		dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%ld\n",
+		dev_err(rdev_to_dev(rdev), "MR Size: %lld > Max supported:%lld\n",
 			length, BNXT_RE_MAX_MR_SIZE);
 		return ERR_PTR(-ENOMEM);
 	}
@@ -3277,64 +3612,68 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
 
+	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed to allocate MR");
+		goto free_mr;
+	}
+	/* The fixed portion of the rkey is the same as the lkey */
+	mr->ib_mr.rkey = mr->qplib_mr.rkey;
+
 	umem = ib_umem_get(ib_pd->uobject->context, start, length,
 			   mr_access_flags, 0);
 	if (IS_ERR(umem)) {
 		dev_err(rdev_to_dev(rdev), "Failed to get umem");
 		rc = -EFAULT;
-		goto free_mr;
+		goto free_mrw;
 	}
 	mr->ib_umem = umem;
 
-	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
-	if (rc) {
-		dev_err(rdev_to_dev(rdev), "Failed to allocate MR");
-		goto release_umem;
-	}
-	/* The fixed portion of the rkey is the same as the lkey */
-	mr->ib_mr.rkey = mr->qplib_mr.rkey;
-
 	mr->qplib_mr.va = virt_addr;
 	umem_pgs = ib_umem_page_count(umem);
 	if (!umem_pgs) {
 		dev_err(rdev_to_dev(rdev), "umem is invalid!");
 		rc = -EINVAL;
-		goto free_mrw;
+		goto free_umem;
 	}
 	mr->qplib_mr.total_size = length;
 
 	pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL);
 	if (!pbl_tbl) {
-		rc = -EINVAL;
-		goto free_mrw;
+		rc = -ENOMEM;
+		goto free_umem;
 	}
-	pbl_tbl_orig = pbl_tbl;
 
-	if (umem->hugetlb) {
-		dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!");
+	page_shift = umem->page_shift;
+
+	if (!bnxt_re_page_size_ok(page_shift)) {
+		dev_err(rdev_to_dev(rdev), "umem page size unsupported!");
 		rc = -EFAULT;
 		goto fail;
 	}
 
-	if (umem->page_shift != PAGE_SHIFT) {
-		dev_err(rdev_to_dev(rdev), "umem page shift unsupported!");
-		rc = -EFAULT;
+	if (!umem->hugetlb && length > BNXT_RE_MAX_MR_SIZE_LOW) {
+		dev_err(rdev_to_dev(rdev), "Requested MR Sz:%llu Max sup:%llu",
+			length,	(u64)BNXT_RE_MAX_MR_SIZE_LOW);
+		rc = -EINVAL;
 		goto fail;
 	}
-	/* Map umem buf ptrs to the PBL */
-	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
-		pages = sg_dma_len(sg) >> umem->page_shift;
-		for (i = 0; i < pages; i++, pbl_tbl++)
-			*pbl_tbl = sg_dma_address(sg) + (i << umem->page_shift);
+	if (umem->hugetlb && length > BNXT_RE_PAGE_SIZE_2M) {
+		page_shift = BNXT_RE_PAGE_SHIFT_2M;
+		dev_warn(rdev_to_dev(rdev), "umem hugetlb set page_size %x",
+			 1 << page_shift);
 	}
-	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig,
-			       umem_pgs, false);
+
+	/* Map umem buf ptrs to the PBL */
+	umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, page_shift);
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl,
+			       umem_pgs, false, 1 << page_shift);
 	if (rc) {
 		dev_err(rdev_to_dev(rdev), "Failed to register user MR");
 		goto fail;
 	}
 
-	kfree(pbl_tbl_orig);
+	kfree(pbl_tbl);
 
 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
 	mr->ib_mr.rkey = mr->qplib_mr.lkey;
@@ -3342,11 +3681,11 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 
 	return &mr->ib_mr;
 fail:
-	kfree(pbl_tbl_orig);
+	kfree(pbl_tbl);
+free_umem:
+	ib_umem_release(umem);
 free_mrw:
 	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
-release_umem:
-	ib_umem_release(umem);
 free_mr:
 	kfree(mr);
 	return ERR_PTR(rc);
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index 1df11ed272ea..e62b7c2c7da6 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -68,6 +68,15 @@ struct bnxt_re_ah {
 	struct bnxt_qplib_ah	qplib_ah;
 };
 
+struct bnxt_re_srq {
+	struct bnxt_re_dev	*rdev;
+	u32			srq_limit;
+	struct ib_srq		ib_srq;
+	struct bnxt_qplib_srq	qplib_srq;
+	struct ib_umem		*umem;
+	spinlock_t		lock;		/* protect srq */
+};
+
 struct bnxt_re_qp {
 	struct list_head	list;
 	struct bnxt_re_dev	*rdev;
@@ -80,6 +89,8 @@ struct bnxt_re_qp {
 	/* QP1 */
 	u32			send_psn;
 	struct ib_ud_header	qp1_hdr;
+	struct bnxt_re_cq	*scq;
+	struct bnxt_re_cq	*rcq;
 };
 
 struct bnxt_re_cq {
@@ -143,6 +154,7 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
 		       struct ib_port_attr *port_attr);
 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
 			       struct ib_port_immutable *immutable);
+void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str);
 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
 		       u16 index, u16 *pkey);
 int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
@@ -164,6 +176,16 @@ struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd,
 int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
 int bnxt_re_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
 int bnxt_re_destroy_ah(struct ib_ah *ah);
+struct ib_srq *bnxt_re_create_srq(struct ib_pd *pd,
+				  struct ib_srq_init_attr *srq_init_attr,
+				  struct ib_udata *udata);
+int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
+		       enum ib_srq_attr_mask srq_attr_mask,
+		       struct ib_udata *udata);
+int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
+int bnxt_re_destroy_srq(struct ib_srq *srq);
+int bnxt_re_post_srq_recv(struct ib_srq *srq, struct ib_recv_wr *recv_wr,
+			  struct ib_recv_wr **bad_recv_wr);
 struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd,
 				struct ib_qp_init_attr *qp_init_attr,
 				struct ib_udata *udata);
@@ -200,4 +222,7 @@ struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev,
 					   struct ib_udata *udata);
 int bnxt_re_dealloc_ucontext(struct ib_ucontext *context);
 int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+
+unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp);
+void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags);
 #endif /* __BNXT_RE_IB_VERBS_H__ */
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index aafc19aa5de1..f6e361750466 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -80,6 +80,79 @@ static DEFINE_MUTEX(bnxt_re_dev_lock);
 static struct workqueue_struct *bnxt_re_wq;
 static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait);
 
+/* SR-IOV helper functions */
+
+static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev)
+{
+	struct bnxt *bp;
+
+	bp = netdev_priv(rdev->en_dev->net);
+	if (BNXT_VF(bp))
+		rdev->is_virtfn = 1;
+}
+
+/* Set the maximum number of each resource that the driver actually wants
+ * to allocate. This may be up to the maximum number the firmware has
+ * reserved for the function. The driver may choose to allocate fewer
+ * resources than the firmware maximum.
+ */
+static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
+{
+	u32 vf_qps = 0, vf_srqs = 0, vf_cqs = 0, vf_mrws = 0, vf_gids = 0;
+	u32 i;
+	u32 vf_pct;
+	u32 num_vfs;
+	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
+
+	rdev->qplib_ctx.qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,
+					  dev_attr->max_qp);
+
+	rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;
+	/* Use max_mr from fw since max_mrw does not get set */
+	rdev->qplib_ctx.mrw_count = min_t(u32, rdev->qplib_ctx.mrw_count,
+					  dev_attr->max_mr);
+	rdev->qplib_ctx.srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,
+					   dev_attr->max_srq);
+	rdev->qplib_ctx.cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT,
+					 dev_attr->max_cq);
+
+	for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
+		rdev->qplib_ctx.tqm_count[i] =
+		rdev->dev_attr.tqm_alloc_reqs[i];
+
+	if (rdev->num_vfs) {
+		/*
+		 * Reserve a set of resources for the PF. Divide the remaining
+		 * resources among the VFs
+		 */
+		vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;
+		num_vfs = 100 * rdev->num_vfs;
+		vf_qps = (rdev->qplib_ctx.qpc_count * vf_pct) / num_vfs;
+		vf_srqs = (rdev->qplib_ctx.srqc_count * vf_pct) / num_vfs;
+		vf_cqs = (rdev->qplib_ctx.cq_count * vf_pct) / num_vfs;
+		/*
+		 * The driver allows many more MRs than other resources. If the
+		 * firmware does also, then reserve a fixed amount for the PF
+		 * and divide the rest among VFs. VFs may use many MRs for NFS
+		 * mounts, ISER, NVME applications, etc. If the firmware
+		 * severely restricts the number of MRs, then let PF have
+		 * half and divide the rest among VFs, as for the other
+		 * resource types.
+		 */
+		if (rdev->qplib_ctx.mrw_count < BNXT_RE_MAX_MRW_COUNT_64K)
+			vf_mrws = rdev->qplib_ctx.mrw_count * vf_pct / num_vfs;
+		else
+			vf_mrws = (rdev->qplib_ctx.mrw_count -
+				   BNXT_RE_RESVD_MR_FOR_PF) / rdev->num_vfs;
+		vf_gids = BNXT_RE_MAX_GID_PER_VF;
+	}
+	rdev->qplib_ctx.vf_res.max_mrw_per_vf = vf_mrws;
+	rdev->qplib_ctx.vf_res.max_gid_per_vf = vf_gids;
+	rdev->qplib_ctx.vf_res.max_qp_per_vf = vf_qps;
+	rdev->qplib_ctx.vf_res.max_srq_per_vf = vf_srqs;
+	rdev->qplib_ctx.vf_res.max_cq_per_vf = vf_cqs;
+}
+
 /* for handling bnxt_en callbacks later */
 static void bnxt_re_stop(void *p)
 {
@@ -91,6 +164,15 @@ static void bnxt_re_start(void *p)
 
 static void bnxt_re_sriov_config(void *p, int num_vfs)
 {
+	struct bnxt_re_dev *rdev = p;
+
+	if (!rdev)
+		return;
+
+	rdev->num_vfs = num_vfs;
+	bnxt_re_set_resource_limits(rdev);
+	bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw,
+				      &rdev->qplib_ctx);
 }
 
 static void bnxt_re_shutdown(void *p)
@@ -417,7 +499,7 @@ static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)
 		return ERR_PTR(-EINVAL);
 
 	if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) {
-		dev_dbg(&pdev->dev,
+		dev_info(&pdev->dev,
 			"%s: probe error: RoCE is not supported on this device",
 			ROCE_DRV_MODULE_NAME);
 		return ERR_PTR(-ENODEV);
@@ -490,6 +572,7 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
 
 	ibdev->query_port		= bnxt_re_query_port;
 	ibdev->get_port_immutable	= bnxt_re_get_port_immutable;
+	ibdev->get_dev_fw_str           = bnxt_re_query_fw_str;
 	ibdev->query_pkey		= bnxt_re_query_pkey;
 	ibdev->query_gid		= bnxt_re_query_gid;
 	ibdev->get_netdev		= bnxt_re_get_netdev;
@@ -505,6 +588,12 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
 	ibdev->query_ah			= bnxt_re_query_ah;
 	ibdev->destroy_ah		= bnxt_re_destroy_ah;
 
+	ibdev->create_srq		= bnxt_re_create_srq;
+	ibdev->modify_srq		= bnxt_re_modify_srq;
+	ibdev->query_srq		= bnxt_re_query_srq;
+	ibdev->destroy_srq		= bnxt_re_destroy_srq;
+	ibdev->post_srq_recv		= bnxt_re_post_srq_recv;
+
 	ibdev->create_qp		= bnxt_re_create_qp;
 	ibdev->modify_qp		= bnxt_re_modify_qp;
 	ibdev->query_qp			= bnxt_re_query_qp;
@@ -541,14 +630,6 @@ static ssize_t show_rev(struct device *device, struct device_attribute *attr,
 	return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
 }
 
-static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
-			   char *buf)
-{
-	struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
-
-	return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->dev_attr.fw_ver);
-}
-
 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
 			char *buf)
 {
@@ -558,12 +639,10 @@ static ssize_t show_hca(struct device *device, struct device_attribute *attr,
 }
 
 static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL);
-static DEVICE_ATTR(fw_rev, 0444, show_fw_ver, NULL);
 static DEVICE_ATTR(hca_type, 0444, show_hca, NULL);
 
 static struct device_attribute *bnxt_re_attributes[] = {
 	&dev_attr_hw_rev,
-	&dev_attr_fw_rev,
 	&dev_attr_hca_type
 };
 
@@ -577,7 +656,6 @@ static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
 	mutex_unlock(&bnxt_re_dev_lock);
 
 	synchronize_rcu();
-	flush_workqueue(bnxt_re_wq);
 
 	ib_dealloc_device(&rdev->ibdev);
 	/* rdev is gone */
@@ -616,10 +694,10 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
 	return rdev;
 }
 
-static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
-			       struct creq_func_event *aeqe)
+static int bnxt_re_handle_unaffi_async_event(struct creq_func_event
+					     *unaffi_async)
 {
-	switch (aeqe->event) {
+	switch (unaffi_async->event) {
 	case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
 		break;
 	case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
@@ -648,6 +726,100 @@ static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
 	return 0;
 }
 
+static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event,
+					 struct bnxt_re_qp *qp)
+{
+	struct ib_event event;
+	unsigned int flags;
+
+	if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+		flags = bnxt_re_lock_cqs(qp);
+		bnxt_qplib_add_flush_qp(&qp->qplib_qp);
+		bnxt_re_unlock_cqs(qp, flags);
+	}
+
+	memset(&event, 0, sizeof(event));
+	if (qp->qplib_qp.srq) {
+		event.device = &qp->rdev->ibdev;
+		event.element.qp = &qp->ib_qp;
+		event.event = IB_EVENT_QP_LAST_WQE_REACHED;
+	}
+
+	if (event.device && qp->ib_qp.event_handler)
+		qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+
+	return 0;
+}
+
+static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,
+					   void *obj)
+{
+	int rc = 0;
+	u8 event;
+
+	if (!obj)
+		return rc; /* QP was already dead, still return success */
+
+	event = affi_async->event;
+	if (event == CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION) {
+		struct bnxt_qplib_qp *lib_qp = obj;
+		struct bnxt_re_qp *qp = container_of(lib_qp, struct bnxt_re_qp,
+						     qplib_qp);
+		rc = bnxt_re_handle_qp_async_event(affi_async, qp);
+	}
+	return rc;
+}
+
+static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
+			       void *aeqe, void *obj)
+{
+	struct creq_qp_event *affi_async;
+	struct creq_func_event *unaffi_async;
+	u8 type;
+	int rc;
+
+	type = ((struct creq_base *)aeqe)->type;
+	if (type == CREQ_BASE_TYPE_FUNC_EVENT) {
+		unaffi_async = aeqe;
+		rc = bnxt_re_handle_unaffi_async_event(unaffi_async);
+	} else {
+		affi_async = aeqe;
+		rc = bnxt_re_handle_affi_async_event(affi_async, obj);
+	}
+
+	return rc;
+}
+
+static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
+				struct bnxt_qplib_srq *handle, u8 event)
+{
+	struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq,
+					       qplib_srq);
+	struct ib_event ib_event;
+	int rc = 0;
+
+	if (!srq) {
+		dev_err(NULL, "%s: SRQ is NULL, SRQN not handled",
+			ROCE_DRV_MODULE_NAME);
+		rc = -EINVAL;
+		goto done;
+	}
+	ib_event.device = &srq->rdev->ibdev;
+	ib_event.element.srq = &srq->ib_srq;
+	if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)
+		ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;
+	else
+		ib_event.event = IB_EVENT_SRQ_ERR;
+
+	if (srq->ib_srq.event_handler) {
+		/* Lock event_handler? */
+		(*srq->ib_srq.event_handler)(&ib_event,
+					     srq->ib_srq.srq_context);
+	}
+done:
+	return rc;
+}
+
 static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
 			       struct bnxt_qplib_cq *handle)
 {
@@ -690,7 +862,8 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
 		rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
 					  i - 1, rdev->msix_entries[i].vector,
 					  rdev->msix_entries[i].db_offset,
-					  &bnxt_re_cqn_handler, NULL);
+					  &bnxt_re_cqn_handler,
+					  &bnxt_re_srqn_handler);
 
 		if (rc) {
 			dev_err(rdev_to_dev(rdev),
@@ -734,7 +907,8 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
 
 	/* Configure and allocate resources for qplib */
 	rdev->qplib_res.rcfw = &rdev->rcfw;
-	rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
+	rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr,
+				     rdev->is_virtfn);
 	if (rc)
 		goto fail;
 
@@ -1035,19 +1209,6 @@ static void bnxt_re_ib_unreg(struct bnxt_re_dev *rdev, bool lock_wait)
 	}
 }
 
-static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
-{
-	u32 i;
-
-	rdev->qplib_ctx.qpc_count = BNXT_RE_MAX_QPC_COUNT;
-	rdev->qplib_ctx.mrw_count = BNXT_RE_MAX_MRW_COUNT;
-	rdev->qplib_ctx.srqc_count = BNXT_RE_MAX_SRQC_COUNT;
-	rdev->qplib_ctx.cq_count = BNXT_RE_MAX_CQ_COUNT;
-	for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
-		rdev->qplib_ctx.tqm_count[i] =
-		rdev->dev_attr.tqm_alloc_reqs[i];
-}
-
 /* worker thread for polling periodic events. Now used for QoS programming*/
 static void bnxt_re_worker(struct work_struct *work)
 {
@@ -1070,6 +1231,9 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
 	}
 	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
 
+	/* Check whether VF or PF */
+	bnxt_re_get_sriov_func_type(rdev);
+
 	rc = bnxt_re_request_msix(rdev);
 	if (rc) {
 		pr_err("Failed to get MSI-X vectors: %#x\n", rc);
@@ -1101,16 +1265,18 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
 				(rdev->en_dev->pdev, &rdev->rcfw,
 				 rdev->msix_entries[BNXT_RE_AEQ_IDX].vector,
 				 rdev->msix_entries[BNXT_RE_AEQ_IDX].db_offset,
-				 0, &bnxt_re_aeq_handler);
+				 rdev->is_virtfn, &bnxt_re_aeq_handler);
 	if (rc) {
 		pr_err("Failed to enable RCFW channel: %#x\n", rc);
 		goto free_ring;
 	}
 
-	rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
+	rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr,
+				     rdev->is_virtfn);
 	if (rc)
 		goto disable_rcfw;
-	bnxt_re_set_resource_limits(rdev);
+	if (!rdev->is_virtfn)
+		bnxt_re_set_resource_limits(rdev);
 
 	rc = bnxt_qplib_alloc_ctx(rdev->en_dev->pdev, &rdev->qplib_ctx, 0);
 	if (rc) {
@@ -1125,7 +1291,8 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
 		goto free_ctx;
 	}
 
-	rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx, 0);
+	rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx,
+				  rdev->is_virtfn);
 	if (rc) {
 		pr_err("Failed to initialize RCFW: %#x\n", rc);
 		goto free_sctx;
@@ -1144,13 +1311,15 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
 		goto fail;
 	}
 
-	rc = bnxt_re_setup_qos(rdev);
-	if (rc)
-		pr_info("RoCE priority not yet configured\n");
+	if (!rdev->is_virtfn) {
+		rc = bnxt_re_setup_qos(rdev);
+		if (rc)
+			pr_info("RoCE priority not yet configured\n");
 
-	INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
-	set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
-	schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
+		INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
+		set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
+		schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
+	}
 
 	/* Register ib dev */
 	rc = bnxt_re_register_ib(rdev);
@@ -1176,6 +1345,7 @@ static int bnxt_re_ib_reg(struct bnxt_re_dev *rdev)
 	set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
 	ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
 			 &rdev->active_width);
+	set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
 	bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
 	bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE);
 
@@ -1253,9 +1423,12 @@ static void bnxt_re_task(struct work_struct *work)
 	switch (re_work->event) {
 	case NETDEV_REGISTER:
 		rc = bnxt_re_ib_reg(rdev);
-		if (rc)
+		if (rc) {
 			dev_err(rdev_to_dev(rdev),
 				"Failed to register with IB: %#x", rc);
+			bnxt_re_remove_one(rdev);
+			bnxt_re_dev_unreg(rdev);
+		}
 		break;
 	case NETDEV_UP:
 		bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
@@ -1277,7 +1450,7 @@ static void bnxt_re_task(struct work_struct *work)
 		break;
 	}
 	smp_mb__before_atomic();
-	clear_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags);
+	atomic_dec(&rdev->sched_count);
 	kfree(re_work);
 }
 
@@ -1339,7 +1512,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
 		/* netdev notifier will call NETDEV_UNREGISTER again later since
 		 * we are still holding the reference to the netdev
 		 */
-		if (test_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags))
+		if (atomic_read(&rdev->sched_count) > 0)
 			goto exit;
 		bnxt_re_ib_unreg(rdev, false);
 		bnxt_re_remove_one(rdev);
@@ -1359,7 +1532,7 @@ static int bnxt_re_netdev_event(struct notifier_block *notifier,
 			re_work->vlan_dev = (real_dev == netdev ?
 					     NULL : netdev);
 			INIT_WORK(&re_work->work, bnxt_re_task);
-			set_bit(BNXT_RE_FLAG_TASK_IN_PROG, &rdev->flags);
+			atomic_inc(&rdev->sched_count);
 			queue_work(bnxt_re_wq, &re_work->work);
 		}
 	}
@@ -1400,7 +1573,7 @@ static int __init bnxt_re_mod_init(void)
 
 static void __exit bnxt_re_mod_exit(void)
 {
-	struct bnxt_re_dev *rdev;
+	struct bnxt_re_dev *rdev, *next;
 	LIST_HEAD(to_be_deleted);
 
 	mutex_lock(&bnxt_re_dev_lock);
@@ -1408,9 +1581,17 @@ static void __exit bnxt_re_mod_exit(void)
 	if (!list_empty(&bnxt_re_dev_list))
 		list_splice_init(&bnxt_re_dev_list, &to_be_deleted);
 	mutex_unlock(&bnxt_re_dev_lock);
-
-	list_for_each_entry(rdev, &to_be_deleted, list) {
+       /*
+	* Cleanup the devices in reverse order so that the VF device
+	* cleanup is done before PF cleanup
+	*/
+	list_for_each_entry_safe_reverse(rdev, next, &to_be_deleted, list) {
 		dev_info(rdev_to_dev(rdev), "Unregistering Device");
+		/*
+		 * Flush out any scheduled tasks before destroying the
+		 * resources
+		 */
+		flush_workqueue(bnxt_re_wq);
 		bnxt_re_dev_stop(rdev);
 		bnxt_re_ib_unreg(rdev, true);
 		bnxt_re_remove_one(rdev);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 61764f7aa79b..3a78faba8d91 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -52,6 +52,7 @@
 
 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
+static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type);
 
 static void bnxt_qplib_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
 {
@@ -87,75 +88,35 @@ static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
 	}
 }
 
-void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
-				 unsigned long *flags)
-	__acquires(&qp->scq->hwq.lock) __acquires(&qp->rcq->hwq.lock)
+static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp,
+				       unsigned long *flags)
+	__acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
 {
-	spin_lock_irqsave(&qp->scq->hwq.lock, *flags);
+	spin_lock_irqsave(&qp->scq->flush_lock, *flags);
 	if (qp->scq == qp->rcq)
-		__acquire(&qp->rcq->hwq.lock);
+		__acquire(&qp->rcq->flush_lock);
 	else
-		spin_lock(&qp->rcq->hwq.lock);
+		spin_lock(&qp->rcq->flush_lock);
 }
 
-void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp,
-				 unsigned long *flags)
-	__releases(&qp->scq->hwq.lock) __releases(&qp->rcq->hwq.lock)
+static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp,
+				       unsigned long *flags)
+	__releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
 {
 	if (qp->scq == qp->rcq)
-		__release(&qp->rcq->hwq.lock);
+		__release(&qp->rcq->flush_lock);
 	else
-		spin_unlock(&qp->rcq->hwq.lock);
-	spin_unlock_irqrestore(&qp->scq->hwq.lock, *flags);
-}
-
-static struct bnxt_qplib_cq *bnxt_qplib_find_buddy_cq(struct bnxt_qplib_qp *qp,
-						      struct bnxt_qplib_cq *cq)
-{
-	struct bnxt_qplib_cq *buddy_cq = NULL;
-
-	if (qp->scq == qp->rcq)
-		buddy_cq = NULL;
-	else if (qp->scq == cq)
-		buddy_cq = qp->rcq;
-	else
-		buddy_cq = qp->scq;
-	return buddy_cq;
-}
-
-static void bnxt_qplib_lock_buddy_cq(struct bnxt_qplib_qp *qp,
-				     struct bnxt_qplib_cq *cq)
-	__acquires(&buddy_cq->hwq.lock)
-{
-	struct bnxt_qplib_cq *buddy_cq = NULL;
-
-	buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq);
-	if (!buddy_cq)
-		__acquire(&cq->hwq.lock);
-	else
-		spin_lock(&buddy_cq->hwq.lock);
-}
-
-static void bnxt_qplib_unlock_buddy_cq(struct bnxt_qplib_qp *qp,
-				       struct bnxt_qplib_cq *cq)
-	__releases(&buddy_cq->hwq.lock)
-{
-	struct bnxt_qplib_cq *buddy_cq = NULL;
-
-	buddy_cq = bnxt_qplib_find_buddy_cq(qp, cq);
-	if (!buddy_cq)
-		__release(&cq->hwq.lock);
-	else
-		spin_unlock(&buddy_cq->hwq.lock);
+		spin_unlock(&qp->rcq->flush_lock);
+	spin_unlock_irqrestore(&qp->scq->flush_lock, *flags);
 }
 
 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
 {
 	unsigned long flags;
 
-	bnxt_qplib_acquire_cq_locks(qp, &flags);
+	bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
 	__bnxt_qplib_add_flush_qp(qp);
-	bnxt_qplib_release_cq_locks(qp, &flags);
+	bnxt_qplib_release_cq_flush_locks(qp, &flags);
 }
 
 static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
@@ -172,11 +133,11 @@ static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
 	}
 }
 
-void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
 {
 	unsigned long flags;
 
-	bnxt_qplib_acquire_cq_locks(qp, &flags);
+	bnxt_qplib_acquire_cq_flush_locks(qp, &flags);
 	__clean_cq(qp->scq, (u64)(unsigned long)qp);
 	qp->sq.hwq.prod = 0;
 	qp->sq.hwq.cons = 0;
@@ -185,7 +146,7 @@ void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
 	qp->rq.hwq.cons = 0;
 
 	__bnxt_qplib_del_flush_qp(qp);
-	bnxt_qplib_release_cq_locks(qp, &flags);
+	bnxt_qplib_release_cq_flush_locks(qp, &flags);
 }
 
 static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
@@ -278,10 +239,11 @@ static void bnxt_qplib_service_nq(unsigned long data)
 	struct nq_base *nqe, **nq_ptr;
 	struct bnxt_qplib_cq *cq;
 	int num_cqne_processed = 0;
+	int num_srqne_processed = 0;
 	u32 sw_cons, raw_cons;
 	u16 type;
 	int budget = nq->budget;
-	u64 q_handle;
+	uintptr_t q_handle;
 
 	/* Service the NQ until empty */
 	raw_cons = hwq->cons;
@@ -320,6 +282,26 @@ static void bnxt_qplib_service_nq(unsigned long data)
 			spin_unlock_bh(&cq->compl_lock);
 			break;
 		}
+		case NQ_BASE_TYPE_SRQ_EVENT:
+		{
+			struct nq_srq_event *nqsrqe =
+						(struct nq_srq_event *)nqe;
+
+			q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
+			q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high)
+				     << 32;
+			bnxt_qplib_arm_srq((struct bnxt_qplib_srq *)q_handle,
+					   DBR_DBR_TYPE_SRQ_ARMENA);
+			if (!nq->srqn_handler(nq,
+					      (struct bnxt_qplib_srq *)q_handle,
+					      nqsrqe->event))
+				num_srqne_processed++;
+			else
+				dev_warn(&nq->pdev->dev,
+					 "QPLIB: SRQ event 0x%x not handled",
+					 nqsrqe->event);
+			break;
+		}
 		case NQ_BASE_TYPE_DBQ_EVENT:
 			break;
 		default:
@@ -384,17 +366,19 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
 					    struct bnxt_qplib_cq *),
 			 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
-					     void *, u8 event))
+					     struct bnxt_qplib_srq *,
+					     u8 event))
 {
 	resource_size_t nq_base;
 	int rc = -1;
 
 	nq->pdev = pdev;
 	nq->vector = msix_vector;
+	if (cqn_handler)
+		nq->cqn_handler = cqn_handler;
 
-	nq->cqn_handler = cqn_handler;
-
-	nq->srqn_handler = srqn_handler;
+	if (srqn_handler)
+		nq->srqn_handler = srqn_handler;
 
 	tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
 
@@ -410,7 +394,6 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 	if (rc) {
 		dev_err(&nq->pdev->dev,
 			"Failed to request IRQ for NQ: %#x", rc);
-		bnxt_qplib_disable_nq(nq);
 		goto fail;
 	}
 
@@ -469,6 +452,240 @@ int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
 	return 0;
 }
 
+/* SRQ */
+static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
+{
+	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
+	struct dbr_dbr db_msg = { 0 };
+	void __iomem *db;
+	u32 sw_prod = 0;
+
+	/* Ring DB */
+	sw_prod = (arm_type == DBR_DBR_TYPE_SRQ_ARM) ? srq->threshold :
+		   HWQ_CMP(srq_hwq->prod, srq_hwq);
+	db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
+				   DBR_DBR_INDEX_MASK);
+	db_msg.type_xid = cpu_to_le32(((srq->id << DBR_DBR_XID_SFT) &
+					DBR_DBR_XID_MASK) | arm_type);
+	db = (arm_type == DBR_DBR_TYPE_SRQ_ARMENA) ?
+		srq->dbr_base : srq->dpi->dbr;
+	wmb(); /* barrier before db ring */
+	__iowrite64_copy(db, &db_msg, sizeof(db_msg) / sizeof(u64));
+}
+
+int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+			   struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct cmdq_destroy_srq req;
+	struct creq_destroy_srq_resp resp;
+	u16 cmd_flags = 0;
+	int rc;
+
+	RCFW_CMD_PREP(req, DESTROY_SRQ, cmd_flags);
+
+	/* Configure the request */
+	req.srq_cid = cpu_to_le32(srq->id);
+
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+					  (void *)&resp, NULL, 0);
+	if (rc)
+		return rc;
+
+	bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
+	kfree(srq->swq);
+	return 0;
+}
+
+int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct cmdq_create_srq req;
+	struct creq_create_srq_resp resp;
+	struct bnxt_qplib_pbl *pbl;
+	u16 cmd_flags = 0;
+	int rc, idx;
+
+	srq->hwq.max_elements = srq->max_wqe;
+	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist,
+				       srq->nmap, &srq->hwq.max_elements,
+				       BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
+				       PAGE_SIZE, HWQ_TYPE_QUEUE);
+	if (rc)
+		goto exit;
+
+	srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq),
+			   GFP_KERNEL);
+	if (!srq->swq) {
+		rc = -ENOMEM;
+		goto fail;
+	}
+
+	RCFW_CMD_PREP(req, CREATE_SRQ, cmd_flags);
+
+	/* Configure the request */
+	req.dpi = cpu_to_le32(srq->dpi->dpi);
+	req.srq_handle = cpu_to_le64((uintptr_t)srq);
+
+	req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements);
+	pbl = &srq->hwq.pbl[PBL_LVL_0];
+	req.pg_size_lvl = cpu_to_le16((((u16)srq->hwq.level &
+				      CMDQ_CREATE_SRQ_LVL_MASK) <<
+				      CMDQ_CREATE_SRQ_LVL_SFT) |
+				      (pbl->pg_size == ROCE_PG_SIZE_4K ?
+				       CMDQ_CREATE_SRQ_PG_SIZE_PG_4K :
+				       pbl->pg_size == ROCE_PG_SIZE_8K ?
+				       CMDQ_CREATE_SRQ_PG_SIZE_PG_8K :
+				       pbl->pg_size == ROCE_PG_SIZE_64K ?
+				       CMDQ_CREATE_SRQ_PG_SIZE_PG_64K :
+				       pbl->pg_size == ROCE_PG_SIZE_2M ?
+				       CMDQ_CREATE_SRQ_PG_SIZE_PG_2M :
+				       pbl->pg_size == ROCE_PG_SIZE_8M ?
+				       CMDQ_CREATE_SRQ_PG_SIZE_PG_8M :
+				       pbl->pg_size == ROCE_PG_SIZE_1G ?
+				       CMDQ_CREATE_SRQ_PG_SIZE_PG_1G :
+				       CMDQ_CREATE_SRQ_PG_SIZE_PG_4K));
+	req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+	req.pd_id = cpu_to_le32(srq->pd->id);
+	req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
+
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+					  (void *)&resp, NULL, 0);
+	if (rc)
+		goto fail;
+
+	spin_lock_init(&srq->lock);
+	srq->start_idx = 0;
+	srq->last_idx = srq->hwq.max_elements - 1;
+	for (idx = 0; idx < srq->hwq.max_elements; idx++)
+		srq->swq[idx].next_idx = idx + 1;
+	srq->swq[srq->last_idx].next_idx = -1;
+
+	srq->id = le32_to_cpu(resp.xid);
+	srq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
+	if (srq->threshold)
+		bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARMENA);
+	srq->arm_req = false;
+
+	return 0;
+fail:
+	bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
+	kfree(srq->swq);
+exit:
+	return rc;
+}
+
+int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
+	u32 sw_prod, sw_cons, count = 0;
+
+	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
+	sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
+
+	count = sw_prod > sw_cons ? sw_prod - sw_cons :
+				    srq_hwq->max_elements - sw_cons + sw_prod;
+	if (count > srq->threshold) {
+		srq->arm_req = false;
+		bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
+	} else {
+		/* Deferred arming */
+		srq->arm_req = true;
+	}
+
+	return 0;
+}
+
+int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct cmdq_query_srq req;
+	struct creq_query_srq_resp resp;
+	struct bnxt_qplib_rcfw_sbuf *sbuf;
+	struct creq_query_srq_resp_sb *sb;
+	u16 cmd_flags = 0;
+	int rc = 0;
+
+	RCFW_CMD_PREP(req, QUERY_SRQ, cmd_flags);
+	req.srq_cid = cpu_to_le32(srq->id);
+
+	/* Configure the request */
+	sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
+	if (!sbuf)
+		return -ENOMEM;
+	sb = sbuf->sb;
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
+					  (void *)sbuf, 0);
+	srq->threshold = le16_to_cpu(sb->srq_limit);
+	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
+
+	return rc;
+}
+
+int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
+			     struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
+	struct rq_wqe *srqe, **srqe_ptr;
+	struct sq_sge *hw_sge;
+	u32 sw_prod, sw_cons, count = 0;
+	int i, rc = 0, next;
+
+	spin_lock(&srq_hwq->lock);
+	if (srq->start_idx == srq->last_idx) {
+		dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!",
+			srq->id);
+		rc = -EINVAL;
+		spin_unlock(&srq_hwq->lock);
+		goto done;
+	}
+	next = srq->start_idx;
+	srq->start_idx = srq->swq[next].next_idx;
+	spin_unlock(&srq_hwq->lock);
+
+	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
+	srqe_ptr = (struct rq_wqe **)srq_hwq->pbl_ptr;
+	srqe = &srqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
+	memset(srqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
+	/* Calculate wqe_size16 and data_len */
+	for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
+	     i < wqe->num_sge; i++, hw_sge++) {
+		hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
+		hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
+		hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
+	}
+	srqe->wqe_type = wqe->type;
+	srqe->flags = wqe->flags;
+	srqe->wqe_size = wqe->num_sge +
+			((offsetof(typeof(*srqe), data) + 15) >> 4);
+	srqe->wr_id[0] = cpu_to_le32((u32)next);
+	srq->swq[next].wr_id = wqe->wr_id;
+
+	srq_hwq->prod++;
+
+	spin_lock(&srq_hwq->lock);
+	sw_prod = HWQ_CMP(srq_hwq->prod, srq_hwq);
+	/* retaining srq_hwq->cons for this logic
+	 * actually the lock is only required to
+	 * read srq_hwq->cons.
+	 */
+	sw_cons = HWQ_CMP(srq_hwq->cons, srq_hwq);
+	count = sw_prod > sw_cons ? sw_prod - sw_cons :
+				    srq_hwq->max_elements - sw_cons + sw_prod;
+	spin_unlock(&srq_hwq->lock);
+	/* Ring DB */
+	bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ);
+	if (srq->arm_req == true && count > srq->threshold) {
+		srq->arm_req = false;
+		bnxt_qplib_arm_srq(srq, DBR_DBR_TYPE_SRQ_ARM);
+	}
+done:
+	return rc;
+}
+
 /* QP */
 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 {
@@ -737,6 +954,12 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 				 pbl->pg_size == ROCE_PG_SIZE_1G ?
 					CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
 				 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
+	} else {
+		/* SRQ */
+		if (qp->srq) {
+			qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
+			req.srq_cid = cpu_to_le32(qp->srq->id);
+		}
 	}
 
 	if (qp->rcq)
@@ -1156,7 +1379,6 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct cmdq_destroy_qp req;
 	struct creq_destroy_qp_resp resp;
-	unsigned long flags;
 	u16 cmd_flags = 0;
 	int rc;
 
@@ -1174,19 +1396,12 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 		return rc;
 	}
 
-	/* Must walk the associated CQs to nullified the QP ptr */
-	spin_lock_irqsave(&qp->scq->hwq.lock, flags);
-
-	__clean_cq(qp->scq, (u64)(unsigned long)qp);
-
-	if (qp->rcq && qp->rcq != qp->scq) {
-		spin_lock(&qp->rcq->hwq.lock);
-		__clean_cq(qp->rcq, (u64)(unsigned long)qp);
-		spin_unlock(&qp->rcq->hwq.lock);
-	}
-
-	spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
+	return 0;
+}
 
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_qp *qp)
+{
 	bnxt_qplib_free_qp_hdr_buf(res, qp);
 	bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
 	kfree(qp->sq.swq);
@@ -1199,7 +1414,6 @@ int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
 	if (qp->orrq.max_elements)
 		bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
 
-	return 0;
 }
 
 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
@@ -1853,9 +2067,6 @@ void bnxt_qplib_mark_qp_error(void *qp_handle)
 	/* Must block new posting of SQ and RQ */
 	qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
 	bnxt_qplib_cancel_phantom_processing(qp);
-
-	/* Add qp to flush list of the CQ */
-	__bnxt_qplib_add_flush_qp(qp);
 }
 
 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
@@ -2031,9 +2242,9 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
 				sw_sq_cons, cqe->wr_id, cqe->status);
 			cqe++;
 			(*budget)--;
-			bnxt_qplib_lock_buddy_cq(qp, cq);
 			bnxt_qplib_mark_qp_error(qp);
-			bnxt_qplib_unlock_buddy_cq(qp, cq);
+			/* Add qp to flush list of the CQ */
+			bnxt_qplib_add_flush_qp(qp);
 		} else {
 			if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
 				/* Before we complete, do WA 9060 */
@@ -2068,6 +2279,16 @@ static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
 	return rc;
 }
 
+static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
+{
+	spin_lock(&srq->hwq.lock);
+	srq->swq[srq->last_idx].next_idx = (int)tag;
+	srq->last_idx = (int)tag;
+	srq->swq[srq->last_idx].next_idx = -1;
+	srq->hwq.cons++; /* Support for SRQE counter */
+	spin_unlock(&srq->hwq.lock);
+}
+
 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 					struct cq_res_rc *hwcqe,
 					struct bnxt_qplib_cqe **pcqe,
@@ -2075,6 +2296,7 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 {
 	struct bnxt_qplib_qp *qp;
 	struct bnxt_qplib_q *rq;
+	struct bnxt_qplib_srq *srq;
 	struct bnxt_qplib_cqe *cqe;
 	u32 wr_id_idx;
 	int rc = 0;
@@ -2102,27 +2324,44 @@ static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
 
 	wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
 				CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
-	rq = &qp->rq;
-	if (wr_id_idx > rq->hwq.max_elements) {
-		dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
-		dev_err(&cq->hwq.pdev->dev,
-			"QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
-			wr_id_idx, rq->hwq.max_elements);
-		return -EINVAL;
-	}
-
-	cqe->wr_id = rq->swq[wr_id_idx].wr_id;
-	cqe++;
-	(*budget)--;
-	rq->hwq.cons++;
-	*pcqe = cqe;
+	if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
+		srq = qp->srq;
+		if (!srq)
+			return -EINVAL;
+		if (wr_id_idx > srq->hwq.max_elements) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process RC ");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
+				wr_id_idx, srq->hwq.max_elements);
+			return -EINVAL;
+		}
+		cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+		bnxt_qplib_release_srqe(srq, wr_id_idx);
+		cqe++;
+		(*budget)--;
+		*pcqe = cqe;
+	} else {
+		rq = &qp->rq;
+		if (wr_id_idx > rq->hwq.max_elements) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process RC ");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
+				wr_id_idx, rq->hwq.max_elements);
+			return -EINVAL;
+		}
+		cqe->wr_id = rq->swq[wr_id_idx].wr_id;
+		cqe++;
+		(*budget)--;
+		rq->hwq.cons++;
+		*pcqe = cqe;
 
-	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
-		qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
-		 /* Add qp to flush list of the CQ */
-		bnxt_qplib_lock_buddy_cq(qp, cq);
-		__bnxt_qplib_add_flush_qp(qp);
-		bnxt_qplib_unlock_buddy_cq(qp, cq);
+		if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
+			qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
+			/* Add qp to flush list of the CQ */
+			bnxt_qplib_add_flush_qp(qp);
+		}
 	}
 
 done:
@@ -2136,6 +2375,7 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 {
 	struct bnxt_qplib_qp *qp;
 	struct bnxt_qplib_q *rq;
+	struct bnxt_qplib_srq *srq;
 	struct bnxt_qplib_cqe *cqe;
 	u32 wr_id_idx;
 	int rc = 0;
@@ -2166,27 +2406,46 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 				  hwcqe->src_qp_high_srq_or_rq_wr_id) &
 				 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
 
-	rq = &qp->rq;
-	if (wr_id_idx > rq->hwq.max_elements) {
-		dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
-		dev_err(&cq->hwq.pdev->dev,
-			"QPLIB: wr_id idx %#x exceeded RQ max %#x",
-			wr_id_idx, rq->hwq.max_elements);
-		return -EINVAL;
-	}
+	if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
+		srq = qp->srq;
+		if (!srq)
+			return -EINVAL;
 
-	cqe->wr_id = rq->swq[wr_id_idx].wr_id;
-	cqe++;
-	(*budget)--;
-	rq->hwq.cons++;
-	*pcqe = cqe;
+		if (wr_id_idx > srq->hwq.max_elements) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process UD ");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
+				wr_id_idx, srq->hwq.max_elements);
+			return -EINVAL;
+		}
+		cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+		bnxt_qplib_release_srqe(srq, wr_id_idx);
+		cqe++;
+		(*budget)--;
+		*pcqe = cqe;
+	} else {
+		rq = &qp->rq;
+		if (wr_id_idx > rq->hwq.max_elements) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process UD ");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
+				wr_id_idx, rq->hwq.max_elements);
+			return -EINVAL;
+		}
+
+		cqe->wr_id = rq->swq[wr_id_idx].wr_id;
+		cqe++;
+		(*budget)--;
+		rq->hwq.cons++;
+		*pcqe = cqe;
 
-	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
-		qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
-		/* Add qp to flush list of the CQ */
-		bnxt_qplib_lock_buddy_cq(qp, cq);
-		__bnxt_qplib_add_flush_qp(qp);
-		bnxt_qplib_unlock_buddy_cq(qp, cq);
+		if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
+			qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
+			/* Add qp to flush list of the CQ */
+			bnxt_qplib_add_flush_qp(qp);
+		}
 	}
 done:
 	return rc;
@@ -2195,11 +2454,9 @@ static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
 bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
 {
 	struct cq_base *hw_cqe, **hw_cqe_ptr;
-	unsigned long flags;
 	u32 sw_cons, raw_cons;
 	bool rc = true;
 
-	spin_lock_irqsave(&cq->hwq.lock, flags);
 	raw_cons = cq->hwq.cons;
 	sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
 	hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
@@ -2207,7 +2464,6 @@ bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
 
 	 /* Check for Valid bit. If the CQE is valid, return false */
 	rc = !CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements);
-	spin_unlock_irqrestore(&cq->hwq.lock, flags);
 	return rc;
 }
 
@@ -2218,6 +2474,7 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
 {
 	struct bnxt_qplib_qp *qp;
 	struct bnxt_qplib_q *rq;
+	struct bnxt_qplib_srq *srq;
 	struct bnxt_qplib_cqe *cqe;
 	u32 wr_id_idx;
 	int rc = 0;
@@ -2256,26 +2513,47 @@ static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
 	cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
 	cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
 
-	rq = &qp->rq;
-	if (wr_id_idx > rq->hwq.max_elements) {
-		dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
-		dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
-			wr_id_idx, rq->hwq.max_elements);
-		return -EINVAL;
-	}
-
-	cqe->wr_id = rq->swq[wr_id_idx].wr_id;
-	cqe++;
-	(*budget)--;
-	rq->hwq.cons++;
-	*pcqe = cqe;
+	if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {
+		srq = qp->srq;
+		if (!srq) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: SRQ used but not defined??");
+			return -EINVAL;
+		}
+		if (wr_id_idx > srq->hwq.max_elements) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process Raw/QP1 ");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x",
+				wr_id_idx, srq->hwq.max_elements);
+			return -EINVAL;
+		}
+		cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+		bnxt_qplib_release_srqe(srq, wr_id_idx);
+		cqe++;
+		(*budget)--;
+		*pcqe = cqe;
+	} else {
+		rq = &qp->rq;
+		if (wr_id_idx > rq->hwq.max_elements) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: ix 0x%x exceeded RQ max 0x%x",
+				wr_id_idx, rq->hwq.max_elements);
+			return -EINVAL;
+		}
+		cqe->wr_id = rq->swq[wr_id_idx].wr_id;
+		cqe++;
+		(*budget)--;
+		rq->hwq.cons++;
+		*pcqe = cqe;
 
-	if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
-		qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
-		/* Add qp to flush list of the CQ */
-		bnxt_qplib_lock_buddy_cq(qp, cq);
-		__bnxt_qplib_add_flush_qp(qp);
-		bnxt_qplib_unlock_buddy_cq(qp, cq);
+		if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
+			qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
+			/* Add qp to flush list of the CQ */
+			bnxt_qplib_add_flush_qp(qp);
+		}
 	}
 
 done:
@@ -2389,9 +2667,7 @@ static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
 	 */
 
 	/* Add qp to flush list of the CQ */
-	bnxt_qplib_lock_buddy_cq(qp, cq);
-	__bnxt_qplib_add_flush_qp(qp);
-	bnxt_qplib_unlock_buddy_cq(qp, cq);
+	bnxt_qplib_add_flush_qp(qp);
 done:
 	return rc;
 }
@@ -2420,7 +2696,7 @@ int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
 	u32 budget = num_cqes;
 	unsigned long flags;
 
-	spin_lock_irqsave(&cq->hwq.lock, flags);
+	spin_lock_irqsave(&cq->flush_lock, flags);
 	list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
 		dev_dbg(&cq->hwq.pdev->dev,
 			"QPLIB: FP: Flushing SQ QP= %p",
@@ -2434,7 +2710,7 @@ int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
 			qp);
 		__flush_rq(&qp->rq, qp, &cqe, &budget);
 	}
-	spin_unlock_irqrestore(&cq->hwq.lock, flags);
+	spin_unlock_irqrestore(&cq->flush_lock, flags);
 
 	return num_cqes - budget;
 }
@@ -2443,11 +2719,9 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 		       int num_cqes, struct bnxt_qplib_qp **lib_qp)
 {
 	struct cq_base *hw_cqe, **hw_cqe_ptr;
-	unsigned long flags;
 	u32 sw_cons, raw_cons;
 	int budget, rc = 0;
 
-	spin_lock_irqsave(&cq->hwq.lock, flags);
 	raw_cons = cq->hwq.cons;
 	budget = num_cqes;
 
@@ -2523,20 +2797,15 @@ int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
 		bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
 	}
 exit:
-	spin_unlock_irqrestore(&cq->hwq.lock, flags);
 	return num_cqes - budget;
 }
 
 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(&cq->hwq.lock, flags);
 	if (arm_type)
 		bnxt_qplib_arm_cq(cq, arm_type);
 	/* Using cq->arm_state variable to track whether to issue cq handler */
 	atomic_set(&cq->arm_state, 1);
-	spin_unlock_irqrestore(&cq->hwq.lock, flags);
 }
 
 void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index c582d4ec8173..ade9f13c0fd1 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -39,6 +39,27 @@
 #ifndef __BNXT_QPLIB_FP_H__
 #define __BNXT_QPLIB_FP_H__
 
+struct bnxt_qplib_srq {
+	struct bnxt_qplib_pd		*pd;
+	struct bnxt_qplib_dpi		*dpi;
+	void __iomem			*dbr_base;
+	u64				srq_handle;
+	u32				id;
+	u32				max_wqe;
+	u32				max_sge;
+	u32				threshold;
+	bool				arm_req;
+	struct bnxt_qplib_cq		*cq;
+	struct bnxt_qplib_hwq		hwq;
+	struct bnxt_qplib_swq		*swq;
+	struct scatterlist		*sglist;
+	int				start_idx;
+	int				last_idx;
+	u32				nmap;
+	u16				eventq_hw_ring_id;
+	spinlock_t			lock; /* protect SRQE link list */
+};
+
 struct bnxt_qplib_sge {
 	u64				addr;
 	u32				lkey;
@@ -79,6 +100,7 @@ static inline u32 get_psne_idx(u32 val)
 
 struct bnxt_qplib_swq {
 	u64				wr_id;
+	int				next_idx;
 	u8				type;
 	u8				flags;
 	u32				start_psn;
@@ -367,6 +389,18 @@ struct bnxt_qplib_cq {
 	struct list_head		sqf_head, rqf_head;
 	atomic_t			arm_state;
 	spinlock_t			compl_lock; /* synch CQ handlers */
+/* Locking Notes:
+ * QP can move to error state from modify_qp, async error event or error
+ * CQE as part of poll_cq. When QP is moved to error state, it gets added
+ * to two flush lists, one each for SQ and RQ.
+ * Each flush list is protected by qplib_cq->flush_lock. Both scq and rcq
+ * flush_locks should be acquired when QP is moved to error. The control path
+ * operations(modify_qp and async error events) are synchronized with poll_cq
+ * using upper level CQ locks (bnxt_re_cq->cq_lock) of both SCQ and RCQ.
+ * The qplib_cq->flush_lock is required to synchronize two instances of poll_cq
+ * of the same QP while manipulating the flush list.
+ */
+	spinlock_t			flush_lock; /* QP flush management */
 };
 
 #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE	sizeof(struct xrrq_irrq)
@@ -404,29 +438,27 @@ struct bnxt_qplib_cq {
 	writel(NQ_DB_CP_FLAGS | ((raw_cons) & ((cp_bit) - 1)), db)
 
 struct bnxt_qplib_nq {
-	struct pci_dev			*pdev;
-
-	int				vector;
-	cpumask_t			mask;
-	int				budget;
-	bool				requested;
-	struct tasklet_struct		worker;
-	struct bnxt_qplib_hwq		hwq;
-
-	u16				bar_reg;
-	u16				bar_reg_off;
-	u16				ring_id;
-	void __iomem			*bar_reg_iomem;
-
-	int				(*cqn_handler)
-						(struct bnxt_qplib_nq *nq,
-						 struct bnxt_qplib_cq *cq);
-	int				(*srqn_handler)
-						(struct bnxt_qplib_nq *nq,
-						 void *srq,
-						 u8 event);
-	struct workqueue_struct         *cqn_wq;
-	char                            name[32];
+	struct pci_dev		*pdev;
+
+	int			vector;
+	cpumask_t		mask;
+	int			budget;
+	bool			requested;
+	struct tasklet_struct	worker;
+	struct bnxt_qplib_hwq	hwq;
+
+	u16			bar_reg;
+	u16			bar_reg_off;
+	u16			ring_id;
+	void __iomem		*bar_reg_iomem;
+
+	int			(*cqn_handler)(struct bnxt_qplib_nq *nq,
+					       struct bnxt_qplib_cq *cq);
+	int			(*srqn_handler)(struct bnxt_qplib_nq *nq,
+						struct bnxt_qplib_srq *srq,
+						u8 event);
+	struct workqueue_struct	*cqn_wq;
+	char			name[32];
 };
 
 struct bnxt_qplib_nq_work {
@@ -441,13 +473,26 @@ int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
 			 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
 					    struct bnxt_qplib_cq *cq),
 			 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
-					     void *srq,
+					     struct bnxt_qplib_srq *srq,
 					     u8 event));
+int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq);
+int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq);
+int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_srq *srq);
+int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+			   struct bnxt_qplib_srq *srq);
+int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
+			     struct bnxt_qplib_swqe *wqe);
 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_qp *qp);
 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
 				struct bnxt_qplib_sge *sge);
 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
@@ -470,7 +515,6 @@ void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq);
 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq);
 void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);
-void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp);
 void bnxt_qplib_acquire_cq_locks(struct bnxt_qplib_qp *qp,
 				 unsigned long *flags);
 void bnxt_qplib_release_cq_locks(struct bnxt_qplib_qp *qp,
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index bb5574adf195..80027a494730 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -93,7 +93,8 @@ static int __send_message(struct bnxt_qplib_rcfw *rcfw, struct cmdq_base *req,
 	opcode = req->opcode;
 	if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->flags) &&
 	    (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
-	     opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW)) {
+	     opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &&
+	     opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {
 		dev_err(&rcfw->pdev->dev,
 			"QPLIB: RCFW not initialized, reject opcode 0x%x",
 			opcode);
@@ -304,9 +305,8 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
 			err_event->res_err_state_reason);
 		if (!qp)
 			break;
-		bnxt_qplib_acquire_cq_locks(qp, &flags);
 		bnxt_qplib_mark_qp_error(qp);
-		bnxt_qplib_release_cq_locks(qp, &flags);
+		rcfw->aeq_handler(rcfw, qp_event, qp);
 		break;
 	default:
 		/* Command Response */
@@ -459,7 +459,11 @@ int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw,
 	int rc;
 
 	RCFW_CMD_PREP(req, INITIALIZE_FW, cmd_flags);
-
+	/* Supply (log-base-2-of-host-page-size - base-page-shift)
+	 * to bono to adjust the doorbell page sizes.
+	 */
+	req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT -
+					   RCFW_DBR_BASE_PAGE_SHIFT);
 	/*
 	 * VFs need not setup the HW context area, PF
 	 * shall setup this area for VF. Skipping the
@@ -615,7 +619,7 @@ int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
 				   int msix_vector,
 				   int cp_bar_reg_off, int virt_fn,
 				   int (*aeq_handler)(struct bnxt_qplib_rcfw *,
-						      struct creq_func_event *))
+						      void *, void *))
 {
 	resource_size_t res_base;
 	struct cmdq_init init;
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
index 2946a7cfae82..c7cce2e4185e 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.h
@@ -49,6 +49,7 @@
 #define RCFW_COMM_SIZE			0x104
 
 #define RCFW_DBR_PCI_BAR_REGION		2
+#define RCFW_DBR_BASE_PAGE_SHIFT	12
 
 #define RCFW_CMD_PREP(req, CMD, cmd_flags)				\
 	do {								\
@@ -167,7 +168,7 @@ struct bnxt_qplib_rcfw {
 #define FIRMWARE_TIMED_OUT		3
 	wait_queue_head_t	waitq;
 	int			(*aeq_handler)(struct bnxt_qplib_rcfw *,
-					       struct creq_func_event *);
+					       void *, void *);
 	u32			seq_num;
 
 	/* Bar region info */
@@ -199,9 +200,8 @@ int bnxt_qplib_enable_rcfw_channel(struct pci_dev *pdev,
 				   struct bnxt_qplib_rcfw *rcfw,
 				   int msix_vector,
 				   int cp_bar_reg_off, int virt_fn,
-				   int (*aeq_handler)
-					(struct bnxt_qplib_rcfw *,
-					 struct creq_func_event *));
+				   int (*aeq_handler)(struct bnxt_qplib_rcfw *,
+						      void *aeqe, void *obj));
 
 struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(
 				struct bnxt_qplib_rcfw *rcfw,
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 4e101704e801..539a5d44e6db 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -104,13 +104,12 @@ static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
 
 	if (!sghead) {
 		for (i = 0; i < pages; i++) {
-			pbl->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
-							    pbl->pg_size,
-							    &pbl->pg_map_arr[i],
-							    GFP_KERNEL);
+			pbl->pg_arr[i] = dma_zalloc_coherent(&pdev->dev,
+							     pbl->pg_size,
+							     &pbl->pg_map_arr[i],
+							     GFP_KERNEL);
 			if (!pbl->pg_arr[i])
 				goto fail;
-			memset(pbl->pg_arr[i], 0, pbl->pg_size);
 			pbl->pg_count++;
 		}
 	} else {
@@ -706,12 +705,8 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res     *res,
 	dpit->max = dbr_len / PAGE_SIZE;
 
 	dpit->app_tbl = kcalloc(dpit->max, sizeof(void *), GFP_KERNEL);
-	if (!dpit->app_tbl) {
-		pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
-		dev_err(&res->pdev->dev,
-			"QPLIB: DPI app tbl allocation failed");
-		return -ENOMEM;
-	}
+	if (!dpit->app_tbl)
+		goto unmap_io;
 
 	bytes = dpit->max >> 3;
 	if (!bytes)
@@ -719,18 +714,18 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res     *res,
 
 	dpit->tbl = kmalloc(bytes, GFP_KERNEL);
 	if (!dpit->tbl) {
-		pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
 		kfree(dpit->app_tbl);
 		dpit->app_tbl = NULL;
-		dev_err(&res->pdev->dev,
-			"QPLIB: DPI tbl allocation failed for size = %d",
-			bytes);
-		return -ENOMEM;
+		goto unmap_io;
 	}
 
 	memset((u8 *)dpit->tbl, 0xFF, bytes);
 
 	return 0;
+
+unmap_io:
+	pci_iounmap(res->pdev, dpit->dbr_bar_reg_iomem);
+	return -ENOMEM;
 }
 
 /* PKEYs */
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index 9543ce51a28a..ee98e5efef84 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -52,20 +52,28 @@ const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
 
 /* Device */
 
-static bool bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw)
+static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw,
+				     char *fw_ver)
 {
-	int rc;
-	u16 pcie_ctl2;
+	struct cmdq_query_version req;
+	struct creq_query_version_resp resp;
+	u16 cmd_flags = 0;
+	int rc = 0;
+
+	RCFW_CMD_PREP(req, QUERY_VERSION, cmd_flags);
 
-	rc = pcie_capability_read_word(rcfw->pdev, PCI_EXP_DEVCTL2,
-				       &pcie_ctl2);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+					  (void *)&resp, NULL, 0);
 	if (rc)
-		return false;
-	return !!(pcie_ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
+		return;
+	fw_ver[0] = resp.fw_maj;
+	fw_ver[1] = resp.fw_minor;
+	fw_ver[2] = resp.fw_bld;
+	fw_ver[3] = resp.fw_rsvd;
 }
 
 int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
-			    struct bnxt_qplib_dev_attr *attr)
+			    struct bnxt_qplib_dev_attr *attr, bool vf)
 {
 	struct cmdq_query_func req;
 	struct creq_query_func_resp resp;
@@ -95,7 +103,8 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 	/* Extract the context from the side buffer */
 	attr->max_qp = le32_to_cpu(sb->max_qp);
 	/* max_qp value reported by FW for PF doesn't include the QP1 for PF */
-	attr->max_qp += 1;
+	if (!vf)
+		attr->max_qp += 1;
 	attr->max_qp_rd_atom =
 		sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
 		BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom;
@@ -130,10 +139,11 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 	attr->max_pkey = le32_to_cpu(sb->max_pkeys);
 
 	attr->max_inline_data = le32_to_cpu(sb->max_inline_data);
-	attr->l2_db_size = (sb->l2_db_space_size + 1) * PAGE_SIZE;
+	attr->l2_db_size = (sb->l2_db_space_size + 1) *
+			    (0x01 << RCFW_DBR_BASE_PAGE_SHIFT);
 	attr->max_sgid = le32_to_cpu(sb->max_gid);
 
-	strlcpy(attr->fw_ver, "20.6.28.0", sizeof(attr->fw_ver));
+	bnxt_qplib_query_version(rcfw, attr->fw_ver);
 
 	for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) {
 		temp = le32_to_cpu(sb->tqm_alloc_reqs[i]);
@@ -144,12 +154,44 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
 		attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);
 	}
 
-	attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw);
+	attr->is_atomic = 0;
 bail:
 	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
 	return rc;
 }
 
+int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,
+				  struct bnxt_qplib_rcfw *rcfw,
+				  struct bnxt_qplib_ctx *ctx)
+{
+	struct cmdq_set_func_resources req;
+	struct creq_set_func_resources_resp resp;
+	u16 cmd_flags = 0;
+	int rc = 0;
+
+	RCFW_CMD_PREP(req, SET_FUNC_RESOURCES, cmd_flags);
+
+	req.number_of_qp = cpu_to_le32(ctx->qpc_count);
+	req.number_of_mrw = cpu_to_le32(ctx->mrw_count);
+	req.number_of_srq =  cpu_to_le32(ctx->srqc_count);
+	req.number_of_cq = cpu_to_le32(ctx->cq_count);
+
+	req.max_qp_per_vf = cpu_to_le32(ctx->vf_res.max_qp_per_vf);
+	req.max_mrw_per_vf = cpu_to_le32(ctx->vf_res.max_mrw_per_vf);
+	req.max_srq_per_vf = cpu_to_le32(ctx->vf_res.max_srq_per_vf);
+	req.max_cq_per_vf = cpu_to_le32(ctx->vf_res.max_cq_per_vf);
+	req.max_gid_per_vf = cpu_to_le32(ctx->vf_res.max_gid_per_vf);
+
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
+					  (void *)&resp,
+					  NULL, 0);
+	if (rc) {
+		dev_err(&res->pdev->dev,
+			"QPLIB: Failed to set function resources");
+	}
+	return rc;
+}
+
 /* SGID */
 int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
 			struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
@@ -604,7 +646,7 @@ int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
 }
 
 int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
-		      u64 *pbl_tbl, int num_pbls, bool block)
+		      u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size)
 {
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct cmdq_register_mr req;
@@ -615,6 +657,9 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 	u32 pg_size;
 
 	if (num_pbls) {
+		/* Allocate memory for the non-leaf pages to store buf ptrs.
+		 * Non-leaf pages always uses system PAGE_SIZE
+		 */
 		pg_ptrs = roundup_pow_of_two(num_pbls);
 		pages = pg_ptrs >> MAX_PBL_LVL_1_PGS_SHIFT;
 		if (!pages)
@@ -632,6 +677,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 			bnxt_qplib_free_hwq(res->pdev, &mr->hwq);
 
 		mr->hwq.max_elements = pages;
+		/* Use system PAGE_SIZE */
 		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL, 0,
 					       &mr->hwq.max_elements,
 					       PAGE_SIZE, 0, PAGE_SIZE,
@@ -652,18 +698,22 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 
 	/* Configure the request */
 	if (mr->hwq.level == PBL_LVL_MAX) {
+		/* No PBL provided, just use system PAGE_SIZE */
 		level = 0;
 		req.pbl = 0;
 		pg_size = PAGE_SIZE;
 	} else {
 		level = mr->hwq.level + 1;
 		req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
-		pg_size = mr->hwq.pbl[PBL_LVL_0].pg_size;
 	}
+	pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE;
 	req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) |
 			       ((ilog2(pg_size) <<
 				 CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) &
 				CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK);
+	req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) <<
+				 CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) &
+				CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK));
 	req.access = (mr->flags & 0xFFFF);
 	req.va = cpu_to_le64(mr->va);
 	req.key = cpu_to_le32(mr->lkey);
@@ -729,3 +779,73 @@ int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids)
 				     0);
 	return 0;
 }
+
+int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,
+			      struct bnxt_qplib_roce_stats *stats)
+{
+	struct cmdq_query_roce_stats req;
+	struct creq_query_roce_stats_resp resp;
+	struct bnxt_qplib_rcfw_sbuf *sbuf;
+	struct creq_query_roce_stats_resp_sb *sb;
+	u16 cmd_flags = 0;
+	int rc = 0;
+
+	RCFW_CMD_PREP(req, QUERY_ROCE_STATS, cmd_flags);
+
+	sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
+	if (!sbuf) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: SP: QUERY_ROCE_STATS alloc side buffer failed");
+		return -ENOMEM;
+	}
+
+	sb = sbuf->sb;
+	req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
+					  (void *)sbuf, 0);
+	if (rc)
+		goto bail;
+	/* Extract the context from the side buffer */
+	stats->to_retransmits = le64_to_cpu(sb->to_retransmits);
+	stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd);
+	stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded);
+	stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd);
+	stats->missing_resp = le64_to_cpu(sb->missing_resp);
+	stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err);
+	stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err);
+	stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err);
+	stats->local_protection_err = le64_to_cpu(sb->local_protection_err);
+	stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err);
+	stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err);
+	stats->remote_access_err = le64_to_cpu(sb->remote_access_err);
+	stats->remote_op_err = le64_to_cpu(sb->remote_op_err);
+	stats->dup_req = le64_to_cpu(sb->dup_req);
+	stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max);
+	stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch);
+	stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe);
+	stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err);
+	stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey);
+	stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err);
+	stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm);
+	stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err);
+	stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey);
+	stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err);
+	stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm);
+	stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err);
+	stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow);
+	stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode);
+	stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic);
+	stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err);
+	stats->res_mem_error = le64_to_cpu(sb->res_mem_error);
+	stats->res_srq_err = le64_to_cpu(sb->res_srq_err);
+	stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err);
+	stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey);
+	stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err);
+	stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err);
+	stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err);
+	stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err);
+	stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err);
+bail:
+	bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
+	return rc;
+}
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index 11322582f5e4..9d3e8b994945 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -45,7 +45,8 @@
 #define PCI_EXP_DEVCTL2_ATOMIC_REQ      0x0040
 
 struct bnxt_qplib_dev_attr {
-	char				fw_ver[32];
+#define FW_VER_ARR_LEN			4
+	u8				fw_ver[FW_VER_ARR_LEN];
 	u16				max_sgid;
 	u16				max_mrw;
 	u32				max_qp;
@@ -127,6 +128,85 @@ struct bnxt_qplib_frpl {
 #define BNXT_QPLIB_ACCESS_ZERO_BASED	BIT(5)
 #define BNXT_QPLIB_ACCESS_ON_DEMAND	BIT(6)
 
+struct bnxt_qplib_roce_stats {
+	u64 to_retransmits;
+	u64 seq_err_naks_rcvd;
+	/* seq_err_naks_rcvd is 64 b */
+	u64 max_retry_exceeded;
+	/* max_retry_exceeded is 64 b */
+	u64 rnr_naks_rcvd;
+	/* rnr_naks_rcvd is 64 b */
+	u64 missing_resp;
+	u64 unrecoverable_err;
+	/* unrecoverable_err is 64 b */
+	u64 bad_resp_err;
+	/* bad_resp_err is 64 b */
+	u64 local_qp_op_err;
+	/* local_qp_op_err is 64 b */
+	u64 local_protection_err;
+	/* local_protection_err is 64 b */
+	u64 mem_mgmt_op_err;
+	/* mem_mgmt_op_err is 64 b */
+	u64 remote_invalid_req_err;
+	/* remote_invalid_req_err is 64 b */
+	u64 remote_access_err;
+	/* remote_access_err is 64 b */
+	u64 remote_op_err;
+	/* remote_op_err is 64 b */
+	u64 dup_req;
+	/* dup_req is 64 b */
+	u64 res_exceed_max;
+	/* res_exceed_max is 64 b */
+	u64 res_length_mismatch;
+	/* res_length_mismatch is 64 b */
+	u64 res_exceeds_wqe;
+	/* res_exceeds_wqe is 64 b */
+	u64 res_opcode_err;
+	/* res_opcode_err is 64 b */
+	u64 res_rx_invalid_rkey;
+	/* res_rx_invalid_rkey is 64 b */
+	u64 res_rx_domain_err;
+	/* res_rx_domain_err is 64 b */
+	u64 res_rx_no_perm;
+	/* res_rx_no_perm is 64 b */
+	u64 res_rx_range_err;
+	/* res_rx_range_err is 64 b */
+	u64 res_tx_invalid_rkey;
+	/* res_tx_invalid_rkey is 64 b */
+	u64 res_tx_domain_err;
+	/* res_tx_domain_err is 64 b */
+	u64 res_tx_no_perm;
+	/* res_tx_no_perm is 64 b */
+	u64 res_tx_range_err;
+	/* res_tx_range_err is 64 b */
+	u64 res_irrq_oflow;
+	/* res_irrq_oflow is 64 b */
+	u64 res_unsup_opcode;
+	/* res_unsup_opcode is 64 b */
+	u64 res_unaligned_atomic;
+	/* res_unaligned_atomic is 64 b */
+	u64 res_rem_inv_err;
+	/* res_rem_inv_err is 64 b */
+	u64 res_mem_error;
+	/* res_mem_error is 64 b */
+	u64 res_srq_err;
+	/* res_srq_err is 64 b */
+	u64 res_cmp_err;
+	/* res_cmp_err is 64 b */
+	u64 res_invalid_dup_rkey;
+	/* res_invalid_dup_rkey is 64 b */
+	u64 res_wqe_format_err;
+	/* res_wqe_format_err is 64 b */
+	u64 res_cq_load_err;
+	/* res_cq_load_err is 64 b */
+	u64 res_srq_load_err;
+	/* res_srq_load_err is 64 b */
+	u64 res_tx_pci_err;
+	/* res_tx_pci_err is 64 b */
+	u64 res_rx_pci_err;
+	/* res_rx_pci_err is 64 b */
+};
+
 int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
 			struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
 			struct bnxt_qplib_gid *gid);
@@ -147,7 +227,10 @@ int bnxt_qplib_add_pkey(struct bnxt_qplib_res *res,
 			struct bnxt_qplib_pkey_tbl *pkey_tbl, u16 *pkey,
 			bool update);
 int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw,
-			    struct bnxt_qplib_dev_attr *attr);
+			    struct bnxt_qplib_dev_attr *attr, bool vf);
+int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,
+				  struct bnxt_qplib_rcfw *rcfw,
+				  struct bnxt_qplib_ctx *ctx);
 int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah);
 int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah);
 int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res,
@@ -155,7 +238,7 @@ int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res,
 int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
 			 bool block);
 int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
-		      u64 *pbl_tbl, int num_pbls, bool block);
+		      u64 *pbl_tbl, int num_pbls, bool block, u32 buf_pg_size);
 int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr);
 int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res,
 				 struct bnxt_qplib_mrw *mr, int max);
@@ -164,4 +247,6 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
 int bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
 				       struct bnxt_qplib_frpl *frpl);
 int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids);
+int bnxt_qplib_get_roce_stats(struct bnxt_qplib_rcfw *rcfw,
+			      struct bnxt_qplib_roce_stats *stats);
 #endif /* __BNXT_QPLIB_SP_H__*/
diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
index c3cba6063a03..3e5a4f760d0e 100644
--- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h
+++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h
@@ -954,6 +954,7 @@ struct cmdq_base {
 	#define CMDQ_BASE_OPCODE_QUERY_VERSION			   0x8bUL
 	#define CMDQ_BASE_OPCODE_MODIFY_CC			   0x8cUL
 	#define CMDQ_BASE_OPCODE_QUERY_CC			   0x8dUL
+	#define CMDQ_BASE_OPCODE_QUERY_ROCE_STATS	   0x8eUL
 	u8 cmd_size;
 	__le16 flags;
 	__le16 cookie;
@@ -1383,8 +1384,20 @@ struct cmdq_register_mr {
 	#define CMDQ_REGISTER_MR_LVL_LVL_0			   0x0UL
 	#define CMDQ_REGISTER_MR_LVL_LVL_1			   0x1UL
 	#define CMDQ_REGISTER_MR_LVL_LVL_2			   0x2UL
+	#define CMDQ_REGISTER_MR_LVL_LAST             CMDQ_REGISTER_MR_LVL_LVL_2
 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK		    0x7cUL
 	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT		    2
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4K    (0xcUL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_8K    (0xdUL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_64K   (0x10UL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_256K  (0x12UL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1M    (0x14UL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_2M    (0x15UL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_4M    (0x16UL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G    (0x1eUL << 2)
+	#define CMDQ_REGISTER_MR_LOG2_PG_SIZE_LAST	\
+					CMDQ_REGISTER_MR_LOG2_PG_SIZE_PG_1G
+	#define CMDQ_REGISTER_MR_UNUSED1             0x80UL
 	u8 access;
 	#define CMDQ_REGISTER_MR_ACCESS_LOCAL_WRITE		    0x1UL
 	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_READ		    0x2UL
@@ -1392,7 +1405,21 @@ struct cmdq_register_mr {
 	#define CMDQ_REGISTER_MR_ACCESS_REMOTE_ATOMIC		    0x8UL
 	#define CMDQ_REGISTER_MR_ACCESS_MW_BIND		    0x10UL
 	#define CMDQ_REGISTER_MR_ACCESS_ZERO_BASED		    0x20UL
-	__le16 unused_1;
+	__le16	log2_pbl_pg_size;
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK   0x1fUL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT    0
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K    0xcUL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K    0xdUL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K   0x10UL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K  0x12UL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M    0x14UL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M    0x15UL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M    0x16UL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G    0x1eUL
+	#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_LAST    \
+				CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G
+	#define CMDQ_REGISTER_MR_UNUSED11_MASK           0xffe0UL
+	#define CMDQ_REGISTER_MR_UNUSED11_SFT            5
 	__le32 key;
 	__le64 pbl;
 	__le64 va;
@@ -1734,7 +1761,30 @@ struct cmdq_initialize_fw {
 	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_2M		   (0x3UL << 4)
 	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_8M		   (0x4UL << 4)
 	#define CMDQ_INITIALIZE_FW_TIM_PG_SIZE_PG_1G		   (0x5UL << 4)
-	__le16 reserved16;
+	/* This value is (log-base-2-of-DBR-page-size - 12).
+	 * 0 for 4KB. HW supported values are enumerated below.
+	 */
+	__le16  log2_dbr_pg_size;
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_MASK	0xfUL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_SFT		0
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4K	0x0UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8K	0x1UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16K	0x2UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32K	0x3UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64K	0x4UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128K	0x5UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_256K	0x6UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_512K	0x7UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_1M	0x8UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_2M	0x9UL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_4M	0xaUL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_8M	0xbUL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_16M	0xcUL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_32M	0xdUL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_64M	0xeUL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M	0xfUL
+	#define CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_LAST		\
+			CMDQ_INITIALIZE_FW_LOG2_DBR_PG_SIZE_PG_128M
 	__le64 qpc_page_dir;
 	__le64 mrw_page_dir;
 	__le64 srq_page_dir;
@@ -1799,6 +1849,16 @@ struct cmdq_set_func_resources {
 	u8 resp_size;
 	u8 reserved8;
 	__le64 resp_addr;
+	__le32 number_of_qp;
+	__le32 number_of_mrw;
+	__le32 number_of_srq;
+	__le32 number_of_cq;
+	__le32 max_qp_per_vf;
+	__le32 max_mrw_per_vf;
+	__le32 max_srq_per_vf;
+	__le32 max_cq_per_vf;
+	__le32 max_gid_per_vf;
+	__le32 stat_ctx_id;
 };
 
 /* Read hardware resource context command (24 bytes) */
@@ -2013,6 +2073,20 @@ struct creq_modify_qp_resp {
 	__le16 reserved48[3];
 };
 
+/* cmdq_query_roce_stats (size:128b/16B) */
+struct cmdq_query_roce_stats {
+	u8	opcode;
+	#define CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS 0x8eUL
+	#define CMDQ_QUERY_ROCE_STATS_OPCODE_LAST	\
+				CMDQ_QUERY_ROCE_STATS_OPCODE_QUERY_ROCE_STATS
+	u8	cmd_size;
+	__le16	flags;
+	__le16	cookie;
+	u8	resp_size;
+	u8	reserved8;
+	__le64	resp_addr;
+};
+
 /* Query QP command response (16 bytes) */
 struct creq_query_qp_resp {
 	u8 type;
@@ -2783,6 +2857,80 @@ struct creq_query_cc_resp_sb {
 	__le64 reserved64_1;
 };
 
+/* creq_query_roce_stats_resp (size:128b/16B) */
+struct creq_query_roce_stats_resp {
+	u8	type;
+	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_MASK    0x3fUL
+	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_SFT     0
+	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT  0x38UL
+	#define CREQ_QUERY_ROCE_STATS_RESP_TYPE_LAST	\
+				CREQ_QUERY_ROCE_STATS_RESP_TYPE_QP_EVENT
+	u8	status;
+	__le16	cookie;
+	__le32	size;
+	u8	v;
+	#define CREQ_QUERY_ROCE_STATS_RESP_V     0x1UL
+	u8	event;
+	#define CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS 0x8eUL
+	#define CREQ_QUERY_ROCE_STATS_RESP_EVENT_LAST	\
+			CREQ_QUERY_ROCE_STATS_RESP_EVENT_QUERY_ROCE_STATS
+	u8	reserved48[6];
+};
+
+/* creq_query_roce_stats_resp_sb (size:2624b/328B) */
+struct creq_query_roce_stats_resp_sb {
+	u8	opcode;
+	#define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS 0x8eUL
+	#define CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_LAST \
+			CREQ_QUERY_ROCE_STATS_RESP_SB_OPCODE_QUERY_ROCE_STATS
+	u8	status;
+	__le16	cookie;
+	__le16	flags;
+	u8	resp_size;
+	u8	rsvd;
+	__le32	num_counters;
+	__le32	rsvd1;
+	__le64	to_retransmits;
+	__le64	seq_err_naks_rcvd;
+	__le64	max_retry_exceeded;
+	__le64	rnr_naks_rcvd;
+	__le64	missing_resp;
+	__le64	unrecoverable_err;
+	__le64	bad_resp_err;
+	__le64	local_qp_op_err;
+	__le64	local_protection_err;
+	__le64	mem_mgmt_op_err;
+	__le64	remote_invalid_req_err;
+	__le64	remote_access_err;
+	__le64	remote_op_err;
+	__le64	dup_req;
+	__le64	res_exceed_max;
+	__le64	res_length_mismatch;
+	__le64	res_exceeds_wqe;
+	__le64	res_opcode_err;
+	__le64	res_rx_invalid_rkey;
+	__le64	res_rx_domain_err;
+	__le64	res_rx_no_perm;
+	__le64	res_rx_range_err;
+	__le64	res_tx_invalid_rkey;
+	__le64	res_tx_domain_err;
+	__le64	res_tx_no_perm;
+	__le64	res_tx_range_err;
+	__le64	res_irrq_oflow;
+	__le64	res_unsup_opcode;
+	__le64	res_unaligned_atomic;
+	__le64	res_rem_inv_err;
+	__le64	res_mem_error;
+	__le64	res_srq_err;
+	__le64	res_cmp_err;
+	__le64	res_invalid_dup_rkey;
+	__le64	res_wqe_format_err;
+	__le64	res_cq_load_err;
+	__le64	res_srq_load_err;
+	__le64	res_tx_pci_err;
+	__le64	res_rx_pci_err;
+};
+
 /* QP error notification event (16 bytes) */
 struct creq_qp_error_notification {
 	u8 type;
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 21db3b48a617..4cf17c650c36 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -257,8 +257,8 @@ static void set_emss(struct c4iw_ep *ep, u16 opt)
 	if (ep->emss < 128)
 		ep->emss = 128;
 	if (ep->emss & 7)
-		pr_warn("Warning: misaligned mtu idx %u mss %u emss=%u\n",
-			TCPOPT_MSS_G(opt), ep->mss, ep->emss);
+		pr_debug("Warning: misaligned mtu idx %u mss %u emss=%u\n",
+			 TCPOPT_MSS_G(opt), ep->mss, ep->emss);
 	pr_debug("mss_idx %u mss %u emss=%u\n", TCPOPT_MSS_G(opt), ep->mss,
 		 ep->emss);
 }
@@ -2733,9 +2733,8 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb)
 		return 0;
 
 	if (cxgb_is_neg_adv(req->status)) {
-		pr_warn("%s Negative advice on abort- tid %u status %d (%s)\n",
-			__func__, ep->hwtid, req->status,
-			neg_adv_str(req->status));
+		pr_debug("Negative advice on abort- tid %u status %d (%s)\n",
+			 ep->hwtid, req->status, neg_adv_str(req->status));
 		ep->stats.abort_neg_adv++;
 		mutex_lock(&dev->rdev.stats.lock);
 		dev->rdev.stats.neg_adv++;
@@ -3567,8 +3566,8 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp)
 	case MORIBUND:
 	case ABORTING:
 	case DEAD:
-		pr_info("%s ignoring disconnect ep %p state %u\n",
-			__func__, ep, ep->com.state);
+		pr_debug("ignoring disconnect ep %p state %u\n",
+			 ep, ep->com.state);
 		break;
 	default:
 		WARN_ONCE(1, "Bad endpoint state %u\n", ep->com.state);
@@ -4097,9 +4096,15 @@ static void process_work(struct work_struct *work)
 		dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *)));
 		opcode = rpl->ot.opcode;
 
-		ret = work_handlers[opcode](dev, skb);
-		if (!ret)
+		if (opcode >= ARRAY_SIZE(work_handlers) ||
+		    !work_handlers[opcode]) {
+			pr_err("No handler for opcode 0x%x.\n", opcode);
 			kfree_skb(skb);
+		} else {
+			ret = work_handlers[opcode](dev, skb);
+			if (!ret)
+				kfree_skb(skb);
+		}
 		process_timedout_eps();
 	}
 }
@@ -4201,8 +4206,8 @@ static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb)
 		return 0;
 	}
 	if (cxgb_is_neg_adv(req->status)) {
-		pr_warn("%s Negative advice on abort- tid %u status %d (%s)\n",
-			__func__, ep->hwtid, req->status,
+		pr_debug("Negative advice on abort- tid %u status %d (%s)\n",
+			 ep->hwtid, req->status,
 			 neg_adv_str(req->status));
 		goto out;
 	}
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index af77d128d242..7a9d0de89d6a 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -66,7 +66,7 @@ MODULE_PARM_DESC(c4iw_wr_log_size_order,
 
 static LIST_HEAD(uld_ctx_list);
 static DEFINE_MUTEX(dev_mutex);
-struct workqueue_struct *reg_workq;
+static struct workqueue_struct *reg_workq;
 
 #define DB_FC_RESUME_SIZE 64
 #define DB_FC_RESUME_DELAY 1
@@ -108,19 +108,19 @@ void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe)
 	idx = (atomic_inc_return(&wq->rdev->wr_log_idx) - 1) &
 		(wq->rdev->wr_log_size - 1);
 	le.poll_sge_ts = cxgb4_read_sge_timestamp(wq->rdev->lldi.ports[0]);
-	getnstimeofday(&le.poll_host_ts);
+	le.poll_host_time = ktime_get();
 	le.valid = 1;
 	le.cqe_sge_ts = CQE_TS(cqe);
 	if (SQ_TYPE(cqe)) {
 		le.qid = wq->sq.qid;
 		le.opcode = CQE_OPCODE(cqe);
-		le.post_host_ts = wq->sq.sw_sq[wq->sq.cidx].host_ts;
+		le.post_host_time = wq->sq.sw_sq[wq->sq.cidx].host_time;
 		le.post_sge_ts = wq->sq.sw_sq[wq->sq.cidx].sge_ts;
 		le.wr_id = CQE_WRID_SQ_IDX(cqe);
 	} else {
 		le.qid = wq->rq.qid;
 		le.opcode = FW_RI_RECEIVE;
-		le.post_host_ts = wq->rq.sw_rq[wq->rq.cidx].host_ts;
+		le.post_host_time = wq->rq.sw_rq[wq->rq.cidx].host_time;
 		le.post_sge_ts = wq->rq.sw_rq[wq->rq.cidx].sge_ts;
 		le.wr_id = CQE_WRID_MSN(cqe);
 	}
@@ -130,9 +130,9 @@ void c4iw_log_wr_stats(struct t4_wq *wq, struct t4_cqe *cqe)
 static int wr_log_show(struct seq_file *seq, void *v)
 {
 	struct c4iw_dev *dev = seq->private;
-	struct timespec prev_ts = {0, 0};
+	ktime_t prev_time;
 	struct wr_log_entry *lep;
-	int prev_ts_set = 0;
+	int prev_time_set = 0;
 	int idx, end;
 
 #define ts2ns(ts) div64_u64((ts) * dev->rdev.lldi.cclk_ps, 1000)
@@ -145,33 +145,29 @@ static int wr_log_show(struct seq_file *seq, void *v)
 	lep = &dev->rdev.wr_log[idx];
 	while (idx != end) {
 		if (lep->valid) {
-			if (!prev_ts_set) {
-				prev_ts_set = 1;
-				prev_ts = lep->poll_host_ts;
+			if (!prev_time_set) {
+				prev_time_set = 1;
+				prev_time = lep->poll_host_time;
 			}
-			seq_printf(seq, "%04u: sec %lu nsec %lu qid %u opcode "
-				   "%u %s 0x%x host_wr_delta sec %lu nsec %lu "
+			seq_printf(seq, "%04u: nsec %llu qid %u opcode "
+				   "%u %s 0x%x host_wr_delta nsec %llu "
 				   "post_sge_ts 0x%llx cqe_sge_ts 0x%llx "
 				   "poll_sge_ts 0x%llx post_poll_delta_ns %llu "
 				   "cqe_poll_delta_ns %llu\n",
 				   idx,
-				   timespec_sub(lep->poll_host_ts,
-						prev_ts).tv_sec,
-				   timespec_sub(lep->poll_host_ts,
-						prev_ts).tv_nsec,
+				   ktime_to_ns(ktime_sub(lep->poll_host_time,
+							 prev_time)),
 				   lep->qid, lep->opcode,
 				   lep->opcode == FW_RI_RECEIVE ?
 							"msn" : "wrid",
 				   lep->wr_id,
-				   timespec_sub(lep->poll_host_ts,
-						lep->post_host_ts).tv_sec,
-				   timespec_sub(lep->poll_host_ts,
-						lep->post_host_ts).tv_nsec,
+				   ktime_to_ns(ktime_sub(lep->poll_host_time,
+							 lep->post_host_time)),
 				   lep->post_sge_ts, lep->cqe_sge_ts,
 				   lep->poll_sge_ts,
 				   ts2ns(lep->poll_sge_ts - lep->post_sge_ts),
 				   ts2ns(lep->poll_sge_ts - lep->cqe_sge_ts));
-			prev_ts = lep->poll_host_ts;
+			prev_time = lep->poll_host_time;
 		}
 		idx++;
 		if (idx > (dev->rdev.wr_log_size - 1))
diff --git a/drivers/infiniband/hw/cxgb4/ev.c b/drivers/infiniband/hw/cxgb4/ev.c
index a252d5c40ae3..3e9d8b277ab9 100644
--- a/drivers/infiniband/hw/cxgb4/ev.c
+++ b/drivers/infiniband/hw/cxgb4/ev.c
@@ -236,7 +236,7 @@ int c4iw_ev_handler(struct c4iw_dev *dev, u32 qid)
 		if (atomic_dec_and_test(&chp->refcnt))
 			wake_up(&chp->wait);
 	} else {
-		pr_warn("%s unknown cqid 0x%x\n", __func__, qid);
+		pr_debug("unknown cqid 0x%x\n", qid);
 		spin_unlock_irqrestore(&dev->lock, flag);
 	}
 	return 0;
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 65dd3726ca02..cc929002c05e 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -153,8 +153,8 @@ struct c4iw_hw_queue {
 };
 
 struct wr_log_entry {
-	struct timespec post_host_ts;
-	struct timespec poll_host_ts;
+	ktime_t post_host_time;
+	ktime_t poll_host_time;
 	u64 post_sge_ts;
 	u64 cqe_sge_ts;
 	u64 poll_sge_ts;
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index d5c92fc520d6..de77b6027d69 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -1042,7 +1042,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		if (c4iw_wr_log) {
 			swsqe->sge_ts = cxgb4_read_sge_timestamp(
 					qhp->rhp->rdev.lldi.ports[0]);
-			getnstimeofday(&swsqe->host_ts);
+			swsqe->host_time = ktime_get();
 		}
 
 		init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16);
@@ -1117,8 +1117,8 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 			qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].sge_ts =
 				cxgb4_read_sge_timestamp(
 						qhp->rhp->rdev.lldi.ports[0]);
-			getnstimeofday(
-				&qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].host_ts);
+			qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].host_time =
+				ktime_get();
 		}
 
 		wqe->recv.opcode = FW_RI_RECV_WR;
diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h
index 79e8ee12c391..8369c7c8de83 100644
--- a/drivers/infiniband/hw/cxgb4/t4.h
+++ b/drivers/infiniband/hw/cxgb4/t4.h
@@ -277,7 +277,7 @@ struct t4_swsqe {
 	int			signaled;
 	u16			idx;
 	int                     flushed;
-	struct timespec         host_ts;
+	ktime_t			host_time;
 	u64                     sge_ts;
 };
 
@@ -318,7 +318,7 @@ struct t4_sq {
 
 struct t4_swrqe {
 	u64 wr_id;
-	struct timespec host_ts;
+	ktime_t	host_time;
 	u64 sge_ts;
 };
 
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index 4f057e8ffe50..e6a60fa59f2b 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -1083,6 +1083,7 @@ static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp,
 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index);
+static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width);
 
 /*
  * Error interrupt table entry.  This is used as input to the interrupt
@@ -6518,11 +6519,12 @@ static void _dc_start(struct hfi1_devdata *dd)
 	if (!dd->dc_shutdown)
 		return;
 
-	/*
-	 * Take the 8051 out of reset, wait until 8051 is ready, and set host
-	 * version bit.
-	 */
-	release_and_wait_ready_8051_firmware(dd);
+	/* Take the 8051 out of reset */
+	write_csr(dd, DC_DC8051_CFG_RST, 0ull);
+	/* Wait until 8051 is ready */
+	if (wait_fm_ready(dd, TIMEOUT_8051_START))
+		dd_dev_err(dd, "%s: timeout starting 8051 firmware\n",
+			   __func__);
 
 	/* Take away reset for LCB and RX FPE (set in lcb_shutdown). */
 	write_csr(dd, DCC_CFG_RESET, 0x10);
@@ -6904,6 +6906,32 @@ void handle_freeze(struct work_struct *work)
 	/* no longer frozen */
 }
 
+/**
+ * update_xmit_counters - update PortXmitWait/PortVlXmitWait
+ * counters.
+ * @ppd: info of physical Hfi port
+ * @link_width: new link width after link up or downgrade
+ *
+ * Update the PortXmitWait and PortVlXmitWait counters after
+ * a link up or downgrade event to reflect a link width change.
+ */
+static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width)
+{
+	int i;
+	u16 tx_width;
+	u16 link_speed;
+
+	tx_width = tx_link_width(link_width);
+	link_speed = get_link_speed(ppd->link_speed_active);
+
+	/*
+	 * There are C_VL_COUNT number of PortVLXmitWait counters.
+	 * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
+	 */
+	for (i = 0; i < C_VL_COUNT + 1; i++)
+		get_xmit_wait_counters(ppd, tx_width, link_speed, i);
+}
+
 /*
  * Handle a link up interrupt from the 8051.
  *
@@ -7525,18 +7553,29 @@ void handle_verify_cap(struct work_struct *work)
 	set_link_state(ppd, HLS_GOING_UP);
 }
 
-/*
- * Apply the link width downgrade enabled policy against the current active
- * link widths.
+/**
+ * apply_link_downgrade_policy - Apply the link width downgrade enabled
+ * policy against the current active link widths.
+ * @ppd: info of physical Hfi port
+ * @refresh_widths: True indicates link downgrade event
+ * @return: True indicates a successful link downgrade. False indicates
+ *	    link downgrade event failed and the link will bounce back to
+ *	    default link width.
  *
- * Called when the enabled policy changes or the active link widths change.
+ * Called when the enabled policy changes or the active link widths
+ * change.
+ * Refresh_widths indicates that a link downgrade occurred. The
+ * link_downgraded variable is set by refresh_widths and
+ * determines the success/failure of the policy application.
  */
-void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
+bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
+				 bool refresh_widths)
 {
 	int do_bounce = 0;
 	int tries;
 	u16 lwde;
 	u16 tx, rx;
+	bool link_downgraded = refresh_widths;
 
 	/* use the hls lock to avoid a race with actual link up */
 	tries = 0;
@@ -7570,6 +7609,7 @@ void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
 	    ppd->link_width_downgrade_rx_active == 0) {
 		/* the 8051 reported a dead link as a downgrade */
 		dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
+		link_downgraded = false;
 	} else if (lwde == 0) {
 		/* downgrade is disabled */
 
@@ -7586,6 +7626,7 @@ void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
 				   ppd->link_width_downgrade_tx_active,
 				   ppd->link_width_downgrade_rx_active);
 			do_bounce = 1;
+			link_downgraded = false;
 		}
 	} else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
 		   (lwde & ppd->link_width_downgrade_rx_active) == 0) {
@@ -7597,6 +7638,7 @@ void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
 			   lwde, ppd->link_width_downgrade_tx_active,
 			   ppd->link_width_downgrade_rx_active);
 		do_bounce = 1;
+		link_downgraded = false;
 	}
 
 done:
@@ -7608,6 +7650,8 @@ void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
 		set_link_state(ppd, HLS_DN_OFFLINE);
 		start_link(ppd);
 	}
+
+	return link_downgraded;
 }
 
 /*
@@ -7621,7 +7665,8 @@ void handle_link_downgrade(struct work_struct *work)
 							link_downgrade_work);
 
 	dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
-	apply_link_downgrade_policy(ppd, 1);
+	if (apply_link_downgrade_policy(ppd, true))
+		update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active);
 }
 
 static char *dcc_err_string(char *buf, int buf_len, u64 flags)
@@ -8263,8 +8308,8 @@ static irqreturn_t sdma_interrupt(int irq, void *data)
 		/* handle the interrupt(s) */
 		sdma_engine_interrupt(sde, status);
 	} else {
-		dd_dev_err_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n",
-				       sde->this_idx);
+		dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n",
+					sde->this_idx);
 	}
 	return IRQ_HANDLED;
 }
@@ -8564,23 +8609,27 @@ int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data)
 }
 
 /*
- * If the 8051 is in reset mode (dd->dc_shutdown == 1), this function
- * will still continue executing.
- *
  * Returns:
  *	< 0 = Linux error, not able to get access
  *	> 0 = 8051 command RETURN_CODE
  */
-static int _do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
-			    u64 *out_data)
+static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
+			   u64 *out_data)
 {
 	u64 reg, completed;
 	int return_code;
 	unsigned long timeout;
 
-	lockdep_assert_held(&dd->dc8051_lock);
 	hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data);
 
+	mutex_lock(&dd->dc8051_lock);
+
+	/* We can't send any commands to the 8051 if it's in reset */
+	if (dd->dc_shutdown) {
+		return_code = -ENODEV;
+		goto fail;
+	}
+
 	/*
 	 * If an 8051 host command timed out previously, then the 8051 is
 	 * stuck.
@@ -8680,29 +8729,6 @@ static int _do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
 	 */
 	write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0);
 
-fail:
-	return return_code;
-}
-
-/*
- * Returns:
- *	< 0 = Linux error, not able to get access
- *	> 0 = 8051 command RETURN_CODE
- */
-static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data,
-			   u64 *out_data)
-{
-	int return_code;
-
-	mutex_lock(&dd->dc8051_lock);
-	/* We can't send any commands to the 8051 if it's in reset */
-	if (dd->dc_shutdown) {
-		return_code = -ENODEV;
-		goto fail;
-	}
-
-	return_code = _do_8051_command(dd, type, in_data, out_data);
-
 fail:
 	mutex_unlock(&dd->dc8051_lock);
 	return return_code;
@@ -8713,17 +8739,16 @@ static int set_physical_link_state(struct hfi1_devdata *dd, u64 state)
 	return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL);
 }
 
-static int _load_8051_config(struct hfi1_devdata *dd, u8 field_id,
-			     u8 lane_id, u32 config_data)
+int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
+		     u8 lane_id, u32 config_data)
 {
 	u64 data;
 	int ret;
 
-	lockdep_assert_held(&dd->dc8051_lock);
 	data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT
 		| (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT
 		| (u64)config_data << LOAD_DATA_DATA_SHIFT;
-	ret = _do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
+	ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL);
 	if (ret != HCMD_SUCCESS) {
 		dd_dev_err(dd,
 			   "load 8051 config: field id %d, lane %d, err %d\n",
@@ -8732,18 +8757,6 @@ static int _load_8051_config(struct hfi1_devdata *dd, u8 field_id,
 	return ret;
 }
 
-int load_8051_config(struct hfi1_devdata *dd, u8 field_id,
-		     u8 lane_id, u32 config_data)
-{
-	int return_code;
-
-	mutex_lock(&dd->dc8051_lock);
-	return_code = _load_8051_config(dd, field_id, lane_id, config_data);
-	mutex_unlock(&dd->dc8051_lock);
-
-	return return_code;
-}
-
 /*
  * Read the 8051 firmware "registers".  Use the RAM directly.  Always
  * set the result, even on error.
@@ -8859,14 +8872,13 @@ int write_host_interface_version(struct hfi1_devdata *dd, u8 version)
 	u32 frame;
 	u32 mask;
 
-	lockdep_assert_held(&dd->dc8051_lock);
 	mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT);
 	read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame);
 	/* Clear, then set field */
 	frame &= ~mask;
 	frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT);
-	return _load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG,
-				 frame);
+	return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG,
+				frame);
 }
 
 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
@@ -9270,6 +9282,14 @@ static int set_local_link_attributes(struct hfi1_pportdata *ppd)
 	if (ret != HCMD_SUCCESS)
 		goto set_local_link_attributes_fail;
 
+	ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
+	if (ret != HCMD_SUCCESS) {
+		dd_dev_err(dd,
+			   "Failed to set host interface version, return 0x%x\n",
+			   ret);
+		goto set_local_link_attributes_fail;
+	}
+
 	/*
 	 * DC supports continuous updates.
 	 */
@@ -10621,6 +10641,14 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 		add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
 
 		handle_linkup_change(dd, 1);
+
+		/*
+		 * After link up, a new link width will have been set.
+		 * Update the xmit counters with regards to the new
+		 * link width.
+		 */
+		update_xmit_counters(ppd, ppd->link_width_active);
+
 		ppd->host_link_state = HLS_UP_INIT;
 		update_statusp(ppd, IB_PORT_INIT);
 		break;
@@ -12984,7 +13012,14 @@ static void disable_intx(struct pci_dev *pdev)
 	pci_intx(pdev, 0);
 }
 
-static void clean_up_interrupts(struct hfi1_devdata *dd)
+/**
+ * hfi1_clean_up_interrupts() - Free all IRQ resources
+ * @dd: valid device data data structure
+ *
+ * Free the MSI or INTx IRQs and assoicated PCI resources,
+ * if they have been allocated.
+ */
+void hfi1_clean_up_interrupts(struct hfi1_devdata *dd)
 {
 	int i;
 
@@ -13345,7 +13380,7 @@ static int set_up_interrupts(struct hfi1_devdata *dd)
 	return 0;
 
 fail:
-	clean_up_interrupts(dd);
+	hfi1_clean_up_interrupts(dd);
 	return ret;
 }
 
@@ -14772,7 +14807,6 @@ void hfi1_start_cleanup(struct hfi1_devdata *dd)
 	aspm_exit(dd);
 	free_cntrs(dd);
 	free_rcverr(dd);
-	clean_up_interrupts(dd);
 	finish_chip_resources(dd);
 }
 
@@ -14944,9 +14978,8 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 
 		if (num_vls < HFI1_MIN_VLS_SUPPORTED ||
 		    num_vls > HFI1_MAX_VLS_SUPPORTED) {
-			hfi1_early_err(&pdev->dev,
-				       "Invalid num_vls %u, using %u VLs\n",
-				    num_vls, HFI1_MAX_VLS_SUPPORTED);
+			dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n",
+				   num_vls, HFI1_MAX_VLS_SUPPORTED);
 			num_vls = HFI1_MAX_VLS_SUPPORTED;
 		}
 		ppd->vls_supported = num_vls;
@@ -15229,7 +15262,7 @@ struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev,
 bail_free_cntrs:
 	free_cntrs(dd);
 bail_clear_intr:
-	clean_up_interrupts(dd);
+	hfi1_clean_up_interrupts(dd);
 bail_cleanup:
 	hfi1_pcie_ddcleanup(dd);
 bail_free:
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index 133e313feca4..c0d70f255050 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -508,6 +508,7 @@
 #define DOWN_REMOTE_REASON_SHIFT 16
 #define DOWN_REMOTE_REASON_MASK  0xff
 
+#define HOST_INTERFACE_VERSION 1
 #define HOST_INTERFACE_VERSION_SHIFT 16
 #define HOST_INTERFACE_VERSION_MASK  0xff
 
@@ -713,7 +714,6 @@ void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor,
 		      u8 *ver_patch);
 int write_host_interface_version(struct hfi1_devdata *dd, u8 version);
 void read_guid(struct hfi1_devdata *dd);
-int release_and_wait_ready_8051_firmware(struct hfi1_devdata *dd);
 int wait_fm_ready(struct hfi1_devdata *dd, u32 mstimeout);
 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason,
 			  u8 neigh_reason, u8 rem_reason);
@@ -736,8 +736,8 @@ int read_8051_config(struct hfi1_devdata *, u8, u8, u32 *);
 int start_link(struct hfi1_pportdata *ppd);
 int bringup_serdes(struct hfi1_pportdata *ppd);
 void set_intr_state(struct hfi1_devdata *dd, u32 enable);
-void apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
-				 int refresh_widths);
+bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
+				 bool refresh_widths);
 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
 		    u32 intr_adjust, u32 npkts);
 int stop_drain_data_vls(struct hfi1_devdata *dd);
diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c
index 2e6e0c516041..852173bf05d0 100644
--- a/drivers/infiniband/hw/hfi1/debugfs.c
+++ b/drivers/infiniband/hw/hfi1/debugfs.c
@@ -1,5 +1,5 @@
 /*
- * Copyright(c) 2015-2017 Intel Corporation.
+ * Copyright(c) 2015-2018 Intel Corporation.
  *
  * This file is provided under a dual BSD/GPLv2 license.  When using or
  * redistributing this file, you may do so under either license.
@@ -1201,6 +1201,13 @@ static int _fault_stats_seq_show(struct seq_file *s, void *v)
 		}
 		hfi1_rcd_put(rcd);
 	}
+	for_each_possible_cpu(j) {
+		struct hfi1_opcode_stats_perctx *sp =
+			per_cpu_ptr(dd->tx_opstats, j);
+
+		n_packets += sp->stats[i].n_packets;
+		n_bytes += sp->stats[i].n_bytes;
+	}
 	if (!n_packets && !n_bytes)
 		return SEQ_SKIP;
 	if (!ibd->fault_opcode->n_rxfaults[i] &&
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index 4f65ac671044..addc68e83606 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -159,22 +159,6 @@ static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
 	return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
 }
 
-const char *get_unit_name(int unit)
-{
-	static char iname[16];
-
-	snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit);
-	return iname;
-}
-
-const char *get_card_name(struct rvt_dev_info *rdi)
-{
-	struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
-	struct hfi1_devdata *dd = container_of(ibdev,
-					       struct hfi1_devdata, verbs_dev);
-	return get_unit_name(dd->unit);
-}
-
 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
 {
 	struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
@@ -272,7 +256,12 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 	u32 mlid_base;
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
 	struct hfi1_devdata *dd = ppd->dd;
-	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
+	struct hfi1_ibdev *verbs_dev = &dd->verbs_dev;
+	struct rvt_dev_info *rdi = &verbs_dev->rdi;
+
+	if ((packet->rhf & RHF_DC_ERR) &&
+	    hfi1_dbg_fault_suppress_err(verbs_dev))
+		return;
 
 	if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
 		return;
@@ -650,9 +639,10 @@ static void __prescan_rxq(struct hfi1_packet *packet)
 	}
 }
 
-static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd)
+static void process_rcv_qp_work(struct hfi1_packet *packet)
 {
 	struct rvt_qp *qp, *nqp;
+	struct hfi1_ctxtdata *rcd = packet->rcd;
 
 	/*
 	 * Iterate over all QPs waiting to respond.
@@ -662,7 +652,8 @@ static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd)
 		list_del_init(&qp->rspwait);
 		if (qp->r_flags & RVT_R_RSP_NAK) {
 			qp->r_flags &= ~RVT_R_RSP_NAK;
-			hfi1_send_rc_ack(rcd, qp, 0);
+			packet->qp = qp;
+			hfi1_send_rc_ack(packet, 0);
 		}
 		if (qp->r_flags & RVT_R_RSP_SEND) {
 			unsigned long flags;
@@ -683,7 +674,7 @@ static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread)
 	if (thread) {
 		if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0)
 			/* allow defered processing */
-			process_rcv_qp_work(packet->rcd);
+			process_rcv_qp_work(packet);
 		cond_resched();
 		return RCV_PKT_OK;
 	} else {
@@ -825,7 +816,7 @@ int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
 			last = RCV_PKT_DONE;
 		process_rcv_update(last, &packet);
 	}
-	process_rcv_qp_work(rcd);
+	process_rcv_qp_work(&packet);
 	rcd->head = packet.rhqoff;
 bail:
 	finish_packet(&packet);
@@ -854,7 +845,7 @@ int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
 			last = RCV_PKT_DONE;
 		process_rcv_update(last, &packet);
 	}
-	process_rcv_qp_work(rcd);
+	process_rcv_qp_work(&packet);
 	rcd->head = packet.rhqoff;
 bail:
 	finish_packet(&packet);
@@ -1084,7 +1075,7 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
 		process_rcv_update(last, &packet);
 	}
 
-	process_rcv_qp_work(rcd);
+	process_rcv_qp_work(&packet);
 	rcd->head = packet.rhqoff;
 
 bail:
@@ -1454,8 +1445,8 @@ static int hfi1_setup_9B_packet(struct hfi1_packet *packet)
 	packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf);
 	packet->pad = ib_bth_get_pad(packet->ohdr);
 	packet->extra_byte = 0;
-	packet->fecn = ib_bth_get_fecn(packet->ohdr);
-	packet->becn = ib_bth_get_becn(packet->ohdr);
+	packet->pkey = ib_bth_get_pkey(packet->ohdr);
+	packet->migrated = ib_bth_is_migration(packet->ohdr);
 
 	return 0;
 drop:
@@ -1508,8 +1499,10 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
 
 	/* Query commonly used fields from packet header */
 	packet->opcode = ib_bth_get_opcode(packet->ohdr);
-	packet->hlen = hdr_len_by_opcode[packet->opcode] + 8 + grh_len;
-	packet->payload = packet->ebuf + packet->hlen - (4 * sizeof(u32));
+	/* hdr_len_by_opcode already has an IB LRH factored in */
+	packet->hlen = hdr_len_by_opcode[packet->opcode] +
+		(LRH_16B_BYTES - LRH_9B_BYTES) + grh_len;
+	packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES;
 	packet->slid = hfi1_16B_get_slid(packet->hdr);
 	packet->dlid = hfi1_16B_get_dlid(packet->hdr);
 	if (unlikely(hfi1_is_16B_mcast(packet->dlid)))
@@ -1520,8 +1513,8 @@ static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
 	packet->sl = ibp->sc_to_sl[packet->sc];
 	packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
 	packet->extra_byte = SIZE_OF_LT;
-	packet->fecn = hfi1_16B_get_fecn(packet->hdr);
-	packet->becn = hfi1_16B_get_becn(packet->hdr);
+	packet->pkey = hfi1_16B_get_pkey(packet->hdr);
+	packet->migrated = opa_bth_is_migration(packet->ohdr);
 
 	if (hfi1_bypass_ingress_pkt_check(packet))
 		goto drop;
@@ -1566,19 +1559,7 @@ int process_receive_ib(struct hfi1_packet *packet)
 	if (hfi1_setup_9B_packet(packet))
 		return RHF_RCV_CONTINUE;
 
-	trace_hfi1_rcvhdr(packet->rcd->ppd->dd,
-			  packet->rcd->ctxt,
-			  rhf_err_flags(packet->rhf),
-			  RHF_RCV_TYPE_IB,
-			  packet->hlen,
-			  packet->tlen,
-			  packet->updegr,
-			  rhf_egr_index(packet->rhf));
-
-	if (unlikely(
-		 (hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
-		 (packet->rhf & RHF_DC_ERR))))
-		return RHF_RCV_CONTINUE;
+	trace_hfi1_rcvhdr(packet);
 
 	if (unlikely(rhf_err_flags(packet->rhf))) {
 		handle_eflags(packet);
@@ -1614,6 +1595,8 @@ int process_receive_bypass(struct hfi1_packet *packet)
 	if (hfi1_setup_bypass_packet(packet))
 		return RHF_RCV_CONTINUE;
 
+	trace_hfi1_rcvhdr(packet);
+
 	if (unlikely(rhf_err_flags(packet->rhf))) {
 		handle_eflags(packet);
 		return RHF_RCV_CONTINUE;
diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c
index 1df7da47f431..41fafebe3b0d 100644
--- a/drivers/infiniband/hw/hfi1/file_ops.c
+++ b/drivers/infiniband/hw/hfi1/file_ops.c
@@ -74,7 +74,7 @@
 static int hfi1_file_open(struct inode *inode, struct file *fp);
 static int hfi1_file_close(struct inode *inode, struct file *fp);
 static ssize_t hfi1_write_iter(struct kiocb *kiocb, struct iov_iter *from);
-static unsigned int hfi1_poll(struct file *fp, struct poll_table_struct *pt);
+static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt);
 static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma);
 
 static u64 kvirt_to_phys(void *addr);
@@ -102,8 +102,8 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd,
 			 struct hfi1_user_info *uinfo,
 			 struct hfi1_ctxtdata **cd);
 static void deallocate_ctxt(struct hfi1_ctxtdata *uctxt);
-static unsigned int poll_urgent(struct file *fp, struct poll_table_struct *pt);
-static unsigned int poll_next(struct file *fp, struct poll_table_struct *pt);
+static __poll_t poll_urgent(struct file *fp, struct poll_table_struct *pt);
+static __poll_t poll_next(struct file *fp, struct poll_table_struct *pt);
 static int user_event_ack(struct hfi1_ctxtdata *uctxt, u16 subctxt,
 			  unsigned long arg);
 static int set_ctxt_pkey(struct hfi1_ctxtdata *uctxt, unsigned long arg);
@@ -196,9 +196,6 @@ static int hfi1_file_open(struct inode *inode, struct file *fp)
 	if (!atomic_inc_not_zero(&dd->user_refcount))
 		return -ENXIO;
 
-	/* Just take a ref now. Not all opens result in a context assign */
-	kobject_get(&dd->kobj);
-
 	/* The real work is performed later in assign_ctxt() */
 
 	fd = kzalloc(sizeof(*fd), GFP_KERNEL);
@@ -208,6 +205,7 @@ static int hfi1_file_open(struct inode *inode, struct file *fp)
 		fd->mm = current->mm;
 		mmgrab(fd->mm);
 		fd->dd = dd;
+		kobject_get(&fd->dd->kobj);
 		fp->private_data = fd;
 	} else {
 		fp->private_data = NULL;
@@ -607,20 +605,20 @@ static int vma_fault(struct vm_fault *vmf)
 	return 0;
 }
 
-static unsigned int hfi1_poll(struct file *fp, struct poll_table_struct *pt)
+static __poll_t hfi1_poll(struct file *fp, struct poll_table_struct *pt)
 {
 	struct hfi1_ctxtdata *uctxt;
-	unsigned pollflag;
+	__poll_t pollflag;
 
 	uctxt = ((struct hfi1_filedata *)fp->private_data)->uctxt;
 	if (!uctxt)
-		pollflag = POLLERR;
+		pollflag = EPOLLERR;
 	else if (uctxt->poll_type == HFI1_POLL_TYPE_URGENT)
 		pollflag = poll_urgent(fp, pt);
 	else  if (uctxt->poll_type == HFI1_POLL_TYPE_ANYRCV)
 		pollflag = poll_next(fp, pt);
 	else /* invalid */
-		pollflag = POLLERR;
+		pollflag = EPOLLERR;
 
 	return pollflag;
 }
@@ -1425,19 +1423,19 @@ static int user_exp_rcv_invalid(struct hfi1_filedata *fd, unsigned long arg,
 	return ret;
 }
 
-static unsigned int poll_urgent(struct file *fp,
+static __poll_t poll_urgent(struct file *fp,
 				struct poll_table_struct *pt)
 {
 	struct hfi1_filedata *fd = fp->private_data;
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
-	unsigned pollflag;
+	__poll_t pollflag;
 
 	poll_wait(fp, &uctxt->wait, pt);
 
 	spin_lock_irq(&dd->uctxt_lock);
 	if (uctxt->urgent != uctxt->urgent_poll) {
-		pollflag = POLLIN | POLLRDNORM;
+		pollflag = EPOLLIN | EPOLLRDNORM;
 		uctxt->urgent_poll = uctxt->urgent;
 	} else {
 		pollflag = 0;
@@ -1448,13 +1446,13 @@ static unsigned int poll_urgent(struct file *fp,
 	return pollflag;
 }
 
-static unsigned int poll_next(struct file *fp,
+static __poll_t poll_next(struct file *fp,
 			      struct poll_table_struct *pt)
 {
 	struct hfi1_filedata *fd = fp->private_data;
 	struct hfi1_ctxtdata *uctxt = fd->uctxt;
 	struct hfi1_devdata *dd = uctxt->dd;
-	unsigned pollflag;
+	__poll_t pollflag;
 
 	poll_wait(fp, &uctxt->wait, pt);
 
@@ -1464,7 +1462,7 @@ static unsigned int poll_next(struct file *fp,
 		hfi1_rcvctrl(dd, HFI1_RCVCTRL_INTRAVAIL_ENB, uctxt);
 		pollflag = 0;
 	} else {
-		pollflag = POLLIN | POLLRDNORM;
+		pollflag = EPOLLIN | EPOLLRDNORM;
 	}
 	spin_unlock_irq(&dd->uctxt_lock);
 
diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c
index 98868df78a7e..2b57ba70ddd6 100644
--- a/drivers/infiniband/hw/hfi1/firmware.c
+++ b/drivers/infiniband/hw/hfi1/firmware.c
@@ -68,7 +68,6 @@
 #define ALT_FW_FABRIC_NAME "hfi1_fabric_d.fw"
 #define ALT_FW_SBUS_NAME "hfi1_sbus_d.fw"
 #define ALT_FW_PCIE_NAME "hfi1_pcie_d.fw"
-#define HOST_INTERFACE_VERSION 1
 
 MODULE_FIRMWARE(DEFAULT_FW_8051_NAME_ASIC);
 MODULE_FIRMWARE(DEFAULT_FW_FABRIC_NAME);
@@ -975,46 +974,6 @@ int wait_fm_ready(struct hfi1_devdata *dd, u32 mstimeout)
 	}
 }
 
-/*
- * Clear all reset bits, releasing the 8051.
- * Wait for firmware to be ready to accept host requests.
- * Then, set host version bit.
- *
- * This function executes even if the 8051 is in reset mode when
- * dd->dc_shutdown == 1.
- *
- * Expects dd->dc8051_lock to be held.
- */
-int release_and_wait_ready_8051_firmware(struct hfi1_devdata *dd)
-{
-	int ret;
-
-	lockdep_assert_held(&dd->dc8051_lock);
-	/* clear all reset bits, releasing the 8051 */
-	write_csr(dd, DC_DC8051_CFG_RST, 0ull);
-
-	/*
-	 * Wait for firmware to be ready to accept host
-	 * requests.
-	 */
-	ret = wait_fm_ready(dd, TIMEOUT_8051_START);
-	if (ret) {
-		dd_dev_err(dd, "8051 start timeout, current FW state 0x%x\n",
-			   get_firmware_state(dd));
-		return ret;
-	}
-
-	ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
-	if (ret != HCMD_SUCCESS) {
-		dd_dev_err(dd,
-			   "Failed to set host interface version, return 0x%x\n",
-			   ret);
-		return -EIO;
-	}
-
-	return 0;
-}
-
 /*
  * Load the 8051 firmware.
  */
@@ -1080,22 +1039,31 @@ static int load_8051_firmware(struct hfi1_devdata *dd,
 	if (ret)
 		return ret;
 
+	/* clear all reset bits, releasing the 8051 */
+	write_csr(dd, DC_DC8051_CFG_RST, 0ull);
+
 	/*
-	 * Clear all reset bits, releasing the 8051.
 	 * DC reset step 5. Wait for firmware to be ready to accept host
 	 * requests.
-	 * Then, set host version bit.
 	 */
-	mutex_lock(&dd->dc8051_lock);
-	ret = release_and_wait_ready_8051_firmware(dd);
-	mutex_unlock(&dd->dc8051_lock);
-	if (ret)
-		return ret;
+	ret = wait_fm_ready(dd, TIMEOUT_8051_START);
+	if (ret) { /* timed out */
+		dd_dev_err(dd, "8051 start timeout, current state 0x%x\n",
+			   get_firmware_state(dd));
+		return -ETIMEDOUT;
+	}
 
 	read_misc_status(dd, &ver_major, &ver_minor, &ver_patch);
 	dd_dev_info(dd, "8051 firmware version %d.%d.%d\n",
 		    (int)ver_major, (int)ver_minor, (int)ver_patch);
 	dd->dc8051_ver = dc8051_ver(ver_major, ver_minor, ver_patch);
+	ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION);
+	if (ret != HCMD_SUCCESS) {
+		dd_dev_err(dd,
+			   "Failed to set host interface version, return 0x%x\n",
+			   ret);
+		return -EIO;
+	}
 
 	return 0;
 }
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index 8ce9118d4a7f..90bc8c76d2ca 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -341,6 +341,7 @@ struct hfi1_packet {
 	u32 slid;
 	u16 tlen;
 	s16 etail;
+	u16 pkey;
 	u8 hlen;
 	u8 numpkt;
 	u8 rsize;
@@ -351,8 +352,7 @@ struct hfi1_packet {
 	u8 sc;
 	u8 sl;
 	u8 opcode;
-	bool becn;
-	bool fecn;
+	bool migrated;
 };
 
 /* Packet types */
@@ -858,6 +858,13 @@ struct hfi1_pportdata {
 	struct work_struct linkstate_active_work;
 	/* Does this port need to prescan for FECNs */
 	bool cc_prescan;
+	/*
+	 * Sample sendWaitCnt & sendWaitVlCnt during link transition
+	 * and counter request.
+	 */
+	u64 port_vl_xmit_wait_last[C_VL_COUNT + 1];
+	u16 prev_link_width;
+	u64 vl_xmit_flit_cnt[C_VL_COUNT + 1];
 };
 
 typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);
@@ -1623,7 +1630,7 @@ static int ingress_pkey_table_search(struct hfi1_pportdata *ppd, u16 pkey)
  * the 'error info' for this failure.
  */
 static void ingress_pkey_table_fail(struct hfi1_pportdata *ppd, u16 pkey,
-				    u16 slid)
+				    u32 slid)
 {
 	struct hfi1_devdata *dd = ppd->dd;
 
@@ -1779,19 +1786,15 @@ void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 static inline bool process_ecn(struct rvt_qp *qp, struct hfi1_packet *pkt,
 			       bool do_cnp)
 {
-	struct ib_other_headers *ohdr = pkt->ohdr;
-
-	u32 bth1;
-	bool becn = false;
-	bool fecn = false;
+	bool becn;
+	bool fecn;
 
 	if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
 		fecn = hfi1_16B_get_fecn(pkt->hdr);
 		becn = hfi1_16B_get_becn(pkt->hdr);
 	} else {
-		bth1 = be32_to_cpu(ohdr->bth[1]);
-		fecn = bth1 & IB_FECN_SMASK;
-		becn = bth1 & IB_BECN_SMASK;
+		fecn = ib_bth_get_fecn(pkt->ohdr);
+		becn = ib_bth_get_becn(pkt->ohdr);
 	}
 	if (unlikely(fecn || becn)) {
 		hfi1_process_ecn_slowpath(qp, pkt, do_cnp);
@@ -1957,6 +1960,7 @@ void hfi1_verbs_unregister_sysfs(struct hfi1_devdata *dd);
 int qsfp_dump(struct hfi1_pportdata *ppd, char *buf, int len);
 
 int hfi1_pcie_init(struct pci_dev *pdev, const struct pci_device_id *ent);
+void hfi1_clean_up_interrupts(struct hfi1_devdata *dd);
 void hfi1_pcie_cleanup(struct pci_dev *pdev);
 int hfi1_pcie_ddinit(struct hfi1_devdata *dd, struct pci_dev *pdev);
 void hfi1_pcie_ddcleanup(struct hfi1_devdata *);
@@ -1971,8 +1975,6 @@ int get_platform_config_field(struct hfi1_devdata *dd,
 			      table_type, int table_index, int field_index,
 			      u32 *data, u32 len);
 
-const char *get_unit_name(int unit);
-const char *get_card_name(struct rvt_dev_info *rdi);
 struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi);
 
 /*
@@ -2122,39 +2124,42 @@ static inline u64 hfi1_pkt_base_sdma_integrity(struct hfi1_devdata *dd)
 
 #define dd_dev_emerg(dd, fmt, ...) \
 	dev_emerg(&(dd)->pcidev->dev, "%s: " fmt, \
-		  get_unit_name((dd)->unit), ##__VA_ARGS__)
+		  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
 
 #define dd_dev_err(dd, fmt, ...) \
 	dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
-			get_unit_name((dd)->unit), ##__VA_ARGS__)
+		rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
 
 #define dd_dev_err_ratelimited(dd, fmt, ...) \
 	dev_err_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
-			get_unit_name((dd)->unit), ##__VA_ARGS__)
+			    rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
+			    ##__VA_ARGS__)
 
 #define dd_dev_warn(dd, fmt, ...) \
 	dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
-			get_unit_name((dd)->unit), ##__VA_ARGS__)
+		 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
 
 #define dd_dev_warn_ratelimited(dd, fmt, ...) \
 	dev_warn_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
-			get_unit_name((dd)->unit), ##__VA_ARGS__)
+			     rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
+			     ##__VA_ARGS__)
 
 #define dd_dev_info(dd, fmt, ...) \
 	dev_info(&(dd)->pcidev->dev, "%s: " fmt, \
-			get_unit_name((dd)->unit), ##__VA_ARGS__)
+		 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
 
 #define dd_dev_info_ratelimited(dd, fmt, ...) \
 	dev_info_ratelimited(&(dd)->pcidev->dev, "%s: " fmt, \
-			get_unit_name((dd)->unit), ##__VA_ARGS__)
+			     rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), \
+			     ##__VA_ARGS__)
 
 #define dd_dev_dbg(dd, fmt, ...) \
 	dev_dbg(&(dd)->pcidev->dev, "%s: " fmt, \
-		get_unit_name((dd)->unit), ##__VA_ARGS__)
+		rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
 
 #define hfi1_dev_porterr(dd, port, fmt, ...) \
 	dev_err(&(dd)->pcidev->dev, "%s: port %u: " fmt, \
-			get_unit_name((dd)->unit), (port), ##__VA_ARGS__)
+		rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), (port), ##__VA_ARGS__)
 
 /*
  * this is used for formatting hw error messages...
@@ -2415,7 +2420,7 @@ static inline void hfi1_make_ib_hdr(struct ib_header *hdr,
 static inline void hfi1_make_16b_hdr(struct hfi1_16b_header *hdr,
 				     u32 slid, u32 dlid,
 				     u16 len, u16 pkey,
-				     u8 becn, u8 fecn, u8 l4,
+				     bool becn, bool fecn, u8 l4,
 				     u8 sc)
 {
 	u32 lrh0 = 0;
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index 8e3b3e7d829a..33eba2356742 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -172,7 +172,7 @@ int hfi1_create_kctxts(struct hfi1_devdata *dd)
 	u16 i;
 	int ret;
 
-	dd->rcd = kzalloc_node(dd->num_rcv_contexts * sizeof(*dd->rcd),
+	dd->rcd = kcalloc_node(dd->num_rcv_contexts, sizeof(*dd->rcd),
 			       GFP_KERNEL, dd->node);
 	if (!dd->rcd)
 		return -ENOMEM;
@@ -439,15 +439,16 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
 		 * The resulting value will be rounded down to the closest
 		 * multiple of dd->rcv_entries.group_size.
 		 */
-		rcd->egrbufs.buffers = kzalloc_node(
-			rcd->egrbufs.count * sizeof(*rcd->egrbufs.buffers),
-			GFP_KERNEL, numa);
+		rcd->egrbufs.buffers =
+			kcalloc_node(rcd->egrbufs.count,
+				     sizeof(*rcd->egrbufs.buffers),
+				     GFP_KERNEL, numa);
 		if (!rcd->egrbufs.buffers)
 			goto bail;
-		rcd->egrbufs.rcvtids = kzalloc_node(
-				rcd->egrbufs.count *
-				sizeof(*rcd->egrbufs.rcvtids),
-				GFP_KERNEL, numa);
+		rcd->egrbufs.rcvtids =
+			kcalloc_node(rcd->egrbufs.count,
+				     sizeof(*rcd->egrbufs.rcvtids),
+				     GFP_KERNEL, numa);
 		if (!rcd->egrbufs.rcvtids)
 			goto bail;
 		rcd->egrbufs.size = eager_buffer_size;
@@ -637,6 +638,15 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
 	ppd->dd = dd;
 	ppd->hw_pidx = hw_pidx;
 	ppd->port = port; /* IB port number, not index */
+	ppd->prev_link_width = LINK_WIDTH_DEFAULT;
+	/*
+	 * There are C_VL_COUNT number of PortVLXmitWait counters.
+	 * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
+	 */
+	for (i = 0; i < C_VL_COUNT + 1; i++) {
+		ppd->port_vl_xmit_wait_last[i] = 0;
+		ppd->vl_xmit_flit_cnt[i] = 0;
+	}
 
 	default_pkey_idx = 1;
 
@@ -1058,8 +1068,9 @@ static void shutdown_device(struct hfi1_devdata *dd)
 	}
 	dd->flags &= ~HFI1_INITTED;
 
-	/* mask interrupts, but not errors */
+	/* mask and clean up interrupts, but not errors */
 	set_intr_state(dd, 0);
+	hfi1_clean_up_interrupts(dd);
 
 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
 		ppd = dd->pport + pidx;
@@ -1218,6 +1229,7 @@ static void __hfi1_free_devdata(struct kobject *kobj)
 	free_percpu(dd->rcv_limit);
 	free_percpu(dd->send_schedule);
 	free_percpu(dd->tx_opstats);
+	sdma_clean(dd, dd->num_sdma);
 	rvt_dealloc_device(&dd->verbs_dev.rdi);
 }
 
@@ -1272,6 +1284,8 @@ struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra)
 			       "Could not allocate unit ID: error %d\n", -ret);
 		goto bail;
 	}
+	rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit);
+
 	/*
 	 * Initialize all locks for the device. This needs to be as early as
 	 * possible so locks are usable.
@@ -1702,6 +1716,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		dd_dev_err(dd, "Failed to create /dev devices: %d\n", -j);
 
 	if (initfail || ret) {
+		hfi1_clean_up_interrupts(dd);
 		stop_timers(dd);
 		flush_workqueue(ib_wq);
 		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
diff --git a/drivers/infiniband/hw/hfi1/iowait.h b/drivers/infiniband/hw/hfi1/iowait.h
index 591697d85eed..3d9c32c7c340 100644
--- a/drivers/infiniband/hw/hfi1/iowait.h
+++ b/drivers/infiniband/hw/hfi1/iowait.h
@@ -371,4 +371,13 @@ static inline void iowait_starve_find_max(struct iowait *w, u8 *max,
 	}
 }
 
+/**
+ * iowait_packet_queued() - determine if a packet is already built
+ * @wait: the wait structure
+ */
+static inline bool iowait_packet_queued(struct iowait *wait)
+{
+	return !list_empty(&wait->tx_head);
+}
+
 #endif
diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c
index cf8dba34fe30..e9962c65c68f 100644
--- a/drivers/infiniband/hw/hfi1/mad.c
+++ b/drivers/infiniband/hw/hfi1/mad.c
@@ -2649,6 +2649,79 @@ static void a0_portstatus(struct hfi1_pportdata *ppd,
 	}
 }
 
+/**
+ * tx_link_width - convert link width bitmask to integer
+ * value representing actual link width.
+ * @link_width: width of active link
+ * @return: return index of the bit set in link_width var
+ *
+ * The function convert and return the index of bit set
+ * that indicate the current link width.
+ */
+u16 tx_link_width(u16 link_width)
+{
+	int n = LINK_WIDTH_DEFAULT;
+	u16 tx_width = n;
+
+	while (link_width && n) {
+		if (link_width & (1 << (n - 1))) {
+			tx_width = n;
+			break;
+		}
+		n--;
+	}
+
+	return tx_width;
+}
+
+/**
+ * get_xmit_wait_counters - Convert HFI 's SendWaitCnt/SendWaitVlCnt
+ * counter in unit of TXE cycle times to flit times.
+ * @ppd: info of physical Hfi port
+ * @link_width: width of active link
+ * @link_speed: speed of active link
+ * @vl: represent VL0-VL7, VL15 for PortVLXmitWait counters request
+ * and if vl value is C_VL_COUNT, it represent SendWaitCnt
+ * counter request
+ * @return: return SendWaitCnt/SendWaitVlCnt counter value per vl.
+ *
+ * Convert SendWaitCnt/SendWaitVlCnt counter from TXE cycle times to
+ * flit times. Call this function to samples these counters. This
+ * function will calculate for previous state transition and update
+ * current state at end of function using ppd->prev_link_width and
+ * ppd->port_vl_xmit_wait_last to port_vl_xmit_wait_curr and link_width.
+ */
+u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd,
+			   u16 link_width, u16 link_speed, int vl)
+{
+	u64 port_vl_xmit_wait_curr;
+	u64 delta_vl_xmit_wait;
+	u64 xmit_wait_val;
+
+	if (vl > C_VL_COUNT)
+		return  0;
+	if (vl < C_VL_COUNT)
+		port_vl_xmit_wait_curr =
+			read_port_cntr(ppd, C_TX_WAIT_VL, vl);
+	else
+		port_vl_xmit_wait_curr =
+			read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL);
+
+	xmit_wait_val =
+		port_vl_xmit_wait_curr -
+		ppd->port_vl_xmit_wait_last[vl];
+	delta_vl_xmit_wait =
+		convert_xmit_counter(xmit_wait_val,
+				     ppd->prev_link_width,
+				     link_speed);
+
+	ppd->vl_xmit_flit_cnt[vl] += delta_vl_xmit_wait;
+	ppd->port_vl_xmit_wait_last[vl] = port_vl_xmit_wait_curr;
+	ppd->prev_link_width = link_width;
+
+	return ppd->vl_xmit_flit_cnt[vl];
+}
+
 static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,
 				  struct ib_device *ibdev,
 				  u8 port, u32 *resp_len)
@@ -2668,6 +2741,8 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 	int vfi;
 	u64 tmp, tmp2;
+	u16 link_width;
+	u16 link_speed;
 
 	response_data_size = sizeof(struct opa_port_status_rsp) +
 				num_vls * sizeof(struct _vls_pctrs);
@@ -2711,8 +2786,16 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,
 	rsp->port_multicast_rcv_pkts =
 		cpu_to_be64(read_dev_cntr(dd, C_DC_MC_RCV_PKTS,
 					  CNTR_INVALID_VL));
+	/*
+	 * Convert PortXmitWait counter from TXE cycle times
+	 * to flit times.
+	 */
+	link_width =
+		tx_link_width(ppd->link_width_downgrade_tx_active);
+	link_speed = get_link_speed(ppd->link_speed_active);
 	rsp->port_xmit_wait =
-		cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL));
+		cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
+						   link_speed, C_VL_COUNT));
 	rsp->port_rcv_fecn =
 		cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL));
 	rsp->port_rcv_becn =
@@ -2777,10 +2860,14 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,
 		rsp->vls[vfi].port_vl_xmit_pkts =
 			cpu_to_be64(read_port_cntr(ppd, C_TX_PKT_VL,
 						   idx_from_vl(vl)));
-
+		/*
+		 * Convert PortVlXmitWait counter from TXE cycle
+		 * times to flit times.
+		 */
 		rsp->vls[vfi].port_vl_xmit_wait =
-			cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL,
-						   idx_from_vl(vl)));
+			cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
+							   link_speed,
+							   idx_from_vl(vl)));
 
 		rsp->vls[vfi].port_vl_rcv_fecn =
 			cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL,
@@ -2910,6 +2997,8 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,
 	unsigned long vl;
 	u32 vl_select_mask;
 	int vfi;
+	u16 link_width;
+	u16 link_speed;
 
 	num_ports = be32_to_cpu(pmp->mad_hdr.attr_mod) >> 24;
 	num_vls = hweight32(be32_to_cpu(req->vl_select_mask));
@@ -2959,8 +3048,16 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,
 	rsp->link_quality_indicator = cpu_to_be32((u32)lq);
 	pma_get_opa_port_dctrs(ibdev, rsp);
 
+	/*
+	 * Convert PortXmitWait counter from TXE
+	 * cycle times to flit times.
+	 */
+	link_width =
+		tx_link_width(ppd->link_width_downgrade_tx_active);
+	link_speed = get_link_speed(ppd->link_speed_active);
 	rsp->port_xmit_wait =
-		cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL));
+		cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
+						   link_speed, C_VL_COUNT));
 	rsp->port_rcv_fecn =
 		cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL));
 	rsp->port_rcv_becn =
@@ -2996,9 +3093,14 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp,
 			cpu_to_be64(read_dev_cntr(dd, C_DC_RX_PKT_VL,
 						  idx_from_vl(vl)));
 
+		/*
+		 * Convert PortVlXmitWait counter from TXE
+		 * cycle times to flit times.
+		 */
 		rsp->vls[vfi].port_vl_xmit_wait =
-			cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL,
-						   idx_from_vl(vl)));
+			cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
+							   link_speed,
+							   idx_from_vl(vl)));
 
 		rsp->vls[vfi].port_vl_rcv_fecn =
 			cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL,
@@ -3416,9 +3518,11 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp,
 	if (counter_select & CS_PORT_MCAST_RCV_PKTS)
 		write_dev_cntr(dd, C_DC_MC_RCV_PKTS, CNTR_INVALID_VL, 0);
 
-	if (counter_select & CS_PORT_XMIT_WAIT)
+	if (counter_select & CS_PORT_XMIT_WAIT) {
 		write_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL, 0);
-
+		ppd->port_vl_xmit_wait_last[C_VL_COUNT] = 0;
+		ppd->vl_xmit_flit_cnt[C_VL_COUNT] = 0;
+	}
 	/* ignore cs_sw_portCongestion for HFIs */
 
 	if (counter_select & CS_PORT_RCV_FECN)
@@ -3491,8 +3595,11 @@ static int pma_set_opa_portstatus(struct opa_pma_mad *pmp,
 		if (counter_select & CS_PORT_RCV_PKTS)
 			write_dev_cntr(dd, C_DC_RX_PKT_VL, idx_from_vl(vl), 0);
 
-		if (counter_select & CS_PORT_XMIT_WAIT)
+		if (counter_select & CS_PORT_XMIT_WAIT) {
 			write_port_cntr(ppd, C_TX_WAIT_VL, idx_from_vl(vl), 0);
+			ppd->port_vl_xmit_wait_last[idx_from_vl(vl)] = 0;
+			ppd->vl_xmit_flit_cnt[idx_from_vl(vl)] = 0;
+		}
 
 		/* sw_port_vl_congestion is 0 for HFIs */
 		if (counter_select & CS_PORT_RCV_FECN)
@@ -4348,11 +4455,7 @@ static int opa_local_smp_check(struct hfi1_ibport *ibp,
 	 */
 	if (pkey == LIM_MGMT_P_KEY || pkey == FULL_MGMT_P_KEY)
 		return 0;
-	/*
-	 * On OPA devices it is okay to lose the upper 16 bits of LID as this
-	 * information is obtained elsewhere. Mask off the upper 16 bits.
-	 */
-	ingress_pkey_table_fail(ppd, pkey, ib_lid_cpu16(0xFFFF & in_wc->slid));
+	ingress_pkey_table_fail(ppd, pkey, in_wc->slid);
 	return 1;
 }
 
diff --git a/drivers/infiniband/hw/hfi1/mad.h b/drivers/infiniband/hw/hfi1/mad.h
index c4938f3d97c8..2f48e6953629 100644
--- a/drivers/infiniband/hw/hfi1/mad.h
+++ b/drivers/infiniband/hw/hfi1/mad.h
@@ -180,6 +180,15 @@ struct opa_mad_notice_attr {
 #define OPA_VLARB_PREEMPT_MATRIX     3
 
 #define IB_PMA_PORT_COUNTERS_CONG       cpu_to_be16(0xFF00)
+#define LINK_SPEED_25G		1
+#define LINK_SPEED_12_5G	2
+#define LINK_WIDTH_DEFAULT	4
+#define DECIMAL_FACTORING	1000
+/*
+ * The default link width is multiplied by 1000
+ * to get accurate value after division.
+ */
+#define FACTOR_LINK_WIDTH	(LINK_WIDTH_DEFAULT * DECIMAL_FACTORING)
 
 struct ib_pma_portcounters_cong {
 	u8 reserved;
@@ -429,5 +438,41 @@ struct sc2vlnt {
 
 void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port);
 void hfi1_handle_trap_timer(struct timer_list *t);
-
+u16 tx_link_width(u16 link_width);
+u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, u16 link_width,
+			   u16 link_speed, int vl);
+/**
+ * get_link_speed - determine whether 12.5G or 25G speed
+ * @link_speed: the speed of active link
+ * @return: Return 2 if link speed identified as 12.5G
+ * or return 1 if link speed is 25G.
+ *
+ * The function indirectly calculate required link speed
+ * value for convert_xmit_counter function. If the link
+ * speed is 25G, the function return as 1 as it is required
+ * by xmit counter conversion formula :-( 25G / link_speed).
+ * This conversion will provide value 1 if current
+ * link speed is 25G or 2 if 12.5G.This is done to avoid
+ * 12.5 float number conversion.
+ */
+static inline u16 get_link_speed(u16 link_speed)
+{
+	return (link_speed == 1) ?
+		 LINK_SPEED_12_5G : LINK_SPEED_25G;
+}
+
+/**
+ * convert_xmit_counter - calculate flit times for given xmit counter
+ * value
+ * @xmit_wait_val: current xmit counter value
+ * @link_width: width of active link
+ * @link_speed: speed of active link
+ * @return: return xmit counter value in flit times.
+ */
+static inline u64 convert_xmit_counter(u64 xmit_wait_val, u16 link_width,
+				       u16 link_speed)
+{
+	return (xmit_wait_val * 2 * (FACTOR_LINK_WIDTH / link_width)
+		 * link_speed) / DECIMAL_FACTORING;
+}
 #endif				/* _HFI1_MAD_H */
diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c
index e7b3ce123da6..70aceefe14d5 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.c
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.c
@@ -77,6 +77,7 @@ static void do_remove(struct mmu_rb_handler *handler,
 static void handle_remove(struct work_struct *work);
 
 static const struct mmu_notifier_ops mn_opts = {
+	.flags = MMU_INVALIDATE_DOES_NOT_BLOCK,
 	.invalidate_range_start = mmu_notifier_range_start,
 };
 
diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c
index 8c7e7a60b715..83d66e862207 100644
--- a/drivers/infiniband/hw/hfi1/pcie.c
+++ b/drivers/infiniband/hw/hfi1/pcie.c
@@ -1034,6 +1034,7 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd)
 	int do_retry, retry_count = 0;
 	int intnum = 0;
 	uint default_pset;
+	uint pset = pcie_pset;
 	u16 target_vector, target_speed;
 	u16 lnkctl2, vendor;
 	u8 div;
@@ -1201,16 +1202,16 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd)
 	 *
 	 * Set Gen3EqPsetReqVec, leave other fields 0.
 	 */
-	if (pcie_pset == UNSET_PSET)
-		pcie_pset = default_pset;
-	if (pcie_pset > 10) {	/* valid range is 0-10, inclusive */
+	if (pset == UNSET_PSET)
+		pset = default_pset;
+	if (pset > 10) {	/* valid range is 0-10, inclusive */
 		dd_dev_err(dd, "%s: Invalid Eq Pset %u, setting to %d\n",
-			   __func__, pcie_pset, default_pset);
-		pcie_pset = default_pset;
+			   __func__, pset, default_pset);
+		pset = default_pset;
 	}
-	dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pcie_pset);
+	dd_dev_info(dd, "%s: using EQ Pset %u\n", __func__, pset);
 	pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL106,
-			       ((1 << pcie_pset) <<
+			       ((1 << pset) <<
 			PCIE_CFG_REG_PL106_GEN3_EQ_PSET_REQ_VEC_SHIFT) |
 			PCIE_CFG_REG_PL106_GEN3_EQ_EVAL2MS_DISABLE_SMASK |
 			PCIE_CFG_REG_PL106_GEN3_EQ_PHASE23_EXIT_MODE_SMASK);
@@ -1240,10 +1241,10 @@ int do_pcie_gen3_transition(struct hfi1_devdata *dd)
 		/* apply static CTLE tunings */
 		u8 pcie_dc, pcie_lf, pcie_hf, pcie_bw;
 
-		pcie_dc = ctle_tunings[pcie_pset][0];
-		pcie_lf = ctle_tunings[pcie_pset][1];
-		pcie_hf = ctle_tunings[pcie_pset][2];
-		pcie_bw = ctle_tunings[pcie_pset][3];
+		pcie_dc = ctle_tunings[pset][0];
+		pcie_lf = ctle_tunings[pset][1];
+		pcie_hf = ctle_tunings[pset][2];
+		pcie_bw = ctle_tunings[pset][3];
 		write_gasket_interrupt(dd, intnum++, 0x0026, 0x0200 | pcie_dc);
 		write_gasket_interrupt(dd, intnum++, 0x0026, 0x0100 | pcie_lf);
 		write_gasket_interrupt(dd, intnum++, 0x0026, 0x0000 | pcie_hf);
diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c
index 4c1198bc5e70..40dac4d16eb8 100644
--- a/drivers/infiniband/hw/hfi1/pio.c
+++ b/drivers/infiniband/hw/hfi1/pio.c
@@ -455,8 +455,8 @@ int init_send_contexts(struct hfi1_devdata *dd)
 	dd->hw_to_sw = kmalloc_array(TXE_NUM_CONTEXTS, sizeof(u8),
 					GFP_KERNEL);
 	dd->send_contexts = kcalloc(dd->num_send_contexts,
-					sizeof(struct send_context_info),
-					GFP_KERNEL);
+				    sizeof(struct send_context_info),
+				    GFP_KERNEL);
 	if (!dd->send_contexts || !dd->hw_to_sw) {
 		kfree(dd->hw_to_sw);
 		kfree(dd->send_contexts);
@@ -856,8 +856,9 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
 		 * so head == tail can mean empty.
 		 */
 		sc->sr_size = sci->credits + 1;
-		sc->sr = kzalloc_node(sizeof(union pio_shadow_ring) *
-				sc->sr_size, GFP_KERNEL, numa);
+		sc->sr = kcalloc_node(sc->sr_size,
+				      sizeof(union pio_shadow_ring),
+				      GFP_KERNEL, numa);
 		if (!sc->sr) {
 			sc_free(sc);
 			return NULL;
@@ -1958,9 +1959,9 @@ int init_pervl_scs(struct hfi1_devdata *dd)
 	hfi1_init_ctxt(dd->vld[15].sc);
 	dd->vld[15].mtu = enum_to_mtu(OPA_MTU_2048);
 
-	dd->kernel_send_context = kzalloc_node(dd->num_send_contexts *
-					sizeof(struct send_context *),
-					GFP_KERNEL, dd->node);
+	dd->kernel_send_context = kcalloc_node(dd->num_send_contexts,
+					       sizeof(struct send_context *),
+					       GFP_KERNEL, dd->node);
 	if (!dd->kernel_send_context)
 		goto freesc15;
 
diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c
index 4b01ccd895b4..d30dd1a5b0a6 100644
--- a/drivers/infiniband/hw/hfi1/qp.c
+++ b/drivers/infiniband/hw/hfi1/qp.c
@@ -556,6 +556,8 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)
 	struct sdma_engine *sde;
 	struct send_context *send_context;
 	struct rvt_ack_entry *e = NULL;
+	struct rvt_srq *srq = qp->ibqp.srq ?
+		ibsrq_to_rvtsrq(qp->ibqp.srq) : NULL;
 
 	sde = qp_to_sdma_engine(qp, priv->s_sc);
 	wqe = rvt_get_swqe_ptr(qp, qp->s_last);
@@ -563,7 +565,7 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)
 	if (qp->s_ack_queue)
 		e = &qp->s_ack_queue[qp->s_tail_ack_queue];
 	seq_printf(s,
-		   "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x\n",
+		   "N %d %s QP %x R %u %s %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x S(%u %u %u %u %u %u %u) R(%u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d OS %x %x E %x %x %x RNR %d %s %d\n",
 		   iter->n,
 		   qp_idle(qp) ? "I" : "B",
 		   qp->ibqp.qp_num,
@@ -571,7 +573,6 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)
 		   qp_type_str[qp->ibqp.qp_type],
 		   qp->state,
 		   wqe ? wqe->wr.opcode : 0,
-		   qp->s_hdrwords,
 		   qp->s_flags,
 		   iowait_sdma_pending(&priv->s_iowait),
 		   iowait_pio_pending(&priv->s_iowait),
@@ -610,7 +611,11 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter)
 		   /* ack queue information */
 		   e ? e->opcode : 0,
 		   e ? e->psn : 0,
-		   e ? e->lpsn : 0);
+		   e ? e->lpsn : 0,
+		   qp->r_min_rnr_timer,
+		   srq ? "SRQ" : "RQ",
+		   srq ? srq->rq.size : qp->r_rq.size
+		);
 }
 
 void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp)
@@ -789,7 +794,6 @@ void notify_error_qp(struct rvt_qp *qp)
 	}
 
 	if (!(qp->s_flags & RVT_S_BUSY)) {
-		qp->s_hdrwords = 0;
 		if (qp->s_rdma_mr) {
 			rvt_put_mr(qp->s_rdma_mr);
 			qp->s_rdma_mr = NULL;
diff --git a/drivers/infiniband/hw/hfi1/qp.h b/drivers/infiniband/hw/hfi1/qp.h
index c06d2f8348e0..b2d4cba8d15b 100644
--- a/drivers/infiniband/hw/hfi1/qp.h
+++ b/drivers/infiniband/hw/hfi1/qp.h
@@ -51,11 +51,24 @@
 #include <rdma/rdmavt_qp.h>
 #include "verbs.h"
 #include "sdma.h"
+#include "verbs_txreq.h"
 
 extern unsigned int hfi1_qp_table_size;
 
 extern const struct rvt_operation_params hfi1_post_parms[];
 
+/*
+ * Send if not busy or waiting for I/O and either
+ * a RC response is pending or we can process send work requests.
+ */
+static inline int hfi1_send_ok(struct rvt_qp *qp)
+{
+	return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) &&
+		(verbs_txreq_queued(qp) ||
+		(qp->s_flags & RVT_S_RESP_PENDING) ||
+		 !(qp->s_flags & RVT_S_ANY_WAIT_SEND));
+}
+
 /*
  * free_ahg - clear ahg from QP
  */
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index af5f7936f7e5..da58046a02ea 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -226,12 +226,10 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 		bth2 = mask_psn(qp->s_ack_psn);
 	}
 	qp->s_rdma_ack_cnt++;
-	qp->s_hdrwords = hwords;
 	ps->s_txreq->sde = priv->s_sde;
 	ps->s_txreq->s_cur_size = len;
+	ps->s_txreq->hdr_dwords = hwords;
 	hfi1_make_ruc_header(qp, ohdr, bth0, bth2, middle, ps);
-	/* pbc */
-	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
 	return 1;
 
 bail:
@@ -302,7 +300,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
 			goto bail;
 		/* We are in the error state, flush the work request. */
-		smp_read_barrier_depends(); /* see post_one_send() */
 		if (qp->s_last == READ_ONCE(qp->s_head))
 			goto bail;
 		/* If DMAs are in progress, we can't flush immediately. */
@@ -346,7 +343,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		newreq = 0;
 		if (qp->s_cur == qp->s_tail) {
 			/* Check if send work queue is empty. */
-			smp_read_barrier_depends(); /* see post_one_send() */
 			if (qp->s_tail == READ_ONCE(qp->s_head)) {
 				clear_ahg(qp);
 				goto bail;
@@ -387,7 +383,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 						       : IB_WC_SUCCESS);
 				if (local_ops)
 					atomic_dec(&qp->local_ops_pending);
-				qp->s_hdrwords = 0;
 				goto done_free_tx;
 			}
 
@@ -690,7 +685,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		bth2 |= IB_BTH_REQ_ACK;
 	}
 	qp->s_len -= len;
-	qp->s_hdrwords = hwords;
+	ps->s_txreq->hdr_dwords = hwords;
 	ps->s_txreq->sde = priv->s_sde;
 	ps->s_txreq->ss = ss;
 	ps->s_txreq->s_cur_size = len;
@@ -701,8 +696,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		bth2,
 		middle,
 		ps);
-	/* pbc */
-	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
 	return 1;
 
 done_free_tx:
@@ -716,7 +709,6 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 bail_no_tx:
 	ps->s_txreq = NULL;
 	qp->s_flags &= ~RVT_S_BUSY;
-	qp->s_hdrwords = 0;
 	return 0;
 }
 
@@ -736,14 +728,16 @@ static inline void hfi1_make_bth_aeth(struct rvt_qp *qp,
 	ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn));
 }
 
-static inline void hfi1_queue_rc_ack(struct rvt_qp *qp, bool is_fecn)
+static inline void hfi1_queue_rc_ack(struct hfi1_packet *packet, bool is_fecn)
 {
-	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	struct rvt_qp *qp = packet->qp;
+	struct hfi1_ibport *ibp;
 	unsigned long flags;
 
 	spin_lock_irqsave(&qp->s_lock, flags);
 	if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
 		goto unlock;
+	ibp = rcd_to_iport(packet->rcd);
 	this_cpu_inc(*ibp->rvp.rc_qacks);
 	qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
 	qp->s_nak_state = qp->r_nak_state;
@@ -757,13 +751,14 @@ static inline void hfi1_queue_rc_ack(struct rvt_qp *qp, bool is_fecn)
 	spin_unlock_irqrestore(&qp->s_lock, flags);
 }
 
-static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp,
+static inline void hfi1_make_rc_ack_9B(struct hfi1_packet *packet,
 				       struct hfi1_opa_header *opa_hdr,
 				       u8 sc5, bool is_fecn,
 				       u64 *pbc_flags, u32 *hwords,
 				       u32 *nwords)
 {
-	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	struct rvt_qp *qp = packet->qp;
+	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 	struct ib_header *hdr = &opa_hdr->ibh;
 	struct ib_other_headers *ohdr;
@@ -804,19 +799,20 @@ static inline void hfi1_make_rc_ack_9B(struct rvt_qp *qp,
 	hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
 }
 
-static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp,
+static inline void hfi1_make_rc_ack_16B(struct hfi1_packet *packet,
 					struct hfi1_opa_header *opa_hdr,
 					u8 sc5, bool is_fecn,
 					u64 *pbc_flags, u32 *hwords,
 					u32 *nwords)
 {
-	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	struct rvt_qp *qp = packet->qp;
+	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
 	struct hfi1_16b_header *hdr = &opa_hdr->opah;
 	struct ib_other_headers *ohdr;
 	u32 bth0, bth1 = 0;
 	u16 len, pkey;
-	u8 becn = !!is_fecn;
+	bool becn = is_fecn;
 	u8 l4 = OPA_16B_L4_IB_LOCAL;
 	u8 extra_bytes;
 
@@ -843,11 +839,11 @@ static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp,
 	/* Convert dwords to flits */
 	len = (*hwords + *nwords) >> 1;
 
-	hfi1_make_16b_hdr(hdr,
-			  ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr),
+	hfi1_make_16b_hdr(hdr, ppd->lid |
+			  (rdma_ah_get_path_bits(&qp->remote_ah_attr) &
+			  ((1 << ppd->lmc) - 1)),
 			  opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr),
-				      16B),
-			  len, pkey, becn, 0, l4, sc5);
+				      16B), len, pkey, becn, 0, l4, sc5);
 
 	bth0 = pkey | (OP(ACKNOWLEDGE) << 24);
 	bth0 |= extra_bytes << 20;
@@ -856,7 +852,7 @@ static inline void hfi1_make_rc_ack_16B(struct rvt_qp *qp,
 	hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
 }
 
-typedef void (*hfi1_make_rc_ack)(struct rvt_qp *qp,
+typedef void (*hfi1_make_rc_ack)(struct hfi1_packet *packet,
 				 struct hfi1_opa_header *opa_hdr,
 				 u8 sc5, bool is_fecn,
 				 u64 *pbc_flags, u32 *hwords,
@@ -876,9 +872,10 @@ static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = {
  * Note that RDMA reads and atomics are handled in the
  * send side QP state and send engine.
  */
-void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd,
-		      struct rvt_qp *qp, bool is_fecn)
+void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn)
 {
+	struct hfi1_ctxtdata *rcd = packet->rcd;
+	struct rvt_qp *qp = packet->qp;
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
@@ -895,14 +892,13 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd,
 
 	/* Don't send ACK or NAK if a RDMA read or atomic is pending. */
 	if (qp->s_flags & RVT_S_RESP_PENDING) {
-		hfi1_queue_rc_ack(qp, is_fecn);
+		hfi1_queue_rc_ack(packet, is_fecn);
 		return;
 	}
 
 	/* Ensure s_rdma_ack_cnt changes are committed */
-	smp_read_barrier_depends();
 	if (qp->s_rdma_ack_cnt) {
-		hfi1_queue_rc_ack(qp, is_fecn);
+		hfi1_queue_rc_ack(packet, is_fecn);
 		return;
 	}
 
@@ -911,7 +907,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd,
 		return;
 
 	/* Make the appropriate header */
-	hfi1_make_rc_ack_tbl[priv->hdr_type](qp, &opa_hdr, sc5, is_fecn,
+	hfi1_make_rc_ack_tbl[priv->hdr_type](packet, &opa_hdr, sc5, is_fecn,
 					     &pbc_flags, &hwords, &nwords);
 
 	plen = 2 /* PBC */ + hwords + nwords;
@@ -925,7 +921,7 @@ void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd,
 		 * so that when enough buffer space becomes available,
 		 * the ACK is sent ahead of other outgoing packets.
 		 */
-		hfi1_queue_rc_ack(qp, is_fecn);
+		hfi1_queue_rc_ack(packet, is_fecn);
 		return;
 	}
 	trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
@@ -1543,7 +1539,7 @@ static void rc_rcv_resp(struct hfi1_packet *packet)
 	void *data = packet->payload;
 	u32 tlen = packet->tlen;
 	struct rvt_qp *qp = packet->qp;
-	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
+	struct hfi1_ibport *ibp;
 	struct ib_other_headers *ohdr = packet->ohdr;
 	struct rvt_swqe *wqe;
 	enum ib_wc_status status;
@@ -1562,7 +1558,6 @@ static void rc_rcv_resp(struct hfi1_packet *packet)
 	trace_hfi1_ack(qp, psn);
 
 	/* Ignore invalid responses. */
-	smp_read_barrier_depends(); /* see post_one_send */
 	if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0)
 		goto ack_done;
 
@@ -1701,6 +1696,7 @@ static void rc_rcv_resp(struct hfi1_packet *packet)
 	goto ack_err;
 
 ack_seq_err:
+	ibp = rcd_to_iport(rcd);
 	rdma_seq_err(qp, ibp, psn, rcd);
 	goto ack_done;
 
@@ -2041,7 +2037,6 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	struct rvt_qp *qp = packet->qp;
 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
 	struct ib_other_headers *ohdr = packet->ohdr;
-	u32 bth0 = be32_to_cpu(ohdr->bth[0]);
 	u32 opcode = packet->opcode;
 	u32 hdrsize = packet->hlen;
 	u32 psn = ib_bth_get_psn(packet->ohdr);
@@ -2239,7 +2234,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 		wc.port_num = 0;
 		/* Signal completion event if the solicited bit is set. */
 		rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
-			     (bth0 & IB_BTH_SOLICITED) != 0);
+			     ib_bth_is_solicited(ohdr));
 		break;
 
 	case OP(RDMA_WRITE_ONLY):
@@ -2483,7 +2478,7 @@ void hfi1_rc_rcv(struct hfi1_packet *packet)
 	qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
 	qp->r_ack_psn = qp->r_psn;
 send_ack:
-	hfi1_send_rc_ack(rcd, qp, is_fecn);
+	hfi1_send_rc_ack(packet, is_fecn);
 }
 
 void hfi1_rc_hdrerr(
diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index 2c7fc6e331ea..3daa94bdae3a 100644
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -225,19 +225,8 @@ int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet)
 	u32 dlid = packet->dlid;
 	u32 slid = packet->slid;
 	u32 sl = packet->sl;
-	int migrated;
-	u32 bth0, bth1;
-	u16 pkey;
-
-	bth0 = be32_to_cpu(packet->ohdr->bth[0]);
-	bth1 = be32_to_cpu(packet->ohdr->bth[1]);
-	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
-		pkey = hfi1_16B_get_pkey(packet->hdr);
-		migrated = bth1 & OPA_BTH_MIG_REQ;
-	} else {
-		pkey = ib_bth_get_pkey(packet->ohdr);
-		migrated = bth0 & IB_BTH_MIG_REQ;
-	}
+	bool migrated = packet->migrated;
+	u16 pkey = packet->pkey;
 
 	if (qp->s_mig_state == IB_MIG_ARMED && migrated) {
 		if (!packet->grh) {
@@ -362,7 +351,6 @@ static void ruc_loopback(struct rvt_qp *sqp)
 	sqp->s_flags |= RVT_S_BUSY;
 
 again:
-	smp_read_barrier_depends(); /* see post_one_send() */
 	if (sqp->s_last == READ_ONCE(sqp->s_head))
 		goto clr_busy;
 	wqe = rvt_get_swqe_ptr(sqp, sqp->s_last);
@@ -757,19 +745,18 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp,
 	u32 slid;
 	u16 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
 	u8 l4 = OPA_16B_L4_IB_LOCAL;
-	u8 extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2),
-				   ps->s_txreq->s_cur_size);
+	u8 extra_bytes = hfi1_get_16b_padding(
+				(ps->s_txreq->hdr_dwords << 2),
+				ps->s_txreq->s_cur_size);
 	u32 nwords = SIZE_OF_CRC + ((ps->s_txreq->s_cur_size +
 				 extra_bytes + SIZE_OF_LT) >> 2);
-	u8 becn = 0;
+	bool becn = false;
 
 	if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
 	    hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) {
 		struct ib_grh *grh;
 		struct ib_global_route *grd =
 			rdma_ah_retrieve_grh(&qp->remote_ah_attr);
-		int hdrwords;
-
 		/*
 		 * Ensure OPA GIDs are transformed to IB gids
 		 * before creating the GRH.
@@ -778,9 +765,10 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp,
 			grd->sgid_index = 0;
 		grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh;
 		l4 = OPA_16B_L4_IB_GLOBAL;
-		hdrwords = qp->s_hdrwords - 4;
-		qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd,
-						hdrwords, nwords);
+		ps->s_txreq->hdr_dwords +=
+			hfi1_make_grh(ibp, grh, grd,
+				      ps->s_txreq->hdr_dwords - LRH_16B_DWORDS,
+				      nwords);
 		middle = 0;
 	}
 
@@ -799,7 +787,7 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp,
 	if (qp->s_flags & RVT_S_ECN) {
 		qp->s_flags &= ~RVT_S_ECN;
 		/* we recently received a FECN, so return a BECN */
-		becn = 1;
+		becn = true;
 	}
 	hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2);
 
@@ -814,7 +802,7 @@ static inline void hfi1_make_ruc_header_16B(struct rvt_qp *qp,
 			  slid,
 			  opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr),
 				      16B),
-			  (qp->s_hdrwords + nwords) >> 1,
+			  (ps->s_txreq->hdr_dwords + nwords) >> 1,
 			  pkey, becn, 0, l4, priv->s_sc);
 }
 
@@ -834,13 +822,13 @@ static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp,
 
 	if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) {
 		struct ib_grh *grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh;
-		int hdrwords = qp->s_hdrwords - 2;
 
 		lrh0 = HFI1_LRH_GRH;
-		qp->s_hdrwords +=
+		ps->s_txreq->hdr_dwords +=
 			hfi1_make_grh(ibp, grh,
 				      rdma_ah_read_grh(&qp->remote_ah_attr),
-				      hdrwords, nwords);
+				      ps->s_txreq->hdr_dwords - LRH_9B_DWORDS,
+				      nwords);
 		middle = 0;
 	}
 	lrh0 |= (priv->s_sc & 0xf) << 12 |
@@ -866,7 +854,7 @@ static inline void hfi1_make_ruc_header_9B(struct rvt_qp *qp,
 	hfi1_make_ruc_bth(qp, ohdr, bth0, bth1, bth2);
 	hfi1_make_ib_hdr(&ps->s_txreq->phdr.hdr.ibh,
 			 lrh0,
-			 qp->s_hdrwords + nwords,
+			 ps->s_txreq->hdr_dwords + nwords,
 			 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B),
 			 ppd_from_ibp(ibp)->lid |
 				rdma_ah_get_path_bits(&qp->remote_ah_attr));
@@ -1031,7 +1019,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread)
 	ps.s_txreq = get_waiting_verbs_txreq(qp);
 	do {
 		/* Check for a constructed packet to be sent. */
-		if (qp->s_hdrwords != 0) {
+		if (ps.s_txreq) {
 			spin_unlock_irqrestore(&qp->s_lock, ps.flags);
 			/*
 			 * If the packet cannot be sent now, return and
@@ -1039,8 +1027,6 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread)
 			 */
 			if (hfi1_verbs_send(qp, &ps))
 				return;
-			/* Record that s_ahg is empty. */
-			qp->s_hdrwords = 0;
 			/* allow other tasks to run */
 			if (schedule_send_yield(qp, &ps))
 				return;
diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c
index 31c8f89b5fc8..1f203309cf24 100644
--- a/drivers/infiniband/hw/hfi1/sdma.c
+++ b/drivers/infiniband/hw/hfi1/sdma.c
@@ -553,7 +553,6 @@ static void sdma_hw_clean_up_task(unsigned long opaque)
 
 static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde)
 {
-	smp_read_barrier_depends(); /* see sdma_update_tail() */
 	return sde->tx_ring[sde->tx_head & sde->sdma_mask];
 }
 
@@ -1276,13 +1275,15 @@ int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines)
 	return -ENOMEM;
 }
 
-/*
- * Clean up allocated memory.
- *
- * This routine is can be called regardless of the success of sdma_init()
+/**
+ * sdma_clean()  Clean up allocated memory
+ * @dd:          struct hfi1_devdata
+ * @num_engines: num sdma engines
  *
+ * This routine can be called regardless of the success of
+ * sdma_init()
  */
-static void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
+void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
 {
 	size_t i;
 	struct sdma_engine *sde;
@@ -1387,7 +1388,8 @@ int sdma_init(struct hfi1_devdata *dd, u8 port)
 		    num_engines, descq_cnt);
 
 	/* alloc memory for array of send engines */
-	dd->per_sdma = kcalloc(num_engines, sizeof(*dd->per_sdma), GFP_KERNEL);
+	dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma),
+				    GFP_KERNEL, dd->node);
 	if (!dd->per_sdma)
 		return ret;
 
@@ -1618,7 +1620,6 @@ void sdma_exit(struct hfi1_devdata *dd)
 		 */
 		sdma_finalput(&sde->state);
 	}
-	sdma_clean(dd, dd->num_sdma);
 }
 
 /*
diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index 374c59784950..46c775f255d1 100644
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -420,6 +420,7 @@ struct sdma_engine {
 int sdma_init(struct hfi1_devdata *dd, u8 port);
 void sdma_start(struct hfi1_devdata *dd);
 void sdma_exit(struct hfi1_devdata *dd);
+void sdma_clean(struct hfi1_devdata *dd, size_t num_engines);
 void sdma_all_running(struct hfi1_devdata *dd);
 void sdma_all_idle(struct hfi1_devdata *dd);
 void sdma_freeze_notify(struct hfi1_devdata *dd, int go_idle);
diff --git a/drivers/infiniband/hw/hfi1/trace.c b/drivers/infiniband/hw/hfi1/trace.c
index 959a80429ee9..89bd9851065b 100644
--- a/drivers/infiniband/hw/hfi1/trace.c
+++ b/drivers/infiniband/hw/hfi1/trace.c
@@ -138,7 +138,7 @@ static const char *parse_syndrome(u8 syndrome)
 }
 
 void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr,
-			     u8 *ack, u8 *becn, u8 *fecn, u8 *mig,
+			     u8 *ack, bool *becn, bool *fecn, u8 *mig,
 			     u8 *se, u8 *pad, u8 *opcode, u8 *tver,
 			     u16 *pkey, u32 *psn, u32 *qpn)
 {
@@ -184,7 +184,7 @@ void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5,
 }
 
 void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr,
-			      u8 *age, u8 *becn, u8 *fecn,
+			      u8 *age, bool *becn, bool *fecn,
 			      u8 *l4, u8 *rc, u8 *sc,
 			      u16 *entropy, u16 *len, u16 *pkey,
 			      u32 *dlid, u32 *slid)
@@ -207,7 +207,7 @@ void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr,
 #define LRH_16B_PRN "age:%d becn:%d fecn:%d l4:%d " \
 		    "rc:%d sc:%d pkey:0x%.4x entropy:0x%.4x"
 const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass,
-			       u8 age, u8 becn, u8 fecn, u8 l4,
+			       u8 age, bool becn, bool fecn, u8 l4,
 			       u8 lnh, const char *lnh_name, u8 lver,
 			       u8 rc, u8 sc, u8 sl, u16 entropy,
 			       u16 len, u16 pkey, u32 dlid, u32 slid)
@@ -235,7 +235,7 @@ const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass,
 	"op:0x%.2x,%s se:%d m:%d pad:%d tver:%d " \
 	"qpn:0x%.6x a:%d psn:0x%.8x"
 const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass,
-			       u8 ack, u8 becn, u8 fecn, u8 mig,
+			       u8 ack, bool becn, bool fecn, u8 mig,
 			       u8 se, u8 pad, u8 opcode, const char *opname,
 			       u8 tver, u16 pkey, u32 psn, u32 qpn)
 {
diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
index fb631278eccd..2847626d3819 100644
--- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
+++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h
@@ -101,7 +101,7 @@ u8 hfi1_trace_opa_hdr_len(struct hfi1_opa_header *opah);
 u8 hfi1_trace_packet_hdr_len(struct hfi1_packet *packet);
 const char *hfi1_trace_get_packet_l4_str(u8 l4);
 void hfi1_trace_parse_9b_bth(struct ib_other_headers *ohdr,
-			     u8 *ack, u8 *becn, u8 *fecn, u8 *mig,
+			     u8 *ack, bool *becn, bool *fecn, u8 *mig,
 			     u8 *se, u8 *pad, u8 *opcode, u8 *tver,
 			     u16 *pkey, u32 *psn, u32 *qpn);
 void hfi1_trace_parse_9b_hdr(struct ib_header *hdr, bool sc5,
@@ -112,19 +112,19 @@ void hfi1_trace_parse_16b_bth(struct ib_other_headers *ohdr,
 			      u8 *pad, u8 *se, u8 *tver,
 			      u32 *psn, u32 *qpn);
 void hfi1_trace_parse_16b_hdr(struct hfi1_16b_header *hdr,
-			      u8 *age, u8 *becn, u8 *fecn,
+			      u8 *age, bool *becn, bool *fecn,
 			      u8 *l4, u8 *rc, u8 *sc,
 			      u16 *entropy, u16 *len, u16 *pkey,
 			      u32 *dlid, u32 *slid);
 
 const char *hfi1_trace_fmt_lrh(struct trace_seq *p, bool bypass,
-			       u8 age, u8 becn, u8 fecn, u8 l4,
+			       u8 age, bool becn, bool fecn, u8 l4,
 			       u8 lnh, const char *lnh_name, u8 lver,
 			       u8 rc, u8 sc, u8 sl, u16 entropy,
 			       u16 len, u16 pkey, u32 dlid, u32 slid);
 
 const char *hfi1_trace_fmt_bth(struct trace_seq *p, bool bypass,
-			       u8 ack, u8 becn, u8 fecn, u8 mig,
+			       u8 ack, bool becn, bool fecn, u8 mig,
 			       u8 se, u8 pad, u8 opcode, const char *opname,
 			       u8 tver, u16 pkey, u32 psn, u32 qpn);
 
@@ -148,8 +148,8 @@ DECLARE_EVENT_CLASS(hfi1_input_ibhdr_template,
 			__field(u8, etype)
 			__field(u8, ack)
 			__field(u8, age)
-			__field(u8, becn)
-			__field(u8, fecn)
+			__field(bool, becn)
+			__field(bool, fecn)
 			__field(u8, l2)
 			__field(u8, l4)
 			__field(u8, lnh)
@@ -290,8 +290,8 @@ DECLARE_EVENT_CLASS(hfi1_output_ibhdr_template,
 			__field(u8, hdr_type)
 			__field(u8, ack)
 			__field(u8, age)
-			__field(u8, becn)
-			__field(u8, fecn)
+			__field(bool, becn)
+			__field(bool, fecn)
 			__field(u8, l4)
 			__field(u8, lnh)
 			__field(u8, lver)
diff --git a/drivers/infiniband/hw/hfi1/trace_rx.h b/drivers/infiniband/hw/hfi1/trace_rx.h
index 4d487fee105d..7eceb57e0415 100644
--- a/drivers/infiniband/hw/hfi1/trace_rx.h
+++ b/drivers/infiniband/hw/hfi1/trace_rx.h
@@ -63,17 +63,9 @@ __print_symbolic(type,                       \
 #define TRACE_SYSTEM hfi1_rx
 
 TRACE_EVENT(hfi1_rcvhdr,
-	    TP_PROTO(struct hfi1_devdata *dd,
-		     u32 ctxt,
-		     u64 eflags,
-		     u32 etype,
-		     u32 hlen,
-		     u32 tlen,
-		     u32 updegr,
-		     u32 etail
-		    ),
-	    TP_ARGS(dd, ctxt, eflags, etype, hlen, tlen, updegr, etail),
-	    TP_STRUCT__entry(DD_DEV_ENTRY(dd)
+	    TP_PROTO(struct hfi1_packet *packet),
+	    TP_ARGS(packet),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(packet->rcd->dd)
 			     __field(u64, eflags)
 			     __field(u32, ctxt)
 			     __field(u32, etype)
@@ -82,14 +74,14 @@ TRACE_EVENT(hfi1_rcvhdr,
 			     __field(u32, updegr)
 			     __field(u32, etail)
 			     ),
-	     TP_fast_assign(DD_DEV_ASSIGN(dd);
-			    __entry->eflags = eflags;
-			    __entry->ctxt = ctxt;
-			    __entry->etype = etype;
-			    __entry->hlen = hlen;
-			    __entry->tlen = tlen;
-			    __entry->updegr = updegr;
-			    __entry->etail = etail;
+	     TP_fast_assign(DD_DEV_ASSIGN(packet->rcd->dd);
+			    __entry->eflags = rhf_err_flags(packet->rhf);
+			    __entry->ctxt = packet->rcd->ctxt;
+			    __entry->etype = packet->etype;
+			    __entry->hlen = packet->hlen;
+			    __entry->tlen = packet->tlen;
+			    __entry->updegr = packet->updegr;
+			    __entry->etail = rhf_egr_index(packet->rhf);
 			    ),
 	     TP_printk(
 		"[%s] ctxt %d eflags 0x%llx etype %d,%s hlen %d tlen %d updegr %d etail %d",
diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c
index 991bbee04821..9d7a3110c14c 100644
--- a/drivers/infiniband/hw/hfi1/uc.c
+++ b/drivers/infiniband/hw/hfi1/uc.c
@@ -79,7 +79,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
 			goto bail;
 		/* We are in the error state, flush the work request. */
-		smp_read_barrier_depends(); /* see post_one_send() */
 		if (qp->s_last == READ_ONCE(qp->s_head))
 			goto bail;
 		/* If DMAs are in progress, we can't flush immediately. */
@@ -119,7 +118,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		    RVT_PROCESS_NEXT_SEND_OK))
 			goto bail;
 		/* Check if send work queue is empty. */
-		smp_read_barrier_depends(); /* see post_one_send() */
 		if (qp->s_cur == READ_ONCE(qp->s_head)) {
 			clear_ahg(qp);
 			goto bail;
@@ -146,7 +144,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 							: IB_WC_SUCCESS);
 			if (local_ops)
 				atomic_dec(&qp->local_ops_pending);
-			qp->s_hdrwords = 0;
 			goto done_free_tx;
 		}
 		/*
@@ -269,14 +266,12 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		break;
 	}
 	qp->s_len -= len;
-	qp->s_hdrwords = hwords;
+	ps->s_txreq->hdr_dwords = hwords;
 	ps->s_txreq->sde = priv->s_sde;
 	ps->s_txreq->ss = &qp->s_sge;
 	ps->s_txreq->s_cur_size = len;
 	hfi1_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24),
 			     mask_psn(qp->s_psn++), middle, ps);
-	/* pbc */
-	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
 	return 1;
 
 done_free_tx:
@@ -290,7 +285,6 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 bail_no_tx:
 	ps->s_txreq = NULL;
 	qp->s_flags &= ~RVT_S_BUSY;
-	qp->s_hdrwords = 0;
 	return 0;
 }
 
@@ -482,8 +476,7 @@ void hfi1_uc_rcv(struct hfi1_packet *packet)
 		wc.port_num = 0;
 		/* Signal completion event if the solicited bit is set. */
 		rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
-			     (ohdr->bth[0] &
-			      cpu_to_be32(IB_BTH_SOLICITED)) != 0);
+			     ib_bth_is_solicited(ohdr));
 		break;
 
 	case OP(RDMA_WRITE_FIRST):
diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c
index beb5091eccca..bcf3b0bebac8 100644
--- a/drivers/infiniband/hw/hfi1/ud.c
+++ b/drivers/infiniband/hw/hfi1/ud.c
@@ -340,15 +340,16 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	extra_bytes = -wqe->length & 3;
 	nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC;
 	/* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
-	qp->s_hdrwords = 7;
+	ps->s_txreq->hdr_dwords = 7;
 	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
-		qp->s_hdrwords++;
+		ps->s_txreq->hdr_dwords++;
 
 	if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
 		grh = &ps->s_txreq->phdr.hdr.ibh.u.l.grh;
-		qp->s_hdrwords += hfi1_make_grh(ibp, grh,
-						rdma_ah_read_grh(ah_attr),
-						qp->s_hdrwords - 2, nwords);
+		ps->s_txreq->hdr_dwords +=
+			hfi1_make_grh(ibp, grh, rdma_ah_read_grh(ah_attr),
+				      ps->s_txreq->hdr_dwords - LRH_9B_DWORDS,
+				      nwords);
 		lrh0 = HFI1_LRH_GRH;
 		ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
 	} else {
@@ -381,7 +382,7 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 		}
 	}
 	hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, false);
-	len = qp->s_hdrwords + nwords;
+	len = ps->s_txreq->hdr_dwords + nwords;
 
 	/* Setup the packet */
 	ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_9B;
@@ -405,12 +406,12 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	ppd = ppd_from_ibp(ibp);
 	ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr;
 	/* header size in dwords 16B LRH+BTH+DETH = (16+12+8)/4. */
-	qp->s_hdrwords = 9;
+	ps->s_txreq->hdr_dwords = 9;
 	if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
-		qp->s_hdrwords++;
+		ps->s_txreq->hdr_dwords++;
 
 	/* SW provides space for CRC and LT for bypass packets. */
-	extra_bytes = hfi1_get_16b_padding((qp->s_hdrwords << 2),
+	extra_bytes = hfi1_get_16b_padding((ps->s_txreq->hdr_dwords << 2),
 					   wqe->length);
 	nwords = ((wqe->length + extra_bytes + SIZE_OF_LT) >> 2) + SIZE_OF_CRC;
 
@@ -428,8 +429,10 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 			grd->sgid_index = 0;
 		}
 		grh = &ps->s_txreq->phdr.hdr.opah.u.l.grh;
-		qp->s_hdrwords += hfi1_make_grh(ibp, grh, grd,
-					qp->s_hdrwords - 4, nwords);
+		ps->s_txreq->hdr_dwords += hfi1_make_grh(
+			ibp, grh, grd,
+			ps->s_txreq->hdr_dwords - LRH_16B_DWORDS,
+			nwords);
 		ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth;
 		l4 = OPA_16B_L4_IB_GLOBAL;
 	} else {
@@ -452,7 +455,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 
 	hfi1_make_bth_deth(qp, wqe, ohdr, &pkey, extra_bytes, true);
 	/* Convert dwords to flits */
-	len = (qp->s_hdrwords + nwords) >> 1;
+	len = (ps->s_txreq->hdr_dwords + nwords) >> 1;
 
 	/* Setup the packet */
 	ps->s_txreq->phdr.hdr.hdr_type = HFI1_PKT_TYPE_16B;
@@ -486,7 +489,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 		if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
 			goto bail;
 		/* We are in the error state, flush the work request. */
-		smp_read_barrier_depends(); /* see post_one_send */
 		if (qp->s_last == READ_ONCE(qp->s_head))
 			goto bail;
 		/* If DMAs are in progress, we can't flush immediately. */
@@ -500,7 +502,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	}
 
 	/* see post_one_send() */
-	smp_read_barrier_depends();
 	if (qp->s_cur == READ_ONCE(qp->s_head))
 		goto bail;
 
@@ -564,8 +565,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 	priv->s_ahg->ahgcount = 0;
 	priv->s_ahg->ahgidx = 0;
 	priv->s_ahg->tx_flags = 0;
-	/* pbc */
-	ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
 
 	return 1;
 
@@ -580,7 +579,6 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
 bail_no_tx:
 	ps->s_txreq = NULL;
 	qp->s_flags &= ~RVT_S_BUSY;
-	qp->s_hdrwords = 0;
 	return 0;
 }
 
@@ -651,7 +649,8 @@ void return_cnp_16B(struct hfi1_ibport *ibp, struct rvt_qp *qp,
 		struct ib_grh *grh = &hdr.u.l.grh;
 
 		grh->version_tclass_flow = old_grh->version_tclass_flow;
-		grh->paylen = cpu_to_be16((hwords - 4 + nwords) << 2);
+		grh->paylen = cpu_to_be16(
+			(hwords - LRH_16B_DWORDS + nwords) << 2);
 		grh->hop_limit = 0xff;
 		grh->sgid = old_grh->dgid;
 		grh->dgid = old_grh->sgid;
@@ -705,7 +704,8 @@ void return_cnp(struct hfi1_ibport *ibp, struct rvt_qp *qp, u32 remote_qpn,
 		struct ib_grh *grh = &hdr.u.l.grh;
 
 		grh->version_tclass_flow = old_grh->version_tclass_flow;
-		grh->paylen = cpu_to_be16((hwords - 2 + SIZE_OF_CRC) << 2);
+		grh->paylen = cpu_to_be16(
+			(hwords - LRH_9B_DWORDS + SIZE_OF_CRC) << 2);
 		grh->hop_limit = 0xff;
 		grh->sgid = old_grh->dgid;
 		grh->dgid = old_grh->sgid;
@@ -1048,8 +1048,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet)
 	wc.port_num = qp->port_num;
 	/* Signal completion event if the solicited bit is set. */
 	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
-		     (ohdr->bth[0] &
-		      cpu_to_be32(IB_BTH_SOLICITED)) != 0);
+		     ib_bth_is_solicited(ohdr));
 	return;
 
 drop:
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index a38785e224cc..471d55c50066 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -835,7 +835,7 @@ static int build_verbs_tx_desc(
 {
 	int ret = 0;
 	struct hfi1_sdma_header *phdr = &tx->phdr;
-	u16 hdrbytes = tx->hdr_dwords << 2;
+	u16 hdrbytes = (tx->hdr_dwords + sizeof(pbc) / 4) << 2;
 	u8 extra_bytes = 0;
 
 	if (tx->phdr.hdr.hdr_type) {
@@ -901,7 +901,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 {
 	struct hfi1_qp_priv *priv = qp->priv;
 	struct hfi1_ahg_info *ahg_info = priv->s_ahg;
-	u32 hdrwords = qp->s_hdrwords;
+	u32 hdrwords = ps->s_txreq->hdr_dwords;
 	u32 len = ps->s_txreq->s_cur_size;
 	u32 plen;
 	struct hfi1_ibdev *dev = ps->dev;
@@ -919,7 +919,7 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 	} else {
 		dwords = (len + 3) >> 2;
 	}
-	plen = hdrwords + dwords + 2;
+	plen = hdrwords + dwords + sizeof(pbc) / 4;
 
 	tx = ps->s_txreq;
 	if (!sdma_txreq_built(&tx->txreq)) {
@@ -1038,7 +1038,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 			u64 pbc)
 {
 	struct hfi1_qp_priv *priv = qp->priv;
-	u32 hdrwords = qp->s_hdrwords;
+	u32 hdrwords = ps->s_txreq->hdr_dwords;
 	struct rvt_sge_state *ss = ps->s_txreq->ss;
 	u32 len = ps->s_txreq->s_cur_size;
 	u32 dwords;
@@ -1064,7 +1064,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,
 		dwords = (len + 3) >> 2;
 		hdr = (u32 *)&ps->s_txreq->phdr.hdr.ibh;
 	}
-	plen = hdrwords + dwords + 2;
+	plen = hdrwords + dwords + sizeof(pbc) / 4;
 
 	/* only RC/UC use complete */
 	switch (qp->ibqp.qp_type) {
@@ -1486,7 +1486,7 @@ static int query_port(struct rvt_dev_info *rdi, u8 port_num,
 	props->max_mtu = mtu_to_enum((!valid_ib_mtu(hfi1_max_mtu) ?
 				      4096 : hfi1_max_mtu), IB_MTU_4096);
 	props->active_mtu = !valid_ib_mtu(ppd->ibmtu) ? props->max_mtu :
-		mtu_to_enum(ppd->ibmtu, IB_MTU_2048);
+		mtu_to_enum(ppd->ibmtu, IB_MTU_4096);
 
 	/*
 	 * sm_lid of 0xFFFF needs special handling so that it can
@@ -1844,7 +1844,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	struct hfi1_ibport *ibp = &ppd->ibport_data;
 	unsigned i;
 	int ret;
-	size_t lcpysz = IB_DEVICE_NAME_MAX;
 
 	for (i = 0; i < dd->num_pports; i++)
 		init_ibport(ppd + i);
@@ -1872,8 +1871,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	 */
 	if (!ib_hfi1_sys_image_guid)
 		ib_hfi1_sys_image_guid = ibdev->node_guid;
-	lcpysz = strlcpy(ibdev->name, class_name(), lcpysz);
-	strlcpy(ibdev->name + lcpysz, "_%d", IB_DEVICE_NAME_MAX - lcpysz);
 	ibdev->owner = THIS_MODULE;
 	ibdev->phys_port_cnt = dd->num_pports;
 	ibdev->dev.parent = &dd->pcidev->dev;
@@ -1893,7 +1890,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
 	 * Fill in rvt info object.
 	 */
 	dd->verbs_dev.rdi.driver_f.port_callback = hfi1_create_port_files;
-	dd->verbs_dev.rdi.driver_f.get_card_name = get_card_name;
 	dd->verbs_dev.rdi.driver_f.get_pci_dev = get_pci_dev;
 	dd->verbs_dev.rdi.driver_f.check_ah = hfi1_check_ah;
 	dd->verbs_dev.rdi.driver_f.notify_new_ah = hfi1_notify_new_ah;
diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index 87d1285a3340..2d787b8346ca 100644
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -105,6 +105,11 @@ enum {
 	HFI1_HAS_GRH = (1 << 0),
 };
 
+#define LRH_16B_BYTES (FIELD_SIZEOF(struct hfi1_16b_header, lrh))
+#define LRH_16B_DWORDS (LRH_16B_BYTES / sizeof(u32))
+#define LRH_9B_BYTES (FIELD_SIZEOF(struct ib_header, lrh))
+#define LRH_9B_DWORDS (LRH_9B_BYTES / sizeof(u32))
+
 struct hfi1_16b_header {
 	u32 lrh[4];
 	union {
@@ -245,17 +250,6 @@ static inline struct rvt_qp *iowait_to_qp(struct  iowait *s_iowait)
 	return priv->owner;
 }
 
-/*
- * Send if not busy or waiting for I/O and either
- * a RC response is pending or we can process send work requests.
- */
-static inline int hfi1_send_ok(struct rvt_qp *qp)
-{
-	return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) &&
-		(qp->s_hdrwords || (qp->s_flags & RVT_S_RESP_PENDING) ||
-		 !(qp->s_flags & RVT_S_ANY_WAIT_SEND));
-}
-
 /*
  * This must be called with s_lock held.
  */
@@ -369,8 +363,7 @@ void hfi1_do_send(struct rvt_qp *qp, bool in_thread);
 void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
 			enum ib_wc_status status);
 
-void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
-		      bool is_fecn);
+void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn);
 
 int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps);
 
@@ -416,6 +409,11 @@ static inline void cacheless_memcpy(void *dst, void *src, size_t n)
 	__copy_user_nocache(dst, (void __user *)src, n, 0);
 }
 
+static inline bool opa_bth_is_migration(struct ib_other_headers *ohdr)
+{
+	return ohdr->bth[1] & cpu_to_be32(OPA_BTH_MIG_REQ);
+}
+
 extern const enum ib_wc_opcode ib_hfi1_wc_opcode[];
 
 extern const u8 hdr_len_by_opcode[];
diff --git a/drivers/infiniband/hw/hfi1/verbs_txreq.h b/drivers/infiniband/hw/hfi1/verbs_txreq.h
index cec7a4b34d16..729244c3086c 100644
--- a/drivers/infiniband/hw/hfi1/verbs_txreq.h
+++ b/drivers/infiniband/hw/hfi1/verbs_txreq.h
@@ -113,6 +113,13 @@ static inline struct verbs_txreq *get_waiting_verbs_txreq(struct rvt_qp *qp)
 	return NULL;
 }
 
+static inline bool verbs_txreq_queued(struct rvt_qp *qp)
+{
+	struct hfi1_qp_priv *priv = qp->priv;
+
+	return iowait_packet_queued(&priv->s_iowait);
+}
+
 void hfi1_put_txreq(struct verbs_txreq *tx);
 int verbs_txreq_init(struct hfi1_ibdev *dev);
 void verbs_txreq_exit(struct hfi1_ibdev *dev);
diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
index ff426a625e13..97bf2cd1cacb 100644
--- a/drivers/infiniband/hw/hns/Makefile
+++ b/drivers/infiniband/hw/hns/Makefile
@@ -5,7 +5,7 @@
 ccflags-y :=  -Idrivers/net/ethernet/hisilicon/hns3
 
 obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
-hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_eq.o hns_roce_pd.o \
+hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
 	hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
 	hns_roce_cq.o hns_roce_alloc.o
 obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c
index 1085cb249bc1..9ebe839d8b24 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cmd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c
@@ -103,6 +103,7 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status,
 	context->out_param = out_param;
 	complete(&context->done);
 }
+EXPORT_SYMBOL_GPL(hns_roce_cmd_event);
 
 /* this should be called with "use_events" */
 static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param,
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
index b1c94223c28b..9549ae51a0dd 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cmd.h
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
@@ -88,6 +88,16 @@ enum {
 	HNS_ROCE_CMD_DESTROY_SRQC_BT0	= 0x38,
 	HNS_ROCE_CMD_DESTROY_SRQC_BT1	= 0x39,
 	HNS_ROCE_CMD_DESTROY_SRQC_BT2	= 0x3a,
+
+	/* EQC commands */
+	HNS_ROCE_CMD_CREATE_AEQC	= 0x80,
+	HNS_ROCE_CMD_MODIFY_AEQC	= 0x81,
+	HNS_ROCE_CMD_QUERY_AEQC		= 0x82,
+	HNS_ROCE_CMD_DESTROY_AEQC	= 0x83,
+	HNS_ROCE_CMD_CREATE_CEQC	= 0x90,
+	HNS_ROCE_CMD_MODIFY_CEQC	= 0x91,
+	HNS_ROCE_CMD_QUERY_CEQC		= 0x92,
+	HNS_ROCE_CMD_DESTROY_CEQC	= 0x93,
 };
 
 enum {
diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h
index 7ecb7a4147a8..319cb74aebaf 100644
--- a/drivers/infiniband/hw/hns/hns_roce_common.h
+++ b/drivers/infiniband/hw/hns/hns_roce_common.h
@@ -43,15 +43,15 @@
 	__raw_writel((__force u32)cpu_to_le32(value), (addr))
 
 #define roce_get_field(origin, mask, shift) \
-	(((origin) & (mask)) >> (shift))
+	(((le32_to_cpu(origin)) & (mask)) >> (shift))
 
 #define roce_get_bit(origin, shift) \
 	roce_get_field((origin), (1ul << (shift)), (shift))
 
 #define roce_set_field(origin, mask, shift, val) \
 	do { \
-		(origin) &= (~(mask)); \
-		(origin) |= (((u32)(val) << (shift)) & (mask)); \
+		(origin) &= ~cpu_to_le32(mask); \
+		(origin) |= cpu_to_le32(((u32)(val) << (shift)) & (mask)); \
 	} while (0)
 
 #define roce_set_bit(origin, shift, val) \
@@ -376,6 +376,12 @@
 #define ROCEE_RX_CMQ_TAIL_REG			0x07024
 #define ROCEE_RX_CMQ_HEAD_REG			0x07028
 
+#define ROCEE_VF_MB_CFG0_REG			0x40
+#define ROCEE_VF_MB_STATUS_REG			0x58
+
+#define ROCEE_VF_EQ_DB_CFG0_REG			0x238
+#define ROCEE_VF_EQ_DB_CFG1_REG			0x23C
+
 #define ROCEE_VF_SMAC_CFG0_REG			0x12000
 #define ROCEE_VF_SMAC_CFG1_REG			0x12004
 
@@ -385,4 +391,9 @@
 #define ROCEE_VF_SGID_CFG3_REG			0x1000c
 #define ROCEE_VF_SGID_CFG4_REG			0x10010
 
+#define ROCEE_VF_ABN_INT_CFG_REG		0x13000
+#define ROCEE_VF_ABN_INT_ST_REG			0x13004
+#define ROCEE_VF_ABN_INT_EN_REG			0x13008
+#define ROCEE_VF_EVENT_INT_EN_REG		0x1300c
+
 #endif /* _HNS_ROCE_COMMON_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 2111b57a3489..bccc9b54c9ce 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -196,15 +196,14 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 	if (ret)
 		dev_err(dev, "HW2SW_CQ failed (%d) for CQN %06lx\n", ret,
 			hr_cq->cqn);
-	if (hr_dev->eq_table.eq) {
-		/* Waiting interrupt process procedure carried out */
-		synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq);
-
-		/* wait for all interrupt processed */
-		if (atomic_dec_and_test(&hr_cq->refcount))
-			complete(&hr_cq->free);
-		wait_for_completion(&hr_cq->free);
-	}
+
+	/* Waiting interrupt process procedure carried out */
+	synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq);
+
+	/* wait for all interrupt processed */
+	if (atomic_dec_and_test(&hr_cq->refcount))
+		complete(&hr_cq->free);
+	wait_for_completion(&hr_cq->free);
 
 	spin_lock_irq(&cq_table->lock);
 	radix_tree_delete(&cq_table->tree, hr_cq->cqn);
@@ -460,6 +459,7 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)
 	++cq->arm_sn;
 	cq->comp(cq);
 }
+EXPORT_SYMBOL_GPL(hns_roce_cq_completion);
 
 void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)
 {
@@ -482,6 +482,7 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)
 	if (atomic_dec_and_test(&cq->refcount))
 		complete(&cq->free);
 }
+EXPORT_SYMBOL_GPL(hns_roce_cq_event);
 
 int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev)
 {
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index b154ce40cded..165a09b314f6 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -62,12 +62,16 @@
 #define HNS_ROCE_CQE_WCMD_EMPTY_BIT		0x2
 #define HNS_ROCE_MIN_CQE_CNT			16
 
-#define HNS_ROCE_MAX_IRQ_NUM			34
+#define HNS_ROCE_MAX_IRQ_NUM			128
 
-#define HNS_ROCE_COMP_VEC_NUM			32
+#define EQ_ENABLE				1
+#define EQ_DISABLE				0
 
-#define HNS_ROCE_AEQE_VEC_NUM			1
-#define HNS_ROCE_AEQE_OF_VEC_NUM		1
+#define HNS_ROCE_CEQ				0
+#define HNS_ROCE_AEQ				1
+
+#define HNS_ROCE_CEQ_ENTRY_SIZE			0x4
+#define HNS_ROCE_AEQ_ENTRY_SIZE			0x10
 
 /* 4G/4K = 1M */
 #define HNS_ROCE_SL_SHIFT			28
@@ -130,6 +134,7 @@ enum hns_roce_event {
 	HNS_ROCE_EVENT_TYPE_DB_OVERFLOW               = 0x12,
 	HNS_ROCE_EVENT_TYPE_MB                        = 0x13,
 	HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW              = 0x14,
+	HNS_ROCE_EVENT_TYPE_FLR			      = 0x15,
 };
 
 /* Local Work Queue Catastrophic Error,SUBTYPE 0x5 */
@@ -173,6 +178,7 @@ enum {
 enum {
 	HNS_ROCE_CAP_FLAG_REREG_MR		= BIT(0),
 	HNS_ROCE_CAP_FLAG_ROCE_V1_V2		= BIT(1),
+	HNS_ROCE_CAP_FLAG_RQ_INLINE		= BIT(2)
 };
 
 enum hns_roce_mtt_type {
@@ -339,8 +345,8 @@ struct hns_roce_cq {
 	struct hns_roce_cq_buf		hr_buf;
 	spinlock_t			lock;
 	struct ib_umem			*umem;
-	void (*comp)(struct hns_roce_cq *);
-	void (*event)(struct hns_roce_cq *, enum hns_roce_event);
+	void (*comp)(struct hns_roce_cq *cq);
+	void (*event)(struct hns_roce_cq *cq, enum hns_roce_event event_type);
 
 	struct hns_roce_uar		*uar;
 	u32				cq_depth;
@@ -441,11 +447,26 @@ struct hns_roce_cmd_mailbox {
 
 struct hns_roce_dev;
 
+struct hns_roce_rinl_sge {
+	void			*addr;
+	u32			len;
+};
+
+struct hns_roce_rinl_wqe {
+	struct hns_roce_rinl_sge *sg_list;
+	u32			 sge_cnt;
+};
+
+struct hns_roce_rinl_buf {
+	struct hns_roce_rinl_wqe *wqe_list;
+	u32			 wqe_cnt;
+};
+
 struct hns_roce_qp {
 	struct ib_qp		ibqp;
 	struct hns_roce_buf	hr_buf;
 	struct hns_roce_wq	rq;
-	__le64			doorbell_qpn;
+	u32			doorbell_qpn;
 	__le32			sq_signal_bits;
 	u32			sq_next_wqe;
 	int			sq_max_wqes_per_wr;
@@ -462,9 +483,11 @@ struct hns_roce_qp {
 	u8			resp_depth;
 	u8			state;
 	u32			access_flags;
+	u32                     atomic_rd_en;
 	u32			pkey_index;
-	void			(*event)(struct hns_roce_qp *,
-					 enum hns_roce_event);
+	u32			qkey;
+	void			(*event)(struct hns_roce_qp *qp,
+					 enum hns_roce_event event_type);
 	unsigned long		qpn;
 
 	atomic_t		refcount;
@@ -472,6 +495,8 @@ struct hns_roce_qp {
 
 	struct hns_roce_sge	sge;
 	u32			next_sge;
+
+	struct hns_roce_rinl_buf rq_inl_buf;
 };
 
 struct hns_roce_sqp {
@@ -485,6 +510,45 @@ struct hns_roce_ib_iboe {
 	u8			phy_port[HNS_ROCE_MAX_PORTS];
 };
 
+enum {
+	HNS_ROCE_EQ_STAT_INVALID  = 0,
+	HNS_ROCE_EQ_STAT_VALID    = 2,
+};
+
+struct hns_roce_ceqe {
+	u32			comp;
+};
+
+struct hns_roce_aeqe {
+	u32 asyn;
+	union {
+		struct {
+			u32 qp;
+			u32 rsv0;
+			u32 rsv1;
+		} qp_event;
+
+		struct {
+			u32 cq;
+			u32 rsv0;
+			u32 rsv1;
+		} cq_event;
+
+		struct {
+			u32 ceqe;
+			u32 rsv0;
+			u32 rsv1;
+		} ce_event;
+
+		struct {
+			__le64  out_param;
+			__le16  token;
+			u8	status;
+			u8	rsv0;
+		} __packed cmd;
+	 } event;
+};
+
 struct hns_roce_eq {
 	struct hns_roce_dev		*hr_dev;
 	void __iomem			*doorbell;
@@ -498,11 +562,31 @@ struct hns_roce_eq {
 	int				log_page_size;
 	int				cons_index;
 	struct hns_roce_buf_list	*buf_list;
+	int				over_ignore;
+	int				coalesce;
+	int				arm_st;
+	u64				eqe_ba;
+	int				eqe_ba_pg_sz;
+	int				eqe_buf_pg_sz;
+	int				hop_num;
+	u64				*bt_l0;	/* Base address table for L0 */
+	u64				**bt_l1; /* Base address table for L1 */
+	u64				**buf;
+	dma_addr_t			l0_dma;
+	dma_addr_t			*l1_dma;
+	dma_addr_t			*buf_dma;
+	u32				l0_last_num; /* L0 last chunk num */
+	u32				l1_last_num; /* L1 last chunk num */
+	int				eq_max_cnt;
+	int				eq_period;
+	int				shift;
+	dma_addr_t			cur_eqe_ba;
+	dma_addr_t			nxt_eqe_ba;
 };
 
 struct hns_roce_eq_table {
 	struct hns_roce_eq	*eq;
-	void __iomem		**eqc_base;
+	void __iomem		**eqc_base; /* only for hw v1 */
 };
 
 struct hns_roce_caps {
@@ -528,7 +612,7 @@ struct hns_roce_caps {
 	u32		min_wqes;
 	int		reserved_cqs;
 	int		num_aeq_vectors;	/* 1 */
-	int		num_comp_vectors;	/* 32 ceq */
+	int		num_comp_vectors;
 	int		num_other_vectors;
 	int		num_mtpts;
 	u32		num_mtt_segs;
@@ -550,7 +634,7 @@ struct hns_roce_caps {
 	u32		pbl_buf_pg_sz;
 	u32		pbl_hop_num;
 	int		aeqe_depth;
-	int		ceqe_depth[HNS_ROCE_COMP_VEC_NUM];
+	int		ceqe_depth;
 	enum ib_mtu	max_mtu;
 	u32		qpc_bt_num;
 	u32		srqc_bt_num;
@@ -574,6 +658,9 @@ struct hns_roce_caps {
 	u32		cqe_ba_pg_sz;
 	u32		cqe_buf_pg_sz;
 	u32		cqe_hop_num;
+	u32		eqe_ba_pg_sz;
+	u32		eqe_buf_pg_sz;
+	u32		eqe_hop_num;
 	u32		chunk_sz;	/* chunk size in non multihop mode*/
 	u64		flags;
 };
@@ -623,6 +710,8 @@ struct hns_roce_hw {
 	int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr);
 	int (*destroy_cq)(struct ib_cq *ibcq);
 	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
+	int (*init_eq)(struct hns_roce_dev *hr_dev);
+	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 };
 
 struct hns_roce_dev {
diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.c b/drivers/infiniband/hw/hns/hns_roce_eq.c
deleted file mode 100644
index d184431e2bf5..000000000000
--- a/drivers/infiniband/hw/hns/hns_roce_eq.c
+++ /dev/null
@@ -1,759 +0,0 @@
-/*
- * Copyright (c) 2016 Hisilicon Limited.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include <linux/platform_device.h>
-#include <linux/interrupt.h>
-#include "hns_roce_common.h"
-#include "hns_roce_device.h"
-#include "hns_roce_eq.h"
-
-static void eq_set_cons_index(struct hns_roce_eq *eq, int req_not)
-{
-	roce_raw_write((eq->cons_index & CONS_INDEX_MASK) |
-		      (req_not << eq->log_entries), eq->doorbell);
-	/* Memory barrier */
-	mb();
-}
-
-static struct hns_roce_aeqe *get_aeqe(struct hns_roce_eq *eq, u32 entry)
-{
-	unsigned long off = (entry & (eq->entries - 1)) *
-			     HNS_ROCE_AEQ_ENTRY_SIZE;
-
-	return (struct hns_roce_aeqe *)((u8 *)
-		(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
-		off % HNS_ROCE_BA_SIZE);
-}
-
-static struct hns_roce_aeqe *next_aeqe_sw(struct hns_roce_eq *eq)
-{
-	struct hns_roce_aeqe *aeqe = get_aeqe(eq, eq->cons_index);
-
-	return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^
-		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
-}
-
-static void hns_roce_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
-					 struct hns_roce_aeqe *aeqe, int qpn)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-
-	dev_warn(dev, "Local Work Queue Catastrophic Error.\n");
-	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
-			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
-	case HNS_ROCE_LWQCE_QPC_ERROR:
-		dev_warn(dev, "QP %d, QPC error.\n", qpn);
-		break;
-	case HNS_ROCE_LWQCE_MTU_ERROR:
-		dev_warn(dev, "QP %d, MTU error.\n", qpn);
-		break;
-	case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
-		dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
-		break;
-	case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
-		dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
-		break;
-	case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
-		dev_warn(dev, "QP %d, WQE shift error\n", qpn);
-		break;
-	case HNS_ROCE_LWQCE_SL_ERROR:
-		dev_warn(dev, "QP %d, SL error.\n", qpn);
-		break;
-	case HNS_ROCE_LWQCE_PORT_ERROR:
-		dev_warn(dev, "QP %d, port error.\n", qpn);
-		break;
-	default:
-		break;
-	}
-}
-
-static void hns_roce_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
-						struct hns_roce_aeqe *aeqe,
-						int qpn)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-
-	dev_warn(dev, "Local Access Violation Work Queue Error.\n");
-	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
-			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
-	case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
-		dev_warn(dev, "QP %d, R_key violation.\n", qpn);
-		break;
-	case HNS_ROCE_LAVWQE_LENGTH_ERROR:
-		dev_warn(dev, "QP %d, length error.\n", qpn);
-		break;
-	case HNS_ROCE_LAVWQE_VA_ERROR:
-		dev_warn(dev, "QP %d, VA error.\n", qpn);
-		break;
-	case HNS_ROCE_LAVWQE_PD_ERROR:
-		dev_err(dev, "QP %d, PD error.\n", qpn);
-		break;
-	case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
-		dev_warn(dev, "QP %d, rw acc error.\n", qpn);
-		break;
-	case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
-		dev_warn(dev, "QP %d, key state error.\n", qpn);
-		break;
-	case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
-		dev_warn(dev, "QP %d, MR operation error.\n", qpn);
-		break;
-	default:
-		break;
-	}
-}
-
-static void hns_roce_qp_err_handle(struct hns_roce_dev *hr_dev,
-				   struct hns_roce_aeqe *aeqe,
-				   int event_type)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-	int phy_port;
-	int qpn;
-
-	qpn = roce_get_field(aeqe->event.qp_event.qp,
-			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
-			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
-	phy_port = roce_get_field(aeqe->event.qp_event.qp,
-			HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M,
-			HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S);
-	if (qpn <= 1)
-		qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port;
-
-	switch (event_type) {
-	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
-		dev_warn(dev, "Invalid Req Local Work Queue Error.\n"
-			      "QP %d, phy_port %d.\n", qpn, phy_port);
-		break;
-	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
-		hns_roce_wq_catas_err_handle(hr_dev, aeqe, qpn);
-		break;
-	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
-		hns_roce_local_wq_access_err_handle(hr_dev, aeqe, qpn);
-		break;
-	default:
-		break;
-	}
-
-	hns_roce_qp_event(hr_dev, qpn, event_type);
-}
-
-static void hns_roce_cq_err_handle(struct hns_roce_dev *hr_dev,
-				   struct hns_roce_aeqe *aeqe,
-				   int event_type)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-	u32 cqn;
-
-	cqn = le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq,
-		    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
-		    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S));
-
-	switch (event_type) {
-	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
-		dev_warn(dev, "CQ 0x%x access err.\n", cqn);
-		break;
-	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
-		dev_warn(dev, "CQ 0x%x overflow\n", cqn);
-		break;
-	case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
-		dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn);
-		break;
-	default:
-		break;
-	}
-
-	hns_roce_cq_event(hr_dev, cqn, event_type);
-}
-
-static void hns_roce_db_overflow_handle(struct hns_roce_dev *hr_dev,
-					struct hns_roce_aeqe *aeqe)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-
-	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
-			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
-	case HNS_ROCE_DB_SUBTYPE_SDB_OVF:
-		dev_warn(dev, "SDB overflow.\n");
-		break;
-	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF:
-		dev_warn(dev, "SDB almost overflow.\n");
-		break;
-	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP:
-		dev_warn(dev, "SDB almost empty.\n");
-		break;
-	case HNS_ROCE_DB_SUBTYPE_ODB_OVF:
-		dev_warn(dev, "ODB overflow.\n");
-		break;
-	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF:
-		dev_warn(dev, "ODB almost overflow.\n");
-		break;
-	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP:
-		dev_warn(dev, "SDB almost empty.\n");
-		break;
-	default:
-		break;
-	}
-}
-
-static int hns_roce_aeq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_aeqe *aeqe;
-	int aeqes_found = 0;
-	int event_type;
-
-	while ((aeqe = next_aeqe_sw(eq))) {
-		dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe,
-			roce_get_field(aeqe->asyn,
-				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
-				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
-		/* Memory barrier */
-		rmb();
-
-		event_type = roce_get_field(aeqe->asyn,
-				HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
-				HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S);
-		switch (event_type) {
-		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
-			dev_warn(dev, "PATH MIG not supported\n");
-			break;
-		case HNS_ROCE_EVENT_TYPE_COMM_EST:
-			dev_warn(dev, "COMMUNICATION established\n");
-			break;
-		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
-			dev_warn(dev, "SQ DRAINED not supported\n");
-			break;
-		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
-			dev_warn(dev, "PATH MIG failed\n");
-			break;
-		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
-		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
-		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
-			hns_roce_qp_err_handle(hr_dev, aeqe, event_type);
-			break;
-		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
-		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
-		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
-			dev_warn(dev, "SRQ not support!\n");
-			break;
-		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
-		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
-		case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
-			hns_roce_cq_err_handle(hr_dev, aeqe, event_type);
-			break;
-		case HNS_ROCE_EVENT_TYPE_PORT_CHANGE:
-			dev_warn(dev, "port change.\n");
-			break;
-		case HNS_ROCE_EVENT_TYPE_MB:
-			hns_roce_cmd_event(hr_dev,
-					   le16_to_cpu(aeqe->event.cmd.token),
-					   aeqe->event.cmd.status,
-					   le64_to_cpu(aeqe->event.cmd.out_param
-					   ));
-			break;
-		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
-			hns_roce_db_overflow_handle(hr_dev, aeqe);
-			break;
-		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
-			dev_warn(dev, "CEQ 0x%lx overflow.\n",
-			roce_get_field(aeqe->event.ce_event.ceqe,
-				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M,
-				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S));
-			break;
-		default:
-			dev_warn(dev, "Unhandled event %d on EQ %d at index %u\n",
-				 event_type, eq->eqn, eq->cons_index);
-			break;
-		}
-
-		eq->cons_index++;
-		aeqes_found = 1;
-
-		if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) {
-			dev_warn(dev, "cons_index overflow, set back to zero\n"
-				);
-			eq->cons_index = 0;
-		}
-	}
-
-	eq_set_cons_index(eq, 0);
-
-	return aeqes_found;
-}
-
-static struct hns_roce_ceqe *get_ceqe(struct hns_roce_eq *eq, u32 entry)
-{
-	unsigned long off = (entry & (eq->entries - 1)) *
-			     HNS_ROCE_CEQ_ENTRY_SIZE;
-
-	return (struct hns_roce_ceqe *)((u8 *)
-			(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
-			off % HNS_ROCE_BA_SIZE);
-}
-
-static struct hns_roce_ceqe *next_ceqe_sw(struct hns_roce_eq *eq)
-{
-	struct hns_roce_ceqe *ceqe = get_ceqe(eq, eq->cons_index);
-
-	return (!!(roce_get_bit(ceqe->ceqe.comp,
-		 HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^
-		 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
-}
-
-static int hns_roce_ceq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
-{
-	struct hns_roce_ceqe *ceqe;
-	int ceqes_found = 0;
-	u32 cqn;
-
-	while ((ceqe = next_ceqe_sw(eq))) {
-		/* Memory barrier */
-		rmb();
-		cqn = roce_get_field(ceqe->ceqe.comp,
-				     HNS_ROCE_CEQE_CEQE_COMP_CQN_M,
-				     HNS_ROCE_CEQE_CEQE_COMP_CQN_S);
-		hns_roce_cq_completion(hr_dev, cqn);
-
-		++eq->cons_index;
-		ceqes_found = 1;
-
-		if (eq->cons_index > 2 * hr_dev->caps.ceqe_depth[eq->eqn] - 1) {
-			dev_warn(&eq->hr_dev->pdev->dev,
-				"cons_index overflow, set back to zero\n");
-			eq->cons_index = 0;
-		}
-	}
-
-	eq_set_cons_index(eq, 0);
-
-	return ceqes_found;
-}
-
-static int hns_roce_aeq_ovf_int(struct hns_roce_dev *hr_dev,
-				struct hns_roce_eq *eq)
-{
-	struct device *dev = &eq->hr_dev->pdev->dev;
-	int eqovf_found = 0;
-	u32 caepaemask_val;
-	u32 cealmovf_val;
-	u32 caepaest_val;
-	u32 aeshift_val;
-	u32 ceshift_val;
-	u32 cemask_val;
-	int i = 0;
-
-	/**
-	 * AEQ overflow ECC mult bit err CEQ overflow alarm
-	 * must clear interrupt, mask irq, clear irq, cancel mask operation
-	 */
-	aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);
-
-	if (roce_get_bit(aeshift_val,
-		ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) {
-		dev_warn(dev, "AEQ overflow!\n");
-
-		/* Set mask */
-		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
-		roce_set_bit(caepaemask_val,
-			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
-			     HNS_ROCE_INT_MASK_ENABLE);
-		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
-
-		/* Clear int state(INT_WC : write 1 clear) */
-		caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG);
-		roce_set_bit(caepaest_val,
-			     ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1);
-		roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val);
-
-		/* Clear mask */
-		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
-		roce_set_bit(caepaemask_val,
-			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
-			     HNS_ROCE_INT_MASK_DISABLE);
-		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
-	}
-
-	/* CEQ almost overflow */
-	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
-		ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG +
-					i * CEQ_REG_OFFSET);
-
-		if (roce_get_bit(ceshift_val,
-		ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) {
-			dev_warn(dev, "CEQ[%d] almost overflow!\n", i);
-			eqovf_found++;
-
-			/* Set mask */
-			cemask_val = roce_read(hr_dev,
-					       ROCEE_CAEP_CE_IRQ_MASK_0_REG +
-					       i * CEQ_REG_OFFSET);
-			roce_set_bit(cemask_val,
-				ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
-				HNS_ROCE_INT_MASK_ENABLE);
-			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
-				   i * CEQ_REG_OFFSET, cemask_val);
-
-			/* Clear int state(INT_WC : write 1 clear) */
-			cealmovf_val = roce_read(hr_dev,
-				       ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
-				       i * CEQ_REG_OFFSET);
-			roce_set_bit(cealmovf_val,
-				     ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S,
-				     1);
-			roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
-				    i * CEQ_REG_OFFSET, cealmovf_val);
-
-			/* Clear mask */
-			cemask_val = roce_read(hr_dev,
-				     ROCEE_CAEP_CE_IRQ_MASK_0_REG +
-				     i * CEQ_REG_OFFSET);
-			roce_set_bit(cemask_val,
-			       ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
-			       HNS_ROCE_INT_MASK_DISABLE);
-			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
-				   i * CEQ_REG_OFFSET, cemask_val);
-		}
-	}
-
-	/* ECC multi-bit error alarm */
-	dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n",
-		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG),
-		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG),
-		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG));
-
-	dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n",
-		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG),
-		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG),
-		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG));
-
-	return eqovf_found;
-}
-
-static int hns_roce_eq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
-{
-	int eqes_found = 0;
-
-	if (likely(eq->type_flag == HNS_ROCE_CEQ))
-		/* CEQ irq routine, CEQ is pulse irq, not clear */
-		eqes_found = hns_roce_ceq_int(hr_dev, eq);
-	else if (likely(eq->type_flag == HNS_ROCE_AEQ))
-		/* AEQ irq routine, AEQ is pulse irq, not clear */
-		eqes_found = hns_roce_aeq_int(hr_dev, eq);
-	else
-		/* AEQ queue overflow irq */
-		eqes_found = hns_roce_aeq_ovf_int(hr_dev, eq);
-
-	return eqes_found;
-}
-
-static irqreturn_t hns_roce_msi_x_interrupt(int irq, void *eq_ptr)
-{
-	int int_work = 0;
-	struct hns_roce_eq  *eq  = eq_ptr;
-	struct hns_roce_dev *hr_dev = eq->hr_dev;
-
-	int_work = hns_roce_eq_int(hr_dev, eq);
-
-	return IRQ_RETVAL(int_work);
-}
-
-static void hns_roce_enable_eq(struct hns_roce_dev *hr_dev, int eq_num,
-			       int enable_flag)
-{
-	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num];
-	u32 val;
-
-	val = readl(eqc);
-
-	if (enable_flag)
-		roce_set_field(val,
-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
-			       HNS_ROCE_EQ_STAT_VALID);
-	else
-		roce_set_field(val,
-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
-			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
-			       HNS_ROCE_EQ_STAT_INVALID);
-	writel(val, eqc);
-}
-
-static int hns_roce_create_eq(struct hns_roce_dev *hr_dev,
-			      struct hns_roce_eq *eq)
-{
-	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn];
-	struct device *dev = &hr_dev->pdev->dev;
-	dma_addr_t tmp_dma_addr;
-	u32 eqconsindx_val = 0;
-	u32 eqcuridx_val = 0;
-	u32 eqshift_val = 0;
-	int num_bas = 0;
-	int ret;
-	int i;
-
-	num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) +
-		   HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
-
-	if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) {
-		dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n",
-			(eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE,
-			num_bas);
-		return -EINVAL;
-	}
-
-	eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL);
-	if (!eq->buf_list)
-		return -ENOMEM;
-
-	for (i = 0; i < num_bas; ++i) {
-		eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE,
-							 &tmp_dma_addr,
-							 GFP_KERNEL);
-		if (!eq->buf_list[i].buf) {
-			ret = -ENOMEM;
-			goto err_out_free_pages;
-		}
-
-		eq->buf_list[i].map = tmp_dma_addr;
-		memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE);
-	}
-	eq->cons_index = 0;
-	roce_set_field(eqshift_val,
-		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
-		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
-		       HNS_ROCE_EQ_STAT_INVALID);
-	roce_set_field(eqshift_val,
-		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M,
-		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S,
-		       eq->log_entries);
-	writel(eqshift_val, eqc);
-
-	/* Configure eq extended address 12~44bit */
-	writel((u32)(eq->buf_list[0].map >> 12), eqc + 4);
-
-	/*
-	 * Configure eq extended address 45~49 bit.
-	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
-	 * using 4K page, and shift more 32 because of
-	 * caculating the high 32 bit value evaluated to hardware.
-	 */
-	roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M,
-		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S,
-		       eq->buf_list[0].map >> 44);
-	roce_set_field(eqcuridx_val,
-		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M,
-		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0);
-	writel(eqcuridx_val, eqc + 8);
-
-	/* Configure eq consumer index */
-	roce_set_field(eqconsindx_val,
-		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M,
-		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0);
-	writel(eqconsindx_val, eqc + 0xc);
-
-	return 0;
-
-err_out_free_pages:
-	for (i = i - 1; i >= 0; i--)
-		dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf,
-				  eq->buf_list[i].map);
-
-	kfree(eq->buf_list);
-	return ret;
-}
-
-static void hns_roce_free_eq(struct hns_roce_dev *hr_dev,
-			     struct hns_roce_eq *eq)
-{
-	int i = 0;
-	int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) +
-		      HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
-
-	if (!eq->buf_list)
-		return;
-
-	for (i = 0; i < npages; ++i)
-		dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE,
-				  eq->buf_list[i].buf, eq->buf_list[i].map);
-
-	kfree(eq->buf_list);
-}
-
-static void hns_roce_int_mask_en(struct hns_roce_dev *hr_dev)
-{
-	int i = 0;
-	u32 aemask_val;
-	int masken = 0;
-
-	/* AEQ INT */
-	aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
-	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
-		     masken);
-	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken);
-	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val);
-
-	/* CEQ INT */
-	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
-		/* IRQ mask */
-		roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
-			   i * CEQ_REG_OFFSET, masken);
-	}
-}
-
-static void hns_roce_ce_int_default_cfg(struct hns_roce_dev *hr_dev)
-{
-	/* Configure ce int interval */
-	roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG,
-		   HNS_ROCE_CEQ_DEFAULT_INTERVAL);
-
-	/* Configure ce int burst num */
-	roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG,
-		   HNS_ROCE_CEQ_DEFAULT_BURST_NUM);
-}
-
-int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev)
-{
-	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
-	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_eq *eq = NULL;
-	int eq_num = 0;
-	int ret = 0;
-	int i = 0;
-	int j = 0;
-
-	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
-	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
-	if (!eq_table->eq)
-		return -ENOMEM;
-
-	eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base),
-				     GFP_KERNEL);
-	if (!eq_table->eqc_base) {
-		ret = -ENOMEM;
-		goto err_eqc_base_alloc_fail;
-	}
-
-	for (i = 0; i < eq_num; i++) {
-		eq = &eq_table->eq[i];
-		eq->hr_dev = hr_dev;
-		eq->eqn = i;
-		eq->irq = hr_dev->irq[i];
-		eq->log_page_size = PAGE_SHIFT;
-
-		if (i < hr_dev->caps.num_comp_vectors) {
-			/* CEQ */
-			eq_table->eqc_base[i] = hr_dev->reg_base +
-						ROCEE_CAEP_CEQC_SHIFT_0_REG +
-						HNS_ROCE_CEQC_REG_OFFSET * i;
-			eq->type_flag = HNS_ROCE_CEQ;
-			eq->doorbell = hr_dev->reg_base +
-				       ROCEE_CAEP_CEQC_CONS_IDX_0_REG +
-				       HNS_ROCE_CEQC_REG_OFFSET * i;
-			eq->entries = hr_dev->caps.ceqe_depth[i];
-			eq->log_entries = ilog2(eq->entries);
-			eq->eqe_size = sizeof(struct hns_roce_ceqe);
-		} else {
-			/* AEQ */
-			eq_table->eqc_base[i] = hr_dev->reg_base +
-						ROCEE_CAEP_AEQC_AEQE_SHIFT_REG;
-			eq->type_flag = HNS_ROCE_AEQ;
-			eq->doorbell = hr_dev->reg_base +
-				       ROCEE_CAEP_AEQE_CONS_IDX_REG;
-			eq->entries = hr_dev->caps.aeqe_depth;
-			eq->log_entries = ilog2(eq->entries);
-			eq->eqe_size = sizeof(struct hns_roce_aeqe);
-		}
-	}
-
-	/* Disable irq */
-	hns_roce_int_mask_en(hr_dev);
-
-	/* Configure CE irq interval and burst num */
-	hns_roce_ce_int_default_cfg(hr_dev);
-
-	for (i = 0; i < eq_num; i++) {
-		ret = hns_roce_create_eq(hr_dev, &eq_table->eq[i]);
-		if (ret) {
-			dev_err(dev, "eq create failed\n");
-			goto err_create_eq_fail;
-		}
-	}
-
-	for (j = 0; j < eq_num; j++) {
-		ret = request_irq(eq_table->eq[j].irq, hns_roce_msi_x_interrupt,
-				  0, hr_dev->irq_names[j], eq_table->eq + j);
-		if (ret) {
-			dev_err(dev, "request irq error!\n");
-			goto err_request_irq_fail;
-		}
-	}
-
-	for (i = 0; i < eq_num; i++)
-		hns_roce_enable_eq(hr_dev, i, EQ_ENABLE);
-
-	return 0;
-
-err_request_irq_fail:
-	for (j = j - 1; j >= 0; j--)
-		free_irq(eq_table->eq[j].irq, eq_table->eq + j);
-
-err_create_eq_fail:
-	for (i = i - 1; i >= 0; i--)
-		hns_roce_free_eq(hr_dev, &eq_table->eq[i]);
-
-	kfree(eq_table->eqc_base);
-
-err_eqc_base_alloc_fail:
-	kfree(eq_table->eq);
-
-	return ret;
-}
-
-void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev)
-{
-	int i;
-	int eq_num;
-	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
-
-	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
-	for (i = 0; i < eq_num; i++) {
-		/* Disable EQ */
-		hns_roce_enable_eq(hr_dev, i, EQ_DISABLE);
-
-		free_irq(eq_table->eq[i].irq, eq_table->eq + i);
-
-		hns_roce_free_eq(hr_dev, &eq_table->eq[i]);
-	}
-
-	kfree(eq_table->eqc_base);
-	kfree(eq_table->eq);
-}
diff --git a/drivers/infiniband/hw/hns/hns_roce_eq.h b/drivers/infiniband/hw/hns/hns_roce_eq.h
deleted file mode 100644
index c6d212d12e03..000000000000
--- a/drivers/infiniband/hw/hns/hns_roce_eq.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright (c) 2016 Hisilicon Limited.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _HNS_ROCE_EQ_H
-#define _HNS_ROCE_EQ_H
-
-#define HNS_ROCE_CEQ			1
-#define HNS_ROCE_AEQ			2
-
-#define HNS_ROCE_CEQ_ENTRY_SIZE		0x4
-#define HNS_ROCE_AEQ_ENTRY_SIZE		0x10
-#define HNS_ROCE_CEQC_REG_OFFSET	0x18
-
-#define HNS_ROCE_CEQ_DEFAULT_INTERVAL	0x10
-#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM	0x10
-
-#define HNS_ROCE_INT_MASK_DISABLE	0
-#define HNS_ROCE_INT_MASK_ENABLE	1
-
-#define EQ_ENABLE			1
-#define EQ_DISABLE			0
-#define CONS_INDEX_MASK			0xffff
-
-#define CEQ_REG_OFFSET			0x18
-
-enum {
-	HNS_ROCE_EQ_STAT_INVALID  = 0,
-	HNS_ROCE_EQ_STAT_VALID    = 2,
-};
-
-struct hns_roce_aeqe {
-	u32 asyn;
-	union {
-		struct {
-			u32 qp;
-			u32 rsv0;
-			u32 rsv1;
-		} qp_event;
-
-		struct {
-			u32 cq;
-			u32 rsv0;
-			u32 rsv1;
-		} cq_event;
-
-		struct {
-			u32 port;
-			u32 rsv0;
-			u32 rsv1;
-		} port_event;
-
-		struct {
-			u32 ceqe;
-			u32 rsv0;
-			u32 rsv1;
-		} ce_event;
-
-		struct {
-			__le64  out_param;
-			__le16  token;
-			u8	status;
-			u8	rsv0;
-		} __packed cmd;
-	 } event;
-};
-
-#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16
-#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M   \
-	(((1UL << 8) - 1) << HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)
-
-#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24
-#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M   \
-	(((1UL << 7) - 1) << HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)
-
-#define HNS_ROCE_AEQE_U32_4_OWNER_S 31
-
-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0
-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M   \
-	(((1UL << 24) - 1) << HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S)
-
-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25
-#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M   \
-	(((1UL << 3) - 1) << HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S)
-
-#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0
-#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M   \
-	(((1UL << 16) - 1) << HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S)
-
-#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0
-#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M   \
-	(((1UL << 5) - 1) << HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S)
-
-struct hns_roce_ceqe {
-	union {
-		int		comp;
-	} ceqe;
-};
-
-#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S	0
-
-#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16
-#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M   \
-	(((1UL << 16) - 1) << HNS_ROCE_CEQE_CEQE_COMP_CQN_S)
-
-#endif /* _HNS_ROCE_EQ_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index af27168faf0f..da13bd7c3ca9 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -33,6 +33,7 @@
 #include <linux/platform_device.h>
 #include <linux/acpi.h>
 #include <linux/etherdevice.h>
+#include <linux/interrupt.h>
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <rdma/ib_umem.h>
@@ -194,23 +195,47 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 
 			memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
 
-			ud_sq_wqe->va0_l = (u32)wr->sg_list[0].addr;
-			ud_sq_wqe->va0_h = (wr->sg_list[0].addr) >> 32;
-			ud_sq_wqe->l_key0 = wr->sg_list[0].lkey;
-
-			ud_sq_wqe->va1_l = (u32)wr->sg_list[1].addr;
-			ud_sq_wqe->va1_h = (wr->sg_list[1].addr) >> 32;
-			ud_sq_wqe->l_key1 = wr->sg_list[1].lkey;
+			ud_sq_wqe->va0_l =
+				       cpu_to_le32((u32)wr->sg_list[0].addr);
+			ud_sq_wqe->va0_h =
+				       cpu_to_le32((wr->sg_list[0].addr) >> 32);
+			ud_sq_wqe->l_key0 =
+				       cpu_to_le32(wr->sg_list[0].lkey);
+
+			ud_sq_wqe->va1_l =
+				       cpu_to_le32((u32)wr->sg_list[1].addr);
+			ud_sq_wqe->va1_h =
+				       cpu_to_le32((wr->sg_list[1].addr) >> 32);
+			ud_sq_wqe->l_key1 =
+				       cpu_to_le32(wr->sg_list[1].lkey);
 			ind++;
 		} else if (ibqp->qp_type == IB_QPT_RC) {
+			u32 tmp_len = 0;
+
 			ctrl = wqe;
 			memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
 			for (i = 0; i < wr->num_sge; i++)
-				ctrl->msg_length += wr->sg_list[i].length;
+				tmp_len += wr->sg_list[i].length;
+
+			ctrl->msg_length =
+			  cpu_to_le32(le32_to_cpu(ctrl->msg_length) + tmp_len);
 
 			ctrl->sgl_pa_h = 0;
 			ctrl->flag = 0;
-			ctrl->imm_data = send_ieth(wr);
+
+			switch (wr->opcode) {
+			case IB_WR_SEND_WITH_IMM:
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				ctrl->imm_data = wr->ex.imm_data;
+				break;
+			case IB_WR_SEND_WITH_INV:
+				ctrl->inv_key =
+					cpu_to_le32(wr->ex.invalidate_rkey);
+				break;
+			default:
+				ctrl->imm_data = 0;
+				break;
+			}
 
 			/*Ctrl field, ctrl set type: sig, solic, imm, fence */
 			/* SO wait for conforming application scenarios */
@@ -257,8 +282,8 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 
 			dseg = wqe;
 			if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
-				if (ctrl->msg_length >
-					hr_dev->caps.max_sq_inline) {
+				if (le32_to_cpu(ctrl->msg_length) >
+				    hr_dev->caps.max_sq_inline) {
 					ret = -EINVAL;
 					*bad_wr = wr;
 					dev_err(dev, "inline len(1-%d)=%d, illegal",
@@ -272,7 +297,7 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 					       wr->sg_list[i].length);
 					wqe += wr->sg_list[i].length;
 				}
-				ctrl->flag |= HNS_ROCE_WQE_INLINE;
+				ctrl->flag |= cpu_to_le32(HNS_ROCE_WQE_INLINE);
 			} else {
 				/*sqe num is two */
 				for (i = 0; i < wr->num_sge; i++)
@@ -305,8 +330,8 @@ static int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 			       SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
 		roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
 
-		doorbell[0] = sq_db.u32_4;
-		doorbell[1] = sq_db.u32_8;
+		doorbell[0] = le32_to_cpu(sq_db.u32_4);
+		doorbell[1] = le32_to_cpu(sq_db.u32_8);
 
 		hns_roce_write64_k(doorbell, qp->sq.db_reg_l);
 		qp->sq_next_wqe = ind;
@@ -402,8 +427,8 @@ static int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 			roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
 				     1);
 
-			doorbell[0] = rq_db.u32_4;
-			doorbell[1] = rq_db.u32_8;
+			doorbell[0] = le32_to_cpu(rq_db.u32_4);
+			doorbell[1] = le32_to_cpu(rq_db.u32_8);
 
 			hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
 		}
@@ -774,7 +799,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
 			goto create_lp_qp_failed;
 		}
 
-		ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, attr_mask,
+		ret = hr_dev->hw->modify_qp(&hr_qp->ibqp, &attr, IB_QP_DEST_QPN,
 					    IB_QPS_INIT, IB_QPS_RTR);
 		if (ret) {
 			dev_err(dev, "modify qp failed(%d)!\n", ret);
@@ -1492,9 +1517,9 @@ static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 	caps->max_sq_inline	= HNS_ROCE_V1_INLINE_SIZE;
 	caps->num_uars		= HNS_ROCE_V1_UAR_NUM;
 	caps->phy_num_uars	= HNS_ROCE_V1_PHY_UAR_NUM;
-	caps->num_aeq_vectors	= HNS_ROCE_AEQE_VEC_NUM;
-	caps->num_comp_vectors	= HNS_ROCE_COMP_VEC_NUM;
-	caps->num_other_vectors	= HNS_ROCE_AEQE_OF_VEC_NUM;
+	caps->num_aeq_vectors	= HNS_ROCE_V1_AEQE_VEC_NUM;
+	caps->num_comp_vectors	= HNS_ROCE_V1_COMP_VEC_NUM;
+	caps->num_other_vectors	= HNS_ROCE_V1_ABNORMAL_VEC_NUM;
 	caps->num_mtpts		= HNS_ROCE_V1_MAX_MTPT_NUM;
 	caps->num_mtt_segs	= HNS_ROCE_V1_MAX_MTT_SEGS;
 	caps->num_pds		= HNS_ROCE_V1_MAX_PD_NUM;
@@ -1529,10 +1554,8 @@ static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 						 caps->num_ports + 1;
 	}
 
-	for (i = 0; i < caps->num_comp_vectors; i++)
-		caps->ceqe_depth[i] = HNS_ROCE_V1_NUM_COMP_EQE;
-
-	caps->aeqe_depth = HNS_ROCE_V1_NUM_ASYNC_EQE;
+	caps->ceqe_depth = HNS_ROCE_V1_COMP_EQE_NUM;
+	caps->aeqe_depth = HNS_ROCE_V1_ASYNC_EQE_NUM;
 	caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev,
 							 ROCEE_ACK_DELAY_REG));
 	caps->max_mtu = IB_MTU_2048;
@@ -2262,7 +2285,7 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
 						CQE_BYTE_4_WQE_INDEX_M,
 						CQE_BYTE_4_WQE_INDEX_S)&
 						((*cur_qp)->sq.wqe_cnt-1));
-		switch (sq_wqe->flag & HNS_ROCE_WQE_OPCODE_MASK) {
+		switch (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_OPCODE_MASK) {
 		case HNS_ROCE_WQE_OPCODE_SEND:
 			wc->opcode = IB_WC_SEND;
 			break;
@@ -2283,7 +2306,7 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
 			wc->status = IB_WC_GENERAL_ERR;
 			break;
 		}
-		wc->wc_flags = (sq_wqe->flag & HNS_ROCE_WQE_IMM ?
+		wc->wc_flags = (le32_to_cpu(sq_wqe->flag) & HNS_ROCE_WQE_IMM ?
 				IB_WC_WITH_IMM : 0);
 
 		wq = &(*cur_qp)->sq;
@@ -2312,15 +2335,16 @@ static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
 		case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
 			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
 			wc->wc_flags = IB_WC_WITH_IMM;
-			wc->ex.imm_data = le32_to_cpu(cqe->immediate_data);
+			wc->ex.imm_data =
+				cpu_to_be32(le32_to_cpu(cqe->immediate_data));
 			break;
 		case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
 			if (roce_get_bit(cqe->cqe_byte_4,
 					 CQE_BYTE_4_IMM_INDICATOR_S)) {
 				wc->opcode = IB_WC_RECV;
 				wc->wc_flags = IB_WC_WITH_IMM;
-				wc->ex.imm_data = le32_to_cpu(
-						  cqe->immediate_data);
+				wc->ex.imm_data = cpu_to_be32(
+					le32_to_cpu(cqe->immediate_data));
 			} else {
 				wc->opcode = IB_WC_RECV;
 				wc->wc_flags = 0;
@@ -3960,6 +3984,732 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq)
 	return ret;
 }
 
+static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not)
+{
+	roce_raw_write((eq->cons_index & HNS_ROCE_V1_CONS_IDX_M) |
+		      (req_not << eq->log_entries), eq->doorbell);
+}
+
+static void hns_roce_v1_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
+					    struct hns_roce_aeqe *aeqe, int qpn)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+
+	dev_warn(dev, "Local Work Queue Catastrophic Error.\n");
+	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
+			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
+	case HNS_ROCE_LWQCE_QPC_ERROR:
+		dev_warn(dev, "QP %d, QPC error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_MTU_ERROR:
+		dev_warn(dev, "QP %d, MTU error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
+		dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
+		dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
+		dev_warn(dev, "QP %d, WQE shift error\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_SL_ERROR:
+		dev_warn(dev, "QP %d, SL error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_PORT_ERROR:
+		dev_warn(dev, "QP %d, port error.\n", qpn);
+		break;
+	default:
+		break;
+	}
+}
+
+static void hns_roce_v1_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
+						   struct hns_roce_aeqe *aeqe,
+						   int qpn)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+
+	dev_warn(dev, "Local Access Violation Work Queue Error.\n");
+	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
+			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
+	case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
+		dev_warn(dev, "QP %d, R_key violation.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_LENGTH_ERROR:
+		dev_warn(dev, "QP %d, length error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_VA_ERROR:
+		dev_warn(dev, "QP %d, VA error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_PD_ERROR:
+		dev_err(dev, "QP %d, PD error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
+		dev_warn(dev, "QP %d, rw acc error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
+		dev_warn(dev, "QP %d, key state error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
+		dev_warn(dev, "QP %d, MR operation error.\n", qpn);
+		break;
+	default:
+		break;
+	}
+}
+
+static void hns_roce_v1_qp_err_handle(struct hns_roce_dev *hr_dev,
+				      struct hns_roce_aeqe *aeqe,
+				      int event_type)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	int phy_port;
+	int qpn;
+
+	qpn = roce_get_field(aeqe->event.qp_event.qp,
+			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
+			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
+	phy_port = roce_get_field(aeqe->event.qp_event.qp,
+				  HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M,
+				  HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S);
+	if (qpn <= 1)
+		qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port;
+
+	switch (event_type) {
+	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
+		dev_warn(dev, "Invalid Req Local Work Queue Error.\n"
+			 "QP %d, phy_port %d.\n", qpn, phy_port);
+		break;
+	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+		hns_roce_v1_wq_catas_err_handle(hr_dev, aeqe, qpn);
+		break;
+	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
+		hns_roce_v1_local_wq_access_err_handle(hr_dev, aeqe, qpn);
+		break;
+	default:
+		break;
+	}
+
+	hns_roce_qp_event(hr_dev, qpn, event_type);
+}
+
+static void hns_roce_v1_cq_err_handle(struct hns_roce_dev *hr_dev,
+				      struct hns_roce_aeqe *aeqe,
+				      int event_type)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	u32 cqn;
+
+	cqn = le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq,
+			  HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
+			  HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S));
+
+	switch (event_type) {
+	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
+		dev_warn(dev, "CQ 0x%x access err.\n", cqn);
+		break;
+	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
+		dev_warn(dev, "CQ 0x%x overflow\n", cqn);
+		break;
+	case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
+		dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn);
+		break;
+	default:
+		break;
+	}
+
+	hns_roce_cq_event(hr_dev, cqn, event_type);
+}
+
+static void hns_roce_v1_db_overflow_handle(struct hns_roce_dev *hr_dev,
+					   struct hns_roce_aeqe *aeqe)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+
+	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
+			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
+	case HNS_ROCE_DB_SUBTYPE_SDB_OVF:
+		dev_warn(dev, "SDB overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF:
+		dev_warn(dev, "SDB almost overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP:
+		dev_warn(dev, "SDB almost empty.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_ODB_OVF:
+		dev_warn(dev, "ODB overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF:
+		dev_warn(dev, "ODB almost overflow.\n");
+		break;
+	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP:
+		dev_warn(dev, "SDB almost empty.\n");
+		break;
+	default:
+		break;
+	}
+}
+
+static struct hns_roce_aeqe *get_aeqe_v1(struct hns_roce_eq *eq, u32 entry)
+{
+	unsigned long off = (entry & (eq->entries - 1)) *
+			     HNS_ROCE_AEQ_ENTRY_SIZE;
+
+	return (struct hns_roce_aeqe *)((u8 *)
+		(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
+		off % HNS_ROCE_BA_SIZE);
+}
+
+static struct hns_roce_aeqe *next_aeqe_sw_v1(struct hns_roce_eq *eq)
+{
+	struct hns_roce_aeqe *aeqe = get_aeqe_v1(eq, eq->cons_index);
+
+	return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^
+		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
+}
+
+static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_eq *eq)
+{
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_aeqe *aeqe;
+	int aeqes_found = 0;
+	int event_type;
+
+	while ((aeqe = next_aeqe_sw_v1(eq))) {
+
+		/* Make sure we read the AEQ entry after we have checked the
+		 * ownership bit
+		 */
+		dma_rmb();
+
+		dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe,
+			roce_get_field(aeqe->asyn,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
+		event_type = roce_get_field(aeqe->asyn,
+					    HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
+					    HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S);
+		switch (event_type) {
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
+			dev_warn(dev, "PATH MIG not supported\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_COMM_EST:
+			dev_warn(dev, "COMMUNICATION established\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
+			dev_warn(dev, "SQ DRAINED not supported\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
+			dev_warn(dev, "PATH MIG failed\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
+		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
+			hns_roce_v1_qp_err_handle(hr_dev, aeqe, event_type);
+			break;
+		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
+		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
+		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
+			dev_warn(dev, "SRQ not support!\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
+		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
+		case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
+			hns_roce_v1_cq_err_handle(hr_dev, aeqe, event_type);
+			break;
+		case HNS_ROCE_EVENT_TYPE_PORT_CHANGE:
+			dev_warn(dev, "port change.\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_MB:
+			hns_roce_cmd_event(hr_dev,
+					   le16_to_cpu(aeqe->event.cmd.token),
+					   aeqe->event.cmd.status,
+					   le64_to_cpu(aeqe->event.cmd.out_param
+					   ));
+			break;
+		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
+			hns_roce_v1_db_overflow_handle(hr_dev, aeqe);
+			break;
+		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
+			dev_warn(dev, "CEQ 0x%lx overflow.\n",
+			roce_get_field(aeqe->event.ce_event.ceqe,
+				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M,
+				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S));
+			break;
+		default:
+			dev_warn(dev, "Unhandled event %d on EQ %d at idx %u.\n",
+				 event_type, eq->eqn, eq->cons_index);
+			break;
+		}
+
+		eq->cons_index++;
+		aeqes_found = 1;
+
+		if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) {
+			dev_warn(dev, "cons_index overflow, set back to 0.\n");
+			eq->cons_index = 0;
+		}
+	}
+
+	set_eq_cons_index_v1(eq, 0);
+
+	return aeqes_found;
+}
+
+static struct hns_roce_ceqe *get_ceqe_v1(struct hns_roce_eq *eq, u32 entry)
+{
+	unsigned long off = (entry & (eq->entries - 1)) *
+			     HNS_ROCE_CEQ_ENTRY_SIZE;
+
+	return (struct hns_roce_ceqe *)((u8 *)
+			(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
+			off % HNS_ROCE_BA_SIZE);
+}
+
+static struct hns_roce_ceqe *next_ceqe_sw_v1(struct hns_roce_eq *eq)
+{
+	struct hns_roce_ceqe *ceqe = get_ceqe_v1(eq, eq->cons_index);
+
+	return (!!(roce_get_bit(ceqe->comp,
+		HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^
+		(!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
+}
+
+static int hns_roce_v1_ceq_int(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_eq *eq)
+{
+	struct hns_roce_ceqe *ceqe;
+	int ceqes_found = 0;
+	u32 cqn;
+
+	while ((ceqe = next_ceqe_sw_v1(eq))) {
+
+		/* Make sure we read CEQ entry after we have checked the
+		 * ownership bit
+		 */
+		dma_rmb();
+
+		cqn = roce_get_field(ceqe->comp,
+				     HNS_ROCE_CEQE_CEQE_COMP_CQN_M,
+				     HNS_ROCE_CEQE_CEQE_COMP_CQN_S);
+		hns_roce_cq_completion(hr_dev, cqn);
+
+		++eq->cons_index;
+		ceqes_found = 1;
+
+		if (eq->cons_index > 2 * hr_dev->caps.ceqe_depth - 1) {
+			dev_warn(&eq->hr_dev->pdev->dev,
+				"cons_index overflow, set back to 0.\n");
+			eq->cons_index = 0;
+		}
+	}
+
+	set_eq_cons_index_v1(eq, 0);
+
+	return ceqes_found;
+}
+
+static irqreturn_t hns_roce_v1_msix_interrupt_eq(int irq, void *eq_ptr)
+{
+	struct hns_roce_eq  *eq  = eq_ptr;
+	struct hns_roce_dev *hr_dev = eq->hr_dev;
+	int int_work = 0;
+
+	if (eq->type_flag == HNS_ROCE_CEQ)
+		/* CEQ irq routine, CEQ is pulse irq, not clear */
+		int_work = hns_roce_v1_ceq_int(hr_dev, eq);
+	else
+		/* AEQ irq routine, AEQ is pulse irq, not clear */
+		int_work = hns_roce_v1_aeq_int(hr_dev, eq);
+
+	return IRQ_RETVAL(int_work);
+}
+
+static irqreturn_t hns_roce_v1_msix_interrupt_abn(int irq, void *dev_id)
+{
+	struct hns_roce_dev *hr_dev = dev_id;
+	struct device *dev = &hr_dev->pdev->dev;
+	int int_work = 0;
+	u32 caepaemask_val;
+	u32 cealmovf_val;
+	u32 caepaest_val;
+	u32 aeshift_val;
+	u32 ceshift_val;
+	u32 cemask_val;
+	int i;
+
+	/*
+	 * Abnormal interrupt:
+	 * AEQ overflow, ECC multi-bit err, CEQ overflow must clear
+	 * interrupt, mask irq, clear irq, cancel mask operation
+	 */
+	aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);
+
+	/* AEQE overflow */
+	if (roce_get_bit(aeshift_val,
+		ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) {
+		dev_warn(dev, "AEQ overflow!\n");
+
+		/* Set mask */
+		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
+		roce_set_bit(caepaemask_val,
+			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
+			     HNS_ROCE_INT_MASK_ENABLE);
+		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
+
+		/* Clear int state(INT_WC : write 1 clear) */
+		caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG);
+		roce_set_bit(caepaest_val,
+			     ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1);
+		roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val);
+
+		/* Clear mask */
+		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
+		roce_set_bit(caepaemask_val,
+			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
+			     HNS_ROCE_INT_MASK_DISABLE);
+		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
+	}
+
+	/* CEQ almost overflow */
+	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
+		ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG +
+					i * CEQ_REG_OFFSET);
+
+		if (roce_get_bit(ceshift_val,
+			ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) {
+			dev_warn(dev, "CEQ[%d] almost overflow!\n", i);
+			int_work++;
+
+			/* Set mask */
+			cemask_val = roce_read(hr_dev,
+					       ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+					       i * CEQ_REG_OFFSET);
+			roce_set_bit(cemask_val,
+				ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
+				HNS_ROCE_INT_MASK_ENABLE);
+			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+				   i * CEQ_REG_OFFSET, cemask_val);
+
+			/* Clear int state(INT_WC : write 1 clear) */
+			cealmovf_val = roce_read(hr_dev,
+				       ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
+				       i * CEQ_REG_OFFSET);
+			roce_set_bit(cealmovf_val,
+				     ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S,
+				     1);
+			roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
+				   i * CEQ_REG_OFFSET, cealmovf_val);
+
+			/* Clear mask */
+			cemask_val = roce_read(hr_dev,
+				     ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+				     i * CEQ_REG_OFFSET);
+			roce_set_bit(cemask_val,
+			       ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
+			       HNS_ROCE_INT_MASK_DISABLE);
+			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+				   i * CEQ_REG_OFFSET, cemask_val);
+		}
+	}
+
+	/* ECC multi-bit error alarm */
+	dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n",
+		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG),
+		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG),
+		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG));
+
+	dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n",
+		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG),
+		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG),
+		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG));
+
+	return IRQ_RETVAL(int_work);
+}
+
+static void hns_roce_v1_int_mask_enable(struct hns_roce_dev *hr_dev)
+{
+	u32 aemask_val;
+	int masken = 0;
+	int i;
+
+	/* AEQ INT */
+	aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
+	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
+		     masken);
+	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken);
+	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val);
+
+	/* CEQ INT */
+	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
+		/* IRQ mask */
+		roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
+			   i * CEQ_REG_OFFSET, masken);
+	}
+}
+
+static void hns_roce_v1_free_eq(struct hns_roce_dev *hr_dev,
+				struct hns_roce_eq *eq)
+{
+	int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) +
+		      HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
+	int i;
+
+	if (!eq->buf_list)
+		return;
+
+	for (i = 0; i < npages; ++i)
+		dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE,
+				  eq->buf_list[i].buf, eq->buf_list[i].map);
+
+	kfree(eq->buf_list);
+}
+
+static void hns_roce_v1_enable_eq(struct hns_roce_dev *hr_dev, int eq_num,
+				  int enable_flag)
+{
+	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num];
+	u32 val;
+
+	val = readl(eqc);
+
+	if (enable_flag)
+		roce_set_field(val,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
+			       HNS_ROCE_EQ_STAT_VALID);
+	else
+		roce_set_field(val,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
+			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
+			       HNS_ROCE_EQ_STAT_INVALID);
+	writel(val, eqc);
+}
+
+static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev,
+				 struct hns_roce_eq *eq)
+{
+	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn];
+	struct device *dev = &hr_dev->pdev->dev;
+	dma_addr_t tmp_dma_addr;
+	u32 eqconsindx_val = 0;
+	u32 eqcuridx_val = 0;
+	u32 eqshift_val = 0;
+	int num_bas;
+	int ret;
+	int i;
+
+	num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) +
+		   HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;
+
+	if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) {
+		dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n",
+			(eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE,
+			num_bas);
+		return -EINVAL;
+	}
+
+	eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL);
+	if (!eq->buf_list)
+		return -ENOMEM;
+
+	for (i = 0; i < num_bas; ++i) {
+		eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE,
+							 &tmp_dma_addr,
+							 GFP_KERNEL);
+		if (!eq->buf_list[i].buf) {
+			ret = -ENOMEM;
+			goto err_out_free_pages;
+		}
+
+		eq->buf_list[i].map = tmp_dma_addr;
+		memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE);
+	}
+	eq->cons_index = 0;
+	roce_set_field(eqshift_val,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
+		       HNS_ROCE_EQ_STAT_INVALID);
+	roce_set_field(eqshift_val,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M,
+		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S,
+		       eq->log_entries);
+	writel(eqshift_val, eqc);
+
+	/* Configure eq extended address 12~44bit */
+	writel((u32)(eq->buf_list[0].map >> 12), eqc + 4);
+
+	/*
+	 * Configure eq extended address 45~49 bit.
+	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
+	 * using 4K page, and shift more 32 because of
+	 * caculating the high 32 bit value evaluated to hardware.
+	 */
+	roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M,
+		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S,
+		       eq->buf_list[0].map >> 44);
+	roce_set_field(eqcuridx_val,
+		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M,
+		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0);
+	writel(eqcuridx_val, eqc + 8);
+
+	/* Configure eq consumer index */
+	roce_set_field(eqconsindx_val,
+		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M,
+		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0);
+	writel(eqconsindx_val, eqc + 0xc);
+
+	return 0;
+
+err_out_free_pages:
+	for (i -= 1; i >= 0; i--)
+		dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf,
+				  eq->buf_list[i].map);
+
+	kfree(eq->buf_list);
+	return ret;
+}
+
+static int hns_roce_v1_init_eq_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
+	struct device *dev = &hr_dev->pdev->dev;
+	struct hns_roce_eq *eq;
+	int irq_num;
+	int eq_num;
+	int ret;
+	int i, j;
+
+	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
+	irq_num = eq_num + hr_dev->caps.num_other_vectors;
+
+	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
+	if (!eq_table->eq)
+		return -ENOMEM;
+
+	eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base),
+				     GFP_KERNEL);
+	if (!eq_table->eqc_base) {
+		ret = -ENOMEM;
+		goto err_eqc_base_alloc_fail;
+	}
+
+	for (i = 0; i < eq_num; i++) {
+		eq = &eq_table->eq[i];
+		eq->hr_dev = hr_dev;
+		eq->eqn = i;
+		eq->irq = hr_dev->irq[i];
+		eq->log_page_size = PAGE_SHIFT;
+
+		if (i < hr_dev->caps.num_comp_vectors) {
+			/* CEQ */
+			eq_table->eqc_base[i] = hr_dev->reg_base +
+						ROCEE_CAEP_CEQC_SHIFT_0_REG +
+						CEQ_REG_OFFSET * i;
+			eq->type_flag = HNS_ROCE_CEQ;
+			eq->doorbell = hr_dev->reg_base +
+				       ROCEE_CAEP_CEQC_CONS_IDX_0_REG +
+				       CEQ_REG_OFFSET * i;
+			eq->entries = hr_dev->caps.ceqe_depth;
+			eq->log_entries = ilog2(eq->entries);
+			eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
+		} else {
+			/* AEQ */
+			eq_table->eqc_base[i] = hr_dev->reg_base +
+						ROCEE_CAEP_AEQC_AEQE_SHIFT_REG;
+			eq->type_flag = HNS_ROCE_AEQ;
+			eq->doorbell = hr_dev->reg_base +
+				       ROCEE_CAEP_AEQE_CONS_IDX_REG;
+			eq->entries = hr_dev->caps.aeqe_depth;
+			eq->log_entries = ilog2(eq->entries);
+			eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
+		}
+	}
+
+	/* Disable irq */
+	hns_roce_v1_int_mask_enable(hr_dev);
+
+	/* Configure ce int interval */
+	roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG,
+		   HNS_ROCE_CEQ_DEFAULT_INTERVAL);
+
+	/* Configure ce int burst num */
+	roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG,
+		   HNS_ROCE_CEQ_DEFAULT_BURST_NUM);
+
+	for (i = 0; i < eq_num; i++) {
+		ret = hns_roce_v1_create_eq(hr_dev, &eq_table->eq[i]);
+		if (ret) {
+			dev_err(dev, "eq create failed\n");
+			goto err_create_eq_fail;
+		}
+	}
+
+	for (j = 0; j < irq_num; j++) {
+		if (j < eq_num)
+			ret = request_irq(hr_dev->irq[j],
+					  hns_roce_v1_msix_interrupt_eq, 0,
+					  hr_dev->irq_names[j],
+					  &eq_table->eq[j]);
+		else
+			ret = request_irq(hr_dev->irq[j],
+					  hns_roce_v1_msix_interrupt_abn, 0,
+					  hr_dev->irq_names[j], hr_dev);
+
+		if (ret) {
+			dev_err(dev, "request irq error!\n");
+			goto err_request_irq_fail;
+		}
+	}
+
+	for (i = 0; i < eq_num; i++)
+		hns_roce_v1_enable_eq(hr_dev, i, EQ_ENABLE);
+
+	return 0;
+
+err_request_irq_fail:
+	for (j -= 1; j >= 0; j--)
+		free_irq(hr_dev->irq[j], &eq_table->eq[j]);
+
+err_create_eq_fail:
+	for (i -= 1; i >= 0; i--)
+		hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]);
+
+	kfree(eq_table->eqc_base);
+
+err_eqc_base_alloc_fail:
+	kfree(eq_table->eq);
+
+	return ret;
+}
+
+static void hns_roce_v1_cleanup_eq_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
+	int irq_num;
+	int eq_num;
+	int i;
+
+	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
+	irq_num = eq_num + hr_dev->caps.num_other_vectors;
+	for (i = 0; i < eq_num; i++) {
+		/* Disable EQ */
+		hns_roce_v1_enable_eq(hr_dev, i, EQ_DISABLE);
+
+		free_irq(hr_dev->irq[i], &eq_table->eq[i]);
+
+		hns_roce_v1_free_eq(hr_dev, &eq_table->eq[i]);
+	}
+	for (i = eq_num; i < irq_num; i++)
+		free_irq(hr_dev->irq[i], hr_dev);
+
+	kfree(eq_table->eqc_base);
+	kfree(eq_table->eq);
+}
+
 static const struct hns_roce_hw hns_roce_hw_v1 = {
 	.reset = hns_roce_v1_reset,
 	.hw_profile = hns_roce_v1_profile,
@@ -3983,6 +4733,8 @@ static const struct hns_roce_hw hns_roce_hw_v1 = {
 	.poll_cq = hns_roce_v1_poll_cq,
 	.dereg_mr = hns_roce_v1_dereg_mr,
 	.destroy_cq = hns_roce_v1_destroy_cq,
+	.init_eq = hns_roce_v1_init_eq_table,
+	.cleanup_eq = hns_roce_v1_cleanup_eq_table,
 };
 
 static const struct of_device_id hns_roce_of_match[] = {
@@ -4060,10 +4812,6 @@ static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev)
 
 	/* get the mapped register base address */
 	res = platform_get_resource(hr_dev->pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		dev_err(dev, "memory resource not found!\n");
-		return -EINVAL;
-	}
 	hr_dev->reg_base = devm_ioremap_resource(dev, res);
 	if (IS_ERR(hr_dev->reg_base))
 		return PTR_ERR(hr_dev->reg_base);
@@ -4132,14 +4880,14 @@ static int hns_roce_get_cfg(struct hns_roce_dev *hr_dev)
 	/* read the interrupt names from the DT or ACPI */
 	ret = device_property_read_string_array(dev, "interrupt-names",
 						hr_dev->irq_names,
-						HNS_ROCE_MAX_IRQ_NUM);
+						HNS_ROCE_V1_MAX_IRQ_NUM);
 	if (ret < 0) {
 		dev_err(dev, "couldn't get interrupt names from DT or ACPI!\n");
 		return ret;
 	}
 
 	/* fetch the interrupt numbers */
-	for (i = 0; i < HNS_ROCE_MAX_IRQ_NUM; i++) {
+	for (i = 0; i < HNS_ROCE_V1_MAX_IRQ_NUM; i++) {
 		hr_dev->irq[i] = platform_get_irq(hr_dev->pdev, i);
 		if (hr_dev->irq[i] <= 0) {
 			dev_err(dev, "platform get of irq[=%d] failed!\n", i);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
index 21a07ef0afc9..e9a2717ea7cd 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
@@ -60,8 +60,13 @@
 #define HNS_ROCE_V1_GID_NUM				16
 #define HNS_ROCE_V1_RESV_QP				8
 
-#define HNS_ROCE_V1_NUM_COMP_EQE			0x8000
-#define HNS_ROCE_V1_NUM_ASYNC_EQE			0x400
+#define HNS_ROCE_V1_MAX_IRQ_NUM				34
+#define HNS_ROCE_V1_COMP_VEC_NUM			32
+#define HNS_ROCE_V1_AEQE_VEC_NUM			1
+#define HNS_ROCE_V1_ABNORMAL_VEC_NUM			1
+
+#define HNS_ROCE_V1_COMP_EQE_NUM			0x8000
+#define HNS_ROCE_V1_ASYNC_EQE_NUM			0x400
 
 #define HNS_ROCE_V1_QPC_ENTRY_SIZE			256
 #define HNS_ROCE_V1_IRRL_ENTRY_SIZE			8
@@ -159,15 +164,50 @@
 #define SDB_INV_CNT_OFFSET				8
 #define SDB_ST_CMP_VAL					8
 
+#define HNS_ROCE_CEQ_DEFAULT_INTERVAL			0x10
+#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM			0x10
+
+#define HNS_ROCE_INT_MASK_DISABLE			0
+#define HNS_ROCE_INT_MASK_ENABLE			1
+
+#define CEQ_REG_OFFSET					0x18
+
+#define HNS_ROCE_CEQE_CEQE_COMP_OWNER_S	0
+
+#define HNS_ROCE_V1_CONS_IDX_M GENMASK(15, 0)
+
+#define HNS_ROCE_CEQE_CEQE_COMP_CQN_S 16
+#define HNS_ROCE_CEQE_CEQE_COMP_CQN_M GENMASK(31, 16)
+
+#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S 16
+#define HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M GENMASK(23, 16)
+
+#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S 24
+#define HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M GENMASK(30, 24)
+
+#define HNS_ROCE_AEQE_U32_4_OWNER_S 31
+
+#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S 0
+#define HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M GENMASK(23, 0)
+
+#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S 25
+#define HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M GENMASK(27, 25)
+
+#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S 0
+#define HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M GENMASK(15, 0)
+
+#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S 0
+#define HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M GENMASK(4, 0)
+
 struct hns_roce_cq_context {
-	u32 cqc_byte_4;
-	u32 cq_bt_l;
-	u32 cqc_byte_12;
-	u32 cur_cqe_ba0_l;
-	u32 cqc_byte_20;
-	u32 cqe_tptr_addr_l;
-	u32 cur_cqe_ba1_l;
-	u32 cqc_byte_32;
+	__le32 cqc_byte_4;
+	__le32 cq_bt_l;
+	__le32 cqc_byte_12;
+	__le32 cur_cqe_ba0_l;
+	__le32 cqc_byte_20;
+	__le32 cqe_tptr_addr_l;
+	__le32 cur_cqe_ba1_l;
+	__le32 cqc_byte_32;
 };
 
 #define CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S 0
@@ -217,17 +257,17 @@ struct hns_roce_cq_context {
 	(((1UL << 16) - 1) << CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S)
 
 struct hns_roce_cqe {
-	u32 cqe_byte_4;
+	__le32 cqe_byte_4;
 	union {
-		u32 r_key;
-		u32 immediate_data;
+		__le32 r_key;
+		__be32 immediate_data;
 	};
-	u32 byte_cnt;
-	u32 cqe_byte_16;
-	u32 cqe_byte_20;
-	u32 s_mac_l;
-	u32 cqe_byte_28;
-	u32 reserved;
+	__le32 byte_cnt;
+	__le32 cqe_byte_16;
+	__le32 cqe_byte_20;
+	__le32 s_mac_l;
+	__le32 cqe_byte_28;
+	__le32 reserved;
 };
 
 #define CQE_BYTE_4_OWNER_S 7
@@ -268,22 +308,22 @@ struct hns_roce_cqe {
 #define CQ_DB_REQ_NOT		(1 << 16)
 
 struct hns_roce_v1_mpt_entry {
-	u32  mpt_byte_4;
-	u32  pbl_addr_l;
-	u32  mpt_byte_12;
-	u32  virt_addr_l;
-	u32  virt_addr_h;
-	u32  length;
-	u32  mpt_byte_28;
-	u32  pa0_l;
-	u32  mpt_byte_36;
-	u32  mpt_byte_40;
-	u32  mpt_byte_44;
-	u32  mpt_byte_48;
-	u32  pa4_l;
-	u32  mpt_byte_56;
-	u32  mpt_byte_60;
-	u32  mpt_byte_64;
+	__le32  mpt_byte_4;
+	__le32  pbl_addr_l;
+	__le32  mpt_byte_12;
+	__le32  virt_addr_l;
+	__le32  virt_addr_h;
+	__le32  length;
+	__le32  mpt_byte_28;
+	__le32  pa0_l;
+	__le32  mpt_byte_36;
+	__le32  mpt_byte_40;
+	__le32  mpt_byte_44;
+	__le32  mpt_byte_48;
+	__le32  pa4_l;
+	__le32  mpt_byte_56;
+	__le32  mpt_byte_60;
+	__le32  mpt_byte_64;
 };
 
 #define MPT_BYTE_4_KEY_STATE_S 0
@@ -368,30 +408,32 @@ struct hns_roce_v1_mpt_entry {
 	(((1UL << 8) - 1) << MPT_BYTE_64_L_KEY_IDX_H_S)
 
 struct hns_roce_wqe_ctrl_seg {
-	__be32 sgl_pa_h;
-	__be32 flag;
-	__be32 imm_data;
-	__be32 msg_length;
+	__le32 sgl_pa_h;
+	__le32 flag;
+	union {
+		__be32 imm_data;
+		__le32 inv_key;
+	};
+	__le32 msg_length;
 };
 
 struct hns_roce_wqe_data_seg {
-	__be64    addr;
-	__be32    lkey;
-	__be32    len;
+	__le64    addr;
+	__le32    lkey;
+	__le32    len;
 };
 
 struct hns_roce_wqe_raddr_seg {
-	__be32 rkey;
-	__be32 len;/* reserved */
-	__be64 raddr;
+	__le32 rkey;
+	__le32 len;/* reserved */
+	__le64 raddr;
 };
 
 struct hns_roce_rq_wqe_ctrl {
-
-	u32 rwqe_byte_4;
-	u32 rocee_sgl_ba_l;
-	u32 rwqe_byte_12;
-	u32 reserved[5];
+	__le32 rwqe_byte_4;
+	__le32 rocee_sgl_ba_l;
+	__le32 rwqe_byte_12;
+	__le32 reserved[5];
 };
 
 #define RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S 16
@@ -403,31 +445,31 @@ struct hns_roce_rq_wqe_ctrl {
 #define GID_LEN					16
 
 struct hns_roce_ud_send_wqe {
-	u32 dmac_h;
-	u32 u32_8;
-	u32 immediate_data;
+	__le32 dmac_h;
+	__le32 u32_8;
+	__le32 immediate_data;
 
-	u32 u32_16;
+	__le32 u32_16;
 	union {
 		unsigned char dgid[GID_LEN];
 		struct {
-			u32 u32_20;
-			u32 u32_24;
-			u32 u32_28;
-			u32 u32_32;
+			__le32 u32_20;
+			__le32 u32_24;
+			__le32 u32_28;
+			__le32 u32_32;
 		};
 	};
 
-	u32 u32_36;
-	u32 u32_40;
+	__le32 u32_36;
+	__le32 u32_40;
 
-	u32 va0_l;
-	u32 va0_h;
-	u32 l_key0;
+	__le32 va0_l;
+	__le32 va0_h;
+	__le32 l_key0;
 
-	u32 va1_l;
-	u32 va1_h;
-	u32 l_key1;
+	__le32 va1_l;
+	__le32 va1_h;
+	__le32 l_key1;
 };
 
 #define UD_SEND_WQE_U32_4_DMAC_0_S 0
@@ -495,16 +537,16 @@ struct hns_roce_ud_send_wqe {
 	(((1UL << 8) - 1) << UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S)
 
 struct hns_roce_sqp_context {
-	u32 qp1c_bytes_4;
-	u32 sq_rq_bt_l;
-	u32 qp1c_bytes_12;
-	u32 qp1c_bytes_16;
-	u32 qp1c_bytes_20;
-	u32 cur_rq_wqe_ba_l;
-	u32 qp1c_bytes_28;
-	u32 qp1c_bytes_32;
-	u32 cur_sq_wqe_ba_l;
-	u32 qp1c_bytes_40;
+	__le32 qp1c_bytes_4;
+	__le32 sq_rq_bt_l;
+	__le32 qp1c_bytes_12;
+	__le32 qp1c_bytes_16;
+	__le32 qp1c_bytes_20;
+	__le32 cur_rq_wqe_ba_l;
+	__le32 qp1c_bytes_28;
+	__le32 qp1c_bytes_32;
+	__le32 cur_sq_wqe_ba_l;
+	__le32 qp1c_bytes_40;
 };
 
 #define QP1C_BYTES_4_QP_STATE_S 0
@@ -586,64 +628,64 @@ struct hns_roce_sqp_context {
 #define HNS_ROCE_WQE_OPCODE_MASK	(15<<16)
 
 struct hns_roce_qp_context {
-	u32 qpc_bytes_4;
-	u32 qpc_bytes_8;
-	u32 qpc_bytes_12;
-	u32 qpc_bytes_16;
-	u32 sq_rq_bt_l;
-	u32 qpc_bytes_24;
-	u32 irrl_ba_l;
-	u32 qpc_bytes_32;
-	u32 qpc_bytes_36;
-	u32 dmac_l;
-	u32 qpc_bytes_44;
-	u32 qpc_bytes_48;
-	u8 dgid[16];
-	u32 qpc_bytes_68;
-	u32 cur_rq_wqe_ba_l;
-	u32 qpc_bytes_76;
-	u32 rx_rnr_time;
-	u32 qpc_bytes_84;
-	u32 qpc_bytes_88;
+	__le32 qpc_bytes_4;
+	__le32 qpc_bytes_8;
+	__le32 qpc_bytes_12;
+	__le32 qpc_bytes_16;
+	__le32 sq_rq_bt_l;
+	__le32 qpc_bytes_24;
+	__le32 irrl_ba_l;
+	__le32 qpc_bytes_32;
+	__le32 qpc_bytes_36;
+	__le32 dmac_l;
+	__le32 qpc_bytes_44;
+	__le32 qpc_bytes_48;
+	u8     dgid[16];
+	__le32 qpc_bytes_68;
+	__le32 cur_rq_wqe_ba_l;
+	__le32 qpc_bytes_76;
+	__le32 rx_rnr_time;
+	__le32 qpc_bytes_84;
+	__le32 qpc_bytes_88;
 	union {
-		u32 rx_sge_len;
-		u32 dma_length;
+		__le32 rx_sge_len;
+		__le32 dma_length;
 	};
 	union {
-		u32 rx_sge_num;
-		u32 rx_send_pktn;
-		u32 r_key;
+		__le32 rx_sge_num;
+		__le32 rx_send_pktn;
+		__le32 r_key;
 	};
-	u32 va_l;
-	u32 va_h;
-	u32 qpc_bytes_108;
-	u32 qpc_bytes_112;
-	u32 rx_cur_sq_wqe_ba_l;
-	u32 qpc_bytes_120;
-	u32 qpc_bytes_124;
-	u32 qpc_bytes_128;
-	u32 qpc_bytes_132;
-	u32 qpc_bytes_136;
-	u32 qpc_bytes_140;
-	u32 qpc_bytes_144;
-	u32 qpc_bytes_148;
+	__le32 va_l;
+	__le32 va_h;
+	__le32 qpc_bytes_108;
+	__le32 qpc_bytes_112;
+	__le32 rx_cur_sq_wqe_ba_l;
+	__le32 qpc_bytes_120;
+	__le32 qpc_bytes_124;
+	__le32 qpc_bytes_128;
+	__le32 qpc_bytes_132;
+	__le32 qpc_bytes_136;
+	__le32 qpc_bytes_140;
+	__le32 qpc_bytes_144;
+	__le32 qpc_bytes_148;
 	union {
-		u32 rnr_retry;
-		u32 ack_time;
+		__le32 rnr_retry;
+		__le32 ack_time;
 	};
-	u32 qpc_bytes_156;
-	u32 pkt_use_len;
-	u32 qpc_bytes_164;
-	u32 qpc_bytes_168;
+	__le32 qpc_bytes_156;
+	__le32 pkt_use_len;
+	__le32 qpc_bytes_164;
+	__le32 qpc_bytes_168;
 	union {
-		u32 sge_use_len;
-		u32 pa_use_len;
+		__le32 sge_use_len;
+		__le32 pa_use_len;
 	};
-	u32 qpc_bytes_176;
-	u32 qpc_bytes_180;
-	u32 tx_cur_sq_wqe_ba_l;
-	u32 qpc_bytes_188;
-	u32 rvd21;
+	__le32 qpc_bytes_176;
+	__le32 qpc_bytes_180;
+	__le32 tx_cur_sq_wqe_ba_l;
+	__le32 qpc_bytes_188;
+	__le32 rvd21;
 };
 
 #define QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S 0
@@ -956,8 +998,8 @@ struct hns_roce_qp_context {
 #define HCR_GO_BIT		15
 
 struct hns_roce_rq_db {
-	u32    u32_4;
-	u32    u32_8;
+	__le32    u32_4;
+	__le32    u32_8;
 };
 
 #define RQ_DOORBELL_U32_4_RQ_HEAD_S 0
@@ -973,8 +1015,8 @@ struct hns_roce_rq_db {
 #define RQ_DOORBELL_U32_8_HW_SYNC_S 31
 
 struct hns_roce_sq_db {
-	u32    u32_4;
-	u32    u32_8;
+	__le32    u32_4;
+	__le32    u32_8;
 };
 
 #define SQ_DOORBELL_U32_4_SQ_HEAD_S 0
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 8e18445714a9..ec638778661c 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -34,6 +34,7 @@
 #include <linux/etherdevice.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
+#include <net/addrconf.h>
 #include <rdma/ib_umem.h>
 
 #include "hnae3.h"
@@ -51,32 +52,108 @@ static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
 	dseg->len  = cpu_to_le32(sg->length);
 }
 
+static int set_rwqe_data_seg(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			     struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
+			     void *wqe, unsigned int *sge_ind,
+			     struct ib_send_wr **bad_wr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_v2_wqe_data_seg *dseg = wqe;
+	struct hns_roce_qp *qp = to_hr_qp(ibqp);
+	int i;
+
+	if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
+		if (le32_to_cpu(rc_sq_wqe->msg_len) >
+		    hr_dev->caps.max_sq_inline) {
+			*bad_wr = wr;
+			dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
+				rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
+			return -EINVAL;
+		}
+
+		for (i = 0; i < wr->num_sge; i++) {
+			memcpy(wqe, ((void *)wr->sg_list[i].addr),
+			       wr->sg_list[i].length);
+			wqe += wr->sg_list[i].length;
+		}
+
+		roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
+			     1);
+	} else {
+		if (wr->num_sge <= 2) {
+			for (i = 0; i < wr->num_sge; i++) {
+				if (likely(wr->sg_list[i].length)) {
+					set_data_seg_v2(dseg, wr->sg_list + i);
+					dseg++;
+				}
+			}
+		} else {
+			roce_set_field(rc_sq_wqe->byte_20,
+				     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
+				     V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
+				     (*sge_ind) & (qp->sge.sge_cnt - 1));
+
+			for (i = 0; i < 2; i++) {
+				if (likely(wr->sg_list[i].length)) {
+					set_data_seg_v2(dseg, wr->sg_list + i);
+					dseg++;
+				}
+			}
+
+			dseg = get_send_extend_sge(qp,
+					    (*sge_ind) & (qp->sge.sge_cnt - 1));
+
+			for (i = 0; i < wr->num_sge - 2; i++) {
+				if (likely(wr->sg_list[i + 2].length)) {
+					set_data_seg_v2(dseg,
+							wr->sg_list + 2 + i);
+					dseg++;
+					(*sge_ind)++;
+				}
+			}
+		}
+
+		roce_set_field(rc_sq_wqe->byte_16,
+			       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
+			       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
+	}
+
+	return 0;
+}
+
 static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 				 struct ib_send_wr **bad_wr)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
+	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
+	struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
 	struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
 	struct hns_roce_qp *qp = to_hr_qp(ibqp);
 	struct hns_roce_v2_wqe_data_seg *dseg;
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_v2_db sq_db;
 	unsigned int sge_ind = 0;
-	unsigned int wqe_sz = 0;
 	unsigned int owner_bit;
 	unsigned long flags;
 	unsigned int ind;
 	void *wqe = NULL;
+	u32 tmp_len = 0;
+	bool loopback;
 	int ret = 0;
+	u8 *smac;
 	int nreq;
 	int i;
 
-	if (unlikely(ibqp->qp_type != IB_QPT_RC)) {
+	if (unlikely(ibqp->qp_type != IB_QPT_RC &&
+		     ibqp->qp_type != IB_QPT_GSI &&
+		     ibqp->qp_type != IB_QPT_UD)) {
 		dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
 		*bad_wr = NULL;
 		return -EOPNOTSUPP;
 	}
 
-	if (unlikely(qp->state != IB_QPS_RTS && qp->state != IB_QPS_SQD)) {
+	if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
+		     qp->state == IB_QPS_RTR)) {
 		dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
 		*bad_wr = wr;
 		return -EINVAL;
@@ -106,161 +183,281 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 								      wr->wr_id;
 
 		owner_bit = ~(qp->sq.head >> ilog2(qp->sq.wqe_cnt)) & 0x1;
-		rc_sq_wqe = wqe;
-		memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
-		for (i = 0; i < wr->num_sge; i++)
-			rc_sq_wqe->msg_len += wr->sg_list[i].length;
 
-		rc_sq_wqe->inv_key_immtdata = send_ieth(wr);
+		/* Corresponding to the QP type, wqe process separately */
+		if (ibqp->qp_type == IB_QPT_GSI) {
+			ud_sq_wqe = wqe;
+			memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
+
+			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
+				       V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
+			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
+				       V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
+			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
+				       V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
+			roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
+				       V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
+			roce_set_field(ud_sq_wqe->byte_48,
+				       V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
+				       V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
+				       ah->av.mac[4]);
+			roce_set_field(ud_sq_wqe->byte_48,
+				       V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
+				       V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
+				       ah->av.mac[5]);
+
+			/* MAC loopback */
+			smac = (u8 *)hr_dev->dev_addr[qp->port];
+			loopback = ether_addr_equal_unaligned(ah->av.mac,
+							      smac) ? 1 : 0;
+
+			roce_set_bit(ud_sq_wqe->byte_40,
+				     V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
+
+			roce_set_field(ud_sq_wqe->byte_4,
+				       V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
+				       V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
+				       HNS_ROCE_V2_WQE_OP_SEND);
 
-		roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_FENCE_S,
-			    (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
+			for (i = 0; i < wr->num_sge; i++)
+				tmp_len += wr->sg_list[i].length;
 
-		roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_SE_S,
-			    (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
+			ud_sq_wqe->msg_len =
+			 cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
 
-		roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_CQE_S,
-			    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
+			switch (wr->opcode) {
+			case IB_WR_SEND_WITH_IMM:
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				ud_sq_wqe->immtdata = wr->ex.imm_data;
+				break;
+			default:
+				ud_sq_wqe->immtdata = 0;
+				break;
+			}
 
-		roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_OWNER_S,
-			     owner_bit);
+			/* Set sig attr */
+			roce_set_bit(ud_sq_wqe->byte_4,
+				   V2_UD_SEND_WQE_BYTE_4_CQE_S,
+				   (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 
-		switch (wr->opcode) {
-		case IB_WR_RDMA_READ:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_RDMA_READ);
-			rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
-			rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
-			break;
-		case IB_WR_RDMA_WRITE:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_RDMA_WRITE);
-			rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
-			rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
-			break;
-		case IB_WR_RDMA_WRITE_WITH_IMM:
-			roce_set_field(rc_sq_wqe->byte_4,
+			/* Set se attr */
+			roce_set_bit(ud_sq_wqe->byte_4,
+				  V2_UD_SEND_WQE_BYTE_4_SE_S,
+				  (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
+
+			roce_set_bit(ud_sq_wqe->byte_4,
+				     V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
+
+			roce_set_field(ud_sq_wqe->byte_16,
+				       V2_UD_SEND_WQE_BYTE_16_PD_M,
+				       V2_UD_SEND_WQE_BYTE_16_PD_S,
+				       to_hr_pd(ibqp->pd)->pdn);
+
+			roce_set_field(ud_sq_wqe->byte_16,
+				       V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
+				       V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
+				       wr->num_sge);
+
+			roce_set_field(ud_sq_wqe->byte_20,
+				     V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
+				     V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
+				     sge_ind & (qp->sge.sge_cnt - 1));
+
+			roce_set_field(ud_sq_wqe->byte_24,
+				       V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
+				       V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
+			ud_sq_wqe->qkey =
+			     cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
+			     qp->qkey : ud_wr(wr)->remote_qkey);
+			roce_set_field(ud_sq_wqe->byte_32,
+				       V2_UD_SEND_WQE_BYTE_32_DQPN_M,
+				       V2_UD_SEND_WQE_BYTE_32_DQPN_S,
+				       ud_wr(wr)->remote_qpn);
+
+			roce_set_field(ud_sq_wqe->byte_36,
+				       V2_UD_SEND_WQE_BYTE_36_VLAN_M,
+				       V2_UD_SEND_WQE_BYTE_36_VLAN_S,
+				       le16_to_cpu(ah->av.vlan));
+			roce_set_field(ud_sq_wqe->byte_36,
+				       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
+				       V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
+				       ah->av.hop_limit);
+			roce_set_field(ud_sq_wqe->byte_36,
+				       V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
+				       V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
+				       0);
+			roce_set_field(ud_sq_wqe->byte_36,
+				       V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
+				       V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
+				       0);
+			roce_set_field(ud_sq_wqe->byte_40,
+				       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
+				       V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S, 0);
+			roce_set_field(ud_sq_wqe->byte_40,
+				       V2_UD_SEND_WQE_BYTE_40_SL_M,
+				       V2_UD_SEND_WQE_BYTE_40_SL_S,
+				      le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
+				      HNS_ROCE_SL_SHIFT);
+			roce_set_field(ud_sq_wqe->byte_40,
+				       V2_UD_SEND_WQE_BYTE_40_PORTN_M,
+				       V2_UD_SEND_WQE_BYTE_40_PORTN_S,
+				       qp->port);
+
+			roce_set_field(ud_sq_wqe->byte_48,
+				       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
+				       V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
+				       hns_get_gid_index(hr_dev, qp->phy_port,
+							 ah->av.gid_index));
+
+			memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
+			       GID_LEN_V2);
+
+			dseg = get_send_extend_sge(qp,
+					    sge_ind & (qp->sge.sge_cnt - 1));
+			for (i = 0; i < wr->num_sge; i++) {
+				set_data_seg_v2(dseg + i, wr->sg_list + i);
+				sge_ind++;
+			}
+
+			ind++;
+		} else if (ibqp->qp_type == IB_QPT_RC) {
+			rc_sq_wqe = wqe;
+			memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
+			for (i = 0; i < wr->num_sge; i++)
+				tmp_len += wr->sg_list[i].length;
+
+			rc_sq_wqe->msg_len =
+			 cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
+
+			switch (wr->opcode) {
+			case IB_WR_SEND_WITH_IMM:
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				rc_sq_wqe->immtdata = wr->ex.imm_data;
+				break;
+			case IB_WR_SEND_WITH_INV:
+				rc_sq_wqe->inv_key =
+					cpu_to_le32(wr->ex.invalidate_rkey);
+				break;
+			default:
+				rc_sq_wqe->immtdata = 0;
+				break;
+			}
+
+			roce_set_bit(rc_sq_wqe->byte_4,
+				     V2_RC_SEND_WQE_BYTE_4_FENCE_S,
+				     (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
+
+			roce_set_bit(rc_sq_wqe->byte_4,
+				  V2_RC_SEND_WQE_BYTE_4_SE_S,
+				  (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
+
+			roce_set_bit(rc_sq_wqe->byte_4,
+				   V2_RC_SEND_WQE_BYTE_4_CQE_S,
+				   (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
+
+			roce_set_bit(rc_sq_wqe->byte_4,
+				     V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
+
+			switch (wr->opcode) {
+			case IB_WR_RDMA_READ:
+				roce_set_field(rc_sq_wqe->byte_4,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					       HNS_ROCE_V2_WQE_OP_RDMA_READ);
+				rc_sq_wqe->rkey =
+					cpu_to_le32(rdma_wr(wr)->rkey);
+				rc_sq_wqe->va =
+					cpu_to_le64(rdma_wr(wr)->remote_addr);
+				break;
+			case IB_WR_RDMA_WRITE:
+				roce_set_field(rc_sq_wqe->byte_4,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					       HNS_ROCE_V2_WQE_OP_RDMA_WRITE);
+				rc_sq_wqe->rkey =
+					cpu_to_le32(rdma_wr(wr)->rkey);
+				rc_sq_wqe->va =
+					cpu_to_le64(rdma_wr(wr)->remote_addr);
+				break;
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				roce_set_field(rc_sq_wqe->byte_4,
 				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
 				       HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM);
-			rc_sq_wqe->rkey = cpu_to_le32(rdma_wr(wr)->rkey);
-			rc_sq_wqe->va = cpu_to_le64(rdma_wr(wr)->remote_addr);
-			break;
-		case IB_WR_SEND:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_SEND);
-			break;
-		case IB_WR_SEND_WITH_INV:
-			roce_set_field(rc_sq_wqe->byte_4,
+				rc_sq_wqe->rkey =
+					cpu_to_le32(rdma_wr(wr)->rkey);
+				rc_sq_wqe->va =
+					cpu_to_le64(rdma_wr(wr)->remote_addr);
+				break;
+			case IB_WR_SEND:
+				roce_set_field(rc_sq_wqe->byte_4,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					       HNS_ROCE_V2_WQE_OP_SEND);
+				break;
+			case IB_WR_SEND_WITH_INV:
+				roce_set_field(rc_sq_wqe->byte_4,
 				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
 				       HNS_ROCE_V2_WQE_OP_SEND_WITH_INV);
-			break;
-		case IB_WR_SEND_WITH_IMM:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM);
-			break;
-		case IB_WR_LOCAL_INV:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_LOCAL_INV);
-			break;
-		case IB_WR_ATOMIC_CMP_AND_SWP:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP);
-			break;
-		case IB_WR_ATOMIC_FETCH_AND_ADD:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD);
-			break;
-		case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
-			roce_set_field(rc_sq_wqe->byte_4,
+				break;
+			case IB_WR_SEND_WITH_IMM:
+				roce_set_field(rc_sq_wqe->byte_4,
+					      V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					      V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					      HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM);
+				break;
+			case IB_WR_LOCAL_INV:
+				roce_set_field(rc_sq_wqe->byte_4,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					       HNS_ROCE_V2_WQE_OP_LOCAL_INV);
+				break;
+			case IB_WR_ATOMIC_CMP_AND_SWP:
+				roce_set_field(rc_sq_wqe->byte_4,
+					  V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					  V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					  HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP);
+				break;
+			case IB_WR_ATOMIC_FETCH_AND_ADD:
+				roce_set_field(rc_sq_wqe->byte_4,
+					 V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					 V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					 HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD);
+				break;
+			case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
+				roce_set_field(rc_sq_wqe->byte_4,
 				      V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 				      V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
 				      HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP);
-			break;
-		case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
-			roce_set_field(rc_sq_wqe->byte_4,
+				break;
+			case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
+				roce_set_field(rc_sq_wqe->byte_4,
 				     V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 				     V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
 				     HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD);
-			break;
-		default:
-			roce_set_field(rc_sq_wqe->byte_4,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
-				       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
-				       HNS_ROCE_V2_WQE_OP_MASK);
-			break;
-		}
-
-		wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
-		dseg = wqe;
-		if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
-			if (rc_sq_wqe->msg_len >
-				hr_dev->caps.max_sq_inline) {
-				ret = -EINVAL;
-				*bad_wr = wr;
-				dev_err(dev, "inline len(1-%d)=%d, illegal",
-					rc_sq_wqe->msg_len,
-					hr_dev->caps.max_sq_inline);
-				goto out;
+				break;
+			default:
+				roce_set_field(rc_sq_wqe->byte_4,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
+					       V2_RC_SEND_WQE_BYTE_4_OPCODE_S,
+					       HNS_ROCE_V2_WQE_OP_MASK);
+				break;
 			}
 
-			for (i = 0; i < wr->num_sge; i++) {
-				memcpy(wqe, ((void *)wr->sg_list[i].addr),
-				       wr->sg_list[i].length);
-				wqe += wr->sg_list[i].length;
-				wqe_sz += wr->sg_list[i].length;
-			}
+			wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
+			dseg = wqe;
 
-			roce_set_bit(rc_sq_wqe->byte_4,
-				     V2_RC_SEND_WQE_BYTE_4_INLINE_S, 1);
+			ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe, wqe,
+						&sge_ind, bad_wr);
+			if (ret)
+				goto out;
+			ind++;
 		} else {
-			if (wr->num_sge <= 2) {
-				for (i = 0; i < wr->num_sge; i++)
-					set_data_seg_v2(dseg + i,
-							wr->sg_list + i);
-			} else {
-				roce_set_field(rc_sq_wqe->byte_20,
-				V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
-				V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
-				sge_ind & (qp->sge.sge_cnt - 1));
-
-				for (i = 0; i < 2; i++)
-					set_data_seg_v2(dseg + i,
-							wr->sg_list + i);
-
-				dseg = get_send_extend_sge(qp,
-					sge_ind & (qp->sge.sge_cnt - 1));
-
-				for (i = 0; i < wr->num_sge - 2; i++) {
-					set_data_seg_v2(dseg + i,
-							wr->sg_list + 2 + i);
-					sge_ind++;
-				}
-			}
-
-			roce_set_field(rc_sq_wqe->byte_16,
-				       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
-				       V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
-				       wr->num_sge);
-			wqe_sz += wr->num_sge *
-				  sizeof(struct hns_roce_v2_wqe_data_seg);
+			dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
+			spin_unlock_irqrestore(&qp->sq.lock, flags);
+			return -EOPNOTSUPP;
 		}
-		ind++;
 	}
 
 out:
@@ -282,7 +479,7 @@ static int hns_roce_v2_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
 			       V2_DB_PARAMETER_SL_S, qp->sl);
 
-		hns_roce_write64_k((__be32 *)&sq_db, qp->sq.db_reg_l);
+		hns_roce_write64_k((__le32 *)&sq_db, qp->sq.db_reg_l);
 
 		qp->sq_next_wqe = ind;
 		qp->next_sge = sge_ind;
@@ -299,6 +496,7 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 	struct hns_roce_v2_wqe_data_seg *dseg;
+	struct hns_roce_rinl_sge *sge_list;
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_v2_db rq_db;
 	unsigned long flags;
@@ -343,10 +541,18 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 		}
 
 		if (i < hr_qp->rq.max_gs) {
-			dseg[i].lkey = cpu_to_be32(HNS_ROCE_INVALID_LKEY);
+			dseg[i].lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
 			dseg[i].addr = 0;
 		}
 
+		/* rq support inline data */
+		sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
+		hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt = (u32)wr->num_sge;
+		for (i = 0; i < wr->num_sge; i++) {
+			sge_list[i].addr = (void *)(u64)wr->sg_list[i].addr;
+			sge_list[i].len = wr->sg_list[i].length;
+		}
+
 		hr_qp->rq.wrid[ind] = wr->wr_id;
 
 		ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
@@ -368,7 +574,7 @@ static int hns_roce_v2_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 		roce_set_field(rq_db.parameter, V2_DB_PARAMETER_CONS_IDX_M,
 			       V2_DB_PARAMETER_CONS_IDX_S, hr_qp->rq.head);
 
-		hns_roce_write64_k((__be32 *)&rq_db, hr_qp->rq.db_reg_l);
+		hns_roce_write64_k((__le32 *)&rq_db, hr_qp->rq.db_reg_l);
 	}
 	spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 
@@ -908,9 +1114,9 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
 	caps->max_sq_inline	= HNS_ROCE_V2_MAX_SQ_INLINE;
 	caps->num_uars		= HNS_ROCE_V2_UAR_NUM;
 	caps->phy_num_uars	= HNS_ROCE_V2_PHY_UAR_NUM;
-	caps->num_aeq_vectors	= 1;
-	caps->num_comp_vectors	= 63;
-	caps->num_other_vectors	= 0;
+	caps->num_aeq_vectors	= HNS_ROCE_V2_AEQE_VEC_NUM;
+	caps->num_comp_vectors	= HNS_ROCE_V2_COMP_VEC_NUM;
+	caps->num_other_vectors	= HNS_ROCE_V2_ABNORMAL_VEC_NUM;
 	caps->num_mtpts		= HNS_ROCE_V2_MAX_MTPT_NUM;
 	caps->num_mtt_segs	= HNS_ROCE_V2_MAX_MTT_SEGS;
 	caps->num_cqe_segs	= HNS_ROCE_V2_MAX_CQE_SEGS;
@@ -955,12 +1161,18 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
 	caps->cqe_ba_pg_sz	= 0;
 	caps->cqe_buf_pg_sz	= 0;
 	caps->cqe_hop_num	= HNS_ROCE_CQE_HOP_NUM;
+	caps->eqe_ba_pg_sz	= 0;
+	caps->eqe_buf_pg_sz	= 0;
+	caps->eqe_hop_num	= HNS_ROCE_EQE_HOP_NUM;
 	caps->chunk_sz		= HNS_ROCE_V2_TABLE_CHUNK_SIZE;
 
 	caps->flags		= HNS_ROCE_CAP_FLAG_REREG_MR |
-				  HNS_ROCE_CAP_FLAG_ROCE_V1_V2;
+				  HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
+				  HNS_ROCE_CAP_FLAG_RQ_INLINE;
 	caps->pkey_table_len[0] = 1;
 	caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
+	caps->ceqe_depth	= HNS_ROCE_V2_COMP_EQE_NUM;
+	caps->aeqe_depth	= HNS_ROCE_V2_ASYNC_EQE_NUM;
 	caps->local_ca_ack_delay = 0;
 	caps->max_mtu = IB_MTU_4096;
 
@@ -1382,6 +1594,8 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
 
 	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
 		       V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
+	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
+		       V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
 	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
 		       V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
 	roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
@@ -1422,6 +1636,15 @@ static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
 
 	roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
 		       V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
+
+	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
+		       V2_CQC_BYTE_56_CQ_MAX_CNT_M,
+		       V2_CQC_BYTE_56_CQ_MAX_CNT_S,
+		       HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
+	roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
+		       V2_CQC_BYTE_56_CQ_PERIOD_M,
+		       V2_CQC_BYTE_56_CQ_PERIOD_S,
+		       HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
 }
 
 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
@@ -1457,6 +1680,40 @@ static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
 	return 0;
 }
 
+static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
+						    struct hns_roce_qp **cur_qp,
+						    struct ib_wc *wc)
+{
+	struct hns_roce_rinl_sge *sge_list;
+	u32 wr_num, wr_cnt, sge_num;
+	u32 sge_cnt, data_len, size;
+	void *wqe_buf;
+
+	wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
+				V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
+	wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
+
+	sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
+	sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
+	wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
+	data_len = wc->byte_len;
+
+	for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
+		size = min(sge_list[sge_cnt].len, data_len);
+		memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
+
+		data_len -= size;
+		wqe_buf += size;
+	}
+
+	if (data_len) {
+		wc->status = IB_WC_LOC_LEN_ERR;
+		return -EAGAIN;
+	}
+
+	return 0;
+}
+
 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 				struct hns_roce_qp **cur_qp, struct ib_wc *wc)
 {
@@ -1469,6 +1726,7 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 	u32 opcode;
 	u32 status;
 	int qpn;
+	int ret;
 
 	/* Find cqe according to consumer index */
 	cqe = next_cqe_sw_v2(hr_cq);
@@ -1636,7 +1894,7 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 		case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
 			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
 			wc->wc_flags = IB_WC_WITH_IMM;
-			wc->ex.imm_data = le32_to_cpu(cqe->rkey_immtdata);
+			wc->ex.imm_data = cqe->immtdata;
 			break;
 		case HNS_ROCE_V2_OPCODE_SEND:
 			wc->opcode = IB_WC_RECV;
@@ -1645,18 +1903,29 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 		case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
 			wc->opcode = IB_WC_RECV;
 			wc->wc_flags = IB_WC_WITH_IMM;
-			wc->ex.imm_data = le32_to_cpu(cqe->rkey_immtdata);
+			wc->ex.imm_data = cqe->immtdata;
 			break;
 		case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
 			wc->opcode = IB_WC_RECV;
 			wc->wc_flags = IB_WC_WITH_INVALIDATE;
-			wc->ex.invalidate_rkey = cqe->rkey_immtdata;
+			wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
 			break;
 		default:
 			wc->status = IB_WC_GENERAL_ERR;
 			break;
 		}
 
+		if ((wc->qp->qp_type == IB_QPT_RC ||
+		     wc->qp->qp_type == IB_QPT_UC) &&
+		    (opcode == HNS_ROCE_V2_OPCODE_SEND ||
+		    opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
+		    opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
+		    (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
+			ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
+			if (ret)
+				return -EAGAIN;
+		}
+
 		/* Update tail pointer, record wr_id */
 		wq = &(*cur_qp)->rq;
 		wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
@@ -1670,6 +1939,21 @@ static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
 		wc->wc_flags |= (roce_get_bit(cqe->byte_32,
 					      V2_CQE_BYTE_32_GRH_S) ?
 					      IB_WC_GRH : 0);
+		wc->port_num = roce_get_field(cqe->byte_32,
+				V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
+		wc->pkey_index = 0;
+		memcpy(wc->smac, cqe->smac, 4);
+		wc->smac[4] = roce_get_field(cqe->byte_28,
+					     V2_CQE_BYTE_28_SMAC_4_M,
+					     V2_CQE_BYTE_28_SMAC_4_S);
+		wc->smac[5] = roce_get_field(cqe->byte_28,
+					     V2_CQE_BYTE_28_SMAC_5_M,
+					     V2_CQE_BYTE_28_SMAC_5_S);
+		wc->vlan_id = 0xffff;
+		wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
+		wc->network_hdr_type = roce_get_field(cqe->byte_28,
+						    V2_CQE_BYTE_28_PORT_TYPE_M,
+						    V2_CQE_BYTE_28_PORT_TYPE_S);
 	}
 
 	return 0;
@@ -1859,8 +2143,39 @@ static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
 	return ret;
 }
 
+static void set_access_flags(struct hns_roce_qp *hr_qp,
+			     struct hns_roce_v2_qp_context *context,
+			     struct hns_roce_v2_qp_context *qpc_mask,
+			     const struct ib_qp_attr *attr, int attr_mask)
+{
+	u8 dest_rd_atomic;
+	u32 access_flags;
+
+	dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
+			 attr->max_dest_rd_atomic : hr_qp->resp_depth;
+
+	access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
+		       attr->qp_access_flags : hr_qp->atomic_rd_en;
+
+	if (!dest_rd_atomic)
+		access_flags &= IB_ACCESS_REMOTE_WRITE;
+
+	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
+		     !!(access_flags & IB_ACCESS_REMOTE_READ));
+	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
+
+	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
+		     !!(access_flags & IB_ACCESS_REMOTE_WRITE));
+	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
+
+	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
+		     !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
+	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
+}
+
 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
 				    const struct ib_qp_attr *attr,
+				    int attr_mask,
 				    struct hns_roce_v2_qp_context *context,
 				    struct hns_roce_v2_qp_context *qpc_mask)
 {
@@ -1877,9 +2192,18 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp,
 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
 		       V2_QPC_BYTE_4_TST_S, 0);
 
-	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
-		       V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
-		       ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
+	if (ibqp->qp_type == IB_QPT_GSI)
+		roce_set_field(context->byte_4_sqpn_tst,
+			       V2_QPC_BYTE_4_SGE_SHIFT_M,
+			       V2_QPC_BYTE_4_SGE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->sge.sge_cnt));
+	else
+		roce_set_field(context->byte_4_sqpn_tst,
+			       V2_QPC_BYTE_4_SGE_SHIFT_M,
+			       V2_QPC_BYTE_4_SGE_SHIFT_S,
+			       hr_qp->sq.max_gs > 2 ?
+			       ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
+
 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
 		       V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
 
@@ -1944,18 +2268,13 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp,
 	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
 	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
 
-	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
-		     !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
-	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
-
-	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
-		     !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE));
-	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
-
-	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
-		     !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC));
-	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
+	if (attr_mask & IB_QP_QKEY) {
+		context->qkey_xrcd = attr->qkey;
+		qpc_mask->qkey_xrcd = 0;
+		hr_qp->qkey = attr->qkey;
+	}
 
+	roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 1);
 	roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
 
 	roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
@@ -2176,9 +2495,17 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp,
 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
 		       V2_QPC_BYTE_4_TST_S, 0);
 
-	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
-		       V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
-		       ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
+	if (ibqp->qp_type == IB_QPT_GSI)
+		roce_set_field(context->byte_4_sqpn_tst,
+			       V2_QPC_BYTE_4_SGE_SHIFT_M,
+			       V2_QPC_BYTE_4_SGE_SHIFT_S,
+			       ilog2((unsigned int)hr_qp->sge.sge_cnt));
+	else
+		roce_set_field(context->byte_4_sqpn_tst,
+			       V2_QPC_BYTE_4_SGE_SHIFT_M,
+			       V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
+			       ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
+
 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
 		       V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
 
@@ -2239,7 +2566,7 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp,
 		       V2_QPC_BYTE_80_RX_CQN_S, 0);
 
 	roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
-		       V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
+		       V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
 	roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
 		       V2_QPC_BYTE_252_TX_CQN_S, 0);
 
@@ -2255,10 +2582,10 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp,
 			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
 	}
 
-	if (attr_mask & IB_QP_PKEY_INDEX)
-		context->qkey_xrcd = attr->pkey_index;
-	else
-		context->qkey_xrcd = hr_qp->pkey_index;
+	if (attr_mask & IB_QP_QKEY) {
+		context->qkey_xrcd = attr->qkey;
+		qpc_mask->qkey_xrcd = 0;
+	}
 
 	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
 		       V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
@@ -2354,7 +2681,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(context->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_S,
-		       hr_qp->sq.max_gs > 2 ? hr_dev->caps.mtt_hop_num : 0);
+		       ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
+		       hr_dev->caps.mtt_hop_num : 0);
 	roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_M,
 		       V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
@@ -2463,11 +2791,14 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 		roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
 	}
 
-	roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
-		       V2_QPC_BYTE_140_RR_MAX_S,
-		       ilog2((unsigned int)attr->max_dest_rd_atomic));
-	roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
-		       V2_QPC_BYTE_140_RR_MAX_S, 0);
+	if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
+	     attr->max_dest_rd_atomic) {
+		roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
+			       V2_QPC_BYTE_140_RR_MAX_S,
+			       fls(attr->max_dest_rd_atomic - 1));
+		roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
+			       V2_QPC_BYTE_140_RR_MAX_S, 0);
+	}
 
 	roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
 		       V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
@@ -2511,8 +2842,13 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
 		       V2_QPC_BYTE_24_TC_S, 0);
 
-	roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
-		       V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
+	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
+		roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
+			       V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
+	else
+		roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
+			       V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
+
 	roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
 		       V2_QPC_BYTE_24_MTU_S, 0);
 
@@ -2557,12 +2893,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 		       V2_QPC_BYTE_168_LP_SGEN_INI_M,
 		       V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
 
-	roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
-		       V2_QPC_BYTE_208_SR_MAX_S,
-		       ilog2((unsigned int)attr->max_rd_atomic));
-	roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
-		       V2_QPC_BYTE_208_SR_MAX_S, 0);
-
 	roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
 		       V2_QPC_BYTE_28_SL_S, rdma_ah_get_sl(&attr->ah_attr));
 	roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
@@ -2625,13 +2955,14 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 		       V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
 
 	page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
-	context->sq_cur_sge_blk_addr = hr_qp->sq.max_gs > 2 ?
+	context->sq_cur_sge_blk_addr =
+		       ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
 				      ((u32)(mtts[hr_qp->sge.offset / page_size]
 				      >> PAGE_ADDR_SHIFT)) : 0;
 	roce_set_field(context->byte_184_irrl_idx,
 		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
 		       V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
-		       hr_qp->sq.max_gs > 2 ?
+		       ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
 		       (mtts[hr_qp->sge.offset / page_size] >>
 		       (32 + PAGE_ADDR_SHIFT)) : 0);
 	qpc_mask->sq_cur_sge_blk_addr = 0;
@@ -2766,6 +3097,14 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 	roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
 		       V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
 
+	if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
+		roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
+			       V2_QPC_BYTE_208_SR_MAX_S,
+			       fls(attr->max_rd_atomic - 1));
+		roce_set_field(qpc_mask->byte_208_irrl,
+			       V2_QPC_BYTE_208_SR_MAX_M,
+			       V2_QPC_BYTE_208_SR_MAX_S, 0);
+	}
 	return 0;
 }
 
@@ -2794,7 +3133,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 	 */
 	memset(qpc_mask, 0xff, sizeof(*qpc_mask));
 	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
-		modify_qp_reset_to_init(ibqp, attr, context, qpc_mask);
+		modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
+					qpc_mask);
 	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
 		modify_qp_init_to_init(ibqp, attr, attr_mask, context,
 				       qpc_mask);
@@ -2829,6 +3169,9 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 		goto out;
 	}
 
+	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
+		set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
+
 	/* Every status migrate must change state */
 	roce_set_field(context->byte_60_qpst_mapid, V2_QPC_BYTE_60_QP_ST_M,
 		       V2_QPC_BYTE_60_QP_ST_S, new_state);
@@ -2845,6 +3188,9 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 
 	hr_qp->state = new_state;
 
+	if (attr_mask & IB_QP_ACCESS_FLAGS)
+		hr_qp->atomic_rd_en = attr->qp_access_flags;
+
 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
 		hr_qp->resp_depth = attr->max_dest_rd_atomic;
 	if (attr_mask & IB_QP_PORT) {
@@ -3098,6 +3444,11 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
 	}
 
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
+		kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
+		kfree(hr_qp->rq_inl_buf.wqe_list);
+	}
+
 	return 0;
 }
 
@@ -3162,6 +3513,1146 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 	return ret;
 }
 
+static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
+{
+	u32 doorbell[2];
+
+	doorbell[0] = 0;
+	doorbell[1] = 0;
+
+	if (eq->type_flag == HNS_ROCE_AEQ) {
+		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
+			       HNS_ROCE_V2_EQ_DB_CMD_S,
+			       eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
+			       HNS_ROCE_EQ_DB_CMD_AEQ :
+			       HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
+	} else {
+		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
+			       HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
+
+		roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
+			       HNS_ROCE_V2_EQ_DB_CMD_S,
+			       eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
+			       HNS_ROCE_EQ_DB_CMD_CEQ :
+			       HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
+	}
+
+	roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
+		       HNS_ROCE_V2_EQ_DB_PARA_S,
+		       (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
+
+	hns_roce_write64_k(doorbell, eq->doorbell);
+}
+
+static void hns_roce_v2_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
+						  struct hns_roce_aeqe *aeqe,
+						  u32 qpn)
+{
+	struct device *dev = hr_dev->dev;
+	int sub_type;
+
+	dev_warn(dev, "Local work queue catastrophic error.\n");
+	sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M,
+				  HNS_ROCE_V2_AEQE_SUB_TYPE_S);
+	switch (sub_type) {
+	case HNS_ROCE_LWQCE_QPC_ERROR:
+		dev_warn(dev, "QP %d, QPC error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_MTU_ERROR:
+		dev_warn(dev, "QP %d, MTU error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
+		dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
+		dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
+		break;
+	case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
+		dev_warn(dev, "QP %d, WQE shift error.\n", qpn);
+		break;
+	default:
+		dev_err(dev, "Unhandled sub_event type %d.\n", sub_type);
+		break;
+	}
+}
+
+static void hns_roce_v2_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
+					    struct hns_roce_aeqe *aeqe, u32 qpn)
+{
+	struct device *dev = hr_dev->dev;
+	int sub_type;
+
+	dev_warn(dev, "Local access violation work queue error.\n");
+	sub_type = roce_get_field(aeqe->asyn, HNS_ROCE_V2_AEQE_SUB_TYPE_M,
+				  HNS_ROCE_V2_AEQE_SUB_TYPE_S);
+	switch (sub_type) {
+	case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
+		dev_warn(dev, "QP %d, R_key violation.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_LENGTH_ERROR:
+		dev_warn(dev, "QP %d, length error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_VA_ERROR:
+		dev_warn(dev, "QP %d, VA error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_PD_ERROR:
+		dev_err(dev, "QP %d, PD error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
+		dev_warn(dev, "QP %d, rw acc error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
+		dev_warn(dev, "QP %d, key state error.\n", qpn);
+		break;
+	case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
+		dev_warn(dev, "QP %d, MR operation error.\n", qpn);
+		break;
+	default:
+		dev_err(dev, "Unhandled sub_event type %d.\n", sub_type);
+		break;
+	}
+}
+
+static void hns_roce_v2_qp_err_handle(struct hns_roce_dev *hr_dev,
+				      struct hns_roce_aeqe *aeqe,
+				      int event_type)
+{
+	struct device *dev = hr_dev->dev;
+	u32 qpn;
+
+	qpn = roce_get_field(aeqe->event.qp_event.qp,
+			     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
+			     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
+
+	switch (event_type) {
+	case HNS_ROCE_EVENT_TYPE_COMM_EST:
+		dev_warn(dev, "Communication established.\n");
+		break;
+	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
+		dev_warn(dev, "Send queue drained.\n");
+		break;
+	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+		hns_roce_v2_wq_catas_err_handle(hr_dev, aeqe, qpn);
+		break;
+	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
+		dev_warn(dev, "Invalid request local work queue error.\n");
+		break;
+	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
+		hns_roce_v2_local_wq_access_err_handle(hr_dev, aeqe, qpn);
+		break;
+	default:
+		break;
+	}
+
+	hns_roce_qp_event(hr_dev, qpn, event_type);
+}
+
+static void hns_roce_v2_cq_err_handle(struct hns_roce_dev *hr_dev,
+				      struct hns_roce_aeqe *aeqe,
+				      int event_type)
+{
+	struct device *dev = hr_dev->dev;
+	u32 cqn;
+
+	cqn = roce_get_field(aeqe->event.cq_event.cq,
+			     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
+			     HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
+
+	switch (event_type) {
+	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
+		dev_warn(dev, "CQ 0x%x access err.\n", cqn);
+		break;
+	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
+		dev_warn(dev, "CQ 0x%x overflow\n", cqn);
+		break;
+	default:
+		break;
+	}
+
+	hns_roce_cq_event(hr_dev, cqn, event_type);
+}
+
+static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)
+{
+	u32 buf_chk_sz;
+	unsigned long off;
+
+	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
+	off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
+
+	return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) +
+		off % buf_chk_sz);
+}
+
+static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)
+{
+	u32 buf_chk_sz;
+	unsigned long off;
+
+	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
+
+	off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
+
+	if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
+		return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) +
+			off % buf_chk_sz);
+	else
+		return (struct hns_roce_aeqe *)((u8 *)
+			(eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
+}
+
+static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
+{
+	struct hns_roce_aeqe *aeqe;
+
+	if (!eq->hop_num)
+		aeqe = get_aeqe_v2(eq, eq->cons_index);
+	else
+		aeqe = mhop_get_aeqe(eq, eq->cons_index);
+
+	return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
+		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
+}
+
+static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_eq *eq)
+{
+	struct device *dev = hr_dev->dev;
+	struct hns_roce_aeqe *aeqe;
+	int aeqe_found = 0;
+	int event_type;
+
+	while ((aeqe = next_aeqe_sw_v2(eq))) {
+
+		/* Make sure we read AEQ entry after we have checked the
+		 * ownership bit
+		 */
+		dma_rmb();
+
+		event_type = roce_get_field(aeqe->asyn,
+					    HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
+					    HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
+
+		switch (event_type) {
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
+			dev_warn(dev, "Path migrated succeeded.\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
+			dev_warn(dev, "Path migration failed.\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_COMM_EST:
+		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
+		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
+		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
+		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
+			hns_roce_v2_qp_err_handle(hr_dev, aeqe, event_type);
+			break;
+		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
+		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
+		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
+			dev_warn(dev, "SRQ not support.\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
+		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
+			hns_roce_v2_cq_err_handle(hr_dev, aeqe, event_type);
+			break;
+		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
+			dev_warn(dev, "DB overflow.\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_MB:
+			hns_roce_cmd_event(hr_dev,
+					le16_to_cpu(aeqe->event.cmd.token),
+					aeqe->event.cmd.status,
+					le64_to_cpu(aeqe->event.cmd.out_param));
+			break;
+		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
+			dev_warn(dev, "CEQ overflow.\n");
+			break;
+		case HNS_ROCE_EVENT_TYPE_FLR:
+			dev_warn(dev, "Function level reset.\n");
+			break;
+		default:
+			dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
+				event_type, eq->eqn, eq->cons_index);
+			break;
+		};
+
+		++eq->cons_index;
+		aeqe_found = 1;
+
+		if (eq->cons_index > (2 * eq->entries - 1)) {
+			dev_warn(dev, "cons_index overflow, set back to 0.\n");
+			eq->cons_index = 0;
+		}
+	}
+
+	set_eq_cons_index_v2(eq);
+	return aeqe_found;
+}
+
+static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)
+{
+	u32 buf_chk_sz;
+	unsigned long off;
+
+	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
+	off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
+
+	return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) +
+		off % buf_chk_sz);
+}
+
+static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)
+{
+	u32 buf_chk_sz;
+	unsigned long off;
+
+	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
+
+	off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
+
+	if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
+		return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) +
+			off % buf_chk_sz);
+	else
+		return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off /
+			buf_chk_sz]) + off % buf_chk_sz);
+}
+
+static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
+{
+	struct hns_roce_ceqe *ceqe;
+
+	if (!eq->hop_num)
+		ceqe = get_ceqe_v2(eq, eq->cons_index);
+	else
+		ceqe = mhop_get_ceqe(eq, eq->cons_index);
+
+	return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
+		(!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
+}
+
+static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
+			       struct hns_roce_eq *eq)
+{
+	struct device *dev = hr_dev->dev;
+	struct hns_roce_ceqe *ceqe;
+	int ceqe_found = 0;
+	u32 cqn;
+
+	while ((ceqe = next_ceqe_sw_v2(eq))) {
+
+		/* Make sure we read CEQ entry after we have checked the
+		 * ownership bit
+		 */
+		dma_rmb();
+
+		cqn = roce_get_field(ceqe->comp,
+				     HNS_ROCE_V2_CEQE_COMP_CQN_M,
+				     HNS_ROCE_V2_CEQE_COMP_CQN_S);
+
+		hns_roce_cq_completion(hr_dev, cqn);
+
+		++eq->cons_index;
+		ceqe_found = 1;
+
+		if (eq->cons_index > (2 * eq->entries - 1)) {
+			dev_warn(dev, "cons_index overflow, set back to 0.\n");
+			eq->cons_index = 0;
+		}
+	}
+
+	set_eq_cons_index_v2(eq);
+
+	return ceqe_found;
+}
+
+static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
+{
+	struct hns_roce_eq *eq = eq_ptr;
+	struct hns_roce_dev *hr_dev = eq->hr_dev;
+	int int_work = 0;
+
+	if (eq->type_flag == HNS_ROCE_CEQ)
+		/* Completion event interrupt */
+		int_work = hns_roce_v2_ceq_int(hr_dev, eq);
+	else
+		/* Asychronous event interrupt */
+		int_work = hns_roce_v2_aeq_int(hr_dev, eq);
+
+	return IRQ_RETVAL(int_work);
+}
+
+static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
+{
+	struct hns_roce_dev *hr_dev = dev_id;
+	struct device *dev = hr_dev->dev;
+	int int_work = 0;
+	u32 int_st;
+	u32 int_en;
+
+	/* Abnormal interrupt */
+	int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
+	int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
+
+	if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
+		dev_err(dev, "AEQ overflow!\n");
+
+		roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
+
+		roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
+
+		int_work = 1;
+	} else if (roce_get_bit(int_st,	HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
+		dev_err(dev, "BUS ERR!\n");
+
+		roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
+
+		roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
+
+		int_work = 1;
+	} else if (roce_get_bit(int_st,	HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
+		dev_err(dev, "OTHER ERR!\n");
+
+		roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
+
+		roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
+
+		int_work = 1;
+	} else
+		dev_err(dev, "There is no abnormal irq found!\n");
+
+	return IRQ_RETVAL(int_work);
+}
+
+static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
+					int eq_num, int enable_flag)
+{
+	int i;
+
+	if (enable_flag == EQ_ENABLE) {
+		for (i = 0; i < eq_num; i++)
+			roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
+				   i * EQ_REG_OFFSET,
+				   HNS_ROCE_V2_VF_EVENT_INT_EN_M);
+
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
+			   HNS_ROCE_V2_VF_ABN_INT_EN_M);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
+			   HNS_ROCE_V2_VF_ABN_INT_CFG_M);
+	} else {
+		for (i = 0; i < eq_num; i++)
+			roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
+				   i * EQ_REG_OFFSET,
+				   HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
+
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
+			   HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
+		roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
+			   HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
+	}
+}
+
+static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
+{
+	struct device *dev = hr_dev->dev;
+	int ret;
+
+	if (eqn < hr_dev->caps.num_comp_vectors)
+		ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
+					0, HNS_ROCE_CMD_DESTROY_CEQC,
+					HNS_ROCE_CMD_TIMEOUT_MSECS);
+	else
+		ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
+					0, HNS_ROCE_CMD_DESTROY_AEQC,
+					HNS_ROCE_CMD_TIMEOUT_MSECS);
+	if (ret)
+		dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
+}
+
+static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
+				  struct hns_roce_eq *eq)
+{
+	struct device *dev = hr_dev->dev;
+	u64 idx;
+	u64 size;
+	u32 buf_chk_sz;
+	u32 bt_chk_sz;
+	u32 mhop_num;
+	int eqe_alloc;
+	int ba_num;
+	int i = 0;
+	int j = 0;
+
+	mhop_num = hr_dev->caps.eqe_hop_num;
+	buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
+	bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
+	ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1) /
+		 buf_chk_sz;
+
+	/* hop_num = 0 */
+	if (mhop_num == HNS_ROCE_HOP_NUM_0) {
+		dma_free_coherent(dev, (unsigned int)(eq->entries *
+				  eq->eqe_size), eq->bt_l0, eq->l0_dma);
+		return;
+	}
+
+	/* hop_num = 1 or hop = 2 */
+	dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
+	if (mhop_num == 1) {
+		for (i = 0; i < eq->l0_last_num; i++) {
+			if (i == eq->l0_last_num - 1) {
+				eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
+				size = (eq->entries - eqe_alloc) * eq->eqe_size;
+				dma_free_coherent(dev, size, eq->buf[i],
+						  eq->buf_dma[i]);
+				break;
+			}
+			dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
+					  eq->buf_dma[i]);
+		}
+	} else if (mhop_num == 2) {
+		for (i = 0; i < eq->l0_last_num; i++) {
+			dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
+					  eq->l1_dma[i]);
+
+			for (j = 0; j < bt_chk_sz / 8; j++) {
+				idx = i * (bt_chk_sz / 8) + j;
+				if ((i == eq->l0_last_num - 1)
+				     && j == eq->l1_last_num - 1) {
+					eqe_alloc = (buf_chk_sz / eq->eqe_size)
+						    * idx;
+					size = (eq->entries - eqe_alloc)
+						* eq->eqe_size;
+					dma_free_coherent(dev, size,
+							  eq->buf[idx],
+							  eq->buf_dma[idx]);
+					break;
+				}
+				dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
+						  eq->buf_dma[idx]);
+			}
+		}
+	}
+	kfree(eq->buf_dma);
+	kfree(eq->buf);
+	kfree(eq->l1_dma);
+	kfree(eq->bt_l1);
+	eq->buf_dma = NULL;
+	eq->buf = NULL;
+	eq->l1_dma = NULL;
+	eq->bt_l1 = NULL;
+}
+
+static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
+				struct hns_roce_eq *eq)
+{
+	u32 buf_chk_sz;
+
+	buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
+
+	if (hr_dev->caps.eqe_hop_num) {
+		hns_roce_mhop_free_eq(hr_dev, eq);
+		return;
+	}
+
+	if (eq->buf_list)
+		dma_free_coherent(hr_dev->dev, buf_chk_sz,
+				  eq->buf_list->buf, eq->buf_list->map);
+}
+
+static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
+				struct hns_roce_eq *eq,
+				void *mb_buf)
+{
+	struct hns_roce_eq_context *eqc;
+
+	eqc = mb_buf;
+	memset(eqc, 0, sizeof(struct hns_roce_eq_context));
+
+	/* init eqc */
+	eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
+	eq->hop_num = hr_dev->caps.eqe_hop_num;
+	eq->cons_index = 0;
+	eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
+	eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
+	eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
+	eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
+	eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
+	eq->shift = ilog2((unsigned int)eq->entries);
+
+	if (!eq->hop_num)
+		eq->eqe_ba = eq->buf_list->map;
+	else
+		eq->eqe_ba = eq->l0_dma;
+
+	/* set eqc state */
+	roce_set_field(eqc->byte_4,
+		       HNS_ROCE_EQC_EQ_ST_M,
+		       HNS_ROCE_EQC_EQ_ST_S,
+		       HNS_ROCE_V2_EQ_STATE_VALID);
+
+	/* set eqe hop num */
+	roce_set_field(eqc->byte_4,
+		       HNS_ROCE_EQC_HOP_NUM_M,
+		       HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
+
+	/* set eqc over_ignore */
+	roce_set_field(eqc->byte_4,
+		       HNS_ROCE_EQC_OVER_IGNORE_M,
+		       HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
+
+	/* set eqc coalesce */
+	roce_set_field(eqc->byte_4,
+		       HNS_ROCE_EQC_COALESCE_M,
+		       HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
+
+	/* set eqc arm_state */
+	roce_set_field(eqc->byte_4,
+		       HNS_ROCE_EQC_ARM_ST_M,
+		       HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
+
+	/* set eqn */
+	roce_set_field(eqc->byte_4,
+		       HNS_ROCE_EQC_EQN_M,
+		       HNS_ROCE_EQC_EQN_S, eq->eqn);
+
+	/* set eqe_cnt */
+	roce_set_field(eqc->byte_4,
+		       HNS_ROCE_EQC_EQE_CNT_M,
+		       HNS_ROCE_EQC_EQE_CNT_S,
+		       HNS_ROCE_EQ_INIT_EQE_CNT);
+
+	/* set eqe_ba_pg_sz */
+	roce_set_field(eqc->byte_8,
+		       HNS_ROCE_EQC_BA_PG_SZ_M,
+		       HNS_ROCE_EQC_BA_PG_SZ_S, eq->eqe_ba_pg_sz);
+
+	/* set eqe_buf_pg_sz */
+	roce_set_field(eqc->byte_8,
+		       HNS_ROCE_EQC_BUF_PG_SZ_M,
+		       HNS_ROCE_EQC_BUF_PG_SZ_S, eq->eqe_buf_pg_sz);
+
+	/* set eq_producer_idx */
+	roce_set_field(eqc->byte_8,
+		       HNS_ROCE_EQC_PROD_INDX_M,
+		       HNS_ROCE_EQC_PROD_INDX_S,
+		       HNS_ROCE_EQ_INIT_PROD_IDX);
+
+	/* set eq_max_cnt */
+	roce_set_field(eqc->byte_12,
+		       HNS_ROCE_EQC_MAX_CNT_M,
+		       HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
+
+	/* set eq_period */
+	roce_set_field(eqc->byte_12,
+		       HNS_ROCE_EQC_PERIOD_M,
+		       HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
+
+	/* set eqe_report_timer */
+	roce_set_field(eqc->eqe_report_timer,
+		       HNS_ROCE_EQC_REPORT_TIMER_M,
+		       HNS_ROCE_EQC_REPORT_TIMER_S,
+		       HNS_ROCE_EQ_INIT_REPORT_TIMER);
+
+	/* set eqe_ba [34:3] */
+	roce_set_field(eqc->eqe_ba0,
+		       HNS_ROCE_EQC_EQE_BA_L_M,
+		       HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
+
+	/* set eqe_ba [64:35] */
+	roce_set_field(eqc->eqe_ba1,
+		       HNS_ROCE_EQC_EQE_BA_H_M,
+		       HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
+
+	/* set eq shift */
+	roce_set_field(eqc->byte_28,
+		       HNS_ROCE_EQC_SHIFT_M,
+		       HNS_ROCE_EQC_SHIFT_S, eq->shift);
+
+	/* set eq MSI_IDX */
+	roce_set_field(eqc->byte_28,
+		       HNS_ROCE_EQC_MSI_INDX_M,
+		       HNS_ROCE_EQC_MSI_INDX_S,
+		       HNS_ROCE_EQ_INIT_MSI_IDX);
+
+	/* set cur_eqe_ba [27:12] */
+	roce_set_field(eqc->byte_28,
+		       HNS_ROCE_EQC_CUR_EQE_BA_L_M,
+		       HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
+
+	/* set cur_eqe_ba [59:28] */
+	roce_set_field(eqc->byte_32,
+		       HNS_ROCE_EQC_CUR_EQE_BA_M_M,
+		       HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
+
+	/* set cur_eqe_ba [63:60] */
+	roce_set_field(eqc->byte_36,
+		       HNS_ROCE_EQC_CUR_EQE_BA_H_M,
+		       HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
+
+	/* set eq consumer idx */
+	roce_set_field(eqc->byte_36,
+		       HNS_ROCE_EQC_CONS_INDX_M,
+		       HNS_ROCE_EQC_CONS_INDX_S,
+		       HNS_ROCE_EQ_INIT_CONS_IDX);
+
+	/* set nex_eqe_ba[43:12] */
+	roce_set_field(eqc->nxt_eqe_ba0,
+		       HNS_ROCE_EQC_NXT_EQE_BA_L_M,
+		       HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
+
+	/* set nex_eqe_ba[63:44] */
+	roce_set_field(eqc->nxt_eqe_ba1,
+		       HNS_ROCE_EQC_NXT_EQE_BA_H_M,
+		       HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
+}
+
+static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
+				  struct hns_roce_eq *eq)
+{
+	struct device *dev = hr_dev->dev;
+	int eq_alloc_done = 0;
+	int eq_buf_cnt = 0;
+	int eqe_alloc;
+	u32 buf_chk_sz;
+	u32 bt_chk_sz;
+	u32 mhop_num;
+	u64 size;
+	u64 idx;
+	int ba_num;
+	int bt_num;
+	int record_i;
+	int record_j;
+	int i = 0;
+	int j = 0;
+
+	mhop_num = hr_dev->caps.eqe_hop_num;
+	buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
+	bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
+
+	ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1)
+		  / buf_chk_sz;
+	bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8);
+
+	/* hop_num = 0 */
+	if (mhop_num == HNS_ROCE_HOP_NUM_0) {
+		if (eq->entries > buf_chk_sz / eq->eqe_size) {
+			dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
+				eq->entries);
+			return -EINVAL;
+		}
+		eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
+					       &(eq->l0_dma), GFP_KERNEL);
+		if (!eq->bt_l0)
+			return -ENOMEM;
+
+		eq->cur_eqe_ba = eq->l0_dma;
+		eq->nxt_eqe_ba = 0;
+
+		memset(eq->bt_l0, 0, eq->entries * eq->eqe_size);
+
+		return 0;
+	}
+
+	eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
+	if (!eq->buf_dma)
+		return -ENOMEM;
+	eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
+	if (!eq->buf)
+		goto err_kcalloc_buf;
+
+	if (mhop_num == 2) {
+		eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
+		if (!eq->l1_dma)
+			goto err_kcalloc_l1_dma;
+
+		eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
+		if (!eq->bt_l1)
+			goto err_kcalloc_bt_l1;
+	}
+
+	/* alloc L0 BT */
+	eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
+	if (!eq->bt_l0)
+		goto err_dma_alloc_l0;
+
+	if (mhop_num == 1) {
+		if (ba_num > (bt_chk_sz / 8))
+			dev_err(dev, "ba_num %d is too large for 1 hop\n",
+				ba_num);
+
+		/* alloc buf */
+		for (i = 0; i < bt_chk_sz / 8; i++) {
+			if (eq_buf_cnt + 1 < ba_num) {
+				size = buf_chk_sz;
+			} else {
+				eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
+				size = (eq->entries - eqe_alloc) * eq->eqe_size;
+			}
+			eq->buf[i] = dma_alloc_coherent(dev, size,
+							&(eq->buf_dma[i]),
+							GFP_KERNEL);
+			if (!eq->buf[i])
+				goto err_dma_alloc_buf;
+
+			memset(eq->buf[i], 0, size);
+			*(eq->bt_l0 + i) = eq->buf_dma[i];
+
+			eq_buf_cnt++;
+			if (eq_buf_cnt >= ba_num)
+				break;
+		}
+		eq->cur_eqe_ba = eq->buf_dma[0];
+		eq->nxt_eqe_ba = eq->buf_dma[1];
+
+	} else if (mhop_num == 2) {
+		/* alloc L1 BT and buf */
+		for (i = 0; i < bt_chk_sz / 8; i++) {
+			eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
+							  &(eq->l1_dma[i]),
+							  GFP_KERNEL);
+			if (!eq->bt_l1[i])
+				goto err_dma_alloc_l1;
+			*(eq->bt_l0 + i) = eq->l1_dma[i];
+
+			for (j = 0; j < bt_chk_sz / 8; j++) {
+				idx = i * bt_chk_sz / 8 + j;
+				if (eq_buf_cnt + 1 < ba_num) {
+					size = buf_chk_sz;
+				} else {
+					eqe_alloc = (buf_chk_sz / eq->eqe_size)
+						    * idx;
+					size = (eq->entries - eqe_alloc)
+						* eq->eqe_size;
+				}
+				eq->buf[idx] = dma_alloc_coherent(dev, size,
+							    &(eq->buf_dma[idx]),
+							    GFP_KERNEL);
+				if (!eq->buf[idx])
+					goto err_dma_alloc_buf;
+
+				memset(eq->buf[idx], 0, size);
+				*(eq->bt_l1[i] + j) = eq->buf_dma[idx];
+
+				eq_buf_cnt++;
+				if (eq_buf_cnt >= ba_num) {
+					eq_alloc_done = 1;
+					break;
+				}
+			}
+
+			if (eq_alloc_done)
+				break;
+		}
+		eq->cur_eqe_ba = eq->buf_dma[0];
+		eq->nxt_eqe_ba = eq->buf_dma[1];
+	}
+
+	eq->l0_last_num = i + 1;
+	if (mhop_num == 2)
+		eq->l1_last_num = j + 1;
+
+	return 0;
+
+err_dma_alloc_l1:
+	dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
+	eq->bt_l0 = NULL;
+	eq->l0_dma = 0;
+	for (i -= 1; i >= 0; i--) {
+		dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
+				  eq->l1_dma[i]);
+
+		for (j = 0; j < bt_chk_sz / 8; j++) {
+			idx = i * bt_chk_sz / 8 + j;
+			dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
+					  eq->buf_dma[idx]);
+		}
+	}
+	goto err_dma_alloc_l0;
+
+err_dma_alloc_buf:
+	dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
+	eq->bt_l0 = NULL;
+	eq->l0_dma = 0;
+
+	if (mhop_num == 1)
+		for (i -= 1; i >= 0; i--)
+			dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
+					  eq->buf_dma[i]);
+	else if (mhop_num == 2) {
+		record_i = i;
+		record_j = j;
+		for (; i >= 0; i--) {
+			dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
+					  eq->l1_dma[i]);
+
+			for (j = 0; j < bt_chk_sz / 8; j++) {
+				if (i == record_i && j >= record_j)
+					break;
+
+				idx = i * bt_chk_sz / 8 + j;
+				dma_free_coherent(dev, buf_chk_sz,
+						  eq->buf[idx],
+						  eq->buf_dma[idx]);
+			}
+		}
+	}
+
+err_dma_alloc_l0:
+	kfree(eq->bt_l1);
+	eq->bt_l1 = NULL;
+
+err_kcalloc_bt_l1:
+	kfree(eq->l1_dma);
+	eq->l1_dma = NULL;
+
+err_kcalloc_l1_dma:
+	kfree(eq->buf);
+	eq->buf = NULL;
+
+err_kcalloc_buf:
+	kfree(eq->buf_dma);
+	eq->buf_dma = NULL;
+
+	return -ENOMEM;
+}
+
+static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
+				 struct hns_roce_eq *eq,
+				 unsigned int eq_cmd)
+{
+	struct device *dev = hr_dev->dev;
+	struct hns_roce_cmd_mailbox *mailbox;
+	u32 buf_chk_sz = 0;
+	int ret;
+
+	/* Allocate mailbox memory */
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	if (!hr_dev->caps.eqe_hop_num) {
+		buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
+
+		eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
+				       GFP_KERNEL);
+		if (!eq->buf_list) {
+			ret = -ENOMEM;
+			goto free_cmd_mbox;
+		}
+
+		eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
+						       &(eq->buf_list->map),
+						       GFP_KERNEL);
+		if (!eq->buf_list->buf) {
+			ret = -ENOMEM;
+			goto err_alloc_buf;
+		}
+
+		memset(eq->buf_list->buf, 0, buf_chk_sz);
+	} else {
+		ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
+		if (ret) {
+			ret = -ENOMEM;
+			goto free_cmd_mbox;
+		}
+	}
+
+	hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
+
+	ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
+				eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
+	if (ret) {
+		dev_err(dev, "[mailbox cmd] creat eqc failed.\n");
+		goto err_cmd_mbox;
+	}
+
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return 0;
+
+err_cmd_mbox:
+	if (!hr_dev->caps.eqe_hop_num)
+		dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
+				  eq->buf_list->map);
+	else {
+		hns_roce_mhop_free_eq(hr_dev, eq);
+		goto free_cmd_mbox;
+	}
+
+err_alloc_buf:
+	kfree(eq->buf_list);
+
+free_cmd_mbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
+static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
+	struct device *dev = hr_dev->dev;
+	struct hns_roce_eq *eq;
+	unsigned int eq_cmd;
+	int irq_num;
+	int eq_num;
+	int other_num;
+	int comp_num;
+	int aeq_num;
+	int i, j, k;
+	int ret;
+
+	other_num = hr_dev->caps.num_other_vectors;
+	comp_num = hr_dev->caps.num_comp_vectors;
+	aeq_num = hr_dev->caps.num_aeq_vectors;
+
+	eq_num = comp_num + aeq_num;
+	irq_num = eq_num + other_num;
+
+	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
+	if (!eq_table->eq)
+		return -ENOMEM;
+
+	for (i = 0; i < irq_num; i++) {
+		hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
+					       GFP_KERNEL);
+		if (!hr_dev->irq_names[i]) {
+			ret = -ENOMEM;
+			goto err_failed_kzalloc;
+		}
+	}
+
+	/* create eq */
+	for (j = 0; j < eq_num; j++) {
+		eq = &eq_table->eq[j];
+		eq->hr_dev = hr_dev;
+		eq->eqn = j;
+		if (j < comp_num) {
+			/* CEQ */
+			eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
+			eq->type_flag = HNS_ROCE_CEQ;
+			eq->entries = hr_dev->caps.ceqe_depth;
+			eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
+			eq->irq = hr_dev->irq[j + other_num + aeq_num];
+			eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
+			eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
+		} else {
+			/* AEQ */
+			eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
+			eq->type_flag = HNS_ROCE_AEQ;
+			eq->entries = hr_dev->caps.aeqe_depth;
+			eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
+			eq->irq = hr_dev->irq[j - comp_num + other_num];
+			eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
+			eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
+		}
+
+		ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
+		if (ret) {
+			dev_err(dev, "eq create failed.\n");
+			goto err_create_eq_fail;
+		}
+	}
+
+	/* enable irq */
+	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
+
+	/* irq contains: abnormal + AEQ + CEQ*/
+	for (k = 0; k < irq_num; k++)
+		if (k < other_num)
+			snprintf((char *)hr_dev->irq_names[k],
+				 HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k);
+		else if (k < (other_num + aeq_num))
+			snprintf((char *)hr_dev->irq_names[k],
+				 HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
+				 k - other_num);
+		else
+			snprintf((char *)hr_dev->irq_names[k],
+				 HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
+				 k - other_num - aeq_num);
+
+	for (k = 0; k < irq_num; k++) {
+		if (k < other_num)
+			ret = request_irq(hr_dev->irq[k],
+					  hns_roce_v2_msix_interrupt_abn,
+					  0, hr_dev->irq_names[k], hr_dev);
+
+		else if (k < (other_num + comp_num))
+			ret = request_irq(eq_table->eq[k - other_num].irq,
+					  hns_roce_v2_msix_interrupt_eq,
+					  0, hr_dev->irq_names[k + aeq_num],
+					  &eq_table->eq[k - other_num]);
+		else
+			ret = request_irq(eq_table->eq[k - other_num].irq,
+					  hns_roce_v2_msix_interrupt_eq,
+					  0, hr_dev->irq_names[k - comp_num],
+					  &eq_table->eq[k - other_num]);
+		if (ret) {
+			dev_err(dev, "Request irq error!\n");
+			goto err_request_irq_fail;
+		}
+	}
+
+	return 0;
+
+err_request_irq_fail:
+	for (k -= 1; k >= 0; k--)
+		if (k < other_num)
+			free_irq(hr_dev->irq[k], hr_dev);
+		else
+			free_irq(eq_table->eq[k - other_num].irq,
+				 &eq_table->eq[k - other_num]);
+
+err_create_eq_fail:
+	for (j -= 1; j >= 0; j--)
+		hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]);
+
+err_failed_kzalloc:
+	for (i -= 1; i >= 0; i--)
+		kfree(hr_dev->irq_names[i]);
+	kfree(eq_table->eq);
+
+	return ret;
+}
+
+static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
+	int irq_num;
+	int eq_num;
+	int i;
+
+	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
+	irq_num = eq_num + hr_dev->caps.num_other_vectors;
+
+	/* Disable irq */
+	hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
+
+	for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
+		free_irq(hr_dev->irq[i], hr_dev);
+
+	for (i = 0; i < eq_num; i++) {
+		hns_roce_v2_destroy_eqc(hr_dev, i);
+
+		free_irq(eq_table->eq[i].irq, &eq_table->eq[i]);
+
+		hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
+	}
+
+	for (i = 0; i < irq_num; i++)
+		kfree(hr_dev->irq_names[i]);
+
+	kfree(eq_table->eq);
+}
+
 static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.cmq_init = hns_roce_v2_cmq_init,
 	.cmq_exit = hns_roce_v2_cmq_exit,
@@ -3183,6 +4674,8 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.post_recv = hns_roce_v2_post_recv,
 	.req_notify_cq = hns_roce_v2_req_notify_cq,
 	.poll_cq = hns_roce_v2_poll_cq,
+	.init_eq = hns_roce_v2_init_eq_table,
+	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
 };
 
 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
@@ -3197,6 +4690,7 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
 				  struct hnae3_handle *handle)
 {
 	const struct pci_device_id *id;
+	int i;
 
 	id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
 	if (!id) {
@@ -3214,8 +4708,15 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
 	hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
 	hr_dev->iboe.phy_port[0] = 0;
 
+	addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
+			    hr_dev->iboe.netdevs[0]->dev_addr);
+
+	for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
+		hr_dev->irq[i] = pci_irq_vector(handle->pdev,
+						i + handle->rinfo.base_vector);
+
 	/* cmd issue mode: 0 is poll, 1 is event */
-	hr_dev->cmd_mod = 0;
+	hr_dev->cmd_mod = 1;
 	hr_dev->loop_idc = 0;
 
 	return 0;
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index 04b7a51b8efb..2bf8a47e3de3 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -53,6 +53,10 @@
 #define HNS_ROCE_V2_MAX_SQ_INLINE		0x20
 #define HNS_ROCE_V2_UAR_NUM			256
 #define HNS_ROCE_V2_PHY_UAR_NUM			1
+#define HNS_ROCE_V2_MAX_IRQ_NUM			65
+#define HNS_ROCE_V2_COMP_VEC_NUM		63
+#define HNS_ROCE_V2_AEQE_VEC_NUM		1
+#define HNS_ROCE_V2_ABNORMAL_VEC_NUM		1
 #define HNS_ROCE_V2_MAX_MTPT_NUM		0x8000
 #define HNS_ROCE_V2_MAX_MTT_SEGS		0x1000000
 #define HNS_ROCE_V2_MAX_CQE_SEGS		0x1000000
@@ -78,6 +82,8 @@
 #define HNS_ROCE_MTT_HOP_NUM			1
 #define HNS_ROCE_CQE_HOP_NUM			1
 #define HNS_ROCE_PBL_HOP_NUM			2
+#define HNS_ROCE_EQE_HOP_NUM			2
+
 #define HNS_ROCE_V2_GID_INDEX_NUM		256
 
 #define HNS_ROCE_V2_TABLE_CHUNK_SIZE		(1 << 18)
@@ -105,6 +111,12 @@
 	(step_idx == 1 && hop_num == 1) || \
 	(step_idx == 2 && hop_num == 2))
 
+enum {
+	NO_ARMED = 0x0,
+	REG_NXT_CEQE = 0x2,
+	REG_NXT_SE_CEQE = 0x3
+};
+
 #define V2_CQ_DB_REQ_NOT_SOL			0
 #define V2_CQ_DB_REQ_NOT			1
 
@@ -212,23 +224,26 @@ enum hns_roce_sgid_type {
 };
 
 struct hns_roce_v2_cq_context {
-	u32	byte_4_pg_ceqn;
-	u32	byte_8_cqn;
-	u32	cqe_cur_blk_addr;
-	u32	byte_16_hop_addr;
-	u32	cqe_nxt_blk_addr;
-	u32	byte_24_pgsz_addr;
-	u32	byte_28_cq_pi;
-	u32	byte_32_cq_ci;
-	u32	cqe_ba;
-	u32	byte_40_cqe_ba;
-	u32	byte_44_db_record;
-	u32	db_record_addr;
-	u32	byte_52_cqe_cnt;
-	u32	byte_56_cqe_period_maxcnt;
-	u32	cqe_report_timer;
-	u32	byte_64_se_cqe_idx;
+	__le32	byte_4_pg_ceqn;
+	__le32	byte_8_cqn;
+	__le32	cqe_cur_blk_addr;
+	__le32	byte_16_hop_addr;
+	__le32	cqe_nxt_blk_addr;
+	__le32	byte_24_pgsz_addr;
+	__le32	byte_28_cq_pi;
+	__le32	byte_32_cq_ci;
+	__le32	cqe_ba;
+	__le32	byte_40_cqe_ba;
+	__le32	byte_44_db_record;
+	__le32	db_record_addr;
+	__le32	byte_52_cqe_cnt;
+	__le32	byte_56_cqe_period_maxcnt;
+	__le32	cqe_report_timer;
+	__le32	byte_64_se_cqe_idx;
 };
+#define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0
+#define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL	0x0
+
 #define	V2_CQC_BYTE_4_CQ_ST_S 0
 #define V2_CQC_BYTE_4_CQ_ST_M GENMASK(1, 0)
 
@@ -313,66 +328,66 @@ enum hns_roce_v2_qp_state {
 };
 
 struct hns_roce_v2_qp_context {
-	u32	byte_4_sqpn_tst;
-	u32	wqe_sge_ba;
-	u32	byte_12_sq_hop;
-	u32	byte_16_buf_ba_pg_sz;
-	u32	byte_20_smac_sgid_idx;
-	u32	byte_24_mtu_tc;
-	u32	byte_28_at_fl;
+	__le32	byte_4_sqpn_tst;
+	__le32	wqe_sge_ba;
+	__le32	byte_12_sq_hop;
+	__le32	byte_16_buf_ba_pg_sz;
+	__le32	byte_20_smac_sgid_idx;
+	__le32	byte_24_mtu_tc;
+	__le32	byte_28_at_fl;
 	u8	dgid[GID_LEN_V2];
-	u32	dmac;
-	u32	byte_52_udpspn_dmac;
-	u32	byte_56_dqpn_err;
-	u32	byte_60_qpst_mapid;
-	u32	qkey_xrcd;
-	u32	byte_68_rq_db;
-	u32	rq_db_record_addr;
-	u32	byte_76_srqn_op_en;
-	u32	byte_80_rnr_rx_cqn;
-	u32	byte_84_rq_ci_pi;
-	u32	rq_cur_blk_addr;
-	u32	byte_92_srq_info;
-	u32	byte_96_rx_reqmsn;
-	u32	rq_nxt_blk_addr;
-	u32	byte_104_rq_sge;
-	u32	byte_108_rx_reqepsn;
-	u32	rq_rnr_timer;
-	u32	rx_msg_len;
-	u32	rx_rkey_pkt_info;
-	u64	rx_va;
-	u32	byte_132_trrl;
-	u32	trrl_ba;
-	u32	byte_140_raq;
-	u32	byte_144_raq;
-	u32	byte_148_raq;
-	u32	byte_152_raq;
-	u32	byte_156_raq;
-	u32	byte_160_sq_ci_pi;
-	u32	sq_cur_blk_addr;
-	u32	byte_168_irrl_idx;
-	u32	byte_172_sq_psn;
-	u32	byte_176_msg_pktn;
-	u32	sq_cur_sge_blk_addr;
-	u32	byte_184_irrl_idx;
-	u32	cur_sge_offset;
-	u32	byte_192_ext_sge;
-	u32	byte_196_sq_psn;
-	u32	byte_200_sq_max;
-	u32	irrl_ba;
-	u32	byte_208_irrl;
-	u32	byte_212_lsn;
-	u32	sq_timer;
-	u32	byte_220_retry_psn_msn;
-	u32	byte_224_retry_msg;
-	u32	rx_sq_cur_blk_addr;
-	u32	byte_232_irrl_sge;
-	u32	irrl_cur_sge_offset;
-	u32	byte_240_irrl_tail;
-	u32	byte_244_rnr_rxack;
-	u32	byte_248_ack_psn;
-	u32	byte_252_err_txcqn;
-	u32	byte_256_sqflush_rqcqe;
+	__le32	dmac;
+	__le32	byte_52_udpspn_dmac;
+	__le32	byte_56_dqpn_err;
+	__le32	byte_60_qpst_mapid;
+	__le32	qkey_xrcd;
+	__le32	byte_68_rq_db;
+	__le32	rq_db_record_addr;
+	__le32	byte_76_srqn_op_en;
+	__le32	byte_80_rnr_rx_cqn;
+	__le32	byte_84_rq_ci_pi;
+	__le32	rq_cur_blk_addr;
+	__le32	byte_92_srq_info;
+	__le32	byte_96_rx_reqmsn;
+	__le32	rq_nxt_blk_addr;
+	__le32	byte_104_rq_sge;
+	__le32	byte_108_rx_reqepsn;
+	__le32	rq_rnr_timer;
+	__le32	rx_msg_len;
+	__le32	rx_rkey_pkt_info;
+	__le64	rx_va;
+	__le32	byte_132_trrl;
+	__le32	trrl_ba;
+	__le32	byte_140_raq;
+	__le32	byte_144_raq;
+	__le32	byte_148_raq;
+	__le32	byte_152_raq;
+	__le32	byte_156_raq;
+	__le32	byte_160_sq_ci_pi;
+	__le32	sq_cur_blk_addr;
+	__le32	byte_168_irrl_idx;
+	__le32	byte_172_sq_psn;
+	__le32	byte_176_msg_pktn;
+	__le32	sq_cur_sge_blk_addr;
+	__le32	byte_184_irrl_idx;
+	__le32	cur_sge_offset;
+	__le32	byte_192_ext_sge;
+	__le32	byte_196_sq_psn;
+	__le32	byte_200_sq_max;
+	__le32	irrl_ba;
+	__le32	byte_208_irrl;
+	__le32	byte_212_lsn;
+	__le32	sq_timer;
+	__le32	byte_220_retry_psn_msn;
+	__le32	byte_224_retry_msg;
+	__le32	rx_sq_cur_blk_addr;
+	__le32	byte_232_irrl_sge;
+	__le32	irrl_cur_sge_offset;
+	__le32	byte_240_irrl_tail;
+	__le32	byte_244_rnr_rxack;
+	__le32	byte_248_ack_psn;
+	__le32	byte_252_err_txcqn;
+	__le32	byte_256_sqflush_rqcqe;
 };
 
 #define	V2_QPC_BYTE_4_TST_S 0
@@ -746,14 +761,17 @@ struct hns_roce_v2_qp_context {
 #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16)
 
 struct hns_roce_v2_cqe {
-	u32	byte_4;
-	u32	rkey_immtdata;
-	u32	byte_12;
-	u32	byte_16;
-	u32	byte_cnt;
-	u32	smac;
-	u32	byte_28;
-	u32	byte_32;
+	__le32	byte_4;
+	union {
+		__le32 rkey;
+		__be32 immtdata;
+	};
+	__le32	byte_12;
+	__le32	byte_16;
+	__le32	byte_cnt;
+	u8	smac[4];
+	__le32	byte_28;
+	__le32	byte_32;
 };
 
 #define	V2_CQE_BYTE_4_OPCODE_S 0
@@ -883,8 +901,8 @@ struct hns_roce_v2_mpt_entry {
 #define V2_DB_PARAMETER_SL_M GENMASK(18, 16)
 
 struct hns_roce_v2_cq_db {
-	u32	byte_4;
-	u32	parameter;
+	__le32	byte_4;
+	__le32	parameter;
 };
 
 #define	V2_CQ_DB_BYTE_4_TAG_S 0
@@ -901,14 +919,101 @@ struct hns_roce_v2_cq_db {
 
 #define V2_CQ_DB_PARAMETER_NOTIFY_S 24
 
+struct hns_roce_v2_ud_send_wqe {
+	__le32	byte_4;
+	__le32	msg_len;
+	__be32	immtdata;
+	__le32	byte_16;
+	__le32	byte_20;
+	__le32	byte_24;
+	__le32	qkey;
+	__le32	byte_32;
+	__le32	byte_36;
+	__le32	byte_40;
+	__le32	dmac;
+	__le32	byte_48;
+	u8	dgid[GID_LEN_V2];
+
+};
+#define	V2_UD_SEND_WQE_BYTE_4_OPCODE_S 0
+#define V2_UD_SEND_WQE_BYTE_4_OPCODE_M GENMASK(4, 0)
+
+#define	V2_UD_SEND_WQE_BYTE_4_OWNER_S 7
+
+#define	V2_UD_SEND_WQE_BYTE_4_CQE_S 8
+
+#define	V2_UD_SEND_WQE_BYTE_4_SE_S 11
+
+#define	V2_UD_SEND_WQE_BYTE_16_PD_S 0
+#define V2_UD_SEND_WQE_BYTE_16_PD_M GENMASK(23, 0)
+
+#define	V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S 24
+#define V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M GENMASK(31, 24)
+
+#define	V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S 0
+#define V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0)
+
+#define	V2_UD_SEND_WQE_BYTE_24_UDPSPN_S 16
+#define V2_UD_SEND_WQE_BYTE_24_UDPSPN_M GENMASK(31, 16)
+
+#define	V2_UD_SEND_WQE_BYTE_32_DQPN_S 0
+#define V2_UD_SEND_WQE_BYTE_32_DQPN_M GENMASK(23, 0)
+
+#define	V2_UD_SEND_WQE_BYTE_36_VLAN_S 0
+#define V2_UD_SEND_WQE_BYTE_36_VLAN_M GENMASK(15, 0)
+
+#define	V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S 16
+#define V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M GENMASK(23, 16)
+
+#define	V2_UD_SEND_WQE_BYTE_36_TCLASS_S 24
+#define V2_UD_SEND_WQE_BYTE_36_TCLASS_M GENMASK(31, 24)
+
+#define	V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S 0
+#define V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M GENMASK(19, 0)
+
+#define	V2_UD_SEND_WQE_BYTE_40_SL_S 20
+#define V2_UD_SEND_WQE_BYTE_40_SL_M GENMASK(23, 20)
+
+#define	V2_UD_SEND_WQE_BYTE_40_PORTN_S 24
+#define V2_UD_SEND_WQE_BYTE_40_PORTN_M GENMASK(26, 24)
+
+#define	V2_UD_SEND_WQE_BYTE_40_LBI_S 31
+
+#define	V2_UD_SEND_WQE_DMAC_0_S 0
+#define V2_UD_SEND_WQE_DMAC_0_M GENMASK(7, 0)
+
+#define	V2_UD_SEND_WQE_DMAC_1_S 8
+#define V2_UD_SEND_WQE_DMAC_1_M GENMASK(15, 8)
+
+#define	V2_UD_SEND_WQE_DMAC_2_S 16
+#define V2_UD_SEND_WQE_DMAC_2_M GENMASK(23, 16)
+
+#define	V2_UD_SEND_WQE_DMAC_3_S 24
+#define V2_UD_SEND_WQE_DMAC_3_M GENMASK(31, 24)
+
+#define	V2_UD_SEND_WQE_BYTE_48_DMAC_4_S 0
+#define V2_UD_SEND_WQE_BYTE_48_DMAC_4_M GENMASK(7, 0)
+
+#define	V2_UD_SEND_WQE_BYTE_48_DMAC_5_S 8
+#define V2_UD_SEND_WQE_BYTE_48_DMAC_5_M GENMASK(15, 8)
+
+#define	V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S 16
+#define V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M GENMASK(23, 16)
+
+#define	V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_S 24
+#define V2_UD_SEND_WQE_BYTE_48_SMAC_INDX_M GENMASK(31, 24)
+
 struct hns_roce_v2_rc_send_wqe {
-	u32		byte_4;
-	u32		msg_len;
-	u32		inv_key_immtdata;
-	u32		byte_16;
-	u32		byte_20;
-	u32		rkey;
-	u64		va;
+	__le32		byte_4;
+	__le32		msg_len;
+	union {
+		__le32  inv_key;
+		__be32  immtdata;
+	};
+	__le32		byte_16;
+	__le32		byte_20;
+	__le32		rkey;
+	__le64		va;
 };
 
 #define	V2_RC_SEND_WQE_BYTE_4_OPCODE_S 0
@@ -936,14 +1041,14 @@ struct hns_roce_v2_rc_send_wqe {
 #define V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M GENMASK(23, 0)
 
 struct hns_roce_v2_wqe_data_seg {
-	__be32    len;
-	__be32    lkey;
-	__be64    addr;
+	__le32    len;
+	__le32    lkey;
+	__le64    addr;
 };
 
 struct hns_roce_v2_db {
-	u32	byte_4;
-	u32	parameter;
+	__le32	byte_4;
+	__le32	parameter;
 };
 
 struct hns_roce_query_version {
@@ -1003,12 +1108,12 @@ struct hns_roce_pf_res {
 #define PF_RES_DATA_5_PF_EQC_BT_NUM_M GENMASK(25, 16)
 
 struct hns_roce_vf_res_a {
-	u32 vf_id;
-	u32 vf_qpc_bt_idx_num;
-	u32 vf_srqc_bt_idx_num;
-	u32 vf_cqc_bt_idx_num;
-	u32 vf_mpt_bt_idx_num;
-	u32 vf_eqc_bt_idx_num;
+	__le32 vf_id;
+	__le32 vf_qpc_bt_idx_num;
+	__le32 vf_srqc_bt_idx_num;
+	__le32 vf_cqc_bt_idx_num;
+	__le32 vf_mpt_bt_idx_num;
+	__le32 vf_eqc_bt_idx_num;
 };
 
 #define VF_RES_A_DATA_1_VF_QPC_BT_IDX_S 0
@@ -1042,11 +1147,11 @@ struct hns_roce_vf_res_a {
 #define VF_RES_A_DATA_5_VF_EQC_NUM_M GENMASK(25, 16)
 
 struct hns_roce_vf_res_b {
-	u32 rsv0;
-	u32 vf_smac_idx_num;
-	u32 vf_sgid_idx_num;
-	u32 vf_qid_idx_sl_num;
-	u32 rsv[2];
+	__le32 rsv0;
+	__le32 vf_smac_idx_num;
+	__le32 vf_sgid_idx_num;
+	__le32 vf_qid_idx_sl_num;
+	__le32 rsv[2];
 };
 
 #define VF_RES_B_DATA_0_VF_ID_S 0
@@ -1078,11 +1183,11 @@ struct hns_roce_vf_res_b {
 #define ROCEE_VF_SGID_CFG4_SGID_TYPE_M GENMASK(1, 0)
 
 struct hns_roce_cfg_bt_attr {
-	u32 vf_qpc_cfg;
-	u32 vf_srqc_cfg;
-	u32 vf_cqc_cfg;
-	u32 vf_mpt_cfg;
-	u32 rsv[2];
+	__le32 vf_qpc_cfg;
+	__le32 vf_srqc_cfg;
+	__le32 vf_cqc_cfg;
+	__le32 vf_mpt_cfg;
+	__le32 rsv[2];
 };
 
 #define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S 0
@@ -1122,16 +1227,13 @@ struct hns_roce_cfg_bt_attr {
 #define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M GENMASK(9, 8)
 
 struct hns_roce_cmq_desc {
-	u16 opcode;
-	u16 flag;
-	u16 retval;
-	u16 rsv;
-	u32 data[6];
+	__le16 opcode;
+	__le16 flag;
+	__le16 retval;
+	__le16 rsv;
+	__le32 data[6];
 };
 
-#define ROCEE_VF_MB_CFG0_REG		0x40
-#define ROCEE_VF_MB_STATUS_REG		0x58
-
 #define HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS	10000
 
 #define HNS_ROCE_HW_RUN_BIT_SHIFT	31
@@ -1174,4 +1276,178 @@ struct hns_roce_v2_priv {
 	struct hns_roce_v2_cmq cmq;
 };
 
+struct hns_roce_eq_context {
+	__le32	byte_4;
+	__le32	byte_8;
+	__le32	byte_12;
+	__le32	eqe_report_timer;
+	__le32	eqe_ba0;
+	__le32	eqe_ba1;
+	__le32	byte_28;
+	__le32	byte_32;
+	__le32	byte_36;
+	__le32	nxt_eqe_ba0;
+	__le32	nxt_eqe_ba1;
+	__le32	rsv[5];
+};
+
+#define HNS_ROCE_AEQ_DEFAULT_BURST_NUM	0x0
+#define HNS_ROCE_AEQ_DEFAULT_INTERVAL	0x0
+#define HNS_ROCE_CEQ_DEFAULT_BURST_NUM	0x0
+#define HNS_ROCE_CEQ_DEFAULT_INTERVAL	0x0
+
+#define HNS_ROCE_V2_EQ_STATE_INVALID		0
+#define HNS_ROCE_V2_EQ_STATE_VALID		1
+#define HNS_ROCE_V2_EQ_STATE_OVERFLOW		2
+#define HNS_ROCE_V2_EQ_STATE_FAILURE		3
+
+#define HNS_ROCE_V2_EQ_OVER_IGNORE_0		0
+#define HNS_ROCE_V2_EQ_OVER_IGNORE_1		1
+
+#define HNS_ROCE_V2_EQ_COALESCE_0		0
+#define HNS_ROCE_V2_EQ_COALESCE_1		1
+
+#define HNS_ROCE_V2_EQ_FIRED			0
+#define HNS_ROCE_V2_EQ_ARMED			1
+#define HNS_ROCE_V2_EQ_ALWAYS_ARMED		3
+
+#define HNS_ROCE_EQ_INIT_EQE_CNT		0
+#define HNS_ROCE_EQ_INIT_PROD_IDX		0
+#define HNS_ROCE_EQ_INIT_REPORT_TIMER		0
+#define HNS_ROCE_EQ_INIT_MSI_IDX		0
+#define HNS_ROCE_EQ_INIT_CONS_IDX		0
+#define HNS_ROCE_EQ_INIT_NXT_EQE_BA		0
+
+#define HNS_ROCE_V2_CEQ_CEQE_OWNER_S		31
+#define HNS_ROCE_V2_AEQ_AEQE_OWNER_S		31
+
+#define HNS_ROCE_V2_COMP_EQE_NUM		0x1000
+#define HNS_ROCE_V2_ASYNC_EQE_NUM		0x1000
+
+#define HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S	0
+#define HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S		1
+#define HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S	2
+
+#define HNS_ROCE_EQ_DB_CMD_AEQ			0x0
+#define HNS_ROCE_EQ_DB_CMD_AEQ_ARMED		0x1
+#define HNS_ROCE_EQ_DB_CMD_CEQ			0x2
+#define HNS_ROCE_EQ_DB_CMD_CEQ_ARMED		0x3
+
+#define EQ_ENABLE				1
+#define EQ_DISABLE				0
+
+#define EQ_REG_OFFSET				0x4
+
+#define HNS_ROCE_INT_NAME_LEN			32
+#define HNS_ROCE_V2_EQN_M GENMASK(23, 0)
+
+#define HNS_ROCE_V2_CONS_IDX_M GENMASK(23, 0)
+
+#define HNS_ROCE_V2_VF_ABN_INT_EN_S 0
+#define HNS_ROCE_V2_VF_ABN_INT_EN_M GENMASK(0, 0)
+#define HNS_ROCE_V2_VF_ABN_INT_ST_M GENMASK(2, 0)
+#define HNS_ROCE_V2_VF_ABN_INT_CFG_M GENMASK(2, 0)
+#define HNS_ROCE_V2_VF_EVENT_INT_EN_M GENMASK(0, 0)
+
+/* WORD0 */
+#define HNS_ROCE_EQC_EQ_ST_S 0
+#define HNS_ROCE_EQC_EQ_ST_M GENMASK(1, 0)
+
+#define HNS_ROCE_EQC_HOP_NUM_S 2
+#define HNS_ROCE_EQC_HOP_NUM_M GENMASK(3, 2)
+
+#define HNS_ROCE_EQC_OVER_IGNORE_S 4
+#define HNS_ROCE_EQC_OVER_IGNORE_M GENMASK(4, 4)
+
+#define HNS_ROCE_EQC_COALESCE_S 5
+#define HNS_ROCE_EQC_COALESCE_M GENMASK(5, 5)
+
+#define HNS_ROCE_EQC_ARM_ST_S 6
+#define HNS_ROCE_EQC_ARM_ST_M GENMASK(7, 6)
+
+#define HNS_ROCE_EQC_EQN_S 8
+#define HNS_ROCE_EQC_EQN_M GENMASK(15, 8)
+
+#define HNS_ROCE_EQC_EQE_CNT_S 16
+#define HNS_ROCE_EQC_EQE_CNT_M GENMASK(31, 16)
+
+/* WORD1 */
+#define HNS_ROCE_EQC_BA_PG_SZ_S 0
+#define HNS_ROCE_EQC_BA_PG_SZ_M GENMASK(3, 0)
+
+#define HNS_ROCE_EQC_BUF_PG_SZ_S 4
+#define HNS_ROCE_EQC_BUF_PG_SZ_M GENMASK(7, 4)
+
+#define HNS_ROCE_EQC_PROD_INDX_S 8
+#define HNS_ROCE_EQC_PROD_INDX_M GENMASK(31, 8)
+
+/* WORD2 */
+#define HNS_ROCE_EQC_MAX_CNT_S 0
+#define HNS_ROCE_EQC_MAX_CNT_M GENMASK(15, 0)
+
+#define HNS_ROCE_EQC_PERIOD_S 16
+#define HNS_ROCE_EQC_PERIOD_M GENMASK(31, 16)
+
+/* WORD3 */
+#define HNS_ROCE_EQC_REPORT_TIMER_S 0
+#define HNS_ROCE_EQC_REPORT_TIMER_M GENMASK(31, 0)
+
+/* WORD4 */
+#define HNS_ROCE_EQC_EQE_BA_L_S 0
+#define HNS_ROCE_EQC_EQE_BA_L_M GENMASK(31, 0)
+
+/* WORD5 */
+#define HNS_ROCE_EQC_EQE_BA_H_S 0
+#define HNS_ROCE_EQC_EQE_BA_H_M GENMASK(28, 0)
+
+/* WORD6 */
+#define HNS_ROCE_EQC_SHIFT_S 0
+#define HNS_ROCE_EQC_SHIFT_M GENMASK(7, 0)
+
+#define HNS_ROCE_EQC_MSI_INDX_S 8
+#define HNS_ROCE_EQC_MSI_INDX_M GENMASK(15, 8)
+
+#define HNS_ROCE_EQC_CUR_EQE_BA_L_S 16
+#define HNS_ROCE_EQC_CUR_EQE_BA_L_M GENMASK(31, 16)
+
+/* WORD7 */
+#define HNS_ROCE_EQC_CUR_EQE_BA_M_S 0
+#define HNS_ROCE_EQC_CUR_EQE_BA_M_M GENMASK(31, 0)
+
+/* WORD8 */
+#define HNS_ROCE_EQC_CUR_EQE_BA_H_S 0
+#define HNS_ROCE_EQC_CUR_EQE_BA_H_M GENMASK(3, 0)
+
+#define HNS_ROCE_EQC_CONS_INDX_S 8
+#define HNS_ROCE_EQC_CONS_INDX_M GENMASK(31, 8)
+
+/* WORD9 */
+#define HNS_ROCE_EQC_NXT_EQE_BA_L_S 0
+#define HNS_ROCE_EQC_NXT_EQE_BA_L_M GENMASK(31, 0)
+
+/* WORD10 */
+#define HNS_ROCE_EQC_NXT_EQE_BA_H_S 0
+#define HNS_ROCE_EQC_NXT_EQE_BA_H_M GENMASK(19, 0)
+
+#define HNS_ROCE_V2_CEQE_COMP_CQN_S 0
+#define HNS_ROCE_V2_CEQE_COMP_CQN_M GENMASK(23, 0)
+
+#define HNS_ROCE_V2_AEQE_EVENT_TYPE_S 0
+#define HNS_ROCE_V2_AEQE_EVENT_TYPE_M GENMASK(7, 0)
+
+#define HNS_ROCE_V2_AEQE_SUB_TYPE_S 8
+#define HNS_ROCE_V2_AEQE_SUB_TYPE_M GENMASK(15, 8)
+
+#define HNS_ROCE_V2_EQ_DB_CMD_S	16
+#define HNS_ROCE_V2_EQ_DB_CMD_M	GENMASK(17, 16)
+
+#define HNS_ROCE_V2_EQ_DB_TAG_S	0
+#define HNS_ROCE_V2_EQ_DB_TAG_M	GENMASK(7, 0)
+
+#define HNS_ROCE_V2_EQ_DB_PARA_S 0
+#define HNS_ROCE_V2_EQ_DB_PARA_M GENMASK(23, 0)
+
+#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S 0
+#define HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M GENMASK(23, 0)
+
 #endif
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index cf02ac2d3596..eb9a69fc7bec 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -200,7 +200,7 @@ static int hns_roce_query_device(struct ib_device *ib_dev,
 
 	memset(props, 0, sizeof(*props));
 
-	props->sys_image_guid = hr_dev->sys_image_guid;
+	props->sys_image_guid = cpu_to_be32(hr_dev->sys_image_guid);
 	props->max_mr_size = (u64)(~(0ULL));
 	props->page_size_cap = hr_dev->caps.page_size_cap;
 	props->vendor_id = hr_dev->vendor_id;
@@ -636,7 +636,6 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
 
 err_unmap_mtt:
-	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
 	if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
 		hns_roce_cleanup_hem_table(hr_dev,
 					   &hr_dev->mr_table.mtt_cqe_table);
@@ -748,12 +747,10 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)
 		goto error_failed_cmd_init;
 	}
 
-	if (hr_dev->cmd_mod) {
-		ret = hns_roce_init_eq_table(hr_dev);
-		if (ret) {
-			dev_err(dev, "eq init failed!\n");
-			goto error_failed_eq_table;
-		}
+	ret = hr_dev->hw->init_eq(hr_dev);
+	if (ret) {
+		dev_err(dev, "eq init failed!\n");
+		goto error_failed_eq_table;
 	}
 
 	if (hr_dev->cmd_mod) {
@@ -805,8 +802,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)
 		hns_roce_cmd_use_polling(hr_dev);
 
 error_failed_use_event:
-	if (hr_dev->cmd_mod)
-		hns_roce_cleanup_eq_table(hr_dev);
+	hr_dev->hw->cleanup_eq(hr_dev);
 
 error_failed_eq_table:
 	hns_roce_cmd_cleanup(hr_dev);
@@ -837,8 +833,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev)
 	if (hr_dev->cmd_mod)
 		hns_roce_cmd_use_polling(hr_dev);
 
-	if (hr_dev->cmd_mod)
-		hns_roce_cleanup_eq_table(hr_dev);
+	hr_dev->hw->cleanup_eq(hr_dev);
 	hns_roce_cmd_cleanup(hr_dev);
 	if (hr_dev->hw->cmq_exit)
 		hr_dev->hw->cmq_exit(hr_dev);
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index 49586ec8126a..088973a05882 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -65,6 +65,7 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
 	if (atomic_dec_and_test(&qp->refcount))
 		complete(&qp->free);
 }
+EXPORT_SYMBOL_GPL(hns_roce_qp_event);
 
 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
 				 enum hns_roce_event type)
@@ -454,6 +455,13 @@ static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev,
 		hr_qp->sge.sge_shift = 4;
 	}
 
+	/* ud sqwqe's sge use extend sge */
+	if (hr_dev->caps.max_sq_sg > 2 && hr_qp->ibqp.qp_type == IB_QPT_GSI) {
+		hr_qp->sge.sge_cnt = roundup_pow_of_two(hr_qp->sq.wqe_cnt *
+				     hr_qp->sq.max_gs);
+		hr_qp->sge.sge_shift = 4;
+	}
+
 	/* Get buf size, SQ and RQ are aligned to PAGE_SIZE */
 	page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
 	hr_qp->sq.offset = 0;
@@ -493,6 +501,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 	int ret = 0;
 	u32 page_shift;
 	u32 npages;
+	int i;
 
 	mutex_init(&hr_qp->mutex);
 	spin_lock_init(&hr_qp->sq.lock);
@@ -500,10 +509,12 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 
 	hr_qp->state = IB_QPS_RESET;
 
+	hr_qp->ibqp.qp_type = init_attr->qp_type;
+
 	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
-		hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
+		hr_qp->sq_signal_bits = cpu_to_le32(IB_SIGNAL_ALL_WR);
 	else
-		hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
+		hr_qp->sq_signal_bits = cpu_to_le32(IB_SIGNAL_REQ_WR);
 
 	ret = hns_roce_set_rq_size(hr_dev, &init_attr->cap, !!ib_pd->uobject,
 				   !!init_attr->srq, hr_qp);
@@ -512,18 +523,48 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 		goto err_out;
 	}
 
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
+		/* allocate recv inline buf */
+		hr_qp->rq_inl_buf.wqe_list = kcalloc(hr_qp->rq.wqe_cnt,
+					       sizeof(struct hns_roce_rinl_wqe),
+					       GFP_KERNEL);
+		if (!hr_qp->rq_inl_buf.wqe_list) {
+			ret = -ENOMEM;
+			goto err_out;
+		}
+
+		hr_qp->rq_inl_buf.wqe_cnt = hr_qp->rq.wqe_cnt;
+
+		/* Firstly, allocate a list of sge space buffer */
+		hr_qp->rq_inl_buf.wqe_list[0].sg_list =
+					kcalloc(hr_qp->rq_inl_buf.wqe_cnt,
+					       init_attr->cap.max_recv_sge *
+					       sizeof(struct hns_roce_rinl_sge),
+					       GFP_KERNEL);
+		if (!hr_qp->rq_inl_buf.wqe_list[0].sg_list) {
+			ret = -ENOMEM;
+			goto err_wqe_list;
+		}
+
+		for (i = 1; i < hr_qp->rq_inl_buf.wqe_cnt; i++)
+			/* Secondly, reallocate the buffer */
+			hr_qp->rq_inl_buf.wqe_list[i].sg_list =
+				&hr_qp->rq_inl_buf.wqe_list[0].sg_list[i *
+				init_attr->cap.max_recv_sge];
+	}
+
 	if (ib_pd->uobject) {
 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
 			dev_err(dev, "ib_copy_from_udata error for create qp\n");
 			ret = -EFAULT;
-			goto err_out;
+			goto err_rq_sge_list;
 		}
 
 		ret = hns_roce_set_user_sq_size(hr_dev, &init_attr->cap, hr_qp,
 						&ucmd);
 		if (ret) {
 			dev_err(dev, "hns_roce_set_user_sq_size error for create qp\n");
-			goto err_out;
+			goto err_rq_sge_list;
 		}
 
 		hr_qp->umem = ib_umem_get(ib_pd->uobject->context,
@@ -532,7 +573,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 		if (IS_ERR(hr_qp->umem)) {
 			dev_err(dev, "ib_umem_get error for create qp\n");
 			ret = PTR_ERR(hr_qp->umem);
-			goto err_out;
+			goto err_rq_sge_list;
 		}
 
 		hr_qp->mtt.mtt_type = MTT_TYPE_WQE;
@@ -566,13 +607,13 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 		    IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) {
 			dev_err(dev, "init_attr->create_flags error!\n");
 			ret = -EINVAL;
-			goto err_out;
+			goto err_rq_sge_list;
 		}
 
 		if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO) {
 			dev_err(dev, "init_attr->create_flags error!\n");
 			ret = -EINVAL;
-			goto err_out;
+			goto err_rq_sge_list;
 		}
 
 		/* Set SQ size */
@@ -580,7 +621,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 						  hr_qp);
 		if (ret) {
 			dev_err(dev, "hns_roce_set_kernel_sq_size error!\n");
-			goto err_out;
+			goto err_rq_sge_list;
 		}
 
 		/* QP doorbell register address */
@@ -596,7 +637,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 				       &hr_qp->hr_buf, page_shift)) {
 			dev_err(dev, "hns_roce_buf_alloc error!\n");
 			ret = -ENOMEM;
-			goto err_out;
+			goto err_rq_sge_list;
 		}
 
 		hr_qp->mtt.mtt_type = MTT_TYPE_WQE;
@@ -678,6 +719,14 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 	else
 		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
 
+err_rq_sge_list:
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE)
+		kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
+
+err_wqe_list:
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE)
+		kfree(hr_qp->rq_inl_buf.wqe_list);
+
 err_out:
 	return ret;
 }
@@ -724,8 +773,13 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd,
 		hr_qp = &hr_sqp->hr_qp;
 		hr_qp->port = init_attr->port_num - 1;
 		hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
-		hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS +
-				     hr_dev->iboe.phy_port[hr_qp->port];
+
+		/* when hw version is v1, the sqpn is allocated */
+		if (hr_dev->caps.max_sq_sg <= 2)
+			hr_qp->ibqp.qp_num = HNS_ROCE_MAX_PORTS +
+					     hr_dev->iboe.phy_port[hr_qp->port];
+		else
+			hr_qp->ibqp.qp_num = 1;
 
 		ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata,
 						hr_qp->ibqp.qp_num, hr_qp);
@@ -883,20 +937,6 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
 }
 EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs);
 
-__be32 send_ieth(struct ib_send_wr *wr)
-{
-	switch (wr->opcode) {
-	case IB_WR_SEND_WITH_IMM:
-	case IB_WR_RDMA_WRITE_WITH_IMM:
-		return cpu_to_le32(wr->ex.imm_data);
-	case IB_WR_SEND_WITH_INV:
-		return cpu_to_le32(wr->ex.invalidate_rkey);
-	default:
-		return 0;
-	}
-}
-EXPORT_SYMBOL_GPL(send_ieth);
-
 static void *get_wqe(struct hns_roce_qp *hr_qp, int offset)
 {
 
diff --git a/drivers/infiniband/hw/i40iw/Kconfig b/drivers/infiniband/hw/i40iw/Kconfig
index f6d20ba88c03..2962979c06e9 100644
--- a/drivers/infiniband/hw/i40iw/Kconfig
+++ b/drivers/infiniband/hw/i40iw/Kconfig
@@ -5,4 +5,3 @@ config INFINIBAND_I40IW
 	select GENERIC_ALLOCATOR
 	---help---
 	Intel(R) Ethernet X722 iWARP Driver
-	INET && I40IW && INFINIBAND && I40E
diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h
index 4ae9131b6350..bcddd7061fc0 100644
--- a/drivers/infiniband/hw/i40iw/i40iw.h
+++ b/drivers/infiniband/hw/i40iw/i40iw.h
@@ -587,5 +587,8 @@ int i40iw_inet6addr_event(struct notifier_block *notifier,
 int i40iw_net_event(struct notifier_block *notifier,
 		    unsigned long event,
 		    void *ptr);
+int i40iw_netdevice_event(struct notifier_block *notifier,
+			  unsigned long event,
+			  void *ptr);
 
 #endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
index 77870f9e1736..abf4cd897849 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c
@@ -92,14 +92,9 @@ void i40iw_free_sqbuf(struct i40iw_sc_vsi *vsi, void *bufp)
 static u8 i40iw_derive_hw_ird_setting(u16 cm_ird)
 {
 	u8 encoded_ird_size;
-	u8 pof2_cm_ird = 1;
-
-	/* round-off to next powerof2 */
-	while (pof2_cm_ird < cm_ird)
-		pof2_cm_ird *= 2;
 
 	/* ird_size field is encoded in qp_ctx */
-	switch (pof2_cm_ird) {
+	switch (cm_ird ? roundup_pow_of_two(cm_ird) : 0) {
 	case I40IW_HW_IRD_SETTING_64:
 		encoded_ird_size = 3;
 		break;
@@ -125,13 +120,16 @@ static u8 i40iw_derive_hw_ird_setting(u16 cm_ird)
  * @conn_ird: connection IRD
  * @conn_ord: connection ORD
  */
-static void i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u16 conn_ird, u16 conn_ord)
+static void i40iw_record_ird_ord(struct i40iw_cm_node *cm_node, u32 conn_ird,
+				 u32 conn_ord)
 {
 	if (conn_ird > I40IW_MAX_IRD_SIZE)
 		conn_ird = I40IW_MAX_IRD_SIZE;
 
 	if (conn_ord > I40IW_MAX_ORD_SIZE)
 		conn_ord = I40IW_MAX_ORD_SIZE;
+	else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)
+		conn_ord = 1;
 
 	cm_node->ird_size = conn_ird;
 	cm_node->ord_size = conn_ord;
@@ -2878,15 +2876,13 @@ static struct i40iw_cm_listener *i40iw_make_listen_node(
  * i40iw_create_cm_node - make a connection node with params
  * @cm_core: cm's core
  * @iwdev: iwarp device structure
- * @private_data_len: len to provate data for mpa request
- * @private_data: pointer to private data for connection
+ * @conn_param: upper layer connection parameters
  * @cm_info: quad info for connection
  */
 static struct i40iw_cm_node *i40iw_create_cm_node(
 					struct i40iw_cm_core *cm_core,
 					struct i40iw_device *iwdev,
-					u16 private_data_len,
-					void *private_data,
+					struct iw_cm_conn_param *conn_param,
 					struct i40iw_cm_info *cm_info)
 {
 	struct i40iw_cm_node *cm_node;
@@ -2894,6 +2890,9 @@ static struct i40iw_cm_node *i40iw_create_cm_node(
 	struct i40iw_cm_node *loopback_remotenode;
 	struct i40iw_cm_info loopback_cm_info;
 
+	u16 private_data_len = conn_param->private_data_len;
+	const void *private_data = conn_param->private_data;
+
 	/* create a CM connection node */
 	cm_node = i40iw_make_cm_node(cm_core, iwdev, cm_info, NULL);
 	if (!cm_node)
@@ -2902,6 +2901,8 @@ static struct i40iw_cm_node *i40iw_create_cm_node(
 	cm_node->tcp_cntxt.client = 1;
 	cm_node->tcp_cntxt.rcv_wscale = I40IW_CM_DEFAULT_RCV_WND_SCALE;
 
+	i40iw_record_ird_ord(cm_node, conn_param->ird, conn_param->ord);
+
 	if (!memcmp(cm_info->loc_addr, cm_info->rem_addr, sizeof(cm_info->loc_addr))) {
 		loopback_remotelistener = i40iw_find_listener(
 						cm_core,
@@ -2935,6 +2936,10 @@ static struct i40iw_cm_node *i40iw_create_cm_node(
 			       private_data_len);
 			loopback_remotenode->pdata.size = private_data_len;
 
+			if (loopback_remotenode->ord_size > cm_node->ird_size)
+				loopback_remotenode->ord_size =
+					cm_node->ird_size;
+
 			cm_node->state = I40IW_CM_STATE_OFFLOADED;
 			cm_node->tcp_cntxt.rcv_nxt =
 				loopback_remotenode->tcp_cntxt.loc_seq_num;
@@ -3691,7 +3696,7 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	cm_node->qhash_set = false;
 	i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
 
-	cm_node->accelerated = 1;
+	cm_node->accelerated = true;
 	status =
 		i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);
 	if (status)
@@ -3815,9 +3820,7 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		    __func__, cm_id->tos, cm_info.user_pri);
 	cm_id->add_ref(cm_id);
 	cm_node = i40iw_create_cm_node(&iwdev->cm_core, iwdev,
-				       conn_param->private_data_len,
-				       (void *)conn_param->private_data,
-				       &cm_info);
+				       conn_param, &cm_info);
 
 	if (IS_ERR(cm_node)) {
 		ret = PTR_ERR(cm_node);
@@ -3849,11 +3852,6 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	}
 
 	cm_node->apbvt_set = true;
-	i40iw_record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord);
-	if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO &&
-	    !cm_node->ord_size)
-		cm_node->ord_size = 1;
-
 	iwqp->cm_node = cm_node;
 	cm_node->iwqp = iwqp;
 	iwqp->cm_id = cm_id;
@@ -4058,7 +4056,7 @@ static void i40iw_cm_event_connected(struct i40iw_cm_event *event)
 	cm_node->qhash_set = false;
 	i40iw_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
 
-	cm_node->accelerated = 1;
+	cm_node->accelerated = true;
 	status = i40iw_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,
 				     0);
 	if (status)
@@ -4242,10 +4240,16 @@ static void i40iw_qhash_ctrl(struct i40iw_device *iwdev,
 }
 
 /**
- * i40iw_cm_disconnect_all - disconnect all connected qp's
+ * i40iw_cm_teardown_connections - teardown QPs
  * @iwdev: device pointer
+ * @ipaddr: Pointer to IPv4 or IPv6 address
+ * @ipv4: flag indicating IPv4 when true
+ * @disconnect_all: flag indicating disconnect all QPs
+ * teardown QPs where source or destination addr matches ip addr
  */
-void i40iw_cm_disconnect_all(struct i40iw_device *iwdev)
+void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr,
+				   struct i40iw_cm_info *nfo,
+				   bool disconnect_all)
 {
 	struct i40iw_cm_core *cm_core = &iwdev->cm_core;
 	struct list_head *list_core_temp;
@@ -4259,8 +4263,13 @@ void i40iw_cm_disconnect_all(struct i40iw_device *iwdev)
 	spin_lock_irqsave(&cm_core->ht_lock, flags);
 	list_for_each_safe(list_node, list_core_temp, &cm_core->connected_nodes) {
 		cm_node = container_of(list_node, struct i40iw_cm_node, list);
-		atomic_inc(&cm_node->ref_count);
-		list_add(&cm_node->connected_entry, &connected_list);
+		if (disconnect_all ||
+		    (nfo->vlan_id == cm_node->vlan_id &&
+		    (!memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16) ||
+		     !memcmp(cm_node->rem_addr, ipaddr, nfo->ipv4 ? 4 : 16)))) {
+			atomic_inc(&cm_node->ref_count);
+			list_add(&cm_node->connected_entry, &connected_list);
+		}
 	}
 	spin_unlock_irqrestore(&cm_core->ht_lock, flags);
 
@@ -4294,6 +4303,9 @@ void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
 	enum i40iw_quad_hash_manage_type op =
 		ifup ? I40IW_QHASH_MANAGE_TYPE_ADD : I40IW_QHASH_MANAGE_TYPE_DELETE;
 
+	nfo.vlan_id = vlan_id;
+	nfo.ipv4 = ipv4;
+
 	/* Disable or enable qhash for listeners */
 	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
 	list_for_each_entry(listen_node, &cm_core->listen_nodes, list) {
@@ -4303,8 +4315,6 @@ void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
 			memcpy(nfo.loc_addr, listen_node->loc_addr,
 			       sizeof(nfo.loc_addr));
 			nfo.loc_port = listen_node->loc_port;
-			nfo.ipv4 = listen_node->ipv4;
-			nfo.vlan_id = listen_node->vlan_id;
 			nfo.user_pri = listen_node->user_pri;
 			if (!list_empty(&listen_node->child_listen_list)) {
 				i40iw_qhash_ctrl(iwdev,
@@ -4326,7 +4336,7 @@ void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
 	}
 	spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
 
-	/* disconnect any connected qp's on ifdown */
+	/* teardown connected qp's on ifdown */
 	if (!ifup)
-		i40iw_cm_disconnect_all(iwdev);
+		i40iw_cm_teardown_connections(iwdev, ipaddr, &nfo, false);
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h
index 0d5840d2c4fc..cf60c451e071 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.h
@@ -276,8 +276,6 @@ struct i40iw_cm_tcp_context {
 	u32 mss;
 	u8 snd_wscale;
 	u8 rcv_wscale;
-
-	struct timeval sent_ts;
 };
 
 enum i40iw_cm_listener_state {
@@ -337,7 +335,7 @@ struct i40iw_cm_node {
 	u16     mpav2_ird_ord;
 	struct iw_cm_id *cm_id;
 	struct list_head list;
-	int accelerated;
+	bool accelerated;
 	struct i40iw_cm_listener *listener;
 	int apbvt_set;
 	int accept_pend;
@@ -455,5 +453,7 @@ int i40iw_arp_table(struct i40iw_device *iwdev,
 
 void i40iw_if_notify(struct i40iw_device *iwdev, struct net_device *netdev,
 		     u32 *ipaddr, bool ipv4, bool ifup);
-void i40iw_cm_disconnect_all(struct i40iw_device *iwdev);
+void i40iw_cm_teardown_connections(struct i40iw_device *iwdev, u32 *ipaddr,
+				   struct i40iw_cm_info *nfo,
+				   bool disconnect_all);
 #endif /* I40IW_CM_H */
diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
index da9821a10e0d..c74fd3309b93 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c
@@ -1893,8 +1893,6 @@ static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,
 static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,
 							  u32 count)
 {
-	if (count > I40IW_MAX_AEQ_ALLOCATE_COUNT)
-		return I40IW_ERR_INVALID_SIZE;
 
 	if (dev->is_pf)
 		i40iw_wr32(dev->hw, I40E_PFPE_AEQALLOC, count);
@@ -3872,7 +3870,6 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_
 	struct i40iw_virt_mem virt_mem;
 	u32 i, mem_size;
 	u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
-	u32 powerof2;
 	u64 sd_needed;
 	u32 loop_count = 0;
 
@@ -3928,8 +3925,10 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_
 		hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;
 		hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;
 
-		hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt = I40IW_MAX_WQ_ENTRIES * qpwanted;
-		hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt = 4 * I40IW_MAX_IRD_SIZE * qpwanted;
+		hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt =
+			roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted);
+		hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt =
+			roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted);
 		hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt =
 			hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
 		hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt =
@@ -3945,16 +3944,10 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_
 		if ((loop_count > 1000) ||
 		    ((!(loop_count % 10)) &&
 		    (qpwanted > qpwantedoriginal * 2 / 3))) {
-			if (qpwanted > FPM_MULTIPLIER) {
-				qpwanted -= FPM_MULTIPLIER;
-				powerof2 = 1;
-				while (powerof2 < qpwanted)
-					powerof2 *= 2;
-				powerof2 /= 2;
-				qpwanted = powerof2;
-			} else {
-				qpwanted /= 2;
-			}
+			if (qpwanted > FPM_MULTIPLIER)
+				qpwanted = roundup_pow_of_two(qpwanted -
+							      FPM_MULTIPLIER);
+			qpwanted >>= 1;
 		}
 		if (mrwanted > FPM_MULTIPLIER * 10)
 			mrwanted -= FPM_MULTIPLIER * 10;
@@ -3962,8 +3955,6 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_
 			pblewanted -= FPM_MULTIPLIER * 1000;
 	} while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000);
 
-	sd_needed = i40iw_est_sd(dev, hmc_info);
-
 	i40iw_debug(dev, I40IW_DEBUG_HMC,
 		    "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
 		    loop_count, sd_needed,
diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h
index 029083cb81d5..4b65e4140bd7 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_d.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_d.h
@@ -97,6 +97,7 @@
 #define RDMA_OPCODE_MASK        0x0f
 #define RDMA_READ_REQ_OPCODE    1
 #define Q2_BAD_FRAME_OFFSET     72
+#define Q2_FPSN_OFFSET          64
 #define CQE_MAJOR_DRV           0x8000
 
 #define I40IW_TERM_SENT 0x01
diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c
index e96bdafbcbb3..61540e14e4b9 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_hw.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_hw.c
@@ -385,6 +385,8 @@ void i40iw_process_aeq(struct i40iw_device *iwdev)
 				iwcq->ibcq.event_handler(&ibevent, iwcq->ibcq.cq_context);
 			}
 			break;
+		case I40IW_AE_LLP_DOUBT_REACHABILITY:
+			break;
 		case I40IW_AE_PRIV_OPERATION_DENIED:
 		case I40IW_AE_STAG_ZERO_INVALID:
 		case I40IW_AE_IB_RREQ_AND_Q1_FULL:
@@ -403,7 +405,6 @@ void i40iw_process_aeq(struct i40iw_device *iwdev)
 		case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
 		case I40IW_AE_LLP_SYN_RECEIVED:
 		case I40IW_AE_LLP_TOO_MANY_RETRIES:
-		case I40IW_AE_LLP_DOUBT_REACHABILITY:
 		case I40IW_AE_LCE_QP_CATASTROPHIC:
 		case I40IW_AE_LCE_FUNCTION_CATASTROPHIC:
 		case I40IW_AE_LCE_CQ_CATASTROPHIC:
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
index e824296713e2..b08862978de8 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_main.c
@@ -99,6 +99,10 @@ static struct notifier_block i40iw_net_notifier = {
 	.notifier_call = i40iw_net_event
 };
 
+static struct notifier_block i40iw_netdevice_notifier = {
+	.notifier_call = i40iw_netdevice_event
+};
+
 /**
  * i40iw_find_i40e_handler - find a handler given a client info
  * @ldev: pointer to a client info
@@ -483,6 +487,7 @@ static enum i40iw_status_code i40iw_create_hmc_objs(struct i40iw_device *iwdev,
 	for (i = 0; i < IW_HMC_OBJ_TYPE_NUM; i++) {
 		info.rsrc_type = iw_hmc_obj_types[i];
 		info.count = dev->hmc_info->hmc_obj[info.rsrc_type].cnt;
+		info.add_sd_cnt = 0;
 		status = i40iw_create_hmc_obj_type(dev, &info);
 		if (status) {
 			i40iw_pr_err("create obj type %d status = %d\n",
@@ -607,7 +612,7 @@ static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev)
 	INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
 	INIT_LIST_HEAD(&cqp->cqp_pending_reqs);
 	/* init the waitq of the cqp_requests and add them to the list */
-	for (i = 0; i < I40IW_CQP_SW_SQSIZE_2048; i++) {
+	for (i = 0; i < sqsize; i++) {
 		init_waitqueue_head(&cqp->cqp_requests[i].waitq);
 		list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs);
 	}
@@ -1285,7 +1290,7 @@ static void i40iw_wait_pe_ready(struct i40iw_hw *hw)
 			      __LINE__, statuscpu2);
 		if ((statuscpu0 == 0x80) && (statuscpu1 == 0x80) && (statuscpu2 == 0x80))
 			break;	/* SUCCESS */
-		mdelay(1000);
+		msleep(1000);
 		retrycount++;
 	} while (retrycount < 14);
 	i40iw_wr32(hw, 0xb4040, 0x4C104C5);
@@ -1393,6 +1398,7 @@ static void i40iw_register_notifiers(void)
 	register_inetaddr_notifier(&i40iw_inetaddr_notifier);
 	register_inet6addr_notifier(&i40iw_inetaddr6_notifier);
 	register_netevent_notifier(&i40iw_net_notifier);
+	register_netdevice_notifier(&i40iw_netdevice_notifier);
 }
 
 /**
@@ -1404,6 +1410,7 @@ static void i40iw_unregister_notifiers(void)
 	unregister_netevent_notifier(&i40iw_net_notifier);
 	unregister_inetaddr_notifier(&i40iw_inetaddr_notifier);
 	unregister_inet6addr_notifier(&i40iw_inetaddr6_notifier);
+	unregister_netdevice_notifier(&i40iw_netdevice_notifier);
 }
 
 /**
@@ -1793,7 +1800,7 @@ static void i40iw_close(struct i40e_info *ldev, struct i40e_client *client, bool
 	if (reset)
 		iwdev->reset = true;
 
-	i40iw_cm_disconnect_all(iwdev);
+	i40iw_cm_teardown_connections(iwdev, NULL, NULL, true);
 	destroy_workqueue(iwdev->virtchnl_wq);
 	i40iw_deinit_device(iwdev);
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c
index 796a815b53fd..4c21197830b3 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_puda.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_puda.c
@@ -48,7 +48,6 @@ static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid);
 static void i40iw_ilq_putback_rcvbuf(struct i40iw_sc_qp *qp, u32 wqe_idx);
 static enum i40iw_status_code i40iw_puda_replenish_rq(struct i40iw_puda_rsrc
 						      *rsrc, bool initial);
-static void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp);
 /**
  * i40iw_puda_get_listbuf - get buffer from puda list
  * @list: list to use for buffers (ILQ or IEQ)
@@ -1378,7 +1377,7 @@ static void i40iw_ieq_handle_exception(struct i40iw_puda_rsrc *ieq,
 	u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
 	u32 rcv_wnd = hw_host_ctx[23];
 	/* first partial seq # in q2 */
-	u32 fps = qp->q2_buf[16];
+	u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
 	struct list_head *rxlist = &pfpdu->rxlist;
 	struct list_head *plist;
 
@@ -1483,7 +1482,7 @@ static void i40iw_ieq_tx_compl(struct i40iw_sc_vsi *vsi, void *sqwrid)
  * @ieq: ieq resource
  * @qp: all pending fpdu buffers
  */
-static void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp)
+void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp)
 {
 	struct i40iw_puda_buf *buf;
 	struct i40iw_pfpdu *pfpdu = &qp->pfpdu;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.h b/drivers/infiniband/hw/i40iw/i40iw_puda.h
index 660aa3edae56..53a7d58c84b5 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_puda.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_puda.h
@@ -184,4 +184,5 @@ enum i40iw_status_code i40iw_cqp_qp_create_cmd(struct i40iw_sc_dev *dev, struct
 enum i40iw_status_code i40iw_cqp_cq_create_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);
 void i40iw_cqp_qp_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp);
 void i40iw_cqp_cq_destroy_cmd(struct i40iw_sc_dev *dev, struct i40iw_sc_cq *cq);
+void i40iw_ieq_cleanup_qp(struct i40iw_puda_rsrc *ieq, struct i40iw_sc_qp *qp);
 #endif
diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c
index 3ec5389a81a1..8afa5a67a86b 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_uk.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_uk.c
@@ -893,20 +893,6 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq,
 	return ret_code;
 }
 
-/**
- * i40iw_qp_roundup - return round up QP WQ depth
- * @wqdepth: WQ depth in quantas to round up
- */
-static int i40iw_qp_round_up(u32 wqdepth)
-{
-	int scount = 1;
-
-	for (wqdepth--; scount <= 16; scount *= 2)
-		wqdepth |= wqdepth >> scount;
-
-	return ++wqdepth;
-}
-
 /**
  * i40iw_get_wqe_shift - get shift count for maximum wqe size
  * @sge: Maximum Scatter Gather Elements wqe
@@ -934,7 +920,7 @@ void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift)
  */
 enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth)
 {
-	*sqdepth = i40iw_qp_round_up((sq_size << shift) + I40IW_SQ_RSVD);
+	*sqdepth = roundup_pow_of_two((sq_size << shift) + I40IW_SQ_RSVD);
 
 	if (*sqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift))
 		*sqdepth = I40IW_QP_SW_MIN_WQSIZE << shift;
@@ -953,7 +939,7 @@ enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth)
  */
 enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth)
 {
-	*rqdepth = i40iw_qp_round_up((rq_size << shift) + I40IW_RQ_RSVD);
+	*rqdepth = roundup_pow_of_two((rq_size << shift) + I40IW_RQ_RSVD);
 
 	if (*rqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift))
 		*rqdepth = I40IW_QP_SW_MIN_WQSIZE << shift;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h
index e73efc59a0ab..b125925641e0 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_user.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_user.h
@@ -59,7 +59,6 @@ enum i40iw_device_capabilities_const {
 	I40IW_MAX_CEQ_ENTRIES =			131071,
 	I40IW_MIN_CQ_SIZE =			1,
 	I40IW_MAX_CQ_SIZE =			1048575,
-	I40IW_MAX_AEQ_ALLOCATE_COUNT =		255,
 	I40IW_DB_ID_ZERO =			0,
 	I40IW_MAX_WQ_FRAGMENT_COUNT =		3,
 	I40IW_MAX_SGE_RD =			1,
@@ -72,7 +71,7 @@ enum i40iw_device_capabilities_const {
 	I40IW_MAX_SQ_PAYLOAD_SIZE =		2145386496,
 	I40IW_MAX_INLINE_DATA_SIZE =		48,
 	I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE =	48,
-	I40IW_MAX_IRD_SIZE =			63,
+	I40IW_MAX_IRD_SIZE =			64,
 	I40IW_MAX_ORD_SIZE =			127,
 	I40IW_MAX_WQ_ENTRIES =			2048,
 	I40IW_Q2_BUFFER_SIZE =			(248 + 100),
diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c
index 8845dba7c438..ddc1056b0b4e 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_utils.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c
@@ -137,7 +137,7 @@ inline u32 i40iw_rd32(struct i40iw_hw *hw, u32 reg)
 }
 
 /**
- * i40iw_inetaddr_event - system notifier for netdev events
+ * i40iw_inetaddr_event - system notifier for ipv4 addr events
  * @notfier: not used
  * @event: event for notifier
  * @ptr: if address
@@ -200,7 +200,7 @@ int i40iw_inetaddr_event(struct notifier_block *notifier,
 }
 
 /**
- * i40iw_inet6addr_event - system notifier for ipv6 netdev events
+ * i40iw_inet6addr_event - system notifier for ipv6 addr events
  * @notfier: not used
  * @event: event for notifier
  * @ptr: if address
@@ -252,7 +252,7 @@ int i40iw_inet6addr_event(struct notifier_block *notifier,
 }
 
 /**
- * i40iw_net_event - system notifier for net events
+ * i40iw_net_event - system notifier for netevents
  * @notfier: not used
  * @event: event for notifier
  * @ptr: neighbor
@@ -296,6 +296,50 @@ int i40iw_net_event(struct notifier_block *notifier, unsigned long event, void *
 	return NOTIFY_DONE;
 }
 
+/**
+ * i40iw_netdevice_event - system notifier for netdev events
+ * @notfier: not used
+ * @event: event for notifier
+ * @ptr: netdev
+ */
+int i40iw_netdevice_event(struct notifier_block *notifier,
+			  unsigned long event,
+			  void *ptr)
+{
+	struct net_device *event_netdev;
+	struct net_device *netdev;
+	struct i40iw_device *iwdev;
+	struct i40iw_handler *hdl;
+
+	event_netdev = netdev_notifier_info_to_dev(ptr);
+
+	hdl = i40iw_find_netdev(event_netdev);
+	if (!hdl)
+		return NOTIFY_DONE;
+
+	iwdev = &hdl->device;
+	if (iwdev->init_state < RDMA_DEV_REGISTERED || iwdev->closing)
+		return NOTIFY_DONE;
+
+	netdev = iwdev->ldev->netdev;
+	if (netdev != event_netdev)
+		return NOTIFY_DONE;
+
+	iwdev->iw_status = 1;
+
+	switch (event) {
+	case NETDEV_DOWN:
+		iwdev->iw_status = 0;
+		/* Fall through */
+	case NETDEV_UP:
+		i40iw_port_ibevent(iwdev);
+		break;
+	default:
+		break;
+	}
+	return NOTIFY_DONE;
+}
+
 /**
  * i40iw_get_cqp_request - get cqp struct
  * @cqp: device cqp ptr
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 3c6f3ce88f89..70024e8e2692 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -412,6 +412,7 @@ void i40iw_free_qp_resources(struct i40iw_device *iwdev,
 {
 	struct i40iw_pbl *iwpbl = &iwqp->iwpbl;
 
+	i40iw_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
 	i40iw_dealloc_push_page(iwdev, &iwqp->sc_qp);
 	if (qp_num)
 		i40iw_free_resource(iwdev, iwdev->allocated_qps, qp_num);
@@ -1637,6 +1638,7 @@ static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd,
 		err_code = -EOVERFLOW;
 		goto err;
 	}
+	stag &= ~I40IW_CQPSQ_STAG_KEY_MASK;
 	iwmr->stag = stag;
 	iwmr->ibmr.rkey = stag;
 	iwmr->ibmr.lkey = stag;
@@ -2242,14 +2244,12 @@ static int i40iw_post_send(struct ib_qp *ibqp,
 				info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
 				info.op.inline_rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
 				info.op.inline_rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
-				info.op.inline_rdma_write.rem_addr.len = ib_wr->sg_list->length;
 				ret = ukqp->ops.iw_inline_rdma_write(ukqp, &info, false);
 			} else {
 				info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
 				info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
 				info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
 				info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
-				info.op.rdma_write.rem_addr.len = ib_wr->sg_list->length;
 				ret = ukqp->ops.iw_rdma_write(ukqp, &info, false);
 			}
 
@@ -2271,7 +2271,6 @@ static int i40iw_post_send(struct ib_qp *ibqp,
 			info.op_type = I40IW_OP_TYPE_RDMA_READ;
 			info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
 			info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
-			info.op.rdma_read.rem_addr.len = ib_wr->sg_list->length;
 			info.op.rdma_read.lo_addr.tag_off = ib_wr->sg_list->addr;
 			info.op.rdma_read.lo_addr.stag = ib_wr->sg_list->lkey;
 			info.op.rdma_read.lo_addr.len = ib_wr->sg_list->length;
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index bf4f14a1b4fc..82adc0d1d30e 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -170,7 +170,7 @@ static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *cont
 	return err;
 }
 
-#define CQ_CREATE_FLAGS_SUPPORTED IB_CQ_FLAGS_TIMESTAMP_COMPLETION
+#define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
 struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
 				struct ib_ucontext *context,
@@ -246,7 +246,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 
 	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,
 			    cq->db.dma, &cq->mcq, vector, 0,
-			    !!(cq->create_flags & IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
+			    !!(cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION));
 	if (err)
 		goto err_dbmap;
 
@@ -601,6 +601,7 @@ static void use_tunnel_data(struct mlx4_ib_qp *qp, struct mlx4_ib_cq *cq, struct
 	wc->dlid_path_bits = 0;
 
 	if (is_eth) {
+		wc->slid = 0;
 		wc->vlan_id = be16_to_cpu(hdr->tun.sl_vid);
 		memcpy(&(wc->smac[0]), (char *)&hdr->tun.mac_31_0, 4);
 		memcpy(&(wc->smac[4]), (char *)&hdr->tun.slid_mac_47_32, 2);
@@ -851,7 +852,6 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
 			}
 		}
 
-		wc->slid	   = be16_to_cpu(cqe->rlid);
 		g_mlpath_rqpn	   = be32_to_cpu(cqe->g_mlpath_rqpn);
 		wc->src_qp	   = g_mlpath_rqpn & 0xffffff;
 		wc->dlid_path_bits = (g_mlpath_rqpn >> 24) & 0x7f;
@@ -860,6 +860,7 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
 		wc->wc_flags	  |= mlx4_ib_ipoib_csum_ok(cqe->status,
 					cqe->checksum) ? IB_WC_IP_CSUM_OK : 0;
 		if (is_eth) {
+			wc->slid = 0;
 			wc->sl  = be16_to_cpu(cqe->sl_vid) >> 13;
 			if (be32_to_cpu(cqe->vlan_my_qpn) &
 					MLX4_CQE_CVLAN_PRESENT_MASK) {
@@ -871,6 +872,7 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
 			memcpy(wc->smac, cqe->smac, ETH_ALEN);
 			wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
 		} else {
+			wc->slid = be16_to_cpu(cqe->rlid);
 			wc->sl  = be16_to_cpu(cqe->sl_vid) >> 12;
 			wc->vlan_id = 0xffff;
 		}
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 8c8a16791a3f..5a0e4fc4785a 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -219,8 +219,6 @@ static int mlx4_ib_update_gids_v1_v2(struct gid_entry *gids,
 			gid_tbl[i].version = 2;
 			if (!ipv6_addr_v4mapped((struct in6_addr *)&gids[i].gid))
 				gid_tbl[i].type = 1;
-			else
-				memset(&gid_tbl[i].gid, 0, 12);
 		}
 	}
 
@@ -366,8 +364,13 @@ static int mlx4_ib_del_gid(struct ib_device *device,
 		if (!gids) {
 			ret = -ENOMEM;
 		} else {
-			for (i = 0; i < MLX4_MAX_PORT_GIDS; i++)
-				memcpy(&gids[i].gid, &port_gid_table->gids[i].gid, sizeof(union ib_gid));
+			for (i = 0; i < MLX4_MAX_PORT_GIDS; i++) {
+				memcpy(&gids[i].gid,
+				       &port_gid_table->gids[i].gid,
+				       sizeof(union ib_gid));
+				gids[i].gid_type =
+				    port_gid_table->gids[i].gid_type;
+			}
 		}
 	}
 	spin_unlock_bh(&iboe->lock);
@@ -589,6 +592,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 		if (props->rss_caps.supported_qpts) {
 			resp.rss_caps.rx_hash_function =
 				MLX4_IB_RX_HASH_FUNC_TOEPLITZ;
+
 			resp.rss_caps.rx_hash_fields_mask =
 				MLX4_IB_RX_HASH_SRC_IPV4 |
 				MLX4_IB_RX_HASH_DST_IPV4 |
@@ -598,6 +602,11 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 				MLX4_IB_RX_HASH_DST_PORT_TCP |
 				MLX4_IB_RX_HASH_SRC_PORT_UDP |
 				MLX4_IB_RX_HASH_DST_PORT_UDP;
+
+			if (dev->dev->caps.tunnel_offload_mode ==
+			    MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
+				resp.rss_caps.rx_hash_fields_mask |=
+					MLX4_IB_RX_HASH_INNER;
 		}
 	}
 
@@ -2995,9 +3004,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
 	kfree(ibdev->ib_uc_qpns_bitmap);
 
 err_steer_qp_release:
-	if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED)
-		mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
-				      ibdev->steer_qpn_count);
+	mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
+			      ibdev->steer_qpn_count);
 err_counter:
 	for (i = 0; i < ibdev->num_ports; ++i)
 		mlx4_ib_delete_counters_table(ibdev, &ibdev->counters_table[i]);
@@ -3102,11 +3110,9 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
 		ibdev->iboe.nb.notifier_call = NULL;
 	}
 
-	if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED) {
-		mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
-				      ibdev->steer_qpn_count);
-		kfree(ibdev->ib_uc_qpns_bitmap);
-	}
+	mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
+			      ibdev->steer_qpn_count);
+	kfree(ibdev->ib_uc_qpns_bitmap);
 
 	iounmap(ibdev->uar_map);
 	for (p = 0; p < ibdev->num_ports; ++p)
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index caf490ab24c8..f045491f2c14 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -734,10 +734,24 @@ static int set_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_rss *rss_ctx,
 		return (-EOPNOTSUPP);
 	}
 
+	if (ucmd->rx_hash_fields_mask & MLX4_IB_RX_HASH_INNER) {
+		if (dev->dev->caps.tunnel_offload_mode ==
+		    MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
+			/*
+			 * Hash according to inner headers if exist, otherwise
+			 * according to outer headers.
+			 */
+			rss_ctx->flags |= MLX4_RSS_BY_INNER_HEADERS_IPONLY;
+		} else {
+			pr_debug("RSS Hash for inner headers isn't supported\n");
+			return (-EOPNOTSUPP);
+		}
+	}
+
 	return 0;
 }
 
-static int create_qp_rss(struct mlx4_ib_dev *dev, struct ib_pd *ibpd,
+static int create_qp_rss(struct mlx4_ib_dev *dev,
 			 struct ib_qp_init_attr *init_attr,
 			 struct mlx4_ib_create_qp_rss *ucmd,
 			 struct mlx4_ib_qp *qp)
@@ -860,7 +874,7 @@ static struct ib_qp *_mlx4_ib_create_qp_rss(struct ib_pd *pd,
 	qp->pri.vid = 0xFFFF;
 	qp->alt.vid = 0xFFFF;
 
-	err = create_qp_rss(to_mdev(pd->device), pd, init_attr, &ucmd, qp);
+	err = create_qp_rss(to_mdev(pd->device), init_attr, &ucmd, qp);
 	if (err) {
 		kfree(qp);
 		return ERR_PTR(err);
@@ -1836,6 +1850,8 @@ static int _mlx4_set_path(struct mlx4_ib_dev *dev,
 			mlx4_ib_gid_index_to_real_index(dev, port,
 							grh->sgid_index);
 
+		if (real_sgid_index < 0)
+			return real_sgid_index;
 		if (real_sgid_index >= dev->dev->caps.gid_table_len[port]) {
 			pr_err("sgid_index (%u) too large. max is %d\n",
 			       real_sgid_index, dev->dev->caps.gid_table_len[port] - 1);
diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c
index 2d32b519bb61..985fa2637390 100644
--- a/drivers/infiniband/hw/mlx5/cong.c
+++ b/drivers/infiniband/hw/mlx5/cong.c
@@ -247,21 +247,30 @@ static void mlx5_ib_set_cc_param_mask_val(void *field, int offset,
 	}
 }
 
-static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, int offset, u32 *var)
+static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, u8 port_num,
+				 int offset, u32 *var)
 {
 	int outlen = MLX5_ST_SZ_BYTES(query_cong_params_out);
 	void *out;
 	void *field;
 	int err;
 	enum mlx5_ib_cong_node_type node;
+	struct mlx5_core_dev *mdev;
+
+	/* Takes a 1-based port number */
+	mdev = mlx5_ib_get_native_port_mdev(dev, port_num + 1, NULL);
+	if (!mdev)
+		return -ENODEV;
 
 	out = kvzalloc(outlen, GFP_KERNEL);
-	if (!out)
-		return -ENOMEM;
+	if (!out) {
+		err = -ENOMEM;
+		goto alloc_err;
+	}
 
 	node = mlx5_ib_param_to_node(offset);
 
-	err = mlx5_cmd_query_cong_params(dev->mdev, node, out, outlen);
+	err = mlx5_cmd_query_cong_params(mdev, node, out, outlen);
 	if (err)
 		goto free;
 
@@ -270,21 +279,32 @@ static int mlx5_ib_get_cc_params(struct mlx5_ib_dev *dev, int offset, u32 *var)
 
 free:
 	kvfree(out);
+alloc_err:
+	mlx5_ib_put_native_port_mdev(dev, port_num + 1);
 	return err;
 }
 
-static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, int offset, u32 var)
+static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u8 port_num,
+				 int offset, u32 var)
 {
 	int inlen = MLX5_ST_SZ_BYTES(modify_cong_params_in);
 	void *in;
 	void *field;
 	enum mlx5_ib_cong_node_type node;
+	struct mlx5_core_dev *mdev;
 	u32 attr_mask = 0;
 	int err;
 
+	/* Takes a 1-based port number */
+	mdev = mlx5_ib_get_native_port_mdev(dev, port_num + 1, NULL);
+	if (!mdev)
+		return -ENODEV;
+
 	in = kvzalloc(inlen, GFP_KERNEL);
-	if (!in)
-		return -ENOMEM;
+	if (!in) {
+		err = -ENOMEM;
+		goto alloc_err;
+	}
 
 	MLX5_SET(modify_cong_params_in, in, opcode,
 		 MLX5_CMD_OP_MODIFY_CONG_PARAMS);
@@ -299,8 +319,10 @@ static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, int offset, u32 var)
 	MLX5_SET(field_select_r_roce_rp, field, field_select_r_roce_rp,
 		 attr_mask);
 
-	err = mlx5_cmd_modify_cong_params(dev->mdev, in, inlen);
+	err = mlx5_cmd_modify_cong_params(mdev, in, inlen);
 	kvfree(in);
+alloc_err:
+	mlx5_ib_put_native_port_mdev(dev, port_num + 1);
 	return err;
 }
 
@@ -324,7 +346,7 @@ static ssize_t set_param(struct file *filp, const char __user *buf,
 	if (kstrtou32(lbuf, 0, &var))
 		return -EINVAL;
 
-	ret = mlx5_ib_set_cc_params(param->dev, offset, var);
+	ret = mlx5_ib_set_cc_params(param->dev, param->port_num, offset, var);
 	return ret ? ret : count;
 }
 
@@ -340,7 +362,7 @@ static ssize_t get_param(struct file *filp, char __user *buf, size_t count,
 	if (*pos)
 		return 0;
 
-	ret = mlx5_ib_get_cc_params(param->dev, offset, &var);
+	ret = mlx5_ib_get_cc_params(param->dev, param->port_num, offset, &var);
 	if (ret)
 		return ret;
 
@@ -362,44 +384,51 @@ static const struct file_operations dbg_cc_fops = {
 	.read	= get_param,
 };
 
-void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev)
+void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num)
 {
 	if (!mlx5_debugfs_root ||
-	    !dev->dbg_cc_params ||
-	    !dev->dbg_cc_params->root)
+	    !dev->port[port_num].dbg_cc_params ||
+	    !dev->port[port_num].dbg_cc_params->root)
 		return;
 
-	debugfs_remove_recursive(dev->dbg_cc_params->root);
-	kfree(dev->dbg_cc_params);
-	dev->dbg_cc_params = NULL;
+	debugfs_remove_recursive(dev->port[port_num].dbg_cc_params->root);
+	kfree(dev->port[port_num].dbg_cc_params);
+	dev->port[port_num].dbg_cc_params = NULL;
 }
 
-int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev)
+int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num)
 {
 	struct mlx5_ib_dbg_cc_params *dbg_cc_params;
+	struct mlx5_core_dev *mdev;
 	int i;
 
 	if (!mlx5_debugfs_root)
 		goto out;
 
-	if (!MLX5_CAP_GEN(dev->mdev, cc_query_allowed) ||
-	    !MLX5_CAP_GEN(dev->mdev, cc_modify_allowed))
+	/* Takes a 1-based port number */
+	mdev = mlx5_ib_get_native_port_mdev(dev, port_num + 1, NULL);
+	if (!mdev)
 		goto out;
 
+	if (!MLX5_CAP_GEN(mdev, cc_query_allowed) ||
+	    !MLX5_CAP_GEN(mdev, cc_modify_allowed))
+		goto put_mdev;
+
 	dbg_cc_params = kzalloc(sizeof(*dbg_cc_params), GFP_KERNEL);
 	if (!dbg_cc_params)
-		goto out;
+		goto err;
 
-	dev->dbg_cc_params = dbg_cc_params;
+	dev->port[port_num].dbg_cc_params = dbg_cc_params;
 
 	dbg_cc_params->root = debugfs_create_dir("cc_params",
-						 dev->mdev->priv.dbg_root);
+						 mdev->priv.dbg_root);
 	if (!dbg_cc_params->root)
 		goto err;
 
 	for (i = 0; i < MLX5_IB_DBG_CC_MAX; i++) {
 		dbg_cc_params->params[i].offset = i;
 		dbg_cc_params->params[i].dev = dev;
+		dbg_cc_params->params[i].port_num = port_num;
 		dbg_cc_params->params[i].dentry =
 			debugfs_create_file(mlx5_ib_dbg_cc_name[i],
 					    0600, dbg_cc_params->root,
@@ -408,11 +437,17 @@ int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev)
 		if (!dbg_cc_params->params[i].dentry)
 			goto err;
 	}
-out:	return 0;
+
+put_mdev:
+	mlx5_ib_put_native_port_mdev(dev, port_num + 1);
+out:
+	return 0;
 
 err:
 	mlx5_ib_warn(dev, "cong debugfs failure\n");
-	mlx5_ib_cleanup_cong_debugfs(dev);
+	mlx5_ib_cleanup_cong_debugfs(dev, port_num);
+	mlx5_ib_put_native_port_mdev(dev, port_num + 1);
+
 	/*
 	 * We don't want to fail driver if debugfs failed to initialize,
 	 * so we are not forwarding error to the user.
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 18705cbcdc8c..15457c9569a7 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -226,7 +226,6 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 		wc->ex.invalidate_rkey = be32_to_cpu(cqe->imm_inval_pkey);
 		break;
 	}
-	wc->slid	   = be16_to_cpu(cqe->slid);
 	wc->src_qp	   = be32_to_cpu(cqe->flags_rqpn) & 0xffffff;
 	wc->dlid_path_bits = cqe->ml_path;
 	g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
@@ -241,10 +240,12 @@ static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 	}
 
 	if (ll != IB_LINK_LAYER_ETHERNET) {
+		wc->slid = be16_to_cpu(cqe->slid);
 		wc->sl = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0xf;
 		return;
 	}
 
+	wc->slid = 0;
 	vlan_present = cqe->l4_l3_hdr_type & 0x1;
 	roce_packet_type   = (be32_to_cpu(cqe->flags_rqpn) >> 24) & 0x3;
 	if (vlan_present) {
@@ -1010,7 +1011,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 	MLX5_SET(cqc, cqc, uar_page, index);
 	MLX5_SET(cqc, cqc, c_eqn, eqn);
 	MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma);
-	if (cq->create_flags & IB_CQ_FLAGS_IGNORE_OVERRUN)
+	if (cq->create_flags & IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN)
 		MLX5_SET(cqc, cqc, oi, 1);
 
 	err = mlx5_core_create_cq(dev->mdev, &cq->mcq, cqb, inlen);
@@ -1177,7 +1178,12 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
 	if (ucmd.reserved0 || ucmd.reserved1)
 		return -EINVAL;
 
-	umem = ib_umem_get(context, ucmd.buf_addr, entries * ucmd.cqe_size,
+	/* check multiplication overflow */
+	if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1)
+		return -EINVAL;
+
+	umem = ib_umem_get(context, ucmd.buf_addr,
+			   (size_t)ucmd.cqe_size * entries,
 			   IB_ACCESS_LOCAL_WRITE, 1);
 	if (IS_ERR(umem)) {
 		err = PTR_ERR(umem);
diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index 1003b0133a49..32a9e9228b13 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -197,10 +197,9 @@ static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt,
 			     vl_15_dropped);
 }
 
-static int process_pma_cmd(struct ib_device *ibdev, u8 port_num,
+static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num,
 			   const struct ib_mad *in_mad, struct ib_mad *out_mad)
 {
-	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	int err;
 	void *out_cnt;
 
@@ -222,7 +221,7 @@ static int process_pma_cmd(struct ib_device *ibdev, u8 port_num,
 		if (!out_cnt)
 			return IB_MAD_RESULT_FAILURE;
 
-		err = mlx5_core_query_vport_counter(dev->mdev, 0, 0,
+		err = mlx5_core_query_vport_counter(mdev, 0, 0,
 						    port_num, out_cnt, sz);
 		if (!err)
 			pma_cnt_ext_assign(pma_cnt_ext, out_cnt);
@@ -235,7 +234,7 @@ static int process_pma_cmd(struct ib_device *ibdev, u8 port_num,
 		if (!out_cnt)
 			return IB_MAD_RESULT_FAILURE;
 
-		err = mlx5_core_query_ib_ppcnt(dev->mdev, port_num,
+		err = mlx5_core_query_ib_ppcnt(mdev, port_num,
 					       out_cnt, sz);
 		if (!err)
 			pma_cnt_assign(pma_cnt, out_cnt);
@@ -255,9 +254,11 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 			u16 *out_mad_pkey_index)
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
-	struct mlx5_core_dev *mdev = dev->mdev;
 	const struct ib_mad *in_mad = (const struct ib_mad *)in;
 	struct ib_mad *out_mad = (struct ib_mad *)out;
+	struct mlx5_core_dev *mdev;
+	u8 mdev_port_num;
+	int ret;
 
 	if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||
 			 *out_mad_size != sizeof(*out_mad)))
@@ -265,14 +266,20 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 
 	memset(out_mad->data, 0, sizeof(out_mad->data));
 
+	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
+	if (!mdev)
+		return IB_MAD_RESULT_FAILURE;
+
 	if (MLX5_CAP_GEN(mdev, vport_counters) &&
 	    in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT &&
 	    in_mad->mad_hdr.method == IB_MGMT_METHOD_GET) {
-		return process_pma_cmd(ibdev, port_num, in_mad, out_mad);
+		ret = process_pma_cmd(mdev, mdev_port_num, in_mad, out_mad);
 	} else {
-		return process_mad(ibdev, mad_flags, port_num, in_wc, in_grh,
+		ret =  process_mad(ibdev, mad_flags, port_num, in_wc, in_grh,
 				   in_mad, out_mad);
 	}
+	mlx5_ib_put_native_port_mdev(dev, port_num);
+	return ret;
 }
 
 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port)
@@ -519,7 +526,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 	int ext_active_speed;
 	int err = -ENOMEM;
 
-	if (port < 1 || port > MLX5_CAP_GEN(mdev, num_ports)) {
+	if (port < 1 || port > dev->num_ports) {
 		mlx5_ib_warn(dev, "invalid port number %d\n", port);
 		return -EINVAL;
 	}
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 262c1aa2e028..7f8bda3a2005 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -50,16 +50,14 @@
 #include <rdma/ib_cache.h>
 #include <linux/mlx5/port.h>
 #include <linux/mlx5/vport.h>
+#include <linux/mlx5/fs.h>
 #include <linux/list.h>
 #include <rdma/ib_smi.h>
 #include <rdma/ib_umem.h>
 #include <linux/in.h>
 #include <linux/etherdevice.h>
-#include <linux/mlx5/fs.h>
-#include <linux/mlx5/vport.h>
 #include "mlx5_ib.h"
 #include "cmd.h"
-#include <linux/mlx5/vport.h>
 
 #define DRIVER_NAME "mlx5_ib"
 #define DRIVER_VERSION "5.0-0"
@@ -72,10 +70,36 @@ static char mlx5_version[] =
 	DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
 	DRIVER_VERSION "\n";
 
+struct mlx5_ib_event_work {
+	struct work_struct	work;
+	struct mlx5_core_dev	*dev;
+	void			*context;
+	enum mlx5_dev_event	event;
+	unsigned long		param;
+};
+
 enum {
 	MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
 };
 
+static struct workqueue_struct *mlx5_ib_event_wq;
+static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
+static LIST_HEAD(mlx5_ib_dev_list);
+/*
+ * This mutex should be held when accessing either of the above lists
+ */
+static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
+
+struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
+{
+	struct mlx5_ib_dev *dev;
+
+	mutex_lock(&mlx5_ib_multiport_mutex);
+	dev = mpi->ibdev;
+	mutex_unlock(&mlx5_ib_multiport_mutex);
+	return dev;
+}
+
 static enum rdma_link_layer
 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
 {
@@ -115,24 +139,32 @@ static int get_port_state(struct ib_device *ibdev,
 static int mlx5_netdev_event(struct notifier_block *this,
 			     unsigned long event, void *ptr)
 {
+	struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
-	struct mlx5_ib_dev *ibdev = container_of(this, struct mlx5_ib_dev,
-						 roce.nb);
+	u8 port_num = roce->native_port_num;
+	struct mlx5_core_dev *mdev;
+	struct mlx5_ib_dev *ibdev;
+
+	ibdev = roce->dev;
+	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
+	if (!mdev)
+		return NOTIFY_DONE;
 
 	switch (event) {
 	case NETDEV_REGISTER:
 	case NETDEV_UNREGISTER:
-		write_lock(&ibdev->roce.netdev_lock);
-		if (ndev->dev.parent == &ibdev->mdev->pdev->dev)
-			ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ?
-					     NULL : ndev;
-		write_unlock(&ibdev->roce.netdev_lock);
+		write_lock(&roce->netdev_lock);
+
+		if (ndev->dev.parent == &mdev->pdev->dev)
+			roce->netdev = (event == NETDEV_UNREGISTER) ?
+					NULL : ndev;
+		write_unlock(&roce->netdev_lock);
 		break;
 
 	case NETDEV_CHANGE:
 	case NETDEV_UP:
 	case NETDEV_DOWN: {
-		struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
+		struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(mdev);
 		struct net_device *upper = NULL;
 
 		if (lag_ndev) {
@@ -140,27 +172,28 @@ static int mlx5_netdev_event(struct notifier_block *this,
 			dev_put(lag_ndev);
 		}
 
-		if ((upper == ndev || (!upper && ndev == ibdev->roce.netdev))
+		if ((upper == ndev || (!upper && ndev == roce->netdev))
 		    && ibdev->ib_active) {
 			struct ib_event ibev = { };
 			enum ib_port_state port_state;
 
-			if (get_port_state(&ibdev->ib_dev, 1, &port_state))
-				return NOTIFY_DONE;
+			if (get_port_state(&ibdev->ib_dev, port_num,
+					   &port_state))
+				goto done;
 
-			if (ibdev->roce.last_port_state == port_state)
-				return NOTIFY_DONE;
+			if (roce->last_port_state == port_state)
+				goto done;
 
-			ibdev->roce.last_port_state = port_state;
+			roce->last_port_state = port_state;
 			ibev.device = &ibdev->ib_dev;
 			if (port_state == IB_PORT_DOWN)
 				ibev.event = IB_EVENT_PORT_ERR;
 			else if (port_state == IB_PORT_ACTIVE)
 				ibev.event = IB_EVENT_PORT_ACTIVE;
 			else
-				return NOTIFY_DONE;
+				goto done;
 
-			ibev.element.port_num = 1;
+			ibev.element.port_num = port_num;
 			ib_dispatch_event(&ibev);
 		}
 		break;
@@ -169,7 +202,8 @@ static int mlx5_netdev_event(struct notifier_block *this,
 	default:
 		break;
 	}
-
+done:
+	mlx5_ib_put_native_port_mdev(ibdev, port_num);
 	return NOTIFY_DONE;
 }
 
@@ -178,22 +212,92 @@ static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
 {
 	struct mlx5_ib_dev *ibdev = to_mdev(device);
 	struct net_device *ndev;
+	struct mlx5_core_dev *mdev;
+
+	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
+	if (!mdev)
+		return NULL;
 
-	ndev = mlx5_lag_get_roce_netdev(ibdev->mdev);
+	ndev = mlx5_lag_get_roce_netdev(mdev);
 	if (ndev)
-		return ndev;
+		goto out;
 
 	/* Ensure ndev does not disappear before we invoke dev_hold()
 	 */
-	read_lock(&ibdev->roce.netdev_lock);
-	ndev = ibdev->roce.netdev;
+	read_lock(&ibdev->roce[port_num - 1].netdev_lock);
+	ndev = ibdev->roce[port_num - 1].netdev;
 	if (ndev)
 		dev_hold(ndev);
-	read_unlock(&ibdev->roce.netdev_lock);
+	read_unlock(&ibdev->roce[port_num - 1].netdev_lock);
 
+out:
+	mlx5_ib_put_native_port_mdev(ibdev, port_num);
 	return ndev;
 }
 
+struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
+						   u8 ib_port_num,
+						   u8 *native_port_num)
+{
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
+							  ib_port_num);
+	struct mlx5_core_dev *mdev = NULL;
+	struct mlx5_ib_multiport_info *mpi;
+	struct mlx5_ib_port *port;
+
+	if (!mlx5_core_mp_enabled(ibdev->mdev) ||
+	    ll != IB_LINK_LAYER_ETHERNET) {
+		if (native_port_num)
+			*native_port_num = ib_port_num;
+		return ibdev->mdev;
+	}
+
+	if (native_port_num)
+		*native_port_num = 1;
+
+	port = &ibdev->port[ib_port_num - 1];
+	if (!port)
+		return NULL;
+
+	spin_lock(&port->mp.mpi_lock);
+	mpi = ibdev->port[ib_port_num - 1].mp.mpi;
+	if (mpi && !mpi->unaffiliate) {
+		mdev = mpi->mdev;
+		/* If it's the master no need to refcount, it'll exist
+		 * as long as the ib_dev exists.
+		 */
+		if (!mpi->is_master)
+			mpi->mdev_refcnt++;
+	}
+	spin_unlock(&port->mp.mpi_lock);
+
+	return mdev;
+}
+
+void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u8 port_num)
+{
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
+							  port_num);
+	struct mlx5_ib_multiport_info *mpi;
+	struct mlx5_ib_port *port;
+
+	if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
+		return;
+
+	port = &ibdev->port[port_num - 1];
+
+	spin_lock(&port->mp.mpi_lock);
+	mpi = ibdev->port[port_num - 1].mp.mpi;
+	if (mpi->is_master)
+		goto out;
+
+	mpi->mdev_refcnt--;
+	if (mpi->unaffiliate)
+		complete(&mpi->unref_comp);
+out:
+	spin_unlock(&port->mp.mpi_lock);
+}
+
 static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,
 				    u8 *active_width)
 {
@@ -256,19 +360,33 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 				struct ib_port_attr *props)
 {
 	struct mlx5_ib_dev *dev = to_mdev(device);
-	struct mlx5_core_dev *mdev = dev->mdev;
+	struct mlx5_core_dev *mdev;
 	struct net_device *ndev, *upper;
 	enum ib_mtu ndev_ib_mtu;
+	bool put_mdev = true;
 	u16 qkey_viol_cntr;
 	u32 eth_prot_oper;
+	u8 mdev_port_num;
 	int err;
 
+	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
+	if (!mdev) {
+		/* This means the port isn't affiliated yet. Get the
+		 * info for the master port instead.
+		 */
+		put_mdev = false;
+		mdev = dev->mdev;
+		mdev_port_num = 1;
+		port_num = 1;
+	}
+
 	/* Possible bad flows are checked before filling out props so in case
 	 * of an error it will still be zeroed out.
 	 */
-	err = mlx5_query_port_eth_proto_oper(mdev, &eth_prot_oper, port_num);
+	err = mlx5_query_port_eth_proto_oper(mdev, &eth_prot_oper,
+					     mdev_port_num);
 	if (err)
-		return err;
+		goto out;
 
 	translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
 				 &props->active_width);
@@ -284,12 +402,16 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 	props->state            = IB_PORT_DOWN;
 	props->phys_state       = 3;
 
-	mlx5_query_nic_vport_qkey_viol_cntr(dev->mdev, &qkey_viol_cntr);
+	mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
 	props->qkey_viol_cntr = qkey_viol_cntr;
 
+	/* If this is a stub query for an unaffiliated port stop here */
+	if (!put_mdev)
+		goto out;
+
 	ndev = mlx5_ib_get_netdev(device, port_num);
 	if (!ndev)
-		return 0;
+		goto out;
 
 	if (mlx5_lag_is_active(dev->mdev)) {
 		rcu_read_lock();
@@ -312,7 +434,10 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 	dev_put(ndev);
 
 	props->active_mtu	= min(props->max_mtu, ndev_ib_mtu);
-	return 0;
+out:
+	if (put_mdev)
+		mlx5_ib_put_native_port_mdev(dev, port_num);
+	return err;
 }
 
 static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
@@ -354,7 +479,7 @@ static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
 
 	return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
 				      roce_l3_type, gid->raw, mac, vlan,
-				      vlan_id);
+				      vlan_id, port_num);
 }
 
 static int mlx5_ib_add_gid(struct ib_device *device, u8 port_num,
@@ -438,11 +563,11 @@ static int mlx5_get_vport_access_method(struct ib_device *ibdev)
 }
 
 static void get_atomic_caps(struct mlx5_ib_dev *dev,
+			    u8 atomic_size_qp,
 			    struct ib_device_attr *props)
 {
 	u8 tmp;
 	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
-	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
 	u8 atomic_req_8B_endianness_mode =
 		MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
 
@@ -459,6 +584,29 @@ static void get_atomic_caps(struct mlx5_ib_dev *dev,
 	}
 }
 
+static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
+			       struct ib_device_attr *props)
+{
+	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
+
+	get_atomic_caps(dev, atomic_size_qp, props);
+}
+
+static void get_atomic_caps_dc(struct mlx5_ib_dev *dev,
+			       struct ib_device_attr *props)
+{
+	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
+
+	get_atomic_caps(dev, atomic_size_qp, props);
+}
+
+bool mlx5_ib_dc_atomic_is_supported(struct mlx5_ib_dev *dev)
+{
+	struct ib_device_attr props = {};
+
+	get_atomic_caps_dc(dev, &props);
+	return (props.atomic_cap == IB_ATOMIC_HCA) ? true : false;
+}
 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
 					__be64 *sys_image_guid)
 {
@@ -587,6 +735,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	int max_rq_sg;
 	int max_sq_sg;
 	u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
+	bool raw_support = !mlx5_core_mp_enabled(mdev);
 	struct mlx5_ib_query_device_resp resp = {};
 	size_t resp_len;
 	u64 max_tso;
@@ -650,7 +799,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	if (MLX5_CAP_GEN(mdev, block_lb_mc))
 		props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
 
-	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads)) {
+	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
 		if (MLX5_CAP_ETH(mdev, csum_cap)) {
 			/* Legacy bit to support old userspace libraries */
 			props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
@@ -682,7 +831,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 						MLX5_RX_HASH_SRC_PORT_TCP |
 						MLX5_RX_HASH_DST_PORT_TCP |
 						MLX5_RX_HASH_SRC_PORT_UDP |
-						MLX5_RX_HASH_DST_PORT_UDP;
+						MLX5_RX_HASH_DST_PORT_UDP |
+						MLX5_RX_HASH_INNER;
 			resp.response_length += sizeof(resp.rss_caps);
 		}
 	} else {
@@ -698,7 +848,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	}
 
 	if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
-	    MLX5_CAP_GEN(dev->mdev, general_notification_event))
+	    MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
+	    raw_support)
 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
 
 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
@@ -706,7 +857,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
 
 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
-	    MLX5_CAP_ETH(dev->mdev, scatter_fcs)) {
+	    MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
+	    raw_support) {
 		/* Legacy bit to support old userspace libraries */
 		props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
@@ -746,7 +898,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	props->max_srq_sge	   = max_rq_sg - 1;
 	props->max_fast_reg_page_list_len =
 		1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
-	get_atomic_caps(dev, props);
+	get_atomic_caps_qp(dev, props);
 	props->masked_atomic_cap   = IB_ATOMIC_NONE;
 	props->max_mcast_grp	   = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
 	props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
@@ -770,7 +922,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 		props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
 
 	if (mlx5_ib_port_link_layer(ibdev, 1) ==
-	    IB_LINK_LAYER_ETHERNET) {
+	    IB_LINK_LAYER_ETHERNET && raw_support) {
 		props->rss_caps.max_rwq_indirection_tables =
 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
 		props->rss_caps.max_rwq_indirection_table_size =
@@ -807,7 +959,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 		resp.response_length += sizeof(resp.cqe_comp_caps);
 	}
 
-	if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen)) {
+	if (field_avail(typeof(resp), packet_pacing_caps, uhw->outlen) &&
+	    raw_support) {
 		if (MLX5_CAP_QOS(mdev, packet_pacing) &&
 		    MLX5_CAP_GEN(mdev, qos)) {
 			resp.packet_pacing_caps.qp_rate_limit_max =
@@ -866,7 +1019,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 		}
 	}
 
-	if (field_avail(typeof(resp), striding_rq_caps, uhw->outlen)) {
+	if (field_avail(typeof(resp), striding_rq_caps, uhw->outlen) &&
+	    raw_support) {
 		resp.response_length += sizeof(resp.striding_rq_caps);
 		if (MLX5_CAP_GEN(mdev, striding_rq)) {
 			resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
@@ -1097,7 +1251,22 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
 	}
 
 	if (!ret && props) {
-		count = mlx5_core_reserved_gids_count(to_mdev(ibdev)->mdev);
+		struct mlx5_ib_dev *dev = to_mdev(ibdev);
+		struct mlx5_core_dev *mdev;
+		bool put_mdev = true;
+
+		mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
+		if (!mdev) {
+			/* If the port isn't affiliated yet query the master.
+			 * The master and slave will have the same values.
+			 */
+			mdev = dev->mdev;
+			port = 1;
+			put_mdev = false;
+		}
+		count = mlx5_core_reserved_gids_count(mdev);
+		if (put_mdev)
+			mlx5_ib_put_native_port_mdev(dev, port);
 		props->gid_tbl_len -= count;
 	}
 	return ret;
@@ -1122,20 +1291,43 @@ static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 
 }
 
-static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
-			      u16 *pkey)
+static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u8 port,
+				   u16 index, u16 *pkey)
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
-	struct mlx5_core_dev *mdev = dev->mdev;
+	struct mlx5_core_dev *mdev;
+	bool put_mdev = true;
+	u8 mdev_port_num;
+	int err;
+
+	mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
+	if (!mdev) {
+		/* The port isn't affiliated yet, get the PKey from the master
+		 * port. For RoCE the PKey tables will be the same.
+		 */
+		put_mdev = false;
+		mdev = dev->mdev;
+		mdev_port_num = 1;
+	}
+
+	err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
+					index, pkey);
+	if (put_mdev)
+		mlx5_ib_put_native_port_mdev(dev, port);
+
+	return err;
+}
 
+static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
+			      u16 *pkey)
+{
 	switch (mlx5_get_vport_access_method(ibdev)) {
 	case MLX5_VPORT_ACCESS_METHOD_MAD:
 		return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
 
 	case MLX5_VPORT_ACCESS_METHOD_HCA:
 	case MLX5_VPORT_ACCESS_METHOD_NIC:
-		return mlx5_query_hca_vport_pkey(mdev, 0, port,  0, index,
-						 pkey);
+		return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
 	default:
 		return -EINVAL;
 	}
@@ -1174,23 +1366,32 @@ static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u8 port_num, u32 mask,
 				u32 value)
 {
 	struct mlx5_hca_vport_context ctx = {};
+	struct mlx5_core_dev *mdev;
+	u8 mdev_port_num;
 	int err;
 
-	err = mlx5_query_hca_vport_context(dev->mdev, 0,
-					   port_num, 0, &ctx);
+	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
+	if (!mdev)
+		return -ENODEV;
+
+	err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
 	if (err)
-		return err;
+		goto out;
 
 	if (~ctx.cap_mask1_perm & mask) {
 		mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
 			     mask, ctx.cap_mask1_perm);
-		return -EINVAL;
+		err = -EINVAL;
+		goto out;
 	}
 
 	ctx.cap_mask1 = value;
 	ctx.cap_mask1_perm = mask;
-	err = mlx5_core_modify_hca_vport_context(dev->mdev, 0,
-						 port_num, 0, &ctx);
+	err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
+						 0, &ctx);
+
+out:
+	mlx5_ib_put_native_port_mdev(dev, port_num);
 
 	return err;
 }
@@ -1241,9 +1442,18 @@ static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
 		    caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
 }
 
+static u16 calc_dynamic_bfregs(int uars_per_sys_page)
+{
+	/* Large page with non 4k uar support might limit the dynamic size */
+	if (uars_per_sys_page == 1  && PAGE_SIZE > 4096)
+		return MLX5_MIN_DYN_BFREGS;
+
+	return MLX5_MAX_DYN_BFREGS;
+}
+
 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
 			     struct mlx5_ib_alloc_ucontext_req_v2 *req,
-			     u32 *num_sys_pages)
+			     struct mlx5_bfreg_info *bfregi)
 {
 	int uars_per_sys_page;
 	int bfregs_per_sys_page;
@@ -1260,16 +1470,21 @@ static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
 
 	uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
 	bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
+	/* This holds the required static allocation asked by the user */
 	req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
-	*num_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
-
 	if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
 		return -EINVAL;
 
-	mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, using %d sys pages\n",
+	bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
+	bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
+	bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
+	bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
+
+	mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
 		    MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
 		    lib_uar_4k ? "yes" : "no", ref_bfregs,
-		    req->total_num_bfregs, *num_sys_pages);
+		    req->total_num_bfregs, bfregi->total_num_bfregs,
+		    bfregi->num_sys_pages);
 
 	return 0;
 }
@@ -1281,13 +1496,17 @@ static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *conte
 	int i;
 
 	bfregi = &context->bfregi;
-	for (i = 0; i < bfregi->num_sys_pages; i++) {
+	for (i = 0; i < bfregi->num_static_sys_pages; i++) {
 		err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
 		if (err)
 			goto error;
 
 		mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
 	}
+
+	for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
+		bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
+
 	return 0;
 
 error:
@@ -1306,12 +1525,16 @@ static int deallocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *con
 
 	bfregi = &context->bfregi;
 	for (i = 0; i < bfregi->num_sys_pages; i++) {
-		err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
-		if (err) {
-			mlx5_ib_warn(dev, "failed to free uar %d\n", i);
-			return err;
+		if (i < bfregi->num_static_sys_pages ||
+		    bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX) {
+			err = mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
+			if (err) {
+				mlx5_ib_warn(dev, "failed to free uar %d, err=%d\n", i, err);
+				return err;
+			}
 		}
 	}
+
 	return 0;
 }
 
@@ -1362,6 +1585,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_ib_alloc_ucontext_req_v2 req = {};
 	struct mlx5_ib_alloc_ucontext_resp resp = {};
+	struct mlx5_core_dev *mdev = dev->mdev;
 	struct mlx5_ib_ucontext *context;
 	struct mlx5_bfreg_info *bfregi;
 	int ver;
@@ -1422,13 +1646,13 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	bfregi = &context->bfregi;
 
 	/* updates req->total_num_bfregs */
-	err = calc_total_bfregs(dev, lib_uar_4k, &req, &bfregi->num_sys_pages);
+	err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
 	if (err)
 		goto out_ctx;
 
 	mutex_init(&bfregi->lock);
 	bfregi->lib_uar_4k = lib_uar_4k;
-	bfregi->count = kcalloc(req.total_num_bfregs, sizeof(*bfregi->count),
+	bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
 				GFP_KERNEL);
 	if (!bfregi->count) {
 		err = -ENOMEM;
@@ -1470,7 +1694,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	mutex_init(&context->db_page_mutex);
 
 	resp.tot_bfregs = req.total_num_bfregs;
-	resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
+	resp.num_ports = dev->num_ports;
 
 	if (field_avail(typeof(resp), cqe_version, udata->outlen))
 		resp.response_length += sizeof(resp.cqe_version);
@@ -1489,6 +1713,12 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 		resp.response_length += sizeof(resp.eth_min_inline);
 	}
 
+	if (field_avail(typeof(resp), clock_info_versions, udata->outlen)) {
+		if (mdev->clock_info)
+			resp.clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
+		resp.response_length += sizeof(resp.clock_info_versions);
+	}
+
 	/*
 	 * We don't want to expose information from the PCI bar that is located
 	 * after 4096 bytes, so if the arch only supports larger pages, let's
@@ -1502,8 +1732,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 			resp.hca_core_clock_offset =
 				offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
 		}
-		resp.response_length += sizeof(resp.hca_core_clock_offset) +
-					sizeof(resp.reserved2);
+		resp.response_length += sizeof(resp.hca_core_clock_offset);
 	}
 
 	if (field_avail(typeof(resp), log_uar_size, udata->outlen))
@@ -1512,6 +1741,11 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
 	if (field_avail(typeof(resp), num_uars_per_page, udata->outlen))
 		resp.response_length += sizeof(resp.num_uars_per_page);
 
+	if (field_avail(typeof(resp), num_dyn_bfregs, udata->outlen)) {
+		resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
+		resp.response_length += sizeof(resp.num_dyn_bfregs);
+	}
+
 	err = ib_copy_to_udata(udata, &resp, resp.response_length);
 	if (err)
 		goto out_td;
@@ -1566,15 +1800,13 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 }
 
 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
-				 struct mlx5_bfreg_info *bfregi,
-				 int idx)
+				 int uar_idx)
 {
 	int fw_uars_per_page;
 
 	fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
 
-	return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) +
-			bfregi->sys_pages[idx] / fw_uars_per_page;
+	return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
 }
 
 static int get_command(unsigned long offset)
@@ -1592,6 +1824,12 @@ static int get_index(unsigned long offset)
 	return get_arg(offset);
 }
 
+/* Index resides in an extra byte to enable larger values than 255 */
+static int get_extended_index(unsigned long offset)
+{
+	return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
+}
+
 static void  mlx5_ib_vma_open(struct vm_area_struct *area)
 {
 	/* vma_open is called when a new VMA is created on top of our VMA.  This
@@ -1733,6 +1971,38 @@ static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
 	}
 }
 
+static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
+					struct vm_area_struct *vma,
+					struct mlx5_ib_ucontext *context)
+{
+	phys_addr_t pfn;
+	int err;
+
+	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+		return -EINVAL;
+
+	if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
+		return -EOPNOTSUPP;
+
+	if (vma->vm_flags & VM_WRITE)
+		return -EPERM;
+
+	if (!dev->mdev->clock_info_page)
+		return -EOPNOTSUPP;
+
+	pfn = page_to_pfn(dev->mdev->clock_info_page);
+	err = remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
+			      vma->vm_page_prot);
+	if (err)
+		return err;
+
+	mlx5_ib_dbg(dev, "mapped clock info at 0x%lx, PA 0x%llx\n",
+		    vma->vm_start,
+		    (unsigned long long)pfn << PAGE_SHIFT);
+
+	return mlx5_ib_set_vma_data(vma, context);
+}
+
 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
 		    struct vm_area_struct *vma,
 		    struct mlx5_ib_ucontext *context)
@@ -1742,21 +2012,29 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
 	unsigned long idx;
 	phys_addr_t pfn, pa;
 	pgprot_t prot;
-	int uars_per_page;
+	u32 bfreg_dyn_idx = 0;
+	u32 uar_index;
+	int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
+	int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
+				bfregi->num_static_sys_pages;
 
 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
 		return -EINVAL;
 
-	uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
-	idx = get_index(vma->vm_pgoff);
-	if (idx % uars_per_page ||
-	    idx * uars_per_page >= bfregi->num_sys_pages) {
-		mlx5_ib_warn(dev, "invalid uar index %lu\n", idx);
+	if (dyn_uar)
+		idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
+	else
+		idx = get_index(vma->vm_pgoff);
+
+	if (idx >= max_valid_idx) {
+		mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
+			     idx, max_valid_idx);
 		return -EINVAL;
 	}
 
 	switch (cmd) {
 	case MLX5_IB_MMAP_WC_PAGE:
+	case MLX5_IB_MMAP_ALLOC_WC:
 /* Some architectures don't support WC memory */
 #if defined(CONFIG_X86)
 		if (!pat_enabled())
@@ -1776,7 +2054,40 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
 		return -EINVAL;
 	}
 
-	pfn = uar_index2pfn(dev, bfregi, idx);
+	if (dyn_uar) {
+		int uars_per_page;
+
+		uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
+		bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
+		if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
+			mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
+				     bfreg_dyn_idx, bfregi->total_num_bfregs);
+			return -EINVAL;
+		}
+
+		mutex_lock(&bfregi->lock);
+		/* Fail if uar already allocated, first bfreg index of each
+		 * page holds its count.
+		 */
+		if (bfregi->count[bfreg_dyn_idx]) {
+			mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
+			mutex_unlock(&bfregi->lock);
+			return -EINVAL;
+		}
+
+		bfregi->count[bfreg_dyn_idx]++;
+		mutex_unlock(&bfregi->lock);
+
+		err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index);
+		if (err) {
+			mlx5_ib_warn(dev, "UAR alloc failed\n");
+			goto free_bfreg;
+		}
+	} else {
+		uar_index = bfregi->sys_pages[idx];
+	}
+
+	pfn = uar_index2pfn(dev, uar_index);
 	mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
 
 	vma->vm_page_prot = prot;
@@ -1785,14 +2096,32 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
 	if (err) {
 		mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%lx, pfn=%pa, mmap_cmd=%s\n",
 			    err, vma->vm_start, &pfn, mmap_cmd2str(cmd));
-		return -EAGAIN;
+		err = -EAGAIN;
+		goto err;
 	}
 
 	pa = pfn << PAGE_SHIFT;
 	mlx5_ib_dbg(dev, "mapped %s at 0x%lx, PA %pa\n", mmap_cmd2str(cmd),
 		    vma->vm_start, &pa);
 
-	return mlx5_ib_set_vma_data(vma, context);
+	err = mlx5_ib_set_vma_data(vma, context);
+	if (err)
+		goto err;
+
+	if (dyn_uar)
+		bfregi->sys_pages[idx] = uar_index;
+	return 0;
+
+err:
+	if (!dyn_uar)
+		return err;
+
+	mlx5_cmd_free_uar(dev->mdev, idx);
+
+free_bfreg:
+	mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
+
+	return err;
 }
 
 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
@@ -1807,6 +2136,7 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
 	case MLX5_IB_MMAP_WC_PAGE:
 	case MLX5_IB_MMAP_NC_PAGE:
 	case MLX5_IB_MMAP_REGULAR_PAGE:
+	case MLX5_IB_MMAP_ALLOC_WC:
 		return uar_mmap(dev, command, vma, context);
 
 	case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
@@ -1835,6 +2165,8 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
 			    vma->vm_start,
 			    (unsigned long long)pfn << PAGE_SHIFT);
 		break;
+	case MLX5_IB_MMAP_CLOCK_INFO:
+		return mlx5_ib_mmap_clock_info_page(dev, vma, context);
 
 	default:
 		return -EINVAL;
@@ -2663,7 +2995,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
 		return ERR_PTR(-ENOMEM);
 
 	if (domain != IB_FLOW_DOMAIN_USER ||
-	    flow_attr->port > MLX5_CAP_GEN(dev->mdev, num_ports) ||
+	    flow_attr->port > dev->num_ports ||
 	    (flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP))
 		return ERR_PTR(-EINVAL);
 
@@ -2928,15 +3260,24 @@ static void delay_drop_handler(struct work_struct *work)
 	mutex_unlock(&delay_drop->lock);
 }
 
-static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
-			  enum mlx5_dev_event event, unsigned long param)
+static void mlx5_ib_handle_event(struct work_struct *_work)
 {
-	struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
+	struct mlx5_ib_event_work *work =
+		container_of(_work, struct mlx5_ib_event_work, work);
+	struct mlx5_ib_dev *ibdev;
 	struct ib_event ibev;
 	bool fatal = false;
-	u8 port = 0;
+	u8 port = (u8)work->param;
 
-	switch (event) {
+	if (mlx5_core_is_mp_slave(work->dev)) {
+		ibdev = mlx5_ib_get_ibdev_from_mpi(work->context);
+		if (!ibdev)
+			goto out;
+	} else {
+		ibdev = work->context;
+	}
+
+	switch (work->event) {
 	case MLX5_DEV_EVENT_SYS_ERROR:
 		ibev.event = IB_EVENT_DEVICE_FATAL;
 		mlx5_ib_handle_internal_error(ibdev);
@@ -2946,39 +3287,32 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 	case MLX5_DEV_EVENT_PORT_UP:
 	case MLX5_DEV_EVENT_PORT_DOWN:
 	case MLX5_DEV_EVENT_PORT_INITIALIZED:
-		port = (u8)param;
-
 		/* In RoCE, port up/down events are handled in
 		 * mlx5_netdev_event().
 		 */
 		if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
 			IB_LINK_LAYER_ETHERNET)
-			return;
+			goto out;
 
-		ibev.event = (event == MLX5_DEV_EVENT_PORT_UP) ?
+		ibev.event = (work->event == MLX5_DEV_EVENT_PORT_UP) ?
 			     IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
 		break;
 
 	case MLX5_DEV_EVENT_LID_CHANGE:
 		ibev.event = IB_EVENT_LID_CHANGE;
-		port = (u8)param;
 		break;
 
 	case MLX5_DEV_EVENT_PKEY_CHANGE:
 		ibev.event = IB_EVENT_PKEY_CHANGE;
-		port = (u8)param;
-
 		schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
 		break;
 
 	case MLX5_DEV_EVENT_GUID_CHANGE:
 		ibev.event = IB_EVENT_GID_CHANGE;
-		port = (u8)param;
 		break;
 
 	case MLX5_DEV_EVENT_CLIENT_REREG:
 		ibev.event = IB_EVENT_CLIENT_REREGISTER;
-		port = (u8)param;
 		break;
 	case MLX5_DEV_EVENT_DELAY_DROP_TIMEOUT:
 		schedule_work(&ibdev->delay_drop.delay_drop_work);
@@ -2990,7 +3324,7 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 	ibev.device	      = &ibdev->ib_dev;
 	ibev.element.port_num = port;
 
-	if (port < 1 || port > ibdev->num_ports) {
+	if (!rdma_is_port_valid(&ibdev->ib_dev, port)) {
 		mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
 		goto out;
 	}
@@ -3000,9 +3334,26 @@ static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
 
 	if (fatal)
 		ibdev->ib_active = false;
-
 out:
-	return;
+	kfree(work);
+}
+
+static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
+			  enum mlx5_dev_event event, unsigned long param)
+{
+	struct mlx5_ib_event_work *work;
+
+	work = kmalloc(sizeof(*work), GFP_ATOMIC);
+	if (!work)
+		return;
+
+	INIT_WORK(&work->work, mlx5_ib_handle_event);
+	work->dev = dev;
+	work->param = param;
+	work->context = context;
+	work->event = event;
+
+	queue_work(mlx5_ib_event_wq, &work->work);
 }
 
 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
@@ -3011,7 +3362,7 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)
 	int err;
 	int port;
 
-	for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
+	for (port = 1; port <= dev->num_ports; port++) {
 		dev->mdev->port_caps[port - 1].has_smi = false;
 		if (MLX5_CAP_GEN(dev->mdev, port_type) ==
 		    MLX5_CAP_PORT_TYPE_IB) {
@@ -3038,16 +3389,15 @@ static void get_ext_port_caps(struct mlx5_ib_dev *dev)
 {
 	int port;
 
-	for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++)
+	for (port = 1; port <= dev->num_ports; port++)
 		mlx5_query_ext_port_caps(dev, port);
 }
 
-static int get_port_caps(struct mlx5_ib_dev *dev)
+static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 {
 	struct ib_device_attr *dprops = NULL;
 	struct ib_port_attr *pprops = NULL;
 	int err = -ENOMEM;
-	int port;
 	struct ib_udata uhw = {.inlen = 0, .outlen = 0};
 
 	pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
@@ -3068,22 +3418,21 @@ static int get_port_caps(struct mlx5_ib_dev *dev)
 		goto out;
 	}
 
-	for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
-		memset(pprops, 0, sizeof(*pprops));
-		err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
-		if (err) {
-			mlx5_ib_warn(dev, "query_port %d failed %d\n",
-				     port, err);
-			break;
-		}
-		dev->mdev->port_caps[port - 1].pkey_table_len =
-						dprops->max_pkeys;
-		dev->mdev->port_caps[port - 1].gid_table_len =
-						pprops->gid_tbl_len;
-		mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
-			    dprops->max_pkeys, pprops->gid_tbl_len);
+	memset(pprops, 0, sizeof(*pprops));
+	err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
+	if (err) {
+		mlx5_ib_warn(dev, "query_port %d failed %d\n",
+			     port, err);
+		goto out;
 	}
 
+	dev->mdev->port_caps[port - 1].pkey_table_len =
+					dprops->max_pkeys;
+	dev->mdev->port_caps[port - 1].gid_table_len =
+					pprops->gid_tbl_len;
+	mlx5_ib_dbg(dev, "port %d: pkey_table_len %d, gid_table_len %d\n",
+		    port, dprops->max_pkeys, pprops->gid_tbl_len);
+
 out:
 	kfree(pprops);
 	kfree(dprops);
@@ -3099,9 +3448,12 @@ static void destroy_umrc_res(struct mlx5_ib_dev *dev)
 	if (err)
 		mlx5_ib_warn(dev, "mr cache cleanup failed\n");
 
-	mlx5_ib_destroy_qp(dev->umrc.qp);
-	ib_free_cq(dev->umrc.cq);
-	ib_dealloc_pd(dev->umrc.pd);
+	if (dev->umrc.qp)
+		mlx5_ib_destroy_qp(dev->umrc.qp);
+	if (dev->umrc.cq)
+		ib_free_cq(dev->umrc.cq);
+	if (dev->umrc.pd)
+		ib_dealloc_pd(dev->umrc.pd);
 }
 
 enum {
@@ -3203,12 +3555,15 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
 
 error_4:
 	mlx5_ib_destroy_qp(qp);
+	dev->umrc.qp = NULL;
 
 error_3:
 	ib_free_cq(cq);
+	dev->umrc.cq = NULL;
 
 error_2:
 	ib_dealloc_pd(pd);
+	dev->umrc.pd = NULL;
 
 error_0:
 	kfree(attr);
@@ -3373,12 +3728,14 @@ static u32 get_core_cap_flags(struct ib_device *ibdev)
 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
 	u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
 	u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
+	bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
 	u32 ret = 0;
 
 	if (ll == IB_LINK_LAYER_INFINIBAND)
 		return RDMA_CORE_PORT_IBA_IB;
 
-	ret = RDMA_CORE_PORT_RAW_PACKET;
+	if (raw_support)
+		ret = RDMA_CORE_PORT_RAW_PACKET;
 
 	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
 		return ret;
@@ -3468,33 +3825,33 @@ static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
 	}
 }
 
-static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev)
+static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
 {
 	int err;
 
-	dev->roce.nb.notifier_call = mlx5_netdev_event;
-	err = register_netdevice_notifier(&dev->roce.nb);
+	dev->roce[port_num].nb.notifier_call = mlx5_netdev_event;
+	err = register_netdevice_notifier(&dev->roce[port_num].nb);
 	if (err) {
-		dev->roce.nb.notifier_call = NULL;
+		dev->roce[port_num].nb.notifier_call = NULL;
 		return err;
 	}
 
 	return 0;
 }
 
-static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev)
+static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
 {
-	if (dev->roce.nb.notifier_call) {
-		unregister_netdevice_notifier(&dev->roce.nb);
-		dev->roce.nb.notifier_call = NULL;
+	if (dev->roce[port_num].nb.notifier_call) {
+		unregister_netdevice_notifier(&dev->roce[port_num].nb);
+		dev->roce[port_num].nb.notifier_call = NULL;
 	}
 }
 
-static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
+static int mlx5_enable_eth(struct mlx5_ib_dev *dev, u8 port_num)
 {
 	int err;
 
-	err = mlx5_add_netdev_notifier(dev);
+	err = mlx5_add_netdev_notifier(dev, port_num);
 	if (err)
 		return err;
 
@@ -3515,7 +3872,7 @@ static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
 		mlx5_nic_vport_disable_roce(dev->mdev);
 
 err_unregister_netdevice_notifier:
-	mlx5_remove_netdev_notifier(dev);
+	mlx5_remove_netdev_notifier(dev, port_num);
 	return err;
 }
 
@@ -3577,11 +3934,12 @@ static const struct mlx5_ib_counter extended_err_cnts[] = {
 
 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
 {
-	unsigned int i;
+	int i;
 
 	for (i = 0; i < dev->num_ports; i++) {
-		mlx5_core_dealloc_q_counter(dev->mdev,
-					    dev->port[i].cnts.set_id);
+		if (dev->port[i].cnts.set_id)
+			mlx5_core_dealloc_q_counter(dev->mdev,
+						    dev->port[i].cnts.set_id);
 		kfree(dev->port[i].cnts.names);
 		kfree(dev->port[i].cnts.offsets);
 	}
@@ -3623,6 +3981,7 @@ static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
 
 err_names:
 	kfree(cnts->names);
+	cnts->names = NULL;
 	return -ENOMEM;
 }
 
@@ -3669,37 +4028,33 @@ static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
 
 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
 {
+	int err = 0;
 	int i;
-	int ret;
 
 	for (i = 0; i < dev->num_ports; i++) {
-		struct mlx5_ib_port *port = &dev->port[i];
+		err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
+		if (err)
+			goto err_alloc;
+
+		mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
+				      dev->port[i].cnts.offsets);
 
-		ret = mlx5_core_alloc_q_counter(dev->mdev,
-						&port->cnts.set_id);
-		if (ret) {
+		err = mlx5_core_alloc_q_counter(dev->mdev,
+						&dev->port[i].cnts.set_id);
+		if (err) {
 			mlx5_ib_warn(dev,
 				     "couldn't allocate queue counter for port %d, err %d\n",
-				     i + 1, ret);
-			goto dealloc_counters;
+				     i + 1, err);
+			goto err_alloc;
 		}
-
-		ret = __mlx5_ib_alloc_counters(dev, &port->cnts);
-		if (ret)
-			goto dealloc_counters;
-
-		mlx5_ib_fill_counters(dev, port->cnts.names,
-				      port->cnts.offsets);
+		dev->port[i].cnts.set_id_valid = true;
 	}
 
 	return 0;
 
-dealloc_counters:
-	while (--i >= 0)
-		mlx5_core_dealloc_q_counter(dev->mdev,
-					    dev->port[i].cnts.set_id);
-
-	return ret;
+err_alloc:
+	mlx5_ib_dealloc_counters(dev);
+	return err;
 }
 
 static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
@@ -3718,7 +4073,7 @@ static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
 					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
 }
 
-static int mlx5_ib_query_q_counters(struct mlx5_ib_dev *dev,
+static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
 				    struct mlx5_ib_port *port,
 				    struct rdma_hw_stats *stats)
 {
@@ -3731,7 +4086,7 @@ static int mlx5_ib_query_q_counters(struct mlx5_ib_dev *dev,
 	if (!out)
 		return -ENOMEM;
 
-	ret = mlx5_core_query_q_counter(dev->mdev,
+	ret = mlx5_core_query_q_counter(mdev,
 					port->cnts.set_id, 0,
 					out, outlen);
 	if (ret)
@@ -3753,28 +4108,43 @@ static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_ib_port *port = &dev->port[port_num - 1];
+	struct mlx5_core_dev *mdev;
 	int ret, num_counters;
+	u8 mdev_port_num;
 
 	if (!stats)
 		return -EINVAL;
 
-	ret = mlx5_ib_query_q_counters(dev, port, stats);
+	num_counters = port->cnts.num_q_counters + port->cnts.num_cong_counters;
+
+	/* q_counters are per IB device, query the master mdev */
+	ret = mlx5_ib_query_q_counters(dev->mdev, port, stats);
 	if (ret)
 		return ret;
-	num_counters = port->cnts.num_q_counters;
 
 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
+		mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
+						    &mdev_port_num);
+		if (!mdev) {
+			/* If port is not affiliated yet, its in down state
+			 * which doesn't have any counters yet, so it would be
+			 * zero. So no need to read from the HCA.
+			 */
+			goto done;
+		}
 		ret = mlx5_lag_query_cong_counters(dev->mdev,
 						   stats->value +
 						   port->cnts.num_q_counters,
 						   port->cnts.num_cong_counters,
 						   port->cnts.offsets +
 						   port->cnts.num_q_counters);
+
+		mlx5_ib_put_native_port_mdev(dev, port_num);
 		if (ret)
 			return ret;
-		num_counters += port->cnts.num_cong_counters;
 	}
 
+done:
 	return num_counters;
 }
 
@@ -3936,64 +4306,303 @@ mlx5_ib_get_vector_affinity(struct ib_device *ibdev, int comp_vector)
 	return mlx5_get_vector_affinity(dev->mdev, comp_vector);
 }
 
-static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
+/* The mlx5_ib_multiport_mutex should be held when calling this function */
+static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
+				      struct mlx5_ib_multiport_info *mpi)
 {
-	struct mlx5_ib_dev *dev;
-	enum rdma_link_layer ll;
-	int port_type_cap;
-	const char *name;
+	u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
+	struct mlx5_ib_port *port = &ibdev->port[port_num];
+	int comps;
 	int err;
 	int i;
 
-	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
-	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
+	mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
 
-	printk_once(KERN_INFO "%s", mlx5_version);
+	spin_lock(&port->mp.mpi_lock);
+	if (!mpi->ibdev) {
+		spin_unlock(&port->mp.mpi_lock);
+		return;
+	}
+	mpi->ibdev = NULL;
 
-	dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
-	if (!dev)
-		return NULL;
+	spin_unlock(&port->mp.mpi_lock);
+	mlx5_remove_netdev_notifier(ibdev, port_num);
+	spin_lock(&port->mp.mpi_lock);
 
-	dev->mdev = mdev;
+	comps = mpi->mdev_refcnt;
+	if (comps) {
+		mpi->unaffiliate = true;
+		init_completion(&mpi->unref_comp);
+		spin_unlock(&port->mp.mpi_lock);
 
-	dev->port = kcalloc(MLX5_CAP_GEN(mdev, num_ports), sizeof(*dev->port),
-			    GFP_KERNEL);
-	if (!dev->port)
-		goto err_dealloc;
+		for (i = 0; i < comps; i++)
+			wait_for_completion(&mpi->unref_comp);
+
+		spin_lock(&port->mp.mpi_lock);
+		mpi->unaffiliate = false;
+	}
+
+	port->mp.mpi = NULL;
+
+	list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
 
-	rwlock_init(&dev->roce.netdev_lock);
-	err = get_port_caps(dev);
+	spin_unlock(&port->mp.mpi_lock);
+
+	err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
+
+	mlx5_ib_dbg(ibdev, "unaffiliated port %d\n", port_num + 1);
+	/* Log an error, still needed to cleanup the pointers and add
+	 * it back to the list.
+	 */
 	if (err)
-		goto err_free_port;
+		mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
+			    port_num + 1);
 
-	if (mlx5_use_mad_ifc(dev))
-		get_ext_port_caps(dev);
+	ibdev->roce[port_num].last_port_state = IB_PORT_DOWN;
+}
 
-	if (!mlx5_lag_is_active(mdev))
-		name = "mlx5_%d";
-	else
-		name = "mlx5_bond_%d";
+/* The mlx5_ib_multiport_mutex should be held when calling this function */
+static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
+				    struct mlx5_ib_multiport_info *mpi)
+{
+	u8 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
+	int err;
 
-	strlcpy(dev->ib_dev.name, name, IB_DEVICE_NAME_MAX);
-	dev->ib_dev.owner		= THIS_MODULE;
-	dev->ib_dev.node_type		= RDMA_NODE_IB_CA;
-	dev->ib_dev.local_dma_lkey	= 0 /* not supported for now */;
-	dev->num_ports		= MLX5_CAP_GEN(mdev, num_ports);
-	dev->ib_dev.phys_port_cnt     = dev->num_ports;
-	dev->ib_dev.num_comp_vectors    =
-		dev->mdev->priv.eq_table.num_comp_vectors;
-	dev->ib_dev.dev.parent		= &mdev->pdev->dev;
+	spin_lock(&ibdev->port[port_num].mp.mpi_lock);
+	if (ibdev->port[port_num].mp.mpi) {
+		mlx5_ib_warn(ibdev, "port %d already affiliated.\n",
+			     port_num + 1);
+		spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
+		return false;
+	}
 
-	dev->ib_dev.uverbs_abi_ver	= MLX5_IB_UVERBS_ABI_VERSION;
-	dev->ib_dev.uverbs_cmd_mask	=
-		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
-		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
-		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
-		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
-		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
-		(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
-		(1ull << IB_USER_VERBS_CMD_DESTROY_AH)		|
-		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
+	ibdev->port[port_num].mp.mpi = mpi;
+	mpi->ibdev = ibdev;
+	spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
+
+	err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
+	if (err)
+		goto unbind;
+
+	err = get_port_caps(ibdev, mlx5_core_native_port_num(mpi->mdev));
+	if (err)
+		goto unbind;
+
+	err = mlx5_add_netdev_notifier(ibdev, port_num);
+	if (err) {
+		mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
+			    port_num + 1);
+		goto unbind;
+	}
+
+	err = mlx5_ib_init_cong_debugfs(ibdev, port_num);
+	if (err)
+		goto unbind;
+
+	return true;
+
+unbind:
+	mlx5_ib_unbind_slave_port(ibdev, mpi);
+	return false;
+}
+
+static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
+{
+	int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
+							  port_num + 1);
+	struct mlx5_ib_multiport_info *mpi;
+	int err;
+	int i;
+
+	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
+		return 0;
+
+	err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
+						     &dev->sys_image_guid);
+	if (err)
+		return err;
+
+	err = mlx5_nic_vport_enable_roce(dev->mdev);
+	if (err)
+		return err;
+
+	mutex_lock(&mlx5_ib_multiport_mutex);
+	for (i = 0; i < dev->num_ports; i++) {
+		bool bound = false;
+
+		/* build a stub multiport info struct for the native port. */
+		if (i == port_num) {
+			mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
+			if (!mpi) {
+				mutex_unlock(&mlx5_ib_multiport_mutex);
+				mlx5_nic_vport_disable_roce(dev->mdev);
+				return -ENOMEM;
+			}
+
+			mpi->is_master = true;
+			mpi->mdev = dev->mdev;
+			mpi->sys_image_guid = dev->sys_image_guid;
+			dev->port[i].mp.mpi = mpi;
+			mpi->ibdev = dev;
+			mpi = NULL;
+			continue;
+		}
+
+		list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
+				    list) {
+			if (dev->sys_image_guid == mpi->sys_image_guid &&
+			    (mlx5_core_native_port_num(mpi->mdev) - 1) == i) {
+				bound = mlx5_ib_bind_slave_port(dev, mpi);
+			}
+
+			if (bound) {
+				dev_dbg(&mpi->mdev->pdev->dev, "removing port from unaffiliated list.\n");
+				mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
+				list_del(&mpi->list);
+				break;
+			}
+		}
+		if (!bound) {
+			get_port_caps(dev, i + 1);
+			mlx5_ib_dbg(dev, "no free port found for port %d\n",
+				    i + 1);
+		}
+	}
+
+	list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
+	mutex_unlock(&mlx5_ib_multiport_mutex);
+	return err;
+}
+
+static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
+{
+	int port_num = mlx5_core_native_port_num(dev->mdev) - 1;
+	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
+							  port_num + 1);
+	int i;
+
+	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
+		return;
+
+	mutex_lock(&mlx5_ib_multiport_mutex);
+	for (i = 0; i < dev->num_ports; i++) {
+		if (dev->port[i].mp.mpi) {
+			/* Destroy the native port stub */
+			if (i == port_num) {
+				kfree(dev->port[i].mp.mpi);
+				dev->port[i].mp.mpi = NULL;
+			} else {
+				mlx5_ib_dbg(dev, "unbinding port_num: %d\n", i + 1);
+				mlx5_ib_unbind_slave_port(dev, dev->port[i].mp.mpi);
+			}
+		}
+	}
+
+	mlx5_ib_dbg(dev, "removing from devlist\n");
+	list_del(&dev->ib_dev_list);
+	mutex_unlock(&mlx5_ib_multiport_mutex);
+
+	mlx5_nic_vport_disable_roce(dev->mdev);
+}
+
+static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
+{
+	mlx5_ib_cleanup_multiport_master(dev);
+#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+	cleanup_srcu_struct(&dev->mr_srcu);
+#endif
+	kfree(dev->port);
+}
+
+static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_core_dev *mdev = dev->mdev;
+	const char *name;
+	int err;
+	int i;
+
+	dev->port = kcalloc(dev->num_ports, sizeof(*dev->port),
+			    GFP_KERNEL);
+	if (!dev->port)
+		return -ENOMEM;
+
+	for (i = 0; i < dev->num_ports; i++) {
+		spin_lock_init(&dev->port[i].mp.mpi_lock);
+		rwlock_init(&dev->roce[i].netdev_lock);
+	}
+
+	err = mlx5_ib_init_multiport_master(dev);
+	if (err)
+		goto err_free_port;
+
+	if (!mlx5_core_mp_enabled(mdev)) {
+		int i;
+
+		for (i = 1; i <= dev->num_ports; i++) {
+			err = get_port_caps(dev, i);
+			if (err)
+				break;
+		}
+	} else {
+		err = get_port_caps(dev, mlx5_core_native_port_num(mdev));
+	}
+	if (err)
+		goto err_mp;
+
+	if (mlx5_use_mad_ifc(dev))
+		get_ext_port_caps(dev);
+
+	if (!mlx5_lag_is_active(mdev))
+		name = "mlx5_%d";
+	else
+		name = "mlx5_bond_%d";
+
+	strlcpy(dev->ib_dev.name, name, IB_DEVICE_NAME_MAX);
+	dev->ib_dev.owner		= THIS_MODULE;
+	dev->ib_dev.node_type		= RDMA_NODE_IB_CA;
+	dev->ib_dev.local_dma_lkey	= 0 /* not supported for now */;
+	dev->ib_dev.phys_port_cnt	= dev->num_ports;
+	dev->ib_dev.num_comp_vectors    =
+		dev->mdev->priv.eq_table.num_comp_vectors;
+	dev->ib_dev.dev.parent		= &mdev->pdev->dev;
+
+	mutex_init(&dev->flow_db.lock);
+	mutex_init(&dev->cap_mask_mutex);
+	INIT_LIST_HEAD(&dev->qp_list);
+	spin_lock_init(&dev->reset_flow_resource_lock);
+
+#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+	err = init_srcu_struct(&dev->mr_srcu);
+	if (err)
+		goto err_free_port;
+#endif
+
+	return 0;
+err_mp:
+	mlx5_ib_cleanup_multiport_master(dev);
+
+err_free_port:
+	kfree(dev->port);
+
+	return -ENOMEM;
+}
+
+static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_core_dev *mdev = dev->mdev;
+	int err;
+
+	dev->ib_dev.uverbs_abi_ver	= MLX5_IB_UVERBS_ABI_VERSION;
+	dev->ib_dev.uverbs_cmd_mask	=
+		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
+		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
+		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
+		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_AH)		|
+		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
 		(1ull << IB_USER_VERBS_CMD_REREG_MR)		|
 		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
 		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|
@@ -4022,8 +4631,6 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	dev->ib_dev.query_device	= mlx5_ib_query_device;
 	dev->ib_dev.query_port		= mlx5_ib_query_port;
 	dev->ib_dev.get_link_layer	= mlx5_ib_port_link_layer;
-	if (ll == IB_LINK_LAYER_ETHERNET)
-		dev->ib_dev.get_netdev	= mlx5_ib_get_netdev;
 	dev->ib_dev.query_gid		= mlx5_ib_query_gid;
 	dev->ib_dev.add_gid		= mlx5_ib_add_gid;
 	dev->ib_dev.del_gid		= mlx5_ib_del_gid;
@@ -4080,8 +4687,6 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 
 	dev->ib_dev.disassociate_ucontext = mlx5_ib_disassociate_ucontext;
 
-	mlx5_ib_internal_fill_odp_caps(dev);
-
 	dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
 
 	if (MLX5_CAP_GEN(mdev, imaicl)) {
@@ -4092,11 +4697,6 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 			(1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
 	}
 
-	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
-		dev->ib_dev.get_hw_stats	= mlx5_ib_get_hw_stats;
-		dev->ib_dev.alloc_hw_stats	= mlx5_ib_alloc_hw_stats;
-	}
-
 	if (MLX5_CAP_GEN(mdev, xrc)) {
 		dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
 		dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
@@ -4111,8 +4711,39 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 			(1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
 			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
 
-	if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
-	    IB_LINK_LAYER_ETHERNET) {
+	err = init_node_data(dev);
+	if (err)
+		return err;
+
+	if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
+	    (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
+	     MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
+		mutex_init(&dev->lb_mutex);
+
+	return 0;
+}
+
+static int mlx5_ib_stage_roce_init(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_core_dev *mdev = dev->mdev;
+	enum rdma_link_layer ll;
+	int port_type_cap;
+	u8 port_num;
+	int err;
+	int i;
+
+	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
+	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
+	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
+
+	if (ll == IB_LINK_LAYER_ETHERNET) {
+		for (i = 0; i < dev->num_ports; i++) {
+			dev->roce[i].dev = dev;
+			dev->roce[i].native_port_num = i + 1;
+			dev->roce[i].last_port_state = IB_PORT_DOWN;
+		}
+
+		dev->ib_dev.get_netdev	= mlx5_ib_get_netdev;
 		dev->ib_dev.create_wq	 = mlx5_ib_create_wq;
 		dev->ib_dev.modify_wq	 = mlx5_ib_modify_wq;
 		dev->ib_dev.destroy_wq	 = mlx5_ib_destroy_wq;
@@ -4124,143 +4755,332 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
 			(1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
 			(1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
+		err = mlx5_enable_eth(dev, port_num);
+		if (err)
+			return err;
 	}
-	err = init_node_data(dev);
-	if (err)
-		goto err_free_port;
 
-	mutex_init(&dev->flow_db.lock);
-	mutex_init(&dev->cap_mask_mutex);
-	INIT_LIST_HEAD(&dev->qp_list);
-	spin_lock_init(&dev->reset_flow_resource_lock);
+	return 0;
+}
+
+static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)
+{
+	struct mlx5_core_dev *mdev = dev->mdev;
+	enum rdma_link_layer ll;
+	int port_type_cap;
+	u8 port_num;
+
+	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
+	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
+	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
 
 	if (ll == IB_LINK_LAYER_ETHERNET) {
-		err = mlx5_enable_eth(dev);
-		if (err)
-			goto err_free_port;
-		dev->roce.last_port_state = IB_PORT_DOWN;
+		mlx5_disable_eth(dev);
+		mlx5_remove_netdev_notifier(dev, port_num);
 	}
+}
 
-	err = create_dev_resources(&dev->devr);
-	if (err)
-		goto err_disable_eth;
+static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
+{
+	return create_dev_resources(&dev->devr);
+}
 
-	err = mlx5_ib_odp_init_one(dev);
-	if (err)
-		goto err_rsrc;
+static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
+{
+	destroy_dev_resources(&dev->devr);
+}
+
+static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)
+{
+	mlx5_ib_internal_fill_odp_caps(dev);
 
+	return mlx5_ib_odp_init_one(dev);
+}
+
+static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
+{
 	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
-		err = mlx5_ib_alloc_counters(dev);
-		if (err)
-			goto err_odp;
+		dev->ib_dev.get_hw_stats	= mlx5_ib_get_hw_stats;
+		dev->ib_dev.alloc_hw_stats	= mlx5_ib_alloc_hw_stats;
+
+		return mlx5_ib_alloc_counters(dev);
 	}
 
-	err = mlx5_ib_init_cong_debugfs(dev);
-	if (err)
-		goto err_cnt;
+	return 0;
+}
+
+static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
+{
+	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
+		mlx5_ib_dealloc_counters(dev);
+}
+
+static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
+{
+	return mlx5_ib_init_cong_debugfs(dev,
+					 mlx5_core_native_port_num(dev->mdev) - 1);
+}
+
+static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
+{
+	mlx5_ib_cleanup_cong_debugfs(dev,
+				     mlx5_core_native_port_num(dev->mdev) - 1);
+}
 
+static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)
+{
 	dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
-	if (IS_ERR(dev->mdev->priv.uar))
-		goto err_cong;
+	if (!dev->mdev->priv.uar)
+		return -ENOMEM;
+	return 0;
+}
+
+static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
+{
+	mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
+}
+
+static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
+{
+	int err;
 
 	err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
 	if (err)
-		goto err_uar_page;
+		return err;
 
 	err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
 	if (err)
-		goto err_bfreg;
+		mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
 
-	err = ib_register_device(&dev->ib_dev, NULL);
-	if (err)
-		goto err_fp_bfreg;
+	return err;
+}
 
-	err = create_umr_res(dev);
-	if (err)
-		goto err_dev;
+static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
+{
+	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
+	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
+}
 
+static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
+{
+	return ib_register_device(&dev->ib_dev, NULL);
+}
+
+static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
+{
+	destroy_umrc_res(dev);
+}
+
+static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
+{
+	ib_unregister_device(&dev->ib_dev);
+}
+
+static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
+{
+	return create_umr_res(dev);
+}
+
+static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
+{
 	init_delay_drop(dev);
 
+	return 0;
+}
+
+static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
+{
+	cancel_delay_drop(dev);
+}
+
+static int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev)
+{
+	int err;
+	int i;
+
 	for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
 		err = device_create_file(&dev->ib_dev.dev,
 					 mlx5_class_attributes[i]);
 		if (err)
-			goto err_delay_drop;
+			return err;
 	}
 
-	if ((MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
-	    (MLX5_CAP_GEN(mdev, disable_local_lb_uc) ||
-	     MLX5_CAP_GEN(mdev, disable_local_lb_mc)))
-		mutex_init(&dev->lb_mutex);
+	return 0;
+}
+
+static void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
+			     const struct mlx5_ib_profile *profile,
+			     int stage)
+{
+	/* Number of stages to cleanup */
+	while (stage) {
+		stage--;
+		if (profile->stage[stage].cleanup)
+			profile->stage[stage].cleanup(dev);
+	}
+
+	ib_dealloc_device((struct ib_device *)dev);
+}
+
+static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num);
 
+static void *__mlx5_ib_add(struct mlx5_core_dev *mdev,
+			   const struct mlx5_ib_profile *profile)
+{
+	struct mlx5_ib_dev *dev;
+	int err;
+	int i;
+
+	printk_once(KERN_INFO "%s", mlx5_version);
+
+	dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
+	if (!dev)
+		return NULL;
+
+	dev->mdev = mdev;
+	dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
+			     MLX5_CAP_GEN(mdev, num_vhca_ports));
+
+	for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
+		if (profile->stage[i].init) {
+			err = profile->stage[i].init(dev);
+			if (err)
+				goto err_out;
+		}
+	}
+
+	dev->profile = profile;
 	dev->ib_active = true;
 
 	return dev;
 
-err_delay_drop:
-	cancel_delay_drop(dev);
-	destroy_umrc_res(dev);
+err_out:
+	__mlx5_ib_remove(dev, profile, i);
 
-err_dev:
-	ib_unregister_device(&dev->ib_dev);
+	return NULL;
+}
 
-err_fp_bfreg:
-	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
+static const struct mlx5_ib_profile pf_profile = {
+	STAGE_CREATE(MLX5_IB_STAGE_INIT,
+		     mlx5_ib_stage_init_init,
+		     mlx5_ib_stage_init_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
+		     mlx5_ib_stage_caps_init,
+		     NULL),
+	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
+		     mlx5_ib_stage_roce_init,
+		     mlx5_ib_stage_roce_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
+		     mlx5_ib_stage_dev_res_init,
+		     mlx5_ib_stage_dev_res_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_ODP,
+		     mlx5_ib_stage_odp_init,
+		     NULL),
+	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
+		     mlx5_ib_stage_counters_init,
+		     mlx5_ib_stage_counters_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
+		     mlx5_ib_stage_cong_debugfs_init,
+		     mlx5_ib_stage_cong_debugfs_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_UAR,
+		     mlx5_ib_stage_uar_init,
+		     mlx5_ib_stage_uar_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
+		     mlx5_ib_stage_bfrag_init,
+		     mlx5_ib_stage_bfrag_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
+		     NULL,
+		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
+		     mlx5_ib_stage_ib_reg_init,
+		     mlx5_ib_stage_ib_reg_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
+		     mlx5_ib_stage_post_ib_reg_umr_init,
+		     NULL),
+	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
+		     mlx5_ib_stage_delay_drop_init,
+		     mlx5_ib_stage_delay_drop_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR,
+		     mlx5_ib_stage_class_attr_init,
+		     NULL),
+};
 
-err_bfreg:
-	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
+static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num)
+{
+	struct mlx5_ib_multiport_info *mpi;
+	struct mlx5_ib_dev *dev;
+	bool bound = false;
+	int err;
 
-err_uar_page:
-	mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
+	mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
+	if (!mpi)
+		return NULL;
 
-err_cong:
-	mlx5_ib_cleanup_cong_debugfs(dev);
-err_cnt:
-	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
-		mlx5_ib_dealloc_counters(dev);
+	mpi->mdev = mdev;
 
-err_odp:
-	mlx5_ib_odp_remove_one(dev);
+	err = mlx5_query_nic_vport_system_image_guid(mdev,
+						     &mpi->sys_image_guid);
+	if (err) {
+		kfree(mpi);
+		return NULL;
+	}
 
-err_rsrc:
-	destroy_dev_resources(&dev->devr);
+	mutex_lock(&mlx5_ib_multiport_mutex);
+	list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
+		if (dev->sys_image_guid == mpi->sys_image_guid)
+			bound = mlx5_ib_bind_slave_port(dev, mpi);
 
-err_disable_eth:
-	if (ll == IB_LINK_LAYER_ETHERNET) {
-		mlx5_disable_eth(dev);
-		mlx5_remove_netdev_notifier(dev);
+		if (bound) {
+			rdma_roce_rescan_device(&dev->ib_dev);
+			break;
+		}
 	}
 
-err_free_port:
-	kfree(dev->port);
+	if (!bound) {
+		list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
+		dev_dbg(&mdev->pdev->dev, "no suitable IB device found to bind to, added to unaffiliated list.\n");
+	} else {
+		mlx5_ib_dbg(dev, "bound port %u\n", port_num + 1);
+	}
+	mutex_unlock(&mlx5_ib_multiport_mutex);
 
-err_dealloc:
-	ib_dealloc_device((struct ib_device *)dev);
+	return mpi;
+}
 
-	return NULL;
+static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
+{
+	enum rdma_link_layer ll;
+	int port_type_cap;
+
+	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
+	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
+
+	if (mlx5_core_is_mp_slave(mdev) && ll == IB_LINK_LAYER_ETHERNET) {
+		u8 port_num = mlx5_core_native_port_num(mdev) - 1;
+
+		return mlx5_ib_add_slave_port(mdev, port_num);
+	}
+
+	return __mlx5_ib_add(mdev, &pf_profile);
 }
 
 static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
 {
-	struct mlx5_ib_dev *dev = context;
-	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
+	struct mlx5_ib_multiport_info *mpi;
+	struct mlx5_ib_dev *dev;
 
-	cancel_delay_drop(dev);
-	mlx5_remove_netdev_notifier(dev);
-	ib_unregister_device(&dev->ib_dev);
-	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
-	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
-	mlx5_put_uars_page(dev->mdev, mdev->priv.uar);
-	mlx5_ib_cleanup_cong_debugfs(dev);
-	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
-		mlx5_ib_dealloc_counters(dev);
-	destroy_umrc_res(dev);
-	mlx5_ib_odp_remove_one(dev);
-	destroy_dev_resources(&dev->devr);
-	if (ll == IB_LINK_LAYER_ETHERNET)
-		mlx5_disable_eth(dev);
-	kfree(dev->port);
-	ib_dealloc_device(&dev->ib_dev);
+	if (mlx5_core_is_mp_slave(mdev)) {
+		mpi = context;
+		mutex_lock(&mlx5_ib_multiport_mutex);
+		if (mpi->ibdev)
+			mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
+		list_del(&mpi->list);
+		mutex_unlock(&mlx5_ib_multiport_mutex);
+		return;
+	}
+
+	dev = context;
+	__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
 }
 
 static struct mlx5_interface mlx5_ib_interface = {
@@ -4277,6 +5097,10 @@ static int __init mlx5_ib_init(void)
 {
 	int err;
 
+	mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
+	if (!mlx5_ib_event_wq)
+		return -ENOMEM;
+
 	mlx5_ib_odp_init();
 
 	err = mlx5_register_interface(&mlx5_ib_interface);
@@ -4287,6 +5111,7 @@ static int __init mlx5_ib_init(void)
 static void __exit mlx5_ib_cleanup(void)
 {
 	mlx5_unregister_interface(&mlx5_ib_interface);
+	destroy_workqueue(mlx5_ib_event_wq);
 }
 
 module_init(mlx5_ib_init);
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 2c5f3533bbc9..a5272499b600 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -70,15 +70,6 @@ enum {
 	MLX5_IB_MMAP_CMD_MASK	= 0xff,
 };
 
-enum mlx5_ib_mmap_cmd {
-	MLX5_IB_MMAP_REGULAR_PAGE		= 0,
-	MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES	= 1,
-	MLX5_IB_MMAP_WC_PAGE			= 2,
-	MLX5_IB_MMAP_NC_PAGE			= 3,
-	/* 5 is chosen in order to be compatible with old versions of libmlx5 */
-	MLX5_IB_MMAP_CORE_CLOCK			= 5,
-};
-
 enum {
 	MLX5_RES_SCAT_DATA32_CQE	= 0x1,
 	MLX5_RES_SCAT_DATA64_CQE	= 0x2,
@@ -112,6 +103,11 @@ enum {
 	MLX5_TM_MAX_SGE			= 1,
 };
 
+enum {
+	MLX5_IB_INVALID_UAR_INDEX	= BIT(31),
+	MLX5_IB_INVALID_BFREG		= BIT(31),
+};
+
 struct mlx5_ib_vma_private_data {
 	struct list_head list;
 	struct vm_area_struct *vma;
@@ -200,6 +196,8 @@ struct mlx5_ib_flow_db {
  * creates the actual hardware QP.
  */
 #define MLX5_IB_QPT_HW_GSI	IB_QPT_RESERVED2
+#define MLX5_IB_QPT_DCI		IB_QPT_RESERVED3
+#define MLX5_IB_QPT_DCT		IB_QPT_RESERVED4
 #define MLX5_IB_WR_UMR		IB_WR_RESERVED1
 
 #define MLX5_IB_UMR_OCTOWORD	       16
@@ -360,12 +358,18 @@ struct mlx5_bf {
 	struct mlx5_sq_bfreg   *bfreg;
 };
 
+struct mlx5_ib_dct {
+	struct mlx5_core_dct    mdct;
+	u32                     *in;
+};
+
 struct mlx5_ib_qp {
 	struct ib_qp		ibqp;
 	union {
 		struct mlx5_ib_qp_trans trans_qp;
 		struct mlx5_ib_raw_packet_qp raw_packet_qp;
 		struct mlx5_ib_rss_qp rss_qp;
+		struct mlx5_ib_dct dct;
 	};
 	struct mlx5_buf		buf;
 
@@ -404,6 +408,8 @@ struct mlx5_ib_qp {
 	u32			rate_limit;
 	u32                     underlay_qpn;
 	bool			tunnel_offload_en;
+	/* storage for qp sub type when core qp type is IB_QPT_DRIVER */
+	enum ib_qp_type		qp_sub_type;
 };
 
 struct mlx5_ib_cq_buf {
@@ -636,10 +642,21 @@ struct mlx5_ib_counters {
 	u32 num_q_counters;
 	u32 num_cong_counters;
 	u16 set_id;
+	bool set_id_valid;
+};
+
+struct mlx5_ib_multiport_info;
+
+struct mlx5_ib_multiport {
+	struct mlx5_ib_multiport_info *mpi;
+	/* To be held when accessing the multiport info */
+	spinlock_t mpi_lock;
 };
 
 struct mlx5_ib_port {
 	struct mlx5_ib_counters cnts;
+	struct mlx5_ib_multiport mp;
+	struct mlx5_ib_dbg_cc_params	*dbg_cc_params;
 };
 
 struct mlx5_roce {
@@ -651,12 +668,15 @@ struct mlx5_roce {
 	struct notifier_block	nb;
 	atomic_t		next_port;
 	enum ib_port_state last_port_state;
+	struct mlx5_ib_dev	*dev;
+	u8			native_port_num;
 };
 
 struct mlx5_ib_dbg_param {
 	int			offset;
 	struct mlx5_ib_dev	*dev;
 	struct dentry		*dentry;
+	u8			port_num;
 };
 
 enum mlx5_ib_dbg_cc_types {
@@ -709,10 +729,51 @@ struct mlx5_ib_delay_drop {
 	struct mlx5_ib_dbg_delay_drop *dbg;
 };
 
+enum mlx5_ib_stages {
+	MLX5_IB_STAGE_INIT,
+	MLX5_IB_STAGE_CAPS,
+	MLX5_IB_STAGE_ROCE,
+	MLX5_IB_STAGE_DEVICE_RESOURCES,
+	MLX5_IB_STAGE_ODP,
+	MLX5_IB_STAGE_COUNTERS,
+	MLX5_IB_STAGE_CONG_DEBUGFS,
+	MLX5_IB_STAGE_UAR,
+	MLX5_IB_STAGE_BFREG,
+	MLX5_IB_STAGE_PRE_IB_REG_UMR,
+	MLX5_IB_STAGE_IB_REG,
+	MLX5_IB_STAGE_POST_IB_REG_UMR,
+	MLX5_IB_STAGE_DELAY_DROP,
+	MLX5_IB_STAGE_CLASS_ATTR,
+	MLX5_IB_STAGE_MAX,
+};
+
+struct mlx5_ib_stage {
+	int (*init)(struct mlx5_ib_dev *dev);
+	void (*cleanup)(struct mlx5_ib_dev *dev);
+};
+
+#define STAGE_CREATE(_stage, _init, _cleanup) \
+	.stage[_stage] = {.init = _init, .cleanup = _cleanup}
+
+struct mlx5_ib_profile {
+	struct mlx5_ib_stage stage[MLX5_IB_STAGE_MAX];
+};
+
+struct mlx5_ib_multiport_info {
+	struct list_head list;
+	struct mlx5_ib_dev *ibdev;
+	struct mlx5_core_dev *mdev;
+	struct completion unref_comp;
+	u64 sys_image_guid;
+	u32 mdev_refcnt;
+	bool is_master;
+	bool unaffiliate;
+};
+
 struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
-	struct mlx5_roce		roce;
+	struct mlx5_roce		roce[MLX5_MAX_PORTS];
 	int				num_ports;
 	/* serialize update of capability mask
 	 */
@@ -746,12 +807,14 @@ struct mlx5_ib_dev {
 	struct mlx5_sq_bfreg	bfreg;
 	struct mlx5_sq_bfreg	fp_bfreg;
 	struct mlx5_ib_delay_drop	delay_drop;
-	struct mlx5_ib_dbg_cc_params	*dbg_cc_params;
+	const struct mlx5_ib_profile	*profile;
 
 	/* protect the user_td */
 	struct mutex		lb_mutex;
 	u32			user_td;
 	u8			umr_fence;
+	struct list_head	ib_dev_list;
+	u64			sys_image_guid;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
@@ -956,13 +1019,14 @@ struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
 						      struct ib_rwq_ind_table_init_attr *init_attr,
 						      struct ib_udata *udata);
 int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
+bool mlx5_ib_dc_atomic_is_supported(struct mlx5_ib_dev *dev);
+
 
 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
 void mlx5_ib_pfault(struct mlx5_core_dev *mdev, void *context,
 		    struct mlx5_pagefault *pfault);
 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
-void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev);
 int __init mlx5_ib_odp_init(void);
 void mlx5_ib_odp_cleanup(void);
 void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
@@ -977,7 +1041,6 @@ static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
 }
 
 static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
-static inline void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *ibdev)	    {}
 static inline int mlx5_ib_odp_init(void) { return 0; }
 static inline void mlx5_ib_odp_cleanup(void)				    {}
 static inline void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) {}
@@ -1001,8 +1064,8 @@ __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
 int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port_num,
 			   int index, enum ib_gid_type *gid_type);
 
-void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev);
-int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev);
+void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
+int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
 
 /* GSI QP helper functions */
 struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
@@ -1021,6 +1084,15 @@ void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
 
 int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
 
+void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,
+			int bfregn);
+struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi);
+struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *dev,
+						   u8 ib_port_num,
+						   u8 *native_port_num);
+void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,
+				  u8 port_num);
+
 static inline void init_query_mad(struct ib_smp *mad)
 {
 	mad->base_version  = 1;
@@ -1052,8 +1124,8 @@ static inline u32 check_cq_create_flags(u32 flags)
 	 * It returns non-zero value for unsupported CQ
 	 * create flags, otherwise it returns zero.
 	 */
-	return (flags & ~(IB_CQ_FLAGS_IGNORE_OVERRUN |
-			  IB_CQ_FLAGS_TIMESTAMP_COMPLETION));
+	return (flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN |
+			  IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION));
 }
 
 static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
@@ -1113,10 +1185,10 @@ static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_suppor
 				MLX5_UARS_IN_PAGE : 1;
 }
 
-static inline int get_num_uars(struct mlx5_ib_dev *dev,
-			       struct mlx5_bfreg_info *bfregi)
+static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
+				      struct mlx5_bfreg_info *bfregi)
 {
-	return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_sys_pages;
+	return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
 }
 
 #endif /* MLX5_IB_H */
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index d109fe8290a7..3e0b3f0238d6 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -739,6 +739,9 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
 {
 	int i;
 
+	if (!dev->cache.wq)
+		return 0;
+
 	dev->cache.stopped = 1;
 	flush_workqueue(dev->cache.wq);
 
@@ -838,7 +841,8 @@ static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
 	*umem = ib_umem_get(pd->uobject->context, start, length,
 			    access_flags, 0);
 	err = PTR_ERR_OR_ZERO(*umem);
-	if (err < 0) {
+	if (err) {
+		*umem = NULL;
 		mlx5_ib_err(dev, "umem get failed (%d)\n", err);
 		return err;
 	}
@@ -1206,6 +1210,9 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	int err;
 	bool use_umr = true;
 
+	if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
+		return ERR_PTR(-EINVAL);
+
 	mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
 		    start, virt_addr, length, access_flags);
 
@@ -1412,6 +1419,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
 		if (err) {
 			mlx5_ib_warn(dev, "Failed to rereg UMR\n");
 			ib_umem_release(mr->umem);
+			mr->umem = NULL;
 			clean_mr(dev, mr);
 			return err;
 		}
@@ -1495,14 +1503,11 @@ static int clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 		u32 key = mr->mmkey.key;
 
 		err = destroy_mkey(dev, mr);
-		kfree(mr);
 		if (err) {
 			mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
 				     key, err);
 			return err;
 		}
-	} else {
-		mlx5_mr_cache_free(dev, mr);
 	}
 
 	return 0;
@@ -1545,6 +1550,11 @@ static int dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 		atomic_sub(npages, &dev->mdev->priv.reg_pages);
 	}
 
+	if (!mr->allocated_from_cache)
+		kfree(mr);
+	else
+		mlx5_mr_cache_free(dev, mr);
+
 	return 0;
 }
 
@@ -1813,7 +1823,6 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
 
 	mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
 	mr->ibmr.length = 0;
-	mr->ndescs = sg_nents;
 
 	for_each_sg(sgl, sg, sg_nents, i) {
 		if (unlikely(i >= mr->max_descs))
@@ -1825,6 +1834,7 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
 
 		sg_offset = 0;
 	}
+	mr->ndescs = i;
 
 	if (sg_offset_p)
 		*sg_offset_p = sg_offset;
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index e2197bdda89c..f1a87a690a4c 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -1207,10 +1207,6 @@ int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)
 {
 	int ret;
 
-	ret = init_srcu_struct(&dev->mr_srcu);
-	if (ret)
-		return ret;
-
 	if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) {
 		ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey);
 		if (ret) {
@@ -1222,11 +1218,6 @@ int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)
 	return 0;
 }
 
-void mlx5_ib_odp_remove_one(struct mlx5_ib_dev *dev)
-{
-	cleanup_srcu_struct(&dev->mr_srcu);
-}
-
 int mlx5_ib_odp_init(void)
 {
 	mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) -
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index cffe5966aef9..a2e1aa86e133 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -493,7 +493,7 @@ enum {
 
 static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
 {
-	return get_num_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
+	return get_num_static_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
 }
 
 static int num_med_bfreg(struct mlx5_ib_dev *dev,
@@ -581,7 +581,7 @@ static int alloc_bfreg(struct mlx5_ib_dev *dev,
 	return bfregn;
 }
 
-static void free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
+void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
 {
 	mutex_lock(&bfregi->lock);
 	bfregi->count[bfregn]--;
@@ -613,6 +613,7 @@ static int to_mlx5_st(enum ib_qp_type type)
 	case IB_QPT_XRC_TGT:		return MLX5_QP_ST_XRC;
 	case IB_QPT_SMI:		return MLX5_QP_ST_QP0;
 	case MLX5_IB_QPT_HW_GSI:	return MLX5_QP_ST_QP1;
+	case MLX5_IB_QPT_DCI:		return MLX5_QP_ST_DCI;
 	case IB_QPT_RAW_IPV6:		return MLX5_QP_ST_RAW_IPV6;
 	case IB_QPT_RAW_PACKET:
 	case IB_QPT_RAW_ETHERTYPE:	return MLX5_QP_ST_RAW_ETHERTYPE;
@@ -627,7 +628,8 @@ static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
 			       struct mlx5_ib_cq *recv_cq);
 
 static int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
-			       struct mlx5_bfreg_info *bfregi, int bfregn)
+			       struct mlx5_bfreg_info *bfregi, int bfregn,
+			       bool dyn_bfreg)
 {
 	int bfregs_per_sys_page;
 	int index_of_sys_page;
@@ -637,8 +639,16 @@ static int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
 				MLX5_NON_FP_BFREGS_PER_UAR;
 	index_of_sys_page = bfregn / bfregs_per_sys_page;
 
-	offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
+	if (dyn_bfreg) {
+		index_of_sys_page += bfregi->num_static_sys_pages;
+		if (bfregn > bfregi->num_dyn_bfregs ||
+		    bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
+			mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
+			return -EINVAL;
+		}
+	}
 
+	offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
 	return bfregi->sys_pages[index_of_sys_page] + offset;
 }
 
@@ -764,7 +774,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	struct mlx5_ib_create_qp ucmd;
 	struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
 	int page_shift = 0;
-	int uar_index;
+	int uar_index = 0;
 	int npages;
 	u32 offset = 0;
 	int bfregn;
@@ -780,12 +790,20 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	}
 
 	context = to_mucontext(pd->uobject->context);
-	/*
-	 * TBD: should come from the verbs when we have the API
-	 */
-	if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
+	if (ucmd.flags & MLX5_QP_FLAG_BFREG_INDEX) {
+		uar_index = bfregn_to_uar_index(dev, &context->bfregi,
+						ucmd.bfreg_index, true);
+		if (uar_index < 0)
+			return uar_index;
+
+		bfregn = MLX5_IB_INVALID_BFREG;
+	} else if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL) {
+		/*
+		 * TBD: should come from the verbs when we have the API
+		 */
 		/* In CROSS_CHANNEL CQ and QP must use the same UAR */
 		bfregn = MLX5_CROSS_CHANNEL_BFREG;
+	}
 	else {
 		bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_HIGH);
 		if (bfregn < 0) {
@@ -804,8 +822,10 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		}
 	}
 
-	uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn);
 	mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
+	if (bfregn != MLX5_IB_INVALID_BFREG)
+		uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn,
+						false);
 
 	qp->rq.offset = 0;
 	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
@@ -845,7 +865,10 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	MLX5_SET(qpc, qpc, page_offset, offset);
 
 	MLX5_SET(qpc, qpc, uar_page, uar_index);
-	resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
+	if (bfregn != MLX5_IB_INVALID_BFREG)
+		resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
+	else
+		resp->bfreg_index = MLX5_IB_INVALID_BFREG;
 	qp->bfregn = bfregn;
 
 	err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
@@ -874,7 +897,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		ib_umem_release(ubuffer->umem);
 
 err_bfreg:
-	free_bfreg(dev, &context->bfregi, bfregn);
+	if (bfregn != MLX5_IB_INVALID_BFREG)
+		mlx5_ib_free_bfreg(dev, &context->bfregi, bfregn);
 	return err;
 }
 
@@ -887,7 +911,13 @@ static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	mlx5_ib_db_unmap_user(context, &qp->db);
 	if (base->ubuffer.umem)
 		ib_umem_release(base->ubuffer.umem);
-	free_bfreg(dev, &context->bfregi, qp->bfregn);
+
+	/*
+	 * Free only the BFREGs which are handled by the kernel.
+	 * BFREGs of UARs allocated dynamically are handled by user.
+	 */
+	if (qp->bfregn != MLX5_IB_INVALID_BFREG)
+		mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
 }
 
 static int create_kernel_qp(struct mlx5_ib_dev *dev,
@@ -1015,6 +1045,7 @@ static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
 {
 	if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) ||
+	    (attr->qp_type == MLX5_IB_QPT_DCI) ||
 	    (attr->qp_type == IB_QPT_XRC_INI))
 		return MLX5_SRQ_RQ;
 	else if (!qp->has_rq)
@@ -1130,7 +1161,7 @@ static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
 	ib_umem_release(sq->ubuffer.umem);
 }
 
-static int get_rq_pas_size(void *qpc)
+static size_t get_rq_pas_size(void *qpc)
 {
 	u32 log_page_size = MLX5_GET(qpc, qpc, log_page_size) + 12;
 	u32 log_rq_stride = MLX5_GET(qpc, qpc, log_rq_stride);
@@ -1146,7 +1177,8 @@ static int get_rq_pas_size(void *qpc)
 }
 
 static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
-				   struct mlx5_ib_rq *rq, void *qpin)
+				   struct mlx5_ib_rq *rq, void *qpin,
+				   size_t qpinlen)
 {
 	struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
 	__be64 *pas;
@@ -1155,9 +1187,12 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
 	void *rqc;
 	void *wq;
 	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
-	int inlen;
+	size_t rq_pas_size = get_rq_pas_size(qpc);
+	size_t inlen;
 	int err;
-	u32 rq_pas_size = get_rq_pas_size(qpc);
+
+	if (qpinlen < rq_pas_size + MLX5_BYTE_OFF(create_qp_in, pas))
+		return -EINVAL;
 
 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + rq_pas_size;
 	in = kvzalloc(inlen, GFP_KERNEL);
@@ -1246,7 +1281,7 @@ static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
 }
 
 static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
-				u32 *in,
+				u32 *in, size_t inlen,
 				struct ib_pd *pd)
 {
 	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
@@ -1278,7 +1313,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
 		if (qp->flags & MLX5_IB_QP_PCI_WRITE_END_PADDING)
 			rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
-		err = create_raw_packet_qp_rq(dev, rq, in);
+		err = create_raw_packet_qp_rq(dev, rq, in, inlen);
 		if (err)
 			goto err_destroy_sq;
 
@@ -1553,6 +1588,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	u32 uidx = MLX5_IB_DEFAULT_UIDX;
 	struct mlx5_ib_create_qp ucmd;
 	struct mlx5_ib_qp_base *base;
+	int mlx5_st;
 	void *qpc;
 	u32 *in;
 	int err;
@@ -1561,6 +1597,10 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 	spin_lock_init(&qp->sq.lock);
 	spin_lock_init(&qp->rq.lock);
 
+	mlx5_st = to_mlx5_st(init_attr->qp_type);
+	if (mlx5_st < 0)
+		return -EINVAL;
+
 	if (init_attr->rwq_ind_tbl) {
 		if (!udata)
 			return -ENOSYS;
@@ -1722,7 +1762,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 
-	MLX5_SET(qpc, qpc, st, to_mlx5_st(init_attr->qp_type));
+	MLX5_SET(qpc, qpc, st, mlx5_st);
 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
 
 	if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR)
@@ -1836,11 +1876,16 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 		}
 	}
 
+	if (inlen < 0) {
+		err = -EINVAL;
+		goto err;
+	}
+
 	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
 	    qp->flags & MLX5_IB_QP_UNDERLAY) {
 		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd.sq_buf_addr;
 		raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
-		err = create_raw_packet_qp(dev, qp, in, pd);
+		err = create_raw_packet_qp(dev, qp, in, inlen, pd);
 	} else {
 		err = mlx5_core_create_qp(dev->mdev, &base->mqp, in, inlen);
 	}
@@ -2086,20 +2131,108 @@ static const char *ib_qp_type_str(enum ib_qp_type type)
 		return "IB_QPT_RAW_PACKET";
 	case MLX5_IB_QPT_REG_UMR:
 		return "MLX5_IB_QPT_REG_UMR";
+	case IB_QPT_DRIVER:
+		return "IB_QPT_DRIVER";
 	case IB_QPT_MAX:
 	default:
 		return "Invalid QP type";
 	}
 }
 
+static struct ib_qp *mlx5_ib_create_dct(struct ib_pd *pd,
+					struct ib_qp_init_attr *attr,
+					struct mlx5_ib_create_qp *ucmd)
+{
+	struct mlx5_ib_dev *dev;
+	struct mlx5_ib_qp *qp;
+	int err = 0;
+	u32 uidx = MLX5_IB_DEFAULT_UIDX;
+	void *dctc;
+
+	if (!attr->srq || !attr->recv_cq)
+		return ERR_PTR(-EINVAL);
+
+	dev = to_mdev(pd->device);
+
+	err = get_qp_user_index(to_mucontext(pd->uobject->context),
+				ucmd, sizeof(*ucmd), &uidx);
+	if (err)
+		return ERR_PTR(err);
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp)
+		return ERR_PTR(-ENOMEM);
+
+	qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
+	if (!qp->dct.in) {
+		err = -ENOMEM;
+		goto err_free;
+	}
+
+	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
+	qp->qp_sub_type = MLX5_IB_QPT_DCT;
+	MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
+	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
+	MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
+	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
+	MLX5_SET(dctc, dctc, user_index, uidx);
+
+	qp->state = IB_QPS_RESET;
+
+	return &qp->ibqp;
+err_free:
+	kfree(qp);
+	return ERR_PTR(err);
+}
+
+static int set_mlx_qp_type(struct mlx5_ib_dev *dev,
+			   struct ib_qp_init_attr *init_attr,
+			   struct mlx5_ib_create_qp *ucmd,
+			   struct ib_udata *udata)
+{
+	enum { MLX_QP_FLAGS = MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI };
+	int err;
+
+	if (!udata)
+		return -EINVAL;
+
+	if (udata->inlen < sizeof(*ucmd)) {
+		mlx5_ib_dbg(dev, "create_qp user command is smaller than expected\n");
+		return -EINVAL;
+	}
+	err = ib_copy_from_udata(ucmd, udata, sizeof(*ucmd));
+	if (err)
+		return err;
+
+	if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCI) {
+		init_attr->qp_type = MLX5_IB_QPT_DCI;
+	} else {
+		if ((ucmd->flags & MLX_QP_FLAGS) == MLX5_QP_FLAG_TYPE_DCT) {
+			init_attr->qp_type = MLX5_IB_QPT_DCT;
+		} else {
+			mlx5_ib_dbg(dev, "Invalid QP flags\n");
+			return -EINVAL;
+		}
+	}
+
+	if (!MLX5_CAP_GEN(dev->mdev, dct)) {
+		mlx5_ib_dbg(dev, "DC transport is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
 struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
-				struct ib_qp_init_attr *init_attr,
+				struct ib_qp_init_attr *verbs_init_attr,
 				struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev;
 	struct mlx5_ib_qp *qp;
 	u16 xrcdn = 0;
 	int err;
+	struct ib_qp_init_attr mlx_init_attr;
+	struct ib_qp_init_attr *init_attr = verbs_init_attr;
 
 	if (pd) {
 		dev = to_mdev(pd->device);
@@ -2124,6 +2257,26 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
 		dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device);
 	}
 
+	if (init_attr->qp_type == IB_QPT_DRIVER) {
+		struct mlx5_ib_create_qp ucmd;
+
+		init_attr = &mlx_init_attr;
+		memcpy(init_attr, verbs_init_attr, sizeof(*verbs_init_attr));
+		err = set_mlx_qp_type(dev, init_attr, &ucmd, udata);
+		if (err)
+			return ERR_PTR(err);
+
+		if (init_attr->qp_type == MLX5_IB_QPT_DCI) {
+			if (init_attr->cap.max_recv_wr ||
+			    init_attr->cap.max_recv_sge) {
+				mlx5_ib_dbg(dev, "DCI QP requires zero size receive queue\n");
+				return ERR_PTR(-EINVAL);
+			}
+		} else {
+			return mlx5_ib_create_dct(pd, init_attr, &ucmd);
+		}
+	}
+
 	switch (init_attr->qp_type) {
 	case IB_QPT_XRC_TGT:
 	case IB_QPT_XRC_INI:
@@ -2145,6 +2298,7 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
 	case IB_QPT_SMI:
 	case MLX5_IB_QPT_HW_GSI:
 	case MLX5_IB_QPT_REG_UMR:
+	case MLX5_IB_QPT_DCI:
 		qp = kzalloc(sizeof(*qp), GFP_KERNEL);
 		if (!qp)
 			return ERR_PTR(-ENOMEM);
@@ -2185,9 +2339,31 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
 		return ERR_PTR(-EINVAL);
 	}
 
+	if (verbs_init_attr->qp_type == IB_QPT_DRIVER)
+		qp->qp_sub_type = init_attr->qp_type;
+
 	return &qp->ibqp;
 }
 
+static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
+{
+	struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
+
+	if (mqp->state == IB_QPS_RTR) {
+		int err;
+
+		err = mlx5_core_destroy_dct(dev->mdev, &mqp->dct.mdct);
+		if (err) {
+			mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
+			return err;
+		}
+	}
+
+	kfree(mqp->dct.in);
+	kfree(mqp);
+	return 0;
+}
+
 int mlx5_ib_destroy_qp(struct ib_qp *qp)
 {
 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
@@ -2196,6 +2372,9 @@ int mlx5_ib_destroy_qp(struct ib_qp *qp)
 	if (unlikely(qp->qp_type == IB_QPT_GSI))
 		return mlx5_ib_gsi_destroy_qp(qp);
 
+	if (mqp->qp_sub_type == MLX5_IB_QPT_DCT)
+		return mlx5_ib_destroy_dct(mqp);
+
 	destroy_qp_common(dev, mqp);
 
 	kfree(mqp);
@@ -2763,7 +2942,8 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	if (!context)
 		return -ENOMEM;
 
-	err = to_mlx5_st(ibqp->qp_type);
+	err = to_mlx5_st(ibqp->qp_type == IB_QPT_DRIVER ?
+			 qp->qp_sub_type : ibqp->qp_type);
 	if (err < 0) {
 		mlx5_ib_dbg(dev, "unsupported qp type %d\n", ibqp->qp_type);
 		goto out;
@@ -2796,8 +2976,9 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 		    (ibqp->qp_type == IB_QPT_XRC_INI) ||
 		    (ibqp->qp_type == IB_QPT_XRC_TGT)) {
 			if (mlx5_lag_is_active(dev->mdev)) {
+				u8 p = mlx5_core_native_port_num(dev->mdev);
 				tx_affinity = (unsigned int)atomic_add_return(1,
-						&dev->roce.next_port) %
+						&dev->roce[p].next_port) %
 						MLX5_MAX_PORTS + 1;
 				context->flags |= cpu_to_be32(tx_affinity << 24);
 			}
@@ -2922,13 +3103,16 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 
 	mlx5_cur = to_mlx5_state(cur_state);
 	mlx5_new = to_mlx5_state(new_state);
-	mlx5_st = to_mlx5_st(ibqp->qp_type);
+	mlx5_st = to_mlx5_st(ibqp->qp_type == IB_QPT_DRIVER ?
+			     qp->qp_sub_type : ibqp->qp_type);
 	if (mlx5_st < 0)
 		goto out;
 
 	if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
-	    !optab[mlx5_cur][mlx5_new])
+	    !optab[mlx5_cur][mlx5_new]) {
+		err = -EINVAL;
 		goto out;
+	}
 
 	op = optab[mlx5_cur][mlx5_new];
 	optpar = ib_mask_to_mlx5_opt(attr_mask);
@@ -2994,6 +3178,139 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 	return err;
 }
 
+static inline bool is_valid_mask(int mask, int req, int opt)
+{
+	if ((mask & req) != req)
+		return false;
+
+	if (mask & ~(req | opt))
+		return false;
+
+	return true;
+}
+
+/* check valid transition for driver QP types
+ * for now the only QP type that this function supports is DCI
+ */
+static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
+				enum ib_qp_attr_mask attr_mask)
+{
+	int req = IB_QP_STATE;
+	int opt = 0;
+
+	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+		req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
+		return is_valid_mask(attr_mask, req, opt);
+	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
+		opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
+		return is_valid_mask(attr_mask, req, opt);
+	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
+		req |= IB_QP_PATH_MTU;
+		opt = IB_QP_PKEY_INDEX;
+		return is_valid_mask(attr_mask, req, opt);
+	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
+		req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
+		       IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
+		opt = IB_QP_MIN_RNR_TIMER;
+		return is_valid_mask(attr_mask, req, opt);
+	} else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
+		opt = IB_QP_MIN_RNR_TIMER;
+		return is_valid_mask(attr_mask, req, opt);
+	} else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
+		return is_valid_mask(attr_mask, req, opt);
+	}
+	return false;
+}
+
+/* mlx5_ib_modify_dct: modify a DCT QP
+ * valid transitions are:
+ * RESET to INIT: must set access_flags, pkey_index and port
+ * INIT  to RTR : must set min_rnr_timer, tclass, flow_label,
+ *			   mtu, gid_index and hop_limit
+ * Other transitions and attributes are illegal
+ */
+static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+			      int attr_mask, struct ib_udata *udata)
+{
+	struct mlx5_ib_qp *qp = to_mqp(ibqp);
+	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
+	enum ib_qp_state cur_state, new_state;
+	int err = 0;
+	int required = IB_QP_STATE;
+	void *dctc;
+
+	if (!(attr_mask & IB_QP_STATE))
+		return -EINVAL;
+
+	cur_state = qp->state;
+	new_state = attr->qp_state;
+
+	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
+	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+		required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
+		if (!is_valid_mask(attr_mask, required, 0))
+			return -EINVAL;
+
+		if (attr->port_num == 0 ||
+		    attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports)) {
+			mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
+				    attr->port_num, dev->num_ports);
+			return -EINVAL;
+		}
+		if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
+			MLX5_SET(dctc, dctc, rre, 1);
+		if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
+			MLX5_SET(dctc, dctc, rwe, 1);
+		if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
+			if (!mlx5_ib_dc_atomic_is_supported(dev))
+				return -EOPNOTSUPP;
+			MLX5_SET(dctc, dctc, rae, 1);
+			MLX5_SET(dctc, dctc, atomic_mode, MLX5_ATOMIC_MODE_DCT_CX);
+		}
+		MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
+		MLX5_SET(dctc, dctc, port, attr->port_num);
+		MLX5_SET(dctc, dctc, counter_set_id, dev->port[attr->port_num - 1].cnts.set_id);
+
+	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
+		struct mlx5_ib_modify_qp_resp resp = {};
+		u32 min_resp_len = offsetof(typeof(resp), dctn) +
+				   sizeof(resp.dctn);
+
+		if (udata->outlen < min_resp_len)
+			return -EINVAL;
+		resp.response_length = min_resp_len;
+
+		required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
+		if (!is_valid_mask(attr_mask, required, 0))
+			return -EINVAL;
+		MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
+		MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
+		MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
+		MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
+		MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
+		MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
+
+		err = mlx5_core_create_dct(dev->mdev, &qp->dct.mdct, qp->dct.in,
+					   MLX5_ST_SZ_BYTES(create_dct_in));
+		if (err)
+			return err;
+		resp.dctn = qp->dct.mdct.mqp.qpn;
+		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		if (err) {
+			mlx5_core_destroy_dct(dev->mdev, &qp->dct.mdct);
+			return err;
+		}
+	} else {
+		mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
+		return -EINVAL;
+	}
+	if (err)
+		qp->state = IB_QPS_ERR;
+	else
+		qp->state = new_state;
+	return err;
+}
+
 int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		      int attr_mask, struct ib_udata *udata)
 {
@@ -3011,8 +3328,14 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	if (unlikely(ibqp->qp_type == IB_QPT_GSI))
 		return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
 
-	qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ?
-		IB_QPT_GSI : ibqp->qp_type;
+	if (ibqp->qp_type == IB_QPT_DRIVER)
+		qp_type = qp->qp_sub_type;
+	else
+		qp_type = (unlikely(ibqp->qp_type == MLX5_IB_QPT_HW_GSI)) ?
+			IB_QPT_GSI : ibqp->qp_type;
+
+	if (qp_type == MLX5_IB_QPT_DCT)
+		return mlx5_ib_modify_dct(ibqp, attr, attr_mask, udata);
 
 	mutex_lock(&qp->mutex);
 
@@ -3031,15 +3354,21 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 			goto out;
 		}
 	} else if (qp_type != MLX5_IB_QPT_REG_UMR &&
-	    !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) {
+		   qp_type != MLX5_IB_QPT_DCI &&
+		   !ib_modify_qp_is_ok(cur_state, new_state, qp_type, attr_mask, ll)) {
 		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
 			    cur_state, new_state, ibqp->qp_type, attr_mask);
 		goto out;
+	} else if (qp_type == MLX5_IB_QPT_DCI &&
+		   !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
+		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
+			    cur_state, new_state, qp_type, attr_mask);
+		goto out;
 	}
 
 	if ((attr_mask & IB_QP_PORT) &&
 	    (attr->port_num == 0 ||
-	     attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports))) {
+	     attr->port_num > dev->num_ports)) {
 		mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
 			    attr->port_num, dev->num_ports);
 		goto out;
@@ -4358,11 +4687,10 @@ static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
 			    struct rdma_ah_attr *ah_attr,
 			    struct mlx5_qp_path *path)
 {
-	struct mlx5_core_dev *dev = ibdev->mdev;
 
 	memset(ah_attr, 0, sizeof(*ah_attr));
 
-	if (!path->port || path->port > MLX5_CAP_GEN(dev, num_ports))
+	if (!path->port || path->port > ibdev->num_ports)
 		return;
 
 	ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, path->port);
@@ -4577,6 +4905,71 @@ static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	return err;
 }
 
+static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
+				struct ib_qp_attr *qp_attr, int qp_attr_mask,
+				struct ib_qp_init_attr *qp_init_attr)
+{
+	struct mlx5_core_dct	*dct = &mqp->dct.mdct;
+	u32 *out;
+	u32 access_flags = 0;
+	int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
+	void *dctc;
+	int err;
+	int supported_mask = IB_QP_STATE |
+			     IB_QP_ACCESS_FLAGS |
+			     IB_QP_PORT |
+			     IB_QP_MIN_RNR_TIMER |
+			     IB_QP_AV |
+			     IB_QP_PATH_MTU |
+			     IB_QP_PKEY_INDEX;
+
+	if (qp_attr_mask & ~supported_mask)
+		return -EINVAL;
+	if (mqp->state != IB_QPS_RTR)
+		return -EINVAL;
+
+	out = kzalloc(outlen, GFP_KERNEL);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_core_dct_query(dev->mdev, dct, out, outlen);
+	if (err)
+		goto out;
+
+	dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
+
+	if (qp_attr_mask & IB_QP_STATE)
+		qp_attr->qp_state = IB_QPS_RTR;
+
+	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
+		if (MLX5_GET(dctc, dctc, rre))
+			access_flags |= IB_ACCESS_REMOTE_READ;
+		if (MLX5_GET(dctc, dctc, rwe))
+			access_flags |= IB_ACCESS_REMOTE_WRITE;
+		if (MLX5_GET(dctc, dctc, rae))
+			access_flags |= IB_ACCESS_REMOTE_ATOMIC;
+		qp_attr->qp_access_flags = access_flags;
+	}
+
+	if (qp_attr_mask & IB_QP_PORT)
+		qp_attr->port_num = MLX5_GET(dctc, dctc, port);
+	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
+		qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
+	if (qp_attr_mask & IB_QP_AV) {
+		qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
+		qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
+		qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
+		qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
+	}
+	if (qp_attr_mask & IB_QP_PATH_MTU)
+		qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
+	if (qp_attr_mask & IB_QP_PKEY_INDEX)
+		qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
+out:
+	kfree(out);
+	return err;
+}
+
 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
 {
@@ -4596,6 +4989,10 @@ int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
 	memset(qp_attr, 0, sizeof(*qp_attr));
 
+	if (unlikely(qp->qp_sub_type == MLX5_IB_QPT_DCT))
+		return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
+					    qp_attr_mask, qp_init_attr);
+
 	mutex_lock(&qp->mutex);
 
 	if (qp->ibqp.qp_type == IB_QPT_RAW_PACKET ||
@@ -4685,13 +5082,10 @@ int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
 	int err;
 
 	err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn);
-	if (err) {
+	if (err)
 		mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
-		return err;
-	}
 
 	kfree(xrcd);
-
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index 6d5fadad9090..3c7522d025f2 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -241,8 +241,8 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
 	struct mlx5_ib_srq *srq;
-	int desc_size;
-	int buf_size;
+	size_t desc_size;
+	size_t buf_size;
 	int err;
 	struct mlx5_srq_attr in = {0};
 	__u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
@@ -266,15 +266,18 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 
 	desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
 		    srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
+	if (desc_size == 0 || srq->msrq.max_gs > desc_size)
+		return ERR_PTR(-EINVAL);
 	desc_size = roundup_pow_of_two(desc_size);
-	desc_size = max_t(int, 32, desc_size);
+	desc_size = max_t(size_t, 32, desc_size);
+	if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg))
+		return ERR_PTR(-EINVAL);
 	srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
 		sizeof(struct mlx5_wqe_data_seg);
 	srq->msrq.wqe_shift = ilog2(desc_size);
 	buf_size = srq->msrq.max * desc_size;
-	mlx5_ib_dbg(dev, "desc_size 0x%x, req wr 0x%x, srq size 0x%x, max_gs 0x%x, max_avail_gather 0x%x\n",
-		    desc_size, init_attr->attr.max_wr, srq->msrq.max, srq->msrq.max_gs,
-		    srq->msrq.max_avail_gather);
+	if (buf_size < desc_size)
+		return ERR_PTR(-EINVAL);
 	in.type = init_attr->srq_type;
 
 	if (pd->uobject)
diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c
index c6fe89d79248..2fe503e86c1d 100644
--- a/drivers/infiniband/hw/mthca/mthca_memfree.c
+++ b/drivers/infiniband/hw/mthca/mthca_memfree.c
@@ -472,7 +472,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar,
 		goto out;
 	}
 
-	ret = get_user_pages(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages, NULL);
+	ret = get_user_pages_fast(uaddr & PAGE_MASK, 1, FOLL_WRITE, pages);
 	if (ret < 0)
 		goto out;
 
@@ -623,13 +623,12 @@ int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type,
 	page = dev->db_tab->page + end;
 
 alloc:
-	page->db_rec = dma_alloc_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,
-					  &page->mapping, GFP_KERNEL);
+	page->db_rec = dma_zalloc_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE,
+					   &page->mapping, GFP_KERNEL);
 	if (!page->db_rec) {
 		ret = -ENOMEM;
 		goto out;
 	}
-	memset(page->db_rec, 0, MTHCA_ICM_PAGE_SIZE);
 
 	ret = mthca_MAP_ICM_page(dev, page->mapping,
 				 mthca_uarc_virt(dev, &dev->driver_uar, i));
diff --git a/drivers/infiniband/hw/mthca/mthca_user.h b/drivers/infiniband/hw/mthca/mthca_user.h
deleted file mode 100644
index 5fe56e810739..000000000000
--- a/drivers/infiniband/hw/mthca/mthca_user.h
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright (c) 2005 Topspin Communications.  All rights reserved.
- * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef MTHCA_USER_H
-#define MTHCA_USER_H
-
-#include <linux/types.h>
-
-/*
- * Increment this value if any changes that break userspace ABI
- * compatibility are made.
- */
-#define MTHCA_UVERBS_ABI_VERSION	1
-
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __u64
- * instead.
- */
-
-struct mthca_alloc_ucontext_resp {
-	__u32 qp_tab_size;
-	__u32 uarc_size;
-};
-
-struct mthca_alloc_pd_resp {
-	__u32 pdn;
-	__u32 reserved;
-};
-
-struct mthca_reg_mr {
-/*
- * Mark the memory region with a DMA attribute that causes
- * in-flight DMA to be flushed when the region is written to:
- */
-#define MTHCA_MR_DMASYNC	0x1
-	__u32 mr_attrs;
-	__u32 reserved;
-};
-
-struct mthca_create_cq {
-	__u32 lkey;
-	__u32 pdn;
-	__u64 arm_db_page;
-	__u64 set_db_page;
-	__u32 arm_db_index;
-	__u32 set_db_index;
-};
-
-struct mthca_create_cq_resp {
-	__u32 cqn;
-	__u32 reserved;
-};
-
-struct mthca_resize_cq {
-	__u32 lkey;
-	__u32 reserved;
-};
-
-struct mthca_create_srq {
-	__u32 lkey;
-	__u32 db_index;
-	__u64 db_page;
-};
-
-struct mthca_create_srq_resp {
-	__u32 srqn;
-	__u32 reserved;
-};
-
-struct mthca_create_qp {
-	__u32 lkey;
-	__u32 reserved;
-	__u64 sq_db_page;
-	__u64 rq_db_page;
-	__u32 sq_db_index;
-	__u32 rq_db_index;
-};
-
-#endif /* MTHCA_USER_H */
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index c56ca2a74df5..6cdfbf8c5674 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -1365,7 +1365,7 @@ static int mini_cm_del_listen(struct nes_cm_core *cm_core,
 static inline int mini_cm_accelerated(struct nes_cm_core *cm_core,
 				      struct nes_cm_node *cm_node)
 {
-	cm_node->accelerated = 1;
+	cm_node->accelerated = true;
 
 	if (cm_node->accept_pend) {
 		BUG_ON(!cm_node->listener);
diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h
index d827d03e3941..b9cc02b4e8d5 100644
--- a/drivers/infiniband/hw/nes/nes_cm.h
+++ b/drivers/infiniband/hw/nes/nes_cm.h
@@ -279,7 +279,6 @@ struct nes_cm_tcp_context {
 	u8  rcv_wscale;
 
 	struct nes_cm_tsa_context tsa_cntxt;
-	struct timeval            sent_ts;
 };
 
 
@@ -341,7 +340,7 @@ struct nes_cm_node {
 	u16                       mpa_frame_size;
 	struct iw_cm_id           *cm_id;
 	struct list_head          list;
-	int                       accelerated;
+	bool                      accelerated;
 	struct nes_cm_listener    *listener;
 	enum nes_cm_conn_type     conn_type;
 	struct nes_vnic           *nesvnic;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index 0ba695a88b62..9904918589a4 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -380,11 +380,10 @@ static int ocrdma_alloc_q(struct ocrdma_dev *dev,
 	q->len = len;
 	q->entry_size = entry_size;
 	q->size = len * entry_size;
-	q->va = dma_alloc_coherent(&dev->nic_info.pdev->dev, q->size,
-				   &q->dma, GFP_KERNEL);
+	q->va = dma_zalloc_coherent(&dev->nic_info.pdev->dev, q->size,
+				    &q->dma, GFP_KERNEL);
 	if (!q->va)
 		return -ENOMEM;
-	memset(q->va, 0, q->size);
 	return 0;
 }
 
@@ -1819,12 +1818,11 @@ int ocrdma_mbx_create_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
 		return -ENOMEM;
 	ocrdma_init_mch(&cmd->cmd.req, OCRDMA_CMD_CREATE_CQ,
 			OCRDMA_SUBSYS_COMMON, sizeof(*cmd));
-	cq->va = dma_alloc_coherent(&pdev->dev, cq->len, &cq->pa, GFP_KERNEL);
+	cq->va = dma_zalloc_coherent(&pdev->dev, cq->len, &cq->pa, GFP_KERNEL);
 	if (!cq->va) {
 		status = -ENOMEM;
 		goto mem_err;
 	}
-	memset(cq->va, 0, cq->len);
 	page_size = cq->len / hw_pages;
 	cmd->cmd.pgsz_pgcnt = (page_size / OCRDMA_MIN_Q_PAGE_SIZE) <<
 					OCRDMA_CREATE_CQ_PAGE_SIZE_SHIFT;
@@ -2212,10 +2210,9 @@ static int ocrdma_set_create_qp_sq_cmd(struct ocrdma_create_qp_req *cmd,
 	qp->sq.max_cnt = max_wqe_allocated;
 	len = (hw_pages * hw_page_size);
 
-	qp->sq.va = dma_alloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL);
+	qp->sq.va = dma_zalloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL);
 	if (!qp->sq.va)
 		return -EINVAL;
-	memset(qp->sq.va, 0, len);
 	qp->sq.len = len;
 	qp->sq.pa = pa;
 	qp->sq.entry_size = dev->attr.wqe_size;
@@ -2263,10 +2260,9 @@ static int ocrdma_set_create_qp_rq_cmd(struct ocrdma_create_qp_req *cmd,
 	qp->rq.max_cnt = max_rqe_allocated;
 	len = (hw_pages * hw_page_size);
 
-	qp->rq.va = dma_alloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL);
+	qp->rq.va = dma_zalloc_coherent(&pdev->dev, len, &pa, GFP_KERNEL);
 	if (!qp->rq.va)
 		return -ENOMEM;
-	memset(qp->rq.va, 0, len);
 	qp->rq.pa = pa;
 	qp->rq.len = len;
 	qp->rq.entry_size = dev->attr.rqe_size;
@@ -2320,11 +2316,10 @@ static int ocrdma_set_create_qp_ird_cmd(struct ocrdma_create_qp_req *cmd,
 	if (dev->attr.ird == 0)
 		return 0;
 
-	qp->ird_q_va = dma_alloc_coherent(&pdev->dev, ird_q_len,
-					&pa, GFP_KERNEL);
+	qp->ird_q_va = dma_zalloc_coherent(&pdev->dev, ird_q_len, &pa,
+					   GFP_KERNEL);
 	if (!qp->ird_q_va)
 		return -ENOMEM;
-	memset(qp->ird_q_va, 0, ird_q_len);
 	ocrdma_build_q_pages(&cmd->ird_addr[0], dev->attr.num_ird_pages,
 			     pa, ird_page_size);
 	for (; i < ird_q_len / dev->attr.rqe_size; i++) {
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_stats.c b/drivers/infiniband/hw/ocrdma/ocrdma_stats.c
index e528d7acb7f6..24d20a4aa262 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_stats.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_stats.c
@@ -73,15 +73,13 @@ bool ocrdma_alloc_stats_resources(struct ocrdma_dev *dev)
 	mem->size = max_t(u32, sizeof(struct ocrdma_rdma_stats_req),
 			sizeof(struct ocrdma_rdma_stats_resp));
 
-	mem->va   = dma_alloc_coherent(&dev->nic_info.pdev->dev, mem->size,
-					 &mem->pa, GFP_KERNEL);
+	mem->va = dma_zalloc_coherent(&dev->nic_info.pdev->dev, mem->size,
+				      &mem->pa, GFP_KERNEL);
 	if (!mem->va) {
 		pr_err("%s: stats mbox allocation failed\n", __func__);
 		return false;
 	}
 
-	memset(mem->va, 0, mem->size);
-
 	/* Alloc debugfs mem */
 	mem->debugfs_mem = kzalloc(OCRDMA_MAX_DBGFS_MEM, GFP_KERNEL);
 	if (!mem->debugfs_mem)
@@ -834,7 +832,7 @@ void ocrdma_add_port_stats(struct ocrdma_dev *dev)
 
 	dev->reset_stats.type = OCRDMA_RESET_STATS;
 	dev->reset_stats.dev = dev;
-	if (!debugfs_create_file("reset_stats", S_IRUSR, dev->dir,
+	if (!debugfs_create_file("reset_stats", 0200, dev->dir,
 				&dev->reset_stats, &ocrdma_dbg_ops))
 		goto err;
 
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 7866fd8051f6..8009bdad4e5b 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -461,7 +461,7 @@ static struct ocrdma_pd *_ocrdma_alloc_pd(struct ocrdma_dev *dev,
 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,
 				 struct ocrdma_pd *pd)
 {
-	return (uctx->cntxt_pd == pd ? true : false);
+	return (uctx->cntxt_pd == pd);
 }
 
 static int _ocrdma_dealloc_pd(struct ocrdma_dev *dev,
@@ -550,13 +550,12 @@ struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
 	INIT_LIST_HEAD(&ctx->mm_head);
 	mutex_init(&ctx->mm_list_lock);
 
-	ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,
-					    &ctx->ah_tbl.pa, GFP_KERNEL);
+	ctx->ah_tbl.va = dma_zalloc_coherent(&pdev->dev, map_len,
+					     &ctx->ah_tbl.pa, GFP_KERNEL);
 	if (!ctx->ah_tbl.va) {
 		kfree(ctx);
 		return ERR_PTR(-ENOMEM);
 	}
-	memset(ctx->ah_tbl.va, 0, map_len);
 	ctx->ah_tbl.len = map_len;
 
 	memset(&resp, 0, sizeof(resp));
@@ -885,13 +884,12 @@ static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
 		return -ENOMEM;
 
 	for (i = 0; i < mr->num_pbls; i++) {
-		va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
+		va = dma_zalloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
 		if (!va) {
 			ocrdma_free_mr_pbl_tbl(dev, mr);
 			status = -ENOMEM;
 			break;
 		}
-		memset(va, 0, dma_len);
 		mr->pbl_table[i].va = va;
 		mr->pbl_table[i].pa = pa;
 	}
diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c
index 50812b33291b..0ffb9b93e22d 100644
--- a/drivers/infiniband/hw/qedr/main.c
+++ b/drivers/infiniband/hw/qedr/main.c
@@ -264,7 +264,7 @@ static int qedr_register_device(struct qedr_dev *dev)
 static int qedr_alloc_mem_sb(struct qedr_dev *dev,
 			     struct qed_sb_info *sb_info, u16 sb_id)
 {
-	struct status_block *sb_virt;
+	struct status_block_e4 *sb_virt;
 	dma_addr_t sb_phys;
 	int rc;
 
@@ -430,59 +430,16 @@ static void qedr_remove_sysfiles(struct qedr_dev *dev)
 
 static void qedr_pci_set_atomic(struct qedr_dev *dev, struct pci_dev *pdev)
 {
-	struct pci_dev *bridge;
-	u32 ctl2, cap2;
-	u16 flags;
-	int rc;
-
-	bridge = pdev->bus->self;
-	if (!bridge)
-		goto disable;
-
-	/* Check atomic routing support all the way to root complex */
-	while (bridge->bus->parent) {
-		rc = pcie_capability_read_word(bridge, PCI_EXP_FLAGS, &flags);
-		if (rc || ((flags & PCI_EXP_FLAGS_VERS) < 2))
-			goto disable;
-
-		rc = pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap2);
-		if (rc)
-			goto disable;
+	int rc = pci_enable_atomic_ops_to_root(pdev,
+					       PCI_EXP_DEVCAP2_ATOMIC_COMP64);
 
-		rc = pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, &ctl2);
-		if (rc)
-			goto disable;
-
-		if (!(cap2 & PCI_EXP_DEVCAP2_ATOMIC_ROUTE) ||
-		    (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK))
-			goto disable;
-		bridge = bridge->bus->parent->self;
+	if (rc) {
+		dev->atomic_cap = IB_ATOMIC_NONE;
+		DP_DEBUG(dev, QEDR_MSG_INIT, "Atomic capability disabled\n");
+	} else {
+		dev->atomic_cap = IB_ATOMIC_GLOB;
+		DP_DEBUG(dev, QEDR_MSG_INIT, "Atomic capability enabled\n");
 	}
-
-	rc = pcie_capability_read_word(bridge, PCI_EXP_FLAGS, &flags);
-	if (rc || ((flags & PCI_EXP_FLAGS_VERS) < 2))
-		goto disable;
-
-	rc = pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap2);
-	if (rc || !(cap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP64))
-		goto disable;
-
-	/* Set atomic operations */
-	pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
-				 PCI_EXP_DEVCTL2_ATOMIC_REQ);
-	dev->atomic_cap = IB_ATOMIC_GLOB;
-
-	DP_DEBUG(dev, QEDR_MSG_INIT, "Atomic capability enabled\n");
-
-	return;
-
-disable:
-	pcie_capability_clear_word(pdev, PCI_EXP_DEVCTL2,
-				   PCI_EXP_DEVCTL2_ATOMIC_REQ);
-	dev->atomic_cap = IB_ATOMIC_NONE;
-
-	DP_DEBUG(dev, QEDR_MSG_INIT, "Atomic capability disabled\n");
-
 }
 
 static const struct qed_rdma_ops *qed_ops;
@@ -876,7 +833,8 @@ static struct qedr_dev *qedr_add(struct qed_dev *cdev, struct pci_dev *pdev,
 
 	dev->num_cnq = dev->ops->rdma_get_min_cnq_msix(cdev);
 	if (!dev->num_cnq) {
-		DP_ERR(dev, "not enough CNQ resources.\n");
+		DP_ERR(dev, "Failed. At least one CNQ is required.\n");
+		rc = -ENOMEM;
 		goto init_err;
 	}
 
diff --git a/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h b/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h
index b7587f10e7de..78b49002fbd2 100644
--- a/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h
+++ b/drivers/infiniband/hw/qedr/qedr_hsi_rdma.h
@@ -164,6 +164,13 @@ struct rdma_srq_sge {
 	__le32 l_key;
 };
 
+/* Rdma doorbell data for flags update */
+struct rdma_pwm_flags_data {
+	__le16 icid; /* internal CID */
+	u8 agg_flags; /* aggregative flags */
+	u8 reserved;
+};
+
 /* Rdma doorbell data for SQ and RQ */
 struct rdma_pwm_val16_data {
 	__le16 icid;
@@ -180,12 +187,16 @@ struct rdma_pwm_val32_data {
 	__le16 icid;
 	u8 agg_flags;
 	u8 params;
-#define RDMA_PWM_VAL32_DATA_AGG_CMD_MASK    0x3
-#define RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT   0
-#define RDMA_PWM_VAL32_DATA_BYPASS_EN_MASK  0x1
-#define RDMA_PWM_VAL32_DATA_BYPASS_EN_SHIFT 2
-#define RDMA_PWM_VAL32_DATA_RESERVED_MASK   0x1F
-#define RDMA_PWM_VAL32_DATA_RESERVED_SHIFT  3
+#define RDMA_PWM_VAL32_DATA_AGG_CMD_MASK		0x3
+#define RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT		0
+#define RDMA_PWM_VAL32_DATA_BYPASS_EN_MASK		0x1
+#define RDMA_PWM_VAL32_DATA_BYPASS_EN_SHIFT		2
+#define RDMA_PWM_VAL32_DATA_CONN_TYPE_IS_IWARP_MASK	0x1
+#define RDMA_PWM_VAL32_DATA_CONN_TYPE_IS_IWARP_SHIFT	3
+#define RDMA_PWM_VAL32_DATA_SET_16B_VAL_MASK		0x1
+#define RDMA_PWM_VAL32_DATA_SET_16B_VAL_SHIFT		4
+#define RDMA_PWM_VAL32_DATA_RESERVED_MASK		0x7
+#define RDMA_PWM_VAL32_DATA_RESERVED_SHIFT		5
 	__le32 value;
 };
 
@@ -478,23 +489,25 @@ struct rdma_sq_fmr_wqe {
 	__le16 dif_app_tag_mask;
 	__le16 dif_runt_crc_value;
 	__le16 dif_flags;
-#define RDMA_SQ_FMR_WQE_DIF_IO_DIRECTION_FLG_MASK    0x1
-#define RDMA_SQ_FMR_WQE_DIF_IO_DIRECTION_FLG_SHIFT   0
-#define RDMA_SQ_FMR_WQE_DIF_BLOCK_SIZE_MASK          0x1
-#define RDMA_SQ_FMR_WQE_DIF_BLOCK_SIZE_SHIFT         1
-#define RDMA_SQ_FMR_WQE_DIF_RUNT_VALID_FLG_MASK      0x1
-#define RDMA_SQ_FMR_WQE_DIF_RUNT_VALID_FLG_SHIFT     2
-#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_CRC_GUARD_MASK  0x1
-#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_CRC_GUARD_SHIFT 3
-#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_REF_TAG_MASK    0x1
-#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_REF_TAG_SHIFT   4
-#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_APP_TAG_MASK    0x1
-#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_APP_TAG_SHIFT   5
-#define RDMA_SQ_FMR_WQE_DIF_CRC_SEED_MASK            0x1
-#define RDMA_SQ_FMR_WQE_DIF_CRC_SEED_SHIFT           6
-#define RDMA_SQ_FMR_WQE_RESERVED4_MASK               0x1FF
-#define RDMA_SQ_FMR_WQE_RESERVED4_SHIFT              7
-	__le32 Reserved5;
+#define RDMA_SQ_FMR_WQE_DIF_IO_DIRECTION_FLG_MASK	0x1
+#define RDMA_SQ_FMR_WQE_DIF_IO_DIRECTION_FLG_SHIFT	0
+#define RDMA_SQ_FMR_WQE_DIF_BLOCK_SIZE_MASK		0x1
+#define RDMA_SQ_FMR_WQE_DIF_BLOCK_SIZE_SHIFT		1
+#define RDMA_SQ_FMR_WQE_DIF_RUNT_VALID_FLG_MASK		0x1
+#define RDMA_SQ_FMR_WQE_DIF_RUNT_VALID_FLG_SHIFT	2
+#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_CRC_GUARD_MASK	0x1
+#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_CRC_GUARD_SHIFT	3
+#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_REF_TAG_MASK	0x1
+#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_REF_TAG_SHIFT	4
+#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_APP_TAG_MASK	0x1
+#define RDMA_SQ_FMR_WQE_DIF_VALIDATE_APP_TAG_SHIFT	5
+#define RDMA_SQ_FMR_WQE_DIF_CRC_SEED_MASK		0x1
+#define RDMA_SQ_FMR_WQE_DIF_CRC_SEED_SHIFT		6
+#define RDMA_SQ_FMR_WQE_DIF_RX_REF_TAG_CONST_MASK	0x1
+#define RDMA_SQ_FMR_WQE_DIF_RX_REF_TAG_CONST_SHIFT	7
+#define RDMA_SQ_FMR_WQE_RESERVED4_MASK			0xFF
+#define RDMA_SQ_FMR_WQE_RESERVED4_SHIFT			8
+	__le32 reserved5;
 };
 
 /* First element (16 bytes) of fmr wqe */
@@ -558,23 +571,25 @@ struct rdma_sq_fmr_wqe_3rd {
 	__le16 dif_app_tag_mask;
 	__le16 dif_runt_crc_value;
 	__le16 dif_flags;
-#define RDMA_SQ_FMR_WQE_3RD_DIF_IO_DIRECTION_FLG_MASK    0x1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_IO_DIRECTION_FLG_SHIFT   0
-#define RDMA_SQ_FMR_WQE_3RD_DIF_BLOCK_SIZE_MASK          0x1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_BLOCK_SIZE_SHIFT         1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_RUNT_VALID_FLG_MASK      0x1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_RUNT_VALID_FLG_SHIFT     2
-#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_CRC_GUARD_MASK  0x1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_CRC_GUARD_SHIFT 3
-#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_REF_TAG_MASK    0x1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_REF_TAG_SHIFT   4
-#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_APP_TAG_MASK    0x1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_APP_TAG_SHIFT   5
-#define RDMA_SQ_FMR_WQE_3RD_DIF_CRC_SEED_MASK            0x1
-#define RDMA_SQ_FMR_WQE_3RD_DIF_CRC_SEED_SHIFT           6
-#define RDMA_SQ_FMR_WQE_3RD_RESERVED4_MASK               0x1FF
-#define RDMA_SQ_FMR_WQE_3RD_RESERVED4_SHIFT              7
-	__le32 Reserved5;
+#define RDMA_SQ_FMR_WQE_3RD_DIF_IO_DIRECTION_FLG_MASK		0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_IO_DIRECTION_FLG_SHIFT		0
+#define RDMA_SQ_FMR_WQE_3RD_DIF_BLOCK_SIZE_MASK			0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_BLOCK_SIZE_SHIFT		1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_RUNT_VALID_FLG_MASK		0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_RUNT_VALID_FLG_SHIFT		2
+#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_CRC_GUARD_MASK		0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_CRC_GUARD_SHIFT	3
+#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_REF_TAG_MASK		0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_REF_TAG_SHIFT		4
+#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_APP_TAG_MASK		0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_VALIDATE_APP_TAG_SHIFT		5
+#define RDMA_SQ_FMR_WQE_3RD_DIF_CRC_SEED_MASK			0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_CRC_SEED_SHIFT			6
+#define RDMA_SQ_FMR_WQE_3RD_DIF_RX_REF_TAG_CONST_MASK		0x1
+#define RDMA_SQ_FMR_WQE_3RD_DIF_RX_REF_TAG_CONST_SHIFT		7
+#define RDMA_SQ_FMR_WQE_3RD_RESERVED4_MASK			0xFF
+#define RDMA_SQ_FMR_WQE_RESERVED4_SHIFT				8
+	__le32 reserved5;
 };
 
 struct rdma_sq_local_inv_wqe {
@@ -606,20 +621,22 @@ struct rdma_sq_rdma_wqe {
 	__le32 xrc_srq;
 	u8 req_type;
 	u8 flags;
-#define RDMA_SQ_RDMA_WQE_COMP_FLG_MASK                  0x1
-#define RDMA_SQ_RDMA_WQE_COMP_FLG_SHIFT                 0
-#define RDMA_SQ_RDMA_WQE_RD_FENCE_FLG_MASK              0x1
-#define RDMA_SQ_RDMA_WQE_RD_FENCE_FLG_SHIFT             1
-#define RDMA_SQ_RDMA_WQE_INV_FENCE_FLG_MASK             0x1
-#define RDMA_SQ_RDMA_WQE_INV_FENCE_FLG_SHIFT            2
-#define RDMA_SQ_RDMA_WQE_SE_FLG_MASK                    0x1
-#define RDMA_SQ_RDMA_WQE_SE_FLG_SHIFT                   3
-#define RDMA_SQ_RDMA_WQE_INLINE_FLG_MASK                0x1
-#define RDMA_SQ_RDMA_WQE_INLINE_FLG_SHIFT               4
-#define RDMA_SQ_RDMA_WQE_DIF_ON_HOST_FLG_MASK           0x1
-#define RDMA_SQ_RDMA_WQE_DIF_ON_HOST_FLG_SHIFT          5
-#define RDMA_SQ_RDMA_WQE_RESERVED0_MASK                 0x3
-#define RDMA_SQ_RDMA_WQE_RESERVED0_SHIFT                6
+#define RDMA_SQ_RDMA_WQE_COMP_FLG_MASK		0x1
+#define RDMA_SQ_RDMA_WQE_COMP_FLG_SHIFT		0
+#define RDMA_SQ_RDMA_WQE_RD_FENCE_FLG_MASK	0x1
+#define RDMA_SQ_RDMA_WQE_RD_FENCE_FLG_SHIFT	1
+#define RDMA_SQ_RDMA_WQE_INV_FENCE_FLG_MASK	0x1
+#define RDMA_SQ_RDMA_WQE_INV_FENCE_FLG_SHIFT	2
+#define RDMA_SQ_RDMA_WQE_SE_FLG_MASK		0x1
+#define RDMA_SQ_RDMA_WQE_SE_FLG_SHIFT		3
+#define RDMA_SQ_RDMA_WQE_INLINE_FLG_MASK	0x1
+#define RDMA_SQ_RDMA_WQE_INLINE_FLG_SHIFT	4
+#define RDMA_SQ_RDMA_WQE_DIF_ON_HOST_FLG_MASK	0x1
+#define RDMA_SQ_RDMA_WQE_DIF_ON_HOST_FLG_SHIFT	5
+#define RDMA_SQ_RDMA_WQE_READ_INV_FLG_MASK	0x1
+#define RDMA_SQ_RDMA_WQE_READ_INV_FLG_SHIFT	6
+#define RDMA_SQ_RDMA_WQE_RESERVED0_MASK		0x1
+#define RDMA_SQ_RDMA_WQE_RESERVED0_SHIFT	7
 	u8 wqe_size;
 	u8 prev_wqe_size;
 	struct regpair remote_va;
diff --git a/drivers/infiniband/hw/qedr/qedr_iw_cm.c b/drivers/infiniband/hw/qedr/qedr_iw_cm.c
index 478b7317b80a..26dc374787f7 100644
--- a/drivers/infiniband/hw/qedr/qedr_iw_cm.c
+++ b/drivers/infiniband/hw/qedr/qedr_iw_cm.c
@@ -458,8 +458,7 @@ qedr_addr6_resolve(struct qedr_dev *dev,
 		}
 		return -EINVAL;
 	}
-	neigh = dst_neigh_lookup(dst, &dst_in);
-
+	neigh = dst_neigh_lookup(dst, &fl6.daddr);
 	if (neigh) {
 		rcu_read_lock();
 		if (neigh->nud_state & NUD_VALID) {
@@ -494,10 +493,14 @@ int qedr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 
 	qp = idr_find(&dev->qpidr, conn_param->qpn);
 
-	laddr = (struct sockaddr_in *)&cm_id->local_addr;
-	raddr = (struct sockaddr_in *)&cm_id->remote_addr;
-	laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr;
-	raddr6 = (struct sockaddr_in6 *)&cm_id->remote_addr;
+	laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
+	raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
+	laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
+	raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr;
+
+	DP_DEBUG(dev, QEDR_MSG_IWARP, "MAPPED %d %d\n",
+		 ntohs(((struct sockaddr_in *)&cm_id->remote_addr)->sin_port),
+		 ntohs(raddr->sin_port));
 
 	DP_DEBUG(dev, QEDR_MSG_IWARP,
 		 "Connect source address: %pISpc, remote address: %pISpc\n",
@@ -599,8 +602,8 @@ int qedr_iw_create_listen(struct iw_cm_id *cm_id, int backlog)
 	int rc;
 	int i;
 
-	laddr = (struct sockaddr_in *)&cm_id->local_addr;
-	laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr;
+	laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
+	laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
 
 	DP_DEBUG(dev, QEDR_MSG_IWARP,
 		 "Create Listener address: %pISpc\n", &cm_id->local_addr);
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index b26aa88dab48..419a158e8fca 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -604,12 +604,11 @@ static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 		return ERR_PTR(-ENOMEM);
 
 	for (i = 0; i < pbl_info->num_pbls; i++) {
-		va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
-					&pa, flags);
+		va = dma_zalloc_coherent(&pdev->dev, pbl_info->pbl_size,
+					 &pa, flags);
 		if (!va)
 			goto err;
 
-		memset(va, 0, pbl_info->pbl_size);
 		pbl_table[i].va = va;
 		pbl_table[i].pa = pa;
 	}
@@ -1842,14 +1841,15 @@ static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
 
 static int qedr_update_qp_state(struct qedr_dev *dev,
 				struct qedr_qp *qp,
+				enum qed_roce_qp_state cur_state,
 				enum qed_roce_qp_state new_state)
 {
 	int status = 0;
 
-	if (new_state == qp->state)
+	if (new_state == cur_state)
 		return 0;
 
-	switch (qp->state) {
+	switch (cur_state) {
 	case QED_ROCE_QP_STATE_RESET:
 		switch (new_state) {
 		case QED_ROCE_QP_STATE_INIT:
@@ -1956,6 +1956,7 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
 	enum ib_qp_state old_qp_state, new_qp_state;
+	enum qed_roce_qp_state cur_state;
 	int rc = 0;
 
 	DP_DEBUG(dev, QEDR_MSG_QP,
@@ -2087,18 +2088,23 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		SET_FIELD(qp_params.modify_flags,
 			  QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
 
-		qp_params.ack_timeout = attr->timeout;
-		if (attr->timeout) {
-			u32 temp;
-
-			temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
-			/* FW requires [msec] */
-			qp_params.ack_timeout = temp;
-		} else {
-			/* Infinite */
+		/* The received timeout value is an exponent used like this:
+		 *    "12.7.34 LOCAL ACK TIMEOUT
+		 *    Value representing the transport (ACK) timeout for use by
+		 *    the remote, expressed as: 4.096 * 2^timeout [usec]"
+		 * The FW expects timeout in msec so we need to divide the usec
+		 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
+		 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
+		 * The value of zero means infinite so we use a 'max_t' to make
+		 * sure that sub 1 msec values will be configured as 1 msec.
+		 */
+		if (attr->timeout)
+			qp_params.ack_timeout =
+					1 << max_t(int, attr->timeout - 8, 0);
+		else
 			qp_params.ack_timeout = 0;
-		}
 	}
+
 	if (attr_mask & IB_QP_RETRY_CNT) {
 		SET_FIELD(qp_params.modify_flags,
 			  QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
@@ -2171,13 +2177,25 @@ int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		qp->dest_qp_num = attr->dest_qp_num;
 	}
 
+	cur_state = qp->state;
+
+	/* Update the QP state before the actual ramrod to prevent a race with
+	 * fast path. Modifying the QP state to error will cause the device to
+	 * flush the CQEs and while polling the flushed CQEs will considered as
+	 * a potential issue if the QP isn't in error state.
+	 */
+	if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
+	    !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
+		qp->state = QED_ROCE_QP_STATE_ERR;
+
 	if (qp->qp_type != IB_QPT_GSI)
 		rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
 					      qp->qed_qp, &qp_params);
 
 	if (attr_mask & IB_QP_STATE) {
 		if ((qp->qp_type != IB_QPT_GSI) && (!udata))
-			rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
+			rc = qedr_update_qp_state(dev, qp, cur_state,
+						  qp_params.new_state);
 		qp->state = qp_params.new_state;
 	}
 
@@ -3035,12 +3053,17 @@ static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 
 	switch (wr->opcode) {
 	case IB_WR_SEND_WITH_IMM:
+		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
+			rc = -EINVAL;
+			*bad_wr = wr;
+			break;
+		}
 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
 		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
 		swqe->wqe_size = 2;
 		swqe2 = qed_chain_produce(&qp->sq.pbl);
 
-		swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
+		swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
 						   wr, bad_wr);
 		swqe->length = cpu_to_le32(length);
@@ -3076,6 +3099,11 @@ static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		break;
 
 	case IB_WR_RDMA_WRITE_WITH_IMM:
+		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
+			rc = -EINVAL;
+			*bad_wr = wr;
+			break;
+		}
 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
 		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
 
@@ -3471,9 +3499,9 @@ static int qedr_poll_cq_req(struct qedr_dev *dev,
 		break;
 	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
 		if (qp->state != QED_ROCE_QP_STATE_ERR)
-			DP_ERR(dev,
-			       "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
-			       cq->icid, qp->icid);
+			DP_DEBUG(dev, QEDR_MSG_CQ,
+				 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
+				 cq->icid, qp->icid);
 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
 				  IB_WC_WR_FLUSH_ERR, 1);
 		break;
@@ -3591,7 +3619,7 @@ static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
 	wc->byte_len = le32_to_cpu(resp->length);
 
 	if (resp->flags & QEDR_RESP_IMM) {
-		wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
+		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
 		wc->wc_flags |= IB_WC_WITH_IMM;
 
 		if (resp->flags & QEDR_RESP_RDMA)
@@ -3725,7 +3753,7 @@ int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 {
 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
 	struct qedr_cq *cq = get_qedr_cq(ibcq);
-	union rdma_cqe *cqe = cq->latest_cqe;
+	union rdma_cqe *cqe;
 	u32 old_cons, new_cons;
 	unsigned long flags;
 	int update = 0;
@@ -3742,6 +3770,7 @@ int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 		return qedr_gsi_poll_cq(ibcq, num_entries, wc);
 
 	spin_lock_irqsave(&cq->cq_lock, flags);
+	cqe = cq->latest_cqe;
 	old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
 	while (num_entries && is_valid_cqe(cq, cqe)) {
 		struct qedr_qp *qp;
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 092ed8103842..0235f76bbc72 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -1428,8 +1428,6 @@ u64 qib_sps_ints(void);
  */
 dma_addr_t qib_map_page(struct pci_dev *, struct page *, unsigned long,
 			  size_t, int);
-const char *qib_get_unit_name(int unit);
-const char *qib_get_card_name(struct rvt_dev_info *rdi);
 struct pci_dev *qib_get_pci_dev(struct rvt_dev_info *rdi);
 
 /*
@@ -1488,15 +1486,15 @@ extern struct mutex qib_mutex;
 
 #define qib_dev_err(dd, fmt, ...) \
 	dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
-		qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
+		rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
 
 #define qib_dev_warn(dd, fmt, ...) \
 	dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
-		qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
+		 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), ##__VA_ARGS__)
 
 #define qib_dev_porterr(dd, port, fmt, ...) \
 	dev_err(&(dd)->pcidev->dev, "%s: IB%u:%u " fmt, \
-		qib_get_unit_name((dd)->unit), (dd)->unit, (port), \
+		rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), (dd)->unit, (port), \
 		##__VA_ARGS__)
 
 #define qib_devinfo(pcidev, fmt, ...) \
diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c
index 33d3335385e8..3117cc5f2a9a 100644
--- a/drivers/infiniband/hw/qib/qib_driver.c
+++ b/drivers/infiniband/hw/qib/qib_driver.c
@@ -81,22 +81,6 @@ MODULE_DESCRIPTION("Intel IB driver");
 
 struct qlogic_ib_stats qib_stats;
 
-const char *qib_get_unit_name(int unit)
-{
-	static char iname[16];
-
-	snprintf(iname, sizeof(iname), "infinipath%u", unit);
-	return iname;
-}
-
-const char *qib_get_card_name(struct rvt_dev_info *rdi)
-{
-	struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi);
-	struct qib_devdata *dd = container_of(ibdev,
-					      struct qib_devdata, verbs_dev);
-	return qib_get_unit_name(dd->unit);
-}
-
 struct pci_dev *qib_get_pci_dev(struct rvt_dev_info *rdi)
 {
 	struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi);
diff --git a/drivers/infiniband/hw/qib/qib_eeprom.c b/drivers/infiniband/hw/qib/qib_eeprom.c
index 33a2e74c8495..5838b3bf34b9 100644
--- a/drivers/infiniband/hw/qib/qib_eeprom.c
+++ b/drivers/infiniband/hw/qib/qib_eeprom.c
@@ -163,8 +163,7 @@ void qib_get_eeprom_info(struct qib_devdata *dd)
 			if (bguid[6] == 0xff) {
 				if (bguid[5] == 0xff) {
 					qib_dev_err(dd,
-						"Can't set %s GUID from base, wraps to OUI!\n",
-						qib_get_unit_name(t));
+						    "Can't set GUID from base, wraps to OUI!\n");
 					dd->base_guid = 0;
 					goto bail;
 				}
diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c
index 2d6a191afec0..52c29db3a2f4 100644
--- a/drivers/infiniband/hw/qib/qib_file_ops.c
+++ b/drivers/infiniband/hw/qib/qib_file_ops.c
@@ -58,7 +58,7 @@ static int qib_open(struct inode *, struct file *);
 static int qib_close(struct inode *, struct file *);
 static ssize_t qib_write(struct file *, const char __user *, size_t, loff_t *);
 static ssize_t qib_write_iter(struct kiocb *, struct iov_iter *);
-static unsigned int qib_poll(struct file *, struct poll_table_struct *);
+static __poll_t qib_poll(struct file *, struct poll_table_struct *);
 static int qib_mmapf(struct file *, struct vm_area_struct *);
 
 /*
@@ -568,20 +568,16 @@ static int qib_tid_free(struct qib_ctxtdata *rcd, unsigned subctxt,
 static int qib_set_part_key(struct qib_ctxtdata *rcd, u16 key)
 {
 	struct qib_pportdata *ppd = rcd->ppd;
-	int i, any = 0, pidx = -1;
+	int i, pidx = -1;
+	bool any = false;
 	u16 lkey = key & 0x7FFF;
-	int ret;
 
-	if (lkey == (QIB_DEFAULT_P_KEY & 0x7FFF)) {
+	if (lkey == (QIB_DEFAULT_P_KEY & 0x7FFF))
 		/* nothing to do; this key always valid */
-		ret = 0;
-		goto bail;
-	}
+		return 0;
 
-	if (!lkey) {
-		ret = -EINVAL;
-		goto bail;
-	}
+	if (!lkey)
+		return -EINVAL;
 
 	/*
 	 * Set the full membership bit, because it has to be
@@ -594,18 +590,14 @@ static int qib_set_part_key(struct qib_ctxtdata *rcd, u16 key)
 	for (i = 0; i < ARRAY_SIZE(rcd->pkeys); i++) {
 		if (!rcd->pkeys[i] && pidx == -1)
 			pidx = i;
-		if (rcd->pkeys[i] == key) {
-			ret = -EEXIST;
-			goto bail;
-		}
+		if (rcd->pkeys[i] == key)
+			return -EEXIST;
 	}
-	if (pidx == -1) {
-		ret = -EBUSY;
-		goto bail;
-	}
-	for (any = i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) {
+	if (pidx == -1)
+		return -EBUSY;
+	for (i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) {
 		if (!ppd->pkeys[i]) {
-			any++;
+			any = true;
 			continue;
 		}
 		if (ppd->pkeys[i] == key) {
@@ -613,44 +605,34 @@ static int qib_set_part_key(struct qib_ctxtdata *rcd, u16 key)
 
 			if (atomic_inc_return(pkrefs) > 1) {
 				rcd->pkeys[pidx] = key;
-				ret = 0;
-				goto bail;
-			} else {
-				/*
-				 * lost race, decrement count, catch below
-				 */
-				atomic_dec(pkrefs);
-				any++;
+				return 0;
 			}
+			/*
+			 * lost race, decrement count, catch below
+			 */
+			atomic_dec(pkrefs);
+			any = true;
 		}
-		if ((ppd->pkeys[i] & 0x7FFF) == lkey) {
+		if ((ppd->pkeys[i] & 0x7FFF) == lkey)
 			/*
 			 * It makes no sense to have both the limited and
 			 * full membership PKEY set at the same time since
 			 * the unlimited one will disable the limited one.
 			 */
-			ret = -EEXIST;
-			goto bail;
-		}
-	}
-	if (!any) {
-		ret = -EBUSY;
-		goto bail;
+			return -EEXIST;
 	}
-	for (any = i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) {
+	if (!any)
+		return -EBUSY;
+	for (i = 0; i < ARRAY_SIZE(ppd->pkeys); i++) {
 		if (!ppd->pkeys[i] &&
 		    atomic_inc_return(&ppd->pkeyrefs[i]) == 1) {
 			rcd->pkeys[pidx] = key;
 			ppd->pkeys[i] = key;
 			(void) ppd->dd->f_set_ib_cfg(ppd, QIB_IB_CFG_PKEYS, 0);
-			ret = 0;
-			goto bail;
+			return 0;
 		}
 	}
-	ret = -EBUSY;
-
-bail:
-	return ret;
+	return -EBUSY;
 }
 
 /**
@@ -1092,18 +1074,18 @@ static int qib_mmapf(struct file *fp, struct vm_area_struct *vma)
 	return ret;
 }
 
-static unsigned int qib_poll_urgent(struct qib_ctxtdata *rcd,
+static __poll_t qib_poll_urgent(struct qib_ctxtdata *rcd,
 				    struct file *fp,
 				    struct poll_table_struct *pt)
 {
 	struct qib_devdata *dd = rcd->dd;
-	unsigned pollflag;
+	__poll_t pollflag;
 
 	poll_wait(fp, &rcd->wait, pt);
 
 	spin_lock_irq(&dd->uctxt_lock);
 	if (rcd->urgent != rcd->urgent_poll) {
-		pollflag = POLLIN | POLLRDNORM;
+		pollflag = EPOLLIN | EPOLLRDNORM;
 		rcd->urgent_poll = rcd->urgent;
 	} else {
 		pollflag = 0;
@@ -1114,12 +1096,12 @@ static unsigned int qib_poll_urgent(struct qib_ctxtdata *rcd,
 	return pollflag;
 }
 
-static unsigned int qib_poll_next(struct qib_ctxtdata *rcd,
+static __poll_t qib_poll_next(struct qib_ctxtdata *rcd,
 				  struct file *fp,
 				  struct poll_table_struct *pt)
 {
 	struct qib_devdata *dd = rcd->dd;
-	unsigned pollflag;
+	__poll_t pollflag;
 
 	poll_wait(fp, &rcd->wait, pt);
 
@@ -1129,26 +1111,26 @@ static unsigned int qib_poll_next(struct qib_ctxtdata *rcd,
 		dd->f_rcvctrl(rcd->ppd, QIB_RCVCTRL_INTRAVAIL_ENB, rcd->ctxt);
 		pollflag = 0;
 	} else
-		pollflag = POLLIN | POLLRDNORM;
+		pollflag = EPOLLIN | EPOLLRDNORM;
 	spin_unlock_irq(&dd->uctxt_lock);
 
 	return pollflag;
 }
 
-static unsigned int qib_poll(struct file *fp, struct poll_table_struct *pt)
+static __poll_t qib_poll(struct file *fp, struct poll_table_struct *pt)
 {
 	struct qib_ctxtdata *rcd;
-	unsigned pollflag;
+	__poll_t pollflag;
 
 	rcd = ctxt_fp(fp);
 	if (!rcd)
-		pollflag = POLLERR;
+		pollflag = EPOLLERR;
 	else if (rcd->poll_type == QIB_POLL_TYPE_URGENT)
 		pollflag = qib_poll_urgent(rcd, fp, pt);
 	else  if (rcd->poll_type == QIB_POLL_TYPE_ANYRCV)
 		pollflag = qib_poll_next(rcd, fp, pt);
 	else /* invalid */
-		pollflag = POLLERR;
+		pollflag = EPOLLERR;
 
 	return pollflag;
 }
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index 85dfbba427f6..3990f386aa32 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -1119,6 +1119,8 @@ struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
 			      "Could not allocate unit ID: error %d\n", -ret);
 		goto bail;
 	}
+	rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s%d", "qib", dd->unit);
+
 	dd->int_counter = alloc_percpu(u64);
 	if (!dd->int_counter) {
 		ret = -ENOMEM;
diff --git a/drivers/infiniband/hw/qib/qib_keys.c b/drivers/infiniband/hw/qib/qib_keys.c
deleted file mode 100644
index 8fdf79f8d4e4..000000000000
--- a/drivers/infiniband/hw/qib/qib_keys.c
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Copyright (c) 2006, 2007, 2009 QLogic Corporation. All rights reserved.
- * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "qib.h"
-
-/**
- * qib_alloc_lkey - allocate an lkey
- * @mr: memory region that this lkey protects
- * @dma_region: 0->normal key, 1->restricted DMA key
- *
- * Returns 0 if successful, otherwise returns -errno.
- *
- * Increments mr reference count as required.
- *
- * Sets the lkey field mr for non-dma regions.
- *
- */
-
-int qib_alloc_lkey(struct rvt_mregion *mr, int dma_region)
-{
-	unsigned long flags;
-	u32 r;
-	u32 n;
-	int ret = 0;
-	struct qib_ibdev *dev = to_idev(mr->pd->device);
-	struct rvt_lkey_table *rkt = &dev->lk_table;
-
-	spin_lock_irqsave(&rkt->lock, flags);
-
-	/* special case for dma_mr lkey == 0 */
-	if (dma_region) {
-		struct rvt_mregion *tmr;
-
-		tmr = rcu_access_pointer(dev->dma_mr);
-		if (!tmr) {
-			qib_get_mr(mr);
-			rcu_assign_pointer(dev->dma_mr, mr);
-			mr->lkey_published = 1;
-		}
-		goto success;
-	}
-
-	/* Find the next available LKEY */
-	r = rkt->next;
-	n = r;
-	for (;;) {
-		if (rkt->table[r] == NULL)
-			break;
-		r = (r + 1) & (rkt->max - 1);
-		if (r == n)
-			goto bail;
-	}
-	rkt->next = (r + 1) & (rkt->max - 1);
-	/*
-	 * Make sure lkey is never zero which is reserved to indicate an
-	 * unrestricted LKEY.
-	 */
-	rkt->gen++;
-	/*
-	 * bits are capped in qib_verbs.c to insure enough bits
-	 * for generation number
-	 */
-	mr->lkey = (r << (32 - ib_rvt_lkey_table_size)) |
-		((((1 << (24 - ib_rvt_lkey_table_size)) - 1) & rkt->gen)
-		 << 8);
-	if (mr->lkey == 0) {
-		mr->lkey |= 1 << 8;
-		rkt->gen++;
-	}
-	qib_get_mr(mr);
-	rcu_assign_pointer(rkt->table[r], mr);
-	mr->lkey_published = 1;
-success:
-	spin_unlock_irqrestore(&rkt->lock, flags);
-out:
-	return ret;
-bail:
-	spin_unlock_irqrestore(&rkt->lock, flags);
-	ret = -ENOMEM;
-	goto out;
-}
-
-/**
- * qib_free_lkey - free an lkey
- * @mr: mr to free from tables
- */
-void qib_free_lkey(struct rvt_mregion *mr)
-{
-	unsigned long flags;
-	u32 lkey = mr->lkey;
-	u32 r;
-	struct qib_ibdev *dev = to_idev(mr->pd->device);
-	struct rvt_lkey_table *rkt = &dev->lk_table;
-
-	spin_lock_irqsave(&rkt->lock, flags);
-	if (!mr->lkey_published)
-		goto out;
-	if (lkey == 0)
-		RCU_INIT_POINTER(dev->dma_mr, NULL);
-	else {
-		r = lkey >> (32 - ib_rvt_lkey_table_size);
-		RCU_INIT_POINTER(rkt->table[r], NULL);
-	}
-	qib_put_mr(mr);
-	mr->lkey_published = 0;
-out:
-	spin_unlock_irqrestore(&rkt->lock, flags);
-}
-
-/**
- * qib_rkey_ok - check the IB virtual address, length, and RKEY
- * @qp: qp for validation
- * @sge: SGE state
- * @len: length of data
- * @vaddr: virtual address to place data
- * @rkey: rkey to check
- * @acc: access flags
- *
- * Return 1 if successful, otherwise 0.
- *
- * increments the reference count upon success
- */
-int qib_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
-		u32 len, u64 vaddr, u32 rkey, int acc)
-{
-	struct rvt_lkey_table *rkt = &to_idev(qp->ibqp.device)->lk_table;
-	struct rvt_mregion *mr;
-	unsigned n, m;
-	size_t off;
-
-	/* We use RKEY == zero for kernel virtual addresses */
-	rcu_read_lock();
-	if (rkey == 0) {
-		struct rvt_pd *pd = ibpd_to_rvtpd(qp->ibqp.pd);
-		struct qib_ibdev *dev = to_idev(pd->ibpd.device);
-
-		if (pd->user)
-			goto bail;
-		mr = rcu_dereference(dev->dma_mr);
-		if (!mr)
-			goto bail;
-		if (unlikely(!atomic_inc_not_zero(&mr->refcount)))
-			goto bail;
-		rcu_read_unlock();
-
-		sge->mr = mr;
-		sge->vaddr = (void *) vaddr;
-		sge->length = len;
-		sge->sge_length = len;
-		sge->m = 0;
-		sge->n = 0;
-		goto ok;
-	}
-
-	mr = rcu_dereference(
-		rkt->table[(rkey >> (32 - ib_rvt_lkey_table_size))]);
-	if (unlikely(!mr || mr->lkey != rkey || qp->ibqp.pd != mr->pd))
-		goto bail;
-
-	off = vaddr - mr->iova;
-	if (unlikely(vaddr < mr->iova || off + len > mr->length ||
-		     (mr->access_flags & acc) == 0))
-		goto bail;
-	if (unlikely(!atomic_inc_not_zero(&mr->refcount)))
-		goto bail;
-	rcu_read_unlock();
-
-	off += mr->offset;
-	if (mr->page_shift) {
-		/*
-		page sizes are uniform power of 2 so no loop is necessary
-		entries_spanned_by_off is the number of times the loop below
-		would have executed.
-		*/
-		size_t entries_spanned_by_off;
-
-		entries_spanned_by_off = off >> mr->page_shift;
-		off -= (entries_spanned_by_off << mr->page_shift);
-		m = entries_spanned_by_off / RVT_SEGSZ;
-		n = entries_spanned_by_off % RVT_SEGSZ;
-	} else {
-		m = 0;
-		n = 0;
-		while (off >= mr->map[m]->segs[n].length) {
-			off -= mr->map[m]->segs[n].length;
-			n++;
-			if (n >= RVT_SEGSZ) {
-				m++;
-				n = 0;
-			}
-		}
-	}
-	sge->mr = mr;
-	sge->vaddr = mr->map[m]->segs[n].vaddr + off;
-	sge->length = mr->map[m]->segs[n].length - off;
-	sge->sge_length = len;
-	sge->m = m;
-	sge->n = n;
-ok:
-	return 1;
-bail:
-	rcu_read_unlock();
-	return 0;
-}
-
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index 8f5754fb8579..c9955d48c50f 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -246,7 +246,6 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 		if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
 			goto bail;
 		/* We are in the error state, flush the work request. */
-		smp_read_barrier_depends(); /* see post_one_send() */
 		if (qp->s_last == READ_ONCE(qp->s_head))
 			goto bail;
 		/* If DMAs are in progress, we can't flush immediately. */
@@ -293,7 +292,6 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 		newreq = 0;
 		if (qp->s_cur == qp->s_tail) {
 			/* Check if send work queue is empty. */
-			smp_read_barrier_depends(); /* see post_one_send() */
 			if (qp->s_tail == READ_ONCE(qp->s_head))
 				goto bail;
 			/*
@@ -434,13 +432,13 @@ int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 				qp->s_state = OP(COMPARE_SWAP);
 				put_ib_ateth_swap(wqe->atomic_wr.swap,
 						  &ohdr->u.atomic_eth);
-				put_ib_ateth_swap(wqe->atomic_wr.compare_add,
-						  &ohdr->u.atomic_eth);
+				put_ib_ateth_compare(wqe->atomic_wr.compare_add,
+						     &ohdr->u.atomic_eth);
 			} else {
 				qp->s_state = OP(FETCH_ADD);
 				put_ib_ateth_swap(wqe->atomic_wr.compare_add,
 						  &ohdr->u.atomic_eth);
-				put_ib_ateth_swap(0, &ohdr->u.atomic_eth);
+				put_ib_ateth_compare(0, &ohdr->u.atomic_eth);
 			}
 			put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
 					   &ohdr->u.atomic_eth);
@@ -1340,7 +1338,6 @@ static void qib_rc_rcv_resp(struct qib_ibport *ibp,
 		goto ack_done;
 
 	/* Ignore invalid responses. */
-	smp_read_barrier_depends(); /* see post_one_send */
 	if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0)
 		goto ack_done;
 
@@ -1916,8 +1913,7 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
 		wc.port_num = 0;
 		/* Signal completion event if the solicited bit is set. */
 		rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
-			     (ohdr->bth[0] &
-			      cpu_to_be32(IB_BTH_SOLICITED)) != 0);
+			     ib_bth_is_solicited(ohdr));
 		break;
 
 	case OP(RDMA_WRITE_FIRST):
diff --git a/drivers/infiniband/hw/qib/qib_ruc.c b/drivers/infiniband/hw/qib/qib_ruc.c
index 9a37e844d4c8..4662cc7bde92 100644
--- a/drivers/infiniband/hw/qib/qib_ruc.c
+++ b/drivers/infiniband/hw/qib/qib_ruc.c
@@ -367,7 +367,6 @@ static void qib_ruc_loopback(struct rvt_qp *sqp)
 	sqp->s_flags |= RVT_S_BUSY;
 
 again:
-	smp_read_barrier_depends(); /* see post_one_send() */
 	if (sqp->s_last == READ_ONCE(sqp->s_head))
 		goto clr_busy;
 	wqe = rvt_get_swqe_ptr(sqp, sqp->s_last);
diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c
index bddcc37ace44..840eec6ebc33 100644
--- a/drivers/infiniband/hw/qib/qib_uc.c
+++ b/drivers/infiniband/hw/qib/qib_uc.c
@@ -60,7 +60,6 @@ int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags)
 		if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
 			goto bail;
 		/* We are in the error state, flush the work request. */
-		smp_read_barrier_depends(); /* see post_one_send() */
 		if (qp->s_last == READ_ONCE(qp->s_head))
 			goto bail;
 		/* If DMAs are in progress, we can't flush immediately. */
@@ -90,7 +89,6 @@ int qib_make_uc_req(struct rvt_qp *qp, unsigned long *flags)
 		    RVT_PROCESS_NEXT_SEND_OK))
 			goto bail;
 		/* Check if send work queue is empty. */
-		smp_read_barrier_depends(); /* see post_one_send() */
 		if (qp->s_cur == READ_ONCE(qp->s_head))
 			goto bail;
 		/*
@@ -403,8 +401,7 @@ void qib_uc_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 		wc.port_num = 0;
 		/* Signal completion event if the solicited bit is set. */
 		rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
-			     (ohdr->bth[0] &
-				cpu_to_be32(IB_BTH_SOLICITED)) != 0);
+			     ib_bth_is_solicited(ohdr));
 		break;
 
 	case OP(RDMA_WRITE_FIRST):
diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c
index 15962ed193ce..3e4ff77260c2 100644
--- a/drivers/infiniband/hw/qib/qib_ud.c
+++ b/drivers/infiniband/hw/qib/qib_ud.c
@@ -252,7 +252,6 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags)
 		if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
 			goto bail;
 		/* We are in the error state, flush the work request. */
-		smp_read_barrier_depends(); /* see post_one_send */
 		if (qp->s_last == READ_ONCE(qp->s_head))
 			goto bail;
 		/* If DMAs are in progress, we can't flush immediately. */
@@ -266,7 +265,6 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags)
 	}
 
 	/* see post_one_send() */
-	smp_read_barrier_depends();
 	if (qp->s_cur == READ_ONCE(qp->s_head))
 		goto bail;
 
@@ -581,8 +579,7 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr,
 	wc.port_num = qp->port_num;
 	/* Signal completion event if the solicited bit is set. */
 	rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
-		     (ohdr->bth[0] &
-			cpu_to_be32(IB_BTH_SOLICITED)) != 0);
+		     ib_bth_is_solicited(ohdr));
 	return;
 
 drop:
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index c55000501582..fabee760407e 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -1571,7 +1571,6 @@ int qib_register_ib_device(struct qib_devdata *dd)
 	if (!ib_qib_sys_image_guid)
 		ib_qib_sys_image_guid = ppd->guid;
 
-	strlcpy(ibdev->name, "qib%d", IB_DEVICE_NAME_MAX);
 	ibdev->owner = THIS_MODULE;
 	ibdev->node_guid = ppd->guid;
 	ibdev->phys_port_cnt = dd->num_pports;
@@ -1586,7 +1585,6 @@ int qib_register_ib_device(struct qib_devdata *dd)
 	 * Fill in rvt info object.
 	 */
 	dd->verbs_dev.rdi.driver_f.port_callback = qib_create_port_files;
-	dd->verbs_dev.rdi.driver_f.get_card_name = qib_get_card_name;
 	dd->verbs_dev.rdi.driver_f.get_pci_dev = qib_get_pci_dev;
 	dd->verbs_dev.rdi.driver_f.check_ah = qib_check_ah;
 	dd->verbs_dev.rdi.driver_f.check_send_wqe = qib_check_send_wqe;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c b/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c
index 685ef2293cb8..4210ca14014d 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_sysfs.c
@@ -45,7 +45,6 @@
 #include "usnic_ib_verbs.h"
 #include "usnic_ib_sysfs.h"
 #include "usnic_log.h"
-#include "usnic_ib_sysfs.h"
 
 static ssize_t usnic_ib_show_board(struct device *device,
 					struct device_attribute *attr,
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index aa2456a4f9bd..a688a5669168 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -47,7 +47,6 @@
 #include "usnic_log.h"
 #include "usnic_uiom.h"
 #include "usnic_transport.h"
-#include "usnic_ib_verbs.h"
 
 #define USNIC_DEFAULT_TRANSPORT USNIC_TRANSPORT_ROCE_CUSTOM
 
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
index 4f7bd3b6a315..44cb1cfba417 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
@@ -93,7 +93,7 @@ struct pvrdma_cq {
 	struct pvrdma_page_dir pdir;
 	u32 cq_handle;
 	bool is_kernel;
-	atomic_t refcnt;
+	refcount_t refcnt;
 	struct completion free;
 };
 
@@ -196,7 +196,7 @@ struct pvrdma_qp {
 	u8 state;
 	bool is_kernel;
 	struct mutex mutex; /* QP state mutex. */
-	atomic_t refcnt;
+	refcount_t refcnt;
 	struct completion free;
 };
 
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
index e529622cefad..f95b97646c25 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
@@ -114,6 +114,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 	union pvrdma_cmd_resp rsp;
 	struct pvrdma_cmd_create_cq *cmd = &req.create_cq;
 	struct pvrdma_cmd_create_cq_resp *resp = &rsp.create_cq_resp;
+	struct pvrdma_create_cq_resp cq_resp = {0};
 	struct pvrdma_create_cq ucmd;
 
 	BUILD_BUG_ON(sizeof(struct pvrdma_cqe) != 64);
@@ -132,8 +133,9 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 	}
 
 	cq->ibcq.cqe = entries;
+	cq->is_kernel = !context;
 
-	if (context) {
+	if (!cq->is_kernel) {
 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
 			ret = -EFAULT;
 			goto err_cq;
@@ -148,8 +150,6 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 
 		npages = ib_umem_page_count(cq->umem);
 	} else {
-		cq->is_kernel = true;
-
 		/* One extra page for shared ring state */
 		npages = 1 + (entries * sizeof(struct pvrdma_cqe) +
 			      PAGE_SIZE - 1) / PAGE_SIZE;
@@ -178,7 +178,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 	else
 		pvrdma_page_dir_insert_umem(&cq->pdir, cq->umem, 0);
 
-	atomic_set(&cq->refcnt, 1);
+	refcount_set(&cq->refcnt, 1);
 	init_completion(&cq->free);
 	spin_lock_init(&cq->cq_lock);
 
@@ -198,15 +198,16 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 
 	cq->ibcq.cqe = resp->cqe;
 	cq->cq_handle = resp->cq_handle;
+	cq_resp.cqn = resp->cq_handle;
 	spin_lock_irqsave(&dev->cq_tbl_lock, flags);
 	dev->cq_tbl[cq->cq_handle % dev->dsr->caps.max_cq] = cq;
 	spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
 
-	if (context) {
+	if (!cq->is_kernel) {
 		cq->uar = &(to_vucontext(context)->uar);
 
 		/* Copy udata back. */
-		if (ib_copy_to_udata(udata, &cq->cq_handle, sizeof(__u32))) {
+		if (ib_copy_to_udata(udata, &cq_resp, sizeof(cq_resp))) {
 			dev_warn(&dev->pdev->dev,
 				 "failed to copy back udata\n");
 			pvrdma_destroy_cq(&cq->ibcq);
@@ -219,7 +220,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 err_page_dir:
 	pvrdma_page_dir_cleanup(dev, &cq->pdir);
 err_umem:
-	if (context)
+	if (!cq->is_kernel)
 		ib_umem_release(cq->umem);
 err_cq:
 	atomic_dec(&dev->num_cqs);
@@ -230,7 +231,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 
 static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq)
 {
-	if (atomic_dec_and_test(&cq->refcnt))
+	if (refcount_dec_and_test(&cq->refcnt))
 		complete(&cq->free);
 	wait_for_completion(&cq->free);
 
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
index e92681878c93..d650a9fcde24 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
@@ -243,13 +243,13 @@ static int pvrdma_register_device(struct pvrdma_dev *dev)
 	mutex_init(&dev->port_mutex);
 	spin_lock_init(&dev->desc_lock);
 
-	dev->cq_tbl = kcalloc(dev->dsr->caps.max_cq, sizeof(void *),
+	dev->cq_tbl = kcalloc(dev->dsr->caps.max_cq, sizeof(struct pvrdma_cq *),
 			      GFP_KERNEL);
 	if (!dev->cq_tbl)
 		return ret;
 	spin_lock_init(&dev->cq_tbl_lock);
 
-	dev->qp_tbl = kcalloc(dev->dsr->caps.max_qp, sizeof(void *),
+	dev->qp_tbl = kcalloc(dev->dsr->caps.max_qp, sizeof(struct pvrdma_qp *),
 			      GFP_KERNEL);
 	if (!dev->qp_tbl)
 		goto err_cq_free;
@@ -333,7 +333,7 @@ static void pvrdma_qp_event(struct pvrdma_dev *dev, u32 qpn, int type)
 	spin_lock_irqsave(&dev->qp_tbl_lock, flags);
 	qp = dev->qp_tbl[qpn % dev->dsr->caps.max_qp];
 	if (qp)
-		atomic_inc(&qp->refcnt);
+		refcount_inc(&qp->refcnt);
 	spin_unlock_irqrestore(&dev->qp_tbl_lock, flags);
 
 	if (qp && qp->ibqp.event_handler) {
@@ -346,7 +346,7 @@ static void pvrdma_qp_event(struct pvrdma_dev *dev, u32 qpn, int type)
 		ibqp->event_handler(&e, ibqp->qp_context);
 	}
 	if (qp) {
-		if (atomic_dec_and_test(&qp->refcnt))
+		if (refcount_dec_and_test(&qp->refcnt))
 			complete(&qp->free);
 	}
 }
@@ -359,7 +359,7 @@ static void pvrdma_cq_event(struct pvrdma_dev *dev, u32 cqn, int type)
 	spin_lock_irqsave(&dev->cq_tbl_lock, flags);
 	cq = dev->cq_tbl[cqn % dev->dsr->caps.max_cq];
 	if (cq)
-		atomic_inc(&cq->refcnt);
+		refcount_inc(&cq->refcnt);
 	spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
 
 	if (cq && cq->ibcq.event_handler) {
@@ -372,7 +372,7 @@ static void pvrdma_cq_event(struct pvrdma_dev *dev, u32 cqn, int type)
 		ibcq->event_handler(&e, ibcq->cq_context);
 	}
 	if (cq) {
-		if (atomic_dec_and_test(&cq->refcnt))
+		if (refcount_dec_and_test(&cq->refcnt))
 			complete(&cq->free);
 	}
 }
@@ -531,13 +531,13 @@ static irqreturn_t pvrdma_intrx_handler(int irq, void *dev_id)
 		spin_lock_irqsave(&dev->cq_tbl_lock, flags);
 		cq = dev->cq_tbl[cqne->info % dev->dsr->caps.max_cq];
 		if (cq)
-			atomic_inc(&cq->refcnt);
+			refcount_inc(&cq->refcnt);
 		spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
 
 		if (cq && cq->ibcq.comp_handler)
 			cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 		if (cq) {
-			if (atomic_dec_and_test(&cq->refcnt))
+			if (refcount_dec_and_test(&cq->refcnt))
 				complete(&cq->free);
 		}
 		pvrdma_idx_ring_inc(&ring->cons_head, ring_slots);
@@ -882,8 +882,8 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	dev_info(&pdev->dev, "device version %d, driver version %d\n",
 		 dev->dsr_version, PVRDMA_VERSION);
 
-	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
-				      &dev->dsrbase, GFP_KERNEL);
+	dev->dsr = dma_zalloc_coherent(&pdev->dev, sizeof(*dev->dsr),
+				       &dev->dsrbase, GFP_KERNEL);
 	if (!dev->dsr) {
 		dev_err(&pdev->dev, "failed to allocate shared region\n");
 		ret = -ENOMEM;
@@ -891,7 +891,6 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	}
 
 	/* Setup the shared region */
-	memset(dev->dsr, 0, sizeof(*dev->dsr));
 	dev->dsr->driver_version = PVRDMA_VERSION;
 	dev->dsr->gos_info.gos_bits = sizeof(void *) == 4 ?
 		PVRDMA_GOS_BITS_32 :
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
index 8519f3212e52..fa96fa4fb829 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
@@ -119,10 +119,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	union pvrdma_cmd_resp rsp;
 	struct pvrdma_cmd_create_mr *cmd = &req.create_mr;
 	struct pvrdma_cmd_create_mr_resp *resp = &rsp.create_mr_resp;
-	int nchunks;
 	int ret;
-	int entry;
-	struct scatterlist *sg;
 
 	if (length == 0 || length > dev->dsr->caps.max_mr_size) {
 		dev_warn(&dev->pdev->dev, "invalid mem region length\n");
@@ -137,13 +134,9 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 		return ERR_CAST(umem);
 	}
 
-	nchunks = 0;
-	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry)
-		nchunks += sg_dma_len(sg) >> PAGE_SHIFT;
-
-	if (nchunks < 0 || nchunks > PVRDMA_PAGE_DIR_MAX_PAGES) {
+	if (umem->npages < 0 || umem->npages > PVRDMA_PAGE_DIR_MAX_PAGES) {
 		dev_warn(&dev->pdev->dev, "overflow %d pages in mem region\n",
-			 nchunks);
+			 umem->npages);
 		ret = -EINVAL;
 		goto err_umem;
 	}
@@ -158,7 +151,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	mr->mmr.size = length;
 	mr->umem = umem;
 
-	ret = pvrdma_page_dir_init(dev, &mr->pdir, nchunks, false);
+	ret = pvrdma_page_dir_init(dev, &mr->pdir, umem->npages, false);
 	if (ret) {
 		dev_warn(&dev->pdev->dev,
 			 "could not allocate page directory\n");
@@ -175,7 +168,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	cmd->length = length;
 	cmd->pd_handle = to_vpd(pd)->pd_handle;
 	cmd->access_flags = access_flags;
-	cmd->nchunks = nchunks;
+	cmd->nchunks = umem->npages;
 	cmd->pdir_dma = mr->pdir.dir_dma;
 
 	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_MR_RESP);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
index 4059308e1454..7bf518bdbf21 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
@@ -245,12 +245,13 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
 		spin_lock_init(&qp->sq.lock);
 		spin_lock_init(&qp->rq.lock);
 		mutex_init(&qp->mutex);
-		atomic_set(&qp->refcnt, 1);
+		refcount_set(&qp->refcnt, 1);
 		init_completion(&qp->free);
 
 		qp->state = IB_QPS_RESET;
+		qp->is_kernel = !(pd->uobject && udata);
 
-		if (pd->uobject && udata) {
+		if (!qp->is_kernel) {
 			dev_dbg(&dev->pdev->dev,
 				"create queuepair from user space\n");
 
@@ -291,8 +292,6 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
 				qp->npages_recv = 0;
 			qp->npages = qp->npages_send + qp->npages_recv;
 		} else {
-			qp->is_kernel = true;
-
 			ret = pvrdma_set_sq_size(to_vdev(pd->device),
 						 &init_attr->cap, qp);
 			if (ret)
@@ -394,7 +393,7 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
 err_pdir:
 	pvrdma_page_dir_cleanup(dev, &qp->pdir);
 err_umem:
-	if (pd->uobject && udata) {
+	if (!qp->is_kernel) {
 		if (qp->rumem)
 			ib_umem_release(qp->rumem);
 		if (qp->sumem)
@@ -428,7 +427,7 @@ static void pvrdma_free_qp(struct pvrdma_qp *qp)
 
 	pvrdma_unlock_cqs(scq, rcq, &scq_flags, &rcq_flags);
 
-	if (atomic_dec_and_test(&qp->refcnt))
+	if (refcount_dec_and_test(&qp->refcnt))
 		complete(&qp->free);
 	wait_for_completion(&qp->free);
 
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
index 5acebb1ef631..af235967a9c2 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
@@ -113,6 +113,7 @@ struct ib_srq *pvrdma_create_srq(struct ib_pd *pd,
 	union pvrdma_cmd_resp rsp;
 	struct pvrdma_cmd_create_srq *cmd = &req.create_srq;
 	struct pvrdma_cmd_create_srq_resp *resp = &rsp.create_srq_resp;
+	struct pvrdma_create_srq_resp srq_resp = {0};
 	struct pvrdma_create_srq ucmd;
 	unsigned long flags;
 	int ret;
@@ -204,12 +205,13 @@ struct ib_srq *pvrdma_create_srq(struct ib_pd *pd,
 	}
 
 	srq->srq_handle = resp->srqn;
+	srq_resp.srqn = resp->srqn;
 	spin_lock_irqsave(&dev->srq_tbl_lock, flags);
 	dev->srq_tbl[srq->srq_handle % dev->dsr->caps.max_srq] = srq;
 	spin_unlock_irqrestore(&dev->srq_tbl_lock, flags);
 
 	/* Copy udata back. */
-	if (ib_copy_to_udata(udata, &srq->srq_handle, sizeof(__u32))) {
+	if (ib_copy_to_udata(udata, &srq_resp, sizeof(srq_resp))) {
 		dev_warn(&dev->pdev->dev, "failed to copy back udata\n");
 		pvrdma_destroy_srq(&srq->ibsrq);
 		return ERR_PTR(-EINVAL);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index 16b96616ef7e..a51463cd2f37 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -447,6 +447,7 @@ struct ib_pd *pvrdma_alloc_pd(struct ib_device *ibdev,
 	union pvrdma_cmd_resp rsp;
 	struct pvrdma_cmd_create_pd *cmd = &req.create_pd;
 	struct pvrdma_cmd_create_pd_resp *resp = &rsp.create_pd_resp;
+	struct pvrdma_alloc_pd_resp pd_resp = {0};
 	int ret;
 	void *ptr;
 
@@ -475,9 +476,10 @@ struct ib_pd *pvrdma_alloc_pd(struct ib_device *ibdev,
 	pd->privileged = !context;
 	pd->pd_handle = resp->pd_handle;
 	pd->pdn = resp->pd_handle;
+	pd_resp.pdn = resp->pd_handle;
 
 	if (context) {
-		if (ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
+		if (ib_copy_to_udata(udata, &pd_resp, sizeof(pd_resp))) {
 			dev_warn(&dev->pdev->dev,
 				 "failed to copy back protection domain\n");
 			pvrdma_dealloc_pd(&pd->ibpd);
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index 97d71e49c092..fb52b669bfce 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -56,7 +56,7 @@
  * rvt_cq_enter - add a new entry to the completion queue
  * @cq: completion queue
  * @entry: work completion entry to add
- * @sig: true if @entry is solicited
+ * @solicited: true if @entry is solicited
  *
  * This may be called with qp->s_lock held.
  */
@@ -101,8 +101,7 @@ void rvt_cq_enter(struct rvt_cq *cq, struct ib_wc *entry, bool solicited)
 		wc->uqueue[head].opcode = entry->opcode;
 		wc->uqueue[head].vendor_err = entry->vendor_err;
 		wc->uqueue[head].byte_len = entry->byte_len;
-		wc->uqueue[head].ex.imm_data =
-			(__u32 __force)entry->ex.imm_data;
+		wc->uqueue[head].ex.imm_data = entry->ex.imm_data;
 		wc->uqueue[head].qp_num = entry->qp->qp_num;
 		wc->uqueue[head].src_qp = entry->src_qp;
 		wc->uqueue[head].wc_flags = entry->wc_flags;
@@ -198,7 +197,7 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 		return ERR_PTR(-EINVAL);
 
 	/* Allocate the completion queue structure. */
-	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
+	cq = kzalloc_node(sizeof(*cq), GFP_KERNEL, rdi->dparms.node);
 	if (!cq)
 		return ERR_PTR(-ENOMEM);
 
@@ -214,7 +213,9 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 		sz += sizeof(struct ib_uverbs_wc) * (entries + 1);
 	else
 		sz += sizeof(struct ib_wc) * (entries + 1);
-	wc = vmalloc_user(sz);
+	wc = udata ?
+		vmalloc_user(sz) :
+		vzalloc_node(sz, rdi->dparms.node);
 	if (!wc) {
 		ret = ERR_PTR(-ENOMEM);
 		goto bail_cq;
@@ -369,7 +370,9 @@ int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
 		sz += sizeof(struct ib_uverbs_wc) * (cqe + 1);
 	else
 		sz += sizeof(struct ib_wc) * (cqe + 1);
-	wc = vmalloc_user(sz);
+	wc = udata ?
+		vmalloc_user(sz) :
+		vzalloc_node(sz, rdi->dparms.node);
 	if (!wc)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/sw/rdmavt/mcast.c b/drivers/infiniband/sw/rdmavt/mcast.c
index b3a38c5e4cad..dd11c6fcd060 100644
--- a/drivers/infiniband/sw/rdmavt/mcast.c
+++ b/drivers/infiniband/sw/rdmavt/mcast.c
@@ -272,7 +272,7 @@ static int rvt_mcast_add(struct rvt_dev_info *rdi, struct rvt_ibport *ibp,
 /**
  * rvt_attach_mcast - attach a qp to a multicast group
  * @ibqp: Infiniband qp
- * @igd: multicast guid
+ * @gid: multicast guid
  * @lid: multicast lid
  *
  * Return: 0 on success
@@ -335,7 +335,7 @@ int rvt_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 /**
  * rvt_detach_mcast - remove a qp from a multicast group
  * @ibqp: Infiniband qp
- * @igd: multicast guid
+ * @gid: multicast guid
  * @lid: multicast lid
  *
  * Return: 0 on success
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 42713511b53b..cc429b567d0a 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -489,11 +489,13 @@ static int rvt_check_refs(struct rvt_mregion *mr, const char *t)
 	unsigned long timeout;
 	struct rvt_dev_info *rdi = ib_to_rvt(mr->pd->device);
 
-	if (percpu_ref_is_zero(&mr->refcount))
-		return 0;
-	/* avoid dma mr */
-	if (mr->lkey)
+	if (mr->lkey) {
+		/* avoid dma mr */
 		rvt_dereg_clean_qps(mr);
+		/* @mr was indexed on rcu protected @lkey_table */
+		synchronize_rcu();
+	}
+
 	timeout = wait_for_completion_timeout(&mr->comp, 5 * HZ);
 	if (!timeout) {
 		rvt_pr_err(rdi,
@@ -768,7 +770,7 @@ struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
 
 /**
  * rvt_map_phys_fmr - set up a fast memory region
- * @ibmfr: the fast memory region to set up
+ * @ibfmr: the fast memory region to set up
  * @page_list: the list of pages to associate with the fast memory region
  * @list_len: the number of pages to associate with the fast memory region
  * @iova: the virtual address of the start of the fast memory region
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 9177df60742a..c82e6bb3d77c 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -269,7 +269,7 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi)
 
 /**
  * free_all_qps - check for QPs still in use
- * @qpt: the QP table to empty
+ * @rdi: rvt device info structure
  *
  * There should not be any QPs still in use.
  * Free memory for table.
@@ -335,9 +335,9 @@ static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
 /**
  * alloc_qpn - Allocate the next available qpn or zero/one for QP type
  *	       IB_QPT_SMI/IB_QPT_GSI
- *@rdi:	rvt device info structure
- *@qpt: queue pair number table pointer
- *@port_num: IB port number, 1 based, comes from core
+ * @rdi: rvt device info structure
+ * @qpt: queue pair number table pointer
+ * @port_num: IB port number, 1 based, comes from core
  *
  * Return: The queue pair number
  */
@@ -1650,9 +1650,9 @@ static inline int rvt_qp_valid_operation(
 
 /**
  * rvt_qp_is_avail - determine queue capacity
- * @qp - the qp
- * @rdi - the rdmavt device
- * @reserved_op - is reserved operation
+ * @qp: the qp
+ * @rdi: the rdmavt device
+ * @reserved_op: is reserved operation
  *
  * This assumes the s_hlock is held but the s_last
  * qp variable is uncontrolled.
@@ -1684,7 +1684,6 @@ static inline int rvt_qp_is_avail(
 	/* non-reserved operations */
 	if (likely(qp->s_avail))
 		return 0;
-	smp_read_barrier_depends(); /* see rc.c */
 	slast = READ_ONCE(qp->s_last);
 	if (qp->s_head >= slast)
 		avail = qp->s_size - (qp->s_head - slast);
@@ -2075,6 +2074,7 @@ void rvt_add_rnr_timer(struct rvt_qp *qp, u32 aeth)
 	lockdep_assert_held(&qp->s_lock);
 	qp->s_flags |= RVT_S_WAIT_RNR;
 	to = rvt_aeth_to_usec(aeth);
+	trace_rvt_rnrnak_add(qp, to);
 	hrtimer_start(&qp->s_rnr_timer,
 		      ns_to_ktime(1000 * to), HRTIMER_MODE_REL);
 }
@@ -2104,17 +2104,14 @@ EXPORT_SYMBOL(rvt_stop_rc_timers);
  * stop an rnr timer and return if the timer
  * had been pending.
  */
-static int rvt_stop_rnr_timer(struct rvt_qp *qp)
+static void rvt_stop_rnr_timer(struct rvt_qp *qp)
 {
-	int rval = 0;
-
 	lockdep_assert_held(&qp->s_lock);
 	/* Remove QP from rnr timer */
 	if (qp->s_flags & RVT_S_WAIT_RNR) {
 		qp->s_flags &= ~RVT_S_WAIT_RNR;
-		rval = hrtimer_try_to_cancel(&qp->s_rnr_timer);
+		trace_rvt_rnrnak_stop(qp, 0);
 	}
-	return rval;
 }
 
 /**
@@ -2167,6 +2164,7 @@ enum hrtimer_restart rvt_rc_rnr_retry(struct hrtimer *t)
 
 	spin_lock_irqsave(&qp->s_lock, flags);
 	rvt_stop_rnr_timer(qp);
+	trace_rvt_rnrnak_timeout(qp, 0);
 	rdi->driver_f.schedule_send(qp);
 	spin_unlock_irqrestore(&qp->s_lock, flags);
 	return HRTIMER_NORESTART;
@@ -2175,8 +2173,8 @@ EXPORT_SYMBOL(rvt_rc_rnr_retry);
 
 /**
  * rvt_qp_iter_init - initial for QP iteration
- * @rdi - rvt devinfo
- * @v - u64 value
+ * @rdi: rvt devinfo
+ * @v: u64 value
  *
  * This returns an iterator suitable for iterating QPs
  * in the system.
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index f7c48e9023de..3707952b4364 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -90,7 +90,7 @@ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 	    srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr)
 		return ERR_PTR(-EINVAL);
 
-	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
+	srq = kzalloc_node(sizeof(*srq), GFP_KERNEL, dev->dparms.node);
 	if (!srq)
 		return ERR_PTR(-ENOMEM);
 
@@ -101,7 +101,10 @@ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 	srq->rq.max_sge = srq_init_attr->attr.max_sge;
 	sz = sizeof(struct ib_sge) * srq->rq.max_sge +
 		sizeof(struct rvt_rwqe);
-	srq->rq.wq = vmalloc_user(sizeof(struct rvt_rwq) + srq->rq.size * sz);
+	srq->rq.wq = udata ?
+		vmalloc_user(sizeof(struct rvt_rwq) + srq->rq.size * sz) :
+		vzalloc_node(sizeof(struct rvt_rwq) + srq->rq.size * sz,
+			     dev->dparms.node);
 	if (!srq->rq.wq) {
 		ret = ERR_PTR(-ENOMEM);
 		goto bail_srq;
@@ -129,16 +132,12 @@ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 			ret = ERR_PTR(err);
 			goto bail_ip;
 		}
-	} else {
-		srq->ip = NULL;
 	}
 
 	/*
 	 * ib_create_srq() will initialize srq->ibsrq.
 	 */
 	spin_lock_init(&srq->rq.lock);
-	srq->rq.wq->head = 0;
-	srq->rq.wq->tail = 0;
 	srq->limit = srq_init_attr->attr.srq_limit;
 
 	spin_lock(&dev->n_srqs_lock);
@@ -200,7 +199,10 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 		sz = sizeof(struct rvt_rwqe) +
 			srq->rq.max_sge * sizeof(struct ib_sge);
 		size = attr->max_wr + 1;
-		wq = vmalloc_user(sizeof(struct rvt_rwq) + size * sz);
+		wq = udata ?
+			vmalloc_user(sizeof(struct rvt_rwq) + size * sz) :
+			vzalloc_node(sizeof(struct rvt_rwq) + size * sz,
+				     dev->dparms.node);
 		if (!wq)
 			return -ENOMEM;
 
diff --git a/drivers/infiniband/sw/rdmavt/trace.h b/drivers/infiniband/sw/rdmavt/trace.h
index bb4b1e710f22..36ddbd291ee0 100644
--- a/drivers/infiniband/sw/rdmavt/trace.h
+++ b/drivers/infiniband/sw/rdmavt/trace.h
@@ -45,8 +45,8 @@
  *
  */
 
-#define RDI_DEV_ENTRY(rdi)   __string(dev, rdi->driver_f.get_card_name(rdi))
-#define RDI_DEV_ASSIGN(rdi)  __assign_str(dev, rdi->driver_f.get_card_name(rdi))
+#define RDI_DEV_ENTRY(rdi)   __string(dev, rvt_get_ibdev_name(rdi))
+#define RDI_DEV_ASSIGN(rdi)  __assign_str(dev, rvt_get_ibdev_name(rdi))
 
 #include "trace_rvt.h"
 #include "trace_qp.h"
diff --git a/drivers/infiniband/sw/rdmavt/trace_qp.h b/drivers/infiniband/sw/rdmavt/trace_qp.h
index 4c77a3119bda..efc9d814b032 100644
--- a/drivers/infiniband/sw/rdmavt/trace_qp.h
+++ b/drivers/infiniband/sw/rdmavt/trace_qp.h
@@ -85,6 +85,48 @@ DEFINE_EVENT(rvt_qphash_template, rvt_qpremove,
 	TP_PROTO(struct rvt_qp *qp, u32 bucket),
 	TP_ARGS(qp, bucket));
 
+DECLARE_EVENT_CLASS(
+	rvt_rnrnak_template,
+	TP_PROTO(struct rvt_qp *qp, u32 to),
+	TP_ARGS(qp, to),
+	TP_STRUCT__entry(
+		RDI_DEV_ENTRY(ib_to_rvt(qp->ibqp.device))
+		__field(u32, qpn)
+		__field(void *, hrtimer)
+		__field(u32, s_flags)
+		__field(u32, to)
+	),
+	TP_fast_assign(
+		RDI_DEV_ASSIGN(ib_to_rvt(qp->ibqp.device))
+		__entry->qpn = qp->ibqp.qp_num;
+		__entry->hrtimer = &qp->s_rnr_timer;
+		__entry->s_flags = qp->s_flags;
+		__entry->to = to;
+	),
+	TP_printk(
+		"[%s] qpn 0x%x hrtimer 0x%p s_flags 0x%x timeout %u us",
+		__get_str(dev),
+		__entry->qpn,
+		__entry->hrtimer,
+		__entry->s_flags,
+		__entry->to
+	)
+);
+
+DEFINE_EVENT(
+	rvt_rnrnak_template, rvt_rnrnak_add,
+	TP_PROTO(struct rvt_qp *qp, u32 to),
+	TP_ARGS(qp, to));
+
+DEFINE_EVENT(
+	rvt_rnrnak_template, rvt_rnrnak_timeout,
+	TP_PROTO(struct rvt_qp *qp, u32 to),
+	TP_ARGS(qp, to));
+
+DEFINE_EVENT(
+	rvt_rnrnak_template, rvt_rnrnak_stop,
+	TP_PROTO(struct rvt_qp *qp, u32 to),
+	TP_ARGS(qp, to));
 
 #endif /* __RVT_TRACE_QP_H */
 
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 64bdd442078a..a4553b2b3696 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -224,7 +224,8 @@ static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
  * rvt_query_pkey - Return a pkey from the table at a given index
  * @ibdev: Verbs IB dev
  * @port_num: Port number, 1 based from ib core
- * @intex: Index into pkey table
+ * @index: Index into pkey table
+ * @pkey: returned pkey from the port pkey table
  *
  * Return: 0 on failure pkey otherwise
  */
@@ -255,7 +256,7 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,
  * rvt_query_gid - Return a gid from the table
  * @ibdev: Verbs IB dev
  * @port_num: Port number, 1 based from ib core
- * @index: = Index in table
+ * @guid_index: Index in table
  * @gid: Gid to return
  *
  * Return: 0 on success
@@ -297,8 +298,8 @@ static inline struct rvt_ucontext *to_iucontext(struct ib_ucontext
 
 /**
  * rvt_alloc_ucontext - Allocate a user context
- * @ibdev: Vers IB dev
- * @data: User data allocated
+ * @ibdev: Verbs IB dev
+ * @udata: User data allocated
  */
 static struct ib_ucontext *rvt_alloc_ucontext(struct ib_device *ibdev,
 					      struct ib_udata *udata)
@@ -413,7 +414,6 @@ static noinline int check_support(struct rvt_dev_info *rdi, int verb)
 		 * required for rdmavt to function.
 		 */
 		if ((!rdi->driver_f.port_callback) ||
-		    (!rdi->driver_f.get_card_name) ||
 		    (!rdi->driver_f.get_pci_dev))
 			return -EINVAL;
 		break;
diff --git a/drivers/infiniband/sw/rdmavt/vt.h b/drivers/infiniband/sw/rdmavt/vt.h
index f363505312be..8823b2e7aac6 100644
--- a/drivers/infiniband/sw/rdmavt/vt.h
+++ b/drivers/infiniband/sw/rdmavt/vt.h
@@ -63,19 +63,19 @@
 
 #define rvt_pr_info(rdi, fmt, ...) \
 	__rvt_pr_info(rdi->driver_f.get_pci_dev(rdi), \
-		      rdi->driver_f.get_card_name(rdi), \
+		      rvt_get_ibdev_name(rdi), \
 		      fmt, \
 		      ##__VA_ARGS__)
 
 #define rvt_pr_warn(rdi, fmt, ...) \
 	__rvt_pr_warn(rdi->driver_f.get_pci_dev(rdi), \
-		      rdi->driver_f.get_card_name(rdi), \
+		      rvt_get_ibdev_name(rdi), \
 		      fmt, \
 		      ##__VA_ARGS__)
 
 #define rvt_pr_err(rdi, fmt, ...) \
 	__rvt_pr_err(rdi->driver_f.get_pci_dev(rdi), \
-		     rdi->driver_f.get_card_name(rdi), \
+		     rvt_get_ibdev_name(rdi), \
 		     fmt, \
 		     ##__VA_ARGS__)
 
diff --git a/drivers/infiniband/sw/rxe/Kconfig b/drivers/infiniband/sw/rxe/Kconfig
index 320bffc980d8..bad4a576d7cf 100644
--- a/drivers/infiniband/sw/rxe/Kconfig
+++ b/drivers/infiniband/sw/rxe/Kconfig
@@ -1,8 +1,8 @@
 config RDMA_RXE
 	tristate "Software RDMA over Ethernet (RoCE) driver"
 	depends on INET && PCI && INFINIBAND
-	depends on NET_UDP_TUNNEL
-	depends on CRYPTO_CRC32
+	select NET_UDP_TUNNEL
+	select CRYPTO_CRC32
 	select DMA_VIRT_OPS
 	---help---
 	This driver implements the InfiniBand RDMA transport over
diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 8c3d30b3092d..b7debb6f2eac 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -77,12 +77,6 @@ void rxe_release(struct kref *kref)
 	ib_dealloc_device(&rxe->ib_dev);
 }
 
-void rxe_dev_put(struct rxe_dev *rxe)
-{
-	kref_put(&rxe->ref_cnt, rxe_release);
-}
-EXPORT_SYMBOL_GPL(rxe_dev_put);
-
 /* initialize rxe device parameters */
 static int rxe_init_device_param(struct rxe_dev *rxe)
 {
diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
index 6447d736d5a4..7d232611303f 100644
--- a/drivers/infiniband/sw/rxe/rxe.h
+++ b/drivers/infiniband/sw/rxe/rxe.h
@@ -57,6 +57,7 @@
 #include "rxe_hdr.h"
 #include "rxe_param.h"
 #include "rxe_verbs.h"
+#include "rxe_loc.h"
 
 #define RXE_UVERBS_ABI_VERSION		(1)
 
@@ -95,7 +96,10 @@ void rxe_remove_all(void);
 
 int rxe_rcv(struct sk_buff *skb);
 
-void rxe_dev_put(struct rxe_dev *rxe);
+static inline void rxe_dev_put(struct rxe_dev *rxe)
+{
+	kref_put(&rxe->ref_cnt, rxe_release);
+}
 struct rxe_dev *net_to_rxe(struct net_device *ndev);
 struct rxe_dev *get_rxe_by_name(const char *name);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c
index 1cc9e2e1365d..7522d1af3ae2 100644
--- a/drivers/infiniband/sw/rxe/rxe_av.c
+++ b/drivers/infiniband/sw/rxe/rxe_av.c
@@ -52,28 +52,24 @@ int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr)
 	return 0;
 }
 
-int rxe_av_from_attr(struct rxe_dev *rxe, u8 port_num,
-		     struct rxe_av *av, struct rdma_ah_attr *attr)
+void rxe_av_from_attr(u8 port_num, struct rxe_av *av,
+		     struct rdma_ah_attr *attr)
 {
 	memset(av, 0, sizeof(*av));
 	memcpy(&av->grh, rdma_ah_read_grh(attr),
 	       sizeof(*rdma_ah_read_grh(attr)));
 	av->port_num = port_num;
-	return 0;
 }
 
-int rxe_av_to_attr(struct rxe_dev *rxe, struct rxe_av *av,
-		   struct rdma_ah_attr *attr)
+void rxe_av_to_attr(struct rxe_av *av, struct rdma_ah_attr *attr)
 {
 	attr->type = RDMA_AH_ATTR_TYPE_ROCE;
 	memcpy(rdma_ah_retrieve_grh(attr), &av->grh, sizeof(av->grh));
 	rdma_ah_set_ah_flags(attr, IB_AH_GRH);
 	rdma_ah_set_port_num(attr, av->port_num);
-	return 0;
 }
 
-int rxe_av_fill_ip_info(struct rxe_dev *rxe,
-			struct rxe_av *av,
+void rxe_av_fill_ip_info(struct rxe_av *av,
 			struct rdma_ah_attr *attr,
 			struct ib_gid_attr *sgid_attr,
 			union ib_gid *sgid)
@@ -81,8 +77,6 @@ int rxe_av_fill_ip_info(struct rxe_dev *rxe,
 	rdma_gid2ip(&av->sgid_addr._sockaddr, sgid);
 	rdma_gid2ip(&av->dgid_addr._sockaddr, &rdma_ah_read_grh(attr)->dgid);
 	av->network_type = ib_gid_to_network_type(sgid_attr->gid_type, sgid);
-
-	return 0;
 }
 
 struct rxe_av *rxe_get_av(struct rxe_pkt_info *pkt)
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index d7472a442a2c..4ef75d5b729b 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -38,14 +38,12 @@
 
 int rxe_av_chk_attr(struct rxe_dev *rxe, struct rdma_ah_attr *attr);
 
-int rxe_av_from_attr(struct rxe_dev *rxe, u8 port_num,
-		     struct rxe_av *av, struct rdma_ah_attr *attr);
+void rxe_av_from_attr(u8 port_num, struct rxe_av *av,
+		     struct rdma_ah_attr *attr);
 
-int rxe_av_to_attr(struct rxe_dev *rxe, struct rxe_av *av,
-		   struct rdma_ah_attr *attr);
+void rxe_av_to_attr(struct rxe_av *av, struct rdma_ah_attr *attr);
 
-int rxe_av_fill_ip_info(struct rxe_dev *rxe,
-			struct rxe_av *av,
+void rxe_av_fill_ip_info(struct rxe_av *av,
 			struct rdma_ah_attr *attr,
 			struct ib_gid_attr *sgid_attr,
 			union ib_gid *sgid);
@@ -237,7 +235,6 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 
 void rxe_release(struct kref *kref);
 
-void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify);
 int rxe_completer(void *arg);
 int rxe_requester(void *arg);
 int rxe_responder(void *arg);
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 59dee10bebcb..159246b03867 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -82,7 +82,7 @@ struct rxe_dev *get_rxe_by_name(const char *name)
 }
 
 
-struct rxe_recv_sockets recv_sockets;
+static struct rxe_recv_sockets recv_sockets;
 
 struct device *rxe_dma_device(struct rxe_dev *rxe)
 {
@@ -452,31 +452,26 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb)
 
 int rxe_send(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, struct sk_buff *skb)
 {
-	struct sk_buff *nskb;
 	struct rxe_av *av;
 	int err;
 
 	av = rxe_get_av(pkt);
 
-	nskb = skb_clone(skb, GFP_ATOMIC);
-	if (!nskb)
-		return -ENOMEM;
-
-	nskb->destructor = rxe_skb_tx_dtor;
-	nskb->sk = pkt->qp->sk->sk;
+	skb->destructor = rxe_skb_tx_dtor;
+	skb->sk = pkt->qp->sk->sk;
 
 	rxe_add_ref(pkt->qp);
 	atomic_inc(&pkt->qp->skb_out);
 
 	if (av->network_type == RDMA_NETWORK_IPV4) {
-		err = ip_local_out(dev_net(skb_dst(skb)->dev), nskb->sk, nskb);
+		err = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
 	} else if (av->network_type == RDMA_NETWORK_IPV6) {
-		err = ip6_local_out(dev_net(skb_dst(skb)->dev), nskb->sk, nskb);
+		err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
 	} else {
 		pr_err("Unknown layer 3 protocol: %d\n", av->network_type);
 		atomic_dec(&pkt->qp->skb_out);
 		rxe_drop_ref(pkt->qp);
-		kfree_skb(nskb);
+		kfree_skb(skb);
 		return -EINVAL;
 	}
 
@@ -485,7 +480,6 @@ int rxe_send(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, struct sk_buff *skb)
 		return -EAGAIN;
 	}
 
-	kfree_skb(skb);
 	return 0;
 }
 
diff --git a/drivers/infiniband/sw/rxe/rxe_net.h b/drivers/infiniband/sw/rxe/rxe_net.h
index 1c06b3bfe1b6..728d8c71b36a 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.h
+++ b/drivers/infiniband/sw/rxe/rxe_net.h
@@ -43,7 +43,6 @@ struct rxe_recv_sockets {
 	struct socket *sk6;
 };
 
-extern struct rxe_recv_sockets recv_sockets;
 extern struct notifier_block rxe_net_notifier;
 void rxe_release_udp_tunnel(struct socket *sk);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index 4469592b839d..2fcf1cab7678 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -633,9 +633,8 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
 		ib_get_cached_gid(&rxe->ib_dev, 1,
 				  rdma_ah_read_grh(&attr->ah_attr)->sgid_index,
 				  &sgid, &sgid_attr);
-		rxe_av_from_attr(rxe, attr->port_num, &qp->pri_av,
-				 &attr->ah_attr);
-		rxe_av_fill_ip_info(rxe, &qp->pri_av, &attr->ah_attr,
+		rxe_av_from_attr(attr->port_num, &qp->pri_av, &attr->ah_attr);
+		rxe_av_fill_ip_info(&qp->pri_av, &attr->ah_attr,
 				    &sgid_attr, &sgid);
 		if (sgid_attr.ndev)
 			dev_put(sgid_attr.ndev);
@@ -648,9 +647,9 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
 		ib_get_cached_gid(&rxe->ib_dev, 1, sgid_index,
 				  &sgid, &sgid_attr);
 
-		rxe_av_from_attr(rxe, attr->alt_port_num, &qp->alt_av,
+		rxe_av_from_attr(attr->alt_port_num, &qp->alt_av,
 				 &attr->alt_ah_attr);
-		rxe_av_fill_ip_info(rxe, &qp->alt_av, &attr->alt_ah_attr,
+		rxe_av_fill_ip_info(&qp->alt_av, &attr->alt_ah_attr,
 				    &sgid_attr, &sgid);
 		if (sgid_attr.ndev)
 			dev_put(sgid_attr.ndev);
@@ -765,8 +764,6 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask,
 /* called by the query qp verb */
 int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask)
 {
-	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
-
 	*attr = qp->attr;
 
 	attr->rq_psn				= qp->resp.psn;
@@ -781,8 +778,8 @@ int rxe_qp_to_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask)
 		attr->cap.max_recv_sge		= qp->rq.max_sge;
 	}
 
-	rxe_av_to_attr(rxe, &qp->pri_av, &attr->ah_attr);
-	rxe_av_to_attr(rxe, &qp->alt_av, &attr->alt_ah_attr);
+	rxe_av_to_attr(&qp->pri_av, &attr->ah_attr);
+	rxe_av_to_attr(&qp->alt_av, &attr->alt_ah_attr);
 
 	if (qp->req.state == QP_STATE_DRAIN) {
 		attr->sq_draining = 1;
@@ -824,9 +821,9 @@ void rxe_qp_destroy(struct rxe_qp *qp)
 }
 
 /* called when the last reference to the qp is dropped */
-void rxe_qp_cleanup(struct rxe_pool_entry *arg)
+static void rxe_qp_do_cleanup(struct work_struct *work)
 {
-	struct rxe_qp *qp = container_of(arg, typeof(*qp), pelem);
+	struct rxe_qp *qp = container_of(work, typeof(*qp), cleanup_work.work);
 
 	rxe_drop_all_mcast_groups(qp);
 
@@ -859,3 +856,11 @@ void rxe_qp_cleanup(struct rxe_pool_entry *arg)
 	kernel_sock_shutdown(qp->sk, SHUT_RDWR);
 	sock_release(qp->sk);
 }
+
+/* called when the last reference to the qp is dropped */
+void rxe_qp_cleanup(struct rxe_pool_entry *arg)
+{
+	struct rxe_qp *qp = container_of(arg, typeof(*qp), pelem);
+
+	execute_in_process_context(rxe_qp_do_cleanup, &qp->cleanup_work);
+}
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index fb8c83e055e1..4c3f899241d4 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -336,7 +336,6 @@ static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb)
 {
 	union ib_gid dgid;
 	union ib_gid *pdgid;
-	u16 index;
 
 	if (skb->protocol == htons(ETH_P_IP)) {
 		ipv6_addr_set_v4mapped(ip_hdr(skb)->daddr,
@@ -348,7 +347,7 @@ static int rxe_match_dgid(struct rxe_dev *rxe, struct sk_buff *skb)
 
 	return ib_find_cached_gid_by_port(&rxe->ib_dev, pdgid,
 					  IB_GID_TYPE_ROCE_UDP_ENCAP,
-					  1, rxe->ndev, &index);
+					  1, rxe->ndev, NULL);
 }
 
 /* rxe_rcv is called from the interface driver */
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 26a7f923045b..7bdaf71b8221 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -594,15 +594,8 @@ int rxe_requester(void *arg)
 	rxe_add_ref(qp);
 
 next_wqe:
-	if (unlikely(!qp->valid)) {
-		rxe_drain_req_pkts(qp, true);
+	if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR))
 		goto exit;
-	}
-
-	if (unlikely(qp->req.state == QP_STATE_ERROR)) {
-		rxe_drain_req_pkts(qp, true);
-		goto exit;
-	}
 
 	if (unlikely(qp->req.state == QP_STATE_RESET)) {
 		qp->req.wqe_index = consumer_index(qp->sq.queue);
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index 4240866a5331..d37bb9b97569 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -863,8 +863,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
 
 			if (pkt->mask & RXE_IMMDT_MASK) {
 				uwc->wc_flags |= IB_WC_WITH_IMM;
-				uwc->ex.imm_data =
-					(__u32 __force)immdt_imm(pkt);
+				uwc->ex.imm_data = immdt_imm(pkt);
 			}
 
 			if (pkt->mask & RXE_IETH_MASK) {
@@ -1210,7 +1209,7 @@ static enum resp_states do_class_d1e_error(struct rxe_qp *qp)
 	}
 }
 
-void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)
+static void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)
 {
 	struct sk_buff *skb;
 
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index d03002b9d84d..f4bab2cd0ec2 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -271,13 +271,12 @@ static int rxe_init_av(struct rxe_dev *rxe, struct rdma_ah_attr *attr,
 		return err;
 	}
 
-	err = rxe_av_from_attr(rxe, rdma_ah_get_port_num(attr), av, attr);
-	if (!err)
-		err = rxe_av_fill_ip_info(rxe, av, attr, &sgid_attr, &sgid);
+	rxe_av_from_attr(rdma_ah_get_port_num(attr), av, attr);
+	rxe_av_fill_ip_info(av, attr, &sgid_attr, &sgid);
 
 	if (sgid_attr.ndev)
 		dev_put(sgid_attr.ndev);
-	return err;
+	return 0;
 }
 
 static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd,
@@ -335,12 +334,11 @@ static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
 
 static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
 {
-	struct rxe_dev *rxe = to_rdev(ibah->device);
 	struct rxe_ah *ah = to_rah(ibah);
 
 	memset(attr, 0, sizeof(*attr));
 	attr->type = ibah->type;
-	rxe_av_to_attr(rxe, &ah->av, attr);
+	rxe_av_to_attr(&ah->av, attr);
 	return 0;
 }
 
@@ -814,6 +812,8 @@ static int rxe_post_send_kernel(struct rxe_qp *qp, struct ib_send_wr *wr,
 			(queue_count(qp->sq.queue) > 1);
 
 	rxe_run_task(&qp->req.task, must_sched);
+	if (unlikely(qp->req.state == QP_STATE_ERROR))
+		rxe_run_task(&qp->comp.task, 1);
 
 	return err;
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 0c2dbe45c729..1019f5e7dbdd 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -35,6 +35,7 @@
 #define RXE_VERBS_H
 
 #include <linux/interrupt.h>
+#include <linux/workqueue.h>
 #include <rdma/rdma_user_rxe.h>
 #include "rxe_pool.h"
 #include "rxe_task.h"
@@ -281,6 +282,8 @@ struct rxe_qp {
 	struct timer_list rnr_nak_timer;
 
 	spinlock_t		state_lock; /* guard requester and completer */
+
+	struct execute_work	cleanup_work;
 };
 
 enum rxe_mem_state {
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 71ea9e26666c..962fbcb57dc7 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -766,12 +766,14 @@ void ipoib_cm_send(struct net_device *dev, struct sk_buff *skb, struct ipoib_cm_
 	skb_orphan(skb);
 	skb_dst_drop(skb);
 
-	if (netif_queue_stopped(dev))
-		if (ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP |
-				     IB_CQ_REPORT_MISSED_EVENTS)) {
+	if (netif_queue_stopped(dev)) {
+		rc = ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP |
+				      IB_CQ_REPORT_MISSED_EVENTS);
+		if (unlikely(rc < 0))
 			ipoib_warn(priv, "IPoIB/CM:request notify on send CQ failed\n");
+		else if (rc)
 			napi_schedule(&priv->send_napi);
-		}
+	}
 
 	rc = post_send(priv, tx, tx->tx_head & (ipoib_sendq_size - 1), tx_req);
 	if (unlikely(rc)) {
@@ -876,7 +878,7 @@ int ipoib_cm_dev_open(struct net_device *dev)
 
 	priv->cm.id = ib_create_cm_id(priv->ca, ipoib_cm_rx_handler, dev);
 	if (IS_ERR(priv->cm.id)) {
-		printk(KERN_WARNING "%s: failed to create CM ID\n", priv->ca->name);
+		pr_warn("%s: failed to create CM ID\n", priv->ca->name);
 		ret = PTR_ERR(priv->cm.id);
 		goto err_cm;
 	}
@@ -884,8 +886,8 @@ int ipoib_cm_dev_open(struct net_device *dev)
 	ret = ib_cm_listen(priv->cm.id, cpu_to_be64(IPOIB_CM_IETF_ID | priv->qp->qp_num),
 			   0);
 	if (ret) {
-		printk(KERN_WARNING "%s: failed to listen on ID 0x%llx\n", priv->ca->name,
-		       IPOIB_CM_IETF_ID | priv->qp->qp_num);
+		pr_warn("%s: failed to listen on ID 0x%llx\n", priv->ca->name,
+			IPOIB_CM_IETF_ID | priv->qp->qp_num);
 		goto err_listen;
 	}
 
@@ -1562,7 +1564,7 @@ static void ipoib_cm_create_srq(struct net_device *dev, int max_sge)
 	priv->cm.srq = ib_create_srq(priv->pd, &srq_init_attr);
 	if (IS_ERR(priv->cm.srq)) {
 		if (PTR_ERR(priv->cm.srq) != -ENOSYS)
-			printk(KERN_WARNING "%s: failed to allocate SRQ, error %ld\n",
+			pr_warn("%s: failed to allocate SRQ, error %ld\n",
 			       priv->ca->name, PTR_ERR(priv->cm.srq));
 		priv->cm.srq = NULL;
 		return;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_fs.c b/drivers/infiniband/ulp/ipoib/ipoib_fs.c
index 11f74cbe6660..ea302b054601 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_fs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_fs.c
@@ -281,8 +281,6 @@ void ipoib_delete_debug_files(struct net_device *dev)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 
-	WARN_ONCE(!priv->mcg_dentry, "null mcg debug file\n");
-	WARN_ONCE(!priv->path_dentry, "null path debug file\n");
 	debugfs_remove(priv->mcg_dentry);
 	debugfs_remove(priv->path_dentry);
 	priv->mcg_dentry = priv->path_dentry = NULL;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index e6151a29c412..10384ea50bed 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -644,7 +644,7 @@ int ipoib_send(struct net_device *dev, struct sk_buff *skb,
 
 	if (netif_queue_stopped(dev))
 		if (ib_req_notify_cq(priv->send_cq, IB_CQ_NEXT_COMP |
-				     IB_CQ_REPORT_MISSED_EVENTS))
+				     IB_CQ_REPORT_MISSED_EVENTS) < 0)
 			ipoib_warn(priv, "request notify on send CQ failed\n");
 
 	rc = post_send(priv, priv->tx_head & (ipoib_sendq_size - 1),
@@ -1085,8 +1085,7 @@ static bool ipoib_dev_addr_changed_valid(struct ipoib_dev_priv *priv)
 
 	netif_addr_unlock_bh(priv->dev);
 
-	err = ib_find_gid(priv->ca, &search_gid, IB_GID_TYPE_IB,
-			  priv->dev, &port, &index);
+	err = ib_find_gid(priv->ca, &search_gid, priv->dev, &port, &index);
 
 	netif_addr_lock_bh(priv->dev);
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 8880351df179..161ba8c76285 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -768,13 +768,30 @@ static void path_rec_completion(int status,
 	if (!status) {
 		struct rdma_ah_attr av;
 
-		if (!ib_init_ah_from_path(priv->ca, priv->port, pathrec, &av))
+		if (!ib_init_ah_attr_from_path(priv->ca, priv->port,
+					       pathrec, &av))
 			ah = ipoib_create_ah(dev, priv->pd, &av);
 	}
 
 	spin_lock_irqsave(&priv->lock, flags);
 
 	if (!IS_ERR_OR_NULL(ah)) {
+		/*
+		 * pathrec.dgid is used as the database key from the LLADDR,
+		 * it must remain unchanged even if the SA returns a different
+		 * GID to use in the AH.
+		 */
+		if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw,
+			   sizeof(union ib_gid))) {
+			ipoib_dbg(
+				priv,
+				"%s got PathRec for gid %pI6 while asked for %pI6\n",
+				dev->name, pathrec->dgid.raw,
+				path->pathrec.dgid.raw);
+			memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,
+			       sizeof(union ib_gid));
+		}
+
 		path->pathrec = *pathrec;
 
 		old_ah   = path->ah;
@@ -840,6 +857,23 @@ static void path_rec_completion(int status,
 	}
 }
 
+static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,
+			  void *gid)
+{
+	path->dev = priv->dev;
+
+	if (rdma_cap_opa_ah(priv->ca, priv->port))
+		path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
+	else
+		path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;
+
+	memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));
+	path->pathrec.sgid	    = priv->local_gid;
+	path->pathrec.pkey	    = cpu_to_be16(priv->pkey);
+	path->pathrec.numb_path     = 1;
+	path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
+}
+
 static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
@@ -852,21 +886,11 @@ static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
 	if (!path)
 		return NULL;
 
-	path->dev = dev;
-
 	skb_queue_head_init(&path->queue);
 
 	INIT_LIST_HEAD(&path->neigh_list);
 
-	if (rdma_cap_opa_ah(priv->ca, priv->port))
-		path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
-	else
-		path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;
-	memcpy(path->pathrec.dgid.raw, gid, sizeof (union ib_gid));
-	path->pathrec.sgid	    = priv->local_gid;
-	path->pathrec.pkey	    = cpu_to_be16(priv->pkey);
-	path->pathrec.numb_path     = 1;
-	path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
+	init_path_rec(priv, path, gid);
 
 	return path;
 }
@@ -1005,6 +1029,10 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
 
 	spin_lock_irqsave(&priv->lock, flags);
 
+	/* no broadcast means that all paths are (going to be) not valid */
+	if (!priv->broadcast)
+		goto drop_and_unlock;
+
 	path = __path_find(dev, phdr->hwaddr + 4);
 	if (!path || !path->valid) {
 		int new_path = 0;
@@ -1014,6 +1042,10 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
 			new_path = 1;
 		}
 		if (path) {
+			if (!new_path)
+				/* make sure there is no changes in the existing path record */
+				init_path_rec(priv, path, phdr->hwaddr + 4);
+
 			if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
 				push_pseudo_header(skb, phdr->hwaddr);
 				__skb_queue_tail(&path->queue, skb);
@@ -1030,8 +1062,7 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
 			} else
 				__path_add(dev, path);
 		} else {
-			++dev->stats.tx_dropped;
-			dev_kfree_skb_any(skb);
+			goto drop_and_unlock;
 		}
 
 		spin_unlock_irqrestore(&priv->lock, flags);
@@ -1051,11 +1082,16 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
 		push_pseudo_header(skb, phdr->hwaddr);
 		__skb_queue_tail(&path->queue, skb);
 	} else {
-		++dev->stats.tx_dropped;
-		dev_kfree_skb_any(skb);
+		goto drop_and_unlock;
 	}
 
 	spin_unlock_irqrestore(&priv->lock, flags);
+	return;
+
+drop_and_unlock:
+	++dev->stats.tx_dropped;
+	dev_kfree_skb_any(skb);
+	spin_unlock_irqrestore(&priv->lock, flags);
 }
 
 static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
@@ -1674,8 +1710,8 @@ static int ipoib_dev_init_default(struct net_device *dev)
 
 	priv->tx_ring = vzalloc(ipoib_sendq_size * sizeof *priv->tx_ring);
 	if (!priv->tx_ring) {
-		printk(KERN_WARNING "%s: failed to allocate TX ring (%d entries)\n",
-		       priv->ca->name, ipoib_sendq_size);
+		pr_warn("%s: failed to allocate TX ring (%d entries)\n",
+			priv->ca->name, ipoib_sendq_size);
 		goto out_rx_ring_cleanup;
 	}
 
@@ -2207,16 +2243,17 @@ static struct net_device *ipoib_add_port(const char *format,
 	int result = -ENOMEM;
 
 	priv = ipoib_intf_alloc(hca, port, format);
-	if (!priv)
+	if (!priv) {
+		pr_warn("%s, %d: ipoib_intf_alloc failed\n", hca->name, port);
 		goto alloc_mem_failed;
+	}
 
 	SET_NETDEV_DEV(priv->dev, hca->dev.parent);
 	priv->dev->dev_id = port - 1;
 
 	result = ib_query_port(hca, port, &attr);
 	if (result) {
-		printk(KERN_WARNING "%s: ib_query_port %d failed\n",
-		       hca->name, port);
+		pr_warn("%s: ib_query_port %d failed\n", hca->name, port);
 		goto device_init_failed;
 	}
 
@@ -2231,8 +2268,8 @@ static struct net_device *ipoib_add_port(const char *format,
 
 	result = ib_query_pkey(hca, port, 0, &priv->pkey);
 	if (result) {
-		printk(KERN_WARNING "%s: ib_query_pkey port %d failed (ret = %d)\n",
-		       hca->name, port, result);
+		pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",
+			hca->name, port, result);
 		goto device_init_failed;
 	}
 
@@ -2249,8 +2286,8 @@ static struct net_device *ipoib_add_port(const char *format,
 
 	result = ib_query_gid(hca, port, 0, &priv->local_gid, NULL);
 	if (result) {
-		printk(KERN_WARNING "%s: ib_query_gid port %d failed (ret = %d)\n",
-		       hca->name, port, result);
+		pr_warn("%s: ib_query_gid port %d failed (ret = %d)\n",
+			hca->name, port, result);
 		goto device_init_failed;
 	}
 
@@ -2260,8 +2297,8 @@ static struct net_device *ipoib_add_port(const char *format,
 
 	result = ipoib_dev_init(priv->dev, hca, port);
 	if (result) {
-		printk(KERN_WARNING "%s: failed to initialize port %d (ret = %d)\n",
-		       hca->name, port, result);
+		pr_warn("%s: failed to initialize port %d (ret = %d)\n",
+			hca->name, port, result);
 		goto device_init_failed;
 	}
 
@@ -2269,10 +2306,13 @@ static struct net_device *ipoib_add_port(const char *format,
 			      priv->ca, ipoib_event);
 	ib_register_event_handler(&priv->event_handler);
 
+	/* call event handler to ensure pkey in sync */
+	queue_work(ipoib_workqueue, &priv->flush_heavy);
+
 	result = register_netdev(priv->dev);
 	if (result) {
-		printk(KERN_WARNING "%s: couldn't register ipoib port %d; error %d\n",
-		       hca->name, port, result);
+		pr_warn("%s: couldn't register ipoib port %d; error %d\n",
+			hca->name, port, result);
 		goto register_failed;
 	}
 
@@ -2337,8 +2377,7 @@ static void ipoib_add_one(struct ib_device *device)
 	}
 
 	if (!count) {
-		pr_err("Failed to init port, removing it\n");
-		ipoib_remove_one(device, dev_list);
+		kfree(dev_list);
 		return;
 	}
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index a1ed25422b72..984a88096f39 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -178,7 +178,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
 	priv->recv_cq = ib_create_cq(priv->ca, ipoib_ib_rx_completion, NULL,
 				     priv, &cq_attr);
 	if (IS_ERR(priv->recv_cq)) {
-		printk(KERN_WARNING "%s: failed to create receive CQ\n", ca->name);
+		pr_warn("%s: failed to create receive CQ\n", ca->name);
 		goto out_cm_dev_cleanup;
 	}
 
@@ -187,7 +187,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
 	priv->send_cq = ib_create_cq(priv->ca, ipoib_ib_tx_completion, NULL,
 				     priv, &cq_attr);
 	if (IS_ERR(priv->send_cq)) {
-		printk(KERN_WARNING "%s: failed to create send CQ\n", ca->name);
+		pr_warn("%s: failed to create send CQ\n", ca->name);
 		goto out_free_recv_cq;
 	}
 
@@ -208,7 +208,7 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
 
 	priv->qp = ib_create_qp(priv->pd, &init_attr);
 	if (IS_ERR(priv->qp)) {
-		printk(KERN_WARNING "%s: failed to create QP\n", ca->name);
+		pr_warn("%s: failed to create QP\n", ca->name);
 		goto out_free_send_cq;
 	}
 
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 19624e023ebd..0336643c2ed6 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -874,7 +874,7 @@ iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
 	iser_info("iser conn %p rc = %d\n", iser_conn, rc);
 
 	if (rc > 0)
-		return 1; /* success, this is the equivalent of POLLOUT */
+		return 1; /* success, this is the equivalent of EPOLLOUT */
 	else if (!rc)
 		return 0; /* timeout */
 	else
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index 2a07692007bd..df49c4eb67f7 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -142,8 +142,7 @@ iser_prepare_write_cmd(struct iscsi_task *task,
 			hdr->write_va = cpu_to_be64(mem_reg->sge.addr + unsol_sz);
 		}
 
-		iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
-			 "VA:%#llX + unsol:%d\n",
+		iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X VA:%#llX + unsol:%d\n",
 			 task->itt, mem_reg->rkey,
 			 (unsigned long long)mem_reg->sge.addr, unsol_sz);
 	}
@@ -436,7 +435,7 @@ int iser_send_data_out(struct iscsi_conn *conn,
 {
 	struct iser_conn *iser_conn = conn->dd_data;
 	struct iscsi_iser_task *iser_task = task->dd_data;
-	struct iser_tx_desc *tx_desc = NULL;
+	struct iser_tx_desc *tx_desc;
 	struct iser_mem_reg *mem_reg;
 	unsigned long buf_offset;
 	unsigned long data_seg_len;
@@ -452,10 +451,8 @@ int iser_send_data_out(struct iscsi_conn *conn,
 		 __func__,(int)itt,(int)data_seg_len,(int)buf_offset);
 
 	tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC);
-	if (tx_desc == NULL) {
-		iser_err("Failed to alloc desc for post dataout\n");
+	if (!tx_desc)
 		return -ENOMEM;
-	}
 
 	tx_desc->type = ISCSI_TX_DATAOUT;
 	tx_desc->cqe.done = iser_dataout_comp;
@@ -475,8 +472,7 @@ int iser_send_data_out(struct iscsi_conn *conn,
 	tx_desc->num_sge = 2;
 
 	if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) {
-		iser_err("Offset:%ld & DSL:%ld in Data-Out "
-			 "inconsistent with total len:%ld, itt:%d\n",
+		iser_err("Offset:%ld & DSL:%ld in Data-Out inconsistent with total len:%ld, itt:%d\n",
 			 buf_offset, data_seg_len,
 			 iser_task->data[ISER_DIR_OUT].data_len, itt);
 		err = -EINVAL;
@@ -614,8 +610,8 @@ iser_check_remote_inv(struct iser_conn *iser_conn,
 			 iser_conn, rkey);
 
 		if (unlikely(!iser_conn->snd_w_inv)) {
-			iser_err("conn %p: unexpected remote invalidation, "
-				 "terminating connection\n", iser_conn);
+			iser_err("conn %p: unexpected remote invalidation, terminating connection\n",
+				 iser_conn);
 			return -EPROTO;
 		}
 
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 1b02283ce20e..fff40b097947 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -2124,6 +2124,9 @@ isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
 	u32 rkey, offset;
 	int ret;
 
+	if (cmd->ctx_init_done)
+		goto rdma_ctx_post;
+
 	if (dir == DMA_FROM_DEVICE) {
 		addr = cmd->write_va;
 		rkey = cmd->write_stag;
@@ -2151,11 +2154,15 @@ isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
 				se_cmd->t_data_sg, se_cmd->t_data_nents,
 				offset, addr, rkey, dir);
 	}
+
 	if (ret < 0) {
 		isert_err("Cmd: %p failed to prepare RDMA res\n", cmd);
 		return ret;
 	}
 
+	cmd->ctx_init_done = true;
+
+rdma_ctx_post:
 	ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr);
 	if (ret < 0)
 		isert_err("Cmd: %p failed to post RDMA res\n", cmd);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index d6fd248320ae..3b296bac4f60 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -126,6 +126,7 @@ struct isert_cmd {
 	struct rdma_rw_ctx	rw;
 	struct work_struct	comp_work;
 	struct scatterlist	sg;
+	bool			ctx_init_done;
 };
 
 static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc)
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
index 4b615c1451e7..15711dcc6f58 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
@@ -710,7 +710,7 @@ vema_get_port(struct opa_vnic_ctrl_port *cport, u8 port_num)
 
 /**
  * opa_vnic_vema_send_trap -- This function sends a trap to the EM
- * @cport: pointer to vnic control port
+ * @adapter: pointer to vnic adapter
  * @data: pointer to trap data filled by calling function
  * @lid:  issuers lid (encap_slid from vesw_port_info)
  *
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 972d4b3c5223..b48843833d69 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -41,6 +41,7 @@
 #include <linux/random.h>
 #include <linux/jiffies.h>
 #include <linux/lockdep.h>
+#include <linux/inet.h>
 #include <rdma/ib_cache.h>
 
 #include <linux/atomic.h>
@@ -144,7 +145,9 @@ static void srp_remove_one(struct ib_device *device, void *client_data);
 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
 		const char *opname);
-static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
+static int srp_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
+static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
+			       struct rdma_cm_event *event);
 
 static struct scsi_transport_template *ib_srp_transport_template;
 static struct workqueue_struct *srp_remove_wq;
@@ -265,8 +268,8 @@ static void srp_qp_event(struct ib_event *event, void *context)
 		 ib_event_msg(event->event), event->event);
 }
 
-static int srp_init_qp(struct srp_target_port *target,
-		       struct ib_qp *qp)
+static int srp_init_ib_qp(struct srp_target_port *target,
+			  struct ib_qp *qp)
 {
 	struct ib_qp_attr *attr;
 	int ret;
@@ -277,7 +280,7 @@ static int srp_init_qp(struct srp_target_port *target,
 
 	ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
 				  target->srp_host->port,
-				  be16_to_cpu(target->pkey),
+				  be16_to_cpu(target->ib_cm.pkey),
 				  &attr->pkey_index);
 	if (ret)
 		goto out;
@@ -298,32 +301,110 @@ static int srp_init_qp(struct srp_target_port *target,
 	return ret;
 }
 
-static int srp_new_cm_id(struct srp_rdma_ch *ch)
+static int srp_new_ib_cm_id(struct srp_rdma_ch *ch)
 {
 	struct srp_target_port *target = ch->target;
 	struct ib_cm_id *new_cm_id;
 
 	new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
-				    srp_cm_handler, ch);
+				    srp_ib_cm_handler, ch);
 	if (IS_ERR(new_cm_id))
 		return PTR_ERR(new_cm_id);
 
-	if (ch->cm_id)
-		ib_destroy_cm_id(ch->cm_id);
-	ch->cm_id = new_cm_id;
+	if (ch->ib_cm.cm_id)
+		ib_destroy_cm_id(ch->ib_cm.cm_id);
+	ch->ib_cm.cm_id = new_cm_id;
 	if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev,
 			    target->srp_host->port))
-		ch->path.rec_type = SA_PATH_REC_TYPE_OPA;
+		ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA;
 	else
-		ch->path.rec_type = SA_PATH_REC_TYPE_IB;
-	ch->path.sgid = target->sgid;
-	ch->path.dgid = target->orig_dgid;
-	ch->path.pkey = target->pkey;
-	ch->path.service_id = target->service_id;
+		ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB;
+	ch->ib_cm.path.sgid = target->sgid;
+	ch->ib_cm.path.dgid = target->ib_cm.orig_dgid;
+	ch->ib_cm.path.pkey = target->ib_cm.pkey;
+	ch->ib_cm.path.service_id = target->ib_cm.service_id;
 
 	return 0;
 }
 
+static const char *inet_ntop(const void *sa, char *dst, unsigned int size)
+{
+	switch (((struct sockaddr *)sa)->sa_family) {
+	case AF_INET:
+		snprintf(dst, size, "%pI4",
+			 &((struct sockaddr_in *)sa)->sin_addr);
+		break;
+	case AF_INET6:
+		snprintf(dst, size, "%pI6",
+			 &((struct sockaddr_in6 *)sa)->sin6_addr);
+		break;
+	default:
+		snprintf(dst, size, "???");
+		break;
+	}
+	return dst;
+}
+
+static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch)
+{
+	struct srp_target_port *target = ch->target;
+	struct rdma_cm_id *new_cm_id;
+	char src_addr[64], dst_addr[64];
+	int ret;
+
+	new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch,
+				   RDMA_PS_TCP, IB_QPT_RC);
+	if (IS_ERR(new_cm_id)) {
+		ret = PTR_ERR(new_cm_id);
+		new_cm_id = NULL;
+		goto out;
+	}
+
+	init_completion(&ch->done);
+	ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ?
+				(struct sockaddr *)&target->rdma_cm.src : NULL,
+				(struct sockaddr *)&target->rdma_cm.dst,
+				SRP_PATH_REC_TIMEOUT_MS);
+	if (ret) {
+		pr_err("No route available from %s to %s (%d)\n",
+		       target->rdma_cm.src_specified ?
+		       inet_ntop(&target->rdma_cm.src, src_addr,
+				 sizeof(src_addr)) : "(any)",
+		       inet_ntop(&target->rdma_cm.dst, dst_addr,
+				 sizeof(dst_addr)),
+		       ret);
+		goto out;
+	}
+	ret = wait_for_completion_interruptible(&ch->done);
+	if (ret < 0)
+		goto out;
+
+	ret = ch->status;
+	if (ret) {
+		pr_err("Resolving address %s failed (%d)\n",
+		       inet_ntop(&target->rdma_cm.dst, dst_addr,
+				 sizeof(dst_addr)),
+		       ret);
+		goto out;
+	}
+
+	swap(ch->rdma_cm.cm_id, new_cm_id);
+
+out:
+	if (new_cm_id)
+		rdma_destroy_id(new_cm_id);
+
+	return ret;
+}
+
+static int srp_new_cm_id(struct srp_rdma_ch *ch)
+{
+	struct srp_target_port *target = ch->target;
+
+	return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) :
+		srp_new_ib_cm_id(ch);
+}
+
 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 {
 	struct srp_device *dev = target->srp_host->srp_dev;
@@ -521,16 +602,25 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 	init_attr->send_cq             = send_cq;
 	init_attr->recv_cq             = recv_cq;
 
-	qp = ib_create_qp(dev->pd, init_attr);
-	if (IS_ERR(qp)) {
-		ret = PTR_ERR(qp);
+	if (target->using_rdma_cm) {
+		ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr);
+		qp = ch->rdma_cm.cm_id->qp;
+	} else {
+		qp = ib_create_qp(dev->pd, init_attr);
+		if (!IS_ERR(qp)) {
+			ret = srp_init_ib_qp(target, qp);
+			if (ret)
+				ib_destroy_qp(qp);
+		} else {
+			ret = PTR_ERR(qp);
+		}
+	}
+	if (ret) {
+		pr_err("QP creation failed for dev %s: %d\n",
+		       dev_name(&dev->dev->dev), ret);
 		goto err_send_cq;
 	}
 
-	ret = srp_init_qp(target, qp);
-	if (ret)
-		goto err_qp;
-
 	if (dev->use_fast_reg) {
 		fr_pool = srp_alloc_fr_pool(target);
 		if (IS_ERR(fr_pool)) {
@@ -574,7 +664,10 @@ static int srp_create_ch_ib(struct srp_rdma_ch *ch)
 	return 0;
 
 err_qp:
-	ib_destroy_qp(qp);
+	if (target->using_rdma_cm)
+		rdma_destroy_qp(ch->rdma_cm.cm_id);
+	else
+		ib_destroy_qp(qp);
 
 err_send_cq:
 	ib_free_cq(send_cq);
@@ -600,9 +693,16 @@ static void srp_free_ch_ib(struct srp_target_port *target,
 	if (!ch->target)
 		return;
 
-	if (ch->cm_id) {
-		ib_destroy_cm_id(ch->cm_id);
-		ch->cm_id = NULL;
+	if (target->using_rdma_cm) {
+		if (ch->rdma_cm.cm_id) {
+			rdma_destroy_id(ch->rdma_cm.cm_id);
+			ch->rdma_cm.cm_id = NULL;
+		}
+	} else {
+		if (ch->ib_cm.cm_id) {
+			ib_destroy_cm_id(ch->ib_cm.cm_id);
+			ch->ib_cm.cm_id = NULL;
+		}
 	}
 
 	/* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
@@ -658,16 +758,16 @@ static void srp_path_rec_completion(int status,
 		shost_printk(KERN_ERR, target->scsi_host,
 			     PFX "Got failed path rec status %d\n", status);
 	else
-		ch->path = *pathrec;
+		ch->ib_cm.path = *pathrec;
 	complete(&ch->done);
 }
 
-static int srp_lookup_path(struct srp_rdma_ch *ch)
+static int srp_ib_lookup_path(struct srp_rdma_ch *ch)
 {
 	struct srp_target_port *target = ch->target;
 	int ret = -ENODEV;
 
-	ch->path.numb_path = 1;
+	ch->ib_cm.path.numb_path = 1;
 
 	init_completion(&ch->done);
 
@@ -678,10 +778,10 @@ static int srp_lookup_path(struct srp_rdma_ch *ch)
 	if (!scsi_host_get(target->scsi_host))
 		goto out;
 
-	ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
+	ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 					       target->srp_host->srp_dev->dev,
 					       target->srp_host->port,
-					       &ch->path,
+					       &ch->ib_cm.path,
 					       IB_SA_PATH_REC_SERVICE_ID |
 					       IB_SA_PATH_REC_DGID	 |
 					       IB_SA_PATH_REC_SGID	 |
@@ -690,8 +790,8 @@ static int srp_lookup_path(struct srp_rdma_ch *ch)
 					       SRP_PATH_REC_TIMEOUT_MS,
 					       GFP_KERNEL,
 					       srp_path_rec_completion,
-					       ch, &ch->path_query);
-	ret = ch->path_query_id;
+					       ch, &ch->ib_cm.path_query);
+	ret = ch->ib_cm.path_query_id;
 	if (ret < 0)
 		goto put;
 
@@ -702,7 +802,10 @@ static int srp_lookup_path(struct srp_rdma_ch *ch)
 	ret = ch->status;
 	if (ret < 0)
 		shost_printk(KERN_WARNING, target->scsi_host,
-			     PFX "Path record query failed\n");
+			     PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n",
+			     ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw,
+			     be16_to_cpu(target->ib_cm.pkey),
+			     be64_to_cpu(target->ib_cm.service_id));
 
 put:
 	scsi_host_put(target->scsi_host);
@@ -711,6 +814,34 @@ static int srp_lookup_path(struct srp_rdma_ch *ch)
 	return ret;
 }
 
+static int srp_rdma_lookup_path(struct srp_rdma_ch *ch)
+{
+	struct srp_target_port *target = ch->target;
+	int ret;
+
+	init_completion(&ch->done);
+
+	ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS);
+	if (ret)
+		return ret;
+
+	wait_for_completion_interruptible(&ch->done);
+
+	if (ch->status != 0)
+		shost_printk(KERN_WARNING, target->scsi_host,
+			     PFX "Path resolution failed\n");
+
+	return ch->status;
+}
+
+static int srp_lookup_path(struct srp_rdma_ch *ch)
+{
+	struct srp_target_port *target = ch->target;
+
+	return target->using_rdma_cm ? srp_rdma_lookup_path(ch) :
+		srp_ib_lookup_path(ch);
+}
+
 static u8 srp_get_subnet_timeout(struct srp_host *host)
 {
 	struct ib_port_attr attr;
@@ -732,48 +863,76 @@ static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
 {
 	struct srp_target_port *target = ch->target;
 	struct {
-		struct ib_cm_req_param param;
-		struct srp_login_req   priv;
+		struct rdma_conn_param	  rdma_param;
+		struct srp_login_req_rdma rdma_req;
+		struct ib_cm_req_param	  ib_param;
+		struct srp_login_req	  ib_req;
 	} *req = NULL;
+	char *ipi, *tpi;
 	int status;
-	u8 subnet_timeout;
-
-	subnet_timeout = srp_get_subnet_timeout(target->srp_host);
 
 	req = kzalloc(sizeof *req, GFP_KERNEL);
 	if (!req)
 		return -ENOMEM;
 
-	req->param.primary_path		      = &ch->path;
-	req->param.alternate_path 	      = NULL;
-	req->param.service_id 		      = target->service_id;
-	req->param.qp_num		      = ch->qp->qp_num;
-	req->param.qp_type		      = ch->qp->qp_type;
-	req->param.private_data 	      = &req->priv;
-	req->param.private_data_len 	      = sizeof req->priv;
-	req->param.flow_control 	      = 1;
-
-	get_random_bytes(&req->param.starting_psn, 4);
-	req->param.starting_psn 	     &= 0xffffff;
+	req->ib_param.flow_control = 1;
+	req->ib_param.retry_count = target->tl_retry_count;
 
 	/*
 	 * Pick some arbitrary defaults here; we could make these
 	 * module parameters if anyone cared about setting them.
 	 */
-	req->param.responder_resources	      = 4;
-	req->param.remote_cm_response_timeout = subnet_timeout + 2;
-	req->param.local_cm_response_timeout  = subnet_timeout + 2;
-	req->param.retry_count                = target->tl_retry_count;
-	req->param.rnr_retry_count 	      = 7;
-	req->param.max_cm_retries 	      = 15;
-
-	req->priv.opcode     	= SRP_LOGIN_REQ;
-	req->priv.tag        	= 0;
-	req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
-	req->priv.req_buf_fmt 	= cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
+	req->ib_param.responder_resources = 4;
+	req->ib_param.rnr_retry_count = 7;
+	req->ib_param.max_cm_retries = 15;
+
+	req->ib_req.opcode = SRP_LOGIN_REQ;
+	req->ib_req.tag = 0;
+	req->ib_req.req_it_iu_len = cpu_to_be32(target->max_iu_len);
+	req->ib_req.req_buf_fmt	= cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 					      SRP_BUF_FORMAT_INDIRECT);
-	req->priv.req_flags	= (multich ? SRP_MULTICHAN_MULTI :
-				   SRP_MULTICHAN_SINGLE);
+	req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI :
+				 SRP_MULTICHAN_SINGLE);
+
+	if (target->using_rdma_cm) {
+		req->rdma_param.flow_control = req->ib_param.flow_control;
+		req->rdma_param.responder_resources =
+			req->ib_param.responder_resources;
+		req->rdma_param.initiator_depth = req->ib_param.initiator_depth;
+		req->rdma_param.retry_count = req->ib_param.retry_count;
+		req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count;
+		req->rdma_param.private_data = &req->rdma_req;
+		req->rdma_param.private_data_len = sizeof(req->rdma_req);
+
+		req->rdma_req.opcode = req->ib_req.opcode;
+		req->rdma_req.tag = req->ib_req.tag;
+		req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len;
+		req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt;
+		req->rdma_req.req_flags	= req->ib_req.req_flags;
+
+		ipi = req->rdma_req.initiator_port_id;
+		tpi = req->rdma_req.target_port_id;
+	} else {
+		u8 subnet_timeout;
+
+		subnet_timeout = srp_get_subnet_timeout(target->srp_host);
+
+		req->ib_param.primary_path = &ch->ib_cm.path;
+		req->ib_param.alternate_path = NULL;
+		req->ib_param.service_id = target->ib_cm.service_id;
+		get_random_bytes(&req->ib_param.starting_psn, 4);
+		req->ib_param.starting_psn &= 0xffffff;
+		req->ib_param.qp_num = ch->qp->qp_num;
+		req->ib_param.qp_type = ch->qp->qp_type;
+		req->ib_param.local_cm_response_timeout = subnet_timeout + 2;
+		req->ib_param.remote_cm_response_timeout = subnet_timeout + 2;
+		req->ib_param.private_data = &req->ib_req;
+		req->ib_param.private_data_len = sizeof(req->ib_req);
+
+		ipi = req->ib_req.initiator_port_id;
+		tpi = req->ib_req.target_port_id;
+	}
+
 	/*
 	 * In the published SRP specification (draft rev. 16a), the
 	 * port identifier format is 8 bytes of ID extension followed
@@ -784,19 +943,15 @@ static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
 	 * recognized by the I/O Class they report.
 	 */
 	if (target->io_class == SRP_REV10_IB_IO_CLASS) {
-		memcpy(req->priv.initiator_port_id,
-		       &target->sgid.global.interface_id, 8);
-		memcpy(req->priv.initiator_port_id + 8,
-		       &target->initiator_ext, 8);
-		memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
-		memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
+		memcpy(ipi,     &target->sgid.global.interface_id, 8);
+		memcpy(ipi + 8, &target->initiator_ext, 8);
+		memcpy(tpi,     &target->ioc_guid, 8);
+		memcpy(tpi + 8, &target->id_ext, 8);
 	} else {
-		memcpy(req->priv.initiator_port_id,
-		       &target->initiator_ext, 8);
-		memcpy(req->priv.initiator_port_id + 8,
-		       &target->sgid.global.interface_id, 8);
-		memcpy(req->priv.target_port_id,     &target->id_ext, 8);
-		memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
+		memcpy(ipi,     &target->initiator_ext, 8);
+		memcpy(ipi + 8, &target->sgid.global.interface_id, 8);
+		memcpy(tpi,     &target->id_ext, 8);
+		memcpy(tpi + 8, &target->ioc_guid, 8);
 	}
 
 	/*
@@ -809,12 +964,14 @@ static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
 			     PFX "Topspin/Cisco initiator port ID workaround "
 			     "activated for target GUID %016llx\n",
 			     be64_to_cpu(target->ioc_guid));
-		memset(req->priv.initiator_port_id, 0, 8);
-		memcpy(req->priv.initiator_port_id + 8,
-		       &target->srp_host->srp_dev->dev->node_guid, 8);
+		memset(ipi, 0, 8);
+		memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8);
 	}
 
-	status = ib_send_cm_req(ch->cm_id, &req->param);
+	if (target->using_rdma_cm)
+		status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param);
+	else
+		status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param);
 
 	kfree(req);
 
@@ -841,14 +998,23 @@ static bool srp_queue_remove_work(struct srp_target_port *target)
 static void srp_disconnect_target(struct srp_target_port *target)
 {
 	struct srp_rdma_ch *ch;
-	int i;
+	int i, ret;
 
 	/* XXX should send SRP_I_LOGOUT request */
 
 	for (i = 0; i < target->ch_count; i++) {
 		ch = &target->ch[i];
 		ch->connected = false;
-		if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) {
+		ret = 0;
+		if (target->using_rdma_cm) {
+			if (ch->rdma_cm.cm_id)
+				rdma_disconnect(ch->rdma_cm.cm_id);
+		} else {
+			if (ch->ib_cm.cm_id)
+				ret = ib_send_cm_dreq(ch->ib_cm.cm_id,
+						      NULL, 0);
+		}
+		if (ret < 0) {
 			shost_printk(KERN_DEBUG, target->scsi_host,
 				     PFX "Sending CM DREQ failed\n");
 		}
@@ -962,6 +1128,7 @@ static void srp_remove_target(struct srp_target_port *target)
 	scsi_remove_host(target->scsi_host);
 	srp_stop_rport_timers(target->rport);
 	srp_disconnect_target(target);
+	kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
 	for (i = 0; i < target->ch_count; i++) {
 		ch = &target->ch[i];
 		srp_free_ch_ib(target, ch);
@@ -2349,7 +2516,7 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
 	struct srp_target_port *target = ch->target;
 	struct ib_qp_attr *qp_attr = NULL;
 	int attr_mask = 0;
-	int ret;
+	int ret = 0;
 	int i;
 
 	if (lrsp->opcode == SRP_LOGIN_RSP) {
@@ -2379,40 +2546,42 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
 			goto error;
 	}
 
-	ret = -ENOMEM;
-	qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
-	if (!qp_attr)
-		goto error;
-
-	qp_attr->qp_state = IB_QPS_RTR;
-	ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
-	if (ret)
-		goto error_free;
-
-	ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
-	if (ret)
-		goto error_free;
-
 	for (i = 0; i < target->queue_size; i++) {
 		struct srp_iu *iu = ch->rx_ring[i];
 
 		ret = srp_post_recv(ch, iu);
 		if (ret)
-			goto error_free;
+			goto error;
 	}
 
-	qp_attr->qp_state = IB_QPS_RTS;
-	ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
-	if (ret)
-		goto error_free;
+	if (!target->using_rdma_cm) {
+		ret = -ENOMEM;
+		qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL);
+		if (!qp_attr)
+			goto error;
 
-	target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
+		qp_attr->qp_state = IB_QPS_RTR;
+		ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
+		if (ret)
+			goto error_free;
 
-	ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
-	if (ret)
-		goto error_free;
+		ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
+		if (ret)
+			goto error_free;
 
-	ret = ib_send_cm_rtu(cm_id, NULL, 0);
+		qp_attr->qp_state = IB_QPS_RTS;
+		ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
+		if (ret)
+			goto error_free;
+
+		target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
+
+		ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
+		if (ret)
+			goto error_free;
+
+		ret = ib_send_cm_rtu(cm_id, NULL, 0);
+	}
 
 error_free:
 	kfree(qp_attr);
@@ -2421,41 +2590,43 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
 	ch->status = ret;
 }
 
-static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
-			       struct ib_cm_event *event,
-			       struct srp_rdma_ch *ch)
+static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id,
+				  struct ib_cm_event *event,
+				  struct srp_rdma_ch *ch)
 {
 	struct srp_target_port *target = ch->target;
 	struct Scsi_Host *shost = target->scsi_host;
 	struct ib_class_port_info *cpi;
 	int opcode;
+	u16 dlid;
 
 	switch (event->param.rej_rcvd.reason) {
 	case IB_CM_REJ_PORT_CM_REDIRECT:
 		cpi = event->param.rej_rcvd.ari;
-		sa_path_set_dlid(&ch->path, ntohs(cpi->redirect_lid));
-		ch->path.pkey = cpi->redirect_pkey;
+		dlid = be16_to_cpu(cpi->redirect_lid);
+		sa_path_set_dlid(&ch->ib_cm.path, dlid);
+		ch->ib_cm.path.pkey = cpi->redirect_pkey;
 		cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
-		memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16);
+		memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16);
 
-		ch->status = sa_path_get_dlid(&ch->path) ?
-			SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
+		ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
 		break;
 
 	case IB_CM_REJ_PORT_REDIRECT:
 		if (srp_target_is_topspin(target)) {
+			union ib_gid *dgid = &ch->ib_cm.path.dgid;
+
 			/*
 			 * Topspin/Cisco SRP gateways incorrectly send
 			 * reject reason code 25 when they mean 24
 			 * (port redirect).
 			 */
-			memcpy(ch->path.dgid.raw,
-			       event->param.rej_rcvd.ari, 16);
+			memcpy(dgid->raw, event->param.rej_rcvd.ari, 16);
 
 			shost_printk(KERN_DEBUG, shost,
 				     PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
-				     be64_to_cpu(ch->path.dgid.global.subnet_prefix),
-				     be64_to_cpu(ch->path.dgid.global.interface_id));
+				     be64_to_cpu(dgid->global.subnet_prefix),
+				     be64_to_cpu(dgid->global.interface_id));
 
 			ch->status = SRP_PORT_REDIRECT;
 		} else {
@@ -2484,7 +2655,8 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
 				shost_printk(KERN_WARNING, shost, PFX
 					     "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
 					     target->sgid.raw,
-					     target->orig_dgid.raw, reason);
+					     target->ib_cm.orig_dgid.raw,
+					     reason);
 		} else
 			shost_printk(KERN_WARNING, shost,
 				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
@@ -2504,7 +2676,7 @@ static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
 	}
 }
 
-static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
+static int srp_ib_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
 {
 	struct srp_rdma_ch *ch = cm_id->context;
 	struct srp_target_port *target = ch->target;
@@ -2527,7 +2699,7 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
 		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
 		comp = 1;
 
-		srp_cm_rej_handler(cm_id, event, ch);
+		srp_ib_cm_rej_handler(cm_id, event, ch);
 		break;
 
 	case IB_CM_DREQ_RECEIVED:
@@ -2565,6 +2737,135 @@ static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
 	return 0;
 }
 
+static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch,
+				    struct rdma_cm_event *event)
+{
+	struct srp_target_port *target = ch->target;
+	struct Scsi_Host *shost = target->scsi_host;
+	int opcode;
+
+	switch (event->status) {
+	case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
+		shost_printk(KERN_WARNING, shost,
+			    "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
+		ch->status = -ECONNRESET;
+		break;
+
+	case IB_CM_REJ_CONSUMER_DEFINED:
+		opcode = *(u8 *) event->param.conn.private_data;
+		if (opcode == SRP_LOGIN_REJ) {
+			struct srp_login_rej *rej =
+				(struct srp_login_rej *)
+				event->param.conn.private_data;
+			u32 reason = be32_to_cpu(rej->reason);
+
+			if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
+				shost_printk(KERN_WARNING, shost,
+					     PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
+			else
+				shost_printk(KERN_WARNING, shost,
+					    PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason);
+		} else {
+			shost_printk(KERN_WARNING, shost,
+				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n",
+				     opcode);
+		}
+		ch->status = -ECONNRESET;
+		break;
+
+	case IB_CM_REJ_STALE_CONN:
+		shost_printk(KERN_WARNING, shost,
+			     "  REJ reason: stale connection\n");
+		ch->status = SRP_STALE_CONN;
+		break;
+
+	default:
+		shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
+			     event->status);
+		ch->status = -ECONNRESET;
+		break;
+	}
+}
+
+static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
+			       struct rdma_cm_event *event)
+{
+	struct srp_rdma_ch *ch = cm_id->context;
+	struct srp_target_port *target = ch->target;
+	int comp = 0;
+
+	switch (event->event) {
+	case RDMA_CM_EVENT_ADDR_RESOLVED:
+		ch->status = 0;
+		comp = 1;
+		break;
+
+	case RDMA_CM_EVENT_ADDR_ERROR:
+		ch->status = -ENXIO;
+		comp = 1;
+		break;
+
+	case RDMA_CM_EVENT_ROUTE_RESOLVED:
+		ch->status = 0;
+		comp = 1;
+		break;
+
+	case RDMA_CM_EVENT_ROUTE_ERROR:
+	case RDMA_CM_EVENT_UNREACHABLE:
+		ch->status = -EHOSTUNREACH;
+		comp = 1;
+		break;
+
+	case RDMA_CM_EVENT_CONNECT_ERROR:
+		shost_printk(KERN_DEBUG, target->scsi_host,
+			     PFX "Sending CM REQ failed\n");
+		comp = 1;
+		ch->status = -ECONNRESET;
+		break;
+
+	case RDMA_CM_EVENT_ESTABLISHED:
+		comp = 1;
+		srp_cm_rep_handler(NULL, event->param.conn.private_data, ch);
+		break;
+
+	case RDMA_CM_EVENT_REJECTED:
+		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
+		comp = 1;
+
+		srp_rdma_cm_rej_handler(ch, event);
+		break;
+
+	case RDMA_CM_EVENT_DISCONNECTED:
+		if (ch->connected) {
+			shost_printk(KERN_WARNING, target->scsi_host,
+				     PFX "received DREQ\n");
+			rdma_disconnect(ch->rdma_cm.cm_id);
+			comp = 1;
+			ch->status = 0;
+			queue_work(system_long_wq, &target->tl_err_work);
+		}
+		break;
+
+	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+		shost_printk(KERN_ERR, target->scsi_host,
+			     PFX "connection closed\n");
+
+		comp = 1;
+		ch->status = 0;
+		break;
+
+	default:
+		shost_printk(KERN_WARNING, target->scsi_host,
+			     PFX "Unhandled CM event %d\n", event->event);
+		break;
+	}
+
+	if (comp)
+		complete(&ch->done);
+
+	return 0;
+}
+
 /**
  * srp_change_queue_depth - setting device queue depth
  * @sdev: scsi device struct
@@ -2717,6 +3018,16 @@ static int srp_reset_host(struct scsi_cmnd *scmnd)
 	return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
 }
 
+static int srp_target_alloc(struct scsi_target *starget)
+{
+	struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
+	struct srp_target_port *target = host_to_target(shost);
+
+	if (target->target_can_queue)
+		starget->can_queue = target->target_can_queue;
+	return 0;
+}
+
 static int srp_slave_alloc(struct scsi_device *sdev)
 {
 	struct Scsi_Host *shost = sdev->host;
@@ -2766,7 +3077,10 @@ static ssize_t show_service_id(struct device *dev,
 {
 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
 
-	return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id));
+	if (target->using_rdma_cm)
+		return -ENOENT;
+	return sprintf(buf, "0x%016llx\n",
+		       be64_to_cpu(target->ib_cm.service_id));
 }
 
 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
@@ -2774,7 +3088,9 @@ static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
 {
 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
 
-	return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey));
+	if (target->using_rdma_cm)
+		return -ENOENT;
+	return sprintf(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey));
 }
 
 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
@@ -2791,7 +3107,9 @@ static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
 	struct srp_rdma_ch *ch = &target->ch[0];
 
-	return sprintf(buf, "%pI6\n", ch->path.dgid.raw);
+	if (target->using_rdma_cm)
+		return -ENOENT;
+	return sprintf(buf, "%pI6\n", ch->ib_cm.path.dgid.raw);
 }
 
 static ssize_t show_orig_dgid(struct device *dev,
@@ -2799,7 +3117,9 @@ static ssize_t show_orig_dgid(struct device *dev,
 {
 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
 
-	return sprintf(buf, "%pI6\n", target->orig_dgid.raw);
+	if (target->using_rdma_cm)
+		return -ENOENT;
+	return sprintf(buf, "%pI6\n", target->ib_cm.orig_dgid.raw);
 }
 
 static ssize_t show_req_lim(struct device *dev,
@@ -2921,6 +3241,7 @@ static struct scsi_host_template srp_template = {
 	.module				= THIS_MODULE,
 	.name				= "InfiniBand SRP initiator",
 	.proc_name			= DRV_NAME,
+	.target_alloc			= srp_target_alloc,
 	.slave_alloc			= srp_slave_alloc,
 	.slave_configure		= srp_slave_configure,
 	.info				= srp_target_info,
@@ -3044,6 +3365,9 @@ static bool srp_conn_unique(struct srp_host *host,
 		if (t != target &&
 		    target->id_ext == t->id_ext &&
 		    target->ioc_guid == t->ioc_guid &&
+		    (!target->using_rdma_cm ||
+		     memcmp(&target->rdma_cm.dst, &t->rdma_cm.dst,
+			    sizeof(target->rdma_cm.dst)) == 0) &&
 		    target->initiator_ext == t->initiator_ext) {
 			ret = false;
 			break;
@@ -3060,6 +3384,9 @@ static bool srp_conn_unique(struct srp_host *host,
  *
  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
  *     pkey=<P_Key>,service_id=<service ID>
+ * or
+ *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,
+ *     [src=<IPv4 address>,]dest=<IPv4 address>:<port number>
  *
  * to the add_target sysfs attribute.
  */
@@ -3080,11 +3407,20 @@ enum {
 	SRP_OPT_COMP_VECTOR	= 1 << 12,
 	SRP_OPT_TL_RETRY_COUNT	= 1 << 13,
 	SRP_OPT_QUEUE_SIZE	= 1 << 14,
-	SRP_OPT_ALL		= (SRP_OPT_ID_EXT	|
-				   SRP_OPT_IOC_GUID	|
-				   SRP_OPT_DGID		|
-				   SRP_OPT_PKEY		|
-				   SRP_OPT_SERVICE_ID),
+	SRP_OPT_IP_SRC		= 1 << 15,
+	SRP_OPT_IP_DEST		= 1 << 16,
+	SRP_OPT_TARGET_CAN_QUEUE= 1 << 17,
+};
+
+static unsigned int srp_opt_mandatory[] = {
+	SRP_OPT_ID_EXT		|
+	SRP_OPT_IOC_GUID	|
+	SRP_OPT_DGID		|
+	SRP_OPT_PKEY		|
+	SRP_OPT_SERVICE_ID,
+	SRP_OPT_ID_EXT		|
+	SRP_OPT_IOC_GUID	|
+	SRP_OPT_IP_DEST,
 };
 
 static const match_table_t srp_opt_tokens = {
@@ -3095,6 +3431,7 @@ static const match_table_t srp_opt_tokens = {
 	{ SRP_OPT_SERVICE_ID,		"service_id=%s"		},
 	{ SRP_OPT_MAX_SECT,		"max_sect=%d" 		},
 	{ SRP_OPT_MAX_CMD_PER_LUN,	"max_cmd_per_lun=%d" 	},
+	{ SRP_OPT_TARGET_CAN_QUEUE,	"target_can_queue=%d"	},
 	{ SRP_OPT_IO_CLASS,		"io_class=%x"		},
 	{ SRP_OPT_INITIATOR_EXT,	"initiator_ext=%s"	},
 	{ SRP_OPT_CMD_SG_ENTRIES,	"cmd_sg_entries=%u"	},
@@ -3103,15 +3440,33 @@ static const match_table_t srp_opt_tokens = {
 	{ SRP_OPT_COMP_VECTOR,		"comp_vector=%u"	},
 	{ SRP_OPT_TL_RETRY_COUNT,	"tl_retry_count=%u"	},
 	{ SRP_OPT_QUEUE_SIZE,		"queue_size=%d"		},
+	{ SRP_OPT_IP_SRC,		"src=%s"		},
+	{ SRP_OPT_IP_DEST,		"dest=%s"		},
 	{ SRP_OPT_ERR,			NULL 			}
 };
 
-static int srp_parse_options(const char *buf, struct srp_target_port *target)
+static int srp_parse_in(struct net *net, struct sockaddr_storage *sa,
+			const char *addr_port_str)
+{
+	char *addr = kstrdup(addr_port_str, GFP_KERNEL);
+	char *port_str = addr;
+	int ret;
+
+	if (!addr)
+		return -ENOMEM;
+	strsep(&port_str, ":");
+	ret = inet_pton_with_scope(net, AF_UNSPEC, addr, port_str, sa);
+	kfree(addr);
+	return ret;
+}
+
+static int srp_parse_options(struct net *net, const char *buf,
+			     struct srp_target_port *target)
 {
 	char *options, *sep_opt;
 	char *p;
-	char dgid[3];
 	substring_t args[MAX_OPT_ARGS];
+	unsigned long long ull;
 	int opt_mask = 0;
 	int token;
 	int ret = -EINVAL;
@@ -3136,7 +3491,13 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 				ret = -ENOMEM;
 				goto out;
 			}
-			target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
+			ret = kstrtoull(p, 16, &ull);
+			if (ret) {
+				pr_warn("invalid id_ext parameter '%s'\n", p);
+				kfree(p);
+				goto out;
+			}
+			target->id_ext = cpu_to_be64(ull);
 			kfree(p);
 			break;
 
@@ -3146,7 +3507,13 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 				ret = -ENOMEM;
 				goto out;
 			}
-			target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
+			ret = kstrtoull(p, 16, &ull);
+			if (ret) {
+				pr_warn("invalid ioc_guid parameter '%s'\n", p);
+				kfree(p);
+				goto out;
+			}
+			target->ioc_guid = cpu_to_be64(ull);
 			kfree(p);
 			break;
 
@@ -3162,16 +3529,10 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 				goto out;
 			}
 
-			for (i = 0; i < 16; ++i) {
-				strlcpy(dgid, p + i * 2, sizeof(dgid));
-				if (sscanf(dgid, "%hhx",
-					   &target->orig_dgid.raw[i]) < 1) {
-					ret = -EINVAL;
-					kfree(p);
-					goto out;
-				}
-			}
+			ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16);
 			kfree(p);
+			if (ret < 0)
+				goto out;
 			break;
 
 		case SRP_OPT_PKEY:
@@ -3179,7 +3540,7 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 				pr_warn("bad P_Key parameter '%s'\n", p);
 				goto out;
 			}
-			target->pkey = cpu_to_be16(token);
+			target->ib_cm.pkey = cpu_to_be16(token);
 			break;
 
 		case SRP_OPT_SERVICE_ID:
@@ -3188,7 +3549,45 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 				ret = -ENOMEM;
 				goto out;
 			}
-			target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
+			ret = kstrtoull(p, 16, &ull);
+			if (ret) {
+				pr_warn("bad service_id parameter '%s'\n", p);
+				kfree(p);
+				goto out;
+			}
+			target->ib_cm.service_id = cpu_to_be64(ull);
+			kfree(p);
+			break;
+
+		case SRP_OPT_IP_SRC:
+			p = match_strdup(args);
+			if (!p) {
+				ret = -ENOMEM;
+				goto out;
+			}
+			ret = srp_parse_in(net, &target->rdma_cm.src.ss, p);
+			if (ret < 0) {
+				pr_warn("bad source parameter '%s'\n", p);
+				kfree(p);
+				goto out;
+			}
+			target->rdma_cm.src_specified = true;
+			kfree(p);
+			break;
+
+		case SRP_OPT_IP_DEST:
+			p = match_strdup(args);
+			if (!p) {
+				ret = -ENOMEM;
+				goto out;
+			}
+			ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p);
+			if (ret < 0) {
+				pr_warn("bad dest parameter '%s'\n", p);
+				kfree(p);
+				goto out;
+			}
+			target->using_rdma_cm = true;
 			kfree(p);
 			break;
 
@@ -3221,6 +3620,15 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 			target->scsi_host->cmd_per_lun = token;
 			break;
 
+		case SRP_OPT_TARGET_CAN_QUEUE:
+			if (match_int(args, &token) || token < 1) {
+				pr_warn("bad max target_can_queue parameter '%s'\n",
+					p);
+				goto out;
+			}
+			target->target_can_queue = token;
+			break;
+
 		case SRP_OPT_IO_CLASS:
 			if (match_hex(args, &token)) {
 				pr_warn("bad IO class parameter '%s'\n", p);
@@ -3242,7 +3650,13 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 				ret = -ENOMEM;
 				goto out;
 			}
-			target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
+			ret = kstrtoull(p, 16, &ull);
+			if (ret) {
+				pr_warn("bad initiator_ext value '%s'\n", p);
+				kfree(p);
+				goto out;
+			}
+			target->initiator_ext = cpu_to_be64(ull);
 			kfree(p);
 			break;
 
@@ -3297,14 +3711,14 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
 		}
 	}
 
-	if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
-		ret = 0;
-	else
-		for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
-			if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
-			    !(srp_opt_tokens[i].token & opt_mask))
-				pr_warn("target creation request is missing parameter '%s'\n",
-					srp_opt_tokens[i].pattern);
+	for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) {
+		if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) {
+			ret = 0;
+			break;
+		}
+	}
+	if (ret)
+		pr_warn("target creation request is missing one or more parameters\n");
 
 	if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
 	    && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
@@ -3345,6 +3759,7 @@ static ssize_t srp_create_target(struct device *dev,
 
 	target = host_to_target(target_host);
 
+	target->net		= kobj_ns_grab_current(KOBJ_NS_TYPE_NET);
 	target->io_class	= SRP_REV16A_IB_IO_CLASS;
 	target->scsi_host	= target_host;
 	target->srp_host	= host;
@@ -3366,18 +3781,29 @@ static ssize_t srp_create_target(struct device *dev,
 	if (ret < 0)
 		goto put;
 
-	ret = srp_parse_options(buf, target);
+	ret = srp_parse_options(target->net, buf, target);
 	if (ret)
 		goto out;
 
 	target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
 
 	if (!srp_conn_unique(target->srp_host, target)) {
-		shost_printk(KERN_INFO, target->scsi_host,
-			     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
-			     be64_to_cpu(target->id_ext),
-			     be64_to_cpu(target->ioc_guid),
-			     be64_to_cpu(target->initiator_ext));
+		if (target->using_rdma_cm) {
+			char dst_addr[64];
+
+			shost_printk(KERN_INFO, target->scsi_host,
+				     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%s\n",
+				     be64_to_cpu(target->id_ext),
+				     be64_to_cpu(target->ioc_guid),
+				     inet_ntop(&target->rdma_cm.dst, dst_addr,
+					       sizeof(dst_addr)));
+		} else {
+			shost_printk(KERN_INFO, target->scsi_host,
+				     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
+				     be64_to_cpu(target->id_ext),
+				     be64_to_cpu(target->ioc_guid),
+				     be64_to_cpu(target->initiator_ext));
+		}
 		ret = -EEXIST;
 		goto out;
 	}
@@ -3478,11 +3904,18 @@ static ssize_t srp_create_target(struct device *dev,
 
 			ret = srp_connect_ch(ch, multich);
 			if (ret) {
+				char dst[64];
+
+				if (target->using_rdma_cm)
+					inet_ntop(&target->rdma_cm.dst, dst,
+						  sizeof(dst));
+				else
+					snprintf(dst, sizeof(dst), "%pI6",
+						 target->ib_cm.orig_dgid.raw);
 				shost_printk(KERN_ERR, target->scsi_host,
-					     PFX "Connection %d/%d to %pI6 failed\n",
+					     PFX "Connection %d/%d to %s failed\n",
 					     ch_start + cpu_idx,
-					     target->ch_count,
-					     ch->target->orig_dgid.raw);
+					     target->ch_count, dst);
 				if (node_idx == 0 && cpu_idx == 0) {
 					goto free_ch;
 				} else {
@@ -3507,13 +3940,25 @@ static ssize_t srp_create_target(struct device *dev,
 		goto err_disconnect;
 
 	if (target->state != SRP_TARGET_REMOVED) {
-		shost_printk(KERN_DEBUG, target->scsi_host, PFX
-			     "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
-			     be64_to_cpu(target->id_ext),
-			     be64_to_cpu(target->ioc_guid),
-			     be16_to_cpu(target->pkey),
-			     be64_to_cpu(target->service_id),
-			     target->sgid.raw, target->orig_dgid.raw);
+		if (target->using_rdma_cm) {
+			char dst[64];
+
+			inet_ntop(&target->rdma_cm.dst, dst, sizeof(dst));
+			shost_printk(KERN_DEBUG, target->scsi_host, PFX
+				     "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %s\n",
+				     be64_to_cpu(target->id_ext),
+				     be64_to_cpu(target->ioc_guid),
+				     target->sgid.raw, dst);
+		} else {
+			shost_printk(KERN_DEBUG, target->scsi_host, PFX
+				     "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
+				     be64_to_cpu(target->id_ext),
+				     be64_to_cpu(target->ioc_guid),
+				     be16_to_cpu(target->ib_cm.pkey),
+				     be64_to_cpu(target->ib_cm.service_id),
+				     target->sgid.raw,
+				     target->ib_cm.orig_dgid.raw);
+		}
 	}
 
 	ret = count;
@@ -3523,8 +3968,16 @@ static ssize_t srp_create_target(struct device *dev,
 
 put:
 	scsi_host_put(target->scsi_host);
-	if (ret < 0)
+	if (ret < 0) {
+		/*
+		 * If a call to srp_remove_target() has not been scheduled,
+		 * drop the network namespace reference now that was obtained
+		 * earlier in this function.
+		 */
+		if (target->state != SRP_TARGET_REMOVED)
+			kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
 		scsi_host_put(target->scsi_host);
+	}
 
 	return ret;
 
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index a814f5ef16f9..a2706086b9c7 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -45,6 +45,7 @@
 #include <rdma/ib_sa.h>
 #include <rdma/ib_cm.h>
 #include <rdma/ib_fmr_pool.h>
+#include <rdma/rdma_cm.h>
 
 enum {
 	SRP_PATH_REC_TIMEOUT_MS	= 1000,
@@ -153,11 +154,18 @@ struct srp_rdma_ch {
 	struct completion	done;
 	int			status;
 
-	struct sa_path_rec	path;
-	struct ib_sa_query     *path_query;
-	int			path_query_id;
+	union {
+		struct ib_cm {
+			struct sa_path_rec	path;
+			struct ib_sa_query	*path_query;
+			int			path_query_id;
+			struct ib_cm_id		*cm_id;
+		} ib_cm;
+		struct rdma_cm {
+			struct rdma_cm_id	*cm_id;
+		} rdma_cm;
+	};
 
-	struct ib_cm_id	       *cm_id;
 	struct srp_iu	      **tx_ring;
 	struct srp_iu	      **rx_ring;
 	struct srp_request     *req_ring;
@@ -182,6 +190,7 @@ struct srp_target_port {
 	/* read only in the hot path */
 	u32			global_rkey;
 	struct srp_rdma_ch	*ch;
+	struct net		*net;
 	u32			ch_count;
 	u32			lkey;
 	enum srp_target_state	state;
@@ -194,7 +203,6 @@ struct srp_target_port {
 	union ib_gid		sgid;
 	__be64			id_ext;
 	__be64			ioc_guid;
-	__be64			service_id;
 	__be64			initiator_ext;
 	u16			io_class;
 	struct srp_host	       *srp_host;
@@ -203,6 +211,7 @@ struct srp_target_port {
 	char			target_name[32];
 	unsigned int		scsi_id;
 	unsigned int		sg_tablesize;
+	unsigned int		target_can_queue;
 	int			mr_pool_size;
 	int			mr_per_cmd;
 	int			queue_size;
@@ -210,8 +219,28 @@ struct srp_target_port {
 	int			comp_vector;
 	int			tl_retry_count;
 
-	union ib_gid		orig_dgid;
-	__be16			pkey;
+	bool			using_rdma_cm;
+
+	union {
+		struct {
+			__be64			service_id;
+			union ib_gid		orig_dgid;
+			__be16			pkey;
+		} ib_cm;
+		struct {
+			union {
+				struct sockaddr_in	ip4;
+				struct sockaddr_in6	ip6;
+				struct sockaddr_storage ss;
+			} src;
+			union {
+				struct sockaddr_in	ip4;
+				struct sockaddr_in6	ip6;
+				struct sockaddr_storage ss;
+			} dst;
+			bool src_specified;
+		} rdma_cm;
+	};
 
 	u32			rq_tmo_jiffies;
 
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index bfa576aa9f03..0373b7c40902 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -41,6 +41,7 @@
 #include <linux/string.h>
 #include <linux/delay.h>
 #include <linux/atomic.h>
+#include <rdma/ib_cache.h>
 #include <scsi/scsi_proto.h>
 #include <scsi/scsi_tcq.h>
 #include <target/target_core_base.h>
@@ -120,7 +121,9 @@ static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
 }
 
 /**
- * srpt_event_handler() - Asynchronous IB event callback function.
+ * srpt_event_handler - asynchronous IB event callback function
+ * @handler: IB event handler registered by ib_register_event_handler().
+ * @event: Description of the event that occurred.
  *
  * Callback function called by the InfiniBand core when an asynchronous IB
  * event occurs. This callback may occur in interrupt context. See also
@@ -132,6 +135,7 @@ static void srpt_event_handler(struct ib_event_handler *handler,
 {
 	struct srpt_device *sdev;
 	struct srpt_port *sport;
+	u8 port_num;
 
 	sdev = ib_get_client_data(event->device, &srpt_client);
 	if (!sdev || sdev->device != event->device)
@@ -142,10 +146,15 @@ static void srpt_event_handler(struct ib_event_handler *handler,
 
 	switch (event->event) {
 	case IB_EVENT_PORT_ERR:
-		if (event->element.port_num <= sdev->device->phys_port_cnt) {
-			sport = &sdev->port[event->element.port_num - 1];
+		port_num = event->element.port_num - 1;
+		if (port_num < sdev->device->phys_port_cnt) {
+			sport = &sdev->port[port_num];
 			sport->lid = 0;
 			sport->sm_lid = 0;
+		} else {
+			WARN(true, "event %d: port_num %d out of range 1..%d\n",
+			     event->event, port_num + 1,
+			     sdev->device->phys_port_cnt);
 		}
 		break;
 	case IB_EVENT_PORT_ACTIVE:
@@ -155,25 +164,31 @@ static void srpt_event_handler(struct ib_event_handler *handler,
 	case IB_EVENT_CLIENT_REREGISTER:
 	case IB_EVENT_GID_CHANGE:
 		/* Refresh port data asynchronously. */
-		if (event->element.port_num <= sdev->device->phys_port_cnt) {
-			sport = &sdev->port[event->element.port_num - 1];
+		port_num = event->element.port_num - 1;
+		if (port_num < sdev->device->phys_port_cnt) {
+			sport = &sdev->port[port_num];
 			if (!sport->lid && !sport->sm_lid)
 				schedule_work(&sport->work);
+		} else {
+			WARN(true, "event %d: port_num %d out of range 1..%d\n",
+			     event->event, port_num + 1,
+			     sdev->device->phys_port_cnt);
 		}
 		break;
 	default:
-		pr_err("received unrecognized IB event %d\n",
-		       event->event);
+		pr_err("received unrecognized IB event %d\n", event->event);
 		break;
 	}
 }
 
 /**
- * srpt_srq_event() - SRQ event callback function.
+ * srpt_srq_event - SRQ event callback function
+ * @event: Description of the event that occurred.
+ * @ctx: Context pointer specified at SRQ creation time.
  */
 static void srpt_srq_event(struct ib_event *event, void *ctx)
 {
-	pr_info("SRQ event %d\n", event->event);
+	pr_debug("SRQ event %d\n", event->event);
 }
 
 static const char *get_ch_state_name(enum rdma_ch_state s)
@@ -194,16 +209,18 @@ static const char *get_ch_state_name(enum rdma_ch_state s)
 }
 
 /**
- * srpt_qp_event() - QP event callback function.
+ * srpt_qp_event - QP event callback function
+ * @event: Description of the event that occurred.
+ * @ch: SRPT RDMA channel.
  */
 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
 {
-	pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n",
-		 event->event, ch->cm_id, ch->sess_name, ch->state);
+	pr_debug("QP event %d on ch=%p sess_name=%s state=%d\n",
+		 event->event, ch, ch->sess_name, ch->state);
 
 	switch (event->event) {
 	case IB_EVENT_COMM_EST:
-		ib_cm_notify(ch->cm_id, event->event);
+		ib_cm_notify(ch->ib_cm.cm_id, event->event);
 		break;
 	case IB_EVENT_QP_LAST_WQE_REACHED:
 		pr_debug("%s-%d, state %s: received Last WQE event.\n",
@@ -217,8 +234,8 @@ static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
 }
 
 /**
- * srpt_set_ioc() - Helper function for initializing an IOUnitInfo structure.
- *
+ * srpt_set_ioc - initialize a IOUnitInfo structure
+ * @c_list: controller list.
  * @slot: one-based slot number.
  * @value: four-bit value.
  *
@@ -241,7 +258,8 @@ static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
 }
 
 /**
- * srpt_get_class_port_info() - Copy ClassPortInfo to a management datagram.
+ * srpt_get_class_port_info - copy ClassPortInfo to a management datagram
+ * @mad: Datagram that will be sent as response to DM_ATTR_CLASS_PORT_INFO.
  *
  * See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture
  * Specification.
@@ -260,7 +278,8 @@ static void srpt_get_class_port_info(struct ib_dm_mad *mad)
 }
 
 /**
- * srpt_get_iou() - Write IOUnitInfo to a management datagram.
+ * srpt_get_iou - write IOUnitInfo to a management datagram
+ * @mad: Datagram that will be sent as response to DM_ATTR_IOU_INFO.
  *
  * See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture
  * Specification. See also section B.7, table B.6 in the SRP r16a document.
@@ -284,7 +303,10 @@ static void srpt_get_iou(struct ib_dm_mad *mad)
 }
 
 /**
- * srpt_get_ioc() - Write IOControllerprofile to a management datagram.
+ * srpt_get_ioc - write IOControllerprofile to a management datagram
+ * @sport: HCA port through which the MAD has been received.
+ * @slot: Slot number specified in DM_ATTR_IOC_PROFILE query.
+ * @mad: Datagram that will be sent as response to DM_ATTR_IOC_PROFILE.
  *
  * See also section 16.3.3.4 IOControllerProfile in the InfiniBand
  * Architecture Specification. See also section B.7, table B.7 in the SRP
@@ -314,7 +336,7 @@ static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
 	if (sdev->use_srq)
 		send_queue_depth = sdev->srq_size;
 	else
-		send_queue_depth = min(SRPT_RQ_SIZE,
+		send_queue_depth = min(MAX_SRPT_RQ_SIZE,
 				       sdev->device->attrs.max_qp_wr);
 
 	memset(iocp, 0, sizeof(*iocp));
@@ -342,7 +364,12 @@ static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
 }
 
 /**
- * srpt_get_svc_entries() - Write ServiceEntries to a management datagram.
+ * srpt_get_svc_entries - write ServiceEntries to a management datagram
+ * @ioc_guid: I/O controller GUID to use in reply.
+ * @slot: I/O controller number.
+ * @hi: End of the range of service entries to be specified in the reply.
+ * @lo: Start of the range of service entries to be specified in the reply..
+ * @mad: Datagram that will be sent as response to DM_ATTR_SVC_ENTRIES.
  *
  * See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
  * Specification. See also section B.7, table B.8 in the SRP r16a document.
@@ -379,8 +406,8 @@ static void srpt_get_svc_entries(u64 ioc_guid,
 }
 
 /**
- * srpt_mgmt_method_get() - Process a received management datagram.
- * @sp:      source port through which the MAD has been received.
+ * srpt_mgmt_method_get - process a received management datagram
+ * @sp:      HCA port through which the MAD has been received.
  * @rq_mad:  received MAD.
  * @rsp_mad: response MAD.
  */
@@ -419,7 +446,9 @@ static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
 }
 
 /**
- * srpt_mad_send_handler() - Post MAD-send callback function.
+ * srpt_mad_send_handler - MAD send completion callback
+ * @mad_agent: Return value of ib_register_mad_agent().
+ * @mad_wc: Work completion reporting that the MAD has been sent.
  */
 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
 				  struct ib_mad_send_wc *mad_wc)
@@ -429,7 +458,10 @@ static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
 }
 
 /**
- * srpt_mad_recv_handler() - MAD reception callback function.
+ * srpt_mad_recv_handler - MAD reception callback function
+ * @mad_agent: Return value of ib_register_mad_agent().
+ * @send_buf: Not used.
+ * @mad_wc: Work completion reporting that a MAD has been received.
  */
 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
 				  struct ib_mad_send_buf *send_buf,
@@ -493,8 +525,18 @@ static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
 	ib_free_recv_mad(mad_wc);
 }
 
+static int srpt_format_guid(char *buf, unsigned int size, const __be64 *guid)
+{
+	const __be16 *g = (const __be16 *)guid;
+
+	return snprintf(buf, size, "%04x:%04x:%04x:%04x",
+			be16_to_cpu(g[0]), be16_to_cpu(g[1]),
+			be16_to_cpu(g[2]), be16_to_cpu(g[3]));
+}
+
 /**
- * srpt_refresh_port() - Configure a HCA port.
+ * srpt_refresh_port - configure a HCA port
+ * @sport: SRPT HCA port.
  *
  * Enable InfiniBand management datagram processing, update the cached sm_lid,
  * lid and gid values, and register a callback function for processing MADs
@@ -507,7 +549,6 @@ static int srpt_refresh_port(struct srpt_port *sport)
 	struct ib_mad_reg_req reg_req;
 	struct ib_port_modify port_modify;
 	struct ib_port_attr port_attr;
-	__be16 *guid;
 	int ret;
 
 	memset(&port_modify, 0, sizeof(port_modify));
@@ -531,11 +572,8 @@ static int srpt_refresh_port(struct srpt_port *sport)
 		goto err_query_port;
 
 	sport->port_guid_wwn.priv = sport;
-	guid = (__be16 *)&sport->gid.global.interface_id;
-	snprintf(sport->port_guid, sizeof(sport->port_guid),
-		 "%04x:%04x:%04x:%04x",
-		 be16_to_cpu(guid[0]), be16_to_cpu(guid[1]),
-		 be16_to_cpu(guid[2]), be16_to_cpu(guid[3]));
+	srpt_format_guid(sport->port_guid, sizeof(sport->port_guid),
+			 &sport->gid.global.interface_id);
 	sport->port_gid_wwn.priv = sport;
 	snprintf(sport->port_gid, sizeof(sport->port_gid),
 		 "0x%016llx%016llx",
@@ -577,7 +615,8 @@ static int srpt_refresh_port(struct srpt_port *sport)
 }
 
 /**
- * srpt_unregister_mad_agent() - Unregister MAD callback functions.
+ * srpt_unregister_mad_agent - unregister MAD callback functions
+ * @sdev: SRPT HCA pointer.
  *
  * Note: It is safe to call this function more than once for the same device.
  */
@@ -602,7 +641,11 @@ static void srpt_unregister_mad_agent(struct srpt_device *sdev)
 }
 
 /**
- * srpt_alloc_ioctx() - Allocate an SRPT I/O context structure.
+ * srpt_alloc_ioctx - allocate a SRPT I/O context structure
+ * @sdev: SRPT HCA pointer.
+ * @ioctx_size: I/O context size.
+ * @dma_size: Size of I/O context DMA buffer.
+ * @dir: DMA data direction.
  */
 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
 					   int ioctx_size, int dma_size,
@@ -633,7 +676,11 @@ static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
 }
 
 /**
- * srpt_free_ioctx() - Free an SRPT I/O context structure.
+ * srpt_free_ioctx - free a SRPT I/O context structure
+ * @sdev: SRPT HCA pointer.
+ * @ioctx: I/O context pointer.
+ * @dma_size: Size of I/O context DMA buffer.
+ * @dir: DMA data direction.
  */
 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
 			    int dma_size, enum dma_data_direction dir)
@@ -647,7 +694,7 @@ static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
 }
 
 /**
- * srpt_alloc_ioctx_ring() - Allocate a ring of SRPT I/O context structures.
+ * srpt_alloc_ioctx_ring - allocate a ring of SRPT I/O context structures
  * @sdev:       Device to allocate the I/O context ring for.
  * @ring_size:  Number of elements in the I/O context ring.
  * @ioctx_size: I/O context size.
@@ -685,7 +732,12 @@ static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
 }
 
 /**
- * srpt_free_ioctx_ring() - Free the ring of SRPT I/O context structures.
+ * srpt_free_ioctx_ring - free the ring of SRPT I/O context structures
+ * @ioctx_ring: I/O context ring to be freed.
+ * @sdev: SRPT HCA pointer.
+ * @ring_size: Number of ring elements.
+ * @dma_size: Size of I/O context DMA buffer.
+ * @dir: DMA data direction.
  */
 static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
 				 struct srpt_device *sdev, int ring_size,
@@ -702,23 +754,9 @@ static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
 }
 
 /**
- * srpt_get_cmd_state() - Get the state of a SCSI command.
- */
-static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)
-{
-	enum srpt_command_state state;
-	unsigned long flags;
-
-	BUG_ON(!ioctx);
-
-	spin_lock_irqsave(&ioctx->spinlock, flags);
-	state = ioctx->state;
-	spin_unlock_irqrestore(&ioctx->spinlock, flags);
-	return state;
-}
-
-/**
- * srpt_set_cmd_state() - Set the state of a SCSI command.
+ * srpt_set_cmd_state - set the state of a SCSI command
+ * @ioctx: Send I/O context.
+ * @new: New I/O context state.
  *
  * Does not modify the state of aborted commands. Returns the previous command
  * state.
@@ -727,21 +765,19 @@ static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
 						  enum srpt_command_state new)
 {
 	enum srpt_command_state previous;
-	unsigned long flags;
 
-	BUG_ON(!ioctx);
-
-	spin_lock_irqsave(&ioctx->spinlock, flags);
 	previous = ioctx->state;
 	if (previous != SRPT_STATE_DONE)
 		ioctx->state = new;
-	spin_unlock_irqrestore(&ioctx->spinlock, flags);
 
 	return previous;
 }
 
 /**
- * srpt_test_and_set_cmd_state() - Test and set the state of a command.
+ * srpt_test_and_set_cmd_state - test and set the state of a command
+ * @ioctx: Send I/O context.
+ * @old: Current I/O context state.
+ * @new: New I/O context state.
  *
  * Returns true if and only if the previous command state was equal to 'old'.
  */
@@ -750,22 +786,23 @@ static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
 					enum srpt_command_state new)
 {
 	enum srpt_command_state previous;
-	unsigned long flags;
 
 	WARN_ON(!ioctx);
 	WARN_ON(old == SRPT_STATE_DONE);
 	WARN_ON(new == SRPT_STATE_NEW);
 
-	spin_lock_irqsave(&ioctx->spinlock, flags);
 	previous = ioctx->state;
 	if (previous == old)
 		ioctx->state = new;
-	spin_unlock_irqrestore(&ioctx->spinlock, flags);
+
 	return previous == old;
 }
 
 /**
- * srpt_post_recv() - Post an IB receive request.
+ * srpt_post_recv - post an IB receive request
+ * @sdev: SRPT HCA pointer.
+ * @ch: SRPT RDMA channel.
+ * @ioctx: Receive I/O context pointer.
  */
 static int srpt_post_recv(struct srpt_device *sdev, struct srpt_rdma_ch *ch,
 			  struct srpt_recv_ioctx *ioctx)
@@ -791,7 +828,8 @@ static int srpt_post_recv(struct srpt_device *sdev, struct srpt_rdma_ch *ch,
 }
 
 /**
- * srpt_zerolength_write() - Perform a zero-length RDMA write.
+ * srpt_zerolength_write - perform a zero-length RDMA write
+ * @ch: SRPT RDMA channel.
  *
  * A quote from the InfiniBand specification: C9-88: For an HCA responder
  * using Reliable Connection service, for each zero-length RDMA READ or WRITE
@@ -802,6 +840,9 @@ static int srpt_zerolength_write(struct srpt_rdma_ch *ch)
 {
 	struct ib_send_wr wr, *bad_wr;
 
+	pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
+		 ch->qp->qp_num);
+
 	memset(&wr, 0, sizeof(wr));
 	wr.opcode = IB_WR_RDMA_WRITE;
 	wr.wr_cqe = &ch->zw_cqe;
@@ -813,13 +854,17 @@ static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
 {
 	struct srpt_rdma_ch *ch = cq->cq_context;
 
+	pr_debug("%s-%d wc->status %d\n", ch->sess_name, ch->qp->qp_num,
+		 wc->status);
+
 	if (wc->status == IB_WC_SUCCESS) {
 		srpt_process_wait_list(ch);
 	} else {
 		if (srpt_set_ch_state(ch, CH_DISCONNECTED))
 			schedule_work(&ch->release_work);
 		else
-			WARN_ONCE(1, "%s-%d\n", ch->sess_name, ch->qp->qp_num);
+			pr_debug("%s-%d: already disconnected.\n",
+				 ch->sess_name, ch->qp->qp_num);
 	}
 }
 
@@ -928,11 +973,13 @@ static inline void *srpt_get_desc_buf(struct srp_cmd *srp_cmd)
 }
 
 /**
- * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request.
+ * srpt_get_desc_tbl - parse the data descriptors of a SRP_CMD request
  * @ioctx: Pointer to the I/O context associated with the request.
  * @srp_cmd: Pointer to the SRP_CMD request data.
  * @dir: Pointer to the variable to which the transfer direction will be
  *   written.
+ * @sg: [out] scatterlist allocated for the parsed SRP_CMD.
+ * @sg_cnt: [out] length of @sg.
  * @data_len: Pointer to the variable to which the total data length of all
  *   descriptors in the SRP_CMD request will be written.
  *
@@ -998,7 +1045,9 @@ static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
 }
 
 /**
- * srpt_init_ch_qp() - Initialize queue pair attributes.
+ * srpt_init_ch_qp - initialize queue pair attributes
+ * @ch: SRPT RDMA channel.
+ * @qp: Queue pair pointer.
  *
  * Initialized the attributes of queue pair 'qp' by allowing local write,
  * remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT.
@@ -1015,7 +1064,12 @@ static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 	attr->qp_state = IB_QPS_INIT;
 	attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE;
 	attr->port_num = ch->sport->port;
-	attr->pkey_index = 0;
+
+	ret = ib_find_cached_pkey(ch->sport->sdev->device, ch->sport->port,
+				  ch->pkey, &attr->pkey_index);
+	if (ret < 0)
+		pr_err("Translating pkey %#x failed (%d) - using index 0\n",
+		       ch->pkey, ret);
 
 	ret = ib_modify_qp(qp, attr,
 			   IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
@@ -1026,7 +1080,7 @@ static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 }
 
 /**
- * srpt_ch_qp_rtr() - Change the state of a channel to 'ready to receive' (RTR).
+ * srpt_ch_qp_rtr - change the state of a channel to 'ready to receive' (RTR)
  * @ch: channel of the queue pair.
  * @qp: queue pair to change the state of.
  *
@@ -1043,7 +1097,7 @@ static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 	int ret;
 
 	qp_attr.qp_state = IB_QPS_RTR;
-	ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
+	ret = ib_cm_init_qp_attr(ch->ib_cm.cm_id, &qp_attr, &attr_mask);
 	if (ret)
 		goto out;
 
@@ -1056,7 +1110,7 @@ static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 }
 
 /**
- * srpt_ch_qp_rts() - Change the state of a channel to 'ready to send' (RTS).
+ * srpt_ch_qp_rts - change the state of a channel to 'ready to send' (RTS)
  * @ch: channel of the queue pair.
  * @qp: queue pair to change the state of.
  *
@@ -1073,7 +1127,7 @@ static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 	int ret;
 
 	qp_attr.qp_state = IB_QPS_RTS;
-	ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
+	ret = ib_cm_init_qp_attr(ch->ib_cm.cm_id, &qp_attr, &attr_mask);
 	if (ret)
 		goto out;
 
@@ -1086,7 +1140,8 @@ static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
 }
 
 /**
- * srpt_ch_qp_err() - Set the channel queue pair state to 'error'.
+ * srpt_ch_qp_err - set the channel queue pair state to 'error'
+ * @ch: SRPT RDMA channel.
  */
 static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
 {
@@ -1097,7 +1152,8 @@ static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
 }
 
 /**
- * srpt_get_send_ioctx() - Obtain an I/O context for sending to the initiator.
+ * srpt_get_send_ioctx - obtain an I/O context for sending to the initiator
+ * @ch: SRPT RDMA channel.
  */
 static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
 {
@@ -1119,11 +1175,9 @@ static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
 		return ioctx;
 
 	BUG_ON(ioctx->ch != ch);
-	spin_lock_init(&ioctx->spinlock);
 	ioctx->state = SRPT_STATE_NEW;
 	ioctx->n_rdma = 0;
 	ioctx->n_rw_ctx = 0;
-	init_completion(&ioctx->tx_done);
 	ioctx->queue_status_only = false;
 	/*
 	 * transport_init_se_cmd() does not initialize all fields, so do it
@@ -1136,14 +1190,12 @@ static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
 }
 
 /**
- * srpt_abort_cmd() - Abort a SCSI command.
+ * srpt_abort_cmd - abort a SCSI command
  * @ioctx:   I/O context associated with the SCSI command.
- * @context: Preferred execution context.
  */
 static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
 {
 	enum srpt_command_state state;
-	unsigned long flags;
 
 	BUG_ON(!ioctx);
 
@@ -1152,7 +1204,6 @@ static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
 	 * the ib_srpt driver, change the state to the next state.
 	 */
 
-	spin_lock_irqsave(&ioctx->spinlock, flags);
 	state = ioctx->state;
 	switch (state) {
 	case SRPT_STATE_NEED_DATA:
@@ -1167,7 +1218,6 @@ static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
 			  __func__, state);
 		break;
 	}
-	spin_unlock_irqrestore(&ioctx->spinlock, flags);
 
 	pr_debug("Aborting cmd with state %d -> %d and tag %lld\n", state,
 		 ioctx->state, ioctx->cmd.tag);
@@ -1206,6 +1256,10 @@ static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
 }
 
 /**
+ * srpt_rdma_read_done - RDMA read completion callback
+ * @cq: Completion queue.
+ * @wc: Work completion.
+ *
  * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
  * the data that has been transferred via IB RDMA had to be postponed until the
  * check_stop_free() callback.  None of this is necessary anymore and needs to
@@ -1233,11 +1287,11 @@ static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
 		target_execute_cmd(&ioctx->cmd);
 	else
 		pr_err("%s[%d]: wrong state = %d\n", __func__,
-		       __LINE__, srpt_get_cmd_state(ioctx));
+		       __LINE__, ioctx->state);
 }
 
 /**
- * srpt_build_cmd_rsp() - Build an SRP_RSP response.
+ * srpt_build_cmd_rsp - build a SRP_RSP response
  * @ch: RDMA channel through which the request has been received.
  * @ioctx: I/O context associated with the SRP_CMD request. The response will
  *   be built in the buffer ioctx->buf points at and hence this function will
@@ -1297,7 +1351,7 @@ static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
 }
 
 /**
- * srpt_build_tskmgmt_rsp() - Build a task management response.
+ * srpt_build_tskmgmt_rsp - build a task management response
  * @ch:       RDMA channel through which the request has been received.
  * @ioctx:    I/O context in which the SRP_RSP response will be built.
  * @rsp_code: RSP_CODE that will be stored in the response.
@@ -1345,7 +1399,10 @@ static int srpt_check_stop_free(struct se_cmd *cmd)
 }
 
 /**
- * srpt_handle_cmd() - Process SRP_CMD.
+ * srpt_handle_cmd - process a SRP_CMD information unit
+ * @ch: SRPT RDMA channel.
+ * @recv_ioctx: Receive I/O context.
+ * @send_ioctx: Send I/O context.
  */
 static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
 			    struct srpt_recv_ioctx *recv_ioctx,
@@ -1427,7 +1484,10 @@ static int srp_tmr_to_tcm(int fn)
 }
 
 /**
- * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit.
+ * srpt_handle_tsk_mgmt - process a SRP_TSK_MGMT information unit
+ * @ch: SRPT RDMA channel.
+ * @recv_ioctx: Receive I/O context.
+ * @send_ioctx: Send I/O context.
  *
  * Returns 0 if and only if the request will be processed by the target core.
  *
@@ -1449,9 +1509,9 @@ static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
 	srp_tsk = recv_ioctx->ioctx.buf;
 	cmd = &send_ioctx->cmd;
 
-	pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld"
-		 " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func,
-		 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess);
+	pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld ch %p sess %p\n",
+		 srp_tsk->tsk_mgmt_func, srp_tsk->task_tag, srp_tsk->tag, ch,
+		 ch->sess);
 
 	srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
 	send_ioctx->cmd.tag = srp_tsk->tag;
@@ -1470,41 +1530,42 @@ static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
 }
 
 /**
- * srpt_handle_new_iu() - Process a newly received information unit.
+ * srpt_handle_new_iu - process a newly received information unit
  * @ch:    RDMA channel through which the information unit has been received.
- * @ioctx: SRPT I/O context associated with the information unit.
+ * @recv_ioctx: Receive I/O context associated with the information unit.
  */
-static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
-			       struct srpt_recv_ioctx *recv_ioctx,
-			       struct srpt_send_ioctx *send_ioctx)
+static bool
+srpt_handle_new_iu(struct srpt_rdma_ch *ch, struct srpt_recv_ioctx *recv_ioctx)
 {
+	struct srpt_send_ioctx *send_ioctx = NULL;
 	struct srp_cmd *srp_cmd;
+	bool res = false;
+	u8 opcode;
 
 	BUG_ON(!ch);
 	BUG_ON(!recv_ioctx);
 
+	if (unlikely(ch->state == CH_CONNECTING))
+		goto push;
+
 	ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
 				   recv_ioctx->ioctx.dma, srp_max_req_size,
 				   DMA_FROM_DEVICE);
 
-	if (unlikely(ch->state == CH_CONNECTING))
-		goto out_wait;
-
-	if (unlikely(ch->state != CH_LIVE))
-		return;
-
 	srp_cmd = recv_ioctx->ioctx.buf;
-	if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) {
-		if (!send_ioctx) {
-			if (!list_empty(&ch->cmd_wait_list))
-				goto out_wait;
-			send_ioctx = srpt_get_send_ioctx(ch);
-		}
+	opcode = srp_cmd->opcode;
+	if (opcode == SRP_CMD || opcode == SRP_TSK_MGMT) {
+		send_ioctx = srpt_get_send_ioctx(ch);
 		if (unlikely(!send_ioctx))
-			goto out_wait;
+			goto push;
 	}
 
-	switch (srp_cmd->opcode) {
+	if (!list_empty(&recv_ioctx->wait_list)) {
+		WARN_ON_ONCE(!ch->processing_wait_list);
+		list_del_init(&recv_ioctx->wait_list);
+	}
+
+	switch (opcode) {
 	case SRP_CMD:
 		srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
 		break;
@@ -1524,16 +1585,22 @@ static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
 		pr_err("Received SRP_RSP\n");
 		break;
 	default:
-		pr_err("received IU with unknown opcode 0x%x\n",
-		       srp_cmd->opcode);
+		pr_err("received IU with unknown opcode 0x%x\n", opcode);
 		break;
 	}
 
 	srpt_post_recv(ch->sport->sdev, ch, recv_ioctx);
-	return;
+	res = true;
+
+out:
+	return res;
 
-out_wait:
-	list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
+push:
+	if (list_empty(&recv_ioctx->wait_list)) {
+		WARN_ON_ONCE(ch->processing_wait_list);
+		list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
+	}
+	goto out;
 }
 
 static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
@@ -1548,10 +1615,10 @@ static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 		req_lim = atomic_dec_return(&ch->req_lim);
 		if (unlikely(req_lim < 0))
 			pr_err("req_lim = %d < 0\n", req_lim);
-		srpt_handle_new_iu(ch, ioctx, NULL);
+		srpt_handle_new_iu(ch, ioctx);
 	} else {
-		pr_info("receiving failed for ioctx %p with status %d\n",
-			ioctx, wc->status);
+		pr_info_ratelimited("receiving failed for ioctx %p with status %d\n",
+				    ioctx, wc->status);
 	}
 }
 
@@ -1562,22 +1629,28 @@ static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
  */
 static void srpt_process_wait_list(struct srpt_rdma_ch *ch)
 {
-	struct srpt_send_ioctx *ioctx;
+	struct srpt_recv_ioctx *recv_ioctx, *tmp;
+
+	WARN_ON_ONCE(ch->state == CH_CONNECTING);
 
-	while (!list_empty(&ch->cmd_wait_list) &&
-	       ch->state >= CH_LIVE &&
-	       (ioctx = srpt_get_send_ioctx(ch)) != NULL) {
-		struct srpt_recv_ioctx *recv_ioctx;
+	if (list_empty(&ch->cmd_wait_list))
+		return;
 
-		recv_ioctx = list_first_entry(&ch->cmd_wait_list,
-					      struct srpt_recv_ioctx,
-					      wait_list);
-		list_del(&recv_ioctx->wait_list);
-		srpt_handle_new_iu(ch, recv_ioctx, ioctx);
+	WARN_ON_ONCE(ch->processing_wait_list);
+	ch->processing_wait_list = true;
+	list_for_each_entry_safe(recv_ioctx, tmp, &ch->cmd_wait_list,
+				 wait_list) {
+		if (!srpt_handle_new_iu(ch, recv_ioctx))
+			break;
 	}
+	ch->processing_wait_list = false;
 }
 
 /**
+ * srpt_send_done - send completion callback
+ * @cq: Completion queue.
+ * @wc: Work completion.
+ *
  * Note: Although this has not yet been observed during tests, at least in
  * theory it is possible that the srpt_get_send_ioctx() call invoked by
  * srpt_handle_new_iu() fails. This is possible because the req_lim_delta
@@ -1619,7 +1692,8 @@ static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
 }
 
 /**
- * srpt_create_ch_ib() - Create receive and send completion queues.
+ * srpt_create_ch_ib - create receive and send completion queues
+ * @ch: SRPT RDMA channel.
  */
 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 {
@@ -1627,7 +1701,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	struct srpt_port *sport = ch->sport;
 	struct srpt_device *sdev = sport->sdev;
 	const struct ib_device_attr *attrs = &sdev->device->attrs;
-	u32 srp_sq_size = sport->port_attrib.srp_sq_size;
+	int sq_size = sport->port_attrib.srp_sq_size;
 	int i, ret;
 
 	WARN_ON(ch->rq_size < 1);
@@ -1638,12 +1712,12 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 		goto out;
 
 retry:
-	ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + srp_sq_size,
+	ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + sq_size,
 			0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
 	if (IS_ERR(ch->cq)) {
 		ret = PTR_ERR(ch->cq);
 		pr_err("failed to create CQ cqe= %d ret= %d\n",
-		       ch->rq_size + srp_sq_size, ret);
+		       ch->rq_size + sq_size, ret);
 		goto out;
 	}
 
@@ -1661,8 +1735,8 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	 * both both, as RDMA contexts will also post completions for the
 	 * RDMA READ case.
 	 */
-	qp_init->cap.max_send_wr = min(srp_sq_size / 2, attrs->max_qp_wr + 0U);
-	qp_init->cap.max_rdma_ctxs = srp_sq_size / 2;
+	qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr);
+	qp_init->cap.max_rdma_ctxs = sq_size / 2;
 	qp_init->cap.max_send_sge = min(attrs->max_sge, SRPT_MAX_SG_PER_WQE);
 	qp_init->port_num = ch->sport->port;
 	if (sdev->use_srq) {
@@ -1676,8 +1750,8 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 	if (IS_ERR(ch->qp)) {
 		ret = PTR_ERR(ch->qp);
 		if (ret == -ENOMEM) {
-			srp_sq_size /= 2;
-			if (srp_sq_size >= MIN_SRPT_SQ_SIZE) {
+			sq_size /= 2;
+			if (sq_size >= MIN_SRPT_SQ_SIZE) {
 				ib_destroy_cq(ch->cq);
 				goto retry;
 			}
@@ -1688,9 +1762,9 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
 
 	atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
 
-	pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
+	pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d ch= %p\n",
 		 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
-		 qp_init->cap.max_send_wr, ch->cm_id);
+		 qp_init->cap.max_send_wr, ch);
 
 	ret = srpt_init_ch_qp(ch, ch->qp);
 	if (ret)
@@ -1718,7 +1792,8 @@ static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
 }
 
 /**
- * srpt_close_ch() - Close an RDMA channel.
+ * srpt_close_ch - close a RDMA channel
+ * @ch: SRPT RDMA channel.
  *
  * Make sure all resources associated with the channel will be deallocated at
  * an appropriate time.
@@ -1743,8 +1818,6 @@ static bool srpt_close_ch(struct srpt_rdma_ch *ch)
 		pr_err("%s-%d: changing queue pair into error state failed: %d\n",
 		       ch->sess_name, ch->qp->qp_num, ret);
 
-	pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
-		 ch->qp->qp_num);
 	ret = srpt_zerolength_write(ch);
 	if (ret < 0) {
 		pr_err("%s-%d: queuing zero-length write failed: %d\n",
@@ -1776,9 +1849,9 @@ static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
 	if (!srpt_set_ch_state(ch, CH_DISCONNECTING))
 		return -ENOTCONN;
 
-	ret = ib_send_cm_dreq(ch->cm_id, NULL, 0);
+	ret = ib_send_cm_dreq(ch->ib_cm.cm_id, NULL, 0);
 	if (ret < 0)
-		ret = ib_send_cm_drep(ch->cm_id, NULL, 0);
+		ret = ib_send_cm_drep(ch->ib_cm.cm_id, NULL, 0);
 
 	if (ret < 0 && srpt_close_ch(ch))
 		ret = 0;
@@ -1786,83 +1859,135 @@ static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
 	return ret;
 }
 
-/*
- * Send DREQ and wait for DREP. Return true if and only if this function
- * changed the state of @ch.
- */
-static bool srpt_disconnect_ch_sync(struct srpt_rdma_ch *ch)
-	__must_hold(&sdev->mutex)
+static bool srpt_ch_closed(struct srpt_port *sport, struct srpt_rdma_ch *ch)
 {
-	DECLARE_COMPLETION_ONSTACK(release_done);
-	struct srpt_device *sdev = ch->sport->sdev;
-	bool wait;
+	struct srpt_nexus *nexus;
+	struct srpt_rdma_ch *ch2;
+	bool res = true;
+
+	rcu_read_lock();
+	list_for_each_entry(nexus, &sport->nexus_list, entry) {
+		list_for_each_entry(ch2, &nexus->ch_list, list) {
+			if (ch2 == ch) {
+				res = false;
+				goto done;
+			}
+		}
+	}
+done:
+	rcu_read_unlock();
 
-	lockdep_assert_held(&sdev->mutex);
+	return res;
+}
+
+/* Send DREQ and wait for DREP. */
+static void srpt_disconnect_ch_sync(struct srpt_rdma_ch *ch)
+{
+	struct srpt_port *sport = ch->sport;
 
 	pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
 		 ch->state);
 
-	WARN_ON(ch->release_done);
-	ch->release_done = &release_done;
-	wait = !list_empty(&ch->list);
+	mutex_lock(&sport->mutex);
 	srpt_disconnect_ch(ch);
-	mutex_unlock(&sdev->mutex);
+	mutex_unlock(&sport->mutex);
 
-	if (!wait)
-		goto out;
-
-	while (wait_for_completion_timeout(&release_done, 180 * HZ) == 0)
+	while (wait_event_timeout(sport->ch_releaseQ, srpt_ch_closed(sport, ch),
+				  5 * HZ) == 0)
 		pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
 			ch->sess_name, ch->qp->qp_num, ch->state);
 
-out:
-	mutex_lock(&sdev->mutex);
-	return wait;
 }
 
-static void srpt_set_enabled(struct srpt_port *sport, bool enabled)
-	__must_hold(&sdev->mutex)
+static void __srpt_close_all_ch(struct srpt_port *sport)
 {
-	struct srpt_device *sdev = sport->sdev;
+	struct srpt_nexus *nexus;
 	struct srpt_rdma_ch *ch;
 
-	lockdep_assert_held(&sdev->mutex);
+	lockdep_assert_held(&sport->mutex);
 
-	if (sport->enabled == enabled)
-		return;
-	sport->enabled = enabled;
-	if (sport->enabled)
-		return;
+	list_for_each_entry(nexus, &sport->nexus_list, entry) {
+		list_for_each_entry(ch, &nexus->ch_list, list) {
+			if (srpt_disconnect_ch(ch) >= 0)
+				pr_info("Closing channel %s-%d because target %s_%d has been disabled\n",
+					ch->sess_name, ch->qp->qp_num,
+					sport->sdev->device->name, sport->port);
+			srpt_close_ch(ch);
+		}
+	}
+}
 
-again:
-	list_for_each_entry(ch, &sdev->rch_list, list) {
-		if (ch->sport == sport) {
-			pr_info("%s: closing channel %s-%d\n",
-				sdev->device->name, ch->sess_name,
-				ch->qp->qp_num);
-			if (srpt_disconnect_ch_sync(ch))
-				goto again;
+/*
+ * Look up (i_port_id, t_port_id) in sport->nexus_list. Create an entry if
+ * it does not yet exist.
+ */
+static struct srpt_nexus *srpt_get_nexus(struct srpt_port *sport,
+					 const u8 i_port_id[16],
+					 const u8 t_port_id[16])
+{
+	struct srpt_nexus *nexus = NULL, *tmp_nexus = NULL, *n;
+
+	for (;;) {
+		mutex_lock(&sport->mutex);
+		list_for_each_entry(n, &sport->nexus_list, entry) {
+			if (memcmp(n->i_port_id, i_port_id, 16) == 0 &&
+			    memcmp(n->t_port_id, t_port_id, 16) == 0) {
+				nexus = n;
+				break;
+			}
 		}
+		if (!nexus && tmp_nexus) {
+			list_add_tail_rcu(&tmp_nexus->entry,
+					  &sport->nexus_list);
+			swap(nexus, tmp_nexus);
+		}
+		mutex_unlock(&sport->mutex);
+
+		if (nexus)
+			break;
+		tmp_nexus = kzalloc(sizeof(*nexus), GFP_KERNEL);
+		if (!tmp_nexus) {
+			nexus = ERR_PTR(-ENOMEM);
+			break;
+		}
+		INIT_LIST_HEAD(&tmp_nexus->ch_list);
+		memcpy(tmp_nexus->i_port_id, i_port_id, 16);
+		memcpy(tmp_nexus->t_port_id, t_port_id, 16);
 	}
 
+	kfree(tmp_nexus);
+
+	return nexus;
+}
+
+static void srpt_set_enabled(struct srpt_port *sport, bool enabled)
+	__must_hold(&sport->mutex)
+{
+	lockdep_assert_held(&sport->mutex);
+
+	if (sport->enabled == enabled)
+		return;
+	sport->enabled = enabled;
+	if (!enabled)
+		__srpt_close_all_ch(sport);
 }
 
 static void srpt_free_ch(struct kref *kref)
 {
 	struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
 
-	kfree(ch);
+	kfree_rcu(ch, rcu);
 }
 
 static void srpt_release_channel_work(struct work_struct *w)
 {
 	struct srpt_rdma_ch *ch;
 	struct srpt_device *sdev;
+	struct srpt_port *sport;
 	struct se_session *se_sess;
 
 	ch = container_of(w, struct srpt_rdma_ch, release_work);
-	pr_debug("%s: %s-%d; release_done = %p\n", __func__, ch->sess_name,
-		 ch->qp->qp_num, ch->release_done);
+	pr_debug("%s-%d\n", ch->sess_name, ch->qp->qp_num);
 
 	sdev = ch->sport->sdev;
 	BUG_ON(!sdev);
@@ -1877,169 +2002,141 @@ static void srpt_release_channel_work(struct work_struct *w)
 	transport_deregister_session(se_sess);
 	ch->sess = NULL;
 
-	ib_destroy_cm_id(ch->cm_id);
+	ib_destroy_cm_id(ch->ib_cm.cm_id);
 
 	srpt_destroy_ch_ib(ch);
 
 	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
 			     ch->sport->sdev, ch->rq_size,
-			     ch->rsp_size, DMA_TO_DEVICE);
+			     ch->max_rsp_size, DMA_TO_DEVICE);
 
 	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_recv_ring,
 			     sdev, ch->rq_size,
 			     srp_max_req_size, DMA_FROM_DEVICE);
 
-	mutex_lock(&sdev->mutex);
-	list_del_init(&ch->list);
-	if (ch->release_done)
-		complete(ch->release_done);
-	mutex_unlock(&sdev->mutex);
+	sport = ch->sport;
+	mutex_lock(&sport->mutex);
+	list_del_rcu(&ch->list);
+	mutex_unlock(&sport->mutex);
 
-	wake_up(&sdev->ch_releaseQ);
+	wake_up(&sport->ch_releaseQ);
 
 	kref_put(&ch->kref, srpt_free_ch);
 }
 
 /**
- * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED.
+ * srpt_cm_req_recv - process the event IB_CM_REQ_RECEIVED
+ * @cm_id: IB/CM connection identifier.
+ * @port_num: Port through which the IB/CM REQ message was received.
+ * @pkey: P_Key of the incoming connection.
+ * @req: SRP login request.
+ * @src_addr: GID of the port that submitted the login request.
  *
  * Ownership of the cm_id is transferred to the target session if this
  * functions returns zero. Otherwise the caller remains the owner of cm_id.
  */
 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
-			    struct ib_cm_req_event_param *param,
-			    void *private_data)
+			    u8 port_num, __be16 pkey,
+			    const struct srp_login_req *req,
+			    const char *src_addr)
 {
 	struct srpt_device *sdev = cm_id->context;
-	struct srpt_port *sport = &sdev->port[param->port - 1];
-	struct srp_login_req *req;
-	struct srp_login_rsp *rsp;
-	struct srp_login_rej *rej;
-	struct ib_cm_rep_param *rep_param;
-	struct srpt_rdma_ch *ch, *tmp_ch;
-	__be16 *guid;
+	struct srpt_port *sport = &sdev->port[port_num - 1];
+	struct srpt_nexus *nexus;
+	struct srp_login_rsp *rsp = NULL;
+	struct srp_login_rej *rej = NULL;
+	struct ib_cm_rep_param *rep_param = NULL;
+	struct srpt_rdma_ch *ch;
+	char i_port_id[36];
 	u32 it_iu_len;
-	int i, ret = 0;
+	int i, ret;
 
 	WARN_ON_ONCE(irqs_disabled());
 
-	if (WARN_ON(!sdev || !private_data))
+	if (WARN_ON(!sdev || !req))
 		return -EINVAL;
 
-	req = (struct srp_login_req *)private_data;
-
 	it_iu_len = be32_to_cpu(req->req_it_iu_len);
 
-	pr_info("Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx,"
-		" t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d"
-		" (guid=0x%llx:0x%llx)\n",
-		be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]),
-		be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]),
-		be64_to_cpu(*(__be64 *)&req->target_port_id[0]),
-		be64_to_cpu(*(__be64 *)&req->target_port_id[8]),
-		it_iu_len,
-		param->port,
-		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),
-		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));
+	pr_info("Received SRP_LOGIN_REQ with i_port_id %pI6, t_port_id %pI6 and it_iu_len %d on port %d (guid=%pI6); pkey %#04x\n",
+		req->initiator_port_id, req->target_port_id, it_iu_len,
+		port_num, &sport->gid, be16_to_cpu(pkey));
 
+	nexus = srpt_get_nexus(sport, req->initiator_port_id,
+			       req->target_port_id);
+	if (IS_ERR(nexus)) {
+		ret = PTR_ERR(nexus);
+		goto out;
+	}
+
+	ret = -ENOMEM;
 	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
 	rej = kzalloc(sizeof(*rej), GFP_KERNEL);
 	rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
-
-	if (!rsp || !rej || !rep_param) {
-		ret = -ENOMEM;
+	if (!rsp || !rej || !rep_param)
 		goto out;
-	}
 
+	ret = -EINVAL;
 	if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
 		rej->reason = cpu_to_be32(
-			      SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
-		ret = -EINVAL;
-		pr_err("rejected SRP_LOGIN_REQ because its"
-		       " length (%d bytes) is out of range (%d .. %d)\n",
+				SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
+		pr_err("rejected SRP_LOGIN_REQ because its length (%d bytes) is out of range (%d .. %d)\n",
 		       it_iu_len, 64, srp_max_req_size);
 		goto reject;
 	}
 
 	if (!sport->enabled) {
-		rej->reason = cpu_to_be32(
-			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
-		ret = -EINVAL;
-		pr_err("rejected SRP_LOGIN_REQ because the target port"
-		       " has not yet been enabled\n");
+		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
+		pr_info("rejected SRP_LOGIN_REQ because target port %s_%d has not yet been enabled\n",
+			sport->sdev->device->name, port_num);
 		goto reject;
 	}
 
-	if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
-		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
-
-		mutex_lock(&sdev->mutex);
-
-		list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
-			if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
-			    && !memcmp(ch->t_port_id, req->target_port_id, 16)
-			    && param->port == ch->sport->port
-			    && param->listen_id == ch->sport->sdev->cm_id
-			    && ch->cm_id) {
-				if (srpt_disconnect_ch(ch) < 0)
-					continue;
-				pr_info("Relogin - closed existing channel %s\n",
-					ch->sess_name);
-				rsp->rsp_flags =
-					SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
-			}
-		}
-
-		mutex_unlock(&sdev->mutex);
-
-	} else
-		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
-
 	if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
 	    || *(__be64 *)(req->target_port_id + 8) !=
 	       cpu_to_be64(srpt_service_guid)) {
 		rej->reason = cpu_to_be32(
-			      SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
-		ret = -ENOMEM;
-		pr_err("rejected SRP_LOGIN_REQ because it"
-		       " has an invalid target port identifier.\n");
+				SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
+		pr_err("rejected SRP_LOGIN_REQ because it has an invalid target port identifier.\n");
 		goto reject;
 	}
 
+	ret = -ENOMEM;
 	ch = kzalloc(sizeof(*ch), GFP_KERNEL);
 	if (!ch) {
-		rej->reason = cpu_to_be32(
-			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
-		pr_err("rejected SRP_LOGIN_REQ because no memory.\n");
-		ret = -ENOMEM;
+		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
+		pr_err("rejected SRP_LOGIN_REQ because out of memory.\n");
 		goto reject;
 	}
 
 	kref_init(&ch->kref);
+	ch->pkey = be16_to_cpu(pkey);
+	ch->nexus = nexus;
 	ch->zw_cqe.done = srpt_zerolength_write_done;
 	INIT_WORK(&ch->release_work, srpt_release_channel_work);
-	memcpy(ch->i_port_id, req->initiator_port_id, 16);
-	memcpy(ch->t_port_id, req->target_port_id, 16);
-	ch->sport = &sdev->port[param->port - 1];
-	ch->cm_id = cm_id;
+	ch->sport = sport;
+	ch->ib_cm.cm_id = cm_id;
 	cm_id->context = ch;
 	/*
 	 * ch->rq_size should be at least as large as the initiator queue
 	 * depth to avoid that the initiator driver has to report QUEUE_FULL
 	 * to the SCSI mid-layer.
 	 */
-	ch->rq_size = min(SRPT_RQ_SIZE, sdev->device->attrs.max_qp_wr);
+	ch->rq_size = min(MAX_SRPT_RQ_SIZE, sdev->device->attrs.max_qp_wr);
 	spin_lock_init(&ch->spinlock);
 	ch->state = CH_CONNECTING;
 	INIT_LIST_HEAD(&ch->cmd_wait_list);
-	ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
+	ch->max_rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
 
 	ch->ioctx_ring = (struct srpt_send_ioctx **)
 		srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
 				      sizeof(*ch->ioctx_ring[0]),
-				      ch->rsp_size, DMA_TO_DEVICE);
-	if (!ch->ioctx_ring)
+				      ch->max_rsp_size, DMA_TO_DEVICE);
+	if (!ch->ioctx_ring) {
+		pr_err("rejected SRP_LOGIN_REQ because creating a new QP SQ ring failed.\n");
+		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
 		goto free_ch;
+	}
 
 	INIT_LIST_HEAD(&ch->free_list);
 	for (i = 0; i < ch->rq_size; i++) {
@@ -2058,59 +2155,88 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 			    cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
 			goto free_ring;
 		}
+		for (i = 0; i < ch->rq_size; i++)
+			INIT_LIST_HEAD(&ch->ioctx_recv_ring[i]->wait_list);
 	}
 
 	ret = srpt_create_ch_ib(ch);
-	if (ret) {
-		rej->reason = cpu_to_be32(
-			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
-		pr_err("rejected SRP_LOGIN_REQ because creating"
-		       " a new RDMA channel failed.\n");
-		goto free_recv_ring;
-	}
-
-	ret = srpt_ch_qp_rtr(ch, ch->qp);
 	if (ret) {
 		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
-		pr_err("rejected SRP_LOGIN_REQ because enabling"
-		       " RTR failed (error code = %d)\n", ret);
-		goto destroy_ib;
+		pr_err("rejected SRP_LOGIN_REQ because creating a new RDMA channel failed.\n");
+		goto free_recv_ring;
 	}
 
-	guid = (__be16 *)&param->primary_path->dgid.global.interface_id;
-	snprintf(ch->ini_guid, sizeof(ch->ini_guid), "%04x:%04x:%04x:%04x",
-		 be16_to_cpu(guid[0]), be16_to_cpu(guid[1]),
-		 be16_to_cpu(guid[2]), be16_to_cpu(guid[3]));
-	snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
-			be64_to_cpu(*(__be64 *)ch->i_port_id),
-			be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
+	strlcpy(ch->sess_name, src_addr, sizeof(ch->sess_name));
+	snprintf(i_port_id, sizeof(i_port_id), "0x%016llx%016llx",
+			be64_to_cpu(*(__be64 *)nexus->i_port_id),
+			be64_to_cpu(*(__be64 *)(nexus->i_port_id + 8)));
 
 	pr_debug("registering session %s\n", ch->sess_name);
 
 	if (sport->port_guid_tpg.se_tpg_wwn)
 		ch->sess = target_alloc_session(&sport->port_guid_tpg, 0, 0,
 						TARGET_PROT_NORMAL,
-						ch->ini_guid, ch, NULL);
+						ch->sess_name, ch, NULL);
 	if (sport->port_gid_tpg.se_tpg_wwn && IS_ERR_OR_NULL(ch->sess))
 		ch->sess = target_alloc_session(&sport->port_gid_tpg, 0, 0,
-					TARGET_PROT_NORMAL, ch->sess_name, ch,
+					TARGET_PROT_NORMAL, i_port_id, ch,
 					NULL);
 	/* Retry without leading "0x" */
 	if (sport->port_gid_tpg.se_tpg_wwn && IS_ERR_OR_NULL(ch->sess))
 		ch->sess = target_alloc_session(&sport->port_gid_tpg, 0, 0,
 						TARGET_PROT_NORMAL,
-						ch->sess_name + 2, ch, NULL);
+						i_port_id + 2, ch, NULL);
 	if (IS_ERR_OR_NULL(ch->sess)) {
-		pr_info("Rejected login because no ACL has been configured yet for initiator %s.\n",
-			ch->sess_name);
-		rej->reason = cpu_to_be32((PTR_ERR(ch->sess) == -ENOMEM) ?
+		ret = PTR_ERR(ch->sess);
+		pr_info("Rejected login for initiator %s: ret = %d.\n",
+			ch->sess_name, ret);
+		rej->reason = cpu_to_be32(ret == -ENOMEM ?
 				SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
 				SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
+		goto reject;
+	}
+
+	mutex_lock(&sport->mutex);
+
+	if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
+		struct srpt_rdma_ch *ch2;
+
+		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
+
+		list_for_each_entry(ch2, &nexus->ch_list, list) {
+			if (srpt_disconnect_ch(ch2) < 0)
+				continue;
+			pr_info("Relogin - closed existing channel %s\n",
+				ch2->sess_name);
+			rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
+		}
+	} else {
+		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
+	}
+
+	list_add_tail_rcu(&ch->list, &nexus->ch_list);
+
+	if (!sport->enabled) {
+		rej->reason = cpu_to_be32(
+				SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
+		pr_info("rejected SRP_LOGIN_REQ because target %s_%d is not enabled\n",
+			sdev->device->name, port_num);
+		mutex_unlock(&sport->mutex);
+		goto reject;
+	}
+
+	mutex_unlock(&sport->mutex);
+
+	ret = srpt_ch_qp_rtr(ch, ch->qp);
+	if (ret) {
+		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
+		pr_err("rejected SRP_LOGIN_REQ because enabling RTR failed (error code = %d)\n",
+		       ret);
 		goto destroy_ib;
 	}
 
-	pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
-		 ch->sess_name, ch->cm_id);
+	pr_debug("Establish connection sess=%p name=%s ch=%p\n", ch->sess,
+		 ch->sess_name, ch);
 
 	/* create srp_login_response */
 	rsp->opcode = SRP_LOGIN_RSP;
@@ -2118,8 +2244,8 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 	rsp->max_it_iu_len = req->req_it_iu_len;
 	rsp->max_ti_iu_len = req->req_it_iu_len;
 	ch->max_ti_iu_len = it_iu_len;
-	rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
-				   | SRP_BUF_FORMAT_INDIRECT);
+	rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
+				   SRP_BUF_FORMAT_INDIRECT);
 	rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
 	atomic_set(&ch->req_lim, ch->rq_size);
 	atomic_set(&ch->req_lim_delta, 0);
@@ -2135,25 +2261,31 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 	rep_param->responder_resources = 4;
 	rep_param->initiator_depth = 4;
 
-	ret = ib_send_cm_rep(cm_id, rep_param);
-	if (ret) {
-		pr_err("sending SRP_LOGIN_REQ response failed"
-		       " (error code = %d)\n", ret);
-		goto release_channel;
-	}
+	/*
+	 * Hold the sport mutex while accepting a connection to avoid that
+	 * srpt_disconnect_ch() is invoked concurrently with this code.
+	 */
+	mutex_lock(&sport->mutex);
+	if (sport->enabled && ch->state == CH_CONNECTING)
+		ret = ib_send_cm_rep(cm_id, rep_param);
+	else
+		ret = -EINVAL;
+	mutex_unlock(&sport->mutex);
 
-	mutex_lock(&sdev->mutex);
-	list_add_tail(&ch->list, &sdev->rch_list);
-	mutex_unlock(&sdev->mutex);
+	switch (ret) {
+	case 0:
+		break;
+	case -EINVAL:
+		goto reject;
+	default:
+		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
+		pr_err("sending SRP_LOGIN_REQ response failed (error code = %d)\n",
+		       ret);
+		goto reject;
+	}
 
 	goto out;
 
-release_channel:
-	srpt_disconnect_ch(ch);
-	transport_deregister_session_configfs(ch->sess);
-	transport_deregister_session(ch->sess);
-	ch->sess = NULL;
-
 destroy_ib:
 	srpt_destroy_ch_ib(ch);
 
@@ -2165,15 +2297,20 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 free_ring:
 	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
 			     ch->sport->sdev, ch->rq_size,
-			     ch->rsp_size, DMA_TO_DEVICE);
+			     ch->max_rsp_size, DMA_TO_DEVICE);
 free_ch:
+	cm_id->context = NULL;
 	kfree(ch);
+	ch = NULL;
+
+	WARN_ON_ONCE(ret == 0);
 
 reject:
+	pr_info("Rejecting login with reason %#x\n", be32_to_cpu(rej->reason));
 	rej->opcode = SRP_LOGIN_REJ;
 	rej->tag = req->tag;
-	rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
-				   | SRP_BUF_FORMAT_INDIRECT);
+	rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
+				   SRP_BUF_FORMAT_INDIRECT);
 
 	ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
 			     (void *)rej, sizeof(*rej));
@@ -2186,6 +2323,19 @@ static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
 	return ret;
 }
 
+static int srpt_ib_cm_req_recv(struct ib_cm_id *cm_id,
+			       struct ib_cm_req_event_param *param,
+			       void *private_data)
+{
+	char sguid[40];
+
+	srpt_format_guid(sguid, sizeof(sguid),
+			 &param->primary_path->dgid.global.interface_id);
+
+	return srpt_cm_req_recv(cm_id, param->port, param->primary_path->pkey,
+				private_data, sguid);
+}
+
 static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
 			     enum ib_cm_rej_reason reason,
 			     const u8 *private_data,
@@ -2206,7 +2356,8 @@ static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
 }
 
 /**
- * srpt_cm_rtu_recv() - Process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event.
+ * srpt_cm_rtu_recv - process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event
+ * @ch: SRPT RDMA channel.
  *
  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
  * and that the recipient may begin transmitting (RTU = ready to use).
@@ -2215,21 +2366,34 @@ static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
 {
 	int ret;
 
-	if (srpt_set_ch_state(ch, CH_LIVE)) {
-		ret = srpt_ch_qp_rts(ch, ch->qp);
+	ret = srpt_ch_qp_rts(ch, ch->qp);
+	if (ret < 0) {
+		pr_err("%s-%d: QP transition to RTS failed\n", ch->sess_name,
+		       ch->qp->qp_num);
+		srpt_close_ch(ch);
+		return;
+	}
 
-		if (ret == 0) {
-			/* Trigger wait list processing. */
-			ret = srpt_zerolength_write(ch);
-			WARN_ONCE(ret < 0, "%d\n", ret);
-		} else {
-			srpt_close_ch(ch);
-		}
+	/*
+	 * Note: calling srpt_close_ch() if the transition to the LIVE state
+	 * fails is not necessary since that means that that function has
+	 * already been invoked from another thread.
+	 */
+	if (!srpt_set_ch_state(ch, CH_LIVE)) {
+		pr_err("%s-%d: channel transition to LIVE state failed\n",
+		       ch->sess_name, ch->qp->qp_num);
+		return;
 	}
+
+	/* Trigger wait list processing. */
+	ret = srpt_zerolength_write(ch);
+	WARN_ONCE(ret < 0, "%d\n", ret);
 }
 
 /**
- * srpt_cm_handler() - IB connection manager callback function.
+ * srpt_cm_handler - IB connection manager callback function
+ * @cm_id: IB/CM connection identifier.
+ * @event: IB/CM event.
  *
  * A non-zero return value will cause the caller destroy the CM ID.
  *
@@ -2246,8 +2410,8 @@ static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
 	ret = 0;
 	switch (event->event) {
 	case IB_CM_REQ_RECEIVED:
-		ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
-				       event->private_data);
+		ret = srpt_ib_cm_req_recv(cm_id, &event->param.req_rcvd,
+					  event->private_data);
 		break;
 	case IB_CM_REJ_RECEIVED:
 		srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason,
@@ -2294,11 +2458,11 @@ static int srpt_write_pending_status(struct se_cmd *se_cmd)
 	struct srpt_send_ioctx *ioctx;
 
 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
-	return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;
+	return ioctx->state == SRPT_STATE_NEED_DATA;
 }
 
 /*
- * srpt_write_pending() - Start data transfer from initiator to target (write).
+ * srpt_write_pending - Start data transfer from initiator to target (write).
  */
 static int srpt_write_pending(struct se_cmd *se_cmd)
 {
@@ -2355,7 +2519,8 @@ static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
 }
 
 /**
- * srpt_queue_response() - Transmits the response to a SCSI command.
+ * srpt_queue_response - transmit the response to a SCSI command
+ * @cmd: SCSI target command.
  *
  * Callback function called by the TCM core. Must not block since it can be
  * invoked on the context of the IB completion handler.
@@ -2369,13 +2534,11 @@ static void srpt_queue_response(struct se_cmd *cmd)
 	struct ib_send_wr send_wr, *first_wr = &send_wr, *bad_wr;
 	struct ib_sge sge;
 	enum srpt_command_state state;
-	unsigned long flags;
 	int resp_len, ret, i;
 	u8 srp_tm_status;
 
 	BUG_ON(!ch);
 
-	spin_lock_irqsave(&ioctx->spinlock, flags);
 	state = ioctx->state;
 	switch (state) {
 	case SRPT_STATE_NEW:
@@ -2390,7 +2553,6 @@ static void srpt_queue_response(struct se_cmd *cmd)
 			ch, ioctx->ioctx.index, ioctx->state);
 		break;
 	}
-	spin_unlock_irqrestore(&ioctx->spinlock, flags);
 
 	if (unlikely(WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT)))
 		return;
@@ -2494,26 +2656,56 @@ static void srpt_refresh_port_work(struct work_struct *work)
 	srpt_refresh_port(sport);
 }
 
+static bool srpt_ch_list_empty(struct srpt_port *sport)
+{
+	struct srpt_nexus *nexus;
+	bool res = true;
+
+	rcu_read_lock();
+	list_for_each_entry(nexus, &sport->nexus_list, entry)
+		if (!list_empty(&nexus->ch_list))
+			res = false;
+	rcu_read_unlock();
+
+	return res;
+}
+
 /**
- * srpt_release_sdev() - Free the channel resources associated with a target.
+ * srpt_release_sport - disable login and wait for associated channels
+ * @sport: SRPT HCA port.
  */
-static int srpt_release_sdev(struct srpt_device *sdev)
+static int srpt_release_sport(struct srpt_port *sport)
 {
-	int i, res;
+	struct srpt_nexus *nexus, *next_n;
+	struct srpt_rdma_ch *ch;
 
 	WARN_ON_ONCE(irqs_disabled());
 
-	BUG_ON(!sdev);
-
-	mutex_lock(&sdev->mutex);
-	for (i = 0; i < ARRAY_SIZE(sdev->port); i++)
-		srpt_set_enabled(&sdev->port[i], false);
-	mutex_unlock(&sdev->mutex);
+	mutex_lock(&sport->mutex);
+	srpt_set_enabled(sport, false);
+	mutex_unlock(&sport->mutex);
+
+	while (wait_event_timeout(sport->ch_releaseQ,
+				  srpt_ch_list_empty(sport), 5 * HZ) <= 0) {
+		pr_info("%s_%d: waiting for session unregistration ...\n",
+			sport->sdev->device->name, sport->port);
+		rcu_read_lock();
+		list_for_each_entry(nexus, &sport->nexus_list, entry) {
+			list_for_each_entry(ch, &nexus->ch_list, list) {
+				pr_info("%s-%d: state %s\n",
+					ch->sess_name, ch->qp->qp_num,
+					get_ch_state_name(ch->state));
+			}
+		}
+		rcu_read_unlock();
+	}
 
-	res = wait_event_interruptible(sdev->ch_releaseQ,
-				       list_empty_careful(&sdev->rch_list));
-	if (res)
-		pr_err("%s: interrupted.\n", __func__);
+	mutex_lock(&sport->mutex);
+	list_for_each_entry_safe(nexus, next_n, &sport->nexus_list, entry) {
+		list_del(&nexus->entry);
+		kfree_rcu(nexus, rcu);
+	}
+	mutex_unlock(&sport->mutex);
 
 	return 0;
 }
@@ -2600,8 +2792,10 @@ static int srpt_alloc_srq(struct srpt_device *sdev)
 	sdev->use_srq = true;
 	sdev->srq = srq;
 
-	for (i = 0; i < sdev->srq_size; ++i)
+	for (i = 0; i < sdev->srq_size; ++i) {
+		INIT_LIST_HEAD(&sdev->ioctx_ring[i]->wait_list);
 		srpt_post_recv(sdev, NULL, sdev->ioctx_ring[i]);
+	}
 
 	return 0;
 }
@@ -2623,7 +2817,8 @@ static int srpt_use_srq(struct srpt_device *sdev, bool use_srq)
 }
 
 /**
- * srpt_add_one() - Infiniband device addition callback function.
+ * srpt_add_one - InfiniBand device addition callback function
+ * @device: Describes a HCA.
  */
 static void srpt_add_one(struct ib_device *device)
 {
@@ -2638,9 +2833,7 @@ static void srpt_add_one(struct ib_device *device)
 		goto err;
 
 	sdev->device = device;
-	INIT_LIST_HEAD(&sdev->rch_list);
-	init_waitqueue_head(&sdev->ch_releaseQ);
-	mutex_init(&sdev->mutex);
+	mutex_init(&sdev->sdev_mutex);
 
 	sdev->pd = ib_alloc_pd(device, 0);
 	if (IS_ERR(sdev->pd))
@@ -2681,6 +2874,9 @@ static void srpt_add_one(struct ib_device *device)
 
 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
 		sport = &sdev->port[i - 1];
+		INIT_LIST_HEAD(&sport->nexus_list);
+		init_waitqueue_head(&sport->ch_releaseQ);
+		mutex_init(&sport->mutex);
 		sport->sdev = sdev;
 		sport->port = i;
 		sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
@@ -2721,7 +2917,9 @@ static void srpt_add_one(struct ib_device *device)
 }
 
 /**
- * srpt_remove_one() - InfiniBand device removal callback function.
+ * srpt_remove_one - InfiniBand device removal callback function
+ * @device: Describes a HCA.
+ * @client_data: The value passed as the third argument to ib_set_client_data().
  */
 static void srpt_remove_one(struct ib_device *device, void *client_data)
 {
@@ -2751,7 +2949,9 @@ static void srpt_remove_one(struct ib_device *device, void *client_data)
 	spin_lock(&srpt_dev_lock);
 	list_del(&sdev->list);
 	spin_unlock(&srpt_dev_lock);
-	srpt_release_sdev(sdev);
+
+	for (i = 0; i < sdev->device->phys_port_cnt; i++)
+		srpt_release_sport(&sdev->port[i]);
 
 	srpt_free_srq(sdev);
 
@@ -2827,7 +3027,8 @@ static void srpt_release_cmd(struct se_cmd *se_cmd)
 }
 
 /**
- * srpt_close_session() - Forcibly close a session.
+ * srpt_close_session - forcibly close a session
+ * @se_sess: SCSI target session.
  *
  * Callback function invoked by the TCM core to clean up sessions associated
  * with a node ACL when the user invokes
@@ -2836,15 +3037,13 @@ static void srpt_release_cmd(struct se_cmd *se_cmd)
 static void srpt_close_session(struct se_session *se_sess)
 {
 	struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
-	struct srpt_device *sdev = ch->sport->sdev;
 
-	mutex_lock(&sdev->mutex);
 	srpt_disconnect_ch_sync(ch);
-	mutex_unlock(&sdev->mutex);
 }
 
 /**
- * srpt_sess_get_index() - Return the value of scsiAttIntrPortIndex (SCSI-MIB).
+ * srpt_sess_get_index - return the value of scsiAttIntrPortIndex (SCSI-MIB)
+ * @se_sess: SCSI target session.
  *
  * A quote from RFC 4455 (SCSI-MIB) about this MIB object:
  * This object represents an arbitrary integer used to uniquely identify a
@@ -2866,7 +3065,7 @@ static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
 	struct srpt_send_ioctx *ioctx;
 
 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
-	return srpt_get_cmd_state(ioctx);
+	return ioctx->state;
 }
 
 static int srpt_parse_guid(u64 *guid, const char *name)
@@ -2883,7 +3082,7 @@ static int srpt_parse_guid(u64 *guid, const char *name)
 }
 
 /**
- * srpt_parse_i_port_id() - Parse an initiator port ID.
+ * srpt_parse_i_port_id - parse an initiator port ID
  * @name: ASCII representation of a 128-bit initiator port ID.
  * @i_port_id: Binary 128-bit port ID.
  */
@@ -3064,18 +3263,24 @@ static ssize_t srpt_tpg_attrib_use_srq_store(struct config_item *item,
 	if (val != !!val)
 		return -EINVAL;
 
-	ret = mutex_lock_interruptible(&sdev->mutex);
+	ret = mutex_lock_interruptible(&sdev->sdev_mutex);
 	if (ret < 0)
 		return ret;
+	ret = mutex_lock_interruptible(&sport->mutex);
+	if (ret < 0)
+		goto unlock_sdev;
 	enabled = sport->enabled;
 	/* Log out all initiator systems before changing 'use_srq'. */
 	srpt_set_enabled(sport, false);
 	sport->port_attrib.use_srq = val;
 	srpt_use_srq(sdev, sport->port_attrib.use_srq);
 	srpt_set_enabled(sport, enabled);
-	mutex_unlock(&sdev->mutex);
+	ret = count;
+	mutex_unlock(&sport->mutex);
+unlock_sdev:
+	mutex_unlock(&sdev->sdev_mutex);
 
-	return count;
+	return ret;
 }
 
 CONFIGFS_ATTR(srpt_tpg_attrib_,  srp_max_rdma_size);
@@ -3104,7 +3309,6 @@ static ssize_t srpt_tpg_enable_store(struct config_item *item,
 {
 	struct se_portal_group *se_tpg = to_tpg(item);
 	struct srpt_port *sport = srpt_tpg_to_sport(se_tpg);
-	struct srpt_device *sdev = sport->sdev;
 	unsigned long tmp;
         int ret;
 
@@ -3119,9 +3323,9 @@ static ssize_t srpt_tpg_enable_store(struct config_item *item,
 		return -EINVAL;
 	}
 
-	mutex_lock(&sdev->mutex);
+	mutex_lock(&sport->mutex);
 	srpt_set_enabled(sport, tmp);
-	mutex_unlock(&sdev->mutex);
+	mutex_unlock(&sport->mutex);
 
 	return count;
 }
@@ -3134,8 +3338,10 @@ static struct configfs_attribute *srpt_tpg_attrs[] = {
 };
 
 /**
- * configfs callback invoked for
- * mkdir /sys/kernel/config/target/$driver/$port/$tpg
+ * srpt_make_tpg - configfs callback invoked for mkdir /sys/kernel/config/target/$driver/$port/$tpg
+ * @wwn: Corresponds to $driver/$port.
+ * @group: Not used.
+ * @name: $tpg.
  */
 static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
 					     struct config_group *group,
@@ -3157,8 +3363,8 @@ static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
 }
 
 /**
- * configfs callback invoked for
- * rmdir /sys/kernel/config/target/$driver/$port/$tpg
+ * srpt_drop_tpg - configfs callback invoked for rmdir /sys/kernel/config/target/$driver/$port/$tpg
+ * @tpg: Target portal group to deregister.
  */
 static void srpt_drop_tpg(struct se_portal_group *tpg)
 {
@@ -3169,8 +3375,10 @@ static void srpt_drop_tpg(struct se_portal_group *tpg)
 }
 
 /**
- * configfs callback invoked for
- * mkdir /sys/kernel/config/target/$driver/$port
+ * srpt_make_tport - configfs callback invoked for mkdir /sys/kernel/config/target/$driver/$port
+ * @tf: Not used.
+ * @group: Not used.
+ * @name: $port.
  */
 static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
 				      struct config_group *group,
@@ -3180,8 +3388,8 @@ static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
 }
 
 /**
- * configfs callback invoked for
- * rmdir /sys/kernel/config/target/$driver/$port
+ * srpt_drop_tport - configfs callback invoked for rmdir /sys/kernel/config/target/$driver/$port
+ * @wwn: $port.
  */
 static void srpt_drop_tport(struct se_wwn *wwn)
 {
@@ -3239,7 +3447,7 @@ static const struct target_core_fabric_ops srpt_template = {
 };
 
 /**
- * srpt_init_module() - Kernel module initialization.
+ * srpt_init_module - kernel module initialization
  *
  * Note: Since ib_register_client() registers callback functions, and since at
  * least one of these callback functions (srpt_add_one()) calls target core
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h
index 673387d365a3..4d9199fd00dc 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.h
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.h
@@ -54,6 +54,8 @@
  */
 #define SRP_SERVICE_NAME_PREFIX		"SRP.T10:"
 
+struct srpt_nexus;
+
 enum {
 	/*
 	 * SRP IOControllerProfile attributes for SRP target ports that have
@@ -114,7 +116,7 @@ enum {
 
 	MIN_SRPT_SQ_SIZE = 16,
 	DEF_SRPT_SQ_SIZE = 4096,
-	SRPT_RQ_SIZE = 128,
+	MAX_SRPT_RQ_SIZE = 128,
 	MIN_SRPT_SRQ_SIZE = 4,
 	DEFAULT_SRPT_SRQ_SIZE = 4095,
 	MAX_SRPT_SRQ_SIZE = 65535,
@@ -134,7 +136,7 @@ enum {
 };
 
 /**
- * enum srpt_command_state - SCSI command state managed by SRPT.
+ * enum srpt_command_state - SCSI command state managed by SRPT
  * @SRPT_STATE_NEW:           New command arrived and is being processed.
  * @SRPT_STATE_NEED_DATA:     Processing a write or bidir command and waiting
  *                            for data arrival.
@@ -158,7 +160,8 @@ enum srpt_command_state {
 };
 
 /**
- * struct srpt_ioctx - Shared SRPT I/O context information.
+ * struct srpt_ioctx - shared SRPT I/O context information
+ * @cqe:   Completion queue element.
  * @buf:   Pointer to the buffer.
  * @dma:   DMA address of the buffer.
  * @index: Index of the I/O context in its ioctx_ring array.
@@ -171,7 +174,7 @@ struct srpt_ioctx {
 };
 
 /**
- * struct srpt_recv_ioctx - SRPT receive I/O context.
+ * struct srpt_recv_ioctx - SRPT receive I/O context
  * @ioctx:     See above.
  * @wait_list: Node for insertion in srpt_rdma_ch.cmd_wait_list.
  */
@@ -187,13 +190,20 @@ struct srpt_rw_ctx {
 };
 
 /**
- * struct srpt_send_ioctx - SRPT send I/O context.
+ * struct srpt_send_ioctx - SRPT send I/O context
  * @ioctx:       See above.
  * @ch:          Channel pointer.
- * @spinlock:    Protects 'state'.
+ * @s_rw_ctx:    @rw_ctxs points here if only a single rw_ctx is needed.
+ * @rw_ctxs:     RDMA read/write contexts.
+ * @rdma_cqe:    RDMA completion queue element.
+ * @free_list:   Node in srpt_rdma_ch.free_list.
  * @state:       I/O context state.
  * @cmd:         Target core command data structure.
  * @sense_data:  SCSI sense data.
+ * @n_rdma:      Number of work requests needed to transfer this ioctx.
+ * @n_rw_ctx:    Size of rw_ctxs array.
+ * @queue_status_only: Send a SCSI status back to the initiator but no data.
+ * @sense_data:  Sense data to be sent to the initiator.
  */
 struct srpt_send_ioctx {
 	struct srpt_ioctx	ioctx;
@@ -204,10 +214,8 @@ struct srpt_send_ioctx {
 
 	struct ib_cqe		rdma_cqe;
 	struct list_head	free_list;
-	spinlock_t		spinlock;
 	enum srpt_command_state	state;
 	struct se_cmd		cmd;
-	struct completion	tx_done;
 	u8			n_rdma;
 	u8			n_rw_ctx;
 	bool			queue_status_only;
@@ -215,7 +223,7 @@ struct srpt_send_ioctx {
 };
 
 /**
- * enum rdma_ch_state - SRP channel state.
+ * enum rdma_ch_state - SRP channel state
  * @CH_CONNECTING:    QP is in RTR state; waiting for RTU.
  * @CH_LIVE:	      QP is in RTS state.
  * @CH_DISCONNECTING: DREQ has been sent and waiting for DREP or DREQ has
@@ -233,17 +241,19 @@ enum rdma_ch_state {
 };
 
 /**
- * struct srpt_rdma_ch - RDMA channel.
- * @cm_id:         IB CM ID associated with the channel.
+ * struct srpt_rdma_ch - RDMA channel
+ * @nexus:         I_T nexus this channel is associated with.
  * @qp:            IB queue pair used for communicating over this channel.
+ * @cm_id:         IB CM ID associated with the channel.
  * @cq:            IB completion queue for this channel.
+ * @zw_cqe:	   Zero-length write CQE.
+ * @rcu:           RCU head.
+ * @kref:	   kref for this channel.
  * @rq_size:       IB receive queue size.
- * @rsp_size	   IB response message size in bytes.
+ * @max_rsp_size:  Maximum size of an RSP response message in bytes.
  * @sq_wr_avail:   number of work requests available in the send queue.
  * @sport:         pointer to the information of the HCA port used by this
  *                 channel.
- * @i_port_id:     128-bit initiator port identifier copied from SRP_LOGIN_REQ.
- * @t_port_id:     128-bit target port identifier copied from SRP_LOGIN_REQ.
  * @max_ti_iu_len: maximum target-to-initiator information unit length.
  * @req_lim:       request limit: maximum number of requests that may be sent
  *                 by the initiator without having received a response.
@@ -251,30 +261,34 @@ enum rdma_ch_state {
  * @spinlock:      Protects free_list and state.
  * @free_list:     Head of list with free send I/O contexts.
  * @state:         channel state. See also enum rdma_ch_state.
+ * @processing_wait_list: Whether or not cmd_wait_list is being processed.
  * @ioctx_ring:    Send ring.
  * @ioctx_recv_ring: Receive I/O context ring.
- * @list:          Node for insertion in the srpt_device.rch_list list.
+ * @list:          Node in srpt_nexus.ch_list.
  * @cmd_wait_list: List of SCSI commands that arrived before the RTU event. This
  *                 list contains struct srpt_ioctx elements and is protected
  *                 against concurrent modification by the cm_id spinlock.
+ * @pkey:          P_Key of the IB partition for this SRP channel.
  * @sess:          Session information associated with this SRP channel.
  * @sess_name:     Session name.
- * @ini_guid:      Initiator port GUID.
  * @release_work:  Allows scheduling of srpt_release_channel().
- * @release_done:  Enables waiting for srpt_release_channel() completion.
  */
 struct srpt_rdma_ch {
-	struct ib_cm_id		*cm_id;
+	struct srpt_nexus	*nexus;
 	struct ib_qp		*qp;
+	union {
+		struct {
+			struct ib_cm_id		*cm_id;
+		} ib_cm;
+	};
 	struct ib_cq		*cq;
 	struct ib_cqe		zw_cqe;
+	struct rcu_head		rcu;
 	struct kref		kref;
 	int			rq_size;
-	u32			rsp_size;
+	u32			max_rsp_size;
 	atomic_t		sq_wr_avail;
 	struct srpt_port	*sport;
-	u8			i_port_id[16];
-	u8			t_port_id[16];
 	int			max_ti_iu_len;
 	atomic_t		req_lim;
 	atomic_t		req_lim_delta;
@@ -285,15 +299,31 @@ struct srpt_rdma_ch {
 	struct srpt_recv_ioctx	**ioctx_recv_ring;
 	struct list_head	list;
 	struct list_head	cmd_wait_list;
+	uint16_t		pkey;
+	bool			processing_wait_list;
 	struct se_session	*sess;
-	u8			sess_name[36];
-	u8			ini_guid[24];
+	u8			sess_name[24];
 	struct work_struct	release_work;
-	struct completion	*release_done;
 };
 
 /**
- * struct srpt_port_attib - Attributes for SRPT port
+ * struct srpt_nexus - I_T nexus
+ * @rcu:       RCU head for this data structure.
+ * @entry:     srpt_port.nexus_list list node.
+ * @ch_list:   struct srpt_rdma_ch list. Protected by srpt_port.mutex.
+ * @i_port_id: 128-bit initiator port identifier copied from SRP_LOGIN_REQ.
+ * @t_port_id: 128-bit target port identifier copied from SRP_LOGIN_REQ.
+ */
+struct srpt_nexus {
+	struct rcu_head		rcu;
+	struct list_head	entry;
+	struct list_head	ch_list;
+	u8			i_port_id[16];
+	u8			t_port_id[16];
+};
+
+/**
+ * struct srpt_port_attib - attributes for SRPT port
  * @srp_max_rdma_size: Maximum size of SRP RDMA transfers for new connections.
  * @srp_max_rsp_size: Maximum size of SRP response messages in bytes.
  * @srp_sq_size: Shared receive queue (SRQ) size.
@@ -307,7 +337,7 @@ struct srpt_port_attrib {
 };
 
 /**
- * struct srpt_port - Information associated by SRPT with a single IB port.
+ * struct srpt_port - information associated by SRPT with a single IB port
  * @sdev:      backpointer to the HCA information.
  * @mad_agent: per-port management datagram processing information.
  * @enabled:   Whether or not this target port is enabled.
@@ -323,7 +353,10 @@ struct srpt_port_attrib {
  * @port_guid_wwn: WWN associated with target port GUID.
  * @port_gid_tpg:  TPG associated with target port GID.
  * @port_gid_wwn:  WWN associated with target port GID.
- * @port_acl_list: Head of the list with all node ACLs for this port.
+ * @port_attrib:   Port attributes that can be accessed through configfs.
+ * @ch_releaseQ:   Enables waiting for removal from nexus_list.
+ * @mutex:	   Protects nexus_list.
+ * @nexus_list:	   Nexus list. See also srpt_nexus.entry.
  */
 struct srpt_port {
 	struct srpt_device	*sdev;
@@ -341,21 +374,22 @@ struct srpt_port {
 	struct se_portal_group	port_gid_tpg;
 	struct se_wwn		port_gid_wwn;
 	struct srpt_port_attrib port_attrib;
+	wait_queue_head_t	ch_releaseQ;
+	struct mutex		mutex;
+	struct list_head	nexus_list;
 };
 
 /**
- * struct srpt_device - Information associated by SRPT with a single HCA.
+ * struct srpt_device - information associated by SRPT with a single HCA
  * @device:        Backpointer to the struct ib_device managed by the IB core.
  * @pd:            IB protection domain.
  * @lkey:          L_Key (local key) with write access to all local memory.
  * @srq:           Per-HCA SRQ (shared receive queue).
  * @cm_id:         Connection identifier.
  * @srq_size:      SRQ size.
+ * @sdev_mutex:	   Serializes use_srq changes.
  * @use_srq:       Whether or not to use SRQ.
  * @ioctx_ring:    Per-HCA SRQ.
- * @rch_list:      Per-device channel list -- see also srpt_rdma_ch.list.
- * @ch_releaseQ:   Enables waiting for removal from rch_list.
- * @mutex:         Protects rch_list.
  * @port:          Information about the ports owned by this HCA.
  * @event_handler: Per-HCA asynchronous IB event handler.
  * @list:          Node in srpt_dev_list.
@@ -367,11 +401,9 @@ struct srpt_device {
 	struct ib_srq		*srq;
 	struct ib_cm_id		*cm_id;
 	int			srq_size;
+	struct mutex		sdev_mutex;
 	bool			use_srq;
 	struct srpt_recv_ioctx	**ioctx_ring;
-	struct list_head	rch_list;
-	wait_queue_head_t	ch_releaseQ;
-	struct mutex		mutex;
 	struct srpt_port	port[2];
 	struct ib_event_handler	event_handler;
 	struct list_head	list;
diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c
index 925571475005..c81c79d01d93 100644
--- a/drivers/input/evdev.c
+++ b/drivers/input/evdev.c
@@ -135,10 +135,7 @@ static void __evdev_flush_queue(struct evdev_client *client, unsigned int type)
 			continue;
 		} else if (head != i) {
 			/* move entry to fill the gap */
-			client->buffer[head].time = ev->time;
-			client->buffer[head].type = ev->type;
-			client->buffer[head].code = ev->code;
-			client->buffer[head].value = ev->value;
+			client->buffer[head] = *ev;
 		}
 
 		num++;
@@ -157,6 +154,7 @@ static void __evdev_queue_syn_dropped(struct evdev_client *client)
 {
 	struct input_event ev;
 	ktime_t time;
+	struct timespec64 ts;
 
 	time = client->clk_type == EV_CLK_REAL ?
 			ktime_get_real() :
@@ -164,7 +162,9 @@ static void __evdev_queue_syn_dropped(struct evdev_client *client)
 				ktime_get() :
 				ktime_get_boottime();
 
-	ev.time = ktime_to_timeval(time);
+	ts = ktime_to_timespec64(time);
+	ev.input_event_sec = ts.tv_sec;
+	ev.input_event_usec = ts.tv_nsec / NSEC_PER_USEC;
 	ev.type = EV_SYN;
 	ev.code = SYN_DROPPED;
 	ev.value = 0;
@@ -241,7 +241,10 @@ static void __pass_event(struct evdev_client *client,
 		 */
 		client->tail = (client->head - 2) & (client->bufsize - 1);
 
-		client->buffer[client->tail].time = event->time;
+		client->buffer[client->tail].input_event_sec =
+						event->input_event_sec;
+		client->buffer[client->tail].input_event_usec =
+						event->input_event_usec;
 		client->buffer[client->tail].type = EV_SYN;
 		client->buffer[client->tail].code = SYN_DROPPED;
 		client->buffer[client->tail].value = 0;
@@ -262,12 +265,15 @@ static void evdev_pass_values(struct evdev_client *client,
 	struct evdev *evdev = client->evdev;
 	const struct input_value *v;
 	struct input_event event;
+	struct timespec64 ts;
 	bool wakeup = false;
 
 	if (client->revoked)
 		return;
 
-	event.time = ktime_to_timeval(ev_time[client->clk_type]);
+	ts = ktime_to_timespec64(ev_time[client->clk_type]);
+	event.input_event_sec = ts.tv_sec;
+	event.input_event_usec = ts.tv_nsec / NSEC_PER_USEC;
 
 	/* Interrupts are disabled, just acquire the lock. */
 	spin_lock(&client->buffer_lock);
@@ -635,21 +641,21 @@ static ssize_t evdev_read(struct file *file, char __user *buffer,
 }
 
 /* No kernel lock - fine */
-static unsigned int evdev_poll(struct file *file, poll_table *wait)
+static __poll_t evdev_poll(struct file *file, poll_table *wait)
 {
 	struct evdev_client *client = file->private_data;
 	struct evdev *evdev = client->evdev;
-	unsigned int mask;
+	__poll_t mask;
 
 	poll_wait(file, &evdev->wait, wait);
 
 	if (evdev->exist && !client->revoked)
-		mask = POLLOUT | POLLWRNORM;
+		mask = EPOLLOUT | EPOLLWRNORM;
 	else
-		mask = POLLHUP | POLLERR;
+		mask = EPOLLHUP | EPOLLERR;
 
 	if (client->packet_head != client->tail)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/input/input-compat.c b/drivers/input/input-compat.c
index 2186f71c9fe5..fda8d6d2a268 100644
--- a/drivers/input/input-compat.c
+++ b/drivers/input/input-compat.c
@@ -24,8 +24,8 @@ int input_event_from_user(const char __user *buffer,
 				   sizeof(struct input_event_compat)))
 			return -EFAULT;
 
-		event->time.tv_sec = compat_event.time.tv_sec;
-		event->time.tv_usec = compat_event.time.tv_usec;
+		event->input_event_sec = compat_event.sec;
+		event->input_event_usec = compat_event.usec;
 		event->type = compat_event.type;
 		event->code = compat_event.code;
 		event->value = compat_event.value;
@@ -44,8 +44,8 @@ int input_event_to_user(char __user *buffer,
 	if (in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
 		struct input_event_compat compat_event;
 
-		compat_event.time.tv_sec = event->time.tv_sec;
-		compat_event.time.tv_usec = event->time.tv_usec;
+		compat_event.sec = event->input_event_sec;
+		compat_event.usec = event->input_event_usec;
 		compat_event.type = event->type;
 		compat_event.code = event->code;
 		compat_event.value = event->value;
diff --git a/drivers/input/input-compat.h b/drivers/input/input-compat.h
index 1563160a7af3..08cd755e73fd 100644
--- a/drivers/input/input-compat.h
+++ b/drivers/input/input-compat.h
@@ -18,7 +18,8 @@
 #ifdef CONFIG_COMPAT
 
 struct input_event_compat {
-	struct compat_timeval time;
+	compat_ulong_t sec;
+	compat_ulong_t usec;
 	__u16 type;
 	__u16 code;
 	__s32 value;
diff --git a/drivers/input/input-polldev.c b/drivers/input/input-polldev.c
index 3664f81655ca..78df5a74822e 100644
--- a/drivers/input/input-polldev.c
+++ b/drivers/input/input-polldev.c
@@ -20,7 +20,6 @@
 MODULE_AUTHOR("Dmitry Torokhov <dtor@mail.ru>");
 MODULE_DESCRIPTION("Generic implementation of a polled input device");
 MODULE_LICENSE("GPL v2");
-MODULE_VERSION("0.1");
 
 static void input_polldev_queue_work(struct input_polled_dev *dev)
 {
diff --git a/drivers/input/input.c b/drivers/input/input.c
index e30642db50d5..9785546420a7 100644
--- a/drivers/input/input.c
+++ b/drivers/input/input.c
@@ -1048,12 +1048,12 @@ static inline void input_wakeup_procfs_readers(void)
 	wake_up(&input_devices_poll_wait);
 }
 
-static unsigned int input_proc_devices_poll(struct file *file, poll_table *wait)
+static __poll_t input_proc_devices_poll(struct file *file, poll_table *wait)
 {
 	poll_wait(file, &input_devices_poll_wait, wait);
 	if (file->f_version != input_devices_state) {
 		file->f_version = input_devices_state;
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	}
 
 	return 0;
diff --git a/drivers/input/joydev.c b/drivers/input/joydev.c
index 7b29a8944039..4c1e427dfabb 100644
--- a/drivers/input/joydev.c
+++ b/drivers/input/joydev.c
@@ -436,14 +436,14 @@ static ssize_t joydev_read(struct file *file, char __user *buf,
 }
 
 /* No kernel lock - fine */
-static unsigned int joydev_poll(struct file *file, poll_table *wait)
+static __poll_t joydev_poll(struct file *file, poll_table *wait)
 {
 	struct joydev_client *client = file->private_data;
 	struct joydev *joydev = client->joydev;
 
 	poll_wait(file, &joydev->wait, wait);
-	return (joydev_data_pending(client) ? (POLLIN | POLLRDNORM) : 0) |
-		(joydev->exist ?  0 : (POLLHUP | POLLERR));
+	return (joydev_data_pending(client) ? (EPOLLIN | EPOLLRDNORM) : 0) |
+		(joydev->exist ?  0 : (EPOLLHUP | EPOLLERR));
 }
 
 static int joydev_handle_JSIOCSAXMAP(struct joydev *joydev,
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index d88d3e0f59fb..9d2688f3f961 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -84,9 +84,6 @@
 #include <linux/usb/input.h>
 #include <linux/usb/quirks.h>
 
-#define DRIVER_AUTHOR "Marko Friedemann <mfr@bmx-chemnitz.de>"
-#define DRIVER_DESC "X-Box pad driver"
-
 #define XPAD_PKT_LEN 64
 
 /* xbox d-pads should map to buttons, as is required for DDR pads
@@ -1943,6 +1940,6 @@ static struct usb_driver xpad_driver = {
 
 module_usb_driver(xpad_driver);
 
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_AUTHOR("Marko Friedemann <mfr@bmx-chemnitz.de>");
+MODULE_DESCRIPTION("X-Box pad driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/input/keyboard/matrix_keypad.c b/drivers/input/keyboard/matrix_keypad.c
index 1f316d66e6f7..41614c185918 100644
--- a/drivers/input/keyboard/matrix_keypad.c
+++ b/drivers/input/keyboard/matrix_keypad.c
@@ -218,8 +218,10 @@ static void matrix_keypad_stop(struct input_dev *dev)
 {
 	struct matrix_keypad *keypad = input_get_drvdata(dev);
 
+	spin_lock_irq(&keypad->lock);
 	keypad->stopped = true;
-	mb();
+	spin_unlock_irq(&keypad->lock);
+
 	flush_work(&keypad->work.work);
 	/*
 	 * matrix_keypad_scan() will leave IRQs enabled;
diff --git a/drivers/input/keyboard/pmic8xxx-keypad.c b/drivers/input/keyboard/pmic8xxx-keypad.c
index 97c5424f49b9..98b24ed18752 100644
--- a/drivers/input/keyboard/pmic8xxx-keypad.c
+++ b/drivers/input/keyboard/pmic8xxx-keypad.c
@@ -697,6 +697,5 @@ module_platform_driver(pmic8xxx_kp_driver);
 
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("PMIC8XXX keypad driver");
-MODULE_VERSION("1.0");
 MODULE_ALIAS("platform:pmic8xxx_keypad");
 MODULE_AUTHOR("Trilok Soni <tsoni@codeaurora.org>");
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index 9f082a388388..62a1312a7387 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -268,20 +268,6 @@ config INPUT_GPIO_BEEPER
 	  To compile this driver as a module, choose M here: the
 	  module will be called gpio-beeper.
 
-config INPUT_GPIO_TILT_POLLED
-	tristate "Polled GPIO tilt switch"
-	depends on GPIOLIB || COMPILE_TEST
-	select INPUT_POLLDEV
-	help
-	  This driver implements support for tilt switches connected
-	  to GPIO pins that are not capable of generating interrupts.
-
-	  The list of gpios to use and the mapping of their states
-	  to specific angles is done via platform data.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called gpio_tilt_polled.
-
 config INPUT_GPIO_DECODER
 	tristate "Polled GPIO Decoder Input driver"
 	depends on GPIOLIB || COMPILE_TEST
@@ -468,7 +454,7 @@ config INPUT_TPS65218_PWRBUTTON
 	tristate "TPS65218 Power button driver"
 	depends on (MFD_TPS65217 || MFD_TPS65218)
 	help
-	  Say Y here if you want to enable power buttong reporting for
+	  Say Y here if you want to enable power button reporting for
 	  TPS65217 and TPS65218 Power Management IC devices.
 
 	  To compile this driver as a module, choose M here. The module will
diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile
index 4b6118d313fe..a8f61af865aa 100644
--- a/drivers/input/misc/Makefile
+++ b/drivers/input/misc/Makefile
@@ -36,7 +36,6 @@ obj-$(CONFIG_INPUT_DRV2665_HAPTICS)	+= drv2665.o
 obj-$(CONFIG_INPUT_DRV2667_HAPTICS)	+= drv2667.o
 obj-$(CONFIG_INPUT_GP2A)		+= gp2ap002a00f.o
 obj-$(CONFIG_INPUT_GPIO_BEEPER)		+= gpio-beeper.o
-obj-$(CONFIG_INPUT_GPIO_TILT_POLLED)	+= gpio_tilt_polled.o
 obj-$(CONFIG_INPUT_GPIO_DECODER)	+= gpio_decoder.o
 obj-$(CONFIG_INPUT_HISI_POWERKEY)	+= hisi_powerkey.o
 obj-$(CONFIG_HP_SDC_RTC)		+= hp_sdc_rtc.o
diff --git a/drivers/input/misc/apanel.c b/drivers/input/misc/apanel.c
index aad1df04c854..094bddf56755 100644
--- a/drivers/input/misc/apanel.c
+++ b/drivers/input/misc/apanel.c
@@ -26,7 +26,6 @@
 #include <linux/leds.h>
 
 #define APANEL_NAME	"Fujitsu Application Panel"
-#define APANEL_VERSION	"1.3.1"
 #define APANEL		"apanel"
 
 /* How often we poll keys - msecs */
@@ -345,7 +344,6 @@ module_exit(apanel_cleanup);
 MODULE_AUTHOR("Stephen Hemminger <shemminger@linux-foundation.org>");
 MODULE_DESCRIPTION(APANEL_NAME " driver");
 MODULE_LICENSE("GPL");
-MODULE_VERSION(APANEL_VERSION);
 
 MODULE_ALIAS("dmi:*:svnFUJITSU:pnLifeBook*:pvr*:rvnFUJITSU:*");
 MODULE_ALIAS("dmi:*:svnFUJITSU:pnLifebook*:pvr*:rvnFUJITSU:*");
diff --git a/drivers/input/misc/ati_remote2.c b/drivers/input/misc/ati_remote2.c
index ebf4448b31b9..ded5b84e336d 100644
--- a/drivers/input/misc/ati_remote2.c
+++ b/drivers/input/misc/ati_remote2.c
@@ -14,10 +14,8 @@
 #include <linux/module.h>
 
 #define DRIVER_DESC    "ATI/Philips USB RF remote driver"
-#define DRIVER_VERSION "0.3"
 
 MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_VERSION(DRIVER_VERSION);
 MODULE_AUTHOR("Ville Syrjala <syrjala@sci.fi>");
 MODULE_LICENSE("GPL");
 
diff --git a/drivers/input/misc/gpio_tilt_polled.c b/drivers/input/misc/gpio_tilt_polled.c
deleted file mode 100644
index 6e217a45e39a..000000000000
--- a/drivers/input/misc/gpio_tilt_polled.c
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- *  Driver for tilt switches connected via GPIO lines
- *  not capable of generating interrupts
- *
- *  Copyright (C) 2011 Heiko Stuebner <heiko@sntech.de>
- *
- *  based on: drivers/input/keyboard/gpio_keys_polled.c
- *
- *  Copyright (C) 2007-2010 Gabor Juhos <juhosg@openwrt.org>
- *  Copyright (C) 2010 Nuno Goncalves <nunojpg@gmail.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/slab.h>
-#include <linux/input.h>
-#include <linux/input-polldev.h>
-#include <linux/ioport.h>
-#include <linux/platform_device.h>
-#include <linux/gpio.h>
-#include <linux/input/gpio_tilt.h>
-
-#define DRV_NAME	"gpio-tilt-polled"
-
-struct gpio_tilt_polled_dev {
-	struct input_polled_dev *poll_dev;
-	struct device *dev;
-	const struct gpio_tilt_platform_data *pdata;
-
-	int last_state;
-
-	int threshold;
-	int count;
-};
-
-static void gpio_tilt_polled_poll(struct input_polled_dev *dev)
-{
-	struct gpio_tilt_polled_dev *tdev = dev->private;
-	const struct gpio_tilt_platform_data *pdata = tdev->pdata;
-	struct input_dev *input = dev->input;
-	struct gpio_tilt_state *tilt_state = NULL;
-	int state, i;
-
-	if (tdev->count < tdev->threshold) {
-		tdev->count++;
-	} else {
-		state = 0;
-		for (i = 0; i < pdata->nr_gpios; i++)
-			state |= (!!gpio_get_value(pdata->gpios[i].gpio) << i);
-
-		if (state != tdev->last_state) {
-			for (i = 0; i < pdata->nr_states; i++)
-				if (pdata->states[i].gpios == state)
-					tilt_state = &pdata->states[i];
-
-			if (tilt_state) {
-				for (i = 0; i < pdata->nr_axes; i++)
-					input_report_abs(input,
-							 pdata->axes[i].axis,
-							 tilt_state->axes[i]);
-
-				input_sync(input);
-			}
-
-			tdev->count = 0;
-			tdev->last_state = state;
-		}
-	}
-}
-
-static void gpio_tilt_polled_open(struct input_polled_dev *dev)
-{
-	struct gpio_tilt_polled_dev *tdev = dev->private;
-	const struct gpio_tilt_platform_data *pdata = tdev->pdata;
-
-	if (pdata->enable)
-		pdata->enable(tdev->dev);
-
-	/* report initial state of the axes */
-	tdev->last_state = -1;
-	tdev->count = tdev->threshold;
-	gpio_tilt_polled_poll(tdev->poll_dev);
-}
-
-static void gpio_tilt_polled_close(struct input_polled_dev *dev)
-{
-	struct gpio_tilt_polled_dev *tdev = dev->private;
-	const struct gpio_tilt_platform_data *pdata = tdev->pdata;
-
-	if (pdata->disable)
-		pdata->disable(tdev->dev);
-}
-
-static int gpio_tilt_polled_probe(struct platform_device *pdev)
-{
-	const struct gpio_tilt_platform_data *pdata =
-			dev_get_platdata(&pdev->dev);
-	struct device *dev = &pdev->dev;
-	struct gpio_tilt_polled_dev *tdev;
-	struct input_polled_dev *poll_dev;
-	struct input_dev *input;
-	int error, i;
-
-	if (!pdata || !pdata->poll_interval)
-		return -EINVAL;
-
-	tdev = kzalloc(sizeof(struct gpio_tilt_polled_dev), GFP_KERNEL);
-	if (!tdev) {
-		dev_err(dev, "no memory for private data\n");
-		return -ENOMEM;
-	}
-
-	error = gpio_request_array(pdata->gpios, pdata->nr_gpios);
-	if (error) {
-		dev_err(dev,
-			"Could not request tilt GPIOs: %d\n", error);
-		goto err_free_tdev;
-	}
-
-	poll_dev = input_allocate_polled_device();
-	if (!poll_dev) {
-		dev_err(dev, "no memory for polled device\n");
-		error = -ENOMEM;
-		goto err_free_gpios;
-	}
-
-	poll_dev->private = tdev;
-	poll_dev->poll = gpio_tilt_polled_poll;
-	poll_dev->poll_interval = pdata->poll_interval;
-	poll_dev->open = gpio_tilt_polled_open;
-	poll_dev->close = gpio_tilt_polled_close;
-
-	input = poll_dev->input;
-
-	input->name = pdev->name;
-	input->phys = DRV_NAME"/input0";
-	input->dev.parent = dev;
-
-	input->id.bustype = BUS_HOST;
-	input->id.vendor = 0x0001;
-	input->id.product = 0x0001;
-	input->id.version = 0x0100;
-
-	__set_bit(EV_ABS, input->evbit);
-	for (i = 0; i < pdata->nr_axes; i++)
-		input_set_abs_params(input, pdata->axes[i].axis,
-				     pdata->axes[i].min, pdata->axes[i].max,
-				     pdata->axes[i].fuzz, pdata->axes[i].flat);
-
-	tdev->threshold = DIV_ROUND_UP(pdata->debounce_interval,
-				       pdata->poll_interval);
-
-	tdev->poll_dev = poll_dev;
-	tdev->dev = dev;
-	tdev->pdata = pdata;
-
-	error = input_register_polled_device(poll_dev);
-	if (error) {
-		dev_err(dev, "unable to register polled device, err=%d\n",
-			error);
-		goto err_free_polldev;
-	}
-
-	platform_set_drvdata(pdev, tdev);
-
-	return 0;
-
-err_free_polldev:
-	input_free_polled_device(poll_dev);
-err_free_gpios:
-	gpio_free_array(pdata->gpios, pdata->nr_gpios);
-err_free_tdev:
-	kfree(tdev);
-
-	return error;
-}
-
-static int gpio_tilt_polled_remove(struct platform_device *pdev)
-{
-	struct gpio_tilt_polled_dev *tdev = platform_get_drvdata(pdev);
-	const struct gpio_tilt_platform_data *pdata = tdev->pdata;
-
-	input_unregister_polled_device(tdev->poll_dev);
-	input_free_polled_device(tdev->poll_dev);
-
-	gpio_free_array(pdata->gpios, pdata->nr_gpios);
-
-	kfree(tdev);
-
-	return 0;
-}
-
-static struct platform_driver gpio_tilt_polled_driver = {
-	.probe	= gpio_tilt_polled_probe,
-	.remove	= gpio_tilt_polled_remove,
-	.driver	= {
-		.name	= DRV_NAME,
-	},
-};
-
-module_platform_driver(gpio_tilt_polled_driver);
-
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Heiko Stuebner <heiko@sntech.de>");
-MODULE_DESCRIPTION("Polled GPIO tilt driver");
-MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/drivers/input/misc/hp_sdc_rtc.c b/drivers/input/misc/hp_sdc_rtc.c
index 1c8c56efc995..49b34de0aed4 100644
--- a/drivers/input/misc/hp_sdc_rtc.c
+++ b/drivers/input/misc/hp_sdc_rtc.c
@@ -408,13 +408,13 @@ static ssize_t hp_sdc_rtc_read(struct file *file, char __user *buf,
 	return retval;
 }
 
-static unsigned int hp_sdc_rtc_poll(struct file *file, poll_table *wait)
+static __poll_t hp_sdc_rtc_poll(struct file *file, poll_table *wait)
 {
         unsigned long l;
 
 	l = 0;
         if (l != 0)
-                return POLLIN | POLLRDNORM;
+                return EPOLLIN | EPOLLRDNORM;
         return 0;
 }
 
diff --git a/drivers/input/misc/keyspan_remote.c b/drivers/input/misc/keyspan_remote.c
index 77c47d6325fe..67482b248b2d 100644
--- a/drivers/input/misc/keyspan_remote.c
+++ b/drivers/input/misc/keyspan_remote.c
@@ -17,11 +17,6 @@
 #include <linux/module.h>
 #include <linux/usb/input.h>
 
-#define DRIVER_VERSION	"v0.1"
-#define DRIVER_AUTHOR	"Michael Downey <downey@zymeta.com>"
-#define DRIVER_DESC	"Driver for the USB Keyspan remote control."
-#define DRIVER_LICENSE	"GPL"
-
 /* Parameters that can be passed to the driver. */
 static int debug;
 module_param(debug, int, 0444);
@@ -590,6 +585,6 @@ static struct usb_driver keyspan_driver =
 module_usb_driver(keyspan_driver);
 
 MODULE_DEVICE_TABLE(usb, keyspan_table);
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_AUTHOR("Michael Downey <downey@zymeta.com>");
+MODULE_DESCRIPTION("Driver for the USB Keyspan remote control.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/misc/uinput.c b/drivers/input/misc/uinput.c
index 39ddd9a73feb..96a887f33698 100644
--- a/drivers/input/misc/uinput.c
+++ b/drivers/input/misc/uinput.c
@@ -84,11 +84,14 @@ static int uinput_dev_event(struct input_dev *dev,
 			    unsigned int type, unsigned int code, int value)
 {
 	struct uinput_device	*udev = input_get_drvdata(dev);
+	struct timespec64	ts;
 
 	udev->buff[udev->head].type = type;
 	udev->buff[udev->head].code = code;
 	udev->buff[udev->head].value = value;
-	do_gettimeofday(&udev->buff[udev->head].time);
+	ktime_get_ts64(&ts);
+	udev->buff[udev->head].input_event_sec = ts.tv_sec;
+	udev->buff[udev->head].input_event_usec = ts.tv_nsec / NSEC_PER_USEC;
 	udev->head = (udev->head + 1) % UINPUT_BUFFER_SIZE;
 
 	wake_up_interruptible(&udev->waitq);
@@ -694,14 +697,14 @@ static ssize_t uinput_read(struct file *file, char __user *buffer,
 	return retval;
 }
 
-static unsigned int uinput_poll(struct file *file, poll_table *wait)
+static __poll_t uinput_poll(struct file *file, poll_table *wait)
 {
 	struct uinput_device *udev = file->private_data;
 
 	poll_wait(file, &udev->waitq, wait);
 
 	if (udev->head != udev->tail)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
@@ -1085,4 +1088,3 @@ MODULE_ALIAS("devname:" UINPUT_NAME);
 MODULE_AUTHOR("Aristeu Sergio Rozanski Filho");
 MODULE_DESCRIPTION("User level driver support for input subsystem");
 MODULE_LICENSE("GPL");
-MODULE_VERSION("0.3");
diff --git a/drivers/input/misc/yealink.c b/drivers/input/misc/yealink.c
index a1e0ff59d2f2..f0c9bf87b4e3 100644
--- a/drivers/input/misc/yealink.c
+++ b/drivers/input/misc/yealink.c
@@ -56,8 +56,6 @@
 #include "yealink.h"
 
 #define DRIVER_VERSION "yld-20051230"
-#define DRIVER_AUTHOR "Henk Vergonet"
-#define DRIVER_DESC "Yealink phone driver"
 
 #define YEALINK_POLLING_FREQUENCY	10	/* in [Hz] */
 
@@ -1006,6 +1004,6 @@ module_usb_driver(yealink_driver);
 
 MODULE_DEVICE_TABLE (usb, usb_table);
 
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_AUTHOR("Henk Vergonet");
+MODULE_DESCRIPTION("Yealink phone driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/input/mouse/cyapa.c b/drivers/input/mouse/cyapa.c
index fd8865c65caf..dfd3873513e4 100644
--- a/drivers/input/mouse/cyapa.c
+++ b/drivers/input/mouse/cyapa.c
@@ -740,7 +740,7 @@ static ssize_t cyapa_show_suspend_scanrate(struct device *dev,
 					   char *buf)
 {
 	struct cyapa *cyapa = dev_get_drvdata(dev);
-	u8 pwr_cmd = cyapa->suspend_power_mode;
+	u8 pwr_cmd;
 	u16 sleep_time;
 	int len;
 	int error;
diff --git a/drivers/input/mouse/cyapa_gen3.c b/drivers/input/mouse/cyapa_gen3.c
index 1cbfa4a6e830..076dda4a66da 100644
--- a/drivers/input/mouse/cyapa_gen3.c
+++ b/drivers/input/mouse/cyapa_gen3.c
@@ -136,49 +136,6 @@ static const u8 bl_exit[] = { 0x00, 0xff, 0xa5, 0x00, 0x01, 0x02, 0x03, 0x04,
 		0x05, 0x06, 0x07 };
 
 
- /* for byte read/write command */
-#define CMD_RESET      0
-#define CMD_POWER_MODE 1
-#define CMD_DEV_STATUS 2
-#define CMD_REPORT_MAX_BASELINE 3
-#define CMD_REPORT_MIN_BASELINE 4
-#define SMBUS_BYTE_CMD(cmd) (((cmd) & 0x3f) << 1)
-#define CYAPA_SMBUS_RESET         SMBUS_BYTE_CMD(CMD_RESET)
-#define CYAPA_SMBUS_POWER_MODE    SMBUS_BYTE_CMD(CMD_POWER_MODE)
-#define CYAPA_SMBUS_DEV_STATUS    SMBUS_BYTE_CMD(CMD_DEV_STATUS)
-#define CYAPA_SMBUS_MAX_BASELINE  SMBUS_BYTE_CMD(CMD_REPORT_MAX_BASELINE)
-#define CYAPA_SMBUS_MIN_BASELINE  SMBUS_BYTE_CMD(CMD_REPORT_MIN_BASELINE)
-
- /* for group registers read/write command */
-#define REG_GROUP_DATA 0
-#define REG_GROUP_CMD 2
-#define REG_GROUP_QUERY 3
-#define SMBUS_GROUP_CMD(grp) (0x80 | (((grp) & 0x07) << 3))
-#define CYAPA_SMBUS_GROUP_DATA	 SMBUS_GROUP_CMD(REG_GROUP_DATA)
-#define CYAPA_SMBUS_GROUP_CMD	 SMBUS_GROUP_CMD(REG_GROUP_CMD)
-#define CYAPA_SMBUS_GROUP_QUERY	 SMBUS_GROUP_CMD(REG_GROUP_QUERY)
-
- /* for register block read/write command */
-#define CMD_BL_STATUS 0
-#define CMD_BL_HEAD 1
-#define CMD_BL_CMD 2
-#define CMD_BL_DATA 3
-#define CMD_BL_ALL 4
-#define CMD_BLK_PRODUCT_ID 5
-#define CMD_BLK_HEAD 6
-#define SMBUS_BLOCK_CMD(cmd) (0xc0 | (((cmd) & 0x1f) << 1))
-
-/* register block read/write command in bootloader mode */
-#define CYAPA_SMBUS_BL_STATUS  SMBUS_BLOCK_CMD(CMD_BL_STATUS)
-#define CYAPA_SMBUS_BL_HEAD    SMBUS_BLOCK_CMD(CMD_BL_HEAD)
-#define CYAPA_SMBUS_BL_CMD     SMBUS_BLOCK_CMD(CMD_BL_CMD)
-#define CYAPA_SMBUS_BL_DATA    SMBUS_BLOCK_CMD(CMD_BL_DATA)
-#define CYAPA_SMBUS_BL_ALL     SMBUS_BLOCK_CMD(CMD_BL_ALL)
-
-/* register block read/write command in operational mode */
-#define CYAPA_SMBUS_BLK_PRODUCT_ID SMBUS_BLOCK_CMD(CMD_BLK_PRODUCT_ID)
-#define CYAPA_SMBUS_BLK_HEAD SMBUS_BLOCK_CMD(CMD_BLK_HEAD)
-
  /* for byte read/write command */
 #define CMD_RESET 0
 #define CMD_POWER_MODE 1
diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c
index 2111a85d0b17..75e757520ef0 100644
--- a/drivers/input/mouse/elan_i2c_core.c
+++ b/drivers/input/mouse/elan_i2c_core.c
@@ -42,7 +42,6 @@
 #include "elan_i2c.h"
 
 #define DRIVER_NAME		"elan_i2c"
-#define ELAN_DRIVER_VERSION	"1.6.3"
 #define ELAN_VENDOR_ID		0x04f3
 #define ETP_MAX_PRESSURE	255
 #define ETP_FWIDTH_REDUCE	90
@@ -1294,4 +1293,3 @@ module_i2c_driver(elan_driver);
 MODULE_AUTHOR("Duson Lin <dusonlin@emc.com.tw>");
 MODULE_DESCRIPTION("Elan I2C/SMBus Touchpad driver");
 MODULE_LICENSE("GPL");
-MODULE_VERSION(ELAN_DRIVER_VERSION);
diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c
index 6a5649e52eed..8ac9e03c05b4 100644
--- a/drivers/input/mouse/psmouse-base.c
+++ b/drivers/input/mouse/psmouse-base.c
@@ -975,6 +975,21 @@ static void psmouse_apply_defaults(struct psmouse *psmouse)
 	psmouse->pt_deactivate = NULL;
 }
 
+static bool psmouse_do_detect(int (*detect)(struct psmouse *, bool),
+			      struct psmouse *psmouse, bool allow_passthrough,
+			      bool set_properties)
+{
+	if (psmouse->ps2dev.serio->id.type == SERIO_PS_PSTHRU &&
+	    !allow_passthrough) {
+		return false;
+	}
+
+	if (set_properties)
+		psmouse_apply_defaults(psmouse);
+
+	return detect(psmouse, set_properties) == 0;
+}
+
 static bool psmouse_try_protocol(struct psmouse *psmouse,
 				 enum psmouse_type type,
 				 unsigned int *max_proto,
@@ -986,15 +1001,8 @@ static bool psmouse_try_protocol(struct psmouse *psmouse,
 	if (!proto)
 		return false;
 
-	if (psmouse->ps2dev.serio->id.type == SERIO_PS_PSTHRU &&
-	    !proto->try_passthru) {
-		return false;
-	}
-
-	if (set_properties)
-		psmouse_apply_defaults(psmouse);
-
-	if (proto->detect(psmouse, set_properties) != 0)
+	if (!psmouse_do_detect(proto->detect, psmouse, proto->try_passthru,
+			       set_properties))
 		return false;
 
 	if (set_properties && proto->init && init_allowed) {
@@ -1027,8 +1035,8 @@ static int psmouse_extensions(struct psmouse *psmouse,
 	 * Always check for focaltech, this is safe as it uses pnp-id
 	 * matching.
 	 */
-	if (psmouse_try_protocol(psmouse, PSMOUSE_FOCALTECH,
-				 &max_proto, set_properties, false)) {
+	if (psmouse_do_detect(focaltech_detect,
+			      psmouse, false, set_properties)) {
 		if (max_proto > PSMOUSE_IMEX &&
 		    IS_ENABLED(CONFIG_MOUSE_PS2_FOCALTECH) &&
 		    (!set_properties || focaltech_init(psmouse) == 0)) {
@@ -1074,8 +1082,8 @@ static int psmouse_extensions(struct psmouse *psmouse,
 	 * probing for IntelliMouse.
 	 */
 	if (max_proto > PSMOUSE_PS2 &&
-	    psmouse_try_protocol(psmouse, PSMOUSE_SYNAPTICS, &max_proto,
-				 set_properties, false)) {
+	    psmouse_do_detect(synaptics_detect,
+			      psmouse, false, set_properties)) {
 		synaptics_hardware = true;
 
 		if (max_proto > PSMOUSE_IMEX) {
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index cd9f61cb3fc6..a246fc686bb7 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -173,7 +173,6 @@ static const char * const smbus_pnp_ids[] = {
 	"LEN0046", /* X250 */
 	"LEN004a", /* W541 */
 	"LEN200f", /* T450s */
-	"LEN2018", /* T460p */
 	NULL
 };
 
@@ -1281,6 +1280,16 @@ static void set_input_params(struct psmouse *psmouse,
 				    INPUT_MT_POINTER |
 				    (cr48_profile_sensor ?
 					INPUT_MT_TRACK : INPUT_MT_SEMI_MT));
+
+		/*
+		 * For semi-mt devices we send ABS_X/Y ourselves instead of
+		 * input_mt_report_pointer_emulation. But
+		 * input_mt_init_slots() resets the fuzz to 0, leading to a
+		 * filtered ABS_MT_POSITION_X but an unfiltered ABS_X
+		 * position. Let's re-initialize ABS_X/Y here.
+		 */
+		if (!cr48_profile_sensor)
+			set_abs_position_params(dev, &priv->info, ABS_X, ABS_Y);
 	}
 
 	if (SYN_CAP_PALMDETECT(info->capabilities))
diff --git a/drivers/input/mousedev.c b/drivers/input/mousedev.c
index 2d7f691ec71c..e08228061bcd 100644
--- a/drivers/input/mousedev.c
+++ b/drivers/input/mousedev.c
@@ -757,17 +757,17 @@ static ssize_t mousedev_read(struct file *file, char __user *buffer,
 }
 
 /* No kernel lock - fine */
-static unsigned int mousedev_poll(struct file *file, poll_table *wait)
+static __poll_t mousedev_poll(struct file *file, poll_table *wait)
 {
 	struct mousedev_client *client = file->private_data;
 	struct mousedev *mousedev = client->mousedev;
-	unsigned int mask;
+	__poll_t mask;
 
 	poll_wait(file, &mousedev->wait, wait);
 
-	mask = mousedev->exist ? POLLOUT | POLLWRNORM : POLLHUP | POLLERR;
+	mask = mousedev->exist ? EPOLLOUT | EPOLLWRNORM : EPOLLHUP | EPOLLERR;
 	if (client->ready || client->buffer)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/input/rmi4/rmi_bus.c b/drivers/input/rmi4/rmi_bus.c
index ae1bffe45c75..c5fa53adba8d 100644
--- a/drivers/input/rmi4/rmi_bus.c
+++ b/drivers/input/rmi4/rmi_bus.c
@@ -427,4 +427,3 @@ MODULE_AUTHOR("Christopher Heiny <cheiny@synaptics.com");
 MODULE_AUTHOR("Andrew Duggan <aduggan@synaptics.com");
 MODULE_DESCRIPTION("RMI bus");
 MODULE_LICENSE("GPL");
-MODULE_VERSION(RMI_DRIVER_VERSION);
diff --git a/drivers/input/rmi4/rmi_driver.c b/drivers/input/rmi4/rmi_driver.c
index 141ea228aac6..f5954981e9ee 100644
--- a/drivers/input/rmi4/rmi_driver.c
+++ b/drivers/input/rmi4/rmi_driver.c
@@ -41,6 +41,13 @@ void rmi_free_function_list(struct rmi_device *rmi_dev)
 
 	rmi_dbg(RMI_DEBUG_CORE, &rmi_dev->dev, "Freeing function list\n");
 
+	/* Doing it in the reverse order so F01 will be removed last */
+	list_for_each_entry_safe_reverse(fn, tmp,
+					 &data->function_list, node) {
+		list_del(&fn->node);
+		rmi_unregister_function(fn);
+	}
+
 	devm_kfree(&rmi_dev->dev, data->irq_memory);
 	data->irq_memory = NULL;
 	data->irq_status = NULL;
@@ -50,13 +57,6 @@ void rmi_free_function_list(struct rmi_device *rmi_dev)
 
 	data->f01_container = NULL;
 	data->f34_container = NULL;
-
-	/* Doing it in the reverse order so F01 will be removed last */
-	list_for_each_entry_safe_reverse(fn, tmp,
-					 &data->function_list, node) {
-		list_del(&fn->node);
-		rmi_unregister_function(fn);
-	}
 }
 
 static int reset_one_function(struct rmi_function *fn)
diff --git a/drivers/input/rmi4/rmi_driver.h b/drivers/input/rmi4/rmi_driver.h
index f1a2a2266022..d31793ae83f0 100644
--- a/drivers/input/rmi4/rmi_driver.h
+++ b/drivers/input/rmi4/rmi_driver.h
@@ -16,8 +16,6 @@
 #include <linux/input.h>
 #include "rmi_bus.h"
 
-#define RMI_DRIVER_VERSION "2.0"
-
 #define SYNAPTICS_INPUT_DEVICE_NAME "Synaptics RMI4 Touch Sensor"
 #define SYNAPTICS_VENDOR_ID 0x06cb
 
diff --git a/drivers/input/rmi4/rmi_f03.c b/drivers/input/rmi4/rmi_f03.c
index ad71a5e768dc..88822196d6b7 100644
--- a/drivers/input/rmi4/rmi_f03.c
+++ b/drivers/input/rmi4/rmi_f03.c
@@ -32,6 +32,7 @@ struct f03_data {
 	struct rmi_function *fn;
 
 	struct serio *serio;
+	bool serio_registered;
 
 	unsigned int overwrite_buttons;
 
@@ -138,6 +139,37 @@ static int rmi_f03_initialize(struct f03_data *f03)
 	return 0;
 }
 
+static int rmi_f03_pt_open(struct serio *serio)
+{
+	struct f03_data *f03 = serio->port_data;
+	struct rmi_function *fn = f03->fn;
+	const u8 ob_len = f03->rx_queue_length * RMI_F03_OB_SIZE;
+	const u16 data_addr = fn->fd.data_base_addr + RMI_F03_OB_OFFSET;
+	u8 obs[RMI_F03_QUEUE_LENGTH * RMI_F03_OB_SIZE];
+	int error;
+
+	/*
+	 * Consume any pending data. Some devices like to spam with
+	 * 0xaa 0x00 announcements which may confuse us as we try to
+	 * probe the device.
+	 */
+	error = rmi_read_block(fn->rmi_dev, data_addr, &obs, ob_len);
+	if (!error)
+		rmi_dbg(RMI_DEBUG_FN, &fn->dev,
+			"%s: Consumed %*ph (%d) from PS2 guest\n",
+			__func__, ob_len, obs, ob_len);
+
+	return fn->rmi_dev->driver->set_irq_bits(fn->rmi_dev, fn->irq_mask);
+}
+
+static void rmi_f03_pt_close(struct serio *serio)
+{
+	struct f03_data *f03 = serio->port_data;
+	struct rmi_function *fn = f03->fn;
+
+	fn->rmi_dev->driver->clear_irq_bits(fn->rmi_dev, fn->irq_mask);
+}
+
 static int rmi_f03_register_pt(struct f03_data *f03)
 {
 	struct serio *serio;
@@ -148,16 +180,19 @@ static int rmi_f03_register_pt(struct f03_data *f03)
 
 	serio->id.type = SERIO_PS_PSTHRU;
 	serio->write = rmi_f03_pt_write;
+	serio->open = rmi_f03_pt_open;
+	serio->close = rmi_f03_pt_close;
 	serio->port_data = f03;
 
-	strlcpy(serio->name, "Synaptics RMI4 PS/2 pass-through",
-		sizeof(serio->name));
-	strlcpy(serio->phys, "synaptics-rmi4-pt/serio1",
-		sizeof(serio->phys));
+	strlcpy(serio->name, "RMI4 PS/2 pass-through", sizeof(serio->name));
+	snprintf(serio->phys, sizeof(serio->phys), "%s/serio0",
+		 dev_name(&f03->fn->dev));
 	serio->dev.parent = &f03->fn->dev;
 
 	f03->serio = serio;
 
+	printk(KERN_INFO "serio: %s port at %s\n",
+		serio->name, dev_name(&f03->fn->dev));
 	serio_register_port(serio);
 
 	return 0;
@@ -184,17 +219,27 @@ static int rmi_f03_probe(struct rmi_function *fn)
 			 f03->device_count);
 
 	dev_set_drvdata(dev, f03);
-
-	error = rmi_f03_register_pt(f03);
-	if (error)
-		return error;
-
 	return 0;
 }
 
 static int rmi_f03_config(struct rmi_function *fn)
 {
-	fn->rmi_dev->driver->set_irq_bits(fn->rmi_dev, fn->irq_mask);
+	struct f03_data *f03 = dev_get_drvdata(&fn->dev);
+	int error;
+
+	if (!f03->serio_registered) {
+		error = rmi_f03_register_pt(f03);
+		if (error)
+			return error;
+
+		f03->serio_registered = true;
+	} else {
+		/*
+		 * We must be re-configuring the sensor, just enable
+		 * interrupts for this function.
+		 */
+		fn->rmi_dev->driver->set_irq_bits(fn->rmi_dev, fn->irq_mask);
+	}
 
 	return 0;
 }
@@ -204,7 +249,7 @@ static int rmi_f03_attention(struct rmi_function *fn, unsigned long *irq_bits)
 	struct rmi_device *rmi_dev = fn->rmi_dev;
 	struct rmi_driver_data *drvdata = dev_get_drvdata(&rmi_dev->dev);
 	struct f03_data *f03 = dev_get_drvdata(&fn->dev);
-	u16 data_addr = fn->fd.data_base_addr;
+	const u16 data_addr = fn->fd.data_base_addr + RMI_F03_OB_OFFSET;
 	const u8 ob_len = f03->rx_queue_length * RMI_F03_OB_SIZE;
 	u8 obs[RMI_F03_QUEUE_LENGTH * RMI_F03_OB_SIZE];
 	u8 ob_status;
@@ -226,8 +271,7 @@ static int rmi_f03_attention(struct rmi_function *fn, unsigned long *irq_bits)
 		drvdata->attn_data.size -= ob_len;
 	} else {
 		/* Grab all of the data registers, and check them for data */
-		error = rmi_read_block(fn->rmi_dev, data_addr + RMI_F03_OB_OFFSET,
-				       &obs, ob_len);
+		error = rmi_read_block(fn->rmi_dev, data_addr, &obs, ob_len);
 		if (error) {
 			dev_err(&fn->dev,
 				"%s: Failed to read F03 output buffers: %d\n",
@@ -266,7 +310,8 @@ static void rmi_f03_remove(struct rmi_function *fn)
 {
 	struct f03_data *f03 = dev_get_drvdata(&fn->dev);
 
-	serio_unregister_port(f03->serio);
+	if (f03->serio_registered)
+		serio_unregister_port(f03->serio);
 }
 
 struct rmi_function_handler rmi_f03_handler = {
diff --git a/drivers/input/rmi4/rmi_f34.c b/drivers/input/rmi4/rmi_f34.c
index 4cfe9703a8e7..f1f5ac539d5d 100644
--- a/drivers/input/rmi4/rmi_f34.c
+++ b/drivers/input/rmi4/rmi_f34.c
@@ -11,7 +11,6 @@
 #include <linux/rmi.h>
 #include <linux/firmware.h>
 #include <asm/unaligned.h>
-#include <asm/unaligned.h>
 #include <linux/bitops.h>
 
 #include "rmi_driver.h"
diff --git a/drivers/input/rmi4/rmi_i2c.c b/drivers/input/rmi4/rmi_i2c.c
index e28663ef9e5a..d4b3f9d0dc2e 100644
--- a/drivers/input/rmi4/rmi_i2c.c
+++ b/drivers/input/rmi4/rmi_i2c.c
@@ -391,4 +391,3 @@ MODULE_AUTHOR("Christopher Heiny <cheiny@synaptics.com>");
 MODULE_AUTHOR("Andrew Duggan <aduggan@synaptics.com>");
 MODULE_DESCRIPTION("RMI I2C driver");
 MODULE_LICENSE("GPL");
-MODULE_VERSION(RMI_DRIVER_VERSION);
diff --git a/drivers/input/rmi4/rmi_spi.c b/drivers/input/rmi4/rmi_spi.c
index d97a85907ed6..76edbf2c1bce 100644
--- a/drivers/input/rmi4/rmi_spi.c
+++ b/drivers/input/rmi4/rmi_spi.c
@@ -528,4 +528,3 @@ MODULE_AUTHOR("Christopher Heiny <cheiny@synaptics.com>");
 MODULE_AUTHOR("Andrew Duggan <aduggan@synaptics.com>");
 MODULE_DESCRIPTION("RMI SPI driver");
 MODULE_LICENSE("GPL");
-MODULE_VERSION(RMI_DRIVER_VERSION);
diff --git a/drivers/input/serio/Kconfig b/drivers/input/serio/Kconfig
index 21488c048fa3..ca4530eb3378 100644
--- a/drivers/input/serio/Kconfig
+++ b/drivers/input/serio/Kconfig
@@ -96,16 +96,6 @@ config SERIO_RPCKBD
 	  To compile this driver as a module, choose M here: the
 	  module will be called rpckbd.
 
-config SERIO_AT32PSIF
-	tristate "AVR32 PSIF PS/2 keyboard and mouse controller"
-	depends on AVR32
-	help
-	  Say Y here if you want to use the PSIF peripheral on AVR32 devices
-	  and connect a PS/2 keyboard and/or mouse to it.
-
-	  To compile this driver as a module, choose M here: the module will
-	  be called at32psif.
-
 config SERIO_AMBAKMI
 	tristate "AMBA KMI keyboard controller"
 	depends on ARM_AMBA
diff --git a/drivers/input/serio/Makefile b/drivers/input/serio/Makefile
index a3ca07621542..67950a5ccb3f 100644
--- a/drivers/input/serio/Makefile
+++ b/drivers/input/serio/Makefile
@@ -13,7 +13,6 @@ obj-$(CONFIG_SERIO_CT82C710)	+= ct82c710.o
 obj-$(CONFIG_SERIO_RPCKBD)	+= rpckbd.o
 obj-$(CONFIG_SERIO_SA1111)	+= sa1111ps2.o
 obj-$(CONFIG_SERIO_AMBAKMI)	+= ambakmi.o
-obj-$(CONFIG_SERIO_AT32PSIF)	+= at32psif.o
 obj-$(CONFIG_SERIO_Q40KBD)	+= q40kbd.o
 obj-$(CONFIG_SERIO_GSCPS2)	+= gscps2.o
 obj-$(CONFIG_HP_SDC)		+= hp_sdc.o
diff --git a/drivers/input/serio/at32psif.c b/drivers/input/serio/at32psif.c
deleted file mode 100644
index e420fd781d44..000000000000
--- a/drivers/input/serio/at32psif.c
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
- * Copyright (C) 2007 Atmel Corporation
- *
- * Driver for the AT32AP700X PS/2 controller (PSIF).
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published
- * by the Free Software Foundation.
- */
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/device.h>
-#include <linux/init.h>
-#include <linux/serio.h>
-#include <linux/interrupt.h>
-#include <linux/delay.h>
-#include <linux/err.h>
-#include <linux/io.h>
-#include <linux/clk.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-/* PSIF register offsets */
-#define PSIF_CR				0x00
-#define PSIF_RHR			0x04
-#define PSIF_THR			0x08
-#define PSIF_SR				0x10
-#define PSIF_IER			0x14
-#define PSIF_IDR			0x18
-#define PSIF_IMR			0x1c
-#define PSIF_PSR			0x24
-
-/* Bitfields in control register. */
-#define PSIF_CR_RXDIS_OFFSET		1
-#define PSIF_CR_RXDIS_SIZE		1
-#define PSIF_CR_RXEN_OFFSET		0
-#define PSIF_CR_RXEN_SIZE		1
-#define PSIF_CR_SWRST_OFFSET		15
-#define PSIF_CR_SWRST_SIZE		1
-#define PSIF_CR_TXDIS_OFFSET		9
-#define PSIF_CR_TXDIS_SIZE		1
-#define PSIF_CR_TXEN_OFFSET		8
-#define PSIF_CR_TXEN_SIZE		1
-
-/* Bitfields in interrupt disable, enable, mask and status register. */
-#define PSIF_NACK_OFFSET		8
-#define PSIF_NACK_SIZE			1
-#define PSIF_OVRUN_OFFSET		5
-#define PSIF_OVRUN_SIZE			1
-#define PSIF_PARITY_OFFSET		9
-#define PSIF_PARITY_SIZE		1
-#define PSIF_RXRDY_OFFSET		4
-#define PSIF_RXRDY_SIZE			1
-#define PSIF_TXEMPTY_OFFSET		1
-#define PSIF_TXEMPTY_SIZE		1
-#define PSIF_TXRDY_OFFSET		0
-#define PSIF_TXRDY_SIZE			1
-
-/* Bitfields in prescale register. */
-#define PSIF_PSR_PRSCV_OFFSET		0
-#define PSIF_PSR_PRSCV_SIZE		12
-
-/* Bitfields in receive hold register. */
-#define PSIF_RHR_RXDATA_OFFSET		0
-#define PSIF_RHR_RXDATA_SIZE		8
-
-/* Bitfields in transmit hold register. */
-#define PSIF_THR_TXDATA_OFFSET		0
-#define PSIF_THR_TXDATA_SIZE		8
-
-/* Bit manipulation macros */
-#define PSIF_BIT(name)					\
-	(1 << PSIF_##name##_OFFSET)
-
-#define PSIF_BF(name, value)				\
-	(((value) & ((1 << PSIF_##name##_SIZE) - 1))	\
-	 << PSIF_##name##_OFFSET)
-
-#define PSIF_BFEXT(name, value)				\
-	(((value) >> PSIF_##name##_OFFSET)		\
-	 & ((1 << PSIF_##name##_SIZE) - 1))
-
-#define PSIF_BFINS(name, value, old)			\
-	(((old) & ~(((1 << PSIF_##name##_SIZE) - 1)	\
-		    << PSIF_##name##_OFFSET))		\
-	 | PSIF_BF(name, value))
-
-/* Register access macros */
-#define psif_readl(port, reg)				\
-	__raw_readl((port)->regs + PSIF_##reg)
-
-#define psif_writel(port, reg, value)			\
-	__raw_writel((value), (port)->regs + PSIF_##reg)
-
-struct psif {
-	struct platform_device	*pdev;
-	struct clk		*pclk;
-	struct serio		*io;
-	void __iomem		*regs;
-	unsigned int		irq;
-	/* Prevent concurrent writes to PSIF THR. */
-	spinlock_t		lock;
-	bool			open;
-};
-
-static irqreturn_t psif_interrupt(int irq, void *_ptr)
-{
-	struct psif *psif = _ptr;
-	int retval = IRQ_NONE;
-	unsigned int io_flags = 0;
-	unsigned long status;
-
-	status = psif_readl(psif, SR);
-
-	if (status & PSIF_BIT(RXRDY)) {
-		unsigned char val = (unsigned char) psif_readl(psif, RHR);
-
-		if (status & PSIF_BIT(PARITY))
-			io_flags |= SERIO_PARITY;
-		if (status & PSIF_BIT(OVRUN))
-			dev_err(&psif->pdev->dev, "overrun read error\n");
-
-		serio_interrupt(psif->io, val, io_flags);
-
-		retval = IRQ_HANDLED;
-	}
-
-	return retval;
-}
-
-static int psif_write(struct serio *io, unsigned char val)
-{
-	struct psif *psif = io->port_data;
-	unsigned long flags;
-	int timeout = 10;
-	int retval = 0;
-
-	spin_lock_irqsave(&psif->lock, flags);
-
-	while (!(psif_readl(psif, SR) & PSIF_BIT(TXEMPTY)) && timeout--)
-		udelay(50);
-
-	if (timeout >= 0) {
-		psif_writel(psif, THR, val);
-	} else {
-		dev_dbg(&psif->pdev->dev, "timeout writing to THR\n");
-		retval = -EBUSY;
-	}
-
-	spin_unlock_irqrestore(&psif->lock, flags);
-
-	return retval;
-}
-
-static int psif_open(struct serio *io)
-{
-	struct psif *psif = io->port_data;
-	int retval;
-
-	retval = clk_enable(psif->pclk);
-	if (retval)
-		return retval;
-
-	psif_writel(psif, CR, PSIF_BIT(CR_TXEN) | PSIF_BIT(CR_RXEN));
-	psif_writel(psif, IER, PSIF_BIT(RXRDY));
-
-	psif->open = true;
-	return retval;
-}
-
-static void psif_close(struct serio *io)
-{
-	struct psif *psif = io->port_data;
-
-	psif->open = false;
-
-	psif_writel(psif, IDR, ~0UL);
-	psif_writel(psif, CR, PSIF_BIT(CR_TXDIS) | PSIF_BIT(CR_RXDIS));
-
-	clk_disable(psif->pclk);
-}
-
-static void psif_set_prescaler(struct psif *psif)
-{
-	unsigned long prscv;
-	unsigned long rate = clk_get_rate(psif->pclk);
-
-	/* PRSCV = Pulse length (100 us) * PSIF module frequency. */
-	prscv = 100 * (rate / 1000000UL);
-
-	if (prscv > ((1<<PSIF_PSR_PRSCV_SIZE) - 1)) {
-		prscv = (1<<PSIF_PSR_PRSCV_SIZE) - 1;
-		dev_dbg(&psif->pdev->dev, "pclk too fast, "
-				"prescaler set to max\n");
-	}
-
-	clk_enable(psif->pclk);
-	psif_writel(psif, PSR, prscv);
-	clk_disable(psif->pclk);
-}
-
-static int __init psif_probe(struct platform_device *pdev)
-{
-	struct resource *regs;
-	struct psif *psif;
-	struct serio *io;
-	struct clk *pclk;
-	int irq;
-	int ret;
-
-	psif = kzalloc(sizeof(struct psif), GFP_KERNEL);
-	if (!psif)
-		return -ENOMEM;
-	psif->pdev = pdev;
-
-	io = kzalloc(sizeof(struct serio), GFP_KERNEL);
-	if (!io) {
-		ret = -ENOMEM;
-		goto out_free_psif;
-	}
-	psif->io = io;
-
-	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!regs) {
-		dev_dbg(&pdev->dev, "no mmio resources defined\n");
-		ret = -ENOMEM;
-		goto out_free_io;
-	}
-
-	psif->regs = ioremap(regs->start, resource_size(regs));
-	if (!psif->regs) {
-		ret = -ENOMEM;
-		dev_dbg(&pdev->dev, "could not map I/O memory\n");
-		goto out_free_io;
-	}
-
-	pclk = clk_get(&pdev->dev, "pclk");
-	if (IS_ERR(pclk)) {
-		dev_dbg(&pdev->dev, "could not get peripheral clock\n");
-		ret = PTR_ERR(pclk);
-		goto out_iounmap;
-	}
-	psif->pclk = pclk;
-
-	/* Reset the PSIF to enter at a known state. */
-	ret = clk_enable(pclk);
-	if (ret) {
-		dev_dbg(&pdev->dev, "could not enable pclk\n");
-		goto out_put_clk;
-	}
-	psif_writel(psif, CR, PSIF_BIT(CR_SWRST));
-	clk_disable(pclk);
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_dbg(&pdev->dev, "could not get irq\n");
-		ret = -ENXIO;
-		goto out_put_clk;
-	}
-	ret = request_irq(irq, psif_interrupt, IRQF_SHARED, "at32psif", psif);
-	if (ret) {
-		dev_dbg(&pdev->dev, "could not request irq %d\n", irq);
-		goto out_put_clk;
-	}
-	psif->irq = irq;
-
-	io->id.type	= SERIO_8042;
-	io->write	= psif_write;
-	io->open	= psif_open;
-	io->close	= psif_close;
-	snprintf(io->name, sizeof(io->name), "AVR32 PS/2 port%d", pdev->id);
-	snprintf(io->phys, sizeof(io->phys), "at32psif/serio%d", pdev->id);
-	io->port_data	= psif;
-	io->dev.parent	= &pdev->dev;
-
-	psif_set_prescaler(psif);
-
-	spin_lock_init(&psif->lock);
-	serio_register_port(psif->io);
-	platform_set_drvdata(pdev, psif);
-
-	dev_info(&pdev->dev, "Atmel AVR32 PSIF PS/2 driver on 0x%08x irq %d\n",
-			(int)psif->regs, psif->irq);
-
-	return 0;
-
-out_put_clk:
-	clk_put(psif->pclk);
-out_iounmap:
-	iounmap(psif->regs);
-out_free_io:
-	kfree(io);
-out_free_psif:
-	kfree(psif);
-	return ret;
-}
-
-static int __exit psif_remove(struct platform_device *pdev)
-{
-	struct psif *psif = platform_get_drvdata(pdev);
-
-	psif_writel(psif, IDR, ~0UL);
-	psif_writel(psif, CR, PSIF_BIT(CR_TXDIS) | PSIF_BIT(CR_RXDIS));
-
-	serio_unregister_port(psif->io);
-	iounmap(psif->regs);
-	free_irq(psif->irq, psif);
-	clk_put(psif->pclk);
-	kfree(psif);
-
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int psif_suspend(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct psif *psif = platform_get_drvdata(pdev);
-
-	if (psif->open) {
-		psif_writel(psif, CR, PSIF_BIT(CR_RXDIS) | PSIF_BIT(CR_TXDIS));
-		clk_disable(psif->pclk);
-	}
-
-	return 0;
-}
-
-static int psif_resume(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct psif *psif = platform_get_drvdata(pdev);
-
-	if (psif->open) {
-		clk_enable(psif->pclk);
-		psif_set_prescaler(psif);
-		psif_writel(psif, CR, PSIF_BIT(CR_RXEN) | PSIF_BIT(CR_TXEN));
-	}
-
-	return 0;
-}
-#endif
-
-static SIMPLE_DEV_PM_OPS(psif_pm_ops, psif_suspend, psif_resume);
-
-static struct platform_driver psif_driver = {
-	.remove		= __exit_p(psif_remove),
-	.driver		= {
-		.name	= "atmel_psif",
-		.pm	= &psif_pm_ops,
-	},
-};
-
-module_platform_driver_probe(psif_driver, psif_probe);
-
-MODULE_AUTHOR("Hans-Christian Egtvedt <egtvedt@samfundet.no>");
-MODULE_DESCRIPTION("Atmel AVR32 PSIF PS/2 driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/input/serio/hil_mlc.c b/drivers/input/serio/hil_mlc.c
index d66d01c5373b..e1423f7648d6 100644
--- a/drivers/input/serio/hil_mlc.c
+++ b/drivers/input/serio/hil_mlc.c
@@ -274,14 +274,12 @@ static int hilse_match(hil_mlc *mlc, int unused)
 /* An LCV used to prevent runaway loops, forces 5 second sleep when reset. */
 static int hilse_init_lcv(hil_mlc *mlc, int unused)
 {
-	struct timeval tv;
+	time64_t now = ktime_get_seconds();
 
-	do_gettimeofday(&tv);
-
-	if (mlc->lcv && (tv.tv_sec - mlc->lcv_tv.tv_sec) < 5)
+	if (mlc->lcv && (now - mlc->lcv_time) < 5)
 		return -1;
 
-	mlc->lcv_tv = tv;
+	mlc->lcv_time = now;
 	mlc->lcv = 0;
 
 	return 0;
@@ -604,8 +602,8 @@ static inline void hilse_setup_input(hil_mlc *mlc, const struct hilse_node *node
 		BUG();
 	}
 	mlc->istarted = 1;
-	mlc->intimeout = node->arg;
-	do_gettimeofday(&(mlc->instart));
+	mlc->intimeout = usecs_to_jiffies(node->arg);
+	mlc->instart = jiffies;
 	mlc->icount = 15;
 	memset(mlc->ipacket, 0, 16 * sizeof(hil_packet));
 	BUG_ON(down_trylock(&mlc->isem));
@@ -710,7 +708,7 @@ static int hilse_donode(hil_mlc *mlc)
 			break;
 		}
 		mlc->ostarted = 0;
-		do_gettimeofday(&(mlc->instart));
+		mlc->instart = jiffies;
 		write_unlock_irqrestore(&mlc->lock, flags);
 		nextidx = HILSEN_NEXT;
 		break;
@@ -731,18 +729,14 @@ static int hilse_donode(hil_mlc *mlc)
 #endif
 
 	while (nextidx & HILSEN_SCHED) {
-		struct timeval tv;
+		unsigned long now = jiffies;
 
 		if (!sched_long)
 			goto sched;
 
-		do_gettimeofday(&tv);
-		tv.tv_usec += USEC_PER_SEC * (tv.tv_sec - mlc->instart.tv_sec);
-		tv.tv_usec -= mlc->instart.tv_usec;
-		if (tv.tv_usec >= mlc->intimeout) goto sched;
-		tv.tv_usec = (mlc->intimeout - tv.tv_usec) * HZ / USEC_PER_SEC;
-		if (!tv.tv_usec) goto sched;
-		mod_timer(&hil_mlcs_kicker, jiffies + tv.tv_usec);
+		if (time_after(now, mlc->instart + mlc->intimeout))
+			 goto sched;
+		mod_timer(&hil_mlcs_kicker, mlc->instart + mlc->intimeout);
 		break;
 	sched:
 		tasklet_schedule(&hil_mlcs_tasklet);
diff --git a/drivers/input/serio/hp_sdc.c b/drivers/input/serio/hp_sdc.c
index 1d7c7d81a5ef..0b8a25c58d02 100644
--- a/drivers/input/serio/hp_sdc.c
+++ b/drivers/input/serio/hp_sdc.c
@@ -193,7 +193,7 @@ static void hp_sdc_take(int irq, void *dev_id, uint8_t status, uint8_t data)
 	curr->seq[curr->idx++] = status;
 	curr->seq[curr->idx++] = data;
 	hp_sdc.rqty -= 2;
-	do_gettimeofday(&hp_sdc.rtv);
+	hp_sdc.rtime = ktime_get();
 
 	if (hp_sdc.rqty <= 0) {
 		/* All data has been gathered. */
@@ -306,13 +306,10 @@ static void hp_sdc_tasklet(unsigned long foo)
 	write_lock_irq(&hp_sdc.rtq_lock);
 
 	if (hp_sdc.rcurr >= 0) {
-		struct timeval tv;
+		ktime_t now = ktime_get();
 
-		do_gettimeofday(&tv);
-		if (tv.tv_sec > hp_sdc.rtv.tv_sec)
-			tv.tv_usec += USEC_PER_SEC;
-
-		if (tv.tv_usec - hp_sdc.rtv.tv_usec > HP_SDC_MAX_REG_DELAY) {
+		if (ktime_after(now, ktime_add_us(hp_sdc.rtime,
+						  HP_SDC_MAX_REG_DELAY))) {
 			hp_sdc_transaction *curr;
 			uint8_t tmp;
 
@@ -321,8 +318,8 @@ static void hp_sdc_tasklet(unsigned long foo)
 			 * we'll need to figure out a way to communicate
 			 * it back to the application. and be less verbose.
 			 */
-			printk(KERN_WARNING PREFIX "read timeout (%ius)!\n",
-			       (int)(tv.tv_usec - hp_sdc.rtv.tv_usec));
+			printk(KERN_WARNING PREFIX "read timeout (%lldus)!\n",
+			       ktime_us_delta(now, hp_sdc.rtime));
 			curr->idx += hp_sdc.rqty;
 			hp_sdc.rqty = 0;
 			tmp = curr->seq[curr->actidx];
@@ -551,7 +548,7 @@ unsigned long hp_sdc_put(void)
 
 			/* Start a new read */
 			hp_sdc.rqty = curr->seq[curr->idx];
-			do_gettimeofday(&hp_sdc.rtv);
+			hp_sdc.rtime = ktime_get();
 			curr->idx++;
 			/* Still need to lock here in case of spurious irq. */
 			write_lock_irq(&hp_sdc.rtq_lock);
diff --git a/drivers/input/serio/hp_sdc_mlc.c b/drivers/input/serio/hp_sdc_mlc.c
index d50f0678bf47..232d30c825bd 100644
--- a/drivers/input/serio/hp_sdc_mlc.c
+++ b/drivers/input/serio/hp_sdc_mlc.c
@@ -149,7 +149,6 @@ static int hp_sdc_mlc_in(hil_mlc *mlc, suseconds_t timeout)
 
 	/* Try to down the semaphore */
 	if (down_trylock(&mlc->isem)) {
-		struct timeval tv;
 		if (priv->emtestmode) {
 			mlc->ipacket[0] =
 				HIL_ERR_INT | (mlc->opacket &
@@ -160,9 +159,7 @@ static int hp_sdc_mlc_in(hil_mlc *mlc, suseconds_t timeout)
 			/* printk(KERN_DEBUG PREFIX ">[%x]\n", mlc->ipacket[0]); */
 			goto wasup;
 		}
-		do_gettimeofday(&tv);
-		tv.tv_usec += USEC_PER_SEC * (tv.tv_sec - mlc->instart.tv_sec);
-		if (tv.tv_usec - mlc->instart.tv_usec > mlc->intimeout) {
+		if (time_after(jiffies, mlc->instart + mlc->intimeout)) {
 			/*	printk("!%i %i",
 				tv.tv_usec - mlc->instart.tv_usec,
 				mlc->intimeout);
diff --git a/drivers/input/serio/serio_raw.c b/drivers/input/serio/serio_raw.c
index 516f9fe77a17..17b7fbecd9fe 100644
--- a/drivers/input/serio/serio_raw.c
+++ b/drivers/input/serio/serio_raw.c
@@ -239,17 +239,17 @@ static ssize_t serio_raw_write(struct file *file, const char __user *buffer,
 	return retval;
 }
 
-static unsigned int serio_raw_poll(struct file *file, poll_table *wait)
+static __poll_t serio_raw_poll(struct file *file, poll_table *wait)
 {
 	struct serio_raw_client *client = file->private_data;
 	struct serio_raw *serio_raw = client->serio_raw;
-	unsigned int mask;
+	__poll_t mask;
 
 	poll_wait(file, &serio_raw->wait, wait);
 
-	mask = serio_raw->dead ? POLLHUP | POLLERR : POLLOUT | POLLWRNORM;
+	mask = serio_raw->dead ? EPOLLHUP | EPOLLERR : EPOLLOUT | EPOLLWRNORM;
 	if (serio_raw->head != serio_raw->tail)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/input/serio/userio.c b/drivers/input/serio/userio.c
index df1fd41860ac..9ab5c45c3a9f 100644
--- a/drivers/input/serio/userio.c
+++ b/drivers/input/serio/userio.c
@@ -248,14 +248,14 @@ static ssize_t userio_char_write(struct file *file, const char __user *buffer,
 	return error ?: count;
 }
 
-static unsigned int userio_char_poll(struct file *file, poll_table *wait)
+static __poll_t userio_char_poll(struct file *file, poll_table *wait)
 {
 	struct userio_device *userio = file->private_data;
 
 	poll_wait(file, &userio->waitq, wait);
 
 	if (userio->head != userio->tail)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/drivers/input/sparse-keymap.c b/drivers/input/sparse-keymap.c
index fd03e55768c9..0cad5e7c559b 100644
--- a/drivers/input/sparse-keymap.c
+++ b/drivers/input/sparse-keymap.c
@@ -21,7 +21,6 @@
 MODULE_AUTHOR("Dmitry Torokhov <dtor@mail.ru>");
 MODULE_DESCRIPTION("Generic support for sparse keymaps");
 MODULE_LICENSE("GPL v2");
-MODULE_VERSION("0.1");
 
 static unsigned int sparse_keymap_get_key_index(struct input_dev *dev,
 						const struct key_entry *k)
diff --git a/drivers/input/tablet/acecad.c b/drivers/input/tablet/acecad.c
index aebb3f9090cd..a86255d08a6b 100644
--- a/drivers/input/tablet/acecad.c
+++ b/drivers/input/tablet/acecad.c
@@ -30,17 +30,9 @@
 #include <linux/module.h>
 #include <linux/usb/input.h>
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v3.2"
-#define DRIVER_DESC    "USB Acecad Flair tablet driver"
-#define DRIVER_LICENSE "GPL"
-#define DRIVER_AUTHOR  "Edouard TISSERANT <edouard.tisserant@wanadoo.fr>"
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_AUTHOR("Edouard TISSERANT <edouard.tisserant@wanadoo.fr>");
+MODULE_DESCRIPTION("USB Acecad Flair tablet driver");
+MODULE_LICENSE("GPL");
 
 #define USB_VENDOR_ID_ACECAD	0x0460
 #define USB_DEVICE_ID_FLAIR	0x0004
diff --git a/drivers/input/tablet/aiptek.c b/drivers/input/tablet/aiptek.c
index 0b55e1f375b3..545fa6e89035 100644
--- a/drivers/input/tablet/aiptek.c
+++ b/drivers/input/tablet/aiptek.c
@@ -78,13 +78,6 @@
 #include <linux/uaccess.h>
 #include <asm/unaligned.h>
 
-/*
- * Version Information
- */
-#define DRIVER_VERSION "v2.3 (May 2, 2007)"
-#define DRIVER_AUTHOR  "Bryan W. Headley/Chris Atenasio/Cedric Brun/Rene van Paassen"
-#define DRIVER_DESC    "Aiptek HyperPen USB Tablet Driver (Linux 2.6.x)"
-
 /*
  * Aiptek status packet:
  *
@@ -1941,8 +1934,8 @@ static struct usb_driver aiptek_driver = {
 
 module_usb_driver(aiptek_driver);
 
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_AUTHOR("Bryan W. Headley/Chris Atenasio/Cedric Brun/Rene van Paassen");
+MODULE_DESCRIPTION("Aiptek HyperPen USB Tablet Driver");
 MODULE_LICENSE("GPL");
 
 module_param(programmableDelay, int, 0);
diff --git a/drivers/input/tablet/hanwang.c b/drivers/input/tablet/hanwang.c
index df4bea96d7ed..4042c41160f4 100644
--- a/drivers/input/tablet/hanwang.c
+++ b/drivers/input/tablet/hanwang.c
@@ -28,13 +28,9 @@
 #include <linux/module.h>
 #include <linux/usb/input.h>
 
-#define DRIVER_AUTHOR   "Xing Wei <weixing@hanwang.com.cn>"
-#define DRIVER_DESC     "USB Hanwang tablet driver"
-#define DRIVER_LICENSE  "GPL"
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_AUTHOR("Xing Wei <weixing@hanwang.com.cn>");
+MODULE_DESCRIPTION("USB Hanwang tablet driver");
+MODULE_LICENSE("GPL");
 
 #define USB_VENDOR_ID_HANWANG		0x0b57
 #define HANWANG_TABLET_INT_CLASS	0x0003
diff --git a/drivers/input/tablet/kbtab.c b/drivers/input/tablet/kbtab.c
index a41c3ff7c9af..75b500651e4e 100644
--- a/drivers/input/tablet/kbtab.c
+++ b/drivers/input/tablet/kbtab.c
@@ -5,21 +5,12 @@
 #include <asm/unaligned.h>
 
 /*
- * Version Information
- * v0.0.1 - Original, extremely basic version, 2.4.xx only
- * v0.0.2 - Updated, works with 2.5.62 and 2.4.20;
- *           - added pressure-threshold modules param code from
- *              Alex Perry <alex.perry@ieee.org>
+ * Pressure-threshold modules param code from Alex Perry <alex.perry@ieee.org>
  */
 
-#define DRIVER_VERSION "v0.0.2"
-#define DRIVER_AUTHOR "Josh Myer <josh@joshisanerd.com>"
-#define DRIVER_DESC "USB KB Gear JamStudio Tablet driver"
-#define DRIVER_LICENSE "GPL"
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_AUTHOR("Josh Myer <josh@joshisanerd.com>");
+MODULE_DESCRIPTION("USB KB Gear JamStudio Tablet driver");
+MODULE_LICENSE("GPL");
 
 #define USB_VENDOR_ID_KBGEAR	0x084e
 
diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig
index 38a226f9fcbd..4f15496fec8b 100644
--- a/drivers/input/touchscreen/Kconfig
+++ b/drivers/input/touchscreen/Kconfig
@@ -795,21 +795,6 @@ config TOUCHSCREEN_WM9713
 	  Say Y here to enable support for the Wolfson Microelectronics
 	  WM9713 touchscreen controller.
 
-config TOUCHSCREEN_WM97XX_ATMEL
-	tristate "WM97xx Atmel accelerated touch"
-	depends on TOUCHSCREEN_WM97XX && AVR32
-	help
-	  Say Y here for support for streaming mode with WM97xx touchscreens
-	  on Atmel AT91 or AVR32 systems with an AC97C module.
-
-	  Be aware that this will use channel B in the controller for
-	  streaming data, this must not conflict with other AC97C drivers.
-
-	  If unsure, say N.
-
-	  To compile this driver as a module, choose M here: the module will
-	  be called atmel-wm97xx.
-
 config TOUCHSCREEN_WM97XX_MAINSTONE
 	tristate "WM97xx Mainstone/Palm accelerated touch"
 	depends on TOUCHSCREEN_WM97XX && ARCH_PXA
diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile
index d2a2b3b7af27..dddae7973436 100644
--- a/drivers/input/touchscreen/Makefile
+++ b/drivers/input/touchscreen/Makefile
@@ -97,7 +97,6 @@ obj-$(CONFIG_TOUCHSCREEN_WM97XX)	+= wm97xx-ts.o
 wm97xx-ts-$(CONFIG_TOUCHSCREEN_WM9705)	+= wm9705.o
 wm97xx-ts-$(CONFIG_TOUCHSCREEN_WM9712)	+= wm9712.o
 wm97xx-ts-$(CONFIG_TOUCHSCREEN_WM9713)	+= wm9713.o
-obj-$(CONFIG_TOUCHSCREEN_WM97XX_ATMEL)	+= atmel-wm97xx.o
 obj-$(CONFIG_TOUCHSCREEN_WM97XX_MAINSTONE)	+= mainstone-wm97xx.o
 obj-$(CONFIG_TOUCHSCREEN_WM97XX_ZYLONITE)	+= zylonite-wm97xx.o
 obj-$(CONFIG_TOUCHSCREEN_W90X900)	+= w90p910_ts.o
diff --git a/drivers/input/touchscreen/ad7877.c b/drivers/input/touchscreen/ad7877.c
index 0381c7809d1b..5cfe477ec992 100644
--- a/drivers/input/touchscreen/ad7877.c
+++ b/drivers/input/touchscreen/ad7877.c
@@ -417,8 +417,10 @@ static irqreturn_t ad7877_irq(int irq, void *handle)
 	return IRQ_HANDLED;
 }
 
-static void ad7877_disable(struct ad7877 *ts)
+static void ad7877_disable(void *data)
 {
+	struct ad7877 *ts = data;
+
 	mutex_lock(&ts->mutex);
 
 	if (!ts->disabled) {
@@ -707,12 +709,17 @@ static int ad7877_probe(struct spi_device *spi)
 		return err;
 	}
 
-	ts = kzalloc(sizeof(struct ad7877), GFP_KERNEL);
-	input_dev = input_allocate_device();
-	if (!ts || !input_dev) {
-		err = -ENOMEM;
-		goto err_free_mem;
-	}
+	ts = devm_kzalloc(&spi->dev, sizeof(struct ad7877), GFP_KERNEL);
+	if (!ts)
+		return -ENOMEM;
+
+	input_dev = devm_input_allocate_device(&spi->dev);
+	if (!input_dev)
+		return -ENOMEM;
+
+	err = devm_add_action_or_reset(&spi->dev, ad7877_disable, ts);
+	if (err)
+		return err;
 
 	spi_set_drvdata(spi, ts);
 	ts->spi = spi;
@@ -761,11 +768,10 @@ static int ad7877_probe(struct spi_device *spi)
 
 	verify = ad7877_read(spi, AD7877_REG_SEQ1);
 
-	if (verify != AD7877_MM_SEQUENCE){
+	if (verify != AD7877_MM_SEQUENCE) {
 		dev_err(&spi->dev, "%s: Failed to probe %s\n",
 			dev_name(&spi->dev), input_dev->name);
-		err = -ENODEV;
-		goto err_free_mem;
+		return -ENODEV;
 	}
 
 	if (gpio3)
@@ -775,47 +781,21 @@ static int ad7877_probe(struct spi_device *spi)
 
 	/* Request AD7877 /DAV GPIO interrupt */
 
-	err = request_threaded_irq(spi->irq, NULL, ad7877_irq,
-				   IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-				   spi->dev.driver->name, ts);
+	err = devm_request_threaded_irq(&spi->dev, spi->irq, NULL, ad7877_irq,
+					IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
+					spi->dev.driver->name, ts);
 	if (err) {
 		dev_dbg(&spi->dev, "irq %d busy?\n", spi->irq);
-		goto err_free_mem;
+		return err;
 	}
 
-	err = sysfs_create_group(&spi->dev.kobj, &ad7877_attr_group);
+	err = devm_device_add_group(&spi->dev, &ad7877_attr_group);
 	if (err)
-		goto err_free_irq;
+		return err;
 
 	err = input_register_device(input_dev);
 	if (err)
-		goto err_remove_attr_group;
-
-	return 0;
-
-err_remove_attr_group:
-	sysfs_remove_group(&spi->dev.kobj, &ad7877_attr_group);
-err_free_irq:
-	free_irq(spi->irq, ts);
-err_free_mem:
-	input_free_device(input_dev);
-	kfree(ts);
-	return err;
-}
-
-static int ad7877_remove(struct spi_device *spi)
-{
-	struct ad7877 *ts = spi_get_drvdata(spi);
-
-	sysfs_remove_group(&spi->dev.kobj, &ad7877_attr_group);
-
-	ad7877_disable(ts);
-	free_irq(ts->spi->irq, ts);
-
-	input_unregister_device(ts->input);
-	kfree(ts);
-
-	dev_dbg(&spi->dev, "unregistered touchscreen\n");
+		return err;
 
 	return 0;
 }
@@ -846,7 +826,6 @@ static struct spi_driver ad7877_driver = {
 		.pm	= &ad7877_pm,
 	},
 	.probe		= ad7877_probe,
-	.remove		= ad7877_remove,
 };
 
 module_spi_driver(ad7877_driver);
diff --git a/drivers/input/touchscreen/atmel-wm97xx.c b/drivers/input/touchscreen/atmel-wm97xx.c
deleted file mode 100644
index 9140a43cfe20..000000000000
--- a/drivers/input/touchscreen/atmel-wm97xx.c
+++ /dev/null
@@ -1,436 +0,0 @@
-/*
- * Atmel AT91 and AVR32 continuous touch screen driver for Wolfson WM97xx AC97
- * codecs.
- *
- * Copyright (C) 2008 - 2009 Atmel Corporation
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- */
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/irq.h>
-#include <linux/interrupt.h>
-#include <linux/wm97xx.h>
-#include <linux/timer.h>
-#include <linux/gpio.h>
-#include <linux/io.h>
-#include <linux/slab.h>
-
-#define AC97C_ICA		0x10
-#define AC97C_CBRHR		0x30
-#define AC97C_CBSR		0x38
-#define AC97C_CBMR		0x3c
-#define AC97C_IER		0x54
-#define AC97C_IDR		0x58
-
-#define AC97C_RXRDY		(1 << 4)
-#define AC97C_OVRUN		(1 << 5)
-
-#define AC97C_CMR_SIZE_20	(0 << 16)
-#define AC97C_CMR_SIZE_18	(1 << 16)
-#define AC97C_CMR_SIZE_16	(2 << 16)
-#define AC97C_CMR_SIZE_10	(3 << 16)
-#define AC97C_CMR_CEM_LITTLE	(1 << 18)
-#define AC97C_CMR_CEM_BIG	(0 << 18)
-#define AC97C_CMR_CENA		(1 << 21)
-
-#define AC97C_INT_CBEVT		(1 << 4)
-
-#define AC97C_SR_CAEVT		(1 << 3)
-
-#define AC97C_CH_MASK(slot)						\
-	(0x7 << (3 * (slot - 3)))
-#define AC97C_CH_ASSIGN(slot, channel)					\
-	(AC97C_CHANNEL_##channel << (3 * (slot - 3)))
-#define AC97C_CHANNEL_NONE	0x0
-#define AC97C_CHANNEL_B		0x2
-
-#define ac97c_writel(chip, reg, val)			\
-	__raw_writel((val), (chip)->regs + AC97C_##reg)
-#define ac97c_readl(chip, reg)				\
-	__raw_readl((chip)->regs + AC97C_##reg)
-
-#ifdef CONFIG_CPU_AT32AP700X
-#define ATMEL_WM97XX_AC97C_IOMEM	(0xfff02800)
-#define ATMEL_WM97XX_AC97C_IRQ		(29)
-#define ATMEL_WM97XX_GPIO_DEFAULT	(32+16) /* Pin 16 on port B. */
-#else
-#error Unknown CPU, this driver only supports AT32AP700X CPUs.
-#endif
-
-struct continuous {
-	u16 id;    /* codec id */
-	u8 code;   /* continuous code */
-	u8 reads;  /* number of coord reads per read cycle */
-	u32 speed; /* number of coords per second */
-};
-
-#define WM_READS(sp) ((sp / HZ) + 1)
-
-static const struct continuous cinfo[] = {
-	{WM9705_ID2, 0, WM_READS(94), 94},
-	{WM9705_ID2, 1, WM_READS(188), 188},
-	{WM9705_ID2, 2, WM_READS(375), 375},
-	{WM9705_ID2, 3, WM_READS(750), 750},
-	{WM9712_ID2, 0, WM_READS(94), 94},
-	{WM9712_ID2, 1, WM_READS(188), 188},
-	{WM9712_ID2, 2, WM_READS(375), 375},
-	{WM9712_ID2, 3, WM_READS(750), 750},
-	{WM9713_ID2, 0, WM_READS(94), 94},
-	{WM9713_ID2, 1, WM_READS(120), 120},
-	{WM9713_ID2, 2, WM_READS(154), 154},
-	{WM9713_ID2, 3, WM_READS(188), 188},
-};
-
-/* Continuous speed index. */
-static int sp_idx;
-
-/*
- * Pen sampling frequency (Hz) in continuous mode.
- */
-static int cont_rate = 188;
-module_param(cont_rate, int, 0);
-MODULE_PARM_DESC(cont_rate, "Sampling rate in continuous mode (Hz)");
-
-/*
- * Pen down detection.
- *
- * This driver can either poll or use an interrupt to indicate a pen down
- * event. If the irq request fails then it will fall back to polling mode.
- */
-static int pen_int = 1;
-module_param(pen_int, int, 0);
-MODULE_PARM_DESC(pen_int, "Pen down detection (1 = interrupt, 0 = polling)");
-
-/*
- * Pressure readback.
- *
- * Set to 1 to read back pen down pressure.
- */
-static int pressure;
-module_param(pressure, int, 0);
-MODULE_PARM_DESC(pressure, "Pressure readback (1 = pressure, 0 = no pressure)");
-
-/*
- * AC97 touch data slot.
- *
- * Touch screen readback data ac97 slot.
- */
-static int ac97_touch_slot = 5;
-module_param(ac97_touch_slot, int, 0);
-MODULE_PARM_DESC(ac97_touch_slot, "Touch screen data slot AC97 number");
-
-/*
- * GPIO line number.
- *
- * Set to GPIO number where the signal from the WM97xx device is hooked up.
- */
-static int atmel_gpio_line = ATMEL_WM97XX_GPIO_DEFAULT;
-module_param(atmel_gpio_line, int, 0);
-MODULE_PARM_DESC(atmel_gpio_line, "GPIO line number connected to WM97xx");
-
-struct atmel_wm97xx {
-	struct wm97xx		*wm;
-	struct timer_list	pen_timer;
-	void __iomem		*regs;
-	unsigned long		ac97c_irq;
-	unsigned long		gpio_pen;
-	unsigned long		gpio_irq;
-	unsigned short		x;
-	unsigned short		y;
-};
-
-static irqreturn_t atmel_wm97xx_channel_b_interrupt(int irq, void *dev_id)
-{
-	struct atmel_wm97xx *atmel_wm97xx = dev_id;
-	struct wm97xx *wm = atmel_wm97xx->wm;
-	int status = ac97c_readl(atmel_wm97xx, CBSR);
-	irqreturn_t retval = IRQ_NONE;
-
-	if (status & AC97C_OVRUN) {
-		dev_dbg(&wm->touch_dev->dev, "AC97C overrun\n");
-		ac97c_readl(atmel_wm97xx, CBRHR);
-		retval = IRQ_HANDLED;
-	} else if (status & AC97C_RXRDY) {
-		u16 data;
-		u16 value;
-		u16 source;
-		u16 pen_down;
-
-		data = ac97c_readl(atmel_wm97xx, CBRHR);
-		value = data & 0x0fff;
-		source = data & WM97XX_ADCSEL_MASK;
-		pen_down = (data & WM97XX_PEN_DOWN) >> 8;
-
-		if (source == WM97XX_ADCSEL_X)
-			atmel_wm97xx->x = value;
-		if (source == WM97XX_ADCSEL_Y)
-			atmel_wm97xx->y = value;
-
-		if (!pressure && source == WM97XX_ADCSEL_Y) {
-			input_report_abs(wm->input_dev, ABS_X, atmel_wm97xx->x);
-			input_report_abs(wm->input_dev, ABS_Y, atmel_wm97xx->y);
-			input_report_key(wm->input_dev, BTN_TOUCH, pen_down);
-			input_sync(wm->input_dev);
-		} else if (pressure && source == WM97XX_ADCSEL_PRES) {
-			input_report_abs(wm->input_dev, ABS_X, atmel_wm97xx->x);
-			input_report_abs(wm->input_dev, ABS_Y, atmel_wm97xx->y);
-			input_report_abs(wm->input_dev, ABS_PRESSURE, value);
-			input_report_key(wm->input_dev, BTN_TOUCH, value);
-			input_sync(wm->input_dev);
-		}
-
-		retval = IRQ_HANDLED;
-	}
-
-	return retval;
-}
-
-static void atmel_wm97xx_acc_pen_up(struct wm97xx *wm)
-{
-	struct atmel_wm97xx *atmel_wm97xx = platform_get_drvdata(wm->touch_dev);
-	struct input_dev *input_dev = wm->input_dev;
-	int pen_down = gpio_get_value(atmel_wm97xx->gpio_pen);
-
-	if (pen_down != 0) {
-		mod_timer(&atmel_wm97xx->pen_timer,
-			  jiffies + msecs_to_jiffies(1));
-	} else {
-		if (pressure)
-			input_report_abs(input_dev, ABS_PRESSURE, 0);
-		input_report_key(input_dev, BTN_TOUCH, 0);
-		input_sync(input_dev);
-	}
-}
-
-static void atmel_wm97xx_pen_timer(struct timer_list *t)
-{
-	struct atmel_wm97xx *atmel_wm97xx = from_timer(atmel_wm97xx, t,
-						       pen_timer);
-
-	atmel_wm97xx_acc_pen_up(atmel_wm97xx->wm);
-}
-
-static int atmel_wm97xx_acc_startup(struct wm97xx *wm)
-{
-	struct atmel_wm97xx *atmel_wm97xx = platform_get_drvdata(wm->touch_dev);
-	int idx = 0;
-
-	if (wm->ac97 == NULL)
-		return -ENODEV;
-
-	for (idx = 0; idx < ARRAY_SIZE(cinfo); idx++) {
-		if (wm->id != cinfo[idx].id)
-			continue;
-
-		sp_idx = idx;
-
-		if (cont_rate <= cinfo[idx].speed)
-			break;
-	}
-
-	wm->acc_rate = cinfo[sp_idx].code;
-	wm->acc_slot = ac97_touch_slot;
-	dev_info(&wm->touch_dev->dev, "atmel accelerated touchscreen driver, "
-			"%d samples/sec\n", cinfo[sp_idx].speed);
-
-	if (pen_int) {
-		unsigned long reg;
-
-		wm->pen_irq = atmel_wm97xx->gpio_irq;
-
-		switch (wm->id) {
-		case WM9712_ID2: /* Fall through. */
-		case WM9713_ID2:
-			/*
-			 * Use GPIO 13 (PEN_DOWN) to assert GPIO line 3
-			 * (PENDOWN).
-			 */
-			wm97xx_config_gpio(wm, WM97XX_GPIO_13, WM97XX_GPIO_IN,
-					WM97XX_GPIO_POL_HIGH,
-					WM97XX_GPIO_STICKY,
-					WM97XX_GPIO_WAKE);
-			wm97xx_config_gpio(wm, WM97XX_GPIO_3, WM97XX_GPIO_OUT,
-					WM97XX_GPIO_POL_HIGH,
-					WM97XX_GPIO_NOTSTICKY,
-					WM97XX_GPIO_NOWAKE);
-		case WM9705_ID2: /* Fall through. */
-			/*
-			 * Enable touch data slot in AC97 controller channel B.
-			 */
-			reg = ac97c_readl(atmel_wm97xx, ICA);
-			reg &= ~AC97C_CH_MASK(wm->acc_slot);
-			reg |= AC97C_CH_ASSIGN(wm->acc_slot, B);
-			ac97c_writel(atmel_wm97xx, ICA, reg);
-
-			/*
-			 * Enable channel and interrupt for RXRDY and OVERRUN.
-			 */
-			ac97c_writel(atmel_wm97xx, CBMR, AC97C_CMR_CENA
-					| AC97C_CMR_CEM_BIG
-					| AC97C_CMR_SIZE_16
-					| AC97C_OVRUN
-					| AC97C_RXRDY);
-			/* Dummy read to empty RXRHR. */
-			ac97c_readl(atmel_wm97xx, CBRHR);
-			/*
-			 * Enable interrupt for channel B in the AC97
-			 * controller.
-			 */
-			ac97c_writel(atmel_wm97xx, IER, AC97C_INT_CBEVT);
-			break;
-		default:
-			dev_err(&wm->touch_dev->dev, "pen down irq not "
-					"supported on this device\n");
-			pen_int = 0;
-			break;
-		}
-	}
-
-	return 0;
-}
-
-static void atmel_wm97xx_acc_shutdown(struct wm97xx *wm)
-{
-	if (pen_int) {
-		struct atmel_wm97xx *atmel_wm97xx =
-			platform_get_drvdata(wm->touch_dev);
-		unsigned long ica;
-
-		switch (wm->id & 0xffff) {
-		case WM9705_ID2: /* Fall through. */
-		case WM9712_ID2: /* Fall through. */
-		case WM9713_ID2:
-			/* Disable slot and turn off channel B interrupts. */
-			ica = ac97c_readl(atmel_wm97xx, ICA);
-			ica &= ~AC97C_CH_MASK(wm->acc_slot);
-			ac97c_writel(atmel_wm97xx, ICA, ica);
-			ac97c_writel(atmel_wm97xx, IDR, AC97C_INT_CBEVT);
-			ac97c_writel(atmel_wm97xx, CBMR, 0);
-			wm->pen_irq = 0;
-			break;
-		default:
-			dev_err(&wm->touch_dev->dev, "unknown codec\n");
-			break;
-		}
-	}
-}
-
-static void atmel_wm97xx_irq_enable(struct wm97xx *wm, int enable)
-{
-	/* Intentionally left empty. */
-}
-
-static struct wm97xx_mach_ops atmel_mach_ops = {
-	.acc_enabled	= 1,
-	.acc_pen_up	= atmel_wm97xx_acc_pen_up,
-	.acc_startup	= atmel_wm97xx_acc_startup,
-	.acc_shutdown	= atmel_wm97xx_acc_shutdown,
-	.irq_enable	= atmel_wm97xx_irq_enable,
-	.irq_gpio	= WM97XX_GPIO_3,
-};
-
-static int __init atmel_wm97xx_probe(struct platform_device *pdev)
-{
-	struct wm97xx *wm = platform_get_drvdata(pdev);
-	struct atmel_wm97xx *atmel_wm97xx;
-	int ret;
-
-	atmel_wm97xx = kzalloc(sizeof(struct atmel_wm97xx), GFP_KERNEL);
-	if (!atmel_wm97xx)
-		return -ENOMEM;
-
-	atmel_wm97xx->wm	= wm;
-	atmel_wm97xx->regs	= (void *)ATMEL_WM97XX_AC97C_IOMEM;
-	atmel_wm97xx->ac97c_irq	= ATMEL_WM97XX_AC97C_IRQ;
-	atmel_wm97xx->gpio_pen	= atmel_gpio_line;
-	atmel_wm97xx->gpio_irq	= gpio_to_irq(atmel_wm97xx->gpio_pen);
-
-	timer_setup(&atmel_wm97xx->pen_timer, atmel_wm97xx_pen_timer, 0);
-
-	ret = request_irq(atmel_wm97xx->ac97c_irq,
-			  atmel_wm97xx_channel_b_interrupt,
-			  IRQF_SHARED, "atmel-wm97xx-ch-b", atmel_wm97xx);
-	if (ret) {
-		dev_dbg(&pdev->dev, "could not request ac97c irq\n");
-		goto err;
-	}
-
-	platform_set_drvdata(pdev, atmel_wm97xx);
-
-	ret = wm97xx_register_mach_ops(wm, &atmel_mach_ops);
-	if (ret)
-		goto err_irq;
-
-	return ret;
-
-err_irq:
-	free_irq(atmel_wm97xx->ac97c_irq, atmel_wm97xx);
-err:
-	kfree(atmel_wm97xx);
-	return ret;
-}
-
-static int __exit atmel_wm97xx_remove(struct platform_device *pdev)
-{
-	struct atmel_wm97xx *atmel_wm97xx = platform_get_drvdata(pdev);
-	struct wm97xx *wm = atmel_wm97xx->wm;
-
-	ac97c_writel(atmel_wm97xx, IDR, AC97C_INT_CBEVT);
-	free_irq(atmel_wm97xx->ac97c_irq, atmel_wm97xx);
-	del_timer_sync(&atmel_wm97xx->pen_timer);
-	wm97xx_unregister_mach_ops(wm);
-	kfree(atmel_wm97xx);
-
-	return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int atmel_wm97xx_suspend(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct atmel_wm97xx *atmel_wm97xx = platform_get_drvdata(pdev);
-
-	ac97c_writel(atmel_wm97xx, IDR, AC97C_INT_CBEVT);
-	disable_irq(atmel_wm97xx->gpio_irq);
-	del_timer_sync(&atmel_wm97xx->pen_timer);
-
-	return 0;
-}
-
-static int atmel_wm97xx_resume(struct device *dev)
-{
-	struct platform_device *pdev = to_platform_device(dev);
-	struct atmel_wm97xx *atmel_wm97xx = platform_get_drvdata(pdev);
-	struct wm97xx *wm = atmel_wm97xx->wm;
-
-	if (wm->input_dev->users) {
-		enable_irq(atmel_wm97xx->gpio_irq);
-		ac97c_writel(atmel_wm97xx, IER, AC97C_INT_CBEVT);
-	}
-
-	return 0;
-}
-#endif
-
-static SIMPLE_DEV_PM_OPS(atmel_wm97xx_pm_ops,
-			 atmel_wm97xx_suspend, atmel_wm97xx_resume);
-
-static struct platform_driver atmel_wm97xx_driver = {
-	.remove		= __exit_p(atmel_wm97xx_remove),
-	.driver		= {
-		.name	= "wm97xx-touch",
-		.pm	= &atmel_wm97xx_pm_ops,
-	},
-};
-
-module_platform_driver_probe(atmel_wm97xx_driver, atmel_wm97xx_probe);
-
-MODULE_AUTHOR("Hans-Christian Egtvedt <egtvedt@samfundet.no>");
-MODULE_DESCRIPTION("wm97xx continuous touch driver for Atmel AT91 and AVR32");
-MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/auo-pixcir-ts.c b/drivers/input/touchscreen/auo-pixcir-ts.c
index 6592fc5d48b4..df8ca856393b 100644
--- a/drivers/input/touchscreen/auo-pixcir-ts.c
+++ b/drivers/input/touchscreen/auo-pixcir-ts.c
@@ -408,8 +408,6 @@ static void auo_pixcir_input_close(struct input_dev *dev)
 	struct auo_pixcir_ts *ts = input_get_drvdata(dev);
 
 	auo_pixcir_stop(ts);
-
-	return;
 }
 
 static int __maybe_unused auo_pixcir_suspend(struct device *dev)
@@ -487,10 +485,8 @@ static struct auo_pixcir_ts_platdata *auo_pixcir_parse_dt(struct device *dev)
 		return ERR_PTR(-ENOENT);
 
 	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata) {
-		dev_err(dev, "failed to allocate platform data\n");
+	if (!pdata)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	pdata->gpio_int = of_get_gpio(np, 0);
 	if (!gpio_is_valid(pdata->gpio_int)) {
diff --git a/drivers/input/touchscreen/colibri-vf50-ts.c b/drivers/input/touchscreen/colibri-vf50-ts.c
index 69c08acae264..82ca5ab65cec 100644
--- a/drivers/input/touchscreen/colibri-vf50-ts.c
+++ b/drivers/input/touchscreen/colibri-vf50-ts.c
@@ -28,7 +28,6 @@
 #include <linux/types.h>
 
 #define DRIVER_NAME			"colibri-vf50-ts"
-#define DRV_VERSION			"1.0"
 
 #define VF_ADC_MAX			((1 << 12) - 1)
 
@@ -382,4 +381,3 @@ module_platform_driver(vf50_touch_driver);
 MODULE_AUTHOR("Sanchayan Maity");
 MODULE_DESCRIPTION("Colibri VF50 Touchscreen driver");
 MODULE_LICENSE("GPL");
-MODULE_VERSION(DRV_VERSION);
diff --git a/drivers/input/touchscreen/da9052_tsi.c b/drivers/input/touchscreen/da9052_tsi.c
index 5a013bb7bcad..b5dfd5944cc3 100644
--- a/drivers/input/touchscreen/da9052_tsi.c
+++ b/drivers/input/touchscreen/da9052_tsi.c
@@ -26,7 +26,6 @@ struct da9052_tsi {
 	struct da9052 *da9052;
 	struct input_dev *dev;
 	struct delayed_work ts_pen_work;
-	struct mutex mutex;
 	bool stopped;
 	bool adc_on;
 };
diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c
index c53a3d7239e7..1e18ca0d1b4e 100644
--- a/drivers/input/touchscreen/edt-ft5x06.c
+++ b/drivers/input/touchscreen/edt-ft5x06.c
@@ -511,6 +511,12 @@ static int edt_ft5x06_factory_mode(struct edt_ft5x06_ts_data *tsdata)
 	int ret;
 	int error;
 
+	if (tsdata->version != EDT_M06) {
+		dev_err(&client->dev,
+			"No factory mode support for non-M06 devices\n");
+		return -EINVAL;
+	}
+
 	disable_irq(client->irq);
 
 	if (!tsdata->raw_buffer) {
@@ -524,9 +530,6 @@ static int edt_ft5x06_factory_mode(struct edt_ft5x06_ts_data *tsdata)
 	}
 
 	/* mode register is 0x3c when in the work mode */
-	if (tsdata->version != EDT_M06)
-		goto m09_out;
-
 	error = edt_ft5x06_register_write(tsdata, WORK_REGISTER_OPMODE, 0x03);
 	if (error) {
 		dev_err(&client->dev,
@@ -559,11 +562,6 @@ static int edt_ft5x06_factory_mode(struct edt_ft5x06_ts_data *tsdata)
 	enable_irq(client->irq);
 
 	return error;
-
-m09_out:
-	dev_err(&client->dev, "No factory mode support for M09/M12/GENERIC_FT\n");
-	return -EINVAL;
-
 }
 
 static int edt_ft5x06_work_mode(struct edt_ft5x06_ts_data *tsdata)
diff --git a/drivers/input/touchscreen/elants_i2c.c b/drivers/input/touchscreen/elants_i2c.c
index a458e5ec9e41..d21ca39b0fdb 100644
--- a/drivers/input/touchscreen/elants_i2c.c
+++ b/drivers/input/touchscreen/elants_i2c.c
@@ -45,7 +45,6 @@
 
 /* Device, Driver information */
 #define DEVICE_NAME	"elants_i2c"
-#define DRV_VERSION	"1.0.9"
 
 /* Convert from rows or columns into resolution */
 #define ELAN_TS_RESOLUTION(n, m)   (((n) - 1) * (m))
@@ -1000,7 +999,7 @@ static ssize_t show_iap_mode(struct device *dev,
 				"Normal" : "Recovery");
 }
 
-static DEVICE_ATTR(calibrate, S_IWUSR, NULL, calibrate_store);
+static DEVICE_ATTR_WO(calibrate);
 static DEVICE_ATTR(iap_mode, S_IRUGO, show_iap_mode, NULL);
 static DEVICE_ATTR(update_fw, S_IWUSR, NULL, write_update_fw);
 
@@ -1406,5 +1405,4 @@ module_i2c_driver(elants_i2c_driver);
 
 MODULE_AUTHOR("Scott Liu <scott.liu@emc.com.tw>");
 MODULE_DESCRIPTION("Elan I2c Touchscreen driver");
-MODULE_VERSION(DRV_VERSION);
 MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c
index 69d0b8cbc71f..9736c83dd418 100644
--- a/drivers/input/touchscreen/goodix.c
+++ b/drivers/input/touchscreen/goodix.c
@@ -22,6 +22,7 @@
 #include <linux/i2c.h>
 #include <linux/input.h>
 #include <linux/input/mt.h>
+#include <linux/input/touchscreen.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/irq.h>
@@ -43,11 +44,7 @@ struct goodix_ts_data {
 	struct i2c_client *client;
 	struct input_dev *input_dev;
 	const struct goodix_chip_data *chip;
-	int abs_x_max;
-	int abs_y_max;
-	bool swapped_x_y;
-	bool inverted_x;
-	bool inverted_y;
+	struct touchscreen_properties prop;
 	unsigned int max_touch_num;
 	unsigned int int_trigger_type;
 	struct gpio_desc *gpiod_int;
@@ -160,7 +157,7 @@ static int goodix_i2c_read(struct i2c_client *client,
 			   u16 reg, u8 *buf, int len)
 {
 	struct i2c_msg msgs[2];
-	u16 wbuf = cpu_to_be16(reg);
+	__be16 wbuf = cpu_to_be16(reg);
 	int ret;
 
 	msgs[0].flags = 0;
@@ -295,18 +292,10 @@ static void goodix_ts_report_touch(struct goodix_ts_data *ts, u8 *coor_data)
 	int input_y = get_unaligned_le16(&coor_data[3]);
 	int input_w = get_unaligned_le16(&coor_data[5]);
 
-	/* Inversions have to happen before axis swapping */
-	if (ts->inverted_x)
-		input_x = ts->abs_x_max - input_x;
-	if (ts->inverted_y)
-		input_y = ts->abs_y_max - input_y;
-	if (ts->swapped_x_y)
-		swap(input_x, input_y);
-
 	input_mt_slot(ts->input_dev, id);
 	input_mt_report_slot_state(ts->input_dev, MT_TOOL_FINGER, true);
-	input_report_abs(ts->input_dev, ABS_MT_POSITION_X, input_x);
-	input_report_abs(ts->input_dev, ABS_MT_POSITION_Y, input_y);
+	touchscreen_report_pos(ts->input_dev, &ts->prop,
+			       input_x, input_y, true);
 	input_report_abs(ts->input_dev, ABS_MT_TOUCH_MAJOR, input_w);
 	input_report_abs(ts->input_dev, ABS_MT_WIDTH_MAJOR, input_w);
 }
@@ -579,44 +568,27 @@ static int goodix_get_gpio_config(struct goodix_ts_data *ts)
 static void goodix_read_config(struct goodix_ts_data *ts)
 {
 	u8 config[GOODIX_CONFIG_MAX_LENGTH];
+	int x_max, y_max;
 	int error;
 
 	error = goodix_i2c_read(ts->client, ts->chip->config_addr,
 				config, ts->chip->config_len);
 	if (error) {
-		dev_warn(&ts->client->dev,
-			 "Error reading config (%d), using defaults\n",
+		dev_warn(&ts->client->dev, "Error reading config: %d\n",
 			 error);
-		ts->abs_x_max = GOODIX_MAX_WIDTH;
-		ts->abs_y_max = GOODIX_MAX_HEIGHT;
-		if (ts->swapped_x_y)
-			swap(ts->abs_x_max, ts->abs_y_max);
 		ts->int_trigger_type = GOODIX_INT_TRIGGER;
 		ts->max_touch_num = GOODIX_MAX_CONTACTS;
 		return;
 	}
 
-	ts->abs_x_max = get_unaligned_le16(&config[RESOLUTION_LOC]);
-	ts->abs_y_max = get_unaligned_le16(&config[RESOLUTION_LOC + 2]);
-	if (ts->swapped_x_y)
-		swap(ts->abs_x_max, ts->abs_y_max);
 	ts->int_trigger_type = config[TRIGGER_LOC] & 0x03;
 	ts->max_touch_num = config[MAX_CONTACTS_LOC] & 0x0f;
-	if (!ts->abs_x_max || !ts->abs_y_max || !ts->max_touch_num) {
-		dev_err(&ts->client->dev,
-			"Invalid config, using defaults\n");
-		ts->abs_x_max = GOODIX_MAX_WIDTH;
-		ts->abs_y_max = GOODIX_MAX_HEIGHT;
-		if (ts->swapped_x_y)
-			swap(ts->abs_x_max, ts->abs_y_max);
-		ts->max_touch_num = GOODIX_MAX_CONTACTS;
-	}
 
-	if (dmi_check_system(rotated_screen)) {
-		ts->inverted_x = true;
-		ts->inverted_y = true;
-		dev_dbg(&ts->client->dev,
-			 "Applying '180 degrees rotated screen' quirk\n");
+	x_max = get_unaligned_le16(&config[RESOLUTION_LOC]);
+	y_max = get_unaligned_le16(&config[RESOLUTION_LOC + 2]);
+	if (x_max && y_max) {
+		input_abs_set_max(ts->input_dev, ABS_MT_POSITION_X, x_max - 1);
+		input_abs_set_max(ts->input_dev, ABS_MT_POSITION_Y, y_max - 1);
 	}
 }
 
@@ -676,32 +648,28 @@ static int goodix_i2c_test(struct i2c_client *client)
 }
 
 /**
- * goodix_request_input_dev - Allocate, populate and register the input device
+ * goodix_configure_dev - Finish device initialization
  *
  * @ts: our goodix_ts_data pointer
  *
- * Must be called during probe
+ * Must be called from probe to finish initialization of the device.
+ * Contains the common initialization code for both devices that
+ * declare gpio pins and devices that do not. It is either called
+ * directly from probe or from request_firmware_wait callback.
  */
-static int goodix_request_input_dev(struct goodix_ts_data *ts)
+static int goodix_configure_dev(struct goodix_ts_data *ts)
 {
 	int error;
 
+	ts->int_trigger_type = GOODIX_INT_TRIGGER;
+	ts->max_touch_num = GOODIX_MAX_CONTACTS;
+
 	ts->input_dev = devm_input_allocate_device(&ts->client->dev);
 	if (!ts->input_dev) {
 		dev_err(&ts->client->dev, "Failed to allocate input device.");
 		return -ENOMEM;
 	}
 
-	input_set_abs_params(ts->input_dev, ABS_MT_POSITION_X,
-			     0, ts->abs_x_max, 0, 0);
-	input_set_abs_params(ts->input_dev, ABS_MT_POSITION_Y,
-			     0, ts->abs_y_max, 0, 0);
-	input_set_abs_params(ts->input_dev, ABS_MT_WIDTH_MAJOR, 0, 255, 0, 0);
-	input_set_abs_params(ts->input_dev, ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0);
-
-	input_mt_init_slots(ts->input_dev, ts->max_touch_num,
-			    INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED);
-
 	ts->input_dev->name = "Goodix Capacitive TouchScreen";
 	ts->input_dev->phys = "input/ts";
 	ts->input_dev->id.bustype = BUS_I2C;
@@ -712,42 +680,49 @@ static int goodix_request_input_dev(struct goodix_ts_data *ts)
 	/* Capacitive Windows/Home button on some devices */
 	input_set_capability(ts->input_dev, EV_KEY, KEY_LEFTMETA);
 
-	error = input_register_device(ts->input_dev);
-	if (error) {
-		dev_err(&ts->client->dev,
-			"Failed to register input device: %d", error);
-		return error;
-	}
+	input_set_capability(ts->input_dev, EV_ABS, ABS_MT_POSITION_X);
+	input_set_capability(ts->input_dev, EV_ABS, ABS_MT_POSITION_Y);
+	input_set_abs_params(ts->input_dev, ABS_MT_WIDTH_MAJOR, 0, 255, 0, 0);
+	input_set_abs_params(ts->input_dev, ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0);
 
-	return 0;
-}
+	/* Read configuration and apply touchscreen parameters */
+	goodix_read_config(ts);
 
-/**
- * goodix_configure_dev - Finish device initialization
- *
- * @ts: our goodix_ts_data pointer
- *
- * Must be called from probe to finish initialization of the device.
- * Contains the common initialization code for both devices that
- * declare gpio pins and devices that do not. It is either called
- * directly from probe or from request_firmware_wait callback.
- */
-static int goodix_configure_dev(struct goodix_ts_data *ts)
-{
-	int error;
+	/* Try overriding touchscreen parameters via device properties */
+	touchscreen_parse_properties(ts->input_dev, true, &ts->prop);
 
-	ts->swapped_x_y = device_property_read_bool(&ts->client->dev,
-						    "touchscreen-swapped-x-y");
-	ts->inverted_x = device_property_read_bool(&ts->client->dev,
-						   "touchscreen-inverted-x");
-	ts->inverted_y = device_property_read_bool(&ts->client->dev,
-						   "touchscreen-inverted-y");
+	if (!ts->prop.max_x || !ts->prop.max_y || !ts->max_touch_num) {
+		dev_err(&ts->client->dev, "Invalid config, using defaults\n");
+		ts->prop.max_x = GOODIX_MAX_WIDTH - 1;
+		ts->prop.max_y = GOODIX_MAX_HEIGHT - 1;
+		ts->max_touch_num = GOODIX_MAX_CONTACTS;
+		input_abs_set_max(ts->input_dev,
+				  ABS_MT_POSITION_X, ts->prop.max_x);
+		input_abs_set_max(ts->input_dev,
+				  ABS_MT_POSITION_Y, ts->prop.max_y);
+	}
 
-	goodix_read_config(ts);
+	if (dmi_check_system(rotated_screen)) {
+		ts->prop.invert_x = true;
+		ts->prop.invert_y = true;
+		dev_dbg(&ts->client->dev,
+			"Applying '180 degrees rotated screen' quirk\n");
+	}
 
-	error = goodix_request_input_dev(ts);
-	if (error)
+	error = input_mt_init_slots(ts->input_dev, ts->max_touch_num,
+				    INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED);
+	if (error) {
+		dev_err(&ts->client->dev,
+			"Failed to initialize MT slots: %d", error);
 		return error;
+	}
+
+	error = input_register_device(ts->input_dev);
+	if (error) {
+		dev_err(&ts->client->dev,
+			"Failed to register input device: %d", error);
+		return error;
+	}
 
 	ts->irq_flags = goodix_irq_flags[ts->int_trigger_type] | IRQF_ONESHOT;
 	error = goodix_request_irq(ts);
@@ -878,8 +853,10 @@ static int __maybe_unused goodix_suspend(struct device *dev)
 	int error;
 
 	/* We need gpio pins to suspend/resume */
-	if (!ts->gpiod_int || !ts->gpiod_rst)
+	if (!ts->gpiod_int || !ts->gpiod_rst) {
+		disable_irq(client->irq);
 		return 0;
+	}
 
 	wait_for_completion(&ts->firmware_loading_complete);
 
@@ -919,8 +896,10 @@ static int __maybe_unused goodix_resume(struct device *dev)
 	struct goodix_ts_data *ts = i2c_get_clientdata(client);
 	int error;
 
-	if (!ts->gpiod_int || !ts->gpiod_rst)
+	if (!ts->gpiod_int || !ts->gpiod_rst) {
+		enable_irq(client->irq);
 		return 0;
+	}
 
 	/*
 	 * Exit sleep mode by outputting HIGH level to INT pin
diff --git a/drivers/input/touchscreen/melfas_mip4.c b/drivers/input/touchscreen/melfas_mip4.c
index 6892f0e28918..430a2bc5f7ca 100644
--- a/drivers/input/touchscreen/melfas_mip4.c
+++ b/drivers/input/touchscreen/melfas_mip4.c
@@ -1611,6 +1611,5 @@ static struct i2c_driver mip4_driver = {
 module_i2c_driver(mip4_driver);
 
 MODULE_DESCRIPTION("MELFAS MIP4 Touchscreen");
-MODULE_VERSION("2016.10.31");
 MODULE_AUTHOR("Sangwon Jee <jeesw@melfas.com>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/mms114.c b/drivers/input/touchscreen/mms114.c
index e5eeb6311f7d..a5ab774da4cc 100644
--- a/drivers/input/touchscreen/mms114.c
+++ b/drivers/input/touchscreen/mms114.c
@@ -1,26 +1,24 @@
-/*
- * Copyright (C) 2012 Samsung Electronics Co.Ltd
- * Author: Joonyoung Shim <jy0922.shim@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+// Melfas MMS114/MMS152 touchscreen device driver
+//
+// Copyright (c) 2012 Samsung Electronics Co., Ltd.
+// Author: Joonyoung Shim <jy0922.shim@samsung.com>
 
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/i2c.h>
 #include <linux/input/mt.h>
+#include <linux/input/touchscreen.h>
 #include <linux/interrupt.h>
-#include <linux/platform_data/mms114.h>
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
 
 /* Write only registers */
 #define MMS114_MODE_CONTROL		0x01
 #define MMS114_OPERATION_MODE_MASK	0xE
-#define MMS114_ACTIVE			(1 << 1)
+#define MMS114_ACTIVE			BIT(1)
 
 #define MMS114_XY_RESOLUTION_H		0x02
 #define MMS114_X_RESOLUTION		0x03
@@ -30,9 +28,12 @@
 
 /* Read only registers */
 #define MMS114_PACKET_SIZE		0x0F
-#define MMS114_INFOMATION		0x10
+#define MMS114_INFORMATION		0x10
 #define MMS114_TSP_REV			0xF0
 
+#define MMS152_FW_REV			0xE1
+#define MMS152_COMPAT_GROUP		0xF2
+
 /* Minimum delay time is 50us between stop and start signal of i2c */
 #define MMS114_I2C_DELAY		50
 
@@ -50,12 +51,20 @@
 #define MMS114_TYPE_TOUCHSCREEN		1
 #define MMS114_TYPE_TOUCHKEY		2
 
+enum mms_type {
+	TYPE_MMS114	= 114,
+	TYPE_MMS152	= 152,
+};
+
 struct mms114_data {
 	struct i2c_client	*client;
 	struct input_dev	*input_dev;
 	struct regulator	*core_reg;
 	struct regulator	*io_reg;
-	const struct mms114_platform_data	*pdata;
+	struct touchscreen_properties props;
+	enum mms_type		type;
+	unsigned int		contact_threshold;
+	unsigned int		moving_threshold;
 
 	/* Use cache data for mode control register(write only) */
 	u8			cache_mode_control;
@@ -143,7 +152,6 @@ static int mms114_write_reg(struct mms114_data *data, unsigned int reg,
 
 static void mms114_process_mt(struct mms114_data *data, struct mms114_touch *touch)
 {
-	const struct mms114_platform_data *pdata = data->pdata;
 	struct i2c_client *client = data->client;
 	struct input_dev *input_dev = data->input_dev;
 	unsigned int id;
@@ -163,16 +171,6 @@ static void mms114_process_mt(struct mms114_data *data, struct mms114_touch *tou
 	id = touch->id - 1;
 	x = touch->x_lo | touch->x_hi << 8;
 	y = touch->y_lo | touch->y_hi << 8;
-	if (x > pdata->x_size || y > pdata->y_size) {
-		dev_dbg(&client->dev,
-			"Wrong touch coordinates (%d, %d)\n", x, y);
-		return;
-	}
-
-	if (pdata->x_invert)
-		x = pdata->x_size - x;
-	if (pdata->y_invert)
-		y = pdata->y_size - y;
 
 	dev_dbg(&client->dev,
 		"id: %d, type: %d, pressed: %d, x: %d, y: %d, width: %d, strength: %d\n",
@@ -183,9 +181,8 @@ static void mms114_process_mt(struct mms114_data *data, struct mms114_touch *tou
 	input_mt_report_slot_state(input_dev, MT_TOOL_FINGER, touch->pressed);
 
 	if (touch->pressed) {
+		touchscreen_report_pos(input_dev, &data->props, x, y, true);
 		input_report_abs(input_dev, ABS_MT_TOUCH_MAJOR, touch->width);
-		input_report_abs(input_dev, ABS_MT_POSITION_X, x);
-		input_report_abs(input_dev, ABS_MT_POSITION_Y, y);
 		input_report_abs(input_dev, ABS_MT_PRESSURE, touch->strength);
 	}
 }
@@ -213,7 +210,7 @@ static irqreturn_t mms114_interrupt(int irq, void *dev_id)
 
 	touch_size = packet_size / MMS114_PACKET_NUM;
 
-	error = __mms114_read_reg(data, MMS114_INFOMATION, packet_size,
+	error = __mms114_read_reg(data, MMS114_INFORMATION, packet_size,
 			(u8 *)touch);
 	if (error < 0)
 		goto out;
@@ -249,21 +246,40 @@ static int mms114_get_version(struct mms114_data *data)
 {
 	struct device *dev = &data->client->dev;
 	u8 buf[6];
+	int group;
 	int error;
 
-	error = __mms114_read_reg(data, MMS114_TSP_REV, 6, buf);
-	if (error < 0)
-		return error;
+	switch (data->type) {
+	case TYPE_MMS152:
+		error = __mms114_read_reg(data, MMS152_FW_REV, 3, buf);
+		if (error)
+			return error;
+
+		group = i2c_smbus_read_byte_data(data->client,
+						  MMS152_COMPAT_GROUP);
+		if (group < 0)
+			return group;
+
+		dev_info(dev, "TSP FW Rev: bootloader 0x%x / core 0x%x / config 0x%x, Compat group: %c\n",
+			 buf[0], buf[1], buf[2], group);
+		break;
+
+	case TYPE_MMS114:
+		error = __mms114_read_reg(data, MMS114_TSP_REV, 6, buf);
+		if (error)
+			return error;
 
-	dev_info(dev, "TSP Rev: 0x%x, HW Rev: 0x%x, Firmware Ver: 0x%x\n",
-		 buf[0], buf[1], buf[3]);
+		dev_info(dev, "TSP Rev: 0x%x, HW Rev: 0x%x, Firmware Ver: 0x%x\n",
+			 buf[0], buf[1], buf[3]);
+		break;
+	}
 
 	return 0;
 }
 
 static int mms114_setup_regs(struct mms114_data *data)
 {
-	const struct mms114_platform_data *pdata = data->pdata;
+	const struct touchscreen_properties *props = &data->props;
 	int val;
 	int error;
 
@@ -271,36 +287,40 @@ static int mms114_setup_regs(struct mms114_data *data)
 	if (error < 0)
 		return error;
 
+	/* MMS152 has no configuration or power on registers */
+	if (data->type == TYPE_MMS152)
+		return 0;
+
 	error = mms114_set_active(data, true);
 	if (error < 0)
 		return error;
 
-	val = (pdata->x_size >> 8) & 0xf;
-	val |= ((pdata->y_size >> 8) & 0xf) << 4;
+	val = (props->max_x >> 8) & 0xf;
+	val |= ((props->max_y >> 8) & 0xf) << 4;
 	error = mms114_write_reg(data, MMS114_XY_RESOLUTION_H, val);
 	if (error < 0)
 		return error;
 
-	val = pdata->x_size & 0xff;
+	val = props->max_x & 0xff;
 	error = mms114_write_reg(data, MMS114_X_RESOLUTION, val);
 	if (error < 0)
 		return error;
 
-	val = pdata->y_size & 0xff;
+	val = props->max_x & 0xff;
 	error = mms114_write_reg(data, MMS114_Y_RESOLUTION, val);
 	if (error < 0)
 		return error;
 
-	if (pdata->contact_threshold) {
+	if (data->contact_threshold) {
 		error = mms114_write_reg(data, MMS114_CONTACT_THRESHOLD,
-				pdata->contact_threshold);
+				data->contact_threshold);
 		if (error < 0)
 			return error;
 	}
 
-	if (pdata->moving_threshold) {
+	if (data->moving_threshold) {
 		error = mms114_write_reg(data, MMS114_MOVING_THRESHOLD,
-				pdata->moving_threshold);
+				data->moving_threshold);
 		if (error < 0)
 			return error;
 	}
@@ -326,7 +346,7 @@ static int mms114_start(struct mms114_data *data)
 		return error;
 	}
 
-	mdelay(MMS114_POWERON_DELAY);
+	msleep(MMS114_POWERON_DELAY);
 
 	error = mms114_setup_regs(data);
 	if (error < 0) {
@@ -335,9 +355,6 @@ static int mms114_start(struct mms114_data *data)
 		return error;
 	}
 
-	if (data->pdata->cfg_pin)
-		data->pdata->cfg_pin(true);
-
 	enable_irq(client->irq);
 
 	return 0;
@@ -350,9 +367,6 @@ static void mms114_stop(struct mms114_data *data)
 
 	disable_irq(client->irq);
 
-	if (data->pdata->cfg_pin)
-		data->pdata->cfg_pin(false);
-
 	error = regulator_disable(data->io_reg);
 	if (error)
 		dev_warn(&client->dev, "Failed to disable vdd: %d\n", error);
@@ -376,67 +390,44 @@ static void mms114_input_close(struct input_dev *dev)
 	mms114_stop(data);
 }
 
-#ifdef CONFIG_OF
-static struct mms114_platform_data *mms114_parse_dt(struct device *dev)
+static int mms114_parse_legacy_bindings(struct mms114_data *data)
 {
-	struct mms114_platform_data *pdata;
-	struct device_node *np = dev->of_node;
-
-	if (!np)
-		return NULL;
+	struct device *dev = &data->client->dev;
+	struct touchscreen_properties *props = &data->props;
 
-	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
-	if (!pdata) {
-		dev_err(dev, "failed to allocate platform data\n");
-		return NULL;
+	if (device_property_read_u32(dev, "x-size", &props->max_x)) {
+		dev_dbg(dev, "failed to get legacy x-size property\n");
+		return -EINVAL;
 	}
 
-	if (of_property_read_u32(np, "x-size", &pdata->x_size)) {
-		dev_err(dev, "failed to get x-size property\n");
-		return NULL;
+	if (device_property_read_u32(dev, "y-size", &props->max_y)) {
+		dev_dbg(dev, "failed to get legacy y-size property\n");
+		return -EINVAL;
 	}
 
-	if (of_property_read_u32(np, "y-size", &pdata->y_size)) {
-		dev_err(dev, "failed to get y-size property\n");
-		return NULL;
-	}
+	device_property_read_u32(dev, "contact-threshold",
+				&data->contact_threshold);
+	device_property_read_u32(dev, "moving-threshold",
+				&data->moving_threshold);
 
-	of_property_read_u32(np, "contact-threshold",
-				&pdata->contact_threshold);
-	of_property_read_u32(np, "moving-threshold",
-				&pdata->moving_threshold);
+	if (device_property_read_bool(dev, "x-invert"))
+		props->invert_x = true;
+	if (device_property_read_bool(dev, "y-invert"))
+		props->invert_y = true;
 
-	if (of_find_property(np, "x-invert", NULL))
-		pdata->x_invert = true;
-	if (of_find_property(np, "y-invert", NULL))
-		pdata->y_invert = true;
+	props->swap_x_y = false;
 
-	return pdata;
-}
-#else
-static inline struct mms114_platform_data *mms114_parse_dt(struct device *dev)
-{
-	return NULL;
+	return 0;
 }
-#endif
 
 static int mms114_probe(struct i2c_client *client,
 				  const struct i2c_device_id *id)
 {
-	const struct mms114_platform_data *pdata;
 	struct mms114_data *data;
 	struct input_dev *input_dev;
+	const void *match_data;
 	int error;
 
-	pdata = dev_get_platdata(&client->dev);
-	if (!pdata)
-		pdata = mms114_parse_dt(&client->dev);
-
-	if (!pdata) {
-		dev_err(&client->dev, "Need platform data\n");
-		return -EINVAL;
-	}
-
 	if (!i2c_check_functionality(client->adapter,
 				I2C_FUNC_PROTOCOL_MANGLING)) {
 		dev_err(&client->dev,
@@ -454,29 +445,63 @@ static int mms114_probe(struct i2c_client *client,
 
 	data->client = client;
 	data->input_dev = input_dev;
-	data->pdata = pdata;
 
-	input_dev->name = "MELFAS MMS114 Touchscreen";
+	/* FIXME: switch to device_get_match_data() when available */
+	match_data = of_device_get_match_data(&client->dev);
+	if (!match_data)
+		return -EINVAL;
+
+	data->type = (enum mms_type)match_data;
+
+	input_set_capability(input_dev, EV_ABS, ABS_MT_POSITION_X);
+	input_set_capability(input_dev, EV_ABS, ABS_MT_POSITION_Y);
+	input_set_abs_params(input_dev, ABS_MT_PRESSURE, 0, 255, 0, 0);
+	input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR,
+			     0, MMS114_MAX_AREA, 0, 0);
+
+	touchscreen_parse_properties(input_dev, true, &data->props);
+	if (!data->props.max_x || !data->props.max_y) {
+		dev_dbg(&client->dev,
+			"missing X/Y size properties, trying legacy bindings\n");
+		error = mms114_parse_legacy_bindings(data);
+		if (error)
+			return error;
+
+		input_set_abs_params(input_dev, ABS_MT_POSITION_X,
+				     0, data->props.max_x, 0, 0);
+		input_set_abs_params(input_dev, ABS_MT_POSITION_Y,
+				     0, data->props.max_y, 0, 0);
+	}
+
+	if (data->type == TYPE_MMS114) {
+		/*
+		 * The firmware handles movement and pressure fuzz, so
+		 * don't duplicate that in software.
+		 */
+		data->moving_threshold = input_abs_get_fuzz(input_dev,
+							    ABS_MT_POSITION_X);
+		data->contact_threshold = input_abs_get_fuzz(input_dev,
+							     ABS_MT_PRESSURE);
+		input_abs_set_fuzz(input_dev, ABS_MT_POSITION_X, 0);
+		input_abs_set_fuzz(input_dev, ABS_MT_POSITION_Y, 0);
+		input_abs_set_fuzz(input_dev, ABS_MT_PRESSURE, 0);
+	}
+
+	input_dev->name = devm_kasprintf(&client->dev, GFP_KERNEL,
+					 "MELFAS MMS%d Touchscreen",
+					 data->type);
+	if (!input_dev->name)
+		return -ENOMEM;
+
 	input_dev->id.bustype = BUS_I2C;
 	input_dev->dev.parent = &client->dev;
 	input_dev->open = mms114_input_open;
 	input_dev->close = mms114_input_close;
 
-	__set_bit(EV_ABS, input_dev->evbit);
-	__set_bit(EV_KEY, input_dev->evbit);
-	__set_bit(BTN_TOUCH, input_dev->keybit);
-	input_set_abs_params(input_dev, ABS_X, 0, data->pdata->x_size, 0, 0);
-	input_set_abs_params(input_dev, ABS_Y, 0, data->pdata->y_size, 0, 0);
-
-	/* For multi touch */
-	input_mt_init_slots(input_dev, MMS114_MAX_TOUCH, 0);
-	input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR,
-			     0, MMS114_MAX_AREA, 0, 0);
-	input_set_abs_params(input_dev, ABS_MT_POSITION_X,
-			     0, data->pdata->x_size, 0, 0);
-	input_set_abs_params(input_dev, ABS_MT_POSITION_Y,
-			     0, data->pdata->y_size, 0, 0);
-	input_set_abs_params(input_dev, ABS_MT_PRESSURE, 0, 255, 0, 0);
+	error = input_mt_init_slots(input_dev, MMS114_MAX_TOUCH,
+				    INPUT_MT_DIRECT);
+	if (error)
+		return error;
 
 	input_set_drvdata(input_dev, data);
 	i2c_set_clientdata(client, data);
@@ -497,9 +522,9 @@ static int mms114_probe(struct i2c_client *client,
 		return error;
 	}
 
-	error = devm_request_threaded_irq(&client->dev, client->irq, NULL,
-			mms114_interrupt, IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-			dev_name(&client->dev), data);
+	error = devm_request_threaded_irq(&client->dev, client->irq,
+					  NULL, mms114_interrupt, IRQF_ONESHOT,
+					  dev_name(&client->dev), data);
 	if (error) {
 		dev_err(&client->dev, "Failed to register interrupt\n");
 		return error;
@@ -569,7 +594,13 @@ MODULE_DEVICE_TABLE(i2c, mms114_id);
 
 #ifdef CONFIG_OF
 static const struct of_device_id mms114_dt_match[] = {
-	{ .compatible = "melfas,mms114" },
+	{
+		.compatible = "melfas,mms114",
+		.data = (void *)TYPE_MMS114,
+	}, {
+		.compatible = "melfas,mms152",
+		.data = (void *)TYPE_MMS152,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, mms114_dt_match);
@@ -590,4 +621,4 @@ module_i2c_driver(mms114_driver);
 /* Module information */
 MODULE_AUTHOR("Joonyoung Shim <jy0922.shim@samsung.com>");
 MODULE_DESCRIPTION("MELFAS mms114 Touchscreen driver");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/input/touchscreen/raydium_i2c_ts.c b/drivers/input/touchscreen/raydium_i2c_ts.c
index 100538d64fff..d1c09e6a2cb6 100644
--- a/drivers/input/touchscreen/raydium_i2c_ts.c
+++ b/drivers/input/touchscreen/raydium_i2c_ts.c
@@ -752,13 +752,20 @@ static int raydium_i2c_fw_update(struct raydium_data *ts)
 {
 	struct i2c_client *client = ts->client;
 	const struct firmware *fw = NULL;
-	const char *fw_file = "raydium.fw";
+	char *fw_file;
 	int error;
 
+	fw_file = kasprintf(GFP_KERNEL, "raydium_%#04x.fw",
+			    le32_to_cpu(ts->info.hw_ver));
+	if (!fw_file)
+		return -ENOMEM;
+
+	dev_dbg(&client->dev, "firmware name: %s\n", fw_file);
+
 	error = request_firmware(&fw, fw_file, &client->dev);
 	if (error) {
 		dev_err(&client->dev, "Unable to open firmware %s\n", fw_file);
-		return error;
+		goto out_free_fw_file;
 	}
 
 	disable_irq(client->irq);
@@ -787,6 +794,9 @@ static int raydium_i2c_fw_update(struct raydium_data *ts)
 
 	release_firmware(fw);
 
+out_free_fw_file:
+	kfree(fw_file);
+
 	return error;
 }
 
diff --git a/drivers/input/touchscreen/silead.c b/drivers/input/touchscreen/silead.c
index 0dbcf105f7db..646b1e768e6b 100644
--- a/drivers/input/touchscreen/silead.c
+++ b/drivers/input/touchscreen/silead.c
@@ -56,7 +56,7 @@
 #define SILEAD_POINT_Y_MSB_OFF	0x01
 #define SILEAD_POINT_X_OFF	0x02
 #define SILEAD_POINT_X_MSB_OFF	0x03
-#define SILEAD_TOUCH_ID_MASK	0xF0
+#define SILEAD_EXTRA_DATA_MASK	0xF0
 
 #define SILEAD_CMD_SLEEP_MIN	10000
 #define SILEAD_CMD_SLEEP_MAX	20000
@@ -109,6 +109,9 @@ static int silead_ts_request_input_dev(struct silead_ts_data *data)
 			    INPUT_MT_DIRECT | INPUT_MT_DROP_UNUSED |
 			    INPUT_MT_TRACK);
 
+	if (device_property_read_bool(dev, "silead,home-button"))
+		input_set_capability(data->input, EV_KEY, KEY_LEFTMETA);
+
 	data->input->name = SILEAD_TS_NAME;
 	data->input->phys = "input/ts";
 	data->input->id.bustype = BUS_I2C;
@@ -139,7 +142,8 @@ static void silead_ts_read_data(struct i2c_client *client)
 	struct input_dev *input = data->input;
 	struct device *dev = &client->dev;
 	u8 *bufp, buf[SILEAD_TS_DATA_LEN];
-	int touch_nr, error, i;
+	int touch_nr, softbutton, error, i;
+	bool softbutton_pressed = false;
 
 	error = i2c_smbus_read_i2c_block_data(client, SILEAD_REG_DATA,
 					      SILEAD_TS_DATA_LEN, buf);
@@ -148,21 +152,40 @@ static void silead_ts_read_data(struct i2c_client *client)
 		return;
 	}
 
-	touch_nr = buf[0];
-	if (touch_nr > data->max_fingers) {
+	if (buf[0] > data->max_fingers) {
 		dev_warn(dev, "More touches reported then supported %d > %d\n",
-			 touch_nr, data->max_fingers);
-		touch_nr = data->max_fingers;
+			 buf[0], data->max_fingers);
+		buf[0] = data->max_fingers;
 	}
 
+	touch_nr = 0;
 	bufp = buf + SILEAD_POINT_DATA_LEN;
-	for (i = 0; i < touch_nr; i++, bufp += SILEAD_POINT_DATA_LEN) {
-		/* Bits 4-7 are the touch id */
-		data->id[i] = (bufp[SILEAD_POINT_X_MSB_OFF] &
-			       SILEAD_TOUCH_ID_MASK) >> 4;
-		touchscreen_set_mt_pos(&data->pos[i], &data->prop,
+	for (i = 0; i < buf[0]; i++, bufp += SILEAD_POINT_DATA_LEN) {
+		softbutton = (bufp[SILEAD_POINT_Y_MSB_OFF] &
+			      SILEAD_EXTRA_DATA_MASK) >> 4;
+
+		if (softbutton) {
+			/*
+			 * For now only respond to softbutton == 0x01, some
+			 * tablets *without* a capacative button send 0x04
+			 * when crossing the edges of the screen.
+			 */
+			if (softbutton == 0x01)
+				softbutton_pressed = true;
+
+			continue;
+		}
+
+		/*
+		 * Bits 4-7 are the touch id, note not all models have
+		 * hardware touch ids so atm we don't use these.
+		 */
+		data->id[touch_nr] = (bufp[SILEAD_POINT_X_MSB_OFF] &
+				      SILEAD_EXTRA_DATA_MASK) >> 4;
+		touchscreen_set_mt_pos(&data->pos[touch_nr], &data->prop,
 			get_unaligned_le16(&bufp[SILEAD_POINT_X_OFF]) & 0xfff,
 			get_unaligned_le16(&bufp[SILEAD_POINT_Y_OFF]) & 0xfff);
+		touch_nr++;
 	}
 
 	input_mt_assign_slots(input, data->slots, data->pos, touch_nr, 0);
@@ -178,6 +201,7 @@ static void silead_ts_read_data(struct i2c_client *client)
 	}
 
 	input_mt_sync_frame(input);
+	input_report_key(input, KEY_LEFTMETA, softbutton_pressed);
 	input_sync(input);
 }
 
diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c
index 2a123e20a42e..efdb1a75a163 100644
--- a/drivers/input/touchscreen/stmfts.c
+++ b/drivers/input/touchscreen/stmfts.c
@@ -682,6 +682,14 @@ static int stmfts_probe(struct i2c_client *client,
 
 	input_set_drvdata(sdata->input, sdata);
 
+	/*
+	 * stmfts_power_on expects interrupt to be disabled, but
+	 * at this point the device is still off and I do not trust
+	 * the status of the irq line that can generate some spurious
+	 * interrupts. To be on the safe side it's better to not enable
+	 * the interrupts during their request.
+	 */
+	irq_set_status_flags(client->irq, IRQ_NOAUTOEN);
 	err = devm_request_threaded_irq(&client->dev, client->irq,
 					NULL, stmfts_irq_handler,
 					IRQF_ONESHOT,
@@ -689,9 +697,6 @@ static int stmfts_probe(struct i2c_client *client,
 	if (err)
 		return err;
 
-	/* stmfts_power_on expects interrupt to be disabled */
-	disable_irq(client->irq);
-
 	dev_dbg(&client->dev, "initializing ST-Microelectronics FTS...\n");
 
 	err = stmfts_power_on(sdata);
diff --git a/drivers/input/touchscreen/usbtouchscreen.c b/drivers/input/touchscreen/usbtouchscreen.c
index 2c41107240de..aa77d243b786 100644
--- a/drivers/input/touchscreen/usbtouchscreen.c
+++ b/drivers/input/touchscreen/usbtouchscreen.c
@@ -55,11 +55,6 @@
 #include <linux/usb/input.h>
 #include <linux/hid.h>
 
-
-#define DRIVER_VERSION		"v0.6"
-#define DRIVER_AUTHOR		"Daniel Ritz <daniel.ritz@gmx.ch>"
-#define DRIVER_DESC		"USB Touchscreen Driver"
-
 static bool swap_xy;
 module_param(swap_xy, bool, 0644);
 MODULE_PARM_DESC(swap_xy, "If set X and Y axes are swapped.");
@@ -1763,8 +1758,8 @@ static struct usb_driver usbtouch_driver = {
 
 module_usb_driver(usbtouch_driver);
 
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_AUTHOR("Daniel Ritz <daniel.ritz@gmx.ch>");
+MODULE_DESCRIPTION("USB Touchscreen Driver");
 MODULE_LICENSE("GPL");
 
 MODULE_ALIAS("touchkitusb");
diff --git a/drivers/input/touchscreen/wdt87xx_i2c.c b/drivers/input/touchscreen/wdt87xx_i2c.c
index d351efd18f89..20f7f3902757 100644
--- a/drivers/input/touchscreen/wdt87xx_i2c.c
+++ b/drivers/input/touchscreen/wdt87xx_i2c.c
@@ -23,7 +23,6 @@
 #include <asm/unaligned.h>
 
 #define WDT87XX_NAME		"wdt87xx_i2c"
-#define WDT87XX_DRV_VER		"0.9.8"
 #define WDT87XX_FW_NAME		"wdt87xx_fw.bin"
 #define WDT87XX_CFG_NAME	"wdt87xx_cfg.bin"
 
@@ -1183,5 +1182,4 @@ module_i2c_driver(wdt87xx_driver);
 
 MODULE_AUTHOR("HN Chen <hn.chen@weidahitech.com>");
 MODULE_DESCRIPTION("WeidaHiTech WDT87XX Touchscreen driver");
-MODULE_VERSION(WDT87XX_DRV_VER);
 MODULE_LICENSE("GPL");
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 97baf88d9505..74788fdeb773 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -527,7 +527,8 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id,
 	struct iommu_dev_data *dev_data = NULL;
 	struct pci_dev *pdev;
 
-	pdev = pci_get_bus_and_slot(PCI_BUS_NUM(devid), devid & 0xff);
+	pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid),
+					   devid & 0xff);
 	if (pdev)
 		dev_data = get_dev_data(&pdev->dev);
 
@@ -616,7 +617,9 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
 		       address, flags);
 		break;
 	default:
-		printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
+		printk(KERN_ERR "UNKNOWN type=0x%02x event[0]=0x%08x "
+		       "event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n",
+		       type, event[0], event[1], event[2], event[3]);
 	}
 
 	memset(__evt, 0, 4 * sizeof(u32));
@@ -1815,7 +1818,8 @@ static bool dma_ops_domain(struct protection_domain *domain)
 	return domain->flags & PD_DMA_OPS_MASK;
 }
 
-static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
+static void set_dte_entry(u16 devid, struct protection_domain *domain,
+			  bool ats, bool ppr)
 {
 	u64 pte_root = 0;
 	u64 flags = 0;
@@ -1832,6 +1836,13 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
 	if (ats)
 		flags |= DTE_FLAG_IOTLB;
 
+	if (ppr) {
+		struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
+
+		if (iommu_feature(iommu, FEATURE_EPHSUP))
+			pte_root |= 1ULL << DEV_ENTRY_PPR;
+	}
+
 	if (domain->flags & PD_IOMMUV2_MASK) {
 		u64 gcr3 = iommu_virt_to_phys(domain->gcr3_tbl);
 		u64 glx  = domain->glx;
@@ -1894,9 +1905,9 @@ static void do_attach(struct iommu_dev_data *dev_data,
 	domain->dev_cnt                 += 1;
 
 	/* Update device table */
-	set_dte_entry(dev_data->devid, domain, ats);
+	set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2);
 	if (alias != dev_data->devid)
-		set_dte_entry(alias, domain, ats);
+		set_dte_entry(alias, domain, ats, dev_data->iommu_v2);
 
 	device_flush_dte(dev_data);
 }
@@ -2275,13 +2286,15 @@ static void update_device_table(struct protection_domain *domain)
 	struct iommu_dev_data *dev_data;
 
 	list_for_each_entry(dev_data, &domain->dev_list, list) {
-		set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
+		set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled,
+			      dev_data->iommu_v2);
 
 		if (dev_data->devid == dev_data->alias)
 			continue;
 
 		/* There is an alias, update device table entry for it */
-		set_dte_entry(dev_data->alias, domain, dev_data->ats.enabled);
+		set_dte_entry(dev_data->alias, domain, dev_data->ats.enabled,
+			      dev_data->iommu_v2);
 	}
 }
 
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 6fe2d0346073..4e4a615bf13f 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -1697,8 +1697,8 @@ static int iommu_init_pci(struct amd_iommu *iommu)
 	u32 range, misc, low, high;
 	int ret;
 
-	iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid),
-					  iommu->devid & 0xff);
+	iommu->dev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(iommu->devid),
+						 iommu->devid & 0xff);
 	if (!iommu->dev)
 		return -ENODEV;
 
@@ -1764,8 +1764,9 @@ static int iommu_init_pci(struct amd_iommu *iommu)
 	if (is_rd890_iommu(iommu->dev)) {
 		int i, j;
 
-		iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,
-				PCI_DEVFN(0, 0));
+		iommu->root_pdev =
+			pci_get_domain_bus_and_slot(0, iommu->dev->bus->number,
+						    PCI_DEVFN(0, 0));
 
 		/*
 		 * Some rd890 systems may not be fully reconfigured by the
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index f6b24c7d8b70..6a877ebd058b 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -98,6 +98,7 @@
 #define FEATURE_HE		(1ULL<<8)
 #define FEATURE_PC		(1ULL<<9)
 #define FEATURE_GAM_VAPIC	(1ULL<<21)
+#define FEATURE_EPHSUP		(1ULL<<50)
 
 #define FEATURE_PASID_SHIFT	32
 #define FEATURE_PASID_MASK	(0x1fULL << FEATURE_PASID_SHIFT)
@@ -192,6 +193,7 @@
 /* macros and definitions for device table entries */
 #define DEV_ENTRY_VALID         0x00
 #define DEV_ENTRY_TRANSLATION   0x01
+#define DEV_ENTRY_PPR           0x34
 #define DEV_ENTRY_IR            0x3d
 #define DEV_ENTRY_IW            0x3e
 #define DEV_ENTRY_NO_PAGE_FAULT	0x62
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 7d94e1d39e5e..1d0b53a04a08 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -427,6 +427,7 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
 }
 
 static const struct mmu_notifier_ops iommu_mn = {
+	.flags			= MMU_INVALIDATE_DOES_NOT_BLOCK,
 	.release		= mn_release,
 	.clear_flush_young      = mn_clear_flush_young,
 	.invalidate_range       = mn_invalidate_range,
@@ -564,7 +565,8 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data)
 	finish      = (iommu_fault->tag >> 9) & 1;
 
 	devid = iommu_fault->device_id;
-	pdev = pci_get_bus_and_slot(PCI_BUS_NUM(devid), devid & 0xff);
+	pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid),
+					   devid & 0xff);
 	if (!pdev)
 		return -ENODEV;
 	dev_data = get_dev_data(&pdev->dev);
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 744592d330ca..3f2f1fc68b52 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -2971,7 +2971,7 @@ static struct platform_driver arm_smmu_driver = {
 };
 module_platform_driver(arm_smmu_driver);
 
-IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3", NULL);
+IOMMU_OF_DECLARE(arm_smmuv3, "arm,smmu-v3");
 
 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index 78d4c6b8f1ba..69e7c60792a8 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -2211,12 +2211,12 @@ static struct platform_driver arm_smmu_driver = {
 };
 module_platform_driver(arm_smmu_driver);
 
-IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", NULL);
-IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", NULL);
-IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", NULL);
-IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", NULL);
-IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", NULL);
-IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", NULL);
+IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1");
+IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2");
+IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400");
+IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401");
+IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500");
+IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2");
 
 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 79c45650f8de..2138102ef611 100644
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1353,8 +1353,15 @@ static const struct iommu_ops exynos_iommu_ops = {
 
 static int __init exynos_iommu_init(void)
 {
+	struct device_node *np;
 	int ret;
 
+	np = of_find_matching_node(NULL, sysmmu_of_match);
+	if (!np)
+		return 0;
+
+	of_node_put(np);
+
 	lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table",
 				LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL);
 	if (!lv2table_kmem_cache) {
@@ -1394,4 +1401,4 @@ static int __init exynos_iommu_init(void)
 }
 core_initcall(exynos_iommu_init);
 
-IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu", NULL);
+IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu");
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 4a2de34895ec..582fd01cb7d1 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -64,7 +64,7 @@
 #define IOAPIC_RANGE_END	(0xfeefffff)
 #define IOVA_START_ADDR		(0x1000)
 
-#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
+#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
 
 #define MAX_AGAW_WIDTH 64
 #define MAX_AGAW_PFN_WIDTH	(MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
@@ -1601,8 +1601,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
 	 * flush. However, device IOTLB doesn't need to be flushed in this case.
 	 */
 	if (!cap_caching_mode(iommu->cap) || !map)
-		iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
-				      addr, mask);
+		iommu_flush_dev_iotlb(domain, addr, mask);
 }
 
 static void iommu_flush_iova(struct iova_domain *iovad)
@@ -4808,7 +4807,7 @@ int __init intel_iommu_init(void)
 	up_write(&dmar_global_lock);
 	pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
 
-#ifdef CONFIG_SWIOTLB
+#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
 	swiotlb = 0;
 #endif
 	dma_ops = &intel_dma_ops;
diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
index ed1cf7c5a43b..99bc9bd64b9e 100644
--- a/drivers/iommu/intel-svm.c
+++ b/drivers/iommu/intel-svm.c
@@ -26,6 +26,10 @@
 #include <linux/interrupt.h>
 #include <asm/page.h>
 
+#define PASID_ENTRY_P		BIT_ULL(0)
+#define PASID_ENTRY_FLPM_5LP	BIT_ULL(9)
+#define PASID_ENTRY_SRE		BIT_ULL(11)
+
 static irqreturn_t prq_event_thread(int irq, void *d);
 
 struct pasid_entry {
@@ -41,6 +45,14 @@ int intel_svm_alloc_pasid_tables(struct intel_iommu *iommu)
 	struct page *pages;
 	int order;
 
+	if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
+			!cap_fl1gp_support(iommu->cap))
+		return -EINVAL;
+
+	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
+			!cap_5lp_support(iommu->cap))
+		return -EINVAL;
+
 	/* Start at 2 because it's defined as 2^(1+PSS) */
 	iommu->pasid_max = 2 << ecap_pss(iommu->ecap);
 
@@ -129,6 +141,7 @@ int intel_svm_enable_prq(struct intel_iommu *iommu)
 		pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n",
 		       iommu->name);
 		dmar_free_hwirq(irq);
+		iommu->pr_irq = 0;
 		goto err;
 	}
 	dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL);
@@ -144,9 +157,11 @@ int intel_svm_finish_prq(struct intel_iommu *iommu)
 	dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL);
 	dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL);
 
-	free_irq(iommu->pr_irq, iommu);
-	dmar_free_hwirq(iommu->pr_irq);
-	iommu->pr_irq = 0;
+	if (iommu->pr_irq) {
+		free_irq(iommu->pr_irq, iommu);
+		dmar_free_hwirq(iommu->pr_irq);
+		iommu->pr_irq = 0;
+	}
 
 	free_pages((unsigned long)iommu->prq, PRQ_ORDER);
 	iommu->prq = NULL;
@@ -190,7 +205,7 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d
 			 * for example, an "address" value of 0x12345f000 will
 			 * flush from 0x123440000 to 0x12347ffff (256KiB). */
 			unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT);
-			unsigned long mask = __rounddown_pow_of_two(address ^ last);;
+			unsigned long mask = __rounddown_pow_of_two(address ^ last);
 
 			desc.high = QI_DEV_EIOTLB_ADDR((address & ~mask) | (mask - 1)) | QI_DEV_EIOTLB_SIZE;
 		} else {
@@ -276,6 +291,7 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 }
 
 static const struct mmu_notifier_ops intel_mmuops = {
+	.flags = MMU_INVALIDATE_DOES_NOT_BLOCK,
 	.release = intel_mm_release,
 	.change_pte = intel_change_pte,
 	.invalidate_range = intel_invalidate_range,
@@ -289,6 +305,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 	struct intel_svm_dev *sdev;
 	struct intel_svm *svm = NULL;
 	struct mm_struct *mm = NULL;
+	u64 pasid_entry_val;
 	int pasid_max;
 	int ret;
 
@@ -395,9 +412,15 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_
 				kfree(sdev);
 				goto out;
 			}
-			iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) | 1;
+			pasid_entry_val = (u64)__pa(mm->pgd) | PASID_ENTRY_P;
 		} else
-			iommu->pasid_table[svm->pasid].val = (u64)__pa(init_mm.pgd) | 1 | (1ULL << 11);
+			pasid_entry_val = (u64)__pa(init_mm.pgd) |
+					  PASID_ENTRY_P | PASID_ENTRY_SRE;
+		if (cpu_feature_enabled(X86_FEATURE_LA57))
+			pasid_entry_val |= PASID_ENTRY_FLPM_5LP;
+
+		iommu->pasid_table[svm->pasid].val = pasid_entry_val;
+
 		wmb();
 		/* In caching mode, we still have to flush with PASID 0 when
 		 * a PASID table entry becomes present. Not entirely clear
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 3de5c0bcb5cc..69fef991c651 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -1303,6 +1303,9 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev)
 	int ret;
 
 	group = iommu_group_get(dev);
+	if (!group)
+		return -ENODEV;
+
 	/*
 	 * Lock the group to make sure the device-count doesn't
 	 * change while we are attaching
@@ -1341,6 +1344,8 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev)
 	struct iommu_group *group;
 
 	group = iommu_group_get(dev);
+	if (!group)
+		return;
 
 	mutex_lock(&group->mutex);
 	if (iommu_group_device_count(group) != 1) {
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 8dce3a9de9d8..40ae6e87cb88 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -1108,18 +1108,8 @@ static void __exit ipmmu_exit(void)
 subsys_initcall(ipmmu_init);
 module_exit(ipmmu_exit);
 
-#ifdef CONFIG_IOMMU_DMA
-static int __init ipmmu_vmsa_iommu_of_setup(struct device_node *np)
-{
-	ipmmu_init();
-	return 0;
-}
-
-IOMMU_OF_DECLARE(ipmmu_vmsa_iommu_of, "renesas,ipmmu-vmsa",
-		 ipmmu_vmsa_iommu_of_setup);
-IOMMU_OF_DECLARE(ipmmu_r8a7795_iommu_of, "renesas,ipmmu-r8a7795",
-		 ipmmu_vmsa_iommu_of_setup);
-#endif
+IOMMU_OF_DECLARE(ipmmu_vmsa_iommu_of, "renesas,ipmmu-vmsa");
+IOMMU_OF_DECLARE(ipmmu_r8a7795_iommu_of, "renesas,ipmmu-r8a7795");
 
 MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");
 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 04f4d51ffacb..0d3350463a3f 100644
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -823,6 +823,8 @@ static int msm_iommu_probe(struct platform_device *pdev)
 		goto fail;
 	}
 
+	bus_set_iommu(&platform_bus_type, &msm_iommu_ops);
+
 	pr_info("device mapped at %p, irq %d with %d ctx banks\n",
 		iommu->base, iommu->irq, iommu->ncb);
 
@@ -875,19 +877,7 @@ static void __exit msm_iommu_driver_exit(void)
 subsys_initcall(msm_iommu_driver_init);
 module_exit(msm_iommu_driver_exit);
 
-static int __init msm_iommu_init(void)
-{
-	bus_set_iommu(&platform_bus_type, &msm_iommu_ops);
-	return 0;
-}
-
-static int __init msm_iommu_of_setup(struct device_node *np)
-{
-	msm_iommu_init();
-	return 0;
-}
-
-IOMMU_OF_DECLARE(msm_iommu_of, "qcom,apq8064-iommu", msm_iommu_of_setup);
+IOMMU_OF_DECLARE(msm_iommu_of, "qcom,apq8064-iommu");
 
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 50947ebb6d17..5c36a8b7656a 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -231,19 +231,3 @@ const struct iommu_ops *of_iommu_configure(struct device *dev,
 
 	return ops;
 }
-
-static int __init of_iommu_init(void)
-{
-	struct device_node *np;
-	const struct of_device_id *match, *matches = &__iommu_of_table;
-
-	for_each_matching_node_and_match(np, matches, &match) {
-		const of_iommu_init_fn init_fn = match->data;
-
-		if (init_fn && init_fn(np))
-			pr_err("Failed to initialise IOMMU %pOF\n", np);
-	}
-
-	return 0;
-}
-postcore_initcall_sync(of_iommu_init);
diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c
index 505548aafeff..50217548c3b8 100644
--- a/drivers/iommu/omap-iommu-debug.c
+++ b/drivers/iommu/omap-iommu-debug.c
@@ -274,8 +274,8 @@ void omap_iommu_debugfs_add(struct omap_iommu *obj)
 	if (!obj->debug_dir)
 		return;
 
-	d = debugfs_create_u8("nr_tlb_entries", 0400, obj->debug_dir,
-			      (u8 *)&obj->nr_tlb_entries);
+	d = debugfs_create_u32("nr_tlb_entries", 0400, obj->debug_dir,
+			       &obj->nr_tlb_entries);
 	if (!d)
 		return;
 
diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
index e07f02d00c68..65b9c99707f8 100644
--- a/drivers/iommu/qcom_iommu.c
+++ b/drivers/iommu/qcom_iommu.c
@@ -947,7 +947,7 @@ static void __exit qcom_iommu_exit(void)
 module_init(qcom_iommu_init);
 module_exit(qcom_iommu_exit);
 
-IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1", NULL);
+IOMMU_OF_DECLARE(qcom_iommu_dev, "qcom,msm-iommu-v1");
 
 MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 3b6449e2cbf1..44d40bc771b5 100644
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -20,6 +20,12 @@
 #include <soc/tegra/ahb.h>
 #include <soc/tegra/mc.h>
 
+struct tegra_smmu_group {
+	struct list_head list;
+	const struct tegra_smmu_group_soc *soc;
+	struct iommu_group *group;
+};
+
 struct tegra_smmu {
 	void __iomem *regs;
 	struct device *dev;
@@ -27,6 +33,8 @@ struct tegra_smmu {
 	struct tegra_mc *mc;
 	const struct tegra_smmu_soc *soc;
 
+	struct list_head groups;
+
 	unsigned long pfn_mask;
 	unsigned long tlb_mask;
 
@@ -703,19 +711,47 @@ static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
 	return mc->smmu;
 }
 
+static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,
+				struct of_phandle_args *args)
+{
+	const struct iommu_ops *ops = smmu->iommu.ops;
+	int err;
+
+	err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops);
+	if (err < 0) {
+		dev_err(dev, "failed to initialize fwspec: %d\n", err);
+		return err;
+	}
+
+	err = ops->of_xlate(dev, args);
+	if (err < 0) {
+		dev_err(dev, "failed to parse SW group ID: %d\n", err);
+		iommu_fwspec_free(dev);
+		return err;
+	}
+
+	return 0;
+}
+
 static int tegra_smmu_add_device(struct device *dev)
 {
 	struct device_node *np = dev->of_node;
+	struct tegra_smmu *smmu = NULL;
 	struct iommu_group *group;
 	struct of_phandle_args args;
 	unsigned int index = 0;
+	int err;
 
 	while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
 					  &args) == 0) {
-		struct tegra_smmu *smmu;
-
 		smmu = tegra_smmu_find(args.np);
 		if (smmu) {
+			err = tegra_smmu_configure(smmu, dev, &args);
+			of_node_put(args.np);
+
+			if (err < 0)
+				return err;
+
 			/*
 			 * Only a single IOMMU master interface is currently
 			 * supported by the Linux kernel, so abort after the
@@ -728,9 +764,13 @@ static int tegra_smmu_add_device(struct device *dev)
 			break;
 		}
 
+		of_node_put(args.np);
 		index++;
 	}
 
+	if (!smmu)
+		return -ENODEV;
+
 	group = iommu_group_get_for_dev(dev);
 	if (IS_ERR(group))
 		return PTR_ERR(group);
@@ -751,6 +791,80 @@ static void tegra_smmu_remove_device(struct device *dev)
 	iommu_group_remove_device(dev);
 }
 
+static const struct tegra_smmu_group_soc *
+tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup)
+{
+	unsigned int i, j;
+
+	for (i = 0; i < smmu->soc->num_groups; i++)
+		for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)
+			if (smmu->soc->groups[i].swgroups[j] == swgroup)
+				return &smmu->soc->groups[i];
+
+	return NULL;
+}
+
+static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu,
+						unsigned int swgroup)
+{
+	const struct tegra_smmu_group_soc *soc;
+	struct tegra_smmu_group *group;
+
+	soc = tegra_smmu_find_group(smmu, swgroup);
+	if (!soc)
+		return NULL;
+
+	mutex_lock(&smmu->lock);
+
+	list_for_each_entry(group, &smmu->groups, list)
+		if (group->soc == soc) {
+			mutex_unlock(&smmu->lock);
+			return group->group;
+		}
+
+	group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);
+	if (!group) {
+		mutex_unlock(&smmu->lock);
+		return NULL;
+	}
+
+	INIT_LIST_HEAD(&group->list);
+	group->soc = soc;
+
+	group->group = iommu_group_alloc();
+	if (IS_ERR(group->group)) {
+		devm_kfree(smmu->dev, group);
+		mutex_unlock(&smmu->lock);
+		return NULL;
+	}
+
+	list_add_tail(&group->list, &smmu->groups);
+	mutex_unlock(&smmu->lock);
+
+	return group->group;
+}
+
+static struct iommu_group *tegra_smmu_device_group(struct device *dev)
+{
+	struct iommu_fwspec *fwspec = dev->iommu_fwspec;
+	struct tegra_smmu *smmu = dev->archdata.iommu;
+	struct iommu_group *group;
+
+	group = tegra_smmu_group_get(smmu, fwspec->ids[0]);
+	if (!group)
+		group = generic_device_group(dev);
+
+	return group;
+}
+
+static int tegra_smmu_of_xlate(struct device *dev,
+			       struct of_phandle_args *args)
+{
+	u32 id = args->args[0];
+
+	return iommu_fwspec_add_ids(dev, &id, 1);
+}
+
 static const struct iommu_ops tegra_smmu_ops = {
 	.capable = tegra_smmu_capable,
 	.domain_alloc = tegra_smmu_domain_alloc,
@@ -759,12 +873,12 @@ static const struct iommu_ops tegra_smmu_ops = {
 	.detach_dev = tegra_smmu_detach_dev,
 	.add_device = tegra_smmu_add_device,
 	.remove_device = tegra_smmu_remove_device,
-	.device_group = generic_device_group,
+	.device_group = tegra_smmu_device_group,
 	.map = tegra_smmu_map,
 	.unmap = tegra_smmu_unmap,
 	.map_sg = default_iommu_map_sg,
 	.iova_to_phys = tegra_smmu_iova_to_phys,
-
+	.of_xlate = tegra_smmu_of_xlate,
 	.pgsize_bitmap = SZ_4K,
 };
 
@@ -913,6 +1027,7 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
 	if (!smmu->asids)
 		return ERR_PTR(-ENOMEM);
 
+	INIT_LIST_HEAD(&smmu->groups);
 	mutex_init(&smmu->lock);
 
 	smmu->regs = mc->regs;
@@ -954,6 +1069,7 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev,
 		return ERR_PTR(err);
 
 	iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);
+	iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
 
 	err = iommu_device_register(&smmu->iommu);
 	if (err) {
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index c70476b34a53..d913aec85109 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -343,4 +343,12 @@ config MESON_IRQ_GPIO
        help
          Support Meson SoC Family GPIO Interrupt Multiplexer
 
+config GOLDFISH_PIC
+       bool "Goldfish programmable interrupt controller"
+       depends on MIPS && (GOLDFISH || COMPILE_TEST)
+       select IRQ_DOMAIN
+       help
+         Say yes here to enable Goldfish interrupt controller driver used
+         for Goldfish based virtual platforms.
+
 endmenu
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index d2df34a54d38..d27e3e3619e0 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -84,3 +84,4 @@ obj-$(CONFIG_QCOM_IRQ_COMBINER)		+= qcom-irq-combiner.o
 obj-$(CONFIG_IRQ_UNIPHIER_AIDET)	+= irq-uniphier-aidet.o
 obj-$(CONFIG_ARCH_SYNQUACER)		+= irq-sni-exiu.o
 obj-$(CONFIG_MESON_IRQ_GPIO)		+= irq-meson-gpio.o
+obj-$(CONFIG_GOLDFISH_PIC) 		+= irq-goldfish-pic.o
diff --git a/drivers/irqchip/irq-bcm2836.c b/drivers/irqchip/irq-bcm2836.c
index 667b9e14b032..dfe4a460340b 100644
--- a/drivers/irqchip/irq-bcm2836.c
+++ b/drivers/irqchip/irq-bcm2836.c
@@ -98,13 +98,35 @@ static struct irq_chip bcm2836_arm_irqchip_gpu = {
 	.irq_unmask	= bcm2836_arm_irqchip_unmask_gpu_irq,
 };
 
-static void bcm2836_arm_irqchip_register_irq(int hwirq, struct irq_chip *chip)
-{
-	int irq = irq_create_mapping(intc.domain, hwirq);
+static int bcm2836_map(struct irq_domain *d, unsigned int irq,
+		       irq_hw_number_t hw)
+{
+	struct irq_chip *chip;
+
+	switch (hw) {
+	case LOCAL_IRQ_CNTPSIRQ:
+	case LOCAL_IRQ_CNTPNSIRQ:
+	case LOCAL_IRQ_CNTHPIRQ:
+	case LOCAL_IRQ_CNTVIRQ:
+		chip = &bcm2836_arm_irqchip_timer;
+		break;
+	case LOCAL_IRQ_GPU_FAST:
+		chip = &bcm2836_arm_irqchip_gpu;
+		break;
+	case LOCAL_IRQ_PMU_FAST:
+		chip = &bcm2836_arm_irqchip_pmu;
+		break;
+	default:
+		pr_warn_once("Unexpected hw irq: %lu\n", hw);
+		return -EINVAL;
+	}
 
 	irq_set_percpu_devid(irq);
-	irq_set_chip_and_handler(irq, chip, handle_percpu_devid_irq);
+	irq_domain_set_info(d, irq, hw, chip, d->host_data,
+			    handle_percpu_devid_irq, NULL, NULL);
 	irq_set_status_flags(irq, IRQ_NOAUTOEN);
+
+	return 0;
 }
 
 static void
@@ -165,7 +187,8 @@ static int bcm2836_cpu_dying(unsigned int cpu)
 #endif
 
 static const struct irq_domain_ops bcm2836_arm_irqchip_intc_ops = {
-	.xlate = irq_domain_xlate_onecell
+	.xlate = irq_domain_xlate_onetwocell,
+	.map = bcm2836_map,
 };
 
 static void
@@ -218,19 +241,6 @@ static int __init bcm2836_arm_irqchip_l1_intc_of_init(struct device_node *node,
 	if (!intc.domain)
 		panic("%pOF: unable to create IRQ domain\n", node);
 
-	bcm2836_arm_irqchip_register_irq(LOCAL_IRQ_CNTPSIRQ,
-					 &bcm2836_arm_irqchip_timer);
-	bcm2836_arm_irqchip_register_irq(LOCAL_IRQ_CNTPNSIRQ,
-					 &bcm2836_arm_irqchip_timer);
-	bcm2836_arm_irqchip_register_irq(LOCAL_IRQ_CNTHPIRQ,
-					 &bcm2836_arm_irqchip_timer);
-	bcm2836_arm_irqchip_register_irq(LOCAL_IRQ_CNTVIRQ,
-					 &bcm2836_arm_irqchip_timer);
-	bcm2836_arm_irqchip_register_irq(LOCAL_IRQ_GPU_FAST,
-					 &bcm2836_arm_irqchip_gpu);
-	bcm2836_arm_irqchip_register_irq(LOCAL_IRQ_PMU_FAST,
-					 &bcm2836_arm_irqchip_pmu);
-
 	bcm2836_arm_irqchip_smp_init();
 
 	set_handle_irq(bcm2836_arm_irqchip_handle_irq);
diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c
index 55cfb986225b..faf734ff4cf3 100644
--- a/drivers/irqchip/irq-bcm7038-l1.c
+++ b/drivers/irqchip/irq-bcm7038-l1.c
@@ -339,9 +339,6 @@ int __init bcm7038_l1_of_init(struct device_node *dn,
 		goto out_unmap;
 	}
 
-	pr_info("registered BCM7038 L1 intc (mem: 0x%p, IRQs: %d)\n",
-		intc->cpus[0]->map_base, IRQS_PER_WORD * intc->n_words);
-
 	return 0;
 
 out_unmap:
diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c
index 983640eba418..8968e5e93fcb 100644
--- a/drivers/irqchip/irq-bcm7120-l2.c
+++ b/drivers/irqchip/irq-bcm7120-l2.c
@@ -318,9 +318,6 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn,
 		}
 	}
 
-	pr_info("registered %s intc (mem: 0x%p, parent IRQ(s): %d)\n",
-			intc_name, data->map_base[0], data->num_parent_irqs);
-
 	return 0;
 
 out_free_domain:
diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c
index 691d20eb0bec..0e65f609352e 100644
--- a/drivers/irqchip/irq-brcmstb-l2.c
+++ b/drivers/irqchip/irq-brcmstb-l2.c
@@ -262,9 +262,6 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np,
 		ct->chip.irq_set_wake = irq_gc_set_wake;
 	}
 
-	pr_info("registered L2 intc (mem: 0x%p, parent irq: %d)\n",
-			base, parent_irq);
-
 	return 0;
 
 out_free_domain:
diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c
index 993a8426a453..1ff38aff9f29 100644
--- a/drivers/irqchip/irq-gic-v2m.c
+++ b/drivers/irqchip/irq-gic-v2m.c
@@ -94,7 +94,7 @@ static struct irq_chip gicv2m_msi_irq_chip = {
 
 static struct msi_domain_info gicv2m_msi_domain_info = {
 	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		   MSI_FLAG_PCI_MSIX),
+		   MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI),
 	.chip	= &gicv2m_msi_irq_chip,
 };
 
@@ -155,18 +155,12 @@ static int gicv2m_irq_gic_domain_alloc(struct irq_domain *domain,
 	return 0;
 }
 
-static void gicv2m_unalloc_msi(struct v2m_data *v2m, unsigned int hwirq)
+static void gicv2m_unalloc_msi(struct v2m_data *v2m, unsigned int hwirq,
+			       int nr_irqs)
 {
-	int pos;
-
-	pos = hwirq - v2m->spi_start;
-	if (pos < 0 || pos >= v2m->nr_spis) {
-		pr_err("Failed to teardown msi. Invalid hwirq %d\n", hwirq);
-		return;
-	}
-
 	spin_lock(&v2m_lock);
-	__clear_bit(pos, v2m->bm);
+	bitmap_release_region(v2m->bm, hwirq - v2m->spi_start,
+			      get_count_order(nr_irqs));
 	spin_unlock(&v2m_lock);
 }
 
@@ -174,13 +168,13 @@ static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 				   unsigned int nr_irqs, void *args)
 {
 	struct v2m_data *v2m = NULL, *tmp;
-	int hwirq, offset, err = 0;
+	int hwirq, offset, i, err = 0;
 
 	spin_lock(&v2m_lock);
 	list_for_each_entry(tmp, &v2m_nodes, entry) {
-		offset = find_first_zero_bit(tmp->bm, tmp->nr_spis);
-		if (offset < tmp->nr_spis) {
-			__set_bit(offset, tmp->bm);
+		offset = bitmap_find_free_region(tmp->bm, tmp->nr_spis,
+						 get_count_order(nr_irqs));
+		if (offset >= 0) {
 			v2m = tmp;
 			break;
 		}
@@ -192,16 +186,21 @@ static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
 
 	hwirq = v2m->spi_start + offset;
 
-	err = gicv2m_irq_gic_domain_alloc(domain, virq, hwirq);
-	if (err) {
-		gicv2m_unalloc_msi(v2m, hwirq);
-		return err;
-	}
+	for (i = 0; i < nr_irqs; i++) {
+		err = gicv2m_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
+		if (err)
+			goto fail;
 
-	irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
-				      &gicv2m_irq_chip, v2m);
+		irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i,
+					      &gicv2m_irq_chip, v2m);
+	}
 
 	return 0;
+
+fail:
+	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
+	gicv2m_unalloc_msi(v2m, hwirq, get_count_order(nr_irqs));
+	return err;
 }
 
 static void gicv2m_irq_domain_free(struct irq_domain *domain,
@@ -210,8 +209,7 @@ static void gicv2m_irq_domain_free(struct irq_domain *domain,
 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
 	struct v2m_data *v2m = irq_data_get_irq_chip_data(d);
 
-	BUG_ON(nr_irqs != 1);
-	gicv2m_unalloc_msi(v2m, d->hwirq);
+	gicv2m_unalloc_msi(v2m, d->hwirq, nr_irqs);
 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
 }
 
diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
index 14a8c0a7e095..25a98de5cfb2 100644
--- a/drivers/irqchip/irq-gic-v3-its-pci-msi.c
+++ b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
@@ -132,6 +132,8 @@ static int __init its_pci_of_msi_init(void)
 
 	for (np = of_find_matching_node(NULL, its_device_id); np;
 	     np = of_find_matching_node(np, its_device_id)) {
+		if (!of_device_is_available(np))
+			continue;
 		if (!of_property_read_bool(np, "msi-controller"))
 			continue;
 
diff --git a/drivers/irqchip/irq-gic-v3-its-platform-msi.c b/drivers/irqchip/irq-gic-v3-its-platform-msi.c
index 833a90fe33ae..8881a053c173 100644
--- a/drivers/irqchip/irq-gic-v3-its-platform-msi.c
+++ b/drivers/irqchip/irq-gic-v3-its-platform-msi.c
@@ -154,6 +154,8 @@ static void __init its_pmsi_of_init(void)
 
 	for (np = of_find_matching_node(NULL, its_device_id); np;
 	     np = of_find_matching_node(np, its_device_id)) {
+		if (!of_device_is_available(np))
+			continue;
 		if (!of_property_read_bool(np, "msi-controller"))
 			continue;
 
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 06f025fd5726..2cbb19cddbf8 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -1412,7 +1412,7 @@ static struct irq_chip its_irq_chip = {
  * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations.
  */
 #define IRQS_PER_CHUNK_SHIFT	5
-#define IRQS_PER_CHUNK		(1 << IRQS_PER_CHUNK_SHIFT)
+#define IRQS_PER_CHUNK		(1UL << IRQS_PER_CHUNK_SHIFT)
 #define ITS_MAX_LPI_NRBITS	16 /* 64K LPIs */
 
 static unsigned long *lpi_bitmap;
@@ -2119,11 +2119,10 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	/*
-	 * At least one bit of EventID is being used, hence a minimum
-	 * of two entries. No, the architecture doesn't let you
-	 * express an ITT with a single entry.
+	 * We allocate at least one chunk worth of LPIs bet device,
+	 * and thus that many ITEs. The device may require less though.
 	 */
-	nr_ites = max(2UL, roundup_pow_of_two(nvecs));
+	nr_ites = max(IRQS_PER_CHUNK, roundup_pow_of_two(nvecs));
 	sz = nr_ites * its->ite_size;
 	sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
 	itt = kzalloc(sz, GFP_KERNEL);
@@ -2495,7 +2494,7 @@ static int its_vpe_set_affinity(struct irq_data *d,
 
 static void its_vpe_schedule(struct its_vpe *vpe)
 {
-	void * __iomem vlpi_base = gic_data_rdist_vlpi_base();
+	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
 	u64 val;
 
 	/* Schedule the VPE */
@@ -2527,7 +2526,7 @@ static void its_vpe_schedule(struct its_vpe *vpe)
 
 static void its_vpe_deschedule(struct its_vpe *vpe)
 {
-	void * __iomem vlpi_base = gic_data_rdist_vlpi_base();
+	void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
 	u32 count = 1000000;	/* 1s! */
 	bool clean;
 	u64 val;
@@ -3314,6 +3313,8 @@ static int __init its_of_probe(struct device_node *node)
 
 	for (np = of_find_matching_node(node, its_device_id); np;
 	     np = of_find_matching_node(np, its_device_id)) {
+		if (!of_device_is_available(np))
+			continue;
 		if (!of_property_read_bool(np, "msi-controller")) {
 			pr_warn("%pOF: no msi-controller property, ITS ignored\n",
 				np);
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index b56c3e23f0af..d99cc07903ec 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -673,7 +673,7 @@ static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
 	       MPIDR_TO_SGI_RS(cluster_id)		|
 	       tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
 
-	pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
+	pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
 	gic_write_sgi1r(val);
 }
 
@@ -688,7 +688,7 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 	 * Ensure that stores to Normal memory are visible to the
 	 * other CPUs before issuing the IPI.
 	 */
-	smp_wmb();
+	wmb();
 
 	for_each_cpu(cpu, mask) {
 		u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu));
@@ -1070,31 +1070,6 @@ static int __init gic_validate_dist_version(void __iomem *dist_base)
 	return 0;
 }
 
-static int get_cpu_number(struct device_node *dn)
-{
-	const __be32 *cell;
-	u64 hwid;
-	int cpu;
-
-	cell = of_get_property(dn, "reg", NULL);
-	if (!cell)
-		return -1;
-
-	hwid = of_read_number(cell, of_n_addr_cells(dn));
-
-	/*
-	 * Non affinity bits must be set to 0 in the DT
-	 */
-	if (hwid & ~MPIDR_HWID_BITMASK)
-		return -1;
-
-	for_each_possible_cpu(cpu)
-		if (cpu_logical_map(cpu) == hwid)
-			return cpu;
-
-	return -1;
-}
-
 /* Create all possible partitions at boot time */
 static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
 {
@@ -1145,8 +1120,8 @@ static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
 			if (WARN_ON(!cpu_node))
 				continue;
 
-			cpu = get_cpu_number(cpu_node);
-			if (WARN_ON(cpu == -1))
+			cpu = of_cpu_node_to_id(cpu_node);
+			if (WARN_ON(cpu < 0))
 				continue;
 
 			pr_cont("%pOF[%d] ", cpu_node, cpu);
@@ -1331,6 +1306,10 @@ gic_acpi_parse_madt_gicc(struct acpi_subtable_header *header,
 	u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
 	void __iomem *redist_base;
 
+	/* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
+	if (!(gicc->flags & ACPI_MADT_ENABLED))
+		return 0;
+
 	redist_base = ioremap(gicc->gicr_base_address, size);
 	if (!redist_base)
 		return -ENOMEM;
@@ -1380,6 +1359,13 @@ static int __init gic_acpi_match_gicc(struct acpi_subtable_header *header,
 	if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address)
 		return 0;
 
+	/*
+	 * It's perfectly valid firmware can pass disabled GICC entry, driver
+	 * should not treat as errors, skip the entry instead of probe fail.
+	 */
+	if (!(gicc->flags & ACPI_MADT_ENABLED))
+		return 0;
+
 	return -ENODEV;
 }
 
diff --git a/drivers/irqchip/irq-goldfish-pic.c b/drivers/irqchip/irq-goldfish-pic.c
new file mode 100644
index 000000000000..2a92f03c73e4
--- /dev/null
+++ b/drivers/irqchip/irq-goldfish-pic.c
@@ -0,0 +1,139 @@
+/*
+ * Driver for MIPS Goldfish Programmable Interrupt Controller.
+ *
+ * Author: Miodrag Dinic <miodrag.dinic@mips.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+
+#define GFPIC_NR_IRQS			32
+
+/* 8..39 Cascaded Goldfish PIC interrupts */
+#define GFPIC_IRQ_BASE			8
+
+#define GFPIC_REG_IRQ_PENDING		0x04
+#define GFPIC_REG_IRQ_DISABLE_ALL	0x08
+#define GFPIC_REG_IRQ_DISABLE		0x0c
+#define GFPIC_REG_IRQ_ENABLE		0x10
+
+struct goldfish_pic_data {
+	void __iomem *base;
+	struct irq_domain *irq_domain;
+};
+
+static void goldfish_pic_cascade(struct irq_desc *desc)
+{
+	struct goldfish_pic_data *gfpic = irq_desc_get_handler_data(desc);
+	struct irq_chip *host_chip = irq_desc_get_chip(desc);
+	u32 pending, hwirq, virq;
+
+	chained_irq_enter(host_chip, desc);
+
+	pending = readl(gfpic->base + GFPIC_REG_IRQ_PENDING);
+	while (pending) {
+		hwirq = __fls(pending);
+		virq = irq_linear_revmap(gfpic->irq_domain, hwirq);
+		generic_handle_irq(virq);
+		pending &= ~(1 << hwirq);
+	}
+
+	chained_irq_exit(host_chip, desc);
+}
+
+static const struct irq_domain_ops goldfish_irq_domain_ops = {
+	.xlate = irq_domain_xlate_onecell,
+};
+
+static int __init goldfish_pic_of_init(struct device_node *of_node,
+				       struct device_node *parent)
+{
+	struct goldfish_pic_data *gfpic;
+	struct irq_chip_generic *gc;
+	struct irq_chip_type *ct;
+	unsigned int parent_irq;
+	int ret = 0;
+
+	gfpic = kzalloc(sizeof(*gfpic), GFP_KERNEL);
+	if (!gfpic) {
+		ret = -ENOMEM;
+		goto out_err;
+	}
+
+	parent_irq = irq_of_parse_and_map(of_node, 0);
+	if (!parent_irq) {
+		pr_err("Failed to map parent IRQ!\n");
+		ret = -EINVAL;
+		goto out_free;
+	}
+
+	gfpic->base = of_iomap(of_node, 0);
+	if (!gfpic->base) {
+		pr_err("Failed to map base address!\n");
+		ret = -ENOMEM;
+		goto out_unmap_irq;
+	}
+
+	/* Mask interrupts. */
+	writel(1, gfpic->base + GFPIC_REG_IRQ_DISABLE_ALL);
+
+	gc = irq_alloc_generic_chip("GFPIC", 1, GFPIC_IRQ_BASE, gfpic->base,
+				    handle_level_irq);
+	if (!gc) {
+		pr_err("Failed to allocate chip structures!\n");
+		ret = -ENOMEM;
+		goto out_iounmap;
+	}
+
+	ct = gc->chip_types;
+	ct->regs.enable = GFPIC_REG_IRQ_ENABLE;
+	ct->regs.disable = GFPIC_REG_IRQ_DISABLE;
+	ct->chip.irq_unmask = irq_gc_unmask_enable_reg;
+	ct->chip.irq_mask = irq_gc_mask_disable_reg;
+
+	irq_setup_generic_chip(gc, IRQ_MSK(GFPIC_NR_IRQS), 0,
+			       IRQ_NOPROBE | IRQ_LEVEL, 0);
+
+	gfpic->irq_domain = irq_domain_add_legacy(of_node, GFPIC_NR_IRQS,
+						  GFPIC_IRQ_BASE, 0,
+						  &goldfish_irq_domain_ops,
+						  NULL);
+	if (!gfpic->irq_domain) {
+		pr_err("Failed to add irqdomain!\n");
+		ret = -ENOMEM;
+		goto out_destroy_generic_chip;
+	}
+
+	irq_set_chained_handler_and_data(parent_irq,
+					 goldfish_pic_cascade, gfpic);
+
+	pr_info("Successfully registered.\n");
+	return 0;
+
+out_destroy_generic_chip:
+	irq_destroy_generic_chip(gc, IRQ_MSK(GFPIC_NR_IRQS),
+				 IRQ_NOPROBE | IRQ_LEVEL, 0);
+out_iounmap:
+	iounmap(gfpic->base);
+out_unmap_irq:
+	irq_dispose_mapping(parent_irq);
+out_free:
+	kfree(gfpic);
+out_err:
+	pr_err("Failed to initialize! (errno = %d)\n", ret);
+	return ret;
+}
+
+IRQCHIP_DECLARE(google_gf_pic, "google,goldfish-pic", goldfish_pic_of_init);
diff --git a/drivers/irqchip/irq-i8259.c b/drivers/irqchip/irq-i8259.c
index 7aafbb091b67..b0d4aab1a58c 100644
--- a/drivers/irqchip/irq-i8259.c
+++ b/drivers/irqchip/irq-i8259.c
@@ -289,14 +289,14 @@ static struct resource pic1_io_resource = {
 	.name = "pic1",
 	.start = PIC_MASTER_CMD,
 	.end = PIC_MASTER_IMR,
-	.flags = IORESOURCE_BUSY
+	.flags = IORESOURCE_IO | IORESOURCE_BUSY
 };
 
 static struct resource pic2_io_resource = {
 	.name = "pic2",
 	.start = PIC_SLAVE_CMD,
 	.end = PIC_SLAVE_IMR,
-	.flags = IORESOURCE_BUSY
+	.flags = IORESOURCE_IO | IORESOURCE_BUSY
 };
 
 static int i8259A_irq_domain_map(struct irq_domain *d, unsigned int virq,
diff --git a/drivers/irqchip/irq-imx-gpcv2.c b/drivers/irqchip/irq-imx-gpcv2.c
index 675eda5ff2b8..4760307ab43f 100644
--- a/drivers/irqchip/irq-imx-gpcv2.c
+++ b/drivers/irqchip/irq-imx-gpcv2.c
@@ -28,20 +28,6 @@ struct gpcv2_irqchip_data {
 
 static struct gpcv2_irqchip_data *imx_gpcv2_instance;
 
-/*
- * Interface for the low level wakeup code.
- */
-u32 imx_gpcv2_get_wakeup_source(u32 **sources)
-{
-	if (!imx_gpcv2_instance)
-		return 0;
-
-	if (sources)
-		*sources = imx_gpcv2_instance->wakeup_sources;
-
-	return IMR_NUM;
-}
-
 static int gpcv2_wakeup_source_save(void)
 {
 	struct gpcv2_irqchip_data *cd;
diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index ef92a4d2038e..d32268cc1174 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -424,8 +424,6 @@ static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq,
 	spin_lock_irqsave(&gic_lock, flags);
 	write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin);
 	write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu)));
-	gic_clear_pcpu_masks(intr);
-	set_bit(intr, per_cpu_ptr(pcpu_masks, cpu));
 	irq_data_update_effective_affinity(data, cpumask_of(cpu));
 	spin_unlock_irqrestore(&gic_lock, flags);
 
diff --git a/drivers/irqchip/irq-ompic.c b/drivers/irqchip/irq-ompic.c
index cf6d0c455518..e66ef4373b1e 100644
--- a/drivers/irqchip/irq-ompic.c
+++ b/drivers/irqchip/irq-ompic.c
@@ -171,9 +171,9 @@ static int __init ompic_of_init(struct device_node *node,
 
 	/* Setup the device */
 	ompic_base = ioremap(res.start, resource_size(&res));
-	if (IS_ERR(ompic_base)) {
+	if (!ompic_base) {
 		pr_err("ompic: unable to map registers");
-		return PTR_ERR(ompic_base);
+		return -ENOMEM;
 	}
 
 	irq = irq_of_parse_and_map(node, 0);
diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c
index 31ab0dee2ce7..36f0fbe36c35 100644
--- a/drivers/irqchip/irq-stm32-exti.c
+++ b/drivers/irqchip/irq-stm32-exti.c
@@ -1,7 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) Maxime Coquelin 2015
+ * Copyright (C) STMicroelectronics 2017
  * Author:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/bitops.h>
diff --git a/drivers/isdn/capi/capi.c b/drivers/isdn/capi/capi.c
index dde8f46bc254..19cd93783c87 100644
--- a/drivers/isdn/capi/capi.c
+++ b/drivers/isdn/capi/capi.c
@@ -724,19 +724,19 @@ capi_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos
 	return count;
 }
 
-static unsigned int
+static __poll_t
 capi_poll(struct file *file, poll_table *wait)
 {
 	struct capidev *cdev = file->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	if (!cdev->ap.applid)
-		return POLLERR;
+		return EPOLLERR;
 
 	poll_wait(file, &(cdev->recvwait), wait);
-	mask = POLLOUT | POLLWRNORM;
+	mask = EPOLLOUT | EPOLLWRNORM;
 	if (!skb_queue_empty(&cdev->recvqueue))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	return mask;
 }
 
diff --git a/drivers/isdn/divert/divert_procfs.c b/drivers/isdn/divert/divert_procfs.c
index 1c5dc345e7c5..342585e04fd3 100644
--- a/drivers/isdn/divert/divert_procfs.c
+++ b/drivers/isdn/divert/divert_procfs.c
@@ -119,15 +119,15 @@ isdn_divert_write(struct file *file, const char __user *buf, size_t count, loff_
 /***************************************/
 /* select routines for various kernels */
 /***************************************/
-static unsigned int
+static __poll_t
 isdn_divert_poll(struct file *file, poll_table *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &(rd_queue), wait);
-	/* mask = POLLOUT | POLLWRNORM; */
+	/* mask = EPOLLOUT | EPOLLWRNORM; */
 	if (*((struct divert_info **) file->private_data)) {
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	}
 	return mask;
 }				/* isdn_divert_poll */
diff --git a/drivers/isdn/hardware/eicon/divamnt.c b/drivers/isdn/hardware/eicon/divamnt.c
index 72e58bf07577..5a95587b3117 100644
--- a/drivers/isdn/hardware/eicon/divamnt.c
+++ b/drivers/isdn/hardware/eicon/divamnt.c
@@ -98,14 +98,14 @@ void diva_os_get_time(dword *sec, dword *usec)
 /*
  * device node operations
  */
-static unsigned int maint_poll(struct file *file, poll_table *wait)
+static __poll_t maint_poll(struct file *file, poll_table *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &msgwaitq, wait);
-	mask = POLLOUT | POLLWRNORM;
+	mask = EPOLLOUT | EPOLLWRNORM;
 	if (file->private_data || diva_dbg_q_length()) {
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	}
 	return (mask);
 }
diff --git a/drivers/isdn/hardware/eicon/divasi.c b/drivers/isdn/hardware/eicon/divasi.c
index 0033d74a7291..525518c945fe 100644
--- a/drivers/isdn/hardware/eicon/divasi.c
+++ b/drivers/isdn/hardware/eicon/divasi.c
@@ -74,7 +74,7 @@ static ssize_t um_idi_read(struct file *file, char __user *buf, size_t count,
 			   loff_t *offset);
 static ssize_t um_idi_write(struct file *file, const char __user *buf,
 			    size_t count, loff_t *offset);
-static unsigned int um_idi_poll(struct file *file, poll_table *wait);
+static __poll_t um_idi_poll(struct file *file, poll_table *wait);
 static int um_idi_open(struct inode *inode, struct file *file);
 static int um_idi_release(struct inode *inode, struct file *file);
 static int remove_entity(void *entity);
@@ -365,36 +365,36 @@ um_idi_write(struct file *file, const char __user *buf, size_t count,
 	return (ret);
 }
 
-static unsigned int um_idi_poll(struct file *file, poll_table *wait)
+static __poll_t um_idi_poll(struct file *file, poll_table *wait)
 {
 	diva_um_idi_os_context_t *p_os;
 
 	if (!file->private_data) {
-		return (POLLERR);
+		return (EPOLLERR);
 	}
 
 	if ((!(p_os =
 	       (diva_um_idi_os_context_t *)
 	       diva_um_id_get_os_context(file->private_data)))
 	    || p_os->aborted) {
-		return (POLLERR);
+		return (EPOLLERR);
 	}
 
 	poll_wait(file, &p_os->read_wait, wait);
 
 	if (p_os->aborted) {
-		return (POLLERR);
+		return (EPOLLERR);
 	}
 
 	switch (diva_user_mode_idi_ind_ready(file->private_data, file)) {
 	case (-1):
-		return (POLLERR);
+		return (EPOLLERR);
 
 	case 0:
 		return (0);
 	}
 
-	return (POLLIN | POLLRDNORM);
+	return (EPOLLIN | EPOLLRDNORM);
 }
 
 static int um_idi_open(struct inode *inode, struct file *file)
diff --git a/drivers/isdn/hardware/eicon/divasmain.c b/drivers/isdn/hardware/eicon/divasmain.c
index b2023e08dcd2..b9980e84f9db 100644
--- a/drivers/isdn/hardware/eicon/divasmain.c
+++ b/drivers/isdn/hardware/eicon/divasmain.c
@@ -650,12 +650,12 @@ static ssize_t divas_read(struct file *file, char __user *buf,
 	return (ret);
 }
 
-static unsigned int divas_poll(struct file *file, poll_table *wait)
+static __poll_t divas_poll(struct file *file, poll_table *wait)
 {
 	if (!file->private_data) {
-		return (POLLERR);
+		return (EPOLLERR);
 	}
-	return (POLLIN | POLLRDNORM);
+	return (EPOLLIN | EPOLLRDNORM);
 }
 
 static const struct file_operations divas_fops = {
diff --git a/drivers/isdn/hardware/eicon/divasproc.c b/drivers/isdn/hardware/eicon/divasproc.c
index b57efd6ad916..f52f4622b10b 100644
--- a/drivers/isdn/hardware/eicon/divasproc.c
+++ b/drivers/isdn/hardware/eicon/divasproc.c
@@ -99,9 +99,9 @@ divas_write(struct file *file, const char __user *buf, size_t count, loff_t *off
 	return (-ENODEV);
 }
 
-static unsigned int divas_poll(struct file *file, poll_table *wait)
+static __poll_t divas_poll(struct file *file, poll_table *wait)
 {
-	return (POLLERR);
+	return (EPOLLERR);
 }
 
 static int divas_open(struct inode *inode, struct file *file)
diff --git a/drivers/isdn/hysdn/hysdn_proclog.c b/drivers/isdn/hysdn/hysdn_proclog.c
index aaca0b3d662e..6e898b90e86e 100644
--- a/drivers/isdn/hysdn/hysdn_proclog.c
+++ b/drivers/isdn/hysdn/hysdn_proclog.c
@@ -281,10 +281,10 @@ hysdn_log_close(struct inode *ino, struct file *filep)
 /*************************************************/
 /* select/poll routine to be able using select() */
 /*************************************************/
-static unsigned int
+static __poll_t
 hysdn_log_poll(struct file *file, poll_table *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	hysdn_card *card = PDE_DATA(file_inode(file));
 	struct procdata *pd = card->proclog;
 
@@ -294,7 +294,7 @@ hysdn_log_poll(struct file *file, poll_table *wait)
 	poll_wait(file, &(pd->rd_queue), wait);
 
 	if (*((struct log_data **) file->private_data))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }				/* hysdn_log_poll */
diff --git a/drivers/isdn/i4l/isdn_common.c b/drivers/isdn/i4l/isdn_common.c
index 8b03d618185e..7c6f3f5d9d9a 100644
--- a/drivers/isdn/i4l/isdn_common.c
+++ b/drivers/isdn/i4l/isdn_common.c
@@ -1227,32 +1227,32 @@ isdn_write(struct file *file, const char __user *buf, size_t count, loff_t *off)
 	return retval;
 }
 
-static unsigned int
+static __poll_t
 isdn_poll(struct file *file, poll_table *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	unsigned int minor = iminor(file_inode(file));
 	int drvidx = isdn_minor2drv(minor - ISDN_MINOR_CTRL);
 
 	mutex_lock(&isdn_mutex);
 	if (minor == ISDN_MINOR_STATUS) {
 		poll_wait(file, &(dev->info_waitq), wait);
-		/* mask = POLLOUT | POLLWRNORM; */
+		/* mask = EPOLLOUT | EPOLLWRNORM; */
 		if (file->private_data) {
-			mask |= POLLIN | POLLRDNORM;
+			mask |= EPOLLIN | EPOLLRDNORM;
 		}
 		goto out;
 	}
 	if (minor >= ISDN_MINOR_CTRL && minor <= ISDN_MINOR_CTRLMAX) {
 		if (drvidx < 0) {
 			/* driver deregistered while file open */
-			mask = POLLHUP;
+			mask = EPOLLHUP;
 			goto out;
 		}
 		poll_wait(file, &(dev->drv[drvidx]->st_waitq), wait);
-		mask = POLLOUT | POLLWRNORM;
+		mask = EPOLLOUT | EPOLLWRNORM;
 		if (dev->drv[drvidx]->stavail) {
-			mask |= POLLIN | POLLRDNORM;
+			mask |= EPOLLIN | EPOLLRDNORM;
 		}
 		goto out;
 	}
@@ -1262,7 +1262,7 @@ isdn_poll(struct file *file, poll_table *wait)
 		goto out;
 	}
 #endif
-	mask = POLLERR;
+	mask = EPOLLERR;
 out:
 	mutex_unlock(&isdn_mutex);
 	return mask;
diff --git a/drivers/isdn/i4l/isdn_ppp.c b/drivers/isdn/i4l/isdn_ppp.c
index e07aefb9151d..a7b275ea5de1 100644
--- a/drivers/isdn/i4l/isdn_ppp.c
+++ b/drivers/isdn/i4l/isdn_ppp.c
@@ -685,10 +685,10 @@ isdn_ppp_ioctl(int min, struct file *file, unsigned int cmd, unsigned long arg)
 	return 0;
 }
 
-unsigned int
+__poll_t
 isdn_ppp_poll(struct file *file, poll_table *wait)
 {
-	u_int mask;
+	__poll_t mask;
 	struct ippp_buf_queue *bf, *bl;
 	u_long flags;
 	struct ippp_struct *is;
@@ -704,12 +704,12 @@ isdn_ppp_poll(struct file *file, poll_table *wait)
 
 	if (!(is->state & IPPP_OPEN)) {
 		if (is->state == IPPP_CLOSEWAIT)
-			return POLLHUP;
+			return EPOLLHUP;
 		printk(KERN_DEBUG "isdn_ppp: device not open\n");
-		return POLLERR;
+		return EPOLLERR;
 	}
 	/* we're always ready to send .. */
-	mask = POLLOUT | POLLWRNORM;
+	mask = EPOLLOUT | EPOLLWRNORM;
 
 	spin_lock_irqsave(&is->buflock, flags);
 	bl = is->last;
@@ -719,7 +719,7 @@ isdn_ppp_poll(struct file *file, poll_table *wait)
 	 */
 	if (bf->next != bl || (is->state & IPPP_NOBLOCK)) {
 		is->state &= ~IPPP_NOBLOCK;
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	}
 	spin_unlock_irqrestore(&is->buflock, flags);
 	return mask;
diff --git a/drivers/isdn/i4l/isdn_ppp.h b/drivers/isdn/i4l/isdn_ppp.h
index 4e9b8935a4eb..34b8a2ce84f3 100644
--- a/drivers/isdn/i4l/isdn_ppp.h
+++ b/drivers/isdn/i4l/isdn_ppp.h
@@ -23,7 +23,7 @@ extern int isdn_ppp_autodial_filter(struct sk_buff *, isdn_net_local *);
 extern int isdn_ppp_xmit(struct sk_buff *, struct net_device *);
 extern void isdn_ppp_receive(isdn_net_dev *, isdn_net_local *, struct sk_buff *);
 extern int isdn_ppp_dev_ioctl(struct net_device *, struct ifreq *, int);
-extern unsigned int isdn_ppp_poll(struct file *, struct poll_table_struct *);
+extern __poll_t isdn_ppp_poll(struct file *, struct poll_table_struct *);
 extern int isdn_ppp_ioctl(int, struct file *, unsigned int, unsigned long);
 extern void isdn_ppp_release(int, struct file *);
 extern int isdn_ppp_dial_slave(char *);
diff --git a/drivers/isdn/mISDN/l1oip_core.c b/drivers/isdn/mISDN/l1oip_core.c
index e3654782a3e2..21d50e4cc5e1 100644
--- a/drivers/isdn/mISDN/l1oip_core.c
+++ b/drivers/isdn/mISDN/l1oip_core.c
@@ -645,8 +645,10 @@ l1oip_socket_thread(void *data)
 {
 	struct l1oip *hc = (struct l1oip *)data;
 	int ret = 0;
-	struct msghdr msg;
 	struct sockaddr_in sin_rx;
+	struct kvec iov;
+	struct msghdr msg = {.msg_name = &sin_rx,
+			     .msg_namelen = sizeof(sin_rx)};
 	unsigned char *recvbuf;
 	size_t recvbuf_size = 1500;
 	int recvlen;
@@ -661,6 +663,9 @@ l1oip_socket_thread(void *data)
 		goto fail;
 	}
 
+	iov.iov_base = recvbuf;
+	iov.iov_len = recvbuf_size;
+
 	/* make daemon */
 	allow_signal(SIGTERM);
 
@@ -697,12 +702,6 @@ l1oip_socket_thread(void *data)
 		goto fail;
 	}
 
-	/* build receive message */
-	msg.msg_name = &sin_rx;
-	msg.msg_namelen = sizeof(sin_rx);
-	msg.msg_control = NULL;
-	msg.msg_controllen = 0;
-
 	/* build send message */
 	hc->sendmsg.msg_name = &hc->sin_remote;
 	hc->sendmsg.msg_namelen = sizeof(hc->sin_remote);
@@ -719,12 +718,9 @@ l1oip_socket_thread(void *data)
 		printk(KERN_DEBUG "%s: socket created and open\n",
 		       __func__);
 	while (!signal_pending(current)) {
-		struct kvec iov = {
-			.iov_base = recvbuf,
-			.iov_len = recvbuf_size,
-		};
-		recvlen = kernel_recvmsg(socket, &msg, &iov, 1,
-					 recvbuf_size, 0);
+		iov_iter_kvec(&msg.msg_iter, READ | ITER_KVEC, &iov, 1,
+				recvbuf_size);
+		recvlen = sock_recvmsg(socket, &msg, 0);
 		if (recvlen > 0) {
 			l1oip_socket_parse(hc, &sin_rx, recvbuf, recvlen);
 		} else {
diff --git a/drivers/isdn/mISDN/timerdev.c b/drivers/isdn/mISDN/timerdev.c
index c50a34340f67..211ed6cffd10 100644
--- a/drivers/isdn/mISDN/timerdev.c
+++ b/drivers/isdn/mISDN/timerdev.c
@@ -141,11 +141,11 @@ mISDN_read(struct file *filep, char __user *buf, size_t count, loff_t *off)
 	return ret;
 }
 
-static unsigned int
+static __poll_t
 mISDN_poll(struct file *filep, poll_table *wait)
 {
 	struct mISDNtimerdev	*dev = filep->private_data;
-	unsigned int		mask = POLLERR;
+	__poll_t		mask = EPOLLERR;
 
 	if (*debug & DEBUG_TIMER)
 		printk(KERN_DEBUG "%s(%p, %p)\n", __func__, filep, wait);
@@ -153,7 +153,7 @@ mISDN_poll(struct file *filep, poll_table *wait)
 		poll_wait(filep, &dev->wait, wait);
 		mask = 0;
 		if (dev->work || !list_empty(&dev->expired))
-			mask |= (POLLIN | POLLRDNORM);
+			mask |= (EPOLLIN | EPOLLRDNORM);
 		if (*debug & DEBUG_TIMER)
 			printk(KERN_DEBUG "%s work(%d) empty(%d)\n", __func__,
 			       dev->work, list_empty(&dev->expired));
diff --git a/drivers/leds/Kconfig b/drivers/leds/Kconfig
index 318a28fd58fe..3e763d2a0cb3 100644
--- a/drivers/leds/Kconfig
+++ b/drivers/leds/Kconfig
@@ -137,6 +137,13 @@ config LEDS_LM3642
 	  converter plus 1.5A constant current driver for a high-current
 	  white LED.
 
+config LEDS_LM3692X
+	tristate "LED support for LM3692x Chips"
+	depends on LEDS_CLASS && I2C && OF
+	select REGMAP_I2C
+	help
+	  This option enables support for the TI LM3692x family
+	  of white LED string drivers used for backlighting.
 
 config LEDS_LOCOMO
 	tristate "LED Support for Locomo device"
@@ -347,7 +354,7 @@ config LEDS_LP8788
 
 config LEDS_LP8860
 	tristate "LED support for the TI LP8860 4 channel LED driver"
-	depends on LEDS_CLASS && I2C
+	depends on LEDS_CLASS && I2C && OF
 	select REGMAP_I2C
 	help
 	  If you say yes here you get support for the TI LP8860 4 channel
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index a2a6b5a4f86d..987884a5b9a5 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -74,6 +74,7 @@ obj-$(CONFIG_LEDS_PM8058)		+= leds-pm8058.o
 obj-$(CONFIG_LEDS_MLXCPLD)		+= leds-mlxcpld.o
 obj-$(CONFIG_LEDS_NIC78BX)		+= leds-nic78bx.o
 obj-$(CONFIG_LEDS_MT6323)		+= leds-mt6323.o
+obj-$(CONFIG_LEDS_LM3692X)		+= leds-lm3692x.o
 
 # LED SPI Drivers
 obj-$(CONFIG_LEDS_DAC124S085)		+= leds-dac124s085.o
diff --git a/drivers/leds/leds-as3645a.c b/drivers/leds/leds-as3645a.c
index 9a257f969300..f883616d9e60 100644
--- a/drivers/leds/leds-as3645a.c
+++ b/drivers/leds/leds-as3645a.c
@@ -360,7 +360,8 @@ static int as3645a_set_flash_brightness(struct led_classdev_flash *fled,
 {
 	struct as3645a *flash = fled_to_as3645a(fled);
 
-	flash->flash_current = as3645a_current_to_reg(flash, true, brightness_ua);
+	flash->flash_current = as3645a_current_to_reg(flash, true,
+						      brightness_ua);
 
 	return as3645a_set_current(flash);
 }
@@ -455,8 +456,8 @@ static int as3645a_detect(struct as3645a *flash)
 
 	/* Verify the chip model and version. */
 	if (model != 0x01 || rfu != 0x00) {
-		dev_err(dev, "AS3645A not detected "
-			"(model %d rfu %d)\n", model, rfu);
+		dev_err(dev, "AS3645A not detected (model %d rfu %d)\n",
+			model, rfu);
 		return -ENODEV;
 	}
 
diff --git a/drivers/leds/leds-blinkm.c b/drivers/leds/leds-blinkm.c
index d03ed6b4176b..851c1920b63c 100644
--- a/drivers/leds/leds-blinkm.c
+++ b/drivers/leds/leds-blinkm.c
@@ -549,8 +549,12 @@ static int blinkm_detect(struct i2c_client *client, struct i2c_board_info *info)
 	/* make sure the blinkM is balanced (read/writes) */
 	while (count > 0) {
 		ret = blinkm_write(client, BLM_GET_ADDR, NULL);
+		if (ret)
+			return ret;
 		usleep_range(5000, 10000);
 		ret = blinkm_read(client, BLM_GET_ADDR, tmpargs);
+		if (ret)
+			return ret;
 		usleep_range(5000, 10000);
 		if (tmpargs[0] == 0x09)
 			count = 0;
diff --git a/drivers/leds/leds-lm3692x.c b/drivers/leds/leds-lm3692x.c
new file mode 100644
index 000000000000..437173d1712c
--- /dev/null
+++ b/drivers/leds/leds-lm3692x.c
@@ -0,0 +1,393 @@
+/*
+ * TI lm3692x LED Driver
+ *
+ * Copyright (C) 2017 Texas Instruments
+ *
+ * Author: Dan Murphy <dmurphy@ti.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * Data sheet is located
+ * http://www.ti.com/lit/ds/snvsa29/snvsa29.pdf
+ */
+
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/leds.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/slab.h>
+#include <uapi/linux/uleds.h>
+
+#define LM3692X_REV		0x0
+#define LM3692X_RESET		0x1
+#define LM3692X_EN		0x10
+#define LM3692X_BRT_CTRL	0x11
+#define LM3692X_PWM_CTRL	0x12
+#define LM3692X_BOOST_CTRL	0x13
+#define LM3692X_AUTO_FREQ_HI	0x15
+#define LM3692X_AUTO_FREQ_LO	0x16
+#define LM3692X_BL_ADJ_THRESH	0x17
+#define LM3692X_BRT_LSB		0x18
+#define LM3692X_BRT_MSB		0x19
+#define LM3692X_FAULT_CTRL	0x1e
+#define LM3692X_FAULT_FLAGS	0x1f
+
+#define LM3692X_SW_RESET	BIT(0)
+#define LM3692X_DEVICE_EN	BIT(0)
+#define LM3692X_LED1_EN		BIT(1)
+#define LM3692X_LED2_EN		BIT(2)
+
+/* Brightness Control Bits */
+#define LM3692X_BL_ADJ_POL	BIT(0)
+#define LM3692X_RAMP_RATE_125us	0x00
+#define LM3692X_RAMP_RATE_250us	BIT(1)
+#define LM3692X_RAMP_RATE_500us BIT(2)
+#define LM3692X_RAMP_RATE_1ms	(BIT(1) | BIT(2))
+#define LM3692X_RAMP_RATE_2ms	BIT(3)
+#define LM3692X_RAMP_RATE_4ms	(BIT(3) | BIT(1))
+#define LM3692X_RAMP_RATE_8ms	(BIT(2) | BIT(3))
+#define LM3692X_RAMP_RATE_16ms	(BIT(1) | BIT(2) | BIT(3))
+#define LM3692X_RAMP_EN		BIT(4)
+#define LM3692X_BRHT_MODE_REG	0x00
+#define LM3692X_BRHT_MODE_PWM	BIT(5)
+#define LM3692X_BRHT_MODE_MULTI_RAMP BIT(6)
+#define LM3692X_BRHT_MODE_RAMP_MULTI (BIT(5) | BIT(6))
+#define LM3692X_MAP_MODE_EXP	BIT(7)
+
+/* PWM Register Bits */
+#define LM3692X_PWM_FILTER_100	BIT(0)
+#define LM3692X_PWM_FILTER_150	BIT(1)
+#define LM3692X_PWM_FILTER_200	(BIT(0) | BIT(1))
+#define LM3692X_PWM_HYSTER_1LSB BIT(2)
+#define LM3692X_PWM_HYSTER_2LSB	BIT(3)
+#define LM3692X_PWM_HYSTER_3LSB (BIT(3) | BIT(2))
+#define LM3692X_PWM_HYSTER_4LSB BIT(4)
+#define LM3692X_PWM_HYSTER_5LSB (BIT(4) | BIT(2))
+#define LM3692X_PWM_HYSTER_6LSB (BIT(4) | BIT(3))
+#define LM3692X_PWM_POLARITY	BIT(5)
+#define LM3692X_PWM_SAMP_4MHZ	BIT(6)
+#define LM3692X_PWM_SAMP_24MHZ	BIT(7)
+
+/* Boost Control Bits */
+#define LM3692X_OCP_PROT_1A	BIT(0)
+#define LM3692X_OCP_PROT_1_25A	BIT(1)
+#define LM3692X_OCP_PROT_1_5A	(BIT(0) | BIT(1))
+#define LM3692X_OVP_21V		BIT(2)
+#define LM3692X_OVP_25V		BIT(3)
+#define LM3692X_OVP_29V		(BIT(2) | BIT(3))
+#define LM3692X_MIN_IND_22UH	BIT(4)
+#define LM3692X_BOOST_SW_1MHZ	BIT(5)
+#define LM3692X_BOOST_SW_NO_SHIFT	BIT(6)
+
+/* Fault Control Bits */
+#define LM3692X_FAULT_CTRL_OVP BIT(0)
+#define LM3692X_FAULT_CTRL_OCP BIT(1)
+#define LM3692X_FAULT_CTRL_TSD BIT(2)
+#define LM3692X_FAULT_CTRL_OPEN BIT(3)
+
+/* Fault Flag Bits */
+#define LM3692X_FAULT_FLAG_OVP BIT(0)
+#define LM3692X_FAULT_FLAG_OCP BIT(1)
+#define LM3692X_FAULT_FLAG_TSD BIT(2)
+#define LM3692X_FAULT_FLAG_SHRT BIT(3)
+#define LM3692X_FAULT_FLAG_OPEN BIT(4)
+
+/**
+ * struct lm3692x_led -
+ * @lock - Lock for reading/writing the device
+ * @client - Pointer to the I2C client
+ * @led_dev - LED class device pointer
+ * @regmap - Devices register map
+ * @enable_gpio - VDDIO/EN gpio to enable communication interface
+ * @regulator - LED supply regulator pointer
+ * @label - LED label
+ */
+struct lm3692x_led {
+	struct mutex lock;
+	struct i2c_client *client;
+	struct led_classdev led_dev;
+	struct regmap *regmap;
+	struct gpio_desc *enable_gpio;
+	struct regulator *regulator;
+	char label[LED_MAX_NAME_SIZE];
+};
+
+static const struct reg_default lm3692x_reg_defs[] = {
+	{LM3692X_EN, 0xf},
+	{LM3692X_BRT_CTRL, 0x61},
+	{LM3692X_PWM_CTRL, 0x73},
+	{LM3692X_BOOST_CTRL, 0x6f},
+	{LM3692X_AUTO_FREQ_HI, 0x0},
+	{LM3692X_AUTO_FREQ_LO, 0x0},
+	{LM3692X_BL_ADJ_THRESH, 0x0},
+	{LM3692X_BRT_LSB, 0x7},
+	{LM3692X_BRT_MSB, 0xff},
+	{LM3692X_FAULT_CTRL, 0x7},
+};
+
+static const struct regmap_config lm3692x_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.max_register = LM3692X_FAULT_FLAGS,
+	.reg_defaults = lm3692x_reg_defs,
+	.num_reg_defaults = ARRAY_SIZE(lm3692x_reg_defs),
+	.cache_type = REGCACHE_RBTREE,
+};
+
+static int lm3692x_fault_check(struct lm3692x_led *led)
+{
+	int ret;
+	unsigned int read_buf;
+
+	ret = regmap_read(led->regmap, LM3692X_FAULT_FLAGS, &read_buf);
+	if (ret)
+		return ret;
+
+	if (read_buf)
+		dev_err(&led->client->dev, "Detected a fault 0x%X\n", read_buf);
+
+	/* The first read may clear the fault.  Check again to see if the fault
+	 * still exits and return that value.
+	 */
+	regmap_read(led->regmap, LM3692X_FAULT_FLAGS, &read_buf);
+	if (read_buf)
+		dev_err(&led->client->dev, "Second read of fault flags 0x%X\n",
+			read_buf);
+
+	return read_buf;
+}
+
+static int lm3692x_brightness_set(struct led_classdev *led_cdev,
+				enum led_brightness brt_val)
+{
+	struct lm3692x_led *led =
+			container_of(led_cdev, struct lm3692x_led, led_dev);
+	int ret;
+	int led_brightness_lsb = (brt_val >> 5);
+
+	mutex_lock(&led->lock);
+
+	ret = lm3692x_fault_check(led);
+	if (ret) {
+		dev_err(&led->client->dev, "Cannot read/clear faults\n");
+		goto out;
+	}
+
+	ret = regmap_write(led->regmap, LM3692X_BRT_MSB, brt_val);
+	if (ret) {
+		dev_err(&led->client->dev, "Cannot write MSB\n");
+		goto out;
+	}
+
+	ret = regmap_write(led->regmap, LM3692X_BRT_LSB, led_brightness_lsb);
+	if (ret) {
+		dev_err(&led->client->dev, "Cannot write LSB\n");
+		goto out;
+	}
+out:
+	mutex_unlock(&led->lock);
+	return ret;
+}
+
+static int lm3692x_init(struct lm3692x_led *led)
+{
+	int ret;
+
+	if (led->regulator) {
+		ret = regulator_enable(led->regulator);
+		if (ret) {
+			dev_err(&led->client->dev,
+				"Failed to enable regulator\n");
+			return ret;
+		}
+	}
+
+	if (led->enable_gpio)
+		gpiod_direction_output(led->enable_gpio, 1);
+
+	ret = lm3692x_fault_check(led);
+	if (ret) {
+		dev_err(&led->client->dev, "Cannot read/clear faults\n");
+		goto out;
+	}
+
+	ret = regmap_write(led->regmap, LM3692X_BRT_CTRL, 0x00);
+	if (ret)
+		goto out;
+
+	/*
+	 * For glitch free operation, the following data should
+	 * only be written while device enable bit is 0
+	 * per Section 7.5.14 of the data sheet
+	 */
+	ret = regmap_write(led->regmap, LM3692X_PWM_CTRL,
+		LM3692X_PWM_FILTER_100 | LM3692X_PWM_SAMP_24MHZ);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(led->regmap, LM3692X_BOOST_CTRL,
+			LM3692X_BRHT_MODE_RAMP_MULTI |
+			LM3692X_BL_ADJ_POL |
+			LM3692X_RAMP_RATE_250us);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(led->regmap, LM3692X_AUTO_FREQ_HI, 0x00);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(led->regmap, LM3692X_AUTO_FREQ_LO, 0x00);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(led->regmap, LM3692X_BL_ADJ_THRESH, 0x00);
+	if (ret)
+		goto out;
+
+	ret = regmap_write(led->regmap, LM3692X_BRT_CTRL,
+			LM3692X_BL_ADJ_POL | LM3692X_PWM_HYSTER_4LSB);
+	if (ret)
+		goto out;
+
+	return ret;
+out:
+	dev_err(&led->client->dev, "Fail writing initialization values\n");
+
+	if (led->enable_gpio)
+		gpiod_direction_output(led->enable_gpio, 0);
+
+	if (led->regulator) {
+		ret = regulator_disable(led->regulator);
+		if (ret)
+			dev_err(&led->client->dev,
+				"Failed to disable regulator\n");
+	}
+
+	return ret;
+}
+
+static int lm3692x_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	int ret;
+	struct lm3692x_led *led;
+	struct device_node *np = client->dev.of_node;
+	struct device_node *child_node;
+	const char *name;
+
+	led = devm_kzalloc(&client->dev, sizeof(*led), GFP_KERNEL);
+	if (!led)
+		return -ENOMEM;
+
+	for_each_available_child_of_node(np, child_node) {
+		led->led_dev.default_trigger = of_get_property(child_node,
+						    "linux,default-trigger",
+						    NULL);
+
+		ret = of_property_read_string(child_node, "label", &name);
+		if (!ret)
+			snprintf(led->label, sizeof(led->label),
+				 "%s:%s", id->name, name);
+		else
+			snprintf(led->label, sizeof(led->label),
+				 "%s::backlight_cluster", id->name);
+	};
+
+	led->enable_gpio = devm_gpiod_get_optional(&client->dev,
+						   "enable", GPIOD_OUT_LOW);
+	if (IS_ERR(led->enable_gpio)) {
+		ret = PTR_ERR(led->enable_gpio);
+		dev_err(&client->dev, "Failed to get enable gpio: %d\n", ret);
+		return ret;
+	}
+
+	led->regulator = devm_regulator_get(&client->dev, "vled");
+	if (IS_ERR(led->regulator))
+		led->regulator = NULL;
+
+	led->client = client;
+	led->led_dev.name = led->label;
+	led->led_dev.brightness_set_blocking = lm3692x_brightness_set;
+
+	mutex_init(&led->lock);
+
+	i2c_set_clientdata(client, led);
+
+	led->regmap = devm_regmap_init_i2c(client, &lm3692x_regmap_config);
+	if (IS_ERR(led->regmap)) {
+		ret = PTR_ERR(led->regmap);
+		dev_err(&client->dev, "Failed to allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = lm3692x_init(led);
+	if (ret)
+		return ret;
+
+	ret = devm_led_classdev_register(&client->dev, &led->led_dev);
+	if (ret) {
+		dev_err(&client->dev, "led register err: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int lm3692x_remove(struct i2c_client *client)
+{
+	struct lm3692x_led *led = i2c_get_clientdata(client);
+	int ret;
+
+	if (led->enable_gpio)
+		gpiod_direction_output(led->enable_gpio, 0);
+
+	if (led->regulator) {
+		ret = regulator_disable(led->regulator);
+		if (ret)
+			dev_err(&led->client->dev,
+				"Failed to disable regulator\n");
+	}
+
+	mutex_destroy(&led->lock);
+
+	return 0;
+}
+
+static const struct i2c_device_id lm3692x_id[] = {
+	{ "lm36922", 0 },
+	{ "lm36923", 1 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, lm3692x_id);
+
+static const struct of_device_id of_lm3692x_leds_match[] = {
+	{ .compatible = "ti,lm36922", },
+	{ .compatible = "ti,lm36923", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, of_lm3692x_leds_match);
+
+static struct i2c_driver lm3692x_driver = {
+	.driver = {
+		.name	= "lm3692x",
+		.of_match_table = of_lm3692x_leds_match,
+	},
+	.probe		= lm3692x_probe,
+	.remove		= lm3692x_remove,
+	.id_table	= lm3692x_id,
+};
+module_i2c_driver(lm3692x_driver);
+
+MODULE_DESCRIPTION("Texas Instruments LM3692X LED driver");
+MODULE_AUTHOR("Dan Murphy <dmurphy@ti.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-lp8860.c b/drivers/leds/leds-lp8860.c
index 3e70775a2d54..39c72a908f3b 100644
--- a/drivers/leds/leds-lp8860.c
+++ b/drivers/leds/leds-lp8860.c
@@ -22,6 +22,7 @@
 #include <linux/of_gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/slab.h>
+#include <uapi/linux/uleds.h>
 
 #define LP8860_DISP_CL1_BRT_MSB		0x00
 #define LP8860_DISP_CL1_BRT_LSB		0x01
@@ -86,8 +87,6 @@
 
 #define LP8860_CLEAR_FAULTS		0x01
 
-#define LP8860_DISP_LED_NAME		"display_cluster"
-
 /**
  * struct lp8860_led -
  * @lock - Lock for reading/writing the device
@@ -98,7 +97,7 @@
  * @enable_gpio - VDDIO/EN gpio to enable communication interface
  * @regulator - LED supply regulator pointer
  * @label - LED label
-**/
+ */
 struct lp8860_led {
 	struct mutex lock;
 	struct i2c_client *client;
@@ -107,7 +106,7 @@ struct lp8860_led {
 	struct regmap *eeprom_regmap;
 	struct gpio_desc *enable_gpio;
 	struct regulator *regulator;
-	const char *label;
+	char label[LED_MAX_NAME_SIZE];
 };
 
 struct lp8860_eeprom_reg {
@@ -247,6 +246,15 @@ static int lp8860_init(struct lp8860_led *led)
 	unsigned int read_buf;
 	int ret, i, reg_count;
 
+	if (led->regulator) {
+		ret = regulator_enable(led->regulator);
+		if (ret) {
+			dev_err(&led->client->dev,
+				"Failed to enable regulator\n");
+			return ret;
+		}
+	}
+
 	if (led->enable_gpio)
 		gpiod_direction_output(led->enable_gpio, 1);
 
@@ -282,12 +290,25 @@ static int lp8860_init(struct lp8860_led *led)
 	ret = regmap_write(led->regmap,
 			LP8860_EEPROM_CNTRL,
 			LP8860_PROGRAM_EEPROM);
-	if (ret)
+	if (ret) {
 		dev_err(&led->client->dev, "Failed programming EEPROM\n");
+		goto out;
+	}
+
+	return ret;
+
 out:
 	if (ret)
 		if (led->enable_gpio)
 			gpiod_direction_output(led->enable_gpio, 0);
+
+	if (led->regulator) {
+		ret = regulator_disable(led->regulator);
+		if (ret)
+			dev_err(&led->client->dev,
+				"Failed to disable regulator\n");
+	}
+
 	return ret;
 }
 
@@ -365,19 +386,25 @@ static int lp8860_probe(struct i2c_client *client,
 	int ret;
 	struct lp8860_led *led;
 	struct device_node *np = client->dev.of_node;
+	struct device_node *child_node;
+	const char *name;
 
 	led = devm_kzalloc(&client->dev, sizeof(*led), GFP_KERNEL);
 	if (!led)
 		return -ENOMEM;
 
-	led->label = LP8860_DISP_LED_NAME;
-
-	if (client->dev.of_node) {
-		ret = of_property_read_string(np, "label", &led->label);
-		if (ret) {
-			dev_err(&client->dev, "Missing label in dt\n");
-			return -EINVAL;
-		}
+	for_each_available_child_of_node(np, child_node) {
+		led->led_dev.default_trigger = of_get_property(child_node,
+						    "linux,default-trigger",
+						    NULL);
+
+		ret = of_property_read_string(child_node, "label", &name);
+		if (!ret)
+			snprintf(led->label, sizeof(led->label), "%s:%s",
+				 id->name, name);
+		else
+			snprintf(led->label, sizeof(led->label),
+				"%s::display_cluster", id->name);
 	}
 
 	led->enable_gpio = devm_gpiod_get_optional(&client->dev,
@@ -394,7 +421,6 @@ static int lp8860_probe(struct i2c_client *client,
 
 	led->client = client;
 	led->led_dev.name = led->label;
-	led->led_dev.max_brightness = LED_FULL;
 	led->led_dev.brightness_set_blocking = lp8860_brightness_set;
 
 	mutex_init(&led->lock);
@@ -421,7 +447,7 @@ static int lp8860_probe(struct i2c_client *client,
 	if (ret)
 		return ret;
 
-	ret = led_classdev_register(&client->dev, &led->led_dev);
+	ret = devm_led_classdev_register(&client->dev, &led->led_dev);
 	if (ret) {
 		dev_err(&client->dev, "led register err: %d\n", ret);
 		return ret;
@@ -435,8 +461,6 @@ static int lp8860_remove(struct i2c_client *client)
 	struct lp8860_led *led = i2c_get_clientdata(client);
 	int ret;
 
-	led_classdev_unregister(&led->led_dev);
-
 	if (led->enable_gpio)
 		gpiod_direction_output(led->enable_gpio, 0);
 
@@ -447,6 +471,8 @@ static int lp8860_remove(struct i2c_client *client)
 				"Failed to disable regulator\n");
 	}
 
+	mutex_destroy(&led->lock);
+
 	return 0;
 }
 
@@ -456,18 +482,16 @@ static const struct i2c_device_id lp8860_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, lp8860_id);
 
-#ifdef CONFIG_OF
 static const struct of_device_id of_lp8860_leds_match[] = {
 	{ .compatible = "ti,lp8860", },
 	{},
 };
 MODULE_DEVICE_TABLE(of, of_lp8860_leds_match);
-#endif
 
 static struct i2c_driver lp8860_driver = {
 	.driver = {
 		.name	= "lp8860",
-		.of_match_table = of_match_ptr(of_lp8860_leds_match),
+		.of_match_table = of_lp8860_leds_match,
 	},
 	.probe		= lp8860_probe,
 	.remove		= lp8860_remove,
@@ -477,4 +501,4 @@ module_i2c_driver(lp8860_driver);
 
 MODULE_DESCRIPTION("Texas Instruments LP8860 LED driver");
 MODULE_AUTHOR("Dan Murphy <dmurphy@ti.com>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/leds-pm8058.c b/drivers/leds/leds-pm8058.c
index a52674327857..8988ba3b2d65 100644
--- a/drivers/leds/leds-pm8058.c
+++ b/drivers/leds/leds-pm8058.c
@@ -106,7 +106,7 @@ static int pm8058_led_probe(struct platform_device *pdev)
 	if (!led)
 		return -ENOMEM;
 
-	led->ledtype = (u32)of_device_get_match_data(&pdev->dev);
+	led->ledtype = (u32)(unsigned long)of_device_get_match_data(&pdev->dev);
 
 	map = dev_get_regmap(pdev->dev.parent, NULL);
 	if (!map) {
diff --git a/drivers/leds/leds-pwm.c b/drivers/leds/leds-pwm.c
index 8d456dc6c5bf..df80c89ebe7f 100644
--- a/drivers/leds/leds-pwm.c
+++ b/drivers/leds/leds-pwm.c
@@ -16,7 +16,6 @@
 #include <linux/kernel.h>
 #include <linux/platform_device.h>
 #include <linux/of_platform.h>
-#include <linux/fb.h>
 #include <linux/leds.h>
 #include <linux/err.h>
 #include <linux/pwm.h>
diff --git a/drivers/leds/trigger/Kconfig b/drivers/leds/trigger/Kconfig
index bb090216b4dc..a2559b4fdfff 100644
--- a/drivers/leds/trigger/Kconfig
+++ b/drivers/leds/trigger/Kconfig
@@ -81,7 +81,7 @@ config LEDS_TRIGGER_ACTIVITY
 	tristate "LED activity Trigger"
 	depends on LEDS_TRIGGERS
 	help
-	  This allows LEDs to be controlled by a immediate CPU usage.
+	  This allows LEDs to be controlled by an immediate CPU usage.
 	  The flash frequency and duty cycle varies from faint flashes to
 	  intense brightness depending on the instant CPU load.
 	  If unsure, say N.
@@ -135,4 +135,11 @@ config LEDS_TRIGGER_PANIC
 	  a different trigger.
 	  If unsure, say Y.
 
+config LEDS_TRIGGER_NETDEV
+	tristate "LED Netdev Trigger"
+	depends on NET && LEDS_TRIGGERS
+	help
+	  This allows LEDs to be controlled by network device activity.
+	  If unsure, say Y.
+
 endif # LEDS_TRIGGERS
diff --git a/drivers/leds/trigger/Makefile b/drivers/leds/trigger/Makefile
index 4a8b6cff7761..f3cfe1950538 100644
--- a/drivers/leds/trigger/Makefile
+++ b/drivers/leds/trigger/Makefile
@@ -12,3 +12,4 @@ obj-$(CONFIG_LEDS_TRIGGER_DEFAULT_ON)	+= ledtrig-default-on.o
 obj-$(CONFIG_LEDS_TRIGGER_TRANSIENT)	+= ledtrig-transient.o
 obj-$(CONFIG_LEDS_TRIGGER_CAMERA)	+= ledtrig-camera.o
 obj-$(CONFIG_LEDS_TRIGGER_PANIC)	+= ledtrig-panic.o
+obj-$(CONFIG_LEDS_TRIGGER_NETDEV)	+= ledtrig-netdev.o
diff --git a/drivers/leds/trigger/ledtrig-netdev.c b/drivers/leds/trigger/ledtrig-netdev.c
new file mode 100644
index 000000000000..6df4781a6308
--- /dev/null
+++ b/drivers/leds/trigger/ledtrig-netdev.c
@@ -0,0 +1,496 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright 2017 Ben Whitten <ben.whitten@gmail.com>
+// Copyright 2007 Oliver Jowett <oliver@opencloud.com>
+//
+// LED Kernel Netdev Trigger
+//
+// Toggles the LED to reflect the link and traffic state of a named net device
+//
+// Derived from ledtrig-timer.c which is:
+//  Copyright 2005-2006 Openedhand Ltd.
+//  Author: Richard Purdie <rpurdie@openedhand.com>
+
+#include <linux/atomic.h>
+#include <linux/ctype.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/leds.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/spinlock.h>
+#include <linux/timer.h>
+#include "../leds.h"
+
+/*
+ * Configurable sysfs attributes:
+ *
+ * device_name - network device name to monitor
+ * interval - duration of LED blink, in milliseconds
+ * link -  LED's normal state reflects whether the link is up
+ *         (has carrier) or not
+ * tx -  LED blinks on transmitted data
+ * rx -  LED blinks on receive data
+ *
+ */
+
+struct led_netdev_data {
+	spinlock_t lock;
+
+	struct delayed_work work;
+	struct notifier_block notifier;
+
+	struct led_classdev *led_cdev;
+	struct net_device *net_dev;
+
+	char device_name[IFNAMSIZ];
+	atomic_t interval;
+	unsigned int last_activity;
+
+	unsigned long mode;
+#define NETDEV_LED_LINK	0
+#define NETDEV_LED_TX	1
+#define NETDEV_LED_RX	2
+#define NETDEV_LED_MODE_LINKUP	3
+};
+
+enum netdev_led_attr {
+	NETDEV_ATTR_LINK,
+	NETDEV_ATTR_TX,
+	NETDEV_ATTR_RX
+};
+
+static void set_baseline_state(struct led_netdev_data *trigger_data)
+{
+	int current_brightness;
+	struct led_classdev *led_cdev = trigger_data->led_cdev;
+
+	current_brightness = led_cdev->brightness;
+	if (current_brightness)
+		led_cdev->blink_brightness = current_brightness;
+	if (!led_cdev->blink_brightness)
+		led_cdev->blink_brightness = led_cdev->max_brightness;
+
+	if (!test_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode))
+		led_set_brightness(led_cdev, LED_OFF);
+	else {
+		if (test_bit(NETDEV_LED_LINK, &trigger_data->mode))
+			led_set_brightness(led_cdev,
+					   led_cdev->blink_brightness);
+		else
+			led_set_brightness(led_cdev, LED_OFF);
+
+		/* If we are looking for RX/TX start periodically
+		 * checking stats
+		 */
+		if (test_bit(NETDEV_LED_TX, &trigger_data->mode) ||
+		    test_bit(NETDEV_LED_RX, &trigger_data->mode))
+			schedule_delayed_work(&trigger_data->work, 0);
+	}
+}
+
+static ssize_t device_name_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	struct led_netdev_data *trigger_data = led_cdev->trigger_data;
+	ssize_t len;
+
+	spin_lock_bh(&trigger_data->lock);
+	len = sprintf(buf, "%s\n", trigger_data->device_name);
+	spin_unlock_bh(&trigger_data->lock);
+
+	return len;
+}
+
+static ssize_t device_name_store(struct device *dev,
+				 struct device_attribute *attr, const char *buf,
+				 size_t size)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	struct led_netdev_data *trigger_data = led_cdev->trigger_data;
+
+	if (size >= IFNAMSIZ)
+		return -EINVAL;
+
+	cancel_delayed_work_sync(&trigger_data->work);
+
+	spin_lock_bh(&trigger_data->lock);
+
+	if (trigger_data->net_dev) {
+		dev_put(trigger_data->net_dev);
+		trigger_data->net_dev = NULL;
+	}
+
+	strncpy(trigger_data->device_name, buf, size);
+	if (size > 0 && trigger_data->device_name[size - 1] == '\n')
+		trigger_data->device_name[size - 1] = 0;
+
+	if (trigger_data->device_name[0] != 0)
+		trigger_data->net_dev =
+		    dev_get_by_name(&init_net, trigger_data->device_name);
+
+	clear_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode);
+	if (trigger_data->net_dev != NULL)
+		if (netif_carrier_ok(trigger_data->net_dev))
+			set_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode);
+
+	trigger_data->last_activity = 0;
+
+	set_baseline_state(trigger_data);
+	spin_unlock_bh(&trigger_data->lock);
+
+	return size;
+}
+
+static DEVICE_ATTR_RW(device_name);
+
+static ssize_t netdev_led_attr_show(struct device *dev, char *buf,
+	enum netdev_led_attr attr)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	struct led_netdev_data *trigger_data = led_cdev->trigger_data;
+	int bit;
+
+	switch (attr) {
+	case NETDEV_ATTR_LINK:
+		bit = NETDEV_LED_LINK;
+		break;
+	case NETDEV_ATTR_TX:
+		bit = NETDEV_LED_TX;
+		break;
+	case NETDEV_ATTR_RX:
+		bit = NETDEV_LED_RX;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return sprintf(buf, "%u\n", test_bit(bit, &trigger_data->mode));
+}
+
+static ssize_t netdev_led_attr_store(struct device *dev, const char *buf,
+	size_t size, enum netdev_led_attr attr)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	struct led_netdev_data *trigger_data = led_cdev->trigger_data;
+	unsigned long state;
+	int ret;
+	int bit;
+
+	ret = kstrtoul(buf, 0, &state);
+	if (ret)
+		return ret;
+
+	switch (attr) {
+	case NETDEV_ATTR_LINK:
+		bit = NETDEV_LED_LINK;
+		break;
+	case NETDEV_ATTR_TX:
+		bit = NETDEV_LED_TX;
+		break;
+	case NETDEV_ATTR_RX:
+		bit = NETDEV_LED_RX;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	cancel_delayed_work_sync(&trigger_data->work);
+
+	if (state)
+		set_bit(bit, &trigger_data->mode);
+	else
+		clear_bit(bit, &trigger_data->mode);
+
+	set_baseline_state(trigger_data);
+
+	return size;
+}
+
+static ssize_t link_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	return netdev_led_attr_show(dev, buf, NETDEV_ATTR_LINK);
+}
+
+static ssize_t link_store(struct device *dev,
+	struct device_attribute *attr, const char *buf, size_t size)
+{
+	return netdev_led_attr_store(dev, buf, size, NETDEV_ATTR_LINK);
+}
+
+static DEVICE_ATTR_RW(link);
+
+static ssize_t tx_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	return netdev_led_attr_show(dev, buf, NETDEV_ATTR_TX);
+}
+
+static ssize_t tx_store(struct device *dev,
+	struct device_attribute *attr, const char *buf, size_t size)
+{
+	return netdev_led_attr_store(dev, buf, size, NETDEV_ATTR_TX);
+}
+
+static DEVICE_ATTR_RW(tx);
+
+static ssize_t rx_show(struct device *dev,
+	struct device_attribute *attr, char *buf)
+{
+	return netdev_led_attr_show(dev, buf, NETDEV_ATTR_RX);
+}
+
+static ssize_t rx_store(struct device *dev,
+	struct device_attribute *attr, const char *buf, size_t size)
+{
+	return netdev_led_attr_store(dev, buf, size, NETDEV_ATTR_RX);
+}
+
+static DEVICE_ATTR_RW(rx);
+
+static ssize_t interval_show(struct device *dev,
+			     struct device_attribute *attr, char *buf)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	struct led_netdev_data *trigger_data = led_cdev->trigger_data;
+
+	return sprintf(buf, "%u\n",
+		       jiffies_to_msecs(atomic_read(&trigger_data->interval)));
+}
+
+static ssize_t interval_store(struct device *dev,
+			      struct device_attribute *attr, const char *buf,
+			      size_t size)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	struct led_netdev_data *trigger_data = led_cdev->trigger_data;
+	unsigned long value;
+	int ret;
+
+	ret = kstrtoul(buf, 0, &value);
+	if (ret)
+		return ret;
+
+	/* impose some basic bounds on the timer interval */
+	if (value >= 5 && value <= 10000) {
+		cancel_delayed_work_sync(&trigger_data->work);
+
+		atomic_set(&trigger_data->interval, msecs_to_jiffies(value));
+		set_baseline_state(trigger_data);	/* resets timer */
+	}
+
+	return size;
+}
+
+static DEVICE_ATTR_RW(interval);
+
+static int netdev_trig_notify(struct notifier_block *nb,
+			      unsigned long evt, void *dv)
+{
+	struct net_device *dev =
+		netdev_notifier_info_to_dev((struct netdev_notifier_info *)dv);
+	struct led_netdev_data *trigger_data = container_of(nb,
+							    struct
+							    led_netdev_data,
+							    notifier);
+
+	if (evt != NETDEV_UP && evt != NETDEV_DOWN && evt != NETDEV_CHANGE
+	    && evt != NETDEV_REGISTER && evt != NETDEV_UNREGISTER
+	    && evt != NETDEV_CHANGENAME)
+		return NOTIFY_DONE;
+
+	if (strcmp(dev->name, trigger_data->device_name))
+		return NOTIFY_DONE;
+
+	cancel_delayed_work_sync(&trigger_data->work);
+
+	spin_lock_bh(&trigger_data->lock);
+
+	clear_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode);
+	switch (evt) {
+	case NETDEV_REGISTER:
+		if (trigger_data->net_dev)
+			dev_put(trigger_data->net_dev);
+		dev_hold(dev);
+		trigger_data->net_dev = dev;
+		break;
+	case NETDEV_CHANGENAME:
+	case NETDEV_UNREGISTER:
+		if (trigger_data->net_dev) {
+			dev_put(trigger_data->net_dev);
+			trigger_data->net_dev = NULL;
+		}
+		break;
+	case NETDEV_UP:
+	case NETDEV_CHANGE:
+		if (netif_carrier_ok(dev))
+			set_bit(NETDEV_LED_MODE_LINKUP, &trigger_data->mode);
+		break;
+	}
+
+	set_baseline_state(trigger_data);
+
+	spin_unlock_bh(&trigger_data->lock);
+
+	return NOTIFY_DONE;
+}
+
+/* here's the real work! */
+static void netdev_trig_work(struct work_struct *work)
+{
+	struct led_netdev_data *trigger_data = container_of(work,
+							    struct
+							    led_netdev_data,
+							    work.work);
+	struct rtnl_link_stats64 *dev_stats;
+	unsigned int new_activity;
+	struct rtnl_link_stats64 temp;
+	unsigned long interval;
+	int invert;
+
+	/* If we dont have a device, insure we are off */
+	if (!trigger_data->net_dev) {
+		led_set_brightness(trigger_data->led_cdev, LED_OFF);
+		return;
+	}
+
+	/* If we are not looking for RX/TX then return  */
+	if (!test_bit(NETDEV_LED_TX, &trigger_data->mode) &&
+	    !test_bit(NETDEV_LED_RX, &trigger_data->mode))
+		return;
+
+	dev_stats = dev_get_stats(trigger_data->net_dev, &temp);
+	new_activity =
+	    (test_bit(NETDEV_LED_TX, &trigger_data->mode) ?
+		dev_stats->tx_packets : 0) +
+	    (test_bit(NETDEV_LED_RX, &trigger_data->mode) ?
+		dev_stats->rx_packets : 0);
+
+	if (trigger_data->last_activity != new_activity) {
+		led_stop_software_blink(trigger_data->led_cdev);
+
+		invert = test_bit(NETDEV_LED_LINK, &trigger_data->mode);
+		interval = jiffies_to_msecs(
+				atomic_read(&trigger_data->interval));
+		/* base state is ON (link present) */
+		led_blink_set_oneshot(trigger_data->led_cdev,
+				      &interval,
+				      &interval,
+				      invert);
+		trigger_data->last_activity = new_activity;
+	}
+
+	schedule_delayed_work(&trigger_data->work,
+			(atomic_read(&trigger_data->interval)*2));
+}
+
+static void netdev_trig_activate(struct led_classdev *led_cdev)
+{
+	struct led_netdev_data *trigger_data;
+	int rc;
+
+	trigger_data = kzalloc(sizeof(struct led_netdev_data), GFP_KERNEL);
+	if (!trigger_data)
+		return;
+
+	spin_lock_init(&trigger_data->lock);
+
+	trigger_data->notifier.notifier_call = netdev_trig_notify;
+	trigger_data->notifier.priority = 10;
+
+	INIT_DELAYED_WORK(&trigger_data->work, netdev_trig_work);
+
+	trigger_data->led_cdev = led_cdev;
+	trigger_data->net_dev = NULL;
+	trigger_data->device_name[0] = 0;
+
+	trigger_data->mode = 0;
+	atomic_set(&trigger_data->interval, msecs_to_jiffies(50));
+	trigger_data->last_activity = 0;
+
+	led_cdev->trigger_data = trigger_data;
+
+	rc = device_create_file(led_cdev->dev, &dev_attr_device_name);
+	if (rc)
+		goto err_out;
+	rc = device_create_file(led_cdev->dev, &dev_attr_link);
+	if (rc)
+		goto err_out_device_name;
+	rc = device_create_file(led_cdev->dev, &dev_attr_rx);
+	if (rc)
+		goto err_out_link;
+	rc = device_create_file(led_cdev->dev, &dev_attr_tx);
+	if (rc)
+		goto err_out_rx;
+	rc = device_create_file(led_cdev->dev, &dev_attr_interval);
+	if (rc)
+		goto err_out_tx;
+	rc = register_netdevice_notifier(&trigger_data->notifier);
+	if (rc)
+		goto err_out_interval;
+	return;
+
+err_out_interval:
+	device_remove_file(led_cdev->dev, &dev_attr_interval);
+err_out_tx:
+	device_remove_file(led_cdev->dev, &dev_attr_tx);
+err_out_rx:
+	device_remove_file(led_cdev->dev, &dev_attr_rx);
+err_out_link:
+	device_remove_file(led_cdev->dev, &dev_attr_link);
+err_out_device_name:
+	device_remove_file(led_cdev->dev, &dev_attr_device_name);
+err_out:
+	led_cdev->trigger_data = NULL;
+	kfree(trigger_data);
+}
+
+static void netdev_trig_deactivate(struct led_classdev *led_cdev)
+{
+	struct led_netdev_data *trigger_data = led_cdev->trigger_data;
+
+	if (trigger_data) {
+		unregister_netdevice_notifier(&trigger_data->notifier);
+
+		device_remove_file(led_cdev->dev, &dev_attr_device_name);
+		device_remove_file(led_cdev->dev, &dev_attr_link);
+		device_remove_file(led_cdev->dev, &dev_attr_rx);
+		device_remove_file(led_cdev->dev, &dev_attr_tx);
+		device_remove_file(led_cdev->dev, &dev_attr_interval);
+
+		cancel_delayed_work_sync(&trigger_data->work);
+
+		if (trigger_data->net_dev)
+			dev_put(trigger_data->net_dev);
+
+		kfree(trigger_data);
+	}
+}
+
+static struct led_trigger netdev_led_trigger = {
+	.name = "netdev",
+	.activate = netdev_trig_activate,
+	.deactivate = netdev_trig_deactivate,
+};
+
+static int __init netdev_trig_init(void)
+{
+	return led_trigger_register(&netdev_led_trigger);
+}
+
+static void __exit netdev_trig_exit(void)
+{
+	led_trigger_unregister(&netdev_led_trigger);
+}
+
+module_init(netdev_trig_init);
+module_exit(netdev_trig_exit);
+
+MODULE_AUTHOR("Ben Whitten <ben.whitten@gmail.com>");
+MODULE_AUTHOR("Oliver Jowett <oliver@opencloud.com>");
+MODULE_DESCRIPTION("Netdev LED trigger");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/trigger/ledtrig-transient.c b/drivers/leds/trigger/ledtrig-transient.c
index 7acce64b692a..9d1769073562 100644
--- a/drivers/leds/trigger/ledtrig-transient.c
+++ b/drivers/leds/trigger/ledtrig-transient.c
@@ -1,22 +1,15 @@
-/*
- * LED Kernel Transient Trigger
- *
- * Copyright (C) 2012 Shuah Khan <shuahkhan@gmail.com>
- *
- * Based on Richard Purdie's ledtrig-timer.c and Atsushi Nemoto's
- * ledtrig-heartbeat.c
- * Design and use-case input from Jonas Bonn <jonas@southpole.se> and
- * Neil Brown <neilb@suse.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
-/*
- * Transient trigger allows one shot timer activation. Please refer to
- * Documentation/leds/ledtrig-transient.txt for details
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// LED Kernel Transient Trigger
+//
+// Transient trigger allows one shot timer activation. Please refer to
+// Documentation/leds/ledtrig-transient.txt for details
+// Copyright (C) 2012 Shuah Khan <shuahkhan@gmail.com>
+//
+// Based on Richard Purdie's ledtrig-timer.c and Atsushi Nemoto's
+// ledtrig-heartbeat.c
+// Design and use-case input from Jonas Bonn <jonas@southpole.se> and
+// Neil Brown <neilb@suse.de>
 
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -238,4 +231,4 @@ module_exit(transient_trig_exit);
 
 MODULE_AUTHOR("Shuah Khan <shuahkhan@gmail.com>");
 MODULE_DESCRIPTION("Transient LED trigger");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/leds/uleds.c b/drivers/leds/uleds.c
index 5e9e8a1fdefb..0c43bfac9598 100644
--- a/drivers/leds/uleds.c
+++ b/drivers/leds/uleds.c
@@ -176,14 +176,14 @@ static ssize_t uleds_read(struct file *file, char __user *buffer, size_t count,
 	return retval;
 }
 
-static unsigned int uleds_poll(struct file *file, poll_table *wait)
+static __poll_t uleds_poll(struct file *file, poll_table *wait)
 {
 	struct uleds_device *udev = file->private_data;
 
 	poll_wait(file, &udev->waitq, wait);
 
 	if (udev->new_data)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/drivers/lightnvm/Kconfig b/drivers/lightnvm/Kconfig
index 2a953efec4e1..10c08982185a 100644
--- a/drivers/lightnvm/Kconfig
+++ b/drivers/lightnvm/Kconfig
@@ -27,13 +27,6 @@ config NVM_DEBUG
 
 	It is required to create/remove targets without IOCTLs.
 
-config NVM_RRPC
-	tristate "Round-robin Hybrid Open-Channel SSD target"
-	---help---
-	Allows an open-channel SSD to be exposed as a block device to the
-	host. The target is implemented using a linear mapping table and
-	cost-based garbage collection. It is optimized for 4K IO sizes.
-
 config NVM_PBLK
 	tristate "Physical Block Device Open-Channel SSD target"
 	---help---
diff --git a/drivers/lightnvm/Makefile b/drivers/lightnvm/Makefile
index 2c3fd9d2c08c..97d9d7c71550 100644
--- a/drivers/lightnvm/Makefile
+++ b/drivers/lightnvm/Makefile
@@ -4,7 +4,6 @@
 #
 
 obj-$(CONFIG_NVM)		:= core.o
-obj-$(CONFIG_NVM_RRPC)		+= rrpc.o
 obj-$(CONFIG_NVM_PBLK)		+= pblk.o
 pblk-y				:= pblk-init.o pblk-core.o pblk-rb.o \
 				   pblk-write.o pblk-cache.o pblk-read.o \
diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c
index 83249b43dd06..dcc9e621e651 100644
--- a/drivers/lightnvm/core.c
+++ b/drivers/lightnvm/core.c
@@ -45,12 +45,6 @@ struct nvm_dev_map {
 	int nr_chnls;
 };
 
-struct nvm_area {
-	struct list_head list;
-	sector_t begin;
-	sector_t end;	/* end is excluded */
-};
-
 static struct nvm_target *nvm_find_target(struct nvm_dev *dev, const char *name)
 {
 	struct nvm_target *tgt;
@@ -62,6 +56,30 @@ static struct nvm_target *nvm_find_target(struct nvm_dev *dev, const char *name)
 	return NULL;
 }
 
+static bool nvm_target_exists(const char *name)
+{
+	struct nvm_dev *dev;
+	struct nvm_target *tgt;
+	bool ret = false;
+
+	down_write(&nvm_lock);
+	list_for_each_entry(dev, &nvm_devices, devices) {
+		mutex_lock(&dev->mlock);
+		list_for_each_entry(tgt, &dev->targets, list) {
+			if (!strcmp(name, tgt->disk->disk_name)) {
+				ret = true;
+				mutex_unlock(&dev->mlock);
+				goto out;
+			}
+		}
+		mutex_unlock(&dev->mlock);
+	}
+
+out:
+	up_write(&nvm_lock);
+	return ret;
+}
+
 static int nvm_reserve_luns(struct nvm_dev *dev, int lun_begin, int lun_end)
 {
 	int i;
@@ -104,7 +122,7 @@ static void nvm_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev, int clear)
 		if (clear) {
 			for (j = 0; j < ch_map->nr_luns; j++) {
 				int lun = j + lun_offs[j];
-				int lunid = (ch * dev->geo.luns_per_chnl) + lun;
+				int lunid = (ch * dev->geo.nr_luns) + lun;
 
 				WARN_ON(!test_and_clear_bit(lunid,
 							dev->lun_map));
@@ -122,7 +140,8 @@ static void nvm_remove_tgt_dev(struct nvm_tgt_dev *tgt_dev, int clear)
 }
 
 static struct nvm_tgt_dev *nvm_create_tgt_dev(struct nvm_dev *dev,
-					      int lun_begin, int lun_end)
+					      u16 lun_begin, u16 lun_end,
+					      u16 op)
 {
 	struct nvm_tgt_dev *tgt_dev = NULL;
 	struct nvm_dev_map *dev_rmap = dev->rmap;
@@ -130,10 +149,10 @@ static struct nvm_tgt_dev *nvm_create_tgt_dev(struct nvm_dev *dev,
 	struct ppa_addr *luns;
 	int nr_luns = lun_end - lun_begin + 1;
 	int luns_left = nr_luns;
-	int nr_chnls = nr_luns / dev->geo.luns_per_chnl;
-	int nr_chnls_mod = nr_luns % dev->geo.luns_per_chnl;
-	int bch = lun_begin / dev->geo.luns_per_chnl;
-	int blun = lun_begin % dev->geo.luns_per_chnl;
+	int nr_chnls = nr_luns / dev->geo.nr_luns;
+	int nr_chnls_mod = nr_luns % dev->geo.nr_luns;
+	int bch = lun_begin / dev->geo.nr_luns;
+	int blun = lun_begin % dev->geo.nr_luns;
 	int lunid = 0;
 	int lun_balanced = 1;
 	int prev_nr_luns;
@@ -154,15 +173,15 @@ static struct nvm_tgt_dev *nvm_create_tgt_dev(struct nvm_dev *dev,
 	if (!luns)
 		goto err_luns;
 
-	prev_nr_luns = (luns_left > dev->geo.luns_per_chnl) ?
-					dev->geo.luns_per_chnl : luns_left;
+	prev_nr_luns = (luns_left > dev->geo.nr_luns) ?
+					dev->geo.nr_luns : luns_left;
 	for (i = 0; i < nr_chnls; i++) {
 		struct nvm_ch_map *ch_rmap = &dev_rmap->chnls[i + bch];
 		int *lun_roffs = ch_rmap->lun_offs;
 		struct nvm_ch_map *ch_map = &dev_map->chnls[i];
 		int *lun_offs;
-		int luns_in_chnl = (luns_left > dev->geo.luns_per_chnl) ?
-					dev->geo.luns_per_chnl : luns_left;
+		int luns_in_chnl = (luns_left > dev->geo.nr_luns) ?
+					dev->geo.nr_luns : luns_left;
 
 		if (lun_balanced && prev_nr_luns != luns_in_chnl)
 			lun_balanced = 0;
@@ -199,8 +218,9 @@ static struct nvm_tgt_dev *nvm_create_tgt_dev(struct nvm_dev *dev,
 	memcpy(&tgt_dev->geo, &dev->geo, sizeof(struct nvm_geo));
 	/* Target device only owns a portion of the physical device */
 	tgt_dev->geo.nr_chnls = nr_chnls;
-	tgt_dev->geo.nr_luns = nr_luns;
-	tgt_dev->geo.luns_per_chnl = (lun_balanced) ? prev_nr_luns : -1;
+	tgt_dev->geo.all_luns = nr_luns;
+	tgt_dev->geo.nr_luns = (lun_balanced) ? prev_nr_luns : -1;
+	tgt_dev->geo.op = op;
 	tgt_dev->total_secs = nr_luns * tgt_dev->geo.sec_per_lun;
 	tgt_dev->q = dev->q;
 	tgt_dev->map = dev_map;
@@ -226,27 +246,79 @@ static const struct block_device_operations nvm_fops = {
 	.owner		= THIS_MODULE,
 };
 
-static struct nvm_tgt_type *nvm_find_target_type(const char *name, int lock)
+static struct nvm_tgt_type *__nvm_find_target_type(const char *name)
 {
-	struct nvm_tgt_type *tmp, *tt = NULL;
+	struct nvm_tgt_type *tt;
 
-	if (lock)
-		down_write(&nvm_tgtt_lock);
+	list_for_each_entry(tt, &nvm_tgt_types, list)
+		if (!strcmp(name, tt->name))
+			return tt;
 
-	list_for_each_entry(tmp, &nvm_tgt_types, list)
-		if (!strcmp(name, tmp->name)) {
-			tt = tmp;
-			break;
-		}
+	return NULL;
+}
+
+static struct nvm_tgt_type *nvm_find_target_type(const char *name)
+{
+	struct nvm_tgt_type *tt;
+
+	down_write(&nvm_tgtt_lock);
+	tt = __nvm_find_target_type(name);
+	up_write(&nvm_tgtt_lock);
 
-	if (lock)
-		up_write(&nvm_tgtt_lock);
 	return tt;
 }
 
+static int nvm_config_check_luns(struct nvm_geo *geo, int lun_begin,
+				 int lun_end)
+{
+	if (lun_begin > lun_end || lun_end >= geo->all_luns) {
+		pr_err("nvm: lun out of bound (%u:%u > %u)\n",
+			lun_begin, lun_end, geo->all_luns - 1);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int __nvm_config_simple(struct nvm_dev *dev,
+			       struct nvm_ioctl_create_simple *s)
+{
+	struct nvm_geo *geo = &dev->geo;
+
+	if (s->lun_begin == -1 && s->lun_end == -1) {
+		s->lun_begin = 0;
+		s->lun_end = geo->all_luns - 1;
+	}
+
+	return nvm_config_check_luns(geo, s->lun_begin, s->lun_end);
+}
+
+static int __nvm_config_extended(struct nvm_dev *dev,
+				 struct nvm_ioctl_create_extended *e)
+{
+	struct nvm_geo *geo = &dev->geo;
+
+	if (e->lun_begin == 0xFFFF && e->lun_end == 0xFFFF) {
+		e->lun_begin = 0;
+		e->lun_end = dev->geo.all_luns - 1;
+	}
+
+	/* op not set falls into target's default */
+	if (e->op == 0xFFFF)
+		e->op = NVM_TARGET_DEFAULT_OP;
+
+	if (e->op < NVM_TARGET_MIN_OP ||
+	    e->op > NVM_TARGET_MAX_OP) {
+		pr_err("nvm: invalid over provisioning value\n");
+		return -EINVAL;
+	}
+
+	return nvm_config_check_luns(geo, e->lun_begin, e->lun_end);
+}
+
 static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 {
-	struct nvm_ioctl_create_simple *s = &create->conf.s;
+	struct nvm_ioctl_create_extended e;
 	struct request_queue *tqueue;
 	struct gendisk *tdisk;
 	struct nvm_tgt_type *tt;
@@ -255,22 +327,41 @@ static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 	void *targetdata;
 	int ret;
 
-	tt = nvm_find_target_type(create->tgttype, 1);
+	switch (create->conf.type) {
+	case NVM_CONFIG_TYPE_SIMPLE:
+		ret = __nvm_config_simple(dev, &create->conf.s);
+		if (ret)
+			return ret;
+
+		e.lun_begin = create->conf.s.lun_begin;
+		e.lun_end = create->conf.s.lun_end;
+		e.op = NVM_TARGET_DEFAULT_OP;
+		break;
+	case NVM_CONFIG_TYPE_EXTENDED:
+		ret = __nvm_config_extended(dev, &create->conf.e);
+		if (ret)
+			return ret;
+
+		e = create->conf.e;
+		break;
+	default:
+		pr_err("nvm: config type not valid\n");
+		return -EINVAL;
+	}
+
+	tt = nvm_find_target_type(create->tgttype);
 	if (!tt) {
 		pr_err("nvm: target type %s not found\n", create->tgttype);
 		return -EINVAL;
 	}
 
-	mutex_lock(&dev->mlock);
-	t = nvm_find_target(dev, create->tgtname);
-	if (t) {
-		pr_err("nvm: target name already exists.\n");
-		mutex_unlock(&dev->mlock);
+	if (nvm_target_exists(create->tgtname)) {
+		pr_err("nvm: target name already exists (%s)\n",
+							create->tgtname);
 		return -EINVAL;
 	}
-	mutex_unlock(&dev->mlock);
 
-	ret = nvm_reserve_luns(dev, s->lun_begin, s->lun_end);
+	ret = nvm_reserve_luns(dev, e.lun_begin, e.lun_end);
 	if (ret)
 		return ret;
 
@@ -280,7 +371,7 @@ static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 		goto err_reserve;
 	}
 
-	tgt_dev = nvm_create_tgt_dev(dev, s->lun_begin, s->lun_end);
+	tgt_dev = nvm_create_tgt_dev(dev, e.lun_begin, e.lun_end, e.op);
 	if (!tgt_dev) {
 		pr_err("nvm: could not create target device\n");
 		ret = -ENOMEM;
@@ -350,7 +441,7 @@ static int nvm_create_tgt(struct nvm_dev *dev, struct nvm_ioctl_create *create)
 err_t:
 	kfree(t);
 err_reserve:
-	nvm_release_luns_err(dev, s->lun_begin, s->lun_end);
+	nvm_release_luns_err(dev, e.lun_begin, e.lun_end);
 	return ret;
 }
 
@@ -420,7 +511,7 @@ static int nvm_register_map(struct nvm_dev *dev)
 	for (i = 0; i < dev->geo.nr_chnls; i++) {
 		struct nvm_ch_map *ch_rmap;
 		int *lun_roffs;
-		int luns_in_chnl = dev->geo.luns_per_chnl;
+		int luns_in_chnl = dev->geo.nr_luns;
 
 		ch_rmap = &rmap->chnls[i];
 
@@ -524,41 +615,12 @@ static void nvm_rq_dev_to_tgt(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 	nvm_ppa_dev_to_tgt(tgt_dev, rqd->ppa_list, rqd->nr_ppas);
 }
 
-void nvm_part_to_tgt(struct nvm_dev *dev, sector_t *entries,
-		     int len)
-{
-	struct nvm_geo *geo = &dev->geo;
-	struct nvm_dev_map *dev_rmap = dev->rmap;
-	u64 i;
-
-	for (i = 0; i < len; i++) {
-		struct nvm_ch_map *ch_rmap;
-		int *lun_roffs;
-		struct ppa_addr gaddr;
-		u64 pba = le64_to_cpu(entries[i]);
-		u64 diff;
-
-		if (!pba)
-			continue;
-
-		gaddr = linear_to_generic_addr(geo, pba);
-		ch_rmap = &dev_rmap->chnls[gaddr.g.ch];
-		lun_roffs = ch_rmap->lun_offs;
-
-		diff = ((ch_rmap->ch_off * geo->luns_per_chnl) +
-				(lun_roffs[gaddr.g.lun])) * geo->sec_per_lun;
-
-		entries[i] -= cpu_to_le64(diff);
-	}
-}
-EXPORT_SYMBOL(nvm_part_to_tgt);
-
 int nvm_register_tgt_type(struct nvm_tgt_type *tt)
 {
 	int ret = 0;
 
 	down_write(&nvm_tgtt_lock);
-	if (nvm_find_target_type(tt->name, 0))
+	if (__nvm_find_target_type(tt->name))
 		ret = -EEXIST;
 	else
 		list_add(&tt->list, &nvm_tgt_types);
@@ -726,112 +788,6 @@ int nvm_submit_io_sync(struct nvm_tgt_dev *tgt_dev, struct nvm_rq *rqd)
 }
 EXPORT_SYMBOL(nvm_submit_io_sync);
 
-int nvm_erase_sync(struct nvm_tgt_dev *tgt_dev, struct ppa_addr *ppas,
-								int nr_ppas)
-{
-	struct nvm_geo *geo = &tgt_dev->geo;
-	struct nvm_rq rqd;
-	int ret;
-
-	memset(&rqd, 0, sizeof(struct nvm_rq));
-
-	rqd.opcode = NVM_OP_ERASE;
-	rqd.flags = geo->plane_mode >> 1;
-
-	ret = nvm_set_rqd_ppalist(tgt_dev, &rqd, ppas, nr_ppas);
-	if (ret)
-		return ret;
-
-	ret = nvm_submit_io_sync(tgt_dev, &rqd);
-	if (ret) {
-		pr_err("rrpr: erase I/O submission failed: %d\n", ret);
-		goto free_ppa_list;
-	}
-
-free_ppa_list:
-	nvm_free_rqd_ppalist(tgt_dev, &rqd);
-
-	return ret;
-}
-EXPORT_SYMBOL(nvm_erase_sync);
-
-int nvm_get_l2p_tbl(struct nvm_tgt_dev *tgt_dev, u64 slba, u32 nlb,
-		    nvm_l2p_update_fn *update_l2p, void *priv)
-{
-	struct nvm_dev *dev = tgt_dev->parent;
-
-	if (!dev->ops->get_l2p_tbl)
-		return 0;
-
-	return dev->ops->get_l2p_tbl(dev, slba, nlb, update_l2p, priv);
-}
-EXPORT_SYMBOL(nvm_get_l2p_tbl);
-
-int nvm_get_area(struct nvm_tgt_dev *tgt_dev, sector_t *lba, sector_t len)
-{
-	struct nvm_dev *dev = tgt_dev->parent;
-	struct nvm_geo *geo = &dev->geo;
-	struct nvm_area *area, *prev, *next;
-	sector_t begin = 0;
-	sector_t max_sectors = (geo->sec_size * dev->total_secs) >> 9;
-
-	if (len > max_sectors)
-		return -EINVAL;
-
-	area = kmalloc(sizeof(struct nvm_area), GFP_KERNEL);
-	if (!area)
-		return -ENOMEM;
-
-	prev = NULL;
-
-	spin_lock(&dev->lock);
-	list_for_each_entry(next, &dev->area_list, list) {
-		if (begin + len > next->begin) {
-			begin = next->end;
-			prev = next;
-			continue;
-		}
-		break;
-	}
-
-	if ((begin + len) > max_sectors) {
-		spin_unlock(&dev->lock);
-		kfree(area);
-		return -EINVAL;
-	}
-
-	area->begin = *lba = begin;
-	area->end = begin + len;
-
-	if (prev) /* insert into sorted order */
-		list_add(&area->list, &prev->list);
-	else
-		list_add(&area->list, &dev->area_list);
-	spin_unlock(&dev->lock);
-
-	return 0;
-}
-EXPORT_SYMBOL(nvm_get_area);
-
-void nvm_put_area(struct nvm_tgt_dev *tgt_dev, sector_t begin)
-{
-	struct nvm_dev *dev = tgt_dev->parent;
-	struct nvm_area *area;
-
-	spin_lock(&dev->lock);
-	list_for_each_entry(area, &dev->area_list, list) {
-		if (area->begin != begin)
-			continue;
-
-		list_del(&area->list);
-		spin_unlock(&dev->lock);
-		kfree(area);
-		return;
-	}
-	spin_unlock(&dev->lock);
-}
-EXPORT_SYMBOL(nvm_put_area);
-
 void nvm_end_io(struct nvm_rq *rqd)
 {
 	struct nvm_tgt_dev *tgt_dev = rqd->dev;
@@ -858,10 +814,10 @@ int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
 	struct nvm_geo *geo = &dev->geo;
 	int blk, offset, pl, blktype;
 
-	if (nr_blks != geo->blks_per_lun * geo->plane_mode)
+	if (nr_blks != geo->nr_chks * geo->plane_mode)
 		return -EINVAL;
 
-	for (blk = 0; blk < geo->blks_per_lun; blk++) {
+	for (blk = 0; blk < geo->nr_chks; blk++) {
 		offset = blk * geo->plane_mode;
 		blktype = blks[offset];
 
@@ -877,7 +833,7 @@ int nvm_bb_tbl_fold(struct nvm_dev *dev, u8 *blks, int nr_blks)
 		blks[blk] = blktype;
 	}
 
-	return geo->blks_per_lun;
+	return geo->nr_chks;
 }
 EXPORT_SYMBOL(nvm_bb_tbl_fold);
 
@@ -892,53 +848,6 @@ int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *tgt_dev, struct ppa_addr ppa,
 }
 EXPORT_SYMBOL(nvm_get_tgt_bb_tbl);
 
-static int nvm_init_slc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
-{
-	struct nvm_geo *geo = &dev->geo;
-	int i;
-
-	dev->lps_per_blk = geo->pgs_per_blk;
-	dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
-	if (!dev->lptbl)
-		return -ENOMEM;
-
-	/* Just a linear array */
-	for (i = 0; i < dev->lps_per_blk; i++)
-		dev->lptbl[i] = i;
-
-	return 0;
-}
-
-static int nvm_init_mlc_tbl(struct nvm_dev *dev, struct nvm_id_group *grp)
-{
-	int i, p;
-	struct nvm_id_lp_mlc *mlc = &grp->lptbl.mlc;
-
-	if (!mlc->num_pairs)
-		return 0;
-
-	dev->lps_per_blk = mlc->num_pairs;
-	dev->lptbl = kcalloc(dev->lps_per_blk, sizeof(int), GFP_KERNEL);
-	if (!dev->lptbl)
-		return -ENOMEM;
-
-	/* The lower page table encoding consists of a list of bytes, where each
-	 * has a lower and an upper half. The first half byte maintains the
-	 * increment value and every value after is an offset added to the
-	 * previous incrementation value
-	 */
-	dev->lptbl[0] = mlc->pairs[0] & 0xF;
-	for (i = 1; i < dev->lps_per_blk; i++) {
-		p = mlc->pairs[i >> 1];
-		if (i & 0x1) /* upper */
-			dev->lptbl[i] = dev->lptbl[i - 1] + ((p & 0xF0) >> 4);
-		else /* lower */
-			dev->lptbl[i] = dev->lptbl[i - 1] + (p & 0xF);
-	}
-
-	return 0;
-}
-
 static int nvm_core_init(struct nvm_dev *dev)
 {
 	struct nvm_id *id = &dev->identity;
@@ -946,66 +855,44 @@ static int nvm_core_init(struct nvm_dev *dev)
 	struct nvm_geo *geo = &dev->geo;
 	int ret;
 
+	memcpy(&geo->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
+
+	if (grp->mtype != 0) {
+		pr_err("nvm: memory type not supported\n");
+		return -EINVAL;
+	}
+
 	/* Whole device values */
 	geo->nr_chnls = grp->num_ch;
-	geo->luns_per_chnl = grp->num_lun;
-
-	/* Generic device values */
-	geo->pgs_per_blk = grp->num_pg;
-	geo->blks_per_lun = grp->num_blk;
-	geo->nr_planes = grp->num_pln;
-	geo->fpg_size = grp->fpg_sz;
-	geo->pfpg_size = grp->fpg_sz * grp->num_pln;
+	geo->nr_luns = grp->num_lun;
+
+	/* Generic device geometry values */
+	geo->ws_min = grp->ws_min;
+	geo->ws_opt = grp->ws_opt;
+	geo->ws_seq = grp->ws_seq;
+	geo->ws_per_chk = grp->ws_per_chk;
+	geo->nr_chks = grp->num_chk;
 	geo->sec_size = grp->csecs;
 	geo->oob_size = grp->sos;
-	geo->sec_per_pg = grp->fpg_sz / grp->csecs;
 	geo->mccap = grp->mccap;
-	memcpy(&geo->ppaf, &id->ppaf, sizeof(struct nvm_addr_format));
-
-	geo->plane_mode = NVM_PLANE_SINGLE;
 	geo->max_rq_size = dev->ops->max_phys_sect * geo->sec_size;
 
-	if (grp->mpos & 0x020202)
-		geo->plane_mode = NVM_PLANE_DOUBLE;
-	if (grp->mpos & 0x040404)
-		geo->plane_mode = NVM_PLANE_QUAD;
+	geo->sec_per_chk = grp->clba;
+	geo->sec_per_lun = geo->sec_per_chk * geo->nr_chks;
+	geo->all_luns = geo->nr_luns * geo->nr_chnls;
 
-	if (grp->mtype != 0) {
-		pr_err("nvm: memory type not supported\n");
-		return -EINVAL;
-	}
-
-	/* calculated values */
+	/* 1.2 spec device geometry values */
+	geo->plane_mode = 1 << geo->ws_seq;
+	geo->nr_planes = geo->ws_opt / geo->ws_min;
+	geo->sec_per_pg = geo->ws_min;
 	geo->sec_per_pl = geo->sec_per_pg * geo->nr_planes;
-	geo->sec_per_blk = geo->sec_per_pl * geo->pgs_per_blk;
-	geo->sec_per_lun = geo->sec_per_blk * geo->blks_per_lun;
-	geo->nr_luns = geo->luns_per_chnl * geo->nr_chnls;
 
-	dev->total_secs = geo->nr_luns * geo->sec_per_lun;
-	dev->lun_map = kcalloc(BITS_TO_LONGS(geo->nr_luns),
+	dev->total_secs = geo->all_luns * geo->sec_per_lun;
+	dev->lun_map = kcalloc(BITS_TO_LONGS(geo->all_luns),
 					sizeof(unsigned long), GFP_KERNEL);
 	if (!dev->lun_map)
 		return -ENOMEM;
 
-	switch (grp->fmtype) {
-	case NVM_ID_FMTYPE_SLC:
-		if (nvm_init_slc_tbl(dev, grp)) {
-			ret = -ENOMEM;
-			goto err_fmtype;
-		}
-		break;
-	case NVM_ID_FMTYPE_MLC:
-		if (nvm_init_mlc_tbl(dev, grp)) {
-			ret = -ENOMEM;
-			goto err_fmtype;
-		}
-		break;
-	default:
-		pr_err("nvm: flash type not supported\n");
-		ret = -EINVAL;
-		goto err_fmtype;
-	}
-
 	INIT_LIST_HEAD(&dev->area_list);
 	INIT_LIST_HEAD(&dev->targets);
 	mutex_init(&dev->mlock);
@@ -1031,7 +918,6 @@ static void nvm_free(struct nvm_dev *dev)
 		dev->ops->destroy_dma_pool(dev->dma_pool);
 
 	nvm_unregister_map(dev);
-	kfree(dev->lptbl);
 	kfree(dev->lun_map);
 	kfree(dev);
 }
@@ -1062,8 +948,8 @@ static int nvm_init(struct nvm_dev *dev)
 
 	pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
 			dev->name, geo->sec_per_pg, geo->nr_planes,
-			geo->pgs_per_blk, geo->blks_per_lun,
-			geo->nr_luns, geo->nr_chnls);
+			geo->ws_per_chk, geo->nr_chks,
+			geo->all_luns, geo->nr_chnls);
 	return 0;
 err:
 	pr_err("nvm: failed to initialize nvm\n");
@@ -1135,7 +1021,6 @@ EXPORT_SYMBOL(nvm_unregister);
 static int __nvm_configure_create(struct nvm_ioctl_create *create)
 {
 	struct nvm_dev *dev;
-	struct nvm_ioctl_create_simple *s;
 
 	down_write(&nvm_lock);
 	dev = nvm_find_nvm_dev(create->dev);
@@ -1146,23 +1031,6 @@ static int __nvm_configure_create(struct nvm_ioctl_create *create)
 		return -EINVAL;
 	}
 
-	if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) {
-		pr_err("nvm: config type not valid\n");
-		return -EINVAL;
-	}
-	s = &create->conf.s;
-
-	if (s->lun_begin == -1 && s->lun_end == -1) {
-		s->lun_begin = 0;
-		s->lun_end = dev->geo.nr_luns - 1;
-	}
-
-	if (s->lun_begin > s->lun_end || s->lun_end >= dev->geo.nr_luns) {
-		pr_err("nvm: lun out of bound (%u:%u > %u)\n",
-			s->lun_begin, s->lun_end, dev->geo.nr_luns - 1);
-		return -EINVAL;
-	}
-
 	return nvm_create_tgt(dev, create);
 }
 
@@ -1262,6 +1130,12 @@ static long nvm_ioctl_dev_create(struct file *file, void __user *arg)
 	if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create)))
 		return -EFAULT;
 
+	if (create.conf.type == NVM_CONFIG_TYPE_EXTENDED &&
+	    create.conf.e.rsv != 0) {
+		pr_err("nvm: reserved config field in use\n");
+		return -EINVAL;
+	}
+
 	create.dev[DISK_NAME_LEN - 1] = '\0';
 	create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0';
 	create.tgtname[DISK_NAME_LEN - 1] = '\0';
diff --git a/drivers/lightnvm/pblk-cache.c b/drivers/lightnvm/pblk-cache.c
index 0d227ef7d1b9..000fcad38136 100644
--- a/drivers/lightnvm/pblk-cache.c
+++ b/drivers/lightnvm/pblk-cache.c
@@ -19,12 +19,16 @@
 
 int pblk_write_to_cache(struct pblk *pblk, struct bio *bio, unsigned long flags)
 {
+	struct request_queue *q = pblk->dev->q;
 	struct pblk_w_ctx w_ctx;
 	sector_t lba = pblk_get_lba(bio);
+	unsigned long start_time = jiffies;
 	unsigned int bpos, pos;
 	int nr_entries = pblk_get_secs(bio);
 	int i, ret;
 
+	generic_start_io_acct(q, WRITE, bio_sectors(bio), &pblk->disk->part0);
+
 	/* Update the write buffer head (mem) with the entries that we can
 	 * write. The write in itself cannot fail, so there is no need to
 	 * rollback from here on.
@@ -67,6 +71,7 @@ int pblk_write_to_cache(struct pblk *pblk, struct bio *bio, unsigned long flags)
 	pblk_rl_inserted(&pblk->rl, nr_entries);
 
 out:
+	generic_end_io_acct(q, WRITE, &pblk->disk->part0, start_time);
 	pblk_write_should_kick(pblk);
 	return ret;
 }
diff --git a/drivers/lightnvm/pblk-core.c b/drivers/lightnvm/pblk-core.c
index 76516ee84e9a..0487b9340c1d 100644
--- a/drivers/lightnvm/pblk-core.c
+++ b/drivers/lightnvm/pblk-core.c
@@ -32,8 +32,8 @@ static void pblk_line_mark_bb(struct work_struct *work)
 		struct pblk_line *line;
 		int pos;
 
-		line = &pblk->lines[pblk_dev_ppa_to_line(*ppa)];
-		pos = pblk_dev_ppa_to_pos(&dev->geo, *ppa);
+		line = &pblk->lines[pblk_ppa_to_line(*ppa)];
+		pos = pblk_ppa_to_pos(&dev->geo, *ppa);
 
 		pr_err("pblk: failed to mark bb, line:%d, pos:%d\n",
 				line->id, pos);
@@ -48,7 +48,7 @@ static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line,
 {
 	struct nvm_tgt_dev *dev = pblk->dev;
 	struct nvm_geo *geo = &dev->geo;
-	int pos = pblk_dev_ppa_to_pos(geo, *ppa);
+	int pos = pblk_ppa_to_pos(geo, *ppa);
 
 	pr_debug("pblk: erase failed: line:%d, pos:%d\n", line->id, pos);
 	atomic_long_inc(&pblk->erase_failed);
@@ -66,7 +66,7 @@ static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd)
 {
 	struct pblk_line *line;
 
-	line = &pblk->lines[pblk_dev_ppa_to_line(rqd->ppa_addr)];
+	line = &pblk->lines[pblk_ppa_to_line(rqd->ppa_addr)];
 	atomic_dec(&line->left_seblks);
 
 	if (rqd->error) {
@@ -144,7 +144,7 @@ void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa)
 	BUG_ON(pblk_ppa_empty(ppa));
 #endif
 
-	line_id = pblk_tgt_ppa_to_line(ppa);
+	line_id = pblk_ppa_to_line(ppa);
 	line = &pblk->lines[line_id];
 	paddr = pblk_dev_ppa_to_line_addr(pblk, ppa);
 
@@ -650,7 +650,7 @@ static int pblk_line_submit_emeta_io(struct pblk *pblk, struct pblk_line *line,
 	} else {
 		for (i = 0; i < rqd.nr_ppas; ) {
 			struct ppa_addr ppa = addr_to_gen_ppa(pblk, paddr, id);
-			int pos = pblk_dev_ppa_to_pos(geo, ppa);
+			int pos = pblk_ppa_to_pos(geo, ppa);
 			int read_type = PBLK_READ_RANDOM;
 
 			if (pblk_io_aligned(pblk, rq_ppas))
@@ -668,7 +668,7 @@ static int pblk_line_submit_emeta_io(struct pblk *pblk, struct pblk_line *line,
 				}
 
 				ppa = addr_to_gen_ppa(pblk, paddr, id);
-				pos = pblk_dev_ppa_to_pos(geo, ppa);
+				pos = pblk_ppa_to_pos(geo, ppa);
 			}
 
 			if (pblk_boundary_paddr_checks(pblk, paddr + min)) {
@@ -742,7 +742,7 @@ static int pblk_line_submit_smeta_io(struct pblk *pblk, struct pblk_line *line,
 		cmd_op = NVM_OP_PWRITE;
 		flags = pblk_set_progr_mode(pblk, PBLK_WRITE);
 		lba_list = emeta_to_lbas(pblk, line->emeta->buf);
-	} else if (dir == PBLK_READ) {
+	} else if (dir == PBLK_READ_RECOV || dir == PBLK_READ) {
 		bio_op = REQ_OP_READ;
 		cmd_op = NVM_OP_PREAD;
 		flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL);
@@ -802,7 +802,7 @@ static int pblk_line_submit_smeta_io(struct pblk *pblk, struct pblk_line *line,
 	if (rqd.error) {
 		if (dir == PBLK_WRITE)
 			pblk_log_write_err(pblk, &rqd);
-		else
+		else if (dir == PBLK_READ)
 			pblk_log_read_err(pblk, &rqd);
 	}
 
@@ -816,7 +816,7 @@ int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line)
 {
 	u64 bpaddr = pblk_line_smeta_start(pblk, line);
 
-	return pblk_line_submit_smeta_io(pblk, line, bpaddr, PBLK_READ);
+	return pblk_line_submit_smeta_io(pblk, line, bpaddr, PBLK_READ_RECOV);
 }
 
 int pblk_line_read_emeta(struct pblk *pblk, struct pblk_line *line,
@@ -854,8 +854,8 @@ static int pblk_blk_erase_sync(struct pblk *pblk, struct ppa_addr ppa)
 		struct nvm_geo *geo = &dev->geo;
 
 		pr_err("pblk: could not sync erase line:%d,blk:%d\n",
-					pblk_dev_ppa_to_line(ppa),
-					pblk_dev_ppa_to_pos(geo, ppa));
+					pblk_ppa_to_line(ppa),
+					pblk_ppa_to_pos(geo, ppa));
 
 		rqd.error = ret;
 		goto out;
@@ -979,7 +979,7 @@ static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line,
 
 	/* Start metadata */
 	smeta_buf->seq_nr = cpu_to_le64(line->seq_nr);
-	smeta_buf->window_wr_lun = cpu_to_le32(geo->nr_luns);
+	smeta_buf->window_wr_lun = cpu_to_le32(geo->all_luns);
 
 	/* Fill metadata among lines */
 	if (cur) {
@@ -1032,7 +1032,7 @@ static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line,
 							lm->sec_per_line);
 		bitmap_or(line->map_bitmap, line->map_bitmap, l_mg->bb_aux,
 							lm->sec_per_line);
-		line->sec_in_line -= geo->sec_per_blk;
+		line->sec_in_line -= geo->sec_per_chk;
 		if (bit >= lm->emeta_bb)
 			nr_bb++;
 	}
@@ -1145,7 +1145,7 @@ int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line)
 	}
 	spin_unlock(&l_mg->free_lock);
 
-	pblk_rl_free_lines_dec(&pblk->rl, line);
+	pblk_rl_free_lines_dec(&pblk->rl, line, true);
 
 	if (!pblk_line_init_bb(pblk, line, 0)) {
 		list_add(&line->list, &l_mg->free_list);
@@ -1233,7 +1233,7 @@ static struct pblk_line *pblk_line_retry(struct pblk *pblk,
 	l_mg->data_line = retry_line;
 	spin_unlock(&l_mg->free_lock);
 
-	pblk_rl_free_lines_dec(&pblk->rl, retry_line);
+	pblk_rl_free_lines_dec(&pblk->rl, line, false);
 
 	if (pblk_line_erase(pblk, retry_line))
 		goto retry;
@@ -1252,7 +1252,6 @@ struct pblk_line *pblk_line_get_first_data(struct pblk *pblk)
 {
 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 	struct pblk_line *line;
-	int is_next = 0;
 
 	spin_lock(&l_mg->free_lock);
 	line = pblk_line_get(pblk);
@@ -1280,7 +1279,6 @@ struct pblk_line *pblk_line_get_first_data(struct pblk *pblk)
 	} else {
 		l_mg->data_next->seq_nr = l_mg->d_seq_nr++;
 		l_mg->data_next->type = PBLK_LINETYPE_DATA;
-		is_next = 1;
 	}
 	spin_unlock(&l_mg->free_lock);
 
@@ -1290,10 +1288,6 @@ struct pblk_line *pblk_line_get_first_data(struct pblk *pblk)
 			return NULL;
 	}
 
-	pblk_rl_free_lines_dec(&pblk->rl, line);
-	if (is_next)
-		pblk_rl_free_lines_dec(&pblk->rl, l_mg->data_next);
-
 retry_setup:
 	if (!pblk_line_init_metadata(pblk, line, NULL)) {
 		line = pblk_line_retry(pblk, line);
@@ -1311,6 +1305,8 @@ struct pblk_line *pblk_line_get_first_data(struct pblk *pblk)
 		goto retry_setup;
 	}
 
+	pblk_rl_free_lines_dec(&pblk->rl, line, true);
+
 	return line;
 }
 
@@ -1395,7 +1391,6 @@ struct pblk_line *pblk_line_replace_data(struct pblk *pblk)
 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 	struct pblk_line *cur, *new = NULL;
 	unsigned int left_seblks;
-	int is_next = 0;
 
 	cur = l_mg->data_line;
 	new = l_mg->data_next;
@@ -1444,6 +1439,8 @@ struct pblk_line *pblk_line_replace_data(struct pblk *pblk)
 		goto retry_setup;
 	}
 
+	pblk_rl_free_lines_dec(&pblk->rl, new, true);
+
 	/* Allocate next line for preparation */
 	spin_lock(&l_mg->free_lock);
 	l_mg->data_next = pblk_line_get(pblk);
@@ -1457,13 +1454,9 @@ struct pblk_line *pblk_line_replace_data(struct pblk *pblk)
 	} else {
 		l_mg->data_next->seq_nr = l_mg->d_seq_nr++;
 		l_mg->data_next->type = PBLK_LINETYPE_DATA;
-		is_next = 1;
 	}
 	spin_unlock(&l_mg->free_lock);
 
-	if (is_next)
-		pblk_rl_free_lines_dec(&pblk->rl, l_mg->data_next);
-
 out:
 	return new;
 }
@@ -1561,8 +1554,8 @@ int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr ppa)
 		struct nvm_geo *geo = &dev->geo;
 
 		pr_err("pblk: could not async erase line:%d,blk:%d\n",
-					pblk_dev_ppa_to_line(ppa),
-					pblk_dev_ppa_to_pos(geo, ppa));
+					pblk_ppa_to_line(ppa),
+					pblk_ppa_to_pos(geo, ppa));
 	}
 
 	return err;
@@ -1746,7 +1739,7 @@ void pblk_up_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
 	struct nvm_tgt_dev *dev = pblk->dev;
 	struct nvm_geo *geo = &dev->geo;
 	struct pblk_lun *rlun;
-	int nr_luns = geo->nr_luns;
+	int nr_luns = geo->all_luns;
 	int bit = -1;
 
 	while ((bit = find_next_bit(lun_bitmap, nr_luns, bit + 1)) < nr_luns) {
@@ -1884,7 +1877,7 @@ void pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas,
 
 		/* If the L2P entry maps to a line, the reference is valid */
 		if (!pblk_ppa_empty(ppa) && !pblk_addr_in_cache(ppa)) {
-			int line_id = pblk_dev_ppa_to_line(ppa);
+			int line_id = pblk_ppa_to_line(ppa);
 			struct pblk_line *line = &pblk->lines[line_id];
 
 			kref_get(&line->ref);
diff --git a/drivers/lightnvm/pblk-gc.c b/drivers/lightnvm/pblk-gc.c
index 9c8e114c8a54..3d899383666e 100644
--- a/drivers/lightnvm/pblk-gc.c
+++ b/drivers/lightnvm/pblk-gc.c
@@ -169,7 +169,14 @@ static void pblk_gc_line_prepare_ws(struct work_struct *work)
 	 * the line untouched. TODO: Implement a recovery routine that scans and
 	 * moves all sectors on the line.
 	 */
-	lba_list = pblk_recov_get_lba_list(pblk, emeta_buf);
+
+	ret = pblk_recov_check_emeta(pblk, emeta_buf);
+	if (ret) {
+		pr_err("pblk: inconsistent emeta (line %d)\n", line->id);
+		goto fail_free_emeta;
+	}
+
+	lba_list = emeta_to_lbas(pblk, emeta_buf);
 	if (!lba_list) {
 		pr_err("pblk: could not interpret emeta (line %d)\n", line->id);
 		goto fail_free_emeta;
@@ -519,22 +526,12 @@ void pblk_gc_should_start(struct pblk *pblk)
 	}
 }
 
-/*
- * If flush_wq == 1 then no lock should be held by the caller since
- * flush_workqueue can sleep
- */
-static void pblk_gc_stop(struct pblk *pblk, int flush_wq)
-{
-	pblk->gc.gc_active = 0;
-	pr_debug("pblk: gc stop\n");
-}
-
 void pblk_gc_should_stop(struct pblk *pblk)
 {
 	struct pblk_gc *gc = &pblk->gc;
 
 	if (gc->gc_active && !gc->gc_forced)
-		pblk_gc_stop(pblk, 0);
+		gc->gc_active = 0;
 }
 
 void pblk_gc_should_kick(struct pblk *pblk)
@@ -660,7 +657,7 @@ void pblk_gc_exit(struct pblk *pblk)
 
 	gc->gc_enabled = 0;
 	del_timer_sync(&gc->gc_timer);
-	pblk_gc_stop(pblk, 1);
+	gc->gc_active = 0;
 
 	if (gc->gc_ts)
 		kthread_stop(gc->gc_ts);
diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c
index 695826a06b5d..93d671ca518e 100644
--- a/drivers/lightnvm/pblk-init.c
+++ b/drivers/lightnvm/pblk-init.c
@@ -169,8 +169,8 @@ static int pblk_set_ppaf(struct pblk *pblk)
 	}
 	ppaf.ch_len = power_len;
 
-	power_len = get_count_order(geo->luns_per_chnl);
-	if (1 << power_len != geo->luns_per_chnl) {
+	power_len = get_count_order(geo->nr_luns);
+	if (1 << power_len != geo->nr_luns) {
 		pr_err("pblk: supports only power-of-two LUN config.\n");
 		return -EINVAL;
 	}
@@ -254,7 +254,7 @@ static int pblk_core_init(struct pblk *pblk)
 	struct nvm_geo *geo = &dev->geo;
 
 	pblk->pgs_in_buffer = NVM_MEM_PAGE_WRITE * geo->sec_per_pg *
-						geo->nr_planes * geo->nr_luns;
+						geo->nr_planes * geo->all_luns;
 
 	if (pblk_init_global_caches(pblk))
 		return -ENOMEM;
@@ -270,21 +270,22 @@ static int pblk_core_init(struct pblk *pblk)
 	if (!pblk->gen_ws_pool)
 		goto free_page_bio_pool;
 
-	pblk->rec_pool = mempool_create_slab_pool(geo->nr_luns, pblk_rec_cache);
+	pblk->rec_pool = mempool_create_slab_pool(geo->all_luns,
+							pblk_rec_cache);
 	if (!pblk->rec_pool)
 		goto free_gen_ws_pool;
 
-	pblk->r_rq_pool = mempool_create_slab_pool(geo->nr_luns,
+	pblk->r_rq_pool = mempool_create_slab_pool(geo->all_luns,
 							pblk_g_rq_cache);
 	if (!pblk->r_rq_pool)
 		goto free_rec_pool;
 
-	pblk->e_rq_pool = mempool_create_slab_pool(geo->nr_luns,
+	pblk->e_rq_pool = mempool_create_slab_pool(geo->all_luns,
 							pblk_g_rq_cache);
 	if (!pblk->e_rq_pool)
 		goto free_r_rq_pool;
 
-	pblk->w_rq_pool = mempool_create_slab_pool(geo->nr_luns,
+	pblk->w_rq_pool = mempool_create_slab_pool(geo->all_luns,
 							pblk_w_rq_cache);
 	if (!pblk->w_rq_pool)
 		goto free_e_rq_pool;
@@ -354,6 +355,8 @@ static void pblk_core_free(struct pblk *pblk)
 	mempool_destroy(pblk->e_rq_pool);
 	mempool_destroy(pblk->w_rq_pool);
 
+	pblk_rwb_free(pblk);
+
 	pblk_free_global_caches(pblk);
 }
 
@@ -409,7 +412,7 @@ static int pblk_bb_discovery(struct nvm_tgt_dev *dev, struct pblk_lun *rlun)
 	u8 *blks;
 	int nr_blks, ret;
 
-	nr_blks = geo->blks_per_lun * geo->plane_mode;
+	nr_blks = geo->nr_chks * geo->plane_mode;
 	blks = kmalloc(nr_blks, GFP_KERNEL);
 	if (!blks)
 		return -ENOMEM;
@@ -482,20 +485,21 @@ static int pblk_luns_init(struct pblk *pblk, struct ppa_addr *luns)
 	int i, ret;
 
 	/* TODO: Implement unbalanced LUN support */
-	if (geo->luns_per_chnl < 0) {
+	if (geo->nr_luns < 0) {
 		pr_err("pblk: unbalanced LUN config.\n");
 		return -EINVAL;
 	}
 
-	pblk->luns = kcalloc(geo->nr_luns, sizeof(struct pblk_lun), GFP_KERNEL);
+	pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun),
+								GFP_KERNEL);
 	if (!pblk->luns)
 		return -ENOMEM;
 
-	for (i = 0; i < geo->nr_luns; i++) {
+	for (i = 0; i < geo->all_luns; i++) {
 		/* Stripe across channels */
 		int ch = i % geo->nr_chnls;
 		int lun_raw = i / geo->nr_chnls;
-		int lunid = lun_raw + ch * geo->luns_per_chnl;
+		int lunid = lun_raw + ch * geo->nr_luns;
 
 		rlun = &pblk->luns[i];
 		rlun->bppa = luns[lunid];
@@ -577,22 +581,37 @@ static unsigned int calc_emeta_len(struct pblk *pblk)
 static void pblk_set_provision(struct pblk *pblk, long nr_free_blks)
 {
 	struct nvm_tgt_dev *dev = pblk->dev;
+	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
+	struct pblk_line_meta *lm = &pblk->lm;
 	struct nvm_geo *geo = &dev->geo;
 	sector_t provisioned;
+	int sec_meta, blk_meta;
 
-	pblk->over_pct = 20;
+	if (geo->op == NVM_TARGET_DEFAULT_OP)
+		pblk->op = PBLK_DEFAULT_OP;
+	else
+		pblk->op = geo->op;
 
 	provisioned = nr_free_blks;
-	provisioned *= (100 - pblk->over_pct);
+	provisioned *= (100 - pblk->op);
 	sector_div(provisioned, 100);
 
+	pblk->op_blks = nr_free_blks - provisioned;
+
 	/* Internally pblk manages all free blocks, but all calculations based
 	 * on user capacity consider only provisioned blocks
 	 */
 	pblk->rl.total_blocks = nr_free_blks;
-	pblk->rl.nr_secs = nr_free_blks * geo->sec_per_blk;
-	pblk->capacity = provisioned * geo->sec_per_blk;
+	pblk->rl.nr_secs = nr_free_blks * geo->sec_per_chk;
+
+	/* Consider sectors used for metadata */
+	sec_meta = (lm->smeta_sec + lm->emeta_sec[0]) * l_mg->nr_free_lines;
+	blk_meta = DIV_ROUND_UP(sec_meta, geo->sec_per_chk);
+
+	pblk->capacity = (provisioned - blk_meta) * geo->sec_per_chk;
+
 	atomic_set(&pblk->rl.free_blocks, nr_free_blks);
+	atomic_set(&pblk->rl.free_user_blocks, nr_free_blks);
 }
 
 static int pblk_lines_alloc_metadata(struct pblk *pblk)
@@ -683,7 +702,7 @@ static int pblk_lines_init(struct pblk *pblk)
 	int i, ret;
 
 	pblk->min_write_pgs = geo->sec_per_pl * (geo->sec_size / PAGE_SIZE);
-	max_write_ppas = pblk->min_write_pgs * geo->nr_luns;
+	max_write_ppas = pblk->min_write_pgs * geo->all_luns;
 	pblk->max_write_pgs = (max_write_ppas < nvm_max_phys_sects(dev)) ?
 				max_write_ppas : nvm_max_phys_sects(dev);
 	pblk_set_sec_per_write(pblk, pblk->min_write_pgs);
@@ -693,26 +712,26 @@ static int pblk_lines_init(struct pblk *pblk)
 		return -EINVAL;
 	}
 
-	div_u64_rem(geo->sec_per_blk, pblk->min_write_pgs, &mod);
+	div_u64_rem(geo->sec_per_chk, pblk->min_write_pgs, &mod);
 	if (mod) {
 		pr_err("pblk: bad configuration of sectors/pages\n");
 		return -EINVAL;
 	}
 
-	l_mg->nr_lines = geo->blks_per_lun;
+	l_mg->nr_lines = geo->nr_chks;
 	l_mg->log_line = l_mg->data_line = NULL;
 	l_mg->l_seq_nr = l_mg->d_seq_nr = 0;
 	l_mg->nr_free_lines = 0;
 	bitmap_zero(&l_mg->meta_bitmap, PBLK_DATA_LINES);
 
-	lm->sec_per_line = geo->sec_per_blk * geo->nr_luns;
-	lm->blk_per_line = geo->nr_luns;
-	lm->blk_bitmap_len = BITS_TO_LONGS(geo->nr_luns) * sizeof(long);
+	lm->sec_per_line = geo->sec_per_chk * geo->all_luns;
+	lm->blk_per_line = geo->all_luns;
+	lm->blk_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
 	lm->sec_bitmap_len = BITS_TO_LONGS(lm->sec_per_line) * sizeof(long);
-	lm->lun_bitmap_len = BITS_TO_LONGS(geo->nr_luns) * sizeof(long);
+	lm->lun_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
 	lm->mid_thrs = lm->sec_per_line / 2;
 	lm->high_thrs = lm->sec_per_line / 4;
-	lm->meta_distance = (geo->nr_luns / 2) * pblk->min_write_pgs;
+	lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs;
 
 	/* Calculate necessary pages for smeta. See comment over struct
 	 * line_smeta definition
@@ -742,12 +761,12 @@ static int pblk_lines_init(struct pblk *pblk)
 		goto add_emeta_page;
 	}
 
-	lm->emeta_bb = geo->nr_luns > i ? geo->nr_luns - i : 0;
+	lm->emeta_bb = geo->all_luns > i ? geo->all_luns - i : 0;
 
 	lm->min_blk_line = 1;
-	if (geo->nr_luns > 1)
+	if (geo->all_luns > 1)
 		lm->min_blk_line += DIV_ROUND_UP(lm->smeta_sec +
-					lm->emeta_sec[0], geo->sec_per_blk);
+					lm->emeta_sec[0], geo->sec_per_chk);
 
 	if (lm->min_blk_line > lm->blk_per_line) {
 		pr_err("pblk: config. not supported. Min. LUN in line:%d\n",
@@ -772,7 +791,7 @@ static int pblk_lines_init(struct pblk *pblk)
 		goto fail_free_bb_template;
 	}
 
-	bb_distance = (geo->nr_luns) * geo->sec_per_pl;
+	bb_distance = (geo->all_luns) * geo->sec_per_pl;
 	for (i = 0; i < lm->sec_per_line; i += bb_distance)
 		bitmap_set(l_mg->bb_template, i, geo->sec_per_pl);
 
@@ -844,7 +863,7 @@ static int pblk_lines_init(struct pblk *pblk)
 	pblk_set_provision(pblk, nr_free_blks);
 
 	/* Cleanup per-LUN bad block lists - managed within lines on run-time */
-	for (i = 0; i < geo->nr_luns; i++)
+	for (i = 0; i < geo->all_luns; i++)
 		kfree(pblk->luns[i].bb_list);
 
 	return 0;
@@ -858,7 +877,7 @@ static int pblk_lines_init(struct pblk *pblk)
 fail_free_meta:
 	pblk_line_meta_free(pblk);
 fail:
-	for (i = 0; i < geo->nr_luns; i++)
+	for (i = 0; i < geo->all_luns; i++)
 		kfree(pblk->luns[i].bb_list);
 
 	return ret;
@@ -866,15 +885,19 @@ static int pblk_lines_init(struct pblk *pblk)
 
 static int pblk_writer_init(struct pblk *pblk)
 {
-	timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0);
-	mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100));
-
 	pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t");
 	if (IS_ERR(pblk->writer_ts)) {
-		pr_err("pblk: could not allocate writer kthread\n");
-		return PTR_ERR(pblk->writer_ts);
+		int err = PTR_ERR(pblk->writer_ts);
+
+		if (err != -EINTR)
+			pr_err("pblk: could not allocate writer kthread (%d)\n",
+					err);
+		return err;
 	}
 
+	timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0);
+	mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100));
+
 	return 0;
 }
 
@@ -910,7 +933,6 @@ static void pblk_tear_down(struct pblk *pblk)
 	pblk_pipeline_stop(pblk);
 	pblk_writer_stop(pblk);
 	pblk_rb_sync_l2p(&pblk->rwb);
-	pblk_rwb_free(pblk);
 	pblk_rl_free(&pblk->rl);
 
 	pr_debug("pblk: consistent tear down\n");
@@ -1025,7 +1047,8 @@ static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
 
 	ret = pblk_writer_init(pblk);
 	if (ret) {
-		pr_err("pblk: could not initialize write thread\n");
+		if (ret != -EINTR)
+			pr_err("pblk: could not initialize write thread\n");
 		goto fail_free_lines;
 	}
 
@@ -1041,13 +1064,14 @@ static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
 
 	blk_queue_write_cache(tqueue, true, false);
 
-	tqueue->limits.discard_granularity = geo->pgs_per_blk * geo->pfpg_size;
+	tqueue->limits.discard_granularity = geo->sec_per_chk * geo->sec_size;
 	tqueue->limits.discard_alignment = 0;
 	blk_queue_max_discard_sectors(tqueue, UINT_MAX >> 9);
 	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, tqueue);
 
-	pr_info("pblk init: luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
-			geo->nr_luns, pblk->l_mg.nr_lines,
+	pr_info("pblk(%s): luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
+			tdisk->disk_name,
+			geo->all_luns, pblk->l_mg.nr_lines,
 			(unsigned long long)pblk->rl.nr_secs,
 			pblk->rwb.nr_entries);
 
diff --git a/drivers/lightnvm/pblk-map.c b/drivers/lightnvm/pblk-map.c
index 6f3ecde2140f..7445e6430c52 100644
--- a/drivers/lightnvm/pblk-map.c
+++ b/drivers/lightnvm/pblk-map.c
@@ -146,7 +146,7 @@ void pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd,
 		return;
 
 	/* Erase blocks that are bad in this line but might not be in next */
-	if (unlikely(ppa_empty(*erase_ppa)) &&
+	if (unlikely(pblk_ppa_empty(*erase_ppa)) &&
 			bitmap_weight(d_line->blk_bitmap, lm->blk_per_line)) {
 		int bit = -1;
 
diff --git a/drivers/lightnvm/pblk-rb.c b/drivers/lightnvm/pblk-rb.c
index b8f78e401482..ec8fc314646b 100644
--- a/drivers/lightnvm/pblk-rb.c
+++ b/drivers/lightnvm/pblk-rb.c
@@ -54,7 +54,7 @@ int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base,
 	rb->seg_size = (1 << power_seg_sz);
 	rb->nr_entries = (1 << power_size);
 	rb->mem = rb->subm = rb->sync = rb->l2p_update = 0;
-	rb->sync_point = EMPTY_ENTRY;
+	rb->flush_point = EMPTY_ENTRY;
 
 	spin_lock_init(&rb->w_lock);
 	spin_lock_init(&rb->s_lock);
@@ -112,7 +112,7 @@ int pblk_rb_init(struct pblk_rb *rb, struct pblk_rb_entry *rb_entry_base,
 	up_write(&pblk_rb_lock);
 
 #ifdef CONFIG_NVM_DEBUG
-	atomic_set(&rb->inflight_sync_point, 0);
+	atomic_set(&rb->inflight_flush_point, 0);
 #endif
 
 	/*
@@ -226,7 +226,7 @@ static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update)
 		pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa,
 							entry->cacheline);
 
-		line = &pblk->lines[pblk_tgt_ppa_to_line(w_ctx->ppa)];
+		line = &pblk->lines[pblk_ppa_to_line(w_ctx->ppa)];
 		kref_put(&line->ref, pblk_line_put);
 		clean_wctx(w_ctx);
 		rb->l2p_update = (rb->l2p_update + 1) & (rb->nr_entries - 1);
@@ -349,35 +349,35 @@ void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data,
 	smp_store_release(&entry->w_ctx.flags, flags);
 }
 
-static int pblk_rb_sync_point_set(struct pblk_rb *rb, struct bio *bio,
+static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio,
 				  unsigned int pos)
 {
 	struct pblk_rb_entry *entry;
-	unsigned int subm, sync_point;
+	unsigned int sync, flush_point;
 
-	subm = READ_ONCE(rb->subm);
+	sync = READ_ONCE(rb->sync);
+
+	if (pos == sync)
+		return 0;
 
 #ifdef CONFIG_NVM_DEBUG
-	atomic_inc(&rb->inflight_sync_point);
+	atomic_inc(&rb->inflight_flush_point);
 #endif
 
-	if (pos == subm)
-		return 0;
+	flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
+	entry = &rb->entries[flush_point];
 
-	sync_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
-	entry = &rb->entries[sync_point];
+	pblk_rb_sync_init(rb, NULL);
 
-	/* Protect syncs */
-	smp_store_release(&rb->sync_point, sync_point);
+	/* Protect flush points */
+	smp_store_release(&rb->flush_point, flush_point);
 
-	if (!bio)
-		return 0;
+	if (bio)
+		bio_list_add(&entry->w_ctx.bios, bio);
 
-	spin_lock_irq(&rb->s_lock);
-	bio_list_add(&entry->w_ctx.bios, bio);
-	spin_unlock_irq(&rb->s_lock);
+	pblk_rb_sync_end(rb, NULL);
 
-	return 1;
+	return bio ? 1 : 0;
 }
 
 static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
@@ -416,7 +416,7 @@ void pblk_rb_flush(struct pblk_rb *rb)
 	struct pblk *pblk = container_of(rb, struct pblk, rwb);
 	unsigned int mem = READ_ONCE(rb->mem);
 
-	if (pblk_rb_sync_point_set(rb, NULL, mem))
+	if (pblk_rb_flush_point_set(rb, NULL, mem))
 		return;
 
 	pblk_write_should_kick(pblk);
@@ -440,7 +440,7 @@ static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries,
 #ifdef CONFIG_NVM_DEBUG
 		atomic_long_inc(&pblk->nr_flush);
 #endif
-		if (pblk_rb_sync_point_set(&pblk->rwb, bio, mem))
+		if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem))
 			*io_ret = NVM_IO_OK;
 	}
 
@@ -606,21 +606,6 @@ unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd,
 			return NVM_IO_ERR;
 		}
 
-		if (flags & PBLK_FLUSH_ENTRY) {
-			unsigned int sync_point;
-
-			sync_point = READ_ONCE(rb->sync_point);
-			if (sync_point == pos) {
-				/* Protect syncs */
-				smp_store_release(&rb->sync_point, EMPTY_ENTRY);
-			}
-
-			flags &= ~PBLK_FLUSH_ENTRY;
-#ifdef CONFIG_NVM_DEBUG
-			atomic_dec(&rb->inflight_sync_point);
-#endif
-		}
-
 		flags &= ~PBLK_WRITTEN_DATA;
 		flags |= PBLK_SUBMITTED_ENTRY;
 
@@ -730,15 +715,24 @@ void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags)
 
 unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
 {
-	unsigned int sync;
-	unsigned int i;
-
+	unsigned int sync, flush_point;
 	lockdep_assert_held(&rb->s_lock);
 
 	sync = READ_ONCE(rb->sync);
+	flush_point = READ_ONCE(rb->flush_point);
 
-	for (i = 0; i < nr_entries; i++)
-		sync = (sync + 1) & (rb->nr_entries - 1);
+	if (flush_point != EMPTY_ENTRY) {
+		unsigned int secs_to_flush;
+
+		secs_to_flush = pblk_rb_ring_count(flush_point, sync,
+					rb->nr_entries);
+		if (secs_to_flush < nr_entries) {
+			/* Protect flush points */
+			smp_store_release(&rb->flush_point, EMPTY_ENTRY);
+		}
+	}
+
+	sync = (sync + nr_entries) & (rb->nr_entries - 1);
 
 	/* Protect from counts */
 	smp_store_release(&rb->sync, sync);
@@ -746,22 +740,27 @@ unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
 	return sync;
 }
 
-unsigned int pblk_rb_sync_point_count(struct pblk_rb *rb)
+/* Calculate how many sectors to submit up to the current flush point. */
+unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb)
 {
-	unsigned int subm, sync_point;
-	unsigned int count;
+	unsigned int subm, sync, flush_point;
+	unsigned int submitted, to_flush;
 
-	/* Protect syncs */
-	sync_point = smp_load_acquire(&rb->sync_point);
-	if (sync_point == EMPTY_ENTRY)
+	/* Protect flush points */
+	flush_point = smp_load_acquire(&rb->flush_point);
+	if (flush_point == EMPTY_ENTRY)
 		return 0;
 
+	/* Protect syncs */
+	sync = smp_load_acquire(&rb->sync);
+
 	subm = READ_ONCE(rb->subm);
+	submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries);
 
 	/* The sync point itself counts as a sector to sync */
-	count = pblk_rb_ring_count(sync_point, subm, rb->nr_entries) + 1;
+	to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1;
 
-	return count;
+	return (submitted < to_flush) ? (to_flush - submitted) : 0;
 }
 
 /*
@@ -801,7 +800,7 @@ int pblk_rb_tear_down_check(struct pblk_rb *rb)
 
 	if ((rb->mem == rb->subm) && (rb->subm == rb->sync) &&
 				(rb->sync == rb->l2p_update) &&
-				(rb->sync_point == EMPTY_ENTRY)) {
+				(rb->flush_point == EMPTY_ENTRY)) {
 		goto out;
 	}
 
@@ -848,7 +847,7 @@ ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
 		queued_entries++;
 	spin_unlock_irq(&rb->s_lock);
 
-	if (rb->sync_point != EMPTY_ENTRY)
+	if (rb->flush_point != EMPTY_ENTRY)
 		offset = scnprintf(buf, PAGE_SIZE,
 			"%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n",
 			rb->nr_entries,
@@ -857,14 +856,14 @@ ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
 			rb->sync,
 			rb->l2p_update,
 #ifdef CONFIG_NVM_DEBUG
-			atomic_read(&rb->inflight_sync_point),
+			atomic_read(&rb->inflight_flush_point),
 #else
 			0,
 #endif
-			rb->sync_point,
+			rb->flush_point,
 			pblk_rb_read_count(rb),
 			pblk_rb_space(rb),
-			pblk_rb_sync_point_count(rb),
+			pblk_rb_flush_point_count(rb),
 			queued_entries);
 	else
 		offset = scnprintf(buf, PAGE_SIZE,
@@ -875,13 +874,13 @@ ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
 			rb->sync,
 			rb->l2p_update,
 #ifdef CONFIG_NVM_DEBUG
-			atomic_read(&rb->inflight_sync_point),
+			atomic_read(&rb->inflight_flush_point),
 #else
 			0,
 #endif
 			pblk_rb_read_count(rb),
 			pblk_rb_space(rb),
-			pblk_rb_sync_point_count(rb),
+			pblk_rb_flush_point_count(rb),
 			queued_entries);
 
 	return offset;
diff --git a/drivers/lightnvm/pblk-read.c b/drivers/lightnvm/pblk-read.c
index ca79d8fb3e60..2f761283f43e 100644
--- a/drivers/lightnvm/pblk-read.c
+++ b/drivers/lightnvm/pblk-read.c
@@ -141,7 +141,7 @@ static void pblk_read_put_rqd_kref(struct pblk *pblk, struct nvm_rq *rqd)
 		struct ppa_addr ppa = ppa_list[i];
 		struct pblk_line *line;
 
-		line = &pblk->lines[pblk_dev_ppa_to_line(ppa)];
+		line = &pblk->lines[pblk_ppa_to_line(ppa)];
 		kref_put(&line->ref, pblk_line_put_wq);
 	}
 }
@@ -158,8 +158,12 @@ static void pblk_end_user_read(struct bio *bio)
 static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd,
 			       bool put_line)
 {
+	struct nvm_tgt_dev *dev = pblk->dev;
 	struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
 	struct bio *bio = rqd->bio;
+	unsigned long start_time = r_ctx->start_time;
+
+	generic_end_io_acct(dev->q, READ, &pblk->disk->part0, start_time);
 
 	if (rqd->error)
 		pblk_log_read_err(pblk, rqd);
@@ -193,9 +197,9 @@ static void pblk_end_io_read(struct nvm_rq *rqd)
 	__pblk_end_io_read(pblk, rqd, true);
 }
 
-static int pblk_fill_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd,
-				      unsigned int bio_init_idx,
-				      unsigned long *read_bitmap)
+static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd,
+				 unsigned int bio_init_idx,
+				 unsigned long *read_bitmap)
 {
 	struct bio *new_bio, *bio = rqd->bio;
 	struct pblk_sec_meta *meta_list = rqd->meta_list;
@@ -270,7 +274,7 @@ static int pblk_fill_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd,
 	i = 0;
 	hole = find_first_zero_bit(read_bitmap, nr_secs);
 	do {
-		int line_id = pblk_dev_ppa_to_line(rqd->ppa_list[i]);
+		int line_id = pblk_ppa_to_line(rqd->ppa_list[i]);
 		struct pblk_line *line = &pblk->lines[line_id];
 
 		kref_put(&line->ref, pblk_line_put);
@@ -306,6 +310,8 @@ static int pblk_fill_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd,
 	return NVM_IO_OK;
 
 err:
+	pr_err("pblk: failed to perform partial read\n");
+
 	/* Free allocated pages in new bio */
 	pblk_bio_free_pages(pblk, bio, 0, new_bio->bi_vcnt);
 	__pblk_end_io_read(pblk, rqd, false);
@@ -357,6 +363,7 @@ static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd,
 int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 {
 	struct nvm_tgt_dev *dev = pblk->dev;
+	struct request_queue *q = dev->q;
 	sector_t blba = pblk_get_lba(bio);
 	unsigned int nr_secs = pblk_get_secs(bio);
 	struct pblk_g_ctx *r_ctx;
@@ -372,6 +379,8 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 		return NVM_IO_ERR;
 	}
 
+	generic_start_io_acct(q, READ, bio_sectors(bio), &pblk->disk->part0);
+
 	bitmap_zero(&read_bitmap, nr_secs);
 
 	rqd = pblk_alloc_rqd(pblk, PBLK_READ);
@@ -383,6 +392,7 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 	rqd->end_io = pblk_end_io_read;
 
 	r_ctx = nvm_rq_to_pdu(rqd);
+	r_ctx->start_time = jiffies;
 	r_ctx->lba = blba;
 
 	/* Save the index for this bio's start. This is needed in case
@@ -422,7 +432,7 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 		int_bio = bio_clone_fast(bio, GFP_KERNEL, pblk_bio_set);
 		if (!int_bio) {
 			pr_err("pblk: could not clone read bio\n");
-			return NVM_IO_ERR;
+			goto fail_end_io;
 		}
 
 		rqd->bio = int_bio;
@@ -433,7 +443,7 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 			pr_err("pblk: read IO submission failed\n");
 			if (int_bio)
 				bio_put(int_bio);
-			return ret;
+			goto fail_end_io;
 		}
 
 		return NVM_IO_OK;
@@ -442,17 +452,14 @@ int pblk_submit_read(struct pblk *pblk, struct bio *bio)
 	/* The read bio request could be partially filled by the write buffer,
 	 * but there are some holes that need to be read from the drive.
 	 */
-	ret = pblk_fill_partial_read_bio(pblk, rqd, bio_init_idx, &read_bitmap);
-	if (ret) {
-		pr_err("pblk: failed to perform partial read\n");
-		return ret;
-	}
-
-	return NVM_IO_OK;
+	return pblk_partial_read_bio(pblk, rqd, bio_init_idx, &read_bitmap);
 
 fail_rqd_free:
 	pblk_free_rqd(pblk, rqd, PBLK_READ);
 	return ret;
+fail_end_io:
+	__pblk_end_io_read(pblk, rqd, false);
+	return ret;
 }
 
 static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
diff --git a/drivers/lightnvm/pblk-recovery.c b/drivers/lightnvm/pblk-recovery.c
index eadb3eb5d4dc..1d5e961bf5e0 100644
--- a/drivers/lightnvm/pblk-recovery.c
+++ b/drivers/lightnvm/pblk-recovery.c
@@ -111,18 +111,18 @@ int pblk_recov_setup_rq(struct pblk *pblk, struct pblk_c_ctx *c_ctx,
 	return 0;
 }
 
-__le64 *pblk_recov_get_lba_list(struct pblk *pblk, struct line_emeta *emeta_buf)
+int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta_buf)
 {
 	u32 crc;
 
 	crc = pblk_calc_emeta_crc(pblk, emeta_buf);
 	if (le32_to_cpu(emeta_buf->crc) != crc)
-		return NULL;
+		return 1;
 
 	if (le32_to_cpu(emeta_buf->header.identifier) != PBLK_MAGIC)
-		return NULL;
+		return 1;
 
-	return emeta_to_lbas(pblk, emeta_buf);
+	return 0;
 }
 
 static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line)
@@ -137,7 +137,7 @@ static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line)
 	u64 nr_valid_lbas, nr_lbas = 0;
 	u64 i;
 
-	lba_list = pblk_recov_get_lba_list(pblk, emeta_buf);
+	lba_list = emeta_to_lbas(pblk, emeta_buf);
 	if (!lba_list)
 		return 1;
 
@@ -149,7 +149,7 @@ static int pblk_recov_l2p_from_emeta(struct pblk *pblk, struct pblk_line *line)
 		struct ppa_addr ppa;
 		int pos;
 
-		ppa = addr_to_pblk_ppa(pblk, i, line->id);
+		ppa = addr_to_gen_ppa(pblk, i, line->id);
 		pos = pblk_ppa_to_pos(geo, ppa);
 
 		/* Do not update bad blocks */
@@ -188,7 +188,7 @@ static int pblk_calc_sec_in_line(struct pblk *pblk, struct pblk_line *line)
 	int nr_bb = bitmap_weight(line->blk_bitmap, lm->blk_per_line);
 
 	return lm->sec_per_line - lm->smeta_sec - lm->emeta_sec[0] -
-				nr_bb * geo->sec_per_blk;
+				nr_bb * geo->sec_per_chk;
 }
 
 struct pblk_recov_alloc {
@@ -263,12 +263,12 @@ static int pblk_recov_read_oob(struct pblk *pblk, struct pblk_line *line,
 		int pos;
 
 		ppa = addr_to_gen_ppa(pblk, r_ptr_int, line->id);
-		pos = pblk_dev_ppa_to_pos(geo, ppa);
+		pos = pblk_ppa_to_pos(geo, ppa);
 
 		while (test_bit(pos, line->blk_bitmap)) {
 			r_ptr_int += pblk->min_write_pgs;
 			ppa = addr_to_gen_ppa(pblk, r_ptr_int, line->id);
-			pos = pblk_dev_ppa_to_pos(geo, ppa);
+			pos = pblk_ppa_to_pos(geo, ppa);
 		}
 
 		for (j = 0; j < pblk->min_write_pgs; j++, i++, r_ptr_int++)
@@ -288,7 +288,7 @@ static int pblk_recov_read_oob(struct pblk *pblk, struct pblk_line *line,
 	/* At this point, the read should not fail. If it does, it is a problem
 	 * we cannot recover from here. Need FTL log.
 	 */
-	if (rqd->error) {
+	if (rqd->error && rqd->error != NVM_RSP_WARN_HIGHECC) {
 		pr_err("pblk: L2P recovery failed (%d)\n", rqd->error);
 		return -EINTR;
 	}
@@ -411,12 +411,12 @@ static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
 		int pos;
 
 		w_ptr = pblk_alloc_page(pblk, line, pblk->min_write_pgs);
-		ppa = addr_to_pblk_ppa(pblk, w_ptr, line->id);
+		ppa = addr_to_gen_ppa(pblk, w_ptr, line->id);
 		pos = pblk_ppa_to_pos(geo, ppa);
 
 		while (test_bit(pos, line->blk_bitmap)) {
 			w_ptr += pblk->min_write_pgs;
-			ppa = addr_to_pblk_ppa(pblk, w_ptr, line->id);
+			ppa = addr_to_gen_ppa(pblk, w_ptr, line->id);
 			pos = pblk_ppa_to_pos(geo, ppa);
 		}
 
@@ -541,12 +541,12 @@ static int pblk_recov_scan_all_oob(struct pblk *pblk, struct pblk_line *line,
 
 		w_ptr = pblk_alloc_page(pblk, line, pblk->min_write_pgs);
 		ppa = addr_to_gen_ppa(pblk, w_ptr, line->id);
-		pos = pblk_dev_ppa_to_pos(geo, ppa);
+		pos = pblk_ppa_to_pos(geo, ppa);
 
 		while (test_bit(pos, line->blk_bitmap)) {
 			w_ptr += pblk->min_write_pgs;
 			ppa = addr_to_gen_ppa(pblk, w_ptr, line->id);
-			pos = pblk_dev_ppa_to_pos(geo, ppa);
+			pos = pblk_ppa_to_pos(geo, ppa);
 		}
 
 		for (j = 0; j < pblk->min_write_pgs; j++, i++, w_ptr++)
@@ -672,12 +672,12 @@ static int pblk_recov_scan_oob(struct pblk *pblk, struct pblk_line *line,
 
 		paddr = pblk_alloc_page(pblk, line, pblk->min_write_pgs);
 		ppa = addr_to_gen_ppa(pblk, paddr, line->id);
-		pos = pblk_dev_ppa_to_pos(geo, ppa);
+		pos = pblk_ppa_to_pos(geo, ppa);
 
 		while (test_bit(pos, line->blk_bitmap)) {
 			paddr += pblk->min_write_pgs;
 			ppa = addr_to_gen_ppa(pblk, paddr, line->id);
-			pos = pblk_dev_ppa_to_pos(geo, ppa);
+			pos = pblk_ppa_to_pos(geo, ppa);
 		}
 
 		for (j = 0; j < pblk->min_write_pgs; j++, i++, paddr++)
@@ -817,7 +817,7 @@ static u64 pblk_line_emeta_start(struct pblk *pblk, struct pblk_line *line)
 
 	while (emeta_secs) {
 		emeta_start--;
-		ppa = addr_to_pblk_ppa(pblk, emeta_start, line->id);
+		ppa = addr_to_gen_ppa(pblk, emeta_start, line->id);
 		pos = pblk_ppa_to_pos(geo, ppa);
 		if (!test_bit(pos, line->blk_bitmap))
 			emeta_secs--;
@@ -938,6 +938,11 @@ struct pblk_line *pblk_recov_l2p(struct pblk *pblk)
 			goto next;
 		}
 
+		if (pblk_recov_check_emeta(pblk, line->emeta->buf)) {
+			pblk_recov_l2p_from_oob(pblk, line);
+			goto next;
+		}
+
 		if (pblk_recov_l2p_from_emeta(pblk, line))
 			pblk_recov_l2p_from_oob(pblk, line);
 
@@ -984,10 +989,8 @@ struct pblk_line *pblk_recov_l2p(struct pblk *pblk)
 	}
 	spin_unlock(&l_mg->free_lock);
 
-	if (is_next) {
+	if (is_next)
 		pblk_line_erase(pblk, l_mg->data_next);
-		pblk_rl_free_lines_dec(&pblk->rl, l_mg->data_next);
-	}
 
 out:
 	if (found_lines != recovered_lines)
diff --git a/drivers/lightnvm/pblk-rl.c b/drivers/lightnvm/pblk-rl.c
index dacc71922260..0d457b162f23 100644
--- a/drivers/lightnvm/pblk-rl.c
+++ b/drivers/lightnvm/pblk-rl.c
@@ -89,17 +89,15 @@ unsigned long pblk_rl_nr_free_blks(struct pblk_rl *rl)
 	return atomic_read(&rl->free_blocks);
 }
 
-/*
- * We check for (i) the number of free blocks in the current LUN and (ii) the
- * total number of free blocks in the pblk instance. This is to even out the
- * number of free blocks on each LUN when GC kicks in.
- *
- * Only the total number of free blocks is used to configure the rate limiter.
- */
-void pblk_rl_update_rates(struct pblk_rl *rl)
+unsigned long pblk_rl_nr_user_free_blks(struct pblk_rl *rl)
+{
+	return atomic_read(&rl->free_user_blocks);
+}
+
+static void __pblk_rl_update_rates(struct pblk_rl *rl,
+				   unsigned long free_blocks)
 {
 	struct pblk *pblk = container_of(rl, struct pblk, rl);
-	unsigned long free_blocks = pblk_rl_nr_free_blks(rl);
 	int max = rl->rb_budget;
 
 	if (free_blocks >= rl->high) {
@@ -132,20 +130,37 @@ void pblk_rl_update_rates(struct pblk_rl *rl)
 		pblk_gc_should_stop(pblk);
 }
 
+void pblk_rl_update_rates(struct pblk_rl *rl)
+{
+	__pblk_rl_update_rates(rl, pblk_rl_nr_user_free_blks(rl));
+}
+
 void pblk_rl_free_lines_inc(struct pblk_rl *rl, struct pblk_line *line)
 {
 	int blk_in_line = atomic_read(&line->blk_in_line);
+	int free_blocks;
 
 	atomic_add(blk_in_line, &rl->free_blocks);
-	pblk_rl_update_rates(rl);
+	free_blocks = atomic_add_return(blk_in_line, &rl->free_user_blocks);
+
+	__pblk_rl_update_rates(rl, free_blocks);
 }
 
-void pblk_rl_free_lines_dec(struct pblk_rl *rl, struct pblk_line *line)
+void pblk_rl_free_lines_dec(struct pblk_rl *rl, struct pblk_line *line,
+			    bool used)
 {
 	int blk_in_line = atomic_read(&line->blk_in_line);
+	int free_blocks;
 
 	atomic_sub(blk_in_line, &rl->free_blocks);
-	pblk_rl_update_rates(rl);
+
+	if (used)
+		free_blocks = atomic_sub_return(blk_in_line,
+							&rl->free_user_blocks);
+	else
+		free_blocks = atomic_read(&rl->free_user_blocks);
+
+	__pblk_rl_update_rates(rl, free_blocks);
 }
 
 int pblk_rl_high_thrs(struct pblk_rl *rl)
@@ -174,16 +189,21 @@ void pblk_rl_free(struct pblk_rl *rl)
 void pblk_rl_init(struct pblk_rl *rl, int budget)
 {
 	struct pblk *pblk = container_of(rl, struct pblk, rl);
+	struct nvm_tgt_dev *dev = pblk->dev;
+	struct nvm_geo *geo = &dev->geo;
+	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
 	struct pblk_line_meta *lm = &pblk->lm;
 	int min_blocks = lm->blk_per_line * PBLK_GC_RSV_LINE;
+	int sec_meta, blk_meta;
+
 	unsigned int rb_windows;
 
-	rl->high = rl->total_blocks / PBLK_USER_HIGH_THRS;
-	rl->high_pw = get_count_order(rl->high);
+	/* Consider sectors used for metadata */
+	sec_meta = (lm->smeta_sec + lm->emeta_sec[0]) * l_mg->nr_free_lines;
+	blk_meta = DIV_ROUND_UP(sec_meta, geo->sec_per_chk);
 
-	rl->low = rl->total_blocks / PBLK_USER_LOW_THRS;
-	if (rl->low < min_blocks)
-		rl->low = min_blocks;
+	rl->high = pblk->op_blks - blk_meta - lm->blk_per_line;
+	rl->high_pw = get_count_order(rl->high);
 
 	rl->rsv_blocks = min_blocks;
 
diff --git a/drivers/lightnvm/pblk-sysfs.c b/drivers/lightnvm/pblk-sysfs.c
index cd49e8875d4e..620bab853579 100644
--- a/drivers/lightnvm/pblk-sysfs.c
+++ b/drivers/lightnvm/pblk-sysfs.c
@@ -28,7 +28,7 @@ static ssize_t pblk_sysfs_luns_show(struct pblk *pblk, char *page)
 	ssize_t sz = 0;
 	int i;
 
-	for (i = 0; i < geo->nr_luns; i++) {
+	for (i = 0; i < geo->all_luns; i++) {
 		int active = 1;
 
 		rlun = &pblk->luns[i];
@@ -49,11 +49,12 @@ static ssize_t pblk_sysfs_luns_show(struct pblk *pblk, char *page)
 
 static ssize_t pblk_sysfs_rate_limiter(struct pblk *pblk, char *page)
 {
-	int free_blocks, total_blocks;
+	int free_blocks, free_user_blocks, total_blocks;
 	int rb_user_max, rb_user_cnt;
 	int rb_gc_max, rb_gc_cnt, rb_budget, rb_state;
 
-	free_blocks = atomic_read(&pblk->rl.free_blocks);
+	free_blocks = pblk_rl_nr_free_blks(&pblk->rl);
+	free_user_blocks = pblk_rl_nr_user_free_blks(&pblk->rl);
 	rb_user_max = pblk->rl.rb_user_max;
 	rb_user_cnt = atomic_read(&pblk->rl.rb_user_cnt);
 	rb_gc_max = pblk->rl.rb_gc_max;
@@ -64,16 +65,16 @@ static ssize_t pblk_sysfs_rate_limiter(struct pblk *pblk, char *page)
 	total_blocks = pblk->rl.total_blocks;
 
 	return snprintf(page, PAGE_SIZE,
-		"u:%u/%u,gc:%u/%u(%u/%u)(stop:<%u,full:>%u,free:%d/%d)-%d\n",
+		"u:%u/%u,gc:%u/%u(%u)(stop:<%u,full:>%u,free:%d/%d/%d)-%d\n",
 				rb_user_cnt,
 				rb_user_max,
 				rb_gc_cnt,
 				rb_gc_max,
 				rb_state,
 				rb_budget,
-				pblk->rl.low,
 				pblk->rl.high,
 				free_blocks,
+				free_user_blocks,
 				total_blocks,
 				READ_ONCE(pblk->rl.rb_user_active));
 }
@@ -238,7 +239,7 @@ static ssize_t pblk_sysfs_lines(struct pblk *pblk, char *page)
 
 	sz = snprintf(page, PAGE_SIZE - sz,
 		"line: nluns:%d, nblks:%d, nsecs:%d\n",
-		geo->nr_luns, lm->blk_per_line, lm->sec_per_line);
+		geo->all_luns, lm->blk_per_line, lm->sec_per_line);
 
 	sz += snprintf(page + sz, PAGE_SIZE - sz,
 		"lines:d:%d,l:%d-f:%d,m:%d/%d,c:%d,b:%d,co:%d(d:%d,l:%d)t:%d\n",
@@ -287,7 +288,7 @@ static ssize_t pblk_sysfs_lines_info(struct pblk *pblk, char *page)
 				"blk_line:%d, sec_line:%d, sec_blk:%d\n",
 					lm->blk_per_line,
 					lm->sec_per_line,
-					geo->sec_per_blk);
+					geo->sec_per_chk);
 
 	return sz;
 }
diff --git a/drivers/lightnvm/pblk-write.c b/drivers/lightnvm/pblk-write.c
index 6c1cafafef53..aae86ed60b98 100644
--- a/drivers/lightnvm/pblk-write.c
+++ b/drivers/lightnvm/pblk-write.c
@@ -21,13 +21,28 @@ static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,
 				    struct pblk_c_ctx *c_ctx)
 {
 	struct bio *original_bio;
+	struct pblk_rb *rwb = &pblk->rwb;
 	unsigned long ret;
 	int i;
 
 	for (i = 0; i < c_ctx->nr_valid; i++) {
 		struct pblk_w_ctx *w_ctx;
+		int pos = c_ctx->sentry + i;
+		int flags;
+
+		w_ctx = pblk_rb_w_ctx(rwb, pos);
+		flags = READ_ONCE(w_ctx->flags);
+
+		if (flags & PBLK_FLUSH_ENTRY) {
+			flags &= ~PBLK_FLUSH_ENTRY;
+			/* Release flags on context. Protect from writes */
+			smp_store_release(&w_ctx->flags, flags);
+
+#ifdef CONFIG_NVM_DEBUG
+			atomic_dec(&rwb->inflight_flush_point);
+#endif
+		}
 
-		w_ctx = pblk_rb_w_ctx(&pblk->rwb, c_ctx->sentry + i);
 		while ((original_bio = bio_list_pop(&w_ctx->bios)))
 			bio_endio(original_bio);
 	}
@@ -439,7 +454,7 @@ static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd)
 	struct pblk_line *meta_line;
 	int err;
 
-	ppa_set_empty(&erase_ppa);
+	pblk_ppa_set_empty(&erase_ppa);
 
 	/* Assign lbas to ppas and populate request structure */
 	err = pblk_setup_w_rq(pblk, rqd, &erase_ppa);
@@ -457,7 +472,7 @@ static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd)
 		return NVM_IO_ERR;
 	}
 
-	if (!ppa_empty(erase_ppa)) {
+	if (!pblk_ppa_empty(erase_ppa)) {
 		/* Submit erase for next data line */
 		if (pblk_blk_erase_async(pblk, erase_ppa)) {
 			struct pblk_line *e_line = pblk_line_get_erase(pblk);
@@ -508,7 +523,7 @@ static int pblk_submit_write(struct pblk *pblk)
 	if (!secs_avail)
 		return 1;
 
-	secs_to_flush = pblk_rb_sync_point_count(&pblk->rwb);
+	secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb);
 	if (!secs_to_flush && secs_avail < pblk->min_write_pgs)
 		return 1;
 
diff --git a/drivers/lightnvm/pblk.h b/drivers/lightnvm/pblk.h
index 59a64d461a5d..8c357fb6538e 100644
--- a/drivers/lightnvm/pblk.h
+++ b/drivers/lightnvm/pblk.h
@@ -51,17 +51,16 @@
 
 #define NR_PHY_IN_LOG (PBLK_EXPOSED_PAGE_SIZE / PBLK_SECTOR)
 
-#define pblk_for_each_lun(pblk, rlun, i) \
-		for ((i) = 0, rlun = &(pblk)->luns[0]; \
-			(i) < (pblk)->nr_luns; (i)++, rlun = &(pblk)->luns[(i)])
-
 /* Static pool sizes */
 #define PBLK_GEN_WS_POOL_SIZE (2)
 
+#define PBLK_DEFAULT_OP (11)
+
 enum {
 	PBLK_READ		= READ,
 	PBLK_WRITE		= WRITE,/* Write from write buffer */
 	PBLK_WRITE_INT,			/* Internal write - no write buffer */
+	PBLK_READ_RECOV,		/* Recovery read - errors allowed */
 	PBLK_ERASE,
 };
 
@@ -114,6 +113,7 @@ struct pblk_c_ctx {
 /* read context */
 struct pblk_g_ctx {
 	void *private;
+	unsigned long start_time;
 	u64 lba;
 };
 
@@ -170,7 +170,7 @@ struct pblk_rb {
 					 * the last submitted entry that has
 					 * been successfully persisted to media
 					 */
-	unsigned int sync_point;	/* Sync point - last entry that must be
+	unsigned int flush_point;	/* Sync point - last entry that must be
 					 * flushed to the media. Used with
 					 * REQ_FLUSH and REQ_FUA
 					 */
@@ -193,7 +193,7 @@ struct pblk_rb {
 	spinlock_t s_lock;		/* Sync lock */
 
 #ifdef CONFIG_NVM_DEBUG
-	atomic_t inflight_sync_point;	/* Not served REQ_FLUSH | REQ_FUA */
+	atomic_t inflight_flush_point;	/* Not served REQ_FLUSH | REQ_FUA */
 #endif
 };
 
@@ -256,9 +256,6 @@ struct pblk_rl {
 	unsigned int high;	/* Upper threshold for rate limiter (free run -
 				 * user I/O rate limiter
 				 */
-	unsigned int low;	/* Lower threshold for rate limiter (user I/O
-				 * rate limiter - stall)
-				 */
 	unsigned int high_pw;	/* High rounded up as a power of 2 */
 
 #define PBLK_USER_HIGH_THRS 8	/* Begin write limit at 12% available blks */
@@ -292,7 +289,9 @@ struct pblk_rl {
 
 	unsigned long long nr_secs;
 	unsigned long total_blocks;
-	atomic_t free_blocks;
+
+	atomic_t free_blocks;		/* Total number of free blocks (+ OP) */
+	atomic_t free_user_blocks;	/* Number of user free blocks (no OP) */
 };
 
 #define PBLK_LINE_EMPTY (~0U)
@@ -583,7 +582,9 @@ struct pblk {
 			    */
 
 	sector_t capacity; /* Device capacity when bad blocks are subtracted */
-	int over_pct;      /* Percentage of device used for over-provisioning */
+
+	int op;      /* Percentage of device used for over-provisioning */
+	int op_blks; /* Number of blocks used for over-provisioning */
 
 	/* pblk provisioning values. Used by rate limiter */
 	struct pblk_rl rl;
@@ -691,7 +692,7 @@ unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries);
 struct pblk_rb_entry *pblk_rb_sync_scan_entry(struct pblk_rb *rb,
 					      struct ppa_addr *ppa);
 void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags);
-unsigned int pblk_rb_sync_point_count(struct pblk_rb *rb);
+unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb);
 
 unsigned int pblk_rb_read_count(struct pblk_rb *rb);
 unsigned int pblk_rb_sync_count(struct pblk_rb *rb);
@@ -812,7 +813,7 @@ int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq);
 void pblk_submit_rec(struct work_struct *work);
 struct pblk_line *pblk_recov_l2p(struct pblk *pblk);
 int pblk_recov_pad(struct pblk *pblk);
-__le64 *pblk_recov_get_lba_list(struct pblk *pblk, struct line_emeta *emeta);
+int pblk_recov_check_emeta(struct pblk *pblk, struct line_emeta *emeta);
 int pblk_recov_setup_rq(struct pblk *pblk, struct pblk_c_ctx *c_ctx,
 			struct pblk_rec_ctx *recovery, u64 *comp_bits,
 			unsigned int comp);
@@ -843,6 +844,7 @@ void pblk_rl_free(struct pblk_rl *rl);
 void pblk_rl_update_rates(struct pblk_rl *rl);
 int pblk_rl_high_thrs(struct pblk_rl *rl);
 unsigned long pblk_rl_nr_free_blks(struct pblk_rl *rl);
+unsigned long pblk_rl_nr_user_free_blks(struct pblk_rl *rl);
 int pblk_rl_user_may_insert(struct pblk_rl *rl, int nr_entries);
 void pblk_rl_inserted(struct pblk_rl *rl, int nr_entries);
 void pblk_rl_user_in(struct pblk_rl *rl, int nr_entries);
@@ -851,7 +853,8 @@ void pblk_rl_gc_in(struct pblk_rl *rl, int nr_entries);
 void pblk_rl_out(struct pblk_rl *rl, int nr_user, int nr_gc);
 int pblk_rl_max_io(struct pblk_rl *rl);
 void pblk_rl_free_lines_inc(struct pblk_rl *rl, struct pblk_line *line);
-void pblk_rl_free_lines_dec(struct pblk_rl *rl, struct pblk_line *line);
+void pblk_rl_free_lines_dec(struct pblk_rl *rl, struct pblk_line *line,
+			    bool used);
 int pblk_rl_is_limit(struct pblk_rl *rl);
 
 /*
@@ -907,28 +910,47 @@ static inline int pblk_pad_distance(struct pblk *pblk)
 	struct nvm_tgt_dev *dev = pblk->dev;
 	struct nvm_geo *geo = &dev->geo;
 
-	return NVM_MEM_PAGE_WRITE * geo->nr_luns * geo->sec_per_pl;
+	return NVM_MEM_PAGE_WRITE * geo->all_luns * geo->sec_per_pl;
 }
 
-static inline int pblk_dev_ppa_to_line(struct ppa_addr p)
+static inline int pblk_ppa_to_line(struct ppa_addr p)
 {
 	return p.g.blk;
 }
 
-static inline int pblk_tgt_ppa_to_line(struct ppa_addr p)
+static inline int pblk_ppa_to_pos(struct nvm_geo *geo, struct ppa_addr p)
 {
-	return p.g.blk;
+	return p.g.lun * geo->nr_chnls + p.g.ch;
 }
 
-static inline int pblk_ppa_to_pos(struct nvm_geo *geo, struct ppa_addr p)
+static inline struct ppa_addr addr_to_gen_ppa(struct pblk *pblk, u64 paddr,
+					      u64 line_id)
 {
-	return p.g.lun * geo->nr_chnls + p.g.ch;
+	struct ppa_addr ppa;
+
+	ppa.ppa = 0;
+	ppa.g.blk = line_id;
+	ppa.g.pg = (paddr & pblk->ppaf.pg_mask) >> pblk->ppaf.pg_offset;
+	ppa.g.lun = (paddr & pblk->ppaf.lun_mask) >> pblk->ppaf.lun_offset;
+	ppa.g.ch = (paddr & pblk->ppaf.ch_mask) >> pblk->ppaf.ch_offset;
+	ppa.g.pl = (paddr & pblk->ppaf.pln_mask) >> pblk->ppaf.pln_offset;
+	ppa.g.sec = (paddr & pblk->ppaf.sec_mask) >> pblk->ppaf.sec_offset;
+
+	return ppa;
 }
 
-/* A block within a line corresponds to the lun */
-static inline int pblk_dev_ppa_to_pos(struct nvm_geo *geo, struct ppa_addr p)
+static inline u64 pblk_dev_ppa_to_line_addr(struct pblk *pblk,
+							struct ppa_addr p)
 {
-	return p.g.lun * geo->nr_chnls + p.g.ch;
+	u64 paddr;
+
+	paddr = (u64)p.g.pg << pblk->ppaf.pg_offset;
+	paddr |= (u64)p.g.lun << pblk->ppaf.lun_offset;
+	paddr |= (u64)p.g.ch << pblk->ppaf.ch_offset;
+	paddr |= (u64)p.g.pl << pblk->ppaf.pln_offset;
+	paddr |= (u64)p.g.sec << pblk->ppaf.sec_offset;
+
+	return paddr;
 }
 
 static inline struct ppa_addr pblk_ppa32_to_ppa64(struct pblk *pblk, u32 ppa32)
@@ -960,24 +982,6 @@ static inline struct ppa_addr pblk_ppa32_to_ppa64(struct pblk *pblk, u32 ppa32)
 	return ppa64;
 }
 
-static inline struct ppa_addr pblk_trans_map_get(struct pblk *pblk,
-								sector_t lba)
-{
-	struct ppa_addr ppa;
-
-	if (pblk->ppaf_bitsize < 32) {
-		u32 *map = (u32 *)pblk->trans_map;
-
-		ppa = pblk_ppa32_to_ppa64(pblk, map[lba]);
-	} else {
-		struct ppa_addr *map = (struct ppa_addr *)pblk->trans_map;
-
-		ppa = map[lba];
-	}
-
-	return ppa;
-}
-
 static inline u32 pblk_ppa64_to_ppa32(struct pblk *pblk, struct ppa_addr ppa64)
 {
 	u32 ppa32 = 0;
@@ -999,33 +1003,36 @@ static inline u32 pblk_ppa64_to_ppa32(struct pblk *pblk, struct ppa_addr ppa64)
 	return ppa32;
 }
 
-static inline void pblk_trans_map_set(struct pblk *pblk, sector_t lba,
-						struct ppa_addr ppa)
+static inline struct ppa_addr pblk_trans_map_get(struct pblk *pblk,
+								sector_t lba)
 {
+	struct ppa_addr ppa;
+
 	if (pblk->ppaf_bitsize < 32) {
 		u32 *map = (u32 *)pblk->trans_map;
 
-		map[lba] = pblk_ppa64_to_ppa32(pblk, ppa);
+		ppa = pblk_ppa32_to_ppa64(pblk, map[lba]);
 	} else {
-		u64 *map = (u64 *)pblk->trans_map;
+		struct ppa_addr *map = (struct ppa_addr *)pblk->trans_map;
 
-		map[lba] = ppa.ppa;
+		ppa = map[lba];
 	}
+
+	return ppa;
 }
 
-static inline u64 pblk_dev_ppa_to_line_addr(struct pblk *pblk,
-							struct ppa_addr p)
+static inline void pblk_trans_map_set(struct pblk *pblk, sector_t lba,
+						struct ppa_addr ppa)
 {
-	u64 paddr;
+	if (pblk->ppaf_bitsize < 32) {
+		u32 *map = (u32 *)pblk->trans_map;
 
-	paddr = 0;
-	paddr |= (u64)p.g.pg << pblk->ppaf.pg_offset;
-	paddr |= (u64)p.g.lun << pblk->ppaf.lun_offset;
-	paddr |= (u64)p.g.ch << pblk->ppaf.ch_offset;
-	paddr |= (u64)p.g.pl << pblk->ppaf.pln_offset;
-	paddr |= (u64)p.g.sec << pblk->ppaf.sec_offset;
+		map[lba] = pblk_ppa64_to_ppa32(pblk, ppa);
+	} else {
+		u64 *map = (u64 *)pblk->trans_map;
 
-	return paddr;
+		map[lba] = ppa.ppa;
+	}
 }
 
 static inline int pblk_ppa_empty(struct ppa_addr ppa_addr)
@@ -1040,10 +1047,7 @@ static inline void pblk_ppa_set_empty(struct ppa_addr *ppa_addr)
 
 static inline bool pblk_ppa_comp(struct ppa_addr lppa, struct ppa_addr rppa)
 {
-	if (lppa.ppa == rppa.ppa)
-		return true;
-
-	return false;
+	return (lppa.ppa == rppa.ppa);
 }
 
 static inline int pblk_addr_in_cache(struct ppa_addr ppa)
@@ -1066,32 +1070,6 @@ static inline struct ppa_addr pblk_cacheline_to_addr(int addr)
 	return p;
 }
 
-static inline struct ppa_addr addr_to_gen_ppa(struct pblk *pblk, u64 paddr,
-					      u64 line_id)
-{
-	struct ppa_addr ppa;
-
-	ppa.ppa = 0;
-	ppa.g.blk = line_id;
-	ppa.g.pg = (paddr & pblk->ppaf.pg_mask) >> pblk->ppaf.pg_offset;
-	ppa.g.lun = (paddr & pblk->ppaf.lun_mask) >> pblk->ppaf.lun_offset;
-	ppa.g.ch = (paddr & pblk->ppaf.ch_mask) >> pblk->ppaf.ch_offset;
-	ppa.g.pl = (paddr & pblk->ppaf.pln_mask) >> pblk->ppaf.pln_offset;
-	ppa.g.sec = (paddr & pblk->ppaf.sec_mask) >> pblk->ppaf.sec_offset;
-
-	return ppa;
-}
-
-static inline struct ppa_addr addr_to_pblk_ppa(struct pblk *pblk, u64 paddr,
-					 u64 line_id)
-{
-	struct ppa_addr ppa;
-
-	ppa = addr_to_gen_ppa(pblk, paddr, line_id);
-
-	return ppa;
-}
-
 static inline u32 pblk_calc_meta_header_crc(struct pblk *pblk,
 					    struct line_header *header)
 {
@@ -1212,10 +1190,10 @@ static inline int pblk_boundary_ppa_checks(struct nvm_tgt_dev *tgt_dev,
 
 		if (!ppa->c.is_cached &&
 				ppa->g.ch < geo->nr_chnls &&
-				ppa->g.lun < geo->luns_per_chnl &&
+				ppa->g.lun < geo->nr_luns &&
 				ppa->g.pl < geo->nr_planes &&
-				ppa->g.blk < geo->blks_per_lun &&
-				ppa->g.pg < geo->pgs_per_blk &&
+				ppa->g.blk < geo->nr_chks &&
+				ppa->g.pg < geo->ws_per_chk &&
 				ppa->g.sec < geo->sec_per_pg)
 			continue;
 
@@ -1245,7 +1223,7 @@ static inline int pblk_check_io(struct pblk *pblk, struct nvm_rq *rqd)
 
 		for (i = 0; i < rqd->nr_ppas; i++) {
 			ppa = ppa_list[i];
-			line = &pblk->lines[pblk_dev_ppa_to_line(ppa)];
+			line = &pblk->lines[pblk_ppa_to_line(ppa)];
 
 			spin_lock(&line->lock);
 			if (line->state != PBLK_LINESTATE_OPEN) {
@@ -1288,11 +1266,6 @@ static inline unsigned int pblk_get_secs(struct bio *bio)
 	return  bio->bi_iter.bi_size / PBLK_EXPOSED_PAGE_SIZE;
 }
 
-static inline sector_t pblk_get_sector(sector_t lba)
-{
-	return lba * NR_PHY_IN_LOG;
-}
-
 static inline void pblk_setup_uuid(struct pblk *pblk)
 {
 	uuid_le uuid;
diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
deleted file mode 100644
index 0993c14be860..000000000000
--- a/drivers/lightnvm/rrpc.c
+++ /dev/null
@@ -1,1625 +0,0 @@
-/*
- * Copyright (C) 2015 IT University of Copenhagen
- * Initial release: Matias Bjorling <m@bjorling.me>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs.
- */
-
-#include "rrpc.h"
-
-static struct kmem_cache *rrpc_gcb_cache, *rrpc_rq_cache;
-static DECLARE_RWSEM(rrpc_lock);
-
-static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
-				struct nvm_rq *rqd, unsigned long flags);
-
-#define rrpc_for_each_lun(rrpc, rlun, i) \
-		for ((i) = 0, rlun = &(rrpc)->luns[0]; \
-			(i) < (rrpc)->nr_luns; (i)++, rlun = &(rrpc)->luns[(i)])
-
-static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_block *rblk = a->rblk;
-	unsigned int pg_offset;
-
-	lockdep_assert_held(&rrpc->rev_lock);
-
-	if (a->addr == ADDR_EMPTY || !rblk)
-		return;
-
-	spin_lock(&rblk->lock);
-
-	div_u64_rem(a->addr, dev->geo.sec_per_blk, &pg_offset);
-	WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages));
-	rblk->nr_invalid_pages++;
-
-	spin_unlock(&rblk->lock);
-
-	rrpc->rev_trans_map[a->addr].addr = ADDR_EMPTY;
-}
-
-static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba,
-							unsigned int len)
-{
-	sector_t i;
-
-	spin_lock(&rrpc->rev_lock);
-	for (i = slba; i < slba + len; i++) {
-		struct rrpc_addr *gp = &rrpc->trans_map[i];
-
-		rrpc_page_invalidate(rrpc, gp);
-		gp->rblk = NULL;
-	}
-	spin_unlock(&rrpc->rev_lock);
-}
-
-static struct nvm_rq *rrpc_inflight_laddr_acquire(struct rrpc *rrpc,
-					sector_t laddr, unsigned int pages)
-{
-	struct nvm_rq *rqd;
-	struct rrpc_inflight_rq *inf;
-
-	rqd = mempool_alloc(rrpc->rq_pool, GFP_ATOMIC);
-	if (!rqd)
-		return ERR_PTR(-ENOMEM);
-
-	inf = rrpc_get_inflight_rq(rqd);
-	if (rrpc_lock_laddr(rrpc, laddr, pages, inf)) {
-		mempool_free(rqd, rrpc->rq_pool);
-		return NULL;
-	}
-
-	return rqd;
-}
-
-static void rrpc_inflight_laddr_release(struct rrpc *rrpc, struct nvm_rq *rqd)
-{
-	struct rrpc_inflight_rq *inf = rrpc_get_inflight_rq(rqd);
-
-	rrpc_unlock_laddr(rrpc, inf);
-
-	mempool_free(rqd, rrpc->rq_pool);
-}
-
-static void rrpc_discard(struct rrpc *rrpc, struct bio *bio)
-{
-	sector_t slba = bio->bi_iter.bi_sector / NR_PHY_IN_LOG;
-	sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE;
-	struct nvm_rq *rqd;
-
-	while (1) {
-		rqd = rrpc_inflight_laddr_acquire(rrpc, slba, len);
-		if (rqd)
-			break;
-
-		schedule();
-	}
-
-	if (IS_ERR(rqd)) {
-		pr_err("rrpc: unable to acquire inflight IO\n");
-		bio_io_error(bio);
-		return;
-	}
-
-	rrpc_invalidate_range(rrpc, slba, len);
-	rrpc_inflight_laddr_release(rrpc, rqd);
-}
-
-static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-
-	return (rblk->next_page == dev->geo.sec_per_blk);
-}
-
-/* Calculate relative addr for the given block, considering instantiated LUNs */
-static u64 block_to_rel_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_lun *rlun = rblk->rlun;
-
-	return rlun->id * dev->geo.sec_per_blk;
-}
-
-static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_tgt_dev *dev,
-					 struct rrpc_addr *gp)
-{
-	struct rrpc_block *rblk = gp->rblk;
-	struct rrpc_lun *rlun = rblk->rlun;
-	u64 addr = gp->addr;
-	struct ppa_addr paddr;
-
-	paddr.ppa = addr;
-	paddr = rrpc_linear_to_generic_addr(&dev->geo, paddr);
-	paddr.g.ch = rlun->bppa.g.ch;
-	paddr.g.lun = rlun->bppa.g.lun;
-	paddr.g.blk = rblk->id;
-
-	return paddr;
-}
-
-/* requires lun->lock taken */
-static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *new_rblk,
-						struct rrpc_block **cur_rblk)
-{
-	struct rrpc *rrpc = rlun->rrpc;
-
-	if (*cur_rblk) {
-		spin_lock(&(*cur_rblk)->lock);
-		WARN_ON(!block_is_full(rrpc, *cur_rblk));
-		spin_unlock(&(*cur_rblk)->lock);
-	}
-	*cur_rblk = new_rblk;
-}
-
-static struct rrpc_block *__rrpc_get_blk(struct rrpc *rrpc,
-							struct rrpc_lun *rlun)
-{
-	struct rrpc_block *rblk = NULL;
-
-	if (list_empty(&rlun->free_list))
-		goto out;
-
-	rblk = list_first_entry(&rlun->free_list, struct rrpc_block, list);
-
-	list_move_tail(&rblk->list, &rlun->used_list);
-	rblk->state = NVM_BLK_ST_TGT;
-	rlun->nr_free_blocks--;
-
-out:
-	return rblk;
-}
-
-static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
-							unsigned long flags)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_block *rblk;
-	int is_gc = flags & NVM_IOTYPE_GC;
-
-	spin_lock(&rlun->lock);
-	if (!is_gc && rlun->nr_free_blocks < rlun->reserved_blocks) {
-		pr_err("nvm: rrpc: cannot give block to non GC request\n");
-		spin_unlock(&rlun->lock);
-		return NULL;
-	}
-
-	rblk = __rrpc_get_blk(rrpc, rlun);
-	if (!rblk) {
-		pr_err("nvm: rrpc: cannot get new block\n");
-		spin_unlock(&rlun->lock);
-		return NULL;
-	}
-	spin_unlock(&rlun->lock);
-
-	bitmap_zero(rblk->invalid_pages, dev->geo.sec_per_blk);
-	rblk->next_page = 0;
-	rblk->nr_invalid_pages = 0;
-	atomic_set(&rblk->data_cmnt_size, 0);
-
-	return rblk;
-}
-
-static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct rrpc_lun *rlun = rblk->rlun;
-
-	spin_lock(&rlun->lock);
-	if (rblk->state & NVM_BLK_ST_TGT) {
-		list_move_tail(&rblk->list, &rlun->free_list);
-		rlun->nr_free_blocks++;
-		rblk->state = NVM_BLK_ST_FREE;
-	} else if (rblk->state & NVM_BLK_ST_BAD) {
-		list_move_tail(&rblk->list, &rlun->bb_list);
-		rblk->state = NVM_BLK_ST_BAD;
-	} else {
-		WARN_ON_ONCE(1);
-		pr_err("rrpc: erroneous type (ch:%d,lun:%d,blk%d-> %u)\n",
-					rlun->bppa.g.ch, rlun->bppa.g.lun,
-					rblk->id, rblk->state);
-		list_move_tail(&rblk->list, &rlun->bb_list);
-	}
-	spin_unlock(&rlun->lock);
-}
-
-static void rrpc_put_blks(struct rrpc *rrpc)
-{
-	struct rrpc_lun *rlun;
-	int i;
-
-	for (i = 0; i < rrpc->nr_luns; i++) {
-		rlun = &rrpc->luns[i];
-		if (rlun->cur)
-			rrpc_put_blk(rrpc, rlun->cur);
-		if (rlun->gc_cur)
-			rrpc_put_blk(rrpc, rlun->gc_cur);
-	}
-}
-
-static struct rrpc_lun *get_next_lun(struct rrpc *rrpc)
-{
-	int next = atomic_inc_return(&rrpc->next_lun);
-
-	return &rrpc->luns[next % rrpc->nr_luns];
-}
-
-static void rrpc_gc_kick(struct rrpc *rrpc)
-{
-	struct rrpc_lun *rlun;
-	unsigned int i;
-
-	for (i = 0; i < rrpc->nr_luns; i++) {
-		rlun = &rrpc->luns[i];
-		queue_work(rrpc->krqd_wq, &rlun->ws_gc);
-	}
-}
-
-/*
- * timed GC every interval.
- */
-static void rrpc_gc_timer(struct timer_list *t)
-{
-	struct rrpc *rrpc = from_timer(rrpc, t, gc_timer);
-
-	rrpc_gc_kick(rrpc);
-	mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10));
-}
-
-static void rrpc_end_sync_bio(struct bio *bio)
-{
-	struct completion *waiting = bio->bi_private;
-
-	if (bio->bi_status)
-		pr_err("nvm: gc request failed (%u).\n", bio->bi_status);
-
-	complete(waiting);
-}
-
-/*
- * rrpc_move_valid_pages -- migrate live data off the block
- * @rrpc: the 'rrpc' structure
- * @block: the block from which to migrate live pages
- *
- * Description:
- *   GC algorithms may call this function to migrate remaining live
- *   pages off the block prior to erasing it. This function blocks
- *   further execution until the operation is complete.
- */
-static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct request_queue *q = dev->q;
-	struct rrpc_rev_addr *rev;
-	struct nvm_rq *rqd;
-	struct bio *bio;
-	struct page *page;
-	int slot;
-	int nr_sec_per_blk = dev->geo.sec_per_blk;
-	u64 phys_addr;
-	DECLARE_COMPLETION_ONSTACK(wait);
-
-	if (bitmap_full(rblk->invalid_pages, nr_sec_per_blk))
-		return 0;
-
-	bio = bio_alloc(GFP_NOIO, 1);
-	if (!bio) {
-		pr_err("nvm: could not alloc bio to gc\n");
-		return -ENOMEM;
-	}
-
-	page = mempool_alloc(rrpc->page_pool, GFP_NOIO);
-
-	while ((slot = find_first_zero_bit(rblk->invalid_pages,
-					    nr_sec_per_blk)) < nr_sec_per_blk) {
-
-		/* Lock laddr */
-		phys_addr = rrpc_blk_to_ppa(rrpc, rblk) + slot;
-
-try:
-		spin_lock(&rrpc->rev_lock);
-		/* Get logical address from physical to logical table */
-		rev = &rrpc->rev_trans_map[phys_addr];
-		/* already updated by previous regular write */
-		if (rev->addr == ADDR_EMPTY) {
-			spin_unlock(&rrpc->rev_lock);
-			continue;
-		}
-
-		rqd = rrpc_inflight_laddr_acquire(rrpc, rev->addr, 1);
-		if (IS_ERR_OR_NULL(rqd)) {
-			spin_unlock(&rrpc->rev_lock);
-			schedule();
-			goto try;
-		}
-
-		spin_unlock(&rrpc->rev_lock);
-
-		/* Perform read to do GC */
-		bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);
-		bio_set_op_attrs(bio,  REQ_OP_READ, 0);
-		bio->bi_private = &wait;
-		bio->bi_end_io = rrpc_end_sync_bio;
-
-		/* TODO: may fail when EXP_PG_SIZE > PAGE_SIZE */
-		bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
-
-		if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {
-			pr_err("rrpc: gc read failed.\n");
-			rrpc_inflight_laddr_release(rrpc, rqd);
-			goto finished;
-		}
-		wait_for_completion_io(&wait);
-		if (bio->bi_status) {
-			rrpc_inflight_laddr_release(rrpc, rqd);
-			goto finished;
-		}
-
-		bio_reset(bio);
-		reinit_completion(&wait);
-
-		bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);
-		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
-		bio->bi_private = &wait;
-		bio->bi_end_io = rrpc_end_sync_bio;
-
-		bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
-
-		/* turn the command around and write the data back to a new
-		 * address
-		 */
-		if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {
-			pr_err("rrpc: gc write failed.\n");
-			rrpc_inflight_laddr_release(rrpc, rqd);
-			goto finished;
-		}
-		wait_for_completion_io(&wait);
-
-		rrpc_inflight_laddr_release(rrpc, rqd);
-		if (bio->bi_status)
-			goto finished;
-
-		bio_reset(bio);
-	}
-
-finished:
-	mempool_free(page, rrpc->page_pool);
-	bio_put(bio);
-
-	if (!bitmap_full(rblk->invalid_pages, nr_sec_per_blk)) {
-		pr_err("nvm: failed to garbage collect block\n");
-		return -EIO;
-	}
-
-	return 0;
-}
-
-static void rrpc_block_gc(struct work_struct *work)
-{
-	struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc,
-									ws_gc);
-	struct rrpc *rrpc = gcb->rrpc;
-	struct rrpc_block *rblk = gcb->rblk;
-	struct rrpc_lun *rlun = rblk->rlun;
-	struct ppa_addr ppa;
-
-	mempool_free(gcb, rrpc->gcb_pool);
-	pr_debug("nvm: block 'ch:%d,lun:%d,blk:%d' being reclaimed\n",
-			rlun->bppa.g.ch, rlun->bppa.g.lun,
-			rblk->id);
-
-	if (rrpc_move_valid_pages(rrpc, rblk))
-		goto put_back;
-
-	ppa.ppa = 0;
-	ppa.g.ch = rlun->bppa.g.ch;
-	ppa.g.lun = rlun->bppa.g.lun;
-	ppa.g.blk = rblk->id;
-
-	if (nvm_erase_sync(rrpc->dev, &ppa, 1))
-		goto put_back;
-
-	rrpc_put_blk(rrpc, rblk);
-
-	return;
-
-put_back:
-	spin_lock(&rlun->lock);
-	list_add_tail(&rblk->prio, &rlun->prio_list);
-	spin_unlock(&rlun->lock);
-}
-
-/* the block with highest number of invalid pages, will be in the beginning
- * of the list
- */
-static struct rrpc_block *rblk_max_invalid(struct rrpc_block *ra,
-							struct rrpc_block *rb)
-{
-	if (ra->nr_invalid_pages == rb->nr_invalid_pages)
-		return ra;
-
-	return (ra->nr_invalid_pages < rb->nr_invalid_pages) ? rb : ra;
-}
-
-/* linearly find the block with highest number of invalid pages
- * requires lun->lock
- */
-static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun)
-{
-	struct list_head *prio_list = &rlun->prio_list;
-	struct rrpc_block *rblk, *max;
-
-	BUG_ON(list_empty(prio_list));
-
-	max = list_first_entry(prio_list, struct rrpc_block, prio);
-	list_for_each_entry(rblk, prio_list, prio)
-		max = rblk_max_invalid(max, rblk);
-
-	return max;
-}
-
-static void rrpc_lun_gc(struct work_struct *work)
-{
-	struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc);
-	struct rrpc *rrpc = rlun->rrpc;
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_block_gc *gcb;
-	unsigned int nr_blocks_need;
-
-	nr_blocks_need = dev->geo.blks_per_lun / GC_LIMIT_INVERSE;
-
-	if (nr_blocks_need < rrpc->nr_luns)
-		nr_blocks_need = rrpc->nr_luns;
-
-	spin_lock(&rlun->lock);
-	while (nr_blocks_need > rlun->nr_free_blocks &&
-					!list_empty(&rlun->prio_list)) {
-		struct rrpc_block *rblk = block_prio_find_max(rlun);
-
-		if (!rblk->nr_invalid_pages)
-			break;
-
-		gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
-		if (!gcb)
-			break;
-
-		list_del_init(&rblk->prio);
-
-		WARN_ON(!block_is_full(rrpc, rblk));
-
-		pr_debug("rrpc: selected block 'ch:%d,lun:%d,blk:%d' for GC\n",
-					rlun->bppa.g.ch, rlun->bppa.g.lun,
-					rblk->id);
-
-		gcb->rrpc = rrpc;
-		gcb->rblk = rblk;
-		INIT_WORK(&gcb->ws_gc, rrpc_block_gc);
-
-		queue_work(rrpc->kgc_wq, &gcb->ws_gc);
-
-		nr_blocks_need--;
-	}
-	spin_unlock(&rlun->lock);
-
-	/* TODO: Hint that request queue can be started again */
-}
-
-static void rrpc_gc_queue(struct work_struct *work)
-{
-	struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc,
-									ws_gc);
-	struct rrpc *rrpc = gcb->rrpc;
-	struct rrpc_block *rblk = gcb->rblk;
-	struct rrpc_lun *rlun = rblk->rlun;
-
-	spin_lock(&rlun->lock);
-	list_add_tail(&rblk->prio, &rlun->prio_list);
-	spin_unlock(&rlun->lock);
-
-	mempool_free(gcb, rrpc->gcb_pool);
-	pr_debug("nvm: block 'ch:%d,lun:%d,blk:%d' full, allow GC (sched)\n",
-					rlun->bppa.g.ch, rlun->bppa.g.lun,
-					rblk->id);
-}
-
-static const struct block_device_operations rrpc_fops = {
-	.owner		= THIS_MODULE,
-};
-
-static struct rrpc_lun *rrpc_get_lun_rr(struct rrpc *rrpc, int is_gc)
-{
-	unsigned int i;
-	struct rrpc_lun *rlun, *max_free;
-
-	if (!is_gc)
-		return get_next_lun(rrpc);
-
-	/* during GC, we don't care about RR, instead we want to make
-	 * sure that we maintain evenness between the block luns.
-	 */
-	max_free = &rrpc->luns[0];
-	/* prevent GC-ing lun from devouring pages of a lun with
-	 * little free blocks. We don't take the lock as we only need an
-	 * estimate.
-	 */
-	rrpc_for_each_lun(rrpc, rlun, i) {
-		if (rlun->nr_free_blocks > max_free->nr_free_blocks)
-			max_free = rlun;
-	}
-
-	return max_free;
-}
-
-static struct rrpc_addr *rrpc_update_map(struct rrpc *rrpc, sector_t laddr,
-					struct rrpc_block *rblk, u64 paddr)
-{
-	struct rrpc_addr *gp;
-	struct rrpc_rev_addr *rev;
-
-	BUG_ON(laddr >= rrpc->nr_sects);
-
-	gp = &rrpc->trans_map[laddr];
-	spin_lock(&rrpc->rev_lock);
-	if (gp->rblk)
-		rrpc_page_invalidate(rrpc, gp);
-
-	gp->addr = paddr;
-	gp->rblk = rblk;
-
-	rev = &rrpc->rev_trans_map[gp->addr];
-	rev->addr = laddr;
-	spin_unlock(&rrpc->rev_lock);
-
-	return gp;
-}
-
-static u64 rrpc_alloc_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	u64 addr = ADDR_EMPTY;
-
-	spin_lock(&rblk->lock);
-	if (block_is_full(rrpc, rblk))
-		goto out;
-
-	addr = rblk->next_page;
-
-	rblk->next_page++;
-out:
-	spin_unlock(&rblk->lock);
-	return addr;
-}
-
-/* Map logical address to a physical page. The mapping implements a round robin
- * approach and allocates a page from the next lun available.
- *
- * Returns rrpc_addr with the physical address and block. Returns NULL if no
- * blocks in the next rlun are available.
- */
-static struct ppa_addr rrpc_map_page(struct rrpc *rrpc, sector_t laddr,
-								int is_gc)
-{
-	struct nvm_tgt_dev *tgt_dev = rrpc->dev;
-	struct rrpc_lun *rlun;
-	struct rrpc_block *rblk, **cur_rblk;
-	struct rrpc_addr *p;
-	struct ppa_addr ppa;
-	u64 paddr;
-	int gc_force = 0;
-
-	ppa.ppa = ADDR_EMPTY;
-	rlun = rrpc_get_lun_rr(rrpc, is_gc);
-
-	if (!is_gc && rlun->nr_free_blocks < rrpc->nr_luns * 4)
-		return ppa;
-
-	/*
-	 * page allocation steps:
-	 * 1. Try to allocate new page from current rblk
-	 * 2a. If succeed, proceed to map it in and return
-	 * 2b. If fail, first try to allocate a new block from media manger,
-	 *     and then retry step 1. Retry until the normal block pool is
-	 *     exhausted.
-	 * 3. If exhausted, and garbage collector is requesting the block,
-	 *    go to the reserved block and retry step 1.
-	 *    In the case that this fails as well, or it is not GC
-	 *    requesting, report not able to retrieve a block and let the
-	 *    caller handle further processing.
-	 */
-
-	spin_lock(&rlun->lock);
-	cur_rblk = &rlun->cur;
-	rblk = rlun->cur;
-retry:
-	paddr = rrpc_alloc_addr(rrpc, rblk);
-
-	if (paddr != ADDR_EMPTY)
-		goto done;
-
-	if (!list_empty(&rlun->wblk_list)) {
-new_blk:
-		rblk = list_first_entry(&rlun->wblk_list, struct rrpc_block,
-									prio);
-		rrpc_set_lun_cur(rlun, rblk, cur_rblk);
-		list_del(&rblk->prio);
-		goto retry;
-	}
-	spin_unlock(&rlun->lock);
-
-	rblk = rrpc_get_blk(rrpc, rlun, gc_force);
-	if (rblk) {
-		spin_lock(&rlun->lock);
-		list_add_tail(&rblk->prio, &rlun->wblk_list);
-		/*
-		 * another thread might already have added a new block,
-		 * Therefore, make sure that one is used, instead of the
-		 * one just added.
-		 */
-		goto new_blk;
-	}
-
-	if (unlikely(is_gc) && !gc_force) {
-		/* retry from emergency gc block */
-		cur_rblk = &rlun->gc_cur;
-		rblk = rlun->gc_cur;
-		gc_force = 1;
-		spin_lock(&rlun->lock);
-		goto retry;
-	}
-
-	pr_err("rrpc: failed to allocate new block\n");
-	return ppa;
-done:
-	spin_unlock(&rlun->lock);
-	p = rrpc_update_map(rrpc, laddr, rblk, paddr);
-	if (!p)
-		return ppa;
-
-	/* return global address */
-	return rrpc_ppa_to_gaddr(tgt_dev, p);
-}
-
-static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct rrpc_block_gc *gcb;
-
-	gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
-	if (!gcb) {
-		pr_err("rrpc: unable to queue block for gc.");
-		return;
-	}
-
-	gcb->rrpc = rrpc;
-	gcb->rblk = rblk;
-
-	INIT_WORK(&gcb->ws_gc, rrpc_gc_queue);
-	queue_work(rrpc->kgc_wq, &gcb->ws_gc);
-}
-
-static struct rrpc_lun *rrpc_ppa_to_lun(struct rrpc *rrpc, struct ppa_addr p)
-{
-	struct rrpc_lun *rlun = NULL;
-	int i;
-
-	for (i = 0; i < rrpc->nr_luns; i++) {
-		if (rrpc->luns[i].bppa.g.ch == p.g.ch &&
-				rrpc->luns[i].bppa.g.lun == p.g.lun) {
-			rlun = &rrpc->luns[i];
-			break;
-		}
-	}
-
-	return rlun;
-}
-
-static void __rrpc_mark_bad_block(struct rrpc *rrpc, struct ppa_addr ppa)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_lun *rlun;
-	struct rrpc_block *rblk;
-
-	rlun = rrpc_ppa_to_lun(rrpc, ppa);
-	rblk = &rlun->blocks[ppa.g.blk];
-	rblk->state = NVM_BLK_ST_BAD;
-
-	nvm_set_tgt_bb_tbl(dev, &ppa, 1, NVM_BLK_T_GRWN_BAD);
-}
-
-static void rrpc_mark_bad_block(struct rrpc *rrpc, struct nvm_rq *rqd)
-{
-	void *comp_bits = &rqd->ppa_status;
-	struct ppa_addr ppa, prev_ppa;
-	int nr_ppas = rqd->nr_ppas;
-	int bit;
-
-	if (rqd->nr_ppas == 1)
-		__rrpc_mark_bad_block(rrpc, rqd->ppa_addr);
-
-	ppa_set_empty(&prev_ppa);
-	bit = -1;
-	while ((bit = find_next_bit(comp_bits, nr_ppas, bit + 1)) < nr_ppas) {
-		ppa = rqd->ppa_list[bit];
-		if (ppa_cmp_blk(ppa, prev_ppa))
-			continue;
-
-		__rrpc_mark_bad_block(rrpc, ppa);
-	}
-}
-
-static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd,
-						sector_t laddr, uint8_t npages)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_addr *p;
-	struct rrpc_block *rblk;
-	int cmnt_size, i;
-
-	for (i = 0; i < npages; i++) {
-		p = &rrpc->trans_map[laddr + i];
-		rblk = p->rblk;
-
-		cmnt_size = atomic_inc_return(&rblk->data_cmnt_size);
-		if (unlikely(cmnt_size == dev->geo.sec_per_blk))
-			rrpc_run_gc(rrpc, rblk);
-	}
-}
-
-static void rrpc_end_io(struct nvm_rq *rqd)
-{
-	struct rrpc *rrpc = rqd->private;
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
-	uint8_t npages = rqd->nr_ppas;
-	sector_t laddr = rrpc_get_laddr(rqd->bio) - npages;
-
-	if (bio_data_dir(rqd->bio) == WRITE) {
-		if (rqd->error == NVM_RSP_ERR_FAILWRITE)
-			rrpc_mark_bad_block(rrpc, rqd);
-
-		rrpc_end_io_write(rrpc, rrqd, laddr, npages);
-	}
-
-	bio_put(rqd->bio);
-
-	if (rrqd->flags & NVM_IOTYPE_GC)
-		return;
-
-	rrpc_unlock_rq(rrpc, rqd);
-
-	if (npages > 1)
-		nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
-
-	mempool_free(rqd, rrpc->rq_pool);
-}
-
-static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
-			struct nvm_rq *rqd, unsigned long flags, int npages)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
-	struct rrpc_addr *gp;
-	sector_t laddr = rrpc_get_laddr(bio);
-	int is_gc = flags & NVM_IOTYPE_GC;
-	int i;
-
-	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
-		nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
-		return NVM_IO_REQUEUE;
-	}
-
-	for (i = 0; i < npages; i++) {
-		/* We assume that mapping occurs at 4KB granularity */
-		BUG_ON(!(laddr + i < rrpc->nr_sects));
-		gp = &rrpc->trans_map[laddr + i];
-
-		if (gp->rblk) {
-			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(dev, gp);
-		} else {
-			BUG_ON(is_gc);
-			rrpc_unlock_laddr(rrpc, r);
-			nvm_dev_dma_free(dev->parent, rqd->ppa_list,
-							rqd->dma_ppa_list);
-			return NVM_IO_DONE;
-		}
-	}
-
-	rqd->opcode = NVM_OP_HBREAD;
-
-	return NVM_IO_OK;
-}
-
-static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
-							unsigned long flags)
-{
-	int is_gc = flags & NVM_IOTYPE_GC;
-	sector_t laddr = rrpc_get_laddr(bio);
-	struct rrpc_addr *gp;
-
-	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))
-		return NVM_IO_REQUEUE;
-
-	BUG_ON(!(laddr < rrpc->nr_sects));
-	gp = &rrpc->trans_map[laddr];
-
-	if (gp->rblk) {
-		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp);
-	} else {
-		BUG_ON(is_gc);
-		rrpc_unlock_rq(rrpc, rqd);
-		return NVM_IO_DONE;
-	}
-
-	rqd->opcode = NVM_OP_HBREAD;
-
-	return NVM_IO_OK;
-}
-
-static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
-			struct nvm_rq *rqd, unsigned long flags, int npages)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
-	struct ppa_addr p;
-	sector_t laddr = rrpc_get_laddr(bio);
-	int is_gc = flags & NVM_IOTYPE_GC;
-	int i;
-
-	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
-		nvm_dev_dma_free(dev->parent, rqd->ppa_list, rqd->dma_ppa_list);
-		return NVM_IO_REQUEUE;
-	}
-
-	for (i = 0; i < npages; i++) {
-		/* We assume that mapping occurs at 4KB granularity */
-		p = rrpc_map_page(rrpc, laddr + i, is_gc);
-		if (p.ppa == ADDR_EMPTY) {
-			BUG_ON(is_gc);
-			rrpc_unlock_laddr(rrpc, r);
-			nvm_dev_dma_free(dev->parent, rqd->ppa_list,
-							rqd->dma_ppa_list);
-			rrpc_gc_kick(rrpc);
-			return NVM_IO_REQUEUE;
-		}
-
-		rqd->ppa_list[i] = p;
-	}
-
-	rqd->opcode = NVM_OP_HBWRITE;
-
-	return NVM_IO_OK;
-}
-
-static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
-				struct nvm_rq *rqd, unsigned long flags)
-{
-	struct ppa_addr p;
-	int is_gc = flags & NVM_IOTYPE_GC;
-	sector_t laddr = rrpc_get_laddr(bio);
-
-	if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))
-		return NVM_IO_REQUEUE;
-
-	p = rrpc_map_page(rrpc, laddr, is_gc);
-	if (p.ppa == ADDR_EMPTY) {
-		BUG_ON(is_gc);
-		rrpc_unlock_rq(rrpc, rqd);
-		rrpc_gc_kick(rrpc);
-		return NVM_IO_REQUEUE;
-	}
-
-	rqd->ppa_addr = p;
-	rqd->opcode = NVM_OP_HBWRITE;
-
-	return NVM_IO_OK;
-}
-
-static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio,
-			struct nvm_rq *rqd, unsigned long flags, uint8_t npages)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-
-	if (npages > 1) {
-		rqd->ppa_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
-							&rqd->dma_ppa_list);
-		if (!rqd->ppa_list) {
-			pr_err("rrpc: not able to allocate ppa list\n");
-			return NVM_IO_ERR;
-		}
-
-		if (bio_op(bio) == REQ_OP_WRITE)
-			return rrpc_write_ppalist_rq(rrpc, bio, rqd, flags,
-									npages);
-
-		return rrpc_read_ppalist_rq(rrpc, bio, rqd, flags, npages);
-	}
-
-	if (bio_op(bio) == REQ_OP_WRITE)
-		return rrpc_write_rq(rrpc, bio, rqd, flags);
-
-	return rrpc_read_rq(rrpc, bio, rqd, flags);
-}
-
-static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
-				struct nvm_rq *rqd, unsigned long flags)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_rq *rrq = nvm_rq_to_pdu(rqd);
-	uint8_t nr_pages = rrpc_get_pages(bio);
-	int bio_size = bio_sectors(bio) << 9;
-	int err;
-
-	if (bio_size < dev->geo.sec_size)
-		return NVM_IO_ERR;
-	else if (bio_size > dev->geo.max_rq_size)
-		return NVM_IO_ERR;
-
-	err = rrpc_setup_rq(rrpc, bio, rqd, flags, nr_pages);
-	if (err)
-		return err;
-
-	bio_get(bio);
-	rqd->bio = bio;
-	rqd->private = rrpc;
-	rqd->nr_ppas = nr_pages;
-	rqd->end_io = rrpc_end_io;
-	rrq->flags = flags;
-
-	err = nvm_submit_io(dev, rqd);
-	if (err) {
-		pr_err("rrpc: I/O submission failed: %d\n", err);
-		bio_put(bio);
-		if (!(flags & NVM_IOTYPE_GC)) {
-			rrpc_unlock_rq(rrpc, rqd);
-			if (rqd->nr_ppas > 1)
-				nvm_dev_dma_free(dev->parent, rqd->ppa_list,
-							rqd->dma_ppa_list);
-		}
-		return NVM_IO_ERR;
-	}
-
-	return NVM_IO_OK;
-}
-
-static blk_qc_t rrpc_make_rq(struct request_queue *q, struct bio *bio)
-{
-	struct rrpc *rrpc = q->queuedata;
-	struct nvm_rq *rqd;
-	int err;
-
-	blk_queue_split(q, &bio);
-
-	if (bio_op(bio) == REQ_OP_DISCARD) {
-		rrpc_discard(rrpc, bio);
-		return BLK_QC_T_NONE;
-	}
-
-	rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL);
-	memset(rqd, 0, sizeof(struct nvm_rq));
-
-	err = rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_NONE);
-	switch (err) {
-	case NVM_IO_OK:
-		return BLK_QC_T_NONE;
-	case NVM_IO_ERR:
-		bio_io_error(bio);
-		break;
-	case NVM_IO_DONE:
-		bio_endio(bio);
-		break;
-	case NVM_IO_REQUEUE:
-		spin_lock(&rrpc->bio_lock);
-		bio_list_add(&rrpc->requeue_bios, bio);
-		spin_unlock(&rrpc->bio_lock);
-		queue_work(rrpc->kgc_wq, &rrpc->ws_requeue);
-		break;
-	}
-
-	mempool_free(rqd, rrpc->rq_pool);
-	return BLK_QC_T_NONE;
-}
-
-static void rrpc_requeue(struct work_struct *work)
-{
-	struct rrpc *rrpc = container_of(work, struct rrpc, ws_requeue);
-	struct bio_list bios;
-	struct bio *bio;
-
-	bio_list_init(&bios);
-
-	spin_lock(&rrpc->bio_lock);
-	bio_list_merge(&bios, &rrpc->requeue_bios);
-	bio_list_init(&rrpc->requeue_bios);
-	spin_unlock(&rrpc->bio_lock);
-
-	while ((bio = bio_list_pop(&bios)))
-		rrpc_make_rq(rrpc->disk->queue, bio);
-}
-
-static void rrpc_gc_free(struct rrpc *rrpc)
-{
-	if (rrpc->krqd_wq)
-		destroy_workqueue(rrpc->krqd_wq);
-
-	if (rrpc->kgc_wq)
-		destroy_workqueue(rrpc->kgc_wq);
-}
-
-static int rrpc_gc_init(struct rrpc *rrpc)
-{
-	rrpc->krqd_wq = alloc_workqueue("rrpc-lun", WQ_MEM_RECLAIM|WQ_UNBOUND,
-								rrpc->nr_luns);
-	if (!rrpc->krqd_wq)
-		return -ENOMEM;
-
-	rrpc->kgc_wq = alloc_workqueue("rrpc-bg", WQ_MEM_RECLAIM, 1);
-	if (!rrpc->kgc_wq)
-		return -ENOMEM;
-
-	timer_setup(&rrpc->gc_timer, rrpc_gc_timer, 0);
-
-	return 0;
-}
-
-static void rrpc_map_free(struct rrpc *rrpc)
-{
-	vfree(rrpc->rev_trans_map);
-	vfree(rrpc->trans_map);
-}
-
-static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private)
-{
-	struct rrpc *rrpc = (struct rrpc *)private;
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_addr *addr = rrpc->trans_map + slba;
-	struct rrpc_rev_addr *raddr = rrpc->rev_trans_map;
-	struct rrpc_lun *rlun;
-	struct rrpc_block *rblk;
-	u64 i;
-
-	for (i = 0; i < nlb; i++) {
-		struct ppa_addr gaddr;
-		u64 pba = le64_to_cpu(entries[i]);
-		unsigned int mod;
-
-		/* LNVM treats address-spaces as silos, LBA and PBA are
-		 * equally large and zero-indexed.
-		 */
-		if (unlikely(pba >= dev->total_secs && pba != U64_MAX)) {
-			pr_err("nvm: L2P data entry is out of bounds!\n");
-			pr_err("nvm: Maybe loaded an old target L2P\n");
-			return -EINVAL;
-		}
-
-		/* Address zero is a special one. The first page on a disk is
-		 * protected. As it often holds internal device boot
-		 * information.
-		 */
-		if (!pba)
-			continue;
-
-		div_u64_rem(pba, rrpc->nr_sects, &mod);
-
-		gaddr = rrpc_recov_addr(dev, pba);
-		rlun = rrpc_ppa_to_lun(rrpc, gaddr);
-		if (!rlun) {
-			pr_err("rrpc: l2p corruption on lba %llu\n",
-							slba + i);
-			return -EINVAL;
-		}
-
-		rblk = &rlun->blocks[gaddr.g.blk];
-		if (!rblk->state) {
-			/* at this point, we don't know anything about the
-			 * block. It's up to the FTL on top to re-etablish the
-			 * block state. The block is assumed to be open.
-			 */
-			list_move_tail(&rblk->list, &rlun->used_list);
-			rblk->state = NVM_BLK_ST_TGT;
-			rlun->nr_free_blocks--;
-		}
-
-		addr[i].addr = pba;
-		addr[i].rblk = rblk;
-		raddr[mod].addr = slba + i;
-	}
-
-	return 0;
-}
-
-static int rrpc_map_init(struct rrpc *rrpc)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	sector_t i;
-	int ret;
-
-	rrpc->trans_map = vzalloc(sizeof(struct rrpc_addr) * rrpc->nr_sects);
-	if (!rrpc->trans_map)
-		return -ENOMEM;
-
-	rrpc->rev_trans_map = vmalloc(sizeof(struct rrpc_rev_addr)
-							* rrpc->nr_sects);
-	if (!rrpc->rev_trans_map)
-		return -ENOMEM;
-
-	for (i = 0; i < rrpc->nr_sects; i++) {
-		struct rrpc_addr *p = &rrpc->trans_map[i];
-		struct rrpc_rev_addr *r = &rrpc->rev_trans_map[i];
-
-		p->addr = ADDR_EMPTY;
-		r->addr = ADDR_EMPTY;
-	}
-
-	/* Bring up the mapping table from device */
-	ret = nvm_get_l2p_tbl(dev, rrpc->soffset, rrpc->nr_sects,
-							rrpc_l2p_update, rrpc);
-	if (ret) {
-		pr_err("nvm: rrpc: could not read L2P table.\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-/* Minimum pages needed within a lun */
-#define PAGE_POOL_SIZE 16
-#define ADDR_POOL_SIZE 64
-
-static int rrpc_core_init(struct rrpc *rrpc)
-{
-	down_write(&rrpc_lock);
-	if (!rrpc_gcb_cache) {
-		rrpc_gcb_cache = kmem_cache_create("rrpc_gcb",
-				sizeof(struct rrpc_block_gc), 0, 0, NULL);
-		if (!rrpc_gcb_cache) {
-			up_write(&rrpc_lock);
-			return -ENOMEM;
-		}
-
-		rrpc_rq_cache = kmem_cache_create("rrpc_rq",
-				sizeof(struct nvm_rq) + sizeof(struct rrpc_rq),
-				0, 0, NULL);
-		if (!rrpc_rq_cache) {
-			kmem_cache_destroy(rrpc_gcb_cache);
-			up_write(&rrpc_lock);
-			return -ENOMEM;
-		}
-	}
-	up_write(&rrpc_lock);
-
-	rrpc->page_pool = mempool_create_page_pool(PAGE_POOL_SIZE, 0);
-	if (!rrpc->page_pool)
-		return -ENOMEM;
-
-	rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->geo.nr_luns,
-								rrpc_gcb_cache);
-	if (!rrpc->gcb_pool)
-		return -ENOMEM;
-
-	rrpc->rq_pool = mempool_create_slab_pool(64, rrpc_rq_cache);
-	if (!rrpc->rq_pool)
-		return -ENOMEM;
-
-	spin_lock_init(&rrpc->inflights.lock);
-	INIT_LIST_HEAD(&rrpc->inflights.reqs);
-
-	return 0;
-}
-
-static void rrpc_core_free(struct rrpc *rrpc)
-{
-	mempool_destroy(rrpc->page_pool);
-	mempool_destroy(rrpc->gcb_pool);
-	mempool_destroy(rrpc->rq_pool);
-}
-
-static void rrpc_luns_free(struct rrpc *rrpc)
-{
-	struct rrpc_lun *rlun;
-	int i;
-
-	if (!rrpc->luns)
-		return;
-
-	for (i = 0; i < rrpc->nr_luns; i++) {
-		rlun = &rrpc->luns[i];
-		vfree(rlun->blocks);
-	}
-
-	kfree(rrpc->luns);
-}
-
-static int rrpc_bb_discovery(struct nvm_tgt_dev *dev, struct rrpc_lun *rlun)
-{
-	struct nvm_geo *geo = &dev->geo;
-	struct rrpc_block *rblk;
-	struct ppa_addr ppa;
-	u8 *blks;
-	int nr_blks;
-	int i;
-	int ret;
-
-	if (!dev->parent->ops->get_bb_tbl)
-		return 0;
-
-	nr_blks = geo->blks_per_lun * geo->plane_mode;
-	blks = kmalloc(nr_blks, GFP_KERNEL);
-	if (!blks)
-		return -ENOMEM;
-
-	ppa.ppa = 0;
-	ppa.g.ch = rlun->bppa.g.ch;
-	ppa.g.lun = rlun->bppa.g.lun;
-
-	ret = nvm_get_tgt_bb_tbl(dev, ppa, blks);
-	if (ret) {
-		pr_err("rrpc: could not get BB table\n");
-		goto out;
-	}
-
-	nr_blks = nvm_bb_tbl_fold(dev->parent, blks, nr_blks);
-	if (nr_blks < 0) {
-		ret = nr_blks;
-		goto out;
-	}
-
-	for (i = 0; i < nr_blks; i++) {
-		if (blks[i] == NVM_BLK_T_FREE)
-			continue;
-
-		rblk = &rlun->blocks[i];
-		list_move_tail(&rblk->list, &rlun->bb_list);
-		rblk->state = NVM_BLK_ST_BAD;
-		rlun->nr_free_blocks--;
-	}
-
-out:
-	kfree(blks);
-	return ret;
-}
-
-static void rrpc_set_lun_ppa(struct rrpc_lun *rlun, struct ppa_addr ppa)
-{
-	rlun->bppa.ppa = 0;
-	rlun->bppa.g.ch = ppa.g.ch;
-	rlun->bppa.g.lun = ppa.g.lun;
-}
-
-static int rrpc_luns_init(struct rrpc *rrpc, struct ppa_addr *luns)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct nvm_geo *geo = &dev->geo;
-	struct rrpc_lun *rlun;
-	int i, j, ret = -EINVAL;
-
-	if (geo->sec_per_blk > MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
-		pr_err("rrpc: number of pages per block too high.");
-		return -EINVAL;
-	}
-
-	spin_lock_init(&rrpc->rev_lock);
-
-	rrpc->luns = kcalloc(rrpc->nr_luns, sizeof(struct rrpc_lun),
-								GFP_KERNEL);
-	if (!rrpc->luns)
-		return -ENOMEM;
-
-	/* 1:1 mapping */
-	for (i = 0; i < rrpc->nr_luns; i++) {
-		rlun = &rrpc->luns[i];
-		rlun->id = i;
-		rrpc_set_lun_ppa(rlun, luns[i]);
-		rlun->blocks = vzalloc(sizeof(struct rrpc_block) *
-							geo->blks_per_lun);
-		if (!rlun->blocks) {
-			ret = -ENOMEM;
-			goto err;
-		}
-
-		INIT_LIST_HEAD(&rlun->free_list);
-		INIT_LIST_HEAD(&rlun->used_list);
-		INIT_LIST_HEAD(&rlun->bb_list);
-
-		for (j = 0; j < geo->blks_per_lun; j++) {
-			struct rrpc_block *rblk = &rlun->blocks[j];
-
-			rblk->id = j;
-			rblk->rlun = rlun;
-			rblk->state = NVM_BLK_T_FREE;
-			INIT_LIST_HEAD(&rblk->prio);
-			INIT_LIST_HEAD(&rblk->list);
-			spin_lock_init(&rblk->lock);
-
-			list_add_tail(&rblk->list, &rlun->free_list);
-		}
-
-		rlun->rrpc = rrpc;
-		rlun->nr_free_blocks = geo->blks_per_lun;
-		rlun->reserved_blocks = 2; /* for GC only */
-
-		INIT_LIST_HEAD(&rlun->prio_list);
-		INIT_LIST_HEAD(&rlun->wblk_list);
-
-		INIT_WORK(&rlun->ws_gc, rrpc_lun_gc);
-		spin_lock_init(&rlun->lock);
-
-		if (rrpc_bb_discovery(dev, rlun))
-			goto err;
-
-	}
-
-	return 0;
-err:
-	return ret;
-}
-
-/* returns 0 on success and stores the beginning address in *begin */
-static int rrpc_area_init(struct rrpc *rrpc, sector_t *begin)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	sector_t size = rrpc->nr_sects * dev->geo.sec_size;
-	int ret;
-
-	size >>= 9;
-
-	ret = nvm_get_area(dev, begin, size);
-	if (!ret)
-		*begin >>= (ilog2(dev->geo.sec_size) - 9);
-
-	return ret;
-}
-
-static void rrpc_area_free(struct rrpc *rrpc)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	sector_t begin = rrpc->soffset << (ilog2(dev->geo.sec_size) - 9);
-
-	nvm_put_area(dev, begin);
-}
-
-static void rrpc_free(struct rrpc *rrpc)
-{
-	rrpc_gc_free(rrpc);
-	rrpc_map_free(rrpc);
-	rrpc_core_free(rrpc);
-	rrpc_luns_free(rrpc);
-	rrpc_area_free(rrpc);
-
-	kfree(rrpc);
-}
-
-static void rrpc_exit(void *private)
-{
-	struct rrpc *rrpc = private;
-
-	del_timer(&rrpc->gc_timer);
-
-	flush_workqueue(rrpc->krqd_wq);
-	flush_workqueue(rrpc->kgc_wq);
-
-	rrpc_free(rrpc);
-}
-
-static sector_t rrpc_capacity(void *private)
-{
-	struct rrpc *rrpc = private;
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	sector_t reserved, provisioned;
-
-	/* cur, gc, and two emergency blocks for each lun */
-	reserved = rrpc->nr_luns * dev->geo.sec_per_blk * 4;
-	provisioned = rrpc->nr_sects - reserved;
-
-	if (reserved > rrpc->nr_sects) {
-		pr_err("rrpc: not enough space available to expose storage.\n");
-		return 0;
-	}
-
-	sector_div(provisioned, 10);
-	return provisioned * 9 * NR_PHY_IN_LOG;
-}
-
-/*
- * Looks up the logical address from reverse trans map and check if its valid by
- * comparing the logical to physical address with the physical address.
- * Returns 0 on free, otherwise 1 if in use
- */
-static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	int offset;
-	struct rrpc_addr *laddr;
-	u64 bpaddr, paddr, pladdr;
-
-	bpaddr = block_to_rel_addr(rrpc, rblk);
-	for (offset = 0; offset < dev->geo.sec_per_blk; offset++) {
-		paddr = bpaddr + offset;
-
-		pladdr = rrpc->rev_trans_map[paddr].addr;
-		if (pladdr == ADDR_EMPTY)
-			continue;
-
-		laddr = &rrpc->trans_map[pladdr];
-
-		if (paddr == laddr->addr) {
-			laddr->rblk = rblk;
-		} else {
-			set_bit(offset, rblk->invalid_pages);
-			rblk->nr_invalid_pages++;
-		}
-	}
-}
-
-static int rrpc_blocks_init(struct rrpc *rrpc)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct rrpc_lun *rlun;
-	struct rrpc_block *rblk;
-	int lun_iter, blk_iter;
-
-	for (lun_iter = 0; lun_iter < rrpc->nr_luns; lun_iter++) {
-		rlun = &rrpc->luns[lun_iter];
-
-		for (blk_iter = 0; blk_iter < dev->geo.blks_per_lun;
-								blk_iter++) {
-			rblk = &rlun->blocks[blk_iter];
-			rrpc_block_map_update(rrpc, rblk);
-		}
-	}
-
-	return 0;
-}
-
-static int rrpc_luns_configure(struct rrpc *rrpc)
-{
-	struct rrpc_lun *rlun;
-	struct rrpc_block *rblk;
-	int i;
-
-	for (i = 0; i < rrpc->nr_luns; i++) {
-		rlun = &rrpc->luns[i];
-
-		rblk = rrpc_get_blk(rrpc, rlun, 0);
-		if (!rblk)
-			goto err;
-		rrpc_set_lun_cur(rlun, rblk, &rlun->cur);
-
-		/* Emergency gc block */
-		rblk = rrpc_get_blk(rrpc, rlun, 1);
-		if (!rblk)
-			goto err;
-		rrpc_set_lun_cur(rlun, rblk, &rlun->gc_cur);
-	}
-
-	return 0;
-err:
-	rrpc_put_blks(rrpc);
-	return -EINVAL;
-}
-
-static struct nvm_tgt_type tt_rrpc;
-
-static void *rrpc_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
-		       int flags)
-{
-	struct request_queue *bqueue = dev->q;
-	struct request_queue *tqueue = tdisk->queue;
-	struct nvm_geo *geo = &dev->geo;
-	struct rrpc *rrpc;
-	sector_t soffset;
-	int ret;
-
-	if (!(dev->identity.dom & NVM_RSP_L2P)) {
-		pr_err("nvm: rrpc: device does not support l2p (%x)\n",
-							dev->identity.dom);
-		return ERR_PTR(-EINVAL);
-	}
-
-	rrpc = kzalloc(sizeof(struct rrpc), GFP_KERNEL);
-	if (!rrpc)
-		return ERR_PTR(-ENOMEM);
-
-	rrpc->dev = dev;
-	rrpc->disk = tdisk;
-
-	bio_list_init(&rrpc->requeue_bios);
-	spin_lock_init(&rrpc->bio_lock);
-	INIT_WORK(&rrpc->ws_requeue, rrpc_requeue);
-
-	rrpc->nr_luns = geo->nr_luns;
-	rrpc->nr_sects = (unsigned long long)geo->sec_per_lun * rrpc->nr_luns;
-
-	/* simple round-robin strategy */
-	atomic_set(&rrpc->next_lun, -1);
-
-	ret = rrpc_area_init(rrpc, &soffset);
-	if (ret < 0) {
-		pr_err("nvm: rrpc: could not initialize area\n");
-		return ERR_PTR(ret);
-	}
-	rrpc->soffset = soffset;
-
-	ret = rrpc_luns_init(rrpc, dev->luns);
-	if (ret) {
-		pr_err("nvm: rrpc: could not initialize luns\n");
-		goto err;
-	}
-
-	ret = rrpc_core_init(rrpc);
-	if (ret) {
-		pr_err("nvm: rrpc: could not initialize core\n");
-		goto err;
-	}
-
-	ret = rrpc_map_init(rrpc);
-	if (ret) {
-		pr_err("nvm: rrpc: could not initialize maps\n");
-		goto err;
-	}
-
-	ret = rrpc_blocks_init(rrpc);
-	if (ret) {
-		pr_err("nvm: rrpc: could not initialize state for blocks\n");
-		goto err;
-	}
-
-	ret = rrpc_luns_configure(rrpc);
-	if (ret) {
-		pr_err("nvm: rrpc: not enough blocks available in LUNs.\n");
-		goto err;
-	}
-
-	ret = rrpc_gc_init(rrpc);
-	if (ret) {
-		pr_err("nvm: rrpc: could not initialize gc\n");
-		goto err;
-	}
-
-	/* inherit the size from the underlying device */
-	blk_queue_logical_block_size(tqueue, queue_physical_block_size(bqueue));
-	blk_queue_max_hw_sectors(tqueue, queue_max_hw_sectors(bqueue));
-
-	pr_info("nvm: rrpc initialized with %u luns and %llu pages.\n",
-			rrpc->nr_luns, (unsigned long long)rrpc->nr_sects);
-
-	mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10));
-
-	return rrpc;
-err:
-	rrpc_free(rrpc);
-	return ERR_PTR(ret);
-}
-
-/* round robin, page-based FTL, and cost-based GC */
-static struct nvm_tgt_type tt_rrpc = {
-	.name		= "rrpc",
-	.version	= {1, 0, 0},
-
-	.make_rq	= rrpc_make_rq,
-	.capacity	= rrpc_capacity,
-
-	.init		= rrpc_init,
-	.exit		= rrpc_exit,
-};
-
-static int __init rrpc_module_init(void)
-{
-	return nvm_register_tgt_type(&tt_rrpc);
-}
-
-static void rrpc_module_exit(void)
-{
-	nvm_unregister_tgt_type(&tt_rrpc);
-}
-
-module_init(rrpc_module_init);
-module_exit(rrpc_module_exit);
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("Block-Device Target for Open-Channel SSDs");
diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h
deleted file mode 100644
index fdb6ff902903..000000000000
--- a/drivers/lightnvm/rrpc.h
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Copyright (C) 2015 IT University of Copenhagen
- * Initial release: Matias Bjorling <m@bjorling.me>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License version
- * 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs.
- */
-
-#ifndef RRPC_H_
-#define RRPC_H_
-
-#include <linux/blkdev.h>
-#include <linux/blk-mq.h>
-#include <linux/bio.h>
-#include <linux/module.h>
-#include <linux/kthread.h>
-#include <linux/vmalloc.h>
-
-#include <linux/lightnvm.h>
-
-/* Run only GC if less than 1/X blocks are free */
-#define GC_LIMIT_INVERSE 10
-#define GC_TIME_SECS 100
-
-#define RRPC_SECTOR (512)
-#define RRPC_EXPOSED_PAGE_SIZE (4096)
-
-#define NR_PHY_IN_LOG (RRPC_EXPOSED_PAGE_SIZE / RRPC_SECTOR)
-
-struct rrpc_inflight {
-	struct list_head reqs;
-	spinlock_t lock;
-};
-
-struct rrpc_inflight_rq {
-	struct list_head list;
-	sector_t l_start;
-	sector_t l_end;
-};
-
-struct rrpc_rq {
-	struct rrpc_inflight_rq inflight_rq;
-	unsigned long flags;
-};
-
-struct rrpc_block {
-	int id;				/* id inside of LUN */
-	struct rrpc_lun *rlun;
-
-	struct list_head prio;		/* LUN CG list */
-	struct list_head list;		/* LUN free, used, bb list */
-
-#define MAX_INVALID_PAGES_STORAGE 8
-	/* Bitmap for invalid page intries */
-	unsigned long invalid_pages[MAX_INVALID_PAGES_STORAGE];
-	/* points to the next writable page within a block */
-	unsigned int next_page;
-	/* number of pages that are invalid, wrt host page size */
-	unsigned int nr_invalid_pages;
-
-	int state;
-
-	spinlock_t lock;
-	atomic_t data_cmnt_size; /* data pages committed to stable storage */
-};
-
-struct rrpc_lun {
-	struct rrpc *rrpc;
-
-	int id;
-	struct ppa_addr bppa;
-
-	struct rrpc_block *cur, *gc_cur;
-	struct rrpc_block *blocks;	/* Reference to block allocation */
-
-	struct list_head prio_list;	/* Blocks that may be GC'ed */
-	struct list_head wblk_list;	/* Queued blocks to be written to */
-
-	/* lun block lists */
-	struct list_head used_list;	/* In-use blocks */
-	struct list_head free_list;	/* Not used blocks i.e. released
-					 * and ready for use
-					 */
-	struct list_head bb_list;	/* Bad blocks. Mutually exclusive with
-					 * free_list and used_list
-					 */
-	unsigned int nr_free_blocks;	/* Number of unused blocks */
-
-	struct work_struct ws_gc;
-
-	int reserved_blocks;
-
-	spinlock_t lock;
-};
-
-struct rrpc {
-	struct nvm_tgt_dev *dev;
-	struct gendisk *disk;
-
-	sector_t soffset; /* logical sector offset */
-
-	int nr_luns;
-	struct rrpc_lun *luns;
-
-	/* calculated values */
-	unsigned long long nr_sects;
-
-	/* Write strategy variables. Move these into each for structure for each
-	 * strategy
-	 */
-	atomic_t next_lun; /* Whenever a page is written, this is updated
-			    * to point to the next write lun
-			    */
-
-	spinlock_t bio_lock;
-	struct bio_list requeue_bios;
-	struct work_struct ws_requeue;
-
-	/* Simple translation map of logical addresses to physical addresses.
-	 * The logical addresses is known by the host system, while the physical
-	 * addresses are used when writing to the disk block device.
-	 */
-	struct rrpc_addr *trans_map;
-	/* also store a reverse map for garbage collection */
-	struct rrpc_rev_addr *rev_trans_map;
-	spinlock_t rev_lock;
-
-	struct rrpc_inflight inflights;
-
-	mempool_t *addr_pool;
-	mempool_t *page_pool;
-	mempool_t *gcb_pool;
-	mempool_t *rq_pool;
-
-	struct timer_list gc_timer;
-	struct workqueue_struct *krqd_wq;
-	struct workqueue_struct *kgc_wq;
-};
-
-struct rrpc_block_gc {
-	struct rrpc *rrpc;
-	struct rrpc_block *rblk;
-	struct work_struct ws_gc;
-};
-
-/* Logical to physical mapping */
-struct rrpc_addr {
-	u64 addr;
-	struct rrpc_block *rblk;
-};
-
-/* Physical to logical mapping */
-struct rrpc_rev_addr {
-	u64 addr;
-};
-
-static inline struct ppa_addr rrpc_linear_to_generic_addr(struct nvm_geo *geo,
-							  struct ppa_addr r)
-{
-	struct ppa_addr l;
-	int secs, pgs;
-	sector_t ppa = r.ppa;
-
-	l.ppa = 0;
-
-	div_u64_rem(ppa, geo->sec_per_pg, &secs);
-	l.g.sec = secs;
-
-	sector_div(ppa, geo->sec_per_pg);
-	div_u64_rem(ppa, geo->pgs_per_blk, &pgs);
-	l.g.pg = pgs;
-
-	return l;
-}
-
-static inline struct ppa_addr rrpc_recov_addr(struct nvm_tgt_dev *dev, u64 pba)
-{
-	return linear_to_generic_addr(&dev->geo, pba);
-}
-
-static inline u64 rrpc_blk_to_ppa(struct rrpc *rrpc, struct rrpc_block *rblk)
-{
-	struct nvm_tgt_dev *dev = rrpc->dev;
-	struct nvm_geo *geo = &dev->geo;
-	struct rrpc_lun *rlun = rblk->rlun;
-
-	return (rlun->id * geo->sec_per_lun) + (rblk->id * geo->sec_per_blk);
-}
-
-static inline sector_t rrpc_get_laddr(struct bio *bio)
-{
-	return bio->bi_iter.bi_sector / NR_PHY_IN_LOG;
-}
-
-static inline unsigned int rrpc_get_pages(struct bio *bio)
-{
-	return  bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE;
-}
-
-static inline sector_t rrpc_get_sector(sector_t laddr)
-{
-	return laddr * NR_PHY_IN_LOG;
-}
-
-static inline int request_intersects(struct rrpc_inflight_rq *r,
-				sector_t laddr_start, sector_t laddr_end)
-{
-	return (laddr_end >= r->l_start) && (laddr_start <= r->l_end);
-}
-
-static int __rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr,
-			     unsigned int pages, struct rrpc_inflight_rq *r)
-{
-	sector_t laddr_end = laddr + pages - 1;
-	struct rrpc_inflight_rq *rtmp;
-
-	WARN_ON(irqs_disabled());
-
-	spin_lock_irq(&rrpc->inflights.lock);
-	list_for_each_entry(rtmp, &rrpc->inflights.reqs, list) {
-		if (unlikely(request_intersects(rtmp, laddr, laddr_end))) {
-			/* existing, overlapping request, come back later */
-			spin_unlock_irq(&rrpc->inflights.lock);
-			return 1;
-		}
-	}
-
-	r->l_start = laddr;
-	r->l_end = laddr_end;
-
-	list_add_tail(&r->list, &rrpc->inflights.reqs);
-	spin_unlock_irq(&rrpc->inflights.lock);
-	return 0;
-}
-
-static inline int rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr,
-				 unsigned int pages,
-				 struct rrpc_inflight_rq *r)
-{
-	BUG_ON((laddr + pages) > rrpc->nr_sects);
-
-	return __rrpc_lock_laddr(rrpc, laddr, pages, r);
-}
-
-static inline struct rrpc_inflight_rq *rrpc_get_inflight_rq(struct nvm_rq *rqd)
-{
-	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
-
-	return &rrqd->inflight_rq;
-}
-
-static inline int rrpc_lock_rq(struct rrpc *rrpc, struct bio *bio,
-							struct nvm_rq *rqd)
-{
-	sector_t laddr = rrpc_get_laddr(bio);
-	unsigned int pages = rrpc_get_pages(bio);
-	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
-
-	return rrpc_lock_laddr(rrpc, laddr, pages, r);
-}
-
-static inline void rrpc_unlock_laddr(struct rrpc *rrpc,
-						struct rrpc_inflight_rq *r)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&rrpc->inflights.lock, flags);
-	list_del_init(&r->list);
-	spin_unlock_irqrestore(&rrpc->inflights.lock, flags);
-}
-
-static inline void rrpc_unlock_rq(struct rrpc *rrpc, struct nvm_rq *rqd)
-{
-	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
-	uint8_t pages = rqd->nr_ppas;
-
-	BUG_ON((r->l_start + pages) > rrpc->nr_sects);
-
-	rrpc_unlock_laddr(rrpc, r);
-}
-
-#endif /* RRPC_H_ */
diff --git a/drivers/macintosh/adb.c b/drivers/macintosh/adb.c
index 289800b5235d..4c8097e0e6fe 100644
--- a/drivers/macintosh/adb.c
+++ b/drivers/macintosh/adb.c
@@ -203,18 +203,17 @@ static int adb_scan_bus(void)
 	}
 
 	/* Now fill in the handler_id field of the adb_handler entries. */
-	printk(KERN_DEBUG "adb devices:");
+	pr_debug("adb devices:\n");
 	for (i = 1; i < 16; i++) {
 		if (adb_handler[i].original_address == 0)
 			continue;
 		adb_request(&req, NULL, ADBREQ_SYNC | ADBREQ_REPLY, 1,
 			    (i << 4) | 0xf);
 		adb_handler[i].handler_id = req.reply[2];
-		printk(" [%d]: %d %x", i, adb_handler[i].original_address,
-		       adb_handler[i].handler_id);
+		pr_debug(" [%d]: %d %x\n", i, adb_handler[i].original_address,
+			 adb_handler[i].handler_id);
 		devmask |= 1 << i;
 	}
-	printk("\n");
 	return devmask;
 }
 
@@ -225,9 +224,9 @@ static int adb_scan_bus(void)
 static int
 adb_probe_task(void *x)
 {
-	printk(KERN_INFO "adb: starting probe task...\n");
+	pr_debug("adb: starting probe task...\n");
 	do_adb_reset_bus();
-	printk(KERN_INFO "adb: finished probe task...\n");
+	pr_debug("adb: finished probe task...\n");
 
 	up(&adb_probe_mutex);
 
@@ -337,7 +336,7 @@ static int __init adb_init(void)
 	    adb_controller->init())
 		adb_controller = NULL;
 	if (adb_controller == NULL) {
-		printk(KERN_WARNING "Warning: no ADB interface detected\n");
+		pr_warn("Warning: no ADB interface detected\n");
 	} else {
 #ifdef CONFIG_PPC
 		if (of_machine_is_compatible("AAPL,PowerBook1998") ||
@@ -480,8 +479,7 @@ adb_register(int default_id, int handler_id, struct adb_ids *ids,
 		    (!handler_id || (handler_id == adb_handler[i].handler_id) || 
 		    try_handler_change(i, handler_id))) {
 			if (adb_handler[i].handler != 0) {
-				printk(KERN_ERR
-				       "Two handlers for ADB device %d\n",
+				pr_err("Two handlers for ADB device %d\n",
 				       default_id);
 				continue;
 			}
@@ -535,10 +533,10 @@ adb_input(unsigned char *buf, int nb, int autopoll)
 		
 	id = buf[0] >> 4;
 	if (dump_adb_input) {
-		printk(KERN_INFO "adb packet: ");
+		pr_info("adb packet: ");
 		for (i = 0; i < nb; ++i)
-			printk(" %x", buf[i]);
-		printk(", id = %d\n", id);
+			pr_cont(" %x", buf[i]);
+		pr_cont(", id = %d\n", id);
 	}
 	write_lock_irqsave(&adb_handler_lock, flags);
 	handler = adb_handler[id].handler;
@@ -884,7 +882,7 @@ static void __init
 adbdev_init(void)
 {
 	if (register_chrdev(ADB_MAJOR, "adb", &adb_fops)) {
-		printk(KERN_ERR "adb: unable to get major %d\n", ADB_MAJOR);
+		pr_err("adb: unable to get major %d\n", ADB_MAJOR);
 		return;
 	}
 
diff --git a/drivers/macintosh/adbhid.c b/drivers/macintosh/adbhid.c
index e091193104f7..a261892c03b3 100644
--- a/drivers/macintosh/adbhid.c
+++ b/drivers/macintosh/adbhid.c
@@ -268,7 +268,7 @@ adbhid_keyboard_input(unsigned char *data, int nb, int apoll)
 	int id = (data[0] >> 4) & 0x0f;
 
 	if (!adbhid[id]) {
-		printk(KERN_ERR "ADB HID on ID %d not yet registered, packet %#02x, %#02x, %#02x, %#02x\n",
+		pr_err("ADB HID on ID %d not yet registered, packet %#02x, %#02x, %#02x, %#02x\n",
 		       id, data[0], data[1], data[2], data[3]);
 		return;
 	}
@@ -320,8 +320,7 @@ adbhid_input_keycode(int id, int scancode, int repeat)
 					ahid->flags &= ~FLAG_CAPSLOCK_TRANSLATE;
 				}
 			} else {
-				printk(KERN_INFO "Spurious caps lock event "
-						 "(scancode 0xff).\n");
+				pr_info("Spurious caps lock event (scancode 0xff).\n");
 			}
 		}
 	}
@@ -397,8 +396,8 @@ adbhid_input_keycode(int id, int scancode, int repeat)
 		input_report_key(adbhid[id]->input, key, !up_flag);
 		input_sync(adbhid[id]->input);
 	} else
-		printk(KERN_INFO "Unhandled ADB key (scancode %#02x) %s.\n", keycode,
-		       up_flag ? "released" : "pressed");
+		pr_info("Unhandled ADB key (scancode %#02x) %s.\n", keycode,
+			up_flag ? "released" : "pressed");
 
 }
 
@@ -408,7 +407,7 @@ adbhid_mouse_input(unsigned char *data, int nb, int autopoll)
 	int id = (data[0] >> 4) & 0x0f;
 
 	if (!adbhid[id]) {
-		printk(KERN_ERR "ADB HID on ID %d not yet registered\n", id);
+		pr_err("ADB HID on ID %d not yet registered\n", id);
 		return;
 	}
 
@@ -506,7 +505,7 @@ adbhid_buttons_input(unsigned char *data, int nb, int autopoll)
 	int id = (data[0] >> 4) & 0x0f;
 
 	if (!adbhid[id]) {
-		printk(KERN_ERR "ADB HID on ID %d not yet registered\n", id);
+		pr_err("ADB HID on ID %d not yet registered\n", id);
 		return;
 	}
 
@@ -534,8 +533,8 @@ adbhid_buttons_input(unsigned char *data, int nb, int autopoll)
 			break;
 
 		default:
-			printk(KERN_INFO "Unhandled ADB_MISC event %02x, %02x, %02x, %02x\n",
-			       data[0], data[1], data[2], data[3]);
+			pr_info("Unhandled ADB_MISC event %02x, %02x, %02x, %02x\n",
+				data[0], data[1], data[2], data[3]);
 			break;
 		}
 	  }
@@ -609,14 +608,14 @@ adbhid_buttons_input(unsigned char *data, int nb, int autopoll)
 				break;
 
 			default:
-				printk(KERN_INFO "Unhandled ADB_MISC event %02x, %02x, %02x, %02x\n",
-				       data[0], data[1], data[2], data[3]);
+				pr_info("Unhandled ADB_MISC event %02x, %02x, %02x, %02x\n",
+					data[0], data[1], data[2], data[3]);
 				break;
 			}
 			break;
 		default:
-			printk(KERN_INFO "Unhandled ADB_MISC event %02x, %02x, %02x, %02x\n",
-			       data[0], data[1], data[2], data[3]);
+			pr_info("Unhandled ADB_MISC event %02x, %02x, %02x, %02x\n",
+				data[0], data[1], data[2], data[3]);
 			break;
 		}
 	  }
@@ -760,7 +759,7 @@ adbhid_input_register(int id, int default_id, int original_handler_id,
 	int i;
 
 	if (adbhid[id]) {
-		printk(KERN_ERR "Trying to reregister ADB HID on ID %d\n", id);
+		pr_err("Trying to reregister ADB HID on ID %d\n", id);
 		return -EEXIST;
 	}
 
@@ -799,24 +798,24 @@ adbhid_input_register(int id, int default_id, int original_handler_id,
 
 		memcpy(hid->keycode, adb_to_linux_keycodes, sizeof(adb_to_linux_keycodes));
 
-		printk(KERN_INFO "Detected ADB keyboard, type ");
+		pr_info("Detected ADB keyboard, type ");
 		switch (original_handler_id) {
 		default:
-			printk("<unknown>.\n");
+			pr_cont("<unknown>.\n");
 			input_dev->id.version = ADB_KEYBOARD_UNKNOWN;
 			break;
 
 		case 0x01: case 0x02: case 0x03: case 0x06: case 0x08:
 		case 0x0C: case 0x10: case 0x18: case 0x1B: case 0x1C:
 		case 0xC0: case 0xC3: case 0xC6:
-			printk("ANSI.\n");
+			pr_cont("ANSI.\n");
 			input_dev->id.version = ADB_KEYBOARD_ANSI;
 			break;
 
 		case 0x04: case 0x05: case 0x07: case 0x09: case 0x0D:
 		case 0x11: case 0x14: case 0x19: case 0x1D: case 0xC1:
 		case 0xC4: case 0xC7:
-			printk("ISO, swapping keys.\n");
+			pr_cont("ISO, swapping keys.\n");
 			input_dev->id.version = ADB_KEYBOARD_ISO;
 			i = hid->keycode[10];
 			hid->keycode[10] = hid->keycode[50];
@@ -825,7 +824,7 @@ adbhid_input_register(int id, int default_id, int original_handler_id,
 
 		case 0x12: case 0x15: case 0x16: case 0x17: case 0x1A:
 		case 0x1E: case 0xC2: case 0xC5: case 0xC8: case 0xC9:
-			printk("JIS.\n");
+			pr_cont("JIS.\n");
 			input_dev->id.version = ADB_KEYBOARD_JIS;
 			break;
 		}
@@ -884,7 +883,7 @@ adbhid_input_register(int id, int default_id, int original_handler_id,
 		/* else fall through */
 
 	default:
-		printk(KERN_INFO "Trying to register unknown ADB device to input layer.\n");
+		pr_info("Trying to register unknown ADB device to input layer.\n");
 		err = -ENODEV;
 		goto fail;
 	}
@@ -1073,12 +1072,12 @@ adbhid_probe(void)
 			    (req.reply[1] == 0x4b) && (req.reply[2] == 0x4f) &&
 			    (req.reply[3] == 0x49) && (req.reply[4] == 0x54)) {
 				if (adb_try_handler_change(id, 0x42)) {
-					printk("\nADB MacAlly 2-button mouse at %d, handler set to 0x42", id);
+					pr_cont("\nADB MacAlly 2-button mouse at %d, handler set to 0x42", id);
 					mouse_kind = ADBMOUSE_MACALLY2;
 				}
 			}
 		}
-		printk("\n");
+		pr_cont("\n");
 
 		adb_get_infos(id, &default_id, &cur_handler_id);
 		reg |= adbhid_input_reregister(id, default_id, org_handler_id,
@@ -1093,12 +1092,12 @@ init_trackpad(int id)
 	struct adb_request req;
 	unsigned char r1_buffer[8];
 
-	printk(" (trackpad)");
+	pr_cont(" (trackpad)");
 
 	adb_request(&req, NULL, ADBREQ_SYNC | ADBREQ_REPLY, 1,
 		    ADB_READREG(id,1));
 	if (req.reply_len < 8)
-	    printk("bad length for reg. 1\n");
+	    pr_cont("bad length for reg. 1\n");
 	else
 	{
 	    memcpy(r1_buffer, &req.reply[1], 8);
@@ -1146,7 +1145,7 @@ init_trackball(int id)
 {
 	struct adb_request req;
 
-	printk(" (trackman/mouseman)");
+	pr_cont(" (trackman/mouseman)");
 
 	adb_request(&req, NULL, ADBREQ_SYNC, 3,
 	ADB_WRITEREG(id,1), 00,0x81);
@@ -1178,7 +1177,7 @@ init_turbomouse(int id)
 {
 	struct adb_request req;
 
-        printk(" (TurboMouse 5)");
+	pr_cont(" (TurboMouse 5)");
 
 	adb_request(&req, NULL, ADBREQ_SYNC, 1, ADB_FLUSH(id));
 
@@ -1214,7 +1213,7 @@ init_microspeed(int id)
 {
 	struct adb_request req;
 
-        printk(" (Microspeed/MacPoint or compatible)");
+	pr_cont(" (Microspeed/MacPoint or compatible)");
 
 	adb_request(&req, NULL, ADBREQ_SYNC, 1, ADB_FLUSH(id));
 
@@ -1254,7 +1253,7 @@ init_ms_a3(int id)
 {
 	struct adb_request req;
 
-	printk(" (Mouse Systems A3 Mouse, or compatible)");
+	pr_cont(" (Mouse Systems A3 Mouse, or compatible)");
 	adb_request(&req, NULL, ADBREQ_SYNC, 3,
 	ADB_WRITEREG(id, 0x2),
 	    0x00,
diff --git a/drivers/macintosh/ams/ams-input.c b/drivers/macintosh/ams/ams-input.c
index 2edae7dfcab2..fe248f6a30c1 100644
--- a/drivers/macintosh/ams/ams-input.c
+++ b/drivers/macintosh/ams/ams-input.c
@@ -91,7 +91,7 @@ static int ams_input_enable(void)
 		return error;
 	}
 
-	joystick = 1;
+	joystick = true;
 
 	return 0;
 }
@@ -104,7 +104,7 @@ static void ams_input_disable(void)
 		ams_info.idev = NULL;
 	}
 
-	joystick = 0;
+	joystick = false;
 }
 
 static ssize_t ams_input_show_joystick(struct device *dev,
diff --git a/drivers/macintosh/macio_asic.c b/drivers/macintosh/macio_asic.c
index 62f541f968f6..07074820a167 100644
--- a/drivers/macintosh/macio_asic.c
+++ b/drivers/macintosh/macio_asic.c
@@ -375,6 +375,7 @@ static struct macio_dev * macio_add_one_device(struct macio_chip *chip,
 	dev->ofdev.dev.of_node = np;
 	dev->ofdev.archdata.dma_mask = 0xffffffffUL;
 	dev->ofdev.dev.dma_mask = &dev->ofdev.archdata.dma_mask;
+	dev->ofdev.dev.coherent_dma_mask = dev->ofdev.archdata.dma_mask;
 	dev->ofdev.dev.parent = parent;
 	dev->ofdev.dev.bus = &macio_bus_type;
 	dev->ofdev.dev.release = macio_release_dev;
diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c
index 899ec1f4c833..e8ae2e54151c 100644
--- a/drivers/macintosh/smu.c
+++ b/drivers/macintosh/smu.c
@@ -1245,10 +1245,10 @@ static ssize_t smu_read(struct file *file, char __user *buf,
 	return -EBADFD;
 }
 
-static unsigned int smu_fpoll(struct file *file, poll_table *wait)
+static __poll_t smu_fpoll(struct file *file, poll_table *wait)
 {
 	struct smu_private *pp = file->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	unsigned long flags;
 
 	if (pp == 0)
@@ -1259,7 +1259,7 @@ static unsigned int smu_fpoll(struct file *file, poll_table *wait)
 
 		spin_lock_irqsave(&pp->lock, flags);
 		if (pp->busy && pp->cmd.status != 1)
-			mask |= POLLIN;
+			mask |= EPOLLIN;
 		spin_unlock_irqrestore(&pp->lock, flags);
 	}
 	if (pp->mode == smu_file_events) {
diff --git a/drivers/macintosh/therm_adt746x.c b/drivers/macintosh/therm_adt746x.c
index f433521a6f9d..d7cd5afa38cd 100644
--- a/drivers/macintosh/therm_adt746x.c
+++ b/drivers/macintosh/therm_adt746x.c
@@ -230,7 +230,7 @@ static void update_fans_speed (struct thermostat *th)
 
 	/* we don't care about local sensor, so we start at sensor 1 */
 	for (i = 1; i < 3; i++) {
-		int started = 0;
+		bool started = false;
 		int fan_number = (th->type == ADT7460 && i == 2);
 		int var = th->temps[i] - th->limits[i];
 
@@ -243,7 +243,7 @@ static void update_fans_speed (struct thermostat *th)
 			if (abs(var - th->last_var[fan_number]) < 2)
 				continue;
 
-			started = 1;
+			started = true;
 			new_speed = fan_speed + ((var-1)*step);
 
 			if (new_speed < fan_speed)
diff --git a/drivers/macintosh/via-pmu-backlight.c b/drivers/macintosh/via-pmu-backlight.c
index 89ed51571b62..50ada02ae75d 100644
--- a/drivers/macintosh/via-pmu-backlight.c
+++ b/drivers/macintosh/via-pmu-backlight.c
@@ -137,7 +137,7 @@ void pmu_backlight_set_sleep(int sleep)
 }
 #endif /* CONFIG_PM */
 
-void __init pmu_backlight_init()
+void __init pmu_backlight_init(void)
 {
 	struct backlight_properties props;
 	struct backlight_device *bd;
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index c4c2b3b85ebc..94c0f3f7df69 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -1799,7 +1799,7 @@ static int powerbook_sleep_grackle(void)
 	struct adb_request req;
 	struct pci_dev *grackle;
 
-	grackle = pci_get_bus_and_slot(0, 0);
+	grackle = pci_get_domain_bus_and_slot(0, 0, 0);
 	if (!grackle)
 		return -ENODEV;
 
@@ -2157,11 +2157,11 @@ pmu_write(struct file *file, const char __user *buf,
 	return 0;
 }
 
-static unsigned int
+static __poll_t
 pmu_fpoll(struct file *filp, poll_table *wait)
 {
 	struct pmu_private *pp = filp->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	unsigned long flags;
 	
 	if (pp == 0)
@@ -2169,7 +2169,7 @@ pmu_fpoll(struct file *filp, poll_table *wait)
 	poll_wait(filp, &pp->wait, wait);
 	spin_lock_irqsave(&pp->lock, flags);
 	if (pp->rb_get != pp->rb_put)
-		mask |= POLLIN;
+		mask |= EPOLLIN;
 	spin_unlock_irqrestore(&pp->lock, flags);
 	return mask;
 }
diff --git a/drivers/macintosh/windfarm_pm112.c b/drivers/macintosh/windfarm_pm112.c
index 96d16fca68b2..fec91db1142e 100644
--- a/drivers/macintosh/windfarm_pm112.c
+++ b/drivers/macintosh/windfarm_pm112.c
@@ -96,14 +96,14 @@ static int cpu_last_target;
 static struct wf_pid_state backside_pid;
 static int backside_tick;
 static struct wf_pid_state slots_pid;
-static int slots_started;
+static bool slots_started;
 static struct wf_pid_state drive_bay_pid;
 static int drive_bay_tick;
 
 static int nr_cores;
 static int have_all_controls;
 static int have_all_sensors;
-static int started;
+static bool started;
 
 static int failure_state;
 #define FAILURE_SENSOR		1
@@ -462,7 +462,7 @@ static void slots_fan_tick(void)
 		/* first time; initialize things */
 		printk(KERN_INFO "windfarm: Slots control loop started.\n");
 		wf_pid_init(&slots_pid, &slots_param);
-		slots_started = 1;
+		slots_started = true;
 	}
 
 	err = slots_power->ops->get_value(slots_power, &power);
@@ -506,7 +506,7 @@ static void pm112_tick(void)
 	int i, last_failure;
 
 	if (!started) {
-		started = 1;
+		started = true;
 		printk(KERN_INFO "windfarm: CPUs control loops started.\n");
 		for (i = 0; i < nr_cores; ++i) {
 			if (create_cpu_loop(i) < 0) {
diff --git a/drivers/macintosh/windfarm_pm121.c b/drivers/macintosh/windfarm_pm121.c
index b350fb86ff08..4d72d8f58cb6 100644
--- a/drivers/macintosh/windfarm_pm121.c
+++ b/drivers/macintosh/windfarm_pm121.c
@@ -246,7 +246,8 @@ enum {
 static struct wf_control *controls[N_CONTROLS] = {};
 
 /* Set to kick the control loop into life */
-static int pm121_all_controls_ok, pm121_all_sensors_ok, pm121_started;
+static int pm121_all_controls_ok, pm121_all_sensors_ok;
+static bool pm121_started;
 
 enum {
 	FAILURE_FAN		= 1 << 0,
@@ -806,7 +807,7 @@ static void pm121_tick(void)
 			pm121_create_sys_fans(i);
 
 		pm121_create_cpu_fans();
-		pm121_started = 1;
+		pm121_started = true;
 	}
 
 	/* skipping ticks */
diff --git a/drivers/macintosh/windfarm_pm72.c b/drivers/macintosh/windfarm_pm72.c
index e88cfb36a74d..833021508c05 100644
--- a/drivers/macintosh/windfarm_pm72.c
+++ b/drivers/macintosh/windfarm_pm72.c
@@ -611,7 +611,7 @@ static void pm72_tick(void)
 	int i, last_failure;
 
 	if (!started) {
-		started = 1;
+		started = true;
 		printk(KERN_INFO "windfarm: CPUs control loops started.\n");
 		for (i = 0; i < nr_chips; ++i) {
 			if (cpu_setup_pid(i) < 0) {
diff --git a/drivers/macintosh/windfarm_pm81.c b/drivers/macintosh/windfarm_pm81.c
index 93faf298a3c5..d9ea45581b9e 100644
--- a/drivers/macintosh/windfarm_pm81.c
+++ b/drivers/macintosh/windfarm_pm81.c
@@ -140,7 +140,8 @@ static struct wf_control *fan_system;
 static struct wf_control *cpufreq_clamp;
 
 /* Set to kick the control loop into life */
-static int wf_smu_all_controls_ok, wf_smu_all_sensors_ok, wf_smu_started;
+static int wf_smu_all_controls_ok, wf_smu_all_sensors_ok;
+static bool wf_smu_started;
 
 /* Failure handling.. could be nicer */
 #define FAILURE_FAN		0x01
@@ -549,7 +550,7 @@ static void wf_smu_tick(void)
 		DBG("wf: creating control loops !\n");
 		wf_smu_create_sys_fans();
 		wf_smu_create_cpu_fans();
-		wf_smu_started = 1;
+		wf_smu_started = true;
 	}
 
 	/* Skipping ticks */
diff --git a/drivers/macintosh/windfarm_pm91.c b/drivers/macintosh/windfarm_pm91.c
index 81fdf40c5b82..7fd73dcb2b0a 100644
--- a/drivers/macintosh/windfarm_pm91.c
+++ b/drivers/macintosh/windfarm_pm91.c
@@ -75,7 +75,8 @@ static struct wf_control *fan_slots;
 static struct wf_control *cpufreq_clamp;
 
 /* Set to kick the control loop into life */
-static int wf_smu_all_controls_ok, wf_smu_all_sensors_ok, wf_smu_started;
+static int wf_smu_all_controls_ok, wf_smu_all_sensors_ok;
+static bool wf_smu_started;
 static bool wf_smu_overtemp;
 
 /* Failure handling.. could be nicer */
@@ -467,7 +468,7 @@ static void wf_smu_tick(void)
 		wf_smu_create_drive_fans();
 		wf_smu_create_slots_fans();
 		wf_smu_create_cpu_fans();
-		wf_smu_started = 1;
+		wf_smu_started = true;
 	}
 
 	/* Skipping ticks */
diff --git a/drivers/macintosh/windfarm_rm31.c b/drivers/macintosh/windfarm_rm31.c
index a0cd9c7f9835..9ce87cc0597f 100644
--- a/drivers/macintosh/windfarm_rm31.c
+++ b/drivers/macintosh/windfarm_rm31.c
@@ -514,7 +514,7 @@ static void rm31_tick(void)
 	int i, last_failure;
 
 	if (!started) {
-		started = 1;
+		started = true;
 		printk(KERN_INFO "windfarm: CPUs control loops started.\n");
 		for (i = 0; i < nr_chips; ++i) {
 			if (cpu_setup_pid(i) < 0) {
diff --git a/drivers/mailbox/mailbox-test.c b/drivers/mailbox/mailbox-test.c
index 93f3d4d61fa7..58bfafc34bc4 100644
--- a/drivers/mailbox/mailbox-test.c
+++ b/drivers/mailbox/mailbox-test.c
@@ -235,7 +235,7 @@ static ssize_t mbox_test_message_read(struct file *filp, char __user *userbuf,
 	return ret;
 }
 
-static unsigned int
+static __poll_t
 mbox_test_message_poll(struct file *filp, struct poll_table_struct *wait)
 {
 	struct mbox_test_device *tdev = filp->private_data;
@@ -243,7 +243,7 @@ mbox_test_message_poll(struct file *filp, struct poll_table_struct *wait)
 	poll_wait(filp, &tdev->waitq, wait);
 
 	if (mbox_test_message_data_ready(tdev))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 
diff --git a/drivers/mailbox/qcom-apcs-ipc-mailbox.c b/drivers/mailbox/qcom-apcs-ipc-mailbox.c
index 9924c6d7f05d..57bde0dfd12f 100644
--- a/drivers/mailbox/qcom-apcs-ipc-mailbox.c
+++ b/drivers/mailbox/qcom-apcs-ipc-mailbox.c
@@ -18,6 +18,7 @@
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/regmap.h>
 #include <linux/mailbox_controller.h>
 
 #define QCOM_APCS_IPC_BITS	32
@@ -26,8 +27,17 @@ struct qcom_apcs_ipc {
 	struct mbox_controller mbox;
 	struct mbox_chan mbox_chans[QCOM_APCS_IPC_BITS];
 
-	void __iomem *reg;
+	struct regmap *regmap;
 	unsigned long offset;
+	struct platform_device *clk;
+};
+
+static const struct regmap_config apcs_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1000,
+	.fast_io = true,
 };
 
 static int qcom_apcs_ipc_send_data(struct mbox_chan *chan, void *data)
@@ -36,9 +46,7 @@ static int qcom_apcs_ipc_send_data(struct mbox_chan *chan, void *data)
 						  struct qcom_apcs_ipc, mbox);
 	unsigned long idx = (unsigned long)chan->con_priv;
 
-	writel(BIT(idx), apcs->reg);
-
-	return 0;
+	return regmap_write(apcs->regmap, apcs->offset, BIT(idx));
 }
 
 static const struct mbox_chan_ops qcom_apcs_ipc_ops = {
@@ -47,7 +55,9 @@ static const struct mbox_chan_ops qcom_apcs_ipc_ops = {
 
 static int qcom_apcs_ipc_probe(struct platform_device *pdev)
 {
+	struct device_node *np = pdev->dev.of_node;
 	struct qcom_apcs_ipc *apcs;
+	struct regmap *regmap;
 	struct resource *res;
 	unsigned long offset;
 	void __iomem *base;
@@ -63,9 +73,14 @@ static int qcom_apcs_ipc_probe(struct platform_device *pdev)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
+	regmap = devm_regmap_init_mmio(&pdev->dev, base, &apcs_regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
 	offset = (unsigned long)of_device_get_match_data(&pdev->dev);
 
-	apcs->reg = base + offset;
+	apcs->regmap = regmap;
+	apcs->offset = offset;
 
 	/* Initialize channel identifiers */
 	for (i = 0; i < ARRAY_SIZE(apcs->mbox_chans); i++)
@@ -82,6 +97,14 @@ static int qcom_apcs_ipc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	if (of_device_is_compatible(np, "qcom,msm8916-apcs-kpss-global")) {
+		apcs->clk = platform_device_register_data(&pdev->dev,
+							  "qcom-apcs-msm8916-clk",
+							  -1, NULL, 0);
+		if (IS_ERR(apcs->clk))
+			dev_err(&pdev->dev, "failed to register APCS clk\n");
+	}
+
 	platform_set_drvdata(pdev, apcs);
 
 	return 0;
@@ -90,8 +113,10 @@ static int qcom_apcs_ipc_probe(struct platform_device *pdev)
 static int qcom_apcs_ipc_remove(struct platform_device *pdev)
 {
 	struct qcom_apcs_ipc *apcs = platform_get_drvdata(pdev);
+	struct platform_device *clk = apcs->clk;
 
 	mbox_controller_unregister(&apcs->mbox);
+	platform_device_unregister(clk);
 
 	return 0;
 }
diff --git a/drivers/mailbox/ti-msgmgr.c b/drivers/mailbox/ti-msgmgr.c
index 54b9e4cb4cfa..78753a87ba4d 100644
--- a/drivers/mailbox/ti-msgmgr.c
+++ b/drivers/mailbox/ti-msgmgr.c
@@ -1,17 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Texas Instruments' Message Manager Driver
  *
- * Copyright (C) 2015-2016 Texas Instruments Incorporated - http://www.ti.com/
+ * Copyright (C) 2015-2017 Texas Instruments Incorporated - http://www.ti.com/
  *	Nishanth Menon
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * This program is distributed "as is" WITHOUT ANY WARRANTY of any
- * kind, whether express or implied; without even the implied warranty
- * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #define pr_fmt(fmt) "%s: " fmt, __func__
@@ -291,7 +283,7 @@ static int ti_msgmgr_send_data(struct mbox_chan *chan, void *data)
 	desc = inst->desc;
 
 	if (desc->max_message_size < message->len) {
-		dev_err(dev, "Queue %s message length %d > max %d\n",
+		dev_err(dev, "Queue %s message length %zu > max %d\n",
 			qinst->name, message->len, desc->max_message_size);
 		return -EINVAL;
 	}
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 83b9362be09c..2c8ac3688815 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -269,6 +269,13 @@ config DM_BIO_PRISON
 
 source "drivers/md/persistent-data/Kconfig"
 
+config DM_UNSTRIPED
+       tristate "Unstriped target"
+       depends on BLK_DEV_DM
+       ---help---
+	  Unstripes I/O so it is issued solely on a single drive in a HW
+	  RAID0 or dm-striped target.
+
 config DM_CRYPT
 	tristate "Crypt target support"
 	depends on BLK_DEV_DM
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index f701bb211783..63255f3ebd97 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -43,6 +43,7 @@ obj-$(CONFIG_BCACHE)		+= bcache/
 obj-$(CONFIG_BLK_DEV_MD)	+= md-mod.o
 obj-$(CONFIG_BLK_DEV_DM)	+= dm-mod.o
 obj-$(CONFIG_BLK_DEV_DM_BUILTIN) += dm-builtin.o
+obj-$(CONFIG_DM_UNSTRIPED)	+= dm-unstripe.o
 obj-$(CONFIG_DM_BUFIO)		+= dm-bufio.o
 obj-$(CONFIG_DM_BIO_PRISON)	+= dm-bio-prison.o
 obj-$(CONFIG_DM_CRYPT)		+= dm-crypt.o
diff --git a/drivers/md/bcache/alloc.c b/drivers/md/bcache/alloc.c
index a0cc1bc6d884..458e1d38577d 100644
--- a/drivers/md/bcache/alloc.c
+++ b/drivers/md/bcache/alloc.c
@@ -287,8 +287,10 @@ do {									\
 			break;						\
 									\
 		mutex_unlock(&(ca)->set->bucket_lock);			\
-		if (kthread_should_stop())				\
+		if (kthread_should_stop()) {				\
+			set_current_state(TASK_RUNNING);		\
 			return 0;					\
+		}							\
 									\
 		schedule();						\
 		mutex_lock(&(ca)->set->bucket_lock);			\
@@ -525,15 +527,21 @@ struct open_bucket {
 
 /*
  * We keep multiple buckets open for writes, and try to segregate different
- * write streams for better cache utilization: first we look for a bucket where
- * the last write to it was sequential with the current write, and failing that
- * we look for a bucket that was last used by the same task.
+ * write streams for better cache utilization: first we try to segregate flash
+ * only volume write streams from cached devices, secondly we look for a bucket
+ * where the last write to it was sequential with the current write, and
+ * failing that we look for a bucket that was last used by the same task.
  *
  * The ideas is if you've got multiple tasks pulling data into the cache at the
  * same time, you'll get better cache utilization if you try to segregate their
  * data and preserve locality.
  *
- * For example, say you've starting Firefox at the same time you're copying a
+ * For example, dirty sectors of flash only volume is not reclaimable, if their
+ * dirty sectors mixed with dirty sectors of cached device, such buckets will
+ * be marked as dirty and won't be reclaimed, though the dirty data of cached
+ * device have been written back to backend device.
+ *
+ * And say you've starting Firefox at the same time you're copying a
  * bunch of files. Firefox will likely end up being fairly hot and stay in the
  * cache awhile, but the data you copied might not be; if you wrote all that
  * data to the same buckets it'd get invalidated at the same time.
@@ -550,7 +558,10 @@ static struct open_bucket *pick_data_bucket(struct cache_set *c,
 	struct open_bucket *ret, *ret_task = NULL;
 
 	list_for_each_entry_reverse(ret, &c->data_buckets, list)
-		if (!bkey_cmp(&ret->key, search))
+		if (UUID_FLASH_ONLY(&c->uuids[KEY_INODE(&ret->key)]) !=
+		    UUID_FLASH_ONLY(&c->uuids[KEY_INODE(search)]))
+			continue;
+		else if (!bkey_cmp(&ret->key, search))
 			goto found;
 		else if (ret->last_write_point == write_point)
 			ret_task = ret;
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 843877e017e1..12e5197f186c 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -320,14 +320,15 @@ struct cached_dev {
 	 */
 	atomic_t		has_dirty;
 
-	struct bch_ratelimit	writeback_rate;
-	struct delayed_work	writeback_rate_update;
-
 	/*
-	 * Internal to the writeback code, so read_dirty() can keep track of
-	 * where it's at.
+	 * Set to zero by things that touch the backing volume-- except
+	 * writeback.  Incremented by writeback.  Used to determine when to
+	 * accelerate idle writeback.
 	 */
-	sector_t		last_read;
+	atomic_t		backing_idle;
+
+	struct bch_ratelimit	writeback_rate;
+	struct delayed_work	writeback_rate_update;
 
 	/* Limit number of writeback bios in flight */
 	struct semaphore	in_flight;
@@ -336,6 +337,14 @@ struct cached_dev {
 
 	struct keybuf		writeback_keys;
 
+	/*
+	 * Order the write-half of writeback operations strongly in dispatch
+	 * order.  (Maintain LBA order; don't allow reads completing out of
+	 * order to re-order the writes...)
+	 */
+	struct closure_waitlist writeback_ordering_wait;
+	atomic_t		writeback_sequence_next;
+
 	/* For tracking sequential IO */
 #define RECENT_IO_BITS	7
 #define RECENT_IO	(1 << RECENT_IO_BITS)
@@ -488,6 +497,7 @@ struct cache_set {
 	int			caches_loaded;
 
 	struct bcache_device	**devices;
+	unsigned		devices_max_used;
 	struct list_head	cached_devs;
 	uint64_t		cached_dev_sectors;
 	struct closure		caching;
@@ -648,10 +658,15 @@ struct cache_set {
 	atomic_long_t		writeback_keys_done;
 	atomic_long_t		writeback_keys_failed;
 
+	atomic_long_t		reclaim;
+	atomic_long_t		flush_write;
+	atomic_long_t		retry_flush_write;
+
 	enum			{
 		ON_ERROR_UNREGISTER,
 		ON_ERROR_PANIC,
 	}			on_error;
+#define DEFAULT_IO_ERROR_LIMIT 8
 	unsigned		error_limit;
 	unsigned		error_decay;
 
@@ -665,6 +680,8 @@ struct cache_set {
 
 #define BUCKET_HASH_BITS	12
 	struct hlist_head	bucket_hash[1 << BUCKET_HASH_BITS];
+
+	DECLARE_HEAP(struct btree *, flush_btree);
 };
 
 struct bbio {
@@ -852,7 +869,7 @@ static inline void wake_up_allocators(struct cache_set *c)
 
 /* Forward declarations */
 
-void bch_count_io_errors(struct cache *, blk_status_t, const char *);
+void bch_count_io_errors(struct cache *, blk_status_t, int, const char *);
 void bch_bbio_count_io_errors(struct cache_set *, struct bio *,
 			      blk_status_t, const char *);
 void bch_bbio_endio(struct cache_set *, struct bio *, blk_status_t,
@@ -907,7 +924,7 @@ void bcache_write_super(struct cache_set *);
 
 int bch_flash_dev_create(struct cache_set *c, uint64_t size);
 
-int bch_cached_dev_attach(struct cached_dev *, struct cache_set *);
+int bch_cached_dev_attach(struct cached_dev *, struct cache_set *, uint8_t *);
 void bch_cached_dev_detach(struct cached_dev *);
 void bch_cached_dev_run(struct cached_dev *);
 void bcache_device_stop(struct bcache_device *);
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 81e8dc3dbe5e..fad9fe8817eb 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -419,7 +419,7 @@ static void do_btree_node_write(struct btree *b)
 	SET_PTR_OFFSET(&k.key, 0, PTR_OFFSET(&k.key, 0) +
 		       bset_sector_offset(&b->keys, i));
 
-	if (!bio_alloc_pages(b->bio, __GFP_NOWARN|GFP_NOWAIT)) {
+	if (!bch_bio_alloc_pages(b->bio, __GFP_NOWARN|GFP_NOWAIT)) {
 		int j;
 		struct bio_vec *bv;
 		void *base = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1));
@@ -432,6 +432,7 @@ static void do_btree_node_write(struct btree *b)
 
 		continue_at(cl, btree_node_write_done, NULL);
 	} else {
+		/* No problem for multipage bvec since the bio is just allocated */
 		b->bio->bi_vcnt = 0;
 		bch_bio_map(b->bio, i);
 
@@ -1678,7 +1679,7 @@ static void bch_btree_gc_finish(struct cache_set *c)
 
 	/* don't reclaim buckets to which writeback keys point */
 	rcu_read_lock();
-	for (i = 0; i < c->nr_uuids; i++) {
+	for (i = 0; i < c->devices_max_used; i++) {
 		struct bcache_device *d = c->devices[i];
 		struct cached_dev *dc;
 		struct keybuf_key *w, *n;
@@ -1803,10 +1804,7 @@ static int bch_gc_thread(void *arg)
 int bch_gc_thread_start(struct cache_set *c)
 {
 	c->gc_thread = kthread_run(bch_gc_thread, c, "bcache_gc");
-	if (IS_ERR(c->gc_thread))
-		return PTR_ERR(c->gc_thread);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(c->gc_thread);
 }
 
 /* Initial partial gc */
@@ -1871,14 +1869,17 @@ void bch_initial_gc_finish(struct cache_set *c)
 	 */
 	for_each_cache(ca, c, i) {
 		for_each_bucket(b, ca) {
-			if (fifo_full(&ca->free[RESERVE_PRIO]))
+			if (fifo_full(&ca->free[RESERVE_PRIO]) &&
+			    fifo_full(&ca->free[RESERVE_BTREE]))
 				break;
 
 			if (bch_can_invalidate_bucket(ca, b) &&
 			    !GC_MARK(b)) {
 				__bch_invalidate_one_bucket(ca, b);
-				fifo_push(&ca->free[RESERVE_PRIO],
-					  b - ca->buckets);
+				if (!fifo_push(&ca->free[RESERVE_PRIO],
+				   b - ca->buckets))
+					fifo_push(&ca->free[RESERVE_BTREE],
+						  b - ca->buckets);
 			}
 		}
 	}
diff --git a/drivers/md/bcache/closure.c b/drivers/md/bcache/closure.c
index 1841d0359bac..7f12920c14f7 100644
--- a/drivers/md/bcache/closure.c
+++ b/drivers/md/bcache/closure.c
@@ -8,6 +8,7 @@
 #include <linux/debugfs.h>
 #include <linux/module.h>
 #include <linux/seq_file.h>
+#include <linux/sched/debug.h>
 
 #include "closure.h"
 
@@ -18,10 +19,6 @@ static inline void closure_put_after_sub(struct closure *cl, int flags)
 	BUG_ON(flags & CLOSURE_GUARD_MASK);
 	BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR));
 
-	/* Must deliver precisely one wakeup */
-	if (r == 1 && (flags & CLOSURE_SLEEPING))
-		wake_up_process(cl->task);
-
 	if (!r) {
 		if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {
 			atomic_set(&cl->remaining,
@@ -100,28 +97,34 @@ bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl)
 }
 EXPORT_SYMBOL(closure_wait);
 
-/**
- * closure_sync - sleep until a closure has nothing left to wait on
- *
- * Sleeps until the refcount hits 1 - the thread that's running the closure owns
- * the last refcount.
- */
-void closure_sync(struct closure *cl)
+struct closure_syncer {
+	struct task_struct	*task;
+	int			done;
+};
+
+static void closure_sync_fn(struct closure *cl)
 {
-	while (1) {
-		__closure_start_sleep(cl);
-		closure_set_ret_ip(cl);
+	cl->s->done = 1;
+	wake_up_process(cl->s->task);
+}
 
-		if ((atomic_read(&cl->remaining) &
-		     CLOSURE_REMAINING_MASK) == 1)
-			break;
+void __sched __closure_sync(struct closure *cl)
+{
+	struct closure_syncer s = { .task = current };
 
+	cl->s = &s;
+	continue_at(cl, closure_sync_fn, NULL);
+
+	while (1) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		if (s.done)
+			break;
 		schedule();
 	}
 
-	__closure_end_sleep(cl);
+	__set_current_state(TASK_RUNNING);
 }
-EXPORT_SYMBOL(closure_sync);
+EXPORT_SYMBOL(__closure_sync);
 
 #ifdef CONFIG_BCACHE_CLOSURES_DEBUG
 
@@ -168,12 +171,10 @@ static int debug_seq_show(struct seq_file *f, void *data)
 			   cl, (void *) cl->ip, cl->fn, cl->parent,
 			   r & CLOSURE_REMAINING_MASK);
 
-		seq_printf(f, "%s%s%s%s\n",
+		seq_printf(f, "%s%s\n",
 			   test_bit(WORK_STRUCT_PENDING_BIT,
 				    work_data_bits(&cl->work)) ? "Q" : "",
-			   r & CLOSURE_RUNNING	? "R" : "",
-			   r & CLOSURE_STACK	? "S" : "",
-			   r & CLOSURE_SLEEPING	? "Sl" : "");
+			   r & CLOSURE_RUNNING	? "R" : "");
 
 		if (r & CLOSURE_WAITING)
 			seq_printf(f, " W %pF\n",
diff --git a/drivers/md/bcache/closure.h b/drivers/md/bcache/closure.h
index ccfbea6f9f6b..3b9dfc9962ad 100644
--- a/drivers/md/bcache/closure.h
+++ b/drivers/md/bcache/closure.h
@@ -103,6 +103,7 @@
  */
 
 struct closure;
+struct closure_syncer;
 typedef void (closure_fn) (struct closure *);
 
 struct closure_waitlist {
@@ -115,10 +116,6 @@ enum closure_state {
 	 * the thread that owns the closure, and cleared by the thread that's
 	 * waking up the closure.
 	 *
-	 * CLOSURE_SLEEPING: Must be set before a thread uses a closure to sleep
-	 * - indicates that cl->task is valid and closure_put() may wake it up.
-	 * Only set or cleared by the thread that owns the closure.
-	 *
 	 * The rest are for debugging and don't affect behaviour:
 	 *
 	 * CLOSURE_RUNNING: Set when a closure is running (i.e. by
@@ -128,22 +125,16 @@ enum closure_state {
 	 * continue_at() and closure_return() clear it for you, if you're doing
 	 * something unusual you can use closure_set_dead() which also helps
 	 * annotate where references are being transferred.
-	 *
-	 * CLOSURE_STACK: Sanity check - remaining should never hit 0 on a
-	 * closure with this flag set
 	 */
 
-	CLOSURE_BITS_START	= (1 << 23),
-	CLOSURE_DESTRUCTOR	= (1 << 23),
-	CLOSURE_WAITING		= (1 << 25),
-	CLOSURE_SLEEPING	= (1 << 27),
-	CLOSURE_RUNNING		= (1 << 29),
-	CLOSURE_STACK		= (1 << 31),
+	CLOSURE_BITS_START	= (1U << 26),
+	CLOSURE_DESTRUCTOR	= (1U << 26),
+	CLOSURE_WAITING		= (1U << 28),
+	CLOSURE_RUNNING		= (1U << 30),
 };
 
 #define CLOSURE_GUARD_MASK					\
-	((CLOSURE_DESTRUCTOR|CLOSURE_WAITING|CLOSURE_SLEEPING|	\
-	  CLOSURE_RUNNING|CLOSURE_STACK) << 1)
+	((CLOSURE_DESTRUCTOR|CLOSURE_WAITING|CLOSURE_RUNNING) << 1)
 
 #define CLOSURE_REMAINING_MASK		(CLOSURE_BITS_START - 1)
 #define CLOSURE_REMAINING_INITIALIZER	(1|CLOSURE_RUNNING)
@@ -152,7 +143,7 @@ struct closure {
 	union {
 		struct {
 			struct workqueue_struct *wq;
-			struct task_struct	*task;
+			struct closure_syncer	*s;
 			struct llist_node	list;
 			closure_fn		*fn;
 		};
@@ -178,7 +169,19 @@ void closure_sub(struct closure *cl, int v);
 void closure_put(struct closure *cl);
 void __closure_wake_up(struct closure_waitlist *list);
 bool closure_wait(struct closure_waitlist *list, struct closure *cl);
-void closure_sync(struct closure *cl);
+void __closure_sync(struct closure *cl);
+
+/**
+ * closure_sync - sleep until a closure a closure has nothing left to wait on
+ *
+ * Sleeps until the refcount hits 1 - the thread that's running the closure owns
+ * the last refcount.
+ */
+static inline void closure_sync(struct closure *cl)
+{
+	if ((atomic_read(&cl->remaining) & CLOSURE_REMAINING_MASK) != 1)
+		__closure_sync(cl);
+}
 
 #ifdef CONFIG_BCACHE_CLOSURES_DEBUG
 
@@ -215,24 +218,6 @@ static inline void closure_set_waiting(struct closure *cl, unsigned long f)
 #endif
 }
 
-static inline void __closure_end_sleep(struct closure *cl)
-{
-	__set_current_state(TASK_RUNNING);
-
-	if (atomic_read(&cl->remaining) & CLOSURE_SLEEPING)
-		atomic_sub(CLOSURE_SLEEPING, &cl->remaining);
-}
-
-static inline void __closure_start_sleep(struct closure *cl)
-{
-	closure_set_ip(cl);
-	cl->task = current;
-	set_current_state(TASK_UNINTERRUPTIBLE);
-
-	if (!(atomic_read(&cl->remaining) & CLOSURE_SLEEPING))
-		atomic_add(CLOSURE_SLEEPING, &cl->remaining);
-}
-
 static inline void closure_set_stopped(struct closure *cl)
 {
 	atomic_sub(CLOSURE_RUNNING, &cl->remaining);
@@ -241,7 +226,6 @@ static inline void closure_set_stopped(struct closure *cl)
 static inline void set_closure_fn(struct closure *cl, closure_fn *fn,
 				  struct workqueue_struct *wq)
 {
-	BUG_ON(object_is_on_stack(cl));
 	closure_set_ip(cl);
 	cl->fn = fn;
 	cl->wq = wq;
@@ -300,7 +284,7 @@ static inline void closure_init(struct closure *cl, struct closure *parent)
 static inline void closure_init_stack(struct closure *cl)
 {
 	memset(cl, 0, sizeof(struct closure));
-	atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER|CLOSURE_STACK);
+	atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER);
 }
 
 /**
@@ -322,6 +306,8 @@ static inline void closure_wake_up(struct closure_waitlist *list)
  * This is because after calling continue_at() you no longer have a ref on @cl,
  * and whatever @cl owns may be freed out from under you - a running closure fn
  * has a ref on its own closure which continue_at() drops.
+ *
+ * Note you are expected to immediately return after using this macro.
  */
 #define continue_at(_cl, _fn, _wq)					\
 do {									\
diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c
index c7a02c4900da..af89408befe8 100644
--- a/drivers/md/bcache/debug.c
+++ b/drivers/md/bcache/debug.c
@@ -116,7 +116,7 @@ void bch_data_verify(struct cached_dev *dc, struct bio *bio)
 		return;
 	check->bi_opf = REQ_OP_READ;
 
-	if (bio_alloc_pages(check, GFP_NOIO))
+	if (bch_bio_alloc_pages(check, GFP_NOIO))
 		goto out_put;
 
 	submit_bio_wait(check);
@@ -251,8 +251,7 @@ void bch_debug_exit(void)
 
 int __init bch_debug_init(struct kobject *kobj)
 {
-	int ret = 0;
-
 	debug = debugfs_create_dir("bcache", NULL);
-	return ret;
+
+	return IS_ERR_OR_NULL(debug);
 }
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index fac97ec2d0e2..a783c5a41ff1 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -51,7 +51,10 @@ void bch_submit_bbio(struct bio *bio, struct cache_set *c,
 
 /* IO errors */
 
-void bch_count_io_errors(struct cache *ca, blk_status_t error, const char *m)
+void bch_count_io_errors(struct cache *ca,
+			 blk_status_t error,
+			 int is_read,
+			 const char *m)
 {
 	/*
 	 * The halflife of an error is:
@@ -94,8 +97,9 @@ void bch_count_io_errors(struct cache *ca, blk_status_t error, const char *m)
 		errors >>= IO_ERROR_SHIFT;
 
 		if (errors < ca->set->error_limit)
-			pr_err("%s: IO error on %s, recovering",
-			       bdevname(ca->bdev, buf), m);
+			pr_err("%s: IO error on %s%s",
+			       bdevname(ca->bdev, buf), m,
+			       is_read ? ", recovering." : ".");
 		else
 			bch_cache_set_error(ca->set,
 					    "%s: too many IO errors %s",
@@ -108,6 +112,7 @@ void bch_bbio_count_io_errors(struct cache_set *c, struct bio *bio,
 {
 	struct bbio *b = container_of(bio, struct bbio, bio);
 	struct cache *ca = PTR_CACHE(c, &b->key, 0);
+	int is_read = (bio_data_dir(bio) == READ ? 1 : 0);
 
 	unsigned threshold = op_is_write(bio_op(bio))
 		? c->congested_write_threshold_us
@@ -129,7 +134,7 @@ void bch_bbio_count_io_errors(struct cache_set *c, struct bio *bio,
 			atomic_inc(&c->congested);
 	}
 
-	bch_count_io_errors(ca, error, m);
+	bch_count_io_errors(ca, error, is_read, m);
 }
 
 void bch_bbio_endio(struct cache_set *c, struct bio *bio,
diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c
index a87165c1d8e5..1b736b860739 100644
--- a/drivers/md/bcache/journal.c
+++ b/drivers/md/bcache/journal.c
@@ -368,6 +368,12 @@ int bch_journal_replay(struct cache_set *s, struct list_head *list)
 }
 
 /* Journalling */
+#define journal_max_cmp(l, r) \
+	(fifo_idx(&c->journal.pin, btree_current_write(l)->journal) < \
+	 fifo_idx(&(c)->journal.pin, btree_current_write(r)->journal))
+#define journal_min_cmp(l, r) \
+	(fifo_idx(&c->journal.pin, btree_current_write(l)->journal) > \
+	 fifo_idx(&(c)->journal.pin, btree_current_write(r)->journal))
 
 static void btree_flush_write(struct cache_set *c)
 {
@@ -375,28 +381,41 @@ static void btree_flush_write(struct cache_set *c)
 	 * Try to find the btree node with that references the oldest journal
 	 * entry, best is our current candidate and is locked if non NULL:
 	 */
-	struct btree *b, *best;
-	unsigned i;
+	struct btree *b;
+	int i;
+
+	atomic_long_inc(&c->flush_write);
+
 retry:
-	best = NULL;
-
-	for_each_cached_btree(b, c, i)
-		if (btree_current_write(b)->journal) {
-			if (!best)
-				best = b;
-			else if (journal_pin_cmp(c,
-					btree_current_write(best)->journal,
-					btree_current_write(b)->journal)) {
-				best = b;
+	spin_lock(&c->journal.lock);
+	if (heap_empty(&c->flush_btree)) {
+		for_each_cached_btree(b, c, i)
+			if (btree_current_write(b)->journal) {
+				if (!heap_full(&c->flush_btree))
+					heap_add(&c->flush_btree, b,
+						 journal_max_cmp);
+				else if (journal_max_cmp(b,
+					 heap_peek(&c->flush_btree))) {
+					c->flush_btree.data[0] = b;
+					heap_sift(&c->flush_btree, 0,
+						  journal_max_cmp);
+				}
 			}
-		}
 
-	b = best;
+		for (i = c->flush_btree.used / 2 - 1; i >= 0; --i)
+			heap_sift(&c->flush_btree, i, journal_min_cmp);
+	}
+
+	b = NULL;
+	heap_pop(&c->flush_btree, b, journal_min_cmp);
+	spin_unlock(&c->journal.lock);
+
 	if (b) {
 		mutex_lock(&b->write_lock);
 		if (!btree_current_write(b)->journal) {
 			mutex_unlock(&b->write_lock);
 			/* We raced */
+			atomic_long_inc(&c->retry_flush_write);
 			goto retry;
 		}
 
@@ -476,6 +495,8 @@ static void journal_reclaim(struct cache_set *c)
 	unsigned iter, n = 0;
 	atomic_t p;
 
+	atomic_long_inc(&c->reclaim);
+
 	while (!atomic_read(&fifo_front(&c->journal.pin)))
 		fifo_pop(&c->journal.pin, p);
 
@@ -819,7 +840,8 @@ int bch_journal_alloc(struct cache_set *c)
 	j->w[0].c = c;
 	j->w[1].c = c;
 
-	if (!(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL)) ||
+	if (!(init_heap(&c->flush_btree, 128, GFP_KERNEL)) ||
+	    !(init_fifo(&j->pin, JOURNAL_PIN, GFP_KERNEL)) ||
 	    !(j->w[0].data = (void *) __get_free_pages(GFP_KERNEL, JSET_BITS)) ||
 	    !(j->w[1].data = (void *) __get_free_pages(GFP_KERNEL, JSET_BITS)))
 		return -ENOMEM;
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index d50c1c97da68..a24c3a95b2c0 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -162,7 +162,7 @@ static void read_moving(struct cache_set *c)
 		bio_set_op_attrs(bio, REQ_OP_READ, 0);
 		bio->bi_end_io	= read_moving_endio;
 
-		if (bio_alloc_pages(bio, GFP_KERNEL))
+		if (bch_bio_alloc_pages(bio, GFP_KERNEL))
 			goto err;
 
 		trace_bcache_gc_copy(&w->key);
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 643c3021624f..6422846b546e 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -576,6 +576,7 @@ static void cache_lookup(struct closure *cl)
 {
 	struct search *s = container_of(cl, struct search, iop.cl);
 	struct bio *bio = &s->bio.bio;
+	struct cached_dev *dc;
 	int ret;
 
 	bch_btree_op_init(&s->op, -1);
@@ -588,6 +589,27 @@ static void cache_lookup(struct closure *cl)
 		return;
 	}
 
+	/*
+	 * We might meet err when searching the btree, If that happens, we will
+	 * get negative ret, in this scenario we should not recover data from
+	 * backing device (when cache device is dirty) because we don't know
+	 * whether bkeys the read request covered are all clean.
+	 *
+	 * And after that happened, s->iop.status is still its initial value
+	 * before we submit s->bio.bio
+	 */
+	if (ret < 0) {
+		BUG_ON(ret == -EINTR);
+		if (s->d && s->d->c &&
+				!UUID_FLASH_ONLY(&s->d->c->uuids[s->d->id])) {
+			dc = container_of(s->d, struct cached_dev, disk);
+			if (dc && atomic_read(&dc->has_dirty))
+				s->recoverable = false;
+		}
+		if (!s->iop.status)
+			s->iop.status = BLK_STS_IOERR;
+	}
+
 	closure_return(cl);
 }
 
@@ -611,8 +633,8 @@ static void request_endio(struct bio *bio)
 static void bio_complete(struct search *s)
 {
 	if (s->orig_bio) {
-		struct request_queue *q = s->orig_bio->bi_disk->queue;
-		generic_end_io_acct(q, bio_data_dir(s->orig_bio),
+		generic_end_io_acct(s->d->disk->queue,
+				    bio_data_dir(s->orig_bio),
 				    &s->d->disk->part0, s->start_time);
 
 		trace_bcache_request_end(s->d, s->orig_bio);
@@ -637,11 +659,11 @@ static void do_bio_hook(struct search *s, struct bio *orig_bio)
 static void search_free(struct closure *cl)
 {
 	struct search *s = container_of(cl, struct search, cl);
-	bio_complete(s);
 
 	if (s->iop.bio)
 		bio_put(s->iop.bio);
 
+	bio_complete(s);
 	closure_debug_destroy(cl);
 	mempool_free(s, s->d->c->search);
 }
@@ -841,7 +863,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s,
 	cache_bio->bi_private	= &s->cl;
 
 	bch_bio_map(cache_bio, NULL);
-	if (bio_alloc_pages(cache_bio, __GFP_NOWARN|GFP_NOIO))
+	if (bch_bio_alloc_pages(cache_bio, __GFP_NOWARN|GFP_NOIO))
 		goto out_put;
 
 	if (reada)
@@ -974,6 +996,7 @@ static blk_qc_t cached_dev_make_request(struct request_queue *q,
 	struct cached_dev *dc = container_of(d, struct cached_dev, disk);
 	int rw = bio_data_dir(bio);
 
+	atomic_set(&dc->backing_idle, 0);
 	generic_start_io_acct(q, rw, bio_sectors(bio), &d->disk->part0);
 
 	bio_set_dev(bio, dc->bdev);
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index b4d28928dec5..f2273143b3cb 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -211,7 +211,7 @@ static void write_bdev_super_endio(struct bio *bio)
 
 static void __write_super(struct cache_sb *sb, struct bio *bio)
 {
-	struct cache_sb *out = page_address(bio->bi_io_vec[0].bv_page);
+	struct cache_sb *out = page_address(bio_first_page_all(bio));
 	unsigned i;
 
 	bio->bi_iter.bi_sector	= SB_SECTOR;
@@ -274,7 +274,9 @@ static void write_super_endio(struct bio *bio)
 {
 	struct cache *ca = bio->bi_private;
 
-	bch_count_io_errors(ca, bio->bi_status, "writing superblock");
+	/* is_read = 0 */
+	bch_count_io_errors(ca, bio->bi_status, 0,
+			    "writing superblock");
 	closure_put(&ca->set->sb_write);
 }
 
@@ -721,6 +723,9 @@ static void bcache_device_attach(struct bcache_device *d, struct cache_set *c,
 	d->c = c;
 	c->devices[id] = d;
 
+	if (id >= c->devices_max_used)
+		c->devices_max_used = id + 1;
+
 	closure_get(&c->caching);
 }
 
@@ -906,6 +911,12 @@ static void cached_dev_detach_finish(struct work_struct *w)
 
 	mutex_lock(&bch_register_lock);
 
+	cancel_delayed_work_sync(&dc->writeback_rate_update);
+	if (!IS_ERR_OR_NULL(dc->writeback_thread)) {
+		kthread_stop(dc->writeback_thread);
+		dc->writeback_thread = NULL;
+	}
+
 	memset(&dc->sb.set_uuid, 0, 16);
 	SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);
 
@@ -946,15 +957,18 @@ void bch_cached_dev_detach(struct cached_dev *dc)
 	cached_dev_put(dc);
 }
 
-int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c)
+int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c,
+			  uint8_t *set_uuid)
 {
 	uint32_t rtime = cpu_to_le32(get_seconds());
 	struct uuid_entry *u;
 	char buf[BDEVNAME_SIZE];
+	struct cached_dev *exist_dc, *t;
 
 	bdevname(dc->bdev, buf);
 
-	if (memcmp(dc->sb.set_uuid, c->sb.set_uuid, 16))
+	if ((set_uuid && memcmp(set_uuid, c->sb.set_uuid, 16)) ||
+	    (!set_uuid && memcmp(dc->sb.set_uuid, c->sb.set_uuid, 16)))
 		return -ENOENT;
 
 	if (dc->disk.c) {
@@ -974,6 +988,16 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c)
 		return -EINVAL;
 	}
 
+	/* Check whether already attached */
+	list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) {
+		if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) {
+			pr_err("Tried to attach %s but duplicate UUID already attached",
+				buf);
+
+			return -EINVAL;
+		}
+	}
+
 	u = uuid_find(c, dc->sb.uuid);
 
 	if (u &&
@@ -1166,7 +1190,7 @@ static void register_bdev(struct cache_sb *sb, struct page *sb_page,
 	dc->bdev->bd_holder = dc;
 
 	bio_init(&dc->sb_bio, dc->sb_bio.bi_inline_vecs, 1);
-	dc->sb_bio.bi_io_vec[0].bv_page = sb_page;
+	bio_first_bvec_all(&dc->sb_bio)->bv_page = sb_page;
 	get_page(sb_page);
 
 	if (cached_dev_init(dc, sb->block_size << 9))
@@ -1183,7 +1207,7 @@ static void register_bdev(struct cache_sb *sb, struct page *sb_page,
 
 	list_add(&dc->list, &uncached_devices);
 	list_for_each_entry(c, &bch_cache_sets, list)
-		bch_cached_dev_attach(dc, c);
+		bch_cached_dev_attach(dc, c, NULL);
 
 	if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
 	    BDEV_STATE(&dc->sb) == BDEV_STATE_STALE)
@@ -1191,7 +1215,7 @@ static void register_bdev(struct cache_sb *sb, struct page *sb_page,
 
 	return;
 err:
-	pr_notice("error opening %s: %s", bdevname(bdev, name), err);
+	pr_notice("error %s: %s", bdevname(bdev, name), err);
 	bcache_device_stop(&dc->disk);
 }
 
@@ -1427,7 +1451,7 @@ static void __cache_set_unregister(struct closure *cl)
 
 	mutex_lock(&bch_register_lock);
 
-	for (i = 0; i < c->nr_uuids; i++)
+	for (i = 0; i < c->devices_max_used; i++)
 		if (c->devices[i]) {
 			if (!UUID_FLASH_ONLY(&c->uuids[i]) &&
 			    test_bit(CACHE_SET_UNREGISTERING, &c->flags)) {
@@ -1490,7 +1514,7 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
 	c->bucket_bits		= ilog2(sb->bucket_size);
 	c->block_bits		= ilog2(sb->block_size);
 	c->nr_uuids		= bucket_bytes(c) / sizeof(struct uuid_entry);
-
+	c->devices_max_used	= 0;
 	c->btree_pages		= bucket_pages(c);
 	if (c->btree_pages > BTREE_MAX_PAGES)
 		c->btree_pages = max_t(int, c->btree_pages / 4,
@@ -1542,7 +1566,7 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
 
 	c->congested_read_threshold_us	= 2000;
 	c->congested_write_threshold_us	= 20000;
-	c->error_limit	= 8 << IO_ERROR_SHIFT;
+	c->error_limit	= DEFAULT_IO_ERROR_LIMIT;
 
 	return c;
 err:
@@ -1705,7 +1729,7 @@ static void run_cache_set(struct cache_set *c)
 	bcache_write_super(c);
 
 	list_for_each_entry_safe(dc, t, &uncached_devices, list)
-		bch_cached_dev_attach(dc, c);
+		bch_cached_dev_attach(dc, c, NULL);
 
 	flash_devs_run(c);
 
@@ -1810,7 +1834,7 @@ void bch_cache_release(struct kobject *kobj)
 		free_fifo(&ca->free[i]);
 
 	if (ca->sb_bio.bi_inline_vecs[0].bv_page)
-		put_page(ca->sb_bio.bi_io_vec[0].bv_page);
+		put_page(bio_first_page_all(&ca->sb_bio));
 
 	if (!IS_ERR_OR_NULL(ca->bdev))
 		blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
@@ -1822,6 +1846,7 @@ void bch_cache_release(struct kobject *kobj)
 static int cache_alloc(struct cache *ca)
 {
 	size_t free;
+	size_t btree_buckets;
 	struct bucket *b;
 
 	__module_get(THIS_MODULE);
@@ -1829,9 +1854,19 @@ static int cache_alloc(struct cache *ca)
 
 	bio_init(&ca->journal.bio, ca->journal.bio.bi_inline_vecs, 8);
 
+	/*
+	 * when ca->sb.njournal_buckets is not zero, journal exists,
+	 * and in bch_journal_replay(), tree node may split,
+	 * so bucket of RESERVE_BTREE type is needed,
+	 * the worst situation is all journal buckets are valid journal,
+	 * and all the keys need to replay,
+	 * so the number of  RESERVE_BTREE type buckets should be as much
+	 * as journal buckets
+	 */
+	btree_buckets = ca->sb.njournal_buckets ?: 8;
 	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;
 
-	if (!init_fifo(&ca->free[RESERVE_BTREE], 8, GFP_KERNEL) ||
+	if (!init_fifo(&ca->free[RESERVE_BTREE], btree_buckets, GFP_KERNEL) ||
 	    !init_fifo_exact(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) ||
 	    !init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL) ||
 	    !init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL) ||
@@ -1859,19 +1894,22 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page,
 	const char *err = NULL; /* must be set for any error case */
 	int ret = 0;
 
+	bdevname(bdev, name);
+
 	memcpy(&ca->sb, sb, sizeof(struct cache_sb));
 	ca->bdev = bdev;
 	ca->bdev->bd_holder = ca;
 
 	bio_init(&ca->sb_bio, ca->sb_bio.bi_inline_vecs, 1);
-	ca->sb_bio.bi_io_vec[0].bv_page = sb_page;
+	bio_first_bvec_all(&ca->sb_bio)->bv_page = sb_page;
 	get_page(sb_page);
 
-	if (blk_queue_discard(bdev_get_queue(ca->bdev)))
+	if (blk_queue_discard(bdev_get_queue(bdev)))
 		ca->discard = CACHE_DISCARD(&ca->sb);
 
 	ret = cache_alloc(ca);
 	if (ret != 0) {
+		blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
 		if (ret == -ENOMEM)
 			err = "cache_alloc(): -ENOMEM";
 		else
@@ -1894,14 +1932,14 @@ static int register_cache(struct cache_sb *sb, struct page *sb_page,
 		goto out;
 	}
 
-	pr_info("registered cache device %s", bdevname(bdev, name));
+	pr_info("registered cache device %s", name);
 
 out:
 	kobject_put(&ca->kobj);
 
 err:
 	if (err)
-		pr_notice("error opening %s: %s", bdevname(bdev, name), err);
+		pr_notice("error %s: %s", name, err);
 
 	return ret;
 }
@@ -1990,6 +2028,7 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
 	if (err)
 		goto err_close;
 
+	err = "failed to register device";
 	if (SB_IS_BDEV(sb)) {
 		struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL);
 		if (!dc)
@@ -2004,7 +2043,7 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
 			goto err_close;
 
 		if (register_cache(sb, sb_page, bdev, ca) != 0)
-			goto err_close;
+			goto err;
 	}
 out:
 	if (sb_page)
@@ -2017,7 +2056,7 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
 err_close:
 	blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
 err:
-	pr_info("error opening %s: %s", path, err);
+	pr_info("error %s: %s", path, err);
 	ret = -EINVAL;
 	goto out;
 }
diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
index b4184092c727..78cd7bd50fdd 100644
--- a/drivers/md/bcache/sysfs.c
+++ b/drivers/md/bcache/sysfs.c
@@ -65,6 +65,9 @@ read_attribute(bset_tree_stats);
 
 read_attribute(state);
 read_attribute(cache_read_races);
+read_attribute(reclaim);
+read_attribute(flush_write);
+read_attribute(retry_flush_write);
 read_attribute(writeback_keys_done);
 read_attribute(writeback_keys_failed);
 read_attribute(io_errors);
@@ -195,7 +198,7 @@ STORE(__cached_dev)
 {
 	struct cached_dev *dc = container_of(kobj, struct cached_dev,
 					     disk.kobj);
-	ssize_t v = size;
+	ssize_t v;
 	struct cache_set *c;
 	struct kobj_uevent_env *env;
 
@@ -215,7 +218,9 @@ STORE(__cached_dev)
 	sysfs_strtoul_clamp(writeback_rate,
 			    dc->writeback_rate.rate, 1, INT_MAX);
 
-	d_strtoul_nonzero(writeback_rate_update_seconds);
+	sysfs_strtoul_clamp(writeback_rate_update_seconds,
+			    dc->writeback_rate_update_seconds,
+			    1, WRITEBACK_RATE_UPDATE_SECS_MAX);
 	d_strtoul(writeback_rate_i_term_inverse);
 	d_strtoul_nonzero(writeback_rate_p_term_inverse);
 
@@ -267,17 +272,20 @@ STORE(__cached_dev)
 	}
 
 	if (attr == &sysfs_attach) {
-		if (bch_parse_uuid(buf, dc->sb.set_uuid) < 16)
+		uint8_t		set_uuid[16];
+
+		if (bch_parse_uuid(buf, set_uuid) < 16)
 			return -EINVAL;
 
+		v = -ENOENT;
 		list_for_each_entry(c, &bch_cache_sets, list) {
-			v = bch_cached_dev_attach(dc, c);
+			v = bch_cached_dev_attach(dc, c, set_uuid);
 			if (!v)
 				return size;
 		}
 
 		pr_err("Can't attach %s: cache set not found", buf);
-		size = v;
+		return v;
 	}
 
 	if (attr == &sysfs_detach && dc->disk.c)
@@ -545,6 +553,15 @@ SHOW(__bch_cache_set)
 	sysfs_print(cache_read_races,
 		    atomic_long_read(&c->cache_read_races));
 
+	sysfs_print(reclaim,
+		    atomic_long_read(&c->reclaim));
+
+	sysfs_print(flush_write,
+		    atomic_long_read(&c->flush_write));
+
+	sysfs_print(retry_flush_write,
+		    atomic_long_read(&c->retry_flush_write));
+
 	sysfs_print(writeback_keys_done,
 		    atomic_long_read(&c->writeback_keys_done));
 	sysfs_print(writeback_keys_failed,
@@ -556,7 +573,7 @@ SHOW(__bch_cache_set)
 
 	/* See count_io_errors for why 88 */
 	sysfs_print(io_error_halflife,	c->error_decay * 88);
-	sysfs_print(io_error_limit,	c->error_limit >> IO_ERROR_SHIFT);
+	sysfs_print(io_error_limit,	c->error_limit);
 
 	sysfs_hprint(congested,
 		     ((uint64_t) bch_get_congested(c)) << 9);
@@ -656,7 +673,7 @@ STORE(__bch_cache_set)
 	}
 
 	if (attr == &sysfs_io_error_limit)
-		c->error_limit = strtoul_or_return(buf) << IO_ERROR_SHIFT;
+		c->error_limit = strtoul_or_return(buf);
 
 	/* See count_io_errors() for why 88 */
 	if (attr == &sysfs_io_error_halflife)
@@ -731,6 +748,9 @@ static struct attribute *bch_cache_set_internal_files[] = {
 
 	&sysfs_bset_tree_stats,
 	&sysfs_cache_read_races,
+	&sysfs_reclaim,
+	&sysfs_flush_write,
+	&sysfs_retry_flush_write,
 	&sysfs_writeback_keys_done,
 	&sysfs_writeback_keys_failed,
 
diff --git a/drivers/md/bcache/util.c b/drivers/md/bcache/util.c
index e548b8b51322..a23cd6a14b74 100644
--- a/drivers/md/bcache/util.c
+++ b/drivers/md/bcache/util.c
@@ -249,6 +249,13 @@ uint64_t bch_next_delay(struct bch_ratelimit *d, uint64_t done)
 		: 0;
 }
 
+/*
+ * Generally it isn't good to access .bi_io_vec and .bi_vcnt directly,
+ * the preferred way is bio_add_page, but in this case, bch_bio_map()
+ * supposes that the bvec table is empty, so it is safe to access
+ * .bi_vcnt & .bi_io_vec in this way even after multipage bvec is
+ * supported.
+ */
 void bch_bio_map(struct bio *bio, void *base)
 {
 	size_t size = bio->bi_iter.bi_size;
@@ -276,6 +283,33 @@ start:		bv->bv_len	= min_t(size_t, PAGE_SIZE - bv->bv_offset,
 	}
 }
 
+/**
+ * bch_bio_alloc_pages - allocates a single page for each bvec in a bio
+ * @bio: bio to allocate pages for
+ * @gfp_mask: flags for allocation
+ *
+ * Allocates pages up to @bio->bi_vcnt.
+ *
+ * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
+ * freed.
+ */
+int bch_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
+{
+	int i;
+	struct bio_vec *bv;
+
+	bio_for_each_segment_all(bv, bio, i) {
+		bv->bv_page = alloc_page(gfp_mask);
+		if (!bv->bv_page) {
+			while (--bv >= bio->bi_io_vec)
+				__free_page(bv->bv_page);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
 /*
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group (Any
  * use permitted, subject to terms of PostgreSQL license; see.)
diff --git a/drivers/md/bcache/util.h b/drivers/md/bcache/util.h
index ed5e8a412eb8..a6763db7f061 100644
--- a/drivers/md/bcache/util.h
+++ b/drivers/md/bcache/util.h
@@ -112,6 +112,8 @@ do {									\
 
 #define heap_full(h)	((h)->used == (h)->size)
 
+#define heap_empty(h)	((h)->used == 0)
+
 #define DECLARE_FIFO(type, name)					\
 	struct {							\
 		size_t front, back, size, mask;				\
@@ -558,6 +560,7 @@ static inline unsigned fract_exp_two(unsigned x, unsigned fract_bits)
 }
 
 void bch_bio_map(struct bio *bio, void *base);
+int bch_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask);
 
 static inline sector_t bdev_sectors(struct block_device *bdev)
 {
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index 56a37884ca8b..f1d2fc15abcc 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -18,17 +18,39 @@
 #include <trace/events/bcache.h>
 
 /* Rate limiting */
-
-static void __update_writeback_rate(struct cached_dev *dc)
+static uint64_t __calc_target_rate(struct cached_dev *dc)
 {
 	struct cache_set *c = dc->disk.c;
+
+	/*
+	 * This is the size of the cache, minus the amount used for
+	 * flash-only devices
+	 */
 	uint64_t cache_sectors = c->nbuckets * c->sb.bucket_size -
 				bcache_flash_devs_sectors_dirty(c);
+
+	/*
+	 * Unfortunately there is no control of global dirty data.  If the
+	 * user states that they want 10% dirty data in the cache, and has,
+	 * e.g., 5 backing volumes of equal size, we try and ensure each
+	 * backing volume uses about 2% of the cache for dirty data.
+	 */
+	uint32_t bdev_share =
+		div64_u64(bdev_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT,
+				c->cached_dev_sectors);
+
 	uint64_t cache_dirty_target =
 		div_u64(cache_sectors * dc->writeback_percent, 100);
-	int64_t target = div64_u64(cache_dirty_target * bdev_sectors(dc->bdev),
-				   c->cached_dev_sectors);
 
+	/* Ensure each backing dev gets at least one dirty share */
+	if (bdev_share < 1)
+		bdev_share = 1;
+
+	return (cache_dirty_target * bdev_share) >> WRITEBACK_SHARE_SHIFT;
+}
+
+static void __update_writeback_rate(struct cached_dev *dc)
+{
 	/*
 	 * PI controller:
 	 * Figures out the amount that should be written per second.
@@ -49,6 +71,7 @@ static void __update_writeback_rate(struct cached_dev *dc)
 	 * This acts as a slow, long-term average that is not subject to
 	 * variations in usage like the p term.
 	 */
+	int64_t target = __calc_target_rate(dc);
 	int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);
 	int64_t error = dirty - target;
 	int64_t proportional_scaled =
@@ -116,6 +139,7 @@ static unsigned writeback_delay(struct cached_dev *dc, unsigned sectors)
 struct dirty_io {
 	struct closure		cl;
 	struct cached_dev	*dc;
+	uint16_t		sequence;
 	struct bio		bio;
 };
 
@@ -194,6 +218,27 @@ static void write_dirty(struct closure *cl)
 {
 	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
 	struct keybuf_key *w = io->bio.bi_private;
+	struct cached_dev *dc = io->dc;
+
+	uint16_t next_sequence;
+
+	if (atomic_read(&dc->writeback_sequence_next) != io->sequence) {
+		/* Not our turn to write; wait for a write to complete */
+		closure_wait(&dc->writeback_ordering_wait, cl);
+
+		if (atomic_read(&dc->writeback_sequence_next) == io->sequence) {
+			/*
+			 * Edge case-- it happened in indeterminate order
+			 * relative to when we were added to wait list..
+			 */
+			closure_wake_up(&dc->writeback_ordering_wait);
+		}
+
+		continue_at(cl, write_dirty, io->dc->writeback_write_wq);
+		return;
+	}
+
+	next_sequence = io->sequence + 1;
 
 	/*
 	 * IO errors are signalled using the dirty bit on the key.
@@ -211,6 +256,9 @@ static void write_dirty(struct closure *cl)
 		closure_bio_submit(&io->bio, cl);
 	}
 
+	atomic_set(&dc->writeback_sequence_next, next_sequence);
+	closure_wake_up(&dc->writeback_ordering_wait);
+
 	continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);
 }
 
@@ -219,8 +267,10 @@ static void read_dirty_endio(struct bio *bio)
 	struct keybuf_key *w = bio->bi_private;
 	struct dirty_io *io = w->private;
 
+	/* is_read = 1 */
 	bch_count_io_errors(PTR_CACHE(io->dc->disk.c, &w->key, 0),
-			    bio->bi_status, "reading dirty data from cache");
+			    bio->bi_status, 1,
+			    "reading dirty data from cache");
 
 	dirty_endio(bio);
 }
@@ -237,10 +287,15 @@ static void read_dirty_submit(struct closure *cl)
 static void read_dirty(struct cached_dev *dc)
 {
 	unsigned delay = 0;
-	struct keybuf_key *w;
+	struct keybuf_key *next, *keys[MAX_WRITEBACKS_IN_PASS], *w;
+	size_t size;
+	int nk, i;
 	struct dirty_io *io;
 	struct closure cl;
+	uint16_t sequence = 0;
 
+	BUG_ON(!llist_empty(&dc->writeback_ordering_wait.list));
+	atomic_set(&dc->writeback_sequence_next, sequence);
 	closure_init_stack(&cl);
 
 	/*
@@ -248,45 +303,109 @@ static void read_dirty(struct cached_dev *dc)
 	 * mempools.
 	 */
 
-	while (!kthread_should_stop()) {
-
-		w = bch_keybuf_next(&dc->writeback_keys);
-		if (!w)
-			break;
-
-		BUG_ON(ptr_stale(dc->disk.c, &w->key, 0));
-
-		if (KEY_START(&w->key) != dc->last_read ||
-		    jiffies_to_msecs(delay) > 50)
-			while (!kthread_should_stop() && delay)
-				delay = schedule_timeout_interruptible(delay);
-
-		dc->last_read	= KEY_OFFSET(&w->key);
-
-		io = kzalloc(sizeof(struct dirty_io) + sizeof(struct bio_vec)
-			     * DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
-			     GFP_KERNEL);
-		if (!io)
-			goto err;
-
-		w->private	= io;
-		io->dc		= dc;
-
-		dirty_init(w);
-		bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
-		io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
-		bio_set_dev(&io->bio, PTR_CACHE(dc->disk.c, &w->key, 0)->bdev);
-		io->bio.bi_end_io	= read_dirty_endio;
-
-		if (bio_alloc_pages(&io->bio, GFP_KERNEL))
-			goto err_free;
+	next = bch_keybuf_next(&dc->writeback_keys);
+
+	while (!kthread_should_stop() && next) {
+		size = 0;
+		nk = 0;
+
+		do {
+			BUG_ON(ptr_stale(dc->disk.c, &next->key, 0));
+
+			/*
+			 * Don't combine too many operations, even if they
+			 * are all small.
+			 */
+			if (nk >= MAX_WRITEBACKS_IN_PASS)
+				break;
+
+			/*
+			 * If the current operation is very large, don't
+			 * further combine operations.
+			 */
+			if (size >= MAX_WRITESIZE_IN_PASS)
+				break;
+
+			/*
+			 * Operations are only eligible to be combined
+			 * if they are contiguous.
+			 *
+			 * TODO: add a heuristic willing to fire a
+			 * certain amount of non-contiguous IO per pass,
+			 * so that we can benefit from backing device
+			 * command queueing.
+			 */
+			if ((nk != 0) && bkey_cmp(&keys[nk-1]->key,
+						&START_KEY(&next->key)))
+				break;
+
+			size += KEY_SIZE(&next->key);
+			keys[nk++] = next;
+		} while ((next = bch_keybuf_next(&dc->writeback_keys)));
+
+		/* Now we have gathered a set of 1..5 keys to write back. */
+		for (i = 0; i < nk; i++) {
+			w = keys[i];
+
+			io = kzalloc(sizeof(struct dirty_io) +
+				     sizeof(struct bio_vec) *
+				     DIV_ROUND_UP(KEY_SIZE(&w->key), PAGE_SECTORS),
+				     GFP_KERNEL);
+			if (!io)
+				goto err;
+
+			w->private	= io;
+			io->dc		= dc;
+			io->sequence    = sequence++;
+
+			dirty_init(w);
+			bio_set_op_attrs(&io->bio, REQ_OP_READ, 0);
+			io->bio.bi_iter.bi_sector = PTR_OFFSET(&w->key, 0);
+			bio_set_dev(&io->bio,
+				    PTR_CACHE(dc->disk.c, &w->key, 0)->bdev);
+			io->bio.bi_end_io	= read_dirty_endio;
+
+			if (bch_bio_alloc_pages(&io->bio, GFP_KERNEL))
+				goto err_free;
+
+			trace_bcache_writeback(&w->key);
+
+			down(&dc->in_flight);
+
+			/* We've acquired a semaphore for the maximum
+			 * simultaneous number of writebacks; from here
+			 * everything happens asynchronously.
+			 */
+			closure_call(&io->cl, read_dirty_submit, NULL, &cl);
+		}
 
-		trace_bcache_writeback(&w->key);
+		delay = writeback_delay(dc, size);
 
-		down(&dc->in_flight);
-		closure_call(&io->cl, read_dirty_submit, NULL, &cl);
+		/* If the control system would wait for at least half a
+		 * second, and there's been no reqs hitting the backing disk
+		 * for awhile: use an alternate mode where we have at most
+		 * one contiguous set of writebacks in flight at a time.  If
+		 * someone wants to do IO it will be quick, as it will only
+		 * have to contend with one operation in flight, and we'll
+		 * be round-tripping data to the backing disk as quickly as
+		 * it can accept it.
+		 */
+		if (delay >= HZ / 2) {
+			/* 3 means at least 1.5 seconds, up to 7.5 if we
+			 * have slowed way down.
+			 */
+			if (atomic_inc_return(&dc->backing_idle) >= 3) {
+				/* Wait for current I/Os to finish */
+				closure_sync(&cl);
+				/* And immediately launch a new set. */
+				delay = 0;
+			}
+		}
 
-		delay = writeback_delay(dc, KEY_SIZE(&w->key));
+		while (!kthread_should_stop() && delay) {
+			schedule_timeout_interruptible(delay);
+			delay = writeback_delay(dc, 0);
+		}
 	}
 
 	if (0) {
@@ -445,18 +564,21 @@ static int bch_writeback_thread(void *arg)
 
 	while (!kthread_should_stop()) {
 		down_write(&dc->writeback_lock);
+		set_current_state(TASK_INTERRUPTIBLE);
 		if (!atomic_read(&dc->has_dirty) ||
 		    (!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&
 		     !dc->writeback_running)) {
 			up_write(&dc->writeback_lock);
-			set_current_state(TASK_INTERRUPTIBLE);
 
-			if (kthread_should_stop())
+			if (kthread_should_stop()) {
+				set_current_state(TASK_RUNNING);
 				return 0;
+			}
 
 			schedule();
 			continue;
 		}
+		set_current_state(TASK_RUNNING);
 
 		searched_full_index = refill_dirty(dc);
 
@@ -533,7 +655,7 @@ void bch_cached_dev_writeback_init(struct cached_dev *dc)
 	dc->writeback_rate.rate		= 1024;
 	dc->writeback_rate_minimum	= 8;
 
-	dc->writeback_rate_update_seconds = 5;
+	dc->writeback_rate_update_seconds = WRITEBACK_RATE_UPDATE_SECS_DEFAULT;
 	dc->writeback_rate_p_term_inverse = 40;
 	dc->writeback_rate_i_term_inverse = 10000;
 
diff --git a/drivers/md/bcache/writeback.h b/drivers/md/bcache/writeback.h
index a9e3ffb4b03c..587b25599856 100644
--- a/drivers/md/bcache/writeback.h
+++ b/drivers/md/bcache/writeback.h
@@ -5,6 +5,19 @@
 #define CUTOFF_WRITEBACK	40
 #define CUTOFF_WRITEBACK_SYNC	70
 
+#define MAX_WRITEBACKS_IN_PASS  5
+#define MAX_WRITESIZE_IN_PASS   5000	/* *512b */
+
+#define WRITEBACK_RATE_UPDATE_SECS_MAX		60
+#define WRITEBACK_RATE_UPDATE_SECS_DEFAULT	5
+
+/*
+ * 14 (16384ths) is chosen here as something that each backing device
+ * should be a reasonable fraction of the share, and not to blow up
+ * until individual backing devices are a petabyte.
+ */
+#define WRITEBACK_SHARE_SHIFT   14
+
 static inline uint64_t bcache_dev_sectors_dirty(struct bcache_device *d)
 {
 	uint64_t i, ret = 0;
@@ -21,7 +34,7 @@ static inline uint64_t  bcache_flash_devs_sectors_dirty(struct cache_set *c)
 
 	mutex_lock(&bch_register_lock);
 
-	for (i = 0; i < c->nr_uuids; i++) {
+	for (i = 0; i < c->devices_max_used; i++) {
 		struct bcache_device *d = c->devices[i];
 
 		if (!d || !UUID_FLASH_ONLY(&c->uuids[i]))
diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c
index c546b567f3b5..aa2032fa80d4 100644
--- a/drivers/md/dm-bufio.c
+++ b/drivers/md/dm-bufio.c
@@ -386,9 +386,6 @@ static void __cache_size_refresh(void)
 static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask,
 			       enum data_mode *data_mode)
 {
-	unsigned noio_flag;
-	void *ptr;
-
 	if (c->block_size <= DM_BUFIO_BLOCK_SIZE_SLAB_LIMIT) {
 		*data_mode = DATA_MODE_SLAB;
 		return kmem_cache_alloc(DM_BUFIO_CACHE(c), gfp_mask);
@@ -412,16 +409,15 @@ static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask,
 	 * all allocations done by this process (including pagetables) are done
 	 * as if GFP_NOIO was specified.
 	 */
+	if (gfp_mask & __GFP_NORETRY) {
+		unsigned noio_flag = memalloc_noio_save();
+		void *ptr = __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL);
 
-	if (gfp_mask & __GFP_NORETRY)
-		noio_flag = memalloc_noio_save();
-
-	ptr = __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL);
-
-	if (gfp_mask & __GFP_NORETRY)
 		memalloc_noio_restore(noio_flag);
+		return ptr;
+	}
 
-	return ptr;
+	return __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL);
 }
 
 /*
@@ -662,7 +658,7 @@ static void submit_io(struct dm_buffer *b, int rw, bio_end_io_t *end_io)
 
 	sector = (b->block << b->c->sectors_per_block_bits) + b->c->start;
 
-	if (rw != WRITE) {
+	if (rw != REQ_OP_WRITE) {
 		n_sectors = 1 << b->c->sectors_per_block_bits;
 		offset = 0;
 	} else {
@@ -740,7 +736,7 @@ static void __write_dirty_buffer(struct dm_buffer *b,
 	b->write_end = b->dirty_end;
 
 	if (!write_list)
-		submit_io(b, WRITE, write_endio);
+		submit_io(b, REQ_OP_WRITE, write_endio);
 	else
 		list_add_tail(&b->write_list, write_list);
 }
@@ -753,7 +749,7 @@ static void __flush_write_list(struct list_head *write_list)
 		struct dm_buffer *b =
 			list_entry(write_list->next, struct dm_buffer, write_list);
 		list_del(&b->write_list);
-		submit_io(b, WRITE, write_endio);
+		submit_io(b, REQ_OP_WRITE, write_endio);
 		cond_resched();
 	}
 	blk_finish_plug(&plug);
@@ -1123,7 +1119,7 @@ static void *new_read(struct dm_bufio_client *c, sector_t block,
 		return NULL;
 
 	if (need_submit)
-		submit_io(b, READ, read_endio);
+		submit_io(b, REQ_OP_READ, read_endio);
 
 	wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
 
@@ -1193,7 +1189,7 @@ void dm_bufio_prefetch(struct dm_bufio_client *c,
 			dm_bufio_unlock(c);
 
 			if (need_submit)
-				submit_io(b, READ, read_endio);
+				submit_io(b, REQ_OP_READ, read_endio);
 			dm_bufio_release(b);
 
 			cond_resched();
@@ -1454,7 +1450,7 @@ void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block)
 		old_block = b->block;
 		__unlink_buffer(b);
 		__link_buffer(b, new_block, b->list_mode);
-		submit_io(b, WRITE, write_endio);
+		submit_io(b, REQ_OP_WRITE, write_endio);
 		wait_on_bit_io(&b->state, B_WRITING,
 			       TASK_UNINTERRUPTIBLE);
 		__unlink_buffer(b);
@@ -1716,7 +1712,7 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 			if (!DM_BUFIO_CACHE_NAME(c)) {
 				r = -ENOMEM;
 				mutex_unlock(&dm_bufio_clients_lock);
-				goto bad_cache;
+				goto bad;
 			}
 		}
 
@@ -1727,7 +1723,7 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 			if (!DM_BUFIO_CACHE(c)) {
 				r = -ENOMEM;
 				mutex_unlock(&dm_bufio_clients_lock);
-				goto bad_cache;
+				goto bad;
 			}
 		}
 	}
@@ -1738,27 +1734,28 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 
 		if (!b) {
 			r = -ENOMEM;
-			goto bad_buffer;
+			goto bad;
 		}
 		__free_buffer_wake(b);
 	}
 
+	c->shrinker.count_objects = dm_bufio_shrink_count;
+	c->shrinker.scan_objects = dm_bufio_shrink_scan;
+	c->shrinker.seeks = 1;
+	c->shrinker.batch = 0;
+	r = register_shrinker(&c->shrinker);
+	if (r)
+		goto bad;
+
 	mutex_lock(&dm_bufio_clients_lock);
 	dm_bufio_client_count++;
 	list_add(&c->client_list, &dm_bufio_all_clients);
 	__cache_size_refresh();
 	mutex_unlock(&dm_bufio_clients_lock);
 
-	c->shrinker.count_objects = dm_bufio_shrink_count;
-	c->shrinker.scan_objects = dm_bufio_shrink_scan;
-	c->shrinker.seeks = 1;
-	c->shrinker.batch = 0;
-	register_shrinker(&c->shrinker);
-
 	return c;
 
-bad_buffer:
-bad_cache:
+bad:
 	while (!list_empty(&c->reserved_buffers)) {
 		struct dm_buffer *b = list_entry(c->reserved_buffers.next,
 						 struct dm_buffer, lru_list);
@@ -1767,6 +1764,7 @@ struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsign
 	}
 	dm_io_client_destroy(c->dm_io);
 bad_dm_io:
+	mutex_destroy(&c->lock);
 	kfree(c);
 bad_client:
 	return ERR_PTR(r);
@@ -1811,6 +1809,7 @@ void dm_bufio_client_destroy(struct dm_bufio_client *c)
 		BUG_ON(c->n_buffers[i]);
 
 	dm_io_client_destroy(c->dm_io);
+	mutex_destroy(&c->lock);
 	kfree(c);
 }
 EXPORT_SYMBOL_GPL(dm_bufio_client_destroy);
diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
index 6a14f945783c..3222e21cbbf8 100644
--- a/drivers/md/dm-core.h
+++ b/drivers/md/dm-core.h
@@ -91,8 +91,7 @@ struct mapped_device {
 	/*
 	 * io objects are allocated from here.
 	 */
-	mempool_t *io_pool;
-
+	struct bio_set *io_bs;
 	struct bio_set *bs;
 
 	/*
@@ -130,8 +129,6 @@ struct mapped_device {
 	struct srcu_struct io_barrier;
 };
 
-void dm_init_md_queue(struct mapped_device *md);
-void dm_init_normal_md_queue(struct mapped_device *md);
 int md_in_flight(struct mapped_device *md);
 void disable_write_same(struct mapped_device *md);
 void disable_write_zeroes(struct mapped_device *md);
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 554d60394c06..8168f737590e 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -1446,7 +1446,6 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
 	bio_for_each_segment_all(bv, clone, i) {
 		BUG_ON(!bv->bv_page);
 		mempool_free(bv->bv_page, cc->page_pool);
-		bv->bv_page = NULL;
 	}
 }
 
@@ -2194,6 +2193,8 @@ static void crypt_dtr(struct dm_target *ti)
 	kzfree(cc->cipher_auth);
 	kzfree(cc->authenc_key);
 
+	mutex_destroy(&cc->bio_alloc_lock);
+
 	/* Must zero key material before freeing */
 	kzfree(cc);
 }
@@ -2703,8 +2704,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad;
 	}
 
-	cc->bs = bioset_create(MIN_IOS, 0, (BIOSET_NEED_BVECS |
-					    BIOSET_NEED_RESCUER));
+	cc->bs = bioset_create(MIN_IOS, 0, BIOSET_NEED_BVECS);
 	if (!cc->bs) {
 		ti->error = "Cannot allocate crypt bioset";
 		goto bad;
diff --git a/drivers/md/dm-delay.c b/drivers/md/dm-delay.c
index 288386bfbfb5..1783d80c9cad 100644
--- a/drivers/md/dm-delay.c
+++ b/drivers/md/dm-delay.c
@@ -229,6 +229,8 @@ static void delay_dtr(struct dm_target *ti)
 	if (dc->dev_write)
 		dm_put_device(ti, dc->dev_write);
 
+	mutex_destroy(&dc->timer_lock);
+
 	kfree(dc);
 }
 
diff --git a/drivers/md/dm-flakey.c b/drivers/md/dm-flakey.c
index b82cb1ab1eaa..1b907b15f5c3 100644
--- a/drivers/md/dm-flakey.c
+++ b/drivers/md/dm-flakey.c
@@ -70,6 +70,11 @@ static int parse_features(struct dm_arg_set *as, struct flakey_c *fc,
 		arg_name = dm_shift_arg(as);
 		argc--;
 
+		if (!arg_name) {
+			ti->error = "Insufficient feature arguments";
+			return -EINVAL;
+		}
+
 		/*
 		 * drop_writes
 		 */
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
index b4357ed4d541..a8d914d5abbe 100644
--- a/drivers/md/dm-io.c
+++ b/drivers/md/dm-io.c
@@ -58,8 +58,7 @@ struct dm_io_client *dm_io_client_create(void)
 	if (!client->pool)
 		goto bad;
 
-	client->bios = bioset_create(min_ios, 0, (BIOSET_NEED_BVECS |
-						  BIOSET_NEED_RESCUER));
+	client->bios = bioset_create(min_ios, 0, BIOSET_NEED_BVECS);
 	if (!client->bios)
 		goto bad;
 
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index e52676fa9832..a89fd8f44453 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -1929,15 +1929,15 @@ static int dm_release(struct inode *inode, struct file *filp)
 	return 0;
 }
 
-static unsigned dm_poll(struct file *filp, poll_table *wait)
+static __poll_t dm_poll(struct file *filp, poll_table *wait)
 {
 	struct dm_file *priv = filp->private_data;
-	unsigned mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(filp, &dm_global_eventq, wait);
 
 	if ((int)(atomic_read(&dm_global_event_nr) - priv->global_event_nr) > 0)
-		mask |= POLLIN;
+		mask |= EPOLLIN;
 
 	return mask;
 }
diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c
index eb45cc3df31d..e6e7c686646d 100644
--- a/drivers/md/dm-kcopyd.c
+++ b/drivers/md/dm-kcopyd.c
@@ -477,8 +477,10 @@ static int run_complete_job(struct kcopyd_job *job)
 	 * If this is the master job, the sub jobs have already
 	 * completed so we can free everything.
 	 */
-	if (job->master_job == job)
+	if (job->master_job == job) {
+		mutex_destroy(&job->lock);
 		mempool_free(job, kc->job_pool);
+	}
 	fn(read_err, write_err, context);
 
 	if (atomic_dec_and_test(&kc->nr_jobs))
@@ -750,6 +752,7 @@ int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
 	 * followed by SPLIT_COUNT sub jobs.
 	 */
 	job = mempool_alloc(kc->job_pool, GFP_NOIO);
+	mutex_init(&job->lock);
 
 	/*
 	 * set up for the read.
@@ -811,7 +814,6 @@ int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,
 	if (job->source.count <= SUB_JOB_SIZE)
 		dispatch_job(job);
 	else {
-		mutex_init(&job->lock);
 		job->progress = 0;
 		split_job(job);
 	}
diff --git a/drivers/md/dm-log-writes.c b/drivers/md/dm-log-writes.c
index 189badbeddaf..3362d866793b 100644
--- a/drivers/md/dm-log-writes.c
+++ b/drivers/md/dm-log-writes.c
@@ -594,7 +594,7 @@ static int log_mark(struct log_writes_c *lc, char *data)
 		return -ENOMEM;
 	}
 
-	block->data = kstrndup(data, maxsize, GFP_KERNEL);
+	block->data = kstrndup(data, maxsize - 1, GFP_KERNEL);
 	if (!block->data) {
 		DMERR("Error copying mark data");
 		kfree(block);
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
index f7810cc869ac..a6b7baf31cdd 100644
--- a/drivers/md/dm-mpath.c
+++ b/drivers/md/dm-mpath.c
@@ -64,36 +64,30 @@ struct priority_group {
 
 /* Multipath context */
 struct multipath {
-	struct list_head list;
-	struct dm_target *ti;
-
-	const char *hw_handler_name;
-	char *hw_handler_params;
+	unsigned long flags;		/* Multipath state flags */
 
 	spinlock_t lock;
-
-	unsigned nr_priority_groups;
-	struct list_head priority_groups;
-
-	wait_queue_head_t pg_init_wait;	/* Wait for pg_init completion */
+	enum dm_queue_mode queue_mode;
 
 	struct pgpath *current_pgpath;
 	struct priority_group *current_pg;
 	struct priority_group *next_pg;	/* Switch to this PG if set */
 
-	unsigned long flags;		/* Multipath state flags */
+	atomic_t nr_valid_paths;	/* Total number of usable paths */
+	unsigned nr_priority_groups;
+	struct list_head priority_groups;
 
+	const char *hw_handler_name;
+	char *hw_handler_params;
+	wait_queue_head_t pg_init_wait;	/* Wait for pg_init completion */
 	unsigned pg_init_retries;	/* Number of times to retry pg_init */
 	unsigned pg_init_delay_msecs;	/* Number of msecs before pg_init retry */
-
-	atomic_t nr_valid_paths;	/* Total number of usable paths */
 	atomic_t pg_init_in_progress;	/* Only one pg_init allowed at once */
 	atomic_t pg_init_count;		/* Number of times pg_init called */
 
-	enum dm_queue_mode queue_mode;
-
 	struct mutex work_mutex;
 	struct work_struct trigger_event;
+	struct dm_target *ti;
 
 	struct work_struct process_queued_bios;
 	struct bio_list queued_bios;
@@ -135,10 +129,10 @@ static struct pgpath *alloc_pgpath(void)
 {
 	struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 
-	if (pgpath) {
-		pgpath->is_active = true;
-		INIT_DELAYED_WORK(&pgpath->activate_path, activate_path_work);
-	}
+	if (!pgpath)
+		return NULL;
+
+	pgpath->is_active = true;
 
 	return pgpath;
 }
@@ -193,13 +187,8 @@ static struct multipath *alloc_multipath(struct dm_target *ti)
 	if (m) {
 		INIT_LIST_HEAD(&m->priority_groups);
 		spin_lock_init(&m->lock);
-		set_bit(MPATHF_QUEUE_IO, &m->flags);
 		atomic_set(&m->nr_valid_paths, 0);
-		atomic_set(&m->pg_init_in_progress, 0);
-		atomic_set(&m->pg_init_count, 0);
-		m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
 		INIT_WORK(&m->trigger_event, trigger_event);
-		init_waitqueue_head(&m->pg_init_wait);
 		mutex_init(&m->work_mutex);
 
 		m->queue_mode = DM_TYPE_NONE;
@@ -221,6 +210,7 @@ static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
 			m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
 		else
 			m->queue_mode = DM_TYPE_REQUEST_BASED;
+
 	} else if (m->queue_mode == DM_TYPE_BIO_BASED) {
 		INIT_WORK(&m->process_queued_bios, process_queued_bios);
 		/*
@@ -232,6 +222,16 @@ static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
 
 	dm_table_set_type(ti->table, m->queue_mode);
 
+	/*
+	 * Init fields that are only used when a scsi_dh is attached
+	 * - must do this unconditionally (really doesn't hurt non-SCSI uses)
+	 */
+	set_bit(MPATHF_QUEUE_IO, &m->flags);
+	atomic_set(&m->pg_init_in_progress, 0);
+	atomic_set(&m->pg_init_count, 0);
+	m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
+	init_waitqueue_head(&m->pg_init_wait);
+
 	return 0;
 }
 
@@ -246,6 +246,7 @@ static void free_multipath(struct multipath *m)
 
 	kfree(m->hw_handler_name);
 	kfree(m->hw_handler_params);
+	mutex_destroy(&m->work_mutex);
 	kfree(m);
 }
 
@@ -264,29 +265,23 @@ static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
 	return dm_per_bio_data(bio, multipath_per_bio_data_size());
 }
 
-static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio)
+static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
 {
 	/* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
-	struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 	void *bio_details = mpio + 1;
-
 	return bio_details;
 }
 
-static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p,
-					struct dm_bio_details **bio_details_p)
+static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
 {
 	struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
-	struct dm_bio_details *bio_details = get_bio_details_from_bio(bio);
+	struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
 
-	memset(mpio, 0, sizeof(*mpio));
-	memset(bio_details, 0, sizeof(*bio_details));
-	dm_bio_record(bio_details, bio);
+	mpio->nr_bytes = bio->bi_iter.bi_size;
+	mpio->pgpath = NULL;
+	*mpio_p = mpio;
 
-	if (mpio_p)
-		*mpio_p = mpio;
-	if (bio_details_p)
-		*bio_details_p = bio_details;
+	dm_bio_record(bio_details, bio);
 }
 
 /*-----------------------------------------------
@@ -516,12 +511,10 @@ static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 		return DM_MAPIO_KILL;
 	} else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
 		   test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
-		if (pg_init_all_paths(m))
-			return DM_MAPIO_DELAY_REQUEUE;
-		return DM_MAPIO_REQUEUE;
+		pg_init_all_paths(m);
+		return DM_MAPIO_DELAY_REQUEUE;
 	}
 
-	memset(mpio, 0, sizeof(*mpio));
 	mpio->pgpath = pgpath;
 	mpio->nr_bytes = nr_bytes;
 
@@ -530,12 +523,23 @@ static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 	clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC);
 	if (IS_ERR(clone)) {
 		/* EBUSY, ENODEV or EWOULDBLOCK: requeue */
-		bool queue_dying = blk_queue_dying(q);
-		if (queue_dying) {
+		if (blk_queue_dying(q)) {
 			atomic_inc(&m->pg_init_in_progress);
 			activate_or_offline_path(pgpath);
+			return DM_MAPIO_DELAY_REQUEUE;
 		}
-		return DM_MAPIO_DELAY_REQUEUE;
+
+		/*
+		 * blk-mq's SCHED_RESTART can cover this requeue, so we
+		 * needn't deal with it by DELAY_REQUEUE. More importantly,
+		 * we have to return DM_MAPIO_REQUEUE so that blk-mq can
+		 * get the queue busy feedback (via BLK_STS_RESOURCE),
+		 * otherwise I/O merging can suffer.
+		 */
+		if (q->mq_ops)
+			return DM_MAPIO_REQUEUE;
+		else
+			return DM_MAPIO_DELAY_REQUEUE;
 	}
 	clone->bio = clone->biotail = NULL;
 	clone->rq_disk = bdev->bd_disk;
@@ -557,9 +561,9 @@ static void multipath_release_clone(struct request *clone)
 /*
  * Map cloned bios (bio-based multipath)
  */
-static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio)
+
+static struct pgpath *__map_bio(struct multipath *m, struct bio *bio)
 {
-	size_t nr_bytes = bio->bi_iter.bi_size;
 	struct pgpath *pgpath;
 	unsigned long flags;
 	bool queue_io;
@@ -568,7 +572,7 @@ static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_m
 	pgpath = READ_ONCE(m->current_pgpath);
 	queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
 	if (!pgpath || !queue_io)
-		pgpath = choose_pgpath(m, nr_bytes);
+		pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
 
 	if ((pgpath && queue_io) ||
 	    (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
@@ -576,14 +580,62 @@ static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_m
 		spin_lock_irqsave(&m->lock, flags);
 		bio_list_add(&m->queued_bios, bio);
 		spin_unlock_irqrestore(&m->lock, flags);
+
 		/* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
 		if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
 			pg_init_all_paths(m);
 		else if (!queue_io)
 			queue_work(kmultipathd, &m->process_queued_bios);
-		return DM_MAPIO_SUBMITTED;
+
+		return ERR_PTR(-EAGAIN);
+	}
+
+	return pgpath;
+}
+
+static struct pgpath *__map_bio_fast(struct multipath *m, struct bio *bio)
+{
+	struct pgpath *pgpath;
+	unsigned long flags;
+
+	/* Do we need to select a new pgpath? */
+	/*
+	 * FIXME: currently only switching path if no path (due to failure, etc)
+	 * - which negates the point of using a path selector
+	 */
+	pgpath = READ_ONCE(m->current_pgpath);
+	if (!pgpath)
+		pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
+
+	if (!pgpath) {
+		if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
+			/* Queue for the daemon to resubmit */
+			spin_lock_irqsave(&m->lock, flags);
+			bio_list_add(&m->queued_bios, bio);
+			spin_unlock_irqrestore(&m->lock, flags);
+			queue_work(kmultipathd, &m->process_queued_bios);
+
+			return ERR_PTR(-EAGAIN);
+		}
+		return NULL;
 	}
 
+	return pgpath;
+}
+
+static int __multipath_map_bio(struct multipath *m, struct bio *bio,
+			       struct dm_mpath_io *mpio)
+{
+	struct pgpath *pgpath;
+
+	if (!m->hw_handler_name)
+		pgpath = __map_bio_fast(m, bio);
+	else
+		pgpath = __map_bio(m, bio);
+
+	if (IS_ERR(pgpath))
+		return DM_MAPIO_SUBMITTED;
+
 	if (!pgpath) {
 		if (must_push_back_bio(m))
 			return DM_MAPIO_REQUEUE;
@@ -592,7 +644,6 @@ static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_m
 	}
 
 	mpio->pgpath = pgpath;
-	mpio->nr_bytes = nr_bytes;
 
 	bio->bi_status = 0;
 	bio_set_dev(bio, pgpath->path.dev->bdev);
@@ -601,7 +652,7 @@ static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_m
 	if (pgpath->pg->ps.type->start_io)
 		pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 					      &pgpath->path,
-					      nr_bytes);
+					      mpio->nr_bytes);
 	return DM_MAPIO_REMAPPED;
 }
 
@@ -610,8 +661,7 @@ static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
 	struct multipath *m = ti->private;
 	struct dm_mpath_io *mpio = NULL;
 
-	multipath_init_per_bio_data(bio, &mpio, NULL);
-
+	multipath_init_per_bio_data(bio, &mpio);
 	return __multipath_map_bio(m, bio, mpio);
 }
 
@@ -649,7 +699,9 @@ static void process_queued_bios(struct work_struct *work)
 
 	blk_start_plug(&plug);
 	while ((bio = bio_list_pop(&bios))) {
-		r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio));
+		struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
+		dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
+		r = __multipath_map_bio(m, bio, mpio);
 		switch (r) {
 		case DM_MAPIO_KILL:
 			bio->bi_status = BLK_STS_IOERR;
@@ -752,38 +804,14 @@ static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 	return 0;
 }
 
-static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
-			       struct dm_target *ti)
+static int setup_scsi_dh(struct block_device *bdev, struct multipath *m,
+			 const char *attached_handler_name, char **error)
 {
+	struct request_queue *q = bdev_get_queue(bdev);
 	int r;
-	struct pgpath *p;
-	struct multipath *m = ti->private;
-	struct request_queue *q = NULL;
-	const char *attached_handler_name;
-
-	/* we need at least a path arg */
-	if (as->argc < 1) {
-		ti->error = "no device given";
-		return ERR_PTR(-EINVAL);
-	}
-
-	p = alloc_pgpath();
-	if (!p)
-		return ERR_PTR(-ENOMEM);
-
-	r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
-			  &p->path.dev);
-	if (r) {
-		ti->error = "error getting device";
-		goto bad;
-	}
-
-	if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name)
-		q = bdev_get_queue(p->path.dev->bdev);
 
 	if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
 retain:
-		attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
 		if (attached_handler_name) {
 			/*
 			 * Clear any hw_handler_params associated with a
@@ -811,26 +839,63 @@ static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps
 			char b[BDEVNAME_SIZE];
 
 			printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
-				bdevname(p->path.dev->bdev, b));
+			       bdevname(bdev, b));
 			goto retain;
 		}
 		if (r < 0) {
-			ti->error = "error attaching hardware handler";
-			dm_put_device(ti, p->path.dev);
-			goto bad;
+			*error = "error attaching hardware handler";
+			return r;
 		}
 
 		if (m->hw_handler_params) {
 			r = scsi_dh_set_params(q, m->hw_handler_params);
 			if (r < 0) {
-				ti->error = "unable to set hardware "
-							"handler parameters";
-				dm_put_device(ti, p->path.dev);
-				goto bad;
+				*error = "unable to set hardware handler parameters";
+				return r;
 			}
 		}
 	}
 
+	return 0;
+}
+
+static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
+				 struct dm_target *ti)
+{
+	int r;
+	struct pgpath *p;
+	struct multipath *m = ti->private;
+	struct request_queue *q;
+	const char *attached_handler_name;
+
+	/* we need at least a path arg */
+	if (as->argc < 1) {
+		ti->error = "no device given";
+		return ERR_PTR(-EINVAL);
+	}
+
+	p = alloc_pgpath();
+	if (!p)
+		return ERR_PTR(-ENOMEM);
+
+	r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
+			  &p->path.dev);
+	if (r) {
+		ti->error = "error getting device";
+		goto bad;
+	}
+
+	q = bdev_get_queue(p->path.dev->bdev);
+	attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
+	if (attached_handler_name || m->hw_handler_name) {
+		INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
+		r = setup_scsi_dh(p->path.dev->bdev, m, attached_handler_name, &ti->error);
+		if (r) {
+			dm_put_device(ti, p->path.dev);
+			goto bad;
+		}
+	}
+
 	r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 	if (r) {
 		dm_put_device(ti, p->path.dev);
@@ -838,7 +903,6 @@ static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps
 	}
 
 	return p;
-
  bad:
 	free_pgpath(p);
 	return ERR_PTR(r);
@@ -1151,16 +1215,19 @@ static void multipath_wait_for_pg_init_completion(struct multipath *m)
 
 static void flush_multipath_work(struct multipath *m)
 {
-	set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
-	smp_mb__after_atomic();
+	if (m->hw_handler_name) {
+		set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
+		smp_mb__after_atomic();
+
+		flush_workqueue(kmpath_handlerd);
+		multipath_wait_for_pg_init_completion(m);
+
+		clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
+		smp_mb__after_atomic();
+	}
 
-	flush_workqueue(kmpath_handlerd);
-	multipath_wait_for_pg_init_completion(m);
 	flush_workqueue(kmultipathd);
 	flush_work(&m->trigger_event);
-
-	clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
-	smp_mb__after_atomic();
 }
 
 static void multipath_dtr(struct dm_target *ti)
@@ -1475,21 +1542,6 @@ static void activate_path_work(struct work_struct *work)
 	activate_or_offline_path(pgpath);
 }
 
-static int noretry_error(blk_status_t error)
-{
-	switch (error) {
-	case BLK_STS_NOTSUPP:
-	case BLK_STS_NOSPC:
-	case BLK_STS_TARGET:
-	case BLK_STS_NEXUS:
-	case BLK_STS_MEDIUM:
-		return 1;
-	}
-
-	/* Anything else could be a path failure, so should be retried */
-	return 0;
-}
-
 static int multipath_end_io(struct dm_target *ti, struct request *clone,
 			    blk_status_t error, union map_info *map_context)
 {
@@ -1508,10 +1560,13 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone,
 	 * request into dm core, which will remake a clone request and
 	 * clone bios for it and resubmit it later.
 	 */
-	if (error && !noretry_error(error)) {
+	if (error && blk_path_error(error)) {
 		struct multipath *m = ti->private;
 
-		r = DM_ENDIO_REQUEUE;
+		if (error == BLK_STS_RESOURCE)
+			r = DM_ENDIO_DELAY_REQUEUE;
+		else
+			r = DM_ENDIO_REQUEUE;
 
 		if (pgpath)
 			fail_path(pgpath);
@@ -1536,7 +1591,7 @@ static int multipath_end_io(struct dm_target *ti, struct request *clone,
 }
 
 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
-		blk_status_t *error)
+				blk_status_t *error)
 {
 	struct multipath *m = ti->private;
 	struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
@@ -1544,7 +1599,7 @@ static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
 	unsigned long flags;
 	int r = DM_ENDIO_DONE;
 
-	if (!*error || noretry_error(*error))
+	if (!*error || !blk_path_error(*error))
 		goto done;
 
 	if (pgpath)
@@ -1561,9 +1616,6 @@ static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
 		goto done;
 	}
 
-	/* Queue for the daemon to resubmit */
-	dm_bio_restore(get_bio_details_from_bio(clone), clone);
-
 	spin_lock_irqsave(&m->lock, flags);
 	bio_list_add(&m->queued_bios, clone);
 	spin_unlock_irqrestore(&m->lock, flags);
@@ -1968,8 +2020,9 @@ static int multipath_busy(struct dm_target *ti)
  *---------------------------------------------------------------*/
 static struct target_type multipath_target = {
 	.name = "multipath",
-	.version = {1, 12, 0},
-	.features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
+	.version = {1, 13, 0},
+	.features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE |
+		    DM_TARGET_PASSES_INTEGRITY,
 	.module = THIS_MODULE,
 	.ctr = multipath_ctr,
 	.dtr = multipath_dtr,
diff --git a/drivers/md/dm-queue-length.c b/drivers/md/dm-queue-length.c
index 23f178641794..969c4f1a3633 100644
--- a/drivers/md/dm-queue-length.c
+++ b/drivers/md/dm-queue-length.c
@@ -195,9 +195,6 @@ static struct dm_path *ql_select_path(struct path_selector *ps, size_t nr_bytes)
 	if (list_empty(&s->valid_paths))
 		goto out;
 
-	/* Change preferred (first in list) path to evenly balance. */
-	list_move_tail(s->valid_paths.next, &s->valid_paths);
-
 	list_for_each_entry(pi, &s->valid_paths, list) {
 		if (!best ||
 		    (atomic_read(&pi->qlen) < atomic_read(&best->qlen)))
@@ -210,6 +207,9 @@ static struct dm_path *ql_select_path(struct path_selector *ps, size_t nr_bytes)
 	if (!best)
 		goto out;
 
+	/* Move most recently used to least preferred to evenly balance. */
+	list_move_tail(&best->list, &s->valid_paths);
+
 	ret = best->path;
 out:
 	spin_unlock_irqrestore(&s->lock, flags);
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index 6319d846e0ad..c1d1034ff7b7 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -29,6 +29,9 @@
  */
 #define	MIN_RAID456_JOURNAL_SPACE (4*2048)
 
+/* Global list of all raid sets */
+static LIST_HEAD(raid_sets);
+
 static bool devices_handle_discard_safely = false;
 
 /*
@@ -105,8 +108,6 @@ struct raid_dev {
 #define CTR_FLAG_JOURNAL_DEV		(1 << __CTR_FLAG_JOURNAL_DEV)
 #define CTR_FLAG_JOURNAL_MODE		(1 << __CTR_FLAG_JOURNAL_MODE)
 
-#define RESUME_STAY_FROZEN_FLAGS (CTR_FLAG_DELTA_DISKS | CTR_FLAG_DATA_OFFSET)
-
 /*
  * Definitions of various constructor flags to
  * be used in checks of valid / invalid flags
@@ -209,6 +210,8 @@ struct raid_dev {
 #define RT_FLAG_UPDATE_SBS		3
 #define RT_FLAG_RESHAPE_RS		4
 #define RT_FLAG_RS_SUSPENDED		5
+#define RT_FLAG_RS_IN_SYNC		6
+#define RT_FLAG_RS_RESYNCING		7
 
 /* Array elements of 64 bit needed for rebuild/failed disk bits */
 #define DISKS_ARRAY_ELEMS ((MAX_RAID_DEVICES + (sizeof(uint64_t) * 8 - 1)) / sizeof(uint64_t) / 8)
@@ -224,8 +227,8 @@ struct rs_layout {
 
 struct raid_set {
 	struct dm_target *ti;
+	struct list_head list;
 
-	uint32_t bitmap_loaded;
 	uint32_t stripe_cache_entries;
 	unsigned long ctr_flags;
 	unsigned long runtime_flags;
@@ -270,6 +273,19 @@ static void rs_config_restore(struct raid_set *rs, struct rs_layout *l)
 	mddev->new_chunk_sectors = l->new_chunk_sectors;
 }
 
+/* Find any raid_set in active slot for @rs on global list */
+static struct raid_set *rs_find_active(struct raid_set *rs)
+{
+	struct raid_set *r;
+	struct mapped_device *md = dm_table_get_md(rs->ti->table);
+
+	list_for_each_entry(r, &raid_sets, list)
+		if (r != rs && dm_table_get_md(r->ti->table) == md)
+			return r;
+
+	return NULL;
+}
+
 /* raid10 algorithms (i.e. formats) */
 #define	ALGORITHM_RAID10_DEFAULT	0
 #define	ALGORITHM_RAID10_NEAR		1
@@ -572,7 +588,7 @@ static const char *raid10_md_layout_to_format(int layout)
 }
 
 /* Return md raid10 algorithm for @name */
-static int raid10_name_to_format(const char *name)
+static const int raid10_name_to_format(const char *name)
 {
 	if (!strcasecmp(name, "near"))
 		return ALGORITHM_RAID10_NEAR;
@@ -675,15 +691,11 @@ static struct raid_type *get_raid_type_by_ll(const int level, const int layout)
 	return NULL;
 }
 
-/*
- * Conditionally change bdev capacity of @rs
- * in case of a disk add/remove reshape
- */
-static void rs_set_capacity(struct raid_set *rs)
+/* Adjust rdev sectors */
+static void rs_set_rdev_sectors(struct raid_set *rs)
 {
 	struct mddev *mddev = &rs->md;
 	struct md_rdev *rdev;
-	struct gendisk *gendisk = dm_disk(dm_table_get_md(rs->ti->table));
 
 	/*
 	 * raid10 sets rdev->sector to the device size, which
@@ -692,8 +704,16 @@ static void rs_set_capacity(struct raid_set *rs)
 	rdev_for_each(rdev, mddev)
 		if (!test_bit(Journal, &rdev->flags))
 			rdev->sectors = mddev->dev_sectors;
+}
 
-	set_capacity(gendisk, mddev->array_sectors);
+/*
+ * Change bdev capacity of @rs in case of a disk add/remove reshape
+ */
+static void rs_set_capacity(struct raid_set *rs)
+{
+	struct gendisk *gendisk = dm_disk(dm_table_get_md(rs->ti->table));
+
+	set_capacity(gendisk, rs->md.array_sectors);
 	revalidate_disk(gendisk);
 }
 
@@ -744,6 +764,7 @@ static struct raid_set *raid_set_alloc(struct dm_target *ti, struct raid_type *r
 
 	mddev_init(&rs->md);
 
+	INIT_LIST_HEAD(&rs->list);
 	rs->raid_disks = raid_devs;
 	rs->delta_disks = 0;
 
@@ -761,6 +782,9 @@ static struct raid_set *raid_set_alloc(struct dm_target *ti, struct raid_type *r
 	for (i = 0; i < raid_devs; i++)
 		md_rdev_init(&rs->dev[i].rdev);
 
+	/* Add @rs to global list. */
+	list_add(&rs->list, &raid_sets);
+
 	/*
 	 * Remaining items to be initialized by further RAID params:
 	 *  rs->md.persistent
@@ -773,6 +797,7 @@ static struct raid_set *raid_set_alloc(struct dm_target *ti, struct raid_type *r
 	return rs;
 }
 
+/* Free all @rs allocations and remove it from global list. */
 static void raid_set_free(struct raid_set *rs)
 {
 	int i;
@@ -790,6 +815,8 @@ static void raid_set_free(struct raid_set *rs)
 			dm_put_device(rs->ti, rs->dev[i].data_dev);
 	}
 
+	list_del(&rs->list);
+
 	kfree(rs);
 }
 
@@ -1002,7 +1029,7 @@ static int validate_raid_redundancy(struct raid_set *rs)
 		    !rs->dev[i].rdev.sb_page)
 			rebuild_cnt++;
 
-	switch (rs->raid_type->level) {
+	switch (rs->md.level) {
 	case 0:
 		break;
 	case 1:
@@ -1017,6 +1044,11 @@ static int validate_raid_redundancy(struct raid_set *rs)
 		break;
 	case 10:
 		copies = raid10_md_layout_to_copies(rs->md.new_layout);
+		if (copies < 2) {
+			DMERR("Bogus raid10 data copies < 2!");
+			return -EINVAL;
+		}
+
 		if (rebuild_cnt < copies)
 			break;
 
@@ -1576,6 +1608,24 @@ static sector_t __rdev_sectors(struct raid_set *rs)
 	return 0;
 }
 
+/* Check that calculated dev_sectors fits all component devices. */
+static int _check_data_dev_sectors(struct raid_set *rs)
+{
+	sector_t ds = ~0;
+	struct md_rdev *rdev;
+
+	rdev_for_each(rdev, &rs->md)
+		if (!test_bit(Journal, &rdev->flags) && rdev->bdev) {
+			ds = min(ds, to_sector(i_size_read(rdev->bdev->bd_inode)));
+			if (ds < rs->md.dev_sectors) {
+				rs->ti->error = "Component device(s) too small";
+				return -EINVAL;
+			}
+		}
+
+	return 0;
+}
+
 /* Calculate the sectors per device and per array used for @rs */
 static int rs_set_dev_and_array_sectors(struct raid_set *rs, bool use_mddev)
 {
@@ -1625,7 +1675,7 @@ static int rs_set_dev_and_array_sectors(struct raid_set *rs, bool use_mddev)
 	mddev->array_sectors = array_sectors;
 	mddev->dev_sectors = dev_sectors;
 
-	return 0;
+	return _check_data_dev_sectors(rs);
 bad:
 	rs->ti->error = "Target length not divisible by number of data devices";
 	return -EINVAL;
@@ -1674,8 +1724,11 @@ static void do_table_event(struct work_struct *ws)
 	struct raid_set *rs = container_of(ws, struct raid_set, md.event_work);
 
 	smp_rmb(); /* Make sure we access most actual mddev properties */
-	if (!rs_is_reshaping(rs))
+	if (!rs_is_reshaping(rs)) {
+		if (rs_is_raid10(rs))
+			rs_set_rdev_sectors(rs);
 		rs_set_capacity(rs);
+	}
 	dm_table_event(rs->ti->table);
 }
 
@@ -1860,7 +1913,7 @@ static bool rs_reshape_requested(struct raid_set *rs)
 	if (rs_takeover_requested(rs))
 		return false;
 
-	if (!mddev->level)
+	if (rs_is_raid0(rs))
 		return false;
 
 	change = mddev->new_layout != mddev->layout ||
@@ -1868,7 +1921,7 @@ static bool rs_reshape_requested(struct raid_set *rs)
 		 rs->delta_disks;
 
 	/* Historical case to support raid1 reshape without delta disks */
-	if (mddev->level == 1) {
+	if (rs_is_raid1(rs)) {
 		if (rs->delta_disks)
 			return !!rs->delta_disks;
 
@@ -1876,7 +1929,7 @@ static bool rs_reshape_requested(struct raid_set *rs)
 		       mddev->raid_disks != rs->raid_disks;
 	}
 
-	if (mddev->level == 10)
+	if (rs_is_raid10(rs))
 		return change &&
 		       !__is_raid10_far(mddev->new_layout) &&
 		       rs->delta_disks >= 0;
@@ -2340,7 +2393,7 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev)
 			DMERR("new device%s provided without 'rebuild'",
 			      new_devs > 1 ? "s" : "");
 			return -EINVAL;
-		} else if (rs_is_recovering(rs)) {
+		} else if (!test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags) && rs_is_recovering(rs)) {
 			DMERR("'rebuild' specified while raid set is not in-sync (recovery_cp=%llu)",
 			      (unsigned long long) mddev->recovery_cp);
 			return -EINVAL;
@@ -2640,12 +2693,19 @@ static int rs_adjust_data_offsets(struct raid_set *rs)
 	 * Make sure we got a minimum amount of free sectors per device
 	 */
 	if (rs->data_offset &&
-	    to_sector(i_size_read(rdev->bdev->bd_inode)) - rdev->sectors < MIN_FREE_RESHAPE_SPACE) {
+	    to_sector(i_size_read(rdev->bdev->bd_inode)) - rs->md.dev_sectors < MIN_FREE_RESHAPE_SPACE) {
 		rs->ti->error = data_offset ? "No space for forward reshape" :
 					      "No space for backward reshape";
 		return -ENOSPC;
 	}
 out:
+	/*
+	 * Raise recovery_cp in case data_offset != 0 to
+	 * avoid false recovery positives in the constructor.
+	 */
+	if (rs->md.recovery_cp < rs->md.dev_sectors)
+		rs->md.recovery_cp += rs->dev[0].rdev.data_offset;
+
 	/* Adjust data offsets on all rdevs but on any raid4/5/6 journal device */
 	rdev_for_each(rdev, &rs->md) {
 		if (!test_bit(Journal, &rdev->flags)) {
@@ -2682,14 +2742,14 @@ static int rs_setup_takeover(struct raid_set *rs)
 	sector_t new_data_offset = rs->dev[0].rdev.data_offset ? 0 : rs->data_offset;
 
 	if (rt_is_raid10(rs->raid_type)) {
-		if (mddev->level == 0) {
+		if (rs_is_raid0(rs)) {
 			/* Userpace reordered disks -> adjust raid_disk indexes */
 			__reorder_raid_disk_indexes(rs);
 
 			/* raid0 -> raid10_far layout */
 			mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_FAR,
 								   rs->raid10_copies);
-		} else if (mddev->level == 1)
+		} else if (rs_is_raid1(rs))
 			/* raid1 -> raid10_near layout */
 			mddev->layout = raid10_format_to_md_layout(rs, ALGORITHM_RAID10_NEAR,
 								   rs->raid_disks);
@@ -2777,6 +2837,23 @@ static int rs_prepare_reshape(struct raid_set *rs)
 	return 0;
 }
 
+/* Get reshape sectors from data_offsets or raid set */
+static sector_t _get_reshape_sectors(struct raid_set *rs)
+{
+	struct md_rdev *rdev;
+	sector_t reshape_sectors = 0;
+
+	rdev_for_each(rdev, &rs->md)
+		if (!test_bit(Journal, &rdev->flags)) {
+			reshape_sectors = (rdev->data_offset > rdev->new_data_offset) ?
+					rdev->data_offset - rdev->new_data_offset :
+					rdev->new_data_offset - rdev->data_offset;
+			break;
+		}
+
+	return max(reshape_sectors, (sector_t) rs->data_offset);
+}
+
 /*
  *
  * - change raid layout
@@ -2788,6 +2865,7 @@ static int rs_setup_reshape(struct raid_set *rs)
 {
 	int r = 0;
 	unsigned int cur_raid_devs, d;
+	sector_t reshape_sectors = _get_reshape_sectors(rs);
 	struct mddev *mddev = &rs->md;
 	struct md_rdev *rdev;
 
@@ -2804,13 +2882,13 @@ static int rs_setup_reshape(struct raid_set *rs)
 	/*
 	 * Adjust array size:
 	 *
-	 * - in case of adding disks, array size has
+	 * - in case of adding disk(s), array size has
 	 *   to grow after the disk adding reshape,
 	 *   which'll hapen in the event handler;
 	 *   reshape will happen forward, so space has to
 	 *   be available at the beginning of each disk
 	 *
-	 * - in case of removing disks, array size
+	 * - in case of removing disk(s), array size
 	 *   has to shrink before starting the reshape,
 	 *   which'll happen here;
 	 *   reshape will happen backward, so space has to
@@ -2841,7 +2919,7 @@ static int rs_setup_reshape(struct raid_set *rs)
 			rdev->recovery_offset = rs_is_raid1(rs) ? 0 : MaxSector;
 		}
 
-		mddev->reshape_backwards = 0; /* adding disks -> forward reshape */
+		mddev->reshape_backwards = 0; /* adding disk(s) -> forward reshape */
 
 	/* Remove disk(s) */
 	} else if (rs->delta_disks < 0) {
@@ -2874,6 +2952,15 @@ static int rs_setup_reshape(struct raid_set *rs)
 		mddev->reshape_backwards = rs->dev[0].rdev.data_offset ? 0 : 1;
 	}
 
+	/*
+	 * Adjust device size for forward reshape
+	 * because md_finish_reshape() reduces it.
+	 */
+	if (!mddev->reshape_backwards)
+		rdev_for_each(rdev, &rs->md)
+			if (!test_bit(Journal, &rdev->flags))
+				rdev->sectors += reshape_sectors;
+
 	return r;
 }
 
@@ -2890,7 +2977,7 @@ static void configure_discard_support(struct raid_set *rs)
 	/*
 	 * XXX: RAID level 4,5,6 require zeroing for safety.
 	 */
-	raid456 = (rs->md.level == 4 || rs->md.level == 5 || rs->md.level == 6);
+	raid456 = rs_is_raid456(rs);
 
 	for (i = 0; i < rs->raid_disks; i++) {
 		struct request_queue *q;
@@ -2915,7 +3002,7 @@ static void configure_discard_support(struct raid_set *rs)
 	 * RAID1 and RAID10 personalities require bio splitting,
 	 * RAID0/4/5/6 don't and process large discard bios properly.
 	 */
-	ti->split_discard_bios = !!(rs->md.level == 1 || rs->md.level == 10);
+	ti->split_discard_bios = !!(rs_is_raid1(rs) || rs_is_raid10(rs));
 	ti->num_discard_bios = 1;
 }
 
@@ -2935,10 +3022,10 @@ static void configure_discard_support(struct raid_set *rs)
 static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 {
 	int r;
-	bool resize;
+	bool resize = false;
 	struct raid_type *rt;
 	unsigned int num_raid_params, num_raid_devs;
-	sector_t calculated_dev_sectors, rdev_sectors;
+	sector_t calculated_dev_sectors, rdev_sectors, reshape_sectors;
 	struct raid_set *rs = NULL;
 	const char *arg;
 	struct rs_layout rs_layout;
@@ -3021,7 +3108,10 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad;
 	}
 
-	resize = calculated_dev_sectors != rdev_sectors;
+
+	reshape_sectors = _get_reshape_sectors(rs);
+	if (calculated_dev_sectors != rdev_sectors)
+		resize = calculated_dev_sectors != (reshape_sectors ? rdev_sectors - reshape_sectors : rdev_sectors);
 
 	INIT_WORK(&rs->md.event_work, do_table_event);
 	ti->private = rs;
@@ -3105,19 +3195,22 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 			goto bad;
 		}
 
-		/*
-		  * We can only prepare for a reshape here, because the
-		  * raid set needs to run to provide the repective reshape
-		  * check functions via its MD personality instance.
-		  *
-		  * So do the reshape check after md_run() succeeded.
-		  */
-		r = rs_prepare_reshape(rs);
-		if (r)
-			return r;
+		/* Out-of-place space has to be available to allow for a reshape unless raid1! */
+		if (reshape_sectors || rs_is_raid1(rs)) {
+			/*
+			  * We can only prepare for a reshape here, because the
+			  * raid set needs to run to provide the repective reshape
+			  * check functions via its MD personality instance.
+			  *
+			  * So do the reshape check after md_run() succeeded.
+			  */
+			r = rs_prepare_reshape(rs);
+			if (r)
+				return r;
 
-		/* Reshaping ain't recovery, so disable recovery */
-		rs_setup_recovery(rs, MaxSector);
+			/* Reshaping ain't recovery, so disable recovery */
+			rs_setup_recovery(rs, MaxSector);
+		}
 		rs_set_cur(rs);
 	} else {
 		/* May not set recovery when a device rebuild is requested */
@@ -3144,13 +3237,20 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	mddev_lock_nointr(&rs->md);
 	r = md_run(&rs->md);
 	rs->md.in_sync = 0; /* Assume already marked dirty */
-
 	if (r) {
 		ti->error = "Failed to run raid array";
 		mddev_unlock(&rs->md);
 		goto bad;
 	}
 
+	r = md_start(&rs->md);
+
+	if (r) {
+		ti->error = "Failed to start raid array";
+		mddev_unlock(&rs->md);
+		goto bad_md_start;
+	}
+
 	rs->callbacks.congested_fn = raid_is_congested;
 	dm_table_add_target_callbacks(ti->table, &rs->callbacks);
 
@@ -3198,6 +3298,7 @@ static int raid_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	mddev_unlock(&rs->md);
 	return 0;
 
+bad_md_start:
 bad_journal_mode_set:
 bad_stripe_cache:
 bad_check_reshape:
@@ -3239,25 +3340,27 @@ static int raid_map(struct dm_target *ti, struct bio *bio)
 }
 
 /* Return string describing the current sync action of @mddev */
-static const char *decipher_sync_action(struct mddev *mddev)
+static const char *decipher_sync_action(struct mddev *mddev, unsigned long recovery)
 {
-	if (test_bit(MD_RECOVERY_FROZEN, &mddev->recovery))
+	if (test_bit(MD_RECOVERY_FROZEN, &recovery))
 		return "frozen";
 
-	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||
-	    (!mddev->ro && test_bit(MD_RECOVERY_NEEDED, &mddev->recovery))) {
-		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
+	/* The MD sync thread can be done with io but still be running */
+	if (!test_bit(MD_RECOVERY_DONE, &recovery) &&
+	    (test_bit(MD_RECOVERY_RUNNING, &recovery) ||
+	     (!mddev->ro && test_bit(MD_RECOVERY_NEEDED, &recovery)))) {
+		if (test_bit(MD_RECOVERY_RESHAPE, &recovery))
 			return "reshape";
 
-		if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
-			if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
+		if (test_bit(MD_RECOVERY_SYNC, &recovery)) {
+			if (!test_bit(MD_RECOVERY_REQUESTED, &recovery))
 				return "resync";
-			else if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
+			else if (test_bit(MD_RECOVERY_CHECK, &recovery))
 				return "check";
 			return "repair";
 		}
 
-		if (test_bit(MD_RECOVERY_RECOVER, &mddev->recovery))
+		if (test_bit(MD_RECOVERY_RECOVER, &recovery))
 			return "recover";
 	}
 
@@ -3274,7 +3377,7 @@ static const char *decipher_sync_action(struct mddev *mddev)
  *  'A' = Alive and in-sync raid set component _or_ alive raid4/5/6 'write_through' journal device
  *  '-' = Non-existing device (i.e. uspace passed '- -' into the ctr)
  */
-static const char *__raid_dev_status(struct raid_set *rs, struct md_rdev *rdev, bool array_in_sync)
+static const char *__raid_dev_status(struct raid_set *rs, struct md_rdev *rdev)
 {
 	if (!rdev->bdev)
 		return "-";
@@ -3282,85 +3385,109 @@ static const char *__raid_dev_status(struct raid_set *rs, struct md_rdev *rdev,
 		return "D";
 	else if (test_bit(Journal, &rdev->flags))
 		return (rs->journal_dev.mode == R5C_JOURNAL_MODE_WRITE_THROUGH) ? "A" : "a";
-	else if (!array_in_sync || !test_bit(In_sync, &rdev->flags))
+	else if (test_bit(RT_FLAG_RS_RESYNCING, &rs->runtime_flags) ||
+		 (!test_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags) &&
+		  !test_bit(In_sync, &rdev->flags)))
 		return "a";
 	else
 		return "A";
 }
 
-/* Helper to return resync/reshape progress for @rs and @array_in_sync */
-static sector_t rs_get_progress(struct raid_set *rs,
-				sector_t resync_max_sectors, bool *array_in_sync)
+/* Helper to return resync/reshape progress for @rs and runtime flags for raid set in sync / resynching */
+static sector_t rs_get_progress(struct raid_set *rs, unsigned long recovery,
+				sector_t resync_max_sectors)
 {
-	sector_t r, curr_resync_completed;
+	sector_t r;
 	struct mddev *mddev = &rs->md;
 
-	curr_resync_completed = mddev->curr_resync_completed ?: mddev->recovery_cp;
-	*array_in_sync = false;
+	clear_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags);
+	clear_bit(RT_FLAG_RS_RESYNCING, &rs->runtime_flags);
 
 	if (rs_is_raid0(rs)) {
 		r = resync_max_sectors;
-		*array_in_sync = true;
+		set_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags);
 
 	} else {
-		r = mddev->reshape_position;
-
-		/* Reshape is relative to the array size */
-		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) ||
-		    r != MaxSector) {
-			if (r == MaxSector) {
-				*array_in_sync = true;
-				r = resync_max_sectors;
-			} else {
-				/* Got to reverse on backward reshape */
-				if (mddev->reshape_backwards)
-					r = mddev->array_sectors - r;
-
-				/* Devide by # of data stripes */
-				sector_div(r, mddev_data_stripes(rs));
-			}
-
-		/* Sync is relative to the component device size */
-		} else if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
-			r = curr_resync_completed;
+		if (!test_bit(MD_RECOVERY_INTR, &recovery) &&
+		    (test_bit(MD_RECOVERY_NEEDED, &recovery) ||
+		     test_bit(MD_RECOVERY_RESHAPE, &recovery) ||
+		     test_bit(MD_RECOVERY_RUNNING, &recovery)))
+			r = mddev->curr_resync_completed;
 		else
 			r = mddev->recovery_cp;
 
-		if ((r == MaxSector) ||
-		    (test_bit(MD_RECOVERY_DONE, &mddev->recovery) &&
-		     (mddev->curr_resync_completed == resync_max_sectors))) {
+		if (r >= resync_max_sectors &&
+		    (!test_bit(MD_RECOVERY_REQUESTED, &recovery) ||
+		     (!test_bit(MD_RECOVERY_FROZEN, &recovery) &&
+		      !test_bit(MD_RECOVERY_NEEDED, &recovery) &&
+		      !test_bit(MD_RECOVERY_RUNNING, &recovery)))) {
 			/*
 			 * Sync complete.
 			 */
-			*array_in_sync = true;
-			r = resync_max_sectors;
-		} else if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
+			/* In case we have finished recovering, the array is in sync. */
+			if (test_bit(MD_RECOVERY_RECOVER, &recovery))
+				set_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags);
+
+		} else if (test_bit(MD_RECOVERY_RECOVER, &recovery)) {
+			/*
+			 * In case we are recovering, the array is not in sync
+			 * and health chars should show the recovering legs.
+			 */
+			;
+
+		} else if (test_bit(MD_RECOVERY_SYNC, &recovery) &&
+			   !test_bit(MD_RECOVERY_REQUESTED, &recovery)) {
+			/*
+			 * If "resync" is occurring, the raid set
+			 * is or may be out of sync hence the health
+			 * characters shall be 'a'.
+			 */
+			set_bit(RT_FLAG_RS_RESYNCING, &rs->runtime_flags);
+
+		} else if (test_bit(MD_RECOVERY_RESHAPE, &recovery) &&
+			   !test_bit(MD_RECOVERY_REQUESTED, &recovery)) {
+			/*
+			 * If "reshape" is occurring, the raid set
+			 * is or may be out of sync hence the health
+			 * characters shall be 'a'.
+			 */
+			set_bit(RT_FLAG_RS_RESYNCING, &rs->runtime_flags);
+
+		} else if (test_bit(MD_RECOVERY_REQUESTED, &recovery)) {
 			/*
 			 * If "check" or "repair" is occurring, the raid set has
 			 * undergone an initial sync and the health characters
 			 * should not be 'a' anymore.
 			 */
-			*array_in_sync = true;
+			set_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags);
+
 		} else {
 			struct md_rdev *rdev;
 
+			/*
+			 * We are idle and recovery is needed, prevent 'A' chars race
+			 * caused by components still set to in-sync by constrcuctor.
+			 */
+			if (test_bit(MD_RECOVERY_NEEDED, &recovery))
+				set_bit(RT_FLAG_RS_RESYNCING, &rs->runtime_flags);
+
 			/*
 			 * The raid set may be doing an initial sync, or it may
 			 * be rebuilding individual components.	 If all the
 			 * devices are In_sync, then it is the raid set that is
 			 * being initialized.
 			 */
+			set_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags);
 			rdev_for_each(rdev, mddev)
 				if (!test_bit(Journal, &rdev->flags) &&
-				    !test_bit(In_sync, &rdev->flags))
-					*array_in_sync = true;
-#if 0
-			r = 0; /* HM FIXME: TESTME: https://bugzilla.redhat.com/show_bug.cgi?id=1210637 ? */
-#endif
+				    !test_bit(In_sync, &rdev->flags)) {
+					clear_bit(RT_FLAG_RS_IN_SYNC, &rs->runtime_flags);
+					break;
+				}
 		}
 	}
 
-	return r;
+	return min(r, resync_max_sectors);
 }
 
 /* Helper to return @dev name or "-" if !@dev */
@@ -3376,7 +3503,7 @@ static void raid_status(struct dm_target *ti, status_type_t type,
 	struct mddev *mddev = &rs->md;
 	struct r5conf *conf = mddev->private;
 	int i, max_nr_stripes = conf ? conf->max_nr_stripes : 0;
-	bool array_in_sync;
+	unsigned long recovery;
 	unsigned int raid_param_cnt = 1; /* at least 1 for chunksize */
 	unsigned int sz = 0;
 	unsigned int rebuild_disks;
@@ -3396,17 +3523,18 @@ static void raid_status(struct dm_target *ti, status_type_t type,
 
 		/* Access most recent mddev properties for status output */
 		smp_rmb();
+		recovery = rs->md.recovery;
 		/* Get sensible max sectors even if raid set not yet started */
 		resync_max_sectors = test_bit(RT_FLAG_RS_PRERESUMED, &rs->runtime_flags) ?
 				      mddev->resync_max_sectors : mddev->dev_sectors;
-		progress = rs_get_progress(rs, resync_max_sectors, &array_in_sync);
+		progress = rs_get_progress(rs, recovery, resync_max_sectors);
 		resync_mismatches = (mddev->last_sync_action && !strcasecmp(mddev->last_sync_action, "check")) ?
 				    atomic64_read(&mddev->resync_mismatches) : 0;
-		sync_action = decipher_sync_action(&rs->md);
+		sync_action = decipher_sync_action(&rs->md, recovery);
 
 		/* HM FIXME: do we want another state char for raid0? It shows 'D'/'A'/'-' now */
 		for (i = 0; i < rs->raid_disks; i++)
-			DMEMIT(__raid_dev_status(rs, &rs->dev[i].rdev, array_in_sync));
+			DMEMIT(__raid_dev_status(rs, &rs->dev[i].rdev));
 
 		/*
 		 * In-sync/Reshape ratio:
@@ -3457,7 +3585,7 @@ static void raid_status(struct dm_target *ti, status_type_t type,
 		 * v1.10.0+:
 		 */
 		DMEMIT(" %s", test_bit(__CTR_FLAG_JOURNAL_DEV, &rs->ctr_flags) ?
-			      __raid_dev_status(rs, &rs->journal_dev.rdev, 0) : "-");
+			      __raid_dev_status(rs, &rs->journal_dev.rdev) : "-");
 		break;
 
 	case STATUSTYPE_TABLE:
@@ -3613,24 +3741,19 @@ static void raid_io_hints(struct dm_target *ti, struct queue_limits *limits)
 	blk_limits_io_opt(limits, chunk_size * mddev_data_stripes(rs));
 }
 
-static void raid_presuspend(struct dm_target *ti)
-{
-	struct raid_set *rs = ti->private;
-
-	md_stop_writes(&rs->md);
-}
-
 static void raid_postsuspend(struct dm_target *ti)
 {
 	struct raid_set *rs = ti->private;
 
 	if (!test_and_set_bit(RT_FLAG_RS_SUSPENDED, &rs->runtime_flags)) {
+		/* Writes have to be stopped before suspending to avoid deadlocks. */
+		if (!test_bit(MD_RECOVERY_FROZEN, &rs->md.recovery))
+			md_stop_writes(&rs->md);
+
 		mddev_lock_nointr(&rs->md);
 		mddev_suspend(&rs->md);
 		mddev_unlock(&rs->md);
 	}
-
-	rs->md.ro = 1;
 }
 
 static void attempt_restore_of_faulty_devices(struct raid_set *rs)
@@ -3807,10 +3930,33 @@ static int raid_preresume(struct dm_target *ti)
 	struct raid_set *rs = ti->private;
 	struct mddev *mddev = &rs->md;
 
-	/* This is a resume after a suspend of the set -> it's already started */
+	/* This is a resume after a suspend of the set -> it's already started. */
 	if (test_and_set_bit(RT_FLAG_RS_PRERESUMED, &rs->runtime_flags))
 		return 0;
 
+	if (!test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags)) {
+		struct raid_set *rs_active = rs_find_active(rs);
+
+		if (rs_active) {
+			/*
+			 * In case no rebuilds have been requested
+			 * and an active table slot exists, copy
+			 * current resynchonization completed and
+			 * reshape position pointers across from
+			 * suspended raid set in the active slot.
+			 *
+			 * This resumes the new mapping at current
+			 * offsets to continue recover/reshape without
+			 * necessarily redoing a raid set partially or
+			 * causing data corruption in case of a reshape.
+			 */
+			if (rs_active->md.curr_resync_completed != MaxSector)
+				mddev->curr_resync_completed = rs_active->md.curr_resync_completed;
+			if (rs_active->md.reshape_position != MaxSector)
+				mddev->reshape_position = rs_active->md.reshape_position;
+		}
+	}
+
 	/*
 	 * The superblocks need to be updated on disk if the
 	 * array is new or new devices got added (thus zeroed
@@ -3842,11 +3988,10 @@ static int raid_preresume(struct dm_target *ti)
 		mddev->resync_min = mddev->recovery_cp;
 	}
 
-	rs_set_capacity(rs);
-
 	/* Check for any reshape request unless new raid set */
-	if (test_and_clear_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags)) {
+	if (test_bit(RT_FLAG_RESHAPE_RS, &rs->runtime_flags)) {
 		/* Initiate a reshape. */
+		rs_set_rdev_sectors(rs);
 		mddev_lock_nointr(mddev);
 		r = rs_start_reshape(rs);
 		mddev_unlock(mddev);
@@ -3872,21 +4017,15 @@ static void raid_resume(struct dm_target *ti)
 		attempt_restore_of_faulty_devices(rs);
 	}
 
-	mddev->ro = 0;
-	mddev->in_sync = 0;
-
-	/*
-	 * Keep the RAID set frozen if reshape/rebuild flags are set.
-	 * The RAID set is unfrozen once the next table load/resume,
-	 * which clears the reshape/rebuild flags, occurs.
-	 * This ensures that the constructor for the inactive table
-	 * retrieves an up-to-date reshape_position.
-	 */
-	if (!(rs->ctr_flags & RESUME_STAY_FROZEN_FLAGS))
-		clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
-
 	if (test_and_clear_bit(RT_FLAG_RS_SUSPENDED, &rs->runtime_flags)) {
+		/* Only reduce raid set size before running a disk removing reshape. */
+		if (mddev->delta_disks < 0)
+			rs_set_capacity(rs);
+
 		mddev_lock_nointr(mddev);
+		clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
+		mddev->ro = 0;
+		mddev->in_sync = 0;
 		mddev_resume(mddev);
 		mddev_unlock(mddev);
 	}
@@ -3894,7 +4033,7 @@ static void raid_resume(struct dm_target *ti)
 
 static struct target_type raid_target = {
 	.name = "raid",
-	.version = {1, 13, 0},
+	.version = {1, 13, 2},
 	.module = THIS_MODULE,
 	.ctr = raid_ctr,
 	.dtr = raid_dtr,
@@ -3903,7 +4042,6 @@ static struct target_type raid_target = {
 	.message = raid_message,
 	.iterate_devices = raid_iterate_devices,
 	.io_hints = raid_io_hints,
-	.presuspend = raid_presuspend,
 	.postsuspend = raid_postsuspend,
 	.preresume = raid_preresume,
 	.resume = raid_resume,
diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c
index 9d32f25489c2..bf0b840645cc 100644
--- a/drivers/md/dm-rq.c
+++ b/drivers/md/dm-rq.c
@@ -315,6 +315,10 @@ static void dm_done(struct request *clone, blk_status_t error, bool mapped)
 		/* The target wants to requeue the I/O */
 		dm_requeue_original_request(tio, false);
 		break;
+	case DM_ENDIO_DELAY_REQUEUE:
+		/* The target wants to requeue the I/O after a delay */
+		dm_requeue_original_request(tio, true);
+		break;
 	default:
 		DMWARN("unimplemented target endio return value: %d", r);
 		BUG();
@@ -395,7 +399,7 @@ static void end_clone_request(struct request *clone, blk_status_t error)
 	dm_complete_request(tio->orig, error);
 }
 
-static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
+static blk_status_t dm_dispatch_clone_request(struct request *clone, struct request *rq)
 {
 	blk_status_t r;
 
@@ -404,9 +408,10 @@ static void dm_dispatch_clone_request(struct request *clone, struct request *rq)
 
 	clone->start_time = jiffies;
 	r = blk_insert_cloned_request(clone->q, clone);
-	if (r)
+	if (r != BLK_STS_OK && r != BLK_STS_RESOURCE && r != BLK_STS_DEV_RESOURCE)
 		/* must complete clone in terms of original request */
 		dm_complete_request(rq, r);
+	return r;
 }
 
 static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
@@ -476,8 +481,10 @@ static int map_request(struct dm_rq_target_io *tio)
 	struct mapped_device *md = tio->md;
 	struct request *rq = tio->orig;
 	struct request *clone = NULL;
+	blk_status_t ret;
 
 	r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);
+check_again:
 	switch (r) {
 	case DM_MAPIO_SUBMITTED:
 		/* The target has taken the I/O to submit by itself later */
@@ -492,7 +499,17 @@ static int map_request(struct dm_rq_target_io *tio)
 		/* The target has remapped the I/O so dispatch it */
 		trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
 				     blk_rq_pos(rq));
-		dm_dispatch_clone_request(clone, rq);
+		ret = dm_dispatch_clone_request(clone, rq);
+		if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) {
+			blk_rq_unprep_clone(clone);
+			tio->ti->type->release_clone_rq(clone);
+			tio->clone = NULL;
+			if (!rq->q->mq_ops)
+				r = DM_MAPIO_DELAY_REQUEUE;
+			else
+				r = DM_MAPIO_REQUEUE;
+			goto check_again;
+		}
 		break;
 	case DM_MAPIO_REQUEUE:
 		/* The target wants to requeue the I/O */
@@ -700,7 +717,6 @@ int dm_old_init_request_queue(struct mapped_device *md, struct dm_table *t)
 	/* disable dm_old_request_fn's merge heuristic by default */
 	md->seq_rq_merge_deadline_usecs = 0;
 
-	dm_init_normal_md_queue(md);
 	blk_queue_softirq_done(md->queue, dm_softirq_done);
 
 	/* Initialize the request-based DM worker thread */
@@ -713,8 +729,6 @@ int dm_old_init_request_queue(struct mapped_device *md, struct dm_table *t)
 		return error;
 	}
 
-	elv_register_queue(md->queue);
-
 	return 0;
 }
 
@@ -758,7 +772,6 @@ static blk_status_t dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
 		/* Undo dm_start_request() before requeuing */
 		rq_end_stats(md, rq);
 		rq_completed(md, rq_data_dir(rq), false);
-		blk_mq_delay_run_hw_queue(hctx, 100/*ms*/);
 		return BLK_STS_RESOURCE;
 	}
 
@@ -810,17 +823,9 @@ int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t)
 		err = PTR_ERR(q);
 		goto out_tag_set;
 	}
-	dm_init_md_queue(md);
-
-	/* backfill 'mq' sysfs registration normally done in blk_register_queue */
-	err = blk_mq_register_dev(disk_to_dev(md->disk), q);
-	if (err)
-		goto out_cleanup_queue;
 
 	return 0;
 
-out_cleanup_queue:
-	blk_cleanup_queue(q);
 out_tag_set:
 	blk_mq_free_tag_set(md->tag_set);
 out_kfree_tag_set:
diff --git a/drivers/md/dm-service-time.c b/drivers/md/dm-service-time.c
index 7b8642045c55..f006a9005593 100644
--- a/drivers/md/dm-service-time.c
+++ b/drivers/md/dm-service-time.c
@@ -282,9 +282,6 @@ static struct dm_path *st_select_path(struct path_selector *ps, size_t nr_bytes)
 	if (list_empty(&s->valid_paths))
 		goto out;
 
-	/* Change preferred (first in list) path to evenly balance. */
-	list_move_tail(s->valid_paths.next, &s->valid_paths);
-
 	list_for_each_entry(pi, &s->valid_paths, list)
 		if (!best || (st_compare_load(pi, best, nr_bytes) < 0))
 			best = pi;
@@ -292,6 +289,9 @@ static struct dm_path *st_select_path(struct path_selector *ps, size_t nr_bytes)
 	if (!best)
 		goto out;
 
+	/* Move most recently used to least preferred to evenly balance. */
+	list_move_tail(&best->list, &s->valid_paths);
+
 	ret = best->path;
 out:
 	spin_unlock_irqrestore(&s->lock, flags);
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
index a0613bd8ed00..216035be5661 100644
--- a/drivers/md/dm-snap.c
+++ b/drivers/md/dm-snap.c
@@ -47,7 +47,7 @@ struct dm_exception_table {
 };
 
 struct dm_snapshot {
-	struct rw_semaphore lock;
+	struct mutex lock;
 
 	struct dm_dev *origin;
 	struct dm_dev *cow;
@@ -439,9 +439,9 @@ static int __find_snapshots_sharing_cow(struct dm_snapshot *snap,
 		if (!bdev_equal(s->cow->bdev, snap->cow->bdev))
 			continue;
 
-		down_read(&s->lock);
+		mutex_lock(&s->lock);
 		active = s->active;
-		up_read(&s->lock);
+		mutex_unlock(&s->lock);
 
 		if (active) {
 			if (snap_src)
@@ -909,7 +909,7 @@ static int remove_single_exception_chunk(struct dm_snapshot *s)
 	int r;
 	chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;
 
-	down_write(&s->lock);
+	mutex_lock(&s->lock);
 
 	/*
 	 * Process chunks (and associated exceptions) in reverse order
@@ -924,7 +924,7 @@ static int remove_single_exception_chunk(struct dm_snapshot *s)
 	b = __release_queued_bios_after_merge(s);
 
 out:
-	up_write(&s->lock);
+	mutex_unlock(&s->lock);
 	if (b)
 		flush_bios(b);
 
@@ -983,9 +983,9 @@ static void snapshot_merge_next_chunks(struct dm_snapshot *s)
 		if (linear_chunks < 0) {
 			DMERR("Read error in exception store: "
 			      "shutting down merge");
-			down_write(&s->lock);
+			mutex_lock(&s->lock);
 			s->merge_failed = 1;
-			up_write(&s->lock);
+			mutex_unlock(&s->lock);
 		}
 		goto shut;
 	}
@@ -1026,10 +1026,10 @@ static void snapshot_merge_next_chunks(struct dm_snapshot *s)
 		previous_count = read_pending_exceptions_done_count();
 	}
 
-	down_write(&s->lock);
+	mutex_lock(&s->lock);
 	s->first_merging_chunk = old_chunk;
 	s->num_merging_chunks = linear_chunks;
-	up_write(&s->lock);
+	mutex_unlock(&s->lock);
 
 	/* Wait until writes to all 'linear_chunks' drain */
 	for (i = 0; i < linear_chunks; i++)
@@ -1071,10 +1071,10 @@ static void merge_callback(int read_err, unsigned long write_err, void *context)
 	return;
 
 shut:
-	down_write(&s->lock);
+	mutex_lock(&s->lock);
 	s->merge_failed = 1;
 	b = __release_queued_bios_after_merge(s);
-	up_write(&s->lock);
+	mutex_unlock(&s->lock);
 	error_bios(b);
 
 	merge_shutdown(s);
@@ -1173,7 +1173,7 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	s->exception_start_sequence = 0;
 	s->exception_complete_sequence = 0;
 	INIT_LIST_HEAD(&s->out_of_order_list);
-	init_rwsem(&s->lock);
+	mutex_init(&s->lock);
 	INIT_LIST_HEAD(&s->list);
 	spin_lock_init(&s->pe_lock);
 	s->state_bits = 0;
@@ -1338,9 +1338,9 @@ static void snapshot_dtr(struct dm_target *ti)
 	/* Check whether exception handover must be cancelled */
 	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
 	if (snap_src && snap_dest && (s == snap_src)) {
-		down_write(&snap_dest->lock);
+		mutex_lock(&snap_dest->lock);
 		snap_dest->valid = 0;
-		up_write(&snap_dest->lock);
+		mutex_unlock(&snap_dest->lock);
 		DMERR("Cancelling snapshot handover.");
 	}
 	up_read(&_origins_lock);
@@ -1371,6 +1371,8 @@ static void snapshot_dtr(struct dm_target *ti)
 
 	dm_exception_store_destroy(s->store);
 
+	mutex_destroy(&s->lock);
+
 	dm_put_device(ti, s->cow);
 
 	dm_put_device(ti, s->origin);
@@ -1458,7 +1460,7 @@ static void pending_complete(void *context, int success)
 
 	if (!success) {
 		/* Read/write error - snapshot is unusable */
-		down_write(&s->lock);
+		mutex_lock(&s->lock);
 		__invalidate_snapshot(s, -EIO);
 		error = 1;
 		goto out;
@@ -1466,14 +1468,14 @@ static void pending_complete(void *context, int success)
 
 	e = alloc_completed_exception(GFP_NOIO);
 	if (!e) {
-		down_write(&s->lock);
+		mutex_lock(&s->lock);
 		__invalidate_snapshot(s, -ENOMEM);
 		error = 1;
 		goto out;
 	}
 	*e = pe->e;
 
-	down_write(&s->lock);
+	mutex_lock(&s->lock);
 	if (!s->valid) {
 		free_completed_exception(e);
 		error = 1;
@@ -1498,7 +1500,7 @@ static void pending_complete(void *context, int success)
 		full_bio->bi_end_io = pe->full_bio_end_io;
 	increment_pending_exceptions_done_count();
 
-	up_write(&s->lock);
+	mutex_unlock(&s->lock);
 
 	/* Submit any pending write bios */
 	if (error) {
@@ -1694,7 +1696,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio)
 
 	/* FIXME: should only take write lock if we need
 	 * to copy an exception */
-	down_write(&s->lock);
+	mutex_lock(&s->lock);
 
 	if (!s->valid || (unlikely(s->snapshot_overflowed) &&
 	    bio_data_dir(bio) == WRITE)) {
@@ -1717,9 +1719,9 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio)
 	if (bio_data_dir(bio) == WRITE) {
 		pe = __lookup_pending_exception(s, chunk);
 		if (!pe) {
-			up_write(&s->lock);
+			mutex_unlock(&s->lock);
 			pe = alloc_pending_exception(s);
-			down_write(&s->lock);
+			mutex_lock(&s->lock);
 
 			if (!s->valid || s->snapshot_overflowed) {
 				free_pending_exception(pe);
@@ -1754,7 +1756,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio)
 		    bio->bi_iter.bi_size ==
 		    (s->store->chunk_size << SECTOR_SHIFT)) {
 			pe->started = 1;
-			up_write(&s->lock);
+			mutex_unlock(&s->lock);
 			start_full_bio(pe, bio);
 			goto out;
 		}
@@ -1764,7 +1766,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio)
 		if (!pe->started) {
 			/* this is protected by snap->lock */
 			pe->started = 1;
-			up_write(&s->lock);
+			mutex_unlock(&s->lock);
 			start_copy(pe);
 			goto out;
 		}
@@ -1774,7 +1776,7 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio)
 	}
 
 out_unlock:
-	up_write(&s->lock);
+	mutex_unlock(&s->lock);
 out:
 	return r;
 }
@@ -1810,7 +1812,7 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
 
 	chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
 
-	down_write(&s->lock);
+	mutex_lock(&s->lock);
 
 	/* Full merging snapshots are redirected to the origin */
 	if (!s->valid)
@@ -1841,12 +1843,12 @@ static int snapshot_merge_map(struct dm_target *ti, struct bio *bio)
 	bio_set_dev(bio, s->origin->bdev);
 
 	if (bio_data_dir(bio) == WRITE) {
-		up_write(&s->lock);
+		mutex_unlock(&s->lock);
 		return do_origin(s->origin, bio);
 	}
 
 out_unlock:
-	up_write(&s->lock);
+	mutex_unlock(&s->lock);
 
 	return r;
 }
@@ -1878,7 +1880,7 @@ static int snapshot_preresume(struct dm_target *ti)
 	down_read(&_origins_lock);
 	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
 	if (snap_src && snap_dest) {
-		down_read(&snap_src->lock);
+		mutex_lock(&snap_src->lock);
 		if (s == snap_src) {
 			DMERR("Unable to resume snapshot source until "
 			      "handover completes.");
@@ -1888,7 +1890,7 @@ static int snapshot_preresume(struct dm_target *ti)
 			      "source is suspended.");
 			r = -EINVAL;
 		}
-		up_read(&snap_src->lock);
+		mutex_unlock(&snap_src->lock);
 	}
 	up_read(&_origins_lock);
 
@@ -1934,11 +1936,11 @@ static void snapshot_resume(struct dm_target *ti)
 
 	(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL);
 	if (snap_src && snap_dest) {
-		down_write(&snap_src->lock);
-		down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING);
+		mutex_lock(&snap_src->lock);
+		mutex_lock_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING);
 		__handover_exceptions(snap_src, snap_dest);
-		up_write(&snap_dest->lock);
-		up_write(&snap_src->lock);
+		mutex_unlock(&snap_dest->lock);
+		mutex_unlock(&snap_src->lock);
 	}
 
 	up_read(&_origins_lock);
@@ -1953,9 +1955,9 @@ static void snapshot_resume(struct dm_target *ti)
 	/* Now we have correct chunk size, reregister */
 	reregister_snapshot(s);
 
-	down_write(&s->lock);
+	mutex_lock(&s->lock);
 	s->active = 1;
-	up_write(&s->lock);
+	mutex_unlock(&s->lock);
 }
 
 static uint32_t get_origin_minimum_chunksize(struct block_device *bdev)
@@ -1995,7 +1997,7 @@ static void snapshot_status(struct dm_target *ti, status_type_t type,
 	switch (type) {
 	case STATUSTYPE_INFO:
 
-		down_write(&snap->lock);
+		mutex_lock(&snap->lock);
 
 		if (!snap->valid)
 			DMEMIT("Invalid");
@@ -2020,7 +2022,7 @@ static void snapshot_status(struct dm_target *ti, status_type_t type,
 				DMEMIT("Unknown");
 		}
 
-		up_write(&snap->lock);
+		mutex_unlock(&snap->lock);
 
 		break;
 
@@ -2086,7 +2088,7 @@ static int __origin_write(struct list_head *snapshots, sector_t sector,
 		if (dm_target_is_snapshot_merge(snap->ti))
 			continue;
 
-		down_write(&snap->lock);
+		mutex_lock(&snap->lock);
 
 		/* Only deal with valid and active snapshots */
 		if (!snap->valid || !snap->active)
@@ -2113,9 +2115,9 @@ static int __origin_write(struct list_head *snapshots, sector_t sector,
 
 		pe = __lookup_pending_exception(snap, chunk);
 		if (!pe) {
-			up_write(&snap->lock);
+			mutex_unlock(&snap->lock);
 			pe = alloc_pending_exception(snap);
-			down_write(&snap->lock);
+			mutex_lock(&snap->lock);
 
 			if (!snap->valid) {
 				free_pending_exception(pe);
@@ -2158,7 +2160,7 @@ static int __origin_write(struct list_head *snapshots, sector_t sector,
 		}
 
 next_snapshot:
-		up_write(&snap->lock);
+		mutex_unlock(&snap->lock);
 
 		if (pe_to_start_now) {
 			start_copy(pe_to_start_now);
diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c
index 29bc51084c82..56059fb56e2d 100644
--- a/drivers/md/dm-stats.c
+++ b/drivers/md/dm-stats.c
@@ -228,6 +228,7 @@ void dm_stats_cleanup(struct dm_stats *stats)
 		dm_stat_free(&s->rcu_head);
 	}
 	free_percpu(stats->last);
+	mutex_destroy(&stats->mutex);
 }
 
 static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end,
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
index aaffd0c0ee9a..7eb3e2a3c07d 100644
--- a/drivers/md/dm-table.c
+++ b/drivers/md/dm-table.c
@@ -866,7 +866,8 @@ EXPORT_SYMBOL(dm_consume_args);
 static bool __table_type_bio_based(enum dm_queue_mode table_type)
 {
 	return (table_type == DM_TYPE_BIO_BASED ||
-		table_type == DM_TYPE_DAX_BIO_BASED);
+		table_type == DM_TYPE_DAX_BIO_BASED ||
+		table_type == DM_TYPE_NVME_BIO_BASED);
 }
 
 static bool __table_type_request_based(enum dm_queue_mode table_type)
@@ -909,21 +910,44 @@ static bool dm_table_supports_dax(struct dm_table *t)
 	return true;
 }
 
+static bool dm_table_does_not_support_partial_completion(struct dm_table *t);
+
+struct verify_rq_based_data {
+	unsigned sq_count;
+	unsigned mq_count;
+};
+
+static int device_is_rq_based(struct dm_target *ti, struct dm_dev *dev,
+			      sector_t start, sector_t len, void *data)
+{
+	struct request_queue *q = bdev_get_queue(dev->bdev);
+	struct verify_rq_based_data *v = data;
+
+	if (q->mq_ops)
+		v->mq_count++;
+	else
+		v->sq_count++;
+
+	return queue_is_rq_based(q);
+}
+
 static int dm_table_determine_type(struct dm_table *t)
 {
 	unsigned i;
 	unsigned bio_based = 0, request_based = 0, hybrid = 0;
-	unsigned sq_count = 0, mq_count = 0;
+	struct verify_rq_based_data v = {.sq_count = 0, .mq_count = 0};
 	struct dm_target *tgt;
-	struct dm_dev_internal *dd;
 	struct list_head *devices = dm_table_get_devices(t);
 	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
 
 	if (t->type != DM_TYPE_NONE) {
 		/* target already set the table's type */
-		if (t->type == DM_TYPE_BIO_BASED)
-			return 0;
+		if (t->type == DM_TYPE_BIO_BASED) {
+			/* possibly upgrade to a variant of bio-based */
+			goto verify_bio_based;
+		}
 		BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);
+		BUG_ON(t->type == DM_TYPE_NVME_BIO_BASED);
 		goto verify_rq_based;
 	}
 
@@ -937,8 +961,8 @@ static int dm_table_determine_type(struct dm_table *t)
 			bio_based = 1;
 
 		if (bio_based && request_based) {
-			DMWARN("Inconsistent table: different target types"
-			       " can't be mixed up");
+			DMERR("Inconsistent table: different target types"
+			      " can't be mixed up");
 			return -EINVAL;
 		}
 	}
@@ -956,11 +980,22 @@ static int dm_table_determine_type(struct dm_table *t)
 	}
 
 	if (bio_based) {
+verify_bio_based:
 		/* We must use this table as bio-based */
 		t->type = DM_TYPE_BIO_BASED;
 		if (dm_table_supports_dax(t) ||
-		    (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED))
+		    (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {
 			t->type = DM_TYPE_DAX_BIO_BASED;
+		} else {
+			/* Check if upgrading to NVMe bio-based is valid or required */
+			tgt = dm_table_get_immutable_target(t);
+			if (tgt && !tgt->max_io_len && dm_table_does_not_support_partial_completion(t)) {
+				t->type = DM_TYPE_NVME_BIO_BASED;
+				goto verify_rq_based; /* must be stacked directly on NVMe (blk-mq) */
+			} else if (list_empty(devices) && live_md_type == DM_TYPE_NVME_BIO_BASED) {
+				t->type = DM_TYPE_NVME_BIO_BASED;
+			}
+		}
 		return 0;
 	}
 
@@ -980,7 +1015,8 @@ static int dm_table_determine_type(struct dm_table *t)
 	 * (e.g. request completion process for partial completion.)
 	 */
 	if (t->num_targets > 1) {
-		DMWARN("Request-based dm doesn't support multiple targets yet");
+		DMERR("%s DM doesn't support multiple targets",
+		      t->type == DM_TYPE_NVME_BIO_BASED ? "nvme bio-based" : "request-based");
 		return -EINVAL;
 	}
 
@@ -997,28 +1033,29 @@ static int dm_table_determine_type(struct dm_table *t)
 		return 0;
 	}
 
-	/* Non-request-stackable devices can't be used for request-based dm */
-	list_for_each_entry(dd, devices, list) {
-		struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
-
-		if (!queue_is_rq_based(q)) {
-			DMERR("table load rejected: including"
-			      " non-request-stackable devices");
-			return -EINVAL;
-		}
+	tgt = dm_table_get_immutable_target(t);
+	if (!tgt) {
+		DMERR("table load rejected: immutable target is required");
+		return -EINVAL;
+	} else if (tgt->max_io_len) {
+		DMERR("table load rejected: immutable target that splits IO is not supported");
+		return -EINVAL;
+	}
 
-		if (q->mq_ops)
-			mq_count++;
-		else
-			sq_count++;
+	/* Non-request-stackable devices can't be used for request-based dm */
+	if (!tgt->type->iterate_devices ||
+	    !tgt->type->iterate_devices(tgt, device_is_rq_based, &v)) {
+		DMERR("table load rejected: including non-request-stackable devices");
+		return -EINVAL;
 	}
-	if (sq_count && mq_count) {
+	if (v.sq_count && v.mq_count) {
 		DMERR("table load rejected: not all devices are blk-mq request-stackable");
 		return -EINVAL;
 	}
-	t->all_blk_mq = mq_count > 0;
+	t->all_blk_mq = v.mq_count > 0;
 
-	if (t->type == DM_TYPE_MQ_REQUEST_BASED && !t->all_blk_mq) {
+	if (!t->all_blk_mq &&
+	    (t->type == DM_TYPE_MQ_REQUEST_BASED || t->type == DM_TYPE_NVME_BIO_BASED)) {
 		DMERR("table load rejected: all devices are not blk-mq request-stackable");
 		return -EINVAL;
 	}
@@ -1079,7 +1116,8 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *
 {
 	enum dm_queue_mode type = dm_table_get_type(t);
 	unsigned per_io_data_size = 0;
-	struct dm_target *tgt;
+	unsigned min_pool_size = 0;
+	struct dm_target *ti;
 	unsigned i;
 
 	if (unlikely(type == DM_TYPE_NONE)) {
@@ -1089,11 +1127,13 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *
 
 	if (__table_type_bio_based(type))
 		for (i = 0; i < t->num_targets; i++) {
-			tgt = t->targets + i;
-			per_io_data_size = max(per_io_data_size, tgt->per_io_data_size);
+			ti = t->targets + i;
+			per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
+			min_pool_size = max(min_pool_size, ti->num_flush_bios);
 		}
 
-	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported, per_io_data_size);
+	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported,
+					   per_io_data_size, min_pool_size);
 	if (!t->mempools)
 		return -ENOMEM;
 
@@ -1705,6 +1745,20 @@ static bool dm_table_all_devices_attribute(struct dm_table *t,
 	return true;
 }
 
+static int device_no_partial_completion(struct dm_target *ti, struct dm_dev *dev,
+					sector_t start, sector_t len, void *data)
+{
+	char b[BDEVNAME_SIZE];
+
+	/* For now, NVMe devices are the only devices of this class */
+	return (strncmp(bdevname(dev->bdev, b), "nvme", 4) == 0);
+}
+
+static bool dm_table_does_not_support_partial_completion(struct dm_table *t)
+{
+	return dm_table_all_devices_attribute(t, device_no_partial_completion);
+}
+
 static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev,
 					 sector_t start, sector_t len, void *data)
 {
@@ -1820,6 +1874,8 @@ void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
 	}
 	blk_queue_write_cache(q, wc, fua);
 
+	if (dm_table_supports_dax(t))
+		queue_flag_set_unlocked(QUEUE_FLAG_DAX, q);
 	if (dm_table_supports_dax_write_cache(t))
 		dax_write_cache(t->md->dax_dev, true);
 
diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c
index f91d771fff4b..629c555890c1 100644
--- a/drivers/md/dm-thin.c
+++ b/drivers/md/dm-thin.c
@@ -492,6 +492,11 @@ static void pool_table_init(void)
 	INIT_LIST_HEAD(&dm_thin_pool_table.pools);
 }
 
+static void pool_table_exit(void)
+{
+	mutex_destroy(&dm_thin_pool_table.mutex);
+}
+
 static void __pool_table_insert(struct pool *pool)
 {
 	BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
@@ -1717,7 +1722,7 @@ static void __remap_and_issue_shared_cell(void *context,
 		    bio_op(bio) == REQ_OP_DISCARD)
 			bio_list_add(&info->defer_bios, bio);
 		else {
-			struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));;
+			struct dm_thin_endio_hook *h = dm_per_bio_data(bio, sizeof(struct dm_thin_endio_hook));
 
 			h->shared_read_entry = dm_deferred_entry_inc(info->tc->pool->shared_read_ds);
 			inc_all_io_entry(info->tc->pool, bio);
@@ -4387,6 +4392,8 @@ static void dm_thin_exit(void)
 	dm_unregister_target(&pool_target);
 
 	kmem_cache_destroy(_new_mapping_cache);
+
+	pool_table_exit();
 }
 
 module_init(dm_thin_init);
diff --git a/drivers/md/dm-unstripe.c b/drivers/md/dm-unstripe.c
new file mode 100644
index 000000000000..65f838fa2e99
--- /dev/null
+++ b/drivers/md/dm-unstripe.c
@@ -0,0 +1,219 @@
+/*
+ * Copyright (C) 2017 Intel Corporation.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/device-mapper.h>
+
+struct unstripe_c {
+	struct dm_dev *dev;
+	sector_t physical_start;
+
+	uint32_t stripes;
+
+	uint32_t unstripe;
+	sector_t unstripe_width;
+	sector_t unstripe_offset;
+
+	uint32_t chunk_size;
+	u8 chunk_shift;
+};
+
+#define DM_MSG_PREFIX "unstriped"
+
+static void cleanup_unstripe(struct unstripe_c *uc, struct dm_target *ti)
+{
+	if (uc->dev)
+		dm_put_device(ti, uc->dev);
+	kfree(uc);
+}
+
+/*
+ * Contruct an unstriped mapping.
+ * <number of stripes> <chunk size> <stripe #> <dev_path> <offset>
+ */
+static int unstripe_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+	struct unstripe_c *uc;
+	sector_t tmp_len;
+	unsigned long long start;
+	char dummy;
+
+	if (argc != 5) {
+		ti->error = "Invalid number of arguments";
+		return -EINVAL;
+	}
+
+	uc = kzalloc(sizeof(*uc), GFP_KERNEL);
+	if (!uc) {
+		ti->error = "Memory allocation for unstriped context failed";
+		return -ENOMEM;
+	}
+
+	if (kstrtouint(argv[0], 10, &uc->stripes) || !uc->stripes) {
+		ti->error = "Invalid stripe count";
+		goto err;
+	}
+
+	if (kstrtouint(argv[1], 10, &uc->chunk_size) || !uc->chunk_size) {
+		ti->error = "Invalid chunk_size";
+		goto err;
+	}
+
+	// FIXME: must support non power of 2 chunk_size, dm-stripe.c does
+	if (!is_power_of_2(uc->chunk_size)) {
+		ti->error = "Non power of 2 chunk_size is not supported yet";
+		goto err;
+	}
+
+	if (kstrtouint(argv[2], 10, &uc->unstripe)) {
+		ti->error = "Invalid stripe number";
+		goto err;
+	}
+
+	if (uc->unstripe > uc->stripes && uc->stripes > 1) {
+		ti->error = "Please provide stripe between [0, # of stripes]";
+		goto err;
+	}
+
+	if (dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &uc->dev)) {
+		ti->error = "Couldn't get striped device";
+		goto err;
+	}
+
+	if (sscanf(argv[4], "%llu%c", &start, &dummy) != 1) {
+		ti->error = "Invalid striped device offset";
+		goto err;
+	}
+	uc->physical_start = start;
+
+	uc->unstripe_offset = uc->unstripe * uc->chunk_size;
+	uc->unstripe_width = (uc->stripes - 1) * uc->chunk_size;
+	uc->chunk_shift = fls(uc->chunk_size) - 1;
+
+	tmp_len = ti->len;
+	if (sector_div(tmp_len, uc->chunk_size)) {
+		ti->error = "Target length not divisible by chunk size";
+		goto err;
+	}
+
+	if (dm_set_target_max_io_len(ti, uc->chunk_size)) {
+		ti->error = "Failed to set max io len";
+		goto err;
+	}
+
+	ti->private = uc;
+	return 0;
+err:
+	cleanup_unstripe(uc, ti);
+	return -EINVAL;
+}
+
+static void unstripe_dtr(struct dm_target *ti)
+{
+	struct unstripe_c *uc = ti->private;
+
+	cleanup_unstripe(uc, ti);
+}
+
+static sector_t map_to_core(struct dm_target *ti, struct bio *bio)
+{
+	struct unstripe_c *uc = ti->private;
+	sector_t sector = bio->bi_iter.bi_sector;
+
+	/* Shift us up to the right "row" on the stripe */
+	sector += uc->unstripe_width * (sector >> uc->chunk_shift);
+
+	/* Account for what stripe we're operating on */
+	sector += uc->unstripe_offset;
+
+	return sector;
+}
+
+static int unstripe_map(struct dm_target *ti, struct bio *bio)
+{
+	struct unstripe_c *uc = ti->private;
+
+	bio_set_dev(bio, uc->dev->bdev);
+	bio->bi_iter.bi_sector = map_to_core(ti, bio) + uc->physical_start;
+
+	return DM_MAPIO_REMAPPED;
+}
+
+static void unstripe_status(struct dm_target *ti, status_type_t type,
+			    unsigned int status_flags, char *result, unsigned int maxlen)
+{
+	struct unstripe_c *uc = ti->private;
+	unsigned int sz = 0;
+
+	switch (type) {
+	case STATUSTYPE_INFO:
+		break;
+
+	case STATUSTYPE_TABLE:
+		DMEMIT("%d %llu %d %s %llu",
+		       uc->stripes, (unsigned long long)uc->chunk_size, uc->unstripe,
+		       uc->dev->name, (unsigned long long)uc->physical_start);
+		break;
+	}
+}
+
+static int unstripe_iterate_devices(struct dm_target *ti,
+				    iterate_devices_callout_fn fn, void *data)
+{
+	struct unstripe_c *uc = ti->private;
+
+	return fn(ti, uc->dev, uc->physical_start, ti->len, data);
+}
+
+static void unstripe_io_hints(struct dm_target *ti,
+			       struct queue_limits *limits)
+{
+	struct unstripe_c *uc = ti->private;
+
+	limits->chunk_sectors = uc->chunk_size;
+}
+
+static struct target_type unstripe_target = {
+	.name = "unstriped",
+	.version = {1, 0, 0},
+	.module = THIS_MODULE,
+	.ctr = unstripe_ctr,
+	.dtr = unstripe_dtr,
+	.map = unstripe_map,
+	.status = unstripe_status,
+	.iterate_devices = unstripe_iterate_devices,
+	.io_hints = unstripe_io_hints,
+};
+
+static int __init dm_unstripe_init(void)
+{
+	int r;
+
+	r = dm_register_target(&unstripe_target);
+	if (r < 0)
+		DMERR("target registration failed");
+
+	return r;
+}
+
+static void __exit dm_unstripe_exit(void)
+{
+	dm_unregister_target(&unstripe_target);
+}
+
+module_init(dm_unstripe_init);
+module_exit(dm_unstripe_exit);
+
+MODULE_DESCRIPTION(DM_NAME " unstriped target");
+MODULE_AUTHOR("Scott Bauer <scott.bauer@intel.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-zoned-metadata.c b/drivers/md/dm-zoned-metadata.c
index 70485de37b66..969954915566 100644
--- a/drivers/md/dm-zoned-metadata.c
+++ b/drivers/md/dm-zoned-metadata.c
@@ -2333,6 +2333,9 @@ static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
 
 	/* Free the zone descriptors */
 	dmz_drop_zones(zmd);
+
+	mutex_destroy(&zmd->mblk_flush_lock);
+	mutex_destroy(&zmd->map_lock);
 }
 
 /*
diff --git a/drivers/md/dm-zoned-target.c b/drivers/md/dm-zoned-target.c
index 6d7bda6f8190..caff02caf083 100644
--- a/drivers/md/dm-zoned-target.c
+++ b/drivers/md/dm-zoned-target.c
@@ -827,6 +827,7 @@ static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 err_cwq:
 	destroy_workqueue(dmz->chunk_wq);
 err_bio:
+	mutex_destroy(&dmz->chunk_lock);
 	bioset_free(dmz->bio_set);
 err_meta:
 	dmz_dtr_metadata(dmz->metadata);
@@ -861,6 +862,8 @@ static void dmz_dtr(struct dm_target *ti)
 
 	dmz_put_zoned_device(ti);
 
+	mutex_destroy(&dmz->chunk_lock);
+
 	kfree(dmz);
 }
 
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index de17b7193299..353ea0ede091 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -60,18 +60,73 @@ void dm_issue_global_event(void)
 }
 
 /*
- * One of these is allocated per bio.
+ * One of these is allocated (on-stack) per original bio.
  */
+struct clone_info {
+	struct dm_table *map;
+	struct bio *bio;
+	struct dm_io *io;
+	sector_t sector;
+	unsigned sector_count;
+};
+
+/*
+ * One of these is allocated per clone bio.
+ */
+#define DM_TIO_MAGIC 7282014
+struct dm_target_io {
+	unsigned magic;
+	struct dm_io *io;
+	struct dm_target *ti;
+	unsigned target_bio_nr;
+	unsigned *len_ptr;
+	bool inside_dm_io;
+	struct bio clone;
+};
+
+/*
+ * One of these is allocated per original bio.
+ * It contains the first clone used for that original.
+ */
+#define DM_IO_MAGIC 5191977
 struct dm_io {
+	unsigned magic;
 	struct mapped_device *md;
 	blk_status_t status;
 	atomic_t io_count;
-	struct bio *bio;
+	struct bio *orig_bio;
 	unsigned long start_time;
 	spinlock_t endio_lock;
 	struct dm_stats_aux stats_aux;
+	/* last member of dm_target_io is 'struct bio' */
+	struct dm_target_io tio;
 };
 
+void *dm_per_bio_data(struct bio *bio, size_t data_size)
+{
+	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
+	if (!tio->inside_dm_io)
+		return (char *)bio - offsetof(struct dm_target_io, clone) - data_size;
+	return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size;
+}
+EXPORT_SYMBOL_GPL(dm_per_bio_data);
+
+struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
+{
+	struct dm_io *io = (struct dm_io *)((char *)data + data_size);
+	if (io->magic == DM_IO_MAGIC)
+		return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone));
+	BUG_ON(io->magic != DM_TIO_MAGIC);
+	return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone));
+}
+EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
+
+unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
+{
+	return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
+}
+EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
+
 #define MINOR_ALLOCED ((void *)-1)
 
 /*
@@ -93,8 +148,8 @@ static int dm_numa_node = DM_NUMA_NODE;
  * For mempools pre-allocation at the table loading time.
  */
 struct dm_md_mempools {
-	mempool_t *io_pool;
 	struct bio_set *bs;
+	struct bio_set *io_bs;
 };
 
 struct table_device {
@@ -103,7 +158,6 @@ struct table_device {
 	struct dm_dev dm_dev;
 };
 
-static struct kmem_cache *_io_cache;
 static struct kmem_cache *_rq_tio_cache;
 static struct kmem_cache *_rq_cache;
 
@@ -170,14 +224,9 @@ static int __init local_init(void)
 {
 	int r = -ENOMEM;
 
-	/* allocate a slab for the dm_ios */
-	_io_cache = KMEM_CACHE(dm_io, 0);
-	if (!_io_cache)
-		return r;
-
 	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
 	if (!_rq_tio_cache)
-		goto out_free_io_cache;
+		return r;
 
 	_rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
 				      __alignof__(struct request), 0, NULL);
@@ -212,8 +261,6 @@ static int __init local_init(void)
 	kmem_cache_destroy(_rq_cache);
 out_free_rq_tio_cache:
 	kmem_cache_destroy(_rq_tio_cache);
-out_free_io_cache:
-	kmem_cache_destroy(_io_cache);
 
 	return r;
 }
@@ -225,7 +272,6 @@ static void local_exit(void)
 
 	kmem_cache_destroy(_rq_cache);
 	kmem_cache_destroy(_rq_tio_cache);
-	kmem_cache_destroy(_io_cache);
 	unregister_blkdev(_major, _name);
 	dm_uevent_exit();
 
@@ -412,13 +458,15 @@ static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 	return dm_get_geometry(md, geo);
 }
 
-static int dm_grab_bdev_for_ioctl(struct mapped_device *md,
-				  struct block_device **bdev,
-				  fmode_t *mode)
+static char *_dm_claim_ptr = "I belong to device-mapper";
+
+static int dm_get_bdev_for_ioctl(struct mapped_device *md,
+				 struct block_device **bdev,
+				 fmode_t *mode)
 {
 	struct dm_target *tgt;
 	struct dm_table *map;
-	int srcu_idx, r;
+	int srcu_idx, r, r2;
 
 retry:
 	r = -ENOTTY;
@@ -444,6 +492,12 @@ static int dm_grab_bdev_for_ioctl(struct mapped_device *md,
 		goto out;
 
 	bdgrab(*bdev);
+	r2 = blkdev_get(*bdev, *mode, _dm_claim_ptr);
+	if (r2 < 0) {
+		r = r2;
+		goto out;
+	}
+
 	dm_put_live_table(md, srcu_idx);
 	return r;
 
@@ -462,7 +516,7 @@ static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
 	struct mapped_device *md = bdev->bd_disk->private_data;
 	int r;
 
-	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
+	r = dm_get_bdev_for_ioctl(md, &bdev, &mode);
 	if (r < 0)
 		return r;
 
@@ -482,22 +536,73 @@ static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
 
 	r =  __blkdev_driver_ioctl(bdev, mode, cmd, arg);
 out:
-	bdput(bdev);
+	blkdev_put(bdev, mode);
 	return r;
 }
 
-static struct dm_io *alloc_io(struct mapped_device *md)
+static void start_io_acct(struct dm_io *io);
+
+static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
 {
-	return mempool_alloc(md->io_pool, GFP_NOIO);
+	struct dm_io *io;
+	struct dm_target_io *tio;
+	struct bio *clone;
+
+	clone = bio_alloc_bioset(GFP_NOIO, 0, md->io_bs);
+	if (!clone)
+		return NULL;
+
+	tio = container_of(clone, struct dm_target_io, clone);
+	tio->inside_dm_io = true;
+	tio->io = NULL;
+
+	io = container_of(tio, struct dm_io, tio);
+	io->magic = DM_IO_MAGIC;
+	io->status = 0;
+	atomic_set(&io->io_count, 1);
+	io->orig_bio = bio;
+	io->md = md;
+	spin_lock_init(&io->endio_lock);
+
+	start_io_acct(io);
+
+	return io;
 }
 
 static void free_io(struct mapped_device *md, struct dm_io *io)
 {
-	mempool_free(io, md->io_pool);
+	bio_put(&io->tio.clone);
+}
+
+static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
+				      unsigned target_bio_nr, gfp_t gfp_mask)
+{
+	struct dm_target_io *tio;
+
+	if (!ci->io->tio.io) {
+		/* the dm_target_io embedded in ci->io is available */
+		tio = &ci->io->tio;
+	} else {
+		struct bio *clone = bio_alloc_bioset(gfp_mask, 0, ci->io->md->bs);
+		if (!clone)
+			return NULL;
+
+		tio = container_of(clone, struct dm_target_io, clone);
+		tio->inside_dm_io = false;
+	}
+
+	tio->magic = DM_TIO_MAGIC;
+	tio->io = ci->io;
+	tio->ti = ti;
+	tio->target_bio_nr = target_bio_nr;
+
+	return tio;
 }
 
 static void free_tio(struct dm_target_io *tio)
 {
+	if (tio->inside_dm_io)
+		return;
 	bio_put(&tio->clone);
 }
 
@@ -510,17 +615,15 @@ int md_in_flight(struct mapped_device *md)
 static void start_io_acct(struct dm_io *io)
 {
 	struct mapped_device *md = io->md;
-	struct bio *bio = io->bio;
-	int cpu;
+	struct bio *bio = io->orig_bio;
 	int rw = bio_data_dir(bio);
 
 	io->start_time = jiffies;
 
-	cpu = part_stat_lock();
-	part_round_stats(md->queue, cpu, &dm_disk(md)->part0);
-	part_stat_unlock();
+	generic_start_io_acct(md->queue, rw, bio_sectors(bio), &dm_disk(md)->part0);
+
 	atomic_set(&dm_disk(md)->part0.in_flight[rw],
-		atomic_inc_return(&md->pending[rw]));
+		   atomic_inc_return(&md->pending[rw]));
 
 	if (unlikely(dm_stats_used(&md->stats)))
 		dm_stats_account_io(&md->stats, bio_data_dir(bio),
@@ -531,7 +634,7 @@ static void start_io_acct(struct dm_io *io)
 static void end_io_acct(struct dm_io *io)
 {
 	struct mapped_device *md = io->md;
-	struct bio *bio = io->bio;
+	struct bio *bio = io->orig_bio;
 	unsigned long duration = jiffies - io->start_time;
 	int pending;
 	int rw = bio_data_dir(bio);
@@ -613,14 +716,13 @@ static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
 static int open_table_device(struct table_device *td, dev_t dev,
 			     struct mapped_device *md)
 {
-	static char *_claim_ptr = "I belong to device-mapper";
 	struct block_device *bdev;
 
 	int r;
 
 	BUG_ON(td->dm_dev.bdev);
 
-	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
+	bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
 	if (IS_ERR(bdev))
 		return PTR_ERR(bdev);
 
@@ -752,15 +854,6 @@ int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
 	return 0;
 }
 
-/*-----------------------------------------------------------------
- * CRUD START:
- *   A more elegant soln is in the works that uses the queue
- *   merge fn, unfortunately there are a couple of changes to
- *   the block layer that I want to make for this.  So in the
- *   interests of getting something for people to use I give
- *   you this clearly demarcated crap.
- *---------------------------------------------------------------*/
-
 static int __noflush_suspending(struct mapped_device *md)
 {
 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
@@ -780,8 +873,7 @@ static void dec_pending(struct dm_io *io, blk_status_t error)
 	/* Push-back supersedes any I/O errors */
 	if (unlikely(error)) {
 		spin_lock_irqsave(&io->endio_lock, flags);
-		if (!(io->status == BLK_STS_DM_REQUEUE &&
-				__noflush_suspending(md)))
+		if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
 			io->status = error;
 		spin_unlock_irqrestore(&io->endio_lock, flags);
 	}
@@ -793,7 +885,8 @@ static void dec_pending(struct dm_io *io, blk_status_t error)
 			 */
 			spin_lock_irqsave(&md->deferred_lock, flags);
 			if (__noflush_suspending(md))
-				bio_list_add_head(&md->deferred, io->bio);
+				/* NOTE early return due to BLK_STS_DM_REQUEUE below */
+				bio_list_add_head(&md->deferred, io->orig_bio);
 			else
 				/* noflush suspend was interrupted. */
 				io->status = BLK_STS_IOERR;
@@ -801,7 +894,7 @@ static void dec_pending(struct dm_io *io, blk_status_t error)
 		}
 
 		io_error = io->status;
-		bio = io->bio;
+		bio = io->orig_bio;
 		end_io_acct(io);
 		free_io(md, io);
 
@@ -817,7 +910,8 @@ static void dec_pending(struct dm_io *io, blk_status_t error)
 			queue_io(md, bio);
 		} else {
 			/* done with normal IO or empty flush */
-			bio->bi_status = io_error;
+			if (io_error)
+				bio->bi_status = io_error;
 			bio_endio(bio);
 		}
 	}
@@ -847,7 +941,7 @@ static void clone_endio(struct bio *bio)
 	struct mapped_device *md = tio->io->md;
 	dm_endio_fn endio = tio->ti->type->end_io;
 
-	if (unlikely(error == BLK_STS_TARGET)) {
+	if (unlikely(error == BLK_STS_TARGET) && md->type != DM_TYPE_NVME_BIO_BASED) {
 		if (bio_op(bio) == REQ_OP_WRITE_SAME &&
 		    !bio->bi_disk->queue->limits.max_write_same_sectors)
 			disable_write_same(md);
@@ -920,7 +1014,15 @@ int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
 		return -EINVAL;
 	}
 
-	ti->max_io_len = (uint32_t) len;
+	/*
+	 * BIO based queue uses its own splitting. When multipage bvecs
+	 * is switched on, size of the incoming bio may be too big to
+	 * be handled in some targets, such as crypt.
+	 *
+	 * When these targets are ready for the big bio, we can remove
+	 * the limit.
+	 */
+	ti->max_io_len = min_t(uint32_t, len, BIO_MAX_PAGES * PAGE_SIZE);
 
 	return 0;
 }
@@ -997,7 +1099,7 @@ static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
 
 /*
  * A target may call dm_accept_partial_bio only from the map routine.  It is
- * allowed for all bio types except REQ_PREFLUSH.
+ * allowed for all bio types except REQ_PREFLUSH and REQ_OP_ZONE_RESET.
  *
  * dm_accept_partial_bio informs the dm that the target only wants to process
  * additional n_sectors sectors of the bio and the rest of the data should be
@@ -1047,7 +1149,7 @@ void dm_remap_zone_report(struct dm_target *ti, struct bio *bio, sector_t start)
 {
 #ifdef CONFIG_BLK_DEV_ZONED
 	struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
-	struct bio *report_bio = tio->io->bio;
+	struct bio *report_bio = tio->io->orig_bio;
 	struct blk_zone_report_hdr *hdr = NULL;
 	struct blk_zone *zone;
 	unsigned int nr_rep = 0;
@@ -1114,67 +1216,15 @@ void dm_remap_zone_report(struct dm_target *ti, struct bio *bio, sector_t start)
 }
 EXPORT_SYMBOL_GPL(dm_remap_zone_report);
 
-/*
- * Flush current->bio_list when the target map method blocks.
- * This fixes deadlocks in snapshot and possibly in other targets.
- */
-struct dm_offload {
-	struct blk_plug plug;
-	struct blk_plug_cb cb;
-};
-
-static void flush_current_bio_list(struct blk_plug_cb *cb, bool from_schedule)
-{
-	struct dm_offload *o = container_of(cb, struct dm_offload, cb);
-	struct bio_list list;
-	struct bio *bio;
-	int i;
-
-	INIT_LIST_HEAD(&o->cb.list);
-
-	if (unlikely(!current->bio_list))
-		return;
-
-	for (i = 0; i < 2; i++) {
-		list = current->bio_list[i];
-		bio_list_init(&current->bio_list[i]);
-
-		while ((bio = bio_list_pop(&list))) {
-			struct bio_set *bs = bio->bi_pool;
-			if (unlikely(!bs) || bs == fs_bio_set ||
-			    !bs->rescue_workqueue) {
-				bio_list_add(&current->bio_list[i], bio);
-				continue;
-			}
-
-			spin_lock(&bs->rescue_lock);
-			bio_list_add(&bs->rescue_list, bio);
-			queue_work(bs->rescue_workqueue, &bs->rescue_work);
-			spin_unlock(&bs->rescue_lock);
-		}
-	}
-}
-
-static void dm_offload_start(struct dm_offload *o)
-{
-	blk_start_plug(&o->plug);
-	o->cb.callback = flush_current_bio_list;
-	list_add(&o->cb.list, &current->plug->cb_list);
-}
-
-static void dm_offload_end(struct dm_offload *o)
-{
-	list_del(&o->cb.list);
-	blk_finish_plug(&o->plug);
-}
-
-static void __map_bio(struct dm_target_io *tio)
+static blk_qc_t __map_bio(struct dm_target_io *tio)
 {
 	int r;
 	sector_t sector;
-	struct dm_offload o;
 	struct bio *clone = &tio->clone;
+	struct dm_io *io = tio->io;
+	struct mapped_device *md = io->md;
 	struct dm_target *ti = tio->ti;
+	blk_qc_t ret = BLK_QC_T_NONE;
 
 	clone->bi_end_io = clone_endio;
 
@@ -1183,44 +1233,37 @@ static void __map_bio(struct dm_target_io *tio)
 	 * anything, the target has assumed ownership of
 	 * this io.
 	 */
-	atomic_inc(&tio->io->io_count);
+	atomic_inc(&io->io_count);
 	sector = clone->bi_iter.bi_sector;
 
-	dm_offload_start(&o);
 	r = ti->type->map(ti, clone);
-	dm_offload_end(&o);
-
 	switch (r) {
 	case DM_MAPIO_SUBMITTED:
 		break;
 	case DM_MAPIO_REMAPPED:
 		/* the bio has been remapped so dispatch it */
 		trace_block_bio_remap(clone->bi_disk->queue, clone,
-				      bio_dev(tio->io->bio), sector);
-		generic_make_request(clone);
+				      bio_dev(io->orig_bio), sector);
+		if (md->type == DM_TYPE_NVME_BIO_BASED)
+			ret = direct_make_request(clone);
+		else
+			ret = generic_make_request(clone);
 		break;
 	case DM_MAPIO_KILL:
-		dec_pending(tio->io, BLK_STS_IOERR);
 		free_tio(tio);
+		dec_pending(io, BLK_STS_IOERR);
 		break;
 	case DM_MAPIO_REQUEUE:
-		dec_pending(tio->io, BLK_STS_DM_REQUEUE);
 		free_tio(tio);
+		dec_pending(io, BLK_STS_DM_REQUEUE);
 		break;
 	default:
 		DMWARN("unimplemented target map return value: %d", r);
 		BUG();
 	}
-}
 
-struct clone_info {
-	struct mapped_device *md;
-	struct dm_table *map;
-	struct bio *bio;
-	struct dm_io *io;
-	sector_t sector;
-	unsigned sector_count;
-};
+	return ret;
+}
 
 static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
 {
@@ -1264,28 +1307,49 @@ static int clone_bio(struct dm_target_io *tio, struct bio *bio,
 	return 0;
 }
 
-static struct dm_target_io *alloc_tio(struct clone_info *ci,
-				      struct dm_target *ti,
-				      unsigned target_bio_nr)
+static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
+				struct dm_target *ti, unsigned num_bios)
 {
 	struct dm_target_io *tio;
-	struct bio *clone;
+	int try;
 
-	clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
-	tio = container_of(clone, struct dm_target_io, clone);
+	if (!num_bios)
+		return;
 
-	tio->io = ci->io;
-	tio->ti = ti;
-	tio->target_bio_nr = target_bio_nr;
+	if (num_bios == 1) {
+		tio = alloc_tio(ci, ti, 0, GFP_NOIO);
+		bio_list_add(blist, &tio->clone);
+		return;
+	}
 
-	return tio;
+	for (try = 0; try < 2; try++) {
+		int bio_nr;
+		struct bio *bio;
+
+		if (try)
+			mutex_lock(&ci->io->md->table_devices_lock);
+		for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
+			tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
+			if (!tio)
+				break;
+
+			bio_list_add(blist, &tio->clone);
+		}
+		if (try)
+			mutex_unlock(&ci->io->md->table_devices_lock);
+		if (bio_nr == num_bios)
+			return;
+
+		while ((bio = bio_list_pop(blist))) {
+			tio = container_of(bio, struct dm_target_io, clone);
+			free_tio(tio);
+		}
+	}
 }
 
-static void __clone_and_map_simple_bio(struct clone_info *ci,
-				       struct dm_target *ti,
-				       unsigned target_bio_nr, unsigned *len)
+static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
+					   struct dm_target_io *tio, unsigned *len)
 {
-	struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
 	struct bio *clone = &tio->clone;
 
 	tio->len_ptr = len;
@@ -1294,16 +1358,22 @@ static void __clone_and_map_simple_bio(struct clone_info *ci,
 	if (len)
 		bio_setup_sector(clone, ci->sector, *len);
 
-	__map_bio(tio);
+	return __map_bio(tio);
 }
 
 static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
 				  unsigned num_bios, unsigned *len)
 {
-	unsigned target_bio_nr;
+	struct bio_list blist = BIO_EMPTY_LIST;
+	struct bio *bio;
+	struct dm_target_io *tio;
 
-	for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
-		__clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
+	alloc_multiple_bios(&blist, ci, ti, num_bios);
+
+	while ((bio = bio_list_pop(&blist))) {
+		tio = container_of(bio, struct dm_target_io, clone);
+		(void) __clone_and_map_simple_bio(ci, tio, len);
+	}
 }
 
 static int __send_empty_flush(struct clone_info *ci)
@@ -1319,32 +1389,22 @@ static int __send_empty_flush(struct clone_info *ci)
 }
 
 static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
-				     sector_t sector, unsigned *len)
+				    sector_t sector, unsigned *len)
 {
 	struct bio *bio = ci->bio;
 	struct dm_target_io *tio;
-	unsigned target_bio_nr;
-	unsigned num_target_bios = 1;
-	int r = 0;
+	int r;
 
-	/*
-	 * Does the target want to receive duplicate copies of the bio?
-	 */
-	if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
-		num_target_bios = ti->num_write_bios(ti, bio);
-
-	for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
-		tio = alloc_tio(ci, ti, target_bio_nr);
-		tio->len_ptr = len;
-		r = clone_bio(tio, bio, sector, *len);
-		if (r < 0) {
-			free_tio(tio);
-			break;
-		}
-		__map_bio(tio);
+	tio = alloc_tio(ci, ti, 0, GFP_NOIO);
+	tio->len_ptr = len;
+	r = clone_bio(tio, bio, sector, *len);
+	if (r < 0) {
+		free_tio(tio);
+		return r;
 	}
+	(void) __map_bio(tio);
 
-	return r;
+	return 0;
 }
 
 typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
@@ -1371,56 +1431,50 @@ static bool is_split_required_for_discard(struct dm_target *ti)
 	return ti->split_discard_bios;
 }
 
-static int __send_changing_extent_only(struct clone_info *ci,
+static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
 				       get_num_bios_fn get_num_bios,
 				       is_split_required_fn is_split_required)
 {
-	struct dm_target *ti;
 	unsigned len;
 	unsigned num_bios;
 
-	do {
-		ti = dm_table_find_target(ci->map, ci->sector);
-		if (!dm_target_is_valid(ti))
-			return -EIO;
-
-		/*
-		 * Even though the device advertised support for this type of
-		 * request, that does not mean every target supports it, and
-		 * reconfiguration might also have changed that since the
-		 * check was performed.
-		 */
-		num_bios = get_num_bios ? get_num_bios(ti) : 0;
-		if (!num_bios)
-			return -EOPNOTSUPP;
+	/*
+	 * Even though the device advertised support for this type of
+	 * request, that does not mean every target supports it, and
+	 * reconfiguration might also have changed that since the
+	 * check was performed.
+	 */
+	num_bios = get_num_bios ? get_num_bios(ti) : 0;
+	if (!num_bios)
+		return -EOPNOTSUPP;
 
-		if (is_split_required && !is_split_required(ti))
-			len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
-		else
-			len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
+	if (is_split_required && !is_split_required(ti))
+		len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
+	else
+		len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
 
-		__send_duplicate_bios(ci, ti, num_bios, &len);
+	__send_duplicate_bios(ci, ti, num_bios, &len);
 
-		ci->sector += len;
-	} while (ci->sector_count -= len);
+	ci->sector += len;
+	ci->sector_count -= len;
 
 	return 0;
 }
 
-static int __send_discard(struct clone_info *ci)
+static int __send_discard(struct clone_info *ci, struct dm_target *ti)
 {
-	return __send_changing_extent_only(ci, get_num_discard_bios,
+	return __send_changing_extent_only(ci, ti, get_num_discard_bios,
 					   is_split_required_for_discard);
 }
 
-static int __send_write_same(struct clone_info *ci)
+static int __send_write_same(struct clone_info *ci, struct dm_target *ti)
 {
-	return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
+	return __send_changing_extent_only(ci, ti, get_num_write_same_bios, NULL);
 }
 
-static int __send_write_zeroes(struct clone_info *ci)
+static int __send_write_zeroes(struct clone_info *ci, struct dm_target *ti)
 {
-	return __send_changing_extent_only(ci, get_num_write_zeroes_bios, NULL);
+	return __send_changing_extent_only(ci, ti, get_num_write_zeroes_bios, NULL);
 }
 
 /*
@@ -1433,17 +1487,17 @@ static int __split_and_process_non_flush(struct clone_info *ci)
 	unsigned len;
 	int r;
 
-	if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
-		return __send_discard(ci);
-	else if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
-		return __send_write_same(ci);
-	else if (unlikely(bio_op(bio) == REQ_OP_WRITE_ZEROES))
-		return __send_write_zeroes(ci);
-
 	ti = dm_table_find_target(ci->map, ci->sector);
 	if (!dm_target_is_valid(ti))
 		return -EIO;
 
+	if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
+		return __send_discard(ci, ti);
+	else if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
+		return __send_write_same(ci, ti);
+	else if (unlikely(bio_op(bio) == REQ_OP_WRITE_ZEROES))
+		return __send_write_zeroes(ci, ti);
+
 	if (bio_op(bio) == REQ_OP_ZONE_REPORT)
 		len = ci->sector_count;
 	else
@@ -1460,34 +1514,33 @@ static int __split_and_process_non_flush(struct clone_info *ci)
 	return 0;
 }
 
+static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
+			    struct dm_table *map, struct bio *bio)
+{
+	ci->map = map;
+	ci->io = alloc_io(md, bio);
+	ci->sector = bio->bi_iter.bi_sector;
+}
+
 /*
  * Entry point to split a bio into clones and submit them to the targets.
  */
-static void __split_and_process_bio(struct mapped_device *md,
-				    struct dm_table *map, struct bio *bio)
+static blk_qc_t __split_and_process_bio(struct mapped_device *md,
+					struct dm_table *map, struct bio *bio)
 {
 	struct clone_info ci;
+	blk_qc_t ret = BLK_QC_T_NONE;
 	int error = 0;
 
 	if (unlikely(!map)) {
 		bio_io_error(bio);
-		return;
+		return ret;
 	}
 
-	ci.map = map;
-	ci.md = md;
-	ci.io = alloc_io(md);
-	ci.io->status = 0;
-	atomic_set(&ci.io->io_count, 1);
-	ci.io->bio = bio;
-	ci.io->md = md;
-	spin_lock_init(&ci.io->endio_lock);
-	ci.sector = bio->bi_iter.bi_sector;
-
-	start_io_acct(ci.io);
+	init_clone_info(&ci, md, map, bio);
 
 	if (bio->bi_opf & REQ_PREFLUSH) {
-		ci.bio = &ci.md->flush_bio;
+		ci.bio = &ci.io->md->flush_bio;
 		ci.sector_count = 0;
 		error = __send_empty_flush(&ci);
 		/* dec_pending submits any data associated with flush */
@@ -1498,32 +1551,95 @@ static void __split_and_process_bio(struct mapped_device *md,
 	} else {
 		ci.bio = bio;
 		ci.sector_count = bio_sectors(bio);
-		while (ci.sector_count && !error)
+		while (ci.sector_count && !error) {
 			error = __split_and_process_non_flush(&ci);
+			if (current->bio_list && ci.sector_count && !error) {
+				/*
+				 * Remainder must be passed to generic_make_request()
+				 * so that it gets handled *after* bios already submitted
+				 * have been completely processed.
+				 * We take a clone of the original to store in
+				 * ci.io->orig_bio to be used by end_io_acct() and
+				 * for dec_pending to use for completion handling.
+				 * As this path is not used for REQ_OP_ZONE_REPORT,
+				 * the usage of io->orig_bio in dm_remap_zone_report()
+				 * won't be affected by this reassignment.
+				 */
+				struct bio *b = bio_clone_bioset(bio, GFP_NOIO,
+								 md->queue->bio_split);
+				ci.io->orig_bio = b;
+				bio_advance(bio, (bio_sectors(bio) - ci.sector_count) << 9);
+				bio_chain(b, bio);
+				ret = generic_make_request(bio);
+				break;
+			}
+		}
 	}
 
 	/* drop the extra reference count */
 	dec_pending(ci.io, errno_to_blk_status(error));
+	return ret;
 }
-/*-----------------------------------------------------------------
- * CRUD END
- *---------------------------------------------------------------*/
 
 /*
- * The request function that just remaps the bio built up by
- * dm_merge_bvec.
+ * Optimized variant of __split_and_process_bio that leverages the
+ * fact that targets that use it do _not_ have a need to split bios.
  */
-static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
+static blk_qc_t __process_bio(struct mapped_device *md,
+			      struct dm_table *map, struct bio *bio)
+{
+	struct clone_info ci;
+	blk_qc_t ret = BLK_QC_T_NONE;
+	int error = 0;
+
+	if (unlikely(!map)) {
+		bio_io_error(bio);
+		return ret;
+	}
+
+	init_clone_info(&ci, md, map, bio);
+
+	if (bio->bi_opf & REQ_PREFLUSH) {
+		ci.bio = &ci.io->md->flush_bio;
+		ci.sector_count = 0;
+		error = __send_empty_flush(&ci);
+		/* dec_pending submits any data associated with flush */
+	} else {
+		struct dm_target *ti = md->immutable_target;
+		struct dm_target_io *tio;
+
+		/*
+		 * Defend against IO still getting in during teardown
+		 * - as was seen for a time with nvme-fcloop
+		 */
+		if (unlikely(WARN_ON_ONCE(!ti || !dm_target_is_valid(ti)))) {
+			error = -EIO;
+			goto out;
+		}
+
+		tio = alloc_tio(&ci, ti, 0, GFP_NOIO);
+		ci.bio = bio;
+		ci.sector_count = bio_sectors(bio);
+		ret = __clone_and_map_simple_bio(&ci, tio, NULL);
+	}
+out:
+	/* drop the extra reference count */
+	dec_pending(ci.io, errno_to_blk_status(error));
+	return ret;
+}
+
+typedef blk_qc_t (process_bio_fn)(struct mapped_device *, struct dm_table *, struct bio *);
+
+static blk_qc_t __dm_make_request(struct request_queue *q, struct bio *bio,
+				  process_bio_fn process_bio)
 {
-	int rw = bio_data_dir(bio);
 	struct mapped_device *md = q->queuedata;
+	blk_qc_t ret = BLK_QC_T_NONE;
 	int srcu_idx;
 	struct dm_table *map;
 
 	map = dm_get_live_table(md, &srcu_idx);
 
-	generic_start_io_acct(q, rw, bio_sectors(bio), &dm_disk(md)->part0);
-
 	/* if we're suspended, we have to queue this io for later */
 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
 		dm_put_live_table(md, srcu_idx);
@@ -1532,12 +1648,27 @@ static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
 			queue_io(md, bio);
 		else
 			bio_io_error(bio);
-		return BLK_QC_T_NONE;
+		return ret;
 	}
 
-	__split_and_process_bio(md, map, bio);
+	ret = process_bio(md, map, bio);
+
 	dm_put_live_table(md, srcu_idx);
-	return BLK_QC_T_NONE;
+	return ret;
+}
+
+/*
+ * The request function that remaps the bio to one target and
+ * splits off any remainder.
+ */
+static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
+{
+	return __dm_make_request(q, bio, __split_and_process_bio);
+}
+
+static blk_qc_t dm_make_request_nvme(struct request_queue *q, struct bio *bio)
+{
+	return __dm_make_request(q, bio, __process_bio);
 }
 
 static int dm_any_congested(void *congested_data, int bdi_bits)
@@ -1618,20 +1749,9 @@ static const struct dax_operations dm_dax_ops;
 
 static void dm_wq_work(struct work_struct *work);
 
-void dm_init_md_queue(struct mapped_device *md)
-{
-	/*
-	 * Initialize data that will only be used by a non-blk-mq DM queue
-	 * - must do so here (in alloc_dev callchain) before queue is used
-	 */
-	md->queue->queuedata = md;
-	md->queue->backing_dev_info->congested_data = md;
-}
-
-void dm_init_normal_md_queue(struct mapped_device *md)
+static void dm_init_normal_md_queue(struct mapped_device *md)
 {
 	md->use_blk_mq = false;
-	dm_init_md_queue(md);
 
 	/*
 	 * Initialize aspects of queue that aren't relevant for blk-mq
@@ -1645,9 +1765,10 @@ static void cleanup_mapped_device(struct mapped_device *md)
 		destroy_workqueue(md->wq);
 	if (md->kworker_task)
 		kthread_stop(md->kworker_task);
-	mempool_destroy(md->io_pool);
 	if (md->bs)
 		bioset_free(md->bs);
+	if (md->io_bs)
+		bioset_free(md->io_bs);
 
 	if (md->dax_dev) {
 		kill_dax(md->dax_dev);
@@ -1673,6 +1794,10 @@ static void cleanup_mapped_device(struct mapped_device *md)
 		md->bdev = NULL;
 	}
 
+	mutex_destroy(&md->suspend_lock);
+	mutex_destroy(&md->type_lock);
+	mutex_destroy(&md->table_devices_lock);
+
 	dm_mq_cleanup_mapped_device(md);
 }
 
@@ -1726,10 +1851,10 @@ static struct mapped_device *alloc_dev(int minor)
 	md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id);
 	if (!md->queue)
 		goto bad;
+	md->queue->queuedata = md;
+	md->queue->backing_dev_info->congested_data = md;
 
-	dm_init_md_queue(md);
-
-	md->disk = alloc_disk_node(1, numa_node_id);
+	md->disk = alloc_disk_node(1, md->numa_node_id);
 	if (!md->disk)
 		goto bad;
 
@@ -1753,7 +1878,7 @@ static struct mapped_device *alloc_dev(int minor)
 		goto bad;
 	md->dax_dev = dax_dev;
 
-	add_disk(md->disk);
+	add_disk_no_queue_reg(md->disk);
 	format_dev_t(md->name, MKDEV(_major, minor));
 
 	md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
@@ -1812,17 +1937,22 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
 {
 	struct dm_md_mempools *p = dm_table_get_md_mempools(t);
 
-	if (md->bs) {
-		/* The md already has necessary mempools. */
-		if (dm_table_bio_based(t)) {
-			/*
-			 * Reload bioset because front_pad may have changed
-			 * because a different table was loaded.
-			 */
+	if (dm_table_bio_based(t)) {
+		/*
+		 * The md may already have mempools that need changing.
+		 * If so, reload bioset because front_pad may have changed
+		 * because a different table was loaded.
+		 */
+		if (md->bs) {
 			bioset_free(md->bs);
-			md->bs = p->bs;
-			p->bs = NULL;
+			md->bs = NULL;
 		}
+		if (md->io_bs) {
+			bioset_free(md->io_bs);
+			md->io_bs = NULL;
+		}
+
+	} else if (md->bs) {
 		/*
 		 * There's no need to reload with request-based dm
 		 * because the size of front_pad doesn't change.
@@ -1834,13 +1964,12 @@ static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
 		goto out;
 	}
 
-	BUG_ON(!p || md->io_pool || md->bs);
+	BUG_ON(!p || md->bs || md->io_bs);
 
-	md->io_pool = p->io_pool;
-	p->io_pool = NULL;
 	md->bs = p->bs;
 	p->bs = NULL;
-
+	md->io_bs = p->io_bs;
+	p->io_bs = NULL;
 out:
 	/* mempool bind completed, no longer need any mempools in the table */
 	dm_table_free_md_mempools(t);
@@ -1886,6 +2015,7 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
 {
 	struct dm_table *old_map;
 	struct request_queue *q = md->queue;
+	bool request_based = dm_table_request_based(t);
 	sector_t size;
 
 	lockdep_assert_held(&md->suspend_lock);
@@ -1909,12 +2039,15 @@ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
 	 * This must be done before setting the queue restrictions,
 	 * because request-based dm may be run just after the setting.
 	 */
-	if (dm_table_request_based(t)) {
+	if (request_based)
 		dm_stop_queue(q);
+
+	if (request_based || md->type == DM_TYPE_NVME_BIO_BASED) {
 		/*
-		 * Leverage the fact that request-based DM targets are
-		 * immutable singletons and establish md->immutable_target
-		 * - used to optimize both dm_request_fn and dm_mq_queue_rq
+		 * Leverage the fact that request-based DM targets and
+		 * NVMe bio based targets are immutable singletons
+		 * - used to optimize both dm_request_fn and dm_mq_queue_rq;
+		 *   and __process_bio.
 		 */
 		md->immutable_target = dm_table_get_immutable_target(t);
 	}
@@ -1954,13 +2087,18 @@ static struct dm_table *__unbind(struct mapped_device *md)
  */
 int dm_create(int minor, struct mapped_device **result)
 {
+	int r;
 	struct mapped_device *md;
 
 	md = alloc_dev(minor);
 	if (!md)
 		return -ENXIO;
 
-	dm_sysfs_init(md);
+	r = dm_sysfs_init(md);
+	if (r) {
+		free_dev(md);
+		return r;
+	}
 
 	*result = md;
 	return 0;
@@ -2013,10 +2151,12 @@ EXPORT_SYMBOL_GPL(dm_get_queue_limits);
 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
 {
 	int r;
+	struct queue_limits limits;
 	enum dm_queue_mode type = dm_get_md_type(md);
 
 	switch (type) {
 	case DM_TYPE_REQUEST_BASED:
+		dm_init_normal_md_queue(md);
 		r = dm_old_init_request_queue(md, t);
 		if (r) {
 			DMERR("Cannot initialize queue for request-based mapped device");
@@ -2034,21 +2174,24 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
 	case DM_TYPE_DAX_BIO_BASED:
 		dm_init_normal_md_queue(md);
 		blk_queue_make_request(md->queue, dm_make_request);
-		/*
-		 * DM handles splitting bios as needed.  Free the bio_split bioset
-		 * since it won't be used (saves 1 process per bio-based DM device).
-		 */
-		bioset_free(md->queue->bio_split);
-		md->queue->bio_split = NULL;
-
-		if (type == DM_TYPE_DAX_BIO_BASED)
-			queue_flag_set_unlocked(QUEUE_FLAG_DAX, md->queue);
+		break;
+	case DM_TYPE_NVME_BIO_BASED:
+		dm_init_normal_md_queue(md);
+		blk_queue_make_request(md->queue, dm_make_request_nvme);
 		break;
 	case DM_TYPE_NONE:
 		WARN_ON_ONCE(true);
 		break;
 	}
 
+	r = dm_calculate_queue_limits(t, &limits);
+	if (r) {
+		DMERR("Cannot calculate initial queue limits");
+		return r;
+	}
+	dm_table_set_restrictions(t, md->queue, &limits);
+	blk_register_queue(md->disk);
+
 	return 0;
 }
 
@@ -2113,7 +2256,6 @@ EXPORT_SYMBOL_GPL(dm_device_name);
 
 static void __dm_destroy(struct mapped_device *md, bool wait)
 {
-	struct request_queue *q = dm_get_md_queue(md);
 	struct dm_table *map;
 	int srcu_idx;
 
@@ -2124,7 +2266,7 @@ static void __dm_destroy(struct mapped_device *md, bool wait)
 	set_bit(DMF_FREEING, &md->flags);
 	spin_unlock(&_minor_lock);
 
-	blk_set_queue_dying(q);
+	blk_set_queue_dying(md->queue);
 
 	if (dm_request_based(md) && md->kworker_task)
 		kthread_flush_worker(&md->kworker);
@@ -2735,11 +2877,12 @@ int dm_noflush_suspending(struct dm_target *ti)
 EXPORT_SYMBOL_GPL(dm_noflush_suspending);
 
 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
-					    unsigned integrity, unsigned per_io_data_size)
+					    unsigned integrity, unsigned per_io_data_size,
+					    unsigned min_pool_size)
 {
 	struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
 	unsigned int pool_size = 0;
-	unsigned int front_pad;
+	unsigned int front_pad, io_front_pad;
 
 	if (!pools)
 		return NULL;
@@ -2747,16 +2890,19 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu
 	switch (type) {
 	case DM_TYPE_BIO_BASED:
 	case DM_TYPE_DAX_BIO_BASED:
-		pool_size = dm_get_reserved_bio_based_ios();
+	case DM_TYPE_NVME_BIO_BASED:
+		pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
 		front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
-	
-		pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
-		if (!pools->io_pool)
+		io_front_pad = roundup(front_pad,  __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
+		pools->io_bs = bioset_create(pool_size, io_front_pad, 0);
+		if (!pools->io_bs)
+			goto out;
+		if (integrity && bioset_integrity_create(pools->io_bs, pool_size))
 			goto out;
 		break;
 	case DM_TYPE_REQUEST_BASED:
 	case DM_TYPE_MQ_REQUEST_BASED:
-		pool_size = dm_get_reserved_rq_based_ios();
+		pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
 		front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
 		/* per_io_data_size is used for blk-mq pdu at queue allocation */
 		break;
@@ -2764,7 +2910,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu
 		BUG();
 	}
 
-	pools->bs = bioset_create(pool_size, front_pad, BIOSET_NEED_RESCUER);
+	pools->bs = bioset_create(pool_size, front_pad, 0);
 	if (!pools->bs)
 		goto out;
 
@@ -2784,10 +2930,10 @@ void dm_free_md_mempools(struct dm_md_mempools *pools)
 	if (!pools)
 		return;
 
-	mempool_destroy(pools->io_pool);
-
 	if (pools->bs)
 		bioset_free(pools->bs);
+	if (pools->io_bs)
+		bioset_free(pools->io_bs);
 
 	kfree(pools);
 }
@@ -2872,7 +3018,7 @@ static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
 	fmode_t mode;
 	int r;
 
-	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
+	r = dm_get_bdev_for_ioctl(md, &bdev, &mode);
 	if (r < 0)
 		return r;
 
@@ -2882,7 +3028,7 @@ static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
 	else
 		r = -EOPNOTSUPP;
 
-	bdput(bdev);
+	blkdev_put(bdev, mode);
 	return r;
 }
 
@@ -2893,7 +3039,7 @@ static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
 	fmode_t mode;
 	int r;
 
-	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
+	r = dm_get_bdev_for_ioctl(md, &bdev, &mode);
 	if (r < 0)
 		return r;
 
@@ -2903,7 +3049,7 @@ static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
 	else
 		r = -EOPNOTSUPP;
 
-	bdput(bdev);
+	blkdev_put(bdev, mode);
 	return r;
 }
 
@@ -2915,7 +3061,7 @@ static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
 	fmode_t mode;
 	int r;
 
-	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
+	r = dm_get_bdev_for_ioctl(md, &bdev, &mode);
 	if (r < 0)
 		return r;
 
@@ -2925,7 +3071,7 @@ static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
 	else
 		r = -EOPNOTSUPP;
 
-	bdput(bdev);
+	blkdev_put(bdev, mode);
 	return r;
 }
 
@@ -2936,7 +3082,7 @@ static int dm_pr_clear(struct block_device *bdev, u64 key)
 	fmode_t mode;
 	int r;
 
-	r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
+	r = dm_get_bdev_for_ioctl(md, &bdev, &mode);
 	if (r < 0)
 		return r;
 
@@ -2946,7 +3092,7 @@ static int dm_pr_clear(struct block_device *bdev, u64 key)
 	else
 		r = -EOPNOTSUPP;
 
-	bdput(bdev);
+	blkdev_put(bdev, mode);
 	return r;
 }
 
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
index 36399bb875dd..114a81b27c37 100644
--- a/drivers/md/dm.h
+++ b/drivers/md/dm.h
@@ -49,7 +49,6 @@ struct dm_md_mempools;
 /*-----------------------------------------------------------------
  * Internal table functions.
  *---------------------------------------------------------------*/
-void dm_table_destroy(struct dm_table *t);
 void dm_table_event_callback(struct dm_table *t,
 			     void (*fn)(void *), void *context);
 struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index);
@@ -206,7 +205,8 @@ void dm_kcopyd_exit(void);
  * Mempool operations
  */
 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
-					    unsigned integrity, unsigned per_bio_data_size);
+					    unsigned integrity, unsigned per_bio_data_size,
+					    unsigned min_pool_size);
 void dm_free_md_mempools(struct dm_md_mempools *pools);
 
 /*
diff --git a/drivers/md/md-multipath.c b/drivers/md/md-multipath.c
index e40065bdbfc8..0a7e99d62c69 100644
--- a/drivers/md/md-multipath.c
+++ b/drivers/md/md-multipath.c
@@ -157,7 +157,7 @@ static void multipath_status(struct seq_file *seq, struct mddev *mddev)
 		seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
 	}
 	rcu_read_unlock();
-	seq_printf (seq, "]");
+	seq_putc(seq, ']');
 }
 
 static int multipath_congested(struct mddev *mddev, int bits)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 4e4dee0ec2de..254e44e44668 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -711,7 +711,7 @@ static struct md_rdev *find_rdev(struct mddev *mddev, dev_t dev)
 	return NULL;
 }
 
-static struct md_rdev *find_rdev_rcu(struct mddev *mddev, dev_t dev)
+struct md_rdev *md_find_rdev_rcu(struct mddev *mddev, dev_t dev)
 {
 	struct md_rdev *rdev;
 
@@ -721,6 +721,7 @@ static struct md_rdev *find_rdev_rcu(struct mddev *mddev, dev_t dev)
 
 	return NULL;
 }
+EXPORT_SYMBOL_GPL(md_find_rdev_rcu);
 
 static struct md_personality *find_pers(int level, char *clevel)
 {
@@ -800,6 +801,9 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev,
 	struct bio *bio;
 	int ff = 0;
 
+	if (!page)
+		return;
+
 	if (test_bit(Faulty, &rdev->flags))
 		return;
 
@@ -5451,6 +5455,7 @@ int md_run(struct mddev *mddev)
 	 * the only valid external interface is through the md
 	 * device.
 	 */
+	mddev->has_superblocks = false;
 	rdev_for_each(rdev, mddev) {
 		if (test_bit(Faulty, &rdev->flags))
 			continue;
@@ -5464,6 +5469,9 @@ int md_run(struct mddev *mddev)
 				set_disk_ro(mddev->gendisk, 1);
 		}
 
+		if (rdev->sb_page)
+			mddev->has_superblocks = true;
+
 		/* perform some consistency tests on the device.
 		 * We don't want the data to overlap the metadata,
 		 * Internal Bitmap issues have been handled elsewhere.
@@ -5496,8 +5504,10 @@ int md_run(struct mddev *mddev)
 	}
 	if (mddev->sync_set == NULL) {
 		mddev->sync_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
-		if (!mddev->sync_set)
-			return -ENOMEM;
+		if (!mddev->sync_set) {
+			err = -ENOMEM;
+			goto abort;
+		}
 	}
 
 	spin_lock(&pers_lock);
@@ -5510,7 +5520,8 @@ int md_run(struct mddev *mddev)
 		else
 			pr_warn("md: personality for level %s is not loaded!\n",
 				mddev->clevel);
-		return -EINVAL;
+		err = -EINVAL;
+		goto abort;
 	}
 	spin_unlock(&pers_lock);
 	if (mddev->level != pers->level) {
@@ -5523,7 +5534,8 @@ int md_run(struct mddev *mddev)
 	    pers->start_reshape == NULL) {
 		/* This personality cannot handle reshaping... */
 		module_put(pers->owner);
-		return -EINVAL;
+		err = -EINVAL;
+		goto abort;
 	}
 
 	if (pers->sync_request) {
@@ -5560,11 +5572,6 @@ int md_run(struct mddev *mddev)
 	if (start_readonly && mddev->ro == 0)
 		mddev->ro = 2; /* read-only, but switch on first write */
 
-	/*
-	 * NOTE: some pers->run(), for example r5l_recovery_log(), wakes
-	 * up mddev->thread. It is important to initialize critical
-	 * resources for mddev->thread BEFORE calling pers->run().
-	 */
 	err = pers->run(mddev);
 	if (err)
 		pr_warn("md: pers->run() failed ...\n");
@@ -5597,7 +5604,7 @@ int md_run(struct mddev *mddev)
 		mddev->private = NULL;
 		module_put(pers->owner);
 		bitmap_destroy(mddev);
-		return err;
+		goto abort;
 	}
 	if (mddev->queue) {
 		bool nonrot = true;
@@ -5659,6 +5666,18 @@ int md_run(struct mddev *mddev)
 	sysfs_notify_dirent_safe(mddev->sysfs_action);
 	sysfs_notify(&mddev->kobj, NULL, "degraded");
 	return 0;
+
+abort:
+	if (mddev->bio_set) {
+		bioset_free(mddev->bio_set);
+		mddev->bio_set = NULL;
+	}
+	if (mddev->sync_set) {
+		bioset_free(mddev->sync_set);
+		mddev->sync_set = NULL;
+	}
+
+	return err;
 }
 EXPORT_SYMBOL_GPL(md_run);
 
@@ -5678,6 +5697,9 @@ static int do_md_run(struct mddev *mddev)
 	if (mddev_is_clustered(mddev))
 		md_allow_write(mddev);
 
+	/* run start up tasks that require md_thread */
+	md_start(mddev);
+
 	md_wakeup_thread(mddev->thread);
 	md_wakeup_thread(mddev->sync_thread); /* possibly kick off a reshape */
 
@@ -5689,6 +5711,21 @@ static int do_md_run(struct mddev *mddev)
 	return err;
 }
 
+int md_start(struct mddev *mddev)
+{
+	int ret = 0;
+
+	if (mddev->pers->start) {
+		set_bit(MD_RECOVERY_WAIT, &mddev->recovery);
+		md_wakeup_thread(mddev->thread);
+		ret = mddev->pers->start(mddev);
+		clear_bit(MD_RECOVERY_WAIT, &mddev->recovery);
+		md_wakeup_thread(mddev->sync_thread);
+	}
+	return ret;
+}
+EXPORT_SYMBOL_GPL(md_start);
+
 static int restart_array(struct mddev *mddev)
 {
 	struct gendisk *disk = mddev->gendisk;
@@ -6997,7 +7034,7 @@ static int set_disk_faulty(struct mddev *mddev, dev_t dev)
 		return -ENODEV;
 
 	rcu_read_lock();
-	rdev = find_rdev_rcu(mddev, dev);
+	rdev = md_find_rdev_rcu(mddev, dev);
 	if (!rdev)
 		err =  -ENODEV;
 	else {
@@ -7871,20 +7908,20 @@ static int md_seq_open(struct inode *inode, struct file *file)
 }
 
 static int md_unloading;
-static unsigned int mdstat_poll(struct file *filp, poll_table *wait)
+static __poll_t mdstat_poll(struct file *filp, poll_table *wait)
 {
 	struct seq_file *seq = filp->private_data;
-	int mask;
+	__poll_t mask;
 
 	if (md_unloading)
-		return POLLIN|POLLRDNORM|POLLERR|POLLPRI;
+		return EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
 	poll_wait(filp, &md_event_waiters, wait);
 
 	/* always allow read */
-	mask = POLLIN | POLLRDNORM;
+	mask = EPOLLIN | EPOLLRDNORM;
 
 	if (seq->poll_event != atomic_read(&md_event_count))
-		mask |= POLLERR | POLLPRI;
+		mask |= EPOLLERR | EPOLLPRI;
 	return mask;
 }
 
@@ -8035,6 +8072,7 @@ EXPORT_SYMBOL(md_done_sync);
 bool md_write_start(struct mddev *mddev, struct bio *bi)
 {
 	int did_change = 0;
+
 	if (bio_data_dir(bi) != WRITE)
 		return true;
 
@@ -8067,6 +8105,8 @@ bool md_write_start(struct mddev *mddev, struct bio *bi)
 	rcu_read_unlock();
 	if (did_change)
 		sysfs_notify_dirent_safe(mddev->sysfs_state);
+	if (!mddev->has_superblocks)
+		return true;
 	wait_event(mddev->sb_wait,
 		   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags) ||
 		   mddev->suspended);
@@ -8169,7 +8209,8 @@ void md_do_sync(struct md_thread *thread)
 	int ret;
 
 	/* just incase thread restarts... */
-	if (test_bit(MD_RECOVERY_DONE, &mddev->recovery))
+	if (test_bit(MD_RECOVERY_DONE, &mddev->recovery) ||
+	    test_bit(MD_RECOVERY_WAIT, &mddev->recovery))
 		return;
 	if (mddev->ro) {/* never try to sync a read-only array */
 		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
@@ -8528,6 +8569,19 @@ void md_do_sync(struct md_thread *thread)
 	set_mask_bits(&mddev->sb_flags, 0,
 		      BIT(MD_SB_CHANGE_PENDING) | BIT(MD_SB_CHANGE_DEVS));
 
+	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
+			!test_bit(MD_RECOVERY_INTR, &mddev->recovery) &&
+			mddev->delta_disks > 0 &&
+			mddev->pers->finish_reshape &&
+			mddev->pers->size &&
+			mddev->queue) {
+		mddev_lock_nointr(mddev);
+		md_set_array_sectors(mddev, mddev->pers->size(mddev, 0, 0));
+		mddev_unlock(mddev);
+		set_capacity(mddev->gendisk, mddev->array_sectors);
+		revalidate_disk(mddev->gendisk);
+	}
+
 	spin_lock(&mddev->lock);
 	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
 		/* We completed so min/max setting can be forgotten if used. */
@@ -8554,6 +8608,10 @@ static int remove_and_add_spares(struct mddev *mddev,
 	int removed = 0;
 	bool remove_some = false;
 
+	if (this && test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
+		/* Mustn't remove devices when resync thread is running */
+		return 0;
+
 	rdev_for_each(rdev, mddev) {
 		if ((this == NULL || rdev == this) &&
 		    rdev->raid_disk >= 0 &&
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 7d6bcf0eba0c..fbc925cce810 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -468,6 +468,8 @@ struct mddev {
 	void (*sync_super)(struct mddev *mddev, struct md_rdev *rdev);
 	struct md_cluster_info		*cluster_info;
 	unsigned int			good_device_nr;	/* good device num within cluster raid */
+
+	bool	has_superblocks:1;
 };
 
 enum recovery_flags {
@@ -485,6 +487,7 @@ enum recovery_flags {
 	MD_RECOVERY_RESHAPE,	/* A reshape is happening */
 	MD_RECOVERY_FROZEN,	/* User request to abort, and not restart, any action */
 	MD_RECOVERY_ERROR,	/* sync-action interrupted because io-error */
+	MD_RECOVERY_WAIT,	/* waiting for pers->start() to finish */
 };
 
 static inline int __must_check mddev_lock(struct mddev *mddev)
@@ -523,7 +526,13 @@ struct md_personality
 	struct list_head list;
 	struct module *owner;
 	bool (*make_request)(struct mddev *mddev, struct bio *bio);
+	/*
+	 * start up works that do NOT require md_thread. tasks that
+	 * requires md_thread should go into start()
+	 */
 	int (*run)(struct mddev *mddev);
+	/* start up works that require md threads */
+	int (*start)(struct mddev *mddev);
 	void (*free)(struct mddev *mddev, void *priv);
 	void (*status)(struct seq_file *seq, struct mddev *mddev);
 	/* error_handler must set ->faulty and clear ->in_sync
@@ -687,6 +696,7 @@ extern int strict_strtoul_scaled(const char *cp, unsigned long *res, int scale);
 
 extern void mddev_init(struct mddev *mddev);
 extern int md_run(struct mddev *mddev);
+extern int md_start(struct mddev *mddev);
 extern void md_stop(struct mddev *mddev);
 extern void md_stop_writes(struct mddev *mddev);
 extern int md_rdev_init(struct md_rdev *rdev);
@@ -702,6 +712,7 @@ extern void md_reload_sb(struct mddev *mddev, int raid_disk);
 extern void md_update_sb(struct mddev *mddev, int force);
 extern void md_kick_rdev_from_array(struct md_rdev * rdev);
 struct md_rdev *md_find_rdev_nr_rcu(struct mddev *mddev, int nr);
+struct md_rdev *md_find_rdev_rcu(struct mddev *mddev, dev_t dev);
 
 static inline void rdev_dec_pending(struct md_rdev *rdev, struct mddev *mddev)
 {
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 6df398e3a008..fe872dc6712e 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -815,6 +815,17 @@ static void flush_pending_writes(struct r1conf *conf)
 		bio = bio_list_get(&conf->pending_bio_list);
 		conf->pending_count = 0;
 		spin_unlock_irq(&conf->device_lock);
+
+		/*
+		 * As this is called in a wait_event() loop (see freeze_array),
+		 * current->state might be TASK_UNINTERRUPTIBLE which will
+		 * cause a warning when we prepare to wait again.  As it is
+		 * rare that this path is taken, it is perfectly safe to force
+		 * us to go around the wait_event() loop again, so the warning
+		 * is a false-positive.  Silence the warning by resetting
+		 * thread state
+		 */
+		__set_current_state(TASK_RUNNING);
 		blk_start_plug(&plug);
 		flush_bio_list(conf, bio);
 		blk_finish_plug(&plug);
@@ -1097,7 +1108,7 @@ static void alloc_behind_master_bio(struct r1bio *r1_bio,
 
 	bio_copy_data(behind_bio, bio);
 skip_copy:
-	r1_bio->behind_master_bio = behind_bio;;
+	r1_bio->behind_master_bio = behind_bio;
 	set_bit(R1BIO_BehindIO, &r1_bio->state);
 
 	return;
@@ -1798,6 +1809,17 @@ static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
 			struct md_rdev *repl =
 				conf->mirrors[conf->raid_disks + number].rdev;
 			freeze_array(conf, 0);
+			if (atomic_read(&repl->nr_pending)) {
+				/* It means that some queued IO of retry_list
+				 * hold repl. Thus, we cannot set replacement
+				 * as NULL, avoiding rdev NULL pointer
+				 * dereference in sync_request_write and
+				 * handle_write_finished.
+				 */
+				err = -EBUSY;
+				unfreeze_array(conf);
+				goto abort;
+			}
 			clear_bit(Replacement, &repl->flags);
 			p->rdev = repl;
 			conf->mirrors[conf->raid_disks + number].rdev = NULL;
diff --git a/drivers/md/raid1.h b/drivers/md/raid1.h
index c7294e7557e0..eb84bc68e2fd 100644
--- a/drivers/md/raid1.h
+++ b/drivers/md/raid1.h
@@ -26,6 +26,18 @@
 #define BARRIER_BUCKETS_NR_BITS		(PAGE_SHIFT - ilog2(sizeof(atomic_t)))
 #define BARRIER_BUCKETS_NR		(1<<BARRIER_BUCKETS_NR_BITS)
 
+/* Note: raid1_info.rdev can be set to NULL asynchronously by raid1_remove_disk.
+ * There are three safe ways to access raid1_info.rdev.
+ * 1/ when holding mddev->reconfig_mutex
+ * 2/ when resync/recovery is known to be happening - i.e. in code that is
+ *    called as part of performing resync/recovery.
+ * 3/ while holding rcu_read_lock(), use rcu_dereference to get the pointer
+ *    and if it is non-NULL, increment rdev->nr_pending before dropping the
+ *    RCU lock.
+ * When .rdev is set to NULL, the nr_pending count checked again and if it has
+ * been incremented, the pointer is put back in .rdev.
+ */
+
 struct raid1_info {
 	struct md_rdev	*rdev;
 	sector_t	head_position;
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index c131835cf008..c5e6c60fc0d4 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -141,7 +141,7 @@ static void r10bio_pool_free(void *r10_bio, void *data)
 #define RESYNC_WINDOW (1024*1024)
 /* maximum number of concurrent requests, memory permitting */
 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
-#define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW)
+#define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW)
 #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)
 
 /*
@@ -900,6 +900,18 @@ static void flush_pending_writes(struct r10conf *conf)
 		bio = bio_list_get(&conf->pending_bio_list);
 		conf->pending_count = 0;
 		spin_unlock_irq(&conf->device_lock);
+
+		/*
+		 * As this is called in a wait_event() loop (see freeze_array),
+		 * current->state might be TASK_UNINTERRUPTIBLE which will
+		 * cause a warning when we prepare to wait again.  As it is
+		 * rare that this path is taken, it is perfectly safe to force
+		 * us to go around the wait_event() loop again, so the warning
+		 * is a false-positive. Silence the warning by resetting
+		 * thread state
+		 */
+		__set_current_state(TASK_RUNNING);
+
 		blk_start_plug(&plug);
 		/* flush any pending bitmap writes to disk
 		 * before proceeding w/ I/O */
@@ -2643,7 +2655,8 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
 		for (m = 0; m < conf->copies; m++) {
 			int dev = r10_bio->devs[m].devnum;
 			rdev = conf->mirrors[dev].rdev;
-			if (r10_bio->devs[m].bio == NULL)
+			if (r10_bio->devs[m].bio == NULL ||
+				r10_bio->devs[m].bio->bi_end_io == NULL)
 				continue;
 			if (!r10_bio->devs[m].bio->bi_status) {
 				rdev_clear_badblocks(
@@ -2658,7 +2671,8 @@ static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
 					md_error(conf->mddev, rdev);
 			}
 			rdev = conf->mirrors[dev].replacement;
-			if (r10_bio->devs[m].repl_bio == NULL)
+			if (r10_bio->devs[m].repl_bio == NULL ||
+				r10_bio->devs[m].repl_bio->bi_end_io == NULL)
 				continue;
 
 			if (!r10_bio->devs[m].repl_bio->bi_status) {
@@ -3770,7 +3784,7 @@ static int raid10_run(struct mddev *mddev)
 		if (fc > 1 || fo > 0) {
 			pr_err("only near layout is supported by clustered"
 				" raid10\n");
-			goto out;
+			goto out_free_conf;
 		}
 	}
 
@@ -4818,17 +4832,11 @@ static void raid10_finish_reshape(struct mddev *mddev)
 		return;
 
 	if (mddev->delta_disks > 0) {
-		sector_t size = raid10_size(mddev, 0, 0);
-		md_set_array_sectors(mddev, size);
 		if (mddev->recovery_cp > mddev->resync_max_sectors) {
 			mddev->recovery_cp = mddev->resync_max_sectors;
 			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
 		}
-		mddev->resync_max_sectors = size;
-		if (mddev->queue) {
-			set_capacity(mddev->gendisk, mddev->array_sectors);
-			revalidate_disk(mddev->gendisk);
-		}
+		mddev->resync_max_sectors = mddev->array_sectors;
 	} else {
 		int d;
 		rcu_read_lock();
diff --git a/drivers/md/raid10.h b/drivers/md/raid10.h
index db2ac22ac1b4..e2e8840de9bf 100644
--- a/drivers/md/raid10.h
+++ b/drivers/md/raid10.h
@@ -2,6 +2,19 @@
 #ifndef _RAID10_H
 #define _RAID10_H
 
+/* Note: raid10_info.rdev can be set to NULL asynchronously by
+ * raid10_remove_disk.
+ * There are three safe ways to access raid10_info.rdev.
+ * 1/ when holding mddev->reconfig_mutex
+ * 2/ when resync/recovery/reshape is known to be happening - i.e. in code
+ *    that is called as part of performing resync/recovery/reshape.
+ * 3/ while holding rcu_read_lock(), use rcu_dereference to get the pointer
+ *    and if it is non-NULL, increment rdev->nr_pending before dropping the
+ *    RCU lock.
+ * When .rdev is set to NULL, the nr_pending count checked again and if it has
+ * been incremented, the pointer is put back in .rdev.
+ */
+
 struct raid10_info {
 	struct md_rdev	*rdev, *replacement;
 	sector_t	head_position;
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index 39f31f07ffe9..3c65f52b68f5 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -1111,9 +1111,6 @@ void r5l_write_stripe_run(struct r5l_log *log)
 
 int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio)
 {
-	if (!log)
-		return -ENODEV;
-
 	if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
 		/*
 		 * in write through (journal only)
@@ -1592,8 +1589,6 @@ void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
 void r5l_quiesce(struct r5l_log *log, int quiesce)
 {
 	struct mddev *mddev;
-	if (!log)
-		return;
 
 	if (quiesce) {
 		/* make sure r5l_write_super_and_discard_space exits */
@@ -2448,7 +2443,6 @@ static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log,
 		raid5_release_stripe(sh);
 	}
 
-	md_wakeup_thread(conf->mddev->thread);
 	/* reuse conf->wait_for_quiescent in recovery */
 	wait_event(conf->wait_for_quiescent,
 		   atomic_read(&conf->active_stripes) == 0);
@@ -2491,10 +2485,10 @@ static int r5l_recovery_log(struct r5l_log *log)
 	ctx->seq += 10000;
 
 	if ((ctx->data_only_stripes == 0) && (ctx->data_parity_stripes == 0))
-		pr_debug("md/raid:%s: starting from clean shutdown\n",
+		pr_info("md/raid:%s: starting from clean shutdown\n",
 			 mdname(mddev));
 	else
-		pr_debug("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
+		pr_info("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
 			 mdname(mddev), ctx->data_only_stripes,
 			 ctx->data_parity_stripes);
 
@@ -3036,6 +3030,23 @@ static int r5l_load_log(struct r5l_log *log)
 	return ret;
 }
 
+int r5l_start(struct r5l_log *log)
+{
+	int ret;
+
+	if (!log)
+		return 0;
+
+	ret = r5l_load_log(log);
+	if (ret) {
+		struct mddev *mddev = log->rdev->mddev;
+		struct r5conf *conf = mddev->private;
+
+		r5l_exit_log(conf);
+	}
+	return ret;
+}
+
 void r5c_update_on_rdev_error(struct mddev *mddev, struct md_rdev *rdev)
 {
 	struct r5conf *conf = mddev->private;
@@ -3138,13 +3149,9 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
 
 	rcu_assign_pointer(conf->log, log);
 
-	if (r5l_load_log(log))
-		goto error;
-
 	set_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
 	return 0;
 
-error:
 	rcu_assign_pointer(conf->log, NULL);
 	md_unregister_thread(&log->reclaim_thread);
 reclaim_thread:
diff --git a/drivers/md/raid5-log.h b/drivers/md/raid5-log.h
index 284578b0a349..a001808a2b77 100644
--- a/drivers/md/raid5-log.h
+++ b/drivers/md/raid5-log.h
@@ -32,6 +32,7 @@ extern struct md_sysfs_entry r5c_journal_mode;
 extern void r5c_update_on_rdev_error(struct mddev *mddev,
 				     struct md_rdev *rdev);
 extern bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect);
+extern int r5l_start(struct r5l_log *log);
 
 extern struct dma_async_tx_descriptor *
 ops_run_partial_parity(struct stripe_head *sh, struct raid5_percpu *percpu,
@@ -42,6 +43,8 @@ extern int ppl_write_stripe(struct r5conf *conf, struct stripe_head *sh);
 extern void ppl_write_stripe_run(struct r5conf *conf);
 extern void ppl_stripe_write_finished(struct stripe_head *sh);
 extern int ppl_modify_log(struct r5conf *conf, struct md_rdev *rdev, bool add);
+extern void ppl_quiesce(struct r5conf *conf, int quiesce);
+extern int ppl_handle_flush_request(struct r5l_log *log, struct bio *bio);
 
 static inline bool raid5_has_ppl(struct r5conf *conf)
 {
@@ -87,6 +90,34 @@ static inline void log_write_stripe_run(struct r5conf *conf)
 		ppl_write_stripe_run(conf);
 }
 
+static inline void log_flush_stripe_to_raid(struct r5conf *conf)
+{
+	if (conf->log)
+		r5l_flush_stripe_to_raid(conf->log);
+	else if (raid5_has_ppl(conf))
+		ppl_write_stripe_run(conf);
+}
+
+static inline int log_handle_flush_request(struct r5conf *conf, struct bio *bio)
+{
+	int ret = -ENODEV;
+
+	if (conf->log)
+		ret = r5l_handle_flush_request(conf->log, bio);
+	else if (raid5_has_ppl(conf))
+		ret = ppl_handle_flush_request(conf->log, bio);
+
+	return ret;
+}
+
+static inline void log_quiesce(struct r5conf *conf, int quiesce)
+{
+	if (conf->log)
+		r5l_quiesce(conf->log, quiesce);
+	else if (raid5_has_ppl(conf))
+		ppl_quiesce(conf, quiesce);
+}
+
 static inline void log_exit(struct r5conf *conf)
 {
 	if (conf->log)
diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c
index 628c0bf7b9fd..42890a08375b 100644
--- a/drivers/md/raid5-ppl.c
+++ b/drivers/md/raid5-ppl.c
@@ -85,6 +85,9 @@
  * (for a single member disk). New io_units are added to the end of the list
  * and the first io_unit is submitted, if it is not submitted already.
  * The current io_unit accepting new stripes is always at the end of the list.
+ *
+ * If write-back cache is enabled for any of the disks in the array, its data
+ * must be flushed before next io_unit is submitted.
  */
 
 #define PPL_SPACE_SIZE (128 * 1024)
@@ -104,6 +107,7 @@ struct ppl_conf {
 	struct kmem_cache *io_kc;
 	mempool_t *io_pool;
 	struct bio_set *bs;
+	struct bio_set *flush_bs;
 
 	/* used only for recovery */
 	int recovered_entries;
@@ -128,6 +132,8 @@ struct ppl_log {
 	sector_t next_io_sector;
 	unsigned int entry_space;
 	bool use_multippl;
+	bool wb_cache_on;
+	unsigned long disk_flush_bitmap;
 };
 
 #define PPL_IO_INLINE_BVECS 32
@@ -145,6 +151,7 @@ struct ppl_io_unit {
 
 	struct list_head stripe_list;	/* stripes added to the io_unit */
 	atomic_t pending_stripes;	/* how many stripes not written to raid */
+	atomic_t pending_flushes;	/* how many disk flushes are in progress */
 
 	bool submitted;			/* true if write to log started */
 
@@ -249,6 +256,7 @@ static struct ppl_io_unit *ppl_new_iounit(struct ppl_log *log,
 	INIT_LIST_HEAD(&io->log_sibling);
 	INIT_LIST_HEAD(&io->stripe_list);
 	atomic_set(&io->pending_stripes, 0);
+	atomic_set(&io->pending_flushes, 0);
 	bio_init(&io->bio, io->biovec, PPL_IO_INLINE_BVECS);
 
 	pplhdr = page_address(io->header_page);
@@ -475,7 +483,18 @@ static void ppl_submit_iounit(struct ppl_io_unit *io)
 	if (log->use_multippl)
 		log->next_io_sector += (PPL_HEADER_SIZE + io->pp_size) >> 9;
 
+	WARN_ON(log->disk_flush_bitmap != 0);
+
 	list_for_each_entry(sh, &io->stripe_list, log_list) {
+		for (i = 0; i < sh->disks; i++) {
+			struct r5dev *dev = &sh->dev[i];
+
+			if ((ppl_conf->child_logs[i].wb_cache_on) &&
+			    (test_bit(R5_Wantwrite, &dev->flags))) {
+				set_bit(i, &log->disk_flush_bitmap);
+			}
+		}
+
 		/* entries for full stripe writes have no partial parity */
 		if (test_bit(STRIPE_FULL_WRITE, &sh->state))
 			continue;
@@ -540,6 +559,7 @@ static void ppl_io_unit_finished(struct ppl_io_unit *io)
 {
 	struct ppl_log *log = io->log;
 	struct ppl_conf *ppl_conf = log->ppl_conf;
+	struct r5conf *conf = ppl_conf->mddev->private;
 	unsigned long flags;
 
 	pr_debug("%s: seq: %llu\n", __func__, io->seq);
@@ -565,6 +585,122 @@ static void ppl_io_unit_finished(struct ppl_io_unit *io)
 	spin_unlock(&ppl_conf->no_mem_stripes_lock);
 
 	local_irq_restore(flags);
+
+	wake_up(&conf->wait_for_quiescent);
+}
+
+static void ppl_flush_endio(struct bio *bio)
+{
+	struct ppl_io_unit *io = bio->bi_private;
+	struct ppl_log *log = io->log;
+	struct ppl_conf *ppl_conf = log->ppl_conf;
+	struct r5conf *conf = ppl_conf->mddev->private;
+	char b[BDEVNAME_SIZE];
+
+	pr_debug("%s: dev: %s\n", __func__, bio_devname(bio, b));
+
+	if (bio->bi_status) {
+		struct md_rdev *rdev;
+
+		rcu_read_lock();
+		rdev = md_find_rdev_rcu(conf->mddev, bio_dev(bio));
+		if (rdev)
+			md_error(rdev->mddev, rdev);
+		rcu_read_unlock();
+	}
+
+	bio_put(bio);
+
+	if (atomic_dec_and_test(&io->pending_flushes)) {
+		ppl_io_unit_finished(io);
+		md_wakeup_thread(conf->mddev->thread);
+	}
+}
+
+static void ppl_do_flush(struct ppl_io_unit *io)
+{
+	struct ppl_log *log = io->log;
+	struct ppl_conf *ppl_conf = log->ppl_conf;
+	struct r5conf *conf = ppl_conf->mddev->private;
+	int raid_disks = conf->raid_disks;
+	int flushed_disks = 0;
+	int i;
+
+	atomic_set(&io->pending_flushes, raid_disks);
+
+	for_each_set_bit(i, &log->disk_flush_bitmap, raid_disks) {
+		struct md_rdev *rdev;
+		struct block_device *bdev = NULL;
+
+		rcu_read_lock();
+		rdev = rcu_dereference(conf->disks[i].rdev);
+		if (rdev && !test_bit(Faulty, &rdev->flags))
+			bdev = rdev->bdev;
+		rcu_read_unlock();
+
+		if (bdev) {
+			struct bio *bio;
+			char b[BDEVNAME_SIZE];
+
+			bio = bio_alloc_bioset(GFP_NOIO, 0, ppl_conf->flush_bs);
+			bio_set_dev(bio, bdev);
+			bio->bi_private = io;
+			bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
+			bio->bi_end_io = ppl_flush_endio;
+
+			pr_debug("%s: dev: %s\n", __func__,
+				 bio_devname(bio, b));
+
+			submit_bio(bio);
+			flushed_disks++;
+		}
+	}
+
+	log->disk_flush_bitmap = 0;
+
+	for (i = flushed_disks ; i < raid_disks; i++) {
+		if (atomic_dec_and_test(&io->pending_flushes))
+			ppl_io_unit_finished(io);
+	}
+}
+
+static inline bool ppl_no_io_unit_submitted(struct r5conf *conf,
+					    struct ppl_log *log)
+{
+	struct ppl_io_unit *io;
+
+	io = list_first_entry_or_null(&log->io_list, struct ppl_io_unit,
+				      log_sibling);
+
+	return !io || !io->submitted;
+}
+
+void ppl_quiesce(struct r5conf *conf, int quiesce)
+{
+	struct ppl_conf *ppl_conf = conf->log_private;
+	int i;
+
+	if (quiesce) {
+		for (i = 0; i < ppl_conf->count; i++) {
+			struct ppl_log *log = &ppl_conf->child_logs[i];
+
+			spin_lock_irq(&log->io_list_lock);
+			wait_event_lock_irq(conf->wait_for_quiescent,
+					    ppl_no_io_unit_submitted(conf, log),
+					    log->io_list_lock);
+			spin_unlock_irq(&log->io_list_lock);
+		}
+	}
+}
+
+int ppl_handle_flush_request(struct r5l_log *log, struct bio *bio)
+{
+	if (bio->bi_iter.bi_size == 0) {
+		bio_endio(bio);
+		return 0;
+	}
+	bio->bi_opf &= ~REQ_PREFLUSH;
+	return -EAGAIN;
 }
 
 void ppl_stripe_write_finished(struct stripe_head *sh)
@@ -574,8 +710,12 @@ void ppl_stripe_write_finished(struct stripe_head *sh)
 	io = sh->ppl_io;
 	sh->ppl_io = NULL;
 
-	if (io && atomic_dec_and_test(&io->pending_stripes))
-		ppl_io_unit_finished(io);
+	if (io && atomic_dec_and_test(&io->pending_stripes)) {
+		if (io->log->disk_flush_bitmap)
+			ppl_do_flush(io);
+		else
+			ppl_io_unit_finished(io);
+	}
 }
 
 static void ppl_xor(int size, struct page *page1, struct page *page2)
@@ -1108,6 +1248,8 @@ static void __ppl_exit_log(struct ppl_conf *ppl_conf)
 
 	if (ppl_conf->bs)
 		bioset_free(ppl_conf->bs);
+	if (ppl_conf->flush_bs)
+		bioset_free(ppl_conf->flush_bs);
 	mempool_destroy(ppl_conf->io_pool);
 	kmem_cache_destroy(ppl_conf->io_kc);
 
@@ -1173,6 +1315,8 @@ static int ppl_validate_rdev(struct md_rdev *rdev)
 
 static void ppl_init_child_log(struct ppl_log *log, struct md_rdev *rdev)
 {
+	struct request_queue *q;
+
 	if ((rdev->ppl.size << 9) >= (PPL_SPACE_SIZE +
 				      PPL_HEADER_SIZE) * 2) {
 		log->use_multippl = true;
@@ -1185,6 +1329,10 @@ static void ppl_init_child_log(struct ppl_log *log, struct md_rdev *rdev)
 				   PPL_HEADER_SIZE;
 	}
 	log->next_io_sector = rdev->ppl.sector;
+
+	q = bdev_get_queue(rdev->bdev);
+	if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
+		log->wb_cache_on = true;
 }
 
 int ppl_init_log(struct r5conf *conf)
@@ -1192,8 +1340,8 @@ int ppl_init_log(struct r5conf *conf)
 	struct ppl_conf *ppl_conf;
 	struct mddev *mddev = conf->mddev;
 	int ret = 0;
+	int max_disks;
 	int i;
-	bool need_cache_flush = false;
 
 	pr_debug("md/raid:%s: enabling distributed Partial Parity Log\n",
 		 mdname(conf->mddev));
@@ -1219,6 +1367,14 @@ int ppl_init_log(struct r5conf *conf)
 		return -EINVAL;
 	}
 
+	max_disks = FIELD_SIZEOF(struct ppl_log, disk_flush_bitmap) *
+		BITS_PER_BYTE;
+	if (conf->raid_disks > max_disks) {
+		pr_warn("md/raid:%s PPL doesn't support over %d disks in the array\n",
+			mdname(mddev), max_disks);
+		return -EINVAL;
+	}
+
 	ppl_conf = kzalloc(sizeof(struct ppl_conf), GFP_KERNEL);
 	if (!ppl_conf)
 		return -ENOMEM;
@@ -1244,6 +1400,12 @@ int ppl_init_log(struct r5conf *conf)
 		goto err;
 	}
 
+	ppl_conf->flush_bs = bioset_create(conf->raid_disks, 0, 0);
+	if (!ppl_conf->flush_bs) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
 	ppl_conf->count = conf->raid_disks;
 	ppl_conf->child_logs = kcalloc(ppl_conf->count, sizeof(struct ppl_log),
 				       GFP_KERNEL);
@@ -1275,23 +1437,14 @@ int ppl_init_log(struct r5conf *conf)
 		log->rdev = rdev;
 
 		if (rdev) {
-			struct request_queue *q;
-
 			ret = ppl_validate_rdev(rdev);
 			if (ret)
 				goto err;
 
-			q = bdev_get_queue(rdev->bdev);
-			if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
-				need_cache_flush = true;
 			ppl_init_child_log(log, rdev);
 		}
 	}
 
-	if (need_cache_flush)
-		pr_warn("md/raid:%s: Volatile write-back cache should be disabled on all member drives when using PPL!\n",
-			mdname(mddev));
-
 	/* load and possibly recover the logs from the member disks */
 	ret = ppl_load(ppl_conf);
 
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 98ce4272ace9..b5d2601483e3 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -2196,15 +2196,16 @@ static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
 static int grow_stripes(struct r5conf *conf, int num)
 {
 	struct kmem_cache *sc;
+	size_t namelen = sizeof(conf->cache_name[0]);
 	int devs = max(conf->raid_disks, conf->previous_raid_disks);
 
 	if (conf->mddev->gendisk)
-		sprintf(conf->cache_name[0],
+		snprintf(conf->cache_name[0], namelen,
 			"raid%d-%s", conf->level, mdname(conf->mddev));
 	else
-		sprintf(conf->cache_name[0],
+		snprintf(conf->cache_name[0], namelen,
 			"raid%d-%p", conf->level, conf->mddev);
-	sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);
+	snprintf(conf->cache_name[1], namelen, "%.27s-alt", conf->cache_name[0]);
 
 	conf->active_name = 0;
 	sc = kmem_cache_create(conf->cache_name[conf->active_name],
@@ -5563,7 +5564,7 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
 	bool do_flush = false;
 
 	if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {
-		int ret = r5l_handle_flush_request(conf->log, bi);
+		int ret = log_handle_flush_request(conf, bi);
 
 		if (ret == 0)
 			return true;
@@ -6168,7 +6169,7 @@ static int handle_active_stripes(struct r5conf *conf, int group,
 				break;
 		if (i == NR_STRIPE_HASH_LOCKS) {
 			spin_unlock_irq(&conf->device_lock);
-			r5l_flush_stripe_to_raid(conf->log);
+			log_flush_stripe_to_raid(conf);
 			spin_lock_irq(&conf->device_lock);
 			return batch_size;
 		}
@@ -6764,9 +6765,7 @@ static void free_conf(struct r5conf *conf)
 
 	log_exit(conf);
 
-	if (conf->shrinker.nr_deferred)
-		unregister_shrinker(&conf->shrinker);
-
+	unregister_shrinker(&conf->shrinker);
 	free_thread_groups(conf);
 	shrink_stripes(conf);
 	raid5_free_percpu(conf);
@@ -8001,13 +8000,7 @@ static void raid5_finish_reshape(struct mddev *mddev)
 
 	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
 
-		if (mddev->delta_disks > 0) {
-			md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
-			if (mddev->queue) {
-				set_capacity(mddev->gendisk, mddev->array_sectors);
-				revalidate_disk(mddev->gendisk);
-			}
-		} else {
+		if (mddev->delta_disks <= 0) {
 			int d;
 			spin_lock_irq(&conf->device_lock);
 			mddev->degraded = raid5_calc_degraded(conf);
@@ -8060,7 +8053,7 @@ static void raid5_quiesce(struct mddev *mddev, int quiesce)
 		wake_up(&conf->wait_for_overlap);
 		unlock_all_device_hash_locks_irq(conf);
 	}
-	r5l_quiesce(conf->log, quiesce);
+	log_quiesce(conf, quiesce);
 }
 
 static void *raid45_takeover_raid0(struct mddev *mddev, int level)
@@ -8364,6 +8357,13 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
 	return err;
 }
 
+static int raid5_start(struct mddev *mddev)
+{
+	struct r5conf *conf = mddev->private;
+
+	return r5l_start(conf->log);
+}
+
 static struct md_personality raid6_personality =
 {
 	.name		= "raid6",
@@ -8371,6 +8371,7 @@ static struct md_personality raid6_personality =
 	.owner		= THIS_MODULE,
 	.make_request	= raid5_make_request,
 	.run		= raid5_run,
+	.start		= raid5_start,
 	.free		= raid5_free,
 	.status		= raid5_status,
 	.error_handler	= raid5_error,
@@ -8395,6 +8396,7 @@ static struct md_personality raid5_personality =
 	.owner		= THIS_MODULE,
 	.make_request	= raid5_make_request,
 	.run		= raid5_run,
+	.start		= raid5_start,
 	.free		= raid5_free,
 	.status		= raid5_status,
 	.error_handler	= raid5_error,
@@ -8420,6 +8422,7 @@ static struct md_personality raid4_personality =
 	.owner		= THIS_MODULE,
 	.make_request	= raid5_make_request,
 	.run		= raid5_run,
+	.start		= raid5_start,
 	.free		= raid5_free,
 	.status		= raid5_status,
 	.error_handler	= raid5_error,
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 2e6123825095..3f8da26032ac 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -450,6 +450,18 @@ enum {
  * HANDLE gets cleared if stripe_handle leaves nothing locked.
  */
 
+/* Note: disk_info.rdev can be set to NULL asynchronously by raid5_remove_disk.
+ * There are three safe ways to access disk_info.rdev.
+ * 1/ when holding mddev->reconfig_mutex
+ * 2/ when resync/recovery/reshape is known to be happening - i.e. in code that
+ *    is called as part of performing resync/recovery/reshape.
+ * 3/ while holding rcu_read_lock(), use rcu_dereference to get the pointer
+ *    and if it is non-NULL, increment rdev->nr_pending before dropping the RCU
+ *    lock.
+ * When .rdev is set to NULL, the nr_pending count checked again and if
+ * it has been incremented, the pointer is put back in .rdev.
+ */
+
 struct disk_info {
 	struct md_rdev	*rdev, *replacement;
 	struct page	*extra_page; /* extra page to use in prexor */
diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig
index edfe99b22d56..86c1a190d946 100644
--- a/drivers/media/Kconfig
+++ b/drivers/media/Kconfig
@@ -80,11 +80,11 @@ config MEDIA_SDR_SUPPORT
 config MEDIA_CEC_SUPPORT
        bool "HDMI CEC support"
        ---help---
-         Enable support for HDMI CEC (Consumer Electronics Control),
-         which is an optional HDMI feature.
+	 Enable support for HDMI CEC (Consumer Electronics Control),
+	 which is an optional HDMI feature.
 
-         Say Y when you have an HDMI receiver, transmitter or a USB CEC
-         adapter that supports HDMI CEC.
+	 Say Y when you have an HDMI receiver, transmitter or a USB CEC
+	 adapter that supports HDMI CEC.
 
 source "drivers/media/cec/Kconfig"
 
@@ -144,6 +144,20 @@ config DVB_CORE
 	default y
 	select CRC32
 
+config DVB_MMAP
+	bool "Enable DVB memory-mapped API (EXPERIMENTAL)"
+	depends on DVB_CORE
+	depends on VIDEO_V4L2=y || VIDEO_V4L2=DVB_CORE
+	select VIDEOBUF2_VMALLOC
+	default n
+	help
+	  This option enables DVB experimental memory-mapped API, which
+	  reduces the number of context switches to read DVB buffers, as
+	  the buffers can use mmap() syscalls.
+
+	  Support for it is experimental. Use with care. If unsure,
+	  say N.
+
 config DVB_NET
 	bool "DVB Network Support"
 	default (NET && INET)
diff --git a/drivers/media/cec/cec-adap.c b/drivers/media/cec/cec-adap.c
index 98f88c43f62c..2b1e540587d6 100644
--- a/drivers/media/cec/cec-adap.c
+++ b/drivers/media/cec/cec-adap.c
@@ -540,7 +540,7 @@ void cec_transmit_done_ts(struct cec_adapter *adap, u8 status,
 	unsigned int attempts_made = arb_lost_cnt + nack_cnt +
 				     low_drive_cnt + error_cnt;
 
-	dprintk(2, "%s: status %02x\n", __func__, status);
+	dprintk(2, "%s: status 0x%02x\n", __func__, status);
 	if (attempts_made < 1)
 		attempts_made = 1;
 
@@ -1788,9 +1788,6 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 	int la_idx = cec_log_addr2idx(adap, dest_laddr);
 	bool from_unregistered = init_laddr == 0xf;
 	struct cec_msg tx_cec_msg = { };
-#ifdef CONFIG_MEDIA_CEC_RC
-	int scancode;
-#endif
 
 	dprintk(2, "%s: %*ph\n", __func__, msg->len, msg->msg);
 
@@ -1886,9 +1883,11 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 		 */
 		case 0x60:
 			if (msg->len == 2)
-				scancode = msg->msg[2];
+				rc_keydown(adap->rc, RC_PROTO_CEC,
+					   msg->msg[2], 0);
 			else
-				scancode = msg->msg[2] << 8 | msg->msg[3];
+				rc_keydown(adap->rc, RC_PROTO_CEC,
+					   msg->msg[2] << 8 | msg->msg[3], 0);
 			break;
 		/*
 		 * Other function messages that are not handled.
@@ -1901,54 +1900,11 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 		 */
 		case 0x56: case 0x57:
 		case 0x67: case 0x68: case 0x69: case 0x6a:
-			scancode = -1;
 			break;
 		default:
-			scancode = msg->msg[2];
-			break;
-		}
-
-		/* Was repeating, but keypress timed out */
-		if (adap->rc_repeating && !adap->rc->keypressed) {
-			adap->rc_repeating = false;
-			adap->rc_last_scancode = -1;
-		}
-		/* Different keypress from last time, ends repeat mode */
-		if (adap->rc_last_scancode != scancode) {
-			rc_keyup(adap->rc);
-			adap->rc_repeating = false;
-		}
-		/* We can't handle this scancode */
-		if (scancode < 0) {
-			adap->rc_last_scancode = scancode;
-			break;
-		}
-
-		/* Send key press */
-		rc_keydown(adap->rc, RC_PROTO_CEC, scancode, 0);
-
-		/* When in repeating mode, we're done */
-		if (adap->rc_repeating)
-			break;
-
-		/*
-		 * We are not repeating, but the new scancode is
-		 * the same as the last one, and this second key press is
-		 * within 550 ms (the 'Follower Safety Timeout') from the
-		 * previous key press, so we now enable the repeating mode.
-		 */
-		if (adap->rc_last_scancode == scancode &&
-		    msg->rx_ts - adap->rc_last_keypress < 550 * NSEC_PER_MSEC) {
-			adap->rc_repeating = true;
+			rc_keydown(adap->rc, RC_PROTO_CEC, msg->msg[2], 0);
 			break;
 		}
-		/*
-		 * Not in repeating mode, so avoid triggering repeat mode
-		 * by calling keyup.
-		 */
-		rc_keyup(adap->rc);
-		adap->rc_last_scancode = scancode;
-		adap->rc_last_keypress = msg->rx_ts;
 #endif
 		break;
 
@@ -1958,8 +1914,6 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg,
 			break;
 #ifdef CONFIG_MEDIA_CEC_RC
 		rc_keyup(adap->rc);
-		adap->rc_repeating = false;
-		adap->rc_last_scancode = -1;
 #endif
 		break;
 
@@ -2053,6 +2007,29 @@ void cec_monitor_all_cnt_dec(struct cec_adapter *adap)
 		WARN_ON(call_op(adap, adap_monitor_all_enable, 0));
 }
 
+/*
+ * Helper functions to keep track of the 'monitor pin' use count.
+ *
+ * These functions are called with adap->lock held.
+ */
+int cec_monitor_pin_cnt_inc(struct cec_adapter *adap)
+{
+	int ret = 0;
+
+	if (adap->monitor_pin_cnt == 0)
+		ret = call_op(adap, adap_monitor_pin_enable, 1);
+	if (ret == 0)
+		adap->monitor_pin_cnt++;
+	return ret;
+}
+
+void cec_monitor_pin_cnt_dec(struct cec_adapter *adap)
+{
+	adap->monitor_pin_cnt--;
+	if (adap->monitor_pin_cnt == 0)
+		WARN_ON(call_op(adap, adap_monitor_pin_enable, 0));
+}
+
 #ifdef CONFIG_DEBUG_FS
 /*
  * Log the current state of the CEC adapter.
diff --git a/drivers/media/cec/cec-api.c b/drivers/media/cec/cec-api.c
index 3dba3aa34a43..492db12b8c4d 100644
--- a/drivers/media/cec/cec-api.c
+++ b/drivers/media/cec/cec-api.c
@@ -43,24 +43,23 @@ static inline struct cec_devnode *cec_devnode_data(struct file *filp)
 
 /* CEC file operations */
 
-static unsigned int cec_poll(struct file *filp,
+static __poll_t cec_poll(struct file *filp,
 			     struct poll_table_struct *poll)
 {
-	struct cec_devnode *devnode = cec_devnode_data(filp);
 	struct cec_fh *fh = filp->private_data;
 	struct cec_adapter *adap = fh->adap;
-	unsigned int res = 0;
+	__poll_t res = 0;
 
-	if (!devnode->registered)
-		return POLLERR | POLLHUP;
+	if (!cec_is_registered(adap))
+		return EPOLLERR | EPOLLHUP;
 	mutex_lock(&adap->lock);
 	if (adap->is_configured &&
 	    adap->transmit_queue_sz < CEC_MAX_MSG_TX_QUEUE_SZ)
-		res |= POLLOUT | POLLWRNORM;
+		res |= EPOLLOUT | EPOLLWRNORM;
 	if (fh->queued_msgs)
-		res |= POLLIN | POLLRDNORM;
+		res |= EPOLLIN | EPOLLRDNORM;
 	if (fh->total_queued_events)
-		res |= POLLPRI;
+		res |= EPOLLPRI;
 	poll_wait(filp, &fh->wait, poll);
 	mutex_unlock(&adap->lock);
 	return res;
@@ -354,6 +353,7 @@ static long cec_s_mode(struct cec_adapter *adap, struct cec_fh *fh,
 	u32 mode;
 	u8 mode_initiator;
 	u8 mode_follower;
+	bool send_pin_event = false;
 	long err = 0;
 
 	if (copy_from_user(&mode, parg, sizeof(mode)))
@@ -433,6 +433,19 @@ static long cec_s_mode(struct cec_adapter *adap, struct cec_fh *fh,
 		}
 	}
 
+	if (!err) {
+		bool old_mon_pin = fh->mode_follower == CEC_MODE_MONITOR_PIN;
+		bool new_mon_pin = mode_follower == CEC_MODE_MONITOR_PIN;
+
+		if (old_mon_pin != new_mon_pin) {
+			send_pin_event = new_mon_pin;
+			if (new_mon_pin)
+				err = cec_monitor_pin_cnt_inc(adap);
+			else
+				cec_monitor_pin_cnt_dec(adap);
+		}
+	}
+
 	if (err) {
 		mutex_unlock(&adap->lock);
 		return err;
@@ -440,11 +453,9 @@ static long cec_s_mode(struct cec_adapter *adap, struct cec_fh *fh,
 
 	if (fh->mode_follower == CEC_MODE_FOLLOWER)
 		adap->follower_cnt--;
-	if (fh->mode_follower == CEC_MODE_MONITOR_PIN)
-		adap->monitor_pin_cnt--;
 	if (mode_follower == CEC_MODE_FOLLOWER)
 		adap->follower_cnt++;
-	if (mode_follower == CEC_MODE_MONITOR_PIN) {
+	if (send_pin_event) {
 		struct cec_event ev = {
 			.flags = CEC_EVENT_FL_INITIAL_STATE,
 		};
@@ -452,7 +463,6 @@ static long cec_s_mode(struct cec_adapter *adap, struct cec_fh *fh,
 		ev.event = adap->cec_pin_is_high ? CEC_EVENT_PIN_CEC_HIGH :
 						   CEC_EVENT_PIN_CEC_LOW;
 		cec_queue_event_fh(fh, &ev, 0);
-		adap->monitor_pin_cnt++;
 	}
 	if (mode_follower == CEC_MODE_EXCL_FOLLOWER ||
 	    mode_follower == CEC_MODE_EXCL_FOLLOWER_PASSTHRU) {
@@ -475,13 +485,12 @@ static long cec_s_mode(struct cec_adapter *adap, struct cec_fh *fh,
 
 static long cec_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 {
-	struct cec_devnode *devnode = cec_devnode_data(filp);
 	struct cec_fh *fh = filp->private_data;
 	struct cec_adapter *adap = fh->adap;
 	bool block = !(filp->f_flags & O_NONBLOCK);
 	void __user *parg = (void __user *)arg;
 
-	if (!devnode->registered)
+	if (!cec_is_registered(adap))
 		return -ENODEV;
 
 	switch (cmd) {
@@ -608,16 +617,15 @@ static int cec_release(struct inode *inode, struct file *filp)
 	if (fh->mode_follower == CEC_MODE_FOLLOWER)
 		adap->follower_cnt--;
 	if (fh->mode_follower == CEC_MODE_MONITOR_PIN)
-		adap->monitor_pin_cnt--;
+		cec_monitor_pin_cnt_dec(adap);
 	if (fh->mode_follower == CEC_MODE_MONITOR_ALL)
 		cec_monitor_all_cnt_dec(adap);
 	mutex_unlock(&adap->lock);
 
 	mutex_lock(&devnode->lock);
 	list_del(&fh->list);
-	if (list_empty(&devnode->fhs) &&
-	    !adap->needs_hpd &&
-	    adap->phys_addr == CEC_PHYS_ADDR_INVALID) {
+	if (cec_is_registered(adap) && list_empty(&devnode->fhs) &&
+	    !adap->needs_hpd && adap->phys_addr == CEC_PHYS_ADDR_INVALID) {
 		WARN_ON(adap->ops->adap_enable(adap, false));
 	}
 	mutex_unlock(&devnode->lock);
diff --git a/drivers/media/cec/cec-core.c b/drivers/media/cec/cec-core.c
index 5870da6a567f..a9f9525db9ae 100644
--- a/drivers/media/cec/cec-core.c
+++ b/drivers/media/cec/cec-core.c
@@ -160,8 +160,9 @@ static int __must_check cec_devnode_register(struct cec_devnode *devnode,
  * This function can safely be called if the device node has never been
  * registered or has already been unregistered.
  */
-static void cec_devnode_unregister(struct cec_devnode *devnode)
+static void cec_devnode_unregister(struct cec_adapter *adap)
 {
+	struct cec_devnode *devnode = &adap->devnode;
 	struct cec_fh *fh;
 
 	mutex_lock(&devnode->lock);
@@ -179,6 +180,11 @@ static void cec_devnode_unregister(struct cec_devnode *devnode)
 	devnode->unregistered = true;
 	mutex_unlock(&devnode->lock);
 
+	mutex_lock(&adap->lock);
+	__cec_s_phys_addr(adap, CEC_PHYS_ADDR_INVALID, false);
+	__cec_s_log_addrs(adap, NULL, false);
+	mutex_unlock(&adap->lock);
+
 	cdev_device_del(&devnode->cdev, &devnode->dev);
 	put_device(&devnode->dev);
 }
@@ -192,7 +198,7 @@ static void cec_cec_notify(struct cec_adapter *adap, u16 pa)
 void cec_register_cec_notifier(struct cec_adapter *adap,
 			       struct cec_notifier *notifier)
 {
-	if (WARN_ON(!adap->devnode.registered))
+	if (WARN_ON(!cec_is_registered(adap)))
 		return;
 
 	adap->notifier = notifier;
@@ -280,7 +286,6 @@ struct cec_adapter *cec_allocate_adapter(const struct cec_adap_ops *ops,
 	adap->rc->priv = adap;
 	adap->rc->map_name = RC_MAP_CEC;
 	adap->rc->timeout = MS_TO_NS(100);
-	adap->rc_last_scancode = -1;
 #endif
 	return adap;
 }
@@ -312,17 +317,6 @@ int cec_register_adapter(struct cec_adapter *adap,
 			adap->rc = NULL;
 			return res;
 		}
-		/*
-		 * The REP_DELAY for CEC is really the time between the initial
-		 * 'User Control Pressed' message and the second. The first
-		 * keypress is always seen as non-repeating, the second
-		 * (provided it has the same UI Command) will start the 'Press
-		 * and Hold' (aka repeat) behavior. By setting REP_DELAY to the
-		 * same value as REP_PERIOD the expected CEC behavior is
-		 * reproduced.
-		 */
-		adap->rc->input_dev->rep[REP_DELAY] =
-			adap->rc->input_dev->rep[REP_PERIOD];
 	}
 #endif
 
@@ -373,7 +367,7 @@ void cec_unregister_adapter(struct cec_adapter *adap)
 	if (adap->notifier)
 		cec_notifier_unregister(adap->notifier);
 #endif
-	cec_devnode_unregister(&adap->devnode);
+	cec_devnode_unregister(adap);
 }
 EXPORT_SYMBOL_GPL(cec_unregister_adapter);
 
@@ -381,9 +375,6 @@ void cec_delete_adapter(struct cec_adapter *adap)
 {
 	if (IS_ERR_OR_NULL(adap))
 		return;
-	mutex_lock(&adap->lock);
-	__cec_s_phys_addr(adap, CEC_PHYS_ADDR_INVALID, false);
-	mutex_unlock(&adap->lock);
 	kthread_stop(adap->kthread);
 	if (adap->kthread_config)
 		kthread_stop(adap->kthread_config);
diff --git a/drivers/media/cec/cec-priv.h b/drivers/media/cec/cec-priv.h
index 70767a7900f2..daf597643af8 100644
--- a/drivers/media/cec/cec-priv.h
+++ b/drivers/media/cec/cec-priv.h
@@ -40,6 +40,8 @@ void cec_put_device(struct cec_devnode *devnode);
 /* cec-adap.c */
 int cec_monitor_all_cnt_inc(struct cec_adapter *adap);
 void cec_monitor_all_cnt_dec(struct cec_adapter *adap);
+int cec_monitor_pin_cnt_inc(struct cec_adapter *adap);
+void cec_monitor_pin_cnt_dec(struct cec_adapter *adap);
 int cec_adap_status(struct seq_file *file, void *priv);
 int cec_thread_func(void *_adap);
 void __cec_s_phys_addr(struct cec_adapter *adap, u16 phys_addr, bool block);
diff --git a/drivers/media/common/Kconfig b/drivers/media/common/Kconfig
index 326df0ad75c0..0cb7d819a5d2 100644
--- a/drivers/media/common/Kconfig
+++ b/drivers/media/common/Kconfig
@@ -16,6 +16,7 @@ config CYPRESS_FIRMWARE
 	tristate "Cypress firmware helper routines"
 	depends on USB
 
+source "drivers/media/common/videobuf2/Kconfig"
 source "drivers/media/common/b2c2/Kconfig"
 source "drivers/media/common/saa7146/Kconfig"
 source "drivers/media/common/siano/Kconfig"
diff --git a/drivers/media/common/Makefile b/drivers/media/common/Makefile
index 2d1b0a025084..e7bc17abbbbc 100644
--- a/drivers/media/common/Makefile
+++ b/drivers/media/common/Makefile
@@ -1,4 +1,4 @@
-obj-y += b2c2/ saa7146/ siano/ v4l2-tpg/
+obj-y += b2c2/ saa7146/ siano/ v4l2-tpg/ videobuf2/
 obj-$(CONFIG_VIDEO_CX2341X) += cx2341x.o
 obj-$(CONFIG_VIDEO_TVEEPROM) += tveeprom.o
 obj-$(CONFIG_CYPRESS_FIRMWARE) += cypress_firmware.o
diff --git a/drivers/media/common/b2c2/Makefile b/drivers/media/common/b2c2/Makefile
index 73df4a334eda..aa2dc2434ee5 100644
--- a/drivers/media/common/b2c2/Makefile
+++ b/drivers/media/common/b2c2/Makefile
@@ -4,6 +4,5 @@ b2c2-flexcop-objs += flexcop-sram.o flexcop-eeprom.o flexcop-misc.o
 b2c2-flexcop-objs += flexcop-hw-filter.o
 obj-$(CONFIG_DVB_B2C2_FLEXCOP) += b2c2-flexcop.o
 
-ccflags-y += -Idrivers/media/dvb-core/
 ccflags-y += -Idrivers/media/dvb-frontends/
 ccflags-y += -Idrivers/media/tuners/
diff --git a/drivers/media/common/b2c2/flexcop-common.h b/drivers/media/common/b2c2/flexcop-common.h
index b7e5e4c17acb..f944c59cf495 100644
--- a/drivers/media/common/b2c2/flexcop-common.h
+++ b/drivers/media/common/b2c2/flexcop-common.h
@@ -13,10 +13,10 @@
 
 #include "flexcop-reg.h"
 
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_net.h"
-#include "dvb_frontend.h"
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
+#include <media/dvb_frontend.h>
 
 #define FC_MAX_FEED 256
 
diff --git a/drivers/media/common/saa7146/saa7146_fops.c b/drivers/media/common/saa7146/saa7146_fops.c
index 8c87d6837c49..d4987fd05d05 100644
--- a/drivers/media/common/saa7146/saa7146_fops.c
+++ b/drivers/media/common/saa7146/saa7146_fops.c
@@ -320,19 +320,19 @@ static int fops_mmap(struct file *file, struct vm_area_struct * vma)
 	return res;
 }
 
-static unsigned int __fops_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t __fops_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct video_device *vdev = video_devdata(file);
 	struct saa7146_fh *fh = file->private_data;
 	struct videobuf_buffer *buf = NULL;
 	struct videobuf_queue *q;
-	unsigned int res = v4l2_ctrl_poll(file, wait);
+	__poll_t res = v4l2_ctrl_poll(file, wait);
 
 	DEB_EE("file:%p, poll:%p\n", file, wait);
 
 	if (vdev->vfl_type == VFL_TYPE_VBI) {
 		if (fh->dev->ext_vv_data->capabilities & V4L2_CAP_SLICED_VBI_OUTPUT)
-			return res | POLLOUT | POLLWRNORM;
+			return res | EPOLLOUT | EPOLLWRNORM;
 		if( 0 == fh->vbi_q.streaming )
 			return res | videobuf_poll_stream(file, &fh->vbi_q, wait);
 		q = &fh->vbi_q;
@@ -346,23 +346,23 @@ static unsigned int __fops_poll(struct file *file, struct poll_table_struct *wai
 
 	if (!buf) {
 		DEB_D("buf == NULL!\n");
-		return res | POLLERR;
+		return res | EPOLLERR;
 	}
 
 	poll_wait(file, &buf->done, wait);
 	if (buf->state == VIDEOBUF_DONE || buf->state == VIDEOBUF_ERROR) {
 		DEB_D("poll succeeded!\n");
-		return res | POLLIN | POLLRDNORM;
+		return res | EPOLLIN | EPOLLRDNORM;
 	}
 
 	DEB_D("nothing to poll for, buf->state:%d\n", buf->state);
 	return res;
 }
 
-static unsigned int fops_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t fops_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct video_device *vdev = video_devdata(file);
-	unsigned int res;
+	__poll_t res;
 
 	mutex_lock(vdev->lock);
 	res = __fops_poll(file, wait);
diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/media/common/saa7146/saa7146_video.c
index 2b631eaa65b3..0dfa0c09d646 100644
--- a/drivers/media/common/saa7146/saa7146_video.c
+++ b/drivers/media/common/saa7146/saa7146_video.c
@@ -4,6 +4,7 @@
 #include <media/v4l2-event.h>
 #include <media/v4l2-ctrls.h>
 #include <linux/module.h>
+#include <linux/kernel.h>
 
 static int max_memory = 32;
 
@@ -86,13 +87,11 @@ static struct saa7146_format formats[] = {
    due to this, it's impossible to provide additional *packed* formats, which are simply byte swapped
    (like V4L2_PIX_FMT_YUYV) ... 8-( */
 
-static int NUM_FORMATS = sizeof(formats)/sizeof(struct saa7146_format);
-
 struct saa7146_format* saa7146_format_by_fourcc(struct saa7146_dev *dev, int fourcc)
 {
-	int i, j = NUM_FORMATS;
+	int i;
 
-	for (i = 0; i < j; i++) {
+	for (i = 0; i < ARRAY_SIZE(formats); i++) {
 		if (formats[i].pixelformat == fourcc) {
 			return formats+i;
 		}
@@ -524,7 +523,7 @@ static int vidioc_s_fbuf(struct file *file, void *fh, const struct v4l2_framebuf
 
 static int vidioc_enum_fmt_vid_cap(struct file *file, void *fh, struct v4l2_fmtdesc *f)
 {
-	if (f->index >= NUM_FORMATS)
+	if (f->index >= ARRAY_SIZE(formats))
 		return -EINVAL;
 	strlcpy((char *)f->description, formats[f->index].name,
 			sizeof(f->description));
@@ -1002,9 +1001,9 @@ const struct v4l2_ioctl_ops saa7146_video_ioctl_ops = {
 	.vidioc_try_fmt_vid_overlay  = vidioc_try_fmt_vid_overlay,
 	.vidioc_s_fmt_vid_overlay    = vidioc_s_fmt_vid_overlay,
 
-	.vidioc_overlay 	     = vidioc_overlay,
-	.vidioc_g_fbuf  	     = vidioc_g_fbuf,
-	.vidioc_s_fbuf  	     = vidioc_s_fbuf,
+	.vidioc_overlay		     = vidioc_overlay,
+	.vidioc_g_fbuf		     = vidioc_g_fbuf,
+	.vidioc_s_fbuf		     = vidioc_s_fbuf,
 	.vidioc_reqbufs              = vidioc_reqbufs,
 	.vidioc_querybuf             = vidioc_querybuf,
 	.vidioc_qbuf                 = vidioc_qbuf,
@@ -1013,7 +1012,7 @@ const struct v4l2_ioctl_ops saa7146_video_ioctl_ops = {
 	.vidioc_s_std                = vidioc_s_std,
 	.vidioc_streamon             = vidioc_streamon,
 	.vidioc_streamoff            = vidioc_streamoff,
-	.vidioc_g_parm 		     = vidioc_g_parm,
+	.vidioc_g_parm		     = vidioc_g_parm,
 	.vidioc_subscribe_event      = v4l2_ctrl_subscribe_event,
 	.vidioc_unsubscribe_event    = v4l2_event_unsubscribe,
 };
diff --git a/drivers/media/common/siano/Makefile b/drivers/media/common/siano/Makefile
index 88e2b7ffc537..b33022e0be56 100644
--- a/drivers/media/common/siano/Makefile
+++ b/drivers/media/common/siano/Makefile
@@ -11,7 +11,3 @@ endif
 ifeq ($(CONFIG_SMS_SIANO_DEBUGFS),y)
   smsdvb-objs += smsdvb-debugfs.o
 endif
-
-ccflags-y += -Idrivers/media/dvb-core
-ccflags-y += $(extra-cflags-y) $(extra-cflags-m)
-
diff --git a/drivers/media/common/siano/smsdvb-debugfs.c b/drivers/media/common/siano/smsdvb-debugfs.c
index 1a8677ade391..40891f4f842b 100644
--- a/drivers/media/common/siano/smsdvb-debugfs.c
+++ b/drivers/media/common/siano/smsdvb-debugfs.c
@@ -1,21 +1,6 @@
-/***********************************************************************
- *
- * Copyright(c) 2013 Mauro Carvalho Chehab
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 2 of the License, or
- * (at your option) any later version.
-
- *  This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- ***********************************************************************/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright(c) 2013 Mauro Carvalho Chehab
 
 #include "smscoreapi.h"
 
@@ -26,10 +11,10 @@
 #include <linux/spinlock.h>
 #include <linux/usb.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
 
 #include "smsdvb.h"
 
@@ -374,7 +359,7 @@ static int smsdvb_stats_wait_read(struct smsdvb_debugfs *debug_data)
 	return rc;
 }
 
-static unsigned int smsdvb_stats_poll(struct file *file, poll_table *wait)
+static __poll_t smsdvb_stats_poll(struct file *file, poll_table *wait)
 {
 	struct smsdvb_debugfs *debug_data = file->private_data;
 	int rc;
@@ -384,12 +369,9 @@ static unsigned int smsdvb_stats_poll(struct file *file, poll_table *wait)
 	poll_wait(file, &debug_data->stats_queue, wait);
 
 	rc = smsdvb_stats_wait_read(debug_data);
-	if (rc > 0)
-		rc = POLLIN | POLLRDNORM;
-
 	kref_put(&debug_data->refcount, smsdvb_debugfs_data_release);
 
-	return rc;
+	return rc > 0 ? EPOLLIN | EPOLLRDNORM : 0;
 }
 
 static ssize_t smsdvb_stats_read(struct file *file, char __user *user_buf,
diff --git a/drivers/media/common/siano/smsdvb-main.c b/drivers/media/common/siano/smsdvb-main.c
index affde1426b7a..c0faad1ba428 100644
--- a/drivers/media/common/siano/smsdvb-main.c
+++ b/drivers/media/common/siano/smsdvb-main.c
@@ -26,10 +26,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #include <linux/init.h>
 #include <asm/div64.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
 
 #include "sms-cards.h"
 
@@ -271,7 +271,7 @@ static void smsdvb_update_per_slices(struct smsdvb_client_t *client,
 	c->post_bit_count.stat[0].uvalue += p->ber_bit_count;
 
 	/* Legacy PER/BER */
-	tmp = p->ets_packets * 65535;
+	tmp = p->ets_packets * 65535ULL;
 	if (p->ts_packets + p->ets_packets)
 		do_div(tmp, p->ts_packets + p->ets_packets);
 	client->legacy_per = tmp;
diff --git a/drivers/media/common/siano/smsir.c b/drivers/media/common/siano/smsir.c
index e77bb0c95e69..56db0a944421 100644
--- a/drivers/media/common/siano/smsir.c
+++ b/drivers/media/common/siano/smsir.c
@@ -1,28 +1,13 @@
-/****************************************************************
-
- Siano Mobile Silicon, Inc.
- MDTV receiver kernel modules.
- Copyright (C) 2006-2009, Uri Shkolnik
-
- Copyright (c) 2010 - Mauro Carvalho Chehab
-	- Ported the driver to use rc-core
-	- IR raw event decoding is now done at rc-core
-	- Code almost re-written
-
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation, either version 2 of the License, or
- (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
- ****************************************************************/
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Siano Mobile Silicon, Inc.
+// MDTV receiver kernel modules.
+// Copyright (C) 2006-2009, Uri Shkolnik
+//
+// Copyright (c) 2010 - Mauro Carvalho Chehab
+//	- Ported the driver to use rc-core
+//	- IR raw event decoding is now done at rc-core
+//	- Code almost re-written
 
 
 #include "smscoreapi.h"
diff --git a/drivers/media/common/siano/smsir.h b/drivers/media/common/siano/smsir.h
index d9abd96ef48b..b2c54c256e86 100644
--- a/drivers/media/common/siano/smsir.h
+++ b/drivers/media/common/siano/smsir.h
@@ -1,28 +1,15 @@
-/****************************************************************
-
-Siano Mobile Silicon, Inc.
-MDTV receiver kernel modules.
-Copyright (C) 2006-2009, Uri Shkolnik
-
- Copyright (c) 2010 - Mauro Carvalho Chehab
-	- Ported the driver to use rc-core
-	- IR raw event decoding is now done at rc-core
-	- Code almost re-written
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 2 of the License, or
-(at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-****************************************************************/
+/*
+ * SPDX-License-Identifier: GPL-2.0+
+ *
+ * Siano Mobile Silicon, Inc.
+ * MDTV receiver kernel modules.
+ * Copyright (C) 2006-2009, Uri Shkolnik
+ *
+ * Copyright (c) 2010 - Mauro Carvalho Chehab
+ *	- Ported the driver to use rc-core
+ *	- IR raw event decoding is now done at rc-core
+ *	- Code almost re-written
+ */
 
 #ifndef __SMS_IR_H__
 #define __SMS_IR_H__
diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c
index 5b5f95c38fe1..43180204fab2 100644
--- a/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c
+++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-colors.c
@@ -36,10 +36,10 @@
  */
 
 #include <linux/videodev2.h>
-#include <media/v4l2-tpg-colors.h>
+#include <media/tpg/v4l2-tpg.h>
 
 /* sRGB colors with range [0-255] */
-const struct color tpg_colors[TPG_COLOR_MAX] = {
+const struct tpg_rbg_color8 tpg_colors[TPG_COLOR_MAX] = {
 	/*
 	 * Colors to test colorspace conversion: converting these colors
 	 * to other colorspaces will never lead to out-of-gamut colors.
@@ -597,7 +597,7 @@ const unsigned short tpg_linear_to_rec709[255 * 16 + 1] = {
 };
 
 /* Generated table */
-const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_SMPTE2084 + 1][TPG_COLOR_CSC_BLACK + 1] = {
+const struct tpg_rbg_color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_SMPTE2084 + 1][TPG_COLOR_CSC_BLACK + 1] = {
 	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_709][0] = { 2939, 2939, 2939 },
 	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_709][1] = { 2953, 2963, 586 },
 	[V4L2_COLORSPACE_SMPTE170M][V4L2_XFER_FUNC_709][2] = { 0, 2967, 2937 },
@@ -1392,7 +1392,7 @@ int main(int argc, char **argv)
 	printf("\n};\n\n");
 
 	printf("/* Generated table */\n");
-	printf("const struct color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_SMPTE2084 + 1][TPG_COLOR_CSC_BLACK + 1] = {\n");
+	printf("const struct tpg_rbg_color16 tpg_csc_colors[V4L2_COLORSPACE_DCI_P3 + 1][V4L2_XFER_FUNC_SMPTE2084 + 1][TPG_COLOR_CSC_BLACK + 1] = {\n");
 	for (c = 0; c <= V4L2_COLORSPACE_DCI_P3; c++) {
 		for (x = 1; x <= V4L2_XFER_FUNC_SMPTE2084; x++) {
 			for (i = 0; i <= TPG_COLOR_CSC_BLACK; i++) {
diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
index f96968c11312..2b3d4ac4dfd4 100644
--- a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
+++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c
@@ -21,7 +21,7 @@
  */
 
 #include <linux/module.h>
-#include <media/v4l2-tpg.h>
+#include <media/tpg/v4l2-tpg.h>
 
 /* Must remain in sync with enum tpg_pattern */
 const char * const tpg_pattern_strings[] = {
diff --git a/drivers/media/common/videobuf2/Kconfig b/drivers/media/common/videobuf2/Kconfig
new file mode 100644
index 000000000000..17c32ea58395
--- /dev/null
+++ b/drivers/media/common/videobuf2/Kconfig
@@ -0,0 +1,34 @@
+# Used by drivers that need Videobuf2 modules
+config VIDEOBUF2_CORE
+	select DMA_SHARED_BUFFER
+	tristate
+
+config VIDEOBUF2_V4L2
+	tristate
+
+config VIDEOBUF2_MEMOPS
+	tristate
+	select FRAME_VECTOR
+
+config VIDEOBUF2_DMA_CONTIG
+	tristate
+	depends on HAS_DMA
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_MEMOPS
+	select DMA_SHARED_BUFFER
+
+config VIDEOBUF2_VMALLOC
+	tristate
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_MEMOPS
+	select DMA_SHARED_BUFFER
+
+config VIDEOBUF2_DMA_SG
+	tristate
+	depends on HAS_DMA
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_MEMOPS
+
+config VIDEOBUF2_DVB
+	tristate
+	select VIDEOBUF2_CORE
diff --git a/drivers/media/common/videobuf2/Makefile b/drivers/media/common/videobuf2/Makefile
new file mode 100644
index 000000000000..77bebe8b202f
--- /dev/null
+++ b/drivers/media/common/videobuf2/Makefile
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+videobuf2-common-objs := videobuf2-core.o
+
+ifeq ($(CONFIG_TRACEPOINTS),y)
+  videobuf2-common-objs += vb2-trace.o
+endif
+
+obj-$(CONFIG_VIDEOBUF2_CORE) += videobuf2-common.o
+obj-$(CONFIG_VIDEOBUF2_V4L2) += videobuf2-v4l2.o
+obj-$(CONFIG_VIDEOBUF2_MEMOPS) += videobuf2-memops.o
+obj-$(CONFIG_VIDEOBUF2_VMALLOC) += videobuf2-vmalloc.o
+obj-$(CONFIG_VIDEOBUF2_DMA_CONTIG) += videobuf2-dma-contig.o
+obj-$(CONFIG_VIDEOBUF2_DMA_SG) += videobuf2-dma-sg.o
+obj-$(CONFIG_VIDEOBUF2_DVB) += videobuf2-dvb.o
diff --git a/drivers/media/v4l2-core/vb2-trace.c b/drivers/media/common/videobuf2/vb2-trace.c
similarity index 100%
rename from drivers/media/v4l2-core/vb2-trace.c
rename to drivers/media/common/videobuf2/vb2-trace.c
diff --git a/drivers/media/v4l2-core/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c
similarity index 96%
rename from drivers/media/v4l2-core/videobuf2-core.c
rename to drivers/media/common/videobuf2/videobuf2-core.c
index a8589d96ef72..debe35fc66b4 100644
--- a/drivers/media/v4l2-core/videobuf2-core.c
+++ b/drivers/media/common/videobuf2/videobuf2-core.c
@@ -14,6 +14,8 @@
  * the Free Software Foundation.
  */
 
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
 #include <linux/err.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -32,10 +34,10 @@
 static int debug;
 module_param(debug, int, 0644);
 
-#define dprintk(level, fmt, arg...)					      \
-	do {								      \
-		if (debug >= level)					      \
-			pr_info("vb2-core: %s: " fmt, __func__, ## arg); \
+#define dprintk(level, fmt, arg...)				\
+	do {							\
+		if (debug >= level)				\
+			pr_info("%s: " fmt, __func__, ## arg);	\
 	} while (0)
 
 #ifdef CONFIG_VIDEO_ADV_DEBUG
@@ -332,6 +334,10 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum vb2_memory memory,
 	struct vb2_buffer *vb;
 	int ret;
 
+	/* Ensure that q->num_buffers+num_buffers is below VB2_MAX_FRAME */
+	num_buffers = min_t(unsigned int, num_buffers,
+			    VB2_MAX_FRAME - q->num_buffers);
+
 	for (buffer = 0; buffer < num_buffers; ++buffer) {
 		/* Allocate videobuf buffer structures */
 		vb = kzalloc(q->buf_struct_size, GFP_KERNEL);
@@ -460,12 +466,12 @@ static int __vb2_queue_free(struct vb2_queue *q, unsigned int buffers)
 				  q->cnt_wait_prepare != q->cnt_wait_finish;
 
 		if (unbalanced || debug) {
-			pr_info("vb2: counters for queue %p:%s\n", q,
+			pr_info("counters for queue %p:%s\n", q,
 				unbalanced ? " UNBALANCED!" : "");
-			pr_info("vb2:     setup: %u start_streaming: %u stop_streaming: %u\n",
+			pr_info("     setup: %u start_streaming: %u stop_streaming: %u\n",
 				q->cnt_queue_setup, q->cnt_start_streaming,
 				q->cnt_stop_streaming);
-			pr_info("vb2:     wait_prepare: %u wait_finish: %u\n",
+			pr_info("     wait_prepare: %u wait_finish: %u\n",
 				q->cnt_wait_prepare, q->cnt_wait_finish);
 		}
 		q->cnt_queue_setup = 0;
@@ -486,23 +492,23 @@ static int __vb2_queue_free(struct vb2_queue *q, unsigned int buffers)
 				  vb->cnt_buf_init != vb->cnt_buf_cleanup;
 
 		if (unbalanced || debug) {
-			pr_info("vb2:   counters for queue %p, buffer %d:%s\n",
+			pr_info("   counters for queue %p, buffer %d:%s\n",
 				q, buffer, unbalanced ? " UNBALANCED!" : "");
-			pr_info("vb2:     buf_init: %u buf_cleanup: %u buf_prepare: %u buf_finish: %u\n",
+			pr_info("     buf_init: %u buf_cleanup: %u buf_prepare: %u buf_finish: %u\n",
 				vb->cnt_buf_init, vb->cnt_buf_cleanup,
 				vb->cnt_buf_prepare, vb->cnt_buf_finish);
-			pr_info("vb2:     buf_queue: %u buf_done: %u\n",
+			pr_info("     buf_queue: %u buf_done: %u\n",
 				vb->cnt_buf_queue, vb->cnt_buf_done);
-			pr_info("vb2:     alloc: %u put: %u prepare: %u finish: %u mmap: %u\n",
+			pr_info("     alloc: %u put: %u prepare: %u finish: %u mmap: %u\n",
 				vb->cnt_mem_alloc, vb->cnt_mem_put,
 				vb->cnt_mem_prepare, vb->cnt_mem_finish,
 				vb->cnt_mem_mmap);
-			pr_info("vb2:     get_userptr: %u put_userptr: %u\n",
+			pr_info("     get_userptr: %u put_userptr: %u\n",
 				vb->cnt_mem_get_userptr, vb->cnt_mem_put_userptr);
-			pr_info("vb2:     attach_dmabuf: %u detach_dmabuf: %u map_dmabuf: %u unmap_dmabuf: %u\n",
+			pr_info("     attach_dmabuf: %u detach_dmabuf: %u map_dmabuf: %u unmap_dmabuf: %u\n",
 				vb->cnt_mem_attach_dmabuf, vb->cnt_mem_detach_dmabuf,
 				vb->cnt_mem_map_dmabuf, vb->cnt_mem_unmap_dmabuf);
-			pr_info("vb2:     get_dmabuf: %u num_users: %u vaddr: %u cookie: %u\n",
+			pr_info("     get_dmabuf: %u num_users: %u vaddr: %u cookie: %u\n",
 				vb->cnt_mem_get_dmabuf,
 				vb->cnt_mem_num_users,
 				vb->cnt_mem_vaddr,
@@ -520,7 +526,7 @@ static int __vb2_queue_free(struct vb2_queue *q, unsigned int buffers)
 
 	q->num_buffers -= buffers;
 	if (!q->num_buffers) {
-		q->memory = 0;
+		q->memory = VB2_MEMORY_UNKNOWN;
 		INIT_LIST_HEAD(&q->queued_list);
 	}
 	return 0;
@@ -662,7 +668,8 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
 		return -EBUSY;
 	}
 
-	if (*count == 0 || q->num_buffers != 0 || q->memory != memory) {
+	if (*count == 0 || q->num_buffers != 0 ||
+	    (q->memory != VB2_MEMORY_UNKNOWN && q->memory != memory)) {
 		/*
 		 * We already have buffers allocated, so first check if they
 		 * are not in use and can be freed.
@@ -696,8 +703,9 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
 	/*
 	 * Make sure the requested values and current defaults are sane.
 	 */
-	num_buffers = min_t(unsigned int, *count, VB2_MAX_FRAME);
-	num_buffers = max_t(unsigned int, num_buffers, q->min_buffers_needed);
+	WARN_ON(q->min_buffers_needed > VB2_MAX_FRAME);
+	num_buffers = max_t(unsigned int, *count, q->min_buffers_needed);
+	num_buffers = min_t(unsigned int, num_buffers, VB2_MAX_FRAME);
 	memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
 	q->memory = memory;
 
@@ -1650,8 +1658,11 @@ static void __vb2_queue_cancel(struct vb2_queue *q)
 	 */
 	if (WARN_ON(atomic_read(&q->owned_by_drv_count))) {
 		for (i = 0; i < q->num_buffers; ++i)
-			if (q->bufs[i]->state == VB2_BUF_STATE_ACTIVE)
+			if (q->bufs[i]->state == VB2_BUF_STATE_ACTIVE) {
+				pr_warn("driver bug: stop_streaming operation is leaving buf %p in active state\n",
+					q->bufs[i]);
 				vb2_buffer_done(q->bufs[i], VB2_BUF_STATE_ERROR);
+			}
 		/* Must be zero now */
 		WARN_ON(atomic_read(&q->owned_by_drv_count));
 	}
@@ -1994,6 +2005,8 @@ int vb2_core_queue_init(struct vb2_queue *q)
 	mutex_init(&q->mmap_lock);
 	init_waitqueue_head(&q->done_wq);
 
+	q->memory = VB2_MEMORY_UNKNOWN;
+
 	if (q->buf_struct_size == 0)
 		q->buf_struct_size = sizeof(struct vb2_buffer);
 
@@ -2018,16 +2031,16 @@ void vb2_core_queue_release(struct vb2_queue *q)
 }
 EXPORT_SYMBOL_GPL(vb2_core_queue_release);
 
-unsigned int vb2_core_poll(struct vb2_queue *q, struct file *file,
+__poll_t vb2_core_poll(struct vb2_queue *q, struct file *file,
 		poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct vb2_buffer *vb = NULL;
 	unsigned long flags;
 
-	if (!q->is_output && !(req_events & (POLLIN | POLLRDNORM)))
+	if (!q->is_output && !(req_events & (EPOLLIN | EPOLLRDNORM)))
 		return 0;
-	if (q->is_output && !(req_events & (POLLOUT | POLLWRNORM)))
+	if (q->is_output && !(req_events & (EPOLLOUT | EPOLLWRNORM)))
 		return 0;
 
 	/*
@@ -2035,18 +2048,18 @@ unsigned int vb2_core_poll(struct vb2_queue *q, struct file *file,
 	 */
 	if (q->num_buffers == 0 && !vb2_fileio_is_active(q)) {
 		if (!q->is_output && (q->io_modes & VB2_READ) &&
-				(req_events & (POLLIN | POLLRDNORM))) {
+				(req_events & (EPOLLIN | EPOLLRDNORM))) {
 			if (__vb2_init_fileio(q, 1))
-				return POLLERR;
+				return EPOLLERR;
 		}
 		if (q->is_output && (q->io_modes & VB2_WRITE) &&
-				(req_events & (POLLOUT | POLLWRNORM))) {
+				(req_events & (EPOLLOUT | EPOLLWRNORM))) {
 			if (__vb2_init_fileio(q, 0))
-				return POLLERR;
+				return EPOLLERR;
 			/*
 			 * Write to OUTPUT queue can be done immediately.
 			 */
-			return POLLOUT | POLLWRNORM;
+			return EPOLLOUT | EPOLLWRNORM;
 		}
 	}
 
@@ -2055,24 +2068,24 @@ unsigned int vb2_core_poll(struct vb2_queue *q, struct file *file,
 	 * error flag is set.
 	 */
 	if (!vb2_is_streaming(q) || q->error)
-		return POLLERR;
+		return EPOLLERR;
 
 	/*
 	 * If this quirk is set and QBUF hasn't been called yet then
-	 * return POLLERR as well. This only affects capture queues, output
+	 * return EPOLLERR as well. This only affects capture queues, output
 	 * queues will always initialize waiting_for_buffers to false.
 	 * This quirk is set by V4L2 for backwards compatibility reasons.
 	 */
 	if (q->quirk_poll_must_check_waiting_for_buffers &&
-	    q->waiting_for_buffers && (req_events & (POLLIN | POLLRDNORM)))
-		return POLLERR;
+	    q->waiting_for_buffers && (req_events & (EPOLLIN | EPOLLRDNORM)))
+		return EPOLLERR;
 
 	/*
 	 * For output streams you can call write() as long as there are fewer
 	 * buffers queued than there are buffers available.
 	 */
 	if (q->is_output && q->fileio && q->queued_count < q->num_buffers)
-		return POLLOUT | POLLWRNORM;
+		return EPOLLOUT | EPOLLWRNORM;
 
 	if (list_empty(&q->done_list)) {
 		/*
@@ -2080,7 +2093,7 @@ unsigned int vb2_core_poll(struct vb2_queue *q, struct file *file,
 		 * return immediately. DQBUF will return -EPIPE.
 		 */
 		if (q->last_buffer_dequeued)
-			return POLLIN | POLLRDNORM;
+			return EPOLLIN | EPOLLRDNORM;
 
 		poll_wait(file, &q->done_wq, wait);
 	}
@@ -2097,8 +2110,8 @@ unsigned int vb2_core_poll(struct vb2_queue *q, struct file *file,
 	if (vb && (vb->state == VB2_BUF_STATE_DONE
 			|| vb->state == VB2_BUF_STATE_ERROR)) {
 		return (q->is_output) ?
-				POLLOUT | POLLWRNORM :
-				POLLIN | POLLRDNORM;
+				EPOLLOUT | EPOLLWRNORM :
+				EPOLLIN | EPOLLRDNORM;
 	}
 	return 0;
 }
@@ -2522,7 +2535,7 @@ static int vb2_thread(void *data)
 				break;
 		call_void_qop(q, wait_finish, q);
 		if (copy_timestamp)
-			vb->timestamp = ktime_get_ns();;
+			vb->timestamp = ktime_get_ns();
 		if (!threadio->stop)
 			ret = vb2_core_qbuf(q, vb->index, NULL);
 		call_void_qop(q, wait_prepare, q);
diff --git a/drivers/media/v4l2-core/videobuf2-dma-contig.c b/drivers/media/common/videobuf2/videobuf2-dma-contig.c
similarity index 99%
rename from drivers/media/v4l2-core/videobuf2-dma-contig.c
rename to drivers/media/common/videobuf2/videobuf2-dma-contig.c
index a9806ba6116d..f1178f6f434d 100644
--- a/drivers/media/v4l2-core/videobuf2-dma-contig.c
+++ b/drivers/media/common/videobuf2/videobuf2-dma-contig.c
@@ -434,8 +434,10 @@ static void vb2_dc_put_userptr(void *buf_priv)
 		pages = frame_vector_pages(buf->vec);
 		/* sgt should exist only if vector contains pages... */
 		BUG_ON(IS_ERR(pages));
-		for (i = 0; i < frame_vector_count(buf->vec); i++)
-			set_page_dirty_lock(pages[i]);
+		if (buf->dma_dir == DMA_FROM_DEVICE ||
+		    buf->dma_dir == DMA_BIDIRECTIONAL)
+			for (i = 0; i < frame_vector_count(buf->vec); i++)
+				set_page_dirty_lock(pages[i]);
 		sg_free_table(sgt);
 		kfree(sgt);
 	}
diff --git a/drivers/media/v4l2-core/videobuf2-dma-sg.c b/drivers/media/common/videobuf2/videobuf2-dma-sg.c
similarity index 99%
rename from drivers/media/v4l2-core/videobuf2-dma-sg.c
rename to drivers/media/common/videobuf2/videobuf2-dma-sg.c
index 6808231a6bdc..753ed3138dcc 100644
--- a/drivers/media/v4l2-core/videobuf2-dma-sg.c
+++ b/drivers/media/common/videobuf2/videobuf2-dma-sg.c
@@ -292,11 +292,10 @@ static void vb2_dma_sg_put_userptr(void *buf_priv)
 	if (buf->vaddr)
 		vm_unmap_ram(buf->vaddr, buf->num_pages);
 	sg_free_table(buf->dma_sgt);
-	while (--i >= 0) {
-		if (buf->dma_dir == DMA_FROM_DEVICE ||
-		    buf->dma_dir == DMA_BIDIRECTIONAL)
+	if (buf->dma_dir == DMA_FROM_DEVICE ||
+	    buf->dma_dir == DMA_BIDIRECTIONAL)
+		while (--i >= 0)
 			set_page_dirty_lock(buf->pages[i]);
-	}
 	vb2_destroy_framevec(buf->vec);
 	kfree(buf);
 }
diff --git a/drivers/media/v4l2-core/videobuf2-dvb.c b/drivers/media/common/videobuf2/videobuf2-dvb.c
similarity index 100%
rename from drivers/media/v4l2-core/videobuf2-dvb.c
rename to drivers/media/common/videobuf2/videobuf2-dvb.c
diff --git a/drivers/media/v4l2-core/videobuf2-memops.c b/drivers/media/common/videobuf2/videobuf2-memops.c
similarity index 100%
rename from drivers/media/v4l2-core/videobuf2-memops.c
rename to drivers/media/common/videobuf2/videobuf2-memops.c
diff --git a/drivers/media/v4l2-core/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c
similarity index 98%
rename from drivers/media/v4l2-core/videobuf2-v4l2.c
rename to drivers/media/common/videobuf2/videobuf2-v4l2.c
index 4075314a6989..886a2d8d5c6c 100644
--- a/drivers/media/v4l2-core/videobuf2-v4l2.c
+++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c
@@ -434,6 +434,8 @@ static int __fill_vb2_buffer(struct vb2_buffer *vb,
 	} else {
 		/* Zero any output buffer flags as this is a capture buffer */
 		vbuf->flags &= ~V4L2_BUFFER_OUT_FLAGS;
+		/* Zero last flag, this is a signal from driver to userspace */
+		vbuf->flags &= ~V4L2_BUF_FLAG_LAST;
 	}
 
 	return 0;
@@ -656,7 +658,7 @@ int vb2_queue_init(struct vb2_queue *q)
 			== V4L2_BUF_FLAG_TIMESTAMP_COPY;
 	/*
 	 * For compatibility with vb1: if QBUF hasn't been called yet, then
-	 * return POLLERR as well. This only affects capture queues, output
+	 * return EPOLLERR as well. This only affects capture queues, output
 	 * queues will always initialize waiting_for_buffers to false.
 	 */
 	q->quirk_poll_must_check_waiting_for_buffers = true;
@@ -671,18 +673,18 @@ void vb2_queue_release(struct vb2_queue *q)
 }
 EXPORT_SYMBOL_GPL(vb2_queue_release);
 
-unsigned int vb2_poll(struct vb2_queue *q, struct file *file, poll_table *wait)
+__poll_t vb2_poll(struct vb2_queue *q, struct file *file, poll_table *wait)
 {
 	struct video_device *vfd = video_devdata(file);
-	unsigned long req_events = poll_requested_events(wait);
-	unsigned int res = 0;
+	__poll_t req_events = poll_requested_events(wait);
+	__poll_t res = 0;
 
 	if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags)) {
 		struct v4l2_fh *fh = file->private_data;
 
 		if (v4l2_event_pending(fh))
-			res = POLLPRI;
-		else if (req_events & POLLPRI)
+			res = EPOLLPRI;
+		else if (req_events & EPOLLPRI)
 			poll_wait(file, &fh->wait, wait);
 	}
 
@@ -904,12 +906,12 @@ ssize_t vb2_fop_read(struct file *file, char __user *buf,
 }
 EXPORT_SYMBOL_GPL(vb2_fop_read);
 
-unsigned int vb2_fop_poll(struct file *file, poll_table *wait)
+__poll_t vb2_fop_poll(struct file *file, poll_table *wait)
 {
 	struct video_device *vdev = video_devdata(file);
 	struct vb2_queue *q = vdev->queue;
 	struct mutex *lock = q->lock ? q->lock : vdev->lock;
-	unsigned res;
+	__poll_t res;
 	void *fileio;
 
 	/*
@@ -919,7 +921,7 @@ unsigned int vb2_fop_poll(struct file *file, poll_table *wait)
 	WARN_ON(!lock);
 
 	if (lock && mutex_lock_interruptible(lock))
-		return POLLERR;
+		return EPOLLERR;
 
 	fileio = q->fileio;
 
diff --git a/drivers/media/v4l2-core/videobuf2-vmalloc.c b/drivers/media/common/videobuf2/videobuf2-vmalloc.c
similarity index 100%
rename from drivers/media/v4l2-core/videobuf2-vmalloc.c
rename to drivers/media/common/videobuf2/videobuf2-vmalloc.c
diff --git a/drivers/media/dvb-core/Kconfig b/drivers/media/dvb-core/Kconfig
index eeef94a0c84e..f004aea352e0 100644
--- a/drivers/media/dvb-core/Kconfig
+++ b/drivers/media/dvb-core/Kconfig
@@ -40,3 +40,16 @@ config DVB_DEMUX_SECTION_LOSS_LOG
 	  be very verbose.
 
 	  If you are unsure about this, say N here.
+
+config DVB_ULE_DEBUG
+	bool "Enable DVB net ULE packet debug messages"
+	depends on DVB_CORE
+	default n
+	help
+	  Enable extra log messages meant to detect problems while
+	  handling DVB network ULE packet loss inside the Kernel.
+
+	  Should not be enabled on normal cases, as logs can
+	  be very verbose.
+
+	  If you are unsure about this, say N here.
diff --git a/drivers/media/dvb-core/Makefile b/drivers/media/dvb-core/Makefile
index 47e2e391bfb8..62b028ded9f7 100644
--- a/drivers/media/dvb-core/Makefile
+++ b/drivers/media/dvb-core/Makefile
@@ -4,9 +4,10 @@
 #
 
 dvb-net-$(CONFIG_DVB_NET) := dvb_net.o
+dvb-vb2-$(CONFIG_DVB_MMAP) := dvb_vb2.o
 
-dvb-core-objs := dvbdev.o dmxdev.o dvb_demux.o		 	\
-		 dvb_ca_en50221.o dvb_frontend.o 		\
-		 $(dvb-net-y) dvb_ringbuffer.o dvb_math.o
+dvb-core-objs := dvbdev.o dmxdev.o dvb_demux.o			\
+		 dvb_ca_en50221.o dvb_frontend.o		\
+		 $(dvb-net-y) dvb_ringbuffer.o $(dvb-vb2-y) dvb_math.o
 
 obj-$(CONFIG_DVB_CORE) += dvb-core.o
diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c
index 3ddd44e1ee77..61a750fae465 100644
--- a/drivers/media/dvb-core/dmxdev.c
+++ b/drivers/media/dvb-core/dmxdev.c
@@ -27,7 +27,8 @@
 #include <linux/ioctl.h>
 #include <linux/wait.h>
 #include <linux/uaccess.h>
-#include "dmxdev.h"
+#include <media/dmxdev.h>
+#include <media/dvb_vb2.h>
 
 static int debug;
 
@@ -127,6 +128,7 @@ static int dvb_dvr_open(struct inode *inode, struct file *file)
 	struct dvb_device *dvbdev = file->private_data;
 	struct dmxdev *dmxdev = dvbdev->priv;
 	struct dmx_frontend *front;
+	bool need_ringbuffer = false;
 
 	dprintk("%s\n", __func__);
 
@@ -138,14 +140,33 @@ static int dvb_dvr_open(struct inode *inode, struct file *file)
 		return -ENODEV;
 	}
 
-	if ((file->f_flags & O_ACCMODE) == O_RDWR) {
+	dmxdev->may_do_mmap = 0;
+
+	/*
+	 * The logic here is a little tricky due to the ifdef.
+	 *
+	 * The ringbuffer is used for both read and mmap.
+	 *
+	 * It is not needed, however, on two situations:
+	 *	- Write devices (access with O_WRONLY);
+	 *	- For duplex device nodes, opened with O_RDWR.
+	 */
+
+	if ((file->f_flags & O_ACCMODE) == O_RDONLY)
+		need_ringbuffer = true;
+	else if ((file->f_flags & O_ACCMODE) == O_RDWR) {
 		if (!(dmxdev->capabilities & DMXDEV_CAP_DUPLEX)) {
+#ifdef CONFIG_DVB_MMAP
+			dmxdev->may_do_mmap = 1;
+			need_ringbuffer = true;
+#else
 			mutex_unlock(&dmxdev->mutex);
 			return -EOPNOTSUPP;
+#endif
 		}
 	}
 
-	if ((file->f_flags & O_ACCMODE) == O_RDONLY) {
+	if (need_ringbuffer) {
 		void *mem;
 
 		if (!dvbdev->readers) {
@@ -158,6 +179,9 @@ static int dvb_dvr_open(struct inode *inode, struct file *file)
 			return -ENOMEM;
 		}
 		dvb_ringbuffer_init(&dmxdev->dvr_buffer, mem, DVR_BUFFER_SIZE);
+		if (dmxdev->may_do_mmap)
+			dvb_vb2_init(&dmxdev->dvr_vb2_ctx, "dvr",
+				     file->f_flags & O_NONBLOCK);
 		dvbdev->readers--;
 	}
 
@@ -195,7 +219,14 @@ static int dvb_dvr_release(struct inode *inode, struct file *file)
 		dmxdev->demux->connect_frontend(dmxdev->demux,
 						dmxdev->dvr_orig_fe);
 	}
-	if ((file->f_flags & O_ACCMODE) == O_RDONLY) {
+
+	if (((file->f_flags & O_ACCMODE) == O_RDONLY) ||
+	    dmxdev->may_do_mmap) {
+		if (dmxdev->may_do_mmap) {
+			if (dvb_vb2_is_streaming(&dmxdev->dvr_vb2_ctx))
+				dvb_vb2_stream_off(&dmxdev->dvr_vb2_ctx);
+			dvb_vb2_release(&dmxdev->dvr_vb2_ctx);
+		}
 		dvbdev->readers++;
 		if (dmxdev->dvr_buffer.data) {
 			void *mem = dmxdev->dvr_buffer.data;
@@ -354,12 +385,14 @@ static void dvb_dmxdev_filter_timer(struct dmxdev_filter *dmxdevfilter)
 
 static int dvb_dmxdev_section_callback(const u8 *buffer1, size_t buffer1_len,
 				       const u8 *buffer2, size_t buffer2_len,
-				       struct dmx_section_filter *filter)
+				       struct dmx_section_filter *filter,
+				       u32 *buffer_flags)
 {
 	struct dmxdev_filter *dmxdevfilter = filter->priv;
 	int ret;
 
-	if (dmxdevfilter->buffer.error) {
+	if (!dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx) &&
+	    dmxdevfilter->buffer.error) {
 		wake_up(&dmxdevfilter->buffer.queue);
 		return 0;
 	}
@@ -370,11 +403,21 @@ static int dvb_dmxdev_section_callback(const u8 *buffer1, size_t buffer1_len,
 	}
 	del_timer(&dmxdevfilter->timer);
 	dprintk("section callback %*ph\n", 6, buffer1);
-	ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer, buffer1,
-				      buffer1_len);
-	if (ret == buffer1_len) {
-		ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer, buffer2,
-					      buffer2_len);
+	if (dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx)) {
+		ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx,
+					  buffer1, buffer1_len,
+					  buffer_flags);
+		if (ret == buffer1_len)
+			ret = dvb_vb2_fill_buffer(&dmxdevfilter->vb2_ctx,
+						  buffer2, buffer2_len,
+						  buffer_flags);
+	} else {
+		ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer,
+					      buffer1, buffer1_len);
+		if (ret == buffer1_len) {
+			ret = dvb_dmxdev_buffer_write(&dmxdevfilter->buffer,
+						      buffer2, buffer2_len);
+		}
 	}
 	if (ret < 0)
 		dmxdevfilter->buffer.error = ret;
@@ -387,10 +430,14 @@ static int dvb_dmxdev_section_callback(const u8 *buffer1, size_t buffer1_len,
 
 static int dvb_dmxdev_ts_callback(const u8 *buffer1, size_t buffer1_len,
 				  const u8 *buffer2, size_t buffer2_len,
-				  struct dmx_ts_feed *feed)
+				  struct dmx_ts_feed *feed,
+				  u32 *buffer_flags)
 {
 	struct dmxdev_filter *dmxdevfilter = feed->priv;
 	struct dvb_ringbuffer *buffer;
+#ifdef CONFIG_DVB_MMAP
+	struct dvb_vb2_ctx *ctx;
+#endif
 	int ret;
 
 	spin_lock(&dmxdevfilter->dev->lock);
@@ -399,19 +446,36 @@ static int dvb_dmxdev_ts_callback(const u8 *buffer1, size_t buffer1_len,
 		return 0;
 	}
 
-	if (dmxdevfilter->params.pes.output == DMX_OUT_TAP
-	    || dmxdevfilter->params.pes.output == DMX_OUT_TSDEMUX_TAP)
+	if (dmxdevfilter->params.pes.output == DMX_OUT_TAP ||
+	    dmxdevfilter->params.pes.output == DMX_OUT_TSDEMUX_TAP) {
 		buffer = &dmxdevfilter->buffer;
-	else
+#ifdef CONFIG_DVB_MMAP
+		ctx = &dmxdevfilter->vb2_ctx;
+#endif
+	} else {
 		buffer = &dmxdevfilter->dev->dvr_buffer;
-	if (buffer->error) {
-		spin_unlock(&dmxdevfilter->dev->lock);
-		wake_up(&buffer->queue);
-		return 0;
+#ifdef CONFIG_DVB_MMAP
+		ctx = &dmxdevfilter->dev->dvr_vb2_ctx;
+#endif
+	}
+
+	if (dvb_vb2_is_streaming(ctx)) {
+		ret = dvb_vb2_fill_buffer(ctx, buffer1, buffer1_len,
+					  buffer_flags);
+		if (ret == buffer1_len)
+			ret = dvb_vb2_fill_buffer(ctx, buffer2, buffer2_len,
+						  buffer_flags);
+	} else {
+		if (buffer->error) {
+			spin_unlock(&dmxdevfilter->dev->lock);
+			wake_up(&buffer->queue);
+			return 0;
+		}
+		ret = dvb_dmxdev_buffer_write(buffer, buffer1, buffer1_len);
+		if (ret == buffer1_len)
+			ret = dvb_dmxdev_buffer_write(buffer,
+						      buffer2, buffer2_len);
 	}
-	ret = dvb_dmxdev_buffer_write(buffer, buffer1, buffer1_len);
-	if (ret == buffer1_len)
-		ret = dvb_dmxdev_buffer_write(buffer, buffer2, buffer2_len);
 	if (ret < 0)
 		buffer->error = ret;
 	spin_unlock(&dmxdevfilter->dev->lock);
@@ -749,7 +813,15 @@ static int dvb_demux_open(struct inode *inode, struct file *file)
 	mutex_init(&dmxdevfilter->mutex);
 	file->private_data = dmxdevfilter;
 
+#ifdef CONFIG_DVB_MMAP
+	dmxdev->may_do_mmap = 1;
+#else
+	dmxdev->may_do_mmap = 0;
+#endif
+
 	dvb_ringbuffer_init(&dmxdevfilter->buffer, NULL, 8192);
+	dvb_vb2_init(&dmxdevfilter->vb2_ctx, "demux_filter",
+		     file->f_flags & O_NONBLOCK);
 	dmxdevfilter->type = DMXDEV_TYPE_NONE;
 	dvb_dmxdev_filter_state_set(dmxdevfilter, DMXDEV_STATE_ALLOCATED);
 	timer_setup(&dmxdevfilter->timer, dvb_dmxdev_filter_timeout, 0);
@@ -765,6 +837,10 @@ static int dvb_dmxdev_filter_free(struct dmxdev *dmxdev,
 {
 	mutex_lock(&dmxdev->mutex);
 	mutex_lock(&dmxdevfilter->mutex);
+	if (dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx))
+		dvb_vb2_stream_off(&dmxdevfilter->vb2_ctx);
+	dvb_vb2_release(&dmxdevfilter->vb2_ctx);
+
 
 	dvb_dmxdev_filter_stop(dmxdevfilter);
 	dvb_dmxdev_filter_reset(dmxdevfilter);
@@ -1052,8 +1128,56 @@ static int dvb_demux_do_ioctl(struct file *file,
 		mutex_unlock(&dmxdevfilter->mutex);
 		break;
 
+#ifdef CONFIG_DVB_MMAP
+	case DMX_REQBUFS:
+		if (mutex_lock_interruptible(&dmxdevfilter->mutex)) {
+			mutex_unlock(&dmxdev->mutex);
+			return -ERESTARTSYS;
+		}
+		ret = dvb_vb2_reqbufs(&dmxdevfilter->vb2_ctx, parg);
+		mutex_unlock(&dmxdevfilter->mutex);
+		break;
+
+	case DMX_QUERYBUF:
+		if (mutex_lock_interruptible(&dmxdevfilter->mutex)) {
+			mutex_unlock(&dmxdev->mutex);
+			return -ERESTARTSYS;
+		}
+		ret = dvb_vb2_querybuf(&dmxdevfilter->vb2_ctx, parg);
+		mutex_unlock(&dmxdevfilter->mutex);
+		break;
+
+	case DMX_EXPBUF:
+		if (mutex_lock_interruptible(&dmxdevfilter->mutex)) {
+			mutex_unlock(&dmxdev->mutex);
+			return -ERESTARTSYS;
+		}
+		ret = dvb_vb2_expbuf(&dmxdevfilter->vb2_ctx, parg);
+		mutex_unlock(&dmxdevfilter->mutex);
+		break;
+
+	case DMX_QBUF:
+		if (mutex_lock_interruptible(&dmxdevfilter->mutex)) {
+			mutex_unlock(&dmxdev->mutex);
+			return -ERESTARTSYS;
+		}
+		ret = dvb_vb2_qbuf(&dmxdevfilter->vb2_ctx, parg);
+		if (ret == 0 && !dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx))
+			ret = dvb_vb2_stream_on(&dmxdevfilter->vb2_ctx);
+		mutex_unlock(&dmxdevfilter->mutex);
+		break;
+
+	case DMX_DQBUF:
+		if (mutex_lock_interruptible(&dmxdevfilter->mutex)) {
+			mutex_unlock(&dmxdev->mutex);
+			return -ERESTARTSYS;
+		}
+		ret = dvb_vb2_dqbuf(&dmxdevfilter->vb2_ctx, parg);
+		mutex_unlock(&dmxdevfilter->mutex);
+		break;
+#endif
 	default:
-		ret = -EINVAL;
+		ret = -ENOTTY;
 		break;
 	}
 	mutex_unlock(&dmxdev->mutex);
@@ -1066,13 +1190,15 @@ static long dvb_demux_ioctl(struct file *file, unsigned int cmd,
 	return dvb_usercopy(file, cmd, arg, dvb_demux_do_ioctl);
 }
 
-static unsigned int dvb_demux_poll(struct file *file, poll_table *wait)
+static __poll_t dvb_demux_poll(struct file *file, poll_table *wait)
 {
 	struct dmxdev_filter *dmxdevfilter = file->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	if ((!dmxdevfilter) || dmxdevfilter->dev->exit)
-		return POLLERR;
+		return EPOLLERR;
+	if (dvb_vb2_is_streaming(&dmxdevfilter->vb2_ctx))
+		return dvb_vb2_poll(&dmxdevfilter->vb2_ctx, file, wait);
 
 	poll_wait(file, &dmxdevfilter->buffer.queue, wait);
 
@@ -1082,19 +1208,44 @@ static unsigned int dvb_demux_poll(struct file *file, poll_table *wait)
 		return 0;
 
 	if (dmxdevfilter->buffer.error)
-		mask |= (POLLIN | POLLRDNORM | POLLPRI | POLLERR);
+		mask |= (EPOLLIN | EPOLLRDNORM | EPOLLPRI | EPOLLERR);
 
 	if (!dvb_ringbuffer_empty(&dmxdevfilter->buffer))
-		mask |= (POLLIN | POLLRDNORM | POLLPRI);
+		mask |= (EPOLLIN | EPOLLRDNORM | EPOLLPRI);
 
 	return mask;
 }
 
-static int dvb_demux_release(struct inode *inode, struct file *file)
+#ifdef CONFIG_DVB_MMAP
+static int dvb_demux_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	struct dmxdev_filter *dmxdevfilter = file->private_data;
 	struct dmxdev *dmxdev = dmxdevfilter->dev;
+	int ret;
+
+	if (!dmxdev->may_do_mmap)
+		return -ENOTTY;
+
+	if (mutex_lock_interruptible(&dmxdev->mutex))
+		return -ERESTARTSYS;
+
+	if (mutex_lock_interruptible(&dmxdevfilter->mutex)) {
+		mutex_unlock(&dmxdev->mutex);
+		return -ERESTARTSYS;
+	}
+	ret = dvb_vb2_mmap(&dmxdevfilter->vb2_ctx, vma);
+
+	mutex_unlock(&dmxdevfilter->mutex);
+	mutex_unlock(&dmxdev->mutex);
 
+	return ret;
+}
+#endif
+
+static int dvb_demux_release(struct inode *inode, struct file *file)
+{
+	struct dmxdev_filter *dmxdevfilter = file->private_data;
+	struct dmxdev *dmxdev = dmxdevfilter->dev;
 	int ret;
 
 	ret = dvb_dmxdev_filter_free(dmxdev, dmxdevfilter);
@@ -1118,6 +1269,9 @@ static const struct file_operations dvb_demux_fops = {
 	.release = dvb_demux_release,
 	.poll = dvb_demux_poll,
 	.llseek = default_llseek,
+#ifdef CONFIG_DVB_MMAP
+	.mmap = dvb_demux_mmap,
+#endif
 };
 
 static const struct dvb_device dvbdev_demux = {
@@ -1146,8 +1300,31 @@ static int dvb_dvr_do_ioctl(struct file *file,
 		ret = dvb_dvr_set_buffer_size(dmxdev, arg);
 		break;
 
+#ifdef CONFIG_DVB_MMAP
+	case DMX_REQBUFS:
+		ret = dvb_vb2_reqbufs(&dmxdev->dvr_vb2_ctx, parg);
+		break;
+
+	case DMX_QUERYBUF:
+		ret = dvb_vb2_querybuf(&dmxdev->dvr_vb2_ctx, parg);
+		break;
+
+	case DMX_EXPBUF:
+		ret = dvb_vb2_expbuf(&dmxdev->dvr_vb2_ctx, parg);
+		break;
+
+	case DMX_QBUF:
+		ret = dvb_vb2_qbuf(&dmxdev->dvr_vb2_ctx, parg);
+		if (ret == 0 && !dvb_vb2_is_streaming(&dmxdev->dvr_vb2_ctx))
+			ret = dvb_vb2_stream_on(&dmxdev->dvr_vb2_ctx);
+		break;
+
+	case DMX_DQBUF:
+		ret = dvb_vb2_dqbuf(&dmxdev->dvr_vb2_ctx, parg);
+		break;
+#endif
 	default:
-		ret = -EINVAL;
+		ret = -ENOTTY;
 		break;
 	}
 	mutex_unlock(&dmxdev->mutex);
@@ -1160,31 +1337,56 @@ static long dvb_dvr_ioctl(struct file *file,
 	return dvb_usercopy(file, cmd, arg, dvb_dvr_do_ioctl);
 }
 
-static unsigned int dvb_dvr_poll(struct file *file, poll_table *wait)
+static __poll_t dvb_dvr_poll(struct file *file, poll_table *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dmxdev *dmxdev = dvbdev->priv;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	dprintk("%s\n", __func__);
 
 	if (dmxdev->exit)
-		return POLLERR;
+		return EPOLLERR;
+	if (dvb_vb2_is_streaming(&dmxdev->dvr_vb2_ctx))
+		return dvb_vb2_poll(&dmxdev->dvr_vb2_ctx, file, wait);
 
 	poll_wait(file, &dmxdev->dvr_buffer.queue, wait);
 
-	if ((file->f_flags & O_ACCMODE) == O_RDONLY) {
+	if (((file->f_flags & O_ACCMODE) == O_RDONLY) ||
+	    dmxdev->may_do_mmap) {
 		if (dmxdev->dvr_buffer.error)
-			mask |= (POLLIN | POLLRDNORM | POLLPRI | POLLERR);
+			mask |= (EPOLLIN | EPOLLRDNORM | EPOLLPRI | EPOLLERR);
 
 		if (!dvb_ringbuffer_empty(&dmxdev->dvr_buffer))
-			mask |= (POLLIN | POLLRDNORM | POLLPRI);
+			mask |= (EPOLLIN | EPOLLRDNORM | EPOLLPRI);
 	} else
-		mask |= (POLLOUT | POLLWRNORM | POLLPRI);
+		mask |= (EPOLLOUT | EPOLLWRNORM | EPOLLPRI);
 
 	return mask;
 }
 
+#ifdef CONFIG_DVB_MMAP
+static int dvb_dvr_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct dvb_device *dvbdev = file->private_data;
+	struct dmxdev *dmxdev = dvbdev->priv;
+	int ret;
+
+	if (!dmxdev->may_do_mmap)
+		return -ENOTTY;
+
+	if (dmxdev->exit)
+		return -ENODEV;
+
+	if (mutex_lock_interruptible(&dmxdev->mutex))
+		return -ERESTARTSYS;
+
+	ret = dvb_vb2_mmap(&dmxdev->dvr_vb2_ctx, vma);
+	mutex_unlock(&dmxdev->mutex);
+	return ret;
+}
+#endif
+
 static const struct file_operations dvb_dvr_fops = {
 	.owner = THIS_MODULE,
 	.read = dvb_dvr_read,
@@ -1194,6 +1396,9 @@ static const struct file_operations dvb_dvr_fops = {
 	.release = dvb_dvr_release,
 	.poll = dvb_dvr_poll,
 	.llseek = default_llseek,
+#ifdef CONFIG_DVB_MMAP
+	.mmap = dvb_dvr_mmap,
+#endif
 };
 
 static const struct dvb_device dvbdev_dvr = {
diff --git a/drivers/media/dvb-core/dvb_ca_en50221.c b/drivers/media/dvb-core/dvb_ca_en50221.c
index d48b61eb01f4..204d0f6c678d 100644
--- a/drivers/media/dvb-core/dvb_ca_en50221.c
+++ b/drivers/media/dvb-core/dvb_ca_en50221.c
@@ -37,8 +37,8 @@
 #include <linux/sched/signal.h>
 #include <linux/kthread.h>
 
-#include "dvb_ca_en50221.h"
-#include "dvb_ringbuffer.h"
+#include <media/dvb_ca_en50221.h>
+#include <media/dvb_ringbuffer.h>
 
 static int dvb_ca_en50221_debug;
 
@@ -786,7 +786,7 @@ static int dvb_ca_en50221_read_data(struct dvb_ca_private *ca, int slot,
  * @ca: CA instance.
  * @slot: Slot to write to.
  * @buf: The data in this buffer is treated as a complete link-level packet to
- * 	 be written.
+ *	 be written.
  * @bytes_write: Size of ebuf.
  *
  * return: Number of bytes written, or < 0 on error.
@@ -1473,6 +1473,9 @@ static ssize_t dvb_ca_en50221_io_write(struct file *file,
 		return -EFAULT;
 	buf += 2;
 	count -= 2;
+
+	if (slot >= ca->slot_count)
+		return -EINVAL;
 	sl = &ca->slot_info[slot];
 
 	/* check if the slot is actually running */
@@ -1782,18 +1785,18 @@ static int dvb_ca_en50221_io_release(struct inode *inode, struct file *file)
  *
  * return: Standard poll mask.
  */
-static unsigned int dvb_ca_en50221_io_poll(struct file *file, poll_table *wait)
+static __poll_t dvb_ca_en50221_io_poll(struct file *file, poll_table *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dvb_ca_private *ca = dvbdev->priv;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	int slot;
 	int result = 0;
 
 	dprintk("%s\n", __func__);
 
 	if (dvb_ca_en50221_io_read_condition(ca, &result, &slot) == 1)
-		mask |= POLLIN;
+		mask |= EPOLLIN;
 
 	/* if there is something, return now */
 	if (mask)
@@ -1803,7 +1806,7 @@ static unsigned int dvb_ca_en50221_io_poll(struct file *file, poll_table *wait)
 	poll_wait(file, &ca->wait_queue, wait);
 
 	if (dvb_ca_en50221_io_read_condition(ca, &result, &slot) == 1)
-		mask |= POLLIN;
+		mask |= EPOLLIN;
 
 	return mask;
 }
diff --git a/drivers/media/dvb-core/dvb_demux.c b/drivers/media/dvb-core/dvb_demux.c
index acade7543b82..f45091246bdc 100644
--- a/drivers/media/dvb-core/dvb_demux.c
+++ b/drivers/media/dvb-core/dvb_demux.c
@@ -30,7 +30,7 @@
 #include <linux/uaccess.h>
 #include <asm/div64.h>
 
-#include "dvb_demux.h"
+#include <media/dvb_demux.h>
 
 static int dvb_demux_tscheck;
 module_param(dvb_demux_tscheck, int, 0644);
@@ -55,6 +55,17 @@ MODULE_PARM_DESC(dvb_demux_feed_err_pkts,
 		dprintk(x);				\
 } while (0)
 
+#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
+#  define dprintk_sect_loss(x...) dprintk(x)
+#else
+#  define dprintk_sect_loss(x...)
+#endif
+
+#define set_buf_flags(__feed, __flag)			\
+	do {						\
+		(__feed)->buffer_flags |= (__flag);	\
+	} while (0)
+
 /******************************************************************************
  * static inlined helper functions
  ******************************************************************************/
@@ -104,30 +115,30 @@ static inline int dvb_dmx_swfilter_payload(struct dvb_demux_feed *feed,
 {
 	int count = payload(buf);
 	int p;
-#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
 	int ccok;
 	u8 cc;
-#endif
 
 	if (count == 0)
 		return -1;
 
 	p = 188 - count;
 
-#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
 	cc = buf[3] & 0x0f;
 	ccok = ((feed->cc + 1) & 0x0f) == cc;
 	feed->cc = cc;
-	if (!ccok)
-		dprintk("missed packet!\n");
-#endif
+	if (!ccok) {
+		set_buf_flags(feed, DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED);
+		dprintk_sect_loss("missed packet: %d instead of %d!\n",
+				  cc, (feed->cc + 1) & 0x0f);
+	}
 
 	if (buf[1] & 0x40)	// PUSI ?
 		feed->peslen = 0xfffa;
 
 	feed->peslen += count;
 
-	return feed->cb.ts(&buf[p], count, NULL, 0, &feed->feed.ts);
+	return feed->cb.ts(&buf[p], count, NULL, 0, &feed->feed.ts,
+			   &feed->buffer_flags);
 }
 
 static int dvb_dmx_swfilter_sectionfilter(struct dvb_demux_feed *feed,
@@ -149,7 +160,7 @@ static int dvb_dmx_swfilter_sectionfilter(struct dvb_demux_feed *feed,
 		return 0;
 
 	return feed->cb.sec(feed->feed.sec.secbuf, feed->feed.sec.seclen,
-			    NULL, 0, &f->filter);
+			    NULL, 0, &f->filter, &feed->buffer_flags);
 }
 
 static inline int dvb_dmx_swfilter_section_feed(struct dvb_demux_feed *feed)
@@ -168,8 +179,10 @@ static inline int dvb_dmx_swfilter_section_feed(struct dvb_demux_feed *feed)
 	if (sec->check_crc) {
 		section_syntax_indicator = ((sec->secbuf[1] & 0x80) != 0);
 		if (section_syntax_indicator &&
-		    demux->check_crc32(feed, sec->secbuf, sec->seclen))
+		    demux->check_crc32(feed, sec->secbuf, sec->seclen)) {
+			set_buf_flags(feed, DMX_BUFFER_FLAG_HAD_CRC32_DISCARD);
 			return -1;
+		}
 	}
 
 	do {
@@ -186,9 +199,8 @@ static void dvb_dmx_swfilter_section_new(struct dvb_demux_feed *feed)
 {
 	struct dmx_section_feed *sec = &feed->feed.sec;
 
-#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
 	if (sec->secbufp < sec->tsfeedp) {
-		int i, n = sec->tsfeedp - sec->secbufp;
+		int n = sec->tsfeedp - sec->secbufp;
 
 		/*
 		 * Section padding is done with 0xff bytes entirely.
@@ -196,15 +208,13 @@ static void dvb_dmx_swfilter_section_new(struct dvb_demux_feed *feed)
 		 * but just first and last.
 		 */
 		if (sec->secbuf[0] != 0xff || sec->secbuf[n - 1] != 0xff) {
-			dprintk("dvb_demux.c section ts padding loss: %d/%d\n",
-			       n, sec->tsfeedp);
-			dprintk("dvb_demux.c pad data:");
-			for (i = 0; i < n; i++)
-				pr_cont(" %02x", sec->secbuf[i]);
-			pr_cont("\n");
+			set_buf_flags(feed,
+				      DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED);
+			dprintk_sect_loss("section ts padding loss: %d/%d\n",
+					  n, sec->tsfeedp);
+			dprintk_sect_loss("pad data: %*ph\n", n, sec->secbuf);
 		}
 	}
-#endif
 
 	sec->tsfeedp = sec->secbufp = sec->seclen = 0;
 	sec->secbuf = sec->secbuf_base;
@@ -239,11 +249,10 @@ static int dvb_dmx_swfilter_section_copy_dump(struct dvb_demux_feed *feed,
 		return 0;
 
 	if (sec->tsfeedp + len > DMX_MAX_SECFEED_SIZE) {
-#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
-		dprintk("dvb_demux.c section buffer full loss: %d/%d\n",
-		       sec->tsfeedp + len - DMX_MAX_SECFEED_SIZE,
-		       DMX_MAX_SECFEED_SIZE);
-#endif
+		set_buf_flags(feed, DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED);
+		dprintk_sect_loss("section buffer full loss: %d/%d\n",
+				  sec->tsfeedp + len - DMX_MAX_SECFEED_SIZE,
+				  DMX_MAX_SECFEED_SIZE);
 		len = DMX_MAX_SECFEED_SIZE - sec->tsfeedp;
 	}
 
@@ -271,12 +280,13 @@ static int dvb_dmx_swfilter_section_copy_dump(struct dvb_demux_feed *feed,
 		sec->seclen = seclen;
 		sec->crc_val = ~0;
 		/* dump [secbuf .. secbuf+seclen) */
-		if (feed->pusi_seen)
+		if (feed->pusi_seen) {
 			dvb_dmx_swfilter_section_feed(feed);
-#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
-		else
-			dprintk("dvb_demux.c pusi not seen, discarding section data\n");
-#endif
+		} else {
+			set_buf_flags(feed,
+				      DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED);
+			dprintk_sect_loss("pusi not seen, discarding section data\n");
+		}
 		sec->secbufp += seclen;	/* secbufp and secbuf moving together is */
 		sec->secbuf += seclen;	/* redundant but saves pointer arithmetic */
 	}
@@ -309,18 +319,30 @@ static int dvb_dmx_swfilter_section_packet(struct dvb_demux_feed *feed,
 	}
 
 	if (!ccok || dc_i) {
-#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
-		dprintk("dvb_demux.c discontinuity detected %d bytes lost\n",
-			count);
+		if (dc_i) {
+			set_buf_flags(feed,
+				      DMX_BUFFER_FLAG_DISCONTINUITY_INDICATOR);
+			dprintk_sect_loss("%d frame with disconnect indicator\n",
+				cc);
+		} else {
+			set_buf_flags(feed,
+				      DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED);
+			dprintk_sect_loss("discontinuity: %d instead of %d. %d bytes lost\n",
+				cc, (feed->cc + 1) & 0x0f, count + 4);
+		}
 		/*
-		 * those bytes under sume circumstances will again be reported
+		 * those bytes under some circumstances will again be reported
 		 * in the following dvb_dmx_swfilter_section_new
 		 */
-#endif
+
 		/*
 		 * Discontinuity detected. Reset pusi_seen to
 		 * stop feeding of suspicious data until next PUSI=1 arrives
+		 *
+		 * FIXME: does it make sense if the MPEG-TS is the one
+		 *	reporting discontinuity?
 		 */
+
 		feed->pusi_seen = false;
 		dvb_dmx_swfilter_section_new(feed);
 	}
@@ -340,12 +362,11 @@ static int dvb_dmx_swfilter_section_packet(struct dvb_demux_feed *feed,
 			dvb_dmx_swfilter_section_new(feed);
 			dvb_dmx_swfilter_section_copy_dump(feed, after,
 							   after_len);
+		} else if (count > 0) {
+			set_buf_flags(feed,
+				      DMX_BUFFER_FLAG_DISCONTINUITY_DETECTED);
+			dprintk_sect_loss("PUSI=1 but %d bytes lost\n", count);
 		}
-#ifdef CONFIG_DVB_DEMUX_SECTION_LOSS_LOG
-		else if (count > 0)
-			dprintk("dvb_demux.c PUSI=1 but %d bytes lost\n",
-				count);
-#endif
 	} else {
 		/* PUSI=0 (is not set), no section boundary */
 		dvb_dmx_swfilter_section_copy_dump(feed, &buf[p], count);
@@ -365,7 +386,8 @@ static inline void dvb_dmx_swfilter_packet_type(struct dvb_demux_feed *feed,
 			if (feed->ts_type & TS_PAYLOAD_ONLY)
 				dvb_dmx_swfilter_payload(feed, buf);
 			else
-				feed->cb.ts(buf, 188, NULL, 0, &feed->feed.ts);
+				feed->cb.ts(buf, 188, NULL, 0, &feed->feed.ts,
+					    &feed->buffer_flags);
 		}
 		/* Used only on full-featured devices */
 		if (feed->ts_type & TS_DECODER)
@@ -414,9 +436,10 @@ static void dvb_dmx_swfilter_packet(struct dvb_demux *demux, const u8 *buf)
 						1024);
 				speed_timedelta = ktime_ms_delta(cur_time,
 							demux->speed_last_time);
-				dprintk("TS speed %llu Kbits/sec \n",
-					div64_u64(speed_bytes,
-						  speed_timedelta));
+				if (speed_timedelta)
+					dprintk("TS speed %llu Kbits/sec \n",
+						div64_u64(speed_bytes,
+							  speed_timedelta));
 			}
 
 			demux->speed_last_time = cur_time;
@@ -425,6 +448,11 @@ static void dvb_dmx_swfilter_packet(struct dvb_demux *demux, const u8 *buf)
 	}
 
 	if (buf[1] & 0x80) {
+		list_for_each_entry(feed, &demux->feed_list, list_head) {
+			if ((feed->pid != pid) && (feed->pid != 0x2000))
+				continue;
+			set_buf_flags(feed, DMX_BUFFER_FLAG_TEI);
+		}
 		dprintk_tscheck("TEI detected. PID=0x%x data1=0x%x\n",
 				pid, buf[1]);
 		/* data in this packet can't be trusted - drop it unless
@@ -440,9 +468,16 @@ static void dvb_dmx_swfilter_packet(struct dvb_demux *demux, const u8 *buf)
 						(demux->cnt_storage[pid] + 1) & 0xf;
 
 				if ((buf[3] & 0xf) != demux->cnt_storage[pid]) {
+					list_for_each_entry(feed, &demux->feed_list, list_head) {
+						if ((feed->pid != pid) && (feed->pid != 0x2000))
+							continue;
+						set_buf_flags(feed,
+							      DMX_BUFFER_PKT_COUNTER_MISMATCH);
+					}
+
 					dprintk_tscheck("TS packet counter mismatch. PID=0x%x expected 0x%x got 0x%x\n",
-						pid, demux->cnt_storage[pid],
-						buf[3] & 0xf);
+							pid, demux->cnt_storage[pid],
+							buf[3] & 0xf);
 					demux->cnt_storage[pid] = buf[3] & 0xf;
 				}
 			}
@@ -461,7 +496,8 @@ static void dvb_dmx_swfilter_packet(struct dvb_demux *demux, const u8 *buf)
 		if (feed->pid == pid)
 			dvb_dmx_swfilter_packet_type(feed, buf);
 		else if (feed->pid == 0x2000)
-			feed->cb.ts(buf, 188, NULL, 0, &feed->feed.ts);
+			feed->cb.ts(buf, 188, NULL, 0, &feed->feed.ts,
+				    &feed->buffer_flags);
 	}
 }
 
@@ -580,7 +616,8 @@ void dvb_dmx_swfilter_raw(struct dvb_demux *demux, const u8 *buf, size_t count)
 
 	spin_lock_irqsave(&demux->lock, flags);
 
-	demux->feed->cb.ts(buf, count, NULL, 0, &demux->feed->feed.ts);
+	demux->feed->cb.ts(buf, count, NULL, 0, &demux->feed->feed.ts,
+			   &demux->feed->buffer_flags);
 
 	spin_unlock_irqrestore(&demux->lock, flags);
 }
@@ -780,6 +817,7 @@ static int dvbdmx_allocate_ts_feed(struct dmx_demux *dmx,
 	feed->demux = demux;
 	feed->pid = 0xffff;
 	feed->peslen = 0xfffa;
+	feed->buffer_flags = 0;
 
 	(*ts_feed) = &feed->feed.ts;
 	(*ts_feed)->parent = dmx;
@@ -1037,6 +1075,7 @@ static int dvbdmx_allocate_section_feed(struct dmx_demux *demux,
 	dvbdmxfeed->cb.sec = callback;
 	dvbdmxfeed->demux = dvbdmx;
 	dvbdmxfeed->pid = 0xffff;
+	dvbdmxfeed->buffer_flags = 0;
 	dvbdmxfeed->feed.sec.secbuf = dvbdmxfeed->feed.sec.secbuf_base;
 	dvbdmxfeed->feed.sec.secbufp = dvbdmxfeed->feed.sec.seclen = 0;
 	dvbdmxfeed->feed.sec.tsfeedp = 0;
diff --git a/drivers/media/dvb-core/dvb_frontend.c b/drivers/media/dvb-core/dvb_frontend.c
index 2afaa8226342..a7ed16e0841d 100644
--- a/drivers/media/dvb-core/dvb_frontend.c
+++ b/drivers/media/dvb-core/dvb_frontend.c
@@ -40,10 +40,11 @@
 #include <linux/jiffies.h>
 #include <linux/kthread.h>
 #include <linux/ktime.h>
+#include <linux/compat.h>
 #include <asm/processor.h>
 
-#include "dvb_frontend.h"
-#include "dvbdev.h"
+#include <media/dvb_frontend.h>
+#include <media/dvbdev.h>
 #include <linux/dvb/version.h>
 
 static int dvb_frontend_debug;
@@ -150,8 +151,7 @@ static void __dvb_frontend_free(struct dvb_frontend *fe)
 
 	dvb_frontend_invoke_release(fe, fe->ops.release);
 
-	if (fepriv)
-		kfree(fepriv);
+	kfree(fepriv);
 }
 
 static void dvb_frontend_free(struct kref *ref)
@@ -982,6 +982,7 @@ static int dvb_frontend_clear_cache(struct dvb_frontend *fe)
 	}
 
 	c->stream_id = NO_STREAM_ID_FILTER;
+	c->scrambling_sequence_index = 0;/* default sequence */
 
 	switch (c->delivery_system) {
 	case SYS_DVBS:
@@ -1072,6 +1073,7 @@ static struct dtv_cmds_h dtv_cmds[DTV_MAX_COMMAND + 1] = {
 
 	_DTV_CMD(DTV_STREAM_ID, 1, 0),
 	_DTV_CMD(DTV_DVBT2_PLP_ID_LEGACY, 1, 0),
+	_DTV_CMD(DTV_SCRAMBLING_SEQUENCE_INDEX, 1, 0),
 	_DTV_CMD(DTV_LNA, 1, 0),
 
 	/* Get */
@@ -1417,6 +1419,11 @@ static int dtv_property_process_get(struct dvb_frontend *fe,
 		tvp->u.data = c->stream_id;
 		break;
 
+	/* Physical layer scrambling support */
+	case DTV_SCRAMBLING_SEQUENCE_INDEX:
+		tvp->u.data = c->scrambling_sequence_index;
+		break;
+
 	/* ATSC-MH */
 	case DTV_ATSCMH_FIC_VER:
 		tvp->u.data = fe->dtv_property_cache.atscmh_fic_ver;
@@ -1900,6 +1907,11 @@ static int dtv_property_process_set(struct dvb_frontend *fe,
 		c->stream_id = data;
 		break;
 
+	/* Physical layer scrambling support */
+	case DTV_SCRAMBLING_SEQUENCE_INDEX:
+		c->scrambling_sequence_index = data;
+		break;
+
 	/* ATSC-MH */
 	case DTV_ATSCMH_PARADE_ID:
 		fe->dtv_property_cache.atscmh_parade_id = data;
@@ -1923,7 +1935,8 @@ static int dtv_property_process_set(struct dvb_frontend *fe,
 	return r;
 }
 
-static int dvb_frontend_ioctl(struct file *file, unsigned int cmd, void *parg)
+static int dvb_frontend_do_ioctl(struct file *file, unsigned int cmd,
+				 void *parg)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dvb_frontend *fe = dvbdev->priv;
@@ -1966,6 +1979,156 @@ static int dvb_frontend_ioctl(struct file *file, unsigned int cmd, void *parg)
 	return err;
 }
 
+static long dvb_frontend_ioctl(struct file *file, unsigned int cmd,
+			       unsigned long arg)
+{
+	struct dvb_device *dvbdev = file->private_data;
+
+	if (!dvbdev)
+		return -ENODEV;
+
+	return dvb_usercopy(file, cmd, arg, dvb_frontend_do_ioctl);
+}
+
+#ifdef CONFIG_COMPAT
+struct compat_dtv_property {
+	__u32 cmd;
+	__u32 reserved[3];
+	union {
+		__u32 data;
+		struct dtv_fe_stats st;
+		struct {
+			__u8 data[32];
+			__u32 len;
+			__u32 reserved1[3];
+			compat_uptr_t reserved2;
+		} buffer;
+	} u;
+	int result;
+} __attribute__ ((packed));
+
+struct compat_dtv_properties {
+	__u32 num;
+	compat_uptr_t props;
+};
+
+#define COMPAT_FE_SET_PROPERTY	   _IOW('o', 82, struct compat_dtv_properties)
+#define COMPAT_FE_GET_PROPERTY	   _IOR('o', 83, struct compat_dtv_properties)
+
+static int dvb_frontend_handle_compat_ioctl(struct file *file, unsigned int cmd,
+					    unsigned long arg)
+{
+	struct dvb_device *dvbdev = file->private_data;
+	struct dvb_frontend *fe = dvbdev->priv;
+	struct dvb_frontend_private *fepriv = fe->frontend_priv;
+	int i, err = 0;
+
+	if (cmd == COMPAT_FE_SET_PROPERTY) {
+		struct compat_dtv_properties prop, *tvps = NULL;
+		struct compat_dtv_property *tvp = NULL;
+
+		if (copy_from_user(&prop, compat_ptr(arg), sizeof(prop)))
+			return -EFAULT;
+
+		tvps = &prop;
+
+		/*
+		 * Put an arbitrary limit on the number of messages that can
+		 * be sent at once
+		 */
+		if (!tvps->num || (tvps->num > DTV_IOCTL_MAX_MSGS))
+			return -EINVAL;
+
+		tvp = memdup_user(compat_ptr(tvps->props), tvps->num * sizeof(*tvp));
+		if (IS_ERR(tvp))
+			return PTR_ERR(tvp);
+
+		for (i = 0; i < tvps->num; i++) {
+			err = dtv_property_process_set(fe, file,
+							(tvp + i)->cmd,
+							(tvp + i)->u.data);
+			if (err < 0) {
+				kfree(tvp);
+				return err;
+			}
+		}
+		kfree(tvp);
+	} else if (cmd == COMPAT_FE_GET_PROPERTY) {
+		struct compat_dtv_properties prop, *tvps = NULL;
+		struct compat_dtv_property *tvp = NULL;
+		struct dtv_frontend_properties getp = fe->dtv_property_cache;
+
+		if (copy_from_user(&prop, compat_ptr(arg), sizeof(prop)))
+			return -EFAULT;
+
+		tvps = &prop;
+
+		/*
+		 * Put an arbitrary limit on the number of messages that can
+		 * be sent at once
+		 */
+		if (!tvps->num || (tvps->num > DTV_IOCTL_MAX_MSGS))
+			return -EINVAL;
+
+		tvp = memdup_user(compat_ptr(tvps->props), tvps->num * sizeof(*tvp));
+		if (IS_ERR(tvp))
+			return PTR_ERR(tvp);
+
+		/*
+		 * Let's use our own copy of property cache, in order to
+		 * avoid mangling with DTV zigzag logic, as drivers might
+		 * return crap, if they don't check if the data is available
+		 * before updating the properties cache.
+		 */
+		if (fepriv->state != FESTATE_IDLE) {
+			err = dtv_get_frontend(fe, &getp, NULL);
+			if (err < 0) {
+				kfree(tvp);
+				return err;
+			}
+		}
+		for (i = 0; i < tvps->num; i++) {
+			err = dtv_property_process_get(
+			    fe, &getp, (struct dtv_property *)tvp + i, file);
+			if (err < 0) {
+				kfree(tvp);
+				return err;
+			}
+		}
+
+		if (copy_to_user((void __user *)compat_ptr(tvps->props), tvp,
+				 tvps->num * sizeof(struct compat_dtv_property))) {
+			kfree(tvp);
+			return -EFAULT;
+		}
+		kfree(tvp);
+	}
+
+	return err;
+}
+
+static long dvb_frontend_compat_ioctl(struct file *file, unsigned int cmd,
+				      unsigned long arg)
+{
+	struct dvb_device *dvbdev = file->private_data;
+	struct dvb_frontend *fe = dvbdev->priv;
+	struct dvb_frontend_private *fepriv = fe->frontend_priv;
+	int err;
+
+	if (cmd == COMPAT_FE_SET_PROPERTY || cmd == COMPAT_FE_GET_PROPERTY) {
+		if (down_interruptible(&fepriv->sem))
+			return -ERESTARTSYS;
+
+		err = dvb_frontend_handle_compat_ioctl(file, cmd, arg);
+
+		up(&fepriv->sem);
+		return err;
+	}
+
+	return dvb_frontend_ioctl(file, cmd, (unsigned long)compat_ptr(arg));
+}
+#endif
+
 static int dtv_set_frontend(struct dvb_frontend *fe)
 {
 	struct dvb_frontend_private *fepriv = fe->frontend_priv;
@@ -2110,7 +2273,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 	struct dvb_frontend *fe = dvbdev->priv;
 	struct dvb_frontend_private *fepriv = fe->frontend_priv;
 	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
-	int i, err;
+	int i, err = -ENOTSUPP;
 
 	dev_dbg(fe->dvb->device, "%s:\n", __func__);
 
@@ -2145,6 +2308,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 			}
 		}
 		kfree(tvp);
+		err = 0;
 		break;
 	}
 	case FE_GET_PROPERTY: {
@@ -2196,6 +2360,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 			return -EFAULT;
 		}
 		kfree(tvp);
+		err = 0;
 		break;
 	}
 
@@ -2470,7 +2635,7 @@ static int dvb_frontend_handle_ioctl(struct file *file,
 }
 
 
-static unsigned int dvb_frontend_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t dvb_frontend_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct dvb_frontend *fe = dvbdev->priv;
@@ -2481,7 +2646,7 @@ static unsigned int dvb_frontend_poll(struct file *file, struct poll_table_struc
 	poll_wait (file, &fepriv->events.wait_queue, wait);
 
 	if (fepriv->events.eventw != fepriv->events.eventr)
-		return (POLLIN | POLLRDNORM | POLLPRI);
+		return (EPOLLIN | EPOLLRDNORM | EPOLLPRI);
 
 	return 0;
 }
@@ -2647,7 +2812,10 @@ static int dvb_frontend_release(struct inode *inode, struct file *file)
 
 static const struct file_operations dvb_frontend_fops = {
 	.owner		= THIS_MODULE,
-	.unlocked_ioctl	= dvb_generic_ioctl,
+	.unlocked_ioctl	= dvb_frontend_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= dvb_frontend_compat_ioctl,
+#endif
 	.poll		= dvb_frontend_poll,
 	.open		= dvb_frontend_open,
 	.release	= dvb_frontend_release,
@@ -2715,7 +2883,6 @@ int dvb_register_frontend(struct dvb_adapter* dvb,
 #if defined(CONFIG_MEDIA_CONTROLLER_DVB)
 		.name = fe->ops.info.name,
 #endif
-		.kernel_ioctl = dvb_frontend_ioctl
 	};
 
 	dev_dbg(dvb->device, "%s:\n", __func__);
diff --git a/drivers/media/dvb-core/dvb_math.c b/drivers/media/dvb-core/dvb_math.c
index a2e1810dd83a..dc90564d7f34 100644
--- a/drivers/media/dvb-core/dvb_math.c
+++ b/drivers/media/dvb-core/dvb_math.c
@@ -19,7 +19,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <asm/bug.h>
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 
 static const unsigned short logtable[256] = {
 	0x0000, 0x0171, 0x02e0, 0x044e, 0x05ba, 0x0725, 0x088e, 0x09f7,
diff --git a/drivers/media/dvb-core/dvb_net.c b/drivers/media/dvb-core/dvb_net.c
index c018e3c06d5d..ba39f9942e1d 100644
--- a/drivers/media/dvb-core/dvb_net.c
+++ b/drivers/media/dvb-core/dvb_net.c
@@ -38,7 +38,7 @@
  *                       Competence Center for Advanced Satellite Communications.
  *                     Bugfixes and robustness improvements.
  *                     Filtering on dest MAC addresses, if present (D-Bit = 0)
- *                     ULE_DEBUG compile-time option.
+ *                     DVB_ULE_DEBUG compile-time option.
  * Apr 2006: cp v3:    Bugfixes and compliency with RFC 4326 (ULE) by
  *                       Christian Praehauser <cpraehaus@cosy.sbg.ac.at>,
  *                       Paris Lodron University of Salzburg.
@@ -64,8 +64,8 @@
 #include <linux/mutex.h>
 #include <linux/sched.h>
 
-#include "dvb_demux.h"
-#include "dvb_net.h"
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
 
 static inline __u32 iov_crc32( __u32 c, struct kvec *iov, unsigned int cnt )
 {
@@ -78,15 +78,18 @@ static inline __u32 iov_crc32( __u32 c, struct kvec *iov, unsigned int cnt )
 
 #define DVB_NET_MULTICAST_MAX 10
 
-#undef ULE_DEBUG
-
-#ifdef ULE_DEBUG
+#ifdef DVB_ULE_DEBUG
+/*
+ * The code inside DVB_ULE_DEBUG keeps a history of the
+ * last 100 TS cells processed.
+ */
+static unsigned char ule_hist[100*TS_SZ] = { 0 };
+static unsigned char *ule_where = ule_hist, ule_dump;
 
 static void hexdump(const unsigned char *buf, unsigned short len)
 {
 	print_hex_dump_debug("", DUMP_PREFIX_OFFSET, 16, 1, buf, len, true);
 }
-
 #endif
 
 struct dvb_net_priv {
@@ -280,11 +283,9 @@ static int handle_ule_extensions( struct dvb_net_priv *p )
 		if (l < 0)
 			return l;	/* Stop extension header processing and discard SNDU. */
 		total_ext_len += l;
-#ifdef ULE_DEBUG
 		pr_debug("ule_next_hdr=%p, ule_sndu_type=%i, l=%i, total_ext_len=%i\n",
 			 p->ule_next_hdr, (int)p->ule_sndu_type,
 			 l, total_ext_len);
-#endif
 
 	} while (p->ule_sndu_type < ETH_P_802_3_MIN);
 
@@ -320,29 +321,21 @@ struct dvb_net_ule_handle {
 	const u8 *ts, *ts_end, *from_where;
 	u8 ts_remain, how_much, new_ts;
 	bool error;
-#ifdef ULE_DEBUG
-	/*
-	 * The code inside ULE_DEBUG keeps a history of the
-	 * last 100 TS cells processed.
-	 */
-	static unsigned char ule_hist[100*TS_SZ];
-	static unsigned char *ule_where = ule_hist, ule_dump;
-#endif
 };
 
 static int dvb_net_ule_new_ts_cell(struct dvb_net_ule_handle *h)
 {
 	/* We are about to process a new TS cell. */
 
-#ifdef ULE_DEBUG
-	if (h->ule_where >= &h->ule_hist[100*TS_SZ])
-		h->ule_where = h->ule_hist;
-	memcpy(h->ule_where, h->ts, TS_SZ);
-	if (h->ule_dump) {
-		hexdump(h->ule_where, TS_SZ);
-		h->ule_dump = 0;
+#ifdef DVB_ULE_DEBUG
+	if (ule_where >= &ule_hist[100*TS_SZ])
+		ule_where = ule_hist;
+	memcpy(ule_where, h->ts, TS_SZ);
+	if (ule_dump) {
+		hexdump(ule_where, TS_SZ);
+		ule_dump = 0;
 	}
-	h->ule_where += TS_SZ;
+	ule_where += TS_SZ;
 #endif
 
 	/*
@@ -660,6 +653,7 @@ static int dvb_net_ule_should_drop(struct dvb_net_ule_handle *h)
 
 
 static void dvb_net_ule_check_crc(struct dvb_net_ule_handle *h,
+				  struct kvec iov[3],
 				  u32 ule_crc, u32 expected_crc)
 {
 	u8 dest_addr[ETH_ALEN];
@@ -672,22 +666,22 @@ static void dvb_net_ule_check_crc(struct dvb_net_ule_handle *h,
 			h->ts_remain > 2 ?
 				*(unsigned short *)h->from_where : 0);
 
-	#ifdef ULE_DEBUG
+	#ifdef DVB_ULE_DEBUG
 		hexdump(iov[0].iov_base, iov[0].iov_len);
 		hexdump(iov[1].iov_base, iov[1].iov_len);
 		hexdump(iov[2].iov_base, iov[2].iov_len);
 
-		if (h->ule_where == h->ule_hist) {
-			hexdump(&h->ule_hist[98*TS_SZ], TS_SZ);
-			hexdump(&h->ule_hist[99*TS_SZ], TS_SZ);
-		} else if (h->ule_where == &h->ule_hist[TS_SZ]) {
-			hexdump(&h->ule_hist[99*TS_SZ], TS_SZ);
-			hexdump(h->ule_hist, TS_SZ);
+		if (ule_where == ule_hist) {
+			hexdump(&ule_hist[98*TS_SZ], TS_SZ);
+			hexdump(&ule_hist[99*TS_SZ], TS_SZ);
+		} else if (ule_where == &ule_hist[TS_SZ]) {
+			hexdump(&ule_hist[99*TS_SZ], TS_SZ);
+			hexdump(ule_hist, TS_SZ);
 		} else {
-			hexdump(h->ule_where - TS_SZ - TS_SZ, TS_SZ);
-			hexdump(h->ule_where - TS_SZ, TS_SZ);
+			hexdump(ule_where - TS_SZ - TS_SZ, TS_SZ);
+			hexdump(ule_where - TS_SZ, TS_SZ);
 		}
-		h->ule_dump = 1;
+		ule_dump = 1;
 	#endif
 
 		h->dev->stats.rx_errors++;
@@ -705,11 +699,9 @@ static void dvb_net_ule_check_crc(struct dvb_net_ule_handle *h,
 
 	if (!h->priv->ule_dbit) {
 		if (dvb_net_ule_should_drop(h)) {
-#ifdef ULE_DEBUG
 			netdev_dbg(h->dev,
 				   "Dropping SNDU: MAC destination address does not match: dest addr: %pM, h->dev addr: %pM\n",
 				   h->priv->ule_skb->data, h->dev->dev_addr);
-#endif
 			dev_kfree_skb(h->priv->ule_skb);
 			return;
 		}
@@ -779,6 +771,8 @@ static void dvb_net_ule(struct net_device *dev, const u8 *buf, size_t buf_len)
 	int ret;
 	struct dvb_net_ule_handle h = {
 		.dev = dev,
+		.priv = netdev_priv(dev),
+		.ethh = NULL,
 		.buf = buf,
 		.buf_len = buf_len,
 		.skipped = 0L,
@@ -788,11 +782,7 @@ static void dvb_net_ule(struct net_device *dev, const u8 *buf, size_t buf_len)
 		.ts_remain = 0,
 		.how_much = 0,
 		.new_ts = 1,
-		.ethh = NULL,
 		.error = false,
-#ifdef ULE_DEBUG
-		.ule_where = ule_hist,
-#endif
 	};
 
 	/*
@@ -860,7 +850,7 @@ static void dvb_net_ule(struct net_device *dev, const u8 *buf, size_t buf_len)
 				       *(tail - 2) << 8 |
 				       *(tail - 1);
 
-			dvb_net_ule_check_crc(&h, ule_crc, expected_crc);
+			dvb_net_ule_check_crc(&h, iov, ule_crc, expected_crc);
 
 			/* Prepare for next SNDU. */
 			reset_ule(h.priv);
@@ -893,7 +883,8 @@ static void dvb_net_ule(struct net_device *dev, const u8 *buf, size_t buf_len)
 
 static int dvb_net_ts_callback(const u8 *buffer1, size_t buffer1_len,
 			       const u8 *buffer2, size_t buffer2_len,
-			       struct dmx_ts_feed *feed)
+			       struct dmx_ts_feed *feed,
+			       u32 *buffer_flags)
 {
 	struct net_device *dev = feed->priv;
 
@@ -1002,7 +993,7 @@ static void dvb_net_sec(struct net_device *dev,
 
 static int dvb_net_sec_callback(const u8 *buffer1, size_t buffer1_len,
 		 const u8 *buffer2, size_t buffer2_len,
-		 struct dmx_section_filter *filter)
+		 struct dmx_section_filter *filter, u32 *buffer_flags)
 {
 	struct net_device *dev = filter->priv;
 
diff --git a/drivers/media/dvb-core/dvb_ringbuffer.c b/drivers/media/dvb-core/dvb_ringbuffer.c
index 53011629c9ad..4330b6fa4af2 100644
--- a/drivers/media/dvb-core/dvb_ringbuffer.c
+++ b/drivers/media/dvb-core/dvb_ringbuffer.c
@@ -29,7 +29,7 @@
 #include <linux/string.h>
 #include <linux/uaccess.h>
 
-#include "dvb_ringbuffer.h"
+#include <media/dvb_ringbuffer.h>
 
 #define PKT_READY 0
 #define PKT_DISPOSED 1
diff --git a/drivers/media/dvb-core/dvb_vb2.c b/drivers/media/dvb-core/dvb_vb2.c
new file mode 100644
index 000000000000..b811adf88afa
--- /dev/null
+++ b/drivers/media/dvb-core/dvb_vb2.c
@@ -0,0 +1,443 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * dvb-vb2.c - dvb-vb2
+ *
+ * Copyright (C) 2015 Samsung Electronics
+ *
+ * Author: jh1009.sung@samsung.com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+
+#include <media/dvbdev.h>
+#include <media/dvb_vb2.h>
+
+#define DVB_V2_MAX_SIZE		(4096 * 188)
+
+static int vb2_debug;
+module_param(vb2_debug, int, 0644);
+
+#define dprintk(level, fmt, arg...)					      \
+	do {								      \
+		if (vb2_debug >= level)					      \
+			pr_info("vb2: %s: " fmt, __func__, ## arg); \
+	} while (0)
+
+static int _queue_setup(struct vb2_queue *vq,
+			unsigned int *nbuffers, unsigned int *nplanes,
+			unsigned int sizes[], struct device *alloc_devs[])
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vq);
+
+	ctx->buf_cnt = *nbuffers;
+	*nplanes = 1;
+	sizes[0] = ctx->buf_siz;
+
+	/*
+	 * videobuf2-vmalloc allocator is context-less so no need to set
+	 * alloc_ctxs array.
+	 */
+
+	dprintk(3, "[%s] count=%d, size=%d\n", ctx->name,
+		*nbuffers, sizes[0]);
+
+	return 0;
+}
+
+static int _buffer_prepare(struct vb2_buffer *vb)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	unsigned long size = ctx->buf_siz;
+
+	if (vb2_plane_size(vb, 0) < size) {
+		dprintk(1, "[%s] data will not fit into plane (%lu < %lu)\n",
+			ctx->name, vb2_plane_size(vb, 0), size);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(vb, 0, size);
+	dprintk(3, "[%s]\n", ctx->name);
+
+	return 0;
+}
+
+static void _buffer_queue(struct vb2_buffer *vb)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	struct dvb_buffer *buf = container_of(vb, struct dvb_buffer, vb);
+	unsigned long flags = 0;
+
+	spin_lock_irqsave(&ctx->slock, flags);
+	list_add_tail(&buf->list, &ctx->dvb_q);
+	spin_unlock_irqrestore(&ctx->slock, flags);
+
+	dprintk(3, "[%s]\n", ctx->name);
+}
+
+static int _start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vq);
+
+	dprintk(3, "[%s] count=%d\n", ctx->name, count);
+	return 0;
+}
+
+static void _stop_streaming(struct vb2_queue *vq)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vq);
+	struct dvb_buffer *buf;
+	unsigned long flags = 0;
+
+	dprintk(3, "[%s]\n", ctx->name);
+
+	spin_lock_irqsave(&ctx->slock, flags);
+	while (!list_empty(&ctx->dvb_q)) {
+		buf = list_entry(ctx->dvb_q.next,
+				 struct dvb_buffer, list);
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_ERROR);
+		list_del(&buf->list);
+	}
+	spin_unlock_irqrestore(&ctx->slock, flags);
+}
+
+static void _dmxdev_lock(struct vb2_queue *vq)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vq);
+
+	mutex_lock(&ctx->mutex);
+	dprintk(3, "[%s]\n", ctx->name);
+}
+
+static void _dmxdev_unlock(struct vb2_queue *vq)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vq);
+
+	if (mutex_is_locked(&ctx->mutex))
+		mutex_unlock(&ctx->mutex);
+	dprintk(3, "[%s]\n", ctx->name);
+}
+
+static const struct vb2_ops dvb_vb2_qops = {
+	.queue_setup		= _queue_setup,
+	.buf_prepare		= _buffer_prepare,
+	.buf_queue		= _buffer_queue,
+	.start_streaming	= _start_streaming,
+	.stop_streaming		= _stop_streaming,
+	.wait_prepare		= _dmxdev_unlock,
+	.wait_finish		= _dmxdev_lock,
+};
+
+static void _fill_dmx_buffer(struct vb2_buffer *vb, void *pb)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	struct dmx_buffer *b = pb;
+
+	b->index = vb->index;
+	b->length = vb->planes[0].length;
+	b->bytesused = vb->planes[0].bytesused;
+	b->offset = vb->planes[0].m.offset;
+	dprintk(3, "[%s]\n", ctx->name);
+}
+
+static int _fill_vb2_buffer(struct vb2_buffer *vb,
+			    const void *pb, struct vb2_plane *planes)
+{
+	struct dvb_vb2_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+	planes[0].bytesused = 0;
+	dprintk(3, "[%s]\n", ctx->name);
+
+	return 0;
+}
+
+static const struct vb2_buf_ops dvb_vb2_buf_ops = {
+	.fill_user_buffer	= _fill_dmx_buffer,
+	.fill_vb2_buffer	= _fill_vb2_buffer,
+};
+
+/*
+ * Videobuf operations
+ */
+int dvb_vb2_init(struct dvb_vb2_ctx *ctx, const char *name, int nonblocking)
+{
+	struct vb2_queue *q = &ctx->vb_q;
+	int ret;
+
+	memset(ctx, 0, sizeof(struct dvb_vb2_ctx));
+	q->type = DVB_BUF_TYPE_CAPTURE;
+	/**capture type*/
+	q->is_output = 0;
+	/**only mmap is supported currently*/
+	q->io_modes = VB2_MMAP;
+	q->drv_priv = ctx;
+	q->buf_struct_size = sizeof(struct dvb_buffer);
+	q->min_buffers_needed = 1;
+	q->ops = &dvb_vb2_qops;
+	q->mem_ops = &vb2_vmalloc_memops;
+	q->buf_ops = &dvb_vb2_buf_ops;
+	q->num_buffers = 0;
+	ret = vb2_core_queue_init(q);
+	if (ret) {
+		ctx->state = DVB_VB2_STATE_NONE;
+		dprintk(1, "[%s] errno=%d\n", ctx->name, ret);
+		return ret;
+	}
+
+	mutex_init(&ctx->mutex);
+	spin_lock_init(&ctx->slock);
+	INIT_LIST_HEAD(&ctx->dvb_q);
+
+	strlcpy(ctx->name, name, DVB_VB2_NAME_MAX);
+	ctx->nonblocking = nonblocking;
+	ctx->state = DVB_VB2_STATE_INIT;
+
+	dprintk(3, "[%s]\n", ctx->name);
+
+	return 0;
+}
+
+int dvb_vb2_release(struct dvb_vb2_ctx *ctx)
+{
+	struct vb2_queue *q = (struct vb2_queue *)&ctx->vb_q;
+
+	if (ctx->state & DVB_VB2_STATE_INIT)
+		vb2_core_queue_release(q);
+
+	ctx->state = DVB_VB2_STATE_NONE;
+	dprintk(3, "[%s]\n", ctx->name);
+
+	return 0;
+}
+
+int dvb_vb2_stream_on(struct dvb_vb2_ctx *ctx)
+{
+	struct vb2_queue *q = &ctx->vb_q;
+	int ret;
+
+	ret = vb2_core_streamon(q, q->type);
+	if (ret) {
+		ctx->state = DVB_VB2_STATE_NONE;
+		dprintk(1, "[%s] errno=%d\n", ctx->name, ret);
+		return ret;
+	}
+	ctx->state |= DVB_VB2_STATE_STREAMON;
+	dprintk(3, "[%s]\n", ctx->name);
+
+	return 0;
+}
+
+int dvb_vb2_stream_off(struct dvb_vb2_ctx *ctx)
+{
+	struct vb2_queue *q = (struct vb2_queue *)&ctx->vb_q;
+	int ret;
+
+	ctx->state &= ~DVB_VB2_STATE_STREAMON;
+	ret = vb2_core_streamoff(q, q->type);
+	if (ret) {
+		ctx->state = DVB_VB2_STATE_NONE;
+		dprintk(1, "[%s] errno=%d\n", ctx->name, ret);
+		return ret;
+	}
+	dprintk(3, "[%s]\n", ctx->name);
+
+	return 0;
+}
+
+int dvb_vb2_is_streaming(struct dvb_vb2_ctx *ctx)
+{
+	return (ctx->state & DVB_VB2_STATE_STREAMON);
+}
+
+int dvb_vb2_fill_buffer(struct dvb_vb2_ctx *ctx,
+			const unsigned char *src, int len,
+			enum dmx_buffer_flags *buffer_flags)
+{
+	unsigned long flags = 0;
+	void *vbuf = NULL;
+	int todo = len;
+	unsigned char *psrc = (unsigned char *)src;
+	int ll = 0;
+
+	/*
+	 * normal case: This func is called twice from demux driver
+	 * one with valid src pointer, second time with NULL pointer
+	 */
+	if (!src || !len)
+		return 0;
+	spin_lock_irqsave(&ctx->slock, flags);
+	if (buffer_flags && *buffer_flags) {
+		ctx->flags |= *buffer_flags;
+		*buffer_flags = 0;
+	}
+	while (todo) {
+		if (!ctx->buf) {
+			if (list_empty(&ctx->dvb_q)) {
+				dprintk(3, "[%s] Buffer overflow!!!\n",
+					ctx->name);
+				break;
+			}
+
+			ctx->buf = list_entry(ctx->dvb_q.next,
+					      struct dvb_buffer, list);
+			ctx->remain = vb2_plane_size(&ctx->buf->vb, 0);
+			ctx->offset = 0;
+		}
+
+		if (!dvb_vb2_is_streaming(ctx)) {
+			vb2_buffer_done(&ctx->buf->vb, VB2_BUF_STATE_ERROR);
+			list_del(&ctx->buf->list);
+			ctx->buf = NULL;
+			break;
+		}
+
+		/* Fill buffer */
+		ll = min(todo, ctx->remain);
+		vbuf = vb2_plane_vaddr(&ctx->buf->vb, 0);
+		memcpy(vbuf + ctx->offset, psrc, ll);
+		todo -= ll;
+		psrc += ll;
+
+		ctx->remain -= ll;
+		ctx->offset += ll;
+
+		if (ctx->remain == 0) {
+			vb2_buffer_done(&ctx->buf->vb, VB2_BUF_STATE_DONE);
+			list_del(&ctx->buf->list);
+			ctx->buf = NULL;
+		}
+	}
+
+	if (ctx->nonblocking && ctx->buf) {
+		vb2_set_plane_payload(&ctx->buf->vb, 0, ll);
+		vb2_buffer_done(&ctx->buf->vb, VB2_BUF_STATE_DONE);
+		list_del(&ctx->buf->list);
+		ctx->buf = NULL;
+	}
+	spin_unlock_irqrestore(&ctx->slock, flags);
+
+	if (todo)
+		dprintk(1, "[%s] %d bytes are dropped.\n", ctx->name, todo);
+	else
+		dprintk(3, "[%s]\n", ctx->name);
+
+	dprintk(3, "[%s] %d bytes are copied\n", ctx->name, len - todo);
+	return (len - todo);
+}
+
+int dvb_vb2_reqbufs(struct dvb_vb2_ctx *ctx, struct dmx_requestbuffers *req)
+{
+	int ret;
+
+	/* Adjust size to a sane value */
+	if (req->size > DVB_V2_MAX_SIZE)
+		req->size = DVB_V2_MAX_SIZE;
+
+	/* FIXME: round req->size to a 188 or 204 multiple */
+
+	ctx->buf_siz = req->size;
+	ctx->buf_cnt = req->count;
+	ret = vb2_core_reqbufs(&ctx->vb_q, VB2_MEMORY_MMAP, &req->count);
+	if (ret) {
+		ctx->state = DVB_VB2_STATE_NONE;
+		dprintk(1, "[%s] count=%d size=%d errno=%d\n", ctx->name,
+			ctx->buf_cnt, ctx->buf_siz, ret);
+		return ret;
+	}
+	ctx->state |= DVB_VB2_STATE_REQBUFS;
+	dprintk(3, "[%s] count=%d size=%d\n", ctx->name,
+		ctx->buf_cnt, ctx->buf_siz);
+
+	return 0;
+}
+
+int dvb_vb2_querybuf(struct dvb_vb2_ctx *ctx, struct dmx_buffer *b)
+{
+	vb2_core_querybuf(&ctx->vb_q, b->index, b);
+	dprintk(3, "[%s] index=%d\n", ctx->name, b->index);
+	return 0;
+}
+
+int dvb_vb2_expbuf(struct dvb_vb2_ctx *ctx, struct dmx_exportbuffer *exp)
+{
+	struct vb2_queue *q = &ctx->vb_q;
+	int ret;
+
+	ret = vb2_core_expbuf(&ctx->vb_q, &exp->fd, q->type, exp->index,
+			      0, exp->flags);
+	if (ret) {
+		dprintk(1, "[%s] index=%d errno=%d\n", ctx->name,
+			exp->index, ret);
+		return ret;
+	}
+	dprintk(3, "[%s] index=%d fd=%d\n", ctx->name, exp->index, exp->fd);
+
+	return 0;
+}
+
+int dvb_vb2_qbuf(struct dvb_vb2_ctx *ctx, struct dmx_buffer *b)
+{
+	int ret;
+
+	ret = vb2_core_qbuf(&ctx->vb_q, b->index, b);
+	if (ret) {
+		dprintk(1, "[%s] index=%d errno=%d\n", ctx->name,
+			b->index, ret);
+		return ret;
+	}
+	dprintk(5, "[%s] index=%d\n", ctx->name, b->index);
+
+	return 0;
+}
+
+int dvb_vb2_dqbuf(struct dvb_vb2_ctx *ctx, struct dmx_buffer *b)
+{
+	unsigned long flags;
+	int ret;
+
+	ret = vb2_core_dqbuf(&ctx->vb_q, &b->index, b, ctx->nonblocking);
+	if (ret) {
+		dprintk(1, "[%s] errno=%d\n", ctx->name, ret);
+		return ret;
+	}
+
+	spin_lock_irqsave(&ctx->slock, flags);
+	b->count = ctx->count++;
+	b->flags = ctx->flags;
+	ctx->flags = 0;
+	spin_unlock_irqrestore(&ctx->slock, flags);
+
+	dprintk(5, "[%s] index=%d, count=%d, flags=%d\n",
+		ctx->name, b->index, ctx->count, b->flags);
+
+
+	return 0;
+}
+
+int dvb_vb2_mmap(struct dvb_vb2_ctx *ctx, struct vm_area_struct *vma)
+{
+	int ret;
+
+	ret = vb2_mmap(&ctx->vb_q, vma);
+	if (ret) {
+		dprintk(1, "[%s] errno=%d\n", ctx->name, ret);
+		return ret;
+	}
+	dprintk(3, "[%s] ret=%d\n", ctx->name, ret);
+
+	return 0;
+}
+
+__poll_t dvb_vb2_poll(struct dvb_vb2_ctx *ctx, struct file *file,
+		      poll_table *wait)
+{
+	dprintk(3, "[%s]\n", ctx->name);
+	return vb2_core_poll(&ctx->vb_q, file, wait);
+}
+
diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c
index 060c60ddfcc3..60e9c2ba26be 100644
--- a/drivers/media/dvb-core/dvbdev.c
+++ b/drivers/media/dvb-core/dvbdev.c
@@ -30,7 +30,7 @@
 #include <linux/fs.h>
 #include <linux/cdev.h>
 #include <linux/mutex.h>
-#include "dvbdev.h"
+#include <media/dvbdev.h>
 
 /* Due to enum tuner_pad_index */
 #include <media/tuner.h>
diff --git a/drivers/media/dvb-frontends/Makefile b/drivers/media/dvb-frontends/Makefile
index d025eb373842..4be59fed4536 100644
--- a/drivers/media/dvb-frontends/Makefile
+++ b/drivers/media/dvb-frontends/Makefile
@@ -3,7 +3,6 @@
 # Makefile for the kernel DVB frontend device drivers.
 #
 
-ccflags-y += -I$(srctree)/drivers/media/dvb-core/
 ccflags-y += -I$(srctree)/drivers/media/tuners/
 
 # FIXME: RTL2832 SDR driver uses power management directly from USB IF driver
diff --git a/drivers/media/dvb-frontends/a8293.h b/drivers/media/dvb-frontends/a8293.h
index 7b90a03fcd0a..bc6f74f10f32 100644
--- a/drivers/media/dvb-frontends/a8293.h
+++ b/drivers/media/dvb-frontends/a8293.h
@@ -17,7 +17,7 @@
 #ifndef A8293_H
 #define A8293_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /*
  * I2C address
diff --git a/drivers/media/dvb-frontends/af9013_priv.h b/drivers/media/dvb-frontends/af9013_priv.h
index 35ca5c9bcacd..688fc3472cf6 100644
--- a/drivers/media/dvb-frontends/af9013_priv.h
+++ b/drivers/media/dvb-frontends/af9013_priv.h
@@ -21,7 +21,7 @@
 #ifndef AF9013_PRIV_H
 #define AF9013_PRIV_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "af9013.h"
 #include <linux/firmware.h>
 #include <linux/math64.h>
diff --git a/drivers/media/dvb-frontends/af9033_priv.h b/drivers/media/dvb-frontends/af9033_priv.h
index 8799cda1ae14..f269abf609f0 100644
--- a/drivers/media/dvb-frontends/af9033_priv.h
+++ b/drivers/media/dvb-frontends/af9033_priv.h
@@ -18,12 +18,12 @@
 #ifndef AF9033_PRIV_H
 #define AF9033_PRIV_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "af9033.h"
 #include <linux/math64.h>
 #include <linux/regmap.h>
 #include <linux/kernel.h>
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 
 struct reg_val {
 	u32 reg;
diff --git a/drivers/media/dvb-frontends/as102_fe.c b/drivers/media/dvb-frontends/as102_fe.c
index b1c84ee914f0..9b2f2da1d989 100644
--- a/drivers/media/dvb-frontends/as102_fe.c
+++ b/drivers/media/dvb-frontends/as102_fe.c
@@ -14,7 +14,7 @@
  * GNU General Public License for more details.
  */
 
-#include <dvb_frontend.h>
+#include <media/dvb_frontend.h>
 
 #include "as102_fe.h"
 
diff --git a/drivers/media/dvb-frontends/ascot2e.c b/drivers/media/dvb-frontends/ascot2e.c
index 0ee0df53b91b..9746c6dd7fb8 100644
--- a/drivers/media/dvb-frontends/ascot2e.c
+++ b/drivers/media/dvb-frontends/ascot2e.c
@@ -24,7 +24,7 @@
 #include <linux/dvb/frontend.h>
 #include <linux/types.h>
 #include "ascot2e.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define MAX_WRITE_REGSIZE 10
 
@@ -155,7 +155,9 @@ static int ascot2e_write_regs(struct ascot2e_priv *priv,
 
 static int ascot2e_write_reg(struct ascot2e_priv *priv, u8 reg, u8 val)
 {
-	return ascot2e_write_regs(priv, reg, &val, 1);
+	u8 tmp = val; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return ascot2e_write_regs(priv, reg, &tmp, 1);
 }
 
 static int ascot2e_read_regs(struct ascot2e_priv *priv,
diff --git a/drivers/media/dvb-frontends/atbm8830.c b/drivers/media/dvb-frontends/atbm8830.c
index 05850b32d6c6..7b0f3239dbba 100644
--- a/drivers/media/dvb-frontends/atbm8830.c
+++ b/drivers/media/dvb-frontends/atbm8830.c
@@ -16,7 +16,7 @@
  */
 
 #include <asm/div64.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "atbm8830.h"
 #include "atbm8830_priv.h"
diff --git a/drivers/media/dvb-frontends/au8522_common.c b/drivers/media/dvb-frontends/au8522_common.c
index 6722838c3707..56605de9923b 100644
--- a/drivers/media/dvb-frontends/au8522_common.c
+++ b/drivers/media/dvb-frontends/au8522_common.c
@@ -23,7 +23,7 @@
 */
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "au8522_priv.h"
 
 static int debug;
diff --git a/drivers/media/dvb-frontends/au8522_dig.c b/drivers/media/dvb-frontends/au8522_dig.c
index 3f3635f5a06a..8f659bd1c79e 100644
--- a/drivers/media/dvb-frontends/au8522_dig.c
+++ b/drivers/media/dvb-frontends/au8522_dig.c
@@ -24,7 +24,7 @@
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/delay.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "au8522.h"
 #include "au8522_priv.h"
 
diff --git a/drivers/media/dvb-frontends/au8522_priv.h b/drivers/media/dvb-frontends/au8522_priv.h
index f5a9438f6ce5..2043c1744753 100644
--- a/drivers/media/dvb-frontends/au8522_priv.h
+++ b/drivers/media/dvb-frontends/au8522_priv.h
@@ -32,7 +32,7 @@
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-mc.h>
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "au8522.h"
 #include "tuner-i2c.h"
 
@@ -99,7 +99,7 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H		0x0A5
 #define AU8522_AGC_CONTROL_RANGE_REG0A6H		0x0A6
 #define AU8522_SYSTEM_GAIN_CONTROL_REG0A7H		0x0A7
-#define AU8522_TUNER_AGC_RF_STOP_REG0A8H  		0x0A8
+#define AU8522_TUNER_AGC_RF_STOP_REG0A8H		0x0A8
 #define AU8522_TUNER_AGC_RF_START_REG0A9H		0x0A9
 #define AU8522_TUNER_RF_AGC_DEFAULT_REG0AAH		0x0AA
 #define AU8522_TUNER_AGC_IF_STOP_REG0ABH		0x0AB
@@ -110,18 +110,18 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 
 /* Receiver registers */
 #define AU8522_FRMREGTHRD1_REG0B0H			0x0B0
-#define AU8522_FRMREGAGC1H_REG0B1H 			0x0B1
-#define AU8522_FRMREGSHIFT1_REG0B2H 			0x0B2
-#define AU8522_TOREGAGC1_REG0B3H 			0x0B3
-#define AU8522_TOREGASHIFT1_REG0B4H 			0x0B4
+#define AU8522_FRMREGAGC1H_REG0B1H			0x0B1
+#define AU8522_FRMREGSHIFT1_REG0B2H			0x0B2
+#define AU8522_TOREGAGC1_REG0B3H			0x0B3
+#define AU8522_TOREGASHIFT1_REG0B4H			0x0B4
 #define AU8522_FRMREGBBH_REG0B5H			0x0B5
-#define AU8522_FRMREGBBM_REG0B6H 			0x0B6
-#define AU8522_FRMREGBBL_REG0B7H     			0x0B7
+#define AU8522_FRMREGBBM_REG0B6H			0x0B6
+#define AU8522_FRMREGBBL_REG0B7H			0x0B7
 /* 0xB8 TO 0xD7 are the filter coefficients */
-#define AU8522_FRMREGTHRD2_REG0D8H 			0x0D8
-#define AU8522_FRMREGAGC2H_REG0D9H 			0x0D9
-#define AU8522_TOREGAGC2_REG0DAH 			0x0DA
-#define AU8522_TOREGSHIFT2_REG0DBH 			0x0DB
+#define AU8522_FRMREGTHRD2_REG0D8H			0x0D8
+#define AU8522_FRMREGAGC2H_REG0D9H			0x0D9
+#define AU8522_TOREGAGC2_REG0DAH			0x0DA
+#define AU8522_TOREGSHIFT2_REG0DBH			0x0DB
 #define AU8522_FRMREGPILOTH_REG0DCH			0x0DC
 #define AU8522_FRMREGPILOTM_REG0DDH			0x0DD
 #define AU8522_FRMREGPILOTL_REG0DEH			0x0DE
@@ -134,9 +134,9 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 #define AU8522_CHIP_MODE_REG0FEH			0x0FE
 
 /* I2C bus control registers */
-#define AU8522_I2C_CONTROL_REG0_REG090H    		0x090
-#define AU8522_I2C_CONTROL_REG1_REG091H    		0x091
-#define AU8522_I2C_STATUS_REG092H          		0x092
+#define AU8522_I2C_CONTROL_REG0_REG090H			0x090
+#define AU8522_I2C_CONTROL_REG1_REG091H			0x091
+#define AU8522_I2C_STATUS_REG092H			0x092
 #define AU8522_I2C_WR_DATA0_REG093H			0x093
 #define AU8522_I2C_WR_DATA1_REG094H			0x094
 #define AU8522_I2C_WR_DATA2_REG095H			0x095
@@ -156,48 +156,48 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 
 #define AU8522_ENA_USB_REG101H				0x101
 
-#define AU8522_I2S_CTRL_0_REG110H  			0x110
-#define AU8522_I2S_CTRL_1_REG111H 			0x111
-#define AU8522_I2S_CTRL_2_REG112H 			0x112
+#define AU8522_I2S_CTRL_0_REG110H			0x110
+#define AU8522_I2S_CTRL_1_REG111H			0x111
+#define AU8522_I2S_CTRL_2_REG112H			0x112
 
-#define AU8522_FRMREGFFECONTROL_REG121H    		0x121
-#define AU8522_FRMREGDFECONTROL_REG122H    		0x122
+#define AU8522_FRMREGFFECONTROL_REG121H			0x121
+#define AU8522_FRMREGDFECONTROL_REG122H			0x122
 
-#define AU8522_CARRFREQOFFSET0_REG201H 			0x201
+#define AU8522_CARRFREQOFFSET0_REG201H			0x201
 #define AU8522_CARRFREQOFFSET1_REG202H			0x202
 
 #define AU8522_DECIMATION_GAIN_REG21AH			0x21A
-#define AU8522_FRMREGIFSLP_REG21BH 			0x21B
-#define AU8522_FRMREGTHRDL2_REG21CH 			0x21C
-#define AU8522_FRMREGSTEP3DB_REG21DH 			0x21D
+#define AU8522_FRMREGIFSLP_REG21BH			0x21B
+#define AU8522_FRMREGTHRDL2_REG21CH			0x21C
+#define AU8522_FRMREGSTEP3DB_REG21DH			0x21D
 #define AU8522_DAGC_GAIN_ADJUSTMENT_REG21EH		0x21E
-#define AU8522_FRMREGPLLMODE_REG21FH 			0x21F
-#define AU8522_FRMREGCSTHRD_REG220H 			0x220
-#define AU8522_FRMREGCRLOCKDMAX_REG221H 		0x221
-#define AU8522_FRMREGCRPERIODMASK_REG222H 		0x222
-#define AU8522_FRMREGCRLOCK0THH_REG223H 		0x223
-#define AU8522_FRMREGCRLOCK1THH_REG224H 		0x224
-#define AU8522_FRMREGCRLOCK0THL_REG225H 		0x225
-#define AU8522_FRMREGCRLOCK1THL_REG226H 		0x226
+#define AU8522_FRMREGPLLMODE_REG21FH			0x21F
+#define AU8522_FRMREGCSTHRD_REG220H			0x220
+#define AU8522_FRMREGCRLOCKDMAX_REG221H			0x221
+#define AU8522_FRMREGCRPERIODMASK_REG222H		0x222
+#define AU8522_FRMREGCRLOCK0THH_REG223H			0x223
+#define AU8522_FRMREGCRLOCK1THH_REG224H			0x224
+#define AU8522_FRMREGCRLOCK0THL_REG225H			0x225
+#define AU8522_FRMREGCRLOCK1THL_REG226H			0x226
 #define AU_FRMREGPLLACQPHASESCL_REG227H			0x227
-#define AU8522_FRMREGFREQFBCTRL_REG228H 		0x228
+#define AU8522_FRMREGFREQFBCTRL_REG228H			0x228
 
 /* Analog TV Decoder */
 #define AU8522_TVDEC_STATUS_REG000H			0x000
 #define AU8522_TVDEC_INT_STATUS_REG001H			0x001
-#define AU8522_TVDEC_MACROVISION_STATUS_REG002H 	0x002
+#define AU8522_TVDEC_MACROVISION_STATUS_REG002H		0x002
 #define AU8522_TVDEC_SHARPNESSREG009H			0x009
 #define AU8522_TVDEC_BRIGHTNESS_REG00AH			0x00A
 #define AU8522_TVDEC_CONTRAST_REG00BH			0x00B
 #define AU8522_TVDEC_SATURATION_CB_REG00CH		0x00C
 #define AU8522_TVDEC_SATURATION_CR_REG00DH		0x00D
 #define AU8522_TVDEC_HUE_H_REG00EH			0x00E
-#define AU8522_TVDEC_HUE_L_REG00FH                   	0x00F
+#define AU8522_TVDEC_HUE_L_REG00FH			0x00F
 #define AU8522_TVDEC_INT_MASK_REG010H			0x010
 #define AU8522_VIDEO_MODE_REG011H			0x011
 #define AU8522_TVDEC_PGA_REG012H			0x012
 #define AU8522_TVDEC_COMB_MODE_REG015H			0x015
-#define AU8522_REG016H                            	0x016
+#define AU8522_REG016H					0x016
 #define AU8522_TVDED_DBG_MODE_REG060H			0x060
 #define AU8522_TVDEC_FORMAT_CTRL1_REG061H		0x061
 #define AU8522_TVDEC_FORMAT_CTRL2_REG062H		0x062
@@ -207,13 +207,13 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 #define AU8522_TVDEC_COMB_VDIF_THR2_REG066H		0x066
 #define AU8522_TVDEC_COMB_VDIF_THR3_REG067H		0x067
 #define AU8522_TVDEC_COMB_NOTCH_THR_REG068H		0x068
-#define AU8522_TVDEC_COMB_HDIF_THR1_REG069H   		0x069
+#define AU8522_TVDEC_COMB_HDIF_THR1_REG069H		0x069
 #define AU8522_TVDEC_COMB_HDIF_THR2_REG06AH		0x06A
-#define AU8522_TVDEC_COMB_HDIF_THR3_REG06BH   		0x06B
-#define AU8522_TVDEC_COMB_DCDIF_THR1_REG06CH  		0x06C
-#define AU8522_TVDEC_COMB_DCDIF_THR2_REG06DH 		0x06D
-#define AU8522_TVDEC_COMB_DCDIF_THR3_REG06EH       	0x06E
-#define AU8522_TVDEC_UV_SEP_THR_REG06FH  		0x06F
+#define AU8522_TVDEC_COMB_HDIF_THR3_REG06BH		0x06B
+#define AU8522_TVDEC_COMB_DCDIF_THR1_REG06CH		0x06C
+#define AU8522_TVDEC_COMB_DCDIF_THR2_REG06DH		0x06D
+#define AU8522_TVDEC_COMB_DCDIF_THR3_REG06EH		0x06E
+#define AU8522_TVDEC_UV_SEP_THR_REG06FH			0x06F
 #define AU8522_TVDEC_COMB_DC_THR1_NTSC_REG070H		0x070
 #define AU8522_TVDEC_COMB_DC_THR2_NTSC_REG073H		0x073
 #define AU8522_TVDEC_DCAGC_CTRL_REG077H			0x077
@@ -229,42 +229,42 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 
 #define AU8522_TVDEC_CHROMA_AGC_REG401H		0x401
 #define AU8522_TVDEC_CHROMA_SFT_REG402H		0x402
-#define AU8522_FILTER_COEF_R410     		0x410
-#define AU8522_FILTER_COEF_R411     		0x411
-#define AU8522_FILTER_COEF_R412     		0x412
-#define AU8522_FILTER_COEF_R413     		0x413
-#define AU8522_FILTER_COEF_R414     		0x414
-#define AU8522_FILTER_COEF_R415     		0x415
-#define AU8522_FILTER_COEF_R416     		0x416
-#define AU8522_FILTER_COEF_R417     		0x417
-#define AU8522_FILTER_COEF_R418     		0x418
-#define AU8522_FILTER_COEF_R419     		0x419
-#define AU8522_FILTER_COEF_R41A     		0x41A
-#define AU8522_FILTER_COEF_R41B     		0x41B
-#define AU8522_FILTER_COEF_R41C     		0x41C
-#define AU8522_FILTER_COEF_R41D     		0x41D
-#define AU8522_FILTER_COEF_R41E     		0x41E
-#define AU8522_FILTER_COEF_R41F     		0x41F
-#define AU8522_FILTER_COEF_R420     		0x420
-#define AU8522_FILTER_COEF_R421     		0x421
-#define AU8522_FILTER_COEF_R422     		0x422
-#define AU8522_FILTER_COEF_R423     		0x423
-#define AU8522_FILTER_COEF_R424     		0x424
-#define AU8522_FILTER_COEF_R425     		0x425
-#define AU8522_FILTER_COEF_R426     		0x426
-#define AU8522_FILTER_COEF_R427     		0x427
-#define AU8522_FILTER_COEF_R428     		0x428
-#define AU8522_FILTER_COEF_R429     		0x429
-#define AU8522_FILTER_COEF_R42A     		0x42A
-#define AU8522_FILTER_COEF_R42B     		0x42B
-#define AU8522_FILTER_COEF_R42C     		0x42C
-#define AU8522_FILTER_COEF_R42D     		0x42D
+#define AU8522_FILTER_COEF_R410			0x410
+#define AU8522_FILTER_COEF_R411			0x411
+#define AU8522_FILTER_COEF_R412			0x412
+#define AU8522_FILTER_COEF_R413			0x413
+#define AU8522_FILTER_COEF_R414			0x414
+#define AU8522_FILTER_COEF_R415			0x415
+#define AU8522_FILTER_COEF_R416			0x416
+#define AU8522_FILTER_COEF_R417			0x417
+#define AU8522_FILTER_COEF_R418			0x418
+#define AU8522_FILTER_COEF_R419			0x419
+#define AU8522_FILTER_COEF_R41A			0x41A
+#define AU8522_FILTER_COEF_R41B			0x41B
+#define AU8522_FILTER_COEF_R41C			0x41C
+#define AU8522_FILTER_COEF_R41D			0x41D
+#define AU8522_FILTER_COEF_R41E			0x41E
+#define AU8522_FILTER_COEF_R41F			0x41F
+#define AU8522_FILTER_COEF_R420			0x420
+#define AU8522_FILTER_COEF_R421			0x421
+#define AU8522_FILTER_COEF_R422			0x422
+#define AU8522_FILTER_COEF_R423			0x423
+#define AU8522_FILTER_COEF_R424			0x424
+#define AU8522_FILTER_COEF_R425			0x425
+#define AU8522_FILTER_COEF_R426			0x426
+#define AU8522_FILTER_COEF_R427			0x427
+#define AU8522_FILTER_COEF_R428			0x428
+#define AU8522_FILTER_COEF_R429			0x429
+#define AU8522_FILTER_COEF_R42A			0x42A
+#define AU8522_FILTER_COEF_R42B			0x42B
+#define AU8522_FILTER_COEF_R42C			0x42C
+#define AU8522_FILTER_COEF_R42D			0x42D
 
 /* VBI Control Registers */
-#define AU8522_TVDEC_VBI_RX_FIFO_CONTAIN_REG004H  	0x004
-#define AU8522_TVDEC_VBI_TX_FIFO_CONTAIN_REG005H  	0x005
-#define AU8522_TVDEC_VBI_RX_FIFO_READ_REG006H      	0x006
-#define AU8522_TVDEC_VBI_FIFO_STATUS_REG007H       	0x007
+#define AU8522_TVDEC_VBI_RX_FIFO_CONTAIN_REG004H	0x004
+#define AU8522_TVDEC_VBI_TX_FIFO_CONTAIN_REG005H	0x005
+#define AU8522_TVDEC_VBI_RX_FIFO_READ_REG006H		0x006
+#define AU8522_TVDEC_VBI_FIFO_STATUS_REG007H		0x007
 #define AU8522_TVDEC_VBI_CTRL_H_REG017H			0x017
 #define AU8522_TVDEC_VBI_CTRL_L_REG018H			0x018
 #define AU8522_TVDEC_VBI_USER_TOTAL_BITS_REG019H	0x019
@@ -272,10 +272,10 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 #define AU8522_TVDEC_VBI_USER_TUNIT_L_REG01BH		0x01B
 #define AU8522_TVDEC_VBI_USER_THRESH1_REG01CH		0x01C
 #define AU8522_TVDEC_VBI_USER_FRAME_PAT2_REG01EH	0x01E
-#define AU8522_TVDEC_VBI_USER_FRAME_PAT1_REG01FH   	0x01F
-#define AU8522_TVDEC_VBI_USER_FRAME_PAT0_REG020H   	0x020
-#define AU8522_TVDEC_VBI_USER_FRAME_MASK2_REG021H 	0x021
-#define AU8522_TVDEC_VBI_USER_FRAME_MASK1_REG022H  	0x022
+#define AU8522_TVDEC_VBI_USER_FRAME_PAT1_REG01FH	0x01F
+#define AU8522_TVDEC_VBI_USER_FRAME_PAT0_REG020H	0x020
+#define AU8522_TVDEC_VBI_USER_FRAME_MASK2_REG021H	0x021
+#define AU8522_TVDEC_VBI_USER_FRAME_MASK1_REG022H	0x022
 #define AU8522_TVDEC_VBI_USER_FRAME_MASK0_REG023H	0x023
 
 #define AU8522_REG071H					0x071
@@ -315,17 +315,17 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 #define AU8522_GPIO_DATA_REG0E2H			0x0E2
 
 /* Audio Control Registers */
-#define AU8522_AUDIOAGC_REG0EEH 			0x0EE
-#define AU8522_AUDIO_STATUS_REG0F0H 			0x0F0
-#define AU8522_AUDIO_MODE_REG0F1H 			0x0F1
-#define AU8522_AUDIO_VOLUME_L_REG0F2H 			0x0F2
-#define AU8522_AUDIO_VOLUME_R_REG0F3H 			0x0F3
-#define AU8522_AUDIO_VOLUME_REG0F4H 			0x0F4
-#define AU8522_FRMREGAUPHASE_REG0F7H 			0x0F7
+#define AU8522_AUDIOAGC_REG0EEH				0x0EE
+#define AU8522_AUDIO_STATUS_REG0F0H			0x0F0
+#define AU8522_AUDIO_MODE_REG0F1H			0x0F1
+#define AU8522_AUDIO_VOLUME_L_REG0F2H			0x0F2
+#define AU8522_AUDIO_VOLUME_R_REG0F3H			0x0F3
+#define AU8522_AUDIO_VOLUME_REG0F4H			0x0F4
+#define AU8522_FRMREGAUPHASE_REG0F7H			0x0F7
 #define AU8522_REG0F9H					0x0F9
 
-#define AU8522_AUDIOAGC2_REG605H 			0x605
-#define AU8522_AUDIOFREQ_REG606H 			0x606
+#define AU8522_AUDIOAGC2_REG605H			0x605
+#define AU8522_AUDIOFREQ_REG606H			0x606
 
 
 /**************************************************************/
@@ -356,53 +356,53 @@ int au8522_led_ctrl(struct au8522_state *state, int led);
 #define AU8522_TVDEC_FORMAT_CTRL2_REG062H_STD_PAL_M		0x02
 
 
-#define AU8522_INPUT_CONTROL_REG081H_ATSC               	0xC4
+#define AU8522_INPUT_CONTROL_REG081H_ATSC			0xC4
 #define AU8522_INPUT_CONTROL_REG081H_ATVRF			0xC4
 #define AU8522_INPUT_CONTROL_REG081H_ATVRF13			0xC4
-#define AU8522_INPUT_CONTROL_REG081H_J83B64             	0xC4
-#define AU8522_INPUT_CONTROL_REG081H_J83B256            	0xC4
-#define AU8522_INPUT_CONTROL_REG081H_CVBS               	0x20
+#define AU8522_INPUT_CONTROL_REG081H_J83B64			0xC4
+#define AU8522_INPUT_CONTROL_REG081H_J83B256			0xC4
+#define AU8522_INPUT_CONTROL_REG081H_CVBS			0x20
 #define AU8522_INPUT_CONTROL_REG081H_CVBS_CH1			0xA2
 #define AU8522_INPUT_CONTROL_REG081H_CVBS_CH2			0xA0
 #define AU8522_INPUT_CONTROL_REG081H_CVBS_CH3			0x69
 #define AU8522_INPUT_CONTROL_REG081H_CVBS_CH4			0x68
-#define AU8522_INPUT_CONTROL_REG081H_CVBS_CH4_SIF        	0x28
+#define AU8522_INPUT_CONTROL_REG081H_CVBS_CH4_SIF		0x28
 /* CH1 AS Y,CH3 AS C */
-#define AU8522_INPUT_CONTROL_REG081H_SVIDEO_CH13        	0x23
+#define AU8522_INPUT_CONTROL_REG081H_SVIDEO_CH13		0x23
 /* CH2 AS Y,CH4 AS C */
-#define AU8522_INPUT_CONTROL_REG081H_SVIDEO_CH24        	0x20
-#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_ATSC        	0x0C
-#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_J83B64      	0x09
-#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_J83B256    		0x09
-#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_CVBS        	0x12
-#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_ATVRF       	0x1A
+#define AU8522_INPUT_CONTROL_REG081H_SVIDEO_CH24		0x20
+#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_ATSC		0x0C
+#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_J83B64		0x09
+#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_J83B256		0x09
+#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_CVBS		0x12
+#define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_ATVRF		0x1A
 #define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_ATVRF13		0x1A
 #define AU8522_MODULE_CLOCK_CONTROL_REG0A3H_SVIDEO		0x02
 
 #define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_CLEAR		0x00
 #define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_SVIDEO		0x9C
-#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_CVBS     	0x9D
+#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_CVBS		0x9D
 #define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_ATSC		0xE8
-#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_J83B256 		0xCA
-#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_J83B64  		0xCA
-#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_ATVRF   		0xDD
+#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_J83B256		0xCA
+#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_J83B64		0xCA
+#define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_ATVRF		0xDD
 #define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_ATVRF13		0xDD
 #define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_PAL		0xDD
 #define AU8522_SYSTEM_MODULE_CONTROL_0_REG0A4H_FM		0xDD
 
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_ATSC		0x80
-#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_J83B256 		0x80
-#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_J83B64  		0x80
+#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_J83B256		0x80
+#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_J83B64		0x80
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_DONGLE_ATSC	0x40
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_DONGLE_J83B256	0x40
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_DONGLE_J83B64	0x40
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_DONGLE_CLEAR	0x00
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_ATVRF		0x01
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_ATVRF13		0x01
-#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_SVIDEO  		0x04
+#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_SVIDEO		0x04
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_CVBS		0x01
-#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_PWM     		0x03
-#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_IIS      	0x09
+#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_PWM		0x03
+#define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_IIS		0x09
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_PAL		0x01
 #define AU8522_SYSTEM_MODULE_CONTROL_1_REG0A5H_FM		0x01
 
diff --git a/drivers/media/dvb-frontends/bcm3510.c b/drivers/media/dvb-frontends/bcm3510.c
index ba63ad170d3c..05df133dc5be 100644
--- a/drivers/media/dvb-frontends/bcm3510.c
+++ b/drivers/media/dvb-frontends/bcm3510.c
@@ -40,7 +40,7 @@
 #include <linux/slab.h>
 #include <linux/mutex.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "bcm3510.h"
 #include "bcm3510_priv.h"
 
diff --git a/drivers/media/dvb-frontends/cx22700.c b/drivers/media/dvb-frontends/cx22700.c
index 2b629e23ceeb..9ffd2c7ac74a 100644
--- a/drivers/media/dvb-frontends/cx22700.c
+++ b/drivers/media/dvb-frontends/cx22700.c
@@ -25,7 +25,7 @@
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/slab.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx22700.h"
 
 
diff --git a/drivers/media/dvb-frontends/cx22702.c b/drivers/media/dvb-frontends/cx22702.c
index c0e54c59cccf..e8b1e6b7e7e5 100644
--- a/drivers/media/dvb-frontends/cx22702.c
+++ b/drivers/media/dvb-frontends/cx22702.c
@@ -31,7 +31,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx22702.h"
 
 struct cx22702_state {
diff --git a/drivers/media/dvb-frontends/cx24110.c b/drivers/media/dvb-frontends/cx24110.c
index cf1bc99d1f32..2f3a1c237489 100644
--- a/drivers/media/dvb-frontends/cx24110.c
+++ b/drivers/media/dvb-frontends/cx24110.c
@@ -27,7 +27,7 @@
 #include <linux/module.h>
 #include <linux/init.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx24110.h"
 
 
diff --git a/drivers/media/dvb-frontends/cx24113.c b/drivers/media/dvb-frontends/cx24113.c
index ee1f704f81f2..037db3e9d2dd 100644
--- a/drivers/media/dvb-frontends/cx24113.c
+++ b/drivers/media/dvb-frontends/cx24113.c
@@ -22,7 +22,7 @@
 #include <linux/module.h>
 #include <linux/init.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx24113.h"
 
 static int debug;
diff --git a/drivers/media/dvb-frontends/cx24116.c b/drivers/media/dvb-frontends/cx24116.c
index 8fb3f095e21c..786c56a4ef76 100644
--- a/drivers/media/dvb-frontends/cx24116.c
+++ b/drivers/media/dvb-frontends/cx24116.c
@@ -41,7 +41,7 @@
 #include <linux/init.h>
 #include <linux/firmware.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx24116.h"
 
 static int debug;
@@ -963,7 +963,7 @@ static int cx24116_send_diseqc_msg(struct dvb_frontend *fe,
 
 	/* Validate length */
 	if (d->msg_len > sizeof(d->msg))
-                return -EINVAL;
+		return -EINVAL;
 
 	/* Dump DiSEqC message */
 	if (debug) {
diff --git a/drivers/media/dvb-frontends/cx24117.c b/drivers/media/dvb-frontends/cx24117.c
index d37cb7762bd6..8935114b75f3 100644
--- a/drivers/media/dvb-frontends/cx24117.c
+++ b/drivers/media/dvb-frontends/cx24117.c
@@ -32,7 +32,7 @@
 #include <linux/firmware.h>
 
 #include "tuner-i2c.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx24117.h"
 
 
diff --git a/drivers/media/dvb-frontends/cx24120.c b/drivers/media/dvb-frontends/cx24120.c
index 7f11dcc94d85..810f68acd69b 100644
--- a/drivers/media/dvb-frontends/cx24120.c
+++ b/drivers/media/dvb-frontends/cx24120.c
@@ -29,7 +29,7 @@
 #include <linux/moduleparam.h>
 #include <linux/init.h>
 #include <linux/firmware.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx24120.h"
 
 #define CX24120_SEARCH_RANGE_KHZ 5000
diff --git a/drivers/media/dvb-frontends/cx24123.c b/drivers/media/dvb-frontends/cx24123.c
index 1d59d1d3bd82..228ba1f4bf63 100644
--- a/drivers/media/dvb-frontends/cx24123.c
+++ b/drivers/media/dvb-frontends/cx24123.c
@@ -24,7 +24,7 @@
 #include <linux/init.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "cx24123.h"
 
 #define XTAL 10111000
diff --git a/drivers/media/dvb-frontends/cxd2820r_priv.h b/drivers/media/dvb-frontends/cxd2820r_priv.h
index 0d096206ac66..61adde4b4b2f 100644
--- a/drivers/media/dvb-frontends/cxd2820r_priv.h
+++ b/drivers/media/dvb-frontends/cxd2820r_priv.h
@@ -23,8 +23,8 @@
 #define CXD2820R_PRIV_H
 
 #include <linux/dvb/version.h>
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "cxd2820r.h"
 #include <linux/gpio.h>
 #include <linux/math64.h>
diff --git a/drivers/media/dvb-frontends/cxd2841er.c b/drivers/media/dvb-frontends/cxd2841er.c
index 48ee9bc00c06..85905d3503ff 100644
--- a/drivers/media/dvb-frontends/cxd2841er.c
+++ b/drivers/media/dvb-frontends/cxd2841er.c
@@ -29,9 +29,10 @@
 #include <linux/math64.h>
 #include <linux/log2.h>
 #include <linux/dynamic_debug.h>
+#include <linux/kernel.h>
 
-#include "dvb_math.h"
-#include "dvb_frontend.h"
+#include <media/dvb_math.h>
+#include <media/dvb_frontend.h>
 #include "cxd2841er.h"
 #include "cxd2841er_priv.h"
 
@@ -257,7 +258,9 @@ static int cxd2841er_write_regs(struct cxd2841er_priv *priv,
 static int cxd2841er_write_reg(struct cxd2841er_priv *priv,
 			       u8 addr, u8 reg, u8 val)
 {
-	return cxd2841er_write_regs(priv, addr, reg, &val, 1);
+	u8 tmp = val; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return cxd2841er_write_regs(priv, addr, reg, &tmp, 1);
 }
 
 static int cxd2841er_read_regs(struct cxd2841er_priv *priv,
@@ -1696,12 +1699,10 @@ static u32 cxd2841er_dvbs_read_snr(struct cxd2841er_priv *priv,
 		min_index = 0;
 		if (delsys == SYS_DVBS) {
 			cn_data = s_cn_data;
-			max_index = sizeof(s_cn_data) /
-				sizeof(s_cn_data[0]) - 1;
+			max_index = ARRAY_SIZE(s_cn_data) - 1;
 		} else {
 			cn_data = s2_cn_data;
-			max_index = sizeof(s2_cn_data) /
-				sizeof(s2_cn_data[0]) - 1;
+			max_index = ARRAY_SIZE(s2_cn_data) - 1;
 		}
 		if (value >= cn_data[min_index].value) {
 			res = cn_data[min_index].cnr_x1000;
@@ -3340,13 +3341,17 @@ static int cxd2841er_set_frontend_s(struct dvb_frontend *fe)
 
 	cxd2841er_tune_done(priv);
 	timeout = ((3000000 + (symbol_rate - 1)) / symbol_rate) + 150;
-	for (i = 0; i < timeout / CXD2841ER_DVBS_POLLING_INVL; i++) {
+
+	i = 0;
+	do {
 		usleep_range(CXD2841ER_DVBS_POLLING_INVL*1000,
 			(CXD2841ER_DVBS_POLLING_INVL + 2) * 1000);
 		cxd2841er_read_status_s(fe, &status);
 		if (status & FE_HAS_LOCK)
 			break;
-	}
+		i++;
+	} while (i < timeout / CXD2841ER_DVBS_POLLING_INVL);
+
 	if (status & FE_HAS_LOCK) {
 		if (cxd2841er_get_carrier_offset_s_s2(
 				priv, &carr_offset)) {
diff --git a/drivers/media/dvb-frontends/dib0070.c b/drivers/media/dvb-frontends/dib0070.c
index d7614b8b8782..932d235118e2 100644
--- a/drivers/media/dvb-frontends/dib0070.c
+++ b/drivers/media/dvb-frontends/dib0070.c
@@ -27,7 +27,7 @@
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "dib0070.h"
 #include "dibx000_common.h"
diff --git a/drivers/media/dvb-frontends/dib0090.c b/drivers/media/dvb-frontends/dib0090.c
index d9d730dfe0b1..64f49c8eb1fb 100644
--- a/drivers/media/dvb-frontends/dib0090.c
+++ b/drivers/media/dvb-frontends/dib0090.c
@@ -27,7 +27,7 @@
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "dib0090.h"
 #include "dibx000_common.h"
diff --git a/drivers/media/dvb-frontends/dib3000mb.c b/drivers/media/dvb-frontends/dib3000mb.c
index 068bec104e29..de3ce2786c72 100644
--- a/drivers/media/dvb-frontends/dib3000mb.c
+++ b/drivers/media/dvb-frontends/dib3000mb.c
@@ -30,7 +30,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "dib3000.h"
 #include "dib3000mb_priv.h"
diff --git a/drivers/media/dvb-frontends/dib3000mc.c b/drivers/media/dvb-frontends/dib3000mc.c
index 4d086a7248e9..7e5d474806a5 100644
--- a/drivers/media/dvb-frontends/dib3000mc.c
+++ b/drivers/media/dvb-frontends/dib3000mc.c
@@ -17,7 +17,7 @@
 #include <linux/slab.h>
 #include <linux/i2c.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "dib3000mc.h"
 
diff --git a/drivers/media/dvb-frontends/dib7000m.c b/drivers/media/dvb-frontends/dib7000m.c
index 5ce9f93a65c3..6a1d357d0c7c 100644
--- a/drivers/media/dvb-frontends/dib7000m.c
+++ b/drivers/media/dvb-frontends/dib7000m.c
@@ -16,7 +16,7 @@
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "dib7000m.h"
 
diff --git a/drivers/media/dvb-frontends/dib7000p.c b/drivers/media/dvb-frontends/dib7000p.c
index 0fbaabe43682..90ace707a80d 100644
--- a/drivers/media/dvb-frontends/dib7000p.c
+++ b/drivers/media/dvb-frontends/dib7000p.c
@@ -16,8 +16,8 @@
 #include <linux/mutex.h>
 #include <asm/div64.h>
 
-#include "dvb_math.h"
-#include "dvb_frontend.h"
+#include <media/dvb_math.h>
+#include <media/dvb_frontend.h>
 
 #include "dib7000p.h"
 
diff --git a/drivers/media/dvb-frontends/dib8000.c b/drivers/media/dvb-frontends/dib8000.c
index 5d9381509b07..e64014f338fb 100644
--- a/drivers/media/dvb-frontends/dib8000.c
+++ b/drivers/media/dvb-frontends/dib8000.c
@@ -16,9 +16,9 @@
 #include <linux/mutex.h>
 #include <asm/div64.h>
 
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "dib8000.h"
 
diff --git a/drivers/media/dvb-frontends/dib9000.c b/drivers/media/dvb-frontends/dib9000.c
index 1b7a4331af05..f9289f488de7 100644
--- a/drivers/media/dvb-frontends/dib9000.c
+++ b/drivers/media/dvb-frontends/dib9000.c
@@ -14,8 +14,8 @@
 #include <linux/i2c.h>
 #include <linux/mutex.h>
 
-#include "dvb_math.h"
-#include "dvb_frontend.h"
+#include <media/dvb_math.h>
+#include <media/dvb_frontend.h>
 
 #include "dib9000.h"
 #include "dibx000_common.h"
diff --git a/drivers/media/dvb-frontends/dibx000_common.c b/drivers/media/dvb-frontends/dibx000_common.c
index bc28184c7fb0..d981233e458f 100644
--- a/drivers/media/dvb-frontends/dibx000_common.c
+++ b/drivers/media/dvb-frontends/dibx000_common.c
@@ -288,8 +288,8 @@ static int dibx000_i2c_gated_gpio67_xfer(struct i2c_adapter *i2c_adap,
 	int ret;
 
 	if (num > 32) {
-		dprintk("%s: too much I2C message to be transmitted (%i).\
-				Maximum is 32", __func__, num);
+		dprintk("%s: too much I2C message to be transmitted (%i). Maximum is 32",
+			__func__, num);
 		return -ENOMEM;
 	}
 
@@ -335,8 +335,8 @@ static int dibx000_i2c_gated_tuner_xfer(struct i2c_adapter *i2c_adap,
 	int ret;
 
 	if (num > 32) {
-		dprintk("%s: too much I2C message to be transmitted (%i).\
-				Maximum is 32", __func__, num);
+		dprintk("%s: too much I2C message to be transmitted (%i). Maximum is 32",
+			__func__, num);
 		return -ENOMEM;
 	}
 
diff --git a/drivers/media/dvb-frontends/drx39xyj/Makefile b/drivers/media/dvb-frontends/drx39xyj/Makefile
index 672e07774955..87f6eddcf657 100644
--- a/drivers/media/dvb-frontends/drx39xyj/Makefile
+++ b/drivers/media/dvb-frontends/drx39xyj/Makefile
@@ -2,5 +2,4 @@ drx39xyj-objs := drxj.o
 
 obj-$(CONFIG_DVB_DRX39XYJ) += drx39xyj.o
 
-ccflags-y += -I$(srctree)/drivers/media/dvb-core/
 ccflags-y += -I$(srctree)/drivers/media/tuners/
diff --git a/drivers/media/dvb-frontends/drx39xyj/drx39xxj.h b/drivers/media/dvb-frontends/drx39xyj/drx39xxj.h
index 11e1ddeeef0a..c0c66ed65b6e 100644
--- a/drivers/media/dvb-frontends/drx39xyj/drx39xxj.h
+++ b/drivers/media/dvb-frontends/drx39xyj/drx39xxj.h
@@ -19,7 +19,7 @@
 #define DRX39XXJ_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "drx_driver.h"
 
 struct drx39xxj_state {
diff --git a/drivers/media/dvb-frontends/drx39xyj/drx_driver.h b/drivers/media/dvb-frontends/drx39xyj/drx_driver.h
index 855685b6b386..1ec20eecc433 100644
--- a/drivers/media/dvb-frontends/drx39xyj/drx_driver.h
+++ b/drivers/media/dvb-frontends/drx39xyj/drx_driver.h
@@ -932,7 +932,7 @@ STRUCTS
  * Used by DRX_CTRL_LOAD_UCODE and DRX_CTRL_VERIFY_UCODE
  */
 struct drxu_code_info {
-	char 			*mc_file;
+	char			*mc_file;
 };
 
 /*
diff --git a/drivers/media/dvb-frontends/drx39xyj/drxj.c b/drivers/media/dvb-frontends/drx39xyj/drxj.c
index 8cbd8cc21059..5706898e84cc 100644
--- a/drivers/media/dvb-frontends/drx39xyj/drxj.c
+++ b/drivers/media/dvb-frontends/drx39xyj/drxj.c
@@ -61,7 +61,7 @@ INCLUDE FILES
 #include <linux/slab.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "drx39xxj.h"
 
 #include "drxj.h"
@@ -2151,9 +2151,13 @@ int drxj_dap_atomic_read_write_block(struct i2c_device_addr *dev_addr,
 	if (read_flag) {
 		/* read data from buffer */
 		for (i = 0; i < (datasize / 2); i++) {
-			drxj_dap_read_reg16(dev_addr,
-					    (DRXJ_HI_ATOMIC_BUF_START + i),
-					   &word, 0);
+			rc = drxj_dap_read_reg16(dev_addr,
+						 (DRXJ_HI_ATOMIC_BUF_START + i),
+						 &word, 0);
+			if (rc) {
+				pr_err("error %d\n", rc);
+				goto rw_error;
+			}
 			data[2 * i] = (u8) (word & 0xFF);
 			data[(2 * i) + 1] = (u8) (word >> 8);
 		}
@@ -11074,7 +11078,7 @@ ctrl_power_mode(struct drx_demod_instance *demod, enum drx_power_mode *mode)
 		}
 	}
 
-	if ((*mode == DRX_POWER_UP)) {
+	if (*mode == DRX_POWER_UP) {
 		/* Restore analog & pin configuration */
 
 		/* Initialize default AFE configuration for VSB */
@@ -11723,7 +11727,7 @@ static int drx_check_firmware(struct drx_demod_instance *demod, u8 *mc_data,
  *		- In case of UCODE_UPLOAD: I2C error.
  *		- In case of UCODE_VERIFY: I2C error or image on device
  *		  is not equal to image provided to this control function.
- * 	-EINVAL:
+ *	-EINVAL:
  *		- Invalid arguments.
  *		- Provided image is corrupt
  */
diff --git a/drivers/media/dvb-frontends/drxd_hard.c b/drivers/media/dvb-frontends/drxd_hard.c
index 0696bc62dcc9..3b7d31a22d82 100644
--- a/drivers/media/dvb-frontends/drxd_hard.c
+++ b/drivers/media/dvb-frontends/drxd_hard.c
@@ -26,7 +26,7 @@
 #include <linux/i2c.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "drxd.h"
 #include "drxd_firm.h"
 
@@ -972,7 +972,6 @@ static int DownloadMicrocode(struct drxd_state *state,
 static int HI_Command(struct drxd_state *state, u16 cmd, u16 * pResult)
 {
 	u32 nrRetries = 0;
-	u16 waitCmd;
 	int status;
 
 	status = Write16(state, HI_RA_RAM_SRV_CMD__A, cmd, 0);
@@ -985,8 +984,8 @@ static int HI_Command(struct drxd_state *state, u16 cmd, u16 * pResult)
 			status = -1;
 			break;
 		}
-		status = Read16(state, HI_RA_RAM_SRV_CMD__A, &waitCmd, 0);
-	} while (waitCmd != 0);
+		status = Read16(state, HI_RA_RAM_SRV_CMD__A, NULL, 0);
+	} while (status != 0);
 
 	if (status >= 0)
 		status = Read16(state, HI_RA_RAM_SRV_RES__A, pResult, 0);
@@ -1298,12 +1297,11 @@ static int InitFT(struct drxd_state *state)
 
 static int SC_WaitForReady(struct drxd_state *state)
 {
-	u16 curCmd;
 	int i;
 
 	for (i = 0; i < DRXD_MAX_RETRIES; i += 1) {
-		int status = Read16(state, SC_RA_RAM_CMD__A, &curCmd, 0);
-		if (status == 0 || curCmd == 0)
+		int status = Read16(state, SC_RA_RAM_CMD__A, NULL, 0);
+		if (status == 0)
 			return status;
 	}
 	return -1;
@@ -1311,15 +1309,15 @@ static int SC_WaitForReady(struct drxd_state *state)
 
 static int SC_SendCommand(struct drxd_state *state, u16 cmd)
 {
-	int status = 0;
+	int status = 0, ret;
 	u16 errCode;
 
 	Write16(state, SC_RA_RAM_CMD__A, cmd, 0);
 	SC_WaitForReady(state);
 
-	Read16(state, SC_RA_RAM_CMD_ADDR__A, &errCode, 0);
+	ret = Read16(state, SC_RA_RAM_CMD_ADDR__A, &errCode, 0);
 
-	if (errCode == 0xFFFF) {
+	if (ret < 0 || errCode == 0xFFFF) {
 		printk(KERN_ERR "Command Error\n");
 		status = -1;
 	}
@@ -1330,13 +1328,13 @@ static int SC_SendCommand(struct drxd_state *state, u16 cmd)
 static int SC_ProcStartCommand(struct drxd_state *state,
 			       u16 subCmd, u16 param0, u16 param1)
 {
-	int status = 0;
+	int ret, status = 0;
 	u16 scExec;
 
 	mutex_lock(&state->mutex);
 	do {
-		Read16(state, SC_COMM_EXEC__A, &scExec, 0);
-		if (scExec != 1) {
+		ret = Read16(state, SC_COMM_EXEC__A, &scExec, 0);
+		if (ret < 0 || scExec != 1) {
 			status = -1;
 			break;
 		}
@@ -2140,7 +2138,6 @@ static int DRX_Start(struct drxd_state *state, s32 off)
 			}
 			break;
 		}
-		status = status;
 		if (status < 0)
 			break;
 
@@ -2251,7 +2248,6 @@ static int DRX_Start(struct drxd_state *state, s32 off)
 			break;
 
 		}
-		status = status;
 		if (status < 0)
 			break;
 
@@ -2318,7 +2314,6 @@ static int DRX_Start(struct drxd_state *state, s32 off)
 			}
 			break;
 		}
-		status = status;
 		if (status < 0)
 			break;
 
diff --git a/drivers/media/dvb-frontends/drxk.h b/drivers/media/dvb-frontends/drxk.h
index b16fedbb53a3..76466f7ec3a0 100644
--- a/drivers/media/dvb-frontends/drxk.h
+++ b/drivers/media/dvb-frontends/drxk.h
@@ -10,7 +10,7 @@
  *
  * @adr:		I2C address of the DRX-K
  * @parallel_ts:	True means that the device uses parallel TS,
- * 			Serial otherwise.
+ *			Serial otherwise.
  * @dynamic_clk:	True means that the clock will be dynamically
  *			adjusted. Static clock otherwise.
  * @enable_merr_cfg:	Enable SIO_PDR_PERR_CFG/SIO_PDR_MVAL_CFG.
@@ -67,8 +67,8 @@ extern struct dvb_frontend *drxk_attach(const struct drxk_config *config,
 static inline struct dvb_frontend *drxk_attach(const struct drxk_config *config,
 					struct i2c_adapter *i2c)
 {
-        printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
-        return NULL;
+	printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
+	return NULL;
 }
 #endif
 
diff --git a/drivers/media/dvb-frontends/drxk_hard.c b/drivers/media/dvb-frontends/drxk_hard.c
index f59ac2e91c59..5a26ad93be10 100644
--- a/drivers/media/dvb-frontends/drxk_hard.c
+++ b/drivers/media/dvb-frontends/drxk_hard.c
@@ -29,10 +29,10 @@
 #include <linux/hardirq.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "drxk.h"
 #include "drxk_hard.h"
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 
 static int power_down_dvbt(struct drxk_state *state, bool set_power_mode);
 static int power_down_qam(struct drxk_state *state);
@@ -6062,7 +6062,7 @@ static int init_drxk(struct drxk_state *state)
 	u16 driver_version;
 
 	dprintk(1, "\n");
-	if ((state->m_drxk_state == DRXK_UNINITIALIZED)) {
+	if (state->m_drxk_state == DRXK_UNINITIALIZED) {
 		drxk_i2c_lock(state);
 		status = power_up_device(state);
 		if (status < 0)
diff --git a/drivers/media/dvb-frontends/ds3000.c b/drivers/media/dvb-frontends/ds3000.c
index bd4f8278c906..2ff90e5eabce 100644
--- a/drivers/media/dvb-frontends/ds3000.c
+++ b/drivers/media/dvb-frontends/ds3000.c
@@ -26,7 +26,7 @@
 #include <linux/init.h>
 #include <linux/firmware.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "ts2020.h"
 #include "ds3000.h"
 
diff --git a/drivers/media/dvb-frontends/dvb-pll.h b/drivers/media/dvb-frontends/dvb-pll.h
index 212e0730f154..ca885e71d2f0 100644
--- a/drivers/media/dvb-frontends/dvb-pll.h
+++ b/drivers/media/dvb-frontends/dvb-pll.h
@@ -7,7 +7,7 @@
 #define __DVB_PLL_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define DVB_PLL_UNDEFINED               0
 #define DVB_PLL_THOMSON_DTT7579         1
diff --git a/drivers/media/dvb-frontends/dvb_dummy_fe.c b/drivers/media/dvb-frontends/dvb_dummy_fe.c
index 50b2b666ef6c..6650d4f61ef2 100644
--- a/drivers/media/dvb-frontends/dvb_dummy_fe.c
+++ b/drivers/media/dvb-frontends/dvb_dummy_fe.c
@@ -20,7 +20,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "dvb_dummy_fe.h"
 
 
diff --git a/drivers/media/dvb-frontends/dvb_dummy_fe.h b/drivers/media/dvb-frontends/dvb_dummy_fe.h
index 86dd7b9d1e57..7aacef4b7c80 100644
--- a/drivers/media/dvb-frontends/dvb_dummy_fe.h
+++ b/drivers/media/dvb-frontends/dvb_dummy_fe.h
@@ -19,7 +19,7 @@
 #define DVB_DUMMY_FE_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #if IS_REACHABLE(CONFIG_DVB_DUMMY_FE)
 extern struct dvb_frontend* dvb_dummy_fe_ofdm_attach(void);
diff --git a/drivers/media/dvb-frontends/ec100.c b/drivers/media/dvb-frontends/ec100.c
index fa2a96d5f94e..c2575fdcc811 100644
--- a/drivers/media/dvb-frontends/ec100.c
+++ b/drivers/media/dvb-frontends/ec100.c
@@ -15,7 +15,7 @@
  *
  */
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "ec100.h"
 
 struct ec100_state {
diff --git a/drivers/media/dvb-frontends/gp8psk-fe.c b/drivers/media/dvb-frontends/gp8psk-fe.c
index efe015df7f1d..a772ef8bfe1c 100644
--- a/drivers/media/dvb-frontends/gp8psk-fe.c
+++ b/drivers/media/dvb-frontends/gp8psk-fe.c
@@ -16,7 +16,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include "gp8psk-fe.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 static int debug;
 module_param(debug, int, 0644);
diff --git a/drivers/media/dvb-frontends/helene.c b/drivers/media/dvb-frontends/helene.c
index 4bf5a551ba40..a0d0b53c91d7 100644
--- a/drivers/media/dvb-frontends/helene.c
+++ b/drivers/media/dvb-frontends/helene.c
@@ -23,7 +23,7 @@
 #include <linux/dvb/frontend.h>
 #include <linux/types.h>
 #include "helene.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define MAX_WRITE_REGSIZE 20
 
@@ -331,7 +331,9 @@ static int helene_write_regs(struct helene_priv *priv,
 
 static int helene_write_reg(struct helene_priv *priv, u8 reg, u8 val)
 {
-	return helene_write_regs(priv, reg, &val, 1);
+	u8 tmp = val; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return helene_write_regs(priv, reg, &tmp, 1);
 }
 
 static int helene_read_regs(struct helene_priv *priv,
diff --git a/drivers/media/dvb-frontends/horus3a.c b/drivers/media/dvb-frontends/horus3a.c
index 68d759c4c52e..5e7e265a52e6 100644
--- a/drivers/media/dvb-frontends/horus3a.c
+++ b/drivers/media/dvb-frontends/horus3a.c
@@ -24,7 +24,7 @@
 #include <linux/dvb/frontend.h>
 #include <linux/types.h>
 #include "horus3a.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define MAX_WRITE_REGSIZE      5
 
@@ -89,7 +89,9 @@ static int horus3a_write_regs(struct horus3a_priv *priv,
 
 static int horus3a_write_reg(struct horus3a_priv *priv, u8 reg, u8 val)
 {
-	return horus3a_write_regs(priv, reg, &val, 1);
+	u8 tmp = val; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return horus3a_write_regs(priv, reg, &tmp, 1);
 }
 
 static int horus3a_enter_power_save(struct horus3a_priv *priv)
diff --git a/drivers/media/dvb-frontends/isl6405.c b/drivers/media/dvb-frontends/isl6405.c
index 2fc8d3c72c11..3bc78f8ffc00 100644
--- a/drivers/media/dvb-frontends/isl6405.c
+++ b/drivers/media/dvb-frontends/isl6405.c
@@ -29,7 +29,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "isl6405.h"
 
 struct isl6405 {
diff --git a/drivers/media/dvb-frontends/isl6421.c b/drivers/media/dvb-frontends/isl6421.c
index 3f3487887672..ae8ec59b665c 100644
--- a/drivers/media/dvb-frontends/isl6421.c
+++ b/drivers/media/dvb-frontends/isl6421.c
@@ -29,7 +29,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "isl6421.h"
 
 struct isl6421 {
diff --git a/drivers/media/dvb-frontends/isl6423.c b/drivers/media/dvb-frontends/isl6423.c
index dca5bebfeeb5..3dd2465d17cf 100644
--- a/drivers/media/dvb-frontends/isl6423.c
+++ b/drivers/media/dvb-frontends/isl6423.c
@@ -26,7 +26,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "isl6423.h"
 
 static unsigned int verbose;
diff --git a/drivers/media/dvb-frontends/itd1000.c b/drivers/media/dvb-frontends/itd1000.c
index 5bb1e73a10b4..04f7f6854f73 100644
--- a/drivers/media/dvb-frontends/itd1000.c
+++ b/drivers/media/dvb-frontends/itd1000.c
@@ -22,7 +22,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "itd1000.h"
 #include "itd1000_priv.h"
@@ -95,8 +95,9 @@ static int itd1000_read_reg(struct itd1000_state *state, u8 reg)
 
 static inline int itd1000_write_reg(struct itd1000_state *state, u8 r, u8 v)
 {
-	int ret = itd1000_write_regs(state, r, &v, 1);
-	state->shadow[r] = v;
+	u8 tmp = v; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+	int ret = itd1000_write_regs(state, r, &tmp, 1);
+	state->shadow[r] = tmp;
 	return ret;
 }
 
diff --git a/drivers/media/dvb-frontends/ix2505v.h b/drivers/media/dvb-frontends/ix2505v.h
index 49ed93e754ed..20b1eb3dda1a 100644
--- a/drivers/media/dvb-frontends/ix2505v.h
+++ b/drivers/media/dvb-frontends/ix2505v.h
@@ -17,7 +17,7 @@
 #define DVB_IX2505V_H
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /**
  * struct ix2505v_config - ix2505 attachment configuration
diff --git a/drivers/media/dvb-frontends/l64781.c b/drivers/media/dvb-frontends/l64781.c
index e5a6c1766664..e056f36e6f0c 100644
--- a/drivers/media/dvb-frontends/l64781.c
+++ b/drivers/media/dvb-frontends/l64781.c
@@ -25,7 +25,7 @@
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/slab.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "l64781.h"
 
 
diff --git a/drivers/media/dvb-frontends/lg2160.h b/drivers/media/dvb-frontends/lg2160.h
index ba99125deac0..df817aec29e2 100644
--- a/drivers/media/dvb-frontends/lg2160.h
+++ b/drivers/media/dvb-frontends/lg2160.h
@@ -19,7 +19,7 @@
 #define _LG2160_H_
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 enum lg_chip_type {
 	LG2160 = 0,
diff --git a/drivers/media/dvb-frontends/lgdt3305.c b/drivers/media/dvb-frontends/lgdt3305.c
index 0af4d9104761..735d73060265 100644
--- a/drivers/media/dvb-frontends/lgdt3305.c
+++ b/drivers/media/dvb-frontends/lgdt3305.c
@@ -20,7 +20,7 @@
 #include <asm/div64.h>
 #include <linux/dvb/frontend.h>
 #include <linux/slab.h>
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 #include "lgdt3305.h"
 
 static int debug;
diff --git a/drivers/media/dvb-frontends/lgdt3305.h b/drivers/media/dvb-frontends/lgdt3305.h
index 2fb60d91f7b4..a54daaee823a 100644
--- a/drivers/media/dvb-frontends/lgdt3305.h
+++ b/drivers/media/dvb-frontends/lgdt3305.h
@@ -19,7 +19,7 @@
 #define _LGDT3305_H_
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 
 enum lgdt3305_mpeg_mode {
diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c
index 724e9aac0f11..6356815cf3e1 100644
--- a/drivers/media/dvb-frontends/lgdt3306a.c
+++ b/drivers/media/dvb-frontends/lgdt3306a.c
@@ -21,7 +21,7 @@
 #include <asm/div64.h>
 #include <linux/kernel.h>
 #include <linux/dvb/frontend.h>
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 #include "lgdt3306a.h"
 #include <linux/i2c-mux.h>
 
diff --git a/drivers/media/dvb-frontends/lgdt3306a.h b/drivers/media/dvb-frontends/lgdt3306a.h
index 6ce337ec5272..8b53044f5bdb 100644
--- a/drivers/media/dvb-frontends/lgdt3306a.h
+++ b/drivers/media/dvb-frontends/lgdt3306a.h
@@ -19,7 +19,7 @@
 #define _LGDT3306A_H_
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 
 enum lgdt3306a_mpeg_mode {
diff --git a/drivers/media/dvb-frontends/lgdt330x.c b/drivers/media/dvb-frontends/lgdt330x.c
index 06f47dc8cd3d..8ad03bd81af5 100644
--- a/drivers/media/dvb-frontends/lgdt330x.c
+++ b/drivers/media/dvb-frontends/lgdt330x.c
@@ -37,8 +37,8 @@
 #include <linux/slab.h>
 #include <asm/byteorder.h>
 
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "lgdt330x_priv.h"
 #include "lgdt330x.h"
 
diff --git a/drivers/media/dvb-frontends/lgs8gl5.c b/drivers/media/dvb-frontends/lgs8gl5.c
index 970e42fdbc1b..f47e5a1af16d 100644
--- a/drivers/media/dvb-frontends/lgs8gl5.c
+++ b/drivers/media/dvb-frontends/lgs8gl5.c
@@ -25,7 +25,7 @@
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/slab.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "lgs8gl5.h"
 
 
diff --git a/drivers/media/dvb-frontends/lgs8gxx.c b/drivers/media/dvb-frontends/lgs8gxx.c
index e6bf60e1138c..84af8a12f26a 100644
--- a/drivers/media/dvb-frontends/lgs8gxx.c
+++ b/drivers/media/dvb-frontends/lgs8gxx.c
@@ -22,7 +22,7 @@
 #include <asm/div64.h>
 #include <linux/firmware.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "lgs8gxx.h"
 #include "lgs8gxx_priv.h"
diff --git a/drivers/media/dvb-frontends/lnbh25.c b/drivers/media/dvb-frontends/lnbh25.c
index cb486e879fdd..0b388502c298 100644
--- a/drivers/media/dvb-frontends/lnbh25.c
+++ b/drivers/media/dvb-frontends/lnbh25.c
@@ -23,7 +23,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "lnbh25.h"
 
 /**
diff --git a/drivers/media/dvb-frontends/lnbp21.c b/drivers/media/dvb-frontends/lnbp21.c
index 392d7be93774..d9966a338a72 100644
--- a/drivers/media/dvb-frontends/lnbp21.c
+++ b/drivers/media/dvb-frontends/lnbp21.c
@@ -29,7 +29,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "lnbp21.h"
 #include "lnbh24.h"
 
diff --git a/drivers/media/dvb-frontends/lnbp22.c b/drivers/media/dvb-frontends/lnbp22.c
index 39326a2ebab2..a62e82bf46f5 100644
--- a/drivers/media/dvb-frontends/lnbp22.c
+++ b/drivers/media/dvb-frontends/lnbp22.c
@@ -30,7 +30,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "lnbp22.h"
 
 static int debug;
diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c
index 50bce68ffd66..65d157fe76d1 100644
--- a/drivers/media/dvb-frontends/m88ds3103.c
+++ b/drivers/media/dvb-frontends/m88ds3103.c
@@ -1262,11 +1262,12 @@ static int m88ds3103_select(struct i2c_mux_core *muxc, u32 chan)
  * New users must use I2C client binding directly!
  */
 struct dvb_frontend *m88ds3103_attach(const struct m88ds3103_config *cfg,
-		struct i2c_adapter *i2c, struct i2c_adapter **tuner_i2c_adapter)
+				      struct i2c_adapter *i2c,
+				      struct i2c_adapter **tuner_i2c_adapter)
 {
 	struct i2c_client *client;
 	struct i2c_board_info board_info;
-	struct m88ds3103_platform_data pdata;
+	struct m88ds3103_platform_data pdata = {};
 
 	pdata.clk = cfg->clock;
 	pdata.i2c_wr_max = cfg->i2c_wr_max;
@@ -1409,6 +1410,8 @@ static int m88ds3103_probe(struct i2c_client *client,
 	case M88DS3103_CHIP_ID:
 		break;
 	default:
+		ret = -ENODEV;
+		dev_err(&client->dev, "Unknown device. Chip_id=%02x\n", dev->chip_id);
 		goto err_kfree;
 	}
 
diff --git a/drivers/media/dvb-frontends/m88ds3103_priv.h b/drivers/media/dvb-frontends/m88ds3103_priv.h
index 07f20c269c67..1ba0b79df311 100644
--- a/drivers/media/dvb-frontends/m88ds3103_priv.h
+++ b/drivers/media/dvb-frontends/m88ds3103_priv.h
@@ -17,9 +17,9 @@
 #ifndef M88DS3103_PRIV_H
 #define M88DS3103_PRIV_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "m88ds3103.h"
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 #include <linux/firmware.h>
 #include <linux/i2c-mux.h>
 #include <linux/regmap.h>
diff --git a/drivers/media/dvb-frontends/m88rs2000.c b/drivers/media/dvb-frontends/m88rs2000.c
index ce6c21d405ee..496ce27fa63a 100644
--- a/drivers/media/dvb-frontends/m88rs2000.c
+++ b/drivers/media/dvb-frontends/m88rs2000.c
@@ -31,7 +31,7 @@
 #include <linux/types.h>
 
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "m88rs2000.h"
 
 struct m88rs2000_state {
@@ -630,13 +630,16 @@ static int m88rs2000_set_frontend(struct dvb_frontend *fe)
 	if (ret < 0)
 		return -ENODEV;
 
-	if (fe->ops.tuner_ops.get_frequency)
+	if (fe->ops.tuner_ops.get_frequency) {
 		ret = fe->ops.tuner_ops.get_frequency(fe, &tuner_freq);
 
-	if (ret < 0)
-		return -ENODEV;
+		if (ret < 0)
+			return -ENODEV;
 
-	offset = (s16)((s32)tuner_freq - c->frequency);
+		offset = (s16)((s32)tuner_freq - c->frequency);
+	} else {
+		offset = 0;
+	}
 
 	/* default mclk value 96.4285 * 2 * 1000 = 192857 */
 	if (((c->frequency % 192857) >= (192857 - 3000)) ||
diff --git a/drivers/media/dvb-frontends/m88rs2000.h b/drivers/media/dvb-frontends/m88rs2000.h
index 1a313b0f5875..b015872c4ff4 100644
--- a/drivers/media/dvb-frontends/m88rs2000.h
+++ b/drivers/media/dvb-frontends/m88rs2000.h
@@ -21,7 +21,7 @@
 #define M88RS2000_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct m88rs2000_config {
 	/* Demodulator i2c address */
diff --git a/drivers/media/dvb-frontends/mb86a16.c b/drivers/media/dvb-frontends/mb86a16.c
index dfe322eccaa1..2969ba6ed9e1 100644
--- a/drivers/media/dvb-frontends/mb86a16.c
+++ b/drivers/media/dvb-frontends/mb86a16.c
@@ -24,7 +24,7 @@
 #include <linux/moduleparam.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mb86a16.h"
 #include "mb86a16_priv.h"
 
@@ -635,6 +635,7 @@ static int sync_chk(struct mb86a16_state *state,
 	return sync;
 err:
 	dprintk(verbose, MB86A16_ERROR, 1, "I2C transfer error");
+	*VIRM = 0;
 	return -EREMOTEIO;
 
 }
@@ -1676,15 +1677,15 @@ static int mb86a16_read_ber(struct dvb_frontend *fe, u32 *ber)
 			 * the deinterleaver output.
 			 * monitored BER is expressed as a 20 bit output in total
 			 */
-			ber_rst = ber_mon >> 3;
+			ber_rst = (ber_mon >> 3) & 0x03;
 			*ber = (((ber_msb << 8) | ber_mid) << 8) | ber_lsb;
 			if (ber_rst == 0)
 				timer =  12500000;
-			if (ber_rst == 1)
+			else if (ber_rst == 1)
 				timer =  25000000;
-			if (ber_rst == 2)
+			else if (ber_rst == 2)
 				timer =  50000000;
-			if (ber_rst == 3)
+			else /* ber_rst == 3 */
 				timer = 100000000;
 
 			*ber /= timer;
@@ -1696,11 +1697,11 @@ static int mb86a16_read_ber(struct dvb_frontend *fe, u32 *ber)
 			 * QPSK demodulator output.
 			 * monitored BER is expressed as a 24 bit output in total
 			 */
-			ber_tim = ber_mon >> 1;
+			ber_tim = (ber_mon >> 1) & 0x01;
 			*ber = (((ber_msb << 8) | ber_mid) << 8) | ber_lsb;
 			if (ber_tim == 0)
 				timer = 16;
-			if (ber_tim == 1)
+			else /* ber_tim == 1 */
 				timer = 24;
 
 			*ber /= 2 ^ timer;
diff --git a/drivers/media/dvb-frontends/mb86a16.h b/drivers/media/dvb-frontends/mb86a16.h
index dbd5f43fa128..f13820bc7a21 100644
--- a/drivers/media/dvb-frontends/mb86a16.h
+++ b/drivers/media/dvb-frontends/mb86a16.h
@@ -22,7 +22,7 @@
 #define __MB86A16_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 
 struct mb86a16_config {
diff --git a/drivers/media/dvb-frontends/mb86a20s.c b/drivers/media/dvb-frontends/mb86a20s.c
index bdaf9d235fed..36e95196dff4 100644
--- a/drivers/media/dvb-frontends/mb86a20s.c
+++ b/drivers/media/dvb-frontends/mb86a20s.c
@@ -17,7 +17,7 @@
 #include <linux/kernel.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mb86a20s.h"
 
 #define NUM_LAYERS 3
@@ -2057,7 +2057,7 @@ static void mb86a20s_release(struct dvb_frontend *fe)
 
 static int mb86a20s_get_frontend_algo(struct dvb_frontend *fe)
 {
-        return DVBFE_ALGO_HW;
+	return DVBFE_ALGO_HW;
 }
 
 static const struct dvb_frontend_ops mb86a20s_ops;
diff --git a/drivers/media/dvb-frontends/mn88472_priv.h b/drivers/media/dvb-frontends/mn88472_priv.h
index fb50f56ba30b..2ec126a42527 100644
--- a/drivers/media/dvb-frontends/mn88472_priv.h
+++ b/drivers/media/dvb-frontends/mn88472_priv.h
@@ -17,8 +17,8 @@
 #ifndef MN88472_PRIV_H
 #define MN88472_PRIV_H
 
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "mn88472.h"
 #include <linux/firmware.h>
 #include <linux/regmap.h>
diff --git a/drivers/media/dvb-frontends/mn88473.c b/drivers/media/dvb-frontends/mn88473.c
index 58247432a628..ca722084e534 100644
--- a/drivers/media/dvb-frontends/mn88473.c
+++ b/drivers/media/dvb-frontends/mn88473.c
@@ -764,7 +764,7 @@ MODULE_DEVICE_TABLE(i2c, mn88473_id_table);
 
 static struct i2c_driver mn88473_driver = {
 	.driver = {
-		.name	             = "mn88473",
+		.name		     = "mn88473",
 		.suppress_bind_attrs = true,
 	},
 	.probe		= mn88473_probe,
diff --git a/drivers/media/dvb-frontends/mn88473_priv.h b/drivers/media/dvb-frontends/mn88473_priv.h
index 5fc463d147c8..d89a86320263 100644
--- a/drivers/media/dvb-frontends/mn88473_priv.h
+++ b/drivers/media/dvb-frontends/mn88473_priv.h
@@ -17,8 +17,8 @@
 #ifndef MN88473_PRIV_H
 #define MN88473_PRIV_H
 
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "mn88473.h"
 #include <linux/math64.h>
 #include <linux/firmware.h>
diff --git a/drivers/media/dvb-frontends/mt312.c b/drivers/media/dvb-frontends/mt312.c
index 961b9a2508e0..e2a3fc521620 100644
--- a/drivers/media/dvb-frontends/mt312.c
+++ b/drivers/media/dvb-frontends/mt312.c
@@ -32,7 +32,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mt312_priv.h"
 #include "mt312.h"
 
@@ -142,7 +142,10 @@ static inline int mt312_readreg(struct mt312_state *state,
 static inline int mt312_writereg(struct mt312_state *state,
 				 const enum mt312_reg_addr reg, const u8 val)
 {
-	return mt312_write(state, reg, &val, 1);
+	u8 tmp = val; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+
+	return mt312_write(state, reg, &tmp, 1);
 }
 
 static inline u32 mt312_div(u32 a, u32 b)
diff --git a/drivers/media/dvb-frontends/mt352.c b/drivers/media/dvb-frontends/mt352.c
index d5fa96f0a6cd..a440b76444d3 100644
--- a/drivers/media/dvb-frontends/mt352.c
+++ b/drivers/media/dvb-frontends/mt352.c
@@ -33,7 +33,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mt352_priv.h"
 #include "mt352.h"
 
diff --git a/drivers/media/dvb-frontends/mxl5xx.c b/drivers/media/dvb-frontends/mxl5xx.c
index 53064e11f5f1..e899821018a0 100644
--- a/drivers/media/dvb-frontends/mxl5xx.c
+++ b/drivers/media/dvb-frontends/mxl5xx.c
@@ -33,7 +33,7 @@
 #include <asm/div64.h>
 #include <asm/unaligned.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mxl5xx.h"
 #include "mxl5xx_regs.h"
 #include "mxl5xx_defs.h"
@@ -636,16 +636,9 @@ static int tune(struct dvb_frontend *fe, bool re_tune,
 		if (r)
 			return r;
 		state->tune_time = jiffies;
-		return 0;
 	}
-	if (*status & FE_HAS_LOCK)
-		return 0;
 
-	r = read_status(fe, status);
-	if (r)
-		return r;
-
-	return 0;
+	return read_status(fe, status);
 }
 
 static enum fe_code_rate conv_fec(enum MXL_HYDRA_FEC_E fec)
diff --git a/drivers/media/dvb-frontends/mxl5xx.h b/drivers/media/dvb-frontends/mxl5xx.h
index 532e08111537..ad4c21846800 100644
--- a/drivers/media/dvb-frontends/mxl5xx.h
+++ b/drivers/media/dvb-frontends/mxl5xx.h
@@ -4,7 +4,7 @@
 #include <linux/types.h>
 #include <linux/i2c.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct mxl5xx_cfg {
 	u8   adr;
diff --git a/drivers/media/dvb-frontends/nxt200x.c b/drivers/media/dvb-frontends/nxt200x.c
index bf6e5cd572c5..7aa74403648e 100644
--- a/drivers/media/dvb-frontends/nxt200x.c
+++ b/drivers/media/dvb-frontends/nxt200x.c
@@ -48,7 +48,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "nxt200x.h"
 
 struct nxt200x_state {
diff --git a/drivers/media/dvb-frontends/nxt6000.c b/drivers/media/dvb-frontends/nxt6000.c
index 1ce5ea28489b..109a635d166a 100644
--- a/drivers/media/dvb-frontends/nxt6000.c
+++ b/drivers/media/dvb-frontends/nxt6000.c
@@ -27,7 +27,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "nxt6000_priv.h"
 #include "nxt6000.h"
 
diff --git a/drivers/media/dvb-frontends/or51132.c b/drivers/media/dvb-frontends/or51132.c
index 5f2549c48eb0..b4c9aadcb552 100644
--- a/drivers/media/dvb-frontends/or51132.c
+++ b/drivers/media/dvb-frontends/or51132.c
@@ -38,8 +38,8 @@
 #include <linux/slab.h>
 #include <asm/byteorder.h>
 
-#include "dvb_math.h"
-#include "dvb_frontend.h"
+#include <media/dvb_math.h>
+#include <media/dvb_frontend.h>
 #include "or51132.h"
 
 static int debug;
diff --git a/drivers/media/dvb-frontends/or51211.c b/drivers/media/dvb-frontends/or51211.c
index d14fa9736ae5..a1b7c301828f 100644
--- a/drivers/media/dvb-frontends/or51211.c
+++ b/drivers/media/dvb-frontends/or51211.c
@@ -36,8 +36,8 @@
 #include <linux/slab.h>
 #include <asm/byteorder.h>
 
-#include "dvb_math.h"
-#include "dvb_frontend.h"
+#include <media/dvb_math.h>
+#include <media/dvb_frontend.h>
 #include "or51211.h"
 
 static int debug;
diff --git a/drivers/media/dvb-frontends/rtl2830_priv.h b/drivers/media/dvb-frontends/rtl2830_priv.h
index 8ec4721d79ac..72d3f3546ff2 100644
--- a/drivers/media/dvb-frontends/rtl2830_priv.h
+++ b/drivers/media/dvb-frontends/rtl2830_priv.h
@@ -18,8 +18,8 @@
 #ifndef RTL2830_PRIV_H
 #define RTL2830_PRIV_H
 
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "rtl2830.h"
 #include <linux/i2c-mux.h>
 #include <linux/math64.h>
diff --git a/drivers/media/dvb-frontends/rtl2832_priv.h b/drivers/media/dvb-frontends/rtl2832_priv.h
index 9a6d01a9c690..bd13d9ad9ab7 100644
--- a/drivers/media/dvb-frontends/rtl2832_priv.h
+++ b/drivers/media/dvb-frontends/rtl2832_priv.h
@@ -26,8 +26,8 @@
 #include <linux/math64.h>
 #include <linux/bitops.h>
 
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "rtl2832.h"
 
 struct rtl2832_dev {
diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.h b/drivers/media/dvb-frontends/rtl2832_sdr.h
index 8f88c2fb8627..d28735c1cb0c 100644
--- a/drivers/media/dvb-frontends/rtl2832_sdr.h
+++ b/drivers/media/dvb-frontends/rtl2832_sdr.h
@@ -27,7 +27,7 @@
 
 #include <linux/i2c.h>
 #include <media/v4l2-subdev.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /**
  * struct rtl2832_sdr_platform_data - Platform data for the rtl2832_sdr driver
diff --git a/drivers/media/dvb-frontends/s5h1409.c b/drivers/media/dvb-frontends/s5h1409.c
index f370c6df0a8b..aced6a956ec5 100644
--- a/drivers/media/dvb-frontends/s5h1409.c
+++ b/drivers/media/dvb-frontends/s5h1409.c
@@ -25,7 +25,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "s5h1409.h"
 
 struct s5h1409_state {
diff --git a/drivers/media/dvb-frontends/s5h1411.c b/drivers/media/dvb-frontends/s5h1411.c
index dd09336a135b..c4b1e9725f3e 100644
--- a/drivers/media/dvb-frontends/s5h1411.c
+++ b/drivers/media/dvb-frontends/s5h1411.c
@@ -25,7 +25,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "s5h1411.h"
 
 struct s5h1411_state {
diff --git a/drivers/media/dvb-frontends/s5h1420.c b/drivers/media/dvb-frontends/s5h1420.c
index fd427a29c001..8b2222530227 100644
--- a/drivers/media/dvb-frontends/s5h1420.c
+++ b/drivers/media/dvb-frontends/s5h1420.c
@@ -30,7 +30,7 @@
 #include <linux/i2c.h>
 
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "s5h1420.h"
 #include "s5h1420_priv.h"
 
diff --git a/drivers/media/dvb-frontends/s5h1432.c b/drivers/media/dvb-frontends/s5h1432.c
index 4de50fe0c638..740a60df0455 100644
--- a/drivers/media/dvb-frontends/s5h1432.c
+++ b/drivers/media/dvb-frontends/s5h1432.c
@@ -20,7 +20,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "s5h1432.h"
 
 struct s5h1432_state {
diff --git a/drivers/media/dvb-frontends/s921.c b/drivers/media/dvb-frontends/s921.c
index 274544a3ae0e..2d75ede77aca 100644
--- a/drivers/media/dvb-frontends/s921.c
+++ b/drivers/media/dvb-frontends/s921.c
@@ -25,7 +25,7 @@
 #include <linux/kernel.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "s921.h"
 
 static int debug = 1;
diff --git a/drivers/media/dvb-frontends/si2165.c b/drivers/media/dvb-frontends/si2165.c
index 528b82a5dd46..2dd336f95cbf 100644
--- a/drivers/media/dvb-frontends/si2165.c
+++ b/drivers/media/dvb-frontends/si2165.c
@@ -1,7 +1,7 @@
 /*
  *  Driver for Silicon Labs Si2161 DVB-T and Si2165 DVB-C/-T Demodulator
  *
- *  Copyright (C) 2013-2014 Matthias Schwarzott <zzam@gentoo.org>
+ *  Copyright (C) 2013-2017 Matthias Schwarzott <zzam@gentoo.org>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -27,8 +27,8 @@
 #include <linux/firmware.h>
 #include <linux/regmap.h>
 
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "si2165_priv.h"
 #include "si2165.h"
 
@@ -57,63 +57,21 @@ struct si2165_state {
 	u32 sys_clk;
 	u32 adc_clk;
 
+	/* DVBv3 stats */
+	u64 ber_prev;
+
 	bool has_dvbc;
 	bool has_dvbt;
 	bool firmware_loaded;
 };
 
-#define DEBUG_OTHER	0x01
-#define DEBUG_I2C_WRITE	0x02
-#define DEBUG_I2C_READ	0x04
-#define DEBUG_REG_READ	0x08
-#define DEBUG_REG_WRITE	0x10
-#define DEBUG_FW_LOAD	0x20
-
-static int debug = 0x00;
-
-#define dprintk(args...) \
-	do { \
-		if (debug & DEBUG_OTHER) \
-			printk(KERN_DEBUG "si2165: " args); \
-	} while (0)
-
-#define deb_i2c_write(args...) \
-	do { \
-		if (debug & DEBUG_I2C_WRITE) \
-			printk(KERN_DEBUG "si2165: i2c write: " args); \
-	} while (0)
-
-#define deb_i2c_read(args...) \
-	do { \
-		if (debug & DEBUG_I2C_READ) \
-			printk(KERN_DEBUG "si2165: i2c read: " args); \
-	} while (0)
-
-#define deb_readreg(args...) \
-	do { \
-		if (debug & DEBUG_REG_READ) \
-			printk(KERN_DEBUG "si2165: reg read: " args); \
-	} while (0)
-
-#define deb_writereg(args...) \
-	do { \
-		if (debug & DEBUG_REG_WRITE) \
-			printk(KERN_DEBUG "si2165: reg write: " args); \
-	} while (0)
-
-#define deb_fw_load(args...) \
-	do { \
-		if (debug & DEBUG_FW_LOAD) \
-			printk(KERN_DEBUG "si2165: fw load: " args); \
-	} while (0)
-
 static int si2165_write(struct si2165_state *state, const u16 reg,
-		       const u8 *src, const int count)
+			const u8 *src, const int count)
 {
 	int ret;
 
-	if (debug & DEBUG_I2C_WRITE)
-		deb_i2c_write("reg: 0x%04x, data: %*ph\n", reg, count, src);
+	dev_dbg(&state->client->dev, "i2c write: reg: 0x%04x, data: %*ph\n",
+		reg, count, src);
 
 	ret = regmap_bulk_write(state->regmap, reg, src, count);
 
@@ -134,30 +92,41 @@ static int si2165_read(struct si2165_state *state,
 		return ret;
 	}
 
-	if (debug & DEBUG_I2C_READ)
-		deb_i2c_read("reg: 0x%04x, data: %*ph\n", reg, count, val);
+	dev_dbg(&state->client->dev, "i2c read: reg: 0x%04x, data: %*ph\n",
+		reg, count, val);
 
 	return 0;
 }
 
 static int si2165_readreg8(struct si2165_state *state,
-		       const u16 reg, u8 *val)
+			   const u16 reg, u8 *val)
 {
 	unsigned int val_tmp;
 	int ret = regmap_read(state->regmap, reg, &val_tmp);
 	*val = (u8)val_tmp;
-	deb_readreg("R(0x%04x)=0x%02x\n", reg, *val);
+	dev_dbg(&state->client->dev, "reg read: R(0x%04x)=0x%02x\n", reg, *val);
 	return ret;
 }
 
 static int si2165_readreg16(struct si2165_state *state,
-		       const u16 reg, u16 *val)
+			    const u16 reg, u16 *val)
 {
 	u8 buf[2];
 
 	int ret = si2165_read(state, reg, buf, 2);
 	*val = buf[0] | buf[1] << 8;
-	deb_readreg("R(0x%04x)=0x%04x\n", reg, *val);
+	dev_dbg(&state->client->dev, "reg read: R(0x%04x)=0x%04x\n", reg, *val);
+	return ret;
+}
+
+static int si2165_readreg24(struct si2165_state *state,
+			    const u16 reg, u32 *val)
+{
+	u8 buf[3];
+
+	int ret = si2165_read(state, reg, buf, 3);
+	*val = buf[0] | buf[1] << 8 | buf[2] << 16;
+	dev_dbg(&state->client->dev, "reg read: R(0x%04x)=0x%06x\n", reg, *val);
 	return ret;
 }
 
@@ -208,7 +177,9 @@ static int si2165_writereg_mask8(struct si2165_state *state, const u16 reg,
 	return si2165_writereg8(state, reg, val);
 }
 
-#define REG16(reg, val) { (reg), (val) & 0xff }, { (reg)+1, (val)>>8 & 0xff }
+#define REG16(reg, val) \
+	{ (reg), (val) & 0xff }, \
+	{ (reg) + 1, (val) >> 8 & 0xff }
 struct si2165_reg_value_pair {
 	u16 reg;
 	u8 val;
@@ -238,7 +209,7 @@ static int si2165_get_tune_settings(struct dvb_frontend *fe,
 
 static int si2165_init_pll(struct si2165_state *state)
 {
-	u32 ref_freq_Hz = state->config.ref_freq_Hz;
+	u32 ref_freq_hz = state->config.ref_freq_hz;
 	u8 divr = 1; /* 1..7 */
 	u8 divp = 1; /* only 1 or 4 */
 	u8 divn = 56; /* 1..63 */
@@ -250,7 +221,7 @@ static int si2165_init_pll(struct si2165_state *state)
 	 * hardcoded values can be deleted if calculation is verified
 	 * or it yields the same values as the windows driver
 	 */
-	switch (ref_freq_Hz) {
+	switch (ref_freq_hz) {
 	case 16000000u:
 		divn = 56;
 		break;
@@ -261,39 +232,39 @@ static int si2165_init_pll(struct si2165_state *state)
 		break;
 	default:
 		/* ref_freq / divr must be between 4 and 16 MHz */
-		if (ref_freq_Hz > 16000000u)
+		if (ref_freq_hz > 16000000u)
 			divr = 2;
 
 		/*
 		 * now select divn and divp such that
 		 * fvco is in 1624..1824 MHz
 		 */
-		if (1624000000u * divr > ref_freq_Hz * 2u * 63u)
+		if (1624000000u * divr > ref_freq_hz * 2u * 63u)
 			divp = 4;
 
 		/* is this already correct regarding rounding? */
-		divn = 1624000000u * divr / (ref_freq_Hz * 2u * divp);
+		divn = 1624000000u * divr / (ref_freq_hz * 2u * divp);
 		break;
 	}
 
 	/* adc_clk and sys_clk depend on xtal and pll settings */
-	state->fvco_hz = ref_freq_Hz / divr
+	state->fvco_hz = ref_freq_hz / divr
 			* 2u * divn * divp;
 	state->adc_clk = state->fvco_hz / (divm * 4u);
 	state->sys_clk = state->fvco_hz / (divl * 2u);
 
-	/* write pll registers 0x00a0..0x00a3 at once */
+	/* write all 4 pll registers 0x00a0..0x00a3 at once */
 	buf[0] = divl;
 	buf[1] = divm;
 	buf[2] = (divn & 0x3f) | ((divp == 1) ? 0x40 : 0x00) | 0x80;
 	buf[3] = divr;
-	return si2165_write(state, 0x00a0, buf, 4);
+	return si2165_write(state, REG_PLL_DIVL, buf, 4);
 }
 
 static int si2165_adjust_pll_divl(struct si2165_state *state, u8 divl)
 {
 	state->sys_clk = state->fvco_hz / (divl * 2u);
-	return si2165_writereg8(state, 0x00a0, divl); /* pll_divl */
+	return si2165_writereg8(state, REG_PLL_DIVL, divl);
 }
 
 static u32 si2165_get_fe_clk(struct si2165_state *state)
@@ -309,18 +280,18 @@ static int si2165_wait_init_done(struct si2165_state *state)
 	int i;
 
 	for (i = 0; i < 3; ++i) {
-		si2165_readreg8(state, 0x0054, &val);
+		si2165_readreg8(state, REG_INIT_DONE, &val);
 		if (val == 0x01)
 			return 0;
 		usleep_range(1000, 50000);
 	}
-	dev_err(&state->client->dev, "%s: init_done was not set\n",
-		KBUILD_MODNAME);
+	dev_err(&state->client->dev, "init_done was not set\n");
 	return ret;
 }
 
 static int si2165_upload_firmware_block(struct si2165_state *state,
-	const u8 *data, u32 len, u32 *poffset, u32 block_count)
+					const u8 *data, u32 len, u32 *poffset,
+					u32 block_count)
 {
 	int ret;
 	u8 buf_ctrl[4] = { 0x00, 0x00, 0x00, 0xc0 };
@@ -333,42 +304,43 @@ static int si2165_upload_firmware_block(struct si2165_state *state,
 	if (len % 4 != 0)
 		return -EINVAL;
 
-	deb_fw_load(
-		"si2165_upload_firmware_block called with len=0x%x offset=0x%x blockcount=0x%x\n",
-				len, offset, block_count);
-	while (offset+12 <= len && cur_block < block_count) {
-		deb_fw_load(
-			"si2165_upload_firmware_block in while len=0x%x offset=0x%x cur_block=0x%x blockcount=0x%x\n",
-					len, offset, cur_block, block_count);
+	dev_dbg(&state->client->dev,
+		"fw load: %s: called with len=0x%x offset=0x%x blockcount=0x%x\n",
+		__func__, len, offset, block_count);
+	while (offset + 12 <= len && cur_block < block_count) {
+		dev_dbg(&state->client->dev,
+			"fw load: %s: in while len=0x%x offset=0x%x cur_block=0x%x blockcount=0x%x\n",
+			__func__, len, offset, cur_block, block_count);
 		wordcount = data[offset];
-		if (wordcount < 1 || data[offset+1] ||
-		    data[offset+2] || data[offset+3]) {
+		if (wordcount < 1 || data[offset + 1] ||
+		    data[offset + 2] || data[offset + 3]) {
 			dev_warn(&state->client->dev,
-				 "%s: bad fw data[0..3] = %*ph\n",
-				KBUILD_MODNAME, 4, data);
+				 "bad fw data[0..3] = %*ph\n",
+				 4, data);
 			return -EINVAL;
 		}
 
 		if (offset + 8 + wordcount * 4 > len) {
 			dev_warn(&state->client->dev,
-				 "%s: len is too small for block len=%d, wordcount=%d\n",
-				KBUILD_MODNAME, len, wordcount);
+				 "len is too small for block len=%d, wordcount=%d\n",
+				len, wordcount);
 			return -EINVAL;
 		}
 
 		buf_ctrl[0] = wordcount - 1;
 
-		ret = si2165_write(state, 0x0364, buf_ctrl, 4);
+		ret = si2165_write(state, REG_DCOM_CONTROL_BYTE, buf_ctrl, 4);
 		if (ret < 0)
 			goto error;
-		ret = si2165_write(state, 0x0368, data+offset+4, 4);
+		ret = si2165_write(state, REG_DCOM_ADDR, data + offset + 4, 4);
 		if (ret < 0)
 			goto error;
 
 		offset += 8;
 
 		while (wordcount > 0) {
-			ret = si2165_write(state, 0x36c, data+offset, 4);
+			ret = si2165_write(state, REG_DCOM_DATA,
+					   data + offset, 4);
 			if (ret < 0)
 				goto error;
 			wordcount--;
@@ -377,15 +349,16 @@ static int si2165_upload_firmware_block(struct si2165_state *state,
 		cur_block++;
 	}
 
-	deb_fw_load(
-		"si2165_upload_firmware_block after while len=0x%x offset=0x%x cur_block=0x%x blockcount=0x%x\n",
-				len, offset, cur_block, block_count);
+	dev_dbg(&state->client->dev,
+		"fw load: %s: after while len=0x%x offset=0x%x cur_block=0x%x blockcount=0x%x\n",
+		__func__, len, offset, cur_block, block_count);
 
 	if (poffset)
 		*poffset = offset;
 
-	deb_fw_load("si2165_upload_firmware_block returned offset=0x%x\n",
-				offset);
+	dev_dbg(&state->client->dev,
+		"fw load: %s: returned offset=0x%x\n",
+		__func__, offset);
 
 	return 0;
 error:
@@ -413,43 +386,40 @@ static int si2165_upload_firmware(struct si2165_state *state)
 		fw_file = SI2165_FIRMWARE_REV_D;
 		break;
 	default:
-		dev_info(&state->client->dev, "%s: no firmware file for revision=%d\n",
-			KBUILD_MODNAME, state->chip_revcode);
+		dev_info(&state->client->dev, "no firmware file for revision=%d\n",
+			 state->chip_revcode);
 		return 0;
 	}
 
 	/* request the firmware, this will block and timeout */
 	ret = request_firmware(&fw, fw_file, &state->client->dev);
 	if (ret) {
-		dev_warn(&state->client->dev, "%s: firmware file '%s' not found\n",
-				KBUILD_MODNAME, fw_file);
+		dev_warn(&state->client->dev, "firmware file '%s' not found\n",
+			 fw_file);
 		goto error;
 	}
 
 	data = fw->data;
 	len = fw->size;
 
-	dev_info(&state->client->dev, "%s: downloading firmware from file '%s' size=%d\n",
-			KBUILD_MODNAME, fw_file, len);
+	dev_info(&state->client->dev, "downloading firmware from file '%s' size=%d\n",
+		 fw_file, len);
 
 	if (len % 4 != 0) {
-		dev_warn(&state->client->dev, "%s: firmware size is not multiple of 4\n",
-				KBUILD_MODNAME);
+		dev_warn(&state->client->dev, "firmware size is not multiple of 4\n");
 		ret = -EINVAL;
 		goto error;
 	}
 
 	/* check header (8 bytes) */
 	if (len < 8) {
-		dev_warn(&state->client->dev, "%s: firmware header is missing\n",
-				KBUILD_MODNAME);
+		dev_warn(&state->client->dev, "firmware header is missing\n");
 		ret = -EINVAL;
 		goto error;
 	}
 
 	if (data[0] != 1 || data[1] != 0) {
-		dev_warn(&state->client->dev, "%s: firmware file version is wrong\n",
-				KBUILD_MODNAME);
+		dev_warn(&state->client->dev, "firmware file version is wrong\n");
 		ret = -EINVAL;
 		goto error;
 	}
@@ -460,45 +430,45 @@ static int si2165_upload_firmware(struct si2165_state *state)
 
 	/* start uploading fw */
 	/* boot/wdog status */
-	ret = si2165_writereg8(state, 0x0341, 0x00);
+	ret = si2165_writereg8(state, REG_WDOG_AND_BOOT, 0x00);
 	if (ret < 0)
 		goto error;
 	/* reset */
-	ret = si2165_writereg8(state, 0x00c0, 0x00);
+	ret = si2165_writereg8(state, REG_RST_ALL, 0x00);
 	if (ret < 0)
 		goto error;
 	/* boot/wdog status */
-	ret = si2165_readreg8(state, 0x0341, val);
+	ret = si2165_readreg8(state, REG_WDOG_AND_BOOT, val);
 	if (ret < 0)
 		goto error;
 
 	/* enable reset on error */
-	ret = si2165_readreg8(state, 0x035c, val);
+	ret = si2165_readreg8(state, REG_EN_RST_ERROR, val);
 	if (ret < 0)
 		goto error;
-	ret = si2165_readreg8(state, 0x035c, val);
+	ret = si2165_readreg8(state, REG_EN_RST_ERROR, val);
 	if (ret < 0)
 		goto error;
-	ret = si2165_writereg8(state, 0x035c, 0x02);
+	ret = si2165_writereg8(state, REG_EN_RST_ERROR, 0x02);
 	if (ret < 0)
 		goto error;
 
 	/* start right after the header */
 	offset = 8;
 
-	dev_info(&state->client->dev, "%s: si2165_upload_firmware extracted patch_version=0x%02x, block_count=0x%02x, crc_expected=0x%04x\n",
-		KBUILD_MODNAME, patch_version, block_count, crc_expected);
+	dev_info(&state->client->dev, "%s: extracted patch_version=0x%02x, block_count=0x%02x, crc_expected=0x%04x\n",
+		 __func__, patch_version, block_count, crc_expected);
 
 	ret = si2165_upload_firmware_block(state, data, len, &offset, 1);
 	if (ret < 0)
 		goto error;
 
-	ret = si2165_writereg8(state, 0x0344, patch_version);
+	ret = si2165_writereg8(state, REG_PATCH_VERSION, patch_version);
 	if (ret < 0)
 		goto error;
 
 	/* reset crc */
-	ret = si2165_writereg8(state, 0x0379, 0x01);
+	ret = si2165_writereg8(state, REG_RST_CRC, 0x01);
 	if (ret)
 		goto error;
 
@@ -506,20 +476,19 @@ static int si2165_upload_firmware(struct si2165_state *state)
 					   &offset, block_count);
 	if (ret < 0) {
 		dev_err(&state->client->dev,
-			"%s: firmware could not be uploaded\n",
-			KBUILD_MODNAME);
+			"firmware could not be uploaded\n");
 		goto error;
 	}
 
 	/* read crc */
-	ret = si2165_readreg16(state, 0x037a, &val16);
+	ret = si2165_readreg16(state, REG_CRC, &val16);
 	if (ret)
 		goto error;
 
 	if (val16 != crc_expected) {
 		dev_err(&state->client->dev,
-			"%s: firmware crc mismatch %04x != %04x\n",
-			KBUILD_MODNAME, val16, crc_expected);
+			"firmware crc mismatch %04x != %04x\n",
+			val16, crc_expected);
 		ret = -EINVAL;
 		goto error;
 	}
@@ -530,23 +499,23 @@ static int si2165_upload_firmware(struct si2165_state *state)
 
 	if (len != offset) {
 		dev_err(&state->client->dev,
-			"%s: firmware len mismatch %04x != %04x\n",
-			KBUILD_MODNAME, len, offset);
+			"firmware len mismatch %04x != %04x\n",
+			len, offset);
 		ret = -EINVAL;
 		goto error;
 	}
 
 	/* reset watchdog error register */
-	ret = si2165_writereg_mask8(state, 0x0341, 0x02, 0x02);
+	ret = si2165_writereg_mask8(state, REG_WDOG_AND_BOOT, 0x02, 0x02);
 	if (ret < 0)
 		goto error;
 
 	/* enable reset on error */
-	ret = si2165_writereg_mask8(state, 0x035c, 0x01, 0x01);
+	ret = si2165_writereg_mask8(state, REG_EN_RST_ERROR, 0x01, 0x01);
 	if (ret < 0)
 		goto error;
 
-	dev_info(&state->client->dev, "%s: fw load finished\n", KBUILD_MODNAME);
+	dev_info(&state->client->dev, "fw load finished\n");
 
 	ret = 0;
 	state->firmware_loaded = true;
@@ -563,46 +532,47 @@ static int si2165_init(struct dvb_frontend *fe)
 {
 	int ret = 0;
 	struct si2165_state *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
 	u8 val;
 	u8 patch_version = 0x00;
 
-	dprintk("%s: called\n", __func__);
+	dev_dbg(&state->client->dev, "%s: called\n", __func__);
 
 	/* powerup */
-	ret = si2165_writereg8(state, 0x0000, state->config.chip_mode);
+	ret = si2165_writereg8(state, REG_CHIP_MODE, state->config.chip_mode);
 	if (ret < 0)
 		goto error;
 	/* dsp_clock_enable */
-	ret = si2165_writereg8(state, 0x0104, 0x01);
+	ret = si2165_writereg8(state, REG_DSP_CLOCK, 0x01);
 	if (ret < 0)
 		goto error;
-	ret = si2165_readreg8(state, 0x0000, &val); /* verify chip_mode */
+	/* verify chip_mode */
+	ret = si2165_readreg8(state, REG_CHIP_MODE, &val);
 	if (ret < 0)
 		goto error;
 	if (val != state->config.chip_mode) {
-		dev_err(&state->client->dev, "%s: could not set chip_mode\n",
-			KBUILD_MODNAME);
+		dev_err(&state->client->dev, "could not set chip_mode\n");
 		return -EINVAL;
 	}
 
 	/* agc */
-	ret = si2165_writereg8(state, 0x018b, 0x00);
+	ret = si2165_writereg8(state, REG_AGC_IF_TRI, 0x00);
 	if (ret < 0)
 		goto error;
-	ret = si2165_writereg8(state, 0x0190, 0x01);
+	ret = si2165_writereg8(state, REG_AGC_IF_SLR, 0x01);
 	if (ret < 0)
 		goto error;
-	ret = si2165_writereg8(state, 0x0170, 0x00);
+	ret = si2165_writereg8(state, REG_AGC2_OUTPUT, 0x00);
 	if (ret < 0)
 		goto error;
-	ret = si2165_writereg8(state, 0x0171, 0x07);
+	ret = si2165_writereg8(state, REG_AGC2_CLKDIV, 0x07);
 	if (ret < 0)
 		goto error;
 	/* rssi pad */
-	ret = si2165_writereg8(state, 0x0646, 0x00);
+	ret = si2165_writereg8(state, REG_RSSI_PAD_CTRL, 0x00);
 	if (ret < 0)
 		goto error;
-	ret = si2165_writereg8(state, 0x0641, 0x00);
+	ret = si2165_writereg8(state, REG_RSSI_ENABLE, 0x00);
 	if (ret < 0)
 		goto error;
 
@@ -611,11 +581,11 @@ static int si2165_init(struct dvb_frontend *fe)
 		goto error;
 
 	/* enable chip_init */
-	ret = si2165_writereg8(state, 0x0050, 0x01);
+	ret = si2165_writereg8(state, REG_CHIP_INIT, 0x01);
 	if (ret < 0)
 		goto error;
 	/* set start_init */
-	ret = si2165_writereg8(state, 0x0096, 0x01);
+	ret = si2165_writereg8(state, REG_START_INIT, 0x01);
 	if (ret < 0)
 		goto error;
 	ret = si2165_wait_init_done(state);
@@ -623,29 +593,30 @@ static int si2165_init(struct dvb_frontend *fe)
 		goto error;
 
 	/* disable chip_init */
-	ret = si2165_writereg8(state, 0x0050, 0x00);
+	ret = si2165_writereg8(state, REG_CHIP_INIT, 0x00);
 	if (ret < 0)
 		goto error;
 
-	/* ber_pkt */
-	ret = si2165_writereg16(state, 0x0470, 0x7530);
+	/* ber_pkt - default 65535 */
+	ret = si2165_writereg16(state, REG_BER_PKT,
+				STATISTICS_PERIOD_PKT_COUNT);
 	if (ret < 0)
 		goto error;
 
-	ret = si2165_readreg8(state, 0x0344, &patch_version);
+	ret = si2165_readreg8(state, REG_PATCH_VERSION, &patch_version);
 	if (ret < 0)
 		goto error;
 
-	ret = si2165_writereg8(state, 0x00cb, 0x00);
+	ret = si2165_writereg8(state, REG_AUTO_RESET, 0x00);
 	if (ret < 0)
 		goto error;
 
 	/* dsp_addr_jump */
-	ret = si2165_writereg32(state, 0x0348, 0xf4000000);
+	ret = si2165_writereg32(state, REG_ADDR_JUMP, 0xf4000000);
 	if (ret < 0)
 		goto error;
 	/* boot/wdog status */
-	ret = si2165_readreg8(state, 0x0341, &val);
+	ret = si2165_readreg8(state, REG_WDOG_AND_BOOT, &val);
 	if (ret < 0)
 		goto error;
 
@@ -656,19 +627,30 @@ static int si2165_init(struct dvb_frontend *fe)
 	}
 
 	/* ts output config */
-	ret = si2165_writereg8(state, 0x04e4, 0x20);
+	ret = si2165_writereg8(state, REG_TS_DATA_MODE, 0x20);
+	if (ret < 0)
+		return ret;
+	ret = si2165_writereg16(state, REG_TS_TRI, 0x00fe);
 	if (ret < 0)
 		return ret;
-	ret = si2165_writereg16(state, 0x04ef, 0x00fe);
+	ret = si2165_writereg24(state, REG_TS_SLR, 0x555555);
 	if (ret < 0)
 		return ret;
-	ret = si2165_writereg24(state, 0x04f4, 0x555555);
+	ret = si2165_writereg8(state, REG_TS_CLK_MODE, 0x01);
 	if (ret < 0)
 		return ret;
-	ret = si2165_writereg8(state, 0x04e5, 0x01);
+	ret = si2165_writereg8(state, REG_TS_PARALLEL_MODE, 0x00);
 	if (ret < 0)
 		return ret;
 
+	c = &state->fe.dtv_property_cache;
+	c->cnr.len = 1;
+	c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	c->post_bit_error.len = 1;
+	c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	c->post_bit_count.len = 1;
+	c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+
 	return 0;
 error:
 	return ret;
@@ -680,11 +662,11 @@ static int si2165_sleep(struct dvb_frontend *fe)
 	struct si2165_state *state = fe->demodulator_priv;
 
 	/* dsp clock disable */
-	ret = si2165_writereg8(state, 0x0104, 0x00);
+	ret = si2165_writereg8(state, REG_DSP_CLOCK, 0x00);
 	if (ret < 0)
 		return ret;
 	/* chip mode */
-	ret = si2165_writereg8(state, 0x0000, SI2165_MODE_OFF);
+	ret = si2165_writereg8(state, REG_CHIP_MODE, SI2165_MODE_OFF);
 	if (ret < 0)
 		return ret;
 	return 0;
@@ -693,18 +675,48 @@ static int si2165_sleep(struct dvb_frontend *fe)
 static int si2165_read_status(struct dvb_frontend *fe, enum fe_status *status)
 {
 	int ret;
-	u8 fec_lock = 0;
+	u8 u8tmp;
+	u32 u32tmp;
 	struct si2165_state *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+	u32 delsys = c->delivery_system;
 
-	if (!state->has_dvbt)
-		return -EINVAL;
+	*status = 0;
+
+	switch (delsys) {
+	case SYS_DVBT:
+		/* check fast signal type */
+		ret = si2165_readreg8(state, REG_CHECK_SIGNAL, &u8tmp);
+		if (ret < 0)
+			return ret;
+		switch (u8tmp & 0x3) {
+		case 0: /* searching */
+		case 1: /* nothing */
+			break;
+		case 2: /* digital signal */
+			*status |= FE_HAS_SIGNAL | FE_HAS_CARRIER;
+			break;
+		}
+		break;
+	case SYS_DVBC_ANNEX_A:
+		/* check packet sync lock */
+		ret = si2165_readreg8(state, REG_PS_LOCK, &u8tmp);
+		if (ret < 0)
+			return ret;
+		if (u8tmp & 0x01) {
+			*status |= FE_HAS_SIGNAL;
+			*status |= FE_HAS_CARRIER;
+			*status |= FE_HAS_VITERBI;
+			*status |= FE_HAS_SYNC;
+		}
+		break;
+	}
 
 	/* check fec_lock */
-	ret = si2165_readreg8(state, 0x4e0, &fec_lock);
+	ret = si2165_readreg8(state, REG_FEC_LOCK, &u8tmp);
 	if (ret < 0)
 		return ret;
-	*status = 0;
-	if (fec_lock & 0x01) {
+	if (u8tmp & 0x01) {
 		*status |= FE_HAS_SIGNAL;
 		*status |= FE_HAS_CARRIER;
 		*status |= FE_HAS_VITERBI;
@@ -712,6 +724,109 @@ static int si2165_read_status(struct dvb_frontend *fe, enum fe_status *status)
 		*status |= FE_HAS_LOCK;
 	}
 
+	/* CNR */
+	if (delsys == SYS_DVBC_ANNEX_A && *status & FE_HAS_VITERBI) {
+		ret = si2165_readreg24(state, REG_C_N, &u32tmp);
+		if (ret < 0)
+			return ret;
+		/*
+		 * svalue =
+		 * 1000 * c_n/dB =
+		 * 1000 * 10 * log10(2^24 / regval) =
+		 * 1000 * 10 * (log10(2^24) - log10(regval)) =
+		 * 1000 * 10 * (intlog10(2^24) - intlog10(regval)) / 2^24
+		 *
+		 * intlog10(x) = log10(x) * 2^24
+		 * intlog10(2^24) = log10(2^24) * 2^24 = 121210686
+		 */
+		u32tmp = (1000 * 10 * (121210686 - (u64)intlog10(u32tmp)))
+				>> 24;
+		c->cnr.stat[0].scale = FE_SCALE_DECIBEL;
+		c->cnr.stat[0].svalue = u32tmp;
+	} else
+		c->cnr.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+
+	/* BER */
+	if (*status & FE_HAS_VITERBI) {
+		if (c->post_bit_error.stat[0].scale == FE_SCALE_NOT_AVAILABLE) {
+			/* start new sampling period to get rid of old data*/
+			ret = si2165_writereg8(state, REG_BER_RST, 0x01);
+			if (ret < 0)
+				return ret;
+
+			/* set scale to enter read code on next call */
+			c->post_bit_error.stat[0].scale = FE_SCALE_COUNTER;
+			c->post_bit_count.stat[0].scale = FE_SCALE_COUNTER;
+			c->post_bit_error.stat[0].uvalue = 0;
+			c->post_bit_count.stat[0].uvalue = 0;
+
+			/*
+			 * reset DVBv3 value to deliver a good result
+			 * for the first call
+			 */
+			state->ber_prev = 0;
+
+		} else {
+			ret = si2165_readreg8(state, REG_BER_AVAIL, &u8tmp);
+			if (ret < 0)
+				return ret;
+
+			if (u8tmp & 1) {
+				u32 biterrcnt;
+
+				ret = si2165_readreg24(state, REG_BER_BIT,
+							&biterrcnt);
+				if (ret < 0)
+					return ret;
+
+				c->post_bit_error.stat[0].uvalue +=
+					biterrcnt;
+				c->post_bit_count.stat[0].uvalue +=
+					STATISTICS_PERIOD_BIT_COUNT;
+
+				/* start new sampling period */
+				ret = si2165_writereg8(state,
+							REG_BER_RST, 0x01);
+				if (ret < 0)
+					return ret;
+
+				dev_dbg(&state->client->dev,
+					"post_bit_error=%u post_bit_count=%u\n",
+					biterrcnt, STATISTICS_PERIOD_BIT_COUNT);
+			}
+		}
+	} else {
+		c->post_bit_error.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+		c->post_bit_count.stat[0].scale = FE_SCALE_NOT_AVAILABLE;
+	}
+
+	return 0;
+}
+
+static int si2165_read_snr(struct dvb_frontend *fe, u16 *snr)
+{
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+
+	if (c->cnr.stat[0].scale == FE_SCALE_DECIBEL)
+		*snr = div_s64(c->cnr.stat[0].svalue, 100);
+	else
+		*snr = 0;
+	return 0;
+}
+
+static int si2165_read_ber(struct dvb_frontend *fe, u32 *ber)
+{
+	struct si2165_state *state = fe->demodulator_priv;
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+
+	if (c->post_bit_error.stat[0].scale != FE_SCALE_COUNTER) {
+		*ber = 0;
+		return 0;
+	}
+
+	*ber = c->post_bit_error.stat[0].uvalue - state->ber_prev;
+	state->ber_prev = c->post_bit_error.stat[0].uvalue;
+
 	return 0;
 }
 
@@ -728,8 +843,8 @@ static int si2165_set_oversamp(struct si2165_state *state, u32 dvb_rate)
 	do_div(oversamp, dvb_rate);
 	reg_value = oversamp & 0x3fffffff;
 
-	dprintk("%s: Write oversamp=%#x\n", __func__, reg_value);
-	return si2165_writereg32(state, 0x00e4, reg_value);
+	dev_dbg(&state->client->dev, "Write oversamp=%#x\n", reg_value);
+	return si2165_writereg32(state, REG_OVERSAMP, reg_value);
 }
 
 static int si2165_set_if_freq_shift(struct si2165_state *state)
@@ -742,8 +857,7 @@ static int si2165_set_if_freq_shift(struct si2165_state *state)
 
 	if (!fe->ops.tuner_ops.get_if_frequency) {
 		dev_err(&state->client->dev,
-			"%s: Error: get_if_frequency() not defined at tuner. Can't work without it!\n",
-			KBUILD_MODNAME);
+			"Error: get_if_frequency() not defined at tuner. Can't work without it!\n");
 		return -EINVAL;
 	}
 
@@ -763,30 +877,29 @@ static int si2165_set_if_freq_shift(struct si2165_state *state)
 	reg_value = reg_value & 0x1fffffff;
 
 	/* if_freq_shift, usbdump contained 0x023ee08f; */
-	return si2165_writereg32(state, 0x00e8, reg_value);
+	return si2165_writereg32(state, REG_IF_FREQ_SHIFT, reg_value);
 }
 
 static const struct si2165_reg_value_pair dvbt_regs[] = {
 	/* standard = DVB-T */
-	{ 0x00ec, 0x01 },
-	{ 0x08f8, 0x00 },
+	{ REG_DVB_STANDARD, 0x01 },
 	/* impulsive_noise_remover */
-	{ 0x031c, 0x01 },
-	{ 0x00cb, 0x00 },
+	{ REG_IMPULSIVE_NOISE_REM, 0x01 },
+	{ REG_AUTO_RESET, 0x00 },
 	/* agc2 */
-	{ 0x016e, 0x41 },
-	{ 0x016c, 0x0e },
-	{ 0x016d, 0x10 },
+	{ REG_AGC2_MIN, 0x41 },
+	{ REG_AGC2_KACQ, 0x0e },
+	{ REG_AGC2_KLOC, 0x10 },
 	/* agc */
-	{ 0x015b, 0x03 },
-	{ 0x0150, 0x78 },
+	{ REG_AGC_UNFREEZE_THR, 0x03 },
+	{ REG_AGC_CRESTF_DBX8, 0x78 },
 	/* agc */
-	{ 0x01a0, 0x78 },
-	{ 0x01c8, 0x68 },
+	{ REG_AAF_CRESTF_DBX8, 0x78 },
+	{ REG_ACI_CRESTF_DBX8, 0x68 },
 	/* freq_sync_range */
-	REG16(0x030c, 0x0064),
+	REG16(REG_FREQ_SYNC_RANGE, 0x0064),
 	/* gp_reg0 */
-	{ 0x0387, 0x00 }
+	{ REG_GP_REG0_MSB, 0x00 }
 };
 
 static int si2165_set_frontend_dvbt(struct dvb_frontend *fe)
@@ -798,7 +911,7 @@ static int si2165_set_frontend_dvbt(struct dvb_frontend *fe)
 	u16 bw10k;
 	u32 bw_hz = p->bandwidth_hz;
 
-	dprintk("%s: called\n", __func__);
+	dev_dbg(&state->client->dev, "%s: called\n", __func__);
 
 	if (!state->has_dvbt)
 		return -EINVAL;
@@ -815,7 +928,7 @@ static int si2165_set_frontend_dvbt(struct dvb_frontend *fe)
 		return ret;
 
 	/* bandwidth in 10KHz steps */
-	ret = si2165_writereg16(state, 0x0308, bw10k);
+	ret = si2165_writereg16(state, REG_T_BANDWIDTH, bw10k);
 	if (ret < 0)
 		return ret;
 	ret = si2165_set_oversamp(state, dvb_rate);
@@ -831,33 +944,30 @@ static int si2165_set_frontend_dvbt(struct dvb_frontend *fe)
 
 static const struct si2165_reg_value_pair dvbc_regs[] = {
 	/* standard = DVB-C */
-	{ 0x00ec, 0x05 },
-	{ 0x08f8, 0x00 },
+	{ REG_DVB_STANDARD, 0x05 },
 
 	/* agc2 */
-	{ 0x016e, 0x50 },
-	{ 0x016c, 0x0e },
-	{ 0x016d, 0x10 },
+	{ REG_AGC2_MIN, 0x50 },
+	{ REG_AGC2_KACQ, 0x0e },
+	{ REG_AGC2_KLOC, 0x10 },
 	/* agc */
-	{ 0x015b, 0x03 },
-	{ 0x0150, 0x68 },
+	{ REG_AGC_UNFREEZE_THR, 0x03 },
+	{ REG_AGC_CRESTF_DBX8, 0x68 },
 	/* agc */
-	{ 0x01a0, 0x68 },
-	{ 0x01c8, 0x50 },
-
-	{ 0x0278, 0x0d },
-
-	{ 0x023a, 0x05 },
-	{ 0x0261, 0x09 },
-	REG16(0x0350, 0x3e80),
-	{ 0x02f4, 0x00 },
-
-	{ 0x00cb, 0x01 },
-	REG16(0x024c, 0x0000),
-	REG16(0x027c, 0x0000),
-	{ 0x0232, 0x03 },
-	{ 0x02f4, 0x0b },
-	{ 0x018b, 0x00 },
+	{ REG_AAF_CRESTF_DBX8, 0x68 },
+	{ REG_ACI_CRESTF_DBX8, 0x50 },
+
+	{ REG_EQ_AUTO_CONTROL, 0x0d },
+
+	{ REG_KP_LOCK, 0x05 },
+	{ REG_CENTRAL_TAP, 0x09 },
+	REG16(REG_UNKNOWN_350, 0x3e80),
+
+	{ REG_AUTO_RESET, 0x01 },
+	REG16(REG_UNKNOWN_24C, 0x0000),
+	REG16(REG_UNKNOWN_27C, 0x0000),
+	{ REG_SWEEP_STEP, 0x03 },
+	{ REG_AGC_IF_TRI, 0x00 },
 };
 
 static int si2165_set_frontend_dvbc(struct dvb_frontend *fe)
@@ -866,7 +976,7 @@ static int si2165_set_frontend_dvbc(struct dvb_frontend *fe)
 	int ret;
 	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
 	const u32 dvb_rate = p->symbol_rate;
-	const u32 bw_hz = p->bandwidth_hz;
+	u8 u8tmp;
 
 	if (!state->has_dvbc)
 		return -EINVAL;
@@ -883,7 +993,32 @@ static int si2165_set_frontend_dvbc(struct dvb_frontend *fe)
 	if (ret < 0)
 		return ret;
 
-	ret = si2165_writereg32(state, 0x00c4, bw_hz);
+	switch (p->modulation) {
+	case QPSK:
+		u8tmp = 0x3;
+		break;
+	case QAM_16:
+		u8tmp = 0x7;
+		break;
+	case QAM_32:
+		u8tmp = 0x8;
+		break;
+	case QAM_64:
+		u8tmp = 0x9;
+		break;
+	case QAM_128:
+		u8tmp = 0xa;
+		break;
+	case QAM_256:
+	default:
+		u8tmp = 0xb;
+		break;
+	}
+	ret = si2165_writereg8(state, REG_REQ_CONSTELLATION, u8tmp);
+	if (ret < 0)
+		return ret;
+
+	ret = si2165_writereg32(state, REG_LOCK_TIMEOUT, 0x007a1200);
 	if (ret < 0)
 		return ret;
 
@@ -894,12 +1029,12 @@ static int si2165_set_frontend_dvbc(struct dvb_frontend *fe)
 	return 0;
 }
 
-static const struct si2165_reg_value_pair agc_rewrite[] = {
-	{ 0x012a, 0x46 },
-	{ 0x012c, 0x00 },
-	{ 0x012e, 0x0a },
-	{ 0x012f, 0xff },
-	{ 0x0123, 0x70 }
+static const struct si2165_reg_value_pair adc_rewrite[] = {
+	{ REG_ADC_RI1, 0x46 },
+	{ REG_ADC_RI3, 0x00 },
+	{ REG_ADC_RI5, 0x0a },
+	{ REG_ADC_RI6, 0xff },
+	{ REG_ADC_RI8, 0x70 }
 };
 
 static int si2165_set_frontend(struct dvb_frontend *fe)
@@ -931,7 +1066,7 @@ static int si2165_set_frontend(struct dvb_frontend *fe)
 	}
 
 	/* dsp_addr_jump */
-	ret = si2165_writereg32(state, 0x0348, 0xf4000000);
+	ret = si2165_writereg32(state, REG_ADDR_JUMP, 0xf4000000);
 	if (ret < 0)
 		return ret;
 
@@ -944,34 +1079,34 @@ static int si2165_set_frontend(struct dvb_frontend *fe)
 		return ret;
 
 	/* boot/wdog status */
-	ret = si2165_readreg8(state, 0x0341, val);
+	ret = si2165_readreg8(state, REG_WDOG_AND_BOOT, val);
 	if (ret < 0)
 		return ret;
-	ret = si2165_writereg8(state, 0x0341, 0x00);
+	ret = si2165_writereg8(state, REG_WDOG_AND_BOOT, 0x00);
 	if (ret < 0)
 		return ret;
 
 	/* reset all */
-	ret = si2165_writereg8(state, 0x00c0, 0x00);
+	ret = si2165_writereg8(state, REG_RST_ALL, 0x00);
 	if (ret < 0)
 		return ret;
 	/* gp_reg0 */
-	ret = si2165_writereg32(state, 0x0384, 0x00000000);
+	ret = si2165_writereg32(state, REG_GP_REG0_LSB, 0x00000000);
 	if (ret < 0)
 		return ret;
 
 	/* write adc values after each reset*/
-	ret = si2165_write_reg_list(state, agc_rewrite,
-				    ARRAY_SIZE(agc_rewrite));
+	ret = si2165_write_reg_list(state, adc_rewrite,
+				    ARRAY_SIZE(adc_rewrite));
 	if (ret < 0)
 		return ret;
 
 	/* start_synchro */
-	ret = si2165_writereg8(state, 0x02e0, 0x01);
+	ret = si2165_writereg8(state, REG_START_SYNCHRO, 0x01);
 	if (ret < 0)
 		return ret;
 	/* boot/wdog status */
-	ret = si2165_readreg8(state, 0x0341, val);
+	ret = si2165_readreg8(state, REG_WDOG_AND_BOOT, val);
 	if (ret < 0)
 		return ret;
 
@@ -998,7 +1133,6 @@ static const struct dvb_frontend_ops si2165_ops = {
 			FE_CAN_QAM_64 |
 			FE_CAN_QAM_128 |
 			FE_CAN_QAM_256 |
-			FE_CAN_QAM_AUTO |
 			FE_CAN_GUARD_INTERVAL_AUTO |
 			FE_CAN_HIERARCHY_AUTO |
 			FE_CAN_MUTE_TS |
@@ -1013,10 +1147,12 @@ static const struct dvb_frontend_ops si2165_ops = {
 
 	.set_frontend      = si2165_set_frontend,
 	.read_status       = si2165_read_status,
+	.read_snr          = si2165_read_snr,
+	.read_ber          = si2165_read_ber,
 };
 
 static int si2165_probe(struct i2c_client *client,
-		const struct i2c_device_id *id)
+			const struct i2c_device_id *id)
 {
 	struct si2165_state *state = NULL;
 	struct si2165_platform_data *pdata = client->dev.platform_data;
@@ -1032,8 +1168,8 @@ static int si2165_probe(struct i2c_client *client,
 	};
 
 	/* allocate memory for the internal state */
-	state = kzalloc(sizeof(struct si2165_state), GFP_KERNEL);
-	if (state == NULL) {
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state) {
 		ret = -ENOMEM;
 		goto error;
 	}
@@ -1049,45 +1185,45 @@ static int si2165_probe(struct i2c_client *client,
 	state->client = client;
 	state->config.i2c_addr = client->addr;
 	state->config.chip_mode = pdata->chip_mode;
-	state->config.ref_freq_Hz = pdata->ref_freq_Hz;
+	state->config.ref_freq_hz = pdata->ref_freq_hz;
 	state->config.inversion = pdata->inversion;
 
-	if (state->config.ref_freq_Hz < 4000000
-	    || state->config.ref_freq_Hz > 27000000) {
-		dev_err(&state->client->dev, "%s: ref_freq of %d Hz not supported by this driver\n",
-			 KBUILD_MODNAME, state->config.ref_freq_Hz);
+	if (state->config.ref_freq_hz < 4000000 ||
+	    state->config.ref_freq_hz > 27000000) {
+		dev_err(&state->client->dev, "ref_freq of %d Hz not supported by this driver\n",
+			state->config.ref_freq_hz);
 		ret = -EINVAL;
 		goto error;
 	}
 
 	/* create dvb_frontend */
 	memcpy(&state->fe.ops, &si2165_ops,
-		sizeof(struct dvb_frontend_ops));
+	       sizeof(struct dvb_frontend_ops));
 	state->fe.ops.release = NULL;
 	state->fe.demodulator_priv = state;
 	i2c_set_clientdata(client, state);
 
 	/* powerup */
-	ret = si2165_writereg8(state, 0x0000, state->config.chip_mode);
+	ret = si2165_writereg8(state, REG_CHIP_MODE, state->config.chip_mode);
 	if (ret < 0)
 		goto nodev_error;
 
-	ret = si2165_readreg8(state, 0x0000, &val);
+	ret = si2165_readreg8(state, REG_CHIP_MODE, &val);
 	if (ret < 0)
 		goto nodev_error;
 	if (val != state->config.chip_mode)
 		goto nodev_error;
 
-	ret = si2165_readreg8(state, 0x0023, &state->chip_revcode);
+	ret = si2165_readreg8(state, REG_CHIP_REVCODE, &state->chip_revcode);
 	if (ret < 0)
 		goto nodev_error;
 
-	ret = si2165_readreg8(state, 0x0118, &state->chip_type);
+	ret = si2165_readreg8(state, REV_CHIP_TYPE, &state->chip_type);
 	if (ret < 0)
 		goto nodev_error;
 
 	/* powerdown */
-	ret = si2165_writereg8(state, 0x0000, SI2165_MODE_OFF);
+	ret = si2165_writereg8(state, REG_CHIP_MODE, SI2165_MODE_OFF);
 	if (ret < 0)
 		goto nodev_error;
 
@@ -1107,18 +1243,18 @@ static int si2165_probe(struct i2c_client *client,
 		state->has_dvbc = true;
 		break;
 	default:
-		dev_err(&state->client->dev, "%s: Unsupported Silicon Labs chip (type %d, rev %d)\n",
-			KBUILD_MODNAME, state->chip_type, state->chip_revcode);
+		dev_err(&state->client->dev, "Unsupported Silicon Labs chip (type %d, rev %d)\n",
+			state->chip_type, state->chip_revcode);
 		goto nodev_error;
 	}
 
 	dev_info(&state->client->dev,
-		"%s: Detected Silicon Labs %s-%c (type %d, rev %d)\n",
-		KBUILD_MODNAME, chip_name, rev_char, state->chip_type,
+		 "Detected Silicon Labs %s-%c (type %d, rev %d)\n",
+		chip_name, rev_char, state->chip_type,
 		state->chip_revcode);
 
 	strlcat(state->fe.ops.info.name, chip_name,
-			sizeof(state->fe.ops.info.name));
+		sizeof(state->fe.ops.info.name));
 
 	n = 0;
 	if (state->has_dvbt) {
@@ -1173,9 +1309,6 @@ static struct i2c_driver si2165_driver = {
 
 module_i2c_driver(si2165_driver);
 
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "Turn on/off frontend debugging (default:off).");
-
 MODULE_DESCRIPTION("Silicon Labs Si2165 DVB-C/-T Demodulator driver");
 MODULE_AUTHOR("Matthias Schwarzott <zzam@gentoo.org>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/media/dvb-frontends/si2165.h b/drivers/media/dvb-frontends/si2165.h
index 76c2ca7d7edb..74a57b7ecd26 100644
--- a/drivers/media/dvb-frontends/si2165.h
+++ b/drivers/media/dvb-frontends/si2165.h
@@ -1,21 +1,22 @@
 /*
-    Driver for Silicon Labs SI2165 DVB-C/-T Demodulator
-
-    Copyright (C) 2013-2014 Matthias Schwarzott <zzam@gentoo.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    References:
-    http://www.silabs.com/Support%20Documents/TechnicalDocs/Si2165-short.pdf
-*/
+ * Driver for Silicon Labs SI2165 DVB-C/-T Demodulator
+ *
+ * Copyright (C) 2013-2017 Matthias Schwarzott <zzam@gentoo.org>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ * References:
+ *   http://www.silabs.com/Support%20Documents/TechnicalDocs/Si2165-short.pdf
+ *
+ */
 
 #ifndef _DVB_SI2165_H
 #define _DVB_SI2165_H
@@ -44,7 +45,7 @@ struct si2165_platform_data {
 	/* frequency of external clock or xtal in Hz
 	 * possible values: 4000000, 16000000, 20000000, 240000000, 27000000
 	 */
-	u32 ref_freq_Hz;
+	u32 ref_freq_hz;
 
 	/* invert the spectrum */
 	bool inversion;
diff --git a/drivers/media/dvb-frontends/si2165_priv.h b/drivers/media/dvb-frontends/si2165_priv.h
index e5932118834b..8c6fbfe441ff 100644
--- a/drivers/media/dvb-frontends/si2165_priv.h
+++ b/drivers/media/dvb-frontends/si2165_priv.h
@@ -1,19 +1,19 @@
 /*
-    Driver for Silicon Labs SI2165 DVB-C/-T Demodulator
-
-    Copyright (C) 2013-2014 Matthias Schwarzott <zzam@gentoo.org>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-*/
+ * Driver for Silicon Labs SI2165 DVB-C/-T Demodulator
+ *
+ * Copyright (C) 2013-2017 Matthias Schwarzott <zzam@gentoo.org>
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ */
 
 #ifndef _DVB_SI2165_PRIV
 #define _DVB_SI2165_PRIV
@@ -22,7 +22,8 @@
 
 struct si2165_config {
 	/* i2c addr
-	 * possible values: 0x64,0x65,0x66,0x67 */
+	 * possible values: 0x64,0x65,0x66,0x67
+	 */
 	u8 i2c_addr;
 
 	/* external clock or XTAL */
@@ -31,10 +32,89 @@ struct si2165_config {
 	/* frequency of external clock or xtal in Hz
 	 * possible values: 4000000, 16000000, 20000000, 240000000, 27000000
 	 */
-	u32 ref_freq_Hz;
+	u32 ref_freq_hz;
 
 	/* invert the spectrum */
 	bool inversion;
 };
 
+#define STATISTICS_PERIOD_PKT_COUNT	30000u
+#define STATISTICS_PERIOD_BIT_COUNT	(STATISTICS_PERIOD_PKT_COUNT * 204 * 8)
+
+#define REG_CHIP_MODE			0x0000
+#define REG_CHIP_REVCODE		0x0023
+#define REV_CHIP_TYPE			0x0118
+#define REG_CHIP_INIT			0x0050
+#define REG_INIT_DONE			0x0054
+#define REG_START_INIT			0x0096
+#define REG_PLL_DIVL			0x00a0
+#define REG_RST_ALL			0x00c0
+#define REG_LOCK_TIMEOUT		0x00c4
+#define REG_AUTO_RESET			0x00cb
+#define REG_OVERSAMP			0x00e4
+#define REG_IF_FREQ_SHIFT		0x00e8
+#define REG_DVB_STANDARD		0x00ec
+#define REG_DSP_CLOCK			0x0104
+#define REG_ADC_RI8			0x0123
+#define REG_ADC_RI1			0x012a
+#define REG_ADC_RI2			0x012b
+#define REG_ADC_RI3			0x012c
+#define REG_ADC_RI4			0x012d
+#define REG_ADC_RI5			0x012e
+#define REG_ADC_RI6			0x012f
+#define REG_AGC_CRESTF_DBX8		0x0150
+#define REG_AGC_UNFREEZE_THR		0x015b
+#define REG_AGC2_MIN			0x016e
+#define REG_AGC2_KACQ			0x016c
+#define REG_AGC2_KLOC			0x016d
+#define REG_AGC2_OUTPUT			0x0170
+#define REG_AGC2_CLKDIV			0x0171
+#define REG_AGC_IF_TRI			0x018b
+#define REG_AGC_IF_SLR			0x0190
+#define REG_AAF_CRESTF_DBX8		0x01a0
+#define REG_ACI_CRESTF_DBX8		0x01c8
+#define REG_SWEEP_STEP			0x0232
+#define REG_KP_LOCK			0x023a
+#define REG_UNKNOWN_24C			0x024c
+#define REG_CENTRAL_TAP			0x0261
+#define REG_C_N				0x026c
+#define REG_EQ_AUTO_CONTROL		0x0278
+#define REG_UNKNOWN_27C			0x027c
+#define REG_START_SYNCHRO		0x02e0
+#define REG_REQ_CONSTELLATION		0x02f4
+#define REG_T_BANDWIDTH			0x0308
+#define REG_FREQ_SYNC_RANGE		0x030c
+#define REG_IMPULSIVE_NOISE_REM		0x031c
+#define REG_WDOG_AND_BOOT		0x0341
+#define REG_PATCH_VERSION		0x0344
+#define REG_ADDR_JUMP			0x0348
+#define REG_UNKNOWN_350			0x0350
+#define REG_EN_RST_ERROR		0x035c
+#define REG_DCOM_CONTROL_BYTE		0x0364
+#define REG_DCOM_ADDR			0x0368
+#define REG_DCOM_DATA			0x036c
+#define REG_RST_CRC			0x0379
+#define REG_GP_REG0_LSB			0x0384
+#define REG_GP_REG0_MSB			0x0387
+#define REG_CRC				0x037a
+#define REG_CHECK_SIGNAL		0x03a8
+#define REG_CBER_RST			0x0424
+#define REG_CBER_BIT			0x0428
+#define REG_CBER_ERR			0x0430
+#define REG_CBER_AVAIL			0x0434
+#define REG_PS_LOCK			0x0440
+#define REG_UNCOR_CNT			0x0468
+#define REG_BER_RST			0x046c
+#define REG_BER_PKT			0x0470
+#define REG_BER_BIT			0x0478
+#define REG_BER_AVAIL			0x047c
+#define REG_FEC_LOCK			0x04e0
+#define REG_TS_DATA_MODE		0x04e4
+#define REG_TS_CLK_MODE			0x04e5
+#define REG_TS_TRI			0x04ef
+#define REG_TS_SLR			0x04f4
+#define REG_RSSI_ENABLE			0x0641
+#define REG_RSSI_PAD_CTRL		0x0646
+#define REG_TS_PARALLEL_MODE		0x08f8
+
 #endif /* _DVB_SI2165_PRIV */
diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c
index 41d9c513b7e8..539399dac551 100644
--- a/drivers/media/dvb-frontends/si2168.c
+++ b/drivers/media/dvb-frontends/si2168.c
@@ -14,6 +14,8 @@
  *    GNU General Public License for more details.
  */
 
+#include <linux/delay.h>
+
 #include "si2168_priv.h"
 
 static const struct dvb_frontend_ops si2168_ops;
@@ -435,6 +437,7 @@ static int si2168_init(struct dvb_frontend *fe)
 		if (ret)
 			goto err;
 
+		udelay(100);
 		memcpy(cmd.args, "\x85", 1);
 		cmd.wlen = 1;
 		cmd.rlen = 1;
diff --git a/drivers/media/dvb-frontends/si2168_priv.h b/drivers/media/dvb-frontends/si2168_priv.h
index 737cf416fbb2..3c8746a20038 100644
--- a/drivers/media/dvb-frontends/si2168_priv.h
+++ b/drivers/media/dvb-frontends/si2168_priv.h
@@ -18,7 +18,7 @@
 #define SI2168_PRIV_H
 
 #include "si2168.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include <linux/firmware.h>
 #include <linux/i2c-mux.h>
 #include <linux/kernel.h>
diff --git a/drivers/media/dvb-frontends/si21xx.c b/drivers/media/dvb-frontends/si21xx.c
index 4e8c3ac4303f..9b32a1b3205e 100644
--- a/drivers/media/dvb-frontends/si21xx.c
+++ b/drivers/media/dvb-frontends/si21xx.c
@@ -16,7 +16,7 @@
 #include <linux/jiffies.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "si21xx.h"
 
 #define	REVISION_REG			0x00
diff --git a/drivers/media/dvb-frontends/si21xx.h b/drivers/media/dvb-frontends/si21xx.h
index 43d480bb6ea2..12fa1d579820 100644
--- a/drivers/media/dvb-frontends/si21xx.h
+++ b/drivers/media/dvb-frontends/si21xx.h
@@ -3,7 +3,7 @@
 #define SI21XX_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct si21xx_config {
 	/* the demodulator's i2c address */
diff --git a/drivers/media/dvb-frontends/sp2.h b/drivers/media/dvb-frontends/sp2.h
index 3901cd74b3f7..1bf60b80566f 100644
--- a/drivers/media/dvb-frontends/sp2.h
+++ b/drivers/media/dvb-frontends/sp2.h
@@ -17,7 +17,7 @@
 #ifndef SP2_H
 #define SP2_H
 
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 /*
  * I2C address
diff --git a/drivers/media/dvb-frontends/sp2_priv.h b/drivers/media/dvb-frontends/sp2_priv.h
index 37fef7bcd63f..c9ee53073ec0 100644
--- a/drivers/media/dvb-frontends/sp2_priv.h
+++ b/drivers/media/dvb-frontends/sp2_priv.h
@@ -18,7 +18,7 @@
 #define SP2_PRIV_H
 
 #include "sp2.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /* state struct */
 struct sp2 {
diff --git a/drivers/media/dvb-frontends/sp8870.c b/drivers/media/dvb-frontends/sp8870.c
index 04454cb78467..9a726f3a4896 100644
--- a/drivers/media/dvb-frontends/sp8870.c
+++ b/drivers/media/dvb-frontends/sp8870.c
@@ -35,7 +35,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "sp8870.h"
 
 
diff --git a/drivers/media/dvb-frontends/sp887x.c b/drivers/media/dvb-frontends/sp887x.c
index d2c402b52c6e..572a297811fe 100644
--- a/drivers/media/dvb-frontends/sp887x.c
+++ b/drivers/media/dvb-frontends/sp887x.c
@@ -17,7 +17,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "sp887x.h"
 
 
diff --git a/drivers/media/dvb-frontends/stb0899_algo.c b/drivers/media/dvb-frontends/stb0899_algo.c
index 3012f196e9bd..bd2defde7a77 100644
--- a/drivers/media/dvb-frontends/stb0899_algo.c
+++ b/drivers/media/dvb-frontends/stb0899_algo.c
@@ -925,8 +925,7 @@ static void stb0899_dvbs2_set_btr_loopbw(struct stb0899_state *state)
 		wn = (4 * zeta * zeta) + 1000000;
 		wn = (2 * (loopbw_percent * 1000) * 40 * zeta) /wn;  /*wn =wn 10^-8*/
 
-		k_indirect = (wn * wn) / K;
-		k_indirect = k_indirect;	  /*kindirect = kindirect 10^-6*/
+		k_indirect = (wn * wn) / K;	/*kindirect = kindirect 10^-6*/
 		k_direct   = (2 * wn * zeta) / K;	/*kDirect = kDirect 10^-2*/
 		k_direct  *= 100;
 
diff --git a/drivers/media/dvb-frontends/stb0899_drv.c b/drivers/media/dvb-frontends/stb0899_drv.c
index 02347598277a..3c654ae16e78 100644
--- a/drivers/media/dvb-frontends/stb0899_drv.c
+++ b/drivers/media/dvb-frontends/stb0899_drv.c
@@ -27,7 +27,7 @@
 #include <linux/string.h>
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "stb0899_drv.h"
 #include "stb0899_priv.h"
@@ -539,7 +539,8 @@ int stb0899_write_regs(struct stb0899_state *state, unsigned int reg, u8 *data,
 
 int stb0899_write_reg(struct stb0899_state *state, unsigned int reg, u8 data)
 {
-	return stb0899_write_regs(state, reg, &data, 1);
+	u8 tmp = data;
+	return stb0899_write_regs(state, reg, &tmp, 1);
 }
 
 /*
@@ -1582,15 +1583,15 @@ static enum dvbfe_algo stb0899_frontend_algo(struct dvb_frontend *fe)
 static const struct dvb_frontend_ops stb0899_ops = {
 	.delsys = { SYS_DVBS, SYS_DVBS2, SYS_DSS },
 	.info = {
-		.name 			= "STB0899 Multistandard",
+		.name			= "STB0899 Multistandard",
 		.frequency_min		= 950000,
-		.frequency_max 		= 2150000,
+		.frequency_max		= 2150000,
 		.frequency_stepsize	= 0,
 		.frequency_tolerance	= 0,
-		.symbol_rate_min 	=  5000000,
-		.symbol_rate_max 	= 45000000,
+		.symbol_rate_min	=  5000000,
+		.symbol_rate_max	= 45000000,
 
-		.caps 			= FE_CAN_INVERSION_AUTO	|
+		.caps			= FE_CAN_INVERSION_AUTO	|
 					  FE_CAN_FEC_AUTO	|
 					  FE_CAN_2G_MODULATION	|
 					  FE_CAN_QPSK
diff --git a/drivers/media/dvb-frontends/stb0899_drv.h b/drivers/media/dvb-frontends/stb0899_drv.h
index 0a72131a57db..f65f9a8266f8 100644
--- a/drivers/media/dvb-frontends/stb0899_drv.h
+++ b/drivers/media/dvb-frontends/stb0899_drv.h
@@ -25,7 +25,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define STB0899_TSMODE_SERIAL		1
 #define STB0899_CLKPOL_FALLING		2
@@ -82,7 +82,7 @@ enum stb0899_inversion {
  * 1. POWER ON/OFF		(index 0)
  * 2. FE_HAS_LOCK/LOCK_LOSS	(index 1)
  *
- * @gpio 	= one of the above listed GPIO's
+ * @gpio	= one of the above listed GPIO's
  * @level	= output state: pulled up or low
  */
 struct stb0899_postproc {
diff --git a/drivers/media/dvb-frontends/stb0899_priv.h b/drivers/media/dvb-frontends/stb0899_priv.h
index 82395b912815..3285cd1ba60a 100644
--- a/drivers/media/dvb-frontends/stb0899_priv.h
+++ b/drivers/media/dvb-frontends/stb0899_priv.h
@@ -22,7 +22,7 @@
 #ifndef __STB0899_PRIV_H
 #define __STB0899_PRIV_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "stb0899_drv.h"
 
 #define FE_ERROR				0
@@ -252,7 +252,7 @@ extern int stb0899_write_s2reg(struct stb0899_state *state,
 extern int stb0899_i2c_gate_ctrl(struct dvb_frontend *fe, int enable);
 
 
-#define STB0899_READ_S2REG(DEVICE, REG) 	(_stb0899_read_s2reg(state, DEVICE, STB0899_BASE_##REG, STB0899_OFF0_##REG))
+#define STB0899_READ_S2REG(DEVICE, REG)		(_stb0899_read_s2reg(state, DEVICE, STB0899_BASE_##REG, STB0899_OFF0_##REG))
 //#define STB0899_WRITE_S2REG(DEVICE, REG, DATA)	(_stb0899_write_s2reg(state, DEVICE, STB0899_BASE_##REG, STB0899_OFF0_##REG, DATA))
 
 /* stb0899_algo.c	*/
diff --git a/drivers/media/dvb-frontends/stb6000.h b/drivers/media/dvb-frontends/stb6000.h
index e94a3d5facf6..1adda72379ff 100644
--- a/drivers/media/dvb-frontends/stb6000.h
+++ b/drivers/media/dvb-frontends/stb6000.h
@@ -24,7 +24,7 @@
 #define __DVB_STB6000_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #if IS_REACHABLE(CONFIG_DVB_STB6000)
 /**
diff --git a/drivers/media/dvb-frontends/stb6100.c b/drivers/media/dvb-frontends/stb6100.c
index 17a955d0031b..3a851f524b16 100644
--- a/drivers/media/dvb-frontends/stb6100.c
+++ b/drivers/media/dvb-frontends/stb6100.c
@@ -25,7 +25,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "stb6100.h"
 
 static unsigned int verbose;
@@ -226,12 +226,14 @@ static int stb6100_write_reg_range(struct stb6100_state *state, u8 buf[], int st
 
 static int stb6100_write_reg(struct stb6100_state *state, u8 reg, u8 data)
 {
+	u8 tmp = data; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
 	if (unlikely(reg >= STB6100_NUMREGS)) {
 		dprintk(verbose, FE_ERROR, 1, "Invalid register offset 0x%x", reg);
 		return -EREMOTEIO;
 	}
-	data = (data & stb6100_template[reg].mask) | stb6100_template[reg].set;
-	return stb6100_write_reg_range(state, &data, reg, 1);
+	tmp = (tmp & stb6100_template[reg].mask) | stb6100_template[reg].set;
+	return stb6100_write_reg_range(state, &tmp, reg, 1);
 }
 
 
diff --git a/drivers/media/dvb-frontends/stb6100.h b/drivers/media/dvb-frontends/stb6100.h
index f7b468b6dc26..6cdae688a23e 100644
--- a/drivers/media/dvb-frontends/stb6100.h
+++ b/drivers/media/dvb-frontends/stb6100.h
@@ -23,7 +23,7 @@
 #define __STB_6100_REG_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define STB6100_LD			0x00
 #define STB6100_LD_LOCK			(1 << 0)
diff --git a/drivers/media/dvb-frontends/stb6100_cfg.h b/drivers/media/dvb-frontends/stb6100_cfg.h
index 2ef67aa768b9..203f9b36c0eb 100644
--- a/drivers/media/dvb-frontends/stb6100_cfg.h
+++ b/drivers/media/dvb-frontends/stb6100_cfg.h
@@ -20,7 +20,7 @@
 */
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 static int stb6100_get_frequency(struct dvb_frontend *fe, u32 *frequency)
 {
diff --git a/drivers/media/dvb-frontends/stb6100_proc.h b/drivers/media/dvb-frontends/stb6100_proc.h
index 50ffa21e3871..fad877b2fc7d 100644
--- a/drivers/media/dvb-frontends/stb6100_proc.h
+++ b/drivers/media/dvb-frontends/stb6100_proc.h
@@ -18,7 +18,7 @@
 */
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 static int stb6100_get_freq(struct dvb_frontend *fe, u32 *frequency)
 {
diff --git a/drivers/media/dvb-frontends/stv0288.c b/drivers/media/dvb-frontends/stv0288.c
index 67f91814b9f7..f947ed947aae 100644
--- a/drivers/media/dvb-frontends/stv0288.c
+++ b/drivers/media/dvb-frontends/stv0288.c
@@ -33,7 +33,7 @@
 #include <linux/jiffies.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "stv0288.h"
 
 struct stv0288_state {
diff --git a/drivers/media/dvb-frontends/stv0288.h b/drivers/media/dvb-frontends/stv0288.h
index 803acb917282..c10227aaa62c 100644
--- a/drivers/media/dvb-frontends/stv0288.h
+++ b/drivers/media/dvb-frontends/stv0288.h
@@ -28,7 +28,7 @@
 #define STV0288_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct stv0288_config {
 	/* the demodulator's i2c address */
diff --git a/drivers/media/dvb-frontends/stv0297.c b/drivers/media/dvb-frontends/stv0297.c
index db94d4d109f9..b823c04e24d3 100644
--- a/drivers/media/dvb-frontends/stv0297.c
+++ b/drivers/media/dvb-frontends/stv0297.c
@@ -27,7 +27,7 @@
 #include <linux/jiffies.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "stv0297.h"
 
 struct stv0297_state {
diff --git a/drivers/media/dvb-frontends/stv0297.h b/drivers/media/dvb-frontends/stv0297.h
index b30632a67333..8fa5ac700fc3 100644
--- a/drivers/media/dvb-frontends/stv0297.h
+++ b/drivers/media/dvb-frontends/stv0297.h
@@ -22,7 +22,7 @@
 #define STV0297_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct stv0297_config
 {
diff --git a/drivers/media/dvb-frontends/stv0299.c b/drivers/media/dvb-frontends/stv0299.c
index b1f3d675d316..633b90e6fe86 100644
--- a/drivers/media/dvb-frontends/stv0299.c
+++ b/drivers/media/dvb-frontends/stv0299.c
@@ -51,7 +51,7 @@
 #include <linux/jiffies.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "stv0299.h"
 
 struct stv0299_state {
diff --git a/drivers/media/dvb-frontends/stv0299.h b/drivers/media/dvb-frontends/stv0299.h
index 0aca30a8ec25..700c124a1699 100644
--- a/drivers/media/dvb-frontends/stv0299.h
+++ b/drivers/media/dvb-frontends/stv0299.h
@@ -46,7 +46,7 @@
 #define STV0299_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define STV0299_LOCKOUTPUT_0  0
 #define STV0299_LOCKOUTPUT_1  1
diff --git a/drivers/media/dvb-frontends/stv0367.c b/drivers/media/dvb-frontends/stv0367.c
index f3529df8211d..5435c908e298 100644
--- a/drivers/media/dvb-frontends/stv0367.c
+++ b/drivers/media/dvb-frontends/stv0367.c
@@ -25,7 +25,7 @@
 #include <linux/slab.h>
 #include <linux/i2c.h>
 
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 
 #include "stv0367.h"
 #include "stv0367_defs.h"
@@ -166,7 +166,9 @@ int stv0367_writeregs(struct stv0367_state *state, u16 reg, u8 *data, int len)
 
 static int stv0367_writereg(struct stv0367_state *state, u16 reg, u8 data)
 {
-	return stv0367_writeregs(state, reg, &data, 1);
+	u8 tmp = data; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return stv0367_writeregs(state, reg, &tmp, 1);
 }
 
 static u8 stv0367_readreg(struct stv0367_state *state, u16 reg)
@@ -1547,7 +1549,6 @@ static int stv0367ter_read_ber(struct dvb_frontend *fe, u32 *ber)
 	} else if (abc == 0x7) {
 		if (Errors <= 4) {
 			temporary = (Errors * 1000000000) / (8 * (1 << 14));
-			temporary =  temporary;
 		} else if (Errors <= 42) {
 			temporary = (Errors * 100000000) / (8 * (1 << 14));
 			temporary = temporary * 10;
@@ -1625,7 +1626,6 @@ static u32 stv0367ter_get_per(struct stv0367_state *state)
 	else if (abc == 0x9) {
 		if (Errors <= 4) {
 			temporary = (Errors * 1000000000) / (8 * (1 << 8));
-			temporary =  temporary;
 		} else if (Errors <= 42) {
 			temporary = (Errors * 100000000) / (8 * (1 << 8));
 			temporary = temporary * 10;
diff --git a/drivers/media/dvb-frontends/stv0367.h b/drivers/media/dvb-frontends/stv0367.h
index 8f7a31481744..14a50ecef6dd 100644
--- a/drivers/media/dvb-frontends/stv0367.h
+++ b/drivers/media/dvb-frontends/stv0367.h
@@ -23,7 +23,7 @@
 #define STV0367_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define STV0367_ICSPEED_53125	53125000
 #define STV0367_ICSPEED_58000	58000000
diff --git a/drivers/media/dvb-frontends/stv0900.h b/drivers/media/dvb-frontends/stv0900.h
index 1571a465e05c..5dbe1e550fe5 100644
--- a/drivers/media/dvb-frontends/stv0900.h
+++ b/drivers/media/dvb-frontends/stv0900.h
@@ -23,7 +23,7 @@
 #define STV0900_H
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct stv0900_reg {
 	u16 addr;
diff --git a/drivers/media/dvb-frontends/stv0900_core.c b/drivers/media/dvb-frontends/stv0900_core.c
index 0b739725e3c0..72f17b97ca04 100644
--- a/drivers/media/dvb-frontends/stv0900_core.c
+++ b/drivers/media/dvb-frontends/stv0900_core.c
@@ -1929,7 +1929,7 @@ struct dvb_frontend *stv0900_attach(const struct stv0900_config *config,
 	switch (demod) {
 	case 0:
 	case 1:
-		init_params.dmd_ref_clk  	= config->xtal;
+		init_params.dmd_ref_clk		= config->xtal;
 		init_params.demod_mode		= config->demod_mode;
 		init_params.rolloff		= STV0900_35;
 		init_params.path1_ts_clock	= config->path1_mode;
diff --git a/drivers/media/dvb-frontends/stv0900_init.h b/drivers/media/dvb-frontends/stv0900_init.h
index 411941442086..550ef4a0f654 100644
--- a/drivers/media/dvb-frontends/stv0900_init.h
+++ b/drivers/media/dvb-frontends/stv0900_init.h
@@ -148,8 +148,8 @@ struct stv0900_short_frames_car_loop_optim_vs_mod {
 
 /* Cut 1.x Tracking carrier loop carrier QPSK 1/2 to 8PSK 9/10 long Frame */
 static const struct stv0900_car_loop_optim FE_STV0900_S2CarLoop[14] = {
-	/*Modcod		2MPon 	2MPoff	5MPon 	5MPoff	10MPon
-				10MPoff	20MPon 	20MPoff	30MPon 	30MPoff */
+	/*Modcod		2MPon	2MPoff	5MPon	5MPoff	10MPon
+				10MPoff	20MPon	20MPoff	30MPon	30MPoff */
 	{ STV0900_QPSK_12,	0x1C,	0x0D,	0x1B,	0x2C,	0x3A,
 				0x1C,	0x2A,	0x3B,	0x2A,	0x1B },
 	{ STV0900_QPSK_35,	0x2C,	0x0D,	0x2B,	0x2C,	0x3A,
@@ -176,15 +176,15 @@ static const struct stv0900_car_loop_optim FE_STV0900_S2CarLoop[14] = {
 				0x0B,	0x39,	0x1A,	0x19,	0x0A },
 	{ STV0900_8PSK_89,	0x3B,	0x3B,	0x0B,	0x2B,	0x2A,
 				0x0B,	0x39,	0x1A,	0x29,	0x39 },
-	{ STV0900_8PSK_910,	0x3B,	0x3B, 	0x0B,	0x2B, 	0x2A,
+	{ STV0900_8PSK_910,	0x3B,	0x3B,	0x0B,	0x2B,	0x2A,
 				0x0B,	0x39,	0x1A,	0x29,	0x39 }
 };
 
 
 /* Cut 2.0 Tracking carrier loop carrier QPSK 1/2 to 8PSK 9/10 long Frame */
 static const struct stv0900_car_loop_optim FE_STV0900_S2CarLoopCut20[14] = {
-	/* Modcod		2MPon 	2MPoff	5MPon 	5MPoff	10MPon
-				10MPoff	20MPon 	20MPoff	30MPon 	30MPoff */
+	/* Modcod		2MPon	2MPoff	5MPon	5MPoff	10MPon
+				10MPoff	20MPon	20MPoff	30MPon	30MPoff */
 	{ STV0900_QPSK_12,	0x1F,	0x3F,	0x1E,	0x3F,	0x3D,
 				0x1F,	0x3D,	0x3E,	0x3D,	0x1E },
 	{ STV0900_QPSK_35,	0x2F,	0x3F,	0x2E,	0x2F,	0x3D,
@@ -211,7 +211,7 @@ static const struct stv0900_car_loop_optim FE_STV0900_S2CarLoopCut20[14] = {
 				0x1e,	0x3c,	0x2d,	0x2c,	0x1d },
 	{ STV0900_8PSK_89,	0x3e,	0x3e,	0x1e,	0x2e,	0x3d,
 				0x1e,	0x0d,	0x2d,	0x3c,	0x1d },
-	{ STV0900_8PSK_910,	0x3e,	0x3e, 	0x1e,	0x2e, 	0x3d,
+	{ STV0900_8PSK_910,	0x3e,	0x3e,	0x1e,	0x2e,	0x3d,
 				0x1e,	0x1d,	0x2d,	0x0d,	0x1d },
 };
 
@@ -219,8 +219,8 @@ static const struct stv0900_car_loop_optim FE_STV0900_S2CarLoopCut20[14] = {
 
 /* Cut 2.0 Tracking carrier loop carrier 16APSK 2/3 to 32APSK 9/10 long Frame */
 static const struct stv0900_car_loop_optim FE_STV0900_S2APSKCarLoopCut20[11] = {
-	/* Modcod		2MPon 	2MPoff	5MPon 	5MPoff	10MPon
-				10MPoff	20MPon 	20MPoff	30MPon 	30MPoff */
+	/* Modcod		2MPon	2MPoff	5MPon	5MPoff	10MPon
+				10MPoff	20MPon	20MPoff	30MPon	30MPoff */
 	{ STV0900_16APSK_23,	0x0C,	0x0C,	0x0C,	0x0C,	0x1D,
 				0x0C,	0x3C,	0x0C,	0x2C,	0x0C },
 	{ STV0900_16APSK_34,	0x0C,	0x0C,	0x0C,	0x0C,	0x0E,
@@ -248,8 +248,8 @@ static const struct stv0900_car_loop_optim FE_STV0900_S2APSKCarLoopCut20[11] = {
 
 /* Cut 2.0 Tracking carrier loop carrier QPSK 1/4 to QPSK 2/5 long Frame */
 static const struct stv0900_car_loop_optim FE_STV0900_S2LowQPCarLoopCut20[3] = {
-	/* Modcod		2MPon 	2MPoff	5MPon 	5MPoff	10MPon
-				10MPoff	20MPon 	20MPoff	30MPon 	30MPoff */
+	/* Modcod		2MPon	2MPoff	5MPon	5MPoff	10MPon
+				10MPoff	20MPon	20MPoff	30MPon	30MPoff */
 	{ STV0900_QPSK_14,	0x0F,	0x3F,	0x0E,	0x3F,	0x2D,
 				0x2F,	0x2D,	0x1F,	0x3D,	0x3E },
 	{ STV0900_QPSK_13,	0x0F,	0x3F,	0x0E,	0x3F,	0x2D,
@@ -275,10 +275,10 @@ struct stv0900_short_frames_car_loop_optim FE_STV0900_S2ShortCarLoop[4] = {
 };
 
 static	const struct stv0900_car_loop_optim FE_STV0900_S2CarLoopCut30[14] = {
-	/*Modcod		2MPon 	2MPoff	5MPon 	5MPoff	10MPon
-				10MPoff	20MPon 	20MPoff	30MPon 	30MPoff	*/
+	/*Modcod		2MPon	2MPoff	5MPon	5MPoff	10MPon
+				10MPoff	20MPon	20MPoff	30MPon	30MPoff	*/
 	{ STV0900_QPSK_12,	0x3C,	0x2C,	0x0C,	0x2C,	0x1B,
-				0x2C,	0x1B,	0x1C,	0x0B, 	0x3B },
+				0x2C,	0x1B,	0x1C,	0x0B,	0x3B },
 	{ STV0900_QPSK_35,	0x0D,	0x0D,	0x0C,	0x0D,	0x1B,
 				0x3C,	0x1B,	0x1C,	0x0B,	0x3B },
 	{ STV0900_QPSK_23,	0x1D,	0x0D,	0x0C,	0x1D,	0x2B,
@@ -309,8 +309,8 @@ static	const struct stv0900_car_loop_optim FE_STV0900_S2CarLoopCut30[14] = {
 
 static	const
 struct stv0900_car_loop_optim FE_STV0900_S2APSKCarLoopCut30[11] = {
-	/*Modcod		2MPon 	2MPoff	5MPon 	5MPoff	10MPon
-				10MPoff	20MPon 	20MPoff	30MPon 	30MPoff	*/
+	/*Modcod		2MPon	2MPoff	5MPon	5MPoff	10MPon
+				10MPoff	20MPon	20MPoff	30MPon	30MPoff	*/
 	{ STV0900_16APSK_23,	0x0A,	0x0A,	0x0A,	0x0A,	0x1A,
 				0x0A,	0x3A,	0x0A,	0x2A,	0x0A },
 	{ STV0900_16APSK_34,	0x0A,	0x0A,	0x0A,	0x0A,	0x0B,
@@ -337,8 +337,8 @@ struct stv0900_car_loop_optim FE_STV0900_S2APSKCarLoopCut30[11] = {
 
 static	const
 struct stv0900_car_loop_optim FE_STV0900_S2LowQPCarLoopCut30[3] = {
-	/*Modcod		2MPon 	2MPoff	5MPon 	5MPoff	10MPon
-				10MPoff	20MPon 	20MPoff	30MPon 	30MPoff*/
+	/*Modcod		2MPon	2MPoff	5MPon	5MPoff	10MPon
+				10MPoff	20MPon	20MPoff	30MPon	30MPoff*/
 	{ STV0900_QPSK_14,	0x0C,	0x3C,	0x0B,	0x3C,	0x2A,
 				0x2C,	0x2A,	0x1C,	0x3A,	0x3B },
 	{ STV0900_QPSK_13,	0x0C,	0x3C,	0x0B,	0x3C,	0x2A,
diff --git a/drivers/media/dvb-frontends/stv0900_priv.h b/drivers/media/dvb-frontends/stv0900_priv.h
index 7a95f955627b..d1fc06ff27d3 100644
--- a/drivers/media/dvb-frontends/stv0900_priv.h
+++ b/drivers/media/dvb-frontends/stv0900_priv.h
@@ -243,7 +243,7 @@ struct stv0900_init_params{
 
 	u8	tun1_maddress;
 	int	tuner1_adc;
-	int 	tuner1_type;
+	int	tuner1_type;
 
 	/* IQ from the tuner1 to the demod */
 	enum stv0900_iq_inversion	tun1_iq_inv;
diff --git a/drivers/media/dvb-frontends/stv090x.c b/drivers/media/dvb-frontends/stv090x.c
index 7ef469c0c866..9133f65d4623 100644
--- a/drivers/media/dvb-frontends/stv090x.c
+++ b/drivers/media/dvb-frontends/stv090x.c
@@ -27,7 +27,7 @@
 #include <linux/mutex.h>
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "stv6110x.h" /* for demodulator internal modes */
 
@@ -677,7 +677,7 @@ static struct stv090x_short_frame_crloop stv090x_s2_short_crl_cut20[] = {
 
 /* Cut 3.0 Short Frame Tracking CR Loop */
 static struct stv090x_short_frame_crloop stv090x_s2_short_crl_cut30[] = {
-	/* MODCOD  	  2M	5M    10M   20M	  30M */
+	/* MODCOD	  2M	5M    10M   20M	  30M */
 	{ STV090x_QPSK,   0x2C, 0x2B, 0x0B, 0x0B, 0x3A },
 	{ STV090x_8PSK,   0x3B, 0x0B, 0x2A, 0x0A, 0x39 },
 	{ STV090x_16APSK, 0x1B, 0x1B, 0x1B, 0x3A, 0x2A },
@@ -701,7 +701,7 @@ static int stv090x_read_reg(struct stv090x_state *state, unsigned int reg)
 	u8 buf;
 
 	struct i2c_msg msg[] = {
-		{ .addr	= config->address, .flags	= 0, 		.buf = b0,   .len = 2 },
+		{ .addr	= config->address, .flags	= 0,		.buf = b0,   .len = 2 },
 		{ .addr	= config->address, .flags	= I2C_M_RD,	.buf = &buf, .len = 1 }
 	};
 
@@ -755,7 +755,9 @@ static int stv090x_write_regs(struct stv090x_state *state, unsigned int reg, u8
 
 static int stv090x_write_reg(struct stv090x_state *state, unsigned int reg, u8 data)
 {
-	return stv090x_write_regs(state, reg, &data, 1);
+	u8 tmp = data; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return stv090x_write_regs(state, reg, &tmp, 1);
 }
 
 static int stv090x_i2c_gate_ctrl(struct stv090x_state *state, int enable)
@@ -2215,13 +2217,12 @@ static int stv090x_get_coldlock(struct stv090x_state *state, s32 timeout_dmd)
 		if (state->config->tuner_get_status) {
 			if (state->config->tuner_get_status(fe, &reg) < 0)
 				goto err_gateoff;
+			if (reg)
+				dprintk(FE_DEBUG, 1, "Tuner phase locked");
+			else
+				dprintk(FE_DEBUG, 1, "Tuner unlocked");
 		}
 
-		if (reg)
-			dprintk(FE_DEBUG, 1, "Tuner phase locked");
-		else
-			dprintk(FE_DEBUG, 1, "Tuner unlocked");
-
 		if (stv090x_i2c_gate_ctrl(state, 0) < 0)
 			goto err;
 
@@ -3429,6 +3430,21 @@ static enum stv090x_signal_state stv090x_algo(struct stv090x_state *state)
 	return -1;
 }
 
+static int stv090x_set_pls(struct stv090x_state *state, u32 pls_code)
+{
+	dprintk(FE_DEBUG, 1, "Set Gold PLS code %d", pls_code);
+	if (STV090x_WRITE_DEMOD(state, PLROOT0, pls_code & 0xff) < 0)
+		goto err;
+	if (STV090x_WRITE_DEMOD(state, PLROOT1, (pls_code >> 8) & 0xff) < 0)
+		goto err;
+	if (STV090x_WRITE_DEMOD(state, PLROOT2, 0x04 | (pls_code >> 16)) < 0)
+		goto err;
+	return 0;
+err:
+	dprintk(FE_ERROR, 1, "I/O error");
+	return -1;
+}
+
 static int stv090x_set_mis(struct stv090x_state *state, int mis)
 {
 	u32 reg;
@@ -3491,6 +3507,7 @@ static enum dvbfe_search stv090x_search(struct dvb_frontend *fe)
 		state->search_range = 5000000;
 	}
 
+	stv090x_set_pls(state, props->scrambling_sequence_index);
 	stv090x_set_mis(state, props->stream_id);
 
 	if (stv090x_algo(state) == STV090x_RANGEOK) {
@@ -4889,11 +4906,11 @@ static const struct dvb_frontend_ops stv090x_ops = {
 	.info = {
 		.name			= "STV090x Multistandard",
 		.frequency_min		= 950000,
-		.frequency_max 		= 2150000,
+		.frequency_max		= 2150000,
 		.frequency_stepsize	= 0,
 		.frequency_tolerance	= 0,
-		.symbol_rate_min 	= 1000000,
-		.symbol_rate_max 	= 45000000,
+		.symbol_rate_min	= 1000000,
+		.symbol_rate_max	= 45000000,
 		.caps			= FE_CAN_INVERSION_AUTO |
 					  FE_CAN_FEC_AUTO       |
 					  FE_CAN_QPSK           |
@@ -4936,7 +4953,7 @@ struct dvb_frontend *stv090x_attach(struct stv090x_config *config,
 	state->frontend.ops			= stv090x_ops;
 	state->frontend.demodulator_priv	= state;
 	state->demod				= demod;
-	state->demod_mode 			= config->demod_mode; /* Single or Dual mode */
+	state->demod_mode			= config->demod_mode; /* Single or Dual mode */
 	state->device				= config->device;
 	state->rolloff				= STV090x_RO_35; /* default */
 
diff --git a/drivers/media/dvb-frontends/stv090x_priv.h b/drivers/media/dvb-frontends/stv090x_priv.h
index 5b780c80d496..fdda2185db9d 100644
--- a/drivers/media/dvb-frontends/stv090x_priv.h
+++ b/drivers/media/dvb-frontends/stv090x_priv.h
@@ -22,7 +22,7 @@
 #ifndef __STV090x_PRIV_H
 #define __STV090x_PRIV_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define FE_ERROR				0
 #define FE_NOTICE				1
@@ -231,7 +231,7 @@ struct stv090x_tab {
 };
 
 struct stv090x_internal {
-	struct i2c_adapter 	*i2c_adap;
+	struct i2c_adapter	*i2c_adap;
 	u8			i2c_addr;
 
 	struct mutex		demod_lock; /* Lock access to shared register */
diff --git a/drivers/media/dvb-frontends/stv0910.c b/drivers/media/dvb-frontends/stv0910.c
index 8bf855c301f5..a2f7c0c1587f 100644
--- a/drivers/media/dvb-frontends/stv0910.c
+++ b/drivers/media/dvb-frontends/stv0910.c
@@ -24,8 +24,7 @@
 #include <linux/i2c.h>
 #include <asm/div64.h>
 
-#include "dvb_math.h"
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "stv0910.h"
 #include "stv0910_regs.h"
 
@@ -138,33 +137,21 @@ struct slookup {
 	u32  reg_value;
 };
 
-static inline int i2c_write(struct i2c_adapter *adap, u8 adr,
-			    u8 *data, int len)
+static int write_reg(struct stv *state, u16 reg, u8 val)
 {
-	struct i2c_msg msg = {.addr = adr, .flags = 0,
-			      .buf = data, .len = len};
+	struct i2c_adapter *adap = state->base->i2c;
+	u8 data[3] = {reg >> 8, reg & 0xff, val};
+	struct i2c_msg msg = {.addr = state->base->adr, .flags = 0,
+			      .buf = data, .len = 3};
 
 	if (i2c_transfer(adap, &msg, 1) != 1) {
 		dev_warn(&adap->dev, "i2c write error ([%02x] %04x: %02x)\n",
-			 adr, (data[0] << 8) | data[1],
-			 (len > 2 ? data[2] : 0));
-		return -EREMOTEIO;
+			 state->base->adr, reg, val);
+		return -EIO;
 	}
 	return 0;
 }
 
-static int i2c_write_reg16(struct i2c_adapter *adap, u8 adr, u16 reg, u8 val)
-{
-	u8 msg[3] = {reg >> 8, reg & 0xff, val};
-
-	return i2c_write(adap, adr, msg, 3);
-}
-
-static int write_reg(struct stv *state, u16 reg, u8 val)
-{
-	return i2c_write_reg16(state->base->i2c, state->base->adr, reg, val);
-}
-
 static inline int i2c_read_regs16(struct i2c_adapter *adapter, u8 adr,
 				  u16 reg, u8 *val, int count)
 {
@@ -177,7 +164,7 @@ static inline int i2c_read_regs16(struct i2c_adapter *adapter, u8 adr,
 	if (i2c_transfer(adapter, msgs, 2) != 2) {
 		dev_warn(&adapter->dev, "i2c read error ([%02x] %04x)\n",
 			 adr, reg);
-		return -EREMOTEIO;
+		return -EIO;
 	}
 	return 0;
 }
@@ -207,6 +194,34 @@ static int write_shared_reg(struct stv *state, u16 reg, u8 mask, u8 val)
 	return status;
 }
 
+static int write_field(struct stv *state, u32 field, u8 val)
+{
+	int status;
+	u8 shift, mask, old, new;
+
+	status = read_reg(state, field >> 16, &old);
+	if (status)
+		return status;
+	mask = field & 0xff;
+	shift = (field >> 12) & 0xf;
+	new = ((val << shift) & mask) | (old & ~mask);
+	if (new == old)
+		return 0;
+	return write_reg(state, field >> 16, new);
+}
+
+#define SET_FIELD(_reg, _val)					\
+	write_field(state, state->nr ? FSTV0910_P2_##_reg :	\
+		    FSTV0910_P1_##_reg, _val)
+
+#define SET_REG(_reg, _val)					\
+	write_reg(state, state->nr ? RSTV0910_P2_##_reg :	\
+		  RSTV0910_P1_##_reg, _val)
+
+#define GET_REG(_reg, _val)					\
+	read_reg(state, state->nr ? RSTV0910_P2_##_reg :	\
+		 RSTV0910_P1_##_reg, _val)
+
 static const struct slookup s1_sn_lookup[] = {
 	{   0,    9242  }, /* C/N=   0dB */
 	{   5,    9105  }, /* C/N= 0.5dB */
@@ -533,10 +548,8 @@ static int get_signal_parameters(struct stv *state)
 
 static int tracking_optimization(struct stv *state)
 {
-	u32 symbol_rate = 0;
 	u8 tmp;
 
-	get_cur_symbol_rate(state, &symbol_rate);
 	read_reg(state, RSTV0910_P2_DMDCFGMD + state->regoff, &tmp);
 	tmp &= ~0xC0;
 
@@ -867,71 +880,90 @@ static int stop(struct stv *state)
 	return 0;
 }
 
-static int init_search_param(struct stv *state)
+static void set_pls(struct stv *state, u32 pls_code)
 {
-	u8 tmp;
-
-	read_reg(state, RSTV0910_P2_PDELCTRL1 + state->regoff, &tmp);
-	tmp |= 0x20; /* Filter_en (no effect if SIS=non-MIS */
-	write_reg(state, RSTV0910_P2_PDELCTRL1 + state->regoff, tmp);
-
-	read_reg(state, RSTV0910_P2_PDELCTRL2 + state->regoff, &tmp);
-	tmp &= ~0x02; /* frame mode = 0 */
-	write_reg(state, RSTV0910_P2_PDELCTRL2 + state->regoff, tmp);
-
-	write_reg(state, RSTV0910_P2_UPLCCST0 + state->regoff, 0xe0);
-	write_reg(state, RSTV0910_P2_ISIBITENA + state->regoff, 0x00);
-
-	read_reg(state, RSTV0910_P2_TSSTATEM + state->regoff, &tmp);
-	tmp &= ~0x01; /* nosync = 0, in case next signal is standard TS */
-	write_reg(state, RSTV0910_P2_TSSTATEM + state->regoff, tmp);
-
-	read_reg(state, RSTV0910_P2_TSCFGL + state->regoff, &tmp);
-	tmp &= ~0x04; /* embindvb = 0 */
-	write_reg(state, RSTV0910_P2_TSCFGL + state->regoff, tmp);
-
-	read_reg(state, RSTV0910_P2_TSINSDELH + state->regoff, &tmp);
-	tmp &= ~0x80; /* syncbyte = 0 */
-	write_reg(state, RSTV0910_P2_TSINSDELH + state->regoff, tmp);
-
-	read_reg(state, RSTV0910_P2_TSINSDELM + state->regoff, &tmp);
-	tmp &= ~0x08; /* token = 0 */
-	write_reg(state, RSTV0910_P2_TSINSDELM + state->regoff, tmp);
+	if (pls_code == state->cur_scrambling_code)
+		return;
+
+	/* PLROOT2 bit 2 = gold code */
+	write_reg(state, RSTV0910_P2_PLROOT0 + state->regoff,
+		  pls_code & 0xff);
+	write_reg(state, RSTV0910_P2_PLROOT1 + state->regoff,
+		  (pls_code >> 8) & 0xff);
+	write_reg(state, RSTV0910_P2_PLROOT2 + state->regoff,
+		  0x04 | ((pls_code >> 16) & 0x03));
+	state->cur_scrambling_code = pls_code;
+}
 
-	read_reg(state, RSTV0910_P2_TSDLYSET2 + state->regoff, &tmp);
-	tmp &= ~0x30; /* hysteresis threshold = 0 */
-	write_reg(state, RSTV0910_P2_TSDLYSET2 + state->regoff, tmp);
+static void set_isi(struct stv *state, u32 isi)
+{
+	if (isi == NO_STREAM_ID_FILTER)
+		return;
+	if (isi == 0x80000000) {
+		SET_FIELD(FORCE_CONTINUOUS, 1);
+		SET_FIELD(TSOUT_NOSYNC, 1);
+	} else {
+		SET_FIELD(FILTER_EN, 1);
+		write_reg(state, RSTV0910_P2_ISIENTRY + state->regoff,
+			  isi & 0xff);
+		write_reg(state, RSTV0910_P2_ISIBITENA + state->regoff, 0xff);
+	}
+	SET_FIELD(ALGOSWRST, 1);
+	SET_FIELD(ALGOSWRST, 0);
+}
 
-	read_reg(state, RSTV0910_P2_PDELCTRL0 + state->regoff, &tmp);
-	tmp = (tmp & ~0x30) | 0x10; /* isi obs mode = 1, observe min ISI */
-	write_reg(state, RSTV0910_P2_PDELCTRL0 + state->regoff, tmp);
+static void set_stream_modes(struct stv *state,
+			     struct dtv_frontend_properties *p)
+{
+	set_isi(state, p->stream_id);
+	set_pls(state, p->scrambling_sequence_index);
+}
 
+static int init_search_param(struct stv *state,
+			     struct dtv_frontend_properties *p)
+{
+	SET_FIELD(FORCE_CONTINUOUS, 0);
+	SET_FIELD(FRAME_MODE, 0);
+	SET_FIELD(FILTER_EN, 0);
+	SET_FIELD(TSOUT_NOSYNC, 0);
+	SET_FIELD(TSFIFO_EMBINDVB, 0);
+	SET_FIELD(TSDEL_SYNCBYTE, 0);
+	SET_REG(UPLCCST0, 0xe0);
+	SET_FIELD(TSINS_TOKEN, 0);
+	SET_FIELD(HYSTERESIS_THRESHOLD, 0);
+	SET_FIELD(ISIOBS_MODE, 1);
+
+	set_stream_modes(state, p);
 	return 0;
 }
 
 static int enable_puncture_rate(struct stv *state, enum fe_code_rate rate)
 {
+	u8 val;
+
 	switch (rate) {
 	case FEC_1_2:
-		return write_reg(state,
-				 RSTV0910_P2_PRVIT + state->regoff, 0x01);
+		val = 0x01;
+		break;
 	case FEC_2_3:
-		return write_reg(state,
-				 RSTV0910_P2_PRVIT + state->regoff, 0x02);
+		val = 0x02;
+		break;
 	case FEC_3_4:
-		return write_reg(state,
-				 RSTV0910_P2_PRVIT + state->regoff, 0x04);
+		val = 0x04;
+		break;
 	case FEC_5_6:
-		return write_reg(state,
-				 RSTV0910_P2_PRVIT + state->regoff, 0x08);
+		val = 0x08;
+		break;
 	case FEC_7_8:
-		return write_reg(state,
-				 RSTV0910_P2_PRVIT + state->regoff, 0x20);
+		val = 0x20;
+		break;
 	case FEC_NONE:
 	default:
-		return write_reg(state,
-				 RSTV0910_P2_PRVIT + state->regoff, 0x2f);
+		val = 0x2f;
+		break;
 	}
+
+	return write_reg(state, RSTV0910_P2_PRVIT + state->regoff, val);
 }
 
 static int set_vth_default(struct stv *state)
@@ -988,7 +1020,6 @@ static int start(struct stv *state, struct dtv_frontend_properties *p)
 	s32 freq;
 	u8  reg_dmdcfgmd;
 	u16 symb;
-	u32 scrambling_code = 1;
 
 	if (p->symbol_rate < 100000 || p->symbol_rate > 70000000)
 		return -EINVAL;
@@ -1000,30 +1031,7 @@ static int start(struct stv *state, struct dtv_frontend_properties *p)
 	if (state->started)
 		write_reg(state, RSTV0910_P2_DMDISTATE + state->regoff, 0x5C);
 
-	init_search_param(state);
-
-	if (p->stream_id != NO_STREAM_ID_FILTER) {
-		/*
-		 * Backwards compatibility to "crazy" API.
-		 * PRBS X root cannot be 0, so this should always work.
-		 */
-		if (p->stream_id & 0xffffff00)
-			scrambling_code = p->stream_id >> 8;
-		write_reg(state, RSTV0910_P2_ISIENTRY + state->regoff,
-			  p->stream_id & 0xff);
-		write_reg(state, RSTV0910_P2_ISIBITENA + state->regoff,
-			  0xff);
-	}
-
-	if (scrambling_code != state->cur_scrambling_code) {
-		write_reg(state, RSTV0910_P2_PLROOT0 + state->regoff,
-			  scrambling_code & 0xff);
-		write_reg(state, RSTV0910_P2_PLROOT1 + state->regoff,
-			  (scrambling_code >> 8) & 0xff);
-		write_reg(state, RSTV0910_P2_PLROOT2 + state->regoff,
-			  (scrambling_code >> 16) & 0x0f);
-		state->cur_scrambling_code = scrambling_code;
-	}
+	init_search_param(state, p);
 
 	if (p->symbol_rate <= 1000000) { /* SR <=1Msps */
 		state->demod_timeout = 3000;
@@ -1241,7 +1249,8 @@ static int gate_ctrl(struct dvb_frontend *fe, int enable)
 	if (write_reg(state, state->nr ? RSTV0910_P2_I2CRPT :
 		      RSTV0910_P1_I2CRPT, i2crpt) < 0) {
 		/* don't hold the I2C bus lock on failure */
-		mutex_unlock(&state->base->i2c_lock);
+		if (!WARN_ON(!mutex_is_locked(&state->base->i2c_lock)))
+			mutex_unlock(&state->base->i2c_lock);
 		dev_err(&state->base->i2c->dev,
 			"%s() write_reg failure (enable=%d)\n",
 			__func__, enable);
@@ -1251,7 +1260,8 @@ static int gate_ctrl(struct dvb_frontend *fe, int enable)
 	state->i2crpt = i2crpt;
 
 	if (!enable)
-		mutex_unlock(&state->base->i2c_lock);
+		if (!WARN_ON(!mutex_is_locked(&state->base->i2c_lock)))
+			mutex_unlock(&state->base->i2c_lock);
 	return 0;
 }
 
@@ -1271,14 +1281,11 @@ static int set_parameters(struct dvb_frontend *fe)
 {
 	int stat = 0;
 	struct stv *state = fe->demodulator_priv;
-	u32 iffreq;
 	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
 
 	stop(state);
 	if (fe->ops.tuner_ops.set_params)
 		fe->ops.tuner_ops.set_params(fe);
-	if (fe->ops.tuner_ops.get_if_frequency)
-		fe->ops.tuner_ops.get_if_frequency(fe, &iffreq);
 	state->symbol_rate = p->symbol_rate;
 	stat = start(state, p);
 	return stat;
@@ -1498,6 +1505,19 @@ static int read_status(struct dvb_frontend *fe, enum fe_status *status)
 				enable_puncture_rate(state,
 						     state->puncture_rate);
 		}
+
+		/* Use highest signaled ModCod for quality */
+		if (state->is_vcm) {
+			u8 tmp;
+			enum fe_stv0910_mod_cod mod_cod;
+
+			read_reg(state, RSTV0910_P2_DMDMODCOD + state->regoff,
+				 &tmp);
+			mod_cod = (enum fe_stv0910_mod_cod)((tmp & 0x7c) >> 2);
+
+			if (mod_cod > state->mod_cod)
+				state->mod_cod = mod_cod;
+		}
 	}
 
 	/* read signal statistics */
@@ -1527,6 +1547,7 @@ static int get_frontend(struct dvb_frontend *fe,
 {
 	struct stv *state = fe->demodulator_priv;
 	u8 tmp;
+	u32 symbolrate;
 
 	if (state->receive_mode == RCVMODE_DVBS2) {
 		u32 mc;
@@ -1580,6 +1601,10 @@ static int get_frontend(struct dvb_frontend *fe,
 		p->rolloff = ROLLOFF_35;
 	}
 
+	if (state->receive_mode != RCVMODE_NONE) {
+		get_cur_symbol_rate(state, &symbolrate);
+		p->symbol_rate = symbolrate;
+	}
 	return 0;
 }
 
diff --git a/drivers/media/dvb-frontends/stv0910_regs.h b/drivers/media/dvb-frontends/stv0910_regs.h
index 32ced4eaf296..f0eb915090bd 100644
--- a/drivers/media/dvb-frontends/stv0910_regs.h
+++ b/drivers/media/dvb-frontends/stv0910_regs.h
@@ -7,15 +7,15 @@
  * Registers Declaration (Internal ST, All Applications )
  * -------------------------
  * Each register (RSTV0910__XXXXX) is defined by its address (2 bytes).
- *
- * Each field (FSTV0910__XXXXX)is defined as follow:
- * [register address -- 2bytes][field sign -- 1byte][field mask -- 1byte]
- *  ======================================================================
+ * Each field (FSTV0910__XXXXX) is defined as follow:
+ * [register address -- 2bytes][field offset -- 4 bits][unused -- 3 bits]
+ * [field sign -- 1 bit][field mask -- 1byte]
+ * =======================================================================
  */
 
 /* MID */
 #define RSTV0910_MID  0xf100
-#define FSTV0910_MCHIP_IDENT  0xf10000f0
+#define FSTV0910_MCHIP_IDENT  0xf10040f0
 #define FSTV0910_MRELEASE  0xf100000f
 
 /* DID */
@@ -24,7 +24,7 @@
 
 /* DACR1 */
 #define RSTV0910_DACR1  0xf113
-#define FSTV0910_DAC_MODE  0xf11300e0
+#define FSTV0910_DAC_MODE  0xf11350e0
 #define FSTV0910_DAC_VALUE1  0xf113000f
 
 /* DACR2 */
@@ -33,283 +33,283 @@
 
 /* PADCFG */
 #define RSTV0910_PADCFG  0xf11a
-#define FSTV0910_AGCRF2_OPD  0xf11a0008
-#define FSTV0910_AGCRF2_XOR  0xf11a0004
-#define FSTV0910_AGCRF1_OPD  0xf11a0002
+#define FSTV0910_AGCRF2_OPD  0xf11a3008
+#define FSTV0910_AGCRF2_XOR  0xf11a2004
+#define FSTV0910_AGCRF1_OPD  0xf11a1002
 #define FSTV0910_AGCRF1_XOR  0xf11a0001
 
 /* OUTCFG2 */
 #define RSTV0910_OUTCFG2  0xf11b
-#define FSTV0910_TS2_ERROR_XOR  0xf11b0080
-#define FSTV0910_TS2_DPN_XOR  0xf11b0040
-#define FSTV0910_TS2_STROUT_XOR  0xf11b0020
-#define FSTV0910_TS2_CLOCKOUT_XOR  0xf11b0010
-#define FSTV0910_TS1_ERROR_XOR  0xf11b0008
-#define FSTV0910_TS1_DPN_XOR  0xf11b0004
-#define FSTV0910_TS1_STROUT_XOR  0xf11b0002
+#define FSTV0910_TS2_ERROR_XOR  0xf11b7080
+#define FSTV0910_TS2_DPN_XOR  0xf11b6040
+#define FSTV0910_TS2_STROUT_XOR  0xf11b5020
+#define FSTV0910_TS2_CLOCKOUT_XOR  0xf11b4010
+#define FSTV0910_TS1_ERROR_XOR  0xf11b3008
+#define FSTV0910_TS1_DPN_XOR  0xf11b2004
+#define FSTV0910_TS1_STROUT_XOR  0xf11b1002
 #define FSTV0910_TS1_CLOCKOUT_XOR  0xf11b0001
 
 /* OUTCFG */
 #define RSTV0910_OUTCFG  0xf11c
-#define FSTV0910_TS2_OUTSER_HZ  0xf11c0020
-#define FSTV0910_TS1_OUTSER_HZ  0xf11c0010
-#define FSTV0910_TS2_OUTPAR_HZ  0xf11c0008
-#define FSTV0910_TS1_OUTPAR_HZ  0xf11c0004
-#define FSTV0910_TS_SERDATA0  0xf11c0002
+#define FSTV0910_TS2_OUTSER_HZ  0xf11c5020
+#define FSTV0910_TS1_OUTSER_HZ  0xf11c4010
+#define FSTV0910_TS2_OUTPAR_HZ  0xf11c3008
+#define FSTV0910_TS1_OUTPAR_HZ  0xf11c2004
+#define FSTV0910_TS_SERDATA0  0xf11c1002
 
 /* IRQSTATUS3 */
 #define RSTV0910_IRQSTATUS3  0xf120
-#define FSTV0910_SPLL_LOCK  0xf1200020
-#define FSTV0910_SSTREAM_LCK_1  0xf1200010
-#define FSTV0910_SSTREAM_LCK_2  0xf1200008
-#define FSTV0910_SDVBS1_PRF_2  0xf1200002
+#define FSTV0910_SPLL_LOCK  0xf1205020
+#define FSTV0910_SSTREAM_LCK_1  0xf1204010
+#define FSTV0910_SSTREAM_LCK_2  0xf1203008
+#define FSTV0910_SDVBS1_PRF_2  0xf1201002
 #define FSTV0910_SDVBS1_PRF_1  0xf1200001
 
 /* IRQSTATUS2 */
 #define RSTV0910_IRQSTATUS2  0xf121
-#define FSTV0910_SSPY_ENDSIM_1  0xf1210080
-#define FSTV0910_SSPY_ENDSIM_2  0xf1210040
-#define FSTV0910_SPKTDEL_ERROR_2  0xf1210010
-#define FSTV0910_SPKTDEL_LOCKB_2  0xf1210008
-#define FSTV0910_SPKTDEL_LOCK_2  0xf1210004
-#define FSTV0910_SPKTDEL_ERROR_1  0xf1210002
+#define FSTV0910_SSPY_ENDSIM_1  0xf1217080
+#define FSTV0910_SSPY_ENDSIM_2  0xf1216040
+#define FSTV0910_SPKTDEL_ERROR_2  0xf1214010
+#define FSTV0910_SPKTDEL_LOCKB_2  0xf1213008
+#define FSTV0910_SPKTDEL_LOCK_2  0xf1212004
+#define FSTV0910_SPKTDEL_ERROR_1  0xf1211002
 #define FSTV0910_SPKTDEL_LOCKB_1  0xf1210001
 
 /* IRQSTATUS1 */
 #define RSTV0910_IRQSTATUS1  0xf122
-#define FSTV0910_SPKTDEL_LOCK_1  0xf1220080
-#define FSTV0910_SFEC_LOCKB_2  0xf1220040
-#define FSTV0910_SFEC_LOCK_2  0xf1220020
-#define FSTV0910_SFEC_LOCKB_1  0xf1220010
-#define FSTV0910_SFEC_LOCK_1  0xf1220008
-#define FSTV0910_SDEMOD_LOCKB_2  0xf1220004
-#define FSTV0910_SDEMOD_LOCK_2  0xf1220002
+#define FSTV0910_SPKTDEL_LOCK_1  0xf1227080
+#define FSTV0910_SFEC_LOCKB_2  0xf1226040
+#define FSTV0910_SFEC_LOCK_2  0xf1225020
+#define FSTV0910_SFEC_LOCKB_1  0xf1224010
+#define FSTV0910_SFEC_LOCK_1  0xf1223008
+#define FSTV0910_SDEMOD_LOCKB_2  0xf1222004
+#define FSTV0910_SDEMOD_LOCK_2  0xf1221002
 #define FSTV0910_SDEMOD_IRQ_2  0xf1220001
 
 /* IRQSTATUS0 */
 #define RSTV0910_IRQSTATUS0  0xf123
-#define FSTV0910_SDEMOD_LOCKB_1  0xf1230080
-#define FSTV0910_SDEMOD_LOCK_1  0xf1230040
-#define FSTV0910_SDEMOD_IRQ_1  0xf1230020
-#define FSTV0910_SBCH_ERRFLAG  0xf1230010
-#define FSTV0910_SDISEQC2_IRQ  0xf1230004
+#define FSTV0910_SDEMOD_LOCKB_1  0xf1237080
+#define FSTV0910_SDEMOD_LOCK_1  0xf1236040
+#define FSTV0910_SDEMOD_IRQ_1  0xf1235020
+#define FSTV0910_SBCH_ERRFLAG  0xf1234010
+#define FSTV0910_SDISEQC2_IRQ  0xf1232004
 #define FSTV0910_SDISEQC1_IRQ  0xf1230001
 
 /* IRQMASK3 */
 #define RSTV0910_IRQMASK3  0xf124
-#define FSTV0910_MPLL_LOCK  0xf1240020
-#define FSTV0910_MSTREAM_LCK_1  0xf1240010
-#define FSTV0910_MSTREAM_LCK_2  0xf1240008
-#define FSTV0910_MDVBS1_PRF_2  0xf1240002
+#define FSTV0910_MPLL_LOCK  0xf1245020
+#define FSTV0910_MSTREAM_LCK_1  0xf1244010
+#define FSTV0910_MSTREAM_LCK_2  0xf1243008
+#define FSTV0910_MDVBS1_PRF_2  0xf1241002
 #define FSTV0910_MDVBS1_PRF_1  0xf1240001
 
 /* IRQMASK2 */
 #define RSTV0910_IRQMASK2  0xf125
-#define FSTV0910_MSPY_ENDSIM_1  0xf1250080
-#define FSTV0910_MSPY_ENDSIM_2  0xf1250040
-#define FSTV0910_MPKTDEL_ERROR_2  0xf1250010
-#define FSTV0910_MPKTDEL_LOCKB_2  0xf1250008
-#define FSTV0910_MPKTDEL_LOCK_2  0xf1250004
-#define FSTV0910_MPKTDEL_ERROR_1  0xf1250002
+#define FSTV0910_MSPY_ENDSIM_1  0xf1257080
+#define FSTV0910_MSPY_ENDSIM_2  0xf1256040
+#define FSTV0910_MPKTDEL_ERROR_2  0xf1254010
+#define FSTV0910_MPKTDEL_LOCKB_2  0xf1253008
+#define FSTV0910_MPKTDEL_LOCK_2  0xf1252004
+#define FSTV0910_MPKTDEL_ERROR_1  0xf1251002
 #define FSTV0910_MPKTDEL_LOCKB_1  0xf1250001
 
 /* IRQMASK1 */
 #define RSTV0910_IRQMASK1  0xf126
-#define FSTV0910_MPKTDEL_LOCK_1  0xf1260080
-#define FSTV0910_MFEC_LOCKB_2  0xf1260040
-#define FSTV0910_MFEC_LOCK_2  0xf1260020
-#define FSTV0910_MFEC_LOCKB_1  0xf1260010
-#define FSTV0910_MFEC_LOCK_1  0xf1260008
-#define FSTV0910_MDEMOD_LOCKB_2  0xf1260004
-#define FSTV0910_MDEMOD_LOCK_2  0xf1260002
+#define FSTV0910_MPKTDEL_LOCK_1  0xf1267080
+#define FSTV0910_MFEC_LOCKB_2  0xf1266040
+#define FSTV0910_MFEC_LOCK_2  0xf1265020
+#define FSTV0910_MFEC_LOCKB_1  0xf1264010
+#define FSTV0910_MFEC_LOCK_1  0xf1263008
+#define FSTV0910_MDEMOD_LOCKB_2  0xf1262004
+#define FSTV0910_MDEMOD_LOCK_2  0xf1261002
 #define FSTV0910_MDEMOD_IRQ_2  0xf1260001
 
 /* IRQMASK0 */
 #define RSTV0910_IRQMASK0  0xf127
-#define FSTV0910_MDEMOD_LOCKB_1  0xf1270080
-#define FSTV0910_MDEMOD_LOCK_1  0xf1270040
-#define FSTV0910_MDEMOD_IRQ_1  0xf1270020
-#define FSTV0910_MBCH_ERRFLAG  0xf1270010
-#define FSTV0910_MDISEQC2_IRQ  0xf1270004
+#define FSTV0910_MDEMOD_LOCKB_1  0xf1277080
+#define FSTV0910_MDEMOD_LOCK_1  0xf1276040
+#define FSTV0910_MDEMOD_IRQ_1  0xf1275020
+#define FSTV0910_MBCH_ERRFLAG  0xf1274010
+#define FSTV0910_MDISEQC2_IRQ  0xf1272004
 #define FSTV0910_MDISEQC1_IRQ  0xf1270001
 
 /* I2CCFG */
 #define RSTV0910_I2CCFG  0xf129
-#define FSTV0910_I2C_FASTMODE  0xf1290008
+#define FSTV0910_I2C_FASTMODE  0xf1293008
 #define FSTV0910_I2CADDR_INC  0xf1290003
 
 /* P1_I2CRPT */
 #define RSTV0910_P1_I2CRPT  0xf12a
-#define FSTV0910_P1_I2CT_ON  0xf12a0080
-#define FSTV0910_P1_ENARPT_LEVEL  0xf12a0070
-#define FSTV0910_P1_SCLT_DELAY  0xf12a0008
-#define FSTV0910_P1_STOP_ENABLE  0xf12a0004
-#define FSTV0910_P1_STOP_SDAT2SDA  0xf12a0002
+#define FSTV0910_P1_I2CT_ON  0xf12a7080
+#define FSTV0910_P1_ENARPT_LEVEL  0xf12a4070
+#define FSTV0910_P1_SCLT_DELAY  0xf12a3008
+#define FSTV0910_P1_STOP_ENABLE  0xf12a2004
+#define FSTV0910_P1_STOP_SDAT2SDA  0xf12a1002
 
 /* P2_I2CRPT */
 #define RSTV0910_P2_I2CRPT  0xf12b
-#define FSTV0910_P2_I2CT_ON  0xf12b0080
-#define FSTV0910_P2_ENARPT_LEVEL  0xf12b0070
-#define FSTV0910_P2_SCLT_DELAY  0xf12b0008
-#define FSTV0910_P2_STOP_ENABLE  0xf12b0004
-#define FSTV0910_P2_STOP_SDAT2SDA  0xf12b0002
+#define FSTV0910_P2_I2CT_ON  0xf12b7080
+#define FSTV0910_P2_ENARPT_LEVEL  0xf12b4070
+#define FSTV0910_P2_SCLT_DELAY  0xf12b3008
+#define FSTV0910_P2_STOP_ENABLE  0xf12b2004
+#define FSTV0910_P2_STOP_SDAT2SDA  0xf12b1002
 
 /* GPIO0CFG */
 #define RSTV0910_GPIO0CFG  0xf140
-#define FSTV0910_GPIO0_OPD  0xf1400080
-#define FSTV0910_GPIO0_CONFIG  0xf140007e
+#define FSTV0910_GPIO0_OPD  0xf1407080
+#define FSTV0910_GPIO0_CONFIG  0xf140107e
 #define FSTV0910_GPIO0_XOR  0xf1400001
 
 /* GPIO1CFG */
 #define RSTV0910_GPIO1CFG  0xf141
-#define FSTV0910_GPIO1_OPD  0xf1410080
-#define FSTV0910_GPIO1_CONFIG  0xf141007e
+#define FSTV0910_GPIO1_OPD  0xf1417080
+#define FSTV0910_GPIO1_CONFIG  0xf141107e
 #define FSTV0910_GPIO1_XOR  0xf1410001
 
 /* GPIO2CFG */
 #define RSTV0910_GPIO2CFG  0xf142
-#define FSTV0910_GPIO2_OPD  0xf1420080
-#define FSTV0910_GPIO2_CONFIG  0xf142007e
+#define FSTV0910_GPIO2_OPD  0xf1427080
+#define FSTV0910_GPIO2_CONFIG  0xf142107e
 #define FSTV0910_GPIO2_XOR  0xf1420001
 
 /* GPIO3CFG */
 #define RSTV0910_GPIO3CFG  0xf143
-#define FSTV0910_GPIO3_OPD  0xf1430080
-#define FSTV0910_GPIO3_CONFIG  0xf143007e
+#define FSTV0910_GPIO3_OPD  0xf1437080
+#define FSTV0910_GPIO3_CONFIG  0xf143107e
 #define FSTV0910_GPIO3_XOR  0xf1430001
 
 /* GPIO4CFG */
 #define RSTV0910_GPIO4CFG  0xf144
-#define FSTV0910_GPIO4_OPD  0xf1440080
-#define FSTV0910_GPIO4_CONFIG  0xf144007e
+#define FSTV0910_GPIO4_OPD  0xf1447080
+#define FSTV0910_GPIO4_CONFIG  0xf144107e
 #define FSTV0910_GPIO4_XOR  0xf1440001
 
 /* GPIO5CFG */
 #define RSTV0910_GPIO5CFG  0xf145
-#define FSTV0910_GPIO5_OPD  0xf1450080
-#define FSTV0910_GPIO5_CONFIG  0xf145007e
+#define FSTV0910_GPIO5_OPD  0xf1457080
+#define FSTV0910_GPIO5_CONFIG  0xf145107e
 #define FSTV0910_GPIO5_XOR  0xf1450001
 
 /* GPIO6CFG */
 #define RSTV0910_GPIO6CFG  0xf146
-#define FSTV0910_GPIO6_OPD  0xf1460080
-#define FSTV0910_GPIO6_CONFIG  0xf146007e
+#define FSTV0910_GPIO6_OPD  0xf1467080
+#define FSTV0910_GPIO6_CONFIG  0xf146107e
 #define FSTV0910_GPIO6_XOR  0xf1460001
 
 /* GPIO7CFG */
 #define RSTV0910_GPIO7CFG  0xf147
-#define FSTV0910_GPIO7_OPD  0xf1470080
-#define FSTV0910_GPIO7_CONFIG  0xf147007e
+#define FSTV0910_GPIO7_OPD  0xf1477080
+#define FSTV0910_GPIO7_CONFIG  0xf147107e
 #define FSTV0910_GPIO7_XOR  0xf1470001
 
 /* GPIO8CFG */
 #define RSTV0910_GPIO8CFG  0xf148
-#define FSTV0910_GPIO8_OPD  0xf1480080
-#define FSTV0910_GPIO8_CONFIG  0xf148007e
+#define FSTV0910_GPIO8_OPD  0xf1487080
+#define FSTV0910_GPIO8_CONFIG  0xf148107e
 #define FSTV0910_GPIO8_XOR  0xf1480001
 
 /* GPIO9CFG */
 #define RSTV0910_GPIO9CFG  0xf149
-#define FSTV0910_GPIO9_OPD  0xf1490080
-#define FSTV0910_GPIO9_CONFIG  0xf149007e
+#define FSTV0910_GPIO9_OPD  0xf1497080
+#define FSTV0910_GPIO9_CONFIG  0xf149107e
 #define FSTV0910_GPIO9_XOR  0xf1490001
 
 /* GPIO10CFG */
 #define RSTV0910_GPIO10CFG  0xf14a
-#define FSTV0910_GPIO10_OPD  0xf14a0080
-#define FSTV0910_GPIO10_CONFIG  0xf14a007e
+#define FSTV0910_GPIO10_OPD  0xf14a7080
+#define FSTV0910_GPIO10_CONFIG  0xf14a107e
 #define FSTV0910_GPIO10_XOR  0xf14a0001
 
 /* GPIO11CFG */
 #define RSTV0910_GPIO11CFG  0xf14b
-#define FSTV0910_GPIO11_OPD  0xf14b0080
-#define FSTV0910_GPIO11_CONFIG  0xf14b007e
+#define FSTV0910_GPIO11_OPD  0xf14b7080
+#define FSTV0910_GPIO11_CONFIG  0xf14b107e
 #define FSTV0910_GPIO11_XOR  0xf14b0001
 
 /* GPIO12CFG */
 #define RSTV0910_GPIO12CFG  0xf14c
-#define FSTV0910_GPIO12_OPD  0xf14c0080
-#define FSTV0910_GPIO12_CONFIG  0xf14c007e
+#define FSTV0910_GPIO12_OPD  0xf14c7080
+#define FSTV0910_GPIO12_CONFIG  0xf14c107e
 #define FSTV0910_GPIO12_XOR  0xf14c0001
 
 /* GPIO13CFG */
 #define RSTV0910_GPIO13CFG  0xf14d
-#define FSTV0910_GPIO13_OPD  0xf14d0080
-#define FSTV0910_GPIO13_CONFIG  0xf14d007e
+#define FSTV0910_GPIO13_OPD  0xf14d7080
+#define FSTV0910_GPIO13_CONFIG  0xf14d107e
 #define FSTV0910_GPIO13_XOR  0xf14d0001
 
 /* GPIO14CFG */
 #define RSTV0910_GPIO14CFG  0xf14e
-#define FSTV0910_GPIO14_OPD  0xf14e0080
-#define FSTV0910_GPIO14_CONFIG  0xf14e007e
+#define FSTV0910_GPIO14_OPD  0xf14e7080
+#define FSTV0910_GPIO14_CONFIG  0xf14e107e
 #define FSTV0910_GPIO14_XOR  0xf14e0001
 
 /* GPIO15CFG */
 #define RSTV0910_GPIO15CFG  0xf14f
-#define FSTV0910_GPIO15_OPD  0xf14f0080
-#define FSTV0910_GPIO15_CONFIG  0xf14f007e
+#define FSTV0910_GPIO15_OPD  0xf14f7080
+#define FSTV0910_GPIO15_CONFIG  0xf14f107e
 #define FSTV0910_GPIO15_XOR  0xf14f0001
 
 /* GPIO16CFG */
 #define RSTV0910_GPIO16CFG  0xf150
-#define FSTV0910_GPIO16_OPD  0xf1500080
-#define FSTV0910_GPIO16_CONFIG  0xf150007e
+#define FSTV0910_GPIO16_OPD  0xf1507080
+#define FSTV0910_GPIO16_CONFIG  0xf150107e
 #define FSTV0910_GPIO16_XOR  0xf1500001
 
 /* GPIO17CFG */
 #define RSTV0910_GPIO17CFG  0xf151
-#define FSTV0910_GPIO17_OPD  0xf1510080
-#define FSTV0910_GPIO17_CONFIG  0xf151007e
+#define FSTV0910_GPIO17_OPD  0xf1517080
+#define FSTV0910_GPIO17_CONFIG  0xf151107e
 #define FSTV0910_GPIO17_XOR  0xf1510001
 
 /* GPIO18CFG */
 #define RSTV0910_GPIO18CFG  0xf152
-#define FSTV0910_GPIO18_OPD  0xf1520080
-#define FSTV0910_GPIO18_CONFIG  0xf152007e
+#define FSTV0910_GPIO18_OPD  0xf1527080
+#define FSTV0910_GPIO18_CONFIG  0xf152107e
 #define FSTV0910_GPIO18_XOR  0xf1520001
 
 /* GPIO19CFG */
 #define RSTV0910_GPIO19CFG  0xf153
-#define FSTV0910_GPIO19_OPD  0xf1530080
-#define FSTV0910_GPIO19_CONFIG  0xf153007e
+#define FSTV0910_GPIO19_OPD  0xf1537080
+#define FSTV0910_GPIO19_CONFIG  0xf153107e
 #define FSTV0910_GPIO19_XOR  0xf1530001
 
 /* GPIO20CFG */
 #define RSTV0910_GPIO20CFG  0xf154
-#define FSTV0910_GPIO20_OPD  0xf1540080
-#define FSTV0910_GPIO20_CONFIG  0xf154007e
+#define FSTV0910_GPIO20_OPD  0xf1547080
+#define FSTV0910_GPIO20_CONFIG  0xf154107e
 #define FSTV0910_GPIO20_XOR  0xf1540001
 
 /* GPIO21CFG */
 #define RSTV0910_GPIO21CFG  0xf155
-#define FSTV0910_GPIO21_OPD  0xf1550080
-#define FSTV0910_GPIO21_CONFIG  0xf155007e
+#define FSTV0910_GPIO21_OPD  0xf1557080
+#define FSTV0910_GPIO21_CONFIG  0xf155107e
 #define FSTV0910_GPIO21_XOR  0xf1550001
 
 /* GPIO22CFG */
 #define RSTV0910_GPIO22CFG  0xf156
-#define FSTV0910_GPIO22_OPD  0xf1560080
-#define FSTV0910_GPIO22_CONFIG  0xf156007e
+#define FSTV0910_GPIO22_OPD  0xf1567080
+#define FSTV0910_GPIO22_CONFIG  0xf156107e
 #define FSTV0910_GPIO22_XOR  0xf1560001
 
 /* STRSTATUS1 */
 #define RSTV0910_STRSTATUS1  0xf16a
-#define FSTV0910_STRSTATUS_SEL2  0xf16a00f0
+#define FSTV0910_STRSTATUS_SEL2  0xf16a40f0
 #define FSTV0910_STRSTATUS_SEL1  0xf16a000f
 
 /* STRSTATUS2 */
 #define RSTV0910_STRSTATUS2  0xf16b
-#define FSTV0910_STRSTATUS_SEL4  0xf16b00f0
+#define FSTV0910_STRSTATUS_SEL4  0xf16b40f0
 #define FSTV0910_STRSTATUS_SEL3  0xf16b000f
 
 /* STRSTATUS3 */
 #define RSTV0910_STRSTATUS3  0xf16c
-#define FSTV0910_STRSTATUS_SEL6  0xf16c00f0
+#define FSTV0910_STRSTATUS_SEL6  0xf16c40f0
 #define FSTV0910_STRSTATUS_SEL5  0xf16c000f
 
 /* FSKTFC2 */
 #define RSTV0910_FSKTFC2  0xf170
-#define FSTV0910_FSKT_KMOD  0xf17000fc
+#define FSTV0910_FSKT_KMOD  0xf17020fc
 #define FSTV0910_FSKT_CAR2  0xf1700003
 
 /* FSKTFC1 */
@@ -330,17 +330,17 @@
 
 /* FSKTCTRL */
 #define RSTV0910_FSKTCTRL  0xf175
-#define FSTV0910_FSKT_PINSEL  0xf1750080
-#define FSTV0910_FSKT_EN_SGN  0xf1750040
-#define FSTV0910_FSKT_MOD_SGN  0xf1750020
-#define FSTV0910_FSKT_MOD_EN  0xf175001c
+#define FSTV0910_FSKT_PINSEL  0xf1757080
+#define FSTV0910_FSKT_EN_SGN  0xf1756040
+#define FSTV0910_FSKT_MOD_SGN  0xf1755020
+#define FSTV0910_FSKT_MOD_EN  0xf175201c
 #define FSTV0910_FSKT_DACMODE  0xf1750003
 
 /* FSKRFC2 */
 #define RSTV0910_FSKRFC2  0xf176
-#define FSTV0910_FSKR_DETSGN  0xf1760040
-#define FSTV0910_FSKR_OUTSGN  0xf1760020
-#define FSTV0910_FSKR_KAGC  0xf176001c
+#define FSTV0910_FSKR_DETSGN  0xf1766040
+#define FSTV0910_FSKR_OUTSGN  0xf1765020
+#define FSTV0910_FSKR_KAGC  0xf176201c
 #define FSTV0910_FSKR_CAR2  0xf1760003
 
 /* FSKRFC1 */
@@ -353,17 +353,17 @@
 
 /* FSKRK1 */
 #define RSTV0910_FSKRK1  0xf179
-#define FSTV0910_FSKR_K1_EXP  0xf17900e0
+#define FSTV0910_FSKR_K1_EXP  0xf17950e0
 #define FSTV0910_FSKR_K1_MANT  0xf179001f
 
 /* FSKRK2 */
 #define RSTV0910_FSKRK2  0xf17a
-#define FSTV0910_FSKR_K2_EXP  0xf17a00e0
+#define FSTV0910_FSKR_K2_EXP  0xf17a50e0
 #define FSTV0910_FSKR_K2_MANT  0xf17a001f
 
 /* FSKRAGCR */
 #define RSTV0910_FSKRAGCR  0xf17b
-#define FSTV0910_FSKR_OUTCTL  0xf17b00c0
+#define FSTV0910_FSKR_OUTCTL  0xf17b60c0
 #define FSTV0910_FSKR_AGC_REF  0xf17b003f
 
 /* FSKRAGC */
@@ -372,12 +372,12 @@
 
 /* FSKRALPHA */
 #define RSTV0910_FSKRALPHA  0xf17d
-#define FSTV0910_FSKR_ALPHA_EXP  0xf17d001c
+#define FSTV0910_FSKR_ALPHA_EXP  0xf17d201c
 #define FSTV0910_FSKR_ALPHA_M  0xf17d0003
 
 /* FSKRPLTH1 */
 #define RSTV0910_FSKRPLTH1  0xf17e
-#define FSTV0910_FSKR_BETA  0xf17e00f0
+#define FSTV0910_FSKR_BETA  0xf17e40f0
 #define FSTV0910_FSKR_PLL_TRESH1  0xf17e000f
 
 /* FSKRPLTH0 */
@@ -386,8 +386,8 @@
 
 /* FSKRDF1 */
 #define RSTV0910_FSKRDF1  0xf180
-#define FSTV0910_FSKR_OUT  0xf1800080
-#define FSTV0910_FSKR_STATE  0xf1800060
+#define FSTV0910_FSKR_OUT  0xf1807080
+#define FSTV0910_FSKR_STATE  0xf1805060
 #define FSTV0910_FSKR_DELTAF1  0xf180001f
 
 /* FSKRDF0 */
@@ -404,7 +404,7 @@
 
 /* FSKRDET1 */
 #define RSTV0910_FSKRDET1  0xf184
-#define FSTV0910_FSKR_DETECT  0xf1840080
+#define FSTV0910_FSKR_DETECT  0xf1847080
 #define FSTV0910_FSKR_CARDET_ACCU1  0xf184000f
 
 /* FSKRDET0 */
@@ -413,7 +413,7 @@
 
 /* FSKRDTH1 */
 #define RSTV0910_FSKRDTH1  0xf186
-#define FSTV0910_FSKR_CARLOSS_THRESH1  0xf18600f0
+#define FSTV0910_FSKR_CARLOSS_THRESH1  0xf18640f0
 #define FSTV0910_FSKR_CARDET_THRESH1  0xf186000f
 
 /* FSKRDTH0 */
@@ -426,7 +426,7 @@
 
 /* NCOARSE */
 #define RSTV0910_NCOARSE  0xf1b3
-#define FSTV0910_CP  0xf1b300f8
+#define FSTV0910_CP  0xf1b330f8
 #define FSTV0910_IDF  0xf1b30007
 
 /* NCOARSE1 */
@@ -439,14 +439,14 @@
 
 /* SYNTCTRL */
 #define RSTV0910_SYNTCTRL  0xf1b6
-#define FSTV0910_STANDBY  0xf1b60080
-#define FSTV0910_BYPASSPLLCORE  0xf1b60040
-#define FSTV0910_STOP_PLL  0xf1b60008
-#define FSTV0910_OSCI_E  0xf1b60002
+#define FSTV0910_STANDBY  0xf1b67080
+#define FSTV0910_BYPASSPLLCORE  0xf1b66040
+#define FSTV0910_STOP_PLL  0xf1b63008
+#define FSTV0910_OSCI_E  0xf1b61002
 
 /* FILTCTRL */
 #define RSTV0910_FILTCTRL  0xf1b7
-#define FSTV0910_INV_CLKFSK  0xf1b70002
+#define FSTV0910_INV_CLKFSK  0xf1b71002
 #define FSTV0910_BYPASS_APPLI  0xf1b70001
 
 /* PLLSTAT */
@@ -455,48 +455,48 @@
 
 /* STOPCLK1 */
 #define RSTV0910_STOPCLK1  0xf1c2
-#define FSTV0910_INV_CLKADCI2  0xf1c20004
+#define FSTV0910_INV_CLKADCI2  0xf1c22004
 #define FSTV0910_INV_CLKADCI1  0xf1c20001
 
 /* STOPCLK2 */
 #define RSTV0910_STOPCLK2  0xf1c3
-#define FSTV0910_STOP_DVBS2FEC2  0xf1c30020
-#define FSTV0910_STOP_DVBS2FEC  0xf1c30010
-#define FSTV0910_STOP_DVBS1FEC2  0xf1c30008
-#define FSTV0910_STOP_DVBS1FEC  0xf1c30004
-#define FSTV0910_STOP_DEMOD2  0xf1c30002
+#define FSTV0910_STOP_DVBS2FEC2  0xf1c35020
+#define FSTV0910_STOP_DVBS2FEC  0xf1c34010
+#define FSTV0910_STOP_DVBS1FEC2  0xf1c33008
+#define FSTV0910_STOP_DVBS1FEC  0xf1c32004
+#define FSTV0910_STOP_DEMOD2  0xf1c31002
 #define FSTV0910_STOP_DEMOD  0xf1c30001
 
 /* PREGCTL */
 #define RSTV0910_PREGCTL  0xf1c8
-#define FSTV0910_REG3V3TO2V5_POFF  0xf1c80080
+#define FSTV0910_REG3V3TO2V5_POFF  0xf1c87080
 
 /* TSTTNR0 */
 #define RSTV0910_TSTTNR0  0xf1df
-#define FSTV0910_FSK_PON  0xf1df0004
+#define FSTV0910_FSK_PON  0xf1df2004
 
 /* TSTTNR1 */
 #define RSTV0910_TSTTNR1  0xf1e0
-#define FSTV0910_ADC1_PON  0xf1e00002
+#define FSTV0910_ADC1_PON  0xf1e01002
 
 /* TSTTNR2 */
 #define RSTV0910_TSTTNR2  0xf1e1
-#define FSTV0910_I2C_DISEQC_PON  0xf1e10020
+#define FSTV0910_I2C_DISEQC_PON  0xf1e15020
 #define FSTV0910_DISEQC_CLKDIV  0xf1e1000f
 
 /* TSTTNR3 */
 #define RSTV0910_TSTTNR3  0xf1e2
-#define FSTV0910_ADC2_PON  0xf1e20002
+#define FSTV0910_ADC2_PON  0xf1e21002
 
 /* P2_IQCONST */
 #define RSTV0910_P2_IQCONST  0xf200
-#define FSTV0910_P2_CONSTEL_SELECT  0xf2000060
+#define FSTV0910_P2_CONSTEL_SELECT  0xf2005060
 #define FSTV0910_P2_IQSYMB_SEL  0xf200001f
 
 /* P2_NOSCFG */
 #define RSTV0910_P2_NOSCFG  0xf201
-#define FSTV0910_P2_DUMMYPL_NOSDATA  0xf2010020
-#define FSTV0910_P2_NOSPLH_BETA  0xf2010018
+#define FSTV0910_P2_DUMMYPL_NOSDATA  0xf2015020
+#define FSTV0910_P2_NOSPLH_BETA  0xf2013018
 #define FSTV0910_P2_NOSDATA_BETA  0xf2010007
 
 /* P2_ISYMB */
@@ -509,18 +509,18 @@
 
 /* P2_AGC1CFG */
 #define RSTV0910_P2_AGC1CFG  0xf204
-#define FSTV0910_P2_DC_FROZEN  0xf2040080
-#define FSTV0910_P2_DC_CORRECT  0xf2040040
-#define FSTV0910_P2_AMM_FROZEN  0xf2040020
-#define FSTV0910_P2_AMM_CORRECT  0xf2040010
-#define FSTV0910_P2_QUAD_FROZEN  0xf2040008
-#define FSTV0910_P2_QUAD_CORRECT  0xf2040004
+#define FSTV0910_P2_DC_FROZEN  0xf2047080
+#define FSTV0910_P2_DC_CORRECT  0xf2046040
+#define FSTV0910_P2_AMM_FROZEN  0xf2045020
+#define FSTV0910_P2_AMM_CORRECT  0xf2044010
+#define FSTV0910_P2_QUAD_FROZEN  0xf2043008
+#define FSTV0910_P2_QUAD_CORRECT  0xf2042004
 
 /* P2_AGC1CN */
 #define RSTV0910_P2_AGC1CN  0xf206
-#define FSTV0910_P2_AGC1_LOCKED  0xf2060080
-#define FSTV0910_P2_AGC1_MINPOWER  0xf2060010
-#define FSTV0910_P2_AGCOUT_FAST  0xf2060008
+#define FSTV0910_P2_AGC1_LOCKED  0xf2067080
+#define FSTV0910_P2_AGC1_MINPOWER  0xf2064010
+#define FSTV0910_P2_AGCOUT_FAST  0xf2063008
 #define FSTV0910_P2_AGCIQ_BETA  0xf2060007
 
 /* P2_AGC1REF */
@@ -561,50 +561,50 @@
 
 /* P2_DEMOD */
 #define RSTV0910_P2_DEMOD  0xf210
-#define FSTV0910_P2_MANUALS2_ROLLOFF  0xf2100080
-#define FSTV0910_P2_SPECINV_CONTROL  0xf2100030
-#define FSTV0910_P2_MANUALSX_ROLLOFF  0xf2100004
+#define FSTV0910_P2_MANUALS2_ROLLOFF  0xf2107080
+#define FSTV0910_P2_SPECINV_CONTROL  0xf2104030
+#define FSTV0910_P2_MANUALSX_ROLLOFF  0xf2102004
 #define FSTV0910_P2_ROLLOFF_CONTROL  0xf2100003
 
 /* P2_DMDMODCOD */
 #define RSTV0910_P2_DMDMODCOD  0xf211
-#define FSTV0910_P2_MANUAL_MODCOD  0xf2110080
-#define FSTV0910_P2_DEMOD_MODCOD  0xf211007c
+#define FSTV0910_P2_MANUAL_MODCOD  0xf2117080
+#define FSTV0910_P2_DEMOD_MODCOD  0xf211207c
 #define FSTV0910_P2_DEMOD_TYPE  0xf2110003
 
 /* P2_DSTATUS */
 #define RSTV0910_P2_DSTATUS  0xf212
-#define FSTV0910_P2_CAR_LOCK  0xf2120080
-#define FSTV0910_P2_TMGLOCK_QUALITY  0xf2120060
-#define FSTV0910_P2_LOCK_DEFINITIF  0xf2120008
+#define FSTV0910_P2_CAR_LOCK  0xf2127080
+#define FSTV0910_P2_TMGLOCK_QUALITY  0xf2125060
+#define FSTV0910_P2_LOCK_DEFINITIF  0xf2123008
 #define FSTV0910_P2_OVADC_DETECT  0xf2120001
 
 /* P2_DSTATUS2 */
 #define RSTV0910_P2_DSTATUS2  0xf213
-#define FSTV0910_P2_DEMOD_DELOCK  0xf2130080
-#define FSTV0910_P2_MODCODRQ_SYNCTAG  0xf2130020
-#define FSTV0910_P2_POLYPH_SATEVENT  0xf2130010
-#define FSTV0910_P2_AGC1_NOSIGNALACK  0xf2130008
-#define FSTV0910_P2_AGC2_OVERFLOW  0xf2130004
-#define FSTV0910_P2_CFR_OVERFLOW  0xf2130002
+#define FSTV0910_P2_DEMOD_DELOCK  0xf2137080
+#define FSTV0910_P2_MODCODRQ_SYNCTAG  0xf2135020
+#define FSTV0910_P2_POLYPH_SATEVENT  0xf2134010
+#define FSTV0910_P2_AGC1_NOSIGNALACK  0xf2133008
+#define FSTV0910_P2_AGC2_OVERFLOW  0xf2132004
+#define FSTV0910_P2_CFR_OVERFLOW  0xf2131002
 #define FSTV0910_P2_GAMMA_OVERUNDER  0xf2130001
 
 /* P2_DMDCFGMD */
 #define RSTV0910_P2_DMDCFGMD  0xf214
-#define FSTV0910_P2_DVBS2_ENABLE  0xf2140080
-#define FSTV0910_P2_DVBS1_ENABLE  0xf2140040
-#define FSTV0910_P2_SCAN_ENABLE  0xf2140010
-#define FSTV0910_P2_CFR_AUTOSCAN  0xf2140008
+#define FSTV0910_P2_DVBS2_ENABLE  0xf2147080
+#define FSTV0910_P2_DVBS1_ENABLE  0xf2146040
+#define FSTV0910_P2_SCAN_ENABLE  0xf2144010
+#define FSTV0910_P2_CFR_AUTOSCAN  0xf2143008
 #define FSTV0910_P2_TUN_RNG  0xf2140003
 
 /* P2_DMDCFG2 */
 #define RSTV0910_P2_DMDCFG2  0xf215
-#define FSTV0910_P2_S1S2_SEQUENTIAL  0xf2150040
-#define FSTV0910_P2_INFINITE_RELOCK  0xf2150010
+#define FSTV0910_P2_S1S2_SEQUENTIAL  0xf2156040
+#define FSTV0910_P2_INFINITE_RELOCK  0xf2154010
 
 /* P2_DMDISTATE */
 #define RSTV0910_P2_DMDISTATE  0xf216
-#define FSTV0910_P2_I2C_NORESETDMODE  0xf2160080
+#define FSTV0910_P2_I2C_NORESETDMODE  0xf2167080
 #define FSTV0910_P2_I2C_DEMOD_MODE  0xf216001f
 
 /* P2_DMDT0M */
@@ -613,27 +613,27 @@
 
 /* P2_DMDSTATE */
 #define RSTV0910_P2_DMDSTATE  0xf21b
-#define FSTV0910_P2_HEADER_MODE  0xf21b0060
+#define FSTV0910_P2_HEADER_MODE  0xf21b5060
 
 /* P2_DMDFLYW */
 #define RSTV0910_P2_DMDFLYW  0xf21c
-#define FSTV0910_P2_I2C_IRQVAL  0xf21c00f0
+#define FSTV0910_P2_I2C_IRQVAL  0xf21c40f0
 #define FSTV0910_P2_FLYWHEEL_CPT  0xf21c000f
 
 /* P2_DSTATUS3 */
 #define RSTV0910_P2_DSTATUS3  0xf21d
-#define FSTV0910_P2_CFR_ZIGZAG  0xf21d0080
-#define FSTV0910_P2_DEMOD_CFGMODE  0xf21d0060
-#define FSTV0910_P2_GAMMA_LOWBAUDRATE  0xf21d0010
+#define FSTV0910_P2_CFR_ZIGZAG  0xf21d7080
+#define FSTV0910_P2_DEMOD_CFGMODE  0xf21d5060
+#define FSTV0910_P2_GAMMA_LOWBAUDRATE  0xf21d4010
 
 /* P2_DMDCFG3 */
 #define RSTV0910_P2_DMDCFG3  0xf21e
-#define FSTV0910_P2_NOSTOP_FIFOFULL  0xf21e0008
+#define FSTV0910_P2_NOSTOP_FIFOFULL  0xf21e3008
 
 /* P2_DMDCFG4 */
 #define RSTV0910_P2_DMDCFG4  0xf21f
-#define FSTV0910_P2_DIS_VITLOCK  0xf21f0080
-#define FSTV0910_P2_DIS_CLKENABLE  0xf21f0004
+#define FSTV0910_P2_DIS_VITLOCK  0xf21f7080
+#define FSTV0910_P2_DIS_CLKENABLE  0xf21f2004
 
 /* P2_CORRELMANT */
 #define RSTV0910_P2_CORRELMANT  0xf220
@@ -645,13 +645,13 @@
 
 /* P2_CORRELEXP */
 #define RSTV0910_P2_CORRELEXP  0xf222
-#define FSTV0910_P2_CORREL_ABSEXP  0xf22200f0
+#define FSTV0910_P2_CORREL_ABSEXP  0xf22240f0
 #define FSTV0910_P2_CORREL_EXP  0xf222000f
 
 /* P2_PLHMODCOD */
 #define RSTV0910_P2_PLHMODCOD  0xf224
-#define FSTV0910_P2_SPECINV_DEMOD  0xf2240080
-#define FSTV0910_P2_PLH_MODCOD  0xf224007c
+#define FSTV0910_P2_SPECINV_DEMOD  0xf2247080
+#define FSTV0910_P2_PLH_MODCOD  0xf224207c
 #define FSTV0910_P2_PLH_TYPE  0xf2240003
 
 /* P2_DMDREG */
@@ -660,19 +660,19 @@
 
 /* P2_AGCNADJ */
 #define RSTV0910_P2_AGCNADJ  0xf226
-#define FSTV0910_P2_RADJOFF_AGC2  0xf2260080
-#define FSTV0910_P2_RADJOFF_AGC1  0xf2260040
+#define FSTV0910_P2_RADJOFF_AGC2  0xf2267080
+#define FSTV0910_P2_RADJOFF_AGC1  0xf2266040
 #define FSTV0910_P2_AGC_NADJ  0xf226013f
 
 /* P2_AGCKS */
 #define RSTV0910_P2_AGCKS  0xf227
-#define FSTV0910_P2_RSADJ_MANUALCFG  0xf2270080
-#define FSTV0910_P2_RSADJ_CCMMODE  0xf2270040
+#define FSTV0910_P2_RSADJ_MANUALCFG  0xf2277080
+#define FSTV0910_P2_RSADJ_CCMMODE  0xf2276040
 #define FSTV0910_P2_RADJ_SPSK  0xf227013f
 
 /* P2_AGCKQ */
 #define RSTV0910_P2_AGCKQ  0xf228
-#define FSTV0910_P2_RADJON_DVBS1  0xf2280040
+#define FSTV0910_P2_RADJON_DVBS1  0xf2286040
 #define FSTV0910_P2_RADJ_QPSK  0xf228013f
 
 /* P2_AGCK8 */
@@ -681,20 +681,20 @@
 
 /* P2_AGCK16 */
 #define RSTV0910_P2_AGCK16  0xf22a
-#define FSTV0910_P2_R2ADJOFF_16APSK  0xf22a0040
-#define FSTV0910_P2_R1ADJOFF_16APSK  0xf22a0020
+#define FSTV0910_P2_R2ADJOFF_16APSK  0xf22a6040
+#define FSTV0910_P2_R1ADJOFF_16APSK  0xf22a5020
 #define FSTV0910_P2_RADJ_16APSK  0xf22a011f
 
 /* P2_AGCK32 */
 #define RSTV0910_P2_AGCK32  0xf22b
-#define FSTV0910_P2_R3ADJOFF_32APSK  0xf22b0080
-#define FSTV0910_P2_R2ADJOFF_32APSK  0xf22b0040
-#define FSTV0910_P2_R1ADJOFF_32APSK  0xf22b0020
+#define FSTV0910_P2_R3ADJOFF_32APSK  0xf22b7080
+#define FSTV0910_P2_R2ADJOFF_32APSK  0xf22b6040
+#define FSTV0910_P2_R1ADJOFF_32APSK  0xf22b5020
 #define FSTV0910_P2_RADJ_32APSK  0xf22b011f
 
 /* P2_AGC2O */
 #define RSTV0910_P2_AGC2O  0xf22c
-#define FSTV0910_P2_CSTENV_MODE  0xf22c00c0
+#define FSTV0910_P2_CSTENV_MODE  0xf22c60c0
 #define FSTV0910_P2_AGC2_COEF  0xf22c0007
 
 /* P2_AGC2REF */
@@ -743,32 +743,32 @@
 
 /* P2_CARCFG */
 #define RSTV0910_P2_CARCFG  0xf238
-#define FSTV0910_P2_ROTAON  0xf2380004
+#define FSTV0910_P2_ROTAON  0xf2382004
 #define FSTV0910_P2_PH_DET_ALGO  0xf2380003
 
 /* P2_ACLC */
 #define RSTV0910_P2_ACLC  0xf239
-#define FSTV0910_P2_CAR_ALPHA_MANT  0xf2390030
+#define FSTV0910_P2_CAR_ALPHA_MANT  0xf2394030
 #define FSTV0910_P2_CAR_ALPHA_EXP  0xf239000f
 
 /* P2_BCLC */
 #define RSTV0910_P2_BCLC  0xf23a
-#define FSTV0910_P2_CAR_BETA_MANT  0xf23a0030
+#define FSTV0910_P2_CAR_BETA_MANT  0xf23a4030
 #define FSTV0910_P2_CAR_BETA_EXP  0xf23a000f
 
 /* P2_ACLCS2 */
 #define RSTV0910_P2_ACLCS2  0xf23b
-#define FSTV0910_P2_CARS2_APLHA_MANTISSE  0xf23b0030
+#define FSTV0910_P2_CARS2_APLHA_MANTISSE  0xf23b4030
 #define FSTV0910_P2_CARS2_ALPHA_EXP  0xf23b000f
 
 /* P2_BCLCS2 */
 #define RSTV0910_P2_BCLCS2  0xf23c
-#define FSTV0910_P2_CARS2_BETA_MANTISSE  0xf23c0030
+#define FSTV0910_P2_CARS2_BETA_MANTISSE  0xf23c4030
 #define FSTV0910_P2_CARS2_BETA_EXP  0xf23c000f
 
 /* P2_CARFREQ */
 #define RSTV0910_P2_CARFREQ  0xf23d
-#define FSTV0910_P2_KC_COARSE_EXP  0xf23d00f0
+#define FSTV0910_P2_KC_COARSE_EXP  0xf23d40f0
 #define FSTV0910_P2_BETA_FREQ  0xf23d000f
 
 /* P2_CARHDR */
@@ -821,7 +821,7 @@
 
 /* P2_CFRINC1 */
 #define RSTV0910_P2_CFRINC1  0xf24a
-#define FSTV0910_P2_MANUAL_CFRINC  0xf24a0080
+#define FSTV0910_P2_MANUAL_CFRINC  0xf24a7080
 #define FSTV0910_P2_CFR_INC1  0xf24a003f
 
 /* P2_CFRINC0 */
@@ -846,18 +846,18 @@
 
 /* P2_TMGCFG */
 #define RSTV0910_P2_TMGCFG  0xf250
-#define FSTV0910_P2_TMGLOCK_BETA  0xf25000c0
-#define FSTV0910_P2_DO_TIMING_CORR  0xf2500010
+#define FSTV0910_P2_TMGLOCK_BETA  0xf25060c0
+#define FSTV0910_P2_DO_TIMING_CORR  0xf2504010
 #define FSTV0910_P2_TMG_MINFREQ  0xf2500003
 
 /* P2_RTC */
 #define RSTV0910_P2_RTC  0xf251
-#define FSTV0910_P2_TMGALPHA_EXP  0xf25100f0
+#define FSTV0910_P2_TMGALPHA_EXP  0xf25140f0
 #define FSTV0910_P2_TMGBETA_EXP  0xf251000f
 
 /* P2_RTCS2 */
 #define RSTV0910_P2_RTCS2  0xf252
-#define FSTV0910_P2_TMGALPHAS2_EXP  0xf25200f0
+#define FSTV0910_P2_TMGALPHAS2_EXP  0xf25240f0
 #define FSTV0910_P2_TMGBETAS2_EXP  0xf252000f
 
 /* P2_TMGTHRISE */
@@ -878,7 +878,7 @@
 
 /* P2_KTTMG */
 #define RSTV0910_P2_KTTMG  0xf257
-#define FSTV0910_P2_KT_TMG_EXP  0xf25700f0
+#define FSTV0910_P2_KT_TMG_EXP  0xf25740f0
 
 /* P2_KREFTMG */
 #define RSTV0910_P2_KREFTMG  0xf258
@@ -886,12 +886,12 @@
 
 /* P2_SFRSTEP */
 #define RSTV0910_P2_SFRSTEP  0xf259
-#define FSTV0910_P2_SFR_SCANSTEP  0xf25900f0
+#define FSTV0910_P2_SFR_SCANSTEP  0xf25940f0
 #define FSTV0910_P2_SFR_CENTERSTEP  0xf259000f
 
 /* P2_TMGCFG2 */
 #define RSTV0910_P2_TMGCFG2  0xf25a
-#define FSTV0910_P2_DIS_AUTOSAMP  0xf25a0008
+#define FSTV0910_P2_DIS_AUTOSAMP  0xf25a3008
 #define FSTV0910_P2_SFRRATIO_FINE  0xf25a0001
 
 /* P2_KREFTMG2 */
@@ -900,9 +900,9 @@
 
 /* P2_TMGCFG3 */
 #define RSTV0910_P2_TMGCFG3  0xf25d
-#define FSTV0910_P2_CONT_TMGCENTER  0xf25d0008
-#define FSTV0910_P2_AUTO_GUP  0xf25d0004
-#define FSTV0910_P2_AUTO_GLOW  0xf25d0002
+#define FSTV0910_P2_CONT_TMGCENTER  0xf25d3008
+#define FSTV0910_P2_AUTO_GUP  0xf25d2004
+#define FSTV0910_P2_AUTO_GLOW  0xf25d1002
 
 /* P2_SFRINIT1 */
 #define RSTV0910_P2_SFRINIT1  0xf25e
@@ -966,11 +966,11 @@
 
 /* P2_TMGOBS */
 #define RSTV0910_P2_TMGOBS  0xf26d
-#define FSTV0910_P2_ROLLOFF_STATUS  0xf26d00c0
+#define FSTV0910_P2_ROLLOFF_STATUS  0xf26d60c0
 
 /* P2_EQUALCFG */
 #define RSTV0910_P2_EQUALCFG  0xf26f
-#define FSTV0910_P2_EQUAL_ON  0xf26f0040
+#define FSTV0910_P2_EQUAL_ON  0xf26f6040
 #define FSTV0910_P2_MU_EQUALDFE  0xf26f0007
 
 /* P2_EQUAI1 */
@@ -1095,33 +1095,33 @@
 
 /* P2_NOSCFGF1 */
 #define RSTV0910_P2_NOSCFGF1  0xf28e
-#define FSTV0910_P2_LOWNOISE_MESURE  0xf28e0080
-#define FSTV0910_P2_NOS_DELFRAME  0xf28e0040
-#define FSTV0910_P2_NOSDATA_MODE  0xf28e0030
-#define FSTV0910_P2_FRAMESEL_TYPESEL  0xf28e000c
+#define FSTV0910_P2_LOWNOISE_MESURE  0xf28e7080
+#define FSTV0910_P2_NOS_DELFRAME  0xf28e6040
+#define FSTV0910_P2_NOSDATA_MODE  0xf28e4030
+#define FSTV0910_P2_FRAMESEL_TYPESEL  0xf28e200c
 #define FSTV0910_P2_FRAMESEL_TYPE  0xf28e0003
 
 /* P2_NOSCFGF2 */
 #define RSTV0910_P2_NOSCFGF2  0xf28f
-#define FSTV0910_P2_DIS_NOSPILOTS  0xf28f0080
-#define FSTV0910_P2_FRAMESEL_MODCODSEL  0xf28f0060
+#define FSTV0910_P2_DIS_NOSPILOTS  0xf28f7080
+#define FSTV0910_P2_FRAMESEL_MODCODSEL  0xf28f5060
 #define FSTV0910_P2_FRAMESEL_MODCOD  0xf28f001f
 
 /* P2_CAR2CFG */
 #define RSTV0910_P2_CAR2CFG  0xf290
-#define FSTV0910_P2_ROTA2ON  0xf2900004
+#define FSTV0910_P2_ROTA2ON  0xf2902004
 #define FSTV0910_P2_PH_DET_ALGO2  0xf2900003
 
 /* P2_CFR2CFR1 */
 #define RSTV0910_P2_CFR2CFR1  0xf291
-#define FSTV0910_P2_EN_S2CAR2CENTER  0xf2910020
+#define FSTV0910_P2_EN_S2CAR2CENTER  0xf2915020
 #define FSTV0910_P2_CFR2TOCFR1_BETA  0xf2910007
 
 /* P2_CAR3CFG */
 #define RSTV0910_P2_CAR3CFG  0xf292
-#define FSTV0910_P2_CARRIER23_MODE  0xf29200c0
-#define FSTV0910_P2_CAR3INTERM_DVBS1  0xf2920020
-#define FSTV0910_P2_ABAMPLIF_MODE  0xf2920018
+#define FSTV0910_P2_CARRIER23_MODE  0xf29260c0
+#define FSTV0910_P2_CAR3INTERM_DVBS1  0xf2925020
+#define FSTV0910_P2_ABAMPLIF_MODE  0xf2923018
 #define FSTV0910_P2_CARRIER3_ALPHA3DL  0xf2920007
 
 /* P2_CFR22 */
@@ -1138,50 +1138,50 @@
 
 /* P2_ACLC2S2Q */
 #define RSTV0910_P2_ACLC2S2Q  0xf297
-#define FSTV0910_P2_ENAB_SPSKSYMB  0xf2970080
-#define FSTV0910_P2_CAR2S2_Q_ALPH_M  0xf2970030
+#define FSTV0910_P2_ENAB_SPSKSYMB  0xf2977080
+#define FSTV0910_P2_CAR2S2_Q_ALPH_M  0xf2974030
 #define FSTV0910_P2_CAR2S2_Q_ALPH_E  0xf297000f
 
 /* P2_ACLC2S28 */
 #define RSTV0910_P2_ACLC2S28  0xf298
-#define FSTV0910_P2_CAR2S2_8_ALPH_M  0xf2980030
+#define FSTV0910_P2_CAR2S2_8_ALPH_M  0xf2984030
 #define FSTV0910_P2_CAR2S2_8_ALPH_E  0xf298000f
 
 /* P2_ACLC2S216A */
 #define RSTV0910_P2_ACLC2S216A  0xf299
-#define FSTV0910_P2_CAR2S2_16A_ALPH_M  0xf2990030
+#define FSTV0910_P2_CAR2S2_16A_ALPH_M  0xf2994030
 #define FSTV0910_P2_CAR2S2_16A_ALPH_E  0xf299000f
 
 /* P2_ACLC2S232A */
 #define RSTV0910_P2_ACLC2S232A  0xf29a
-#define FSTV0910_P2_CAR2S2_32A_ALPH_M  0xf29a0030
+#define FSTV0910_P2_CAR2S2_32A_ALPH_M  0xf29a4030
 #define FSTV0910_P2_CAR2S2_32A_ALPH_E  0xf29a000f
 
 /* P2_BCLC2S2Q */
 #define RSTV0910_P2_BCLC2S2Q  0xf29c
-#define FSTV0910_P2_CAR2S2_Q_BETA_M  0xf29c0030
+#define FSTV0910_P2_CAR2S2_Q_BETA_M  0xf29c4030
 #define FSTV0910_P2_CAR2S2_Q_BETA_E  0xf29c000f
 
 /* P2_BCLC2S28 */
 #define RSTV0910_P2_BCLC2S28  0xf29d
-#define FSTV0910_P2_CAR2S2_8_BETA_M  0xf29d0030
+#define FSTV0910_P2_CAR2S2_8_BETA_M  0xf29d4030
 #define FSTV0910_P2_CAR2S2_8_BETA_E  0xf29d000f
 
 /* P2_BCLC2S216A */
 #define RSTV0910_P2_BCLC2S216A  0xf29e
-#define FSTV0910_P2_DVBS2S216A_NIP  0xf29e0080
-#define FSTV0910_P2_CAR2S2_16A_BETA_M  0xf29e0030
+#define FSTV0910_P2_DVBS2S216A_NIP  0xf29e7080
+#define FSTV0910_P2_CAR2S2_16A_BETA_M  0xf29e4030
 #define FSTV0910_P2_CAR2S2_16A_BETA_E  0xf29e000f
 
 /* P2_BCLC2S232A */
 #define RSTV0910_P2_BCLC2S232A  0xf29f
-#define FSTV0910_P2_DVBS2S232A_NIP  0xf29f0080
-#define FSTV0910_P2_CAR2S2_32A_BETA_M  0xf29f0030
+#define FSTV0910_P2_DVBS2S232A_NIP  0xf29f7080
+#define FSTV0910_P2_CAR2S2_32A_BETA_M  0xf29f4030
 #define FSTV0910_P2_CAR2S2_32A_BETA_E  0xf29f000f
 
 /* P2_PLROOT2 */
 #define RSTV0910_P2_PLROOT2  0xf2ac
-#define FSTV0910_P2_PLSCRAMB_MODE  0xf2ac000c
+#define FSTV0910_P2_PLSCRAMB_MODE  0xf2ac200c
 #define FSTV0910_P2_PLSCRAMB_ROOT2  0xf2ac0003
 
 /* P2_PLROOT1 */
@@ -1198,100 +1198,100 @@
 
 /* P2_MODCODLST1 */
 #define RSTV0910_P2_MODCODLST1  0xf2b1
-#define FSTV0910_P2_SYMBRATE_FILTER  0xf2b10008
-#define FSTV0910_P2_NRESET_MODCODLST  0xf2b10004
+#define FSTV0910_P2_SYMBRATE_FILTER  0xf2b13008
+#define FSTV0910_P2_NRESET_MODCODLST  0xf2b12004
 #define FSTV0910_P2_DIS_32PSK_9_10  0xf2b10003
 
 /* P2_MODCODLST2 */
 #define RSTV0910_P2_MODCODLST2  0xf2b2
-#define FSTV0910_P2_DIS_32PSK_8_9  0xf2b200f0
+#define FSTV0910_P2_DIS_32PSK_8_9  0xf2b240f0
 #define FSTV0910_P2_DIS_32PSK_5_6  0xf2b2000f
 
 /* P2_MODCODLST3 */
 #define RSTV0910_P2_MODCODLST3  0xf2b3
-#define FSTV0910_P2_DIS_32PSK_4_5  0xf2b300f0
+#define FSTV0910_P2_DIS_32PSK_4_5  0xf2b340f0
 #define FSTV0910_P2_DIS_32PSK_3_4  0xf2b3000f
 
 /* P2_MODCODLST4 */
 #define RSTV0910_P2_MODCODLST4  0xf2b4
-#define FSTV0910_P2_DUMMYPL_PILOT  0xf2b40080
-#define FSTV0910_P2_DUMMYPL_NOPILOT  0xf2b40040
-#define FSTV0910_P2_DIS_16PSK_9_10  0xf2b40030
+#define FSTV0910_P2_DUMMYPL_PILOT  0xf2b47080
+#define FSTV0910_P2_DUMMYPL_NOPILOT  0xf2b46040
+#define FSTV0910_P2_DIS_16PSK_9_10  0xf2b44030
 #define FSTV0910_P2_DIS_16PSK_8_9  0xf2b4000f
 
 /* P2_MODCODLST5 */
 #define RSTV0910_P2_MODCODLST5  0xf2b5
-#define FSTV0910_P2_DIS_16PSK_5_6  0xf2b500f0
+#define FSTV0910_P2_DIS_16PSK_5_6  0xf2b540f0
 #define FSTV0910_P2_DIS_16PSK_4_5  0xf2b5000f
 
 /* P2_MODCODLST6 */
 #define RSTV0910_P2_MODCODLST6  0xf2b6
-#define FSTV0910_P2_DIS_16PSK_3_4  0xf2b600f0
+#define FSTV0910_P2_DIS_16PSK_3_4  0xf2b640f0
 #define FSTV0910_P2_DIS_16PSK_2_3  0xf2b6000f
 
 /* P2_MODCODLST7 */
 #define RSTV0910_P2_MODCODLST7  0xf2b7
-#define FSTV0910_P2_MODCOD_NNOSFILTER  0xf2b70080
-#define FSTV0910_P2_DIS_8PSK_9_10  0xf2b70030
+#define FSTV0910_P2_MODCOD_NNOSFILTER  0xf2b77080
+#define FSTV0910_P2_DIS_8PSK_9_10  0xf2b74030
 #define FSTV0910_P2_DIS_8PSK_8_9  0xf2b7000f
 
 /* P2_MODCODLST8 */
 #define RSTV0910_P2_MODCODLST8  0xf2b8
-#define FSTV0910_P2_DIS_8PSK_5_6  0xf2b800f0
+#define FSTV0910_P2_DIS_8PSK_5_6  0xf2b840f0
 #define FSTV0910_P2_DIS_8PSK_3_4  0xf2b8000f
 
 /* P2_MODCODLST9 */
 #define RSTV0910_P2_MODCODLST9  0xf2b9
-#define FSTV0910_P2_DIS_8PSK_2_3  0xf2b900f0
+#define FSTV0910_P2_DIS_8PSK_2_3  0xf2b940f0
 #define FSTV0910_P2_DIS_8PSK_3_5  0xf2b9000f
 
 /* P2_MODCODLSTA */
 #define RSTV0910_P2_MODCODLSTA  0xf2ba
-#define FSTV0910_P2_NOSFILTER_LIMITE  0xf2ba0080
-#define FSTV0910_P2_DIS_QPSK_9_10  0xf2ba0030
+#define FSTV0910_P2_NOSFILTER_LIMITE  0xf2ba7080
+#define FSTV0910_P2_DIS_QPSK_9_10  0xf2ba4030
 #define FSTV0910_P2_DIS_QPSK_8_9  0xf2ba000f
 
 /* P2_MODCODLSTB */
 #define RSTV0910_P2_MODCODLSTB  0xf2bb
-#define FSTV0910_P2_DIS_QPSK_5_6  0xf2bb00f0
+#define FSTV0910_P2_DIS_QPSK_5_6  0xf2bb40f0
 #define FSTV0910_P2_DIS_QPSK_4_5  0xf2bb000f
 
 /* P2_MODCODLSTC */
 #define RSTV0910_P2_MODCODLSTC  0xf2bc
-#define FSTV0910_P2_DIS_QPSK_3_4  0xf2bc00f0
+#define FSTV0910_P2_DIS_QPSK_3_4  0xf2bc40f0
 #define FSTV0910_P2_DIS_QPSK_2_3  0xf2bc000f
 
 /* P2_MODCODLSTD */
 #define RSTV0910_P2_MODCODLSTD  0xf2bd
-#define FSTV0910_P2_DIS_QPSK_3_5  0xf2bd00f0
+#define FSTV0910_P2_DIS_QPSK_3_5  0xf2bd40f0
 #define FSTV0910_P2_DIS_QPSK_1_2  0xf2bd000f
 
 /* P2_MODCODLSTE */
 #define RSTV0910_P2_MODCODLSTE  0xf2be
-#define FSTV0910_P2_DIS_QPSK_2_5  0xf2be00f0
+#define FSTV0910_P2_DIS_QPSK_2_5  0xf2be40f0
 #define FSTV0910_P2_DIS_QPSK_1_3  0xf2be000f
 
 /* P2_MODCODLSTF */
 #define RSTV0910_P2_MODCODLSTF  0xf2bf
-#define FSTV0910_P2_DIS_QPSK_1_4  0xf2bf00f0
-#define FSTV0910_P2_DEMOD_INVMODLST  0xf2bf0008
-#define FSTV0910_P2_DEMODOUT_ENABLE  0xf2bf0004
-#define FSTV0910_P2_DDEMOD_NSET  0xf2bf0002
+#define FSTV0910_P2_DIS_QPSK_1_4  0xf2bf40f0
+#define FSTV0910_P2_DEMOD_INVMODLST  0xf2bf3008
+#define FSTV0910_P2_DEMODOUT_ENABLE  0xf2bf2004
+#define FSTV0910_P2_DDEMOD_NSET  0xf2bf1002
 #define FSTV0910_P2_MODCOD_NSTOCK  0xf2bf0001
 
 /* P2_GAUSSR0 */
 #define RSTV0910_P2_GAUSSR0  0xf2c0
-#define FSTV0910_P2_EN_CCIMODE  0xf2c00080
+#define FSTV0910_P2_EN_CCIMODE  0xf2c07080
 #define FSTV0910_P2_R0_GAUSSIEN  0xf2c0007f
 
 /* P2_CCIR0 */
 #define RSTV0910_P2_CCIR0  0xf2c1
-#define FSTV0910_P2_CCIDETECT_PLHONLY  0xf2c10080
+#define FSTV0910_P2_CCIDETECT_PLHONLY  0xf2c17080
 #define FSTV0910_P2_R0_CCI  0xf2c1007f
 
 /* P2_CCIQUANT */
 #define RSTV0910_P2_CCIQUANT  0xf2c2
-#define FSTV0910_P2_CCI_BETA  0xf2c200e0
+#define FSTV0910_P2_CCI_BETA  0xf2c250e0
 #define FSTV0910_P2_CCI_QUANT  0xf2c2001f
 
 /* P2_CCITHRES */
@@ -1304,24 +1304,24 @@
 
 /* P2_DSTATUS4 */
 #define RSTV0910_P2_DSTATUS4  0xf2c5
-#define FSTV0910_P2_RAINFADE_DETECT  0xf2c50080
-#define FSTV0910_P2_NOTHRES2_FAIL  0xf2c50040
-#define FSTV0910_P2_NOTHRES1_FAIL  0xf2c50020
-#define FSTV0910_P2_DMDPROG_ERROR  0xf2c50004
-#define FSTV0910_P2_CSTENV_DETECT  0xf2c50002
+#define FSTV0910_P2_RAINFADE_DETECT  0xf2c57080
+#define FSTV0910_P2_NOTHRES2_FAIL  0xf2c56040
+#define FSTV0910_P2_NOTHRES1_FAIL  0xf2c55020
+#define FSTV0910_P2_DMDPROG_ERROR  0xf2c52004
+#define FSTV0910_P2_CSTENV_DETECT  0xf2c51002
 #define FSTV0910_P2_DETECTION_TRIAX  0xf2c50001
 
 /* P2_DMDRESCFG */
 #define RSTV0910_P2_DMDRESCFG  0xf2c6
-#define FSTV0910_P2_DMDRES_RESET  0xf2c60080
-#define FSTV0910_P2_DMDRES_STRALL  0xf2c60008
-#define FSTV0910_P2_DMDRES_NEWONLY  0xf2c60004
-#define FSTV0910_P2_DMDRES_NOSTORE  0xf2c60002
+#define FSTV0910_P2_DMDRES_RESET  0xf2c67080
+#define FSTV0910_P2_DMDRES_STRALL  0xf2c63008
+#define FSTV0910_P2_DMDRES_NEWONLY  0xf2c62004
+#define FSTV0910_P2_DMDRES_NOSTORE  0xf2c61002
 
 /* P2_DMDRESADR */
 #define RSTV0910_P2_DMDRESADR  0xf2c7
-#define FSTV0910_P2_DMDRES_VALIDCFR  0xf2c70040
-#define FSTV0910_P2_DMDRES_MEMFULL  0xf2c70030
+#define FSTV0910_P2_DMDRES_VALIDCFR  0xf2c76040
+#define FSTV0910_P2_DMDRES_MEMFULL  0xf2c74030
 #define FSTV0910_P2_DMDRES_RESNBR  0xf2c7000f
 
 /* P2_DMDRESDATA7 */
@@ -1390,29 +1390,29 @@
 
 /* P2_FFECFG */
 #define RSTV0910_P2_FFECFG  0xf2d8
-#define FSTV0910_P2_EQUALFFE_ON  0xf2d80040
-#define FSTV0910_P2_EQUAL_USEDSYMB  0xf2d80030
+#define FSTV0910_P2_EQUALFFE_ON  0xf2d86040
+#define FSTV0910_P2_EQUAL_USEDSYMB  0xf2d84030
 #define FSTV0910_P2_MU_EQUALFFE  0xf2d80007
 
 /* P2_TNRCFG2 */
 #define RSTV0910_P2_TNRCFG2  0xf2e1
-#define FSTV0910_P2_TUN_IQSWAP  0xf2e10080
+#define FSTV0910_P2_TUN_IQSWAP  0xf2e17080
 
 /* P2_SMAPCOEF7 */
 #define RSTV0910_P2_SMAPCOEF7  0xf300
-#define FSTV0910_P2_DIS_QSCALE  0xf3000080
+#define FSTV0910_P2_DIS_QSCALE  0xf3007080
 #define FSTV0910_P2_SMAPCOEF_Q_LLR12  0xf300017f
 
 /* P2_SMAPCOEF6 */
 #define RSTV0910_P2_SMAPCOEF6  0xf301
-#define FSTV0910_P2_DIS_AGC2SCALE  0xf3010080
-#define FSTV0910_P2_ADJ_8PSKLLR1  0xf3010004
-#define FSTV0910_P2_OLD_8PSKLLR1  0xf3010002
+#define FSTV0910_P2_DIS_AGC2SCALE  0xf3017080
+#define FSTV0910_P2_ADJ_8PSKLLR1  0xf3012004
+#define FSTV0910_P2_OLD_8PSKLLR1  0xf3011002
 #define FSTV0910_P2_DIS_AB8PSK  0xf3010001
 
 /* P2_SMAPCOEF5 */
 #define RSTV0910_P2_SMAPCOEF5  0xf302
-#define FSTV0910_P2_DIS_8SCALE  0xf3020080
+#define FSTV0910_P2_DIS_8SCALE  0xf3027080
 #define FSTV0910_P2_SMAPCOEF_8P_LLR23  0xf302017f
 
 /* P2_SMAPCOEF4 */
@@ -1425,17 +1425,17 @@
 
 /* P2_SMAPCOEF2 */
 #define RSTV0910_P2_SMAPCOEF2  0xf305
-#define FSTV0910_P2_SMAPCOEF_32APSK_R2R3  0xf30501f0
+#define FSTV0910_P2_SMAPCOEF_32APSK_R2R3  0xf30541f0
 #define FSTV0910_P2_SMAPCOEF_32APSK_LLR2  0xf305010f
 
 /* P2_SMAPCOEF1 */
 #define RSTV0910_P2_SMAPCOEF1  0xf306
-#define FSTV0910_P2_DIS_16SCALE  0xf3060080
+#define FSTV0910_P2_DIS_16SCALE  0xf3067080
 #define FSTV0910_P2_SMAPCOEF_32_LLR34  0xf306017f
 
 /* P2_SMAPCOEF0 */
 #define RSTV0910_P2_SMAPCOEF0  0xf307
-#define FSTV0910_P2_DIS_32SCALE  0xf3070080
+#define FSTV0910_P2_DIS_32SCALE  0xf3077080
 #define FSTV0910_P2_SMAPCOEF_32_LLR15  0xf307017f
 
 /* P2_NOSTHRES1 */
@@ -1452,20 +1452,20 @@
 
 /* P2_RAINFADE */
 #define RSTV0910_P2_RAINFADE  0xf30c
-#define FSTV0910_P2_NOSTHRES_DATAT  0xf30c0080
-#define FSTV0910_P2_RAINFADE_CNLIMIT  0xf30c0070
+#define FSTV0910_P2_NOSTHRES_DATAT  0xf30c7080
+#define FSTV0910_P2_RAINFADE_CNLIMIT  0xf30c4070
 #define FSTV0910_P2_RAINFADE_TIMEOUT  0xf30c0007
 
 /* P2_NOSRAMCFG */
 #define RSTV0910_P2_NOSRAMCFG  0xf30d
-#define FSTV0910_P2_NOSRAM_ACTIVATION  0xf30d0030
-#define FSTV0910_P2_NOSRAM_CNRONLY  0xf30d0008
+#define FSTV0910_P2_NOSRAM_ACTIVATION  0xf30d4030
+#define FSTV0910_P2_NOSRAM_CNRONLY  0xf30d3008
 #define FSTV0910_P2_NOSRAM_LGNCNR1  0xf30d0007
 
 /* P2_NOSRAMPOS */
 #define RSTV0910_P2_NOSRAMPOS  0xf30e
-#define FSTV0910_P2_NOSRAM_LGNCNR0  0xf30e00f0
-#define FSTV0910_P2_NOSRAM_VALIDE  0xf30e0004
+#define FSTV0910_P2_NOSRAM_LGNCNR0  0xf30e40f0
+#define FSTV0910_P2_NOSRAM_VALIDE  0xf30e2004
 #define FSTV0910_P2_NOSRAM_CNRVAL1  0xf30e0003
 
 /* P2_NOSRAMVAL */
@@ -1494,16 +1494,16 @@
 
 /* P2_VITSCALE */
 #define RSTV0910_P2_VITSCALE  0xf332
-#define FSTV0910_P2_NVTH_NOSRANGE  0xf3320080
-#define FSTV0910_P2_VERROR_MAXMODE  0xf3320040
-#define FSTV0910_P2_NSLOWSN_LOCKED  0xf3320008
-#define FSTV0910_P2_DIS_RSFLOCK  0xf3320002
+#define FSTV0910_P2_NVTH_NOSRANGE  0xf3327080
+#define FSTV0910_P2_VERROR_MAXMODE  0xf3326040
+#define FSTV0910_P2_NSLOWSN_LOCKED  0xf3323008
+#define FSTV0910_P2_DIS_RSFLOCK  0xf3321002
 
 /* P2_FECM */
 #define RSTV0910_P2_FECM  0xf333
-#define FSTV0910_P2_DSS_DVB  0xf3330080
-#define FSTV0910_P2_DSS_SRCH  0xf3330010
-#define FSTV0910_P2_SYNCVIT  0xf3330002
+#define FSTV0910_P2_DSS_DVB  0xf3337080
+#define FSTV0910_P2_DSS_SRCH  0xf3334010
+#define FSTV0910_P2_SYNCVIT  0xf3331002
 #define FSTV0910_P2_IQINV  0xf3330001
 
 /* P2_VTH12 */
@@ -1540,26 +1540,26 @@
 
 /* P2_PRVIT */
 #define RSTV0910_P2_PRVIT  0xf33c
-#define FSTV0910_P2_DIS_VTHLOCK  0xf33c0040
-#define FSTV0910_P2_E7_8VIT  0xf33c0020
-#define FSTV0910_P2_E6_7VIT  0xf33c0010
-#define FSTV0910_P2_E5_6VIT  0xf33c0008
-#define FSTV0910_P2_E3_4VIT  0xf33c0004
-#define FSTV0910_P2_E2_3VIT  0xf33c0002
+#define FSTV0910_P2_DIS_VTHLOCK  0xf33c6040
+#define FSTV0910_P2_E7_8VIT  0xf33c5020
+#define FSTV0910_P2_E6_7VIT  0xf33c4010
+#define FSTV0910_P2_E5_6VIT  0xf33c3008
+#define FSTV0910_P2_E3_4VIT  0xf33c2004
+#define FSTV0910_P2_E2_3VIT  0xf33c1002
 #define FSTV0910_P2_E1_2VIT  0xf33c0001
 
 /* P2_VAVSRVIT */
 #define RSTV0910_P2_VAVSRVIT  0xf33d
-#define FSTV0910_P2_AMVIT  0xf33d0080
-#define FSTV0910_P2_FROZENVIT  0xf33d0040
-#define FSTV0910_P2_SNVIT  0xf33d0030
-#define FSTV0910_P2_TOVVIT  0xf33d000c
+#define FSTV0910_P2_AMVIT  0xf33d7080
+#define FSTV0910_P2_FROZENVIT  0xf33d6040
+#define FSTV0910_P2_SNVIT  0xf33d4030
+#define FSTV0910_P2_TOVVIT  0xf33d200c
 #define FSTV0910_P2_HYPVIT  0xf33d0003
 
 /* P2_VSTATUSVIT */
 #define RSTV0910_P2_VSTATUSVIT  0xf33e
-#define FSTV0910_P2_PRFVIT  0xf33e0010
-#define FSTV0910_P2_LOCKEDVIT  0xf33e0008
+#define FSTV0910_P2_PRFVIT  0xf33e4010
+#define FSTV0910_P2_LOCKEDVIT  0xf33e3008
 
 /* P2_VTHINUSE */
 #define RSTV0910_P2_VTHINUSE  0xf33f
@@ -1599,32 +1599,32 @@
 
 /* P2_PDELCTRL0 */
 #define RSTV0910_P2_PDELCTRL0  0xf34f
-#define FSTV0910_P2_ISIOBS_MODE  0xf34f0030
+#define FSTV0910_P2_ISIOBS_MODE  0xf34f4030
 
 /* P2_PDELCTRL1 */
 #define RSTV0910_P2_PDELCTRL1  0xf350
-#define FSTV0910_P2_INV_MISMASK  0xf3500080
-#define FSTV0910_P2_FILTER_EN  0xf3500020
-#define FSTV0910_P2_HYSTEN  0xf3500008
-#define FSTV0910_P2_HYSTSWRST  0xf3500004
-#define FSTV0910_P2_EN_MIS00  0xf3500002
+#define FSTV0910_P2_INV_MISMASK  0xf3507080
+#define FSTV0910_P2_FILTER_EN  0xf3505020
+#define FSTV0910_P2_HYSTEN  0xf3503008
+#define FSTV0910_P2_HYSTSWRST  0xf3502004
+#define FSTV0910_P2_EN_MIS00  0xf3501002
 #define FSTV0910_P2_ALGOSWRST  0xf3500001
 
 /* P2_PDELCTRL2 */
 #define RSTV0910_P2_PDELCTRL2  0xf351
-#define FSTV0910_P2_FORCE_CONTINUOUS  0xf3510080
-#define FSTV0910_P2_RESET_UPKO_COUNT  0xf3510040
-#define FSTV0910_P2_USER_PKTDELIN_NB  0xf3510020
-#define FSTV0910_P2_FRAME_MODE  0xf3510002
+#define FSTV0910_P2_FORCE_CONTINUOUS  0xf3517080
+#define FSTV0910_P2_RESET_UPKO_COUNT  0xf3516040
+#define FSTV0910_P2_USER_PKTDELIN_NB  0xf3515020
+#define FSTV0910_P2_FRAME_MODE  0xf3511002
 
 /* P2_HYSTTHRESH */
 #define RSTV0910_P2_HYSTTHRESH  0xf354
-#define FSTV0910_P2_DELIN_LOCKTHRES  0xf35400f0
+#define FSTV0910_P2_DELIN_LOCKTHRES  0xf35440f0
 #define FSTV0910_P2_DELIN_UNLOCKTHRES  0xf354000f
 
 /* P2_UPLCCST0 */
 #define RSTV0910_P2_UPLCCST0  0xf358
-#define FSTV0910_P2_UPL_CST0  0xf35800f8
+#define FSTV0910_P2_UPL_CST0  0xf35830f8
 #define FSTV0910_P2_UPL_MODE  0xf3580007
 
 /* P2_ISIENTRY */
@@ -1673,16 +1673,16 @@
 
 /* P2_PDELSTATUS1 */
 #define RSTV0910_P2_PDELSTATUS1  0xf369
-#define FSTV0910_P2_PKTDELIN_DELOCK  0xf3690080
-#define FSTV0910_P2_SYNCDUPDFL_BADDFL  0xf3690040
-#define FSTV0910_P2_UNACCEPTED_STREAM  0xf3690010
-#define FSTV0910_P2_BCH_ERROR_FLAG  0xf3690008
-#define FSTV0910_P2_PKTDELIN_LOCK  0xf3690002
+#define FSTV0910_P2_PKTDELIN_DELOCK  0xf3697080
+#define FSTV0910_P2_SYNCDUPDFL_BADDFL  0xf3696040
+#define FSTV0910_P2_UNACCEPTED_STREAM  0xf3694010
+#define FSTV0910_P2_BCH_ERROR_FLAG  0xf3693008
+#define FSTV0910_P2_PKTDELIN_LOCK  0xf3691002
 #define FSTV0910_P2_FIRST_LOCK  0xf3690001
 
 /* P2_PDELSTATUS2 */
 #define RSTV0910_P2_PDELSTATUS2  0xf36a
-#define FSTV0910_P2_FRAME_MODCOD  0xf36a007c
+#define FSTV0910_P2_FRAME_MODCOD  0xf36a207c
 #define FSTV0910_P2_FRAME_TYPE  0xf36a0003
 
 /* P2_BBFCRCKO1 */
@@ -1703,92 +1703,92 @@
 
 /* P2_PDELCTRL3 */
 #define RSTV0910_P2_PDELCTRL3  0xf36f
-#define FSTV0910_P2_NOFIFO_BCHERR  0xf36f0020
-#define FSTV0910_P2_PKTDELIN_DELACMERR  0xf36f0010
+#define FSTV0910_P2_NOFIFO_BCHERR  0xf36f5020
+#define FSTV0910_P2_PKTDELIN_DELACMERR  0xf36f4010
 
 /* P2_TSSTATEM */
 #define RSTV0910_P2_TSSTATEM  0xf370
-#define FSTV0910_P2_TSDIL_ON  0xf3700080
-#define FSTV0910_P2_TSRS_ON  0xf3700020
-#define FSTV0910_P2_TSDESCRAMB_ON  0xf3700010
-#define FSTV0910_P2_TSFRAME_MODE  0xf3700008
-#define FSTV0910_P2_TS_DISABLE  0xf3700004
-#define FSTV0910_P2_TSACM_MODE  0xf3700002
+#define FSTV0910_P2_TSDIL_ON  0xf3707080
+#define FSTV0910_P2_TSRS_ON  0xf3705020
+#define FSTV0910_P2_TSDESCRAMB_ON  0xf3704010
+#define FSTV0910_P2_TSFRAME_MODE  0xf3703008
+#define FSTV0910_P2_TS_DISABLE  0xf3702004
+#define FSTV0910_P2_TSACM_MODE  0xf3701002
 #define FSTV0910_P2_TSOUT_NOSYNC  0xf3700001
 
 /* P2_TSSTATEL */
 #define RSTV0910_P2_TSSTATEL  0xf371
-#define FSTV0910_P2_TSNOSYNCBYTE  0xf3710080
-#define FSTV0910_P2_TSPARITY_ON  0xf3710040
-#define FSTV0910_P2_TSISSYI_ON  0xf3710008
-#define FSTV0910_P2_TSNPD_ON  0xf3710004
-#define FSTV0910_P2_TSCRC8_ON  0xf3710002
+#define FSTV0910_P2_TSNOSYNCBYTE  0xf3717080
+#define FSTV0910_P2_TSPARITY_ON  0xf3716040
+#define FSTV0910_P2_TSISSYI_ON  0xf3713008
+#define FSTV0910_P2_TSNPD_ON  0xf3712004
+#define FSTV0910_P2_TSCRC8_ON  0xf3711002
 #define FSTV0910_P2_TSDSS_PACKET  0xf3710001
 
 /* P2_TSCFGH */
 #define RSTV0910_P2_TSCFGH  0xf372
-#define FSTV0910_P2_TSFIFO_DVBCI  0xf3720080
-#define FSTV0910_P2_TSFIFO_SERIAL  0xf3720040
-#define FSTV0910_P2_TSFIFO_TEIUPDATE  0xf3720020
-#define FSTV0910_P2_TSFIFO_DUTY50  0xf3720010
-#define FSTV0910_P2_TSFIFO_HSGNLOUT  0xf3720008
-#define FSTV0910_P2_TSFIFO_ERRMODE  0xf3720006
+#define FSTV0910_P2_TSFIFO_DVBCI  0xf3727080
+#define FSTV0910_P2_TSFIFO_SERIAL  0xf3726040
+#define FSTV0910_P2_TSFIFO_TEIUPDATE  0xf3725020
+#define FSTV0910_P2_TSFIFO_DUTY50  0xf3724010
+#define FSTV0910_P2_TSFIFO_HSGNLOUT  0xf3723008
+#define FSTV0910_P2_TSFIFO_ERRMODE  0xf3721006
 #define FSTV0910_P2_RST_HWARE  0xf3720001
 
 /* P2_TSCFGM */
 #define RSTV0910_P2_TSCFGM  0xf373
-#define FSTV0910_P2_TSFIFO_MANSPEED  0xf37300c0
-#define FSTV0910_P2_TSFIFO_PERMDATA  0xf3730020
-#define FSTV0910_P2_TSFIFO_NONEWSGNL  0xf3730010
+#define FSTV0910_P2_TSFIFO_MANSPEED  0xf37360c0
+#define FSTV0910_P2_TSFIFO_PERMDATA  0xf3735020
+#define FSTV0910_P2_TSFIFO_NONEWSGNL  0xf3734010
 #define FSTV0910_P2_TSFIFO_INVDATA  0xf3730001
 
 /* P2_TSCFGL */
 #define RSTV0910_P2_TSCFGL  0xf374
-#define FSTV0910_P2_TSFIFO_BCLKDEL1CK  0xf37400c0
-#define FSTV0910_P2_BCHERROR_MODE  0xf3740030
-#define FSTV0910_P2_TSFIFO_NSGNL2DATA  0xf3740008
-#define FSTV0910_P2_TSFIFO_EMBINDVB  0xf3740004
+#define FSTV0910_P2_TSFIFO_BCLKDEL1CK  0xf37460c0
+#define FSTV0910_P2_BCHERROR_MODE  0xf3744030
+#define FSTV0910_P2_TSFIFO_NSGNL2DATA  0xf3743008
+#define FSTV0910_P2_TSFIFO_EMBINDVB  0xf3742004
 #define FSTV0910_P2_TSFIFO_BITSPEED  0xf3740003
 
 /* P2_TSSYNC */
 #define RSTV0910_P2_TSSYNC  0xf375
-#define FSTV0910_P2_TSFIFO_SYNCMODE  0xf3750018
+#define FSTV0910_P2_TSFIFO_SYNCMODE  0xf3753018
 
 /* P2_TSINSDELH */
 #define RSTV0910_P2_TSINSDELH  0xf376
-#define FSTV0910_P2_TSDEL_SYNCBYTE  0xf3760080
-#define FSTV0910_P2_TSDEL_XXHEADER  0xf3760040
-#define FSTV0910_P2_TSDEL_DATAFIELD  0xf3760010
-#define FSTV0910_P2_TSINSDEL_RSPARITY  0xf3760002
+#define FSTV0910_P2_TSDEL_SYNCBYTE  0xf3767080
+#define FSTV0910_P2_TSDEL_XXHEADER  0xf3766040
+#define FSTV0910_P2_TSDEL_DATAFIELD  0xf3764010
+#define FSTV0910_P2_TSINSDEL_RSPARITY  0xf3761002
 #define FSTV0910_P2_TSINSDEL_CRC8  0xf3760001
 
 /* P2_TSINSDELM */
 #define RSTV0910_P2_TSINSDELM  0xf377
-#define FSTV0910_P2_TSINS_EMODCOD  0xf3770010
-#define FSTV0910_P2_TSINS_TOKEN  0xf3770008
-#define FSTV0910_P2_TSINS_XXXERR  0xf3770004
-#define FSTV0910_P2_TSINS_MATYPE  0xf3770002
+#define FSTV0910_P2_TSINS_EMODCOD  0xf3774010
+#define FSTV0910_P2_TSINS_TOKEN  0xf3773008
+#define FSTV0910_P2_TSINS_XXXERR  0xf3772004
+#define FSTV0910_P2_TSINS_MATYPE  0xf3771002
 #define FSTV0910_P2_TSINS_UPL  0xf3770001
 
 /* P2_TSINSDELL */
 #define RSTV0910_P2_TSINSDELL  0xf378
-#define FSTV0910_P2_TSINS_DFL  0xf3780080
-#define FSTV0910_P2_TSINS_SYNCD  0xf3780040
-#define FSTV0910_P2_TSINS_BLOCLEN  0xf3780020
-#define FSTV0910_P2_TSINS_SIGPCOUNT  0xf3780010
-#define FSTV0910_P2_TSINS_FIFO  0xf3780008
-#define FSTV0910_P2_TSINS_REALPACK  0xf3780004
-#define FSTV0910_P2_TSINS_TSCONFIG  0xf3780002
+#define FSTV0910_P2_TSINS_DFL  0xf3787080
+#define FSTV0910_P2_TSINS_SYNCD  0xf3786040
+#define FSTV0910_P2_TSINS_BLOCLEN  0xf3785020
+#define FSTV0910_P2_TSINS_SIGPCOUNT  0xf3784010
+#define FSTV0910_P2_TSINS_FIFO  0xf3783008
+#define FSTV0910_P2_TSINS_REALPACK  0xf3782004
+#define FSTV0910_P2_TSINS_TSCONFIG  0xf3781002
 #define FSTV0910_P2_TSINS_LATENCY  0xf3780001
 
 /* P2_TSDIVN */
 #define RSTV0910_P2_TSDIVN  0xf379
-#define FSTV0910_P2_TSFIFO_SPEEDMODE  0xf37900c0
+#define FSTV0910_P2_TSFIFO_SPEEDMODE  0xf37960c0
 #define FSTV0910_P2_TSFIFO_RISEOK  0xf3790007
 
 /* P2_TSCFG4 */
 #define RSTV0910_P2_TSCFG4  0xf37a
-#define FSTV0910_P2_TSFIFO_TSSPEEDMODE  0xf37a00c0
+#define FSTV0910_P2_TSFIFO_TSSPEEDMODE  0xf37a60c0
 
 /* P2_TSSPEED */
 #define RSTV0910_P2_TSSPEED  0xf380
@@ -1796,18 +1796,18 @@
 
 /* P2_TSSTATUS */
 #define RSTV0910_P2_TSSTATUS  0xf381
-#define FSTV0910_P2_TSFIFO_LINEOK  0xf3810080
-#define FSTV0910_P2_TSFIFO_ERROR  0xf3810040
-#define FSTV0910_P2_TSFIFO_NOSYNC  0xf3810010
-#define FSTV0910_P2_TSREGUL_ERROR  0xf3810004
+#define FSTV0910_P2_TSFIFO_LINEOK  0xf3817080
+#define FSTV0910_P2_TSFIFO_ERROR  0xf3816040
+#define FSTV0910_P2_TSFIFO_NOSYNC  0xf3814010
+#define FSTV0910_P2_TSREGUL_ERROR  0xf3812004
 #define FSTV0910_P2_DIL_READY  0xf3810001
 
 /* P2_TSSTATUS2 */
 #define RSTV0910_P2_TSSTATUS2  0xf382
-#define FSTV0910_P2_TSFIFO_DEMODSEL  0xf3820080
-#define FSTV0910_P2_TSFIFOSPEED_STORE  0xf3820040
-#define FSTV0910_P2_DILXX_RESET  0xf3820020
-#define FSTV0910_P2_SCRAMBDETECT  0xf3820002
+#define FSTV0910_P2_TSFIFO_DEMODSEL  0xf3827080
+#define FSTV0910_P2_TSFIFOSPEED_STORE  0xf3826040
+#define FSTV0910_P2_DILXX_RESET  0xf3825020
+#define FSTV0910_P2_SCRAMBDETECT  0xf3821002
 
 /* P2_TSBITRATE1 */
 #define RSTV0910_P2_TSBITRATE1  0xf383
@@ -1819,7 +1819,7 @@
 
 /* P2_TSPACKLEN1 */
 #define RSTV0910_P2_TSPACKLEN1  0xf385
-#define FSTV0910_P2_TSFIFO_PACKCPT  0xf38500e0
+#define FSTV0910_P2_TSFIFO_PACKCPT  0xf38550e0
 
 /* P2_TSDLY2 */
 #define RSTV0910_P2_TSDLY2  0xf389
@@ -1839,8 +1839,8 @@
 
 /* P2_TSBUFSTAT2 */
 #define RSTV0910_P2_TSBUFSTAT2  0xf38d
-#define FSTV0910_P2_TSISCR_3BYTES  0xf38d0080
-#define FSTV0910_P2_TSISCR_NEWDATA  0xf38d0040
+#define FSTV0910_P2_TSISCR_3BYTES  0xf38d7080
+#define FSTV0910_P2_TSISCR_NEWDATA  0xf38d6040
 #define FSTV0910_P2_TSISCR_BUFSTAT2  0xf38d003f
 
 /* P2_TSBUFSTAT1 */
@@ -1853,13 +1853,13 @@
 
 /* P2_TSDEBUGL */
 #define RSTV0910_P2_TSDEBUGL  0xf391
-#define FSTV0910_P2_TSFIFO_ERROR_EVNT  0xf3910004
+#define FSTV0910_P2_TSFIFO_ERROR_EVNT  0xf3912004
 #define FSTV0910_P2_TSFIFO_OVERFLOWM  0xf3910001
 
 /* P2_TSDLYSET2 */
 #define RSTV0910_P2_TSDLYSET2  0xf392
-#define FSTV0910_P2_SOFFIFO_OFFSET  0xf39200c0
-#define FSTV0910_P2_HYSTERESIS_THRESHOLD  0xf3920030
+#define FSTV0910_P2_SOFFIFO_OFFSET  0xf39260c0
+#define FSTV0910_P2_HYSTERESIS_THRESHOLD  0xf3924030
 #define FSTV0910_P2_SOFFIFO_SYMBOFFS2  0xf392000f
 
 /* P2_TSDLYSET1 */
@@ -1872,12 +1872,12 @@
 
 /* P2_ERRCTRL1 */
 #define RSTV0910_P2_ERRCTRL1  0xf398
-#define FSTV0910_P2_ERR_SOURCE1  0xf39800f0
+#define FSTV0910_P2_ERR_SOURCE1  0xf39840f0
 #define FSTV0910_P2_NUM_EVENT1  0xf3980007
 
 /* P2_ERRCNT12 */
 #define RSTV0910_P2_ERRCNT12  0xf399
-#define FSTV0910_P2_ERRCNT1_OLDVALUE  0xf3990080
+#define FSTV0910_P2_ERRCNT1_OLDVALUE  0xf3997080
 #define FSTV0910_P2_ERR_CNT12  0xf399007f
 
 /* P2_ERRCNT11 */
@@ -1890,12 +1890,12 @@
 
 /* P2_ERRCTRL2 */
 #define RSTV0910_P2_ERRCTRL2  0xf39c
-#define FSTV0910_P2_ERR_SOURCE2  0xf39c00f0
+#define FSTV0910_P2_ERR_SOURCE2  0xf39c40f0
 #define FSTV0910_P2_NUM_EVENT2  0xf39c0007
 
 /* P2_ERRCNT22 */
 #define RSTV0910_P2_ERRCNT22  0xf39d
-#define FSTV0910_P2_ERRCNT2_OLDVALUE  0xf39d0080
+#define FSTV0910_P2_ERRCNT2_OLDVALUE  0xf39d7080
 #define FSTV0910_P2_ERR_CNT22  0xf39d007f
 
 /* P2_ERRCNT21 */
@@ -1908,39 +1908,39 @@
 
 /* P2_FECSPY */
 #define RSTV0910_P2_FECSPY  0xf3a0
-#define FSTV0910_P2_SPY_ENABLE  0xf3a00080
-#define FSTV0910_P2_NO_SYNCBYTE  0xf3a00040
-#define FSTV0910_P2_SERIAL_MODE  0xf3a00020
-#define FSTV0910_P2_UNUSUAL_PACKET  0xf3a00010
-#define FSTV0910_P2_BERMETER_DATAMODE  0xf3a0000c
-#define FSTV0910_P2_BERMETER_LMODE  0xf3a00002
+#define FSTV0910_P2_SPY_ENABLE  0xf3a07080
+#define FSTV0910_P2_NO_SYNCBYTE  0xf3a06040
+#define FSTV0910_P2_SERIAL_MODE  0xf3a05020
+#define FSTV0910_P2_UNUSUAL_PACKET  0xf3a04010
+#define FSTV0910_P2_BERMETER_DATAMODE  0xf3a0200c
+#define FSTV0910_P2_BERMETER_LMODE  0xf3a01002
 #define FSTV0910_P2_BERMETER_RESET  0xf3a00001
 
 /* P2_FSPYCFG */
 #define RSTV0910_P2_FSPYCFG  0xf3a1
-#define FSTV0910_P2_FECSPY_INPUT  0xf3a100c0
-#define FSTV0910_P2_RST_ON_ERROR  0xf3a10020
-#define FSTV0910_P2_ONE_SHOT  0xf3a10010
-#define FSTV0910_P2_I2C_MODE  0xf3a1000c
+#define FSTV0910_P2_FECSPY_INPUT  0xf3a160c0
+#define FSTV0910_P2_RST_ON_ERROR  0xf3a15020
+#define FSTV0910_P2_ONE_SHOT  0xf3a14010
+#define FSTV0910_P2_I2C_MODE  0xf3a1200c
 #define FSTV0910_P2_SPY_HYSTERESIS  0xf3a10003
 
 /* P2_FSPYDATA */
 #define RSTV0910_P2_FSPYDATA  0xf3a2
-#define FSTV0910_P2_SPY_STUFFING  0xf3a20080
-#define FSTV0910_P2_SPY_CNULLPKT  0xf3a20020
+#define FSTV0910_P2_SPY_STUFFING  0xf3a27080
+#define FSTV0910_P2_SPY_CNULLPKT  0xf3a25020
 #define FSTV0910_P2_SPY_OUTDATA_MODE  0xf3a2001f
 
 /* P2_FSPYOUT */
 #define RSTV0910_P2_FSPYOUT  0xf3a3
-#define FSTV0910_P2_FSPY_DIRECT  0xf3a30080
+#define FSTV0910_P2_FSPY_DIRECT  0xf3a37080
 #define FSTV0910_P2_STUFF_MODE  0xf3a30007
 
 /* P2_FSTATUS */
 #define RSTV0910_P2_FSTATUS  0xf3a4
-#define FSTV0910_P2_SPY_ENDSIM  0xf3a40080
-#define FSTV0910_P2_VALID_SIM  0xf3a40040
-#define FSTV0910_P2_FOUND_SIGNAL  0xf3a40020
-#define FSTV0910_P2_DSS_SYNCBYTE  0xf3a40010
+#define FSTV0910_P2_SPY_ENDSIM  0xf3a47080
+#define FSTV0910_P2_VALID_SIM  0xf3a46040
+#define FSTV0910_P2_FOUND_SIGNAL  0xf3a45020
+#define FSTV0910_P2_DSS_SYNCBYTE  0xf3a44010
 #define FSTV0910_P2_RESULT_STATE  0xf3a4000f
 
 /* P2_FBERCPT4 */
@@ -1977,8 +1977,8 @@
 
 /* P2_FSPYBER */
 #define RSTV0910_P2_FSPYBER  0xf3b2
-#define FSTV0910_P2_FSPYBER_SYNCBYTE  0xf3b20010
-#define FSTV0910_P2_FSPYBER_UNSYNC  0xf3b20008
+#define FSTV0910_P2_FSPYBER_SYNCBYTE  0xf3b24010
+#define FSTV0910_P2_FSPYBER_UNSYNC  0xf3b23008
 #define FSTV0910_P2_FSPYBER_CTIME  0xf3b20007
 
 /* P2_SFERROR */
@@ -1987,60 +1987,60 @@
 
 /* P2_SFECSTATUS */
 #define RSTV0910_P2_SFECSTATUS  0xf3c3
-#define FSTV0910_P2_SFEC_ON  0xf3c30080
-#define FSTV0910_P2_SFEC_OFF  0xf3c30040
-#define FSTV0910_P2_LOCKEDSFEC  0xf3c30008
-#define FSTV0910_P2_SFEC_DELOCK  0xf3c30004
-#define FSTV0910_P2_SFEC_DEMODSEL  0xf3c30002
+#define FSTV0910_P2_SFEC_ON  0xf3c37080
+#define FSTV0910_P2_SFEC_OFF  0xf3c36040
+#define FSTV0910_P2_LOCKEDSFEC  0xf3c33008
+#define FSTV0910_P2_SFEC_DELOCK  0xf3c32004
+#define FSTV0910_P2_SFEC_DEMODSEL  0xf3c31002
 #define FSTV0910_P2_SFEC_OVFON  0xf3c30001
 
 /* P2_SFKDIV12 */
 #define RSTV0910_P2_SFKDIV12  0xf3c4
-#define FSTV0910_P2_SFECKDIV12_MAN  0xf3c40080
+#define FSTV0910_P2_SFECKDIV12_MAN  0xf3c47080
 
 /* P2_SFKDIV23 */
 #define RSTV0910_P2_SFKDIV23  0xf3c5
-#define FSTV0910_P2_SFECKDIV23_MAN  0xf3c50080
+#define FSTV0910_P2_SFECKDIV23_MAN  0xf3c57080
 
 /* P2_SFKDIV34 */
 #define RSTV0910_P2_SFKDIV34  0xf3c6
-#define FSTV0910_P2_SFECKDIV34_MAN  0xf3c60080
+#define FSTV0910_P2_SFECKDIV34_MAN  0xf3c67080
 
 /* P2_SFKDIV56 */
 #define RSTV0910_P2_SFKDIV56  0xf3c7
-#define FSTV0910_P2_SFECKDIV56_MAN  0xf3c70080
+#define FSTV0910_P2_SFECKDIV56_MAN  0xf3c77080
 
 /* P2_SFKDIV67 */
 #define RSTV0910_P2_SFKDIV67  0xf3c8
-#define FSTV0910_P2_SFECKDIV67_MAN  0xf3c80080
+#define FSTV0910_P2_SFECKDIV67_MAN  0xf3c87080
 
 /* P2_SFKDIV78 */
 #define RSTV0910_P2_SFKDIV78  0xf3c9
-#define FSTV0910_P2_SFECKDIV78_MAN  0xf3c90080
+#define FSTV0910_P2_SFECKDIV78_MAN  0xf3c97080
 
 /* P2_SFSTATUS */
 #define RSTV0910_P2_SFSTATUS  0xf3cc
-#define FSTV0910_P2_SFEC_LINEOK  0xf3cc0080
-#define FSTV0910_P2_SFEC_ERROR  0xf3cc0040
-#define FSTV0910_P2_SFEC_DATA7  0xf3cc0020
-#define FSTV0910_P2_SFEC_PKTDNBRFAIL  0xf3cc0010
-#define FSTV0910_P2_TSSFEC_DEMODSEL  0xf3cc0008
-#define FSTV0910_P2_SFEC_NOSYNC  0xf3cc0004
-#define FSTV0910_P2_SFEC_UNREGULA  0xf3cc0002
+#define FSTV0910_P2_SFEC_LINEOK  0xf3cc7080
+#define FSTV0910_P2_SFEC_ERROR  0xf3cc6040
+#define FSTV0910_P2_SFEC_DATA7  0xf3cc5020
+#define FSTV0910_P2_SFEC_PKTDNBRFAIL  0xf3cc4010
+#define FSTV0910_P2_TSSFEC_DEMODSEL  0xf3cc3008
+#define FSTV0910_P2_SFEC_NOSYNC  0xf3cc2004
+#define FSTV0910_P2_SFEC_UNREGULA  0xf3cc1002
 #define FSTV0910_P2_SFEC_READY  0xf3cc0001
 
 /* P2_SFDLYSET2 */
 #define RSTV0910_P2_SFDLYSET2  0xf3d0
-#define FSTV0910_P2_SFEC_DISABLE  0xf3d00002
+#define FSTV0910_P2_SFEC_DISABLE  0xf3d01002
 
 /* P2_SFERRCTRL */
 #define RSTV0910_P2_SFERRCTRL  0xf3d8
-#define FSTV0910_P2_SFEC_ERR_SOURCE  0xf3d800f0
+#define FSTV0910_P2_SFEC_ERR_SOURCE  0xf3d840f0
 #define FSTV0910_P2_SFEC_NUM_EVENT  0xf3d80007
 
 /* P2_SFERRCNT2 */
 #define RSTV0910_P2_SFERRCNT2  0xf3d9
-#define FSTV0910_P2_SFERRC_OLDVALUE  0xf3d90080
+#define FSTV0910_P2_SFERRC_OLDVALUE  0xf3d97080
 #define FSTV0910_P2_SFEC_ERR_CNT2  0xf3d9007f
 
 /* P2_SFERRCNT1 */
@@ -2053,13 +2053,13 @@
 
 /* P1_IQCONST */
 #define RSTV0910_P1_IQCONST  0xf400
-#define FSTV0910_P1_CONSTEL_SELECT  0xf4000060
+#define FSTV0910_P1_CONSTEL_SELECT  0xf4005060
 #define FSTV0910_P1_IQSYMB_SEL  0xf400001f
 
 /* P1_NOSCFG */
 #define RSTV0910_P1_NOSCFG  0xf401
-#define FSTV0910_P1_DUMMYPL_NOSDATA  0xf4010020
-#define FSTV0910_P1_NOSPLH_BETA  0xf4010018
+#define FSTV0910_P1_DUMMYPL_NOSDATA  0xf4015020
+#define FSTV0910_P1_NOSPLH_BETA  0xf4013018
 #define FSTV0910_P1_NOSDATA_BETA  0xf4010007
 
 /* P1_ISYMB */
@@ -2072,18 +2072,18 @@
 
 /* P1_AGC1CFG */
 #define RSTV0910_P1_AGC1CFG  0xf404
-#define FSTV0910_P1_DC_FROZEN  0xf4040080
-#define FSTV0910_P1_DC_CORRECT  0xf4040040
-#define FSTV0910_P1_AMM_FROZEN  0xf4040020
-#define FSTV0910_P1_AMM_CORRECT  0xf4040010
-#define FSTV0910_P1_QUAD_FROZEN  0xf4040008
-#define FSTV0910_P1_QUAD_CORRECT  0xf4040004
+#define FSTV0910_P1_DC_FROZEN  0xf4047080
+#define FSTV0910_P1_DC_CORRECT  0xf4046040
+#define FSTV0910_P1_AMM_FROZEN  0xf4045020
+#define FSTV0910_P1_AMM_CORRECT  0xf4044010
+#define FSTV0910_P1_QUAD_FROZEN  0xf4043008
+#define FSTV0910_P1_QUAD_CORRECT  0xf4042004
 
 /* P1_AGC1CN */
 #define RSTV0910_P1_AGC1CN  0xf406
-#define FSTV0910_P1_AGC1_LOCKED  0xf4060080
-#define FSTV0910_P1_AGC1_MINPOWER  0xf4060010
-#define FSTV0910_P1_AGCOUT_FAST  0xf4060008
+#define FSTV0910_P1_AGC1_LOCKED  0xf4067080
+#define FSTV0910_P1_AGC1_MINPOWER  0xf4064010
+#define FSTV0910_P1_AGCOUT_FAST  0xf4063008
 #define FSTV0910_P1_AGCIQ_BETA  0xf4060007
 
 /* P1_AGC1REF */
@@ -2124,50 +2124,50 @@
 
 /* P1_DEMOD */
 #define RSTV0910_P1_DEMOD  0xf410
-#define FSTV0910_P1_MANUALS2_ROLLOFF  0xf4100080
-#define FSTV0910_P1_SPECINV_CONTROL  0xf4100030
-#define FSTV0910_P1_MANUALSX_ROLLOFF  0xf4100004
+#define FSTV0910_P1_MANUALS2_ROLLOFF  0xf4107080
+#define FSTV0910_P1_SPECINV_CONTROL  0xf4104030
+#define FSTV0910_P1_MANUALSX_ROLLOFF  0xf4102004
 #define FSTV0910_P1_ROLLOFF_CONTROL  0xf4100003
 
 /* P1_DMDMODCOD */
 #define RSTV0910_P1_DMDMODCOD  0xf411
-#define FSTV0910_P1_MANUAL_MODCOD  0xf4110080
-#define FSTV0910_P1_DEMOD_MODCOD  0xf411007c
+#define FSTV0910_P1_MANUAL_MODCOD  0xf4117080
+#define FSTV0910_P1_DEMOD_MODCOD  0xf411207c
 #define FSTV0910_P1_DEMOD_TYPE  0xf4110003
 
 /* P1_DSTATUS */
 #define RSTV0910_P1_DSTATUS  0xf412
-#define FSTV0910_P1_CAR_LOCK  0xf4120080
-#define FSTV0910_P1_TMGLOCK_QUALITY  0xf4120060
-#define FSTV0910_P1_LOCK_DEFINITIF  0xf4120008
+#define FSTV0910_P1_CAR_LOCK  0xf4127080
+#define FSTV0910_P1_TMGLOCK_QUALITY  0xf4125060
+#define FSTV0910_P1_LOCK_DEFINITIF  0xf4123008
 #define FSTV0910_P1_OVADC_DETECT  0xf4120001
 
 /* P1_DSTATUS2 */
 #define RSTV0910_P1_DSTATUS2  0xf413
-#define FSTV0910_P1_DEMOD_DELOCK  0xf4130080
-#define FSTV0910_P1_MODCODRQ_SYNCTAG  0xf4130020
-#define FSTV0910_P1_POLYPH_SATEVENT  0xf4130010
-#define FSTV0910_P1_AGC1_NOSIGNALACK  0xf4130008
-#define FSTV0910_P1_AGC2_OVERFLOW  0xf4130004
-#define FSTV0910_P1_CFR_OVERFLOW  0xf4130002
+#define FSTV0910_P1_DEMOD_DELOCK  0xf4137080
+#define FSTV0910_P1_MODCODRQ_SYNCTAG  0xf4135020
+#define FSTV0910_P1_POLYPH_SATEVENT  0xf4134010
+#define FSTV0910_P1_AGC1_NOSIGNALACK  0xf4133008
+#define FSTV0910_P1_AGC2_OVERFLOW  0xf4132004
+#define FSTV0910_P1_CFR_OVERFLOW  0xf4131002
 #define FSTV0910_P1_GAMMA_OVERUNDER  0xf4130001
 
 /* P1_DMDCFGMD */
 #define RSTV0910_P1_DMDCFGMD  0xf414
-#define FSTV0910_P1_DVBS2_ENABLE  0xf4140080
-#define FSTV0910_P1_DVBS1_ENABLE  0xf4140040
-#define FSTV0910_P1_SCAN_ENABLE  0xf4140010
-#define FSTV0910_P1_CFR_AUTOSCAN  0xf4140008
+#define FSTV0910_P1_DVBS2_ENABLE  0xf4147080
+#define FSTV0910_P1_DVBS1_ENABLE  0xf4146040
+#define FSTV0910_P1_SCAN_ENABLE  0xf4144010
+#define FSTV0910_P1_CFR_AUTOSCAN  0xf4143008
 #define FSTV0910_P1_TUN_RNG  0xf4140003
 
 /* P1_DMDCFG2 */
 #define RSTV0910_P1_DMDCFG2  0xf415
-#define FSTV0910_P1_S1S2_SEQUENTIAL  0xf4150040
-#define FSTV0910_P1_INFINITE_RELOCK  0xf4150010
+#define FSTV0910_P1_S1S2_SEQUENTIAL  0xf4156040
+#define FSTV0910_P1_INFINITE_RELOCK  0xf4154010
 
 /* P1_DMDISTATE */
 #define RSTV0910_P1_DMDISTATE  0xf416
-#define FSTV0910_P1_I2C_NORESETDMODE  0xf4160080
+#define FSTV0910_P1_I2C_NORESETDMODE  0xf4167080
 #define FSTV0910_P1_I2C_DEMOD_MODE  0xf416001f
 
 /* P1_DMDT0M */
@@ -2176,27 +2176,27 @@
 
 /* P1_DMDSTATE */
 #define RSTV0910_P1_DMDSTATE  0xf41b
-#define FSTV0910_P1_HEADER_MODE  0xf41b0060
+#define FSTV0910_P1_HEADER_MODE  0xf41b5060
 
 /* P1_DMDFLYW */
 #define RSTV0910_P1_DMDFLYW  0xf41c
-#define FSTV0910_P1_I2C_IRQVAL  0xf41c00f0
+#define FSTV0910_P1_I2C_IRQVAL  0xf41c40f0
 #define FSTV0910_P1_FLYWHEEL_CPT  0xf41c000f
 
 /* P1_DSTATUS3 */
 #define RSTV0910_P1_DSTATUS3  0xf41d
-#define FSTV0910_P1_CFR_ZIGZAG  0xf41d0080
-#define FSTV0910_P1_DEMOD_CFGMODE  0xf41d0060
-#define FSTV0910_P1_GAMMA_LOWBAUDRATE  0xf41d0010
+#define FSTV0910_P1_CFR_ZIGZAG  0xf41d7080
+#define FSTV0910_P1_DEMOD_CFGMODE  0xf41d5060
+#define FSTV0910_P1_GAMMA_LOWBAUDRATE  0xf41d4010
 
 /* P1_DMDCFG3 */
 #define RSTV0910_P1_DMDCFG3  0xf41e
-#define FSTV0910_P1_NOSTOP_FIFOFULL  0xf41e0008
+#define FSTV0910_P1_NOSTOP_FIFOFULL  0xf41e3008
 
 /* P1_DMDCFG4 */
 #define RSTV0910_P1_DMDCFG4  0xf41f
-#define FSTV0910_P1_DIS_VITLOCK  0xf41f0080
-#define FSTV0910_P1_DIS_CLKENABLE  0xf41f0004
+#define FSTV0910_P1_DIS_VITLOCK  0xf41f7080
+#define FSTV0910_P1_DIS_CLKENABLE  0xf41f2004
 
 /* P1_CORRELMANT */
 #define RSTV0910_P1_CORRELMANT  0xf420
@@ -2208,13 +2208,13 @@
 
 /* P1_CORRELEXP */
 #define RSTV0910_P1_CORRELEXP  0xf422
-#define FSTV0910_P1_CORREL_ABSEXP  0xf42200f0
+#define FSTV0910_P1_CORREL_ABSEXP  0xf42240f0
 #define FSTV0910_P1_CORREL_EXP  0xf422000f
 
 /* P1_PLHMODCOD */
 #define RSTV0910_P1_PLHMODCOD  0xf424
-#define FSTV0910_P1_SPECINV_DEMOD  0xf4240080
-#define FSTV0910_P1_PLH_MODCOD  0xf424007c
+#define FSTV0910_P1_SPECINV_DEMOD  0xf4247080
+#define FSTV0910_P1_PLH_MODCOD  0xf424207c
 #define FSTV0910_P1_PLH_TYPE  0xf4240003
 
 /* P1_DMDREG */
@@ -2223,19 +2223,19 @@
 
 /* P1_AGCNADJ */
 #define RSTV0910_P1_AGCNADJ  0xf426
-#define FSTV0910_P1_RADJOFF_AGC2  0xf4260080
-#define FSTV0910_P1_RADJOFF_AGC1  0xf4260040
+#define FSTV0910_P1_RADJOFF_AGC2  0xf4267080
+#define FSTV0910_P1_RADJOFF_AGC1  0xf4266040
 #define FSTV0910_P1_AGC_NADJ  0xf426013f
 
 /* P1_AGCKS */
 #define RSTV0910_P1_AGCKS  0xf427
-#define FSTV0910_P1_RSADJ_MANUALCFG  0xf4270080
-#define FSTV0910_P1_RSADJ_CCMMODE  0xf4270040
+#define FSTV0910_P1_RSADJ_MANUALCFG  0xf4277080
+#define FSTV0910_P1_RSADJ_CCMMODE  0xf4276040
 #define FSTV0910_P1_RADJ_SPSK  0xf427013f
 
 /* P1_AGCKQ */
 #define RSTV0910_P1_AGCKQ  0xf428
-#define FSTV0910_P1_RADJON_DVBS1  0xf4280040
+#define FSTV0910_P1_RADJON_DVBS1  0xf4286040
 #define FSTV0910_P1_RADJ_QPSK  0xf428013f
 
 /* P1_AGCK8 */
@@ -2244,20 +2244,20 @@
 
 /* P1_AGCK16 */
 #define RSTV0910_P1_AGCK16  0xf42a
-#define FSTV0910_P1_R2ADJOFF_16APSK  0xf42a0040
-#define FSTV0910_P1_R1ADJOFF_16APSK  0xf42a0020
+#define FSTV0910_P1_R2ADJOFF_16APSK  0xf42a6040
+#define FSTV0910_P1_R1ADJOFF_16APSK  0xf42a5020
 #define FSTV0910_P1_RADJ_16APSK  0xf42a011f
 
 /* P1_AGCK32 */
 #define RSTV0910_P1_AGCK32  0xf42b
-#define FSTV0910_P1_R3ADJOFF_32APSK  0xf42b0080
-#define FSTV0910_P1_R2ADJOFF_32APSK  0xf42b0040
-#define FSTV0910_P1_R1ADJOFF_32APSK  0xf42b0020
+#define FSTV0910_P1_R3ADJOFF_32APSK  0xf42b7080
+#define FSTV0910_P1_R2ADJOFF_32APSK  0xf42b6040
+#define FSTV0910_P1_R1ADJOFF_32APSK  0xf42b5020
 #define FSTV0910_P1_RADJ_32APSK  0xf42b011f
 
 /* P1_AGC2O */
 #define RSTV0910_P1_AGC2O  0xf42c
-#define FSTV0910_P1_CSTENV_MODE  0xf42c00c0
+#define FSTV0910_P1_CSTENV_MODE  0xf42c60c0
 #define FSTV0910_P1_AGC2_COEF  0xf42c0007
 
 /* P1_AGC2REF */
@@ -2306,32 +2306,32 @@
 
 /* P1_CARCFG */
 #define RSTV0910_P1_CARCFG  0xf438
-#define FSTV0910_P1_ROTAON  0xf4380004
+#define FSTV0910_P1_ROTAON  0xf4382004
 #define FSTV0910_P1_PH_DET_ALGO  0xf4380003
 
 /* P1_ACLC */
 #define RSTV0910_P1_ACLC  0xf439
-#define FSTV0910_P1_CAR_ALPHA_MANT  0xf4390030
+#define FSTV0910_P1_CAR_ALPHA_MANT  0xf4394030
 #define FSTV0910_P1_CAR_ALPHA_EXP  0xf439000f
 
 /* P1_BCLC */
 #define RSTV0910_P1_BCLC  0xf43a
-#define FSTV0910_P1_CAR_BETA_MANT  0xf43a0030
+#define FSTV0910_P1_CAR_BETA_MANT  0xf43a4030
 #define FSTV0910_P1_CAR_BETA_EXP  0xf43a000f
 
 /* P1_ACLCS2 */
 #define RSTV0910_P1_ACLCS2  0xf43b
-#define FSTV0910_P1_CARS2_APLHA_MANTISSE  0xf43b0030
+#define FSTV0910_P1_CARS2_APLHA_MANTISSE  0xf43b4030
 #define FSTV0910_P1_CARS2_ALPHA_EXP  0xf43b000f
 
 /* P1_BCLCS2 */
 #define RSTV0910_P1_BCLCS2  0xf43c
-#define FSTV0910_P1_CARS2_BETA_MANTISSE  0xf43c0030
+#define FSTV0910_P1_CARS2_BETA_MANTISSE  0xf43c4030
 #define FSTV0910_P1_CARS2_BETA_EXP  0xf43c000f
 
 /* P1_CARFREQ */
 #define RSTV0910_P1_CARFREQ  0xf43d
-#define FSTV0910_P1_KC_COARSE_EXP  0xf43d00f0
+#define FSTV0910_P1_KC_COARSE_EXP  0xf43d40f0
 #define FSTV0910_P1_BETA_FREQ  0xf43d000f
 
 /* P1_CARHDR */
@@ -2384,7 +2384,7 @@
 
 /* P1_CFRINC1 */
 #define RSTV0910_P1_CFRINC1  0xf44a
-#define FSTV0910_P1_MANUAL_CFRINC  0xf44a0080
+#define FSTV0910_P1_MANUAL_CFRINC  0xf44a7080
 #define FSTV0910_P1_CFR_INC1  0xf44a003f
 
 /* P1_CFRINC0 */
@@ -2409,18 +2409,18 @@
 
 /* P1_TMGCFG */
 #define RSTV0910_P1_TMGCFG  0xf450
-#define FSTV0910_P1_TMGLOCK_BETA  0xf45000c0
-#define FSTV0910_P1_DO_TIMING_CORR  0xf4500010
+#define FSTV0910_P1_TMGLOCK_BETA  0xf45060c0
+#define FSTV0910_P1_DO_TIMING_CORR  0xf4504010
 #define FSTV0910_P1_TMG_MINFREQ  0xf4500003
 
 /* P1_RTC */
 #define RSTV0910_P1_RTC  0xf451
-#define FSTV0910_P1_TMGALPHA_EXP  0xf45100f0
+#define FSTV0910_P1_TMGALPHA_EXP  0xf45140f0
 #define FSTV0910_P1_TMGBETA_EXP  0xf451000f
 
 /* P1_RTCS2 */
 #define RSTV0910_P1_RTCS2  0xf452
-#define FSTV0910_P1_TMGALPHAS2_EXP  0xf45200f0
+#define FSTV0910_P1_TMGALPHAS2_EXP  0xf45240f0
 #define FSTV0910_P1_TMGBETAS2_EXP  0xf452000f
 
 /* P1_TMGTHRISE */
@@ -2441,7 +2441,7 @@
 
 /* P1_KTTMG */
 #define RSTV0910_P1_KTTMG  0xf457
-#define FSTV0910_P1_KT_TMG_EXP  0xf45700f0
+#define FSTV0910_P1_KT_TMG_EXP  0xf45740f0
 
 /* P1_KREFTMG */
 #define RSTV0910_P1_KREFTMG  0xf458
@@ -2449,12 +2449,12 @@
 
 /* P1_SFRSTEP */
 #define RSTV0910_P1_SFRSTEP  0xf459
-#define FSTV0910_P1_SFR_SCANSTEP  0xf45900f0
+#define FSTV0910_P1_SFR_SCANSTEP  0xf45940f0
 #define FSTV0910_P1_SFR_CENTERSTEP  0xf459000f
 
 /* P1_TMGCFG2 */
 #define RSTV0910_P1_TMGCFG2  0xf45a
-#define FSTV0910_P1_DIS_AUTOSAMP  0xf45a0008
+#define FSTV0910_P1_DIS_AUTOSAMP  0xf45a3008
 #define FSTV0910_P1_SFRRATIO_FINE  0xf45a0001
 
 /* P1_KREFTMG2 */
@@ -2463,9 +2463,9 @@
 
 /* P1_TMGCFG3 */
 #define RSTV0910_P1_TMGCFG3  0xf45d
-#define FSTV0910_P1_CONT_TMGCENTER  0xf45d0008
-#define FSTV0910_P1_AUTO_GUP  0xf45d0004
-#define FSTV0910_P1_AUTO_GLOW  0xf45d0002
+#define FSTV0910_P1_CONT_TMGCENTER  0xf45d3008
+#define FSTV0910_P1_AUTO_GUP  0xf45d2004
+#define FSTV0910_P1_AUTO_GLOW  0xf45d1002
 
 /* P1_SFRINIT1 */
 #define RSTV0910_P1_SFRINIT1  0xf45e
@@ -2529,11 +2529,11 @@
 
 /* P1_TMGOBS */
 #define RSTV0910_P1_TMGOBS  0xf46d
-#define FSTV0910_P1_ROLLOFF_STATUS  0xf46d00c0
+#define FSTV0910_P1_ROLLOFF_STATUS  0xf46d60c0
 
 /* P1_EQUALCFG */
 #define RSTV0910_P1_EQUALCFG  0xf46f
-#define FSTV0910_P1_EQUAL_ON  0xf46f0040
+#define FSTV0910_P1_EQUAL_ON  0xf46f6040
 #define FSTV0910_P1_MU_EQUALDFE  0xf46f0007
 
 /* P1_EQUAI1 */
@@ -2658,33 +2658,33 @@
 
 /* P1_NOSCFGF1 */
 #define RSTV0910_P1_NOSCFGF1  0xf48e
-#define FSTV0910_P1_LOWNOISE_MESURE  0xf48e0080
-#define FSTV0910_P1_NOS_DELFRAME  0xf48e0040
-#define FSTV0910_P1_NOSDATA_MODE  0xf48e0030
-#define FSTV0910_P1_FRAMESEL_TYPESEL  0xf48e000c
+#define FSTV0910_P1_LOWNOISE_MESURE  0xf48e7080
+#define FSTV0910_P1_NOS_DELFRAME  0xf48e6040
+#define FSTV0910_P1_NOSDATA_MODE  0xf48e4030
+#define FSTV0910_P1_FRAMESEL_TYPESEL  0xf48e200c
 #define FSTV0910_P1_FRAMESEL_TYPE  0xf48e0003
 
 /* P1_NOSCFGF2 */
 #define RSTV0910_P1_NOSCFGF2  0xf48f
-#define FSTV0910_P1_DIS_NOSPILOTS  0xf48f0080
-#define FSTV0910_P1_FRAMESEL_MODCODSEL  0xf48f0060
+#define FSTV0910_P1_DIS_NOSPILOTS  0xf48f7080
+#define FSTV0910_P1_FRAMESEL_MODCODSEL  0xf48f5060
 #define FSTV0910_P1_FRAMESEL_MODCOD  0xf48f001f
 
 /* P1_CAR2CFG */
 #define RSTV0910_P1_CAR2CFG  0xf490
-#define FSTV0910_P1_ROTA2ON  0xf4900004
+#define FSTV0910_P1_ROTA2ON  0xf4902004
 #define FSTV0910_P1_PH_DET_ALGO2  0xf4900003
 
 /* P1_CFR2CFR1 */
 #define RSTV0910_P1_CFR2CFR1  0xf491
-#define FSTV0910_P1_EN_S2CAR2CENTER  0xf4910020
+#define FSTV0910_P1_EN_S2CAR2CENTER  0xf4915020
 #define FSTV0910_P1_CFR2TOCFR1_BETA  0xf4910007
 
 /* P1_CAR3CFG */
 #define RSTV0910_P1_CAR3CFG  0xf492
-#define FSTV0910_P1_CARRIER23_MODE  0xf49200c0
-#define FSTV0910_P1_CAR3INTERM_DVBS1  0xf4920020
-#define FSTV0910_P1_ABAMPLIF_MODE  0xf4920018
+#define FSTV0910_P1_CARRIER23_MODE  0xf49260c0
+#define FSTV0910_P1_CAR3INTERM_DVBS1  0xf4925020
+#define FSTV0910_P1_ABAMPLIF_MODE  0xf4923018
 #define FSTV0910_P1_CARRIER3_ALPHA3DL  0xf4920007
 
 /* P1_CFR22 */
@@ -2701,50 +2701,50 @@
 
 /* P1_ACLC2S2Q */
 #define RSTV0910_P1_ACLC2S2Q  0xf497
-#define FSTV0910_P1_ENAB_SPSKSYMB  0xf4970080
-#define FSTV0910_P1_CAR2S2_Q_ALPH_M  0xf4970030
+#define FSTV0910_P1_ENAB_SPSKSYMB  0xf4977080
+#define FSTV0910_P1_CAR2S2_Q_ALPH_M  0xf4974030
 #define FSTV0910_P1_CAR2S2_Q_ALPH_E  0xf497000f
 
 /* P1_ACLC2S28 */
 #define RSTV0910_P1_ACLC2S28  0xf498
-#define FSTV0910_P1_CAR2S2_8_ALPH_M  0xf4980030
+#define FSTV0910_P1_CAR2S2_8_ALPH_M  0xf4984030
 #define FSTV0910_P1_CAR2S2_8_ALPH_E  0xf498000f
 
 /* P1_ACLC2S216A */
 #define RSTV0910_P1_ACLC2S216A  0xf499
-#define FSTV0910_P1_CAR2S2_16A_ALPH_M  0xf4990030
+#define FSTV0910_P1_CAR2S2_16A_ALPH_M  0xf4994030
 #define FSTV0910_P1_CAR2S2_16A_ALPH_E  0xf499000f
 
 /* P1_ACLC2S232A */
 #define RSTV0910_P1_ACLC2S232A  0xf49a
-#define FSTV0910_P1_CAR2S2_32A_ALPH_M  0xf49a0030
+#define FSTV0910_P1_CAR2S2_32A_ALPH_M  0xf49a4030
 #define FSTV0910_P1_CAR2S2_32A_ALPH_E  0xf49a000f
 
 /* P1_BCLC2S2Q */
 #define RSTV0910_P1_BCLC2S2Q  0xf49c
-#define FSTV0910_P1_CAR2S2_Q_BETA_M  0xf49c0030
+#define FSTV0910_P1_CAR2S2_Q_BETA_M  0xf49c4030
 #define FSTV0910_P1_CAR2S2_Q_BETA_E  0xf49c000f
 
 /* P1_BCLC2S28 */
 #define RSTV0910_P1_BCLC2S28  0xf49d
-#define FSTV0910_P1_CAR2S2_8_BETA_M  0xf49d0030
+#define FSTV0910_P1_CAR2S2_8_BETA_M  0xf49d4030
 #define FSTV0910_P1_CAR2S2_8_BETA_E  0xf49d000f
 
 /* P1_BCLC2S216A */
 #define RSTV0910_P1_BCLC2S216A  0xf49e
-#define FSTV0910_P1_DVBS2S216A_NIP  0xf49e0080
-#define FSTV0910_P1_CAR2S2_16A_BETA_M  0xf49e0030
+#define FSTV0910_P1_DVBS2S216A_NIP  0xf49e7080
+#define FSTV0910_P1_CAR2S2_16A_BETA_M  0xf49e4030
 #define FSTV0910_P1_CAR2S2_16A_BETA_E  0xf49e000f
 
 /* P1_BCLC2S232A */
 #define RSTV0910_P1_BCLC2S232A  0xf49f
-#define FSTV0910_P1_DVBS2S232A_NIP  0xf49f0080
-#define FSTV0910_P1_CAR2S2_32A_BETA_M  0xf49f0030
+#define FSTV0910_P1_DVBS2S232A_NIP  0xf49f7080
+#define FSTV0910_P1_CAR2S2_32A_BETA_M  0xf49f4030
 #define FSTV0910_P1_CAR2S2_32A_BETA_E  0xf49f000f
 
 /* P1_PLROOT2 */
 #define RSTV0910_P1_PLROOT2  0xf4ac
-#define FSTV0910_P1_PLSCRAMB_MODE  0xf4ac000c
+#define FSTV0910_P1_PLSCRAMB_MODE  0xf4ac200c
 #define FSTV0910_P1_PLSCRAMB_ROOT2  0xf4ac0003
 
 /* P1_PLROOT1 */
@@ -2761,100 +2761,100 @@
 
 /* P1_MODCODLST1 */
 #define RSTV0910_P1_MODCODLST1  0xf4b1
-#define FSTV0910_P1_SYMBRATE_FILTER  0xf4b10008
-#define FSTV0910_P1_NRESET_MODCODLST  0xf4b10004
+#define FSTV0910_P1_SYMBRATE_FILTER  0xf4b13008
+#define FSTV0910_P1_NRESET_MODCODLST  0xf4b12004
 #define FSTV0910_P1_DIS_32PSK_9_10  0xf4b10003
 
 /* P1_MODCODLST2 */
 #define RSTV0910_P1_MODCODLST2  0xf4b2
-#define FSTV0910_P1_DIS_32PSK_8_9  0xf4b200f0
+#define FSTV0910_P1_DIS_32PSK_8_9  0xf4b240f0
 #define FSTV0910_P1_DIS_32PSK_5_6  0xf4b2000f
 
 /* P1_MODCODLST3 */
 #define RSTV0910_P1_MODCODLST3  0xf4b3
-#define FSTV0910_P1_DIS_32PSK_4_5  0xf4b300f0
+#define FSTV0910_P1_DIS_32PSK_4_5  0xf4b340f0
 #define FSTV0910_P1_DIS_32PSK_3_4  0xf4b3000f
 
 /* P1_MODCODLST4 */
 #define RSTV0910_P1_MODCODLST4  0xf4b4
-#define FSTV0910_P1_DUMMYPL_PILOT  0xf4b40080
-#define FSTV0910_P1_DUMMYPL_NOPILOT  0xf4b40040
-#define FSTV0910_P1_DIS_16PSK_9_10  0xf4b40030
+#define FSTV0910_P1_DUMMYPL_PILOT  0xf4b47080
+#define FSTV0910_P1_DUMMYPL_NOPILOT  0xf4b46040
+#define FSTV0910_P1_DIS_16PSK_9_10  0xf4b44030
 #define FSTV0910_P1_DIS_16PSK_8_9  0xf4b4000f
 
 /* P1_MODCODLST5 */
 #define RSTV0910_P1_MODCODLST5  0xf4b5
-#define FSTV0910_P1_DIS_16PSK_5_6  0xf4b500f0
+#define FSTV0910_P1_DIS_16PSK_5_6  0xf4b540f0
 #define FSTV0910_P1_DIS_16PSK_4_5  0xf4b5000f
 
 /* P1_MODCODLST6 */
 #define RSTV0910_P1_MODCODLST6  0xf4b6
-#define FSTV0910_P1_DIS_16PSK_3_4  0xf4b600f0
+#define FSTV0910_P1_DIS_16PSK_3_4  0xf4b640f0
 #define FSTV0910_P1_DIS_16PSK_2_3  0xf4b6000f
 
 /* P1_MODCODLST7 */
 #define RSTV0910_P1_MODCODLST7  0xf4b7
-#define FSTV0910_P1_MODCOD_NNOSFILTER  0xf4b70080
-#define FSTV0910_P1_DIS_8PSK_9_10  0xf4b70030
+#define FSTV0910_P1_MODCOD_NNOSFILTER  0xf4b77080
+#define FSTV0910_P1_DIS_8PSK_9_10  0xf4b74030
 #define FSTV0910_P1_DIS_8PSK_8_9  0xf4b7000f
 
 /* P1_MODCODLST8 */
 #define RSTV0910_P1_MODCODLST8  0xf4b8
-#define FSTV0910_P1_DIS_8PSK_5_6  0xf4b800f0
+#define FSTV0910_P1_DIS_8PSK_5_6  0xf4b840f0
 #define FSTV0910_P1_DIS_8PSK_3_4  0xf4b8000f
 
 /* P1_MODCODLST9 */
 #define RSTV0910_P1_MODCODLST9  0xf4b9
-#define FSTV0910_P1_DIS_8PSK_2_3  0xf4b900f0
+#define FSTV0910_P1_DIS_8PSK_2_3  0xf4b940f0
 #define FSTV0910_P1_DIS_8PSK_3_5  0xf4b9000f
 
 /* P1_MODCODLSTA */
 #define RSTV0910_P1_MODCODLSTA  0xf4ba
-#define FSTV0910_P1_NOSFILTER_LIMITE  0xf4ba0080
-#define FSTV0910_P1_DIS_QPSK_9_10  0xf4ba0030
+#define FSTV0910_P1_NOSFILTER_LIMITE  0xf4ba7080
+#define FSTV0910_P1_DIS_QPSK_9_10  0xf4ba4030
 #define FSTV0910_P1_DIS_QPSK_8_9  0xf4ba000f
 
 /* P1_MODCODLSTB */
 #define RSTV0910_P1_MODCODLSTB  0xf4bb
-#define FSTV0910_P1_DIS_QPSK_5_6  0xf4bb00f0
+#define FSTV0910_P1_DIS_QPSK_5_6  0xf4bb40f0
 #define FSTV0910_P1_DIS_QPSK_4_5  0xf4bb000f
 
 /* P1_MODCODLSTC */
 #define RSTV0910_P1_MODCODLSTC  0xf4bc
-#define FSTV0910_P1_DIS_QPSK_3_4  0xf4bc00f0
+#define FSTV0910_P1_DIS_QPSK_3_4  0xf4bc40f0
 #define FSTV0910_P1_DIS_QPSK_2_3  0xf4bc000f
 
 /* P1_MODCODLSTD */
 #define RSTV0910_P1_MODCODLSTD  0xf4bd
-#define FSTV0910_P1_DIS_QPSK_3_5  0xf4bd00f0
+#define FSTV0910_P1_DIS_QPSK_3_5  0xf4bd40f0
 #define FSTV0910_P1_DIS_QPSK_1_2  0xf4bd000f
 
 /* P1_MODCODLSTE */
 #define RSTV0910_P1_MODCODLSTE  0xf4be
-#define FSTV0910_P1_DIS_QPSK_2_5  0xf4be00f0
+#define FSTV0910_P1_DIS_QPSK_2_5  0xf4be40f0
 #define FSTV0910_P1_DIS_QPSK_1_3  0xf4be000f
 
 /* P1_MODCODLSTF */
 #define RSTV0910_P1_MODCODLSTF  0xf4bf
-#define FSTV0910_P1_DIS_QPSK_1_4  0xf4bf00f0
-#define FSTV0910_P1_DEMOD_INVMODLST  0xf4bf0008
-#define FSTV0910_P1_DEMODOUT_ENABLE  0xf4bf0004
-#define FSTV0910_P1_DDEMOD_NSET  0xf4bf0002
+#define FSTV0910_P1_DIS_QPSK_1_4  0xf4bf40f0
+#define FSTV0910_P1_DEMOD_INVMODLST  0xf4bf3008
+#define FSTV0910_P1_DEMODOUT_ENABLE  0xf4bf2004
+#define FSTV0910_P1_DDEMOD_NSET  0xf4bf1002
 #define FSTV0910_P1_MODCOD_NSTOCK  0xf4bf0001
 
 /* P1_GAUSSR0 */
 #define RSTV0910_P1_GAUSSR0  0xf4c0
-#define FSTV0910_P1_EN_CCIMODE  0xf4c00080
+#define FSTV0910_P1_EN_CCIMODE  0xf4c07080
 #define FSTV0910_P1_R0_GAUSSIEN  0xf4c0007f
 
 /* P1_CCIR0 */
 #define RSTV0910_P1_CCIR0  0xf4c1
-#define FSTV0910_P1_CCIDETECT_PLHONLY  0xf4c10080
+#define FSTV0910_P1_CCIDETECT_PLHONLY  0xf4c17080
 #define FSTV0910_P1_R0_CCI  0xf4c1007f
 
 /* P1_CCIQUANT */
 #define RSTV0910_P1_CCIQUANT  0xf4c2
-#define FSTV0910_P1_CCI_BETA  0xf4c200e0
+#define FSTV0910_P1_CCI_BETA  0xf4c250e0
 #define FSTV0910_P1_CCI_QUANT  0xf4c2001f
 
 /* P1_CCITHRES */
@@ -2867,24 +2867,24 @@
 
 /* P1_DSTATUS4 */
 #define RSTV0910_P1_DSTATUS4  0xf4c5
-#define FSTV0910_P1_RAINFADE_DETECT  0xf4c50080
-#define FSTV0910_P1_NOTHRES2_FAIL  0xf4c50040
-#define FSTV0910_P1_NOTHRES1_FAIL  0xf4c50020
-#define FSTV0910_P1_DMDPROG_ERROR  0xf4c50004
-#define FSTV0910_P1_CSTENV_DETECT  0xf4c50002
+#define FSTV0910_P1_RAINFADE_DETECT  0xf4c57080
+#define FSTV0910_P1_NOTHRES2_FAIL  0xf4c56040
+#define FSTV0910_P1_NOTHRES1_FAIL  0xf4c55020
+#define FSTV0910_P1_DMDPROG_ERROR  0xf4c52004
+#define FSTV0910_P1_CSTENV_DETECT  0xf4c51002
 #define FSTV0910_P1_DETECTION_TRIAX  0xf4c50001
 
 /* P1_DMDRESCFG */
 #define RSTV0910_P1_DMDRESCFG  0xf4c6
-#define FSTV0910_P1_DMDRES_RESET  0xf4c60080
-#define FSTV0910_P1_DMDRES_STRALL  0xf4c60008
-#define FSTV0910_P1_DMDRES_NEWONLY  0xf4c60004
-#define FSTV0910_P1_DMDRES_NOSTORE  0xf4c60002
+#define FSTV0910_P1_DMDRES_RESET  0xf4c67080
+#define FSTV0910_P1_DMDRES_STRALL  0xf4c63008
+#define FSTV0910_P1_DMDRES_NEWONLY  0xf4c62004
+#define FSTV0910_P1_DMDRES_NOSTORE  0xf4c61002
 
 /* P1_DMDRESADR */
 #define RSTV0910_P1_DMDRESADR  0xf4c7
-#define FSTV0910_P1_DMDRES_VALIDCFR  0xf4c70040
-#define FSTV0910_P1_DMDRES_MEMFULL  0xf4c70030
+#define FSTV0910_P1_DMDRES_VALIDCFR  0xf4c76040
+#define FSTV0910_P1_DMDRES_MEMFULL  0xf4c74030
 #define FSTV0910_P1_DMDRES_RESNBR  0xf4c7000f
 
 /* P1_DMDRESDATA7 */
@@ -2953,29 +2953,29 @@
 
 /* P1_FFECFG */
 #define RSTV0910_P1_FFECFG  0xf4d8
-#define FSTV0910_P1_EQUALFFE_ON  0xf4d80040
-#define FSTV0910_P1_EQUAL_USEDSYMB  0xf4d80030
+#define FSTV0910_P1_EQUALFFE_ON  0xf4d86040
+#define FSTV0910_P1_EQUAL_USEDSYMB  0xf4d84030
 #define FSTV0910_P1_MU_EQUALFFE  0xf4d80007
 
 /* P1_TNRCFG2 */
 #define RSTV0910_P1_TNRCFG2  0xf4e1
-#define FSTV0910_P1_TUN_IQSWAP  0xf4e10080
+#define FSTV0910_P1_TUN_IQSWAP  0xf4e17080
 
 /* P1_SMAPCOEF7 */
 #define RSTV0910_P1_SMAPCOEF7  0xf500
-#define FSTV0910_P1_DIS_QSCALE  0xf5000080
+#define FSTV0910_P1_DIS_QSCALE  0xf5007080
 #define FSTV0910_P1_SMAPCOEF_Q_LLR12  0xf500017f
 
 /* P1_SMAPCOEF6 */
 #define RSTV0910_P1_SMAPCOEF6  0xf501
-#define FSTV0910_P1_DIS_AGC2SCALE  0xf5010080
-#define FSTV0910_P1_ADJ_8PSKLLR1  0xf5010004
-#define FSTV0910_P1_OLD_8PSKLLR1  0xf5010002
+#define FSTV0910_P1_DIS_AGC2SCALE  0xf5017080
+#define FSTV0910_P1_ADJ_8PSKLLR1  0xf5012004
+#define FSTV0910_P1_OLD_8PSKLLR1  0xf5011002
 #define FSTV0910_P1_DIS_AB8PSK  0xf5010001
 
 /* P1_SMAPCOEF5 */
 #define RSTV0910_P1_SMAPCOEF5  0xf502
-#define FSTV0910_P1_DIS_8SCALE  0xf5020080
+#define FSTV0910_P1_DIS_8SCALE  0xf5027080
 #define FSTV0910_P1_SMAPCOEF_8P_LLR23  0xf502017f
 
 /* P1_SMAPCOEF4 */
@@ -2988,17 +2988,17 @@
 
 /* P1_SMAPCOEF2 */
 #define RSTV0910_P1_SMAPCOEF2  0xf505
-#define FSTV0910_P1_SMAPCOEF_32APSK_R2R3  0xf50501f0
+#define FSTV0910_P1_SMAPCOEF_32APSK_R2R3  0xf50541f0
 #define FSTV0910_P1_SMAPCOEF_32APSK_LLR2  0xf505010f
 
 /* P1_SMAPCOEF1 */
 #define RSTV0910_P1_SMAPCOEF1  0xf506
-#define FSTV0910_P1_DIS_16SCALE  0xf5060080
+#define FSTV0910_P1_DIS_16SCALE  0xf5067080
 #define FSTV0910_P1_SMAPCOEF_32_LLR34  0xf506017f
 
 /* P1_SMAPCOEF0 */
 #define RSTV0910_P1_SMAPCOEF0  0xf507
-#define FSTV0910_P1_DIS_32SCALE  0xf5070080
+#define FSTV0910_P1_DIS_32SCALE  0xf5077080
 #define FSTV0910_P1_SMAPCOEF_32_LLR15  0xf507017f
 
 /* P1_NOSTHRES1 */
@@ -3015,20 +3015,20 @@
 
 /* P1_RAINFADE */
 #define RSTV0910_P1_RAINFADE  0xf50c
-#define FSTV0910_P1_NOSTHRES_DATAT  0xf50c0080
-#define FSTV0910_P1_RAINFADE_CNLIMIT  0xf50c0070
+#define FSTV0910_P1_NOSTHRES_DATAT  0xf50c7080
+#define FSTV0910_P1_RAINFADE_CNLIMIT  0xf50c4070
 #define FSTV0910_P1_RAINFADE_TIMEOUT  0xf50c0007
 
 /* P1_NOSRAMCFG */
 #define RSTV0910_P1_NOSRAMCFG  0xf50d
-#define FSTV0910_P1_NOSRAM_ACTIVATION  0xf50d0030
-#define FSTV0910_P1_NOSRAM_CNRONLY  0xf50d0008
+#define FSTV0910_P1_NOSRAM_ACTIVATION  0xf50d4030
+#define FSTV0910_P1_NOSRAM_CNRONLY  0xf50d3008
 #define FSTV0910_P1_NOSRAM_LGNCNR1  0xf50d0007
 
 /* P1_NOSRAMPOS */
 #define RSTV0910_P1_NOSRAMPOS  0xf50e
-#define FSTV0910_P1_NOSRAM_LGNCNR0  0xf50e00f0
-#define FSTV0910_P1_NOSRAM_VALIDE  0xf50e0004
+#define FSTV0910_P1_NOSRAM_LGNCNR0  0xf50e40f0
+#define FSTV0910_P1_NOSRAM_VALIDE  0xf50e2004
 #define FSTV0910_P1_NOSRAM_CNRVAL1  0xf50e0003
 
 /* P1_NOSRAMVAL */
@@ -3057,16 +3057,16 @@
 
 /* P1_VITSCALE */
 #define RSTV0910_P1_VITSCALE  0xf532
-#define FSTV0910_P1_NVTH_NOSRANGE  0xf5320080
-#define FSTV0910_P1_VERROR_MAXMODE  0xf5320040
-#define FSTV0910_P1_NSLOWSN_LOCKED  0xf5320008
-#define FSTV0910_P1_DIS_RSFLOCK  0xf5320002
+#define FSTV0910_P1_NVTH_NOSRANGE  0xf5327080
+#define FSTV0910_P1_VERROR_MAXMODE  0xf5326040
+#define FSTV0910_P1_NSLOWSN_LOCKED  0xf5323008
+#define FSTV0910_P1_DIS_RSFLOCK  0xf5321002
 
 /* P1_FECM */
 #define RSTV0910_P1_FECM  0xf533
-#define FSTV0910_P1_DSS_DVB  0xf5330080
-#define FSTV0910_P1_DSS_SRCH  0xf5330010
-#define FSTV0910_P1_SYNCVIT  0xf5330002
+#define FSTV0910_P1_DSS_DVB  0xf5337080
+#define FSTV0910_P1_DSS_SRCH  0xf5334010
+#define FSTV0910_P1_SYNCVIT  0xf5331002
 #define FSTV0910_P1_IQINV  0xf5330001
 
 /* P1_VTH12 */
@@ -3103,26 +3103,26 @@
 
 /* P1_PRVIT */
 #define RSTV0910_P1_PRVIT  0xf53c
-#define FSTV0910_P1_DIS_VTHLOCK  0xf53c0040
-#define FSTV0910_P1_E7_8VIT  0xf53c0020
-#define FSTV0910_P1_E6_7VIT  0xf53c0010
-#define FSTV0910_P1_E5_6VIT  0xf53c0008
-#define FSTV0910_P1_E3_4VIT  0xf53c0004
-#define FSTV0910_P1_E2_3VIT  0xf53c0002
+#define FSTV0910_P1_DIS_VTHLOCK  0xf53c6040
+#define FSTV0910_P1_E7_8VIT  0xf53c5020
+#define FSTV0910_P1_E6_7VIT  0xf53c4010
+#define FSTV0910_P1_E5_6VIT  0xf53c3008
+#define FSTV0910_P1_E3_4VIT  0xf53c2004
+#define FSTV0910_P1_E2_3VIT  0xf53c1002
 #define FSTV0910_P1_E1_2VIT  0xf53c0001
 
 /* P1_VAVSRVIT */
 #define RSTV0910_P1_VAVSRVIT  0xf53d
-#define FSTV0910_P1_AMVIT  0xf53d0080
-#define FSTV0910_P1_FROZENVIT  0xf53d0040
-#define FSTV0910_P1_SNVIT  0xf53d0030
-#define FSTV0910_P1_TOVVIT  0xf53d000c
+#define FSTV0910_P1_AMVIT  0xf53d7080
+#define FSTV0910_P1_FROZENVIT  0xf53d6040
+#define FSTV0910_P1_SNVIT  0xf53d4030
+#define FSTV0910_P1_TOVVIT  0xf53d200c
 #define FSTV0910_P1_HYPVIT  0xf53d0003
 
 /* P1_VSTATUSVIT */
 #define RSTV0910_P1_VSTATUSVIT  0xf53e
-#define FSTV0910_P1_PRFVIT  0xf53e0010
-#define FSTV0910_P1_LOCKEDVIT  0xf53e0008
+#define FSTV0910_P1_PRFVIT  0xf53e4010
+#define FSTV0910_P1_LOCKEDVIT  0xf53e3008
 
 /* P1_VTHINUSE */
 #define RSTV0910_P1_VTHINUSE  0xf53f
@@ -3162,32 +3162,32 @@
 
 /* P1_PDELCTRL0 */
 #define RSTV0910_P1_PDELCTRL0  0xf54f
-#define FSTV0910_P1_ISIOBS_MODE  0xf54f0030
+#define FSTV0910_P1_ISIOBS_MODE  0xf54f4030
 
 /* P1_PDELCTRL1 */
 #define RSTV0910_P1_PDELCTRL1  0xf550
-#define FSTV0910_P1_INV_MISMASK  0xf5500080
-#define FSTV0910_P1_FILTER_EN  0xf5500020
-#define FSTV0910_P1_HYSTEN  0xf5500008
-#define FSTV0910_P1_HYSTSWRST  0xf5500004
-#define FSTV0910_P1_EN_MIS00  0xf5500002
+#define FSTV0910_P1_INV_MISMASK  0xf5507080
+#define FSTV0910_P1_FILTER_EN  0xf5505020
+#define FSTV0910_P1_HYSTEN  0xf5503008
+#define FSTV0910_P1_HYSTSWRST  0xf5502004
+#define FSTV0910_P1_EN_MIS00  0xf5501002
 #define FSTV0910_P1_ALGOSWRST  0xf5500001
 
 /* P1_PDELCTRL2 */
 #define RSTV0910_P1_PDELCTRL2  0xf551
-#define FSTV0910_P1_FORCE_CONTINUOUS  0xf5510080
-#define FSTV0910_P1_RESET_UPKO_COUNT  0xf5510040
-#define FSTV0910_P1_USER_PKTDELIN_NB  0xf5510020
-#define FSTV0910_P1_FRAME_MODE  0xf5510002
+#define FSTV0910_P1_FORCE_CONTINUOUS  0xf5517080
+#define FSTV0910_P1_RESET_UPKO_COUNT  0xf5516040
+#define FSTV0910_P1_USER_PKTDELIN_NB  0xf5515020
+#define FSTV0910_P1_FRAME_MODE  0xf5511002
 
 /* P1_HYSTTHRESH */
 #define RSTV0910_P1_HYSTTHRESH  0xf554
-#define FSTV0910_P1_DELIN_LOCKTHRES  0xf55400f0
+#define FSTV0910_P1_DELIN_LOCKTHRES  0xf55440f0
 #define FSTV0910_P1_DELIN_UNLOCKTHRES  0xf554000f
 
 /* P1_UPLCCST0 */
 #define RSTV0910_P1_UPLCCST0  0xf558
-#define FSTV0910_P1_UPL_CST0  0xf55800f8
+#define FSTV0910_P1_UPL_CST0  0xf55830f8
 #define FSTV0910_P1_UPL_MODE  0xf5580007
 
 /* P1_ISIENTRY */
@@ -3236,16 +3236,16 @@
 
 /* P1_PDELSTATUS1 */
 #define RSTV0910_P1_PDELSTATUS1  0xf569
-#define FSTV0910_P1_PKTDELIN_DELOCK  0xf5690080
-#define FSTV0910_P1_SYNCDUPDFL_BADDFL  0xf5690040
-#define FSTV0910_P1_UNACCEPTED_STREAM  0xf5690010
-#define FSTV0910_P1_BCH_ERROR_FLAG  0xf5690008
-#define FSTV0910_P1_PKTDELIN_LOCK  0xf5690002
+#define FSTV0910_P1_PKTDELIN_DELOCK  0xf5697080
+#define FSTV0910_P1_SYNCDUPDFL_BADDFL  0xf5696040
+#define FSTV0910_P1_UNACCEPTED_STREAM  0xf5694010
+#define FSTV0910_P1_BCH_ERROR_FLAG  0xf5693008
+#define FSTV0910_P1_PKTDELIN_LOCK  0xf5691002
 #define FSTV0910_P1_FIRST_LOCK  0xf5690001
 
 /* P1_PDELSTATUS2 */
 #define RSTV0910_P1_PDELSTATUS2  0xf56a
-#define FSTV0910_P1_FRAME_MODCOD  0xf56a007c
+#define FSTV0910_P1_FRAME_MODCOD  0xf56a207c
 #define FSTV0910_P1_FRAME_TYPE  0xf56a0003
 
 /* P1_BBFCRCKO1 */
@@ -3266,92 +3266,92 @@
 
 /* P1_PDELCTRL3 */
 #define RSTV0910_P1_PDELCTRL3  0xf56f
-#define FSTV0910_P1_NOFIFO_BCHERR  0xf56f0020
-#define FSTV0910_P1_PKTDELIN_DELACMERR  0xf56f0010
+#define FSTV0910_P1_NOFIFO_BCHERR  0xf56f5020
+#define FSTV0910_P1_PKTDELIN_DELACMERR  0xf56f4010
 
 /* P1_TSSTATEM */
 #define RSTV0910_P1_TSSTATEM  0xf570
-#define FSTV0910_P1_TSDIL_ON  0xf5700080
-#define FSTV0910_P1_TSRS_ON  0xf5700020
-#define FSTV0910_P1_TSDESCRAMB_ON  0xf5700010
-#define FSTV0910_P1_TSFRAME_MODE  0xf5700008
-#define FSTV0910_P1_TS_DISABLE  0xf5700004
-#define FSTV0910_P1_TSACM_MODE  0xf5700002
+#define FSTV0910_P1_TSDIL_ON  0xf5707080
+#define FSTV0910_P1_TSRS_ON  0xf5705020
+#define FSTV0910_P1_TSDESCRAMB_ON  0xf5704010
+#define FSTV0910_P1_TSFRAME_MODE  0xf5703008
+#define FSTV0910_P1_TS_DISABLE  0xf5702004
+#define FSTV0910_P1_TSACM_MODE  0xf5701002
 #define FSTV0910_P1_TSOUT_NOSYNC  0xf5700001
 
 /* P1_TSSTATEL */
 #define RSTV0910_P1_TSSTATEL  0xf571
-#define FSTV0910_P1_TSNOSYNCBYTE  0xf5710080
-#define FSTV0910_P1_TSPARITY_ON  0xf5710040
-#define FSTV0910_P1_TSISSYI_ON  0xf5710008
-#define FSTV0910_P1_TSNPD_ON  0xf5710004
-#define FSTV0910_P1_TSCRC8_ON  0xf5710002
+#define FSTV0910_P1_TSNOSYNCBYTE  0xf5717080
+#define FSTV0910_P1_TSPARITY_ON  0xf5716040
+#define FSTV0910_P1_TSISSYI_ON  0xf5713008
+#define FSTV0910_P1_TSNPD_ON  0xf5712004
+#define FSTV0910_P1_TSCRC8_ON  0xf5711002
 #define FSTV0910_P1_TSDSS_PACKET  0xf5710001
 
 /* P1_TSCFGH */
 #define RSTV0910_P1_TSCFGH  0xf572
-#define FSTV0910_P1_TSFIFO_DVBCI  0xf5720080
-#define FSTV0910_P1_TSFIFO_SERIAL  0xf5720040
-#define FSTV0910_P1_TSFIFO_TEIUPDATE  0xf5720020
-#define FSTV0910_P1_TSFIFO_DUTY50  0xf5720010
-#define FSTV0910_P1_TSFIFO_HSGNLOUT  0xf5720008
-#define FSTV0910_P1_TSFIFO_ERRMODE  0xf5720006
+#define FSTV0910_P1_TSFIFO_DVBCI  0xf5727080
+#define FSTV0910_P1_TSFIFO_SERIAL  0xf5726040
+#define FSTV0910_P1_TSFIFO_TEIUPDATE  0xf5725020
+#define FSTV0910_P1_TSFIFO_DUTY50  0xf5724010
+#define FSTV0910_P1_TSFIFO_HSGNLOUT  0xf5723008
+#define FSTV0910_P1_TSFIFO_ERRMODE  0xf5721006
 #define FSTV0910_P1_RST_HWARE  0xf5720001
 
 /* P1_TSCFGM */
 #define RSTV0910_P1_TSCFGM  0xf573
-#define FSTV0910_P1_TSFIFO_MANSPEED  0xf57300c0
-#define FSTV0910_P1_TSFIFO_PERMDATA  0xf5730020
-#define FSTV0910_P1_TSFIFO_NONEWSGNL  0xf5730010
+#define FSTV0910_P1_TSFIFO_MANSPEED  0xf57360c0
+#define FSTV0910_P1_TSFIFO_PERMDATA  0xf5735020
+#define FSTV0910_P1_TSFIFO_NONEWSGNL  0xf5734010
 #define FSTV0910_P1_TSFIFO_INVDATA  0xf5730001
 
 /* P1_TSCFGL */
 #define RSTV0910_P1_TSCFGL  0xf574
-#define FSTV0910_P1_TSFIFO_BCLKDEL1CK  0xf57400c0
-#define FSTV0910_P1_BCHERROR_MODE  0xf5740030
-#define FSTV0910_P1_TSFIFO_NSGNL2DATA  0xf5740008
-#define FSTV0910_P1_TSFIFO_EMBINDVB  0xf5740004
+#define FSTV0910_P1_TSFIFO_BCLKDEL1CK  0xf57460c0
+#define FSTV0910_P1_BCHERROR_MODE  0xf5744030
+#define FSTV0910_P1_TSFIFO_NSGNL2DATA  0xf5743008
+#define FSTV0910_P1_TSFIFO_EMBINDVB  0xf5742004
 #define FSTV0910_P1_TSFIFO_BITSPEED  0xf5740003
 
 /* P1_TSSYNC */
 #define RSTV0910_P1_TSSYNC  0xf575
-#define FSTV0910_P1_TSFIFO_SYNCMODE  0xf5750018
+#define FSTV0910_P1_TSFIFO_SYNCMODE  0xf5753018
 
 /* P1_TSINSDELH */
 #define RSTV0910_P1_TSINSDELH  0xf576
-#define FSTV0910_P1_TSDEL_SYNCBYTE  0xf5760080
-#define FSTV0910_P1_TSDEL_XXHEADER  0xf5760040
-#define FSTV0910_P1_TSDEL_DATAFIELD  0xf5760010
-#define FSTV0910_P1_TSINSDEL_RSPARITY  0xf5760002
+#define FSTV0910_P1_TSDEL_SYNCBYTE  0xf5767080
+#define FSTV0910_P1_TSDEL_XXHEADER  0xf5766040
+#define FSTV0910_P1_TSDEL_DATAFIELD  0xf5764010
+#define FSTV0910_P1_TSINSDEL_RSPARITY  0xf5761002
 #define FSTV0910_P1_TSINSDEL_CRC8  0xf5760001
 
 /* P1_TSINSDELM */
 #define RSTV0910_P1_TSINSDELM  0xf577
-#define FSTV0910_P1_TSINS_EMODCOD  0xf5770010
-#define FSTV0910_P1_TSINS_TOKEN  0xf5770008
-#define FSTV0910_P1_TSINS_XXXERR  0xf5770004
-#define FSTV0910_P1_TSINS_MATYPE  0xf5770002
+#define FSTV0910_P1_TSINS_EMODCOD  0xf5774010
+#define FSTV0910_P1_TSINS_TOKEN  0xf5773008
+#define FSTV0910_P1_TSINS_XXXERR  0xf5772004
+#define FSTV0910_P1_TSINS_MATYPE  0xf5771002
 #define FSTV0910_P1_TSINS_UPL  0xf5770001
 
 /* P1_TSINSDELL */
 #define RSTV0910_P1_TSINSDELL  0xf578
-#define FSTV0910_P1_TSINS_DFL  0xf5780080
-#define FSTV0910_P1_TSINS_SYNCD  0xf5780040
-#define FSTV0910_P1_TSINS_BLOCLEN  0xf5780020
-#define FSTV0910_P1_TSINS_SIGPCOUNT  0xf5780010
-#define FSTV0910_P1_TSINS_FIFO  0xf5780008
-#define FSTV0910_P1_TSINS_REALPACK  0xf5780004
-#define FSTV0910_P1_TSINS_TSCONFIG  0xf5780002
+#define FSTV0910_P1_TSINS_DFL  0xf5787080
+#define FSTV0910_P1_TSINS_SYNCD  0xf5786040
+#define FSTV0910_P1_TSINS_BLOCLEN  0xf5785020
+#define FSTV0910_P1_TSINS_SIGPCOUNT  0xf5784010
+#define FSTV0910_P1_TSINS_FIFO  0xf5783008
+#define FSTV0910_P1_TSINS_REALPACK  0xf5782004
+#define FSTV0910_P1_TSINS_TSCONFIG  0xf5781002
 #define FSTV0910_P1_TSINS_LATENCY  0xf5780001
 
 /* P1_TSDIVN */
 #define RSTV0910_P1_TSDIVN  0xf579
-#define FSTV0910_P1_TSFIFO_SPEEDMODE  0xf57900c0
+#define FSTV0910_P1_TSFIFO_SPEEDMODE  0xf57960c0
 #define FSTV0910_P1_TSFIFO_RISEOK  0xf5790007
 
 /* P1_TSCFG4 */
 #define RSTV0910_P1_TSCFG4  0xf57a
-#define FSTV0910_P1_TSFIFO_TSSPEEDMODE  0xf57a00c0
+#define FSTV0910_P1_TSFIFO_TSSPEEDMODE  0xf57a60c0
 
 /* P1_TSSPEED */
 #define RSTV0910_P1_TSSPEED  0xf580
@@ -3359,18 +3359,18 @@
 
 /* P1_TSSTATUS */
 #define RSTV0910_P1_TSSTATUS  0xf581
-#define FSTV0910_P1_TSFIFO_LINEOK  0xf5810080
-#define FSTV0910_P1_TSFIFO_ERROR  0xf5810040
-#define FSTV0910_P1_TSFIFO_NOSYNC  0xf5810010
-#define FSTV0910_P1_TSREGUL_ERROR  0xf5810004
+#define FSTV0910_P1_TSFIFO_LINEOK  0xf5817080
+#define FSTV0910_P1_TSFIFO_ERROR  0xf5816040
+#define FSTV0910_P1_TSFIFO_NOSYNC  0xf5814010
+#define FSTV0910_P1_TSREGUL_ERROR  0xf5812004
 #define FSTV0910_P1_DIL_READY  0xf5810001
 
 /* P1_TSSTATUS2 */
 #define RSTV0910_P1_TSSTATUS2  0xf582
-#define FSTV0910_P1_TSFIFO_DEMODSEL  0xf5820080
-#define FSTV0910_P1_TSFIFOSPEED_STORE  0xf5820040
-#define FSTV0910_P1_DILXX_RESET  0xf5820020
-#define FSTV0910_P1_SCRAMBDETECT  0xf5820002
+#define FSTV0910_P1_TSFIFO_DEMODSEL  0xf5827080
+#define FSTV0910_P1_TSFIFOSPEED_STORE  0xf5826040
+#define FSTV0910_P1_DILXX_RESET  0xf5825020
+#define FSTV0910_P1_SCRAMBDETECT  0xf5821002
 
 /* P1_TSBITRATE1 */
 #define RSTV0910_P1_TSBITRATE1  0xf583
@@ -3382,7 +3382,7 @@
 
 /* P1_TSPACKLEN1 */
 #define RSTV0910_P1_TSPACKLEN1  0xf585
-#define FSTV0910_P1_TSFIFO_PACKCPT  0xf58500e0
+#define FSTV0910_P1_TSFIFO_PACKCPT  0xf58550e0
 
 /* P1_TSDLY2 */
 #define RSTV0910_P1_TSDLY2  0xf589
@@ -3402,8 +3402,8 @@
 
 /* P1_TSBUFSTAT2 */
 #define RSTV0910_P1_TSBUFSTAT2  0xf58d
-#define FSTV0910_P1_TSISCR_3BYTES  0xf58d0080
-#define FSTV0910_P1_TSISCR_NEWDATA  0xf58d0040
+#define FSTV0910_P1_TSISCR_3BYTES  0xf58d7080
+#define FSTV0910_P1_TSISCR_NEWDATA  0xf58d6040
 #define FSTV0910_P1_TSISCR_BUFSTAT2  0xf58d003f
 
 /* P1_TSBUFSTAT1 */
@@ -3416,13 +3416,13 @@
 
 /* P1_TSDEBUGL */
 #define RSTV0910_P1_TSDEBUGL  0xf591
-#define FSTV0910_P1_TSFIFO_ERROR_EVNT  0xf5910004
+#define FSTV0910_P1_TSFIFO_ERROR_EVNT  0xf5912004
 #define FSTV0910_P1_TSFIFO_OVERFLOWM  0xf5910001
 
 /* P1_TSDLYSET2 */
 #define RSTV0910_P1_TSDLYSET2  0xf592
-#define FSTV0910_P1_SOFFIFO_OFFSET  0xf59200c0
-#define FSTV0910_P1_HYSTERESIS_THRESHOLD  0xf5920030
+#define FSTV0910_P1_SOFFIFO_OFFSET  0xf59260c0
+#define FSTV0910_P1_HYSTERESIS_THRESHOLD  0xf5924030
 #define FSTV0910_P1_SOFFIFO_SYMBOFFS2  0xf592000f
 
 /* P1_TSDLYSET1 */
@@ -3435,12 +3435,12 @@
 
 /* P1_ERRCTRL1 */
 #define RSTV0910_P1_ERRCTRL1  0xf598
-#define FSTV0910_P1_ERR_SOURCE1  0xf59800f0
+#define FSTV0910_P1_ERR_SOURCE1  0xf59840f0
 #define FSTV0910_P1_NUM_EVENT1  0xf5980007
 
 /* P1_ERRCNT12 */
 #define RSTV0910_P1_ERRCNT12  0xf599
-#define FSTV0910_P1_ERRCNT1_OLDVALUE  0xf5990080
+#define FSTV0910_P1_ERRCNT1_OLDVALUE  0xf5997080
 #define FSTV0910_P1_ERR_CNT12  0xf599007f
 
 /* P1_ERRCNT11 */
@@ -3453,12 +3453,12 @@
 
 /* P1_ERRCTRL2 */
 #define RSTV0910_P1_ERRCTRL2  0xf59c
-#define FSTV0910_P1_ERR_SOURCE2  0xf59c00f0
+#define FSTV0910_P1_ERR_SOURCE2  0xf59c40f0
 #define FSTV0910_P1_NUM_EVENT2  0xf59c0007
 
 /* P1_ERRCNT22 */
 #define RSTV0910_P1_ERRCNT22  0xf59d
-#define FSTV0910_P1_ERRCNT2_OLDVALUE  0xf59d0080
+#define FSTV0910_P1_ERRCNT2_OLDVALUE  0xf59d7080
 #define FSTV0910_P1_ERR_CNT22  0xf59d007f
 
 /* P1_ERRCNT21 */
@@ -3471,39 +3471,39 @@
 
 /* P1_FECSPY */
 #define RSTV0910_P1_FECSPY  0xf5a0
-#define FSTV0910_P1_SPY_ENABLE  0xf5a00080
-#define FSTV0910_P1_NO_SYNCBYTE  0xf5a00040
-#define FSTV0910_P1_SERIAL_MODE  0xf5a00020
-#define FSTV0910_P1_UNUSUAL_PACKET  0xf5a00010
-#define FSTV0910_P1_BERMETER_DATAMODE  0xf5a0000c
-#define FSTV0910_P1_BERMETER_LMODE  0xf5a00002
+#define FSTV0910_P1_SPY_ENABLE  0xf5a07080
+#define FSTV0910_P1_NO_SYNCBYTE  0xf5a06040
+#define FSTV0910_P1_SERIAL_MODE  0xf5a05020
+#define FSTV0910_P1_UNUSUAL_PACKET  0xf5a04010
+#define FSTV0910_P1_BERMETER_DATAMODE  0xf5a0200c
+#define FSTV0910_P1_BERMETER_LMODE  0xf5a01002
 #define FSTV0910_P1_BERMETER_RESET  0xf5a00001
 
 /* P1_FSPYCFG */
 #define RSTV0910_P1_FSPYCFG  0xf5a1
-#define FSTV0910_P1_FECSPY_INPUT  0xf5a100c0
-#define FSTV0910_P1_RST_ON_ERROR  0xf5a10020
-#define FSTV0910_P1_ONE_SHOT  0xf5a10010
-#define FSTV0910_P1_I2C_MODE  0xf5a1000c
+#define FSTV0910_P1_FECSPY_INPUT  0xf5a160c0
+#define FSTV0910_P1_RST_ON_ERROR  0xf5a15020
+#define FSTV0910_P1_ONE_SHOT  0xf5a14010
+#define FSTV0910_P1_I2C_MODE  0xf5a1200c
 #define FSTV0910_P1_SPY_HYSTERESIS  0xf5a10003
 
 /* P1_FSPYDATA */
 #define RSTV0910_P1_FSPYDATA  0xf5a2
-#define FSTV0910_P1_SPY_STUFFING  0xf5a20080
-#define FSTV0910_P1_SPY_CNULLPKT  0xf5a20020
+#define FSTV0910_P1_SPY_STUFFING  0xf5a27080
+#define FSTV0910_P1_SPY_CNULLPKT  0xf5a25020
 #define FSTV0910_P1_SPY_OUTDATA_MODE  0xf5a2001f
 
 /* P1_FSPYOUT */
 #define RSTV0910_P1_FSPYOUT  0xf5a3
-#define FSTV0910_P1_FSPY_DIRECT  0xf5a30080
+#define FSTV0910_P1_FSPY_DIRECT  0xf5a37080
 #define FSTV0910_P1_STUFF_MODE  0xf5a30007
 
 /* P1_FSTATUS */
 #define RSTV0910_P1_FSTATUS  0xf5a4
-#define FSTV0910_P1_SPY_ENDSIM  0xf5a40080
-#define FSTV0910_P1_VALID_SIM  0xf5a40040
-#define FSTV0910_P1_FOUND_SIGNAL  0xf5a40020
-#define FSTV0910_P1_DSS_SYNCBYTE  0xf5a40010
+#define FSTV0910_P1_SPY_ENDSIM  0xf5a47080
+#define FSTV0910_P1_VALID_SIM  0xf5a46040
+#define FSTV0910_P1_FOUND_SIGNAL  0xf5a45020
+#define FSTV0910_P1_DSS_SYNCBYTE  0xf5a44010
 #define FSTV0910_P1_RESULT_STATE  0xf5a4000f
 
 /* P1_FBERCPT4 */
@@ -3540,8 +3540,8 @@
 
 /* P1_FSPYBER */
 #define RSTV0910_P1_FSPYBER  0xf5b2
-#define FSTV0910_P1_FSPYBER_SYNCBYTE  0xf5b20010
-#define FSTV0910_P1_FSPYBER_UNSYNC  0xf5b20008
+#define FSTV0910_P1_FSPYBER_SYNCBYTE  0xf5b24010
+#define FSTV0910_P1_FSPYBER_UNSYNC  0xf5b23008
 #define FSTV0910_P1_FSPYBER_CTIME  0xf5b20007
 
 /* P1_SFERROR */
@@ -3550,60 +3550,60 @@
 
 /* P1_SFECSTATUS */
 #define RSTV0910_P1_SFECSTATUS  0xf5c3
-#define FSTV0910_P1_SFEC_ON  0xf5c30080
-#define FSTV0910_P1_SFEC_OFF  0xf5c30040
-#define FSTV0910_P1_LOCKEDSFEC  0xf5c30008
-#define FSTV0910_P1_SFEC_DELOCK  0xf5c30004
-#define FSTV0910_P1_SFEC_DEMODSEL  0xf5c30002
+#define FSTV0910_P1_SFEC_ON  0xf5c37080
+#define FSTV0910_P1_SFEC_OFF  0xf5c36040
+#define FSTV0910_P1_LOCKEDSFEC  0xf5c33008
+#define FSTV0910_P1_SFEC_DELOCK  0xf5c32004
+#define FSTV0910_P1_SFEC_DEMODSEL  0xf5c31002
 #define FSTV0910_P1_SFEC_OVFON  0xf5c30001
 
 /* P1_SFKDIV12 */
 #define RSTV0910_P1_SFKDIV12  0xf5c4
-#define FSTV0910_P1_SFECKDIV12_MAN  0xf5c40080
+#define FSTV0910_P1_SFECKDIV12_MAN  0xf5c47080
 
 /* P1_SFKDIV23 */
 #define RSTV0910_P1_SFKDIV23  0xf5c5
-#define FSTV0910_P1_SFECKDIV23_MAN  0xf5c50080
+#define FSTV0910_P1_SFECKDIV23_MAN  0xf5c57080
 
 /* P1_SFKDIV34 */
 #define RSTV0910_P1_SFKDIV34  0xf5c6
-#define FSTV0910_P1_SFECKDIV34_MAN  0xf5c60080
+#define FSTV0910_P1_SFECKDIV34_MAN  0xf5c67080
 
 /* P1_SFKDIV56 */
 #define RSTV0910_P1_SFKDIV56  0xf5c7
-#define FSTV0910_P1_SFECKDIV56_MAN  0xf5c70080
+#define FSTV0910_P1_SFECKDIV56_MAN  0xf5c77080
 
 /* P1_SFKDIV67 */
 #define RSTV0910_P1_SFKDIV67  0xf5c8
-#define FSTV0910_P1_SFECKDIV67_MAN  0xf5c80080
+#define FSTV0910_P1_SFECKDIV67_MAN  0xf5c87080
 
 /* P1_SFKDIV78 */
 #define RSTV0910_P1_SFKDIV78  0xf5c9
-#define FSTV0910_P1_SFECKDIV78_MAN  0xf5c90080
+#define FSTV0910_P1_SFECKDIV78_MAN  0xf5c97080
 
 /* P1_SFSTATUS */
 #define RSTV0910_P1_SFSTATUS  0xf5cc
-#define FSTV0910_P1_SFEC_LINEOK  0xf5cc0080
-#define FSTV0910_P1_SFEC_ERROR  0xf5cc0040
-#define FSTV0910_P1_SFEC_DATA7  0xf5cc0020
-#define FSTV0910_P1_SFEC_PKTDNBRFAIL  0xf5cc0010
-#define FSTV0910_P1_TSSFEC_DEMODSEL  0xf5cc0008
-#define FSTV0910_P1_SFEC_NOSYNC  0xf5cc0004
-#define FSTV0910_P1_SFEC_UNREGULA  0xf5cc0002
+#define FSTV0910_P1_SFEC_LINEOK  0xf5cc7080
+#define FSTV0910_P1_SFEC_ERROR  0xf5cc6040
+#define FSTV0910_P1_SFEC_DATA7  0xf5cc5020
+#define FSTV0910_P1_SFEC_PKTDNBRFAIL  0xf5cc4010
+#define FSTV0910_P1_TSSFEC_DEMODSEL  0xf5cc3008
+#define FSTV0910_P1_SFEC_NOSYNC  0xf5cc2004
+#define FSTV0910_P1_SFEC_UNREGULA  0xf5cc1002
 #define FSTV0910_P1_SFEC_READY  0xf5cc0001
 
 /* P1_SFDLYSET2 */
 #define RSTV0910_P1_SFDLYSET2  0xf5d0
-#define FSTV0910_P1_SFEC_DISABLE  0xf5d00002
+#define FSTV0910_P1_SFEC_DISABLE  0xf5d01002
 
 /* P1_SFERRCTRL */
 #define RSTV0910_P1_SFERRCTRL  0xf5d8
-#define FSTV0910_P1_SFEC_ERR_SOURCE  0xf5d800f0
+#define FSTV0910_P1_SFEC_ERR_SOURCE  0xf5d840f0
 #define FSTV0910_P1_SFEC_NUM_EVENT  0xf5d80007
 
 /* P1_SFERRCNT2 */
 #define RSTV0910_P1_SFERRCNT2  0xf5d9
-#define FSTV0910_P1_SFERRC_OLDVALUE  0xf5d90080
+#define FSTV0910_P1_SFERRC_OLDVALUE  0xf5d97080
 #define FSTV0910_P1_SFEC_ERR_CNT2  0xf5d9007f
 
 /* P1_SFERRCNT1 */
@@ -3616,66 +3616,66 @@
 
 /* RCCFG2 */
 #define RSTV0910_RCCFG2  0xf600
-#define FSTV0910_TSRCFIFO_DVBCI  0xf6000080
-#define FSTV0910_TSRCFIFO_SERIAL  0xf6000040
-#define FSTV0910_TSRCFIFO_DISABLE  0xf6000020
-#define FSTV0910_TSFIFO_2TORC  0xf6000010
-#define FSTV0910_TSRCFIFO_HSGNLOUT  0xf6000008
-#define FSTV0910_TSRCFIFO_ERRMODE  0xf6000006
+#define FSTV0910_TSRCFIFO_DVBCI  0xf6007080
+#define FSTV0910_TSRCFIFO_SERIAL  0xf6006040
+#define FSTV0910_TSRCFIFO_DISABLE  0xf6005020
+#define FSTV0910_TSFIFO_2TORC  0xf6004010
+#define FSTV0910_TSRCFIFO_HSGNLOUT  0xf6003008
+#define FSTV0910_TSRCFIFO_ERRMODE  0xf6001006
 
 /* RCCFG1 */
 #define RSTV0910_RCCFG1  0xf601
-#define FSTV0910_TSRCFIFO_MANSPEED  0xf60100c0
-#define FSTV0910_TSRCFIFO_PERMDATA  0xf6010020
-#define FSTV0910_TSRCFIFO_NONEWSGNL  0xf6010010
+#define FSTV0910_TSRCFIFO_MANSPEED  0xf60160c0
+#define FSTV0910_TSRCFIFO_PERMDATA  0xf6015020
+#define FSTV0910_TSRCFIFO_NONEWSGNL  0xf6014010
 #define FSTV0910_TSRCFIFO_INVDATA  0xf6010001
 
 /* RCCFG0 */
 #define RSTV0910_RCCFG0  0xf602
-#define FSTV0910_TSRCFIFO_BCLKDEL1CK  0xf60200c0
-#define FSTV0910_TSRCFIFO_DUTY50  0xf6020010
-#define FSTV0910_TSRCFIFO_NSGNL2DATA  0xf6020008
-#define FSTV0910_TSRCFIFO_NPDSGNL  0xf6020004
+#define FSTV0910_TSRCFIFO_BCLKDEL1CK  0xf60260c0
+#define FSTV0910_TSRCFIFO_DUTY50  0xf6024010
+#define FSTV0910_TSRCFIFO_NSGNL2DATA  0xf6023008
+#define FSTV0910_TSRCFIFO_NPDSGNL  0xf6022004
 
 /* RCINSDEL2 */
 #define RSTV0910_RCINSDEL2  0xf603
-#define FSTV0910_TSRCDEL_SYNCBYTE  0xf6030080
-#define FSTV0910_TSRCDEL_XXHEADER  0xf6030040
-#define FSTV0910_TSRCDEL_BBHEADER  0xf6030020
-#define FSTV0910_TSRCDEL_DATAFIELD  0xf6030010
-#define FSTV0910_TSRCINSDEL_ISCR  0xf6030008
-#define FSTV0910_TSRCINSDEL_NPD  0xf6030004
-#define FSTV0910_TSRCINSDEL_RSPARITY  0xf6030002
+#define FSTV0910_TSRCDEL_SYNCBYTE  0xf6037080
+#define FSTV0910_TSRCDEL_XXHEADER  0xf6036040
+#define FSTV0910_TSRCDEL_BBHEADER  0xf6035020
+#define FSTV0910_TSRCDEL_DATAFIELD  0xf6034010
+#define FSTV0910_TSRCINSDEL_ISCR  0xf6033008
+#define FSTV0910_TSRCINSDEL_NPD  0xf6032004
+#define FSTV0910_TSRCINSDEL_RSPARITY  0xf6031002
 #define FSTV0910_TSRCINSDEL_CRC8  0xf6030001
 
 /* RCINSDEL1 */
 #define RSTV0910_RCINSDEL1  0xf604
-#define FSTV0910_TSRCINS_BBPADDING  0xf6040080
-#define FSTV0910_TSRCINS_BCHFEC  0xf6040040
-#define FSTV0910_TSRCINS_EMODCOD  0xf6040010
-#define FSTV0910_TSRCINS_TOKEN  0xf6040008
-#define FSTV0910_TSRCINS_XXXERR  0xf6040004
-#define FSTV0910_TSRCINS_MATYPE  0xf6040002
+#define FSTV0910_TSRCINS_BBPADDING  0xf6047080
+#define FSTV0910_TSRCINS_BCHFEC  0xf6046040
+#define FSTV0910_TSRCINS_EMODCOD  0xf6044010
+#define FSTV0910_TSRCINS_TOKEN  0xf6043008
+#define FSTV0910_TSRCINS_XXXERR  0xf6042004
+#define FSTV0910_TSRCINS_MATYPE  0xf6041002
 #define FSTV0910_TSRCINS_UPL  0xf6040001
 
 /* RCINSDEL0 */
 #define RSTV0910_RCINSDEL0  0xf605
-#define FSTV0910_TSRCINS_DFL  0xf6050080
-#define FSTV0910_TSRCINS_SYNCD  0xf6050040
-#define FSTV0910_TSRCINS_BLOCLEN  0xf6050020
-#define FSTV0910_TSRCINS_SIGPCOUNT  0xf6050010
-#define FSTV0910_TSRCINS_FIFO  0xf6050008
-#define FSTV0910_TSRCINS_REALPACK  0xf6050004
-#define FSTV0910_TSRCINS_TSCONFIG  0xf6050002
+#define FSTV0910_TSRCINS_DFL  0xf6057080
+#define FSTV0910_TSRCINS_SYNCD  0xf6056040
+#define FSTV0910_TSRCINS_BLOCLEN  0xf6055020
+#define FSTV0910_TSRCINS_SIGPCOUNT  0xf6054010
+#define FSTV0910_TSRCINS_FIFO  0xf6053008
+#define FSTV0910_TSRCINS_REALPACK  0xf6052004
+#define FSTV0910_TSRCINS_TSCONFIG  0xf6051002
 #define FSTV0910_TSRCINS_LATENCY  0xf6050001
 
 /* RCSTATUS */
 #define RSTV0910_RCSTATUS  0xf606
-#define FSTV0910_TSRCFIFO_LINEOK  0xf6060080
-#define FSTV0910_TSRCFIFO_ERROR  0xf6060040
-#define FSTV0910_TSRCREGUL_ERROR  0xf6060010
-#define FSTV0910_TSRCFIFO_DEMODSEL  0xf6060008
-#define FSTV0910_TSRCFIFOSPEED_STORE  0xf6060004
+#define FSTV0910_TSRCFIFO_LINEOK  0xf6067080
+#define FSTV0910_TSRCFIFO_ERROR  0xf6066040
+#define FSTV0910_TSRCREGUL_ERROR  0xf6064010
+#define FSTV0910_TSRCFIFO_DEMODSEL  0xf6063008
+#define FSTV0910_TSRCFIFOSPEED_STORE  0xf6062004
 #define FSTV0910_TSRCSPEED_IMPOSSIBLE  0xf6060001
 
 /* RCSPEED */
@@ -3684,47 +3684,47 @@
 
 /* TSGENERAL */
 #define RSTV0910_TSGENERAL  0xf630
-#define FSTV0910_TSFIFO_DISTS2PAR  0xf6300040
-#define FSTV0910_MUXSTREAM_OUTMODE  0xf6300008
-#define FSTV0910_TSFIFO_PERMPARAL  0xf6300006
+#define FSTV0910_TSFIFO_DISTS2PAR  0xf6306040
+#define FSTV0910_MUXSTREAM_OUTMODE  0xf6303008
+#define FSTV0910_TSFIFO_PERMPARAL  0xf6301006
 
 /* P1_DISIRQCFG */
 #define RSTV0910_P1_DISIRQCFG  0xf700
-#define FSTV0910_P1_ENRXEND  0xf7000040
-#define FSTV0910_P1_ENRXFIFO8B  0xf7000020
-#define FSTV0910_P1_ENTRFINISH  0xf7000010
-#define FSTV0910_P1_ENTIMEOUT  0xf7000008
-#define FSTV0910_P1_ENTXEND  0xf7000004
-#define FSTV0910_P1_ENTXFIFO64B  0xf7000002
+#define FSTV0910_P1_ENRXEND  0xf7006040
+#define FSTV0910_P1_ENRXFIFO8B  0xf7005020
+#define FSTV0910_P1_ENTRFINISH  0xf7004010
+#define FSTV0910_P1_ENTIMEOUT  0xf7003008
+#define FSTV0910_P1_ENTXEND  0xf7002004
+#define FSTV0910_P1_ENTXFIFO64B  0xf7001002
 #define FSTV0910_P1_ENGAPBURST  0xf7000001
 
 /* P1_DISIRQSTAT */
 #define RSTV0910_P1_DISIRQSTAT  0xf701
-#define FSTV0910_P1_IRQRXEND  0xf7010040
-#define FSTV0910_P1_IRQRXFIFO8B  0xf7010020
-#define FSTV0910_P1_IRQTRFINISH  0xf7010010
-#define FSTV0910_P1_IRQTIMEOUT  0xf7010008
-#define FSTV0910_P1_IRQTXEND  0xf7010004
-#define FSTV0910_P1_IRQTXFIFO64B  0xf7010002
+#define FSTV0910_P1_IRQRXEND  0xf7016040
+#define FSTV0910_P1_IRQRXFIFO8B  0xf7015020
+#define FSTV0910_P1_IRQTRFINISH  0xf7014010
+#define FSTV0910_P1_IRQTIMEOUT  0xf7013008
+#define FSTV0910_P1_IRQTXEND  0xf7012004
+#define FSTV0910_P1_IRQTXFIFO64B  0xf7011002
 #define FSTV0910_P1_IRQGAPBURST  0xf7010001
 
 /* P1_DISTXCFG */
 #define RSTV0910_P1_DISTXCFG  0xf702
-#define FSTV0910_P1_DISTX_RESET  0xf7020080
-#define FSTV0910_P1_TIM_OFF  0xf7020040
-#define FSTV0910_P1_TIM_CMD  0xf7020030
-#define FSTV0910_P1_ENVELOP  0xf7020008
-#define FSTV0910_P1_DIS_PRECHARGE  0xf7020004
+#define FSTV0910_P1_DISTX_RESET  0xf7027080
+#define FSTV0910_P1_TIM_OFF  0xf7026040
+#define FSTV0910_P1_TIM_CMD  0xf7024030
+#define FSTV0910_P1_ENVELOP  0xf7023008
+#define FSTV0910_P1_DIS_PRECHARGE  0xf7022004
 #define FSTV0910_P1_DISEQC_MODE  0xf7020003
 
 /* P1_DISTXSTATUS */
 #define RSTV0910_P1_DISTXSTATUS  0xf703
-#define FSTV0910_P1_TX_FIFO_FULL  0xf7030040
-#define FSTV0910_P1_TX_IDLE  0xf7030020
-#define FSTV0910_P1_GAP_BURST  0xf7030010
-#define FSTV0910_P1_TX_FIFO64B  0xf7030008
-#define FSTV0910_P1_TX_END  0xf7030004
-#define FSTV0910_P1_TR_TIMEOUT  0xf7030002
+#define FSTV0910_P1_TX_FIFO_FULL  0xf7036040
+#define FSTV0910_P1_TX_IDLE  0xf7035020
+#define FSTV0910_P1_GAP_BURST  0xf7034010
+#define FSTV0910_P1_TX_FIFO64B  0xf7033008
+#define FSTV0910_P1_TX_END  0xf7032004
+#define FSTV0910_P1_TR_TIMEOUT  0xf7031002
 #define FSTV0910_P1_TR_FINISH  0xf7030001
 
 /* P1_DISTXBYTES */
@@ -3741,7 +3741,7 @@
 
 /* P1_DISTIMEOCFG */
 #define RSTV0910_P1_DISTIMEOCFG  0xf708
-#define FSTV0910_P1_RXCHOICE  0xf7080006
+#define FSTV0910_P1_RXCHOICE  0xf7081006
 #define FSTV0910_P1_TIMEOUT_OFF  0xf7080001
 
 /* P1_DISTIMEOUT */
@@ -3750,30 +3750,30 @@
 
 /* P1_DISRXCFG */
 #define RSTV0910_P1_DISRXCFG  0xf70a
-#define FSTV0910_P1_DISRX_RESET  0xf70a0080
-#define FSTV0910_P1_EXTENVELOP  0xf70a0040
-#define FSTV0910_P1_PINSELECT  0xf70a0038
-#define FSTV0910_P1_IGNORE_SHORT22K  0xf70a0004
-#define FSTV0910_P1_SIGNED_RXIN  0xf70a0002
+#define FSTV0910_P1_DISRX_RESET  0xf70a7080
+#define FSTV0910_P1_EXTENVELOP  0xf70a6040
+#define FSTV0910_P1_PINSELECT  0xf70a3038
+#define FSTV0910_P1_IGNORE_SHORT22K  0xf70a2004
+#define FSTV0910_P1_SIGNED_RXIN  0xf70a1002
 #define FSTV0910_P1_DISRX_ON  0xf70a0001
 
 /* P1_DISRXSTAT1 */
 #define RSTV0910_P1_DISRXSTAT1  0xf70b
-#define FSTV0910_P1_RXEND  0xf70b0080
-#define FSTV0910_P1_RXACTIVE  0xf70b0040
-#define FSTV0910_P1_RXDETECT  0xf70b0020
-#define FSTV0910_P1_CONTTONE  0xf70b0010
-#define FSTV0910_P1_8BFIFOREADY  0xf70b0008
-#define FSTV0910_P1_FIFOEMPTY  0xf70b0004
+#define FSTV0910_P1_RXEND  0xf70b7080
+#define FSTV0910_P1_RXACTIVE  0xf70b6040
+#define FSTV0910_P1_RXDETECT  0xf70b5020
+#define FSTV0910_P1_CONTTONE  0xf70b4010
+#define FSTV0910_P1_8BFIFOREADY  0xf70b3008
+#define FSTV0910_P1_FIFOEMPTY  0xf70b2004
 
 /* P1_DISRXSTAT0 */
 #define RSTV0910_P1_DISRXSTAT0  0xf70c
-#define FSTV0910_P1_RXFAIL  0xf70c0080
-#define FSTV0910_P1_FIFOPFAIL  0xf70c0040
-#define FSTV0910_P1_RXNONBYTE  0xf70c0020
-#define FSTV0910_P1_FIFOOVF  0xf70c0010
-#define FSTV0910_P1_SHORT22K  0xf70c0008
-#define FSTV0910_P1_RXMSGLOST  0xf70c0004
+#define FSTV0910_P1_RXFAIL  0xf70c7080
+#define FSTV0910_P1_FIFOPFAIL  0xf70c6040
+#define FSTV0910_P1_RXNONBYTE  0xf70c5020
+#define FSTV0910_P1_FIFOOVF  0xf70c4010
+#define FSTV0910_P1_SHORT22K  0xf70c3008
+#define FSTV0910_P1_RXMSGLOST  0xf70c2004
 
 /* P1_DISRXBYTES */
 #define RSTV0910_P1_DISRXBYTES  0xf70d
@@ -3825,41 +3825,41 @@
 
 /* P2_DISIRQCFG */
 #define RSTV0910_P2_DISIRQCFG  0xf740
-#define FSTV0910_P2_ENRXEND  0xf7400040
-#define FSTV0910_P2_ENRXFIFO8B  0xf7400020
-#define FSTV0910_P2_ENTRFINISH  0xf7400010
-#define FSTV0910_P2_ENTIMEOUT  0xf7400008
-#define FSTV0910_P2_ENTXEND  0xf7400004
-#define FSTV0910_P2_ENTXFIFO64B  0xf7400002
+#define FSTV0910_P2_ENRXEND  0xf7406040
+#define FSTV0910_P2_ENRXFIFO8B  0xf7405020
+#define FSTV0910_P2_ENTRFINISH  0xf7404010
+#define FSTV0910_P2_ENTIMEOUT  0xf7403008
+#define FSTV0910_P2_ENTXEND  0xf7402004
+#define FSTV0910_P2_ENTXFIFO64B  0xf7401002
 #define FSTV0910_P2_ENGAPBURST  0xf7400001
 
 /* P2_DISIRQSTAT */
 #define RSTV0910_P2_DISIRQSTAT  0xf741
-#define FSTV0910_P2_IRQRXEND  0xf7410040
-#define FSTV0910_P2_IRQRXFIFO8B  0xf7410020
-#define FSTV0910_P2_IRQTRFINISH  0xf7410010
-#define FSTV0910_P2_IRQTIMEOUT  0xf7410008
-#define FSTV0910_P2_IRQTXEND  0xf7410004
-#define FSTV0910_P2_IRQTXFIFO64B  0xf7410002
+#define FSTV0910_P2_IRQRXEND  0xf7416040
+#define FSTV0910_P2_IRQRXFIFO8B  0xf7415020
+#define FSTV0910_P2_IRQTRFINISH  0xf7414010
+#define FSTV0910_P2_IRQTIMEOUT  0xf7413008
+#define FSTV0910_P2_IRQTXEND  0xf7412004
+#define FSTV0910_P2_IRQTXFIFO64B  0xf7411002
 #define FSTV0910_P2_IRQGAPBURST  0xf7410001
 
 /* P2_DISTXCFG */
 #define RSTV0910_P2_DISTXCFG  0xf742
-#define FSTV0910_P2_DISTX_RESET  0xf7420080
-#define FSTV0910_P2_TIM_OFF  0xf7420040
-#define FSTV0910_P2_TIM_CMD  0xf7420030
-#define FSTV0910_P2_ENVELOP  0xf7420008
-#define FSTV0910_P2_DIS_PRECHARGE  0xf7420004
+#define FSTV0910_P2_DISTX_RESET  0xf7427080
+#define FSTV0910_P2_TIM_OFF  0xf7426040
+#define FSTV0910_P2_TIM_CMD  0xf7424030
+#define FSTV0910_P2_ENVELOP  0xf7423008
+#define FSTV0910_P2_DIS_PRECHARGE  0xf7422004
 #define FSTV0910_P2_DISEQC_MODE  0xf7420003
 
 /* P2_DISTXSTATUS */
 #define RSTV0910_P2_DISTXSTATUS  0xf743
-#define FSTV0910_P2_TX_FIFO_FULL  0xf7430040
-#define FSTV0910_P2_TX_IDLE  0xf7430020
-#define FSTV0910_P2_GAP_BURST  0xf7430010
-#define FSTV0910_P2_TX_FIFO64B  0xf7430008
-#define FSTV0910_P2_TX_END  0xf7430004
-#define FSTV0910_P2_TR_TIMEOUT  0xf7430002
+#define FSTV0910_P2_TX_FIFO_FULL  0xf7436040
+#define FSTV0910_P2_TX_IDLE  0xf7435020
+#define FSTV0910_P2_GAP_BURST  0xf7434010
+#define FSTV0910_P2_TX_FIFO64B  0xf7433008
+#define FSTV0910_P2_TX_END  0xf7432004
+#define FSTV0910_P2_TR_TIMEOUT  0xf7431002
 #define FSTV0910_P2_TR_FINISH  0xf7430001
 
 /* P2_DISTXBYTES */
@@ -3876,7 +3876,7 @@
 
 /* P2_DISTIMEOCFG */
 #define RSTV0910_P2_DISTIMEOCFG  0xf748
-#define FSTV0910_P2_RXCHOICE  0xf7480006
+#define FSTV0910_P2_RXCHOICE  0xf7481006
 #define FSTV0910_P2_TIMEOUT_OFF  0xf7480001
 
 /* P2_DISTIMEOUT */
@@ -3885,30 +3885,30 @@
 
 /* P2_DISRXCFG */
 #define RSTV0910_P2_DISRXCFG  0xf74a
-#define FSTV0910_P2_DISRX_RESET  0xf74a0080
-#define FSTV0910_P2_EXTENVELOP  0xf74a0040
-#define FSTV0910_P2_PINSELECT  0xf74a0038
-#define FSTV0910_P2_IGNORE_SHORT22K  0xf74a0004
-#define FSTV0910_P2_SIGNED_RXIN  0xf74a0002
+#define FSTV0910_P2_DISRX_RESET  0xf74a7080
+#define FSTV0910_P2_EXTENVELOP  0xf74a6040
+#define FSTV0910_P2_PINSELECT  0xf74a3038
+#define FSTV0910_P2_IGNORE_SHORT22K  0xf74a2004
+#define FSTV0910_P2_SIGNED_RXIN  0xf74a1002
 #define FSTV0910_P2_DISRX_ON  0xf74a0001
 
 /* P2_DISRXSTAT1 */
 #define RSTV0910_P2_DISRXSTAT1  0xf74b
-#define FSTV0910_P2_RXEND  0xf74b0080
-#define FSTV0910_P2_RXACTIVE  0xf74b0040
-#define FSTV0910_P2_RXDETECT  0xf74b0020
-#define FSTV0910_P2_CONTTONE  0xf74b0010
-#define FSTV0910_P2_8BFIFOREADY  0xf74b0008
-#define FSTV0910_P2_FIFOEMPTY  0xf74b0004
+#define FSTV0910_P2_RXEND  0xf74b7080
+#define FSTV0910_P2_RXACTIVE  0xf74b6040
+#define FSTV0910_P2_RXDETECT  0xf74b5020
+#define FSTV0910_P2_CONTTONE  0xf74b4010
+#define FSTV0910_P2_8BFIFOREADY  0xf74b3008
+#define FSTV0910_P2_FIFOEMPTY  0xf74b2004
 
 /* P2_DISRXSTAT0 */
 #define RSTV0910_P2_DISRXSTAT0  0xf74c
-#define FSTV0910_P2_RXFAIL  0xf74c0080
-#define FSTV0910_P2_FIFOPFAIL  0xf74c0040
-#define FSTV0910_P2_RXNONBYTE  0xf74c0020
-#define FSTV0910_P2_FIFOOVF  0xf74c0010
-#define FSTV0910_P2_SHORT22K  0xf74c0008
-#define FSTV0910_P2_RXMSGLOST  0xf74c0004
+#define FSTV0910_P2_RXFAIL  0xf74c7080
+#define FSTV0910_P2_FIFOPFAIL  0xf74c6040
+#define FSTV0910_P2_RXNONBYTE  0xf74c5020
+#define FSTV0910_P2_FIFOOVF  0xf74c4010
+#define FSTV0910_P2_SHORT22K  0xf74c3008
+#define FSTV0910_P2_RXMSGLOST  0xf74c2004
 
 /* P2_DISRXBYTES */
 #define RSTV0910_P2_DISRXBYTES  0xf74d
@@ -4168,75 +4168,75 @@
 
 /* SELSATUR6 */
 #define RSTV0910_SELSATUR6  0xfa34
-#define FSTV0910_SSAT_SF27  0xfa340008
-#define FSTV0910_SSAT_SF26  0xfa340004
-#define FSTV0910_SSAT_SF25  0xfa340002
+#define FSTV0910_SSAT_SF27  0xfa343008
+#define FSTV0910_SSAT_SF26  0xfa342004
+#define FSTV0910_SSAT_SF25  0xfa341002
 #define FSTV0910_SSAT_SF24  0xfa340001
 
 /* SELSATUR5 */
 #define RSTV0910_SELSATUR5  0xfa35
-#define FSTV0910_SSAT_SF22  0xfa350080
-#define FSTV0910_SSAT_SF21  0xfa350040
-#define FSTV0910_SSAT_SF20  0xfa350020
-#define FSTV0910_SSAT_SF19  0xfa350010
-#define FSTV0910_SSAT_SF18  0xfa350008
-#define FSTV0910_SSAT_SF16  0xfa350004
-#define FSTV0910_SSAT_SF15  0xfa350002
+#define FSTV0910_SSAT_SF22  0xfa357080
+#define FSTV0910_SSAT_SF21  0xfa356040
+#define FSTV0910_SSAT_SF20  0xfa355020
+#define FSTV0910_SSAT_SF19  0xfa354010
+#define FSTV0910_SSAT_SF18  0xfa353008
+#define FSTV0910_SSAT_SF16  0xfa352004
+#define FSTV0910_SSAT_SF15  0xfa351002
 #define FSTV0910_SSAT_SF14  0xfa350001
 
 /* SELSATUR4 */
 #define RSTV0910_SELSATUR4  0xfa36
-#define FSTV0910_SSAT_SF13  0xfa360080
-#define FSTV0910_SSAT_SF12  0xfa360040
-#define FSTV0910_SSAT_SF10  0xfa360020
-#define FSTV0910_SSAT_SF9  0xfa360010
-#define FSTV0910_SSAT_SF8  0xfa360008
-#define FSTV0910_SSAT_SF7  0xfa360004
-#define FSTV0910_SSAT_SF6  0xfa360002
+#define FSTV0910_SSAT_SF13  0xfa367080
+#define FSTV0910_SSAT_SF12  0xfa366040
+#define FSTV0910_SSAT_SF10  0xfa365020
+#define FSTV0910_SSAT_SF9  0xfa364010
+#define FSTV0910_SSAT_SF8  0xfa363008
+#define FSTV0910_SSAT_SF7  0xfa362004
+#define FSTV0910_SSAT_SF6  0xfa361002
 #define FSTV0910_SSAT_SF5  0xfa360001
 
 /* SELSATUR3 */
 #define RSTV0910_SELSATUR3  0xfa37
-#define FSTV0910_SSAT_SF4  0xfa370080
-#define FSTV0910_SSAT_SF3  0xfa370040
-#define FSTV0910_SSAT_SF2  0xfa370020
-#define FSTV0910_SSAT_SF1  0xfa370010
-#define FSTV0910_SSAT_NF28  0xfa370008
-#define FSTV0910_SSAT_NF27  0xfa370004
-#define FSTV0910_SSAT_NF26  0xfa370002
+#define FSTV0910_SSAT_SF4  0xfa377080
+#define FSTV0910_SSAT_SF3  0xfa376040
+#define FSTV0910_SSAT_SF2  0xfa375020
+#define FSTV0910_SSAT_SF1  0xfa374010
+#define FSTV0910_SSAT_NF28  0xfa373008
+#define FSTV0910_SSAT_NF27  0xfa372004
+#define FSTV0910_SSAT_NF26  0xfa371002
 #define FSTV0910_SSAT_NF25  0xfa370001
 
 /* SELSATUR2 */
 #define RSTV0910_SELSATUR2  0xfa38
-#define FSTV0910_SSAT_NF24  0xfa380080
-#define FSTV0910_SSAT_NF23  0xfa380040
-#define FSTV0910_SSAT_NF22  0xfa380020
-#define FSTV0910_SSAT_NF21  0xfa380010
-#define FSTV0910_SSAT_NF20  0xfa380008
-#define FSTV0910_SSAT_NF19  0xfa380004
-#define FSTV0910_SSAT_NF18  0xfa380002
+#define FSTV0910_SSAT_NF24  0xfa387080
+#define FSTV0910_SSAT_NF23  0xfa386040
+#define FSTV0910_SSAT_NF22  0xfa385020
+#define FSTV0910_SSAT_NF21  0xfa384010
+#define FSTV0910_SSAT_NF20  0xfa383008
+#define FSTV0910_SSAT_NF19  0xfa382004
+#define FSTV0910_SSAT_NF18  0xfa381002
 #define FSTV0910_SSAT_NF17  0xfa380001
 
 /* SELSATUR1 */
 #define RSTV0910_SELSATUR1  0xfa39
-#define FSTV0910_SSAT_NF16  0xfa390080
-#define FSTV0910_SSAT_NF15  0xfa390040
-#define FSTV0910_SSAT_NF14  0xfa390020
-#define FSTV0910_SSAT_NF13  0xfa390010
-#define FSTV0910_SSAT_NF12  0xfa390008
-#define FSTV0910_SSAT_NF11  0xfa390004
-#define FSTV0910_SSAT_NF10  0xfa390002
+#define FSTV0910_SSAT_NF16  0xfa397080
+#define FSTV0910_SSAT_NF15  0xfa396040
+#define FSTV0910_SSAT_NF14  0xfa395020
+#define FSTV0910_SSAT_NF13  0xfa394010
+#define FSTV0910_SSAT_NF12  0xfa393008
+#define FSTV0910_SSAT_NF11  0xfa392004
+#define FSTV0910_SSAT_NF10  0xfa391002
 #define FSTV0910_SSAT_NF9  0xfa390001
 
 /* SELSATUR0 */
 #define RSTV0910_SELSATUR0  0xfa3a
-#define FSTV0910_SSAT_NF8  0xfa3a0080
-#define FSTV0910_SSAT_NF7  0xfa3a0040
-#define FSTV0910_SSAT_NF6  0xfa3a0020
-#define FSTV0910_SSAT_NF5  0xfa3a0010
-#define FSTV0910_SSAT_NF4  0xfa3a0008
-#define FSTV0910_SSAT_NF3  0xfa3a0004
-#define FSTV0910_SSAT_NF2  0xfa3a0002
+#define FSTV0910_SSAT_NF8  0xfa3a7080
+#define FSTV0910_SSAT_NF7  0xfa3a6040
+#define FSTV0910_SSAT_NF6  0xfa3a5020
+#define FSTV0910_SSAT_NF5  0xfa3a4010
+#define FSTV0910_SSAT_NF4  0xfa3a3008
+#define FSTV0910_SSAT_NF3  0xfa3a2004
+#define FSTV0910_SSAT_NF2  0xfa3a1002
 #define FSTV0910_SSAT_NF1  0xfa3a0001
 
 /* GAINLLR_NF1 */
@@ -4449,14 +4449,14 @@
 
 /* CFGEXT */
 #define RSTV0910_CFGEXT  0xfa80
-#define FSTV0910_BYPBCH  0xfa800040
-#define FSTV0910_BYPLDPC  0xfa800020
-#define FSTV0910_SHORTMULT  0xfa800004
+#define FSTV0910_BYPBCH  0xfa806040
+#define FSTV0910_BYPLDPC  0xfa805020
+#define FSTV0910_SHORTMULT  0xfa802004
 
 /* GENCFG */
 #define RSTV0910_GENCFG  0xfa86
-#define FSTV0910_BROADCAST  0xfa860010
-#define FSTV0910_CROSSINPUT  0xfa860002
+#define FSTV0910_BROADCAST  0xfa864010
+#define FSTV0910_CROSSINPUT  0xfa861002
 #define FSTV0910_DDEMOD  0xfa860001
 
 /* LDPCERR1 */
@@ -4469,7 +4469,7 @@
 
 /* BCHERR */
 #define RSTV0910_BCHERR  0xfa98
-#define FSTV0910_ERRORFLAG  0xfa980010
+#define FSTV0910_ERRORFLAG  0xfa984010
 #define FSTV0910_BCH_ERRORS_COUNTER  0xfa98000f
 
 /* P1_MAXEXTRAITER */
@@ -4706,22 +4706,22 @@
 
 /* TSTRES0 */
 #define RSTV0910_TSTRES0  0xff11
-#define FSTV0910_FRESFEC  0xff110080
-#define FSTV0910_FRESSYM1  0xff110008
-#define FSTV0910_FRESSYM2  0xff110004
+#define FSTV0910_FRESFEC  0xff117080
+#define FSTV0910_FRESSYM1  0xff113008
+#define FSTV0910_FRESSYM2  0xff112004
 
 /* TSTOUT */
 #define RSTV0910_TSTOUT  0xff12
-#define FSTV0910_TS  0xff12003e
+#define FSTV0910_TS  0xff12103e
 #define FSTV0910_TEST_OUT  0xff120001
 
 /* TSTIN */
 #define RSTV0910_TSTIN  0xff13
-#define FSTV0910_TEST_IN  0xff130080
+#define FSTV0910_TEST_IN  0xff137080
 
 /* P2_TSTDMD */
 #define RSTV0910_P2_TSTDMD  0xff20
-#define FSTV0910_P2_CFRINIT_INVZIGZAG  0xff200008
+#define FSTV0910_P2_CFRINIT_INVZIGZAG  0xff203008
 
 /* P2_TCTL1 */
 #define RSTV0910_P2_TCTL1  0xff24
@@ -4729,15 +4729,15 @@
 
 /* P2_TCTL4 */
 #define RSTV0910_P2_TCTL4  0xff28
-#define FSTV0910_P2_CFR2TOCFR1_DVBS1  0xff2800c0
+#define FSTV0910_P2_CFR2TOCFR1_DVBS1  0xff2860c0
 
 /* P2_TPKTDELIN */
 #define RSTV0910_P2_TPKTDELIN  0xff37
-#define FSTV0910_P2_CFG_RSPARITYON  0xff370080
+#define FSTV0910_P2_CFG_RSPARITYON  0xff377080
 
 /* P1_TSTDMD */
 #define RSTV0910_P1_TSTDMD  0xff40
-#define FSTV0910_P1_CFRINIT_INVZIGZAG  0xff400008
+#define FSTV0910_P1_CFRINIT_INVZIGZAG  0xff403008
 
 /* P1_TCTL1 */
 #define RSTV0910_P1_TCTL1  0xff44
@@ -4745,16 +4745,16 @@
 
 /* P1_TCTL4 */
 #define RSTV0910_P1_TCTL4  0xff48
-#define FSTV0910_P1_CFR2TOCFR1_DVBS1  0xff4800c0
+#define FSTV0910_P1_CFR2TOCFR1_DVBS1  0xff4860c0
 
 /* P1_TPKTDELIN */
 #define RSTV0910_P1_TPKTDELIN  0xff57
-#define FSTV0910_P1_CFG_RSPARITYON  0xff570080
+#define FSTV0910_P1_CFG_RSPARITYON  0xff577080
 
 /* TSTTSRS */
 #define RSTV0910_TSTTSRS  0xff6d
-#define FSTV0910_TSTRS_DISRS2  0xff6d0002
+#define FSTV0910_TSTRS_DISRS2  0xff6d1002
 #define FSTV0910_TSTRS_DISRS1  0xff6d0001
 
-#define STV0910_NBREGS		975
-#define STV0910_NBFIELDS		1818
+#define STV0910_NBREGS  975
+#define STV0910_NBFIELDS  1818
diff --git a/drivers/media/dvb-frontends/stv6110.h b/drivers/media/dvb-frontends/stv6110.h
index ab73124c0dec..ecfc1faba15c 100644
--- a/drivers/media/dvb-frontends/stv6110.h
+++ b/drivers/media/dvb-frontends/stv6110.h
@@ -22,7 +22,7 @@
 #define __DVB_STV6110_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /* registers */
 #define RSTV6110_CTRL1		0
diff --git a/drivers/media/dvb-frontends/stv6110x.c b/drivers/media/dvb-frontends/stv6110x.c
index 66eba38f1014..d8950028d021 100644
--- a/drivers/media/dvb-frontends/stv6110x.c
+++ b/drivers/media/dvb-frontends/stv6110x.c
@@ -26,7 +26,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "stv6110x_reg.h"
 #include "stv6110x.h"
@@ -46,7 +46,7 @@ static int stv6110x_read_reg(struct stv6110x_state *stv6110x, u8 reg, u8 *data)
 	u8 b0[] = { reg };
 	u8 b1[] = { 0 };
 	struct i2c_msg msg[] = {
-		{ .addr = config->addr, .flags = 0, 	   .buf = b0, .len = 1 },
+		{ .addr = config->addr, .flags = 0,	   .buf = b0, .len = 1 },
 		{ .addr = config->addr, .flags = I2C_M_RD, .buf = b1, .len = 1 }
 	};
 
@@ -97,7 +97,9 @@ static int stv6110x_write_regs(struct stv6110x_state *stv6110x, int start, u8 da
 
 static int stv6110x_write_reg(struct stv6110x_state *stv6110x, u8 reg, u8 data)
 {
-	return stv6110x_write_regs(stv6110x, reg, &data, 1);
+	u8 tmp = data; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return stv6110x_write_regs(stv6110x, reg, &tmp, 1);
 }
 
 static int stv6110x_init(struct dvb_frontend *fe)
diff --git a/drivers/media/dvb-frontends/stv6110x_priv.h b/drivers/media/dvb-frontends/stv6110x_priv.h
index a993aba27b7e..109dfaf4ba42 100644
--- a/drivers/media/dvb-frontends/stv6110x_priv.h
+++ b/drivers/media/dvb-frontends/stv6110x_priv.h
@@ -48,11 +48,11 @@
 
 #define STV6110x_SETFIELD(mask, bitf, val)				\
 	(mask = (mask & (~(((1 << STV6110x_WIDTH_##bitf) - 1) <<	\
-				  STV6110x_OFFST_##bitf))) | 		\
+				  STV6110x_OFFST_##bitf))) |		\
 			  (val << STV6110x_OFFST_##bitf))
 
 #define STV6110x_GETFIELD(bitf, val)					\
-	((val >> STV6110x_OFFST_##bitf) & 				\
+	((val >> STV6110x_OFFST_##bitf) &				\
 	((1 << STV6110x_WIDTH_##bitf) - 1))
 
 #define MAKEWORD16(a, b)			(((a) << 8) | (b))
@@ -68,7 +68,7 @@
 struct stv6110x_state {
 	struct i2c_adapter		*i2c;
 	const struct stv6110x_config	*config;
-	u8 				regs[8];
+	u8				regs[8];
 
 	const struct stv6110x_devctl	*devctl;
 };
diff --git a/drivers/media/dvb-frontends/stv6111.c b/drivers/media/dvb-frontends/stv6111.c
index e3e90070e293..9b715b6fe152 100644
--- a/drivers/media/dvb-frontends/stv6111.c
+++ b/drivers/media/dvb-frontends/stv6111.c
@@ -25,7 +25,7 @@
 
 #include "stv6111.h"
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct stv {
 	struct i2c_adapter *i2c;
@@ -424,6 +424,7 @@ static int set_bandwidth(struct dvb_frontend *fe, u32 cutoff_frequency)
 {
 	struct stv *state = fe->tuner_priv;
 	u32 index = (cutoff_frequency + 999999) / 1000000;
+	int stat = 0;
 
 	if (index < 6)
 		index = 6;
@@ -435,12 +436,14 @@ static int set_bandwidth(struct dvb_frontend *fe, u32 cutoff_frequency)
 	state->reg[0x08] = (state->reg[0x08] & ~0xFC) | ((index - 6) << 2);
 	state->reg[0x09] = (state->reg[0x09] & ~0x0C) | 0x08;
 	if (fe->ops.i2c_gate_ctrl)
-		fe->ops.i2c_gate_ctrl(fe, 1);
-	write_regs(state, 0x08, 2);
-	wait_for_call_done(state, 0x08);
-	if (fe->ops.i2c_gate_ctrl)
+		stat = fe->ops.i2c_gate_ctrl(fe, 1);
+	if (!stat) {
+		write_regs(state, 0x08, 2);
+		wait_for_call_done(state, 0x08);
+	}
+	if (fe->ops.i2c_gate_ctrl && !stat)
 		fe->ops.i2c_gate_ctrl(fe, 0);
-	return 0;
+	return stat;
 }
 
 static int set_lof(struct stv *state, u32 local_frequency, u32 cutoff_frequency)
@@ -518,6 +521,7 @@ static int set_params(struct dvb_frontend *fe)
 	struct stv *state = fe->tuner_priv;
 	struct dtv_frontend_properties *p = &fe->dtv_property_cache;
 	u32 freq, cutoff;
+	int stat = 0;
 
 	if (p->delivery_system != SYS_DVBS && p->delivery_system != SYS_DVBS2)
 		return -EINVAL;
@@ -526,9 +530,10 @@ static int set_params(struct dvb_frontend *fe)
 	cutoff = 5000000 + muldiv32(p->symbol_rate, 135, 200);
 
 	if (fe->ops.i2c_gate_ctrl)
-		fe->ops.i2c_gate_ctrl(fe, 1);
-	set_lof(state, freq, cutoff);
-	if (fe->ops.i2c_gate_ctrl)
+		stat = fe->ops.i2c_gate_ctrl(fe, 1);
+	if (!stat)
+		set_lof(state, freq, cutoff);
+	if (fe->ops.i2c_gate_ctrl && !stat)
 		fe->ops.i2c_gate_ctrl(fe, 0);
 	return 0;
 }
@@ -575,14 +580,17 @@ static int get_rf_strength(struct dvb_frontend *fe, u16 *st)
 	if ((state->reg[0x03] & 0x60) == 0) {
 		/* RF Mode, Read AGC ADC */
 		u8 reg = 0;
+		int stat = 0;
 
 		if (fe->ops.i2c_gate_ctrl)
-			fe->ops.i2c_gate_ctrl(fe, 1);
-		write_reg(state, 0x02, state->reg[0x02] | 0x20);
-		read_reg(state, 2, &reg);
-		if (reg & 0x20)
+			stat = fe->ops.i2c_gate_ctrl(fe, 1);
+		if (!stat) {
+			write_reg(state, 0x02, state->reg[0x02] | 0x20);
 			read_reg(state, 2, &reg);
-		if (fe->ops.i2c_gate_ctrl)
+			if (reg & 0x20)
+				read_reg(state, 2, &reg);
+		}
+		if (fe->ops.i2c_gate_ctrl && !stat)
 			fe->ops.i2c_gate_ctrl(fe, 0);
 
 		if ((state->reg[0x02] & 0x80) == 0)
@@ -652,7 +660,8 @@ struct dvb_frontend *stv6111_attach(struct dvb_frontend *fe,
 				    struct i2c_adapter *i2c, u8 adr)
 {
 	struct stv *state;
-	int stat;
+	int stat = -ENODEV;
+	int gatestat = 0;
 
 	state = kzalloc(sizeof(*state), GFP_KERNEL);
 	if (!state)
@@ -663,9 +672,10 @@ struct dvb_frontend *stv6111_attach(struct dvb_frontend *fe,
 	init_state(state);
 
 	if (fe->ops.i2c_gate_ctrl)
-		fe->ops.i2c_gate_ctrl(fe, 1);
-	stat = attach_init(state);
-	if (fe->ops.i2c_gate_ctrl)
+		gatestat = fe->ops.i2c_gate_ctrl(fe, 1);
+	if (!gatestat)
+		stat = attach_init(state);
+	if (fe->ops.i2c_gate_ctrl && !gatestat)
 		fe->ops.i2c_gate_ctrl(fe, 0);
 	if (stat < 0) {
 		kfree(state);
diff --git a/drivers/media/dvb-frontends/tc90522.c b/drivers/media/dvb-frontends/tc90522.c
index 4687e1546af2..5572b39614d5 100644
--- a/drivers/media/dvb-frontends/tc90522.c
+++ b/drivers/media/dvb-frontends/tc90522.c
@@ -30,7 +30,7 @@
 #include <linux/kernel.h>
 #include <linux/math64.h>
 #include <linux/dvb/frontend.h>
-#include "dvb_math.h"
+#include <media/dvb_math.h>
 #include "tc90522.h"
 
 #define TC90522_I2C_THRU_REG 0xfe
diff --git a/drivers/media/dvb-frontends/tc90522.h b/drivers/media/dvb-frontends/tc90522.h
index b1cbddfa6ee6..10e585f32499 100644
--- a/drivers/media/dvb-frontends/tc90522.h
+++ b/drivers/media/dvb-frontends/tc90522.h
@@ -25,7 +25,7 @@
 #define TC90522_H
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /* I2C device types */
 #define TC90522_I2C_DEV_SAT "tc90522sat"
diff --git a/drivers/media/dvb-frontends/tda10021.c b/drivers/media/dvb-frontends/tda10021.c
index 32ba8401e743..4f588ebde39d 100644
--- a/drivers/media/dvb-frontends/tda10021.c
+++ b/drivers/media/dvb-frontends/tda10021.c
@@ -29,7 +29,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda1002x.h"
 
 
diff --git a/drivers/media/dvb-frontends/tda10023.c b/drivers/media/dvb-frontends/tda10023.c
index 8028007c68eb..6c84916234e3 100644
--- a/drivers/media/dvb-frontends/tda10023.c
+++ b/drivers/media/dvb-frontends/tda10023.c
@@ -35,7 +35,7 @@
 
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda1002x.h"
 
 #define REG0_INIT_VAL 0x23
@@ -211,7 +211,7 @@ static int tda10023_set_symbolrate (struct tda10023_state* state, u32 sr)
 
 		BDRX=1<<(24+NDEC);
 		BDRX*=sr;
-		do_div(BDRX, state->sysclk); 	/* BDRX/=SYSCLK; */
+		do_div(BDRX, state->sysclk);	/* BDRX/=SYSCLK; */
 
 		BDR=(s32)BDRX;
 	}
diff --git a/drivers/media/dvb-frontends/tda10048.c b/drivers/media/dvb-frontends/tda10048.c
index 143b39b5f6c9..de82a2558e15 100644
--- a/drivers/media/dvb-frontends/tda10048.c
+++ b/drivers/media/dvb-frontends/tda10048.c
@@ -27,8 +27,8 @@
 #include <linux/delay.h>
 #include <linux/math64.h>
 #include <asm/div64.h>
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "tda10048.h"
 
 #define TDA10048_DEFAULT_FIRMWARE "dvb-fe-tda10048-1.0.fw"
diff --git a/drivers/media/dvb-frontends/tda1004x.c b/drivers/media/dvb-frontends/tda1004x.c
index e674508c349c..58e3beff5adc 100644
--- a/drivers/media/dvb-frontends/tda1004x.c
+++ b/drivers/media/dvb-frontends/tda1004x.c
@@ -36,7 +36,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda1004x.h"
 
 static int debug;
diff --git a/drivers/media/dvb-frontends/tda10071_priv.h b/drivers/media/dvb-frontends/tda10071_priv.h
index b9c3601802ba..67c46e8a7201 100644
--- a/drivers/media/dvb-frontends/tda10071_priv.h
+++ b/drivers/media/dvb-frontends/tda10071_priv.h
@@ -21,7 +21,7 @@
 #ifndef TDA10071_PRIV
 #define TDA10071_PRIV
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda10071.h"
 #include <linux/firmware.h>
 #include <linux/regmap.h>
diff --git a/drivers/media/dvb-frontends/tda10086.c b/drivers/media/dvb-frontends/tda10086.c
index b6d16c05904d..1a95c521e97f 100644
--- a/drivers/media/dvb-frontends/tda10086.c
+++ b/drivers/media/dvb-frontends/tda10086.c
@@ -27,7 +27,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda10086.h"
 
 #define SACLK 96000000
diff --git a/drivers/media/dvb-frontends/tda18271c2dd.c b/drivers/media/dvb-frontends/tda18271c2dd.c
index 2d2778be2d2f..2e1d36ae943b 100644
--- a/drivers/media/dvb-frontends/tda18271c2dd.c
+++ b/drivers/media/dvb-frontends/tda18271c2dd.c
@@ -27,7 +27,7 @@
 #include <linux/i2c.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda18271c2dd.h"
 
 /* Max transfer size done by I2C transfer functions */
@@ -674,7 +674,6 @@ static int PowerScan(struct tda_state *state,
 			Count = 200000;
 			wait = true;
 		}
-		status = status;
 		if (status < 0)
 			break;
 		if (CID_Gain >= CID_Target) {
diff --git a/drivers/media/dvb-frontends/tda18271c2dd.h b/drivers/media/dvb-frontends/tda18271c2dd.h
index 289653db68e4..afeb9536e9c9 100644
--- a/drivers/media/dvb-frontends/tda18271c2dd.h
+++ b/drivers/media/dvb-frontends/tda18271c2dd.h
@@ -9,8 +9,8 @@ struct dvb_frontend *tda18271c2dd_attach(struct dvb_frontend *fe,
 static inline struct dvb_frontend *tda18271c2dd_attach(struct dvb_frontend *fe,
 					 struct i2c_adapter *i2c, u8 adr)
 {
-        printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
-        return NULL;
+	printk(KERN_WARNING "%s: driver disabled by Kconfig\n", __func__);
+	return NULL;
 }
 #endif
 
diff --git a/drivers/media/dvb-frontends/tda665x.c b/drivers/media/dvb-frontends/tda665x.c
index a63dec44295b..3ef7140ed7f3 100644
--- a/drivers/media/dvb-frontends/tda665x.c
+++ b/drivers/media/dvb-frontends/tda665x.c
@@ -22,7 +22,7 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda665x.h"
 
 struct tda665x_state {
diff --git a/drivers/media/dvb-frontends/tda8083.c b/drivers/media/dvb-frontends/tda8083.c
index aa3200d3c352..29b4f64c030c 100644
--- a/drivers/media/dvb-frontends/tda8083.c
+++ b/drivers/media/dvb-frontends/tda8083.c
@@ -30,7 +30,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/jiffies.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda8083.h"
 
 
diff --git a/drivers/media/dvb-frontends/tda8261.c b/drivers/media/dvb-frontends/tda8261.c
index 4eb294f330bc..f72a54e7eb23 100644
--- a/drivers/media/dvb-frontends/tda8261.c
+++ b/drivers/media/dvb-frontends/tda8261.c
@@ -23,7 +23,7 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda8261.h"
 
 struct tda8261_state {
diff --git a/drivers/media/dvb-frontends/tda826x.h b/drivers/media/dvb-frontends/tda826x.h
index 6a7bed12e741..0ef35ff3807f 100644
--- a/drivers/media/dvb-frontends/tda826x.h
+++ b/drivers/media/dvb-frontends/tda826x.h
@@ -24,7 +24,7 @@
 #define __DVB_TDA826X_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /**
  * Attach a tda826x tuner to the supplied frontend structure.
diff --git a/drivers/media/dvb-frontends/ts2020.c b/drivers/media/dvb-frontends/ts2020.c
index 931e5c98da8a..c55882a8da06 100644
--- a/drivers/media/dvb-frontends/ts2020.c
+++ b/drivers/media/dvb-frontends/ts2020.c
@@ -19,7 +19,7 @@
     Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "ts2020.h"
 #include <linux/regmap.h>
 #include <linux/math64.h>
@@ -368,7 +368,7 @@ static int ts2020_read_tuner_gain(struct dvb_frontend *fe, unsigned v_agc,
 		gain2 = clamp_t(long, gain2, 0, 13);
 		v_agc = clamp_t(long, v_agc, 400, 1100);
 
-		*_gain = -(gain1 * 2330 +
+		*_gain = -((__s64)gain1 * 2330 +
 			   gain2 * 3500 +
 			   v_agc * 24 / 10 * 10 +
 			   10000);
@@ -386,7 +386,7 @@ static int ts2020_read_tuner_gain(struct dvb_frontend *fe, unsigned v_agc,
 		gain3 = clamp_t(long, gain3, 0, 6);
 		v_agc = clamp_t(long, v_agc, 600, 1600);
 
-		*_gain = -(gain1 * 2650 +
+		*_gain = -((__s64)gain1 * 2650 +
 			   gain2 * 3380 +
 			   gain3 * 2850 +
 			   v_agc * 176 / 100 * 10 -
diff --git a/drivers/media/dvb-frontends/tua6100.h b/drivers/media/dvb-frontends/tua6100.h
index 6c098a894ea6..a342bd9c7fbf 100644
--- a/drivers/media/dvb-frontends/tua6100.h
+++ b/drivers/media/dvb-frontends/tua6100.h
@@ -28,7 +28,7 @@
 #define __DVB_TUA6100_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #if IS_REACHABLE(CONFIG_DVB_TUA6100)
 extern struct dvb_frontend *tua6100_attach(struct dvb_frontend *fe, int addr, struct i2c_adapter *i2c);
diff --git a/drivers/media/dvb-frontends/ves1820.c b/drivers/media/dvb-frontends/ves1820.c
index 178363704bd4..1d8979289915 100644
--- a/drivers/media/dvb-frontends/ves1820.c
+++ b/drivers/media/dvb-frontends/ves1820.c
@@ -27,7 +27,7 @@
 #include <linux/slab.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "ves1820.h"
 
 
diff --git a/drivers/media/dvb-frontends/ves1x93.c b/drivers/media/dvb-frontends/ves1x93.c
index d0ee52f66a8e..0c7b3286b04d 100644
--- a/drivers/media/dvb-frontends/ves1x93.c
+++ b/drivers/media/dvb-frontends/ves1x93.c
@@ -30,7 +30,7 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "ves1x93.h"
 
 
diff --git a/drivers/media/dvb-frontends/zd1301_demod.h b/drivers/media/dvb-frontends/zd1301_demod.h
index 6cd8f6f9c415..63c13fa4a54b 100644
--- a/drivers/media/dvb-frontends/zd1301_demod.h
+++ b/drivers/media/dvb-frontends/zd1301_demod.h
@@ -19,7 +19,7 @@
 
 #include <linux/platform_device.h>
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /**
  * struct zd1301_demod_platform_data - Platform data for the zd1301_demod driver
diff --git a/drivers/media/dvb-frontends/zl10036.h b/drivers/media/dvb-frontends/zl10036.h
index ec90ca927739..a1129ab74296 100644
--- a/drivers/media/dvb-frontends/zl10036.h
+++ b/drivers/media/dvb-frontends/zl10036.h
@@ -18,7 +18,7 @@
 #define DVB_ZL10036_H
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct zl10036_config {
 	u8 tuner_address;
diff --git a/drivers/media/dvb-frontends/zl10039.c b/drivers/media/dvb-frontends/zl10039.c
index 623355fc2666..6293bd920fa6 100644
--- a/drivers/media/dvb-frontends/zl10039.c
+++ b/drivers/media/dvb-frontends/zl10039.c
@@ -21,7 +21,7 @@
 #include <linux/slab.h>
 #include <linux/dvb/frontend.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "zl10039.h"
 
 static int debug;
@@ -134,7 +134,9 @@ static inline int zl10039_writereg(struct zl10039_state *state,
 				const enum zl10039_reg_addr reg,
 				const u8 val)
 {
-	return zl10039_write(state, reg, &val, 1);
+	const u8 tmp = val; /* see gcc.gnu.org/bugzilla/show_bug.cgi?id=81715 */
+
+	return zl10039_write(state, reg, &tmp, 1);
 }
 
 static int zl10039_init(struct dvb_frontend *fe)
diff --git a/drivers/media/dvb-frontends/zl10353.c b/drivers/media/dvb-frontends/zl10353.c
index 1c689f7f4ab8..c9901f45deb7 100644
--- a/drivers/media/dvb-frontends/zl10353.c
+++ b/drivers/media/dvb-frontends/zl10353.c
@@ -23,7 +23,7 @@
 #include <linux/slab.h>
 #include <asm/div64.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "zl10353_priv.h"
 #include "zl10353.h"
 
diff --git a/drivers/media/firewire/Makefile b/drivers/media/firewire/Makefile
index 239481344d7c..f96049f5fa90 100644
--- a/drivers/media/firewire/Makefile
+++ b/drivers/media/firewire/Makefile
@@ -2,5 +2,3 @@ obj-$(CONFIG_DVB_FIREDTV) += firedtv.o
 
 firedtv-y += firedtv-avc.o firedtv-ci.o firedtv-dvb.o firedtv-fe.o firedtv-fw.o
 firedtv-$(CONFIG_DVB_FIREDTV_INPUT)    += firedtv-rc.o
-
-ccflags-y += -Idrivers/media/dvb-core
diff --git a/drivers/media/firewire/firedtv-avc.c b/drivers/media/firewire/firedtv-avc.c
index 5bde6c209cd7..1c933b2cf760 100644
--- a/drivers/media/firewire/firedtv-avc.c
+++ b/drivers/media/firewire/firedtv-avc.c
@@ -24,7 +24,7 @@
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 
-#include <dvb_frontend.h>
+#include <media/dvb_frontend.h>
 
 #include "firedtv.h"
 
@@ -47,7 +47,7 @@
 #define AVC_OPCODE_DSIT			0xc8
 #define AVC_OPCODE_DSD			0xcb
 
-#define DESCRIPTOR_TUNER_STATUS 	0x80
+#define DESCRIPTOR_TUNER_STATUS		0x80
 #define DESCRIPTOR_SUBUNIT_IDENTIFIER	0x00
 
 #define SFE_VENDOR_DE_COMPANYID_0	0x00 /* OUI of Digital Everywhere */
@@ -688,7 +688,7 @@ int avc_tuner_get_ts(struct firedtv *fdtv)
 	c->operand[2] = 0xff;	/* status */
 	c->operand[3] = 0x20;	/* system id = DVB */
 	c->operand[4] = 0x00;	/* antenna number */
-	c->operand[5] = 0x0; 	/* system_specific_search_flags */
+	c->operand[5] = 0x0;	/* system_specific_search_flags */
 	c->operand[6] = sl;	/* system_specific_multiplex selection_length */
 	/*
 	 * operand[7]: valid_flags[0]
diff --git a/drivers/media/firewire/firedtv-ci.c b/drivers/media/firewire/firedtv-ci.c
index edbb30fdd9d9..8dc5a7495abe 100644
--- a/drivers/media/firewire/firedtv-ci.c
+++ b/drivers/media/firewire/firedtv-ci.c
@@ -15,7 +15,7 @@
 #include <linux/fs.h>
 #include <linux/module.h>
 
-#include <dvbdev.h>
+#include <media/dvbdev.h>
 
 #include "firedtv.h"
 
@@ -207,9 +207,9 @@ static int fdtv_ca_ioctl(struct file *file, unsigned int cmd, void *arg)
 	return err;
 }
 
-static unsigned int fdtv_ca_io_poll(struct file *file, poll_table *wait)
+static __poll_t fdtv_ca_io_poll(struct file *file, poll_table *wait)
 {
-	return POLLIN;
+	return EPOLLIN;
 }
 
 static const struct file_operations fdtv_ca_fops = {
diff --git a/drivers/media/firewire/firedtv-dvb.c b/drivers/media/firewire/firedtv-dvb.c
index f710e17953e3..2f7ac79215cc 100644
--- a/drivers/media/firewire/firedtv-dvb.c
+++ b/drivers/media/firewire/firedtv-dvb.c
@@ -18,10 +18,10 @@
 #include <linux/mutex.h>
 #include <linux/types.h>
 
-#include <dmxdev.h>
-#include <dvb_demux.h>
-#include <dvbdev.h>
-#include <dvb_frontend.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvbdev.h>
+#include <media/dvb_frontend.h>
 
 #include "firedtv.h"
 
diff --git a/drivers/media/firewire/firedtv-fe.c b/drivers/media/firewire/firedtv-fe.c
index 17acda6bcb6e..a2ef4ede8ebe 100644
--- a/drivers/media/firewire/firedtv-fe.c
+++ b/drivers/media/firewire/firedtv-fe.c
@@ -16,7 +16,7 @@
 #include <linux/string.h>
 #include <linux/types.h>
 
-#include <dvb_frontend.h>
+#include <media/dvb_frontend.h>
 
 #include "firedtv.h"
 
@@ -165,7 +165,7 @@ void fdtv_frontend_init(struct firedtv *fdtv, const char *name)
 	ops->read_snr			= fdtv_read_snr;
 	ops->read_ucblocks		= fdtv_read_uncorrected_blocks;
 
-	ops->diseqc_send_master_cmd 	= fdtv_diseqc_send_master_cmd;
+	ops->diseqc_send_master_cmd	= fdtv_diseqc_send_master_cmd;
 	ops->diseqc_send_burst		= fdtv_diseqc_send_burst;
 	ops->set_tone			= fdtv_set_tone;
 	ops->set_voltage		= fdtv_set_voltage;
@@ -220,7 +220,7 @@ void fdtv_frontend_init(struct firedtv *fdtv, const char *name)
 		fi->symbol_rate_min	= 870000;
 		fi->symbol_rate_max	= 6900000;
 
-		fi->caps 		= FE_CAN_INVERSION_AUTO |
+		fi->caps		= FE_CAN_INVERSION_AUTO |
 					  FE_CAN_QAM_16		|
 					  FE_CAN_QAM_32		|
 					  FE_CAN_QAM_64		|
@@ -236,7 +236,7 @@ void fdtv_frontend_init(struct firedtv *fdtv, const char *name)
 		fi->frequency_max	= 861000000;
 		fi->frequency_stepsize	= 62500;
 
-		fi->caps 		= FE_CAN_INVERSION_AUTO		|
+		fi->caps		= FE_CAN_INVERSION_AUTO		|
 					  FE_CAN_FEC_2_3		|
 					  FE_CAN_TRANSMISSION_MODE_AUTO |
 					  FE_CAN_GUARD_INTERVAL_AUTO	|
diff --git a/drivers/media/firewire/firedtv-fw.c b/drivers/media/firewire/firedtv-fw.c
index 247f0e7cb5f7..92f4112d2e37 100644
--- a/drivers/media/firewire/firedtv-fw.c
+++ b/drivers/media/firewire/firedtv-fw.c
@@ -21,7 +21,7 @@
 
 #include <asm/page.h>
 
-#include <dvb_demux.h>
+#include <media/dvb_demux.h>
 
 #include "firedtv.h"
 
diff --git a/drivers/media/firewire/firedtv.h b/drivers/media/firewire/firedtv.h
index 345d1eda8c05..876cdec8329b 100644
--- a/drivers/media/firewire/firedtv.h
+++ b/drivers/media/firewire/firedtv.h
@@ -24,12 +24,12 @@
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 
-#include <demux.h>
-#include <dmxdev.h>
-#include <dvb_demux.h>
-#include <dvb_frontend.h>
-#include <dvb_net.h>
-#include <dvbdev.h>
+#include <media/demux.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
+#include <media/dvbdev.h>
 
 struct firedtv_tuner_status {
 	unsigned active_system:8;
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index cb5d7ff82915..9f18cd296841 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -665,6 +665,14 @@ config VIDEO_OV7670
 	  OV7670 VGA camera.  It currently only works with the M88ALP01
 	  controller.
 
+config VIDEO_OV7740
+	tristate "OmniVision OV7740 sensor support"
+	depends on I2C && VIDEO_V4L2
+	depends on MEDIA_CAMERA_SUPPORT
+	---help---
+	  This is a Video4Linux2 sensor-level driver for the OmniVision
+	  OV7740 VGA camera sensor.
+
 config VIDEO_OV9650
 	tristate "OmniVision OV9650/OV9652 sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
@@ -777,12 +785,12 @@ config VIDEO_S5K6A3
 	  camera sensor.
 
 config VIDEO_S5K4ECGX
-        tristate "Samsung S5K4ECGX sensor support"
-        depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	tristate "Samsung S5K4ECGX sensor support"
+	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	select CRC32
-        ---help---
-          This is a V4L2 sensor-level driver for Samsung S5K4ECGX 5M
-          camera sensor with an embedded SoC image signal processor.
+	---help---
+	  This is a V4L2 sensor-level driver for Samsung S5K4ECGX 5M
+	  camera sensor with an embedded SoC image signal processor.
 
 config VIDEO_S5K5BAF
 	tristate "Samsung S5K5BAF sensor support"
@@ -813,14 +821,6 @@ config VIDEO_ADP1653
 	  This is a driver for the ADP1653 flash controller. It is used for
 	  example in Nokia N900.
 
-config VIDEO_AS3645A
-	tristate "AS3645A flash driver support"
-	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
-	depends on MEDIA_CAMERA_SUPPORT
-	---help---
-	  This is a driver for the AS3645A and LM3555 flash controllers. It has
-	  build in control for flash, torch and indicator LEDs.
-
 config VIDEO_LM3560
 	tristate "LM3560 dual flash driver support"
 	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index 548a9efce966..c0f94cd8d56d 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -68,6 +68,7 @@ obj-$(CONFIG_VIDEO_OV5670) += ov5670.o
 obj-$(CONFIG_VIDEO_OV6650) += ov6650.o
 obj-$(CONFIG_VIDEO_OV7640) += ov7640.o
 obj-$(CONFIG_VIDEO_OV7670) += ov7670.o
+obj-$(CONFIG_VIDEO_OV7740) += ov7740.o
 obj-$(CONFIG_VIDEO_OV9650) += ov9650.o
 obj-$(CONFIG_VIDEO_OV13858) += ov13858.o
 obj-$(CONFIG_VIDEO_MT9M032) += mt9m032.o
@@ -84,7 +85,6 @@ obj-$(CONFIG_VIDEO_S5K4ECGX)	+= s5k4ecgx.o
 obj-$(CONFIG_VIDEO_S5K5BAF)	+= s5k5baf.o
 obj-$(CONFIG_VIDEO_S5C73M3)	+= s5c73m3/
 obj-$(CONFIG_VIDEO_ADP1653)	+= adp1653.o
-obj-$(CONFIG_VIDEO_AS3645A)	+= as3645a.o
 obj-$(CONFIG_VIDEO_LM3560)	+= lm3560.o
 obj-$(CONFIG_VIDEO_LM3646)	+= lm3646.o
 obj-$(CONFIG_VIDEO_SMIAPP_PLL)	+= smiapp-pll.o
diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c
index 6fb818a775db..25d24a3f10a7 100644
--- a/drivers/media/i2c/adv7180.c
+++ b/drivers/media/i2c/adv7180.c
@@ -1366,11 +1366,9 @@ static int adv7180_probe(struct i2c_client *client,
 err_free_ctrl:
 	adv7180_exit_controls(state);
 err_unregister_vpp_client:
-	if (state->chip_info->flags & ADV7180_FLAG_I2P)
-		i2c_unregister_device(state->vpp_client);
+	i2c_unregister_device(state->vpp_client);
 err_unregister_csi_client:
-	if (state->chip_info->flags & ADV7180_FLAG_MIPI_CSI2)
-		i2c_unregister_device(state->csi_client);
+	i2c_unregister_device(state->csi_client);
 	mutex_destroy(&state->mutex);
 	return ret;
 }
@@ -1388,10 +1386,8 @@ static int adv7180_remove(struct i2c_client *client)
 	media_entity_cleanup(&sd->entity);
 	adv7180_exit_controls(state);
 
-	if (state->chip_info->flags & ADV7180_FLAG_I2P)
-		i2c_unregister_device(state->vpp_client);
-	if (state->chip_info->flags & ADV7180_FLAG_MIPI_CSI2)
-		i2c_unregister_device(state->csi_client);
+	i2c_unregister_device(state->vpp_client);
+	i2c_unregister_device(state->csi_client);
 
 	adv7180_set_power_pin(state, false);
 
diff --git a/drivers/media/i2c/adv7343.c b/drivers/media/i2c/adv7343.c
index 11f9029433cf..4a441ee99dd8 100644
--- a/drivers/media/i2c/adv7343.c
+++ b/drivers/media/i2c/adv7343.c
@@ -100,7 +100,7 @@ static const u8 adv7343_init_reg_val[] = {
 };
 
 /*
- * 			    2^32
+ *			    2^32
  * FSC(reg) =  FSC (HZ) * --------
  *			  27000000
  */
diff --git a/drivers/media/i2c/adv7393.c b/drivers/media/i2c/adv7393.c
index f19ad4ecd11e..b6234c8231c9 100644
--- a/drivers/media/i2c/adv7393.c
+++ b/drivers/media/i2c/adv7393.c
@@ -103,7 +103,7 @@ static const u8 adv7393_init_reg_val[] = {
 };
 
 /*
- * 			    2^32
+ *			    2^32
  * FSC(reg) =  FSC (HZ) * --------
  *			  27000000
  */
diff --git a/drivers/media/i2c/adv748x/adv748x-afe.c b/drivers/media/i2c/adv748x/adv748x-afe.c
index 4aa8e45b5cd3..5188178588c9 100644
--- a/drivers/media/i2c/adv748x/adv748x-afe.c
+++ b/drivers/media/i2c/adv748x/adv748x-afe.c
@@ -267,6 +267,7 @@ static int adv748x_afe_g_input_status(struct v4l2_subdev *sd, u32 *status)
 	ret = adv748x_afe_status(afe, status, NULL);
 
 	mutex_unlock(&state->mutex);
+
 	return ret;
 }
 
diff --git a/drivers/media/i2c/adv748x/adv748x-core.c b/drivers/media/i2c/adv748x/adv748x-core.c
index 5ee14f2c2747..fd92c9e4b519 100644
--- a/drivers/media/i2c/adv748x/adv748x-core.c
+++ b/drivers/media/i2c/adv748x/adv748x-core.c
@@ -225,10 +225,8 @@ static void adv748x_unregister_clients(struct adv748x_state *state)
 {
 	unsigned int i;
 
-	for (i = 1; i < ARRAY_SIZE(state->i2c_clients); ++i) {
-		if (state->i2c_clients[i])
-			i2c_unregister_device(state->i2c_clients[i]);
-	}
+	for (i = 1; i < ARRAY_SIZE(state->i2c_clients); ++i)
+		i2c_unregister_device(state->i2c_clients[i]);
 }
 
 static int adv748x_initialise_clients(struct adv748x_state *state)
@@ -646,14 +644,12 @@ static int adv748x_parse_dt(struct adv748x_state *state)
 
 	for_each_endpoint_of_node(state->dev->of_node, ep_np) {
 		of_graph_parse_endpoint(ep_np, &ep);
-		adv_info(state, "Endpoint %s on port %d",
-				of_node_full_name(ep.local_node),
-				ep.port);
+		adv_info(state, "Endpoint %pOF on port %d", ep.local_node,
+			 ep.port);
 
 		if (ep.port >= ADV748X_PORT_MAX) {
-			adv_err(state, "Invalid endpoint %s on port %d",
-				of_node_full_name(ep.local_node),
-				ep.port);
+			adv_err(state, "Invalid endpoint %pOF on port %d",
+				ep.local_node, ep.port);
 
 			continue;
 		}
diff --git a/drivers/media/i2c/adv748x/adv748x-csi2.c b/drivers/media/i2c/adv748x/adv748x-csi2.c
index 979825d4a419..820b44ed56a8 100644
--- a/drivers/media/i2c/adv748x/adv748x-csi2.c
+++ b/drivers/media/i2c/adv748x/adv748x-csi2.c
@@ -223,13 +223,12 @@ static const struct v4l2_subdev_ops adv748x_csi2_ops = {
 
 int adv748x_csi2_set_pixelrate(struct v4l2_subdev *sd, s64 rate)
 {
-	struct v4l2_ctrl *ctrl;
+	struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd);
 
-	ctrl = v4l2_ctrl_find(sd->ctrl_handler, V4L2_CID_PIXEL_RATE);
-	if (!ctrl)
+	if (!tx->pixel_rate)
 		return -EINVAL;
 
-	return v4l2_ctrl_s_ctrl_int64(ctrl, rate);
+	return v4l2_ctrl_s_ctrl_int64(tx->pixel_rate, rate);
 }
 
 static int adv748x_csi2_s_ctrl(struct v4l2_ctrl *ctrl)
@@ -251,8 +250,10 @@ static int adv748x_csi2_init_controls(struct adv748x_csi2 *tx)
 
 	v4l2_ctrl_handler_init(&tx->ctrl_hdl, 1);
 
-	v4l2_ctrl_new_std(&tx->ctrl_hdl, &adv748x_csi2_ctrl_ops,
-			  V4L2_CID_PIXEL_RATE, 1, INT_MAX, 1, 1);
+	tx->pixel_rate = v4l2_ctrl_new_std(&tx->ctrl_hdl,
+					   &adv748x_csi2_ctrl_ops,
+					   V4L2_CID_PIXEL_RATE, 1, INT_MAX,
+					   1, 1);
 
 	tx->sd.ctrl_handler = &tx->ctrl_hdl;
 	if (tx->ctrl_hdl.error) {
diff --git a/drivers/media/i2c/adv748x/adv748x.h b/drivers/media/i2c/adv748x/adv748x.h
index cc4151b5b31e..6789e2f3bc8c 100644
--- a/drivers/media/i2c/adv748x/adv748x.h
+++ b/drivers/media/i2c/adv748x/adv748x.h
@@ -97,6 +97,7 @@ struct adv748x_csi2 {
 
 	struct media_pad pads[ADV748X_CSI2_NR_PADS];
 	struct v4l2_ctrl_handler ctrl_hdl;
+	struct v4l2_ctrl *pixel_rate;
 	struct v4l2_subdev sd;
 };
 
diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c
index c786cd125417..1544920ec52d 100644
--- a/drivers/media/i2c/adv7604.c
+++ b/drivers/media/i2c/adv7604.c
@@ -1982,6 +1982,7 @@ static void adv76xx_cec_tx_raw_status(struct v4l2_subdev *sd, u8 tx_raw_status)
 			 __func__);
 		cec_transmit_done(state->cec_adap, CEC_TX_STATUS_ARB_LOST,
 				  1, 0, 0, 0);
+		return;
 	}
 	if (tx_raw_status & 0x04) {
 		u8 status;
diff --git a/drivers/media/i2c/as3645a.c b/drivers/media/i2c/as3645a.c
deleted file mode 100644
index af5db71a0888..000000000000
--- a/drivers/media/i2c/as3645a.c
+++ /dev/null
@@ -1,880 +0,0 @@
-/*
- * drivers/media/i2c/as3645a.c - AS3645A and LM3555 flash controllers driver
- *
- * Copyright (C) 2008-2011 Nokia Corporation
- * Copyright (c) 2011, Intel Corporation.
- *
- * Contact: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * TODO:
- * - Check hardware FSTROBE control when sensor driver add support for this
- *
- */
-
-#include <linux/delay.h>
-#include <linux/i2c.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/slab.h>
-
-#include <media/i2c/as3645a.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-
-#define AS_TIMER_MS_TO_CODE(t)			(((t) - 100) / 50)
-#define AS_TIMER_CODE_TO_MS(c)			(50 * (c) + 100)
-
-/* Register definitions */
-
-/* Read-only Design info register: Reset state: xxxx 0001 */
-#define AS_DESIGN_INFO_REG			0x00
-#define AS_DESIGN_INFO_FACTORY(x)		(((x) >> 4))
-#define AS_DESIGN_INFO_MODEL(x)			((x) & 0x0f)
-
-/* Read-only Version control register: Reset state: 0000 0000
- * for first engineering samples
- */
-#define AS_VERSION_CONTROL_REG			0x01
-#define AS_VERSION_CONTROL_RFU(x)		(((x) >> 4))
-#define AS_VERSION_CONTROL_VERSION(x)		((x) & 0x0f)
-
-/* Read / Write	(Indicator and timer register): Reset state: 0000 1111 */
-#define AS_INDICATOR_AND_TIMER_REG		0x02
-#define AS_INDICATOR_AND_TIMER_TIMEOUT_SHIFT	0
-#define AS_INDICATOR_AND_TIMER_VREF_SHIFT	4
-#define AS_INDICATOR_AND_TIMER_INDICATOR_SHIFT	6
-
-/* Read / Write	(Current set register): Reset state: 0110 1001 */
-#define AS_CURRENT_SET_REG			0x03
-#define AS_CURRENT_ASSIST_LIGHT_SHIFT		0
-#define AS_CURRENT_LED_DET_ON			(1 << 3)
-#define AS_CURRENT_FLASH_CURRENT_SHIFT		4
-
-/* Read / Write	(Control register): Reset state: 1011 0100 */
-#define AS_CONTROL_REG				0x04
-#define AS_CONTROL_MODE_SETTING_SHIFT		0
-#define AS_CONTROL_STROBE_ON			(1 << 2)
-#define AS_CONTROL_OUT_ON			(1 << 3)
-#define AS_CONTROL_EXT_TORCH_ON			(1 << 4)
-#define AS_CONTROL_STROBE_TYPE_EDGE		(0 << 5)
-#define AS_CONTROL_STROBE_TYPE_LEVEL		(1 << 5)
-#define AS_CONTROL_COIL_PEAK_SHIFT		6
-
-/* Read only (D3 is read / write) (Fault and info): Reset state: 0000 x000 */
-#define AS_FAULT_INFO_REG			0x05
-#define AS_FAULT_INFO_INDUCTOR_PEAK_LIMIT	(1 << 1)
-#define AS_FAULT_INFO_INDICATOR_LED		(1 << 2)
-#define AS_FAULT_INFO_LED_AMOUNT		(1 << 3)
-#define AS_FAULT_INFO_TIMEOUT			(1 << 4)
-#define AS_FAULT_INFO_OVER_TEMPERATURE		(1 << 5)
-#define AS_FAULT_INFO_SHORT_CIRCUIT		(1 << 6)
-#define AS_FAULT_INFO_OVER_VOLTAGE		(1 << 7)
-
-/* Boost register */
-#define AS_BOOST_REG				0x0d
-#define AS_BOOST_CURRENT_DISABLE		(0 << 0)
-#define AS_BOOST_CURRENT_ENABLE			(1 << 0)
-
-/* Password register is used to unlock boost register writing */
-#define AS_PASSWORD_REG				0x0f
-#define AS_PASSWORD_UNLOCK_VALUE		0x55
-
-enum as_mode {
-	AS_MODE_EXT_TORCH = 0 << AS_CONTROL_MODE_SETTING_SHIFT,
-	AS_MODE_INDICATOR = 1 << AS_CONTROL_MODE_SETTING_SHIFT,
-	AS_MODE_ASSIST = 2 << AS_CONTROL_MODE_SETTING_SHIFT,
-	AS_MODE_FLASH = 3 << AS_CONTROL_MODE_SETTING_SHIFT,
-};
-
-/*
- * struct as3645a
- *
- * @subdev:		V4L2 subdev
- * @pdata:		Flash platform data
- * @power_lock:		Protects power_count
- * @power_count:	Power reference count
- * @led_mode:		V4L2 flash LED mode
- * @timeout:		Flash timeout in microseconds
- * @flash_current:	Flash current (0=200mA ... 15=500mA). Maximum
- *			values are 400mA for two LEDs and 500mA for one LED.
- * @assist_current:	Torch/Assist light current (0=20mA, 1=40mA ... 7=160mA)
- * @indicator_current:	Indicator LED current (0=0mA, 1=2.5mA ... 4=10mA)
- * @strobe_source:	Flash strobe source (software or external)
- */
-struct as3645a {
-	struct v4l2_subdev subdev;
-	const struct as3645a_platform_data *pdata;
-
-	struct mutex power_lock;
-	int power_count;
-
-	/* Controls */
-	struct v4l2_ctrl_handler ctrls;
-
-	enum v4l2_flash_led_mode led_mode;
-	unsigned int timeout;
-	u8 flash_current;
-	u8 assist_current;
-	u8 indicator_current;
-	enum v4l2_flash_strobe_source strobe_source;
-};
-
-#define to_as3645a(sd) container_of(sd, struct as3645a, subdev)
-
-/* Return negative errno else zero on success */
-static int as3645a_write(struct as3645a *flash, u8 addr, u8 val)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->subdev);
-	int rval;
-
-	rval = i2c_smbus_write_byte_data(client, addr, val);
-
-	dev_dbg(&client->dev, "Write Addr:%02X Val:%02X %s\n", addr, val,
-		rval < 0 ? "fail" : "ok");
-
-	return rval;
-}
-
-/* Return negative errno else a data byte received from the device. */
-static int as3645a_read(struct as3645a *flash, u8 addr)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->subdev);
-	int rval;
-
-	rval = i2c_smbus_read_byte_data(client, addr);
-
-	dev_dbg(&client->dev, "Read Addr:%02X Val:%02X %s\n", addr, rval,
-		rval < 0 ? "fail" : "ok");
-
-	return rval;
-}
-
-/* -----------------------------------------------------------------------------
- * Hardware configuration and trigger
- */
-
-/*
- * as3645a_set_config - Set flash configuration registers
- * @flash: The flash
- *
- * Configure the hardware with flash, assist and indicator currents, as well as
- * flash timeout.
- *
- * Return 0 on success, or a negative error code if an I2C communication error
- * occurred.
- */
-static int as3645a_set_config(struct as3645a *flash)
-{
-	int ret;
-	u8 val;
-
-	val = (flash->flash_current << AS_CURRENT_FLASH_CURRENT_SHIFT)
-	    | (flash->assist_current << AS_CURRENT_ASSIST_LIGHT_SHIFT)
-	    | AS_CURRENT_LED_DET_ON;
-
-	ret = as3645a_write(flash, AS_CURRENT_SET_REG, val);
-	if (ret < 0)
-		return ret;
-
-	val = AS_TIMER_MS_TO_CODE(flash->timeout / 1000)
-		    << AS_INDICATOR_AND_TIMER_TIMEOUT_SHIFT;
-
-	val |= (flash->pdata->vref << AS_INDICATOR_AND_TIMER_VREF_SHIFT)
-	    |  ((flash->indicator_current ? flash->indicator_current - 1 : 0)
-		 << AS_INDICATOR_AND_TIMER_INDICATOR_SHIFT);
-
-	return as3645a_write(flash, AS_INDICATOR_AND_TIMER_REG, val);
-}
-
-/*
- * as3645a_set_control - Set flash control register
- * @flash: The flash
- * @mode: Desired output mode
- * @on: Desired output state
- *
- * Configure the hardware with output mode and state.
- *
- * Return 0 on success, or a negative error code if an I2C communication error
- * occurred.
- */
-static int
-as3645a_set_control(struct as3645a *flash, enum as_mode mode, bool on)
-{
-	u8 reg;
-
-	/* Configure output parameters and operation mode. */
-	reg = (flash->pdata->peak << AS_CONTROL_COIL_PEAK_SHIFT)
-	    | (on ? AS_CONTROL_OUT_ON : 0)
-	    | mode;
-
-	if (flash->led_mode == V4L2_FLASH_LED_MODE_FLASH &&
-	    flash->strobe_source == V4L2_FLASH_STROBE_SOURCE_EXTERNAL) {
-		reg |= AS_CONTROL_STROBE_TYPE_LEVEL
-		    |  AS_CONTROL_STROBE_ON;
-	}
-
-	return as3645a_write(flash, AS_CONTROL_REG, reg);
-}
-
-/*
- * as3645a_set_output - Configure output and operation mode
- * @flash: Flash controller
- * @strobe: Strobe the flash (only valid in flash mode)
- *
- * Turn the LEDs output on/off and set the operation mode based on the current
- * parameters.
- *
- * The AS3645A can't control the indicator LED independently of the flash/torch
- * LED. If the flash controller is in V4L2_FLASH_LED_MODE_NONE mode, set the
- * chip to indicator mode. Otherwise set it to assist light (torch) or flash
- * mode.
- *
- * In indicator and assist modes, turn the output on/off based on the indicator
- * and torch currents. In software strobe flash mode, turn the output on/off
- * based on the strobe parameter.
- */
-static int as3645a_set_output(struct as3645a *flash, bool strobe)
-{
-	enum as_mode mode;
-	bool on;
-
-	switch (flash->led_mode) {
-	case V4L2_FLASH_LED_MODE_NONE:
-		on = flash->indicator_current != 0;
-		mode = AS_MODE_INDICATOR;
-		break;
-	case V4L2_FLASH_LED_MODE_TORCH:
-		on = true;
-		mode = AS_MODE_ASSIST;
-		break;
-	case V4L2_FLASH_LED_MODE_FLASH:
-		on = strobe;
-		mode = AS_MODE_FLASH;
-		break;
-	default:
-		BUG();
-	}
-
-	/* Configure output parameters and operation mode. */
-	return as3645a_set_control(flash, mode, on);
-}
-
-/* -----------------------------------------------------------------------------
- * V4L2 controls
- */
-
-static int as3645a_is_active(struct as3645a *flash)
-{
-	int ret;
-
-	ret = as3645a_read(flash, AS_CONTROL_REG);
-	return ret < 0 ? ret : !!(ret & AS_CONTROL_OUT_ON);
-}
-
-static int as3645a_read_fault(struct as3645a *flash)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->subdev);
-	int rval;
-
-	/* NOTE: reading register clear fault status */
-	rval = as3645a_read(flash, AS_FAULT_INFO_REG);
-	if (rval < 0)
-		return rval;
-
-	if (rval & AS_FAULT_INFO_INDUCTOR_PEAK_LIMIT)
-		dev_dbg(&client->dev, "Inductor Peak limit fault\n");
-
-	if (rval & AS_FAULT_INFO_INDICATOR_LED)
-		dev_dbg(&client->dev,
-			"Indicator LED fault: Short circuit or open loop\n");
-
-	dev_dbg(&client->dev, "%u connected LEDs\n",
-		rval & AS_FAULT_INFO_LED_AMOUNT ? 2 : 1);
-
-	if (rval & AS_FAULT_INFO_TIMEOUT)
-		dev_dbg(&client->dev, "Timeout fault\n");
-
-	if (rval & AS_FAULT_INFO_OVER_TEMPERATURE)
-		dev_dbg(&client->dev, "Over temperature fault\n");
-
-	if (rval & AS_FAULT_INFO_SHORT_CIRCUIT)
-		dev_dbg(&client->dev, "Short circuit fault\n");
-
-	if (rval & AS_FAULT_INFO_OVER_VOLTAGE)
-		dev_dbg(&client->dev,
-			"Over voltage fault: Indicates missing capacitor or open connection\n");
-
-	return rval;
-}
-
-static int as3645a_get_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct as3645a *flash =
-		container_of(ctrl->handler, struct as3645a, ctrls);
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->subdev);
-	int value;
-
-	switch (ctrl->id) {
-	case V4L2_CID_FLASH_FAULT:
-		value = as3645a_read_fault(flash);
-		if (value < 0)
-			return value;
-
-		ctrl->cur.val = 0;
-		if (value & AS_FAULT_INFO_SHORT_CIRCUIT)
-			ctrl->cur.val |= V4L2_FLASH_FAULT_SHORT_CIRCUIT;
-		if (value & AS_FAULT_INFO_OVER_TEMPERATURE)
-			ctrl->cur.val |= V4L2_FLASH_FAULT_OVER_TEMPERATURE;
-		if (value & AS_FAULT_INFO_TIMEOUT)
-			ctrl->cur.val |= V4L2_FLASH_FAULT_TIMEOUT;
-		if (value & AS_FAULT_INFO_OVER_VOLTAGE)
-			ctrl->cur.val |= V4L2_FLASH_FAULT_OVER_VOLTAGE;
-		if (value & AS_FAULT_INFO_INDUCTOR_PEAK_LIMIT)
-			ctrl->cur.val |= V4L2_FLASH_FAULT_OVER_CURRENT;
-		if (value & AS_FAULT_INFO_INDICATOR_LED)
-			ctrl->cur.val |= V4L2_FLASH_FAULT_INDICATOR;
-		break;
-
-	case V4L2_CID_FLASH_STROBE_STATUS:
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH) {
-			ctrl->cur.val = 0;
-			break;
-		}
-
-		value = as3645a_is_active(flash);
-		if (value < 0)
-			return value;
-
-		ctrl->cur.val = value;
-		break;
-	}
-
-	dev_dbg(&client->dev, "G_CTRL %08x:%d\n", ctrl->id, ctrl->cur.val);
-
-	return 0;
-}
-
-static int as3645a_set_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct as3645a *flash =
-		container_of(ctrl->handler, struct as3645a, ctrls);
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->subdev);
-	int ret;
-
-	dev_dbg(&client->dev, "S_CTRL %08x:%d\n", ctrl->id, ctrl->val);
-
-	/* If a control that doesn't apply to the current mode is modified,
-	 * we store the value and return immediately. The setting will be
-	 * applied when the LED mode is changed. Otherwise we apply the setting
-	 * immediately.
-	 */
-
-	switch (ctrl->id) {
-	case V4L2_CID_FLASH_LED_MODE:
-		if (flash->indicator_current)
-			return -EBUSY;
-
-		ret = as3645a_set_config(flash);
-		if (ret < 0)
-			return ret;
-
-		flash->led_mode = ctrl->val;
-		return as3645a_set_output(flash, false);
-
-	case V4L2_CID_FLASH_STROBE_SOURCE:
-		flash->strobe_source = ctrl->val;
-
-		/* Applies to flash mode only. */
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH)
-			break;
-
-		return as3645a_set_output(flash, false);
-
-	case V4L2_CID_FLASH_STROBE:
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH)
-			return -EBUSY;
-
-		return as3645a_set_output(flash, true);
-
-	case V4L2_CID_FLASH_STROBE_STOP:
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH)
-			return -EBUSY;
-
-		return as3645a_set_output(flash, false);
-
-	case V4L2_CID_FLASH_TIMEOUT:
-		flash->timeout = ctrl->val;
-
-		/* Applies to flash mode only. */
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH)
-			break;
-
-		return as3645a_set_config(flash);
-
-	case V4L2_CID_FLASH_INTENSITY:
-		flash->flash_current = (ctrl->val - AS3645A_FLASH_INTENSITY_MIN)
-				     / AS3645A_FLASH_INTENSITY_STEP;
-
-		/* Applies to flash mode only. */
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH)
-			break;
-
-		return as3645a_set_config(flash);
-
-	case V4L2_CID_FLASH_TORCH_INTENSITY:
-		flash->assist_current =
-			(ctrl->val - AS3645A_TORCH_INTENSITY_MIN)
-			/ AS3645A_TORCH_INTENSITY_STEP;
-
-		/* Applies to torch mode only. */
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_TORCH)
-			break;
-
-		return as3645a_set_config(flash);
-
-	case V4L2_CID_FLASH_INDICATOR_INTENSITY:
-		if (flash->led_mode != V4L2_FLASH_LED_MODE_NONE)
-			return -EBUSY;
-
-		flash->indicator_current =
-			(ctrl->val - AS3645A_INDICATOR_INTENSITY_MIN)
-			/ AS3645A_INDICATOR_INTENSITY_STEP;
-
-		ret = as3645a_set_config(flash);
-		if (ret < 0)
-			return ret;
-
-		if ((ctrl->val == 0) == (ctrl->cur.val == 0))
-			break;
-
-		return as3645a_set_output(flash, false);
-	}
-
-	return 0;
-}
-
-static const struct v4l2_ctrl_ops as3645a_ctrl_ops = {
-	.g_volatile_ctrl = as3645a_get_ctrl,
-	.s_ctrl = as3645a_set_ctrl,
-};
-
-/* -----------------------------------------------------------------------------
- * V4L2 subdev core operations
- */
-
-/* Put device into know state. */
-static int as3645a_setup(struct as3645a *flash)
-{
-	struct i2c_client *client = v4l2_get_subdevdata(&flash->subdev);
-	int ret;
-
-	/* clear errors */
-	ret = as3645a_read(flash, AS_FAULT_INFO_REG);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(&client->dev, "Fault info: %02x\n", ret);
-
-	ret = as3645a_set_config(flash);
-	if (ret < 0)
-		return ret;
-
-	ret = as3645a_set_output(flash, false);
-	if (ret < 0)
-		return ret;
-
-	/* read status */
-	ret = as3645a_read_fault(flash);
-	if (ret < 0)
-		return ret;
-
-	dev_dbg(&client->dev, "AS_INDICATOR_AND_TIMER_REG: %02x\n",
-		as3645a_read(flash, AS_INDICATOR_AND_TIMER_REG));
-	dev_dbg(&client->dev, "AS_CURRENT_SET_REG: %02x\n",
-		as3645a_read(flash, AS_CURRENT_SET_REG));
-	dev_dbg(&client->dev, "AS_CONTROL_REG: %02x\n",
-		as3645a_read(flash, AS_CONTROL_REG));
-
-	return ret & ~AS_FAULT_INFO_LED_AMOUNT ? -EIO : 0;
-}
-
-static int __as3645a_set_power(struct as3645a *flash, int on)
-{
-	int ret;
-
-	if (!on)
-		as3645a_set_control(flash, AS_MODE_EXT_TORCH, false);
-
-	if (flash->pdata->set_power) {
-		ret = flash->pdata->set_power(&flash->subdev, on);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (!on)
-		return 0;
-
-	ret = as3645a_setup(flash);
-	if (ret < 0) {
-		if (flash->pdata->set_power)
-			flash->pdata->set_power(&flash->subdev, 0);
-	}
-
-	return ret;
-}
-
-static int as3645a_set_power(struct v4l2_subdev *sd, int on)
-{
-	struct as3645a *flash = to_as3645a(sd);
-	int ret = 0;
-
-	mutex_lock(&flash->power_lock);
-
-	if (flash->power_count == !on) {
-		ret = __as3645a_set_power(flash, !!on);
-		if (ret < 0)
-			goto done;
-	}
-
-	flash->power_count += on ? 1 : -1;
-	WARN_ON(flash->power_count < 0);
-
-done:
-	mutex_unlock(&flash->power_lock);
-	return ret;
-}
-
-static int as3645a_registered(struct v4l2_subdev *sd)
-{
-	struct as3645a *flash = to_as3645a(sd);
-	struct i2c_client *client = v4l2_get_subdevdata(sd);
-	int rval, man, model, rfu, version;
-	const char *vendor;
-
-	/* Power up the flash driver and read manufacturer ID, model ID, RFU
-	 * and version.
-	 */
-	rval = as3645a_set_power(&flash->subdev, 1);
-	if (rval < 0)
-		return rval;
-
-	rval = as3645a_read(flash, AS_DESIGN_INFO_REG);
-	if (rval < 0)
-		goto power_off;
-
-	man = AS_DESIGN_INFO_FACTORY(rval);
-	model = AS_DESIGN_INFO_MODEL(rval);
-
-	rval = as3645a_read(flash, AS_VERSION_CONTROL_REG);
-	if (rval < 0)
-		goto power_off;
-
-	rfu = AS_VERSION_CONTROL_RFU(rval);
-	version = AS_VERSION_CONTROL_VERSION(rval);
-
-	/* Verify the chip model and version. */
-	if (model != 0x01 || rfu != 0x00) {
-		dev_err(&client->dev,
-			"AS3645A not detected (model %d rfu %d)\n", model, rfu);
-		rval = -ENODEV;
-		goto power_off;
-	}
-
-	switch (man) {
-	case 1:
-		vendor = "AMS, Austria Micro Systems";
-		break;
-	case 2:
-		vendor = "ADI, Analog Devices Inc.";
-		break;
-	case 3:
-		vendor = "NSC, National Semiconductor";
-		break;
-	case 4:
-		vendor = "NXP";
-		break;
-	case 5:
-		vendor = "TI, Texas Instrument";
-		break;
-	default:
-		vendor = "Unknown";
-	}
-
-	dev_info(&client->dev, "Chip vendor: %s (%d) Version: %d\n", vendor,
-		 man, version);
-
-	rval = as3645a_write(flash, AS_PASSWORD_REG, AS_PASSWORD_UNLOCK_VALUE);
-	if (rval < 0)
-		goto power_off;
-
-	rval = as3645a_write(flash, AS_BOOST_REG, AS_BOOST_CURRENT_DISABLE);
-	if (rval < 0)
-		goto power_off;
-
-	/* Setup default values. This makes sure that the chip is in a known
-	 * state, in case the power rail can't be controlled.
-	 */
-	rval = as3645a_setup(flash);
-
-power_off:
-	as3645a_set_power(&flash->subdev, 0);
-
-	return rval;
-}
-
-static int as3645a_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	return as3645a_set_power(sd, 1);
-}
-
-static int as3645a_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
-{
-	return as3645a_set_power(sd, 0);
-}
-
-static const struct v4l2_subdev_core_ops as3645a_core_ops = {
-	.s_power		= as3645a_set_power,
-};
-
-static const struct v4l2_subdev_ops as3645a_ops = {
-	.core = &as3645a_core_ops,
-};
-
-static const struct v4l2_subdev_internal_ops as3645a_internal_ops = {
-	.registered = as3645a_registered,
-	.open = as3645a_open,
-	.close = as3645a_close,
-};
-
-/* -----------------------------------------------------------------------------
- *  I2C driver
- */
-#ifdef CONFIG_PM
-
-static int as3645a_suspend(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
-	struct as3645a *flash = to_as3645a(subdev);
-	int rval;
-
-	if (flash->power_count == 0)
-		return 0;
-
-	rval = __as3645a_set_power(flash, 0);
-
-	dev_dbg(&client->dev, "Suspend %s\n", rval < 0 ? "failed" : "ok");
-
-	return rval;
-}
-
-static int as3645a_resume(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
-	struct as3645a *flash = to_as3645a(subdev);
-	int rval;
-
-	if (flash->power_count == 0)
-		return 0;
-
-	rval = __as3645a_set_power(flash, 1);
-
-	dev_dbg(&client->dev, "Resume %s\n", rval < 0 ? "fail" : "ok");
-
-	return rval;
-}
-
-#else
-
-#define as3645a_suspend	NULL
-#define as3645a_resume	NULL
-
-#endif /* CONFIG_PM */
-
-/*
- * as3645a_init_controls - Create controls
- * @flash: The flash
- *
- * The number of LEDs reported in platform data is used to compute default
- * limits. Parameters passed through platform data can override those limits.
- */
-static int as3645a_init_controls(struct as3645a *flash)
-{
-	const struct as3645a_platform_data *pdata = flash->pdata;
-	struct v4l2_ctrl *ctrl;
-	int maximum;
-
-	v4l2_ctrl_handler_init(&flash->ctrls, 10);
-
-	/* V4L2_CID_FLASH_LED_MODE */
-	v4l2_ctrl_new_std_menu(&flash->ctrls, &as3645a_ctrl_ops,
-			       V4L2_CID_FLASH_LED_MODE, 2, ~7,
-			       V4L2_FLASH_LED_MODE_NONE);
-
-	/* V4L2_CID_FLASH_STROBE_SOURCE */
-	v4l2_ctrl_new_std_menu(&flash->ctrls, &as3645a_ctrl_ops,
-			       V4L2_CID_FLASH_STROBE_SOURCE,
-			       pdata->ext_strobe ? 1 : 0,
-			       pdata->ext_strobe ? ~3 : ~1,
-			       V4L2_FLASH_STROBE_SOURCE_SOFTWARE);
-
-	flash->strobe_source = V4L2_FLASH_STROBE_SOURCE_SOFTWARE;
-
-	/* V4L2_CID_FLASH_STROBE */
-	v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-			  V4L2_CID_FLASH_STROBE, 0, 0, 0, 0);
-
-	/* V4L2_CID_FLASH_STROBE_STOP */
-	v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-			  V4L2_CID_FLASH_STROBE_STOP, 0, 0, 0, 0);
-
-	/* V4L2_CID_FLASH_STROBE_STATUS */
-	ctrl = v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-				 V4L2_CID_FLASH_STROBE_STATUS, 0, 1, 1, 1);
-	if (ctrl != NULL)
-		ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE;
-
-	/* V4L2_CID_FLASH_TIMEOUT */
-	maximum = pdata->timeout_max;
-
-	v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-			  V4L2_CID_FLASH_TIMEOUT, AS3645A_FLASH_TIMEOUT_MIN,
-			  maximum, AS3645A_FLASH_TIMEOUT_STEP, maximum);
-
-	flash->timeout = maximum;
-
-	/* V4L2_CID_FLASH_INTENSITY */
-	maximum = pdata->flash_max_current;
-
-	v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-			  V4L2_CID_FLASH_INTENSITY, AS3645A_FLASH_INTENSITY_MIN,
-			  maximum, AS3645A_FLASH_INTENSITY_STEP, maximum);
-
-	flash->flash_current = (maximum - AS3645A_FLASH_INTENSITY_MIN)
-			     / AS3645A_FLASH_INTENSITY_STEP;
-
-	/* V4L2_CID_FLASH_TORCH_INTENSITY */
-	maximum = pdata->torch_max_current;
-
-	v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-			  V4L2_CID_FLASH_TORCH_INTENSITY,
-			  AS3645A_TORCH_INTENSITY_MIN, maximum,
-			  AS3645A_TORCH_INTENSITY_STEP,
-			  AS3645A_TORCH_INTENSITY_MIN);
-
-	flash->assist_current = 0;
-
-	/* V4L2_CID_FLASH_INDICATOR_INTENSITY */
-	v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-			  V4L2_CID_FLASH_INDICATOR_INTENSITY,
-			  AS3645A_INDICATOR_INTENSITY_MIN,
-			  AS3645A_INDICATOR_INTENSITY_MAX,
-			  AS3645A_INDICATOR_INTENSITY_STEP,
-			  AS3645A_INDICATOR_INTENSITY_MIN);
-
-	flash->indicator_current = 0;
-
-	/* V4L2_CID_FLASH_FAULT */
-	ctrl = v4l2_ctrl_new_std(&flash->ctrls, &as3645a_ctrl_ops,
-				 V4L2_CID_FLASH_FAULT, 0,
-				 V4L2_FLASH_FAULT_OVER_VOLTAGE |
-				 V4L2_FLASH_FAULT_TIMEOUT |
-				 V4L2_FLASH_FAULT_OVER_TEMPERATURE |
-				 V4L2_FLASH_FAULT_SHORT_CIRCUIT, 0, 0);
-	if (ctrl != NULL)
-		ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE;
-
-	flash->subdev.ctrl_handler = &flash->ctrls;
-
-	return flash->ctrls.error;
-}
-
-static int as3645a_probe(struct i2c_client *client,
-			 const struct i2c_device_id *devid)
-{
-	struct as3645a *flash;
-	int ret;
-
-	if (client->dev.platform_data == NULL)
-		return -ENODEV;
-
-	flash = devm_kzalloc(&client->dev, sizeof(*flash), GFP_KERNEL);
-	if (flash == NULL)
-		return -ENOMEM;
-
-	flash->pdata = client->dev.platform_data;
-
-	v4l2_i2c_subdev_init(&flash->subdev, client, &as3645a_ops);
-	flash->subdev.internal_ops = &as3645a_internal_ops;
-	flash->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-
-	ret = as3645a_init_controls(flash);
-	if (ret < 0)
-		goto done;
-
-	ret = media_entity_pads_init(&flash->subdev.entity, 0, NULL);
-	if (ret < 0)
-		goto done;
-
-	flash->subdev.entity.function = MEDIA_ENT_F_FLASH;
-
-	mutex_init(&flash->power_lock);
-
-	flash->led_mode = V4L2_FLASH_LED_MODE_NONE;
-
-done:
-	if (ret < 0)
-		v4l2_ctrl_handler_free(&flash->ctrls);
-
-	return ret;
-}
-
-static int as3645a_remove(struct i2c_client *client)
-{
-	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
-	struct as3645a *flash = to_as3645a(subdev);
-
-	v4l2_device_unregister_subdev(subdev);
-	v4l2_ctrl_handler_free(&flash->ctrls);
-	media_entity_cleanup(&flash->subdev.entity);
-	mutex_destroy(&flash->power_lock);
-
-	return 0;
-}
-
-static const struct i2c_device_id as3645a_id_table[] = {
-	{ AS3645A_NAME, 0 },
-	{ },
-};
-MODULE_DEVICE_TABLE(i2c, as3645a_id_table);
-
-static const struct dev_pm_ops as3645a_pm_ops = {
-	.suspend = as3645a_suspend,
-	.resume = as3645a_resume,
-};
-
-static struct i2c_driver as3645a_i2c_driver = {
-	.driver	= {
-		.name = AS3645A_NAME,
-		.pm   = &as3645a_pm_ops,
-	},
-	.probe	= as3645a_probe,
-	.remove	= as3645a_remove,
-	.id_table = as3645a_id_table,
-};
-
-module_i2c_driver(as3645a_i2c_driver);
-
-MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
-MODULE_DESCRIPTION("LED flash driver for AS3645A, LM3555 and their clones");
-MODULE_LICENSE("GPL");
diff --git a/drivers/media/i2c/cx25840/Makefile b/drivers/media/i2c/cx25840/Makefile
index 898eb13340ae..ac545812fc6a 100644
--- a/drivers/media/i2c/cx25840/Makefile
+++ b/drivers/media/i2c/cx25840/Makefile
@@ -2,5 +2,3 @@ cx25840-objs    := cx25840-core.o cx25840-audio.o cx25840-firmware.o \
 		   cx25840-vbi.o cx25840-ir.o
 
 obj-$(CONFIG_VIDEO_CX25840) += cx25840.o
-
-ccflags-y += -Idrivers/media/i2c
diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c
index f38bf819d805..98be63ae8590 100644
--- a/drivers/media/i2c/cx25840/cx25840-core.c
+++ b/drivers/media/i2c/cx25840/cx25840-core.c
@@ -201,14 +201,14 @@ static int cx23885_s_io_pin_config(struct v4l2_subdev *sd, size_t n,
 			} else {
 				/* IRQ_N */
 				if (p[i].flags &
-					(V4L2_SUBDEV_IO_PIN_DISABLE |
-					 V4L2_SUBDEV_IO_PIN_INPUT)) {
+					(BIT(V4L2_SUBDEV_IO_PIN_DISABLE) |
+					 BIT(V4L2_SUBDEV_IO_PIN_INPUT))) {
 					pin_ctrl &= ~(0x1 << 25);
 				} else {
 					pin_ctrl |= (0x1 << 25);
 				}
 				if (p[i].flags &
-					V4L2_SUBDEV_IO_PIN_ACTIVE_LOW) {
+					BIT(V4L2_SUBDEV_IO_PIN_ACTIVE_LOW)) {
 					pin_ctrl &= ~(0x1 << 24);
 				} else {
 					pin_ctrl |= (0x1 << 24);
@@ -224,7 +224,7 @@ static int cx23885_s_io_pin_config(struct v4l2_subdev *sd, size_t n,
 			} else {
 				/* GPIO19 */
 				gpio_oe &= ~(0x1 << 0);
-				if (p[i].flags & V4L2_SUBDEV_IO_PIN_SET_VALUE) {
+				if (p[i].flags & BIT(V4L2_SUBDEV_IO_PIN_SET_VALUE)) {
 					gpio_data &= ~(0x1 << 0);
 					gpio_data |= ((p[i].value & 0x1) << 0);
 				}
@@ -236,7 +236,7 @@ static int cx23885_s_io_pin_config(struct v4l2_subdev *sd, size_t n,
 			if (p[i].function != CX23885_PAD_GPIO20) {
 				/* IR_TX */
 				gpio_oe |= (0x1 << 1);
-				if (p[i].flags & V4L2_SUBDEV_IO_PIN_DISABLE)
+				if (p[i].flags & BIT(V4L2_SUBDEV_IO_PIN_DISABLE))
 					pin_ctrl &= ~(0x1 << 10);
 				else
 					pin_ctrl |= (0x1 << 10);
@@ -245,7 +245,7 @@ static int cx23885_s_io_pin_config(struct v4l2_subdev *sd, size_t n,
 			} else {
 				/* GPIO20 */
 				gpio_oe &= ~(0x1 << 1);
-				if (p[i].flags & V4L2_SUBDEV_IO_PIN_SET_VALUE) {
+				if (p[i].flags & BIT(V4L2_SUBDEV_IO_PIN_SET_VALUE)) {
 					gpio_data &= ~(0x1 << 1);
 					gpio_data |= ((p[i].value & 0x1) << 1);
 				}
@@ -263,7 +263,7 @@ static int cx23885_s_io_pin_config(struct v4l2_subdev *sd, size_t n,
 			} else {
 				/* GPIO21 */
 				gpio_oe &= ~(0x1 << 2);
-				if (p[i].flags & V4L2_SUBDEV_IO_PIN_SET_VALUE) {
+				if (p[i].flags & BIT(V4L2_SUBDEV_IO_PIN_SET_VALUE)) {
 					gpio_data &= ~(0x1 << 2);
 					gpio_data |= ((p[i].value & 0x1) << 2);
 				}
@@ -281,7 +281,7 @@ static int cx23885_s_io_pin_config(struct v4l2_subdev *sd, size_t n,
 			} else {
 				/* GPIO22 */
 				gpio_oe &= ~(0x1 << 3);
-				if (p[i].flags & V4L2_SUBDEV_IO_PIN_SET_VALUE) {
+				if (p[i].flags & BIT(V4L2_SUBDEV_IO_PIN_SET_VALUE)) {
 					gpio_data &= ~(0x1 << 3);
 					gpio_data |= ((p[i].value & 0x1) << 3);
 				}
@@ -299,7 +299,7 @@ static int cx23885_s_io_pin_config(struct v4l2_subdev *sd, size_t n,
 			} else {
 				/* GPIO23 */
 				gpio_oe &= ~(0x1 << 4);
-				if (p[i].flags & V4L2_SUBDEV_IO_PIN_SET_VALUE) {
+				if (p[i].flags & BIT(V4L2_SUBDEV_IO_PIN_SET_VALUE)) {
 					gpio_data &= ~(0x1 << 4);
 					gpio_data |= ((p[i].value & 0x1) << 4);
 				}
@@ -1263,7 +1263,7 @@ static int set_input(struct i2c_client *client, enum cx25840_video_input vid_inp
 static int set_v4lstd(struct i2c_client *client)
 {
 	struct cx25840_state *state = to_state(i2c_get_clientdata(client));
-	u8 fmt = 0; 	/* zero is autodetect */
+	u8 fmt = 0;	/* zero is autodetect */
 	u8 pal_m = 0;
 
 	/* First tests should be against specific std */
@@ -1395,8 +1395,9 @@ static int cx25840_set_fmt(struct v4l2_subdev *sd,
 	 * height. Without that margin the cx23885 fails in this
 	 * check.
 	 */
-	if ((fmt->width * 16 < Hsrc) || (Hsrc < fmt->width) ||
-			(Vlines * 8 < Vsrc) || (Vsrc + 1 < Vlines)) {
+	if ((fmt->width == 0) || (Vlines == 0) ||
+	    (fmt->width * 16 < Hsrc) || (Hsrc < fmt->width) ||
+	    (Vlines * 8 < Vsrc) || (Vsrc + 1 < Vlines)) {
 		v4l_err(client, "%dx%d is not a valid size!\n",
 				fmt->width, fmt->height);
 		return -ERANGE;
@@ -1757,11 +1758,11 @@ static int cx25840_g_std(struct v4l2_subdev *sd, v4l2_std_id *std)
 		/* 0111 */ V4L2_STD_PAL_Nc,
 		/* 1000 */ V4L2_STD_PAL_60,
 
-		/* 1001 */ V4L2_STD_UNKNOWN,
-		/* 1010 */ V4L2_STD_UNKNOWN,
 		/* 1001 */ V4L2_STD_UNKNOWN,
 		/* 1010 */ V4L2_STD_UNKNOWN,
 		/* 1011 */ V4L2_STD_UNKNOWN,
+		/* 1100 */ V4L2_STD_SECAM,
+		/* 1101 */ V4L2_STD_UNKNOWN,
 		/* 1110 */ V4L2_STD_UNKNOWN,
 		/* 1111 */ V4L2_STD_UNKNOWN
 	};
@@ -2064,10 +2065,10 @@ static void cx23885_dif_setup(struct i2c_client *client, u32 ifHz)
 
 	/* Assuming TV */
 	/* Calculate the PLL frequency word based on the adjusted ifHz */
-        pll_freq = div_u64((u64)ifHz * 268435456, 50000000);
-        pll_freq_word = (u32)pll_freq;
+	pll_freq = div_u64((u64)ifHz * 268435456, 50000000);
+	pll_freq_word = (u32)pll_freq;
 
-        cx25840_write4(client, DIF_PLL_FREQ_WORD,  pll_freq_word);
+	cx25840_write4(client, DIF_PLL_FREQ_WORD,  pll_freq_word);
 
 	/* Round down to the nearest 100KHz */
 	ifHz = (ifHz / 100000) * 100000;
diff --git a/drivers/media/i2c/cx25840/cx25840-core.h b/drivers/media/i2c/cx25840/cx25840-core.h
index 55432ed42714..fb13a624d2e3 100644
--- a/drivers/media/i2c/cx25840/cx25840-core.h
+++ b/drivers/media/i2c/cx25840/cx25840-core.h
@@ -118,7 +118,7 @@ static inline bool is_cx23888(struct cx25840_state *state)
 }
 
 /* ----------------------------------------------------------------------- */
-/* cx25850-core.c 							   */
+/* cx25850-core.c							   */
 int cx25840_write(struct i2c_client *client, u16 addr, u8 value);
 int cx25840_write4(struct i2c_client *client, u16 addr, u32 value);
 u8 cx25840_read(struct i2c_client *client, u16 addr);
diff --git a/drivers/media/i2c/cx25840/cx25840-ir.c b/drivers/media/i2c/cx25840/cx25840-ir.c
index 9b65c7d2fa84..ad7f66c7aac8 100644
--- a/drivers/media/i2c/cx25840/cx25840-ir.c
+++ b/drivers/media/i2c/cx25840/cx25840-ir.c
@@ -28,7 +28,7 @@ static unsigned int ir_debug;
 module_param(ir_debug, int, 0644);
 MODULE_PARM_DESC(ir_debug, "enable integrated IR debug messages");
 
-#define CX25840_IR_REG_BASE 	0x200
+#define CX25840_IR_REG_BASE	0x200
 
 #define CX25840_IR_CNTRL_REG	0x200
 #define CNTRL_WIN_3_3	0x00000000
@@ -131,7 +131,7 @@ static inline struct cx25840_ir_state *to_ir_state(struct v4l2_subdev *sd)
  * Rx and Tx Clock Divider register computations
  *
  * Note the largest clock divider value of 0xffff corresponds to:
- * 	(0xffff + 1) * 1000 / 108/2 MHz = 1,213,629.629... ns
+ *	(0xffff + 1) * 1000 / 108/2 MHz = 1,213,629.629... ns
  * which fits in 21 bits, so we'll use unsigned int for time arguments.
  */
 static inline u16 count_to_clock_divider(unsigned int d)
@@ -187,7 +187,7 @@ static inline unsigned int clock_divider_to_freq(unsigned int divider,
  * Low Pass Filter register calculations
  *
  * Note the largest count value of 0xffff corresponds to:
- * 	0xffff * 1000 / 108/2 MHz = 1,213,611.11... ns
+ *	0xffff * 1000 / 108/2 MHz = 1,213,611.11... ns
  * which fits in 21 bits, so we'll use unsigned int for time arguments.
  */
 static inline u16 count_to_lpf_count(unsigned int d)
diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c
index ed01e8bd4331..8dbbf0f917df 100644
--- a/drivers/media/i2c/dw9714.c
+++ b/drivers/media/i2c/dw9714.c
@@ -42,7 +42,6 @@
 
 /* dw9714 device structure */
 struct dw9714_device {
-	struct i2c_client *client;
 	struct v4l2_ctrl_handler ctrls_vcm;
 	struct v4l2_subdev sd;
 	u16 current_val;
@@ -61,7 +60,7 @@ static inline struct dw9714_device *sd_to_dw9714_vcm(struct v4l2_subdev *subdev)
 static int dw9714_i2c_write(struct i2c_client *client, u16 data)
 {
 	int ret;
-	u16 val = cpu_to_be16(data);
+	__be16 val = cpu_to_be16(data);
 
 	ret = i2c_master_send(client, (const char *)&val, sizeof(val));
 	if (ret != sizeof(val)) {
@@ -73,7 +72,7 @@ static int dw9714_i2c_write(struct i2c_client *client, u16 data)
 
 static int dw9714_t_focus_vcm(struct dw9714_device *dw9714_dev, u16 val)
 {
-	struct i2c_client *client = dw9714_dev->client;
+	struct i2c_client *client = v4l2_get_subdevdata(&dw9714_dev->sd);
 
 	dw9714_dev->current_val = val;
 
@@ -96,13 +95,11 @@ static const struct v4l2_ctrl_ops dw9714_vcm_ctrl_ops = {
 
 static int dw9714_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 {
-	struct dw9714_device *dw9714_dev = sd_to_dw9714_vcm(sd);
-	struct device *dev = &dw9714_dev->client->dev;
 	int rval;
 
-	rval = pm_runtime_get_sync(dev);
+	rval = pm_runtime_get_sync(sd->dev);
 	if (rval < 0) {
-		pm_runtime_put_noidle(dev);
+		pm_runtime_put_noidle(sd->dev);
 		return rval;
 	}
 
@@ -111,10 +108,7 @@ static int dw9714_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 
 static int dw9714_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
 {
-	struct dw9714_device *dw9714_dev = sd_to_dw9714_vcm(sd);
-	struct device *dev = &dw9714_dev->client->dev;
-
-	pm_runtime_put(dev);
+	pm_runtime_put(sd->dev);
 
 	return 0;
 }
@@ -137,7 +131,6 @@ static int dw9714_init_controls(struct dw9714_device *dev_vcm)
 {
 	struct v4l2_ctrl_handler *hdl = &dev_vcm->ctrls_vcm;
 	const struct v4l2_ctrl_ops *ops = &dw9714_vcm_ctrl_ops;
-	struct i2c_client *client = dev_vcm->client;
 
 	v4l2_ctrl_handler_init(hdl, 1);
 
@@ -145,7 +138,7 @@ static int dw9714_init_controls(struct dw9714_device *dev_vcm)
 			  0, DW9714_MAX_FOCUS_POS, DW9714_FOCUS_STEPS, 0);
 
 	if (hdl->error)
-		dev_err(&client->dev, "%s fail error: 0x%x\n",
+		dev_err(dev_vcm->sd.dev, "%s fail error: 0x%x\n",
 			__func__, hdl->error);
 	dev_vcm->sd.ctrl_handler = hdl;
 	return hdl->error;
@@ -161,8 +154,6 @@ static int dw9714_probe(struct i2c_client *client)
 	if (dw9714_dev == NULL)
 		return -ENOMEM;
 
-	dw9714_dev->client = client;
-
 	v4l2_i2c_subdev_init(&dw9714_dev->sd, client, &dw9714_ops);
 	dw9714_dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	dw9714_dev->sd.internal_ops = &dw9714_int_ops;
@@ -183,6 +174,7 @@ static int dw9714_probe(struct i2c_client *client)
 
 	pm_runtime_set_active(&client->dev);
 	pm_runtime_enable(&client->dev);
+	pm_runtime_idle(&client->dev);
 
 	return 0;
 
diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c
index 2f71af2f90bf..664e8acdf2a0 100644
--- a/drivers/media/i2c/imx274.c
+++ b/drivers/media/i2c/imx274.c
@@ -634,7 +634,7 @@ static int imx274_regmap_util_write_table_8(struct regmap *regmap,
 					    const struct reg_8 table[],
 					    u16 wait_ms_addr, u16 end_addr)
 {
-	int err;
+	int err = 0;
 	const struct reg_8 *next;
 	u8 val;
 
@@ -655,6 +655,8 @@ static int imx274_regmap_util_write_table_8(struct regmap *regmap,
 				err = regmap_bulk_write(regmap, range_start,
 							&range_vals[0],
 							range_count);
+			else
+				err = 0;
 
 			if (err)
 				return err;
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
index 8b5f7d0435e4..193020d64e51 100644
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -18,6 +18,20 @@
  *      Brian Rogers <brian_rogers@comcast.net>
  * modified for AVerMedia Cardbus by
  *      Oldrich Jedlicka <oldium.pro@seznam.cz>
+ * Zilog Transmitter portions/ideas were derived from GPLv2+ sources:
+ *  - drivers/char/pctv_zilogir.[ch] from Hauppauge Broadway product
+ *	Copyright 2011 Hauppauge Computer works
+ *  - drivers/staging/media/lirc/lirc_zilog.c
+ *	Copyright (c) 2000 Gerd Knorr <kraxel@goldbach.in-berlin.de>
+ *	Michal Kochanowicz <mkochano@pld.org.pl>
+ *	Christoph Bartelmus <lirc@bartelmus.de>
+ *	Ulrich Mueller <ulrich.mueller42@web.de>
+ *	Stefan Jahn <stefan@lkcc.org>
+ *	Jerome Brock <jbrock@users.sourceforge.net>
+ *	Thomas Reitmayr (treitmayr@yahoo.com)
+ *	Mark Weaver <mark@npsl.co.uk>
+ *	Jarod Wilson <jarod@redhat.com>
+ *	Copyright (C) 2011 Andy Walls <awalls@md.metrocast.net>
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -46,18 +60,11 @@
 #include <media/rc-core.h>
 #include <media/i2c/ir-kbd-i2c.h>
 
-/* ----------------------------------------------------------------------- */
-/* insmod parameters                                                       */
-
-static int debug;
-module_param(debug, int, 0644);    /* debug level (0,1,2) */
-
+#define FLAG_TX		1
+#define FLAG_HDPVR	2
 
-#define MODULE_NAME "ir-kbd-i2c"
-#define dprintk(level, fmt, arg...)	if (debug >= level) \
-	printk(KERN_DEBUG MODULE_NAME ": " fmt , ## arg)
-
-/* ----------------------------------------------------------------------- */
+static bool enable_hdpvr;
+module_param(enable_hdpvr, bool, 0644);
 
 static int get_key_haup_common(struct IR_i2c *ir, enum rc_proto *protocol,
 			       u32 *scancode, u8 *ptoggle, int size)
@@ -96,7 +103,8 @@ static int get_key_haup_common(struct IR_i2c *ir, enum rc_proto *protocol,
 		if (!range)
 			code += 64;
 
-		dprintk(1, "ir hauppauge (rc5): s%d r%d t%d dev=%d code=%d\n",
+		dev_dbg(&ir->rc->dev,
+			"ir hauppauge (rc5): s%d r%d t%d dev=%d code=%d\n",
 			start, range, toggle, dev, code);
 
 		*protocol = RC_PROTO_RC5;
@@ -113,13 +121,15 @@ static int get_key_haup_common(struct IR_i2c *ir, enum rc_proto *protocol,
 			*ptoggle = (dev & 0x80) != 0;
 			*protocol = RC_PROTO_RC6_MCE;
 			dev &= 0x7f;
-			dprintk(1, "ir hauppauge (rc6-mce): t%d vendor=%d dev=%d code=%d\n",
-						*ptoggle, vendor, dev, code);
+			dev_dbg(&ir->rc->dev,
+				"ir hauppauge (rc6-mce): t%d vendor=%d dev=%d code=%d\n",
+				*ptoggle, vendor, dev, code);
 		} else {
 			*ptoggle = 0;
 			*protocol = RC_PROTO_RC6_6A_32;
-			dprintk(1, "ir hauppauge (rc6-6a-32): vendor=%d dev=%d code=%d\n",
-							vendor, dev, code);
+			dev_dbg(&ir->rc->dev,
+				"ir hauppauge (rc6-6a-32): vendor=%d dev=%d code=%d\n",
+				vendor, dev, code);
 		}
 
 		*scancode = RC_SCANCODE_RC6_6A(vendor, dev, code);
@@ -162,7 +172,7 @@ static int get_key_pixelview(struct IR_i2c *ir, enum rc_proto *protocol,
 
 	/* poll IR chip */
 	if (1 != i2c_master_recv(ir->c, &b, 1)) {
-		dprintk(1,"read error\n");
+		dev_dbg(&ir->rc->dev, "read error\n");
 		return -EIO;
 	}
 
@@ -179,13 +189,12 @@ static int get_key_fusionhdtv(struct IR_i2c *ir, enum rc_proto *protocol,
 
 	/* poll IR chip */
 	if (4 != i2c_master_recv(ir->c, buf, 4)) {
-		dprintk(1,"read error\n");
+		dev_dbg(&ir->rc->dev, "read error\n");
 		return -EIO;
 	}
 
-	if(buf[0] !=0 || buf[1] !=0 || buf[2] !=0 || buf[3] != 0)
-		dprintk(2, "%s: 0x%2x 0x%2x 0x%2x 0x%2x\n", __func__,
-			buf[0], buf[1], buf[2], buf[3]);
+	if (buf[0] != 0 || buf[1] != 0 || buf[2] != 0 || buf[3] != 0)
+		dev_dbg(&ir->rc->dev, "%s: %*ph\n", __func__, 4, buf);
 
 	/* no key pressed or signal from other ir remote */
 	if(buf[0] != 0x1 ||  buf[1] != 0xfe)
@@ -204,7 +213,7 @@ static int get_key_knc1(struct IR_i2c *ir, enum rc_proto *protocol,
 
 	/* poll IR chip */
 	if (1 != i2c_master_recv(ir->c, &b, 1)) {
-		dprintk(1,"read error\n");
+		dev_dbg(&ir->rc->dev, "read error\n");
 		return -EIO;
 	}
 
@@ -212,7 +221,7 @@ static int get_key_knc1(struct IR_i2c *ir, enum rc_proto *protocol,
 	   down, while 0xff indicates that no button is hold
 	   down. 0xfe sequences are sometimes interrupted by 0xFF */
 
-	dprintk(2,"key %02x\n", b);
+	dev_dbg(&ir->rc->dev, "key %02x\n", b);
 
 	if (b == 0xff)
 		return 0;
@@ -237,7 +246,7 @@ static int get_key_avermedia_cardbus(struct IR_i2c *ir, enum rc_proto *protocol,
 				  .buf = &key, .len = 1} };
 	subaddr = 0x0d;
 	if (2 != i2c_transfer(ir->c->adapter, msg, 2)) {
-		dprintk(1, "read error\n");
+		dev_dbg(&ir->rc->dev, "read error\n");
 		return -EIO;
 	}
 
@@ -247,18 +256,17 @@ static int get_key_avermedia_cardbus(struct IR_i2c *ir, enum rc_proto *protocol,
 	subaddr = 0x0b;
 	msg[1].buf = &keygroup;
 	if (2 != i2c_transfer(ir->c->adapter, msg, 2)) {
-		dprintk(1, "read error\n");
+		dev_dbg(&ir->rc->dev, "read error\n");
 		return -EIO;
 	}
 
 	if (keygroup == 0xff)
 		return 0;
 
-	dprintk(1, "read key 0x%02x/0x%02x\n", key, keygroup);
+	dev_dbg(&ir->rc->dev, "read key 0x%02x/0x%02x\n", key, keygroup);
 	if (keygroup < 2 || keygroup > 4) {
-		/* Only a warning */
-		dprintk(1, "warning: invalid key group 0x%02x for key 0x%02x\n",
-								keygroup, key);
+		dev_warn(&ir->rc->dev, "warning: invalid key group 0x%02x for key 0x%02x\n",
+			 keygroup, key);
 	}
 	key |= (keygroup & 1) << 6;
 
@@ -279,15 +287,15 @@ static int ir_key_poll(struct IR_i2c *ir)
 	u8 toggle;
 	int rc;
 
-	dprintk(3, "%s\n", __func__);
+	dev_dbg(&ir->rc->dev, "%s\n", __func__);
 	rc = ir->get_key(ir, &protocol, &scancode, &toggle);
 	if (rc < 0) {
-		dprintk(2,"error\n");
+		dev_warn(&ir->rc->dev, "error %d\n", rc);
 		return rc;
 	}
 
 	if (rc) {
-		dprintk(1, "%s: proto = 0x%04x, scancode = 0x%08x\n",
+		dev_dbg(&ir->rc->dev, "%s: proto = 0x%04x, scancode = 0x%08x\n",
 			__func__, protocol, scancode);
 		rc_keydown(ir->rc, protocol, scancode, toggle);
 	}
@@ -299,17 +307,416 @@ static void ir_work(struct work_struct *work)
 	int rc;
 	struct IR_i2c *ir = container_of(work, struct IR_i2c, work.work);
 
-	rc = ir_key_poll(ir);
-	if (rc == -ENODEV) {
-		rc_unregister_device(ir->rc);
-		ir->rc = NULL;
-		return;
+	/*
+	 * If the transmit code is holding the lock, skip polling for
+	 * IR, we'll get it to it next time round
+	 */
+	if (mutex_trylock(&ir->lock)) {
+		rc = ir_key_poll(ir);
+		mutex_unlock(&ir->lock);
+		if (rc == -ENODEV) {
+			rc_unregister_device(ir->rc);
+			ir->rc = NULL;
+			return;
+		}
 	}
 
 	schedule_delayed_work(&ir->work, msecs_to_jiffies(ir->polling_interval));
 }
 
-/* ----------------------------------------------------------------------- */
+static int ir_open(struct rc_dev *dev)
+{
+	struct IR_i2c *ir = dev->priv;
+
+	schedule_delayed_work(&ir->work, 0);
+
+	return 0;
+}
+
+static void ir_close(struct rc_dev *dev)
+{
+	struct IR_i2c *ir = dev->priv;
+
+	cancel_delayed_work_sync(&ir->work);
+}
+
+/* Zilog Transmit Interface */
+#define XTAL_FREQ		18432000
+
+#define ZILOG_SEND		0x80
+#define ZILOG_UIR_END		0x40
+#define ZILOG_INIT_END		0x20
+#define ZILOG_LIR_END		0x10
+
+#define ZILOG_STATUS_OK		0x80
+#define ZILOG_STATUS_TX		0x40
+#define ZILOG_STATUS_SET	0x20
+
+/*
+ * As you can see here, very few different lengths of pulse and space
+ * can be encoded. This means that the hardware does not work well with
+ * recorded IR. It's best to work with generated IR, like from ir-ctl or
+ * the in-kernel encoders.
+ */
+struct code_block {
+	u8	length;
+	u16	pulse[7];	/* not aligned */
+	u8	carrier_pulse;
+	u8	carrier_space;
+	u16	space[8];	/* not aligned */
+	u8	codes[61];
+	u8	csum[2];
+} __packed;
+
+static int send_data_block(struct IR_i2c *ir, int cmd,
+			   struct code_block *code_block)
+{
+	int i, j, ret;
+	u8 buf[5], *p;
+
+	p = &code_block->length;
+	for (i = 0; p < code_block->csum; i++)
+		code_block->csum[i & 1] ^= *p++;
+
+	p = &code_block->length;
+
+	for (i = 0; i < sizeof(*code_block);) {
+		int tosend = sizeof(*code_block) - i;
+
+		if (tosend > 4)
+			tosend = 4;
+		buf[0] = i + 1;
+		for (j = 0; j < tosend; ++j)
+			buf[1 + j] = p[i + j];
+		dev_dbg(&ir->rc->dev, "%*ph", tosend + 1, buf);
+		ret = i2c_master_send(ir->tx_c, buf, tosend + 1);
+		if (ret != tosend + 1) {
+			dev_dbg(&ir->rc->dev,
+				"i2c_master_send failed with %d\n", ret);
+			return ret < 0 ? ret : -EIO;
+		}
+		i += tosend;
+	}
+
+	buf[0] = 0;
+	buf[1] = cmd;
+	ret = i2c_master_send(ir->tx_c, buf, 2);
+	if (ret != 2) {
+		dev_err(&ir->rc->dev, "i2c_master_send failed with %d\n", ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	usleep_range(2000, 5000);
+
+	ret = i2c_master_send(ir->tx_c, buf, 1);
+	if (ret != 1) {
+		dev_err(&ir->rc->dev, "i2c_master_send failed with %d\n", ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	return 0;
+}
+
+static int zilog_init(struct IR_i2c *ir)
+{
+	struct code_block code_block = { .length = sizeof(code_block) };
+	u8 buf[4];
+	int ret;
+
+	put_unaligned_be16(0x1000, &code_block.pulse[3]);
+
+	ret = send_data_block(ir, ZILOG_INIT_END, &code_block);
+	if (ret)
+		return ret;
+
+	ret = i2c_master_recv(ir->tx_c, buf, 4);
+	if (ret != 4) {
+		dev_err(&ir->c->dev, "failed to retrieve firmware version: %d\n",
+			ret);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	dev_info(&ir->c->dev, "Zilog/Hauppauge IR blaster firmware version %d.%d.%d\n",
+		 buf[1], buf[2], buf[3]);
+
+	return 0;
+}
+
+/*
+ * If the last slot for pulse is the same as the current slot for pulse,
+ * then use slot no 7.
+ */
+static void copy_codes(u8 *dst, u8 *src, unsigned int count)
+{
+	u8 c, last = 0xff;
+
+	while (count--) {
+		c = *src++;
+		if ((c & 0xf0) == last) {
+			*dst++ = 0x70 | (c & 0xf);
+		} else {
+			*dst++ = c;
+			last = c & 0xf0;
+		}
+	}
+}
+
+/*
+ * When looking for repeats, we don't care about the trailing space. This
+ * is set to the shortest possible anyway.
+ */
+static int cmp_no_trail(u8 *a, u8 *b, unsigned int count)
+{
+	while (--count) {
+		if (*a++ != *b++)
+			return 1;
+	}
+
+	return (*a & 0xf0) - (*b & 0xf0);
+}
+
+static int find_slot(u16 *array, unsigned int size, u16 val)
+{
+	int i;
+
+	for (i = 0; i < size; i++) {
+		if (get_unaligned_be16(&array[i]) == val) {
+			return i;
+		} else if (!array[i]) {
+			put_unaligned_be16(val, &array[i]);
+			return i;
+		}
+	}
+
+	return -1;
+}
+
+static int zilog_ir_format(struct rc_dev *rcdev, unsigned int *txbuf,
+			   unsigned int count, struct code_block *code_block)
+{
+	struct IR_i2c *ir = rcdev->priv;
+	int rep, i, l, p = 0, s, c = 0;
+	bool repeating;
+	u8 codes[174];
+
+	code_block->carrier_pulse = DIV_ROUND_CLOSEST(
+			ir->duty_cycle * XTAL_FREQ / 1000, ir->carrier);
+	code_block->carrier_space = DIV_ROUND_CLOSEST(
+			(100 - ir->duty_cycle) * XTAL_FREQ / 1000, ir->carrier);
+
+	for (i = 0; i < count; i++) {
+		if (c >= ARRAY_SIZE(codes) - 1) {
+			dev_warn(&rcdev->dev, "IR too long, cannot transmit\n");
+			return -EINVAL;
+		}
+
+		/*
+		 * Lengths more than 142220us cannot be encoded; also
+		 * this checks for multiply overflow
+		 */
+		if (txbuf[i] > 142220)
+			return -EINVAL;
+
+		l = DIV_ROUND_CLOSEST((XTAL_FREQ / 1000) * txbuf[i], 40000);
+
+		if (i & 1) {
+			s = find_slot(code_block->space,
+				      ARRAY_SIZE(code_block->space), l);
+			if (s == -1) {
+				dev_warn(&rcdev->dev, "Too many different lengths spaces, cannot transmit");
+				return -EINVAL;
+			}
+
+			/* We have a pulse and space */
+			codes[c++] = (p << 4) | s;
+		} else {
+			p = find_slot(code_block->pulse,
+				      ARRAY_SIZE(code_block->pulse), l);
+			if (p == -1) {
+				dev_warn(&rcdev->dev, "Too many different lengths pulses, cannot transmit");
+				return -EINVAL;
+			}
+		}
+	}
+
+	/* We have to encode the trailing pulse. Find the shortest space */
+	s = 0;
+	for (i = 1; i < ARRAY_SIZE(code_block->space); i++) {
+		u16 d = get_unaligned_be16(&code_block->space[i]);
+
+		if (get_unaligned_be16(&code_block->space[s]) > d)
+			s = i;
+	}
+
+	codes[c++] = (p << 4) | s;
+
+	dev_dbg(&rcdev->dev, "generated %d codes\n", c);
+
+	/*
+	 * Are the last N codes (so pulse + space) repeating 3 times?
+	 * if so we can shorten the codes list and use code 0xc0 to repeat
+	 * them.
+	 */
+	repeating = false;
+
+	for (rep = c / 3; rep >= 1; rep--) {
+		if (!memcmp(&codes[c - rep * 3], &codes[c - rep * 2], rep) &&
+		    !cmp_no_trail(&codes[c - rep], &codes[c - rep * 2], rep)) {
+			repeating = true;
+			break;
+		}
+	}
+
+	if (repeating) {
+		/* first copy any leading non-repeating */
+		int leading = c - rep * 3;
+
+		if (leading + rep >= ARRAY_SIZE(code_block->codes) - 3) {
+			dev_warn(&rcdev->dev, "IR too long, cannot transmit\n");
+			return -EINVAL;
+		}
+
+		dev_dbg(&rcdev->dev, "found trailing %d repeat\n", rep);
+		copy_codes(code_block->codes, codes, leading);
+		code_block->codes[leading] = 0x82;
+		copy_codes(code_block->codes + leading + 1, codes + leading,
+			   rep);
+		c = leading + 1 + rep;
+		code_block->codes[c++] = 0xc0;
+	} else {
+		if (c >= ARRAY_SIZE(code_block->codes) - 3) {
+			dev_warn(&rcdev->dev, "IR too long, cannot transmit\n");
+			return -EINVAL;
+		}
+
+		dev_dbg(&rcdev->dev, "found no trailing repeat\n");
+		code_block->codes[0] = 0x82;
+		copy_codes(code_block->codes + 1, codes, c);
+		c++;
+		code_block->codes[c++] = 0xc4;
+	}
+
+	while (c < ARRAY_SIZE(code_block->codes))
+		code_block->codes[c++] = 0x83;
+
+	return 0;
+}
+
+static int zilog_tx(struct rc_dev *rcdev, unsigned int *txbuf,
+		    unsigned int count)
+{
+	struct IR_i2c *ir = rcdev->priv;
+	struct code_block code_block = { .length = sizeof(code_block) };
+	u8 buf[2];
+	int ret, i;
+
+	ret = zilog_ir_format(rcdev, txbuf, count, &code_block);
+	if (ret)
+		return ret;
+
+	ret = mutex_lock_interruptible(&ir->lock);
+	if (ret)
+		return ret;
+
+	ret = send_data_block(ir, ZILOG_UIR_END, &code_block);
+	if (ret)
+		goto out_unlock;
+
+	ret = i2c_master_recv(ir->tx_c, buf, 1);
+	if (ret != 1) {
+		dev_err(&ir->rc->dev, "i2c_master_recv failed with %d\n", ret);
+		goto out_unlock;
+	}
+
+	dev_dbg(&ir->rc->dev, "code set status: %02x\n", buf[0]);
+
+	if (buf[0] != (ZILOG_STATUS_OK | ZILOG_STATUS_SET)) {
+		dev_err(&ir->rc->dev, "unexpected IR TX response %02x\n",
+			buf[0]);
+		ret = -EIO;
+		goto out_unlock;
+	}
+
+	buf[0] = 0x00;
+	buf[1] = ZILOG_SEND;
+
+	ret = i2c_master_send(ir->tx_c, buf, 2);
+	if (ret != 2) {
+		dev_err(&ir->rc->dev, "i2c_master_send failed with %d\n", ret);
+		if (ret >= 0)
+			ret = -EIO;
+		goto out_unlock;
+	}
+
+	dev_dbg(&ir->rc->dev, "send command sent\n");
+
+	/*
+	 * This bit NAKs until the device is ready, so we retry it
+	 * sleeping a bit each time.  This seems to be what the windows
+	 * driver does, approximately.
+	 * Try for up to 1s.
+	 */
+	for (i = 0; i < 20; ++i) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_timeout(msecs_to_jiffies(50));
+		ret = i2c_master_send(ir->tx_c, buf, 1);
+		if (ret == 1)
+			break;
+		dev_dbg(&ir->rc->dev,
+			"NAK expected: i2c_master_send failed with %d (try %d)\n",
+			ret, i + 1);
+	}
+
+	if (ret != 1) {
+		dev_err(&ir->rc->dev,
+			"IR TX chip never got ready: last i2c_master_send failed with %d\n",
+			ret);
+		if (ret >= 0)
+			ret = -EIO;
+		goto out_unlock;
+	}
+
+	i = i2c_master_recv(ir->tx_c, buf, 1);
+	if (i != 1) {
+		dev_err(&ir->rc->dev, "i2c_master_recv failed with %d\n", ret);
+		ret = -EIO;
+		goto out_unlock;
+	} else if (buf[0] != ZILOG_STATUS_OK) {
+		dev_err(&ir->rc->dev, "unexpected IR TX response #2: %02x\n",
+			buf[0]);
+		ret = -EIO;
+		goto out_unlock;
+	}
+	dev_dbg(&ir->rc->dev, "transmit complete\n");
+
+	/* Oh good, it worked */
+	ret = count;
+out_unlock:
+	mutex_unlock(&ir->lock);
+
+	return ret;
+}
+
+static int zilog_tx_carrier(struct rc_dev *dev, u32 carrier)
+{
+	struct IR_i2c *ir = dev->priv;
+
+	if (carrier > 500000 || carrier < 20000)
+		return -EINVAL;
+
+	ir->carrier = carrier;
+
+	return 0;
+}
+
+static int zilog_tx_duty_cycle(struct rc_dev *dev, u32 duty_cycle)
+{
+	struct IR_i2c *ir = dev->priv;
+
+	ir->duty_cycle = duty_cycle;
+
+	return 0;
+}
 
 static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
@@ -322,6 +729,11 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	unsigned short addr = client->addr;
 	int err;
 
+	if ((id->driver_data & FLAG_HDPVR) && !enable_hdpvr) {
+		dev_err(&client->dev, "IR for HDPVR is known to cause problems during recording, use enable_hdpvr modparam to enable\n");
+		return -ENODEV;
+	}
+
 	ir = devm_kzalloc(&client->dev, sizeof(*ir), GFP_KERNEL);
 	if (!ir)
 		return -ENOMEM;
@@ -433,18 +845,15 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 
 	/* Make sure we are all setup before going on */
 	if (!name || !ir->get_key || !rc_proto || !ir_codes) {
-		dprintk(1, ": Unsupported device at address 0x%02x\n",
-			addr);
+		dev_warn(&client->dev, "Unsupported device at address 0x%02x\n",
+			 addr);
 		err = -ENODEV;
 		goto err_out_free;
 	}
 
-	/* Sets name */
-	snprintf(ir->name, sizeof(ir->name), "i2c IR (%s)", name);
 	ir->ir_codes = ir_codes;
 
-	snprintf(ir->phys, sizeof(ir->phys), "%s/%s/ir0",
-		 dev_name(&adap->dev),
+	snprintf(ir->phys, sizeof(ir->phys), "%s/%s", dev_name(&adap->dev),
 		 dev_name(&client->dev));
 
 	/*
@@ -453,7 +862,11 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	 */
 	rc->input_id.bustype = BUS_I2C;
 	rc->input_phys       = ir->phys;
-	rc->device_name	     = ir->name;
+	rc->device_name	     = name;
+	rc->dev.parent       = &client->dev;
+	rc->priv             = ir;
+	rc->open             = ir_open;
+	rc->close            = ir_close;
 
 	/*
 	 * Initialize the other fields of rc_dev
@@ -461,22 +874,35 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	rc->map_name       = ir->ir_codes;
 	rc->allowed_protocols = rc_proto;
 	if (!rc->driver_name)
-		rc->driver_name = MODULE_NAME;
+		rc->driver_name = KBUILD_MODNAME;
+
+	mutex_init(&ir->lock);
+
+	INIT_DELAYED_WORK(&ir->work, ir_work);
+
+	if (id->driver_data & FLAG_TX) {
+		ir->tx_c = i2c_new_dummy(client->adapter, 0x70);
+		if (!ir->tx_c) {
+			dev_err(&client->dev, "failed to setup tx i2c address");
+		} else if (!zilog_init(ir)) {
+			ir->carrier = 38000;
+			ir->duty_cycle = 40;
+			rc->tx_ir = zilog_tx;
+			rc->s_tx_carrier = zilog_tx_carrier;
+			rc->s_tx_duty_cycle = zilog_tx_duty_cycle;
+		}
+	}
 
 	err = rc_register_device(rc);
 	if (err)
 		goto err_out_free;
 
-	printk(MODULE_NAME ": %s detected at %s [%s]\n",
-	       ir->name, ir->phys, adap->name);
-
-	/* start polling via eventd */
-	INIT_DELAYED_WORK(&ir->work, ir_work);
-	schedule_delayed_work(&ir->work, 0);
-
 	return 0;
 
  err_out_free:
+	if (ir->tx_c)
+		i2c_unregister_device(ir->tx_c);
+
 	/* Only frees rc if it were allocated internally */
 	rc_free_device(rc);
 	return err;
@@ -489,6 +915,9 @@ static int ir_remove(struct i2c_client *client)
 	/* kill outstanding polls */
 	cancel_delayed_work_sync(&ir->work);
 
+	if (ir->tx_c)
+		i2c_unregister_device(ir->tx_c);
+
 	/* unregister device */
 	rc_unregister_device(ir->rc);
 
@@ -500,10 +929,11 @@ static const struct i2c_device_id ir_kbd_id[] = {
 	/* Generic entry for any IR receiver */
 	{ "ir_video", 0 },
 	/* IR device specific entries should be added here */
-	{ "ir_rx_z8f0811_haup", 0 },
-	{ "ir_rx_z8f0811_hdpvr", 0 },
+	{ "ir_z8f0811_haup", FLAG_TX },
+	{ "ir_z8f0811_hdpvr", FLAG_TX | FLAG_HDPVR },
 	{ }
 };
+MODULE_DEVICE_TABLE(i2c, ir_kbd_id);
 
 static struct i2c_driver ir_kbd_driver = {
 	.driver = {
diff --git a/drivers/media/i2c/ks0127.c b/drivers/media/i2c/ks0127.c
index ab536c4a7115..5905ed6f8397 100644
--- a/drivers/media/i2c/ks0127.c
+++ b/drivers/media/i2c/ks0127.c
@@ -195,7 +195,7 @@ struct adjust {
 struct ks0127 {
 	struct v4l2_subdev sd;
 	v4l2_std_id	norm;
-	u8 		regs[256];
+	u8		regs[256];
 };
 
 static inline struct ks0127 *to_ks0127(struct v4l2_subdev *sd)
diff --git a/drivers/media/i2c/mt9m111.c b/drivers/media/i2c/mt9m111.c
index b1665d97e0fd..efda1aa95ca0 100644
--- a/drivers/media/i2c/mt9m111.c
+++ b/drivers/media/i2c/mt9m111.c
@@ -92,6 +92,7 @@
  */
 #define MT9M111_OPER_MODE_CTRL		0x106
 #define MT9M111_OUTPUT_FORMAT_CTRL	0x108
+#define MT9M111_TPG_CTRL		0x148
 #define MT9M111_REDUCER_XZOOM_B		0x1a0
 #define MT9M111_REDUCER_XSIZE_B		0x1a1
 #define MT9M111_REDUCER_YZOOM_B		0x1a3
@@ -124,6 +125,7 @@
 #define MT9M111_OUTFMT_AVG_CHROMA	(1 << 2)
 #define MT9M111_OUTFMT_SWAP_YCbCr_C_Y_RGB_EVEN	(1 << 1)
 #define MT9M111_OUTFMT_SWAP_YCbCr_Cb_Cr_RGB_R_B	(1 << 0)
+#define MT9M111_TPG_SEL_MASK		GENMASK(2, 0)
 
 /*
  * Camera control register addresses (0x200..0x2ff not implemented)
@@ -215,6 +217,9 @@ struct mt9m111 {
 	int power_count;
 	const struct mt9m111_datafmt *fmt;
 	int lastpage;	/* PageMap cache value */
+#ifdef CONFIG_MEDIA_CONTROLLER
+	struct media_pad pad;
+#endif
 };
 
 /* Find a data format by a pixel code */
@@ -703,6 +708,25 @@ static int mt9m111_set_autowhitebalance(struct mt9m111 *mt9m111, int on)
 	return reg_clear(OPER_MODE_CTRL, MT9M111_OPMODE_AUTOWHITEBAL_EN);
 }
 
+static const char * const mt9m111_test_pattern_menu[] = {
+	"Disabled",
+	"Vertical monochrome gradient",
+	"Flat color type 1",
+	"Flat color type 2",
+	"Flat color type 3",
+	"Flat color type 4",
+	"Flat color type 5",
+	"Color bar",
+};
+
+static int mt9m111_set_test_pattern(struct mt9m111 *mt9m111, int val)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&mt9m111->subdev);
+
+	return mt9m111_reg_mask(client, MT9M111_TPG_CTRL, val,
+				MT9M111_TPG_SEL_MASK);
+}
+
 static int mt9m111_s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct mt9m111 *mt9m111 = container_of(ctrl->handler,
@@ -721,6 +745,8 @@ static int mt9m111_s_ctrl(struct v4l2_ctrl *ctrl)
 		return mt9m111_set_autoexposure(mt9m111, ctrl->val);
 	case V4L2_CID_AUTO_WHITE_BALANCE:
 		return mt9m111_set_autowhitebalance(mt9m111, ctrl->val);
+	case V4L2_CID_TEST_PATTERN:
+		return mt9m111_set_test_pattern(mt9m111, ctrl->val);
 	}
 
 	return -EINVAL;
@@ -951,6 +977,8 @@ static int mt9m111_probe(struct i2c_client *client,
 	mt9m111->ctx = &context_b;
 
 	v4l2_i2c_subdev_init(&mt9m111->subdev, client, &mt9m111_subdev_ops);
+	mt9m111->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+
 	v4l2_ctrl_handler_init(&mt9m111->hdl, 5);
 	v4l2_ctrl_new_std(&mt9m111->hdl, &mt9m111_ctrl_ops,
 			V4L2_CID_VFLIP, 0, 1, 1, 0);
@@ -963,12 +991,24 @@ static int mt9m111_probe(struct i2c_client *client,
 	v4l2_ctrl_new_std_menu(&mt9m111->hdl,
 			&mt9m111_ctrl_ops, V4L2_CID_EXPOSURE_AUTO, 1, 0,
 			V4L2_EXPOSURE_AUTO);
+	v4l2_ctrl_new_std_menu_items(&mt9m111->hdl,
+			&mt9m111_ctrl_ops, V4L2_CID_TEST_PATTERN,
+			ARRAY_SIZE(mt9m111_test_pattern_menu) - 1, 0, 0,
+			mt9m111_test_pattern_menu);
 	mt9m111->subdev.ctrl_handler = &mt9m111->hdl;
 	if (mt9m111->hdl.error) {
 		ret = mt9m111->hdl.error;
 		goto out_clkput;
 	}
 
+#ifdef CONFIG_MEDIA_CONTROLLER
+	mt9m111->pad.flags = MEDIA_PAD_FL_SOURCE;
+	mt9m111->subdev.entity.function = MEDIA_ENT_F_CAM_SENSOR;
+	ret = media_entity_pads_init(&mt9m111->subdev.entity, 1, &mt9m111->pad);
+	if (ret < 0)
+		goto out_hdlfree;
+#endif
+
 	/* Second stage probe - when a capture adapter is there */
 	mt9m111->rect.left	= MT9M111_MIN_DARK_COLS;
 	mt9m111->rect.top	= MT9M111_MIN_DARK_ROWS;
@@ -980,16 +1020,20 @@ static int mt9m111_probe(struct i2c_client *client,
 
 	ret = mt9m111_video_probe(client);
 	if (ret < 0)
-		goto out_hdlfree;
+		goto out_entityclean;
 
 	mt9m111->subdev.dev = &client->dev;
 	ret = v4l2_async_register_subdev(&mt9m111->subdev);
 	if (ret < 0)
-		goto out_hdlfree;
+		goto out_entityclean;
 
 	return 0;
 
+out_entityclean:
+#ifdef CONFIG_MEDIA_CONTROLLER
+	media_entity_cleanup(&mt9m111->subdev.entity);
 out_hdlfree:
+#endif
 	v4l2_ctrl_handler_free(&mt9m111->hdl);
 out_clkput:
 	v4l2_clk_put(mt9m111->clk);
@@ -1002,6 +1046,7 @@ static int mt9m111_remove(struct i2c_client *client)
 	struct mt9m111 *mt9m111 = to_mt9m111(client);
 
 	v4l2_async_unregister_subdev(&mt9m111->subdev);
+	media_entity_cleanup(&mt9m111->subdev.entity);
 	v4l2_clk_put(mt9m111->clk);
 	v4l2_ctrl_handler_free(&mt9m111->hdl);
 
diff --git a/drivers/media/i2c/mt9v011.c b/drivers/media/i2c/mt9v011.c
index 9ed1b26b6549..5e29064fae91 100644
--- a/drivers/media/i2c/mt9v011.c
+++ b/drivers/media/i2c/mt9v011.c
@@ -1,9 +1,8 @@
-/*
- * mt9v011 -Micron 1/4-Inch VGA Digital Image Sensor
- *
- * Copyright (c) 2009 Mauro Carvalho Chehab
- * This code is placed under the terms of the GNU General Public License v2
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// mt9v011 -Micron 1/4-Inch VGA Digital Image Sensor
+//
+// Copyright (c) 2009 Mauro Carvalho Chehab <mchehab@kernel.org>
 
 #include <linux/i2c.h>
 #include <linux/slab.h>
@@ -17,7 +16,7 @@
 
 MODULE_DESCRIPTION("Micron mt9v011 sensor driver");
 MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
 
 static int debug;
 module_param(debug, int, 0);
diff --git a/drivers/media/i2c/mt9v032.c b/drivers/media/i2c/mt9v032.c
index 8a430640c85d..4de63b2df334 100644
--- a/drivers/media/i2c/mt9v032.c
+++ b/drivers/media/i2c/mt9v032.c
@@ -294,14 +294,22 @@ static int mt9v032_power_on(struct mt9v032 *mt9v032)
 	/* Reset the chip and stop data read out */
 	ret = regmap_write(map, MT9V032_RESET, 1);
 	if (ret < 0)
-		return ret;
+		goto err;
 
 	ret = regmap_write(map, MT9V032_RESET, 0);
 	if (ret < 0)
-		return ret;
+		goto err;
+
+	ret = regmap_write(map, MT9V032_CHIP_CONTROL,
+			   MT9V032_CHIP_CONTROL_MASTER_MODE);
+	if (ret < 0)
+		goto err;
+
+	return 0;
 
-	return regmap_write(map, MT9V032_CHIP_CONTROL,
-			    MT9V032_CHIP_CONTROL_MASTER_MODE);
+err:
+	clk_disable_unprepare(mt9v032->clk);
+	return ret;
 }
 
 static void mt9v032_power_off(struct mt9v032 *mt9v032)
@@ -876,6 +884,9 @@ static int mt9v032_registered(struct v4l2_subdev *subdev)
 
 	/* Read and check the sensor version */
 	ret = regmap_read(mt9v032->regmap, MT9V032_CHIP_VERSION, &version);
+
+	mt9v032_power_off(mt9v032);
+
 	if (ret < 0) {
 		dev_err(&client->dev, "Failed reading chip version\n");
 		return ret;
@@ -894,8 +905,6 @@ static int mt9v032_registered(struct v4l2_subdev *subdev)
 		return -ENODEV;
 	}
 
-	mt9v032_power_off(mt9v032);
-
 	dev_info(&client->dev, "%s detected at address 0x%02x\n",
 		 mt9v032->version->name, client->addr);
 
diff --git a/drivers/media/i2c/ov2640.c b/drivers/media/i2c/ov2640.c
index 518868388d65..4c3b92763243 100644
--- a/drivers/media/i2c/ov2640.c
+++ b/drivers/media/i2c/ov2640.c
@@ -1147,9 +1147,7 @@ static int ov2640_probe(struct i2c_client *client,
 	return 0;
 
 err_videoprobe:
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	media_entity_cleanup(&priv->subdev.entity);
-#endif
 err_hdl:
 	v4l2_ctrl_handler_free(&priv->hdl);
 err_clk:
@@ -1163,9 +1161,7 @@ static int ov2640_remove(struct i2c_client *client)
 
 	v4l2_async_unregister_subdev(&priv->subdev);
 	v4l2_ctrl_handler_free(&priv->hdl);
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	media_entity_cleanup(&priv->subdev.entity);
-#endif
 	v4l2_device_unregister_subdev(&priv->subdev);
 	clk_disable_unprepare(priv->clk);
 	return 0;
diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c
index 122dd6c5eb38..4715edc8ca33 100644
--- a/drivers/media/i2c/ov2659.c
+++ b/drivers/media/i2c/ov2659.c
@@ -1474,9 +1474,7 @@ static int ov2659_probe(struct i2c_client *client,
 
 error:
 	v4l2_ctrl_handler_free(&ov2659->ctrls);
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	media_entity_cleanup(&sd->entity);
-#endif
 	mutex_destroy(&ov2659->lock);
 	return ret;
 }
@@ -1488,9 +1486,7 @@ static int ov2659_remove(struct i2c_client *client)
 
 	v4l2_ctrl_handler_free(&ov2659->ctrls);
 	v4l2_async_unregister_subdev(sd);
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	media_entity_cleanup(&sd->entity);
-#endif
 	mutex_destroy(&ov2659->lock);
 
 	return 0;
diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
index c89ed6609738..e2dd352224c7 100644
--- a/drivers/media/i2c/ov5640.c
+++ b/drivers/media/i2c/ov5640.c
@@ -34,13 +34,19 @@
 
 #define OV5640_DEFAULT_SLAVE_ID 0x3c
 
+#define OV5640_REG_SYS_CTRL0		0x3008
 #define OV5640_REG_CHIP_ID		0x300a
+#define OV5640_REG_IO_MIPI_CTRL00	0x300e
+#define OV5640_REG_PAD_OUTPUT_ENABLE01	0x3017
+#define OV5640_REG_PAD_OUTPUT_ENABLE02	0x3018
 #define OV5640_REG_PAD_OUTPUT00		0x3019
+#define OV5640_REG_SYSTEM_CONTROL1	0x302e
 #define OV5640_REG_SC_PLL_CTRL0		0x3034
 #define OV5640_REG_SC_PLL_CTRL1		0x3035
 #define OV5640_REG_SC_PLL_CTRL2		0x3036
 #define OV5640_REG_SC_PLL_CTRL3		0x3037
 #define OV5640_REG_SLAVE_ID		0x3100
+#define OV5640_REG_SCCB_SYS_CTRL1	0x3103
 #define OV5640_REG_SYS_ROOT_DIVIDER	0x3108
 #define OV5640_REG_AWB_R_GAIN		0x3400
 #define OV5640_REG_AWB_G_GAIN		0x3402
@@ -70,8 +76,11 @@
 #define OV5640_REG_HZ5060_CTRL01	0x3c01
 #define OV5640_REG_SIGMADELTA_CTRL0C	0x3c0c
 #define OV5640_REG_FRAME_CTRL01		0x4202
+#define OV5640_REG_FORMAT_CONTROL00	0x4300
+#define OV5640_REG_POLARITY_CTRL00	0x4740
 #define OV5640_REG_MIPI_CTRL00		0x4800
 #define OV5640_REG_DEBUG_MODE		0x4814
+#define OV5640_REG_ISP_FORMAT_MUX_CTRL	0x501f
 #define OV5640_REG_PRE_ISP_TEST_SET1	0x503d
 #define OV5640_REG_SDE_CTRL0		0x5580
 #define OV5640_REG_SDE_CTRL1		0x5581
@@ -99,6 +108,18 @@ enum ov5640_frame_rate {
 	OV5640_NUM_FRAMERATES,
 };
 
+struct ov5640_pixfmt {
+	u32 code;
+	u32 colorspace;
+};
+
+static const struct ov5640_pixfmt ov5640_formats[] = {
+	{ MEDIA_BUS_FMT_UYVY8_2X8, V4L2_COLORSPACE_SRGB, },
+	{ MEDIA_BUS_FMT_YUYV8_2X8, V4L2_COLORSPACE_SRGB, },
+	{ MEDIA_BUS_FMT_RGB565_2X8_LE, V4L2_COLORSPACE_SRGB, },
+	{ MEDIA_BUS_FMT_RGB565_2X8_BE, V4L2_COLORSPACE_SRGB, },
+};
+
 /*
  * FIXME: remove this when a subdev API becomes available
  * to set the MIPI CSI-2 virtual channel.
@@ -982,7 +1003,111 @@ static int ov5640_get_gain(struct ov5640_dev *sensor)
 	return gain & 0x3ff;
 }
 
-static int ov5640_set_stream(struct ov5640_dev *sensor, bool on)
+static int ov5640_set_stream_dvp(struct ov5640_dev *sensor, bool on)
+{
+	int ret;
+	unsigned int flags = sensor->ep.bus.parallel.flags;
+	u8 pclk_pol = 0;
+	u8 hsync_pol = 0;
+	u8 vsync_pol = 0;
+
+	/*
+	 * Note about parallel port configuration.
+	 *
+	 * When configured in parallel mode, the OV5640 will
+	 * output 10 bits data on DVP data lines [9:0].
+	 * If only 8 bits data are wanted, the 8 bits data lines
+	 * of the camera interface must be physically connected
+	 * on the DVP data lines [9:2].
+	 *
+	 * Control lines polarity can be configured through
+	 * devicetree endpoint control lines properties.
+	 * If no endpoint control lines properties are set,
+	 * polarity will be as below:
+	 * - VSYNC:	active high
+	 * - HREF:	active low
+	 * - PCLK:	active low
+	 */
+
+	if (on) {
+		/*
+		 * reset MIPI PCLK/SERCLK divider
+		 *
+		 * SC PLL CONTRL1 0
+		 * - [3..0]:	MIPI PCLK/SERCLK divider
+		 */
+		ret = ov5640_mod_reg(sensor, OV5640_REG_SC_PLL_CTRL1, 0x0f, 0);
+		if (ret)
+			return ret;
+
+		/*
+		 * configure parallel port control lines polarity
+		 *
+		 * POLARITY CTRL0
+		 * - [5]:	PCLK polarity (0: active low, 1: active high)
+		 * - [1]:	HREF polarity (0: active low, 1: active high)
+		 * - [0]:	VSYNC polarity (mismatch here between
+		 *		datasheet and hardware, 0 is active high
+		 *		and 1 is active low...)
+		 */
+		if (flags & V4L2_MBUS_PCLK_SAMPLE_RISING)
+			pclk_pol = 1;
+		if (flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH)
+			hsync_pol = 1;
+		if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
+			vsync_pol = 1;
+
+		ret = ov5640_write_reg(sensor,
+				       OV5640_REG_POLARITY_CTRL00,
+				       (pclk_pol << 5) |
+				       (hsync_pol << 1) |
+				       vsync_pol);
+
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * powerdown MIPI TX/RX PHY & disable MIPI
+	 *
+	 * MIPI CONTROL 00
+	 * 4:	 PWDN PHY TX
+	 * 3:	 PWDN PHY RX
+	 * 2:	 MIPI enable
+	 */
+	ret = ov5640_write_reg(sensor,
+			       OV5640_REG_IO_MIPI_CTRL00, on ? 0x18 : 0);
+	if (ret)
+		return ret;
+
+	/*
+	 * enable VSYNC/HREF/PCLK DVP control lines
+	 * & D[9:6] DVP data lines
+	 *
+	 * PAD OUTPUT ENABLE 01
+	 * - 6:		VSYNC output enable
+	 * - 5:		HREF output enable
+	 * - 4:		PCLK output enable
+	 * - [3:0]:	D[9:6] output enable
+	 */
+	ret = ov5640_write_reg(sensor,
+			       OV5640_REG_PAD_OUTPUT_ENABLE01,
+			       on ? 0x7f : 0);
+	if (ret)
+		return ret;
+
+	/*
+	 * enable D[5:0] DVP data lines
+	 *
+	 * PAD OUTPUT ENABLE 02
+	 * - [7:2]:	D[5:0] output enable
+	 */
+	return ov5640_write_reg(sensor,
+				OV5640_REG_PAD_OUTPUT_ENABLE02,
+				on ? 0xfc : 0);
+}
+
+static int ov5640_set_stream_mipi(struct ov5640_dev *sensor, bool on)
 {
 	int ret;
 
@@ -1524,7 +1649,7 @@ static int ov5640_restore_mode(struct ov5640_dev *sensor)
 
 static void ov5640_power(struct ov5640_dev *sensor, bool enable)
 {
-	gpiod_set_value(sensor->pwdn_gpio, enable ? 0 : 1);
+	gpiod_set_value_cansleep(sensor->pwdn_gpio, enable ? 0 : 1);
 }
 
 static void ov5640_reset(struct ov5640_dev *sensor)
@@ -1532,7 +1657,7 @@ static void ov5640_reset(struct ov5640_dev *sensor)
 	if (!sensor->reset_gpio)
 		return;
 
-	gpiod_set_value(sensor->reset_gpio, 0);
+	gpiod_set_value_cansleep(sensor->reset_gpio, 0);
 
 	/* camera power cycle */
 	ov5640_power(sensor, false);
@@ -1540,56 +1665,89 @@ static void ov5640_reset(struct ov5640_dev *sensor)
 	ov5640_power(sensor, true);
 	usleep_range(5000, 10000);
 
-	gpiod_set_value(sensor->reset_gpio, 1);
+	gpiod_set_value_cansleep(sensor->reset_gpio, 1);
 	usleep_range(1000, 2000);
 
-	gpiod_set_value(sensor->reset_gpio, 0);
+	gpiod_set_value_cansleep(sensor->reset_gpio, 0);
 	usleep_range(5000, 10000);
 }
 
-static int ov5640_set_power(struct ov5640_dev *sensor, bool on)
+static int ov5640_set_power_on(struct ov5640_dev *sensor)
 {
-	int ret = 0;
+	struct i2c_client *client = sensor->i2c_client;
+	int ret;
 
-	if (on) {
-		clk_prepare_enable(sensor->xclk);
+	ret = clk_prepare_enable(sensor->xclk);
+	if (ret) {
+		dev_err(&client->dev, "%s: failed to enable clock\n",
+			__func__);
+		return ret;
+	}
 
-		ret = regulator_bulk_enable(OV5640_NUM_SUPPLIES,
-					    sensor->supplies);
-		if (ret)
-			goto xclk_off;
+	ret = regulator_bulk_enable(OV5640_NUM_SUPPLIES,
+				    sensor->supplies);
+	if (ret) {
+		dev_err(&client->dev, "%s: failed to enable regulators\n",
+			__func__);
+		goto xclk_off;
+	}
+
+	ov5640_reset(sensor);
+	ov5640_power(sensor, true);
 
-		ov5640_reset(sensor);
-		ov5640_power(sensor, true);
+	ret = ov5640_init_slave_id(sensor);
+	if (ret)
+		goto power_off;
 
-		ret = ov5640_init_slave_id(sensor);
+	return 0;
+
+power_off:
+	ov5640_power(sensor, false);
+	regulator_bulk_disable(OV5640_NUM_SUPPLIES, sensor->supplies);
+xclk_off:
+	clk_disable_unprepare(sensor->xclk);
+	return ret;
+}
+
+static void ov5640_set_power_off(struct ov5640_dev *sensor)
+{
+	ov5640_power(sensor, false);
+	regulator_bulk_disable(OV5640_NUM_SUPPLIES, sensor->supplies);
+	clk_disable_unprepare(sensor->xclk);
+}
+
+static int ov5640_set_power(struct ov5640_dev *sensor, bool on)
+{
+	int ret = 0;
+
+	if (on) {
+		ret = ov5640_set_power_on(sensor);
 		if (ret)
-			goto power_off;
+			return ret;
 
 		ret = ov5640_restore_mode(sensor);
 		if (ret)
 			goto power_off;
 
-		/*
-		 * start streaming briefly followed by stream off in
-		 * order to coax the clock lane into LP-11 state.
-		 */
-		ret = ov5640_set_stream(sensor, true);
-		if (ret)
-			goto power_off;
-		usleep_range(1000, 2000);
-		ret = ov5640_set_stream(sensor, false);
-		if (ret)
-			goto power_off;
+		if (sensor->ep.bus_type == V4L2_MBUS_CSI2) {
+			/*
+			 * start streaming briefly followed by stream off in
+			 * order to coax the clock lane into LP-11 state.
+			 */
+			ret = ov5640_set_stream_mipi(sensor, true);
+			if (ret)
+				goto power_off;
+			usleep_range(1000, 2000);
+			ret = ov5640_set_stream_mipi(sensor, false);
+			if (ret)
+				goto power_off;
+		}
 
 		return 0;
 	}
 
 power_off:
-	ov5640_power(sensor, false);
-	regulator_bulk_disable(OV5640_NUM_SUPPLIES, sensor->supplies);
-xclk_off:
-	clk_disable_unprepare(sensor->xclk);
+	ov5640_set_power_off(sensor);
 	return ret;
 }
 
@@ -1693,17 +1851,23 @@ static int ov5640_try_fmt_internal(struct v4l2_subdev *sd,
 {
 	struct ov5640_dev *sensor = to_ov5640_dev(sd);
 	const struct ov5640_mode_info *mode;
+	int i;
 
 	mode = ov5640_find_mode(sensor, fr, fmt->width, fmt->height, true);
 	if (!mode)
 		return -EINVAL;
-
 	fmt->width = mode->width;
 	fmt->height = mode->height;
-	fmt->code = sensor->fmt.code;
 
 	if (new_mode)
 		*new_mode = mode;
+
+	for (i = 0; i < ARRAY_SIZE(ov5640_formats); i++)
+		if (ov5640_formats[i].code == fmt->code)
+			break;
+	if (i >= ARRAY_SIZE(ov5640_formats))
+		fmt->code = ov5640_formats[0].code;
+
 	return 0;
 }
 
@@ -1746,6 +1910,45 @@ static int ov5640_set_fmt(struct v4l2_subdev *sd,
 	return ret;
 }
 
+static int ov5640_set_framefmt(struct ov5640_dev *sensor,
+			       struct v4l2_mbus_framefmt *format)
+{
+	int ret = 0;
+	bool is_rgb = false;
+	u8 val;
+
+	switch (format->code) {
+	case MEDIA_BUS_FMT_UYVY8_2X8:
+		/* YUV422, UYVY */
+		val = 0x3f;
+		break;
+	case MEDIA_BUS_FMT_YUYV8_2X8:
+		/* YUV422, YUYV */
+		val = 0x30;
+		break;
+	case MEDIA_BUS_FMT_RGB565_2X8_LE:
+		/* RGB565 {g[2:0],b[4:0]},{r[4:0],g[5:3]} */
+		val = 0x6F;
+		is_rgb = true;
+		break;
+	case MEDIA_BUS_FMT_RGB565_2X8_BE:
+		/* RGB565 {r[4:0],g[5:3]},{g[2:0],b[4:0]} */
+		val = 0x61;
+		is_rgb = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* FORMAT CONTROL00: YUV and RGB formatting */
+	ret = ov5640_write_reg(sensor, OV5640_REG_FORMAT_CONTROL00, val);
+	if (ret)
+		return ret;
+
+	/* FORMAT MUX CONTROL: ISP YUV or RGB */
+	return ov5640_write_reg(sensor, OV5640_REG_ISP_FORMAT_MUX_CTRL,
+				is_rgb ? 0x01 : 0x00);
+}
 
 /*
  * Sensor Controls.
@@ -1854,6 +2057,7 @@ static int ov5640_set_ctrl_exposure(struct ov5640_dev *sensor, int exp)
 		if (ret < 0)
 			return ret;
 		max_exp += ret;
+		ret = 0;
 
 		if (ctrls->exposure->val < max_exp)
 			ret = ov5640_set_exposure(sensor, ctrls->exposure->val);
@@ -2131,15 +2335,12 @@ static int ov5640_enum_mbus_code(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_pad_config *cfg,
 				  struct v4l2_subdev_mbus_code_enum *code)
 {
-	struct ov5640_dev *sensor = to_ov5640_dev(sd);
-
 	if (code->pad != 0)
 		return -EINVAL;
-	if (code->index != 0)
+	if (code->index >= ARRAY_SIZE(ov5640_formats))
 		return -EINVAL;
 
-	code->code = sensor->fmt.code;
-
+	code->code = ov5640_formats[code->index].code;
 	return 0;
 }
 
@@ -2155,9 +2356,17 @@ static int ov5640_s_stream(struct v4l2_subdev *sd, int enable)
 			ret = ov5640_set_mode(sensor, sensor->current_mode);
 			if (ret)
 				goto out;
+
+			ret = ov5640_set_framefmt(sensor, &sensor->fmt);
+			if (ret)
+				goto out;
 		}
 
-		ret = ov5640_set_stream(sensor, enable);
+		if (sensor->ep.bus_type == V4L2_MBUS_CSI2)
+			ret = ov5640_set_stream_mipi(sensor, enable);
+		else
+			ret = ov5640_set_stream_dvp(sensor, enable);
+
 		if (!ret)
 			sensor->streaming = enable;
 	}
@@ -2202,6 +2411,34 @@ static int ov5640_get_regulators(struct ov5640_dev *sensor)
 				       sensor->supplies);
 }
 
+static int ov5640_check_chip_id(struct ov5640_dev *sensor)
+{
+	struct i2c_client *client = sensor->i2c_client;
+	int ret = 0;
+	u16 chip_id;
+
+	ret = ov5640_set_power_on(sensor);
+	if (ret)
+		return ret;
+
+	ret = ov5640_read_reg16(sensor, OV5640_REG_CHIP_ID, &chip_id);
+	if (ret) {
+		dev_err(&client->dev, "%s: failed to read chip identifier\n",
+			__func__);
+		goto power_off;
+	}
+
+	if (chip_id != 0x5640) {
+		dev_err(&client->dev, "%s: wrong chip identifier, expected 0x5640, got 0x%x\n",
+			__func__, chip_id);
+		ret = -ENXIO;
+	}
+
+power_off:
+	ov5640_set_power_off(sensor);
+	return ret;
+}
+
 static int ov5640_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
@@ -2242,11 +2479,6 @@ static int ov5640_probe(struct i2c_client *client,
 		return ret;
 	}
 
-	if (sensor->ep.bus_type != V4L2_MBUS_CSI2) {
-		dev_err(dev, "invalid bus type, must be MIPI CSI2\n");
-		return -EINVAL;
-	}
-
 	/* get system clock (xclk) */
 	sensor->xclk = devm_clk_get(dev, "xclk");
 	if (IS_ERR(sensor->xclk)) {
@@ -2284,6 +2516,10 @@ static int ov5640_probe(struct i2c_client *client,
 
 	mutex_init(&sensor->lock);
 
+	ret = ov5640_check_chip_id(sensor);
+	if (ret)
+		goto entity_cleanup;
+
 	ret = ov5640_init_controls(sensor);
 	if (ret)
 		goto entity_cleanup;
diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c
index 950a0acf85fb..28571de1c2f6 100644
--- a/drivers/media/i2c/ov7670.c
+++ b/drivers/media/i2c/ov7670.c
@@ -163,6 +163,11 @@ MODULE_PARM_DESC(debug, "Debug level (0-1)");
 #define   DBLV_X6	  0x10	  /* clock x6 */
 #define   DBLV_X8	  0x11	  /* clock x8 */
 
+#define REG_SCALING_XSC	0x70	/* Test pattern and horizontal scale factor */
+#define   TEST_PATTTERN_0 0x80
+#define REG_SCALING_YSC	0x71	/* Test pattern and vertical scale factor */
+#define   TEST_PATTTERN_1 0x80
+
 #define REG_REG76	0x76	/* OV's name */
 #define   R76_BLKPCOR	  0x80	  /* Black pixel correction enable */
 #define   R76_WHTPCOR	  0x40	  /* White pixel correction enable */
@@ -292,7 +297,8 @@ static struct regval_list ov7670_default_regs[] = {
 
 	{ REG_COM3, 0 },	{ REG_COM14, 0 },
 	/* Mystery scaling numbers */
-	{ 0x70, 0x3a },		{ 0x71, 0x35 },
+	{ REG_SCALING_XSC, 0x3a },
+	{ REG_SCALING_YSC, 0x35 },
 	{ 0x72, 0x11 },		{ 0x73, 0xf0 },
 	{ 0xa2, 0x02 },		{ REG_COM10, 0x0 },
 
@@ -406,12 +412,12 @@ static struct regval_list ov7670_fmt_yuv422[] = {
 	{ REG_COM1, 0 },	/* CCIR601 */
 	{ REG_COM15, COM15_R00FF },
 	{ REG_COM9, 0x48 }, /* 32x gain ceiling; 0x8 is reserved bit */
-	{ 0x4f, 0x80 }, 	/* "matrix coefficient 1" */
-	{ 0x50, 0x80 }, 	/* "matrix coefficient 2" */
+	{ 0x4f, 0x80 },		/* "matrix coefficient 1" */
+	{ 0x50, 0x80 },		/* "matrix coefficient 2" */
 	{ 0x51, 0    },		/* vb */
-	{ 0x52, 0x22 }, 	/* "matrix coefficient 4" */
-	{ 0x53, 0x5e }, 	/* "matrix coefficient 5" */
-	{ 0x54, 0x80 }, 	/* "matrix coefficient 6" */
+	{ 0x52, 0x22 },		/* "matrix coefficient 4" */
+	{ 0x53, 0x5e },		/* "matrix coefficient 5" */
+	{ 0x54, 0x80 },		/* "matrix coefficient 6" */
 	{ REG_COM13, COM13_GAMMA|COM13_UVSAT },
 	{ 0xff, 0xff },
 };
@@ -421,13 +427,13 @@ static struct regval_list ov7670_fmt_rgb565[] = {
 	{ REG_RGB444, 0 },	/* No RGB444 please */
 	{ REG_COM1, 0x0 },	/* CCIR601 */
 	{ REG_COM15, COM15_RGB565 },
-	{ REG_COM9, 0x38 }, 	/* 16x gain ceiling; 0x8 is reserved bit */
-	{ 0x4f, 0xb3 }, 	/* "matrix coefficient 1" */
-	{ 0x50, 0xb3 }, 	/* "matrix coefficient 2" */
+	{ REG_COM9, 0x38 },	/* 16x gain ceiling; 0x8 is reserved bit */
+	{ 0x4f, 0xb3 },		/* "matrix coefficient 1" */
+	{ 0x50, 0xb3 },		/* "matrix coefficient 2" */
 	{ 0x51, 0    },		/* vb */
-	{ 0x52, 0x3d }, 	/* "matrix coefficient 4" */
-	{ 0x53, 0xa7 }, 	/* "matrix coefficient 5" */
-	{ 0x54, 0xe4 }, 	/* "matrix coefficient 6" */
+	{ 0x52, 0x3d },		/* "matrix coefficient 4" */
+	{ 0x53, 0xa7 },		/* "matrix coefficient 5" */
+	{ 0x54, 0xe4 },		/* "matrix coefficient 6" */
 	{ REG_COM13, COM13_GAMMA|COM13_UVSAT },
 	{ 0xff, 0xff },
 };
@@ -437,13 +443,13 @@ static struct regval_list ov7670_fmt_rgb444[] = {
 	{ REG_RGB444, R444_ENABLE },	/* Enable xxxxrrrr ggggbbbb */
 	{ REG_COM1, 0x0 },	/* CCIR601 */
 	{ REG_COM15, COM15_R01FE|COM15_RGB565 }, /* Data range needed? */
-	{ REG_COM9, 0x38 }, 	/* 16x gain ceiling; 0x8 is reserved bit */
-	{ 0x4f, 0xb3 }, 	/* "matrix coefficient 1" */
-	{ 0x50, 0xb3 }, 	/* "matrix coefficient 2" */
+	{ REG_COM9, 0x38 },	/* 16x gain ceiling; 0x8 is reserved bit */
+	{ 0x4f, 0xb3 },		/* "matrix coefficient 1" */
+	{ 0x50, 0xb3 },		/* "matrix coefficient 2" */
 	{ 0x51, 0    },		/* vb */
-	{ 0x52, 0x3d }, 	/* "matrix coefficient 4" */
-	{ 0x53, 0xa7 }, 	/* "matrix coefficient 5" */
-	{ 0x54, 0xe4 }, 	/* "matrix coefficient 6" */
+	{ 0x52, 0x3d },		/* "matrix coefficient 4" */
+	{ 0x53, 0xa7 },		/* "matrix coefficient 5" */
+	{ 0x54, 0xe4 },		/* "matrix coefficient 6" */
 	{ REG_COM13, COM13_GAMMA|COM13_UVSAT|0x2 },  /* Magic rsvd bit */
 	{ 0xff, 0xff },
 };
@@ -568,6 +574,19 @@ static int ov7670_write(struct v4l2_subdev *sd, unsigned char reg,
 		return ov7670_write_i2c(sd, reg, value);
 }
 
+static int ov7670_update_bits(struct v4l2_subdev *sd, unsigned char reg,
+		unsigned char mask, unsigned char value)
+{
+	unsigned char orig;
+	int ret;
+
+	ret = ov7670_read(sd, reg, &orig);
+	if (ret)
+		return ret;
+
+	return ov7670_write(sd, reg, (orig & ~mask) | (value & mask));
+}
+
 /*
  * Write a list of register settings; ff/ff stops the process.
  */
@@ -648,7 +667,7 @@ static struct ov7670_format_struct {
 	{
 		.mbus_code	= MEDIA_BUS_FMT_YUYV8_2X8,
 		.colorspace	= V4L2_COLORSPACE_SRGB,
-		.regs 		= ov7670_fmt_yuv422,
+		.regs		= ov7670_fmt_yuv422,
 		.cmatrix	= { 128, -128, 0, -34, -94, 128 },
 	},
 	{
@@ -666,7 +685,7 @@ static struct ov7670_format_struct {
 	{
 		.mbus_code	= MEDIA_BUS_FMT_SBGGR8_1X8,
 		.colorspace	= V4L2_COLORSPACE_SRGB,
-		.regs 		= ov7670_fmt_raw,
+		.regs		= ov7670_fmt_raw,
 		.cmatrix	= { 0, 0, 0, 0, 0, 0 },
 	},
 };
@@ -1470,6 +1489,25 @@ static int ov7670_s_autoexp(struct v4l2_subdev *sd,
 	return ret;
 }
 
+static const char * const ov7670_test_pattern_menu[] = {
+	"No test output",
+	"Shifting \"1\"",
+	"8-bar color bar",
+	"Fade to gray color bar",
+};
+
+static int ov7670_s_test_pattern(struct v4l2_subdev *sd, int value)
+{
+	int ret;
+
+	ret = ov7670_update_bits(sd, REG_SCALING_XSC, TEST_PATTTERN_0,
+				value & BIT(0) ? TEST_PATTTERN_0 : 0);
+	if (ret)
+		return ret;
+
+	return ov7670_update_bits(sd, REG_SCALING_YSC, TEST_PATTTERN_1,
+				value & BIT(1) ? TEST_PATTTERN_1 : 0);
+}
 
 static int ov7670_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
 {
@@ -1516,6 +1554,8 @@ static int ov7670_s_ctrl(struct v4l2_ctrl *ctrl)
 			return ov7670_s_exp(sd, info->exposure->val);
 		}
 		return ov7670_s_autoexp(sd, ctrl->val);
+	case V4L2_CID_TEST_PATTERN:
+		return ov7670_s_test_pattern(sd, ctrl->val);
 	}
 	return -EINVAL;
 }
@@ -1770,6 +1810,10 @@ static int ov7670_probe(struct i2c_client *client,
 	info->auto_exposure = v4l2_ctrl_new_std_menu(&info->hdl, &ov7670_ctrl_ops,
 			V4L2_CID_EXPOSURE_AUTO, V4L2_EXPOSURE_MANUAL, 0,
 			V4L2_EXPOSURE_AUTO);
+	v4l2_ctrl_new_std_menu_items(&info->hdl, &ov7670_ctrl_ops,
+			V4L2_CID_TEST_PATTERN,
+			ARRAY_SIZE(ov7670_test_pattern_menu) - 1, 0, 0,
+			ov7670_test_pattern_menu);
 	sd->ctrl_handler = &info->hdl;
 	if (info->hdl.error) {
 		ret = info->hdl.error;
@@ -1802,9 +1846,7 @@ static int ov7670_probe(struct i2c_client *client,
 	return 0;
 
 entity_cleanup:
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	media_entity_cleanup(&info->sd.entity);
-#endif
 hdl_free:
 	v4l2_ctrl_handler_free(&info->hdl);
 power_off:
@@ -1820,12 +1862,10 @@ static int ov7670_remove(struct i2c_client *client)
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov7670_info *info = to_state(sd);
 
-	v4l2_device_unregister_subdev(sd);
+	v4l2_async_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&info->hdl);
 	clk_disable_unprepare(info->clk);
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	media_entity_cleanup(&info->sd.entity);
-#endif
 	ov7670_s_power(sd, 0);
 	return 0;
 }
diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c
new file mode 100644
index 000000000000..fc9dbbcae56e
--- /dev/null
+++ b/drivers/media/i2c/ov7740.c
@@ -0,0 +1,1214 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2017 Microchip Corporation.
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-image-sizes.h>
+#include <media/v4l2-subdev.h>
+
+#define REG_OUTSIZE_LSB 0x34
+
+/* OV7740 register tables */
+#define REG_GAIN	0x00	/* Gain lower 8 bits (rest in vref) */
+#define REG_BGAIN	0x01	/* blue gain */
+#define REG_RGAIN	0x02	/* red gain */
+#define REG_GGAIN	0x03	/* green gain */
+#define REG_REG04	0x04	/* analog setting, dont change*/
+#define REG_BAVG	0x05	/* b channel average */
+#define REG_GAVG	0x06	/* g channel average */
+#define REG_RAVG	0x07	/* r channel average */
+
+#define REG_REG0C	0x0C	/* filp enable */
+#define REG0C_IMG_FLIP		0x80
+#define REG0C_IMG_MIRROR	0x40
+
+#define REG_REG0E	0x0E	/* blc line */
+#define REG_HAEC	0x0F	/* auto exposure cntrl */
+#define REG_AEC		0x10	/* auto exposure cntrl */
+
+#define REG_CLK		0x11	/* Clock control */
+#define REG_REG55	0x55	/* Clock PLL DIV/PreDiv */
+
+#define REG_REG12	0x12
+
+#define REG_REG13	0x13	/* auto/manual AGC, AEC, Write Balance*/
+#define REG13_AEC_EN	0x01
+#define REG13_AGC_EN	0x04
+
+#define REG_REG14	0x14
+#define REG_CTRL15	0x15
+#define REG15_GAIN_MSB	0x03
+
+#define REG_REG16	0x16
+
+#define REG_MIDH	0x1C	/* manufacture id byte */
+#define REG_MIDL	0x1D	/* manufacture id byre */
+#define REG_PIDH	0x0A	/* Product ID MSB */
+#define REG_PIDL	0x0B	/* Product ID LSB */
+
+#define REG_84		0x84	/* lots of stuff */
+#define REG_REG38	0x38	/* sub-addr */
+
+#define REG_AHSTART	0x17	/* Horiz start high bits */
+#define REG_AHSIZE	0x18
+#define REG_AVSTART	0x19	/* Vert start high bits */
+#define REG_AVSIZE	0x1A
+#define REG_PSHFT	0x1b	/* Pixel delay after HREF */
+
+#define REG_HOUTSIZE	0x31
+#define REG_VOUTSIZE	0x32
+#define REG_HVSIZEOFF	0x33
+#define REG_REG34	0x34	/* DSP output size H/V LSB*/
+
+#define REG_ISP_CTRL00	0x80
+#define ISPCTRL00_AWB_EN	0x10
+#define ISPCTRL00_AWB_GAIN_EN	0x04
+
+#define	REG_YGAIN	0xE2	/* ygain for contrast control */
+
+#define	REG_YBRIGHT	  0xE3
+#define	REG_SGNSET	  0xE4
+#define	SGNSET_YBRIGHT_MASK	  0x08
+
+#define REG_USAT	0xDD
+#define REG_VSAT	0xDE
+
+
+struct ov7740 {
+	struct v4l2_subdev subdev;
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	struct media_pad pad;
+#endif
+	struct v4l2_mbus_framefmt format;
+	const struct ov7740_pixfmt *fmt;  /* Current format */
+	const struct ov7740_framesize *frmsize;
+	struct regmap *regmap;
+	struct clk *xvclk;
+	struct v4l2_ctrl_handler ctrl_handler;
+	struct {
+		/* gain cluster */
+		struct v4l2_ctrl *auto_gain;
+		struct v4l2_ctrl *gain;
+	};
+	struct {
+		struct v4l2_ctrl *auto_wb;
+		struct v4l2_ctrl *blue_balance;
+		struct v4l2_ctrl *red_balance;
+	};
+	struct {
+		struct v4l2_ctrl *hflip;
+		struct v4l2_ctrl *vflip;
+	};
+	struct {
+		/* exposure cluster */
+		struct v4l2_ctrl *auto_exposure;
+		struct v4l2_ctrl *exposure;
+	};
+	struct {
+		/* saturation/hue cluster */
+		struct v4l2_ctrl *saturation;
+		struct v4l2_ctrl *hue;
+	};
+	struct v4l2_ctrl *brightness;
+	struct v4l2_ctrl *contrast;
+
+	struct mutex mutex;	/* To serialize asynchronus callbacks */
+	bool streaming;		/* Streaming on/off */
+
+	struct gpio_desc *resetb_gpio;
+	struct gpio_desc *pwdn_gpio;
+};
+
+struct ov7740_pixfmt {
+	u32 mbus_code;
+	enum v4l2_colorspace colorspace;
+	const struct reg_sequence *regs;
+	u32 reg_num;
+};
+
+struct ov7740_framesize {
+	u16 width;
+	u16 height;
+	const struct reg_sequence *regs;
+	u32 reg_num;
+};
+
+static const struct reg_sequence ov7740_vga[] = {
+	{0x55, 0x40},
+	{0x11, 0x02},
+
+	{0xd5, 0x10},
+	{0x0c, 0x12},
+	{0x0d, 0x34},
+	{0x17, 0x25},
+	{0x18, 0xa0},
+	{0x19, 0x03},
+	{0x1a, 0xf0},
+	{0x1b, 0x89},
+	{0x22, 0x03},
+	{0x29, 0x18},
+	{0x2b, 0xf8},
+	{0x2c, 0x01},
+	{REG_HOUTSIZE, 0xa0},
+	{REG_VOUTSIZE, 0xf0},
+	{0x33, 0xc4},
+	{REG_OUTSIZE_LSB, 0x0},
+	{0x35, 0x05},
+	{0x04, 0x60},
+	{0x27, 0x80},
+	{0x3d, 0x0f},
+	{0x3e, 0x80},
+	{0x3f, 0x40},
+	{0x40, 0x7f},
+	{0x41, 0x6a},
+	{0x42, 0x29},
+	{0x44, 0x22},
+	{0x45, 0x41},
+	{0x47, 0x02},
+	{0x49, 0x64},
+	{0x4a, 0xa1},
+	{0x4b, 0x40},
+	{0x4c, 0x1a},
+	{0x4d, 0x50},
+	{0x4e, 0x13},
+	{0x64, 0x00},
+	{0x67, 0x88},
+	{0x68, 0x1a},
+
+	{0x14, 0x28},
+	{0x24, 0x3c},
+	{0x25, 0x30},
+	{0x26, 0x72},
+	{0x50, 0x97},
+	{0x51, 0x1f},
+	{0x52, 0x00},
+	{0x53, 0x00},
+	{0x20, 0x00},
+	{0x21, 0xcf},
+	{0x50, 0x4b},
+	{0x38, 0x14},
+	{0xe9, 0x00},
+	{0x56, 0x55},
+	{0x57, 0xff},
+	{0x58, 0xff},
+	{0x59, 0xff},
+	{0x5f, 0x04},
+	{0xec, 0x00},
+	{0x13, 0xff},
+
+	{0x81, 0x3f},
+	{0x82, 0x32},
+	{0x38, 0x11},
+	{0x84, 0x70},
+	{0x85, 0x00},
+	{0x86, 0x03},
+	{0x87, 0x01},
+	{0x88, 0x05},
+	{0x89, 0x30},
+	{0x8d, 0x30},
+	{0x8f, 0x85},
+	{0x93, 0x30},
+	{0x95, 0x85},
+	{0x99, 0x30},
+	{0x9b, 0x85},
+
+	{0x9c, 0x08},
+	{0x9d, 0x12},
+	{0x9e, 0x23},
+	{0x9f, 0x45},
+	{0xa0, 0x55},
+	{0xa1, 0x64},
+	{0xa2, 0x72},
+	{0xa3, 0x7f},
+	{0xa4, 0x8b},
+	{0xa5, 0x95},
+	{0xa6, 0xa7},
+	{0xa7, 0xb5},
+	{0xa8, 0xcb},
+	{0xa9, 0xdd},
+	{0xaa, 0xec},
+	{0xab, 0x1a},
+
+	{0xce, 0x78},
+	{0xcf, 0x6e},
+	{0xd0, 0x0a},
+	{0xd1, 0x0c},
+	{0xd2, 0x84},
+	{0xd3, 0x90},
+	{0xd4, 0x1e},
+
+	{0x5a, 0x24},
+	{0x5b, 0x1f},
+	{0x5c, 0x88},
+	{0x5d, 0x60},
+
+	{0xac, 0x6e},
+	{0xbe, 0xff},
+	{0xbf, 0x00},
+
+	{0x0f, 0x1d},
+	{0x0f, 0x1f},
+};
+
+static const struct ov7740_framesize ov7740_framesizes[] = {
+	{
+		.width		= VGA_WIDTH,
+		.height		= VGA_HEIGHT,
+		.regs		= ov7740_vga,
+		.reg_num	= ARRAY_SIZE(ov7740_vga),
+	},
+};
+
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+static int ov7740_get_register(struct v4l2_subdev *sd,
+			       struct v4l2_dbg_register *reg)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+	struct regmap *regmap = ov7740->regmap;
+	unsigned int val = 0;
+	int ret;
+
+	ret = regmap_read(regmap, reg->reg & 0xff, &val);
+	reg->val = val;
+	reg->size = 1;
+
+	return 0;
+}
+
+static int ov7740_set_register(struct v4l2_subdev *sd,
+			       const struct v4l2_dbg_register *reg)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+	struct regmap *regmap = ov7740->regmap;
+
+	regmap_write(regmap, reg->reg & 0xff, reg->val & 0xff);
+
+	return 0;
+}
+#endif
+
+static int ov7740_set_power(struct ov7740 *ov7740, int on)
+{
+	int ret;
+
+	if (on) {
+		ret = clk_prepare_enable(ov7740->xvclk);
+		if (ret)
+			return ret;
+
+		if (ov7740->pwdn_gpio)
+			gpiod_direction_output(ov7740->pwdn_gpio, 0);
+
+		if (ov7740->resetb_gpio) {
+			gpiod_set_value(ov7740->resetb_gpio, 1);
+			usleep_range(500, 1000);
+			gpiod_set_value(ov7740->resetb_gpio, 0);
+			usleep_range(3000, 5000);
+		}
+	} else {
+		clk_disable_unprepare(ov7740->xvclk);
+
+		if (ov7740->pwdn_gpio)
+			gpiod_direction_output(ov7740->pwdn_gpio, 0);
+	}
+
+	return 0;
+}
+
+static struct v4l2_subdev_core_ops ov7740_subdev_core_ops = {
+	.log_status = v4l2_ctrl_subdev_log_status,
+#ifdef CONFIG_VIDEO_ADV_DEBUG
+	.g_register = ov7740_get_register,
+	.s_register = ov7740_set_register,
+#endif
+	.subscribe_event = v4l2_ctrl_subdev_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static int ov7740_set_white_balance(struct ov7740 *ov7740, int awb)
+{
+	struct regmap *regmap = ov7740->regmap;
+	unsigned int value;
+	int ret;
+
+	ret = regmap_read(regmap, REG_ISP_CTRL00, &value);
+	if (!ret) {
+		if (awb)
+			value |= (ISPCTRL00_AWB_EN | ISPCTRL00_AWB_GAIN_EN);
+		else
+			value &= ~(ISPCTRL00_AWB_EN | ISPCTRL00_AWB_GAIN_EN);
+		ret = regmap_write(regmap, REG_ISP_CTRL00, value);
+		if (ret)
+			return ret;
+	}
+
+	if (!awb) {
+		ret = regmap_write(regmap, REG_BGAIN,
+				   ov7740->blue_balance->val);
+		if (ret)
+			return ret;
+
+		ret = regmap_write(regmap, REG_RGAIN, ov7740->red_balance->val);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int ov7740_set_saturation(struct regmap *regmap, int value)
+{
+	int ret;
+
+	ret = regmap_write(regmap, REG_USAT, (unsigned char)value);
+	if (ret)
+		return ret;
+
+	return regmap_write(regmap, REG_VSAT, (unsigned char)value);
+}
+
+static int ov7740_set_gain(struct regmap *regmap, int value)
+{
+	int ret;
+
+	ret = regmap_write(regmap, REG_GAIN, value & 0xff);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(regmap, REG_CTRL15,
+				 REG15_GAIN_MSB, (value >> 8) & 0x3);
+	if (!ret)
+		ret = regmap_update_bits(regmap, REG_REG13, REG13_AGC_EN, 0);
+
+	return ret;
+}
+
+static int ov7740_set_autogain(struct regmap *regmap, int value)
+{
+	unsigned int reg;
+	int ret;
+
+	ret = regmap_read(regmap, REG_REG13, &reg);
+	if (ret)
+		return ret;
+	if (value)
+		reg |= REG13_AGC_EN;
+	else
+		reg &= ~REG13_AGC_EN;
+	return regmap_write(regmap, REG_REG13, reg);
+}
+
+static int ov7740_set_brightness(struct regmap *regmap, int value)
+{
+	/* Turn off AEC/AGC */
+	regmap_update_bits(regmap, REG_REG13, REG13_AEC_EN, 0);
+	regmap_update_bits(regmap, REG_REG13, REG13_AGC_EN, 0);
+
+	if (value >= 0) {
+		regmap_write(regmap, REG_YBRIGHT, (unsigned char)value);
+		regmap_update_bits(regmap, REG_SGNSET, SGNSET_YBRIGHT_MASK, 0);
+	} else{
+		regmap_write(regmap, REG_YBRIGHT, (unsigned char)(-value));
+		regmap_update_bits(regmap, REG_SGNSET, SGNSET_YBRIGHT_MASK, 1);
+	}
+
+	return 0;
+}
+
+static int ov7740_set_contrast(struct regmap *regmap, int value)
+{
+	return regmap_write(regmap, REG_YGAIN, (unsigned char)value);
+}
+
+static int ov7740_get_gain(struct ov7740 *ov7740, struct v4l2_ctrl *ctrl)
+{
+	struct regmap *regmap = ov7740->regmap;
+	unsigned int value0, value1;
+	int ret;
+
+	if (!ctrl->val)
+		return 0;
+
+	ret = regmap_read(regmap, REG_GAIN, &value0);
+	if (ret)
+		return ret;
+	ret = regmap_read(regmap, REG_CTRL15, &value1);
+	if (ret)
+		return ret;
+
+	ov7740->gain->val = (value1 << 8) | (value0 & 0xff);
+
+	return 0;
+}
+
+static int ov7740_set_exp(struct regmap *regmap, int value)
+{
+	int ret;
+
+	/* Turn off AEC/AGC */
+	ret = regmap_update_bits(regmap, REG_REG13,
+				 REG13_AEC_EN | REG13_AGC_EN, 0);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(regmap, REG_AEC, (unsigned char)value);
+	if (ret)
+		return ret;
+
+	return regmap_write(regmap, REG_HAEC, (unsigned char)(value >> 8));
+}
+
+static int ov7740_set_autoexp(struct regmap *regmap,
+			      enum v4l2_exposure_auto_type value)
+{
+	unsigned int reg;
+	int ret;
+
+	ret = regmap_read(regmap, REG_REG13, &reg);
+	if (!ret) {
+		if (value == V4L2_EXPOSURE_AUTO)
+			reg |= (REG13_AEC_EN | REG13_AGC_EN);
+		else
+			reg &= ~(REG13_AEC_EN | REG13_AGC_EN);
+		ret = regmap_write(regmap, REG_REG13, reg);
+	}
+
+	return ret;
+}
+
+
+static int ov7740_get_volatile_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct ov7740 *ov7740 = container_of(ctrl->handler,
+					     struct ov7740, ctrl_handler);
+	int ret;
+
+	switch (ctrl->id) {
+	case V4L2_CID_AUTOGAIN:
+		ret = ov7740_get_gain(ov7740, ctrl);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int ov7740_set_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct ov7740 *ov7740 = container_of(ctrl->handler,
+					     struct ov7740, ctrl_handler);
+	struct i2c_client *client = v4l2_get_subdevdata(&ov7740->subdev);
+	struct regmap *regmap = ov7740->regmap;
+	int ret;
+	u8 val = 0;
+
+	if (pm_runtime_get_if_in_use(&client->dev) <= 0)
+		return 0;
+
+	switch (ctrl->id) {
+	case V4L2_CID_AUTO_WHITE_BALANCE:
+		ret = ov7740_set_white_balance(ov7740, ctrl->val);
+		break;
+	case V4L2_CID_SATURATION:
+		ret = ov7740_set_saturation(regmap, ctrl->val);
+		break;
+	case V4L2_CID_BRIGHTNESS:
+		ret = ov7740_set_brightness(regmap, ctrl->val);
+		break;
+	case V4L2_CID_CONTRAST:
+		ret = ov7740_set_contrast(regmap, ctrl->val);
+		break;
+	case V4L2_CID_VFLIP:
+		ret = regmap_update_bits(regmap, REG_REG0C,
+					 REG0C_IMG_FLIP, val);
+		break;
+	case V4L2_CID_HFLIP:
+		val = ctrl->val ? REG0C_IMG_MIRROR : 0x00;
+		ret = regmap_update_bits(regmap, REG_REG0C,
+					 REG0C_IMG_MIRROR, val);
+		break;
+	case V4L2_CID_AUTOGAIN:
+		if (!ctrl->val)
+			return ov7740_set_gain(regmap, ov7740->gain->val);
+
+		ret = ov7740_set_autogain(regmap, ctrl->val);
+		break;
+
+	case V4L2_CID_EXPOSURE_AUTO:
+		if (ctrl->val == V4L2_EXPOSURE_MANUAL)
+			return ov7740_set_exp(regmap, ov7740->exposure->val);
+
+		ret = ov7740_set_autoexp(regmap, ctrl->val);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	pm_runtime_put(&client->dev);
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops ov7740_ctrl_ops = {
+	.g_volatile_ctrl = ov7740_get_volatile_ctrl,
+	.s_ctrl = ov7740_set_ctrl,
+};
+
+static int ov7740_start_streaming(struct ov7740 *ov7740)
+{
+	int ret;
+
+	if (ov7740->fmt) {
+		ret = regmap_multi_reg_write(ov7740->regmap,
+					     ov7740->fmt->regs,
+					     ov7740->fmt->reg_num);
+		if (ret)
+			return ret;
+	}
+
+	if (ov7740->frmsize) {
+		ret = regmap_multi_reg_write(ov7740->regmap,
+					     ov7740->frmsize->regs,
+					     ov7740->frmsize->reg_num);
+		if (ret)
+			return ret;
+	}
+
+	return __v4l2_ctrl_handler_setup(ov7740->subdev.ctrl_handler);
+}
+
+static int ov7740_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
+	int ret = 0;
+
+	mutex_lock(&ov7740->mutex);
+	if (ov7740->streaming == enable) {
+		mutex_unlock(&ov7740->mutex);
+		return 0;
+	}
+
+	if (enable) {
+		ret = pm_runtime_get_sync(&client->dev);
+		if (ret < 0) {
+			pm_runtime_put_noidle(&client->dev);
+			goto err_unlock;
+		}
+
+		ret = ov7740_start_streaming(ov7740);
+		if (ret)
+			goto err_rpm_put;
+	} else {
+		pm_runtime_put(&client->dev);
+	}
+
+	ov7740->streaming = enable;
+
+	mutex_unlock(&ov7740->mutex);
+	return ret;
+
+err_rpm_put:
+	pm_runtime_put(&client->dev);
+err_unlock:
+	mutex_unlock(&ov7740->mutex);
+	return ret;
+}
+
+static int ov7740_get_parm(struct v4l2_subdev *sd,
+			   struct v4l2_streamparm *parms)
+{
+	struct v4l2_captureparm *cp = &parms->parm.capture;
+	struct v4l2_fract *tpf = &cp->timeperframe;
+
+	if (parms->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	memset(cp, 0, sizeof(struct v4l2_captureparm));
+	cp->capability = V4L2_CAP_TIMEPERFRAME;
+
+	tpf->numerator = 1;
+	tpf->denominator = 60;
+
+	return 0;
+}
+
+static int ov7740_set_parm(struct v4l2_subdev *sd,
+			   struct v4l2_streamparm *parms)
+{
+	struct v4l2_captureparm *cp = &parms->parm.capture;
+	struct v4l2_fract *tpf = &cp->timeperframe;
+
+	if (parms->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+	if (cp->extendedmode != 0)
+		return -EINVAL;
+
+	cp->capability = V4L2_CAP_TIMEPERFRAME;
+
+	tpf->numerator = 1;
+	tpf->denominator = 60;
+
+	return 0;
+}
+
+static struct v4l2_subdev_video_ops ov7740_subdev_video_ops = {
+	.s_stream = ov7740_set_stream,
+	.s_parm = ov7740_set_parm,
+	.g_parm = ov7740_get_parm,
+};
+
+static const struct reg_sequence ov7740_format_yuyv[] = {
+	{0x12, 0x00},
+	{0x36, 0x3f},
+	{0x80, 0x7f},
+	{0x83, 0x01},
+};
+
+static const struct reg_sequence ov7740_format_bggr8[] = {
+	{0x36, 0x2f},
+	{0x80, 0x01},
+	{0x83, 0x04},
+};
+
+static const struct ov7740_pixfmt ov7740_formats[] = {
+	{
+		.mbus_code = MEDIA_BUS_FMT_YUYV8_2X8,
+		.colorspace = V4L2_COLORSPACE_SRGB,
+		.regs = ov7740_format_yuyv,
+		.reg_num = ARRAY_SIZE(ov7740_format_yuyv),
+	},
+	{
+		.mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8,
+		.colorspace = V4L2_COLORSPACE_SRGB,
+		.regs = ov7740_format_bggr8,
+		.reg_num = ARRAY_SIZE(ov7740_format_bggr8),
+	}
+};
+#define N_OV7740_FMTS ARRAY_SIZE(ov7740_formats)
+
+static int ov7740_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->pad || code->index >= N_OV7740_FMTS)
+		return -EINVAL;
+
+	code->code = ov7740_formats[code->index].mbus_code;
+
+	return 0;
+}
+
+static int ov7740_enum_frame_interval(struct v4l2_subdev *sd,
+				struct v4l2_subdev_pad_config *cfg,
+				struct v4l2_subdev_frame_interval_enum *fie)
+{
+	if (fie->pad)
+		return -EINVAL;
+
+	if (fie->index >= 1)
+		return -EINVAL;
+
+	if ((fie->width != VGA_WIDTH) || (fie->height != VGA_HEIGHT))
+		return -EINVAL;
+
+	fie->interval.numerator = 1;
+	fie->interval.denominator = 60;
+
+	return 0;
+}
+
+static int ov7740_enum_frame_size(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_pad_config *cfg,
+				  struct v4l2_subdev_frame_size_enum *fse)
+{
+	if (fse->pad)
+		return -EINVAL;
+
+	if (fse->index > 0)
+		return -EINVAL;
+
+	fse->min_width = fse->max_width = VGA_WIDTH;
+	fse->min_height = fse->max_height = VGA_HEIGHT;
+
+	return 0;
+}
+
+static int ov7740_try_fmt_internal(struct v4l2_subdev *sd,
+				   struct v4l2_mbus_framefmt *fmt,
+				   const struct ov7740_pixfmt **ret_fmt,
+				   const struct ov7740_framesize **ret_frmsize)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+	const struct ov7740_framesize *fsize = &ov7740_framesizes[0];
+	int index, i;
+
+	for (index = 0; index < N_OV7740_FMTS; index++) {
+		if (ov7740_formats[index].mbus_code == fmt->code)
+			break;
+	}
+	if (index >= N_OV7740_FMTS) {
+		/* default to first format */
+		index = 0;
+		fmt->code = ov7740_formats[0].mbus_code;
+	}
+	if (ret_fmt != NULL)
+		*ret_fmt = ov7740_formats + index;
+
+	for (i = 0; i < ARRAY_SIZE(ov7740_framesizes); i++) {
+		if ((fsize->width >= fmt->width) &&
+		    (fsize->height >= fmt->height)) {
+			fmt->width = fsize->width;
+			fmt->height = fsize->height;
+			break;
+		}
+
+		fsize++;
+	}
+
+	if (ret_frmsize != NULL)
+		*ret_frmsize = fsize;
+
+	fmt->field = V4L2_FIELD_NONE;
+	fmt->colorspace = ov7740_formats[index].colorspace;
+
+	ov7740->format = *fmt;
+
+	return 0;
+}
+
+static int ov7740_set_fmt(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_pad_config *cfg,
+			  struct v4l2_subdev_format *format)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+	const struct ov7740_pixfmt *ovfmt;
+	const struct ov7740_framesize *fsize;
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+	struct v4l2_mbus_framefmt *mbus_fmt;
+#endif
+	int ret;
+
+	mutex_lock(&ov7740->mutex);
+	if (format->pad) {
+		ret = -EINVAL;
+		goto error;
+	}
+
+	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
+		ret = ov7740_try_fmt_internal(sd, &format->format, NULL, NULL);
+		if (ret)
+			goto error;
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+		mbus_fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad);
+		*mbus_fmt = format->format;
+
+		mutex_unlock(&ov7740->mutex);
+		return 0;
+#else
+		ret = -ENOTTY;
+		goto error;
+#endif
+	}
+
+	ret = ov7740_try_fmt_internal(sd, &format->format, &ovfmt, &fsize);
+	if (ret)
+		goto error;
+
+	ov7740->fmt = ovfmt;
+	ov7740->frmsize = fsize;
+
+	mutex_unlock(&ov7740->mutex);
+	return 0;
+
+error:
+	mutex_unlock(&ov7740->mutex);
+	return ret;
+}
+
+static int ov7740_get_fmt(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_pad_config *cfg,
+			  struct v4l2_subdev_format *format)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+	struct v4l2_mbus_framefmt *mbus_fmt;
+#endif
+	int ret = 0;
+
+	mutex_lock(&ov7740->mutex);
+	if (format->which == V4L2_SUBDEV_FORMAT_TRY) {
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+		mbus_fmt = v4l2_subdev_get_try_format(sd, cfg, 0);
+		format->format = *mbus_fmt;
+		ret = 0;
+#else
+		ret = -ENOTTY;
+#endif
+	} else {
+		format->format = ov7740->format;
+	}
+	mutex_unlock(&ov7740->mutex);
+
+	return ret;
+}
+
+static const struct v4l2_subdev_pad_ops ov7740_subdev_pad_ops = {
+	.enum_frame_interval = ov7740_enum_frame_interval,
+	.enum_frame_size = ov7740_enum_frame_size,
+	.enum_mbus_code = ov7740_enum_mbus_code,
+	.get_fmt = ov7740_get_fmt,
+	.set_fmt = ov7740_set_fmt,
+};
+
+static const struct v4l2_subdev_ops ov7740_subdev_ops = {
+	.core	= &ov7740_subdev_core_ops,
+	.video	= &ov7740_subdev_video_ops,
+	.pad	= &ov7740_subdev_pad_ops,
+};
+
+static void ov7740_get_default_format(struct v4l2_subdev *sd,
+				      struct v4l2_mbus_framefmt *format)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+
+	format->width = ov7740->frmsize->width;
+	format->height = ov7740->frmsize->height;
+	format->colorspace = ov7740->fmt->colorspace;
+	format->code = ov7740->fmt->mbus_code;
+	format->field = V4L2_FIELD_NONE;
+}
+
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+static int ov7740_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+	struct v4l2_mbus_framefmt *format =
+				v4l2_subdev_get_try_format(sd, fh->pad, 0);
+
+	mutex_lock(&ov7740->mutex);
+	ov7740_get_default_format(sd, format);
+	mutex_unlock(&ov7740->mutex);
+
+	return 0;
+}
+
+static const struct v4l2_subdev_internal_ops ov7740_subdev_internal_ops = {
+	.open = ov7740_open,
+};
+#endif
+
+static int ov7740_probe_dt(struct i2c_client *client,
+			   struct ov7740 *ov7740)
+{
+	ov7740->resetb_gpio = devm_gpiod_get_optional(&client->dev, "reset",
+			GPIOD_OUT_HIGH);
+	if (IS_ERR(ov7740->resetb_gpio)) {
+		dev_info(&client->dev, "can't get %s GPIO\n", "reset");
+		return PTR_ERR(ov7740->resetb_gpio);
+	}
+
+	ov7740->pwdn_gpio = devm_gpiod_get_optional(&client->dev, "powerdown",
+			GPIOD_OUT_LOW);
+	if (IS_ERR(ov7740->pwdn_gpio)) {
+		dev_info(&client->dev, "can't get %s GPIO\n", "powerdown");
+		return PTR_ERR(ov7740->pwdn_gpio);
+	}
+
+	return 0;
+}
+
+static int ov7740_detect(struct ov7740 *ov7740)
+{
+	struct regmap *regmap = ov7740->regmap;
+	unsigned int midh, midl, pidh, pidl;
+	int ret;
+
+	ret = regmap_read(regmap, REG_MIDH, &midh);
+	if (ret)
+		return ret;
+	if (midh != 0x7f)
+		return -ENODEV;
+
+	ret = regmap_read(regmap, REG_MIDL, &midl);
+	if (ret)
+		return ret;
+	if (midl != 0xa2)
+		return -ENODEV;
+
+	ret = regmap_read(regmap, REG_PIDH, &pidh);
+	if (ret)
+		return ret;
+	if (pidh != 0x77)
+		return -ENODEV;
+
+	ret = regmap_read(regmap, REG_PIDL, &pidl);
+	if (ret)
+		return ret;
+	if ((pidl != 0x40) && (pidl != 0x41) && (pidl != 0x42))
+		return -ENODEV;
+
+	return 0;
+}
+
+static int ov7740_init_controls(struct ov7740 *ov7740)
+{
+	struct i2c_client *client = v4l2_get_subdevdata(&ov7740->subdev);
+	struct v4l2_ctrl_handler *ctrl_hdlr = &ov7740->ctrl_handler;
+	int ret;
+
+	ret = v4l2_ctrl_handler_init(ctrl_hdlr, 2);
+	if (ret < 0)
+		return ret;
+
+	ctrl_hdlr->lock = &ov7740->mutex;
+	ov7740->auto_wb = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					  V4L2_CID_AUTO_WHITE_BALANCE,
+					  0, 1, 1, 1);
+	ov7740->blue_balance = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					       V4L2_CID_BLUE_BALANCE,
+					       0, 0xff, 1, 0x80);
+	ov7740->red_balance = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					      V4L2_CID_RED_BALANCE,
+					      0, 0xff, 1, 0x80);
+
+	ov7740->brightness = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					     V4L2_CID_BRIGHTNESS,
+					     -255, 255, 1, 0);
+	ov7740->contrast = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					   V4L2_CID_CONTRAST,
+					   0, 127, 1, 0x20);
+	ov7740->saturation = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+			  V4L2_CID_SATURATION, 0, 256, 1, 0x80);
+	ov7740->hflip = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					V4L2_CID_HFLIP, 0, 1, 1, 0);
+	ov7740->vflip = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					V4L2_CID_VFLIP, 0, 1, 1, 0);
+	ov7740->gain = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+				       V4L2_CID_GAIN, 0, 1023, 1, 500);
+	ov7740->auto_gain = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					    V4L2_CID_AUTOGAIN, 0, 1, 1, 1);
+	ov7740->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
+					   V4L2_CID_EXPOSURE, 0, 65535, 1, 500);
+	ov7740->auto_exposure = v4l2_ctrl_new_std_menu(ctrl_hdlr,
+					&ov7740_ctrl_ops,
+					V4L2_CID_EXPOSURE_AUTO,
+					V4L2_EXPOSURE_MANUAL, 0,
+					V4L2_EXPOSURE_AUTO);
+
+	ov7740->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
+	ov7740->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
+
+	v4l2_ctrl_auto_cluster(3, &ov7740->auto_wb, 0, false);
+	v4l2_ctrl_auto_cluster(2, &ov7740->auto_gain, 0, true);
+	v4l2_ctrl_auto_cluster(2, &ov7740->auto_exposure,
+			       V4L2_EXPOSURE_MANUAL, false);
+	v4l2_ctrl_cluster(2, &ov7740->hflip);
+
+	ret = v4l2_ctrl_handler_setup(ctrl_hdlr);
+	if (ret) {
+		dev_err(&client->dev, "%s control init failed (%d)\n",
+			__func__, ret);
+		goto error;
+	}
+
+	ov7740->subdev.ctrl_handler = ctrl_hdlr;
+	return 0;
+
+error:
+	v4l2_ctrl_handler_free(ctrl_hdlr);
+	mutex_destroy(&ov7740->mutex);
+	return ret;
+}
+
+static void ov7740_free_controls(struct ov7740 *ov7740)
+{
+	v4l2_ctrl_handler_free(ov7740->subdev.ctrl_handler);
+	mutex_destroy(&ov7740->mutex);
+}
+
+#define OV7740_MAX_REGISTER     0xff
+static const struct regmap_config ov7740_regmap_config = {
+	.reg_bits	= 8,
+	.val_bits	= 8,
+	.max_register	= OV7740_MAX_REGISTER,
+};
+
+static int ov7740_probe(struct i2c_client *client,
+			const struct i2c_device_id *id)
+{
+	struct ov7740 *ov7740;
+	struct v4l2_subdev *sd;
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_BYTE_DATA)) {
+		dev_err(&client->dev,
+			"OV7740: I2C-Adapter doesn't support SMBUS\n");
+		return -EIO;
+	}
+
+	ov7740 = devm_kzalloc(&client->dev, sizeof(*ov7740), GFP_KERNEL);
+	if (!ov7740)
+		return -ENOMEM;
+
+	ov7740->xvclk = devm_clk_get(&client->dev, "xvclk");
+	if (IS_ERR(ov7740->xvclk)) {
+		ret = PTR_ERR(ov7740->xvclk);
+		dev_err(&client->dev,
+			"OV7740: fail to get xvclk: %d\n", ret);
+		return ret;
+	}
+
+	ret = ov7740_probe_dt(client, ov7740);
+	if (ret)
+		return ret;
+
+	ov7740->regmap = devm_regmap_init_i2c(client, &ov7740_regmap_config);
+	if (IS_ERR(ov7740->regmap)) {
+		ret = PTR_ERR(ov7740->regmap);
+		dev_err(&client->dev, "Failed to allocate register map: %d\n",
+			ret);
+		return ret;
+	}
+
+	sd = &ov7740->subdev;
+	client->flags |= I2C_CLIENT_SCCB;
+	v4l2_i2c_subdev_init(sd, client, &ov7740_subdev_ops);
+
+#ifdef CONFIG_VIDEO_V4L2_SUBDEV_API
+	sd->internal_ops = &ov7740_subdev_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+#endif
+
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	ov7740->pad.flags = MEDIA_PAD_FL_SOURCE;
+	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
+	ret = media_entity_pads_init(&sd->entity, 1, &ov7740->pad);
+	if (ret)
+		return ret;
+#endif
+
+	ret = ov7740_set_power(ov7740, 1);
+	if (ret)
+		return ret;
+
+	ret = ov7740_detect(ov7740);
+	if (ret)
+		goto error_detect;
+
+	mutex_init(&ov7740->mutex);
+
+	ret = ov7740_init_controls(ov7740);
+	if (ret)
+		goto error_init_controls;
+
+	v4l_info(client, "chip found @ 0x%02x (%s)\n",
+			client->addr << 1, client->adapter->name);
+
+	ov7740->fmt = &ov7740_formats[0];
+	ov7740->frmsize = &ov7740_framesizes[0];
+
+	ov7740_get_default_format(sd, &ov7740->format);
+
+	ret = v4l2_async_register_subdev(sd);
+	if (ret)
+		goto error_async_register;
+
+	pm_runtime_set_active(&client->dev);
+	pm_runtime_enable(&client->dev);
+	pm_runtime_idle(&client->dev);
+
+	return 0;
+
+error_async_register:
+	v4l2_ctrl_handler_free(ov7740->subdev.ctrl_handler);
+error_init_controls:
+	ov7740_free_controls(ov7740);
+error_detect:
+	ov7740_set_power(ov7740, 0);
+	media_entity_cleanup(&ov7740->subdev.entity);
+
+	return ret;
+}
+
+static int ov7740_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+
+	mutex_destroy(&ov7740->mutex);
+	v4l2_ctrl_handler_free(ov7740->subdev.ctrl_handler);
+	media_entity_cleanup(&ov7740->subdev.entity);
+	v4l2_async_unregister_subdev(sd);
+	ov7740_free_controls(ov7740);
+
+	pm_runtime_get_sync(&client->dev);
+	pm_runtime_disable(&client->dev);
+	pm_runtime_set_suspended(&client->dev);
+	pm_runtime_put_noidle(&client->dev);
+
+	ov7740_set_power(ov7740, 0);
+	return 0;
+}
+
+static int __maybe_unused ov7740_runtime_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+
+	ov7740_set_power(ov7740, 0);
+
+	return 0;
+}
+
+static int __maybe_unused ov7740_runtime_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
+
+	return ov7740_set_power(ov7740, 1);
+}
+
+static const struct i2c_device_id ov7740_id[] = {
+	{ "ov7740", 0 },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(i2c, ov7740_id);
+
+static const struct dev_pm_ops ov7740_pm_ops = {
+	SET_RUNTIME_PM_OPS(ov7740_runtime_suspend, ov7740_runtime_resume, NULL)
+};
+
+static const struct of_device_id ov7740_of_match[] = {
+	{.compatible = "ovti,ov7740", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, ov7740_of_match);
+
+static struct i2c_driver ov7740_i2c_driver = {
+	.driver = {
+		.name = "ov7740",
+		.pm = &ov7740_pm_ops,
+		.of_match_table = of_match_ptr(ov7740_of_match),
+	},
+	.probe    = ov7740_probe,
+	.remove   = ov7740_remove,
+	.id_table = ov7740_id,
+};
+module_i2c_driver(ov7740_i2c_driver);
+
+MODULE_DESCRIPTION("The V4L2 driver for Omnivision 7740 sensor");
+MODULE_AUTHOR("Songjun Wu <songjun.wu@atmel.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c
index 69433e1e2533..e519f278d5f9 100644
--- a/drivers/media/i2c/ov9650.c
+++ b/drivers/media/i2c/ov9650.c
@@ -886,10 +886,12 @@ static int __g_volatile_ctrl(struct ov965x *ov965x, struct v4l2_ctrl *ctrl)
 		if (ctrl->val == V4L2_EXPOSURE_MANUAL)
 			return 0;
 		ret = ov965x_read(client, REG_COM1, &reg0);
-		if (!ret)
-			ret = ov965x_read(client, REG_AECH, &reg1);
-		if (!ret)
-			ret = ov965x_read(client, REG_AECHM, &reg2);
+		if (ret < 0)
+			return ret;
+		ret = ov965x_read(client, REG_AECH, &reg1);
+		if (ret < 0)
+			return ret;
+		ret = ov965x_read(client, REG_AECHM, &reg2);
 		if (ret < 0)
 			return ret;
 		exposure = ((reg2 & 0x3f) << 10) | (reg1 << 2) |
diff --git a/drivers/media/i2c/saa6588.c b/drivers/media/i2c/saa6588.c
index 00640233a5e3..c3089bd34df2 100644
--- a/drivers/media/i2c/saa6588.c
+++ b/drivers/media/i2c/saa6588.c
@@ -413,7 +413,7 @@ static long saa6588_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
 	case SAA6588_CMD_POLL:
 		a->result = 0;
 		if (s->data_available_for_read)
-			a->result |= POLLIN | POLLRDNORM;
+			a->result |= EPOLLIN | EPOLLRDNORM;
 		poll_wait(a->instance, &s->read_queue, a->event_list);
 		break;
 
diff --git a/drivers/media/i2c/saa6752hs.c b/drivers/media/i2c/saa6752hs.c
index 7202d3a3219a..170cc65c4f23 100644
--- a/drivers/media/i2c/saa6752hs.c
+++ b/drivers/media/i2c/saa6752hs.c
@@ -72,8 +72,8 @@ struct saa6752hs_mpeg_params {
 	/* video */
 	enum v4l2_mpeg_video_aspect	vi_aspect;
 	enum v4l2_mpeg_video_bitrate_mode vi_bitrate_mode;
-	__u32 				vi_bitrate;
-	__u32 				vi_bitrate_peak;
+	__u32				vi_bitrate;
+	__u32				vi_bitrate_peak;
 };
 
 static const struct v4l2_format v4l2_format_table[] =
@@ -98,8 +98,8 @@ struct saa6752hs_state {
 		struct v4l2_ctrl *video_bitrate;
 		struct v4l2_ctrl *video_bitrate_peak;
 	};
-	u32 			      revision;
-	int 			      has_ac3;
+	u32			      revision;
+	int			      has_ac3;
 	struct saa6752hs_mpeg_params  params;
 	enum saa6752hs_videoformat    video_format;
 	v4l2_std_id                   standard;
diff --git a/drivers/media/i2c/saa7115.c b/drivers/media/i2c/saa7115.c
index d863b04aa2a8..e216cd768409 100644
--- a/drivers/media/i2c/saa7115.c
+++ b/drivers/media/i2c/saa7115.c
@@ -1,37 +1,27 @@
-/* saa711x - Philips SAA711x video decoder driver
- * This driver can work with saa7111, saa7111a, saa7113, saa7114,
- *			     saa7115 and saa7118.
- *
- * Based on saa7114 driver by Maxim Yevtyushkin, which is based on
- * the saa7111 driver by Dave Perks.
- *
- * Copyright (C) 1998 Dave Perks <dperks@ibm.net>
- * Copyright (C) 2002 Maxim Yevtyushkin <max@linuxmedialabs.com>
- *
- * Slight changes for video timing and attachment output by
- * Wolfgang Scherr <scherr@net4you.net>
- *
- * Moved over to the linux >= 2.4.x i2c protocol (1/1/2003)
- * by Ronald Bultje <rbultje@ronald.bitfreak.net>
- *
- * Added saa7115 support by Kevin Thayer <nufan_wfk at yahoo.com>
- * (2/17/2003)
- *
- * VBI support (2004) and cleanups (2005) by Hans Verkuil <hverkuil@xs4all.nl>
- *
- * Copyright (c) 2005-2006 Mauro Carvalho Chehab <mchehab@infradead.org>
- *	SAA7111, SAA7113 and SAA7118 support
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// saa711x - Philips SAA711x video decoder driver
+// This driver can work with saa7111, saa7111a, saa7113, saa7114,
+//			     saa7115 and saa7118.
+//
+// Based on saa7114 driver by Maxim Yevtyushkin, which is based on
+// the saa7111 driver by Dave Perks.
+//
+// Copyright (C) 1998 Dave Perks <dperks@ibm.net>
+// Copyright (C) 2002 Maxim Yevtyushkin <max@linuxmedialabs.com>
+//
+// Slight changes for video timing and attachment output by
+// Wolfgang Scherr <scherr@net4you.net>
+//
+// Moved over to the linux >= 2.4.x i2c protocol (1/1/2003)
+// by Ronald Bultje <rbultje@ronald.bitfreak.net>
+//
+// Added saa7115 support by Kevin Thayer <nufan_wfk at yahoo.com>
+// (2/17/2003)
+//
+// VBI support (2004) and cleanups (2005) by Hans Verkuil <hverkuil@xs4all.nl>
+//
+// Copyright (c) 2005-2006 Mauro Carvalho Chehab <mchehab@infradead.org>
+//	SAA7111, SAA7113 and SAA7118 support
 
 #include "saa711x_regs.h"
 
@@ -758,7 +748,7 @@ static int saa711x_s_clock_freq(struct v4l2_subdev *sd, u32 freq)
 	u32 acni;
 	u32 hz;
 	u64 f;
-	u8 acc = 0; 	/* reg 0x3a, audio clock control */
+	u8 acc = 0;	/* reg 0x3a, audio clock control */
 
 	/* Checks for chips that don't have audio clock (saa7111, saa7113) */
 	if (!saa711x_has_reg(state->ident, R_30_AUD_MAST_CLK_CYCLES_PER_FIELD))
diff --git a/drivers/media/i2c/saa711x_regs.h b/drivers/media/i2c/saa711x_regs.h
index 730ca90b30ac..a50d480e101a 100644
--- a/drivers/media/i2c/saa711x_regs.h
+++ b/drivers/media/i2c/saa711x_regs.h
@@ -1,16 +1,8 @@
-/* saa711x - Philips SAA711x video decoder register specifications
+/*
+ * SPDX-License-Identifier: GPL-2.0+
+ * saa711x - Philips SAA711x video decoder register specifications
  *
  * Copyright (c) 2006 Mauro Carvalho Chehab <mchehab@infradead.org>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #define R_00_CHIP_VERSION                             0x00
diff --git a/drivers/media/i2c/saa7127.c b/drivers/media/i2c/saa7127.c
index 01784d441ae6..e58a150cec5c 100644
--- a/drivers/media/i2c/saa7127.c
+++ b/drivers/media/i2c/saa7127.c
@@ -132,109 +132,109 @@ struct i2c_reg_value {
 };
 
 static const struct i2c_reg_value saa7129_init_config_extra[] = {
-	{ SAA7127_REG_OUTPUT_PORT_CONTROL, 		0x38 },
-	{ SAA7127_REG_VTRIG, 				0xfa },
+	{ SAA7127_REG_OUTPUT_PORT_CONTROL,		0x38 },
+	{ SAA7127_REG_VTRIG,				0xfa },
 	{ 0, 0 }
 };
 
 static const struct i2c_reg_value saa7127_init_config_common[] = {
-	{ SAA7127_REG_WIDESCREEN_CONFIG, 		0x0d },
-	{ SAA7127_REG_WIDESCREEN_ENABLE, 		0x00 },
-	{ SAA7127_REG_COPYGEN_0, 			0x77 },
-	{ SAA7127_REG_COPYGEN_1, 			0x41 },
-	{ SAA7127_REG_COPYGEN_2, 			0x00 },	/* Macrovision enable/disable */
-	{ SAA7127_REG_OUTPUT_PORT_CONTROL, 		0xbf },
-	{ SAA7127_REG_GAIN_LUMINANCE_RGB, 		0x00 },
-	{ SAA7127_REG_GAIN_COLORDIFF_RGB, 		0x00 },
-	{ SAA7127_REG_INPUT_PORT_CONTROL_1, 		0x80 },	/* for color bars */
-	{ SAA7127_REG_LINE_21_ODD_0, 			0x77 },
-	{ SAA7127_REG_LINE_21_ODD_1, 			0x41 },
-	{ SAA7127_REG_LINE_21_EVEN_0, 			0x88 },
-	{ SAA7127_REG_LINE_21_EVEN_1, 			0x41 },
-	{ SAA7127_REG_RCV_PORT_CONTROL, 		0x12 },
-	{ SAA7127_REG_VTRIG, 				0xf9 },
-	{ SAA7127_REG_HTRIG_HI, 			0x00 },
-	{ SAA7127_REG_RCV2_OUTPUT_START, 		0x41 },
-	{ SAA7127_REG_RCV2_OUTPUT_END, 			0xc3 },
-	{ SAA7127_REG_RCV2_OUTPUT_MSBS, 		0x00 },
-	{ SAA7127_REG_TTX_REQUEST_H_START, 		0x3e },
-	{ SAA7127_REG_TTX_REQUEST_H_DELAY_LENGTH, 	0xb8 },
-	{ SAA7127_REG_CSYNC_ADVANCE_VSYNC_SHIFT,  	0x03 },
-	{ SAA7127_REG_TTX_ODD_REQ_VERT_START, 		0x15 },
-	{ SAA7127_REG_TTX_ODD_REQ_VERT_END, 		0x16 },
-	{ SAA7127_REG_TTX_EVEN_REQ_VERT_START, 		0x15 },
-	{ SAA7127_REG_TTX_EVEN_REQ_VERT_END, 		0x16 },
-	{ SAA7127_REG_FIRST_ACTIVE, 			0x1a },
-	{ SAA7127_REG_LAST_ACTIVE, 			0x01 },
-	{ SAA7127_REG_MSB_VERTICAL, 			0xc0 },
-	{ SAA7127_REG_DISABLE_TTX_LINE_LO_0, 		0x00 },
-	{ SAA7127_REG_DISABLE_TTX_LINE_LO_1, 		0x00 },
+	{ SAA7127_REG_WIDESCREEN_CONFIG,		0x0d },
+	{ SAA7127_REG_WIDESCREEN_ENABLE,		0x00 },
+	{ SAA7127_REG_COPYGEN_0,			0x77 },
+	{ SAA7127_REG_COPYGEN_1,			0x41 },
+	{ SAA7127_REG_COPYGEN_2,			0x00 },	/* Macrovision enable/disable */
+	{ SAA7127_REG_OUTPUT_PORT_CONTROL,		0xbf },
+	{ SAA7127_REG_GAIN_LUMINANCE_RGB,		0x00 },
+	{ SAA7127_REG_GAIN_COLORDIFF_RGB,		0x00 },
+	{ SAA7127_REG_INPUT_PORT_CONTROL_1,		0x80 },	/* for color bars */
+	{ SAA7127_REG_LINE_21_ODD_0,			0x77 },
+	{ SAA7127_REG_LINE_21_ODD_1,			0x41 },
+	{ SAA7127_REG_LINE_21_EVEN_0,			0x88 },
+	{ SAA7127_REG_LINE_21_EVEN_1,			0x41 },
+	{ SAA7127_REG_RCV_PORT_CONTROL,			0x12 },
+	{ SAA7127_REG_VTRIG,				0xf9 },
+	{ SAA7127_REG_HTRIG_HI,				0x00 },
+	{ SAA7127_REG_RCV2_OUTPUT_START,		0x41 },
+	{ SAA7127_REG_RCV2_OUTPUT_END,			0xc3 },
+	{ SAA7127_REG_RCV2_OUTPUT_MSBS,			0x00 },
+	{ SAA7127_REG_TTX_REQUEST_H_START,		0x3e },
+	{ SAA7127_REG_TTX_REQUEST_H_DELAY_LENGTH,	0xb8 },
+	{ SAA7127_REG_CSYNC_ADVANCE_VSYNC_SHIFT,	0x03 },
+	{ SAA7127_REG_TTX_ODD_REQ_VERT_START,		0x15 },
+	{ SAA7127_REG_TTX_ODD_REQ_VERT_END,		0x16 },
+	{ SAA7127_REG_TTX_EVEN_REQ_VERT_START,		0x15 },
+	{ SAA7127_REG_TTX_EVEN_REQ_VERT_END,		0x16 },
+	{ SAA7127_REG_FIRST_ACTIVE,			0x1a },
+	{ SAA7127_REG_LAST_ACTIVE,			0x01 },
+	{ SAA7127_REG_MSB_VERTICAL,			0xc0 },
+	{ SAA7127_REG_DISABLE_TTX_LINE_LO_0,		0x00 },
+	{ SAA7127_REG_DISABLE_TTX_LINE_LO_1,		0x00 },
 	{ 0, 0 }
 };
 
 #define SAA7127_60HZ_DAC_CONTROL 0x15
 static const struct i2c_reg_value saa7127_init_config_60hz[] = {
-	{ SAA7127_REG_BURST_START, 			0x19 },
+	{ SAA7127_REG_BURST_START,			0x19 },
 	/* BURST_END is also used as a chip ID in saa7127_probe */
-	{ SAA7127_REG_BURST_END, 			0x1d },
-	{ SAA7127_REG_CHROMA_PHASE, 			0xa3 },
-	{ SAA7127_REG_GAINU, 				0x98 },
-	{ SAA7127_REG_GAINV, 				0xd3 },
-	{ SAA7127_REG_BLACK_LEVEL, 			0x39 },
-	{ SAA7127_REG_BLANKING_LEVEL, 			0x2e },
-	{ SAA7127_REG_VBI_BLANKING, 			0x2e },
-	{ SAA7127_REG_DAC_CONTROL, 			0x15 },
-	{ SAA7127_REG_BURST_AMP, 			0x4d },
-	{ SAA7127_REG_SUBC3, 				0x1f },
-	{ SAA7127_REG_SUBC2, 				0x7c },
-	{ SAA7127_REG_SUBC1, 				0xf0 },
-	{ SAA7127_REG_SUBC0, 				0x21 },
-	{ SAA7127_REG_MULTI, 				0x90 },
-	{ SAA7127_REG_CLOSED_CAPTION, 			0x11 },
+	{ SAA7127_REG_BURST_END,			0x1d },
+	{ SAA7127_REG_CHROMA_PHASE,			0xa3 },
+	{ SAA7127_REG_GAINU,				0x98 },
+	{ SAA7127_REG_GAINV,				0xd3 },
+	{ SAA7127_REG_BLACK_LEVEL,			0x39 },
+	{ SAA7127_REG_BLANKING_LEVEL,			0x2e },
+	{ SAA7127_REG_VBI_BLANKING,			0x2e },
+	{ SAA7127_REG_DAC_CONTROL,			0x15 },
+	{ SAA7127_REG_BURST_AMP,			0x4d },
+	{ SAA7127_REG_SUBC3,				0x1f },
+	{ SAA7127_REG_SUBC2,				0x7c },
+	{ SAA7127_REG_SUBC1,				0xf0 },
+	{ SAA7127_REG_SUBC0,				0x21 },
+	{ SAA7127_REG_MULTI,				0x90 },
+	{ SAA7127_REG_CLOSED_CAPTION,			0x11 },
 	{ 0, 0 }
 };
 
 #define SAA7127_50HZ_PAL_DAC_CONTROL 0x02
 static struct i2c_reg_value saa7127_init_config_50hz_pal[] = {
-	{ SAA7127_REG_BURST_START, 			0x21 },
+	{ SAA7127_REG_BURST_START,			0x21 },
 	/* BURST_END is also used as a chip ID in saa7127_probe */
-	{ SAA7127_REG_BURST_END, 			0x1d },
-	{ SAA7127_REG_CHROMA_PHASE, 			0x3f },
-	{ SAA7127_REG_GAINU, 				0x7d },
-	{ SAA7127_REG_GAINV, 				0xaf },
-	{ SAA7127_REG_BLACK_LEVEL, 			0x33 },
-	{ SAA7127_REG_BLANKING_LEVEL, 			0x35 },
-	{ SAA7127_REG_VBI_BLANKING, 			0x35 },
-	{ SAA7127_REG_DAC_CONTROL, 			0x02 },
-	{ SAA7127_REG_BURST_AMP, 			0x2f },
-	{ SAA7127_REG_SUBC3, 				0xcb },
-	{ SAA7127_REG_SUBC2, 				0x8a },
-	{ SAA7127_REG_SUBC1, 				0x09 },
-	{ SAA7127_REG_SUBC0, 				0x2a },
-	{ SAA7127_REG_MULTI, 				0xa0 },
-	{ SAA7127_REG_CLOSED_CAPTION, 			0x00 },
+	{ SAA7127_REG_BURST_END,			0x1d },
+	{ SAA7127_REG_CHROMA_PHASE,			0x3f },
+	{ SAA7127_REG_GAINU,				0x7d },
+	{ SAA7127_REG_GAINV,				0xaf },
+	{ SAA7127_REG_BLACK_LEVEL,			0x33 },
+	{ SAA7127_REG_BLANKING_LEVEL,			0x35 },
+	{ SAA7127_REG_VBI_BLANKING,			0x35 },
+	{ SAA7127_REG_DAC_CONTROL,			0x02 },
+	{ SAA7127_REG_BURST_AMP,			0x2f },
+	{ SAA7127_REG_SUBC3,				0xcb },
+	{ SAA7127_REG_SUBC2,				0x8a },
+	{ SAA7127_REG_SUBC1,				0x09 },
+	{ SAA7127_REG_SUBC0,				0x2a },
+	{ SAA7127_REG_MULTI,				0xa0 },
+	{ SAA7127_REG_CLOSED_CAPTION,			0x00 },
 	{ 0, 0 }
 };
 
 #define SAA7127_50HZ_SECAM_DAC_CONTROL 0x08
 static struct i2c_reg_value saa7127_init_config_50hz_secam[] = {
-	{ SAA7127_REG_BURST_START, 			0x21 },
+	{ SAA7127_REG_BURST_START,			0x21 },
 	/* BURST_END is also used as a chip ID in saa7127_probe */
-	{ SAA7127_REG_BURST_END, 			0x1d },
-	{ SAA7127_REG_CHROMA_PHASE, 			0x3f },
-	{ SAA7127_REG_GAINU, 				0x6a },
-	{ SAA7127_REG_GAINV, 				0x81 },
-	{ SAA7127_REG_BLACK_LEVEL, 			0x33 },
-	{ SAA7127_REG_BLANKING_LEVEL, 			0x35 },
-	{ SAA7127_REG_VBI_BLANKING, 			0x35 },
-	{ SAA7127_REG_DAC_CONTROL, 			0x08 },
-	{ SAA7127_REG_BURST_AMP, 			0x2f },
-	{ SAA7127_REG_SUBC3, 				0xb2 },
-	{ SAA7127_REG_SUBC2, 				0x3b },
-	{ SAA7127_REG_SUBC1, 				0xa3 },
-	{ SAA7127_REG_SUBC0, 				0x28 },
-	{ SAA7127_REG_MULTI, 				0x90 },
-	{ SAA7127_REG_CLOSED_CAPTION, 			0x00 },
+	{ SAA7127_REG_BURST_END,			0x1d },
+	{ SAA7127_REG_CHROMA_PHASE,			0x3f },
+	{ SAA7127_REG_GAINU,				0x6a },
+	{ SAA7127_REG_GAINV,				0x81 },
+	{ SAA7127_REG_BLACK_LEVEL,			0x33 },
+	{ SAA7127_REG_BLANKING_LEVEL,			0x35 },
+	{ SAA7127_REG_VBI_BLANKING,			0x35 },
+	{ SAA7127_REG_DAC_CONTROL,			0x08 },
+	{ SAA7127_REG_BURST_AMP,			0x2f },
+	{ SAA7127_REG_SUBC3,				0xb2 },
+	{ SAA7127_REG_SUBC2,				0x3b },
+	{ SAA7127_REG_SUBC1,				0xa3 },
+	{ SAA7127_REG_SUBC0,				0x28 },
+	{ SAA7127_REG_MULTI,				0x90 },
+	{ SAA7127_REG_CLOSED_CAPTION,			0x00 },
 	{ 0, 0 }
 };
 
diff --git a/drivers/media/i2c/saa717x.c b/drivers/media/i2c/saa717x.c
index 102467e00fb3..668c39cc29e8 100644
--- a/drivers/media/i2c/saa717x.c
+++ b/drivers/media/i2c/saa717x.c
@@ -82,13 +82,13 @@ static inline struct v4l2_subdev *to_sd(struct v4l2_ctrl *ctrl)
 /* ----------------------------------------------------------------------- */
 
 /* for audio mode */
-#define TUNER_AUDIO_MONO   	0  /* LL */
-#define TUNER_AUDIO_STEREO 	1  /* LR */
-#define TUNER_AUDIO_LANG1  	2  /* LL */
-#define TUNER_AUDIO_LANG2  	3  /* RR */
+#define TUNER_AUDIO_MONO	0  /* LL */
+#define TUNER_AUDIO_STEREO	1  /* LR */
+#define TUNER_AUDIO_LANG1	2  /* LL */
+#define TUNER_AUDIO_LANG2	3  /* RR */
 
-#define SAA717X_NTSC_WIDTH   	(704)
-#define SAA717X_NTSC_HEIGHT  	(480)
+#define SAA717X_NTSC_WIDTH	(704)
+#define SAA717X_NTSC_HEIGHT	(480)
 
 /* ----------------------------------------------------------------------- */
 
diff --git a/drivers/media/i2c/smiapp/smiapp-core.c b/drivers/media/i2c/smiapp/smiapp-core.c
index e6b717b83b18..3b7ace395ee6 100644
--- a/drivers/media/i2c/smiapp/smiapp-core.c
+++ b/drivers/media/i2c/smiapp/smiapp-core.c
@@ -1791,7 +1791,7 @@ static int smiapp_set_format_source(struct v4l2_subdev *subdev,
 	if (csi_format->compressed == old_csi_format->compressed)
 		return 0;
 
-	valid_link_freqs = 
+	valid_link_freqs =
 		&sensor->valid_link_freqs[sensor->csi_format->compressed
 					  - sensor->compressed_min_bpp];
 
diff --git a/drivers/media/i2c/tda7432.c b/drivers/media/i2c/tda7432.c
index d87168adee45..1c5c61d829d6 100644
--- a/drivers/media/i2c/tda7432.c
+++ b/drivers/media/i2c/tda7432.c
@@ -36,7 +36,6 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-ctrls.h>
-#include <media/i2c-addr.h>
 
 #ifndef VIDEO_AUDIO_BALANCE
 # define VIDEO_AUDIO_BALANCE 32
diff --git a/drivers/media/i2c/ths7303.c b/drivers/media/i2c/ths7303.c
index 71a31352135c..8206bf7a5a8f 100644
--- a/drivers/media/i2c/ths7303.c
+++ b/drivers/media/i2c/ths7303.c
@@ -319,7 +319,7 @@ static const struct v4l2_subdev_core_ops ths7303_core_ops = {
 
 static const struct v4l2_subdev_ops ths7303_ops = {
 	.core	= &ths7303_core_ops,
-	.video 	= &ths7303_video_ops,
+	.video	= &ths7303_video_ops,
 };
 
 static int ths7303_probe(struct i2c_client *client,
diff --git a/drivers/media/i2c/tvaudio.c b/drivers/media/i2c/tvaudio.c
index 16a1e08ce06c..772164b848ef 100644
--- a/drivers/media/i2c/tvaudio.c
+++ b/drivers/media/i2c/tvaudio.c
@@ -40,8 +40,6 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-ctrls.h>
 
-#include <media/i2c-addr.h>
-
 /* ---------------------------------------------------------------------- */
 /* insmod args                                                            */
 
@@ -136,7 +134,7 @@ struct CHIPSTATE {
 	/* thread */
 	struct task_struct   *thread;
 	struct timer_list    wt;
-	int 		     audmode;
+	int		     audmode;
 };
 
 static inline struct CHIPSTATE *to_state(struct v4l2_subdev *sd)
diff --git a/drivers/media/i2c/tvp514x.c b/drivers/media/i2c/tvp514x.c
index d575b3e7e835..8b0aa9297bde 100644
--- a/drivers/media/i2c/tvp514x.c
+++ b/drivers/media/i2c/tvp514x.c
@@ -1131,9 +1131,7 @@ tvp514x_probe(struct i2c_client *client, const struct i2c_device_id *id)
 done:
 	if (ret < 0) {
 		v4l2_ctrl_handler_free(&decoder->hdl);
-#if defined(CONFIG_MEDIA_CONTROLLER)
 		media_entity_cleanup(&decoder->sd.entity);
-#endif
 	}
 	return ret;
 }
@@ -1151,9 +1149,7 @@ static int tvp514x_remove(struct i2c_client *client)
 	struct tvp514x_decoder *decoder = to_decoder(sd);
 
 	v4l2_async_unregister_subdev(&decoder->sd);
-#if defined(CONFIG_MEDIA_CONTROLLER)
 	media_entity_cleanup(&decoder->sd.entity);
-#endif
 	v4l2_ctrl_handler_free(&decoder->hdl);
 	return 0;
 }
diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c
index 7b79a7498751..2476d812f669 100644
--- a/drivers/media/i2c/tvp5150.c
+++ b/drivers/media/i2c/tvp5150.c
@@ -1,9 +1,8 @@
-/*
- * tvp5150 - Texas Instruments TVP5150A/AM1 and TVP5151 video decoder driver
- *
- * Copyright (c) 2005,2006 Mauro Carvalho Chehab (mchehab@infradead.org)
- * This code is placed under the terms of the GNU General Public License v2
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// tvp5150 - Texas Instruments TVP5150A/AM1 and TVP5151 video decoder driver
+//
+// Copyright (c) 2005,2006 Mauro Carvalho Chehab <mchehab@infradead.org>
 
 #include <dt-bindings/media/tvp5150.h>
 #include <linux/i2c.h>
@@ -30,7 +29,7 @@
 
 MODULE_DESCRIPTION("Texas Instruments TVP5150A/TVP5150AM1/TVP5151 video decoder driver");
 MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
 
 
 static int debug;
@@ -506,80 +505,77 @@ static struct i2c_vbi_ram_value vbi_ram_default[] =
 	/* FIXME: Current api doesn't handle all VBI types, those not
 	   yet supported are placed under #if 0 */
 #if 0
-	{0x010, /* Teletext, SECAM, WST System A */
+	[0] = {0x010, /* Teletext, SECAM, WST System A */
 		{V4L2_SLICED_TELETEXT_SECAM,6,23,1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xe7, 0x2e, 0x20, 0x26,
 		  0xe6, 0xb4, 0x0e, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
 #endif
-	{0x030, /* Teletext, PAL, WST System B */
+	[1] = {0x030, /* Teletext, PAL, WST System B */
 		{V4L2_SLICED_TELETEXT_B,6,22,1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0x27, 0x2e, 0x20, 0x2b,
 		  0xa6, 0x72, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
 #if 0
-	{0x050, /* Teletext, PAL, WST System C */
+	[2] = {0x050, /* Teletext, PAL, WST System C */
 		{V4L2_SLICED_TELETEXT_PAL_C,6,22,1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xe7, 0x2e, 0x20, 0x22,
 		  0xa6, 0x98, 0x0d, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
-	{0x070, /* Teletext, NTSC, WST System B */
+	[3] = {0x070, /* Teletext, NTSC, WST System B */
 		{V4L2_SLICED_TELETEXT_NTSC_B,10,21,1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0x27, 0x2e, 0x20, 0x23,
 		  0x69, 0x93, 0x0d, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
-	{0x090, /* Tetetext, NTSC NABTS System C */
+	[4] = {0x090, /* Tetetext, NTSC NABTS System C */
 		{V4L2_SLICED_TELETEXT_NTSC_C,10,21,1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xe7, 0x2e, 0x20, 0x22,
 		  0x69, 0x93, 0x0d, 0x00, 0x00, 0x00, 0x15, 0x00 }
 	},
-	{0x0b0, /* Teletext, NTSC-J, NABTS System D */
+	[5] = {0x0b0, /* Teletext, NTSC-J, NABTS System D */
 		{V4L2_SLICED_TELETEXT_NTSC_D,10,21,1},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xa7, 0x2e, 0x20, 0x23,
 		  0x69, 0x93, 0x0d, 0x00, 0x00, 0x00, 0x10, 0x00 }
 	},
-	{0x0d0, /* Closed Caption, PAL/SECAM */
+	[6] = {0x0d0, /* Closed Caption, PAL/SECAM */
 		{V4L2_SLICED_CAPTION_625,22,22,1},
 		{ 0xaa, 0x2a, 0xff, 0x3f, 0x04, 0x51, 0x6e, 0x02,
 		  0xa6, 0x7b, 0x09, 0x00, 0x00, 0x00, 0x27, 0x00 }
 	},
 #endif
-	{0x0f0, /* Closed Caption, NTSC */
+	[7] = {0x0f0, /* Closed Caption, NTSC */
 		{V4L2_SLICED_CAPTION_525,21,21,1},
 		{ 0xaa, 0x2a, 0xff, 0x3f, 0x04, 0x51, 0x6e, 0x02,
 		  0x69, 0x8c, 0x09, 0x00, 0x00, 0x00, 0x27, 0x00 }
 	},
-	{0x110, /* Wide Screen Signal, PAL/SECAM */
+	[8] = {0x110, /* Wide Screen Signal, PAL/SECAM */
 		{V4L2_SLICED_WSS_625,23,23,1},
 		{ 0x5b, 0x55, 0xc5, 0xff, 0x00, 0x71, 0x6e, 0x42,
 		  0xa6, 0xcd, 0x0f, 0x00, 0x00, 0x00, 0x3a, 0x00 }
 	},
 #if 0
-	{0x130, /* Wide Screen Signal, NTSC C */
+	[9] = {0x130, /* Wide Screen Signal, NTSC C */
 		{V4L2_SLICED_WSS_525,20,20,1},
 		{ 0x38, 0x00, 0x3f, 0x00, 0x00, 0x71, 0x6e, 0x43,
 		  0x69, 0x7c, 0x08, 0x00, 0x00, 0x00, 0x39, 0x00 }
 	},
-	{0x150, /* Vertical Interval Timecode (VITC), PAL/SECAM */
+	[10] = {0x150, /* Vertical Interval Timecode (VITC), PAL/SECAM */
 		{V4l2_SLICED_VITC_625,6,22,0},
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x6d, 0x49,
 		  0xa6, 0x85, 0x08, 0x00, 0x00, 0x00, 0x4c, 0x00 }
 	},
-	{0x170, /* Vertical Interval Timecode (VITC), NTSC */
+	[11] = {0x170, /* Vertical Interval Timecode (VITC), NTSC */
 		{V4l2_SLICED_VITC_525,10,20,0},
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x8f, 0x6d, 0x49,
 		  0x69, 0x94, 0x08, 0x00, 0x00, 0x00, 0x4c, 0x00 }
 	},
 #endif
-	{0x190, /* Video Program System (VPS), PAL */
+	[12] = {0x190, /* Video Program System (VPS), PAL */
 		{V4L2_SLICED_VPS,16,16,0},
 		{ 0xaa, 0xaa, 0xff, 0xff, 0xba, 0xce, 0x2b, 0x0d,
 		  0xa6, 0xda, 0x0b, 0x00, 0x00, 0x00, 0x60, 0x00 }
 	},
 	/* 0x1d0 User programmable */
-
-	/* End of struct */
-	{ (u16)-1 }
 };
 
 static int tvp5150_write_inittab(struct v4l2_subdev *sd,
@@ -592,10 +588,10 @@ static int tvp5150_write_inittab(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int tvp5150_vdp_init(struct v4l2_subdev *sd,
-				const struct i2c_vbi_ram_value *regs)
+static int tvp5150_vdp_init(struct v4l2_subdev *sd)
 {
 	unsigned int i;
+	int j;
 
 	/* Disable Full Field */
 	tvp5150_write(sd, TVP5150_FULL_FIELD_ENA, 0);
@@ -605,14 +601,17 @@ static int tvp5150_vdp_init(struct v4l2_subdev *sd,
 		tvp5150_write(sd, i, 0xff);
 
 	/* Load Ram Table */
-	while (regs->reg != (u16)-1) {
+	for (j = 0; j < ARRAY_SIZE(vbi_ram_default); j++) {
+		const struct i2c_vbi_ram_value *regs = &vbi_ram_default[j];
+
+		if (!regs->type.vbi_type)
+			continue;
+
 		tvp5150_write(sd, TVP5150_CONF_RAM_ADDR_HIGH, regs->reg >> 8);
 		tvp5150_write(sd, TVP5150_CONF_RAM_ADDR_LOW, regs->reg);
 
 		for (i = 0; i < 16; i++)
 			tvp5150_write(sd, TVP5150_VDP_CONF_RAM_DATA, regs->values[i]);
-
-		regs++;
 	}
 	return 0;
 }
@@ -621,19 +620,23 @@ static int tvp5150_vdp_init(struct v4l2_subdev *sd,
 static int tvp5150_g_sliced_vbi_cap(struct v4l2_subdev *sd,
 				struct v4l2_sliced_vbi_cap *cap)
 {
-	const struct i2c_vbi_ram_value *regs = vbi_ram_default;
-	int line;
+	int line, i;
 
 	dev_dbg_lvl(sd->dev, 1, debug, "g_sliced_vbi_cap\n");
 	memset(cap, 0, sizeof *cap);
 
-	while (regs->reg != (u16)-1 ) {
-		for (line=regs->type.ini_line;line<=regs->type.end_line;line++) {
+	for (i = 0; i < ARRAY_SIZE(vbi_ram_default); i++) {
+		const struct i2c_vbi_ram_value *regs = &vbi_ram_default[i];
+
+		if (!regs->type.vbi_type)
+			continue;
+
+		for (line = regs->type.ini_line;
+		     line <= regs->type.end_line;
+		     line++) {
 			cap->service_lines[0][line] |= regs->type.vbi_type;
 		}
 		cap->service_set |= regs->type.vbi_type;
-
-		regs++;
 	}
 	return 0;
 }
@@ -652,14 +655,13 @@ static int tvp5150_g_sliced_vbi_cap(struct v4l2_subdev *sd,
  *	MSB = field2
  */
 static int tvp5150_set_vbi(struct v4l2_subdev *sd,
-			const struct i2c_vbi_ram_value *regs,
 			unsigned int type,u8 flags, int line,
 			const int fields)
 {
 	struct tvp5150 *decoder = to_tvp5150(sd);
 	v4l2_std_id std = decoder->norm;
 	u8 reg;
-	int pos = 0;
+	int i, pos = 0;
 
 	if (std == V4L2_STD_ALL) {
 		dev_err(sd->dev, "VBI can't be configured without knowing number of lines\n");
@@ -672,19 +674,19 @@ static int tvp5150_set_vbi(struct v4l2_subdev *sd,
 	if (line < 6 || line > 27)
 		return 0;
 
-	while (regs->reg != (u16)-1) {
+	for (i = 0; i < ARRAY_SIZE(vbi_ram_default); i++) {
+		const struct i2c_vbi_ram_value *regs =  &vbi_ram_default[i];
+
+		if (!regs->type.vbi_type)
+			continue;
+
 		if ((type & regs->type.vbi_type) &&
 		    (line >= regs->type.ini_line) &&
 		    (line <= regs->type.end_line))
 			break;
-
-		regs++;
 		pos++;
 	}
 
-	if (regs->reg == (u16)-1)
-		return 0;
-
 	type = pos | (flags & 0xf0);
 	reg = ((line - 6) << 1) + TVP5150_LINE_MODE_INI;
 
@@ -697,8 +699,7 @@ static int tvp5150_set_vbi(struct v4l2_subdev *sd,
 	return type;
 }
 
-static int tvp5150_get_vbi(struct v4l2_subdev *sd,
-			const struct i2c_vbi_ram_value *regs, int line)
+static int tvp5150_get_vbi(struct v4l2_subdev *sd, int line)
 {
 	struct tvp5150 *decoder = to_tvp5150(sd);
 	v4l2_std_id std = decoder->norm;
@@ -727,8 +728,8 @@ static int tvp5150_get_vbi(struct v4l2_subdev *sd,
 			return 0;
 		}
 		pos = ret & 0x0f;
-		if (pos < 0x0f)
-			type |= regs[pos].type.vbi_type;
+		if (pos < ARRAY_SIZE(vbi_ram_default))
+			type |= vbi_ram_default[pos].type.vbi_type;
 	}
 
 	return type;
@@ -789,7 +790,7 @@ static int tvp5150_reset(struct v4l2_subdev *sd, u32 val)
 	tvp5150_write_inittab(sd, tvp5150_init_default);
 
 	/* Initializes VDP registers */
-	tvp5150_vdp_init(sd, vbi_ram_default);
+	tvp5150_vdp_init(sd);
 
 	/* Selects decoder input */
 	tvp5150_selmux(sd);
@@ -1122,8 +1123,8 @@ static int tvp5150_s_sliced_fmt(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_f
 		for (i = 0; i <= 23; i++) {
 			svbi->service_lines[1][i] = 0;
 			svbi->service_lines[0][i] =
-				tvp5150_set_vbi(sd, vbi_ram_default,
-				       svbi->service_lines[0][i], 0xf0, i, 3);
+				tvp5150_set_vbi(sd, svbi->service_lines[0][i],
+						0xf0, i, 3);
 		}
 		/* Enables FIFO */
 		tvp5150_write(sd, TVP5150_FIFO_OUT_CTRL, 1);
@@ -1149,7 +1150,7 @@ static int tvp5150_g_sliced_fmt(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_f
 
 	for (i = 0; i <= 23; i++) {
 		svbi->service_lines[0][i] =
-			tvp5150_get_vbi(sd, vbi_ram_default, i);
+			tvp5150_get_vbi(sd, i);
 		mask |= svbi->service_lines[0][i];
 	}
 	svbi->service_set = mask;
diff --git a/drivers/media/i2c/tvp5150_reg.h b/drivers/media/i2c/tvp5150_reg.h
index 30a48c28d05a..c43b7b844021 100644
--- a/drivers/media/i2c/tvp5150_reg.h
+++ b/drivers/media/i2c/tvp5150_reg.h
@@ -1,8 +1,9 @@
 /*
+ * SPDX-License-Identifier: GPL-2.0
+ *
  * tvp5150 - Texas Instruments TVP5150A/AM1 video decoder registers
  *
- * Copyright (c) 2005,2006 Mauro Carvalho Chehab (mchehab@infradead.org)
- * This code is placed under the terms of the GNU General Public License v2
+ * Copyright (c) 2005,2006 Mauro Carvalho Chehab <mchehab@infradead.org>
  */
 
 #define TVP5150_VD_IN_SRC_SEL_1      0x00 /* Video input source selection #1 */
@@ -66,10 +67,10 @@
 
 #define VIDEO_STD_NTSC_MJ_BIT_AS                 0x01
 #define VIDEO_STD_PAL_BDGHIN_BIT_AS              0x03
-#define VIDEO_STD_PAL_M_BIT_AS		         0x05
+#define VIDEO_STD_PAL_M_BIT_AS			 0x05
 #define VIDEO_STD_PAL_COMBINATION_N_BIT_AS	 0x07
 #define VIDEO_STD_NTSC_4_43_BIT_AS		 0x09
-#define VIDEO_STD_SECAM_BIT_AS		         0x0b
+#define VIDEO_STD_SECAM_BIT_AS			 0x0b
 
 /* Reserved 29h-2bh */
 
diff --git a/drivers/media/i2c/tvp7002_reg.h b/drivers/media/i2c/tvp7002_reg.h
index 933673561fa2..3c8c8b0a6a4c 100644
--- a/drivers/media/i2c/tvp7002_reg.h
+++ b/drivers/media/i2c/tvp7002_reg.h
@@ -109,15 +109,15 @@
 #define TVP7002_L_FRAME_STAT_LSBS	0x37
 #define TVP7002_L_FRAME_STAT_MSBS	0x38
 #define TVP7002_CLK_L_STAT_LSBS		0x39
-#define TVP7002_CLK_L_STAT_MSBS      	0x3a
+#define TVP7002_CLK_L_STAT_MSBS		0x3a
 #define TVP7002_HSYNC_W			0x3b
 #define TVP7002_VSYNC_W                 0x3c
-#define TVP7002_L_LENGTH_TOL 		0x3d
+#define TVP7002_L_LENGTH_TOL		0x3d
 /* Reserved 0x3e */
 #define TVP7002_VIDEO_BWTH_CTL		0x3f
 #define TVP7002_AVID_START_PIXEL_LSBS	0x40
 #define TVP7002_AVID_START_PIXEL_MSBS   0x41
-#define TVP7002_AVID_STOP_PIXEL_LSBS  	0x42
+#define TVP7002_AVID_STOP_PIXEL_LSBS	0x42
 #define TVP7002_AVID_STOP_PIXEL_MSBS    0x43
 #define TVP7002_VBLK_F_0_START_L_OFF	0x44
 #define TVP7002_VBLK_F_1_START_L_OFF    0x45
diff --git a/drivers/media/i2c/vpx3220.c b/drivers/media/i2c/vpx3220.c
index 67de79b2d550..c3549fa55b62 100644
--- a/drivers/media/i2c/vpx3220.c
+++ b/drivers/media/i2c/vpx3220.c
@@ -201,7 +201,7 @@ static const unsigned short init_pal[] = {
 				 * skipped by the VFE) */
 	0x8b, 16,		/* Horizontal begin */
 	0x8c, 768,		/* Horizontal length */
-	0x8d, 784, 		/* Number of pixels
+	0x8d, 784,		/* Number of pixels
 				 * Must be >= Horizontal begin + Horizontal length */
 	0x8f, 0xc00,		/* Disable window 2 */
 	0xf0, 0x77,		/* 13.5 MHz transport, Forced
diff --git a/drivers/media/media-devnode.c b/drivers/media/media-devnode.c
index 423248f577b6..67ac51eff15c 100644
--- a/drivers/media/media-devnode.c
+++ b/drivers/media/media-devnode.c
@@ -99,13 +99,13 @@ static ssize_t media_write(struct file *filp, const char __user *buf,
 	return devnode->fops->write(filp, buf, sz, off);
 }
 
-static unsigned int media_poll(struct file *filp,
+static __poll_t media_poll(struct file *filp,
 			       struct poll_table_struct *poll)
 {
 	struct media_devnode *devnode = media_devnode_data(filp);
 
 	if (!media_devnode_is_registered(devnode))
-		return POLLERR | POLLHUP;
+		return EPOLLERR | EPOLLHUP;
 	if (!devnode->fops->poll)
 		return DEFAULT_POLLMASK;
 	return devnode->fops->poll(filp, poll);
diff --git a/drivers/media/mmc/siano/Makefile b/drivers/media/mmc/siano/Makefile
index 0e01f973db6b..5fc345645a80 100644
--- a/drivers/media/mmc/siano/Makefile
+++ b/drivers/media/mmc/siano/Makefile
@@ -1,6 +1,4 @@
 obj-$(CONFIG_SMS_SDIO_DRV) += smssdio.o
 
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/common/siano
-ccflags-y += $(extra-cflags-y) $(extra-cflags-m)
 
diff --git a/drivers/media/pci/Kconfig b/drivers/media/pci/Kconfig
index da28e68c87d8..5932e225f9c0 100644
--- a/drivers/media/pci/Kconfig
+++ b/drivers/media/pci/Kconfig
@@ -54,5 +54,7 @@ source "drivers/media/pci/smipcie/Kconfig"
 source "drivers/media/pci/netup_unidvb/Kconfig"
 endif
 
+source "drivers/media/pci/intel/ipu3/Kconfig"
+
 endif #MEDIA_PCI_SUPPORT
 endif #PCI
diff --git a/drivers/media/pci/Makefile b/drivers/media/pci/Makefile
index 1ab759e9b1bb..1c5ab07a8cff 100644
--- a/drivers/media/pci/Makefile
+++ b/drivers/media/pci/Makefile
@@ -14,7 +14,8 @@ obj-y        +=	ttpci/		\
 		ddbridge/	\
 		saa7146/	\
 		smipcie/	\
-		netup_unidvb/
+		netup_unidvb/	\
+		intel/
 
 obj-$(CONFIG_VIDEO_IVTV) += ivtv/
 obj-$(CONFIG_VIDEO_ZORAN) += zoran/
diff --git a/drivers/media/pci/b2c2/Makefile b/drivers/media/pci/b2c2/Makefile
index 35d6835ae43e..b43b9167db5a 100644
--- a/drivers/media/pci/b2c2/Makefile
+++ b/drivers/media/pci/b2c2/Makefile
@@ -6,5 +6,4 @@ endif
 b2c2-flexcop-pci-objs += flexcop-pci.o
 obj-$(CONFIG_DVB_B2C2_FLEXCOP_PCI) += b2c2-flexcop-pci.o
 
-ccflags-y += -Idrivers/media/dvb-core/
 ccflags-y += -Idrivers/media/common/b2c2/
diff --git a/drivers/media/pci/bt8xx/Makefile b/drivers/media/pci/bt8xx/Makefile
index 009f1dc1521f..7f1c3beb1bbc 100644
--- a/drivers/media/pci/bt8xx/Makefile
+++ b/drivers/media/pci/bt8xx/Makefile
@@ -6,8 +6,6 @@ bttv-objs      :=      bttv-driver.o bttv-cards.o bttv-if.o \
 obj-$(CONFIG_VIDEO_BT848) += bttv.o
 obj-$(CONFIG_DVB_BT8XX) += bt878.o dvb-bt8xx.o dst.o dst_ca.o
 
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/common
 ccflags-y += -Idrivers/media/tuners
diff --git a/drivers/media/pci/bt8xx/bt878.c b/drivers/media/pci/bt8xx/bt878.c
index a5f52137d306..f5f87e03f94b 100644
--- a/drivers/media/pci/bt8xx/bt878.c
+++ b/drivers/media/pci/bt8xx/bt878.c
@@ -40,8 +40,8 @@
 #include <linux/vmalloc.h>
 #include <linux/init.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
 #include "bt878.h"
 #include "dst_priv.h"
 
@@ -422,8 +422,7 @@ static int bt878_probe(struct pci_dev *dev, const struct pci_device_id *pci_id)
 	       bt878_num);
 	if (bt878_num >= BT878_MAX) {
 		printk(KERN_ERR "bt878: Too many devices inserted\n");
-		result = -ENOMEM;
-		goto fail0;
+		return -ENOMEM;
 	}
 	if (pci_enable_device(dev))
 		return -EIO;
diff --git a/drivers/media/pci/bt8xx/bttv-cards.c b/drivers/media/pci/bt8xx/bttv-cards.c
index 5cc42b426715..1902732f90e1 100644
--- a/drivers/media/pci/bt8xx/bttv-cards.c
+++ b/drivers/media/pci/bt8xx/bttv-cards.c
@@ -141,6 +141,13 @@ MODULE_PARM_DESC(audiodev, "specify audio device:\n"
 MODULE_PARM_DESC(saa6588, "if 1, then load the saa6588 RDS module, default (0) is to use the card definition.");
 MODULE_PARM_DESC(no_overlay, "allow override overlay default (0 disables, 1 enables) [some VIA/SIS chipsets are known to have problem with overlay]");
 
+
+/* I2C addresses list */
+#define I2C_ADDR_TDA7432	0x8a
+#define I2C_ADDR_MSP3400	0x80
+#define I2C_ADDR_MSP3400_ALT	0x88
+
+
 /* ----------------------------------------------------------------------- */
 /* list of card IDs for bt878+ cards                                       */
 
@@ -366,8 +373,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 15,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 2, 0, 0, 0 },
-		.gpiomute 	= 10,
+		.gpiomux	= { 2, 0, 0, 0 },
+		.gpiomute	= 10,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -378,8 +385,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 7,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 1, 2, 3 },
-		.gpiomute 	= 4,
+		.gpiomux	= { 0, 1, 2, 3 },
+		.gpiomute	= 4,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -390,8 +397,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 7,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 4, 0, 2, 3 },
-		.gpiomute 	= 1,
+		.gpiomux	= { 4, 0, 2, 3 },
+		.gpiomute	= 1,
 		.no_msp34xx	= 1,
 		.tuner_type     = TUNER_PHILIPS_NTSC,
 		.tuner_addr	= ADDR_UNSET,
@@ -407,7 +414,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0 },
+		.gpiomux	= { 0 },
 		.tuner_type	= TUNER_ABSENT,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -418,8 +425,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 3,
 		.muxsel		= MUXSEL(2, 3, 1, 0),
-		.gpiomux 	= { 0, 1, 0, 1 },
-		.gpiomute 	= 3,
+		.gpiomux	= { 0, 1, 0, 1 },
+		.gpiomute	= 3,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -430,7 +437,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 3,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
 		.gpiomask	= 0x0f,
-		.gpiomux 	= { 0x0c, 0x04, 0x08, 0x04 },
+		.gpiomux	= { 0x0c, 0x04, 0x08, 0x04 },
 		/*                0x04 for some cards ?? */
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -444,7 +451,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 3,
 		.gpiomask	= 0,
 		.muxsel		= MUXSEL(2, 3, 1, 0, 0),
-		.gpiomux 	= { 0 },
+		.gpiomux	= { 0 },
 		.tuner_type	= TUNER_ABSENT,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -457,8 +464,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xc00,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0xc00, 0x800, 0x400 },
-		.gpiomute 	= 0xc00,
+		.gpiomux	= { 0, 0xc00, 0x800, 0x400 },
+		.gpiomute	= 0xc00,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -470,7 +477,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 3,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 1, 1, 2, 3 },
+		.gpiomux	= { 1, 1, 2, 3 },
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_TEMIC_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -482,8 +489,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x0f, /* old: 7 */
 		.muxsel		= MUXSEL(2, 0, 1, 1),
-		.gpiomux 	= { 0, 1, 2, 3 },
-		.gpiomute 	= 4,
+		.gpiomux	= { 0, 1, 2, 3 },
+		.gpiomute	= 4,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -495,8 +502,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x3014f,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x20001,0x10001, 0, 0 },
-		.gpiomute 	= 10,
+		.gpiomux	= { 0x20001,0x10001, 0, 0 },
+		.gpiomute	= 10,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -509,7 +516,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 15,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 13, 14, 11, 7 },
+		.gpiomux	= { 13, 14, 11, 7 },
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -520,7 +527,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 15,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 13, 14, 11, 7 },
+		.gpiomux	= { 13, 14, 11, 7 },
 		.msp34xx_alt    = 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL,
@@ -535,8 +542,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 7,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 2, 1, 3 }, /* old: {0, 1, 2, 3, 4} */
-		.gpiomute 	= 4,
+		.gpiomux	= { 0, 2, 1, 3 }, /* old: {0, 1, 2, 3, 4} */
+		.gpiomute	= 4,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -548,8 +555,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 15,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0, 1, 0 },
-		.gpiomute 	= 10,
+		.gpiomux	= { 0, 0, 1, 0 },
+		.gpiomute	= 10,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -564,7 +571,7 @@ struct tvcard bttv_tvcards[] = {
 		.muxsel		= MUXSEL(2, 3, 1, 1),
 		/* 2003-10-20 by "Anton A. Arapov" <arapov@mail.ru> */
 		.gpiomux        = { 0x001e00, 0, 0x018000, 0x014000 },
-		.gpiomute 	= 0x002000,
+		.gpiomute	= 0x002000,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr     = ADDR_UNSET,
@@ -576,8 +583,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x8300f8,
 		.muxsel		= MUXSEL(2, 3, 1, 1, 0),
-		.gpiomux 	= { 0x4fa007,0xcfa007,0xcfa007,0xcfa007 },
-		.gpiomute 	= 0xcfa007,
+		.gpiomux	= { 0x4fa007,0xcfa007,0xcfa007,0xcfa007 },
+		.gpiomute	= 0xcfa007,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 		.volume_gpio	= winview_volume,
@@ -590,7 +597,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 1, 0, 0, 0 },
+		.gpiomux	= { 1, 0, 0, 0 },
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -601,7 +608,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= NO_SVHS,
 		.gpiomask	= 0x8dff00,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0 },
+		.gpiomux	= { 0 },
 		.no_msp34xx	= 1,
 		.tuner_type	= TUNER_ABSENT,
 		.tuner_addr	= ADDR_UNSET,
@@ -624,8 +631,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x1800,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0x800, 0x1000, 0x1000 },
-		.gpiomute 	= 0x1800,
+		.gpiomux	= { 0, 0x800, 0x1000, 0x1000 },
+		.gpiomute	= 0x1800,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL_I,
 		.tuner_addr	= ADDR_UNSET,
@@ -637,8 +644,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xc00,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 1, 0x800, 0x400 },
-		.gpiomute 	= 0xc00,
+		.gpiomux	= { 0, 1, 0x800, 0x400 },
+		.gpiomute	= 0xc00,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -652,7 +659,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask	= 7,
 		.muxsel		= MUXSEL(2, 3, 0), /* input 2 is digital */
 		/* .digital_mode= DIGITAL_MODE_CAMERA, */
-		.gpiomux 	= { 0, 0, 0, 0 },
+		.gpiomux	= { 0, 0, 0, 0 },
 		.no_msp34xx	= 1,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_ALPS_TSBB5_PAL_I,
@@ -667,8 +674,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xe00,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= {0x400, 0x400, 0x400, 0x400 },
-		.gpiomute 	= 0xc00,
+		.gpiomux	= {0x400, 0x400, 0x400, 0x400 },
+		.gpiomute	= 0xc00,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -683,7 +690,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x1f0fff,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0x20000, 0x30000, 0x10000, 0 },
-		.gpiomute 	= 0x40000,
+		.gpiomute	= 0x40000,
 		.tuner_type	= TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
 		.audio_mode_gpio= terratv_audio,
@@ -695,8 +702,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 3,
 		.gpiomask	= 7,
 		.muxsel		= MUXSEL(2, 0, 1, 1),
-		.gpiomux 	= { 0, 1, 2, 3 },
-		.gpiomute 	= 4,
+		.gpiomux	= { 0, 1, 2, 3 },
+		.gpiomute	= 4,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -707,8 +714,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x1800,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0x800, 0x1000, 0x1000 },
-		.gpiomute 	= 0x1800,
+		.gpiomux	= { 0, 0x800, 0x1000, 0x1000 },
+		.gpiomute	= 0x1800,
 		.pll            = PLL_28,
 		.tuner_type	= TUNER_PHILIPS_SECAM,
 		.tuner_addr	= ADDR_UNSET,
@@ -722,8 +729,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x1f0fff,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x20000, 0x30000, 0x10000, 0x00000 },
-		.gpiomute 	= 0x40000,
+		.gpiomux	= { 0x20000, 0x30000, 0x10000, 0x00000 },
+		.gpiomute	= 0x40000,
 		.tuner_type	= TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
 		.audio_mode_gpio= terratv_audio,
@@ -767,7 +774,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 1, /* was: 4 */
 		.gpiomask	= 0,
 		.muxsel		= MUXSEL(2, 3, 1, 0, 0),
-		.gpiomux 	= { 0 },
+		.gpiomux	= { 0 },
 		.tuner_type	= TUNER_ABSENT,
 		.tuner_addr	= ADDR_UNSET,
 		.muxsel_hook    = PXC200_muxsel,
@@ -780,8 +787,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x1800,  /* 0x8dfe00 */
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0x0800, 0x1000, 0x1000 },
-		.gpiomute 	= 0x1800,
+		.gpiomux	= { 0, 0x0800, 0x1000, 0x1000 },
+		.gpiomute	= 0x1800,
 		.pll            = PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -793,7 +800,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 3,
 		.gpiomask	= 1,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 1, 0, 0, 0 },
+		.gpiomux	= { 1, 0, 0, 0 },
 		.pll            = PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -807,7 +814,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0 },
+		.gpiomux	= { 0 },
 		.tuner_type	= TUNER_ABSENT,
 		.tuner_addr	= ADDR_UNSET,
 	},
@@ -818,8 +825,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xffff00,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x500, 0, 0x300, 0x900 },
-		.gpiomute 	= 0x900,
+		.gpiomux	= { 0x500, 0, 0x300, 0x900 },
+		.gpiomute	= 0x900,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -833,8 +840,8 @@ struct tvcard bttv_tvcards[] = {
 		.muxsel		= MUXSEL(2, 3, 1, 1, 0),
 		/* Alexander Varakin <avarakin@hotmail.com> [stereo version] */
 		.gpiomask	= 0xb33000,
-		.gpiomux 	= { 0x122000,0x1000,0x0000,0x620000 },
-		.gpiomute 	= 0x800000,
+		.gpiomux	= { 0x122000,0x1000,0x0000,0x620000 },
+		.gpiomute	= 0x800000,
 		/* Audio Routing for "WinFast 2000 XP" (no tv stereo !)
 			gpio23 -- hef4052:nEnable (0x800000)
 			gpio12 -- hef4052:A1
@@ -860,8 +867,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x1800,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0x800, 0x1000, 0x1000 },
-		.gpiomute 	= 0x1800,
+		.gpiomux	= { 0, 0x800, 0x1000, 0x1000 },
+		.gpiomute	= 0x1800,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -875,8 +882,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x1800,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0x800, 0x1000, 0x1000 },
-		.gpiomute 	= 0x1800,
+		.gpiomux	= { 0, 0x800, 0x1000, 0x1000 },
+		.gpiomute	= 0x1800,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -889,8 +896,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xff,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x21, 0x20, 0x24, 0x2c },
-		.gpiomute 	= 0x29,
+		.gpiomux	= { 0x21, 0x20, 0x24, 0x2c },
+		.gpiomute	= 0x29,
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
@@ -903,8 +910,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x551e00,
 		.muxsel		= MUXSEL(2, 3, 1, 0),
-		.gpiomux 	= { 0x551400, 0x551200, 0, 0 },
-		.gpiomute 	= 0x551c00,
+		.gpiomux	= { 0x551400, 0x551200, 0, 0 },
+		.gpiomute	= 0x551c00,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL_I,
 		.tuner_addr	= ADDR_UNSET,
@@ -917,8 +924,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x03000F,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 2, 0xd0001, 0, 0 },
-		.gpiomute 	= 1,
+		.gpiomux	= { 2, 0xd0001, 0, 0 },
+		.gpiomute	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -932,8 +939,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 7,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 4, 0, 2, 3 },
-		.gpiomute 	= 1,
+		.gpiomux	= { 4, 0, 2, 3 },
+		.gpiomute	= 1,
 		.no_msp34xx	= 1,
 		.tuner_type     = TUNER_PHILIPS_NTSC,
 		.tuner_addr	= ADDR_UNSET,
@@ -947,7 +954,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 15,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 13, 4, 11, 7 },
+		.gpiomux	= { 13, 4, 11, 7 },
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -961,7 +968,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0, 0, 0},
+		.gpiomux	= { 0, 0, 0, 0},
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL_I,
@@ -974,8 +981,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xe00b,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0xff9ff6, 0xff9ff6, 0xff1ff7, 0 },
-		.gpiomute 	= 0xff3ffc,
+		.gpiomux	= { 0xff9ff6, 0xff9ff6, 0xff1ff7, 0 },
+		.gpiomute	= 0xff3ffc,
 		.no_msp34xx	= 1,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -989,8 +996,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= NO_SVHS,
 		.gpiomask	= 3,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 1, 1, 0, 2 },
-		.gpiomute 	= 3,
+		.gpiomux	= { 1, 1, 0, 2 },
+		.gpiomute	= 3,
 		.no_msp34xx	= 1,
 		.pll		= PLL_NONE,
 		.tuner_type	= UNSET,
@@ -1003,7 +1010,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 3,
 		.gpiomask	= 0,
 		.muxsel		= MUXSEL(2, 3, 1, 0, 0),
-		.gpiomux 	= { 0 },
+		.gpiomux	= { 0 },
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_ABSENT,
@@ -1016,8 +1023,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xbcf03f,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0xbc803f, 0xbc903f, 0xbcb03f, 0 },
-		.gpiomute 	= 0xbcb03f,
+		.gpiomux	= { 0xbc803f, 0xbc903f, 0xbcb03f, 0 },
+		.gpiomute	= 0xbcb03f,
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_TEMIC_4039FR5_NTSC,
@@ -1030,8 +1037,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x70000,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x20000, 0x30000, 0x10000, 0 },
-		.gpiomute 	= 0x40000,
+		.gpiomux	= { 0x20000, 0x30000, 0x10000, 0 },
+		.gpiomute	= 0x40000,
 		.no_msp34xx	= 1,
 		.pll		= PLL_35,
 		.tuner_type	= TUNER_PHILIPS_PAL_I,
@@ -1047,8 +1054,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 15,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= {2,0,0,0 },
-		.gpiomute 	= 1,
+		.gpiomux	= {2,0,0,0 },
+		.gpiomute	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
@@ -1060,7 +1067,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x010f00,
 		.muxsel		= MUXSEL(2, 3, 0, 0),
-		.gpiomux 	= {0x10000, 0, 0x10000, 0 },
+		.gpiomux	= {0x10000, 0, 0x10000, 0 },
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_ALPS_TSHC6_NTSC,
@@ -1076,8 +1083,8 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask	= 0xAA0000,
 		.muxsel		= MUXSEL(2, 3, 1, 1, 0), /* in 4 is digital */
 		/* .digital_mode= DIGITAL_MODE_CAMERA, */
-		.gpiomux 	= { 0x20000, 0, 0x80000, 0x80000 },
-		.gpiomute 	= 0xa8000,
+		.gpiomux	= { 0x20000, 0, 0x80000, 0x80000 },
+		.gpiomute	= 0xa8000,
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL_I,
@@ -1101,7 +1108,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 7,
 		.muxsel         = MUXSEL(2, 0, 1, 1),
 		.gpiomux        = { 0, 1, 2, 3 },
-		.gpiomute 	= 4,
+		.gpiomute	= 4,
 		.pll            = PLL_28,
 		.tuner_type     = UNSET /* TUNER_ALPS_TMDH2_NTSC */,
 		.tuner_addr	= ADDR_UNSET,
@@ -1116,8 +1123,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs           = 3,
 		.gpiomask       = 0x03000F,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 1, 0xd0001, 0, 0 },
-		.gpiomute 	= 10,
+		.gpiomux	= { 1, 0xd0001, 0, 0 },
+		.gpiomute	= 10,
 				/* sound path (5 sources):
 				MUX1 (mask 0x03), Enable Pin 0x08 (0=enable, 1=disable)
 					0= ext. Audio IN
@@ -1140,8 +1147,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x1c,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0, 0, 0x10, 8 },
-		.gpiomute 	= 4,
+		.gpiomux	= { 0, 0, 0x10, 8 },
+		.gpiomute	= 4,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1159,8 +1166,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x18e0,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x0000,0x0800,0x1000,0x1000 },
-		.gpiomute 	= 0x18e0,
+		.gpiomux	= { 0x0000,0x0800,0x1000,0x1000 },
+		.gpiomute	= 0x18e0,
 			/* For cards with tda9820/tda9821:
 				0x0000: Tuner normal stereo
 				0x0080: Tuner A2 SAP (second audio program = Zweikanalton)
@@ -1179,7 +1186,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0xF,
 		.muxsel         = MUXSEL(2, 3, 1, 0),
 		.gpiomux        = { 2, 0, 0, 0 },
-		.gpiomute 	= 10,
+		.gpiomute	= 10,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_TEMIC_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1195,7 +1202,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x1800,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 0x800, 0x1000, 0x1000 },
-		.gpiomute 	= 0x1800,
+		.gpiomute	= 0x1800,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1225,7 +1232,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0xe00,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0x400, 0x400, 0x400, 0x400 },
-		.gpiomute 	= 0x800,
+		.gpiomute	= 0x800,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_TEMIC_4036FY5_NTSC,
 		.tuner_addr	= ADDR_UNSET,
@@ -1239,7 +1246,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x03000F,
 		.muxsel		= MUXSEL(2, 3, 1, 0),
 		.gpiomux        = { 2, 0, 0, 0 },
-		.gpiomute 	= 1,
+		.gpiomute	= 1,
 		.pll            = PLL_28,
 		.tuner_type	= TUNER_TEMIC_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1256,7 +1263,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 11,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 2, 0, 0, 1 },
-		.gpiomute 	= 8,
+		.gpiomute	= 8,
 		.pll            = PLL_35,
 		.tuner_type     = TUNER_TEMIC_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1286,7 +1293,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0xFF,
 		.muxsel         = MUXSEL(2, 3, 1, 0),
 		.gpiomux        = { 1, 0, 4, 4 },
-		.gpiomute 	= 9,
+		.gpiomute	= 9,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1299,8 +1306,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xf03f,
 		.muxsel		= MUXSEL(2, 3, 1, 0),
-		.gpiomux 	= { 0xbffe, 0, 0xbfff, 0 },
-		.gpiomute 	= 0xbffe,
+		.gpiomux	= { 0xbffe, 0, 0xbfff, 0 },
+		.gpiomute	= 0xbffe,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_TEMIC_4006FN5_MULTI_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1315,7 +1322,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= NO_SVHS,
 		.gpiomask	= 1,
 		.muxsel		= MUXSEL(2, 3, 0, 1),
-		.gpiomux 	= { 0, 0, 1, 0 },
+		.gpiomux	= { 0, 0, 1, 0 },
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_TEMIC_4006FN5_MULTI_PAL,
@@ -1332,8 +1339,8 @@ struct tvcard bttv_tvcards[] = {
 				/* Radio changed from 1e80 to 0x800 to make
 				FlyVideo2000S in .hu happy (gm)*/
 				/* -dk-???: set mute=0x1800 for tda9874h daughterboard */
-		.gpiomux 	= { 0x0000,0x0800,0x1000,0x1000 },
-		.gpiomute 	= 0x1800,
+		.gpiomux	= { 0x0000,0x0800,0x1000,0x1000 },
+		.gpiomute	= 0x1800,
 		.audio_mode_gpio= fv2000s_audio,
 		.no_msp34xx	= 1,
 		.pll            = PLL_28,
@@ -1347,8 +1354,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0xffff00,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x500, 0x500, 0x300, 0x900 },
-		.gpiomute 	= 0x900,
+		.gpiomux	= { 0x500, 0x500, 0x300, 0x900 },
+		.gpiomute	= 0x900,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -1382,7 +1389,7 @@ struct tvcard bttv_tvcards[] = {
 		/* 0x100000: 1=MSP enabled (0=disable again)
 		* 0x010000: Connected to "S0" on tda9880 (0=Pal/BG, 1=NTSC) */
 		.gpiomux        = {0x947fff, 0x987fff,0x947fff,0x947fff },
-		.gpiomute 	= 0x947fff,
+		.gpiomute	= 0x947fff,
 		/* tvtuner, radio,   external,internal, mute,  stereo
 		* tuner, Composit, SVid, Composit-on-Svid-adapter */
 		.muxsel         = MUXSEL(2, 3, 0, 1),
@@ -1402,7 +1409,7 @@ struct tvcard bttv_tvcards[] = {
 		/* 0x100000: 1=MSP enabled (0=disable again)
 		* 0x010000: Connected to "S0" on tda9880 (0=Pal/BG, 1=NTSC) */
 		.gpiomux        = {0x947fff, 0x987fff,0x947fff,0x947fff },
-		.gpiomute 	= 0x947fff,
+		.gpiomute	= 0x947fff,
 		/* tvtuner, radio,   external,internal, mute,  stereo
 		* tuner, Composit, SVid, Composit-on-Svid-adapter */
 		.muxsel         = MUXSEL(2, 3, 0, 1),
@@ -1431,7 +1438,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 15,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 0, 11, 7 }, /* TV and Radio with same GPIO ! */
-		.gpiomute 	= 13,
+		.gpiomute	= 13,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_LG_PAL_I_FM,
 		.tuner_addr	= ADDR_UNSET,
@@ -1466,8 +1473,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x3f,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 0x01, 0x00, 0x03, 0x03 },
-		.gpiomute 	= 0x09,
+		.gpiomux	= { 0x01, 0x00, 0x03, 0x03 },
+		.gpiomute	= 0x09,
 		.no_msp34xx	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_PAL,
@@ -1518,7 +1525,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x1C800F,  /* Bit0-2: Audio select, 8-12:remote control 14:remote valid 15:remote reset */
 		.muxsel         = MUXSEL(2, 1, 1),
 		.gpiomux        = { 0, 1, 2, 2 },
-		.gpiomute 	= 4,
+		.gpiomute	= 4,
 		.tuner_type     = TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
 		.pll		= PLL_28,
@@ -1535,7 +1542,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x140007,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 1, 2, 3 },
-		.gpiomute 	= 4,
+		.gpiomute	= 4,
 		.tuner_type     = TUNER_PHILIPS_NTSC,
 		.tuner_addr	= ADDR_UNSET,
 		.audio_mode_gpio= windvr_audio,
@@ -1568,7 +1575,7 @@ struct tvcard bttv_tvcards[] = {
 						* gpiomux =1: lower volume, 2+3: mute
 						* btwincap uses 0x80000/0x80003
 						*/
-		.gpiomute 	= 4,
+		.gpiomute	= 4,
 		.no_msp34xx     = 1,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_PAL,
@@ -1619,7 +1626,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x0f0f80,
 		.muxsel         = MUXSEL(2, 3, 1, 0),
 		.gpiomux        = {0x030000, 0x010000, 0, 0 },
-		.gpiomute 	= 0x020000,
+		.gpiomute	= 0x020000,
 		.no_msp34xx     = 1,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_NTSC_M,
@@ -1822,7 +1829,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 7,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 1, 2, 3},
-		.gpiomute 	= 4,
+		.gpiomute	= 4,
 		.tuner_type     = TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
 		.pll            = PLL_28,
@@ -1865,7 +1872,7 @@ struct tvcard bttv_tvcards[] = {
 		.muxsel         = MUXSEL(2, 3, 1, 0),
 		/*                  Tuner, Radio, external, internal, off,  on */
 		.gpiomux        = { 0x08,  0x0f,  0x0a,     0x08 },
-		.gpiomute 	= 0x0f,
+		.gpiomute	= 0x0f,
 		.no_msp34xx     = 1,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_NTSC,
@@ -2132,7 +2139,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x008007,
 		.muxsel         = MUXSEL(2, 3, 0, 0),
 		.gpiomux        = { 0, 0, 0, 0 },
-		.gpiomute 	= 0x000003,
+		.gpiomute	= 0x000003,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -2175,7 +2182,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x008007,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 1, 2, 2 },
-		.gpiomute 	= 3,
+		.gpiomute	= 3,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -2290,7 +2297,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0xFF,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 2, 0, 0, 0 },
-		.gpiomute 	= 10,
+		.gpiomute	= 10,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_PAL,
 		.tuner_addr	= ADDR_UNSET,
@@ -2319,7 +2326,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x3f,
 		.muxsel         = MUXSEL(2, 3, 1, 0),
 		.gpiomux        = {0x31, 0x31, 0x31, 0x31 },
-		.gpiomute 	= 0x31,
+		.gpiomute	= 0x31,
 		.no_msp34xx     = 1,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_PHILIPS_NTSC_M,
@@ -2433,7 +2440,7 @@ struct tvcard bttv_tvcards[] = {
 		.muxsel		= MUXSEL(2, 3, 1),
 		.gpiomask       = 0x00e00007,
 		.gpiomux        = { 0x00400005, 0, 0x00000001, 0 },
-		.gpiomute 	= 0x00c00007,
+		.gpiomute	= 0x00c00007,
 		.no_msp34xx     = 1,
 		.no_tda7432     = 1,
 		.has_dvb        = 1,
@@ -2448,7 +2455,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask	= 0x01fe00,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0x001e00, 0, 0x018000, 0x014000 },
-		.gpiomute 	= 0x002000,
+		.gpiomute	= 0x002000,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_YMEC_TVF66T5_B_DFF,
 		.tuner_addr	= 0xc1 >>1,
@@ -2463,7 +2470,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x001c0007,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 1, 2, 2 },
-		.gpiomute 	= 3,
+		.gpiomute	= 3,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_TENA_9533_DI,
 		.tuner_addr	= ADDR_UNSET,
@@ -2498,7 +2505,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x3f,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0x21, 0x20, 0x24, 0x2c },
-		.gpiomute 	= 0x29,
+		.gpiomute	= 0x29,
 		.no_msp34xx     = 1,
 		.pll            = PLL_28,
 		.tuner_type     = TUNER_YMEC_TVF_5533MF,
@@ -2542,8 +2549,8 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 15,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 2, 0, 0, 0 },
-		.gpiomute 	= 1,
+		.gpiomux	= { 2, 0, 0, 0 },
+		.gpiomute	= 1,
 		.pll		= PLL_28,
 		.tuner_type	= TUNER_PHILIPS_NTSC,
 		.tuner_addr	= ADDR_UNSET,
@@ -2556,7 +2563,7 @@ struct tvcard bttv_tvcards[] = {
 		.svhs		= 2,
 		.gpiomask	= 0x108007,
 		.muxsel		= MUXSEL(2, 3, 1, 1),
-		.gpiomux 	= { 100000, 100002, 100002, 100000 },
+		.gpiomux	= { 100000, 100002, 100002, 100000 },
 		.no_msp34xx	= 1,
 		.no_tda7432     = 1,
 		.pll		= PLL_28,
@@ -2592,7 +2599,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 7,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 1, 2, 3 },
-		.gpiomute 	= 4,
+		.gpiomute	= 4,
 		.tuner_type     = TUNER_TEMIC_4009FR5_PAL,
 		.tuner_addr     = ADDR_UNSET,
 		.pll            = PLL_28,
@@ -2628,7 +2635,7 @@ struct tvcard bttv_tvcards[] = {
 		.muxsel		= MUXSEL(2, 3, 1),
 		.gpiomask       = 0x00e00007,
 		.gpiomux        = { 0x00400005, 0, 0x00000001, 0 },
-		.gpiomute 	= 0x00c00007,
+		.gpiomute	= 0x00c00007,
 		.no_msp34xx     = 1,
 		.no_tda7432     = 1,
 	},
@@ -2672,7 +2679,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x008007,
 		.muxsel         = MUXSEL(2, 3, 1, 1),
 		.gpiomux        = { 0, 1, 2, 2 }, /* CONTVFMi */
-		.gpiomute 	= 3, /* CONTVFMi */
+		.gpiomute	= 3, /* CONTVFMi */
 		.tuner_type     = TUNER_PHILIPS_FM1216ME_MK3, /* TCL MK3 */
 		.tuner_addr     = ADDR_UNSET,
 		.pll            = PLL_28,
@@ -2695,7 +2702,7 @@ struct tvcard bttv_tvcards[] = {
 		.gpiomask       = 0x060040,
 		.muxsel         = MUXSEL(2, 3, 3),
 		.gpiomux        = { 0x60000, 0x60000, 0x20000, 0x20000 },
-		.gpiomute 	= 0,
+		.gpiomute	= 0,
 		.tuner_type	= TUNER_TCL_MF02GIP_5N,
 		.tuner_addr     = ADDR_UNSET,
 		.pll            = PLL_28,
@@ -2745,8 +2752,8 @@ struct tvcard bttv_tvcards[] = {
 		/* Bruno Christo <bchristo@inf.ufsm.br>
 		 *
 		 * GeoVision GV-800(S) has 4 Conexant Fusion 878A:
-		 * 	1 audio input  per BT878A = 4 audio inputs
-		 * 	4 video inputs per BT878A = 16 video inputs
+		 *	1 audio input  per BT878A = 4 audio inputs
+		 *	4 video inputs per BT878A = 16 video inputs
 		 * This is the first BT878A chip of the GV-800(S). It's the
 		 * "master" chip and it controls the video inputs through an
 		 * analog multiplexer (a CD22M3494) via some GPIO pins. The
@@ -2772,8 +2779,8 @@ struct tvcard bttv_tvcards[] = {
 		/* Bruno Christo <bchristo@inf.ufsm.br>
 		 *
 		 * GeoVision GV-800(S) has 4 Conexant Fusion 878A:
-		 * 	1 audio input  per BT878A = 4 audio inputs
-		 * 	4 video inputs per BT878A = 16 video inputs
+		 *	1 audio input  per BT878A = 4 audio inputs
+		 *	4 video inputs per BT878A = 16 video inputs
 		 * The 3 other BT878A chips are "slave" chips of the GV-800(S)
 		 * and should use this card type.
 		 * The audio input is not working yet.
@@ -4777,9 +4784,9 @@ static void gv800s_write(struct bttv *btv,
 	* GPIO bits 0-9 are used for the analog switch:
 	*   00 - 03:	camera selector
 	*   04 - 06:	878A (controller) selector
-	*   16: 	cselect
+	*   16:		cselect
 	*   17:		strobe
-	*   18: 	data (1->on, 0->off)
+	*   18:		data (1->on, 0->off)
 	*   19:		reset
 	*/
 	const u32 ADDRESS = ((xaddr&0xf) | (yaddr&3)<<4);
@@ -4875,7 +4882,7 @@ void __init bttv_check_chipset(void)
 	int pcipci_fail = 0;
 	struct pci_dev *dev = NULL;
 
-	if (pci_pci_problems & (PCIPCI_FAIL|PCIAGP_FAIL)) 	/* should check if target is AGP */
+	if (pci_pci_problems & (PCIPCI_FAIL|PCIAGP_FAIL))	/* should check if target is AGP */
 		pcipci_fail = 1;
 	if (pci_pci_problems & (PCIPCI_TRITON|PCIPCI_NATOMA|PCIPCI_VIAETBF))
 		triton1 = 1;
diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
index b366a7e1d976..f697698fe38d 100644
--- a/drivers/media/pci/bt8xx/bttv-driver.c
+++ b/drivers/media/pci/bt8xx/bttv-driver.c
@@ -2955,48 +2955,48 @@ static ssize_t bttv_read(struct file *file, char __user *data,
 	return retval;
 }
 
-static unsigned int bttv_poll(struct file *file, poll_table *wait)
+static __poll_t bttv_poll(struct file *file, poll_table *wait)
 {
 	struct bttv_fh *fh = file->private_data;
 	struct bttv_buffer *buf;
 	enum v4l2_field field;
-	unsigned int rc = 0;
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t rc = 0;
+	__poll_t req_events = poll_requested_events(wait);
 
 	if (v4l2_event_pending(&fh->fh))
-		rc = POLLPRI;
-	else if (req_events & POLLPRI)
+		rc = EPOLLPRI;
+	else if (req_events & EPOLLPRI)
 		poll_wait(file, &fh->fh.wait, wait);
 
-	if (!(req_events & (POLLIN | POLLRDNORM)))
+	if (!(req_events & (EPOLLIN | EPOLLRDNORM)))
 		return rc;
 
 	if (V4L2_BUF_TYPE_VBI_CAPTURE == fh->type) {
 		if (!check_alloc_btres_lock(fh->btv,fh,RESOURCE_VBI))
-			return rc | POLLERR;
+			return rc | EPOLLERR;
 		return rc | videobuf_poll_stream(file, &fh->vbi, wait);
 	}
 
 	if (check_btres(fh,RESOURCE_VIDEO_STREAM)) {
 		/* streaming capture */
 		if (list_empty(&fh->cap.stream))
-			return rc | POLLERR;
+			return rc | EPOLLERR;
 		buf = list_entry(fh->cap.stream.next,struct bttv_buffer,vb.stream);
 	} else {
 		/* read() capture */
 		if (NULL == fh->cap.read_buf) {
 			/* need to capture a new frame */
 			if (locked_btres(fh->btv,RESOURCE_VIDEO_STREAM))
-				return rc | POLLERR;
+				return rc | EPOLLERR;
 			fh->cap.read_buf = videobuf_sg_alloc(fh->cap.msize);
 			if (NULL == fh->cap.read_buf)
-				return rc | POLLERR;
+				return rc | EPOLLERR;
 			fh->cap.read_buf->memory = V4L2_MEMORY_USERPTR;
 			field = videobuf_next_field(&fh->cap);
 			if (0 != fh->cap.ops->buf_prepare(&fh->cap,fh->cap.read_buf,field)) {
 				kfree (fh->cap.read_buf);
 				fh->cap.read_buf = NULL;
-				return rc | POLLERR;
+				return rc | EPOLLERR;
 			}
 			fh->cap.ops->buf_queue(&fh->cap,fh->cap.read_buf);
 			fh->cap.read_off = 0;
@@ -3007,7 +3007,7 @@ static unsigned int bttv_poll(struct file *file, poll_table *wait)
 	poll_wait(file, &buf->vb.done, wait);
 	if (buf->vb.state == VIDEOBUF_DONE ||
 	    buf->vb.state == VIDEOBUF_ERROR)
-		rc = rc | POLLIN|POLLRDNORM;
+		rc = rc | EPOLLIN|EPOLLRDNORM;
 	return rc;
 }
 
@@ -3329,17 +3329,17 @@ static ssize_t radio_read(struct file *file, char __user *data,
 	return cmd.result;
 }
 
-static unsigned int radio_poll(struct file *file, poll_table *wait)
+static __poll_t radio_poll(struct file *file, poll_table *wait)
 {
 	struct bttv_fh *fh = file->private_data;
 	struct bttv *btv = fh->btv;
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct saa6588_command cmd;
-	unsigned int res = 0;
+	__poll_t res = 0;
 
 	if (v4l2_event_pending(&fh->fh))
-		res = POLLPRI;
-	else if (req_events & POLLPRI)
+		res = EPOLLPRI;
+	else if (req_events & EPOLLPRI)
 		poll_wait(file, &fh->fh.wait, wait);
 	radio_enable(btv);
 	cmd.instance = file;
diff --git a/drivers/media/pci/bt8xx/bttv-input.c b/drivers/media/pci/bt8xx/bttv-input.c
index ac7674700685..da49c5567db5 100644
--- a/drivers/media/pci/bt8xx/bttv-input.c
+++ b/drivers/media/pci/bt8xx/bttv-input.c
@@ -349,12 +349,12 @@ static int get_key_pv951(struct IR_i2c *ir, enum rc_proto *protocol,
 	 * NOTE:
 	 * lirc_i2c maps the pv951 code as:
 	 *	addr = 0x61D6
-	 * 	cmd = bit_reverse (b)
+	 *	cmd = bit_reverse (b)
 	 * So, it seems that this device uses NEC extended
 	 * I decided to not fix the table, due to two reasons:
-	 * 	1) Without the actual device, this is only a guess;
-	 * 	2) As the addr is not reported via I2C, nor can be changed,
-	 * 	   the device is bound to the vendor-provided RC.
+	 *	1) Without the actual device, this is only a guess;
+	 *	2) As the addr is not reported via I2C, nor can be changed,
+	 *	   the device is bound to the vendor-provided RC.
 	 */
 
 	*protocol = RC_PROTO_UNKNOWN;
diff --git a/drivers/media/pci/bt8xx/bttv.h b/drivers/media/pci/bt8xx/bttv.h
index eb67e362acf7..a27384adadd2 100644
--- a/drivers/media/pci/bt8xx/bttv.h
+++ b/drivers/media/pci/bt8xx/bttv.h
@@ -18,7 +18,6 @@
 #include <linux/videodev2.h>
 #include <linux/i2c.h>
 #include <media/v4l2-device.h>
-#include <media/i2c-addr.h>
 #include <media/tuner.h>
 
 /* ---------------------------------------------------------- */
@@ -166,7 +165,7 @@
 #define BTTV_BOARD_PV_M4900                0x8b
 #define BTTV_BOARD_OSPREY440               0x8c
 #define BTTV_BOARD_ASOUND_SKYEYE	   0x8d
-#define BTTV_BOARD_SABRENT_TVFM   	   0x8e
+#define BTTV_BOARD_SABRENT_TVFM		   0x8e
 #define BTTV_BOARD_HAUPPAUGE_IMPACTVCB     0x8f
 #define BTTV_BOARD_MACHTV_MAGICTV          0x90
 #define BTTV_BOARD_SSAI_SECURITY	   0x91
@@ -266,7 +265,7 @@ extern struct tvcard bttv_tvcards[];
  * that they are changed to octal.  One should not use hex number, macros, or
  * anything else with this macro.  Just use plain integers from 0 to 3.
  */
-#define _MUXSELf(a)      	0##a << 30
+#define _MUXSELf(a)		0##a << 30
 #define _MUXSELe(a, b...)	0##a << 28 | _MUXSELf(b)
 #define _MUXSELd(a, b...)	0##a << 26 | _MUXSELe(b)
 #define _MUXSELc(a, b...)	0##a << 24 | _MUXSELd(b)
diff --git a/drivers/media/pci/bt8xx/bttvp.h b/drivers/media/pci/bt8xx/bttvp.h
index cb1b5e611130..7a86e7295166 100644
--- a/drivers/media/pci/bt8xx/bttvp.h
+++ b/drivers/media/pci/bt8xx/bttvp.h
@@ -141,7 +141,7 @@ struct bttv_ir {
 	bool			rc5_gpio;   /* Is RC5 legacy GPIO enabled? */
 	u32                     last_bit;   /* last raw bit seen */
 	u32                     code;       /* raw code under construction */
-	ktime_t          				base_time;  /* time of last seen code */
+	ktime_t						base_time;  /* time of last seen code */
 	bool                    active;     /* building raw code */
 };
 
@@ -400,8 +400,8 @@ struct bttv {
 	int                        i2c_state, i2c_rc;
 	int                        i2c_done;
 	wait_queue_head_t          i2c_queue;
-	struct v4l2_subdev 	  *sd_msp34xx;
-	struct v4l2_subdev 	  *sd_tvaudio;
+	struct v4l2_subdev	  *sd_msp34xx;
+	struct v4l2_subdev	  *sd_tvaudio;
 	struct v4l2_subdev	  *sd_tda7432;
 
 	/* video4linux (1) */
diff --git a/drivers/media/pci/bt8xx/dst.c b/drivers/media/pci/bt8xx/dst.c
index 7166d2279465..4f0bba9e4c48 100644
--- a/drivers/media/pci/bt8xx/dst.c
+++ b/drivers/media/pci/bt8xx/dst.c
@@ -28,7 +28,7 @@
 #include <linux/vmalloc.h>
 #include <linux/delay.h>
 #include <asm/div64.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "dst_priv.h"
 #include "dst_common.h"
 
diff --git a/drivers/media/pci/bt8xx/dst_ca.c b/drivers/media/pci/bt8xx/dst_ca.c
index 530b3e9764ce..0a7623c0fc8e 100644
--- a/drivers/media/pci/bt8xx/dst_ca.c
+++ b/drivers/media/pci/bt8xx/dst_ca.c
@@ -25,8 +25,8 @@
 #include <linux/mutex.h>
 #include <linux/string.h>
 #include <linux/dvb/ca.h>
-#include "dvbdev.h"
-#include "dvb_frontend.h"
+#include <media/dvbdev.h>
+#include <media/dvb_frontend.h>
 #include "dst_ca.h"
 #include "dst_common.h"
 
diff --git a/drivers/media/pci/bt8xx/dvb-bt8xx.c b/drivers/media/pci/bt8xx/dvb-bt8xx.c
index ad617871ce9b..f60d69ac515b 100644
--- a/drivers/media/pci/bt8xx/dvb-bt8xx.c
+++ b/drivers/media/pci/bt8xx/dvb-bt8xx.c
@@ -26,10 +26,10 @@
 #include <linux/slab.h>
 #include <linux/i2c.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
 #include "dvb-bt8xx.h"
 #include "bt878.h"
 
diff --git a/drivers/media/pci/bt8xx/dvb-bt8xx.h b/drivers/media/pci/bt8xx/dvb-bt8xx.h
index 0ec538e23b4e..3184b3f3a85e 100644
--- a/drivers/media/pci/bt8xx/dvb-bt8xx.h
+++ b/drivers/media/pci/bt8xx/dvb-bt8xx.h
@@ -23,8 +23,8 @@
 
 #include <linux/i2c.h>
 #include <linux/mutex.h>
-#include "dvbdev.h"
-#include "dvb_net.h"
+#include <media/dvbdev.h>
+#include <media/dvb_net.h>
 #include "bttv.h"
 #include "mt352.h"
 #include "sp887x.h"
diff --git a/drivers/media/pci/cobalt/Kconfig b/drivers/media/pci/cobalt/Kconfig
index 70343829a125..aa35cbc0a904 100644
--- a/drivers/media/pci/cobalt/Kconfig
+++ b/drivers/media/pci/cobalt/Kconfig
@@ -6,6 +6,7 @@ config VIDEO_COBALT
 	depends on SND
 	depends on MTD
 	select I2C_ALGOBIT
+	select SND_PCM
 	select VIDEO_ADV7604
 	select VIDEO_ADV7511
 	select VIDEO_ADV7842
diff --git a/drivers/media/pci/cx18/Makefile b/drivers/media/pci/cx18/Makefile
index 98914a40f6ac..9c82c2df05e1 100644
--- a/drivers/media/pci/cx18/Makefile
+++ b/drivers/media/pci/cx18/Makefile
@@ -9,6 +9,5 @@ cx18-alsa-objs := cx18-alsa-main.o cx18-alsa-pcm.o
 obj-$(CONFIG_VIDEO_CX18) += cx18.o
 obj-$(CONFIG_VIDEO_CX18_ALSA) += cx18-alsa.o
 
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
 ccflags-y += -Idrivers/media/tuners
diff --git a/drivers/media/pci/cx18/cx18-alsa-pcm.c b/drivers/media/pci/cx18/cx18-alsa-pcm.c
index aadd76466aec..4f31042a442a 100644
--- a/drivers/media/pci/cx18/cx18-alsa-pcm.c
+++ b/drivers/media/pci/cx18/cx18-alsa-pcm.c
@@ -41,7 +41,7 @@ MODULE_PARM_DESC(pcm_debug, "enable debug messages for pcm");
 #define dprintk(fmt, arg...) do {					\
 	    if (pcm_debug)						\
 		printk(KERN_INFO "cx18-alsa-pcm %s: " fmt,		\
-				  __func__, ##arg); 			\
+				  __func__, ##arg);			\
 	} while (0)
 
 static const struct snd_pcm_hardware snd_cx18_hw_capture = {
diff --git a/drivers/media/pci/cx18/cx18-av-audio.c b/drivers/media/pci/cx18/cx18-av-audio.c
index 8b95e9aae576..3abc54cbe4a1 100644
--- a/drivers/media/pci/cx18/cx18-av-audio.c
+++ b/drivers/media/pci/cx18/cx18-av-audio.c
@@ -31,7 +31,7 @@ static int set_audclk_freq(struct cx18 *cx, u32 freq)
 	 * would ideally be:
 	 *
 	 * NTSC Color subcarrier freq * 8 =
-	 * 	4.5 MHz/286 * 455/2 * 8 = 28.63636363... MHz
+	 *	4.5 MHz/286 * 455/2 * 8 = 28.63636363... MHz
 	 *
 	 * The accidents of history and rationale that explain from where this
 	 * combination of magic numbers originate can be found in:
diff --git a/drivers/media/pci/cx18/cx18-av-core.c b/drivers/media/pci/cx18/cx18-av-core.c
index cf8817e9c8b9..eda343322ee0 100644
--- a/drivers/media/pci/cx18/cx18-av-core.c
+++ b/drivers/media/pci/cx18/cx18-av-core.c
@@ -236,10 +236,10 @@ static void cx18_av_initialize(struct v4l2_subdev *sd)
 	 */
 	cx18_av_and_or4(cx, CXADEC_AFE_CTRL, 0xFF000000, 0x00005D00);
 
-/* 	if(dwEnable && dw3DCombAvailable) { */
-/*      	CxDevWrReg(CXADEC_SRC_COMB_CFG, 0x7728021F); */
+/*	if(dwEnable && dw3DCombAvailable) { */
+/*		CxDevWrReg(CXADEC_SRC_COMB_CFG, 0x7728021F); */
 /*    } else { */
-/*      	CxDevWrReg(CXADEC_SRC_COMB_CFG, 0x6628021F); */
+/*		CxDevWrReg(CXADEC_SRC_COMB_CFG, 0x6628021F); */
 /*    } */
 	cx18_av_write4(cx, CXADEC_SRC_COMB_CFG, 0x6628021F);
 	default_volume = cx18_av_read(cx, 0x8d4);
@@ -319,13 +319,13 @@ void cx18_av_std_setup(struct cx18 *cx)
 		 * vblank656: half lines after line 625/mid-313 of blanked video
 		 * vblank:    half lines, after line 5/317, of blanked video
 		 * vactive:   half lines of active video +
-		 * 		5 half lines after the end of active video
+		 *		5 half lines after the end of active video
 		 *
 		 * As far as I can tell:
 		 * vblank656 starts counting from the falling edge of the first
-		 * 	vsync pulse (start of line 1 or mid-313)
+		 *	vsync pulse (start of line 1 or mid-313)
 		 * vblank starts counting from the after the 5 vsync pulses and
-		 * 	5 or 4 equalization pulses (start of line 6 or 318)
+		 *	5 or 4 equalization pulses (start of line 6 or 318)
 		 *
 		 * For 625 line systems the driver will extract VBI information
 		 * from lines 6-23 and lines 318-335 (but the slicer can only
@@ -395,9 +395,9 @@ void cx18_av_std_setup(struct cx18 *cx)
 		 *
 		 * As far as I can tell:
 		 * vblank656 starts counting from the falling edge of the first
-		 * 	vsync pulse (start of line 4 or mid-266)
+		 *	vsync pulse (start of line 4 or mid-266)
 		 * vblank starts counting from the after the 6 vsync pulses and
-		 * 	6 or 5 equalization pulses (start of line 10 or 272)
+		 *	6 or 5 equalization pulses (start of line 10 or 272)
 		 *
 		 * For 525 line systems the driver will extract VBI information
 		 * from lines 10-21 and lines 273-284.
@@ -851,7 +851,7 @@ static int cx18_av_s_std(struct v4l2_subdev *sd, v4l2_std_id norm)
 	struct cx18_av_state *state = to_cx18_av_state(sd);
 	struct cx18 *cx = v4l2_get_subdevdata(sd);
 
-	u8 fmt = 0; 	/* zero is autodetect */
+	u8 fmt = 0;	/* zero is autodetect */
 	u8 pal_m = 0;
 
 	if (state->radio == 0 && state->std == norm)
diff --git a/drivers/media/pci/cx18/cx18-av-core.h b/drivers/media/pci/cx18/cx18-av-core.h
index c976ce6e7a78..1a37f269d2e4 100644
--- a/drivers/media/pci/cx18/cx18-av-core.h
+++ b/drivers/media/pci/cx18/cx18-av-core.h
@@ -349,7 +349,7 @@ static inline struct v4l2_subdev *to_sd(struct v4l2_ctrl *ctrl)
 }
 
 /* ----------------------------------------------------------------------- */
-/* cx18_av-core.c 							   */
+/* cx18_av-core.c							   */
 int cx18_av_write(struct cx18 *cx, u16 addr, u8 value);
 int cx18_av_write4(struct cx18 *cx, u16 addr, u32 value);
 int cx18_av_write4_noretry(struct cx18 *cx, u16 addr, u32 value);
diff --git a/drivers/media/pci/cx18/cx18-cards.c b/drivers/media/pci/cx18/cx18-cards.c
index 11e898e66ce9..c2cf965d639e 100644
--- a/drivers/media/pci/cx18/cx18-cards.c
+++ b/drivers/media/pci/cx18/cx18-cards.c
@@ -388,7 +388,7 @@ static const struct cx18_card cx18_card_cnxt_raptor_pal = {
 		{ CX18_CARD_INPUT_COMPOSITE2, 2, CX18_AV_COMPOSITE6 },
 	},
 	.audio_inputs = {
-		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5, 	    0 },
+		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5,	    0 },
 		{ CX18_CARD_INPUT_LINE_IN1,  CX18_AV_AUDIO_SERIAL1, 1 },
 		{ CX18_CARD_INPUT_LINE_IN2,  CX18_AV_AUDIO_SERIAL2, 1 },
 	},
@@ -439,7 +439,7 @@ static const struct cx18_card cx18_card_toshiba_qosmio_dvbt = {
 		{ CX18_CARD_INPUT_COMPOSITE1, 1, CX18_AV_COMPOSITE1 },
 	},
 	.audio_inputs = {
-		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5, 	    0 },
+		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5,	    0 },
 		{ CX18_CARD_INPUT_LINE_IN1,  CX18_AV_AUDIO_SERIAL1, 1 },
 	},
 	.tuners = {
@@ -485,7 +485,7 @@ static const struct cx18_card cx18_card_leadtek_pvr2100 = {
 		{ CX18_CARD_INPUT_COMPONENT1, 1, CX18_AV_COMPONENT1 },
 	},
 	.audio_inputs = {
-		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5, 	    0 },
+		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5,	    0 },
 		{ CX18_CARD_INPUT_LINE_IN1,  CX18_AV_AUDIO_SERIAL1, 1 },
 	},
 	.tuners = {
@@ -538,7 +538,7 @@ static const struct cx18_card cx18_card_leadtek_dvr3100h = {
 		{ CX18_CARD_INPUT_COMPONENT1, 1, CX18_AV_COMPONENT1 },
 	},
 	.audio_inputs = {
-		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5, 	    0 },
+		{ CX18_CARD_INPUT_AUD_TUNER, CX18_AV_AUDIO5,	    0 },
 		{ CX18_CARD_INPUT_LINE_IN1,  CX18_AV_AUDIO_SERIAL1, 1 },
 	},
 	.tuners = {
diff --git a/drivers/media/pci/cx18/cx18-cards.h b/drivers/media/pci/cx18/cx18-cards.h
index 667e2d7b1d03..02d0fb703a41 100644
--- a/drivers/media/pci/cx18/cx18-cards.h
+++ b/drivers/media/pci/cx18/cx18-cards.h
@@ -25,30 +25,24 @@
 #define CX18_HW_418_AV			(1 << 4)
 #define CX18_HW_GPIO_MUX		(1 << 5)
 #define CX18_HW_GPIO_RESET_CTRL		(1 << 6)
-#define CX18_HW_Z8F0811_IR_TX_HAUP	(1 << 7)
-#define CX18_HW_Z8F0811_IR_RX_HAUP	(1 << 8)
-#define CX18_HW_Z8F0811_IR_HAUP	(CX18_HW_Z8F0811_IR_RX_HAUP | \
-				 CX18_HW_Z8F0811_IR_TX_HAUP)
-
-#define CX18_HW_IR_ANY (CX18_HW_Z8F0811_IR_RX_HAUP | \
-			CX18_HW_Z8F0811_IR_TX_HAUP)
+#define CX18_HW_Z8F0811_IR_HAUP		(1 << 7)
 
 /* video inputs */
 #define	CX18_CARD_INPUT_VID_TUNER	1
-#define	CX18_CARD_INPUT_SVIDEO1 	2
-#define	CX18_CARD_INPUT_SVIDEO2 	3
-#define	CX18_CARD_INPUT_COMPOSITE1 	4
-#define	CX18_CARD_INPUT_COMPOSITE2 	5
-#define	CX18_CARD_INPUT_COMPONENT1 	6
+#define	CX18_CARD_INPUT_SVIDEO1		2
+#define	CX18_CARD_INPUT_SVIDEO2		3
+#define	CX18_CARD_INPUT_COMPOSITE1	4
+#define	CX18_CARD_INPUT_COMPOSITE2	5
+#define	CX18_CARD_INPUT_COMPONENT1	6
 
 /* audio inputs */
 #define	CX18_CARD_INPUT_AUD_TUNER	1
-#define	CX18_CARD_INPUT_LINE_IN1 	2
-#define	CX18_CARD_INPUT_LINE_IN2 	3
+#define	CX18_CARD_INPUT_LINE_IN1	2
+#define	CX18_CARD_INPUT_LINE_IN2	3
 
 #define CX18_CARD_MAX_VIDEO_INPUTS 6
 #define CX18_CARD_MAX_AUDIO_INPUTS 3
-#define CX18_CARD_MAX_TUNERS  	   2
+#define CX18_CARD_MAX_TUNERS	   2
 
 /* V4L2 capability aliases */
 #define CX18_CAP_ENCODER (V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_TUNER | \
@@ -57,7 +51,7 @@
 			  V4L2_CAP_SLICED_VBI_CAPTURE)
 
 struct cx18_card_video_input {
-	u8  video_type; 	/* video input type */
+	u8  video_type;		/* video input type */
 	u8  audio_index;	/* index in cx18_card_audio_input array */
 	u32 video_input;	/* hardware video input */
 };
@@ -80,7 +74,7 @@ struct cx18_card_pci_info {
 /* The mask is the set of bits used by the operation */
 
 struct cx18_gpio_init { /* set initial GPIO DIR and OUT values */
-	u32 direction; 	/* DIR setting. Leave to 0 if no init is needed */
+	u32 direction;	/* DIR setting. Leave to 0 if no init is needed */
 	u32 initial_value;
 };
 
@@ -92,16 +86,16 @@ struct cx18_gpio_i2c_slave_reset {
 	u32 ir_reset_mask;  /* GPIO to reset the Zilog Z8F0811 IR contoller */
 };
 
-struct cx18_gpio_audio_input { 	/* select tuner/line in input */
-	u32 mask; 		/* leave to 0 if not supported */
+struct cx18_gpio_audio_input {	/* select tuner/line in input */
+	u32 mask;		/* leave to 0 if not supported */
 	u32 tuner;
 	u32 linein;
 	u32 radio;
 };
 
 struct cx18_card_tuner {
-	v4l2_std_id std; 	/* standard for which the tuner is suitable */
-	int 	    tuner; 	/* tuner ID (from tuner.h) */
+	v4l2_std_id std;	/* standard for which the tuner is suitable */
+	int	    tuner;	/* tuner ID (from tuner.h) */
 };
 
 struct cx18_card_tuner_i2c {
@@ -134,8 +128,8 @@ struct cx18_card {
 	struct cx18_card_audio_input radio_input;
 
 	/* GPIO card-specific settings */
-	u8 xceive_pin; 		/* XCeive tuner GPIO reset pin */
-	struct cx18_gpio_init 		 gpio_init;
+	u8 xceive_pin;		/* XCeive tuner GPIO reset pin */
+	struct cx18_gpio_init		 gpio_init;
 	struct cx18_gpio_i2c_slave_reset gpio_i2c_slave_reset;
 	struct cx18_gpio_audio_input    gpio_audio_input;
 
diff --git a/drivers/media/pci/cx18/cx18-driver.h b/drivers/media/pci/cx18/cx18-driver.h
index 7be2088c45fe..0b707faca543 100644
--- a/drivers/media/pci/cx18/cx18-driver.h
+++ b/drivers/media/pci/cx18/cx18-driver.h
@@ -50,12 +50,12 @@
 #include "cx23418.h"
 
 /* DVB */
-#include "demux.h"
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
-#include "dvbdev.h"
+#include <media/demux.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
+#include <media/dvbdev.h>
 
 /* Videobuf / YUV support */
 #include <media/videobuf-core.h>
@@ -75,8 +75,8 @@
 /* Supported cards */
 #define CX18_CARD_HVR_1600_ESMT	      0	/* Hauppauge HVR 1600 (ESMT memory) */
 #define CX18_CARD_HVR_1600_SAMSUNG    1	/* Hauppauge HVR 1600 (Samsung memory) */
-#define CX18_CARD_COMPRO_H900 	      2	/* Compro VideoMate H900 */
-#define CX18_CARD_YUAN_MPC718 	      3	/* Yuan MPC718 */
+#define CX18_CARD_COMPRO_H900	      2	/* Compro VideoMate H900 */
+#define CX18_CARD_YUAN_MPC718	      3	/* Yuan MPC718 */
 #define CX18_CARD_CNXT_RAPTOR_PAL     4	/* Conexant Raptor PAL */
 #define CX18_CARD_TOSHIBA_QOSMIO_DVBT 5 /* Toshiba Qosmio Interal DVB-T/Analog*/
 #define CX18_CARD_LEADTEK_PVR2100     6 /* Leadtek WinFast PVR2100 */
@@ -99,9 +99,9 @@
 #define PCI_DEVICE_ID_CX23418 0x5b7a
 
 /* subsystem vendor ID */
-#define CX18_PCI_ID_HAUPPAUGE 		0x0070
-#define CX18_PCI_ID_COMPRO 		0x185b
-#define CX18_PCI_ID_YUAN 		0x12ab
+#define CX18_PCI_ID_HAUPPAUGE		0x0070
+#define CX18_PCI_ID_COMPRO		0x185b
+#define CX18_PCI_ID_YUAN		0x12ab
 #define CX18_PCI_ID_CONEXANT		0x14f1
 #define CX18_PCI_ID_TOSHIBA		0x1179
 #define CX18_PCI_ID_LEADTEK		0x107D
@@ -260,7 +260,7 @@ struct cx18_options {
 #define CX18_F_M_NEED_SWAP  0	/* mdl buffer data must be endianness swapped */
 
 /* per-stream, s_flags */
-#define CX18_F_S_CLAIMED 	3	/* this stream is claimed */
+#define CX18_F_S_CLAIMED	3	/* this stream is claimed */
 #define CX18_F_S_STREAMING      4	/* the fw is decoding/encoding this stream */
 #define CX18_F_S_INTERNAL_USE	5	/* this stream is used internally (sliced VBI processing) */
 #define CX18_F_S_STREAMOFF	7	/* signal end of stream EOS */
@@ -268,12 +268,12 @@ struct cx18_options {
 #define CX18_F_S_STOPPING	9	/* telling the fw to stop capturing */
 
 /* per-cx18, i_flags */
-#define CX18_F_I_LOADED_FW		0 	/* Loaded firmware 1st time */
-#define CX18_F_I_EOS			4 	/* End of encoder stream */
-#define CX18_F_I_RADIO_USER		5 	/* radio tuner is selected */
-#define CX18_F_I_ENC_PAUSED		13 	/* the encoder is paused */
-#define CX18_F_I_INITED			21 	/* set after first open */
-#define CX18_F_I_FAILED			22 	/* set if first open failed */
+#define CX18_F_I_LOADED_FW		0	/* Loaded firmware 1st time */
+#define CX18_F_I_EOS			4	/* End of encoder stream */
+#define CX18_F_I_RADIO_USER		5	/* radio tuner is selected */
+#define CX18_F_I_ENC_PAUSED		13	/* the encoder is paused */
+#define CX18_F_I_INITED			21	/* set after first open */
+#define CX18_F_I_FAILED			22	/* set if first open failed */
 
 /* These are the VBI types as they appear in the embedded VBI private packets. */
 #define CX18_SLICED_TYPE_TELETEXT_B     (1)
@@ -370,7 +370,7 @@ struct cx18_stream {
 	   is not actually created. */
 	struct video_device video_dev;	/* v4l2_dev is NULL when stream not created */
 	struct cx18_dvb *dvb;		/* DVB / Digital Transport */
-	struct cx18 *cx; 		/* for ease of use */
+	struct cx18 *cx;		/* for ease of use */
 	const char *name;		/* name of the stream */
 	int type;			/* stream type */
 	u32 handle;			/* task handle */
@@ -525,14 +525,14 @@ struct vbi_info {
 	 * into the MPEG PS stream.
 	 *
 	 * In each sliced_mpeg_data[] buffer is:
-	 * 	16 byte MPEG-2 PS Program Pack Header
-	 * 	16 byte MPEG-2 Private Stream 1 PES Header
-	 * 	 4 byte magic number: "itv0" or "ITV0"
-	 * 	 4 byte first  field line mask, if "itv0"
-	 * 	 4 byte second field line mask, if "itv0"
-	 * 	36 lines, if "ITV0"; or <36 lines, if "itv0"; of sliced VBI data
+	 *	16 byte MPEG-2 PS Program Pack Header
+	 *	16 byte MPEG-2 Private Stream 1 PES Header
+	 *	 4 byte magic number: "itv0" or "ITV0"
+	 *	 4 byte first  field line mask, if "itv0"
+	 *	 4 byte second field line mask, if "itv0"
+	 *	36 lines, if "ITV0"; or <36 lines, if "itv0"; of sliced VBI data
 	 *
-	 * 	Each line in the payload is
+	 *	Each line in the payload is
 	 *	 1 byte line header derived from the SDID (WSS, CC, VPS, etc.)
 	 *	42 bytes of line data
 	 *
@@ -583,7 +583,7 @@ struct cx18 {
 	u8 nof_inputs;		/* number of video inputs */
 	u8 nof_audio_inputs;	/* number of audio inputs */
 	u32 v4l2_cap;		/* V4L2 capabilities of card */
-	u32 hw_flags; 		/* Hardware description of the board */
+	u32 hw_flags;		/* Hardware description of the board */
 	unsigned int free_mdl_idx;
 	struct cx18_scb __iomem *scb; /* pointer to SCB */
 	struct mutex epu2apu_mb_lock; /* protect driver to chip mailbox in SCB*/
@@ -602,10 +602,10 @@ struct cx18 {
 	u32 dualwatch_stereo_mode;
 
 	struct mutex serialize_lock;    /* mutex used to serialize open/close/start/stop/ioctl operations */
-	struct cx18_options options; 	/* User options */
+	struct cx18_options options;	/* User options */
 	int stream_buffers[CX18_MAX_STREAMS]; /* # of buffers for each stream */
 	int stream_buf_size[CX18_MAX_STREAMS]; /* Stream buffer size */
-	struct cx18_stream streams[CX18_MAX_STREAMS]; 	/* Stream data */
+	struct cx18_stream streams[CX18_MAX_STREAMS];	/* Stream data */
 	struct snd_cx18_card *alsa; /* ALSA interface for PCM capture stream */
 	void (*pcm_announce_callback)(struct snd_cx18_card *card, u8 *pcm_data,
 				      size_t num_bytes);
diff --git a/drivers/media/pci/cx18/cx18-fileops.c b/drivers/media/pci/cx18/cx18-fileops.c
index 4f9c2395941b..a3f44e30f821 100644
--- a/drivers/media/pci/cx18/cx18-fileops.c
+++ b/drivers/media/pci/cx18/cx18-fileops.c
@@ -602,18 +602,18 @@ ssize_t cx18_v4l2_read(struct file *filp, char __user *buf, size_t count,
 	return cx18_read_pos(s, buf, count, pos, filp->f_flags & O_NONBLOCK);
 }
 
-unsigned int cx18_v4l2_enc_poll(struct file *filp, poll_table *wait)
+__poll_t cx18_v4l2_enc_poll(struct file *filp, poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct cx18_open_id *id = file2id(filp);
 	struct cx18 *cx = id->cx;
 	struct cx18_stream *s = &cx->streams[id->type];
 	int eof = test_bit(CX18_F_S_STREAMOFF, &s->s_flags);
-	unsigned res = 0;
+	__poll_t res = 0;
 
 	/* Start a capture if there is none */
 	if (!eof && !test_bit(CX18_F_S_STREAMING, &s->s_flags) &&
-			(req_events & (POLLIN | POLLRDNORM))) {
+			(req_events & (EPOLLIN | EPOLLRDNORM))) {
 		int rc;
 
 		mutex_lock(&cx->serialize_lock);
@@ -622,33 +622,33 @@ unsigned int cx18_v4l2_enc_poll(struct file *filp, poll_table *wait)
 		if (rc) {
 			CX18_DEBUG_INFO("Could not start capture for %s (%d)\n",
 					s->name, rc);
-			return POLLERR;
+			return EPOLLERR;
 		}
 		CX18_DEBUG_FILE("Encoder poll started capture\n");
 	}
 
 	if ((s->vb_type == V4L2_BUF_TYPE_VIDEO_CAPTURE) &&
 		(id->type == CX18_ENC_STREAM_TYPE_YUV)) {
-		int videobuf_poll = videobuf_poll_stream(filp, &s->vbuf_q, wait);
+		__poll_t videobuf_poll = videobuf_poll_stream(filp, &s->vbuf_q, wait);
 
 		if (v4l2_event_pending(&id->fh))
-			res |= POLLPRI;
-                if (eof && videobuf_poll == POLLERR)
-			return res | POLLHUP;
+			res |= EPOLLPRI;
+		if (eof && videobuf_poll == EPOLLERR)
+			return res | EPOLLHUP;
 		return res | videobuf_poll;
 	}
 
 	/* add stream's waitq to the poll list */
 	CX18_DEBUG_HI_FILE("Encoder poll\n");
 	if (v4l2_event_pending(&id->fh))
-		res |= POLLPRI;
+		res |= EPOLLPRI;
 	else
 		poll_wait(filp, &s->waitq, wait);
 
 	if (atomic_read(&s->q_full.depth))
-		return res | POLLIN | POLLRDNORM;
+		return res | EPOLLIN | EPOLLRDNORM;
 	if (eof)
-		return res | POLLHUP;
+		return res | EPOLLHUP;
 	return res;
 }
 
diff --git a/drivers/media/pci/cx18/cx18-fileops.h b/drivers/media/pci/cx18/cx18-fileops.h
index 37ef34e866cb..5b44d30efd8f 100644
--- a/drivers/media/pci/cx18/cx18-fileops.h
+++ b/drivers/media/pci/cx18/cx18-fileops.h
@@ -23,7 +23,7 @@ ssize_t cx18_v4l2_read(struct file *filp, char __user *buf, size_t count,
 ssize_t cx18_v4l2_write(struct file *filp, const char __user *buf, size_t count,
 		       loff_t *pos);
 int cx18_v4l2_close(struct file *filp);
-unsigned int cx18_v4l2_enc_poll(struct file *filp, poll_table *wait);
+__poll_t cx18_v4l2_enc_poll(struct file *filp, poll_table *wait);
 int cx18_start_capture(struct cx18_open_id *id);
 void cx18_stop_capture(struct cx18_open_id *id, int gop_end);
 void cx18_mute(struct cx18 *cx);
diff --git a/drivers/media/pci/cx18/cx18-firmware.c b/drivers/media/pci/cx18/cx18-firmware.c
index 1b34ea1c3730..498a1854b3b0 100644
--- a/drivers/media/pci/cx18/cx18-firmware.c
+++ b/drivers/media/pci/cx18/cx18-firmware.c
@@ -23,65 +23,65 @@
 #include "cx18-cards.h"
 #include <linux/firmware.h>
 
-#define CX18_PROC_SOFT_RESET 		0xc70010
-#define CX18_DDR_SOFT_RESET          	0xc70014
-#define CX18_CLOCK_SELECT1           	0xc71000
-#define CX18_CLOCK_SELECT2           	0xc71004
-#define CX18_HALF_CLOCK_SELECT1      	0xc71008
-#define CX18_HALF_CLOCK_SELECT2      	0xc7100C
-#define CX18_CLOCK_POLARITY1         	0xc71010
-#define CX18_CLOCK_POLARITY2         	0xc71014
-#define CX18_ADD_DELAY_ENABLE1       	0xc71018
-#define CX18_ADD_DELAY_ENABLE2       	0xc7101C
-#define CX18_CLOCK_ENABLE1           	0xc71020
-#define CX18_CLOCK_ENABLE2           	0xc71024
-
-#define CX18_REG_BUS_TIMEOUT_EN      	0xc72024
-
-#define CX18_FAST_CLOCK_PLL_INT      	0xc78000
-#define CX18_FAST_CLOCK_PLL_FRAC     	0xc78004
-#define CX18_FAST_CLOCK_PLL_POST     	0xc78008
-#define CX18_FAST_CLOCK_PLL_PRESCALE 	0xc7800C
+#define CX18_PROC_SOFT_RESET		0xc70010
+#define CX18_DDR_SOFT_RESET		0xc70014
+#define CX18_CLOCK_SELECT1		0xc71000
+#define CX18_CLOCK_SELECT2		0xc71004
+#define CX18_HALF_CLOCK_SELECT1		0xc71008
+#define CX18_HALF_CLOCK_SELECT2		0xc7100C
+#define CX18_CLOCK_POLARITY1		0xc71010
+#define CX18_CLOCK_POLARITY2		0xc71014
+#define CX18_ADD_DELAY_ENABLE1		0xc71018
+#define CX18_ADD_DELAY_ENABLE2		0xc7101C
+#define CX18_CLOCK_ENABLE1		0xc71020
+#define CX18_CLOCK_ENABLE2		0xc71024
+
+#define CX18_REG_BUS_TIMEOUT_EN		0xc72024
+
+#define CX18_FAST_CLOCK_PLL_INT		0xc78000
+#define CX18_FAST_CLOCK_PLL_FRAC	0xc78004
+#define CX18_FAST_CLOCK_PLL_POST	0xc78008
+#define CX18_FAST_CLOCK_PLL_PRESCALE	0xc7800C
 #define CX18_FAST_CLOCK_PLL_ADJUST_BANDWIDTH 0xc78010
 
-#define CX18_SLOW_CLOCK_PLL_INT      	0xc78014
-#define CX18_SLOW_CLOCK_PLL_FRAC     	0xc78018
-#define CX18_SLOW_CLOCK_PLL_POST     	0xc7801C
+#define CX18_SLOW_CLOCK_PLL_INT		0xc78014
+#define CX18_SLOW_CLOCK_PLL_FRAC	0xc78018
+#define CX18_SLOW_CLOCK_PLL_POST	0xc7801C
 #define CX18_MPEG_CLOCK_PLL_INT		0xc78040
 #define CX18_MPEG_CLOCK_PLL_FRAC	0xc78044
 #define CX18_MPEG_CLOCK_PLL_POST	0xc78048
-#define CX18_PLL_POWER_DOWN          	0xc78088
+#define CX18_PLL_POWER_DOWN		0xc78088
 #define CX18_SW1_INT_STATUS             0xc73104
 #define CX18_SW1_INT_ENABLE_PCI         0xc7311C
 #define CX18_SW2_INT_SET                0xc73140
 #define CX18_SW2_INT_STATUS             0xc73144
-#define CX18_ADEC_CONTROL            	0xc78120
+#define CX18_ADEC_CONTROL		0xc78120
 
-#define CX18_DDR_REQUEST_ENABLE      	0xc80000
-#define CX18_DDR_CHIP_CONFIG         	0xc80004
-#define CX18_DDR_REFRESH            	0xc80008
-#define CX18_DDR_TIMING1             	0xc8000C
-#define CX18_DDR_TIMING2             	0xc80010
+#define CX18_DDR_REQUEST_ENABLE		0xc80000
+#define CX18_DDR_CHIP_CONFIG		0xc80004
+#define CX18_DDR_REFRESH		0xc80008
+#define CX18_DDR_TIMING1		0xc8000C
+#define CX18_DDR_TIMING2		0xc80010
 #define CX18_DDR_POWER_REG		0xc8001C
 
-#define CX18_DDR_TUNE_LANE           	0xc80048
-#define CX18_DDR_INITIAL_EMRS        	0xc80054
-#define CX18_DDR_MB_PER_ROW_7        	0xc8009C
-#define CX18_DDR_BASE_63_ADDR        	0xc804FC
-
-#define CX18_WMB_CLIENT02            	0xc90108
-#define CX18_WMB_CLIENT05            	0xc90114
-#define CX18_WMB_CLIENT06            	0xc90118
-#define CX18_WMB_CLIENT07            	0xc9011C
-#define CX18_WMB_CLIENT08            	0xc90120
-#define CX18_WMB_CLIENT09            	0xc90124
-#define CX18_WMB_CLIENT10            	0xc90128
-#define CX18_WMB_CLIENT11            	0xc9012C
-#define CX18_WMB_CLIENT12            	0xc90130
-#define CX18_WMB_CLIENT13            	0xc90134
-#define CX18_WMB_CLIENT14            	0xc90138
-
-#define CX18_DSP0_INTERRUPT_MASK     	0xd0004C
+#define CX18_DDR_TUNE_LANE		0xc80048
+#define CX18_DDR_INITIAL_EMRS		0xc80054
+#define CX18_DDR_MB_PER_ROW_7		0xc8009C
+#define CX18_DDR_BASE_63_ADDR		0xc804FC
+
+#define CX18_WMB_CLIENT02		0xc90108
+#define CX18_WMB_CLIENT05		0xc90114
+#define CX18_WMB_CLIENT06		0xc90118
+#define CX18_WMB_CLIENT07		0xc9011C
+#define CX18_WMB_CLIENT08		0xc90120
+#define CX18_WMB_CLIENT09		0xc90124
+#define CX18_WMB_CLIENT10		0xc90128
+#define CX18_WMB_CLIENT11		0xc9012C
+#define CX18_WMB_CLIENT12		0xc90130
+#define CX18_WMB_CLIENT13		0xc90134
+#define CX18_WMB_CLIENT14		0xc90138
+
+#define CX18_DSP0_INTERRUPT_MASK	0xd0004C
 
 #define APU_ROM_SYNC1 0x6D676553 /* "mgeS" */
 #define APU_ROM_SYNC2 0x72646548 /* "rdeH" */
@@ -229,7 +229,7 @@ void cx18_init_power(struct cx18 *cx, int lowpwr)
 	 * would ideally be:
 	 *
 	 * NTSC Color subcarrier freq * 8 =
-	 * 	4.5 MHz/286 * 455/2 * 8 = 28.63636363... MHz
+	 *	4.5 MHz/286 * 455/2 * 8 = 28.63636363... MHz
 	 *
 	 * The accidents of history and rationale that explain from where this
 	 * combination of magic numbers originate can be found in:
diff --git a/drivers/media/pci/cx18/cx18-i2c.c b/drivers/media/pci/cx18/cx18-i2c.c
index 7f588eeac60f..f0eb181f2b94 100644
--- a/drivers/media/pci/cx18/cx18-i2c.c
+++ b/drivers/media/pci/cx18/cx18-i2c.c
@@ -47,8 +47,7 @@ static const u8 hw_addrs[] = {
 	0,				/* CX18_HW_418_AV */
 	0,				/* CX18_HW_GPIO_MUX */
 	0,				/* CX18_HW_GPIO_RESET_CTRL */
-	CX18_Z8F0811_IR_TX_I2C_ADDR,	/* CX18_HW_Z8F0811_IR_TX_HAUP */
-	CX18_Z8F0811_IR_RX_I2C_ADDR,	/* CX18_HW_Z8F0811_IR_RX_HAUP */
+	CX18_Z8F0811_IR_RX_I2C_ADDR,	/* CX18_HW_Z8F0811_IR_HAUP */
 };
 
 /* This array should match the CX18_HW_ defines */
@@ -61,8 +60,7 @@ static const u8 hw_bus[] = {
 	0,	/* CX18_HW_418_AV */
 	0,	/* CX18_HW_GPIO_MUX */
 	0,	/* CX18_HW_GPIO_RESET_CTRL */
-	0,	/* CX18_HW_Z8F0811_IR_TX_HAUP */
-	0,	/* CX18_HW_Z8F0811_IR_RX_HAUP */
+	0,	/* CX18_HW_Z8F0811_IR_HAUP */
 };
 
 /* This array should match the CX18_HW_ defines */
@@ -74,8 +72,7 @@ static const char * const hw_devicenames[] = {
 	"cx23418_AV",
 	"gpio_mux",
 	"gpio_reset_ctrl",
-	"ir_tx_z8f0811_haup",
-	"ir_rx_z8f0811_haup",
+	"ir_z8f0811_haup",
 };
 
 static int cx18_i2c_new_ir(struct cx18 *cx, struct i2c_adapter *adap, u32 hw,
@@ -90,7 +87,7 @@ static int cx18_i2c_new_ir(struct cx18 *cx, struct i2c_adapter *adap, u32 hw,
 
 	/* Our default information for ir-kbd-i2c.c to use */
 	switch (hw) {
-	case CX18_HW_Z8F0811_IR_RX_HAUP:
+	case CX18_HW_Z8F0811_IR_HAUP:
 		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
 		init_data->type = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
@@ -129,7 +126,7 @@ int cx18_i2c_register(struct cx18 *cx, unsigned idx)
 		return sd != NULL ? 0 : -1;
 	}
 
-	if (hw & CX18_HW_IR_ANY)
+	if (hw == CX18_HW_Z8F0811_IR_HAUP)
 		return cx18_i2c_new_ir(cx, adap, hw, type, hw_addrs[idx]);
 
 	/* Is it not an I2C device or one we do not wish to register? */
diff --git a/drivers/media/pci/cx18/cx18-mailbox.c b/drivers/media/pci/cx18/cx18-mailbox.c
index 763f960fc918..f66dd63e1994 100644
--- a/drivers/media/pci/cx18/cx18-mailbox.c
+++ b/drivers/media/pci/cx18/cx18-mailbox.c
@@ -35,7 +35,7 @@ struct cx18_api_info {
 	u32 cmd;
 	u8 flags;		/* Flags, see above */
 	u8 rpu;			/* Processing unit */
-	const char *name; 	/* The name of the command */
+	const char *name;	/* The name of the command */
 };
 
 #define API_ENTRY(rpu, x, f) { (x), (f), (rpu), #x }
@@ -43,9 +43,9 @@ struct cx18_api_info {
 static const struct cx18_api_info api_info[] = {
 	/* MPEG encoder API */
 	API_ENTRY(CPU, CX18_CPU_SET_CHANNEL_TYPE,		0),
-	API_ENTRY(CPU, CX18_EPU_DEBUG, 				0),
-	API_ENTRY(CPU, CX18_CREATE_TASK, 			0),
-	API_ENTRY(CPU, CX18_DESTROY_TASK, 			0),
+	API_ENTRY(CPU, CX18_EPU_DEBUG,				0),
+	API_ENTRY(CPU, CX18_CREATE_TASK,			0),
+	API_ENTRY(CPU, CX18_DESTROY_TASK,			0),
 	API_ENTRY(CPU, CX18_CPU_CAPTURE_START,                  API_SLOW),
 	API_ENTRY(CPU, CX18_CPU_CAPTURE_STOP,                   API_SLOW),
 	API_ENTRY(CPU, CX18_CPU_CAPTURE_PAUSE,                  0),
diff --git a/drivers/media/pci/cx18/cx18-streams.c b/drivers/media/pci/cx18/cx18-streams.c
index f35f78d66985..a594cfdeca20 100644
--- a/drivers/media/pci/cx18/cx18-streams.c
+++ b/drivers/media/pci/cx18/cx18-streams.c
@@ -29,7 +29,7 @@
 #include "cx18-scb.h"
 #include "cx18-dvb.h"
 
-#define CX18_DSP0_INTERRUPT_MASK     	0xd0004C
+#define CX18_DSP0_INTERRUPT_MASK	0xd0004C
 
 static const struct v4l2_file_operations cx18_v4l2_enc_fops = {
 	.owner = THIS_MODULE,
@@ -116,7 +116,7 @@ static int cx18_prepare_buffer(struct videobuf_queue *q,
 	unsigned int width, unsigned int height,
 	enum v4l2_field field)
 {
-        struct cx18 *cx = s->cx;
+	struct cx18 *cx = s->cx;
 	int rc = 0;
 
 	/* check settings */
diff --git a/drivers/media/pci/cx18/cx18-vbi.c b/drivers/media/pci/cx18/cx18-vbi.c
index 72c74d60c6fb..81f1e27436fd 100644
--- a/drivers/media/pci/cx18/cx18-vbi.c
+++ b/drivers/media/pci/cx18/cx18-vbi.c
@@ -47,7 +47,7 @@ static void copy_vbi_data(struct cx18 *cx, int lines, u32 pts_stamp)
 		0x00, 0x00, 0x01, 0xbd,		    /* Priv Stream 1 start */
 		0x00, 0x1a,			    /* length */
 		0x84, 0x80, 0x07,		    /* flags, hdr data len */
-		0x21, 0x00, 0x5d, 0x63, 0xa7, 	    /* PTS, markers */
+		0x21, 0x00, 0x5d, 0x63, 0xa7,	    /* PTS, markers */
 		0xff, 0xff			    /* stuffing */
 	};
 	const int sd = sizeof(mpeg_hdr_data);	/* start of vbi data */
diff --git a/drivers/media/pci/cx18/cx23418.h b/drivers/media/pci/cx18/cx23418.h
index 901ed7fac10f..15205b662952 100644
--- a/drivers/media/pci/cx18/cx23418.h
+++ b/drivers/media/pci/cx18/cx23418.h
@@ -19,10 +19,10 @@
 
 #include <media/drv-intf/cx2341x.h>
 
-#define MGR_CMD_MASK            		0x40000000
+#define MGR_CMD_MASK				0x40000000
 /* The MSB of the command code indicates that this is the completion of a
    command */
-#define MGR_CMD_MASK_ACK        		(MGR_CMD_MASK | 0x80000000)
+#define MGR_CMD_MASK_ACK			(MGR_CMD_MASK | 0x80000000)
 
 /* Description: This command creates a new instance of a certain task
    IN[0]  - Task ID. This is one of the XPU_CMD_MASK_YYY where XPU is
@@ -30,26 +30,26 @@
    OUT[0] - Task handle. This handle is passed along with commands to
 	    dispatch to the right instance of the task
    ReturnCode - One of the ERR_SYS_... */
-#define CX18_CREATE_TASK      			(MGR_CMD_MASK | 0x0001)
+#define CX18_CREATE_TASK			(MGR_CMD_MASK | 0x0001)
 
 /* Description: This command destroys an instance of a task
    IN[0] - Task handle. Hanlde of the task to destroy
    ReturnCode - One of the ERR_SYS_... */
-#define CX18_DESTROY_TASK     			(MGR_CMD_MASK | 0x0002)
+#define CX18_DESTROY_TASK			(MGR_CMD_MASK | 0x0002)
 
 /* All commands for CPU have the following mask set */
-#define CPU_CMD_MASK                        	0x20000000
-#define CPU_CMD_MASK_DEBUG       		(CPU_CMD_MASK | 0x00000000)
-#define CPU_CMD_MASK_ACK                    	(CPU_CMD_MASK | 0x80000000)
-#define CPU_CMD_MASK_CAPTURE                	(CPU_CMD_MASK | 0x00020000)
-#define CPU_CMD_MASK_TS                     	(CPU_CMD_MASK | 0x00040000)
+#define CPU_CMD_MASK				0x20000000
+#define CPU_CMD_MASK_DEBUG			(CPU_CMD_MASK | 0x00000000)
+#define CPU_CMD_MASK_ACK			(CPU_CMD_MASK | 0x80000000)
+#define CPU_CMD_MASK_CAPTURE			(CPU_CMD_MASK | 0x00020000)
+#define CPU_CMD_MASK_TS				(CPU_CMD_MASK | 0x00040000)
 
-#define EPU_CMD_MASK                        	0x02000000
-#define EPU_CMD_MASK_DEBUG       		(EPU_CMD_MASK | 0x000000)
-#define EPU_CMD_MASK_DE                     	(EPU_CMD_MASK | 0x040000)
+#define EPU_CMD_MASK				0x02000000
+#define EPU_CMD_MASK_DEBUG			(EPU_CMD_MASK | 0x000000)
+#define EPU_CMD_MASK_DE				(EPU_CMD_MASK | 0x040000)
 
-#define APU_CMD_MASK 				0x10000000
-#define APU_CMD_MASK_ACK 			(APU_CMD_MASK | 0x80000000)
+#define APU_CMD_MASK				0x10000000
+#define APU_CMD_MASK_ACK			(APU_CMD_MASK | 0x80000000)
 
 #define CX18_APU_ENCODING_METHOD_MPEG		(0 << 28)
 #define CX18_APU_ENCODING_METHOD_AC3		(1 << 28)
@@ -67,7 +67,7 @@
 
 /* Description: Command APU to reset the AI
    ReturnCode - ??? */
-#define CX18_APU_RESETAI 			(APU_CMD_MASK | 0x05)
+#define CX18_APU_RESETAI			(APU_CMD_MASK | 0x05)
 
 /* Description: This command indicates that a Memory Descriptor List has been
    filled with the requested channel type
@@ -75,13 +75,13 @@
    IN[1] - Offset of the MDL_ACK from the beginning of the local DDR.
    IN[2] - Number of CNXT_MDL_ACK structures in the array pointed to by IN[1]
    ReturnCode - One of the ERR_DE_... */
-#define CX18_EPU_DMA_DONE              		(EPU_CMD_MASK_DE | 0x0001)
+#define CX18_EPU_DMA_DONE			(EPU_CMD_MASK_DE | 0x0001)
 
 /* Something interesting happened
    IN[0] - A value to log
    IN[1] - An offset of a string in the MiniMe memory;
 	   0/zero/NULL means "I have nothing to say" */
-#define CX18_EPU_DEBUG 				(EPU_CMD_MASK_DEBUG | 0x0003)
+#define CX18_EPU_DEBUG				(EPU_CMD_MASK_DEBUG | 0x0003)
 
 /* Reads memory/registers (32-bit)
    IN[0] - Address
@@ -91,40 +91,40 @@
 /* Description: This command starts streaming with the set channel type
    IN[0] - Task handle. Handle of the task to start
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_CAPTURE_START               	(CPU_CMD_MASK_CAPTURE | 0x0002)
+#define CX18_CPU_CAPTURE_START			(CPU_CMD_MASK_CAPTURE | 0x0002)
 
 /* Description: This command stops streaming with the set channel type
    IN[0] - Task handle. Handle of the task to stop
    IN[1] - 0 = stop at end of GOP, 1 = stop at end of frame (MPEG only)
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_CAPTURE_STOP                	(CPU_CMD_MASK_CAPTURE | 0x0003)
+#define CX18_CPU_CAPTURE_STOP			(CPU_CMD_MASK_CAPTURE | 0x0003)
 
 /* Description: This command pauses streaming with the set channel type
    IN[0] - Task handle. Handle of the task to pause
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_CAPTURE_PAUSE               	(CPU_CMD_MASK_CAPTURE | 0x0007)
+#define CX18_CPU_CAPTURE_PAUSE			(CPU_CMD_MASK_CAPTURE | 0x0007)
 
 /* Description: This command resumes streaming with the set channel type
    IN[0] - Task handle. Handle of the task to resume
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_CAPTURE_RESUME              	(CPU_CMD_MASK_CAPTURE | 0x0008)
-
-#define CAPTURE_CHANNEL_TYPE_NONE  		0
-#define CAPTURE_CHANNEL_TYPE_MPEG  		1
-#define CAPTURE_CHANNEL_TYPE_INDEX 		2
-#define CAPTURE_CHANNEL_TYPE_YUV   		3
-#define CAPTURE_CHANNEL_TYPE_PCM   		4
-#define CAPTURE_CHANNEL_TYPE_VBI   		5
+#define CX18_CPU_CAPTURE_RESUME			(CPU_CMD_MASK_CAPTURE | 0x0008)
+
+#define CAPTURE_CHANNEL_TYPE_NONE		0
+#define CAPTURE_CHANNEL_TYPE_MPEG		1
+#define CAPTURE_CHANNEL_TYPE_INDEX		2
+#define CAPTURE_CHANNEL_TYPE_YUV		3
+#define CAPTURE_CHANNEL_TYPE_PCM		4
+#define CAPTURE_CHANNEL_TYPE_VBI		5
 #define CAPTURE_CHANNEL_TYPE_SLICED_VBI		6
 #define CAPTURE_CHANNEL_TYPE_TS			7
-#define CAPTURE_CHANNEL_TYPE_MAX   		15
+#define CAPTURE_CHANNEL_TYPE_MAX		15
 
 /* Description: This command sets the channel type. This can only be done
    when stopped.
    IN[0] - Task handle. Handle of the task to start
    IN[1] - Channel Type. See Below.
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_SET_CHANNEL_TYPE      		(CPU_CMD_MASK_CAPTURE + 1)
+#define CX18_CPU_SET_CHANNEL_TYPE		(CPU_CMD_MASK_CAPTURE + 1)
 
 /* Description: Set stream output type
    IN[0] - task handle. Handle of the task to start
@@ -140,7 +140,7 @@
    IN[4] - reserved
    IN[5] - frame rate, 0 - 29.97f/s, 1 - 25f/s
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_SET_VIDEO_IN                	(CPU_CMD_MASK_CAPTURE | 0x0004)
+#define CX18_CPU_SET_VIDEO_IN			(CPU_CMD_MASK_CAPTURE | 0x0004)
 
 /* Description: Set video frame rate
    IN[0] - task handle. Handle of the task to start
@@ -149,7 +149,7 @@
    IN[3] - video peak rate
    IN[4] - system mux rate
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_SET_VIDEO_RATE              	(CPU_CMD_MASK_CAPTURE | 0x0005)
+#define CX18_CPU_SET_VIDEO_RATE			(CPU_CMD_MASK_CAPTURE | 0x0005)
 
 /* Description: Set video output resolution
    IN[0] - task handle
@@ -166,7 +166,7 @@
 				3 = horizontal/vertical, 4 = diagonal
    IN[3] - strength, temporal 0 - 31, spatial 0 - 15
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_SET_FILTER_PARAM            	(CPU_CMD_MASK_CAPTURE | 0x0009)
+#define CX18_CPU_SET_FILTER_PARAM		(CPU_CMD_MASK_CAPTURE | 0x0009)
 
 /* Description: This command set spatial filter type
    IN[0] - Task handle.
@@ -174,7 +174,7 @@
 		      3 = 2D H/V separable, 4 = 2D symmetric non-separable
    IN[2] - chroma type: 0 - disable, 1 = 1D horizontal
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_SET_SPATIAL_FILTER_TYPE     	(CPU_CMD_MASK_CAPTURE | 0x000C)
+#define CX18_CPU_SET_SPATIAL_FILTER_TYPE	(CPU_CMD_MASK_CAPTURE | 0x000C)
 
 /* Description: This command set coring levels for median filter
    IN[0] - Task handle.
@@ -183,16 +183,16 @@
    IN[3] - chroma_high
    IN[4] - chroma_low
    ReturnCode - One of the ERR_CAPTURE_... */
-#define CX18_CPU_SET_MEDIAN_CORING           	(CPU_CMD_MASK_CAPTURE | 0x000E)
+#define CX18_CPU_SET_MEDIAN_CORING		(CPU_CMD_MASK_CAPTURE | 0x000E)
 
 /* Description: This command set the picture type mask for index file
    IN[0] - Task handle (ignored by firmware)
-   IN[1] - 	0 = disable index file output
+   IN[1] -	0 = disable index file output
 			1 = output I picture
 			2 = P picture
 			4 = B picture
 			other = illegal */
-#define CX18_CPU_SET_INDEXTABLE         	(CPU_CMD_MASK_CAPTURE | 0x0010)
+#define CX18_CPU_SET_INDEXTABLE			(CPU_CMD_MASK_CAPTURE | 0x0010)
 
 /* Description: Set audio parameters
    IN[0] - task handle. Handle of the task to start
@@ -218,7 +218,7 @@
 /* Description: Set stream output type
    IN[0] - task handle. Handle of the task to start
    IN[1] - subType
-	    SET_INITIAL_SCR      		1
+	    SET_INITIAL_SCR			1
 	    SET_QUALITY_MODE            2
 	    SET_VIM_PROTECT_MODE        3
 	    SET_PTS_CORRECTION          4
@@ -311,7 +311,7 @@
 			bit 0:	output user data, 1 - enable
 			bit 1:	output private stream, 1 - enable
 			bit 2:	mux option, 0 - in GOP, 1 - in picture
-			bit[7:0] 	private stream ID
+			bit[7:0]	private stream ID
    IN[5] - insertion period while mux option is in picture
    ReturnCode - VBI data offset */
 #define CX18_CPU_SET_SLICED_VBI_PARAM		(CPU_CMD_MASK_CAPTURE | 0x0020)
@@ -344,13 +344,13 @@
 #define CX18_CPU_SET_VFC_PARAM                  (CPU_CMD_MASK_CAPTURE | 0x0023)
 
 /* Below is the list of commands related to the data exchange */
-#define CPU_CMD_MASK_DE 			(CPU_CMD_MASK | 0x040000)
+#define CPU_CMD_MASK_DE				(CPU_CMD_MASK | 0x040000)
 
 /* Description: This command provides the physical base address of the local
    DDR as viewed by EPU
    IN[0] - Physical offset where EPU has the local DDR mapped
    ReturnCode - One of the ERR_DE_... */
-#define CPU_CMD_DE_SetBase 			(CPU_CMD_MASK_DE | 0x0001)
+#define CPU_CMD_DE_SetBase			(CPU_CMD_MASK_DE | 0x0001)
 
 /* Description: This command provides the offsets in the device memory where
    the 2 cx18_mdl_ack blocks reside
@@ -360,7 +360,7 @@
    IN[2] - Offset of the second cx18_mdl_ack from the beginning of the
 	   local DDR.
    ReturnCode - One of the ERR_DE_... */
-#define CX18_CPU_DE_SET_MDL_ACK                	(CPU_CMD_MASK_DE | 0x0002)
+#define CX18_CPU_DE_SET_MDL_ACK			(CPU_CMD_MASK_DE | 0x0002)
 
 /* Description: This command provides the offset to a Memory Descriptor List
    IN[0] - Task handle. Handle of the task to start
@@ -369,13 +369,13 @@
    IN[3] - Buffer ID
    IN[4] - Total buffer length
    ReturnCode - One of the ERR_DE_... */
-#define CX18_CPU_DE_SET_MDL                   	(CPU_CMD_MASK_DE | 0x0005)
+#define CX18_CPU_DE_SET_MDL			(CPU_CMD_MASK_DE | 0x0005)
 
 /* Description: This command requests return of all current Memory
    Descriptor Lists to the driver
    IN[0] - Task handle. Handle of the task to start
    ReturnCode - One of the ERR_DE_... */
-#define CX18_CPU_DE_RELEASE_MDL               	(CPU_CMD_MASK_DE | 0x0006)
+#define CX18_CPU_DE_RELEASE_MDL			(CPU_CMD_MASK_DE | 0x0006)
 
 /* Description: This command signals the cpu that the dat buffer has been
    consumed and ready for re-use.
diff --git a/drivers/media/pci/cx23885/Makefile b/drivers/media/pci/cx23885/Makefile
index b8bf7806124b..130f0aa29ac6 100644
--- a/drivers/media/pci/cx23885/Makefile
+++ b/drivers/media/pci/cx23885/Makefile
@@ -8,9 +8,7 @@ cx23885-objs	:= cx23885-cards.o cx23885-video.o cx23885-vbi.o \
 obj-$(CONFIG_VIDEO_CX23885) += cx23885.o
 obj-$(CONFIG_MEDIA_ALTERA_CI) += altera-ci.o
 
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
 
 ccflags-y += $(extra-cflags-y) $(extra-cflags-m)
diff --git a/drivers/media/pci/cx23885/altera-ci.c b/drivers/media/pci/cx23885/altera-ci.c
index 5c94e312cba3..70aec9bb7e95 100644
--- a/drivers/media/pci/cx23885/altera-ci.c
+++ b/drivers/media/pci/cx23885/altera-ci.c
@@ -51,10 +51,10 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <dvb_demux.h>
-#include <dvb_frontend.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
 #include "altera-ci.h"
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 /* FPGA regs */
 #define NETUP_CI_INT_CTRL	0x00
diff --git a/drivers/media/pci/cx23885/cimax2.c b/drivers/media/pci/cx23885/cimax2.c
index 5e8e134d81c2..19c005f4a57d 100644
--- a/drivers/media/pci/cx23885/cimax2.c
+++ b/drivers/media/pci/cx23885/cimax2.c
@@ -21,7 +21,7 @@
 
 #include "cx23885.h"
 #include "cimax2.h"
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 /* Max transfer size done by I2C transfer functions */
 #define MAX_XFER_SIZE  64
@@ -54,7 +54,7 @@
 #define NETUP_CI_CTL		0x04
 #define NETUP_CI_RD		1
 
-#define NETUP_IRQ_DETAM 	0x1
+#define NETUP_IRQ_DETAM		0x1
 #define NETUP_IRQ_IRQAM		0x4
 
 static unsigned int ci_dbg;
diff --git a/drivers/media/pci/cx23885/cimax2.h b/drivers/media/pci/cx23885/cimax2.h
index 565e958f6f8d..167ffe205b5d 100644
--- a/drivers/media/pci/cx23885/cimax2.h
+++ b/drivers/media/pci/cx23885/cimax2.h
@@ -21,7 +21,7 @@
 
 #ifndef CIMAX2_H
 #define CIMAX2_H
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 extern int netup_ci_read_attribute_mem(struct dvb_ca_en50221 *en50221,
 						int slot, int addr);
diff --git a/drivers/media/pci/cx23885/cx23885-cards.c b/drivers/media/pci/cx23885/cx23885-cards.c
index 28eab9c518c5..3622521431f5 100644
--- a/drivers/media/pci/cx23885/cx23885-cards.c
+++ b/drivers/media/pci/cx23885/cx23885-cards.c
@@ -1816,13 +1816,13 @@ int cx23885_ir_init(struct cx23885_dev *dev)
 {
 	static struct v4l2_subdev_io_pin_config ir_rxtx_pin_cfg[] = {
 		{
-			.flags	  = V4L2_SUBDEV_IO_PIN_INPUT,
+			.flags	  = BIT(V4L2_SUBDEV_IO_PIN_INPUT),
 			.pin	  = CX23885_PIN_IR_RX_GPIO19,
 			.function = CX23885_PAD_IR_RX,
 			.value	  = 0,
 			.strength = CX25840_PIN_DRIVE_MEDIUM,
 		}, {
-			.flags	  = V4L2_SUBDEV_IO_PIN_OUTPUT,
+			.flags	  = BIT(V4L2_SUBDEV_IO_PIN_OUTPUT),
 			.pin	  = CX23885_PIN_IR_TX_GPIO20,
 			.function = CX23885_PAD_IR_TX,
 			.value	  = 0,
@@ -1833,7 +1833,7 @@ int cx23885_ir_init(struct cx23885_dev *dev)
 
 	static struct v4l2_subdev_io_pin_config ir_rx_pin_cfg[] = {
 		{
-			.flags	  = V4L2_SUBDEV_IO_PIN_INPUT,
+			.flags	  = BIT(V4L2_SUBDEV_IO_PIN_INPUT),
 			.pin	  = CX23885_PIN_IR_RX_GPIO19,
 			.function = CX23885_PAD_IR_RX,
 			.value	  = 0,
diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c
index e795ddeb7fe2..700422b538c0 100644
--- a/drivers/media/pci/cx23885/cx23885-dvb.c
+++ b/drivers/media/pci/cx23885/cx23885-dvb.c
@@ -27,7 +27,7 @@
 
 #include <media/v4l2-common.h>
 
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 #include "s5h1409.h"
 #include "s5h1411.h"
 #include "mt2131.h"
@@ -1852,8 +1852,8 @@ static int dvb_register(struct cx23885_tsport *port)
 			/* attach frontend */
 			memset(&si2165_pdata, 0, sizeof(si2165_pdata));
 			si2165_pdata.fe = &fe0->dvb.frontend;
-			si2165_pdata.chip_mode = SI2165_MODE_PLL_XTAL,
-			si2165_pdata.ref_freq_Hz = 16000000,
+			si2165_pdata.chip_mode = SI2165_MODE_PLL_XTAL;
+			si2165_pdata.ref_freq_hz = 16000000;
 			memset(&info, 0, sizeof(struct i2c_board_info));
 			strlcpy(info.type, "si2165", I2C_NAME_SIZE);
 			info.addr = 0x64;
diff --git a/drivers/media/pci/cx23885/cx23885-input.c b/drivers/media/pci/cx23885/cx23885-input.c
index 944b70831f12..0f4e54294bb7 100644
--- a/drivers/media/pci/cx23885/cx23885-input.c
+++ b/drivers/media/pci/cx23885/cx23885-input.c
@@ -340,14 +340,23 @@ int cx23885_input_init(struct cx23885_dev *dev)
 	kernel_ir->cx = dev;
 	kernel_ir->name = kasprintf(GFP_KERNEL, "cx23885 IR (%s)",
 				    cx23885_boards[dev->board].name);
+	if (!kernel_ir->name) {
+		ret = -ENOMEM;
+		goto err_out_free;
+	}
+
 	kernel_ir->phys = kasprintf(GFP_KERNEL, "pci-%s/ir0",
 				    pci_name(dev->pci));
+	if (!kernel_ir->phys) {
+		ret = -ENOMEM;
+		goto err_out_free_name;
+	}
 
 	/* input device */
 	rc = rc_allocate_device(RC_DRIVER_IR_RAW);
 	if (!rc) {
 		ret = -ENOMEM;
-		goto err_out_free;
+		goto err_out_free_phys;
 	}
 
 	kernel_ir->rc = rc;
@@ -382,9 +391,11 @@ int cx23885_input_init(struct cx23885_dev *dev)
 	cx23885_input_ir_stop(dev);
 	dev->kernel_ir = NULL;
 	rc_free_device(rc);
-err_out_free:
+err_out_free_phys:
 	kfree(kernel_ir->phys);
+err_out_free_name:
 	kfree(kernel_ir->name);
+err_out_free:
 	kfree(kernel_ir);
 	return ret;
 }
diff --git a/drivers/media/pci/cx23885/cx23885-video.c b/drivers/media/pci/cx23885/cx23885-video.c
index ecc580af0148..a03dcb662953 100644
--- a/drivers/media/pci/cx23885/cx23885-video.c
+++ b/drivers/media/pci/cx23885/cx23885-video.c
@@ -1146,7 +1146,7 @@ static struct video_device cx23885_vbi_template;
 static struct video_device cx23885_video_template = {
 	.name                 = "cx23885-video",
 	.fops                 = &video_fops,
-	.ioctl_ops 	      = &video_ioctl_ops,
+	.ioctl_ops	      = &video_ioctl_ops,
 	.tvnorms              = CX23885_NORMS,
 };
 
diff --git a/drivers/media/pci/cx23885/cx23885.h b/drivers/media/pci/cx23885/cx23885.h
index 6aab713e0476..2a17209eb4f6 100644
--- a/drivers/media/pci/cx23885/cx23885.h
+++ b/drivers/media/pci/cx23885/cx23885.h
@@ -357,7 +357,7 @@ struct cx23885_audio_dev {
 
 struct cx23885_dev {
 	atomic_t                   refcount;
-	struct v4l2_device 	   v4l2_dev;
+	struct v4l2_device	   v4l2_dev;
 	struct v4l2_ctrl_handler   ctrl_handler;
 
 	/* pci stuff */
@@ -407,7 +407,7 @@ struct cx23885_dev {
 	unsigned int               tuner_bus;
 	unsigned int               radio_type;
 	unsigned char              radio_addr;
-	struct v4l2_subdev 	   *sd_cx25840;
+	struct v4l2_subdev	   *sd_cx25840;
 	struct work_struct	   cx25840_work;
 
 	/* Infrared */
diff --git a/drivers/media/pci/cx23885/cx23888-ir.c b/drivers/media/pci/cx23885/cx23888-ir.c
index 040323b0f945..00329f668b59 100644
--- a/drivers/media/pci/cx23885/cx23888-ir.c
+++ b/drivers/media/pci/cx23885/cx23888-ir.c
@@ -29,7 +29,7 @@ static unsigned int ir_888_debug;
 module_param(ir_888_debug, int, 0644);
 MODULE_PARM_DESC(ir_888_debug, "enable debug messages [CX23888 IR controller]");
 
-#define CX23888_IR_REG_BASE 	0x170000
+#define CX23888_IR_REG_BASE	0x170000
 /*
  * These CX23888 register offsets have a straightforward one to one mapping
  * to the CX23885 register offsets of 0x200 through 0x218
@@ -170,7 +170,7 @@ static inline int cx23888_ir_and_or4(struct cx23885_dev *dev, u32 addr,
  * Rx and Tx Clock Divider register computations
  *
  * Note the largest clock divider value of 0xffff corresponds to:
- * 	(0xffff + 1) * 1000 / 108/2 MHz = 1,213,629.629... ns
+ *	(0xffff + 1) * 1000 / 108/2 MHz = 1,213,629.629... ns
  * which fits in 21 bits, so we'll use unsigned int for time arguments.
  */
 static inline u16 count_to_clock_divider(unsigned int d)
@@ -226,7 +226,7 @@ static inline unsigned int clock_divider_to_freq(unsigned int divider,
  * Low Pass Filter register calculations
  *
  * Note the largest count value of 0xffff corresponds to:
- * 	0xffff * 1000 / 108/2 MHz = 1,213,611.11... ns
+ *	0xffff * 1000 / 108/2 MHz = 1,213,611.11... ns
  * which fits in 21 bits, so we'll use unsigned int for time arguments.
  */
 static inline u16 count_to_lpf_count(unsigned int d)
diff --git a/drivers/media/pci/cx25821/Makefile b/drivers/media/pci/cx25821/Makefile
index d14d65b1b042..94633d02ac7f 100644
--- a/drivers/media/pci/cx25821/Makefile
+++ b/drivers/media/pci/cx25821/Makefile
@@ -5,5 +5,3 @@ cx25821-y   := cx25821-core.o cx25821-cards.o cx25821-i2c.o \
 
 obj-$(CONFIG_VIDEO_CX25821) += cx25821.o
 obj-$(CONFIG_VIDEO_CX25821_ALSA) += cx25821-alsa.o
-
-ccflags-y += -Idrivers/media/i2c
diff --git a/drivers/media/pci/cx25821/cx25821-alsa.c b/drivers/media/pci/cx25821/cx25821-alsa.c
index 2b34990e86f2..a45bf0331eeb 100644
--- a/drivers/media/pci/cx25821/cx25821-alsa.c
+++ b/drivers/media/pci/cx25821/cx25821-alsa.c
@@ -152,8 +152,8 @@ static int cx25821_alsa_dma_init(struct cx25821_audio_dev *chip, int nr_pages)
 		return -ENOMEM;
 	}
 
-	dprintk(1, "vmalloc is at addr 0x%08lx, size=%d\n",
-				(unsigned long)buf->vaddr,
+	dprintk(1, "vmalloc is at addr 0x%p, size=%d\n",
+				buf->vaddr,
 				nr_pages << PAGE_SHIFT);
 
 	memset(buf->vaddr, 0, nr_pages << PAGE_SHIFT);
diff --git a/drivers/media/pci/cx88/Makefile b/drivers/media/pci/cx88/Makefile
index 86646eee4e6b..d0f45d652d6e 100644
--- a/drivers/media/pci/cx88/Makefile
+++ b/drivers/media/pci/cx88/Makefile
@@ -10,8 +10,5 @@ obj-$(CONFIG_VIDEO_CX88_ALSA) += cx88-alsa.o
 obj-$(CONFIG_VIDEO_CX88_BLACKBIRD) += cx88-blackbird.o
 obj-$(CONFIG_VIDEO_CX88_DVB) += cx88-dvb.o
 obj-$(CONFIG_VIDEO_CX88_VP3054) += cx88-vp3054-i2c.o
-
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/pci/cx88/cx88-blackbird.c b/drivers/media/pci/cx88/cx88-blackbird.c
index e3101f04941c..0e0952e60795 100644
--- a/drivers/media/pci/cx88/cx88-blackbird.c
+++ b/drivers/media/pci/cx88/cx88-blackbird.c
@@ -805,8 +805,7 @@ static int vidioc_querycap(struct file *file, void  *priv,
 
 	strcpy(cap->driver, "cx88_blackbird");
 	sprintf(cap->bus_info, "PCI:%s", pci_name(dev->pci));
-	cx88_querycap(file, core, cap);
-	return 0;
+	return cx88_querycap(file, core, cap);
 }
 
 static int vidioc_enum_fmt_vid_cap(struct file *file, void  *priv,
diff --git a/drivers/media/pci/cx88/cx88-video.c b/drivers/media/pci/cx88/cx88-video.c
index 7d25ecd4404b..9be682cdb644 100644
--- a/drivers/media/pci/cx88/cx88-video.c
+++ b/drivers/media/pci/cx88/cx88-video.c
@@ -806,8 +806,8 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-void cx88_querycap(struct file *file, struct cx88_core *core,
-		   struct v4l2_capability *cap)
+int cx88_querycap(struct file *file, struct cx88_core *core,
+		  struct v4l2_capability *cap)
 {
 	struct video_device *vdev = video_devdata(file);
 
@@ -825,11 +825,14 @@ void cx88_querycap(struct file *file, struct cx88_core *core,
 	case VFL_TYPE_VBI:
 		cap->device_caps |= V4L2_CAP_VBI_CAPTURE;
 		break;
+	default:
+		return -EINVAL;
 	}
 	cap->capabilities = cap->device_caps | V4L2_CAP_VIDEO_CAPTURE |
 		V4L2_CAP_VBI_CAPTURE | V4L2_CAP_DEVICE_CAPS;
 	if (core->board.radio.type == CX88_RADIO)
 		cap->capabilities |= V4L2_CAP_RADIO;
+	return 0;
 }
 EXPORT_SYMBOL(cx88_querycap);
 
@@ -841,8 +844,7 @@ static int vidioc_querycap(struct file *file, void  *priv,
 
 	strcpy(cap->driver, "cx8800");
 	sprintf(cap->bus_info, "PCI:%s", pci_name(dev->pci));
-	cx88_querycap(file, core, cap);
-	return 0;
+	return cx88_querycap(file, core, cap);
 }
 
 static int vidioc_enum_fmt_vid_cap(struct file *file, void  *priv,
diff --git a/drivers/media/pci/cx88/cx88.h b/drivers/media/pci/cx88/cx88.h
index 6777926f20f2..07a33f02fef4 100644
--- a/drivers/media/pci/cx88/cx88.h
+++ b/drivers/media/pci/cx88/cx88.h
@@ -734,7 +734,7 @@ int cx8802_start_dma(struct cx8802_dev    *dev,
 int cx88_enum_input(struct cx88_core *core, struct v4l2_input *i);
 int cx88_set_freq(struct cx88_core  *core, const struct v4l2_frequency *f);
 int cx88_video_mux(struct cx88_core *core, unsigned int input);
-void cx88_querycap(struct file *file, struct cx88_core *core,
-		   struct v4l2_capability *cap);
+int cx88_querycap(struct file *file, struct cx88_core *core,
+		  struct v4l2_capability *cap);
 
 #endif
diff --git a/drivers/media/pci/ddbridge/Makefile b/drivers/media/pci/ddbridge/Makefile
index 7453b65104ff..f58fdec50eab 100644
--- a/drivers/media/pci/ddbridge/Makefile
+++ b/drivers/media/pci/ddbridge/Makefile
@@ -3,12 +3,11 @@
 # Makefile for the ddbridge device driver
 #
 
-ddbridge-objs := ddbridge-main.o ddbridge-core.o ddbridge-hw.o \
-		ddbridge-i2c.o ddbridge-maxs8.o
+ddbridge-objs := ddbridge-main.o ddbridge-core.o ddbridge-ci.o \
+		ddbridge-hw.o ddbridge-i2c.o ddbridge-max.o
 
 obj-$(CONFIG_DVB_DDBRIDGE) += ddbridge.o
 
-ccflags-y += -Idrivers/media/dvb-core/
 ccflags-y += -Idrivers/media/dvb-frontends/
 ccflags-y += -Idrivers/media/tuners/
 
diff --git a/drivers/media/pci/ddbridge/ddbridge-ci.c b/drivers/media/pci/ddbridge/ddbridge-ci.c
new file mode 100644
index 000000000000..5828111487b0
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-ci.c
@@ -0,0 +1,359 @@
+/*
+ * ddbridge-ci.c: Digital Devices bridge CI (DuoFlex, CI Bridge) support
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * To obtain the license, point your browser to
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+#include "ddbridge.h"
+#include "ddbridge-regs.h"
+#include "ddbridge-ci.h"
+#include "ddbridge-io.h"
+#include "ddbridge-i2c.h"
+
+#include "cxd2099.h"
+
+/* Octopus CI internal CI interface */
+
+static int wait_ci_ready(struct ddb_ci *ci)
+{
+	u32 count = 10;
+
+	ndelay(500);
+	do {
+		if (ddbreadl(ci->port->dev,
+			     CI_CONTROL(ci->nr)) & CI_READY)
+			break;
+		usleep_range(1, 2);
+		if ((--count) == 0)
+			return -1;
+	} while (1);
+	return 0;
+}
+
+static int read_attribute_mem(struct dvb_ca_en50221 *ca,
+			      int slot, int address)
+{
+	struct ddb_ci *ci = ca->data;
+	u32 val, off = (address >> 1) & (CI_BUFFER_SIZE - 1);
+
+	if (address > CI_BUFFER_SIZE)
+		return -1;
+	ddbwritel(ci->port->dev, CI_READ_CMD | (1 << 16) | address,
+		  CI_DO_READ_ATTRIBUTES(ci->nr));
+	wait_ci_ready(ci);
+	val = 0xff & ddbreadl(ci->port->dev, CI_BUFFER(ci->nr) + off);
+	return val;
+}
+
+static int write_attribute_mem(struct dvb_ca_en50221 *ca, int slot,
+			       int address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, CI_WRITE_CMD | (value << 16) | address,
+		  CI_DO_ATTRIBUTE_RW(ci->nr));
+	wait_ci_ready(ci);
+	return 0;
+}
+
+static int read_cam_control(struct dvb_ca_en50221 *ca,
+			    int slot, u8 address)
+{
+	u32 count = 100;
+	struct ddb_ci *ci = ca->data;
+	u32 res;
+
+	ddbwritel(ci->port->dev, CI_READ_CMD | address,
+		  CI_DO_IO_RW(ci->nr));
+	ndelay(500);
+	do {
+		res = ddbreadl(ci->port->dev, CI_READDATA(ci->nr));
+		if (res & CI_READY)
+			break;
+		usleep_range(1, 2);
+		if ((--count) == 0)
+			return -1;
+	} while (1);
+	return 0xff & res;
+}
+
+static int write_cam_control(struct dvb_ca_en50221 *ca, int slot,
+			     u8 address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, CI_WRITE_CMD | (value << 16) | address,
+		  CI_DO_IO_RW(ci->nr));
+	wait_ci_ready(ci);
+	return 0;
+}
+
+static int slot_reset(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, CI_POWER_ON,
+		  CI_CONTROL(ci->nr));
+	msleep(100);
+	ddbwritel(ci->port->dev, CI_POWER_ON | CI_RESET_CAM,
+		  CI_CONTROL(ci->nr));
+	ddbwritel(ci->port->dev, CI_ENABLE | CI_POWER_ON | CI_RESET_CAM,
+		  CI_CONTROL(ci->nr));
+	usleep_range(20, 25);
+	ddbwritel(ci->port->dev, CI_ENABLE | CI_POWER_ON,
+		  CI_CONTROL(ci->nr));
+	return 0;
+}
+
+static int slot_shutdown(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	ddbwritel(ci->port->dev, 0, CI_CONTROL(ci->nr));
+	msleep(300);
+	return 0;
+}
+
+static int slot_ts_enable(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+	u32 val = ddbreadl(ci->port->dev, CI_CONTROL(ci->nr));
+
+	ddbwritel(ci->port->dev, val | CI_BYPASS_DISABLE,
+		  CI_CONTROL(ci->nr));
+	return 0;
+}
+
+static int poll_slot_status(struct dvb_ca_en50221 *ca, int slot, int open)
+{
+	struct ddb_ci *ci = ca->data;
+	u32 val = ddbreadl(ci->port->dev, CI_CONTROL(ci->nr));
+	int stat = 0;
+
+	if (val & CI_CAM_DETECT)
+		stat |= DVB_CA_EN50221_POLL_CAM_PRESENT;
+	if (val & CI_CAM_READY)
+		stat |= DVB_CA_EN50221_POLL_CAM_READY;
+	return stat;
+}
+
+static struct dvb_ca_en50221 en_templ = {
+	.read_attribute_mem  = read_attribute_mem,
+	.write_attribute_mem = write_attribute_mem,
+	.read_cam_control    = read_cam_control,
+	.write_cam_control   = write_cam_control,
+	.slot_reset          = slot_reset,
+	.slot_shutdown       = slot_shutdown,
+	.slot_ts_enable      = slot_ts_enable,
+	.poll_slot_status    = poll_slot_status,
+};
+
+static void ci_attach(struct ddb_port *port)
+{
+	struct ddb_ci *ci = NULL;
+
+	ci = kzalloc(sizeof(*ci), GFP_KERNEL);
+	if (!ci)
+		return;
+	memcpy(&ci->en, &en_templ, sizeof(en_templ));
+	ci->en.data = ci;
+	port->en = &ci->en;
+	ci->port = port;
+	ci->nr = port->nr - 2;
+}
+
+/* DuoFlex Dual CI support */
+
+static int write_creg(struct ddb_ci *ci, u8 data, u8 mask)
+{
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+
+	ci->port->creg = (ci->port->creg & ~mask) | data;
+	return i2c_write_reg(i2c, adr, 0x02, ci->port->creg);
+}
+
+static int read_attribute_mem_xo2(struct dvb_ca_en50221 *ca,
+				  int slot, int address)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+	int res;
+	u8 val;
+
+	res = i2c_read_reg16(i2c, adr, 0x8000 | address, &val);
+	return res ? res : val;
+}
+
+static int write_attribute_mem_xo2(struct dvb_ca_en50221 *ca, int slot,
+				   int address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+
+	return i2c_write_reg16(i2c, adr, 0x8000 | address, value);
+}
+
+static int read_cam_control_xo2(struct dvb_ca_en50221 *ca,
+				int slot, u8 address)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+	u8 val;
+	int res;
+
+	res = i2c_read_reg(i2c, adr, 0x20 | (address & 3), &val);
+	return res ? res : val;
+}
+
+static int write_cam_control_xo2(struct dvb_ca_en50221 *ca, int slot,
+				 u8 address, u8 value)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+
+	return i2c_write_reg(i2c, adr, 0x20 | (address & 3), value);
+}
+
+static int slot_reset_xo2(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	dev_dbg(ci->port->dev->dev, "%s\n", __func__);
+	write_creg(ci, 0x01, 0x01);
+	write_creg(ci, 0x04, 0x04);
+	msleep(20);
+	write_creg(ci, 0x02, 0x02);
+	write_creg(ci, 0x00, 0x04);
+	write_creg(ci, 0x18, 0x18);
+	return 0;
+}
+
+static int slot_shutdown_xo2(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	dev_dbg(ci->port->dev->dev, "%s\n", __func__);
+	write_creg(ci, 0x10, 0xff);
+	write_creg(ci, 0x08, 0x08);
+	return 0;
+}
+
+static int slot_ts_enable_xo2(struct dvb_ca_en50221 *ca, int slot)
+{
+	struct ddb_ci *ci = ca->data;
+
+	dev_dbg(ci->port->dev->dev, "%s\n", __func__);
+	write_creg(ci, 0x00, 0x10);
+	return 0;
+}
+
+static int poll_slot_status_xo2(struct dvb_ca_en50221 *ca, int slot, int open)
+{
+	struct ddb_ci *ci = ca->data;
+	struct i2c_adapter *i2c = &ci->port->i2c->adap;
+	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
+	u8 val = 0;
+	int stat = 0;
+
+	i2c_read_reg(i2c, adr, 0x01, &val);
+
+	if (val & 2)
+		stat |= DVB_CA_EN50221_POLL_CAM_PRESENT;
+	if (val & 1)
+		stat |= DVB_CA_EN50221_POLL_CAM_READY;
+	return stat;
+}
+
+static struct dvb_ca_en50221 en_xo2_templ = {
+	.read_attribute_mem  = read_attribute_mem_xo2,
+	.write_attribute_mem = write_attribute_mem_xo2,
+	.read_cam_control    = read_cam_control_xo2,
+	.write_cam_control   = write_cam_control_xo2,
+	.slot_reset          = slot_reset_xo2,
+	.slot_shutdown       = slot_shutdown_xo2,
+	.slot_ts_enable      = slot_ts_enable_xo2,
+	.poll_slot_status    = poll_slot_status_xo2,
+};
+
+static void ci_xo2_attach(struct ddb_port *port)
+{
+	struct ddb_ci *ci;
+
+	ci = kzalloc(sizeof(*ci), GFP_KERNEL);
+	if (!ci)
+		return;
+	memcpy(&ci->en, &en_xo2_templ, sizeof(en_xo2_templ));
+	ci->en.data = ci;
+	port->en = &ci->en;
+	ci->port = port;
+	ci->nr = port->nr - 2;
+	ci->port->creg = 0;
+	write_creg(ci, 0x10, 0xff);
+	write_creg(ci, 0x08, 0x08);
+}
+
+static struct cxd2099_cfg cxd_cfg = {
+	.bitrate =  72000,
+	.adr     =  0x40,
+	.polarity = 1,
+	.clock_mode = 1,
+	.max_i2c = 512,
+};
+
+int ddb_ci_attach(struct ddb_port *port, u32 bitrate)
+{
+	switch (port->type) {
+	case DDB_CI_EXTERNAL_SONY:
+		cxd_cfg.bitrate = bitrate;
+		port->en = cxd2099_attach(&cxd_cfg, port, &port->i2c->adap);
+		if (!port->en)
+			return -ENODEV;
+		break;
+
+	case DDB_CI_EXTERNAL_XO2:
+	case DDB_CI_EXTERNAL_XO2_B:
+		ci_xo2_attach(port);
+		if (!port->en)
+			return -ENODEV;
+		break;
+
+	case DDB_CI_INTERNAL:
+		ci_attach(port);
+		if (!port->en)
+			return -ENODEV;
+		break;
+	}
+
+	dvb_ca_en50221_init(port->dvb[0].adap, port->en, 0, 1);
+	return 0;
+}
+
+void ddb_ci_detach(struct ddb_port *port)
+{
+	if (port->dvb[0].dev)
+		dvb_unregister_device(port->dvb[0].dev);
+	if (port->en) {
+		dvb_ca_en50221_release(port->en);
+		kfree(port->en->data);
+		port->en = NULL;
+	}
+}
diff --git a/drivers/media/pci/ddbridge/ddbridge-ci.h b/drivers/media/pci/ddbridge/ddbridge-ci.h
new file mode 100644
index 000000000000..35a39182dd83
--- /dev/null
+++ b/drivers/media/pci/ddbridge/ddbridge-ci.h
@@ -0,0 +1,31 @@
+/*
+ * ddbridge-ci.h: Digital Devices bridge CI (DuoFlex, CI Bridge) support
+ *
+ * Copyright (C) 2010-2017 Digital Devices GmbH
+ *                         Marcus Metzler <mocm@metzlerbros.de>
+ *                         Ralph Metzler <rjkm@metzlerbros.de>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 only, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * To obtain the license, point your browser to
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+#ifndef __DDBRIDGE_CI_H__
+#define __DDBRIDGE_CI_H__
+
+#include "ddbridge.h"
+
+/******************************************************************************/
+
+int ddb_ci_attach(struct ddb_port *port, u32 bitrate);
+void ddb_ci_detach(struct ddb_port *port);
+
+#endif /* __DDBRIDGE_CI_H__ */
diff --git a/drivers/media/pci/ddbridge/ddbridge-core.c b/drivers/media/pci/ddbridge/ddbridge-core.c
index f4bd4908acdd..f9bee36f1cad 100644
--- a/drivers/media/pci/ddbridge/ddbridge-core.c
+++ b/drivers/media/pci/ddbridge/ddbridge-core.c
@@ -37,7 +37,8 @@
 #include "ddbridge.h"
 #include "ddbridge-i2c.h"
 #include "ddbridge-regs.h"
-#include "ddbridge-maxs8.h"
+#include "ddbridge-max.h"
+#include "ddbridge-ci.h"
 #include "ddbridge-io.h"
 
 #include "tda18271c2dd.h"
@@ -112,7 +113,6 @@ static void ddb_set_dma_tables(struct ddb *dev)
 	}
 }
 
-
 /****************************************************************************/
 /****************************************************************************/
 /****************************************************************************/
@@ -210,8 +210,9 @@ static int ddb_redirect(u32 i, u32 p)
 		if (input->redi) {
 			input2->redi = input->redi;
 			input->redi = NULL;
-		} else
+		} else {
 			input2->redi = input;
+		}
 	}
 	input->redo = port->output;
 	port->output->redi = input;
@@ -357,9 +358,9 @@ static void calc_con(struct ddb_output *output, u32 *con, u32 *con2, u32 flags)
 				max_bitrate = 0;
 				gap = 0;
 				if (bitrate != 72000) {
-					if (bitrate >= 96000)
+					if (bitrate >= 96000) {
 						*con |= 0x800;
-					else {
+					} else {
 						*con |= 0x1000;
 						nco = (bitrate * 8192 + 71999)
 							/ 72000;
@@ -506,7 +507,6 @@ static void ddb_input_start(struct ddb_input *input)
 	}
 }
 
-
 static void ddb_input_start_all(struct ddb_input *input)
 {
 	struct ddb_input *i = input;
@@ -572,7 +572,7 @@ static ssize_t ddb_output_write(struct ddb_output *output,
 	while (left) {
 		len = output->dma->size - output->dma->coff;
 		if ((((output->dma->cbuf + 1) % output->dma->num) == idx) &&
-		    (off == 0)) {
+		    off == 0) {
 			if (len <= 188)
 				break;
 			len -= 188;
@@ -593,7 +593,8 @@ static ssize_t ddb_output_write(struct ddb_output *output,
 				   buf, len))
 			return -EIO;
 		if (alt_dma)
-			dma_sync_single_for_device(dev->dev,
+			dma_sync_single_for_device(
+				dev->dev,
 				output->dma->pbuf[output->dma->cbuf],
 				output->dma->size, DMA_TO_DEVICE);
 		left -= len;
@@ -632,7 +633,7 @@ static u32 ddb_input_avail(struct ddb_input *input)
 }
 
 static ssize_t ddb_input_read(struct ddb_input *input,
-		__user u8 *buf, size_t count)
+			      __user u8 *buf, size_t count)
 {
 	struct ddb *dev = input->port->dev;
 	u32 left = count;
@@ -648,7 +649,8 @@ static ssize_t ddb_input_read(struct ddb_input *input,
 		if (free > left)
 			free = left;
 		if (alt_dma)
-			dma_sync_single_for_cpu(dev->dev,
+			dma_sync_single_for_cpu(
+				dev->dev,
 				input->dma->pbuf[input->dma->cbuf],
 				input->dma->size, DMA_FROM_DEVICE);
 		ret = copy_to_user(buf, input->dma->vbuf[input->dma->cbuf] +
@@ -732,20 +734,20 @@ static ssize_t ts_read(struct file *file, __user char *buf,
 	return (count && (left == count)) ? -EAGAIN : (count - left);
 }
 
-static unsigned int ts_poll(struct file *file, poll_table *wait)
+static __poll_t ts_poll(struct file *file, poll_table *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct ddb_output *output = dvbdev->priv;
 	struct ddb_input *input = output->port->input[0];
 
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	poll_wait(file, &input->dma->wq, wait);
 	poll_wait(file, &output->dma->wq, wait);
 	if (ddb_input_avail(input) >= 188)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	if (ddb_output_free(output) >= 188)
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 	return mask;
 }
 
@@ -792,8 +794,10 @@ static int ts_open(struct inode *inode, struct file *file)
 	} else if ((file->f_flags & O_ACCMODE) == O_WRONLY) {
 		if (!output)
 			return -EINVAL;
-	} else
+	} else {
 		return -EINVAL;
+	}
+
 	err = dvb_generic_open(inode, file);
 	if (err < 0)
 		return err;
@@ -822,7 +826,6 @@ static struct dvb_device dvbdev_ci = {
 	.fops    = &ci_fops,
 };
 
-
 /****************************************************************************/
 /****************************************************************************/
 
@@ -848,21 +851,20 @@ static int demod_attach_drxk(struct ddb_input *input)
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
 	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
 	struct device *dev = input->port->dev->dev;
-	struct dvb_frontend *fe;
 	struct drxk_config config;
 
 	memset(&config, 0, sizeof(config));
 	config.adr = 0x29 + (input->nr & 1);
 	config.microcode_name = "drxk_a3.mc";
 
-	fe = dvb->fe = dvb_attach(drxk_attach, &config, i2c);
-	if (!fe) {
+	dvb->fe = dvb_attach(drxk_attach, &config, i2c);
+	if (!dvb->fe) {
 		dev_err(dev, "No DRXK found!\n");
 		return -ENODEV;
 	}
-	fe->sec_priv = input;
-	dvb->i2c_gate_ctrl = fe->ops.i2c_gate_ctrl;
-	fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
+	dvb->fe->sec_priv = input;
+	dvb->i2c_gate_ctrl = dvb->fe->ops.i2c_gate_ctrl;
+	dvb->fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
 	return 0;
 }
 
@@ -912,19 +914,18 @@ static int demod_attach_stv0367(struct ddb_input *input)
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
 	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
 	struct device *dev = input->port->dev->dev;
-	struct dvb_frontend *fe;
 
 	/* attach frontend */
-	fe = dvb->fe = dvb_attach(stv0367ddb_attach,
-		&ddb_stv0367_config[(input->nr & 1)], i2c);
+	dvb->fe = dvb_attach(stv0367ddb_attach,
+			     &ddb_stv0367_config[(input->nr & 1)], i2c);
 
 	if (!dvb->fe) {
 		dev_err(dev, "No stv0367 found!\n");
 		return -ENODEV;
 	}
-	fe->sec_priv = input;
-	dvb->i2c_gate_ctrl = fe->ops.i2c_gate_ctrl;
-	fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
+	dvb->fe->sec_priv = input;
+	dvb->i2c_gate_ctrl = dvb->fe->ops.i2c_gate_ctrl;
+	dvb->fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
 	return 0;
 }
 
@@ -956,7 +957,6 @@ static int demod_attach_cxd28xx(struct ddb_input *input, int par, int osc24)
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
 	struct ddb_dvb *dvb = &input->port->dvb[input->nr & 1];
 	struct device *dev = input->port->dev->dev;
-	struct dvb_frontend *fe;
 	struct cxd2841er_config cfg;
 
 	/* the cxd2841er driver expects 8bit/shifted I2C addresses */
@@ -971,15 +971,15 @@ static int demod_attach_cxd28xx(struct ddb_input *input, int par, int osc24)
 		cfg.flags |= CXD2841ER_TS_SERIAL;
 
 	/* attach frontend */
-	fe = dvb->fe = dvb_attach(cxd2841er_attach_t_c, &cfg, i2c);
+	dvb->fe = dvb_attach(cxd2841er_attach_t_c, &cfg, i2c);
 
 	if (!dvb->fe) {
 		dev_err(dev, "No cxd2837/38/43/54 found!\n");
 		return -ENODEV;
 	}
-	fe->sec_priv = input;
-	dvb->i2c_gate_ctrl = fe->ops.i2c_gate_ctrl;
-	fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
+	dvb->fe->sec_priv = input;
+	dvb->i2c_gate_ctrl = dvb->fe->ops.i2c_gate_ctrl;
+	dvb->fe->ops.i2c_gate_ctrl = locked_gate_ctrl;
 	return 0;
 }
 
@@ -1020,7 +1020,7 @@ static int tuner_attach_tda18212(struct ddb_input *input, u32 porttype)
 
 	/* perform tuner init/attach */
 	client = i2c_new_device(adapter, &board_info);
-	if (client == NULL || client->dev.driver == NULL)
+	if (!client || !client->dev.driver)
 		goto err;
 
 	if (!try_module_get(client->dev.driver->owner)) {
@@ -1032,7 +1032,7 @@ static int tuner_attach_tda18212(struct ddb_input *input, u32 porttype)
 
 	return 0;
 err:
-	dev_notice(dev, "TDA18212 tuner not found. Device is not fully operational.\n");
+	dev_err(dev, "TDA18212 tuner not found. Device is not fully operational.\n");
 	return -ENODEV;
 }
 
@@ -1114,6 +1114,7 @@ static int demod_attach_stv0900(struct ddb_input *input, int type)
 			0, (input->nr & 1) ?
 			(0x09 - type) : (0x0b - type))) {
 		dev_err(dev, "No LNBH24 found!\n");
+		dvb_frontend_detach(dvb->fe);
 		return -ENODEV;
 	}
 	return 0;
@@ -1135,7 +1136,7 @@ static int tuner_attach_stv6110(struct ddb_input *input, int type)
 		return -ENODEV;
 	}
 	dev_info(dev, "attach tuner input %d adr %02x\n",
-		input->nr, tunerconf->addr);
+		 input->nr, tunerconf->addr);
 
 	feconf->tuner_init          = ctl->tuner_init;
 	feconf->tuner_sleep         = ctl->tuner_sleep;
@@ -1196,6 +1197,7 @@ static int demod_attach_stv0910(struct ddb_input *input, int type)
 		lnbcfg.i2c_address = (((input->nr & 1) ? 0x09 : 0x08) << 1);
 		if (!dvb_attach(lnbh25_attach, dvb->fe, &lnbcfg, i2c)) {
 			dev_err(dev, "No LNBH25 found!\n");
+			dvb_frontend_detach(dvb->fe);
 			return -ENODEV;
 		}
 	}
@@ -1261,19 +1263,22 @@ static void dvb_input_detach(struct ddb_input *input)
 			dvb_unregister_frontend(dvb->fe);
 		/* fallthrough */
 	case 0x30:
-		if (dvb->fe2)
-			dvb_frontend_detach(dvb->fe2);
-		if (dvb->fe)
-			dvb_frontend_detach(dvb->fe);
-		dvb->fe = dvb->fe2 = NULL;
-		/* fallthrough */
-	case 0x20:
 		client = dvb->i2c_client[0];
 		if (client) {
 			module_put(client->dev.driver->owner);
 			i2c_unregister_device(client);
+			dvb->i2c_client[0] = NULL;
+			client = NULL;
 		}
 
+		if (dvb->fe2)
+			dvb_frontend_detach(dvb->fe2);
+		if (dvb->fe)
+			dvb_frontend_detach(dvb->fe);
+		dvb->fe = NULL;
+		dvb->fe2 = NULL;
+		/* fallthrough */
+	case 0x20:
 		dvb_net_release(&dvb->dvbnet);
 		/* fallthrough */
 	case 0x12:
@@ -1409,7 +1414,8 @@ static int dvb_input_attach(struct ddb_input *input)
 		DMX_SECTION_FILTERING | DMX_MEMORY_BASED_FILTERING;
 	dvbdemux->start_feed = start_feed;
 	dvbdemux->stop_feed = stop_feed;
-	dvbdemux->filternum = dvbdemux->feednum = 256;
+	dvbdemux->filternum = 256;
+	dvbdemux->feednum = 256;
 	ret = dvb_dmx_init(dvbdemux);
 	if (ret < 0)
 		return ret;
@@ -1419,7 +1425,7 @@ static int dvb_input_attach(struct ddb_input *input)
 	dvb->dmxdev.demux = &dvbdemux->dmx;
 	ret = dvb_dmxdev_init(&dvb->dmxdev, adap);
 	if (ret < 0)
-		return ret;
+		goto err_detach;
 	dvb->attached = 0x11;
 
 	dvb->mem_frontend.source = DMX_MEMORY_FE;
@@ -1428,66 +1434,62 @@ static int dvb_input_attach(struct ddb_input *input)
 	dvb->demux.dmx.add_frontend(&dvb->demux.dmx, &dvb->hw_frontend);
 	ret = dvbdemux->dmx.connect_frontend(&dvbdemux->dmx, &dvb->hw_frontend);
 	if (ret < 0)
-		return ret;
+		goto err_detach;
 	dvb->attached = 0x12;
 
 	ret = dvb_net_init(adap, &dvb->dvbnet, dvb->dmxdev.demux);
 	if (ret < 0)
-		return ret;
+		goto err_detach;
 	dvb->attached = 0x20;
 
-	dvb->fe = dvb->fe2 = NULL;
+	dvb->fe = NULL;
+	dvb->fe2 = NULL;
 	switch (port->type) {
 	case DDB_TUNER_MXL5XX:
-		if (fe_attach_mxl5xx(input) < 0)
-			return -ENODEV;
+		if (ddb_fe_attach_mxl5xx(input) < 0)
+			goto err_detach;
 		break;
 	case DDB_TUNER_DVBS_ST:
 		if (demod_attach_stv0900(input, 0) < 0)
-			return -ENODEV;
+			goto err_detach;
 		if (tuner_attach_stv6110(input, 0) < 0)
-			return -ENODEV;
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBS_ST_AA:
 		if (demod_attach_stv0900(input, 1) < 0)
-			return -ENODEV;
+			goto err_detach;
 		if (tuner_attach_stv6110(input, 1) < 0)
-			return -ENODEV;
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBS_STV0910:
 		if (demod_attach_stv0910(input, 0) < 0)
-			return -ENODEV;
+			goto err_detach;
 		if (tuner_attach_stv6111(input, 0) < 0)
-			return -ENODEV;
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBS_STV0910_PR:
 		if (demod_attach_stv0910(input, 1) < 0)
-			return -ENODEV;
+			goto err_detach;
 		if (tuner_attach_stv6111(input, 1) < 0)
-			return -ENODEV;
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBS_STV0910_P:
 		if (demod_attach_stv0910(input, 0) < 0)
-			return -ENODEV;
+			goto err_detach;
 		if (tuner_attach_stv6111(input, 1) < 0)
-			return -ENODEV;
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBCT_TR:
 		if (demod_attach_drxk(input) < 0)
-			return -ENODEV;
+			goto err_detach;
 		if (tuner_attach_tda18271(input) < 0)
-			return -ENODEV;
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBCT_ST:
 		if (demod_attach_stv0367(input) < 0)
-			return -ENODEV;
-		if (tuner_attach_tda18212(input, port->type) < 0) {
-			if (dvb->fe2)
-				dvb_frontend_detach(dvb->fe2);
-			if (dvb->fe)
-				dvb_frontend_detach(dvb->fe);
-			return -ENODEV;
-		}
+			goto err_detach;
+		if (tuner_attach_tda18212(input, port->type) < 0)
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBC2T2I_SONY_P:
 		if (input->port->dev->link[input->port->lnr].info->ts_quirks &
@@ -1505,14 +1507,9 @@ static int dvb_input_attach(struct ddb_input *input)
 		else
 			par = 1;
 		if (demod_attach_cxd28xx(input, par, osc24) < 0)
-			return -ENODEV;
-		if (tuner_attach_tda18212(input, port->type) < 0) {
-			if (dvb->fe2)
-				dvb_frontend_detach(dvb->fe2);
-			if (dvb->fe)
-				dvb_frontend_detach(dvb->fe);
-			return -ENODEV;
-		}
+			goto err_detach;
+		if (tuner_attach_tda18212(input, port->type) < 0)
+			goto err_tuner;
 		break;
 	case DDB_TUNER_DVBC2T2I_SONY:
 		osc24 = 1;
@@ -1521,14 +1518,9 @@ static int dvb_input_attach(struct ddb_input *input)
 	case DDB_TUNER_DVBC2T2_SONY:
 	case DDB_TUNER_ISDBT_SONY:
 		if (demod_attach_cxd28xx(input, 0, osc24) < 0)
-			return -ENODEV;
-		if (tuner_attach_tda18212(input, port->type) < 0) {
-			if (dvb->fe2)
-				dvb_frontend_detach(dvb->fe2);
-			if (dvb->fe)
-				dvb_frontend_detach(dvb->fe);
-			return -ENODEV;
-		}
+			goto err_detach;
+		if (tuner_attach_tda18212(input, port->type) < 0)
+			goto err_tuner;
 		break;
 	default:
 		return 0;
@@ -1537,11 +1529,13 @@ static int dvb_input_attach(struct ddb_input *input)
 
 	if (dvb->fe) {
 		if (dvb_register_frontend(adap, dvb->fe) < 0)
-			return -ENODEV;
+			goto err_detach;
 
 		if (dvb->fe2) {
-			if (dvb_register_frontend(adap, dvb->fe2) < 0)
-				return -ENODEV;
+			if (dvb_register_frontend(adap, dvb->fe2) < 0) {
+				dvb_unregister_frontend(dvb->fe);
+				goto err_detach;
+			}
 			dvb->fe2->tuner_priv = dvb->fe->tuner_priv;
 			memcpy(&dvb->fe2->ops.tuner_ops,
 			       &dvb->fe->ops.tuner_ops,
@@ -1551,6 +1545,22 @@ static int dvb_input_attach(struct ddb_input *input)
 
 	dvb->attached = 0x31;
 	return 0;
+
+err_tuner:
+	dev_err(port->dev->dev, "tuner attach failed!\n");
+
+	if (dvb->fe2)
+		dvb_frontend_detach(dvb->fe2);
+	if (dvb->fe)
+		dvb_frontend_detach(dvb->fe);
+err_detach:
+	dvb_input_detach(input);
+
+	/* return error from ret if set */
+	if (ret < 0)
+		return ret;
+
+	return -ENODEV;
 }
 
 static int port_has_encti(struct ddb_port *port)
@@ -1703,11 +1713,11 @@ static int init_xo2_ci(struct ddb_port *port)
 
 	if (data[0] > 1)  {
 		dev_info(dev->dev, "Port %d: invalid XO2 CI %02x\n",
-			port->nr, data[0]);
+			 port->nr, data[0]);
 		return -1;
 	}
 	dev_info(dev->dev, "Port %d: DuoFlex CI %u.%u\n",
-		port->nr, data[0], data[1]);
+		 port->nr, data[0], data[1]);
 
 	i2c_read_reg(i2c, 0x10, 0x08, &val);
 	if (val != 0) {
@@ -1718,7 +1728,6 @@ static int init_xo2_ci(struct ddb_port *port)
 	i2c_write_reg(i2c, 0x10, 0x08, 3);
 	usleep_range(2000, 3000);
 
-
 	/* speed: 0=55,1=75,2=90,3=104 MBit/s */
 	i2c_write_reg(i2c, 0x10, 0x09, 1);
 
@@ -1821,7 +1830,7 @@ static void ddb_port_probe(struct ddb_port *port)
 				  port->i2c->regs + I2C_TIMING);
 		} else {
 			dev_info(dev->dev, "Port %d: Uninitialized DuoFlex\n",
-			       port->nr);
+				 port->nr);
 			return;
 		}
 	} else if (port_has_xo2(port, &type, &id)) {
@@ -1912,338 +1921,10 @@ static void ddb_port_probe(struct ddb_port *port)
 	}
 }
 
-
-/****************************************************************************/
-/****************************************************************************/
-/****************************************************************************/
-
-static int wait_ci_ready(struct ddb_ci *ci)
-{
-	u32 count = 10;
-
-	ndelay(500);
-	do {
-		if (ddbreadl(ci->port->dev,
-			     CI_CONTROL(ci->nr)) & CI_READY)
-			break;
-		usleep_range(1, 2);
-		if ((--count) == 0)
-			return -1;
-	} while (1);
-	return 0;
-}
-
-static int read_attribute_mem(struct dvb_ca_en50221 *ca,
-			      int slot, int address)
-{
-	struct ddb_ci *ci = ca->data;
-	u32 val, off = (address >> 1) & (CI_BUFFER_SIZE - 1);
-
-	if (address > CI_BUFFER_SIZE)
-		return -1;
-	ddbwritel(ci->port->dev, CI_READ_CMD | (1 << 16) | address,
-		  CI_DO_READ_ATTRIBUTES(ci->nr));
-	wait_ci_ready(ci);
-	val = 0xff & ddbreadl(ci->port->dev, CI_BUFFER(ci->nr) + off);
-	return val;
-}
-
-static int write_attribute_mem(struct dvb_ca_en50221 *ca, int slot,
-			       int address, u8 value)
-{
-	struct ddb_ci *ci = ca->data;
-
-	ddbwritel(ci->port->dev, CI_WRITE_CMD | (value << 16) | address,
-		  CI_DO_ATTRIBUTE_RW(ci->nr));
-	wait_ci_ready(ci);
-	return 0;
-}
-
-static int read_cam_control(struct dvb_ca_en50221 *ca,
-			    int slot, u8 address)
-{
-	u32 count = 100;
-	struct ddb_ci *ci = ca->data;
-	u32 res;
-
-	ddbwritel(ci->port->dev, CI_READ_CMD | address,
-		  CI_DO_IO_RW(ci->nr));
-	ndelay(500);
-	do {
-		res = ddbreadl(ci->port->dev, CI_READDATA(ci->nr));
-		if (res & CI_READY)
-			break;
-		usleep_range(1, 2);
-		if ((--count) == 0)
-			return -1;
-	} while (1);
-	return 0xff & res;
-}
-
-static int write_cam_control(struct dvb_ca_en50221 *ca, int slot,
-			     u8 address, u8 value)
-{
-	struct ddb_ci *ci = ca->data;
-
-	ddbwritel(ci->port->dev, CI_WRITE_CMD | (value << 16) | address,
-		  CI_DO_IO_RW(ci->nr));
-	wait_ci_ready(ci);
-	return 0;
-}
-
-static int slot_reset(struct dvb_ca_en50221 *ca, int slot)
-{
-	struct ddb_ci *ci = ca->data;
-
-	ddbwritel(ci->port->dev, CI_POWER_ON,
-		  CI_CONTROL(ci->nr));
-	msleep(100);
-	ddbwritel(ci->port->dev, CI_POWER_ON | CI_RESET_CAM,
-		  CI_CONTROL(ci->nr));
-	ddbwritel(ci->port->dev, CI_ENABLE | CI_POWER_ON | CI_RESET_CAM,
-		  CI_CONTROL(ci->nr));
-	udelay(20);
-	ddbwritel(ci->port->dev, CI_ENABLE | CI_POWER_ON,
-		  CI_CONTROL(ci->nr));
-	return 0;
-}
-
-static int slot_shutdown(struct dvb_ca_en50221 *ca, int slot)
-{
-	struct ddb_ci *ci = ca->data;
-
-	ddbwritel(ci->port->dev, 0, CI_CONTROL(ci->nr));
-	msleep(300);
-	return 0;
-}
-
-static int slot_ts_enable(struct dvb_ca_en50221 *ca, int slot)
-{
-	struct ddb_ci *ci = ca->data;
-	u32 val = ddbreadl(ci->port->dev, CI_CONTROL(ci->nr));
-
-	ddbwritel(ci->port->dev, val | CI_BYPASS_DISABLE,
-		  CI_CONTROL(ci->nr));
-	return 0;
-}
-
-static int poll_slot_status(struct dvb_ca_en50221 *ca, int slot, int open)
-{
-	struct ddb_ci *ci = ca->data;
-	u32 val = ddbreadl(ci->port->dev, CI_CONTROL(ci->nr));
-	int stat = 0;
-
-	if (val & CI_CAM_DETECT)
-		stat |= DVB_CA_EN50221_POLL_CAM_PRESENT;
-	if (val & CI_CAM_READY)
-		stat |= DVB_CA_EN50221_POLL_CAM_READY;
-	return stat;
-}
-
-static struct dvb_ca_en50221 en_templ = {
-	.read_attribute_mem  = read_attribute_mem,
-	.write_attribute_mem = write_attribute_mem,
-	.read_cam_control    = read_cam_control,
-	.write_cam_control   = write_cam_control,
-	.slot_reset          = slot_reset,
-	.slot_shutdown       = slot_shutdown,
-	.slot_ts_enable      = slot_ts_enable,
-	.poll_slot_status    = poll_slot_status,
-};
-
-static void ci_attach(struct ddb_port *port)
-{
-	struct ddb_ci *ci = NULL;
-
-	ci = kzalloc(sizeof(*ci), GFP_KERNEL);
-	if (!ci)
-		return;
-	memcpy(&ci->en, &en_templ, sizeof(en_templ));
-	ci->en.data = ci;
-	port->en = &ci->en;
-	ci->port = port;
-	ci->nr = port->nr - 2;
-}
-
 /****************************************************************************/
 /****************************************************************************/
 /****************************************************************************/
 
-static int write_creg(struct ddb_ci *ci, u8 data, u8 mask)
-{
-	struct i2c_adapter *i2c = &ci->port->i2c->adap;
-	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
-
-	ci->port->creg = (ci->port->creg & ~mask) | data;
-	return i2c_write_reg(i2c, adr, 0x02, ci->port->creg);
-}
-
-static int read_attribute_mem_xo2(struct dvb_ca_en50221 *ca,
-				  int slot, int address)
-{
-	struct ddb_ci *ci = ca->data;
-	struct i2c_adapter *i2c = &ci->port->i2c->adap;
-	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
-	int res;
-	u8 val;
-
-	res = i2c_read_reg16(i2c, adr, 0x8000 | address, &val);
-	return res ? res : val;
-}
-
-static int write_attribute_mem_xo2(struct dvb_ca_en50221 *ca, int slot,
-				   int address, u8 value)
-{
-	struct ddb_ci *ci = ca->data;
-	struct i2c_adapter *i2c = &ci->port->i2c->adap;
-	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
-
-	return i2c_write_reg16(i2c, adr, 0x8000 | address, value);
-}
-
-static int read_cam_control_xo2(struct dvb_ca_en50221 *ca,
-				int slot, u8 address)
-{
-	struct ddb_ci *ci = ca->data;
-	struct i2c_adapter *i2c = &ci->port->i2c->adap;
-	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
-	u8 val;
-	int res;
-
-	res = i2c_read_reg(i2c, adr, 0x20 | (address & 3), &val);
-	return res ? res : val;
-}
-
-static int write_cam_control_xo2(struct dvb_ca_en50221 *ca, int slot,
-				 u8 address, u8 value)
-{
-	struct ddb_ci *ci = ca->data;
-	struct i2c_adapter *i2c = &ci->port->i2c->adap;
-	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
-
-	return i2c_write_reg(i2c, adr, 0x20 | (address & 3), value);
-}
-
-static int slot_reset_xo2(struct dvb_ca_en50221 *ca, int slot)
-{
-	struct ddb_ci *ci = ca->data;
-
-	dev_dbg(ci->port->dev->dev, "%s\n", __func__);
-	write_creg(ci, 0x01, 0x01);
-	write_creg(ci, 0x04, 0x04);
-	msleep(20);
-	write_creg(ci, 0x02, 0x02);
-	write_creg(ci, 0x00, 0x04);
-	write_creg(ci, 0x18, 0x18);
-	return 0;
-}
-
-static int slot_shutdown_xo2(struct dvb_ca_en50221 *ca, int slot)
-{
-	struct ddb_ci *ci = ca->data;
-
-	dev_dbg(ci->port->dev->dev, "%s\n", __func__);
-	write_creg(ci, 0x10, 0xff);
-	write_creg(ci, 0x08, 0x08);
-	return 0;
-}
-
-static int slot_ts_enable_xo2(struct dvb_ca_en50221 *ca, int slot)
-{
-	struct ddb_ci *ci = ca->data;
-
-	dev_info(ci->port->dev->dev, "%s\n", __func__);
-	write_creg(ci, 0x00, 0x10);
-	return 0;
-}
-
-static int poll_slot_status_xo2(struct dvb_ca_en50221 *ca, int slot, int open)
-{
-	struct ddb_ci *ci = ca->data;
-	struct i2c_adapter *i2c = &ci->port->i2c->adap;
-	u8 adr = (ci->port->type == DDB_CI_EXTERNAL_XO2) ? 0x12 : 0x13;
-	u8 val = 0;
-	int stat = 0;
-
-	i2c_read_reg(i2c, adr, 0x01, &val);
-
-	if (val & 2)
-		stat |= DVB_CA_EN50221_POLL_CAM_PRESENT;
-	if (val & 1)
-		stat |= DVB_CA_EN50221_POLL_CAM_READY;
-	return stat;
-}
-
-static struct dvb_ca_en50221 en_xo2_templ = {
-	.read_attribute_mem  = read_attribute_mem_xo2,
-	.write_attribute_mem = write_attribute_mem_xo2,
-	.read_cam_control    = read_cam_control_xo2,
-	.write_cam_control   = write_cam_control_xo2,
-	.slot_reset          = slot_reset_xo2,
-	.slot_shutdown       = slot_shutdown_xo2,
-	.slot_ts_enable      = slot_ts_enable_xo2,
-	.poll_slot_status    = poll_slot_status_xo2,
-};
-
-static void ci_xo2_attach(struct ddb_port *port)
-{
-	struct ddb_ci *ci;
-
-	ci = kzalloc(sizeof(*ci), GFP_KERNEL);
-	if (!ci)
-		return;
-	memcpy(&ci->en, &en_xo2_templ, sizeof(en_xo2_templ));
-	ci->en.data = ci;
-	port->en = &ci->en;
-	ci->port = port;
-	ci->nr = port->nr - 2;
-	ci->port->creg = 0;
-	write_creg(ci, 0x10, 0xff);
-	write_creg(ci, 0x08, 0x08);
-}
-
-/****************************************************************************/
-/****************************************************************************/
-/****************************************************************************/
-
-static struct cxd2099_cfg cxd_cfg = {
-	.bitrate =  72000,
-	.adr     =  0x40,
-	.polarity = 1,
-	.clock_mode = 1,
-	.max_i2c = 512,
-};
-
-static int ddb_ci_attach(struct ddb_port *port)
-{
-	switch (port->type) {
-	case DDB_CI_EXTERNAL_SONY:
-		cxd_cfg.bitrate = ci_bitrate;
-		port->en = cxd2099_attach(&cxd_cfg, port, &port->i2c->adap);
-		if (!port->en)
-			return -ENODEV;
-		dvb_ca_en50221_init(port->dvb[0].adap,
-				    port->en, 0, 1);
-		break;
-
-	case DDB_CI_EXTERNAL_XO2:
-	case DDB_CI_EXTERNAL_XO2_B:
-		ci_xo2_attach(port);
-		if (!port->en)
-			return -ENODEV;
-		dvb_ca_en50221_init(port->dvb[0].adap, port->en, 0, 1);
-		break;
-
-	case DDB_CI_INTERNAL:
-		ci_attach(port);
-		if (!port->en)
-			return -ENODEV;
-		dvb_ca_en50221_init(port->dvb[0].adap, port->en, 0, 1);
-		break;
-	}
-	return 0;
-}
-
 static int ddb_port_attach(struct ddb_port *port)
 {
 	int ret = 0;
@@ -2254,20 +1935,22 @@ static int ddb_port_attach(struct ddb_port *port)
 		if (ret < 0)
 			break;
 		ret = dvb_input_attach(port->input[1]);
-		if (ret < 0)
+		if (ret < 0) {
+			dvb_input_detach(port->input[0]);
 			break;
+		}
 		port->input[0]->redi = port->input[0];
 		port->input[1]->redi = port->input[1];
 		break;
 	case DDB_PORT_CI:
-		ret = ddb_ci_attach(port);
+		ret = ddb_ci_attach(port, ci_bitrate);
 		if (ret < 0)
 			break;
 		/* fall-through */
 	case DDB_PORT_LOOP:
 		ret = dvb_register_device(port->dvb[0].adap,
 					  &port->dvb[0].dev,
-					  &dvbdev_ci, (void *) port->output,
+					  &dvbdev_ci, (void *)port->output,
 					  DVB_DEVICE_SEC, 0);
 		break;
 	default:
@@ -2281,7 +1964,7 @@ static int ddb_port_attach(struct ddb_port *port)
 
 int ddb_ports_attach(struct ddb *dev)
 {
-	int i, ret = 0;
+	int i, numports, err_ports = 0, ret = 0;
 	struct ddb_port *port;
 
 	if (dev->port_num) {
@@ -2291,11 +1974,31 @@ int ddb_ports_attach(struct ddb *dev)
 			return ret;
 		}
 	}
+
+	numports = dev->port_num;
+
 	for (i = 0; i < dev->port_num; i++) {
 		port = &dev->port[i];
-		ret = ddb_port_attach(port);
+		if (port->class != DDB_PORT_NONE) {
+			ret = ddb_port_attach(port);
+			if (ret)
+				err_ports++;
+		} else {
+			numports--;
+		}
 	}
-	return ret;
+
+	if (err_ports) {
+		if (err_ports == numports) {
+			dev_err(dev->dev, "All connected ports failed to initialise!\n");
+			return -ENODEV;
+		}
+
+		dev_warn(dev->dev, "%d of %d connected ports failed to initialise!\n",
+			 err_ports, numports);
+	}
+
+	return 0;
 }
 
 void ddb_ports_detach(struct ddb *dev)
@@ -2308,25 +2011,18 @@ void ddb_ports_detach(struct ddb *dev)
 
 		switch (port->class) {
 		case DDB_PORT_TUNER:
-			dvb_input_detach(port->input[0]);
 			dvb_input_detach(port->input[1]);
+			dvb_input_detach(port->input[0]);
 			break;
 		case DDB_PORT_CI:
 		case DDB_PORT_LOOP:
-			if (port->dvb[0].dev)
-				dvb_unregister_device(port->dvb[0].dev);
-			if (port->en) {
-				dvb_ca_en50221_release(port->en);
-				kfree(port->en);
-				port->en = NULL;
-			}
+			ddb_ci_detach(port);
 			break;
 		}
 	}
 	dvb_unregister_adapters(dev);
 }
 
-
 /* Copy input DMA pointers to output DMA and ACK. */
 
 static void input_write_output(struct ddb_input *input,
@@ -2353,16 +2049,18 @@ static void input_write_dvb(struct ddb_input *input,
 	struct ddb *dev = input->port->dev;
 	int ack = 1;
 
-	dma = dma2 = input->dma;
-	/* if there also is an output connected, do not ACK.
+	dma = input->dma;
+	dma2 = input->dma;
+	/*
+	 * if there also is an output connected, do not ACK.
 	 * input_write_output will ACK.
 	 */
 	if (input->redo) {
 		dma2 = input->redo->dma;
 		ack = 0;
 	}
-	while (dma->cbuf != ((dma->stat >> 11) & 0x1f)
-	       || (4 & dma->ctrl)) {
+	while (dma->cbuf != ((dma->stat >> 11) & 0x1f) ||
+	       (4 & dma->ctrl)) {
 		if (4 & dma->ctrl) {
 			/* dev_err(dev->dev, "Overflow dma %d\n", dma->nr); */
 			ack = 1;
@@ -2385,7 +2083,7 @@ static void input_write_dvb(struct ddb_input *input,
 static void input_work(struct work_struct *work)
 {
 	struct ddb_dma *dma = container_of(work, struct ddb_dma, work);
-	struct ddb_input *input = (struct ddb_input *) dma->io;
+	struct ddb_input *input = (struct ddb_input *)dma->io;
 	struct ddb *dev = input->port->dev;
 	unsigned long flags;
 
@@ -2407,11 +2105,11 @@ static void input_work(struct work_struct *work)
 
 static void input_handler(unsigned long data)
 {
-	struct ddb_input *input = (struct ddb_input *) data;
+	struct ddb_input *input = (struct ddb_input *)data;
 	struct ddb_dma *dma = input->dma;
 
-
-	/* If there is no input connected, input_tasklet() will
+	/*
+	 * If there is no input connected, input_tasklet() will
 	 * just copy pointers and ACK. So, there is no need to go
 	 * through the tasklet scheduler.
 	 */
@@ -2423,7 +2121,7 @@ static void input_handler(unsigned long data)
 
 static void output_handler(unsigned long data)
 {
-	struct ddb_output *output = (struct ddb_output *) data;
+	struct ddb_output *output = (struct ddb_output *)data;
 	struct ddb_dma *dma = output->dma;
 	struct ddb *dev = output->port->dev;
 
@@ -2512,10 +2210,10 @@ static void ddb_input_init(struct ddb_port *port, int nr, int pnr, int anr)
 			dma_nr += 32 + (port->lnr - 1) * 8;
 
 		dev_dbg(dev->dev, "init link %u, input %u, handler %u\n",
-			 port->lnr, nr, dma_nr + base);
+			port->lnr, nr, dma_nr + base);
 
 		dev->handler[0][dma_nr + base] = input_handler;
-		dev->handler_data[0][dma_nr + base] = (unsigned long) input;
+		dev->handler_data[0][dma_nr + base] = (unsigned long)input;
 		ddb_dma_init(input, dma_nr, 0);
 	}
 }
@@ -2534,14 +2232,14 @@ static void ddb_output_init(struct ddb_port *port, int nr)
 		(rm->output->base + rm->output->size * nr);
 
 	dev_dbg(dev->dev, "init link %u, output %u, regs %08x\n",
-		 port->lnr, nr, output->regs);
+		port->lnr, nr, output->regs);
 
 	if (dev->has_dma) {
 		const struct ddb_regmap *rm0 = io_regmap(output, 0);
 		u32 base = rm0->irq_base_odma;
 
 		dev->handler[0][nr + base] = output_handler;
-		dev->handler_data[0][nr + base] = (unsigned long) output;
+		dev->handler_data[0][nr + base] = (unsigned long)output;
 		ddb_dma_init(output, nr, 1);
 	}
 }
@@ -2609,7 +2307,7 @@ void ddb_ports_init(struct ddb *dev)
 			port->dvb[0].adap = &dev->adap[2 * p];
 			port->dvb[1].adap = &dev->adap[2 * p + 1];
 
-			if ((port->class == DDB_PORT_NONE) && i && p &&
+			if (port->class == DDB_PORT_NONE && i && p &&
 			    dev->port[p - 1].type == DDB_CI_EXTERNAL_XO2) {
 				port->class = DDB_PORT_CI;
 				port->type = DDB_CI_EXTERNAL_XO2_B;
@@ -2618,8 +2316,8 @@ void ddb_ports_init(struct ddb *dev)
 			}
 
 			dev_info(dev->dev, "Port %u: Link %u, Link Port %u (TAB %u): %s\n",
-				port->pnr, port->lnr, port->nr, port->nr + 1,
-				port->name);
+				 port->pnr, port->lnr, port->nr, port->nr + 1,
+				 port->name);
 
 			if (port->class == DDB_PORT_CI &&
 			    port->type == DDB_CI_EXTERNAL_XO2) {
@@ -2741,7 +2439,7 @@ static void irq_handle_io(struct ddb *dev, u32 s)
 
 irqreturn_t ddb_irq_handler0(int irq, void *dev_id)
 {
-	struct ddb *dev = (struct ddb *) dev_id;
+	struct ddb *dev = (struct ddb *)dev_id;
 	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
 
 	do {
@@ -2758,7 +2456,7 @@ irqreturn_t ddb_irq_handler0(int irq, void *dev_id)
 
 irqreturn_t ddb_irq_handler1(int irq, void *dev_id)
 {
-	struct ddb *dev = (struct ddb *) dev_id;
+	struct ddb *dev = (struct ddb *)dev_id;
 	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
 
 	do {
@@ -2775,7 +2473,7 @@ irqreturn_t ddb_irq_handler1(int irq, void *dev_id)
 
 irqreturn_t ddb_irq_handler(int irq, void *dev_id)
 {
-	struct ddb *dev = (struct ddb *) dev_id;
+	struct ddb *dev = (struct ddb *)dev_id;
 	u32 s = ddbreadl(dev, INTERRUPT_STATUS);
 	int ret = IRQ_HANDLED;
 
@@ -2812,7 +2510,7 @@ static int reg_wait(struct ddb *dev, u32 reg, u32 bit)
 }
 
 static int flashio(struct ddb *dev, u32 lnr, u8 *wbuf, u32 wlen, u8 *rbuf,
-	u32 rlen)
+		   u32 rlen)
 {
 	u32 data, shift;
 	u32 tag = DDB_LINK_TAG(lnr);
@@ -2823,7 +2521,7 @@ static int flashio(struct ddb *dev, u32 lnr, u8 *wbuf, u32 wlen, u8 *rbuf,
 		ddbwritel(dev, 1, tag | SPI_CONTROL);
 	while (wlen > 4) {
 		/* FIXME: check for big-endian */
-		data = swab32(*(u32 *) wbuf);
+		data = swab32(*(u32 *)wbuf);
 		wbuf += 4;
 		wlen -= 4;
 		ddbwritel(dev, data, tag | SPI_DATA);
@@ -2863,12 +2561,12 @@ static int flashio(struct ddb *dev, u32 lnr, u8 *wbuf, u32 wlen, u8 *rbuf,
 		if (reg_wait(dev, tag | SPI_CONTROL, 4))
 			goto fail;
 		data = ddbreadl(dev, tag | SPI_DATA);
-		*(u32 *) rbuf = swab32(data);
+		*(u32 *)rbuf = swab32(data);
 		rbuf += 4;
 		rlen -= 4;
 	}
 	ddbwritel(dev, 0x0003 | ((rlen << (8 + 3)) & 0x1F00),
-		tag | SPI_CONTROL);
+		  tag | SPI_CONTROL);
 	ddbwritel(dev, 0xffffffff, tag | SPI_DATA);
 	if (reg_wait(dev, tag | SPI_CONTROL, 4))
 		goto fail;
@@ -3011,7 +2709,7 @@ static ssize_t fan_store(struct device *device, struct device_attribute *d,
 }
 
 static ssize_t fanspeed_show(struct device *device,
-			struct device_attribute *attr, char *buf)
+			     struct device_attribute *attr, char *buf)
 {
 	struct ddb *dev = dev_get_drvdata(device);
 	int num = attr->attr.name[8] - 0x30;
@@ -3049,7 +2747,7 @@ static ssize_t temp_show(struct device *device,
 }
 
 static ssize_t ctemp_show(struct device *device,
-		struct device_attribute *attr, char *buf)
+			  struct device_attribute *attr, char *buf)
 {
 	struct ddb *dev = dev_get_drvdata(device);
 	struct i2c_adapter *adap;
@@ -3076,7 +2774,6 @@ static ssize_t led_show(struct device *device,
 	return sprintf(buf, "%d\n", dev->leds & (1 << num) ? 1 : 0);
 }
 
-
 static void ddb_set_led(struct ddb *dev, int num, int val)
 {
 	if (!dev->link[0].info->led_num)
@@ -3163,7 +2860,7 @@ static ssize_t bsnr_show(struct device *device,
 }
 
 static ssize_t bpsnr_show(struct device *device,
-			 struct device_attribute *attr, char *buf)
+			  struct device_attribute *attr, char *buf)
 {
 	struct ddb *dev = dev_get_drvdata(device);
 	unsigned char snr[32];
@@ -3208,7 +2905,6 @@ static ssize_t gap_show(struct device *device,
 	int num = attr->attr.name[3] - 0x30;
 
 	return sprintf(buf, "%d\n", dev->port[num].gap);
-
 }
 
 static ssize_t gap_store(struct device *device, struct device_attribute *attr,
@@ -3254,7 +2950,7 @@ static ssize_t regmap_show(struct device *device,
 }
 
 static ssize_t fmode_show(struct device *device,
-			 struct device_attribute *attr, char *buf)
+			  struct device_attribute *attr, char *buf)
 {
 	int num = attr->attr.name[5] - 0x30;
 	struct ddb *dev = dev_get_drvdata(device);
@@ -3272,7 +2968,7 @@ static ssize_t devid_show(struct device *device,
 }
 
 static ssize_t fmode_store(struct device *device, struct device_attribute *attr,
-			  const char *buf, size_t count)
+			   const char *buf, size_t count)
 {
 	struct ddb *dev = dev_get_drvdata(device);
 	int num = attr->attr.name[5] - 0x30;
@@ -3282,7 +2978,7 @@ static ssize_t fmode_store(struct device *device, struct device_attribute *attr,
 		return -EINVAL;
 	if (val > 3)
 		return -EINVAL;
-	lnb_init_fmode(dev, &dev->link[num], val);
+	ddb_lnb_init_fmode(dev, &dev->link[num], val);
 	return count;
 }
 
@@ -3387,7 +3083,7 @@ static void ddb_device_attrs_del(struct ddb *dev)
 		device_remove_file(dev->ddb_dev, &ddb_attrs_snr[i]);
 		device_remove_file(dev->ddb_dev, &ddb_attrs_ctemp[i]);
 	}
-	for (i = 0; ddb_attrs[i].attr.name != NULL; i++)
+	for (i = 0; ddb_attrs[i].attr.name; i++)
 		device_remove_file(dev->ddb_dev, &ddb_attrs[i]);
 }
 
@@ -3395,7 +3091,7 @@ static int ddb_device_attrs_add(struct ddb *dev)
 {
 	int i;
 
-	for (i = 0; ddb_attrs[i].attr.name != NULL; i++)
+	for (i = 0; ddb_attrs[i].attr.name; i++)
 		if (device_create_file(dev->ddb_dev, &ddb_attrs[i]))
 			goto fail;
 	for (i = 0; i < dev->link[0].info->temp_num; i++)
@@ -3447,8 +3143,9 @@ int ddb_device_create(struct ddb *dev)
 		device_destroy(&ddb_class, MKDEV(ddb_major, dev->nr));
 		ddbs[dev->nr] = NULL;
 		dev->ddb_dev = ERR_PTR(-ENODEV);
-	} else
+	} else {
 		ddb_num++;
+	}
 fail:
 	mutex_unlock(&ddb_mutex);
 	return res;
@@ -3500,7 +3197,7 @@ static void tempmon_setfan(struct ddb_link *link)
 
 static void temp_handler(unsigned long data)
 {
-	struct ddb_link *link = (struct ddb_link *) data;
+	struct ddb_link *link = (struct ddb_link *)data;
 
 	spin_lock(&link->temp_lock);
 	tempmon_setfan(link);
@@ -3519,10 +3216,10 @@ static int tempmon_init(struct ddb_link *link, int first_time)
 			30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80 };
 
 		memcpy(link->temp_tab, temperature_table,
-			sizeof(temperature_table));
+		       sizeof(temperature_table));
 	}
 	dev->handler[l][link->info->tempmon_irq] = temp_handler;
-	dev->handler_data[l][link->info->tempmon_irq] = (unsigned long) link;
+	dev->handler_data[l][link->info->tempmon_irq] = (unsigned long)link;
 	ddblwritel(link, (TEMPMON_CONTROL_OVERTEMP | TEMPMON_CONTROL_AUTOSCAN |
 			  TEMPMON_CONTROL_INTENABLE),
 		   TEMPMON_CONTROL);
@@ -3574,11 +3271,11 @@ static int ddb_init_boards(struct ddb *dev)
 			ddbwritel(dev, 0, DDB_LINK_TAG(l) | BOARD_CONTROL);
 			msleep(100);
 			ddbwritel(dev, info->board_control_2,
-				DDB_LINK_TAG(l) | BOARD_CONTROL);
+				  DDB_LINK_TAG(l) | BOARD_CONTROL);
 			usleep_range(2000, 3000);
 			ddbwritel(dev,
-				info->board_control_2 | info->board_control,
-				DDB_LINK_TAG(l) | BOARD_CONTROL);
+				  info->board_control_2 | info->board_control,
+				  DDB_LINK_TAG(l) | BOARD_CONTROL);
 			usleep_range(2000, 3000);
 		}
 		ddb_init_tempmon(link);
@@ -3598,7 +3295,7 @@ int ddb_init(struct ddb *dev)
 	ddb_init_boards(dev);
 
 	if (ddb_i2c_init(dev) < 0)
-		goto fail;
+		goto fail1;
 	ddb_ports_init(dev);
 	if (ddb_buffers_alloc(dev) < 0) {
 		dev_info(dev->dev, "Could not allocate buffer memory\n");
@@ -3616,14 +3313,14 @@ int ddb_init(struct ddb *dev)
 	return 0;
 
 fail3:
-	ddb_ports_detach(dev);
 	dev_err(dev->dev, "fail3\n");
-	ddb_ports_release(dev);
+	ddb_ports_detach(dev);
+	ddb_buffers_free(dev);
 fail2:
 	dev_err(dev->dev, "fail2\n");
-	ddb_buffers_free(dev);
+	ddb_ports_release(dev);
 	ddb_i2c_release(dev);
-fail:
+fail1:
 	dev_err(dev->dev, "fail1\n");
 	return -1;
 }
diff --git a/drivers/media/pci/ddbridge/ddbridge-hw.c b/drivers/media/pci/ddbridge/ddbridge-hw.c
index 48248bcd59c2..c6d14925e2fc 100644
--- a/drivers/media/pci/ddbridge/ddbridge-hw.c
+++ b/drivers/media/pci/ddbridge/ddbridge-hw.c
@@ -185,7 +185,7 @@ static const struct ddb_info ddb_ctv7 = {
 	.board_control_2 = 4,
 };
 
-static const struct ddb_info ddb_satixS2v3 = {
+static const struct ddb_info ddb_satixs2v3 = {
 	.type     = DDB_OCTOPUS,
 	.name     = "Mystique SaTiX-S2 V3 DVB adapter",
 	.regmap   = &octopus_map,
@@ -336,7 +336,7 @@ static const struct ddb_device_id ddb_device_ids[] = {
 	DDB_DEVID(0x0006, 0x0022, ddb_v7),
 	DDB_DEVID(0x0006, 0x0024, ddb_v7a),
 	DDB_DEVID(0x0003, 0x0030, ddb_dvbct),
-	DDB_DEVID(0x0003, 0xdb03, ddb_satixS2v3),
+	DDB_DEVID(0x0003, 0xdb03, ddb_satixs2v3),
 	DDB_DEVID(0x0006, 0x0031, ddb_ctv7),
 	DDB_DEVID(0x0006, 0x0032, ddb_ctv7),
 	DDB_DEVID(0x0006, 0x0033, ddb_ctv7),
@@ -367,8 +367,8 @@ const struct ddb_info *get_ddb_info(u16 vendor, u16 device,
 		if (vendor == id->vendor &&
 		    device == id->device &&
 		    subvendor == id->subvendor &&
-		    ((subdevice == id->subdevice) ||
-		     (id->subdevice == 0xffff)))
+		    (subdevice == id->subdevice ||
+		     id->subdevice == 0xffff))
 			return id->info;
 	}
 
diff --git a/drivers/media/pci/ddbridge/ddbridge-i2c.c b/drivers/media/pci/ddbridge/ddbridge-i2c.c
index e4d39c3270ae..82a9a0e806fc 100644
--- a/drivers/media/pci/ddbridge/ddbridge-i2c.c
+++ b/drivers/media/pci/ddbridge/ddbridge-i2c.c
@@ -81,7 +81,7 @@ static int ddb_i2c_cmd(struct ddb_i2c *i2c, u32 adr, u32 cmd)
 static int ddb_i2c_master_xfer(struct i2c_adapter *adapter,
 			       struct i2c_msg msg[], int num)
 {
-	struct ddb_i2c *i2c = (struct ddb_i2c *) i2c_get_adapdata(adapter);
+	struct ddb_i2c *i2c = (struct ddb_i2c *)i2c_get_adapdata(adapter);
 	struct ddb *dev = i2c->dev;
 	u8 addr = 0;
 
@@ -149,7 +149,7 @@ void ddb_i2c_release(struct ddb *dev)
 
 static void i2c_handler(unsigned long priv)
 {
-	struct ddb_i2c *i2c = (struct ddb_i2c *) priv;
+	struct ddb_i2c *i2c = (struct ddb_i2c *)priv;
 
 	complete(&i2c->completion);
 }
@@ -171,20 +171,20 @@ static int ddb_i2c_add(struct ddb *dev, struct ddb_i2c *i2c,
 		(regmap->i2c->base + regmap->i2c->size * i);
 	ddbwritel(dev, I2C_SPEED_100, i2c->regs + I2C_TIMING);
 	ddbwritel(dev, ((i2c->rbuf & 0xffff) << 16) | (i2c->wbuf & 0xffff),
-		i2c->regs + I2C_TASKADDRESS);
+		  i2c->regs + I2C_TASKADDRESS);
 	init_completion(&i2c->completion);
 
 	adap = &i2c->adap;
 	i2c_set_adapdata(adap, i2c);
 #ifdef I2C_ADAP_CLASS_TV_DIGITAL
-	adap->class = I2C_ADAP_CLASS_TV_DIGITAL|I2C_CLASS_TV_ANALOG;
+	adap->class = I2C_ADAP_CLASS_TV_DIGITAL | I2C_CLASS_TV_ANALOG;
 #else
 #ifdef I2C_CLASS_TV_ANALOG
 	adap->class = I2C_CLASS_TV_ANALOG;
 #endif
 #endif
 	snprintf(adap->name, I2C_NAME_SIZE, "ddbridge_%02x.%x.%x",
-		dev->nr, i2c->link, i);
+		 dev->nr, i2c->link, i);
 	adap->algo = &ddb_i2c_algo;
 	adap->algo_data = (void *)i2c;
 	adap->dev.parent = dev->dev;
@@ -210,7 +210,7 @@ int ddb_i2c_init(struct ddb *dev)
 			if (!(dev->link[l].info->i2c_mask & (1 << i)))
 				continue;
 			i2c = &dev->i2c[num];
-			dev->handler_data[l][i + base] = (unsigned long) i2c;
+			dev->handler_data[l][i + base] = (unsigned long)i2c;
 			dev->handler[l][i + base] = i2c_handler;
 			stat = ddb_i2c_add(dev, i2c, regmap, l, i, num);
 			if (stat)
@@ -224,7 +224,9 @@ int ddb_i2c_init(struct ddb *dev)
 			adap = &i2c->adap;
 			i2c_del_adapter(adap);
 		}
-	} else
+	} else {
 		dev->i2c_num = num;
+	}
+
 	return stat;
 }
diff --git a/drivers/media/pci/ddbridge/ddbridge-main.c b/drivers/media/pci/ddbridge/ddbridge-main.c
index ccac7fe31336..26497d6b1395 100644
--- a/drivers/media/pci/ddbridge/ddbridge-main.c
+++ b/drivers/media/pci/ddbridge/ddbridge-main.c
@@ -107,7 +107,7 @@ static void ddb_irq_exit(struct ddb *dev)
 
 static void ddb_remove(struct pci_dev *pdev)
 {
-	struct ddb *dev = (struct ddb *) pci_get_drvdata(pdev);
+	struct ddb *dev = (struct ddb *)pci_get_drvdata(pdev);
 
 	ddb_device_destroy(dev);
 	ddb_ports_detach(dev);
@@ -132,9 +132,10 @@ static void ddb_irq_msi(struct ddb *dev, int nr)
 		if (stat >= 1) {
 			dev->msi = stat;
 			dev_info(dev->dev, "using %d MSI interrupt(s)\n",
-				dev->msi);
-		} else
+				 dev->msi);
+		} else {
 			dev_info(dev->dev, "MSI not available.\n");
+		}
 	}
 }
 #endif
@@ -160,11 +161,11 @@ static int ddb_irq_init(struct ddb *dev)
 		irq_flag = 0;
 	if (dev->msi == 2) {
 		stat = request_irq(dev->pdev->irq, ddb_irq_handler0,
-				   irq_flag, "ddbridge", (void *) dev);
+				   irq_flag, "ddbridge", (void *)dev);
 		if (stat < 0)
 			return stat;
 		stat = request_irq(dev->pdev->irq + 1, ddb_irq_handler1,
-				   irq_flag, "ddbridge", (void *) dev);
+				   irq_flag, "ddbridge", (void *)dev);
 		if (stat < 0) {
 			free_irq(dev->pdev->irq, dev);
 			return stat;
@@ -173,7 +174,7 @@ static int ddb_irq_init(struct ddb *dev)
 #endif
 	{
 		stat = request_irq(dev->pdev->irq, ddb_irq_handler,
-				   irq_flag, "ddbridge", (void *) dev);
+				   irq_flag, "ddbridge", (void *)dev);
 		if (stat < 0)
 			return stat;
 	}
@@ -188,7 +189,7 @@ static int ddb_irq_init(struct ddb *dev)
 }
 
 static int ddb_probe(struct pci_dev *pdev,
-			       const struct pci_device_id *id)
+		     const struct pci_device_id *id)
 {
 	struct ddb *dev;
 	int stat = 0;
@@ -202,8 +203,8 @@ static int ddb_probe(struct pci_dev *pdev,
 		if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))
 			return -ENODEV;
 
-	dev = vzalloc(sizeof(struct ddb));
-	if (dev == NULL)
+	dev = vzalloc(sizeof(*dev));
+	if (!dev)
 		return -ENOMEM;
 
 	mutex_init(&dev->mutex);
@@ -242,7 +243,7 @@ static int ddb_probe(struct pci_dev *pdev,
 	dev->link[0].ids.regmapid = ddbreadl(dev, 4);
 
 	dev_info(&pdev->dev, "HW %08x REGMAP %08x\n",
-		dev->link[0].ids.hwid, dev->link[0].ids.regmapid);
+		 dev->link[0].ids.hwid, dev->link[0].ids.regmapid);
 
 	ddbwritel(dev, 0, DMA_BASE_READ);
 	ddbwritel(dev, 0, DMA_BASE_WRITE);
@@ -317,7 +318,7 @@ static __init int module_init_ddbridge(void)
 	if (ddb_class_create() < 0)
 		return -1;
 	ddb_wq = create_workqueue("ddbridge");
-	if (ddb_wq == NULL)
+	if (!ddb_wq)
 		goto exit1;
 	stat = pci_register_driver(&ddb_pci_driver);
 	if (stat < 0)
diff --git a/drivers/media/pci/ddbridge/ddbridge-maxs8.c b/drivers/media/pci/ddbridge/ddbridge-max.c
similarity index 92%
rename from drivers/media/pci/ddbridge/ddbridge-maxs8.c
rename to drivers/media/pci/ddbridge/ddbridge-max.c
index f8a53bc7c86c..dc6b81488746 100644
--- a/drivers/media/pci/ddbridge/ddbridge-maxs8.c
+++ b/drivers/media/pci/ddbridge/ddbridge-max.c
@@ -1,5 +1,5 @@
 /*
- * ddbridge-maxs8.c: Digital Devices bridge MaxS4/8 support
+ * ddbridge-max.c: Digital Devices bridge MAX card support
  *
  * Copyright (C) 2010-2017 Digital Devices GmbH
  *                         Ralph Metzler <rjkm@metzlerbros.de>
@@ -34,7 +34,7 @@
 #include "ddbridge-regs.h"
 #include "ddbridge-io.h"
 
-#include "ddbridge-maxs8.h"
+#include "ddbridge-max.h"
 #include "mxl5xx.h"
 
 /******************************************************************************/
@@ -68,7 +68,7 @@ static int lnb_command(struct ddb *dev, u32 link, u32 lnb, u32 cmd)
 	}
 	if (c == 10)
 		dev_info(dev->dev, "%s lnb = %08x  cmd = %08x\n",
-			__func__, lnb, cmd);
+			 __func__, lnb, cmd);
 	return 0;
 }
 
@@ -123,7 +123,7 @@ static int lnb_set_sat(struct ddb *dev, u32 link, u32 input, u32 sat, u32 band,
 }
 
 static int lnb_set_tone(struct ddb *dev, u32 link, u32 input,
-	enum fe_sec_tone_mode tone)
+			enum fe_sec_tone_mode tone)
 {
 	int s = 0;
 	u32 mask = (1ULL << input);
@@ -149,7 +149,7 @@ static int lnb_set_tone(struct ddb *dev, u32 link, u32 input,
 }
 
 static int lnb_set_voltage(struct ddb *dev, u32 link, u32 input,
-	enum fe_sec_voltage voltage)
+			   enum fe_sec_voltage voltage)
 {
 	int s = 0;
 
@@ -291,34 +291,45 @@ static int max_set_voltage(struct dvb_frontend *fe, enum fe_sec_voltage voltage)
 
 		if (nv != ov) {
 			if (nv) {
-				lnb_set_voltage(dev,
-					port->lnr, 0, SEC_VOLTAGE_13);
+				lnb_set_voltage(
+					dev, port->lnr,
+					0, SEC_VOLTAGE_13);
 				if (fmode == 1) {
-					lnb_set_voltage(dev, port->lnr,
+					lnb_set_voltage(
+						dev, port->lnr,
 						0, SEC_VOLTAGE_13);
 					if (old_quattro) {
-						lnb_set_voltage(dev, port->lnr,
+						lnb_set_voltage(
+							dev, port->lnr,
 							1, SEC_VOLTAGE_18);
-						lnb_set_voltage(dev, port->lnr,
+						lnb_set_voltage(
+							dev, port->lnr,
 							2, SEC_VOLTAGE_13);
 					} else {
-						lnb_set_voltage(dev, port->lnr,
+						lnb_set_voltage(
+							dev, port->lnr,
 							1, SEC_VOLTAGE_13);
-						lnb_set_voltage(dev, port->lnr,
+						lnb_set_voltage(
+							dev, port->lnr,
 							2, SEC_VOLTAGE_18);
 					}
-					lnb_set_voltage(dev, port->lnr,
+					lnb_set_voltage(
+						dev, port->lnr,
 						3, SEC_VOLTAGE_18);
 				}
 			} else {
-				lnb_set_voltage(dev, port->lnr,
+				lnb_set_voltage(
+					dev, port->lnr,
 					0, SEC_VOLTAGE_OFF);
 				if (fmode == 1) {
-					lnb_set_voltage(dev, port->lnr,
+					lnb_set_voltage(
+						dev, port->lnr,
 						1, SEC_VOLTAGE_OFF);
-					lnb_set_voltage(dev, port->lnr,
+					lnb_set_voltage(
+						dev, port->lnr,
 						2, SEC_VOLTAGE_OFF);
-					lnb_set_voltage(dev, port->lnr,
+					lnb_set_voltage(
+						dev, port->lnr,
 						3, SEC_VOLTAGE_OFF);
 				}
 			}
@@ -331,7 +342,6 @@ static int max_set_voltage(struct dvb_frontend *fe, enum fe_sec_voltage voltage)
 
 static int max_enable_high_lnb_voltage(struct dvb_frontend *fe, long arg)
 {
-
 	return 0;
 }
 
@@ -350,7 +360,7 @@ static int mxl_fw_read(void *priv, u8 *buf, u32 len)
 	return ddbridge_flashread(dev, link->nr, buf, 0xc0000, len);
 }
 
-int lnb_init_fmode(struct ddb *dev, struct ddb_link *link, u32 fm)
+int ddb_lnb_init_fmode(struct ddb *dev, struct ddb_link *link, u32 fm)
 {
 	u32 l = link->nr;
 
@@ -394,7 +404,7 @@ static struct mxl5xx_cfg mxl5xx = {
 	.fw_read  = mxl_fw_read,
 };
 
-int fe_attach_mxl5xx(struct ddb_input *input)
+int ddb_fe_attach_mxl5xx(struct ddb_input *input)
 {
 	struct ddb *dev = input->port->dev;
 	struct i2c_adapter *i2c = &input->port->i2c->adap;
@@ -414,7 +424,7 @@ int fe_attach_mxl5xx(struct ddb_input *input)
 		tuner = 0;
 
 	dvb->fe = dvb_attach(mxl5xx_attach, i2c, &cfg,
-		demod, tuner, &dvb->set_input);
+			     demod, tuner, &dvb->set_input);
 
 	if (!dvb->fe) {
 		dev_err(dev->dev, "No MXL5XX found!\n");
@@ -430,7 +440,7 @@ int fe_attach_mxl5xx(struct ddb_input *input)
 		lnb_command(dev, port->lnr, input->nr, LNB_CMD_INIT);
 		lnb_set_voltage(dev, port->lnr, input->nr, SEC_VOLTAGE_OFF);
 	}
-	lnb_init_fmode(dev, link, fmode);
+	ddb_lnb_init_fmode(dev, link, fmode);
 
 	dvb->fe->ops.set_voltage = max_set_voltage;
 	dvb->fe->ops.enable_high_lnb_voltage = max_enable_high_lnb_voltage;
diff --git a/drivers/media/pci/ddbridge/ddbridge-maxs8.h b/drivers/media/pci/ddbridge/ddbridge-max.h
similarity index 73%
rename from drivers/media/pci/ddbridge/ddbridge-maxs8.h
rename to drivers/media/pci/ddbridge/ddbridge-max.h
index bb8884811a46..bf8bf38739f6 100644
--- a/drivers/media/pci/ddbridge/ddbridge-maxs8.h
+++ b/drivers/media/pci/ddbridge/ddbridge-max.h
@@ -1,5 +1,5 @@
 /*
- * ddbridge-maxs8.h: Digital Devices bridge MaxS4/8 support
+ * ddbridge-max.h: Digital Devices bridge MAX card support
  *
  * Copyright (C) 2010-2017 Digital Devices GmbH
  *                         Ralph Metzler <rjkm@metzlerbros.de>
@@ -16,14 +16,14 @@
  *
  */
 
-#ifndef _DDBRIDGE_MAXS8_H_
-#define _DDBRIDGE_MAXS8_H_
+#ifndef _DDBRIDGE_MAX_H_
+#define _DDBRIDGE_MAX_H_
 
 #include "ddbridge.h"
 
 /******************************************************************************/
 
-int lnb_init_fmode(struct ddb *dev, struct ddb_link *link, u32 fm);
-int fe_attach_mxl5xx(struct ddb_input *input);
+int ddb_lnb_init_fmode(struct ddb *dev, struct ddb_link *link, u32 fm);
+int ddb_fe_attach_mxl5xx(struct ddb_input *input);
 
-#endif /* _DDBRIDGE_MAXS8_H */
+#endif /* _DDBRIDGE_MAX_H */
diff --git a/drivers/media/pci/ddbridge/ddbridge-regs.h b/drivers/media/pci/ddbridge/ddbridge-regs.h
index 9d44f8d3af75..23d74ff83fe4 100644
--- a/drivers/media/pci/ddbridge/ddbridge-regs.h
+++ b/drivers/media/pci/ddbridge/ddbridge-regs.h
@@ -95,27 +95,27 @@
 #define DMA_BASE_WRITE        (0x100)
 #define DMA_BASE_READ         (0x140)
 
-#define TS_CONTROL(_io)         (_io->regs + 0x00)
-#define TS_CONTROL2(_io)        (_io->regs + 0x04)
+#define TS_CONTROL(_io)         ((_io)->regs + 0x00)
+#define TS_CONTROL2(_io)        ((_io)->regs + 0x04)
 
 /* ------------------------------------------------------------------------- */
 /* DMA  Buffer */
 
-#define DMA_BUFFER_CONTROL(_dma)       (_dma->regs + 0x00)
-#define DMA_BUFFER_ACK(_dma)           (_dma->regs + 0x04)
-#define DMA_BUFFER_CURRENT(_dma)       (_dma->regs + 0x08)
-#define DMA_BUFFER_SIZE(_dma)          (_dma->regs + 0x0c)
+#define DMA_BUFFER_CONTROL(_dma)       ((_dma)->regs + 0x00)
+#define DMA_BUFFER_ACK(_dma)           ((_dma)->regs + 0x04)
+#define DMA_BUFFER_CURRENT(_dma)       ((_dma)->regs + 0x08)
+#define DMA_BUFFER_SIZE(_dma)          ((_dma)->regs + 0x0c)
 
 /* ------------------------------------------------------------------------- */
 /* CI Interface (only CI-Bridge) */
 
-#define CI_BASE                     (0x400)
-#define CI_CONTROL(i)               (CI_BASE + (i) * 32 + 0x00)
+#define CI_BASE                         (0x400)
+#define CI_CONTROL(i)                   (CI_BASE + (i) * 32 + 0x00)
 
-#define CI_DO_ATTRIBUTE_RW(i)       (CI_BASE + (i) * 32 + 0x04)
-#define CI_DO_IO_RW(i)              (CI_BASE + (i) * 32 + 0x08)
-#define CI_READDATA(i)              (CI_BASE + (i) * 32 + 0x0c)
-#define CI_DO_READ_ATTRIBUTES(i)    (CI_BASE + (i) * 32 + 0x10)
+#define CI_DO_ATTRIBUTE_RW(i)           (CI_BASE + (i) * 32 + 0x04)
+#define CI_DO_IO_RW(i)                  (CI_BASE + (i) * 32 + 0x08)
+#define CI_READDATA(i)                  (CI_BASE + (i) * 32 + 0x0c)
+#define CI_DO_READ_ATTRIBUTES(i)        (CI_BASE + (i) * 32 + 0x10)
 
 #define CI_RESET_CAM                    (0x00000001)
 #define CI_POWER_ON                     (0x00000002)
@@ -132,7 +132,7 @@
 #define CI_BUFFER_BASE                  (0x3000)
 #define CI_BUFFER_SIZE                  (0x0800)
 
-#define CI_BUFFER(i)                  (CI_BUFFER_BASE + (i) * CI_BUFFER_SIZE)
+#define CI_BUFFER(i)                    (CI_BUFFER_BASE + (i) * CI_BUFFER_SIZE)
 
 /* ------------------------------------------------------------------------- */
 /* LNB commands (mxl5xx / Max S8) */
@@ -140,7 +140,7 @@
 #define LNB_BASE			(0x400)
 #define LNB_CONTROL(i)			(LNB_BASE + (i) * 0x20 + 0x00)
 
-#define LNB_CMD				(7ULL <<  0)
+#define LNB_CMD				(7ULL << 0)
 #define LNB_CMD_NOP			0
 #define LNB_CMD_INIT			1
 #define LNB_CMD_LOW			3
@@ -148,8 +148,8 @@
 #define LNB_CMD_OFF			5
 #define LNB_CMD_DISEQC			6
 
-#define LNB_BUSY			(1ULL <<  4)
-#define LNB_TONE			(1ULL << 15)
+#define LNB_BUSY			BIT_ULL(4)
+#define LNB_TONE			BIT_ULL(15)
 
 #define LNB_BUF_LEVEL(i)		(LNB_BASE + (i) * 0x20 + 0x10)
 #define LNB_BUF_WRITE(i)		(LNB_BASE + (i) * 0x20 + 0x14)
diff --git a/drivers/media/pci/ddbridge/ddbridge.h b/drivers/media/pci/ddbridge/ddbridge.h
index e9afa96bd9df..095457737bc1 100644
--- a/drivers/media/pci/ddbridge/ddbridge.h
+++ b/drivers/media/pci/ddbridge/ddbridge.h
@@ -55,15 +55,15 @@
 #include <linux/device.h>
 #include <linux/io.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_ringbuffer.h"
-#include "dvb_ca_en50221.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_ringbuffer.h>
+#include <media/dvb_ca_en50221.h>
+#include <media/dvb_net.h>
 
-#define DDBRIDGE_VERSION "0.9.31intermediate-integrated"
+#define DDBRIDGE_VERSION "0.9.32-integrated"
 
 #define DDB_MAX_I2C    32
 #define DDB_MAX_PORT   32
@@ -143,11 +143,11 @@ struct ddb_info {
 #define DMA_MAX_BUFS 32      /* hardware table limit */
 
 #define INPUT_DMA_BUFS 8
-#define INPUT_DMA_SIZE (128*47*21)
+#define INPUT_DMA_SIZE (128 * 47 * 21)
 #define INPUT_DMA_IRQ_DIV 1
 
 #define OUTPUT_DMA_BUFS 8
-#define OUTPUT_DMA_SIZE (128*47*21)
+#define OUTPUT_DMA_SIZE (128 * 47 * 21)
 #define OUTPUT_DMA_IRQ_DIV 1
 
 struct ddb;
@@ -166,7 +166,7 @@ struct ddb_dma {
 	u32                    bufval;
 
 	struct work_struct     work;
-	spinlock_t             lock;
+	spinlock_t             lock; /* DMA lock */
 	wait_queue_head_t      wq;
 	int                    running;
 	u32                    stat;
@@ -196,17 +196,16 @@ struct ddb_dvb {
 
 	int (*i2c_gate_ctrl)(struct dvb_frontend *, int);
 	int (*set_voltage)(struct dvb_frontend *fe,
-		enum fe_sec_voltage voltage);
+			   enum fe_sec_voltage voltage);
 	int (*set_input)(struct dvb_frontend *fe, int input);
 	int (*diseqc_send_master_cmd)(struct dvb_frontend *fe,
-		struct dvb_diseqc_master_cmd *cmd);
+				      struct dvb_diseqc_master_cmd *cmd);
 };
 
 struct ddb_ci {
 	struct dvb_ca_en50221  en;
 	struct ddb_port       *port;
 	u32                    nr;
-	struct mutex           lock;
 };
 
 struct ddb_io {
@@ -240,7 +239,7 @@ struct ddb_port {
 	u32                    regs;
 	u32                    lnr;
 	struct ddb_i2c        *i2c;
-	struct mutex           i2c_gate_lock;
+	struct mutex           i2c_gate_lock; /* I2C access lock */
 	u32                    class;
 #define DDB_PORT_NONE           0
 #define DDB_PORT_CI             1
@@ -297,7 +296,7 @@ struct ddb_port {
 #define TS_CAPTURE_LEN  (4096)
 
 struct ddb_lnb {
-	struct mutex           lock;
+	struct mutex           lock; /* lock lnb access */
 	u32                    tone;
 	enum fe_sec_voltage    oldvoltage[4];
 	u32                    voltage[4];
@@ -310,54 +309,54 @@ struct ddb_link {
 	const struct ddb_info *info;
 	u32                    nr;
 	u32                    regs;
-	spinlock_t             lock;
-	struct mutex           flash_mutex;
+	spinlock_t             lock; /* lock link access */
+	struct mutex           flash_mutex; /* lock flash access */
 	struct ddb_lnb         lnb;
 	struct tasklet_struct  tasklet;
 	struct ddb_ids         ids;
 
-	spinlock_t             temp_lock;
+	spinlock_t             temp_lock; /* lock temp chip access */
 	int                    overtemperature_error;
 	u8                     temp_tab[11];
 };
 
 struct ddb {
-	struct pci_dev        *pdev;
-	struct platform_device *pfdev;
-	struct device         *dev;
+	struct pci_dev          *pdev;
+	struct platform_device  *pfdev;
+	struct device           *dev;
 
-	int                    msi;
+	int                      msi;
 	struct workqueue_struct *wq;
-	u32                    has_dma;
-
-	struct ddb_link        link[DDB_MAX_LINK];
-	unsigned char __iomem *regs;
-	u32                    regs_len;
-	u32                    port_num;
-	struct ddb_port        port[DDB_MAX_PORT];
-	u32                    i2c_num;
-	struct ddb_i2c         i2c[DDB_MAX_I2C];
-	struct ddb_input       input[DDB_MAX_INPUT];
-	struct ddb_output      output[DDB_MAX_OUTPUT];
-	struct dvb_adapter     adap[DDB_MAX_INPUT];
-	struct ddb_dma         idma[DDB_MAX_INPUT];
-	struct ddb_dma         odma[DDB_MAX_OUTPUT];
-
-	void                   (*handler[4][256])(unsigned long);
-	unsigned long          handler_data[4][256];
-
-	struct device         *ddb_dev;
-	u32                    ddb_dev_users;
-	u32                    nr;
-	u8                     iobuf[1028];
-
-	u8                     leds;
-	u32                    ts_irq;
-	u32                    i2c_irq;
-
-	struct mutex           mutex;
-
-	u8                     tsbuf[TS_CAPTURE_LEN];
+	u32                      has_dma;
+
+	struct ddb_link          link[DDB_MAX_LINK];
+	unsigned char __iomem   *regs;
+	u32                      regs_len;
+	u32                      port_num;
+	struct ddb_port          port[DDB_MAX_PORT];
+	u32                      i2c_num;
+	struct ddb_i2c           i2c[DDB_MAX_I2C];
+	struct ddb_input         input[DDB_MAX_INPUT];
+	struct ddb_output        output[DDB_MAX_OUTPUT];
+	struct dvb_adapter       adap[DDB_MAX_INPUT];
+	struct ddb_dma           idma[DDB_MAX_INPUT];
+	struct ddb_dma           odma[DDB_MAX_OUTPUT];
+
+	void                     (*handler[4][256])(unsigned long);
+	unsigned long            handler_data[4][256];
+
+	struct device           *ddb_dev;
+	u32                      ddb_dev_users;
+	u32                      nr;
+	u8                       iobuf[1028];
+
+	u8                       leds;
+	u32                      ts_irq;
+	u32                      i2c_irq;
+
+	struct mutex             mutex; /* lock access to global ddb array */
+
+	u8                       tsbuf[TS_CAPTURE_LEN];
 };
 
 /****************************************************************************/
diff --git a/drivers/media/pci/dm1105/Makefile b/drivers/media/pci/dm1105/Makefile
index 327585143c83..d22c2547ee86 100644
--- a/drivers/media/pci/dm1105/Makefile
+++ b/drivers/media/pci/dm1105/Makefile
@@ -1,3 +1,3 @@
 obj-$(CONFIG_DVB_DM1105) += dm1105.o
 
-ccflags-y += -Idrivers/media/dvb-core/ -Idrivers/media/dvb-frontends
+ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c
index 7c3900dec368..c9db108751a7 100644
--- a/drivers/media/pci/dm1105/dm1105.c
+++ b/drivers/media/pci/dm1105/dm1105.c
@@ -27,12 +27,12 @@
 #include <linux/slab.h>
 #include <media/rc-core.h>
 
-#include "demux.h"
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
-#include "dvbdev.h"
+#include <media/demux.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
+#include <media/dvbdev.h>
 #include "dvb-pll.h"
 
 #include "stv0299.h"
diff --git a/drivers/media/pci/intel/Makefile b/drivers/media/pci/intel/Makefile
new file mode 100644
index 000000000000..745c8b2a7819
--- /dev/null
+++ b/drivers/media/pci/intel/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for the IPU3 cio2 and ImGU drivers
+#
+
+obj-y	+= ipu3/
diff --git a/drivers/media/pci/intel/ipu3/Kconfig b/drivers/media/pci/intel/ipu3/Kconfig
new file mode 100644
index 000000000000..a82d3fe277d2
--- /dev/null
+++ b/drivers/media/pci/intel/ipu3/Kconfig
@@ -0,0 +1,19 @@
+config VIDEO_IPU3_CIO2
+	tristate "Intel ipu3-cio2 driver"
+	depends on VIDEO_V4L2 && PCI
+	depends on VIDEO_V4L2_SUBDEV_API
+	depends on X86 || COMPILE_TEST
+	depends on MEDIA_CONTROLLER
+	depends on HAS_DMA
+	depends on ACPI
+	select V4L2_FWNODE
+	select VIDEOBUF2_DMA_SG
+
+	---help---
+	This is the Intel IPU3 CIO2 CSI-2 receiver unit, found in Intel
+	Skylake and Kaby Lake SoCs and used for capturing images and
+	video from a camera sensor.
+
+	Say Y or M here if you have a Skylake/Kaby Lake SoC with MIPI CSI-2
+	connected camera.
+	The module will be called ipu3-cio2.
diff --git a/drivers/media/pci/intel/ipu3/Makefile b/drivers/media/pci/intel/ipu3/Makefile
new file mode 100644
index 000000000000..20186e3ff2ae
--- /dev/null
+++ b/drivers/media/pci/intel/ipu3/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_VIDEO_IPU3_CIO2) += ipu3-cio2.o
diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2.c b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
new file mode 100644
index 000000000000..6c4444b31f4b
--- /dev/null
+++ b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
@@ -0,0 +1,2048 @@
+/*
+ * Copyright (c) 2017 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Based partially on Intel IPU4 driver written by
+ *  Sakari Ailus <sakari.ailus@linux.intel.com>
+ *  Samu Onkalo <samu.onkalo@intel.com>
+ *  Jouni Högander <jouni.hogander@intel.com>
+ *  Jouni Ukkonen <jouni.ukkonen@intel.com>
+ *  Antti Laakso <antti.laakso@intel.com>
+ * et al.
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/vmalloc.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-dma-sg.h>
+
+#include "ipu3-cio2.h"
+
+struct ipu3_cio2_fmt {
+	u32 mbus_code;
+	u32 fourcc;
+	u8 mipicode;
+};
+
+/*
+ * These are raw formats used in Intel's third generation of
+ * Image Processing Unit known as IPU3.
+ * 10bit raw bayer packed, 32 bytes for every 25 pixels,
+ * last LSB 6 bits unused.
+ */
+static const struct ipu3_cio2_fmt formats[] = {
+	{	/* put default entry at beginning */
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
+		.fourcc		= V4L2_PIX_FMT_IPU3_SGRBG10,
+		.mipicode	= 0x2b,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
+		.fourcc		= V4L2_PIX_FMT_IPU3_SGBRG10,
+		.mipicode	= 0x2b,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
+		.fourcc		= V4L2_PIX_FMT_IPU3_SBGGR10,
+		.mipicode	= 0x2b,
+	}, {
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
+		.fourcc		= V4L2_PIX_FMT_IPU3_SRGGB10,
+		.mipicode	= 0x2b,
+	},
+};
+
+/*
+ * cio2_find_format - lookup color format by fourcc or/and media bus code
+ * @pixelformat: fourcc to match, ignored if null
+ * @mbus_code: media bus code to match, ignored if null
+ */
+static const struct ipu3_cio2_fmt *cio2_find_format(const u32 *pixelformat,
+						    const u32 *mbus_code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(formats); i++) {
+		if (pixelformat && *pixelformat != formats[i].fourcc)
+			continue;
+		if (mbus_code && *mbus_code != formats[i].mbus_code)
+			continue;
+
+		return &formats[i];
+	}
+
+	return NULL;
+}
+
+static inline u32 cio2_bytesperline(const unsigned int width)
+{
+	/*
+	 * 64 bytes for every 50 pixels, the line length
+	 * in bytes is multiple of 64 (line end alignment).
+	 */
+	return DIV_ROUND_UP(width, 50) * 64;
+}
+
+/**************** FBPT operations ****************/
+
+static void cio2_fbpt_exit_dummy(struct cio2_device *cio2)
+{
+	if (cio2->dummy_lop) {
+		dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,
+				  cio2->dummy_lop, cio2->dummy_lop_bus_addr);
+		cio2->dummy_lop = NULL;
+	}
+	if (cio2->dummy_page) {
+		dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,
+				  cio2->dummy_page, cio2->dummy_page_bus_addr);
+		cio2->dummy_page = NULL;
+	}
+}
+
+static int cio2_fbpt_init_dummy(struct cio2_device *cio2)
+{
+	unsigned int i;
+
+	cio2->dummy_page = dma_alloc_coherent(&cio2->pci_dev->dev,
+					      CIO2_PAGE_SIZE,
+					      &cio2->dummy_page_bus_addr,
+					      GFP_KERNEL);
+	cio2->dummy_lop = dma_alloc_coherent(&cio2->pci_dev->dev,
+					     CIO2_PAGE_SIZE,
+					     &cio2->dummy_lop_bus_addr,
+					     GFP_KERNEL);
+	if (!cio2->dummy_page || !cio2->dummy_lop) {
+		cio2_fbpt_exit_dummy(cio2);
+		return -ENOMEM;
+	}
+	/*
+	 * List of Pointers(LOP) contains 1024x32b pointers to 4KB page each
+	 * Initialize each entry to dummy_page bus base address.
+	 */
+	for (i = 0; i < CIO2_PAGE_SIZE / sizeof(*cio2->dummy_lop); i++)
+		cio2->dummy_lop[i] = cio2->dummy_page_bus_addr >> PAGE_SHIFT;
+
+	return 0;
+}
+
+static void cio2_fbpt_entry_enable(struct cio2_device *cio2,
+				   struct cio2_fbpt_entry entry[CIO2_MAX_LOPS])
+{
+	/*
+	 * The CPU first initializes some fields in fbpt, then sets
+	 * the VALID bit, this barrier is to ensure that the DMA(device)
+	 * does not see the VALID bit enabled before other fields are
+	 * initialized; otherwise it could lead to havoc.
+	 */
+	dma_wmb();
+
+	/*
+	 * Request interrupts for start and completion
+	 * Valid bit is applicable only to 1st entry
+	 */
+	entry[0].first_entry.ctrl = CIO2_FBPT_CTRL_VALID |
+		CIO2_FBPT_CTRL_IOC | CIO2_FBPT_CTRL_IOS;
+}
+
+/* Initialize fpbt entries to point to dummy frame */
+static void cio2_fbpt_entry_init_dummy(struct cio2_device *cio2,
+				       struct cio2_fbpt_entry
+				       entry[CIO2_MAX_LOPS])
+{
+	unsigned int i;
+
+	entry[0].first_entry.first_page_offset = 0;
+	entry[1].second_entry.num_of_pages =
+		CIO2_PAGE_SIZE / sizeof(u32) * CIO2_MAX_LOPS;
+	entry[1].second_entry.last_page_available_bytes = CIO2_PAGE_SIZE - 1;
+
+	for (i = 0; i < CIO2_MAX_LOPS; i++)
+		entry[i].lop_page_addr = cio2->dummy_lop_bus_addr >> PAGE_SHIFT;
+
+	cio2_fbpt_entry_enable(cio2, entry);
+}
+
+/* Initialize fpbt entries to point to a given buffer */
+static void cio2_fbpt_entry_init_buf(struct cio2_device *cio2,
+				     struct cio2_buffer *b,
+				     struct cio2_fbpt_entry
+				     entry[CIO2_MAX_LOPS])
+{
+	struct vb2_buffer *vb = &b->vbb.vb2_buf;
+	unsigned int length = vb->planes[0].length;
+	int remaining, i;
+
+	entry[0].first_entry.first_page_offset = b->offset;
+	remaining = length + entry[0].first_entry.first_page_offset;
+	entry[1].second_entry.num_of_pages =
+		DIV_ROUND_UP(remaining, CIO2_PAGE_SIZE);
+	/*
+	 * last_page_available_bytes has the offset of the last byte in the
+	 * last page which is still accessible by DMA. DMA cannot access
+	 * beyond this point. Valid range for this is from 0 to 4095.
+	 * 0 indicates 1st byte in the page is DMA accessible.
+	 * 4095 (CIO2_PAGE_SIZE - 1) means every single byte in the last page
+	 * is available for DMA transfer.
+	 */
+	entry[1].second_entry.last_page_available_bytes =
+			(remaining & ~PAGE_MASK) ?
+				(remaining & ~PAGE_MASK) - 1 :
+				CIO2_PAGE_SIZE - 1;
+	/* Fill FBPT */
+	remaining = length;
+	i = 0;
+	while (remaining > 0) {
+		entry->lop_page_addr = b->lop_bus_addr[i] >> PAGE_SHIFT;
+		remaining -= CIO2_PAGE_SIZE / sizeof(u32) * CIO2_PAGE_SIZE;
+		entry++;
+		i++;
+	}
+
+	/*
+	 * The first not meaningful FBPT entry should point to a valid LOP
+	 */
+	entry->lop_page_addr = cio2->dummy_lop_bus_addr >> PAGE_SHIFT;
+
+	cio2_fbpt_entry_enable(cio2, entry);
+}
+
+static int cio2_fbpt_init(struct cio2_device *cio2, struct cio2_queue *q)
+{
+	struct device *dev = &cio2->pci_dev->dev;
+
+	q->fbpt = dma_alloc_coherent(dev, CIO2_FBPT_SIZE, &q->fbpt_bus_addr,
+				     GFP_KERNEL);
+	if (!q->fbpt)
+		return -ENOMEM;
+
+	memset(q->fbpt, 0, CIO2_FBPT_SIZE);
+
+	return 0;
+}
+
+static void cio2_fbpt_exit(struct cio2_queue *q, struct device *dev)
+{
+	dma_free_coherent(dev, CIO2_FBPT_SIZE, q->fbpt, q->fbpt_bus_addr);
+}
+
+/**************** CSI2 hardware setup ****************/
+
+/*
+ * The CSI2 receiver has several parameters affecting
+ * the receiver timings. These depend on the MIPI bus frequency
+ * F in Hz (sensor transmitter rate) as follows:
+ *     register value = (A/1e9 + B * UI) / COUNT_ACC
+ * where
+ *      UI = 1 / (2 * F) in seconds
+ *      COUNT_ACC = counter accuracy in seconds
+ *      For IPU3 COUNT_ACC = 0.0625
+ *
+ * A and B are coefficients from the table below,
+ * depending whether the register minimum or maximum value is
+ * calculated.
+ *                                     Minimum     Maximum
+ * Clock lane                          A     B     A     B
+ * reg_rx_csi_dly_cnt_termen_clane     0     0    38     0
+ * reg_rx_csi_dly_cnt_settle_clane    95    -8   300   -16
+ * Data lanes
+ * reg_rx_csi_dly_cnt_termen_dlane0    0     0    35     4
+ * reg_rx_csi_dly_cnt_settle_dlane0   85    -2   145    -6
+ * reg_rx_csi_dly_cnt_termen_dlane1    0     0    35     4
+ * reg_rx_csi_dly_cnt_settle_dlane1   85    -2   145    -6
+ * reg_rx_csi_dly_cnt_termen_dlane2    0     0    35     4
+ * reg_rx_csi_dly_cnt_settle_dlane2   85    -2   145    -6
+ * reg_rx_csi_dly_cnt_termen_dlane3    0     0    35     4
+ * reg_rx_csi_dly_cnt_settle_dlane3   85    -2   145    -6
+ *
+ * We use the minimum values of both A and B.
+ */
+
+/*
+ * shift for keeping value range suitable for 32-bit integer arithmetics
+ */
+#define LIMIT_SHIFT	8
+
+static s32 cio2_rx_timing(s32 a, s32 b, s64 freq, int def)
+{
+	const u32 accinv = 16; /* invert of counter resolution */
+	const u32 uiinv = 500000000; /* 1e9 / 2 */
+	s32 r;
+
+	freq >>= LIMIT_SHIFT;
+
+	if (WARN_ON(freq <= 0 || freq > S32_MAX))
+		return def;
+	/*
+	 * b could be 0, -2 or -8, so |accinv * b| is always
+	 * less than (1 << ds) and thus |r| < 500000000.
+	 */
+	r = accinv * b * (uiinv >> LIMIT_SHIFT);
+	r = r / (s32)freq;
+	/* max value of a is 95 */
+	r += accinv * a;
+
+	return r;
+};
+
+/* Calculate the the delay value for termination enable of clock lane HS Rx */
+static int cio2_csi2_calc_timing(struct cio2_device *cio2, struct cio2_queue *q,
+				 struct cio2_csi2_timing *timing)
+{
+	struct device *dev = &cio2->pci_dev->dev;
+	struct v4l2_querymenu qm = {.id = V4L2_CID_LINK_FREQ, };
+	struct v4l2_ctrl *link_freq;
+	s64 freq;
+	int r;
+
+	if (!q->sensor)
+		return -ENODEV;
+
+	link_freq = v4l2_ctrl_find(q->sensor->ctrl_handler, V4L2_CID_LINK_FREQ);
+	if (!link_freq) {
+		dev_err(dev, "failed to find LINK_FREQ\n");
+		return -EPIPE;
+	}
+
+	qm.index = v4l2_ctrl_g_ctrl(link_freq);
+	r = v4l2_querymenu(q->sensor->ctrl_handler, &qm);
+	if (r) {
+		dev_err(dev, "failed to get menu item\n");
+		return r;
+	}
+
+	if (!qm.value) {
+		dev_err(dev, "error invalid link_freq\n");
+		return -EINVAL;
+	}
+	freq = qm.value;
+
+	timing->clk_termen = cio2_rx_timing(CIO2_CSIRX_DLY_CNT_TERMEN_CLANE_A,
+					    CIO2_CSIRX_DLY_CNT_TERMEN_CLANE_B,
+					    freq,
+					    CIO2_CSIRX_DLY_CNT_TERMEN_DEFAULT);
+	timing->clk_settle = cio2_rx_timing(CIO2_CSIRX_DLY_CNT_SETTLE_CLANE_A,
+					    CIO2_CSIRX_DLY_CNT_SETTLE_CLANE_B,
+					    freq,
+					    CIO2_CSIRX_DLY_CNT_SETTLE_DEFAULT);
+	timing->dat_termen = cio2_rx_timing(CIO2_CSIRX_DLY_CNT_TERMEN_DLANE_A,
+					    CIO2_CSIRX_DLY_CNT_TERMEN_DLANE_B,
+					    freq,
+					    CIO2_CSIRX_DLY_CNT_TERMEN_DEFAULT);
+	timing->dat_settle = cio2_rx_timing(CIO2_CSIRX_DLY_CNT_SETTLE_DLANE_A,
+					    CIO2_CSIRX_DLY_CNT_SETTLE_DLANE_B,
+					    freq,
+					    CIO2_CSIRX_DLY_CNT_SETTLE_DEFAULT);
+
+	dev_dbg(dev, "freq ct value is %d\n", timing->clk_termen);
+	dev_dbg(dev, "freq cs value is %d\n", timing->clk_settle);
+	dev_dbg(dev, "freq dt value is %d\n", timing->dat_termen);
+	dev_dbg(dev, "freq ds value is %d\n", timing->dat_settle);
+
+	return 0;
+};
+
+static int cio2_hw_init(struct cio2_device *cio2, struct cio2_queue *q)
+{
+	static const int NUM_VCS = 4;
+	static const int SID;	/* Stream id */
+	static const int ENTRY;
+	static const int FBPT_WIDTH = DIV_ROUND_UP(CIO2_MAX_LOPS,
+					CIO2_FBPT_SUBENTRY_UNIT);
+	const u32 num_buffers1 = CIO2_MAX_BUFFERS - 1;
+	const struct ipu3_cio2_fmt *fmt;
+	void __iomem *const base = cio2->base;
+	u8 lanes, csi2bus = q->csi2.port;
+	u8 sensor_vc = SENSOR_VIR_CH_DFLT;
+	struct cio2_csi2_timing timing;
+	int i, r;
+
+	fmt = cio2_find_format(NULL, &q->subdev_fmt.code);
+	if (!fmt)
+		return -EINVAL;
+
+	lanes = q->csi2.lanes;
+
+	r = cio2_csi2_calc_timing(cio2, q, &timing);
+	if (r)
+		return r;
+
+	writel(timing.clk_termen, q->csi_rx_base +
+		CIO2_REG_CSIRX_DLY_CNT_TERMEN(CIO2_CSIRX_DLY_CNT_CLANE_IDX));
+	writel(timing.clk_settle, q->csi_rx_base +
+		CIO2_REG_CSIRX_DLY_CNT_SETTLE(CIO2_CSIRX_DLY_CNT_CLANE_IDX));
+
+	for (i = 0; i < lanes; i++) {
+		writel(timing.dat_termen, q->csi_rx_base +
+			CIO2_REG_CSIRX_DLY_CNT_TERMEN(i));
+		writel(timing.dat_settle, q->csi_rx_base +
+			CIO2_REG_CSIRX_DLY_CNT_SETTLE(i));
+	}
+
+	writel(CIO2_PBM_WMCTRL1_MIN_2CK |
+	       CIO2_PBM_WMCTRL1_MID1_2CK |
+	       CIO2_PBM_WMCTRL1_MID2_2CK, base + CIO2_REG_PBM_WMCTRL1);
+	writel(CIO2_PBM_WMCTRL2_HWM_2CK << CIO2_PBM_WMCTRL2_HWM_2CK_SHIFT |
+	       CIO2_PBM_WMCTRL2_LWM_2CK << CIO2_PBM_WMCTRL2_LWM_2CK_SHIFT |
+	       CIO2_PBM_WMCTRL2_OBFFWM_2CK <<
+	       CIO2_PBM_WMCTRL2_OBFFWM_2CK_SHIFT |
+	       CIO2_PBM_WMCTRL2_TRANSDYN << CIO2_PBM_WMCTRL2_TRANSDYN_SHIFT |
+	       CIO2_PBM_WMCTRL2_OBFF_MEM_EN, base + CIO2_REG_PBM_WMCTRL2);
+	writel(CIO2_PBM_ARB_CTRL_LANES_DIV <<
+	       CIO2_PBM_ARB_CTRL_LANES_DIV_SHIFT |
+	       CIO2_PBM_ARB_CTRL_LE_EN |
+	       CIO2_PBM_ARB_CTRL_PLL_POST_SHTDN <<
+	       CIO2_PBM_ARB_CTRL_PLL_POST_SHTDN_SHIFT |
+	       CIO2_PBM_ARB_CTRL_PLL_AHD_WK_UP <<
+	       CIO2_PBM_ARB_CTRL_PLL_AHD_WK_UP_SHIFT,
+	       base + CIO2_REG_PBM_ARB_CTRL);
+	writel(CIO2_CSIRX_STATUS_DLANE_HS_MASK,
+	       q->csi_rx_base + CIO2_REG_CSIRX_STATUS_DLANE_HS);
+	writel(CIO2_CSIRX_STATUS_DLANE_LP_MASK,
+	       q->csi_rx_base + CIO2_REG_CSIRX_STATUS_DLANE_LP);
+
+	writel(CIO2_FB_HPLL_FREQ, base + CIO2_REG_FB_HPLL_FREQ);
+	writel(CIO2_ISCLK_RATIO, base + CIO2_REG_ISCLK_RATIO);
+
+	/* Configure MIPI backend */
+	for (i = 0; i < NUM_VCS; i++)
+		writel(1, q->csi_rx_base + CIO2_REG_MIPIBE_SP_LUT_ENTRY(i));
+
+	/* There are 16 short packet LUT entry */
+	for (i = 0; i < 16; i++)
+		writel(CIO2_MIPIBE_LP_LUT_ENTRY_DISREGARD,
+		       q->csi_rx_base + CIO2_REG_MIPIBE_LP_LUT_ENTRY(i));
+	writel(CIO2_MIPIBE_GLOBAL_LUT_DISREGARD,
+	       q->csi_rx_base + CIO2_REG_MIPIBE_GLOBAL_LUT_DISREGARD);
+
+	writel(CIO2_INT_EN_EXT_IE_MASK, base + CIO2_REG_INT_EN_EXT_IE);
+	writel(CIO2_IRQCTRL_MASK, q->csi_rx_base + CIO2_REG_IRQCTRL_MASK);
+	writel(CIO2_IRQCTRL_MASK, q->csi_rx_base + CIO2_REG_IRQCTRL_ENABLE);
+	writel(0, q->csi_rx_base + CIO2_REG_IRQCTRL_EDGE);
+	writel(0, q->csi_rx_base + CIO2_REG_IRQCTRL_LEVEL_NOT_PULSE);
+	writel(CIO2_INT_EN_EXT_OE_MASK, base + CIO2_REG_INT_EN_EXT_OE);
+
+	writel(CIO2_REG_INT_EN_IRQ | CIO2_INT_IOC(CIO2_DMA_CHAN) |
+	       CIO2_REG_INT_EN_IOS(CIO2_DMA_CHAN),
+	       base + CIO2_REG_INT_EN);
+
+	writel((CIO2_PXM_PXF_FMT_CFG_BPP_10 | CIO2_PXM_PXF_FMT_CFG_PCK_64B)
+	       << CIO2_PXM_PXF_FMT_CFG_SID0_SHIFT,
+	       base + CIO2_REG_PXM_PXF_FMT_CFG0(csi2bus));
+	writel(SID << CIO2_MIPIBE_LP_LUT_ENTRY_SID_SHIFT |
+	       sensor_vc << CIO2_MIPIBE_LP_LUT_ENTRY_VC_SHIFT |
+	       fmt->mipicode << CIO2_MIPIBE_LP_LUT_ENTRY_FORMAT_TYPE_SHIFT,
+	       q->csi_rx_base + CIO2_REG_MIPIBE_LP_LUT_ENTRY(ENTRY));
+	writel(0, q->csi_rx_base + CIO2_REG_MIPIBE_COMP_FORMAT(sensor_vc));
+	writel(0, q->csi_rx_base + CIO2_REG_MIPIBE_FORCE_RAW8);
+	writel(0, base + CIO2_REG_PXM_SID2BID0(csi2bus));
+
+	writel(lanes, q->csi_rx_base + CIO2_REG_CSIRX_NOF_ENABLED_LANES);
+	writel(CIO2_CGC_PRIM_TGE |
+	       CIO2_CGC_SIDE_TGE |
+	       CIO2_CGC_XOSC_TGE |
+	       CIO2_CGC_D3I3_TGE |
+	       CIO2_CGC_CSI2_INTERFRAME_TGE |
+	       CIO2_CGC_CSI2_PORT_DCGE |
+	       CIO2_CGC_SIDE_DCGE |
+	       CIO2_CGC_PRIM_DCGE |
+	       CIO2_CGC_ROSC_DCGE |
+	       CIO2_CGC_XOSC_DCGE |
+	       CIO2_CGC_CLKGATE_HOLDOFF << CIO2_CGC_CLKGATE_HOLDOFF_SHIFT |
+	       CIO2_CGC_CSI_CLKGATE_HOLDOFF
+	       << CIO2_CGC_CSI_CLKGATE_HOLDOFF_SHIFT, base + CIO2_REG_CGC);
+	writel(CIO2_LTRCTRL_LTRDYNEN, base + CIO2_REG_LTRCTRL);
+	writel(CIO2_LTRVAL0_VAL << CIO2_LTRVAL02_VAL_SHIFT |
+	       CIO2_LTRVAL0_SCALE << CIO2_LTRVAL02_SCALE_SHIFT |
+	       CIO2_LTRVAL1_VAL << CIO2_LTRVAL13_VAL_SHIFT |
+	       CIO2_LTRVAL1_SCALE << CIO2_LTRVAL13_SCALE_SHIFT,
+	       base + CIO2_REG_LTRVAL01);
+	writel(CIO2_LTRVAL2_VAL << CIO2_LTRVAL02_VAL_SHIFT |
+	       CIO2_LTRVAL2_SCALE << CIO2_LTRVAL02_SCALE_SHIFT |
+	       CIO2_LTRVAL3_VAL << CIO2_LTRVAL13_VAL_SHIFT |
+	       CIO2_LTRVAL3_SCALE << CIO2_LTRVAL13_SCALE_SHIFT,
+	       base + CIO2_REG_LTRVAL23);
+
+	for (i = 0; i < CIO2_NUM_DMA_CHAN; i++) {
+		writel(0, base + CIO2_REG_CDMABA(i));
+		writel(0, base + CIO2_REG_CDMAC0(i));
+		writel(0, base + CIO2_REG_CDMAC1(i));
+	}
+
+	/* Enable DMA */
+	writel(q->fbpt_bus_addr >> PAGE_SHIFT,
+	       base + CIO2_REG_CDMABA(CIO2_DMA_CHAN));
+
+	writel(num_buffers1 << CIO2_CDMAC0_FBPT_LEN_SHIFT |
+	       FBPT_WIDTH << CIO2_CDMAC0_FBPT_WIDTH_SHIFT |
+	       CIO2_CDMAC0_DMA_INTR_ON_FE |
+	       CIO2_CDMAC0_FBPT_UPDATE_FIFO_FULL |
+	       CIO2_CDMAC0_DMA_EN |
+	       CIO2_CDMAC0_DMA_INTR_ON_FS |
+	       CIO2_CDMAC0_DMA_HALTED, base + CIO2_REG_CDMAC0(CIO2_DMA_CHAN));
+
+	writel(1 << CIO2_CDMAC1_LINENUMUPDATE_SHIFT,
+	       base + CIO2_REG_CDMAC1(CIO2_DMA_CHAN));
+
+	writel(0, base + CIO2_REG_PBM_FOPN_ABORT);
+
+	writel(CIO2_PXM_FRF_CFG_CRC_TH << CIO2_PXM_FRF_CFG_CRC_TH_SHIFT |
+	       CIO2_PXM_FRF_CFG_MSK_ECC_DPHY_NR |
+	       CIO2_PXM_FRF_CFG_MSK_ECC_RE |
+	       CIO2_PXM_FRF_CFG_MSK_ECC_DPHY_NE,
+	       base + CIO2_REG_PXM_FRF_CFG(q->csi2.port));
+
+	/* Clear interrupts */
+	writel(CIO2_IRQCTRL_MASK, q->csi_rx_base + CIO2_REG_IRQCTRL_CLEAR);
+	writel(~0, base + CIO2_REG_INT_STS_EXT_OE);
+	writel(~0, base + CIO2_REG_INT_STS_EXT_IE);
+	writel(~0, base + CIO2_REG_INT_STS);
+
+	/* Enable devices, starting from the last device in the pipe */
+	writel(1, q->csi_rx_base + CIO2_REG_MIPIBE_ENABLE);
+	writel(1, q->csi_rx_base + CIO2_REG_CSIRX_ENABLE);
+
+	return 0;
+}
+
+static void cio2_hw_exit(struct cio2_device *cio2, struct cio2_queue *q)
+{
+	void __iomem *base = cio2->base;
+	unsigned int i, maxloops = 1000;
+
+	/* Disable CSI receiver and MIPI backend devices */
+	writel(0, q->csi_rx_base + CIO2_REG_CSIRX_ENABLE);
+	writel(0, q->csi_rx_base + CIO2_REG_MIPIBE_ENABLE);
+
+	/* Halt DMA */
+	writel(0, base + CIO2_REG_CDMAC0(CIO2_DMA_CHAN));
+	do {
+		if (readl(base + CIO2_REG_CDMAC0(CIO2_DMA_CHAN)) &
+		    CIO2_CDMAC0_DMA_HALTED)
+			break;
+		usleep_range(1000, 2000);
+	} while (--maxloops);
+	if (!maxloops)
+		dev_err(&cio2->pci_dev->dev,
+			"DMA %i can not be halted\n", CIO2_DMA_CHAN);
+
+	for (i = 0; i < CIO2_NUM_PORTS; i++) {
+		writel(readl(base + CIO2_REG_PXM_FRF_CFG(i)) |
+		       CIO2_PXM_FRF_CFG_ABORT, base + CIO2_REG_PXM_FRF_CFG(i));
+		writel(readl(base + CIO2_REG_PBM_FOPN_ABORT) |
+		       CIO2_PBM_FOPN_ABORT(i), base + CIO2_REG_PBM_FOPN_ABORT);
+	}
+}
+
+static void cio2_buffer_done(struct cio2_device *cio2, unsigned int dma_chan)
+{
+	struct device *dev = &cio2->pci_dev->dev;
+	struct cio2_queue *q = cio2->cur_queue;
+	int buffers_found = 0;
+	u64 ns = ktime_get_ns();
+
+	if (dma_chan >= CIO2_QUEUES) {
+		dev_err(dev, "bad DMA channel %i\n", dma_chan);
+		return;
+	}
+
+	/* Find out which buffer(s) are ready */
+	do {
+		struct cio2_fbpt_entry *const entry =
+			&q->fbpt[q->bufs_first * CIO2_MAX_LOPS];
+		struct cio2_buffer *b;
+
+		if (entry->first_entry.ctrl & CIO2_FBPT_CTRL_VALID)
+			break;
+
+		b = q->bufs[q->bufs_first];
+		if (b) {
+			unsigned int bytes = entry[1].second_entry.num_of_bytes;
+
+			q->bufs[q->bufs_first] = NULL;
+			atomic_dec(&q->bufs_queued);
+			dev_dbg(&cio2->pci_dev->dev,
+				"buffer %i done\n", b->vbb.vb2_buf.index);
+
+			b->vbb.vb2_buf.timestamp = ns;
+			b->vbb.field = V4L2_FIELD_NONE;
+			b->vbb.sequence = atomic_read(&q->frame_sequence);
+			if (b->vbb.vb2_buf.planes[0].length != bytes)
+				dev_warn(dev, "buffer length is %d received %d\n",
+					 b->vbb.vb2_buf.planes[0].length,
+					 bytes);
+			vb2_buffer_done(&b->vbb.vb2_buf, VB2_BUF_STATE_DONE);
+		}
+		atomic_inc(&q->frame_sequence);
+		cio2_fbpt_entry_init_dummy(cio2, entry);
+		q->bufs_first = (q->bufs_first + 1) % CIO2_MAX_BUFFERS;
+		buffers_found++;
+	} while (1);
+
+	if (buffers_found == 0)
+		dev_warn(&cio2->pci_dev->dev,
+			 "no ready buffers found on DMA channel %u\n",
+			 dma_chan);
+}
+
+static void cio2_queue_event_sof(struct cio2_device *cio2, struct cio2_queue *q)
+{
+	/*
+	 * For the user space camera control algorithms it is essential
+	 * to know when the reception of a frame has begun. That's often
+	 * the best timing information to get from the hardware.
+	 */
+	struct v4l2_event event = {
+		.type = V4L2_EVENT_FRAME_SYNC,
+		.u.frame_sync.frame_sequence = atomic_read(&q->frame_sequence),
+	};
+
+	v4l2_event_queue(q->subdev.devnode, &event);
+}
+
+static const char *const cio2_irq_errs[] = {
+	"single packet header error corrected",
+	"multiple packet header errors detected",
+	"payload checksum (CRC) error",
+	"fifo overflow",
+	"reserved short packet data type detected",
+	"reserved long packet data type detected",
+	"incomplete long packet detected",
+	"frame sync error",
+	"line sync error",
+	"DPHY start of transmission error",
+	"DPHY synchronization error",
+	"escape mode error",
+	"escape mode trigger event",
+	"escape mode ultra-low power state for data lane(s)",
+	"escape mode ultra-low power state exit for clock lane",
+	"inter-frame short packet discarded",
+	"inter-frame long packet discarded",
+	"non-matching Long Packet stalled",
+};
+
+static const char *const cio2_port_errs[] = {
+	"ECC recoverable",
+	"DPHY not recoverable",
+	"ECC not recoverable",
+	"CRC error",
+	"INTERFRAMEDATA",
+	"PKT2SHORT",
+	"PKT2LONG",
+};
+
+static irqreturn_t cio2_irq(int irq, void *cio2_ptr)
+{
+	struct cio2_device *cio2 = cio2_ptr;
+	void __iomem *const base = cio2->base;
+	struct device *dev = &cio2->pci_dev->dev;
+	u32 int_status, int_clear;
+
+	int_status = readl(base + CIO2_REG_INT_STS);
+	int_clear = int_status;
+
+	if (!int_status)
+		return IRQ_NONE;
+
+	if (int_status & CIO2_INT_IOOE) {
+		/*
+		 * Interrupt on Output Error:
+		 * 1) SRAM is full and FS received, or
+		 * 2) An invalid bit detected by DMA.
+		 */
+		u32 oe_status, oe_clear;
+
+		oe_clear = readl(base + CIO2_REG_INT_STS_EXT_OE);
+		oe_status = oe_clear;
+
+		if (oe_status & CIO2_INT_EXT_OE_DMAOE_MASK) {
+			dev_err(dev, "DMA output error: 0x%x\n",
+				(oe_status & CIO2_INT_EXT_OE_DMAOE_MASK)
+				>> CIO2_INT_EXT_OE_DMAOE_SHIFT);
+			oe_status &= ~CIO2_INT_EXT_OE_DMAOE_MASK;
+		}
+		if (oe_status & CIO2_INT_EXT_OE_OES_MASK) {
+			dev_err(dev, "DMA output error on CSI2 buses: 0x%x\n",
+				(oe_status & CIO2_INT_EXT_OE_OES_MASK)
+				>> CIO2_INT_EXT_OE_OES_SHIFT);
+			oe_status &= ~CIO2_INT_EXT_OE_OES_MASK;
+		}
+		writel(oe_clear, base + CIO2_REG_INT_STS_EXT_OE);
+		if (oe_status)
+			dev_warn(dev, "unknown interrupt 0x%x on OE\n",
+				 oe_status);
+		int_status &= ~CIO2_INT_IOOE;
+	}
+
+	if (int_status & CIO2_INT_IOC_MASK) {
+		/* DMA IO done -- frame ready */
+		u32 clr = 0;
+		unsigned int d;
+
+		for (d = 0; d < CIO2_NUM_DMA_CHAN; d++)
+			if (int_status & CIO2_INT_IOC(d)) {
+				clr |= CIO2_INT_IOC(d);
+				cio2_buffer_done(cio2, d);
+			}
+		int_status &= ~clr;
+	}
+
+	if (int_status & CIO2_INT_IOS_IOLN_MASK) {
+		/* DMA IO starts or reached specified line */
+		u32 clr = 0;
+		unsigned int d;
+
+		for (d = 0; d < CIO2_NUM_DMA_CHAN; d++)
+			if (int_status & CIO2_INT_IOS_IOLN(d)) {
+				clr |= CIO2_INT_IOS_IOLN(d);
+				if (d == CIO2_DMA_CHAN)
+					cio2_queue_event_sof(cio2,
+							     cio2->cur_queue);
+			}
+		int_status &= ~clr;
+	}
+
+	if (int_status & (CIO2_INT_IOIE | CIO2_INT_IOIRQ)) {
+		/* CSI2 receiver (error) interrupt */
+		u32 ie_status, ie_clear;
+		unsigned int port;
+
+		ie_clear = readl(base + CIO2_REG_INT_STS_EXT_IE);
+		ie_status = ie_clear;
+
+		for (port = 0; port < CIO2_NUM_PORTS; port++) {
+			u32 port_status = (ie_status >> (port * 8)) & 0xff;
+			u32 err_mask = BIT_MASK(ARRAY_SIZE(cio2_port_errs)) - 1;
+			void __iomem *const csi_rx_base =
+						base + CIO2_REG_PIPE_BASE(port);
+			unsigned int i;
+
+			while (port_status & err_mask) {
+				i = ffs(port_status) - 1;
+				dev_err(dev, "port %i error %s\n",
+					port, cio2_port_errs[i]);
+				ie_status &= ~BIT(port * 8 + i);
+				port_status &= ~BIT(i);
+			}
+
+			if (ie_status & CIO2_INT_EXT_IE_IRQ(port)) {
+				u32 csi2_status, csi2_clear;
+
+				csi2_status = readl(csi_rx_base +
+						CIO2_REG_IRQCTRL_STATUS);
+				csi2_clear = csi2_status;
+				err_mask =
+					BIT_MASK(ARRAY_SIZE(cio2_irq_errs)) - 1;
+
+				while (csi2_status & err_mask) {
+					i = ffs(csi2_status) - 1;
+					dev_err(dev,
+						"CSI-2 receiver port %i: %s\n",
+							port, cio2_irq_errs[i]);
+					csi2_status &= ~BIT(i);
+				}
+
+				writel(csi2_clear,
+				       csi_rx_base + CIO2_REG_IRQCTRL_CLEAR);
+				if (csi2_status)
+					dev_warn(dev,
+						 "unknown CSI2 error 0x%x on port %i\n",
+						 csi2_status, port);
+
+				ie_status &= ~CIO2_INT_EXT_IE_IRQ(port);
+			}
+		}
+
+		writel(ie_clear, base + CIO2_REG_INT_STS_EXT_IE);
+		if (ie_status)
+			dev_warn(dev, "unknown interrupt 0x%x on IE\n",
+				 ie_status);
+
+		int_status &= ~(CIO2_INT_IOIE | CIO2_INT_IOIRQ);
+	}
+
+	writel(int_clear, base + CIO2_REG_INT_STS);
+	if (int_status)
+		dev_warn(dev, "unknown interrupt 0x%x on INT\n", int_status);
+
+	return IRQ_HANDLED;
+}
+
+/**************** Videobuf2 interface ****************/
+
+static void cio2_vb2_return_all_buffers(struct cio2_queue *q,
+					enum vb2_buffer_state state)
+{
+	unsigned int i;
+
+	for (i = 0; i < CIO2_MAX_BUFFERS; i++) {
+		if (q->bufs[i]) {
+			atomic_dec(&q->bufs_queued);
+			vb2_buffer_done(&q->bufs[i]->vbb.vb2_buf,
+					state);
+		}
+	}
+}
+
+static int cio2_vb2_queue_setup(struct vb2_queue *vq,
+				unsigned int *num_buffers,
+				unsigned int *num_planes,
+				unsigned int sizes[],
+				struct device *alloc_devs[])
+{
+	struct cio2_device *cio2 = vb2_get_drv_priv(vq);
+	struct cio2_queue *q = vb2q_to_cio2_queue(vq);
+	unsigned int i;
+
+	*num_planes = q->format.num_planes;
+
+	for (i = 0; i < *num_planes; ++i) {
+		sizes[i] = q->format.plane_fmt[i].sizeimage;
+		alloc_devs[i] = &cio2->pci_dev->dev;
+	}
+
+	*num_buffers = clamp_val(*num_buffers, 1, CIO2_MAX_BUFFERS);
+
+	/* Initialize buffer queue */
+	for (i = 0; i < CIO2_MAX_BUFFERS; i++) {
+		q->bufs[i] = NULL;
+		cio2_fbpt_entry_init_dummy(cio2, &q->fbpt[i * CIO2_MAX_LOPS]);
+	}
+	atomic_set(&q->bufs_queued, 0);
+	q->bufs_first = 0;
+	q->bufs_next = 0;
+
+	return 0;
+}
+
+/* Called after each buffer is allocated */
+static int cio2_vb2_buf_init(struct vb2_buffer *vb)
+{
+	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);
+	struct device *dev = &cio2->pci_dev->dev;
+	struct cio2_buffer *b =
+		container_of(vb, struct cio2_buffer, vbb.vb2_buf);
+	static const unsigned int entries_per_page =
+		CIO2_PAGE_SIZE / sizeof(u32);
+	unsigned int pages = DIV_ROUND_UP(vb->planes[0].length, CIO2_PAGE_SIZE);
+	unsigned int lops = DIV_ROUND_UP(pages + 1, entries_per_page);
+	struct sg_table *sg;
+	struct sg_page_iter sg_iter;
+	int i, j;
+
+	if (lops <= 0 || lops > CIO2_MAX_LOPS) {
+		dev_err(dev, "%s: bad buffer size (%i)\n", __func__,
+			vb->planes[0].length);
+		return -ENOSPC;		/* Should never happen */
+	}
+
+	memset(b->lop, 0, sizeof(b->lop));
+	/* Allocate LOP table */
+	for (i = 0; i < lops; i++) {
+		b->lop[i] = dma_alloc_coherent(dev, CIO2_PAGE_SIZE,
+					       &b->lop_bus_addr[i], GFP_KERNEL);
+		if (!b->lop[i])
+			goto fail;
+	}
+
+	/* Fill LOP */
+	sg = vb2_dma_sg_plane_desc(vb, 0);
+	if (!sg)
+		return -ENOMEM;
+
+	if (sg->nents && sg->sgl)
+		b->offset = sg->sgl->offset;
+
+	i = j = 0;
+	for_each_sg_page(sg->sgl, &sg_iter, sg->nents, 0) {
+		if (!pages--)
+			break;
+		b->lop[i][j] = sg_page_iter_dma_address(&sg_iter) >> PAGE_SHIFT;
+		j++;
+		if (j == entries_per_page) {
+			i++;
+			j = 0;
+		}
+	}
+
+	b->lop[i][j] = cio2->dummy_page_bus_addr >> PAGE_SHIFT;
+	return 0;
+fail:
+	for (i--; i >= 0; i--)
+		dma_free_coherent(dev, CIO2_PAGE_SIZE,
+				  b->lop[i], b->lop_bus_addr[i]);
+	return -ENOMEM;
+}
+
+/* Transfer buffer ownership to cio2 */
+static void cio2_vb2_buf_queue(struct vb2_buffer *vb)
+{
+	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);
+	struct cio2_queue *q =
+		container_of(vb->vb2_queue, struct cio2_queue, vbq);
+	struct cio2_buffer *b =
+		container_of(vb, struct cio2_buffer, vbb.vb2_buf);
+	struct cio2_fbpt_entry *entry;
+	unsigned long flags;
+	unsigned int i, j, next = q->bufs_next;
+	int bufs_queued = atomic_inc_return(&q->bufs_queued);
+	u32 fbpt_rp;
+
+	dev_dbg(&cio2->pci_dev->dev, "queue buffer %d\n", vb->index);
+
+	/*
+	 * This code queues the buffer to the CIO2 DMA engine, which starts
+	 * running once streaming has started. It is possible that this code
+	 * gets pre-empted due to increased CPU load. Upon this, the driver
+	 * does not get an opportunity to queue new buffers to the CIO2 DMA
+	 * engine. When the DMA engine encounters an FBPT entry without the
+	 * VALID bit set, the DMA engine halts, which requires a restart of
+	 * the DMA engine and sensor, to continue streaming.
+	 * This is not desired and is highly unlikely given that there are
+	 * 32 FBPT entries that the DMA engine needs to process, to run into
+	 * an FBPT entry, without the VALID bit set. We try to mitigate this
+	 * by disabling interrupts for the duration of this queueing.
+	 */
+	local_irq_save(flags);
+
+	fbpt_rp = (readl(cio2->base + CIO2_REG_CDMARI(CIO2_DMA_CHAN))
+		   >> CIO2_CDMARI_FBPT_RP_SHIFT)
+		   & CIO2_CDMARI_FBPT_RP_MASK;
+
+	/*
+	 * fbpt_rp is the fbpt entry that the dma is currently working
+	 * on, but since it could jump to next entry at any time,
+	 * assume that we might already be there.
+	 */
+	fbpt_rp = (fbpt_rp + 1) % CIO2_MAX_BUFFERS;
+
+	if (bufs_queued <= 1 || fbpt_rp == next)
+		/* Buffers were drained */
+		next = (fbpt_rp + 1) % CIO2_MAX_BUFFERS;
+
+	for (i = 0; i < CIO2_MAX_BUFFERS; i++) {
+		/*
+		 * We have allocated CIO2_MAX_BUFFERS circularly for the
+		 * hw, the user has requested N buffer queue. The driver
+		 * ensures N <= CIO2_MAX_BUFFERS and guarantees that whenever
+		 * user queues a buffer, there necessarily is a free buffer.
+		 */
+		if (!q->bufs[next]) {
+			q->bufs[next] = b;
+			entry = &q->fbpt[next * CIO2_MAX_LOPS];
+			cio2_fbpt_entry_init_buf(cio2, b, entry);
+			local_irq_restore(flags);
+			q->bufs_next = (next + 1) % CIO2_MAX_BUFFERS;
+			for (j = 0; j < vb->num_planes; j++)
+				vb2_set_plane_payload(vb, j,
+					q->format.plane_fmt[j].sizeimage);
+			return;
+		}
+
+		dev_dbg(&cio2->pci_dev->dev, "entry %i was full!\n", next);
+		next = (next + 1) % CIO2_MAX_BUFFERS;
+	}
+
+	local_irq_restore(flags);
+	dev_err(&cio2->pci_dev->dev, "error: all cio2 entries were full!\n");
+	atomic_dec(&q->bufs_queued);
+	vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
+}
+
+/* Called when each buffer is freed */
+static void cio2_vb2_buf_cleanup(struct vb2_buffer *vb)
+{
+	struct cio2_device *cio2 = vb2_get_drv_priv(vb->vb2_queue);
+	struct cio2_buffer *b =
+		container_of(vb, struct cio2_buffer, vbb.vb2_buf);
+	unsigned int i;
+
+	/* Free LOP table */
+	for (i = 0; i < CIO2_MAX_LOPS; i++) {
+		if (b->lop[i])
+			dma_free_coherent(&cio2->pci_dev->dev, CIO2_PAGE_SIZE,
+					  b->lop[i], b->lop_bus_addr[i]);
+	}
+}
+
+static int cio2_vb2_start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct cio2_queue *q = vb2q_to_cio2_queue(vq);
+	struct cio2_device *cio2 = vb2_get_drv_priv(vq);
+	int r;
+
+	cio2->cur_queue = q;
+	atomic_set(&q->frame_sequence, 0);
+
+	r = pm_runtime_get_sync(&cio2->pci_dev->dev);
+	if (r < 0) {
+		dev_info(&cio2->pci_dev->dev, "failed to set power %d\n", r);
+		pm_runtime_put_noidle(&cio2->pci_dev->dev);
+		return r;
+	}
+
+	r = media_pipeline_start(&q->vdev.entity, &q->pipe);
+	if (r)
+		goto fail_pipeline;
+
+	r = cio2_hw_init(cio2, q);
+	if (r)
+		goto fail_hw;
+
+	/* Start streaming on sensor */
+	r = v4l2_subdev_call(q->sensor, video, s_stream, 1);
+	if (r)
+		goto fail_csi2_subdev;
+
+	cio2->streaming = true;
+
+	return 0;
+
+fail_csi2_subdev:
+	cio2_hw_exit(cio2, q);
+fail_hw:
+	media_pipeline_stop(&q->vdev.entity);
+fail_pipeline:
+	dev_dbg(&cio2->pci_dev->dev, "failed to start streaming (%d)\n", r);
+	cio2_vb2_return_all_buffers(q, VB2_BUF_STATE_QUEUED);
+	pm_runtime_put(&cio2->pci_dev->dev);
+
+	return r;
+}
+
+static void cio2_vb2_stop_streaming(struct vb2_queue *vq)
+{
+	struct cio2_queue *q = vb2q_to_cio2_queue(vq);
+	struct cio2_device *cio2 = vb2_get_drv_priv(vq);
+
+	if (v4l2_subdev_call(q->sensor, video, s_stream, 0))
+		dev_err(&cio2->pci_dev->dev,
+			"failed to stop sensor streaming\n");
+
+	cio2_hw_exit(cio2, q);
+	cio2_vb2_return_all_buffers(q, VB2_BUF_STATE_ERROR);
+	media_pipeline_stop(&q->vdev.entity);
+	pm_runtime_put(&cio2->pci_dev->dev);
+	cio2->streaming = false;
+}
+
+static const struct vb2_ops cio2_vb2_ops = {
+	.buf_init = cio2_vb2_buf_init,
+	.buf_queue = cio2_vb2_buf_queue,
+	.buf_cleanup = cio2_vb2_buf_cleanup,
+	.queue_setup = cio2_vb2_queue_setup,
+	.start_streaming = cio2_vb2_start_streaming,
+	.stop_streaming = cio2_vb2_stop_streaming,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
+};
+
+/**************** V4L2 interface ****************/
+
+static int cio2_v4l2_querycap(struct file *file, void *fh,
+			      struct v4l2_capability *cap)
+{
+	struct cio2_device *cio2 = video_drvdata(file);
+
+	strlcpy(cap->driver, CIO2_NAME, sizeof(cap->driver));
+	strlcpy(cap->card, CIO2_DEVICE_NAME, sizeof(cap->card));
+	snprintf(cap->bus_info, sizeof(cap->bus_info),
+		 "PCI:%s", pci_name(cio2->pci_dev));
+
+	return 0;
+}
+
+static int cio2_v4l2_enum_fmt(struct file *file, void *fh,
+			      struct v4l2_fmtdesc *f)
+{
+	if (f->index >= ARRAY_SIZE(formats))
+		return -EINVAL;
+
+	f->pixelformat = formats[f->index].fourcc;
+
+	return 0;
+}
+
+/* The format is validated in cio2_video_link_validate() */
+static int cio2_v4l2_g_fmt(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct cio2_queue *q = file_to_cio2_queue(file);
+
+	f->fmt.pix_mp = q->format;
+
+	return 0;
+}
+
+static int cio2_v4l2_try_fmt(struct file *file, void *fh, struct v4l2_format *f)
+{
+	const struct ipu3_cio2_fmt *fmt;
+	struct v4l2_pix_format_mplane *mpix = &f->fmt.pix_mp;
+
+	fmt = cio2_find_format(&mpix->pixelformat, NULL);
+	if (!fmt)
+		fmt = &formats[0];
+
+	/* Only supports up to 4224x3136 */
+	if (mpix->width > CIO2_IMAGE_MAX_WIDTH)
+		mpix->width = CIO2_IMAGE_MAX_WIDTH;
+	if (mpix->height > CIO2_IMAGE_MAX_LENGTH)
+		mpix->height = CIO2_IMAGE_MAX_LENGTH;
+
+	mpix->num_planes = 1;
+	mpix->pixelformat = fmt->fourcc;
+	mpix->colorspace = V4L2_COLORSPACE_RAW;
+	mpix->field = V4L2_FIELD_NONE;
+	memset(mpix->reserved, 0, sizeof(mpix->reserved));
+	mpix->plane_fmt[0].bytesperline = cio2_bytesperline(mpix->width);
+	mpix->plane_fmt[0].sizeimage = mpix->plane_fmt[0].bytesperline *
+							mpix->height;
+	memset(mpix->plane_fmt[0].reserved, 0,
+	       sizeof(mpix->plane_fmt[0].reserved));
+
+	/* use default */
+	mpix->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	mpix->quantization = V4L2_QUANTIZATION_DEFAULT;
+	mpix->xfer_func = V4L2_XFER_FUNC_DEFAULT;
+
+	return 0;
+}
+
+static int cio2_v4l2_s_fmt(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct cio2_queue *q = file_to_cio2_queue(file);
+
+	cio2_v4l2_try_fmt(file, fh, f);
+	q->format = f->fmt.pix_mp;
+
+	return 0;
+}
+
+static int
+cio2_video_enum_input(struct file *file, void *fh, struct v4l2_input *input)
+{
+	if (input->index > 0)
+		return -EINVAL;
+
+	strlcpy(input->name, "camera", sizeof(input->name));
+	input->type = V4L2_INPUT_TYPE_CAMERA;
+
+	return 0;
+}
+
+static int
+cio2_video_g_input(struct file *file, void *fh, unsigned int *input)
+{
+	*input = 0;
+
+	return 0;
+}
+
+static int
+cio2_video_s_input(struct file *file, void *fh, unsigned int input)
+{
+	return input == 0 ? 0 : -EINVAL;
+}
+
+static const struct v4l2_file_operations cio2_v4l2_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = video_ioctl2,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
+	.poll = vb2_fop_poll,
+	.mmap = vb2_fop_mmap,
+};
+
+static const struct v4l2_ioctl_ops cio2_v4l2_ioctl_ops = {
+	.vidioc_querycap = cio2_v4l2_querycap,
+	.vidioc_enum_fmt_vid_cap_mplane = cio2_v4l2_enum_fmt,
+	.vidioc_g_fmt_vid_cap_mplane = cio2_v4l2_g_fmt,
+	.vidioc_s_fmt_vid_cap_mplane = cio2_v4l2_s_fmt,
+	.vidioc_try_fmt_vid_cap_mplane = cio2_v4l2_try_fmt,
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_enum_input = cio2_video_enum_input,
+	.vidioc_g_input	= cio2_video_g_input,
+	.vidioc_s_input	= cio2_video_s_input,
+};
+
+static int cio2_subdev_subscribe_event(struct v4l2_subdev *sd,
+				       struct v4l2_fh *fh,
+				       struct v4l2_event_subscription *sub)
+{
+	if (sub->type != V4L2_EVENT_FRAME_SYNC)
+		return -EINVAL;
+
+	/* Line number. For now only zero accepted. */
+	if (sub->id != 0)
+		return -EINVAL;
+
+	return v4l2_event_subscribe(fh, sub, 0, NULL);
+}
+
+static int cio2_subdev_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)
+{
+	struct v4l2_mbus_framefmt *format;
+	const struct v4l2_mbus_framefmt fmt_default = {
+		.width = 1936,
+		.height = 1096,
+		.code = formats[0].mbus_code,
+		.field = V4L2_FIELD_NONE,
+		.colorspace = V4L2_COLORSPACE_RAW,
+		.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT,
+		.quantization = V4L2_QUANTIZATION_DEFAULT,
+		.xfer_func = V4L2_XFER_FUNC_DEFAULT,
+	};
+
+	/* Initialize try_fmt */
+	format = v4l2_subdev_get_try_format(sd, fh->pad, CIO2_PAD_SINK);
+	*format = fmt_default;
+
+	/* same as sink */
+	format = v4l2_subdev_get_try_format(sd, fh->pad, CIO2_PAD_SOURCE);
+	*format = fmt_default;
+
+	return 0;
+}
+
+/*
+ * cio2_subdev_get_fmt - Handle get format by pads subdev method
+ * @sd : pointer to v4l2 subdev structure
+ * @cfg: V4L2 subdev pad config
+ * @fmt: pointer to v4l2 subdev format structure
+ * return -EINVAL or zero on success
+ */
+static int cio2_subdev_get_fmt(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_format *fmt)
+{
+	struct cio2_queue *q = container_of(sd, struct cio2_queue, subdev);
+	struct v4l2_subdev_format format;
+	int ret;
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		fmt->format = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
+		return 0;
+	}
+
+	if (fmt->pad == CIO2_PAD_SINK) {
+		format.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		ret = v4l2_subdev_call(sd, pad, get_fmt, NULL,
+				       &format);
+
+		if (ret)
+			return ret;
+		/* update colorspace etc */
+		q->subdev_fmt.colorspace = format.format.colorspace;
+		q->subdev_fmt.ycbcr_enc = format.format.ycbcr_enc;
+		q->subdev_fmt.quantization = format.format.quantization;
+		q->subdev_fmt.xfer_func = format.format.xfer_func;
+	}
+
+	fmt->format = q->subdev_fmt;
+
+	return 0;
+}
+
+/*
+ * cio2_subdev_set_fmt - Handle set format by pads subdev method
+ * @sd : pointer to v4l2 subdev structure
+ * @cfg: V4L2 subdev pad config
+ * @fmt: pointer to v4l2 subdev format structure
+ * return -EINVAL or zero on success
+ */
+static int cio2_subdev_set_fmt(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_pad_config *cfg,
+			       struct v4l2_subdev_format *fmt)
+{
+	struct cio2_queue *q = container_of(sd, struct cio2_queue, subdev);
+
+	/*
+	 * Only allow setting sink pad format;
+	 * source always propagates from sink
+	 */
+	if (fmt->pad == CIO2_PAD_SOURCE)
+		return cio2_subdev_get_fmt(sd, cfg, fmt);
+
+	if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {
+		*v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format;
+	} else {
+		/* It's the sink, allow changing frame size */
+		q->subdev_fmt.width = fmt->format.width;
+		q->subdev_fmt.height = fmt->format.height;
+		q->subdev_fmt.code = fmt->format.code;
+		fmt->format = q->subdev_fmt;
+	}
+
+	return 0;
+}
+
+static int cio2_subdev_enum_mbus_code(struct v4l2_subdev *sd,
+				      struct v4l2_subdev_pad_config *cfg,
+				      struct v4l2_subdev_mbus_code_enum *code)
+{
+	if (code->index >= ARRAY_SIZE(formats))
+		return -EINVAL;
+
+	code->code = formats[code->index].mbus_code;
+	return 0;
+}
+
+static int cio2_subdev_link_validate_get_format(struct media_pad *pad,
+						struct v4l2_subdev_format *fmt)
+{
+	if (is_media_entity_v4l2_subdev(pad->entity)) {
+		struct v4l2_subdev *sd =
+			media_entity_to_v4l2_subdev(pad->entity);
+
+		fmt->which = V4L2_SUBDEV_FORMAT_ACTIVE;
+		fmt->pad = pad->index;
+		return v4l2_subdev_call(sd, pad, get_fmt, NULL, fmt);
+	}
+
+	return -EINVAL;
+}
+
+static int cio2_video_link_validate(struct media_link *link)
+{
+	struct video_device *vd = container_of(link->sink->entity,
+						struct video_device, entity);
+	struct cio2_queue *q = container_of(vd, struct cio2_queue, vdev);
+	struct cio2_device *cio2 = video_get_drvdata(vd);
+	struct v4l2_subdev_format source_fmt;
+	int ret;
+
+	if (!media_entity_remote_pad(link->sink->entity->pads)) {
+		dev_info(&cio2->pci_dev->dev,
+			 "video node %s pad not connected\n", vd->name);
+		return -ENOTCONN;
+	}
+
+	ret = cio2_subdev_link_validate_get_format(link->source, &source_fmt);
+	if (ret < 0)
+		return 0;
+
+	if (source_fmt.format.width != q->format.width ||
+	    source_fmt.format.height != q->format.height) {
+		dev_err(&cio2->pci_dev->dev,
+			"Wrong width or height %ux%u (%ux%u expected)\n",
+			q->format.width, q->format.height,
+			source_fmt.format.width, source_fmt.format.height);
+		return -EINVAL;
+	}
+
+	if (!cio2_find_format(&q->format.pixelformat, &source_fmt.format.code))
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops cio2_subdev_core_ops = {
+	.subscribe_event = cio2_subdev_subscribe_event,
+	.unsubscribe_event = v4l2_event_subdev_unsubscribe,
+};
+
+static const struct v4l2_subdev_internal_ops cio2_subdev_internal_ops = {
+	.open = cio2_subdev_open,
+};
+
+static const struct v4l2_subdev_pad_ops cio2_subdev_pad_ops = {
+	.link_validate = v4l2_subdev_link_validate_default,
+	.get_fmt = cio2_subdev_get_fmt,
+	.set_fmt = cio2_subdev_set_fmt,
+	.enum_mbus_code = cio2_subdev_enum_mbus_code,
+};
+
+static const struct v4l2_subdev_ops cio2_subdev_ops = {
+	.core = &cio2_subdev_core_ops,
+	.pad = &cio2_subdev_pad_ops,
+};
+
+/******* V4L2 sub-device asynchronous registration callbacks***********/
+
+struct sensor_async_subdev {
+	struct v4l2_async_subdev asd;
+	struct csi2_bus_info csi2;
+};
+
+/* The .bound() notifier callback when a match is found */
+static int cio2_notifier_bound(struct v4l2_async_notifier *notifier,
+			       struct v4l2_subdev *sd,
+			       struct v4l2_async_subdev *asd)
+{
+	struct cio2_device *cio2 = container_of(notifier,
+					struct cio2_device, notifier);
+	struct sensor_async_subdev *s_asd = container_of(asd,
+					struct sensor_async_subdev, asd);
+	struct cio2_queue *q;
+
+	if (cio2->queue[s_asd->csi2.port].sensor)
+		return -EBUSY;
+
+	q = &cio2->queue[s_asd->csi2.port];
+
+	q->csi2 = s_asd->csi2;
+	q->sensor = sd;
+	q->csi_rx_base = cio2->base + CIO2_REG_PIPE_BASE(q->csi2.port);
+
+	return 0;
+}
+
+/* The .unbind callback */
+static void cio2_notifier_unbind(struct v4l2_async_notifier *notifier,
+				 struct v4l2_subdev *sd,
+				 struct v4l2_async_subdev *asd)
+{
+	struct cio2_device *cio2 = container_of(notifier,
+						struct cio2_device, notifier);
+	struct sensor_async_subdev *s_asd = container_of(asd,
+					struct sensor_async_subdev, asd);
+
+	cio2->queue[s_asd->csi2.port].sensor = NULL;
+}
+
+/* .complete() is called after all subdevices have been located */
+static int cio2_notifier_complete(struct v4l2_async_notifier *notifier)
+{
+	struct cio2_device *cio2 = container_of(notifier, struct cio2_device,
+						notifier);
+	struct sensor_async_subdev *s_asd;
+	struct cio2_queue *q;
+	unsigned int i, pad;
+	int ret;
+
+	for (i = 0; i < notifier->num_subdevs; i++) {
+		s_asd = container_of(cio2->notifier.subdevs[i],
+				     struct sensor_async_subdev, asd);
+		q = &cio2->queue[s_asd->csi2.port];
+
+		for (pad = 0; pad < q->sensor->entity.num_pads; pad++)
+			if (q->sensor->entity.pads[pad].flags &
+						MEDIA_PAD_FL_SOURCE)
+				break;
+
+		if (pad == q->sensor->entity.num_pads) {
+			dev_err(&cio2->pci_dev->dev,
+				"failed to find src pad for %s\n",
+				q->sensor->name);
+			return -ENXIO;
+		}
+
+		ret = media_create_pad_link(
+				&q->sensor->entity, pad,
+				&q->subdev.entity, CIO2_PAD_SINK,
+				0);
+		if (ret) {
+			dev_err(&cio2->pci_dev->dev,
+				"failed to create link for %s\n",
+				cio2->queue[i].sensor->name);
+			return ret;
+		}
+	}
+
+	return v4l2_device_register_subdev_nodes(&cio2->v4l2_dev);
+}
+
+static const struct v4l2_async_notifier_operations cio2_async_ops = {
+	.bound = cio2_notifier_bound,
+	.unbind = cio2_notifier_unbind,
+	.complete = cio2_notifier_complete,
+};
+
+static int cio2_fwnode_parse(struct device *dev,
+			     struct v4l2_fwnode_endpoint *vep,
+			     struct v4l2_async_subdev *asd)
+{
+	struct sensor_async_subdev *s_asd =
+			container_of(asd, struct sensor_async_subdev, asd);
+
+	if (vep->bus_type != V4L2_MBUS_CSI2) {
+		dev_err(dev, "Only CSI2 bus type is currently supported\n");
+		return -EINVAL;
+	}
+
+	s_asd->csi2.port = vep->base.port;
+	s_asd->csi2.lanes = vep->bus.mipi_csi2.num_data_lanes;
+
+	return 0;
+}
+
+static int cio2_notifier_init(struct cio2_device *cio2)
+{
+	int ret;
+
+	ret = v4l2_async_notifier_parse_fwnode_endpoints(
+		&cio2->pci_dev->dev, &cio2->notifier,
+		sizeof(struct sensor_async_subdev),
+		cio2_fwnode_parse);
+	if (ret < 0)
+		return ret;
+
+	if (!cio2->notifier.num_subdevs)
+		return -ENODEV;	/* no endpoint */
+
+	cio2->notifier.ops = &cio2_async_ops;
+	ret = v4l2_async_notifier_register(&cio2->v4l2_dev, &cio2->notifier);
+	if (ret) {
+		dev_err(&cio2->pci_dev->dev,
+			"failed to register async notifier : %d\n", ret);
+		v4l2_async_notifier_cleanup(&cio2->notifier);
+	}
+
+	return ret;
+}
+
+static void cio2_notifier_exit(struct cio2_device *cio2)
+{
+	v4l2_async_notifier_unregister(&cio2->notifier);
+	v4l2_async_notifier_cleanup(&cio2->notifier);
+}
+
+/**************** Queue initialization ****************/
+static const struct media_entity_operations cio2_media_ops = {
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+static const struct media_entity_operations cio2_video_entity_ops = {
+	.link_validate = cio2_video_link_validate,
+};
+
+static int cio2_queue_init(struct cio2_device *cio2, struct cio2_queue *q)
+{
+	static const u32 default_width = 1936;
+	static const u32 default_height = 1096;
+	const struct ipu3_cio2_fmt dflt_fmt = formats[0];
+
+	struct video_device *vdev = &q->vdev;
+	struct vb2_queue *vbq = &q->vbq;
+	struct v4l2_subdev *subdev = &q->subdev;
+	struct v4l2_mbus_framefmt *fmt;
+	int r;
+
+	/* Initialize miscellaneous variables */
+	mutex_init(&q->lock);
+
+	/* Initialize formats to default values */
+	fmt = &q->subdev_fmt;
+	fmt->width = default_width;
+	fmt->height = default_height;
+	fmt->code = dflt_fmt.mbus_code;
+	fmt->field = V4L2_FIELD_NONE;
+
+	q->format.width = default_width;
+	q->format.height = default_height;
+	q->format.pixelformat = dflt_fmt.fourcc;
+	q->format.colorspace = V4L2_COLORSPACE_RAW;
+	q->format.field = V4L2_FIELD_NONE;
+	q->format.num_planes = 1;
+	q->format.plane_fmt[0].bytesperline =
+				cio2_bytesperline(q->format.width);
+	q->format.plane_fmt[0].sizeimage = q->format.plane_fmt[0].bytesperline *
+						q->format.height;
+
+	/* Initialize fbpt */
+	r = cio2_fbpt_init(cio2, q);
+	if (r)
+		goto fail_fbpt;
+
+	/* Initialize media entities */
+	q->subdev_pads[CIO2_PAD_SINK].flags = MEDIA_PAD_FL_SINK |
+		MEDIA_PAD_FL_MUST_CONNECT;
+	q->subdev_pads[CIO2_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
+	subdev->entity.ops = &cio2_media_ops;
+	subdev->internal_ops = &cio2_subdev_internal_ops;
+	r = media_entity_pads_init(&subdev->entity, CIO2_PADS, q->subdev_pads);
+	if (r) {
+		dev_err(&cio2->pci_dev->dev,
+			"failed initialize subdev media entity (%d)\n", r);
+		goto fail_subdev_media_entity;
+	}
+
+	q->vdev_pad.flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT;
+	vdev->entity.ops = &cio2_video_entity_ops;
+	r = media_entity_pads_init(&vdev->entity, 1, &q->vdev_pad);
+	if (r) {
+		dev_err(&cio2->pci_dev->dev,
+			"failed initialize videodev media entity (%d)\n", r);
+		goto fail_vdev_media_entity;
+	}
+
+	/* Initialize subdev */
+	v4l2_subdev_init(subdev, &cio2_subdev_ops);
+	subdev->flags = V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
+	subdev->owner = THIS_MODULE;
+	snprintf(subdev->name, sizeof(subdev->name),
+		 CIO2_ENTITY_NAME " %td", q - cio2->queue);
+	v4l2_set_subdevdata(subdev, cio2);
+	r = v4l2_device_register_subdev(&cio2->v4l2_dev, subdev);
+	if (r) {
+		dev_err(&cio2->pci_dev->dev,
+			"failed initialize subdev (%d)\n", r);
+		goto fail_subdev;
+	}
+
+	/* Initialize vbq */
+	vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+	vbq->io_modes = VB2_USERPTR | VB2_MMAP | VB2_DMABUF;
+	vbq->ops = &cio2_vb2_ops;
+	vbq->mem_ops = &vb2_dma_sg_memops;
+	vbq->buf_struct_size = sizeof(struct cio2_buffer);
+	vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	vbq->min_buffers_needed = 1;
+	vbq->drv_priv = cio2;
+	vbq->lock = &q->lock;
+	r = vb2_queue_init(vbq);
+	if (r) {
+		dev_err(&cio2->pci_dev->dev,
+			"failed to initialize videobuf2 queue (%d)\n", r);
+		goto fail_vbq;
+	}
+
+	/* Initialize vdev */
+	snprintf(vdev->name, sizeof(vdev->name),
+		 "%s %td", CIO2_NAME, q - cio2->queue);
+	vdev->release = video_device_release_empty;
+	vdev->fops = &cio2_v4l2_fops;
+	vdev->ioctl_ops = &cio2_v4l2_ioctl_ops;
+	vdev->lock = &cio2->lock;
+	vdev->v4l2_dev = &cio2->v4l2_dev;
+	vdev->queue = &q->vbq;
+	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_STREAMING;
+	video_set_drvdata(vdev, cio2);
+	r = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
+	if (r) {
+		dev_err(&cio2->pci_dev->dev,
+			"failed to register video device (%d)\n", r);
+		goto fail_vdev;
+	}
+
+	/* Create link from CIO2 subdev to output node */
+	r = media_create_pad_link(
+		&subdev->entity, CIO2_PAD_SOURCE, &vdev->entity, 0,
+		MEDIA_LNK_FL_ENABLED | MEDIA_LNK_FL_IMMUTABLE);
+	if (r)
+		goto fail_link;
+
+	return 0;
+
+fail_link:
+	video_unregister_device(&q->vdev);
+fail_vdev:
+	vb2_queue_release(vbq);
+fail_vbq:
+	v4l2_device_unregister_subdev(subdev);
+fail_subdev:
+	media_entity_cleanup(&vdev->entity);
+fail_vdev_media_entity:
+	media_entity_cleanup(&subdev->entity);
+fail_subdev_media_entity:
+	cio2_fbpt_exit(q, &cio2->pci_dev->dev);
+fail_fbpt:
+	mutex_destroy(&q->lock);
+
+	return r;
+}
+
+static void cio2_queue_exit(struct cio2_device *cio2, struct cio2_queue *q)
+{
+	video_unregister_device(&q->vdev);
+	media_entity_cleanup(&q->vdev.entity);
+	vb2_queue_release(&q->vbq);
+	v4l2_device_unregister_subdev(&q->subdev);
+	media_entity_cleanup(&q->subdev.entity);
+	cio2_fbpt_exit(q, &cio2->pci_dev->dev);
+	mutex_destroy(&q->lock);
+}
+
+static int cio2_queues_init(struct cio2_device *cio2)
+{
+	int i, r;
+
+	for (i = 0; i < CIO2_QUEUES; i++) {
+		r = cio2_queue_init(cio2, &cio2->queue[i]);
+		if (r)
+			break;
+	}
+
+	if (i == CIO2_QUEUES)
+		return 0;
+
+	for (i--; i >= 0; i--)
+		cio2_queue_exit(cio2, &cio2->queue[i]);
+
+	return r;
+}
+
+static void cio2_queues_exit(struct cio2_device *cio2)
+{
+	unsigned int i;
+
+	for (i = 0; i < CIO2_QUEUES; i++)
+		cio2_queue_exit(cio2, &cio2->queue[i]);
+}
+
+/**************** PCI interface ****************/
+
+static int cio2_pci_config_setup(struct pci_dev *dev)
+{
+	u16 pci_command;
+	int r = pci_enable_msi(dev);
+
+	if (r) {
+		dev_err(&dev->dev, "failed to enable MSI (%d)\n", r);
+		return r;
+	}
+
+	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
+	pci_command |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+		PCI_COMMAND_INTX_DISABLE;
+	pci_write_config_word(dev, PCI_COMMAND, pci_command);
+
+	return 0;
+}
+
+static int cio2_pci_probe(struct pci_dev *pci_dev,
+			  const struct pci_device_id *id)
+{
+	struct cio2_device *cio2;
+	void __iomem *const *iomap;
+	int r;
+
+	cio2 = devm_kzalloc(&pci_dev->dev, sizeof(*cio2), GFP_KERNEL);
+	if (!cio2)
+		return -ENOMEM;
+	cio2->pci_dev = pci_dev;
+
+	r = pcim_enable_device(pci_dev);
+	if (r) {
+		dev_err(&pci_dev->dev, "failed to enable device (%d)\n", r);
+		return r;
+	}
+
+	dev_info(&pci_dev->dev, "device 0x%x (rev: 0x%x)\n",
+		 pci_dev->device, pci_dev->revision);
+
+	r = pcim_iomap_regions(pci_dev, 1 << CIO2_PCI_BAR, pci_name(pci_dev));
+	if (r) {
+		dev_err(&pci_dev->dev, "failed to remap I/O memory (%d)\n", r);
+		return -ENODEV;
+	}
+
+	iomap = pcim_iomap_table(pci_dev);
+	if (!iomap) {
+		dev_err(&pci_dev->dev, "failed to iomap table\n");
+		return -ENODEV;
+	}
+
+	cio2->base = iomap[CIO2_PCI_BAR];
+
+	pci_set_drvdata(pci_dev, cio2);
+
+	pci_set_master(pci_dev);
+
+	r = pci_set_dma_mask(pci_dev, CIO2_DMA_MASK);
+	if (r) {
+		dev_err(&pci_dev->dev, "failed to set DMA mask (%d)\n", r);
+		return -ENODEV;
+	}
+
+	r = cio2_pci_config_setup(pci_dev);
+	if (r)
+		return -ENODEV;
+
+	r = cio2_fbpt_init_dummy(cio2);
+	if (r)
+		return r;
+
+	mutex_init(&cio2->lock);
+
+	cio2->media_dev.dev = &cio2->pci_dev->dev;
+	strlcpy(cio2->media_dev.model, CIO2_DEVICE_NAME,
+		sizeof(cio2->media_dev.model));
+	snprintf(cio2->media_dev.bus_info, sizeof(cio2->media_dev.bus_info),
+		 "PCI:%s", pci_name(cio2->pci_dev));
+	cio2->media_dev.hw_revision = 0;
+
+	media_device_init(&cio2->media_dev);
+	r = media_device_register(&cio2->media_dev);
+	if (r < 0)
+		goto fail_mutex_destroy;
+
+	cio2->v4l2_dev.mdev = &cio2->media_dev;
+	r = v4l2_device_register(&pci_dev->dev, &cio2->v4l2_dev);
+	if (r) {
+		dev_err(&pci_dev->dev,
+			"failed to register V4L2 device (%d)\n", r);
+		goto fail_media_device_unregister;
+	}
+
+	r = cio2_queues_init(cio2);
+	if (r)
+		goto fail_v4l2_device_unregister;
+
+	/* Register notifier for subdevices we care */
+	r = cio2_notifier_init(cio2);
+	if (r)
+		goto fail_cio2_queue_exit;
+
+	r = devm_request_irq(&pci_dev->dev, pci_dev->irq, cio2_irq,
+			     IRQF_SHARED, CIO2_NAME, cio2);
+	if (r) {
+		dev_err(&pci_dev->dev, "failed to request IRQ (%d)\n", r);
+		goto fail;
+	}
+
+	pm_runtime_put_noidle(&pci_dev->dev);
+	pm_runtime_allow(&pci_dev->dev);
+
+	return 0;
+
+fail:
+	cio2_notifier_exit(cio2);
+fail_cio2_queue_exit:
+	cio2_queues_exit(cio2);
+fail_v4l2_device_unregister:
+	v4l2_device_unregister(&cio2->v4l2_dev);
+fail_media_device_unregister:
+	media_device_unregister(&cio2->media_dev);
+	media_device_cleanup(&cio2->media_dev);
+fail_mutex_destroy:
+	mutex_destroy(&cio2->lock);
+	cio2_fbpt_exit_dummy(cio2);
+
+	return r;
+}
+
+static void cio2_pci_remove(struct pci_dev *pci_dev)
+{
+	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);
+	unsigned int i;
+
+	cio2_notifier_exit(cio2);
+	cio2_fbpt_exit_dummy(cio2);
+	for (i = 0; i < CIO2_QUEUES; i++)
+		cio2_queue_exit(cio2, &cio2->queue[i]);
+	v4l2_device_unregister(&cio2->v4l2_dev);
+	media_device_unregister(&cio2->media_dev);
+	media_device_cleanup(&cio2->media_dev);
+	mutex_destroy(&cio2->lock);
+}
+
+static int __maybe_unused cio2_runtime_suspend(struct device *dev)
+{
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);
+	void __iomem *const base = cio2->base;
+	u16 pm;
+
+	writel(CIO2_D0I3C_I3, base + CIO2_REG_D0I3C);
+	dev_dbg(dev, "cio2 runtime suspend.\n");
+
+	pci_read_config_word(pci_dev, pci_dev->pm_cap + CIO2_PMCSR_OFFSET, &pm);
+	pm = (pm >> CIO2_PMCSR_D0D3_SHIFT) << CIO2_PMCSR_D0D3_SHIFT;
+	pm |= CIO2_PMCSR_D3;
+	pci_write_config_word(pci_dev, pci_dev->pm_cap + CIO2_PMCSR_OFFSET, pm);
+
+	return 0;
+}
+
+static int __maybe_unused cio2_runtime_resume(struct device *dev)
+{
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);
+	void __iomem *const base = cio2->base;
+	u16 pm;
+
+	writel(CIO2_D0I3C_RR, base + CIO2_REG_D0I3C);
+	dev_dbg(dev, "cio2 runtime resume.\n");
+
+	pci_read_config_word(pci_dev, pci_dev->pm_cap + CIO2_PMCSR_OFFSET, &pm);
+	pm = (pm >> CIO2_PMCSR_D0D3_SHIFT) << CIO2_PMCSR_D0D3_SHIFT;
+	pci_write_config_word(pci_dev, pci_dev->pm_cap + CIO2_PMCSR_OFFSET, pm);
+
+	return 0;
+}
+
+/*
+ * Helper function to advance all the elements of a circular buffer by "start"
+ * positions
+ */
+static void arrange(void *ptr, size_t elem_size, size_t elems, size_t start)
+{
+	struct {
+		size_t begin, end;
+	} arr[2] = {
+		{ 0, start - 1 },
+		{ start, elems - 1 },
+	};
+
+#define CHUNK_SIZE(a) ((a)->end - (a)->begin + 1)
+
+	/* Loop as long as we have out-of-place entries */
+	while (CHUNK_SIZE(&arr[0]) && CHUNK_SIZE(&arr[1])) {
+		size_t size0, i;
+
+		/*
+		 * Find the number of entries that can be arranged on this
+		 * iteration.
+		 */
+		size0 = min(CHUNK_SIZE(&arr[0]), CHUNK_SIZE(&arr[1]));
+
+		/* Swap the entries in two parts of the array. */
+		for (i = 0; i < size0; i++) {
+			u8 *d = ptr + elem_size * (arr[1].begin + i);
+			u8 *s = ptr + elem_size * (arr[0].begin + i);
+			size_t j;
+
+			for (j = 0; j < elem_size; j++)
+				swap(d[j], s[j]);
+		}
+
+		if (CHUNK_SIZE(&arr[0]) > CHUNK_SIZE(&arr[1])) {
+			/* The end of the first array remains unarranged. */
+			arr[0].begin += size0;
+		} else {
+			/*
+			 * The first array is fully arranged so we proceed
+			 * handling the next one.
+			 */
+			arr[0].begin = arr[1].begin;
+			arr[0].end = arr[1].begin + size0 - 1;
+			arr[1].begin += size0;
+		}
+	}
+}
+
+static void cio2_fbpt_rearrange(struct cio2_device *cio2, struct cio2_queue *q)
+{
+	unsigned int i, j;
+
+	for (i = 0, j = q->bufs_first; i < CIO2_MAX_BUFFERS;
+		i++, j = (j + 1) % CIO2_MAX_BUFFERS)
+		if (q->bufs[j])
+			break;
+
+	if (i == CIO2_MAX_BUFFERS)
+		return;
+
+	if (j) {
+		arrange(q->fbpt, sizeof(struct cio2_fbpt_entry) * CIO2_MAX_LOPS,
+			CIO2_MAX_BUFFERS, j);
+		arrange(q->bufs, sizeof(struct cio2_buffer *),
+			CIO2_MAX_BUFFERS, j);
+	}
+
+	/*
+	 * DMA clears the valid bit when accessing the buffer.
+	 * When stopping stream in suspend callback, some of the buffers
+	 * may be in invalid state. After resume, when DMA meets the invalid
+	 * buffer, it will halt and stop receiving new data.
+	 * To avoid DMA halting, set the valid bit for all buffers in FBPT.
+	 */
+	for (i = 0; i < CIO2_MAX_BUFFERS; i++)
+		cio2_fbpt_entry_enable(cio2, q->fbpt + i * CIO2_MAX_LOPS);
+}
+
+static int __maybe_unused cio2_suspend(struct device *dev)
+{
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);
+	struct cio2_queue *q = cio2->cur_queue;
+
+	dev_dbg(dev, "cio2 suspend\n");
+	if (!cio2->streaming)
+		return 0;
+
+	/* Stop stream */
+	cio2_hw_exit(cio2, q);
+
+	pm_runtime_force_suspend(dev);
+
+	/*
+	 * Upon resume, hw starts to process the fbpt entries from beginning,
+	 * so relocate the queued buffs to the fbpt head before suspend.
+	 */
+	cio2_fbpt_rearrange(cio2, q);
+	q->bufs_first = 0;
+	q->bufs_next = 0;
+
+	return 0;
+}
+
+static int __maybe_unused cio2_resume(struct device *dev)
+{
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct cio2_device *cio2 = pci_get_drvdata(pci_dev);
+	int r = 0;
+	struct cio2_queue *q = cio2->cur_queue;
+
+	dev_dbg(dev, "cio2 resume\n");
+	if (!cio2->streaming)
+		return 0;
+	/* Start stream */
+	r = pm_runtime_force_resume(&cio2->pci_dev->dev);
+	if (r < 0) {
+		dev_err(&cio2->pci_dev->dev,
+			"failed to set power %d\n", r);
+		return r;
+	}
+
+	r = cio2_hw_init(cio2, q);
+	if (r)
+		dev_err(dev, "fail to init cio2 hw\n");
+
+	return r;
+}
+
+static const struct dev_pm_ops cio2_pm_ops = {
+	SET_RUNTIME_PM_OPS(&cio2_runtime_suspend, &cio2_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(&cio2_suspend, &cio2_resume)
+};
+
+static const struct pci_device_id cio2_pci_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, CIO2_PCI_ID) },
+	{ 0 }
+};
+
+MODULE_DEVICE_TABLE(pci, cio2_pci_id_table);
+
+static struct pci_driver cio2_pci_driver = {
+	.name = CIO2_NAME,
+	.id_table = cio2_pci_id_table,
+	.probe = cio2_pci_probe,
+	.remove = cio2_pci_remove,
+	.driver = {
+		.pm = &cio2_pm_ops,
+	},
+};
+
+module_pci_driver(cio2_pci_driver);
+
+MODULE_AUTHOR("Tuukka Toivonen <tuukka.toivonen@intel.com>");
+MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>");
+MODULE_AUTHOR("Jian Xu Zheng <jian.xu.zheng@intel.com>");
+MODULE_AUTHOR("Yuning Pu <yuning.pu@intel.com>");
+MODULE_AUTHOR("Yong Zhi <yong.zhi@intel.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("IPU3 CIO2 driver");
diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2.h b/drivers/media/pci/intel/ipu3/ipu3-cio2.h
new file mode 100644
index 000000000000..78a5799f08e7
--- /dev/null
+++ b/drivers/media/pci/intel/ipu3/ipu3-cio2.h
@@ -0,0 +1,449 @@
+/*
+ * Copyright (c) 2017 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version
+ * 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __IPU3_CIO2_H
+#define __IPU3_CIO2_H
+
+#define CIO2_NAME					"ipu3-cio2"
+#define CIO2_DEVICE_NAME				"Intel IPU3 CIO2"
+#define CIO2_ENTITY_NAME				"ipu3-csi2"
+#define CIO2_PCI_ID					0x9d32
+#define CIO2_PCI_BAR					0
+#define CIO2_DMA_MASK					DMA_BIT_MASK(39)
+#define CIO2_IMAGE_MAX_WIDTH				4224
+#define CIO2_IMAGE_MAX_LENGTH				3136
+
+#define CIO2_IMAGE_MAX_WIDTH				4224
+#define CIO2_IMAGE_MAX_LENGTH				3136
+
+/* 32MB = 8xFBPT_entry */
+#define CIO2_MAX_LOPS					8
+#define CIO2_MAX_BUFFERS			(PAGE_SIZE / 16 / CIO2_MAX_LOPS)
+
+#define CIO2_PAD_SINK					0
+#define CIO2_PAD_SOURCE					1
+#define CIO2_PADS					2
+
+#define CIO2_NUM_DMA_CHAN				20
+#define CIO2_NUM_PORTS					4 /* DPHYs */
+
+/* 1 for each sensor */
+#define CIO2_QUEUES					CIO2_NUM_PORTS
+
+/* Register and bit field definitions */
+#define CIO2_REG_PIPE_BASE(n)			((n) * 0x0400)	/* n = 0..3 */
+#define CIO2_REG_CSIRX_BASE				0x000
+#define CIO2_REG_MIPIBE_BASE				0x100
+#define CIO2_REG_PIXELGEN_BAS				0x200
+#define CIO2_REG_IRQCTRL_BASE				0x300
+#define CIO2_REG_GPREG_BASE				0x1000
+
+/* base register: CIO2_REG_PIPE_BASE(pipe) * CIO2_REG_CSIRX_BASE */
+#define CIO2_REG_CSIRX_ENABLE			(CIO2_REG_CSIRX_BASE + 0x0)
+#define CIO2_REG_CSIRX_NOF_ENABLED_LANES	(CIO2_REG_CSIRX_BASE + 0x4)
+#define CIO2_REG_CSIRX_SP_IF_CONFIG		(CIO2_REG_CSIRX_BASE + 0x10)
+#define CIO2_REG_CSIRX_LP_IF_CONFIG		(CIO2_REG_CSIRX_BASE + 0x14)
+#define CIO2_CSIRX_IF_CONFIG_FILTEROUT			0x00
+#define CIO2_CSIRX_IF_CONFIG_FILTEROUT_VC_INACTIVE	0x01
+#define CIO2_CSIRX_IF_CONFIG_PASS			0x02
+#define CIO2_CSIRX_IF_CONFIG_FLAG_ERROR			BIT(2)
+#define CIO2_REG_CSIRX_STATUS			(CIO2_REG_CSIRX_BASE + 0x18)
+#define CIO2_REG_CSIRX_STATUS_DLANE_HS		(CIO2_REG_CSIRX_BASE + 0x1c)
+#define CIO2_CSIRX_STATUS_DLANE_HS_MASK			0xff
+#define CIO2_REG_CSIRX_STATUS_DLANE_LP		(CIO2_REG_CSIRX_BASE + 0x20)
+#define CIO2_CSIRX_STATUS_DLANE_LP_MASK			0xffffff
+/* Termination enable and settle in 0.0625ns units, lane=0..3 or -1 for clock */
+#define CIO2_REG_CSIRX_DLY_CNT_TERMEN(lane) \
+				(CIO2_REG_CSIRX_BASE + 0x2c + 8 * (lane))
+#define CIO2_REG_CSIRX_DLY_CNT_SETTLE(lane) \
+				(CIO2_REG_CSIRX_BASE + 0x30 + 8 * (lane))
+/* base register: CIO2_REG_PIPE_BASE(pipe) * CIO2_REG_MIPIBE_BASE */
+#define CIO2_REG_MIPIBE_ENABLE		(CIO2_REG_MIPIBE_BASE + 0x0)
+#define CIO2_REG_MIPIBE_STATUS		(CIO2_REG_MIPIBE_BASE + 0x4)
+#define CIO2_REG_MIPIBE_COMP_FORMAT(vc) \
+				(CIO2_REG_MIPIBE_BASE + 0x8 + 0x4 * (vc))
+#define CIO2_REG_MIPIBE_FORCE_RAW8	(CIO2_REG_MIPIBE_BASE + 0x20)
+#define CIO2_REG_MIPIBE_FORCE_RAW8_ENABLE		BIT(0)
+#define CIO2_REG_MIPIBE_FORCE_RAW8_USE_TYPEID		BIT(1)
+#define CIO2_REG_MIPIBE_FORCE_RAW8_TYPEID_SHIFT		2
+
+#define CIO2_REG_MIPIBE_IRQ_STATUS	(CIO2_REG_MIPIBE_BASE + 0x24)
+#define CIO2_REG_MIPIBE_IRQ_CLEAR	(CIO2_REG_MIPIBE_BASE + 0x28)
+#define CIO2_REG_MIPIBE_GLOBAL_LUT_DISREGARD (CIO2_REG_MIPIBE_BASE + 0x68)
+#define CIO2_MIPIBE_GLOBAL_LUT_DISREGARD		1
+#define CIO2_REG_MIPIBE_PKT_STALL_STATUS (CIO2_REG_MIPIBE_BASE + 0x6c)
+#define CIO2_REG_MIPIBE_PARSE_GSP_THROUGH_LP_LUT_REG_IDX \
+					(CIO2_REG_MIPIBE_BASE + 0x70)
+#define CIO2_REG_MIPIBE_SP_LUT_ENTRY(vc) \
+				       (CIO2_REG_MIPIBE_BASE + 0x74 + 4 * (vc))
+#define CIO2_REG_MIPIBE_LP_LUT_ENTRY(m)	/* m = 0..15 */ \
+					(CIO2_REG_MIPIBE_BASE + 0x84 + 4 * (m))
+#define CIO2_MIPIBE_LP_LUT_ENTRY_DISREGARD		1
+#define CIO2_MIPIBE_LP_LUT_ENTRY_SID_SHIFT		1
+#define CIO2_MIPIBE_LP_LUT_ENTRY_VC_SHIFT		5
+#define CIO2_MIPIBE_LP_LUT_ENTRY_FORMAT_TYPE_SHIFT	7
+
+/* base register: CIO2_REG_PIPE_BASE(pipe) * CIO2_REG_IRQCTRL_BASE */
+/* IRQ registers are 18-bit wide, see cio2_irq_error for bit definitions */
+#define CIO2_REG_IRQCTRL_EDGE		(CIO2_REG_IRQCTRL_BASE + 0x00)
+#define CIO2_REG_IRQCTRL_MASK		(CIO2_REG_IRQCTRL_BASE + 0x04)
+#define CIO2_REG_IRQCTRL_STATUS		(CIO2_REG_IRQCTRL_BASE + 0x08)
+#define CIO2_REG_IRQCTRL_CLEAR		(CIO2_REG_IRQCTRL_BASE + 0x0c)
+#define CIO2_REG_IRQCTRL_ENABLE		(CIO2_REG_IRQCTRL_BASE + 0x10)
+#define CIO2_REG_IRQCTRL_LEVEL_NOT_PULSE	(CIO2_REG_IRQCTRL_BASE + 0x14)
+
+#define CIO2_REG_GPREG_SRST		(CIO2_REG_GPREG_BASE + 0x0)
+#define CIO2_GPREG_SRST_ALL				0xffff	/* Reset all */
+#define CIO2_REG_FB_HPLL_FREQ		(CIO2_REG_GPREG_BASE + 0x08)
+#define CIO2_REG_ISCLK_RATIO		(CIO2_REG_GPREG_BASE + 0xc)
+
+#define CIO2_REG_CGC					0x1400
+#define CIO2_CGC_CSI2_TGE				BIT(0)
+#define CIO2_CGC_PRIM_TGE				BIT(1)
+#define CIO2_CGC_SIDE_TGE				BIT(2)
+#define CIO2_CGC_XOSC_TGE				BIT(3)
+#define CIO2_CGC_MPLL_SHUTDOWN_EN			BIT(4)
+#define CIO2_CGC_D3I3_TGE				BIT(5)
+#define CIO2_CGC_CSI2_INTERFRAME_TGE			BIT(6)
+#define CIO2_CGC_CSI2_PORT_DCGE				BIT(8)
+#define CIO2_CGC_CSI2_DCGE				BIT(9)
+#define CIO2_CGC_SIDE_DCGE				BIT(10)
+#define CIO2_CGC_PRIM_DCGE				BIT(11)
+#define CIO2_CGC_ROSC_DCGE				BIT(12)
+#define CIO2_CGC_XOSC_DCGE				BIT(13)
+#define CIO2_CGC_FLIS_DCGE				BIT(14)
+#define CIO2_CGC_CLKGATE_HOLDOFF_SHIFT			20
+#define CIO2_CGC_CSI_CLKGATE_HOLDOFF_SHIFT		24
+#define CIO2_REG_D0I3C					0x1408
+#define CIO2_D0I3C_I3					BIT(2)	/* Set D0I3 */
+#define CIO2_D0I3C_RR					BIT(3)	/* Restore? */
+#define CIO2_REG_SWRESET				0x140c
+#define CIO2_SWRESET_SWRESET				1
+#define CIO2_REG_SENSOR_ACTIVE				0x1410
+#define CIO2_REG_INT_STS				0x1414
+#define CIO2_REG_INT_STS_EXT_OE				0x1418
+#define CIO2_INT_EXT_OE_DMAOE_SHIFT			0
+#define CIO2_INT_EXT_OE_DMAOE_MASK			0x7ffff
+#define CIO2_INT_EXT_OE_OES_SHIFT			24
+#define CIO2_INT_EXT_OE_OES_MASK	(0xf << CIO2_INT_EXT_OE_OES_SHIFT)
+#define CIO2_REG_INT_EN					0x1420
+#define CIO2_REG_INT_EN_IRQ				(1 << 24)
+#define CIO2_REG_INT_EN_IOS(dma)	(1 << (((dma) >> 1) + 12))
+/*
+ * Interrupt on completion bit, Eg. DMA 0-3 maps to bit 0-3,
+ * DMA4 & DMA5 map to bit 4 ... DMA18 & DMA19 map to bit 11 Et cetera
+ */
+#define CIO2_INT_IOC(dma)	(1 << ((dma) < 4 ? (dma) : ((dma) >> 1) + 2))
+#define CIO2_INT_IOC_SHIFT				0
+#define CIO2_INT_IOC_MASK		(0x7ff << CIO2_INT_IOC_SHIFT)
+#define CIO2_INT_IOS_IOLN(dma)		(1 << (((dma) >> 1) + 12))
+#define CIO2_INT_IOS_IOLN_SHIFT				12
+#define CIO2_INT_IOS_IOLN_MASK		(0x3ff << CIO2_INT_IOS_IOLN_SHIFT)
+#define CIO2_INT_IOIE					BIT(22)
+#define CIO2_INT_IOOE					BIT(23)
+#define CIO2_INT_IOIRQ					BIT(24)
+#define CIO2_REG_INT_EN_EXT_OE				0x1424
+#define CIO2_REG_DMA_DBG				0x1448
+#define CIO2_REG_DMA_DBG_DMA_INDEX_SHIFT		0
+#define CIO2_REG_PBM_ARB_CTRL				0x1460
+#define CIO2_PBM_ARB_CTRL_LANES_DIV			0 /* 4-4-2-2 lanes */
+#define CIO2_PBM_ARB_CTRL_LANES_DIV_SHIFT		0
+#define CIO2_PBM_ARB_CTRL_LE_EN				BIT(7)
+#define CIO2_PBM_ARB_CTRL_PLL_POST_SHTDN		2
+#define CIO2_PBM_ARB_CTRL_PLL_POST_SHTDN_SHIFT		8
+#define CIO2_PBM_ARB_CTRL_PLL_AHD_WK_UP			480
+#define CIO2_PBM_ARB_CTRL_PLL_AHD_WK_UP_SHIFT		16
+#define CIO2_REG_PBM_WMCTRL1				0x1464
+#define CIO2_PBM_WMCTRL1_MIN_2CK_SHIFT			0
+#define CIO2_PBM_WMCTRL1_MID1_2CK_SHIFT			8
+#define CIO2_PBM_WMCTRL1_MID2_2CK_SHIFT			16
+#define CIO2_PBM_WMCTRL1_TS_COUNT_DISABLE		BIT(31)
+#define CIO2_PBM_WMCTRL1_MIN_2CK	(4 << CIO2_PBM_WMCTRL1_MIN_2CK_SHIFT)
+#define CIO2_PBM_WMCTRL1_MID1_2CK	(16 << CIO2_PBM_WMCTRL1_MID1_2CK_SHIFT)
+#define CIO2_PBM_WMCTRL1_MID2_2CK	(21 << CIO2_PBM_WMCTRL1_MID2_2CK_SHIFT)
+#define CIO2_REG_PBM_WMCTRL2				0x1468
+#define CIO2_PBM_WMCTRL2_HWM_2CK			40
+#define CIO2_PBM_WMCTRL2_HWM_2CK_SHIFT			0
+#define CIO2_PBM_WMCTRL2_LWM_2CK			22
+#define CIO2_PBM_WMCTRL2_LWM_2CK_SHIFT			8
+#define CIO2_PBM_WMCTRL2_OBFFWM_2CK			2
+#define CIO2_PBM_WMCTRL2_OBFFWM_2CK_SHIFT		16
+#define CIO2_PBM_WMCTRL2_TRANSDYN			1
+#define CIO2_PBM_WMCTRL2_TRANSDYN_SHIFT			24
+#define CIO2_PBM_WMCTRL2_DYNWMEN			BIT(28)
+#define CIO2_PBM_WMCTRL2_OBFF_MEM_EN			BIT(29)
+#define CIO2_PBM_WMCTRL2_OBFF_CPU_EN			BIT(30)
+#define CIO2_PBM_WMCTRL2_DRAINNOW			BIT(31)
+#define CIO2_REG_PBM_TS_COUNT				0x146c
+#define CIO2_REG_PBM_FOPN_ABORT				0x1474
+/* below n = 0..3 */
+#define CIO2_PBM_FOPN_ABORT(n)				(0x1 << 8 * (n))
+#define CIO2_PBM_FOPN_FORCE_ABORT(n)			(0x2 << 8 * (n))
+#define CIO2_PBM_FOPN_FRAMEOPEN(n)			(0x8 << 8 * (n))
+#define CIO2_REG_LTRCTRL				0x1480
+#define CIO2_LTRCTRL_LTRDYNEN				BIT(16)
+#define CIO2_LTRCTRL_LTRSTABLETIME_SHIFT		8
+#define CIO2_LTRCTRL_LTRSTABLETIME_MASK			0xff
+#define CIO2_LTRCTRL_LTRSEL1S3				BIT(7)
+#define CIO2_LTRCTRL_LTRSEL1S2				BIT(6)
+#define CIO2_LTRCTRL_LTRSEL1S1				BIT(5)
+#define CIO2_LTRCTRL_LTRSEL1S0				BIT(4)
+#define CIO2_LTRCTRL_LTRSEL2S3				BIT(3)
+#define CIO2_LTRCTRL_LTRSEL2S2				BIT(2)
+#define CIO2_LTRCTRL_LTRSEL2S1				BIT(1)
+#define CIO2_LTRCTRL_LTRSEL2S0				BIT(0)
+#define CIO2_REG_LTRVAL23				0x1484
+#define CIO2_REG_LTRVAL01				0x1488
+#define CIO2_LTRVAL02_VAL_SHIFT				0
+#define CIO2_LTRVAL02_SCALE_SHIFT			10
+#define CIO2_LTRVAL13_VAL_SHIFT				16
+#define CIO2_LTRVAL13_SCALE_SHIFT			26
+
+#define CIO2_LTRVAL0_VAL				175
+/* Value times 1024 ns */
+#define CIO2_LTRVAL0_SCALE				2
+#define CIO2_LTRVAL1_VAL				90
+#define CIO2_LTRVAL1_SCALE				2
+#define CIO2_LTRVAL2_VAL				90
+#define CIO2_LTRVAL2_SCALE				2
+#define CIO2_LTRVAL3_VAL				90
+#define CIO2_LTRVAL3_SCALE				2
+
+#define CIO2_REG_CDMABA(n)		(0x1500 + 0x10 * (n))	/* n = 0..19 */
+#define CIO2_REG_CDMARI(n)		(0x1504 + 0x10 * (n))
+#define CIO2_CDMARI_FBPT_RP_SHIFT			0
+#define CIO2_CDMARI_FBPT_RP_MASK			0xff
+#define CIO2_REG_CDMAC0(n)		(0x1508 + 0x10 * (n))
+#define CIO2_CDMAC0_FBPT_LEN_SHIFT			0
+#define CIO2_CDMAC0_FBPT_WIDTH_SHIFT			8
+#define CIO2_CDMAC0_FBPT_NS				BIT(25)
+#define CIO2_CDMAC0_DMA_INTR_ON_FS			BIT(26)
+#define CIO2_CDMAC0_DMA_INTR_ON_FE			BIT(27)
+#define CIO2_CDMAC0_FBPT_UPDATE_FIFO_FULL		BIT(28)
+#define CIO2_CDMAC0_FBPT_FIFO_FULL_FIX_DIS		BIT(29)
+#define CIO2_CDMAC0_DMA_EN				BIT(30)
+#define CIO2_CDMAC0_DMA_HALTED				BIT(31)
+#define CIO2_REG_CDMAC1(n)		(0x150c + 0x10 * (n))
+#define CIO2_CDMAC1_LINENUMINT_SHIFT			0
+#define CIO2_CDMAC1_LINENUMUPDATE_SHIFT			16
+/* n = 0..3 */
+#define CIO2_REG_PXM_PXF_FMT_CFG0(n)	(0x1700 + 0x30 * (n))
+#define CIO2_PXM_PXF_FMT_CFG_SID0_SHIFT			0
+#define CIO2_PXM_PXF_FMT_CFG_SID1_SHIFT			16
+#define CIO2_PXM_PXF_FMT_CFG_PCK_64B			(0 << 0)
+#define CIO2_PXM_PXF_FMT_CFG_PCK_32B			(1 << 0)
+#define CIO2_PXM_PXF_FMT_CFG_BPP_08			(0 << 2)
+#define CIO2_PXM_PXF_FMT_CFG_BPP_10			(1 << 2)
+#define CIO2_PXM_PXF_FMT_CFG_BPP_12			(2 << 2)
+#define CIO2_PXM_PXF_FMT_CFG_BPP_14			(3 << 2)
+#define CIO2_PXM_PXF_FMT_CFG_SPEC_4PPC			(0 << 4)
+#define CIO2_PXM_PXF_FMT_CFG_SPEC_3PPC_RGBA		(1 << 4)
+#define CIO2_PXM_PXF_FMT_CFG_SPEC_3PPC_ARGB		(2 << 4)
+#define CIO2_PXM_PXF_FMT_CFG_SPEC_PLANAR2		(3 << 4)
+#define CIO2_PXM_PXF_FMT_CFG_SPEC_PLANAR3		(4 << 4)
+#define CIO2_PXM_PXF_FMT_CFG_SPEC_NV16			(5 << 4)
+#define CIO2_PXM_PXF_FMT_CFG_PSWAP4_1ST_AB		(1 << 7)
+#define CIO2_PXM_PXF_FMT_CFG_PSWAP4_1ST_CD		(1 << 8)
+#define CIO2_PXM_PXF_FMT_CFG_PSWAP4_2ND_AC		(1 << 9)
+#define CIO2_PXM_PXF_FMT_CFG_PSWAP4_2ND_BD		(1 << 10)
+#define CIO2_REG_INT_STS_EXT_IE				0x17e4
+#define CIO2_REG_INT_EN_EXT_IE				0x17e8
+#define CIO2_INT_EXT_IE_ECC_RE(n)			(0x01 << (8 * (n)))
+#define CIO2_INT_EXT_IE_DPHY_NR(n)			(0x02 << (8 * (n)))
+#define CIO2_INT_EXT_IE_ECC_NR(n)			(0x04 << (8 * (n)))
+#define CIO2_INT_EXT_IE_CRCERR(n)			(0x08 << (8 * (n)))
+#define CIO2_INT_EXT_IE_INTERFRAMEDATA(n)		(0x10 << (8 * (n)))
+#define CIO2_INT_EXT_IE_PKT2SHORT(n)			(0x20 << (8 * (n)))
+#define CIO2_INT_EXT_IE_PKT2LONG(n)			(0x40 << (8 * (n)))
+#define CIO2_INT_EXT_IE_IRQ(n)				(0x80 << (8 * (n)))
+#define CIO2_REG_PXM_FRF_CFG(n)				(0x1720 + 0x30 * (n))
+#define CIO2_PXM_FRF_CFG_FNSEL				BIT(0)
+#define CIO2_PXM_FRF_CFG_FN_RST				BIT(1)
+#define CIO2_PXM_FRF_CFG_ABORT				BIT(2)
+#define CIO2_PXM_FRF_CFG_CRC_TH_SHIFT			3
+#define CIO2_PXM_FRF_CFG_MSK_ECC_DPHY_NR		BIT(8)
+#define CIO2_PXM_FRF_CFG_MSK_ECC_RE			BIT(9)
+#define CIO2_PXM_FRF_CFG_MSK_ECC_DPHY_NE		BIT(10)
+#define CIO2_PXM_FRF_CFG_EVEN_ODD_MODE_SHIFT		11
+#define CIO2_PXM_FRF_CFG_MASK_CRC_THRES			BIT(13)
+#define CIO2_PXM_FRF_CFG_MASK_CSI_ACCEPT		BIT(14)
+#define CIO2_PXM_FRF_CFG_CIOHC_FS_MODE			BIT(15)
+#define CIO2_PXM_FRF_CFG_CIOHC_FRST_FRM_SHIFT		16
+#define CIO2_REG_PXM_SID2BID0(n)			(0x1724 + 0x30 * (n))
+#define CIO2_FB_HPLL_FREQ				0x2
+#define CIO2_ISCLK_RATIO				0xc
+
+#define CIO2_IRQCTRL_MASK				0x3ffff
+
+#define CIO2_INT_EN_EXT_OE_MASK				0x8f0fffff
+
+#define CIO2_CGC_CLKGATE_HOLDOFF			3
+#define CIO2_CGC_CSI_CLKGATE_HOLDOFF			5
+
+#define CIO2_PXM_FRF_CFG_CRC_TH				16
+
+#define CIO2_INT_EN_EXT_IE_MASK				0xffffffff
+
+#define CIO2_DMA_CHAN					0
+
+#define CIO2_CSIRX_DLY_CNT_CLANE_IDX			-1
+
+#define CIO2_CSIRX_DLY_CNT_TERMEN_CLANE_A		0
+#define CIO2_CSIRX_DLY_CNT_TERMEN_CLANE_B		0
+#define CIO2_CSIRX_DLY_CNT_SETTLE_CLANE_A		95
+#define CIO2_CSIRX_DLY_CNT_SETTLE_CLANE_B		-8
+
+#define CIO2_CSIRX_DLY_CNT_TERMEN_DLANE_A		0
+#define CIO2_CSIRX_DLY_CNT_TERMEN_DLANE_B		0
+#define CIO2_CSIRX_DLY_CNT_SETTLE_DLANE_A		85
+#define CIO2_CSIRX_DLY_CNT_SETTLE_DLANE_B		-2
+
+#define CIO2_CSIRX_DLY_CNT_TERMEN_DEFAULT		0x4
+#define CIO2_CSIRX_DLY_CNT_SETTLE_DEFAULT		0x570
+
+#define CIO2_PMCSR_OFFSET				4
+#define CIO2_PMCSR_D0D3_SHIFT				2
+#define CIO2_PMCSR_D3					0x3
+
+struct cio2_csi2_timing {
+	s32 clk_termen;
+	s32 clk_settle;
+	s32 dat_termen;
+	s32 dat_settle;
+};
+
+struct cio2_buffer {
+	struct vb2_v4l2_buffer vbb;
+	u32 *lop[CIO2_MAX_LOPS];
+	dma_addr_t lop_bus_addr[CIO2_MAX_LOPS];
+	unsigned int offset;
+};
+
+struct csi2_bus_info {
+	u32 port;
+	u32 lanes;
+};
+
+struct cio2_queue {
+	/* mutex to be used by vb2_queue */
+	struct mutex lock;
+	struct media_pipeline pipe;
+	struct csi2_bus_info csi2;
+	struct v4l2_subdev *sensor;
+	void __iomem *csi_rx_base;
+
+	/* Subdev, /dev/v4l-subdevX */
+	struct v4l2_subdev subdev;
+	struct media_pad subdev_pads[CIO2_PADS];
+	struct v4l2_mbus_framefmt subdev_fmt;
+	atomic_t frame_sequence;
+
+	/* Video device, /dev/videoX */
+	struct video_device vdev;
+	struct media_pad vdev_pad;
+	struct v4l2_pix_format_mplane format;
+	struct vb2_queue vbq;
+
+	/* Buffer queue handling */
+	struct cio2_fbpt_entry *fbpt;	/* Frame buffer pointer table */
+	dma_addr_t fbpt_bus_addr;
+	struct cio2_buffer *bufs[CIO2_MAX_BUFFERS];
+	unsigned int bufs_first;	/* Index of the first used entry */
+	unsigned int bufs_next;	/* Index of the first unused entry */
+	atomic_t bufs_queued;
+};
+
+struct cio2_device {
+	struct pci_dev *pci_dev;
+	void __iomem *base;
+	struct v4l2_device v4l2_dev;
+	struct cio2_queue queue[CIO2_QUEUES];
+	struct cio2_queue *cur_queue;
+	/* mutex to be used by video_device */
+	struct mutex lock;
+
+	bool streaming;
+	struct v4l2_async_notifier notifier;
+	struct media_device media_dev;
+
+	/*
+	 * Safety net to catch DMA fetch ahead
+	 * when reaching the end of LOP
+	 */
+	void *dummy_page;
+	/* DMA handle of dummy_page */
+	dma_addr_t dummy_page_bus_addr;
+	/* single List of Pointers (LOP) page */
+	u32 *dummy_lop;
+	/* DMA handle of dummy_lop */
+	dma_addr_t dummy_lop_bus_addr;
+};
+
+/**************** Virtual channel ****************/
+/*
+ * This should come from sensor driver. No
+ * driver interface nor requirement yet.
+ */
+#define SENSOR_VIR_CH_DFLT		0
+
+/**************** FBPT operations ****************/
+#define CIO2_FBPT_SIZE			(CIO2_MAX_BUFFERS * CIO2_MAX_LOPS * \
+					 sizeof(struct cio2_fbpt_entry))
+
+#define CIO2_FBPT_SUBENTRY_UNIT		4
+#define CIO2_PAGE_SIZE			4096
+
+/* cio2 fbpt first_entry ctrl status */
+#define CIO2_FBPT_CTRL_VALID		BIT(0)
+#define CIO2_FBPT_CTRL_IOC		BIT(1)
+#define CIO2_FBPT_CTRL_IOS		BIT(2)
+#define CIO2_FBPT_CTRL_SUCCXFAIL	BIT(3)
+#define CIO2_FBPT_CTRL_CMPLCODE_SHIFT	4
+
+/*
+ * Frame Buffer Pointer Table(FBPT) entry
+ * each entry describe an output buffer and consists of
+ * several sub-entries
+ */
+struct __packed cio2_fbpt_entry {
+	union {
+		struct __packed {
+			u32 ctrl; /* status ctrl */
+			u16 cur_line_num; /* current line # written to DDR */
+			u16 frame_num; /* updated by DMA upon FE */
+			u32 first_page_offset; /* offset for 1st page in LOP */
+		} first_entry;
+		/* Second entry per buffer */
+		struct __packed {
+			u32 timestamp;
+			u32 num_of_bytes;
+			/* the number of bytes for write on last page */
+			u16 last_page_available_bytes;
+			/* the number of pages allocated for this buf */
+			u16 num_of_pages;
+		} second_entry;
+	};
+	u32 lop_page_addr;	/* Points to list of pointers (LOP) table */
+};
+
+static inline struct cio2_queue *file_to_cio2_queue(struct file *file)
+{
+	return container_of(video_devdata(file), struct cio2_queue, vdev);
+}
+
+static inline struct cio2_queue *vb2q_to_cio2_queue(struct vb2_queue *vq)
+{
+	return container_of(vq, struct cio2_queue, vbq);
+}
+
+#endif
diff --git a/drivers/media/pci/ivtv/Makefile b/drivers/media/pci/ivtv/Makefile
index 48f8a23f9a0f..5de95dbe3256 100644
--- a/drivers/media/pci/ivtv/Makefile
+++ b/drivers/media/pci/ivtv/Makefile
@@ -10,8 +10,6 @@ obj-$(CONFIG_VIDEO_IVTV) += ivtv.o
 obj-$(CONFIG_VIDEO_IVTV_ALSA) += ivtv-alsa.o
 obj-$(CONFIG_VIDEO_FB_IVTV) += ivtvfb.o
 
-ccflags-y += -I$(srctree)/drivers/media/i2c
 ccflags-y += -I$(srctree)/drivers/media/tuners
-ccflags-y += -I$(srctree)/drivers/media/dvb-core
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends
 
diff --git a/drivers/media/pci/ivtv/ivtv-cards.c b/drivers/media/pci/ivtv/ivtv-cards.c
index 410d97bdf541..c63792964a03 100644
--- a/drivers/media/pci/ivtv/ivtv-cards.c
+++ b/drivers/media/pci/ivtv/ivtv-cards.c
@@ -65,7 +65,7 @@ static struct ivtv_card_tuner_i2c ivtv_i2c_tda8290 = {
 
 /********************** card configuration *******************************/
 
-/* Please add new PCI IDs to: http://pci-ids.ucw.cz/ 
+/* Please add new PCI IDs to: http://pci-ids.ucw.cz/
    This keeps the PCI ID database up to date. Note that the entries
    must be added under vendor 0x4444 (Conexant) as subsystem IDs.
    New vendor IDs should still be added to the vendor ID list. */
diff --git a/drivers/media/pci/ivtv/ivtv-cards.h b/drivers/media/pci/ivtv/ivtv-cards.h
index e6f5c02981f1..1557a6ea4dd9 100644
--- a/drivers/media/pci/ivtv/ivtv-cards.h
+++ b/drivers/media/pci/ivtv/ivtv-cards.h
@@ -22,26 +22,26 @@
 #define IVTV_CARDS_H
 
 /* Supported cards */
-#define IVTV_CARD_PVR_250 	      0	/* WinTV PVR 250 */
-#define IVTV_CARD_PVR_350 	      1	/* encoder, decoder, tv-out */
-#define IVTV_CARD_PVR_150 	      2	/* WinTV PVR 150 and PVR 500 (really just two
+#define IVTV_CARD_PVR_250	      0	/* WinTV PVR 250 */
+#define IVTV_CARD_PVR_350	      1	/* encoder, decoder, tv-out */
+#define IVTV_CARD_PVR_150	      2	/* WinTV PVR 150 and PVR 500 (really just two
 					   PVR150s on one PCI board) */
-#define IVTV_CARD_M179    	      3	/* AVerMedia M179 (encoder only) */
-#define IVTV_CARD_MPG600  	      4	/* Kuroutoshikou ITVC16-STVLP/YUAN MPG600, encoder only */
-#define IVTV_CARD_MPG160  	      5	/* Kuroutoshikou ITVC15-STVLP/YUAN MPG160
+#define IVTV_CARD_M179		      3	/* AVerMedia M179 (encoder only) */
+#define IVTV_CARD_MPG600	      4	/* Kuroutoshikou ITVC16-STVLP/YUAN MPG600, encoder only */
+#define IVTV_CARD_MPG160	      5	/* Kuroutoshikou ITVC15-STVLP/YUAN MPG160
 					   cx23415 based, but does not have tv-out */
-#define IVTV_CARD_PG600 	      6	/* YUAN PG600/DIAMONDMM PVR-550 based on the CX Falcon 2 */
-#define IVTV_CARD_AVC2410 	      7	/* Adaptec AVC-2410 */
-#define IVTV_CARD_AVC2010 	      8	/* Adaptec AVD-2010 (No Tuner) */
-#define IVTV_CARD_TG5000TV   	      9 /* NAGASE TRANSGEAR 5000TV, encoder only */
+#define IVTV_CARD_PG600		      6	/* YUAN PG600/DIAMONDMM PVR-550 based on the CX Falcon 2 */
+#define IVTV_CARD_AVC2410	      7	/* Adaptec AVC-2410 */
+#define IVTV_CARD_AVC2010	      8	/* Adaptec AVD-2010 (No Tuner) */
+#define IVTV_CARD_TG5000TV	      9 /* NAGASE TRANSGEAR 5000TV, encoder only */
 #define IVTV_CARD_VA2000MAX_SNT6     10 /* VA2000MAX-STN6 */
-#define IVTV_CARD_CX23416GYC 	     11 /* Kuroutoshikou CX23416GYC-STVLP (Yuan MPG600GR OEM) */
-#define IVTV_CARD_GV_MVPRX   	     12 /* I/O Data GV-MVP/RX, RX2, RX2W */
-#define IVTV_CARD_GV_MVPRX2E 	     13 /* I/O Data GV-MVP/RX2E */
+#define IVTV_CARD_CX23416GYC	     11 /* Kuroutoshikou CX23416GYC-STVLP (Yuan MPG600GR OEM) */
+#define IVTV_CARD_GV_MVPRX	     12 /* I/O Data GV-MVP/RX, RX2, RX2W */
+#define IVTV_CARD_GV_MVPRX2E	     13 /* I/O Data GV-MVP/RX2E */
 #define IVTV_CARD_GOTVIEW_PCI_DVD    14	/* GotView PCI DVD */
 #define IVTV_CARD_GOTVIEW_PCI_DVD2   15	/* GotView PCI DVD2 */
 #define IVTV_CARD_YUAN_MPC622        16	/* Yuan MPC622 miniPCI */
-#define IVTV_CARD_DCTMTVP1 	     17 /* DIGITAL COWBOY DCT-MTVP1 */
+#define IVTV_CARD_DCTMTVP1	     17 /* DIGITAL COWBOY DCT-MTVP1 */
 #define IVTV_CARD_PG600V2	     18 /* Yuan PG600V2/GotView PCI DVD Lite */
 #define IVTV_CARD_CLUB3D	     19 /* Club3D ZAP-TV1x01 */
 #define IVTV_CARD_AVERTV_MCE116	     20 /* AVerTV MCE 116 Plus */
@@ -52,7 +52,7 @@
 #define IVTV_CARD_BUFFALO_MV5L       25 /* Buffalo PC-MV5L/PCI card */
 #define IVTV_CARD_AVER_ULTRA1500MCE  26 /* AVerMedia UltraTV 1500 MCE */
 #define IVTV_CARD_KIKYOU             27 /* Sony VAIO Giga Pocket (ENX Kikyou) */
-#define IVTV_CARD_LAST 		     27
+#define IVTV_CARD_LAST		     27
 
 /* Variants of existing cards but with the same PCI IDs. The driver
    detects these based on other device information.
@@ -61,7 +61,7 @@
    must be adjusted accordingly. */
 
 /* PVR-350 V1 (uses saa7114) */
-#define IVTV_CARD_PVR_350_V1 	     (IVTV_CARD_LAST+1)
+#define IVTV_CARD_PVR_350_V1	     (IVTV_CARD_LAST+1)
 /* 2 variants of Kuroutoshikou CX23416GYC-STVLP (Yuan MPG600GR OEM) */
 #define IVTV_CARD_CX23416GYC_NOGR    (IVTV_CARD_LAST+2)
 #define IVTV_CARD_CX23416GYC_NOGRYCS (IVTV_CARD_LAST+3)
@@ -72,22 +72,22 @@
 #define PCI_DEVICE_ID_IVTV16 0x0016
 
 /* subsystem vendor ID */
-#define IVTV_PCI_ID_HAUPPAUGE 		0x0070
-#define IVTV_PCI_ID_HAUPPAUGE_ALT1 	0x0270
-#define IVTV_PCI_ID_HAUPPAUGE_ALT2 	0x4070
-#define IVTV_PCI_ID_ADAPTEC 		0x9005
-#define IVTV_PCI_ID_ASUSTEK 		0x1043
-#define IVTV_PCI_ID_AVERMEDIA 		0x1461
+#define IVTV_PCI_ID_HAUPPAUGE		0x0070
+#define IVTV_PCI_ID_HAUPPAUGE_ALT1	0x0270
+#define IVTV_PCI_ID_HAUPPAUGE_ALT2	0x4070
+#define IVTV_PCI_ID_ADAPTEC		0x9005
+#define IVTV_PCI_ID_ASUSTEK		0x1043
+#define IVTV_PCI_ID_AVERMEDIA		0x1461
 #define IVTV_PCI_ID_YUAN1		0x12ab
-#define IVTV_PCI_ID_YUAN2 		0xff01
-#define IVTV_PCI_ID_YUAN3 		0xffab
-#define IVTV_PCI_ID_YUAN4 		0xfbab
-#define IVTV_PCI_ID_DIAMONDMM 		0xff92
-#define IVTV_PCI_ID_IODATA 		0x10fc
-#define IVTV_PCI_ID_MELCO 		0x1154
+#define IVTV_PCI_ID_YUAN2		0xff01
+#define IVTV_PCI_ID_YUAN3		0xffab
+#define IVTV_PCI_ID_YUAN4		0xfbab
+#define IVTV_PCI_ID_DIAMONDMM		0xff92
+#define IVTV_PCI_ID_IODATA		0x10fc
+#define IVTV_PCI_ID_MELCO		0x1154
 #define IVTV_PCI_ID_GOTVIEW1		0xffac
-#define IVTV_PCI_ID_GOTVIEW2 		0xffad
-#define IVTV_PCI_ID_SONY 		0x104d
+#define IVTV_PCI_ID_GOTVIEW2		0xffad
+#define IVTV_PCI_ID_SONY		0x104d
 
 /* hardware flags, no gaps allowed */
 #define IVTV_HW_CX25840			(1 << 0)
@@ -109,41 +109,33 @@
 #define IVTV_HW_I2C_IR_RX_AVER		(1 << 16)
 #define IVTV_HW_I2C_IR_RX_HAUP_EXT	(1 << 17) /* External before internal */
 #define IVTV_HW_I2C_IR_RX_HAUP_INT	(1 << 18)
-#define IVTV_HW_Z8F0811_IR_TX_HAUP	(1 << 19)
-#define IVTV_HW_Z8F0811_IR_RX_HAUP	(1 << 20)
-#define IVTV_HW_I2C_IR_RX_ADAPTEC	(1 << 21)
-
-#define IVTV_HW_Z8F0811_IR_HAUP	(IVTV_HW_Z8F0811_IR_RX_HAUP | \
-				 IVTV_HW_Z8F0811_IR_TX_HAUP)
+#define IVTV_HW_Z8F0811_IR_HAUP		(1 << 19)
+#define IVTV_HW_I2C_IR_RX_ADAPTEC	(1 << 20)
 
 #define IVTV_HW_SAA711X   (IVTV_HW_SAA7115 | IVTV_HW_SAA7114)
 
-#define IVTV_HW_IR_RX_ANY (IVTV_HW_I2C_IR_RX_AVER | \
-			   IVTV_HW_I2C_IR_RX_HAUP_EXT | \
-			   IVTV_HW_I2C_IR_RX_HAUP_INT | \
-			   IVTV_HW_Z8F0811_IR_RX_HAUP | \
-			   IVTV_HW_I2C_IR_RX_ADAPTEC)
-
-#define IVTV_HW_IR_TX_ANY (IVTV_HW_Z8F0811_IR_TX_HAUP)
-
-#define IVTV_HW_IR_ANY	  (IVTV_HW_IR_RX_ANY | IVTV_HW_IR_TX_ANY)
+#define IVTV_HW_IR_ANY (IVTV_HW_I2C_IR_RX_AVER | \
+			IVTV_HW_I2C_IR_RX_HAUP_EXT | \
+			IVTV_HW_I2C_IR_RX_HAUP_INT | \
+			IVTV_HW_Z8F0811_IR_HAUP | \
+			IVTV_HW_I2C_IR_RX_ADAPTEC)
 
 /* video inputs */
 #define	IVTV_CARD_INPUT_VID_TUNER	1
-#define	IVTV_CARD_INPUT_SVIDEO1 	2
-#define	IVTV_CARD_INPUT_SVIDEO2 	3
-#define	IVTV_CARD_INPUT_COMPOSITE1 	4
-#define	IVTV_CARD_INPUT_COMPOSITE2 	5
-#define	IVTV_CARD_INPUT_COMPOSITE3 	6
+#define	IVTV_CARD_INPUT_SVIDEO1		2
+#define	IVTV_CARD_INPUT_SVIDEO2		3
+#define	IVTV_CARD_INPUT_COMPOSITE1	4
+#define	IVTV_CARD_INPUT_COMPOSITE2	5
+#define	IVTV_CARD_INPUT_COMPOSITE3	6
 
 /* audio inputs */
 #define	IVTV_CARD_INPUT_AUD_TUNER	1
-#define	IVTV_CARD_INPUT_LINE_IN1 	2
-#define	IVTV_CARD_INPUT_LINE_IN2 	3
+#define	IVTV_CARD_INPUT_LINE_IN1	2
+#define	IVTV_CARD_INPUT_LINE_IN2	3
 
 #define IVTV_CARD_MAX_VIDEO_INPUTS 6
 #define IVTV_CARD_MAX_AUDIO_INPUTS 3
-#define IVTV_CARD_MAX_TUNERS  	   3
+#define IVTV_CARD_MAX_TUNERS	   3
 
 /* SAA71XX HW inputs */
 #define IVTV_SAA71XX_COMPOSITE0 0
@@ -180,7 +172,7 @@
 			  V4L2_CAP_SLICED_VBI_OUTPUT | V4L2_CAP_VIDEO_OUTPUT_OVERLAY)
 
 struct ivtv_card_video_input {
-	u8  video_type; 	/* video input type */
+	u8  video_type;		/* video input type */
 	u8  audio_index;	/* index in ivtv_card_audio_input array */
 	u16 video_input;	/* hardware video input */
 };
@@ -207,55 +199,55 @@ struct ivtv_card_pci_info {
 
 /* The mask is the set of bits used by the operation */
 
-struct ivtv_gpio_init { 	/* set initial GPIO DIR and OUT values */
-	u16 direction; 		/* DIR setting. Leave to 0 if no init is needed */
+struct ivtv_gpio_init {		/* set initial GPIO DIR and OUT values */
+	u16 direction;		/* DIR setting. Leave to 0 if no init is needed */
 	u16 initial_value;
 };
 
-struct ivtv_gpio_video_input { 	/* select tuner/line in input */
-	u16 mask; 		/* leave to 0 if not supported */
+struct ivtv_gpio_video_input {	/* select tuner/line in input */
+	u16 mask;		/* leave to 0 if not supported */
 	u16 tuner;
 	u16 composite;
 	u16 svideo;
 };
 
-struct ivtv_gpio_audio_input { 	/* select tuner/line in input */
-	u16 mask; 		/* leave to 0 if not supported */
+struct ivtv_gpio_audio_input {	/* select tuner/line in input */
+	u16 mask;		/* leave to 0 if not supported */
 	u16 tuner;
 	u16 linein;
 	u16 radio;
 };
 
 struct ivtv_gpio_audio_mute {
-	u16 mask; 		/* leave to 0 if not supported */
+	u16 mask;		/* leave to 0 if not supported */
 	u16 mute;		/* set this value to mute, 0 to unmute */
 };
 
 struct ivtv_gpio_audio_mode {
-	u16 mask; 		/* leave to 0 if not supported */
-	u16 mono; 		/* set audio to mono */
-	u16 stereo; 		/* set audio to stereo */
+	u16 mask;		/* leave to 0 if not supported */
+	u16 mono;		/* set audio to mono */
+	u16 stereo;		/* set audio to stereo */
 	u16 lang1;		/* set audio to the first language */
 	u16 lang2;		/* set audio to the second language */
-	u16 both; 		/* both languages are output */
+	u16 both;		/* both languages are output */
 };
 
 struct ivtv_gpio_audio_freq {
-	u16 mask; 		/* leave to 0 if not supported */
+	u16 mask;		/* leave to 0 if not supported */
 	u16 f32000;
 	u16 f44100;
 	u16 f48000;
 };
 
 struct ivtv_gpio_audio_detect {
-	u16 mask; 		/* leave to 0 if not supported */
-	u16 stereo; 		/* if the input matches this value then
+	u16 mask;		/* leave to 0 if not supported */
+	u16 stereo;		/* if the input matches this value then
 				   stereo is detected */
 };
 
 struct ivtv_card_tuner {
-	v4l2_std_id std; 	/* standard for which the tuner is suitable */
-	int 	    tuner; 	/* tuner ID (from tuner.h) */
+	v4l2_std_id std;	/* standard for which the tuner is suitable */
+	int	    tuner;	/* tuner ID (from tuner.h) */
 };
 
 struct ivtv_card_tuner_i2c {
@@ -280,17 +272,17 @@ struct ivtv_card {
 	struct ivtv_card_audio_input radio_input;
 	int nof_outputs;
 	const struct ivtv_card_output *video_outputs;
-	u8 gr_config; 		/* config byte for the ghost reduction device */
-	u8 xceive_pin; 		/* XCeive tuner GPIO reset pin */
+	u8 gr_config;		/* config byte for the ghost reduction device */
+	u8 xceive_pin;		/* XCeive tuner GPIO reset pin */
 
 	/* GPIO card-specific settings */
-	struct ivtv_gpio_init 		gpio_init;
+	struct ivtv_gpio_init		gpio_init;
 	struct ivtv_gpio_video_input	gpio_video_input;
-	struct ivtv_gpio_audio_input 	gpio_audio_input;
-	struct ivtv_gpio_audio_mute 	gpio_audio_mute;
-	struct ivtv_gpio_audio_mode 	gpio_audio_mode;
-	struct ivtv_gpio_audio_freq 	gpio_audio_freq;
-	struct ivtv_gpio_audio_detect 	gpio_audio_detect;
+	struct ivtv_gpio_audio_input	gpio_audio_input;
+	struct ivtv_gpio_audio_mute	gpio_audio_mute;
+	struct ivtv_gpio_audio_mode	gpio_audio_mode;
+	struct ivtv_gpio_audio_freq	gpio_audio_freq;
+	struct ivtv_gpio_audio_detect	gpio_audio_detect;
 
 	struct ivtv_card_tuner tuners[IVTV_CARD_MAX_TUNERS];
 	struct ivtv_card_tuner_i2c *i2c;
diff --git a/drivers/media/pci/ivtv/ivtv-driver.h b/drivers/media/pci/ivtv/ivtv-driver.h
index d27c5c2c07ea..cafba6b1055d 100644
--- a/drivers/media/pci/ivtv/ivtv-driver.h
+++ b/drivers/media/pci/ivtv/ivtv-driver.h
@@ -76,7 +76,7 @@
 #define IVTV_ENCODER_SIZE	0x00800000	/* Total size is 0x01000000, but only first half is used */
 #define IVTV_DECODER_OFFSET	0x01000000
 #define IVTV_DECODER_SIZE	0x00800000	/* Total size is 0x01000000, but only first half is used */
-#define IVTV_REG_OFFSET 	0x02000000
+#define IVTV_REG_OFFSET		0x02000000
 #define IVTV_REG_SIZE		0x00010000
 
 /* Maximum ivtv driver instances. Some people have a huge number of
@@ -97,26 +97,26 @@
 #define IVTV_DMA_SG_OSD_ENT	(2883584/PAGE_SIZE)	/* sg entities */
 
 /* DMA Registers */
-#define IVTV_REG_DMAXFER 	(0x0000)
-#define IVTV_REG_DMASTATUS 	(0x0004)
-#define IVTV_REG_DECDMAADDR 	(0x0008)
-#define IVTV_REG_ENCDMAADDR 	(0x000c)
-#define IVTV_REG_DMACONTROL 	(0x0010)
-#define IVTV_REG_IRQSTATUS 	(0x0040)
-#define IVTV_REG_IRQMASK 	(0x0048)
+#define IVTV_REG_DMAXFER	(0x0000)
+#define IVTV_REG_DMASTATUS	(0x0004)
+#define IVTV_REG_DECDMAADDR	(0x0008)
+#define IVTV_REG_ENCDMAADDR	(0x000c)
+#define IVTV_REG_DMACONTROL	(0x0010)
+#define IVTV_REG_IRQSTATUS	(0x0040)
+#define IVTV_REG_IRQMASK	(0x0048)
 
 /* Setup Registers */
-#define IVTV_REG_ENC_SDRAM_REFRESH 	(0x07F8)
-#define IVTV_REG_ENC_SDRAM_PRECHARGE 	(0x07FC)
-#define IVTV_REG_DEC_SDRAM_REFRESH 	(0x08F8)
-#define IVTV_REG_DEC_SDRAM_PRECHARGE 	(0x08FC)
-#define IVTV_REG_VDM 			(0x2800)
-#define IVTV_REG_AO 			(0x2D00)
-#define IVTV_REG_BYTEFLUSH 		(0x2D24)
-#define IVTV_REG_SPU 			(0x9050)
-#define IVTV_REG_HW_BLOCKS 		(0x9054)
-#define IVTV_REG_VPU 			(0x9058)
-#define IVTV_REG_APU 			(0xA064)
+#define IVTV_REG_ENC_SDRAM_REFRESH	(0x07F8)
+#define IVTV_REG_ENC_SDRAM_PRECHARGE	(0x07FC)
+#define IVTV_REG_DEC_SDRAM_REFRESH	(0x08F8)
+#define IVTV_REG_DEC_SDRAM_PRECHARGE	(0x08FC)
+#define IVTV_REG_VDM			(0x2800)
+#define IVTV_REG_AO			(0x2D00)
+#define IVTV_REG_BYTEFLUSH		(0x2D24)
+#define IVTV_REG_SPU			(0x9050)
+#define IVTV_REG_HW_BLOCKS		(0x9054)
+#define IVTV_REG_VPU			(0x9058)
+#define IVTV_REG_APU			(0xA064)
 
 /* Other registers */
 #define IVTV_REG_DEC_LINE_FIELD		(0x28C0)
@@ -158,7 +158,7 @@ extern int ivtv_fw_debug;
 
 #define IVTV_DEBUG_HIGH_VOL(x, type, fmt, args...) \
 	do { \
-		if (((x) & ivtv_debug) && (ivtv_debug & IVTV_DBGFLG_HIGHVOL)) 	\
+		if (((x) & ivtv_debug) && (ivtv_debug & IVTV_DBGFLG_HIGHVOL))	\
 			v4l2_info(&itv->v4l2_dev, " " type ": " fmt , ##args);	\
 	} while (0)
 #define IVTV_DEBUG_HI_WARN(fmt, args...)  IVTV_DEBUG_HIGH_VOL(IVTV_DBGFLG_WARN,  "warn",  fmt , ## args)
@@ -226,9 +226,9 @@ struct ivtv_mailbox_data {
 /* per-stream, s_flags */
 #define IVTV_F_S_DMA_PENDING	0	/* this stream has pending DMA */
 #define IVTV_F_S_DMA_HAS_VBI	1       /* the current DMA request also requests VBI data */
-#define IVTV_F_S_NEEDS_DATA	2 	/* this decoding stream needs more data */
+#define IVTV_F_S_NEEDS_DATA	2	/* this decoding stream needs more data */
 
-#define IVTV_F_S_CLAIMED 	3	/* this stream is claimed */
+#define IVTV_F_S_CLAIMED	3	/* this stream is claimed */
 #define IVTV_F_S_STREAMING      4	/* the fw is decoding/encoding this stream */
 #define IVTV_F_S_INTERNAL_USE	5	/* this stream is used internally (sliced VBI processing) */
 #define IVTV_F_S_PASSTHROUGH	6	/* this stream is in passthrough mode */
@@ -239,35 +239,35 @@ struct ivtv_mailbox_data {
 #define IVTV_F_S_PIO_HAS_VBI	1       /* the current PIO request also requests VBI data */
 
 /* per-ivtv, i_flags */
-#define IVTV_F_I_DMA		   0 	/* DMA in progress */
-#define IVTV_F_I_UDMA		   1 	/* UDMA in progress */
-#define IVTV_F_I_UDMA_PENDING	   2 	/* UDMA pending */
-#define IVTV_F_I_SPEED_CHANGE	   3 	/* a speed change is in progress */
-#define IVTV_F_I_EOS		   4 	/* end of encoder stream reached */
-#define IVTV_F_I_RADIO_USER	   5 	/* the radio tuner is selected */
-#define IVTV_F_I_DIG_RST	   6 	/* reset digitizer */
-#define IVTV_F_I_DEC_YUV	   7 	/* YUV instead of MPG is being decoded */
-#define IVTV_F_I_UPDATE_CC	   9  	/* CC should be updated */
-#define IVTV_F_I_UPDATE_WSS	   10 	/* WSS should be updated */
-#define IVTV_F_I_UPDATE_VPS	   11 	/* VPS should be updated */
-#define IVTV_F_I_DECODING_YUV	   12 	/* this stream is YUV frame decoding */
-#define IVTV_F_I_ENC_PAUSED	   13 	/* the encoder is paused */
-#define IVTV_F_I_VALID_DEC_TIMINGS 14 	/* last_dec_timing is valid */
-#define IVTV_F_I_HAVE_WORK  	   15	/* used in the interrupt handler: there is work to be done */
+#define IVTV_F_I_DMA		   0	/* DMA in progress */
+#define IVTV_F_I_UDMA		   1	/* UDMA in progress */
+#define IVTV_F_I_UDMA_PENDING	   2	/* UDMA pending */
+#define IVTV_F_I_SPEED_CHANGE	   3	/* a speed change is in progress */
+#define IVTV_F_I_EOS		   4	/* end of encoder stream reached */
+#define IVTV_F_I_RADIO_USER	   5	/* the radio tuner is selected */
+#define IVTV_F_I_DIG_RST	   6	/* reset digitizer */
+#define IVTV_F_I_DEC_YUV	   7	/* YUV instead of MPG is being decoded */
+#define IVTV_F_I_UPDATE_CC	   9	/* CC should be updated */
+#define IVTV_F_I_UPDATE_WSS	   10	/* WSS should be updated */
+#define IVTV_F_I_UPDATE_VPS	   11	/* VPS should be updated */
+#define IVTV_F_I_DECODING_YUV	   12	/* this stream is YUV frame decoding */
+#define IVTV_F_I_ENC_PAUSED	   13	/* the encoder is paused */
+#define IVTV_F_I_VALID_DEC_TIMINGS 14	/* last_dec_timing is valid */
+#define IVTV_F_I_HAVE_WORK	   15	/* used in the interrupt handler: there is work to be done */
 #define IVTV_F_I_WORK_HANDLER_VBI  16	/* there is work to be done for VBI */
 #define IVTV_F_I_WORK_HANDLER_YUV  17	/* there is work to be done for YUV */
 #define IVTV_F_I_WORK_HANDLER_PIO  18	/* there is work to be done for PIO */
 #define IVTV_F_I_PIO		   19	/* PIO in progress */
-#define IVTV_F_I_DEC_PAUSED	   20 	/* the decoder is paused */
-#define IVTV_F_I_INITED		   21 	/* set after first open */
-#define IVTV_F_I_FAILED		   22 	/* set if first open failed */
+#define IVTV_F_I_DEC_PAUSED	   20	/* the decoder is paused */
+#define IVTV_F_I_INITED		   21	/* set after first open */
+#define IVTV_F_I_FAILED		   22	/* set if first open failed */
 #define IVTV_F_I_WORK_HANDLER_PCM  23	/* there is work to be done for PCM */
 
 /* Event notifications */
 #define IVTV_F_I_EV_DEC_STOPPED	   28	/* decoder stopped event */
-#define IVTV_F_I_EV_VSYNC	   29 	/* VSYNC event */
-#define IVTV_F_I_EV_VSYNC_FIELD    30 	/* VSYNC event field (0 = first, 1 = second field) */
-#define IVTV_F_I_EV_VSYNC_ENABLED  31 	/* VSYNC event enabled */
+#define IVTV_F_I_EV_VSYNC	   29	/* VSYNC event */
+#define IVTV_F_I_EV_VSYNC_FIELD    30	/* VSYNC event field (0 = first, 1 = second field) */
+#define IVTV_F_I_EV_VSYNC_ENABLED  31	/* VSYNC event enabled */
 
 /* Scatter-Gather array element, used in DMA transfers */
 struct ivtv_sg_element {
@@ -330,13 +330,13 @@ struct ivtv_stream {
 	/* These first four fields are always set, even if the stream
 	   is not actually created. */
 	struct video_device vdev;	/* vdev.v4l2_dev is NULL if there is no device */
-	struct ivtv *itv; 		/* for ease of use */
+	struct ivtv *itv;		/* for ease of use */
 	const char *name;		/* name of the stream */
 	int type;			/* stream type */
 	u32 caps;			/* V4L2 capabilities */
 
 	struct v4l2_fh *fh;		/* pointer to the streaming filehandle */
-	spinlock_t qlock; 		/* locks access to the queues */
+	spinlock_t qlock;		/* locks access to the queues */
 	unsigned long s_flags;		/* status flags, see above */
 	int dma;			/* can be PCI_DMA_TODEVICE, PCI_DMA_FROMDEVICE or PCI_DMA_NONE */
 	u32 pending_offset;
@@ -564,7 +564,7 @@ struct vbi_info {
 
 	/* Raw VBI compatibility hack */
 
-	u32 frame; 				/* frame counter hack needed for backwards compatibility
+	u32 frame;				/* frame counter hack needed for backwards compatibility
 						   of old VBI software */
 
 	/* Sliced VBI output data */
@@ -620,7 +620,7 @@ struct ivtv {
 	u8 nof_inputs;			/* number of video inputs */
 	u8 nof_audio_inputs;		/* number of audio inputs */
 	u32 v4l2_cap;			/* V4L2 capabilities of card */
-	u32 hw_flags; 			/* hardware description of the board */
+	u32 hw_flags;			/* hardware description of the board */
 	v4l2_std_id tuner_std;		/* the norm of the card's tuner (fixed) */
 	struct v4l2_subdev *sd_video;	/* controlling video decoder subdev */
 	struct v4l2_subdev *sd_audio;	/* controlling audio subdev */
@@ -629,7 +629,7 @@ struct ivtv {
 	volatile void __iomem *enc_mem; /* pointer to mapped encoder memory */
 	volatile void __iomem *dec_mem; /* pointer to mapped decoder memory */
 	volatile void __iomem *reg_mem; /* pointer to mapped registers */
-	struct ivtv_options options; 	/* user options */
+	struct ivtv_options options;	/* user options */
 
 	struct v4l2_device v4l2_dev;
 	struct cx2341x_handler cxhdl;
@@ -668,7 +668,7 @@ struct ivtv {
 
 	/* Streams */
 	int stream_buf_size[IVTV_MAX_STREAMS];          /* stream buffer size */
-	struct ivtv_stream streams[IVTV_MAX_STREAMS]; 	/* stream data */
+	struct ivtv_stream streams[IVTV_MAX_STREAMS];	/* stream data */
 	atomic_t capturing;		/* count number of active capture streams */
 	atomic_t decoding;		/* count number of active decoding streams */
 
@@ -704,7 +704,7 @@ struct ivtv {
 	/* Mailbox */
 	struct ivtv_mailbox_data enc_mbox;              /* encoder mailboxes */
 	struct ivtv_mailbox_data dec_mbox;              /* decoder mailboxes */
-	struct ivtv_api_cache api_cache[256]; 		/* cached API commands */
+	struct ivtv_api_cache api_cache[256];		/* cached API commands */
 
 
 	/* I2C */
@@ -828,7 +828,7 @@ static inline int ivtv_raw_vbi(const struct ivtv *itv)
 
 /* Call the specified callback for all subdevs matching hw (if 0, then
    match them all). Ignore any errors. */
-#define ivtv_call_hw(itv, hw, o, f, args...) 				\
+#define ivtv_call_hw(itv, hw, o, f, args...)				\
 	v4l2_device_mask_call_all(&(itv)->v4l2_dev, hw, o, f, ##args)
 
 #define ivtv_call_all(itv, o, f, args...) ivtv_call_hw(itv, 0, o, f , ##args)
diff --git a/drivers/media/pci/ivtv/ivtv-fileops.c b/drivers/media/pci/ivtv/ivtv-fileops.c
index c9bd018e53de..6196daae4b3e 100644
--- a/drivers/media/pci/ivtv/ivtv-fileops.c
+++ b/drivers/media/pci/ivtv/ivtv-fileops.c
@@ -730,12 +730,12 @@ ssize_t ivtv_v4l2_write(struct file *filp, const char __user *user_buf, size_t c
 	return res;
 }
 
-unsigned int ivtv_v4l2_dec_poll(struct file *filp, poll_table *wait)
+__poll_t ivtv_v4l2_dec_poll(struct file *filp, poll_table *wait)
 {
 	struct ivtv_open_id *id = fh2id(filp->private_data);
 	struct ivtv *itv = id->itv;
 	struct ivtv_stream *s = &itv->streams[id->type];
-	int res = 0;
+	__poll_t res = 0;
 
 	/* add stream's waitq to the poll list */
 	IVTV_DEBUG_HI_FILE("Decoder poll\n");
@@ -747,7 +747,7 @@ unsigned int ivtv_v4l2_dec_poll(struct file *filp, poll_table *wait)
 		/* Turn off the old-style vsync events */
 		clear_bit(IVTV_F_I_EV_VSYNC_ENABLED, &itv->i_flags);
 		if (v4l2_event_pending(&id->fh))
-			res = POLLPRI;
+			res = EPOLLPRI;
 	} else {
 		/* This is the old-style API which is here only for backwards
 		   compatibility. */
@@ -755,28 +755,28 @@ unsigned int ivtv_v4l2_dec_poll(struct file *filp, poll_table *wait)
 		set_bit(IVTV_F_I_EV_VSYNC_ENABLED, &itv->i_flags);
 		if (test_bit(IVTV_F_I_EV_VSYNC, &itv->i_flags) ||
 		    test_bit(IVTV_F_I_EV_DEC_STOPPED, &itv->i_flags))
-			res = POLLPRI;
+			res = EPOLLPRI;
 	}
 
 	/* Allow write if buffers are available for writing */
 	if (s->q_free.buffers)
-		res |= POLLOUT | POLLWRNORM;
+		res |= EPOLLOUT | EPOLLWRNORM;
 	return res;
 }
 
-unsigned int ivtv_v4l2_enc_poll(struct file *filp, poll_table *wait)
+__poll_t ivtv_v4l2_enc_poll(struct file *filp, poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct ivtv_open_id *id = fh2id(filp->private_data);
 	struct ivtv *itv = id->itv;
 	struct ivtv_stream *s = &itv->streams[id->type];
 	int eof = test_bit(IVTV_F_S_STREAMOFF, &s->s_flags);
-	unsigned res = 0;
+	__poll_t res = 0;
 
 	/* Start a capture if there is none */
 	if (!eof && !test_bit(IVTV_F_S_STREAMING, &s->s_flags) &&
 			s->type != IVTV_ENC_STREAM_TYPE_RAD &&
-			(req_events & (POLLIN | POLLRDNORM))) {
+			(req_events & (EPOLLIN | EPOLLRDNORM))) {
 		int rc;
 
 		mutex_lock(&itv->serialize_lock);
@@ -785,7 +785,7 @@ unsigned int ivtv_v4l2_enc_poll(struct file *filp, poll_table *wait)
 		if (rc) {
 			IVTV_DEBUG_INFO("Could not start capture for %s (%d)\n",
 					s->name, rc);
-			return POLLERR;
+			return EPOLLERR;
 		}
 		IVTV_DEBUG_FILE("Encoder poll started capture\n");
 	}
@@ -794,14 +794,14 @@ unsigned int ivtv_v4l2_enc_poll(struct file *filp, poll_table *wait)
 	IVTV_DEBUG_HI_FILE("Encoder poll\n");
 	poll_wait(filp, &s->waitq, wait);
 	if (v4l2_event_pending(&id->fh))
-		res |= POLLPRI;
+		res |= EPOLLPRI;
 	else
 		poll_wait(filp, &id->fh.wait, wait);
 
 	if (s->q_full.length || s->q_io.length)
-		return res | POLLIN | POLLRDNORM;
+		return res | EPOLLIN | EPOLLRDNORM;
 	if (eof)
-		return res | POLLHUP;
+		return res | EPOLLHUP;
 	return res;
 }
 
diff --git a/drivers/media/pci/ivtv/ivtv-fileops.h b/drivers/media/pci/ivtv/ivtv-fileops.h
index 5e08800772ca..e0029b2b8d09 100644
--- a/drivers/media/pci/ivtv/ivtv-fileops.h
+++ b/drivers/media/pci/ivtv/ivtv-fileops.h
@@ -28,8 +28,8 @@ ssize_t ivtv_v4l2_read(struct file *filp, char __user *buf, size_t count,
 ssize_t ivtv_v4l2_write(struct file *filp, const char __user *buf, size_t count,
 		       loff_t * pos);
 int ivtv_v4l2_close(struct file *filp);
-unsigned int ivtv_v4l2_enc_poll(struct file *filp, poll_table * wait);
-unsigned int ivtv_v4l2_dec_poll(struct file *filp, poll_table * wait);
+__poll_t ivtv_v4l2_enc_poll(struct file *filp, poll_table * wait);
+__poll_t ivtv_v4l2_dec_poll(struct file *filp, poll_table * wait);
 int ivtv_start_capture(struct ivtv_open_id *id);
 void ivtv_stop_capture(struct ivtv_open_id *id, int gop_end);
 int ivtv_start_decoding(struct ivtv_open_id *id, int speed);
diff --git a/drivers/media/pci/ivtv/ivtv-firmware.c b/drivers/media/pci/ivtv/ivtv-firmware.c
index ba279fdb3df8..9f05472fca20 100644
--- a/drivers/media/pci/ivtv/ivtv-firmware.c
+++ b/drivers/media/pci/ivtv/ivtv-firmware.c
@@ -28,26 +28,26 @@
 #include <linux/firmware.h>
 #include <media/i2c/saa7127.h>
 
-#define IVTV_MASK_SPU_ENABLE 		0xFFFFFFFE
-#define IVTV_MASK_VPU_ENABLE15 		0xFFFFFFF6
-#define IVTV_MASK_VPU_ENABLE16 		0xFFFFFFFB
-#define IVTV_CMD_VDM_STOP 		0x00000000
-#define IVTV_CMD_AO_STOP 		0x00000005
-#define IVTV_CMD_APU_PING 		0x00000000
-#define IVTV_CMD_VPU_STOP15 		0xFFFFFFFE
-#define IVTV_CMD_VPU_STOP16 		0xFFFFFFEE
-#define IVTV_CMD_HW_BLOCKS_RST 		0xFFFFFFFF
-#define IVTV_CMD_SPU_STOP 		0x00000001
-#define IVTV_CMD_SDRAM_PRECHARGE_INIT 	0x0000001A
-#define IVTV_CMD_SDRAM_REFRESH_INIT 	0x80000640
-#define IVTV_SDRAM_SLEEPTIME 		600
-
-#define IVTV_DECODE_INIT_MPEG_FILENAME 	"v4l-cx2341x-init.mpg"
-#define IVTV_DECODE_INIT_MPEG_SIZE 	(152*1024)
+#define IVTV_MASK_SPU_ENABLE		0xFFFFFFFE
+#define IVTV_MASK_VPU_ENABLE15		0xFFFFFFF6
+#define IVTV_MASK_VPU_ENABLE16		0xFFFFFFFB
+#define IVTV_CMD_VDM_STOP		0x00000000
+#define IVTV_CMD_AO_STOP		0x00000005
+#define IVTV_CMD_APU_PING		0x00000000
+#define IVTV_CMD_VPU_STOP15		0xFFFFFFFE
+#define IVTV_CMD_VPU_STOP16		0xFFFFFFEE
+#define IVTV_CMD_HW_BLOCKS_RST		0xFFFFFFFF
+#define IVTV_CMD_SPU_STOP		0x00000001
+#define IVTV_CMD_SDRAM_PRECHARGE_INIT	0x0000001A
+#define IVTV_CMD_SDRAM_REFRESH_INIT	0x80000640
+#define IVTV_SDRAM_SLEEPTIME		600
+
+#define IVTV_DECODE_INIT_MPEG_FILENAME	"v4l-cx2341x-init.mpg"
+#define IVTV_DECODE_INIT_MPEG_SIZE	(152*1024)
 
 /* Encoder/decoder firmware sizes */
-#define IVTV_FW_ENC_SIZE 		(376836)
-#define IVTV_FW_DEC_SIZE 		(256*1024)
+#define IVTV_FW_ENC_SIZE		(376836)
+#define IVTV_FW_DEC_SIZE		(256*1024)
 
 static int load_fw_direct(const char *fn, volatile u8 __iomem *mem, struct ivtv *itv, long size)
 {
diff --git a/drivers/media/pci/ivtv/ivtv-i2c.c b/drivers/media/pci/ivtv/ivtv-i2c.c
index 893962ac85de..522cd111e399 100644
--- a/drivers/media/pci/ivtv/ivtv-i2c.c
+++ b/drivers/media/pci/ivtv/ivtv-i2c.c
@@ -76,22 +76,22 @@
 
 #define IVTV_CS53L32A_I2C_ADDR		0x11
 #define IVTV_M52790_I2C_ADDR		0x48
-#define IVTV_CX25840_I2C_ADDR 		0x44
-#define IVTV_SAA7115_I2C_ADDR 		0x21
-#define IVTV_SAA7127_I2C_ADDR 		0x44
-#define IVTV_SAA717x_I2C_ADDR 		0x21
-#define IVTV_MSP3400_I2C_ADDR 		0x40
-#define IVTV_HAUPPAUGE_I2C_ADDR 	0x50
-#define IVTV_WM8739_I2C_ADDR 		0x1a
+#define IVTV_CX25840_I2C_ADDR		0x44
+#define IVTV_SAA7115_I2C_ADDR		0x21
+#define IVTV_SAA7127_I2C_ADDR		0x44
+#define IVTV_SAA717x_I2C_ADDR		0x21
+#define IVTV_MSP3400_I2C_ADDR		0x40
+#define IVTV_HAUPPAUGE_I2C_ADDR		0x50
+#define IVTV_WM8739_I2C_ADDR		0x1a
 #define IVTV_WM8775_I2C_ADDR		0x1b
 #define IVTV_TEA5767_I2C_ADDR		0x60
-#define IVTV_UPD64031A_I2C_ADDR 	0x12
-#define IVTV_UPD64083_I2C_ADDR 		0x5c
-#define IVTV_VP27SMPX_I2C_ADDR      	0x5b
-#define IVTV_M52790_I2C_ADDR      	0x48
+#define IVTV_UPD64031A_I2C_ADDR		0x12
+#define IVTV_UPD64083_I2C_ADDR		0x5c
+#define IVTV_VP27SMPX_I2C_ADDR		0x5b
+#define IVTV_M52790_I2C_ADDR		0x48
 #define IVTV_AVERMEDIA_IR_RX_I2C_ADDR	0x40
-#define IVTV_HAUP_EXT_IR_RX_I2C_ADDR 	0x1a
-#define IVTV_HAUP_INT_IR_RX_I2C_ADDR 	0x18
+#define IVTV_HAUP_EXT_IR_RX_I2C_ADDR	0x1a
+#define IVTV_HAUP_INT_IR_RX_I2C_ADDR	0x18
 #define IVTV_Z8F0811_IR_TX_I2C_ADDR	0x70
 #define IVTV_Z8F0811_IR_RX_I2C_ADDR	0x71
 #define IVTV_ADAPTEC_IR_ADDR		0x6b
@@ -117,8 +117,7 @@ static const u8 hw_addrs[] = {
 	IVTV_AVERMEDIA_IR_RX_I2C_ADDR,	/* IVTV_HW_I2C_IR_RX_AVER */
 	IVTV_HAUP_EXT_IR_RX_I2C_ADDR,	/* IVTV_HW_I2C_IR_RX_HAUP_EXT */
 	IVTV_HAUP_INT_IR_RX_I2C_ADDR,	/* IVTV_HW_I2C_IR_RX_HAUP_INT */
-	IVTV_Z8F0811_IR_TX_I2C_ADDR,	/* IVTV_HW_Z8F0811_IR_TX_HAUP */
-	IVTV_Z8F0811_IR_RX_I2C_ADDR,	/* IVTV_HW_Z8F0811_IR_RX_HAUP */
+	IVTV_Z8F0811_IR_RX_I2C_ADDR,	/* IVTV_HW_Z8F0811_IR_HAUP */
 	IVTV_ADAPTEC_IR_ADDR,		/* IVTV_HW_I2C_IR_RX_ADAPTEC */
 };
 
@@ -143,8 +142,7 @@ static const char * const hw_devicenames[] = {
 	"ir_video",		/* IVTV_HW_I2C_IR_RX_AVER */
 	"ir_video",		/* IVTV_HW_I2C_IR_RX_HAUP_EXT */
 	"ir_video",		/* IVTV_HW_I2C_IR_RX_HAUP_INT */
-	"ir_tx_z8f0811_haup",	/* IVTV_HW_Z8F0811_IR_TX_HAUP */
-	"ir_rx_z8f0811_haup",	/* IVTV_HW_Z8F0811_IR_RX_HAUP */
+	"ir_z8f0811_haup",	/* IVTV_HW_Z8F0811_IR_HAUP */
 	"ir_video",		/* IVTV_HW_I2C_IR_RX_ADAPTEC */
 };
 
@@ -181,18 +179,8 @@ static int ivtv_i2c_new_ir(struct ivtv *itv, u32 hw, const char *type, u8 addr)
 	struct IR_i2c_init_data *init_data = &itv->ir_i2c_init_data;
 	unsigned short addr_list[2] = { addr, I2C_CLIENT_END };
 
-	/* Only allow one IR transmitter to be registered per board */
-	if (hw & IVTV_HW_IR_TX_ANY) {
-		if (itv->hw_flags & IVTV_HW_IR_TX_ANY)
-			return -1;
-		memset(&info, 0, sizeof(struct i2c_board_info));
-		strlcpy(info.type, type, I2C_NAME_SIZE);
-		return i2c_new_probed_device(adap, &info, addr_list, NULL)
-							   == NULL ? -1 : 0;
-	}
-
 	/* Only allow one IR receiver to be registered per board */
-	if (itv->hw_flags & IVTV_HW_IR_RX_ANY)
+	if (itv->hw_flags & IVTV_HW_IR_ANY)
 		return -1;
 
 	/* Our default information for ir-kbd-i2c.c to use */
@@ -211,7 +199,7 @@ static int ivtv_i2c_new_ir(struct ivtv *itv, u32 hw, const char *type, u8 addr)
 		init_data->type = RC_PROTO_BIT_RC5;
 		init_data->name = itv->card_name;
 		break;
-	case IVTV_HW_Z8F0811_IR_RX_HAUP:
+	case IVTV_HW_Z8F0811_IR_HAUP:
 		/* Default to grey remote */
 		init_data->ir_codes = RC_MAP_HAUPPAUGE;
 		init_data->internal_get_key_func = IR_KBD_GET_KEY_HAUP_XVR;
diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c
index 670462d195b5..4cdc6d2be85d 100644
--- a/drivers/media/pci/ivtv/ivtv-ioctl.c
+++ b/drivers/media/pci/ivtv/ivtv-ioctl.c
@@ -1884,65 +1884,65 @@ static long ivtv_default(struct file *file, void *fh, bool valid_prio,
 }
 
 static const struct v4l2_ioctl_ops ivtv_ioctl_ops = {
-	.vidioc_querycap    		    = ivtv_querycap,
-	.vidioc_s_audio     		    = ivtv_s_audio,
-	.vidioc_g_audio     		    = ivtv_g_audio,
-	.vidioc_enumaudio   		    = ivtv_enumaudio,
-	.vidioc_s_audout     		    = ivtv_s_audout,
-	.vidioc_g_audout     		    = ivtv_g_audout,
-	.vidioc_enum_input   		    = ivtv_enum_input,
-	.vidioc_enum_output   		    = ivtv_enum_output,
-	.vidioc_enumaudout   		    = ivtv_enumaudout,
-	.vidioc_cropcap       		    = ivtv_cropcap,
+	.vidioc_querycap		    = ivtv_querycap,
+	.vidioc_s_audio			    = ivtv_s_audio,
+	.vidioc_g_audio			    = ivtv_g_audio,
+	.vidioc_enumaudio		    = ivtv_enumaudio,
+	.vidioc_s_audout		    = ivtv_s_audout,
+	.vidioc_g_audout		    = ivtv_g_audout,
+	.vidioc_enum_input		    = ivtv_enum_input,
+	.vidioc_enum_output		    = ivtv_enum_output,
+	.vidioc_enumaudout		    = ivtv_enumaudout,
+	.vidioc_cropcap			    = ivtv_cropcap,
 	.vidioc_s_selection		    = ivtv_s_selection,
 	.vidioc_g_selection		    = ivtv_g_selection,
-	.vidioc_g_input      		    = ivtv_g_input,
-	.vidioc_s_input      		    = ivtv_s_input,
-	.vidioc_g_output     		    = ivtv_g_output,
-	.vidioc_s_output     		    = ivtv_s_output,
-	.vidioc_g_frequency 		    = ivtv_g_frequency,
-	.vidioc_s_frequency  		    = ivtv_s_frequency,
-	.vidioc_s_tuner      		    = ivtv_s_tuner,
-	.vidioc_g_tuner      		    = ivtv_g_tuner,
-	.vidioc_g_enc_index 		    = ivtv_g_enc_index,
+	.vidioc_g_input			    = ivtv_g_input,
+	.vidioc_s_input			    = ivtv_s_input,
+	.vidioc_g_output		    = ivtv_g_output,
+	.vidioc_s_output		    = ivtv_s_output,
+	.vidioc_g_frequency		    = ivtv_g_frequency,
+	.vidioc_s_frequency		    = ivtv_s_frequency,
+	.vidioc_s_tuner			    = ivtv_s_tuner,
+	.vidioc_g_tuner			    = ivtv_g_tuner,
+	.vidioc_g_enc_index		    = ivtv_g_enc_index,
 	.vidioc_g_fbuf			    = ivtv_g_fbuf,
 	.vidioc_s_fbuf			    = ivtv_s_fbuf,
-	.vidioc_g_std 			    = ivtv_g_std,
-	.vidioc_s_std 			    = ivtv_s_std,
+	.vidioc_g_std			    = ivtv_g_std,
+	.vidioc_s_std			    = ivtv_s_std,
 	.vidioc_overlay			    = ivtv_overlay,
 	.vidioc_log_status		    = ivtv_log_status,
-	.vidioc_enum_fmt_vid_cap 	    = ivtv_enum_fmt_vid_cap,
-	.vidioc_encoder_cmd  		    = ivtv_encoder_cmd,
-	.vidioc_try_encoder_cmd 	    = ivtv_try_encoder_cmd,
+	.vidioc_enum_fmt_vid_cap	    = ivtv_enum_fmt_vid_cap,
+	.vidioc_encoder_cmd		    = ivtv_encoder_cmd,
+	.vidioc_try_encoder_cmd		    = ivtv_try_encoder_cmd,
 	.vidioc_decoder_cmd		    = ivtv_decoder_cmd,
 	.vidioc_try_decoder_cmd		    = ivtv_try_decoder_cmd,
-	.vidioc_enum_fmt_vid_out 	    = ivtv_enum_fmt_vid_out,
-	.vidioc_g_fmt_vid_cap 		    = ivtv_g_fmt_vid_cap,
+	.vidioc_enum_fmt_vid_out	    = ivtv_enum_fmt_vid_out,
+	.vidioc_g_fmt_vid_cap		    = ivtv_g_fmt_vid_cap,
 	.vidioc_g_fmt_vbi_cap		    = ivtv_g_fmt_vbi_cap,
 	.vidioc_g_fmt_sliced_vbi_cap        = ivtv_g_fmt_sliced_vbi_cap,
 	.vidioc_g_fmt_vid_out               = ivtv_g_fmt_vid_out,
 	.vidioc_g_fmt_vid_out_overlay       = ivtv_g_fmt_vid_out_overlay,
 	.vidioc_g_fmt_sliced_vbi_out        = ivtv_g_fmt_sliced_vbi_out,
-	.vidioc_s_fmt_vid_cap  		    = ivtv_s_fmt_vid_cap,
-	.vidioc_s_fmt_vbi_cap 		    = ivtv_s_fmt_vbi_cap,
+	.vidioc_s_fmt_vid_cap		    = ivtv_s_fmt_vid_cap,
+	.vidioc_s_fmt_vbi_cap		    = ivtv_s_fmt_vbi_cap,
 	.vidioc_s_fmt_sliced_vbi_cap        = ivtv_s_fmt_sliced_vbi_cap,
 	.vidioc_s_fmt_vid_out               = ivtv_s_fmt_vid_out,
 	.vidioc_s_fmt_vid_out_overlay       = ivtv_s_fmt_vid_out_overlay,
 	.vidioc_s_fmt_sliced_vbi_out        = ivtv_s_fmt_sliced_vbi_out,
-	.vidioc_try_fmt_vid_cap  	    = ivtv_try_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap		    = ivtv_try_fmt_vid_cap,
 	.vidioc_try_fmt_vbi_cap		    = ivtv_try_fmt_vbi_cap,
 	.vidioc_try_fmt_sliced_vbi_cap      = ivtv_try_fmt_sliced_vbi_cap,
-	.vidioc_try_fmt_vid_out 	    = ivtv_try_fmt_vid_out,
+	.vidioc_try_fmt_vid_out		    = ivtv_try_fmt_vid_out,
 	.vidioc_try_fmt_vid_out_overlay     = ivtv_try_fmt_vid_out_overlay,
-	.vidioc_try_fmt_sliced_vbi_out 	    = ivtv_try_fmt_sliced_vbi_out,
-	.vidioc_g_sliced_vbi_cap 	    = ivtv_g_sliced_vbi_cap,
+	.vidioc_try_fmt_sliced_vbi_out	    = ivtv_try_fmt_sliced_vbi_out,
+	.vidioc_g_sliced_vbi_cap	    = ivtv_g_sliced_vbi_cap,
 #ifdef CONFIG_VIDEO_ADV_DEBUG
-	.vidioc_g_register 		    = ivtv_g_register,
-	.vidioc_s_register 		    = ivtv_s_register,
+	.vidioc_g_register		    = ivtv_g_register,
+	.vidioc_s_register		    = ivtv_s_register,
 #endif
-	.vidioc_default 		    = ivtv_default,
-	.vidioc_subscribe_event 	    = ivtv_subscribe_event,
-	.vidioc_unsubscribe_event 	    = v4l2_event_unsubscribe,
+	.vidioc_default			    = ivtv_default,
+	.vidioc_subscribe_event		    = ivtv_subscribe_event,
+	.vidioc_unsubscribe_event	    = v4l2_event_unsubscribe,
 };
 
 void ivtv_set_funcs(struct video_device *vdev)
diff --git a/drivers/media/pci/ivtv/ivtv-mailbox.c b/drivers/media/pci/ivtv/ivtv-mailbox.c
index 9a2506a5edbe..f317c8f0938d 100644
--- a/drivers/media/pci/ivtv/ivtv-mailbox.c
+++ b/drivers/media/pci/ivtv/ivtv-mailbox.c
@@ -28,118 +28,118 @@
 #define IVTV_MBOX_FIRMWARE_DONE 0x00000004
 #define IVTV_MBOX_DRIVER_DONE   0x00000002
 #define IVTV_MBOX_DRIVER_BUSY   0x00000001
-#define IVTV_MBOX_FREE 		0x00000000
+#define IVTV_MBOX_FREE		0x00000000
 
 /* Firmware mailbox standard timeout */
-#define IVTV_API_STD_TIMEOUT 	0x02000000
+#define IVTV_API_STD_TIMEOUT	0x02000000
 
-#define API_CACHE 	 (1 << 0) 	/* Allow the command to be stored in the cache */
-#define API_RESULT	 (1 << 1) 	/* Allow 1 second for this cmd to end */
+#define API_CACHE	 (1 << 0)	/* Allow the command to be stored in the cache */
+#define API_RESULT	 (1 << 1)	/* Allow 1 second for this cmd to end */
 #define API_FAST_RESULT	 (3 << 1)	/* Allow 0.1 second for this cmd to end */
-#define API_DMA 	 (1 << 3)	/* DMA mailbox, has special handling */
-#define API_HIGH_VOL 	 (1 << 5)	/* High volume command (i.e. called during encoding or decoding) */
-#define API_NO_WAIT_MB 	 (1 << 4)	/* Command may not wait for a free mailbox */
+#define API_DMA		 (1 << 3)	/* DMA mailbox, has special handling */
+#define API_HIGH_VOL	 (1 << 5)	/* High volume command (i.e. called during encoding or decoding) */
+#define API_NO_WAIT_MB	 (1 << 4)	/* Command may not wait for a free mailbox */
 #define API_NO_WAIT_RES	 (1 << 5)	/* Command may not wait for the result */
 #define API_NO_POLL	 (1 << 6)	/* Avoid pointless polling */
 
 struct ivtv_api_info {
 	int flags;		/* Flags, see above */
-	const char *name; 	/* The name of the command */
+	const char *name;	/* The name of the command */
 };
 
 #define API_ENTRY(x, f) [x] = { (f), #x }
 
 static const struct ivtv_api_info api_info[256] = {
 	/* MPEG encoder API */
-	API_ENTRY(CX2341X_ENC_PING_FW, 			API_FAST_RESULT),
-	API_ENTRY(CX2341X_ENC_START_CAPTURE, 		API_RESULT | API_NO_POLL),
-	API_ENTRY(CX2341X_ENC_STOP_CAPTURE, 		API_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_AUDIO_ID, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_VIDEO_ID, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_PCR_ID, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_FRAME_RATE, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_FRAME_SIZE, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_BIT_RATE, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_GOP_PROPERTIES, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_ASPECT_RATIO, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_DNR_FILTER_MODE, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_DNR_FILTER_PROPS, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_CORING_LEVELS, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_SPATIAL_FILTER_TYPE, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_VBI_LINE, 		API_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_STREAM_TYPE, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_OUTPUT_PORT, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_AUDIO_PROPERTIES, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_HALT_FW, 			API_FAST_RESULT),
-	API_ENTRY(CX2341X_ENC_GET_VERSION, 		API_FAST_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_GOP_CLOSURE, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_GET_SEQ_END, 		API_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_PGM_INDEX_INFO, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_VBI_CONFIG, 		API_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_DMA_BLOCK_SIZE, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_GET_PREV_DMA_INFO_MB_10, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_ENC_GET_PREV_DMA_INFO_MB_9, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_ENC_SCHED_DMA_TO_HOST, 	API_DMA | API_HIGH_VOL),
-	API_ENTRY(CX2341X_ENC_INITIALIZE_INPUT, 	API_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_FRAME_DROP_RATE, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_PAUSE_ENCODER, 		API_RESULT),
-	API_ENTRY(CX2341X_ENC_REFRESH_INPUT, 		API_NO_WAIT_MB | API_HIGH_VOL),
-	API_ENTRY(CX2341X_ENC_SET_COPYRIGHT, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_EVENT_NOTIFICATION, 	API_RESULT),
-	API_ENTRY(CX2341X_ENC_SET_NUM_VSYNC_LINES, 	API_CACHE),
-	API_ENTRY(CX2341X_ENC_SET_PLACEHOLDER, 		API_CACHE),
-	API_ENTRY(CX2341X_ENC_MUTE_VIDEO, 		API_RESULT),
-	API_ENTRY(CX2341X_ENC_MUTE_AUDIO, 		API_RESULT),
+	API_ENTRY(CX2341X_ENC_PING_FW,			API_FAST_RESULT),
+	API_ENTRY(CX2341X_ENC_START_CAPTURE,		API_RESULT | API_NO_POLL),
+	API_ENTRY(CX2341X_ENC_STOP_CAPTURE,		API_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_AUDIO_ID,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_VIDEO_ID,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_PCR_ID,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_FRAME_RATE,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_FRAME_SIZE,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_BIT_RATE,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_GOP_PROPERTIES,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_ASPECT_RATIO,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_DNR_FILTER_MODE,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_DNR_FILTER_PROPS,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_CORING_LEVELS,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_SPATIAL_FILTER_TYPE,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_VBI_LINE,		API_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_STREAM_TYPE,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_OUTPUT_PORT,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_AUDIO_PROPERTIES,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_HALT_FW,			API_FAST_RESULT),
+	API_ENTRY(CX2341X_ENC_GET_VERSION,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_GOP_CLOSURE,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_GET_SEQ_END,		API_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_PGM_INDEX_INFO,	API_FAST_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_VBI_CONFIG,		API_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_DMA_BLOCK_SIZE,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_GET_PREV_DMA_INFO_MB_10,	API_FAST_RESULT),
+	API_ENTRY(CX2341X_ENC_GET_PREV_DMA_INFO_MB_9,	API_FAST_RESULT),
+	API_ENTRY(CX2341X_ENC_SCHED_DMA_TO_HOST,	API_DMA | API_HIGH_VOL),
+	API_ENTRY(CX2341X_ENC_INITIALIZE_INPUT,		API_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_FRAME_DROP_RATE,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_PAUSE_ENCODER,		API_RESULT),
+	API_ENTRY(CX2341X_ENC_REFRESH_INPUT,		API_NO_WAIT_MB | API_HIGH_VOL),
+	API_ENTRY(CX2341X_ENC_SET_COPYRIGHT,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_EVENT_NOTIFICATION,	API_RESULT),
+	API_ENTRY(CX2341X_ENC_SET_NUM_VSYNC_LINES,	API_CACHE),
+	API_ENTRY(CX2341X_ENC_SET_PLACEHOLDER,		API_CACHE),
+	API_ENTRY(CX2341X_ENC_MUTE_VIDEO,		API_RESULT),
+	API_ENTRY(CX2341X_ENC_MUTE_AUDIO,		API_RESULT),
 	API_ENTRY(CX2341X_ENC_SET_VERT_CROP_LINE,	API_FAST_RESULT),
-	API_ENTRY(CX2341X_ENC_MISC, 			API_FAST_RESULT),
+	API_ENTRY(CX2341X_ENC_MISC,			API_FAST_RESULT),
 	/* Obsolete PULLDOWN API command */
-	API_ENTRY(0xb1, 				API_CACHE),
+	API_ENTRY(0xb1,					API_CACHE),
 
 	/* MPEG decoder API */
-	API_ENTRY(CX2341X_DEC_PING_FW, 			API_FAST_RESULT),
-	API_ENTRY(CX2341X_DEC_START_PLAYBACK, 		API_RESULT | API_NO_POLL),
-	API_ENTRY(CX2341X_DEC_STOP_PLAYBACK, 		API_RESULT),
-	API_ENTRY(CX2341X_DEC_SET_PLAYBACK_SPEED, 	API_RESULT),
-	API_ENTRY(CX2341X_DEC_STEP_VIDEO, 		API_RESULT),
-	API_ENTRY(CX2341X_DEC_SET_DMA_BLOCK_SIZE, 	API_CACHE),
-	API_ENTRY(CX2341X_DEC_GET_XFER_INFO, 		API_FAST_RESULT),
-	API_ENTRY(CX2341X_DEC_GET_DMA_STATUS, 		API_FAST_RESULT),
-	API_ENTRY(CX2341X_DEC_SCHED_DMA_FROM_HOST, 	API_DMA | API_HIGH_VOL),
-	API_ENTRY(CX2341X_DEC_PAUSE_PLAYBACK, 		API_RESULT),
-	API_ENTRY(CX2341X_DEC_HALT_FW, 			API_FAST_RESULT),
-	API_ENTRY(CX2341X_DEC_SET_STANDARD, 		API_CACHE),
-	API_ENTRY(CX2341X_DEC_GET_VERSION, 		API_FAST_RESULT),
-	API_ENTRY(CX2341X_DEC_SET_STREAM_INPUT, 	API_CACHE),
-	API_ENTRY(CX2341X_DEC_GET_TIMING_INFO, 		API_RESULT /*| API_NO_WAIT_RES*/),
-	API_ENTRY(CX2341X_DEC_SET_AUDIO_MODE, 		API_CACHE),
-	API_ENTRY(CX2341X_DEC_SET_EVENT_NOTIFICATION, 	API_RESULT),
-	API_ENTRY(CX2341X_DEC_SET_DISPLAY_BUFFERS, 	API_CACHE),
-	API_ENTRY(CX2341X_DEC_EXTRACT_VBI, 		API_RESULT),
-	API_ENTRY(CX2341X_DEC_SET_DECODER_SOURCE, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_DEC_SET_PREBUFFERING, 	API_CACHE),
+	API_ENTRY(CX2341X_DEC_PING_FW,			API_FAST_RESULT),
+	API_ENTRY(CX2341X_DEC_START_PLAYBACK,		API_RESULT | API_NO_POLL),
+	API_ENTRY(CX2341X_DEC_STOP_PLAYBACK,		API_RESULT),
+	API_ENTRY(CX2341X_DEC_SET_PLAYBACK_SPEED,	API_RESULT),
+	API_ENTRY(CX2341X_DEC_STEP_VIDEO,		API_RESULT),
+	API_ENTRY(CX2341X_DEC_SET_DMA_BLOCK_SIZE,	API_CACHE),
+	API_ENTRY(CX2341X_DEC_GET_XFER_INFO,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_DEC_GET_DMA_STATUS,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_DEC_SCHED_DMA_FROM_HOST,	API_DMA | API_HIGH_VOL),
+	API_ENTRY(CX2341X_DEC_PAUSE_PLAYBACK,		API_RESULT),
+	API_ENTRY(CX2341X_DEC_HALT_FW,			API_FAST_RESULT),
+	API_ENTRY(CX2341X_DEC_SET_STANDARD,		API_CACHE),
+	API_ENTRY(CX2341X_DEC_GET_VERSION,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_DEC_SET_STREAM_INPUT,		API_CACHE),
+	API_ENTRY(CX2341X_DEC_GET_TIMING_INFO,		API_RESULT /*| API_NO_WAIT_RES*/),
+	API_ENTRY(CX2341X_DEC_SET_AUDIO_MODE,		API_CACHE),
+	API_ENTRY(CX2341X_DEC_SET_EVENT_NOTIFICATION,	API_RESULT),
+	API_ENTRY(CX2341X_DEC_SET_DISPLAY_BUFFERS,	API_CACHE),
+	API_ENTRY(CX2341X_DEC_EXTRACT_VBI,		API_RESULT),
+	API_ENTRY(CX2341X_DEC_SET_DECODER_SOURCE,	API_FAST_RESULT),
+	API_ENTRY(CX2341X_DEC_SET_PREBUFFERING,		API_CACHE),
 
 	/* OSD API */
-	API_ENTRY(CX2341X_OSD_GET_FRAMEBUFFER, 		API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_GET_PIXEL_FORMAT, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_PIXEL_FORMAT, 	API_CACHE),
-	API_ENTRY(CX2341X_OSD_GET_STATE, 		API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_STATE, 		API_CACHE),
-	API_ENTRY(CX2341X_OSD_GET_OSD_COORDS, 		API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_OSD_COORDS, 		API_CACHE),
-	API_ENTRY(CX2341X_OSD_GET_SCREEN_COORDS, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_SCREEN_COORDS, 	API_CACHE),
-	API_ENTRY(CX2341X_OSD_GET_GLOBAL_ALPHA, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_GLOBAL_ALPHA, 	API_CACHE),
-	API_ENTRY(CX2341X_OSD_SET_BLEND_COORDS, 	API_CACHE),
-	API_ENTRY(CX2341X_OSD_GET_FLICKER_STATE, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_FLICKER_STATE, 	API_CACHE),
-	API_ENTRY(CX2341X_OSD_BLT_COPY, 		API_RESULT),
-	API_ENTRY(CX2341X_OSD_BLT_FILL, 		API_RESULT),
-	API_ENTRY(CX2341X_OSD_BLT_TEXT, 		API_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_FRAMEBUFFER_WINDOW, 	API_CACHE),
-	API_ENTRY(CX2341X_OSD_SET_CHROMA_KEY, 		API_CACHE),
-	API_ENTRY(CX2341X_OSD_GET_ALPHA_CONTENT_INDEX, 	API_FAST_RESULT),
-	API_ENTRY(CX2341X_OSD_SET_ALPHA_CONTENT_INDEX, 	API_CACHE)
+	API_ENTRY(CX2341X_OSD_GET_FRAMEBUFFER,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_GET_PIXEL_FORMAT,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_PIXEL_FORMAT,		API_CACHE),
+	API_ENTRY(CX2341X_OSD_GET_STATE,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_STATE,		API_CACHE),
+	API_ENTRY(CX2341X_OSD_GET_OSD_COORDS,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_OSD_COORDS,		API_CACHE),
+	API_ENTRY(CX2341X_OSD_GET_SCREEN_COORDS,	API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_SCREEN_COORDS,	API_CACHE),
+	API_ENTRY(CX2341X_OSD_GET_GLOBAL_ALPHA,		API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_GLOBAL_ALPHA,		API_CACHE),
+	API_ENTRY(CX2341X_OSD_SET_BLEND_COORDS,		API_CACHE),
+	API_ENTRY(CX2341X_OSD_GET_FLICKER_STATE,	API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_FLICKER_STATE,	API_CACHE),
+	API_ENTRY(CX2341X_OSD_BLT_COPY,			API_RESULT),
+	API_ENTRY(CX2341X_OSD_BLT_FILL,			API_RESULT),
+	API_ENTRY(CX2341X_OSD_BLT_TEXT,			API_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_FRAMEBUFFER_WINDOW,	API_CACHE),
+	API_ENTRY(CX2341X_OSD_SET_CHROMA_KEY,		API_CACHE),
+	API_ENTRY(CX2341X_OSD_GET_ALPHA_CONTENT_INDEX,	API_FAST_RESULT),
+	API_ENTRY(CX2341X_OSD_SET_ALPHA_CONTENT_INDEX,	API_CACHE)
 };
 
 static int try_mailbox(struct ivtv *itv, struct ivtv_mailbox_data *mbdata, int mb)
diff --git a/drivers/media/pci/mantis/Makefile b/drivers/media/pci/mantis/Makefile
index a684dc2ec79e..b5ef39692cb0 100644
--- a/drivers/media/pci/mantis/Makefile
+++ b/drivers/media/pci/mantis/Makefile
@@ -26,4 +26,4 @@ obj-$(CONFIG_MANTIS_CORE)	+= mantis_core.o
 obj-$(CONFIG_DVB_MANTIS)	+= mantis.o
 obj-$(CONFIG_DVB_HOPPER)	+= hopper.o
 
-ccflags-y += -Idrivers/media/dvb-core/ -Idrivers/media/dvb-frontends/
+ccflags-y += -Idrivers/media/dvb-frontends/
diff --git a/drivers/media/pci/mantis/hopper_cards.c b/drivers/media/pci/mantis/hopper_cards.c
index ed855e3df558..89759cb80ecb 100644
--- a/drivers/media/pci/mantis/hopper_cards.c
+++ b/drivers/media/pci/mantis/hopper_cards.c
@@ -26,11 +26,11 @@
 #include <asm/irq.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "hopper_vp3028.h"
diff --git a/drivers/media/pci/mantis/hopper_vp3028.c b/drivers/media/pci/mantis/hopper_vp3028.c
index 1032db6bb789..d58ae0097fea 100644
--- a/drivers/media/pci/mantis/hopper_vp3028.c
+++ b/drivers/media/pci/mantis/hopper_vp3028.c
@@ -22,11 +22,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "zl10353.h"
 #include "mantis_common.h"
diff --git a/drivers/media/pci/mantis/mantis_ca.c b/drivers/media/pci/mantis/mantis_ca.c
index 60c6c2f24066..4f0ba457c7e5 100644
--- a/drivers/media/pci/mantis/mantis_ca.c
+++ b/drivers/media/pci/mantis/mantis_ca.c
@@ -24,11 +24,11 @@
 #include <linux/interrupt.h>
 #include <asm/io.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_link.h"
diff --git a/drivers/media/pci/mantis/mantis_cards.c b/drivers/media/pci/mantis/mantis_cards.c
index 4ce8a90d69dc..7eb75cb7d75a 100644
--- a/drivers/media/pci/mantis/mantis_cards.c
+++ b/drivers/media/pci/mantis/mantis_cards.c
@@ -27,11 +27,11 @@
 #include <linux/interrupt.h>
 #include <media/rc-map.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 
diff --git a/drivers/media/pci/mantis/mantis_dma.c b/drivers/media/pci/mantis/mantis_dma.c
index 2ce310b0a022..84406a428330 100644
--- a/drivers/media/pci/mantis/mantis_dma.c
+++ b/drivers/media/pci/mantis/mantis_dma.c
@@ -28,11 +28,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_reg.h"
diff --git a/drivers/media/pci/mantis/mantis_dvb.c b/drivers/media/pci/mantis/mantis_dvb.c
index 0db4de3a2285..54dbaa700fa3 100644
--- a/drivers/media/pci/mantis/mantis_dvb.c
+++ b/drivers/media/pci/mantis/mantis_dvb.c
@@ -26,11 +26,11 @@
 #include <linux/pci.h>
 #include <linux/i2c.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_dma.h"
diff --git a/drivers/media/pci/mantis/mantis_evm.c b/drivers/media/pci/mantis/mantis_evm.c
index 909ff54868a3..443ac5ab4902 100644
--- a/drivers/media/pci/mantis/mantis_evm.c
+++ b/drivers/media/pci/mantis/mantis_evm.c
@@ -25,11 +25,11 @@
 #include <linux/interrupt.h>
 #include <asm/io.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_link.h"
diff --git a/drivers/media/pci/mantis/mantis_hif.c b/drivers/media/pci/mantis/mantis_hif.c
index 10c68df7e16f..bf61f8c5a59f 100644
--- a/drivers/media/pci/mantis/mantis_hif.c
+++ b/drivers/media/pci/mantis/mantis_hif.c
@@ -25,11 +25,11 @@
 #include <linux/interrupt.h>
 #include <asm/io.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 
diff --git a/drivers/media/pci/mantis/mantis_i2c.c b/drivers/media/pci/mantis/mantis_i2c.c
index 496c10dfc4df..6528a2180119 100644
--- a/drivers/media/pci/mantis/mantis_i2c.c
+++ b/drivers/media/pci/mantis/mantis_i2c.c
@@ -23,11 +23,11 @@
 #include <linux/pci.h>
 #include <linux/i2c.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_reg.h"
diff --git a/drivers/media/pci/mantis/mantis_input.c b/drivers/media/pci/mantis/mantis_input.c
index 7519dcc934dd..5b472e9b9542 100644
--- a/drivers/media/pci/mantis/mantis_input.c
+++ b/drivers/media/pci/mantis/mantis_input.c
@@ -17,11 +17,11 @@
 #include <media/rc-core.h>
 #include <linux/pci.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_input.h"
diff --git a/drivers/media/pci/mantis/mantis_ioc.c b/drivers/media/pci/mantis/mantis_ioc.c
index 24fcdc63d6d5..f45c2340a493 100644
--- a/drivers/media/pci/mantis/mantis_ioc.c
+++ b/drivers/media/pci/mantis/mantis_ioc.c
@@ -26,11 +26,11 @@
 #include <linux/interrupt.h>
 #include <asm/io.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_reg.h"
diff --git a/drivers/media/pci/mantis/mantis_link.h b/drivers/media/pci/mantis/mantis_link.h
index 2a814774a001..c6698976fc2f 100644
--- a/drivers/media/pci/mantis/mantis_link.h
+++ b/drivers/media/pci/mantis/mantis_link.h
@@ -23,7 +23,7 @@
 
 #include <linux/mutex.h>
 #include <linux/workqueue.h>
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 enum mantis_sbuf_status {
 	MANTIS_SBUF_DATA_AVAIL		= 1,
diff --git a/drivers/media/pci/mantis/mantis_pci.c b/drivers/media/pci/mantis/mantis_pci.c
index 9e89e045213a..d590524b4171 100644
--- a/drivers/media/pci/mantis/mantis_pci.c
+++ b/drivers/media/pci/mantis/mantis_pci.c
@@ -34,11 +34,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_reg.h"
diff --git a/drivers/media/pci/mantis/mantis_pcmcia.c b/drivers/media/pci/mantis/mantis_pcmcia.c
index b2dbc7b2e0f6..2a316b988c07 100644
--- a/drivers/media/pci/mantis/mantis_pcmcia.c
+++ b/drivers/media/pci/mantis/mantis_pcmcia.c
@@ -25,11 +25,11 @@
 #include <linux/interrupt.h>
 #include <asm/io.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_link.h" /* temporary due to physical layer stuff */
diff --git a/drivers/media/pci/mantis/mantis_reg.h b/drivers/media/pci/mantis/mantis_reg.h
index 7761f9dc7fe0..762ed9f7a08e 100644
--- a/drivers/media/pci/mantis/mantis_reg.h
+++ b/drivers/media/pci/mantis/mantis_reg.h
@@ -166,12 +166,12 @@
 #define MANTIS_CARD_PLUGOUT		(0x01 <<  0)
 
 #define MANTIS_GPIF_BRADDR		0xa0
-#define MANTIS_GPIF_PCMCIAREG		(0x01 		<< 27)
-#define MANTIS_GPIF_PCMCIAIOM		(0x01 		<< 26)
+#define MANTIS_GPIF_PCMCIAREG		(0x01		<< 27)
+#define MANTIS_GPIF_PCMCIAIOM		(0x01		<< 26)
 #define MANTIS_GPIF_BR_ADDR		(0xfffffff	<<  0)
 
 #define MANTIS_GPIF_BRBYTES		0xa4
-#define MANTIS_GPIF_BRCNT		(0xfff 		<<  0)
+#define MANTIS_GPIF_BRCNT		(0xfff		<<  0)
 
 #define MANTIS_PCMCIA_RESET		0xa8
 #define MANTIS_PCMCIA_RSTVAL		(0xff << 0)
diff --git a/drivers/media/pci/mantis/mantis_uart.c b/drivers/media/pci/mantis/mantis_uart.c
index f1c96aec8c7b..18f81c135996 100644
--- a/drivers/media/pci/mantis/mantis_uart.c
+++ b/drivers/media/pci/mantis/mantis_uart.c
@@ -27,11 +27,11 @@
 #include <linux/interrupt.h>
 #include <linux/pci.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_reg.h"
diff --git a/drivers/media/pci/mantis/mantis_vp1033.c b/drivers/media/pci/mantis/mantis_vp1033.c
index 12a6adb2bd7e..54d2ab409cc5 100644
--- a/drivers/media/pci/mantis/mantis_vp1033.c
+++ b/drivers/media/pci/mantis/mantis_vp1033.c
@@ -22,11 +22,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "stv0299.h"
 #include "mantis_common.h"
diff --git a/drivers/media/pci/mantis/mantis_vp1034.c b/drivers/media/pci/mantis/mantis_vp1034.c
index e4972ff823c2..26672a49b86f 100644
--- a/drivers/media/pci/mantis/mantis_vp1034.c
+++ b/drivers/media/pci/mantis/mantis_vp1034.c
@@ -23,11 +23,11 @@
 #include <linux/interrupt.h>
 #include <asm/io.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mb86a16.h"
 #include "mantis_common.h"
diff --git a/drivers/media/pci/mantis/mantis_vp1034.h b/drivers/media/pci/mantis/mantis_vp1034.h
index 764b1c66ea1b..35af4e5dcc8c 100644
--- a/drivers/media/pci/mantis/mantis_vp1034.h
+++ b/drivers/media/pci/mantis/mantis_vp1034.h
@@ -21,7 +21,7 @@
 #ifndef __MANTIS_VP1034_H
 #define __MANTIS_VP1034_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mantis_common.h"
 
 
diff --git a/drivers/media/pci/mantis/mantis_vp1041.c b/drivers/media/pci/mantis/mantis_vp1041.c
index 7082fcbc94a1..0eeccc2d19a5 100644
--- a/drivers/media/pci/mantis/mantis_vp1041.c
+++ b/drivers/media/pci/mantis/mantis_vp1041.c
@@ -22,11 +22,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "mantis_common.h"
 #include "mantis_ioc.h"
@@ -47,70 +47,70 @@ static const struct stb0899_s1_reg vp1041_stb0899_s1_init_1[] = {
 	/* 0x0000000b, *//* SYSREG */
 	{ STB0899_DEV_ID		, 0x30 },
 	{ STB0899_DISCNTRL1		, 0x32 },
-	{ STB0899_DISCNTRL2     	, 0x80 },
-	{ STB0899_DISRX_ST0     	, 0x04 },
-	{ STB0899_DISRX_ST1     	, 0x00 },
-	{ STB0899_DISPARITY     	, 0x00 },
+	{ STB0899_DISCNTRL2		, 0x80 },
+	{ STB0899_DISRX_ST0		, 0x04 },
+	{ STB0899_DISRX_ST1		, 0x00 },
+	{ STB0899_DISPARITY		, 0x00 },
 	{ STB0899_DISSTATUS		, 0x20 },
-	{ STB0899_DISF22        	, 0x99 },
-	{ STB0899_DISF22RX      	, 0xa8 },
+	{ STB0899_DISF22		, 0x99 },
+	{ STB0899_DISF22RX		, 0xa8 },
 	/* SYSREG ? */
-	{ STB0899_ACRPRESC      	, 0x11 },
-	{ STB0899_ACRDIV1       	, 0x0a },
-	{ STB0899_ACRDIV2       	, 0x05 },
-	{ STB0899_DACR1         	, 0x00 },
-	{ STB0899_DACR2         	, 0x00 },
-	{ STB0899_OUTCFG        	, 0x00 },
-	{ STB0899_MODECFG       	, 0x00 },
+	{ STB0899_ACRPRESC		, 0x11 },
+	{ STB0899_ACRDIV1		, 0x0a },
+	{ STB0899_ACRDIV2		, 0x05 },
+	{ STB0899_DACR1			, 0x00 },
+	{ STB0899_DACR2			, 0x00 },
+	{ STB0899_OUTCFG		, 0x00 },
+	{ STB0899_MODECFG		, 0x00 },
 	{ STB0899_IRQSTATUS_3		, 0xfe },
 	{ STB0899_IRQSTATUS_2		, 0x03 },
 	{ STB0899_IRQSTATUS_1		, 0x7c },
 	{ STB0899_IRQSTATUS_0		, 0xf4 },
-	{ STB0899_IRQMSK_3      	, 0xf3 },
-	{ STB0899_IRQMSK_2      	, 0xfc },
-	{ STB0899_IRQMSK_1      	, 0xff },
+	{ STB0899_IRQMSK_3		, 0xf3 },
+	{ STB0899_IRQMSK_2		, 0xfc },
+	{ STB0899_IRQMSK_1		, 0xff },
 	{ STB0899_IRQMSK_0		, 0xff },
 	{ STB0899_IRQCFG		, 0x00 },
-	{ STB0899_I2CCFG        	, 0x88 },
-	{ STB0899_I2CRPT        	, 0x58 },
+	{ STB0899_I2CCFG		, 0x88 },
+	{ STB0899_I2CRPT		, 0x58 },
 	{ STB0899_IOPVALUE5		, 0x00 },
 	{ STB0899_IOPVALUE4		, 0x33 },
 	{ STB0899_IOPVALUE3		, 0x6d },
 	{ STB0899_IOPVALUE2		, 0x90 },
 	{ STB0899_IOPVALUE1		, 0x60 },
 	{ STB0899_IOPVALUE0		, 0x00 },
-	{ STB0899_GPIO00CFG     	, 0x82 },
-	{ STB0899_GPIO01CFG     	, 0x82 },
-	{ STB0899_GPIO02CFG     	, 0x82 },
-	{ STB0899_GPIO03CFG     	, 0x82 },
-	{ STB0899_GPIO04CFG     	, 0x82 },
-	{ STB0899_GPIO05CFG     	, 0x82 },
-	{ STB0899_GPIO06CFG     	, 0x82 },
-	{ STB0899_GPIO07CFG     	, 0x82 },
-	{ STB0899_GPIO08CFG     	, 0x82 },
-	{ STB0899_GPIO09CFG     	, 0x82 },
-	{ STB0899_GPIO10CFG     	, 0x82 },
-	{ STB0899_GPIO11CFG     	, 0x82 },
-	{ STB0899_GPIO12CFG     	, 0x82 },
-	{ STB0899_GPIO13CFG     	, 0x82 },
-	{ STB0899_GPIO14CFG     	, 0x82 },
-	{ STB0899_GPIO15CFG     	, 0x82 },
-	{ STB0899_GPIO16CFG     	, 0x82 },
-	{ STB0899_GPIO17CFG     	, 0x82 },
-	{ STB0899_GPIO18CFG     	, 0x82 },
-	{ STB0899_GPIO19CFG     	, 0x82 },
-	{ STB0899_GPIO20CFG     	, 0x82 },
-	{ STB0899_SDATCFG       	, 0xb8 },
-	{ STB0899_SCLTCFG       	, 0xba },
-	{ STB0899_AGCRFCFG      	, 0x1c }, /* 0x11 */
-	{ STB0899_GPIO22        	, 0x82 }, /* AGCBB2CFG */
-	{ STB0899_GPIO21        	, 0x91 }, /* AGCBB1CFG */
-	{ STB0899_DIRCLKCFG     	, 0x82 },
-	{ STB0899_CLKOUT27CFG   	, 0x7e },
-	{ STB0899_STDBYCFG      	, 0x82 },
-	{ STB0899_CS0CFG        	, 0x82 },
-	{ STB0899_CS1CFG        	, 0x82 },
-	{ STB0899_DISEQCOCFG    	, 0x20 },
+	{ STB0899_GPIO00CFG		, 0x82 },
+	{ STB0899_GPIO01CFG		, 0x82 },
+	{ STB0899_GPIO02CFG		, 0x82 },
+	{ STB0899_GPIO03CFG		, 0x82 },
+	{ STB0899_GPIO04CFG		, 0x82 },
+	{ STB0899_GPIO05CFG		, 0x82 },
+	{ STB0899_GPIO06CFG		, 0x82 },
+	{ STB0899_GPIO07CFG		, 0x82 },
+	{ STB0899_GPIO08CFG		, 0x82 },
+	{ STB0899_GPIO09CFG		, 0x82 },
+	{ STB0899_GPIO10CFG		, 0x82 },
+	{ STB0899_GPIO11CFG		, 0x82 },
+	{ STB0899_GPIO12CFG		, 0x82 },
+	{ STB0899_GPIO13CFG		, 0x82 },
+	{ STB0899_GPIO14CFG		, 0x82 },
+	{ STB0899_GPIO15CFG		, 0x82 },
+	{ STB0899_GPIO16CFG		, 0x82 },
+	{ STB0899_GPIO17CFG		, 0x82 },
+	{ STB0899_GPIO18CFG		, 0x82 },
+	{ STB0899_GPIO19CFG		, 0x82 },
+	{ STB0899_GPIO20CFG		, 0x82 },
+	{ STB0899_SDATCFG		, 0xb8 },
+	{ STB0899_SCLTCFG		, 0xba },
+	{ STB0899_AGCRFCFG		, 0x1c }, /* 0x11 */
+	{ STB0899_GPIO22		, 0x82 }, /* AGCBB2CFG */
+	{ STB0899_GPIO21		, 0x91 }, /* AGCBB1CFG */
+	{ STB0899_DIRCLKCFG		, 0x82 },
+	{ STB0899_CLKOUT27CFG		, 0x7e },
+	{ STB0899_STDBYCFG		, 0x82 },
+	{ STB0899_CS0CFG		, 0x82 },
+	{ STB0899_CS1CFG		, 0x82 },
+	{ STB0899_DISEQCOCFG		, 0x20 },
 	{ STB0899_GPIO32CFG		, 0x82 },
 	{ STB0899_GPIO33CFG		, 0x82 },
 	{ STB0899_GPIO34CFG		, 0x82 },
@@ -119,35 +119,35 @@ static const struct stb0899_s1_reg vp1041_stb0899_s1_init_1[] = {
 	{ STB0899_GPIO37CFG		, 0x82 },
 	{ STB0899_GPIO38CFG		, 0x82 },
 	{ STB0899_GPIO39CFG		, 0x82 },
-	{ STB0899_NCOARSE       	, 0x17 }, /* 0x15 = 27 Mhz Clock, F/3 = 198MHz, F/6 = 99MHz */
-	{ STB0899_SYNTCTRL      	, 0x02 }, /* 0x00 = CLK from CLKI, 0x02 = CLK from XTALI */
-	{ STB0899_FILTCTRL      	, 0x00 },
-	{ STB0899_SYSCTRL       	, 0x01 },
-	{ STB0899_STOPCLK1      	, 0x20 },
-	{ STB0899_STOPCLK2      	, 0x00 },
+	{ STB0899_NCOARSE		, 0x17 }, /* 0x15 = 27 Mhz Clock, F/3 = 198MHz, F/6 = 99MHz */
+	{ STB0899_SYNTCTRL		, 0x02 }, /* 0x00 = CLK from CLKI, 0x02 = CLK from XTALI */
+	{ STB0899_FILTCTRL		, 0x00 },
+	{ STB0899_SYSCTRL		, 0x01 },
+	{ STB0899_STOPCLK1		, 0x20 },
+	{ STB0899_STOPCLK2		, 0x00 },
 	{ STB0899_INTBUFSTATUS		, 0x00 },
-	{ STB0899_INTBUFCTRL    	, 0x0a },
+	{ STB0899_INTBUFCTRL		, 0x0a },
 	{ 0xffff			, 0xff },
 };
 
 static const struct stb0899_s1_reg vp1041_stb0899_s1_init_3[] = {
-	{ STB0899_DEMOD         	, 0x00 },
-	{ STB0899_RCOMPC        	, 0xc9 },
-	{ STB0899_AGC1CN        	, 0x01 },
-	{ STB0899_AGC1REF       	, 0x10 },
+	{ STB0899_DEMOD			, 0x00 },
+	{ STB0899_RCOMPC		, 0xc9 },
+	{ STB0899_AGC1CN		, 0x01 },
+	{ STB0899_AGC1REF		, 0x10 },
 	{ STB0899_RTC			, 0x23 },
-	{ STB0899_TMGCFG        	, 0x4e },
-	{ STB0899_AGC2REF       	, 0x34 },
-	{ STB0899_TLSR          	, 0x84 },
-	{ STB0899_CFD           	, 0xf7 },
+	{ STB0899_TMGCFG		, 0x4e },
+	{ STB0899_AGC2REF		, 0x34 },
+	{ STB0899_TLSR			, 0x84 },
+	{ STB0899_CFD			, 0xf7 },
 	{ STB0899_ACLC			, 0x87 },
-	{ STB0899_BCLC          	, 0x94 },
-	{ STB0899_EQON          	, 0x41 },
-	{ STB0899_LDT           	, 0xf1 },
-	{ STB0899_LDT2          	, 0xe3 },
-	{ STB0899_EQUALREF      	, 0xb4 },
-	{ STB0899_TMGRAMP       	, 0x10 },
-	{ STB0899_TMGTHD        	, 0x30 },
+	{ STB0899_BCLC			, 0x94 },
+	{ STB0899_EQON			, 0x41 },
+	{ STB0899_LDT			, 0xf1 },
+	{ STB0899_LDT2			, 0xe3 },
+	{ STB0899_EQUALREF		, 0xb4 },
+	{ STB0899_TMGRAMP		, 0x10 },
+	{ STB0899_TMGTHD		, 0x30 },
 	{ STB0899_IDCCOMP		, 0xfd },
 	{ STB0899_QDCCOMP		, 0xff },
 	{ STB0899_POWERI		, 0x0c },
@@ -166,12 +166,12 @@ static const struct stb0899_s1_reg vp1041_stb0899_s1_init_3[] = {
 	{ STB0899_NIRL			, 0x80 },
 	{ STB0899_ISYMB			, 0x1d },
 	{ STB0899_QSYMB			, 0xa6 },
-	{ STB0899_SFRH          	, 0x2f },
-	{ STB0899_SFRM          	, 0x68 },
-	{ STB0899_SFRL          	, 0x40 },
-	{ STB0899_SFRUPH        	, 0x2f },
-	{ STB0899_SFRUPM        	, 0x68 },
-	{ STB0899_SFRUPL        	, 0x40 },
+	{ STB0899_SFRH			, 0x2f },
+	{ STB0899_SFRM			, 0x68 },
+	{ STB0899_SFRL			, 0x40 },
+	{ STB0899_SFRUPH		, 0x2f },
+	{ STB0899_SFRUPM		, 0x68 },
+	{ STB0899_SFRUPL		, 0x40 },
 	{ STB0899_EQUAI1		, 0x02 },
 	{ STB0899_EQUAQ1		, 0xff },
 	{ STB0899_EQUAI2		, 0x04 },
@@ -183,7 +183,7 @@ static const struct stb0899_s1_reg vp1041_stb0899_s1_init_3[] = {
 	{ STB0899_EQUAI5		, 0x08 },
 	{ STB0899_EQUAQ5		, 0xf5 },
 	{ STB0899_DSTATUS2		, 0x00 },
-	{ STB0899_VSTATUS       	, 0x00 },
+	{ STB0899_VSTATUS		, 0x00 },
 	{ STB0899_VERROR		, 0x86 },
 	{ STB0899_IQSWAP		, 0x2a },
 	{ STB0899_ECNT1M		, 0x00 },
@@ -192,26 +192,26 @@ static const struct stb0899_s1_reg vp1041_stb0899_s1_init_3[] = {
 	{ STB0899_ECNT2L		, 0x00 },
 	{ STB0899_ECNT3M		, 0x0a },
 	{ STB0899_ECNT3L		, 0xad },
-	{ STB0899_FECAUTO1      	, 0x06 },
+	{ STB0899_FECAUTO1		, 0x06 },
 	{ STB0899_FECM			, 0x01 },
-	{ STB0899_VTH12         	, 0xb0 },
-	{ STB0899_VTH23         	, 0x7a },
+	{ STB0899_VTH12			, 0xb0 },
+	{ STB0899_VTH23			, 0x7a },
 	{ STB0899_VTH34			, 0x58 },
-	{ STB0899_VTH56         	, 0x38 },
-	{ STB0899_VTH67         	, 0x34 },
-	{ STB0899_VTH78         	, 0x24 },
-	{ STB0899_PRVIT         	, 0xff },
-	{ STB0899_VITSYNC       	, 0x19 },
-	{ STB0899_RSULC         	, 0xb1 }, /* DVB = 0xb1, DSS = 0xa1 */
-	{ STB0899_TSULC         	, 0x42 },
-	{ STB0899_RSLLC         	, 0x41 },
+	{ STB0899_VTH56			, 0x38 },
+	{ STB0899_VTH67			, 0x34 },
+	{ STB0899_VTH78			, 0x24 },
+	{ STB0899_PRVIT			, 0xff },
+	{ STB0899_VITSYNC		, 0x19 },
+	{ STB0899_RSULC			, 0xb1 }, /* DVB = 0xb1, DSS = 0xa1 */
+	{ STB0899_TSULC			, 0x42 },
+	{ STB0899_RSLLC			, 0x41 },
 	{ STB0899_TSLPL			, 0x12 },
-	{ STB0899_TSCFGH        	, 0x0c },
-	{ STB0899_TSCFGM        	, 0x00 },
-	{ STB0899_TSCFGL        	, 0x00 },
+	{ STB0899_TSCFGH		, 0x0c },
+	{ STB0899_TSCFGM		, 0x00 },
+	{ STB0899_TSCFGL		, 0x00 },
 	{ STB0899_TSOUT			, 0x69 }, /* 0x0d for CAM */
-	{ STB0899_RSSYNCDEL     	, 0x00 },
-	{ STB0899_TSINHDELH     	, 0x02 },
+	{ STB0899_RSSYNCDEL		, 0x00 },
+	{ STB0899_TSINHDELH		, 0x02 },
 	{ STB0899_TSINHDELM		, 0x00 },
 	{ STB0899_TSINHDELL		, 0x00 },
 	{ STB0899_TSLLSTKM		, 0x1b },
@@ -222,18 +222,18 @@ static const struct stb0899_s1_reg vp1041_stb0899_s1_init_3[] = {
 	{ STB0899_PCKLENLL		, 0xcc },
 	{ STB0899_RSPCKLEN		, 0xbd },
 	{ STB0899_TSSTATUS		, 0x90 },
-	{ STB0899_ERRCTRL1      	, 0xb6 },
-	{ STB0899_ERRCTRL2      	, 0x95 },
-	{ STB0899_ERRCTRL3      	, 0x8d },
+	{ STB0899_ERRCTRL1		, 0xb6 },
+	{ STB0899_ERRCTRL2		, 0x95 },
+	{ STB0899_ERRCTRL3		, 0x8d },
 	{ STB0899_DMONMSK1		, 0x27 },
 	{ STB0899_DMONMSK0		, 0x03 },
-	{ STB0899_DEMAPVIT      	, 0x5c },
+	{ STB0899_DEMAPVIT		, 0x5c },
 	{ STB0899_PLPARM		, 0x19 },
-	{ STB0899_PDELCTRL      	, 0x48 },
-	{ STB0899_PDELCTRL2     	, 0x00 },
-	{ STB0899_BBHCTRL1      	, 0x00 },
-	{ STB0899_BBHCTRL2      	, 0x00 },
-	{ STB0899_HYSTTHRESH    	, 0x77 },
+	{ STB0899_PDELCTRL		, 0x48 },
+	{ STB0899_PDELCTRL2		, 0x00 },
+	{ STB0899_BBHCTRL1		, 0x00 },
+	{ STB0899_BBHCTRL2		, 0x00 },
+	{ STB0899_HYSTTHRESH		, 0x77 },
 	{ STB0899_MATCSTM		, 0x00 },
 	{ STB0899_MATCSTL		, 0x00 },
 	{ STB0899_UPLCSTM		, 0x00 },
@@ -270,7 +270,7 @@ static struct stb0899_config vp1041_stb0899_config = {
 	.init_s2_fec		= stb0899_s2_init_4,
 	.init_tst		= stb0899_s1_init_5,
 
-	.demod_address 		= 0x68, /*  0xd0 >> 1 */
+	.demod_address		= 0x68, /*  0xd0 >> 1 */
 
 	.xtal_freq		= 27000000,
 	.inversion		= IQ_SWAP_ON,
diff --git a/drivers/media/pci/mantis/mantis_vp2033.c b/drivers/media/pci/mantis/mantis_vp2033.c
index 8d48b5abe04a..d98e0a3edaab 100644
--- a/drivers/media/pci/mantis/mantis_vp2033.c
+++ b/drivers/media/pci/mantis/mantis_vp2033.c
@@ -22,11 +22,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "tda1002x.h"
 #include "mantis_common.h"
diff --git a/drivers/media/pci/mantis/mantis_vp2040.c b/drivers/media/pci/mantis/mantis_vp2040.c
index 8dd17d7c0881..2c52f3d4e2bc 100644
--- a/drivers/media/pci/mantis/mantis_vp2040.c
+++ b/drivers/media/pci/mantis/mantis_vp2040.c
@@ -22,11 +22,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "tda1002x.h"
 #include "mantis_common.h"
diff --git a/drivers/media/pci/mantis/mantis_vp3028.h b/drivers/media/pci/mantis/mantis_vp3028.h
index b07be6adc522..34130d29e0aa 100644
--- a/drivers/media/pci/mantis/mantis_vp3028.h
+++ b/drivers/media/pci/mantis/mantis_vp3028.h
@@ -21,7 +21,7 @@
 #ifndef __MANTIS_VP3028_H
 #define __MANTIS_VP3028_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mantis_common.h"
 #include "zl10353.h"
 
diff --git a/drivers/media/pci/mantis/mantis_vp3030.c b/drivers/media/pci/mantis/mantis_vp3030.c
index 5c1dd925bdd5..14f6e153000c 100644
--- a/drivers/media/pci/mantis/mantis_vp3030.c
+++ b/drivers/media/pci/mantis/mantis_vp3030.c
@@ -22,11 +22,11 @@
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "zl10353.h"
 #include "tda665x.h"
diff --git a/drivers/media/pci/meye/meye.c b/drivers/media/pci/meye/meye.c
index 23999a8cef37..dedcdb573427 100644
--- a/drivers/media/pci/meye/meye.c
+++ b/drivers/media/pci/meye/meye.c
@@ -1423,14 +1423,14 @@ static long vidioc_default(struct file *file, void *fh, bool valid_prio,
 
 }
 
-static unsigned int meye_poll(struct file *file, poll_table *wait)
+static __poll_t meye_poll(struct file *file, poll_table *wait)
 {
-	unsigned int res = v4l2_ctrl_poll(file, wait);
+	__poll_t res = v4l2_ctrl_poll(file, wait);
 
 	mutex_lock(&meye.lock);
 	poll_wait(file, &meye.proc_list, wait);
 	if (kfifo_len(&meye.doneq))
-		res |= POLLIN | POLLRDNORM;
+		res |= EPOLLIN | EPOLLRDNORM;
 	mutex_unlock(&meye.lock);
 	return res;
 }
@@ -1536,7 +1536,7 @@ static const struct v4l2_ioctl_ops meye_ioctl_ops = {
 static const struct video_device meye_template = {
 	.name		= "meye",
 	.fops		= &meye_fops,
-	.ioctl_ops 	= &meye_ioctl_ops,
+	.ioctl_ops	= &meye_ioctl_ops,
 	.release	= video_device_release_empty,
 };
 
diff --git a/drivers/media/pci/netup_unidvb/Makefile b/drivers/media/pci/netup_unidvb/Makefile
index 07d3f1eb728b..944c3e164157 100644
--- a/drivers/media/pci/netup_unidvb/Makefile
+++ b/drivers/media/pci/netup_unidvb/Makefile
@@ -6,5 +6,4 @@ netup-unidvb-objs += netup_unidvb_spi.o
 
 obj-$(CONFIG_DVB_NETUP_UNIDVB) += netup-unidvb.o
 
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb.h b/drivers/media/pci/netup_unidvb/netup_unidvb.h
index 39b08ecda1fc..3253ac324841 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb.h
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb.h
@@ -24,7 +24,7 @@
 #include <media/v4l2-common.h>
 #include <media/v4l2-device.h>
 #include <media/videobuf2-dvb.h>
-#include <dvb_ca_en50221.h>
+#include <media/dvb_ca_en50221.h>
 
 #define NETUP_UNIDVB_NAME	"netup_unidvb"
 #define NETUP_UNIDVB_VERSION	"0.0.1"
diff --git a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
index 509d69e6ca4a..ead59fabd15f 100644
--- a/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
+++ b/drivers/media/pci/netup_unidvb/netup_unidvb_core.c
@@ -862,7 +862,7 @@ static int netup_unidvb_initdev(struct pci_dev *pci_dev,
 		PCI_EXP_DEVCTL_NOSNOOP_EN, 0);
 	/* Adjust PCIe completion timeout. */
 	pcie_capability_clear_and_set_word(pci_dev,
-		PCI_EXP_DEVCTL2, 0xf, 0x2);
+		PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0x2);
 
 	if (netup_unidvb_request_mmio(pci_dev)) {
 		dev_err(&pci_dev->dev,
diff --git a/drivers/media/pci/ngene/Makefile b/drivers/media/pci/ngene/Makefile
index dbdf284970f8..e4208f5ed215 100644
--- a/drivers/media/pci/ngene/Makefile
+++ b/drivers/media/pci/ngene/Makefile
@@ -7,7 +7,6 @@ ngene-objs := ngene-core.o ngene-i2c.o ngene-cards.o ngene-dvb.o
 
 obj-$(CONFIG_DVB_NGENE) += ngene.o
 
-ccflags-y += -Idrivers/media/dvb-core/
 ccflags-y += -Idrivers/media/dvb-frontends/
 ccflags-y += -Idrivers/media/tuners/
 
diff --git a/drivers/media/pci/ngene/ngene.h b/drivers/media/pci/ngene/ngene.h
index 7c7cd217333d..02dbd18f92d0 100644
--- a/drivers/media/pci/ngene/ngene.h
+++ b/drivers/media/pci/ngene/ngene.h
@@ -29,13 +29,13 @@
 
 #include <linux/dvb/frontend.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_ca_en50221.h"
-#include "dvb_frontend.h"
-#include "dvb_ringbuffer.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_ca_en50221.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_ringbuffer.h>
+#include <media/dvb_net.h>
 #include "cxd2099.h"
 
 #define DEVICE_NAME "ngene"
diff --git a/drivers/media/pci/pluto2/Makefile b/drivers/media/pci/pluto2/Makefile
index 524bf841f42b..3c2aea1ac752 100644
--- a/drivers/media/pci/pluto2/Makefile
+++ b/drivers/media/pci/pluto2/Makefile
@@ -1,3 +1,3 @@
 obj-$(CONFIG_DVB_PLUTO2) += pluto2.o
 
-ccflags-y += -Idrivers/media/dvb-core/ -Idrivers/media/dvb-frontends/
+ccflags-y += -Idrivers/media/dvb-frontends/
diff --git a/drivers/media/pci/pluto2/pluto2.c b/drivers/media/pci/pluto2/pluto2.c
index 39dcba2b620c..5e6fe686f420 100644
--- a/drivers/media/pci/pluto2/pluto2.c
+++ b/drivers/media/pci/pluto2/pluto2.c
@@ -4,7 +4,7 @@
  * Copyright (C) 2005 Andreas Oberritter <obi@linuxtv.org>
  *
  * based on pluto2.c 1.10 - http://instinct-wp8.no-ip.org/pluto/
- * 	by Dany Salman <salmandany@yahoo.fr>
+ *	by Dany Salman <salmandany@yahoo.fr>
  *	Copyright (c) 2004 TDF
  *
  * This program is free software; you can redistribute it and/or modify
@@ -29,12 +29,12 @@
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
 
-#include "demux.h"
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
-#include "dvbdev.h"
+#include <media/demux.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
+#include <media/dvbdev.h>
 #include "tda1004x.h"
 
 DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);
diff --git a/drivers/media/pci/pt1/Makefile b/drivers/media/pci/pt1/Makefile
index 98e391295afe..ab873ae088a0 100644
--- a/drivers/media/pci/pt1/Makefile
+++ b/drivers/media/pci/pt1/Makefile
@@ -2,4 +2,4 @@ earth-pt1-objs := pt1.o va1j5jf8007s.o va1j5jf8007t.o
 
 obj-$(CONFIG_DVB_PT1) += earth-pt1.o
 
-ccflags-y += -Idrivers/media/dvb-core -Idrivers/media/dvb-frontends
+ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/pci/pt1/pt1.c b/drivers/media/pci/pt1/pt1.c
index b6b1a8d20d86..4f6867af8311 100644
--- a/drivers/media/pci/pt1/pt1.c
+++ b/drivers/media/pci/pt1/pt1.c
@@ -4,7 +4,7 @@
  * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
  *
  * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- * 	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
+ *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -27,11 +27,11 @@
 #include <linux/freezer.h>
 #include <linux/ratelimit.h>
 
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dmxdev.h"
-#include "dvb_net.h"
-#include "dvb_frontend.h"
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dmxdev.h>
+#include <media/dvb_net.h>
+#include <media/dvb_frontend.h>
 
 #include "va1j5jf8007t.h"
 #include "va1j5jf8007s.h"
@@ -116,8 +116,8 @@ static u32 pt1_read_reg(struct pt1 *pt1, int reg)
 	return readl(pt1->regs + reg * 4);
 }
 
-static int pt1_nr_tables = 8;
-module_param_named(nr_tables, pt1_nr_tables, int, 0);
+static unsigned int pt1_nr_tables = 8;
+module_param_named(nr_tables, pt1_nr_tables, uint, 0);
 
 static void pt1_increment_table_count(struct pt1 *pt1)
 {
@@ -443,6 +443,9 @@ static int pt1_init_tables(struct pt1 *pt1)
 	int i, ret;
 	u32 first_pfn, pfn;
 
+	if (!pt1_nr_tables)
+		return 0;
+
 	tables = vmalloc(sizeof(struct pt1_table) * pt1_nr_tables);
 	if (tables == NULL)
 		return -ENOMEM;
@@ -450,12 +453,10 @@ static int pt1_init_tables(struct pt1 *pt1)
 	pt1_init_table_count(pt1);
 
 	i = 0;
-	if (pt1_nr_tables) {
-		ret = pt1_init_table(pt1, &tables[0], &first_pfn);
-		if (ret)
-			goto err;
-		i++;
-	}
+	ret = pt1_init_table(pt1, &tables[0], &first_pfn);
+	if (ret)
+		goto err;
+	i++;
 
 	while (i < pt1_nr_tables) {
 		ret = pt1_init_table(pt1, &tables[i], &pfn);
diff --git a/drivers/media/pci/pt1/va1j5jf8007s.c b/drivers/media/pci/pt1/va1j5jf8007s.c
index f75f69556be7..f49867aef054 100644
--- a/drivers/media/pci/pt1/va1j5jf8007s.c
+++ b/drivers/media/pci/pt1/va1j5jf8007s.c
@@ -4,7 +4,7 @@
  * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
  *
  * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- * 	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
+ *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,7 +21,7 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "va1j5jf8007s.h"
 
 enum va1j5jf8007s_tune_state {
diff --git a/drivers/media/pci/pt1/va1j5jf8007s.h b/drivers/media/pci/pt1/va1j5jf8007s.h
index efbe6ccae8b4..f8ce5609095d 100644
--- a/drivers/media/pci/pt1/va1j5jf8007s.h
+++ b/drivers/media/pci/pt1/va1j5jf8007s.h
@@ -4,7 +4,7 @@
  * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
  *
  * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- * 	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
+ *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
diff --git a/drivers/media/pci/pt1/va1j5jf8007t.c b/drivers/media/pci/pt1/va1j5jf8007t.c
index 63fda79a75c0..a52984a6f9b3 100644
--- a/drivers/media/pci/pt1/va1j5jf8007t.c
+++ b/drivers/media/pci/pt1/va1j5jf8007t.c
@@ -4,7 +4,7 @@
  * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
  *
  * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- * 	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
+ *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -21,8 +21,8 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
-#include "dvb_math.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_math.h>
 #include "va1j5jf8007t.h"
 
 enum va1j5jf8007t_tune_state {
diff --git a/drivers/media/pci/pt1/va1j5jf8007t.h b/drivers/media/pci/pt1/va1j5jf8007t.h
index 6fb119c6e73a..95eb7d294d20 100644
--- a/drivers/media/pci/pt1/va1j5jf8007t.h
+++ b/drivers/media/pci/pt1/va1j5jf8007t.h
@@ -4,7 +4,7 @@
  * Copyright (C) 2009 HIRANO Takahito <hiranotaka@zng.info>
  *
  * based on pt1dvr - http://pt1dvr.sourceforge.jp/
- * 	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
+ *	by Tomoaki Ishikawa <tomy@users.sourceforge.jp>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
diff --git a/drivers/media/pci/pt3/Makefile b/drivers/media/pci/pt3/Makefile
index aded8752ac2b..8698d5dfaf52 100644
--- a/drivers/media/pci/pt3/Makefile
+++ b/drivers/media/pci/pt3/Makefile
@@ -4,6 +4,5 @@ earth-pt3-objs += pt3.o pt3_i2c.o pt3_dma.o
 
 obj-$(CONFIG_DVB_PT3) += earth-pt3.o
 
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
 ccflags-y += -Idrivers/media/tuners
diff --git a/drivers/media/pci/pt3/pt3.c b/drivers/media/pci/pt3/pt3.c
index 34044a45fecc..da74828805bc 100644
--- a/drivers/media/pci/pt3/pt3.c
+++ b/drivers/media/pci/pt3/pt3.c
@@ -23,10 +23,10 @@
 #include <linux/string.h>
 #include <linux/sched/signal.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
 
 #include "pt3.h"
 
diff --git a/drivers/media/pci/pt3/pt3.h b/drivers/media/pci/pt3/pt3.h
index 1b3f2ad25db3..fbe8d9b847b0 100644
--- a/drivers/media/pci/pt3/pt3.h
+++ b/drivers/media/pci/pt3/pt3.h
@@ -20,9 +20,9 @@
 #include <linux/atomic.h>
 #include <linux/types.h>
 
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dmxdev.h"
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dmxdev.h>
 
 #include "tc90522.h"
 #include "mxl301rf.h"
diff --git a/drivers/media/pci/pt3/pt3_i2c.c b/drivers/media/pci/pt3/pt3_i2c.c
index ec6a8a2e4744..b66138c7b364 100644
--- a/drivers/media/pci/pt3/pt3_i2c.c
+++ b/drivers/media/pci/pt3/pt3_i2c.c
@@ -85,7 +85,6 @@ static void put_byte_write(struct pt3_i2cbuf *cbuf, u8 val)
 {
 	u8 mask;
 
-	mask = 0x80;
 	for (mask = 0x80; mask > 0; mask >>= 1)
 		cmdbuf_add(cbuf, (val & mask) ? I_DATA_H_NOP : I_DATA_L_NOP);
 	cmdbuf_add(cbuf, I_DATA_H_ACK0);
diff --git a/drivers/media/pci/saa7134/Makefile b/drivers/media/pci/saa7134/Makefile
index dbaadddf4320..82ac7f31221b 100644
--- a/drivers/media/pci/saa7134/Makefile
+++ b/drivers/media/pci/saa7134/Makefile
@@ -12,8 +12,6 @@ obj-$(CONFIG_VIDEO_SAA7134_ALSA) += saa7134-alsa.o
 
 obj-$(CONFIG_VIDEO_SAA7134_DVB) += saa7134-dvb.o
 
-ccflags-y += -I$(srctree)/drivers/media/i2c
 ccflags-y += -I$(srctree)/drivers/media/tuners
-ccflags-y += -I$(srctree)/drivers/media/dvb-core
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends
 ccflags-y += -I$(srctree)/drivers/media/usb/go7007
diff --git a/drivers/media/pci/saa7134/saa7134-cards.c b/drivers/media/pci/saa7134/saa7134-cards.c
index 9965d3531c80..9d6688a82b50 100644
--- a/drivers/media/pci/saa7134/saa7134-cards.c
+++ b/drivers/media/pci/saa7134/saa7134-cards.c
@@ -323,7 +323,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr	= ADDR_UNSET,
 		.radio_addr	= ADDR_UNSET,
-		.empress_addr 	= 0x20,
+		.empress_addr	= 0x20,
 
 		.inputs         = {{
 			.type = SAA7134_INPUT_COMPOSITE1,
@@ -454,7 +454,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr	= ADDR_UNSET,
 		.radio_addr	= ADDR_UNSET,
-		.empress_addr 	= 0x20,
+		.empress_addr	= 0x20,
 		.tda9887_conf	= TDA9887_PRESENT,
 		.gpiomask	= 0x820000,
 		.inputs		= {{
@@ -849,7 +849,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr	= ADDR_UNSET,
 		.radio_addr	= ADDR_UNSET,
-		.empress_addr 	= 0x20,
+		.empress_addr	= 0x20,
 		.inputs         = {{
 			.type = SAA7134_INPUT_COMPOSITE1,
 			.vmux = 4,
@@ -1006,7 +1006,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr	= ADDR_UNSET,
 		.radio_addr	= ADDR_UNSET,
-		.empress_addr 	= 0x20,
+		.empress_addr	= 0x20,
 		.inputs         = {{
 			.type = SAA7134_INPUT_COMPOSITE1,
 			.vmux = 1,
@@ -1767,7 +1767,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr	= ADDR_UNSET,
 		.radio_addr	= ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = {{
 			.type = SAA7134_INPUT_TV,
@@ -2412,7 +2412,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type	= UNSET,
 		.tuner_addr	= ADDR_UNSET,
 		.radio_addr	= ADDR_UNSET,
-		.empress_addr 	= 0x21,
+		.empress_addr	= 0x21,
 		.inputs		= {{
 			.type = SAA7134_INPUT_COMPOSITE0,
 			.vmux   = 0,
@@ -3978,13 +3978,13 @@ struct saa7134_board saa7134_boards[] = {
 	[SAA7134_BOARD_BEHOLD_407] = {
 		/*       Beholder Intl. Ltd. 2008      */
 		/*Dmitry Belimov <d.belimov@gmail.com> */
-		.name 		= "Beholder BeholdTV 407",
-		.audio_clock 	= 0x00187de7,
-		.tuner_type 	= TUNER_PHILIPS_FM1216ME_MK3,
-		.radio_type 	= UNSET,
-		.tuner_addr 	= ADDR_UNSET,
-		.radio_addr 	= ADDR_UNSET,
-		.tda9887_conf 	= TDA9887_PRESENT,
+		.name		= "Beholder BeholdTV 407",
+		.audio_clock	= 0x00187de7,
+		.tuner_type	= TUNER_PHILIPS_FM1216ME_MK3,
+		.radio_type	= UNSET,
+		.tuner_addr	= ADDR_UNSET,
+		.radio_addr	= ADDR_UNSET,
+		.tda9887_conf	= TDA9887_PRESENT,
 		.gpiomask       = 0x00008000,
 		.inputs = {{
 			.type = SAA7134_INPUT_SVIDEO,
@@ -4006,13 +4006,13 @@ struct saa7134_board saa7134_boards[] = {
 	[SAA7134_BOARD_BEHOLD_407FM] = {
 		/*       Beholder Intl. Ltd. 2008      */
 		/*Dmitry Belimov <d.belimov@gmail.com> */
-		.name 		= "Beholder BeholdTV 407 FM",
-		.audio_clock 	= 0x00187de7,
-		.tuner_type 	= TUNER_PHILIPS_FM1216ME_MK3,
-		.radio_type 	= UNSET,
-		.tuner_addr 	= ADDR_UNSET,
-		.radio_addr 	= ADDR_UNSET,
-		.tda9887_conf 	= TDA9887_PRESENT,
+		.name		= "Beholder BeholdTV 407 FM",
+		.audio_clock	= 0x00187de7,
+		.tuner_type	= TUNER_PHILIPS_FM1216ME_MK3,
+		.radio_type	= UNSET,
+		.tuner_addr	= ADDR_UNSET,
+		.radio_addr	= ADDR_UNSET,
+		.tda9887_conf	= TDA9887_PRESENT,
 		.gpiomask       = 0x00008000,
 		.inputs = {{
 			.type = SAA7134_INPUT_SVIDEO,
@@ -4103,7 +4103,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.gpiomask       = 0x00008000,
 		.inputs         = {{
@@ -4166,7 +4166,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.gpiomask       = 0x00008000,
 		.inputs         = {{
@@ -4196,7 +4196,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.gpiomask       = 0x00008000,
 		.inputs         = {{
@@ -4366,7 +4366,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = {{
 			.type = SAA7134_INPUT_TV,
@@ -4394,7 +4394,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = {{
 			.type = SAA7134_INPUT_TV,
@@ -4422,7 +4422,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = {{
 			.type = SAA7134_INPUT_TV,
@@ -4450,7 +4450,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = {{
 			.type = SAA7134_INPUT_TV,
@@ -4481,7 +4481,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.empress_addr 	= 0x20,
+		.empress_addr	= 0x20,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = { {
 			.type = SAA7134_INPUT_TV,
@@ -4517,7 +4517,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.empress_addr 	= 0x20,
+		.empress_addr	= 0x20,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = { {
 			.type = SAA7134_INPUT_TV,
@@ -4554,8 +4554,8 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
-		.empress_addr 	= 0x20,
+		.rds_addr	= 0x10,
+		.empress_addr	= 0x20,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.inputs         = { {
 			.type = SAA7134_INPUT_TV,
@@ -5297,7 +5297,7 @@ struct saa7134_board saa7134_boards[] = {
 		.radio_type     = UNSET,
 		.tuner_addr     = ADDR_UNSET,
 		.radio_addr     = ADDR_UNSET,
-		.rds_addr 	= 0x10,
+		.rds_addr	= 0x10,
 		.tda9887_conf   = TDA9887_PRESENT,
 		.gpiomask       = 0x00008000,
 		.inputs         = {{
diff --git a/drivers/media/pci/saa7134/saa7134-dvb.c b/drivers/media/pci/saa7134/saa7134-dvb.c
index 731dee0a66e7..a7a63d608dde 100644
--- a/drivers/media/pci/saa7134/saa7134-dvb.c
+++ b/drivers/media/pci/saa7134/saa7134-dvb.c
@@ -29,7 +29,7 @@
 
 #include <media/v4l2-common.h>
 #include "dvb-pll.h"
-#include <dvb_frontend.h>
+#include <media/dvb_frontend.h>
 
 #include "mt352.h"
 #include "mt352_priv.h" /* FIXME */
@@ -1389,7 +1389,7 @@ static int dvb_init(struct saa7134_dev *dev)
 			if (configure_tda827x_fe(dev, &lifeview_trio_config,
 						 &tda827x_cfg_0) < 0)
 				goto detach_frontend;
-		} else {  		/* satellite */
+		} else {		/* satellite */
 			fe0->dvb.frontend = dvb_attach(tda10086_attach, &flydvbs, &dev->i2c_adap);
 			if (fe0->dvb.frontend) {
 				if (dvb_attach(tda826x_attach, fe0->dvb.frontend, 0x63,
@@ -1659,7 +1659,7 @@ static int dvb_init(struct saa7134_dev *dev)
 			if (configure_tda827x_fe(dev, &asus_tiger_3in1_config,
 							&tda827x_cfg_2) < 0)
 				goto detach_frontend;
-		} else {  		/* satellite */
+		} else {		/* satellite */
 			fe0->dvb.frontend = dvb_attach(tda10086_attach,
 						&flydvbs, &dev->i2c_adap);
 			if (fe0->dvb.frontend) {
diff --git a/drivers/media/pci/saa7134/saa7134-input.c b/drivers/media/pci/saa7134/saa7134-input.c
index 2d5abeddc079..33ee8322895e 100644
--- a/drivers/media/pci/saa7134/saa7134-input.c
+++ b/drivers/media/pci/saa7134/saa7134-input.c
@@ -43,7 +43,8 @@ MODULE_PARM_DESC(pinnacle_remote, "Specify Pinnacle PCTV remote: 0=coloured, 1=g
 	} while (0)
 #define ir_dbg(ir, fmt, arg...) do { \
 	if (ir_debug) \
-		printk(KERN_DEBUG pr_fmt("ir %s: " fmt), ir->name, ## arg); \
+		printk(KERN_DEBUG pr_fmt("ir %s: " fmt), ir->rc->device_name, \
+		       ## arg); \
 	} while (0)
 
 /* Helper function for raw decoding at GPIO16 or GPIO18 */
diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c
index 82d2a24644e4..1ca6a32ad10e 100644
--- a/drivers/media/pci/saa7134/saa7134-video.c
+++ b/drivers/media/pci/saa7134/saa7134-video.c
@@ -1227,11 +1227,11 @@ static ssize_t radio_read(struct file *file, char __user *data,
 	return cmd.result;
 }
 
-static unsigned int radio_poll(struct file *file, poll_table *wait)
+static __poll_t radio_poll(struct file *file, poll_table *wait)
 {
 	struct saa7134_dev *dev = video_drvdata(file);
 	struct saa6588_command cmd;
-	unsigned int rc = v4l2_ctrl_poll(file, wait);
+	__poll_t rc = v4l2_ctrl_poll(file, wait);
 
 	cmd.instance = file;
 	cmd.event_list = wait;
@@ -1531,6 +1531,8 @@ int saa7134_querycap(struct file *file, void *priv,
 	case VFL_TYPE_VBI:
 		cap->device_caps |= vbi_caps;
 		break;
+	default:
+		return -EINVAL;
 	}
 	cap->capabilities = radio_caps | video_caps | vbi_caps |
 		cap->device_caps | V4L2_CAP_DEVICE_CAPS;
@@ -2041,14 +2043,14 @@ static const struct v4l2_ioctl_ops radio_ioctl_ops = {
 struct video_device saa7134_video_template = {
 	.name				= "saa7134-video",
 	.fops				= &video_fops,
-	.ioctl_ops 			= &video_ioctl_ops,
+	.ioctl_ops			= &video_ioctl_ops,
 	.tvnorms			= SAA7134_NORMS,
 };
 
 struct video_device saa7134_radio_template = {
 	.name			= "saa7134-radio",
 	.fops			= &radio_fops,
-	.ioctl_ops 		= &radio_ioctl_ops,
+	.ioctl_ops		= &radio_ioctl_ops,
 };
 
 static const struct v4l2_ctrl_ops saa7134_ctrl_ops = {
diff --git a/drivers/media/pci/saa7134/saa7134.h b/drivers/media/pci/saa7134/saa7134.h
index 39c36e6aefbe..d99e937a98c1 100644
--- a/drivers/media/pci/saa7134/saa7134.h
+++ b/drivers/media/pci/saa7134/saa7134.h
@@ -261,8 +261,8 @@ struct saa7134_card_ir {
 #define SAA7134_BOARD_SABRENT_TV_PCB05     115
 #define SAA7134_BOARD_10MOONSTVMASTER3     116
 #define SAA7134_BOARD_AVERMEDIA_SUPER_007  117
-#define SAA7134_BOARD_BEHOLD_401  	118
-#define SAA7134_BOARD_BEHOLD_403  	119
+#define SAA7134_BOARD_BEHOLD_401	118
+#define SAA7134_BOARD_BEHOLD_403	119
 #define SAA7134_BOARD_BEHOLD_403FM	120
 #define SAA7134_BOARD_BEHOLD_405	121
 #define SAA7134_BOARD_BEHOLD_405FM	122
@@ -581,7 +581,7 @@ struct saa7134_dev {
 	/* config info */
 	unsigned int               board;
 	unsigned int               tuner_type;
-	unsigned int 		   radio_type;
+	unsigned int		   radio_type;
 	unsigned char		   tuner_addr;
 	unsigned char		   radio_addr;
 
@@ -592,7 +592,7 @@ struct saa7134_dev {
 	struct i2c_adapter         i2c_adap;
 	struct i2c_client          i2c_client;
 	unsigned char              eedata[256];
-	int 			   has_rds;
+	int			   has_rds;
 
 	/* video overlay */
 	struct v4l2_framebuffer    ovbuf;
diff --git a/drivers/media/pci/saa7146/hexium_gemini.c b/drivers/media/pci/saa7146/hexium_gemini.c
index d31a2d4494d1..5817d9cde4d0 100644
--- a/drivers/media/pci/saa7146/hexium_gemini.c
+++ b/drivers/media/pci/saa7146/hexium_gemini.c
@@ -27,6 +27,7 @@
 
 #include <media/drv-intf/saa7146_vv.h>
 #include <linux/module.h>
+#include <linux/kernel.h>
 
 static int debug;
 module_param(debug, int, 0);
@@ -69,8 +70,8 @@ struct hexium
 	struct video_device	video_dev;
 	struct i2c_adapter	i2c_adapter;
 
-	int 		cur_input;	/* current input */
-	v4l2_std_id 	cur_std;	/* current standard */
+	int		cur_input;	/* current input */
+	v4l2_std_id	cur_std;	/* current standard */
 };
 
 /* Samsung KS0127B decoder default registers */
@@ -137,19 +138,19 @@ static struct hexium_data hexium_input_select[] = {
    are currently *not* supported*/
 static struct saa7146_standard hexium_standards[] = {
 	{
-		.name	= "PAL", 	.id	= V4L2_STD_PAL,
-		.v_offset	= 28,	.v_field 	= 288,
-		.h_offset	= 1,	.h_pixels 	= 680,
+		.name	= "PAL",	.id	= V4L2_STD_PAL,
+		.v_offset	= 28,	.v_field	= 288,
+		.h_offset	= 1,	.h_pixels	= 680,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}, {
-		.name	= "NTSC", 	.id	= V4L2_STD_NTSC,
-		.v_offset	= 28,	.v_field 	= 240,
-		.h_offset	= 1,	.h_pixels 	= 640,
+		.name	= "NTSC",	.id	= V4L2_STD_NTSC,
+		.v_offset	= 28,	.v_field	= 240,
+		.h_offset	= 1,	.h_pixels	= 640,
 		.v_max_out	= 480,	.h_max_out	= 640,
 	}, {
-		.name	= "SECAM", 	.id	= V4L2_STD_SECAM,
-		.v_offset	= 28,	.v_field 	= 288,
-		.h_offset	= 1,	.h_pixels 	= 720,
+		.name	= "SECAM",	.id	= V4L2_STD_SECAM,
+		.v_offset	= 28,	.v_field	= 288,
+		.h_offset	= 1,	.h_pixels	= 720,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}
 };
@@ -388,7 +389,7 @@ static struct saa7146_ext_vv vv_data = {
 	.inputs = HEXIUM_INPUTS,
 	.capabilities = 0,
 	.stds = &hexium_standards[0],
-	.num_stds = sizeof(hexium_standards) / sizeof(struct saa7146_standard),
+	.num_stds = ARRAY_SIZE(hexium_standards),
 	.std_callback = &std_callback,
 };
 
diff --git a/drivers/media/pci/saa7146/hexium_orion.c b/drivers/media/pci/saa7146/hexium_orion.c
index 043318aa19e2..0a05176c18ab 100644
--- a/drivers/media/pci/saa7146/hexium_orion.c
+++ b/drivers/media/pci/saa7146/hexium_orion.c
@@ -27,6 +27,7 @@
 
 #include <media/drv-intf/saa7146_vv.h>
 #include <linux/module.h>
+#include <linux/kernel.h>
 
 static int debug;
 module_param(debug, int, 0);
@@ -187,19 +188,19 @@ static struct {
 
 static struct saa7146_standard hexium_standards[] = {
 	{
-		.name	= "PAL", 	.id	= V4L2_STD_PAL,
-		.v_offset	= 16,	.v_field 	= 288,
-		.h_offset	= 1,	.h_pixels 	= 680,
+		.name	= "PAL",	.id	= V4L2_STD_PAL,
+		.v_offset	= 16,	.v_field	= 288,
+		.h_offset	= 1,	.h_pixels	= 680,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}, {
-		.name	= "NTSC", 	.id	= V4L2_STD_NTSC,
-		.v_offset	= 16,	.v_field 	= 240,
-		.h_offset	= 1,	.h_pixels 	= 640,
+		.name	= "NTSC",	.id	= V4L2_STD_NTSC,
+		.v_offset	= 16,	.v_field	= 240,
+		.h_offset	= 1,	.h_pixels	= 640,
 		.v_max_out	= 480,	.h_max_out	= 640,
 	}, {
-		.name	= "SECAM", 	.id	= V4L2_STD_SECAM,
-		.v_offset	= 16,	.v_field 	= 288,
-		.h_offset	= 1,	.h_pixels 	= 720,
+		.name	= "SECAM",	.id	= V4L2_STD_SECAM,
+		.v_offset	= 16,	.v_field	= 288,
+		.h_offset	= 1,	.h_pixels	= 720,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}
 };
@@ -460,7 +461,7 @@ static struct saa7146_ext_vv vv_data = {
 	.inputs = HEXIUM_INPUTS,
 	.capabilities = 0,
 	.stds = &hexium_standards[0],
-	.num_stds = sizeof(hexium_standards) / sizeof(struct saa7146_standard),
+	.num_stds = ARRAY_SIZE(hexium_standards),
 	.std_callback = &std_callback,
 };
 
diff --git a/drivers/media/pci/saa7146/mxb.c b/drivers/media/pci/saa7146/mxb.c
index 930218cc2de1..6b5582b7c595 100644
--- a/drivers/media/pci/saa7146/mxb.c
+++ b/drivers/media/pci/saa7146/mxb.c
@@ -3,7 +3,7 @@
 
     Copyright (C) 1998-2006 Michael Hunold <michael@mihu.de>
 
-    Visit http://www.themm.net/~mihu/linux/saa7146/mxb.html 
+    Visit http://www.themm.net/~mihu/linux/saa7146/mxb.html
     for further details about this card.
 
     This program is free software; you can redistribute it and/or modify
@@ -30,6 +30,7 @@
 #include <media/v4l2-common.h>
 #include <media/i2c/saa7115.h>
 #include <linux/module.h>
+#include <linux/kernel.h>
 
 #include "tea6415c.h"
 #include "tea6420.h"
@@ -792,24 +793,24 @@ static int std_callback(struct saa7146_dev *dev, struct saa7146_standard *standa
 
 static struct saa7146_standard standard[] = {
 	{
-		.name	= "PAL-BG", 	.id	= V4L2_STD_PAL_BG,
-		.v_offset	= 0x17,	.v_field 	= 288,
-		.h_offset	= 0x14,	.h_pixels 	= 680,
+		.name	= "PAL-BG",	.id	= V4L2_STD_PAL_BG,
+		.v_offset	= 0x17,	.v_field	= 288,
+		.h_offset	= 0x14,	.h_pixels	= 680,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}, {
-		.name	= "PAL-I", 	.id	= V4L2_STD_PAL_I,
-		.v_offset	= 0x17,	.v_field 	= 288,
-		.h_offset	= 0x14,	.h_pixels 	= 680,
+		.name	= "PAL-I",	.id	= V4L2_STD_PAL_I,
+		.v_offset	= 0x17,	.v_field	= 288,
+		.h_offset	= 0x14,	.h_pixels	= 680,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}, {
-		.name	= "NTSC", 	.id	= V4L2_STD_NTSC,
-		.v_offset	= 0x16,	.v_field 	= 240,
-		.h_offset	= 0x06,	.h_pixels 	= 708,
+		.name	= "NTSC",	.id	= V4L2_STD_NTSC,
+		.v_offset	= 0x16,	.v_field	= 240,
+		.h_offset	= 0x06,	.h_pixels	= 708,
 		.v_max_out	= 480,	.h_max_out	= 640,
 	}, {
-		.name	= "SECAM", 	.id	= V4L2_STD_SECAM,
-		.v_offset	= 0x14,	.v_field 	= 288,
-		.h_offset	= 0x14,	.h_pixels 	= 720,
+		.name	= "SECAM",	.id	= V4L2_STD_SECAM,
+		.v_offset	= 0x14,	.v_field	= 288,
+		.h_offset	= 0x14,	.h_pixels	= 720,
 		.v_max_out	= 576,	.h_max_out	= 768,
 	}
 };
@@ -837,7 +838,7 @@ static struct saa7146_ext_vv vv_data = {
 	.inputs		= MXB_INPUTS,
 	.capabilities	= V4L2_CAP_TUNER | V4L2_CAP_VBI_CAPTURE | V4L2_CAP_AUDIO,
 	.stds		= &standard[0],
-	.num_stds	= sizeof(standard)/sizeof(struct saa7146_standard),
+	.num_stds	= ARRAY_SIZE(standard),
 	.std_callback	= &std_callback,
 };
 
diff --git a/drivers/media/pci/saa7164/Makefile b/drivers/media/pci/saa7164/Makefile
index 3896bcdb99d2..dea076744ec9 100644
--- a/drivers/media/pci/saa7164/Makefile
+++ b/drivers/media/pci/saa7164/Makefile
@@ -5,9 +5,5 @@ saa7164-objs	:= saa7164-cards.o saa7164-core.o saa7164-i2c.o saa7164-dvb.o \
 
 obj-$(CONFIG_VIDEO_SAA7164) += saa7164.o
 
-ccflags-y += -I$(srctree)/drivers/media/i2c
 ccflags-y += -I$(srctree)/drivers/media/tuners
-ccflags-y += -I$(srctree)/drivers/media/dvb-core
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends
-
-ccflags-y += $(extra-cflags-y) $(extra-cflags-m)
diff --git a/drivers/media/pci/saa7164/saa7164-encoder.c b/drivers/media/pci/saa7164/saa7164-encoder.c
index f21c245a54f7..32136ebe4f61 100644
--- a/drivers/media/pci/saa7164/saa7164-encoder.c
+++ b/drivers/media/pci/saa7164/saa7164-encoder.c
@@ -909,13 +909,13 @@ static ssize_t fops_read(struct file *file, char __user *buffer,
 	return ret;
 }
 
-static unsigned int fops_poll(struct file *file, poll_table *wait)
+static __poll_t fops_poll(struct file *file, poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct saa7164_encoder_fh *fh =
 		(struct saa7164_encoder_fh *)file->private_data;
 	struct saa7164_port *port = fh->port;
-	unsigned int mask = v4l2_ctrl_poll(file, wait);
+	__poll_t mask = v4l2_ctrl_poll(file, wait);
 
 	port->last_poll_msecs_diff = port->last_poll_msecs;
 	port->last_poll_msecs = jiffies_to_msecs(jiffies);
@@ -925,13 +925,13 @@ static unsigned int fops_poll(struct file *file, poll_table *wait)
 	saa7164_histogram_update(&port->poll_interval,
 		port->last_poll_msecs_diff);
 
-	if (!(req_events & (POLLIN | POLLRDNORM)))
+	if (!(req_events & (EPOLLIN | EPOLLRDNORM)))
 		return mask;
 
 	if (atomic_cmpxchg(&fh->v4l_reading, 0, 1) == 0) {
 		if (atomic_inc_return(&port->v4l_reader_count) == 1) {
 			if (saa7164_encoder_initialize(port) < 0)
-				return mask | POLLERR;
+				return mask | EPOLLERR;
 			saa7164_encoder_start_streaming(port);
 			msleep(200);
 		}
@@ -939,7 +939,7 @@ static unsigned int fops_poll(struct file *file, poll_table *wait)
 
 	/* Pull the first buffer from the used list */
 	if (!list_empty(&port->list_buf_used.list))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/media/pci/saa7164/saa7164-vbi.c b/drivers/media/pci/saa7164/saa7164-vbi.c
index 9255d7d23947..64ab91c24c18 100644
--- a/drivers/media/pci/saa7164/saa7164-vbi.c
+++ b/drivers/media/pci/saa7164/saa7164-vbi.c
@@ -614,11 +614,11 @@ static ssize_t fops_read(struct file *file, char __user *buffer,
 	return ret;
 }
 
-static unsigned int fops_poll(struct file *file, poll_table *wait)
+static __poll_t fops_poll(struct file *file, poll_table *wait)
 {
 	struct saa7164_vbi_fh *fh = (struct saa7164_vbi_fh *)file->private_data;
 	struct saa7164_port *port = fh->port;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	port->last_poll_msecs_diff = port->last_poll_msecs;
 	port->last_poll_msecs = jiffies_to_msecs(jiffies);
@@ -650,7 +650,7 @@ static unsigned int fops_poll(struct file *file, poll_table *wait)
 
 	/* Pull the first buffer from the used list */
 	if (!list_empty(&port->list_buf_used.list))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
diff --git a/drivers/media/pci/saa7164/saa7164.h b/drivers/media/pci/saa7164/saa7164.h
index 81b3f0e19993..f3358f43195f 100644
--- a/drivers/media/pci/saa7164/saa7164.h
+++ b/drivers/media/pci/saa7164/saa7164.h
@@ -50,11 +50,11 @@
 
 #include <media/tuner.h>
 #include <media/tveeprom.h>
-#include <dvb_demux.h>
-#include <dvb_frontend.h>
-#include <dvb_net.h>
-#include <dvbdev.h>
-#include <dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
+#include <media/dvbdev.h>
+#include <media/dmxdev.h>
 #include <media/v4l2-common.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-device.h>
diff --git a/drivers/media/pci/smipcie/Makefile b/drivers/media/pci/smipcie/Makefile
index 6006aac3c41f..214ebfe12cf7 100644
--- a/drivers/media/pci/smipcie/Makefile
+++ b/drivers/media/pci/smipcie/Makefile
@@ -5,6 +5,5 @@ smipcie-objs	:= smipcie-main.o smipcie-ir.o
 obj-$(CONFIG_DVB_SMIPCIE) += smipcie.o
 
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
 
diff --git a/drivers/media/pci/smipcie/smipcie.h b/drivers/media/pci/smipcie/smipcie.h
index c8368c78ddd5..a6c5b1bd7edb 100644
--- a/drivers/media/pci/smipcie/smipcie.h
+++ b/drivers/media/pci/smipcie/smipcie.h
@@ -29,12 +29,12 @@
 #include <linux/slab.h>
 #include <media/rc-core.h>
 
-#include "demux.h"
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
-#include "dvbdev.h"
+#include <media/demux.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
+#include <media/dvbdev.h>
 
 /* -------- Register Base -------- */
 #define    MSI_CONTROL_REG_BASE                 0x0800
diff --git a/drivers/media/pci/solo6x10/solo6x10-core.c b/drivers/media/pci/solo6x10/solo6x10-core.c
index ca0873e47bea..19ffd2ed3cc7 100644
--- a/drivers/media/pci/solo6x10/solo6x10-core.c
+++ b/drivers/media/pci/solo6x10/solo6x10-core.c
@@ -47,18 +47,19 @@ MODULE_PARM_DESC(full_eeprom, "Allow access to full 128B EEPROM (dangerous)");
 
 static void solo_set_time(struct solo_dev *solo_dev)
 {
-	struct timespec ts;
+	struct timespec64 ts;
 
-	ktime_get_ts(&ts);
+	ktime_get_ts64(&ts);
 
-	solo_reg_write(solo_dev, SOLO_TIMER_SEC, ts.tv_sec);
-	solo_reg_write(solo_dev, SOLO_TIMER_USEC, ts.tv_nsec / NSEC_PER_USEC);
+	/* no overflow because we use monotonic timestamps */
+	solo_reg_write(solo_dev, SOLO_TIMER_SEC, (u32)ts.tv_sec);
+	solo_reg_write(solo_dev, SOLO_TIMER_USEC, (u32)ts.tv_nsec / NSEC_PER_USEC);
 }
 
 static void solo_timer_sync(struct solo_dev *solo_dev)
 {
 	u32 sec, usec;
-	struct timespec ts;
+	struct timespec64 ts;
 	long diff;
 
 	if (solo_dev->type != SOLO_DEV_6110)
@@ -72,11 +73,11 @@ static void solo_timer_sync(struct solo_dev *solo_dev)
 	sec = solo_reg_read(solo_dev, SOLO_TIMER_SEC);
 	usec = solo_reg_read(solo_dev, SOLO_TIMER_USEC);
 
-	ktime_get_ts(&ts);
+	ktime_get_ts64(&ts);
 
-	diff = (long)ts.tv_sec - (long)sec;
+	diff = (s32)ts.tv_sec - (s32)sec;
 	diff = (diff * 1000000)
-		+ ((long)(ts.tv_nsec / NSEC_PER_USEC) - (long)usec);
+		+ ((s32)(ts.tv_nsec / NSEC_PER_USEC) - (s32)usec);
 
 	if (diff > 1000 || diff < -1000) {
 		solo_set_time(solo_dev);
diff --git a/drivers/media/pci/solo6x10/solo6x10-gpio.c b/drivers/media/pci/solo6x10/solo6x10-gpio.c
index 3d0d1aa2f6a8..7b4641a2cb84 100644
--- a/drivers/media/pci/solo6x10/solo6x10-gpio.c
+++ b/drivers/media/pci/solo6x10/solo6x10-gpio.c
@@ -162,7 +162,9 @@ static void solo_gpiochip_set(struct gpio_chip *chip,
 
 int solo_gpio_init(struct solo_dev *solo_dev)
 {
+#ifdef CONFIG_GPIOLIB
 	int ret;
+#endif
 
 	solo_gpio_config(solo_dev);
 #ifdef CONFIG_GPIOLIB
diff --git a/drivers/media/pci/ttpci/Makefile b/drivers/media/pci/ttpci/Makefile
index 0b805339c123..58ca12732aad 100644
--- a/drivers/media/pci/ttpci/Makefile
+++ b/drivers/media/pci/ttpci/Makefile
@@ -18,5 +18,5 @@ obj-$(CONFIG_DVB_BUDGET_CI) += budget-ci.o
 obj-$(CONFIG_DVB_BUDGET_PATCH) += budget-patch.o
 obj-$(CONFIG_DVB_AV7110) += dvb-ttpci.o
 
-ccflags-y += -Idrivers/media/dvb-core/ -Idrivers/media/dvb-frontends/
+ccflags-y += -Idrivers/media/dvb-frontends/
 ccflags-y += -Idrivers/media/tuners
diff --git a/drivers/media/pci/ttpci/av7110.c b/drivers/media/pci/ttpci/av7110.c
index 6d415bdeef18..d6816effb878 100644
--- a/drivers/media/pci/ttpci/av7110.c
+++ b/drivers/media/pci/ttpci/av7110.c
@@ -53,7 +53,7 @@
 
 #include <linux/dvb/frontend.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "ttpci-eeprom.h"
 #include "av7110.h"
@@ -324,14 +324,15 @@ static int DvbDmxFilterCallback(u8 *buffer1, size_t buffer1_len,
 		}
 		return dvbdmxfilter->feed->cb.sec(buffer1, buffer1_len,
 						  buffer2, buffer2_len,
-						  &dvbdmxfilter->filter);
+						  &dvbdmxfilter->filter, NULL);
 	case DMX_TYPE_TS:
 		if (!(dvbdmxfilter->feed->ts_type & TS_PACKET))
 			return 0;
 		if (dvbdmxfilter->feed->ts_type & TS_PAYLOAD_ONLY)
 			return dvbdmxfilter->feed->cb.ts(buffer1, buffer1_len,
 							 buffer2, buffer2_len,
-							 &dvbdmxfilter->feed->feed.ts);
+							 &dvbdmxfilter->feed->feed.ts,
+							 NULL);
 		else
 			av7110_p2t_write(buffer1, buffer1_len,
 					 dvbdmxfilter->feed->pid,
diff --git a/drivers/media/pci/ttpci/av7110.h b/drivers/media/pci/ttpci/av7110.h
index cbb150d6cbb1..8606ef5ebbe2 100644
--- a/drivers/media/pci/ttpci/av7110.h
+++ b/drivers/media/pci/ttpci/av7110.h
@@ -17,14 +17,14 @@
 #include <linux/dvb/net.h>
 #include <linux/mutex.h>
 
-#include "dvbdev.h"
-#include "demux.h"
-#include "dvb_demux.h"
-#include "dmxdev.h"
+#include <media/dvbdev.h>
+#include <media/demux.h>
+#include <media/dvb_demux.h>
+#include <media/dmxdev.h>
 #include "dvb_filter.h"
-#include "dvb_net.h"
-#include "dvb_ringbuffer.h"
-#include "dvb_frontend.h"
+#include <media/dvb_net.h>
+#include <media/dvb_ringbuffer.h>
+#include <media/dvb_frontend.h>
 #include "ves1820.h"
 #include "ves1x93.h"
 #include "stv0299.h"
@@ -52,7 +52,7 @@ extern int av7110_debug;
 enum {AV_PES_STREAM, PS_STREAM, TS_STREAM, PES_STREAM};
 
 enum av7110_video_mode {
-	AV7110_VIDEO_MODE_PAL 	= 0,
+	AV7110_VIDEO_MODE_PAL	= 0,
 	AV7110_VIDEO_MODE_NTSC	= 1
 };
 
diff --git a/drivers/media/pci/ttpci/av7110_av.c b/drivers/media/pci/ttpci/av7110_av.c
index 2aa4ba675194..ef1bc17cdc4d 100644
--- a/drivers/media/pci/ttpci/av7110_av.c
+++ b/drivers/media/pci/ttpci/av7110_av.c
@@ -99,7 +99,7 @@ int av7110_record_cb(struct dvb_filter_pes2ts *p2t, u8 *buf, size_t len)
 		buf[4] = buf[5] = 0;
 	if (dvbdmxfeed->ts_type & TS_PAYLOAD_ONLY)
 		return dvbdmxfeed->cb.ts(buf, len, NULL, 0,
-					 &dvbdmxfeed->feed.ts);
+					 &dvbdmxfeed->feed.ts, NULL);
 	else
 		return dvb_filter_pes2ts(p2t, buf, len, 1);
 }
@@ -109,7 +109,7 @@ static int dvb_filter_pes2ts_cb(void *priv, unsigned char *data)
 	struct dvb_demux_feed *dvbdmxfeed = (struct dvb_demux_feed *) priv;
 
 	dvbdmxfeed->cb.ts(data, 188, NULL, 0,
-			  &dvbdmxfeed->feed.ts);
+			  &dvbdmxfeed->feed.ts, NULL);
 	return 0;
 }
 
@@ -814,7 +814,7 @@ static void p_to_t(u8 const *buf, long int length, u16 pid, u8 *counter,
 			memcpy(obuf + l, buf + c, TS_SIZE - l);
 			c = length;
 		}
-		feed->cb.ts(obuf, 188, NULL, 0, &feed->feed.ts);
+		feed->cb.ts(obuf, 188, NULL, 0, &feed->feed.ts, NULL);
 		pes_start = 0;
 	}
 }
@@ -937,11 +937,11 @@ static int dvb_video_get_event (struct av7110 *av7110, struct video_event *event
  * DVB device file operations
  ******************************************************************************/
 
-static unsigned int dvb_video_poll(struct file *file, poll_table *wait)
+static __poll_t dvb_video_poll(struct file *file, poll_table *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct av7110 *av7110 = dvbdev->priv;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	dprintk(2, "av7110:%p, \n", av7110);
 
@@ -951,15 +951,15 @@ static unsigned int dvb_video_poll(struct file *file, poll_table *wait)
 	poll_wait(file, &av7110->video_events.wait_queue, wait);
 
 	if (av7110->video_events.eventw != av7110->video_events.eventr)
-		mask = POLLPRI;
+		mask = EPOLLPRI;
 
 	if ((file->f_flags & O_ACCMODE) != O_RDONLY) {
 		if (av7110->playing) {
 			if (FREE_COND)
-				mask |= (POLLOUT | POLLWRNORM);
+				mask |= (EPOLLOUT | EPOLLWRNORM);
 		} else {
 			/* if not playing: may play if asked for */
-			mask |= (POLLOUT | POLLWRNORM);
+			mask |= (EPOLLOUT | EPOLLWRNORM);
 		}
 	}
 
@@ -989,11 +989,11 @@ static ssize_t dvb_video_write(struct file *file, const char __user *buf,
 		return dvb_play(av7110, buf, count, file->f_flags & O_NONBLOCK, 1);
 }
 
-static unsigned int dvb_audio_poll(struct file *file, poll_table *wait)
+static __poll_t dvb_audio_poll(struct file *file, poll_table *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct av7110 *av7110 = dvbdev->priv;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	dprintk(2, "av7110:%p, \n", av7110);
 
@@ -1001,9 +1001,9 @@ static unsigned int dvb_audio_poll(struct file *file, poll_table *wait)
 
 	if (av7110->playing) {
 		if (dvb_ringbuffer_free(&av7110->aout) >= 20 * 1024)
-			mask |= (POLLOUT | POLLWRNORM);
+			mask |= (EPOLLOUT | EPOLLWRNORM);
 	} else /* if not playing: may play if asked for */
-		mask = (POLLOUT | POLLWRNORM);
+		mask = (EPOLLOUT | EPOLLWRNORM);
 
 	return mask;
 }
diff --git a/drivers/media/pci/ttpci/av7110_ca.c b/drivers/media/pci/ttpci/av7110_ca.c
index 1fe49171d823..d8c2f1b34d74 100644
--- a/drivers/media/pci/ttpci/av7110_ca.c
+++ b/drivers/media/pci/ttpci/av7110_ca.c
@@ -223,13 +223,13 @@ static int dvb_ca_open(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int dvb_ca_poll (struct file *file, poll_table *wait)
+static __poll_t dvb_ca_poll (struct file *file, poll_table *wait)
 {
 	struct dvb_device *dvbdev = file->private_data;
 	struct av7110 *av7110 = dvbdev->priv;
 	struct dvb_ringbuffer *rbuf = &av7110->ci_rbuffer;
 	struct dvb_ringbuffer *wbuf = &av7110->ci_wbuffer;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	dprintk(8, "av7110:%p\n",av7110);
 
@@ -237,10 +237,10 @@ static unsigned int dvb_ca_poll (struct file *file, poll_table *wait)
 	poll_wait(file, &wbuf->queue, wait);
 
 	if (!dvb_ringbuffer_empty(rbuf))
-		mask |= (POLLIN | POLLRDNORM);
+		mask |= (EPOLLIN | EPOLLRDNORM);
 
 	if (dvb_ringbuffer_free(wbuf) > 1024)
-		mask |= (POLLOUT | POLLWRNORM);
+		mask |= (EPOLLOUT | EPOLLWRNORM);
 
 	return mask;
 }
diff --git a/drivers/media/pci/ttpci/budget-av.c b/drivers/media/pci/ttpci/budget-av.c
index ac83fff9fe0b..abc98f1ad26c 100644
--- a/drivers/media/pci/ttpci/budget-av.c
+++ b/drivers/media/pci/ttpci/budget-av.c
@@ -51,7 +51,7 @@
 #include <linux/input.h>
 #include <linux/spinlock.h>
 
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 #define DEBICICAM		0x02420000
 
@@ -1181,14 +1181,14 @@ static u8 read_pwm(struct budget_av *budget_av)
 #define SUBID_DVBS_KNC1_PLUS		0x0011
 #define SUBID_DVBS_TYPHOON		0x4f56
 #define SUBID_DVBS_CINERGY1200		0x1154
-#define SUBID_DVBS_CYNERGY1200N 	0x1155
+#define SUBID_DVBS_CYNERGY1200N		0x1155
 #define SUBID_DVBS_TV_STAR		0x0014
 #define SUBID_DVBS_TV_STAR_PLUS_X4	0x0015
 #define SUBID_DVBS_TV_STAR_CI		0x0016
 #define SUBID_DVBS2_KNC1		0x0018
 #define SUBID_DVBS2_KNC1_OEM		0x0019
-#define SUBID_DVBS_EASYWATCH_1  	0x001a
-#define SUBID_DVBS_EASYWATCH_2  	0x001b
+#define SUBID_DVBS_EASYWATCH_1		0x001a
+#define SUBID_DVBS_EASYWATCH_2		0x001b
 #define SUBID_DVBS2_EASYWATCH		0x001d
 #define SUBID_DVBS_EASYWATCH		0x001e
 
diff --git a/drivers/media/pci/ttpci/budget-ci.c b/drivers/media/pci/ttpci/budget-ci.c
index 57af11804fd6..ec8f92540f7c 100644
--- a/drivers/media/pci/ttpci/budget-ci.c
+++ b/drivers/media/pci/ttpci/budget-ci.c
@@ -35,7 +35,7 @@
 
 #include "budget.h"
 
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 #include "stv0299.h"
 #include "stv0297.h"
 #include "tda1004x.h"
@@ -1050,70 +1050,70 @@ static const struct stb0899_s1_reg tt3200_stb0899_s1_init_1[] = {
 
 	{ STB0899_DEV_ID		, 0x81 },
 	{ STB0899_DISCNTRL1		, 0x32 },
-	{ STB0899_DISCNTRL2     	, 0x80 },
-	{ STB0899_DISRX_ST0     	, 0x04 },
-	{ STB0899_DISRX_ST1     	, 0x00 },
-	{ STB0899_DISPARITY     	, 0x00 },
+	{ STB0899_DISCNTRL2		, 0x80 },
+	{ STB0899_DISRX_ST0		, 0x04 },
+	{ STB0899_DISRX_ST1		, 0x00 },
+	{ STB0899_DISPARITY		, 0x00 },
 	{ STB0899_DISSTATUS		, 0x20 },
-	{ STB0899_DISF22        	, 0x8c },
-	{ STB0899_DISF22RX      	, 0x9a },
+	{ STB0899_DISF22		, 0x8c },
+	{ STB0899_DISF22RX		, 0x9a },
 	{ STB0899_SYSREG		, 0x0b },
-	{ STB0899_ACRPRESC      	, 0x11 },
-	{ STB0899_ACRDIV1       	, 0x0a },
-	{ STB0899_ACRDIV2       	, 0x05 },
-	{ STB0899_DACR1         	, 0x00 },
-	{ STB0899_DACR2         	, 0x00 },
-	{ STB0899_OUTCFG        	, 0x00 },
-	{ STB0899_MODECFG       	, 0x00 },
+	{ STB0899_ACRPRESC		, 0x11 },
+	{ STB0899_ACRDIV1		, 0x0a },
+	{ STB0899_ACRDIV2		, 0x05 },
+	{ STB0899_DACR1			, 0x00 },
+	{ STB0899_DACR2			, 0x00 },
+	{ STB0899_OUTCFG		, 0x00 },
+	{ STB0899_MODECFG		, 0x00 },
 	{ STB0899_IRQSTATUS_3		, 0x30 },
 	{ STB0899_IRQSTATUS_2		, 0x00 },
 	{ STB0899_IRQSTATUS_1		, 0x00 },
 	{ STB0899_IRQSTATUS_0		, 0x00 },
-	{ STB0899_IRQMSK_3      	, 0xf3 },
-	{ STB0899_IRQMSK_2      	, 0xfc },
-	{ STB0899_IRQMSK_1      	, 0xff },
+	{ STB0899_IRQMSK_3		, 0xf3 },
+	{ STB0899_IRQMSK_2		, 0xfc },
+	{ STB0899_IRQMSK_1		, 0xff },
 	{ STB0899_IRQMSK_0		, 0xff },
 	{ STB0899_IRQCFG		, 0x00 },
-	{ STB0899_I2CCFG        	, 0x88 },
-	{ STB0899_I2CRPT        	, 0x48 }, /* 12k Pullup, Repeater=16, Stop=disabled */
+	{ STB0899_I2CCFG		, 0x88 },
+	{ STB0899_I2CRPT		, 0x48 }, /* 12k Pullup, Repeater=16, Stop=disabled */
 	{ STB0899_IOPVALUE5		, 0x00 },
 	{ STB0899_IOPVALUE4		, 0x20 },
 	{ STB0899_IOPVALUE3		, 0xc9 },
 	{ STB0899_IOPVALUE2		, 0x90 },
 	{ STB0899_IOPVALUE1		, 0x40 },
 	{ STB0899_IOPVALUE0		, 0x00 },
-	{ STB0899_GPIO00CFG     	, 0x82 },
-	{ STB0899_GPIO01CFG     	, 0x82 },
-	{ STB0899_GPIO02CFG     	, 0x82 },
-	{ STB0899_GPIO03CFG     	, 0x82 },
-	{ STB0899_GPIO04CFG     	, 0x82 },
-	{ STB0899_GPIO05CFG     	, 0x82 },
-	{ STB0899_GPIO06CFG     	, 0x82 },
-	{ STB0899_GPIO07CFG     	, 0x82 },
-	{ STB0899_GPIO08CFG     	, 0x82 },
-	{ STB0899_GPIO09CFG     	, 0x82 },
-	{ STB0899_GPIO10CFG     	, 0x82 },
-	{ STB0899_GPIO11CFG     	, 0x82 },
-	{ STB0899_GPIO12CFG     	, 0x82 },
-	{ STB0899_GPIO13CFG     	, 0x82 },
-	{ STB0899_GPIO14CFG     	, 0x82 },
-	{ STB0899_GPIO15CFG     	, 0x82 },
-	{ STB0899_GPIO16CFG     	, 0x82 },
-	{ STB0899_GPIO17CFG     	, 0x82 },
-	{ STB0899_GPIO18CFG     	, 0x82 },
-	{ STB0899_GPIO19CFG     	, 0x82 },
-	{ STB0899_GPIO20CFG     	, 0x82 },
-	{ STB0899_SDATCFG       	, 0xb8 },
-	{ STB0899_SCLTCFG       	, 0xba },
-	{ STB0899_AGCRFCFG      	, 0x1c }, /* 0x11 */
-	{ STB0899_GPIO22        	, 0x82 }, /* AGCBB2CFG */
-	{ STB0899_GPIO21        	, 0x91 }, /* AGCBB1CFG */
-	{ STB0899_DIRCLKCFG     	, 0x82 },
-	{ STB0899_CLKOUT27CFG   	, 0x7e },
-	{ STB0899_STDBYCFG      	, 0x82 },
-	{ STB0899_CS0CFG        	, 0x82 },
-	{ STB0899_CS1CFG        	, 0x82 },
-	{ STB0899_DISEQCOCFG    	, 0x20 },
+	{ STB0899_GPIO00CFG		, 0x82 },
+	{ STB0899_GPIO01CFG		, 0x82 },
+	{ STB0899_GPIO02CFG		, 0x82 },
+	{ STB0899_GPIO03CFG		, 0x82 },
+	{ STB0899_GPIO04CFG		, 0x82 },
+	{ STB0899_GPIO05CFG		, 0x82 },
+	{ STB0899_GPIO06CFG		, 0x82 },
+	{ STB0899_GPIO07CFG		, 0x82 },
+	{ STB0899_GPIO08CFG		, 0x82 },
+	{ STB0899_GPIO09CFG		, 0x82 },
+	{ STB0899_GPIO10CFG		, 0x82 },
+	{ STB0899_GPIO11CFG		, 0x82 },
+	{ STB0899_GPIO12CFG		, 0x82 },
+	{ STB0899_GPIO13CFG		, 0x82 },
+	{ STB0899_GPIO14CFG		, 0x82 },
+	{ STB0899_GPIO15CFG		, 0x82 },
+	{ STB0899_GPIO16CFG		, 0x82 },
+	{ STB0899_GPIO17CFG		, 0x82 },
+	{ STB0899_GPIO18CFG		, 0x82 },
+	{ STB0899_GPIO19CFG		, 0x82 },
+	{ STB0899_GPIO20CFG		, 0x82 },
+	{ STB0899_SDATCFG		, 0xb8 },
+	{ STB0899_SCLTCFG		, 0xba },
+	{ STB0899_AGCRFCFG		, 0x1c }, /* 0x11 */
+	{ STB0899_GPIO22		, 0x82 }, /* AGCBB2CFG */
+	{ STB0899_GPIO21		, 0x91 }, /* AGCBB1CFG */
+	{ STB0899_DIRCLKCFG		, 0x82 },
+	{ STB0899_CLKOUT27CFG		, 0x7e },
+	{ STB0899_STDBYCFG		, 0x82 },
+	{ STB0899_CS0CFG		, 0x82 },
+	{ STB0899_CS1CFG		, 0x82 },
+	{ STB0899_DISEQCOCFG		, 0x20 },
 	{ STB0899_GPIO32CFG		, 0x82 },
 	{ STB0899_GPIO33CFG		, 0x82 },
 	{ STB0899_GPIO34CFG		, 0x82 },
@@ -1122,35 +1122,35 @@ static const struct stb0899_s1_reg tt3200_stb0899_s1_init_1[] = {
 	{ STB0899_GPIO37CFG		, 0x82 },
 	{ STB0899_GPIO38CFG		, 0x82 },
 	{ STB0899_GPIO39CFG		, 0x82 },
-	{ STB0899_NCOARSE       	, 0x15 }, /* 0x15 = 27 Mhz Clock, F/3 = 198MHz, F/6 = 99MHz */
-	{ STB0899_SYNTCTRL      	, 0x02 }, /* 0x00 = CLK from CLKI, 0x02 = CLK from XTALI */
-	{ STB0899_FILTCTRL      	, 0x00 },
-	{ STB0899_SYSCTRL       	, 0x00 },
-	{ STB0899_STOPCLK1      	, 0x20 },
-	{ STB0899_STOPCLK2      	, 0x00 },
+	{ STB0899_NCOARSE		, 0x15 }, /* 0x15 = 27 Mhz Clock, F/3 = 198MHz, F/6 = 99MHz */
+	{ STB0899_SYNTCTRL		, 0x02 }, /* 0x00 = CLK from CLKI, 0x02 = CLK from XTALI */
+	{ STB0899_FILTCTRL		, 0x00 },
+	{ STB0899_SYSCTRL		, 0x00 },
+	{ STB0899_STOPCLK1		, 0x20 },
+	{ STB0899_STOPCLK2		, 0x00 },
 	{ STB0899_INTBUFSTATUS		, 0x00 },
-	{ STB0899_INTBUFCTRL    	, 0x0a },
+	{ STB0899_INTBUFCTRL		, 0x0a },
 	{ 0xffff			, 0xff },
 };
 
 static const struct stb0899_s1_reg tt3200_stb0899_s1_init_3[] = {
-	{ STB0899_DEMOD         	, 0x00 },
-	{ STB0899_RCOMPC        	, 0xc9 },
-	{ STB0899_AGC1CN        	, 0x41 },
-	{ STB0899_AGC1REF       	, 0x10 },
+	{ STB0899_DEMOD			, 0x00 },
+	{ STB0899_RCOMPC		, 0xc9 },
+	{ STB0899_AGC1CN		, 0x41 },
+	{ STB0899_AGC1REF		, 0x10 },
 	{ STB0899_RTC			, 0x7a },
-	{ STB0899_TMGCFG        	, 0x4e },
-	{ STB0899_AGC2REF       	, 0x34 },
-	{ STB0899_TLSR          	, 0x84 },
-	{ STB0899_CFD           	, 0xc7 },
+	{ STB0899_TMGCFG		, 0x4e },
+	{ STB0899_AGC2REF		, 0x34 },
+	{ STB0899_TLSR			, 0x84 },
+	{ STB0899_CFD			, 0xc7 },
 	{ STB0899_ACLC			, 0x87 },
-	{ STB0899_BCLC          	, 0x94 },
-	{ STB0899_EQON          	, 0x41 },
-	{ STB0899_LDT           	, 0xdd },
-	{ STB0899_LDT2          	, 0xc9 },
-	{ STB0899_EQUALREF      	, 0xb4 },
-	{ STB0899_TMGRAMP       	, 0x10 },
-	{ STB0899_TMGTHD        	, 0x30 },
+	{ STB0899_BCLC			, 0x94 },
+	{ STB0899_EQON			, 0x41 },
+	{ STB0899_LDT			, 0xdd },
+	{ STB0899_LDT2			, 0xc9 },
+	{ STB0899_EQUALREF		, 0xb4 },
+	{ STB0899_TMGRAMP		, 0x10 },
+	{ STB0899_TMGTHD		, 0x30 },
 	{ STB0899_IDCCOMP		, 0xfb },
 	{ STB0899_QDCCOMP		, 0x03 },
 	{ STB0899_POWERI		, 0x3b },
@@ -1169,12 +1169,12 @@ static const struct stb0899_s1_reg tt3200_stb0899_s1_init_3[] = {
 	{ STB0899_NIRL			, 0x05 },
 	{ STB0899_ISYMB			, 0x17 },
 	{ STB0899_QSYMB			, 0xfa },
-	{ STB0899_SFRH          	, 0x2f },
-	{ STB0899_SFRM          	, 0x68 },
-	{ STB0899_SFRL          	, 0x40 },
-	{ STB0899_SFRUPH        	, 0x2f },
-	{ STB0899_SFRUPM        	, 0x68 },
-	{ STB0899_SFRUPL        	, 0x40 },
+	{ STB0899_SFRH			, 0x2f },
+	{ STB0899_SFRM			, 0x68 },
+	{ STB0899_SFRL			, 0x40 },
+	{ STB0899_SFRUPH		, 0x2f },
+	{ STB0899_SFRUPM		, 0x68 },
+	{ STB0899_SFRUPL		, 0x40 },
 	{ STB0899_EQUAI1		, 0xfd },
 	{ STB0899_EQUAQ1		, 0x04 },
 	{ STB0899_EQUAI2		, 0x0f },
@@ -1186,7 +1186,7 @@ static const struct stb0899_s1_reg tt3200_stb0899_s1_init_3[] = {
 	{ STB0899_EQUAI5		, 0xbd },
 	{ STB0899_EQUAQ5		, 0xf7 },
 	{ STB0899_DSTATUS2		, 0x00 },
-	{ STB0899_VSTATUS       	, 0x00 },
+	{ STB0899_VSTATUS		, 0x00 },
 	{ STB0899_VERROR		, 0xff },
 	{ STB0899_IQSWAP		, 0x2a },
 	{ STB0899_ECNT1M		, 0x00 },
@@ -1195,26 +1195,26 @@ static const struct stb0899_s1_reg tt3200_stb0899_s1_init_3[] = {
 	{ STB0899_ECNT2L		, 0x00 },
 	{ STB0899_ECNT3M		, 0x00 },
 	{ STB0899_ECNT3L		, 0x00 },
-	{ STB0899_FECAUTO1      	, 0x06 },
+	{ STB0899_FECAUTO1		, 0x06 },
 	{ STB0899_FECM			, 0x01 },
-	{ STB0899_VTH12         	, 0xf0 },
-	{ STB0899_VTH23         	, 0xa0 },
+	{ STB0899_VTH12			, 0xf0 },
+	{ STB0899_VTH23			, 0xa0 },
 	{ STB0899_VTH34			, 0x78 },
-	{ STB0899_VTH56         	, 0x4e },
-	{ STB0899_VTH67         	, 0x48 },
-	{ STB0899_VTH78         	, 0x38 },
-	{ STB0899_PRVIT         	, 0xff },
-	{ STB0899_VITSYNC       	, 0x19 },
-	{ STB0899_RSULC         	, 0xb1 }, /* DVB = 0xb1, DSS = 0xa1 */
-	{ STB0899_TSULC         	, 0x42 },
-	{ STB0899_RSLLC         	, 0x40 },
+	{ STB0899_VTH56			, 0x4e },
+	{ STB0899_VTH67			, 0x48 },
+	{ STB0899_VTH78			, 0x38 },
+	{ STB0899_PRVIT			, 0xff },
+	{ STB0899_VITSYNC		, 0x19 },
+	{ STB0899_RSULC			, 0xb1 }, /* DVB = 0xb1, DSS = 0xa1 */
+	{ STB0899_TSULC			, 0x42 },
+	{ STB0899_RSLLC			, 0x40 },
 	{ STB0899_TSLPL			, 0x12 },
-	{ STB0899_TSCFGH        	, 0x0c },
-	{ STB0899_TSCFGM        	, 0x00 },
-	{ STB0899_TSCFGL        	, 0x0c },
+	{ STB0899_TSCFGH		, 0x0c },
+	{ STB0899_TSCFGM		, 0x00 },
+	{ STB0899_TSCFGL		, 0x0c },
 	{ STB0899_TSOUT			, 0x4d }, /* 0x0d for CAM */
-	{ STB0899_RSSYNCDEL     	, 0x00 },
-	{ STB0899_TSINHDELH     	, 0x02 },
+	{ STB0899_RSSYNCDEL		, 0x00 },
+	{ STB0899_TSINHDELH		, 0x02 },
 	{ STB0899_TSINHDELM		, 0x00 },
 	{ STB0899_TSINHDELL		, 0x00 },
 	{ STB0899_TSLLSTKM		, 0x00 },
@@ -1225,18 +1225,18 @@ static const struct stb0899_s1_reg tt3200_stb0899_s1_init_3[] = {
 	{ STB0899_PCKLENLL		, 0xcc },
 	{ STB0899_RSPCKLEN		, 0xcc },
 	{ STB0899_TSSTATUS		, 0x80 },
-	{ STB0899_ERRCTRL1      	, 0xb6 },
-	{ STB0899_ERRCTRL2      	, 0x96 },
-	{ STB0899_ERRCTRL3      	, 0x89 },
+	{ STB0899_ERRCTRL1		, 0xb6 },
+	{ STB0899_ERRCTRL2		, 0x96 },
+	{ STB0899_ERRCTRL3		, 0x89 },
 	{ STB0899_DMONMSK1		, 0x27 },
 	{ STB0899_DMONMSK0		, 0x03 },
-	{ STB0899_DEMAPVIT      	, 0x5c },
+	{ STB0899_DEMAPVIT		, 0x5c },
 	{ STB0899_PLPARM		, 0x1f },
-	{ STB0899_PDELCTRL      	, 0x48 },
-	{ STB0899_PDELCTRL2     	, 0x00 },
-	{ STB0899_BBHCTRL1      	, 0x00 },
-	{ STB0899_BBHCTRL2      	, 0x00 },
-	{ STB0899_HYSTTHRESH    	, 0x77 },
+	{ STB0899_PDELCTRL		, 0x48 },
+	{ STB0899_PDELCTRL2		, 0x00 },
+	{ STB0899_BBHCTRL1		, 0x00 },
+	{ STB0899_BBHCTRL2		, 0x00 },
+	{ STB0899_HYSTTHRESH		, 0x77 },
 	{ STB0899_MATCSTM		, 0x00 },
 	{ STB0899_MATCSTL		, 0x00 },
 	{ STB0899_UPLCSTM		, 0x00 },
@@ -1275,7 +1275,7 @@ static struct stb0899_config tt3200_config = {
 
 	.postproc		= NULL,
 
-	.demod_address 		= 0x68,
+	.demod_address		= 0x68,
 
 	.xtal_freq		= 27000000,
 	.inversion		= IQ_SWAP_ON,
diff --git a/drivers/media/pci/ttpci/budget.h b/drivers/media/pci/ttpci/budget.h
index fae83866b199..a7463daf39f1 100644
--- a/drivers/media/pci/ttpci/budget.h
+++ b/drivers/media/pci/ttpci/budget.h
@@ -3,13 +3,13 @@
 #ifndef __BUDGET_DVB__
 #define __BUDGET_DVB__
 
-#include "dvb_frontend.h"
-#include "dvbdev.h"
-#include "demux.h"
-#include "dvb_demux.h"
-#include "dmxdev.h"
+#include <media/dvb_frontend.h>
+#include <media/dvbdev.h>
+#include <media/demux.h>
+#include <media/dvb_demux.h>
+#include <media/dmxdev.h>
 #include "dvb_filter.h"
-#include "dvb_net.h"
+#include <media/dvb_net.h>
 
 #include <linux/module.h>
 #include <linux/mutex.h>
diff --git a/drivers/media/pci/ttpci/dvb_filter.h b/drivers/media/pci/ttpci/dvb_filter.h
index 3d410d02a987..67a3c6333bca 100644
--- a/drivers/media/pci/ttpci/dvb_filter.h
+++ b/drivers/media/pci/ttpci/dvb_filter.h
@@ -19,7 +19,7 @@
 
 #include <linux/slab.h>
 
-#include "demux.h"
+#include <media/demux.h>
 
 typedef int (dvb_filter_pes2ts_cb_t) (void *, unsigned char *);
 
diff --git a/drivers/media/pci/tw5864/tw5864-video.c b/drivers/media/pci/tw5864/tw5864-video.c
index e7bd2b8484e3..ff2b7da90c08 100644
--- a/drivers/media/pci/tw5864/tw5864-video.c
+++ b/drivers/media/pci/tw5864/tw5864-video.c
@@ -730,7 +730,7 @@ static int tw5864_frameinterval_get(struct tw5864_input *input,
 		frameinterval->denominator = 25;
 		break;
 	default:
-	        dev_warn(&dev->pci->dev, "tw5864_frameinterval_get requested for unknown std %d\n",
+		dev_warn(&dev->pci->dev, "tw5864_frameinterval_get requested for unknown std %d\n",
 			 input->std);
 		return -EINVAL;
 	}
diff --git a/drivers/media/pci/zoran/videocodec.c b/drivers/media/pci/zoran/videocodec.c
index 303289a7fd3f..5ff23ef89215 100644
--- a/drivers/media/pci/zoran/videocodec.c
+++ b/drivers/media/pci/zoran/videocodec.c
@@ -325,7 +325,6 @@ static int proc_videocodecs_show(struct seq_file *m, void *v)
 	seq_printf(m, "<S>lave or attached <M>aster name  type flags    magic    ");
 	seq_printf(m, "(connected as)\n");
 
-	h = codeclist_top;
 	while (h) {
 		seq_printf(m, "S %32s %04x %08lx %08lx (TEMPLATE)\n",
 			      h->codec->name, h->codec->type,
diff --git a/drivers/media/pci/zoran/zoran_driver.c b/drivers/media/pci/zoran/zoran_driver.c
index d07840072337..8d4e7d930a66 100644
--- a/drivers/media/pci/zoran/zoran_driver.c
+++ b/drivers/media/pci/zoran/zoran_driver.c
@@ -2501,22 +2501,22 @@ static int zoran_s_jpegcomp(struct file *file, void *__fh,
 	return res;
 }
 
-static unsigned int
+static __poll_t
 zoran_poll (struct file *file,
 	    poll_table  *wait)
 {
 	struct zoran_fh *fh = file->private_data;
 	struct zoran *zr = fh->zr;
-	int res = v4l2_ctrl_poll(file, wait);
+	__poll_t res = v4l2_ctrl_poll(file, wait);
 	int frame;
 	unsigned long flags;
 
 	/* we should check whether buffers are ready to be synced on
 	 * (w/o waits - O_NONBLOCK) here
-	 * if ready for read (sync), return POLLIN|POLLRDNORM,
-	 * if ready for write (sync), return POLLOUT|POLLWRNORM,
-	 * if error, return POLLERR,
-	 * if no buffers queued or so, return POLLNVAL
+	 * if ready for read (sync), return EPOLLIN|EPOLLRDNORM,
+	 * if ready for write (sync), return EPOLLOUT|EPOLLWRNORM,
+	 * if error, return EPOLLERR,
+	 * if no buffers queued or so, return EPOLLNVAL
 	 */
 
 	switch (fh->map_mode) {
@@ -2536,7 +2536,7 @@ zoran_poll (struct file *file,
 		if (fh->buffers.active != ZORAN_FREE &&
 		    /* Buffer ready to DQBUF? */
 		    zr->v4l_buffers.buffer[frame].state == BUZ_STATE_DONE)
-			res |= POLLIN | POLLRDNORM;
+			res |= EPOLLIN | EPOLLRDNORM;
 		spin_unlock_irqrestore(&zr->spinlock, flags);
 
 		break;
@@ -2557,9 +2557,9 @@ zoran_poll (struct file *file,
 		if (fh->buffers.active != ZORAN_FREE &&
 		    zr->jpg_buffers.buffer[frame].state == BUZ_STATE_DONE) {
 			if (fh->map_mode == ZORAN_MAP_MODE_JPG_REC)
-				res |= POLLIN | POLLRDNORM;
+				res |= EPOLLIN | EPOLLRDNORM;
 			else
-				res |= POLLOUT | POLLWRNORM;
+				res |= EPOLLOUT | EPOLLWRNORM;
 		}
 		spin_unlock_irqrestore(&zr->spinlock, flags);
 
@@ -2570,7 +2570,7 @@ zoran_poll (struct file *file,
 			KERN_ERR
 			"%s: %s - internal error, unknown map_mode=%d\n",
 			ZR_DEVNAME(zr), __func__, fh->map_mode);
-		res |= POLLERR;
+		res |= EPOLLERR;
 	}
 
 	return res;
@@ -2792,21 +2792,21 @@ zoran_mmap (struct file           *file,
 }
 
 static const struct v4l2_ioctl_ops zoran_ioctl_ops = {
-	.vidioc_querycap    		    = zoran_querycap,
+	.vidioc_querycap		    = zoran_querycap,
 	.vidioc_s_selection		    = zoran_s_selection,
 	.vidioc_g_selection		    = zoran_g_selection,
-	.vidioc_enum_input     		    = zoran_enum_input,
-	.vidioc_g_input      		    = zoran_g_input,
-	.vidioc_s_input      		    = zoran_s_input,
-	.vidioc_enum_output    		    = zoran_enum_output,
-	.vidioc_g_output     		    = zoran_g_output,
-	.vidioc_s_output     		    = zoran_s_output,
+	.vidioc_enum_input		    = zoran_enum_input,
+	.vidioc_g_input			    = zoran_g_input,
+	.vidioc_s_input			    = zoran_s_input,
+	.vidioc_enum_output		    = zoran_enum_output,
+	.vidioc_g_output		    = zoran_g_output,
+	.vidioc_s_output		    = zoran_s_output,
 	.vidioc_g_fbuf			    = zoran_g_fbuf,
 	.vidioc_s_fbuf			    = zoran_s_fbuf,
-	.vidioc_g_std 			    = zoran_g_std,
-	.vidioc_s_std 			    = zoran_s_std,
-	.vidioc_g_jpegcomp 		    = zoran_g_jpegcomp,
-	.vidioc_s_jpegcomp 		    = zoran_s_jpegcomp,
+	.vidioc_g_std			    = zoran_g_std,
+	.vidioc_s_std			    = zoran_s_std,
+	.vidioc_g_jpegcomp		    = zoran_g_jpegcomp,
+	.vidioc_s_jpegcomp		    = zoran_s_jpegcomp,
 	.vidioc_overlay			    = zoran_overlay,
 	.vidioc_reqbufs			    = zoran_reqbufs,
 	.vidioc_querybuf		    = zoran_querybuf,
@@ -2814,18 +2814,18 @@ static const struct v4l2_ioctl_ops zoran_ioctl_ops = {
 	.vidioc_dqbuf			    = zoran_dqbuf,
 	.vidioc_streamon		    = zoran_streamon,
 	.vidioc_streamoff		    = zoran_streamoff,
-	.vidioc_enum_fmt_vid_cap 	    = zoran_enum_fmt_vid_cap,
-	.vidioc_enum_fmt_vid_out 	    = zoran_enum_fmt_vid_out,
-	.vidioc_enum_fmt_vid_overlay 	    = zoran_enum_fmt_vid_overlay,
-	.vidioc_g_fmt_vid_cap 		    = zoran_g_fmt_vid_cap,
+	.vidioc_enum_fmt_vid_cap	    = zoran_enum_fmt_vid_cap,
+	.vidioc_enum_fmt_vid_out	    = zoran_enum_fmt_vid_out,
+	.vidioc_enum_fmt_vid_overlay	    = zoran_enum_fmt_vid_overlay,
+	.vidioc_g_fmt_vid_cap		    = zoran_g_fmt_vid_cap,
 	.vidioc_g_fmt_vid_out               = zoran_g_fmt_vid_out,
 	.vidioc_g_fmt_vid_overlay           = zoran_g_fmt_vid_overlay,
-	.vidioc_s_fmt_vid_cap  		    = zoran_s_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap		    = zoran_s_fmt_vid_cap,
 	.vidioc_s_fmt_vid_out               = zoran_s_fmt_vid_out,
 	.vidioc_s_fmt_vid_overlay           = zoran_s_fmt_vid_overlay,
-	.vidioc_try_fmt_vid_cap  	    = zoran_try_fmt_vid_cap,
-	.vidioc_try_fmt_vid_out 	    = zoran_try_fmt_vid_out,
-	.vidioc_try_fmt_vid_overlay 	    = zoran_try_fmt_vid_overlay,
+	.vidioc_try_fmt_vid_cap		    = zoran_try_fmt_vid_cap,
+	.vidioc_try_fmt_vid_out		    = zoran_try_fmt_vid_out,
+	.vidioc_try_fmt_vid_overlay	    = zoran_try_fmt_vid_overlay,
 	.vidioc_subscribe_event             = v4l2_ctrl_subscribe_event,
 	.vidioc_unsubscribe_event           = v4l2_event_unsubscribe,
 };
diff --git a/drivers/media/pci/zoran/zr36057.h b/drivers/media/pci/zoran/zr36057.h
index c9ffef15532d..c8acb21dcb5c 100644
--- a/drivers/media/pci/zoran/zr36057.h
+++ b/drivers/media/pci/zoran/zr36057.h
@@ -103,8 +103,8 @@
 #define ZR36057_ICR_IntPinEn            (1<<24)
 
 #define ZR36057_I2CBR           0x044	/* I2C Bus Register */
-#define ZR36057_I2CBR_SDA       	(1<<1)
-#define ZR36057_I2CBR_SCL       	(1<<0)
+#define ZR36057_I2CBR_SDA		(1<<1)
+#define ZR36057_I2CBR_SCL		(1<<0)
 
 #define ZR36057_JMC             0x100	/* JPEG Mode and Control */
 #define ZR36057_JMC_JPG                 (1 << 31)
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index fd0c99859d6f..614fbef08ddc 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -353,10 +353,10 @@ config VIDEO_STI_HVA_DEBUGFS
 	depends on DEBUG_FS
 	help
 	  Select this to see information about the internal state and the last
-          operation of STMicroelectronics HVA multi-format video encoder in
-          debugfs.
+	  operation of STMicroelectronics HVA multi-format video encoder in
+	  debugfs.
 
-          Choose N unless you know you need this.
+	  Choose N unless you know you need this.
 
 config VIDEO_STI_DELTA
 	tristate "STMicroelectronics DELTA multi-format video decoder V4L2 driver"
@@ -586,10 +586,10 @@ config VIDEO_SAMSUNG_S5P_CEC
        select CEC_CORE
        select CEC_NOTIFIER
        ---help---
-         This is a driver for Samsung S5P HDMI CEC interface. It uses the
-         generic CEC framework interface.
-         CEC bus is present in the HDMI connector and enables communication
-         between compatible devices.
+	 This is a driver for Samsung S5P HDMI CEC interface. It uses the
+	 generic CEC framework interface.
+	 CEC bus is present in the HDMI connector and enables communication
+	 between compatible devices.
 
 config VIDEO_STI_HDMI_CEC
        tristate "STMicroelectronics STiH4xx HDMI CEC driver"
@@ -597,10 +597,10 @@ config VIDEO_STI_HDMI_CEC
        select CEC_CORE
        select CEC_NOTIFIER
        ---help---
-         This is a driver for STIH4xx HDMI CEC interface. It uses the
-         generic CEC framework interface.
-         CEC bus is present in the HDMI connector and enables communication
-         between compatible devices.
+	 This is a driver for STIH4xx HDMI CEC interface. It uses the
+	 generic CEC framework interface.
+	 CEC bus is present in the HDMI connector and enables communication
+	 between compatible devices.
 
 config VIDEO_STM32_HDMI_CEC
        tristate "STMicroelectronics STM32 HDMI CEC driver"
@@ -609,10 +609,10 @@ config VIDEO_STM32_HDMI_CEC
        select REGMAP_MMIO
        select CEC_CORE
        ---help---
-         This is a driver for STM32 interface. It uses the
-         generic CEC framework interface.
-         CEC bus is present in the HDMI connector and enables communication
-         between compatible devices.
+	 This is a driver for STM32 interface. It uses the
+	 generic CEC framework interface.
+	 CEC bus is present in the HDMI connector and enables communication
+	 between compatible devices.
 
 config VIDEO_TEGRA_HDMI_CEC
        tristate "Tegra HDMI CEC driver"
@@ -620,10 +620,10 @@ config VIDEO_TEGRA_HDMI_CEC
        select CEC_CORE
        select CEC_NOTIFIER
        ---help---
-         This is a driver for the Tegra HDMI CEC interface. It uses the
-         generic CEC framework interface.
-         The CEC bus is present in the HDMI connector and enables communication
-         between compatible devices.
+	 This is a driver for the Tegra HDMI CEC interface. It uses the
+	 generic CEC framework interface.
+	 The CEC bus is present in the HDMI connector and enables communication
+	 between compatible devices.
 
 endif #CEC_PLATFORM_DRIVERS
 
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 003b0bb2cddf..7f3080437be6 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -23,7 +23,7 @@ obj-$(CONFIG_VIDEO_TI_VPE)		+= ti-vpe/
 obj-$(CONFIG_VIDEO_TI_CAL)		+= ti-vpe/
 
 obj-$(CONFIG_VIDEO_MX2_EMMAPRP)		+= mx2_emmaprp.o
-obj-$(CONFIG_VIDEO_CODA) 		+= coda/
+obj-$(CONFIG_VIDEO_CODA)		+= coda/
 
 obj-$(CONFIG_VIDEO_SH_VEU)		+= sh_veu.o
 
@@ -33,8 +33,8 @@ obj-$(CONFIG_VIDEO_MEM2MEM_DEINTERLACE)	+= m2m-deinterlace.o
 
 obj-$(CONFIG_VIDEO_MUX)			+= video-mux.o
 
-obj-$(CONFIG_VIDEO_S3C_CAMIF) 		+= s3c-camif/
-obj-$(CONFIG_VIDEO_SAMSUNG_EXYNOS4_IS) 	+= exynos4-is/
+obj-$(CONFIG_VIDEO_S3C_CAMIF)		+= s3c-camif/
+obj-$(CONFIG_VIDEO_SAMSUNG_EXYNOS4_IS)	+= exynos4-is/
 obj-$(CONFIG_VIDEO_SAMSUNG_S5P_JPEG)	+= s5p-jpeg/
 obj-$(CONFIG_VIDEO_SAMSUNG_S5P_MFC)	+= s5p-mfc/
 
@@ -45,13 +45,13 @@ obj-$(CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC)	+= exynos-gsc/
 obj-$(CONFIG_VIDEO_STI_BDISP)		+= sti/bdisp/
 obj-$(CONFIG_VIDEO_STI_HVA)		+= sti/hva/
 obj-$(CONFIG_DVB_C8SECTPFE)		+= sti/c8sectpfe/
-obj-$(CONFIG_VIDEO_STI_HDMI_CEC) 	+= sti/cec/
+obj-$(CONFIG_VIDEO_STI_HDMI_CEC)	+= sti/cec/
 
 obj-$(CONFIG_VIDEO_STI_DELTA)		+= sti/delta/
 
 obj-$(CONFIG_VIDEO_TEGRA_HDMI_CEC)	+= tegra-cec/
 
-obj-y 					+= stm32/
+obj-y					+= stm32/
 
 obj-y                                   += blackfin/
 
@@ -62,9 +62,9 @@ obj-$(CONFIG_VIDEO_SH_VOU)		+= sh_vou.o
 obj-$(CONFIG_SOC_CAMERA)		+= soc_camera/
 
 obj-$(CONFIG_VIDEO_RCAR_DRIF)		+= rcar_drif.o
-obj-$(CONFIG_VIDEO_RENESAS_FCP) 	+= rcar-fcp.o
+obj-$(CONFIG_VIDEO_RENESAS_FCP)		+= rcar-fcp.o
 obj-$(CONFIG_VIDEO_RENESAS_FDP1)	+= rcar_fdp1.o
-obj-$(CONFIG_VIDEO_RENESAS_JPU) 	+= rcar_jpu.o
+obj-$(CONFIG_VIDEO_RENESAS_JPU)		+= rcar_jpu.o
 obj-$(CONFIG_VIDEO_RENESAS_VSP1)	+= vsp1/
 
 obj-$(CONFIG_VIDEO_ROCKCHIP_RGA)	+= rockchip/rga/
@@ -82,8 +82,6 @@ obj-$(CONFIG_VIDEO_ATMEL_ISI)		+= atmel/
 
 obj-$(CONFIG_VIDEO_STM32_DCMI)		+= stm32/
 
-ccflags-y += -I$(srctree)/drivers/media/i2c
-
 obj-$(CONFIG_VIDEO_MEDIATEK_VPU)	+= mtk-vpu/
 
 obj-$(CONFIG_VIDEO_MEDIATEK_VCODEC)	+= mtk-vcodec/
diff --git a/drivers/media/platform/am437x/am437x-vpfe.c b/drivers/media/platform/am437x/am437x-vpfe.c
index 0997c640191d..601ae6487617 100644
--- a/drivers/media/platform/am437x/am437x-vpfe.c
+++ b/drivers/media/platform/am437x/am437x-vpfe.c
@@ -2304,8 +2304,8 @@ vpfe_async_bound(struct v4l2_async_notifier *notifier,
 	vpfe_dbg(1, vpfe, "vpfe_async_bound\n");
 
 	for (i = 0; i < ARRAY_SIZE(vpfe->cfg->asd); i++) {
-		if (vpfe->cfg->asd[i]->match.fwnode.fwnode ==
-		    asd[i].match.fwnode.fwnode) {
+		if (vpfe->cfg->asd[i]->match.fwnode ==
+		    asd[i].match.fwnode) {
 			sdinfo = &vpfe->cfg->sub_devs[i];
 			vpfe->sd[i] = subdev;
 			vpfe->sd[i]->grp_id = sdinfo->grp_id;
@@ -2510,7 +2510,7 @@ vpfe_get_pdata(struct platform_device *pdev)
 		}
 
 		pdata->asd[i]->match_type = V4L2_ASYNC_MATCH_FWNODE;
-		pdata->asd[i]->match.fwnode.fwnode = of_fwnode_handle(rem);
+		pdata->asd[i]->match.fwnode = of_fwnode_handle(rem);
 		of_node_put(rem);
 	}
 
diff --git a/drivers/media/platform/arv.c b/drivers/media/platform/arv.c
index 8fe59bf6cd3f..1e865fea803c 100644
--- a/drivers/media/platform/arv.c
+++ b/drivers/media/platform/arv.c
@@ -56,7 +56,7 @@
 
 #define VERSION	"0.0.5"
 
-#define ar_inl(addr) 		inl((unsigned long)(addr))
+#define ar_inl(addr)		inl((unsigned long)(addr))
 #define ar_outl(val, addr)	outl((unsigned long)(val), (unsigned long)(addr))
 
 extern struct cpuinfo_m32r	boot_cpu_data;
@@ -66,7 +66,7 @@ extern struct cpuinfo_m32r	boot_cpu_data;
  *	Note that M32700UT does not support CIF mode, but QVGA is
  *	supported by M32700UT hardware using VGA mode of AR LSI.
  *
- * 	Supported: VGA  (Normal mode, Interlace mode)
+ *	Supported: VGA  (Normal mode, Interlace mode)
  *		   QVGA (Always Interlace mode of VGA)
  *
  */
@@ -210,8 +210,8 @@ static void init_iic(void)
 	 * ICU Setting (iic)
 	 */
 	/* I2C Setting */
-	ar_outl(0x0, PLDI2CCR);      	/* I2CCR Disable                   */
-	ar_outl(0x0300, PLDI2CMOD); 	/* I2CMOD ACK/8b-data/7b-addr/auto */
+	ar_outl(0x0, PLDI2CCR);		/* I2CCR Disable                   */
+	ar_outl(0x0300, PLDI2CMOD);	/* I2CMOD ACK/8b-data/7b-addr/auto */
 	ar_outl(0x1, PLDI2CACK);	/* I2CACK ACK                      */
 
 	/* I2C CLK */
@@ -222,7 +222,7 @@ static void init_iic(void)
 		ar_outl(244, PLDI2CFREQ);	/* BCLK = 50MHz */
 	else
 		ar_outl(244, PLDI2CFREQ);	/* default: BCLK = 50MHz */
-	ar_outl(0x1, PLDI2CCR); 	/* I2CCR Enable */
+	ar_outl(0x1, PLDI2CCR);		/* I2CCR Enable */
 }
 
 /**************************************************************************
@@ -300,9 +300,9 @@ static ssize_t ar_read(struct file *file, char *buf, size_t count, loff_t *ppos)
 	ar_outl(ARDATA32, M32R_DMA0CSA_PORTL);
 	ar_outl(ARDATA32, M32R_DMA0RSA_PORTL);
 	ar_outl(ar->line_buff, M32R_DMA0CDA_PORTL);	/* destination addr. */
-	ar_outl(ar->line_buff, M32R_DMA0RDA_PORTL); 	/* reload address */
-	ar_outl(ar->line_bytes, M32R_DMA0CBCUT_PORTL); 	/* byte count (bytes) */
-	ar_outl(ar->line_bytes, M32R_DMA0RBCUT_PORTL); 	/* reload count (bytes) */
+	ar_outl(ar->line_buff, M32R_DMA0RDA_PORTL);	/* reload address */
+	ar_outl(ar->line_bytes, M32R_DMA0CBCUT_PORTL);	/* byte count (bytes) */
+	ar_outl(ar->line_bytes, M32R_DMA0RBCUT_PORTL);	/* reload count (bytes) */
 
 	/*
 	 * Okay, kick AR LSI to invoke an interrupt
@@ -364,7 +364,7 @@ static ssize_t ar_read(struct file *file, char *buf, size_t count, loff_t *ppos)
 
 	/*
 	 * convert YUV422 to YUV422P
-	 * 	+--------------------+
+	 *	+--------------------+
 	 *	|  Y0,Y1,...	     |
 	 *	|  ..............Yn  |
 	 *	+--------------------+
@@ -533,9 +533,9 @@ static void ar_interrupt(int irq, void *dev)
 	line_count = ar_inl(ARVHCOUNT);			/* line number */
 	if (ar->mode == AR_MODE_INTERLACE && ar->size == AR_SIZE_VGA) {
 		/* operations for interlace mode */
-		if (line_count < (AR_HEIGHT_VGA / 2)) 	/* even line */
+		if (line_count < (AR_HEIGHT_VGA / 2))	/* even line */
 			line_number = (line_count << 1);
-		else 					/* odd line */
+		else					/* odd line */
 			line_number =
 			(((line_count - (AR_HEIGHT_VGA / 2)) << 1) + 1);
 	} else {
@@ -568,7 +568,7 @@ static void ar_interrupt(int irq, void *dev)
 		 * if captured all line of a frame, disable AR interrupt
 		 * and wake a process up.
 		 */
-		if (line_number == (ar->height - 1)) { 	/* end  of line */
+		if (line_number == (ar->height - 1)) {	/* end  of line */
 
 			ar->start_capture = 0;
 
@@ -590,7 +590,7 @@ static void ar_interrupt(int irq, void *dev)
 
 /*
  * ar_initialize()
- * 	ar_initialize() is called by video_register_device() and
+ *	ar_initialize() is called by video_register_device() and
  *	initializes AR LSI and peripherals.
  *
  *	-1 is returned in all failures.
@@ -718,14 +718,14 @@ static const struct v4l2_file_operations ar_fops = {
 };
 
 static const struct v4l2_ioctl_ops ar_ioctl_ops = {
-	.vidioc_querycap    		    = ar_querycap,
-	.vidioc_g_input      		    = ar_g_input,
-	.vidioc_s_input      		    = ar_s_input,
-	.vidioc_enum_input   		    = ar_enum_input,
-	.vidioc_enum_fmt_vid_cap 	    = ar_enum_fmt_vid_cap,
-	.vidioc_g_fmt_vid_cap 		    = ar_g_fmt_vid_cap,
-	.vidioc_s_fmt_vid_cap  		    = ar_s_fmt_vid_cap,
-	.vidioc_try_fmt_vid_cap  	    = ar_try_fmt_vid_cap,
+	.vidioc_querycap		    = ar_querycap,
+	.vidioc_g_input			    = ar_g_input,
+	.vidioc_s_input			    = ar_s_input,
+	.vidioc_enum_input		    = ar_enum_input,
+	.vidioc_enum_fmt_vid_cap	    = ar_enum_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap		    = ar_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap		    = ar_s_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap		    = ar_try_fmt_vid_cap,
 };
 
 #define ALIGN4(x)	((((int)(x)) & 0x3) == 0)
@@ -776,9 +776,9 @@ static int __init ar_init(void)
 	video_set_drvdata(&ar->vdev, ar);
 
 	if (vga) {
-		ar->width 	= AR_WIDTH_VGA;
-		ar->height 	= AR_HEIGHT_VGA;
-		ar->size 	= AR_SIZE_VGA;
+		ar->width	= AR_WIDTH_VGA;
+		ar->height	= AR_HEIGHT_VGA;
+		ar->size	= AR_SIZE_VGA;
 		ar->frame_bytes = AR_FRAME_BYTES_VGA;
 		ar->line_bytes	= AR_LINE_BYTES_VGA;
 		if (vga_interlace)
@@ -786,9 +786,9 @@ static int __init ar_init(void)
 		else
 			ar->mode = AR_MODE_NORMAL;
 	} else {
-		ar->width 	= AR_WIDTH_QVGA;
-		ar->height 	= AR_HEIGHT_QVGA;
-		ar->size 	= AR_SIZE_QVGA;
+		ar->width	= AR_WIDTH_QVGA;
+		ar->height	= AR_HEIGHT_QVGA;
+		ar->size	= AR_SIZE_QVGA;
 		ar->frame_bytes = AR_FRAME_BYTES_QVGA;
 		ar->line_bytes	= AR_LINE_BYTES_QVGA;
 		ar->mode	= AR_MODE_INTERLACE;
diff --git a/drivers/media/platform/atmel/atmel-isc.c b/drivers/media/platform/atmel/atmel-isc.c
index 13f1c1c797b0..34676409ca08 100644
--- a/drivers/media/platform/atmel/atmel-isc.c
+++ b/drivers/media/platform/atmel/atmel-isc.c
@@ -2039,10 +2039,10 @@ static int isc_parse_dt(struct device *dev, struct isc_device *isc)
 
 	INIT_LIST_HEAD(&isc->subdev_entities);
 
-	for (; ;) {
+	while (1) {
 		epn = of_graph_get_next_endpoint(np, epn);
 		if (!epn)
-			break;
+			return 0;
 
 		rem = of_graph_get_remote_port_parent(epn);
 		if (!rem) {
@@ -2088,7 +2088,7 @@ static int isc_parse_dt(struct device *dev, struct isc_device *isc)
 			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_PPOL_LOW;
 
 		subdev_entity->asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
-		subdev_entity->asd->match.fwnode.fwnode =
+		subdev_entity->asd->match.fwnode =
 			of_fwnode_handle(rem);
 		list_add_tail(&subdev_entity->list, &isc->subdev_entities);
 	}
diff --git a/drivers/media/platform/atmel/atmel-isi.c b/drivers/media/platform/atmel/atmel-isi.c
index e900995143a3..9958918e2449 100644
--- a/drivers/media/platform/atmel/atmel-isi.c
+++ b/drivers/media/platform/atmel/atmel-isi.c
@@ -1128,7 +1128,7 @@ static int isi_graph_parse(struct atmel_isi *isi, struct device_node *node)
 		/* Remote node to connect */
 		isi->entity.node = remote;
 		isi->entity.asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
-		isi->entity.asd.match.fwnode.fwnode = of_fwnode_handle(remote);
+		isi->entity.asd.match.fwnode = of_fwnode_handle(remote);
 		return 0;
 	}
 }
diff --git a/drivers/media/platform/blackfin/ppi.c b/drivers/media/platform/blackfin/ppi.c
index 478eb2f7d723..d3dc765c1609 100644
--- a/drivers/media/platform/blackfin/ppi.c
+++ b/drivers/media/platform/blackfin/ppi.c
@@ -52,7 +52,7 @@ static irqreturn_t ppi_irq_err(int irq, void *dev_id)
 		struct bfin_ppi_regs *reg = info->base;
 		unsigned short status;
 
-		/* register on bf561 is cleared when read 
+		/* register on bf561 is cleared when read
 		 * others are W1C
 		 */
 		status = bfin_read16(&reg->status);
diff --git a/drivers/media/platform/coda/coda-bit.c b/drivers/media/platform/coda/coda-bit.c
index bfc4ecf6f068..9fe113cb901f 100644
--- a/drivers/media/platform/coda/coda-bit.c
+++ b/drivers/media/platform/coda/coda-bit.c
@@ -395,6 +395,7 @@ static int coda_alloc_framebuffers(struct coda_ctx *ctx,
 
 	if (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264 ||
 	    ctx->codec->dst_fourcc == V4L2_PIX_FMT_H264 ||
+	    ctx->codec->src_fourcc == V4L2_PIX_FMT_MPEG4 ||
 	    ctx->codec->dst_fourcc == V4L2_PIX_FMT_MPEG4) {
 		width = round_up(q_data->width, 16);
 		height = round_up(q_data->height, 16);
@@ -413,8 +414,10 @@ static int coda_alloc_framebuffers(struct coda_ctx *ctx,
 			size = round_up(ysize, 4096) + ysize / 2;
 		else
 			size = ysize + ysize / 2;
-		if (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264 &&
-		    dev->devtype->product != CODA_DX6)
+		/* Add space for mvcol buffers */
+		if (dev->devtype->product != CODA_DX6 &&
+		    (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264 ||
+		     (ctx->codec->src_fourcc == V4L2_PIX_FMT_MPEG4 && i == 0)))
 			size += ysize / 4;
 		name = kasprintf(GFP_KERNEL, "fb%d", i);
 		if (!name) {
@@ -452,18 +455,16 @@ static int coda_alloc_framebuffers(struct coda_ctx *ctx,
 		coda_parabuf_write(ctx, i * 3 + 1, cb);
 		coda_parabuf_write(ctx, i * 3 + 2, cr);
 
-		/* mvcol buffer for h.264 */
-		if (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264 &&
-		    dev->devtype->product != CODA_DX6)
+		if (dev->devtype->product == CODA_DX6)
+			continue;
+
+		/* mvcol buffer for h.264 and mpeg4 */
+		if (ctx->codec->src_fourcc == V4L2_PIX_FMT_H264)
 			coda_parabuf_write(ctx, 96 + i, mvcol);
+		if (ctx->codec->src_fourcc == V4L2_PIX_FMT_MPEG4 && i == 0)
+			coda_parabuf_write(ctx, 97, mvcol);
 	}
 
-	/* mvcol buffer for mpeg4 */
-	if ((dev->devtype->product != CODA_DX6) &&
-	    (ctx->codec->src_fourcc == V4L2_PIX_FMT_MPEG4))
-		coda_parabuf_write(ctx, 97, ctx->internal_frames[0].paddr +
-					    ysize + ysize/4 + ysize/4);
-
 	return 0;
 }
 
diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c
index 15eb5dc4dff9..e8a7554a61d2 100644
--- a/drivers/media/platform/coda/coda-common.c
+++ b/drivers/media/platform/coda/coda-common.c
@@ -486,8 +486,8 @@ static int coda_try_fmt_vdoa(struct coda_ctx *ctx, struct v4l2_format *f,
 		return 0;
 	}
 
-	err = vdoa_context_configure(NULL, f->fmt.pix.width, f->fmt.pix.height,
-				     f->fmt.pix.pixelformat);
+	err = vdoa_context_configure(NULL, round_up(f->fmt.pix.width, 16),
+				     f->fmt.pix.height, f->fmt.pix.pixelformat);
 	if (err) {
 		*use_vdoa = false;
 		return 0;
@@ -730,7 +730,8 @@ static int coda_s_fmt(struct coda_ctx *ctx, struct v4l2_format *f,
 	if (ctx->tiled_map_type == GDI_TILED_FRAME_MB_RASTER_MAP &&
 	    !coda_try_fmt_vdoa(ctx, f, &ctx->use_vdoa) &&
 	    ctx->use_vdoa)
-		vdoa_context_configure(ctx->vdoa, f->fmt.pix.width,
+		vdoa_context_configure(ctx->vdoa,
+				       round_up(f->fmt.pix.width, 16),
 				       f->fmt.pix.height,
 				       f->fmt.pix.pixelformat);
 	else
@@ -1884,6 +1885,12 @@ static int coda_queue_init(struct coda_ctx *ctx, struct vb2_queue *vq)
 	 * that videobuf2 will keep the value of bytesused intact.
 	 */
 	vq->allow_zero_bytesused = 1;
+	/*
+	 * We might be fine with no buffers on some of the queues, but that
+	 * would need to be reflected in job_ready(). Currently we expect all
+	 * queues to have at least one buffer queued.
+	 */
+	vq->min_buffers_needed = 1;
 	vq->dev = &ctx->dev->plat_dev->dev;
 
 	return vb2_queue_init(vq);
diff --git a/drivers/media/platform/coda/coda_regs.h b/drivers/media/platform/coda/coda_regs.h
index 38df5fd9a2fa..3b650b8aabe9 100644
--- a/drivers/media/platform/coda/coda_regs.h
+++ b/drivers/media/platform/coda/coda_regs.h
@@ -125,7 +125,7 @@
 #define		CODA9_MODE_ENCODE_H264		8
 #define		CODA9_MODE_ENCODE_MP4		11
 #define		CODA9_MODE_ENCODE_MJPG		13
-#define 	CODA_MODE_INVALID		0xffff
+#define		CODA_MODE_INVALID		0xffff
 #define CODA_REG_BIT_INT_ENABLE		0x170
 #define		CODA_INT_INTERRUPT_ENABLE	(1 << 3)
 #define CODA_REG_BIT_INT_REASON			0x174
@@ -254,7 +254,6 @@
 #define		CODA9_STD_H264					0
 #define		CODA_STD_H263					1
 #define		CODA_STD_H264					2
-#define		CODA_STD_MJPG					3
 #define		CODA9_STD_MPEG4					3
 
 #define CODA_CMD_ENC_SEQ_SRC_SIZE				0x190
diff --git a/drivers/media/platform/coda/imx-vdoa.c b/drivers/media/platform/coda/imx-vdoa.c
index 8eb3e0c05473..85a66e4e2f9a 100644
--- a/drivers/media/platform/coda/imx-vdoa.c
+++ b/drivers/media/platform/coda/imx-vdoa.c
@@ -86,7 +86,6 @@ struct vdoa_data {
 	struct device		*dev;
 	struct clk		*vdoa_clk;
 	void __iomem		*regs;
-	int			irq;
 };
 
 struct vdoa_q_data {
@@ -293,6 +292,7 @@ static int vdoa_probe(struct platform_device *pdev)
 {
 	struct vdoa_data *vdoa;
 	struct resource *res;
+	int ret;
 
 	dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
 
@@ -316,12 +316,12 @@ static int vdoa_probe(struct platform_device *pdev)
 	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 	if (!res)
 		return -EINVAL;
-	vdoa->irq = devm_request_threaded_irq(&pdev->dev, res->start, NULL,
+	ret = devm_request_threaded_irq(&pdev->dev, res->start, NULL,
 					vdoa_irq_handler, IRQF_ONESHOT,
 					"vdoa", vdoa);
-	if (vdoa->irq < 0) {
+	if (ret < 0) {
 		dev_err(vdoa->dev, "Failed to get irq\n");
-		return vdoa->irq;
+		return ret;
 	}
 
 	platform_set_drvdata(pdev, vdoa);
diff --git a/drivers/media/platform/davinci/dm355_ccdc.c b/drivers/media/platform/davinci/dm355_ccdc.c
index 89cb3094d7e6..238d01b7f066 100644
--- a/drivers/media/platform/davinci/dm355_ccdc.c
+++ b/drivers/media/platform/davinci/dm355_ccdc.c
@@ -20,10 +20,10 @@
  * pre-process the Bayer RGB data, before writing it to SDRAM.
  *
  * TODO: 1) Raw bayer parameter settings and bayer capture
- * 	 2) Split module parameter structure to module specific ioctl structs
+ *	 2) Split module parameter structure to module specific ioctl structs
  *	 3) add support for lense shading correction
  *	 4) investigate if enum used for user space type definition
- * 	    to be replaced by #defines or integer
+ *	    to be replaced by #defines or integer
  */
 #include <linux/platform_device.h>
 #include <linux/uaccess.h>
diff --git a/drivers/media/platform/davinci/dm355_ccdc_regs.h b/drivers/media/platform/davinci/dm355_ccdc_regs.h
index a753ce262583..20ba390763b5 100644
--- a/drivers/media/platform/davinci/dm355_ccdc_regs.h
+++ b/drivers/media/platform/davinci/dm355_ccdc_regs.h
@@ -107,7 +107,7 @@
 #define CCDC_RAW_IP_MODE			0
 #define CCDC_VDHDOUT_INPUT			0
 #define CCDC_YCINSWP_RAW			(0 << 4)
-#define CCDC_EXWEN_DISABLE 			0
+#define CCDC_EXWEN_DISABLE			0
 #define CCDC_DATAPOL_NORMAL			0
 #define CCDC_CCDCFG_FIDMD_LATCH_VSYNC		0
 #define CCDC_CCDCFG_FIDMD_NO_LATCH_VSYNC	(1 << 6)
@@ -152,7 +152,7 @@
 #define CCDC_ALAW_GAMMA_WD_MASK			7
 #define CCDC_REC656IF_BT656_EN			3
 
-#define CCDC_FMTCFG_FMTMODE_MASK 		3
+#define CCDC_FMTCFG_FMTMODE_MASK		3
 #define CCDC_FMTCFG_FMTMODE_SHIFT		1
 #define CCDC_FMTCFG_LNUM_MASK			3
 #define CCDC_FMTCFG_LNUM_SHIFT			4
@@ -196,7 +196,7 @@
 #define CCDC_LATCH_ON_VSYNC_DISABLE		(1 << 15)
 #define CCDC_LATCH_ON_VSYNC_ENABLE		(0 << 15)
 #define CCDC_FPC_ENABLE				(1 << 15)
-#define CCDC_FPC_FPC_NUM_MASK 			0x7FFF
+#define CCDC_FPC_FPC_NUM_MASK			0x7FFF
 #define CCDC_DATA_PACK_ENABLE			(1 << 11)
 #define CCDC_FMT_HORZ_FMTLNH_MASK		0x1FFF
 #define CCDC_FMT_HORZ_FMTSPH_MASK		0x1FFF
diff --git a/drivers/media/platform/davinci/dm644x_ccdc.c b/drivers/media/platform/davinci/dm644x_ccdc.c
index 5fa0a1f32536..592d3fc91e26 100644
--- a/drivers/media/platform/davinci/dm644x_ccdc.c
+++ b/drivers/media/platform/davinci/dm644x_ccdc.c
@@ -22,9 +22,9 @@
  * may be supported using the same module.
  *
  * TODO: Test Raw bayer parameter settings and bayer capture
- * 	 Split module parameter structure to module specific ioctl structs
- * 	 investigate if enum used for user space type definition
- * 	 to be replaced by #defines or integer
+ *	 Split module parameter structure to module specific ioctl structs
+ *	 investigate if enum used for user space type definition
+ *	 to be replaced by #defines or integer
  */
 #include <linux/platform_device.h>
 #include <linux/uaccess.h>
diff --git a/drivers/media/platform/davinci/dm644x_ccdc_regs.h b/drivers/media/platform/davinci/dm644x_ccdc_regs.h
index bece0bd9c9de..ffd89c7ea2b6 100644
--- a/drivers/media/platform/davinci/dm644x_ccdc_regs.h
+++ b/drivers/media/platform/davinci/dm644x_ccdc_regs.h
@@ -97,7 +97,7 @@
 #define CCDC_LATCH_ON_VSYNC_DISABLE		(1 << 15)
 #define CCDC_FPC_ENABLE				(1 << 15)
 #define CCDC_FPC_DISABLE			0
-#define CCDC_FPC_FPC_NUM_MASK 			0x7FFF
+#define CCDC_FPC_FPC_NUM_MASK			0x7FFF
 #define CCDC_DATA_PACK_ENABLE			(1 << 11)
 #define CCDC_FMTCFG_VPIN_MASK			7
 #define CCDC_FMTCFG_VPIN_SHIFT			12
@@ -143,7 +143,7 @@
 #define CCDC_REC656IF_BT656_EN			3
 #define CCDC_SYN_MODE_VD_POL_NEGATIVE		(1 << 2)
 #define CCDC_CCDCFG_Y8POS_SHIFT			11
-#define CCDC_CCDCFG_BW656_10BIT 		(1 << 5)
+#define CCDC_CCDCFG_BW656_10BIT			(1 << 5)
 #define CCDC_SDOFST_FIELD_INTERLEAVED		0x249
 #define CCDC_NO_CULLING				0xffff00ff
 #endif
diff --git a/drivers/media/platform/davinci/isif_regs.h b/drivers/media/platform/davinci/isif_regs.h
index a3564abe08ae..97d3ba1614d6 100644
--- a/drivers/media/platform/davinci/isif_regs.h
+++ b/drivers/media/platform/davinci/isif_regs.h
@@ -35,7 +35,7 @@
 #define LINCFG0					0x44
 #define LINCFG1					0x48
 #define CCOLP					0x4c
-#define CRGAIN 					0x50
+#define CRGAIN					0x50
 #define CGRGAIN					0x54
 #define CGBGAIN					0x58
 #define CBGAIN					0x5c
@@ -46,7 +46,7 @@
 #define VDINT0					0x70
 #define VDINT1					0x74
 #define VDINT2					0x78
-#define MISC 					0x7c
+#define MISC					0x7c
 #define CGAMMAWD				0x80
 #define REC656IF				0x84
 #define CCDCFG					0x88
@@ -191,7 +191,7 @@
 #define ISIF_VD_POL_SHIFT			2
 #define ISIF_DATAPOL_NORMAL			0
 #define ISIF_DATAPOL_SHIFT			6
-#define ISIF_EXWEN_DISABLE 			0
+#define ISIF_EXWEN_DISABLE			0
 #define ISIF_EXWEN_SHIFT			5
 #define ISIF_FRM_FMT_SHIFT			7
 #define ISIF_DATASFT_SHIFT			8
diff --git a/drivers/media/platform/davinci/vpfe_capture.c b/drivers/media/platform/davinci/vpfe_capture.c
index 7b3f6f8e3dc8..6f44abf7fa31 100644
--- a/drivers/media/platform/davinci/vpfe_capture.c
+++ b/drivers/media/platform/davinci/vpfe_capture.c
@@ -26,8 +26,8 @@
  *
  *
  *    decoder(TVP5146/		YUV/
- * 	     MT9T001)   -->  Raw Bayer RGB ---> MUX -> VPFE (CCDC/ISIF)
- *    				data input              |      |
+ *	     MT9T001)   -->  Raw Bayer RGB ---> MUX -> VPFE (CCDC/ISIF)
+ *				data input              |      |
  *							V      |
  *						      SDRAM    |
  *							       V
@@ -47,7 +47,7 @@
  *    block such as IPIPE (on DM355 only).
  *
  *    Features supported
- *  		- MMAP IO
+ *		- MMAP IO
  *		- Capture using TVP5146 over BT.656
  *		- support for interfacing decoders using sub device model
  *		- Work with DM355 or DM6446 CCDC to do Raw Bayer RGB/YUV
@@ -730,7 +730,7 @@ static int vpfe_mmap(struct file *file, struct vm_area_struct *vma)
 /*
  * vpfe_poll: It is used for select/poll system call
  */
-static unsigned int vpfe_poll(struct file *file, poll_table *wait)
+static __poll_t vpfe_poll(struct file *file, poll_table *wait)
 {
 	struct vpfe_device *vpfe_dev = video_drvdata(file);
 
@@ -1794,7 +1794,7 @@ static int vpfe_probe(struct platform_device *pdev)
 	vfd->fops		= &vpfe_fops;
 	vfd->ioctl_ops		= &vpfe_ioctl_ops;
 	vfd->tvnorms		= 0;
-	vfd->v4l2_dev 		= &vpfe_dev->v4l2_dev;
+	vfd->v4l2_dev		= &vpfe_dev->v4l2_dev;
 	snprintf(vfd->name, sizeof(vfd->name),
 		 "%s_V%d.%d.%d",
 		 CAPTURE_DRV_NAME,
diff --git a/drivers/media/platform/davinci/vpif.h b/drivers/media/platform/davinci/vpif.h
index 9956e6788693..2466c7c77deb 100644
--- a/drivers/media/platform/davinci/vpif.h
+++ b/drivers/media/platform/davinci/vpif.h
@@ -226,11 +226,11 @@ static inline void vpif_clr_bit(u32 reg, u32 bit)
 	(VPIF_INT_BOTH << VPIF_CH1_INT_CTRL_SHIFT)), VPIF_CH1_CTRL))
 
 /* enabled interrupt on both the fields on vpid_ch0_ctrl register */
-#define channel2_intr_assert() 	(regw((regr(VPIF_CH2_CTRL)|\
+#define channel2_intr_assert()	(regw((regr(VPIF_CH2_CTRL)|\
 	(VPIF_INT_BOTH << VPIF_CH2_INT_CTRL_SHIFT)), VPIF_CH2_CTRL))
 
 /* enabled interrupt on both the fields on vpid_ch1_ctrl register */
-#define channel3_intr_assert() 	(regw((regr(VPIF_CH3_CTRL)|\
+#define channel3_intr_assert()	(regw((regr(VPIF_CH3_CTRL)|\
 	(VPIF_INT_BOTH << VPIF_CH3_INT_CTRL_SHIFT)), VPIF_CH3_CTRL))
 
 #define VPIF_CH_FID_MASK	(0x20)
diff --git a/drivers/media/platform/davinci/vpif_capture.c b/drivers/media/platform/davinci/vpif_capture.c
index fca4dc829f73..9364cdf62f54 100644
--- a/drivers/media/platform/davinci/vpif_capture.c
+++ b/drivers/media/platform/davinci/vpif_capture.c
@@ -1390,16 +1390,16 @@ static int vpif_async_bound(struct v4l2_async_notifier *notifier,
 
 	for (i = 0; i < vpif_obj.config->asd_sizes[0]; i++) {
 		struct v4l2_async_subdev *_asd = vpif_obj.config->asd[i];
-		const struct fwnode_handle *fwnode = _asd->match.fwnode.fwnode;
+		const struct fwnode_handle *fwnode = _asd->match.fwnode;
 
 		if (fwnode == subdev->fwnode) {
 			vpif_obj.sd[i] = subdev;
 			vpif_obj.config->chan_config->inputs[i].subdev_name =
 				(char *)to_of_node(subdev->fwnode)->full_name;
 			vpif_dbg(2, debug,
-				 "%s: setting input %d subdev_name = %pOF\n",
+				 "%s: setting input %d subdev_name = %s\n",
 				 __func__, i,
-				 to_of_node(subdev->fwnode));
+				vpif_obj.config->chan_config->inputs[i].subdev_name);
 			return 0;
 		}
 	}
@@ -1550,6 +1550,8 @@ vpif_capture_get_pdata(struct platform_device *pdev)
 					    sizeof(*chan->inputs) *
 					    VPIF_CAPTURE_NUM_CHANNELS,
 					    GFP_KERNEL);
+		if (!chan->inputs)
+			return NULL;
 
 		chan->input_count++;
 		chan->inputs[i].input.type = V4L2_INPUT_TYPE_CAMERA;
@@ -1593,7 +1595,7 @@ vpif_capture_get_pdata(struct platform_device *pdev)
 		}
 
 		pdata->asd[i]->match_type = V4L2_ASYNC_MATCH_FWNODE;
-		pdata->asd[i]->match.fwnode.fwnode = of_fwnode_handle(rem);
+		pdata->asd[i]->match.fwnode = of_fwnode_handle(rem);
 		of_node_put(rem);
 	}
 
diff --git a/drivers/media/platform/davinci/vpss.c b/drivers/media/platform/davinci/vpss.c
index f2d27b932999..b73886519f4f 100644
--- a/drivers/media/platform/davinci/vpss.c
+++ b/drivers/media/platform/davinci/vpss.c
@@ -59,9 +59,9 @@ MODULE_AUTHOR("Texas Instruments");
 #define DM365_ISP5_INTSEL1		0x10
 #define DM365_ISP5_INTSEL2		0x14
 #define DM365_ISP5_INTSEL3		0x18
-#define DM365_ISP5_CCDCMUX 		0x20
-#define DM365_ISP5_PG_FRAME_SIZE 	0x28
-#define DM365_VPBE_CLK_CTRL 		0x00
+#define DM365_ISP5_CCDCMUX		0x20
+#define DM365_ISP5_PG_FRAME_SIZE	0x28
+#define DM365_VPBE_CLK_CTRL		0x00
 
 #define VPSS_CLK_CTRL			0x01c40044
 #define VPSS_CLK_CTRL_VENCCLKEN		BIT(3)
@@ -78,8 +78,8 @@ MODULE_AUTHOR("Texas Instruments");
 #define DM365_ISP5_INTSEL3_DEFAULT	0x00000015
 
 /* masks and shifts for DM365*/
-#define DM365_CCDC_PG_VD_POL_SHIFT 	0
-#define DM365_CCDC_PG_HD_POL_SHIFT 	1
+#define DM365_CCDC_PG_VD_POL_SHIFT	0
+#define DM365_CCDC_PG_HD_POL_SHIFT	1
 
 #define CCD_SRC_SEL_MASK		(BIT_MASK(5) | BIT_MASK(4))
 #define CCD_SRC_SEL_SHIFT		4
diff --git a/drivers/media/platform/exynos-gsc/gsc-m2m.c b/drivers/media/platform/exynos-gsc/gsc-m2m.c
index 2a2994ef15d5..e9ff27949a91 100644
--- a/drivers/media/platform/exynos-gsc/gsc-m2m.c
+++ b/drivers/media/platform/exynos-gsc/gsc-m2m.c
@@ -298,9 +298,7 @@ static int gsc_m2m_querycap(struct file *file, void *fh,
 	strlcpy(cap->card, GSC_MODULE_NAME " gscaler", sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
 		 dev_name(&gsc->pdev->dev));
-	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE |
-		V4L2_CAP_VIDEO_CAPTURE_MPLANE |	V4L2_CAP_VIDEO_OUTPUT_MPLANE;
-
+	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE;
 	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
 	return 0;
 }
@@ -707,12 +705,12 @@ static int gsc_m2m_release(struct file *file)
 	return 0;
 }
 
-static unsigned int gsc_m2m_poll(struct file *file,
+static __poll_t gsc_m2m_poll(struct file *file,
 					struct poll_table_struct *wait)
 {
 	struct gsc_ctx *ctx = fh_to_ctx(file->private_data);
 	struct gsc_dev *gsc = ctx->gsc_dev;
-	unsigned int ret;
+	__poll_t ret;
 
 	if (mutex_lock_interruptible(&gsc->lock))
 		return -ERESTARTSYS;
diff --git a/drivers/media/platform/exynos4-is/Kconfig b/drivers/media/platform/exynos4-is/Kconfig
index 46a7d242a1a5..7b2c49e5a592 100644
--- a/drivers/media/platform/exynos4-is/Kconfig
+++ b/drivers/media/platform/exynos4-is/Kconfig
@@ -41,7 +41,7 @@ config VIDEO_S5P_MIPI_CSIS
 	  To compile this driver as a module, choose M here: the
 	  module will be called s5p-csis.
 
-if SOC_EXYNOS4212 || SOC_EXYNOS4412 || SOC_EXYNOS5250
+if SOC_EXYNOS4412 || SOC_EXYNOS5250
 
 config VIDEO_EXYNOS_FIMC_LITE
 	tristate "EXYNOS FIMC-LITE camera interface driver"
diff --git a/drivers/media/platform/exynos4-is/fimc-core.c b/drivers/media/platform/exynos4-is/fimc-core.c
index 099c735a39b7..d8d8c9902b19 100644
--- a/drivers/media/platform/exynos4-is/fimc-core.c
+++ b/drivers/media/platform/exynos4-is/fimc-core.c
@@ -1211,7 +1211,7 @@ static const struct fimc_drvdata fimc_drvdata_exynos4210 = {
 	.out_buf_count	= 32,
 };
 
-/* EXYNOS4212, EXYNOS4412 */
+/* EXYNOS4412 */
 static const struct fimc_drvdata fimc_drvdata_exynos4x12 = {
 	.num_entities	= 4,
 	.lclk_frequency	= 166000000UL,
@@ -1246,7 +1246,7 @@ static struct platform_driver fimc_driver = {
 	.driver = {
 		.of_match_table = fimc_of_match,
 		.name		= FIMC_DRIVER_NAME,
-		.pm     	= &fimc_pm_ops,
+		.pm		= &fimc_pm_ops,
 	}
 };
 
diff --git a/drivers/media/platform/exynos4-is/fimc-core.h b/drivers/media/platform/exynos4-is/fimc-core.h
index c0373aede81a..82d514df97f0 100644
--- a/drivers/media/platform/exynos4-is/fimc-core.h
+++ b/drivers/media/platform/exynos4-is/fimc-core.h
@@ -303,7 +303,7 @@ struct fimc_m2m_device {
  * @input: capture input type, grp_id of the attached subdev
  * @user_subdev_api: true if subdevs are not configured by the host driver
  * @inh_sensor_ctrls: a flag indicating v4l2 controls are inherited from
- * 		      an image sensor subdev
+ *		      an image sensor subdev
  */
 struct fimc_vid_cap {
 	struct fimc_ctx			*ctx;
diff --git a/drivers/media/platform/exynos4-is/fimc-isp.c b/drivers/media/platform/exynos4-is/fimc-isp.c
index fd793d3ac072..9a48c0f69320 100644
--- a/drivers/media/platform/exynos4-is/fimc-isp.c
+++ b/drivers/media/platform/exynos4-is/fimc-isp.c
@@ -366,16 +366,16 @@ static int fimc_isp_subdev_s_power(struct v4l2_subdev *sd, int on)
 static int fimc_isp_subdev_open(struct v4l2_subdev *sd,
 				struct v4l2_subdev_fh *fh)
 {
-	struct v4l2_mbus_framefmt fmt;
 	struct v4l2_mbus_framefmt *format;
+	struct v4l2_mbus_framefmt fmt = {
+		.colorspace = V4L2_COLORSPACE_SRGB,
+		.code = fimc_isp_formats[0].mbus_code,
+		.width = DEFAULT_PREVIEW_STILL_WIDTH + FIMC_ISP_CAC_MARGIN_WIDTH,
+		.height = DEFAULT_PREVIEW_STILL_HEIGHT + FIMC_ISP_CAC_MARGIN_HEIGHT,
+		.field = V4L2_FIELD_NONE,
+	};
 
 	format = v4l2_subdev_get_try_format(sd, fh->pad, FIMC_ISP_SD_PAD_SINK);
-
-	fmt.colorspace = V4L2_COLORSPACE_SRGB;
-	fmt.code = fimc_isp_formats[0].mbus_code;
-	fmt.width = DEFAULT_PREVIEW_STILL_WIDTH + FIMC_ISP_CAC_MARGIN_WIDTH;
-	fmt.height = DEFAULT_PREVIEW_STILL_HEIGHT + FIMC_ISP_CAC_MARGIN_HEIGHT;
-	fmt.field = V4L2_FIELD_NONE;
 	*format = fmt;
 
 	format = v4l2_subdev_get_try_format(sd, fh->pad, FIMC_ISP_SD_PAD_SRC_FIFO);
diff --git a/drivers/media/platform/exynos4-is/fimc-lite.c b/drivers/media/platform/exynos4-is/fimc-lite.c
index 4a3c9948ca54..70d5f5586a5d 100644
--- a/drivers/media/platform/exynos4-is/fimc-lite.c
+++ b/drivers/media/platform/exynos4-is/fimc-lite.c
@@ -1462,10 +1462,7 @@ static void fimc_lite_clk_put(struct fimc_lite *fimc)
 static int fimc_lite_clk_get(struct fimc_lite *fimc)
 {
 	fimc->clock = clk_get(&fimc->pdev->dev, FLITE_CLK_NAME);
-	if (IS_ERR(fimc->clock))
-		return PTR_ERR(fimc->clock);
-
-	return 0;
+	return PTR_ERR_OR_ZERO(fimc->clock);
 }
 
 static const struct of_device_id flite_of_match[];
@@ -1646,7 +1643,7 @@ static const struct dev_pm_ops fimc_lite_pm_ops = {
 			   NULL)
 };
 
-/* EXYNOS4212, EXYNOS4412 */
+/* EXYNOS4412 */
 static struct flite_drvdata fimc_lite_drvdata_exynos4 = {
 	.max_width		= 8192,
 	.max_height		= 8192,
diff --git a/drivers/media/platform/exynos4-is/fimc-lite.h b/drivers/media/platform/exynos4-is/fimc-lite.h
index 9ae1e96a1bc7..3e238b8c834a 100644
--- a/drivers/media/platform/exynos4-is/fimc-lite.h
+++ b/drivers/media/platform/exynos4-is/fimc-lite.h
@@ -56,9 +56,9 @@ enum {
  * @max_height: maximum camera interface input height in pixels
  * @out_width_align: minimum output width alignment in pixels
  * @win_hor_offs_align: minimum camera interface crop window horizontal
- * 			offset alignment in pixels
+ *			offset alignment in pixels
  * @out_hor_offs_align: minimum output DMA compose rectangle horizontal
- * 			offset alignment in pixels
+ *			offset alignment in pixels
  * @max_dma_bufs: number of output DMA buffer start address registers
  * @num_instances: total number of FIMC-LITE IP instances available
  */
diff --git a/drivers/media/platform/exynos4-is/fimc-m2m.c b/drivers/media/platform/exynos4-is/fimc-m2m.c
index 9027d0b0d2bd..a19f8b164a47 100644
--- a/drivers/media/platform/exynos4-is/fimc-m2m.c
+++ b/drivers/media/platform/exynos4-is/fimc-m2m.c
@@ -236,15 +236,7 @@ static int fimc_m2m_querycap(struct file *file, void *fh,
 				     struct v4l2_capability *cap)
 {
 	struct fimc_dev *fimc = video_drvdata(file);
-	unsigned int caps;
-
-	/*
-	 * This is only a mem-to-mem video device. The capture and output
-	 * device capability flags are left only for backward compatibility
-	 * and are scheduled for removal.
-	 */
-	caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE |
-		V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_VIDEO_OUTPUT_MPLANE;
+	unsigned int caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_M2M_MPLANE;
 
 	__fimc_vidioc_querycap(&fimc->pdev->dev, cap, caps);
 	return 0;
diff --git a/drivers/media/platform/exynos4-is/media-dev.c b/drivers/media/platform/exynos4-is/media-dev.c
index 0ef583cfc424..78b48a1fa26c 100644
--- a/drivers/media/platform/exynos4-is/media-dev.c
+++ b/drivers/media/platform/exynos4-is/media-dev.c
@@ -456,7 +456,7 @@ static int fimc_md_parse_port_node(struct fimc_md *fmd,
 	}
 
 	fmd->sensor[index].asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
-	fmd->sensor[index].asd.match.fwnode.fwnode = of_fwnode_handle(rem);
+	fmd->sensor[index].asd.match.fwnode = of_fwnode_handle(rem);
 	fmd->async_subdevs[index] = &fmd->sensor[index].asd;
 
 	fmd->num_sensors++;
@@ -1364,7 +1364,7 @@ static int subdev_notifier_bound(struct v4l2_async_notifier *notifier,
 
 	/* Find platform data for this sensor subdev */
 	for (i = 0; i < ARRAY_SIZE(fmd->sensor); i++)
-		if (fmd->sensor[i].asd.match.fwnode.fwnode ==
+		if (fmd->sensor[i].asd.match.fwnode ==
 		    of_fwnode_handle(subdev->dev->of_node))
 			si = &fmd->sensor[i];
 
diff --git a/drivers/media/platform/fsl-viu.c b/drivers/media/platform/fsl-viu.c
index dba21215dc84..200c47c69a75 100644
--- a/drivers/media/platform/fsl-viu.c
+++ b/drivers/media/platform/fsl-viu.c
@@ -1263,18 +1263,18 @@ static ssize_t viu_read(struct file *file, char __user *data, size_t count,
 	return 0;
 }
 
-static unsigned int viu_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t viu_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct viu_fh *fh = file->private_data;
 	struct videobuf_queue *q = &fh->vb_vidq;
 	struct viu_dev *dev = fh->dev;
-	unsigned long req_events = poll_requested_events(wait);
-	unsigned int res = v4l2_ctrl_poll(file, wait);
+	__poll_t req_events = poll_requested_events(wait);
+	__poll_t res = v4l2_ctrl_poll(file, wait);
 
 	if (V4L2_BUF_TYPE_VIDEO_CAPTURE != fh->type)
-		return POLLERR;
+		return EPOLLERR;
 
-	if (!(req_events & (POLLIN | POLLRDNORM)))
+	if (!(req_events & (EPOLLIN | EPOLLRDNORM)))
 		return res;
 
 	mutex_lock(&dev->lock);
diff --git a/drivers/media/platform/m2m-deinterlace.c b/drivers/media/platform/m2m-deinterlace.c
index c8a12493f395..1e4195144f39 100644
--- a/drivers/media/platform/m2m-deinterlace.c
+++ b/drivers/media/platform/m2m-deinterlace.c
@@ -384,16 +384,16 @@ static void deinterlace_device_run(void *priv)
 	 * 4 possible field conversions are possible at the moment:
 	 *  V4L2_FIELD_SEQ_TB --> V4L2_FIELD_INTERLACED_TB:
 	 *	two separate fields in the same input buffer are interlaced
-	 * 	in the output buffer using weaving. Top field comes first.
+	 *	in the output buffer using weaving. Top field comes first.
 	 *  V4L2_FIELD_SEQ_TB --> V4L2_FIELD_NONE:
-	 * 	top field from the input buffer is copied to the output buffer
-	 * 	using line doubling. Bottom field from the input buffer is discarded.
+	 *	top field from the input buffer is copied to the output buffer
+	 *	using line doubling. Bottom field from the input buffer is discarded.
 	 * V4L2_FIELD_SEQ_BT --> V4L2_FIELD_INTERLACED_BT:
 	 *	two separate fields in the same input buffer are interlaced
-	 * 	in the output buffer using weaving. Bottom field comes first.
+	 *	in the output buffer using weaving. Bottom field comes first.
 	 * V4L2_FIELD_SEQ_BT --> V4L2_FIELD_NONE:
-	 * 	bottom field from the input buffer is copied to the output buffer
-	 * 	using line doubling. Top field from the input buffer is discarded.
+	 *	bottom field from the input buffer is copied to the output buffer
+	 *	using line doubling. Top field from the input buffer is discarded.
 	 */
 	switch (dst_q_data->fmt->fourcc) {
 	case V4L2_PIX_FMT_YUV420:
@@ -950,11 +950,11 @@ static int deinterlace_release(struct file *file)
 	return 0;
 }
 
-static unsigned int deinterlace_poll(struct file *file,
+static __poll_t deinterlace_poll(struct file *file,
 				 struct poll_table_struct *wait)
 {
 	struct deinterlace_ctx *ctx = file->private_data;
-	int ret;
+	__poll_t ret;
 
 	deinterlace_lock(ctx);
 	ret = v4l2_m2m_poll(file, ctx->m2m_ctx, wait);
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c
index 46768c056193..0c28d0b995cc 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_util.c
@@ -115,3 +115,6 @@ struct mtk_vcodec_ctx *mtk_vcodec_get_curr_ctx(struct mtk_vcodec_dev *dev)
 	return ctx;
 }
 EXPORT_SYMBOL(mtk_vcodec_get_curr_ctx);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Mediatek video codec driver");
diff --git a/drivers/media/platform/mtk-vcodec/vdec_vpu_if.h b/drivers/media/platform/mtk-vcodec/vdec_vpu_if.h
index 0dc9ed01fffe..cd37bb2a610f 100644
--- a/drivers/media/platform/mtk-vcodec/vdec_vpu_if.h
+++ b/drivers/media/platform/mtk-vcodec/vdec_vpu_if.h
@@ -26,7 +26,7 @@
  * @inst_addr	: VPU decoder instance address
  * @signaled    : 1 - Host has received ack message from VPU, 0 - not received
  * @ctx         : context for v4l2 layer integration
- * @dev	        : platform device of VPU
+ * @dev		: platform device of VPU
  * @wq          : wait queue to wait VPU message ack
  * @handler     : ipi handler for each decoder
  */
diff --git a/drivers/media/platform/mx2_emmaprp.c b/drivers/media/platform/mx2_emmaprp.c
index 4a2b1afa19c4..5a8eff60e95f 100644
--- a/drivers/media/platform/mx2_emmaprp.c
+++ b/drivers/media/platform/mx2_emmaprp.c
@@ -838,12 +838,12 @@ static int emmaprp_release(struct file *file)
 	return 0;
 }
 
-static unsigned int emmaprp_poll(struct file *file,
+static __poll_t emmaprp_poll(struct file *file,
 				 struct poll_table_struct *wait)
 {
 	struct emmaprp_dev *pcdev = video_drvdata(file);
 	struct emmaprp_ctx *ctx = file->private_data;
-	unsigned int res;
+	__poll_t res;
 
 	mutex_lock(&pcdev->dev_mutex);
 	res = v4l2_m2m_poll(file, ctx->m2m_ctx, wait);
diff --git a/drivers/media/platform/omap/omap_vout.c b/drivers/media/platform/omap/omap_vout.c
index 6f1b0c799e58..a795a9fae899 100644
--- a/drivers/media/platform/omap/omap_vout.c
+++ b/drivers/media/platform/omap/omap_vout.c
@@ -839,7 +839,7 @@ static void omap_vout_buffer_release(struct videobuf_queue *q,
 /*
  *  File operations
  */
-static unsigned int omap_vout_poll(struct file *file,
+static __poll_t omap_vout_poll(struct file *file,
 				   struct poll_table_struct *wait)
 {
 	struct omap_vout_device *vout = file->private_data;
@@ -1774,8 +1774,8 @@ static int vidioc_g_fbuf(struct file *file, void *fh,
 }
 
 static const struct v4l2_ioctl_ops vout_ioctl_ops = {
-	.vidioc_querycap      			= vidioc_querycap,
-	.vidioc_enum_fmt_vid_out 		= vidioc_enum_fmt_vid_out,
+	.vidioc_querycap			= vidioc_querycap,
+	.vidioc_enum_fmt_vid_out		= vidioc_enum_fmt_vid_out,
 	.vidioc_g_fmt_vid_out			= vidioc_g_fmt_vid_out,
 	.vidioc_try_fmt_vid_out			= vidioc_try_fmt_vid_out,
 	.vidioc_s_fmt_vid_out			= vidioc_s_fmt_vid_out,
@@ -1795,12 +1795,12 @@ static const struct v4l2_ioctl_ops vout_ioctl_ops = {
 };
 
 static const struct v4l2_file_operations omap_vout_fops = {
-	.owner 		= THIS_MODULE,
+	.owner		= THIS_MODULE,
 	.poll		= omap_vout_poll,
 	.unlocked_ioctl	= video_ioctl2,
-	.mmap 		= omap_vout_mmap,
-	.open 		= omap_vout_open,
-	.release 	= omap_vout_release,
+	.mmap		= omap_vout_mmap,
+	.open		= omap_vout_open,
+	.release	= omap_vout_release,
 };
 
 /* Init functions used during driver initialization */
diff --git a/drivers/media/platform/omap3isp/isp.c b/drivers/media/platform/omap3isp/isp.c
index b7ff3842afc0..8eb000e3d8fd 100644
--- a/drivers/media/platform/omap3isp/isp.c
+++ b/drivers/media/platform/omap3isp/isp.c
@@ -920,7 +920,7 @@ static void isp_pipeline_suspend(struct isp_pipeline *pipe)
 
 /*
  * isp_pipeline_is_last - Verify if entity has an enabled link to the output
- * 			  video node
+ *			  video node
  * @me: ISP module's media entity
  *
  * Returns 1 if the entity has an enabled link to the output video node or 0
diff --git a/drivers/media/platform/omap3isp/ispvideo.c b/drivers/media/platform/omap3isp/ispvideo.c
index 218e6d7ae93a..a751c89a3ea8 100644
--- a/drivers/media/platform/omap3isp/ispvideo.c
+++ b/drivers/media/platform/omap3isp/ispvideo.c
@@ -1383,11 +1383,11 @@ static int isp_video_release(struct file *file)
 	return 0;
 }
 
-static unsigned int isp_video_poll(struct file *file, poll_table *wait)
+static __poll_t isp_video_poll(struct file *file, poll_table *wait)
 {
 	struct isp_video_fh *vfh = to_isp_video_fh(file->private_data);
 	struct isp_video *video = video_drvdata(file);
-	int ret;
+	__poll_t ret;
 
 	mutex_lock(&video->queue_lock);
 	ret = vb2_poll(&vfh->queue, file, wait);
diff --git a/drivers/media/platform/pxa_camera.c b/drivers/media/platform/pxa_camera.c
index 295f34ad1080..c71a00736541 100644
--- a/drivers/media/platform/pxa_camera.c
+++ b/drivers/media/platform/pxa_camera.c
@@ -32,7 +32,6 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/dmaengine.h>
-#include <linux/dma-mapping.h>
 #include <linux/dma/pxa-dma.h>
 
 #include <media/v4l2-async.h>
@@ -647,16 +646,16 @@ static unsigned int pxa_mbus_config_compatible(const struct v4l2_mbus_config *cf
 }
 
 /**
- * struct soc_camera_format_xlate - match between host and sensor formats
+ * struct pxa_camera_format_xlate - match between host and sensor formats
  * @code: code of a sensor provided format
  * @host_fmt: host format after host translation from code
  *
  * Host and sensor translation structure. Used in table of host and sensor
- * formats matchings in soc_camera_device. A host can override the generic list
+ * formats matchings in pxa_camera_device. A host can override the generic list
  * generation by implementing get_formats(), and use it for format checks and
  * format setup.
  */
-struct soc_camera_format_xlate {
+struct pxa_camera_format_xlate {
 	u32 code;
 	const struct pxa_mbus_pixelfmt *host_fmt;
 };
@@ -693,8 +692,8 @@ struct pxa_camera_dev {
 	struct v4l2_async_notifier notifier;
 	struct vb2_queue	vb2_vq;
 	struct v4l2_subdev	*sensor;
-	struct soc_camera_format_xlate *user_formats;
-	const struct soc_camera_format_xlate *current_fmt;
+	struct pxa_camera_format_xlate *user_formats;
+	const struct pxa_camera_format_xlate *current_fmt;
 	struct v4l2_pix_format	current_pix;
 
 	struct v4l2_async_subdev asd;
@@ -743,8 +742,8 @@ static const char *pxa_cam_driver_description = "PXA_Camera";
 /*
  * Format translation functions
  */
-static const struct soc_camera_format_xlate
-*pxa_mbus_xlate_by_fourcc(struct soc_camera_format_xlate *user_formats,
+static const struct pxa_camera_format_xlate
+*pxa_mbus_xlate_by_fourcc(struct pxa_camera_format_xlate *user_formats,
 			  unsigned int fourcc)
 {
 	unsigned int i;
@@ -755,17 +754,17 @@ static const struct soc_camera_format_xlate
 	return NULL;
 }
 
-static struct soc_camera_format_xlate *pxa_mbus_build_fmts_xlate(
+static struct pxa_camera_format_xlate *pxa_mbus_build_fmts_xlate(
 	struct v4l2_device *v4l2_dev, struct v4l2_subdev *subdev,
 	int (*get_formats)(struct v4l2_device *, unsigned int,
-			   struct soc_camera_format_xlate *xlate))
+			   struct pxa_camera_format_xlate *xlate))
 {
 	unsigned int i, fmts = 0, raw_fmts = 0;
 	int ret;
 	struct v4l2_subdev_mbus_code_enum code = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
-	struct soc_camera_format_xlate *user_formats;
+	struct pxa_camera_format_xlate *user_formats;
 
 	while (!v4l2_subdev_call(subdev, pad, enum_mbus_code, NULL, &code)) {
 		raw_fmts++;
@@ -1722,7 +1721,7 @@ static bool pxa_camera_packing_supported(const struct pxa_mbus_pixelfmt *fmt)
 
 static int pxa_camera_get_formats(struct v4l2_device *v4l2_dev,
 				  unsigned int idx,
-				  struct soc_camera_format_xlate *xlate)
+				  struct pxa_camera_format_xlate *xlate)
 {
 	struct pxa_camera_dev *pcdev = v4l2_dev_to_pcdev(v4l2_dev);
 	int formats = 0, ret;
@@ -1794,7 +1793,7 @@ static int pxa_camera_get_formats(struct v4l2_device *v4l2_dev,
 
 static int pxa_camera_build_formats(struct pxa_camera_dev *pcdev)
 {
-	struct soc_camera_format_xlate *xlate;
+	struct pxa_camera_format_xlate *xlate;
 
 	xlate = pxa_mbus_build_fmts_xlate(&pcdev->v4l2_dev, pcdev->sensor,
 					  pxa_camera_get_formats);
@@ -1883,7 +1882,7 @@ static int pxac_vidioc_try_fmt_vid_cap(struct file *filp, void *priv,
 				      struct v4l2_format *f)
 {
 	struct pxa_camera_dev *pcdev = video_drvdata(filp);
-	const struct soc_camera_format_xlate *xlate;
+	const struct pxa_camera_format_xlate *xlate;
 	struct v4l2_pix_format *pix = &f->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_format format = {
@@ -1947,7 +1946,7 @@ static int pxac_vidioc_s_fmt_vid_cap(struct file *filp, void *priv,
 				    struct v4l2_format *f)
 {
 	struct pxa_camera_dev *pcdev = video_drvdata(filp);
-	const struct soc_camera_format_xlate *xlate;
+	const struct pxa_camera_format_xlate *xlate;
 	struct v4l2_pix_format *pix = &f->fmt.pix;
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
@@ -2336,7 +2335,7 @@ static int pxa_camera_pdata_from_dt(struct device *dev,
 	asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
 	remote = of_graph_get_remote_port(np);
 	if (remote) {
-		asd->match.fwnode.fwnode = of_fwnode_handle(remote);
+		asd->match.fwnode = of_fwnode_handle(remote);
 		of_node_put(remote);
 	} else {
 		dev_notice(dev, "no remote for %pOF\n", np);
@@ -2492,7 +2491,7 @@ static int pxa_camera_probe(struct platform_device *pdev)
 	dev_set_drvdata(&pdev->dev, pcdev);
 	err = v4l2_device_register(&pdev->dev, &pcdev->v4l2_dev);
 	if (err)
-		goto exit_free_dma;
+		goto exit_deactivate;
 
 	pcdev->asds[0] = &pcdev->asd;
 	pcdev->notifier.subdevs = pcdev->asds;
@@ -2528,6 +2527,8 @@ static int pxa_camera_probe(struct platform_device *pdev)
 	v4l2_clk_unregister(pcdev->mclk_clk);
 exit_free_v4l2dev:
 	v4l2_device_unregister(&pcdev->v4l2_dev);
+exit_deactivate:
+	pxa_camera_deactivate(pcdev);
 exit_free_dma:
 	dma_release_channel(pcdev->dma_chans[2]);
 exit_free_dma_u:
diff --git a/drivers/media/platform/qcom/camss-8x16/camss.c b/drivers/media/platform/qcom/camss-8x16/camss.c
index 390a42c17b66..05f06c98aa64 100644
--- a/drivers/media/platform/qcom/camss-8x16/camss.c
+++ b/drivers/media/platform/qcom/camss-8x16/camss.c
@@ -341,7 +341,7 @@ static int camss_of_parse_ports(struct device *dev,
 		}
 
 		csd->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
-		csd->asd.match.fwnode.fwnode = of_fwnode_handle(remote);
+		csd->asd.match.fwnode = of_fwnode_handle(remote);
 	}
 
 	return notifier->num_subdevs;
diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h
index a0fe80df0cbd..0360d295f4c8 100644
--- a/drivers/media/platform/qcom/venus/core.h
+++ b/drivers/media/platform/qcom/venus/core.h
@@ -144,8 +144,8 @@ struct venc_controls {
 	u32 h264_min_qp;
 	u32 h264_max_qp;
 	u32 h264_loop_filter_mode;
-	u32 h264_loop_filter_alpha;
-	u32 h264_loop_filter_beta;
+	s32 h264_loop_filter_alpha;
+	s32 h264_loop_filter_beta;
 
 	u32 vp8_min_qp;
 	u32 vp8_max_qp;
diff --git a/drivers/media/platform/qcom/venus/hfi_cmds.c b/drivers/media/platform/qcom/venus/hfi_cmds.c
index b83c5b8ddccb..1cfeb7743041 100644
--- a/drivers/media/platform/qcom/venus/hfi_cmds.c
+++ b/drivers/media/platform/qcom/venus/hfi_cmds.c
@@ -421,12 +421,12 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	pkt->shdr.hdr.pkt_type = HFI_CMD_SESSION_SET_PROPERTY;
 	pkt->shdr.session_id = hash32_ptr(cookie);
 	pkt->num_properties = 1;
+	pkt->data[0] = ptype;
 
 	switch (ptype) {
 	case HFI_PROPERTY_CONFIG_FRAME_RATE: {
 		struct hfi_framerate *in = pdata, *frate = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_FRAME_RATE;
 		frate->buffer_type = in->buffer_type;
 		frate->framerate = in->framerate;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*frate);
@@ -436,7 +436,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_uncompressed_format_select *in = pdata;
 		struct hfi_uncompressed_format_select *hfi = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_UNCOMPRESSED_FORMAT_SELECT;
 		hfi->buffer_type = in->buffer_type;
 		hfi->format = in->format;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*hfi);
@@ -445,7 +444,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_FRAME_SIZE: {
 		struct hfi_framesize *in = pdata, *fsize = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_FRAME_SIZE;
 		fsize->buffer_type = in->buffer_type;
 		fsize->height = in->height;
 		fsize->width = in->width;
@@ -455,7 +453,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_REALTIME: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_REALTIME;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -463,7 +460,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_BUFFER_COUNT_ACTUAL: {
 		struct hfi_buffer_count_actual *in = pdata, *count = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_BUFFER_COUNT_ACTUAL;
 		count->count_actual = in->count_actual;
 		count->type = in->type;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*count);
@@ -472,7 +468,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_BUFFER_SIZE_ACTUAL: {
 		struct hfi_buffer_size_actual *in = pdata, *sz = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_BUFFER_SIZE_ACTUAL;
 		sz->size = in->size;
 		sz->type = in->type;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*sz);
@@ -482,8 +477,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_buffer_display_hold_count_actual *in = pdata;
 		struct hfi_buffer_display_hold_count_actual *count = prop_data;
 
-		pkt->data[0] =
-			HFI_PROPERTY_PARAM_BUFFER_DISPLAY_HOLD_COUNT_ACTUAL;
 		count->hold_count = in->hold_count;
 		count->type = in->type;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*count);
@@ -493,7 +486,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_nal_stream_format_select *in = pdata;
 		struct hfi_nal_stream_format_select *fmt = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_NAL_STREAM_FORMAT_SELECT;
 		fmt->format = in->format;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*fmt);
 		break;
@@ -510,7 +502,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_OUTPUT_ORDER;
 		pkt->data[1] = *in;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -518,7 +509,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VDEC_PICTURE_TYPE_DECODE: {
 		struct hfi_enable_picture *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_PICTURE_TYPE_DECODE;
 		en->picture_type = in->picture_type;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -526,8 +516,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VDEC_OUTPUT2_KEEP_ASPECT_RATIO: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] =
-			HFI_PROPERTY_PARAM_VDEC_OUTPUT2_KEEP_ASPECT_RATIO;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -536,7 +524,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_enable *in = pdata;
 		struct hfi_enable *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VDEC_POST_LOOP_DEBLOCKER;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -544,7 +531,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VDEC_MULTI_STREAM: {
 		struct hfi_multi_stream *in = pdata, *multi = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_MULTI_STREAM;
 		multi->buffer_type = in->buffer_type;
 		multi->enable = in->enable;
 		multi->width = in->width;
@@ -556,8 +542,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_display_picture_buffer_count *in = pdata;
 		struct hfi_display_picture_buffer_count *count = prop_data;
 
-		pkt->data[0] =
-			HFI_PROPERTY_PARAM_VDEC_DISPLAY_PICTURE_BUFFER_COUNT;
 		count->count = in->count;
 		count->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*count);
@@ -576,7 +560,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_DIVX_FORMAT;
 		pkt->data[1] = *in;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -584,7 +567,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VDEC_MB_ERROR_MAP_REPORTING: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VDEC_MB_ERROR_MAP_REPORTING;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -592,7 +574,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VDEC_CONTINUE_DATA_TRANSFER: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_CONTINUE_DATA_TRANSFER;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -600,7 +581,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VDEC_THUMBNAIL_MODE: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_THUMBNAIL_MODE;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -608,14 +588,11 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_SYNC_FRAME_SEQUENCE_HEADER: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] =
-			HFI_PROPERTY_CONFIG_VENC_SYNC_FRAME_SEQUENCE_HEADER;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
 	}
 	case HFI_PROPERTY_CONFIG_VENC_REQUEST_SYNC_FRAME:
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_REQUEST_SYNC_FRAME;
 		pkt->shdr.hdr.size += sizeof(u32);
 		break;
 	case HFI_PROPERTY_PARAM_VENC_MPEG4_SHORT_HEADER:
@@ -625,7 +602,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_TARGET_BITRATE: {
 		struct hfi_bitrate *in = pdata, *brate = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_TARGET_BITRATE;
 		brate->bitrate = in->bitrate;
 		brate->layer_id = in->layer_id;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*brate);
@@ -634,7 +610,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_MAX_BITRATE: {
 		struct hfi_bitrate *in = pdata, *hfi = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_MAX_BITRATE;
 		hfi->bitrate = in->bitrate;
 		hfi->layer_id = in->layer_id;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*hfi);
@@ -643,7 +618,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT: {
 		struct hfi_profile_level *in = pdata, *pl = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT;
 		pl->level = in->level;
 		pl->profile = in->profile;
 		if (pl->profile <= 0)
@@ -660,7 +634,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_H264_ENTROPY_CONTROL: {
 		struct hfi_h264_entropy_control *in = pdata, *hfi = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_H264_ENTROPY_CONTROL;
 		hfi->entropy_mode = in->entropy_mode;
 		if (hfi->entropy_mode == HFI_H264_ENTROPY_CABAC)
 			hfi->cabac_model = in->cabac_model;
@@ -682,7 +655,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_RATE_CONTROL;
 		pkt->data[1] = *in;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -690,7 +662,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_MPEG4_TIME_RESOLUTION: {
 		struct hfi_mpeg4_time_resolution *in = pdata, *res = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_MPEG4_TIME_RESOLUTION;
 		res->time_increment_resolution = in->time_increment_resolution;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*res);
 		break;
@@ -698,7 +669,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_MPEG4_HEADER_EXTENSION: {
 		struct hfi_mpeg4_header_extension *in = pdata, *ext = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_MPEG4_HEADER_EXTENSION;
 		ext->header_extension = in->header_extension;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*ext);
 		break;
@@ -716,7 +686,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_H264_DEBLOCK_CONTROL;
 		db->mode = in->mode;
 		db->slice_alpha_offset = in->slice_alpha_offset;
 		db->slice_beta_offset = in->slice_beta_offset;
@@ -726,7 +695,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_SESSION_QP: {
 		struct hfi_quantization *in = pdata, *quant = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_SESSION_QP;
 		quant->qp_i = in->qp_i;
 		quant->qp_p = in->qp_p;
 		quant->qp_b = in->qp_b;
@@ -738,7 +706,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_quantization_range *in = pdata, *range = prop_data;
 		u32 min_qp, max_qp;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE;
 		min_qp = in->min_qp;
 		max_qp = in->max_qp;
 
@@ -764,8 +731,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_VC1_PERF_CFG: {
 		struct hfi_vc1e_perf_cfg_type *in = pdata, *perf = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_VC1_PERF_CFG;
-
 		memcpy(perf->search_range_x_subsampled,
 		       in->search_range_x_subsampled,
 		       sizeof(perf->search_range_x_subsampled));
@@ -780,7 +745,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_max_num_b_frames *bframes = prop_data;
 		u32 *in = pdata;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_MAX_NUM_B_FRAMES;
 		bframes->max_num_b_frames = *in;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*bframes);
 		break;
@@ -788,7 +752,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_INTRA_PERIOD: {
 		struct hfi_intra_period *in = pdata, *intra = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_INTRA_PERIOD;
 		intra->pframes = in->pframes;
 		intra->bframes = in->bframes;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*intra);
@@ -797,7 +760,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_IDR_PERIOD: {
 		struct hfi_idr_period *in = pdata, *idr = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_IDR_PERIOD;
 		idr->idr_period = in->idr_period;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*idr);
 		break;
@@ -806,7 +768,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_conceal_color *color = prop_data;
 		u32 *in = pdata;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_CONCEAL_COLOR;
 		color->conceal_color = *in;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*color);
 		break;
@@ -835,7 +796,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VPE_OPERATIONS;
 		ops->rotation = in->rotation;
 		ops->flip = in->flip;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*ops);
@@ -856,7 +816,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_INTRA_REFRESH;
 		intra->mode = in->mode;
 		intra->air_mbs = in->air_mbs;
 		intra->air_ref = in->air_ref;
@@ -878,7 +837,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_MULTI_SLICE_CONTROL;
 		multi->multi_slice = in->multi_slice;
 		multi->slice_size = in->slice_size;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*multi);
@@ -887,7 +845,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_SLICE_DELIVERY_MODE: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_SLICE_DELIVERY_MODE;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -895,7 +852,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_H264_VUI_TIMING_INFO: {
 		struct hfi_h264_vui_timing_info *in = pdata, *vui = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_H264_VUI_TIMING_INFO;
 		vui->enable = in->enable;
 		vui->fixed_framerate = in->fixed_framerate;
 		vui->time_scale = in->time_scale;
@@ -905,7 +861,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VPE_DEINTERLACE: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VPE_DEINTERLACE;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -913,7 +868,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_H264_GENERATE_AUDNAL: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_H264_GENERATE_AUDNAL;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -921,7 +875,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_BUFFER_ALLOC_MODE: {
 		struct hfi_buffer_alloc_mode *in = pdata, *mode = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_BUFFER_ALLOC_MODE;
 		mode->type = in->type;
 		mode->mode = in->mode;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*mode);
@@ -930,7 +883,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VDEC_FRAME_ASSEMBLY: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_FRAME_ASSEMBLY;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -938,8 +890,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_H264_VUI_BITSTREAM_RESTRC: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] =
-			HFI_PROPERTY_PARAM_VENC_H264_VUI_BITSTREAM_RESTRC;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -947,7 +897,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_PRESERVE_TEXT_QUALITY: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_PRESERVE_TEXT_QUALITY;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -956,7 +905,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_scs_threshold *thres = prop_data;
 		u32 *in = pdata;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_SCS_THRESHOLD;
 		thres->threshold_value = *in;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*thres);
 		break;
@@ -974,7 +922,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_MVC_BUFFER_LAYOUT;
 		mvc->layout_type = in->layout_type;
 		mvc->bright_view_first = in->bright_view_first;
 		mvc->ngap = in->ngap;
@@ -994,7 +941,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_LTRMODE;
 		ltr->ltr_mode = in->ltr_mode;
 		ltr->ltr_count = in->ltr_count;
 		ltr->trust_mode = in->trust_mode;
@@ -1004,7 +950,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_USELTRFRAME: {
 		struct hfi_ltr_use *in = pdata, *ltr_use = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_USELTRFRAME;
 		ltr_use->frames = in->frames;
 		ltr_use->ref_ltr = in->ref_ltr;
 		ltr_use->use_constrnt = in->use_constrnt;
@@ -1014,7 +959,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_MARKLTRFRAME: {
 		struct hfi_ltr_mark *in = pdata, *ltr_mark = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_MARKLTRFRAME;
 		ltr_mark->mark_frame = in->mark_frame;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*ltr_mark);
 		break;
@@ -1022,7 +966,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_HIER_P_MAX_NUM_ENH_LAYER: {
 		u32 *in = pdata;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_HIER_P_MAX_NUM_ENH_LAYER;
 		pkt->data[1] = *in;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -1030,7 +973,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_HIER_P_ENH_LAYER: {
 		u32 *in = pdata;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_HIER_P_ENH_LAYER;
 		pkt->data[1] = *in;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -1038,7 +980,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_DISABLE_RC_TIMESTAMP: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_DISABLE_RC_TIMESTAMP;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -1046,7 +987,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_INITIAL_QP: {
 		struct hfi_initial_quantization *in = pdata, *quant = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_INITIAL_QP;
 		quant->init_qp_enable = in->init_qp_enable;
 		quant->qp_i = in->qp_i;
 		quant->qp_p = in->qp_p;
@@ -1058,7 +998,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_vpe_color_space_conversion *in = pdata;
 		struct hfi_vpe_color_space_conversion *csc = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VPE_COLOR_SPACE_CONVERSION;
 		memcpy(csc->csc_matrix, in->csc_matrix,
 		       sizeof(csc->csc_matrix));
 		memcpy(csc->csc_bias, in->csc_bias, sizeof(csc->csc_bias));
@@ -1069,8 +1008,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_VPX_ERROR_RESILIENCE_MODE: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] =
-			HFI_PROPERTY_PARAM_VENC_VPX_ERROR_RESILIENCE_MODE;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -1078,7 +1015,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_H264_NAL_SVC_EXT: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_H264_NAL_SVC_EXT;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -1086,7 +1022,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_CONFIG_VENC_PERF_MODE: {
 		u32 *in = pdata;
 
-		pkt->data[0] = HFI_PROPERTY_CONFIG_VENC_PERF_MODE;
 		pkt->data[1] = *in;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -1094,7 +1029,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_HIER_B_MAX_NUM_ENH_LAYER: {
 		u32 *in = pdata;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_HIER_B_MAX_NUM_ENH_LAYER;
 		pkt->data[1] = *in;
 		pkt->shdr.hdr.size += sizeof(u32) * 2;
 		break;
@@ -1102,7 +1036,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VDEC_NONCP_OUTPUT2: {
 		struct hfi_enable *in = pdata, *en = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_NONCP_OUTPUT2;
 		en->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*en);
 		break;
@@ -1110,7 +1043,6 @@ static int pkt_session_set_property_1x(struct hfi_session_set_property_pkt *pkt,
 	case HFI_PROPERTY_PARAM_VENC_HIER_P_HYBRID_MODE: {
 		struct hfi_hybrid_hierp *in = pdata, *hierp = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_HIER_P_HYBRID_MODE;
 		hierp->layers = in->layers;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*hierp);
 		break;
@@ -1185,6 +1117,7 @@ pkt_session_set_property_3xx(struct hfi_session_set_property_pkt *pkt,
 	pkt->shdr.hdr.pkt_type = HFI_CMD_SESSION_SET_PROPERTY;
 	pkt->shdr.session_id = hash32_ptr(cookie);
 	pkt->num_properties = 1;
+	pkt->data[0] = ptype;
 
 	/*
 	 * Any session set property which is different in 3XX packetization
@@ -1196,7 +1129,6 @@ pkt_session_set_property_3xx(struct hfi_session_set_property_pkt *pkt,
 		struct hfi_multi_stream *in = pdata;
 		struct hfi_multi_stream_3x *multi = prop_data;
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VDEC_MULTI_STREAM;
 		multi->buffer_type = in->buffer_type;
 		multi->enable = in->enable;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*multi);
@@ -1218,7 +1150,6 @@ pkt_session_set_property_3xx(struct hfi_session_set_property_pkt *pkt,
 			break;
 		}
 
-		pkt->data[0] = HFI_PROPERTY_PARAM_VENC_INTRA_REFRESH;
 		intra->mode = in->mode;
 		intra->mbs = in->cir_mbs;
 		pkt->shdr.hdr.size += sizeof(u32) + sizeof(*intra);
diff --git a/drivers/media/platform/qcom/venus/hfi_helper.h b/drivers/media/platform/qcom/venus/hfi_helper.h
index 8d282dba9e57..55d8eb21403a 100644
--- a/drivers/media/platform/qcom/venus/hfi_helper.h
+++ b/drivers/media/platform/qcom/venus/hfi_helper.h
@@ -585,8 +585,8 @@ struct hfi_enable {
 
 struct hfi_h264_db_control {
 	u32 mode;
-	u32 slice_alpha_offset;
-	u32 slice_beta_offset;
+	s32 slice_alpha_offset;
+	s32 slice_beta_offset;
 };
 
 #define HFI_H264_ENTROPY_CAVLC			0x1
diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c
index 3fcf0e9b7b29..e3a10a852cad 100644
--- a/drivers/media/platform/qcom/venus/venc.c
+++ b/drivers/media/platform/qcom/venus/venc.c
@@ -234,6 +234,16 @@ static int venc_v4l2_to_hfi(int id, int value)
 		case 3:
 			return HFI_VPX_PROFILE_VERSION_3;
 		}
+	case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE:
+		switch (value) {
+		case V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED:
+		default:
+			return HFI_H264_DB_MODE_ALL_BOUNDARY;
+		case V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_DISABLED:
+			return HFI_H264_DB_MODE_DISABLE;
+		case V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_DISABLED_AT_SLICE_BOUNDARY:
+			return HFI_H264_DB_MODE_SKIP_SLICE_BOUNDARY;
+		}
 	}
 
 	return 0;
@@ -641,6 +651,8 @@ static int venc_set_properties(struct venus_inst *inst)
 
 	if (inst->fmt_cap->pixfmt == V4L2_PIX_FMT_H264) {
 		struct hfi_h264_vui_timing_info info;
+		struct hfi_h264_entropy_control entropy;
+		struct hfi_h264_db_control deblock;
 
 		ptype = HFI_PROPERTY_PARAM_VENC_H264_VUI_TIMING_INFO;
 		info.enable = 1;
@@ -650,10 +662,36 @@ static int venc_set_properties(struct venus_inst *inst)
 		ret = hfi_session_set_property(inst, ptype, &info);
 		if (ret)
 			return ret;
+
+		ptype = HFI_PROPERTY_PARAM_VENC_H264_ENTROPY_CONTROL;
+		entropy.entropy_mode = venc_v4l2_to_hfi(
+					  V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE,
+					  ctr->h264_entropy_mode);
+		entropy.cabac_model = HFI_H264_CABAC_MODEL_0;
+
+		ret = hfi_session_set_property(inst, ptype, &entropy);
+		if (ret)
+			return ret;
+
+		ptype = HFI_PROPERTY_PARAM_VENC_H264_DEBLOCK_CONTROL;
+		deblock.mode = venc_v4l2_to_hfi(
+				      V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE,
+				      ctr->h264_loop_filter_mode);
+		deblock.slice_alpha_offset = ctr->h264_loop_filter_alpha;
+		deblock.slice_beta_offset = ctr->h264_loop_filter_beta;
+
+		ret = hfi_session_set_property(inst, ptype, &deblock);
+		if (ret)
+			return ret;
 	}
 
+	/* IDR periodicity, n:
+	 * n = 0 - only the first I-frame is IDR frame
+	 * n = 1 - all I-frames will be IDR frames
+	 * n > 1 - every n-th I-frame will be IDR frame
+	 */
 	ptype = HFI_PROPERTY_CONFIG_VENC_IDR_PERIOD;
-	idrp.idr_period = ctr->gop_size;
+	idrp.idr_period = 0;
 	ret = hfi_session_set_property(inst, ptype, &idrp);
 	if (ret)
 		return ret;
@@ -667,10 +705,6 @@ static int venc_set_properties(struct venus_inst *inst)
 			return ret;
 	}
 
-	/* intra_period = pframes + bframes + 1 */
-	if (!ctr->num_p_frames)
-		ctr->num_p_frames = 2 * 15 - 1,
-
 	ptype = HFI_PROPERTY_CONFIG_VENC_INTRA_PERIOD;
 	intra_period.pframes = ctr->num_p_frames;
 	intra_period.bframes = ctr->num_b_frames;
@@ -767,6 +801,10 @@ static int venc_init_session(struct venus_inst *inst)
 	if (ret)
 		goto deinit;
 
+	ret = venc_set_properties(inst);
+	if (ret)
+		goto deinit;
+
 	return 0;
 deinit:
 	hfi_session_deinit(inst);
diff --git a/drivers/media/platform/qcom/venus/venc_ctrls.c b/drivers/media/platform/qcom/venus/venc_ctrls.c
index ab0fe51ff0f7..21e938a28662 100644
--- a/drivers/media/platform/qcom/venus/venc_ctrls.c
+++ b/drivers/media/platform/qcom/venus/venc_ctrls.c
@@ -30,10 +30,57 @@
 #define AT_SLICE_BOUNDARY	\
 	V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_DISABLED_AT_SLICE_BOUNDARY
 
+static int venc_calc_bpframes(u32 gop_size, u32 conseq_b, u32 *bf, u32 *pf)
+{
+	u32 half = (gop_size - 1) >> 1;
+	u32 b, p, ratio;
+	bool found = false;
+
+	if (!gop_size)
+		return -EINVAL;
+
+	*bf = *pf = 0;
+
+	if (!conseq_b) {
+		*pf = gop_size -  1;
+		return 0;
+	}
+
+	b = p = half;
+
+	for (; b <= gop_size - 1; b++, p--) {
+		if (b % p)
+			continue;
+
+		ratio = b / p;
+
+		if (ratio == conseq_b) {
+			found = true;
+			break;
+		}
+
+		if (ratio > conseq_b)
+			break;
+	}
+
+	if (!found)
+		return -EINVAL;
+
+	if (b + p + 1 != gop_size)
+		return -EINVAL;
+
+	*bf = b;
+	*pf = p;
+
+	return 0;
+}
+
 static int venc_op_s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct venus_inst *inst = ctrl_to_inst(ctrl);
 	struct venc_controls *ctr = &inst->controls.enc;
+	u32 bframes;
+	int ret;
 
 	switch (ctrl->id) {
 	case V4L2_CID_MPEG_VIDEO_BITRATE_MODE:
@@ -102,6 +149,11 @@ static int venc_op_s_ctrl(struct v4l2_ctrl *ctrl)
 	case V4L2_CID_MPEG_VIDEO_CYCLIC_INTRA_REFRESH_MB:
 		break;
 	case V4L2_CID_MPEG_VIDEO_GOP_SIZE:
+		ret = venc_calc_bpframes(ctrl->val, ctr->num_b_frames, &bframes,
+					 &ctr->num_p_frames);
+		if (ret)
+			return ret;
+
 		ctr->gop_size = ctrl->val;
 		break;
 	case V4L2_CID_MPEG_VIDEO_H264_I_PERIOD:
@@ -114,7 +166,12 @@ static int venc_op_s_ctrl(struct v4l2_ctrl *ctrl)
 		ctr->vp8_max_qp = ctrl->val;
 		break;
 	case V4L2_CID_MPEG_VIDEO_B_FRAMES:
-		ctr->num_b_frames = ctrl->val;
+		ret = venc_calc_bpframes(ctr->gop_size, ctrl->val, &bframes,
+					 &ctr->num_p_frames);
+		if (ret)
+			return ret;
+
+		ctr->num_b_frames = bframes;
 		break;
 	default:
 		return -EINVAL;
diff --git a/drivers/media/platform/rcar-vin/rcar-core.c b/drivers/media/platform/rcar-vin/rcar-core.c
index 108d776f3265..f1fc7978d6d1 100644
--- a/drivers/media/platform/rcar-vin/rcar-core.c
+++ b/drivers/media/platform/rcar-vin/rcar-core.c
@@ -187,7 +187,7 @@ static int rvin_digital_graph_init(struct rvin_dev *vin)
 		return -ENODEV;
 
 	vin_dbg(vin, "Found digital subdevice %pOF\n",
-		to_of_node(vin->digital->asd.match.fwnode.fwnode));
+		to_of_node(vin->digital->asd.match.fwnode));
 
 	vin->notifier.ops = &rvin_digital_notify_ops;
 	ret = v4l2_async_notifier_register(&vin->v4l2_dev, &vin->notifier);
diff --git a/drivers/media/platform/rcar_drif.c b/drivers/media/platform/rcar_drif.c
index 63c94f4028a7..b2e080ef5391 100644
--- a/drivers/media/platform/rcar_drif.c
+++ b/drivers/media/platform/rcar_drif.c
@@ -1107,7 +1107,7 @@ static int rcar_drif_notify_bound(struct v4l2_async_notifier *notifier,
 	struct rcar_drif_sdr *sdr =
 		container_of(notifier, struct rcar_drif_sdr, notifier);
 
-	if (sdr->ep.asd.match.fwnode.fwnode !=
+	if (sdr->ep.asd.match.fwnode !=
 	    of_fwnode_handle(subdev->dev->of_node)) {
 		rdrif_err(sdr, "subdev %s cannot bind\n", subdev->name);
 		return -EINVAL;
@@ -1235,7 +1235,7 @@ static int rcar_drif_parse_subdevs(struct rcar_drif_sdr *sdr)
 		return -EINVAL;
 	}
 
-	sdr->ep.asd.match.fwnode.fwnode = fwnode;
+	sdr->ep.asd.match.fwnode = fwnode;
 	sdr->ep.asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
 	notifier->num_subdevs++;
 
diff --git a/drivers/media/platform/s3c-camif/camif-capture.c b/drivers/media/platform/s3c-camif/camif-capture.c
index 25c7a7d42292..437395a61065 100644
--- a/drivers/media/platform/s3c-camif/camif-capture.c
+++ b/drivers/media/platform/s3c-camif/camif-capture.c
@@ -590,12 +590,12 @@ static int s3c_camif_close(struct file *file)
 	return ret;
 }
 
-static unsigned int s3c_camif_poll(struct file *file,
+static __poll_t s3c_camif_poll(struct file *file,
 				   struct poll_table_struct *wait)
 {
 	struct camif_vp *vp = video_drvdata(file);
 	struct camif_dev *camif = vp->camif;
-	int ret;
+	__poll_t ret;
 
 	mutex_lock(&camif->lock);
 	if (vp->owner && vp->owner != file->private_data)
diff --git a/drivers/media/platform/s5p-jpeg/jpeg-core.c b/drivers/media/platform/s5p-jpeg/jpeg-core.c
index faac8161b683..79b63da27f53 100644
--- a/drivers/media/platform/s5p-jpeg/jpeg-core.c
+++ b/drivers/media/platform/s5p-jpeg/jpeg-core.c
@@ -3086,7 +3086,7 @@ static int s5p_jpeg_runtime_resume(struct device *dev)
 	for (i = 0; i < jpeg->variant->num_clocks; i++) {
 		ret = clk_prepare_enable(jpeg->clocks[i]);
 		if (ret) {
-			while (--i > 0)
+			while (--i >= 0)
 				clk_disable_unprepare(jpeg->clocks[i]);
 			return ret;
 		}
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc.c b/drivers/media/platform/s5p-mfc/s5p_mfc.c
index bc68dbbcaec1..d5b94fc0040e 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc.c
@@ -988,14 +988,14 @@ static int s5p_mfc_release(struct file *file)
 }
 
 /* Poll */
-static unsigned int s5p_mfc_poll(struct file *file,
+static __poll_t s5p_mfc_poll(struct file *file,
 				 struct poll_table_struct *wait)
 {
 	struct s5p_mfc_ctx *ctx = fh_to_ctx(file->private_data);
 	struct s5p_mfc_dev *dev = ctx->dev;
 	struct vb2_queue *src_q, *dst_q;
 	struct vb2_buffer *src_vb = NULL, *dst_vb = NULL;
-	unsigned int rc = 0;
+	__poll_t rc = 0;
 	unsigned long flags;
 
 	mutex_lock(&dev->mfc_mutex);
@@ -1008,7 +1008,7 @@ static unsigned int s5p_mfc_poll(struct file *file,
 	 */
 	if ((!src_q->streaming || list_empty(&src_q->queued_list))
 		&& (!dst_q->streaming || list_empty(&dst_q->queued_list))) {
-		rc = POLLERR;
+		rc = EPOLLERR;
 		goto end;
 	}
 	mutex_unlock(&dev->mfc_mutex);
@@ -1017,14 +1017,14 @@ static unsigned int s5p_mfc_poll(struct file *file,
 	poll_wait(file, &dst_q->done_wq, wait);
 	mutex_lock(&dev->mfc_mutex);
 	if (v4l2_event_pending(&ctx->fh))
-		rc |= POLLPRI;
+		rc |= EPOLLPRI;
 	spin_lock_irqsave(&src_q->done_lock, flags);
 	if (!list_empty(&src_q->done_list))
 		src_vb = list_first_entry(&src_q->done_list, struct vb2_buffer,
 								done_entry);
 	if (src_vb && (src_vb->state == VB2_BUF_STATE_DONE
 				|| src_vb->state == VB2_BUF_STATE_ERROR))
-		rc |= POLLOUT | POLLWRNORM;
+		rc |= EPOLLOUT | EPOLLWRNORM;
 	spin_unlock_irqrestore(&src_q->done_lock, flags);
 	spin_lock_irqsave(&dst_q->done_lock, flags);
 	if (!list_empty(&dst_q->done_list))
@@ -1032,7 +1032,7 @@ static unsigned int s5p_mfc_poll(struct file *file,
 								done_entry);
 	if (dst_vb && (dst_vb->state == VB2_BUF_STATE_DONE
 				|| dst_vb->state == VB2_BUF_STATE_ERROR))
-		rc |= POLLIN | POLLRDNORM;
+		rc |= EPOLLIN | EPOLLRDNORM;
 	spin_unlock_irqrestore(&dst_q->done_lock, flags);
 end:
 	mutex_unlock(&dev->mfc_mutex);
@@ -1309,6 +1309,12 @@ static int s5p_mfc_probe(struct platform_device *pdev)
 		goto err_dma;
 	}
 
+	/*
+	 * Load fails if fs isn't mounted. Try loading anyway.
+	 * _open() will load it, it it fails now. Ignore failure.
+	 */
+	s5p_mfc_load_firmware(dev);
+
 	mutex_init(&dev->mfc_mutex);
 	init_waitqueue_head(&dev->queue);
 	dev->hw_lock = 0;
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_common.h b/drivers/media/platform/s5p-mfc/s5p_mfc_common.h
index 4220914529b2..76119a8cc477 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc_common.h
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc_common.h
@@ -290,6 +290,8 @@ struct s5p_mfc_priv_buf {
  * @mfc_cmds:		cmd structure holding HW commands function pointers
  * @mfc_regs:		structure holding MFC registers
  * @fw_ver:		loaded firmware sub-version
+ * @fw_get_done		flag set when request_firmware() is complete and
+ *			copied into fw_buf
  * risc_on:		flag indicates RISC is on or off
  *
  */
@@ -336,6 +338,7 @@ struct s5p_mfc_dev {
 	struct s5p_mfc_hw_cmds *mfc_cmds;
 	const struct s5p_mfc_regs *mfc_regs;
 	enum s5p_mfc_fw_ver fw_ver;
+	bool fw_get_done;
 	bool risc_on; /* indicates if RISC is on or off */
 };
 
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_ctrl.c b/drivers/media/platform/s5p-mfc/s5p_mfc_ctrl.c
index 69ef9c23a99a..f95cd76af537 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc_ctrl.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc_ctrl.c
@@ -55,6 +55,9 @@ int s5p_mfc_load_firmware(struct s5p_mfc_dev *dev)
 	 * into kernel. */
 	mfc_debug_enter();
 
+	if (dev->fw_get_done)
+		return 0;
+
 	for (i = MFC_FW_MAX_VERSIONS - 1; i >= 0; i--) {
 		if (!dev->variant->fw_name[i])
 			continue;
@@ -75,13 +78,9 @@ int s5p_mfc_load_firmware(struct s5p_mfc_dev *dev)
 		release_firmware(fw_blob);
 		return -ENOMEM;
 	}
-	if (!dev->fw_buf.virt) {
-		mfc_err("MFC firmware is not allocated\n");
-		release_firmware(fw_blob);
-		return -EINVAL;
-	}
 	memcpy(dev->fw_buf.virt, fw_blob->data, fw_blob->size);
 	wmb();
+	dev->fw_get_done = true;
 	release_firmware(fw_blob);
 	mfc_debug_leave();
 	return 0;
@@ -93,6 +92,7 @@ int s5p_mfc_release_firmware(struct s5p_mfc_dev *dev)
 	/* Before calling this function one has to make sure
 	 * that MFC is no longer processing */
 	s5p_mfc_release_priv_buf(dev, &dev->fw_buf);
+	dev->fw_get_done = false;
 	return 0;
 }
 
diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c b/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c
index 2a5fd7c42cd5..0d5d465561be 100644
--- a/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c
+++ b/drivers/media/platform/s5p-mfc/s5p_mfc_enc.c
@@ -2080,7 +2080,7 @@ int s5p_mfc_enc_ctrls_setup(struct s5p_mfc_ctx *ctx)
 
 			if (cfg.type == V4L2_CTRL_TYPE_MENU) {
 				cfg.step = 0;
-				cfg.menu_skip_mask = cfg.menu_skip_mask;
+				cfg.menu_skip_mask = controls[i].menu_skip_mask;
 				cfg.qmenu = mfc51_get_menu(cfg.id);
 			} else {
 				cfg.step = controls[i].step;
diff --git a/drivers/media/platform/sh_veu.c b/drivers/media/platform/sh_veu.c
index dedc1b024f6f..976ea0bb5b6c 100644
--- a/drivers/media/platform/sh_veu.c
+++ b/drivers/media/platform/sh_veu.c
@@ -1016,7 +1016,7 @@ static int sh_veu_release(struct file *file)
 	return 0;
 }
 
-static unsigned int sh_veu_poll(struct file *file,
+static __poll_t sh_veu_poll(struct file *file,
 				struct poll_table_struct *wait)
 {
 	struct sh_veu_file *veu_file = file->private_data;
diff --git a/drivers/media/platform/sh_vou.c b/drivers/media/platform/sh_vou.c
index 871da2a2a91c..4dccf29e9d78 100644
--- a/drivers/media/platform/sh_vou.c
+++ b/drivers/media/platform/sh_vou.c
@@ -1181,7 +1181,7 @@ static int sh_vou_release(struct file *file)
 
 /* sh_vou display ioctl operations */
 static const struct v4l2_ioctl_ops sh_vou_ioctl_ops = {
-	.vidioc_querycap        	= sh_vou_querycap,
+	.vidioc_querycap		= sh_vou_querycap,
 	.vidioc_enum_fmt_vid_out	= sh_vou_enum_fmt_vid_out,
 	.vidioc_g_fmt_vid_out		= sh_vou_g_fmt_vid_out,
 	.vidioc_s_fmt_vid_out		= sh_vou_s_fmt_vid_out,
diff --git a/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c b/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
index 36762ec954e7..242342fd7ede 100644
--- a/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
+++ b/drivers/media/platform/soc_camera/sh_mobile_ceu_camera.c
@@ -451,13 +451,18 @@ static void sh_mobile_ceu_stop_streaming(struct vb2_queue *q)
 	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
 	struct sh_mobile_ceu_dev *pcdev = ici->priv;
 	struct list_head *buf_head, *tmp;
+	struct vb2_v4l2_buffer *vbuf;
 
 	spin_lock_irq(&pcdev->lock);
 
 	pcdev->active = NULL;
 
-	list_for_each_safe(buf_head, tmp, &pcdev->capture)
+	list_for_each_safe(buf_head, tmp, &pcdev->capture) {
+		vbuf = &list_entry(buf_head, struct sh_mobile_ceu_buffer,
+				   queue)->vb;
+		vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_DONE);
 		list_del_init(buf_head);
+	}
 
 	spin_unlock_irq(&pcdev->lock);
 
@@ -1553,7 +1558,7 @@ static int sh_mobile_ceu_set_liveselection(struct soc_camera_device *icd,
 	return ret;
 }
 
-static unsigned int sh_mobile_ceu_poll(struct file *file, poll_table *pt)
+static __poll_t sh_mobile_ceu_poll(struct file *file, poll_table *pt)
 {
 	struct soc_camera_device *icd = file->private_data;
 
diff --git a/drivers/media/platform/soc_camera/soc_camera.c b/drivers/media/platform/soc_camera/soc_camera.c
index 916ff68b73d4..c86dd2fdab84 100644
--- a/drivers/media/platform/soc_camera/soc_camera.c
+++ b/drivers/media/platform/soc_camera/soc_camera.c
@@ -805,14 +805,14 @@ static int soc_camera_mmap(struct file *file, struct vm_area_struct *vma)
 	return err;
 }
 
-static unsigned int soc_camera_poll(struct file *file, poll_table *pt)
+static __poll_t soc_camera_poll(struct file *file, poll_table *pt)
 {
 	struct soc_camera_device *icd = file->private_data;
 	struct soc_camera_host *ici = to_soc_camera_host(icd->parent);
-	unsigned res = POLLERR;
+	__poll_t res = EPOLLERR;
 
 	if (icd->streamer != file)
-		return POLLERR;
+		return EPOLLERR;
 
 	mutex_lock(&ici->host_lock);
 	res = ici->ops->poll(file, pt);
@@ -1517,7 +1517,7 @@ static int soc_of_bind(struct soc_camera_host *ici,
 	if (!info)
 		return -ENOMEM;
 
-	info->sasd.asd.match.fwnode.fwnode = of_fwnode_handle(remote);
+	info->sasd.asd.match.fwnode = of_fwnode_handle(remote);
 	info->sasd.asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
 	info->subdev = &info->sasd.asd;
 
diff --git a/drivers/media/platform/soc_camera/soc_scale_crop.c b/drivers/media/platform/soc_camera/soc_scale_crop.c
index 270ec613c27c..6164102e6f9f 100644
--- a/drivers/media/platform/soc_camera/soc_scale_crop.c
+++ b/drivers/media/platform/soc_camera/soc_scale_crop.c
@@ -420,3 +420,7 @@ void soc_camera_calc_client_output(struct soc_camera_device *icd,
 	mf->height = soc_camera_shift_scale(rect->height, shift, scale_v);
 }
 EXPORT_SYMBOL(soc_camera_calc_client_output);
+
+MODULE_DESCRIPTION("soc-camera scaling-cropping functions");
+MODULE_AUTHOR("Guennadi Liakhovetski <kernel@pengutronix.de>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/sti/bdisp/bdisp-debug.c b/drivers/media/platform/sti/bdisp/bdisp-debug.c
index 2cc289e4dea1..c6a4e2de5c0c 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-debug.c
+++ b/drivers/media/platform/sti/bdisp/bdisp-debug.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/debugfs.h>
diff --git a/drivers/media/platform/sti/bdisp/bdisp-filter.h b/drivers/media/platform/sti/bdisp/bdisp-filter.h
index 53e52fb4127f..d25adb57e3d0 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-filter.h
+++ b/drivers/media/platform/sti/bdisp/bdisp-filter.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #define BDISP_HF_NB             64
diff --git a/drivers/media/platform/sti/bdisp/bdisp-hw.c b/drivers/media/platform/sti/bdisp/bdisp-hw.c
index b7892f3efd98..a5eb592e12c0 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-hw.c
+++ b/drivers/media/platform/sti/bdisp/bdisp-hw.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/delay.h>
@@ -1064,7 +1064,7 @@ static void bdisp_hw_save_request(struct bdisp_ctx *ctx)
 		if (!copy_node[i]) {
 			copy_node[i] = devm_kzalloc(ctx->bdisp_dev->dev,
 						    sizeof(*copy_node[i]),
-						    GFP_KERNEL);
+						    GFP_ATOMIC);
 			if (!copy_node[i])
 				return;
 		}
diff --git a/drivers/media/platform/sti/bdisp/bdisp-reg.h b/drivers/media/platform/sti/bdisp/bdisp-reg.h
index e7e1a425f65a..b07ecc903707 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-reg.h
+++ b/drivers/media/platform/sti/bdisp/bdisp-reg.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 struct bdisp_node {
diff --git a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
index 7e9ed9c7b3e1..bf4ca16db440 100644
--- a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
+++ b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/errno.h>
diff --git a/drivers/media/platform/sti/bdisp/bdisp.h b/drivers/media/platform/sti/bdisp/bdisp.h
index b3fbf9902595..e309cde379ca 100644
--- a/drivers/media/platform/sti/bdisp/bdisp.h
+++ b/drivers/media/platform/sti/bdisp/bdisp.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2014
  * Authors: Fabien Dessenne <fabien.dessenne@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
diff --git a/drivers/media/platform/sti/c8sectpfe/Kconfig b/drivers/media/platform/sti/c8sectpfe/Kconfig
index 7420a50572d3..740190f8a3b6 100644
--- a/drivers/media/platform/sti/c8sectpfe/Kconfig
+++ b/drivers/media/platform/sti/c8sectpfe/Kconfig
@@ -1,6 +1,6 @@
 config DVB_C8SECTPFE
 	tristate "STMicroelectronics C8SECTPFE DVB support"
-	depends on PINCTRL && DVB_CORE && I2C
+	depends on PINCTRL && DVB_CORE && I2C && HAS_DMA
 	depends on ARCH_STI || ARCH_MULTIPLATFORM || COMPILE_TEST
 	select FW_LOADER
 	select DEBUG_FS
diff --git a/drivers/media/platform/sti/c8sectpfe/Makefile b/drivers/media/platform/sti/c8sectpfe/Makefile
index b642b4fd5045..34d69472b6f0 100644
--- a/drivers/media/platform/sti/c8sectpfe/Makefile
+++ b/drivers/media/platform/sti/c8sectpfe/Makefile
@@ -4,7 +4,6 @@ c8sectpfe-y += c8sectpfe-core.o c8sectpfe-common.o c8sectpfe-dvb.o \
 
 obj-$(CONFIG_DVB_C8SECTPFE) += c8sectpfe.o
 
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/common
-ccflags-y += -Idrivers/media/dvb-core/ -Idrivers/media/dvb-frontends/ \
-		-Idrivers/media/tuners/
+ccflags-y += -Idrivers/media/dvb-frontends/
+ccflags-y += -Idrivers/media/tuners/
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.c b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.c
index 2dfbe8ab5214..5df67da25525 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.c
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * c8sectpfe-common.c - C8SECTPFE STi DVB driver
  *
@@ -5,10 +6,6 @@
  *
  *   Author: Peter Griffin <peter.griffin@linaro.org>
  *
- *      This program is free software; you can redistribute it and/or
- *      modify it under the terms of the GNU General Public License as
- *      published by the Free Software Foundation; either version 2 of
- *      the License, or (at your option) any later version.
  */
 #include <linux/completion.h>
 #include <linux/delay.h>
@@ -24,11 +21,11 @@
 #include <linux/time.h>
 #include <linux/wait.h>
 
-#include "dmxdev.h"
-#include "dvbdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvbdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #include "c8sectpfe-common.h"
 #include "c8sectpfe-core.h"
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.h b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.h
index da21c0ac0fc1..5ab7ca448cf9 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.h
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-common.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * c8sectpfe-common.h - C8SECTPFE STi DVB driver
  *
@@ -5,10 +6,6 @@
  *
  *   Author: Peter Griffin <peter.griffin@linaro.org>
  *
- *      This program is free software; you can redistribute it and/or
- *      modify it under the terms of the GNU General Public License as
- *      published by the Free Software Foundation; either version 2 of
- *      the License, or (at your option) any later version.
  */
 #ifndef _C8SECTPFE_COMMON_H_
 #define _C8SECTPFE_COMMON_H_
@@ -18,10 +15,10 @@
 #include <linux/gpio.h>
 #include <linux/version.h>
 
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 /* Maximum number of channels */
 #define C8SECTPFE_MAXADAPTER (4)
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.c b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.c
index a0acee7671b1..3c05b3dc49ec 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.c
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * c8sectpfe-core.c - C8SECTPFE STi DVB driver
  *
@@ -6,10 +7,6 @@
  *   Author:Peter Bennett <peter.bennett@st.com>
  *	    Peter Griffin <peter.griffin@linaro.org>
  *
- *	This program is free software; you can redistribute it and/or
- *	modify it under the terms of the GNU General Public License as
- *	published by the Free Software Foundation; either version 2 of
- *	the License, or (at your option) any later version.
  */
 #include <linux/atomic.h>
 #include <linux/clk.h>
@@ -38,10 +35,10 @@
 #include "c8sectpfe-core.h"
 #include "c8sectpfe-common.h"
 #include "c8sectpfe-debugfs.h"
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 
 #define FIRMWARE_MEMDMA "pti_memdma_h407.elf"
 MODULE_FIRMWARE(FIRMWARE_MEMDMA);
@@ -83,7 +80,7 @@ static void c8sectpfe_timer_interrupt(struct timer_list *t)
 static void channel_swdemux_tsklet(unsigned long data)
 {
 	struct channel_info *channel = (struct channel_info *)data;
-	struct c8sectpfei *fei = channel->fei;
+	struct c8sectpfei *fei;
 	unsigned long wp, rp;
 	int pos, num_packets, n, size;
 	u8 *buf;
@@ -91,6 +88,8 @@ static void channel_swdemux_tsklet(unsigned long data)
 	if (unlikely(!channel || !channel->irec))
 		return;
 
+	fei = channel->fei;
+
 	wp = readl(channel->irec + DMA_PRDS_BUSWP_TP(0));
 	rp = readl(channel->irec + DMA_PRDS_BUSRP_TP(0));
 
@@ -691,7 +690,7 @@ static int c8sectpfe_probe(struct platform_device *pdev)
 	if (IS_ERR(fei->sram))
 		return PTR_ERR(fei->sram);
 
-	fei->sram_size = res->end - res->start;
+	fei->sram_size = resource_size(res);
 
 	fei->idle_irq = platform_get_irq_byname(pdev, "c8sectpfe-idle-irq");
 	if (fei->idle_irq < 0) {
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.h b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.h
index 39e7a221a941..3dbb3a287cc0 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.h
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-core.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * c8sectpfe-core.h - C8SECTPFE STi DVB driver
  *
@@ -6,10 +7,6 @@
  *   Author:Peter Bennett <peter.bennett@st.com>
  *	    Peter Griffin <peter.griffin@linaro.org>
  *
- *	This program is free software; you can redistribute it and/or
- *	modify it under the terms of the GNU General Public License as
- *	published by the Free Software Foundation; either version 2 of
- *	the License, or (at your option) any later version.
  */
 #ifndef _C8SECTPFE_CORE_H_
 #define _C8SECTPFE_CORE_H_
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.c b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.c
index e9ba13db49cd..8f0ddcbeed9d 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.c
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * c8sectpfe-debugfs.c - C8SECTPFE STi DVB driver
  *
@@ -5,14 +6,6 @@
  *
  * Author: Peter Griffin <peter.griffin@linaro.org>
  *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2  of
- * the License as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 #include <linux/debugfs.h>
 #include <linux/device.h>
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.h b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.h
index 8af1ac1378c8..b8c30bcc8df9 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.h
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-debugfs.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /**
  * c8sectpfe-debugfs.h - C8SECTPFE STi DVB driver debugfs header
  *
@@ -5,14 +6,6 @@
  *
  * Authors: Peter Griffin <peter.griffin@linaro.org>
  *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2  of
- * the License as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #ifndef __C8SECTPFE_DEBUG_H
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c
index 2c0015b1264d..075d4695ee4d 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *  c8sectpfe-dvb.c - C8SECTPFE STi DVB driver
  *
@@ -5,16 +6,6 @@
  *
  *  Author Peter Griffin <peter.griffin@linaro.org>
  *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *
- *  GNU General Public License for more details.
  */
 #include <linux/completion.h>
 #include <linux/delay.h>
diff --git a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.h b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.h
index bd366dbc82b3..3d87a9ae8702 100644
--- a/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.h
+++ b/drivers/media/platform/sti/c8sectpfe/c8sectpfe-dvb.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * c8sectpfe-common.h - C8SECTPFE STi DVB driver
  *
@@ -5,10 +6,6 @@
  *
  *   Author: Peter Griffin <peter.griffin@linaro.org>
  *
- *      This program is free software; you can redistribute it and/or
- *      modify it under the terms of the GNU General Public License as
- *      published by the Free Software Foundation; either version 2 of
- *      the License, or (at your option) any later version.
  */
 #ifndef _C8SECTPFE_DVB_H_
 #define _C8SECTPFE_DVB_H_
diff --git a/drivers/media/platform/sti/cec/stih-cec.c b/drivers/media/platform/sti/cec/stih-cec.c
index 70160df36de9..d34099f75990 100644
--- a/drivers/media/platform/sti/cec/stih-cec.c
+++ b/drivers/media/platform/sti/cec/stih-cec.c
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * STIH4xx CEC driver
  * Copyright (C) STMicroelectronics SA 2016
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 #include <linux/clk.h>
 #include <linux/interrupt.h>
diff --git a/drivers/media/platform/sti/delta/delta-cfg.h b/drivers/media/platform/sti/delta/delta-cfg.h
index c6388f575800..f47c6e6ff083 100644
--- a/drivers/media/platform/sti/delta/delta-cfg.h
+++ b/drivers/media/platform/sti/delta/delta-cfg.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef DELTA_CFG_H
diff --git a/drivers/media/platform/sti/delta/delta-debug.c b/drivers/media/platform/sti/delta/delta-debug.c
index a7ebf2cc7783..4b2eb6b63aa2 100644
--- a/drivers/media/platform/sti/delta/delta-debug.c
+++ b/drivers/media/platform/sti/delta/delta-debug.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Hugues Fruchet <hugues.fruchet@st.com>
  *          Fabrice Lecoultre <fabrice.lecoultre@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include "delta.h"
diff --git a/drivers/media/platform/sti/delta/delta-debug.h b/drivers/media/platform/sti/delta/delta-debug.h
index 955c1587ac2d..fa90252623e1 100644
--- a/drivers/media/platform/sti/delta/delta-debug.h
+++ b/drivers/media/platform/sti/delta/delta-debug.h
@@ -1,9 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Hugues Fruchet <hugues.fruchet@st.com>
  *          Fabrice Lecoultre <fabrice.lecoultre@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef DELTA_DEBUG_H
diff --git a/drivers/media/platform/sti/delta/delta-ipc.c b/drivers/media/platform/sti/delta/delta-ipc.c
index 41e4a4c259b3..a4603d573c34 100644
--- a/drivers/media/platform/sti/delta/delta-ipc.c
+++ b/drivers/media/platform/sti/delta/delta-ipc.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/rpmsg.h>
diff --git a/drivers/media/platform/sti/delta/delta-ipc.h b/drivers/media/platform/sti/delta/delta-ipc.h
index cef2019c72d4..9fba6b5d169a 100644
--- a/drivers/media/platform/sti/delta/delta-ipc.h
+++ b/drivers/media/platform/sti/delta/delta-ipc.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef DELTA_IPC_H
diff --git a/drivers/media/platform/sti/delta/delta-mem.c b/drivers/media/platform/sti/delta/delta-mem.c
index d7b53d31caa6..aeccd50583da 100644
--- a/drivers/media/platform/sti/delta/delta-mem.c
+++ b/drivers/media/platform/sti/delta/delta-mem.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include "delta.h"
diff --git a/drivers/media/platform/sti/delta/delta-mem.h b/drivers/media/platform/sti/delta/delta-mem.h
index f8ca109e1241..ff7d02f00b28 100644
--- a/drivers/media/platform/sti/delta/delta-mem.h
+++ b/drivers/media/platform/sti/delta/delta-mem.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef DELTA_MEM_H
diff --git a/drivers/media/platform/sti/delta/delta-mjpeg-dec.c b/drivers/media/platform/sti/delta/delta-mjpeg-dec.c
index 84ea43c0eb46..0533d4a083d2 100644
--- a/drivers/media/platform/sti/delta/delta-mjpeg-dec.c
+++ b/drivers/media/platform/sti/delta/delta-mjpeg-dec.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2013
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/slab.h>
diff --git a/drivers/media/platform/sti/delta/delta-mjpeg-fw.h b/drivers/media/platform/sti/delta/delta-mjpeg-fw.h
index de803d0c2fe8..5a9404f4d055 100644
--- a/drivers/media/platform/sti/delta/delta-mjpeg-fw.h
+++ b/drivers/media/platform/sti/delta/delta-mjpeg-fw.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef DELTA_MJPEG_FW_H
diff --git a/drivers/media/platform/sti/delta/delta-mjpeg-hdr.c b/drivers/media/platform/sti/delta/delta-mjpeg-hdr.c
index a8fd8fa0ecb5..90e5b2f72c82 100644
--- a/drivers/media/platform/sti/delta/delta-mjpeg-hdr.c
+++ b/drivers/media/platform/sti/delta/delta-mjpeg-hdr.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2013
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include "delta.h"
diff --git a/drivers/media/platform/sti/delta/delta-mjpeg.h b/drivers/media/platform/sti/delta/delta-mjpeg.h
index 18e6b37217ee..43f7a88b6e59 100644
--- a/drivers/media/platform/sti/delta/delta-mjpeg.h
+++ b/drivers/media/platform/sti/delta/delta-mjpeg.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2013
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef DELTA_MJPEG_H
diff --git a/drivers/media/platform/sti/delta/delta-v4l2.c b/drivers/media/platform/sti/delta/delta-v4l2.c
index b2dc3d223a9c..232d508c5b66 100644
--- a/drivers/media/platform/sti/delta/delta-v4l2.c
+++ b/drivers/media/platform/sti/delta/delta-v4l2.c
@@ -1,9 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Hugues Fruchet <hugues.fruchet@st.com>
  *          Jean-Christophe Trotin <jean-christophe.trotin@st.com>
  *          for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
diff --git a/drivers/media/platform/sti/delta/delta.h b/drivers/media/platform/sti/delta/delta.h
index 60c073246a01..2ba99922c05b 100644
--- a/drivers/media/platform/sti/delta/delta.h
+++ b/drivers/media/platform/sti/delta/delta.h
@@ -1,7 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Author: Hugues Fruchet <hugues.fruchet@st.com> for STMicroelectronics.
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef DELTA_H
diff --git a/drivers/media/platform/sti/hva/hva-debugfs.c b/drivers/media/platform/sti/hva/hva-debugfs.c
index 83a6258a155b..9f7e8ac875d1 100644
--- a/drivers/media/platform/sti/hva/hva-debugfs.c
+++ b/drivers/media/platform/sti/hva/hva-debugfs.c
@@ -1,8 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/debugfs.h>
diff --git a/drivers/media/platform/sti/hva/hva-h264.c b/drivers/media/platform/sti/hva/hva-h264.c
index 17f1eb0ba957..b61a5d337d2a 100644
--- a/drivers/media/platform/sti/hva/hva-h264.c
+++ b/drivers/media/platform/sti/hva/hva-h264.c
@@ -1,8 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include "hva.h"
diff --git a/drivers/media/platform/sti/hva/hva-hw.c b/drivers/media/platform/sti/hva/hva-hw.c
index ec25bdcfa3d1..7917fd2c4bd4 100644
--- a/drivers/media/platform/sti/hva/hva-hw.c
+++ b/drivers/media/platform/sti/hva/hva-hw.c
@@ -1,8 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/clk.h>
diff --git a/drivers/media/platform/sti/hva/hva-hw.h b/drivers/media/platform/sti/hva/hva-hw.h
index b46017dcfae9..b298990264d5 100644
--- a/drivers/media/platform/sti/hva/hva-hw.h
+++ b/drivers/media/platform/sti/hva/hva-hw.h
@@ -1,8 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef HVA_HW_H
diff --git a/drivers/media/platform/sti/hva/hva-mem.c b/drivers/media/platform/sti/hva/hva-mem.c
index 821c78ed208c..caf50cd4bb77 100644
--- a/drivers/media/platform/sti/hva/hva-mem.c
+++ b/drivers/media/platform/sti/hva/hva-mem.c
@@ -1,8 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include "hva.h"
diff --git a/drivers/media/platform/sti/hva/hva-mem.h b/drivers/media/platform/sti/hva/hva-mem.h
index a95c728a45e6..fec549dff2b3 100644
--- a/drivers/media/platform/sti/hva/hva-mem.h
+++ b/drivers/media/platform/sti/hva/hva-mem.h
@@ -1,8 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef HVA_MEM_H
diff --git a/drivers/media/platform/sti/hva/hva-v4l2.c b/drivers/media/platform/sti/hva/hva-v4l2.c
index 1c4fc33cbcb5..2ab0b5cc5c22 100644
--- a/drivers/media/platform/sti/hva/hva-v4l2.c
+++ b/drivers/media/platform/sti/hva/hva-v4l2.c
@@ -1,8 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/module.h>
diff --git a/drivers/media/platform/sti/hva/hva.h b/drivers/media/platform/sti/hva/hva.h
index 0d749b257a21..1226d60cc367 100644
--- a/drivers/media/platform/sti/hva/hva.h
+++ b/drivers/media/platform/sti/hva/hva.h
@@ -1,8 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) STMicroelectronics SA 2015
  * Authors: Yannick Fertre <yannick.fertre@st.com>
  *          Hugues Fruchet <hugues.fruchet@st.com>
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #ifndef HVA_H
@@ -245,7 +245,7 @@ struct hva_enc;
  * @dbg:             context debug info
  */
 struct hva_ctx {
-	struct hva_dev		        *hva_dev;
+	struct hva_dev			*hva_dev;
 	struct v4l2_fh			fh;
 	struct v4l2_ctrl_handler	ctrl_handler;
 	struct hva_controls		ctrls;
diff --git a/drivers/media/platform/stm32/stm32-dcmi.c b/drivers/media/platform/stm32/stm32-dcmi.c
index ac4c450a6c7d..9460b3080dca 100644
--- a/drivers/media/platform/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/stm32/stm32-dcmi.c
@@ -1520,7 +1520,7 @@ static int dcmi_graph_parse(struct stm32_dcmi *dcmi, struct device_node *node)
 		/* Remote node to connect */
 		dcmi->entity.node = remote;
 		dcmi->entity.asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
-		dcmi->entity.asd.match.fwnode.fwnode = of_fwnode_handle(remote);
+		dcmi->entity.asd.match.fwnode = of_fwnode_handle(remote);
 		return 0;
 	}
 }
diff --git a/drivers/media/platform/tegra-cec/tegra_cec.c b/drivers/media/platform/tegra-cec/tegra_cec.c
index 807c94c70049..aba488cd0e64 100644
--- a/drivers/media/platform/tegra-cec/tegra_cec.c
+++ b/drivers/media/platform/tegra-cec/tegra_cec.c
@@ -172,16 +172,13 @@ static irqreturn_t tegra_cec_irq_handler(int irq, void *data)
 		}
 	}
 
-	if (status & (TEGRA_CEC_INT_STAT_RX_REGISTER_OVERRUN |
-		      TEGRA_CEC_INT_STAT_RX_BUS_ANOMALY_DETECTED |
-		      TEGRA_CEC_INT_STAT_RX_START_BIT_DETECTED |
-		      TEGRA_CEC_INT_STAT_RX_BUS_ERROR_DETECTED)) {
+	if (status & TEGRA_CEC_INT_STAT_RX_START_BIT_DETECTED) {
 		cec_write(cec, TEGRA_CEC_INT_STAT,
-			  (TEGRA_CEC_INT_STAT_RX_REGISTER_OVERRUN |
-			   TEGRA_CEC_INT_STAT_RX_BUS_ANOMALY_DETECTED |
-			   TEGRA_CEC_INT_STAT_RX_START_BIT_DETECTED |
-			   TEGRA_CEC_INT_STAT_RX_BUS_ERROR_DETECTED));
-	} else if (status & TEGRA_CEC_INT_STAT_RX_REGISTER_FULL) {
+			  TEGRA_CEC_INT_STAT_RX_START_BIT_DETECTED);
+		cec->rx_done = false;
+		cec->rx_buf_cnt = 0;
+	}
+	if (status & TEGRA_CEC_INT_STAT_RX_REGISTER_FULL) {
 		u32 v;
 
 		cec_write(cec, TEGRA_CEC_INT_STAT,
@@ -255,7 +252,7 @@ static int tegra_cec_adap_enable(struct cec_adapter *adap, bool enable)
 		  TEGRA_CEC_INT_MASK_TX_BUS_ANOMALY_DETECTED |
 		  TEGRA_CEC_INT_MASK_TX_FRAME_TRANSMITTED |
 		  TEGRA_CEC_INT_MASK_RX_REGISTER_FULL |
-		  TEGRA_CEC_INT_MASK_RX_REGISTER_OVERRUN);
+		  TEGRA_CEC_INT_MASK_RX_START_BIT_DETECTED);
 
 	cec_write(cec, TEGRA_CEC_HW_CONTROL, TEGRA_CEC_HWCTRL_TX_RX_MODE);
 	return 0;
@@ -493,3 +490,8 @@ static struct platform_driver tegra_cec_driver = {
 };
 
 module_platform_driver(tegra_cec_driver);
+
+MODULE_DESCRIPTION("Tegra HDMI CEC driver");
+MODULE_AUTHOR("NVIDIA CORPORATION");
+MODULE_AUTHOR("Cisco Systems, Inc. and/or its affiliates");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/platform/ti-vpe/cal.c b/drivers/media/platform/ti-vpe/cal.c
index 8b586c864524..d1febe5baa6d 100644
--- a/drivers/media/platform/ti-vpe/cal.c
+++ b/drivers/media/platform/ti-vpe/cal.c
@@ -28,10 +28,7 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-ioctl.h>
-#include <media/v4l2-ctrls.h>
 #include <media/v4l2-fh.h>
-#include <media/v4l2-event.h>
-#include <media/v4l2-common.h>
 #include <media/videobuf2-core.h>
 #include <media/videobuf2-dma-contig.h>
 #include "cal_regs.h"
@@ -1705,7 +1702,7 @@ static int of_cal_create_instance(struct cal_ctx *ctx, int inst)
 		goto cleanup_exit;
 	}
 	asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
-	asd->match.fwnode.fwnode = of_fwnode_handle(sensor_node);
+	asd->match.fwnode = of_fwnode_handle(sensor_node);
 
 	remote_ep = of_graph_get_remote_endpoint(ep_node);
 	if (!remote_ep) {
diff --git a/drivers/media/platform/via-camera.c b/drivers/media/platform/via-camera.c
index 805d4a8fc17e..f77be9302120 100644
--- a/drivers/media/platform/via-camera.c
+++ b/drivers/media/platform/via-camera.c
@@ -764,7 +764,7 @@ static ssize_t viacam_read(struct file *filp, char __user *buffer,
 }
 
 
-static unsigned int viacam_poll(struct file *filp, struct poll_table_struct *pt)
+static __poll_t viacam_poll(struct file *filp, struct poll_table_struct *pt)
 {
 	struct via_camera *cam = video_drvdata(filp);
 
diff --git a/drivers/media/platform/via-camera.h b/drivers/media/platform/via-camera.h
index 2d67f8ce258d..54f16318b1b3 100644
--- a/drivers/media/platform/via-camera.h
+++ b/drivers/media/platform/via-camera.h
@@ -29,7 +29,7 @@
 #define   VCR_CI_BSS	  0x00000002  /* WTF "bit stream selection" */
 #define   VCR_CI_3BUFS	  0x00000004  /* 1 = 3 buffers, 0 = 2 buffers */
 #define   VCR_CI_VIPEN	  0x00000008  /* VIP enable */
-#define   VCR_CI_CCIR601_8  0	        /* CCIR601 input stream, 8 bit */
+#define   VCR_CI_CCIR601_8  0		/* CCIR601 input stream, 8 bit */
 #define   VCR_CI_CCIR656_8  0x00000010  /* ... CCIR656, 8 bit */
 #define   VCR_CI_CCIR601_16 0x00000020  /* ... CCIR601, 16 bit */
 #define   VCR_CI_CCIR656_16 0x00000030  /* ... CCIR656, 16 bit */
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index dca528a316e7..2e9981b18166 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -22,6 +22,11 @@
 #include <media/media-device.h>
 #include <media/v4l2-device.h>
 
+/* VIMC-specific controls */
+#define VIMC_CID_VIMC_BASE		(0x00f00000 | 0xf000)
+#define VIMC_CID_VIMC_CLASS		(0x00f00000 | 1)
+#define VIMC_CID_TEST_PATTERN		(VIMC_CID_VIMC_BASE + 0)
+
 #define VIMC_FRAME_MAX_WIDTH 4096
 #define VIMC_FRAME_MAX_HEIGHT 2160
 #define VIMC_FRAME_MIN_WIDTH 16
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index 02e68c8fc02b..457e211514c6 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -22,8 +22,9 @@
 #include <linux/platform_device.h>
 #include <linux/v4l2-mediabus.h>
 #include <linux/vmalloc.h>
+#include <media/v4l2-ctrls.h>
 #include <media/v4l2-subdev.h>
-#include <media/v4l2-tpg.h>
+#include <media/tpg/v4l2-tpg.h>
 
 #include "vimc-common.h"
 
@@ -38,6 +39,7 @@ struct vimc_sen_device {
 	u8 *frame;
 	/* The active format */
 	struct v4l2_mbus_framefmt mbus_format;
+	struct v4l2_ctrl_handler hdl;
 };
 
 static const struct v4l2_mbus_framefmt fmt_default = {
@@ -291,6 +293,31 @@ static const struct v4l2_subdev_ops vimc_sen_ops = {
 	.video = &vimc_sen_video_ops,
 };
 
+static int vimc_sen_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct vimc_sen_device *vsen =
+		container_of(ctrl->handler, struct vimc_sen_device, hdl);
+
+	switch (ctrl->id) {
+	case VIMC_CID_TEST_PATTERN:
+		tpg_s_pattern(&vsen->tpg, ctrl->val);
+		break;
+	case V4L2_CID_HFLIP:
+		tpg_s_hflip(&vsen->tpg, ctrl->val);
+		break;
+	case V4L2_CID_VFLIP:
+		tpg_s_vflip(&vsen->tpg, ctrl->val);
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops vimc_sen_ctrl_ops = {
+	.s_ctrl = vimc_sen_s_ctrl,
+};
+
 static void vimc_sen_comp_unbind(struct device *comp, struct device *master,
 				 void *master_data)
 {
@@ -299,10 +326,28 @@ static void vimc_sen_comp_unbind(struct device *comp, struct device *master,
 				container_of(ved, struct vimc_sen_device, ved);
 
 	vimc_ent_sd_unregister(ved, &vsen->sd);
+	v4l2_ctrl_handler_free(&vsen->hdl);
 	tpg_free(&vsen->tpg);
 	kfree(vsen);
 }
 
+/* Image Processing Controls */
+static const struct v4l2_ctrl_config vimc_sen_ctrl_class = {
+	.flags = V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_WRITE_ONLY,
+	.id = VIMC_CID_VIMC_CLASS,
+	.name = "VIMC Controls",
+	.type = V4L2_CTRL_TYPE_CTRL_CLASS,
+};
+
+static const struct v4l2_ctrl_config vimc_sen_ctrl_test_pattern = {
+	.ops = &vimc_sen_ctrl_ops,
+	.id = VIMC_CID_TEST_PATTERN,
+	.name = "Test Pattern",
+	.type = V4L2_CTRL_TYPE_MENU,
+	.max = TPG_PAT_NOISE,
+	.qmenu = tpg_pattern_strings,
+};
+
 static int vimc_sen_comp_bind(struct device *comp, struct device *master,
 			      void *master_data)
 {
@@ -316,6 +361,20 @@ static int vimc_sen_comp_bind(struct device *comp, struct device *master,
 	if (!vsen)
 		return -ENOMEM;
 
+	v4l2_ctrl_handler_init(&vsen->hdl, 4);
+
+	v4l2_ctrl_new_custom(&vsen->hdl, &vimc_sen_ctrl_class, NULL);
+	v4l2_ctrl_new_custom(&vsen->hdl, &vimc_sen_ctrl_test_pattern, NULL);
+	v4l2_ctrl_new_std(&vsen->hdl, &vimc_sen_ctrl_ops,
+			  V4L2_CID_VFLIP, 0, 1, 1, 0);
+	v4l2_ctrl_new_std(&vsen->hdl, &vimc_sen_ctrl_ops,
+			  V4L2_CID_HFLIP, 0, 1, 1, 0);
+	vsen->sd.ctrl_handler = &vsen->hdl;
+	if (vsen->hdl.error) {
+		ret = vsen->hdl.error;
+		goto err_free_vsen;
+	}
+
 	/* Initialize ved and sd */
 	ret = vimc_ent_sd_register(&vsen->ved, &vsen->sd, v4l2_dev,
 				   pdata->entity_name,
@@ -323,7 +382,7 @@ static int vimc_sen_comp_bind(struct device *comp, struct device *master,
 				   (const unsigned long[1]) {MEDIA_PAD_FL_SOURCE},
 				   &vimc_sen_ops);
 	if (ret)
-		goto err_free_vsen;
+		goto err_free_hdl;
 
 	dev_set_drvdata(comp, &vsen->ved);
 	vsen->dev = comp;
@@ -342,6 +401,8 @@ static int vimc_sen_comp_bind(struct device *comp, struct device *master,
 
 err_unregister_ent_sd:
 	vimc_ent_sd_unregister(&vsen->ved,  &vsen->sd);
+err_free_hdl:
+	v4l2_ctrl_handler_free(&vsen->hdl);
 err_free_vsen:
 	kfree(vsen);
 
diff --git a/drivers/media/platform/vivid/vivid-core.c b/drivers/media/platform/vivid/vivid-core.c
index 5f316a5e38db..9f036c5f51b0 100644
--- a/drivers/media/platform/vivid/vivid-core.c
+++ b/drivers/media/platform/vivid/vivid-core.c
@@ -416,7 +416,7 @@ static ssize_t vivid_radio_write(struct file *file, const char __user *buf,
 	return vivid_radio_tx_write(file, buf, size, offset);
 }
 
-static unsigned int vivid_radio_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t vivid_radio_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct video_device *vdev = video_devdata(file);
 
@@ -995,7 +995,7 @@ static int vivid_create_instance(struct platform_device *pdev, int inst)
 
 	dev->edid_max_blocks = dev->edid_blocks = 2;
 	memcpy(dev->edid, vivid_hdmi_edid, sizeof(vivid_hdmi_edid));
-	ktime_get_ts(&dev->radio_rds_init_ts);
+	dev->radio_rds_init_time = ktime_get();
 
 	/* create all controls */
 	ret = vivid_create_controls(dev, ccs_cap == -1, ccs_out == -1, no_error_inj,
diff --git a/drivers/media/platform/vivid/vivid-core.h b/drivers/media/platform/vivid/vivid-core.h
index 5cdf95bdc4d1..c90e4a0ab94e 100644
--- a/drivers/media/platform/vivid/vivid-core.h
+++ b/drivers/media/platform/vivid/vivid-core.h
@@ -27,7 +27,7 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-dev.h>
 #include <media/v4l2-ctrls.h>
-#include <media/v4l2-tpg.h>
+#include <media/tpg/v4l2-tpg.h>
 #include "vivid-rds-gen.h"
 #include "vivid-vbi-gen.h"
 
@@ -154,6 +154,7 @@ struct vivid_dev {
 	struct v4l2_ctrl_handler	ctrl_hdl_streaming;
 	struct v4l2_ctrl_handler	ctrl_hdl_sdtv_cap;
 	struct v4l2_ctrl_handler	ctrl_hdl_loop_cap;
+	struct v4l2_ctrl_handler	ctrl_hdl_fb;
 	struct video_device		vid_cap_dev;
 	struct v4l2_ctrl_handler	ctrl_hdl_vid_cap;
 	struct video_device		vid_out_dev;
@@ -510,7 +511,7 @@ struct vivid_dev {
 
 	/* Shared between radio receiver and transmitter */
 	bool				radio_rds_loop;
-	struct timespec			radio_rds_init_ts;
+	ktime_t				radio_rds_init_time;
 
 	/* CEC */
 	struct cec_adapter		*cec_rx_adap;
diff --git a/drivers/media/platform/vivid/vivid-ctrls.c b/drivers/media/platform/vivid/vivid-ctrls.c
index 34731f71cc00..3f9d354827af 100644
--- a/drivers/media/platform/vivid/vivid-ctrls.c
+++ b/drivers/media/platform/vivid/vivid-ctrls.c
@@ -120,9 +120,6 @@ static int vivid_user_gen_s_ctrl(struct v4l2_ctrl *ctrl)
 		clear_bit(V4L2_FL_REGISTERED, &dev->radio_rx_dev.flags);
 		clear_bit(V4L2_FL_REGISTERED, &dev->radio_tx_dev.flags);
 		break;
-	case VIVID_CID_CLEAR_FB:
-		vivid_clear_fb(dev);
-		break;
 	case VIVID_CID_BUTTON:
 		dev->button_pressed = 30;
 		break;
@@ -274,8 +271,28 @@ static const struct v4l2_ctrl_config vivid_ctrl_disconnect = {
 	.type = V4L2_CTRL_TYPE_BUTTON,
 };
 
+
+/* Framebuffer Controls */
+
+static int vivid_fb_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct vivid_dev *dev = container_of(ctrl->handler,
+					     struct vivid_dev, ctrl_hdl_fb);
+
+	switch (ctrl->id) {
+	case VIVID_CID_CLEAR_FB:
+		vivid_clear_fb(dev);
+		break;
+	}
+	return 0;
+}
+
+static const struct v4l2_ctrl_ops vivid_fb_ctrl_ops = {
+	.s_ctrl = vivid_fb_s_ctrl,
+};
+
 static const struct v4l2_ctrl_config vivid_ctrl_clear_fb = {
-	.ops = &vivid_user_gen_ctrl_ops,
+	.ops = &vivid_fb_ctrl_ops,
 	.id = VIVID_CID_CLEAR_FB,
 	.name = "Clear Framebuffer",
 	.type = V4L2_CTRL_TYPE_BUTTON,
@@ -1357,6 +1374,7 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap,
 	struct v4l2_ctrl_handler *hdl_streaming = &dev->ctrl_hdl_streaming;
 	struct v4l2_ctrl_handler *hdl_sdtv_cap = &dev->ctrl_hdl_sdtv_cap;
 	struct v4l2_ctrl_handler *hdl_loop_cap = &dev->ctrl_hdl_loop_cap;
+	struct v4l2_ctrl_handler *hdl_fb = &dev->ctrl_hdl_fb;
 	struct v4l2_ctrl_handler *hdl_vid_cap = &dev->ctrl_hdl_vid_cap;
 	struct v4l2_ctrl_handler *hdl_vid_out = &dev->ctrl_hdl_vid_out;
 	struct v4l2_ctrl_handler *hdl_vbi_cap = &dev->ctrl_hdl_vbi_cap;
@@ -1384,10 +1402,12 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap,
 	v4l2_ctrl_new_custom(hdl_sdtv_cap, &vivid_ctrl_class, NULL);
 	v4l2_ctrl_handler_init(hdl_loop_cap, 1);
 	v4l2_ctrl_new_custom(hdl_loop_cap, &vivid_ctrl_class, NULL);
+	v4l2_ctrl_handler_init(hdl_fb, 1);
+	v4l2_ctrl_new_custom(hdl_fb, &vivid_ctrl_class, NULL);
 	v4l2_ctrl_handler_init(hdl_vid_cap, 55);
 	v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_class, NULL);
 	v4l2_ctrl_handler_init(hdl_vid_out, 26);
-	if (!no_error_inj)
+	if (!no_error_inj || dev->has_fb)
 		v4l2_ctrl_new_custom(hdl_vid_out, &vivid_ctrl_class, NULL);
 	v4l2_ctrl_handler_init(hdl_vbi_cap, 21);
 	v4l2_ctrl_new_custom(hdl_vbi_cap, &vivid_ctrl_class, NULL);
@@ -1561,7 +1581,7 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap,
 		v4l2_ctrl_new_custom(hdl_loop_cap, &vivid_ctrl_loop_video, NULL);
 
 	if (dev->has_fb)
-		v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_clear_fb, NULL);
+		v4l2_ctrl_new_custom(hdl_fb, &vivid_ctrl_clear_fb, NULL);
 
 	if (dev->has_radio_rx) {
 		v4l2_ctrl_new_custom(hdl_radio_rx, &vivid_ctrl_radio_hw_seek_mode, NULL);
@@ -1658,6 +1678,7 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap,
 		v4l2_ctrl_add_handler(hdl_vid_cap, hdl_streaming, NULL);
 		v4l2_ctrl_add_handler(hdl_vid_cap, hdl_sdtv_cap, NULL);
 		v4l2_ctrl_add_handler(hdl_vid_cap, hdl_loop_cap, NULL);
+		v4l2_ctrl_add_handler(hdl_vid_cap, hdl_fb, NULL);
 		if (hdl_vid_cap->error)
 			return hdl_vid_cap->error;
 		dev->vid_cap_dev.ctrl_handler = hdl_vid_cap;
@@ -1666,6 +1687,7 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap,
 		v4l2_ctrl_add_handler(hdl_vid_out, hdl_user_gen, NULL);
 		v4l2_ctrl_add_handler(hdl_vid_out, hdl_user_aud, NULL);
 		v4l2_ctrl_add_handler(hdl_vid_out, hdl_streaming, NULL);
+		v4l2_ctrl_add_handler(hdl_vid_out, hdl_fb, NULL);
 		if (hdl_vid_out->error)
 			return hdl_vid_out->error;
 		dev->vid_out_dev.ctrl_handler = hdl_vid_out;
@@ -1725,4 +1747,5 @@ void vivid_free_controls(struct vivid_dev *dev)
 	v4l2_ctrl_handler_free(&dev->ctrl_hdl_streaming);
 	v4l2_ctrl_handler_free(&dev->ctrl_hdl_sdtv_cap);
 	v4l2_ctrl_handler_free(&dev->ctrl_hdl_loop_cap);
+	v4l2_ctrl_handler_free(&dev->ctrl_hdl_fb);
 }
diff --git a/drivers/media/platform/vivid/vivid-radio-rx.c b/drivers/media/platform/vivid/vivid-radio-rx.c
index 47c36c26096b..f834f7df8cf9 100644
--- a/drivers/media/platform/vivid/vivid-radio-rx.c
+++ b/drivers/media/platform/vivid/vivid-radio-rx.c
@@ -38,9 +38,9 @@ ssize_t vivid_radio_rx_read(struct file *file, char __user *buf,
 			 size_t size, loff_t *offset)
 {
 	struct vivid_dev *dev = video_drvdata(file);
-	struct timespec ts;
 	struct v4l2_rds_data *data = dev->rds_gen.data;
 	bool use_alternates;
+	ktime_t timestamp;
 	unsigned blk;
 	int perc;
 	int i;
@@ -64,17 +64,16 @@ ssize_t vivid_radio_rx_read(struct file *file, char __user *buf,
 	}
 
 retry:
-	ktime_get_ts(&ts);
-	use_alternates = ts.tv_sec % 10 >= 5;
+	timestamp = ktime_sub(ktime_get(), dev->radio_rds_init_time);
+	blk = ktime_divns(timestamp, VIVID_RDS_NSEC_PER_BLK);
+	use_alternates = (blk % VIVID_RDS_GEN_BLOCKS) & 1;
+
 	if (dev->radio_rx_rds_last_block == 0 ||
 	    dev->radio_rx_rds_use_alternates != use_alternates) {
 		dev->radio_rx_rds_use_alternates = use_alternates;
 		/* Re-init the RDS generator */
 		vivid_radio_rds_init(dev);
 	}
-	ts = timespec_sub(ts, dev->radio_rds_init_ts);
-	blk = ts.tv_sec * 100 + ts.tv_nsec / 10000000;
-	blk = (blk * VIVID_RDS_GEN_BLOCKS) / 500;
 	if (blk >= dev->radio_rx_rds_last_block + VIVID_RDS_GEN_BLOCKS)
 		dev->radio_rx_rds_last_block = blk - VIVID_RDS_GEN_BLOCKS + 1;
 
@@ -141,9 +140,9 @@ ssize_t vivid_radio_rx_read(struct file *file, char __user *buf,
 	return i;
 }
 
-unsigned int vivid_radio_rx_poll(struct file *file, struct poll_table_struct *wait)
+__poll_t vivid_radio_rx_poll(struct file *file, struct poll_table_struct *wait)
 {
-	return POLLIN | POLLRDNORM | v4l2_ctrl_poll(file, wait);
+	return EPOLLIN | EPOLLRDNORM | v4l2_ctrl_poll(file, wait);
 }
 
 int vivid_radio_rx_enum_freq_bands(struct file *file, void *fh, struct v4l2_frequency_band *band)
diff --git a/drivers/media/platform/vivid/vivid-radio-rx.h b/drivers/media/platform/vivid/vivid-radio-rx.h
index 1077d8f061eb..2b33edb60942 100644
--- a/drivers/media/platform/vivid/vivid-radio-rx.h
+++ b/drivers/media/platform/vivid/vivid-radio-rx.h
@@ -21,7 +21,7 @@
 #define _VIVID_RADIO_RX_H_
 
 ssize_t vivid_radio_rx_read(struct file *, char __user *, size_t, loff_t *);
-unsigned int vivid_radio_rx_poll(struct file *file, struct poll_table_struct *wait);
+__poll_t vivid_radio_rx_poll(struct file *file, struct poll_table_struct *wait);
 
 int vivid_radio_rx_enum_freq_bands(struct file *file, void *fh, struct v4l2_frequency_band *band);
 int vivid_radio_rx_s_hw_freq_seek(struct file *file, void *fh, const struct v4l2_hw_freq_seek *a);
diff --git a/drivers/media/platform/vivid/vivid-radio-tx.c b/drivers/media/platform/vivid/vivid-radio-tx.c
index 0e8025b7b4dd..308b13f85dc0 100644
--- a/drivers/media/platform/vivid/vivid-radio-tx.c
+++ b/drivers/media/platform/vivid/vivid-radio-tx.c
@@ -37,7 +37,7 @@ ssize_t vivid_radio_tx_write(struct file *file, const char __user *buf,
 {
 	struct vivid_dev *dev = video_drvdata(file);
 	struct v4l2_rds_data *data = dev->rds_gen.data;
-	struct timespec ts;
+	ktime_t timestamp;
 	unsigned blk;
 	int i;
 
@@ -58,10 +58,8 @@ ssize_t vivid_radio_tx_write(struct file *file, const char __user *buf,
 	dev->radio_tx_rds_owner = file->private_data;
 
 retry:
-	ktime_get_ts(&ts);
-	ts = timespec_sub(ts, dev->radio_rds_init_ts);
-	blk = ts.tv_sec * 100 + ts.tv_nsec / 10000000;
-	blk = (blk * VIVID_RDS_GEN_BLOCKS) / 500;
+	timestamp = ktime_sub(ktime_get(), dev->radio_rds_init_time);
+	blk = ktime_divns(timestamp, VIVID_RDS_NSEC_PER_BLK);
 	if (blk - VIVID_RDS_GEN_BLOCKS >= dev->radio_tx_rds_last_block)
 		dev->radio_tx_rds_last_block = blk - VIVID_RDS_GEN_BLOCKS + 1;
 
@@ -105,9 +103,9 @@ ssize_t vivid_radio_tx_write(struct file *file, const char __user *buf,
 	return i;
 }
 
-unsigned int vivid_radio_tx_poll(struct file *file, struct poll_table_struct *wait)
+__poll_t vivid_radio_tx_poll(struct file *file, struct poll_table_struct *wait)
 {
-	return POLLOUT | POLLWRNORM | v4l2_ctrl_poll(file, wait);
+	return EPOLLOUT | EPOLLWRNORM | v4l2_ctrl_poll(file, wait);
 }
 
 int vidioc_g_modulator(struct file *file, void *fh, struct v4l2_modulator *a)
diff --git a/drivers/media/platform/vivid/vivid-radio-tx.h b/drivers/media/platform/vivid/vivid-radio-tx.h
index 7f8ff7547119..3c3343d70cbc 100644
--- a/drivers/media/platform/vivid/vivid-radio-tx.h
+++ b/drivers/media/platform/vivid/vivid-radio-tx.h
@@ -21,7 +21,7 @@
 #define _VIVID_RADIO_TX_H_
 
 ssize_t vivid_radio_tx_write(struct file *, const char __user *, size_t, loff_t *);
-unsigned int vivid_radio_tx_poll(struct file *file, struct poll_table_struct *wait);
+__poll_t vivid_radio_tx_poll(struct file *file, struct poll_table_struct *wait);
 
 int vidioc_g_modulator(struct file *file, void *fh, struct v4l2_modulator *a);
 int vidioc_s_modulator(struct file *file, void *fh, const struct v4l2_modulator *a);
diff --git a/drivers/media/platform/vivid/vivid-rds-gen.c b/drivers/media/platform/vivid/vivid-rds-gen.c
index 53c7777dc001..996e35e28d37 100644
--- a/drivers/media/platform/vivid/vivid-rds-gen.c
+++ b/drivers/media/platform/vivid/vivid-rds-gen.c
@@ -103,7 +103,7 @@ void vivid_rds_generate(struct vivid_rds_gen *rds)
 			 * EN 50067:1998 to convert a UTC date to an RDS Modified
 			 * Julian Day.
 			 */
-			time_to_tm(get_seconds(), 0, &tm);
+			time64_to_tm(ktime_get_real_seconds(), 0, &tm);
 			l = tm.tm_mon <= 1;
 			date = 14956 + tm.tm_mday + ((tm.tm_year - l) * 1461) / 4 +
 				((tm.tm_mon + 2 + l * 12) * 306001) / 10000;
diff --git a/drivers/media/platform/vivid/vivid-rds-gen.h b/drivers/media/platform/vivid/vivid-rds-gen.h
index eff4bf552ed3..e55e3b22b7ca 100644
--- a/drivers/media/platform/vivid/vivid-rds-gen.h
+++ b/drivers/media/platform/vivid/vivid-rds-gen.h
@@ -29,6 +29,7 @@
 #define VIVID_RDS_GEN_GROUPS 57
 #define VIVID_RDS_GEN_BLKS_PER_GRP 4
 #define VIVID_RDS_GEN_BLOCKS (VIVID_RDS_GEN_BLKS_PER_GRP * VIVID_RDS_GEN_GROUPS)
+#define VIVID_RDS_NSEC_PER_BLK (u32)(5ull * NSEC_PER_SEC / VIVID_RDS_GEN_BLOCKS)
 
 struct vivid_rds_gen {
 	struct v4l2_rds_data	data[VIVID_RDS_GEN_BLOCKS];
diff --git a/drivers/media/platform/vivid/vivid-vbi-gen.c b/drivers/media/platform/vivid/vivid-vbi-gen.c
index a2159de83d0b..02c79d7cedab 100644
--- a/drivers/media/platform/vivid/vivid-vbi-gen.c
+++ b/drivers/media/platform/vivid/vivid-vbi-gen.c
@@ -190,7 +190,7 @@ static void vivid_vbi_gen_set_time_of_day(u8 *packet)
 	struct tm tm;
 	u8 checksum, i;
 
-	time_to_tm(get_seconds(), 0, &tm);
+	time64_to_tm(ktime_get_real_seconds(), 0, &tm);
 	packet[0] = calc_parity(0x07);
 	packet[1] = calc_parity(0x01);
 	packet[2] = calc_parity(0x40 | tm.tm_min);
diff --git a/drivers/media/platform/vivid/vivid-vid-cap.c b/drivers/media/platform/vivid/vivid-vid-cap.c
index 01419455e545..0fbbcde19f0d 100644
--- a/drivers/media/platform/vivid/vivid-vid-cap.c
+++ b/drivers/media/platform/vivid/vivid-vid-cap.c
@@ -93,11 +93,6 @@ static const struct v4l2_fract webcam_intervals[VIVID_WEBCAM_IVALS] = {
 	{  1, 60 },
 };
 
-static const struct v4l2_discrete_probe webcam_probe = {
-	webcam_sizes,
-	VIVID_WEBCAM_SIZES
-};
-
 static int vid_cap_queue_setup(struct vb2_queue *vq,
 		       unsigned *nbuffers, unsigned *nplanes,
 		       unsigned sizes[], struct device *alloc_devs[])
@@ -578,7 +573,9 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv,
 	mp->field = vivid_field_cap(dev, mp->field);
 	if (vivid_is_webcam(dev)) {
 		const struct v4l2_frmsize_discrete *sz =
-			v4l2_find_nearest_format(&webcam_probe, mp->width, mp->height);
+			v4l2_find_nearest_format(webcam_sizes,
+						 VIVID_WEBCAM_SIZES,
+						 mp->width, mp->height);
 
 		w = sz->width;
 		h = sz->height;
diff --git a/drivers/media/platform/vsp1/vsp1_drm.c b/drivers/media/platform/vsp1/vsp1_drm.c
index 4dfbeac8f42c..7ce69f23f50a 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/vsp1/vsp1_drm.c
@@ -84,8 +84,12 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 	struct vsp1_drm_pipeline *drm_pipe;
 	struct vsp1_pipeline *pipe;
 	struct vsp1_bru *bru;
+	struct vsp1_entity *entity;
+	struct vsp1_entity *next;
+	struct vsp1_dl_list *dl;
 	struct v4l2_subdev_format format;
 	const char *bru_name;
+	unsigned long flags;
 	unsigned int i;
 	int ret;
 
@@ -250,6 +254,29 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 	vsp1_write(vsp1, VI6_DISP_IRQ_STA, 0);
 	vsp1_write(vsp1, VI6_DISP_IRQ_ENB, 0);
 
+	/* Configure all entities in the pipeline. */
+	dl = vsp1_dl_list_get(pipe->output->dlm);
+
+	list_for_each_entry_safe(entity, next, &pipe->entities, list_pipe) {
+		vsp1_entity_route_setup(entity, pipe, dl);
+
+		if (entity->ops->configure) {
+			entity->ops->configure(entity, pipe, dl,
+					       VSP1_ENTITY_PARAMS_INIT);
+			entity->ops->configure(entity, pipe, dl,
+					       VSP1_ENTITY_PARAMS_RUNTIME);
+			entity->ops->configure(entity, pipe, dl,
+					       VSP1_ENTITY_PARAMS_PARTITION);
+		}
+	}
+
+	vsp1_dl_list_commit(dl);
+
+	/* Start the pipeline. */
+	spin_lock_irqsave(&pipe->irqlock, flags);
+	vsp1_pipeline_run(pipe);
+	spin_unlock_irqrestore(&pipe->irqlock, flags);
+
 	dev_dbg(vsp1->dev, "%s: pipeline enabled\n", __func__);
 
 	return 0;
@@ -488,7 +515,6 @@ void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index)
 	struct vsp1_entity *next;
 	struct vsp1_dl_list *dl;
 	const char *bru_name;
-	unsigned long flags;
 	unsigned int i;
 	int ret;
 
@@ -570,15 +596,6 @@ void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index)
 	}
 
 	vsp1_dl_list_commit(dl);
-
-	/* Start or stop the pipeline if needed. */
-	if (!drm_pipe->enabled && pipe->num_inputs) {
-		spin_lock_irqsave(&pipe->irqlock, flags);
-		vsp1_pipeline_run(pipe);
-		spin_unlock_irqrestore(&pipe->irqlock, flags);
-	} else if (drm_pipe->enabled && !pipe->num_inputs) {
-		vsp1_pipeline_stop(pipe);
-	}
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_flush);
 
diff --git a/drivers/media/platform/vsp1/vsp1_drv.c b/drivers/media/platform/vsp1/vsp1_drv.c
index 962e4c304076..eed9516e25e1 100644
--- a/drivers/media/platform/vsp1/vsp1_drv.c
+++ b/drivers/media/platform/vsp1/vsp1_drv.c
@@ -571,7 +571,13 @@ static int __maybe_unused vsp1_pm_suspend(struct device *dev)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
 
-	vsp1_pipelines_suspend(vsp1);
+	/*
+	 * When used as part of a display pipeline, the VSP is stopped and
+	 * restarted explicitly by the DU.
+	 */
+	if (!vsp1->drm)
+		vsp1_pipelines_suspend(vsp1);
+
 	pm_runtime_force_suspend(vsp1->dev);
 
 	return 0;
@@ -582,7 +588,13 @@ static int __maybe_unused vsp1_pm_resume(struct device *dev)
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
 
 	pm_runtime_force_resume(vsp1->dev);
-	vsp1_pipelines_resume(vsp1);
+
+	/*
+	 * When used as part of a display pipeline, the VSP is stopped and
+	 * restarted explicitly by the DU.
+	 */
+	if (!vsp1->drm)
+		vsp1_pipelines_resume(vsp1);
 
 	return 0;
 }
diff --git a/drivers/media/platform/xilinx/xilinx-vipp.c b/drivers/media/platform/xilinx/xilinx-vipp.c
index d881cf09876d..6bb28cd49dae 100644
--- a/drivers/media/platform/xilinx/xilinx-vipp.c
+++ b/drivers/media/platform/xilinx/xilinx-vipp.c
@@ -76,20 +76,16 @@ static int xvip_graph_build_one(struct xvip_composite_device *xdev,
 	struct xvip_graph_entity *ent;
 	struct v4l2_fwnode_link link;
 	struct device_node *ep = NULL;
-	struct device_node *next;
 	int ret = 0;
 
 	dev_dbg(xdev->dev, "creating links for entity %s\n", local->name);
 
 	while (1) {
 		/* Get the next endpoint and parse its link. */
-		next = of_graph_get_next_endpoint(entity->node, ep);
-		if (next == NULL)
+		ep = of_graph_get_next_endpoint(entity->node, ep);
+		if (ep == NULL)
 			break;
 
-		of_node_put(ep);
-		ep = next;
-
 		dev_dbg(xdev->dev, "processing endpoint %pOF\n", ep);
 
 		ret = v4l2_fwnode_parse_link(of_fwnode_handle(ep), &link);
@@ -200,7 +196,6 @@ static int xvip_graph_build_dma(struct xvip_composite_device *xdev)
 	struct xvip_graph_entity *ent;
 	struct v4l2_fwnode_link link;
 	struct device_node *ep = NULL;
-	struct device_node *next;
 	struct xvip_dma *dma;
 	int ret = 0;
 
@@ -208,13 +203,10 @@ static int xvip_graph_build_dma(struct xvip_composite_device *xdev)
 
 	while (1) {
 		/* Get the next endpoint and parse its link. */
-		next = of_graph_get_next_endpoint(node, ep);
-		if (next == NULL)
+		ep = of_graph_get_next_endpoint(node, ep);
+		if (ep == NULL)
 			break;
 
-		of_node_put(ep);
-		ep = next;
-
 		dev_dbg(xdev->dev, "processing endpoint %pOF\n", ep);
 
 		ret = v4l2_fwnode_parse_link(of_fwnode_handle(ep), &link);
@@ -395,7 +387,7 @@ static int xvip_graph_parse_one(struct xvip_composite_device *xdev,
 
 		entity->node = remote;
 		entity->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
-		entity->asd.match.fwnode.fwnode = of_fwnode_handle(remote);
+		entity->asd.match.fwnode = of_fwnode_handle(remote);
 		list_add_tail(&entity->list, &xdev->entities);
 		xdev->num_subdevs++;
 	}
diff --git a/drivers/media/radio/radio-aimslab.c b/drivers/media/radio/radio-aimslab.c
index ea9308796741..5ef635e72e10 100644
--- a/drivers/media/radio/radio-aimslab.c
+++ b/drivers/media/radio/radio-aimslab.c
@@ -26,7 +26,7 @@
  * Fully tested with the Keene USB FM Transmitter and the v4l2-compliance tool.
  */
 
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/ioport.h>	/* request_region		*/
 #include <linux/delay.h>	/* msleep			*/
diff --git a/drivers/media/radio/radio-aztech.c b/drivers/media/radio/radio-aztech.c
index f445327f282d..9e12c6027359 100644
--- a/drivers/media/radio/radio-aztech.c
+++ b/drivers/media/radio/radio-aztech.c
@@ -15,7 +15,7 @@
  * Fully tested with the Keene USB FM Transmitter and the v4l2-compliance tool.
 */
 
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/ioport.h>	/* request_region		*/
 #include <linux/delay.h>	/* udelay			*/
diff --git a/drivers/media/radio/radio-cadet.c b/drivers/media/radio/radio-cadet.c
index 7575e5370a49..5b82e63885cd 100644
--- a/drivers/media/radio/radio-cadet.c
+++ b/drivers/media/radio/radio-cadet.c
@@ -30,7 +30,7 @@
  *		Changed API to V4L2
  */
 
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/ioport.h>	/* request_region		*/
 #include <linux/delay.h>	/* udelay			*/
@@ -481,21 +481,21 @@ static int cadet_release(struct file *file)
 	return 0;
 }
 
-static unsigned int cadet_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t cadet_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct cadet *dev = video_drvdata(file);
-	unsigned long req_events = poll_requested_events(wait);
-	unsigned int res = v4l2_ctrl_poll(file, wait);
+	__poll_t req_events = poll_requested_events(wait);
+	__poll_t res = v4l2_ctrl_poll(file, wait);
 
 	poll_wait(file, &dev->read_queue, wait);
-	if (dev->rdsstat == 0 && (req_events & (POLLIN | POLLRDNORM))) {
+	if (dev->rdsstat == 0 && (req_events & (EPOLLIN | EPOLLRDNORM))) {
 		mutex_lock(&dev->lock);
 		if (dev->rdsstat == 0)
 			cadet_start_rds(dev);
 		mutex_unlock(&dev->lock);
 	}
 	if (cadet_has_rds_data(dev))
-		res |= POLLIN | POLLRDNORM;
+		res |= EPOLLIN | EPOLLRDNORM;
 	return res;
 }
 
@@ -503,7 +503,7 @@ static unsigned int cadet_poll(struct file *file, struct poll_table_struct *wait
 static const struct v4l2_file_operations cadet_fops = {
 	.owner		= THIS_MODULE,
 	.open		= cadet_open,
-	.release       	= cadet_release,
+	.release	= cadet_release,
 	.read		= cadet_read,
 	.unlocked_ioctl	= video_ioctl2,
 	.poll		= cadet_poll,
diff --git a/drivers/media/radio/radio-gemtek.c b/drivers/media/radio/radio-gemtek.c
index ddc12b16f77c..3ff4c4e1435f 100644
--- a/drivers/media/radio/radio-gemtek.c
+++ b/drivers/media/radio/radio-gemtek.c
@@ -22,7 +22,7 @@
  * Fully tested with the Keene USB FM Transmitter and the v4l2-compliance tool.
  */
 
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/ioport.h>	/* request_region		*/
 #include <linux/delay.h>	/* udelay			*/
@@ -102,9 +102,9 @@ struct gemtek {
 	u32 bu2614data;
 };
 
-#define BU2614_FREQ_BITS 	16 /* D0..D15, Frequency data		*/
+#define BU2614_FREQ_BITS	16 /* D0..D15, Frequency data		*/
 #define BU2614_PORT_BITS	3 /* P0..P2, Output port control data	*/
-#define BU2614_VOID_BITS	4 /* unused 				*/
+#define BU2614_VOID_BITS	4 /* unused				*/
 #define BU2614_FMES_BITS	1 /* CT, Frequency measurement beginning data */
 #define BU2614_STDF_BITS	3 /* R0..R2, Standard frequency data	*/
 #define BU2614_SWIN_BITS	1 /* S, Switch between FMIN / AMIN	*/
@@ -113,7 +113,7 @@ struct gemtek {
 #define BU2614_FMUN_BITS	1 /* GT, Frequency measurement time & unlock */
 #define BU2614_TEST_BITS	1 /* TS, Test data is input		*/
 
-#define BU2614_FREQ_SHIFT 	0
+#define BU2614_FREQ_SHIFT	0
 #define BU2614_PORT_SHIFT	(BU2614_FREQ_BITS + BU2614_FREQ_SHIFT)
 #define BU2614_VOID_SHIFT	(BU2614_PORT_BITS + BU2614_PORT_SHIFT)
 #define BU2614_FMES_SHIFT	(BU2614_VOID_BITS + BU2614_VOID_SHIFT)
diff --git a/drivers/media/radio/radio-maxiradio.c b/drivers/media/radio/radio-maxiradio.c
index 3aa5ad391581..95f06f3b35dc 100644
--- a/drivers/media/radio/radio-maxiradio.c
+++ b/drivers/media/radio/radio-maxiradio.c
@@ -13,7 +13,7 @@
  * anybody does please mail me.
  *
  * For the pdf file see:
- * http://www.nxp.com/acrobat_download2/expired_datasheets/TEA5757_5759_3.pdf 
+ * http://www.nxp.com/acrobat_download2/expired_datasheets/TEA5757_5759_3.pdf
  *
  *
  * CHANGES:
diff --git a/drivers/media/radio/radio-mr800.c b/drivers/media/radio/radio-mr800.c
index c9f59129af79..dc6c4f985911 100644
--- a/drivers/media/radio/radio-mr800.c
+++ b/drivers/media/radio/radio-mr800.c
@@ -31,22 +31,22 @@
  * http://www.spinics.net/lists/linux-usb-devel/msg10109.html
  *
  * Version 0.01:	First working version.
- * 			It's required to blacklist AverMedia USB Radio
- * 			in usbhid/hid-quirks.c
+ *			It's required to blacklist AverMedia USB Radio
+ *			in usbhid/hid-quirks.c
  * Version 0.10:	A lot of cleanups and fixes: unpluging the device,
- * 			few mutex locks were added, codinstyle issues, etc.
- * 			Added stereo support. Thanks to
- * 			Douglas Schilling Landgraf <dougsland@gmail.com> and
- * 			David Ellingsworth <david@identd.dyndns.org>
- * 			for discussion, help and support.
+ *			few mutex locks were added, codinstyle issues, etc.
+ *			Added stereo support. Thanks to
+ *			Douglas Schilling Landgraf <dougsland@gmail.com> and
+ *			David Ellingsworth <david@identd.dyndns.org>
+ *			for discussion, help and support.
  * Version 0.11:	Converted to v4l2_device.
  *
  * Many things to do:
- * 	- Correct power management of device (suspend & resume)
- * 	- Add code for scanning and smooth tuning
- * 	- Add code for sensitivity value
- * 	- Correct mistakes
- * 	- In Japan another FREQ_MIN and FREQ_MAX
+ *	- Correct power management of device (suspend & resume)
+ *	- Add code for scanning and smooth tuning
+ *	- Add code for sensitivity value
+ *	- Correct mistakes
+ *	- In Japan another FREQ_MIN and FREQ_MAX
  */
 
 /* kernel includes */
diff --git a/drivers/media/radio/radio-rtrack2.c b/drivers/media/radio/radio-rtrack2.c
index 09cfbc373c92..abeaedd8d437 100644
--- a/drivers/media/radio/radio-rtrack2.c
+++ b/drivers/media/radio/radio-rtrack2.c
@@ -12,7 +12,7 @@
  * Fully tested with actual hardware and the v4l2-compliance tool.
  */
 
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/ioport.h>	/* request_region		*/
 #include <linux/delay.h>	/* udelay			*/
diff --git a/drivers/media/radio/radio-sf16fmi.c b/drivers/media/radio/radio-sf16fmi.c
index 28a89466cddc..fc4e63d36e4c 100644
--- a/drivers/media/radio/radio-sf16fmi.c
+++ b/drivers/media/radio/radio-sf16fmi.c
@@ -17,7 +17,7 @@
  */
 
 #include <linux/kernel.h>	/* __setup			*/
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/ioport.h>	/* request_region		*/
 #include <linux/delay.h>	/* udelay			*/
@@ -110,7 +110,7 @@ static inline int fmi_getsigstr(struct fmi *fmi)
 	val = fmi->mute ? 0x00 : 0x08;	/* mute/unmute */
 	outb(val, fmi->io);
 	outb(val | 0x10, fmi->io);
-	msleep(143); 		/* was schedule_timeout(HZ/7) */
+	msleep(143);		/* was schedule_timeout(HZ/7) */
 	res = (int)inb(fmi->io + 1);
 	outb(val, fmi->io);
 
diff --git a/drivers/media/radio/radio-sf16fmr2.c b/drivers/media/radio/radio-sf16fmr2.c
index de79d5569c2a..7b07d42a9909 100644
--- a/drivers/media/radio/radio-sf16fmr2.c
+++ b/drivers/media/radio/radio-sf16fmr2.c
@@ -7,7 +7,7 @@
  */
 
 #include <linux/delay.h>
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/slab.h>
 #include <linux/ioport.h>	/* request_region		*/
diff --git a/drivers/media/radio/radio-si476x.c b/drivers/media/radio/radio-si476x.c
index 540ac887a63c..b52e678c6901 100644
--- a/drivers/media/radio/radio-si476x.c
+++ b/drivers/media/radio/radio-si476x.c
@@ -759,7 +759,7 @@ static int si476x_radio_s_hw_freq_seek(struct file *file, void *priv,
 {
 	int err;
 	enum si476x_func func;
-	u32 rangelow, rangehigh;
+	u32 rangelow = seek->rangelow, rangehigh = seek->rangehigh;
 	struct si476x_radio *radio = video_drvdata(file);
 
 	if (file->f_flags & O_NONBLOCK)
@@ -771,23 +771,21 @@ static int si476x_radio_s_hw_freq_seek(struct file *file, void *priv,
 
 	si476x_core_lock(radio->core);
 
-	if (!seek->rangelow) {
+	if (!rangelow) {
 		err = regmap_read(radio->core->regmap,
 				  SI476X_PROP_SEEK_BAND_BOTTOM,
 				  &rangelow);
-		if (!err)
-			rangelow = si476x_to_v4l2(radio->core, rangelow);
-		else
+		if (err)
 			goto unlock;
+		rangelow = si476x_to_v4l2(radio->core, rangelow);
 	}
-	if (!seek->rangehigh) {
+	if (!rangehigh) {
 		err = regmap_read(radio->core->regmap,
 				  SI476X_PROP_SEEK_BAND_TOP,
 				  &rangehigh);
-		if (!err)
-			rangehigh = si476x_to_v4l2(radio->core, rangehigh);
-		else
+		if (err)
 			goto unlock;
+		rangehigh = si476x_to_v4l2(radio->core, rangehigh);
 	}
 
 	if (rangelow > rangehigh) {
@@ -1153,22 +1151,22 @@ static ssize_t si476x_radio_fops_read(struct file *file, char __user *buf,
 	return rval;
 }
 
-static unsigned int si476x_radio_fops_poll(struct file *file,
+static __poll_t si476x_radio_fops_poll(struct file *file,
 				struct poll_table_struct *pts)
 {
 	struct si476x_radio *radio = video_drvdata(file);
-	unsigned long req_events = poll_requested_events(pts);
-	unsigned int err = v4l2_ctrl_poll(file, pts);
+	__poll_t req_events = poll_requested_events(pts);
+	__poll_t err = v4l2_ctrl_poll(file, pts);
 
-	if (req_events & (POLLIN | POLLRDNORM)) {
+	if (req_events & (EPOLLIN | EPOLLRDNORM)) {
 		if (atomic_read(&radio->core->is_alive))
 			poll_wait(file, &radio->core->rds_read_queue, pts);
 
 		if (!atomic_read(&radio->core->is_alive))
-			err = POLLHUP;
+			err = EPOLLHUP;
 
 		if (!kfifo_is_empty(&radio->core->rds_fifo))
-			err = POLLIN | POLLRDNORM;
+			err = EPOLLIN | EPOLLRDNORM;
 	}
 
 	return err;
diff --git a/drivers/media/radio/radio-tea5764.c b/drivers/media/radio/radio-tea5764.c
index bc7e69e7e32e..afb763256fd6 100644
--- a/drivers/media/radio/radio-tea5764.c
+++ b/drivers/media/radio/radio-tea5764.c
@@ -417,7 +417,7 @@ static const struct v4l2_ioctl_ops tea5764_ioctl_ops = {
 static const struct video_device tea5764_radio_template = {
 	.name		= "TEA5764 FM-Radio",
 	.fops           = &tea5764_fops,
-	.ioctl_ops 	= &tea5764_ioctl_ops,
+	.ioctl_ops	= &tea5764_ioctl_ops,
 	.release	= video_device_release_empty,
 };
 
diff --git a/drivers/media/radio/radio-terratec.c b/drivers/media/radio/radio-terratec.c
index be10a802e3a9..4f116ea294fb 100644
--- a/drivers/media/radio/radio-terratec.c
+++ b/drivers/media/radio/radio-terratec.c
@@ -20,7 +20,7 @@
  * Converted to V4L2 API by Mauro Carvalho Chehab <mchehab@infradead.org>
  */
 
-#include <linux/module.h>	/* Modules 			*/
+#include <linux/module.h>	/* Modules			*/
 #include <linux/init.h>		/* Initdata			*/
 #include <linux/ioport.h>	/* request_region		*/
 #include <linux/videodev2.h>	/* kernel radio structs		*/
@@ -45,12 +45,12 @@ static int radio_nr = -1;
 module_param(radio_nr, int, 0444);
 MODULE_PARM_DESC(radio_nr, "Radio device number");
 
-#define WRT_DIS 	0x00
+#define WRT_DIS		0x00
 #define CLK_OFF		0x00
 #define IIC_DATA	0x01
 #define IIC_CLK		0x02
 #define DATA		0x04
-#define CLK_ON 		0x08
+#define CLK_ON		0x08
 #define WRT_EN		0x10
 
 static struct radio_isa_card *terratec_alloc(void)
diff --git a/drivers/media/radio/radio-wl1273.c b/drivers/media/radio/radio-wl1273.c
index 3cbdc085c65d..58e944591602 100644
--- a/drivers/media/radio/radio-wl1273.c
+++ b/drivers/media/radio/radio-wl1273.c
@@ -1089,7 +1089,7 @@ static ssize_t wl1273_fm_fops_write(struct file *file, const char __user *buf,
 	return r;
 }
 
-static unsigned int wl1273_fm_fops_poll(struct file *file,
+static __poll_t wl1273_fm_fops_poll(struct file *file,
 					struct poll_table_struct *pts)
 {
 	struct wl1273_device *radio = video_get_drvdata(video_devdata(file));
@@ -1104,10 +1104,10 @@ static unsigned int wl1273_fm_fops_poll(struct file *file,
 		poll_wait(file, &radio->read_queue, pts);
 
 		if (radio->rd_index != radio->wr_index)
-			return POLLIN | POLLRDNORM;
+			return EPOLLIN | EPOLLRDNORM;
 
 	} else if (core->mode == WL1273_MODE_TX) {
-		return POLLOUT | POLLWRNORM;
+		return EPOLLOUT | EPOLLWRNORM;
 	}
 
 	return 0;
diff --git a/drivers/media/radio/si470x/radio-si470x-common.c b/drivers/media/radio/si470x/radio-si470x-common.c
index c89a7d5b8c55..e0054e0f410d 100644
--- a/drivers/media/radio/si470x/radio-si470x-common.c
+++ b/drivers/media/radio/si470x/radio-si470x-common.c
@@ -33,18 +33,18 @@
  *		- switched from bit structs to bit masks
  *		- header file cleaned and integrated
  * 2008-01-14	Tobias Lorenz <tobias.lorenz@gmx.net>
- * 		Version 1.0.2
- * 		- hex values are now lower case
- * 		- commented USB ID for ADS/Tech moved on todo list
- * 		- blacklisted si470x in hid-quirks.c
- * 		- rds buffer handling functions integrated into *_work, *_read
- * 		- rds_command in si470x_poll exchanged against simple retval
- * 		- check for firmware version 15
- * 		- code order and prototypes still remain the same
- * 		- spacing and bottom of band codes remain the same
+ *		Version 1.0.2
+ *		- hex values are now lower case
+ *		- commented USB ID for ADS/Tech moved on todo list
+ *		- blacklisted si470x in hid-quirks.c
+ *		- rds buffer handling functions integrated into *_work, *_read
+ *		- rds_command in si470x_poll exchanged against simple retval
+ *		- check for firmware version 15
+ *		- code order and prototypes still remain the same
+ *		- spacing and bottom of band codes remain the same
  * 2008-01-16	Tobias Lorenz <tobias.lorenz@gmx.net>
  *		Version 1.0.3
- * 		- code reordered to avoid function prototypes
+ *		- code reordered to avoid function prototypes
  *		- switch/case defaults are now more user-friendly
  *		- unified comment style
  *		- applied all checkpatch.pl v1.12 suggestions
@@ -88,8 +88,8 @@
  *		- more safety checks, let si470x_get_freq return errno
  *		- vidioc behavior corrected according to v4l2 spec
  * 2008-10-20	Alexey Klimov <klimov.linux@gmail.com>
- * 		- add support for KWorld USB FM Radio FM700
- * 		- blacklisted KWorld radio in hid-core.c and hid-ids.h
+ *		- add support for KWorld USB FM Radio FM700
+ *		- blacklisted KWorld radio in hid-core.c and hid-ids.h
  * 2008-12-03	Mark Lord <mlord@pobox.com>
  *		- add support for DealExtreme USB Radio
  * 2009-01-31	Bob Ross <pigiron@gmx.com>
@@ -507,14 +507,14 @@ static ssize_t si470x_fops_read(struct file *file, char __user *buf,
 /*
  * si470x_fops_poll - poll RDS data
  */
-static unsigned int si470x_fops_poll(struct file *file,
+static __poll_t si470x_fops_poll(struct file *file,
 		struct poll_table_struct *pts)
 {
 	struct si470x_device *radio = video_drvdata(file);
-	unsigned long req_events = poll_requested_events(pts);
-	int retval = v4l2_ctrl_poll(file, pts);
+	__poll_t req_events = poll_requested_events(pts);
+	__poll_t retval = v4l2_ctrl_poll(file, pts);
 
-	if (req_events & (POLLIN | POLLRDNORM)) {
+	if (req_events & (EPOLLIN | EPOLLRDNORM)) {
 		/* switch on rds reception */
 		if ((radio->registers[SYSCONFIG1] & SYSCONFIG1_RDS) == 0)
 			si470x_rds_on(radio);
@@ -522,7 +522,7 @@ static unsigned int si470x_fops_poll(struct file *file,
 		poll_wait(file, &radio->read_queue, pts);
 
 		if (radio->rd_index != radio->wr_index)
-			retval |= POLLIN | POLLRDNORM;
+			retval |= EPOLLIN | EPOLLRDNORM;
 	}
 
 	return retval;
diff --git a/drivers/media/radio/tea575x.c b/drivers/media/radio/tea575x.c
index 4dc2067bce14..7412fe1b10c6 100644
--- a/drivers/media/radio/tea575x.c
+++ b/drivers/media/radio/tea575x.c
@@ -498,7 +498,7 @@ static const struct v4l2_ioctl_ops tea575x_ioctl_ops = {
 };
 
 static const struct video_device tea575x_radio = {
-	.ioctl_ops 	= &tea575x_ioctl_ops,
+	.ioctl_ops	= &tea575x_ioctl_ops,
 	.release        = video_device_release_empty,
 };
 
diff --git a/drivers/media/radio/wl128x/fmdrv_common.h b/drivers/media/radio/wl128x/fmdrv_common.h
index 7f1514eb1c07..552e22ea6bf3 100644
--- a/drivers/media/radio/wl128x/fmdrv_common.h
+++ b/drivers/media/radio/wl128x/fmdrv_common.h
@@ -311,11 +311,11 @@ struct fm_event_msg_hdr {
 #define FM_RDS_GROUP_TYPE_MASK_15B	    ((unsigned long)1<<31)
 
 /* RX Alternate Frequency info */
-#define FM_RDS_MIN_AF		          1
-#define FM_RDS_MAX_AF		        204
-#define FM_RDS_MAX_AF_JAPAN	        140
-#define FM_RDS_1_AF_FOLLOWS	        225
-#define FM_RDS_25_AF_FOLLOWS	        249
+#define FM_RDS_MIN_AF			  1
+#define FM_RDS_MAX_AF			204
+#define FM_RDS_MAX_AF_JAPAN		140
+#define FM_RDS_1_AF_FOLLOWS		225
+#define FM_RDS_25_AF_FOLLOWS		249
 
 /* RDS system type (RDS/RBDS) */
 #define FM_RDS_SYSTEM_RDS		0
diff --git a/drivers/media/radio/wl128x/fmdrv_v4l2.c b/drivers/media/radio/wl128x/fmdrv_v4l2.c
index fc5a7abc83d2..dccdf6558e6a 100644
--- a/drivers/media/radio/wl128x/fmdrv_v4l2.c
+++ b/drivers/media/radio/wl128x/fmdrv_v4l2.c
@@ -102,7 +102,7 @@ static ssize_t fm_v4l2_fops_write(struct file *file, const char __user * buf,
 	return sizeof(rds);
 }
 
-static u32 fm_v4l2_fops_poll(struct file *file, struct poll_table_struct *pts)
+static __poll_t fm_v4l2_fops_poll(struct file *file, struct poll_table_struct *pts)
 {
 	int ret;
 	struct fmdev *fmdev;
@@ -112,7 +112,7 @@ static u32 fm_v4l2_fops_poll(struct file *file, struct poll_table_struct *pts)
 	ret = fmc_is_rds_data_available(fmdev, file, pts);
 	mutex_unlock(&fmdev->mutex);
 	if (ret < 0)
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/drivers/media/rc/Kconfig b/drivers/media/rc/Kconfig
index afb3456d4e20..f14ead5954e0 100644
--- a/drivers/media/rc/Kconfig
+++ b/drivers/media/rc/Kconfig
@@ -16,34 +16,21 @@ menuconfig RC_CORE
 if RC_CORE
 source "drivers/media/rc/keymaps/Kconfig"
 
-menuconfig RC_DECODERS
-        bool "Remote controller decoders"
-	depends on RC_CORE
-	default y
-
-if RC_DECODERS
 config LIRC
-	tristate "LIRC interface driver"
+	bool "LIRC user interface"
 	depends on RC_CORE
-
 	---help---
-	   Enable this option to build the Linux Infrared Remote
-	   Control (LIRC) core device interface driver. The LIRC
-	   interface passes raw IR to and from userspace, where the
-	   LIRC daemon handles protocol decoding for IR reception and
-	   encoding for IR transmitting (aka "blasting").
+	   Enable this option to enable the Linux Infrared Remote
+	   Control user interface (e.g. /dev/lirc*). This interface
+	   passes raw IR to and from userspace, which is needed for
+	   IR transmitting (aka "blasting") and for the lirc daemon.
 
-config IR_LIRC_CODEC
-	tristate "Enable IR to LIRC bridge"
+menuconfig RC_DECODERS
+	bool "Remote controller decoders"
 	depends on RC_CORE
-	depends on LIRC
 	default y
 
-	---help---
-	   Enable this option to pass raw IR to and from userspace via
-	   the LIRC interface.
-
-
+if RC_DECODERS
 config IR_NEC_DECODER
 	tristate "Enable IR raw decoder for the NEC protocol"
 	depends on RC_CORE
@@ -465,9 +452,9 @@ config IR_SERIAL_TRANSMITTER
 	   Serial Port Transmitter support
 
 config IR_SIR
-        tristate "Built-in SIR IrDA port"
-        depends on RC_CORE
-        ---help---
+	tristate "Built-in SIR IrDA port"
+	depends on RC_CORE
+	---help---
 	   Say Y if you want to use a IrDA SIR port Transceivers.
 
 	   To compile this driver as a module, choose M here: the module will
diff --git a/drivers/media/rc/Makefile b/drivers/media/rc/Makefile
index 10026477a677..0e857816ac2d 100644
--- a/drivers/media/rc/Makefile
+++ b/drivers/media/rc/Makefile
@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
-rc-core-objs	:= rc-main.o rc-ir-raw.o
 
 obj-y += keymaps/
 
 obj-$(CONFIG_RC_CORE) += rc-core.o
-obj-$(CONFIG_LIRC) += lirc_dev.o
+rc-core-y := rc-main.o rc-ir-raw.o
+rc-core-$(CONFIG_LIRC) += lirc_dev.o
 obj-$(CONFIG_IR_NEC_DECODER) += ir-nec-decoder.o
 obj-$(CONFIG_IR_RC5_DECODER) += ir-rc5-decoder.o
 obj-$(CONFIG_IR_RC6_DECODER) += ir-rc6-decoder.o
@@ -13,7 +13,6 @@ obj-$(CONFIG_IR_SONY_DECODER) += ir-sony-decoder.o
 obj-$(CONFIG_IR_SANYO_DECODER) += ir-sanyo-decoder.o
 obj-$(CONFIG_IR_SHARP_DECODER) += ir-sharp-decoder.o
 obj-$(CONFIG_IR_MCE_KBD_DECODER) += ir-mce_kbd-decoder.o
-obj-$(CONFIG_IR_LIRC_CODEC) += ir-lirc-codec.o
 obj-$(CONFIG_IR_XMP_DECODER) += ir-xmp-decoder.o
 
 # stand-alone IR receivers/transmitters
diff --git a/drivers/media/rc/iguanair.c b/drivers/media/rc/iguanair.c
index 30e24da67226..7daac8bab83b 100644
--- a/drivers/media/rc/iguanair.c
+++ b/drivers/media/rc/iguanair.c
@@ -347,26 +347,23 @@ static int iguanair_set_tx_mask(struct rc_dev *dev, uint32_t mask)
 static int iguanair_tx(struct rc_dev *dev, unsigned *txbuf, unsigned count)
 {
 	struct iguanair *ir = dev->priv;
-	uint8_t space;
-	unsigned i, size, periods, bytes;
+	unsigned int i, size, p, periods;
 	int rc;
 
 	mutex_lock(&ir->lock);
 
 	/* convert from us to carrier periods */
-	for (i = space = size = 0; i < count; i++) {
+	for (i = size = 0; i < count; i++) {
 		periods = DIV_ROUND_CLOSEST(txbuf[i] * ir->carrier, 1000000);
-		bytes = DIV_ROUND_UP(periods, 127);
-		if (size + bytes > ir->bufsize) {
-			rc = -EINVAL;
-			goto out;
-		}
 		while (periods) {
-			unsigned p = min(periods, 127u);
-			ir->packet->payload[size++] = p | space;
+			p = min(periods, 127u);
+			if (size >= ir->bufsize) {
+				rc = -EINVAL;
+				goto out;
+			}
+			ir->packet->payload[size++] = p | ((i & 1) ? 0x80 : 0);
 			periods -= p;
 		}
-		space ^= 0x80;
 	}
 
 	ir->packet->header.start = 0;
diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c
index eb943e862515..950d068ba806 100644
--- a/drivers/media/rc/imon.c
+++ b/drivers/media/rc/imon.c
@@ -27,6 +27,7 @@
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/ktime.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
@@ -37,7 +38,6 @@
 #include <linux/usb/input.h>
 #include <media/rc-core.h>
 
-#include <linux/time.h>
 #include <linux/timer.h>
 
 #define MOD_AUTHOR	"Jarod Wilson <jarod@wilsonet.com>"
@@ -1168,29 +1168,6 @@ static int imon_ir_change_protocol(struct rc_dev *rc, u64 *rc_proto)
 	return retval;
 }
 
-static inline int tv2int(const struct timeval *a, const struct timeval *b)
-{
-	int usecs = 0;
-	int sec   = 0;
-
-	if (b->tv_usec > a->tv_usec) {
-		usecs = 1000000;
-		sec--;
-	}
-
-	usecs += a->tv_usec - b->tv_usec;
-
-	sec += a->tv_sec - b->tv_sec;
-	sec *= 1000;
-	usecs /= 1000;
-	sec += usecs;
-
-	if (sec < 0)
-		sec = 1000;
-
-	return sec;
-}
-
 /*
  * The directional pad behaves a bit differently, depending on whether this is
  * one of the older ffdc devices or a newer device. Newer devices appear to
@@ -1201,16 +1178,16 @@ static inline int tv2int(const struct timeval *a, const struct timeval *b)
  */
 static int stabilize(int a, int b, u16 timeout, u16 threshold)
 {
-	struct timeval ct;
-	static struct timeval prev_time = {0, 0};
-	static struct timeval hit_time  = {0, 0};
+	ktime_t ct;
+	static ktime_t prev_time;
+	static ktime_t hit_time;
 	static int x, y, prev_result, hits;
 	int result = 0;
-	int msec, msec_hit;
+	long msec, msec_hit;
 
-	do_gettimeofday(&ct);
-	msec = tv2int(&ct, &prev_time);
-	msec_hit = tv2int(&ct, &hit_time);
+	ct = ktime_get();
+	msec = ktime_ms_delta(ct, prev_time);
+	msec_hit = ktime_ms_delta(ct, hit_time);
 
 	if (msec > 100) {
 		x = 0;
@@ -1688,9 +1665,9 @@ static void imon_incoming_scancode(struct imon_context *ictx,
 	u32 kc;
 	u64 scancode;
 	int press_type = 0;
-	int msec;
-	struct timeval t;
-	static struct timeval prev_time = { 0, 0 };
+	long msec;
+	ktime_t t;
+	static ktime_t prev_time;
 	u8 ktype;
 
 	/* filter out junk data on the older 0xffdc imon devices */
@@ -1783,10 +1760,10 @@ static void imon_incoming_scancode(struct imon_context *ictx,
 	/* Only panel type events left to process now */
 	spin_lock_irqsave(&ictx->kc_lock, flags);
 
-	do_gettimeofday(&t);
+	t = ktime_get();
 	/* KEY_MUTE repeats from knob need to be suppressed */
 	if (ictx->kc == KEY_MUTE && ictx->kc == ictx->last_keycode) {
-		msec = tv2int(&t, &prev_time);
+		msec = ktime_ms_delta(t, prev_time);
 		if (msec < ictx->idev->rep[REP_DELAY]) {
 			spin_unlock_irqrestore(&ictx->kc_lock, flags);
 			return;
@@ -1956,6 +1933,7 @@ static void imon_get_ffdc_type(struct imon_context *ictx)
 		break;
 	/* iMON VFD, iMON IR */
 	case 0x24:
+	case 0x30:
 	case 0x85:
 		dev_info(ictx->dev, "0xffdc iMON VFD, iMON IR");
 		detected_display_type = IMON_DISPLAY_TYPE_VFD;
@@ -1974,6 +1952,11 @@ static void imon_get_ffdc_type(struct imon_context *ictx)
 		detected_display_type = IMON_DISPLAY_TYPE_LCD;
 		allowed_protos = RC_PROTO_BIT_RC6_MCE;
 		break;
+	/* no display, iMON IR */
+	case 0x26:
+		dev_info(ictx->dev, "0xffdc iMON Inside, iMON IR");
+		ictx->display_supported = false;
+		break;
 	default:
 		dev_info(ictx->dev, "Unknown 0xffdc device, defaulting to VFD and iMON IR");
 		detected_display_type = IMON_DISPLAY_TYPE_VFD;
diff --git a/drivers/media/rc/ir-jvc-decoder.c b/drivers/media/rc/ir-jvc-decoder.c
index 22c8aee3df4f..c03c776cfa54 100644
--- a/drivers/media/rc/ir-jvc-decoder.c
+++ b/drivers/media/rc/ir-jvc-decoder.c
@@ -212,6 +212,7 @@ static struct ir_raw_handler jvc_handler = {
 	.protocols	= RC_PROTO_BIT_JVC,
 	.decode		= ir_jvc_decode,
 	.encode		= ir_jvc_encode,
+	.carrier	= 38000,
 };
 
 static int __init ir_jvc_decode_init(void)
diff --git a/drivers/media/rc/ir-lirc-codec.c b/drivers/media/rc/ir-lirc-codec.c
deleted file mode 100644
index 4fd4521693d9..000000000000
--- a/drivers/media/rc/ir-lirc-codec.c
+++ /dev/null
@@ -1,448 +0,0 @@
-/* ir-lirc-codec.c - rc-core to classic lirc interface bridge
- *
- * Copyright (C) 2010 by Jarod Wilson <jarod@redhat.com>
- *
- * This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
-
-#include <linux/sched.h>
-#include <linux/wait.h>
-#include <linux/module.h>
-#include <media/lirc.h>
-#include <media/lirc_dev.h>
-#include <media/rc-core.h>
-#include "rc-core-priv.h"
-
-#define LIRCBUF_SIZE 256
-
-/**
- * ir_lirc_decode() - Send raw IR data to lirc_dev to be relayed to the
- *		      lircd userspace daemon for decoding.
- * @dev:	the struct rc_dev descriptor of the device
- * @ev:		the struct ir_raw_event descriptor of the pulse/space
- *
- * This function returns -EINVAL if the lirc interfaces aren't wired up.
- */
-static int ir_lirc_decode(struct rc_dev *dev, struct ir_raw_event ev)
-{
-	struct lirc_codec *lirc = &dev->raw->lirc;
-	int sample;
-
-	if (!dev->raw->lirc.ldev || !dev->raw->lirc.ldev->buf)
-		return -EINVAL;
-
-	/* Packet start */
-	if (ev.reset) {
-		/* Userspace expects a long space event before the start of
-		 * the signal to use as a sync.  This may be done with repeat
-		 * packets and normal samples.  But if a reset has been sent
-		 * then we assume that a long time has passed, so we send a
-		 * space with the maximum time value. */
-		sample = LIRC_SPACE(LIRC_VALUE_MASK);
-		IR_dprintk(2, "delivering reset sync space to lirc_dev\n");
-
-	/* Carrier reports */
-	} else if (ev.carrier_report) {
-		sample = LIRC_FREQUENCY(ev.carrier);
-		IR_dprintk(2, "carrier report (freq: %d)\n", sample);
-
-	/* Packet end */
-	} else if (ev.timeout) {
-
-		if (lirc->gap)
-			return 0;
-
-		lirc->gap_start = ktime_get();
-		lirc->gap = true;
-		lirc->gap_duration = ev.duration;
-
-		if (!lirc->send_timeout_reports)
-			return 0;
-
-		sample = LIRC_TIMEOUT(ev.duration / 1000);
-		IR_dprintk(2, "timeout report (duration: %d)\n", sample);
-
-	/* Normal sample */
-	} else {
-
-		if (lirc->gap) {
-			int gap_sample;
-
-			lirc->gap_duration += ktime_to_ns(ktime_sub(ktime_get(),
-				lirc->gap_start));
-
-			/* Convert to ms and cap by LIRC_VALUE_MASK */
-			do_div(lirc->gap_duration, 1000);
-			lirc->gap_duration = min(lirc->gap_duration,
-							(u64)LIRC_VALUE_MASK);
-
-			gap_sample = LIRC_SPACE(lirc->gap_duration);
-			lirc_buffer_write(dev->raw->lirc.ldev->buf,
-					  (unsigned char *)&gap_sample);
-			lirc->gap = false;
-		}
-
-		sample = ev.pulse ? LIRC_PULSE(ev.duration / 1000) :
-					LIRC_SPACE(ev.duration / 1000);
-		IR_dprintk(2, "delivering %uus %s to lirc_dev\n",
-			   TO_US(ev.duration), TO_STR(ev.pulse));
-	}
-
-	lirc_buffer_write(dev->raw->lirc.ldev->buf,
-			  (unsigned char *) &sample);
-	wake_up(&dev->raw->lirc.ldev->buf->wait_poll);
-
-	return 0;
-}
-
-static ssize_t ir_lirc_transmit_ir(struct file *file, const char __user *buf,
-				   size_t n, loff_t *ppos)
-{
-	struct lirc_codec *lirc;
-	struct rc_dev *dev;
-	unsigned int *txbuf; /* buffer with values to transmit */
-	ssize_t ret = -EINVAL;
-	size_t count;
-	ktime_t start;
-	s64 towait;
-	unsigned int duration = 0; /* signal duration in us */
-	int i;
-
-	start = ktime_get();
-
-	lirc = lirc_get_pdata(file);
-	if (!lirc)
-		return -EFAULT;
-
-	if (n < sizeof(unsigned) || n % sizeof(unsigned))
-		return -EINVAL;
-
-	count = n / sizeof(unsigned);
-	if (count > LIRCBUF_SIZE || count % 2 == 0)
-		return -EINVAL;
-
-	txbuf = memdup_user(buf, n);
-	if (IS_ERR(txbuf))
-		return PTR_ERR(txbuf);
-
-	dev = lirc->dev;
-	if (!dev) {
-		ret = -EFAULT;
-		goto out;
-	}
-
-	if (!dev->tx_ir) {
-		ret = -EINVAL;
-		goto out;
-	}
-
-	for (i = 0; i < count; i++) {
-		if (txbuf[i] > IR_MAX_DURATION / 1000 - duration || !txbuf[i]) {
-			ret = -EINVAL;
-			goto out;
-		}
-
-		duration += txbuf[i];
-	}
-
-	ret = dev->tx_ir(dev, txbuf, count);
-	if (ret < 0)
-		goto out;
-
-	for (duration = i = 0; i < ret; i++)
-		duration += txbuf[i];
-
-	ret *= sizeof(unsigned int);
-
-	/*
-	 * The lircd gap calculation expects the write function to
-	 * wait for the actual IR signal to be transmitted before
-	 * returning.
-	 */
-	towait = ktime_us_delta(ktime_add_us(start, duration), ktime_get());
-	if (towait > 0) {
-		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(usecs_to_jiffies(towait));
-	}
-
-out:
-	kfree(txbuf);
-	return ret;
-}
-
-static long ir_lirc_ioctl(struct file *filep, unsigned int cmd,
-			unsigned long arg)
-{
-	struct lirc_codec *lirc;
-	struct rc_dev *dev;
-	u32 __user *argp = (u32 __user *)(arg);
-	int ret = 0;
-	__u32 val = 0, tmp;
-
-	lirc = lirc_get_pdata(filep);
-	if (!lirc)
-		return -EFAULT;
-
-	dev = lirc->dev;
-	if (!dev)
-		return -EFAULT;
-
-	if (_IOC_DIR(cmd) & _IOC_WRITE) {
-		ret = get_user(val, argp);
-		if (ret)
-			return ret;
-	}
-
-	switch (cmd) {
-
-	/* legacy support */
-	case LIRC_GET_SEND_MODE:
-		if (!dev->tx_ir)
-			return -ENOTTY;
-
-		val = LIRC_MODE_PULSE;
-		break;
-
-	case LIRC_SET_SEND_MODE:
-		if (!dev->tx_ir)
-			return -ENOTTY;
-
-		if (val != LIRC_MODE_PULSE)
-			return -EINVAL;
-		return 0;
-
-	/* TX settings */
-	case LIRC_SET_TRANSMITTER_MASK:
-		if (!dev->s_tx_mask)
-			return -ENOTTY;
-
-		return dev->s_tx_mask(dev, val);
-
-	case LIRC_SET_SEND_CARRIER:
-		if (!dev->s_tx_carrier)
-			return -ENOTTY;
-
-		return dev->s_tx_carrier(dev, val);
-
-	case LIRC_SET_SEND_DUTY_CYCLE:
-		if (!dev->s_tx_duty_cycle)
-			return -ENOTTY;
-
-		if (val <= 0 || val >= 100)
-			return -EINVAL;
-
-		return dev->s_tx_duty_cycle(dev, val);
-
-	/* RX settings */
-	case LIRC_SET_REC_CARRIER:
-		if (!dev->s_rx_carrier_range)
-			return -ENOTTY;
-
-		if (val <= 0)
-			return -EINVAL;
-
-		return dev->s_rx_carrier_range(dev,
-					       dev->raw->lirc.carrier_low,
-					       val);
-
-	case LIRC_SET_REC_CARRIER_RANGE:
-		if (!dev->s_rx_carrier_range)
-			return -ENOTTY;
-
-		if (val <= 0)
-			return -EINVAL;
-
-		dev->raw->lirc.carrier_low = val;
-		return 0;
-
-	case LIRC_GET_REC_RESOLUTION:
-		if (!dev->rx_resolution)
-			return -ENOTTY;
-
-		val = dev->rx_resolution / 1000;
-		break;
-
-	case LIRC_SET_WIDEBAND_RECEIVER:
-		if (!dev->s_learning_mode)
-			return -ENOTTY;
-
-		return dev->s_learning_mode(dev, !!val);
-
-	case LIRC_SET_MEASURE_CARRIER_MODE:
-		if (!dev->s_carrier_report)
-			return -ENOTTY;
-
-		return dev->s_carrier_report(dev, !!val);
-
-	/* Generic timeout support */
-	case LIRC_GET_MIN_TIMEOUT:
-		if (!dev->max_timeout)
-			return -ENOTTY;
-		val = DIV_ROUND_UP(dev->min_timeout, 1000);
-		break;
-
-	case LIRC_GET_MAX_TIMEOUT:
-		if (!dev->max_timeout)
-			return -ENOTTY;
-		val = dev->max_timeout / 1000;
-		break;
-
-	case LIRC_SET_REC_TIMEOUT:
-		if (!dev->max_timeout)
-			return -ENOTTY;
-
-		/* Check for multiply overflow */
-		if (val > U32_MAX / 1000)
-			return -EINVAL;
-
-		tmp = val * 1000;
-
-		if (tmp < dev->min_timeout || tmp > dev->max_timeout)
-			return -EINVAL;
-
-		if (dev->s_timeout)
-			ret = dev->s_timeout(dev, tmp);
-		if (!ret)
-			dev->timeout = tmp;
-		break;
-
-	case LIRC_SET_REC_TIMEOUT_REPORTS:
-		if (!dev->timeout)
-			return -ENOTTY;
-
-		lirc->send_timeout_reports = !!val;
-		break;
-
-	default:
-		return lirc_dev_fop_ioctl(filep, cmd, arg);
-	}
-
-	if (_IOC_DIR(cmd) & _IOC_READ)
-		ret = put_user(val, argp);
-
-	return ret;
-}
-
-static const struct file_operations lirc_fops = {
-	.owner		= THIS_MODULE,
-	.write		= ir_lirc_transmit_ir,
-	.unlocked_ioctl	= ir_lirc_ioctl,
-#ifdef CONFIG_COMPAT
-	.compat_ioctl	= ir_lirc_ioctl,
-#endif
-	.read		= lirc_dev_fop_read,
-	.poll		= lirc_dev_fop_poll,
-	.open		= lirc_dev_fop_open,
-	.release	= lirc_dev_fop_close,
-	.llseek		= no_llseek,
-};
-
-static int ir_lirc_register(struct rc_dev *dev)
-{
-	struct lirc_dev *ldev;
-	int rc = -ENOMEM;
-	unsigned long features = 0;
-
-	ldev = lirc_allocate_device();
-	if (!ldev)
-		return rc;
-
-	if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
-		features |= LIRC_CAN_REC_MODE2;
-		if (dev->rx_resolution)
-			features |= LIRC_CAN_GET_REC_RESOLUTION;
-	}
-
-	if (dev->tx_ir) {
-		features |= LIRC_CAN_SEND_PULSE;
-		if (dev->s_tx_mask)
-			features |= LIRC_CAN_SET_TRANSMITTER_MASK;
-		if (dev->s_tx_carrier)
-			features |= LIRC_CAN_SET_SEND_CARRIER;
-		if (dev->s_tx_duty_cycle)
-			features |= LIRC_CAN_SET_SEND_DUTY_CYCLE;
-	}
-
-	if (dev->s_rx_carrier_range)
-		features |= LIRC_CAN_SET_REC_CARRIER |
-			LIRC_CAN_SET_REC_CARRIER_RANGE;
-
-	if (dev->s_learning_mode)
-		features |= LIRC_CAN_USE_WIDEBAND_RECEIVER;
-
-	if (dev->s_carrier_report)
-		features |= LIRC_CAN_MEASURE_CARRIER;
-
-	if (dev->max_timeout)
-		features |= LIRC_CAN_SET_REC_TIMEOUT;
-
-	snprintf(ldev->name, sizeof(ldev->name), "ir-lirc-codec (%s)",
-		 dev->driver_name);
-	ldev->features = features;
-	ldev->data = &dev->raw->lirc;
-	ldev->buf = NULL;
-	ldev->code_length = sizeof(struct ir_raw_event) * 8;
-	ldev->chunk_size = sizeof(int);
-	ldev->buffer_size = LIRCBUF_SIZE;
-	ldev->fops = &lirc_fops;
-	ldev->dev.parent = &dev->dev;
-	ldev->rdev = dev;
-	ldev->owner = THIS_MODULE;
-
-	rc = lirc_register_device(ldev);
-	if (rc < 0)
-		goto out;
-
-	dev->raw->lirc.ldev = ldev;
-	dev->raw->lirc.dev = dev;
-	return 0;
-
-out:
-	lirc_free_device(ldev);
-	return rc;
-}
-
-static int ir_lirc_unregister(struct rc_dev *dev)
-{
-	struct lirc_codec *lirc = &dev->raw->lirc;
-
-	lirc_unregister_device(lirc->ldev);
-	lirc->ldev = NULL;
-
-	return 0;
-}
-
-static struct ir_raw_handler lirc_handler = {
-	.protocols	= 0,
-	.decode		= ir_lirc_decode,
-	.raw_register	= ir_lirc_register,
-	.raw_unregister	= ir_lirc_unregister,
-};
-
-static int __init ir_lirc_codec_init(void)
-{
-	ir_raw_handler_register(&lirc_handler);
-
-	printk(KERN_INFO "IR LIRC bridge handler initialized\n");
-	return 0;
-}
-
-static void __exit ir_lirc_codec_exit(void)
-{
-	ir_raw_handler_unregister(&lirc_handler);
-}
-
-module_init(ir_lirc_codec_init);
-module_exit(ir_lirc_codec_exit);
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Jarod Wilson <jarod@redhat.com>");
-MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
-MODULE_DESCRIPTION("LIRC IR handler bridge");
diff --git a/drivers/media/rc/ir-mce_kbd-decoder.c b/drivers/media/rc/ir-mce_kbd-decoder.c
index 69d6264d54e6..2c3df02e05ff 100644
--- a/drivers/media/rc/ir-mce_kbd-decoder.c
+++ b/drivers/media/rc/ir-mce_kbd-decoder.c
@@ -215,6 +215,7 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 	struct mce_kbd_dec *data = &dev->raw->mce_kbd;
 	u32 scancode;
 	unsigned long delay;
+	struct lirc_scancode lsc = {};
 
 	if (!is_timing_event(ev)) {
 		if (ev.reset)
@@ -326,18 +327,22 @@ static int ir_mce_kbd_decode(struct rc_dev *dev, struct ir_raw_event ev)
 			mod_timer(&data->rx_timeout, jiffies + delay);
 			/* Pass data to keyboard buffer parser */
 			ir_mce_kbd_process_keyboard_data(data->idev, scancode);
+			lsc.rc_proto = RC_PROTO_MCIR2_KBD;
 			break;
 		case MCIR2_MOUSE_NBITS:
 			scancode = data->body & 0x1fffff;
 			IR_dprintk(1, "mouse data 0x%06x\n", scancode);
 			/* Pass data to mouse buffer parser */
 			ir_mce_kbd_process_mouse_data(data->idev, scancode);
+			lsc.rc_proto = RC_PROTO_MCIR2_MSE;
 			break;
 		default:
 			IR_dprintk(1, "not keyboard or mouse data\n");
 			goto out;
 		}
 
+		lsc.scancode = scancode;
+		ir_lirc_scancode_event(dev, &lsc);
 		data->state = STATE_INACTIVE;
 		input_event(data->idev, EV_MSC, MSC_SCAN, scancode);
 		input_sync(data->idev);
@@ -358,9 +363,6 @@ static int ir_mce_kbd_register(struct rc_dev *dev)
 	struct input_dev *idev;
 	int i, ret;
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW_TX)
-		return 0;
-
 	idev = input_allocate_device();
 	if (!idev)
 		return -ENOMEM;
@@ -415,9 +417,6 @@ static int ir_mce_kbd_unregister(struct rc_dev *dev)
 	struct mce_kbd_dec *mce_kbd = &dev->raw->mce_kbd;
 	struct input_dev *idev = mce_kbd->idev;
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW_TX)
-		return 0;
-
 	del_timer_sync(&mce_kbd->rx_timeout);
 	input_unregister_device(idev);
 
@@ -425,7 +424,7 @@ static int ir_mce_kbd_unregister(struct rc_dev *dev)
 }
 
 static const struct ir_raw_timings_manchester ir_mce_kbd_timings = {
-	.leader		= MCIR2_PREFIX_PULSE,
+	.leader_pulse	= MCIR2_PREFIX_PULSE,
 	.invert		= 1,
 	.clock		= MCIR2_UNIT,
 	.trailer_space	= MCIR2_UNIT * 10,
@@ -453,11 +452,11 @@ static int ir_mce_kbd_encode(enum rc_proto protocol, u32 scancode,
 	if (protocol == RC_PROTO_MCIR2_KBD) {
 		raw = scancode |
 		      ((u64)MCIR2_KEYBOARD_HEADER << MCIR2_KEYBOARD_NBITS);
-		len = MCIR2_KEYBOARD_NBITS + MCIR2_HEADER_NBITS + 1;
+		len = MCIR2_KEYBOARD_NBITS + MCIR2_HEADER_NBITS;
 	} else {
 		raw = scancode |
 		      ((u64)MCIR2_MOUSE_HEADER << MCIR2_MOUSE_NBITS);
-		len = MCIR2_MOUSE_NBITS + MCIR2_HEADER_NBITS + 1;
+		len = MCIR2_MOUSE_NBITS + MCIR2_HEADER_NBITS;
 	}
 
 	ret = ir_raw_gen_manchester(&e, max, &ir_mce_kbd_timings, len, raw);
@@ -473,6 +472,7 @@ static struct ir_raw_handler mce_kbd_handler = {
 	.encode		= ir_mce_kbd_encode,
 	.raw_register	= ir_mce_kbd_register,
 	.raw_unregister	= ir_mce_kbd_unregister,
+	.carrier	= 36000,
 };
 
 static int __init ir_mce_kbd_decode_init(void)
diff --git a/drivers/media/rc/ir-nec-decoder.c b/drivers/media/rc/ir-nec-decoder.c
index 6880c190dcd2..31d7bafe7bda 100644
--- a/drivers/media/rc/ir-nec-decoder.c
+++ b/drivers/media/rc/ir-nec-decoder.c
@@ -1,16 +1,7 @@
-/* ir-nec-decoder.c - handle NEC IR Pulse/Space protocol
- *
- * Copyright (C) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// ir-nec-decoder.c - handle NEC IR Pulse/Space protocol
+//
+// Copyright (C) 2010 by Mauro Carvalho Chehab
 
 #include <linux/bitrev.h>
 #include <linux/module.h>
@@ -263,6 +254,7 @@ static struct ir_raw_handler nec_handler = {
 							RC_PROTO_BIT_NEC32,
 	.decode		= ir_nec_decode,
 	.encode		= ir_nec_encode,
+	.carrier	= 38000,
 };
 
 static int __init ir_nec_decode_init(void)
@@ -281,7 +273,7 @@ static void __exit ir_nec_decode_exit(void)
 module_init(ir_nec_decode_init);
 module_exit(ir_nec_decode_exit);
 
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Mauro Carvalho Chehab");
 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
 MODULE_DESCRIPTION("NEC IR protocol decoder");
diff --git a/drivers/media/rc/ir-rc5-decoder.c b/drivers/media/rc/ir-rc5-decoder.c
index 1292f534de43..11a28f8772da 100644
--- a/drivers/media/rc/ir-rc5-decoder.c
+++ b/drivers/media/rc/ir-rc5-decoder.c
@@ -1,17 +1,8 @@
-/* ir-rc5-decoder.c - decoder for RC5(x) and StreamZap protocols
- *
- * Copyright (C) 2010 by Mauro Carvalho Chehab
- * Copyright (C) 2010 by Jarod Wilson <jarod@redhat.com>
- *
- * This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// ir-rc5-decoder.c - decoder for RC5(x) and StreamZap protocols
+//
+// Copyright (C) 2010 by Mauro Carvalho Chehab
+// Copyright (C) 2010 by Jarod Wilson <jarod@redhat.com>
 
 /*
  * This decoder handles the 14 bit RC5 protocol, 15 bit "StreamZap" protocol
@@ -182,16 +173,14 @@ static int ir_rc5_decode(struct rc_dev *dev, struct ir_raw_event ev)
 }
 
 static const struct ir_raw_timings_manchester ir_rc5_timings = {
-	.leader			= RC5_UNIT,
-	.pulse_space_start	= 0,
+	.leader_pulse		= RC5_UNIT,
 	.clock			= RC5_UNIT,
 	.trailer_space		= RC5_UNIT * 10,
 };
 
 static const struct ir_raw_timings_manchester ir_rc5x_timings[2] = {
 	{
-		.leader			= RC5_UNIT,
-		.pulse_space_start	= 0,
+		.leader_pulse		= RC5_UNIT,
 		.clock			= RC5_UNIT,
 		.trailer_space		= RC5X_SPACE,
 	},
@@ -202,8 +191,7 @@ static const struct ir_raw_timings_manchester ir_rc5x_timings[2] = {
 };
 
 static const struct ir_raw_timings_manchester ir_rc5_sz_timings = {
-	.leader				= RC5_UNIT,
-	.pulse_space_start		= 0,
+	.leader_pulse			= RC5_UNIT,
 	.clock				= RC5_UNIT,
 	.trailer_space			= RC5_UNIT * 10,
 };
@@ -237,9 +225,9 @@ static int ir_rc5_encode(enum rc_proto protocol, u32 scancode,
 		/* encode data */
 		data = !commandx << 12 | system << 6 | command;
 
-		/* Modulate the data */
+		/* First bit is encoded by leader_pulse */
 		ret = ir_raw_gen_manchester(&e, max, &ir_rc5_timings,
-					    RC5_NBITS, data);
+					    RC5_NBITS - 1, data);
 		if (ret < 0)
 			return ret;
 	} else if (protocol == RC_PROTO_RC5X_20) {
@@ -252,10 +240,11 @@ static int ir_rc5_encode(enum rc_proto protocol, u32 scancode,
 		/* encode data */
 		data = commandx << 18 | system << 12 | command << 6 | xdata;
 
-		/* Modulate the data */
+		/* First bit is encoded by leader_pulse */
 		pre_space_data = data >> (RC5X_NBITS - CHECK_RC5X_NBITS);
 		ret = ir_raw_gen_manchester(&e, max, &ir_rc5x_timings[0],
-					    CHECK_RC5X_NBITS, pre_space_data);
+					    CHECK_RC5X_NBITS - 1,
+					    pre_space_data);
 		if (ret < 0)
 			return ret;
 		ret = ir_raw_gen_manchester(&e, max - (e - events),
@@ -266,8 +255,10 @@ static int ir_rc5_encode(enum rc_proto protocol, u32 scancode,
 			return ret;
 	} else if (protocol == RC_PROTO_RC5_SZ) {
 		/* RC5-SZ scancode is raw enough for Manchester as it is */
+		/* First bit is encoded by leader_pulse */
 		ret = ir_raw_gen_manchester(&e, max, &ir_rc5_sz_timings,
-					    RC5_SZ_NBITS, scancode & 0x2fff);
+					    RC5_SZ_NBITS - 1,
+					    scancode & 0x2fff);
 		if (ret < 0)
 			return ret;
 	} else {
@@ -282,6 +273,7 @@ static struct ir_raw_handler rc5_handler = {
 							RC_PROTO_BIT_RC5_SZ,
 	.decode		= ir_rc5_decode,
 	.encode		= ir_rc5_encode,
+	.carrier	= 36000,
 };
 
 static int __init ir_rc5_decode_init(void)
@@ -300,7 +292,7 @@ static void __exit ir_rc5_decode_exit(void)
 module_init(ir_rc5_decode_init);
 module_exit(ir_rc5_decode_exit);
 
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Mauro Carvalho Chehab and Jarod Wilson");
 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
 MODULE_DESCRIPTION("RC5(x/sz) IR protocol decoder");
diff --git a/drivers/media/rc/ir-rc6-decoder.c b/drivers/media/rc/ir-rc6-decoder.c
index 5d0d2fe3b7a7..55bb19bbd4e9 100644
--- a/drivers/media/rc/ir-rc6-decoder.c
+++ b/drivers/media/rc/ir-rc6-decoder.c
@@ -288,13 +288,8 @@ static int ir_rc6_decode(struct rc_dev *dev, struct ir_raw_event ev)
 
 static const struct ir_raw_timings_manchester ir_rc6_timings[4] = {
 	{
-		.leader			= RC6_PREFIX_PULSE,
-		.pulse_space_start	= 0,
-		.clock			= RC6_UNIT,
-		.invert			= 1,
-		.trailer_space		= RC6_PREFIX_SPACE,
-	},
-	{
+		.leader_pulse		= RC6_PREFIX_PULSE,
+		.leader_space		= RC6_PREFIX_SPACE,
 		.clock			= RC6_UNIT,
 		.invert			= 1,
 	},
@@ -329,27 +324,22 @@ static int ir_rc6_encode(enum rc_proto protocol, u32 scancode,
 	struct ir_raw_event *e = events;
 
 	if (protocol == RC_PROTO_RC6_0) {
-		/* Modulate the preamble */
-		ret = ir_raw_gen_manchester(&e, max, &ir_rc6_timings[0], 0, 0);
-		if (ret < 0)
-			return ret;
-
 		/* Modulate the header (Start Bit & Mode-0) */
 		ret = ir_raw_gen_manchester(&e, max - (e - events),
-					    &ir_rc6_timings[1],
+					    &ir_rc6_timings[0],
 					    RC6_HEADER_NBITS, (1 << 3));
 		if (ret < 0)
 			return ret;
 
 		/* Modulate Trailer Bit */
 		ret = ir_raw_gen_manchester(&e, max - (e - events),
-					    &ir_rc6_timings[2], 1, 0);
+					    &ir_rc6_timings[1], 1, 0);
 		if (ret < 0)
 			return ret;
 
 		/* Modulate rest of the data */
 		ret = ir_raw_gen_manchester(&e, max - (e - events),
-					    &ir_rc6_timings[3], RC6_0_NBITS,
+					    &ir_rc6_timings[2], RC6_0_NBITS,
 					    scancode);
 		if (ret < 0)
 			return ret;
@@ -372,27 +362,22 @@ static int ir_rc6_encode(enum rc_proto protocol, u32 scancode,
 			return -EINVAL;
 		}
 
-		/* Modulate the preamble */
-		ret = ir_raw_gen_manchester(&e, max, &ir_rc6_timings[0], 0, 0);
-		if (ret < 0)
-			return ret;
-
 		/* Modulate the header (Start Bit & Header-version 6 */
 		ret = ir_raw_gen_manchester(&e, max - (e - events),
-					    &ir_rc6_timings[1],
+					    &ir_rc6_timings[0],
 					    RC6_HEADER_NBITS, (1 << 3 | 6));
 		if (ret < 0)
 			return ret;
 
 		/* Modulate Trailer Bit */
 		ret = ir_raw_gen_manchester(&e, max - (e - events),
-					    &ir_rc6_timings[2], 1, 0);
+					    &ir_rc6_timings[1], 1, 0);
 		if (ret < 0)
 			return ret;
 
 		/* Modulate rest of the data */
 		ret = ir_raw_gen_manchester(&e, max - (e - events),
-					    &ir_rc6_timings[3],
+					    &ir_rc6_timings[2],
 					    bits,
 					    scancode);
 		if (ret < 0)
@@ -408,6 +393,7 @@ static struct ir_raw_handler rc6_handler = {
 			  RC_PROTO_BIT_RC6_MCE,
 	.decode		= ir_rc6_decode,
 	.encode		= ir_rc6_encode,
+	.carrier	= 36000,
 };
 
 static int __init ir_rc6_decode_init(void)
diff --git a/drivers/media/rc/ir-sanyo-decoder.c b/drivers/media/rc/ir-sanyo-decoder.c
index d94e07b02f3b..ded39cdfc6ef 100644
--- a/drivers/media/rc/ir-sanyo-decoder.c
+++ b/drivers/media/rc/ir-sanyo-decoder.c
@@ -1,25 +1,16 @@
-/* ir-sanyo-decoder.c - handle SANYO IR Pulse/Space protocol
- *
- * Copyright (C) 2011 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- * This protocol uses the NEC protocol timings. However, data is formatted as:
- *	13 bits Custom Code
- *	13 bits NOT(Custom Code)
- *	8 bits Key data
- *	8 bits NOT(Key data)
- *
- * According with LIRC, this protocol is used on Sanyo, Aiwa and Chinon
- * Information for this protocol is available at the Sanyo LC7461 datasheet.
- */
+// SPDX-License-Identifier: GPL-2.0
+// ir-sanyo-decoder.c - handle SANYO IR Pulse/Space protocol
+//
+// Copyright (C) 2011 by Mauro Carvalho Chehab
+//
+// This protocol uses the NEC protocol timings. However, data is formatted as:
+//	13 bits Custom Code
+//	13 bits NOT(Custom Code)
+//	8 bits Key data
+//	8 bits NOT(Key data)
+//
+// According with LIRC, this protocol is used on Sanyo, Aiwa and Chinon
+// Information for this protocol is available at the Sanyo LC7461 datasheet.
 
 #include <linux/module.h>
 #include <linux/bitrev.h>
@@ -218,6 +209,7 @@ static struct ir_raw_handler sanyo_handler = {
 	.protocols	= RC_PROTO_BIT_SANYO,
 	.decode		= ir_sanyo_decode,
 	.encode		= ir_sanyo_encode,
+	.carrier	= 38000,
 };
 
 static int __init ir_sanyo_decode_init(void)
@@ -236,7 +228,7 @@ static void __exit ir_sanyo_decode_exit(void)
 module_init(ir_sanyo_decode_init);
 module_exit(ir_sanyo_decode_exit);
 
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Mauro Carvalho Chehab");
 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
 MODULE_DESCRIPTION("SANYO IR protocol decoder");
diff --git a/drivers/media/rc/ir-sharp-decoder.c b/drivers/media/rc/ir-sharp-decoder.c
index 7140dd6160ee..df296991906c 100644
--- a/drivers/media/rc/ir-sharp-decoder.c
+++ b/drivers/media/rc/ir-sharp-decoder.c
@@ -226,6 +226,7 @@ static struct ir_raw_handler sharp_handler = {
 	.protocols	= RC_PROTO_BIT_SHARP,
 	.decode		= ir_sharp_decode,
 	.encode		= ir_sharp_encode,
+	.carrier	= 38000,
 };
 
 static int __init ir_sharp_decode_init(void)
diff --git a/drivers/media/rc/ir-sony-decoder.c b/drivers/media/rc/ir-sony-decoder.c
index a47ced763031..e4bcff21c025 100644
--- a/drivers/media/rc/ir-sony-decoder.c
+++ b/drivers/media/rc/ir-sony-decoder.c
@@ -221,6 +221,7 @@ static struct ir_raw_handler sony_handler = {
 							RC_PROTO_BIT_SONY20,
 	.decode		= ir_sony_decode,
 	.encode		= ir_sony_encode,
+	.carrier	= 40000,
 };
 
 static int __init ir_sony_decode_init(void)
diff --git a/drivers/media/rc/ir-spi.c b/drivers/media/rc/ir-spi.c
index 29ed0638cb74..a32a84ae2d0b 100644
--- a/drivers/media/rc/ir-spi.c
+++ b/drivers/media/rc/ir-spi.c
@@ -1,13 +1,8 @@
-/*
- * Copyright (c) 2016 Samsung Electronics Co., Ltd.
- * Author: Andi Shyti <andi.shyti@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * SPI driven IR LED device driver
- */
+// SPDX-License-Identifier: GPL-2.0
+// SPI driven IR LED device driver
+//
+// Copyright (c) 2016 Samsung Electronics Co., Ltd.
+// Copyright (c) Andi Shyti <andi.shyti@samsung.com>
 
 #include <linux/delay.h>
 #include <linux/fs.h>
diff --git a/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c b/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c
index 2d303c2cee3b..732687ce0637 100644
--- a/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c
+++ b/drivers/media/rc/keymaps/rc-adstech-dvb-t-pci.c
@@ -1,14 +1,9 @@
-/* adstech-dvb-t-pci.h - Keytable for adstech_dvb_t_pci Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// adstech-dvb-t-pci.h - Keytable for adstech_dvb_t_pci Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-apac-viewcomp.c b/drivers/media/rc/keymaps/rc-apac-viewcomp.c
index 65bc8957d9c3..af2e7fdc7b85 100644
--- a/drivers/media/rc/keymaps/rc-apac-viewcomp.c
+++ b/drivers/media/rc/keymaps/rc-apac-viewcomp.c
@@ -1,14 +1,9 @@
-/* apac-viewcomp.h - Keytable for apac_viewcomp Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// apac-viewcomp.h - Keytable for apac_viewcomp Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-asus-pc39.c b/drivers/media/rc/keymaps/rc-asus-pc39.c
index 530e1d1158d1..13a935c3ac59 100644
--- a/drivers/media/rc/keymaps/rc-asus-pc39.c
+++ b/drivers/media/rc/keymaps/rc-asus-pc39.c
@@ -1,14 +1,9 @@
-/* asus-pc39.h - Keytable for asus_pc39 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// asus-pc39.h - Keytable for asus_pc39 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-asus-ps3-100.c b/drivers/media/rc/keymaps/rc-asus-ps3-100.c
index c91ba332984c..7f836fcc68ac 100644
--- a/drivers/media/rc/keymaps/rc-asus-ps3-100.c
+++ b/drivers/media/rc/keymaps/rc-asus-ps3-100.c
@@ -1,14 +1,9 @@
-/* asus-ps3-100.h - Keytable for asus_ps3_100 Remote Controller
- *
- * Copyright (c) 2012 by Mauro Carvalho Chehab
- *
- * Based on a previous patch from Remi Schwartz <remi.schwartz@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// asus-ps3-100.h - Keytable for asus_ps3_100 Remote Controller
+//
+// Copyright (c) 2012 by Mauro Carvalho Chehab
+//
+// Based on a previous patch from Remi Schwartz <remi.schwartz@gmail.com>
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c b/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c
index 11b4bdd2392b..b4b7932c0c5a 100644
--- a/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c
+++ b/drivers/media/rc/keymaps/rc-ati-tv-wonder-hd-600.c
@@ -1,14 +1,9 @@
-/* ati-tv-wonder-hd-600.h - Keytable for ati_tv_wonder_hd_600 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// ati-tv-wonder-hd-600.h - Keytable for ati_tv_wonder_hd_600 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-avermedia-a16d.c b/drivers/media/rc/keymaps/rc-avermedia-a16d.c
index 510dc90ebf49..5549c043cfe4 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-a16d.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-a16d.c
@@ -1,14 +1,9 @@
-/* avermedia-a16d.h - Keytable for avermedia_a16d Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// avermedia-a16d.h - Keytable for avermedia_a16d Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-avermedia-cardbus.c b/drivers/media/rc/keymaps/rc-avermedia-cardbus.c
index 4bbc1e68d1b8..74edcd82e685 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-cardbus.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-cardbus.c
@@ -1,14 +1,9 @@
-/* avermedia-cardbus.h - Keytable for avermedia_cardbus Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// avermedia-cardbus.h - Keytable for avermedia_cardbus Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-avermedia-dvbt.c b/drivers/media/rc/keymaps/rc-avermedia-dvbt.c
index f6b8547dbad3..796184160a48 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-dvbt.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-dvbt.c
@@ -1,14 +1,9 @@
-/* avermedia-dvbt.h - Keytable for avermedia_dvbt Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// avermedia-dvbt.h - Keytable for avermedia_dvbt Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-avermedia-m135a.c b/drivers/media/rc/keymaps/rc-avermedia-m135a.c
index 6d5a73b7ccec..f6977df1a75b 100644
--- a/drivers/media/rc/keymaps/rc-avermedia-m135a.c
+++ b/drivers/media/rc/keymaps/rc-avermedia-m135a.c
@@ -1,13 +1,8 @@
-/* avermedia-m135a.c - Keytable for Avermedia M135A Remote Controllers
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- * Copyright (c) 2010 by Herton Ronaldo Krzesinski <herton@mandriva.com.br>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// avermedia-m135a.c - Keytable for Avermedia M135A Remote Controllers
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
+// Copyright (c) 2010 by Herton Ronaldo Krzesinski <herton@mandriva.com.br>
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-avermedia.c b/drivers/media/rc/keymaps/rc-avermedia.c
index 6503f11c7df5..631ff52564f0 100644
--- a/drivers/media/rc/keymaps/rc-avermedia.c
+++ b/drivers/media/rc/keymaps/rc-avermedia.c
@@ -1,14 +1,9 @@
-/* avermedia.h - Keytable for avermedia Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// avermedia.h - Keytable for avermedia Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-avertv-303.c b/drivers/media/rc/keymaps/rc-avertv-303.c
index fbdd7ada57ce..47ca8b7ea532 100644
--- a/drivers/media/rc/keymaps/rc-avertv-303.c
+++ b/drivers/media/rc/keymaps/rc-avertv-303.c
@@ -1,14 +1,9 @@
-/* avertv-303.h - Keytable for avertv_303 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// avertv-303.h - Keytable for avertv_303 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-behold-columbus.c b/drivers/media/rc/keymaps/rc-behold-columbus.c
index d256743be998..e73057945bd1 100644
--- a/drivers/media/rc/keymaps/rc-behold-columbus.c
+++ b/drivers/media/rc/keymaps/rc-behold-columbus.c
@@ -1,14 +1,9 @@
-/* behold-columbus.h - Keytable for behold_columbus Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// behold-columbus.h - Keytable for behold_columbus Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
@@ -35,12 +30,12 @@ static struct rc_map_table behold_columbus[] = {
 
 	/*  0x01    0x02    0x03  0x0D    *
 	 *   1       2       3   Stereo   *
-	 *                        	  *
+	 *				  *
 	 *  0x04    0x05    0x06  0x19    *
 	 *   4       5       6   Snapshot *
-	 *                        	  *
+	 *				  *
 	 *  0x07    0x08    0x09  0x10    *
-	 *   7       8       9    Zoom 	  *
+	 *   7       8       9    Zoom	  *
 	 *                                */
 	{ 0x01, KEY_1 },
 	{ 0x02, KEY_2 },
diff --git a/drivers/media/rc/keymaps/rc-behold.c b/drivers/media/rc/keymaps/rc-behold.c
index 93dc795adc67..9b1b57e3c875 100644
--- a/drivers/media/rc/keymaps/rc-behold.c
+++ b/drivers/media/rc/keymaps/rc-behold.c
@@ -1,14 +1,9 @@
-/* behold.h - Keytable for behold Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// behold.h - Keytable for behold Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-budget-ci-old.c b/drivers/media/rc/keymaps/rc-budget-ci-old.c
index 81ea1424d9e5..56f051af6154 100644
--- a/drivers/media/rc/keymaps/rc-budget-ci-old.c
+++ b/drivers/media/rc/keymaps/rc-budget-ci-old.c
@@ -1,14 +1,9 @@
-/* budget-ci-old.h - Keytable for budget_ci_old Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// budget-ci-old.h - Keytable for budget_ci_old Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-cinergy-1400.c b/drivers/media/rc/keymaps/rc-cinergy-1400.c
index bcb96b3dda85..dacb13c53bb4 100644
--- a/drivers/media/rc/keymaps/rc-cinergy-1400.c
+++ b/drivers/media/rc/keymaps/rc-cinergy-1400.c
@@ -1,14 +1,9 @@
-/* cinergy-1400.h - Keytable for cinergy_1400 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// cinergy-1400.h - Keytable for cinergy_1400 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-cinergy.c b/drivers/media/rc/keymaps/rc-cinergy.c
index fd56c402aae5..6ab2e51b764d 100644
--- a/drivers/media/rc/keymaps/rc-cinergy.c
+++ b/drivers/media/rc/keymaps/rc-cinergy.c
@@ -1,14 +1,9 @@
-/* cinergy.h - Keytable for cinergy Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// cinergy.h - Keytable for cinergy Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-dib0700-nec.c b/drivers/media/rc/keymaps/rc-dib0700-nec.c
index 1b4df106b7b5..4ee801acb089 100644
--- a/drivers/media/rc/keymaps/rc-dib0700-nec.c
+++ b/drivers/media/rc/keymaps/rc-dib0700-nec.c
@@ -1,19 +1,14 @@
-/* rc-dvb0700-big.c - Keytable for devices in dvb0700
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * TODO: This table is a real mess, as it merges RC codes from several
- * devices into a big table. It also has both RC-5 and NEC codes inside.
- * It should be broken into small tables, and the protocols should properly
- * be identificated.
- *
- * The table were imported from dib0700_devices.c.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// rc-dvb0700-big.c - Keytable for devices in dvb0700
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
+//
+// TODO: This table is a real mess, as it merges RC codes from several
+// devices into a big table. It also has both RC-5 and NEC codes inside.
+// It should be broken into small tables, and the protocols should properly
+// be identificated.
+//
+// The table were imported from dib0700_devices.c.
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-dib0700-rc5.c b/drivers/media/rc/keymaps/rc-dib0700-rc5.c
index b0f8151bb824..ef4085a0fda3 100644
--- a/drivers/media/rc/keymaps/rc-dib0700-rc5.c
+++ b/drivers/media/rc/keymaps/rc-dib0700-rc5.c
@@ -1,19 +1,14 @@
-/* rc-dvb0700-big.c - Keytable for devices in dvb0700
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * TODO: This table is a real mess, as it merges RC codes from several
- * devices into a big table. It also has both RC-5 and NEC codes inside.
- * It should be broken into small tables, and the protocols should properly
- * be identificated.
- *
- * The table were imported from dib0700_devices.c.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// rc-dvb0700-big.c - Keytable for devices in dvb0700
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
+//
+// TODO: This table is a real mess, as it merges RC codes from several
+// devices into a big table. It also has both RC-5 and NEC codes inside.
+// It should be broken into small tables, and the protocols should properly
+// be identificated.
+//
+// The table were imported from dib0700_devices.c.
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-dm1105-nec.c b/drivers/media/rc/keymaps/rc-dm1105-nec.c
index c353445d10ed..d853cd9a0936 100644
--- a/drivers/media/rc/keymaps/rc-dm1105-nec.c
+++ b/drivers/media/rc/keymaps/rc-dm1105-nec.c
@@ -1,14 +1,9 @@
-/* dm1105-nec.h - Keytable for dm1105_nec Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// dm1105-nec.h - Keytable for dm1105_nec Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c b/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c
index 5bafd5b70f5e..cdc1d8c990cb 100644
--- a/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c
+++ b/drivers/media/rc/keymaps/rc-dntv-live-dvb-t.c
@@ -1,14 +1,9 @@
-/* dntv-live-dvb-t.h - Keytable for dntv_live_dvb_t Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// dntv-live-dvb-t.h - Keytable for dntv_live_dvb_t Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c b/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c
index 360167c8829b..38e1d1b837da 100644
--- a/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c
+++ b/drivers/media/rc/keymaps/rc-dntv-live-dvbt-pro.c
@@ -1,14 +1,9 @@
-/* dntv-live-dvbt-pro.h - Keytable for dntv_live_dvbt_pro Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// dntv-live-dvbt-pro.h - Keytable for dntv_live_dvbt_pro Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-em-terratec.c b/drivers/media/rc/keymaps/rc-em-terratec.c
index 18e1a2679c20..cbbba21484fb 100644
--- a/drivers/media/rc/keymaps/rc-em-terratec.c
+++ b/drivers/media/rc/keymaps/rc-em-terratec.c
@@ -1,14 +1,9 @@
-/* em-terratec.h - Keytable for em_terratec Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// em-terratec.h - Keytable for em_terratec Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c b/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c
index 72ffd5cb0108..e4e78c1f4123 100644
--- a/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c
+++ b/drivers/media/rc/keymaps/rc-encore-enltv-fm53.c
@@ -1,14 +1,9 @@
-/* encore-enltv-fm53.h - Keytable for encore_enltv_fm53 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// encore-enltv-fm53.h - Keytable for encore_enltv_fm53 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-encore-enltv.c b/drivers/media/rc/keymaps/rc-encore-enltv.c
index e0381e7aa964..5b4e832d5fac 100644
--- a/drivers/media/rc/keymaps/rc-encore-enltv.c
+++ b/drivers/media/rc/keymaps/rc-encore-enltv.c
@@ -1,14 +1,9 @@
-/* encore-enltv.h - Keytable for encore_enltv Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// encore-enltv.h - Keytable for encore_enltv Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-encore-enltv2.c b/drivers/media/rc/keymaps/rc-encore-enltv2.c
index e9b0bfba319c..c3d4437a6fda 100644
--- a/drivers/media/rc/keymaps/rc-encore-enltv2.c
+++ b/drivers/media/rc/keymaps/rc-encore-enltv2.c
@@ -1,14 +1,9 @@
-/* encore-enltv2.h - Keytable for encore_enltv2 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// encore-enltv2.h - Keytable for encore_enltv2 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-evga-indtube.c b/drivers/media/rc/keymaps/rc-evga-indtube.c
index b77c5e908668..f4398444330b 100644
--- a/drivers/media/rc/keymaps/rc-evga-indtube.c
+++ b/drivers/media/rc/keymaps/rc-evga-indtube.c
@@ -1,14 +1,9 @@
-/* evga-indtube.h - Keytable for evga_indtube Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// evga-indtube.h - Keytable for evga_indtube Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-eztv.c b/drivers/media/rc/keymaps/rc-eztv.c
index 5013b3b2aa93..0e481d51fcb5 100644
--- a/drivers/media/rc/keymaps/rc-eztv.c
+++ b/drivers/media/rc/keymaps/rc-eztv.c
@@ -1,14 +1,9 @@
-/* eztv.h - Keytable for eztv Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// eztv.h - Keytable for eztv Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-flydvb.c b/drivers/media/rc/keymaps/rc-flydvb.c
index 418b32521273..45940d7c92d0 100644
--- a/drivers/media/rc/keymaps/rc-flydvb.c
+++ b/drivers/media/rc/keymaps/rc-flydvb.c
@@ -1,14 +1,9 @@
-/* flydvb.h - Keytable for flydvb Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// flydvb.h - Keytable for flydvb Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-flyvideo.c b/drivers/media/rc/keymaps/rc-flyvideo.c
index 93fb87ecf061..b2d4e4c7b192 100644
--- a/drivers/media/rc/keymaps/rc-flyvideo.c
+++ b/drivers/media/rc/keymaps/rc-flyvideo.c
@@ -1,14 +1,9 @@
-/* flyvideo.h - Keytable for flyvideo Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// flyvideo.h - Keytable for flyvideo Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c b/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c
index 9ed3f749262b..1c63fc7d4576 100644
--- a/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c
+++ b/drivers/media/rc/keymaps/rc-fusionhdtv-mce.c
@@ -1,14 +1,9 @@
-/* fusionhdtv-mce.h - Keytable for fusionhdtv_mce Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// fusionhdtv-mce.h - Keytable for fusionhdtv_mce Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-gadmei-rm008z.c b/drivers/media/rc/keymaps/rc-gadmei-rm008z.c
index 3443b721d092..4a0a9786914f 100644
--- a/drivers/media/rc/keymaps/rc-gadmei-rm008z.c
+++ b/drivers/media/rc/keymaps/rc-gadmei-rm008z.c
@@ -1,14 +1,9 @@
-/* gadmei-rm008z.h - Keytable for gadmei_rm008z Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// gadmei-rm008z.h - Keytable for gadmei_rm008z Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c b/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c
index d140e8d45bcc..cc876a85cc31 100644
--- a/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c
+++ b/drivers/media/rc/keymaps/rc-genius-tvgo-a11mce.c
@@ -1,14 +1,9 @@
-/* genius-tvgo-a11mce.h - Keytable for genius_tvgo_a11mce Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// genius-tvgo-a11mce.h - Keytable for genius_tvgo_a11mce Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-gotview7135.c b/drivers/media/rc/keymaps/rc-gotview7135.c
index 51230fbb52ba..6b94bd39d977 100644
--- a/drivers/media/rc/keymaps/rc-gotview7135.c
+++ b/drivers/media/rc/keymaps/rc-gotview7135.c
@@ -1,14 +1,9 @@
-/* gotview7135.h - Keytable for gotview7135 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// gotview7135.h - Keytable for gotview7135 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-hauppauge.c b/drivers/media/rc/keymaps/rc-hauppauge.c
index 890164b68d64..582aa9012443 100644
--- a/drivers/media/rc/keymaps/rc-hauppauge.c
+++ b/drivers/media/rc/keymaps/rc-hauppauge.c
@@ -1,20 +1,15 @@
-/* rc-hauppauge.c - Keytable for Hauppauge Remote Controllers
- *
- * keymap imported from ir-keymaps.c
- *
- * This map currently contains the code for four different RCs:
- *	- New Hauppauge Gray;
- *	- Old Hauppauge Gray (with a golden screen for media keys);
- *	- Hauppauge Black;
- *	- DSR-0112 remote bundled with Haupauge MiniStick.
- *
- * Copyright (c) 2010-2011 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// rc-hauppauge.c - Keytable for Hauppauge Remote Controllers
+//
+// keymap imported from ir-keymaps.c
+//
+// This map currently contains the code for four different RCs:
+//	- New Hauppauge Gray;
+//	- Old Hauppauge Gray (with a golden screen for media keys);
+//	- Hauppauge Black;
+//	- DSR-0112 remote bundled with Haupauge MiniStick.
+//
+// Copyright (c) 2010-2011 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-iodata-bctv7e.c b/drivers/media/rc/keymaps/rc-iodata-bctv7e.c
index 8cf87a15c4f2..6ced43458f2a 100644
--- a/drivers/media/rc/keymaps/rc-iodata-bctv7e.c
+++ b/drivers/media/rc/keymaps/rc-iodata-bctv7e.c
@@ -1,14 +1,9 @@
-/* iodata-bctv7e.h - Keytable for iodata_bctv7e Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// iodata-bctv7e.h - Keytable for iodata_bctv7e Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-kaiomy.c b/drivers/media/rc/keymaps/rc-kaiomy.c
index e791f1e1b43b..f0f88df18606 100644
--- a/drivers/media/rc/keymaps/rc-kaiomy.c
+++ b/drivers/media/rc/keymaps/rc-kaiomy.c
@@ -1,14 +1,9 @@
-/* kaiomy.h - Keytable for kaiomy Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// kaiomy.h - Keytable for kaiomy Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-kworld-315u.c b/drivers/media/rc/keymaps/rc-kworld-315u.c
index 71dce0138f0e..ed0e0586dea2 100644
--- a/drivers/media/rc/keymaps/rc-kworld-315u.c
+++ b/drivers/media/rc/keymaps/rc-kworld-315u.c
@@ -1,14 +1,9 @@
-/* kworld-315u.h - Keytable for kworld_315u Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// kworld-315u.h - Keytable for kworld_315u Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c b/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c
index e0322ed16c94..453e04377de7 100644
--- a/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c
+++ b/drivers/media/rc/keymaps/rc-kworld-plus-tv-analog.c
@@ -1,14 +1,9 @@
-/* kworld-plus-tv-analog.h - Keytable for kworld_plus_tv_analog Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// kworld-plus-tv-analog.h - Keytable for kworld_plus_tv_analog Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-manli.c b/drivers/media/rc/keymaps/rc-manli.c
index da566902a4dd..29c9feaf413b 100644
--- a/drivers/media/rc/keymaps/rc-manli.c
+++ b/drivers/media/rc/keymaps/rc-manli.c
@@ -1,14 +1,9 @@
-/* manli.h - Keytable for manli Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// manli.h - Keytable for manli Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c b/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c
index dfa0ed1d7667..78cf2c286083 100644
--- a/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c
+++ b/drivers/media/rc/keymaps/rc-msi-tvanywhere-plus.c
@@ -1,14 +1,9 @@
-/* msi-tvanywhere-plus.h - Keytable for msi_tvanywhere_plus Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// msi-tvanywhere-plus.h - Keytable for msi_tvanywhere_plus Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-msi-tvanywhere.c b/drivers/media/rc/keymaps/rc-msi-tvanywhere.c
index 2111816a3f59..359a57be3a66 100644
--- a/drivers/media/rc/keymaps/rc-msi-tvanywhere.c
+++ b/drivers/media/rc/keymaps/rc-msi-tvanywhere.c
@@ -1,14 +1,9 @@
-/* msi-tvanywhere.h - Keytable for msi_tvanywhere Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// msi-tvanywhere.h - Keytable for msi_tvanywhere Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-nebula.c b/drivers/media/rc/keymaps/rc-nebula.c
index 109b6e1a8b1a..17d7c1b324da 100644
--- a/drivers/media/rc/keymaps/rc-nebula.c
+++ b/drivers/media/rc/keymaps/rc-nebula.c
@@ -1,14 +1,9 @@
-/* nebula.h - Keytable for nebula Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// nebula.h - Keytable for nebula Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c b/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c
index bb2d3a2962c0..76beef44a8d7 100644
--- a/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c
+++ b/drivers/media/rc/keymaps/rc-nec-terratec-cinergy-xs.c
@@ -1,14 +1,9 @@
-/* nec-terratec-cinergy-xs.h - Keytable for nec_terratec_cinergy_xs Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// nec-terratec-cinergy-xs.h - Keytable for nec_terratec_cinergy_xs Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-norwood.c b/drivers/media/rc/keymaps/rc-norwood.c
index cd25df336749..3765705c5549 100644
--- a/drivers/media/rc/keymaps/rc-norwood.c
+++ b/drivers/media/rc/keymaps/rc-norwood.c
@@ -1,14 +1,9 @@
-/* norwood.h - Keytable for norwood Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// norwood.h - Keytable for norwood Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-npgtech.c b/drivers/media/rc/keymaps/rc-npgtech.c
index 140bbc20a764..abaf7f6d4cb7 100644
--- a/drivers/media/rc/keymaps/rc-npgtech.c
+++ b/drivers/media/rc/keymaps/rc-npgtech.c
@@ -1,14 +1,9 @@
-/* npgtech.h - Keytable for npgtech Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// npgtech.h - Keytable for npgtech Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pctv-sedna.c b/drivers/media/rc/keymaps/rc-pctv-sedna.c
index 52b4558b7bd0..e3462c5c8984 100644
--- a/drivers/media/rc/keymaps/rc-pctv-sedna.c
+++ b/drivers/media/rc/keymaps/rc-pctv-sedna.c
@@ -1,14 +1,9 @@
-/* pctv-sedna.h - Keytable for pctv_sedna Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// pctv-sedna.h - Keytable for pctv_sedna Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pinnacle-color.c b/drivers/media/rc/keymaps/rc-pinnacle-color.c
index 973c9c34e304..63c2851e9dfe 100644
--- a/drivers/media/rc/keymaps/rc-pinnacle-color.c
+++ b/drivers/media/rc/keymaps/rc-pinnacle-color.c
@@ -1,14 +1,9 @@
-/* pinnacle-color.h - Keytable for pinnacle_color Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// pinnacle-color.h - Keytable for pinnacle_color Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pinnacle-grey.c b/drivers/media/rc/keymaps/rc-pinnacle-grey.c
index 22e44b0d2a93..31794d4180db 100644
--- a/drivers/media/rc/keymaps/rc-pinnacle-grey.c
+++ b/drivers/media/rc/keymaps/rc-pinnacle-grey.c
@@ -1,14 +1,9 @@
-/* pinnacle-grey.h - Keytable for pinnacle_grey Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// pinnacle-grey.h - Keytable for pinnacle_grey Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c b/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c
index 186dcf8e0491..876aeb6e1d9c 100644
--- a/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c
+++ b/drivers/media/rc/keymaps/rc-pinnacle-pctv-hd.c
@@ -1,14 +1,9 @@
-/* pinnacle-pctv-hd.h - Keytable for pinnacle_pctv_hd Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// pinnacle-pctv-hd.h - Keytable for pinnacle_pctv_hd Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pixelview-002t.c b/drivers/media/rc/keymaps/rc-pixelview-002t.c
index b235ada2e28f..4ed85f61d0ee 100644
--- a/drivers/media/rc/keymaps/rc-pixelview-002t.c
+++ b/drivers/media/rc/keymaps/rc-pixelview-002t.c
@@ -1,14 +1,9 @@
-/* rc-pixelview-mk12.h - Keytable for pixelview Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// rc-pixelview-mk12.h - Keytable for pixelview Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pixelview-mk12.c b/drivers/media/rc/keymaps/rc-pixelview-mk12.c
index 453d52d663fe..6ded64b732a5 100644
--- a/drivers/media/rc/keymaps/rc-pixelview-mk12.c
+++ b/drivers/media/rc/keymaps/rc-pixelview-mk12.c
@@ -1,14 +1,9 @@
-/* rc-pixelview-mk12.h - Keytable for pixelview Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// rc-pixelview-mk12.h - Keytable for pixelview Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pixelview-new.c b/drivers/media/rc/keymaps/rc-pixelview-new.c
index ef97095ec8f1..791130f108ff 100644
--- a/drivers/media/rc/keymaps/rc-pixelview-new.c
+++ b/drivers/media/rc/keymaps/rc-pixelview-new.c
@@ -1,14 +1,9 @@
-/* pixelview-new.h - Keytable for pixelview_new Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// pixelview-new.h - Keytable for pixelview_new Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pixelview.c b/drivers/media/rc/keymaps/rc-pixelview.c
index cfd8f80d3617..988919735165 100644
--- a/drivers/media/rc/keymaps/rc-pixelview.c
+++ b/drivers/media/rc/keymaps/rc-pixelview.c
@@ -1,14 +1,9 @@
-/* pixelview.h - Keytable for pixelview Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// pixelview.h - Keytable for pixelview Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-powercolor-real-angel.c b/drivers/media/rc/keymaps/rc-powercolor-real-angel.c
index b63f82bcf29a..4988e71c524c 100644
--- a/drivers/media/rc/keymaps/rc-powercolor-real-angel.c
+++ b/drivers/media/rc/keymaps/rc-powercolor-real-angel.c
@@ -1,14 +1,9 @@
-/* powercolor-real-angel.h - Keytable for powercolor_real_angel Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// powercolor-real-angel.h - Keytable for powercolor_real_angel Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-proteus-2309.c b/drivers/media/rc/keymaps/rc-proteus-2309.c
index be34c517e4e1..d2c13d0e7bff 100644
--- a/drivers/media/rc/keymaps/rc-proteus-2309.c
+++ b/drivers/media/rc/keymaps/rc-proteus-2309.c
@@ -1,14 +1,9 @@
-/* proteus-2309.h - Keytable for proteus_2309 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// proteus-2309.h - Keytable for proteus_2309 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-purpletv.c b/drivers/media/rc/keymaps/rc-purpletv.c
index 84c40b97ee00..c8011f4d96ea 100644
--- a/drivers/media/rc/keymaps/rc-purpletv.c
+++ b/drivers/media/rc/keymaps/rc-purpletv.c
@@ -1,14 +1,9 @@
-/* purpletv.h - Keytable for purpletv Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// purpletv.h - Keytable for purpletv Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-pv951.c b/drivers/media/rc/keymaps/rc-pv951.c
index be190ddebfc4..5235ee899c30 100644
--- a/drivers/media/rc/keymaps/rc-pv951.c
+++ b/drivers/media/rc/keymaps/rc-pv951.c
@@ -1,14 +1,9 @@
-/* pv951.h - Keytable for pv951 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// pv951.h - Keytable for pv951 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c b/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c
index 957fa21747ea..1cf786649675 100644
--- a/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c
+++ b/drivers/media/rc/keymaps/rc-real-audio-220-32-keys.c
@@ -1,14 +1,9 @@
-/* real-audio-220-32-keys.h - Keytable for real_audio_220_32_keys Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// real-audio-220-32-keys.h - Keytable for real_audio_220_32_keys Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-tbs-nec.c b/drivers/media/rc/keymaps/rc-tbs-nec.c
index 05facc043272..42766cb877c3 100644
--- a/drivers/media/rc/keymaps/rc-tbs-nec.c
+++ b/drivers/media/rc/keymaps/rc-tbs-nec.c
@@ -1,14 +1,9 @@
-/* tbs-nec.h - Keytable for tbs_nec Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// tbs-nec.h - Keytable for tbs_nec Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c b/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c
index 3d0f6f7e5bea..6cf53a56bce4 100644
--- a/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c
+++ b/drivers/media/rc/keymaps/rc-terratec-cinergy-xs.c
@@ -1,14 +1,9 @@
-/* terratec-cinergy-xs.h - Keytable for terratec_cinergy_xs Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// terratec-cinergy-xs.h - Keytable for terratec_cinergy_xs Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-tevii-nec.c b/drivers/media/rc/keymaps/rc-tevii-nec.c
index 31f8a0fd1f2c..58fcc72f528e 100644
--- a/drivers/media/rc/keymaps/rc-tevii-nec.c
+++ b/drivers/media/rc/keymaps/rc-tevii-nec.c
@@ -1,14 +1,9 @@
-/* tevii-nec.h - Keytable for tevii_nec Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// tevii-nec.h - Keytable for tevii_nec Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-tt-1500.c b/drivers/media/rc/keymaps/rc-tt-1500.c
index 374c230705d2..52f239d2c025 100644
--- a/drivers/media/rc/keymaps/rc-tt-1500.c
+++ b/drivers/media/rc/keymaps/rc-tt-1500.c
@@ -1,14 +1,9 @@
-/* tt-1500.h - Keytable for tt_1500 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// tt-1500.h - Keytable for tt_1500 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-videomate-s350.c b/drivers/media/rc/keymaps/rc-videomate-s350.c
index b4f103269872..e4d4dff06a24 100644
--- a/drivers/media/rc/keymaps/rc-videomate-s350.c
+++ b/drivers/media/rc/keymaps/rc-videomate-s350.c
@@ -1,14 +1,9 @@
-/* videomate-s350.h - Keytable for videomate_s350 Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// videomate-s350.h - Keytable for videomate_s350 Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c b/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c
index c431fdf44057..7c4890944407 100644
--- a/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c
+++ b/drivers/media/rc/keymaps/rc-videomate-tv-pvr.c
@@ -1,14 +1,9 @@
-/* videomate-tv-pvr.h - Keytable for videomate_tv_pvr Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// videomate-tv-pvr.h - Keytable for videomate_tv_pvr Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c b/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c
index 5a437e61bd5d..e443192dbe14 100644
--- a/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c
+++ b/drivers/media/rc/keymaps/rc-winfast-usbii-deluxe.c
@@ -1,14 +1,9 @@
-/* winfast-usbii-deluxe.h - Keytable for winfast_usbii_deluxe Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// winfast-usbii-deluxe.h - Keytable for winfast_usbii_deluxe Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
@@ -42,7 +37,7 @@ static struct rc_map_table winfast_usbii_deluxe[] = {
 	{ 0x60, KEY_CHANNELDOWN},	/* CHANNELDOWN */
 	{ 0x61, KEY_LAST},		/* LAST CHANNEL (RECALL) */
 
-	{ 0x72, KEY_VIDEO}, 		/* INPUT MODES (TV/FM) */
+	{ 0x72, KEY_VIDEO},		/* INPUT MODES (TV/FM) */
 
 	{ 0x70, KEY_POWER2},		/* TV ON/OFF */
 
diff --git a/drivers/media/rc/keymaps/rc-winfast.c b/drivers/media/rc/keymaps/rc-winfast.c
index 53685d1f9a47..ee7f4c349fd6 100644
--- a/drivers/media/rc/keymaps/rc-winfast.c
+++ b/drivers/media/rc/keymaps/rc-winfast.c
@@ -1,14 +1,9 @@
-/* winfast.h - Keytable for winfast Remote Controller
- *
- * keymap imported from ir-keymaps.c
- *
- * Copyright (c) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// winfast.h - Keytable for winfast Remote Controller
+//
+// keymap imported from ir-keymaps.c
+//
+// Copyright (c) 2010 by Mauro Carvalho Chehab
 
 #include <media/rc-map.h>
 #include <linux/module.h>
diff --git a/drivers/media/rc/lirc_dev.c b/drivers/media/rc/lirc_dev.c
index e16d1138ca48..cc863044c880 100644
--- a/drivers/media/rc/lirc_dev.c
+++ b/drivers/media/rc/lirc_dev.c
@@ -18,466 +18,775 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/module.h>
-#include <linux/sched/signal.h>
-#include <linux/ioctl.h>
-#include <linux/poll.h>
 #include <linux/mutex.h>
 #include <linux/device.h>
-#include <linux/cdev.h>
 #include <linux/idr.h>
+#include <linux/poll.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
 
-#include <media/rc-core.h>
-#include <media/lirc.h>
-#include <media/lirc_dev.h>
+#include "rc-core-priv.h"
+#include <uapi/linux/lirc.h>
 
-#define LOGHEAD		"lirc_dev (%s[%d]): "
+#define LIRCBUF_SIZE	256
 
 static dev_t lirc_base_dev;
 
 /* Used to keep track of allocated lirc devices */
-#define LIRC_MAX_DEVICES 256
 static DEFINE_IDA(lirc_ida);
 
 /* Only used for sysfs but defined to void otherwise */
 static struct class *lirc_class;
 
-static void lirc_release_device(struct device *ld)
+/**
+ * ir_lirc_raw_event() - Send raw IR data to lirc to be relayed to userspace
+ *
+ * @dev:	the struct rc_dev descriptor of the device
+ * @ev:		the struct ir_raw_event descriptor of the pulse/space
+ */
+void ir_lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev)
 {
-	struct lirc_dev *d = container_of(ld, struct lirc_dev, dev);
-
-	put_device(d->dev.parent);
+	unsigned long flags;
+	struct lirc_fh *fh;
+	int sample;
 
-	if (d->buf_internal) {
-		lirc_buffer_free(d->buf);
-		kfree(d->buf);
-		d->buf = NULL;
-	}
-	kfree(d);
-	module_put(THIS_MODULE);
-}
+	/* Packet start */
+	if (ev.reset) {
+		/*
+		 * Userspace expects a long space event before the start of
+		 * the signal to use as a sync.  This may be done with repeat
+		 * packets and normal samples.  But if a reset has been sent
+		 * then we assume that a long time has passed, so we send a
+		 * space with the maximum time value.
+		 */
+		sample = LIRC_SPACE(LIRC_VALUE_MASK);
+		IR_dprintk(2, "delivering reset sync space to lirc_dev\n");
 
-static int lirc_allocate_buffer(struct lirc_dev *d)
-{
-	int err;
+	/* Carrier reports */
+	} else if (ev.carrier_report) {
+		sample = LIRC_FREQUENCY(ev.carrier);
+		IR_dprintk(2, "carrier report (freq: %d)\n", sample);
 
-	if (d->buf) {
-		d->buf_internal = false;
-		return 0;
-	}
+	/* Packet end */
+	} else if (ev.timeout) {
+		if (dev->gap)
+			return;
 
-	d->buf = kmalloc(sizeof(*d->buf), GFP_KERNEL);
-	if (!d->buf)
-		return -ENOMEM;
+		dev->gap_start = ktime_get();
+		dev->gap = true;
+		dev->gap_duration = ev.duration;
 
-	err = lirc_buffer_init(d->buf, d->chunk_size, d->buffer_size);
-	if (err) {
-		kfree(d->buf);
-		d->buf = NULL;
-		return err;
-	}
+		sample = LIRC_TIMEOUT(ev.duration / 1000);
+		IR_dprintk(2, "timeout report (duration: %d)\n", sample);
 
-	d->buf_internal = true;
-	return 0;
-}
+	/* Normal sample */
+	} else {
+		if (dev->gap) {
+			dev->gap_duration += ktime_to_ns(ktime_sub(ktime_get(),
+							 dev->gap_start));
+
+			/* Convert to ms and cap by LIRC_VALUE_MASK */
+			do_div(dev->gap_duration, 1000);
+			dev->gap_duration = min_t(u64, dev->gap_duration,
+						  LIRC_VALUE_MASK);
+
+			spin_lock_irqsave(&dev->lirc_fh_lock, flags);
+			list_for_each_entry(fh, &dev->lirc_fh, list)
+				kfifo_put(&fh->rawir,
+					  LIRC_SPACE(dev->gap_duration));
+			spin_unlock_irqrestore(&dev->lirc_fh_lock, flags);
+			dev->gap = false;
+		}
 
-struct lirc_dev *
-lirc_allocate_device(void)
-{
-	struct lirc_dev *d;
-
-	d = kzalloc(sizeof(*d), GFP_KERNEL);
-	if (d) {
-		mutex_init(&d->mutex);
-		device_initialize(&d->dev);
-		d->dev.class = lirc_class;
-		d->dev.release = lirc_release_device;
-		__module_get(THIS_MODULE);
+		sample = ev.pulse ? LIRC_PULSE(ev.duration / 1000) :
+					LIRC_SPACE(ev.duration / 1000);
+		IR_dprintk(2, "delivering %uus %s to lirc_dev\n",
+			   TO_US(ev.duration), TO_STR(ev.pulse));
 	}
 
-	return d;
-}
-EXPORT_SYMBOL(lirc_allocate_device);
-
-void lirc_free_device(struct lirc_dev *d)
-{
-	if (!d)
-		return;
-
-	put_device(&d->dev);
+	spin_lock_irqsave(&dev->lirc_fh_lock, flags);
+	list_for_each_entry(fh, &dev->lirc_fh, list) {
+		if (LIRC_IS_TIMEOUT(sample) && !fh->send_timeout_reports)
+			continue;
+		if (kfifo_put(&fh->rawir, sample))
+			wake_up_poll(&fh->wait_poll, EPOLLIN | EPOLLRDNORM);
+	}
+	spin_unlock_irqrestore(&dev->lirc_fh_lock, flags);
 }
-EXPORT_SYMBOL(lirc_free_device);
 
-int lirc_register_device(struct lirc_dev *d)
+/**
+ * ir_lirc_scancode_event() - Send scancode data to lirc to be relayed to
+ *		userspace. This can be called in atomic context.
+ * @dev:	the struct rc_dev descriptor of the device
+ * @lsc:	the struct lirc_scancode describing the decoded scancode
+ */
+void ir_lirc_scancode_event(struct rc_dev *dev, struct lirc_scancode *lsc)
 {
-	int minor;
-	int err;
+	unsigned long flags;
+	struct lirc_fh *fh;
 
-	if (!d) {
-		pr_err("driver pointer must be not NULL!\n");
-		return -EBADRQC;
-	}
+	lsc->timestamp = ktime_get_ns();
 
-	if (!d->dev.parent) {
-		pr_err("dev parent pointer not filled in!\n");
-		return -EINVAL;
+	spin_lock_irqsave(&dev->lirc_fh_lock, flags);
+	list_for_each_entry(fh, &dev->lirc_fh, list) {
+		if (kfifo_put(&fh->scancodes, *lsc))
+			wake_up_poll(&fh->wait_poll, EPOLLIN | EPOLLRDNORM);
 	}
+	spin_unlock_irqrestore(&dev->lirc_fh_lock, flags);
+}
+EXPORT_SYMBOL_GPL(ir_lirc_scancode_event);
 
-	if (!d->fops) {
-		pr_err("fops pointer not filled in!\n");
-		return -EINVAL;
-	}
+static int ir_lirc_open(struct inode *inode, struct file *file)
+{
+	struct rc_dev *dev = container_of(inode->i_cdev, struct rc_dev,
+					  lirc_cdev);
+	struct lirc_fh *fh = kzalloc(sizeof(*fh), GFP_KERNEL);
+	unsigned long flags;
+	int retval;
 
-	if (!d->buf && d->chunk_size < 1) {
-		pr_err("chunk_size must be set!\n");
-		return -EINVAL;
-	}
+	if (!fh)
+		return -ENOMEM;
 
-	if (!d->buf && d->buffer_size < 1) {
-		pr_err("buffer_size must be set!\n");
-		return -EINVAL;
-	}
+	get_device(&dev->dev);
 
-	if (d->code_length < 1 || d->code_length > (BUFLEN * 8)) {
-		dev_err(&d->dev, "code length must be less than %d bits\n",
-			BUFLEN * 8);
-		return -EBADRQC;
+	if (!dev->registered) {
+		retval = -ENODEV;
+		goto out_fh;
 	}
 
-	if (!d->buf && !(d->fops && d->fops->read &&
-			 d->fops->poll && d->fops->unlocked_ioctl)) {
-		dev_err(&d->dev, "undefined read, poll, ioctl\n");
-		return -EBADRQC;
+	if (dev->driver_type == RC_DRIVER_IR_RAW) {
+		if (kfifo_alloc(&fh->rawir, MAX_IR_EVENT_SIZE, GFP_KERNEL)) {
+			retval = -ENOMEM;
+			goto out_fh;
+		}
 	}
 
-	/* some safety check 8-) */
-	d->name[sizeof(d->name) - 1] = '\0';
-
-	if (d->features == 0)
-		d->features = LIRC_CAN_REC_LIRCCODE;
-
-	if (LIRC_CAN_REC(d->features)) {
-		err = lirc_allocate_buffer(d);
-		if (err)
-			return err;
+	if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
+		if (kfifo_alloc(&fh->scancodes, 32, GFP_KERNEL)) {
+			retval = -ENOMEM;
+			goto out_rawir;
+		}
 	}
 
-	minor = ida_simple_get(&lirc_ida, 0, LIRC_MAX_DEVICES, GFP_KERNEL);
-	if (minor < 0)
-		return minor;
+	fh->send_mode = LIRC_MODE_PULSE;
+	fh->rc = dev;
+	fh->send_timeout_reports = true;
 
-	d->minor = minor;
-	d->dev.devt = MKDEV(MAJOR(lirc_base_dev), d->minor);
-	dev_set_name(&d->dev, "lirc%d", d->minor);
+	if (dev->driver_type == RC_DRIVER_SCANCODE)
+		fh->rec_mode = LIRC_MODE_SCANCODE;
+	else
+		fh->rec_mode = LIRC_MODE_MODE2;
 
-	cdev_init(&d->cdev, d->fops);
-	d->cdev.owner = d->owner;
-	d->attached = true;
+	retval = rc_open(dev);
+	if (retval)
+		goto out_kfifo;
 
-	err = cdev_device_add(&d->cdev, &d->dev);
-	if (err) {
-		ida_simple_remove(&lirc_ida, minor);
-		return err;
-	}
+	init_waitqueue_head(&fh->wait_poll);
 
-	get_device(d->dev.parent);
+	file->private_data = fh;
+	spin_lock_irqsave(&dev->lirc_fh_lock, flags);
+	list_add(&fh->list, &dev->lirc_fh);
+	spin_unlock_irqrestore(&dev->lirc_fh_lock, flags);
 
-	dev_info(&d->dev, "lirc_dev: driver %s registered at minor = %d\n",
-		 d->name, d->minor);
+	nonseekable_open(inode, file);
 
 	return 0;
+out_kfifo:
+	if (dev->driver_type != RC_DRIVER_IR_RAW_TX)
+		kfifo_free(&fh->scancodes);
+out_rawir:
+	if (dev->driver_type == RC_DRIVER_IR_RAW)
+		kfifo_free(&fh->rawir);
+out_fh:
+	kfree(fh);
+	put_device(&dev->dev);
+
+	return retval;
 }
-EXPORT_SYMBOL(lirc_register_device);
 
-void lirc_unregister_device(struct lirc_dev *d)
+static int ir_lirc_close(struct inode *inode, struct file *file)
 {
-	if (!d)
-		return;
-
-	dev_dbg(&d->dev, "lirc_dev: driver %s unregistered from minor = %d\n",
-		d->name, d->minor);
+	struct lirc_fh *fh = file->private_data;
+	struct rc_dev *dev = fh->rc;
+	unsigned long flags;
 
-	mutex_lock(&d->mutex);
+	spin_lock_irqsave(&dev->lirc_fh_lock, flags);
+	list_del(&fh->list);
+	spin_unlock_irqrestore(&dev->lirc_fh_lock, flags);
 
-	d->attached = false;
-	if (d->open) {
-		dev_dbg(&d->dev, LOGHEAD "releasing opened driver\n",
-			d->name, d->minor);
-		wake_up_interruptible(&d->buf->wait_poll);
-	}
+	if (dev->driver_type == RC_DRIVER_IR_RAW)
+		kfifo_free(&fh->rawir);
+	if (dev->driver_type != RC_DRIVER_IR_RAW_TX)
+		kfifo_free(&fh->scancodes);
+	kfree(fh);
 
-	mutex_unlock(&d->mutex);
+	rc_close(dev);
+	put_device(&dev->dev);
 
-	cdev_device_del(&d->cdev, &d->dev);
-	ida_simple_remove(&lirc_ida, d->minor);
-	put_device(&d->dev);
+	return 0;
 }
-EXPORT_SYMBOL(lirc_unregister_device);
 
-int lirc_dev_fop_open(struct inode *inode, struct file *file)
+static ssize_t ir_lirc_transmit_ir(struct file *file, const char __user *buf,
+				   size_t n, loff_t *ppos)
 {
-	struct lirc_dev *d = container_of(inode->i_cdev, struct lirc_dev, cdev);
-	int retval;
+	struct lirc_fh *fh = file->private_data;
+	struct rc_dev *dev = fh->rc;
+	unsigned int *txbuf;
+	struct ir_raw_event *raw = NULL;
+	ssize_t ret;
+	size_t count;
+	ktime_t start;
+	s64 towait;
+	unsigned int duration = 0; /* signal duration in us */
+	int i;
+
+	ret = mutex_lock_interruptible(&dev->lock);
+	if (ret)
+		return ret;
 
-	dev_dbg(&d->dev, LOGHEAD "open called\n", d->name, d->minor);
+	if (!dev->registered) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
 
-	retval = mutex_lock_interruptible(&d->mutex);
-	if (retval)
-		return retval;
+	start = ktime_get();
 
-	if (!d->attached) {
-		retval = -ENODEV;
-		goto out;
+	if (!dev->tx_ir) {
+		ret = -EINVAL;
+		goto out_unlock;
 	}
 
-	if (d->open) {
-		retval = -EBUSY;
-		goto out;
-	}
+	if (fh->send_mode == LIRC_MODE_SCANCODE) {
+		struct lirc_scancode scan;
 
-	if (d->rdev) {
-		retval = rc_open(d->rdev);
-		if (retval)
-			goto out;
-	}
+		if (n != sizeof(scan)) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
 
-	if (d->buf)
-		lirc_buffer_clear(d->buf);
+		if (copy_from_user(&scan, buf, sizeof(scan))) {
+			ret = -EFAULT;
+			goto out_unlock;
+		}
 
-	d->open++;
+		if (scan.flags || scan.keycode || scan.timestamp) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
 
-	lirc_init_pdata(inode, file);
-	nonseekable_open(inode, file);
-	mutex_unlock(&d->mutex);
+		/*
+		 * The scancode field in lirc_scancode is 64-bit simply
+		 * to future-proof it, since there are IR protocols encode
+		 * use more than 32 bits. For now only 32-bit protocols
+		 * are supported.
+		 */
+		if (scan.scancode > U32_MAX ||
+		    !rc_validate_scancode(scan.rc_proto, scan.scancode)) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
 
-	return 0;
+		raw = kmalloc_array(LIRCBUF_SIZE, sizeof(*raw), GFP_KERNEL);
+		if (!raw) {
+			ret = -ENOMEM;
+			goto out_unlock;
+		}
 
-out:
-	mutex_unlock(&d->mutex);
-	return retval;
-}
-EXPORT_SYMBOL(lirc_dev_fop_open);
+		ret = ir_raw_encode_scancode(scan.rc_proto, scan.scancode,
+					     raw, LIRCBUF_SIZE);
+		if (ret < 0)
+			goto out_kfree_raw;
 
-int lirc_dev_fop_close(struct inode *inode, struct file *file)
-{
-	struct lirc_dev *d = file->private_data;
+		count = ret;
 
-	mutex_lock(&d->mutex);
+		txbuf = kmalloc_array(count, sizeof(unsigned int), GFP_KERNEL);
+		if (!txbuf) {
+			ret = -ENOMEM;
+			goto out_kfree_raw;
+		}
 
-	rc_close(d->rdev);
-	d->open--;
+		for (i = 0; i < count; i++)
+			/* Convert from NS to US */
+			txbuf[i] = DIV_ROUND_UP(raw[i].duration, 1000);
 
-	mutex_unlock(&d->mutex);
+		if (dev->s_tx_carrier) {
+			int carrier = ir_raw_encode_carrier(scan.rc_proto);
 
-	return 0;
-}
-EXPORT_SYMBOL(lirc_dev_fop_close);
+			if (carrier > 0)
+				dev->s_tx_carrier(dev, carrier);
+		}
+	} else {
+		if (n < sizeof(unsigned int) || n % sizeof(unsigned int)) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
 
-unsigned int lirc_dev_fop_poll(struct file *file, poll_table *wait)
-{
-	struct lirc_dev *d = file->private_data;
-	unsigned int ret;
+		count = n / sizeof(unsigned int);
+		if (count > LIRCBUF_SIZE || count % 2 == 0) {
+			ret = -EINVAL;
+			goto out_unlock;
+		}
 
-	if (!d->attached)
-		return POLLHUP | POLLERR;
+		txbuf = memdup_user(buf, n);
+		if (IS_ERR(txbuf)) {
+			ret = PTR_ERR(txbuf);
+			goto out_unlock;
+		}
+	}
 
-	if (d->buf) {
-		poll_wait(file, &d->buf->wait_poll, wait);
+	for (i = 0; i < count; i++) {
+		if (txbuf[i] > IR_MAX_DURATION / 1000 - duration || !txbuf[i]) {
+			ret = -EINVAL;
+			goto out_kfree;
+		}
 
-		if (lirc_buffer_empty(d->buf))
-			ret = 0;
-		else
-			ret = POLLIN | POLLRDNORM;
-	} else {
-		ret = POLLERR;
+		duration += txbuf[i];
 	}
 
-	dev_dbg(&d->dev, LOGHEAD "poll result = %d\n", d->name, d->minor, ret);
+	ret = dev->tx_ir(dev, txbuf, count);
+	if (ret < 0)
+		goto out_kfree;
+
+	kfree(txbuf);
+	kfree(raw);
+	mutex_unlock(&dev->lock);
+
+	/*
+	 * The lircd gap calculation expects the write function to
+	 * wait for the actual IR signal to be transmitted before
+	 * returning.
+	 */
+	towait = ktime_us_delta(ktime_add_us(start, duration),
+				ktime_get());
+	if (towait > 0) {
+		set_current_state(TASK_INTERRUPTIBLE);
+		schedule_timeout(usecs_to_jiffies(towait));
+	}
 
+	return n;
+out_kfree:
+	kfree(txbuf);
+out_kfree_raw:
+	kfree(raw);
+out_unlock:
+	mutex_unlock(&dev->lock);
 	return ret;
 }
-EXPORT_SYMBOL(lirc_dev_fop_poll);
 
-long lirc_dev_fop_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+static long ir_lirc_ioctl(struct file *file, unsigned int cmd,
+			  unsigned long arg)
 {
-	struct lirc_dev *d = file->private_data;
-	__u32 mode;
-	int result;
-
-	dev_dbg(&d->dev, LOGHEAD "ioctl called (0x%x)\n",
-		d->name, d->minor, cmd);
+	struct lirc_fh *fh = file->private_data;
+	struct rc_dev *dev = fh->rc;
+	u32 __user *argp = (u32 __user *)(arg);
+	u32 val = 0;
+	int ret;
+
+	if (_IOC_DIR(cmd) & _IOC_WRITE) {
+		ret = get_user(val, argp);
+		if (ret)
+			return ret;
+	}
 
-	result = mutex_lock_interruptible(&d->mutex);
-	if (result)
-		return result;
+	ret = mutex_lock_interruptible(&dev->lock);
+	if (ret)
+		return ret;
 
-	if (!d->attached) {
-		result = -ENODEV;
+	if (!dev->registered) {
+		ret = -ENODEV;
 		goto out;
 	}
 
 	switch (cmd) {
 	case LIRC_GET_FEATURES:
-		result = put_user(d->features, (__u32 __user *)arg);
-		break;
-	case LIRC_GET_REC_MODE:
-		if (!LIRC_CAN_REC(d->features)) {
-			result = -ENOTTY;
-			break;
+		if (dev->driver_type == RC_DRIVER_SCANCODE)
+			val |= LIRC_CAN_REC_SCANCODE;
+
+		if (dev->driver_type == RC_DRIVER_IR_RAW) {
+			val |= LIRC_CAN_REC_MODE2;
+			if (dev->rx_resolution)
+				val |= LIRC_CAN_GET_REC_RESOLUTION;
+		}
+
+		if (dev->tx_ir) {
+			val |= LIRC_CAN_SEND_PULSE;
+			if (dev->s_tx_mask)
+				val |= LIRC_CAN_SET_TRANSMITTER_MASK;
+			if (dev->s_tx_carrier)
+				val |= LIRC_CAN_SET_SEND_CARRIER;
+			if (dev->s_tx_duty_cycle)
+				val |= LIRC_CAN_SET_SEND_DUTY_CYCLE;
 		}
 
-		result = put_user(LIRC_REC2MODE
-				  (d->features & LIRC_CAN_REC_MASK),
-				  (__u32 __user *)arg);
+		if (dev->s_rx_carrier_range)
+			val |= LIRC_CAN_SET_REC_CARRIER |
+				LIRC_CAN_SET_REC_CARRIER_RANGE;
+
+		if (dev->s_learning_mode)
+			val |= LIRC_CAN_USE_WIDEBAND_RECEIVER;
+
+		if (dev->s_carrier_report)
+			val |= LIRC_CAN_MEASURE_CARRIER;
+
+		if (dev->max_timeout)
+			val |= LIRC_CAN_SET_REC_TIMEOUT;
+
 		break;
+
+	/* mode support */
+	case LIRC_GET_REC_MODE:
+		if (dev->driver_type == RC_DRIVER_IR_RAW_TX)
+			ret = -ENOTTY;
+		else
+			val = fh->rec_mode;
+		break;
+
 	case LIRC_SET_REC_MODE:
-		if (!LIRC_CAN_REC(d->features)) {
-			result = -ENOTTY;
+		switch (dev->driver_type) {
+		case RC_DRIVER_IR_RAW_TX:
+			ret = -ENOTTY;
+			break;
+		case RC_DRIVER_SCANCODE:
+			if (val != LIRC_MODE_SCANCODE)
+				ret = -EINVAL;
+			break;
+		case RC_DRIVER_IR_RAW:
+			if (!(val == LIRC_MODE_MODE2 ||
+			      val == LIRC_MODE_SCANCODE))
+				ret = -EINVAL;
 			break;
 		}
 
-		result = get_user(mode, (__u32 __user *)arg);
-		if (!result && !(LIRC_MODE2REC(mode) & d->features))
-			result = -EINVAL;
-		/*
-		 * FIXME: We should actually set the mode somehow but
-		 * for now, lirc_serial doesn't support mode changing either
-		 */
+		if (!ret)
+			fh->rec_mode = val;
+		break;
+
+	case LIRC_GET_SEND_MODE:
+		if (!dev->tx_ir)
+			ret = -ENOTTY;
+		else
+			val = fh->send_mode;
+		break;
+
+	case LIRC_SET_SEND_MODE:
+		if (!dev->tx_ir)
+			ret = -ENOTTY;
+		else if (!(val == LIRC_MODE_PULSE || val == LIRC_MODE_SCANCODE))
+			ret = -EINVAL;
+		else
+			fh->send_mode = val;
+		break;
+
+	/* TX settings */
+	case LIRC_SET_TRANSMITTER_MASK:
+		if (!dev->s_tx_mask)
+			ret = -ENOTTY;
+		else
+			ret = dev->s_tx_mask(dev, val);
+		break;
+
+	case LIRC_SET_SEND_CARRIER:
+		if (!dev->s_tx_carrier)
+			ret = -ENOTTY;
+		else
+			ret = dev->s_tx_carrier(dev, val);
+		break;
+
+	case LIRC_SET_SEND_DUTY_CYCLE:
+		if (!dev->s_tx_duty_cycle)
+			ret = -ENOTTY;
+		else if (val <= 0 || val >= 100)
+			ret = -EINVAL;
+		else
+			ret = dev->s_tx_duty_cycle(dev, val);
+		break;
+
+	/* RX settings */
+	case LIRC_SET_REC_CARRIER:
+		if (!dev->s_rx_carrier_range)
+			ret = -ENOTTY;
+		else if (val <= 0)
+			ret = -EINVAL;
+		else
+			ret = dev->s_rx_carrier_range(dev, fh->carrier_low,
+						      val);
+		break;
+
+	case LIRC_SET_REC_CARRIER_RANGE:
+		if (!dev->s_rx_carrier_range)
+			ret = -ENOTTY;
+		else if (val <= 0)
+			ret = -EINVAL;
+		else
+			fh->carrier_low = val;
+		break;
+
+	case LIRC_GET_REC_RESOLUTION:
+		if (!dev->rx_resolution)
+			ret = -ENOTTY;
+		else
+			val = dev->rx_resolution / 1000;
 		break;
-	case LIRC_GET_LENGTH:
-		result = put_user(d->code_length, (__u32 __user *)arg);
+
+	case LIRC_SET_WIDEBAND_RECEIVER:
+		if (!dev->s_learning_mode)
+			ret = -ENOTTY;
+		else
+			ret = dev->s_learning_mode(dev, !!val);
 		break;
+
+	case LIRC_SET_MEASURE_CARRIER_MODE:
+		if (!dev->s_carrier_report)
+			ret = -ENOTTY;
+		else
+			ret = dev->s_carrier_report(dev, !!val);
+		break;
+
+	/* Generic timeout support */
+	case LIRC_GET_MIN_TIMEOUT:
+		if (!dev->max_timeout)
+			ret = -ENOTTY;
+		else
+			val = DIV_ROUND_UP(dev->min_timeout, 1000);
+		break;
+
+	case LIRC_GET_MAX_TIMEOUT:
+		if (!dev->max_timeout)
+			ret = -ENOTTY;
+		else
+			val = dev->max_timeout / 1000;
+		break;
+
+	case LIRC_SET_REC_TIMEOUT:
+		if (!dev->max_timeout) {
+			ret = -ENOTTY;
+		} else if (val > U32_MAX / 1000) {
+			/* Check for multiply overflow */
+			ret = -EINVAL;
+		} else {
+			u32 tmp = val * 1000;
+
+			if (tmp < dev->min_timeout || tmp > dev->max_timeout)
+				ret = -EINVAL;
+			else if (dev->s_timeout)
+				ret = dev->s_timeout(dev, tmp);
+			else if (!ret)
+				dev->timeout = tmp;
+		}
+		break;
+
+	case LIRC_SET_REC_TIMEOUT_REPORTS:
+		if (!dev->timeout)
+			ret = -ENOTTY;
+		else
+			fh->send_timeout_reports = !!val;
+		break;
+
 	default:
-		result = -ENOTTY;
+		ret = -ENOTTY;
 	}
 
+	if (!ret && _IOC_DIR(cmd) & _IOC_READ)
+		ret = put_user(val, argp);
+
 out:
-	mutex_unlock(&d->mutex);
-	return result;
+	mutex_unlock(&dev->lock);
+	return ret;
 }
-EXPORT_SYMBOL(lirc_dev_fop_ioctl);
 
-ssize_t lirc_dev_fop_read(struct file *file,
-			  char __user *buffer,
-			  size_t length,
-			  loff_t *ppos)
+static __poll_t ir_lirc_poll(struct file *file, struct poll_table_struct *wait)
 {
-	struct lirc_dev *d = file->private_data;
-	unsigned char *buf;
-	int ret, written = 0;
-	DECLARE_WAITQUEUE(wait, current);
+	struct lirc_fh *fh = file->private_data;
+	struct rc_dev *rcdev = fh->rc;
+	__poll_t events = 0;
+
+	poll_wait(file, &fh->wait_poll, wait);
+
+	if (!rcdev->registered) {
+		events = EPOLLHUP | EPOLLERR;
+	} else if (rcdev->driver_type != RC_DRIVER_IR_RAW_TX) {
+		if (fh->rec_mode == LIRC_MODE_SCANCODE &&
+		    !kfifo_is_empty(&fh->scancodes))
+			events = EPOLLIN | EPOLLRDNORM;
+
+		if (fh->rec_mode == LIRC_MODE_MODE2 &&
+		    !kfifo_is_empty(&fh->rawir))
+			events = EPOLLIN | EPOLLRDNORM;
+	}
 
-	buf = kzalloc(d->buf->chunk_size, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
+	return events;
+}
 
-	dev_dbg(&d->dev, LOGHEAD "read called\n", d->name, d->minor);
+static ssize_t ir_lirc_read_mode2(struct file *file, char __user *buffer,
+				  size_t length)
+{
+	struct lirc_fh *fh = file->private_data;
+	struct rc_dev *rcdev = fh->rc;
+	unsigned int copied;
+	int ret;
 
-	ret = mutex_lock_interruptible(&d->mutex);
-	if (ret) {
-		kfree(buf);
-		return ret;
-	}
+	if (length < sizeof(unsigned int) || length % sizeof(unsigned int))
+		return -EINVAL;
 
-	if (!d->attached) {
-		ret = -ENODEV;
-		goto out_locked;
-	}
+	do {
+		if (kfifo_is_empty(&fh->rawir)) {
+			if (file->f_flags & O_NONBLOCK)
+				return -EAGAIN;
 
-	if (!LIRC_CAN_REC(d->features)) {
-		ret = -EINVAL;
-		goto out_locked;
-	}
+			ret = wait_event_interruptible(fh->wait_poll,
+					!kfifo_is_empty(&fh->rawir) ||
+					!rcdev->registered);
+			if (ret)
+				return ret;
+		}
 
-	if (length % d->buf->chunk_size) {
-		ret = -EINVAL;
-		goto out_locked;
-	}
+		if (!rcdev->registered)
+			return -ENODEV;
 
-	/*
-	 * we add ourselves to the task queue before buffer check
-	 * to avoid losing scan code (in case when queue is awaken somewhere
-	 * between while condition checking and scheduling)
-	 */
-	add_wait_queue(&d->buf->wait_poll, &wait);
+		ret = mutex_lock_interruptible(&rcdev->lock);
+		if (ret)
+			return ret;
+		ret = kfifo_to_user(&fh->rawir, buffer, length, &copied);
+		mutex_unlock(&rcdev->lock);
+		if (ret)
+			return ret;
+	} while (copied == 0);
 
-	/*
-	 * while we didn't provide 'length' bytes, device is opened in blocking
-	 * mode and 'copy_to_user' is happy, wait for data.
-	 */
-	while (written < length && ret == 0) {
-		if (lirc_buffer_empty(d->buf)) {
-			/* According to the read(2) man page, 'written' can be
-			 * returned as less than 'length', instead of blocking
-			 * again, returning -EWOULDBLOCK, or returning
-			 * -ERESTARTSYS
-			 */
-			if (written)
-				break;
-			if (file->f_flags & O_NONBLOCK) {
-				ret = -EWOULDBLOCK;
-				break;
-			}
-			if (signal_pending(current)) {
-				ret = -ERESTARTSYS;
-				break;
-			}
-
-			mutex_unlock(&d->mutex);
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule();
-			set_current_state(TASK_RUNNING);
-
-			ret = mutex_lock_interruptible(&d->mutex);
-			if (ret) {
-				remove_wait_queue(&d->buf->wait_poll, &wait);
-				goto out_unlocked;
-			}
-
-			if (!d->attached) {
-				ret = -ENODEV;
-				goto out_locked;
-			}
-		} else {
-			lirc_buffer_read(d->buf, buf);
-			ret = copy_to_user((void __user *)buffer+written, buf,
-					   d->buf->chunk_size);
-			if (!ret)
-				written += d->buf->chunk_size;
-			else
-				ret = -EFAULT;
+	return copied;
+}
+
+static ssize_t ir_lirc_read_scancode(struct file *file, char __user *buffer,
+				     size_t length)
+{
+	struct lirc_fh *fh = file->private_data;
+	struct rc_dev *rcdev = fh->rc;
+	unsigned int copied;
+	int ret;
+
+	if (length < sizeof(struct lirc_scancode) ||
+	    length % sizeof(struct lirc_scancode))
+		return -EINVAL;
+
+	do {
+		if (kfifo_is_empty(&fh->scancodes)) {
+			if (file->f_flags & O_NONBLOCK)
+				return -EAGAIN;
+
+			ret = wait_event_interruptible(fh->wait_poll,
+					!kfifo_is_empty(&fh->scancodes) ||
+					!rcdev->registered);
+			if (ret)
+				return ret;
 		}
-	}
 
-	remove_wait_queue(&d->buf->wait_poll, &wait);
+		if (!rcdev->registered)
+			return -ENODEV;
+
+		ret = mutex_lock_interruptible(&rcdev->lock);
+		if (ret)
+			return ret;
+		ret = kfifo_to_user(&fh->scancodes, buffer, length, &copied);
+		mutex_unlock(&rcdev->lock);
+		if (ret)
+			return ret;
+	} while (copied == 0);
+
+	return copied;
+}
 
-out_locked:
-	mutex_unlock(&d->mutex);
+static ssize_t ir_lirc_read(struct file *file, char __user *buffer,
+			    size_t length, loff_t *ppos)
+{
+	struct lirc_fh *fh = file->private_data;
+	struct rc_dev *rcdev = fh->rc;
+
+	if (rcdev->driver_type == RC_DRIVER_IR_RAW_TX)
+		return -EINVAL;
 
-out_unlocked:
-	kfree(buf);
+	if (!rcdev->registered)
+		return -ENODEV;
 
-	return ret ? ret : written;
+	if (fh->rec_mode == LIRC_MODE_MODE2)
+		return ir_lirc_read_mode2(file, buffer, length);
+	else /* LIRC_MODE_SCANCODE */
+		return ir_lirc_read_scancode(file, buffer, length);
 }
-EXPORT_SYMBOL(lirc_dev_fop_read);
 
-void lirc_init_pdata(struct inode *inode, struct file *file)
+static const struct file_operations lirc_fops = {
+	.owner		= THIS_MODULE,
+	.write		= ir_lirc_transmit_ir,
+	.unlocked_ioctl	= ir_lirc_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= ir_lirc_ioctl,
+#endif
+	.read		= ir_lirc_read,
+	.poll		= ir_lirc_poll,
+	.open		= ir_lirc_open,
+	.release	= ir_lirc_close,
+	.llseek		= no_llseek,
+};
+
+static void lirc_release_device(struct device *ld)
 {
-	struct lirc_dev *d = container_of(inode->i_cdev, struct lirc_dev, cdev);
+	struct rc_dev *rcdev = container_of(ld, struct rc_dev, lirc_dev);
 
-	file->private_data = d;
+	put_device(&rcdev->dev);
 }
-EXPORT_SYMBOL(lirc_init_pdata);
 
-void *lirc_get_pdata(struct file *file)
+int ir_lirc_register(struct rc_dev *dev)
 {
-	struct lirc_dev *d = file->private_data;
+	int err, minor;
+
+	minor = ida_simple_get(&lirc_ida, 0, RC_DEV_MAX, GFP_KERNEL);
+	if (minor < 0)
+		return minor;
+
+	device_initialize(&dev->lirc_dev);
+	dev->lirc_dev.class = lirc_class;
+	dev->lirc_dev.parent = &dev->dev;
+	dev->lirc_dev.release = lirc_release_device;
+	dev->lirc_dev.devt = MKDEV(MAJOR(lirc_base_dev), minor);
+	dev_set_name(&dev->lirc_dev, "lirc%d", minor);
 
-	return d->data;
+	INIT_LIST_HEAD(&dev->lirc_fh);
+	spin_lock_init(&dev->lirc_fh_lock);
+
+	cdev_init(&dev->lirc_cdev, &lirc_fops);
+
+	err = cdev_device_add(&dev->lirc_cdev, &dev->lirc_dev);
+	if (err)
+		goto out_ida;
+
+	get_device(&dev->dev);
+
+	dev_info(&dev->dev, "lirc_dev: driver %s registered at minor = %d",
+		 dev->driver_name, minor);
+
+	return 0;
+
+out_ida:
+	ida_simple_remove(&lirc_ida, minor);
+	return err;
 }
-EXPORT_SYMBOL(lirc_get_pdata);
 
+void ir_lirc_unregister(struct rc_dev *dev)
+{
+	unsigned long flags;
+	struct lirc_fh *fh;
+
+	dev_dbg(&dev->dev, "lirc_dev: driver %s unregistered from minor = %d\n",
+		dev->driver_name, MINOR(dev->lirc_dev.devt));
+
+	spin_lock_irqsave(&dev->lirc_fh_lock, flags);
+	list_for_each_entry(fh, &dev->lirc_fh, list)
+		wake_up_poll(&fh->wait_poll, EPOLLHUP | EPOLLERR);
+	spin_unlock_irqrestore(&dev->lirc_fh_lock, flags);
 
-static int __init lirc_dev_init(void)
+	cdev_device_del(&dev->lirc_cdev, &dev->lirc_dev);
+	ida_simple_remove(&lirc_ida, MINOR(dev->lirc_dev.devt));
+}
+
+int __init lirc_dev_init(void)
 {
 	int retval;
 
@@ -487,7 +796,7 @@ static int __init lirc_dev_init(void)
 		return PTR_ERR(lirc_class);
 	}
 
-	retval = alloc_chrdev_region(&lirc_base_dev, 0, LIRC_MAX_DEVICES,
+	retval = alloc_chrdev_region(&lirc_base_dev, 0, RC_DEV_MAX,
 				     "BaseRemoteCtl");
 	if (retval) {
 		class_destroy(lirc_class);
@@ -501,16 +810,10 @@ static int __init lirc_dev_init(void)
 	return 0;
 }
 
-static void __exit lirc_dev_exit(void)
+void __exit lirc_dev_exit(void)
 {
 	class_destroy(lirc_class);
-	unregister_chrdev_region(lirc_base_dev, LIRC_MAX_DEVICES);
-	pr_info("module unloaded\n");
+	unregister_chrdev_region(lirc_base_dev, RC_DEV_MAX);
 }
 
-module_init(lirc_dev_init);
-module_exit(lirc_dev_exit);
-
-MODULE_DESCRIPTION("LIRC base driver module");
-MODULE_AUTHOR("Artur Lipowski");
-MODULE_LICENSE("GPL");
+MODULE_ALIAS("lirc_dev");
diff --git a/drivers/media/rc/rc-core-priv.h b/drivers/media/rc/rc-core-priv.h
index ae4dd0c27731..458e9eb2d6a9 100644
--- a/drivers/media/rc/rc-core-priv.h
+++ b/drivers/media/rc/rc-core-priv.h
@@ -1,27 +1,34 @@
 /*
+ * SPDX-License-Identifier: GPL-2.0
  * Remote Controller core raw events header
  *
  * Copyright (C) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
  */
 
 #ifndef _RC_CORE_PRIV
 #define _RC_CORE_PRIV
 
+#define	RC_DEV_MAX		256
 /* Define the max number of pulse/space transitions to buffer */
 #define	MAX_IR_EVENT_SIZE	512
 
 #include <linux/slab.h>
 #include <media/rc-core.h>
 
+/**
+ * rc_open - Opens a RC device
+ *
+ * @rdev: pointer to struct rc_dev.
+ */
+int rc_open(struct rc_dev *rdev);
+
+/**
+ * rc_close - Closes a RC device
+ *
+ * @rdev: pointer to struct rc_dev.
+ */
+void rc_close(struct rc_dev *rdev);
+
 struct ir_raw_handler {
 	struct list_head list;
 
@@ -29,8 +36,9 @@ struct ir_raw_handler {
 	int (*decode)(struct rc_dev *dev, struct ir_raw_event event);
 	int (*encode)(enum rc_proto protocol, u32 scancode,
 		      struct ir_raw_event *events, unsigned int max);
+	u32 carrier;
 
-	/* These two should only be used by the lirc decoder */
+	/* These two should only be used by the mce kbd decoder */
 	int (*raw_register)(struct rc_dev *dev);
 	int (*raw_unregister)(struct rc_dev *dev);
 };
@@ -104,17 +112,6 @@ struct ir_raw_event_ctrl {
 		unsigned count;
 		unsigned wanted_bits;
 	} mce_kbd;
-	struct lirc_codec {
-		struct rc_dev *dev;
-		struct lirc_dev *ldev;
-		int carrier_low;
-
-		ktime_t gap_start;
-		u64 gap_duration;
-		bool gap;
-		bool send_timeout_reports;
-
-	} lirc;
 	struct xmp_dec {
 		int state;
 		unsigned count;
@@ -156,6 +153,7 @@ static inline bool is_timing_event(struct ir_raw_event ev)
 #define TO_STR(is_pulse)		((is_pulse) ? "pulse" : "space")
 
 /* functions for IR encoders */
+bool rc_validate_scancode(enum rc_proto proto, u32 scancode);
 
 static inline void init_ir_raw_event_duration(struct ir_raw_event *ev,
 					      unsigned int pulse,
@@ -168,17 +166,17 @@ static inline void init_ir_raw_event_duration(struct ir_raw_event *ev,
 
 /**
  * struct ir_raw_timings_manchester - Manchester coding timings
- * @leader:		duration of leader pulse (if any) 0 if continuing
- *			existing signal (see @pulse_space_start)
- * @pulse_space_start:	1 for starting with pulse (0 for starting with space)
+ * @leader_pulse:	duration of leader pulse (if any) 0 if continuing
+ *			existing signal
+ * @leader_space:	duration of leader space (if any)
  * @clock:		duration of each pulse/space in ns
  * @invert:		if set clock logic is inverted
  *			(0 = space + pulse, 1 = pulse + space)
  * @trailer_space:	duration of trailer space in ns
  */
 struct ir_raw_timings_manchester {
-	unsigned int leader;
-	unsigned int pulse_space_start:1;
+	unsigned int leader_pulse;
+	unsigned int leader_space;
 	unsigned int clock;
 	unsigned int invert:1;
 	unsigned int trailer_space;
@@ -270,8 +268,30 @@ void ir_raw_event_free(struct rc_dev *dev);
 void ir_raw_event_unregister(struct rc_dev *dev);
 int ir_raw_handler_register(struct ir_raw_handler *ir_raw_handler);
 void ir_raw_handler_unregister(struct ir_raw_handler *ir_raw_handler);
+void ir_raw_load_modules(u64 *protocols);
 void ir_raw_init(void);
 
+/*
+ * lirc interface
+ */
+#ifdef CONFIG_LIRC
+int lirc_dev_init(void);
+void lirc_dev_exit(void);
+void ir_lirc_raw_event(struct rc_dev *dev, struct ir_raw_event ev);
+void ir_lirc_scancode_event(struct rc_dev *dev, struct lirc_scancode *lsc);
+int ir_lirc_register(struct rc_dev *dev);
+void ir_lirc_unregister(struct rc_dev *dev);
+#else
+static inline int lirc_dev_init(void) { return 0; }
+static inline void lirc_dev_exit(void) {}
+static inline void ir_lirc_raw_event(struct rc_dev *dev,
+				     struct ir_raw_event ev) { }
+static inline void ir_lirc_scancode_event(struct rc_dev *dev,
+					  struct lirc_scancode *lsc) { }
+static inline int ir_lirc_register(struct rc_dev *dev) { return 0; }
+static inline void ir_lirc_unregister(struct rc_dev *dev) { }
+#endif
+
 /*
  * Decoder initialization code
  *
diff --git a/drivers/media/rc/rc-ir-raw.c b/drivers/media/rc/rc-ir-raw.c
index d78483a504c9..18504870b9f0 100644
--- a/drivers/media/rc/rc-ir-raw.c
+++ b/drivers/media/rc/rc-ir-raw.c
@@ -1,16 +1,7 @@
-/* rc-ir-raw.c - handle IR pulse/space events
- *
- * Copyright (C) 2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// rc-ir-raw.c - handle IR pulse/space events
+//
+// Copyright (C) 2010 by Mauro Carvalho Chehab
 
 #include <linux/export.h>
 #include <linux/kthread.h>
@@ -40,6 +31,7 @@ static int ir_raw_event_thread(void *data)
 				if (raw->dev->enabled_protocols &
 				    handler->protocols || !handler->protocols)
 					handler->decode(raw->dev, ev);
+			ir_lirc_raw_event(raw->dev, ev);
 			raw->prev_ev = ev;
 		}
 		mutex_unlock(&ir_raw_handler_lock);
@@ -254,19 +246,16 @@ int ir_raw_gen_manchester(struct ir_raw_event **ev, unsigned int max,
 
 	i = BIT_ULL(n - 1);
 
-	if (timings->leader) {
+	if (timings->leader_pulse) {
 		if (!max--)
 			return ret;
-		if (timings->pulse_space_start) {
-			init_ir_raw_event_duration((*ev)++, 1, timings->leader);
-
+		init_ir_raw_event_duration((*ev), 1, timings->leader_pulse);
+		if (timings->leader_space) {
 			if (!max--)
 				return ret;
-			init_ir_raw_event_duration((*ev), 0, timings->leader);
-		} else {
-			init_ir_raw_event_duration((*ev), 1, timings->leader);
+			init_ir_raw_event_duration(++(*ev), 0,
+						   timings->leader_space);
 		}
-		i >>= 1;
 	} else {
 		/* continue existing signal */
 		--(*ev);
@@ -457,6 +446,8 @@ int ir_raw_encode_scancode(enum rc_proto protocol, u32 scancode,
 	int ret = -EINVAL;
 	u64 mask = 1ULL << protocol;
 
+	ir_raw_load_modules(&mask);
+
 	mutex_lock(&ir_raw_handler_lock);
 	list_for_each_entry(handler, &ir_raw_handler_list, list) {
 		if (handler->protocols & mask && handler->encode) {
@@ -493,21 +484,44 @@ static void edge_handle(struct timer_list *t)
 	ir_raw_event_handle(dev);
 }
 
+/**
+ * ir_raw_encode_carrier() - Get carrier used for protocol
+ *
+ * @protocol:		protocol
+ *
+ * Attempts to find the carrier for the specified protocol
+ *
+ * Returns:	The carrier in Hz
+ *		-EINVAL if the protocol is invalid, or if no
+ *		compatible encoder was found.
+ */
+int ir_raw_encode_carrier(enum rc_proto protocol)
+{
+	struct ir_raw_handler *handler;
+	int ret = -EINVAL;
+	u64 mask = BIT_ULL(protocol);
+
+	mutex_lock(&ir_raw_handler_lock);
+	list_for_each_entry(handler, &ir_raw_handler_list, list) {
+		if (handler->protocols & mask && handler->encode) {
+			ret = handler->carrier;
+			break;
+		}
+	}
+	mutex_unlock(&ir_raw_handler_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(ir_raw_encode_carrier);
+
 /*
  * Used to (un)register raw event clients
  */
 int ir_raw_event_prepare(struct rc_dev *dev)
 {
-	static bool raw_init; /* 'false' default value, raw decoders loaded? */
-
 	if (!dev)
 		return -EINVAL;
 
-	if (!raw_init) {
-		request_module("ir-lirc-codec");
-		raw_init = true;
-	}
-
 	dev->raw = kzalloc(sizeof(*dev->raw), GFP_KERNEL);
 	if (!dev->raw)
 		return -ENOMEM;
@@ -525,19 +539,11 @@ int ir_raw_event_register(struct rc_dev *dev)
 	struct ir_raw_handler *handler;
 	struct task_struct *thread;
 
-	/*
-	 * raw transmitters do not need any event registration
-	 * because the event is coming from userspace
-	 */
-	if (dev->driver_type != RC_DRIVER_IR_RAW_TX) {
-		thread = kthread_run(ir_raw_event_thread, dev->raw, "rc%u",
-				     dev->minor);
+	thread = kthread_run(ir_raw_event_thread, dev->raw, "rc%u", dev->minor);
+	if (IS_ERR(thread))
+		return PTR_ERR(thread);
 
-		if (IS_ERR(thread))
-			return PTR_ERR(thread);
-
-		dev->raw->thread = thread;
-	}
+	dev->raw->thread = thread;
 
 	mutex_lock(&ir_raw_handler_lock);
 	list_add_tail(&dev->raw->list, &ir_raw_client_list);
diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c
index c144b77eac98..1db8d38fed7c 100644
--- a/drivers/media/rc/rc-main.c
+++ b/drivers/media/rc/rc-main.c
@@ -1,16 +1,7 @@
-/* rc-main.c - Remote Controller core module
- *
- * Copyright (C) 2009-2010 by Mauro Carvalho Chehab
- *
- * This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// rc-main.c - Remote Controller core module
+//
+// Copyright (C) 2009-2010 by Mauro Carvalho Chehab
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
@@ -29,7 +20,6 @@
 /* Sizes are in bytes, 256 bytes allows for 32 entries on x64 */
 #define IR_TAB_MIN_SIZE	256
 #define IR_TAB_MAX_SIZE	8192
-#define RC_DEV_MAX	256
 
 static const struct {
 	const char *name;
@@ -607,6 +597,7 @@ static void ir_do_keyup(struct rc_dev *dev, bool sync)
 		return;
 
 	IR_dprintk(1, "keyup key 0x%04x\n", dev->last_keycode);
+	del_timer(&dev->timer_repeat);
 	input_report_key(dev->input_dev, dev->last_keycode, 0);
 	led_trigger_event(led_feedback, LED_OFF);
 	if (sync)
@@ -660,6 +651,31 @@ static void ir_timer_keyup(struct timer_list *t)
 	spin_unlock_irqrestore(&dev->keylock, flags);
 }
 
+/**
+ * ir_timer_repeat() - generates a repeat event after a timeout
+ *
+ * @t:		a pointer to the struct timer_list
+ *
+ * This routine will generate a soft repeat event every REP_PERIOD
+ * milliseconds.
+ */
+static void ir_timer_repeat(struct timer_list *t)
+{
+	struct rc_dev *dev = from_timer(dev, t, timer_repeat);
+	struct input_dev *input = dev->input_dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->keylock, flags);
+	if (dev->keypressed) {
+		input_event(input, EV_KEY, dev->last_keycode, 2);
+		input_sync(input);
+		if (input->rep[REP_PERIOD])
+			mod_timer(&dev->timer_repeat, jiffies +
+				  msecs_to_jiffies(input->rep[REP_PERIOD]));
+	}
+	spin_unlock_irqrestore(&dev->keylock, flags);
+}
+
 /**
  * rc_repeat() - signals that a key is still pressed
  * @dev:	the struct rc_dev descriptor of the device
@@ -672,19 +688,25 @@ void rc_repeat(struct rc_dev *dev)
 {
 	unsigned long flags;
 	unsigned int timeout = protocols[dev->last_protocol].repeat_period;
+	struct lirc_scancode sc = {
+		.scancode = dev->last_scancode, .rc_proto = dev->last_protocol,
+		.keycode = dev->keypressed ? dev->last_keycode : KEY_RESERVED,
+		.flags = LIRC_SCANCODE_FLAG_REPEAT |
+			 (dev->last_toggle ? LIRC_SCANCODE_FLAG_TOGGLE : 0)
+	};
 
-	spin_lock_irqsave(&dev->keylock, flags);
+	ir_lirc_scancode_event(dev, &sc);
 
-	if (!dev->keypressed)
-		goto out;
+	spin_lock_irqsave(&dev->keylock, flags);
 
 	input_event(dev->input_dev, EV_MSC, MSC_SCAN, dev->last_scancode);
 	input_sync(dev->input_dev);
 
-	dev->keyup_jiffies = jiffies + msecs_to_jiffies(timeout);
-	mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
+	if (dev->keypressed) {
+		dev->keyup_jiffies = jiffies + msecs_to_jiffies(timeout);
+		mod_timer(&dev->timer_keyup, dev->keyup_jiffies);
+	}
 
-out:
 	spin_unlock_irqrestore(&dev->keylock, flags);
 }
 EXPORT_SYMBOL_GPL(rc_repeat);
@@ -707,19 +729,27 @@ static void ir_do_keydown(struct rc_dev *dev, enum rc_proto protocol,
 			  dev->last_protocol != protocol ||
 			  dev->last_scancode != scancode ||
 			  dev->last_toggle   != toggle);
+	struct lirc_scancode sc = {
+		.scancode = scancode, .rc_proto = protocol,
+		.flags = toggle ? LIRC_SCANCODE_FLAG_TOGGLE : 0,
+		.keycode = keycode
+	};
+
+	ir_lirc_scancode_event(dev, &sc);
 
 	if (new_event && dev->keypressed)
 		ir_do_keyup(dev, false);
 
 	input_event(dev->input_dev, EV_MSC, MSC_SCAN, scancode);
 
+	dev->last_protocol = protocol;
+	dev->last_scancode = scancode;
+	dev->last_toggle = toggle;
+	dev->last_keycode = keycode;
+
 	if (new_event && keycode != KEY_RESERVED) {
 		/* Register a keypress */
 		dev->keypressed = true;
-		dev->last_protocol = protocol;
-		dev->last_scancode = scancode;
-		dev->last_toggle = toggle;
-		dev->last_keycode = keycode;
 
 		IR_dprintk(1, "%s: key down event, key 0x%04x, protocol 0x%04x, scancode 0x%08x\n",
 			   dev->device_name, keycode, protocol, scancode);
@@ -728,6 +758,22 @@ static void ir_do_keydown(struct rc_dev *dev, enum rc_proto protocol,
 		led_trigger_event(led_feedback, LED_FULL);
 	}
 
+	/*
+	 * For CEC, start sending repeat messages as soon as the first
+	 * repeated message is sent, as long as REP_DELAY = 0 and REP_PERIOD
+	 * is non-zero. Otherwise, the input layer will generate repeat
+	 * messages.
+	 */
+	if (!new_event && keycode != KEY_RESERVED &&
+	    dev->allowed_protocols == RC_PROTO_BIT_CEC &&
+	    !timer_pending(&dev->timer_repeat) &&
+	    dev->input_dev->rep[REP_PERIOD] &&
+	    !dev->input_dev->rep[REP_DELAY]) {
+		input_event(dev->input_dev, EV_KEY, keycode, 2);
+		mod_timer(&dev->timer_repeat, jiffies +
+			  msecs_to_jiffies(dev->input_dev->rep[REP_PERIOD]));
+	}
+
 	input_sync(dev->input_dev);
 }
 
@@ -784,6 +830,51 @@ void rc_keydown_notimeout(struct rc_dev *dev, enum rc_proto protocol,
 }
 EXPORT_SYMBOL_GPL(rc_keydown_notimeout);
 
+/**
+ * rc_validate_scancode() - checks that a scancode is valid for a protocol.
+ *	For nec, it should do the opposite of ir_nec_bytes_to_scancode()
+ * @proto:	protocol
+ * @scancode:	scancode
+ */
+bool rc_validate_scancode(enum rc_proto proto, u32 scancode)
+{
+	switch (proto) {
+	/*
+	 * NECX has a 16-bit address; if the lower 8 bits match the upper
+	 * 8 bits inverted, then the address would match regular nec.
+	 */
+	case RC_PROTO_NECX:
+		if ((((scancode >> 16) ^ ~(scancode >> 8)) & 0xff) == 0)
+			return false;
+		break;
+	/*
+	 * NEC32 has a 16 bit address and 16 bit command. If the lower 8 bits
+	 * of the command match the upper 8 bits inverted, then it would
+	 * be either NEC or NECX.
+	 */
+	case RC_PROTO_NEC32:
+		if ((((scancode >> 8) ^ ~scancode) & 0xff) == 0)
+			return false;
+		break;
+	/*
+	 * If the customer code (top 32-bit) is 0x800f, it is MCE else it
+	 * is regular mode-6a 32 bit
+	 */
+	case RC_PROTO_RC6_MCE:
+		if ((scancode & 0xffff0000) != 0x800f0000)
+			return false;
+		break;
+	case RC_PROTO_RC6_6A_32:
+		if ((scancode & 0xffff0000) == 0x800f0000)
+			return false;
+		break;
+	default:
+		break;
+	}
+
+	return true;
+}
+
 /**
  * rc_validate_filter() - checks that the scancode and mask are valid and
  *			  provides sensible defaults
@@ -803,26 +894,8 @@ static int rc_validate_filter(struct rc_dev *dev,
 
 	mask = protocols[protocol].scancode_bits;
 
-	switch (protocol) {
-	case RC_PROTO_NECX:
-		if ((((s >> 16) ^ ~(s >> 8)) & 0xff) == 0)
-			return -EINVAL;
-		break;
-	case RC_PROTO_NEC32:
-		if ((((s >> 24) ^ ~(s >> 16)) & 0xff) == 0)
-			return -EINVAL;
-		break;
-	case RC_PROTO_RC6_MCE:
-		if ((s & 0xffff0000) != 0x800f0000)
-			return -EINVAL;
-		break;
-	case RC_PROTO_RC6_6A_32:
-		if ((s & 0xffff0000) == 0x800f0000)
-			return -EINVAL;
-		break;
-	default:
-		break;
-	}
+	if (!rc_validate_scancode(protocol, s))
+		return -EINVAL;
 
 	filter->data &= mask;
 	filter->mask &= mask;
@@ -845,17 +918,20 @@ int rc_open(struct rc_dev *rdev)
 
 	mutex_lock(&rdev->lock);
 
-	if (!rdev->users++ && rdev->open != NULL)
-		rval = rdev->open(rdev);
+	if (!rdev->registered) {
+		rval = -ENODEV;
+	} else {
+		if (!rdev->users++ && rdev->open)
+			rval = rdev->open(rdev);
 
-	if (rval)
-		rdev->users--;
+		if (rval)
+			rdev->users--;
+	}
 
 	mutex_unlock(&rdev->lock);
 
 	return rval;
 }
-EXPORT_SYMBOL_GPL(rc_open);
 
 static int ir_open(struct input_dev *idev)
 {
@@ -869,13 +945,12 @@ void rc_close(struct rc_dev *rdev)
 	if (rdev) {
 		mutex_lock(&rdev->lock);
 
-		if (!--rdev->users && rdev->close != NULL)
+		if (!--rdev->users && rdev->close && rdev->registered)
 			rdev->close(rdev);
 
 		mutex_unlock(&rdev->lock);
 	}
 }
-EXPORT_SYMBOL_GPL(rc_close);
 
 static void ir_close(struct input_dev *idev)
 {
@@ -950,23 +1025,6 @@ struct rc_filter_attribute {
 		.mask = (_mask),					\
 	}
 
-static bool lirc_is_present(void)
-{
-#if defined(CONFIG_LIRC_MODULE)
-	struct module *lirc;
-
-	mutex_lock(&module_mutex);
-	lirc = find_module("lirc_dev");
-	mutex_unlock(&module_mutex);
-
-	return lirc ? true : false;
-#elif defined(CONFIG_LIRC)
-	return true;
-#else
-	return false;
-#endif
-}
-
 /**
  * show_protocols() - shows the current IR protocol(s)
  * @device:	the device descriptor
@@ -1011,8 +1069,10 @@ static ssize_t show_protocols(struct device *device,
 			allowed &= ~proto_names[i].type;
 	}
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW && lirc_is_present())
+#ifdef CONFIG_LIRC
+	if (dev->driver_type == RC_DRIVER_IR_RAW)
 		tmp += sprintf(tmp, "[lirc] ");
+#endif
 
 	if (tmp != buf)
 		tmp--;
@@ -1091,7 +1151,7 @@ static int parse_protocol_change(u64 *protocols, const char *buf)
 	return count;
 }
 
-static void ir_raw_load_modules(u64 *protocols)
+void ir_raw_load_modules(u64 *protocols)
 {
 	u64 available;
 	int i, ret;
@@ -1581,6 +1641,7 @@ struct rc_dev *rc_allocate_device(enum rc_driver_type type)
 		input_set_drvdata(dev->input_dev, dev);
 
 		timer_setup(&dev->timer_keyup, ir_timer_keyup, 0);
+		timer_setup(&dev->timer_repeat, ir_timer_repeat, 0);
 
 		spin_lock_init(&dev->rc_map.lock);
 		spin_lock_init(&dev->keylock);
@@ -1714,7 +1775,10 @@ static int rc_setup_rx_device(struct rc_dev *dev)
 	 * to avoid wrong repetition of the keycodes. Note that this must be
 	 * set after the call to input_register_device().
 	 */
-	dev->input_dev->rep[REP_DELAY] = 500;
+	if (dev->allowed_protocols == RC_PROTO_BIT_CEC)
+		dev->input_dev->rep[REP_DELAY] = 0;
+	else
+		dev->input_dev->rep[REP_DELAY] = 500;
 
 	/*
 	 * As a repeat event on protocols like RC-5 and NEC take as long as
@@ -1768,8 +1832,7 @@ int rc_register_device(struct rc_dev *dev)
 		dev->sysfs_groups[attr++] = &rc_dev_wakeup_filter_attr_grp;
 	dev->sysfs_groups[attr++] = NULL;
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW ||
-	    dev->driver_type == RC_DRIVER_IR_RAW_TX) {
+	if (dev->driver_type == RC_DRIVER_IR_RAW) {
 		rc = ir_raw_event_prepare(dev);
 		if (rc < 0)
 			goto out_minor;
@@ -1796,19 +1859,30 @@ int rc_register_device(struct rc_dev *dev)
 			goto out_dev;
 	}
 
-	if (dev->driver_type == RC_DRIVER_IR_RAW ||
-	    dev->driver_type == RC_DRIVER_IR_RAW_TX) {
-		rc = ir_raw_event_register(dev);
+	/* Ensure that the lirc kfifo is setup before we start the thread */
+	if (dev->allowed_protocols != RC_PROTO_BIT_CEC) {
+		rc = ir_lirc_register(dev);
 		if (rc < 0)
 			goto out_rx;
 	}
 
+	if (dev->driver_type == RC_DRIVER_IR_RAW) {
+		rc = ir_raw_event_register(dev);
+		if (rc < 0)
+			goto out_lirc;
+	}
+
+	dev->registered = true;
+
 	IR_dprintk(1, "Registered rc%u (driver: %s)\n",
 		   dev->minor,
 		   dev->driver_name ? dev->driver_name : "unknown");
 
 	return 0;
 
+out_lirc:
+	if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
+		ir_lirc_unregister(dev);
 out_rx:
 	rc_free_rx_device(dev);
 out_dev:
@@ -1856,12 +1930,24 @@ void rc_unregister_device(struct rc_dev *dev)
 		return;
 
 	del_timer_sync(&dev->timer_keyup);
+	del_timer_sync(&dev->timer_repeat);
 
 	if (dev->driver_type == RC_DRIVER_IR_RAW)
 		ir_raw_event_unregister(dev);
 
 	rc_free_rx_device(dev);
 
+	mutex_lock(&dev->lock);
+	dev->registered = false;
+	mutex_unlock(&dev->lock);
+
+	/*
+	 * lirc device should be freed with dev->registered = false, so
+	 * that userspace polling will get notified.
+	 */
+	if (dev->allowed_protocols != RC_PROTO_BIT_CEC)
+		ir_lirc_unregister(dev);
+
 	device_del(&dev->dev);
 
 	ida_simple_remove(&rc_ida, dev->minor);
@@ -1884,6 +1970,13 @@ static int __init rc_core_init(void)
 		return rc;
 	}
 
+	rc = lirc_dev_init();
+	if (rc) {
+		pr_err("rc_core: unable to init lirc\n");
+		class_unregister(&rc_class);
+		return 0;
+	}
+
 	led_trigger_register_simple("rc-feedback", &led_feedback);
 	rc_map_register(&empty_map);
 
@@ -1892,6 +1985,7 @@ static int __init rc_core_init(void)
 
 static void __exit rc_core_exit(void)
 {
+	lirc_dev_exit();
 	class_unregister(&rc_class);
 	led_trigger_unregister_simple(led_feedback);
 	rc_map_unregister(&empty_map);
@@ -1905,4 +1999,4 @@ EXPORT_SYMBOL_GPL(rc_core_debug);
 module_param_named(debug, rc_core_debug, int, 0644);
 
 MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/rc/winbond-cir.c b/drivers/media/rc/winbond-cir.c
index 3ca7ab48293d..0adf0991f5ab 100644
--- a/drivers/media/rc/winbond-cir.c
+++ b/drivers/media/rc/winbond-cir.c
@@ -1044,7 +1044,7 @@ wbcir_probe(struct pnp_dev *device, const struct pnp_device_id *dev_id)
 	data->irq = pnp_irq(device, 0);
 
 	if (data->wbase == 0 || data->ebase == 0 ||
-	    data->sbase == 0 || data->irq == 0) {
+	    data->sbase == 0 || data->irq == -1) {
 		err = -ENODEV;
 		dev_err(dev, "Invalid resources\n");
 		goto exit_free_data;
diff --git a/drivers/media/tuners/Kconfig b/drivers/media/tuners/Kconfig
index 05998f0254c6..6687514df97f 100644
--- a/drivers/media/tuners/Kconfig
+++ b/drivers/media/tuners/Kconfig
@@ -26,6 +26,13 @@ config MEDIA_TUNER_SIMPLE
 	help
 	  Say Y here to include support for various simple tuners.
 
+config MEDIA_TUNER_TDA18250
+	tristate "NXP TDA18250 silicon tuner"
+	depends on MEDIA_SUPPORT && I2C
+	default m if !MEDIA_SUBDRV_AUTOSELECT
+	help
+	  Say Y here to include support for TDA18250 tuner.
+
 config MEDIA_TUNER_TDA8290
 	tristate "TDA 8290/8295 + 8275(a)/18271 tuner combo"
 	depends on MEDIA_SUPPORT && I2C
diff --git a/drivers/media/tuners/Makefile b/drivers/media/tuners/Makefile
index 7be96511532d..0ff21f1c7eed 100644
--- a/drivers/media/tuners/Makefile
+++ b/drivers/media/tuners/Makefile
@@ -42,6 +42,6 @@ obj-$(CONFIG_MEDIA_TUNER_R820T) += r820t.o
 obj-$(CONFIG_MEDIA_TUNER_MXL301RF) += mxl301rf.o
 obj-$(CONFIG_MEDIA_TUNER_QM1D1C0042) += qm1d1c0042.o
 obj-$(CONFIG_MEDIA_TUNER_M88RS6000T) += m88rs6000t.o
+obj-$(CONFIG_MEDIA_TUNER_TDA18250) += tda18250.o
 
-ccflags-y += -I$(srctree)/drivers/media/dvb-core
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends
diff --git a/drivers/media/tuners/e4000.h b/drivers/media/tuners/e4000.h
index aa9340c05b43..9a65208c5bc3 100644
--- a/drivers/media/tuners/e4000.h
+++ b/drivers/media/tuners/e4000.h
@@ -21,7 +21,7 @@
 #ifndef E4000_H
 #define E4000_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /*
  * I2C address
diff --git a/drivers/media/tuners/fc0011.h b/drivers/media/tuners/fc0011.h
index a36871c44c8c..ebae37cc6f5f 100644
--- a/drivers/media/tuners/fc0011.h
+++ b/drivers/media/tuners/fc0011.h
@@ -2,7 +2,7 @@
 #ifndef LINUX_FC0011_H_
 #define LINUX_FC0011_H_
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 
 /** struct fc0011_config - fc0011 hardware config
diff --git a/drivers/media/tuners/fc0012.h b/drivers/media/tuners/fc0012.h
index 64d07a2adb2e..29e84c434de1 100644
--- a/drivers/media/tuners/fc0012.h
+++ b/drivers/media/tuners/fc0012.h
@@ -17,7 +17,7 @@
 #ifndef _FC0012_H_
 #define _FC0012_H_
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "fc001x-common.h"
 
 struct fc0012_config {
diff --git a/drivers/media/tuners/fc0013.h b/drivers/media/tuners/fc0013.h
index 4431e7ceb43d..2d039250c783 100644
--- a/drivers/media/tuners/fc0013.h
+++ b/drivers/media/tuners/fc0013.h
@@ -18,7 +18,7 @@
 #ifndef _FC0013_H_
 #define _FC0013_H_
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "fc001x-common.h"
 
 #if IS_REACHABLE(CONFIG_MEDIA_TUNER_FC0013)
diff --git a/drivers/media/tuners/fc2580.h b/drivers/media/tuners/fc2580.h
index 862ea46995d7..a04fba6b0b8a 100644
--- a/drivers/media/tuners/fc2580.h
+++ b/drivers/media/tuners/fc2580.h
@@ -21,7 +21,7 @@
 #ifndef FC2580_H
 #define FC2580_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include <media/v4l2-subdev.h>
 #include <linux/i2c.h>
 
diff --git a/drivers/media/tuners/it913x.h b/drivers/media/tuners/it913x.h
index 226f657228fb..3cb219a4a645 100644
--- a/drivers/media/tuners/it913x.h
+++ b/drivers/media/tuners/it913x.h
@@ -19,7 +19,7 @@
 #ifndef IT913X_H
 #define IT913X_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /**
  * struct it913x_platform_data - Platform data for the it913x driver
diff --git a/drivers/media/tuners/m88rs6000t.h b/drivers/media/tuners/m88rs6000t.h
index 264c13e2cd39..318b48c8f843 100644
--- a/drivers/media/tuners/m88rs6000t.h
+++ b/drivers/media/tuners/m88rs6000t.h
@@ -17,7 +17,7 @@
 #ifndef _M88RS6000T_H_
 #define _M88RS6000T_H_
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct m88rs6000t_config {
 	/*
diff --git a/drivers/media/tuners/max2165.c b/drivers/media/tuners/max2165.c
index a86c08114915..20ceb72e530b 100644
--- a/drivers/media/tuners/max2165.c
+++ b/drivers/media/tuners/max2165.c
@@ -23,7 +23,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "max2165.h"
 #include "max2165_priv.h"
diff --git a/drivers/media/tuners/mc44s803.c b/drivers/media/tuners/mc44s803.c
index 12f545ef1243..403c6b2aa53b 100644
--- a/drivers/media/tuners/mc44s803.c
+++ b/drivers/media/tuners/mc44s803.c
@@ -21,7 +21,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "mc44s803.h"
 #include "mc44s803_priv.h"
diff --git a/drivers/media/tuners/mt2060.c b/drivers/media/tuners/mt2060.c
index 4983eeb39f36..3d3c6815b6a7 100644
--- a/drivers/media/tuners/mt2060.c
+++ b/drivers/media/tuners/mt2060.c
@@ -23,7 +23,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "mt2060.h"
 #include "mt2060_priv.h"
diff --git a/drivers/media/tuners/mt2063.c b/drivers/media/tuners/mt2063.c
index 5c87c5c6a455..80dc3e241b4a 100644
--- a/drivers/media/tuners/mt2063.c
+++ b/drivers/media/tuners/mt2063.c
@@ -1168,7 +1168,7 @@ static u32 mt2063_set_dnc_output_enable(struct mt2063_state *state,
 
 /*
  * MT2063_SetReceiverMode() - Set the MT2063 receiver mode, according with
- * 			      the selected enum mt2063_delivery_sys type.
+ *			      the selected enum mt2063_delivery_sys type.
  *
  *  (DNC1GC & DNC2GC are the values, which are used, when the specific
  *   DNC Output is selected, the other is always off)
@@ -1544,7 +1544,7 @@ static u32 MT2063_Tune(struct mt2063_state *state, u32 f_in)
 	 * Save original LO1 and LO2 register values
 	 */
 	ofLO1 = state->AS_Data.f_LO1;
-	ofLO2 = state->AS_Data.f_LO2; 
+	ofLO2 = state->AS_Data.f_LO2;
 
 	/*
 	 * Find and set RF Band setting
diff --git a/drivers/media/tuners/mt2063.h b/drivers/media/tuners/mt2063.h
index 0e3e3b0525bb..30d03cd76061 100644
--- a/drivers/media/tuners/mt2063.h
+++ b/drivers/media/tuners/mt2063.h
@@ -2,7 +2,7 @@
 #ifndef __MT2063_H__
 #define __MT2063_H__
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct mt2063_config {
 	u8 tuner_address;
diff --git a/drivers/media/tuners/mt20xx.h b/drivers/media/tuners/mt20xx.h
index 9912362b415e..3cc41a57dca9 100644
--- a/drivers/media/tuners/mt20xx.h
+++ b/drivers/media/tuners/mt20xx.h
@@ -18,7 +18,7 @@
 #define __MT20XX_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #if IS_REACHABLE(CONFIG_MEDIA_TUNER_MT20XX)
 extern struct dvb_frontend *microtune_attach(struct dvb_frontend *fe,
diff --git a/drivers/media/tuners/mt2131.c b/drivers/media/tuners/mt2131.c
index dd85d58fa8d0..659bf19dc434 100644
--- a/drivers/media/tuners/mt2131.c
+++ b/drivers/media/tuners/mt2131.c
@@ -21,7 +21,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "mt2131.h"
 #include "mt2131_priv.h"
diff --git a/drivers/media/tuners/mt2266.c b/drivers/media/tuners/mt2266.c
index 88edcc031e3c..f4545b7f5da2 100644
--- a/drivers/media/tuners/mt2266.c
+++ b/drivers/media/tuners/mt2266.c
@@ -20,7 +20,7 @@
 #include <linux/i2c.h>
 #include <linux/slab.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mt2266.h"
 
 #define I2C_ADDRESS 0x60
diff --git a/drivers/media/tuners/mxl301rf.h b/drivers/media/tuners/mxl301rf.h
index 19e68405f00d..d32d4e8dc448 100644
--- a/drivers/media/tuners/mxl301rf.h
+++ b/drivers/media/tuners/mxl301rf.h
@@ -17,7 +17,7 @@
 #ifndef MXL301RF_H
 #define MXL301RF_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct mxl301rf_config {
 	struct dvb_frontend *fe;
diff --git a/drivers/media/tuners/mxl5005s.c b/drivers/media/tuners/mxl5005s.c
index dd59c2c0e4a5..355ef2959b7d 100644
--- a/drivers/media/tuners/mxl5005s.c
+++ b/drivers/media/tuners/mxl5005s.c
@@ -63,7 +63,7 @@
 #include <linux/string.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mxl5005s.h"
 
 static int debug;
@@ -1677,10 +1677,10 @@ static u16 MXL5005_TunerConfig(struct dvb_frontend *fe,
 	u8	AGC_Mode,	/* AGC Mode - Dual AGC: 0, Single AGC: 1 */
 	u16	TOP,		/* 0: Dual AGC; Value: take over point */
 	u16	IF_OUT_LOAD,	/* IF Out Load Resistor (200 / 300 Ohms) */
-	u8	CLOCK_OUT, 	/* 0: turn off clk out; 1: turn on clock out */
+	u8	CLOCK_OUT,	/* 0: turn off clk out; 1: turn on clock out */
 	u8	DIV_OUT,	/* 0: Div-1; 1: Div-4 */
-	u8	CAPSELECT, 	/* 0: disable On-Chip pulling cap; 1: enable */
-	u8	EN_RSSI, 	/* 0: disable RSSI; 1: enable RSSI */
+	u8	CAPSELECT,	/* 0: disable On-Chip pulling cap; 1: enable */
+	u8	EN_RSSI,	/* 0: disable RSSI; 1: enable RSSI */
 
 	/* Modulation Type; */
 	/* 0 - Default;	1 - DVB-T; 2 - ATSC; 3 - QAM; 4 - Analog Cable */
@@ -3591,10 +3591,11 @@ static u16 MXL_GetInitRegister(struct dvb_frontend *fe, u8 *RegNum,
 	u16 status = 0;
 	int i ;
 
-	u8 RegAddr[] = {
+	static const u8 RegAddr[] = {
 		11, 12, 13, 22, 32, 43, 44, 53, 56, 59, 73,
 		76, 77, 91, 134, 135, 137, 147,
-		156, 166, 167, 168, 25 };
+		156, 166, 167, 168, 25
+	};
 
 	*count = ARRAY_SIZE(RegAddr);
 
@@ -3616,11 +3617,15 @@ static u16 MXL_GetCHRegister(struct dvb_frontend *fe, u8 *RegNum, u8 *RegVal,
 
 /* add 77, 166, 167, 168 register for 2.6.12 */
 #ifdef _MXL_PRODUCTION
-	u8 RegAddr[] = {14, 15, 16, 17, 22, 43, 65, 68, 69, 70, 73, 92, 93, 106,
-	   107, 108, 109, 110, 111, 112, 136, 138, 149, 77, 166, 167, 168 } ;
+	static const u8 RegAddr[] = {
+		14, 15, 16, 17, 22, 43, 65, 68, 69, 70, 73, 92, 93, 106,
+		107, 108, 109, 110, 111, 112, 136, 138, 149, 77, 166, 167, 168
+	};
 #else
-	u8 RegAddr[] = {14, 15, 16, 17, 22, 43, 68, 69, 70, 73, 92, 93, 106,
-	   107, 108, 109, 110, 111, 112, 136, 138, 149, 77, 166, 167, 168 } ;
+	static const u8 RegAddr[] = {
+		14, 15, 16, 17, 22, 43, 68, 69, 70, 73, 92, 93, 106,
+		107, 108, 109, 110, 111, 112, 136, 138, 149, 77, 166, 167, 168
+	};
 	/*
 	u8 RegAddr[171];
 	for (i = 0; i <= 170; i++)
diff --git a/drivers/media/tuners/mxl5005s.h b/drivers/media/tuners/mxl5005s.h
index d842734f2dcd..9ac0811a162e 100644
--- a/drivers/media/tuners/mxl5005s.h
+++ b/drivers/media/tuners/mxl5005s.h
@@ -24,7 +24,7 @@
 #define __MXL5005S_H
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct mxl5005s_config {
 
diff --git a/drivers/media/tuners/mxl5007t.h b/drivers/media/tuners/mxl5007t.h
index 273f61aeb8be..f7f16b86fcae 100644
--- a/drivers/media/tuners/mxl5007t.h
+++ b/drivers/media/tuners/mxl5007t.h
@@ -17,7 +17,7 @@
 #ifndef __MXL5007T_H__
 #define __MXL5007T_H__
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /* ------------------------------------------------------------------------- */
 
diff --git a/drivers/media/tuners/qm1d1c0042.h b/drivers/media/tuners/qm1d1c0042.h
index 4f5c18816c44..8331f8baa094 100644
--- a/drivers/media/tuners/qm1d1c0042.h
+++ b/drivers/media/tuners/qm1d1c0042.h
@@ -17,7 +17,7 @@
 #ifndef QM1D1C0042_H
 #define QM1D1C0042_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 
 struct qm1d1c0042_config {
diff --git a/drivers/media/tuners/qt1010.c b/drivers/media/tuners/qt1010.c
index ee33b7cc7682..b92be882ab3c 100644
--- a/drivers/media/tuners/qt1010.c
+++ b/drivers/media/tuners/qt1010.c
@@ -224,7 +224,7 @@ static int qt1010_set_params(struct dvb_frontend *fe)
 static int qt1010_init_meas1(struct qt1010_priv *priv,
 			     u8 oper, u8 reg, u8 reg_init_val, u8 *retval)
 {
-	u8 i, val1, val2;
+	u8 i, val1, uninitialized_var(val2);
 	int err;
 
 	qt1010_i2c_oper_t i2c_data[] = {
@@ -259,7 +259,7 @@ static int qt1010_init_meas1(struct qt1010_priv *priv,
 static int qt1010_init_meas2(struct qt1010_priv *priv,
 			    u8 reg_init_val, u8 *retval)
 {
-	u8 i, val;
+	u8 i, uninitialized_var(val);
 	int err;
 	qt1010_i2c_oper_t i2c_data[] = {
 		{ QT1010_WR, 0x07, reg_init_val },
diff --git a/drivers/media/tuners/qt1010.h b/drivers/media/tuners/qt1010.h
index 276e59e85032..24216c2a8154 100644
--- a/drivers/media/tuners/qt1010.h
+++ b/drivers/media/tuners/qt1010.h
@@ -18,7 +18,7 @@
 #ifndef QT1010_H
 #define QT1010_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct qt1010_config {
 	u8 i2c_address;
diff --git a/drivers/media/tuners/r820t.c b/drivers/media/tuners/r820t.c
index ba80376a3b86..bc9299059f48 100644
--- a/drivers/media/tuners/r820t.c
+++ b/drivers/media/tuners/r820t.c
@@ -1,36 +1,24 @@
-/*
- * Rafael Micro R820T driver
- *
- * Copyright (C) 2013 Mauro Carvalho Chehab
- *
- * This driver was written from scratch, based on an existing driver
- * that it is part of rtl-sdr git tree, released under GPLv2:
- *	https://groups.google.com/forum/#!topic/ultra-cheap-sdr/Y3rBEOFtHug
- *	https://github.com/n1gp/gr-baz
- *
- * From what I understood from the threads, the original driver was converted
- * to userspace from a Realtek tree. I couldn't find the original tree.
- * However, the original driver look awkward on my eyes. So, I decided to
- * write a new version from it from the scratch, while trying to reproduce
- * everything found there.
- *
- * TODO:
- *	After locking, the original driver seems to have some routines to
- *		improve reception. This was not implemented here yet.
- *
- *	RF Gain set/get is not implemented.
- *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; either version 2 of the License, or
- *    (at your option) any later version.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *    GNU General Public License for more details.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+// Rafael Micro R820T driver
+//
+// Copyright (C) 2013 Mauro Carvalho Chehab
+//
+// This driver was written from scratch, based on an existing driver
+// that it is part of rtl-sdr git tree, released under GPLv2:
+//	https://groups.google.com/forum/#!topic/ultra-cheap-sdr/Y3rBEOFtHug
+//	https://github.com/n1gp/gr-baz
+//
+// From what I understood from the threads, the original driver was converted
+// to userspace from a Realtek tree. I couldn't find the original tree.
+// However, the original driver look awkward on my eyes. So, I decided to
+// write a new version from it from the scratch, while trying to reproduce
+// everything found there.
+//
+// TODO:
+//	After locking, the original driver seems to have some routines to
+//		improve reception. This was not implemented here yet.
+//
+//	RF Gain set/get is not implemented.
 
 #include <linux/videodev2.h>
 #include <linux/mutex.h>
@@ -396,9 +384,11 @@ static int r820t_write(struct r820t_priv *priv, u8 reg, const u8 *val,
 	return 0;
 }
 
-static int r820t_write_reg(struct r820t_priv *priv, u8 reg, u8 val)
+static inline int r820t_write_reg(struct r820t_priv *priv, u8 reg, u8 val)
 {
-	return r820t_write(priv, reg, &val, 1);
+	u8 tmp = val; /* work around GCC PR81715 with asan-stack=1 */
+
+	return r820t_write(priv, reg, &tmp, 1);
 }
 
 static int r820t_read_cache_reg(struct r820t_priv *priv, int reg)
@@ -411,17 +401,18 @@ static int r820t_read_cache_reg(struct r820t_priv *priv, int reg)
 		return -EINVAL;
 }
 
-static int r820t_write_reg_mask(struct r820t_priv *priv, u8 reg, u8 val,
+static inline int r820t_write_reg_mask(struct r820t_priv *priv, u8 reg, u8 val,
 				u8 bit_mask)
 {
+	u8 tmp = val;
 	int rc = r820t_read_cache_reg(priv, reg);
 
 	if (rc < 0)
 		return rc;
 
-	val = (rc & ~bit_mask) | (val & bit_mask);
+	tmp = (rc & ~bit_mask) | (tmp & bit_mask);
 
-	return r820t_write(priv, reg, &val, 1);
+	return r820t_write(priv, reg, &tmp, 1);
 }
 
 static int r820t_read(struct r820t_priv *priv, u8 reg, u8 *val, int len)
@@ -2388,4 +2379,4 @@ EXPORT_SYMBOL_GPL(r820t_attach);
 
 MODULE_DESCRIPTION("Rafael Micro r820t silicon tuner driver");
 MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/tuners/r820t.h b/drivers/media/tuners/r820t.h
index fdcab91405de..4f91dbb29c3a 100644
--- a/drivers/media/tuners/r820t.h
+++ b/drivers/media/tuners/r820t.h
@@ -21,7 +21,7 @@
 #ifndef R820T_H
 #define R820T_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 enum r820t_chip {
 	CHIP_R820T,
diff --git a/drivers/media/tuners/si2157.c b/drivers/media/tuners/si2157.c
index e35b1faf0ddc..9e34d31d724d 100644
--- a/drivers/media/tuners/si2157.c
+++ b/drivers/media/tuners/si2157.c
@@ -532,7 +532,7 @@ MODULE_DEVICE_TABLE(i2c, si2157_id_table);
 
 static struct i2c_driver si2157_driver = {
 	.driver = {
-		.name	             = "si2157",
+		.name		     = "si2157",
 		.suppress_bind_attrs = true,
 	},
 	.probe		= si2157_probe,
diff --git a/drivers/media/tuners/si2157.h b/drivers/media/tuners/si2157.h
index 76807f5b3cf8..de597fa47db6 100644
--- a/drivers/media/tuners/si2157.h
+++ b/drivers/media/tuners/si2157.h
@@ -18,7 +18,7 @@
 #define SI2157_H
 
 #include <media/media-device.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /*
  * I2C address
diff --git a/drivers/media/tuners/tda18212.h b/drivers/media/tuners/tda18212.h
index 6391dafd0c9d..9ed4367c21fc 100644
--- a/drivers/media/tuners/tda18212.h
+++ b/drivers/media/tuners/tda18212.h
@@ -21,7 +21,7 @@
 #ifndef TDA18212_H
 #define TDA18212_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct tda18212_config {
 	u16 if_dvbt_6;
diff --git a/drivers/media/tuners/tda18218.h b/drivers/media/tuners/tda18218.h
index 9c0e3fd7ed7f..0427c6f34c40 100644
--- a/drivers/media/tuners/tda18218.h
+++ b/drivers/media/tuners/tda18218.h
@@ -17,7 +17,7 @@
 #ifndef TDA18218_H
 #define TDA18218_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct tda18218_config {
 	u8 i2c_address;
diff --git a/drivers/media/tuners/tda18250.c b/drivers/media/tuners/tda18250.c
new file mode 100644
index 000000000000..20d12b063380
--- /dev/null
+++ b/drivers/media/tuners/tda18250.c
@@ -0,0 +1,902 @@
+/*
+ * NXP TDA18250 silicon tuner driver
+ *
+ * Copyright (C) 2017 Olli Salonen <olli.salonen@iki.fi>
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ */
+
+#include "tda18250_priv.h"
+#include <linux/regmap.h>
+
+static const struct dvb_tuner_ops tda18250_ops;
+
+static int tda18250_power_control(struct dvb_frontend *fe,
+		unsigned int power_state)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	int ret;
+	unsigned int utmp;
+
+	dev_dbg(&client->dev, "power state: %d", power_state);
+
+	switch (power_state) {
+	case TDA18250_POWER_NORMAL:
+		ret = regmap_write_bits(dev->regmap, R06_POWER2, 0x07, 0x00);
+		if (ret)
+			goto err;
+		ret = regmap_write_bits(dev->regmap, R25_REF, 0xc0, 0xc0);
+		if (ret)
+			goto err;
+		break;
+	case TDA18250_POWER_STANDBY:
+		if (dev->loopthrough) {
+			ret = regmap_write_bits(dev->regmap,
+					R25_REF, 0xc0, 0x80);
+			if (ret)
+				goto err;
+			ret = regmap_write_bits(dev->regmap,
+					R06_POWER2, 0x07, 0x02);
+			if (ret)
+				goto err;
+			ret = regmap_write_bits(dev->regmap,
+					R10_LT1, 0x80, 0x00);
+			if (ret)
+				goto err;
+		} else {
+			ret = regmap_write_bits(dev->regmap,
+					R25_REF, 0xc0, 0x80);
+			if (ret)
+				goto err;
+			ret = regmap_write_bits(dev->regmap,
+					R06_POWER2, 0x07, 0x01);
+			if (ret)
+				goto err;
+			ret = regmap_read(dev->regmap,
+					R0D_AGC12, &utmp);
+			if (ret)
+				goto err;
+			ret = regmap_write_bits(dev->regmap,
+					R0D_AGC12, 0x03, 0x03);
+			if (ret)
+				goto err;
+			ret = regmap_write_bits(dev->regmap,
+					R10_LT1, 0x80, 0x80);
+			if (ret)
+				goto err;
+			ret = regmap_write_bits(dev->regmap,
+					R0D_AGC12, 0x03, utmp & 0x03);
+			if (ret)
+				goto err;
+		}
+		break;
+	default:
+		ret = -EINVAL;
+		goto err;
+	}
+
+	return 0;
+err:
+	return ret;
+}
+
+static int tda18250_wait_for_irq(struct dvb_frontend *fe,
+		int maxwait, int step, u8 irq)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	int ret;
+	unsigned long timeout;
+	bool triggered;
+	unsigned int utmp;
+
+	triggered = false;
+	timeout = jiffies + msecs_to_jiffies(maxwait);
+	while (!time_after(jiffies, timeout)) {
+		// check for the IRQ
+		ret = regmap_read(dev->regmap, R08_IRQ1, &utmp);
+		if (ret)
+			goto err;
+		if ((utmp & irq) == irq) {
+			triggered = true;
+			break;
+		}
+		msleep(step);
+	}
+
+	dev_dbg(&client->dev, "waited IRQ (0x%02x) %d ms, triggered: %s", irq,
+			jiffies_to_msecs(jiffies) -
+			(jiffies_to_msecs(timeout) - maxwait),
+			triggered ? "true" : "false");
+
+	if (!triggered)
+		return -ETIMEDOUT;
+
+	return 0;
+err:
+	return ret;
+}
+
+static int tda18250_init(struct dvb_frontend *fe)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	int ret, i;
+
+	/* default values for various regs */
+	static const u8 init_regs[][2] = {
+		{ R0C_AGC11, 0xc7 },
+		{ R0D_AGC12, 0x5d },
+		{ R0E_AGC13, 0x40 },
+		{ R0F_AGC14, 0x0e },
+		{ R10_LT1, 0x47 },
+		{ R11_LT2, 0x4e },
+		{ R12_AGC21, 0x26 },
+		{ R13_AGC22, 0x60 },
+		{ R18_AGC32, 0x37 },
+		{ R19_AGC33, 0x09 },
+		{ R1A_AGCK, 0x00 },
+		{ R1E_WI_FI, 0x29 },
+		{ R1F_RF_BPF, 0x06 },
+		{ R20_IR_MIX, 0xc6 },
+		{ R21_IF_AGC, 0x00 },
+		{ R2C_PS1, 0x75 },
+		{ R2D_PS2, 0x06 },
+		{ R2E_PS3, 0x07 },
+		{ R30_RSSI2, 0x0e },
+		{ R31_IRQ_CTRL, 0x00 },
+		{ R39_SD5, 0x00 },
+		{ R3B_REGU, 0x55 },
+		{ R3C_RCCAL1, 0xa7 },
+		{ R3F_IRCAL2, 0x85 },
+		{ R40_IRCAL3, 0x87 },
+		{ R41_IRCAL4, 0xc0 },
+		{ R43_PD1, 0x40 },
+		{ R44_PD2, 0xc0 },
+		{ R46_CPUMP, 0x0c },
+		{ R47_LNAPOL, 0x64 },
+		{ R4B_XTALOSC1, 0x30 },
+		{ R59_AGC2_UP2, 0x05 },
+		{ R5B_AGC_AUTO, 0x07 },
+		{ R5C_AGC_DEBUG, 0x00 },
+	};
+
+	/* crystal related regs depend on frequency */
+	static const u8 xtal_regs[][5] = {
+					/* reg:   4d    4e    4f    50    51 */
+		[TDA18250_XTAL_FREQ_16MHZ]  = { 0x3e, 0x80, 0x50, 0x00, 0x20 },
+		[TDA18250_XTAL_FREQ_24MHZ]  = { 0x5d, 0xc0, 0xec, 0x00, 0x18 },
+		[TDA18250_XTAL_FREQ_25MHZ]  = { 0x61, 0xa8, 0xec, 0x80, 0x19 },
+		[TDA18250_XTAL_FREQ_27MHZ]  = { 0x69, 0x78, 0x8d, 0x80, 0x1b },
+		[TDA18250_XTAL_FREQ_30MHZ]  = { 0x75, 0x30, 0x8f, 0x00, 0x1e },
+	};
+
+	dev_dbg(&client->dev, "\n");
+
+	ret = tda18250_power_control(fe, TDA18250_POWER_NORMAL);
+	if (ret)
+		goto err;
+
+	msleep(20);
+
+	if (dev->warm)
+		goto warm;
+
+	/* set initial register values */
+	for (i = 0; i < ARRAY_SIZE(init_regs); i++) {
+		ret = regmap_write(dev->regmap, init_regs[i][0],
+				init_regs[i][1]);
+		if (ret)
+			goto err;
+	}
+
+	/* set xtal related regs */
+	ret = regmap_bulk_write(dev->regmap, R4D_XTALFLX1,
+			xtal_regs[dev->xtal_freq], 5);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R10_LT1, 0x80,
+			dev->loopthrough ? 0x00 : 0x80);
+	if (ret)
+		goto err;
+
+	/* clear IRQ */
+	ret = regmap_write(dev->regmap, R0A_IRQ3, TDA18250_IRQ_HW_INIT);
+	if (ret)
+		goto err;
+
+	/* start HW init */
+	ret = regmap_write(dev->regmap, R2A_MSM1, 0x70);
+	if (ret)
+		goto err;
+
+	ret = regmap_write(dev->regmap, R2B_MSM2, 0x01);
+	if (ret)
+		goto err;
+
+	ret = tda18250_wait_for_irq(fe, 500, 10, TDA18250_IRQ_HW_INIT);
+	if (ret)
+		goto err;
+
+	/* tuner calibration */
+	ret = regmap_write(dev->regmap, R2A_MSM1, 0x02);
+	if (ret)
+		goto err;
+
+	ret = regmap_write(dev->regmap, R2B_MSM2, 0x01);
+	if (ret)
+		goto err;
+
+	ret = tda18250_wait_for_irq(fe, 500, 10, TDA18250_IRQ_CAL);
+	if (ret)
+		goto err;
+
+	dev->warm = true;
+
+warm:
+	/* power up LNA */
+	ret = regmap_write_bits(dev->regmap, R0C_AGC11, 0x80, 0x00);
+	if (ret)
+		goto err;
+
+	return 0;
+err:
+	dev_dbg(&client->dev, "failed=%d", ret);
+	return ret;
+}
+
+static int tda18250_set_agc(struct dvb_frontend *fe)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+	int ret;
+	u8 utmp, utmp2;
+
+	dev_dbg(&client->dev, "\n");
+
+	ret = regmap_write_bits(dev->regmap, R1F_RF_BPF, 0x87, 0x06);
+	if (ret)
+		goto err;
+
+	utmp = ((c->frequency < 100000000) &&
+			((c->delivery_system == SYS_DVBC_ANNEX_A) ||
+			(c->delivery_system == SYS_DVBC_ANNEX_C)) &&
+			(c->bandwidth_hz == 6000000)) ? 0x80 : 0x00;
+	ret = regmap_write(dev->regmap, R5A_H3H5, utmp);
+	if (ret)
+		goto err;
+
+	/* AGC1 */
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		utmp = 4;
+		break;
+	default: /* DVB-C/QAM */
+		switch (c->bandwidth_hz) {
+		case 6000000:
+			utmp = (c->frequency < 800000000) ? 6 : 4;
+			break;
+		default: /* 7.935 and 8 MHz */
+			utmp = (c->frequency < 100000000) ? 2 : 3;
+			break;
+		}
+		break;
+	}
+
+	ret = regmap_write_bits(dev->regmap, R0C_AGC11, 0x07, utmp);
+	if (ret)
+		goto err;
+
+	/* AGC2 */
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		utmp = (c->frequency < 320000000) ? 20 : 16;
+		utmp2 = (c->frequency < 320000000) ? 22 : 18;
+		break;
+	default: /* DVB-C/QAM */
+		switch (c->bandwidth_hz) {
+		case 6000000:
+			if (c->frequency < 600000000) {
+				utmp = 18;
+				utmp2 = 22;
+			} else if (c->frequency < 800000000) {
+				utmp = 16;
+				utmp2 = 20;
+			} else {
+				utmp = 14;
+				utmp2 = 16;
+			}
+			break;
+		default: /* 7.935 and 8 MHz */
+			utmp = (c->frequency < 320000000) ? 16 : 18;
+			utmp2 = (c->frequency < 320000000) ? 18 : 20;
+			break;
+		}
+		break;
+	}
+	ret = regmap_write_bits(dev->regmap, R58_AGC2_UP1, 0x1f, utmp2+8);
+	if (ret)
+		goto err;
+	ret = regmap_write_bits(dev->regmap, R13_AGC22, 0x1f, utmp);
+	if (ret)
+		goto err;
+	ret = regmap_write_bits(dev->regmap, R14_AGC23, 0x1f, utmp2);
+	if (ret)
+		goto err;
+
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		utmp = 98;
+		break;
+	default: /* DVB-C/QAM */
+		utmp = 90;
+		break;
+	}
+	ret = regmap_write_bits(dev->regmap, R16_AGC25, 0xf8, utmp);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R12_AGC21, 0x60,
+			(c->frequency > 800000000) ? 0x40 : 0x20);
+	if (ret)
+		goto err;
+
+	/* AGC3 */
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		utmp = (c->frequency < 320000000) ? 5 : 7;
+		utmp2 = (c->frequency < 320000000) ? 10 : 12;
+		break;
+	default: /* DVB-C/QAM */
+		utmp = 7;
+		utmp2 = 12;
+		break;
+	}
+	ret = regmap_write(dev->regmap, R17_AGC31, (utmp << 4) | utmp2);
+	if (ret)
+		goto err;
+
+	/* S2D */
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		if (c->bandwidth_hz == 8000000)
+			utmp = 0x04;
+		else
+			utmp = (c->frequency < 320000000) ? 0x04 : 0x02;
+		break;
+	default: /* DVB-C/QAM */
+		if (c->bandwidth_hz == 6000000)
+			utmp = ((c->frequency > 172544000) &&
+				(c->frequency < 320000000)) ? 0x04 : 0x02;
+		else /* 7.935 and 8 MHz */
+			utmp = ((c->frequency > 320000000) &&
+				(c->frequency < 600000000)) ? 0x02 : 0x04;
+		break;
+	}
+	ret = regmap_write_bits(dev->regmap, R20_IR_MIX, 0x06, utmp);
+	if (ret)
+		goto err;
+
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		utmp = 0;
+		break;
+	default: /* DVB-C/QAM */
+		utmp = (c->frequency < 600000000) ? 0 : 3;
+		break;
+	}
+	ret = regmap_write_bits(dev->regmap, R16_AGC25, 0x03, utmp);
+	if (ret)
+		goto err;
+
+	utmp = 0x09;
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		if (c->bandwidth_hz == 8000000)
+			utmp = 0x0c;
+		break;
+	default: /* DVB-C/QAM */
+		utmp = 0x0c;
+		break;
+	}
+	ret = regmap_write_bits(dev->regmap, R0F_AGC14, 0x3f, utmp);
+	if (ret)
+		goto err;
+
+	return 0;
+err:
+	dev_dbg(&client->dev, "failed=%d", ret);
+	return ret;
+}
+
+static int tda18250_pll_calc(struct dvb_frontend *fe, u8 *rdiv,
+		u8 *ndiv, u8 *icp)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+	int ret;
+	unsigned int uval, exp, lopd, scale;
+	unsigned long fvco;
+
+	ret = regmap_read(dev->regmap, R34_MD1, &uval);
+	if (ret)
+		goto err;
+
+	exp = (uval & 0x70) >> 4;
+	if (exp > 5)
+		exp = 0;
+	lopd = 1 << (exp - 1);
+	scale = uval & 0x0f;
+	fvco = lopd * scale * ((c->frequency / 1000) + dev->if_frequency);
+
+	switch (dev->xtal_freq) {
+	case TDA18250_XTAL_FREQ_16MHZ:
+		*rdiv = 1;
+		*ndiv = 0;
+		*icp = (fvco < 6622000) ? 0x05 : 0x02;
+	break;
+	case TDA18250_XTAL_FREQ_24MHZ:
+	case TDA18250_XTAL_FREQ_25MHZ:
+		*rdiv = 3;
+		*ndiv = 1;
+		*icp = (fvco < 6622000) ? 0x05 : 0x02;
+	break;
+	case TDA18250_XTAL_FREQ_27MHZ:
+		if (fvco < 6643000) {
+			*rdiv = 2;
+			*ndiv = 0;
+			*icp = 0x05;
+		} else if (fvco < 6811000) {
+			*rdiv = 2;
+			*ndiv = 0;
+			*icp = 0x06;
+		} else {
+			*rdiv = 3;
+			*ndiv = 1;
+			*icp = 0x02;
+		}
+	break;
+	case TDA18250_XTAL_FREQ_30MHZ:
+		*rdiv = 2;
+		*ndiv = 0;
+		*icp = (fvco < 6811000) ? 0x05 : 0x02;
+	break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(&client->dev,
+			"lopd=%d scale=%u fvco=%lu, rdiv=%d ndiv=%d icp=%d",
+			lopd, scale, fvco, *rdiv, *ndiv, *icp);
+	return 0;
+err:
+	return ret;
+}
+
+static int tda18250_set_params(struct dvb_frontend *fe)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	struct dtv_frontend_properties *c = &fe->dtv_property_cache;
+	u32 if_khz;
+	int ret;
+	unsigned int i, j;
+	u8 utmp;
+	u8 buf[3];
+
+	#define REG      0
+	#define MASK     1
+	#define DVBT_6   2
+	#define DVBT_7   3
+	#define DVBT_8   4
+	#define DVBC_6   5
+	#define DVBC_8   6
+	#define ATSC     7
+
+	static const u8 delsys_params[][16] = {
+		[REG]    = { 0x22, 0x23, 0x24, 0x21, 0x0d, 0x0c, 0x0f, 0x14,
+			     0x0e, 0x12, 0x58, 0x59, 0x1a, 0x19, 0x1e, 0x30 },
+		[MASK]   = { 0x77, 0xff, 0xff, 0x87, 0xf0, 0x78, 0x07, 0xe0,
+			     0x60, 0x0f, 0x60, 0x0f, 0x33, 0x30, 0x80, 0x06 },
+		[DVBT_6] = { 0x51, 0x03, 0x83, 0x82, 0x40, 0x48, 0x01, 0xe0,
+			     0x60, 0x0f, 0x60, 0x05, 0x03, 0x10, 0x00, 0x04 },
+		[DVBT_7] = { 0x52, 0x03, 0x85, 0x82, 0x40, 0x48, 0x01, 0xe0,
+			     0x60, 0x0f, 0x60, 0x05, 0x03, 0x10, 0x00, 0x04 },
+		[DVBT_8] = { 0x53, 0x03, 0x87, 0x82, 0x40, 0x48, 0x06, 0xe0,
+			     0x60, 0x07, 0x60, 0x05, 0x03, 0x10, 0x00, 0x04 },
+		[DVBC_6] = { 0x32, 0x05, 0x86, 0x82, 0x50, 0x00, 0x06, 0x60,
+			     0x40, 0x0e, 0x60, 0x05, 0x33, 0x10, 0x00, 0x04 },
+		[DVBC_8] = { 0x53, 0x03, 0x88, 0x82, 0x50, 0x00, 0x06, 0x60,
+			     0x40, 0x0e, 0x60, 0x05, 0x33, 0x10, 0x00, 0x04 },
+		[ATSC]   = { 0x51, 0x03, 0x83, 0x82, 0x40, 0x48, 0x01, 0xe0,
+			     0x40, 0x0e, 0x60, 0x05, 0x03, 0x00, 0x80, 0x04 },
+	};
+
+	dev_dbg(&client->dev,
+			"delivery_system=%d frequency=%u bandwidth_hz=%u",
+			c->delivery_system, c->frequency, c->bandwidth_hz);
+
+
+	switch (c->delivery_system) {
+	case SYS_ATSC:
+		j = ATSC;
+		if_khz = dev->if_atsc;
+		break;
+	case SYS_DVBT:
+	case SYS_DVBT2:
+		if (c->bandwidth_hz == 0) {
+			ret = -EINVAL;
+			goto err;
+		} else if (c->bandwidth_hz <= 6000000) {
+			j = DVBT_6;
+			if_khz = dev->if_dvbt_6;
+		} else if (c->bandwidth_hz <= 7000000) {
+			j = DVBT_7;
+			if_khz = dev->if_dvbt_7;
+		} else if (c->bandwidth_hz <= 8000000) {
+			j = DVBT_8;
+			if_khz = dev->if_dvbt_8;
+		} else {
+			ret = -EINVAL;
+			goto err;
+		}
+		break;
+	case SYS_DVBC_ANNEX_A:
+	case SYS_DVBC_ANNEX_C:
+		if (c->bandwidth_hz == 0) {
+			ret = -EINVAL;
+			goto err;
+		} else if (c->bandwidth_hz <= 6000000) {
+			j = DVBC_6;
+			if_khz = dev->if_dvbc_6;
+		} else if (c->bandwidth_hz <= 8000000) {
+			j = DVBC_8;
+			if_khz = dev->if_dvbc_8;
+		} else {
+			ret = -EINVAL;
+			goto err;
+		}
+		break;
+	default:
+		ret = -EINVAL;
+		dev_err(&client->dev, "unsupported delivery system=%d",
+				c->delivery_system);
+		goto err;
+	}
+
+	/* set delivery system dependent registers */
+	for (i = 0; i < 16; i++) {
+		ret = regmap_write_bits(dev->regmap, delsys_params[REG][i],
+			 delsys_params[MASK][i],  delsys_params[j][i]);
+		if (ret)
+			goto err;
+	}
+
+	/* set IF if needed */
+	if (dev->if_frequency != if_khz) {
+		utmp = DIV_ROUND_CLOSEST(if_khz, 50);
+		ret = regmap_write(dev->regmap, R26_IF, utmp);
+		if (ret)
+			goto err;
+		dev->if_frequency = if_khz;
+		dev_dbg(&client->dev, "set IF=%u kHz", if_khz);
+
+	}
+
+	ret = tda18250_set_agc(fe);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R1A_AGCK, 0x03, 0x01);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R14_AGC23, 0x40, 0x00);
+	if (ret)
+		goto err;
+
+	/* set frequency */
+	buf[0] = ((c->frequency / 1000) >> 16) & 0xff;
+	buf[1] = ((c->frequency / 1000) >>  8) & 0xff;
+	buf[2] = ((c->frequency / 1000) >>  0) & 0xff;
+	ret = regmap_bulk_write(dev->regmap, R27_RF1, buf, 3);
+	if (ret)
+		goto err;
+
+	ret = regmap_write(dev->regmap, R0A_IRQ3, TDA18250_IRQ_TUNE);
+	if (ret)
+		goto err;
+
+	/* initial tune */
+	ret = regmap_write(dev->regmap, R2A_MSM1, 0x01);
+	if (ret)
+		goto err;
+
+	ret = regmap_write(dev->regmap, R2B_MSM2, 0x01);
+	if (ret)
+		goto err;
+
+	ret = tda18250_wait_for_irq(fe, 500, 10, TDA18250_IRQ_TUNE);
+	if (ret)
+		goto err;
+
+	/* calc ndiv and rdiv */
+	ret = tda18250_pll_calc(fe, &buf[0], &buf[1], &buf[2]);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R4F_XTALFLX3, 0xe0,
+			(buf[0] << 6) | (buf[1] << 5));
+	if (ret)
+		goto err;
+
+	/* clear IRQ */
+	ret = regmap_write(dev->regmap, R0A_IRQ3, TDA18250_IRQ_TUNE);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R46_CPUMP, 0x07, 0x00);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R39_SD5, 0x03, 0x00);
+	if (ret)
+		goto err;
+
+	/* tune again */
+	ret = regmap_write(dev->regmap, R2A_MSM1, 0x01); /* tune */
+	if (ret)
+		goto err;
+
+	ret = regmap_write(dev->regmap, R2B_MSM2, 0x01); /* go */
+	if (ret)
+		goto err;
+
+	ret = tda18250_wait_for_irq(fe, 500, 10, TDA18250_IRQ_TUNE);
+	if (ret)
+		goto err;
+
+	/* pll locking */
+	msleep(20);
+
+	ret = regmap_write_bits(dev->regmap, R2B_MSM2, 0x04, 0x04);
+	if (ret)
+		goto err;
+
+	msleep(20);
+
+	/* restore AGCK */
+	ret = regmap_write_bits(dev->regmap, R1A_AGCK, 0x03, 0x03);
+	if (ret)
+		goto err;
+
+	ret = regmap_write_bits(dev->regmap, R14_AGC23, 0x40, 0x40);
+	if (ret)
+		goto err;
+
+	/* charge pump */
+	ret = regmap_write_bits(dev->regmap, R46_CPUMP, 0x07, buf[2]);
+
+	return 0;
+err:
+	return ret;
+}
+
+static int tda18250_get_if_frequency(struct dvb_frontend *fe, u32 *frequency)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+
+	*frequency = dev->if_frequency * 1000;
+	return 0;
+}
+
+static int tda18250_sleep(struct dvb_frontend *fe)
+{
+	struct i2c_client *client = fe->tuner_priv;
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	int ret;
+
+	dev_dbg(&client->dev, "\n");
+
+	/* power down LNA */
+	ret = regmap_write_bits(dev->regmap, R0C_AGC11, 0x80, 0x00);
+	if (ret)
+		return ret;
+
+	/* set if freq to 0 in order to make sure it's set after wake up */
+	dev->if_frequency = 0;
+
+	ret = tda18250_power_control(fe, TDA18250_POWER_STANDBY);
+	return ret;
+}
+
+static const struct dvb_tuner_ops tda18250_ops = {
+	.info = {
+		.name           = "NXP TDA18250",
+		.frequency_min  = 42000000,
+		.frequency_max  = 870000000,
+	},
+
+	.init = tda18250_init,
+	.set_params = tda18250_set_params,
+	.get_if_frequency = tda18250_get_if_frequency,
+	.sleep = tda18250_sleep,
+};
+
+static int tda18250_probe(struct i2c_client *client,
+		const struct i2c_device_id *id)
+{
+	struct tda18250_config *cfg = client->dev.platform_data;
+	struct dvb_frontend *fe = cfg->fe;
+	struct tda18250_dev *dev;
+	int ret;
+	unsigned char chip_id[3];
+
+	/* some registers are always read from HW */
+	static const struct regmap_range tda18250_yes_ranges[] = {
+		regmap_reg_range(R05_POWER1, R0B_IRQ4),
+		regmap_reg_range(R21_IF_AGC, R21_IF_AGC),
+		regmap_reg_range(R2A_MSM1, R2B_MSM2),
+		regmap_reg_range(R2F_RSSI1, R31_IRQ_CTRL),
+	};
+
+	static const struct regmap_access_table tda18250_volatile_table = {
+		.yes_ranges = tda18250_yes_ranges,
+		.n_yes_ranges = ARRAY_SIZE(tda18250_yes_ranges),
+	};
+
+	static const struct regmap_config tda18250_regmap_config = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = TDA18250_NUM_REGS - 1,
+		.volatile_table = &tda18250_volatile_table,
+	};
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	i2c_set_clientdata(client, dev);
+
+	dev->fe = cfg->fe;
+	dev->loopthrough = cfg->loopthrough;
+	if (cfg->xtal_freq < TDA18250_XTAL_FREQ_MAX) {
+		dev->xtal_freq = cfg->xtal_freq;
+	} else {
+		ret = -EINVAL;
+		dev_err(&client->dev, "xtal_freq invalid=%d", cfg->xtal_freq);
+		goto err_kfree;
+	}
+	dev->if_dvbt_6 = cfg->if_dvbt_6;
+	dev->if_dvbt_7 = cfg->if_dvbt_7;
+	dev->if_dvbt_8 = cfg->if_dvbt_8;
+	dev->if_dvbc_6 = cfg->if_dvbc_6;
+	dev->if_dvbc_8 = cfg->if_dvbc_8;
+	dev->if_atsc = cfg->if_atsc;
+
+	dev->if_frequency = 0;
+	dev->warm = false;
+
+	dev->regmap = devm_regmap_init_i2c(client, &tda18250_regmap_config);
+	if (IS_ERR(dev->regmap)) {
+		ret = PTR_ERR(dev->regmap);
+		goto err_kfree;
+	}
+
+	/* read the three chip ID registers */
+	regmap_bulk_read(dev->regmap, R00_ID1, &chip_id, 3);
+	dev_dbg(&client->dev, "chip_id=%02x:%02x:%02x",
+			chip_id[0], chip_id[1], chip_id[2]);
+
+	switch (chip_id[0]) {
+	case 0xc7:
+		dev->slave = false;
+		break;
+	case 0x47:
+		dev->slave = true;
+		break;
+	default:
+		ret = -ENODEV;
+		goto err_kfree;
+	}
+
+	if (chip_id[1] != 0x4a) {
+		ret = -ENODEV;
+		goto err_kfree;
+	}
+
+	switch (chip_id[2]) {
+	case 0x20:
+		dev_info(&client->dev,
+				"NXP TDA18250AHN/%s successfully identified",
+				dev->slave ? "S" : "M");
+		break;
+	case 0x21:
+		dev_info(&client->dev,
+				"NXP TDA18250BHN/%s successfully identified",
+				dev->slave ? "S" : "M");
+		break;
+	default:
+		ret = -ENODEV;
+		goto err_kfree;
+	}
+
+	fe->tuner_priv = client;
+	memcpy(&fe->ops.tuner_ops, &tda18250_ops,
+			sizeof(struct dvb_tuner_ops));
+
+	/* put the tuner in standby */
+	tda18250_power_control(fe, TDA18250_POWER_STANDBY);
+
+	return 0;
+err_kfree:
+	kfree(dev);
+err:
+	dev_dbg(&client->dev, "failed=%d", ret);
+	return ret;
+}
+
+static int tda18250_remove(struct i2c_client *client)
+{
+	struct tda18250_dev *dev = i2c_get_clientdata(client);
+	struct dvb_frontend *fe = dev->fe;
+
+	dev_dbg(&client->dev, "\n");
+
+	memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops));
+	fe->tuner_priv = NULL;
+	kfree(dev);
+
+	return 0;
+}
+
+static const struct i2c_device_id tda18250_id_table[] = {
+	{"tda18250", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, tda18250_id_table);
+
+static struct i2c_driver tda18250_driver = {
+	.driver = {
+		.name	= "tda18250",
+	},
+	.probe		= tda18250_probe,
+	.remove		= tda18250_remove,
+	.id_table	= tda18250_id_table,
+};
+
+module_i2c_driver(tda18250_driver);
+
+MODULE_DESCRIPTION("NXP TDA18250 silicon tuner driver");
+MODULE_AUTHOR("Olli Salonen <olli.salonen@iki.fi>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/tuners/tda18250.h b/drivers/media/tuners/tda18250.h
new file mode 100644
index 000000000000..961806a81f9f
--- /dev/null
+++ b/drivers/media/tuners/tda18250.h
@@ -0,0 +1,51 @@
+/*
+ * NXP TDA18250BHN silicon tuner driver
+ *
+ * Copyright (C) 2017 Olli Salonen <olli.salonen@iki.fi>
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ */
+
+#ifndef TDA18250_H
+#define TDA18250_H
+
+#include <linux/kconfig.h>
+#include <media/media-device.h>
+#include <media/dvb_frontend.h>
+
+#define TDA18250_XTAL_FREQ_16MHZ 0
+#define TDA18250_XTAL_FREQ_24MHZ 1
+#define TDA18250_XTAL_FREQ_25MHZ 2
+#define TDA18250_XTAL_FREQ_27MHZ 3
+#define TDA18250_XTAL_FREQ_30MHZ 4
+#define TDA18250_XTAL_FREQ_MAX 5
+
+struct tda18250_config {
+	u16 if_dvbt_6;
+	u16 if_dvbt_7;
+	u16 if_dvbt_8;
+	u16 if_dvbc_6;
+	u16 if_dvbc_8;
+	u16 if_atsc;
+	u8 xtal_freq;
+	bool loopthrough;
+
+	/*
+	 * frontend
+	 */
+	struct dvb_frontend *fe;
+
+#if defined(CONFIG_MEDIA_CONTROLLER)
+	struct media_device *mdev;
+#endif
+};
+
+#endif
diff --git a/drivers/media/tuners/tda18250_priv.h b/drivers/media/tuners/tda18250_priv.h
new file mode 100644
index 000000000000..4a6f801701a7
--- /dev/null
+++ b/drivers/media/tuners/tda18250_priv.h
@@ -0,0 +1,145 @@
+/*
+ * NXP TDA18250BHN silicon tuner driver
+ *
+ * Copyright (C) 2017 Olli Salonen <olli.salonen@iki.fi>
+ *
+ *    This program is free software; you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation; either version 2 of the License, or
+ *    (at your option) any later version.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ */
+
+#ifndef TDA18250_PRIV_H
+#define TDA18250_PRIV_H
+
+#include "tda18250.h"
+
+#define R00_ID1		0x00	/* ID byte 1 */
+#define R01_ID2		0x01	/* ID byte 2 */
+#define R02_ID3		0x02	/* ID byte 3 */
+#define R03_THERMO1	0x03	/* Thermo byte 1 */
+#define R04_THERMO2	0x04	/* Thermo byte 2 */
+#define R05_POWER1	0x05	/* Power byte 1 */
+#define R06_POWER2	0x06	/* Power byte 2 */
+#define R07_GPIO	0x07	/* GPIO */
+#define R08_IRQ1	0x08	/* IRQ */
+#define R09_IRQ2	0x09	/* IRQ */
+#define R0A_IRQ3	0x0a	/* IRQ */
+#define R0B_IRQ4	0x0b	/* IRQ */
+#define R0C_AGC11	0x0c	/* AGC1 byte 1 */
+#define R0D_AGC12	0x0d	/* AGC1 byte 2 */
+#define R0E_AGC13	0x0e	/* AGC1 byte 3 */
+#define R0F_AGC14	0x0f	/* AGC1 byte 4 */
+#define R10_LT1		0x10	/* LT byte 1 */
+#define R11_LT2		0x11	/* LT byte 2 */
+#define R12_AGC21	0x12	/* AGC2 byte 1 */
+#define R13_AGC22	0x13	/* AGC2 byte 2 */
+#define R14_AGC23	0x14	/* AGC2 byte 3 */
+#define R15_AGC24	0x15	/* AGC2 byte 4 */
+#define R16_AGC25	0x16	/* AGC2 byte 5 */
+#define R17_AGC31	0x17	/* AGC3 byte 1 */
+#define R18_AGC32	0x18	/* AGC3 byte 2 */
+#define R19_AGC33	0x19	/* AGC3 byte 3 */
+#define R1A_AGCK	0x1a
+#define R1B_GAIN1	0x1b
+#define R1C_GAIN2	0x1c
+#define R1D_GAIN3	0x1d
+#define R1E_WI_FI	0x1e	/* Wireless Filter */
+#define R1F_RF_BPF	0x1f	/* RF Band Pass Filter */
+#define R20_IR_MIX	0x20	/* IR Mixer */
+#define R21_IF_AGC	0x21
+#define R22_IF1		0x22	/* IF byte 1 */
+#define R23_IF2		0x23	/* IF byte 2 */
+#define R24_IF3		0x24	/* IF byte 3 */
+#define R25_REF		0x25	/* reference byte */
+#define R26_IF		0x26	/* IF frequency */
+#define R27_RF1		0x27	/* RF frequency byte 1 */
+#define R28_RF2		0x28	/* RF frequency byte 2 */
+#define R29_RF3		0x29	/* RF frequency byte 3 */
+#define R2A_MSM1	0x2a
+#define R2B_MSM2	0x2b
+#define R2C_PS1		0x2c	/* power saving mode byte 1 */
+#define R2D_PS2		0x2d	/* power saving mode byte 2 */
+#define R2E_PS3		0x2e	/* power saving mode byte 3 */
+#define R2F_RSSI1	0x2f
+#define R30_RSSI2	0x30
+#define R31_IRQ_CTRL	0x31
+#define R32_DUMMY	0x32
+#define R33_TEST	0x33
+#define R34_MD1		0x34
+#define R35_SD1		0x35
+#define R36_SD2		0x36
+#define R37_SD3		0x37
+#define R38_SD4		0x38
+#define R39_SD5		0x39
+#define R3A_SD_TEST	0x3a
+#define R3B_REGU	0x3b
+#define R3C_RCCAL1	0x3c
+#define R3D_RCCAL2	0x3d
+#define R3E_IRCAL1	0x3e
+#define R3F_IRCAL2	0x3f
+#define R40_IRCAL3	0x40
+#define R41_IRCAL4	0x41
+#define R42_IRCAL5	0x42
+#define R43_PD1		0x43	/* power down byte 1 */
+#define R44_PD2		0x44	/* power down byte 2 */
+#define R45_PD		0x45	/* power down */
+#define R46_CPUMP	0x46	/* charge pump */
+#define R47_LNAPOL	0x47	/* LNA polar casc */
+#define R48_SMOOTH1	0x48	/* smooth test byte 1 */
+#define R49_SMOOTH2	0x49	/* smooth test byte 2 */
+#define R4A_SMOOTH3	0x4a	/* smooth test byte 3 */
+#define R4B_XTALOSC1	0x4b
+#define R4C_XTALOSC2	0x4c
+#define R4D_XTALFLX1	0x4d
+#define R4E_XTALFLX2	0x4e
+#define R4F_XTALFLX3	0x4f
+#define R50_XTALFLX4	0x50
+#define R51_XTALFLX5	0x51
+#define R52_IRLOOP0	0x52
+#define R53_IRLOOP1	0x53
+#define R54_IRLOOP2	0x54
+#define R55_IRLOOP3	0x55
+#define R56_IRLOOP4	0x56
+#define R57_PLL_LOG	0x57
+#define R58_AGC2_UP1	0x58
+#define R59_AGC2_UP2	0x59
+#define R5A_H3H5	0x5a
+#define R5B_AGC_AUTO	0x5b
+#define R5C_AGC_DEBUG	0x5c
+
+#define TDA18250_NUM_REGS 93
+
+#define TDA18250_POWER_STANDBY 0
+#define TDA18250_POWER_NORMAL 1
+
+#define TDA18250_IRQ_CAL     0x81
+#define TDA18250_IRQ_HW_INIT 0x82
+#define TDA18250_IRQ_TUNE    0x88
+
+struct tda18250_dev {
+	struct mutex i2c_mutex;
+	struct dvb_frontend *fe;
+	struct i2c_adapter *i2c;
+	struct regmap *regmap;
+	u8 xtal_freq;
+	/* IF in kHz */
+	u16 if_dvbt_6;
+	u16 if_dvbt_7;
+	u16 if_dvbt_8;
+	u16 if_dvbc_6;
+	u16 if_dvbc_8;
+	u16 if_atsc;
+	u16 if_frequency;
+	bool slave;
+	bool loopthrough;
+	bool warm;
+	u8 regs[TDA18250_NUM_REGS];
+};
+
+#endif
diff --git a/drivers/media/tuners/tda18271.h b/drivers/media/tuners/tda18271.h
index 0a846333ce57..7e07966c5ace 100644
--- a/drivers/media/tuners/tda18271.h
+++ b/drivers/media/tuners/tda18271.h
@@ -22,7 +22,7 @@
 #define __TDA18271_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 struct tda18271_std_map_item {
 	u16 if_freq;
diff --git a/drivers/media/tuners/tda827x.h b/drivers/media/tuners/tda827x.h
index abf2e2fe5350..a08d3f9fcea1 100644
--- a/drivers/media/tuners/tda827x.h
+++ b/drivers/media/tuners/tda827x.h
@@ -25,7 +25,7 @@
 #define __DVB_TDA827X_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda8290.h"
 
 struct tda827x_config
@@ -36,7 +36,7 @@ struct tda827x_config
 
 	/* interface to tda829x driver */
 	enum tda8290_lna config;
-	int 	     switch_addr;
+	int	     switch_addr;
 
 	void (*agcf)(struct dvb_frontend *fe);
 };
diff --git a/drivers/media/tuners/tda8290.c b/drivers/media/tuners/tda8290.c
index a59c567c55d6..9f1f1d2b8bdc 100644
--- a/drivers/media/tuners/tda8290.c
+++ b/drivers/media/tuners/tda8290.c
@@ -63,8 +63,8 @@ static int tda8290_i2c_bridge(struct dvb_frontend *fe, int close)
 {
 	struct tda8290_priv *priv = fe->analog_demod_priv;
 
-	unsigned char  enable[2] = { 0x21, 0xC0 };
-	unsigned char disable[2] = { 0x21, 0x00 };
+	static unsigned char  enable[2] = { 0x21, 0xC0 };
+	static unsigned char disable[2] = { 0x21, 0x00 };
 	unsigned char *msg;
 
 	if (close) {
@@ -84,9 +84,9 @@ static int tda8295_i2c_bridge(struct dvb_frontend *fe, int close)
 {
 	struct tda8290_priv *priv = fe->analog_demod_priv;
 
-	unsigned char  enable[2] = { 0x45, 0xc1 };
-	unsigned char disable[2] = { 0x46, 0x00 };
-	unsigned char buf[3] = { 0x45, 0x01, 0x00 };
+	static unsigned char  enable[2] = { 0x45, 0xc1 };
+	static unsigned char disable[2] = { 0x46, 0x00 };
+	static unsigned char buf[3] = { 0x45, 0x01, 0x00 };
 	unsigned char *msg;
 
 	if (close) {
@@ -178,25 +178,25 @@ static void tda8290_set_params(struct dvb_frontend *fe,
 {
 	struct tda8290_priv *priv = fe->analog_demod_priv;
 
-	unsigned char soft_reset[]  = { 0x00, 0x00 };
+	static unsigned char soft_reset[]  = { 0x00, 0x00 };
 	unsigned char easy_mode[]   = { 0x01, priv->tda8290_easy_mode };
-	unsigned char expert_mode[] = { 0x01, 0x80 };
-	unsigned char agc_out_on[]  = { 0x02, 0x00 };
-	unsigned char gainset_off[] = { 0x28, 0x14 };
-	unsigned char if_agc_spd[]  = { 0x0f, 0x88 };
-	unsigned char adc_head_6[]  = { 0x05, 0x04 };
-	unsigned char adc_head_9[]  = { 0x05, 0x02 };
-	unsigned char adc_head_12[] = { 0x05, 0x01 };
-	unsigned char pll_bw_nom[]  = { 0x0d, 0x47 };
-	unsigned char pll_bw_low[]  = { 0x0d, 0x27 };
-	unsigned char gainset_2[]   = { 0x28, 0x64 };
-	unsigned char agc_rst_on[]  = { 0x0e, 0x0b };
-	unsigned char agc_rst_off[] = { 0x0e, 0x09 };
-	unsigned char if_agc_set[]  = { 0x0f, 0x81 };
-	unsigned char addr_adc_sat  = 0x1a;
-	unsigned char addr_agc_stat = 0x1d;
-	unsigned char addr_pll_stat = 0x1b;
-	unsigned char adc_sat, agc_stat,
+	static unsigned char expert_mode[] = { 0x01, 0x80 };
+	static unsigned char agc_out_on[]  = { 0x02, 0x00 };
+	static unsigned char gainset_off[] = { 0x28, 0x14 };
+	static unsigned char if_agc_spd[]  = { 0x0f, 0x88 };
+	static unsigned char adc_head_6[]  = { 0x05, 0x04 };
+	static unsigned char adc_head_9[]  = { 0x05, 0x02 };
+	static unsigned char adc_head_12[] = { 0x05, 0x01 };
+	static unsigned char pll_bw_nom[]  = { 0x0d, 0x47 };
+	static unsigned char pll_bw_low[]  = { 0x0d, 0x27 };
+	static unsigned char gainset_2[]   = { 0x28, 0x64 };
+	static unsigned char agc_rst_on[]  = { 0x0e, 0x0b };
+	static unsigned char agc_rst_off[] = { 0x0e, 0x09 };
+	static unsigned char if_agc_set[]  = { 0x0f, 0x81 };
+	static unsigned char addr_adc_sat  = 0x1a;
+	static unsigned char addr_agc_stat = 0x1d;
+	static unsigned char addr_pll_stat = 0x1b;
+	static unsigned char adc_sat = 0, agc_stat = 0,
 		      pll_stat;
 	int i;
 
@@ -468,9 +468,9 @@ static void tda8290_standby(struct dvb_frontend *fe)
 {
 	struct tda8290_priv *priv = fe->analog_demod_priv;
 
-	unsigned char cb1[] = { 0x30, 0xD0 };
-	unsigned char tda8290_standby[] = { 0x00, 0x02 };
-	unsigned char tda8290_agc_tri[] = { 0x02, 0x20 };
+	static unsigned char cb1[] = { 0x30, 0xD0 };
+	static unsigned char tda8290_standby[] = { 0x00, 0x02 };
+	static unsigned char tda8290_agc_tri[] = { 0x02, 0x20 };
 	struct i2c_msg msg = {.addr = priv->tda827x_addr, .flags=0, .buf=cb1, .len = 2};
 
 	if (fe->ops.analog_ops.i2c_gate_ctrl)
@@ -495,9 +495,9 @@ static void tda8290_init_if(struct dvb_frontend *fe)
 {
 	struct tda8290_priv *priv = fe->analog_demod_priv;
 
-	unsigned char set_VS[] = { 0x30, 0x6F };
-	unsigned char set_GP00_CF[] = { 0x20, 0x01 };
-	unsigned char set_GP01_CF[] = { 0x20, 0x0B };
+	static unsigned char set_VS[] = { 0x30, 0x6F };
+	static unsigned char set_GP00_CF[] = { 0x20, 0x01 };
+	static unsigned char set_GP01_CF[] = { 0x20, 0x0B };
 
 	if ((priv->cfg.config == TDA8290_LNA_GP0_HIGH_ON) ||
 	    (priv->cfg.config == TDA8290_LNA_GP0_HIGH_OFF))
@@ -539,10 +539,12 @@ static void tda8295_init_if(struct dvb_frontend *fe)
 static void tda8290_init_tuner(struct dvb_frontend *fe)
 {
 	struct tda8290_priv *priv = fe->analog_demod_priv;
-	unsigned char tda8275_init[]  = { 0x00, 0x00, 0x00, 0x40, 0xdC, 0x04, 0xAf,
-					  0x3F, 0x2A, 0x04, 0xFF, 0x00, 0x00, 0x40 };
-	unsigned char tda8275a_init[] = { 0x00, 0x00, 0x00, 0x00, 0xdC, 0x05, 0x8b,
-					  0x0c, 0x04, 0x20, 0xFF, 0x00, 0x00, 0x4b };
+	static unsigned char tda8275_init[]  =
+		{ 0x00, 0x00, 0x00, 0x40, 0xdC, 0x04, 0xAf,
+		  0x3F, 0x2A, 0x04, 0xFF, 0x00, 0x00, 0x40 };
+	static unsigned char tda8275a_init[] =
+		 { 0x00, 0x00, 0x00, 0x00, 0xdC, 0x05, 0x8b,
+		   0x0c, 0x04, 0x20, 0xFF, 0x00, 0x00, 0x4b };
 	struct i2c_msg msg = {.addr = priv->tda827x_addr, .flags=0,
 			      .buf=tda8275_init, .len = 14};
 	if (priv->ver & TDA8275A)
@@ -834,11 +836,11 @@ int tda829x_probe(struct i2c_adapter *i2c_adap, u8 i2c_addr)
 		.addr = i2c_addr,
 	};
 
-	unsigned char soft_reset[]   = { 0x00, 0x00 };
-	unsigned char easy_mode_b[]  = { 0x01, 0x02 };
-	unsigned char easy_mode_g[]  = { 0x01, 0x04 };
-	unsigned char restore_9886[] = { 0x00, 0xd6, 0x30 };
-	unsigned char addr_dto_lsb = 0x07;
+	static unsigned char soft_reset[]   = { 0x00, 0x00 };
+	static unsigned char easy_mode_b[]  = { 0x01, 0x02 };
+	static unsigned char easy_mode_g[]  = { 0x01, 0x04 };
+	static unsigned char restore_9886[] = { 0x00, 0xd6, 0x30 };
+	static unsigned char addr_dto_lsb = 0x07;
 	unsigned char data;
 #define PROBE_BUFFER_SIZE 8
 	unsigned char buf[PROBE_BUFFER_SIZE];
diff --git a/drivers/media/tuners/tda8290.h b/drivers/media/tuners/tda8290.h
index 901b8cac7105..5db79f16ad7d 100644
--- a/drivers/media/tuners/tda8290.h
+++ b/drivers/media/tuners/tda8290.h
@@ -18,7 +18,7 @@
 #define __TDA8290_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "tda18271.h"
 
 enum tda8290_lna {
diff --git a/drivers/media/tuners/tda9887.c b/drivers/media/tuners/tda9887.c
index c0e815f8b951..9777da03e308 100644
--- a/drivers/media/tuners/tda9887.c
+++ b/drivers/media/tuners/tda9887.c
@@ -31,7 +31,7 @@ struct tda9887_priv {
 	struct tuner_i2c_props i2c_props;
 	struct list_head hybrid_tuner_instance_list;
 
-	unsigned char 	   data[4];
+	unsigned char	   data[4];
 	unsigned int       config;
 	unsigned int       mode;
 	unsigned int       audmode;
@@ -94,7 +94,7 @@ struct tvnorm {
 #define cAudioGain6             0x80    // bit c7
 
 #define cTopMask                0x1f    // bit c0:4
-#define cTopDefault		0x10 	// bit c0:4
+#define cTopDefault		0x10	// bit c0:4
 
 //// third reg (e)
 #define cAudioIF_4_5             0x00    // bit e0:1
diff --git a/drivers/media/tuners/tda9887.h b/drivers/media/tuners/tda9887.h
index 95070eca02ca..2a143f8c6477 100644
--- a/drivers/media/tuners/tda9887.h
+++ b/drivers/media/tuners/tda9887.h
@@ -18,7 +18,7 @@
 #define __TDA9887_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /* ------------------------------------------------------------------------ */
 #if IS_REACHABLE(CONFIG_MEDIA_TUNER_TDA9887)
diff --git a/drivers/media/tuners/tea5761.c b/drivers/media/tuners/tea5761.c
index a9b1bb134409..88b3e80c38ad 100644
--- a/drivers/media/tuners/tea5761.c
+++ b/drivers/media/tuners/tea5761.c
@@ -1,11 +1,8 @@
-/*
- * For Philips TEA5761 FM Chip
- * I2C address is allways 0x20 (0x10 at 7-bit mode).
- *
- * Copyright (c) 2005-2007 Mauro Carvalho Chehab (mchehab@infradead.org)
- * This code is placed under the terms of the GNUv2 General Public License
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+// For Philips TEA5761 FM Chip
+// I2C address is always 0x20 (0x10 at 7-bit mode).
+//
+// Copyright (c) 2005-2007 Mauro Carvalho Chehab (mchehab@infradead.org)
 
 #include <linux/i2c.h>
 #include <linux/slab.h>
@@ -341,4 +338,4 @@ EXPORT_SYMBOL_GPL(tea5761_autodetection);
 
 MODULE_DESCRIPTION("Philips TEA5761 FM tuner driver");
 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@infradead.org>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/tuners/tea5761.h b/drivers/media/tuners/tea5761.h
index 2d624d9919e3..4bcf835fc613 100644
--- a/drivers/media/tuners/tea5761.h
+++ b/drivers/media/tuners/tea5761.h
@@ -18,7 +18,7 @@
 #define __TEA5761_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #if IS_REACHABLE(CONFIG_MEDIA_TUNER_TEA5761)
 extern int tea5761_autodetection(struct i2c_adapter* i2c_adap, u8 i2c_addr);
diff --git a/drivers/media/tuners/tea5767.c b/drivers/media/tuners/tea5767.c
index 525b7ab90c80..2b2c064d7dc3 100644
--- a/drivers/media/tuners/tea5767.c
+++ b/drivers/media/tuners/tea5767.c
@@ -1,14 +1,11 @@
-/*
- * For Philips TEA5767 FM Chip used on some TV Cards like Prolink Pixelview
- * I2C address is allways 0xC0.
- *
- *
- * Copyright (c) 2005 Mauro Carvalho Chehab (mchehab@infradead.org)
- * This code is placed under the terms of the GNU General Public License
- *
- * tea5767 autodetection thanks to Torsten Seeboth and Atsushi Nakagawa
- * from their contributions on DScaler.
- */
+// SPDX-License-Identifier: GPL-2.0
+// For Philips TEA5767 FM Chip used on some TV Cards like Prolink Pixelview
+// I2C address is always 0xC0.
+//
+// Copyright (c) 2005 Mauro Carvalho Chehab (mchehab@infradead.org)
+//
+// tea5767 autodetection thanks to Torsten Seeboth and Atsushi Nakagawa
+// from their contributions on DScaler.
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
@@ -473,4 +470,4 @@ EXPORT_SYMBOL_GPL(tea5767_autodetection);
 
 MODULE_DESCRIPTION("Philips TEA5767 FM tuner driver");
 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@infradead.org>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/tuners/tea5767.h b/drivers/media/tuners/tea5767.h
index 4f6f6c92db78..216a3192a35f 100644
--- a/drivers/media/tuners/tea5767.h
+++ b/drivers/media/tuners/tea5767.h
@@ -18,7 +18,7 @@
 #define __TEA5767_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 enum tea5767_xtal {
 	TEA5767_LOW_LO_32768    = 0,
diff --git a/drivers/media/tuners/tua9001.h b/drivers/media/tuners/tua9001.h
index 7b0548181cdc..4df2c16592fb 100644
--- a/drivers/media/tuners/tua9001.h
+++ b/drivers/media/tuners/tua9001.h
@@ -17,7 +17,7 @@
 #ifndef TUA9001_H
 #define TUA9001_H
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /*
  * I2C address
diff --git a/drivers/media/tuners/tuner-i2c.h b/drivers/media/tuners/tuner-i2c.h
index bda67a5a76f2..56dc2339a989 100644
--- a/drivers/media/tuners/tuner-i2c.h
+++ b/drivers/media/tuners/tuner-i2c.h
@@ -75,7 +75,7 @@ static inline int tuner_i2c_xfer_send_recv(struct tuner_i2c_props *props,
  *
  * state structure must contain the following:
  *
- * 	struct list_head	hybrid_tuner_instance_list;
+ *	struct list_head	hybrid_tuner_instance_list;
  *	struct tuner_i2c_props	i2c_props;
  *
  * hybrid_tuner_instance_list (both within state structure and globally)
diff --git a/drivers/media/tuners/tuner-simple.c b/drivers/media/tuners/tuner-simple.c
index cf44d3657f55..36b88f820239 100644
--- a/drivers/media/tuners/tuner-simple.c
+++ b/drivers/media/tuners/tuner-simple.c
@@ -53,7 +53,7 @@ MODULE_PARM_DESC(dtv_input, "specify dtv rf input, 0 for autoselect");
 /* tv tuner system standard selection for Philips FQ1216ME
    this value takes the low bits of control byte 2
    from datasheet "1999 Nov 16" (supersedes "1999 Mar 23")
-     standard 		BG	DK	I	L	L`
+     standard		BG	DK	I	L	L`
      picture carrier	38.90	38.90	38.90	38.90	33.95
      colour		34.47	34.47	34.47	34.47	38.38
      sound 1		33.40	32.40	32.90	32.40	40.45
diff --git a/drivers/media/tuners/tuner-simple.h b/drivers/media/tuners/tuner-simple.h
index 6399b45b0590..fd71b3490dc8 100644
--- a/drivers/media/tuners/tuner-simple.h
+++ b/drivers/media/tuners/tuner-simple.h
@@ -18,7 +18,7 @@
 #define __TUNER_SIMPLE_H__
 
 #include <linux/i2c.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #if IS_REACHABLE(CONFIG_MEDIA_TUNER_SIMPLE)
 extern struct dvb_frontend *simple_tuner_attach(struct dvb_frontend *fe,
diff --git a/drivers/media/tuners/tuner-xc2028-types.h b/drivers/media/tuners/tuner-xc2028-types.h
index 7e4798783db7..bb0437c36c03 100644
--- a/drivers/media/tuners/tuner-xc2028-types.h
+++ b/drivers/media/tuners/tuner-xc2028-types.h
@@ -1,10 +1,11 @@
-/* tuner-xc2028_types
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ * tuner-xc2028_types
  *
  * This file includes internal tipes to be used inside tuner-xc2028.
  * Shouldn't be included outside tuner-xc2028
  *
  * Copyright (c) 2007-2008 Mauro Carvalho Chehab (mchehab@infradead.org)
- * This code is placed under the terms of the GNU General Public License v2
  */
 
 /* xc3028 firmware types */
diff --git a/drivers/media/tuners/tuner-xc2028.c b/drivers/media/tuners/tuner-xc2028.c
index b5b62b08159e..fca85e08ebd7 100644
--- a/drivers/media/tuners/tuner-xc2028.c
+++ b/drivers/media/tuners/tuner-xc2028.c
@@ -1,12 +1,10 @@
-/* tuner-xc2028
- *
- * Copyright (c) 2007-2008 Mauro Carvalho Chehab (mchehab@infradead.org)
- *
- * Copyright (c) 2007 Michel Ludwig (michel.ludwig@gmail.com)
- *       - frontend interface
- *
- * This code is placed under the terms of the GNU General Public License v2
- */
+// SPDX-License-Identifier: GPL-2.0
+// tuner-xc2028
+//
+// Copyright (c) 2007-2008 Mauro Carvalho Chehab (mchehab@infradead.org)
+//
+// Copyright (c) 2007 Michel Ludwig (michel.ludwig@gmail.com)
+//       - frontend interface
 
 #include <linux/i2c.h>
 #include <asm/div64.h>
@@ -22,7 +20,7 @@
 #include "tuner-xc2028-types.h"
 
 #include <linux/dvb/frontend.h>
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 /* Max transfer size done by I2C transfer functions */
 #define MAX_XFER_SIZE  80
@@ -89,7 +87,7 @@ struct firmware_properties {
 	v4l2_std_id	std_req;
 	__u16		int_freq;
 	unsigned int	scode_table;
-	int 		scode_nr;
+	int		scode_nr;
 };
 
 enum xc2028_state {
@@ -139,7 +137,7 @@ struct xc2028_data {
 				       ibuf, isize);			\
 	if (isize != _rc)						\
 		tuner_err("i2c input error: rc = %d (should be %d)\n",	\
-			   _rc, (int)isize); 				\
+			   _rc, (int)isize);				\
 	if (priv->ctrl.msleep)						\
 		msleep(priv->ctrl.msleep);				\
 	_rc;								\
@@ -174,7 +172,7 @@ static int xc2028_get_reg(struct xc2028_data *priv, u16 reg, u16 *val)
 	return 0;
 }
 
-#define dump_firm_type(t) 	dump_firm_type_and_int_freq(t, 0)
+#define dump_firm_type(t)	dump_firm_type_and_int_freq(t, 0)
 static void dump_firm_type_and_int_freq(unsigned int type, u16 int_freq)
 {
 	if (type & BASE)
@@ -1521,6 +1519,6 @@ EXPORT_SYMBOL(xc2028_attach);
 MODULE_DESCRIPTION("Xceive xc2028/xc3028 tuner driver");
 MODULE_AUTHOR("Michel Ludwig <michel.ludwig@gmail.com>");
 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@infradead.org>");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
 MODULE_FIRMWARE(XC2028_DEFAULT_FIRMWARE);
 MODULE_FIRMWARE(XC3028L_DEFAULT_FIRMWARE);
diff --git a/drivers/media/tuners/tuner-xc2028.h b/drivers/media/tuners/tuner-xc2028.h
index 98e4effca896..03fd6d4233a4 100644
--- a/drivers/media/tuners/tuner-xc2028.h
+++ b/drivers/media/tuners/tuner-xc2028.h
@@ -1,13 +1,14 @@
-/* tuner-xc2028
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ * tuner-xc2028
  *
  * Copyright (c) 2007-2008 Mauro Carvalho Chehab (mchehab@infradead.org)
- * This code is placed under the terms of the GNU General Public License v2
  */
 
 #ifndef __TUNER_XC2028_H__
 #define __TUNER_XC2028_H__
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #define XC2028_DEFAULT_FIRMWARE "xc3028-v27.fw"
 #define XC3028L_DEFAULT_FIRMWARE "xc3028L-v36.fw"
@@ -47,7 +48,7 @@ struct xc2028_ctrl {
 
 struct xc2028_config {
 	struct i2c_adapter *i2c_adap;
-	u8 		   i2c_addr;
+	u8		   i2c_addr;
 	struct xc2028_ctrl *ctrl;
 };
 
diff --git a/drivers/media/tuners/xc4000.c b/drivers/media/tuners/xc4000.c
index e30948e4ff87..f0fa8da08afa 100644
--- a/drivers/media/tuners/xc4000.c
+++ b/drivers/media/tuners/xc4000.c
@@ -27,7 +27,7 @@
 #include <linux/mutex.h>
 #include <asm/unaligned.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "xc4000.h"
 #include "tuner-i2c.h"
@@ -1036,7 +1036,10 @@ static int check_firmware(struct dvb_frontend *fe, unsigned int type,
 		dprintk(1, "load scode failed %d\n", rc);
 
 check_device:
-	rc = xc4000_readreg(priv, XREG_PRODUCT_ID, &hwmodel);
+	if (xc4000_readreg(priv, XREG_PRODUCT_ID, &hwmodel) < 0) {
+		printk(KERN_ERR "Unable to read tuner registers.\n");
+		goto fail;
+	}
 
 	if (xc_get_version(priv, &hw_major, &hw_minor, &fw_major,
 			   &fw_minor) != 0) {
diff --git a/drivers/media/tuners/xc5000.c b/drivers/media/tuners/xc5000.c
index 0e7e4fdf9e50..f7a8d05d1758 100644
--- a/drivers/media/tuners/xc5000.c
+++ b/drivers/media/tuners/xc5000.c
@@ -25,7 +25,7 @@
 #include <linux/dvb/frontend.h>
 #include <linux/i2c.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "xc5000.h"
 #include "tuner-i2c.h"
@@ -685,8 +685,8 @@ static void xc_debug_dump(struct xc5000_priv *priv)
 		(totalgain % 256) * 100 / 256);
 
 	if (priv->pll_register_no) {
-		xc5000_readreg(priv, priv->pll_register_no, &regval);
-		dprintk(1, "*** PLL lock status = 0x%04x\n", regval);
+		if (!xc5000_readreg(priv, priv->pll_register_no, &regval))
+			dprintk(1, "*** PLL lock status = 0x%04x\n", regval);
 	}
 }
 
@@ -831,15 +831,16 @@ static int xc5000_is_firmware_loaded(struct dvb_frontend *fe)
 	u16 id;
 
 	ret = xc5000_readreg(priv, XREG_PRODUCT_ID, &id);
-	if (ret == 0) {
+	if (!ret) {
 		if (id == XC_PRODUCT_ID_FW_NOT_LOADED)
 			ret = -ENOENT;
 		else
 			ret = 0;
+		dprintk(1, "%s() returns id = 0x%x\n", __func__, id);
+	} else {
+		dprintk(1, "%s() returns error %d\n", __func__, ret);
 	}
 
-	dprintk(1, "%s() returns %s id = 0x%x\n", __func__,
-		ret == 0 ? "True" : "False", id);
 	return ret;
 }
 
@@ -935,7 +936,10 @@ static int xc5000_set_tv_freq(struct dvb_frontend *fe)
 
 	if (priv->pll_register_no != 0) {
 		msleep(20);
-		xc5000_readreg(priv, priv->pll_register_no, &pll_lock_status);
+		ret = xc5000_readreg(priv, priv->pll_register_no,
+				     &pll_lock_status);
+		if (ret)
+			return ret;
 		if (pll_lock_status > 63) {
 			/* PLL is unlocked, force reload of the firmware */
 			dprintk(1, "xc5000: PLL not locked (0x%x).  Reloading...\n",
@@ -1190,8 +1194,10 @@ static int xc_load_fw_and_init_tuner(struct dvb_frontend *fe, int force)
 		}
 
 		if (priv->pll_register_no) {
-			xc5000_readreg(priv, priv->pll_register_no,
-				       &pll_lock_status);
+			ret = xc5000_readreg(priv, priv->pll_register_no,
+					     &pll_lock_status);
+			if (ret)
+				continue;
 			if (pll_lock_status > 63) {
 				/* PLL is unlocked, force reload of the firmware */
 				printk(KERN_ERR
diff --git a/drivers/media/usb/as102/Makefile b/drivers/media/usb/as102/Makefile
index 56bd2d00b920..b0b319622edb 100644
--- a/drivers/media/usb/as102/Makefile
+++ b/drivers/media/usb/as102/Makefile
@@ -4,5 +4,4 @@ dvb-as102-objs := as102_drv.o as102_fw.o as10x_cmd.o as10x_cmd_stream.o \
 
 obj-$(CONFIG_DVB_AS102) += dvb-as102.o
 
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/usb/as102/as102_drv.c b/drivers/media/usb/as102/as102_drv.c
index 9dd7c7cb06b1..48b0c4e4dac1 100644
--- a/drivers/media/usb/as102/as102_drv.c
+++ b/drivers/media/usb/as102/as102_drv.c
@@ -27,7 +27,7 @@
 #include "as10x_cmd.h"
 #include "as102_fe.h"
 #include "as102_fw.h"
-#include "dvbdev.h"
+#include <media/dvbdev.h>
 
 int dual_tuner;
 module_param_named(dual_tuner, dual_tuner, int, 0644);
diff --git a/drivers/media/usb/as102/as102_drv.h b/drivers/media/usb/as102/as102_drv.h
index 8def19d9ab92..c92a1e4f6a20 100644
--- a/drivers/media/usb/as102/as102_drv.h
+++ b/drivers/media/usb/as102/as102_drv.h
@@ -16,9 +16,9 @@
 #ifndef _AS102_DRV_H
 #define _AS102_DRV_H
 #include <linux/usb.h>
-#include <dvb_demux.h>
-#include <dvb_frontend.h>
-#include <dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dmxdev.h>
 #include "as10x_handle.h"
 #include "as10x_cmd.h"
 #include "as102_usb_drv.h"
diff --git a/drivers/media/usb/as102/as10x_cmd_cfg.c b/drivers/media/usb/as102/as10x_cmd_cfg.c
index c87f2ca223a2..fabbfead96d8 100644
--- a/drivers/media/usb/as102/as10x_cmd_cfg.c
+++ b/drivers/media/usb/as102/as10x_cmd_cfg.c
@@ -133,9 +133,9 @@ int as10x_cmd_set_context(struct as10x_bus_adapter_t *adap, uint16_t tag,
  * as10x_cmd_eLNA_change_mode - send eLNA change mode command to AS10x
  * @adap:      pointer to AS10x bus adapter
  * @mode:      mode selected:
- *	        - ON    : 0x0 => eLNA always ON
- *	        - OFF   : 0x1 => eLNA always OFF
- *	        - AUTO  : 0x2 => eLNA follow hysteresis parameters
+ *		- ON    : 0x0 => eLNA always ON
+ *		- OFF   : 0x1 => eLNA always OFF
+ *		- AUTO  : 0x2 => eLNA follow hysteresis parameters
  *				 to be ON or OFF
  *
  * Return 0 on success or negative value in case of error.
diff --git a/drivers/media/usb/au0828/Kconfig b/drivers/media/usb/au0828/Kconfig
index 70521e0b4c53..bfaa806633df 100644
--- a/drivers/media/usb/au0828/Kconfig
+++ b/drivers/media/usb/au0828/Kconfig
@@ -1,7 +1,7 @@
 
 config VIDEO_AU0828
 	tristate "Auvitek AU0828 support"
-	depends on I2C && INPUT && DVB_CORE && USB
+	depends on I2C && INPUT && DVB_CORE && USB && VIDEO_V4L2
 	select I2C_ALGOBIT
 	select VIDEO_TVEEPROM
 	select VIDEOBUF2_VMALLOC
diff --git a/drivers/media/usb/au0828/Makefile b/drivers/media/usb/au0828/Makefile
index c06ef6601f2d..5691881c56c0 100644
--- a/drivers/media/usb/au0828/Makefile
+++ b/drivers/media/usb/au0828/Makefile
@@ -12,7 +12,6 @@ endif
 obj-$(CONFIG_VIDEO_AU0828) += au0828.o
 
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
 
 ccflags-y += $(extra-cflags-y) $(extra-cflags-m)
diff --git a/drivers/media/usb/au0828/au0828-cards.h b/drivers/media/usb/au0828/au0828-cards.h
index 1f4412ee6da4..dbd8a90ee76f 100644
--- a/drivers/media/usb/au0828/au0828-cards.h
+++ b/drivers/media/usb/au0828/au0828-cards.h
@@ -17,7 +17,7 @@
 
 #define AU0828_BOARD_UNKNOWN		0
 #define AU0828_BOARD_HAUPPAUGE_HVR950Q	1
-#define AU0828_BOARD_HAUPPAUGE_HVR850 	2
+#define AU0828_BOARD_HAUPPAUGE_HVR850	2
 #define AU0828_BOARD_DVICO_FUSIONHDTV7	3
 #define AU0828_BOARD_HAUPPAUGE_HVR950Q_MXL	4
 #define AU0828_BOARD_HAUPPAUGE_WOODBURY	5
diff --git a/drivers/media/usb/au0828/au0828-input.c b/drivers/media/usb/au0828/au0828-input.c
index af68afe085b5..832ed9f25784 100644
--- a/drivers/media/usb/au0828/au0828-input.c
+++ b/drivers/media/usb/au0828/au0828-input.c
@@ -1,21 +1,10 @@
-/*
-  handle au0828 IR remotes via linux kernel input layer.
-
-   Copyright (C) 2014 Mauro Carvalho Chehab <mchehab@samsung.com>
-   Copyright (c) 2014 Samsung Electronics Co., Ltd.
-
-  Based on em28xx-input.c.
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License as published by
-  the Free Software Foundation; either version 2 of the License, or
-  (at your option) any later version.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// handle au0828 IR remotes via linux kernel input layer.
+//
+// Copyright (c) 2014 Mauro Carvalho Chehab <mchehab@samsung.com>
+// Copyright (c) 2014 Samsung Electronics Co., Ltd.
+//
+// Based on em28xx-input.c.
 
 #include "au0828.h"
 
diff --git a/drivers/media/usb/au0828/au0828-video.c b/drivers/media/usb/au0828/au0828-video.c
index a240153821e0..c765d546114d 100644
--- a/drivers/media/usb/au0828/au0828-video.c
+++ b/drivers/media/usb/au0828/au0828-video.c
@@ -1797,7 +1797,7 @@ static const struct v4l2_ioctl_ops video_ioctl_ops = {
 static const struct video_device au0828_video_template = {
 	.fops                       = &au0828_v4l_fops,
 	.release                    = video_device_release_empty,
-	.ioctl_ops 		    = &video_ioctl_ops,
+	.ioctl_ops		    = &video_ioctl_ops,
 	.tvnorms                    = V4L2_STD_NTSC_M | V4L2_STD_PAL_M,
 };
 
diff --git a/drivers/media/usb/au0828/au0828.h b/drivers/media/usb/au0828/au0828.h
index f6f37e8ef51d..004eadef55c7 100644
--- a/drivers/media/usb/au0828/au0828.h
+++ b/drivers/media/usb/au0828/au0828.h
@@ -33,12 +33,12 @@
 #include <media/media-device.h>
 
 /* DVB */
-#include "demux.h"
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
-#include "dvbdev.h"
+#include <media/demux.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
+#include <media/dvbdev.h>
 
 #include "au0828-reg.h"
 #include "au0828-cards.h"
@@ -190,7 +190,7 @@ struct au0828_dev {
 	struct i2c_adapter		i2c_adap;
 	struct i2c_algorithm		i2c_algo;
 	struct i2c_client		i2c_client;
-	u32 				i2c_rc;
+	u32				i2c_rc;
 
 	/* Digital */
 	struct au0828_dvb		dvb;
@@ -293,8 +293,8 @@ struct au0828_dev {
 /* ----------------------------------------------------------- */
 #define au0828_read(dev, reg) au0828_readreg(dev, reg)
 #define au0828_write(dev, reg, value) au0828_writereg(dev, reg, value)
-#define au0828_andor(dev, reg, mask, value) 				\
-	 au0828_writereg(dev, reg, 					\
+#define au0828_andor(dev, reg, mask, value)				\
+	 au0828_writereg(dev, reg,					\
 	(au0828_readreg(dev, reg) & ~(mask)) | ((value) & (mask)))
 
 #define au0828_set(dev, reg, bit) au0828_andor(dev, (reg), (bit), (bit))
diff --git a/drivers/media/usb/b2c2/Makefile b/drivers/media/usb/b2c2/Makefile
index 2778c19a45eb..f3cef05f37b6 100644
--- a/drivers/media/usb/b2c2/Makefile
+++ b/drivers/media/usb/b2c2/Makefile
@@ -1,5 +1,4 @@
 b2c2-flexcop-usb-objs := flexcop-usb.o
 obj-$(CONFIG_DVB_B2C2_FLEXCOP_USB) += b2c2-flexcop-usb.o
 
-ccflags-y += -Idrivers/media/dvb-core/
 ccflags-y += -Idrivers/media/common/b2c2/
diff --git a/drivers/media/usb/cpia2/cpia2.h b/drivers/media/usb/cpia2/cpia2.h
index 81f72c0b561f..ab238ac8bfc0 100644
--- a/drivers/media/usb/cpia2/cpia2.h
+++ b/drivers/media/usb/cpia2/cpia2.h
@@ -444,7 +444,7 @@ int cpia2_allocate_buffers(struct camera_data *cam);
 void cpia2_free_buffers(struct camera_data *cam);
 long cpia2_read(struct camera_data *cam,
 		char __user *buf, unsigned long count, int noblock);
-unsigned int cpia2_poll(struct camera_data *cam,
+__poll_t cpia2_poll(struct camera_data *cam,
 			struct file *filp, poll_table *wait);
 int cpia2_remap_buffer(struct camera_data *cam, struct vm_area_struct *vma);
 void cpia2_set_property_flip(struct camera_data *cam, int prop_val);
diff --git a/drivers/media/usb/cpia2/cpia2_core.c b/drivers/media/usb/cpia2/cpia2_core.c
index 0efba0da0a45..3dfbb545c0e3 100644
--- a/drivers/media/usb/cpia2/cpia2_core.c
+++ b/drivers/media/usb/cpia2/cpia2_core.c
@@ -2370,12 +2370,12 @@ long cpia2_read(struct camera_data *cam,
  *  cpia2_poll
  *
  *****************************************************************************/
-unsigned int cpia2_poll(struct camera_data *cam, struct file *filp,
+__poll_t cpia2_poll(struct camera_data *cam, struct file *filp,
 			poll_table *wait)
 {
-	unsigned int status = v4l2_ctrl_poll(filp, wait);
+	__poll_t status = v4l2_ctrl_poll(filp, wait);
 
-	if ((poll_requested_events(wait) & (POLLIN | POLLRDNORM)) &&
+	if ((poll_requested_events(wait) & (EPOLLIN | EPOLLRDNORM)) &&
 			!cam->streaming) {
 		/* Start streaming */
 		cpia2_usb_stream_start(cam,
@@ -2385,7 +2385,7 @@ unsigned int cpia2_poll(struct camera_data *cam, struct file *filp,
 	poll_wait(filp, &cam->wq_stream, wait);
 
 	if (cam->curbuff->status == FRAME_READY)
-		status |= POLLIN | POLLRDNORM;
+		status |= EPOLLIN | EPOLLRDNORM;
 
 	return status;
 }
diff --git a/drivers/media/usb/cpia2/cpia2_usb.c b/drivers/media/usb/cpia2/cpia2_usb.c
index 6089036049d9..f3a1e5b1e57c 100644
--- a/drivers/media/usb/cpia2/cpia2_usb.c
+++ b/drivers/media/usb/cpia2/cpia2_usb.c
@@ -33,13 +33,13 @@
 
 static int frame_sizes[] = {
 	0,	// USBIF_CMDONLY
-	0, 	// USBIF_BULK
-	128, 	// USBIF_ISO_1
-	384, 	// USBIF_ISO_2
-	640, 	// USBIF_ISO_3
-	768, 	// USBIF_ISO_4
-	896, 	// USBIF_ISO_5
-	1023, 	// USBIF_ISO_6
+	0,	// USBIF_BULK
+	128,	// USBIF_ISO_1
+	384,	// USBIF_ISO_2
+	640,	// USBIF_ISO_3
+	768,	// USBIF_ISO_4
+	896,	// USBIF_ISO_5
+	1023,	// USBIF_ISO_6
 };
 
 #define FRAMES_PER_DESC    10
diff --git a/drivers/media/usb/cpia2/cpia2_v4l.c b/drivers/media/usb/cpia2/cpia2_v4l.c
index 3dedd83f0b19..99f106b13280 100644
--- a/drivers/media/usb/cpia2/cpia2_v4l.c
+++ b/drivers/media/usb/cpia2/cpia2_v4l.c
@@ -169,10 +169,10 @@ static ssize_t cpia2_v4l_read(struct file *file, char __user *buf, size_t count,
  *  cpia2_v4l_poll
  *
  *****************************************************************************/
-static unsigned int cpia2_v4l_poll(struct file *filp, struct poll_table_struct *wait)
+static __poll_t cpia2_v4l_poll(struct file *filp, struct poll_table_struct *wait)
 {
 	struct camera_data *cam = video_drvdata(filp);
-	unsigned int res;
+	__poll_t res;
 
 	mutex_lock(&cam->v4l2_lock);
 	res = cpia2_poll(cam, filp, wait);
@@ -808,7 +808,7 @@ static int cpia2_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 	struct camera_data *cam = video_drvdata(file);
 
 	if(buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
-	   buf->index > cam->num_frames)
+	   buf->index >= cam->num_frames)
 		return -EINVAL;
 
 	buf->m.offset = cam->buffers[buf->index].data - cam->frame_buffer;
@@ -859,7 +859,7 @@ static int cpia2_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf)
 
 	if(buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE ||
 	   buf->memory != V4L2_MEMORY_MMAP ||
-	   buf->index > cam->num_frames)
+	   buf->index >= cam->num_frames)
 		return -EINVAL;
 
 	DBG("QBUF #%d\n", buf->index);
diff --git a/drivers/media/usb/cx231xx/Makefile b/drivers/media/usb/cx231xx/Makefile
index 19e8c35d6a77..c023d97407de 100644
--- a/drivers/media/usb/cx231xx/Makefile
+++ b/drivers/media/usb/cx231xx/Makefile
@@ -9,8 +9,6 @@ obj-$(CONFIG_VIDEO_CX231XX) += cx231xx.o
 obj-$(CONFIG_VIDEO_CX231XX_ALSA) += cx231xx-alsa.o
 obj-$(CONFIG_VIDEO_CX231XX_DVB) += cx231xx-dvb.o
 
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
 ccflags-y += -Idrivers/media/usb/dvb-usb
diff --git a/drivers/media/usb/cx231xx/cx231xx-417.c b/drivers/media/usb/cx231xx/cx231xx-417.c
index d538fa407742..b80e6857e2eb 100644
--- a/drivers/media/usb/cx231xx/cx231xx-417.c
+++ b/drivers/media/usb/cx231xx/cx231xx-417.c
@@ -1812,20 +1812,20 @@ static ssize_t mpeg_read(struct file *file, char __user *data,
 				    file->f_flags & O_NONBLOCK);
 }
 
-static unsigned int mpeg_poll(struct file *file,
+static __poll_t mpeg_poll(struct file *file,
 	struct poll_table_struct *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct cx231xx_fh *fh = file->private_data;
 	struct cx231xx *dev = fh->dev;
-	unsigned int res = 0;
+	__poll_t res = 0;
 
 	if (v4l2_event_pending(&fh->fh))
-		res |= POLLPRI;
+		res |= EPOLLPRI;
 	else
 		poll_wait(file, &fh->fh.wait, wait);
 
-	if (!(req_events & (POLLIN | POLLRDNORM)))
+	if (!(req_events & (EPOLLIN | EPOLLRDNORM)))
 		return res;
 
 	mutex_lock(&dev->lock);
diff --git a/drivers/media/usb/cx231xx/cx231xx-audio.c b/drivers/media/usb/cx231xx/cx231xx-audio.c
index 06f10d7fc4b0..d96236d786d1 100644
--- a/drivers/media/usb/cx231xx/cx231xx-audio.c
+++ b/drivers/media/usb/cx231xx/cx231xx-audio.c
@@ -404,9 +404,9 @@ static int snd_pcm_alloc_vmalloc_buffer(struct snd_pcm_substream *subs,
 }
 
 static const struct snd_pcm_hardware snd_cx231xx_hw_capture = {
-	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER 	|
-	    SNDRV_PCM_INFO_MMAP 		|
-	    SNDRV_PCM_INFO_INTERLEAVED 		|
+	.info = SNDRV_PCM_INFO_BLOCK_TRANSFER	|
+	    SNDRV_PCM_INFO_MMAP			|
+	    SNDRV_PCM_INFO_INTERLEAVED		|
 	    SNDRV_PCM_INFO_MMAP_VALID,
 
 	.formats = SNDRV_PCM_FMTBIT_S16_LE,
diff --git a/drivers/media/usb/cx231xx/cx231xx-avcore.c b/drivers/media/usb/cx231xx/cx231xx-avcore.c
index 2f52d66b4dae..fdd3c221fa0d 100644
--- a/drivers/media/usb/cx231xx/cx231xx-avcore.c
+++ b/drivers/media/usb/cx231xx/cx231xx-avcore.c
@@ -105,7 +105,7 @@ void uninitGPIO(struct cx231xx *dev)
 
 /******************************************************************************
  *                    A F E - B L O C K    C O N T R O L   functions          *
- * 				[ANALOG FRONT END]			      *
+ *				[ANALOG FRONT END]			      *
  ******************************************************************************/
 static int afe_write_byte(struct cx231xx *dev, u16 saddr, u8 data)
 {
@@ -2168,7 +2168,7 @@ int cx231xx_tuner_post_channel_change(struct cx231xx *dev)
 }
 
 /******************************************************************************
- *        	    I 2 S - B L O C K    C O N T R O L   functions            *
+ *		    I 2 S - B L O C K    C O N T R O L   functions            *
  ******************************************************************************/
 int cx231xx_i2s_blk_initialize(struct cx231xx *dev)
 {
diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c
index 54d9d0cb326f..f9ec7fedcd5b 100644
--- a/drivers/media/usb/cx231xx/cx231xx-cards.c
+++ b/drivers/media/usb/cx231xx/cx231xx-cards.c
@@ -31,7 +31,7 @@
 #include <media/v4l2-common.h>
 
 #include <media/drv-intf/cx25840.h>
-#include "dvb-usb-ids.h"
+#include <media/dvb-usb-ids.h>
 #include "xc5000.h"
 #include "tda18271.h"
 
@@ -896,6 +896,32 @@ struct cx231xx_board cx231xx_boards[] = {
 			},
 		},
 	},
+	[CX231XX_BOARD_THE_IMAGING_SOURCE_DFG_USB2_PRO] = {
+		.name = "The Imaging Source DFG/USB2pro",
+		.tuner_type = TUNER_ABSENT,
+		.decoder = CX231XX_AVDECODER,
+		.output_mode = OUT_MODE_VIP11,
+		.demod_xfer_mode = 0,
+		.ctl_pin_status_mask = 0xFFFFFFC4,
+		.agc_analog_digital_select_gpio = 0x0c,
+		.gpio_pin_status_mask = 0x4001000,
+		.norm = V4L2_STD_PAL,
+		.no_alt_vanc = 1,
+		.external_av = 1,
+		.input = {{
+			.type = CX231XX_VMUX_COMPOSITE1,
+			.vmux = CX231XX_VIN_1_1,
+			.amux = CX231XX_AMUX_LINE_IN,
+			.gpio = NULL,
+		}, {
+			.type = CX231XX_VMUX_SVIDEO,
+			.vmux = CX231XX_VIN_2_1 |
+				(CX231XX_VIN_2_2 << 8) |
+				CX25840_SVIDEO_ON,
+			.amux = CX231XX_AMUX_LINE_IN,
+			.gpio = NULL,
+		} },
+	},
 };
 const unsigned int cx231xx_bcount = ARRAY_SIZE(cx231xx_boards);
 
@@ -967,6 +993,8 @@ struct usb_device_id cx231xx_id_table[] = {
 	.driver_info = CX231XX_BOARD_EVROMEDIA_FULL_HYBRID_FULLHD},
 	{USB_DEVICE(0x15f4, 0x0135),
 	.driver_info = CX231XX_BOARD_ASTROMETA_T2HYBRID},
+	{USB_DEVICE(0x199e, 0x8002),
+	 .driver_info = CX231XX_BOARD_THE_IMAGING_SOURCE_DFG_USB2_PRO},
 	{},
 };
 
diff --git a/drivers/media/usb/cx231xx/cx231xx-core.c b/drivers/media/usb/cx231xx/cx231xx-core.c
index f372ad3917a8..4f43668df15d 100644
--- a/drivers/media/usb/cx231xx/cx231xx-core.c
+++ b/drivers/media/usb/cx231xx/cx231xx-core.c
@@ -56,7 +56,7 @@ MODULE_PARM_DESC(alt, "alternate setting to use for video endpoint");
 			 dev->name, __func__ , ##arg); } while (0)
 
 /*****************************************************************
-*             Device control list functions     				 *
+*             Device control list functions					 *
 ******************************************************************/
 
 LIST_HEAD(cx231xx_devlist);
diff --git a/drivers/media/usb/cx231xx/cx231xx-dvb.c b/drivers/media/usb/cx231xx/cx231xx-dvb.c
index 54abc1a7c8e1..fb5654062b1a 100644
--- a/drivers/media/usb/cx231xx/cx231xx-dvb.c
+++ b/drivers/media/usb/cx231xx/cx231xx-dvb.c
@@ -762,8 +762,8 @@ static int dvb_init(struct cx231xx *dev)
 		/* attach demod */
 		memset(&si2165_pdata, 0, sizeof(si2165_pdata));
 		si2165_pdata.fe = &dev->dvb->frontend;
-		si2165_pdata.chip_mode = SI2165_MODE_PLL_XTAL,
-		si2165_pdata.ref_freq_Hz = 16000000,
+		si2165_pdata.chip_mode = SI2165_MODE_PLL_XTAL;
+		si2165_pdata.ref_freq_hz = 16000000;
 
 		memset(&info, 0, sizeof(struct i2c_board_info));
 		strlcpy(info.type, "si2165", I2C_NAME_SIZE);
@@ -809,8 +809,8 @@ static int dvb_init(struct cx231xx *dev)
 		/* attach demod */
 		memset(&si2165_pdata, 0, sizeof(si2165_pdata));
 		si2165_pdata.fe = &dev->dvb->frontend;
-		si2165_pdata.chip_mode = SI2165_MODE_PLL_EXT,
-		si2165_pdata.ref_freq_Hz = 24000000,
+		si2165_pdata.chip_mode = SI2165_MODE_PLL_EXT;
+		si2165_pdata.ref_freq_hz = 24000000;
 
 		memset(&info, 0, sizeof(struct i2c_board_info));
 		strlcpy(info.type, "si2165", I2C_NAME_SIZE);
diff --git a/drivers/media/usb/cx231xx/cx231xx-i2c.c b/drivers/media/usb/cx231xx/cx231xx-i2c.c
index 23648dab7be8..6e1bef2a45bb 100644
--- a/drivers/media/usb/cx231xx/cx231xx-i2c.c
+++ b/drivers/media/usb/cx231xx/cx231xx-i2c.c
@@ -51,7 +51,7 @@ do {							\
 	if (i2c_debug >= lvl) {				\
 		printk(KERN_DEBUG "%s at %s: " fmt,	\
 		       dev->name, __func__ , ##args);	\
-      } 						\
+      }							\
 } while (0)
 
 static inline int get_real_i2c_port(struct cx231xx *dev, int bus_nr)
diff --git a/drivers/media/usb/cx231xx/cx231xx-input.c b/drivers/media/usb/cx231xx/cx231xx-input.c
index 02ebeb16055f..3e9b73a6b7c9 100644
--- a/drivers/media/usb/cx231xx/cx231xx-input.c
+++ b/drivers/media/usb/cx231xx/cx231xx-input.c
@@ -1,22 +1,12 @@
-/*
- *   cx231xx IR glue driver
- *
- *   Copyright (C) 2010 Mauro Carvalho Chehab
- *
- *   Polaris (cx231xx) has its support for IR's with a design close to MCE.
- *   however, a few designs are using an external I2C chip for IR, instead
- *   of using the one provided by the chip.
- *   This driver provides support for those extra devices
- *
- *   This program is free software; you can redistribute it and/or
- *   modify it under the terms of the GNU General Public License as
- *   published by the Free Software Foundation version 2.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *   General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// cx231xx IR glue driver
+//
+// Copyright (c) 2010 Mauro Carvalho Chehab <mchehab@kernel.org>
+//
+// Polaris (cx231xx) has its support for IR's with a design close to MCE.
+// however, a few designs are using an external I2C chip for IR, instead
+// of using the one provided by the chip.
+// This driver provides support for those extra devices
 
 #include "cx231xx.h"
 #include <linux/slab.h>
diff --git a/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.h b/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.h
index 4511dc5d199c..8f00b1d38277 100644
--- a/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.h
+++ b/drivers/media/usb/cx231xx/cx231xx-pcb-cfg.h
@@ -144,7 +144,7 @@ enum AVDEC_STATUS{
 #define SOURCE_EXTERNAL         0x8
 #define SOURCE_TS_BDA			0x10
 #define SOURCE_TS_ENCODE		0x20
-#define SOURCE_TS_EXTERNAL   	0x40
+#define SOURCE_TS_EXTERNAL	0x40
 
 /***************************************************************************
 				* interface information define *
diff --git a/drivers/media/usb/cx231xx/cx231xx-reg.h b/drivers/media/usb/cx231xx/cx231xx-reg.h
index 750c5d37d569..db5af8d51b61 100644
--- a/drivers/media/usb/cx231xx/cx231xx-reg.h
+++ b/drivers/media/usb/cx231xx/cx231xx-reg.h
@@ -1433,16 +1433,16 @@
 #define      FLD_AC97_SHUTDOWN        0x00000001
 
 /* Cx231xx redefine */
-#define      QPSK_IAGC_CTL1  		0x94c
-#define      QPSK_IAGC_CTL2  		0x950
-#define      QPSK_FEPR_FREQ  		0x954
-#define      QPSK_BTL_CTL1  		0x958
-#define      QPSK_BTL_CTL2  		0x95c
-#define      QPSK_CTL_CTL1  		0x960
-#define      QPSK_CTL_CTL2  		0x964
-#define      QPSK_MF_FAGC_CTL 		0x968
-#define      QPSK_EQ_CTL  		0x96c
-#define      QPSK_LOCK_CTL  		0x970
+#define      QPSK_IAGC_CTL1		0x94c
+#define      QPSK_IAGC_CTL2		0x950
+#define      QPSK_FEPR_FREQ		0x954
+#define      QPSK_BTL_CTL1		0x958
+#define      QPSK_BTL_CTL2		0x95c
+#define      QPSK_CTL_CTL1		0x960
+#define      QPSK_CTL_CTL2		0x964
+#define      QPSK_MF_FAGC_CTL		0x968
+#define      QPSK_EQ_CTL		0x96c
+#define      QPSK_LOCK_CTL		0x970
 
 /*****************************************************************************/
 #define      FM1_DFT_CTL              0x9a8
diff --git a/drivers/media/usb/cx231xx/cx231xx-video.c b/drivers/media/usb/cx231xx/cx231xx-video.c
index 226059fc672b..5b321b8ada3a 100644
--- a/drivers/media/usb/cx231xx/cx231xx-video.c
+++ b/drivers/media/usb/cx231xx/cx231xx-video.c
@@ -39,7 +39,7 @@
 #include <media/drv-intf/msp3400.h>
 #include <media/tuner.h>
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 
 #include "cx231xx-vbi.h"
 
@@ -1756,6 +1756,8 @@ static int cx231xx_v4l2_open(struct file *filp)
 	case VFL_TYPE_RADIO:
 		radio = 1;
 		break;
+	default:
+		return -EINVAL;
 	}
 
 	cx231xx_videodbg("open dev=%s type=%s users=%d\n",
@@ -2006,29 +2008,29 @@ cx231xx_v4l2_read(struct file *filp, char __user *buf, size_t count,
  * cx231xx_v4l2_poll()
  * will allocate buffers when called for the first time
  */
-static unsigned int cx231xx_v4l2_poll(struct file *filp, poll_table *wait)
+static __poll_t cx231xx_v4l2_poll(struct file *filp, poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct cx231xx_fh *fh = filp->private_data;
 	struct cx231xx *dev = fh->dev;
-	unsigned res = 0;
+	__poll_t res = 0;
 	int rc;
 
 	rc = check_dev(dev);
 	if (rc < 0)
-		return POLLERR;
+		return EPOLLERR;
 
 	rc = res_get(fh);
 
 	if (unlikely(rc < 0))
-		return POLLERR;
+		return EPOLLERR;
 
 	if (v4l2_event_pending(&fh->fh))
-		res |= POLLPRI;
+		res |= EPOLLPRI;
 	else
 		poll_wait(filp, &fh->fh.wait, wait);
 
-	if (!(req_events & (POLLIN | POLLRDNORM)))
+	if (!(req_events & (EPOLLIN | EPOLLRDNORM)))
 		return res;
 
 	if ((V4L2_BUF_TYPE_VIDEO_CAPTURE == fh->type) ||
@@ -2038,7 +2040,7 @@ static unsigned int cx231xx_v4l2_poll(struct file *filp, poll_table *wait)
 		mutex_unlock(&dev->lock);
 		return res;
 	}
-	return res | POLLERR;
+	return res | EPOLLERR;
 }
 
 /*
diff --git a/drivers/media/usb/cx231xx/cx231xx.h b/drivers/media/usb/cx231xx/cx231xx.h
index 72d5937a087e..65b039cf80be 100644
--- a/drivers/media/usb/cx231xx/cx231xx.h
+++ b/drivers/media/usb/cx231xx/cx231xx.h
@@ -80,6 +80,7 @@
 #define CX231XX_BOARD_TERRATEC_GRABBY 22
 #define CX231XX_BOARD_EVROMEDIA_FULL_HYBRID_FULLHD 23
 #define CX231XX_BOARD_ASTROMETA_T2HYBRID 24
+#define CX231XX_BOARD_THE_IMAGING_SOURCE_DFG_USB2_PRO 25
 
 /* Limits minimum and default number of buffers */
 #define CX231XX_MIN_BUF                 4
diff --git a/drivers/media/usb/dvb-usb-v2/Makefile b/drivers/media/usb/dvb-usb-v2/Makefile
index bed44601f324..58c0140e19de 100644
--- a/drivers/media/usb/dvb-usb-v2/Makefile
+++ b/drivers/media/usb/dvb-usb-v2/Makefile
@@ -44,7 +44,6 @@ obj-$(CONFIG_DVB_USB_DVBSKY) += dvb-usb-dvbsky.o
 dvb-usb-zd1301-objs := zd1301.o
 obj-$(CONFIG_DVB_USB_ZD1301) += zd1301.o
 
-ccflags-y += -I$(srctree)/drivers/media/dvb-core
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends
 ccflags-y += -I$(srctree)/drivers/media/tuners
 ccflags-y += -I$(srctree)/drivers/media/common
diff --git a/drivers/media/usb/dvb-usb-v2/anysee.h b/drivers/media/usb/dvb-usb-v2/anysee.h
index 393e2fce2aed..2312c55619ca 100644
--- a/drivers/media/usb/dvb-usb-v2/anysee.h
+++ b/drivers/media/usb/dvb-usb-v2/anysee.h
@@ -32,7 +32,7 @@
 
 #define DVB_USB_LOG_PREFIX "anysee"
 #include "dvb_usb.h"
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 enum cmd {
 	CMD_I2C_READ            = 0x33,
diff --git a/drivers/media/usb/dvb-usb-v2/az6007.c b/drivers/media/usb/dvb-usb-v2/az6007.c
index 1414d59e85ba..746926364535 100644
--- a/drivers/media/usb/dvb-usb-v2/az6007.c
+++ b/drivers/media/usb/dvb-usb-v2/az6007.c
@@ -23,7 +23,7 @@
 
 #include "drxk.h"
 #include "mt2063.h"
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 #include "dvb_usb.h"
 #include "cypress_firmware.h"
 
diff --git a/drivers/media/usb/dvb-usb-v2/dvb_usb.h b/drivers/media/usb/dvb-usb-v2/dvb_usb.h
index 0005bdb2207d..d2e80537b2f7 100644
--- a/drivers/media/usb/dvb-usb-v2/dvb_usb.h
+++ b/drivers/media/usb/dvb-usb-v2/dvb_usb.h
@@ -27,11 +27,11 @@
 #include <media/rc-core.h>
 #include <media/media-device.h>
 
-#include "dvb_frontend.h"
-#include "dvb_demux.h"
-#include "dvb_net.h"
-#include "dmxdev.h"
-#include "dvb-usb-ids.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
+#include <media/dmxdev.h>
+#include <media/dvb-usb-ids.h>
 
 /*
  * device file: /dev/dvb/adapter[0-1]/frontend[0-2]
diff --git a/drivers/media/usb/dvb-usb-v2/dvbsky.c b/drivers/media/usb/dvb-usb-v2/dvbsky.c
index 131b6c08e199..43eb82884555 100644
--- a/drivers/media/usb/dvb-usb-v2/dvbsky.c
+++ b/drivers/media/usb/dvb-usb-v2/dvbsky.c
@@ -309,7 +309,7 @@ static int dvbsky_s960_attach(struct dvb_usb_adapter *adap)
 	struct dvb_usb_device *d = adap_to_d(adap);
 	int ret = 0;
 	/* demod I2C adapter */
-	struct i2c_adapter *i2c_adapter;
+	struct i2c_adapter *i2c_adapter = NULL;
 	struct i2c_client *client;
 	struct i2c_board_info info;
 	struct ts2020_config ts2020_config = {};
@@ -431,7 +431,7 @@ static int dvbsky_s960c_attach(struct dvb_usb_adapter *adap)
 	struct dvb_usb_device *d = adap_to_d(adap);
 	int ret = 0;
 	/* demod I2C adapter */
-	struct i2c_adapter *i2c_adapter;
+	struct i2c_adapter *i2c_adapter = NULL;
 	struct i2c_client *client_tuner, *client_ci;
 	struct i2c_board_info info;
 	struct sp2_config sp2_config;
@@ -661,6 +661,65 @@ static int dvbsky_t330_attach(struct dvb_usb_adapter *adap)
 	return ret;
 }
 
+static int dvbsky_mygica_t230c_attach(struct dvb_usb_adapter *adap)
+{
+	struct dvbsky_state *state = adap_to_priv(adap);
+	struct dvb_usb_device *d = adap_to_d(adap);
+	struct i2c_adapter *i2c_adapter;
+	struct i2c_client *client_demod, *client_tuner;
+	struct i2c_board_info info;
+	struct si2168_config si2168_config;
+	struct si2157_config si2157_config;
+
+	/* attach demod */
+	memset(&si2168_config, 0, sizeof(si2168_config));
+	si2168_config.i2c_adapter = &i2c_adapter;
+	si2168_config.fe = &adap->fe[0];
+	si2168_config.ts_mode = SI2168_TS_PARALLEL;
+	si2168_config.ts_clock_inv = 1;
+	memset(&info, 0, sizeof(struct i2c_board_info));
+	strlcpy(info.type, "si2168", sizeof(info.type));
+	info.addr = 0x64;
+	info.platform_data = &si2168_config;
+
+	request_module("si2168");
+	client_demod = i2c_new_device(&d->i2c_adap, &info);
+	if (!client_demod || !client_demod->dev.driver)
+		goto fail_demod_device;
+	if (!try_module_get(client_demod->dev.driver->owner))
+		goto fail_demod_module;
+
+	/* attach tuner */
+	memset(&si2157_config, 0, sizeof(si2157_config));
+	si2157_config.fe = adap->fe[0];
+	si2157_config.if_port = 0;
+	memset(&info, 0, sizeof(struct i2c_board_info));
+	strlcpy(info.type, "si2141", sizeof(info.type));
+	info.addr = 0x60;
+	info.platform_data = &si2157_config;
+
+	request_module("si2157");
+	client_tuner = i2c_new_device(i2c_adapter, &info);
+	if (!client_tuner || !client_tuner->dev.driver)
+		goto fail_tuner_device;
+	if (!try_module_get(client_tuner->dev.driver->owner))
+		goto fail_tuner_module;
+
+	state->i2c_client_demod = client_demod;
+	state->i2c_client_tuner = client_tuner;
+	return 0;
+
+fail_tuner_module:
+	i2c_unregister_device(client_tuner);
+fail_tuner_device:
+	module_put(client_demod->dev.driver->owner);
+fail_demod_module:
+	i2c_unregister_device(client_demod);
+fail_demod_device:
+	return -ENODEV;
+}
+
+
 static int dvbsky_identify_state(struct dvb_usb_device *d, const char **name)
 {
 	dvbsky_gpio_ctrl(d, 0x04, 1);
@@ -826,6 +885,32 @@ static struct dvb_usb_device_properties dvbsky_t330_props = {
 	}
 };
 
+static struct dvb_usb_device_properties mygica_t230c_props = {
+	.driver_name = KBUILD_MODNAME,
+	.owner = THIS_MODULE,
+	.adapter_nr = adapter_nr,
+	.size_of_priv = sizeof(struct dvbsky_state),
+
+	.generic_bulk_ctrl_endpoint = 0x01,
+	.generic_bulk_ctrl_endpoint_response = 0x81,
+	.generic_bulk_ctrl_delay = DVBSKY_MSG_DELAY,
+
+	.i2c_algo         = &dvbsky_i2c_algo,
+	.frontend_attach  = dvbsky_mygica_t230c_attach,
+	.init             = dvbsky_init,
+	.get_rc_config    = dvbsky_get_rc_config,
+	.streaming_ctrl   = dvbsky_streaming_ctrl,
+	.identify_state	  = dvbsky_identify_state,
+	.exit             = dvbsky_exit,
+
+	.num_adapters = 1,
+	.adapter = {
+		{
+			.stream = DVB_USB_STREAM_BULK(0x82, 8, 4096),
+		}
+	}
+};
+
 static const struct usb_device_id dvbsky_id_table[] = {
 	{ DVB_USB_DEVICE(0x0572, 0x6831,
 		&dvbsky_s960_props, "DVBSky S960/S860", RC_MAP_DVBSKY) },
@@ -858,6 +943,9 @@ static const struct usb_device_id dvbsky_id_table[] = {
 	{ DVB_USB_DEVICE(USB_VID_TERRATEC, USB_PID_TERRATEC_CINERGY_S2_R4,
 		&dvbsky_s960_props, "Terratec Cinergy S2 Rev.4",
 		RC_MAP_DVBSKY) },
+	{ DVB_USB_DEVICE(USB_VID_CONEXANT, USB_PID_MYGICA_T230C,
+		&mygica_t230c_props, "MyGica Mini DVB-T2 USB Stick T230C",
+		RC_MAP_TOTAL_MEDIA_IN_HAND_02) },
 	{ }
 };
 MODULE_DEVICE_TABLE(usb, dvbsky_id_table);
diff --git a/drivers/media/usb/dvb-usb-v2/lmedm04.c b/drivers/media/usb/dvb-usb-v2/lmedm04.c
index 5e320fa4a795..be26c029546b 100644
--- a/drivers/media/usb/dvb-usb-v2/lmedm04.c
+++ b/drivers/media/usb/dvb-usb-v2/lmedm04.c
@@ -494,18 +494,23 @@ static int lme2510_pid_filter(struct dvb_usb_adapter *adap, int index, u16 pid,
 
 static int lme2510_return_status(struct dvb_usb_device *d)
 {
-	int ret = 0;
+	int ret;
 	u8 *data;
 
-	data = kzalloc(10, GFP_KERNEL);
+	data = kzalloc(6, GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
-	ret |= usb_control_msg(d->udev, usb_rcvctrlpipe(d->udev, 0),
-			0x06, 0x80, 0x0302, 0x00, data, 0x0006, 200);
-	info("Firmware Status: %x (%x)", ret , data[2]);
+	ret = usb_control_msg(d->udev, usb_rcvctrlpipe(d->udev, 0),
+			      0x06, 0x80, 0x0302, 0x00,
+			      data, 0x6, 200);
+	if (ret != 6)
+		ret = -EINVAL;
+	else
+		ret = data[2];
+
+	info("Firmware Status: %6ph", data);
 
-	ret = (ret < 0) ? -ENODEV : data[2];
 	kfree(data);
 	return ret;
 }
@@ -1071,8 +1076,6 @@ static int dm04_lme2510_frontend_attach(struct dvb_usb_adapter *adap)
 
 		if (adap->fe[0]) {
 			info("FE Found M88RS2000");
-			dvb_attach(ts2020_attach, adap->fe[0], &ts2020_config,
-					&d->i2c_adap);
 			st->i2c_tuner_gate_w = 5;
 			st->i2c_tuner_gate_r = 5;
 			st->i2c_tuner_addr = 0x60;
@@ -1138,17 +1141,18 @@ static int dm04_lme2510_tuner(struct dvb_usb_adapter *adap)
 			ret = st->tuner_config;
 		break;
 	case TUNER_RS2000:
-		ret = st->tuner_config;
+		if (dvb_attach(ts2020_attach, adap->fe[0],
+			       &ts2020_config, &d->i2c_adap))
+			ret = st->tuner_config;
 		break;
 	default:
 		break;
 	}
 
-	if (ret)
+	if (ret) {
 		info("TUN Found %s tuner", tun_msg[ret]);
-	else {
-		info("TUN No tuner found --- resetting device");
-		lme_coldreset(d);
+	} else {
+		info("TUN No tuner found");
 		return -ENODEV;
 	}
 
@@ -1189,6 +1193,7 @@ static int lme2510_get_adapter_count(struct dvb_usb_device *d)
 static int lme2510_identify_state(struct dvb_usb_device *d, const char **name)
 {
 	struct lme2510_state *st = d->priv;
+	int status;
 
 	usb_reset_configuration(d->udev);
 
@@ -1197,12 +1202,16 @@ static int lme2510_identify_state(struct dvb_usb_device *d, const char **name)
 
 	st->dvb_usb_lme2510_firmware = dvb_usb_lme2510_firmware;
 
-	if (lme2510_return_status(d) == 0x44) {
+	status = lme2510_return_status(d);
+	if (status == 0x44) {
 		*name = lme_firmware_switch(d, 0);
 		return COLD;
 	}
 
-	return 0;
+	if (status != 0x47)
+		return -EINVAL;
+
+	return WARM;
 }
 
 static int lme2510_get_stream_config(struct dvb_frontend *fe, u8 *ts_type,
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.c b/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.c
index f0ed37da73d4..221cf46b4140 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.c
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.c
@@ -477,10 +477,15 @@ static int mxl111sf_demod_read_signal_strength(struct dvb_frontend *fe,
 {
 	struct mxl111sf_demod_state *state = fe->demodulator_priv;
 	enum fe_modulation modulation;
+	int ret;
 	u16 snr;
 
-	mxl111sf_demod_calc_snr(state, &snr);
-	mxl1x1sf_demod_get_tps_modulation(state, &modulation);
+	ret = mxl111sf_demod_calc_snr(state, &snr);
+	if (ret < 0)
+		return ret;
+	ret = mxl1x1sf_demod_get_tps_modulation(state, &modulation);
+	if (ret < 0)
+		return ret;
 
 	switch (modulation) {
 	case QPSK:
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h b/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h
index 9cb4972ce7a3..95888b8885c4 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf-demod.h
@@ -17,7 +17,7 @@
 #ifndef __MXL111SF_DEMOD_H__
 #define __MXL111SF_DEMOD_H__
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mxl111sf.h"
 
 struct mxl111sf_demod_config {
diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h b/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h
index 11ea07a73271..87c1b1642115 100644
--- a/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h
+++ b/drivers/media/usb/dvb-usb-v2/mxl111sf-tuner.h
@@ -17,7 +17,7 @@
 #ifndef __MXL111SF_TUNER_H__
 #define __MXL111SF_TUNER_H__
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "mxl111sf.h"
 
 enum mxl_if_freq {
diff --git a/drivers/media/usb/dvb-usb/Kconfig b/drivers/media/usb/dvb-usb/Kconfig
index 959fa09dfd92..2651ae277347 100644
--- a/drivers/media/usb/dvb-usb/Kconfig
+++ b/drivers/media/usb/dvb-usb/Kconfig
@@ -86,6 +86,7 @@ config DVB_USB_DIB0700
 	select DVB_USB_DIB3000MC if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_S5H1411 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_LGDT3305 if MEDIA_SUBDRV_AUTOSELECT
+	select DVB_MN88472 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_TUNER_DIB0070 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_TUNER_DIB0090 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_MT2060 if MEDIA_SUBDRV_AUTOSELECT
@@ -94,6 +95,7 @@ config DVB_USB_DIB0700
 	select MEDIA_TUNER_XC5000 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_XC4000 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_MXL5007T if MEDIA_SUBDRV_AUTOSELECT
+	select MEDIA_TUNER_TDA18250 if MEDIA_SUBDRV_AUTOSELECT
 	help
 	  Support for USB2.0/1.1 DVB receivers based on the DiB0700 USB bridge. The
 	  USB bridge is also present in devices having the DiB7700 DVB-T-USB
diff --git a/drivers/media/usb/dvb-usb/Makefile b/drivers/media/usb/dvb-usb/Makefile
index 16de1e4f36a4..9ad2618408ef 100644
--- a/drivers/media/usb/dvb-usb/Makefile
+++ b/drivers/media/usb/dvb-usb/Makefile
@@ -80,7 +80,6 @@ obj-$(CONFIG_DVB_USB_AZ6027) += dvb-usb-az6027.o
 dvb-usb-technisat-usb2-objs := technisat-usb2.o
 obj-$(CONFIG_DVB_USB_TECHNISAT_USB2) += dvb-usb-technisat-usb2.o
 
-ccflags-y += -I$(srctree)/drivers/media/dvb-core
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends/
 # due to tuner-xc3028
 ccflags-y += -I$(srctree)/drivers/media/tuners
diff --git a/drivers/media/usb/dvb-usb/az6027.c b/drivers/media/usb/dvb-usb/az6027.c
index 2e711362847e..f0d10ac03a37 100644
--- a/drivers/media/usb/dvb-usb/az6027.c
+++ b/drivers/media/usb/dvb-usb/az6027.c
@@ -17,7 +17,7 @@
 
 #include "stb6100.h"
 #include "stb6100_cfg.h"
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 int dvb_usb_az6027_debug;
 module_param_named(debug, dvb_usb_az6027_debug, int, 0644);
@@ -36,70 +36,70 @@ static const struct stb0899_s1_reg az6027_stb0899_s1_init_1[] = {
 	/* 0x0000000b, SYSREG */
 	{ STB0899_DEV_ID		, 0x30 },
 	{ STB0899_DISCNTRL1		, 0x32 },
-	{ STB0899_DISCNTRL2     	, 0x80 },
-	{ STB0899_DISRX_ST0     	, 0x04 },
-	{ STB0899_DISRX_ST1     	, 0x00 },
-	{ STB0899_DISPARITY     	, 0x00 },
+	{ STB0899_DISCNTRL2		, 0x80 },
+	{ STB0899_DISRX_ST0		, 0x04 },
+	{ STB0899_DISRX_ST1		, 0x00 },
+	{ STB0899_DISPARITY		, 0x00 },
 	{ STB0899_DISSTATUS		, 0x20 },
-	{ STB0899_DISF22        	, 0x99 },
-	{ STB0899_DISF22RX      	, 0xa8 },
+	{ STB0899_DISF22		, 0x99 },
+	{ STB0899_DISF22RX		, 0xa8 },
 	/* SYSREG ? */
-	{ STB0899_ACRPRESC      	, 0x11 },
-	{ STB0899_ACRDIV1       	, 0x0a },
-	{ STB0899_ACRDIV2       	, 0x05 },
-	{ STB0899_DACR1         	, 0x00 },
-	{ STB0899_DACR2         	, 0x00 },
-	{ STB0899_OUTCFG        	, 0x00 },
-	{ STB0899_MODECFG       	, 0x00 },
+	{ STB0899_ACRPRESC		, 0x11 },
+	{ STB0899_ACRDIV1		, 0x0a },
+	{ STB0899_ACRDIV2		, 0x05 },
+	{ STB0899_DACR1			, 0x00 },
+	{ STB0899_DACR2			, 0x00 },
+	{ STB0899_OUTCFG		, 0x00 },
+	{ STB0899_MODECFG		, 0x00 },
 	{ STB0899_IRQSTATUS_3		, 0xfe },
 	{ STB0899_IRQSTATUS_2		, 0x03 },
 	{ STB0899_IRQSTATUS_1		, 0x7c },
 	{ STB0899_IRQSTATUS_0		, 0xf4 },
-	{ STB0899_IRQMSK_3      	, 0xf3 },
-	{ STB0899_IRQMSK_2      	, 0xfc },
-	{ STB0899_IRQMSK_1      	, 0xff },
+	{ STB0899_IRQMSK_3		, 0xf3 },
+	{ STB0899_IRQMSK_2		, 0xfc },
+	{ STB0899_IRQMSK_1		, 0xff },
 	{ STB0899_IRQMSK_0		, 0xff },
 	{ STB0899_IRQCFG		, 0x00 },
-	{ STB0899_I2CCFG        	, 0x88 },
-	{ STB0899_I2CRPT        	, 0x58 },
+	{ STB0899_I2CCFG		, 0x88 },
+	{ STB0899_I2CRPT		, 0x58 },
 	{ STB0899_IOPVALUE5		, 0x00 },
 	{ STB0899_IOPVALUE4		, 0x33 },
 	{ STB0899_IOPVALUE3		, 0x6d },
 	{ STB0899_IOPVALUE2		, 0x90 },
 	{ STB0899_IOPVALUE1		, 0x60 },
 	{ STB0899_IOPVALUE0		, 0x00 },
-	{ STB0899_GPIO00CFG     	, 0x82 },
-	{ STB0899_GPIO01CFG     	, 0x82 },
-	{ STB0899_GPIO02CFG     	, 0x82 },
-	{ STB0899_GPIO03CFG     	, 0x82 },
-	{ STB0899_GPIO04CFG     	, 0x82 },
-	{ STB0899_GPIO05CFG     	, 0x82 },
-	{ STB0899_GPIO06CFG     	, 0x82 },
-	{ STB0899_GPIO07CFG     	, 0x82 },
-	{ STB0899_GPIO08CFG     	, 0x82 },
-	{ STB0899_GPIO09CFG     	, 0x82 },
-	{ STB0899_GPIO10CFG     	, 0x82 },
-	{ STB0899_GPIO11CFG     	, 0x82 },
-	{ STB0899_GPIO12CFG     	, 0x82 },
-	{ STB0899_GPIO13CFG     	, 0x82 },
-	{ STB0899_GPIO14CFG     	, 0x82 },
-	{ STB0899_GPIO15CFG     	, 0x82 },
-	{ STB0899_GPIO16CFG     	, 0x82 },
-	{ STB0899_GPIO17CFG     	, 0x82 },
-	{ STB0899_GPIO18CFG     	, 0x82 },
-	{ STB0899_GPIO19CFG     	, 0x82 },
-	{ STB0899_GPIO20CFG     	, 0x82 },
-	{ STB0899_SDATCFG       	, 0xb8 },
-	{ STB0899_SCLTCFG       	, 0xba },
-	{ STB0899_AGCRFCFG      	, 0x1c }, /* 0x11 */
-	{ STB0899_GPIO22        	, 0x82 }, /* AGCBB2CFG */
-	{ STB0899_GPIO21        	, 0x91 }, /* AGCBB1CFG */
-	{ STB0899_DIRCLKCFG     	, 0x82 },
-	{ STB0899_CLKOUT27CFG   	, 0x7e },
-	{ STB0899_STDBYCFG      	, 0x82 },
-	{ STB0899_CS0CFG        	, 0x82 },
-	{ STB0899_CS1CFG        	, 0x82 },
-	{ STB0899_DISEQCOCFG    	, 0x20 },
+	{ STB0899_GPIO00CFG		, 0x82 },
+	{ STB0899_GPIO01CFG		, 0x82 },
+	{ STB0899_GPIO02CFG		, 0x82 },
+	{ STB0899_GPIO03CFG		, 0x82 },
+	{ STB0899_GPIO04CFG		, 0x82 },
+	{ STB0899_GPIO05CFG		, 0x82 },
+	{ STB0899_GPIO06CFG		, 0x82 },
+	{ STB0899_GPIO07CFG		, 0x82 },
+	{ STB0899_GPIO08CFG		, 0x82 },
+	{ STB0899_GPIO09CFG		, 0x82 },
+	{ STB0899_GPIO10CFG		, 0x82 },
+	{ STB0899_GPIO11CFG		, 0x82 },
+	{ STB0899_GPIO12CFG		, 0x82 },
+	{ STB0899_GPIO13CFG		, 0x82 },
+	{ STB0899_GPIO14CFG		, 0x82 },
+	{ STB0899_GPIO15CFG		, 0x82 },
+	{ STB0899_GPIO16CFG		, 0x82 },
+	{ STB0899_GPIO17CFG		, 0x82 },
+	{ STB0899_GPIO18CFG		, 0x82 },
+	{ STB0899_GPIO19CFG		, 0x82 },
+	{ STB0899_GPIO20CFG		, 0x82 },
+	{ STB0899_SDATCFG		, 0xb8 },
+	{ STB0899_SCLTCFG		, 0xba },
+	{ STB0899_AGCRFCFG		, 0x1c }, /* 0x11 */
+	{ STB0899_GPIO22		, 0x82 }, /* AGCBB2CFG */
+	{ STB0899_GPIO21		, 0x91 }, /* AGCBB1CFG */
+	{ STB0899_DIRCLKCFG		, 0x82 },
+	{ STB0899_CLKOUT27CFG		, 0x7e },
+	{ STB0899_STDBYCFG		, 0x82 },
+	{ STB0899_CS0CFG		, 0x82 },
+	{ STB0899_CS1CFG		, 0x82 },
+	{ STB0899_DISEQCOCFG		, 0x20 },
 	{ STB0899_GPIO32CFG		, 0x82 },
 	{ STB0899_GPIO33CFG		, 0x82 },
 	{ STB0899_GPIO34CFG		, 0x82 },
@@ -108,35 +108,35 @@ static const struct stb0899_s1_reg az6027_stb0899_s1_init_1[] = {
 	{ STB0899_GPIO37CFG		, 0x82 },
 	{ STB0899_GPIO38CFG		, 0x82 },
 	{ STB0899_GPIO39CFG		, 0x82 },
-	{ STB0899_NCOARSE       	, 0x17 }, /* 0x15 = 27 Mhz Clock, F/3 = 198MHz, F/6 = 99MHz */
-	{ STB0899_SYNTCTRL      	, 0x02 }, /* 0x00 = CLK from CLKI, 0x02 = CLK from XTALI */
-	{ STB0899_FILTCTRL      	, 0x00 },
-	{ STB0899_SYSCTRL       	, 0x01 },
-	{ STB0899_STOPCLK1      	, 0x20 },
-	{ STB0899_STOPCLK2      	, 0x00 },
+	{ STB0899_NCOARSE		, 0x17 }, /* 0x15 = 27 Mhz Clock, F/3 = 198MHz, F/6 = 99MHz */
+	{ STB0899_SYNTCTRL		, 0x02 }, /* 0x00 = CLK from CLKI, 0x02 = CLK from XTALI */
+	{ STB0899_FILTCTRL		, 0x00 },
+	{ STB0899_SYSCTRL		, 0x01 },
+	{ STB0899_STOPCLK1		, 0x20 },
+	{ STB0899_STOPCLK2		, 0x00 },
 	{ STB0899_INTBUFSTATUS		, 0x00 },
-	{ STB0899_INTBUFCTRL    	, 0x0a },
+	{ STB0899_INTBUFCTRL		, 0x0a },
 	{ 0xffff			, 0xff },
 };
 
 static const struct stb0899_s1_reg az6027_stb0899_s1_init_3[] = {
-	{ STB0899_DEMOD         	, 0x00 },
-	{ STB0899_RCOMPC        	, 0xc9 },
-	{ STB0899_AGC1CN        	, 0x01 },
-	{ STB0899_AGC1REF       	, 0x10 },
+	{ STB0899_DEMOD			, 0x00 },
+	{ STB0899_RCOMPC		, 0xc9 },
+	{ STB0899_AGC1CN		, 0x01 },
+	{ STB0899_AGC1REF		, 0x10 },
 	{ STB0899_RTC			, 0x23 },
-	{ STB0899_TMGCFG        	, 0x4e },
-	{ STB0899_AGC2REF       	, 0x34 },
-	{ STB0899_TLSR          	, 0x84 },
-	{ STB0899_CFD           	, 0xf7 },
+	{ STB0899_TMGCFG		, 0x4e },
+	{ STB0899_AGC2REF		, 0x34 },
+	{ STB0899_TLSR			, 0x84 },
+	{ STB0899_CFD			, 0xf7 },
 	{ STB0899_ACLC			, 0x87 },
-	{ STB0899_BCLC          	, 0x94 },
-	{ STB0899_EQON          	, 0x41 },
-	{ STB0899_LDT           	, 0xf1 },
-	{ STB0899_LDT2          	, 0xe3 },
-	{ STB0899_EQUALREF      	, 0xb4 },
-	{ STB0899_TMGRAMP       	, 0x10 },
-	{ STB0899_TMGTHD        	, 0x30 },
+	{ STB0899_BCLC			, 0x94 },
+	{ STB0899_EQON			, 0x41 },
+	{ STB0899_LDT			, 0xf1 },
+	{ STB0899_LDT2			, 0xe3 },
+	{ STB0899_EQUALREF		, 0xb4 },
+	{ STB0899_TMGRAMP		, 0x10 },
+	{ STB0899_TMGTHD		, 0x30 },
 	{ STB0899_IDCCOMP		, 0xfd },
 	{ STB0899_QDCCOMP		, 0xff },
 	{ STB0899_POWERI		, 0x0c },
@@ -155,12 +155,12 @@ static const struct stb0899_s1_reg az6027_stb0899_s1_init_3[] = {
 	{ STB0899_NIRL			, 0x80 },
 	{ STB0899_ISYMB			, 0x1d },
 	{ STB0899_QSYMB			, 0xa6 },
-	{ STB0899_SFRH          	, 0x2f },
-	{ STB0899_SFRM          	, 0x68 },
-	{ STB0899_SFRL          	, 0x40 },
-	{ STB0899_SFRUPH        	, 0x2f },
-	{ STB0899_SFRUPM        	, 0x68 },
-	{ STB0899_SFRUPL        	, 0x40 },
+	{ STB0899_SFRH			, 0x2f },
+	{ STB0899_SFRM			, 0x68 },
+	{ STB0899_SFRL			, 0x40 },
+	{ STB0899_SFRUPH		, 0x2f },
+	{ STB0899_SFRUPM		, 0x68 },
+	{ STB0899_SFRUPL		, 0x40 },
 	{ STB0899_EQUAI1		, 0x02 },
 	{ STB0899_EQUAQ1		, 0xff },
 	{ STB0899_EQUAI2		, 0x04 },
@@ -172,7 +172,7 @@ static const struct stb0899_s1_reg az6027_stb0899_s1_init_3[] = {
 	{ STB0899_EQUAI5		, 0x08 },
 	{ STB0899_EQUAQ5		, 0xf5 },
 	{ STB0899_DSTATUS2		, 0x00 },
-	{ STB0899_VSTATUS       	, 0x00 },
+	{ STB0899_VSTATUS		, 0x00 },
 	{ STB0899_VERROR		, 0x86 },
 	{ STB0899_IQSWAP		, 0x2a },
 	{ STB0899_ECNT1M		, 0x00 },
@@ -181,26 +181,26 @@ static const struct stb0899_s1_reg az6027_stb0899_s1_init_3[] = {
 	{ STB0899_ECNT2L		, 0x00 },
 	{ STB0899_ECNT3M		, 0x0a },
 	{ STB0899_ECNT3L		, 0xad },
-	{ STB0899_FECAUTO1      	, 0x06 },
+	{ STB0899_FECAUTO1		, 0x06 },
 	{ STB0899_FECM			, 0x01 },
-	{ STB0899_VTH12         	, 0xb0 },
-	{ STB0899_VTH23         	, 0x7a },
+	{ STB0899_VTH12			, 0xb0 },
+	{ STB0899_VTH23			, 0x7a },
 	{ STB0899_VTH34			, 0x58 },
-	{ STB0899_VTH56         	, 0x38 },
-	{ STB0899_VTH67         	, 0x34 },
-	{ STB0899_VTH78         	, 0x24 },
-	{ STB0899_PRVIT         	, 0xff },
-	{ STB0899_VITSYNC       	, 0x19 },
-	{ STB0899_RSULC         	, 0xb1 }, /* DVB = 0xb1, DSS = 0xa1 */
-	{ STB0899_TSULC         	, 0x42 },
-	{ STB0899_RSLLC         	, 0x41 },
+	{ STB0899_VTH56			, 0x38 },
+	{ STB0899_VTH67			, 0x34 },
+	{ STB0899_VTH78			, 0x24 },
+	{ STB0899_PRVIT			, 0xff },
+	{ STB0899_VITSYNC		, 0x19 },
+	{ STB0899_RSULC			, 0xb1 }, /* DVB = 0xb1, DSS = 0xa1 */
+	{ STB0899_TSULC			, 0x42 },
+	{ STB0899_RSLLC			, 0x41 },
 	{ STB0899_TSLPL			, 0x12 },
-	{ STB0899_TSCFGH        	, 0x0c },
-	{ STB0899_TSCFGM        	, 0x00 },
-	{ STB0899_TSCFGL        	, 0x00 },
+	{ STB0899_TSCFGH		, 0x0c },
+	{ STB0899_TSCFGM		, 0x00 },
+	{ STB0899_TSCFGL		, 0x00 },
 	{ STB0899_TSOUT			, 0x69 }, /* 0x0d for CAM */
-	{ STB0899_RSSYNCDEL     	, 0x00 },
-	{ STB0899_TSINHDELH     	, 0x02 },
+	{ STB0899_RSSYNCDEL		, 0x00 },
+	{ STB0899_TSINHDELH		, 0x02 },
 	{ STB0899_TSINHDELM		, 0x00 },
 	{ STB0899_TSINHDELL		, 0x00 },
 	{ STB0899_TSLLSTKM		, 0x1b },
@@ -211,18 +211,18 @@ static const struct stb0899_s1_reg az6027_stb0899_s1_init_3[] = {
 	{ STB0899_PCKLENLL		, 0xcc },
 	{ STB0899_RSPCKLEN		, 0xbd },
 	{ STB0899_TSSTATUS		, 0x90 },
-	{ STB0899_ERRCTRL1      	, 0xb6 },
-	{ STB0899_ERRCTRL2      	, 0x95 },
-	{ STB0899_ERRCTRL3      	, 0x8d },
+	{ STB0899_ERRCTRL1		, 0xb6 },
+	{ STB0899_ERRCTRL2		, 0x95 },
+	{ STB0899_ERRCTRL3		, 0x8d },
 	{ STB0899_DMONMSK1		, 0x27 },
 	{ STB0899_DMONMSK0		, 0x03 },
-	{ STB0899_DEMAPVIT      	, 0x5c },
+	{ STB0899_DEMAPVIT		, 0x5c },
 	{ STB0899_PLPARM		, 0x19 },
-	{ STB0899_PDELCTRL      	, 0x48 },
-	{ STB0899_PDELCTRL2     	, 0x00 },
-	{ STB0899_BBHCTRL1      	, 0x00 },
-	{ STB0899_BBHCTRL2      	, 0x00 },
-	{ STB0899_HYSTTHRESH    	, 0x77 },
+	{ STB0899_PDELCTRL		, 0x48 },
+	{ STB0899_PDELCTRL2		, 0x00 },
+	{ STB0899_BBHCTRL1		, 0x00 },
+	{ STB0899_BBHCTRL2		, 0x00 },
+	{ STB0899_HYSTTHRESH		, 0x77 },
 	{ STB0899_MATCSTM		, 0x00 },
 	{ STB0899_MATCSTL		, 0x00 },
 	{ STB0899_UPLCSTM		, 0x00 },
@@ -261,7 +261,7 @@ static struct stb0899_config az6027_stb0899_config = {
 	.init_s2_fec		= stb0899_s2_init_4,
 	.init_tst		= stb0899_s1_init_5,
 
-	.demod_address 		= 0xd0, /* 0x68, 0xd0 >> 1 */
+	.demod_address		= 0xd0, /* 0x68, 0xd0 >> 1 */
 
 	.xtal_freq		= 27000000,
 	.inversion		= IQ_SWAP_ON,
@@ -1181,9 +1181,9 @@ static struct dvb_usb_device_properties az6027_properties = {
 /* usb specific object needed to register this driver with the usb subsystem */
 static struct usb_driver az6027_usb_driver = {
 	.name		= "dvb_usb_az6027",
-	.probe 		= az6027_usb_probe,
-	.disconnect 	= az6027_usb_disconnect,
-	.id_table 	= az6027_usb_table,
+	.probe		= az6027_usb_probe,
+	.disconnect	= az6027_usb_disconnect,
+	.id_table	= az6027_usb_table,
 };
 
 module_usb_driver(az6027_usb_driver);
diff --git a/drivers/media/usb/dvb-usb/cxusb.c b/drivers/media/usb/dvb-usb/cxusb.c
index 37dea0adc695..2abd15c6df81 100644
--- a/drivers/media/usb/dvb-usb/cxusb.c
+++ b/drivers/media/usb/dvb-usb/cxusb.c
@@ -26,6 +26,7 @@
 #include <media/tuner.h>
 #include <linux/vmalloc.h>
 #include <linux/slab.h>
+#include <linux/kernel.h>
 
 #include "cxusb.h"
 
@@ -56,7 +57,7 @@ DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);
 #define deb_i2c(args...)    dprintk(dvb_usb_cxusb_debug, 0x02, args)
 
 static int cxusb_ctrl_msg(struct dvb_usb_device *d,
-			  u8 cmd, u8 *wbuf, int wlen, u8 *rbuf, int rlen)
+			  u8 cmd, const u8 *wbuf, int wlen, u8 *rbuf, int rlen)
 {
 	struct cxusb_state *st = d->priv;
 	int ret;
@@ -290,7 +291,8 @@ static int cxusb_aver_power_ctrl(struct dvb_usb_device *d, int onoff)
 		/* FIXME: We don't know why, but we need to configure the
 		 * lgdt3303 with the register settings below on resume */
 		int i;
-		u8 buf, bufs[] = {
+		u8 buf;
+		static const u8 bufs[] = {
 			0x0e, 0x2, 0x00, 0x7f,
 			0x0e, 0x2, 0x02, 0xfe,
 			0x0e, 0x2, 0x02, 0x01,
@@ -303,7 +305,7 @@ static int cxusb_aver_power_ctrl(struct dvb_usb_device *d, int onoff)
 			0x0e, 0x2, 0x47, 0x88,
 		};
 		msleep(20);
-		for (i = 0; i < sizeof(bufs)/sizeof(u8); i += 4/sizeof(u8)) {
+		for (i = 0; i < ARRAY_SIZE(bufs); i += 4 / sizeof(u8)) {
 			ret = cxusb_ctrl_msg(d, CMD_I2C_WRITE,
 					     bufs+i, 4, &buf, 1);
 			if (ret)
@@ -677,6 +679,8 @@ static int dvico_bluebird_xc2028_callback(void *ptr, int component,
 	case XC2028_RESET_CLK:
 		deb_info("%s: XC2028_RESET_CLK %d\n", __func__, arg);
 		break;
+	case XC2028_I2C_FLUSH:
+		break;
 	default:
 		deb_info("%s: unknown command %d, arg %d\n", __func__,
 			 command, arg);
diff --git a/drivers/media/usb/dvb-usb/dib0700.h b/drivers/media/usb/dvb-usb/dib0700.h
index f89ab3b5a6c4..3a9d4c288cc2 100644
--- a/drivers/media/usb/dvb-usb/dib0700.h
+++ b/drivers/media/usb/dvb-usb/dib0700.h
@@ -51,6 +51,8 @@ struct dib0700_state {
 	int (*read_status)(struct dvb_frontend *, enum fe_status *);
 	int (*sleep)(struct dvb_frontend* fe);
 	u8 buf[255];
+	struct i2c_client *i2c_client_demod;
+	struct i2c_client *i2c_client_tuner;
 };
 
 extern int dib0700_get_version(struct dvb_usb_device *d, u32 *hwversion,
diff --git a/drivers/media/usb/dvb-usb/dib0700_core.c b/drivers/media/usb/dvb-usb/dib0700_core.c
index 1ee7ec558293..94bd176104c1 100644
--- a/drivers/media/usb/dvb-usb/dib0700_core.c
+++ b/drivers/media/usb/dvb-usb/dib0700_core.c
@@ -911,10 +911,34 @@ static int dib0700_probe(struct usb_interface *intf,
 	return -ENODEV;
 }
 
+static void dib0700_disconnect(struct usb_interface *intf)
+{
+	struct dvb_usb_device *d = usb_get_intfdata(intf);
+	struct dib0700_state *st = d->priv;
+	struct i2c_client *client;
+
+	/* remove I2C client for tuner */
+	client = st->i2c_client_tuner;
+	if (client) {
+		module_put(client->dev.driver->owner);
+		i2c_unregister_device(client);
+	}
+
+	/* remove I2C client for demodulator */
+	client = st->i2c_client_demod;
+	if (client) {
+		module_put(client->dev.driver->owner);
+		i2c_unregister_device(client);
+	}
+
+	dvb_usb_device_exit(intf);
+}
+
+
 static struct usb_driver dib0700_driver = {
 	.name       = "dvb_usb_dib0700",
 	.probe      = dib0700_probe,
-	.disconnect = dvb_usb_device_exit,
+	.disconnect = dib0700_disconnect,
 	.id_table   = dib0700_usb_id_table,
 };
 
diff --git a/drivers/media/usb/dvb-usb/dib0700_devices.c b/drivers/media/usb/dvb-usb/dib0700_devices.c
index 366b05529915..3d99e141d566 100644
--- a/drivers/media/usb/dvb-usb/dib0700_devices.c
+++ b/drivers/media/usb/dvb-usb/dib0700_devices.c
@@ -23,6 +23,9 @@
 #include "dib0090.h"
 #include "lgdt3305.h"
 #include "mxl5007t.h"
+#include "mn88472.h"
+#include "tda18250.h"
+
 
 static int force_lna_activation;
 module_param(force_lna_activation, int, 0644);
@@ -430,6 +433,7 @@ static int stk7700ph_xc3028_callback(void *ptr, int component,
 		state->dib7000p_ops.set_gpio(adap->fe_adap[0].fe, 8, 0, 1);
 		break;
 	case XC2028_RESET_CLK:
+	case XC2028_I2C_FLUSH:
 		break;
 	default:
 		err("%s: unknown command %d, arg %d\n", __func__,
@@ -3725,6 +3729,90 @@ static int mxl5007t_tuner_attach(struct dvb_usb_adapter *adap)
 			  &hcw_mxl5007t_config) == NULL ? -ENODEV : 0;
 }
 
+static int xbox_one_attach(struct dvb_usb_adapter *adap)
+{
+	struct dib0700_state *st = adap->dev->priv;
+	struct i2c_client *client_demod, *client_tuner;
+	struct dvb_usb_device *d = adap->dev;
+	struct mn88472_config mn88472_config = { };
+	struct tda18250_config tda18250_config;
+	struct i2c_board_info info;
+
+	st->fw_use_new_i2c_api = 1;
+	st->disable_streaming_master_mode = 1;
+
+	/* fe power enable */
+	dib0700_set_gpio(adap->dev, GPIO6, GPIO_OUT, 0);
+	msleep(30);
+	dib0700_set_gpio(adap->dev, GPIO6, GPIO_OUT, 1);
+	msleep(30);
+
+	/* demod reset */
+	dib0700_set_gpio(adap->dev, GPIO10, GPIO_OUT, 1);
+	msleep(30);
+	dib0700_set_gpio(adap->dev, GPIO10, GPIO_OUT, 0);
+	msleep(30);
+	dib0700_set_gpio(adap->dev, GPIO10, GPIO_OUT, 1);
+	msleep(30);
+
+	/* attach demod */
+	mn88472_config.fe = &adap->fe_adap[0].fe;
+	mn88472_config.i2c_wr_max = 22;
+	mn88472_config.xtal = 20500000;
+	mn88472_config.ts_mode = PARALLEL_TS_MODE;
+	mn88472_config.ts_clock = FIXED_TS_CLOCK;
+	memset(&info, 0, sizeof(struct i2c_board_info));
+	strlcpy(info.type, "mn88472", I2C_NAME_SIZE);
+	info.addr = 0x18;
+	info.platform_data = &mn88472_config;
+	request_module(info.type);
+	client_demod = i2c_new_device(&d->i2c_adap, &info);
+	if (client_demod == NULL || client_demod->dev.driver == NULL)
+		goto fail_demod_device;
+	if (!try_module_get(client_demod->dev.driver->owner))
+		goto fail_demod_module;
+
+	st->i2c_client_demod = client_demod;
+
+	adap->fe_adap[0].fe = mn88472_config.get_dvb_frontend(client_demod);
+
+	/* attach tuner */
+	memset(&tda18250_config, 0, sizeof(tda18250_config));
+	tda18250_config.if_dvbt_6 = 3950;
+	tda18250_config.if_dvbt_7 = 4450;
+	tda18250_config.if_dvbt_8 = 4950;
+	tda18250_config.if_dvbc_6 = 4950;
+	tda18250_config.if_dvbc_8 = 4950;
+	tda18250_config.if_atsc = 4079;
+	tda18250_config.loopthrough = true;
+	tda18250_config.xtal_freq = TDA18250_XTAL_FREQ_27MHZ;
+	tda18250_config.fe = adap->fe_adap[0].fe;
+
+	memset(&info, 0, sizeof(struct i2c_board_info));
+	strlcpy(info.type, "tda18250", I2C_NAME_SIZE);
+	info.addr = 0x60;
+	info.platform_data = &tda18250_config;
+
+	request_module(info.type);
+	client_tuner = i2c_new_device(&adap->dev->i2c_adap, &info);
+	if (client_tuner == NULL || client_tuner->dev.driver == NULL)
+		goto fail_tuner_device;
+	if (!try_module_get(client_tuner->dev.driver->owner))
+		goto fail_tuner_module;
+
+	st->i2c_client_tuner = client_tuner;
+	return 0;
+
+fail_tuner_module:
+	i2c_unregister_device(client_tuner);
+fail_tuner_device:
+	module_put(client_demod->dev.driver->owner);
+fail_demod_module:
+	i2c_unregister_device(client_demod);
+fail_demod_device:
+	return -ENODEV;
+}
+
 
 /* DVB-USB and USB stuff follows */
 struct usb_device_id dib0700_usb_id_table[] = {
@@ -3816,7 +3904,8 @@ struct usb_device_id dib0700_usb_id_table[] = {
 	{ USB_DEVICE(USB_VID_PCTV,      USB_PID_PCTV_2002E_SE) },
 	{ USB_DEVICE(USB_VID_PCTV,      USB_PID_DIBCOM_STK8096PVR) },
 	{ USB_DEVICE(USB_VID_DIBCOM,    USB_PID_DIBCOM_STK8096PVR) },
-	{ USB_DEVICE(USB_VID_HAMA,	USB_PID_HAMA_DVBT_HYBRID) },
+/* 85 */{ USB_DEVICE(USB_VID_HAMA,	USB_PID_HAMA_DVBT_HYBRID) },
+	{ USB_DEVICE(USB_VID_MICROSOFT,	USB_PID_XBOX_ONE_TUNER) },
 	{ 0 }		/* Terminating entry */
 };
 MODULE_DEVICE_TABLE(usb, dib0700_usb_id_table);
@@ -5040,6 +5129,25 @@ struct dvb_usb_device_properties dib0700_devices[] = {
 				RC_PROTO_BIT_NEC,
 			.change_protocol  = dib0700_change_protocol,
 		},
+	}, { DIB0700_DEFAULT_DEVICE_PROPERTIES,
+		.num_adapters = 1,
+		.adapter = {
+			{
+				DIB0700_NUM_FRONTENDS(1),
+				.fe = {{
+					.frontend_attach = xbox_one_attach,
+
+					DIB0700_DEFAULT_STREAMING_CONFIG(0x82),
+				} },
+			},
+		},
+		.num_device_descs = 1,
+		.devices = {
+			{ "Microsoft Xbox One Digital TV Tuner",
+				{ &dib0700_usb_id_table[86], NULL },
+				{ NULL },
+			},
+		},
 	},
 };
 
diff --git a/drivers/media/usb/dvb-usb/dvb-usb.h b/drivers/media/usb/dvb-usb/dvb-usb.h
index e71fc86b4fb2..317ed6a82d19 100644
--- a/drivers/media/usb/dvb-usb/dvb-usb.h
+++ b/drivers/media/usb/dvb-usb/dvb-usb.h
@@ -17,14 +17,14 @@
 #include <linux/mutex.h>
 #include <media/rc-core.h>
 
-#include "dvb_frontend.h"
-#include "dvb_demux.h"
-#include "dvb_net.h"
-#include "dmxdev.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
+#include <media/dmxdev.h>
 
 #include "dvb-pll.h"
 
-#include "dvb-usb-ids.h"
+#include <media/dvb-usb-ids.h>
 
 /* debug */
 #ifdef CONFIG_DVB_USB_DEBUG
diff --git a/drivers/media/usb/dvb-usb/dw2102.c b/drivers/media/usb/dvb-usb/dw2102.c
index b421329b21fa..346946f35b1a 100644
--- a/drivers/media/usb/dvb-usb/dw2102.c
+++ b/drivers/media/usb/dvb-usb/dw2102.c
@@ -13,7 +13,7 @@
  *
  * see Documentation/dvb/README.dvb-usb for more information
  */
-#include "dvb-usb-ids.h"
+#include <media/dvb-usb-ids.h>
 #include "dw2102.h"
 #include "si21xx.h"
 #include "stv0299.h"
diff --git a/drivers/media/usb/dvb-usb/friio-fe.c b/drivers/media/usb/dvb-usb/friio-fe.c
index b6046e0e07f6..b2830c157548 100644
--- a/drivers/media/usb/dvb-usb/friio-fe.c
+++ b/drivers/media/usb/dvb-usb/friio-fe.c
@@ -13,6 +13,7 @@
 #include <linux/init.h>
 #include <linux/string.h>
 #include <linux/slab.h>
+#include <linux/kernel.h>
 
 #include "friio.h"
 
@@ -340,8 +341,6 @@ static u8 init_code[][2] = {
 	{0x76, 0x0C},
 };
 
-static const int init_code_len = sizeof(init_code) / sizeof(u8[2]);
-
 static int jdvbt90502_init(struct dvb_frontend *fe)
 {
 	int i = -1;
@@ -355,7 +354,7 @@ static int jdvbt90502_init(struct dvb_frontend *fe)
 	msg.addr = state->config.demod_address;
 	msg.flags = 0;
 	msg.len = 2;
-	for (i = 0; i < init_code_len; i++) {
+	for (i = 0; i < ARRAY_SIZE(init_code); i++) {
 		msg.buf = init_code[i];
 		ret = i2c_transfer(state->i2c, &msg, 1);
 		if (ret != 1)
diff --git a/drivers/media/usb/dvb-usb/pctv452e.c b/drivers/media/usb/dvb-usb/pctv452e.c
index 601ade7ca48d..0af74383083d 100644
--- a/drivers/media/usb/dvb-usb/pctv452e.c
+++ b/drivers/media/usb/dvb-usb/pctv452e.c
@@ -26,7 +26,7 @@
 /* FE Power */
 #include "lnbp22.h"
 
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 #include "ttpci-eeprom.h"
 
 static int debug;
@@ -913,6 +913,14 @@ static int pctv452e_frontend_attach(struct dvb_usb_adapter *a)
 						&a->dev->i2c_adap);
 	if (!a->fe_adap[0].fe)
 		return -ENODEV;
+
+	/*
+	 * dvb_frontend will call dvb_detach for both stb0899_detach
+	 * and stb0899_release but we only do dvb_attach(stb0899_attach).
+	 * Increment the module refcount instead.
+	 */
+	symbol_get(stb0899_attach);
+
 	if ((dvb_attach(lnbp22_attach, a->fe_adap[0].fe,
 					&a->dev->i2c_adap)) == NULL)
 		err("Cannot attach lnbp22\n");
diff --git a/drivers/media/usb/dvb-usb/ttusb2.c b/drivers/media/usb/dvb-usb/ttusb2.c
index e7020f245f53..12de89665d60 100644
--- a/drivers/media/usb/dvb-usb/ttusb2.c
+++ b/drivers/media/usb/dvb-usb/ttusb2.c
@@ -34,7 +34,7 @@
 #include "tda827x.h"
 #include "lnbp21.h"
 /* CA */
-#include "dvb_ca_en50221.h"
+#include <media/dvb_ca_en50221.h>
 
 /* debug */
 static int dvb_usb_ttusb2_debug;
diff --git a/drivers/media/usb/em28xx/Kconfig b/drivers/media/usb/em28xx/Kconfig
index 4cc029f18aa8..451e076525d3 100644
--- a/drivers/media/usb/em28xx/Kconfig
+++ b/drivers/media/usb/em28xx/Kconfig
@@ -71,10 +71,10 @@ config VIDEO_EM28XX_DVB
 	  Empiatech em28xx chips.
 
 config VIDEO_EM28XX_RC
-        tristate "EM28XX Remote Controller support"
-        depends on RC_CORE
-        depends on VIDEO_EM28XX
-        depends on !(RC_CORE=m && VIDEO_EM28XX=y)
-        default VIDEO_EM28XX
-        ---help---
-          Enables Remote Controller support on em28xx driver.
+	tristate "EM28XX Remote Controller support"
+	depends on RC_CORE
+	depends on VIDEO_EM28XX
+	depends on !(RC_CORE=m && VIDEO_EM28XX=y)
+	default VIDEO_EM28XX
+	---help---
+	  Enables Remote Controller support on em28xx driver.
diff --git a/drivers/media/usb/em28xx/Makefile b/drivers/media/usb/em28xx/Makefile
index 86bfc35e2ed4..8a224007d755 100644
--- a/drivers/media/usb/em28xx/Makefile
+++ b/drivers/media/usb/em28xx/Makefile
@@ -11,7 +11,5 @@ obj-$(CONFIG_VIDEO_EM28XX_ALSA) += em28xx-alsa.o
 obj-$(CONFIG_VIDEO_EM28XX_DVB) += em28xx-dvb.o
 obj-$(CONFIG_VIDEO_EM28XX_RC) += em28xx-rc.o
 
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c
index 4c57fd7929cb..34e16f6ab4ac 100644
--- a/drivers/media/usb/em28xx/em28xx-cards.c
+++ b/drivers/media/usb/em28xx/em28xx-cards.c
@@ -36,7 +36,6 @@
 #include <media/i2c/saa7115.h>
 #include <dt-bindings/media/tvp5150.h>
 #include <media/i2c/tvaudio.h>
-#include <media/i2c-addr.h>
 #include <media/tveeprom.h>
 #include <media/v4l2-common.h>
 #include <sound/ac97_codec.h>
diff --git a/drivers/media/usb/em28xx/em28xx-dvb.c b/drivers/media/usb/em28xx/em28xx-dvb.c
index 9950a740e04e..8a81c94a8a27 100644
--- a/drivers/media/usb/em28xx/em28xx-dvb.c
+++ b/drivers/media/usb/em28xx/em28xx-dvb.c
@@ -28,9 +28,9 @@
 #include <linux/usb.h>
 
 #include <media/v4l2-common.h>
-#include <dvb_demux.h>
-#include <dvb_net.h>
-#include <dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
+#include <media/dmxdev.h>
 #include <media/tuner.h>
 #include "tuner-simple.h"
 #include <linux/gpio.h>
@@ -2105,6 +2105,8 @@ static int em28xx_dvb_fini(struct em28xx *dev)
 		}
 	}
 
+	em28xx_unregister_dvb(dvb);
+
 	/* remove I2C SEC */
 	client = dvb->i2c_client_sec;
 	if (client) {
@@ -2126,7 +2128,6 @@ static int em28xx_dvb_fini(struct em28xx *dev)
 		i2c_unregister_device(client);
 	}
 
-	em28xx_unregister_dvb(dvb);
 	kfree(dvb);
 	dev->dvb = NULL;
 	kref_put(&dev->ref, em28xx_free_device);
diff --git a/drivers/media/usb/gspca/autogain_functions.c b/drivers/media/usb/gspca/autogain_functions.c
index 427db745e027..6dfab2b077f7 100644
--- a/drivers/media/usb/gspca/autogain_functions.c
+++ b/drivers/media/usb/gspca/autogain_functions.c
@@ -32,7 +32,7 @@ int gspca_expo_autogain(
 	int i, steps, retval = 0;
 
 	if (v4l2_ctrl_g_ctrl(gspca_dev->autogain) == 0)
-	        return 0;
+		return 0;
 
 	orig_gain = gain = v4l2_ctrl_g_ctrl(gspca_dev->gain);
 	orig_exposure = exposure = v4l2_ctrl_g_ctrl(gspca_dev->exposure);
@@ -41,8 +41,8 @@ int gspca_expo_autogain(
 	   desired lumination fast (with the risc of a slight overshoot) */
 	steps = abs(desired_avg_lum - avg_lum) / deadzone;
 
-	PDEBUG(D_FRAM, "autogain: lum: %d, desired: %d, steps: %d",
-		avg_lum, desired_avg_lum, steps);
+	gspca_dbg(gspca_dev, D_FRAM, "autogain: lum: %d, desired: %d, steps: %d\n",
+		  avg_lum, desired_avg_lum, steps);
 
 	for (i = 0; i < steps; i++) {
 		if (avg_lum > desired_avg_lum) {
@@ -75,17 +75,17 @@ int gspca_expo_autogain(
 	}
 
 	if (gain != orig_gain) {
-	        v4l2_ctrl_s_ctrl(gspca_dev->gain, gain);
+		v4l2_ctrl_s_ctrl(gspca_dev->gain, gain);
 		retval = 1;
 	}
 	if (exposure != orig_exposure) {
-	        v4l2_ctrl_s_ctrl(gspca_dev->exposure, exposure);
+		v4l2_ctrl_s_ctrl(gspca_dev->exposure, exposure);
 		retval = 1;
 	}
 
 	if (retval)
-		PDEBUG(D_FRAM, "autogain: changed gain: %d, expo: %d",
-			gain, exposure);
+		gspca_dbg(gspca_dev, D_FRAM, "autogain: changed gain: %d, expo: %d\n",
+			  gain, exposure);
 	return retval;
 }
 EXPORT_SYMBOL(gspca_expo_autogain);
@@ -112,7 +112,7 @@ int gspca_coarse_grained_expo_autogain(
 	int steps, retval = 0;
 
 	if (v4l2_ctrl_g_ctrl(gspca_dev->autogain) == 0)
-	        return 0;
+		return 0;
 
 	orig_gain = gain = v4l2_ctrl_g_ctrl(gspca_dev->gain);
 	orig_exposure = exposure = v4l2_ctrl_g_ctrl(gspca_dev->exposure);
@@ -126,8 +126,8 @@ int gspca_coarse_grained_expo_autogain(
 	   desired lumination fast (with the risc of a slight overshoot) */
 	steps = (desired_avg_lum - avg_lum) / deadzone;
 
-	PDEBUG(D_FRAM, "autogain: lum: %d, desired: %d, steps: %d",
-		avg_lum, desired_avg_lum, steps);
+	gspca_dbg(gspca_dev, D_FRAM, "autogain: lum: %d, desired: %d, steps: %d\n",
+		  avg_lum, desired_avg_lum, steps);
 
 	if ((gain + steps) > gain_high &&
 	    exposure < gspca_dev->exposure->maximum) {
@@ -158,17 +158,17 @@ int gspca_coarse_grained_expo_autogain(
 	}
 
 	if (gain != orig_gain) {
-	        v4l2_ctrl_s_ctrl(gspca_dev->gain, gain);
+		v4l2_ctrl_s_ctrl(gspca_dev->gain, gain);
 		retval = 1;
 	}
 	if (exposure != orig_exposure) {
-	        v4l2_ctrl_s_ctrl(gspca_dev->exposure, exposure);
+		v4l2_ctrl_s_ctrl(gspca_dev->exposure, exposure);
 		retval = 1;
 	}
 
 	if (retval)
-		PDEBUG(D_FRAM, "autogain: changed gain: %d, expo: %d",
-			gain, exposure);
+		gspca_dbg(gspca_dev, D_FRAM, "autogain: changed gain: %d, expo: %d\n",
+			  gain, exposure);
 	return retval;
 }
 EXPORT_SYMBOL(gspca_coarse_grained_expo_autogain);
diff --git a/drivers/media/usb/gspca/benq.c b/drivers/media/usb/gspca/benq.c
index 60a728203b3b..8a8db5eb6d5f 100644
--- a/drivers/media/usb/gspca/benq.c
+++ b/drivers/media/usb/gspca/benq.c
@@ -152,7 +152,7 @@ static void sd_isoc_irq(struct urb *urb)
 	u8 *data;
 	int i, st;
 
-	PDEBUG(D_PACK, "sd isoc irq");
+	gspca_dbg(gspca_dev, D_PACK, "sd isoc irq\n");
 	if (!gspca_dev->streaming)
 		return;
 	if (urb->status != 0) {
@@ -180,9 +180,9 @@ static void sd_isoc_irq(struct urb *urb)
 		/* check the packet status and length */
 		if (urb0->iso_frame_desc[i].actual_length != SD_PKT_SZ
 		    || urb->iso_frame_desc[i].actual_length != SD_PKT_SZ) {
-			PERR("ISOC bad lengths %d / %d",
-				urb0->iso_frame_desc[i].actual_length,
-				urb->iso_frame_desc[i].actual_length);
+			gspca_err(gspca_dev, "ISOC bad lengths %d / %d\n",
+				  urb0->iso_frame_desc[i].actual_length,
+				  urb->iso_frame_desc[i].actual_length);
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			continue;
 		}
diff --git a/drivers/media/usb/gspca/conex.c b/drivers/media/usb/gspca/conex.c
index bdcdf7999c56..6df4e204e291 100644
--- a/drivers/media/usb/gspca/conex.c
+++ b/drivers/media/usb/gspca/conex.c
@@ -70,7 +70,7 @@ static void reg_r(struct gspca_dev *gspca_dev,
 	struct usb_device *dev = gspca_dev->dev;
 
 	if (len > USB_BUF_SZ) {
-		PERR("reg_r: buffer overflow\n");
+		gspca_err(gspca_dev, "reg_r: buffer overflow\n");
 		return;
 	}
 
@@ -81,8 +81,8 @@ static void reg_r(struct gspca_dev *gspca_dev,
 			0,
 			index, gspca_dev->usb_buf, len,
 			500);
-	PDEBUG(D_USBI, "reg read [%02x] -> %02x ..",
-			index, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "reg read [%02x] -> %02x ..\n",
+		  index, gspca_dev->usb_buf[0]);
 }
 
 /* the bytes to write are in gspca_dev->usb_buf */
@@ -109,10 +109,11 @@ static void reg_w(struct gspca_dev *gspca_dev,
 	struct usb_device *dev = gspca_dev->dev;
 
 	if (len > USB_BUF_SZ) {
-		PERR("reg_w: buffer overflow\n");
+		gspca_err(gspca_dev, "reg_w: buffer overflow\n");
 		return;
 	}
-	PDEBUG(D_USBO, "reg write [%02x] = %02x..", index, *buffer);
+	gspca_dbg(gspca_dev, D_USBO, "reg write [%02x] = %02x..\n",
+		  index, *buffer);
 
 	memcpy(gspca_dev->usb_buf, buffer, len);
 	usb_control_msg(dev,
@@ -683,7 +684,7 @@ static void cx11646_jpeg(struct gspca_dev*gspca_dev)
 		reg_w_val(gspca_dev, 0x0053, 0x00);
 	} while (--retry);
 	if (retry == 0)
-		PERR("Damned Errors sending jpeg Table");
+		gspca_err(gspca_dev, "Damned Errors sending jpeg Table\n");
 	/* send the qtable now */
 	reg_r(gspca_dev, 0x0001, 1);		/* -> 0x18 */
 	length = 8;
diff --git a/drivers/media/usb/gspca/cpia1.c b/drivers/media/usb/gspca/cpia1.c
index e91d00762e94..2b09af8865f4 100644
--- a/drivers/media/usb/gspca/cpia1.c
+++ b/drivers/media/usb/gspca/cpia1.c
@@ -419,7 +419,8 @@ static int cpia_usb_transferCmd(struct gspca_dev *gspca_dev, u8 *command)
 		pipe = usb_sndctrlpipe(gspca_dev->dev, 0);
 		requesttype = USB_TYPE_VENDOR | USB_RECIP_DEVICE;
 	} else {
-		PERR("Unexpected first byte of command: %x", command[0]);
+		gspca_err(gspca_dev, "Unexpected first byte of command: %x\n",
+			  command[0]);
 		return -EINVAL;
 	}
 
@@ -542,7 +543,7 @@ static int do_command(struct gspca_dev *gspca_dev, u16 command,
 			input_report_key(gspca_dev->input_dev, KEY_CAMERA, a);
 			input_sync(gspca_dev->input_dev);
 #endif
-	        	sd->params.qx3.button = a;
+			sd->params.qx3.button = a;
 		}
 		if (sd->params.qx3.button) {
 			/* button pressed - unlock the latch */
@@ -700,11 +701,11 @@ static void reset_camera_params(struct gspca_dev *gspca_dev)
 
 static void printstatus(struct gspca_dev *gspca_dev, struct cam_params *params)
 {
-	PDEBUG(D_PROBE, "status: %02x %02x %02x %02x %02x %02x %02x %02x",
-	       params->status.systemState, params->status.grabState,
-	       params->status.streamState, params->status.fatalError,
-	       params->status.cmdError, params->status.debugFlags,
-	       params->status.vpStatus, params->status.errorCode);
+	gspca_dbg(gspca_dev, D_PROBE, "status: %02x %02x %02x %02x %02x %02x %02x %02x\n",
+		  params->status.systemState, params->status.grabState,
+		  params->status.streamState, params->status.fatalError,
+		  params->status.cmdError, params->status.debugFlags,
+		  params->status.vpStatus, params->status.errorCode);
 }
 
 static int goto_low_power(struct gspca_dev *gspca_dev)
@@ -722,14 +723,14 @@ static int goto_low_power(struct gspca_dev *gspca_dev)
 
 	if (sd->params.status.systemState != LO_POWER_STATE) {
 		if (sd->params.status.systemState != WARM_BOOT_STATE) {
-			PERR("unexpected state after lo power cmd: %02x",
-			     sd->params.status.systemState);
+			gspca_err(gspca_dev, "unexpected state after lo power cmd: %02x\n",
+				  sd->params.status.systemState);
 			printstatus(gspca_dev, &sd->params);
 		}
 		return -EIO;
 	}
 
-	PDEBUG(D_CONF, "camera now in LOW power state");
+	gspca_dbg(gspca_dev, D_CONF, "camera now in LOW power state\n");
 	return 0;
 }
 
@@ -752,13 +753,13 @@ static int goto_high_power(struct gspca_dev *gspca_dev)
 		return ret;
 
 	if (sd->params.status.systemState != HI_POWER_STATE) {
-		PERR("unexpected state after hi power cmd: %02x",
-		     sd->params.status.systemState);
+		gspca_err(gspca_dev, "unexpected state after hi power cmd: %02x\n",
+			  sd->params.status.systemState);
 		printstatus(gspca_dev, &sd->params);
 		return -EIO;
 	}
 
-	PDEBUG(D_CONF, "camera now in HIGH power state");
+	gspca_dbg(gspca_dev, D_CONF, "camera now in HIGH power state\n");
 	return 0;
 }
 
@@ -1301,7 +1302,7 @@ static void monitor_exposure(struct gspca_dev *gspca_dev)
 			sd->params.exposure.coarseExpHi = new_exposure >> 8;
 			setexp = 1;
 			sd->exposure_status = EXPOSURE_NORMAL;
-			PDEBUG(D_CONF, "Automatically decreasing sensor_fps");
+			gspca_dbg(gspca_dev, D_CONF, "Automatically decreasing sensor_fps\n");
 
 		} else if ((sd->exposure_status == EXPOSURE_VERY_LIGHT ||
 			    sd->exposure_status == EXPOSURE_LIGHT) &&
@@ -1330,7 +1331,7 @@ static void monitor_exposure(struct gspca_dev *gspca_dev)
 			sd->params.exposure.coarseExpHi = new_exposure >> 8;
 			setexp = 1;
 			sd->exposure_status = EXPOSURE_NORMAL;
-			PDEBUG(D_CONF, "Automatically increasing sensor_fps");
+			gspca_dbg(gspca_dev, D_CONF, "Automatically increasing sensor_fps\n");
 		}
 	} else {
 		/* Flicker control off */
@@ -1348,7 +1349,7 @@ static void monitor_exposure(struct gspca_dev *gspca_dev)
 				setexp = 1;
 			}
 			sd->exposure_status = EXPOSURE_NORMAL;
-			PDEBUG(D_CONF, "Automatically decreasing sensor_fps");
+			gspca_dbg(gspca_dev, D_CONF, "Automatically decreasing sensor_fps\n");
 
 		} else if ((sd->exposure_status == EXPOSURE_VERY_LIGHT ||
 			    sd->exposure_status == EXPOSURE_LIGHT) &&
@@ -1365,7 +1366,7 @@ static void monitor_exposure(struct gspca_dev *gspca_dev)
 				setexp = 1;
 			}
 			sd->exposure_status = EXPOSURE_NORMAL;
-			PDEBUG(D_CONF, "Automatically increasing sensor_fps");
+			gspca_dbg(gspca_dev, D_CONF, "Automatically increasing sensor_fps\n");
 		}
 	}
 
@@ -1433,8 +1434,8 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	sd->mainsFreq = FREQ_DEF == V4L2_CID_POWER_LINE_FREQUENCY_60HZ;
 	reset_camera_params(gspca_dev);
 
-	PDEBUG(D_PROBE, "cpia CPiA camera detected (vid/pid 0x%04X:0x%04X)",
-	       id->idVendor, id->idProduct);
+	gspca_dbg(gspca_dev, D_PROBE, "cpia CPiA camera detected (vid/pid 0x%04X:0x%04X)\n",
+		  id->idVendor, id->idProduct);
 
 	cam = &gspca_dev->cam;
 	cam->cam_mode = mode;
@@ -1445,8 +1446,8 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	sd->params.version.firmwareVersion = 0;
 	get_version_information(gspca_dev);
 	if (sd->params.version.firmwareVersion != 1) {
-		PERR("only firmware version 1 is supported (got: %d)",
-		     sd->params.version.firmwareVersion);
+		gspca_err(gspca_dev, "only firmware version 1 is supported (got: %d)\n",
+			  sd->params.version.firmwareVersion);
 		return -ENODEV;
 	}
 
@@ -1471,8 +1472,8 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	/* Start the camera in low power mode */
 	if (goto_low_power(gspca_dev)) {
 		if (sd->params.status.systemState != WARM_BOOT_STATE) {
-			PERR("unexpected systemstate: %02x",
-			     sd->params.status.systemState);
+			gspca_err(gspca_dev, "unexpected systemstate: %02x\n",
+				  sd->params.status.systemState);
 			printstatus(gspca_dev, &sd->params);
 			return -ENODEV;
 		}
@@ -1519,8 +1520,9 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		return ret;
 
 	if (sd->params.status.fatalError) {
-		PERR("fatal_error: %04x, vp_status: %04x",
-		     sd->params.status.fatalError, sd->params.status.vpStatus);
+		gspca_err(gspca_dev, "fatal_error: %04x, vp_status: %04x\n",
+			  sd->params.status.fatalError,
+			  sd->params.status.vpStatus);
 		return -EIO;
 	}
 
@@ -1667,18 +1669,18 @@ static int sd_init(struct gspca_dev *gspca_dev)
 
 	sd_stopN(gspca_dev);
 
-	PDEBUG(D_PROBE, "CPIA Version:             %d.%02d (%d.%d)",
-			sd->params.version.firmwareVersion,
-			sd->params.version.firmwareRevision,
-			sd->params.version.vcVersion,
-			sd->params.version.vcRevision);
-	PDEBUG(D_PROBE, "CPIA PnP-ID:              %04x:%04x:%04x",
-			sd->params.pnpID.vendor, sd->params.pnpID.product,
-			sd->params.pnpID.deviceRevision);
-	PDEBUG(D_PROBE, "VP-Version:               %d.%d %04x",
-			sd->params.vpVersion.vpVersion,
-			sd->params.vpVersion.vpRevision,
-			sd->params.vpVersion.cameraHeadID);
+	gspca_dbg(gspca_dev, D_PROBE, "CPIA Version:             %d.%02d (%d.%d)\n",
+		  sd->params.version.firmwareVersion,
+		  sd->params.version.firmwareRevision,
+		  sd->params.version.vcVersion,
+		  sd->params.version.vcRevision);
+	gspca_dbg(gspca_dev, D_PROBE, "CPIA PnP-ID:              %04x:%04x:%04x",
+		  sd->params.pnpID.vendor, sd->params.pnpID.product,
+		  sd->params.pnpID.deviceRevision);
+	gspca_dbg(gspca_dev, D_PROBE, "VP-Version:               %d.%d %04x",
+		  sd->params.vpVersion.vpVersion,
+		  sd->params.vpVersion.vpRevision,
+		  sd->params.vpVersion.cameraHeadID);
 
 	return 0;
 }
diff --git a/drivers/media/usb/gspca/dtcs033.c b/drivers/media/usb/gspca/dtcs033.c
index 96bfd4e0f0eb..cdf27cf0112a 100644
--- a/drivers/media/usb/gspca/dtcs033.c
+++ b/drivers/media/usb/gspca/dtcs033.c
@@ -71,18 +71,18 @@ static int reg_reqs(struct gspca_dev *gspca_dev,
 
 		if (gspca_dev->usb_err < 0) {
 
-			PERR("usb error request no: %d / %d\n",
-				i, n_reqs);
+			gspca_err(gspca_dev, "usb error request no: %d / %d\n",
+				  i, n_reqs);
 		} else if (preq->bRequestType & USB_DIR_IN) {
 
-			PDEBUG(D_STREAM,
-			"USB IN (%d) returned[%d] %02X %02X %02X %s",
-				i,
-				preq->wLength,
-				gspca_dev->usb_buf[0],
-				gspca_dev->usb_buf[1],
-				gspca_dev->usb_buf[2],
-				preq->wLength > 3 ? "...\n" : "\n");
+			gspca_dbg(gspca_dev, D_STREAM,
+				  "USB IN (%d) returned[%d] %02X %02X %02X %s\n",
+				  i,
+				  preq->wLength,
+				  gspca_dev->usb_buf[0],
+				  gspca_dev->usb_buf[1],
+				  gspca_dev->usb_buf[2],
+				  preq->wLength > 3 ? "...\n" : "\n");
 		}
 
 		i++;
@@ -176,12 +176,12 @@ static void dtcs033_setexposure(struct gspca_dev *gspca_dev,
 	reg_rw(gspca_dev,
 		bRequestType, bRequest, wValue, wIndex, 0);
 	if (gspca_dev->usb_err < 0)
-		PERR("usb error in setexposure(gain) sequence.\n");
+		gspca_err(gspca_dev, "usb error in setexposure(gain) sequence\n");
 
 	reg_rw(gspca_dev,
 		bRequestType, bRequest, (xtimeVal<<4), 0x6300, 0);
 	if (gspca_dev->usb_err < 0)
-		PERR("usb error in setexposure(time) sequence.\n");
+		gspca_err(gspca_dev, "usb error in setexposure(time) sequence\n");
 }
 
 /* specific webcam descriptor */
@@ -239,8 +239,8 @@ static int dtcs033_init_controls(struct gspca_dev *gspca_dev)
 				V4L2_CID_GAIN,
 				14,  33,  1,  24);/* [dB] */
 	if (hdl->error) {
-		PERR("Could not initialize controls: %d\n",
-			hdl->error);
+		gspca_err(gspca_dev, "Could not initialize controls: %d\n",
+			  hdl->error);
 		return hdl->error;
 	}
 
diff --git a/drivers/media/usb/gspca/etoms.c b/drivers/media/usb/gspca/etoms.c
index 8f84292936e9..48b28897051f 100644
--- a/drivers/media/usb/gspca/etoms.c
+++ b/drivers/media/usb/gspca/etoms.c
@@ -160,7 +160,7 @@ static void reg_r(struct gspca_dev *gspca_dev,
 	struct usb_device *dev = gspca_dev->dev;
 
 	if (len > USB_BUF_SZ) {
-		PERR("reg_r: buffer overflow\n");
+		gspca_err(gspca_dev, "reg_r: buffer overflow\n");
 		return;
 	}
 
@@ -170,8 +170,8 @@ static void reg_r(struct gspca_dev *gspca_dev,
 			USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
 			0,
 			index, gspca_dev->usb_buf, len, 500);
-	PDEBUG(D_USBI, "reg read [%02x] -> %02x ..",
-			index, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "reg read [%02x] -> %02x ..\n",
+		  index, gspca_dev->usb_buf[0]);
 }
 
 static void reg_w_val(struct gspca_dev *gspca_dev,
@@ -200,7 +200,8 @@ static void reg_w(struct gspca_dev *gspca_dev,
 		pr_err("reg_w: buffer overflow\n");
 		return;
 	}
-	PDEBUG(D_USBO, "reg write [%02x] = %02x..", index, *buffer);
+	gspca_dbg(gspca_dev, D_USBO, "reg write [%02x] = %02x..\n",
+		  index, *buffer);
 
 	memcpy(gspca_dev->usb_buf, buffer, len);
 	usb_control_msg(dev,
@@ -268,7 +269,7 @@ static int et_video(struct gspca_dev *gspca_dev,
 		     : 0);		/* stopvideo */
 	ret = Et_WaitStatus(gspca_dev);
 	if (ret != 0)
-		PERR("timeout video on/off");
+		gspca_err(gspca_dev, "timeout video on/off\n");
 	return ret;
 }
 
@@ -277,7 +278,7 @@ static void Et_init2(struct gspca_dev *gspca_dev)
 	__u8 value;
 	static const __u8 FormLine[] = { 0x84, 0x03, 0x14, 0xf4, 0x01, 0x05 };
 
-	PDEBUG(D_STREAM, "Open Init2 ET");
+	gspca_dbg(gspca_dev, D_STREAM, "Open Init2 ET\n");
 	reg_w_val(gspca_dev, ET_GPIO_DIR_CTRL, 0x2f);
 	reg_w_val(gspca_dev, ET_GPIO_OUT, 0x10);
 	reg_r(gspca_dev, ET_GPIO_IN, 1);
@@ -416,8 +417,6 @@ static void setcolors(struct gspca_dev *gspca_dev, s32 val)
 		i2c_w(gspca_dev, PAS106_REG13, &i2cflags, 1, 3);
 		i2c_w(gspca_dev, PAS106_REG9, I2cc, sizeof I2cc, 1);
 	}
-/*	PDEBUG(D_CONF , "Etoms red %d blue %d green %d",
-		I2cc[3], I2cc[0], green); */
 }
 
 static s32 getcolors(struct gspca_dev *gspca_dev)
@@ -451,7 +450,7 @@ static void Et_init1(struct gspca_dev *gspca_dev)
 /*	__u8 I2c0 [] = {0x0a, 0x12, 0x05, 0xfe, 0xfe, 0xc0, 0x01, 0x00};
 						 * 1/60000 hmm ?? */
 
-	PDEBUG(D_STREAM, "Open Init1 ET");
+	gspca_dbg(gspca_dev, D_STREAM, "Open Init1 ET\n\n");
 	reg_w_val(gspca_dev, ET_GPIO_DIR_CTRL, 7);
 	reg_r(gspca_dev, ET_GPIO_IN, 1);
 	reg_w_val(gspca_dev, ET_RESET_ALL, 1);
@@ -463,9 +462,9 @@ static void Et_init1(struct gspca_dev *gspca_dev)
 		value = ET_COMP_VAL1;
 	else
 		value = ET_COMP_VAL0;
-	PDEBUG(D_STREAM, "Open mode %d Compression %d",
-	       gspca_dev->cam.cam_mode[(int) gspca_dev->curr_mode].priv,
-	       value);
+	gspca_dbg(gspca_dev, D_STREAM, "Open mode %d Compression %d\n",
+		  gspca_dev->cam.cam_mode[(int) gspca_dev->curr_mode].priv,
+		  value);
 	reg_w_val(gspca_dev, ET_COMP, value);
 	reg_w_val(gspca_dev, ET_MAXQt, 0x1d);
 	reg_w_val(gspca_dev, ET_MINQt, 0x02);
@@ -604,7 +603,8 @@ static __u8 Et_getgainG(struct gspca_dev *gspca_dev)
 
 	if (sd->sensor == SENSOR_PAS106) {
 		i2c_r(gspca_dev, PAS106_REG0e);
-		PDEBUG(D_CONF, "Etoms gain G %d", gspca_dev->usb_buf[0]);
+		gspca_dbg(gspca_dev, D_CONF, "Etoms gain G %d\n",
+			  gspca_dev->usb_buf[0]);
 		return gspca_dev->usb_buf[0];
 	}
 	return 0x1f;
@@ -652,11 +652,11 @@ static void do_autogain(struct gspca_dev *gspca_dev)
 	b = ((b << 7) >> 10);
 	g = ((g << 9) + (g << 7) + (g << 5)) >> 10;
 	luma = LIMIT(r + g + b);
-	PDEBUG(D_FRAM, "Etoms luma G %d", luma);
+	gspca_dbg(gspca_dev, D_FRAM, "Etoms luma G %d\n", luma);
 	if (luma < luma_mean - luma_delta || luma > luma_mean + luma_delta) {
 		Gbright += (luma_mean - luma) >> spring;
 		Gbright = BLIMIT(Gbright);
-		PDEBUG(D_FRAM, "Etoms Gbright %d", Gbright);
+		gspca_dbg(gspca_dev, D_FRAM, "Etoms Gbright %d\n", Gbright);
 		Et_setgainG(gspca_dev, (__u8) Gbright);
 	}
 }
@@ -673,10 +673,10 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 	seqframe = data[0] & 0x3f;
 	len = (int) (((data[0] & 0xc0) << 2) | data[1]);
 	if (seqframe == 0x3f) {
-		PDEBUG(D_FRAM,
-		       "header packet found datalength %d !!", len);
-		PDEBUG(D_FRAM, "G %d R %d G %d B %d",
-		       data[2], data[3], data[4], data[5]);
+		gspca_dbg(gspca_dev, D_FRAM,
+			  "header packet found datalength %d !!\n", len);
+		gspca_dbg(gspca_dev, D_FRAM, "G %d R %d G %d B %d",
+			  data[2], data[3], data[4], data[5]);
 		data += 30;
 		/* don't change datalength as the chips provided it */
 		gspca_frame_add(gspca_dev, LAST_PACKET, NULL, 0);
diff --git a/drivers/media/usb/gspca/finepix.c b/drivers/media/usb/gspca/finepix.c
index 7bb469aa61a7..1ef1239eb3db 100644
--- a/drivers/media/usb/gspca/finepix.c
+++ b/drivers/media/usb/gspca/finepix.c
@@ -89,7 +89,7 @@ static void dostream(struct work_struct *work)
 	int ret = 0;
 	int len;
 
-	PDEBUG(D_STREAM, "dostream started");
+	gspca_dbg(gspca_dev, D_STREAM, "dostream started\n");
 
 	/* loop reading a frame */
 again:
@@ -160,7 +160,7 @@ static void dostream(struct work_struct *work)
 	}
 
 out:
-	PDEBUG(D_STREAM, "dostream stopped");
+	gspca_dbg(gspca_dev, D_STREAM, "dostream stopped\n");
 }
 
 /* this function is called at probe time */
diff --git a/drivers/media/usb/gspca/gl860/gl860.c b/drivers/media/usb/gspca/gl860/gl860.c
index cea8d7f51c3c..262200af76a3 100644
--- a/drivers/media/usb/gspca/gl860/gl860.c
+++ b/drivers/media/usb/gspca/gl860/gl860.c
@@ -582,7 +582,7 @@ int gl860_RTx(struct gspca_dev *gspca_dev,
 		pr_err("ctrl transfer failed %4d [p%02x r%d v%04x i%04x len%d]\n",
 		       r, pref, req, val, index, len);
 	else if (len > 1 && r < len)
-		PERR("short ctrl transfer %d/%d", r, len);
+		gspca_err(gspca_dev, "short ctrl transfer %d/%d\n", r, len);
 
 	msleep(1);
 
@@ -661,7 +661,7 @@ static int gl860_guess_sensor(struct gspca_dev *gspca_dev,
 		ctrl_out(gspca_dev, 0x40, 1, 0x006a, 0x000d, 0, NULL);
 		msleep(56);
 
-		PDEBUG(D_PROBE, "probing for sensor MI2020 or OVXXXX");
+		gspca_dbg(gspca_dev, D_PROBE, "probing for sensor MI2020 or OVXXXX\n");
 		nOV = 0;
 		for (ntry = 0; ntry < 4; ntry++) {
 			ctrl_out(gspca_dev, 0x40, 1, 0x0040, 0x0000, 0, NULL);
@@ -671,14 +671,14 @@ static int gl860_guess_sensor(struct gspca_dev *gspca_dev,
 			ctrl_out(gspca_dev, 0x40, 1, 0x7a00, 0x8030, 0, NULL);
 			msleep(10);
 			ctrl_in(gspca_dev, 0xc0, 2, 0x7a00, 0x8030, 1, &probe);
-			PDEBUG(D_PROBE, "probe=0x%02x", probe);
+			gspca_dbg(gspca_dev, D_PROBE, "probe=0x%02x\n", probe);
 			if (probe == 0xff)
 				nOV++;
 		}
 
 		if (nOV) {
-			PDEBUG(D_PROBE, "0xff -> OVXXXX");
-			PDEBUG(D_PROBE, "probing for sensor OV2640 or OV9655");
+			gspca_dbg(gspca_dev, D_PROBE, "0xff -> OVXXXX\n");
+			gspca_dbg(gspca_dev, D_PROBE, "probing for sensor OV2640 or OV9655");
 
 			nb26 = nb96 = 0;
 			for (ntry = 0; ntry < 4; ntry++) {
@@ -694,22 +694,23 @@ static int gl860_guess_sensor(struct gspca_dev *gspca_dev,
 						1, &probe);
 
 				if (probe == 0x26 || probe == 0x40) {
-					PDEBUG(D_PROBE,
-						"probe=0x%02x -> OV2640",
-						probe);
+					gspca_dbg(gspca_dev, D_PROBE,
+						  "probe=0x%02x -> OV2640\n",
+						  probe);
 					sd->sensor = ID_OV2640;
 					nb26 += 4;
 					break;
 				}
 				if (probe == 0x96 || probe == 0x55) {
-					PDEBUG(D_PROBE,
-						"probe=0x%02x -> OV9655",
-						probe);
+					gspca_dbg(gspca_dev, D_PROBE,
+						  "probe=0x%02x -> OV9655\n",
+						  probe);
 					sd->sensor = ID_OV9655;
 					nb96 += 4;
 					break;
 				}
-				PDEBUG(D_PROBE, "probe=0x%02x", probe);
+				gspca_dbg(gspca_dev, D_PROBE, "probe=0x%02x\n",
+					  probe);
 				if (probe == 0x00)
 					nb26++;
 				if (probe == 0xff)
@@ -719,21 +720,21 @@ static int gl860_guess_sensor(struct gspca_dev *gspca_dev,
 			if (nb26 < 4 && nb96 < 4)
 				return -1;
 		} else {
-			PDEBUG(D_PROBE, "Not any 0xff -> MI2020");
+			gspca_dbg(gspca_dev, D_PROBE, "Not any 0xff -> MI2020\n");
 			sd->sensor = ID_MI2020;
 		}
 	}
 
 	if (_MI1320_) {
-		PDEBUG(D_PROBE, "05e3:f191 sensor MI1320 (1.3M)");
+		gspca_dbg(gspca_dev, D_PROBE, "05e3:f191 sensor MI1320 (1.3M)\n");
 	} else if (_MI2020_) {
-		PDEBUG(D_PROBE, "05e3:0503 sensor MI2020 (2.0M)");
+		gspca_dbg(gspca_dev, D_PROBE, "05e3:0503 sensor MI2020 (2.0M)\n");
 	} else if (_OV9655_) {
-		PDEBUG(D_PROBE, "05e3:0503 sensor OV9655 (1.3M)");
+		gspca_dbg(gspca_dev, D_PROBE, "05e3:0503 sensor OV9655 (1.3M)\n");
 	} else if (_OV2640_) {
-		PDEBUG(D_PROBE, "05e3:0503 sensor OV2640 (2.0M)");
+		gspca_dbg(gspca_dev, D_PROBE, "05e3:0503 sensor OV2640 (2.0M)\n");
 	} else {
-		PDEBUG(D_PROBE, "***** Unknown sensor *****");
+		gspca_dbg(gspca_dev, D_PROBE, "***** Unknown sensor *****\n");
 		return -1;
 	}
 
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index 961343873fd0..d29773b8f696 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -63,18 +63,18 @@ static void PDEBUG_MODE(struct gspca_dev *gspca_dev, int debug, char *txt,
 			__u32 pixfmt, int w, int h)
 {
 	if ((pixfmt >> 24) >= '0' && (pixfmt >> 24) <= 'z') {
-		PDEBUG(debug, "%s %c%c%c%c %dx%d",
-			txt,
-			pixfmt & 0xff,
-			(pixfmt >> 8) & 0xff,
-			(pixfmt >> 16) & 0xff,
-			pixfmt >> 24,
-			w, h);
+		gspca_dbg(gspca_dev, debug, "%s %c%c%c%c %dx%d\n",
+			  txt,
+			  pixfmt & 0xff,
+			  (pixfmt >> 8) & 0xff,
+			  (pixfmt >> 16) & 0xff,
+			  pixfmt >> 24,
+			  w, h);
 	} else {
-		PDEBUG(debug, "%s 0x%08x %dx%d",
-			txt,
-			pixfmt,
-			w, h);
+		gspca_dbg(gspca_dev, debug, "%s 0x%08x %dx%d\n",
+			  txt,
+			  pixfmt,
+			  w, h);
 	}
 }
 
@@ -122,7 +122,7 @@ static void int_irq(struct urb *urb)
 	case 0:
 		if (gspca_dev->sd_desc->int_pkt_scan(gspca_dev,
 		    urb->transfer_buffer, urb->actual_length) < 0) {
-			PERR("Unknown packet received");
+			gspca_err(gspca_dev, "Unknown packet received\n");
 		}
 		break;
 
@@ -136,7 +136,8 @@ static void int_irq(struct urb *urb)
 		break;
 
 	default:
-		PERR("URB error %i, resubmitting", urb->status);
+		gspca_err(gspca_dev, "URB error %i, resubmitting\n",
+			  urb->status);
 		urb->status = 0;
 		ret = 0;
 	}
@@ -197,8 +198,8 @@ static int alloc_and_submit_int_urb(struct gspca_dev *gspca_dev,
 
 	buffer_len = le16_to_cpu(ep->wMaxPacketSize);
 	interval = ep->bInterval;
-	PDEBUG(D_CONF, "found int in endpoint: 0x%x, buffer_len=%u, interval=%u",
-		ep->bEndpointAddress, buffer_len, interval);
+	gspca_dbg(gspca_dev, D_CONF, "found int in endpoint: 0x%x, buffer_len=%u, interval=%u\n",
+		  ep->bEndpointAddress, buffer_len, interval);
 
 	dev = gspca_dev->dev;
 
@@ -221,7 +222,8 @@ static int alloc_and_submit_int_urb(struct gspca_dev *gspca_dev,
 	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
 	ret = usb_submit_urb(urb, GFP_KERNEL);
 	if (ret < 0) {
-		PERR("submit int URB failed with error %i", ret);
+		gspca_err(gspca_dev, "submit int URB failed with error %i\n",
+			  ret);
 		goto error_submit;
 	}
 	gspca_dev->int_urb = urb;
@@ -307,7 +309,7 @@ static void fill_frame(struct gspca_dev *gspca_dev,
 		if (gspca_dev->frozen)
 			return;
 #endif
-		PERR("urb status: %d", urb->status);
+		gspca_err(gspca_dev, "urb status: %d\n", urb->status);
 		urb->status = 0;
 		goto resubmit;
 	}
@@ -330,8 +332,8 @@ static void fill_frame(struct gspca_dev *gspca_dev,
 		}
 
 		/* let the packet be analyzed by the subdriver */
-		PDEBUG(D_PACK, "packet [%d] o:%d l:%d",
-			i, urb->iso_frame_desc[i].offset, len);
+		gspca_dbg(gspca_dev, D_PACK, "packet [%d] o:%d l:%d\n",
+			  i, urb->iso_frame_desc[i].offset, len);
 		data = (u8 *) urb->transfer_buffer
 					+ urb->iso_frame_desc[i].offset;
 		pkt_scan(gspca_dev, data, len);
@@ -353,7 +355,7 @@ static void isoc_irq(struct urb *urb)
 {
 	struct gspca_dev *gspca_dev = (struct gspca_dev *) urb->context;
 
-	PDEBUG(D_PACK, "isoc irq");
+	gspca_dbg(gspca_dev, D_PACK, "isoc irq\n");
 	if (!gspca_dev->streaming)
 		return;
 	fill_frame(gspca_dev, urb);
@@ -367,7 +369,7 @@ static void bulk_irq(struct urb *urb)
 	struct gspca_dev *gspca_dev = (struct gspca_dev *) urb->context;
 	int st;
 
-	PDEBUG(D_PACK, "bulk irq");
+	gspca_dbg(gspca_dev, D_PACK, "bulk irq\n");
 	if (!gspca_dev->streaming)
 		return;
 	switch (urb->status) {
@@ -380,12 +382,12 @@ static void bulk_irq(struct urb *urb)
 		if (gspca_dev->frozen)
 			return;
 #endif
-		PERR("urb status: %d", urb->status);
+		gspca_err(gspca_dev, "urb status: %d\n", urb->status);
 		urb->status = 0;
 		goto resubmit;
 	}
 
-	PDEBUG(D_PACK, "packet l:%d", urb->actual_length);
+	gspca_dbg(gspca_dev, D_PACK, "packet l:%d\n", urb->actual_length);
 	gspca_dev->sd_desc->pkt_scan(gspca_dev,
 				urb->transfer_buffer,
 				urb->actual_length);
@@ -418,7 +420,7 @@ void gspca_frame_add(struct gspca_dev *gspca_dev,
 	struct gspca_frame *frame;
 	int i, j;
 
-	PDEBUG(D_PACK, "add t:%d l:%d",	packet_type, len);
+	gspca_dbg(gspca_dev, D_PACK, "add t:%d l:%d\n",	packet_type, len);
 
 	if (packet_type == FIRST_PACKET) {
 		i = atomic_read(&gspca_dev->fr_i);
@@ -452,9 +454,9 @@ void gspca_frame_add(struct gspca_dev *gspca_dev,
 	/* append the packet to the frame buffer */
 	if (len > 0) {
 		if (gspca_dev->image_len + len > gspca_dev->frsz) {
-			PERR("frame overflow %d > %d",
-				gspca_dev->image_len + len,
-				gspca_dev->frsz);
+			gspca_err(gspca_dev, "frame overflow %d > %d\n",
+				  gspca_dev->image_len + len,
+				  gspca_dev->frsz);
 			packet_type = DISCARD_PACKET;
 		} else {
 /* !! image is NULL only when last pkt is LAST or DISCARD
@@ -484,8 +486,8 @@ void gspca_frame_add(struct gspca_dev *gspca_dev,
 		i = (i + 1) % GSPCA_MAX_FRAMES;
 		atomic_set(&gspca_dev->fr_i, i);
 		wake_up_interruptible(&gspca_dev->wq);	/* event = new frame */
-		PDEBUG(D_FRAM, "frame complete len:%d",
-			frame->v4l2_buf.bytesused);
+		gspca_dbg(gspca_dev, D_FRAM, "frame complete len:%d\n",
+			  frame->v4l2_buf.bytesused);
 		gspca_dev->image = NULL;
 		gspca_dev->image_len = 0;
 	}
@@ -500,7 +502,7 @@ static int frame_alloc(struct gspca_dev *gspca_dev, struct file *file,
 	int i;
 
 	frsz = gspca_dev->pixfmt.sizeimage;
-	PDEBUG(D_STREAM, "frame alloc frsz: %d", frsz);
+	gspca_dbg(gspca_dev, D_STREAM, "frame alloc frsz: %d\n", frsz);
 	frsz = PAGE_ALIGN(frsz);
 	if (count >= GSPCA_MAX_FRAMES)
 		count = GSPCA_MAX_FRAMES - 1;
@@ -535,7 +537,7 @@ static void frame_free(struct gspca_dev *gspca_dev)
 {
 	int i;
 
-	PDEBUG(D_STREAM, "frame free");
+	gspca_dbg(gspca_dev, D_STREAM, "frame free\n");
 	if (gspca_dev->frbuf != NULL) {
 		vfree(gspca_dev->frbuf);
 		gspca_dev->frbuf = NULL;
@@ -553,7 +555,7 @@ static void destroy_urbs(struct gspca_dev *gspca_dev)
 	struct urb *urb;
 	unsigned int i;
 
-	PDEBUG(D_STREAM, "kill transfer");
+	gspca_dbg(gspca_dev, D_STREAM, "kill transfer\n");
 	for (i = 0; i < MAX_NURBS; i++) {
 		urb = gspca_dev->urb[i];
 		if (urb == NULL)
@@ -594,7 +596,7 @@ static void gspca_stream_off(struct gspca_dev *gspca_dev)
 	gspca_input_create_urb(gspca_dev);
 	if (gspca_dev->sd_desc->stop0)
 		gspca_dev->sd_desc->stop0(gspca_dev);
-	PDEBUG(D_STREAM, "stream off OK");
+	gspca_dbg(gspca_dev, D_STREAM, "stream off OK\n");
 }
 
 /*
@@ -653,7 +655,7 @@ static u32 which_bandwidth(struct gspca_dev *gspca_dev)
 			bandwidth *= 30;		/* 30 fps */
 	}
 
-	PDEBUG(D_STREAM, "min bandwidth: %d", bandwidth);
+	gspca_dbg(gspca_dev, D_STREAM, "min bandwidth: %d\n", bandwidth);
 	return bandwidth;
 }
 
@@ -713,8 +715,8 @@ static int build_isoc_ep_tb(struct gspca_dev *gspca_dev,
 		}
 		if (!found)
 			break;
-		PDEBUG(D_STREAM, "alt %d bandwidth %d",
-				ep_tb->alt, ep_tb->bandwidth);
+		gspca_dbg(gspca_dev, D_STREAM, "alt %d bandwidth %d\n",
+			  ep_tb->alt, ep_tb->bandwidth);
 		last_bw = ep_tb->bandwidth;
 		i++;
 		ep_tb++;
@@ -732,7 +734,7 @@ static int build_isoc_ep_tb(struct gspca_dev *gspca_dev,
 			gspca_dev->dev->speed == USB_SPEED_FULL &&
 			last_bw >= 1000000 &&
 			i > 1) {
-		PDEBUG(D_STREAM, "dev has usb audio, skipping highest alt");
+		gspca_dbg(gspca_dev, D_STREAM, "dev has usb audio, skipping highest alt\n");
 		i--;
 		ep_tb--;
 	}
@@ -772,16 +774,16 @@ static int create_urbs(struct gspca_dev *gspca_dev,
 		if (npkt == 0)
 			npkt = 32;		/* default value */
 		bsize = psize * npkt;
-		PDEBUG(D_STREAM,
-			"isoc %d pkts size %d = bsize:%d",
-			npkt, psize, bsize);
+		gspca_dbg(gspca_dev, D_STREAM,
+			  "isoc %d pkts size %d = bsize:%d\n",
+			  npkt, psize, bsize);
 		nurbs = DEF_NURBS;
 	} else {				/* bulk */
 		npkt = 0;
 		bsize = gspca_dev->cam.bulk_size;
 		if (bsize == 0)
 			bsize = psize;
-		PDEBUG(D_STREAM, "bulk bsize:%d", bsize);
+		gspca_dbg(gspca_dev, D_STREAM, "bulk bsize:%d\n", bsize);
 		if (gspca_dev->cam.bulk_nurbs != 0)
 			nurbs = gspca_dev->cam.bulk_nurbs;
 		else
@@ -900,7 +902,8 @@ static int gspca_init_transfer(struct gspca_dev *gspca_dev)
 			}
 		}
 		if (!gspca_dev->cam.no_urb_create) {
-			PDEBUG(D_STREAM, "init transfer alt %d", alt);
+			gspca_dbg(gspca_dev, D_STREAM, "init transfer alt %d\n",
+				  alt);
 			ret = create_urbs(gspca_dev,
 				alt_xfer(&intf->altsetting[alt], xfer,
 					 gspca_dev->xfer_ep));
@@ -952,7 +955,8 @@ static int gspca_init_transfer(struct gspca_dev *gspca_dev)
 		/* the bandwidth is not wide enough
 		 * negotiate or try a lower alternate setting */
 retry:
-		PERR("alt %d - bandwidth not wide enough, trying again", alt);
+		gspca_err(gspca_dev, "alt %d - bandwidth not wide enough, trying again\n",
+			  alt);
 		msleep(20);	/* wait for kill complete */
 		if (gspca_dev->sd_desc->isoc_nego) {
 			ret = gspca_dev->sd_desc->isoc_nego(gspca_dev);
@@ -1282,7 +1286,7 @@ static int dev_open(struct file *file)
 	struct gspca_dev *gspca_dev = video_drvdata(file);
 	int ret;
 
-	PDEBUG(D_STREAM, "[%s] open", current->comm);
+	gspca_dbg(gspca_dev, D_STREAM, "[%s] open\n", current->comm);
 
 	/* protect the subdriver against rmmod */
 	if (!try_module_get(gspca_dev->module))
@@ -1298,7 +1302,7 @@ static int dev_close(struct file *file)
 {
 	struct gspca_dev *gspca_dev = video_drvdata(file);
 
-	PDEBUG(D_STREAM, "[%s] close", current->comm);
+	gspca_dbg(gspca_dev, D_STREAM, "[%s] close\n", current->comm);
 
 	/* Needed for gspca_stream_off, always lock before queue_lock! */
 	if (mutex_lock_interruptible(&gspca_dev->usb_lock))
@@ -1319,7 +1323,7 @@ static int dev_close(struct file *file)
 	mutex_unlock(&gspca_dev->queue_lock);
 	mutex_unlock(&gspca_dev->usb_lock);
 
-	PDEBUG(D_STREAM, "close done");
+	gspca_dbg(gspca_dev, D_STREAM, "close done\n");
 
 	return v4l2_fh_release(file);
 }
@@ -1440,7 +1444,7 @@ static int vidioc_reqbufs(struct file *file, void *priv,
 	}
 out:
 	mutex_unlock(&gspca_dev->queue_lock);
-	PDEBUG(D_STREAM, "reqbufs st:%d c:%d", ret, rb->count);
+	gspca_dbg(gspca_dev, D_STREAM, "reqbufs st:%d c:%d\n", ret, rb->count);
 	return ret;
 }
 
@@ -1598,7 +1602,8 @@ static int dev_mmap(struct file *file, struct vm_area_struct *vma)
 
 	start = vma->vm_start;
 	size = vma->vm_end - vma->vm_start;
-	PDEBUG(D_STREAM, "mmap start:%08x size:%d", (int) start, (int) size);
+	gspca_dbg(gspca_dev, D_STREAM, "mmap start:%08x size:%d\n",
+		  (int) start, (int)size);
 
 	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
 		return -ERESTARTSYS;
@@ -1610,7 +1615,7 @@ static int dev_mmap(struct file *file, struct vm_area_struct *vma)
 	frame = NULL;
 	for (i = 0; i < gspca_dev->nframes; ++i) {
 		if (gspca_dev->frame[i].v4l2_buf.memory != V4L2_MEMORY_MMAP) {
-			PDEBUG(D_STREAM, "mmap bad memory type");
+			gspca_dbg(gspca_dev, D_STREAM, "mmap bad memory type\n");
 			break;
 		}
 		if ((gspca_dev->frame[i].v4l2_buf.m.offset >> PAGE_SHIFT)
@@ -1620,12 +1625,12 @@ static int dev_mmap(struct file *file, struct vm_area_struct *vma)
 		}
 	}
 	if (frame == NULL) {
-		PDEBUG(D_STREAM, "mmap no frame buffer found");
+		gspca_dbg(gspca_dev, D_STREAM, "mmap no frame buffer found\n");
 		ret = -EINVAL;
 		goto out;
 	}
 	if (size != frame->v4l2_buf.length) {
-		PDEBUG(D_STREAM, "mmap bad size");
+		gspca_dbg(gspca_dev, D_STREAM, "mmap bad size\n");
 		ret = -EINVAL;
 		goto out;
 	}
@@ -1693,7 +1698,7 @@ static int vidioc_dqbuf(struct file *file, void *priv,
 	struct gspca_frame *frame;
 	int i, j, ret;
 
-	PDEBUG(D_FRAM, "dqbuf");
+	gspca_dbg(gspca_dev, D_FRAM, "dqbuf\n");
 
 	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
 		return -ERESTARTSYS;
@@ -1731,14 +1736,14 @@ static int vidioc_dqbuf(struct file *file, void *priv,
 
 	frame->v4l2_buf.flags &= ~V4L2_BUF_FLAG_DONE;
 	memcpy(v4l2_buf, &frame->v4l2_buf, sizeof *v4l2_buf);
-	PDEBUG(D_FRAM, "dqbuf %d", j);
+	gspca_dbg(gspca_dev, D_FRAM, "dqbuf %d\n", j);
 	ret = 0;
 
 	if (gspca_dev->memory == V4L2_MEMORY_USERPTR) {
 		if (copy_to_user((__u8 __user *) frame->v4l2_buf.m.userptr,
 				 frame->data,
 				 frame->v4l2_buf.bytesused)) {
-			PERR("dqbuf cp to user failed");
+			gspca_err(gspca_dev, "dqbuf cp to user failed\n");
 			ret = -EFAULT;
 		}
 	}
@@ -1769,27 +1774,27 @@ static int vidioc_qbuf(struct file *file, void *priv,
 	struct gspca_frame *frame;
 	int i, index, ret;
 
-	PDEBUG(D_FRAM, "qbuf %d", v4l2_buf->index);
+	gspca_dbg(gspca_dev, D_FRAM, "qbuf %d\n", v4l2_buf->index);
 
 	if (mutex_lock_interruptible(&gspca_dev->queue_lock))
 		return -ERESTARTSYS;
 
 	index = v4l2_buf->index;
 	if ((unsigned) index >= gspca_dev->nframes) {
-		PDEBUG(D_FRAM,
-			"qbuf idx %d >= %d", index, gspca_dev->nframes);
+		gspca_dbg(gspca_dev, D_FRAM,
+			  "qbuf idx %d >= %d\n", index, gspca_dev->nframes);
 		ret = -EINVAL;
 		goto out;
 	}
 	if (v4l2_buf->memory != gspca_dev->memory) {
-		PDEBUG(D_FRAM, "qbuf bad memory type");
+		gspca_dbg(gspca_dev, D_FRAM, "qbuf bad memory type\n");
 		ret = -EINVAL;
 		goto out;
 	}
 
 	frame = &gspca_dev->frame[index];
 	if (frame->v4l2_buf.flags & BUF_ALL_FLAGS) {
-		PDEBUG(D_FRAM, "qbuf bad state");
+		gspca_dbg(gspca_dev, D_FRAM, "qbuf bad state\n");
 		ret = -EINVAL;
 		goto out;
 	}
@@ -1823,7 +1828,7 @@ static int read_alloc(struct gspca_dev *gspca_dev,
 	struct v4l2_buffer v4l2_buf;
 	int i, ret;
 
-	PDEBUG(D_STREAM, "read alloc");
+	gspca_dbg(gspca_dev, D_STREAM, "read alloc\n");
 
 	if (mutex_lock_interruptible(&gspca_dev->usb_lock))
 		return -ERESTARTSYS;
@@ -1837,7 +1842,8 @@ static int read_alloc(struct gspca_dev *gspca_dev,
 		rb.memory = GSPCA_MEMORY_READ;
 		ret = vidioc_reqbufs(file, gspca_dev, &rb);
 		if (ret != 0) {
-			PDEBUG(D_STREAM, "read reqbuf err %d", ret);
+			gspca_dbg(gspca_dev, D_STREAM, "read reqbuf err %d\n",
+				  ret);
 			goto out;
 		}
 		memset(&v4l2_buf, 0, sizeof v4l2_buf);
@@ -1847,7 +1853,8 @@ static int read_alloc(struct gspca_dev *gspca_dev,
 			v4l2_buf.index = i;
 			ret = vidioc_qbuf(file, gspca_dev, &v4l2_buf);
 			if (ret != 0) {
-				PDEBUG(D_STREAM, "read qbuf err: %d", ret);
+				gspca_dbg(gspca_dev, D_STREAM, "read qbuf err: %d\n",
+					  ret);
 				goto out;
 			}
 		}
@@ -1856,28 +1863,28 @@ static int read_alloc(struct gspca_dev *gspca_dev,
 	/* start streaming */
 	ret = vidioc_streamon(file, gspca_dev, V4L2_BUF_TYPE_VIDEO_CAPTURE);
 	if (ret != 0)
-		PDEBUG(D_STREAM, "read streamon err %d", ret);
+		gspca_dbg(gspca_dev, D_STREAM, "read streamon err %d\n", ret);
 out:
 	mutex_unlock(&gspca_dev->usb_lock);
 	return ret;
 }
 
-static unsigned int dev_poll(struct file *file, poll_table *wait)
+static __poll_t dev_poll(struct file *file, poll_table *wait)
 {
 	struct gspca_dev *gspca_dev = video_drvdata(file);
-	unsigned long req_events = poll_requested_events(wait);
-	int ret = 0;
+	__poll_t req_events = poll_requested_events(wait);
+	__poll_t ret = 0;
 
-	PDEBUG(D_FRAM, "poll");
+	gspca_dbg(gspca_dev, D_FRAM, "poll\n");
 
-	if (req_events & POLLPRI)
+	if (req_events & EPOLLPRI)
 		ret |= v4l2_ctrl_poll(file, wait);
 
-	if (req_events & (POLLIN | POLLRDNORM)) {
+	if (req_events & (EPOLLIN | EPOLLRDNORM)) {
 		/* if reqbufs is not done, the user would use read() */
 		if (gspca_dev->memory == GSPCA_MEMORY_NO) {
 			if (read_alloc(gspca_dev, file) != 0) {
-				ret |= POLLERR;
+				ret |= EPOLLERR;
 				goto out;
 			}
 		}
@@ -1886,17 +1893,17 @@ static unsigned int dev_poll(struct file *file, poll_table *wait)
 
 		/* check if an image has been received */
 		if (mutex_lock_interruptible(&gspca_dev->queue_lock) != 0) {
-			ret |= POLLERR;
+			ret |= EPOLLERR;
 			goto out;
 		}
 		if (gspca_dev->fr_o != atomic_read(&gspca_dev->fr_i))
-			ret |= POLLIN | POLLRDNORM;
+			ret |= EPOLLIN | EPOLLRDNORM;
 		mutex_unlock(&gspca_dev->queue_lock);
 	}
 
 out:
 	if (!gspca_dev->present)
-		ret |= POLLHUP;
+		ret |= EPOLLHUP;
 
 	return ret;
 }
@@ -1910,7 +1917,7 @@ static ssize_t dev_read(struct file *file, char __user *data,
 	struct timeval timestamp;
 	int n, ret, ret2;
 
-	PDEBUG(D_FRAM, "read (%zd)", count);
+	gspca_dbg(gspca_dev, D_FRAM, "read (%zd)\n", count);
 	if (gspca_dev->memory == GSPCA_MEMORY_NO) { /* first time ? */
 		ret = read_alloc(gspca_dev, file);
 		if (ret != 0)
@@ -1927,7 +1934,8 @@ static ssize_t dev_read(struct file *file, char __user *data,
 		v4l2_buf.memory = GSPCA_MEMORY_READ;
 		ret = vidioc_dqbuf(file, gspca_dev, &v4l2_buf);
 		if (ret != 0) {
-			PDEBUG(D_STREAM, "read dqbuf err %d", ret);
+			gspca_dbg(gspca_dev, D_STREAM, "read dqbuf err %d\n",
+				  ret);
 			return ret;
 		}
 
@@ -1940,7 +1948,8 @@ static ssize_t dev_read(struct file *file, char __user *data,
 			break;
 		ret = vidioc_qbuf(file, gspca_dev, &v4l2_buf);
 		if (ret != 0) {
-			PDEBUG(D_STREAM, "read qbuf err %d", ret);
+			gspca_dbg(gspca_dev, D_STREAM, "read qbuf err %d\n",
+				  ret);
 			return ret;
 		}
 	}
@@ -1950,7 +1959,8 @@ static ssize_t dev_read(struct file *file, char __user *data,
 		count = frame->v4l2_buf.bytesused;
 	ret = copy_to_user(data, frame->data, count);
 	if (ret != 0) {
-		PERR("read cp to user lack %d / %zd", ret, count);
+		gspca_err(gspca_dev, "read cp to user lack %d / %zd\n",
+			  ret, count);
 		ret = -EFAULT;
 		goto out;
 	}
@@ -2128,7 +2138,8 @@ int gspca_dev_probe2(struct usb_interface *intf,
 	}
 
 	usb_set_intfdata(intf, gspca_dev);
-	PDEBUG(D_PROBE, "%s created", video_device_node_name(&gspca_dev->vdev));
+	gspca_dbg(gspca_dev, D_PROBE, "%s created\n",
+		  video_device_node_name(&gspca_dev->vdev));
 
 	gspca_input_create_urb(gspca_dev);
 
@@ -2183,8 +2194,8 @@ void gspca_disconnect(struct usb_interface *intf)
 	struct input_dev *input_dev;
 #endif
 
-	PDEBUG(D_PROBE, "%s disconnect",
-		video_device_node_name(&gspca_dev->vdev));
+	gspca_dbg(gspca_dev, D_PROBE, "%s disconnect\n",
+		  video_device_node_name(&gspca_dev->vdev));
 
 	mutex_lock(&gspca_dev->usb_lock);
 
diff --git a/drivers/media/usb/gspca/gspca.h b/drivers/media/usb/gspca/gspca.h
index 9e0cf711642b..249cb38a542f 100644
--- a/drivers/media/usb/gspca/gspca.h
+++ b/drivers/media/usb/gspca/gspca.h
@@ -26,11 +26,12 @@
 extern int gspca_debug;
 
 
-#define PDEBUG(level, fmt, ...) \
-	v4l2_dbg(level, gspca_debug, &gspca_dev->v4l2_dev, fmt, ##__VA_ARGS__)
+#define gspca_dbg(gspca_dev, level, fmt, ...)			\
+	v4l2_dbg(level, gspca_debug, &(gspca_dev)->v4l2_dev,	\
+		 fmt, ##__VA_ARGS__)
 
-#define PERR(fmt, ...) \
-	v4l2_err(&gspca_dev->v4l2_dev, fmt, ##__VA_ARGS__)
+#define gspca_err(gspca_dev, fmt, ...)				\
+	v4l2_err(&(gspca_dev)->v4l2_dev, fmt, ##__VA_ARGS__)
 
 #define GSPCA_MAX_FRAMES 16	/* maximum number of video frame buffers */
 /* image transfers */
diff --git a/drivers/media/usb/gspca/jeilinj.c b/drivers/media/usb/gspca/jeilinj.c
index 34e043b7d1bc..86d0a0a45631 100644
--- a/drivers/media/usb/gspca/jeilinj.c
+++ b/drivers/media/usb/gspca/jeilinj.c
@@ -262,7 +262,7 @@ static int jlj_start(struct gspca_dev *gspca_dev)
 	msleep(2);
 	setfreq(gspca_dev, v4l2_ctrl_g_ctrl(sd->freq));
 	if (gspca_dev->usb_err < 0)
-		PERR("Start streaming command failed");
+		gspca_err(gspca_dev, "Start streaming command failed\n");
 	return gspca_dev->usb_err;
 }
 
@@ -273,17 +273,18 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 	int packet_type;
 	u32 header_marker;
 
-	PDEBUG(D_STREAM, "Got %d bytes out of %d for Block 0",
-			len, JEILINJ_MAX_TRANSFER);
+	gspca_dbg(gspca_dev, D_STREAM, "Got %d bytes out of %d for Block 0\n",
+		  len, JEILINJ_MAX_TRANSFER);
 	if (len != JEILINJ_MAX_TRANSFER) {
-		PDEBUG(D_PACK, "bad length");
+		gspca_dbg(gspca_dev, D_PACK, "bad length\n");
 		goto discard;
 	}
 	/* check if it's start of frame */
 	header_marker = ((u32 *)data)[0];
 	if (header_marker == FRAME_START) {
 		sd->blocks_left = data[0x0a] - 1;
-		PDEBUG(D_STREAM, "blocks_left = 0x%x", sd->blocks_left);
+		gspca_dbg(gspca_dev, D_STREAM, "blocks_left = 0x%x\n",
+			  sd->blocks_left);
 		/* Start a new frame, and add the JPEG header, first thing */
 		gspca_frame_add(gspca_dev, FIRST_PACKET,
 				sd->jpeg_hdr, JPEG_HDR_SZ);
@@ -292,8 +293,8 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 				data + FRAME_HEADER_LEN,
 				JEILINJ_MAX_TRANSFER - FRAME_HEADER_LEN);
 	} else if (sd->blocks_left > 0) {
-		PDEBUG(D_STREAM, "%d blocks remaining for frame",
-				sd->blocks_left);
+		gspca_dbg(gspca_dev, D_STREAM, "%d blocks remaining for frame\n",
+			  sd->blocks_left);
 		sd->blocks_left -= 1;
 		if (sd->blocks_left == 0)
 			packet_type = LAST_PACKET;
@@ -378,8 +379,8 @@ static int sd_start(struct gspca_dev *gspca_dev)
 			gspca_dev->pixfmt.width,
 			0x21);          /* JPEG 422 */
 	jpeg_set_qual(dev->jpeg_hdr, dev->quality);
-	PDEBUG(D_STREAM, "Start streaming at %dx%d",
-		gspca_dev->pixfmt.height, gspca_dev->pixfmt.width);
+	gspca_dbg(gspca_dev, D_STREAM, "Start streaming at %dx%d\n",
+		  gspca_dev->pixfmt.height, gspca_dev->pixfmt.width);
 	jlj_start(gspca_dev);
 	return gspca_dev->usb_err;
 }
diff --git a/drivers/media/usb/gspca/jl2005bcd.c b/drivers/media/usb/gspca/jl2005bcd.c
index 17c7a953564c..d668589598d6 100644
--- a/drivers/media/usb/gspca/jl2005bcd.c
+++ b/drivers/media/usb/gspca/jl2005bcd.c
@@ -149,7 +149,8 @@ static int jl2005c_start_new_frame(struct gspca_dev *gspca_dev)
 			return retval;
 		i++;
 	}
-	PDEBUG(D_FRAM, "frame_brightness is 0x%02x", gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_FRAM, "frame_brightness is 0x%02x\n",
+		  gspca_dev->usb_buf[0]);
 	return retval;
 }
 
@@ -176,10 +177,11 @@ static int jl2005c_get_firmware_id(struct gspca_dev *gspca_dev)
 	int retval = -1;
 	unsigned char regs_to_read[] = {0x57, 0x02, 0x03, 0x5d, 0x5e, 0x5f};
 
-	PDEBUG(D_PROBE, "Running jl2005c_get_firmware_id");
+	gspca_dbg(gspca_dev, D_PROBE, "Running jl2005c_get_firmware_id\n");
 	/* Read the first ID byte once for warmup */
 	retval = jl2005c_read_reg(gspca_dev, regs_to_read[0]);
-	PDEBUG(D_PROBE, "response is %02x", gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_PROBE, "response is %02x\n",
+		  gspca_dev->usb_buf[0]);
 	if (retval < 0)
 		return retval;
 	/* Now actually get the ID string */
@@ -189,13 +191,13 @@ static int jl2005c_get_firmware_id(struct gspca_dev *gspca_dev)
 			return retval;
 		sd->firmware_id[i] = gspca_dev->usb_buf[0];
 	}
-	PDEBUG(D_PROBE, "firmware ID is %02x%02x%02x%02x%02x%02x",
-						sd->firmware_id[0],
-						sd->firmware_id[1],
-						sd->firmware_id[2],
-						sd->firmware_id[3],
-						sd->firmware_id[4],
-						sd->firmware_id[5]);
+	gspca_dbg(gspca_dev, D_PROBE, "firmware ID is %02x%02x%02x%02x%02x%02x\n",
+		  sd->firmware_id[0],
+		  sd->firmware_id[1],
+		  sd->firmware_id[2],
+		  sd->firmware_id[3],
+		  sd->firmware_id[4],
+		  sd->firmware_id[5]);
 	return 0;
 }
 
@@ -341,9 +343,9 @@ static void jl2005c_dostream(struct work_struct *work)
 				usb_rcvbulkpipe(gspca_dev->dev, 0x82),
 				buffer, JL2005C_MAX_TRANSFER, &act_len,
 				JL2005C_DATA_TIMEOUT);
-			PDEBUG(D_PACK,
-				"Got %d bytes out of %d for header",
-					act_len, JL2005C_MAX_TRANSFER);
+			gspca_dbg(gspca_dev, D_PACK,
+				  "Got %d bytes out of %d for header\n",
+				  act_len, JL2005C_MAX_TRANSFER);
 			if (ret < 0 || act_len < JL2005C_MAX_TRANSFER)
 				goto quit_stream;
 			/* Check whether we actually got the first blodk */
@@ -354,7 +356,8 @@ static void jl2005c_dostream(struct work_struct *work)
 			/* total size to fetch is byte 7, times blocksize
 			 * of which we already got act_len */
 			bytes_left = buffer[0x07] * dev->block_size - act_len;
-			PDEBUG(D_PACK, "bytes_left = 0x%x", bytes_left);
+			gspca_dbg(gspca_dev, D_PACK, "bytes_left = 0x%x\n",
+				  bytes_left);
 			/* We keep the header. It has other information, too.*/
 			packet_type = FIRST_PACKET;
 			gspca_frame_add(gspca_dev, packet_type,
@@ -370,9 +373,9 @@ static void jl2005c_dostream(struct work_struct *work)
 				JL2005C_DATA_TIMEOUT);
 			if (ret < 0 || act_len < data_len)
 				goto quit_stream;
-			PDEBUG(D_PACK,
-				"Got %d bytes out of %d for frame",
-						data_len, bytes_left);
+			gspca_dbg(gspca_dev, D_PACK,
+				  "Got %d bytes out of %d for frame\n",
+				  data_len, bytes_left);
 			bytes_left -= data_len;
 			if (bytes_left == 0) {
 				packet_type = LAST_PACKET;
@@ -449,19 +452,19 @@ static int sd_start(struct gspca_dev *gspca_dev)
 
 	switch (gspca_dev->pixfmt.width) {
 	case 640:
-		PDEBUG(D_STREAM, "Start streaming at vga resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at vga resolution\n");
 		jl2005c_stream_start_vga_lg(gspca_dev);
 		break;
 	case 320:
-		PDEBUG(D_STREAM, "Start streaming at qvga resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at qvga resolution\n");
 		jl2005c_stream_start_vga_small(gspca_dev);
 		break;
 	case 352:
-		PDEBUG(D_STREAM, "Start streaming at cif resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at cif resolution\n");
 		jl2005c_stream_start_cif_lg(gspca_dev);
 		break;
 	case 176:
-		PDEBUG(D_STREAM, "Start streaming at qcif resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at qcif resolution\n");
 		jl2005c_stream_start_cif_small(gspca_dev);
 		break;
 	default:
diff --git a/drivers/media/usb/gspca/kinect.c b/drivers/media/usb/gspca/kinect.c
index 2f28b38c5479..0cfdf8a1e19d 100644
--- a/drivers/media/usb/gspca/kinect.c
+++ b/drivers/media/usb/gspca/kinect.c
@@ -151,8 +151,9 @@ static int send_cmd(struct gspca_dev *gspca_dev, uint16_t cmd, void *cmdbuf,
 	memcpy(obuf+sizeof(*chdr), cmdbuf, cmd_len);
 
 	res = kinect_write(udev, obuf, cmd_len + sizeof(*chdr));
-	PDEBUG(D_USBO, "Control cmd=%04x tag=%04x len=%04x: %d", cmd,
-		sd->cam_tag, cmd_len, res);
+	gspca_dbg(gspca_dev, D_USBO, "Control cmd=%04x tag=%04x len=%04x: %d\n",
+		  cmd,
+		  sd->cam_tag, cmd_len, res);
 	if (res < 0) {
 		pr_err("send_cmd: Output control transfer failed (%d)\n", res);
 		return res;
@@ -161,7 +162,7 @@ static int send_cmd(struct gspca_dev *gspca_dev, uint16_t cmd, void *cmdbuf,
 	do {
 		actual_len = kinect_read(udev, ibuf, 0x200);
 	} while (actual_len == 0);
-	PDEBUG(D_USBO, "Control reply: %d", actual_len);
+	gspca_dbg(gspca_dev, D_USBO, "Control reply: %d\n", actual_len);
 	if (actual_len < sizeof(*rhdr)) {
 		pr_err("send_cmd: Input control transfer failed (%d)\n",
 		       actual_len);
@@ -213,7 +214,7 @@ static int write_register(struct gspca_dev *gspca_dev, uint16_t reg,
 	cmd[0] = cpu_to_le16(reg);
 	cmd[1] = cpu_to_le16(data);
 
-	PDEBUG(D_USBO, "Write Reg 0x%04x <= 0x%02x", reg, data);
+	gspca_dbg(gspca_dev, D_USBO, "Write Reg 0x%04x <= 0x%02x\n", reg, data);
 	res = send_cmd(gspca_dev, 0x03, cmd, 4, reply, 4);
 	if (res < 0)
 		return res;
@@ -274,7 +275,7 @@ static int sd_config_depth(struct gspca_dev *gspca_dev,
 /* this function is called at probe and resume time */
 static int sd_init(struct gspca_dev *gspca_dev)
 {
-	PDEBUG(D_PROBE, "Kinect Camera device.");
+	gspca_dbg(gspca_dev, D_PROBE, "Kinect Camera device.\n");
 
 	return 0;
 }
diff --git a/drivers/media/usb/gspca/konica.c b/drivers/media/usb/gspca/konica.c
index 31b2117e8f1d..989ae997f66d 100644
--- a/drivers/media/usb/gspca/konica.c
+++ b/drivers/media/usb/gspca/konica.c
@@ -263,7 +263,7 @@ static void sd_isoc_irq(struct urb *urb)
 	u8 *data;
 	int i, st;
 
-	PDEBUG(D_PACK, "sd isoc irq");
+	gspca_dbg(gspca_dev, D_PACK, "sd isoc irq\n");
 	if (!gspca_dev->streaming)
 		return;
 
@@ -274,7 +274,7 @@ static void sd_isoc_irq(struct urb *urb)
 		if (gspca_dev->frozen)
 			return;
 #endif
-		PERR("urb status: %d", urb->status);
+		gspca_err(gspca_dev, "urb status: %d\n", urb->status);
 		st = usb_submit_urb(urb, GFP_ATOMIC);
 		if (st < 0)
 			pr_err("resubmit urb error %d\n", st);
@@ -292,30 +292,31 @@ static void sd_isoc_irq(struct urb *urb)
 	sd->last_data_urb = NULL;
 
 	if (!data_urb || data_urb->start_frame != status_urb->start_frame) {
-		PERR("lost sync on frames");
+		gspca_err(gspca_dev, "lost sync on frames\n");
 		goto resubmit;
 	}
 
 	if (data_urb->number_of_packets != status_urb->number_of_packets) {
-		PERR("no packets does not match, data: %d, status: %d",
-		     data_urb->number_of_packets,
-		     status_urb->number_of_packets);
+		gspca_err(gspca_dev, "no packets does not match, data: %d, status: %d\n",
+			  data_urb->number_of_packets,
+			  status_urb->number_of_packets);
 		goto resubmit;
 	}
 
 	for (i = 0; i < status_urb->number_of_packets; i++) {
 		if (data_urb->iso_frame_desc[i].status ||
 		    status_urb->iso_frame_desc[i].status) {
-			PERR("pkt %d data-status %d, status-status %d", i,
-			     data_urb->iso_frame_desc[i].status,
-			     status_urb->iso_frame_desc[i].status);
+			gspca_err(gspca_dev, "pkt %d data-status %d, status-status %d\n",
+				  i,
+				  data_urb->iso_frame_desc[i].status,
+				  status_urb->iso_frame_desc[i].status);
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			continue;
 		}
 
 		if (status_urb->iso_frame_desc[i].actual_length != 1) {
-			PERR("bad status packet length %d",
-			     status_urb->iso_frame_desc[i].actual_length);
+			gspca_err(gspca_dev, "bad status packet length %d\n",
+				  status_urb->iso_frame_desc[i].actual_length);
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			continue;
 		}
@@ -360,11 +361,12 @@ static void sd_isoc_irq(struct urb *urb)
 	if (data_urb) {
 		st = usb_submit_urb(data_urb, GFP_ATOMIC);
 		if (st < 0)
-			PERR("usb_submit_urb(data_urb) ret %d", st);
+			gspca_err(gspca_dev, "usb_submit_urb(data_urb) ret %d\n",
+				  st);
 	}
 	st = usb_submit_urb(status_urb, GFP_ATOMIC);
 	if (st < 0)
-		PERR("usb_submit_urb(status_urb) ret %d\n", st);
+		gspca_err(gspca_dev, "usb_submit_urb(status_urb) ret %d\n", st);
 }
 
 static int sd_s_ctrl(struct v4l2_ctrl *ctrl)
diff --git a/drivers/media/usb/gspca/m5602/m5602_core.c b/drivers/media/usb/gspca/m5602/m5602_core.c
index f1dcd9021983..b83ec4285a0b 100644
--- a/drivers/media/usb/gspca/m5602/m5602_core.c
+++ b/drivers/media/usb/gspca/m5602/m5602_core.c
@@ -66,8 +66,8 @@ int m5602_read_bridge(struct sd *sd, const u8 address, u8 *i2c_data)
 			      1, M5602_URB_MSG_TIMEOUT);
 	*i2c_data = buf[0];
 
-	PDEBUG(D_CONF, "Reading bridge register 0x%x containing 0x%x",
-	       address, *i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Reading bridge register 0x%x containing 0x%x\n",
+		  address, *i2c_data);
 
 	/* usb_control_msg(...) returns the number of bytes sent upon success,
 	mask that and return zero instead*/
@@ -82,8 +82,8 @@ int m5602_write_bridge(struct sd *sd, const u8 address, const u8 i2c_data)
 	struct usb_device *udev = sd->gspca_dev.dev;
 	__u8 *buf = sd->gspca_dev.usb_buf;
 
-	PDEBUG(D_CONF, "Writing bridge register 0x%x with 0x%x",
-	       address, i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Writing bridge register 0x%x with 0x%x\n",
+		  address, i2c_data);
 
 	memcpy(buf, bridge_urb_skeleton,
 	       sizeof(bridge_urb_skeleton));
@@ -154,8 +154,8 @@ int m5602_read_sensor(struct sd *sd, const u8 address,
 
 		err = m5602_read_bridge(sd, M5602_XB_I2C_DATA, &(i2c_data[i]));
 
-		PDEBUG(D_CONF, "Reading sensor register 0x%x containing 0x%x ",
-		       address, *i2c_data);
+		gspca_dbg(gspca_dev, D_CONF, "Reading sensor register 0x%x containing 0x%x\n",
+			  address, *i2c_data);
 	}
 	return err;
 }
@@ -187,8 +187,8 @@ int m5602_write_sensor(struct sd *sd, const u8 address,
 		memcpy(p, sensor_urb_skeleton + 16, 4);
 		p[3] = i2c_data[i];
 		p += 4;
-		PDEBUG(D_CONF, "Writing sensor register 0x%x with 0x%x",
-		       address, i2c_data[i]);
+		gspca_dbg(gspca_dev, D_CONF, "Writing sensor register 0x%x with 0x%x\n",
+			  address, i2c_data[i]);
 	}
 
 	/* Copy the tailer */
@@ -264,7 +264,7 @@ static int m5602_init(struct gspca_dev *gspca_dev)
 	struct sd *sd = (struct sd *) gspca_dev;
 	int err;
 
-	PDEBUG(D_CONF, "Initializing ALi m5602 webcam");
+	gspca_dbg(gspca_dev, D_CONF, "Initializing ALi m5602 webcam\n");
 	/* Run the init sequence */
 	err = sd->sensor->init(sd);
 
@@ -299,7 +299,7 @@ static int m5602_start_transfer(struct gspca_dev *gspca_dev)
 			      0x04, 0x40, 0x19, 0x0000, buf,
 			      sizeof(buffer), M5602_URB_MSG_TIMEOUT);
 
-	PDEBUG(D_STREAM, "Transfer started");
+	gspca_dbg(gspca_dev, D_STREAM, "Transfer started\n");
 	return (err < 0) ? err : 0;
 }
 
@@ -309,14 +309,14 @@ static void m5602_urb_complete(struct gspca_dev *gspca_dev,
 	struct sd *sd = (struct sd *) gspca_dev;
 
 	if (len < 6) {
-		PDEBUG(D_PACK, "Packet is less than 6 bytes");
+		gspca_dbg(gspca_dev, D_PACK, "Packet is less than 6 bytes\n");
 		return;
 	}
 
 	/* Frame delimiter: ff xx xx xx ff ff */
 	if (data[0] == 0xff && data[4] == 0xff && data[5] == 0xff &&
 	    data[2] != sd->frame_id) {
-		PDEBUG(D_FRAM, "Frame delimiter detected");
+		gspca_dbg(gspca_dev, D_FRAM, "Frame delimiter detected\n");
 		sd->frame_id = data[2];
 
 		/* Remove the extra fluff appended on each header */
@@ -331,8 +331,8 @@ static void m5602_urb_complete(struct gspca_dev *gspca_dev,
 		/* Create a new frame */
 		gspca_frame_add(gspca_dev, FIRST_PACKET, data, len);
 
-		PDEBUG(D_FRAM, "Starting new frame %d",
-		       sd->frame_count);
+		gspca_dbg(gspca_dev, D_FRAM, "Starting new frame %d\n",
+			  sd->frame_count);
 
 	} else {
 		int cur_frame_len;
@@ -343,8 +343,8 @@ static void m5602_urb_complete(struct gspca_dev *gspca_dev,
 		len -= 4;
 
 		if (cur_frame_len + len <= gspca_dev->frsz) {
-			PDEBUG(D_FRAM, "Continuing frame %d copying %d bytes",
-			       sd->frame_count, len);
+			gspca_dbg(gspca_dev, D_FRAM, "Continuing frame %d copying %d bytes\n",
+				  sd->frame_count, len);
 
 			gspca_frame_add(gspca_dev, INTER_PACKET,
 					data, len);
@@ -397,7 +397,7 @@ static int m5602_configure(struct gspca_dev *gspca_dev,
 	return 0;
 
 fail:
-	PERR("ALi m5602 webcam failed");
+	gspca_err(gspca_dev, "ALi m5602 webcam failed\n");
 	cam->cam_mode = NULL;
 	cam->nmodes = 0;
 
diff --git a/drivers/media/usb/gspca/m5602/m5602_mt9m111.c b/drivers/media/usb/gspca/m5602/m5602_mt9m111.c
index 7d01ddd7ed01..c9947c4a0f63 100644
--- a/drivers/media/usb/gspca/m5602/m5602_mt9m111.c
+++ b/drivers/media/usb/gspca/m5602/m5602_mt9m111.c
@@ -212,7 +212,7 @@ int mt9m111_probe(struct sd *sd)
 		return -ENODEV;
 	}
 
-	PDEBUG(D_PROBE, "Probing for a mt9m111 sensor");
+	gspca_dbg(gspca_dev, D_PROBE, "Probing for a mt9m111 sensor\n");
 
 	/* Do the preinit */
 	for (i = 0; i < ARRAY_SIZE(preinit_mt9m111); i++) {
@@ -375,11 +375,11 @@ int mt9m111_start(struct sd *sd)
 
 	switch (width) {
 	case 640:
-		PDEBUG(D_CONF, "Configuring camera for VGA mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for VGA mode\n");
 		break;
 
 	case 320:
-		PDEBUG(D_CONF, "Configuring camera for QVGA mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for QVGA mode\n");
 		break;
 	}
 	return err;
@@ -398,7 +398,8 @@ static int mt9m111_set_hvflip(struct gspca_dev *gspca_dev)
 	int hflip;
 	int vflip;
 
-	PDEBUG(D_CONF, "Set hvflip to %d %d", sd->hflip->val, sd->vflip->val);
+	gspca_dbg(gspca_dev, D_CONF, "Set hvflip to %d %d\n",
+		  sd->hflip->val, sd->vflip->val);
 
 	/* The mt9m111 is flipped by default */
 	hflip = !sd->hflip->val;
@@ -439,7 +440,7 @@ static int mt9m111_set_auto_white_balance(struct gspca_dev *gspca_dev,
 
 	err = m5602_write_sensor(sd, MT9M111_CP_OPERATING_MODE_CTL, data, 2);
 
-	PDEBUG(D_CONF, "Set auto white balance %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto white balance %d\n", val);
 	return err;
 }
 
@@ -472,8 +473,8 @@ static int mt9m111_set_gain(struct gspca_dev *gspca_dev, __s32 val)
 
 	data[1] = (tmp & 0xff);
 	data[0] = (tmp & 0xff00) >> 8;
-	PDEBUG(D_CONF, "tmp=%d, data[1]=%d, data[0]=%d", tmp,
-	       data[1], data[0]);
+	gspca_dbg(gspca_dev, D_CONF, "tmp=%d, data[1]=%d, data[0]=%d\n", tmp,
+		  data[1], data[0]);
 
 	err = m5602_write_sensor(sd, MT9M111_SC_GLOBAL_GAIN,
 				   data, 2);
@@ -490,7 +491,7 @@ static int mt9m111_set_green_balance(struct gspca_dev *gspca_dev, __s32 val)
 	data[1] = (val & 0xff);
 	data[0] = (val & 0xff00) >> 8;
 
-	PDEBUG(D_CONF, "Set green balance %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set green balance %d\n", val);
 	err = m5602_write_sensor(sd, MT9M111_SC_GREEN_1_GAIN,
 				 data, 2);
 	if (err < 0)
@@ -508,7 +509,7 @@ static int mt9m111_set_blue_balance(struct gspca_dev *gspca_dev, __s32 val)
 	data[1] = (val & 0xff);
 	data[0] = (val & 0xff00) >> 8;
 
-	PDEBUG(D_CONF, "Set blue balance %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set blue balance %d\n", val);
 
 	return m5602_write_sensor(sd, MT9M111_SC_BLUE_GAIN,
 				  data, 2);
@@ -522,7 +523,7 @@ static int mt9m111_set_red_balance(struct gspca_dev *gspca_dev, __s32 val)
 	data[1] = (val & 0xff);
 	data[0] = (val & 0xff00) >> 8;
 
-	PDEBUG(D_CONF, "Set red balance %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set red balance %d\n", val);
 
 	return m5602_write_sensor(sd, MT9M111_SC_RED_GAIN,
 				  data, 2);
diff --git a/drivers/media/usb/gspca/m5602/m5602_ov7660.c b/drivers/media/usb/gspca/m5602/m5602_ov7660.c
index 672b7a520695..aa1f569c82fd 100644
--- a/drivers/media/usb/gspca/m5602/m5602_ov7660.c
+++ b/drivers/media/usb/gspca/m5602/m5602_ov7660.c
@@ -330,7 +330,7 @@ static int ov7660_set_gain(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data = val;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Setting gain to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Setting gain to %d\n", val);
 
 	err = m5602_write_sensor(sd, OV7660_GAIN, &i2c_data, 1);
 	return err;
@@ -343,7 +343,7 @@ static int ov7660_set_auto_white_balance(struct gspca_dev *gspca_dev,
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set auto white balance to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto white balance to %d\n", val);
 
 	err = m5602_read_sensor(sd, OV7660_COM8, &i2c_data, 1);
 	if (err < 0)
@@ -361,7 +361,7 @@ static int ov7660_set_auto_gain(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set auto gain control to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto gain control to %d\n", val);
 
 	err = m5602_read_sensor(sd, OV7660_COM8, &i2c_data, 1);
 	if (err < 0)
@@ -379,7 +379,7 @@ static int ov7660_set_auto_exposure(struct gspca_dev *gspca_dev,
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set auto exposure control to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto exposure control to %d\n", val);
 
 	err = m5602_read_sensor(sd, OV7660_COM8, &i2c_data, 1);
 	if (err < 0)
@@ -397,7 +397,8 @@ static int ov7660_set_hvflip(struct gspca_dev *gspca_dev)
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set hvflip to %d, %d", sd->hflip->val, sd->vflip->val);
+	gspca_dbg(gspca_dev, D_CONF, "Set hvflip to %d, %d\n",
+		  sd->hflip->val, sd->vflip->val);
 
 	i2c_data = (sd->hflip->val << 5) | (sd->vflip->val << 4);
 
diff --git a/drivers/media/usb/gspca/m5602/m5602_ov9650.c b/drivers/media/usb/gspca/m5602/m5602_ov9650.c
index 4544d3a1ad58..2ffbb54e89f9 100644
--- a/drivers/media/usb/gspca/m5602/m5602_ov9650.c
+++ b/drivers/media/usb/gspca/m5602/m5602_ov9650.c
@@ -311,7 +311,7 @@ int ov9650_probe(struct sd *sd)
 		return -ENODEV;
 	}
 
-	PDEBUG(D_PROBE, "Probing for an ov9650 sensor");
+	gspca_dbg(gspca_dev, D_PROBE, "Probing for an ov9650 sensor\n");
 
 	/* Run the pre-init before probing the sensor */
 	for (i = 0; i < ARRAY_SIZE(preinit_ov9650) && !err; i++) {
@@ -505,7 +505,7 @@ int ov9650_start(struct sd *sd)
 
 	switch (width) {
 	case 640:
-		PDEBUG(D_CONF, "Configuring camera for VGA mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for VGA mode\n");
 
 		data = OV9650_VGA_SELECT | OV9650_RGB_SELECT |
 		       OV9650_RAW_RGB_SELECT;
@@ -513,7 +513,7 @@ int ov9650_start(struct sd *sd)
 		break;
 
 	case 352:
-		PDEBUG(D_CONF, "Configuring camera for CIF mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for CIF mode\n");
 
 		data = OV9650_CIF_SELECT | OV9650_RGB_SELECT |
 				OV9650_RAW_RGB_SELECT;
@@ -521,7 +521,7 @@ int ov9650_start(struct sd *sd)
 		break;
 
 	case 320:
-		PDEBUG(D_CONF, "Configuring camera for QVGA mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for QVGA mode\n");
 
 		data = OV9650_QVGA_SELECT | OV9650_RGB_SELECT |
 				OV9650_RAW_RGB_SELECT;
@@ -529,7 +529,7 @@ int ov9650_start(struct sd *sd)
 		break;
 
 	case 176:
-		PDEBUG(D_CONF, "Configuring camera for QCIF mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for QCIF mode\n");
 
 		data = OV9650_QCIF_SELECT | OV9650_RGB_SELECT |
 			OV9650_RAW_RGB_SELECT;
@@ -558,7 +558,7 @@ static int ov9650_set_exposure(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data;
 	int err;
 
-	PDEBUG(D_CONF, "Set exposure to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set exposure to %d\n", val);
 
 	/* The 6 MSBs */
 	i2c_data = (val >> 10) & 0x3f;
@@ -586,7 +586,7 @@ static int ov9650_set_gain(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Setting gain to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Setting gain to %d\n", val);
 
 	/* The 2 MSB */
 	/* Read the OV9650_VREF register first to avoid
@@ -614,7 +614,7 @@ static int ov9650_set_red_balance(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set red gain to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set red gain to %d\n", val);
 
 	i2c_data = val & 0xff;
 	err = m5602_write_sensor(sd, OV9650_RED, &i2c_data, 1);
@@ -627,7 +627,7 @@ static int ov9650_set_blue_balance(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set blue gain to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set blue gain to %d\n", val);
 
 	i2c_data = val & 0xff;
 	err = m5602_write_sensor(sd, OV9650_BLUE, &i2c_data, 1);
@@ -642,7 +642,7 @@ static int ov9650_set_hvflip(struct gspca_dev *gspca_dev)
 	int hflip = sd->hflip->val;
 	int vflip = sd->vflip->val;
 
-	PDEBUG(D_CONF, "Set hvflip to %d %d", hflip, vflip);
+	gspca_dbg(gspca_dev, D_CONF, "Set hvflip to %d %d\n", hflip, vflip);
 
 	if (dmi_check_system(ov9650_flip_dmi_table))
 		vflip = !vflip;
@@ -666,7 +666,7 @@ static int ov9650_set_auto_exposure(struct gspca_dev *gspca_dev,
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set auto exposure control to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto exposure control to %d\n", val);
 
 	err = m5602_read_sensor(sd, OV9650_COM8, &i2c_data, 1);
 	if (err < 0)
@@ -685,7 +685,7 @@ static int ov9650_set_auto_white_balance(struct gspca_dev *gspca_dev,
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set auto white balance to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto white balance to %d\n", val);
 
 	err = m5602_read_sensor(sd, OV9650_COM8, &i2c_data, 1);
 	if (err < 0)
@@ -703,7 +703,7 @@ static int ov9650_set_auto_gain(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set auto gain control to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto gain control to %d\n", val);
 
 	err = m5602_read_sensor(sd, OV9650_COM8, &i2c_data, 1);
 	if (err < 0)
diff --git a/drivers/media/usb/gspca/m5602/m5602_po1030.c b/drivers/media/usb/gspca/m5602/m5602_po1030.c
index a0a90dd34ca8..37d2891e5f5b 100644
--- a/drivers/media/usb/gspca/m5602/m5602_po1030.c
+++ b/drivers/media/usb/gspca/m5602/m5602_po1030.c
@@ -171,7 +171,7 @@ int po1030_probe(struct sd *sd)
 		return -ENODEV;
 	}
 
-	PDEBUG(D_PROBE, "Probing for a po1030 sensor");
+	gspca_dbg(gspca_dev, D_PROBE, "Probing for a po1030 sensor\n");
 
 	/* Run the pre-init to actually probe the unit */
 	for (i = 0; i < ARRAY_SIZE(preinit_po1030); i++) {
@@ -410,11 +410,11 @@ static int po1030_set_exposure(struct gspca_dev *gspca_dev, __s32 val)
 	u8 i2c_data;
 	int err;
 
-	PDEBUG(D_CONF, "Set exposure to %d", val & 0xffff);
+	gspca_dbg(gspca_dev, D_CONF, "Set exposure to %d\n", val & 0xffff);
 
 	i2c_data = ((val & 0xff00) >> 8);
-	PDEBUG(D_CONF, "Set exposure to high byte to 0x%x",
-	       i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Set exposure to high byte to 0x%x\n",
+		  i2c_data);
 
 	err = m5602_write_sensor(sd, PO1030_INTEGLINES_H,
 				  &i2c_data, 1);
@@ -422,8 +422,8 @@ static int po1030_set_exposure(struct gspca_dev *gspca_dev, __s32 val)
 		return err;
 
 	i2c_data = (val & 0xff);
-	PDEBUG(D_CONF, "Set exposure to low byte to 0x%x",
-	       i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Set exposure to low byte to 0x%x\n",
+		  i2c_data);
 	err = m5602_write_sensor(sd, PO1030_INTEGLINES_M,
 				  &i2c_data, 1);
 
@@ -437,7 +437,7 @@ static int po1030_set_gain(struct gspca_dev *gspca_dev, __s32 val)
 	int err;
 
 	i2c_data = val & 0xff;
-	PDEBUG(D_CONF, "Set global gain to %d", i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Set global gain to %d\n", i2c_data);
 	err = m5602_write_sensor(sd, PO1030_GLOBALGAIN,
 				 &i2c_data, 1);
 	return err;
@@ -449,7 +449,8 @@ static int po1030_set_hvflip(struct gspca_dev *gspca_dev)
 	u8 i2c_data;
 	int err;
 
-	PDEBUG(D_CONF, "Set hvflip %d %d", sd->hflip->val, sd->vflip->val);
+	gspca_dbg(gspca_dev, D_CONF, "Set hvflip %d %d\n",
+		  sd->hflip->val, sd->vflip->val);
 	err = m5602_read_sensor(sd, PO1030_CONTROL2, &i2c_data, 1);
 	if (err < 0)
 		return err;
@@ -470,7 +471,7 @@ static int po1030_set_red_balance(struct gspca_dev *gspca_dev, __s32 val)
 	int err;
 
 	i2c_data = val & 0xff;
-	PDEBUG(D_CONF, "Set red gain to %d", i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Set red gain to %d\n", i2c_data);
 	err = m5602_write_sensor(sd, PO1030_RED_GAIN,
 				  &i2c_data, 1);
 	return err;
@@ -483,7 +484,7 @@ static int po1030_set_blue_balance(struct gspca_dev *gspca_dev, __s32 val)
 	int err;
 
 	i2c_data = val & 0xff;
-	PDEBUG(D_CONF, "Set blue gain to %d", i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Set blue gain to %d\n", i2c_data);
 	err = m5602_write_sensor(sd, PO1030_BLUE_GAIN,
 				  &i2c_data, 1);
 
@@ -497,7 +498,7 @@ static int po1030_set_green_balance(struct gspca_dev *gspca_dev, __s32 val)
 	int err;
 
 	i2c_data = val & 0xff;
-	PDEBUG(D_CONF, "Set green gain to %d", i2c_data);
+	gspca_dbg(gspca_dev, D_CONF, "Set green gain to %d\n", i2c_data);
 
 	err = m5602_write_sensor(sd, PO1030_GREEN_1_GAIN,
 			   &i2c_data, 1);
@@ -519,7 +520,7 @@ static int po1030_set_auto_white_balance(struct gspca_dev *gspca_dev,
 	if (err < 0)
 		return err;
 
-	PDEBUG(D_CONF, "Set auto white balance to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto white balance to %d\n", val);
 	i2c_data = (i2c_data & 0xfe) | (val & 0x01);
 	err = m5602_write_sensor(sd, PO1030_AUTOCTRL1, &i2c_data, 1);
 	return err;
@@ -536,7 +537,7 @@ static int po1030_set_auto_exposure(struct gspca_dev *gspca_dev,
 	if (err < 0)
 		return err;
 
-	PDEBUG(D_CONF, "Set auto exposure to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set auto exposure to %d\n", val);
 	val = (val == V4L2_EXPOSURE_AUTO);
 	i2c_data = (i2c_data & 0xfd) | ((val & 0x01) << 1);
 	return m5602_write_sensor(sd, PO1030_AUTOCTRL1, &i2c_data, 1);
diff --git a/drivers/media/usb/gspca/m5602/m5602_s5k4aa.c b/drivers/media/usb/gspca/m5602/m5602_s5k4aa.c
index 8447b9c5f8e0..cec4a5838aec 100644
--- a/drivers/media/usb/gspca/m5602/m5602_s5k4aa.c
+++ b/drivers/media/usb/gspca/m5602/m5602_s5k4aa.c
@@ -357,7 +357,7 @@ int s5k4aa_probe(struct sd *sd)
 		return -ENODEV;
 	}
 
-	PDEBUG(D_PROBE, "Probing for a s5k4aa sensor");
+	gspca_dbg(gspca_dev, D_PROBE, "Probing for a s5k4aa sensor\n");
 
 	/* Preinit the sensor */
 	for (i = 0; i < ARRAY_SIZE(preinit_s5k4aa) && !err; i++) {
@@ -419,7 +419,7 @@ int s5k4aa_start(struct sd *sd)
 
 	switch (cam->cam_mode[sd->gspca_dev.curr_mode].width) {
 	case 1280:
-		PDEBUG(D_CONF, "Configuring camera for SXGA mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for SXGA mode\n");
 
 		for (i = 0; i < ARRAY_SIZE(SXGA_s5k4aa); i++) {
 			switch (SXGA_s5k4aa[i][0]) {
@@ -452,7 +452,7 @@ int s5k4aa_start(struct sd *sd)
 		break;
 
 	case 640:
-		PDEBUG(D_CONF, "Configuring camera for VGA mode");
+		gspca_dbg(gspca_dev, D_CONF, "Configuring camera for VGA mode\n");
 
 		for (i = 0; i < ARRAY_SIZE(VGA_s5k4aa); i++) {
 			switch (VGA_s5k4aa[i][0]) {
@@ -568,7 +568,7 @@ static int s5k4aa_set_exposure(struct gspca_dev *gspca_dev, __s32 val)
 	u8 data = S5K4AA_PAGE_MAP_2;
 	int err;
 
-	PDEBUG(D_CONF, "Set exposure to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set exposure to %d\n", val);
 	err = m5602_write_sensor(sd, S5K4AA_PAGE_MAP, &data, 1);
 	if (err < 0)
 		return err;
@@ -590,7 +590,7 @@ static int s5k4aa_set_hvflip(struct gspca_dev *gspca_dev)
 	int hflip = sd->hflip->val;
 	int vflip = sd->vflip->val;
 
-	PDEBUG(D_CONF, "Set hvflip %d %d", hflip, vflip);
+	gspca_dbg(gspca_dev, D_CONF, "Set hvflip %d %d\n", hflip, vflip);
 	err = m5602_write_sensor(sd, S5K4AA_PAGE_MAP, &data, 1);
 	if (err < 0)
 		return err;
@@ -640,7 +640,7 @@ static int s5k4aa_set_gain(struct gspca_dev *gspca_dev, __s32 val)
 	u8 data = S5K4AA_PAGE_MAP_2;
 	int err;
 
-	PDEBUG(D_CONF, "Set gain to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set gain to %d\n", val);
 	err = m5602_write_sensor(sd, S5K4AA_PAGE_MAP, &data, 1);
 	if (err < 0)
 		return err;
@@ -657,7 +657,7 @@ static int s5k4aa_set_brightness(struct gspca_dev *gspca_dev, __s32 val)
 	u8 data = S5K4AA_PAGE_MAP_2;
 	int err;
 
-	PDEBUG(D_CONF, "Set brightness to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set brightness to %d\n", val);
 	err = m5602_write_sensor(sd, S5K4AA_PAGE_MAP, &data, 1);
 	if (err < 0)
 		return err;
@@ -672,7 +672,7 @@ static int s5k4aa_set_noise(struct gspca_dev *gspca_dev, __s32 val)
 	u8 data = S5K4AA_PAGE_MAP_2;
 	int err;
 
-	PDEBUG(D_CONF, "Set noise to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set noise to %d\n", val);
 	err = m5602_write_sensor(sd, S5K4AA_PAGE_MAP, &data, 1);
 	if (err < 0)
 		return err;
diff --git a/drivers/media/usb/gspca/m5602/m5602_s5k83a.c b/drivers/media/usb/gspca/m5602/m5602_s5k83a.c
index 6ad8d4849680..3d8ab18138aa 100644
--- a/drivers/media/usb/gspca/m5602/m5602_s5k83a.c
+++ b/drivers/media/usb/gspca/m5602/m5602_s5k83a.c
@@ -187,7 +187,7 @@ int s5k83a_probe(struct sd *sd)
 		return -ENODEV;
 	}
 
-	PDEBUG(D_PROBE, "Probing for a s5k83a sensor");
+	gspca_dbg(gspca_dev, D_PROBE, "Probing for a s5k83a sensor\n");
 
 	/* Preinit the sensor */
 	for (i = 0; i < ARRAY_SIZE(preinit_s5k83a) && !err; i++) {
diff --git a/drivers/media/usb/gspca/mars.c b/drivers/media/usb/gspca/mars.c
index 25df55e840c7..a537cb195c46 100644
--- a/drivers/media/usb/gspca/mars.c
+++ b/drivers/media/usb/gspca/mars.c
@@ -378,8 +378,8 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 			    || data[5 + p] == 0x65
 			    || data[5 + p] == 0x66
 			    || data[5 + p] == 0x67) {
-				PDEBUG(D_PACK, "sof offset: %d len: %d",
-					p, len);
+				gspca_dbg(gspca_dev, D_PACK, "sof offset: %d len: %d\n",
+					  p, len);
 				gspca_frame_add(gspca_dev, LAST_PACKET,
 						data, p);
 
diff --git a/drivers/media/usb/gspca/mr97310a.c b/drivers/media/usb/gspca/mr97310a.c
index 8b0e32a649ac..bea196361215 100644
--- a/drivers/media/usb/gspca/mr97310a.c
+++ b/drivers/media/usb/gspca/mr97310a.c
@@ -221,10 +221,11 @@ static int cam_get_response16(struct gspca_dev *gspca_dev, u8 reg, int verbose)
 		return err_code;
 
 	if (verbose)
-		PDEBUG(D_PROBE, "Register: %02x reads %02x%02x%02x", reg,
-		       gspca_dev->usb_buf[0],
-		       gspca_dev->usb_buf[1],
-		       gspca_dev->usb_buf[2]);
+		gspca_dbg(gspca_dev, D_PROBE, "Register: %02x reads %02x%02x%02x\n",
+			  reg,
+			  gspca_dev->usb_buf[0],
+			  gspca_dev->usb_buf[1],
+			  gspca_dev->usb_buf[2]);
 
 	return 0;
 }
@@ -284,7 +285,7 @@ static int zero_the_pointer(struct gspca_dev *gspca_dev)
 			return err_code;
 	}
 	if (status != 0x0a)
-		PERR("status is %02x", status);
+		gspca_err(gspca_dev, "status is %02x\n", status);
 
 	tries = 0;
 	while (tries < 4) {
@@ -325,7 +326,7 @@ static void stream_stop(struct gspca_dev *gspca_dev)
 	gspca_dev->usb_buf[0] = 0x01;
 	gspca_dev->usb_buf[1] = 0x00;
 	if (mr_write(gspca_dev, 2) < 0)
-		PERR("Stream Stop failed");
+		gspca_err(gspca_dev, "Stream Stop failed\n");
 }
 
 static void lcd_stop(struct gspca_dev *gspca_dev)
@@ -333,7 +334,7 @@ static void lcd_stop(struct gspca_dev *gspca_dev)
 	gspca_dev->usb_buf[0] = 0x19;
 	gspca_dev->usb_buf[1] = 0x54;
 	if (mr_write(gspca_dev, 2) < 0)
-		PERR("LCD Stop failed");
+		gspca_err(gspca_dev, "LCD Stop failed\n");
 }
 
 static int isoc_enable(struct gspca_dev *gspca_dev)
@@ -413,8 +414,8 @@ static int sd_config(struct gspca_dev *gspca_dev,
 			       gspca_dev->usb_buf[1]);
 			return -ENODEV;
 		}
-		PDEBUG(D_PROBE, "MR97310A CIF camera detected, sensor: %d",
-		       sd->sensor_type);
+		gspca_dbg(gspca_dev, D_PROBE, "MR97310A CIF camera detected, sensor: %d\n",
+			  sd->sensor_type);
 	} else {
 		sd->cam_type = CAM_TYPE_VGA;
 
@@ -458,7 +459,7 @@ static int sd_config(struct gspca_dev *gspca_dev,
 			switch (gspca_dev->usb_buf[1]) {
 			case 0x50:
 				sd->sensor_type = 0;
-				PDEBUG(D_PROBE, "sensor_type corrected to 0");
+				gspca_dbg(gspca_dev, D_PROBE, "sensor_type corrected to 0\n");
 				break;
 			case 0x20:
 				/* Nothing to do here. */
@@ -470,16 +471,16 @@ static int sd_config(struct gspca_dev *gspca_dev,
 				pr_err("Please report this\n");
 			}
 		}
-		PDEBUG(D_PROBE, "MR97310A VGA camera detected, sensor: %d",
-		       sd->sensor_type);
+		gspca_dbg(gspca_dev, D_PROBE, "MR97310A VGA camera detected, sensor: %d\n",
+			  sd->sensor_type);
 	}
 	/* Stop streaming as we've started it only to probe the sensor type. */
 	sd_stopN(gspca_dev);
 
 	if (force_sensor_type != -1) {
 		sd->sensor_type = !!force_sensor_type;
-		PDEBUG(D_PROBE, "Forcing sensor type to: %d",
-		       sd->sensor_type);
+		gspca_dbg(gspca_dev, D_PROBE, "Forcing sensor type to: %d\n",
+			  sd->sensor_type);
 	}
 
 	return 0;
diff --git a/drivers/media/usb/gspca/nw80x.c b/drivers/media/usb/gspca/nw80x.c
index 5d2d0bcb038d..bedc04a72e97 100644
--- a/drivers/media/usb/gspca/nw80x.c
+++ b/drivers/media/usb/gspca/nw80x.c
@@ -1543,10 +1543,11 @@ static void reg_w(struct gspca_dev *gspca_dev,
 	if (gspca_dev->usb_err < 0)
 		return;
 	if (len == 1)
-		PDEBUG(D_USBO, "SET 00 0000 %04x %02x", index, *data);
+		gspca_dbg(gspca_dev, D_USBO, "SET 00 0000 %04x %02x\n",
+			  index, *data);
 	else
-		PDEBUG(D_USBO, "SET 00 0000 %04x %02x %02x ...",
-				index, *data, data[1]);
+		gspca_dbg(gspca_dev, D_USBO, "SET 00 0000 %04x %02x %02x ...\n",
+			  index, *data, data[1]);
 	memcpy(gspca_dev->usb_buf, data, len);
 	ret = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
 			0x00,
@@ -1583,12 +1584,12 @@ static void reg_r(struct gspca_dev *gspca_dev,
 		return;
 	}
 	if (len == 1)
-		PDEBUG(D_USBI, "GET 00 0000 %04x %02x",
-				index, gspca_dev->usb_buf[0]);
+		gspca_dbg(gspca_dev, D_USBI, "GET 00 0000 %04x %02x\n",
+			  index, gspca_dev->usb_buf[0]);
 	else
-		PDEBUG(D_USBI, "GET 00 0000 %04x %02x %02x ..",
-				index, gspca_dev->usb_buf[0],
-				gspca_dev->usb_buf[1]);
+		gspca_dbg(gspca_dev, D_USBI, "GET 00 0000 %04x %02x %02x ..\n",
+			  index, gspca_dev->usb_buf[0],
+			  gspca_dev->usb_buf[1]);
 }
 
 static void i2c_w(struct gspca_dev *gspca_dev,
@@ -1762,8 +1763,8 @@ static int sd_config(struct gspca_dev *gspca_dev,
 			sd->webcam = P35u;
 	} else if (id->idVendor == 0x06a5 && id->idProduct == 0xd800) {
 		reg_r(gspca_dev, 0x0403, 1);		/* GPIO */
-		PDEBUG(D_PROBE, "et31x110 sensor type %02x",
-				gspca_dev->usb_buf[0]);
+		gspca_dbg(gspca_dev, D_PROBE, "et31x110 sensor type %02x\n",
+			  gspca_dev->usb_buf[0]);
 		switch (gspca_dev->usb_buf[0] >> 1) {
 		case 0x00:				/* ?? */
 			if (sd->webcam == Generic800)
@@ -1785,7 +1786,8 @@ static int sd_config(struct gspca_dev *gspca_dev,
 		gspca_dev->usb_err = -ENODEV;
 		return gspca_dev->usb_err;
 	}
-	PDEBUG(D_PROBE, "Bridge nw80%d - type: %d", sd->bridge, sd->webcam);
+	gspca_dbg(gspca_dev, D_PROBE, "Bridge nw80%d - type: %d\n",
+		  sd->bridge, sd->webcam);
 
 	if (sd->bridge == BRIDGE_NW800) {
 		switch (sd->webcam) {
diff --git a/drivers/media/usb/gspca/ov519.c b/drivers/media/usb/gspca/ov519.c
index 1b30434b72ef..10fcbe9e8614 100644
--- a/drivers/media/usb/gspca/ov519.c
+++ b/drivers/media/usb/gspca/ov519.c
@@ -2016,8 +2016,8 @@ static void reg_w(struct sd *sd, u16 index, u16 value)
 		req = 0x0a;
 		/* fall through */
 	case BRIDGE_W9968CF:
-		PDEBUG(D_USBO, "SET %02x %04x %04x",
-				req, value, index);
+		gspca_dbg(gspca_dev, D_USBO, "SET %02x %04x %04x\n",
+			  req, value, index);
 		ret = usb_control_msg(sd->gspca_dev.dev,
 			usb_sndctrlpipe(sd->gspca_dev.dev, 0),
 			req,
@@ -2028,8 +2028,8 @@ static void reg_w(struct sd *sd, u16 index, u16 value)
 		req = 1;
 	}
 
-	PDEBUG(D_USBO, "SET %02x 0000 %04x %02x",
-			req, index, value);
+	gspca_dbg(gspca_dev, D_USBO, "SET %02x 0000 %04x %02x\n",
+		  req, index, value);
 	sd->gspca_dev.usb_buf[0] = value;
 	ret = usb_control_msg(sd->gspca_dev.dev,
 			usb_sndctrlpipe(sd->gspca_dev.dev, 0),
@@ -2039,7 +2039,7 @@ static void reg_w(struct sd *sd, u16 index, u16 value)
 			sd->gspca_dev.usb_buf, 1, 500);
 leave:
 	if (ret < 0) {
-		PERR("reg_w %02x failed %d\n", index, ret);
+		gspca_err(gspca_dev, "reg_w %02x failed %d\n", index, ret);
 		sd->gspca_dev.usb_err = ret;
 		return;
 	}
@@ -2078,10 +2078,10 @@ static int reg_r(struct sd *sd, u16 index)
 
 	if (ret >= 0) {
 		ret = sd->gspca_dev.usb_buf[0];
-		PDEBUG(D_USBI, "GET %02x 0000 %04x %02x",
-			req, index, ret);
+		gspca_dbg(gspca_dev, D_USBI, "GET %02x 0000 %04x %02x\n",
+			  req, index, ret);
 	} else {
-		PERR("reg_r %02x failed %d\n", index, ret);
+		gspca_err(gspca_dev, "reg_r %02x failed %d\n", index, ret);
 		sd->gspca_dev.usb_err = ret;
 	}
 
@@ -2109,7 +2109,7 @@ static int reg_r8(struct sd *sd,
 	if (ret >= 0) {
 		ret = sd->gspca_dev.usb_buf[0];
 	} else {
-		PERR("reg_r8 %02x failed %d\n", index, ret);
+		gspca_err(gspca_dev, "reg_r8 %02x failed %d\n", index, ret);
 		sd->gspca_dev.usb_err = ret;
 	}
 
@@ -2165,7 +2165,7 @@ static void ov518_reg_w32(struct sd *sd, u16 index, u32 value, int n)
 			0, index,
 			sd->gspca_dev.usb_buf, n, 500);
 	if (ret < 0) {
-		PERR("reg_w32 %02x failed %d\n", index, ret);
+		gspca_err(gspca_dev, "reg_w32 %02x failed %d\n", index, ret);
 		sd->gspca_dev.usb_err = ret;
 	}
 }
@@ -2175,7 +2175,7 @@ static void ov511_i2c_w(struct sd *sd, u8 reg, u8 value)
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 	int rc, retries;
 
-	PDEBUG(D_USBO, "ov511_i2c_w %02x %02x", reg, value);
+	gspca_dbg(gspca_dev, D_USBO, "ov511_i2c_w %02x %02x\n", reg, value);
 
 	/* Three byte write cycle */
 	for (retries = 6; ; ) {
@@ -2198,7 +2198,7 @@ static void ov511_i2c_w(struct sd *sd, u8 reg, u8 value)
 		if ((rc & 2) == 0) /* Ack? */
 			break;
 		if (--retries < 0) {
-			PDEBUG(D_USBO, "i2c write retries exhausted");
+			gspca_dbg(gspca_dev, D_USBO, "i2c write retries exhausted\n");
 			return;
 		}
 	}
@@ -2231,7 +2231,7 @@ static int ov511_i2c_r(struct sd *sd, u8 reg)
 		reg_w(sd, R511_I2C_CTL, 0x10);
 
 		if (--retries < 0) {
-			PDEBUG(D_USBI, "i2c write retries exhausted");
+			gspca_dbg(gspca_dev, D_USBI, "i2c write retries exhausted\n");
 			return -1;
 		}
 	}
@@ -2255,14 +2255,14 @@ static int ov511_i2c_r(struct sd *sd, u8 reg)
 		reg_w(sd, R511_I2C_CTL, 0x10);
 
 		if (--retries < 0) {
-			PDEBUG(D_USBI, "i2c read retries exhausted");
+			gspca_dbg(gspca_dev, D_USBI, "i2c read retries exhausted\n");
 			return -1;
 		}
 	}
 
 	value = reg_r(sd, R51x_I2C_DATA);
 
-	PDEBUG(D_USBI, "ov511_i2c_r %02x %02x", reg, value);
+	gspca_dbg(gspca_dev, D_USBI, "ov511_i2c_r %02x %02x\n", reg, value);
 
 	/* This is needed to make i2c_w() work */
 	reg_w(sd, R511_I2C_CTL, 0x05);
@@ -2281,7 +2281,7 @@ static void ov518_i2c_w(struct sd *sd,
 {
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 
-	PDEBUG(D_USBO, "ov518_i2c_w %02x %02x", reg, value);
+	gspca_dbg(gspca_dev, D_USBO, "ov518_i2c_w %02x %02x\n", reg, value);
 
 	/* Select camera register */
 	reg_w(sd, R51x_I2C_SADDR_3, reg);
@@ -2321,7 +2321,7 @@ static int ov518_i2c_r(struct sd *sd, u8 reg)
 	reg_r8(sd, R518_I2C_CTL);
 
 	value = reg_r(sd, R51x_I2C_DATA);
-	PDEBUG(D_USBI, "ov518_i2c_r %02x %02x", reg, value);
+	gspca_dbg(gspca_dev, D_USBI, "ov518_i2c_r %02x %02x\n", reg, value);
 	return value;
 }
 
@@ -2340,11 +2340,11 @@ static void ovfx2_i2c_w(struct sd *sd, u8 reg, u8 value)
 			(u16) value, (u16) reg, NULL, 0, 500);
 
 	if (ret < 0) {
-		PERR("ovfx2_i2c_w %02x failed %d\n", reg, ret);
+		gspca_err(gspca_dev, "ovfx2_i2c_w %02x failed %d\n", reg, ret);
 		sd->gspca_dev.usb_err = ret;
 	}
 
-	PDEBUG(D_USBO, "ovfx2_i2c_w %02x %02x", reg, value);
+	gspca_dbg(gspca_dev, D_USBO, "ovfx2_i2c_w %02x %02x\n", reg, value);
 }
 
 static int ovfx2_i2c_r(struct sd *sd, u8 reg)
@@ -2363,9 +2363,10 @@ static int ovfx2_i2c_r(struct sd *sd, u8 reg)
 
 	if (ret >= 0) {
 		ret = sd->gspca_dev.usb_buf[0];
-		PDEBUG(D_USBI, "ovfx2_i2c_r %02x %02x", reg, ret);
+		gspca_dbg(gspca_dev, D_USBI, "ovfx2_i2c_r %02x %02x\n",
+			  reg, ret);
 	} else {
-		PERR("ovfx2_i2c_r %02x failed %d\n", reg, ret);
+		gspca_err(gspca_dev, "ovfx2_i2c_r %02x failed %d\n", reg, ret);
 		sd->gspca_dev.usb_err = ret;
 	}
 
@@ -2464,7 +2465,7 @@ static inline void ov51x_stop(struct sd *sd)
 {
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 
-	PDEBUG(D_STREAM, "stopping");
+	gspca_dbg(gspca_dev, D_STREAM, "stopping\n");
 	sd->stopped = 1;
 	switch (sd->bridge) {
 	case BRIDGE_OV511:
@@ -2495,7 +2496,7 @@ static inline void ov51x_restart(struct sd *sd)
 {
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 
-	PDEBUG(D_STREAM, "restarting");
+	gspca_dbg(gspca_dev, D_STREAM, "restarting\n");
 	if (!sd->stopped)
 		return;
 	sd->stopped = 0;
@@ -2546,7 +2547,8 @@ static int init_ov_sensor(struct sd *sd, u8 slave)
 	for (i = 0; i < i2c_detect_tries; i++) {
 		if (i2c_r(sd, OV7610_REG_ID_HIGH) == 0x7f &&
 		    i2c_r(sd, OV7610_REG_ID_LOW) == 0xa2) {
-			PDEBUG(D_PROBE, "I2C synced in %d attempt(s)", i);
+			gspca_dbg(gspca_dev, D_PROBE, "I2C synced in %d attempt(s)\n",
+				  i);
 			return 0;
 		}
 
@@ -2617,11 +2619,11 @@ static void ov_hires_configure(struct sd *sd)
 	int high, low;
 
 	if (sd->bridge != BRIDGE_OVFX2) {
-		PERR("error hires sensors only supported with ovfx2\n");
+		gspca_err(gspca_dev, "error hires sensors only supported with ovfx2\n");
 		return;
 	}
 
-	PDEBUG(D_PROBE, "starting ov hires configuration");
+	gspca_dbg(gspca_dev, D_PROBE, "starting ov hires configuration\n");
 
 	/* Detect sensor (sub)type */
 	high = i2c_r(sd, 0x0a);
@@ -2631,28 +2633,29 @@ static void ov_hires_configure(struct sd *sd)
 	case 0x96:
 		switch (low) {
 		case 0x40:
-			PDEBUG(D_PROBE, "Sensor is a OV2610");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is a OV2610\n");
 			sd->sensor = SEN_OV2610;
 			return;
 		case 0x41:
-			PDEBUG(D_PROBE, "Sensor is a OV2610AE");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is a OV2610AE\n");
 			sd->sensor = SEN_OV2610AE;
 			return;
 		case 0xb1:
-			PDEBUG(D_PROBE, "Sensor is a OV9600");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is a OV9600\n");
 			sd->sensor = SEN_OV9600;
 			return;
 		}
 		break;
 	case 0x36:
 		if ((low & 0x0f) == 0x00) {
-			PDEBUG(D_PROBE, "Sensor is a OV3610");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is a OV3610\n");
 			sd->sensor = SEN_OV3610;
 			return;
 		}
 		break;
 	}
-	PERR("Error unknown sensor type: %02x%02x\n", high, low);
+	gspca_err(gspca_dev, "Error unknown sensor type: %02x%02x\n",
+		  high, low);
 }
 
 /* This initializes the OV8110, OV8610 sensor. The OV8110 uses
@@ -2663,18 +2666,19 @@ static void ov8xx0_configure(struct sd *sd)
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 	int rc;
 
-	PDEBUG(D_PROBE, "starting ov8xx0 configuration");
+	gspca_dbg(gspca_dev, D_PROBE, "starting ov8xx0 configuration\n");
 
 	/* Detect sensor (sub)type */
 	rc = i2c_r(sd, OV7610_REG_COM_I);
 	if (rc < 0) {
-		PERR("Error detecting sensor type");
+		gspca_err(gspca_dev, "Error detecting sensor type\n");
 		return;
 	}
 	if ((rc & 3) == 1)
 		sd->sensor = SEN_OV8610;
 	else
-		PERR("Unknown image sensor version: %d\n", rc & 3);
+		gspca_err(gspca_dev, "Unknown image sensor version: %d\n",
+			  rc & 3);
 }
 
 /* This initializes the OV7610, OV7620, or OV76BE sensor. The OV76BE uses
@@ -2685,7 +2689,7 @@ static void ov7xx0_configure(struct sd *sd)
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 	int rc, high, low;
 
-	PDEBUG(D_PROBE, "starting OV7xx0 configuration");
+	gspca_dbg(gspca_dev, D_PROBE, "starting OV7xx0 configuration\n");
 
 	/* Detect sensor (sub)type */
 	rc = i2c_r(sd, OV7610_REG_COM_I);
@@ -2693,7 +2697,7 @@ static void ov7xx0_configure(struct sd *sd)
 	/* add OV7670 here
 	 * it appears to be wrongly detected as a 7610 by default */
 	if (rc < 0) {
-		PERR("Error detecting sensor type\n");
+		gspca_err(gspca_dev, "Error detecting sensor type\n");
 		return;
 	}
 	if ((rc & 3) == 3) {
@@ -2702,65 +2706,68 @@ static void ov7xx0_configure(struct sd *sd)
 		low = i2c_r(sd, 0x0b);
 		/* info("%x, %x", high, low); */
 		if (high == 0x76 && (low & 0xf0) == 0x70) {
-			PDEBUG(D_PROBE, "Sensor is an OV76%02x", low);
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV76%02x\n",
+				  low);
 			sd->sensor = SEN_OV7670;
 		} else {
-			PDEBUG(D_PROBE, "Sensor is an OV7610");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV7610\n");
 			sd->sensor = SEN_OV7610;
 		}
 	} else if ((rc & 3) == 1) {
 		/* I don't know what's different about the 76BE yet. */
 		if (i2c_r(sd, 0x15) & 1) {
-			PDEBUG(D_PROBE, "Sensor is an OV7620AE");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV7620AE\n");
 			sd->sensor = SEN_OV7620AE;
 		} else {
-			PDEBUG(D_PROBE, "Sensor is an OV76BE");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV76BE\n");
 			sd->sensor = SEN_OV76BE;
 		}
 	} else if ((rc & 3) == 0) {
 		/* try to read product id registers */
 		high = i2c_r(sd, 0x0a);
 		if (high < 0) {
-			PERR("Error detecting camera chip PID\n");
+			gspca_err(gspca_dev, "Error detecting camera chip PID\n");
 			return;
 		}
 		low = i2c_r(sd, 0x0b);
 		if (low < 0) {
-			PERR("Error detecting camera chip VER\n");
+			gspca_err(gspca_dev, "Error detecting camera chip VER\n");
 			return;
 		}
 		if (high == 0x76) {
 			switch (low) {
 			case 0x30:
-				PERR("Sensor is an OV7630/OV7635\n");
-				PERR("7630 is not supported by this driver\n");
+				gspca_err(gspca_dev, "Sensor is an OV7630/OV7635\n");
+				gspca_err(gspca_dev, "7630 is not supported by this driver\n");
 				return;
 			case 0x40:
-				PDEBUG(D_PROBE, "Sensor is an OV7645");
+				gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV7645\n");
 				sd->sensor = SEN_OV7640; /* FIXME */
 				break;
 			case 0x45:
-				PDEBUG(D_PROBE, "Sensor is an OV7645B");
+				gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV7645B\n");
 				sd->sensor = SEN_OV7640; /* FIXME */
 				break;
 			case 0x48:
-				PDEBUG(D_PROBE, "Sensor is an OV7648");
+				gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV7648\n");
 				sd->sensor = SEN_OV7648;
 				break;
 			case 0x60:
-				PDEBUG(D_PROBE, "Sensor is a OV7660");
+				gspca_dbg(gspca_dev, D_PROBE, "Sensor is a OV7660\n");
 				sd->sensor = SEN_OV7660;
 				break;
 			default:
-				PERR("Unknown sensor: 0x76%02x\n", low);
+				gspca_err(gspca_dev, "Unknown sensor: 0x76%02x\n",
+					  low);
 				return;
 			}
 		} else {
-			PDEBUG(D_PROBE, "Sensor is an OV7620");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV7620\n");
 			sd->sensor = SEN_OV7620;
 		}
 	} else {
-		PERR("Unknown image sensor version: %d\n", rc & 3);
+		gspca_err(gspca_dev, "Unknown image sensor version: %d\n",
+			  rc & 3);
 	}
 }
 
@@ -2770,12 +2777,12 @@ static void ov6xx0_configure(struct sd *sd)
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 	int rc;
 
-	PDEBUG(D_PROBE, "starting OV6xx0 configuration");
+	gspca_dbg(gspca_dev, D_PROBE, "starting OV6xx0 configuration\n");
 
 	/* Detect sensor (sub)type */
 	rc = i2c_r(sd, OV7610_REG_COM_I);
 	if (rc < 0) {
-		PERR("Error detecting sensor type\n");
+		gspca_err(gspca_dev, "Error detecting sensor type\n");
 		return;
 	}
 
@@ -2789,22 +2796,23 @@ static void ov6xx0_configure(struct sd *sd)
 		break;
 	case 0x01:
 		sd->sensor = SEN_OV6620;
-		PDEBUG(D_PROBE, "Sensor is an OV6620");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV6620\n");
 		break;
 	case 0x02:
 		sd->sensor = SEN_OV6630;
-		PDEBUG(D_PROBE, "Sensor is an OV66308AE");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV66308AE\n");
 		break;
 	case 0x03:
 		sd->sensor = SEN_OV66308AF;
-		PDEBUG(D_PROBE, "Sensor is an OV66308AF");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor is an OV66308AF\n");
 		break;
 	case 0x90:
 		sd->sensor = SEN_OV6630;
 		pr_warn("WARNING: Sensor is an OV66307. Your camera may have been misdetected in previous driver versions.\n");
 		break;
 	default:
-		PERR("FATAL: Unknown sensor version: 0x%02x\n", rc);
+		gspca_err(gspca_dev, "FATAL: Unknown sensor version: 0x%02x\n",
+			  rc);
 		return;
 	}
 
@@ -2906,7 +2914,7 @@ static void ov51x_upload_quan_tables(struct sd *sd)
 	unsigned char val0, val1;
 	int i, size, reg = R51x_COMP_LUT_BEGIN;
 
-	PDEBUG(D_PROBE, "Uploading quantization tables");
+	gspca_dbg(gspca_dev, D_PROBE, "Uploading quantization tables\n");
 
 	if (sd->bridge == BRIDGE_OV511 || sd->bridge == BRIDGE_OV511PLUS) {
 		pYTable = yQuanTable511;
@@ -2984,7 +2992,8 @@ static void ov511_configure(struct gspca_dev *gspca_dev)
 		{ 0x77, 0x04 },
 	};
 
-	PDEBUG(D_PROBE, "Device custom id %x", reg_r(sd, R51x_SYS_CUST_ID));
+	gspca_dbg(gspca_dev, D_PROBE, "Device custom id %x\n",
+		  reg_r(sd, R51x_SYS_CUST_ID));
 
 	write_regvals(sd, init_511, ARRAY_SIZE(init_511));
 
@@ -3054,7 +3063,7 @@ static void ov518_configure(struct gspca_dev *gspca_dev)
 
 	/* First 5 bits of custom ID reg are a revision ID on OV518 */
 	sd->revision = reg_r(sd, R51x_SYS_CUST_ID) & 0x1f;
-	PDEBUG(D_PROBE, "Device revision %d", sd->revision);
+	gspca_dbg(gspca_dev, D_PROBE, "Device revision %d\n", sd->revision);
 
 	write_regvals(sd, init_518, ARRAY_SIZE(init_518));
 
@@ -3295,7 +3304,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	} else if (init_ov_sensor(sd, OV_HIRES_SID) >= 0) {
 		ov_hires_configure(sd);
 	} else {
-		PERR("Can't determine sensor slave IDs\n");
+		gspca_err(gspca_dev, "Can't determine sensor slave IDs\n");
 		goto error;
 	}
 
@@ -3428,7 +3437,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	}
 	return gspca_dev->usb_err;
 error:
-	PERR("OV519 Config failed");
+	gspca_err(gspca_dev, "OV519 Config failed\n");
 	return -EINVAL;
 }
 
@@ -3463,7 +3472,7 @@ static void ov511_mode_init_regs(struct sd *sd)
 	intf = usb_ifnum_to_if(sd->gspca_dev.dev, sd->gspca_dev.iface);
 	alt = usb_altnum_to_altsetting(intf, sd->gspca_dev.alt);
 	if (!alt) {
-		PERR("Couldn't get altsetting\n");
+		gspca_err(gspca_dev, "Couldn't get altsetting\n");
 		sd->gspca_dev.usb_err = -EIO;
 		return;
 	}
@@ -3589,7 +3598,7 @@ static void ov518_mode_init_regs(struct sd *sd)
 	intf = usb_ifnum_to_if(sd->gspca_dev.dev, sd->gspca_dev.iface);
 	alt = usb_altnum_to_altsetting(intf, sd->gspca_dev.alt);
 	if (!alt) {
-		PERR("Couldn't get altsetting\n");
+		gspca_err(gspca_dev, "Couldn't get altsetting\n");
 		sd->gspca_dev.usb_err = -EIO;
 		return;
 	}
@@ -3861,8 +3870,8 @@ static void ov519_mode_init_regs(struct sd *sd)
 		}
 		break;
 	case SEN_OV7670:		/* guesses, based on 7640 */
-		PDEBUG(D_STREAM, "Setting framerate to %d fps",
-				 (sd->frame_rate == 0) ? 15 : sd->frame_rate);
+		gspca_dbg(gspca_dev, D_STREAM, "Setting framerate to %d fps\n",
+			  (sd->frame_rate == 0) ? 15 : sd->frame_rate);
 		reg_w(sd, 0xa4, 0x10);
 		switch (sd->frame_rate) {
 		case 30:
@@ -4323,10 +4332,10 @@ static void ov511_pkt_scan(struct gspca_dev *gspca_dev,
 			/* Frame end */
 			if ((in[9] + 1) * 8 != gspca_dev->pixfmt.width ||
 			    (in[10] + 1) * 8 != gspca_dev->pixfmt.height) {
-				PERR("Invalid frame size, got: %dx%d, requested: %dx%d\n",
-					(in[9] + 1) * 8, (in[10] + 1) * 8,
-					gspca_dev->pixfmt.width,
-					gspca_dev->pixfmt.height);
+				gspca_err(gspca_dev, "Invalid frame size, got: %dx%d, requested: %dx%d\n",
+					  (in[9] + 1) * 8, (in[10] + 1) * 8,
+					  gspca_dev->pixfmt.width,
+					  gspca_dev->pixfmt.height);
 				gspca_dev->last_packet_type = DISCARD_PACKET;
 				return;
 			}
@@ -4374,8 +4383,8 @@ static void ov518_pkt_scan(struct gspca_dev *gspca_dev,
 		   except that they may contain part of the footer), are
 		   numbered 0 */
 		else if (sd->packet_nr == 0 || data[len]) {
-			PERR("Invalid packet nr: %d (expect: %d)",
-				(int)data[len], (int)sd->packet_nr);
+			gspca_err(gspca_dev, "Invalid packet nr: %d (expect: %d)\n",
+				  (int)data[len], (int)sd->packet_nr);
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			return;
 		}
@@ -4918,7 +4927,7 @@ static int sd_init_controls(struct gspca_dev *gspca_dev)
 			QUALITY_MIN, QUALITY_MAX, 1, QUALITY_DEF);
 
 	if (hdl->error) {
-		PERR("Could not initialize controls\n");
+		gspca_err(gspca_dev, "Could not initialize controls\n");
 		return hdl->error;
 	}
 	if (gspca_dev->autogain)
diff --git a/drivers/media/usb/gspca/ov534.c b/drivers/media/usb/gspca/ov534.c
index 32849ff86b09..f293921a1f2b 100644
--- a/drivers/media/usb/gspca/ov534.c
+++ b/drivers/media/usb/gspca/ov534.c
@@ -612,7 +612,7 @@ static void ov534_reg_write(struct gspca_dev *gspca_dev, u16 reg, u8 val)
 	if (gspca_dev->usb_err < 0)
 		return;
 
-	PDEBUG(D_USBO, "SET 01 0000 %04x %02x", reg, val);
+	gspca_dbg(gspca_dev, D_USBO, "SET 01 0000 %04x %02x\n", reg, val);
 	gspca_dev->usb_buf[0] = val;
 	ret = usb_control_msg(udev,
 			      usb_sndctrlpipe(udev, 0),
@@ -637,7 +637,8 @@ static u8 ov534_reg_read(struct gspca_dev *gspca_dev, u16 reg)
 			      0x01,
 			      USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
 			      0x00, reg, gspca_dev->usb_buf, 1, CTRL_TIMEOUT);
-	PDEBUG(D_USBI, "GET 01 0000 %04x %02x", reg, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "GET 01 0000 %04x %02x\n",
+		  reg, gspca_dev->usb_buf[0]);
 	if (ret < 0) {
 		pr_err("read failed %d\n", ret);
 		gspca_dev->usb_err = ret;
@@ -651,7 +652,7 @@ static void ov534_set_led(struct gspca_dev *gspca_dev, int status)
 {
 	u8 data;
 
-	PDEBUG(D_CONF, "led status: %d", status);
+	gspca_dbg(gspca_dev, D_CONF, "led status: %d\n", status);
 
 	data = ov534_reg_read(gspca_dev, 0x21);
 	data |= 0x80;
@@ -689,8 +690,8 @@ static int sccb_check_status(struct gspca_dev *gspca_dev)
 		case 0x03:
 			break;
 		default:
-			PERR("sccb status 0x%02x, attempt %d/5",
-			       data, i + 1);
+			gspca_err(gspca_dev, "sccb status 0x%02x, attempt %d/5\n",
+				  data, i + 1);
 		}
 	}
 	return 0;
@@ -698,7 +699,7 @@ static int sccb_check_status(struct gspca_dev *gspca_dev)
 
 static void sccb_reg_write(struct gspca_dev *gspca_dev, u8 reg, u8 val)
 {
-	PDEBUG(D_USBO, "sccb write: %02x %02x", reg, val);
+	gspca_dbg(gspca_dev, D_USBO, "sccb write: %02x %02x\n", reg, val);
 	ov534_reg_write(gspca_dev, OV534_REG_SUBADDR, reg);
 	ov534_reg_write(gspca_dev, OV534_REG_WRITE, val);
 	ov534_reg_write(gspca_dev, OV534_REG_OPERATION, OV534_OP_WRITE_3);
@@ -800,7 +801,7 @@ static void set_frame_rate(struct gspca_dev *gspca_dev)
 	sccb_reg_write(gspca_dev, 0x0d, r->r0d);
 	ov534_reg_write(gspca_dev, 0xe5, r->re5);
 
-	PDEBUG(D_PROBE, "frame_rate: %d", r->fps);
+	gspca_dbg(gspca_dev, D_PROBE, "frame_rate: %d\n", r->fps);
 }
 
 static void sethue(struct gspca_dev *gspca_dev, s32 val)
@@ -1283,7 +1284,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	sensor_id = sccb_reg_read(gspca_dev, 0x0a) << 8;
 	sccb_reg_read(gspca_dev, 0x0b);
 	sensor_id |= sccb_reg_read(gspca_dev, 0x0b);
-	PDEBUG(D_PROBE, "Sensor ID: %04x", sensor_id);
+	gspca_dbg(gspca_dev, D_PROBE, "Sensor ID: %04x\n", sensor_id);
 
 	if ((sensor_id & 0xfff0) == 0x7670) {
 		sd->sensor = SENSOR_OV767x;
@@ -1407,19 +1408,19 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 
 		/* Verify UVC header.  Header length is always 12 */
 		if (data[0] != 12 || len < 12) {
-			PDEBUG(D_PACK, "bad header");
+			gspca_dbg(gspca_dev, D_PACK, "bad header\n");
 			goto discard;
 		}
 
 		/* Check errors */
 		if (data[1] & UVC_STREAM_ERR) {
-			PDEBUG(D_PACK, "payload error");
+			gspca_dbg(gspca_dev, D_PACK, "payload error\n");
 			goto discard;
 		}
 
 		/* Extract PTS and FID */
 		if (!(data[1] & UVC_STREAM_PTS)) {
-			PDEBUG(D_PACK, "PTS not present");
+			gspca_dbg(gspca_dev, D_PACK, "PTS not present\n");
 			goto discard;
 		}
 		this_pts = (data[5] << 24) | (data[4] << 16)
@@ -1442,7 +1443,7 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 			 && gspca_dev->image_len + len - 12 !=
 				   gspca_dev->pixfmt.width *
 					gspca_dev->pixfmt.height * 2) {
-				PDEBUG(D_PACK, "wrong sized frame");
+				gspca_dbg(gspca_dev, D_PACK, "wrong sized frame\n");
 				goto discard;
 			}
 			gspca_frame_add(gspca_dev, LAST_PACKET,
diff --git a/drivers/media/usb/gspca/ov534_9.c b/drivers/media/usb/gspca/ov534_9.c
index b2a92e518118..3d1364d2f83e 100644
--- a/drivers/media/usb/gspca/ov534_9.c
+++ b/drivers/media/usb/gspca/ov534_9.c
@@ -1133,7 +1133,7 @@ static void reg_w_i(struct gspca_dev *gspca_dev, u16 reg, u8 val)
 
 static void reg_w(struct gspca_dev *gspca_dev, u16 reg, u8 val)
 {
-	PDEBUG(D_USBO, "reg_w [%04x] = %02x", reg, val);
+	gspca_dbg(gspca_dev, D_USBO, "reg_w [%04x] = %02x\n", reg, val);
 	reg_w_i(gspca_dev, reg, val);
 }
 
@@ -1149,7 +1149,8 @@ static u8 reg_r(struct gspca_dev *gspca_dev, u16 reg)
 			      0x01,
 			      USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
 			      0x00, reg, gspca_dev->usb_buf, 1, CTRL_TIMEOUT);
-	PDEBUG(D_USBI, "reg_r [%04x] -> %02x", reg, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "reg_r [%04x] -> %02x\n",
+		  reg, gspca_dev->usb_buf[0]);
 	if (ret < 0) {
 		pr_err("reg_r err %d\n", ret);
 		gspca_dev->usb_err = ret;
@@ -1174,9 +1175,9 @@ static int sccb_check_status(struct gspca_dev *gspca_dev)
 		case 0x03:
 			break;
 		default:
-			PDEBUG(D_USBI|D_USBO,
-				"sccb status 0x%02x, attempt %d/5",
-				data, i + 1);
+			gspca_dbg(gspca_dev, D_USBI|D_USBO,
+				  "sccb status 0x%02x, attempt %d/5\n",
+				  data, i + 1);
 		}
 	}
 	return 0;
@@ -1184,7 +1185,7 @@ static int sccb_check_status(struct gspca_dev *gspca_dev)
 
 static void sccb_write(struct gspca_dev *gspca_dev, u8 reg, u8 val)
 {
-	PDEBUG(D_USBO, "sccb_write [%02x] = %02x", reg, val);
+	gspca_dbg(gspca_dev, D_USBO, "sccb_write [%02x] = %02x\n", reg, val);
 	reg_w_i(gspca_dev, OV534_REG_SUBADDR, reg);
 	reg_w_i(gspca_dev, OV534_REG_WRITE, val);
 	reg_w_i(gspca_dev, OV534_REG_OPERATION, OV534_OP_WRITE_3);
@@ -1238,7 +1239,7 @@ static void set_led(struct gspca_dev *gspca_dev, int status)
 {
 	u8 data;
 
-	PDEBUG(D_CONF, "led status: %d", status);
+	gspca_dbg(gspca_dev, D_CONF, "led status: %d\n", status);
 
 	data = reg_r(gspca_dev, 0x21);
 	data |= 0x80;
@@ -1423,7 +1424,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	sensor_id = sccb_read(gspca_dev, 0x0a) << 8;
 	sccb_read(gspca_dev, 0x0b);
 	sensor_id |= sccb_read(gspca_dev, 0x0b);
-	PDEBUG(D_PROBE, "Sensor ID: %04x", sensor_id);
+	gspca_dbg(gspca_dev, D_PROBE, "Sensor ID: %04x\n", sensor_id);
 
 	/* initialize */
 	if ((sensor_id & 0xfff0) == 0x9650) {
@@ -1651,19 +1652,19 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 
 		/* Verify UVC header.  Header length is always 12 */
 		if (data[0] != 12 || len < 12) {
-			PDEBUG(D_PACK, "bad header");
+			gspca_dbg(gspca_dev, D_PACK, "bad header\n");
 			goto discard;
 		}
 
 		/* Check errors */
 		if (data[1] & UVC_STREAM_ERR) {
-			PDEBUG(D_PACK, "payload error");
+			gspca_dbg(gspca_dev, D_PACK, "payload error\n");
 			goto discard;
 		}
 
 		/* Extract PTS and FID */
 		if (!(data[1] & UVC_STREAM_PTS)) {
-			PDEBUG(D_PACK, "PTS not present");
+			gspca_dbg(gspca_dev, D_PACK, "PTS not present\n");
 			goto discard;
 		}
 		this_pts = (data[5] << 24) | (data[4] << 16)
diff --git a/drivers/media/usb/gspca/pac207.c b/drivers/media/usb/gspca/pac207.c
index 01c185d367e5..a1df7af8fef5 100644
--- a/drivers/media/usb/gspca/pac207.c
+++ b/drivers/media/usb/gspca/pac207.c
@@ -166,17 +166,17 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	idreg[1] = pac207_read_reg(gspca_dev, 0x0001);
 	idreg[0] = ((idreg[0] & 0x0f) << 4) | ((idreg[1] & 0xf0) >> 4);
 	idreg[1] = idreg[1] & 0x0f;
-	PDEBUG(D_PROBE, "Pixart Sensor ID 0x%02X Chips ID 0x%02X",
-		idreg[0], idreg[1]);
+	gspca_dbg(gspca_dev, D_PROBE, "Pixart Sensor ID 0x%02X Chips ID 0x%02X\n",
+		  idreg[0], idreg[1]);
 
 	if (idreg[0] != 0x27) {
-		PDEBUG(D_PROBE, "Error invalid sensor ID!");
+		gspca_dbg(gspca_dev, D_PROBE, "Error invalid sensor ID!\n");
 		return -ENODEV;
 	}
 
-	PDEBUG(D_PROBE,
-		"Pixart PAC207BCA Image Processor and Control Chip detected (vid/pid 0x%04X:0x%04X)",
-		id->idVendor, id->idProduct);
+	gspca_dbg(gspca_dev, D_PROBE,
+		  "Pixart PAC207BCA Image Processor and Control Chip detected (vid/pid 0x%04X:0x%04X)\n",
+		  id->idVendor, id->idProduct);
 
 	cam = &gspca_dev->cam;
 	cam->cam_mode = sif_mode;
@@ -315,9 +315,9 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		mode = 0x02;
 	if (gspca_dev->pixfmt.width == 176) {	/* 176x144 */
 		mode |= 0x01;
-		PDEBUG(D_STREAM, "pac207_start mode 176x144");
+		gspca_dbg(gspca_dev, D_STREAM, "pac207_start mode 176x144\n");
 	} else {				/* 352x288 */
-		PDEBUG(D_STREAM, "pac207_start mode 352x288");
+		gspca_dbg(gspca_dev, D_STREAM, "pac207_start mode 352x288\n");
 	}
 	pac207_write_reg(gspca_dev, 0x41, mode);
 
diff --git a/drivers/media/usb/gspca/pac7302.c b/drivers/media/usb/gspca/pac7302.c
index 595535e143e6..b8ff201c7bb9 100644
--- a/drivers/media/usb/gspca/pac7302.c
+++ b/drivers/media/usb/gspca/pac7302.c
@@ -339,7 +339,7 @@ static void reg_w_var(struct gspca_dev *gspca_dev,
 			break;
 		default:
 			if (len > USB_BUF_SZ) {
-				PERR("Incorrect variable sequence");
+				gspca_err(gspca_dev, "Incorrect variable sequence\n");
 				return;
 			}
 			while (len > 0) {
diff --git a/drivers/media/usb/gspca/pac7311.c b/drivers/media/usb/gspca/pac7311.c
index 8bac2d9326bf..44db4f4afa22 100644
--- a/drivers/media/usb/gspca/pac7311.c
+++ b/drivers/media/usb/gspca/pac7311.c
@@ -258,7 +258,7 @@ static void reg_w_var(struct gspca_dev *gspca_dev,
 			break;
 		default:
 			if (len > USB_BUF_SZ) {
-				PERR("Incorrect variable sequence");
+				gspca_err(gspca_dev, "Incorrect variable sequence\n");
 				return;
 			}
 			while (len > 0) {
diff --git a/drivers/media/usb/gspca/pac_common.h b/drivers/media/usb/gspca/pac_common.h
index 4047bcb6c2b5..31f2a42af4dd 100644
--- a/drivers/media/usb/gspca/pac_common.h
+++ b/drivers/media/usb/gspca/pac_common.h
@@ -107,10 +107,9 @@ static unsigned char *pac_find_sof(struct gspca_dev *gspca_dev, u8 *sof_read,
 			switch (m[i]) {
 			case 0x96:
 				/* Pattern found */
-				PDEBUG(D_FRAM,
-					"SOF found, bytes to analyze: %u."
-					" Frame starts at byte #%u",
-					len, i + 1);
+				gspca_dbg(gspca_dev, D_FRAM,
+					  "SOF found, bytes to analyze: %u - Frame starts at byte #%u\n",
+					  len, i + 1);
 				*sof_read = 0;
 				return m + i + 1;
 				break;
diff --git a/drivers/media/usb/gspca/sn9c2028.c b/drivers/media/usb/gspca/sn9c2028.c
index 5d32dd359d84..a1f7189545e1 100644
--- a/drivers/media/usb/gspca/sn9c2028.c
+++ b/drivers/media/usb/gspca/sn9c2028.c
@@ -69,8 +69,9 @@ static int sn9c2028_command(struct gspca_dev *gspca_dev, u8 *command)
 {
 	int rc;
 
-	PDEBUG(D_USBO, "sending command %02x%02x%02x%02x%02x%02x", command[0],
-	       command[1], command[2], command[3], command[4], command[5]);
+	gspca_dbg(gspca_dev, D_USBO, "sending command %02x%02x%02x%02x%02x%02x\n",
+		  command[0], command[1], command[2],
+		  command[3], command[4], command[5]);
 
 	memcpy(gspca_dev->usb_buf, command, 6);
 	rc = usb_control_msg(gspca_dev->dev,
@@ -100,7 +101,8 @@ static int sn9c2028_read1(struct gspca_dev *gspca_dev)
 		pr_err("read1 error %d\n", rc);
 		return (rc < 0) ? rc : -EIO;
 	}
-	PDEBUG(D_USBI, "read1 response %02x", gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "read1 response %02x\n",
+		  gspca_dev->usb_buf[0]);
 	return gspca_dev->usb_buf[0];
 }
 
@@ -117,8 +119,8 @@ static int sn9c2028_read4(struct gspca_dev *gspca_dev, u8 *reading)
 		return (rc < 0) ? rc : -EIO;
 	}
 	memcpy(reading, gspca_dev->usb_buf, 4);
-	PDEBUG(D_USBI, "read4 response %02x%02x%02x%02x", reading[0],
-	       reading[1], reading[2], reading[3]);
+	gspca_dbg(gspca_dev, D_USBI, "read4 response %02x%02x%02x%02x\n",
+		  reading[0], reading[1], reading[2], reading[3]);
 	return rc;
 }
 
@@ -175,32 +177,32 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	struct sd *sd = (struct sd *) gspca_dev;
 	struct cam *cam = &gspca_dev->cam;
 
-	PDEBUG(D_PROBE, "SN9C2028 camera detected (vid/pid 0x%04X:0x%04X)",
-	       id->idVendor, id->idProduct);
+	gspca_dbg(gspca_dev, D_PROBE, "SN9C2028 camera detected (vid/pid 0x%04X:0x%04X)\n",
+		  id->idVendor, id->idProduct);
 
 	sd->model = id->idProduct;
 
 	switch (sd->model) {
 	case 0x7005:
-		PDEBUG(D_PROBE, "Genius Smart 300 camera");
+		gspca_dbg(gspca_dev, D_PROBE, "Genius Smart 300 camera\n");
 		break;
 	case 0x7003:
-		PDEBUG(D_PROBE, "Genius Videocam Live v2");
+		gspca_dbg(gspca_dev, D_PROBE, "Genius Videocam Live v2\n");
 		break;
 	case 0x8000:
-		PDEBUG(D_PROBE, "DC31VC");
+		gspca_dbg(gspca_dev, D_PROBE, "DC31VC\n");
 		break;
 	case 0x8001:
-		PDEBUG(D_PROBE, "Spy camera");
+		gspca_dbg(gspca_dev, D_PROBE, "Spy camera\n");
 		break;
 	case 0x8003:
-		PDEBUG(D_PROBE, "CIF camera");
+		gspca_dbg(gspca_dev, D_PROBE, "CIF camera\n");
 		break;
 	case 0x8008:
-		PDEBUG(D_PROBE, "Mini-Shotz ms-350 camera");
+		gspca_dbg(gspca_dev, D_PROBE, "Mini-Shotz ms-350 camera\n");
 		break;
 	case 0x800a:
-		PDEBUG(D_PROBE, "Vivitar 3350b type camera");
+		gspca_dbg(gspca_dev, D_PROBE, "Vivitar 3350b type camera\n");
 		cam->input_flags = V4L2_IN_ST_VFLIP | V4L2_IN_ST_HFLIP;
 		break;
 	}
@@ -849,13 +851,13 @@ static void sd_stopN(struct gspca_dev *gspca_dev)
 
 	result = sn9c2028_read1(gspca_dev);
 	if (result < 0)
-		PERR("Camera Stop read failed");
+		gspca_err(gspca_dev, "Camera Stop read failed\n");
 
 	memset(data, 0, 6);
 	data[0] = 0x14;
 	result = sn9c2028_command(gspca_dev, data);
 	if (result < 0)
-		PERR("Camera Stop command failed");
+		gspca_err(gspca_dev, "Camera Stop command failed\n");
 }
 
 static void do_autogain(struct gspca_dev *gspca_dev, int avg_lum)
diff --git a/drivers/media/usb/gspca/sn9c2028.h b/drivers/media/usb/gspca/sn9c2028.h
index 85761aa7c8b2..29f1571f9242 100644
--- a/drivers/media/usb/gspca/sn9c2028.h
+++ b/drivers/media/usb/gspca/sn9c2028.h
@@ -43,10 +43,9 @@ static unsigned char *sn9c2028_find_sof(struct gspca_dev *gspca_dev,
 			if (sd->sof_read == 12)
 				sd->avg_lum = (m[i] << 8) + sd->avg_lum_l;
 			if (sd->sof_read == sizeof(sn9c2028_sof_marker)) {
-				PDEBUG(D_FRAM,
-					"SOF found, bytes to analyze: %u."
-					" Frame starts at byte #%u",
-					len, i + 1);
+				gspca_dbg(gspca_dev, D_FRAM,
+					  "SOF found, bytes to analyze: %u - Frame starts at byte #%u\n",
+					  len, i + 1);
 				sd->sof_read = 0;
 				return m + i + 1;
 			}
diff --git a/drivers/media/usb/gspca/sn9c20x.c b/drivers/media/usb/gspca/sn9c20x.c
index c605f78d6186..cfa2a04d9f3f 100644
--- a/drivers/media/usb/gspca/sn9c20x.c
+++ b/drivers/media/usb/gspca/sn9c20x.c
@@ -2153,7 +2153,7 @@ static void qual_upd(struct work_struct *work)
 
 	/* To protect gspca_dev->usb_buf and gspca_dev->usb_err */
 	mutex_lock(&gspca_dev->usb_lock);
-	PDEBUG(D_STREAM, "qual_upd %d%%", qual);
+	gspca_dbg(gspca_dev, D_STREAM, "qual_upd %d%%\n", qual);
 	gspca_dev->usb_err = 0;
 	set_quality(gspca_dev, qual);
 	mutex_unlock(&gspca_dev->usb_lock);
diff --git a/drivers/media/usb/gspca/sonixj.c b/drivers/media/usb/gspca/sonixj.c
index 5eeaf16ac5e8..df8d8482b795 100644
--- a/drivers/media/usb/gspca/sonixj.c
+++ b/drivers/media/usb/gspca/sonixj.c
@@ -1155,7 +1155,7 @@ static void reg_r(struct gspca_dev *gspca_dev,
 	if (gspca_dev->usb_err < 0)
 		return;
 	if (len > USB_BUF_SZ) {
-		PERR("reg_r: buffer overflow\n");
+		gspca_err(gspca_dev, "reg_r: buffer overflow\n");
 		return;
 	}
 
@@ -1166,7 +1166,8 @@ static void reg_r(struct gspca_dev *gspca_dev,
 			value, 0,
 			gspca_dev->usb_buf, len,
 			500);
-	PDEBUG(D_USBI, "reg_r [%02x] -> %02x", value, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "reg_r [%02x] -> %02x\n",
+		  value, gspca_dev->usb_buf[0]);
 	if (ret < 0) {
 		pr_err("reg_r err %d\n", ret);
 		gspca_dev->usb_err = ret;
@@ -1181,7 +1182,7 @@ static void reg_w1(struct gspca_dev *gspca_dev,
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "reg_w1 [%04x] = %02x", value, data);
+	gspca_dbg(gspca_dev, D_USBO, "reg_w1 [%04x] = %02x\n", value, data);
 	gspca_dev->usb_buf[0] = data;
 	ret = usb_control_msg(gspca_dev->dev,
 			usb_sndctrlpipe(gspca_dev->dev, 0),
@@ -1205,11 +1206,11 @@ static void reg_w(struct gspca_dev *gspca_dev,
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "reg_w [%04x] = %02x %02x ..",
-		value, buffer[0], buffer[1]);
+	gspca_dbg(gspca_dev, D_USBO, "reg_w [%04x] = %02x %02x ..\n",
+		  value, buffer[0], buffer[1]);
 
 	if (len > USB_BUF_SZ) {
-		PERR("reg_w: buffer overflow\n");
+		gspca_err(gspca_dev, "reg_w: buffer overflow\n");
 		return;
 	}
 
@@ -1235,7 +1236,7 @@ static void i2c_w1(struct gspca_dev *gspca_dev, u8 reg, u8 val)
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "i2c_w1 [%02x] = %02x", reg, val);
+	gspca_dbg(gspca_dev, D_USBO, "i2c_w1 [%02x] = %02x\n", reg, val);
 	switch (sd->sensor) {
 	case SENSOR_ADCM1700:
 	case SENSOR_OM6802:
@@ -1276,8 +1277,8 @@ static void i2c_w8(struct gspca_dev *gspca_dev,
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "i2c_w8 [%02x] = %02x ..",
-		buffer[2], buffer[3]);
+	gspca_dbg(gspca_dev, D_USBO, "i2c_w8 [%02x] = %02x ..\n",
+		  buffer[2], buffer[3]);
 	memcpy(gspca_dev->usb_buf, buffer, 8);
 	ret = usb_control_msg(gspca_dev->dev,
 			usb_sndctrlpipe(gspca_dev->dev, 0),
@@ -1349,7 +1350,7 @@ static void hv7131r_probe(struct gspca_dev *gspca_dev)
 	if (gspca_dev->usb_buf[0] == 0x02	/* chip ID (02 is R) */
 	    && gspca_dev->usb_buf[1] == 0x09
 	    && gspca_dev->usb_buf[2] == 0x01) {
-		PDEBUG(D_PROBE, "Sensor HV7131R found");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor HV7131R found\n");
 		return;
 	}
 	pr_warn("Erroneous HV7131R ID 0x%02x 0x%02x 0x%02x\n",
@@ -1396,18 +1397,19 @@ static void mi0360_probe(struct gspca_dev *gspca_dev)
 		return;
 	switch (val) {
 	case 0x8221:
-		PDEBUG(D_PROBE, "Sensor mi0360b");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor mi0360b\n");
 		sd->sensor = SENSOR_MI0360B;
 		break;
 	case 0x823a:
-		PDEBUG(D_PROBE, "Sensor mt9v111");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor mt9v111\n");
 		sd->sensor = SENSOR_MT9V111;
 		break;
 	case 0x8243:
-		PDEBUG(D_PROBE, "Sensor mi0360");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor mi0360\n");
 		break;
 	default:
-		PDEBUG(D_PROBE, "Unknown sensor %04x - forced to mi0360", val);
+		gspca_dbg(gspca_dev, D_PROBE, "Unknown sensor %04x - forced to mi0360\n",
+			  val);
 		break;
 	}
 }
@@ -1432,10 +1434,10 @@ static void ov7630_probe(struct gspca_dev *gspca_dev)
 /*fixme: only valid for 0c45:613e?*/
 		gspca_dev->cam.input_flags =
 				V4L2_IN_ST_VFLIP | V4L2_IN_ST_HFLIP;
-		PDEBUG(D_PROBE, "Sensor soi768");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor soi768\n");
 		return;
 	}
-	PDEBUG(D_PROBE, "Sensor ov%04x", val);
+	gspca_dbg(gspca_dev, D_PROBE, "Sensor ov%04x\n", val);
 }
 
 static void ov7648_probe(struct gspca_dev *gspca_dev)
@@ -1452,7 +1454,7 @@ static void ov7648_probe(struct gspca_dev *gspca_dev)
 	reg_w1(gspca_dev, 0x01, 0x29);
 	reg_w1(gspca_dev, 0x17, 0x42);
 	if ((val & 0xff00) == 0x7600) {		/* ov76xx */
-		PDEBUG(D_PROBE, "Sensor ov%04x", val);
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor ov%04x\n", val);
 		return;
 	}
 
@@ -1467,7 +1469,7 @@ static void ov7648_probe(struct gspca_dev *gspca_dev)
 	if (gspca_dev->usb_err < 0)
 		return;
 	if (val == 0x1030) {			/* po1030 */
-		PDEBUG(D_PROBE, "Sensor po1030");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor po1030\n");
 		sd->sensor = SENSOR_PO1030;
 		return;
 	}
@@ -1490,7 +1492,7 @@ static void po2030n_probe(struct gspca_dev *gspca_dev)
 	reg_w1(gspca_dev, 0x01, 0x29);		/* reset */
 	reg_w1(gspca_dev, 0x17, 0x42);
 	if (val == 0x99) {			/* gc0307 (?) */
-		PDEBUG(D_PROBE, "Sensor gc0307");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor gc0307\n");
 		sd->sensor = SENSOR_GC0307;
 		return;
 	}
@@ -1506,7 +1508,7 @@ static void po2030n_probe(struct gspca_dev *gspca_dev)
 	if (gspca_dev->usb_err < 0)
 		return;
 	if (val == 0x2030) {
-		PDEBUG(D_PROBE, "Sensor po2030n");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor po2030n\n");
 /*		sd->sensor = SENSOR_PO2030N; */
 	} else {
 		pr_err("Unknown sensor ID %04x\n", val);
@@ -1558,7 +1560,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	regF1 = gspca_dev->usb_buf[0];
 	if (gspca_dev->usb_err < 0)
 		return gspca_dev->usb_err;
-	PDEBUG(D_PROBE, "Sonix chip id: %02x", regF1);
+	gspca_dbg(gspca_dev, D_PROBE, "Sonix chip id: %02x\n", regF1);
 	if (gspca_dev->audio)
 		regGpio[1] |= 0x04;		/* with audio */
 	switch (sd->bridge) {
@@ -1758,10 +1760,10 @@ static u32 expo_adjust(struct gspca_dev *gspca_dev,
 				| ((expo & 0x0003) << 4);
 		i2c_w8(gspca_dev, expoMo10);
 		i2c_w8(gspca_dev, gainMo);
-		PDEBUG(D_FRAM, "set exposure %d",
-			((expoMo10[3] & 0x07) << 10)
-			| (expoMof[3] << 2)
-			| ((expoMo10[3] & 0x30) >> 4));
+		gspca_dbg(gspca_dev, D_FRAM, "set exposure %d\n",
+			  ((expoMo10[3] & 0x07) << 10)
+			  | (expoMof[3] << 2)
+			  | ((expoMo10[3] & 0x30) >> 4));
 		break;
 	    }
 	case SENSOR_MT9V111: {
@@ -1789,7 +1791,7 @@ static u32 expo_adjust(struct gspca_dev *gspca_dev,
 		gainOm[3] = expo >> 2;
 		i2c_w8(gspca_dev, gainOm);
 		reg_w1(gspca_dev, 0x96, expo >> 5);
-		PDEBUG(D_FRAM, "set exposure %d", gainOm[3]);
+		gspca_dbg(gspca_dev, D_FRAM, "set exposure %d\n", gainOm[3]);
 		break;
 	    }
 	}
@@ -2162,7 +2164,7 @@ static void qual_upd(struct work_struct *work)
 
 	/* To protect gspca_dev->usb_buf and gspca_dev->usb_err */
 	mutex_lock(&gspca_dev->usb_lock);
-	PDEBUG(D_STREAM, "qual_upd %d%%", sd->quality);
+	gspca_dbg(gspca_dev, D_STREAM, "qual_upd %d%%\n", sd->quality);
 	gspca_dev->usb_err = 0;
 	setjpegqual(gspca_dev);
 	mutex_unlock(&gspca_dev->usb_lock);
@@ -2584,7 +2586,7 @@ static void do_autogain(struct gspca_dev *gspca_dev)
 	sd->ag_cnt = AG_CNT_START;
 
 	delta = atomic_read(&sd->avg_lum);
-	PDEBUG(D_FRAM, "mean lum %d", delta);
+	gspca_dbg(gspca_dev, D_FRAM, "mean lum %d\n", delta);
 
 	if (sd->sensor == SENSOR_PO2030N) {
 		gspca_expo_autogain(gspca_dev, delta, luma_mean, luma_delta,
diff --git a/drivers/media/usb/gspca/spca1528.c b/drivers/media/usb/gspca/spca1528.c
index 327ec901abe1..d25924e430f3 100644
--- a/drivers/media/usb/gspca/spca1528.c
+++ b/drivers/media/usb/gspca/spca1528.c
@@ -75,8 +75,8 @@ static void reg_r(struct gspca_dev *gspca_dev,
 			index,
 			gspca_dev->usb_buf, len,
 			500);
-	PDEBUG(D_USBI, "GET %02x 0000 %04x %02x", req, index,
-			 gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "GET %02x 0000 %04x %02x\n", req, index,
+		  gspca_dev->usb_buf[0]);
 	if (ret < 0) {
 		pr_err("reg_r err %d\n", ret);
 		gspca_dev->usb_err = ret;
@@ -93,7 +93,7 @@ static void reg_w(struct gspca_dev *gspca_dev,
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "SET %02x %04x %04x", req, value, index);
+	gspca_dbg(gspca_dev, D_USBO, "SET %02x %04x %04x\n", req, value, index);
 	ret = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
 			req,
 			USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
@@ -116,7 +116,8 @@ static void reg_wb(struct gspca_dev *gspca_dev,
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "SET %02x %04x %04x %02x", req, value, index, byte);
+	gspca_dbg(gspca_dev, D_USBO, "SET %02x %04x %04x %02x\n",
+		  req, value, index, byte);
 	gspca_dev->usb_buf[0] = byte;
 	ret = usb_control_msg(dev, usb_sndctrlpipe(dev, 0),
 			req,
@@ -142,7 +143,7 @@ static void wait_status_0(struct gspca_dev *gspca_dev)
 		w += 15;
 		msleep(w);
 	} while (--i > 0);
-	PERR("wait_status_0 timeout");
+	gspca_err(gspca_dev, "wait_status_0 timeout\n");
 	gspca_dev->usb_err = -ETIME;
 }
 
@@ -160,7 +161,7 @@ static void wait_status_1(struct gspca_dev *gspca_dev)
 			return;
 		}
 	} while (--i > 0);
-	PERR("wait_status_1 timeout");
+	gspca_err(gspca_dev, "wait_status_1 timeout\n");
 	gspca_dev->usb_err = -ETIME;
 }
 
@@ -216,8 +217,8 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	reg_r(gspca_dev, 0x20, 0x0000, 1);
 	reg_r(gspca_dev, 0x20, 0x0000, 5);
 	reg_r(gspca_dev, 0x23, 0x0000, 64);
-	PDEBUG(D_PROBE, "%s%s", &gspca_dev->usb_buf[0x1c],
-				&gspca_dev->usb_buf[0x30]);
+	gspca_dbg(gspca_dev, D_PROBE, "%s%s\n", &gspca_dev->usb_buf[0x1c],
+		  &gspca_dev->usb_buf[0x30]);
 	reg_r(gspca_dev, 0x23, 0x0001, 64);
 	return gspca_dev->usb_err;
 }
diff --git a/drivers/media/usb/gspca/spca500.c b/drivers/media/usb/gspca/spca500.c
index da2d9027914c..e90d2f3b4a67 100644
--- a/drivers/media/usb/gspca/spca500.c
+++ b/drivers/media/usb/gspca/spca500.c
@@ -328,7 +328,8 @@ static int reg_w(struct gspca_dev *gspca_dev,
 {
 	int ret;
 
-	PDEBUG(D_USBO, "reg write: [0x%02x] = 0x%02x", index, value);
+	gspca_dbg(gspca_dev, D_USBO, "reg write: [0x%02x] = 0x%02x\n",
+		  index, value);
 	ret = usb_control_msg(gspca_dev->dev,
 			usb_sndctrlpipe(gspca_dev->dev, 0),
 			req,
@@ -423,15 +424,15 @@ static int spca50x_setup_qtable(struct gspca_dev *gspca_dev,
 static void spca500_ping310(struct gspca_dev *gspca_dev)
 {
 	reg_r(gspca_dev, 0x0d04, 2);
-	PDEBUG(D_STREAM, "ClickSmart310 ping 0x0d04 0x%02x 0x%02x",
-		gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
+	gspca_dbg(gspca_dev, D_STREAM, "ClickSmart310 ping 0x0d04 0x%02x 0x%02x\n",
+		  gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
 }
 
 static void spca500_clksmart310_init(struct gspca_dev *gspca_dev)
 {
 	reg_r(gspca_dev, 0x0d05, 2);
-	PDEBUG(D_STREAM, "ClickSmart310 init 0x0d05 0x%02x 0x%02x",
-		gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
+	gspca_dbg(gspca_dev, D_STREAM, "ClickSmart310 init 0x0d05 0x%02x 0x%02x\n",
+		  gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
 	reg_w(gspca_dev, 0x00, 0x8167, 0x5a);
 	spca500_ping310(gspca_dev);
 
@@ -485,7 +486,7 @@ static int spca500_full_reset(struct gspca_dev *gspca_dev)
 		return err;
 	err = reg_r_wait(gspca_dev, 0x06, 0, 0);
 	if (err < 0) {
-		PERR("reg_r_wait() failed");
+		gspca_err(gspca_dev, "reg_r_wait() failed\n");
 		return err;
 	}
 	/* all ok */
@@ -501,7 +502,7 @@ static int spca500_full_reset(struct gspca_dev *gspca_dev)
 static int spca500_synch310(struct gspca_dev *gspca_dev)
 {
 	if (usb_set_interface(gspca_dev->dev, gspca_dev->iface, 0) < 0) {
-		PERR("Set packet size: set interface error");
+		gspca_err(gspca_dev, "Set packet size: set interface error\n");
 		goto error;
 	}
 	spca500_ping310(gspca_dev);
@@ -509,13 +510,14 @@ static int spca500_synch310(struct gspca_dev *gspca_dev)
 	reg_r(gspca_dev, 0x0d00, 1);
 
 	/* need alt setting here */
-	PDEBUG(D_PACK, "ClickSmart310 sync alt: %d", gspca_dev->alt);
+	gspca_dbg(gspca_dev, D_PACK, "ClickSmart310 sync alt: %d\n",
+		  gspca_dev->alt);
 
 	/* Windoze use pipe with altsetting 6 why 7 here */
 	if (usb_set_interface(gspca_dev->dev,
 				gspca_dev->iface,
 				gspca_dev->alt) < 0) {
-		PERR("Set packet size: set interface error");
+		gspca_err(gspca_dev, "Set packet size: set interface error\n");
 		goto error;
 	}
 	return 0;
@@ -540,7 +542,7 @@ static void spca500_reinit(struct gspca_dev *gspca_dev)
 	err = spca50x_setup_qtable(gspca_dev, 0x00, 0x8800, 0x8840,
 				 qtable_pocketdv);
 	if (err < 0)
-		PERR("spca50x_setup_qtable failed on init");
+		gspca_err(gspca_dev, "spca50x_setup_qtable failed on init\n");
 
 	/* set qtable index */
 	reg_w(gspca_dev, 0x00, 0x8880, 2);
@@ -587,12 +589,12 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	struct sd *sd = (struct sd *) gspca_dev;
 
 	/* initialisation of spca500 based cameras is deferred */
-	PDEBUG(D_STREAM, "SPCA500 init");
+	gspca_dbg(gspca_dev, D_STREAM, "SPCA500 init\n");
 	if (sd->subtype == LogitechClickSmart310)
 		spca500_clksmart310_init(gspca_dev);
 /*	else
 		spca500_initialise(gspca_dev); */
-	PDEBUG(D_STREAM, "SPCA500 init done");
+	gspca_dbg(gspca_dev, D_STREAM, "SPCA500 init done\n");
 	return 0;
 }
 
@@ -619,10 +621,10 @@ static int sd_start(struct gspca_dev *gspca_dev)
 
 	/* is there a sensor here ? */
 	reg_r(gspca_dev, 0x8a04, 1);
-	PDEBUG(D_STREAM, "Spca500 Sensor Address 0x%02x",
-		gspca_dev->usb_buf[0]);
-	PDEBUG(D_STREAM, "Spca500 curr_mode: %d Xmult: 0x%02x, Ymult: 0x%02x",
-		gspca_dev->curr_mode, xmult, ymult);
+	gspca_dbg(gspca_dev, D_STREAM, "Spca500 Sensor Address 0x%02x\n",
+		  gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_STREAM, "Spca500 curr_mode: %d Xmult: 0x%02x, Ymult: 0x%02x",
+		  gspca_dev->curr_mode, xmult, ymult);
 
 	/* setup qtable */
 	switch (sd->subtype) {
@@ -636,7 +638,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 					   0x00, 0x8800, 0x8840,
 					   qtable_creative_pccam);
 		if (err < 0)
-			PERR("spca50x_setup_qtable failed");
+			gspca_err(gspca_dev, "spca50x_setup_qtable failed\n");
 		/* Init SDRAM - needed for SDRAM access */
 		reg_w(gspca_dev, 0x00, 0x870a, 0x04);
 
@@ -644,7 +646,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		reg_w(gspca_dev, 0x00, 0x8000, 0x0004);
 		msleep(500);
 		if (reg_r_wait(gspca_dev, 0, 0x8000, 0x44) != 0)
-			PERR("reg_r_wait() failed");
+			gspca_err(gspca_dev, "reg_r_wait() failed\n");
 
 		reg_r(gspca_dev, 0x816b, 1);
 		Data = gspca_dev->usb_buf[0];
@@ -657,13 +659,13 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		/* enable drop packet */
 		err = reg_w(gspca_dev, 0x00, 0x850a, 0x0001);
 		if (err < 0)
-			PERR("failed to enable drop packet");
+			gspca_err(gspca_dev, "failed to enable drop packet\n");
 		reg_w(gspca_dev, 0x00, 0x8880, 3);
 		err = spca50x_setup_qtable(gspca_dev,
 					   0x00, 0x8800, 0x8840,
 					   qtable_creative_pccam);
 		if (err < 0)
-			PERR("spca50x_setup_qtable failed");
+			gspca_err(gspca_dev, "spca50x_setup_qtable failed\n");
 
 		/* Init SDRAM - needed for SDRAM access */
 		reg_w(gspca_dev, 0x00, 0x870a, 0x04);
@@ -672,7 +674,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		reg_w(gspca_dev, 0x00, 0x8000, 0x0004);
 
 		if (reg_r_wait(gspca_dev, 0, 0x8000, 0x44) != 0)
-			PERR("reg_r_wait() failed");
+			gspca_err(gspca_dev, "reg_r_wait() failed\n");
 
 		reg_r(gspca_dev, 0x816b, 1);
 		Data = gspca_dev->usb_buf[0];
@@ -686,18 +688,18 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		/* do a full reset */
 		err = spca500_full_reset(gspca_dev);
 		if (err < 0)
-			PERR("spca500_full_reset failed");
+			gspca_err(gspca_dev, "spca500_full_reset failed\n");
 
 		/* enable drop packet */
 		err = reg_w(gspca_dev, 0x00, 0x850a, 0x0001);
 		if (err < 0)
-			PERR("failed to enable drop packet");
+			gspca_err(gspca_dev, "failed to enable drop packet\n");
 		reg_w(gspca_dev, 0x00, 0x8880, 3);
 		err = spca50x_setup_qtable(gspca_dev,
 					   0x00, 0x8800, 0x8840,
 					   qtable_creative_pccam);
 		if (err < 0)
-			PERR("spca50x_setup_qtable failed");
+			gspca_err(gspca_dev, "spca50x_setup_qtable failed\n");
 
 		spca500_setmode(gspca_dev, xmult, ymult);
 		reg_w(gspca_dev, 0x20, 0x0001, 0x0004);
@@ -706,7 +708,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		reg_w(gspca_dev, 0x00, 0x8000, 0x0004);
 
 		if (reg_r_wait(gspca_dev, 0, 0x8000, 0x44) != 0)
-			PERR("reg_r_wait() failed");
+			gspca_err(gspca_dev, "reg_r_wait() failed\n");
 
 		reg_r(gspca_dev, 0x816b, 1);
 		Data = gspca_dev->usb_buf[0];
@@ -719,7 +721,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		/* do a full reset */
 		err = spca500_full_reset(gspca_dev);
 		if (err < 0)
-			PERR("spca500_full_reset failed");
+			gspca_err(gspca_dev, "spca500_full_reset failed\n");
 		/* enable drop packet */
 		reg_w(gspca_dev, 0x00, 0x850a, 0x0001);
 		reg_w(gspca_dev, 0x00, 0x8880, 0);
@@ -727,7 +729,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 					   0x00, 0x8800, 0x8840,
 					   qtable_kodak_ez200);
 		if (err < 0)
-			PERR("spca50x_setup_qtable failed");
+			gspca_err(gspca_dev, "spca50x_setup_qtable failed\n");
 		spca500_setmode(gspca_dev, xmult, ymult);
 
 		reg_w(gspca_dev, 0x20, 0x0001, 0x0004);
@@ -736,7 +738,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		reg_w(gspca_dev, 0x00, 0x8000, 0x0004);
 
 		if (reg_r_wait(gspca_dev, 0, 0x8000, 0x44) != 0)
-			PERR("reg_r_wait() failed");
+			gspca_err(gspca_dev, "reg_r_wait() failed\n");
 
 		reg_r(gspca_dev, 0x816b, 1);
 		Data = gspca_dev->usb_buf[0];
@@ -762,7 +764,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 		err = spca50x_setup_qtable(gspca_dev,
 				   0x00, 0x8800, 0x8840, qtable_pocketdv);
 		if (err < 0)
-			PERR("spca50x_setup_qtable failed");
+			gspca_err(gspca_dev, "spca50x_setup_qtable failed\n");
 		reg_w(gspca_dev, 0x00, 0x8880, 2);
 
 		/* familycam Quicksmart pocketDV stuff */
@@ -792,7 +794,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 					0x00, 0x8800,
 					0x8840, qtable_creative_pccam);
 		if (err < 0)
-			PERR("spca50x_setup_qtable failed");
+			gspca_err(gspca_dev, "spca50x_setup_qtable failed\n");
 		reg_w(gspca_dev, 0x00, 0x8880, 3);
 		reg_w(gspca_dev, 0x00, 0x800a, 0x00);
 		/* Init SDRAM - needed for SDRAM access */
@@ -820,8 +822,8 @@ static void sd_stopN(struct gspca_dev *gspca_dev)
 	/* switch to video camera mode */
 	reg_w(gspca_dev, 0x00, 0x8000, 0x0004);
 	reg_r(gspca_dev, 0x8000, 1);
-	PDEBUG(D_STREAM, "stop SPCA500 done reg8000: 0x%2x",
-		gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_STREAM, "stop SPCA500 done reg8000: 0x%2x\n",
+		  gspca_dev->usb_buf[0]);
 }
 
 static void sd_pkt_scan(struct gspca_dev *gspca_dev,
diff --git a/drivers/media/usb/gspca/spca501.c b/drivers/media/usb/gspca/spca501.c
index ae5a80987553..2cce74b166d8 100644
--- a/drivers/media/usb/gspca/spca501.c
+++ b/drivers/media/usb/gspca/spca501.c
@@ -1763,8 +1763,8 @@ static int reg_write(struct gspca_dev *gspca_dev,
 			req,
 			USB_TYPE_VENDOR | USB_RECIP_DEVICE,
 			value, index, NULL, 0, 500);
-	PDEBUG(D_USBO, "reg write: 0x%02x 0x%02x 0x%02x",
-		req, index, value);
+	gspca_dbg(gspca_dev, D_USBO, "reg write: 0x%02x 0x%02x 0x%02x\n",
+		  req, index, value);
 	if (ret < 0)
 		pr_err("reg write: error %d\n", ret);
 	return ret;
@@ -1779,8 +1779,8 @@ static int write_vector(struct gspca_dev *gspca_dev, const __u16 data[][3])
 		ret = reg_write(gspca_dev, data[i][0], data[i][2],
 								data[i][1]);
 		if (ret < 0) {
-			PERR("Reg write failed for 0x%02x,0x%02x,0x%02x",
-				data[i][0], data[i][1], data[i][2]);
+			gspca_err(gspca_dev, "Reg write failed for 0x%02x,0x%02x,0x%02x\n",
+				  data[i][0], data[i][1], data[i][2]);
 			return ret;
 		}
 		i++;
@@ -1852,7 +1852,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 			goto error;
 		break;
 	}
-	PDEBUG(D_STREAM, "Initializing SPCA501 finished");
+	gspca_dbg(gspca_dev, D_STREAM, "Initializing SPCA501 finished\n");
 	return 0;
 error:
 	return -EINVAL;
diff --git a/drivers/media/usb/gspca/spca505.c b/drivers/media/usb/gspca/spca505.c
index 1553cc766c04..07aae9cd5a30 100644
--- a/drivers/media/usb/gspca/spca505.c
+++ b/drivers/media/usb/gspca/spca505.c
@@ -551,8 +551,8 @@ static int reg_write(struct gspca_dev *gspca_dev,
 			req,
 			USB_TYPE_VENDOR | USB_RECIP_DEVICE,
 			value, index, NULL, 0, 500);
-	PDEBUG(D_USBO, "reg write: 0x%02x,0x%02x:0x%02x, %d",
-		req, index, value, ret);
+	gspca_dbg(gspca_dev, D_USBO, "reg write: 0x%02x,0x%02x:0x%02x, %d\n",
+		  req, index, value, ret);
 	if (ret < 0)
 		pr_err("reg write: error %d\n", ret);
 	return ret;
@@ -650,7 +650,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	ret = reg_read(gspca_dev, 0x06, 0x16);
 
 	if (ret < 0) {
-		PERR("register read failed err: %d", ret);
+		gspca_err(gspca_dev, "register read failed err: %d\n", ret);
 		return ret;
 	}
 	if (ret != 0x0101) {
diff --git a/drivers/media/usb/gspca/spca506.c b/drivers/media/usb/gspca/spca506.c
index 843c93f5acf3..6332b3f0b918 100644
--- a/drivers/media/usb/gspca/spca506.c
+++ b/drivers/media/usb/gspca/spca506.c
@@ -126,7 +126,7 @@ static void spca506_SetNormeInput(struct gspca_dev *gspca_dev,
 	__u8 setbit1 = 0x00;
 	__u8 videomask = 0x00;
 
-	PDEBUG(D_STREAM, "** Open Set Norme **");
+	gspca_dbg(gspca_dev, D_STREAM, "** Open Set Norme **\n");
 	spca506_Initi2c(gspca_dev);
 	/* NTSC bit0 -> 1(525 l) PAL SECAM bit0 -> 0 (625 l) */
 	/* Composite channel bit1 -> 1 S-video bit 1 -> 0 */
@@ -153,8 +153,9 @@ static void spca506_SetNormeInput(struct gspca_dev *gspca_dev,
 
 	sd->norme = norme;
 	sd->channel = channel;
-	PDEBUG(D_STREAM, "Set Video Byte to 0x%2x", videomask);
-	PDEBUG(D_STREAM, "Set Norme: %08x Channel %d", norme, channel);
+	gspca_dbg(gspca_dev, D_STREAM, "Set Video Byte to 0x%2x\n", videomask);
+	gspca_dbg(gspca_dev, D_STREAM, "Set Norme: %08x Channel %d",
+		  norme, channel);
 }
 
 static void spca506_GetNormeInput(struct gspca_dev *gspca_dev,
@@ -166,7 +167,8 @@ static void spca506_GetNormeInput(struct gspca_dev *gspca_dev,
 	   we use your own copy in spca50x struct */
 	*norme = sd->norme;
 	*channel = sd->channel;
-	PDEBUG(D_STREAM, "Get Norme: %d Channel %d", *norme, *channel);
+	gspca_dbg(gspca_dev, D_STREAM, "Get Norme: %d Channel %d\n",
+		  *norme, *channel);
 }
 
 static void spca506_Setsize(struct gspca_dev *gspca_dev, __u16 code,
@@ -174,7 +176,7 @@ static void spca506_Setsize(struct gspca_dev *gspca_dev, __u16 code,
 {
 	struct usb_device *dev = gspca_dev->dev;
 
-	PDEBUG(D_STREAM, "** SetSize **");
+	gspca_dbg(gspca_dev, D_STREAM, "** SetSize **\n");
 	reg_w(dev, 0x04, (0x18 | (code & 0x07)), 0x0000);
 	/* Soft snap 0x40 Hard 0x41 */
 	reg_w(dev, 0x04, 0x41, 0x0001);
@@ -317,7 +319,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	spca506_WriteI2c(gspca_dev, 0x00, 0x60);
 	spca506_WriteI2c(gspca_dev, 0x05, 0x61);
 	spca506_WriteI2c(gspca_dev, 0x9f, 0x62);
-	PDEBUG(D_STREAM, "** Close Init *");
+	gspca_dbg(gspca_dev, D_STREAM, "** Close Init *\n");
 	return 0;
 }
 
@@ -445,7 +447,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	reg_w(dev, 0x02, 0x01, 0x0000);
 	reg_w(dev, 0x03, 0x12, 0x0000);
 	reg_r(gspca_dev, 0x04, 0x0001, 2);
-	PDEBUG(D_STREAM, "webcam started");
+	gspca_dbg(gspca_dev, D_STREAM, "webcam started\n");
 	spca506_GetNormeInput(gspca_dev, &norme, &channel);
 	spca506_SetNormeInput(gspca_dev, norme, channel);
 	return 0;
diff --git a/drivers/media/usb/gspca/spca508.c b/drivers/media/usb/gspca/spca508.c
index 1e0ba6b24e21..d80fd397eaf2 100644
--- a/drivers/media/usb/gspca/spca508.c
+++ b/drivers/media/usb/gspca/spca508.c
@@ -1247,8 +1247,8 @@ static int reg_write(struct gspca_dev *gspca_dev, u16 index, u16 value)
 			0,		/* request */
 			USB_TYPE_VENDOR | USB_RECIP_DEVICE,
 			value, index, NULL, 0, 500);
-	PDEBUG(D_USBO, "reg write i:0x%04x = 0x%02x",
-		index, value);
+	gspca_dbg(gspca_dev, D_USBO, "reg write i:0x%04x = 0x%02x\n",
+		  index, value);
 	if (ret < 0)
 		pr_err("reg write: error %d\n", ret);
 	return ret;
@@ -1269,8 +1269,8 @@ static int reg_read(struct gspca_dev *gspca_dev,
 			index,
 			gspca_dev->usb_buf, 1,
 			500);			/* timeout */
-	PDEBUG(D_USBI, "reg read i:%04x --> %02x",
-		index, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "reg read i:%04x --> %02x\n",
+		  index, gspca_dev->usb_buf[0]);
 	if (ret < 0) {
 		pr_err("reg_read err %d\n", ret);
 		return ret;
@@ -1309,7 +1309,8 @@ static int ssi_w(struct gspca_dev *gspca_dev,
 		if (gspca_dev->usb_buf[0] == 0)
 			break;
 		if (--retry <= 0) {
-			PERR("ssi_w busy %02x", gspca_dev->usb_buf[0]);
+			gspca_err(gspca_dev, "ssi_w busy %02x\n",
+				  gspca_dev->usb_buf[0]);
 			ret = -1;
 			break;
 		}
@@ -1365,14 +1366,17 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	 * is a 508. */
 	data1 = reg_read(gspca_dev, 0x8104);
 	data2 = reg_read(gspca_dev, 0x8105);
-	PDEBUG(D_PROBE, "Webcam Vendor ID: 0x%02x%02x", data2, data1);
+	gspca_dbg(gspca_dev, D_PROBE, "Webcam Vendor ID: 0x%02x%02x\n",
+		  data2, data1);
 
 	data1 = reg_read(gspca_dev, 0x8106);
 	data2 = reg_read(gspca_dev, 0x8107);
-	PDEBUG(D_PROBE, "Webcam Product ID: 0x%02x%02x", data2, data1);
+	gspca_dbg(gspca_dev, D_PROBE, "Webcam Product ID: 0x%02x%02x\n",
+		  data2, data1);
 
 	data1 = reg_read(gspca_dev, 0x8621);
-	PDEBUG(D_PROBE, "Window 1 average luminance: %d", data1);
+	gspca_dbg(gspca_dev, D_PROBE, "Window 1 average luminance: %d\n",
+		  data1);
 
 	cam = &gspca_dev->cam;
 	cam->cam_mode = sif_mode;
diff --git a/drivers/media/usb/gspca/spca561.c b/drivers/media/usb/gspca/spca561.c
index 4ff704cf9ed6..f389a8d0937d 100644
--- a/drivers/media/usb/gspca/spca561.c
+++ b/drivers/media/usb/gspca/spca561.c
@@ -290,7 +290,8 @@ static void reg_w_val(struct gspca_dev *gspca_dev, __u16 index, __u8 value)
 			      0,		/* request */
 			      USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE,
 			      value, index, NULL, 0, 500);
-	PDEBUG(D_USBO, "reg write: 0x%02x:0x%02x", index, value);
+	gspca_dbg(gspca_dev, D_USBO, "reg write: 0x%02x:0x%02x\n",
+		  index, value);
 	if (ret < 0)
 		pr_err("reg write: error %d\n", ret);
 }
@@ -420,7 +421,7 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	data2 = gspca_dev->usb_buf[0];
 	product = (data2 << 8) | data1;
 	if (vendor != id->idVendor || product != id->idProduct) {
-		PDEBUG(D_PROBE, "Bad vendor / product from device");
+		gspca_dbg(gspca_dev, D_PROBE, "Bad vendor / product from device\n");
 		return -EINVAL;
 	}
 
@@ -442,13 +443,13 @@ static int sd_config(struct gspca_dev *gspca_dev,
 /* this function is called at probe and resume time */
 static int sd_init_12a(struct gspca_dev *gspca_dev)
 {
-	PDEBUG(D_STREAM, "Chip revision: 012a");
+	gspca_dbg(gspca_dev, D_STREAM, "Chip revision: 012a\n");
 	init_161rev12A(gspca_dev);
 	return 0;
 }
 static int sd_init_72a(struct gspca_dev *gspca_dev)
 {
-	PDEBUG(D_STREAM, "Chip revision: 072a");
+	gspca_dbg(gspca_dev, D_STREAM, "Chip revision: 072a\n");
 	write_vector(gspca_dev, rev72a_reset);
 	msleep(200);
 	write_vector(gspca_dev, rev72a_init_data1);
@@ -679,25 +680,16 @@ static void do_autogain(struct gspca_dev *gspca_dev)
 		y = (77 * R + 75 * (Gr + Gb) + 29 * B) >> 8;
 		/* u= (128*B-(43*(Gr+Gb+R))) >> 8; */
 		/* v= (128*R-(53*(Gr+Gb))-21*B) >> 8; */
-		/* PDEBUG(D_CONF,"reading Y %d U %d V %d ",y,u,v); */
 
 		if (y < luma_mean - luma_delta ||
 		    y > luma_mean + luma_delta) {
 			expotimes = i2c_read(gspca_dev, 0x09, 0x10);
 			pixelclk = 0x0800;
 			expotimes = expotimes & 0x07ff;
-			/* PDEBUG(D_PACK,
-				"Exposition Times 0x%03X Clock 0x%04X ",
-				expotimes,pixelclk); */
 			gainG = i2c_read(gspca_dev, 0x35, 0x10);
-			/* PDEBUG(D_PACK,
-				"reading Gain register %d", gainG); */
 
 			expotimes += (luma_mean - y) >> spring;
 			gainG += (luma_mean - y) / 50;
-			/* PDEBUG(D_PACK,
-				"compute expotimes %d gain %d",
-				expotimes,gainG); */
 
 			if (gainG > 0x3f)
 				gainG = 0x3f;
@@ -728,7 +720,7 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 
 		/* This should never happen */
 		if (len < 2) {
-			PERR("Short SOF packet, ignoring");
+			gspca_err(gspca_dev, "Short SOF packet, ignoring\n\n\n\n\n");
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			return;
 		}
diff --git a/drivers/media/usb/gspca/sq905.c b/drivers/media/usb/gspca/sq905.c
index f1da34a10ce8..cc8ff41b8ab3 100644
--- a/drivers/media/usb/gspca/sq905.c
+++ b/drivers/media/usb/gspca/sq905.c
@@ -246,9 +246,9 @@ static void sq905_dostream(struct work_struct *work)
 			ret = sq905_read_data(gspca_dev, buffer, data_len, 1);
 			if (ret < 0)
 				goto quit_stream;
-			PDEBUG(D_PACK,
-				"Got %d bytes out of %d for frame",
-				data_len, bytes_left);
+			gspca_dbg(gspca_dev, D_PACK,
+				  "Got %d bytes out of %d for frame\n",
+				  data_len, bytes_left);
 			bytes_left -= data_len;
 			data = buffer;
 			if (!header_read) {
@@ -345,7 +345,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	ret = sq905_command(gspca_dev, SQ905_CLEAR);
 	if (ret < 0)
 		return ret;
-	PDEBUG(D_CONF, "SQ905 camera ID %08x detected", ident);
+	gspca_dbg(gspca_dev, D_CONF, "SQ905 camera ID %08x detected\n", ident);
 	gspca_dev->cam.cam_mode = sq905_mode;
 	gspca_dev->cam.nmodes = ARRAY_SIZE(sq905_mode);
 	if (!(ident & SQ905_HIRES_MASK))
@@ -369,20 +369,20 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	switch (gspca_dev->curr_mode) {
 	default:
 /*	case 2: */
-		PDEBUG(D_STREAM, "Start streaming at high resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at high resolution\n");
 		ret = sq905_command(&dev->gspca_dev, SQ905_CAPTURE_HIGH);
 		break;
 	case 1:
-		PDEBUG(D_STREAM, "Start streaming at medium resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at medium resolution\n");
 		ret = sq905_command(&dev->gspca_dev, SQ905_CAPTURE_MED);
 		break;
 	case 0:
-		PDEBUG(D_STREAM, "Start streaming at low resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at low resolution\n");
 		ret = sq905_command(&dev->gspca_dev, SQ905_CAPTURE_LOW);
 	}
 
 	if (ret < 0) {
-		PERR("Start streaming command failed");
+		gspca_err(gspca_dev, "Start streaming command failed\n");
 		return ret;
 	}
 	/* Start the workqueue function to do the streaming */
diff --git a/drivers/media/usb/gspca/sq905c.c b/drivers/media/usb/gspca/sq905c.c
index 8b4e4948a0cb..5e1269eb7c50 100644
--- a/drivers/media/usb/gspca/sq905c.c
+++ b/drivers/media/usb/gspca/sq905c.c
@@ -154,15 +154,16 @@ static void sq905c_dostream(struct work_struct *work)
 				usb_rcvbulkpipe(gspca_dev->dev, 0x81),
 				buffer, FRAME_HEADER_LEN, &act_len,
 				SQ905C_DATA_TIMEOUT);
-		PDEBUG(D_STREAM,
-			"Got %d bytes out of %d for header",
-			act_len, FRAME_HEADER_LEN);
+		gspca_dbg(gspca_dev, D_STREAM,
+			  "Got %d bytes out of %d for header\n",
+			  act_len, FRAME_HEADER_LEN);
 		if (ret < 0 || act_len < FRAME_HEADER_LEN)
 			goto quit_stream;
 		/* size is read from 4 bytes starting 0x40, little endian */
 		bytes_left = buffer[0x40]|(buffer[0x41]<<8)|(buffer[0x42]<<16)
 					|(buffer[0x43]<<24);
-		PDEBUG(D_STREAM, "bytes_left = 0x%x", bytes_left);
+		gspca_dbg(gspca_dev, D_STREAM, "bytes_left = 0x%x\n",
+			  bytes_left);
 		/* We keep the header. It has other information, too. */
 		packet_type = FIRST_PACKET;
 		gspca_frame_add(gspca_dev, packet_type,
@@ -176,9 +177,9 @@ static void sq905c_dostream(struct work_struct *work)
 				SQ905C_DATA_TIMEOUT);
 			if (ret < 0 || act_len < data_len)
 				goto quit_stream;
-			PDEBUG(D_STREAM,
-				"Got %d bytes out of %d for frame",
-				data_len, bytes_left);
+			gspca_dbg(gspca_dev, D_STREAM,
+				  "Got %d bytes out of %d for frame\n",
+				  data_len, bytes_left);
 			bytes_left -= data_len;
 			if (bytes_left == 0)
 				packet_type = LAST_PACKET;
@@ -205,25 +206,25 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	struct sd *dev = (struct sd *) gspca_dev;
 	int ret;
 
-	PDEBUG(D_PROBE,
-	       "SQ9050 camera detected (vid/pid 0x%04X:0x%04X)",
-	       id->idVendor, id->idProduct);
+	gspca_dbg(gspca_dev, D_PROBE,
+		  "SQ9050 camera detected (vid/pid 0x%04X:0x%04X)\n",
+		  id->idVendor, id->idProduct);
 
 	ret = sq905c_command(gspca_dev, SQ905C_GET_ID, 0);
 	if (ret < 0) {
-		PERR("Get version command failed");
+		gspca_err(gspca_dev, "Get version command failed\n");
 		return ret;
 	}
 
 	ret = sq905c_read(gspca_dev, 0xf5, 0, 20);
 	if (ret < 0) {
-		PERR("Reading version command failed");
+		gspca_err(gspca_dev, "Reading version command failed\n");
 		return ret;
 	}
 	/* Note we leave out the usb id and the manufacturing date */
-	PDEBUG(D_PROBE,
-	       "SQ9050 ID string: %02x - %*ph",
-		gspca_dev->usb_buf[3], 6, gspca_dev->usb_buf + 14);
+	gspca_dbg(gspca_dev, D_PROBE,
+		  "SQ9050 ID string: %02x - %*ph\n",
+		  gspca_dev->usb_buf[3], 6, gspca_dev->usb_buf + 14);
 
 	cam->cam_mode = sq905c_mode;
 	cam->nmodes = 2;
@@ -267,19 +268,19 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	/* "Open the shutter" and set size, to start capture */
 	switch (gspca_dev->pixfmt.width) {
 	case 640:
-		PDEBUG(D_STREAM, "Start streaming at high resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at high resolution\n");
 		dev->cap_mode++;
 		ret = sq905c_command(gspca_dev, SQ905C_CAPTURE_HI,
 						SQ905C_CAPTURE_INDEX);
 		break;
 	default: /* 320 */
-	PDEBUG(D_STREAM, "Start streaming at medium resolution");
+		gspca_dbg(gspca_dev, D_STREAM, "Start streaming at medium resolution\n");
 		ret = sq905c_command(gspca_dev, SQ905C_CAPTURE_MED,
 						SQ905C_CAPTURE_INDEX);
 	}
 
 	if (ret < 0) {
-		PERR("Start streaming command failed");
+		gspca_err(gspca_dev, "Start streaming command failed\n");
 		return ret;
 	}
 	/* Start the workqueue function to do the streaming */
diff --git a/drivers/media/usb/gspca/sq930x.c b/drivers/media/usb/gspca/sq930x.c
index aa9a9411b801..d7cbcf2b3947 100644
--- a/drivers/media/usb/gspca/sq930x.c
+++ b/drivers/media/usb/gspca/sq930x.c
@@ -443,7 +443,7 @@ static void reg_w(struct gspca_dev *gspca_dev, u16 value, u16 index)
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "reg_w v: %04x i: %04x", value, index);
+	gspca_dbg(gspca_dev, D_USBO, "reg_w v: %04x i: %04x\n", value, index);
 	ret = usb_control_msg(gspca_dev->dev,
 			usb_sndctrlpipe(gspca_dev->dev, 0),
 			0x0c,			/* request */
@@ -464,8 +464,8 @@ static void reg_wb(struct gspca_dev *gspca_dev, u16 value, u16 index,
 
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "reg_wb v: %04x i: %04x %02x...%02x",
-			value, index, *data, data[len - 1]);
+	gspca_dbg(gspca_dev, D_USBO, "reg_wb v: %04x i: %04x %02x...%02x\n",
+		  value, index, *data, data[len - 1]);
 	memcpy(gspca_dev->usb_buf, data, len);
 	ret = usb_control_msg(gspca_dev->dev,
 			usb_sndctrlpipe(gspca_dev->dev, 0),
@@ -510,8 +510,8 @@ static void i2c_write(struct sd *sd,
 		*buf++ = cmd->val;
 	}
 
-	PDEBUG(D_USBO, "i2c_w v: %04x i: %04x %02x...%02x",
-			val, idx, gspca_dev->usb_buf[0], buf[-1]);
+	gspca_dbg(gspca_dev, D_USBO, "i2c_w v: %04x i: %04x %02x...%02x\n",
+		  val, idx, gspca_dev->usb_buf[0], buf[-1]);
 	ret = usb_control_msg(gspca_dev->dev,
 			usb_sndctrlpipe(gspca_dev->dev, 0),
 			0x0c,			/* request */
@@ -538,7 +538,7 @@ static void ucbus_write(struct gspca_dev *gspca_dev,
 		return;
 
 	if ((batchsize - 1) * 3 > USB_BUF_SZ) {
-		PERR("Bug: usb_buf overflow\n");
+		gspca_err(gspca_dev, "Bug: usb_buf overflow\n");
 		gspca_dev->usb_err = -ENOMEM;
 		return;
 	}
@@ -560,12 +560,12 @@ static void ucbus_write(struct gspca_dev *gspca_dev,
 			*buf++ = cmd->bw_data;
 		}
 		if (buf != gspca_dev->usb_buf)
-			PDEBUG(D_USBO, "ucbus v: %04x i: %04x %02x...%02x",
-					val, idx,
-					gspca_dev->usb_buf[0], buf[-1]);
+			gspca_dbg(gspca_dev, D_USBO, "ucbus v: %04x i: %04x %02x...%02x\n",
+				  val, idx,
+				  gspca_dev->usb_buf[0], buf[-1]);
 		else
-			PDEBUG(D_USBO, "ucbus v: %04x i: %04x",
-					val, idx);
+			gspca_dbg(gspca_dev, D_USBO, "ucbus v: %04x i: %04x\n",
+				  val, idx);
 		ret = usb_control_msg(gspca_dev->dev,
 				usb_sndctrlpipe(gspca_dev->dev, 0),
 				0x0c,			/* request */
@@ -691,7 +691,7 @@ static void mt9v111_init(struct gspca_dev *gspca_dev)
 			 || gspca_dev->usb_err != 0)
 				break;
 			if (--nwait < 0) {
-				PDEBUG(D_PROBE, "mt9v111_init timeout");
+				gspca_dbg(gspca_dev, D_PROBE, "mt9v111_init timeout\n");
 				gspca_dev->usb_err = -ETIME;
 				return;
 			}
@@ -857,7 +857,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
  * 6: c8 / c9 / ca / cf = mode webcam?, sensor? webcam?
  * 7: 00
  */
-	PDEBUG(D_PROBE, "info: %*ph", 8, gspca_dev->usb_buf);
+	gspca_dbg(gspca_dev, D_PROBE, "info: %*ph\n", 8, gspca_dev->usb_buf);
 
 	bridge_init(sd);
 
@@ -870,7 +870,8 @@ static int sd_init(struct gspca_dev *gspca_dev)
 			cmos_probe(gspca_dev);
 	}
 	if (gspca_dev->usb_err >= 0) {
-		PDEBUG(D_PROBE, "Sensor %s", sensor_tb[sd->sensor].name);
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor %s\n",
+			  sensor_tb[sd->sensor].name);
 		global_init(sd, 1);
 	}
 	return gspca_dev->usb_err;
diff --git a/drivers/media/usb/gspca/stk014.c b/drivers/media/usb/gspca/stk014.c
index daf45db6c404..0d8f489ddef2 100644
--- a/drivers/media/usb/gspca/stk014.c
+++ b/drivers/media/usb/gspca/stk014.c
@@ -290,8 +290,8 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	set_par(gspca_dev, 0x01000000);
 	set_par(gspca_dev, 0x01000000);
 	if (gspca_dev->usb_err >= 0)
-		PDEBUG(D_STREAM, "camera started alt: 0x%02x",
-				gspca_dev->alt);
+		gspca_dbg(gspca_dev, D_STREAM, "camera started alt: 0x%02x\n",
+			  gspca_dev->alt);
 out:
 	return gspca_dev->usb_err;
 }
@@ -312,7 +312,7 @@ static void sd_stopN(struct gspca_dev *gspca_dev)
 	reg_w(gspca_dev, 0x0640, 0);
 	reg_w(gspca_dev, 0x0650, 0);
 	reg_w(gspca_dev, 0x0660, 0);
-	PDEBUG(D_STREAM, "camera stopped");
+	gspca_dbg(gspca_dev, D_STREAM, "camera stopped\n");
 }
 
 static void sd_pkt_scan(struct gspca_dev *gspca_dev,
diff --git a/drivers/media/usb/gspca/stk1135.c b/drivers/media/usb/gspca/stk1135.c
index 3ab5ec2ca4bd..6f52a480c2d8 100644
--- a/drivers/media/usb/gspca/stk1135.c
+++ b/drivers/media/usb/gspca/stk1135.c
@@ -67,7 +67,8 @@ static u8 reg_r(struct gspca_dev *gspca_dev, u16 index)
 			gspca_dev->usb_buf, 1,
 			500);
 
-	PDEBUG(D_USBI, "reg_r 0x%x=0x%02x", index, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_USBI, "reg_r 0x%x=0x%02x\n",
+		  index, gspca_dev->usb_buf[0]);
 	if (ret < 0) {
 		pr_err("reg_r 0x%x err %d\n", index, ret);
 		gspca_dev->usb_err = ret;
@@ -93,7 +94,7 @@ static void reg_w(struct gspca_dev *gspca_dev, u16 index, u8 val)
 			NULL,
 			0,
 			500);
-	PDEBUG(D_USBO, "reg_w 0x%x:=0x%02x", index, val);
+	gspca_dbg(gspca_dev, D_USBO, "reg_w 0x%x:=0x%02x\n", index, val);
 	if (ret < 0) {
 		pr_err("reg_w 0x%x err %d\n", index, ret);
 		gspca_dev->usb_err = ret;
@@ -468,8 +469,8 @@ static int sd_start(struct gspca_dev *gspca_dev)
 	reg_w_mask(gspca_dev, STK1135_REG_SCTRL, 0x80, 0x80);
 
 	if (gspca_dev->usb_err >= 0)
-		PDEBUG(D_STREAM, "camera started alt: 0x%02x",
-				gspca_dev->alt);
+		gspca_dbg(gspca_dev, D_STREAM, "camera started alt: 0x%02x\n",
+			  gspca_dev->alt);
 
 	sd->pkt_seq = 0;
 
@@ -484,7 +485,7 @@ static void sd_stopN(struct gspca_dev *gspca_dev)
 
 	stk1135_camera_disable(gspca_dev);
 
-	PDEBUG(D_STREAM, "camera stopped");
+	gspca_dbg(gspca_dev, D_STREAM, "camera stopped\n");
 }
 
 static void sd_pkt_scan(struct gspca_dev *gspca_dev,
@@ -499,7 +500,7 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 	u8 seq;
 
 	if (len < 4) {
-		PDEBUG(D_PACK, "received short packet (less than 4 bytes)");
+		gspca_dbg(gspca_dev, D_PACK, "received short packet (less than 4 bytes)\n");
 		return;
 	}
 
@@ -515,7 +516,7 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 	if (!(hdr->flags & STK1135_HDR_FRAME_START)) {
 		seq = hdr->seq & STK1135_HDR_SEQ_MASK;
 		if (seq != sd->pkt_seq) {
-			PDEBUG(D_PACK, "received out-of-sequence packet");
+			gspca_dbg(gspca_dev, D_PACK, "received out-of-sequence packet\n");
 			/* resync sequence and discard packet */
 			sd->pkt_seq = seq;
 			gspca_dev->last_packet_type = DISCARD_PACKET;
diff --git a/drivers/media/usb/gspca/stv0680.c b/drivers/media/usb/gspca/stv0680.c
index 29a65d05cbb2..3ff5ed74bd9f 100644
--- a/drivers/media/usb/gspca/stv0680.c
+++ b/drivers/media/usb/gspca/stv0680.c
@@ -82,8 +82,8 @@ static int stv_sndctrl(struct gspca_dev *gspca_dev, int set, u8 req, u16 val,
 static int stv0680_handle_error(struct gspca_dev *gspca_dev, int ret)
 {
 	stv_sndctrl(gspca_dev, 0, 0x80, 0, 0x02); /* Get Last Error */
-	PERR("last error: %i,  command = 0x%x",
-	       gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
+	gspca_err(gspca_dev, "last error: %i,  command = 0x%x\n",
+		  gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
 	return ret;
 }
 
@@ -94,7 +94,7 @@ static int stv0680_get_video_mode(struct gspca_dev *gspca_dev)
 	gspca_dev->usb_buf[0] = 0x0f;
 
 	if (stv_sndctrl(gspca_dev, 0, 0x87, 0, 0x08) != 0x08) {
-		PERR("Get_Camera_Mode failed");
+		gspca_err(gspca_dev, "Get_Camera_Mode failed\n");
 		return stv0680_handle_error(gspca_dev, -EIO);
 	}
 
@@ -112,13 +112,13 @@ static int stv0680_set_video_mode(struct gspca_dev *gspca_dev, u8 mode)
 	gspca_dev->usb_buf[0] = mode;
 
 	if (stv_sndctrl(gspca_dev, 3, 0x07, 0x0100, 0x08) != 0x08) {
-		PERR("Set_Camera_Mode failed");
+		gspca_err(gspca_dev, "Set_Camera_Mode failed\n");
 		return stv0680_handle_error(gspca_dev, -EIO);
 	}
 
 	/* Verify we got what we've asked for */
 	if (stv0680_get_video_mode(gspca_dev) != mode) {
-		PERR("Error setting camera video mode!");
+		gspca_err(gspca_dev, "Error setting camera video mode!\n");
 		return -EIO;
 	}
 
@@ -142,7 +142,7 @@ static int sd_config(struct gspca_dev *gspca_dev,
 	/* ping camera to be sure STV0680 is present */
 	if (stv_sndctrl(gspca_dev, 0, 0x88, 0x5678, 0x02) != 0x02 ||
 	    gspca_dev->usb_buf[0] != 0x56 || gspca_dev->usb_buf[1] != 0x78) {
-		PERR("STV(e): camera ping failed!!");
+		gspca_err(gspca_dev, "STV(e): camera ping failed!!\n");
 		return stv0680_handle_error(gspca_dev, -ENODEV);
 	}
 
@@ -152,7 +152,7 @@ static int sd_config(struct gspca_dev *gspca_dev,
 
 	if (stv_sndctrl(gspca_dev, 2, 0x06, 0x0200, 0x22) != 0x22 ||
 	    gspca_dev->usb_buf[7] != 0xa0 || gspca_dev->usb_buf[8] != 0x23) {
-		PERR("Could not get descriptor 0200.");
+		gspca_err(gspca_dev, "Could not get descriptor 0200\n");
 		return stv0680_handle_error(gspca_dev, -ENODEV);
 	}
 	if (stv_sndctrl(gspca_dev, 0, 0x8a, 0, 0x02) != 0x02)
@@ -163,17 +163,17 @@ static int sd_config(struct gspca_dev *gspca_dev,
 		return stv0680_handle_error(gspca_dev, -ENODEV);
 
 	if (!(gspca_dev->usb_buf[7] & 0x09)) {
-		PERR("Camera supports neither CIF nor QVGA mode");
+		gspca_err(gspca_dev, "Camera supports neither CIF nor QVGA mode\n");
 		return -ENODEV;
 	}
 	if (gspca_dev->usb_buf[7] & 0x01)
-		PDEBUG(D_PROBE, "Camera supports CIF mode");
+		gspca_dbg(gspca_dev, D_PROBE, "Camera supports CIF mode\n");
 	if (gspca_dev->usb_buf[7] & 0x02)
-		PDEBUG(D_PROBE, "Camera supports VGA mode");
+		gspca_dbg(gspca_dev, D_PROBE, "Camera supports VGA mode\n");
 	if (gspca_dev->usb_buf[7] & 0x04)
-		PDEBUG(D_PROBE, "Camera supports QCIF mode");
+		gspca_dbg(gspca_dev, D_PROBE, "Camera supports QCIF mode\n");
 	if (gspca_dev->usb_buf[7] & 0x08)
-		PDEBUG(D_PROBE, "Camera supports QVGA mode");
+		gspca_dbg(gspca_dev, D_PROBE, "Camera supports QVGA mode\n");
 
 	if (gspca_dev->usb_buf[7] & 0x01)
 		sd->video_mode = 0x00; /* CIF */
@@ -181,12 +181,12 @@ static int sd_config(struct gspca_dev *gspca_dev,
 		sd->video_mode = 0x03; /* QVGA */
 
 	/* FW rev, ASIC rev, sensor ID  */
-	PDEBUG(D_PROBE, "Firmware rev is %i.%i",
-	       gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
-	PDEBUG(D_PROBE, "ASIC rev is %i.%i",
-	       gspca_dev->usb_buf[2], gspca_dev->usb_buf[3]);
-	PDEBUG(D_PROBE, "Sensor ID is %i",
-	       (gspca_dev->usb_buf[4]*16) + (gspca_dev->usb_buf[5]>>4));
+	gspca_dbg(gspca_dev, D_PROBE, "Firmware rev is %i.%i\n",
+		  gspca_dev->usb_buf[0], gspca_dev->usb_buf[1]);
+	gspca_dbg(gspca_dev, D_PROBE, "ASIC rev is %i.%i",
+		  gspca_dev->usb_buf[2], gspca_dev->usb_buf[3]);
+	gspca_dbg(gspca_dev, D_PROBE, "Sensor ID is %i",
+		  (gspca_dev->usb_buf[4]*16) + (gspca_dev->usb_buf[5]>>4));
 
 
 	ret = stv0680_get_video_mode(gspca_dev);
diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx.c b/drivers/media/usb/gspca/stv06xx/stv06xx.c
index e72c3e1ab9ff..6080a35310ca 100644
--- a/drivers/media/usb/gspca/stv06xx/stv06xx.c
+++ b/drivers/media/usb/gspca/stv06xx/stv06xx.c
@@ -51,8 +51,8 @@ int stv06xx_write_bridge(struct sd *sd, u16 address, u16 i2c_data)
 			      0x04, 0x40, address, 0, buf, len,
 			      STV06XX_URB_MSG_TIMEOUT);
 
-	PDEBUG(D_CONF, "Written 0x%x to address 0x%x, status: %d",
-	       i2c_data, address, err);
+	gspca_dbg(gspca_dev, D_CONF, "Written 0x%x to address 0x%x, status: %d\n",
+		  i2c_data, address, err);
 
 	return (err < 0) ? err : 0;
 }
@@ -70,8 +70,8 @@ int stv06xx_read_bridge(struct sd *sd, u16 address, u8 *i2c_data)
 
 	*i2c_data = buf[0];
 
-	PDEBUG(D_CONF, "Reading 0x%x from address 0x%x, status %d",
-	       *i2c_data, address, err);
+	gspca_dbg(gspca_dev, D_CONF, "Reading 0x%x from address 0x%x, status %d\n",
+		  *i2c_data, address, err);
 
 	return (err < 0) ? err : 0;
 }
@@ -113,15 +113,16 @@ int stv06xx_write_sensor_bytes(struct sd *sd, const u8 *data, u8 len)
 	struct usb_device *udev = sd->gspca_dev.dev;
 	__u8 *buf = sd->gspca_dev.usb_buf;
 
-	PDEBUG(D_CONF, "I2C: Command buffer contains %d entries", len);
+	gspca_dbg(gspca_dev, D_CONF, "I2C: Command buffer contains %d entries\n",
+		  len);
 	for (i = 0; i < len;) {
 		/* Build the command buffer */
 		memset(buf, 0, I2C_BUFFER_LENGTH);
 		for (j = 0; j < I2C_MAX_BYTES && i < len; j++, i++) {
 			buf[j] = data[2*i];
 			buf[0x10 + j] = data[2*i+1];
-			PDEBUG(D_CONF, "I2C: Writing 0x%02x to reg 0x%02x",
-			data[2*i+1], data[2*i]);
+			gspca_dbg(gspca_dev, D_CONF, "I2C: Writing 0x%02x to reg 0x%02x\n",
+				  data[2*i+1], data[2*i]);
 		}
 		buf[0x20] = sd->sensor->i2c_addr;
 		buf[0x21] = j - 1; /* Number of commands to send - 1 */
@@ -143,7 +144,8 @@ int stv06xx_write_sensor_words(struct sd *sd, const u16 *data, u8 len)
 	struct usb_device *udev = sd->gspca_dev.dev;
 	__u8 *buf = sd->gspca_dev.usb_buf;
 
-	PDEBUG(D_CONF, "I2C: Command buffer contains %d entries", len);
+	gspca_dbg(gspca_dev, D_CONF, "I2C: Command buffer contains %d entries\n",
+		  len);
 
 	for (i = 0; i < len;) {
 		/* Build the command buffer */
@@ -152,8 +154,8 @@ int stv06xx_write_sensor_words(struct sd *sd, const u16 *data, u8 len)
 			buf[j] = data[2*i];
 			buf[0x10 + j * 2] = data[2*i+1];
 			buf[0x10 + j * 2 + 1] = data[2*i+1] >> 8;
-			PDEBUG(D_CONF, "I2C: Writing 0x%04x to reg 0x%02x",
-				data[2*i+1], data[2*i]);
+			gspca_dbg(gspca_dev, D_CONF, "I2C: Writing 0x%04x to reg 0x%02x\n",
+				  data[2*i+1], data[2*i]);
 		}
 		buf[0x20] = sd->sensor->i2c_addr;
 		buf[0x21] = j - 1; /* Number of commands to send - 1 */
@@ -205,8 +207,8 @@ int stv06xx_read_sensor(struct sd *sd, const u8 address, u16 *value)
 	else
 		*value = buf[0];
 
-	PDEBUG(D_CONF, "I2C: Read 0x%x from address 0x%x, status: %d",
-	       *value, address, err);
+	gspca_dbg(gspca_dev, D_CONF, "I2C: Read 0x%x from address 0x%x, status: %d\n",
+		  *value, address, err);
 
 	return (err < 0) ? err : 0;
 }
@@ -249,7 +251,7 @@ static int stv06xx_init(struct gspca_dev *gspca_dev)
 	struct sd *sd = (struct sd *) gspca_dev;
 	int err;
 
-	PDEBUG(D_PROBE, "Initializing camera");
+	gspca_dbg(gspca_dev, D_PROBE, "Initializing camera\n");
 
 	/* Let the usb init settle for a bit
 	   before performing the initialization */
@@ -268,7 +270,7 @@ static int stv06xx_init_controls(struct gspca_dev *gspca_dev)
 {
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_PROBE, "Initializing controls");
+	gspca_dbg(gspca_dev, D_PROBE, "Initializing controls\n");
 
 	gspca_dev->vdev.ctrl_handler = &gspca_dev->ctrl_handler;
 	return sd->sensor->init_controls(sd);
@@ -285,7 +287,7 @@ static int stv06xx_start(struct gspca_dev *gspca_dev)
 	intf = usb_ifnum_to_if(sd->gspca_dev.dev, sd->gspca_dev.iface);
 	alt = usb_altnum_to_altsetting(intf, sd->gspca_dev.alt);
 	if (!alt) {
-		PERR("Couldn't get altsetting");
+		gspca_err(gspca_dev, "Couldn't get altsetting\n");
 		return -EIO;
 	}
 
@@ -304,9 +306,9 @@ static int stv06xx_start(struct gspca_dev *gspca_dev)
 
 out:
 	if (err < 0)
-		PDEBUG(D_STREAM, "Starting stream failed");
+		gspca_dbg(gspca_dev, D_STREAM, "Starting stream failed\n");
 	else
-		PDEBUG(D_STREAM, "Started streaming");
+		gspca_dbg(gspca_dev, D_STREAM, "Started streaming\n");
 
 	return (err < 0) ? err : 0;
 }
@@ -343,7 +345,7 @@ static int stv06xx_isoc_nego(struct gspca_dev *gspca_dev)
 
 	ret = usb_set_interface(gspca_dev->dev, gspca_dev->iface, 1);
 	if (ret < 0)
-		PERR("set alt 1 err %d", ret);
+		gspca_err(gspca_dev, "set alt 1 err %d\n", ret);
 
 	return ret;
 }
@@ -362,9 +364,9 @@ static void stv06xx_stopN(struct gspca_dev *gspca_dev)
 
 out:
 	if (err < 0)
-		PDEBUG(D_STREAM, "Failed to stop stream");
+		gspca_dbg(gspca_dev, D_STREAM, "Failed to stop stream\n");
 	else
-		PDEBUG(D_STREAM, "Stopped streaming");
+		gspca_dbg(gspca_dev, D_STREAM, "Stopped streaming\n");
 }
 
 /*
@@ -385,7 +387,7 @@ static void stv06xx_pkt_scan(struct gspca_dev *gspca_dev,
 {
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_PACK, "Packet of length %d arrived", len);
+	gspca_dbg(gspca_dev, D_PACK, "Packet of length %d arrived\n", len);
 
 	/* A packet may contain several frames
 	   loop until the whole packet is reached */
@@ -393,7 +395,7 @@ static void stv06xx_pkt_scan(struct gspca_dev *gspca_dev,
 		int id, chunk_len;
 
 		if (len < 4) {
-			PDEBUG(D_PACK, "Packet is smaller than 4 bytes");
+			gspca_dbg(gspca_dev, D_PACK, "Packet is smaller than 4 bytes\n");
 			return;
 		}
 
@@ -402,13 +404,14 @@ static void stv06xx_pkt_scan(struct gspca_dev *gspca_dev,
 
 		/* Capture the chunk length */
 		chunk_len = (data[2] << 8) | data[3];
-		PDEBUG(D_PACK, "Chunk id: %x, length: %d", id, chunk_len);
+		gspca_dbg(gspca_dev, D_PACK, "Chunk id: %x, length: %d\n",
+			  id, chunk_len);
 
 		data += 4;
 		len -= 4;
 
 		if (len < chunk_len) {
-			PERR("URB packet length is smaller than the specified chunk length");
+			gspca_err(gspca_dev, "URB packet length is smaller than the specified chunk length\n");
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			return;
 		}
@@ -421,7 +424,7 @@ static void stv06xx_pkt_scan(struct gspca_dev *gspca_dev,
 		case 0x0200:
 		case 0x4200:
 frame_data:
-			PDEBUG(D_PACK, "Frame data packet detected");
+			gspca_dbg(gspca_dev, D_PACK, "Frame data packet detected\n");
 
 			if (sd->to_skip) {
 				int skip = (sd->to_skip < chunk_len) ?
@@ -440,7 +443,7 @@ static void stv06xx_pkt_scan(struct gspca_dev *gspca_dev,
 		case 0x8005:
 		case 0xc001:
 		case 0xc005:
-			PDEBUG(D_PACK, "Starting new frame");
+			gspca_dbg(gspca_dev, D_PACK, "Starting new frame\n");
 
 			/* Create a new frame, chunk length should be zero */
 			gspca_frame_add(gspca_dev, FIRST_PACKET,
@@ -450,40 +453,41 @@ static void stv06xx_pkt_scan(struct gspca_dev *gspca_dev,
 				sd->to_skip = gspca_dev->pixfmt.width * 4;
 
 			if (chunk_len)
-				PERR("Chunk length is non-zero on a SOF");
+				gspca_err(gspca_dev, "Chunk length is non-zero on a SOF\n");
 			break;
 
 		case 0x8002:
 		case 0x8006:
 		case 0xc002:
-			PDEBUG(D_PACK, "End of frame detected");
+			gspca_dbg(gspca_dev, D_PACK, "End of frame detected\n");
 
 			/* Complete the last frame (if any) */
 			gspca_frame_add(gspca_dev, LAST_PACKET,
 					NULL, 0);
 
 			if (chunk_len)
-				PERR("Chunk length is non-zero on a EOF");
+				gspca_err(gspca_dev, "Chunk length is non-zero on a EOF\n");
 			break;
 
 		case 0x0005:
-			PDEBUG(D_PACK, "Chunk 0x005 detected");
+			gspca_dbg(gspca_dev, D_PACK, "Chunk 0x005 detected\n");
 			/* Unknown chunk with 11 bytes of data,
 			   occurs just before end of each frame
 			   in compressed mode */
 			break;
 
 		case 0x0100:
-			PDEBUG(D_PACK, "Chunk 0x0100 detected");
+			gspca_dbg(gspca_dev, D_PACK, "Chunk 0x0100 detected\n");
 			/* Unknown chunk with 2 bytes of data,
 			   occurs 2-3 times per USB interrupt */
 			break;
 		case 0x42ff:
-			PDEBUG(D_PACK, "Chunk 0x42ff detected");
+			gspca_dbg(gspca_dev, D_PACK, "Chunk 0x42ff detected\n");
 			/* Special chunk seen sometimes on the ST6422 */
 			break;
 		default:
-			PDEBUG(D_PACK, "Unknown chunk 0x%04x detected", id);
+			gspca_dbg(gspca_dev, D_PACK, "Unknown chunk 0x%04x detected\n",
+				  id);
 			/* Unknown chunk */
 		}
 		data    += chunk_len;
@@ -539,7 +543,7 @@ static int stv06xx_config(struct gspca_dev *gspca_dev,
 {
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_PROBE, "Configuring camera");
+	gspca_dbg(gspca_dev, D_PROBE, "Configuring camera\n");
 
 	sd->bridge = id->driver_info;
 	gspca_dev->sd_desc = &sd_desc;
@@ -575,7 +579,7 @@ static int stv06xx_config(struct gspca_dev *gspca_dev,
 
 /* -- module initialisation -- */
 static const struct usb_device_id device_table[] = {
-	{USB_DEVICE(0x046d, 0x0840), .driver_info = BRIDGE_STV600 }, 	/* QuickCam Express */
+	{USB_DEVICE(0x046d, 0x0840), .driver_info = BRIDGE_STV600 },	/* QuickCam Express */
 	{USB_DEVICE(0x046d, 0x0850), .driver_info = BRIDGE_STV610 },	/* LEGO cam / QuickCam Web */
 	{USB_DEVICE(0x046d, 0x0870), .driver_info = BRIDGE_STV602 },	/* Dexxa WebCam USB */
 	{USB_DEVICE(0x046D, 0x08F0), .driver_info = BRIDGE_ST6422 },	/* QuickCam Messenger */
@@ -598,7 +602,7 @@ static void sd_disconnect(struct usb_interface *intf)
 	struct gspca_dev *gspca_dev = usb_get_intfdata(intf);
 	struct sd *sd = (struct sd *) gspca_dev;
 	void *priv = sd->sensor_priv;
-	PDEBUG(D_PROBE, "Disconnecting the stv06xx device");
+	gspca_dbg(gspca_dev, D_PROBE, "Disconnecting the stv06xx device\n");
 
 	sd->sensor = NULL;
 	gspca_disconnect(intf);
diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx.h b/drivers/media/usb/gspca/stv06xx/stv06xx.h
index f9d74e4d7cf9..480186706bba 100644
--- a/drivers/media/usb/gspca/stv06xx/stv06xx.h
+++ b/drivers/media/usb/gspca/stv06xx/stv06xx.h
@@ -59,7 +59,7 @@
 
 /* Refers to the CIF 352x288 and QCIF 176x144 */
 /* 1: 288 lines, 2: 144 lines */
-#define STV_Y_CTRL		        0x15c3
+#define STV_Y_CTRL			0x15c3
 
 #define STV_RESET                       0x1620
 
diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c b/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c
index 28252f6c4afd..d8db2c89718f 100644
--- a/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c
+++ b/drivers/media/usb/gspca/stv06xx/stv06xx_hdcs.c
@@ -251,8 +251,8 @@ static int hdcs_set_exposure(struct gspca_dev *gspca_dev, __s32 val)
 		if (err < 0)
 			return err;
 	}
-	PDEBUG(D_CONF, "Writing exposure %d, rowexp %d, srowexp %d",
-	       val, rowexp, srowexp);
+	gspca_dbg(gspca_dev, D_CONF, "Writing exposure %d, rowexp %d, srowexp %d\n",
+		  val, rowexp, srowexp);
 	return err;
 }
 
@@ -276,7 +276,7 @@ static int hdcs_set_gains(struct sd *sd, u8 g)
 
 static int hdcs_set_gain(struct gspca_dev *gspca_dev, __s32 val)
 {
-	PDEBUG(D_CONF, "Writing gain %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Writing gain %d\n", val);
 	return hdcs_set_gains((struct sd *) gspca_dev,
 			       val & 0xff);
 }
@@ -465,7 +465,7 @@ static int hdcs_start(struct sd *sd)
 {
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 
-	PDEBUG(D_STREAM, "Starting stream");
+	gspca_dbg(gspca_dev, D_STREAM, "Starting stream\n");
 
 	return hdcs_set_state(sd, HDCS_STATE_RUN);
 }
@@ -474,7 +474,7 @@ static int hdcs_stop(struct sd *sd)
 {
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 
-	PDEBUG(D_STREAM, "Halting stream");
+	gspca_dbg(gspca_dev, D_STREAM, "Halting stream\n");
 
 	return hdcs_set_state(sd, HDCS_STATE_SLEEP);
 }
diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx_pb0100.c b/drivers/media/usb/gspca/stv06xx/stv06xx_pb0100.c
index e1ce96e9405f..7374aeb0a67a 100644
--- a/drivers/media/usb/gspca/stv06xx/stv06xx_pb0100.c
+++ b/drivers/media/usb/gspca/stv06xx/stv06xx_pb0100.c
@@ -229,7 +229,7 @@ static int pb0100_start(struct sd *sd)
 	}
 
 	err = stv06xx_write_sensor(sd, PB_CONTROL, BIT(5)|BIT(3)|BIT(1));
-	PDEBUG(D_STREAM, "Started stream, status: %d", err);
+	gspca_dbg(gspca_dev, D_STREAM, "Started stream, status: %d\n", err);
 
 	return (err < 0) ? err : 0;
 }
@@ -247,7 +247,7 @@ static int pb0100_stop(struct sd *sd)
 	/* Set bit 1 to zero */
 	err = stv06xx_write_sensor(sd, PB_CONTROL, BIT(5)|BIT(3));
 
-	PDEBUG(D_STREAM, "Halting stream");
+	gspca_dbg(gspca_dev, D_STREAM, "Halting stream\n");
 out:
 	return (err < 0) ? err : 0;
 }
@@ -332,7 +332,8 @@ static int pb0100_set_gain(struct gspca_dev *gspca_dev, __s32 val)
 	err = stv06xx_write_sensor(sd, PB_G1GAIN, val);
 	if (!err)
 		err = stv06xx_write_sensor(sd, PB_G2GAIN, val);
-	PDEBUG(D_CONF, "Set green gain to %d, status: %d", val, err);
+	gspca_dbg(gspca_dev, D_CONF, "Set green gain to %d, status: %d\n",
+		  val, err);
 
 	if (!err)
 		err = pb0100_set_red_balance(gspca_dev, ctrls->red->val);
@@ -355,7 +356,8 @@ static int pb0100_set_red_balance(struct gspca_dev *gspca_dev, __s32 val)
 		val = 255;
 
 	err = stv06xx_write_sensor(sd, PB_RGAIN, val);
-	PDEBUG(D_CONF, "Set red gain to %d, status: %d", val, err);
+	gspca_dbg(gspca_dev, D_CONF, "Set red gain to %d, status: %d\n",
+		  val, err);
 
 	return err;
 }
@@ -373,7 +375,8 @@ static int pb0100_set_blue_balance(struct gspca_dev *gspca_dev, __s32 val)
 		val = 255;
 
 	err = stv06xx_write_sensor(sd, PB_BGAIN, val);
-	PDEBUG(D_CONF, "Set blue gain to %d, status: %d", val, err);
+	gspca_dbg(gspca_dev, D_CONF, "Set blue gain to %d, status: %d\n",
+		  val, err);
 
 	return err;
 }
@@ -384,7 +387,8 @@ static int pb0100_set_exposure(struct gspca_dev *gspca_dev, __s32 val)
 	int err;
 
 	err = stv06xx_write_sensor(sd, PB_RINTTIME, val);
-	PDEBUG(D_CONF, "Set exposure to %d, status: %d", val, err);
+	gspca_dbg(gspca_dev, D_CONF, "Set exposure to %d, status: %d\n",
+		  val, err);
 
 	return err;
 }
@@ -404,8 +408,8 @@ static int pb0100_set_autogain(struct gspca_dev *gspca_dev, __s32 val)
 		val = 0;
 
 	err = stv06xx_write_sensor(sd, PB_EXPGAIN, val);
-	PDEBUG(D_CONF, "Set autogain to %d (natural: %d), status: %d",
-	       val, ctrls->natural->val, err);
+	gspca_dbg(gspca_dev, D_CONF, "Set autogain to %d (natural: %d), status: %d\n",
+		  val, ctrls->natural->val, err);
 
 	return err;
 }
@@ -426,7 +430,8 @@ static int pb0100_set_autogain_target(struct gspca_dev *gspca_dev, __s32 val)
 	if (!err)
 		err = stv06xx_write_sensor(sd, PB_R22, darkpixels);
 
-	PDEBUG(D_CONF, "Set autogain target to %d, status: %d", val, err);
+	gspca_dbg(gspca_dev, D_CONF, "Set autogain target to %d, status: %d\n",
+		  val, err);
 
 	return err;
 }
diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx_st6422.c b/drivers/media/usb/gspca/stv06xx/stv06xx_st6422.c
index 4b76070515b5..51a135c2f9f7 100644
--- a/drivers/media/usb/gspca/stv06xx/stv06xx_st6422.c
+++ b/drivers/media/usb/gspca/stv06xx/stv06xx_st6422.c
@@ -277,7 +277,7 @@ static int st6422_stop(struct sd *sd)
 {
 	struct gspca_dev *gspca_dev = (struct gspca_dev *)sd;
 
-	PDEBUG(D_STREAM, "Halting stream");
+	gspca_dbg(gspca_dev, D_STREAM, "Halting stream\n");
 
 	return 0;
 }
diff --git a/drivers/media/usb/gspca/stv06xx/stv06xx_vv6410.c b/drivers/media/usb/gspca/stv06xx/stv06xx_vv6410.c
index d265e6b00994..b2f16c2754fb 100644
--- a/drivers/media/usb/gspca/stv06xx/stv06xx_vv6410.c
+++ b/drivers/media/usb/gspca/stv06xx/stv06xx_vv6410.c
@@ -129,7 +129,7 @@ static int vv6410_start(struct sd *sd)
 	u32 priv = cam->cam_mode[sd->gspca_dev.curr_mode].priv;
 
 	if (priv & VV6410_SUBSAMPLE) {
-		PDEBUG(D_CONF, "Enabling subsampling");
+		gspca_dbg(gspca_dev, D_CONF, "Enabling subsampling\n");
 		stv06xx_write_bridge(sd, STV_Y_CTRL, 0x02);
 		stv06xx_write_bridge(sd, STV_X_CTRL, 0x06);
 
@@ -150,7 +150,7 @@ static int vv6410_start(struct sd *sd)
 	if (err < 0)
 		return err;
 
-	PDEBUG(D_STREAM, "Starting stream");
+	gspca_dbg(gspca_dev, D_STREAM, "Starting stream\n");
 
 	return 0;
 }
@@ -169,7 +169,7 @@ static int vv6410_stop(struct sd *sd)
 	if (err < 0)
 		return err;
 
-	PDEBUG(D_STREAM, "Halting stream");
+	gspca_dbg(gspca_dev, D_STREAM, "Halting stream\n");
 
 	return 0;
 }
@@ -203,7 +203,7 @@ static int vv6410_set_hflip(struct gspca_dev *gspca_dev, __s32 val)
 	else
 		i2c_data &= ~VV6410_HFLIP;
 
-	PDEBUG(D_CONF, "Set horizontal flip to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set horizontal flip to %d\n", val);
 	err = stv06xx_write_sensor(sd, VV6410_DATAFORMAT, i2c_data);
 
 	return (err < 0) ? err : 0;
@@ -224,7 +224,7 @@ static int vv6410_set_vflip(struct gspca_dev *gspca_dev, __s32 val)
 	else
 		i2c_data &= ~VV6410_VFLIP;
 
-	PDEBUG(D_CONF, "Set vertical flip to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set vertical flip to %d\n", val);
 	err = stv06xx_write_sensor(sd, VV6410_DATAFORMAT, i2c_data);
 
 	return (err < 0) ? err : 0;
@@ -235,7 +235,7 @@ static int vv6410_set_analog_gain(struct gspca_dev *gspca_dev, __s32 val)
 	int err;
 	struct sd *sd = (struct sd *) gspca_dev;
 
-	PDEBUG(D_CONF, "Set analog gain to %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Set analog gain to %d\n", val);
 	err = stv06xx_write_sensor(sd, VV6410_ANALOGGAIN, 0xf0 | (val & 0xf));
 
 	return (err < 0) ? err : 0;
@@ -252,8 +252,8 @@ static int vv6410_set_exposure(struct gspca_dev *gspca_dev, __s32 val)
 	fine = val % VV6410_CIF_LINELENGTH;
 	coarse = min(512, val / VV6410_CIF_LINELENGTH);
 
-	PDEBUG(D_CONF, "Set coarse exposure to %d, fine exposure to %d",
-	       coarse, fine);
+	gspca_dbg(gspca_dev, D_CONF, "Set coarse exposure to %d, fine exposure to %d\n",
+		  coarse, fine);
 
 	err = stv06xx_write_sensor(sd, VV6410_FINEH, fine >> 8);
 	if (err < 0)
diff --git a/drivers/media/usb/gspca/sunplus.c b/drivers/media/usb/gspca/sunplus.c
index 8c2785aea3cd..437a3367ab97 100644
--- a/drivers/media/usb/gspca/sunplus.c
+++ b/drivers/media/usb/gspca/sunplus.c
@@ -248,7 +248,7 @@ static void reg_r(struct gspca_dev *gspca_dev,
 	int ret;
 
 	if (len > USB_BUF_SZ) {
-		PERR("reg_r: buffer overflow\n");
+		gspca_err(gspca_dev, "reg_r: buffer overflow\n");
 		return;
 	}
 	if (gspca_dev->usb_err < 0)
@@ -311,8 +311,8 @@ static void reg_w_riv(struct gspca_dev *gspca_dev,
 		gspca_dev->usb_err = ret;
 		return;
 	}
-	PDEBUG(D_USBO, "reg_w_riv: 0x%02x,0x%04x:0x%04x",
-		req, index, value);
+	gspca_dbg(gspca_dev, D_USBO, "reg_w_riv: 0x%02x,0x%04x:0x%04x\n",
+		  req, index, value);
 }
 
 static void write_vector(struct gspca_dev *gspca_dev,
@@ -343,12 +343,14 @@ static void spca504_acknowledged_command(struct gspca_dev *gspca_dev,
 {
 	reg_w_riv(gspca_dev, req, idx, val);
 	reg_r(gspca_dev, 0x01, 0x0001, 1);
-	PDEBUG(D_FRAM, "before wait 0x%04x", gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_FRAM, "before wait 0x%04x\n",
+		  gspca_dev->usb_buf[0]);
 	reg_w_riv(gspca_dev, req, idx, val);
 
 	msleep(200);
 	reg_r(gspca_dev, 0x01, 0x0001, 1);
-	PDEBUG(D_FRAM, "after wait 0x%04x", gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_FRAM, "after wait 0x%04x\n",
+		  gspca_dev->usb_buf[0]);
 }
 
 static void spca504_read_info(struct gspca_dev *gspca_dev)
@@ -363,10 +365,10 @@ static void spca504_read_info(struct gspca_dev *gspca_dev)
 		reg_r(gspca_dev, 0, i, 1);
 		info[i] = gspca_dev->usb_buf[0];
 	}
-	PDEBUG(D_STREAM,
-		"Read info: %d %d %d %d %d %d. Should be 1,0,2,2,0,0",
-		info[0], info[1], info[2],
-		info[3], info[4], info[5]);
+	gspca_dbg(gspca_dev, D_STREAM,
+		  "Read info: %d %d %d %d %d %d. Should be 1,0,2,2,0,0\n",
+		  info[0], info[1], info[2],
+		  info[3], info[4], info[5]);
 }
 
 static void spca504A_acknowledged_command(struct gspca_dev *gspca_dev,
@@ -379,8 +381,8 @@ static void spca504A_acknowledged_command(struct gspca_dev *gspca_dev,
 	reg_r(gspca_dev, 0x01, 0x0001, 1);
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_FRAM, "Status 0x%02x Need 0x%02x",
-			gspca_dev->usb_buf[0], endcode);
+	gspca_dbg(gspca_dev, D_FRAM, "Status 0x%02x Need 0x%02x\n",
+		  gspca_dev->usb_buf[0], endcode);
 	if (!count)
 		return;
 	count = 200;
@@ -391,8 +393,8 @@ static void spca504A_acknowledged_command(struct gspca_dev *gspca_dev,
 		reg_r(gspca_dev, 0x01, 0x0001, 1);
 		status = gspca_dev->usb_buf[0];
 		if (status == endcode) {
-			PDEBUG(D_FRAM, "status 0x%04x after wait %d",
-				status, 200 - count);
+			gspca_dbg(gspca_dev, D_FRAM, "status 0x%04x after wait %d\n",
+				  status, 200 - count);
 				break;
 		}
 	}
@@ -435,8 +437,8 @@ static void spca50x_GetFirmware(struct gspca_dev *gspca_dev)
 
 	data = gspca_dev->usb_buf;
 	reg_r(gspca_dev, 0x20, 0, 5);
-	PDEBUG(D_STREAM, "FirmWare: %d %d %d %d %d",
-		data[0], data[1], data[2], data[3], data[4]);
+	gspca_dbg(gspca_dev, D_STREAM, "FirmWare: %d %d %d %d %d\n",
+		  data[0], data[1], data[2], data[3], data[4]);
 	reg_r(gspca_dev, 0x23, 0, 64);
 	reg_r(gspca_dev, 0x23, 1, 64);
 }
@@ -651,7 +653,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 		spca504B_WaitCmdStatus(gspca_dev);
 		break;
 	case BRIDGE_SPCA504C:	/* pccam600 */
-		PDEBUG(D_STREAM, "Opening SPCA504 (PC-CAM 600)");
+		gspca_dbg(gspca_dev, D_STREAM, "Opening SPCA504 (PC-CAM 600)\n");
 		reg_w_riv(gspca_dev, 0xe0, 0x0000, 0x0000);
 		reg_w_riv(gspca_dev, 0xe0, 0x0000, 0x0001);	/* reset */
 		spca504_wait_status(gspca_dev);
@@ -666,7 +668,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 		break;
 	default:
 /*	case BRIDGE_SPCA504: */
-		PDEBUG(D_STREAM, "Opening SPCA504");
+		gspca_dbg(gspca_dev, D_STREAM, "Opening SPCA504\n");
 		if (sd->subtype == AiptekMiniPenCam13) {
 			spca504_read_info(gspca_dev);
 
diff --git a/drivers/media/usb/gspca/t613.c b/drivers/media/usb/gspca/t613.c
index 46fb76349000..0ae557cd15ef 100644
--- a/drivers/media/usb/gspca/t613.c
+++ b/drivers/media/usb/gspca/t613.c
@@ -490,7 +490,7 @@ static void setcolors(struct gspca_dev *gspca_dev, s32 val)
 
 static void setgamma(struct gspca_dev *gspca_dev, s32 val)
 {
-	PDEBUG(D_CONF, "Gamma: %d", val);
+	gspca_dbg(gspca_dev, D_CONF, "Gamma: %d\n", val);
 	reg_w_ixbuf(gspca_dev, 0x90,
 		gamma_table[val], sizeof gamma_table[0]);
 }
@@ -592,19 +592,19 @@ static int sd_init(struct gspca_dev *gspca_dev)
 			| reg_r(gspca_dev, 0x07);
 	switch (sensor_id & 0xff0f) {
 	case 0x0801:
-		PDEBUG(D_PROBE, "sensor tas5130a");
+		gspca_dbg(gspca_dev, D_PROBE, "sensor tas5130a\n");
 		sd->sensor = SENSOR_TAS5130A;
 		break;
 	case 0x0802:
-		PDEBUG(D_PROBE, "sensor lt168g");
+		gspca_dbg(gspca_dev, D_PROBE, "sensor lt168g\n");
 		sd->sensor = SENSOR_LT168G;
 		break;
 	case 0x0803:
-		PDEBUG(D_PROBE, "sensor 'other'");
+		gspca_dbg(gspca_dev, D_PROBE, "sensor 'other'\n");
 		sd->sensor = SENSOR_OTHER;
 		break;
 	case 0x0807:
-		PDEBUG(D_PROBE, "sensor om6802");
+		gspca_dbg(gspca_dev, D_PROBE, "sensor om6802\n");
 		sd->sensor = SENSOR_OM6802;
 		break;
 	default:
@@ -632,8 +632,8 @@ static int sd_init(struct gspca_dev *gspca_dev)
 	i = 0;
 	while (read_indexs[i] != 0x00) {
 		test_byte = reg_r(gspca_dev, read_indexs[i]);
-		PDEBUG(D_STREAM, "Reg 0x%02x = 0x%02x", read_indexs[i],
-		       test_byte);
+		gspca_dbg(gspca_dev, D_STREAM, "Reg 0x%02x = 0x%02x\n",
+			  read_indexs[i], test_byte);
 		i++;
 	}
 
@@ -643,8 +643,8 @@ static int sd_init(struct gspca_dev *gspca_dev)
 
 	if (sd->sensor == SENSOR_LT168G) {
 		test_byte = reg_r(gspca_dev, 0x80);
-		PDEBUG(D_STREAM, "Reg 0x%02x = 0x%02x", 0x80,
-		       test_byte);
+		gspca_dbg(gspca_dev, D_STREAM, "Reg 0x%02x = 0x%02x\n", 0x80,
+			  test_byte);
 		reg_w(gspca_dev, 0x6c80);
 	}
 
@@ -665,8 +665,8 @@ static int sd_init(struct gspca_dev *gspca_dev)
 
 	if (sd->sensor == SENSOR_LT168G) {
 		test_byte = reg_r(gspca_dev, 0x80);
-		PDEBUG(D_STREAM, "Reg 0x%02x = 0x%02x", 0x80,
-		       test_byte);
+		gspca_dbg(gspca_dev, D_STREAM, "Reg 0x%02x = 0x%02x\n", 0x80,
+			  test_byte);
 		reg_w(gspca_dev, 0x6c80);
 	}
 
@@ -737,7 +737,7 @@ static void poll_sensor(struct gspca_dev *gspca_dev)
 		 0xa1, 0xb1, 0xda, 0x6b, 0xdb, 0x98, 0xdf, 0x0c,
 		 0xc2, 0x80, 0xc3, 0x10};
 
-	PDEBUG(D_STREAM, "[Sensor requires polling]");
+	gspca_dbg(gspca_dev, D_STREAM, "[Sensor requires polling]\n");
 	reg_w_buf(gspca_dev, poll1, sizeof poll1);
 	reg_w_buf(gspca_dev, poll2, sizeof poll2);
 	reg_w_buf(gspca_dev, noise03, sizeof noise03);
diff --git a/drivers/media/usb/gspca/topro.c b/drivers/media/usb/gspca/topro.c
index 983fc6b500af..82e2be14cad8 100644
--- a/drivers/media/usb/gspca/topro.c
+++ b/drivers/media/usb/gspca/topro.c
@@ -1453,7 +1453,7 @@ static void set_dqt(struct gspca_dev *gspca_dev, u8 q)
 	struct sd *sd = (struct sd *) gspca_dev;
 
 	/* update the jpeg quantization tables */
-	PDEBUG(D_STREAM, "q %d -> %d", sd->quality, q);
+	gspca_dbg(gspca_dev, D_STREAM, "q %d -> %d\n", sd->quality, q);
 	sd->quality = q;
 	if (q > 16)
 		q = 16;
@@ -4053,7 +4053,7 @@ static int sd_init(struct gspca_dev *gspca_dev)
 				ARRAY_SIZE(tp6810_preinit));
 	msleep(15);
 	reg_r(gspca_dev, TP6800_R18_GPIO_DATA);
-	PDEBUG(D_PROBE, "gpio: %02x", gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_PROBE, "gpio: %02x\n", gspca_dev->usb_buf[0]);
 /* values:
  *	0x80: snapshot button
  *	0x40: LED
@@ -4627,7 +4627,7 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 			if (*data == 0xaa || *data == 0x00)
 				return;
 			if (*data > 0xc0) {
-				PDEBUG(D_FRAM, "bad frame");
+				gspca_dbg(gspca_dev, D_FRAM, "bad frame\n");
 				gspca_dev->last_packet_type = DISCARD_PACKET;
 				return;
 			}
diff --git a/drivers/media/usb/gspca/touptek.c b/drivers/media/usb/gspca/touptek.c
index b8af4370d27c..d1b9032d7863 100644
--- a/drivers/media/usb/gspca/touptek.c
+++ b/drivers/media/usb/gspca/touptek.c
@@ -195,15 +195,15 @@ static const struct v4l2_pix_format vga_mode[] = {
 static int val_reply(struct gspca_dev *gspca_dev, const char *reply, int rc)
 {
 	if (rc < 0) {
-		PERR("reply has error %d", rc);
+		gspca_err(gspca_dev, "reply has error %d\n", rc);
 		return -EIO;
 	}
 	if (rc != 1) {
-		PERR("Bad reply size %d", rc);
+		gspca_err(gspca_dev, "Bad reply size %d\n", rc);
 		return -EIO;
 	}
 	if (reply[0] != 0x08) {
-		PERR("Bad reply 0x%02x", (int)reply[0]);
+		gspca_err(gspca_dev, "Bad reply 0x%02x\n", (int)reply[0]);
 		return -EIO;
 	}
 	return 0;
@@ -214,21 +214,21 @@ static void reg_w(struct gspca_dev *gspca_dev, u16 value, u16 index)
 	char *buff = gspca_dev->usb_buf;
 	int rc;
 
-	PDEBUG(D_USBO,
-		"reg_w bReq=0x0B, bReqT=0xC0, wVal=0x%04X, wInd=0x%04X\n",
-		value, index);
+	gspca_dbg(gspca_dev, D_USBO,
+		  "reg_w bReq=0x0B, bReqT=0xC0, wVal=0x%04X, wInd=0x%04X\n\n",
+		  value, index);
 	rc = usb_control_msg(gspca_dev->dev, usb_rcvctrlpipe(gspca_dev->dev, 0),
 		0x0B, 0xC0, value, index, buff, 1, 500);
-	PDEBUG(D_USBO, "rc=%d, ret={0x%02x}", rc, (int)buff[0]);
+	gspca_dbg(gspca_dev, D_USBO, "rc=%d, ret={0x%02x}\n", rc, (int)buff[0]);
 	if (rc < 0) {
-		PERR("Failed reg_w(0x0B, 0xC0, 0x%04X, 0x%04X) w/ rc %d\n",
-			value, index, rc);
+		gspca_err(gspca_dev, "Failed reg_w(0x0B, 0xC0, 0x%04X, 0x%04X) w/ rc %d\n",
+			  value, index, rc);
 		gspca_dev->usb_err = rc;
 		return;
 	}
 	if (val_reply(gspca_dev, buff, rc)) {
-		PERR("Bad reply to reg_w(0x0B, 0xC0, 0x%04X, 0x%04X\n",
-			value, index);
+		gspca_err(gspca_dev, "Bad reply to reg_w(0x0B, 0xC0, 0x%04X, 0x%04X\n",
+			  value, index);
 		gspca_dev->usb_err = -EIO;
 	}
 }
@@ -254,11 +254,11 @@ static void setexposure(struct gspca_dev *gspca_dev, s32 val)
 	else if (w == 3264)
 		value = val * 3 / 2;
 	else {
-		PERR("Invalid width %u\n", w);
+		gspca_err(gspca_dev, "Invalid width %u\n", w);
 		gspca_dev->usb_err = -EINVAL;
 		return;
 	}
-	PDEBUG(D_STREAM, "exposure: 0x%04X ms\n", value);
+	gspca_dbg(gspca_dev, D_STREAM, "exposure: 0x%04X ms\n\n", value);
 	/* Wonder if theres a good reason for sending it twice */
 	/* probably not but leave it in because...why not */
 	reg_w(gspca_dev, value, REG_COARSE_INTEGRATION_TIME_);
@@ -286,9 +286,9 @@ static void setggain(struct gspca_dev *gspca_dev, u16 global_gain)
 	u16 normalized;
 
 	normalized = gainify(global_gain);
-	PDEBUG(D_STREAM, "gain G1/G2 (0x%04X): 0x%04X (src 0x%04X)\n",
-		 REG_GREEN1_GAIN,
-		 normalized, global_gain);
+	gspca_dbg(gspca_dev, D_STREAM, "gain G1/G2 (0x%04X): 0x%04X (src 0x%04X)\n\n",
+		  REG_GREEN1_GAIN,
+		  normalized, global_gain);
 
 	reg_w(gspca_dev, normalized, REG_GREEN1_GAIN);
 	reg_w(gspca_dev, normalized, REG_GREEN2_GAIN);
@@ -302,13 +302,13 @@ static void setbgain(struct gspca_dev *gspca_dev,
 	normalized = global_gain +
 		((u32)global_gain) * gain / GAIN_MAX;
 	if (normalized > GAIN_MAX) {
-		PDEBUG(D_STREAM, "Truncating blue 0x%04X w/ value 0x%04X\n",
-			 GAIN_MAX, normalized);
+		gspca_dbg(gspca_dev, D_STREAM, "Truncating blue 0x%04X w/ value 0x%04X\n\n",
+			  GAIN_MAX, normalized);
 		normalized = GAIN_MAX;
 	}
 	normalized = gainify(normalized);
-	PDEBUG(D_STREAM, "gain B (0x%04X): 0x%04X w/ source 0x%04X\n",
-		 REG_BLUE_GAIN, normalized, gain);
+	gspca_dbg(gspca_dev, D_STREAM, "gain B (0x%04X): 0x%04X w/ source 0x%04X\n\n",
+		  REG_BLUE_GAIN, normalized, gain);
 
 	reg_w(gspca_dev, normalized, REG_BLUE_GAIN);
 }
@@ -321,13 +321,13 @@ static void setrgain(struct gspca_dev *gspca_dev,
 	normalized = global_gain +
 		((u32)global_gain) * gain / GAIN_MAX;
 	if (normalized > GAIN_MAX) {
-		PDEBUG(D_STREAM, "Truncating gain 0x%04X w/ value 0x%04X\n",
-			 GAIN_MAX, normalized);
+		gspca_dbg(gspca_dev, D_STREAM, "Truncating gain 0x%04X w/ value 0x%04X\n\n",
+			  GAIN_MAX, normalized);
 		normalized = GAIN_MAX;
 	}
 	normalized = gainify(normalized);
-	PDEBUG(D_STREAM, "gain R (0x%04X): 0x%04X w / source 0x%04X\n",
-		 REG_RED_GAIN, normalized, gain);
+	gspca_dbg(gspca_dev, D_STREAM, "gain R (0x%04X): 0x%04X w / source 0x%04X\n\n",
+		  REG_RED_GAIN, normalized, gain);
 
 	reg_w(gspca_dev, normalized, REG_RED_GAIN);
 }
@@ -336,7 +336,7 @@ static void configure_wh(struct gspca_dev *gspca_dev)
 {
 	unsigned int w = gspca_dev->pixfmt.width;
 
-	PDEBUG(D_STREAM, "configure_wh\n");
+	gspca_dbg(gspca_dev, D_STREAM, "configure_wh\n\n");
 
 	if (w == 800) {
 		static const struct cmd reg_init_res[] = {
@@ -372,7 +372,7 @@ static void configure_wh(struct gspca_dev *gspca_dev)
 		reg_w_buf(gspca_dev,
 			       reg_init_res, ARRAY_SIZE(reg_init_res));
 	} else {
-		PERR("bad width %u\n", w);
+		gspca_err(gspca_dev, "bad width %u\n", w);
 		gspca_dev->usb_err = -EINVAL;
 		return;
 	}
@@ -392,7 +392,7 @@ static void configure_wh(struct gspca_dev *gspca_dev)
 		reg_w(gspca_dev, 0x0B4B, REG_FRAME_LENGTH_LINES_);
 		reg_w(gspca_dev, 0x1F40, REG_LINE_LENGTH_PCK_);
 	} else {
-		PERR("bad width %u\n", w);
+		gspca_err(gspca_dev, "bad width %u\n", w);
 		gspca_dev->usb_err = -EINVAL;
 		return;
 	}
@@ -425,14 +425,15 @@ static void configure_encrypted(struct gspca_dev *gspca_dev)
 		{0x0100, REG_MODE_SELECT},
 	};
 
-	PDEBUG(D_STREAM, "Encrypted begin, w = %u\n", gspca_dev->pixfmt.width);
+	gspca_dbg(gspca_dev, D_STREAM, "Encrypted begin, w = %u\n\n",
+		  gspca_dev->pixfmt.width);
 	reg_w_buf(gspca_dev, reg_init_begin, ARRAY_SIZE(reg_init_begin));
 	configure_wh(gspca_dev);
 	reg_w_buf(gspca_dev, reg_init_end, ARRAY_SIZE(reg_init_end));
 	reg_w(gspca_dev, 0x0100, REG_GROUPED_PARAMETER_HOLD);
 	reg_w(gspca_dev, 0x0000, REG_GROUPED_PARAMETER_HOLD);
 
-	PDEBUG(D_STREAM, "Encrypted end\n");
+	gspca_dbg(gspca_dev, D_STREAM, "Encrypted end\n\n");
 }
 
 static int configure(struct gspca_dev *gspca_dev)
@@ -440,7 +441,7 @@ static int configure(struct gspca_dev *gspca_dev)
 	int rc;
 	char *buff = gspca_dev->usb_buf;
 
-	PDEBUG(D_STREAM, "configure()\n");
+	gspca_dbg(gspca_dev, D_STREAM, "configure()\n\n");
 
 	/*
 	 * First driver sets a sort of encryption key
@@ -458,7 +459,7 @@ static int configure(struct gspca_dev *gspca_dev)
 	rc = usb_control_msg(gspca_dev->dev, usb_rcvctrlpipe(gspca_dev->dev, 0),
 			     0x16, 0xC0, 0x0000, 0x0000, buff, 2, 500);
 	if (val_reply(gspca_dev, buff, rc)) {
-		PERR("failed key req");
+		gspca_err(gspca_dev, "failed key req\n");
 		return -EIO;
 	}
 
@@ -475,21 +476,24 @@ static int configure(struct gspca_dev *gspca_dev)
 	rc = usb_control_msg(gspca_dev->dev, usb_sndctrlpipe(gspca_dev->dev, 0),
 			     0x01, 0x40, 0x0001, 0x000F, NULL, 0, 500);
 	if (rc < 0) {
-		PERR("failed to replay packet 176 w/ rc %d\n", rc);
+		gspca_err(gspca_dev, "failed to replay packet 176 w/ rc %d\n",
+			  rc);
 		return rc;
 	}
 
 	rc = usb_control_msg(gspca_dev->dev, usb_sndctrlpipe(gspca_dev->dev, 0),
 			     0x01, 0x40, 0x0000, 0x000F, NULL, 0, 500);
 	if (rc < 0) {
-		PERR("failed to replay packet 178 w/ rc %d\n", rc);
+		gspca_err(gspca_dev, "failed to replay packet 178 w/ rc %d\n",
+			  rc);
 		return rc;
 	}
 
 	rc = usb_control_msg(gspca_dev->dev, usb_sndctrlpipe(gspca_dev->dev, 0),
 			     0x01, 0x40, 0x0001, 0x000F, NULL, 0, 500);
 	if (rc < 0) {
-		PERR("failed to replay packet 180 w/ rc %d\n", rc);
+		gspca_err(gspca_dev, "failed to replay packet 180 w/ rc %d\n",
+			  rc);
 		return rc;
 	}
 
@@ -511,11 +515,12 @@ static int configure(struct gspca_dev *gspca_dev)
 	rc = usb_control_msg(gspca_dev->dev, usb_sndctrlpipe(gspca_dev->dev, 0),
 			     0x01, 0x40, 0x0003, 0x000F, NULL, 0, 500);
 	if (rc < 0) {
-		PERR("failed to replay final packet w/ rc %d\n", rc);
+		gspca_err(gspca_dev, "failed to replay final packet w/ rc %d\n",
+			  rc);
 		return rc;
 	}
 
-	PDEBUG(D_STREAM, "Configure complete\n");
+	gspca_dbg(gspca_dev, D_STREAM, "Configure complete\n\n");
 	return 0;
 }
 
@@ -545,7 +550,7 @@ static int sd_start(struct gspca_dev *gspca_dev)
 
 	rc = configure(gspca_dev);
 	if (rc < 0) {
-		PERR("Failed configure");
+		gspca_err(gspca_dev, "Failed configure\n");
 		return rc;
 	}
 	/* First two frames have messed up gains
@@ -563,13 +568,13 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 		/* can we finish a frame? */
 		if (sd->this_f + len == gspca_dev->pixfmt.sizeimage) {
 			gspca_frame_add(gspca_dev, LAST_PACKET, data, len);
-			PDEBUG(D_FRAM, "finish frame sz %u/%u w/ len %u\n",
-				 sd->this_f, gspca_dev->pixfmt.sizeimage, len);
+			gspca_dbg(gspca_dev, D_FRAM, "finish frame sz %u/%u w/ len %u\n\n",
+				  sd->this_f, gspca_dev->pixfmt.sizeimage, len);
 		/* lost some data, discard the frame */
 		} else {
 			gspca_frame_add(gspca_dev, DISCARD_PACKET, NULL, 0);
-			PDEBUG(D_FRAM, "abort frame sz %u/%u w/ len %u\n",
-				 sd->this_f, gspca_dev->pixfmt.sizeimage, len);
+			gspca_dbg(gspca_dev, D_FRAM, "abort frame sz %u/%u w/ len %u\n\n",
+				  sd->this_f, gspca_dev->pixfmt.sizeimage, len);
 		}
 		sd->this_f = 0;
 	} else {
@@ -641,7 +646,7 @@ static int sd_init_controls(struct gspca_dev *gspca_dev)
 			V4L2_CID_RED_BALANCE, 0, 1023, 1, 295);
 
 	if (hdl->error) {
-		PERR("Could not initialize controls\n");
+		gspca_err(gspca_dev, "Could not initialize controls\n");
 		return hdl->error;
 	}
 	return 0;
diff --git a/drivers/media/usb/gspca/vc032x.c b/drivers/media/usb/gspca/vc032x.c
index b935febf7146..6b11597977c9 100644
--- a/drivers/media/usb/gspca/vc032x.c
+++ b/drivers/media/usb/gspca/vc032x.c
@@ -2926,11 +2926,12 @@ static void reg_r(struct gspca_dev *gspca_dev,
 	if (gspca_dev->usb_err < 0)
 		return;
 	if (len == 1)
-		PDEBUG(D_USBI, "GET %02x 0001 %04x %02x", req, index,
-				gspca_dev->usb_buf[0]);
+		gspca_dbg(gspca_dev, D_USBI, "GET %02x 0001 %04x %02x\n",
+			  req, index,
+			  gspca_dev->usb_buf[0]);
 	else
-		PDEBUG(D_USBI, "GET %02x 0001 %04x %*ph",
-				req, index, 3, gspca_dev->usb_buf);
+		gspca_dbg(gspca_dev, D_USBI, "GET %02x 0001 %04x %*ph\n",
+			  req, index, 3, gspca_dev->usb_buf);
 }
 
 static void reg_w_i(struct gspca_dev *gspca_dev,
@@ -2960,7 +2961,7 @@ static void reg_w(struct gspca_dev *gspca_dev,
 {
 	if (gspca_dev->usb_err < 0)
 		return;
-	PDEBUG(D_USBO, "SET %02x %04x %04x", req, value, index);
+	gspca_dbg(gspca_dev, D_USBO, "SET %02x %04x %04x\n", req, value, index);
 	reg_w_i(gspca_dev, req, value, index);
 }
 
@@ -2992,8 +2993,8 @@ static u16 read_sensor_register(struct gspca_dev *gspca_dev,
 	reg_r(gspca_dev, 0xa1, 0xb33c, 1);
 	hdata = gspca_dev->usb_buf[0];
 	if (hdata != 0 && mdata != 0 && ldata != 0)
-		PDEBUG(D_PROBE, "Read Sensor %02x%02x %02x",
-			hdata, mdata, ldata);
+		gspca_dbg(gspca_dev, D_PROBE, "Read Sensor %02x%02x %02x\n",
+			  hdata, mdata, ldata);
 	reg_r(gspca_dev, 0xa1, 0xb334, 1);
 	if (gspca_dev->usb_buf[0] == 0x02)
 		return (hdata << 8) + mdata;
@@ -3015,8 +3016,8 @@ static int vc032x_probe_sensor(struct gspca_dev *gspca_dev)
 	}
 
 	reg_r(gspca_dev, 0xa1, 0xbfcf, 1);
-	PDEBUG(D_PROBE, "vc032%d check sensor header %02x",
-		sd->bridge == BRIDGE_VC0321 ? 1 : 3, gspca_dev->usb_buf[0]);
+	gspca_dbg(gspca_dev, D_PROBE, "vc032%d check sensor header %02x\n",
+		  sd->bridge == BRIDGE_VC0321 ? 1 : 3, gspca_dev->usb_buf[0]);
 	if (sd->bridge == BRIDGE_VC0321) {
 		ptsensor_info = vc0321_probe_data;
 		n = ARRAY_SIZE(vc0321_probe_data);
@@ -3036,7 +3037,8 @@ static int vc032x_probe_sensor(struct gspca_dev *gspca_dev)
 		if (value == 0 && ptsensor_info->IdAdd == 0x82)
 			value = read_sensor_register(gspca_dev, 0x83);
 		if (value != 0) {
-			PDEBUG(D_PROBE, "Sensor ID %04x (%d)", value, i);
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor ID %04x (%d)\n",
+				  value, i);
 			if (value == ptsensor_info->VpId)
 				return ptsensor_info->sensorId;
 
@@ -3063,9 +3065,10 @@ static void i2c_write(struct gspca_dev *gspca_dev,
 	if (gspca_dev->usb_err < 0)
 		return;
 	if (size == 1)
-		PDEBUG(D_USBO, "i2c_w %02x %02x", reg, *val);
+		gspca_dbg(gspca_dev, D_USBO, "i2c_w %02x %02x\n", reg, *val);
 	else
-		PDEBUG(D_USBO, "i2c_w %02x %02x%02x", reg, *val, val[1]);
+		gspca_dbg(gspca_dev, D_USBO, "i2c_w %02x %02x%02x\n",
+			  reg, *val, val[1]);
 	reg_r_i(gspca_dev, 0xa1, 0xb33f, 1);
 /*fixme:should check if (!(gspca_dev->usb_buf[0] & 0x02)) error*/
 	reg_w_i(gspca_dev, 0xa0, size, 0xb334);
@@ -3170,35 +3173,35 @@ static int sd_init(struct gspca_dev *gspca_dev)
 		pr_err("Unknown sensor...\n");
 		return -EINVAL;
 	case SENSOR_HV7131R:
-		PDEBUG(D_PROBE, "Find Sensor HV7131R");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor HV7131R\n");
 		break;
 	case SENSOR_MI0360:
-		PDEBUG(D_PROBE, "Find Sensor MI0360");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor MI0360\n");
 		sd->bridge = BRIDGE_VC0323;
 		break;
 	case SENSOR_MI1310_SOC:
-		PDEBUG(D_PROBE, "Find Sensor MI1310_SOC");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor MI1310_SOC\n");
 		break;
 	case SENSOR_MI1320:
-		PDEBUG(D_PROBE, "Find Sensor MI1320");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor MI1320\n");
 		break;
 	case SENSOR_MI1320_SOC:
-		PDEBUG(D_PROBE, "Find Sensor MI1320_SOC");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor MI1320_SOC\n");
 		break;
 	case SENSOR_OV7660:
-		PDEBUG(D_PROBE, "Find Sensor OV7660");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor OV7660\n");
 		break;
 	case SENSOR_OV7670:
-		PDEBUG(D_PROBE, "Find Sensor OV7670");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor OV7670\n");
 		break;
 	case SENSOR_PO1200:
-		PDEBUG(D_PROBE, "Find Sensor PO1200");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor PO1200\n");
 		break;
 	case SENSOR_PO3130NC:
-		PDEBUG(D_PROBE, "Find Sensor PO3130NC");
+		gspca_dbg(gspca_dev, D_PROBE, "Find Sensor PO3130NC\n");
 		break;
 	case SENSOR_POxxxx:
-		PDEBUG(D_PROBE, "Sensor POxxxx");
+		gspca_dbg(gspca_dev, D_PROBE, "Sensor POxxxx\n");
 		break;
 	}
 	sd->sensor = sensor;
@@ -3624,8 +3627,8 @@ static void sd_pkt_scan(struct gspca_dev *gspca_dev,
 	struct sd *sd = (struct sd *) gspca_dev;
 
 	if (data[0] == 0xff && data[1] == 0xd8) {
-		PDEBUG(D_PACK,
-			"vc032x header packet found len %d", len);
+		gspca_dbg(gspca_dev, D_PACK,
+			  "vc032x header packet found len %d\n", len);
 		gspca_frame_add(gspca_dev, LAST_PACKET, NULL, 0);
 		data += sd->image_offset;
 		len -= sd->image_offset;
diff --git a/drivers/media/usb/gspca/w996Xcf.c b/drivers/media/usb/gspca/w996Xcf.c
index 728d2322c433..abfab3de1866 100644
--- a/drivers/media/usb/gspca/w996Xcf.c
+++ b/drivers/media/usb/gspca/w996Xcf.c
@@ -245,7 +245,7 @@ static void w9968cf_smbus_read_ack(struct sd *sd)
 	sda = w9968cf_read_sb(sd);
 	w9968cf_write_sb(sd, 0x0012); /* SDE=1, SDA=1, SCL=0 */
 	if (sda >= 0 && (sda & 0x08)) {
-		PDEBUG(D_USBI, "Did not receive i2c ACK");
+		gspca_dbg(gspca_dev, D_USBI, "Did not receive i2c ACK\n");
 		sd->gspca_dev.usb_err = -EIO;
 	}
 }
@@ -297,7 +297,7 @@ static void w9968cf_i2c_w(struct sd *sd, u8 reg, u8 value)
 
 	w9968cf_write_fsb(sd, data);
 
-	PDEBUG(D_USBO, "i2c 0x%02x -> [0x%02x]", value, reg);
+	gspca_dbg(gspca_dev, D_USBO, "i2c 0x%02x -> [0x%02x]\n", value, reg);
 }
 
 /* SMBus protocol: S Addr Wr [A] Subaddr [A] P S Addr+1 Rd [A] [Value] NA P */
@@ -331,9 +331,10 @@ static int w9968cf_i2c_r(struct sd *sd, u8 reg)
 
 	if (sd->gspca_dev.usb_err >= 0) {
 		ret = value;
-		PDEBUG(D_USBI, "i2c [0x%02X] -> 0x%02X", reg, value);
+		gspca_dbg(gspca_dev, D_USBI, "i2c [0x%02X] -> 0x%02X\n",
+			  reg, value);
 	} else
-		PERR("i2c read [0x%02x] failed", reg);
+		gspca_err(gspca_dev, "i2c read [0x%02x] failed\n", reg);
 
 	return ret;
 }
diff --git a/drivers/media/usb/gspca/xirlink_cit.c b/drivers/media/usb/gspca/xirlink_cit.c
index 68656e7986c7..58deb0c38826 100644
--- a/drivers/media/usb/gspca/xirlink_cit.c
+++ b/drivers/media/usb/gspca/xirlink_cit.c
@@ -704,7 +704,8 @@ static int cit_read_reg(struct gspca_dev *gspca_dev, u16 index, int verbose)
 	}
 
 	if (verbose)
-		PDEBUG(D_PROBE, "Register %04x value: %02x", index, buf[0]);
+		gspca_dbg(gspca_dev, D_PROBE, "Register %04x value: %02x\n",
+			  index, buf[0]);
 
 	return 0;
 }
@@ -1471,10 +1472,11 @@ static int cit_get_clock_div(struct gspca_dev *gspca_dev)
 			fps[clock_div - 1] * 3 / 2)
 		clock_div--;
 
-	PDEBUG(D_PROBE,
-	       "PacketSize: %d, res: %dx%d -> using clockdiv: %d (%d fps)",
-	       packet_size, gspca_dev->pixfmt.width, gspca_dev->pixfmt.height,
-	       clock_div, fps[clock_div]);
+	gspca_dbg(gspca_dev, D_PROBE,
+		  "PacketSize: %d, res: %dx%d -> using clockdiv: %d (%d fps)\n",
+		  packet_size,
+		  gspca_dev->pixfmt.width, gspca_dev->pixfmt.height,
+		  clock_div, fps[clock_div]);
 
 	return clock_div;
 }
@@ -2865,17 +2867,17 @@ static u8 *cit_find_sof(struct gspca_dev *gspca_dev, u8 *data, int len)
 				sd->sof_read = 0;
 				if (data[i] == 0xff) {
 					if (i >= 4)
-						PDEBUG(D_FRAM,
-						       "header found at offset: %d: %02x %02x 00 %3ph\n",
-						       i - 1,
-						       data[i - 4],
-						       data[i - 3],
-						       &data[i]);
+						gspca_dbg(gspca_dev, D_FRAM,
+							  "header found at offset: %d: %02x %02x 00 %3ph\n\n",
+							  i - 1,
+							  data[i - 4],
+							  data[i - 3],
+							  &data[i]);
 					else
-						PDEBUG(D_FRAM,
-						       "header found at offset: %d: 00 %3ph\n",
-						       i - 1,
-						       &data[i]);
+						gspca_dbg(gspca_dev, D_FRAM,
+							  "header found at offset: %d: 00 %3ph\n\n",
+							  i - 1,
+							  &data[i]);
 					return data + i + (sd->sof_len - 1);
 				}
 				break;
diff --git a/drivers/media/usb/gspca/zc3xx.c b/drivers/media/usb/gspca/zc3xx.c
index e2d486bd8c28..25b4dbe8e049 100644
--- a/drivers/media/usb/gspca/zc3xx.c
+++ b/drivers/media/usb/gspca/zc3xx.c
@@ -6041,7 +6041,7 @@ static int sif_probe(struct gspca_dev *gspca_dev)
 	msleep(150);
 	checkword = ((i2c_read(gspca_dev, 0x00) & 0x0f) << 4)
 			| ((i2c_read(gspca_dev, 0x01) & 0xf0) >> 4);
-	PDEBUG(D_PROBE, "probe sif 0x%04x", checkword);
+	gspca_dbg(gspca_dev, D_PROBE, "probe sif 0x%04x\n", checkword);
 	if (checkword == 0x0007) {
 		send_unknown(gspca_dev, SENSOR_PAS106);
 		return 0x0f;			/* PAS106 */
@@ -6129,7 +6129,7 @@ static int vga_2wr_probe(struct gspca_dev *gspca_dev)
 	i2c_write(gspca_dev, 0x12, 0x80, 0x00);	/* sensor reset */
 	retword = i2c_read(gspca_dev, 0x0a) << 8;
 	retword |= i2c_read(gspca_dev, 0x0b);
-	PDEBUG(D_PROBE, "probe 2wr ov vga 0x%04x", retword);
+	gspca_dbg(gspca_dev, D_PROBE, "probe 2wr ov vga 0x%04x\n", retword);
 	switch (retword) {
 	case 0x7631:				/* OV7630C */
 		reg_w(gspca_dev, 0x06, 0x0010);
@@ -6186,7 +6186,7 @@ static int vga_3wr_probe(struct gspca_dev *gspca_dev)
 	reg_w(gspca_dev, 0x02, 0x0010);
 	retword = reg_r(gspca_dev, 0x000b) << 8;
 	retword |= reg_r(gspca_dev, 0x000a);
-	PDEBUG(D_PROBE, "probe 3wr vga 1 0x%04x", retword);
+	gspca_dbg(gspca_dev, D_PROBE, "probe 3wr vga 1 0x%04x\n", retword);
 	reg_r(gspca_dev, 0x0010);
 	if ((retword & 0xff00) == 0x6400)
 		return 0x02;		/* TAS5130C */
@@ -6206,7 +6206,7 @@ static int vga_3wr_probe(struct gspca_dev *gspca_dev)
 	reg_w(gspca_dev, 0x01, 0x0012);
 	retword = i2c_read(gspca_dev, 0x00);
 	if (retword != 0) {
-		PDEBUG(D_PROBE, "probe 3wr vga type 0a");
+		gspca_dbg(gspca_dev, D_PROBE, "probe 3wr vga type 0a\n");
 		return 0x0a;			/* PB0330 */
 	}
 
@@ -6220,7 +6220,8 @@ static int vga_3wr_probe(struct gspca_dev *gspca_dev)
 	reg_w(gspca_dev, 0x01, 0x0012);
 	retword = i2c_read(gspca_dev, 0x00);
 	if (retword != 0) {
-		PDEBUG(D_PROBE, "probe 3wr vga type %02x", retword);
+		gspca_dbg(gspca_dev, D_PROBE, "probe 3wr vga type %02x\n",
+			  retword);
 		if (retword == 0x0011)			/* gc0303 */
 			return 0x0303;
 		if (retword == 0x0029)			/* gc0305 */
@@ -6251,12 +6252,13 @@ static int vga_3wr_probe(struct gspca_dev *gspca_dev)
 	reg_w(gspca_dev, 0x05, 0x0012);
 	retword = i2c_read(gspca_dev, 0x00) << 8;	/* ID 0 */
 	retword |= i2c_read(gspca_dev, 0x01);		/* ID 1 */
-	PDEBUG(D_PROBE, "probe 3wr vga 2 0x%04x", retword);
+	gspca_dbg(gspca_dev, D_PROBE, "probe 3wr vga 2 0x%04x\n", retword);
 	if (retword == 0x2030) {
 		u8 retbyte;
 
 		retbyte = i2c_read(gspca_dev, 0x02);	/* revision number */
-		PDEBUG(D_PROBE, "sensor PO2030 rev 0x%02x", retbyte);
+		gspca_dbg(gspca_dev, D_PROBE, "sensor PO2030 rev 0x%02x\n",
+			  retbyte);
 
 		send_unknown(gspca_dev, SENSOR_PO2030);
 		return retword;
@@ -6272,7 +6274,8 @@ static int vga_3wr_probe(struct gspca_dev *gspca_dev)
 	reg_w(gspca_dev, 0xd3, 0x008b);
 	retword = i2c_read(gspca_dev, 0x01);
 	if (retword != 0) {
-		PDEBUG(D_PROBE, "probe 3wr vga type 0a ? ret: %04x", retword);
+		gspca_dbg(gspca_dev, D_PROBE, "probe 3wr vga type 0a ? ret: %04x\n",
+			  retword);
 		return 0x16;			/* adcm2700 (6100/6200) */
 	}
 	return -1;
@@ -6490,19 +6493,20 @@ static int sd_init(struct gspca_dev *gspca_dev)
 
 	sensor = zcxx_probeSensor(gspca_dev);
 	if (sensor >= 0)
-		PDEBUG(D_PROBE, "probe sensor -> %04x", sensor);
+		gspca_dbg(gspca_dev, D_PROBE, "probe sensor -> %04x\n", sensor);
 	if ((unsigned) force_sensor < SENSOR_MAX) {
 		sd->sensor = force_sensor;
-		PDEBUG(D_PROBE, "sensor forced to %d", force_sensor);
+		gspca_dbg(gspca_dev, D_PROBE, "sensor forced to %d\n",
+			  force_sensor);
 	} else {
 		switch (sensor) {
 		case -1:
 			switch (sd->sensor) {
 			case SENSOR_MC501CB:
-				PDEBUG(D_PROBE, "Sensor MC501CB");
+				gspca_dbg(gspca_dev, D_PROBE, "Sensor MC501CB\n");
 				break;
 			case SENSOR_GC0303:
-				PDEBUG(D_PROBE, "Sensor GC0303");
+				gspca_dbg(gspca_dev, D_PROBE, "Sensor GC0303\n");
 				break;
 			default:
 				pr_warn("Unknown sensor - set to TAS5130C\n");
@@ -6512,100 +6516,101 @@ static int sd_init(struct gspca_dev *gspca_dev)
 		case 0:
 			/* check the sensor type */
 			sensor = i2c_read(gspca_dev, 0x00);
-			PDEBUG(D_PROBE, "Sensor hv7131 type %d", sensor);
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor hv7131 type %d\n",
+				  sensor);
 			switch (sensor) {
 			case 0:			/* hv7131b */
 			case 1:			/* hv7131e */
-				PDEBUG(D_PROBE, "Find Sensor HV7131B");
+				gspca_dbg(gspca_dev, D_PROBE, "Find Sensor HV7131B\n");
 				sd->sensor = SENSOR_HV7131B;
 				break;
 			default:
 /*			case 2:			 * hv7131r */
-				PDEBUG(D_PROBE, "Find Sensor HV7131R");
+				gspca_dbg(gspca_dev, D_PROBE, "Find Sensor HV7131R\n");
 				sd->sensor = SENSOR_HV7131R;
 				break;
 			}
 			break;
 		case 0x02:
-			PDEBUG(D_PROBE, "Sensor TAS5130C");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor TAS5130C\n");
 			sd->sensor = SENSOR_TAS5130C;
 			break;
 		case 0x04:
-			PDEBUG(D_PROBE, "Find Sensor CS2102");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor CS2102\n");
 			sd->sensor = SENSOR_CS2102;
 			break;
 		case 0x08:
-			PDEBUG(D_PROBE, "Find Sensor HDCS2020");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor HDCS2020\n");
 			sd->sensor = SENSOR_HDCS2020;
 			break;
 		case 0x0a:
-			PDEBUG(D_PROBE,
-				"Find Sensor PB0330. Chip revision %x",
-				sd->chip_revision);
+			gspca_dbg(gspca_dev, D_PROBE,
+				  "Find Sensor PB0330. Chip revision %x\n",
+				  sd->chip_revision);
 			sd->sensor = SENSOR_PB0330;
 			break;
 		case 0x0c:
-			PDEBUG(D_PROBE, "Find Sensor ICM105A");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor ICM105A\n");
 			sd->sensor = SENSOR_ICM105A;
 			break;
 		case 0x0e:
-			PDEBUG(D_PROBE, "Find Sensor PAS202B");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor PAS202B\n");
 			sd->sensor = SENSOR_PAS202B;
 			break;
 		case 0x0f:
-			PDEBUG(D_PROBE, "Find Sensor PAS106");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor PAS106\n");
 			sd->sensor = SENSOR_PAS106;
 			break;
 		case 0x10:
 		case 0x12:
-			PDEBUG(D_PROBE, "Find Sensor TAS5130C");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor TAS5130C\n");
 			sd->sensor = SENSOR_TAS5130C;
 			break;
 		case 0x11:
-			PDEBUG(D_PROBE, "Find Sensor HV7131R");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor HV7131R\n");
 			sd->sensor = SENSOR_HV7131R;
 			break;
 		case 0x13:
 		case 0x15:
-			PDEBUG(D_PROBE,
-				"Sensor MT9V111. Chip revision %04x",
-				sd->chip_revision);
+			gspca_dbg(gspca_dev, D_PROBE,
+				  "Sensor MT9V111. Chip revision %04x\n",
+				  sd->chip_revision);
 			sd->sensor = sd->bridge == BRIDGE_ZC301
 					? SENSOR_MT9V111_1
 					: SENSOR_MT9V111_3;
 			break;
 		case 0x14:
-			PDEBUG(D_PROBE,
-				"Find Sensor CS2102K?. Chip revision %x",
-				sd->chip_revision);
+			gspca_dbg(gspca_dev, D_PROBE,
+				  "Find Sensor CS2102K?. Chip revision %x\n",
+				  sd->chip_revision);
 			sd->sensor = SENSOR_CS2102K;
 			break;
 		case 0x16:
-			PDEBUG(D_PROBE, "Find Sensor ADCM2700");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor ADCM2700\n");
 			sd->sensor = SENSOR_ADCM2700;
 			break;
 		case 0x29:
-			PDEBUG(D_PROBE, "Find Sensor GC0305");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor GC0305\n");
 			sd->sensor = SENSOR_GC0305;
 			break;
 		case 0x0303:
-			PDEBUG(D_PROBE, "Sensor GC0303");
+			gspca_dbg(gspca_dev, D_PROBE, "Sensor GC0303\n");
 			sd->sensor =  SENSOR_GC0303;
 			break;
 		case 0x2030:
-			PDEBUG(D_PROBE, "Find Sensor PO2030");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor PO2030\n");
 			sd->sensor = SENSOR_PO2030;
 			break;
 		case 0x7620:
-			PDEBUG(D_PROBE, "Find Sensor OV7620");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor OV7620\n");
 			sd->sensor = SENSOR_OV7620;
 			break;
 		case 0x7631:
-			PDEBUG(D_PROBE, "Find Sensor OV7630C");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor OV7630C\n");
 			sd->sensor = SENSOR_OV7630C;
 			break;
 		case 0x7648:
-			PDEBUG(D_PROBE, "Find Sensor OV7648");
+			gspca_dbg(gspca_dev, D_PROBE, "Find Sensor OV7648\n");
 			sd->sensor = SENSOR_OV7620;	/* same sensor (?) */
 			break;
 		default:
diff --git a/drivers/media/usb/hdpvr/Makefile b/drivers/media/usb/hdpvr/Makefile
index 9b8d1463c526..644dd99ffce3 100644
--- a/drivers/media/usb/hdpvr/Makefile
+++ b/drivers/media/usb/hdpvr/Makefile
@@ -1,7 +1,3 @@
 hdpvr-objs	:= hdpvr-control.o hdpvr-core.o hdpvr-video.o hdpvr-i2c.o
 
 obj-$(CONFIG_VIDEO_HDPVR) += hdpvr.o
-
-ccflags-y += -Idrivers/media/i2c
-
-ccflags-y += $(extra-cflags-y) $(extra-cflags-m)
diff --git a/drivers/media/usb/hdpvr/hdpvr-core.c b/drivers/media/usb/hdpvr/hdpvr-core.c
index dbe29c6c4d8b..29ac7fc5b039 100644
--- a/drivers/media/usb/hdpvr/hdpvr-core.c
+++ b/drivers/media/usb/hdpvr/hdpvr-core.c
@@ -292,7 +292,7 @@ static int hdpvr_probe(struct usb_interface *interface,
 	/* register v4l2_device early so it can be used for printks */
 	if (v4l2_device_register(&interface->dev, &dev->v4l2_dev)) {
 		dev_err(&interface->dev, "v4l2_device_register failed\n");
-		goto error;
+		goto error_free_dev;
 	}
 
 	mutex_init(&dev->io_mutex);
@@ -301,7 +301,7 @@ static int hdpvr_probe(struct usb_interface *interface,
 	dev->usbc_buf = kmalloc(64, GFP_KERNEL);
 	if (!dev->usbc_buf) {
 		v4l2_err(&dev->v4l2_dev, "Out of memory\n");
-		goto error;
+		goto error_v4l2_unregister;
 	}
 
 	init_waitqueue_head(&dev->wait_buffer);
@@ -339,13 +339,13 @@ static int hdpvr_probe(struct usb_interface *interface,
 	}
 	if (!dev->bulk_in_endpointAddr) {
 		v4l2_err(&dev->v4l2_dev, "Could not find bulk-in endpoint\n");
-		goto error;
+		goto error_put_usb;
 	}
 
 	/* init the device */
 	if (hdpvr_device_init(dev)) {
 		v4l2_err(&dev->v4l2_dev, "device init failed\n");
-		goto error;
+		goto error_put_usb;
 	}
 
 	mutex_lock(&dev->io_mutex);
@@ -353,7 +353,7 @@ static int hdpvr_probe(struct usb_interface *interface,
 		mutex_unlock(&dev->io_mutex);
 		v4l2_err(&dev->v4l2_dev,
 			 "allocating transfer buffers failed\n");
-		goto error;
+		goto error_put_usb;
 	}
 	mutex_unlock(&dev->io_mutex);
 
@@ -361,19 +361,12 @@ static int hdpvr_probe(struct usb_interface *interface,
 	retval = hdpvr_register_i2c_adapter(dev);
 	if (retval < 0) {
 		v4l2_err(&dev->v4l2_dev, "i2c adapter register failed\n");
-		goto error;
-	}
-
-	client = hdpvr_register_ir_rx_i2c(dev);
-	if (!client) {
-		v4l2_err(&dev->v4l2_dev, "i2c IR RX device register failed\n");
-		retval = -ENODEV;
-		goto reg_fail;
+		goto error_free_buffers;
 	}
 
-	client = hdpvr_register_ir_tx_i2c(dev);
+	client = hdpvr_register_ir_i2c(dev);
 	if (!client) {
-		v4l2_err(&dev->v4l2_dev, "i2c IR TX device register failed\n");
+		v4l2_err(&dev->v4l2_dev, "i2c IR device register failed\n");
 		retval = -ENODEV;
 		goto reg_fail;
 	}
@@ -394,13 +387,17 @@ static int hdpvr_probe(struct usb_interface *interface,
 reg_fail:
 #if IS_ENABLED(CONFIG_I2C)
 	i2c_del_adapter(&dev->i2c_adapter);
+error_free_buffers:
 #endif
+	hdpvr_free_buffers(dev);
+error_put_usb:
+	usb_put_dev(dev->udev);
+	kfree(dev->usbc_buf);
+error_v4l2_unregister:
+	v4l2_device_unregister(&dev->v4l2_dev);
+error_free_dev:
+	kfree(dev);
 error:
-	if (dev) {
-		flush_work(&dev->worker);
-		/* this frees allocated memory */
-		hdpvr_delete(dev);
-	}
 	return retval;
 }
 
diff --git a/drivers/media/usb/hdpvr/hdpvr-i2c.c b/drivers/media/usb/hdpvr/hdpvr-i2c.c
index 1db49ed5eaf1..4720d79b0282 100644
--- a/drivers/media/usb/hdpvr/hdpvr-i2c.c
+++ b/drivers/media/usb/hdpvr/hdpvr-i2c.c
@@ -32,24 +32,11 @@
 #define Z8F0811_IR_RX_I2C_ADDR	0x71
 
 
-struct i2c_client *hdpvr_register_ir_tx_i2c(struct hdpvr_device *dev)
+struct i2c_client *hdpvr_register_ir_i2c(struct hdpvr_device *dev)
 {
 	struct IR_i2c_init_data *init_data = &dev->ir_i2c_init_data;
-	struct i2c_board_info hdpvr_ir_tx_i2c_board_info = {
-		I2C_BOARD_INFO("ir_tx_z8f0811_hdpvr", Z8F0811_IR_TX_I2C_ADDR),
-	};
-
-	init_data->name = "HD-PVR";
-	hdpvr_ir_tx_i2c_board_info.platform_data = init_data;
-
-	return i2c_new_device(&dev->i2c_adapter, &hdpvr_ir_tx_i2c_board_info);
-}
-
-struct i2c_client *hdpvr_register_ir_rx_i2c(struct hdpvr_device *dev)
-{
-	struct IR_i2c_init_data *init_data = &dev->ir_i2c_init_data;
-	struct i2c_board_info hdpvr_ir_rx_i2c_board_info = {
-		I2C_BOARD_INFO("ir_rx_z8f0811_hdpvr", Z8F0811_IR_RX_I2C_ADDR),
+	struct i2c_board_info info = {
+		I2C_BOARD_INFO("ir_z8f0811_hdpvr", Z8F0811_IR_RX_I2C_ADDR),
 	};
 
 	/* Our default information for ir-kbd-i2c.c to use */
@@ -59,9 +46,9 @@ struct i2c_client *hdpvr_register_ir_rx_i2c(struct hdpvr_device *dev)
 			  RC_PROTO_BIT_RC6_6A_32;
 	init_data->name = "HD-PVR";
 	init_data->polling_interval = 405; /* ms, duplicated from Windows */
-	hdpvr_ir_rx_i2c_board_info.platform_data = init_data;
+	info.platform_data = init_data;
 
-	return i2c_new_device(&dev->i2c_adapter, &hdpvr_ir_rx_i2c_board_info);
+	return i2c_new_device(&dev->i2c_adapter, &info);
 }
 
 static int hdpvr_i2c_read(struct hdpvr_device *dev, int bus,
diff --git a/drivers/media/usb/hdpvr/hdpvr-video.c b/drivers/media/usb/hdpvr/hdpvr-video.c
index 7fb036d6a86e..77c3d331ff31 100644
--- a/drivers/media/usb/hdpvr/hdpvr-video.c
+++ b/drivers/media/usb/hdpvr/hdpvr-video.c
@@ -521,14 +521,14 @@ static ssize_t hdpvr_read(struct file *file, char __user *buffer, size_t count,
 	return ret;
 }
 
-static unsigned int hdpvr_poll(struct file *filp, poll_table *wait)
+static __poll_t hdpvr_poll(struct file *filp, poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct hdpvr_buffer *buf = NULL;
 	struct hdpvr_device *dev = video_drvdata(filp);
-	unsigned int mask = v4l2_ctrl_poll(filp, wait);
+	__poll_t mask = v4l2_ctrl_poll(filp, wait);
 
-	if (!(req_events & (POLLIN | POLLRDNORM)))
+	if (!(req_events & (EPOLLIN | EPOLLRDNORM)))
 		return mask;
 
 	mutex_lock(&dev->io_mutex);
@@ -553,7 +553,7 @@ static unsigned int hdpvr_poll(struct file *filp, poll_table *wait)
 		buf = hdpvr_get_next_buffer(dev);
 	}
 	if (buf && buf->status == BUFSTAT_READY)
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	return mask;
 }
@@ -941,18 +941,18 @@ static int hdpvr_s_ctrl(struct v4l2_ctrl *ctrl)
 		return 0;
 	case V4L2_CID_MPEG_VIDEO_ENCODING:
 		return 0;
-/* 	case V4L2_CID_MPEG_VIDEO_B_FRAMES: */
-/* 		if (ctrl->value == 0 && !(opt->gop_mode & 0x2)) { */
-/* 			opt->gop_mode |= 0x2; */
-/* 			hdpvr_config_call(dev, CTRL_GOP_MODE_VALUE, */
-/* 					  opt->gop_mode); */
-/* 		} */
-/* 		if (ctrl->value == 128 && opt->gop_mode & 0x2) { */
-/* 			opt->gop_mode &= ~0x2; */
-/* 			hdpvr_config_call(dev, CTRL_GOP_MODE_VALUE, */
-/* 					  opt->gop_mode); */
-/* 		} */
-/* 		break; */
+/*	case V4L2_CID_MPEG_VIDEO_B_FRAMES: */
+/*		if (ctrl->value == 0 && !(opt->gop_mode & 0x2)) { */
+/*			opt->gop_mode |= 0x2; */
+/*			hdpvr_config_call(dev, CTRL_GOP_MODE_VALUE, */
+/*					  opt->gop_mode); */
+/*		} */
+/*		if (ctrl->value == 128 && opt->gop_mode & 0x2) { */
+/*			opt->gop_mode &= ~0x2; */
+/*			hdpvr_config_call(dev, CTRL_GOP_MODE_VALUE, */
+/*					  opt->gop_mode); */
+/*		} */
+/*		break; */
 	case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: {
 		uint peak_bitrate = dev->video_bitrate_peak->val / 100000;
 		uint bitrate = dev->video_bitrate->val / 100000;
@@ -1154,7 +1154,7 @@ static void hdpvr_device_release(struct video_device *vdev)
 static const struct video_device hdpvr_video_template = {
 	.fops			= &hdpvr_fops,
 	.release		= hdpvr_device_release,
-	.ioctl_ops 		= &hdpvr_ioctl_ops,
+	.ioctl_ops		= &hdpvr_ioctl_ops,
 	.tvnorms		= V4L2_STD_ALL,
 };
 
diff --git a/drivers/media/usb/hdpvr/hdpvr.h b/drivers/media/usb/hdpvr/hdpvr.h
index a12e0af1d4e1..1d65b4185f57 100644
--- a/drivers/media/usb/hdpvr/hdpvr.h
+++ b/drivers/media/usb/hdpvr/hdpvr.h
@@ -232,15 +232,15 @@ enum {
 
 
 /* :0 s 38 d3 0000 0000 0001 1 = 00 */
-/* 		ret = usb_control_msg(dev->udev, */
-/* 				      usb_sndctrlpipe(dev->udev, 0), */
-/* 				      0xd3, 0x38, */
-/* 				      0, 0, */
-/* 				      "\0", 1, */
-/* 				      1000); */
+/*		ret = usb_control_msg(dev->udev, */
+/*				      usb_sndctrlpipe(dev->udev, 0), */
+/*				      0xd3, 0x38, */
+/*				      0, 0, */
+/*				      "\0", 1, */
+/*				      1000); */
 
-/* 		info("control request returned %d", ret); */
-/* 		msleep(5000); */
+/*		info("control request returned %d", ret); */
+/*		msleep(5000); */
 
 
 	/* :0 s b8 81 1400 0003 0005 5 <
@@ -320,8 +320,7 @@ int hdpvr_cancel_queue(struct hdpvr_device *dev);
 /* i2c adapter registration */
 int hdpvr_register_i2c_adapter(struct hdpvr_device *dev);
 
-struct i2c_client *hdpvr_register_ir_rx_i2c(struct hdpvr_device *dev);
-struct i2c_client *hdpvr_register_ir_tx_i2c(struct hdpvr_device *dev);
+struct i2c_client *hdpvr_register_ir_i2c(struct hdpvr_device *dev);
 
 /*========================================================================*/
 /* buffer management */
diff --git a/drivers/media/usb/pulse8-cec/pulse8-cec.c b/drivers/media/usb/pulse8-cec/pulse8-cec.c
index 50146f263d90..350635826aae 100644
--- a/drivers/media/usb/pulse8-cec/pulse8-cec.c
+++ b/drivers/media/usb/pulse8-cec/pulse8-cec.c
@@ -329,7 +329,7 @@ static int pulse8_setup(struct pulse8 *pulse8, struct serio *serio,
 	u8 cmd[2];
 	int err;
 	struct tm tm;
-	time_t date;
+	time64_t date;
 
 	pulse8->vers = 0;
 
@@ -349,7 +349,7 @@ static int pulse8_setup(struct pulse8 *pulse8, struct serio *serio,
 	if (err)
 		return err;
 	date = (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
-	time_to_tm(date, 0, &tm);
+	time64_to_tm(date, 0, &tm);
 	dev_info(pulse8->dev, "Firmware build date %04ld.%02d.%02d %02d:%02d:%02d\n",
 		 tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
 		 tm.tm_hour, tm.tm_min, tm.tm_sec);
diff --git a/drivers/media/usb/pvrusb2/Makefile b/drivers/media/usb/pvrusb2/Makefile
index 0d84064036b2..9facf6873404 100644
--- a/drivers/media/usb/pvrusb2/Makefile
+++ b/drivers/media/usb/pvrusb2/Makefile
@@ -17,7 +17,5 @@ pvrusb2-objs	:= pvrusb2-i2c-core.o \
 
 obj-$(CONFIG_VIDEO_PVRUSB2) += pvrusb2.o
 
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-devattr.c b/drivers/media/usb/pvrusb2/pvrusb2-devattr.c
index 51b3312eaea1..71537097c13f 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-devattr.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-devattr.c
@@ -319,12 +319,12 @@ static struct tda829x_config tda829x_no_probe = {
 };
 
 static struct tda18271_std_map hauppauge_tda18271_dvbt_std_map = {
-        .dvbt_6   = { .if_freq = 3300, .agc_mode = 3, .std = 4,
-                      .if_lvl = 1, .rfagc_top = 0x37, },
-        .dvbt_7   = { .if_freq = 3800, .agc_mode = 3, .std = 5,
-                      .if_lvl = 1, .rfagc_top = 0x37, },
-        .dvbt_8   = { .if_freq = 4300, .agc_mode = 3, .std = 6,
-                      .if_lvl = 1, .rfagc_top = 0x37, },
+	.dvbt_6   = { .if_freq = 3300, .agc_mode = 3, .std = 4,
+		      .if_lvl = 1, .rfagc_top = 0x37, },
+	.dvbt_7   = { .if_freq = 3800, .agc_mode = 3, .std = 5,
+		      .if_lvl = 1, .rfagc_top = 0x37, },
+	.dvbt_8   = { .if_freq = 4300, .agc_mode = 3, .std = 6,
+		      .if_lvl = 1, .rfagc_top = 0x37, },
 };
 
 static struct tda18271_config hauppauge_tda18271_dvb_config = {
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-dvb.c b/drivers/media/usb/pvrusb2/pvrusb2-dvb.c
index 56c750535ee7..4b32b2141169 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-dvb.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-dvb.c
@@ -18,7 +18,7 @@
 #include <linux/freezer.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
-#include "dvbdev.h"
+#include <media/dvbdev.h>
 #include "pvrusb2-debug.h"
 #include "pvrusb2-hdw-internal.h"
 #include "pvrusb2-hdw.h"
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-dvb.h b/drivers/media/usb/pvrusb2/pvrusb2-dvb.h
index b500c86d4178..e7f71fb94a6e 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-dvb.h
+++ b/drivers/media/usb/pvrusb2/pvrusb2-dvb.h
@@ -2,10 +2,10 @@
 #ifndef __PVRUSB2_DVB_H__
 #define __PVRUSB2_DVB_H__
 
-#include "dvb_frontend.h"
-#include "dvb_demux.h"
-#include "dvb_net.h"
-#include "dmxdev.h"
+#include <media/dvb_frontend.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
+#include <media/dmxdev.h>
 #include "pvrusb2-context.h"
 
 #define PVR2_DVB_BUFFER_COUNT 32
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
index 8289ee482f49..e0353161ccd6 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
@@ -2351,7 +2351,8 @@ struct pvr2_hdw *pvr2_hdw_create(struct usb_interface *intf,
 
 	if (hdw_desc == NULL) {
 		pvr2_trace(PVR2_TRACE_INIT, "pvr2_hdw_create: No device description pointer, unable to continue.");
-		pvr2_trace(PVR2_TRACE_INIT, "If you have a new device type, please contact Mike Isely <isely@pobox.com> to get it included in the driver\n");
+		pvr2_trace(PVR2_TRACE_INIT,
+			   "If you have a new device type, please contact Mike Isely <isely@pobox.com> to get it included in the driver");
 		goto fail;
 	}
 
@@ -3648,6 +3649,12 @@ static int pvr2_send_request_ex(struct pvr2_hdw *hdw,
 				  hdw);
 		hdw->ctl_write_urb->actual_length = 0;
 		hdw->ctl_write_pend_flag = !0;
+		if (usb_urb_ep_type_check(hdw->ctl_write_urb)) {
+			pvr2_trace(
+				PVR2_TRACE_ERROR_LEGS,
+				"Invalid write control endpoint");
+			return -EINVAL;
+		}
 		status = usb_submit_urb(hdw->ctl_write_urb,GFP_KERNEL);
 		if (status < 0) {
 			pvr2_trace(PVR2_TRACE_ERROR_LEGS,
@@ -3672,6 +3679,12 @@ status);
 				  hdw);
 		hdw->ctl_read_urb->actual_length = 0;
 		hdw->ctl_read_pend_flag = !0;
+		if (usb_urb_ep_type_check(hdw->ctl_read_urb)) {
+			pvr2_trace(
+				PVR2_TRACE_ERROR_LEGS,
+				"Invalid read control endpoint");
+			return -EINVAL;
+		}
 		status = usb_submit_urb(hdw->ctl_read_urb,GFP_KERNEL);
 		if (status < 0) {
 			pvr2_trace(PVR2_TRACE_ERROR_LEGS,
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c b/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
index ff7b4d1d385d..f3003ca05f4b 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-i2c-core.c
@@ -585,17 +585,10 @@ static void pvr2_i2c_register_ir(struct pvr2_hdw *hdw)
 		init_data->type = RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_MCE |
 							RC_PROTO_BIT_RC6_6A_32;
 		init_data->name = hdw->hdw_desc->description;
-		/* IR Receiver */
-		info.addr          = 0x71;
-		info.platform_data = init_data;
-		strlcpy(info.type, "ir_rx_z8f0811_haup", I2C_NAME_SIZE);
-		pvr2_trace(PVR2_TRACE_INFO, "Binding %s to i2c address 0x%02x.",
-			   info.type, info.addr);
-		i2c_new_device(&hdw->i2c_adap, &info);
-		/* IR Trasmitter */
-		info.addr          = 0x70;
+		/* IR Transceiver */
+		info.addr = 0x71;
 		info.platform_data = init_data;
-		strlcpy(info.type, "ir_tx_z8f0811_haup", I2C_NAME_SIZE);
+		strlcpy(info.type, "ir_z8f0811_haup", I2C_NAME_SIZE);
 		pvr2_trace(PVR2_TRACE_INFO, "Binding %s to i2c address 0x%02x.",
 			   info.type, info.addr);
 		i2c_new_device(&hdw->i2c_adap, &info);
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
index 4320bda9352d..9fdc57c1658f 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
@@ -78,18 +78,6 @@ static int vbi_nr[PVR_NUM] = {[0 ... PVR_NUM-1] = -1};
 module_param_array(vbi_nr, int, NULL, 0444);
 MODULE_PARM_DESC(vbi_nr, "Offset for device's vbi dev minor");
 
-static struct v4l2_fmtdesc pvr_fmtdesc [] = {
-	{
-		.index          = 0,
-		.type           = V4L2_BUF_TYPE_VIDEO_CAPTURE,
-		.flags          = V4L2_FMT_FLAG_COMPRESSED,
-		.description    = "MPEG1/2",
-		// This should really be V4L2_PIX_FMT_MPEG, but xawtv
-		// breaks when I do that.
-		.pixelformat    = 0, // V4L2_PIX_FMT_MPEG,
-	}
-};
-
 #define PVR_FORMAT_PIX  0
 #define PVR_FORMAT_VBI  1
 
@@ -99,17 +87,11 @@ static struct v4l2_format pvr_format [] = {
 		.fmt    = {
 			.pix        = {
 				.width          = 720,
-				.height             = 576,
-				// This should really be V4L2_PIX_FMT_MPEG,
-				// but xawtv breaks when I do that.
-				.pixelformat    = 0, // V4L2_PIX_FMT_MPEG,
+				.height         = 576,
+				.pixelformat    = V4L2_PIX_FMT_MPEG,
 				.field          = V4L2_FIELD_INTERLACED,
-				.bytesperline   = 0,  // doesn't make sense
-						      // here
-				//FIXME : Don't know what to put here...
-				.sizeimage          = (32*1024),
-				.colorspace     = 0, // doesn't make sense here
-				.priv           = 0
+				/* FIXME : Don't know what to put here... */
+				.sizeimage      = 32 * 1024,
 			}
 		}
 	},
@@ -153,6 +135,8 @@ static int pvr2_querycap(struct file *file, void *priv, struct v4l2_capability *
 	case VFL_TYPE_RADIO:
 		cap->device_caps = V4L2_CAP_RADIO;
 		break;
+	default:
+		return -EINVAL;
 	}
 	cap->device_caps |= V4L2_CAP_TUNER | V4L2_CAP_READWRITE;
 	return 0;
@@ -407,11 +391,11 @@ static int pvr2_g_frequency(struct file *file, void *priv, struct v4l2_frequency
 
 static int pvr2_enum_fmt_vid_cap(struct file *file, void *priv, struct v4l2_fmtdesc *fd)
 {
-	/* Only one format is supported : mpeg.*/
-	if (fd->index != 0)
+	/* Only one format is supported: MPEG. */
+	if (fd->index)
 		return -EINVAL;
 
-	memcpy(fd, pvr_fmtdesc, sizeof(struct v4l2_fmtdesc));
+	fd->pixelformat = V4L2_PIX_FMT_MPEG;
 	return 0;
 }
 
@@ -1190,26 +1174,26 @@ static ssize_t pvr2_v4l2_read(struct file *file,
 }
 
 
-static unsigned int pvr2_v4l2_poll(struct file *file, poll_table *wait)
+static __poll_t pvr2_v4l2_poll(struct file *file, poll_table *wait)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	struct pvr2_v4l2_fh *fh = file->private_data;
 	int ret;
 
 	if (fh->fw_mode_flag) {
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 		return mask;
 	}
 
 	if (!fh->rhp) {
 		ret = pvr2_v4l2_iosetup(fh);
-		if (ret) return POLLERR;
+		if (ret) return EPOLLERR;
 	}
 
 	poll_wait(file,&fh->wait_data,wait);
 
 	if (pvr2_ioread_avail(fh->rhp) >= 0) {
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	}
 
 	return mask;
diff --git a/drivers/media/usb/pwc/pwc.h b/drivers/media/usb/pwc/pwc.h
index 3c73bdaae450..67010010d2a2 100644
--- a/drivers/media/usb/pwc/pwc.h
+++ b/drivers/media/usb/pwc/pwc.h
@@ -50,7 +50,7 @@
 
 /* Version block */
 #define PWC_VERSION	"10.0.15"
-#define PWC_NAME 	"pwc"
+#define PWC_NAME	"pwc"
 #define PFX		PWC_NAME ": "
 
 
@@ -120,10 +120,10 @@
 #define MAX_ISO_BUFS		3
 #define ISO_FRAMES_PER_DESC	10
 #define ISO_MAX_FRAME_SIZE	960
-#define ISO_BUFFER_SIZE 	(ISO_FRAMES_PER_DESC * ISO_MAX_FRAME_SIZE)
+#define ISO_BUFFER_SIZE		(ISO_FRAMES_PER_DESC * ISO_MAX_FRAME_SIZE)
 
 /* Maximum size after decompression is 640x480 YUV data, 1.5 * 640 * 480 */
-#define PWC_FRAME_SIZE 		(460800 + TOUCAM_HEADER_SIZE + TOUCAM_TRAILER_SIZE)
+#define PWC_FRAME_SIZE		(460800 + TOUCAM_HEADER_SIZE + TOUCAM_TRAILER_SIZE)
 
 /* Absolute minimum and maximum number of buffers available for mmap() */
 #define MIN_FRAMES		2
diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c
index 7fee5766587a..8c2a86d71e8a 100644
--- a/drivers/media/usb/s2255/s2255drv.c
+++ b/drivers/media/usb/s2255/s2255drv.c
@@ -350,7 +350,7 @@ static void s2255_fillbuff(struct s2255_vc *vc, struct s2255_buffer *buf,
 			   int jpgsize);
 static int s2255_set_mode(struct s2255_vc *vc, struct s2255_mode *mode);
 static int s2255_board_shutdown(struct s2255_dev *dev);
-static void s2255_fwload_start(struct s2255_dev *dev, int reset);
+static void s2255_fwload_start(struct s2255_dev *dev);
 static void s2255_destroy(struct s2255_dev *dev);
 static long s2255_vendor_req(struct s2255_dev *dev, unsigned char req,
 			     u16 index, u16 value, void *buf,
@@ -476,7 +476,7 @@ static void planar422p_to_yuv_packed(const unsigned char *in,
 static void s2255_reset_dsppower(struct s2255_dev *dev)
 {
 	s2255_vendor_req(dev, 0x40, 0x0000, 0x0001, NULL, 0, 1);
-	msleep(20);
+	msleep(50);
 	s2255_vendor_req(dev, 0x50, 0x0000, 0x0000, NULL, 0, 1);
 	msleep(600);
 	s2255_vendor_req(dev, 0x10, 0x0000, 0x0000, NULL, 0, 1);
@@ -1450,7 +1450,7 @@ static int s2255_open(struct file *file)
 	case S2255_FW_FAILED:
 		s2255_dev_err(&dev->udev->dev,
 			"firmware load failed. retrying.\n");
-		s2255_fwload_start(dev, 1);
+		s2255_fwload_start(dev);
 		wait_event_timeout(dev->fw_data->wait_fw,
 				   ((atomic_read(&dev->fw_data->fw_state)
 				     == S2255_FW_SUCCESS) ||
@@ -2209,10 +2209,9 @@ static void s2255_stop_readpipe(struct s2255_dev *dev)
 	return;
 }
 
-static void s2255_fwload_start(struct s2255_dev *dev, int reset)
+static void s2255_fwload_start(struct s2255_dev *dev)
 {
-	if (reset)
-		s2255_reset_dsppower(dev);
+	s2255_reset_dsppower(dev);
 	dev->fw_data->fw_size = dev->fw_data->fw->size;
 	atomic_set(&dev->fw_data->fw_state, S2255_FW_NOTLOADED);
 	memcpy(dev->fw_data->pfw_data,
@@ -2337,7 +2336,7 @@ static int s2255_probe(struct usb_interface *interface,
 	retval = s2255_board_init(dev);
 	if (retval)
 		goto errorBOARDINIT;
-	s2255_fwload_start(dev, 0);
+	s2255_fwload_start(dev);
 	/* loads v4l specific */
 	retval = s2255_probe_v4l(dev);
 	if (retval)
diff --git a/drivers/media/usb/siano/Makefile b/drivers/media/usb/siano/Makefile
index 758b6a090c59..7d48864e2782 100644
--- a/drivers/media/usb/siano/Makefile
+++ b/drivers/media/usb/siano/Makefile
@@ -1,6 +1,5 @@
 obj-$(CONFIG_SMS_USB_DRV) += smsusb.o
 
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/common/siano
 ccflags-y += $(extra-cflags-y) $(extra-cflags-m)
 
diff --git a/drivers/media/usb/siano/smsusb.c b/drivers/media/usb/siano/smsusb.c
index d07349cf9489..f13e4b01b5a5 100644
--- a/drivers/media/usb/siano/smsusb.c
+++ b/drivers/media/usb/siano/smsusb.c
@@ -61,7 +61,7 @@ struct smsusb_device_t {
 	struct usb_device *udev;
 	struct smscore_device_t *coredev;
 
-	struct smsusb_urb_t 	surbs[MAX_URBS];
+	struct smsusb_urb_t	surbs[MAX_URBS];
 
 	int		response_alignment;
 	int		buffer_size;
diff --git a/drivers/media/usb/stk1160/Makefile b/drivers/media/usb/stk1160/Makefile
index 613471528749..b943db01ccf7 100644
--- a/drivers/media/usb/stk1160/Makefile
+++ b/drivers/media/usb/stk1160/Makefile
@@ -1,10 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
-stk1160-y := 	stk1160-core.o \
+stk1160-y :=	stk1160-core.o \
 		stk1160-v4l.o \
 		stk1160-video.o \
 		stk1160-i2c.o \
 		stk1160-ac97.o
 
 obj-$(CONFIG_VIDEO_STK1160) += stk1160.o
-
-ccflags-y += -Idrivers/media/i2c
diff --git a/drivers/media/usb/stkwebcam/stk-sensor.c b/drivers/media/usb/stkwebcam/stk-sensor.c
index c1d4505f84ea..9a7dbeff1337 100644
--- a/drivers/media/usb/stkwebcam/stk-sensor.c
+++ b/drivers/media/usb/stkwebcam/stk-sensor.c
@@ -397,12 +397,12 @@ int stk_sensor_init(struct stk_camera *dev)
 /* V4L2_PIX_FMT_UYVY */
 static struct regval ov_fmt_uyvy[] = {
 	{REG_TSLB, TSLB_YLAST|0x08 },
-	{ 0x4f, 0x80 }, 	/* "matrix coefficient 1" */
-	{ 0x50, 0x80 }, 	/* "matrix coefficient 2" */
+	{ 0x4f, 0x80 },		/* "matrix coefficient 1" */
+	{ 0x50, 0x80 },		/* "matrix coefficient 2" */
 	{ 0x51, 0    },		/* vb */
-	{ 0x52, 0x22 }, 	/* "matrix coefficient 4" */
-	{ 0x53, 0x5e }, 	/* "matrix coefficient 5" */
-	{ 0x54, 0x80 }, 	/* "matrix coefficient 6" */
+	{ 0x52, 0x22 },		/* "matrix coefficient 4" */
+	{ 0x53, 0x5e },		/* "matrix coefficient 5" */
+	{ 0x54, 0x80 },		/* "matrix coefficient 6" */
 	{REG_COM13, COM13_UVSAT|COM13_CMATRIX},
 	{REG_COM15, COM15_R00FF },
 	{0xff, 0xff}, /* END MARKER */
@@ -410,12 +410,12 @@ static struct regval ov_fmt_uyvy[] = {
 /* V4L2_PIX_FMT_YUYV */
 static struct regval ov_fmt_yuyv[] = {
 	{REG_TSLB, 0 },
-	{ 0x4f, 0x80 }, 	/* "matrix coefficient 1" */
-	{ 0x50, 0x80 }, 	/* "matrix coefficient 2" */
+	{ 0x4f, 0x80 },		/* "matrix coefficient 1" */
+	{ 0x50, 0x80 },		/* "matrix coefficient 2" */
 	{ 0x51, 0    },		/* vb */
-	{ 0x52, 0x22 }, 	/* "matrix coefficient 4" */
-	{ 0x53, 0x5e }, 	/* "matrix coefficient 5" */
-	{ 0x54, 0x80 }, 	/* "matrix coefficient 6" */
+	{ 0x52, 0x22 },		/* "matrix coefficient 4" */
+	{ 0x53, 0x5e },		/* "matrix coefficient 5" */
+	{ 0x54, 0x80 },		/* "matrix coefficient 6" */
 	{REG_COM13, COM13_UVSAT|COM13_CMATRIX},
 	{REG_COM15, COM15_R00FF },
 	{0xff, 0xff}, /* END MARKER */
@@ -426,13 +426,13 @@ static struct regval ov_fmt_rgbr[] = {
 	{ REG_RGB444, 0 },	/* No RGB444 please */
 	{REG_TSLB, 0x00},
 	{ REG_COM1, 0x0 },
-	{ REG_COM9, 0x38 }, 	/* 16x gain ceiling; 0x8 is reserved bit */
-	{ 0x4f, 0xb3 }, 	/* "matrix coefficient 1" */
-	{ 0x50, 0xb3 }, 	/* "matrix coefficient 2" */
+	{ REG_COM9, 0x38 },	/* 16x gain ceiling; 0x8 is reserved bit */
+	{ 0x4f, 0xb3 },		/* "matrix coefficient 1" */
+	{ 0x50, 0xb3 },		/* "matrix coefficient 2" */
 	{ 0x51, 0    },		/* vb */
-	{ 0x52, 0x3d }, 	/* "matrix coefficient 4" */
-	{ 0x53, 0xa7 }, 	/* "matrix coefficient 5" */
-	{ 0x54, 0xe4 }, 	/* "matrix coefficient 6" */
+	{ 0x52, 0x3d },		/* "matrix coefficient 4" */
+	{ 0x53, 0xa7 },		/* "matrix coefficient 5" */
+	{ 0x54, 0xe4 },		/* "matrix coefficient 6" */
 	{ REG_COM13, COM13_GAMMA },
 	{ REG_COM15, COM15_RGB565|COM15_R00FF },
 	{ 0xff, 0xff },
@@ -443,13 +443,13 @@ static struct regval ov_fmt_rgbp[] = {
 	{ REG_RGB444, 0 },	/* No RGB444 please */
 	{REG_TSLB, TSLB_BYTEORD },
 	{ REG_COM1, 0x0 },
-	{ REG_COM9, 0x38 }, 	/* 16x gain ceiling; 0x8 is reserved bit */
-	{ 0x4f, 0xb3 }, 	/* "matrix coefficient 1" */
-	{ 0x50, 0xb3 }, 	/* "matrix coefficient 2" */
+	{ REG_COM9, 0x38 },	/* 16x gain ceiling; 0x8 is reserved bit */
+	{ 0x4f, 0xb3 },		/* "matrix coefficient 1" */
+	{ 0x50, 0xb3 },		/* "matrix coefficient 2" */
 	{ 0x51, 0    },		/* vb */
-	{ 0x52, 0x3d }, 	/* "matrix coefficient 4" */
-	{ 0x53, 0xa7 }, 	/* "matrix coefficient 5" */
-	{ 0x54, 0xe4 }, 	/* "matrix coefficient 6" */
+	{ 0x52, 0x3d },		/* "matrix coefficient 4" */
+	{ 0x53, 0xa7 },		/* "matrix coefficient 5" */
+	{ 0x54, 0xe4 },		/* "matrix coefficient 6" */
 	{ REG_COM13, COM13_GAMMA },
 	{ REG_COM15, COM15_RGB565|COM15_R00FF },
 	{ 0xff, 0xff },
diff --git a/drivers/media/usb/stkwebcam/stk-webcam.c b/drivers/media/usb/stkwebcam/stk-webcam.c
index c0bba773db25..22389b56ec24 100644
--- a/drivers/media/usb/stkwebcam/stk-webcam.c
+++ b/drivers/media/usb/stkwebcam/stk-webcam.c
@@ -721,18 +721,18 @@ static ssize_t v4l_stk_read(struct file *fp, char __user *buf,
 	return ret;
 }
 
-static unsigned int v4l_stk_poll(struct file *fp, poll_table *wait)
+static __poll_t v4l_stk_poll(struct file *fp, poll_table *wait)
 {
 	struct stk_camera *dev = video_drvdata(fp);
-	unsigned res = v4l2_ctrl_poll(fp, wait);
+	__poll_t res = v4l2_ctrl_poll(fp, wait);
 
 	poll_wait(fp, &dev->wait_frame, wait);
 
 	if (!is_present(dev))
-		return POLLERR;
+		return EPOLLERR;
 
 	if (!list_empty(&dev->sio_full))
-		return res | POLLIN | POLLRDNORM;
+		return res | EPOLLIN | EPOLLRDNORM;
 
 	return res;
 }
@@ -1241,7 +1241,6 @@ static void stk_v4l_dev_release(struct video_device *vd)
 	if (dev->sio_bufs != NULL || dev->isobufs != NULL)
 		pr_err("We are leaking memory\n");
 	usb_put_intf(dev->interface);
-	kfree(dev);
 }
 
 static const struct video_device stk_v4l_data = {
@@ -1391,6 +1390,7 @@ static void stk_camera_disconnect(struct usb_interface *interface)
 	video_unregister_device(&dev->vdev);
 	v4l2_ctrl_handler_free(&dev->hdl);
 	v4l2_device_unregister(&dev->v4l2_dev);
+	kfree(dev);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/media/usb/tm6000/Makefile b/drivers/media/usb/tm6000/Makefile
index 05322a72e862..744c039e621a 100644
--- a/drivers/media/usb/tm6000/Makefile
+++ b/drivers/media/usb/tm6000/Makefile
@@ -10,7 +10,5 @@ obj-$(CONFIG_VIDEO_TM6000) += tm6000.o
 obj-$(CONFIG_VIDEO_TM6000_ALSA) += tm6000-alsa.o
 obj-$(CONFIG_VIDEO_TM6000_DVB) += tm6000-dvb.o
 
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/tuners
-ccflags-y += -Idrivers/media/dvb-core
 ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/usb/tm6000/tm6000-alsa.c b/drivers/media/usb/tm6000/tm6000-alsa.c
index 3717a6844ea8..f18cffae4c85 100644
--- a/drivers/media/usb/tm6000/tm6000-alsa.c
+++ b/drivers/media/usb/tm6000/tm6000-alsa.c
@@ -1,14 +1,8 @@
-/*
- *
- *  Support for audio capture for tm5600/6000/6010
- *    (c) 2007-2008 Mauro Carvalho Chehab
- *
- *  Based on cx88-alsa.c
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+// Support for audio capture for tm5600/6000/6010
+// Copyright (c) 2007-2008 Mauro Carvalho Chehab <mchehab@kernel.org>
+//
+// Based on cx88-alsa.c
 
 #include <linux/module.h>
 #include <linux/init.h>
@@ -57,7 +51,7 @@ MODULE_PARM_DESC(index, "Index value for tm6000x capture interface(s).");
 
 MODULE_DESCRIPTION("ALSA driver module for tm5600/tm6000/tm6010 based TV cards");
 MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
 MODULE_SUPPORTED_DEVICE("{{Trident,tm5600},{{Trident,tm6000},{{Trident,tm6010}");
 static unsigned int debug;
 module_param(debug, int, 0644);
diff --git a/drivers/media/usb/tm6000/tm6000-cards.c b/drivers/media/usb/tm6000/tm6000-cards.c
index 77347541904d..4d5f4cc4887e 100644
--- a/drivers/media/usb/tm6000/tm6000-cards.c
+++ b/drivers/media/usb/tm6000/tm6000-cards.c
@@ -1,17 +1,7 @@
-/*
- *  tm6000-cards.c - driver for TM5600/TM6000/TM6010 USB video capture devices
- *
- *  Copyright (C) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// tm6000-cards.c - driver for TM5600/TM6000/TM6010 USB video capture devices
+//
+// Copyright (c) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
 
 #include <linux/init.h>
 #include <linux/module.h>
@@ -23,7 +13,6 @@
 #include <media/v4l2-common.h>
 #include <media/tuner.h>
 #include <media/i2c/tvaudio.h>
-#include <media/i2c-addr.h>
 #include <media/rc-map.h>
 
 #include "tm6000.h"
@@ -1405,4 +1394,4 @@ module_usb_driver(tm6000_usb_driver);
 
 MODULE_DESCRIPTION("Trident TVMaster TM5600/TM6000/TM6010 USB2 adapter");
 MODULE_AUTHOR("Mauro Carvalho Chehab");
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/usb/tm6000/tm6000-core.c b/drivers/media/usb/tm6000/tm6000-core.c
index 8c265bd80faa..23a1332d98e6 100644
--- a/drivers/media/usb/tm6000/tm6000-core.c
+++ b/drivers/media/usb/tm6000/tm6000-core.c
@@ -1,20 +1,10 @@
-/*
- *  tm6000-core.c - driver for TM5600/TM6000/TM6010 USB video capture devices
- *
- *  Copyright (C) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
- *
- *  Copyright (C) 2007 Michel Ludwig <michel.ludwig@gmail.com>
- *      - DVB-T support
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// tm6000-core.c - driver for TM5600/TM6000/TM6010 USB video capture devices
+//
+// Copyright (c) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
+//
+// Copyright (c) 2007 Michel Ludwig <michel.ludwig@gmail.com>
+//     - DVB-T support
 
 #include <linux/module.h>
 #include <linux/kernel.h>
diff --git a/drivers/media/usb/tm6000/tm6000-i2c.c b/drivers/media/usb/tm6000/tm6000-i2c.c
index cbcc1472f1c7..c9a62bbff27a 100644
--- a/drivers/media/usb/tm6000/tm6000-i2c.c
+++ b/drivers/media/usb/tm6000/tm6000-i2c.c
@@ -1,20 +1,10 @@
-/*
- *  tm6000-i2c.c - driver for TM5600/TM6000/TM6010 USB video capture devices
- *
- *  Copyright (C) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
- *
- *  Copyright (C) 2007 Michel Ludwig <michel.ludwig@gmail.com>
- *	- Fix SMBus Read Byte command
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// tm6000-i2c.c - driver for TM5600/TM6000/TM6010 USB video capture devices
+//
+// Copyright (c) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
+//
+// Copyright (c) 2007 Michel Ludwig <michel.ludwig@gmail.com>
+//	- Fix SMBus Read Byte command
 
 #include <linux/module.h>
 #include <linux/kernel.h>
diff --git a/drivers/media/usb/tm6000/tm6000-regs.h b/drivers/media/usb/tm6000/tm6000-regs.h
index ab3fb74c476c..21587fcf11e3 100644
--- a/drivers/media/usb/tm6000/tm6000-regs.h
+++ b/drivers/media/usb/tm6000/tm6000-regs.h
@@ -1,16 +1,8 @@
 /*
- *  tm6000-regs.h - driver for TM5600/TM6000/TM6010 USB video capture devices
+ * SPDX-License-Identifier: GPL-2.0
+ * tm6000-regs.h - driver for TM5600/TM6000/TM6010 USB video capture devices
  *
- *  Copyright (C) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
+ * Copyright (c) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
  */
 
 /*
diff --git a/drivers/media/usb/tm6000/tm6000-stds.c b/drivers/media/usb/tm6000/tm6000-stds.c
index aa43810d17f9..c0c75951246b 100644
--- a/drivers/media/usb/tm6000/tm6000-stds.c
+++ b/drivers/media/usb/tm6000/tm6000-stds.c
@@ -1,17 +1,7 @@
-/*
- *  tm6000-stds.c - driver for TM5600/TM6000/TM6010 USB video capture devices
- *
- *  Copyright (C) 2007 Mauro Carvalho Chehab
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// tm6000-stds.c - driver for TM5600/TM6000/TM6010 USB video capture devices
+//
+// Copyright (c) 2007 Mauro Carvalho Chehab <mchehab@kernel.org>
 
 #include <linux/module.h>
 #include <linux/kernel.h>
diff --git a/drivers/media/usb/tm6000/tm6000-usb-isoc.h b/drivers/media/usb/tm6000/tm6000-usb-isoc.h
index 6a13a27c55d7..5c615b0a7a46 100644
--- a/drivers/media/usb/tm6000/tm6000-usb-isoc.h
+++ b/drivers/media/usb/tm6000/tm6000-usb-isoc.h
@@ -1,16 +1,8 @@
 /*
- *  tm6000-buf.c - driver for TM5600/TM6000/TM6010 USB video capture devices
+ * SPDX-License-Identifier: GPL-2.0
+ * tm6000-buf.c - driver for TM5600/TM6000/TM6010 USB video capture devices
  *
- *  Copyright (C) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
+ * Copyright (c) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
  */
 
 #include <linux/videodev2.h>
diff --git a/drivers/media/usb/tm6000/tm6000-video.c b/drivers/media/usb/tm6000/tm6000-video.c
index 9fa25de6b5a9..8314d3fa9241 100644
--- a/drivers/media/usb/tm6000/tm6000-video.c
+++ b/drivers/media/usb/tm6000/tm6000-video.c
@@ -1,20 +1,10 @@
-/*
- *   tm6000-video.c - driver for TM5600/TM6000/TM6010 USB video capture devices
- *
- *  Copyright (C) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
- *
- *  Copyright (C) 2007 Michel Ludwig <michel.ludwig@gmail.com>
- *	- Fixed module load/unload
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+// tm6000-video.c - driver for TM5600/TM6000/TM6010 USB video capture devices
+//
+// Copyright (c) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
+//
+// Copyright (c) 2007 Michel Ludwig <michel.ludwig@gmail.com>
+//	- Fixed module load/unload
 
 #include <linux/module.h>
 #include <linux/delay.h>
@@ -1323,6 +1313,8 @@ static int __tm6000_open(struct file *file)
 	case VFL_TYPE_RADIO:
 		radio = 1;
 		break;
+	default:
+		return -EINVAL;
 	}
 
 	/* If more than one user, mutex should be added */
@@ -1423,45 +1415,45 @@ tm6000_read(struct file *file, char __user *data, size_t count, loff_t *pos)
 	return 0;
 }
 
-static unsigned int
+static __poll_t
 __tm6000_poll(struct file *file, struct poll_table_struct *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct tm6000_fh        *fh = file->private_data;
 	struct tm6000_buffer    *buf;
-	int res = 0;
+	__poll_t res = 0;
 
 	if (v4l2_event_pending(&fh->fh))
-		res = POLLPRI;
-	else if (req_events & POLLPRI)
+		res = EPOLLPRI;
+	else if (req_events & EPOLLPRI)
 		poll_wait(file, &fh->fh.wait, wait);
 	if (V4L2_BUF_TYPE_VIDEO_CAPTURE != fh->type)
-		return res | POLLERR;
+		return res | EPOLLERR;
 
 	if (!!is_res_streaming(fh->dev, fh))
-		return res | POLLERR;
+		return res | EPOLLERR;
 
 	if (!is_res_read(fh->dev, fh)) {
 		/* streaming capture */
 		if (list_empty(&fh->vb_vidq.stream))
-			return res | POLLERR;
+			return res | EPOLLERR;
 		buf = list_entry(fh->vb_vidq.stream.next, struct tm6000_buffer, vb.stream);
 		poll_wait(file, &buf->vb.done, wait);
 		if (buf->vb.state == VIDEOBUF_DONE ||
 		    buf->vb.state == VIDEOBUF_ERROR)
-			return res | POLLIN | POLLRDNORM;
-	} else if (req_events & (POLLIN | POLLRDNORM)) {
+			return res | EPOLLIN | EPOLLRDNORM;
+	} else if (req_events & (EPOLLIN | EPOLLRDNORM)) {
 		/* read() capture */
 		return res | videobuf_poll_stream(file, &fh->vb_vidq, wait);
 	}
 	return res;
 }
 
-static unsigned int tm6000_poll(struct file *file, struct poll_table_struct *wait)
+static __poll_t tm6000_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct tm6000_fh *fh = file->private_data;
 	struct tm6000_core *dev = fh->dev;
-	unsigned int res;
+	__poll_t res;
 
 	mutex_lock(&dev->lock);
 	res = __tm6000_poll(file, wait);
diff --git a/drivers/media/usb/tm6000/tm6000.h b/drivers/media/usb/tm6000/tm6000.h
index 7ec478d75f55..e1e45770e28d 100644
--- a/drivers/media/usb/tm6000/tm6000.h
+++ b/drivers/media/usb/tm6000/tm6000.h
@@ -1,19 +1,11 @@
 /*
- *  tm6000.h - driver for TM5600/TM6000/TM6010 USB video capture devices
+ * SPDX-License-Identifier: GPL-2.0
+ * tm6000.h - driver for TM5600/TM6000/TM6010 USB video capture devices
  *
- *  Copyright (C) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
+ * Copyright (c) 2006-2007 Mauro Carvalho Chehab <mchehab@infradead.org>
  *
- *  Copyright (C) 2007 Michel Ludwig <michel.ludwig@gmail.com>
+ * Copyright (c) 2007 Michel Ludwig <michel.ludwig@gmail.com>
  *	- DVB-T support
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation version 2
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
  */
 
 #include <linux/videodev2.h>
@@ -27,9 +19,9 @@
 #include <media/v4l2-fh.h>
 
 #include <linux/dvb/frontend.h>
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dmxdev.h"
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dmxdev.h>
 
 /* Inputs */
 enum tm6000_itype {
@@ -185,7 +177,7 @@ struct tm6000_core {
 	struct tm6000_capabilities	caps;
 
 	/* Used to load alsa/dvb */
-        struct work_struct		request_module_wk;
+	struct work_struct		request_module_wk;
 
 	/* Tuner configuration */
 	int				tuner_type;		/* type of the tuner */
diff --git a/drivers/media/usb/ttusb-budget/Makefile b/drivers/media/usb/ttusb-budget/Makefile
index f47bbf62dcde..fe4372dddd0e 100644
--- a/drivers/media/usb/ttusb-budget/Makefile
+++ b/drivers/media/usb/ttusb-budget/Makefile
@@ -1,3 +1,3 @@
 obj-$(CONFIG_DVB_TTUSB_BUDGET) += dvb-ttusb-budget.o
 
-ccflags-y += -Idrivers/media/dvb-core/ -Idrivers/media/dvb-frontends
+ccflags-y += -Idrivers/media/dvb-frontends
diff --git a/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c b/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c
index a142b9dc0feb..eed56895c2b9 100644
--- a/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c
+++ b/drivers/media/usb/ttusb-budget/dvb-ttusb-budget.c
@@ -22,10 +22,10 @@
 #include <linux/mutex.h>
 #include <linux/firmware.h>
 
-#include "dvb_frontend.h"
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_net.h"
+#include <media/dvb_frontend.h>
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_net.h>
 #include "ves1820.h"
 #include "cx22700.h"
 #include "tda1004x.h"
@@ -102,7 +102,6 @@ struct ttusb {
 	unsigned int isoc_in_pipe;
 
 	void *iso_buffer;
-	dma_addr_t iso_dma_handle;
 
 	struct urb *iso_urb[ISO_BUF_COUNT];
 
@@ -792,26 +791,17 @@ static void ttusb_free_iso_urbs(struct ttusb *ttusb)
 
 	for (i = 0; i < ISO_BUF_COUNT; i++)
 		usb_free_urb(ttusb->iso_urb[i]);
-
-	pci_free_consistent(NULL,
-			    ISO_FRAME_SIZE * FRAMES_PER_ISO_BUF *
-			    ISO_BUF_COUNT, ttusb->iso_buffer,
-			    ttusb->iso_dma_handle);
+	kfree(ttusb->iso_buffer);
 }
 
 static int ttusb_alloc_iso_urbs(struct ttusb *ttusb)
 {
 	int i;
 
-	ttusb->iso_buffer = pci_zalloc_consistent(NULL,
-						  ISO_FRAME_SIZE * FRAMES_PER_ISO_BUF * ISO_BUF_COUNT,
-						  &ttusb->iso_dma_handle);
-
-	if (!ttusb->iso_buffer) {
-		dprintk("%s: pci_alloc_consistent - not enough memory\n",
-			__func__);
+	ttusb->iso_buffer = kcalloc(FRAMES_PER_ISO_BUF * ISO_BUF_COUNT,
+			ISO_FRAME_SIZE, GFP_KERNEL);
+	if (!ttusb->iso_buffer)
 		return -ENOMEM;
-	}
 
 	for (i = 0; i < ISO_BUF_COUNT; i++) {
 		struct urb *urb;
diff --git a/drivers/media/usb/ttusb-dec/Makefile b/drivers/media/usb/ttusb-dec/Makefile
index 5352740d2353..dde9168b5e5f 100644
--- a/drivers/media/usb/ttusb-dec/Makefile
+++ b/drivers/media/usb/ttusb-dec/Makefile
@@ -1,3 +1 @@
 obj-$(CONFIG_DVB_TTUSB_DEC) += ttusb_dec.o ttusbdecfe.o
-
-ccflags-y += -Idrivers/media/dvb-core/
diff --git a/drivers/media/usb/ttusb-dec/ttusb_dec.c b/drivers/media/usb/ttusb-dec/ttusb_dec.c
index cdefb5dfbbdc..44ca66cb9b8f 100644
--- a/drivers/media/usb/ttusb-dec/ttusb_dec.c
+++ b/drivers/media/usb/ttusb-dec/ttusb_dec.c
@@ -30,10 +30,10 @@
 
 #include <linux/mutex.h>
 
-#include "dmxdev.h"
-#include "dvb_demux.h"
-#include "dvb_frontend.h"
-#include "dvb_net.h"
+#include <media/dmxdev.h>
+#include <media/dvb_demux.h>
+#include <media/dvb_frontend.h>
+#include <media/dvb_net.h>
 #include "ttusbdecfe.h"
 
 static int debug;
@@ -127,7 +127,6 @@ struct ttusb_dec {
 	struct urb		*irq_urb;
 	dma_addr_t		irq_dma_handle;
 	void			*iso_buffer;
-	dma_addr_t		iso_dma_handle;
 	struct urb		*iso_urb[ISO_BUF_COUNT];
 	int			iso_stream_count;
 	struct mutex		iso_mutex;
@@ -429,7 +428,7 @@ static int ttusb_dec_audio_pes2ts_cb(void *priv, unsigned char *data)
 	struct ttusb_dec *dec = priv;
 
 	dec->audio_filter->feed->cb.ts(data, 188, NULL, 0,
-				       &dec->audio_filter->feed->feed.ts);
+				       &dec->audio_filter->feed->feed.ts, NULL);
 
 	return 0;
 }
@@ -439,7 +438,7 @@ static int ttusb_dec_video_pes2ts_cb(void *priv, unsigned char *data)
 	struct ttusb_dec *dec = priv;
 
 	dec->video_filter->feed->cb.ts(data, 188, NULL, 0,
-				       &dec->video_filter->feed->feed.ts);
+				       &dec->video_filter->feed->feed.ts, NULL);
 
 	return 0;
 }
@@ -491,7 +490,7 @@ static void ttusb_dec_process_pva(struct ttusb_dec *dec, u8 *pva, int length)
 
 		if (output_pva) {
 			dec->video_filter->feed->cb.ts(pva, length, NULL, 0,
-				&dec->video_filter->feed->feed.ts);
+				&dec->video_filter->feed->feed.ts, NULL);
 			return;
 		}
 
@@ -552,7 +551,7 @@ static void ttusb_dec_process_pva(struct ttusb_dec *dec, u8 *pva, int length)
 	case 0x02:		/* MainAudioStream */
 		if (output_pva) {
 			dec->audio_filter->feed->cb.ts(pva, length, NULL, 0,
-				&dec->audio_filter->feed->feed.ts);
+				&dec->audio_filter->feed->feed.ts, NULL);
 			return;
 		}
 
@@ -590,7 +589,7 @@ static void ttusb_dec_process_filter(struct ttusb_dec *dec, u8 *packet,
 
 	if (filter)
 		filter->feed->cb.sec(&packet[2], length - 2, NULL, 0,
-				     &filter->filter);
+				     &filter->filter, NULL);
 }
 
 static void ttusb_dec_process_packet(struct ttusb_dec *dec)
@@ -1185,11 +1184,7 @@ static void ttusb_dec_free_iso_urbs(struct ttusb_dec *dec)
 
 	for (i = 0; i < ISO_BUF_COUNT; i++)
 		usb_free_urb(dec->iso_urb[i]);
-
-	pci_free_consistent(NULL,
-			    ISO_FRAME_SIZE * (FRAMES_PER_ISO_BUF *
-					      ISO_BUF_COUNT),
-			    dec->iso_buffer, dec->iso_dma_handle);
+	kfree(dec->iso_buffer);
 }
 
 static int ttusb_dec_alloc_iso_urbs(struct ttusb_dec *dec)
@@ -1198,15 +1193,10 @@ static int ttusb_dec_alloc_iso_urbs(struct ttusb_dec *dec)
 
 	dprintk("%s\n", __func__);
 
-	dec->iso_buffer = pci_zalloc_consistent(NULL,
-						ISO_FRAME_SIZE * (FRAMES_PER_ISO_BUF * ISO_BUF_COUNT),
-						&dec->iso_dma_handle);
-
-	if (!dec->iso_buffer) {
-		dprintk("%s: pci_alloc_consistent - not enough memory\n",
-			__func__);
+	dec->iso_buffer = kcalloc(FRAMES_PER_ISO_BUF * ISO_BUF_COUNT,
+			ISO_FRAME_SIZE, GFP_KERNEL);
+	if (!dec->iso_buffer)
 		return -ENOMEM;
-	}
 
 	for (i = 0; i < ISO_BUF_COUNT; i++) {
 		struct urb *urb;
diff --git a/drivers/media/usb/ttusb-dec/ttusbdecfe.c b/drivers/media/usb/ttusb-dec/ttusbdecfe.c
index 09693caa15e2..6ea05d909024 100644
--- a/drivers/media/usb/ttusb-dec/ttusbdecfe.c
+++ b/drivers/media/usb/ttusb-dec/ttusbdecfe.c
@@ -15,7 +15,7 @@
  *
  */
 
-#include "dvb_frontend.h"
+#include <media/dvb_frontend.h>
 #include "ttusbdecfe.h"
 
 
diff --git a/drivers/media/usb/usbtv/Kconfig b/drivers/media/usb/usbtv/Kconfig
index b833c5b9094e..14a0941fa0d0 100644
--- a/drivers/media/usb/usbtv/Kconfig
+++ b/drivers/media/usb/usbtv/Kconfig
@@ -1,11 +1,11 @@
 config VIDEO_USBTV
-        tristate "USBTV007 video capture support"
-        depends on VIDEO_V4L2 && SND
-        select SND_PCM
-        select VIDEOBUF2_VMALLOC
+	tristate "USBTV007 video capture support"
+	depends on VIDEO_V4L2 && SND
+	select SND_PCM
+	select VIDEOBUF2_VMALLOC
 
-        ---help---
-          This is a video4linux2 driver for USBTV007 based video capture devices.
+	---help---
+	  This is a video4linux2 driver for USBTV007 based video capture devices.
 
-          To compile this driver as a module, choose M here: the
-          module will be called usbtv
+	  To compile this driver as a module, choose M here: the
+	  module will be called usbtv
diff --git a/drivers/media/usb/usbvision/Makefile b/drivers/media/usb/usbvision/Makefile
index 9b3a5581df42..494d030b4979 100644
--- a/drivers/media/usb/usbvision/Makefile
+++ b/drivers/media/usb/usbvision/Makefile
@@ -2,5 +2,4 @@ usbvision-objs  := usbvision-core.o usbvision-video.o usbvision-i2c.o usbvision-
 
 obj-$(CONFIG_VIDEO_USBVISION) += usbvision.o
 
-ccflags-y += -Idrivers/media/i2c
 ccflags-y += -Idrivers/media/tuners
diff --git a/drivers/media/usb/usbvision/usbvision-video.c b/drivers/media/usb/usbvision/usbvision-video.c
index 960272d3c924..0f5954a1fea2 100644
--- a/drivers/media/usb/usbvision/usbvision-video.c
+++ b/drivers/media/usb/usbvision/usbvision-video.c
@@ -72,7 +72,6 @@
 #define DRIVER_NAME "usbvision"
 #define DRIVER_ALIAS "USBVision"
 #define DRIVER_DESC "USBVision USB Video Device Driver for Linux"
-#define DRIVER_LICENSE "GPL"
 #define USBVISION_VERSION_STRING "0.9.11"
 
 #define	ENABLE_HEXDUMP	0	/* Enable if you need it */
@@ -141,7 +140,7 @@ MODULE_PARM_DESC(radio_nr, "Set radio device number (/dev/radioX).  Default: -1
 /* Misc stuff */
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_LICENSE("GPL");
 MODULE_VERSION(USBVISION_VERSION_STRING);
 MODULE_ALIAS(DRIVER_ALIAS);
 
diff --git a/drivers/media/usb/uvc/Makefile b/drivers/media/usb/uvc/Makefile
index a4fe5b5d533f..4f9eee4f81ab 100644
--- a/drivers/media/usb/uvc/Makefile
+++ b/drivers/media/usb/uvc/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 uvcvideo-objs  := uvc_driver.o uvc_queue.o uvc_v4l2.o uvc_video.o uvc_ctrl.o \
-		  uvc_status.o uvc_isight.o uvc_debugfs.o
+		  uvc_status.o uvc_isight.o uvc_debugfs.o uvc_metadata.o
 ifeq ($(CONFIG_MEDIA_CONTROLLER),y)
 uvcvideo-objs  += uvc_entity.o
 endif
diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c
index 28b91b7d756f..fd387bf3f02d 100644
--- a/drivers/media/usb/uvc/uvc_driver.c
+++ b/drivers/media/usb/uvc/uvc_driver.c
@@ -24,6 +24,7 @@
 #include <asm/unaligned.h>
 
 #include <media/v4l2-common.h>
+#include <media/v4l2-ioctl.h>
 
 #include "uvcvideo.h"
 
@@ -93,6 +94,11 @@ static struct uvc_format_desc uvc_fmts[] = {
 		.guid		= UVC_GUID_FORMAT_Y8,
 		.fcc		= V4L2_PIX_FMT_GREY,
 	},
+	{
+		.name		= "Greyscale 8-bit (D3DFMT_L8)",
+		.guid		= UVC_GUID_FORMAT_D3DFMT_L8,
+		.fcc		= V4L2_PIX_FMT_GREY,
+	},
 	{
 		.name		= "Greyscale 10-bit (Y10 )",
 		.guid		= UVC_GUID_FORMAT_Y10,
@@ -957,11 +963,11 @@ static int uvc_parse_vendor_control(struct uvc_device *dev,
 		 *	Size of this descriptor, in bytes: 24+p+n*2
 		 * ----------------------------------------------------------
 		 * 23+p+n	bmControlsType	N	Bitmap
-		 * 	Individual bits in the set are defined:
-		 * 	0: Absolute
-		 * 	1: Relative
+		 *	Individual bits in the set are defined:
+		 *	0: Absolute
+		 *	1: Relative
 		 *
-		 * 	This bitset is mapped exactly the same as bmControls.
+		 *	This bitset is mapped exactly the same as bmControls.
 		 * ----------------------------------------------------------
 		 * 23+p+n*2	bReserved	1	Boolean
 		 * ----------------------------------------------------------
@@ -1877,6 +1883,7 @@ static void uvc_unregister_video(struct uvc_device *dev)
 			continue;
 
 		video_unregister_device(&stream->vdev);
+		video_unregister_device(&stream->meta.vdev);
 
 		uvc_debugfs_cleanup_stream(stream);
 	}
@@ -1884,63 +1891,95 @@ static void uvc_unregister_video(struct uvc_device *dev)
 	kref_put(&dev->ref, uvc_delete);
 }
 
-static int uvc_register_video(struct uvc_device *dev,
-		struct uvc_streaming *stream)
+int uvc_register_video_device(struct uvc_device *dev,
+			      struct uvc_streaming *stream,
+			      struct video_device *vdev,
+			      struct uvc_video_queue *queue,
+			      enum v4l2_buf_type type,
+			      const struct v4l2_file_operations *fops,
+			      const struct v4l2_ioctl_ops *ioctl_ops)
 {
-	struct video_device *vdev = &stream->vdev;
 	int ret;
 
 	/* Initialize the video buffers queue. */
-	ret = uvc_queue_init(&stream->queue, stream->type, !uvc_no_drop_param);
+	ret = uvc_queue_init(queue, type, !uvc_no_drop_param);
 	if (ret)
 		return ret;
 
-	/* Initialize the streaming interface with default streaming
-	 * parameters.
-	 */
-	ret = uvc_video_init(stream);
-	if (ret < 0) {
-		uvc_printk(KERN_ERR, "Failed to initialize the device "
-			"(%d).\n", ret);
-		return ret;
-	}
-
-	uvc_debugfs_init_stream(stream);
-
 	/* Register the device with V4L. */
 
-	/* We already hold a reference to dev->udev. The video device will be
+	/*
+	 * We already hold a reference to dev->udev. The video device will be
 	 * unregistered before the reference is released, so we don't need to
 	 * get another one.
 	 */
 	vdev->v4l2_dev = &dev->vdev;
-	vdev->fops = &uvc_fops;
-	vdev->ioctl_ops = &uvc_ioctl_ops;
+	vdev->fops = fops;
+	vdev->ioctl_ops = ioctl_ops;
 	vdev->release = uvc_release;
 	vdev->prio = &stream->chain->prio;
-	if (stream->type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
+	if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT)
 		vdev->vfl_dir = VFL_DIR_TX;
+	else
+		vdev->vfl_dir = VFL_DIR_RX;
+
+	switch (type) {
+	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
+	default:
+		vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
+		break;
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
+		vdev->device_caps = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING;
+		break;
+	case V4L2_BUF_TYPE_META_CAPTURE:
+		vdev->device_caps = V4L2_CAP_META_CAPTURE | V4L2_CAP_STREAMING;
+		break;
+	}
+
 	strlcpy(vdev->name, dev->name, sizeof vdev->name);
 
-	/* Set the driver data before calling video_register_device, otherwise
-	 * uvc_v4l2_open might race us.
+	/*
+	 * Set the driver data before calling video_register_device, otherwise
+	 * the file open() handler might race us.
 	 */
 	video_set_drvdata(vdev, stream);
 
 	ret = video_register_device(vdev, VFL_TYPE_GRABBER, -1);
 	if (ret < 0) {
-		uvc_printk(KERN_ERR, "Failed to register video device (%d).\n",
+		uvc_printk(KERN_ERR, "Failed to register %s device (%d).\n",
+			   v4l2_type_names[type], ret);
+		return ret;
+	}
+
+	kref_get(&dev->ref);
+	return 0;
+}
+
+static int uvc_register_video(struct uvc_device *dev,
+		struct uvc_streaming *stream)
+{
+	int ret;
+
+	/* Initialize the streaming interface with default parameters. */
+	ret = uvc_video_init(stream);
+	if (ret < 0) {
+		uvc_printk(KERN_ERR, "Failed to initialize the device (%d).\n",
 			   ret);
 		return ret;
 	}
 
 	if (stream->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		stream->chain->caps |= V4L2_CAP_VIDEO_CAPTURE;
+		stream->chain->caps |= V4L2_CAP_VIDEO_CAPTURE
+			| V4L2_CAP_META_CAPTURE;
 	else
 		stream->chain->caps |= V4L2_CAP_VIDEO_OUTPUT;
 
-	kref_get(&dev->ref);
-	return 0;
+	uvc_debugfs_init_stream(stream);
+
+	/* Register the device with V4L. */
+	return uvc_register_video_device(dev, stream, &stream->vdev,
+					 &stream->queue, stream->type,
+					 &uvc_fops, &uvc_ioctl_ops);
 }
 
 /*
@@ -1969,6 +2008,11 @@ static int uvc_register_terms(struct uvc_device *dev,
 		if (ret < 0)
 			return ret;
 
+		/* Register a metadata node, but ignore a possible failure,
+		 * complete registration of video nodes anyway.
+		 */
+		uvc_meta_register(stream);
+
 		term->vdev = &stream->vdev;
 	}
 
@@ -2001,11 +2045,19 @@ static int uvc_register_chains(struct uvc_device *dev)
  * USB probe, disconnect, suspend and resume
  */
 
+struct uvc_device_info {
+	u32	quirks;
+	u32	meta_format;
+};
+
 static int uvc_probe(struct usb_interface *intf,
 		     const struct usb_device_id *id)
 {
 	struct usb_device *udev = interface_to_usbdev(intf);
 	struct uvc_device *dev;
+	const struct uvc_device_info *info =
+		(const struct uvc_device_info *)id->driver_info;
+	u32 quirks = info ? info->quirks : 0;
 	int function;
 	int ret;
 
@@ -2032,7 +2084,9 @@ static int uvc_probe(struct usb_interface *intf,
 	dev->intf = usb_get_intf(intf);
 	dev->intfnum = intf->cur_altsetting->desc.bInterfaceNumber;
 	dev->quirks = (uvc_quirks_param == -1)
-		    ? id->driver_info : uvc_quirks_param;
+		    ? quirks : uvc_quirks_param;
+	if (info)
+		dev->meta_format = info->meta_format;
 
 	if (udev->product != NULL)
 		strlcpy(dev->name, udev->product, sizeof dev->name);
@@ -2073,7 +2127,7 @@ static int uvc_probe(struct usb_interface *intf,
 		le16_to_cpu(udev->descriptor.idVendor),
 		le16_to_cpu(udev->descriptor.idProduct));
 
-	if (dev->quirks != id->driver_info) {
+	if (dev->quirks != quirks) {
 		uvc_printk(KERN_INFO, "Forcing device quirks to 0x%x by module "
 			"parameter for testing purpose.\n", dev->quirks);
 		uvc_printk(KERN_INFO, "Please report required quirks to the "
@@ -2271,6 +2325,28 @@ MODULE_PARM_DESC(timeout, "Streaming control requests timeout");
  * Driver initialization and cleanup
  */
 
+static const struct uvc_device_info uvc_quirk_probe_minmax = {
+	.quirks = UVC_QUIRK_PROBE_MINMAX,
+};
+
+static const struct uvc_device_info uvc_quirk_fix_bandwidth = {
+	.quirks = UVC_QUIRK_FIX_BANDWIDTH,
+};
+
+static const struct uvc_device_info uvc_quirk_probe_def = {
+	.quirks = UVC_QUIRK_PROBE_DEF,
+};
+
+static const struct uvc_device_info uvc_quirk_stream_no_fid = {
+	.quirks = UVC_QUIRK_STREAM_NO_FID,
+};
+
+static const struct uvc_device_info uvc_quirk_force_y8 = {
+	.quirks = UVC_QUIRK_FORCE_Y8,
+};
+
+#define UVC_QUIRK_INFO(q) (kernel_ulong_t)&(struct uvc_device_info){.quirks = q}
+
 /*
  * The Logitech cameras listed below have their interface class set to
  * VENDOR_SPEC because they don't announce themselves as UVC devices, even
@@ -2285,7 +2361,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* Genius eFace 2025 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2294,7 +2370,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* Microsoft Lifecam NX-6000 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2303,7 +2379,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* Microsoft Lifecam NX-3000 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2312,7 +2388,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* Microsoft Lifecam VX-7000 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2321,7 +2397,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* Logitech Quickcam Fusion */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2378,7 +2454,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_RESTORE_CTRLS_ON_INIT },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_RESTORE_CTRLS_ON_INIT) },
 	/* Chicony CNF7129 (Asus EEE 100HE) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2387,7 +2463,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_RESTRICT_FRAME_RATE },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_RESTRICT_FRAME_RATE) },
 	/* Alcor Micro AU3820 (Future Boy PC USB Webcam) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2396,7 +2472,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* Dell XPS m1530 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2405,7 +2481,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info 		= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* Dell SP2008WFP Monitor */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2414,7 +2490,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info 		= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* Dell Alienware X51 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2423,7 +2499,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info	= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* Dell Studio Hybrid 140g (OmniVision webcam) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2432,7 +2508,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* Dell XPS M1330 (OmniVision OV7670 webcam) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2441,7 +2517,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* Apple Built-In iSight */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2450,8 +2526,8 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info 		= UVC_QUIRK_PROBE_MINMAX
-				| UVC_QUIRK_BUILTIN_ISIGHT },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_PROBE_MINMAX
+					| UVC_QUIRK_BUILTIN_ISIGHT) },
 	/* Apple Built-In iSight via iBridge */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2460,7 +2536,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* Foxlink ("HP Webcam" on HP Mini 5103) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2469,7 +2545,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_FIX_BANDWIDTH },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_fix_bandwidth },
 	/* Genesys Logic USB 2.0 PC Camera */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2478,7 +2554,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Hercules Classic Silver */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2487,7 +2563,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_FIX_BANDWIDTH },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_fix_bandwidth },
 	/* ViMicro Vega */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2496,7 +2572,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_FIX_BANDWIDTH },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_fix_bandwidth },
 	/* ViMicro - Minoru3D */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2505,7 +2581,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_FIX_BANDWIDTH },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_fix_bandwidth },
 	/* ViMicro Venus - Minoru3D */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2514,7 +2590,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_FIX_BANDWIDTH },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_fix_bandwidth },
 	/* Ophir Optronics - SPCAM 620U */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2523,7 +2599,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* MT6227 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2532,8 +2608,8 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX
-				| UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_PROBE_MINMAX
+					| UVC_QUIRK_PROBE_DEF) },
 	/* IMC Networks (Medion Akoya) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2542,7 +2618,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* JMicron USB2.0 XGA WebCam */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2551,7 +2627,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* Syntek (HP Spartan) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2560,7 +2636,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Syntek (Samsung Q310) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2569,7 +2645,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Syntek (Packard Bell EasyNote MX52 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2578,7 +2654,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Syntek (Asus F9SG) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2587,7 +2663,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Syntek (Asus U3S) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2596,7 +2672,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Syntek (JAOtech Smart Terminal) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2605,7 +2681,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Miricle 307K */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2614,7 +2690,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Lenovo Thinkpad SL400/SL500 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2623,7 +2699,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STREAM_NO_FID },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_stream_no_fid },
 	/* Aveo Technology USB 2.0 Camera */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2632,8 +2708,8 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX
-				| UVC_QUIRK_PROBE_EXTRAFIELDS },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_PROBE_MINMAX
+					| UVC_QUIRK_PROBE_EXTRAFIELDS) },
 	/* Aveo Technology USB 2.0 Camera (Tasco USB Microscope) */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2650,7 +2726,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_EXTRAFIELDS },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_PROBE_EXTRAFIELDS) },
 	/* Manta MM-353 Plako */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2659,7 +2735,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* FSC WebCam V30S */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2668,7 +2744,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* Arkmicro unbranded */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2677,7 +2753,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_DEF },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_def },
 	/* The Imaging Source USB CCD cameras */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2696,7 +2772,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_STATUS_INTERVAL },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_STATUS_INTERVAL) },
 	/* MSI StarCam 370i */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2705,7 +2781,16 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
+	/* Generalplus Technology Inc. 808 Camera */
+	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
+				| USB_DEVICE_ID_MATCH_INT_INFO,
+	  .idVendor		= 0x1b3f,
+	  .idProduct		= 0x2002,
+	  .bInterfaceClass	= USB_CLASS_VIDEO,
+	  .bInterfaceSubClass	= 1,
+	  .bInterfaceProtocol	= 0,
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_probe_minmax },
 	/* SiGma Micro USB Web Camera */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2714,8 +2799,8 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_PROBE_MINMAX
-				| UVC_QUIRK_IGNORE_SELECTOR_UNIT },
+	  .driver_info		= UVC_QUIRK_INFO(UVC_QUIRK_PROBE_MINMAX
+					| UVC_QUIRK_IGNORE_SELECTOR_UNIT) },
 	/* Oculus VR Positional Tracker DK2 */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2724,7 +2809,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_FORCE_Y8 },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_force_y8 },
 	/* Oculus VR Rift Sensor */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
 				| USB_DEVICE_ID_MATCH_INT_INFO,
@@ -2733,7 +2818,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .bInterfaceClass	= USB_CLASS_VENDOR_SPEC,
 	  .bInterfaceSubClass	= 1,
 	  .bInterfaceProtocol	= 0,
-	  .driver_info		= UVC_QUIRK_FORCE_Y8 },
+	  .driver_info		= (kernel_ulong_t)&uvc_quirk_force_y8 },
 	/* Generic USB Video Class */
 	{ USB_INTERFACE_INFO(USB_CLASS_VIDEO, 1, UVC_PC_PROTOCOL_UNDEFINED) },
 	{ USB_INTERFACE_INFO(USB_CLASS_VIDEO, 1, UVC_PC_PROTOCOL_15) },
diff --git a/drivers/media/usb/uvc/uvc_isight.c b/drivers/media/usb/uvc/uvc_isight.c
index 8510e7259e76..5059fbf41020 100644
--- a/drivers/media/usb/uvc/uvc_isight.c
+++ b/drivers/media/usb/uvc/uvc_isight.c
@@ -27,11 +27,11 @@
  *
  * Offset   Size (bytes)	Description
  * ------------------------------------------------------------------
- * 0x00 	1   	Header length
- * 0x01 	1   	Flags (UVC-compliant)
- * 0x02 	4   	Always equal to '11223344'
- * 0x06 	8   	Always equal to 'deadbeefdeadface'
- * 0x0e 	16  	Unknown
+ * 0x00	1	Header length
+ * 0x01	1	Flags (UVC-compliant)
+ * 0x02	4	Always equal to '11223344'
+ * 0x06	8	Always equal to 'deadbeefdeadface'
+ * 0x0e	16	Unknown
  *
  * The header can be prefixed by an optional, unknown-purpose byte.
  */
@@ -100,7 +100,7 @@ static int isight_decode(struct uvc_video_queue *queue, struct uvc_buffer *buf,
 }
 
 void uvc_video_decode_isight(struct urb *urb, struct uvc_streaming *stream,
-		struct uvc_buffer *buf)
+			struct uvc_buffer *buf, struct uvc_buffer *meta_buf)
 {
 	int ret, i;
 
diff --git a/drivers/media/usb/uvc/uvc_metadata.c b/drivers/media/usb/uvc/uvc_metadata.c
new file mode 100644
index 000000000000..cd1aec19cc5b
--- /dev/null
+++ b/drivers/media/usb/uvc/uvc_metadata.c
@@ -0,0 +1,179 @@
+/*
+ *      uvc_metadata.c  --  USB Video Class driver - Metadata handling
+ *
+ *      Copyright (C) 2016
+ *          Guennadi Liakhovetski (guennadi.liakhovetski@intel.com)
+ *
+ *      This program is free software; you can redistribute it and/or modify
+ *      it under the terms of the GNU General Public License as published by
+ *      the Free Software Foundation; either version 2 of the License, or
+ *      (at your option) any later version.
+ */
+
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/usb.h>
+#include <linux/videodev2.h>
+
+#include <media/v4l2-ioctl.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/videobuf2-vmalloc.h>
+
+#include "uvcvideo.h"
+
+/* -----------------------------------------------------------------------------
+ * V4L2 ioctls
+ */
+
+static int uvc_meta_v4l2_querycap(struct file *file, void *fh,
+				  struct v4l2_capability *cap)
+{
+	struct v4l2_fh *vfh = file->private_data;
+	struct uvc_streaming *stream = video_get_drvdata(vfh->vdev);
+	struct uvc_video_chain *chain = stream->chain;
+
+	strlcpy(cap->driver, "uvcvideo", sizeof(cap->driver));
+	strlcpy(cap->card, vfh->vdev->name, sizeof(cap->card));
+	usb_make_path(stream->dev->udev, cap->bus_info, sizeof(cap->bus_info));
+	cap->capabilities = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_STREAMING
+			  | chain->caps;
+
+	return 0;
+}
+
+static int uvc_meta_v4l2_get_format(struct file *file, void *fh,
+				    struct v4l2_format *format)
+{
+	struct v4l2_fh *vfh = file->private_data;
+	struct uvc_streaming *stream = video_get_drvdata(vfh->vdev);
+	struct v4l2_meta_format *fmt = &format->fmt.meta;
+
+	if (format->type != vfh->vdev->queue->type)
+		return -EINVAL;
+
+	memset(fmt, 0, sizeof(*fmt));
+
+	fmt->dataformat = stream->meta.format;
+	fmt->buffersize = UVC_METATADA_BUF_SIZE;
+
+	return 0;
+}
+
+static int uvc_meta_v4l2_try_format(struct file *file, void *fh,
+				    struct v4l2_format *format)
+{
+	struct v4l2_fh *vfh = file->private_data;
+	struct uvc_streaming *stream = video_get_drvdata(vfh->vdev);
+	struct uvc_device *dev = stream->dev;
+	struct v4l2_meta_format *fmt = &format->fmt.meta;
+	u32 fmeta = fmt->dataformat;
+
+	if (format->type != vfh->vdev->queue->type)
+		return -EINVAL;
+
+	memset(fmt, 0, sizeof(*fmt));
+
+	fmt->dataformat = fmeta == dev->meta_format ? fmeta : V4L2_META_FMT_UVC;
+	fmt->buffersize = UVC_METATADA_BUF_SIZE;
+
+	return 0;
+}
+
+static int uvc_meta_v4l2_set_format(struct file *file, void *fh,
+				    struct v4l2_format *format)
+{
+	struct v4l2_fh *vfh = file->private_data;
+	struct uvc_streaming *stream = video_get_drvdata(vfh->vdev);
+	struct v4l2_meta_format *fmt = &format->fmt.meta;
+	int ret;
+
+	ret = uvc_meta_v4l2_try_format(file, fh, format);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * We could in principle switch at any time, also during streaming.
+	 * Metadata buffers would still be perfectly parseable, but it's more
+	 * consistent and cleaner to disallow that.
+	 */
+	mutex_lock(&stream->mutex);
+
+	if (uvc_queue_allocated(&stream->queue))
+		ret = -EBUSY;
+	else
+		stream->meta.format = fmt->dataformat;
+
+	mutex_unlock(&stream->mutex);
+
+	return ret;
+}
+
+static int uvc_meta_v4l2_enum_formats(struct file *file, void *fh,
+				      struct v4l2_fmtdesc *fdesc)
+{
+	struct v4l2_fh *vfh = file->private_data;
+	struct uvc_streaming *stream = video_get_drvdata(vfh->vdev);
+	struct uvc_device *dev = stream->dev;
+	u32 index = fdesc->index;
+
+	if (fdesc->type != vfh->vdev->queue->type ||
+	    index > 1U || (index && !dev->meta_format))
+		return -EINVAL;
+
+	memset(fdesc, 0, sizeof(*fdesc));
+
+	fdesc->type = vfh->vdev->queue->type;
+	fdesc->index = index;
+	fdesc->pixelformat = index ? dev->meta_format : V4L2_META_FMT_UVC;
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops uvc_meta_ioctl_ops = {
+	.vidioc_querycap		= uvc_meta_v4l2_querycap,
+	.vidioc_g_fmt_meta_cap		= uvc_meta_v4l2_get_format,
+	.vidioc_s_fmt_meta_cap		= uvc_meta_v4l2_set_format,
+	.vidioc_try_fmt_meta_cap	= uvc_meta_v4l2_try_format,
+	.vidioc_enum_fmt_meta_cap	= uvc_meta_v4l2_enum_formats,
+	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
+	.vidioc_querybuf		= vb2_ioctl_querybuf,
+	.vidioc_qbuf			= vb2_ioctl_qbuf,
+	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
+	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
+	.vidioc_streamon		= vb2_ioctl_streamon,
+	.vidioc_streamoff		= vb2_ioctl_streamoff,
+};
+
+/* -----------------------------------------------------------------------------
+ * V4L2 File Operations
+ */
+
+static const struct v4l2_file_operations uvc_meta_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = video_ioctl2,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
+	.poll = vb2_fop_poll,
+	.mmap = vb2_fop_mmap,
+};
+
+int uvc_meta_register(struct uvc_streaming *stream)
+{
+	struct uvc_device *dev = stream->dev;
+	struct video_device *vdev = &stream->meta.vdev;
+	struct uvc_video_queue *queue = &stream->meta.queue;
+
+	stream->meta.format = V4L2_META_FMT_UVC;
+
+	/*
+	 * The video interface queue uses manual locking and thus does not set
+	 * the queue pointer. Set it manually here.
+	 */
+	vdev->queue = &queue->queue;
+
+	return uvc_register_video_device(dev, stream, vdev, queue,
+					 V4L2_BUF_TYPE_META_CAPTURE,
+					 &uvc_meta_fops, &uvc_meta_ioctl_ops);
+}
diff --git a/drivers/media/usb/uvc/uvc_queue.c b/drivers/media/usb/uvc/uvc_queue.c
index c8d78b2f3de4..fecccb5e7628 100644
--- a/drivers/media/usb/uvc/uvc_queue.c
+++ b/drivers/media/usb/uvc/uvc_queue.c
@@ -79,8 +79,19 @@ static int uvc_queue_setup(struct vb2_queue *vq,
 			   unsigned int sizes[], struct device *alloc_devs[])
 {
 	struct uvc_video_queue *queue = vb2_get_drv_priv(vq);
-	struct uvc_streaming *stream = uvc_queue_to_stream(queue);
-	unsigned size = stream->ctrl.dwMaxVideoFrameSize;
+	struct uvc_streaming *stream;
+	unsigned int size;
+
+	switch (vq->type) {
+	case V4L2_BUF_TYPE_META_CAPTURE:
+		size = UVC_METATADA_BUF_SIZE;
+		break;
+
+	default:
+		stream = uvc_queue_to_stream(queue);
+		size = stream->ctrl.dwMaxVideoFrameSize;
+		break;
+	}
 
 	/*
 	 * When called with plane sizes, validate them. The driver supports
@@ -114,7 +125,7 @@ static int uvc_buffer_prepare(struct vb2_buffer *vb)
 	buf->error = 0;
 	buf->mem = vb2_plane_vaddr(vb, 0);
 	buf->length = vb2_plane_size(vb, 0);
-	if (vb->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
+	if (vb->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
 		buf->bytesused = 0;
 	else
 		buf->bytesused = vb2_get_plane_payload(vb, 0);
@@ -177,10 +188,10 @@ static int uvc_start_streaming(struct vb2_queue *vq, unsigned int count)
 static void uvc_stop_streaming(struct vb2_queue *vq)
 {
 	struct uvc_video_queue *queue = vb2_get_drv_priv(vq);
-	struct uvc_streaming *stream = uvc_queue_to_stream(queue);
 	unsigned long flags;
 
-	uvc_video_enable(stream, 0);
+	if (vq->type != V4L2_BUF_TYPE_META_CAPTURE)
+		uvc_video_enable(uvc_queue_to_stream(queue), 0);
 
 	spin_lock_irqsave(&queue->irqlock, flags);
 	uvc_queue_return_buffers(queue, UVC_BUF_STATE_ERROR);
@@ -198,20 +209,39 @@ static const struct vb2_ops uvc_queue_qops = {
 	.stop_streaming = uvc_stop_streaming,
 };
 
+static const struct vb2_ops uvc_meta_queue_qops = {
+	.queue_setup = uvc_queue_setup,
+	.buf_prepare = uvc_buffer_prepare,
+	.buf_queue = uvc_buffer_queue,
+	.wait_prepare = vb2_ops_wait_prepare,
+	.wait_finish = vb2_ops_wait_finish,
+	.stop_streaming = uvc_stop_streaming,
+};
+
 int uvc_queue_init(struct uvc_video_queue *queue, enum v4l2_buf_type type,
 		    int drop_corrupted)
 {
 	int ret;
 
 	queue->queue.type = type;
-	queue->queue.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF;
+	queue->queue.io_modes = VB2_MMAP | VB2_USERPTR;
 	queue->queue.drv_priv = queue;
 	queue->queue.buf_struct_size = sizeof(struct uvc_buffer);
-	queue->queue.ops = &uvc_queue_qops;
 	queue->queue.mem_ops = &vb2_vmalloc_memops;
 	queue->queue.timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC
 		| V4L2_BUF_FLAG_TSTAMP_SRC_SOE;
 	queue->queue.lock = &queue->mutex;
+
+	switch (type) {
+	case V4L2_BUF_TYPE_META_CAPTURE:
+		queue->queue.ops = &uvc_meta_queue_qops;
+		break;
+	default:
+		queue->queue.io_modes |= VB2_DMABUF;
+		queue->queue.ops = &uvc_queue_qops;
+		break;
+	}
+
 	ret = vb2_queue_init(&queue->queue);
 	if (ret)
 		return ret;
@@ -340,10 +370,10 @@ unsigned long uvc_queue_get_unmapped_area(struct uvc_video_queue *queue,
 }
 #endif
 
-unsigned int uvc_queue_poll(struct uvc_video_queue *queue, struct file *file,
+__poll_t uvc_queue_poll(struct uvc_video_queue *queue, struct file *file,
 			    poll_table *wait)
 {
-	unsigned int ret;
+	__poll_t ret;
 
 	mutex_lock(&queue->mutex);
 	ret = vb2_poll(&queue->queue, file, wait);
diff --git a/drivers/media/usb/uvc/uvc_status.c b/drivers/media/usb/uvc/uvc_status.c
index f552ab997956..1ef20e74b7ac 100644
--- a/drivers/media/usb/uvc/uvc_status.c
+++ b/drivers/media/usb/uvc/uvc_status.c
@@ -93,8 +93,9 @@ static void uvc_event_streaming(struct uvc_device *dev, __u8 *data, int len)
 			data[1], data[3] ? "pressed" : "released", len);
 		uvc_input_report_key(dev, KEY_CAMERA, data[3]);
 	} else {
-		uvc_trace(UVC_TRACE_STATUS, "Stream %u error event %02x %02x "
-			"len %d.\n", data[1], data[2], data[3], len);
+		uvc_trace(UVC_TRACE_STATUS,
+			  "Stream %u error event %02x len %d.\n",
+			  data[1], data[2], len);
 	}
 }
 
diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c
index 3e7e283a44a8..a13ad4e178be 100644
--- a/drivers/media/usb/uvc/uvc_v4l2.c
+++ b/drivers/media/usb/uvc/uvc_v4l2.c
@@ -568,10 +568,6 @@ static int uvc_ioctl_querycap(struct file *file, void *fh,
 	usb_make_path(stream->dev->udev, cap->bus_info, sizeof(cap->bus_info));
 	cap->capabilities = V4L2_CAP_DEVICE_CAPS | V4L2_CAP_STREAMING
 			  | chain->caps;
-	if (stream->type == V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		cap->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
-	else
-		cap->device_caps = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING;
 
 	return 0;
 }
@@ -1284,36 +1280,30 @@ struct uvc_xu_control_mapping32 {
 static int uvc_v4l2_get_xu_mapping(struct uvc_xu_control_mapping *kp,
 			const struct uvc_xu_control_mapping32 __user *up)
 {
-	compat_caddr_t p;
+	struct uvc_xu_control_mapping32 *p = (void *)kp;
+	compat_caddr_t info;
+	u32 count;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    __copy_from_user(kp, up, offsetof(typeof(*up), menu_info)) ||
-	    __get_user(kp->menu_count, &up->menu_count))
+	if (copy_from_user(p, up, sizeof(*p)))
 		return -EFAULT;
 
-	memset(kp->reserved, 0, sizeof(kp->reserved));
-
-	if (kp->menu_count == 0) {
-		kp->menu_info = NULL;
-		return 0;
-	}
-
-	if (__get_user(p, &up->menu_info))
-		return -EFAULT;
-	kp->menu_info = compat_ptr(p);
+	count = p->menu_count;
+	info = p->menu_info;
 
+	memset(kp->reserved, 0, sizeof(kp->reserved));
+	kp->menu_info = count ? compat_ptr(info) : NULL;
+	kp->menu_count = count;
 	return 0;
 }
 
 static int uvc_v4l2_put_xu_mapping(const struct uvc_xu_control_mapping *kp,
 			struct uvc_xu_control_mapping32 __user *up)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    __copy_to_user(up, kp, offsetof(typeof(*up), menu_info)) ||
-	    __put_user(kp->menu_count, &up->menu_count))
+	if (copy_to_user(up, kp, offsetof(typeof(*up), menu_info)) ||
+	    put_user(kp->menu_count, &up->menu_count))
 		return -EFAULT;
 
-	if (__clear_user(up->reserved, sizeof(up->reserved)))
+	if (clear_user(up->reserved, sizeof(up->reserved)))
 		return -EFAULT;
 
 	return 0;
@@ -1330,31 +1320,26 @@ struct uvc_xu_control_query32 {
 static int uvc_v4l2_get_xu_query(struct uvc_xu_control_query *kp,
 			const struct uvc_xu_control_query32 __user *up)
 {
-	compat_caddr_t p;
+	struct uvc_xu_control_query32 v;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
-	    __copy_from_user(kp, up, offsetof(typeof(*up), data)))
+	if (copy_from_user(&v, up, sizeof(v)))
 		return -EFAULT;
 
-	if (kp->size == 0) {
-		kp->data = NULL;
-		return 0;
-	}
-
-	if (__get_user(p, &up->data))
-		return -EFAULT;
-	kp->data = compat_ptr(p);
-
+	*kp = (struct uvc_xu_control_query){
+		.unit = v.unit,
+		.selector = v.selector,
+		.query = v.query,
+		.size = v.size,
+		.data = v.size ? compat_ptr(v.data) : NULL
+	};
 	return 0;
 }
 
 static int uvc_v4l2_put_xu_query(const struct uvc_xu_control_query *kp,
 			struct uvc_xu_control_query32 __user *up)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
-	    __copy_to_user(up, kp, offsetof(typeof(*up), data)))
+	if (copy_to_user(up, kp, offsetof(typeof(*up), data)))
 		return -EFAULT;
-
 	return 0;
 }
 
@@ -1423,7 +1408,7 @@ static int uvc_v4l2_mmap(struct file *file, struct vm_area_struct *vma)
 	return uvc_queue_mmap(&stream->queue, vma);
 }
 
-static unsigned int uvc_v4l2_poll(struct file *file, poll_table *wait)
+static __poll_t uvc_v4l2_poll(struct file *file, poll_table *wait)
 {
 	struct uvc_fh *handle = file->private_data;
 	struct uvc_streaming *stream = handle->stream;
diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c
index fb86d6af398d..5441553f74e1 100644
--- a/drivers/media/usb/uvc/uvc_video.c
+++ b/drivers/media/usb/uvc/uvc_video.c
@@ -369,12 +369,12 @@ static int uvc_commit_video(struct uvc_streaming *stream,
  * Clocks and timestamps
  */
 
-static inline void uvc_video_get_ts(struct timespec *ts)
+static inline ktime_t uvc_video_get_time(void)
 {
 	if (uvc_clock_param == CLOCK_MONOTONIC)
-		ktime_get_ts(ts);
+		return ktime_get();
 	else
-		ktime_get_real_ts(ts);
+		return ktime_get_real();
 }
 
 static void
@@ -386,7 +386,7 @@ uvc_video_clock_decode(struct uvc_streaming *stream, struct uvc_buffer *buf,
 	bool has_pts = false;
 	bool has_scr = false;
 	unsigned long flags;
-	struct timespec ts;
+	ktime_t time;
 	u16 host_sof;
 	u16 dev_sof;
 
@@ -436,7 +436,7 @@ uvc_video_clock_decode(struct uvc_streaming *stream, struct uvc_buffer *buf,
 	stream->clock.last_sof = dev_sof;
 
 	host_sof = usb_get_current_frame_number(stream->dev->udev);
-	uvc_video_get_ts(&ts);
+	time = uvc_video_get_time();
 
 	/* The UVC specification allows device implementations that can't obtain
 	 * the USB frame number to keep their own frame counters as long as they
@@ -473,7 +473,7 @@ uvc_video_clock_decode(struct uvc_streaming *stream, struct uvc_buffer *buf,
 	sample->dev_stc = get_unaligned_le32(&data[header_size - 6]);
 	sample->dev_sof = dev_sof;
 	sample->host_sof = host_sof;
-	sample->host_ts = ts;
+	sample->host_time = time;
 
 	/* Update the sliding window head and count. */
 	stream->clock.head = (stream->clock.head + 1) % stream->clock.size;
@@ -613,14 +613,12 @@ void uvc_video_clock_update(struct uvc_streaming *stream,
 	struct uvc_clock_sample *first;
 	struct uvc_clock_sample *last;
 	unsigned long flags;
-	struct timespec ts;
+	u64 timestamp;
 	u32 delta_stc;
 	u32 y1, y2;
 	u32 x1, x2;
 	u32 mean;
 	u32 sof;
-	u32 div;
-	u32 rem;
 	u64 y;
 
 	if (!uvc_hw_timestamps_param)
@@ -667,9 +665,8 @@ void uvc_video_clock_update(struct uvc_streaming *stream,
 	if (x1 == x2)
 		goto done;
 
-	ts = timespec_sub(last->host_ts, first->host_ts);
 	y1 = NSEC_PER_SEC;
-	y2 = (ts.tv_sec + 1) * NSEC_PER_SEC + ts.tv_nsec;
+	y2 = (u32)ktime_to_ns(ktime_sub(last->host_time, first->host_time)) + y1;
 
 	/* Interpolated and host SOF timestamps can wrap around at slightly
 	 * different times. Handle this by adding or removing 2048 to or from
@@ -686,24 +683,18 @@ void uvc_video_clock_update(struct uvc_streaming *stream,
 	  - (u64)y2 * (u64)x1;
 	y = div_u64(y, x2 - x1);
 
-	div = div_u64_rem(y, NSEC_PER_SEC, &rem);
-	ts.tv_sec = first->host_ts.tv_sec - 1 + div;
-	ts.tv_nsec = first->host_ts.tv_nsec + rem;
-	if (ts.tv_nsec >= NSEC_PER_SEC) {
-		ts.tv_sec++;
-		ts.tv_nsec -= NSEC_PER_SEC;
-	}
+	timestamp = ktime_to_ns(first->host_time) + y - y1;
 
 	uvc_trace(UVC_TRACE_CLOCK, "%s: SOF %u.%06llu y %llu ts %llu "
 		  "buf ts %llu (x1 %u/%u/%u x2 %u/%u/%u y1 %u y2 %u)\n",
 		  stream->dev->name,
 		  sof >> 16, div_u64(((u64)sof & 0xffff) * 1000000LLU, 65536),
-		  y, timespec_to_ns(&ts), vbuf->vb2_buf.timestamp,
+		  y, timestamp, vbuf->vb2_buf.timestamp,
 		  x1, first->host_sof, first->dev_sof,
 		  x2, last->host_sof, last->dev_sof, y1, y2);
 
 	/* Update the V4L2 buffer. */
-	vbuf->vb2_buf.timestamp = timespec_to_ns(&ts);
+	vbuf->vb2_buf.timestamp = timestamp;
 
 done:
 	spin_unlock_irqrestore(&clock->lock, flags);
@@ -725,7 +716,7 @@ static void uvc_video_stats_decode(struct uvc_streaming *stream,
 
 	if (stream->stats.stream.nb_frames == 0 &&
 	    stream->stats.frame.nb_packets == 0)
-		ktime_get_ts(&stream->stats.stream.start_ts);
+		stream->stats.stream.start_ts = ktime_get();
 
 	switch (data[1] & (UVC_STREAM_PTS | UVC_STREAM_SCR)) {
 	case UVC_STREAM_PTS | UVC_STREAM_SCR:
@@ -865,16 +856,13 @@ size_t uvc_video_stats_dump(struct uvc_streaming *stream, char *buf,
 {
 	unsigned int scr_sof_freq;
 	unsigned int duration;
-	struct timespec ts;
 	size_t count = 0;
 
-	ts = timespec_sub(stream->stats.stream.stop_ts,
-			  stream->stats.stream.start_ts);
-
 	/* Compute the SCR.SOF frequency estimate. At the nominal 1kHz SOF
 	 * frequency this will not overflow before more than 1h.
 	 */
-	duration = ts.tv_sec * 1000 + ts.tv_nsec / 1000000;
+	duration = ktime_ms_delta(stream->stats.stream.stop_ts,
+				  stream->stats.stream.start_ts);
 	if (duration != 0)
 		scr_sof_freq = stream->stats.stream.scr_sof_count * 1000
 			     / duration;
@@ -915,7 +903,7 @@ static void uvc_video_stats_start(struct uvc_streaming *stream)
 
 static void uvc_video_stats_stop(struct uvc_streaming *stream)
 {
-	ktime_get_ts(&stream->stats.stream.stop_ts);
+	stream->stats.stream.stop_ts = ktime_get();
 }
 
 /* ------------------------------------------------------------------------
@@ -1010,8 +998,6 @@ static int uvc_video_decode_start(struct uvc_streaming *stream,
 	 * when the EOF bit is set to force synchronisation on the next packet.
 	 */
 	if (buf->state != UVC_BUF_STATE_ACTIVE) {
-		struct timespec ts;
-
 		if (fid == stream->last_fid) {
 			uvc_trace(UVC_TRACE_FRAME, "Dropping payload (out of "
 				"sync).\n");
@@ -1021,11 +1007,9 @@ static int uvc_video_decode_start(struct uvc_streaming *stream,
 			return -ENODATA;
 		}
 
-		uvc_video_get_ts(&ts);
-
 		buf->buf.field = V4L2_FIELD_NONE;
 		buf->buf.sequence = stream->sequence;
-		buf->buf.vb2_buf.timestamp = timespec_to_ns(&ts);
+		buf->buf.vb2_buf.timestamp = uvc_video_get_time();
 
 		/* TODO: Handle PTS and SCR. */
 		buf->state = UVC_BUF_STATE_ACTIVE;
@@ -1077,6 +1061,7 @@ static void uvc_video_decode_data(struct uvc_streaming *stream,
 	/* Complete the current frame if the buffer size was exceeded. */
 	if (len > maxlen) {
 		uvc_trace(UVC_TRACE_FRAME, "Frame complete (overflow).\n");
+		buf->error = 1;
 		buf->state = UVC_BUF_STATE_READY;
 	}
 }
@@ -1134,6 +1119,84 @@ static int uvc_video_encode_data(struct uvc_streaming *stream,
 	return nbytes;
 }
 
+/* ------------------------------------------------------------------------
+ * Metadata
+ */
+
+/*
+ * Additionally to the payload headers we also want to provide the user with USB
+ * Frame Numbers and system time values. The resulting buffer is thus composed
+ * of blocks, containing a 64-bit timestamp in  nanoseconds, a 16-bit USB Frame
+ * Number, and a copy of the payload header.
+ *
+ * Ideally we want to capture all payload headers for each frame. However, their
+ * number is unknown and unbound. We thus drop headers that contain no vendor
+ * data and that either contain no SCR value or an SCR value identical to the
+ * previous header.
+ */
+static void uvc_video_decode_meta(struct uvc_streaming *stream,
+				  struct uvc_buffer *meta_buf,
+				  const u8 *mem, unsigned int length)
+{
+	struct uvc_meta_buf *meta;
+	size_t len_std = 2;
+	bool has_pts, has_scr;
+	unsigned long flags;
+	unsigned int sof;
+	ktime_t time;
+	const u8 *scr;
+
+	if (!meta_buf || length == 2)
+		return;
+
+	if (meta_buf->length - meta_buf->bytesused <
+	    length + sizeof(meta->ns) + sizeof(meta->sof)) {
+		meta_buf->error = 1;
+		return;
+	}
+
+	has_pts = mem[1] & UVC_STREAM_PTS;
+	has_scr = mem[1] & UVC_STREAM_SCR;
+
+	if (has_pts) {
+		len_std += 4;
+		scr = mem + 6;
+	} else {
+		scr = mem + 2;
+	}
+
+	if (has_scr)
+		len_std += 6;
+
+	if (stream->meta.format == V4L2_META_FMT_UVC)
+		length = len_std;
+
+	if (length == len_std && (!has_scr ||
+				  !memcmp(scr, stream->clock.last_scr, 6)))
+		return;
+
+	meta = (struct uvc_meta_buf *)((u8 *)meta_buf->mem + meta_buf->bytesused);
+	local_irq_save(flags);
+	time = uvc_video_get_time();
+	sof = usb_get_current_frame_number(stream->dev->udev);
+	local_irq_restore(flags);
+	put_unaligned(ktime_to_ns(time), &meta->ns);
+	put_unaligned(sof, &meta->sof);
+
+	if (has_scr)
+		memcpy(stream->clock.last_scr, scr, 6);
+
+	memcpy(&meta->length, mem, length);
+	meta_buf->bytesused += length + sizeof(meta->ns) + sizeof(meta->sof);
+
+	uvc_trace(UVC_TRACE_FRAME,
+		  "%s(): t-sys %lluns, SOF %u, len %u, flags 0x%x, PTS %u, STC %u frame SOF %u\n",
+		  __func__, time, meta->sof, meta->length, meta->flags,
+		  has_pts ? *(u32 *)meta->buf : 0,
+		  has_scr ? *(u32 *)scr : 0,
+		  has_scr ? *(u32 *)(scr + 4) & 0x7ff : 0);
+}
+
 /* ------------------------------------------------------------------------
  * URB handling
  */
@@ -1152,8 +1215,29 @@ static void uvc_video_validate_buffer(const struct uvc_streaming *stream,
 /*
  * Completion handler for video URBs.
  */
+
+static void uvc_video_next_buffers(struct uvc_streaming *stream,
+		struct uvc_buffer **video_buf, struct uvc_buffer **meta_buf)
+{
+	if (*meta_buf) {
+		struct vb2_v4l2_buffer *vb2_meta = &(*meta_buf)->buf;
+		const struct vb2_v4l2_buffer *vb2_video = &(*video_buf)->buf;
+
+		vb2_meta->sequence = vb2_video->sequence;
+		vb2_meta->field = vb2_video->field;
+		vb2_meta->vb2_buf.timestamp = vb2_video->vb2_buf.timestamp;
+
+		(*meta_buf)->state = UVC_BUF_STATE_READY;
+		if (!(*meta_buf)->error)
+			(*meta_buf)->error = (*video_buf)->error;
+		*meta_buf = uvc_queue_next_buffer(&stream->meta.queue,
+						  *meta_buf);
+	}
+	*video_buf = uvc_queue_next_buffer(&stream->queue, *video_buf);
+}
+
 static void uvc_video_decode_isoc(struct urb *urb, struct uvc_streaming *stream,
-	struct uvc_buffer *buf)
+			struct uvc_buffer *buf, struct uvc_buffer *meta_buf)
 {
 	u8 *mem;
 	int ret, i;
@@ -1175,14 +1259,15 @@ static void uvc_video_decode_isoc(struct urb *urb, struct uvc_streaming *stream,
 				urb->iso_frame_desc[i].actual_length);
 			if (ret == -EAGAIN) {
 				uvc_video_validate_buffer(stream, buf);
-				buf = uvc_queue_next_buffer(&stream->queue,
-							    buf);
+				uvc_video_next_buffers(stream, &buf, &meta_buf);
 			}
 		} while (ret == -EAGAIN);
 
 		if (ret < 0)
 			continue;
 
+		uvc_video_decode_meta(stream, meta_buf, mem, ret);
+
 		/* Decode the payload data. */
 		uvc_video_decode_data(stream, buf, mem + ret,
 			urb->iso_frame_desc[i].actual_length - ret);
@@ -1193,13 +1278,13 @@ static void uvc_video_decode_isoc(struct urb *urb, struct uvc_streaming *stream,
 
 		if (buf->state == UVC_BUF_STATE_READY) {
 			uvc_video_validate_buffer(stream, buf);
-			buf = uvc_queue_next_buffer(&stream->queue, buf);
+			uvc_video_next_buffers(stream, &buf, &meta_buf);
 		}
 	}
 }
 
 static void uvc_video_decode_bulk(struct urb *urb, struct uvc_streaming *stream,
-	struct uvc_buffer *buf)
+			struct uvc_buffer *buf, struct uvc_buffer *meta_buf)
 {
 	u8 *mem;
 	int len, ret;
@@ -1222,8 +1307,7 @@ static void uvc_video_decode_bulk(struct urb *urb, struct uvc_streaming *stream,
 		do {
 			ret = uvc_video_decode_start(stream, buf, mem, len);
 			if (ret == -EAGAIN)
-				buf = uvc_queue_next_buffer(&stream->queue,
-							    buf);
+				uvc_video_next_buffers(stream, &buf, &meta_buf);
 		} while (ret == -EAGAIN);
 
 		/* If an error occurred skip the rest of the payload. */
@@ -1233,6 +1317,8 @@ static void uvc_video_decode_bulk(struct urb *urb, struct uvc_streaming *stream,
 			memcpy(stream->bulk.header, mem, ret);
 			stream->bulk.header_size = ret;
 
+			uvc_video_decode_meta(stream, meta_buf, mem, ret);
+
 			mem += ret;
 			len -= ret;
 		}
@@ -1256,7 +1342,7 @@ static void uvc_video_decode_bulk(struct urb *urb, struct uvc_streaming *stream,
 			uvc_video_decode_end(stream, buf, stream->bulk.header,
 				stream->bulk.payload_size);
 			if (buf->state == UVC_BUF_STATE_READY)
-				uvc_queue_next_buffer(&stream->queue, buf);
+				uvc_video_next_buffers(stream, &buf, &meta_buf);
 		}
 
 		stream->bulk.header_size = 0;
@@ -1266,7 +1352,7 @@ static void uvc_video_decode_bulk(struct urb *urb, struct uvc_streaming *stream,
 }
 
 static void uvc_video_encode_bulk(struct urb *urb, struct uvc_streaming *stream,
-	struct uvc_buffer *buf)
+	struct uvc_buffer *buf, struct uvc_buffer *meta_buf)
 {
 	u8 *mem = urb->transfer_buffer;
 	int len = stream->urb_size, ret;
@@ -1312,7 +1398,10 @@ static void uvc_video_complete(struct urb *urb)
 {
 	struct uvc_streaming *stream = urb->context;
 	struct uvc_video_queue *queue = &stream->queue;
+	struct uvc_video_queue *qmeta = &stream->meta.queue;
+	struct vb2_queue *vb2_qmeta = stream->meta.vdev.queue;
 	struct uvc_buffer *buf = NULL;
+	struct uvc_buffer *buf_meta = NULL;
 	unsigned long flags;
 	int ret;
 
@@ -1331,6 +1420,8 @@ static void uvc_video_complete(struct urb *urb)
 	case -ECONNRESET:	/* usb_unlink_urb() called. */
 	case -ESHUTDOWN:	/* The endpoint is being disabled. */
 		uvc_queue_cancel(queue, urb->status == -ESHUTDOWN);
+		if (vb2_qmeta)
+			uvc_queue_cancel(qmeta, urb->status == -ESHUTDOWN);
 		return;
 	}
 
@@ -1340,7 +1431,15 @@ static void uvc_video_complete(struct urb *urb)
 				       queue);
 	spin_unlock_irqrestore(&queue->irqlock, flags);
 
-	stream->decode(urb, stream, buf);
+	if (vb2_qmeta) {
+		spin_lock_irqsave(&qmeta->irqlock, flags);
+		if (!list_empty(&qmeta->irqqueue))
+			buf_meta = list_first_entry(&qmeta->irqqueue,
+						    struct uvc_buffer, queue);
+		spin_unlock_irqrestore(&qmeta->irqlock, flags);
+	}
+
+	stream->decode(urb, stream, buf, buf_meta);
 
 	if ((ret = usb_submit_urb(urb, GFP_ATOMIC)) < 0) {
 		uvc_printk(KERN_ERR, "Failed to resubmit video URB (%d).\n",
@@ -1469,13 +1568,13 @@ static unsigned int uvc_endpoint_max_bpi(struct usb_device *dev,
 	case USB_SPEED_HIGH:
 		psize = usb_endpoint_maxp(&ep->desc);
 		mult = usb_endpoint_maxp_mult(&ep->desc);
-		return (psize & 0x07ff) * mult;
+		return psize * mult;
 	case USB_SPEED_WIRELESS:
 		psize = usb_endpoint_maxp(&ep->desc);
 		return psize;
 	default:
 		psize = usb_endpoint_maxp(&ep->desc);
-		return psize & 0x07ff;
+		return psize;
 	}
 }
 
diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h
index 05398784d1c8..d9e7c70788d0 100644
--- a/drivers/media/usb/uvc/uvcvideo.h
+++ b/drivers/media/usb/uvc/uvcvideo.h
@@ -154,6 +154,11 @@
 	{ 'I',  'N',  'V',  'I', 0xdb, 0x57, 0x49, 0x5e, \
 	 0x8e, 0x3f, 0xf4, 0x79, 0x53, 0x2b, 0x94, 0x6f}
 
+#define UVC_GUID_FORMAT_D3DFMT_L8 \
+	{0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, \
+	 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}
+
+
 /* ------------------------------------------------------------------------
  * Driver specific constants.
  */
@@ -452,8 +457,8 @@ struct uvc_stats_frame {
 };
 
 struct uvc_stats_stream {
-	struct timespec start_ts;	/* Stream start timestamp */
-	struct timespec stop_ts;	/* Stream stop timestamp */
+	ktime_t start_ts;		/* Stream start timestamp */
+	ktime_t stop_ts;		/* Stream stop timestamp */
 
 	unsigned int nb_frames;		/* Number of frames */
 
@@ -474,6 +479,8 @@ struct uvc_stats_stream {
 	unsigned int max_sof;		/* Maximum STC.SOF value */
 };
 
+#define UVC_METATADA_BUF_SIZE 1024
+
 struct uvc_streaming {
 	struct list_head list;
 	struct uvc_device *dev;
@@ -505,7 +512,13 @@ struct uvc_streaming {
 	unsigned int frozen : 1;
 	struct uvc_video_queue queue;
 	void (*decode) (struct urb *urb, struct uvc_streaming *video,
-			struct uvc_buffer *buf);
+			struct uvc_buffer *buf, struct uvc_buffer *meta_buf);
+
+	struct {
+		struct video_device vdev;
+		struct uvc_video_queue queue;
+		__u32 format;
+	} meta;
 
 	/* Context data used by the bulk completion handler. */
 	struct {
@@ -536,8 +549,8 @@ struct uvc_streaming {
 		struct uvc_clock_sample {
 			u32 dev_stc;
 			u16 dev_sof;
-			struct timespec host_ts;
 			u16 host_sof;
+			ktime_t host_time;
 		} *samples;
 
 		unsigned int head;
@@ -547,6 +560,8 @@ struct uvc_streaming {
 		u16 last_sof;
 		u16 sof_offset;
 
+		u8 last_scr[6];
+
 		spinlock_t lock;
 	} clock;
 };
@@ -556,6 +571,7 @@ struct uvc_device {
 	struct usb_interface *intf;
 	unsigned long warnings;
 	__u32 quirks;
+	__u32 meta_format;
 	int intfnum;
 	char name[32];
 
@@ -678,7 +694,7 @@ extern struct uvc_buffer *uvc_queue_next_buffer(struct uvc_video_queue *queue,
 		struct uvc_buffer *buf);
 extern int uvc_queue_mmap(struct uvc_video_queue *queue,
 		struct vm_area_struct *vma);
-extern unsigned int uvc_queue_poll(struct uvc_video_queue *queue,
+extern __poll_t uvc_queue_poll(struct uvc_video_queue *queue,
 		struct file *file, poll_table *wait);
 #ifndef CONFIG_MMU
 extern unsigned long uvc_queue_get_unmapped_area(struct uvc_video_queue *queue,
@@ -710,6 +726,15 @@ extern int uvc_query_ctrl(struct uvc_device *dev, __u8 query, __u8 unit,
 void uvc_video_clock_update(struct uvc_streaming *stream,
 			    struct vb2_v4l2_buffer *vbuf,
 			    struct uvc_buffer *buf);
+int uvc_meta_register(struct uvc_streaming *stream);
+
+int uvc_register_video_device(struct uvc_device *dev,
+			      struct uvc_streaming *stream,
+			      struct video_device *vdev,
+			      struct uvc_video_queue *queue,
+			      enum v4l2_buf_type type,
+			      const struct v4l2_file_operations *fops,
+			      const struct v4l2_ioctl_ops *ioctl_ops);
 
 /* Status */
 extern int uvc_status_init(struct uvc_device *dev);
@@ -764,7 +789,7 @@ extern struct usb_host_endpoint *uvc_find_endpoint(
 
 /* Quirks support */
 void uvc_video_decode_isight(struct urb *urb, struct uvc_streaming *stream,
-		struct uvc_buffer *buf);
+		struct uvc_buffer *buf, struct uvc_buffer *meta_buf);
 
 /* debugfs and statistics */
 void uvc_debugfs_init(void);
diff --git a/drivers/media/usb/zr364xx/zr364xx.c b/drivers/media/usb/zr364xx/zr364xx.c
index 1d888661fd03..8b7c19943d46 100644
--- a/drivers/media/usb/zr364xx/zr364xx.c
+++ b/drivers/media/usb/zr364xx/zr364xx.c
@@ -1287,12 +1287,12 @@ static int zr364xx_mmap(struct file *file, struct vm_area_struct *vma)
 	return ret;
 }
 
-static unsigned int zr364xx_poll(struct file *file,
+static __poll_t zr364xx_poll(struct file *file,
 			       struct poll_table_struct *wait)
 {
 	struct zr364xx_camera *cam = video_drvdata(file);
 	struct videobuf_queue *q = &cam->vb_vidq;
-	unsigned res = v4l2_ctrl_poll(file, wait);
+	__poll_t res = v4l2_ctrl_poll(file, wait);
 
 	_DBG("%s\n", __func__);
 
diff --git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig
index a35c33686abf..8e37e7c5e0f7 100644
--- a/drivers/media/v4l2-core/Kconfig
+++ b/drivers/media/v4l2-core/Kconfig
@@ -7,6 +7,7 @@ config VIDEO_V4L2
 	tristate
 	depends on (I2C || I2C=n) && VIDEO_DEV
 	select RATIONAL
+	select VIDEOBUF2_V4L2 if VIDEOBUF2_CORE
 	default (I2C || I2C=n) && VIDEO_DEV
 
 config VIDEO_ADV_DEBUG
@@ -41,8 +42,8 @@ config VIDEO_TUNER
 
 # Used by drivers that need v4l2-mem2mem.ko
 config V4L2_MEM2MEM_DEV
-        tristate
-        depends on VIDEOBUF2_CORE
+	tristate
+	depends on VIDEOBUF2_CORE
 
 # Used by LED subsystem flash drivers
 config V4L2_FLASH_LED_CLASS
@@ -79,35 +80,3 @@ config VIDEOBUF_DMA_CONTIG
 config VIDEOBUF_DVB
 	tristate
 	select VIDEOBUF_GEN
-
-# Used by drivers that need Videobuf2 modules
-config VIDEOBUF2_CORE
-	select DMA_SHARED_BUFFER
-	tristate
-
-config VIDEOBUF2_MEMOPS
-	tristate
-	select FRAME_VECTOR
-
-config VIDEOBUF2_DMA_CONTIG
-	tristate
-	depends on HAS_DMA
-	select VIDEOBUF2_CORE
-	select VIDEOBUF2_MEMOPS
-	select DMA_SHARED_BUFFER
-
-config VIDEOBUF2_VMALLOC
-	tristate
-	select VIDEOBUF2_CORE
-	select VIDEOBUF2_MEMOPS
-	select DMA_SHARED_BUFFER
-
-config VIDEOBUF2_DMA_SG
-	tristate
-	depends on HAS_DMA
-	select VIDEOBUF2_CORE
-	select VIDEOBUF2_MEMOPS
-
-config VIDEOBUF2_DVB
-	tristate
-	select VIDEOBUF2_CORE
diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile
index 77303286aef7..7df54582e956 100644
--- a/drivers/media/v4l2-core/Makefile
+++ b/drivers/media/v4l2-core/Makefile
@@ -13,7 +13,7 @@ ifeq ($(CONFIG_COMPAT),y)
 endif
 obj-$(CONFIG_V4L2_FWNODE) += v4l2-fwnode.o
 ifeq ($(CONFIG_TRACEPOINTS),y)
-  videodev-objs += vb2-trace.o v4l2-trace.o
+  videodev-objs += v4l2-trace.o
 endif
 videodev-$(CONFIG_MEDIA_CONTROLLER) += v4l2-mc.o
 
@@ -33,14 +33,5 @@ obj-$(CONFIG_VIDEOBUF_DMA_CONTIG) += videobuf-dma-contig.o
 obj-$(CONFIG_VIDEOBUF_VMALLOC) += videobuf-vmalloc.o
 obj-$(CONFIG_VIDEOBUF_DVB) += videobuf-dvb.o
 
-obj-$(CONFIG_VIDEOBUF2_CORE) += videobuf2-core.o videobuf2-v4l2.o
-obj-$(CONFIG_VIDEOBUF2_MEMOPS) += videobuf2-memops.o
-obj-$(CONFIG_VIDEOBUF2_VMALLOC) += videobuf2-vmalloc.o
-obj-$(CONFIG_VIDEOBUF2_DMA_CONTIG) += videobuf2-dma-contig.o
-obj-$(CONFIG_VIDEOBUF2_DMA_SG) += videobuf2-dma-sg.o
-obj-$(CONFIG_VIDEOBUF2_DVB) += videobuf2-dvb.o
-
-ccflags-y += -I$(srctree)/drivers/media/dvb-core
 ccflags-y += -I$(srctree)/drivers/media/dvb-frontends
 ccflags-y += -I$(srctree)/drivers/media/tuners
-
diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c
index e5acfab470a5..2b08d03b251d 100644
--- a/drivers/media/v4l2-core/v4l2-async.c
+++ b/drivers/media/v4l2-core/v4l2-async.c
@@ -68,12 +68,12 @@ static bool match_i2c(struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
 static bool match_devname(struct v4l2_subdev *sd,
 			  struct v4l2_async_subdev *asd)
 {
-	return !strcmp(asd->match.device_name.name, dev_name(sd->dev));
+	return !strcmp(asd->match.device_name, dev_name(sd->dev));
 }
 
 static bool match_fwnode(struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
 {
-	return sd->fwnode == asd->match.fwnode.fwnode;
+	return sd->fwnode == asd->match.fwnode;
 }
 
 static bool match_custom(struct v4l2_subdev *sd, struct v4l2_async_subdev *asd)
@@ -319,7 +319,7 @@ static bool __v4l2_async_notifier_fwnode_has_async_subdev(
 		if (asd->match_type != V4L2_ASYNC_MATCH_FWNODE)
 			continue;
 
-		if (asd->match.fwnode.fwnode == fwnode)
+		if (asd->match.fwnode == fwnode)
 			return true;
 	}
 
@@ -330,7 +330,7 @@ static bool __v4l2_async_notifier_fwnode_has_async_subdev(
 		if (sd->asd->match_type != V4L2_ASYNC_MATCH_FWNODE)
 			continue;
 
-		if (sd->asd->match.fwnode.fwnode == fwnode)
+		if (sd->asd->match.fwnode == fwnode)
 			return true;
 	}
 
@@ -355,8 +355,8 @@ static bool v4l2_async_notifier_fwnode_has_async_subdev(
 		struct v4l2_async_subdev *other_asd = notifier->subdevs[j];
 
 		if (other_asd->match_type == V4L2_ASYNC_MATCH_FWNODE &&
-		    asd->match.fwnode.fwnode ==
-		    other_asd->match.fwnode.fwnode)
+		    asd->match.fwnode ==
+		    other_asd->match.fwnode)
 			return true;
 	}
 
@@ -395,7 +395,7 @@ static int __v4l2_async_notifier_register(struct v4l2_async_notifier *notifier)
 			break;
 		case V4L2_ASYNC_MATCH_FWNODE:
 			if (v4l2_async_notifier_fwnode_has_async_subdev(
-				    notifier, asd->match.fwnode.fwnode, i)) {
+				    notifier, asd->match.fwnode, i)) {
 				dev_err(dev,
 					"fwnode has already been registered or in notifier's subdev list\n");
 				ret = -EEXIST;
@@ -510,7 +510,7 @@ void v4l2_async_notifier_cleanup(struct v4l2_async_notifier *notifier)
 
 		switch (asd->match_type) {
 		case V4L2_ASYNC_MATCH_FWNODE:
-			fwnode_handle_put(asd->match.fwnode.fwnode);
+			fwnode_handle_put(asd->match.fwnode);
 			break;
 		default:
 			WARN_ON_ONCE(true);
diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c
index a5ea1f517291..8650ad92b64d 100644
--- a/drivers/media/v4l2-core/v4l2-common.c
+++ b/drivers/media/v4l2-core/v4l2-common.c
@@ -320,20 +320,6 @@ static unsigned int clamp_align(unsigned int x, unsigned int min,
 	return x;
 }
 
-/* Bound an image to have a width between wmin and wmax, and height between
- * hmin and hmax, inclusive.  Additionally, the width will be a multiple of
- * 2^walign, the height will be a multiple of 2^halign, and the overall size
- * (width*height) will be a multiple of 2^salign.  The image may be shrunk
- * or enlarged to fit the alignment constraints.
- *
- * The width or height maximum must not be smaller than the corresponding
- * minimum.  The alignments must not be so high there are no possible image
- * sizes within the allowed bounds.  wmin and hmin must be at least 1
- * (don't use 0).  If you don't care about a certain alignment, specify 0,
- * as 2^0 is 1 and one byte alignment is equivalent to no alignment.  If
- * you only want to adjust downward, specify a maximum that's the same as
- * the initial value.
- */
 void v4l_bound_align_image(u32 *w, unsigned int wmin, unsigned int wmax,
 			   unsigned int walign,
 			   u32 *h, unsigned int hmin, unsigned int hmax,
@@ -371,18 +357,19 @@ void v4l_bound_align_image(u32 *w, unsigned int wmin, unsigned int wmax,
 }
 EXPORT_SYMBOL_GPL(v4l_bound_align_image);
 
-const struct v4l2_frmsize_discrete *v4l2_find_nearest_format(
-		const struct v4l2_discrete_probe *probe,
-		s32 width, s32 height)
+const struct v4l2_frmsize_discrete *
+v4l2_find_nearest_format(const struct v4l2_frmsize_discrete *sizes,
+			  size_t num_sizes,
+			  s32 width, s32 height)
 {
 	int i;
 	u32 error, min_error = UINT_MAX;
 	const struct v4l2_frmsize_discrete *size, *best = NULL;
 
-	if (!probe)
-		return best;
+	if (!sizes)
+		return NULL;
 
-	for (i = 0, size = probe->sizes; i < probe->num_sizes; i++, size++) {
+	for (i = 0, size = sizes; i < num_sizes; i++, size++) {
 		error = abs(size->width - width) + abs(size->height - height);
 		if (error < min_error) {
 			min_error = error;
diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
index 821f2aa299ae..5198c9eeb348 100644
--- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
+++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
@@ -18,8 +18,18 @@
 #include <linux/videodev2.h>
 #include <linux/v4l2-subdev.h>
 #include <media/v4l2-dev.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-ctrls.h>
 #include <media/v4l2-ioctl.h>
 
+/* Use the same argument order as copy_in_user */
+#define assign_in_user(to, from)					\
+({									\
+	typeof(*from) __assign_tmp;					\
+									\
+	get_user(__assign_tmp, from) || put_user(__assign_tmp, to);	\
+})
+
 static long native_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	long ret = -ENOIOCTLCMD;
@@ -33,12 +43,12 @@ static long native_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 
 struct v4l2_clip32 {
 	struct v4l2_rect        c;
-	compat_caddr_t 		next;
+	compat_caddr_t		next;
 };
 
 struct v4l2_window32 {
 	struct v4l2_rect        w;
-	__u32		  	field;	/* enum v4l2_field */
+	__u32			field;	/* enum v4l2_field */
 	__u32			chromakey;
 	compat_caddr_t		clips; /* actually struct v4l2_clip32 * */
 	__u32			clipcount;
@@ -46,135 +56,75 @@ struct v4l2_window32 {
 	__u8                    global_alpha;
 };
 
-static int get_v4l2_window32(struct v4l2_window *kp, struct v4l2_window32 __user *up)
-{
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_window32)) ||
-		copy_from_user(&kp->w, &up->w, sizeof(up->w)) ||
-		get_user(kp->field, &up->field) ||
-		get_user(kp->chromakey, &up->chromakey) ||
-		get_user(kp->clipcount, &up->clipcount) ||
-		get_user(kp->global_alpha, &up->global_alpha))
-			return -EFAULT;
-	if (kp->clipcount > 2048)
-		return -EINVAL;
-	if (kp->clipcount) {
-		struct v4l2_clip32 __user *uclips;
-		struct v4l2_clip __user *kclips;
-		int n = kp->clipcount;
-		compat_caddr_t p;
-
-		if (get_user(p, &up->clips))
-			return -EFAULT;
-		uclips = compat_ptr(p);
-		kclips = compat_alloc_user_space(n * sizeof(struct v4l2_clip));
-		kp->clips = kclips;
-		while (--n >= 0) {
-			if (copy_in_user(&kclips->c, &uclips->c, sizeof(uclips->c)))
-				return -EFAULT;
-			if (put_user(n ? kclips + 1 : NULL, &kclips->next))
-				return -EFAULT;
-			uclips += 1;
-			kclips += 1;
-		}
-	} else
-		kp->clips = NULL;
-	return 0;
-}
-
-static int put_v4l2_window32(struct v4l2_window *kp, struct v4l2_window32 __user *up)
-{
-	if (copy_to_user(&up->w, &kp->w, sizeof(kp->w)) ||
-		put_user(kp->field, &up->field) ||
-		put_user(kp->chromakey, &up->chromakey) ||
-		put_user(kp->clipcount, &up->clipcount) ||
-		put_user(kp->global_alpha, &up->global_alpha))
-			return -EFAULT;
-	return 0;
-}
-
-static inline int get_v4l2_pix_format(struct v4l2_pix_format *kp, struct v4l2_pix_format __user *up)
-{
-	if (copy_from_user(kp, up, sizeof(struct v4l2_pix_format)))
-		return -EFAULT;
-	return 0;
-}
-
-static inline int get_v4l2_pix_format_mplane(struct v4l2_pix_format_mplane *kp,
-				struct v4l2_pix_format_mplane __user *up)
-{
-	if (copy_from_user(kp, up, sizeof(struct v4l2_pix_format_mplane)))
-		return -EFAULT;
-	return 0;
-}
-
-static inline int put_v4l2_pix_format(struct v4l2_pix_format *kp, struct v4l2_pix_format __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_pix_format)))
-		return -EFAULT;
-	return 0;
-}
-
-static inline int put_v4l2_pix_format_mplane(struct v4l2_pix_format_mplane *kp,
-				struct v4l2_pix_format_mplane __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_pix_format_mplane)))
-		return -EFAULT;
-	return 0;
-}
-
-static inline int get_v4l2_vbi_format(struct v4l2_vbi_format *kp, struct v4l2_vbi_format __user *up)
+static int get_v4l2_window32(struct v4l2_window __user *kp,
+			     struct v4l2_window32 __user *up,
+			     void __user *aux_buf, u32 aux_space)
 {
-	if (copy_from_user(kp, up, sizeof(struct v4l2_vbi_format)))
+	struct v4l2_clip32 __user *uclips;
+	struct v4l2_clip __user *kclips;
+	compat_caddr_t p;
+	u32 clipcount;
+
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    copy_in_user(&kp->w, &up->w, sizeof(up->w)) ||
+	    assign_in_user(&kp->field, &up->field) ||
+	    assign_in_user(&kp->chromakey, &up->chromakey) ||
+	    assign_in_user(&kp->global_alpha, &up->global_alpha) ||
+	    get_user(clipcount, &up->clipcount) ||
+	    put_user(clipcount, &kp->clipcount))
 		return -EFAULT;
-	return 0;
-}
+	if (clipcount > 2048)
+		return -EINVAL;
+	if (!clipcount)
+		return put_user(NULL, &kp->clips);
 
-static inline int put_v4l2_vbi_format(struct v4l2_vbi_format *kp, struct v4l2_vbi_format __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_vbi_format)))
+	if (get_user(p, &up->clips))
 		return -EFAULT;
-	return 0;
-}
-
-static inline int get_v4l2_sliced_vbi_format(struct v4l2_sliced_vbi_format *kp, struct v4l2_sliced_vbi_format __user *up)
-{
-	if (copy_from_user(kp, up, sizeof(struct v4l2_sliced_vbi_format)))
+	uclips = compat_ptr(p);
+	if (aux_space < clipcount * sizeof(*kclips))
 		return -EFAULT;
-	return 0;
-}
-
-static inline int put_v4l2_sliced_vbi_format(struct v4l2_sliced_vbi_format *kp, struct v4l2_sliced_vbi_format __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_sliced_vbi_format)))
+	kclips = aux_buf;
+	if (put_user(kclips, &kp->clips))
 		return -EFAULT;
-	return 0;
-}
 
-static inline int get_v4l2_sdr_format(struct v4l2_sdr_format *kp, struct v4l2_sdr_format __user *up)
-{
-	if (copy_from_user(kp, up, sizeof(struct v4l2_sdr_format)))
-		return -EFAULT;
-	return 0;
-}
-
-static inline int put_v4l2_sdr_format(struct v4l2_sdr_format *kp, struct v4l2_sdr_format __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_sdr_format)))
-		return -EFAULT;
+	while (clipcount--) {
+		if (copy_in_user(&kclips->c, &uclips->c, sizeof(uclips->c)))
+			return -EFAULT;
+		if (put_user(clipcount ? kclips + 1 : NULL, &kclips->next))
+			return -EFAULT;
+		uclips++;
+		kclips++;
+	}
 	return 0;
 }
 
-static inline int get_v4l2_meta_format(struct v4l2_meta_format *kp, struct v4l2_meta_format __user *up)
+static int put_v4l2_window32(struct v4l2_window __user *kp,
+			     struct v4l2_window32 __user *up)
 {
-	if (copy_from_user(kp, up, sizeof(struct v4l2_meta_format)))
+	struct v4l2_clip __user *kclips = kp->clips;
+	struct v4l2_clip32 __user *uclips;
+	compat_caddr_t p;
+	u32 clipcount;
+
+	if (copy_in_user(&up->w, &kp->w, sizeof(kp->w)) ||
+	    assign_in_user(&up->field, &kp->field) ||
+	    assign_in_user(&up->chromakey, &kp->chromakey) ||
+	    assign_in_user(&up->global_alpha, &kp->global_alpha) ||
+	    get_user(clipcount, &kp->clipcount) ||
+	    put_user(clipcount, &up->clipcount))
 		return -EFAULT;
-	return 0;
-}
+	if (!clipcount)
+		return 0;
 
-static inline int put_v4l2_meta_format(struct v4l2_meta_format *kp, struct v4l2_meta_format __user *up)
-{
-	if (copy_to_user(up, kp, sizeof(struct v4l2_meta_format)))
+	if (get_user(p, &up->clips))
 		return -EFAULT;
+	uclips = compat_ptr(p);
+	while (clipcount--) {
+		if (copy_in_user(&uclips->c, &kclips->c, sizeof(uclips->c)))
+			return -EFAULT;
+		uclips++;
+		kclips++;
+	}
 	return 0;
 }
 
@@ -209,101 +159,164 @@ struct v4l2_create_buffers32 {
 	__u32			reserved[8];
 };
 
-static int __get_v4l2_format32(struct v4l2_format *kp, struct v4l2_format32 __user *up)
+static int __bufsize_v4l2_format(struct v4l2_format32 __user *up, u32 *size)
 {
-	if (get_user(kp->type, &up->type))
+	u32 type;
+
+	if (get_user(type, &up->type))
 		return -EFAULT;
 
-	switch (kp->type) {
+	switch (type) {
+	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
+	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: {
+		u32 clipcount;
+
+		if (get_user(clipcount, &up->fmt.win.clipcount))
+			return -EFAULT;
+		if (clipcount > 2048)
+			return -EINVAL;
+		*size = clipcount * sizeof(struct v4l2_clip);
+		return 0;
+	}
+	default:
+		*size = 0;
+		return 0;
+	}
+}
+
+static int bufsize_v4l2_format(struct v4l2_format32 __user *up, u32 *size)
+{
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)))
+		return -EFAULT;
+	return __bufsize_v4l2_format(up, size);
+}
+
+static int __get_v4l2_format32(struct v4l2_format __user *kp,
+			       struct v4l2_format32 __user *up,
+			       void __user *aux_buf, u32 aux_space)
+{
+	u32 type;
+
+	if (get_user(type, &up->type) || put_user(type, &kp->type))
+		return -EFAULT;
+
+	switch (type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		return get_v4l2_pix_format(&kp->fmt.pix, &up->fmt.pix);
+		return copy_in_user(&kp->fmt.pix, &up->fmt.pix,
+				    sizeof(kp->fmt.pix)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		return get_v4l2_pix_format_mplane(&kp->fmt.pix_mp,
-						  &up->fmt.pix_mp);
+		return copy_in_user(&kp->fmt.pix_mp, &up->fmt.pix_mp,
+				    sizeof(kp->fmt.pix_mp)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
-		return get_v4l2_window32(&kp->fmt.win, &up->fmt.win);
+		return get_v4l2_window32(&kp->fmt.win, &up->fmt.win,
+					 aux_buf, aux_space);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
-		return get_v4l2_vbi_format(&kp->fmt.vbi, &up->fmt.vbi);
+		return copy_in_user(&kp->fmt.vbi, &up->fmt.vbi,
+				    sizeof(kp->fmt.vbi)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
-		return get_v4l2_sliced_vbi_format(&kp->fmt.sliced, &up->fmt.sliced);
+		return copy_in_user(&kp->fmt.sliced, &up->fmt.sliced,
+				    sizeof(kp->fmt.sliced)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		return get_v4l2_sdr_format(&kp->fmt.sdr, &up->fmt.sdr);
+		return copy_in_user(&kp->fmt.sdr, &up->fmt.sdr,
+				    sizeof(kp->fmt.sdr)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		return get_v4l2_meta_format(&kp->fmt.meta, &up->fmt.meta);
+		return copy_in_user(&kp->fmt.meta, &up->fmt.meta,
+				    sizeof(kp->fmt.meta)) ? -EFAULT : 0;
 	default:
-		pr_info("compat_ioctl32: unexpected VIDIOC_FMT type %d\n",
-								kp->type);
 		return -EINVAL;
 	}
 }
 
-static int get_v4l2_format32(struct v4l2_format *kp, struct v4l2_format32 __user *up)
+static int get_v4l2_format32(struct v4l2_format __user *kp,
+			     struct v4l2_format32 __user *up,
+			     void __user *aux_buf, u32 aux_space)
+{
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)))
+		return -EFAULT;
+	return __get_v4l2_format32(kp, up, aux_buf, aux_space);
+}
+
+static int bufsize_v4l2_create(struct v4l2_create_buffers32 __user *up,
+			       u32 *size)
 {
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_format32)))
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)))
 		return -EFAULT;
-	return __get_v4l2_format32(kp, up);
+	return __bufsize_v4l2_format(&up->format, size);
 }
 
-static int get_v4l2_create32(struct v4l2_create_buffers *kp, struct v4l2_create_buffers32 __user *up)
+static int get_v4l2_create32(struct v4l2_create_buffers __user *kp,
+			     struct v4l2_create_buffers32 __user *up,
+			     void __user *aux_buf, u32 aux_space)
 {
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_create_buffers32)) ||
-	    copy_from_user(kp, up, offsetof(struct v4l2_create_buffers32, format)))
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    copy_in_user(kp, up,
+			 offsetof(struct v4l2_create_buffers32, format)))
 		return -EFAULT;
-	return __get_v4l2_format32(&kp->format, &up->format);
+	return __get_v4l2_format32(&kp->format, &up->format,
+				   aux_buf, aux_space);
 }
 
-static int __put_v4l2_format32(struct v4l2_format *kp, struct v4l2_format32 __user *up)
+static int __put_v4l2_format32(struct v4l2_format __user *kp,
+			       struct v4l2_format32 __user *up)
 {
-	if (put_user(kp->type, &up->type))
+	u32 type;
+
+	if (get_user(type, &kp->type))
 		return -EFAULT;
 
-	switch (kp->type) {
+	switch (type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		return put_v4l2_pix_format(&kp->fmt.pix, &up->fmt.pix);
+		return copy_in_user(&up->fmt.pix, &kp->fmt.pix,
+				    sizeof(kp->fmt.pix)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		return put_v4l2_pix_format_mplane(&kp->fmt.pix_mp,
-						  &up->fmt.pix_mp);
+		return copy_in_user(&up->fmt.pix_mp, &kp->fmt.pix_mp,
+				    sizeof(kp->fmt.pix_mp)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
 		return put_v4l2_window32(&kp->fmt.win, &up->fmt.win);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
-		return put_v4l2_vbi_format(&kp->fmt.vbi, &up->fmt.vbi);
+		return copy_in_user(&up->fmt.vbi, &kp->fmt.vbi,
+				    sizeof(kp->fmt.vbi)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE:
 	case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
-		return put_v4l2_sliced_vbi_format(&kp->fmt.sliced, &up->fmt.sliced);
+		return copy_in_user(&up->fmt.sliced, &kp->fmt.sliced,
+				    sizeof(kp->fmt.sliced)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		return put_v4l2_sdr_format(&kp->fmt.sdr, &up->fmt.sdr);
+		return copy_in_user(&up->fmt.sdr, &kp->fmt.sdr,
+				    sizeof(kp->fmt.sdr)) ? -EFAULT : 0;
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		return put_v4l2_meta_format(&kp->fmt.meta, &up->fmt.meta);
+		return copy_in_user(&up->fmt.meta, &kp->fmt.meta,
+				    sizeof(kp->fmt.meta)) ? -EFAULT : 0;
 	default:
-		pr_info("compat_ioctl32: unexpected VIDIOC_FMT type %d\n",
-								kp->type);
 		return -EINVAL;
 	}
 }
 
-static int put_v4l2_format32(struct v4l2_format *kp, struct v4l2_format32 __user *up)
+static int put_v4l2_format32(struct v4l2_format __user *kp,
+			     struct v4l2_format32 __user *up)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_format32)))
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)))
 		return -EFAULT;
 	return __put_v4l2_format32(kp, up);
 }
 
-static int put_v4l2_create32(struct v4l2_create_buffers *kp, struct v4l2_create_buffers32 __user *up)
+static int put_v4l2_create32(struct v4l2_create_buffers __user *kp,
+			     struct v4l2_create_buffers32 __user *up)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_create_buffers32)) ||
-	    copy_to_user(up, kp, offsetof(struct v4l2_create_buffers32, format)) ||
-	    copy_to_user(up->reserved, kp->reserved, sizeof(kp->reserved)))
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
+	    copy_in_user(up, kp,
+			 offsetof(struct v4l2_create_buffers32, format)) ||
+	    copy_in_user(up->reserved, kp->reserved, sizeof(kp->reserved)))
 		return -EFAULT;
 	return __put_v4l2_format32(&kp->format, &up->format);
 }
@@ -317,25 +330,28 @@ struct v4l2_standard32 {
 	__u32		     reserved[4];
 };
 
-static int get_v4l2_standard32(struct v4l2_standard *kp, struct v4l2_standard32 __user *up)
+static int get_v4l2_standard32(struct v4l2_standard __user *kp,
+			       struct v4l2_standard32 __user *up)
 {
 	/* other fields are not set by the user, nor used by the driver */
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_standard32)) ||
-		get_user(kp->index, &up->index))
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    assign_in_user(&kp->index, &up->index))
 		return -EFAULT;
 	return 0;
 }
 
-static int put_v4l2_standard32(struct v4l2_standard *kp, struct v4l2_standard32 __user *up)
+static int put_v4l2_standard32(struct v4l2_standard __user *kp,
+			       struct v4l2_standard32 __user *up)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_standard32)) ||
-		put_user(kp->index, &up->index) ||
-		put_user(kp->id, &up->id) ||
-		copy_to_user(up->name, kp->name, 24) ||
-		copy_to_user(&up->frameperiod, &kp->frameperiod, sizeof(kp->frameperiod)) ||
-		put_user(kp->framelines, &up->framelines) ||
-		copy_to_user(up->reserved, kp->reserved, 4 * sizeof(__u32)))
-			return -EFAULT;
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
+	    assign_in_user(&up->index, &kp->index) ||
+	    assign_in_user(&up->id, &kp->id) ||
+	    copy_in_user(up->name, kp->name, sizeof(up->name)) ||
+	    copy_in_user(&up->frameperiod, &kp->frameperiod,
+			 sizeof(up->frameperiod)) ||
+	    assign_in_user(&up->framelines, &kp->framelines) ||
+	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)))
+		return -EFAULT;
 	return 0;
 }
 
@@ -374,136 +390,186 @@ struct v4l2_buffer32 {
 	__u32			reserved;
 };
 
-static int get_v4l2_plane32(struct v4l2_plane __user *up, struct v4l2_plane32 __user *up32,
-				enum v4l2_memory memory)
+static int get_v4l2_plane32(struct v4l2_plane __user *up,
+			    struct v4l2_plane32 __user *up32,
+			    enum v4l2_memory memory)
 {
-	void __user *up_pln;
-	compat_long_t p;
+	compat_ulong_t p;
 
 	if (copy_in_user(up, up32, 2 * sizeof(__u32)) ||
-		copy_in_user(&up->data_offset, &up32->data_offset,
-				sizeof(__u32)))
+	    copy_in_user(&up->data_offset, &up32->data_offset,
+			 sizeof(up->data_offset)))
 		return -EFAULT;
 
-	if (memory == V4L2_MEMORY_USERPTR) {
-		if (get_user(p, &up32->m.userptr))
-			return -EFAULT;
-		up_pln = compat_ptr(p);
-		if (put_user((unsigned long)up_pln, &up->m.userptr))
+	switch (memory) {
+	case V4L2_MEMORY_MMAP:
+	case V4L2_MEMORY_OVERLAY:
+		if (copy_in_user(&up->m.mem_offset, &up32->m.mem_offset,
+				 sizeof(up32->m.mem_offset)))
 			return -EFAULT;
-	} else if (memory == V4L2_MEMORY_DMABUF) {
-		if (copy_in_user(&up->m.fd, &up32->m.fd, sizeof(int)))
+		break;
+	case V4L2_MEMORY_USERPTR:
+		if (get_user(p, &up32->m.userptr) ||
+		    put_user((unsigned long)compat_ptr(p), &up->m.userptr))
 			return -EFAULT;
-	} else {
-		if (copy_in_user(&up->m.mem_offset, &up32->m.mem_offset,
-					sizeof(__u32)))
+		break;
+	case V4L2_MEMORY_DMABUF:
+		if (copy_in_user(&up->m.fd, &up32->m.fd, sizeof(up32->m.fd)))
 			return -EFAULT;
+		break;
 	}
 
 	return 0;
 }
 
-static int put_v4l2_plane32(struct v4l2_plane __user *up, struct v4l2_plane32 __user *up32,
-				enum v4l2_memory memory)
+static int put_v4l2_plane32(struct v4l2_plane __user *up,
+			    struct v4l2_plane32 __user *up32,
+			    enum v4l2_memory memory)
 {
+	unsigned long p;
+
 	if (copy_in_user(up32, up, 2 * sizeof(__u32)) ||
-		copy_in_user(&up32->data_offset, &up->data_offset,
-				sizeof(__u32)))
+	    copy_in_user(&up32->data_offset, &up->data_offset,
+			 sizeof(up->data_offset)))
 		return -EFAULT;
 
-	/* For MMAP, driver might've set up the offset, so copy it back.
-	 * USERPTR stays the same (was userspace-provided), so no copying. */
-	if (memory == V4L2_MEMORY_MMAP)
+	switch (memory) {
+	case V4L2_MEMORY_MMAP:
+	case V4L2_MEMORY_OVERLAY:
 		if (copy_in_user(&up32->m.mem_offset, &up->m.mem_offset,
-					sizeof(__u32)))
+				 sizeof(up->m.mem_offset)))
 			return -EFAULT;
-	/* For DMABUF, driver might've set up the fd, so copy it back. */
-	if (memory == V4L2_MEMORY_DMABUF)
-		if (copy_in_user(&up32->m.fd, &up->m.fd,
-					sizeof(int)))
+		break;
+	case V4L2_MEMORY_USERPTR:
+		if (get_user(p, &up->m.userptr) ||
+		    put_user((compat_ulong_t)ptr_to_compat((__force void *)p),
+			     &up32->m.userptr))
 			return -EFAULT;
+		break;
+	case V4L2_MEMORY_DMABUF:
+		if (copy_in_user(&up32->m.fd, &up->m.fd, sizeof(up->m.fd)))
+			return -EFAULT;
+		break;
+	}
+
+	return 0;
+}
+
+static int bufsize_v4l2_buffer(struct v4l2_buffer32 __user *up, u32 *size)
+{
+	u32 type;
+	u32 length;
+
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    get_user(type, &up->type) ||
+	    get_user(length, &up->length))
+		return -EFAULT;
 
+	if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
+		if (length > VIDEO_MAX_PLANES)
+			return -EINVAL;
+
+		/*
+		 * We don't really care if userspace decides to kill itself
+		 * by passing a very big length value
+		 */
+		*size = length * sizeof(struct v4l2_plane);
+	} else {
+		*size = 0;
+	}
 	return 0;
 }
 
-static int get_v4l2_buffer32(struct v4l2_buffer *kp, struct v4l2_buffer32 __user *up)
+static int get_v4l2_buffer32(struct v4l2_buffer __user *kp,
+			     struct v4l2_buffer32 __user *up,
+			     void __user *aux_buf, u32 aux_space)
 {
+	u32 type;
+	u32 length;
+	enum v4l2_memory memory;
 	struct v4l2_plane32 __user *uplane32;
 	struct v4l2_plane __user *uplane;
 	compat_caddr_t p;
 	int ret;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_buffer32)) ||
-		get_user(kp->index, &up->index) ||
-		get_user(kp->type, &up->type) ||
-		get_user(kp->flags, &up->flags) ||
-		get_user(kp->memory, &up->memory) ||
-		get_user(kp->length, &up->length))
-			return -EFAULT;
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    assign_in_user(&kp->index, &up->index) ||
+	    get_user(type, &up->type) ||
+	    put_user(type, &kp->type) ||
+	    assign_in_user(&kp->flags, &up->flags) ||
+	    get_user(memory, &up->memory) ||
+	    put_user(memory, &kp->memory) ||
+	    get_user(length, &up->length) ||
+	    put_user(length, &kp->length))
+		return -EFAULT;
 
-	if (V4L2_TYPE_IS_OUTPUT(kp->type))
-		if (get_user(kp->bytesused, &up->bytesused) ||
-			get_user(kp->field, &up->field) ||
-			get_user(kp->timestamp.tv_sec, &up->timestamp.tv_sec) ||
-			get_user(kp->timestamp.tv_usec,
-					&up->timestamp.tv_usec))
+	if (V4L2_TYPE_IS_OUTPUT(type))
+		if (assign_in_user(&kp->bytesused, &up->bytesused) ||
+		    assign_in_user(&kp->field, &up->field) ||
+		    assign_in_user(&kp->timestamp.tv_sec,
+				   &up->timestamp.tv_sec) ||
+		    assign_in_user(&kp->timestamp.tv_usec,
+				   &up->timestamp.tv_usec))
 			return -EFAULT;
 
-	if (V4L2_TYPE_IS_MULTIPLANAR(kp->type)) {
-		unsigned int num_planes;
+	if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
+		u32 num_planes = length;
 
-		if (kp->length == 0) {
-			kp->m.planes = NULL;
-			/* num_planes == 0 is legal, e.g. when userspace doesn't
-			 * need planes array on DQBUF*/
-			return 0;
-		} else if (kp->length > VIDEO_MAX_PLANES) {
-			return -EINVAL;
+		if (num_planes == 0) {
+			/*
+			 * num_planes == 0 is legal, e.g. when userspace doesn't
+			 * need planes array on DQBUF
+			 */
+			return put_user(NULL, &kp->m.planes);
 		}
+		if (num_planes > VIDEO_MAX_PLANES)
+			return -EINVAL;
 
 		if (get_user(p, &up->m.planes))
 			return -EFAULT;
 
 		uplane32 = compat_ptr(p);
 		if (!access_ok(VERIFY_READ, uplane32,
-				kp->length * sizeof(struct v4l2_plane32)))
+			       num_planes * sizeof(*uplane32)))
 			return -EFAULT;
 
-		/* We don't really care if userspace decides to kill itself
-		 * by passing a very big num_planes value */
-		uplane = compat_alloc_user_space(kp->length *
-						 sizeof(struct v4l2_plane));
-		kp->m.planes = (__force struct v4l2_plane *)uplane;
+		/*
+		 * We don't really care if userspace decides to kill itself
+		 * by passing a very big num_planes value
+		 */
+		if (aux_space < num_planes * sizeof(*uplane))
+			return -EFAULT;
 
-		for (num_planes = 0; num_planes < kp->length; num_planes++) {
-			ret = get_v4l2_plane32(uplane, uplane32, kp->memory);
+		uplane = aux_buf;
+		if (put_user((__force struct v4l2_plane *)uplane,
+			     &kp->m.planes))
+			return -EFAULT;
+
+		while (num_planes--) {
+			ret = get_v4l2_plane32(uplane, uplane32, memory);
 			if (ret)
 				return ret;
-			++uplane;
-			++uplane32;
+			uplane++;
+			uplane32++;
 		}
 	} else {
-		switch (kp->memory) {
+		switch (memory) {
 		case V4L2_MEMORY_MMAP:
-			if (get_user(kp->m.offset, &up->m.offset))
+		case V4L2_MEMORY_OVERLAY:
+			if (assign_in_user(&kp->m.offset, &up->m.offset))
 				return -EFAULT;
 			break;
-		case V4L2_MEMORY_USERPTR:
-			{
-			compat_long_t tmp;
+		case V4L2_MEMORY_USERPTR: {
+			compat_ulong_t userptr;
 
-			if (get_user(tmp, &up->m.userptr))
-				return -EFAULT;
-
-			kp->m.userptr = (unsigned long)compat_ptr(tmp);
-			}
-			break;
-		case V4L2_MEMORY_OVERLAY:
-			if (get_user(kp->m.offset, &up->m.offset))
+			if (get_user(userptr, &up->m.userptr) ||
+			    put_user((unsigned long)compat_ptr(userptr),
+				     &kp->m.userptr))
 				return -EFAULT;
 			break;
+		}
 		case V4L2_MEMORY_DMABUF:
-			if (get_user(kp->m.fd, &up->m.fd))
+			if (assign_in_user(&kp->m.fd, &up->m.fd))
 				return -EFAULT;
 			break;
 		}
@@ -512,65 +578,70 @@ static int get_v4l2_buffer32(struct v4l2_buffer *kp, struct v4l2_buffer32 __user
 	return 0;
 }
 
-static int put_v4l2_buffer32(struct v4l2_buffer *kp, struct v4l2_buffer32 __user *up)
+static int put_v4l2_buffer32(struct v4l2_buffer __user *kp,
+			     struct v4l2_buffer32 __user *up)
 {
+	u32 type;
+	u32 length;
+	enum v4l2_memory memory;
 	struct v4l2_plane32 __user *uplane32;
 	struct v4l2_plane __user *uplane;
 	compat_caddr_t p;
-	int num_planes;
 	int ret;
 
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_buffer32)) ||
-		put_user(kp->index, &up->index) ||
-		put_user(kp->type, &up->type) ||
-		put_user(kp->flags, &up->flags) ||
-		put_user(kp->memory, &up->memory))
-			return -EFAULT;
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
+	    assign_in_user(&up->index, &kp->index) ||
+	    get_user(type, &kp->type) ||
+	    put_user(type, &up->type) ||
+	    assign_in_user(&up->flags, &kp->flags) ||
+	    get_user(memory, &kp->memory) ||
+	    put_user(memory, &up->memory))
+		return -EFAULT;
 
-	if (put_user(kp->bytesused, &up->bytesused) ||
-		put_user(kp->field, &up->field) ||
-		put_user(kp->timestamp.tv_sec, &up->timestamp.tv_sec) ||
-		put_user(kp->timestamp.tv_usec, &up->timestamp.tv_usec) ||
-		copy_to_user(&up->timecode, &kp->timecode, sizeof(struct v4l2_timecode)) ||
-		put_user(kp->sequence, &up->sequence) ||
-		put_user(kp->reserved2, &up->reserved2) ||
-		put_user(kp->reserved, &up->reserved) ||
-		put_user(kp->length, &up->length))
-			return -EFAULT;
+	if (assign_in_user(&up->bytesused, &kp->bytesused) ||
+	    assign_in_user(&up->field, &kp->field) ||
+	    assign_in_user(&up->timestamp.tv_sec, &kp->timestamp.tv_sec) ||
+	    assign_in_user(&up->timestamp.tv_usec, &kp->timestamp.tv_usec) ||
+	    copy_in_user(&up->timecode, &kp->timecode, sizeof(kp->timecode)) ||
+	    assign_in_user(&up->sequence, &kp->sequence) ||
+	    assign_in_user(&up->reserved2, &kp->reserved2) ||
+	    assign_in_user(&up->reserved, &kp->reserved) ||
+	    get_user(length, &kp->length) ||
+	    put_user(length, &up->length))
+		return -EFAULT;
+
+	if (V4L2_TYPE_IS_MULTIPLANAR(type)) {
+		u32 num_planes = length;
 
-	if (V4L2_TYPE_IS_MULTIPLANAR(kp->type)) {
-		num_planes = kp->length;
 		if (num_planes == 0)
 			return 0;
 
-		uplane = (__force struct v4l2_plane __user *)kp->m.planes;
+		if (get_user(uplane, ((__force struct v4l2_plane __user **)&kp->m.planes)))
+			return -EFAULT;
 		if (get_user(p, &up->m.planes))
 			return -EFAULT;
 		uplane32 = compat_ptr(p);
 
-		while (--num_planes >= 0) {
-			ret = put_v4l2_plane32(uplane, uplane32, kp->memory);
+		while (num_planes--) {
+			ret = put_v4l2_plane32(uplane, uplane32, memory);
 			if (ret)
 				return ret;
 			++uplane;
 			++uplane32;
 		}
 	} else {
-		switch (kp->memory) {
+		switch (memory) {
 		case V4L2_MEMORY_MMAP:
-			if (put_user(kp->m.offset, &up->m.offset))
+		case V4L2_MEMORY_OVERLAY:
+			if (assign_in_user(&up->m.offset, &kp->m.offset))
 				return -EFAULT;
 			break;
 		case V4L2_MEMORY_USERPTR:
-			if (put_user(kp->m.userptr, &up->m.userptr))
-				return -EFAULT;
-			break;
-		case V4L2_MEMORY_OVERLAY:
-			if (put_user(kp->m.offset, &up->m.offset))
+			if (assign_in_user(&up->m.userptr, &kp->m.userptr))
 				return -EFAULT;
 			break;
 		case V4L2_MEMORY_DMABUF:
-			if (put_user(kp->m.fd, &up->m.fd))
+			if (assign_in_user(&up->m.fd, &kp->m.fd))
 				return -EFAULT;
 			break;
 		}
@@ -582,7 +653,7 @@ static int put_v4l2_buffer32(struct v4l2_buffer *kp, struct v4l2_buffer32 __user
 struct v4l2_framebuffer32 {
 	__u32			capability;
 	__u32			flags;
-	compat_caddr_t 		base;
+	compat_caddr_t		base;
 	struct {
 		__u32		width;
 		__u32		height;
@@ -595,30 +666,33 @@ struct v4l2_framebuffer32 {
 	} fmt;
 };
 
-static int get_v4l2_framebuffer32(struct v4l2_framebuffer *kp, struct v4l2_framebuffer32 __user *up)
+static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp,
+				  struct v4l2_framebuffer32 __user *up)
 {
-	u32 tmp;
-
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_framebuffer32)) ||
-		get_user(tmp, &up->base) ||
-		get_user(kp->capability, &up->capability) ||
-		get_user(kp->flags, &up->flags) ||
-		copy_from_user(&kp->fmt, &up->fmt, sizeof(up->fmt)))
-			return -EFAULT;
-	kp->base = (__force void *)compat_ptr(tmp);
+	compat_caddr_t tmp;
+
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    get_user(tmp, &up->base) ||
+	    put_user((__force void *)compat_ptr(tmp), &kp->base) ||
+	    assign_in_user(&kp->capability, &up->capability) ||
+	    assign_in_user(&kp->flags, &up->flags) ||
+	    copy_in_user(&kp->fmt, &up->fmt, sizeof(kp->fmt)))
+		return -EFAULT;
 	return 0;
 }
 
-static int put_v4l2_framebuffer32(struct v4l2_framebuffer *kp, struct v4l2_framebuffer32 __user *up)
+static int put_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp,
+				  struct v4l2_framebuffer32 __user *up)
 {
-	u32 tmp = (u32)((unsigned long)kp->base);
-
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_framebuffer32)) ||
-		put_user(tmp, &up->base) ||
-		put_user(kp->capability, &up->capability) ||
-		put_user(kp->flags, &up->flags) ||
-		copy_to_user(&up->fmt, &kp->fmt, sizeof(up->fmt)))
-			return -EFAULT;
+	void *base;
+
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
+	    get_user(base, &kp->base) ||
+	    put_user(ptr_to_compat(base), &up->base) ||
+	    assign_in_user(&up->capability, &kp->capability) ||
+	    assign_in_user(&up->flags, &kp->flags) ||
+	    copy_in_user(&up->fmt, &kp->fmt, sizeof(kp->fmt)))
+		return -EFAULT;
 	return 0;
 }
 
@@ -634,18 +708,22 @@ struct v4l2_input32 {
 	__u32	     reserved[3];
 };
 
-/* The 64-bit v4l2_input struct has extra padding at the end of the struct.
-   Otherwise it is identical to the 32-bit version. */
-static inline int get_v4l2_input32(struct v4l2_input *kp, struct v4l2_input32 __user *up)
+/*
+ * The 64-bit v4l2_input struct has extra padding at the end of the struct.
+ * Otherwise it is identical to the 32-bit version.
+ */
+static inline int get_v4l2_input32(struct v4l2_input __user *kp,
+				   struct v4l2_input32 __user *up)
 {
-	if (copy_from_user(kp, up, sizeof(struct v4l2_input32)))
+	if (copy_in_user(kp, up, sizeof(*up)))
 		return -EFAULT;
 	return 0;
 }
 
-static inline int put_v4l2_input32(struct v4l2_input *kp, struct v4l2_input32 __user *up)
+static inline int put_v4l2_input32(struct v4l2_input __user *kp,
+				   struct v4l2_input32 __user *up)
 {
-	if (copy_to_user(up, kp, sizeof(struct v4l2_input32)))
+	if (copy_in_user(up, kp, sizeof(*up)))
 		return -EFAULT;
 	return 0;
 }
@@ -669,60 +747,95 @@ struct v4l2_ext_control32 {
 	};
 } __attribute__ ((packed));
 
-/* The following function really belong in v4l2-common, but that causes
-   a circular dependency between modules. We need to think about this, but
-   for now this will do. */
-
-/* Return non-zero if this control is a pointer type. Currently only
-   type STRING is a pointer type. */
-static inline int ctrl_is_pointer(u32 id)
+/* Return true if this control is a pointer type. */
+static inline bool ctrl_is_pointer(struct file *file, u32 id)
 {
-	switch (id) {
-	case V4L2_CID_RDS_TX_PS_NAME:
-	case V4L2_CID_RDS_TX_RADIO_TEXT:
-		return 1;
-	default:
-		return 0;
+	struct video_device *vdev = video_devdata(file);
+	struct v4l2_fh *fh = NULL;
+	struct v4l2_ctrl_handler *hdl = NULL;
+	struct v4l2_query_ext_ctrl qec = { id };
+	const struct v4l2_ioctl_ops *ops = vdev->ioctl_ops;
+
+	if (test_bit(V4L2_FL_USES_V4L2_FH, &vdev->flags))
+		fh = file->private_data;
+
+	if (fh && fh->ctrl_handler)
+		hdl = fh->ctrl_handler;
+	else if (vdev->ctrl_handler)
+		hdl = vdev->ctrl_handler;
+
+	if (hdl) {
+		struct v4l2_ctrl *ctrl = v4l2_ctrl_find(hdl, id);
+
+		return ctrl && ctrl->is_ptr;
 	}
+
+	if (!ops || !ops->vidioc_query_ext_ctrl)
+		return false;
+
+	return !ops->vidioc_query_ext_ctrl(file, fh, &qec) &&
+		(qec.flags & V4L2_CTRL_FLAG_HAS_PAYLOAD);
+}
+
+static int bufsize_v4l2_ext_controls(struct v4l2_ext_controls32 __user *up,
+				     u32 *size)
+{
+	u32 count;
+
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    get_user(count, &up->count))
+		return -EFAULT;
+	if (count > V4L2_CID_MAX_CTRLS)
+		return -EINVAL;
+	*size = count * sizeof(struct v4l2_ext_control);
+	return 0;
 }
 
-static int get_v4l2_ext_controls32(struct v4l2_ext_controls *kp, struct v4l2_ext_controls32 __user *up)
+static int get_v4l2_ext_controls32(struct file *file,
+				   struct v4l2_ext_controls __user *kp,
+				   struct v4l2_ext_controls32 __user *up,
+				   void __user *aux_buf, u32 aux_space)
 {
 	struct v4l2_ext_control32 __user *ucontrols;
 	struct v4l2_ext_control __user *kcontrols;
-	unsigned int n;
+	u32 count;
+	u32 n;
 	compat_caddr_t p;
 
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_ext_controls32)) ||
-		get_user(kp->which, &up->which) ||
-		get_user(kp->count, &up->count) ||
-		get_user(kp->error_idx, &up->error_idx) ||
-		copy_from_user(kp->reserved, up->reserved,
-			       sizeof(kp->reserved)))
-			return -EFAULT;
-	if (kp->count == 0) {
-		kp->controls = NULL;
-		return 0;
-	} else if (kp->count > V4L2_CID_MAX_CTRLS) {
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    assign_in_user(&kp->which, &up->which) ||
+	    get_user(count, &up->count) ||
+	    put_user(count, &kp->count) ||
+	    assign_in_user(&kp->error_idx, &up->error_idx) ||
+	    copy_in_user(kp->reserved, up->reserved, sizeof(kp->reserved)))
+		return -EFAULT;
+
+	if (count == 0)
+		return put_user(NULL, &kp->controls);
+	if (count > V4L2_CID_MAX_CTRLS)
 		return -EINVAL;
-	}
 	if (get_user(p, &up->controls))
 		return -EFAULT;
 	ucontrols = compat_ptr(p);
-	if (!access_ok(VERIFY_READ, ucontrols,
-			kp->count * sizeof(struct v4l2_ext_control32)))
+	if (!access_ok(VERIFY_READ, ucontrols, count * sizeof(*ucontrols)))
 		return -EFAULT;
-	kcontrols = compat_alloc_user_space(kp->count *
-					    sizeof(struct v4l2_ext_control));
-	kp->controls = (__force struct v4l2_ext_control *)kcontrols;
-	for (n = 0; n < kp->count; n++) {
+	if (aux_space < count * sizeof(*kcontrols))
+		return -EFAULT;
+	kcontrols = aux_buf;
+	if (put_user((__force struct v4l2_ext_control *)kcontrols,
+		     &kp->controls))
+		return -EFAULT;
+
+	for (n = 0; n < count; n++) {
 		u32 id;
 
 		if (copy_in_user(kcontrols, ucontrols, sizeof(*ucontrols)))
 			return -EFAULT;
+
 		if (get_user(id, &kcontrols->id))
 			return -EFAULT;
-		if (ctrl_is_pointer(id)) {
+
+		if (ctrl_is_pointer(file, id)) {
 			void __user *s;
 
 			if (get_user(p, &ucontrols->string))
@@ -737,43 +850,55 @@ static int get_v4l2_ext_controls32(struct v4l2_ext_controls *kp, struct v4l2_ext
 	return 0;
 }
 
-static int put_v4l2_ext_controls32(struct v4l2_ext_controls *kp, struct v4l2_ext_controls32 __user *up)
+static int put_v4l2_ext_controls32(struct file *file,
+				   struct v4l2_ext_controls __user *kp,
+				   struct v4l2_ext_controls32 __user *up)
 {
 	struct v4l2_ext_control32 __user *ucontrols;
-	struct v4l2_ext_control __user *kcontrols =
-		(__force struct v4l2_ext_control __user *)kp->controls;
-	int n = kp->count;
+	struct v4l2_ext_control __user *kcontrols;
+	u32 count;
+	u32 n;
 	compat_caddr_t p;
 
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_ext_controls32)) ||
-		put_user(kp->which, &up->which) ||
-		put_user(kp->count, &up->count) ||
-		put_user(kp->error_idx, &up->error_idx) ||
-		copy_to_user(up->reserved, kp->reserved, sizeof(up->reserved)))
-			return -EFAULT;
-	if (!kp->count)
-		return 0;
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
+	    assign_in_user(&up->which, &kp->which) ||
+	    get_user(count, &kp->count) ||
+	    put_user(count, &up->count) ||
+	    assign_in_user(&up->error_idx, &kp->error_idx) ||
+	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)) ||
+	    get_user(kcontrols, &kp->controls))
+		return -EFAULT;
 
+	if (!count)
+		return 0;
 	if (get_user(p, &up->controls))
 		return -EFAULT;
 	ucontrols = compat_ptr(p);
-	if (!access_ok(VERIFY_WRITE, ucontrols,
-			n * sizeof(struct v4l2_ext_control32)))
+	if (!access_ok(VERIFY_WRITE, ucontrols, count * sizeof(*ucontrols)))
 		return -EFAULT;
 
-	while (--n >= 0) {
-		unsigned size = sizeof(*ucontrols);
+	for (n = 0; n < count; n++) {
+		unsigned int size = sizeof(*ucontrols);
 		u32 id;
 
-		if (get_user(id, &kcontrols->id))
+		if (get_user(id, &kcontrols->id) ||
+		    put_user(id, &ucontrols->id) ||
+		    assign_in_user(&ucontrols->size, &kcontrols->size) ||
+		    copy_in_user(&ucontrols->reserved2, &kcontrols->reserved2,
+				 sizeof(ucontrols->reserved2)))
 			return -EFAULT;
-		/* Do not modify the pointer when copying a pointer control.
-		   The contents of the pointer was changed, not the pointer
-		   itself. */
-		if (ctrl_is_pointer(id))
+
+		/*
+		 * Do not modify the pointer when copying a pointer control.
+		 * The contents of the pointer was changed, not the pointer
+		 * itself.
+		 */
+		if (ctrl_is_pointer(file, id))
 			size -= sizeof(ucontrols->value64);
+
 		if (copy_in_user(ucontrols, kcontrols, size))
 			return -EFAULT;
+
 		ucontrols++;
 		kcontrols++;
 	}
@@ -793,18 +918,19 @@ struct v4l2_event32 {
 	__u32				reserved[8];
 };
 
-static int put_v4l2_event32(struct v4l2_event *kp, struct v4l2_event32 __user *up)
+static int put_v4l2_event32(struct v4l2_event __user *kp,
+			    struct v4l2_event32 __user *up)
 {
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_event32)) ||
-		put_user(kp->type, &up->type) ||
-		copy_to_user(&up->u, &kp->u, sizeof(kp->u)) ||
-		put_user(kp->pending, &up->pending) ||
-		put_user(kp->sequence, &up->sequence) ||
-		put_user(kp->timestamp.tv_sec, &up->timestamp.tv_sec) ||
-		put_user(kp->timestamp.tv_nsec, &up->timestamp.tv_nsec) ||
-		put_user(kp->id, &up->id) ||
-		copy_to_user(up->reserved, kp->reserved, 8 * sizeof(__u32)))
-			return -EFAULT;
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
+	    assign_in_user(&up->type, &kp->type) ||
+	    copy_in_user(&up->u, &kp->u, sizeof(kp->u)) ||
+	    assign_in_user(&up->pending, &kp->pending) ||
+	    assign_in_user(&up->sequence, &kp->sequence) ||
+	    assign_in_user(&up->timestamp.tv_sec, &kp->timestamp.tv_sec) ||
+	    assign_in_user(&up->timestamp.tv_nsec, &kp->timestamp.tv_nsec) ||
+	    assign_in_user(&up->id, &kp->id) ||
+	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)))
+		return -EFAULT;
 	return 0;
 }
 
@@ -816,32 +942,35 @@ struct v4l2_edid32 {
 	compat_caddr_t edid;
 };
 
-static int get_v4l2_edid32(struct v4l2_edid *kp, struct v4l2_edid32 __user *up)
+static int get_v4l2_edid32(struct v4l2_edid __user *kp,
+			   struct v4l2_edid32 __user *up)
 {
-	u32 tmp;
-
-	if (!access_ok(VERIFY_READ, up, sizeof(struct v4l2_edid32)) ||
-		get_user(kp->pad, &up->pad) ||
-		get_user(kp->start_block, &up->start_block) ||
-		get_user(kp->blocks, &up->blocks) ||
-		get_user(tmp, &up->edid) ||
-		copy_from_user(kp->reserved, up->reserved, sizeof(kp->reserved)))
-			return -EFAULT;
-	kp->edid = (__force u8 *)compat_ptr(tmp);
+	compat_uptr_t tmp;
+
+	if (!access_ok(VERIFY_READ, up, sizeof(*up)) ||
+	    assign_in_user(&kp->pad, &up->pad) ||
+	    assign_in_user(&kp->start_block, &up->start_block) ||
+	    assign_in_user(&kp->blocks, &up->blocks) ||
+	    get_user(tmp, &up->edid) ||
+	    put_user(compat_ptr(tmp), &kp->edid) ||
+	    copy_in_user(kp->reserved, up->reserved, sizeof(kp->reserved)))
+		return -EFAULT;
 	return 0;
 }
 
-static int put_v4l2_edid32(struct v4l2_edid *kp, struct v4l2_edid32 __user *up)
+static int put_v4l2_edid32(struct v4l2_edid __user *kp,
+			   struct v4l2_edid32 __user *up)
 {
-	u32 tmp = (u32)((unsigned long)kp->edid);
-
-	if (!access_ok(VERIFY_WRITE, up, sizeof(struct v4l2_edid32)) ||
-		put_user(kp->pad, &up->pad) ||
-		put_user(kp->start_block, &up->start_block) ||
-		put_user(kp->blocks, &up->blocks) ||
-		put_user(tmp, &up->edid) ||
-		copy_to_user(up->reserved, kp->reserved, sizeof(up->reserved)))
-			return -EFAULT;
+	void *edid;
+
+	if (!access_ok(VERIFY_WRITE, up, sizeof(*up)) ||
+	    assign_in_user(&up->pad, &kp->pad) ||
+	    assign_in_user(&up->start_block, &kp->start_block) ||
+	    assign_in_user(&up->blocks, &kp->blocks) ||
+	    get_user(edid, &kp->edid) ||
+	    put_user(ptr_to_compat(edid), &up->edid) ||
+	    copy_in_user(up->reserved, kp->reserved, sizeof(up->reserved)))
+		return -EFAULT;
 	return 0;
 }
 
@@ -857,7 +986,7 @@ static int put_v4l2_edid32(struct v4l2_edid *kp, struct v4l2_edid32 __user *up)
 #define VIDIOC_ENUMINPUT32	_IOWR('V', 26, struct v4l2_input32)
 #define VIDIOC_G_EDID32		_IOWR('V', 40, struct v4l2_edid32)
 #define VIDIOC_S_EDID32		_IOWR('V', 41, struct v4l2_edid32)
-#define VIDIOC_TRY_FMT32      	_IOWR('V', 64, struct v4l2_format32)
+#define VIDIOC_TRY_FMT32	_IOWR('V', 64, struct v4l2_format32)
 #define VIDIOC_G_EXT_CTRLS32    _IOWR('V', 71, struct v4l2_ext_controls32)
 #define VIDIOC_S_EXT_CTRLS32    _IOWR('V', 72, struct v4l2_ext_controls32)
 #define VIDIOC_TRY_EXT_CTRLS32  _IOWR('V', 73, struct v4l2_ext_controls32)
@@ -873,22 +1002,23 @@ static int put_v4l2_edid32(struct v4l2_edid *kp, struct v4l2_edid32 __user *up)
 #define VIDIOC_G_OUTPUT32	_IOR ('V', 46, s32)
 #define VIDIOC_S_OUTPUT32	_IOWR('V', 47, s32)
 
+static int alloc_userspace(unsigned int size, u32 aux_space,
+			   void __user **up_native)
+{
+	*up_native = compat_alloc_user_space(size + aux_space);
+	if (!*up_native)
+		return -ENOMEM;
+	if (clear_user(*up_native, size))
+		return -EFAULT;
+	return 0;
+}
+
 static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
-	union {
-		struct v4l2_format v2f;
-		struct v4l2_buffer v2b;
-		struct v4l2_framebuffer v2fb;
-		struct v4l2_input v2i;
-		struct v4l2_standard v2s;
-		struct v4l2_ext_controls v2ecs;
-		struct v4l2_event v2ev;
-		struct v4l2_create_buffers v2crt;
-		struct v4l2_edid v2edid;
-		unsigned long vx;
-		int vi;
-	} karg;
 	void __user *up = compat_ptr(arg);
+	void __user *up_native = NULL;
+	void __user *aux_buf;
+	u32 aux_space;
 	int compatible_arg = 1;
 	long err = 0;
 
@@ -927,30 +1057,52 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar
 	case VIDIOC_STREAMOFF:
 	case VIDIOC_S_INPUT:
 	case VIDIOC_S_OUTPUT:
-		err = get_user(karg.vi, (s32 __user *)up);
+		err = alloc_userspace(sizeof(unsigned int), 0, &up_native);
+		if (!err && assign_in_user((unsigned int __user *)up_native,
+					   (compat_uint_t __user *)up))
+			err = -EFAULT;
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_INPUT:
 	case VIDIOC_G_OUTPUT:
+		err = alloc_userspace(sizeof(unsigned int), 0, &up_native);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_EDID:
 	case VIDIOC_S_EDID:
-		err = get_v4l2_edid32(&karg.v2edid, up);
+		err = alloc_userspace(sizeof(struct v4l2_edid), 0, &up_native);
+		if (!err)
+			err = get_v4l2_edid32(up_native, up);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_FMT:
 	case VIDIOC_S_FMT:
 	case VIDIOC_TRY_FMT:
-		err = get_v4l2_format32(&karg.v2f, up);
+		err = bufsize_v4l2_format(up, &aux_space);
+		if (!err)
+			err = alloc_userspace(sizeof(struct v4l2_format),
+					      aux_space, &up_native);
+		if (!err) {
+			aux_buf = up_native + sizeof(struct v4l2_format);
+			err = get_v4l2_format32(up_native, up,
+						aux_buf, aux_space);
+		}
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_CREATE_BUFS:
-		err = get_v4l2_create32(&karg.v2crt, up);
+		err = bufsize_v4l2_create(up, &aux_space);
+		if (!err)
+			err = alloc_userspace(sizeof(struct v4l2_create_buffers),
+					      aux_space, &up_native);
+		if (!err) {
+			aux_buf = up_native + sizeof(struct v4l2_create_buffers);
+			err = get_v4l2_create32(up_native, up,
+						aux_buf, aux_space);
+		}
 		compatible_arg = 0;
 		break;
 
@@ -958,36 +1110,63 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar
 	case VIDIOC_QUERYBUF:
 	case VIDIOC_QBUF:
 	case VIDIOC_DQBUF:
-		err = get_v4l2_buffer32(&karg.v2b, up);
+		err = bufsize_v4l2_buffer(up, &aux_space);
+		if (!err)
+			err = alloc_userspace(sizeof(struct v4l2_buffer),
+					      aux_space, &up_native);
+		if (!err) {
+			aux_buf = up_native + sizeof(struct v4l2_buffer);
+			err = get_v4l2_buffer32(up_native, up,
+						aux_buf, aux_space);
+		}
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_S_FBUF:
-		err = get_v4l2_framebuffer32(&karg.v2fb, up);
+		err = alloc_userspace(sizeof(struct v4l2_framebuffer), 0,
+				      &up_native);
+		if (!err)
+			err = get_v4l2_framebuffer32(up_native, up);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_FBUF:
+		err = alloc_userspace(sizeof(struct v4l2_framebuffer), 0,
+				      &up_native);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_ENUMSTD:
-		err = get_v4l2_standard32(&karg.v2s, up);
+		err = alloc_userspace(sizeof(struct v4l2_standard), 0,
+				      &up_native);
+		if (!err)
+			err = get_v4l2_standard32(up_native, up);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_ENUMINPUT:
-		err = get_v4l2_input32(&karg.v2i, up);
+		err = alloc_userspace(sizeof(struct v4l2_input), 0, &up_native);
+		if (!err)
+			err = get_v4l2_input32(up_native, up);
 		compatible_arg = 0;
 		break;
 
 	case VIDIOC_G_EXT_CTRLS:
 	case VIDIOC_S_EXT_CTRLS:
 	case VIDIOC_TRY_EXT_CTRLS:
-		err = get_v4l2_ext_controls32(&karg.v2ecs, up);
+		err = bufsize_v4l2_ext_controls(up, &aux_space);
+		if (!err)
+			err = alloc_userspace(sizeof(struct v4l2_ext_controls),
+					      aux_space, &up_native);
+		if (!err) {
+			aux_buf = up_native + sizeof(struct v4l2_ext_controls);
+			err = get_v4l2_ext_controls32(file, up_native, up,
+						      aux_buf, aux_space);
+		}
 		compatible_arg = 0;
 		break;
 	case VIDIOC_DQEVENT:
+		err = alloc_userspace(sizeof(struct v4l2_event), 0, &up_native);
 		compatible_arg = 0;
 		break;
 	}
@@ -996,26 +1175,26 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar
 
 	if (compatible_arg)
 		err = native_ioctl(file, cmd, (unsigned long)up);
-	else {
-		mm_segment_t old_fs = get_fs();
+	else
+		err = native_ioctl(file, cmd, (unsigned long)up_native);
 
-		set_fs(KERNEL_DS);
-		err = native_ioctl(file, cmd, (unsigned long)&karg);
-		set_fs(old_fs);
-	}
+	if (err == -ENOTTY)
+		return err;
 
-	/* Special case: even after an error we need to put the
-	   results back for these ioctls since the error_idx will
-	   contain information on which control failed. */
+	/*
+	 * Special case: even after an error we need to put the
+	 * results back for these ioctls since the error_idx will
+	 * contain information on which control failed.
+	 */
 	switch (cmd) {
 	case VIDIOC_G_EXT_CTRLS:
 	case VIDIOC_S_EXT_CTRLS:
 	case VIDIOC_TRY_EXT_CTRLS:
-		if (put_v4l2_ext_controls32(&karg.v2ecs, up))
+		if (put_v4l2_ext_controls32(file, up_native, up))
 			err = -EFAULT;
 		break;
 	case VIDIOC_S_EDID:
-		if (put_v4l2_edid32(&karg.v2edid, up))
+		if (put_v4l2_edid32(up_native, up))
 			err = -EFAULT;
 		break;
 	}
@@ -1027,43 +1206,46 @@ static long do_video_ioctl(struct file *file, unsigned int cmd, unsigned long ar
 	case VIDIOC_S_OUTPUT:
 	case VIDIOC_G_INPUT:
 	case VIDIOC_G_OUTPUT:
-		err = put_user(((s32)karg.vi), (s32 __user *)up);
+		if (assign_in_user((compat_uint_t __user *)up,
+				   ((unsigned int __user *)up_native)))
+			err = -EFAULT;
 		break;
 
 	case VIDIOC_G_FBUF:
-		err = put_v4l2_framebuffer32(&karg.v2fb, up);
+		err = put_v4l2_framebuffer32(up_native, up);
 		break;
 
 	case VIDIOC_DQEVENT:
-		err = put_v4l2_event32(&karg.v2ev, up);
+		err = put_v4l2_event32(up_native, up);
 		break;
 
 	case VIDIOC_G_EDID:
-		err = put_v4l2_edid32(&karg.v2edid, up);
+		err = put_v4l2_edid32(up_native, up);
 		break;
 
 	case VIDIOC_G_FMT:
 	case VIDIOC_S_FMT:
 	case VIDIOC_TRY_FMT:
-		err = put_v4l2_format32(&karg.v2f, up);
+		err = put_v4l2_format32(up_native, up);
 		break;
 
 	case VIDIOC_CREATE_BUFS:
-		err = put_v4l2_create32(&karg.v2crt, up);
+		err = put_v4l2_create32(up_native, up);
 		break;
 
+	case VIDIOC_PREPARE_BUF:
 	case VIDIOC_QUERYBUF:
 	case VIDIOC_QBUF:
 	case VIDIOC_DQBUF:
-		err = put_v4l2_buffer32(&karg.v2b, up);
+		err = put_v4l2_buffer32(up_native, up);
 		break;
 
 	case VIDIOC_ENUMSTD:
-		err = put_v4l2_standard32(&karg.v2s, up);
+		err = put_v4l2_standard32(up_native, up);
 		break;
 
 	case VIDIOC_ENUMINPUT:
-		err = put_v4l2_input32(&karg.v2i, up);
+		err = put_v4l2_input32(up_native, up);
 		break;
 	}
 	return err;
diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c
index cbb2ef43945f..ce08b50b8290 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls.c
@@ -3457,12 +3457,12 @@ int v4l2_ctrl_subdev_subscribe_event(struct v4l2_subdev *sd, struct v4l2_fh *fh,
 }
 EXPORT_SYMBOL(v4l2_ctrl_subdev_subscribe_event);
 
-unsigned int v4l2_ctrl_poll(struct file *file, struct poll_table_struct *wait)
+__poll_t v4l2_ctrl_poll(struct file *file, struct poll_table_struct *wait)
 {
 	struct v4l2_fh *fh = file->private_data;
 
 	if (v4l2_event_pending(fh))
-		return POLLPRI;
+		return EPOLLPRI;
 	poll_wait(file, &fh->wait, wait);
 	return 0;
 }
diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c
index c647ba648805..0301fe426a43 100644
--- a/drivers/media/v4l2-core/v4l2-dev.c
+++ b/drivers/media/v4l2-core/v4l2-dev.c
@@ -102,7 +102,7 @@ static DECLARE_BITMAP(devnode_nums[VFL_TYPE_MAX], VIDEO_NUM_DEVICES);
 
 #ifdef CONFIG_VIDEO_FIXED_MINOR_RANGES
 /* Return the bitmap corresponding to vfl_type. */
-static inline unsigned long *devnode_bits(int vfl_type)
+static inline unsigned long *devnode_bits(enum vfl_devnode_type vfl_type)
 {
 	/* Any types not assigned to fixed minor ranges must be mapped to
 	   one single bitmap for the purposes of finding a free node number
@@ -113,7 +113,7 @@ static inline unsigned long *devnode_bits(int vfl_type)
 }
 #else
 /* Return the bitmap corresponding to vfl_type. */
-static inline unsigned long *devnode_bits(int vfl_type)
+static inline unsigned long *devnode_bits(enum vfl_devnode_type vfl_type)
 {
 	return devnode_nums[vfl_type];
 }
@@ -331,10 +331,10 @@ static ssize_t v4l2_write(struct file *filp, const char __user *buf,
 	return ret;
 }
 
-static unsigned int v4l2_poll(struct file *filp, struct poll_table_struct *poll)
+static __poll_t v4l2_poll(struct file *filp, struct poll_table_struct *poll)
 {
 	struct video_device *vdev = video_devdata(filp);
-	unsigned int res = POLLERR | POLLHUP;
+	__poll_t res = EPOLLERR | EPOLLHUP;
 
 	if (!vdev->fops->poll)
 		return DEFAULT_POLLMASK;
@@ -821,8 +821,10 @@ static int video_register_media_controller(struct video_device *vdev, int type)
 	return 0;
 }
 
-int __video_register_device(struct video_device *vdev, int type, int nr,
-		int warn_if_nr_in_use, struct module *owner)
+int __video_register_device(struct video_device *vdev,
+			    enum vfl_devnode_type type,
+			    int nr, int warn_if_nr_in_use,
+			    struct module *owner)
 {
 	int i = 0;
 	int ret;
diff --git a/drivers/media/v4l2-core/v4l2-dv-timings.c b/drivers/media/v4l2-core/v4l2-dv-timings.c
index 930f9c53a64e..e2ee5f00c445 100644
--- a/drivers/media/v4l2-core/v4l2-dv-timings.c
+++ b/drivers/media/v4l2-core/v4l2-dv-timings.c
@@ -249,7 +249,7 @@ EXPORT_SYMBOL_GPL(v4l2_find_dv_timings_cea861_vic);
  * @t2: with this struct.
  * @pclock_delta: the allowed pixelclock deviation.
  * @match_reduced_fps: if true, then fail if V4L2_DV_FL_REDUCED_FPS does not
- * 	match.
+ *	match.
  *
  * Compare t1 with t2 with a given margin of error for the pixelclock.
  */
diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c
index fb72c7ac04d4..d630640642ee 100644
--- a/drivers/media/v4l2-core/v4l2-fwnode.c
+++ b/drivers/media/v4l2-core/v4l2-fwnode.c
@@ -359,9 +359,9 @@ static int v4l2_async_notifier_fwnode_parse_endpoint(
 		return -ENOMEM;
 
 	asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
-	asd->match.fwnode.fwnode =
+	asd->match.fwnode =
 		fwnode_graph_get_remote_port_parent(endpoint);
-	if (!asd->match.fwnode.fwnode) {
+	if (!asd->match.fwnode) {
 		dev_warn(dev, "bad remote port parent\n");
 		ret = -EINVAL;
 		goto out_err;
@@ -393,7 +393,7 @@ static int v4l2_async_notifier_fwnode_parse_endpoint(
 	return 0;
 
 out_err:
-	fwnode_handle_put(asd->match.fwnode.fwnode);
+	fwnode_handle_put(asd->match.fwnode);
 	kfree(asd);
 
 	return ret == -ENOTCONN ? 0 : ret;
@@ -566,7 +566,7 @@ static int v4l2_fwnode_reference_parse(
 		}
 
 		notifier->subdevs[notifier->num_subdevs] = asd;
-		asd->match.fwnode.fwnode = args.fwnode;
+		asd->match.fwnode = args.fwnode;
 		asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
 		notifier->num_subdevs++;
 	}
@@ -853,7 +853,7 @@ static int v4l2_fwnode_reference_parse_int_props(
 		}
 
 		notifier->subdevs[notifier->num_subdevs] = asd;
-		asd->match.fwnode.fwnode = fwnode;
+		asd->match.fwnode = fwnode;
 		asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
 		notifier->num_subdevs++;
 	}
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index 79614992ee21..260288ca4f55 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -46,37 +46,37 @@ struct std_descr {
 };
 
 static const struct std_descr standards[] = {
-	{ V4L2_STD_NTSC, 	"NTSC"      },
-	{ V4L2_STD_NTSC_M, 	"NTSC-M"    },
-	{ V4L2_STD_NTSC_M_JP, 	"NTSC-M-JP" },
+	{ V4L2_STD_NTSC,	"NTSC"      },
+	{ V4L2_STD_NTSC_M,	"NTSC-M"    },
+	{ V4L2_STD_NTSC_M_JP,	"NTSC-M-JP" },
 	{ V4L2_STD_NTSC_M_KR,	"NTSC-M-KR" },
-	{ V4L2_STD_NTSC_443, 	"NTSC-443"  },
-	{ V4L2_STD_PAL, 	"PAL"       },
-	{ V4L2_STD_PAL_BG, 	"PAL-BG"    },
-	{ V4L2_STD_PAL_B, 	"PAL-B"     },
-	{ V4L2_STD_PAL_B1, 	"PAL-B1"    },
-	{ V4L2_STD_PAL_G, 	"PAL-G"     },
-	{ V4L2_STD_PAL_H, 	"PAL-H"     },
-	{ V4L2_STD_PAL_I, 	"PAL-I"     },
-	{ V4L2_STD_PAL_DK, 	"PAL-DK"    },
-	{ V4L2_STD_PAL_D, 	"PAL-D"     },
-	{ V4L2_STD_PAL_D1, 	"PAL-D1"    },
-	{ V4L2_STD_PAL_K, 	"PAL-K"     },
-	{ V4L2_STD_PAL_M, 	"PAL-M"     },
-	{ V4L2_STD_PAL_N, 	"PAL-N"     },
-	{ V4L2_STD_PAL_Nc, 	"PAL-Nc"    },
-	{ V4L2_STD_PAL_60, 	"PAL-60"    },
-	{ V4L2_STD_SECAM, 	"SECAM"     },
-	{ V4L2_STD_SECAM_B, 	"SECAM-B"   },
-	{ V4L2_STD_SECAM_G, 	"SECAM-G"   },
-	{ V4L2_STD_SECAM_H, 	"SECAM-H"   },
-	{ V4L2_STD_SECAM_DK, 	"SECAM-DK"  },
-	{ V4L2_STD_SECAM_D, 	"SECAM-D"   },
-	{ V4L2_STD_SECAM_K, 	"SECAM-K"   },
-	{ V4L2_STD_SECAM_K1, 	"SECAM-K1"  },
-	{ V4L2_STD_SECAM_L, 	"SECAM-L"   },
-	{ V4L2_STD_SECAM_LC, 	"SECAM-Lc"  },
-	{ 0, 			"Unknown"   }
+	{ V4L2_STD_NTSC_443,	"NTSC-443"  },
+	{ V4L2_STD_PAL,		"PAL"       },
+	{ V4L2_STD_PAL_BG,	"PAL-BG"    },
+	{ V4L2_STD_PAL_B,	"PAL-B"     },
+	{ V4L2_STD_PAL_B1,	"PAL-B1"    },
+	{ V4L2_STD_PAL_G,	"PAL-G"     },
+	{ V4L2_STD_PAL_H,	"PAL-H"     },
+	{ V4L2_STD_PAL_I,	"PAL-I"     },
+	{ V4L2_STD_PAL_DK,	"PAL-DK"    },
+	{ V4L2_STD_PAL_D,	"PAL-D"     },
+	{ V4L2_STD_PAL_D1,	"PAL-D1"    },
+	{ V4L2_STD_PAL_K,	"PAL-K"     },
+	{ V4L2_STD_PAL_M,	"PAL-M"     },
+	{ V4L2_STD_PAL_N,	"PAL-N"     },
+	{ V4L2_STD_PAL_Nc,	"PAL-Nc"    },
+	{ V4L2_STD_PAL_60,	"PAL-60"    },
+	{ V4L2_STD_SECAM,	"SECAM"     },
+	{ V4L2_STD_SECAM_B,	"SECAM-B"   },
+	{ V4L2_STD_SECAM_G,	"SECAM-G"   },
+	{ V4L2_STD_SECAM_H,	"SECAM-H"   },
+	{ V4L2_STD_SECAM_DK,	"SECAM-DK"  },
+	{ V4L2_STD_SECAM_D,	"SECAM-D"   },
+	{ V4L2_STD_SECAM_K,	"SECAM-K"   },
+	{ V4L2_STD_SECAM_K1,	"SECAM-K1"  },
+	{ V4L2_STD_SECAM_L,	"SECAM-L"   },
+	{ V4L2_STD_SECAM_LC,	"SECAM-Lc"  },
+	{ 0,			"Unknown"   }
 };
 
 /* video4linux standard ID conversion to standard name
@@ -1202,6 +1202,10 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_PIX_FMT_SGBRG10P:	descr = "10-bit Bayer GBGB/RGRG Packed"; break;
 	case V4L2_PIX_FMT_SGRBG10P:	descr = "10-bit Bayer GRGR/BGBG Packed"; break;
 	case V4L2_PIX_FMT_SRGGB10P:	descr = "10-bit Bayer RGRG/GBGB Packed"; break;
+	case V4L2_PIX_FMT_IPU3_SBGGR10: descr = "10-bit bayer BGGR IPU3 Packed"; break;
+	case V4L2_PIX_FMT_IPU3_SGBRG10: descr = "10-bit bayer GBRG IPU3 Packed"; break;
+	case V4L2_PIX_FMT_IPU3_SGRBG10: descr = "10-bit bayer GRBG IPU3 Packed"; break;
+	case V4L2_PIX_FMT_IPU3_SRGGB10: descr = "10-bit bayer RGGB IPU3 Packed"; break;
 	case V4L2_PIX_FMT_SBGGR10ALAW8:	descr = "8-bit Bayer BGBG/GRGR (A-law)"; break;
 	case V4L2_PIX_FMT_SGBRG10ALAW8:	descr = "8-bit Bayer GBGB/RGRG (A-law)"; break;
 	case V4L2_PIX_FMT_SGRBG10ALAW8:	descr = "8-bit Bayer GRGR/BGBG (A-law)"; break;
@@ -1246,6 +1250,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	case V4L2_TCH_FMT_TU08:		descr = "8-bit unsigned touch data"; break;
 	case V4L2_META_FMT_VSP1_HGO:	descr = "R-Car VSP1 1-D Histogram"; break;
 	case V4L2_META_FMT_VSP1_HGT:	descr = "R-Car VSP1 2-D Histogram"; break;
+	case V4L2_META_FMT_UVC:		descr = "UVC payload header metadata"; break;
 
 	default:
 		/* Compressed formats */
@@ -1311,52 +1316,50 @@ static int v4l_enum_fmt(const struct v4l2_ioctl_ops *ops,
 				struct file *file, void *fh, void *arg)
 {
 	struct v4l2_fmtdesc *p = arg;
-	struct video_device *vfd = video_devdata(file);
-	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
-	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
-	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
-	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
-	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
-	int ret = -EINVAL;
+	int ret = check_fmt(file, p->type);
+
+	if (ret)
+		return ret;
+	ret = -EINVAL;
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_enum_fmt_vid_cap))
+		if (unlikely(!ops->vidioc_enum_fmt_vid_cap))
 			break;
 		ret = ops->vidioc_enum_fmt_vid_cap(file, fh, arg);
 		break;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_enum_fmt_vid_cap_mplane))
+		if (unlikely(!ops->vidioc_enum_fmt_vid_cap_mplane))
 			break;
 		ret = ops->vidioc_enum_fmt_vid_cap_mplane(file, fh, arg);
 		break;
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_enum_fmt_vid_overlay))
+		if (unlikely(!ops->vidioc_enum_fmt_vid_overlay))
 			break;
 		ret = ops->vidioc_enum_fmt_vid_overlay(file, fh, arg);
 		break;
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_enum_fmt_vid_out))
+		if (unlikely(!ops->vidioc_enum_fmt_vid_out))
 			break;
 		ret = ops->vidioc_enum_fmt_vid_out(file, fh, arg);
 		break;
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_enum_fmt_vid_out_mplane))
+		if (unlikely(!ops->vidioc_enum_fmt_vid_out_mplane))
 			break;
 		ret = ops->vidioc_enum_fmt_vid_out_mplane(file, fh, arg);
 		break;
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
-		if (unlikely(!is_rx || !is_sdr || !ops->vidioc_enum_fmt_sdr_cap))
+		if (unlikely(!ops->vidioc_enum_fmt_sdr_cap))
 			break;
 		ret = ops->vidioc_enum_fmt_sdr_cap(file, fh, arg);
 		break;
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		if (unlikely(!is_tx || !is_sdr || !ops->vidioc_enum_fmt_sdr_out))
+		if (unlikely(!ops->vidioc_enum_fmt_sdr_out))
 			break;
 		ret = ops->vidioc_enum_fmt_sdr_out(file, fh, arg);
 		break;
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_enum_fmt_meta_cap))
+		if (unlikely(!ops->vidioc_enum_fmt_meta_cap))
 			break;
 		ret = ops->vidioc_enum_fmt_meta_cap(file, fh, arg);
 		break;
@@ -1370,13 +1373,10 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 				struct file *file, void *fh, void *arg)
 {
 	struct v4l2_format *p = arg;
-	struct video_device *vfd = video_devdata(file);
-	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
-	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
-	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
-	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
-	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
-	int ret;
+	int ret = check_fmt(file, p->type);
+
+	if (ret)
+		return ret;
 
 	/*
 	 * fmt can't be cleared for these overlay types due to the 'clips'
@@ -1404,7 +1404,7 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_g_fmt_vid_cap))
+		if (unlikely(!ops->vidioc_g_fmt_vid_cap))
 			break;
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		ret = ops->vidioc_g_fmt_vid_cap(file, fh, arg);
@@ -1412,23 +1412,15 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		return ret;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_g_fmt_vid_cap_mplane))
-			break;
 		return ops->vidioc_g_fmt_vid_cap_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_g_fmt_vid_overlay))
-			break;
 		return ops->vidioc_g_fmt_vid_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
-		if (unlikely(!is_rx || is_vid || !ops->vidioc_g_fmt_vbi_cap))
-			break;
 		return ops->vidioc_g_fmt_vbi_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE:
-		if (unlikely(!is_rx || is_vid || !ops->vidioc_g_fmt_sliced_vbi_cap))
-			break;
 		return ops->vidioc_g_fmt_sliced_vbi_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_g_fmt_vid_out))
+		if (unlikely(!ops->vidioc_g_fmt_vid_out))
 			break;
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		ret = ops->vidioc_g_fmt_vid_out(file, fh, arg);
@@ -1436,32 +1428,18 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops,
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		return ret;
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_g_fmt_vid_out_mplane))
-			break;
 		return ops->vidioc_g_fmt_vid_out_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_g_fmt_vid_out_overlay))
-			break;
 		return ops->vidioc_g_fmt_vid_out_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
-		if (unlikely(!is_tx || is_vid || !ops->vidioc_g_fmt_vbi_out))
-			break;
 		return ops->vidioc_g_fmt_vbi_out(file, fh, arg);
 	case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
-		if (unlikely(!is_tx || is_vid || !ops->vidioc_g_fmt_sliced_vbi_out))
-			break;
 		return ops->vidioc_g_fmt_sliced_vbi_out(file, fh, arg);
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
-		if (unlikely(!is_rx || !is_sdr || !ops->vidioc_g_fmt_sdr_cap))
-			break;
 		return ops->vidioc_g_fmt_sdr_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		if (unlikely(!is_tx || !is_sdr || !ops->vidioc_g_fmt_sdr_out))
-			break;
 		return ops->vidioc_g_fmt_sdr_out(file, fh, arg);
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_g_fmt_meta_cap))
-			break;
 		return ops->vidioc_g_fmt_meta_cap(file, fh, arg);
 	}
 	return -EINVAL;
@@ -1487,12 +1465,10 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 {
 	struct v4l2_format *p = arg;
 	struct video_device *vfd = video_devdata(file);
-	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
-	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
-	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
-	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
-	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
-	int ret;
+	int ret = check_fmt(file, p->type);
+
+	if (ret)
+		return ret;
 
 	ret = v4l_enable_media_source(vfd);
 	if (ret)
@@ -1501,37 +1477,37 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_s_fmt_vid_cap))
+		if (unlikely(!ops->vidioc_s_fmt_vid_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix);
 		ret = ops->vidioc_s_fmt_vid_cap(file, fh, arg);
 		/* just in case the driver zeroed it again */
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
-		if (is_tch)
+		if (vfd->vfl_type == VFL_TYPE_TOUCH)
 			v4l_pix_format_touch(&p->fmt.pix);
 		return ret;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_s_fmt_vid_cap_mplane))
+		if (unlikely(!ops->vidioc_s_fmt_vid_cap_mplane))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_s_fmt_vid_cap_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_s_fmt_vid_overlay))
+		if (unlikely(!ops->vidioc_s_fmt_vid_overlay))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.win);
 		return ops->vidioc_s_fmt_vid_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
-		if (unlikely(!is_rx || is_vid || !ops->vidioc_s_fmt_vbi_cap))
+		if (unlikely(!ops->vidioc_s_fmt_vbi_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.vbi);
 		return ops->vidioc_s_fmt_vbi_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE:
-		if (unlikely(!is_rx || is_vid || !ops->vidioc_s_fmt_sliced_vbi_cap))
+		if (unlikely(!ops->vidioc_s_fmt_sliced_vbi_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sliced);
 		return ops->vidioc_s_fmt_sliced_vbi_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_s_fmt_vid_out))
+		if (unlikely(!ops->vidioc_s_fmt_vid_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix);
 		ret = ops->vidioc_s_fmt_vid_out(file, fh, arg);
@@ -1539,37 +1515,37 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops,
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		return ret;
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_s_fmt_vid_out_mplane))
+		if (unlikely(!ops->vidioc_s_fmt_vid_out_mplane))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_s_fmt_vid_out_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_s_fmt_vid_out_overlay))
+		if (unlikely(!ops->vidioc_s_fmt_vid_out_overlay))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.win);
 		return ops->vidioc_s_fmt_vid_out_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
-		if (unlikely(!is_tx || is_vid || !ops->vidioc_s_fmt_vbi_out))
+		if (unlikely(!ops->vidioc_s_fmt_vbi_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.vbi);
 		return ops->vidioc_s_fmt_vbi_out(file, fh, arg);
 	case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
-		if (unlikely(!is_tx || is_vid || !ops->vidioc_s_fmt_sliced_vbi_out))
+		if (unlikely(!ops->vidioc_s_fmt_sliced_vbi_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sliced);
 		return ops->vidioc_s_fmt_sliced_vbi_out(file, fh, arg);
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
-		if (unlikely(!is_rx || !is_sdr || !ops->vidioc_s_fmt_sdr_cap))
+		if (unlikely(!ops->vidioc_s_fmt_sdr_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sdr);
 		return ops->vidioc_s_fmt_sdr_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		if (unlikely(!is_tx || !is_sdr || !ops->vidioc_s_fmt_sdr_out))
+		if (unlikely(!ops->vidioc_s_fmt_sdr_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sdr);
 		return ops->vidioc_s_fmt_sdr_out(file, fh, arg);
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_s_fmt_meta_cap))
+		if (unlikely(!ops->vidioc_s_fmt_meta_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.meta);
 		return ops->vidioc_s_fmt_meta_cap(file, fh, arg);
@@ -1581,19 +1557,16 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 				struct file *file, void *fh, void *arg)
 {
 	struct v4l2_format *p = arg;
-	struct video_device *vfd = video_devdata(file);
-	bool is_vid = vfd->vfl_type == VFL_TYPE_GRABBER;
-	bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR;
-	bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
-	bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
-	bool is_tx = vfd->vfl_dir != VFL_DIR_RX;
-	int ret;
+	int ret = check_fmt(file, p->type);
+
+	if (ret)
+		return ret;
 
 	v4l_sanitize_format(p);
 
 	switch (p->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
-		if (unlikely(!is_rx || (!is_vid && !is_tch) || !ops->vidioc_try_fmt_vid_cap))
+		if (unlikely(!ops->vidioc_try_fmt_vid_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix);
 		ret = ops->vidioc_try_fmt_vid_cap(file, fh, arg);
@@ -1601,27 +1574,27 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		return ret;
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_try_fmt_vid_cap_mplane))
+		if (unlikely(!ops->vidioc_try_fmt_vid_cap_mplane))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_try_fmt_vid_cap_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OVERLAY:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_try_fmt_vid_overlay))
+		if (unlikely(!ops->vidioc_try_fmt_vid_overlay))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.win);
 		return ops->vidioc_try_fmt_vid_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_CAPTURE:
-		if (unlikely(!is_rx || is_vid || !ops->vidioc_try_fmt_vbi_cap))
+		if (unlikely(!ops->vidioc_try_fmt_vbi_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.vbi);
 		return ops->vidioc_try_fmt_vbi_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE:
-		if (unlikely(!is_rx || is_vid || !ops->vidioc_try_fmt_sliced_vbi_cap))
+		if (unlikely(!ops->vidioc_try_fmt_sliced_vbi_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sliced);
 		return ops->vidioc_try_fmt_sliced_vbi_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_try_fmt_vid_out))
+		if (unlikely(!ops->vidioc_try_fmt_vid_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix);
 		ret = ops->vidioc_try_fmt_vid_out(file, fh, arg);
@@ -1629,37 +1602,37 @@ static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops,
 		p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC;
 		return ret;
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_try_fmt_vid_out_mplane))
+		if (unlikely(!ops->vidioc_try_fmt_vid_out_mplane))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.pix_mp.xfer_func);
 		return ops->vidioc_try_fmt_vid_out_mplane(file, fh, arg);
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY:
-		if (unlikely(!is_tx || !is_vid || !ops->vidioc_try_fmt_vid_out_overlay))
+		if (unlikely(!ops->vidioc_try_fmt_vid_out_overlay))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.win);
 		return ops->vidioc_try_fmt_vid_out_overlay(file, fh, arg);
 	case V4L2_BUF_TYPE_VBI_OUTPUT:
-		if (unlikely(!is_tx || is_vid || !ops->vidioc_try_fmt_vbi_out))
+		if (unlikely(!ops->vidioc_try_fmt_vbi_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.vbi);
 		return ops->vidioc_try_fmt_vbi_out(file, fh, arg);
 	case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
-		if (unlikely(!is_tx || is_vid || !ops->vidioc_try_fmt_sliced_vbi_out))
+		if (unlikely(!ops->vidioc_try_fmt_sliced_vbi_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sliced);
 		return ops->vidioc_try_fmt_sliced_vbi_out(file, fh, arg);
 	case V4L2_BUF_TYPE_SDR_CAPTURE:
-		if (unlikely(!is_rx || !is_sdr || !ops->vidioc_try_fmt_sdr_cap))
+		if (unlikely(!ops->vidioc_try_fmt_sdr_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sdr);
 		return ops->vidioc_try_fmt_sdr_cap(file, fh, arg);
 	case V4L2_BUF_TYPE_SDR_OUTPUT:
-		if (unlikely(!is_tx || !is_sdr || !ops->vidioc_try_fmt_sdr_out))
+		if (unlikely(!ops->vidioc_try_fmt_sdr_out))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.sdr);
 		return ops->vidioc_try_fmt_sdr_out(file, fh, arg);
 	case V4L2_BUF_TYPE_META_CAPTURE:
-		if (unlikely(!is_rx || !is_vid || !ops->vidioc_try_fmt_meta_cap))
+		if (unlikely(!ops->vidioc_try_fmt_meta_cap))
 			break;
 		CLEAR_AFTER_FIELD(p, fmt.meta);
 		return ops->vidioc_try_fmt_meta_cap(file, fh, arg);
@@ -2539,7 +2512,7 @@ struct v4l2_ioctl_info {
 #define INFO_FL_CLEAR(v4l2_struct, field)			\
 	((offsetof(struct v4l2_struct, field) +			\
 	  sizeof(((struct v4l2_struct *)0)->field)) << 16)
-#define INFO_FL_CLEAR_MASK 	(_IOC_SIZEMASK << 16)
+#define INFO_FL_CLEAR_MASK	(_IOC_SIZEMASK << 16)
 
 #define IOCTL_INFO_STD(_ioctl, _vidioc, _debug, _flags)			\
 	[_IOC_NR(_ioctl)] = {						\
@@ -2927,8 +2900,11 @@ video_usercopy(struct file *file, unsigned int cmd, unsigned long arg,
 
 	/* Handles IOCTL */
 	err = func(file, cmd, parg);
-	if (err == -ENOIOCTLCMD)
+	if (err == -ENOTTY || err == -ENOIOCTLCMD) {
 		err = -ENOTTY;
+		goto out;
+	}
+
 	if (err == 0) {
 		if (cmd == VIDIOC_DQBUF)
 			trace_v4l2_dqbuf(video_devdata(file)->minor, parg);
diff --git a/drivers/media/v4l2-core/v4l2-mc.c b/drivers/media/v4l2-core/v4l2-mc.c
index 303980b71aae..1d550afeda13 100644
--- a/drivers/media/v4l2-core/v4l2-mc.c
+++ b/drivers/media/v4l2-core/v4l2-mc.c
@@ -25,8 +25,6 @@
 #include <media/v4l2-fh.h>
 #include <media/v4l2-mc.h>
 #include <media/v4l2-subdev.h>
-#include <media/media-device.h>
-#include <media/v4l2-mc.h>
 #include <media/videobuf2-core.h>
 
 int v4l2_mc_create_media_graph(struct media_device *mdev)
diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c
index bc580fbe18fa..c4f963d96a79 100644
--- a/drivers/media/v4l2-core/v4l2-mem2mem.c
+++ b/drivers/media/v4l2-core/v4l2-mem2mem.c
@@ -500,24 +500,24 @@ int v4l2_m2m_streamoff(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_streamoff);
 
-unsigned int v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+__poll_t v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 			   struct poll_table_struct *wait)
 {
 	struct video_device *vfd = video_devdata(file);
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct vb2_queue *src_q, *dst_q;
 	struct vb2_buffer *src_vb = NULL, *dst_vb = NULL;
-	unsigned int rc = 0;
+	__poll_t rc = 0;
 	unsigned long flags;
 
 	if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags)) {
 		struct v4l2_fh *fh = file->private_data;
 
 		if (v4l2_event_pending(fh))
-			rc = POLLPRI;
-		else if (req_events & POLLPRI)
+			rc = EPOLLPRI;
+		else if (req_events & EPOLLPRI)
 			poll_wait(file, &fh->wait, wait);
-		if (!(req_events & (POLLOUT | POLLWRNORM | POLLIN | POLLRDNORM)))
+		if (!(req_events & (EPOLLOUT | EPOLLWRNORM | EPOLLIN | EPOLLRDNORM)))
 			return rc;
 	}
 
@@ -531,7 +531,7 @@ unsigned int v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 	 */
 	if ((!src_q->streaming || list_empty(&src_q->queued_list))
 		&& (!dst_q->streaming || list_empty(&dst_q->queued_list))) {
-		rc |= POLLERR;
+		rc |= EPOLLERR;
 		goto end;
 	}
 
@@ -548,7 +548,7 @@ unsigned int v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		 */
 		if (dst_q->last_buffer_dequeued) {
 			spin_unlock_irqrestore(&dst_q->done_lock, flags);
-			return rc | POLLIN | POLLRDNORM;
+			return rc | EPOLLIN | EPOLLRDNORM;
 		}
 
 		poll_wait(file, &dst_q->done_wq, wait);
@@ -561,7 +561,7 @@ unsigned int v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 						done_entry);
 	if (src_vb && (src_vb->state == VB2_BUF_STATE_DONE
 			|| src_vb->state == VB2_BUF_STATE_ERROR))
-		rc |= POLLOUT | POLLWRNORM;
+		rc |= EPOLLOUT | EPOLLWRNORM;
 	spin_unlock_irqrestore(&src_q->done_lock, flags);
 
 	spin_lock_irqsave(&dst_q->done_lock, flags);
@@ -570,7 +570,7 @@ unsigned int v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 						done_entry);
 	if (dst_vb && (dst_vb->state == VB2_BUF_STATE_DONE
 			|| dst_vb->state == VB2_BUF_STATE_ERROR))
-		rc |= POLLIN | POLLRDNORM;
+		rc |= EPOLLIN | EPOLLRDNORM;
 	spin_unlock_irqrestore(&dst_q->done_lock, flags);
 
 end:
@@ -794,11 +794,11 @@ int v4l2_m2m_fop_mmap(struct file *file, struct vm_area_struct *vma)
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_fop_mmap);
 
-unsigned int v4l2_m2m_fop_poll(struct file *file, poll_table *wait)
+__poll_t v4l2_m2m_fop_poll(struct file *file, poll_table *wait)
 {
 	struct v4l2_fh *fh = file->private_data;
 	struct v4l2_m2m_ctx *m2m_ctx = fh->m2m_ctx;
-	unsigned int ret;
+	__poll_t ret;
 
 	if (m2m_ctx->q_lock)
 		mutex_lock(m2m_ctx->q_lock);
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index 43fefa73e0a3..c5639817db34 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -469,19 +469,19 @@ static long subdev_compat_ioctl32(struct file *file, unsigned int cmd,
 }
 #endif
 
-static unsigned int subdev_poll(struct file *file, poll_table *wait)
+static __poll_t subdev_poll(struct file *file, poll_table *wait)
 {
 	struct video_device *vdev = video_devdata(file);
 	struct v4l2_subdev *sd = vdev_to_v4l2_subdev(vdev);
 	struct v4l2_fh *fh = file->private_data;
 
 	if (!(sd->flags & V4L2_SUBDEV_FL_HAS_EVENTS))
-		return POLLERR;
+		return EPOLLERR;
 
 	poll_wait(file, &fh->wait, wait);
 
 	if (v4l2_event_pending(fh))
-		return POLLPRI;
+		return EPOLLPRI;
 
 	return 0;
 }
diff --git a/drivers/media/v4l2-core/videobuf-core.c b/drivers/media/v4l2-core/videobuf-core.c
index e87fb13b22dc..2b3981842b4b 100644
--- a/drivers/media/v4l2-core/videobuf-core.c
+++ b/drivers/media/v4l2-core/videobuf-core.c
@@ -1118,24 +1118,24 @@ ssize_t videobuf_read_stream(struct videobuf_queue *q,
 }
 EXPORT_SYMBOL_GPL(videobuf_read_stream);
 
-unsigned int videobuf_poll_stream(struct file *file,
+__poll_t videobuf_poll_stream(struct file *file,
 				  struct videobuf_queue *q,
 				  poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct videobuf_buffer *buf = NULL;
-	unsigned int rc = 0;
+	__poll_t rc = 0;
 
 	videobuf_queue_lock(q);
 	if (q->streaming) {
 		if (!list_empty(&q->stream))
 			buf = list_entry(q->stream.next,
 					 struct videobuf_buffer, stream);
-	} else if (req_events & (POLLIN | POLLRDNORM)) {
+	} else if (req_events & (EPOLLIN | EPOLLRDNORM)) {
 		if (!q->reading)
 			__videobuf_read_start(q);
 		if (!q->reading) {
-			rc = POLLERR;
+			rc = EPOLLERR;
 		} else if (NULL == q->read_buf) {
 			q->read_buf = list_entry(q->stream.next,
 						 struct videobuf_buffer,
@@ -1146,7 +1146,7 @@ unsigned int videobuf_poll_stream(struct file *file,
 		buf = q->read_buf;
 	}
 	if (!buf)
-		rc = POLLERR;
+		rc = EPOLLERR;
 
 	if (0 == rc) {
 		poll_wait(file, &buf->done, wait);
@@ -1157,10 +1157,10 @@ unsigned int videobuf_poll_stream(struct file *file,
 			case V4L2_BUF_TYPE_VBI_OUTPUT:
 			case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT:
 			case V4L2_BUF_TYPE_SDR_OUTPUT:
-				rc = POLLOUT | POLLWRNORM;
+				rc = EPOLLOUT | EPOLLWRNORM;
 				break;
 			default:
-				rc = POLLIN | POLLRDNORM;
+				rc = EPOLLIN | EPOLLRDNORM;
 				break;
 			}
 		}
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index ffc350258041..19a0e83f260d 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -84,6 +84,16 @@ config OMAP_GPMC_DEBUG
 	  bootloader or else the GPMC timings won't be identical with the
 	  bootloader timings.
 
+config TI_EMIF_SRAM
+	tristate "Texas Instruments EMIF SRAM driver"
+	depends on (SOC_AM33XX || SOC_AM43XX) && SRAM
+	help
+	  This driver is for the EMIF module available on Texas Instruments
+	  AM33XX and AM43XX SoCs and is required for PM. Certain parts of
+	  the EMIF PM code must run from on-chip SRAM late in the suspend
+	  sequence so this driver provides several relocatable PM functions
+	  for the SoC PM code to use.
+
 config MVEBU_DEVBUS
 	bool "Marvell EBU Device Bus Controller"
 	default y
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index 929a601d4cd1..66f55240830e 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -23,3 +23,11 @@ obj-$(CONFIG_DA8XX_DDRCTL)	+= da8xx-ddrctl.o
 
 obj-$(CONFIG_SAMSUNG_MC)	+= samsung/
 obj-$(CONFIG_TEGRA_MC)		+= tegra/
+obj-$(CONFIG_TI_EMIF_SRAM)	+= ti-emif-sram.o
+ti-emif-sram-objs		:= ti-emif-pm.o ti-emif-sram-pm.o
+
+AFLAGS_ti-emif-sram-pm.o	:=-Wa,-march=armv7-a
+
+include drivers/memory/Makefile.asm-offsets
+
+drivers/memory/ti-emif-sram-pm.o: include/generated/ti-emif-asm-offsets.h
diff --git a/drivers/memory/Makefile.asm-offsets b/drivers/memory/Makefile.asm-offsets
new file mode 100644
index 000000000000..843ff60ccb5a
--- /dev/null
+++ b/drivers/memory/Makefile.asm-offsets
@@ -0,0 +1,5 @@
+drivers/memory/emif-asm-offsets.s: drivers/memory/emif-asm-offsets.c
+	$(call if_changed_dep,cc_s_c)
+
+include/generated/ti-emif-asm-offsets.h: drivers/memory/emif-asm-offsets.s FORCE
+	$(call filechk,offsets,__TI_EMIF_ASM_OFFSETS_H__)
diff --git a/drivers/memory/brcmstb_dpfe.c b/drivers/memory/brcmstb_dpfe.c
index 0a7bdbed3a6f..e9c1485c32b9 100644
--- a/drivers/memory/brcmstb_dpfe.c
+++ b/drivers/memory/brcmstb_dpfe.c
@@ -45,8 +45,16 @@
 #define REG_TO_DCPU_MBOX	0x10
 #define REG_TO_HOST_MBOX	0x14
 
+/* Macros to process offsets returned by the DCPU */
+#define DRAM_MSG_ADDR_OFFSET	0x0
+#define DRAM_MSG_TYPE_OFFSET	0x1c
+#define DRAM_MSG_ADDR_MASK	((1UL << DRAM_MSG_TYPE_OFFSET) - 1)
+#define DRAM_MSG_TYPE_MASK	((1UL << \
+				 (BITS_PER_LONG - DRAM_MSG_TYPE_OFFSET)) - 1)
+
 /* Message RAM */
-#define DCPU_MSG_RAM(x)		(0x100 + (x) * sizeof(u32))
+#define DCPU_MSG_RAM_START	0x100
+#define DCPU_MSG_RAM(x)		(DCPU_MSG_RAM_START + (x) * sizeof(u32))
 
 /* DRAM Info Offsets & Masks */
 #define DRAM_INFO_INTERVAL	0x0
@@ -255,6 +263,40 @@ static unsigned int get_msg_chksum(const u32 msg[])
 	return sum;
 }
 
+static void __iomem *get_msg_ptr(struct private_data *priv, u32 response,
+				 char *buf, ssize_t *size)
+{
+	unsigned int msg_type;
+	unsigned int offset;
+	void __iomem *ptr = NULL;
+
+	msg_type = (response >> DRAM_MSG_TYPE_OFFSET) & DRAM_MSG_TYPE_MASK;
+	offset = (response >> DRAM_MSG_ADDR_OFFSET) & DRAM_MSG_ADDR_MASK;
+
+	/*
+	 * msg_type == 1: the offset is relative to the message RAM
+	 * msg_type == 0: the offset is relative to the data RAM (this is the
+	 *                previous way of passing data)
+	 * msg_type is anything else: there's critical hardware problem
+	 */
+	switch (msg_type) {
+	case 1:
+		ptr = priv->regs + DCPU_MSG_RAM_START + offset;
+		break;
+	case 0:
+		ptr = priv->dmem + offset;
+		break;
+	default:
+		dev_emerg(priv->dev, "invalid message reply from DCPU: %#x\n",
+			response);
+		if (buf && size)
+			*size = sprintf(buf,
+				"FATAL: communication error with DCPU\n");
+	}
+
+	return ptr;
+}
+
 static int __send_command(struct private_data *priv, unsigned int cmd,
 			  u32 result[])
 {
@@ -507,7 +549,7 @@ static ssize_t show_info(struct device *dev, struct device_attribute *devattr,
 {
 	u32 response[MSG_FIELD_MAX];
 	unsigned int info;
-	int ret;
+	ssize_t ret;
 
 	ret = generic_show(DPFE_CMD_GET_INFO, response, dev, buf);
 	if (ret)
@@ -528,18 +570,19 @@ static ssize_t show_refresh(struct device *dev,
 	u32 response[MSG_FIELD_MAX];
 	void __iomem *info;
 	struct private_data *priv;
-	unsigned int offset;
 	u8 refresh, sr_abort, ppre, thermal_offs, tuf;
 	u32 mr4;
-	int ret;
+	ssize_t ret;
 
 	ret = generic_show(DPFE_CMD_GET_REFRESH, response, dev, buf);
 	if (ret)
 		return ret;
 
 	priv = dev_get_drvdata(dev);
-	offset = response[MSG_ARG0];
-	info = priv->dmem + offset;
+
+	info = get_msg_ptr(priv, response[MSG_ARG0], buf, &ret);
+	if (!info)
+		return ret;
 
 	mr4 = readl_relaxed(info + DRAM_INFO_MR4) & DRAM_INFO_MR4_MASK;
 
@@ -561,7 +604,6 @@ static ssize_t store_refresh(struct device *dev, struct device_attribute *attr,
 	u32 response[MSG_FIELD_MAX];
 	struct private_data *priv;
 	void __iomem *info;
-	unsigned int offset;
 	unsigned long val;
 	int ret;
 
@@ -574,8 +616,10 @@ static ssize_t store_refresh(struct device *dev, struct device_attribute *attr,
 	if (ret)
 		return ret;
 
-	offset = response[MSG_ARG0];
-	info = priv->dmem + offset;
+	info = get_msg_ptr(priv, response[MSG_ARG0], NULL, NULL);
+	if (!info)
+		return -EIO;
+
 	writel_relaxed(val, info + DRAM_INFO_INTERVAL);
 
 	return count;
@@ -587,23 +631,25 @@ static ssize_t show_vendor(struct device *dev, struct device_attribute *devattr,
 	u32 response[MSG_FIELD_MAX];
 	struct private_data *priv;
 	void __iomem *info;
-	unsigned int offset;
-	int ret;
+	ssize_t ret;
 
 	ret = generic_show(DPFE_CMD_GET_VENDOR, response, dev, buf);
 	if (ret)
 		return ret;
 
-	offset = response[MSG_ARG0];
 	priv = dev_get_drvdata(dev);
-	info = priv->dmem + offset;
+
+	info = get_msg_ptr(priv, response[MSG_ARG0], buf, &ret);
+	if (!info)
+		return ret;
 
 	return sprintf(buf, "%#x %#x %#x %#x %#x\n",
 		       readl_relaxed(info + DRAM_VENDOR_MR5) & DRAM_VENDOR_MASK,
 		       readl_relaxed(info + DRAM_VENDOR_MR6) & DRAM_VENDOR_MASK,
 		       readl_relaxed(info + DRAM_VENDOR_MR7) & DRAM_VENDOR_MASK,
 		       readl_relaxed(info + DRAM_VENDOR_MR8) & DRAM_VENDOR_MASK,
-		       readl_relaxed(info + DRAM_VENDOR_ERROR));
+		       readl_relaxed(info + DRAM_VENDOR_ERROR) &
+				     DRAM_VENDOR_MASK);
 }
 
 static int brcmstb_dpfe_resume(struct platform_device *pdev)
diff --git a/drivers/memory/emif-asm-offsets.c b/drivers/memory/emif-asm-offsets.c
new file mode 100644
index 000000000000..71a89d5d3efd
--- /dev/null
+++ b/drivers/memory/emif-asm-offsets.c
@@ -0,0 +1,92 @@
+/*
+ * TI AM33XX EMIF PM Assembly Offsets
+ *
+ * Copyright (C) 2016-2017 Texas Instruments Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/ti-emif-sram.h>
+
+int main(void)
+{
+	DEFINE(EMIF_SDCFG_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_sdcfg_val));
+	DEFINE(EMIF_TIMING1_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_timing1_val));
+	DEFINE(EMIF_TIMING2_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_timing2_val));
+	DEFINE(EMIF_TIMING3_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_timing3_val));
+	DEFINE(EMIF_REF_CTRL_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_ref_ctrl_val));
+	DEFINE(EMIF_ZQCFG_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_zqcfg_val));
+	DEFINE(EMIF_PMCR_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_pmcr_val));
+	DEFINE(EMIF_PMCR_SHDW_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_pmcr_shdw_val));
+	DEFINE(EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_rd_wr_level_ramp_ctrl));
+	DEFINE(EMIF_RD_WR_EXEC_THRESH_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_rd_wr_exec_thresh));
+	DEFINE(EMIF_COS_CONFIG_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_cos_config));
+	DEFINE(EMIF_PRIORITY_TO_COS_MAPPING_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_priority_to_cos_mapping));
+	DEFINE(EMIF_CONNECT_ID_SERV_1_MAP_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_connect_id_serv_1_map));
+	DEFINE(EMIF_CONNECT_ID_SERV_2_MAP_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_connect_id_serv_2_map));
+	DEFINE(EMIF_OCP_CONFIG_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_ocp_config_val));
+	DEFINE(EMIF_LPDDR2_NVM_TIM_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_lpddr2_nvm_tim));
+	DEFINE(EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_lpddr2_nvm_tim_shdw));
+	DEFINE(EMIF_DLL_CALIB_CTRL_VAL_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_dll_calib_ctrl_val));
+	DEFINE(EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_dll_calib_ctrl_val_shdw));
+	DEFINE(EMIF_DDR_PHY_CTLR_1_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_ddr_phy_ctlr_1));
+	DEFINE(EMIF_EXT_PHY_CTRL_VALS_OFFSET,
+	       offsetof(struct emif_regs_amx3, emif_ext_phy_ctrl_vals));
+	DEFINE(EMIF_REGS_AMX3_SIZE, sizeof(struct emif_regs_amx3));
+
+	BLANK();
+
+	DEFINE(EMIF_PM_BASE_ADDR_VIRT_OFFSET,
+	       offsetof(struct ti_emif_pm_data, ti_emif_base_addr_virt));
+	DEFINE(EMIF_PM_BASE_ADDR_PHYS_OFFSET,
+	       offsetof(struct ti_emif_pm_data, ti_emif_base_addr_phys));
+	DEFINE(EMIF_PM_CONFIG_OFFSET,
+	       offsetof(struct ti_emif_pm_data, ti_emif_sram_config));
+	DEFINE(EMIF_PM_REGS_VIRT_OFFSET,
+	       offsetof(struct ti_emif_pm_data, regs_virt));
+	DEFINE(EMIF_PM_REGS_PHYS_OFFSET,
+	       offsetof(struct ti_emif_pm_data, regs_phys));
+	DEFINE(EMIF_PM_DATA_SIZE, sizeof(struct ti_emif_pm_data));
+
+	BLANK();
+
+	DEFINE(EMIF_PM_SAVE_CONTEXT_OFFSET,
+	       offsetof(struct ti_emif_pm_functions, save_context));
+	DEFINE(EMIF_PM_RESTORE_CONTEXT_OFFSET,
+	       offsetof(struct ti_emif_pm_functions, restore_context));
+	DEFINE(EMIF_PM_ENTER_SR_OFFSET,
+	       offsetof(struct ti_emif_pm_functions, enter_sr));
+	DEFINE(EMIF_PM_EXIT_SR_OFFSET,
+	       offsetof(struct ti_emif_pm_functions, exit_sr));
+	DEFINE(EMIF_PM_ABORT_SR_OFFSET,
+	       offsetof(struct ti_emif_pm_functions, abort_sr));
+	DEFINE(EMIF_PM_FUNCTIONS_SIZE, sizeof(struct ti_emif_pm_functions));
+
+	return 0;
+}
diff --git a/drivers/memory/emif.h b/drivers/memory/emif.h
index bfe08bae961a..9e9f8037955d 100644
--- a/drivers/memory/emif.h
+++ b/drivers/memory/emif.h
@@ -555,6 +555,9 @@
 #define READ_LATENCY_SHDW_SHIFT				0
 #define READ_LATENCY_SHDW_MASK				(0x1f << 0)
 
+#define EMIF_SRAM_AM33_REG_LAYOUT			0x00000000
+#define EMIF_SRAM_AM43_REG_LAYOUT			0x00000001
+
 #ifndef __ASSEMBLY__
 /*
  * Structure containing shadow of important registers in EMIF
@@ -585,5 +588,19 @@ struct emif_regs {
 	u32 ext_phy_ctrl_3_shdw;
 	u32 ext_phy_ctrl_4_shdw;
 };
+
+struct ti_emif_pm_functions;
+
+extern unsigned int ti_emif_sram;
+extern unsigned int ti_emif_sram_sz;
+extern struct ti_emif_pm_data ti_emif_pm_sram_data;
+extern struct emif_regs_amx3 ti_emif_regs_amx3;
+
+void ti_emif_save_context(void);
+void ti_emif_restore_context(void);
+void ti_emif_enter_sr(void);
+void ti_emif_exit_sr(void);
+void ti_emif_abort_sr(void);
+
 #endif /* __ASSEMBLY__ */
 #endif /* __EMIF_H */
diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c
index a385a35c7de9..90a66b3f7ae1 100644
--- a/drivers/memory/omap-gpmc.c
+++ b/drivers/memory/omap-gpmc.c
@@ -32,7 +32,6 @@
 #include <linux/pm_runtime.h>
 
 #include <linux/platform_data/mtd-nand-omap2.h>
-#include <linux/platform_data/mtd-onenand-omap2.h>
 
 #include <asm/mach-types.h>
 
@@ -1138,6 +1137,112 @@ struct gpmc_nand_ops *gpmc_omap_get_nand_ops(struct gpmc_nand_regs *reg, int cs)
 }
 EXPORT_SYMBOL_GPL(gpmc_omap_get_nand_ops);
 
+static void gpmc_omap_onenand_calc_sync_timings(struct gpmc_timings *t,
+						struct gpmc_settings *s,
+						int freq, int latency)
+{
+	struct gpmc_device_timings dev_t;
+	const int t_cer  = 15;
+	const int t_avdp = 12;
+	const int t_cez  = 20; /* max of t_cez, t_oez */
+	const int t_wpl  = 40;
+	const int t_wph  = 30;
+	int min_gpmc_clk_period, t_ces, t_avds, t_avdh, t_ach, t_aavdh, t_rdyo;
+
+	switch (freq) {
+	case 104:
+		min_gpmc_clk_period = 9600; /* 104 MHz */
+		t_ces   = 3;
+		t_avds  = 4;
+		t_avdh  = 2;
+		t_ach   = 3;
+		t_aavdh = 6;
+		t_rdyo  = 6;
+		break;
+	case 83:
+		min_gpmc_clk_period = 12000; /* 83 MHz */
+		t_ces   = 5;
+		t_avds  = 4;
+		t_avdh  = 2;
+		t_ach   = 6;
+		t_aavdh = 6;
+		t_rdyo  = 9;
+		break;
+	case 66:
+		min_gpmc_clk_period = 15000; /* 66 MHz */
+		t_ces   = 6;
+		t_avds  = 5;
+		t_avdh  = 2;
+		t_ach   = 6;
+		t_aavdh = 6;
+		t_rdyo  = 11;
+		break;
+	default:
+		min_gpmc_clk_period = 18500; /* 54 MHz */
+		t_ces   = 7;
+		t_avds  = 7;
+		t_avdh  = 7;
+		t_ach   = 9;
+		t_aavdh = 7;
+		t_rdyo  = 15;
+		break;
+	}
+
+	/* Set synchronous read timings */
+	memset(&dev_t, 0, sizeof(dev_t));
+
+	if (!s->sync_write) {
+		dev_t.t_avdp_w = max(t_avdp, t_cer) * 1000;
+		dev_t.t_wpl = t_wpl * 1000;
+		dev_t.t_wph = t_wph * 1000;
+		dev_t.t_aavdh = t_aavdh * 1000;
+	}
+	dev_t.ce_xdelay = true;
+	dev_t.avd_xdelay = true;
+	dev_t.oe_xdelay = true;
+	dev_t.we_xdelay = true;
+	dev_t.clk = min_gpmc_clk_period;
+	dev_t.t_bacc = dev_t.clk;
+	dev_t.t_ces = t_ces * 1000;
+	dev_t.t_avds = t_avds * 1000;
+	dev_t.t_avdh = t_avdh * 1000;
+	dev_t.t_ach = t_ach * 1000;
+	dev_t.cyc_iaa = (latency + 1);
+	dev_t.t_cez_r = t_cez * 1000;
+	dev_t.t_cez_w = dev_t.t_cez_r;
+	dev_t.cyc_aavdh_oe = 1;
+	dev_t.t_rdyo = t_rdyo * 1000 + min_gpmc_clk_period;
+
+	gpmc_calc_timings(t, s, &dev_t);
+}
+
+int gpmc_omap_onenand_set_timings(struct device *dev, int cs, int freq,
+				  int latency,
+				  struct gpmc_onenand_info *info)
+{
+	int ret;
+	struct gpmc_timings gpmc_t;
+	struct gpmc_settings gpmc_s;
+
+	gpmc_read_settings_dt(dev->of_node, &gpmc_s);
+
+	info->sync_read = gpmc_s.sync_read;
+	info->sync_write = gpmc_s.sync_write;
+	info->burst_len = gpmc_s.burst_len;
+
+	if (!gpmc_s.sync_read && !gpmc_s.sync_write)
+		return 0;
+
+	gpmc_omap_onenand_calc_sync_timings(&gpmc_t, &gpmc_s, freq, latency);
+
+	ret = gpmc_cs_program_settings(cs, &gpmc_s);
+	if (ret < 0)
+		return ret;
+
+	return gpmc_cs_set_timings(cs, &gpmc_t, &gpmc_s);
+}
+EXPORT_SYMBOL_GPL(gpmc_omap_onenand_set_timings);
+
 int gpmc_get_client_irq(unsigned irq_config)
 {
 	if (!gpmc_irq_domain) {
@@ -1916,41 +2021,6 @@ static void __maybe_unused gpmc_read_timings_dt(struct device_node *np,
 		of_property_read_bool(np, "gpmc,time-para-granularity");
 }
 
-#if IS_ENABLED(CONFIG_MTD_ONENAND)
-static int gpmc_probe_onenand_child(struct platform_device *pdev,
-				 struct device_node *child)
-{
-	u32 val;
-	struct omap_onenand_platform_data *gpmc_onenand_data;
-
-	if (of_property_read_u32(child, "reg", &val) < 0) {
-		dev_err(&pdev->dev, "%pOF has no 'reg' property\n",
-			child);
-		return -ENODEV;
-	}
-
-	gpmc_onenand_data = devm_kzalloc(&pdev->dev, sizeof(*gpmc_onenand_data),
-					 GFP_KERNEL);
-	if (!gpmc_onenand_data)
-		return -ENOMEM;
-
-	gpmc_onenand_data->cs = val;
-	gpmc_onenand_data->of_node = child;
-	gpmc_onenand_data->dma_channel = -1;
-
-	if (!of_property_read_u32(child, "dma-channel", &val))
-		gpmc_onenand_data->dma_channel = val;
-
-	return gpmc_onenand_init(gpmc_onenand_data);
-}
-#else
-static int gpmc_probe_onenand_child(struct platform_device *pdev,
-				    struct device_node *child)
-{
-	return 0;
-}
-#endif
-
 /**
  * gpmc_probe_generic_child - configures the gpmc for a child device
  * @pdev:	pointer to gpmc platform device
@@ -2053,6 +2123,16 @@ static int gpmc_probe_generic_child(struct platform_device *pdev,
 		}
 	}
 
+	if (of_node_cmp(child->name, "onenand") == 0) {
+		/* Warn about older DT blobs with no compatible property */
+		if (!of_property_read_bool(child, "compatible")) {
+			dev_warn(&pdev->dev,
+				 "Incompatible OneNAND node: missing compatible");
+			ret = -EINVAL;
+			goto err;
+		}
+	}
+
 	if (of_device_is_compatible(child, "ti,omap2-nand")) {
 		/* NAND specific setup */
 		val = 8;
@@ -2077,8 +2157,9 @@ static int gpmc_probe_generic_child(struct platform_device *pdev,
 	} else {
 		ret = of_property_read_u32(child, "bank-width",
 					   &gpmc_s.device_width);
-		if (ret < 0) {
-			dev_err(&pdev->dev, "%pOF has no 'bank-width' property\n",
+		if (ret < 0 && !gpmc_s.device_width) {
+			dev_err(&pdev->dev,
+				"%pOF has no 'gpmc,device-width' property\n",
 				child);
 			goto err;
 		}
@@ -2188,11 +2269,7 @@ static void gpmc_probe_dt_children(struct platform_device *pdev)
 		if (!child->name)
 			continue;
 
-		if (of_node_cmp(child->name, "onenand") == 0)
-			ret = gpmc_probe_onenand_child(pdev, child);
-		else
-			ret = gpmc_probe_generic_child(pdev, child);
-
+		ret = gpmc_probe_generic_child(pdev, child);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to probe DT child '%s': %d\n",
 				child->name, ret);
diff --git a/drivers/memory/tegra/Makefile b/drivers/memory/tegra/Makefile
index b44e8627a5e0..ce87a9470034 100644
--- a/drivers/memory/tegra/Makefile
+++ b/drivers/memory/tegra/Makefile
@@ -10,3 +10,4 @@ tegra-mc-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210.o
 obj-$(CONFIG_TEGRA_MC) += tegra-mc.o
 
 obj-$(CONFIG_TEGRA124_EMC) += tegra124-emc.o
+obj-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186.o
diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c
index ba8fff3d66a6..b20e6e3e208e 100644
--- a/drivers/memory/tegra/tegra114.c
+++ b/drivers/memory/tegra/tegra114.c
@@ -912,11 +912,26 @@ static const struct tegra_smmu_swgroup tegra114_swgroups[] = {
 	{ .name = "tsec",      .swgroup = TEGRA_SWGROUP_TSEC,      .reg = 0x294 },
 };
 
+static const unsigned int tegra114_group_display[] = {
+	TEGRA_SWGROUP_DC,
+	TEGRA_SWGROUP_DCB,
+};
+
+static const struct tegra_smmu_group_soc tegra114_groups[] = {
+	{
+		.name = "display",
+		.swgroups = tegra114_group_display,
+		.num_swgroups = ARRAY_SIZE(tegra114_group_display),
+	},
+};
+
 static const struct tegra_smmu_soc tegra114_smmu_soc = {
 	.clients = tegra114_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra114_mc_clients),
 	.swgroups = tegra114_swgroups,
 	.num_swgroups = ARRAY_SIZE(tegra114_swgroups),
+	.groups = tegra114_groups,
+	.num_groups = ARRAY_SIZE(tegra114_groups),
 	.supports_round_robin_arbitration = false,
 	.supports_request_limit = false,
 	.num_tlb_lines = 32,
diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c
index 5a58e440f4a7..8b6360eabb8a 100644
--- a/drivers/memory/tegra/tegra124.c
+++ b/drivers/memory/tegra/tegra124.c
@@ -999,12 +999,27 @@ static const struct tegra_smmu_swgroup tegra124_swgroups[] = {
 	{ .name = "vi",        .swgroup = TEGRA_SWGROUP_VI,        .reg = 0x280 },
 };
 
+static const unsigned int tegra124_group_display[] = {
+	TEGRA_SWGROUP_DC,
+	TEGRA_SWGROUP_DCB,
+};
+
+static const struct tegra_smmu_group_soc tegra124_groups[] = {
+	{
+		.name = "display",
+		.swgroups = tegra124_group_display,
+		.num_swgroups = ARRAY_SIZE(tegra124_group_display),
+	},
+};
+
 #ifdef CONFIG_ARCH_TEGRA_124_SOC
 static const struct tegra_smmu_soc tegra124_smmu_soc = {
 	.clients = tegra124_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra124_mc_clients),
 	.swgroups = tegra124_swgroups,
 	.num_swgroups = ARRAY_SIZE(tegra124_swgroups),
+	.groups = tegra124_groups,
+	.num_groups = ARRAY_SIZE(tegra124_groups),
 	.supports_round_robin_arbitration = true,
 	.supports_request_limit = true,
 	.num_tlb_lines = 32,
@@ -1029,6 +1044,8 @@ static const struct tegra_smmu_soc tegra132_smmu_soc = {
 	.num_clients = ARRAY_SIZE(tegra124_mc_clients),
 	.swgroups = tegra124_swgroups,
 	.num_swgroups = ARRAY_SIZE(tegra124_swgroups),
+	.groups = tegra124_groups,
+	.num_groups = ARRAY_SIZE(tegra124_groups),
 	.supports_round_robin_arbitration = true,
 	.supports_request_limit = true,
 	.num_tlb_lines = 32,
diff --git a/drivers/memory/tegra/tegra186.c b/drivers/memory/tegra/tegra186.c
new file mode 100644
index 000000000000..7254fb596979
--- /dev/null
+++ b/drivers/memory/tegra/tegra186.c
@@ -0,0 +1,600 @@
+/*
+ * Copyright (C) 2017 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include <dt-bindings/memory/tegra186-mc.h>
+
+struct tegra_mc {
+	struct device *dev;
+	void __iomem *regs;
+};
+
+struct tegra_mc_client {
+	const char *name;
+	unsigned int sid;
+	struct {
+		unsigned int override;
+		unsigned int security;
+	} regs;
+};
+
+static const struct tegra_mc_client tegra186_mc_clients[] = {
+	{
+		.name = "ptcr",
+		.sid = TEGRA186_SID_PASSTHROUGH,
+		.regs = {
+			.override = 0x000,
+			.security = 0x004,
+		},
+	}, {
+		.name = "afir",
+		.sid = TEGRA186_SID_AFI,
+		.regs = {
+			.override = 0x070,
+			.security = 0x074,
+		},
+	}, {
+		.name = "hdar",
+		.sid = TEGRA186_SID_HDA,
+		.regs = {
+			.override = 0x0a8,
+			.security = 0x0ac,
+		},
+	}, {
+		.name = "host1xdmar",
+		.sid = TEGRA186_SID_HOST1X,
+		.regs = {
+			.override = 0x0b0,
+			.security = 0x0b4,
+		},
+	}, {
+		.name = "nvencsrd",
+		.sid = TEGRA186_SID_NVENC,
+		.regs = {
+			.override = 0x0e0,
+			.security = 0x0e4,
+		},
+	}, {
+		.name = "satar",
+		.sid = TEGRA186_SID_SATA,
+		.regs = {
+			.override = 0x0f8,
+			.security = 0x0fc,
+		},
+	}, {
+		.name = "mpcorer",
+		.sid = TEGRA186_SID_PASSTHROUGH,
+		.regs = {
+			.override = 0x138,
+			.security = 0x13c,
+		},
+	}, {
+		.name = "nvencswr",
+		.sid = TEGRA186_SID_NVENC,
+		.regs = {
+			.override = 0x158,
+			.security = 0x15c,
+		},
+	}, {
+		.name = "afiw",
+		.sid = TEGRA186_SID_AFI,
+		.regs = {
+			.override = 0x188,
+			.security = 0x18c,
+		},
+	}, {
+		.name = "hdaw",
+		.sid = TEGRA186_SID_HDA,
+		.regs = {
+			.override = 0x1a8,
+			.security = 0x1ac,
+		},
+	}, {
+		.name = "mpcorew",
+		.sid = TEGRA186_SID_PASSTHROUGH,
+		.regs = {
+			.override = 0x1c8,
+			.security = 0x1cc,
+		},
+	}, {
+		.name = "sataw",
+		.sid = TEGRA186_SID_SATA,
+		.regs = {
+			.override = 0x1e8,
+			.security = 0x1ec,
+		},
+	}, {
+		.name = "ispra",
+		.sid = TEGRA186_SID_ISP,
+		.regs = {
+			.override = 0x220,
+			.security = 0x224,
+		},
+	}, {
+		.name = "ispwa",
+		.sid = TEGRA186_SID_ISP,
+		.regs = {
+			.override = 0x230,
+			.security = 0x234,
+		},
+	}, {
+		.name = "ispwb",
+		.sid = TEGRA186_SID_ISP,
+		.regs = {
+			.override = 0x238,
+			.security = 0x23c,
+		},
+	}, {
+		.name = "xusb_hostr",
+		.sid = TEGRA186_SID_XUSB_HOST,
+		.regs = {
+			.override = 0x250,
+			.security = 0x254,
+		},
+	}, {
+		.name = "xusb_hostw",
+		.sid = TEGRA186_SID_XUSB_HOST,
+		.regs = {
+			.override = 0x258,
+			.security = 0x25c,
+		},
+	}, {
+		.name = "xusb_devr",
+		.sid = TEGRA186_SID_XUSB_DEV,
+		.regs = {
+			.override = 0x260,
+			.security = 0x264,
+		},
+	}, {
+		.name = "xusb_devw",
+		.sid = TEGRA186_SID_XUSB_DEV,
+		.regs = {
+			.override = 0x268,
+			.security = 0x26c,
+		},
+	}, {
+		.name = "tsecsrd",
+		.sid = TEGRA186_SID_TSEC,
+		.regs = {
+			.override = 0x2a0,
+			.security = 0x2a4,
+		},
+	}, {
+		.name = "tsecswr",
+		.sid = TEGRA186_SID_TSEC,
+		.regs = {
+			.override = 0x2a8,
+			.security = 0x2ac,
+		},
+	}, {
+		.name = "gpusrd",
+		.sid = TEGRA186_SID_GPU,
+		.regs = {
+			.override = 0x2c0,
+			.security = 0x2c4,
+		},
+	}, {
+		.name = "gpuswr",
+		.sid = TEGRA186_SID_GPU,
+		.regs = {
+			.override = 0x2c8,
+			.security = 0x2cc,
+		},
+	}, {
+		.name = "sdmmcra",
+		.sid = TEGRA186_SID_SDMMC1,
+		.regs = {
+			.override = 0x300,
+			.security = 0x304,
+		},
+	}, {
+		.name = "sdmmcraa",
+		.sid = TEGRA186_SID_SDMMC2,
+		.regs = {
+			.override = 0x308,
+			.security = 0x30c,
+		},
+	}, {
+		.name = "sdmmcr",
+		.sid = TEGRA186_SID_SDMMC3,
+		.regs = {
+			.override = 0x310,
+			.security = 0x314,
+		},
+	}, {
+		.name = "sdmmcrab",
+		.sid = TEGRA186_SID_SDMMC4,
+		.regs = {
+			.override = 0x318,
+			.security = 0x31c,
+		},
+	}, {
+		.name = "sdmmcwa",
+		.sid = TEGRA186_SID_SDMMC1,
+		.regs = {
+			.override = 0x320,
+			.security = 0x324,
+		},
+	}, {
+		.name = "sdmmcwaa",
+		.sid = TEGRA186_SID_SDMMC2,
+		.regs = {
+			.override = 0x328,
+			.security = 0x32c,
+		},
+	}, {
+		.name = "sdmmcw",
+		.sid = TEGRA186_SID_SDMMC3,
+		.regs = {
+			.override = 0x330,
+			.security = 0x334,
+		},
+	}, {
+		.name = "sdmmcwab",
+		.sid = TEGRA186_SID_SDMMC4,
+		.regs = {
+			.override = 0x338,
+			.security = 0x33c,
+		},
+	}, {
+		.name = "vicsrd",
+		.sid = TEGRA186_SID_VIC,
+		.regs = {
+			.override = 0x360,
+			.security = 0x364,
+		},
+	}, {
+		.name = "vicswr",
+		.sid = TEGRA186_SID_VIC,
+		.regs = {
+			.override = 0x368,
+			.security = 0x36c,
+		},
+	}, {
+		.name = "viw",
+		.sid = TEGRA186_SID_VI,
+		.regs = {
+			.override = 0x390,
+			.security = 0x394,
+		},
+	}, {
+		.name = "nvdecsrd",
+		.sid = TEGRA186_SID_NVDEC,
+		.regs = {
+			.override = 0x3c0,
+			.security = 0x3c4,
+		},
+	}, {
+		.name = "nvdecswr",
+		.sid = TEGRA186_SID_NVDEC,
+		.regs = {
+			.override = 0x3c8,
+			.security = 0x3cc,
+		},
+	}, {
+		.name = "aper",
+		.sid = TEGRA186_SID_APE,
+		.regs = {
+			.override = 0x3d0,
+			.security = 0x3d4,
+		},
+	}, {
+		.name = "apew",
+		.sid = TEGRA186_SID_APE,
+		.regs = {
+			.override = 0x3d8,
+			.security = 0x3dc,
+		},
+	}, {
+		.name = "nvjpgsrd",
+		.sid = TEGRA186_SID_NVJPG,
+		.regs = {
+			.override = 0x3f0,
+			.security = 0x3f4,
+		},
+	}, {
+		.name = "nvjpgswr",
+		.sid = TEGRA186_SID_NVJPG,
+		.regs = {
+			.override = 0x3f8,
+			.security = 0x3fc,
+		},
+	}, {
+		.name = "sesrd",
+		.sid = TEGRA186_SID_SE,
+		.regs = {
+			.override = 0x400,
+			.security = 0x404,
+		},
+	}, {
+		.name = "seswr",
+		.sid = TEGRA186_SID_SE,
+		.regs = {
+			.override = 0x408,
+			.security = 0x40c,
+		},
+	}, {
+		.name = "etrr",
+		.sid = TEGRA186_SID_ETR,
+		.regs = {
+			.override = 0x420,
+			.security = 0x424,
+		},
+	}, {
+		.name = "etrw",
+		.sid = TEGRA186_SID_ETR,
+		.regs = {
+			.override = 0x428,
+			.security = 0x42c,
+		},
+	}, {
+		.name = "tsecsrdb",
+		.sid = TEGRA186_SID_TSECB,
+		.regs = {
+			.override = 0x430,
+			.security = 0x434,
+		},
+	}, {
+		.name = "tsecswrb",
+		.sid = TEGRA186_SID_TSECB,
+		.regs = {
+			.override = 0x438,
+			.security = 0x43c,
+		},
+	}, {
+		.name = "gpusrd2",
+		.sid = TEGRA186_SID_GPU,
+		.regs = {
+			.override = 0x440,
+			.security = 0x444,
+		},
+	}, {
+		.name = "gpuswr2",
+		.sid = TEGRA186_SID_GPU,
+		.regs = {
+			.override = 0x448,
+			.security = 0x44c,
+		},
+	}, {
+		.name = "axisr",
+		.sid = TEGRA186_SID_GPCDMA_0,
+		.regs = {
+			.override = 0x460,
+			.security = 0x464,
+		},
+	}, {
+		.name = "axisw",
+		.sid = TEGRA186_SID_GPCDMA_0,
+		.regs = {
+			.override = 0x468,
+			.security = 0x46c,
+		},
+	}, {
+		.name = "eqosr",
+		.sid = TEGRA186_SID_EQOS,
+		.regs = {
+			.override = 0x470,
+			.security = 0x474,
+		},
+	}, {
+		.name = "eqosw",
+		.sid = TEGRA186_SID_EQOS,
+		.regs = {
+			.override = 0x478,
+			.security = 0x47c,
+		},
+	}, {
+		.name = "ufshcr",
+		.sid = TEGRA186_SID_UFSHC,
+		.regs = {
+			.override = 0x480,
+			.security = 0x484,
+		},
+	}, {
+		.name = "ufshcw",
+		.sid = TEGRA186_SID_UFSHC,
+		.regs = {
+			.override = 0x488,
+			.security = 0x48c,
+		},
+	}, {
+		.name = "nvdisplayr",
+		.sid = TEGRA186_SID_NVDISPLAY,
+		.regs = {
+			.override = 0x490,
+			.security = 0x494,
+		},
+	}, {
+		.name = "bpmpr",
+		.sid = TEGRA186_SID_BPMP,
+		.regs = {
+			.override = 0x498,
+			.security = 0x49c,
+		},
+	}, {
+		.name = "bpmpw",
+		.sid = TEGRA186_SID_BPMP,
+		.regs = {
+			.override = 0x4a0,
+			.security = 0x4a4,
+		},
+	}, {
+		.name = "bpmpdmar",
+		.sid = TEGRA186_SID_BPMP,
+		.regs = {
+			.override = 0x4a8,
+			.security = 0x4ac,
+		},
+	}, {
+		.name = "bpmpdmaw",
+		.sid = TEGRA186_SID_BPMP,
+		.regs = {
+			.override = 0x4b0,
+			.security = 0x4b4,
+		},
+	}, {
+		.name = "aonr",
+		.sid = TEGRA186_SID_AON,
+		.regs = {
+			.override = 0x4b8,
+			.security = 0x4bc,
+		},
+	}, {
+		.name = "aonw",
+		.sid = TEGRA186_SID_AON,
+		.regs = {
+			.override = 0x4c0,
+			.security = 0x4c4,
+		},
+	}, {
+		.name = "aondmar",
+		.sid = TEGRA186_SID_AON,
+		.regs = {
+			.override = 0x4c8,
+			.security = 0x4cc,
+		},
+	}, {
+		.name = "aondmaw",
+		.sid = TEGRA186_SID_AON,
+		.regs = {
+			.override = 0x4d0,
+			.security = 0x4d4,
+		},
+	}, {
+		.name = "scer",
+		.sid = TEGRA186_SID_SCE,
+		.regs = {
+			.override = 0x4d8,
+			.security = 0x4dc,
+		},
+	}, {
+		.name = "scew",
+		.sid = TEGRA186_SID_SCE,
+		.regs = {
+			.override = 0x4e0,
+			.security = 0x4e4,
+		},
+	}, {
+		.name = "scedmar",
+		.sid = TEGRA186_SID_SCE,
+		.regs = {
+			.override = 0x4e8,
+			.security = 0x4ec,
+		},
+	}, {
+		.name = "scedmaw",
+		.sid = TEGRA186_SID_SCE,
+		.regs = {
+			.override = 0x4f0,
+			.security = 0x4f4,
+		},
+	}, {
+		.name = "apedmar",
+		.sid = TEGRA186_SID_APE,
+		.regs = {
+			.override = 0x4f8,
+			.security = 0x4fc,
+		},
+	}, {
+		.name = "apedmaw",
+		.sid = TEGRA186_SID_APE,
+		.regs = {
+			.override = 0x500,
+			.security = 0x504,
+		},
+	}, {
+		.name = "nvdisplayr1",
+		.sid = TEGRA186_SID_NVDISPLAY,
+		.regs = {
+			.override = 0x508,
+			.security = 0x50c,
+		},
+	}, {
+		.name = "vicsrd1",
+		.sid = TEGRA186_SID_VIC,
+		.regs = {
+			.override = 0x510,
+			.security = 0x514,
+		},
+	}, {
+		.name = "nvdecsrd1",
+		.sid = TEGRA186_SID_NVDEC,
+		.regs = {
+			.override = 0x518,
+			.security = 0x51c,
+		},
+	},
+};
+
+static int tegra186_mc_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct tegra_mc *mc;
+	unsigned int i;
+	int err = 0;
+
+	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
+	if (!mc)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mc->regs = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mc->regs))
+		return PTR_ERR(mc->regs);
+
+	mc->dev = &pdev->dev;
+
+	for (i = 0; i < ARRAY_SIZE(tegra186_mc_clients); i++) {
+		const struct tegra_mc_client *client = &tegra186_mc_clients[i];
+		u32 override, security;
+
+		override = readl(mc->regs + client->regs.override);
+		security = readl(mc->regs + client->regs.security);
+
+		dev_dbg(&pdev->dev, "client %s: override: %x security: %x\n",
+			client->name, override, security);
+
+		dev_dbg(&pdev->dev, "setting SID %u for %s\n", client->sid,
+			client->name);
+		writel(client->sid, mc->regs + client->regs.override);
+
+		override = readl(mc->regs + client->regs.override);
+		security = readl(mc->regs + client->regs.security);
+
+		dev_dbg(&pdev->dev, "client %s: override: %x security: %x\n",
+			client->name, override, security);
+	}
+
+	platform_set_drvdata(pdev, mc);
+
+	return err;
+}
+
+static const struct of_device_id tegra186_mc_of_match[] = {
+	{ .compatible = "nvidia,tegra186-mc", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, tegra186_mc_of_match);
+
+static struct platform_driver tegra186_mc_driver = {
+	.driver = {
+		.name = "tegra186-mc",
+		.of_match_table = tegra186_mc_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.prevent_deferred_probe = true,
+	.probe = tegra186_mc_probe,
+};
+module_platform_driver(tegra186_mc_driver);
+
+MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra186 Memory Controller driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/tegra/tegra210.c b/drivers/memory/tegra/tegra210.c
index 5e144abe4c18..d398bcd3fc57 100644
--- a/drivers/memory/tegra/tegra210.c
+++ b/drivers/memory/tegra/tegra210.c
@@ -1059,11 +1059,26 @@ static const struct tegra_smmu_swgroup tegra210_swgroups[] = {
 	{ .name = "tsecb",     .swgroup = TEGRA_SWGROUP_TSECB,     .reg = 0xad4 },
 };
 
+static const unsigned int tegra210_group_display[] = {
+	TEGRA_SWGROUP_DC,
+	TEGRA_SWGROUP_DCB,
+};
+
+static const struct tegra_smmu_group_soc tegra210_groups[] = {
+	{
+		.name = "display",
+		.swgroups = tegra210_group_display,
+		.num_swgroups = ARRAY_SIZE(tegra210_group_display),
+	},
+};
+
 static const struct tegra_smmu_soc tegra210_smmu_soc = {
 	.clients = tegra210_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra210_mc_clients),
 	.swgroups = tegra210_swgroups,
 	.num_swgroups = ARRAY_SIZE(tegra210_swgroups),
+	.groups = tegra210_groups,
+	.num_groups = ARRAY_SIZE(tegra210_groups),
 	.supports_round_robin_arbitration = true,
 	.supports_request_limit = true,
 	.num_tlb_lines = 32,
diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c
index b44737840e70..d756c837f23e 100644
--- a/drivers/memory/tegra/tegra30.c
+++ b/drivers/memory/tegra/tegra30.c
@@ -934,11 +934,26 @@ static const struct tegra_smmu_swgroup tegra30_swgroups[] = {
 	{ .name = "isp",  .swgroup = TEGRA_SWGROUP_ISP,  .reg = 0x258 },
 };
 
+static const unsigned int tegra30_group_display[] = {
+	TEGRA_SWGROUP_DC,
+	TEGRA_SWGROUP_DCB,
+};
+
+static const struct tegra_smmu_group_soc tegra30_groups[] = {
+	{
+		.name = "display",
+		.swgroups = tegra30_group_display,
+		.num_swgroups = ARRAY_SIZE(tegra30_group_display),
+	},
+};
+
 static const struct tegra_smmu_soc tegra30_smmu_soc = {
 	.clients = tegra30_mc_clients,
 	.num_clients = ARRAY_SIZE(tegra30_mc_clients),
 	.swgroups = tegra30_swgroups,
 	.num_swgroups = ARRAY_SIZE(tegra30_swgroups),
+	.groups = tegra30_groups,
+	.num_groups = ARRAY_SIZE(tegra30_groups),
 	.supports_round_robin_arbitration = false,
 	.supports_request_limit = false,
 	.num_tlb_lines = 16,
diff --git a/drivers/memory/ti-emif-pm.c b/drivers/memory/ti-emif-pm.c
new file mode 100644
index 000000000000..62a86c4bcd0b
--- /dev/null
+++ b/drivers/memory/ti-emif-pm.c
@@ -0,0 +1,324 @@
+/*
+ * TI AM33XX SRAM EMIF Driver
+ *
+ * Copyright (C) 2016-2017 Texas Instruments Inc.
+ *	Dave Gerlach
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/err.h>
+#include <linux/genalloc.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/sram.h>
+#include <linux/ti-emif-sram.h>
+
+#include "emif.h"
+
+#define TI_EMIF_SRAM_SYMBOL_OFFSET(sym) ((unsigned long)(sym) - \
+					 (unsigned long)&ti_emif_sram)
+
+#define EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES		0x00a0
+
+struct ti_emif_data {
+	phys_addr_t ti_emif_sram_phys;
+	phys_addr_t ti_emif_sram_data_phys;
+	unsigned long ti_emif_sram_virt;
+	unsigned long ti_emif_sram_data_virt;
+	struct gen_pool *sram_pool_code;
+	struct gen_pool	*sram_pool_data;
+	struct ti_emif_pm_data pm_data;
+	struct ti_emif_pm_functions pm_functions;
+};
+
+static struct ti_emif_data *emif_instance;
+
+static u32 sram_suspend_address(struct ti_emif_data *emif_data,
+				unsigned long addr)
+{
+	return (emif_data->ti_emif_sram_virt +
+		TI_EMIF_SRAM_SYMBOL_OFFSET(addr));
+}
+
+static phys_addr_t sram_resume_address(struct ti_emif_data *emif_data,
+				       unsigned long addr)
+{
+	return ((unsigned long)emif_data->ti_emif_sram_phys +
+		TI_EMIF_SRAM_SYMBOL_OFFSET(addr));
+}
+
+static void ti_emif_free_sram(struct ti_emif_data *emif_data)
+{
+	gen_pool_free(emif_data->sram_pool_code, emif_data->ti_emif_sram_virt,
+		      ti_emif_sram_sz);
+	gen_pool_free(emif_data->sram_pool_data,
+		      emif_data->ti_emif_sram_data_virt,
+		      sizeof(struct emif_regs_amx3));
+}
+
+static int ti_emif_alloc_sram(struct device *dev,
+			      struct ti_emif_data *emif_data)
+{
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	emif_data->sram_pool_code = of_gen_pool_get(np, "sram", 0);
+	if (!emif_data->sram_pool_code) {
+		dev_err(dev, "Unable to get sram pool for ocmcram code\n");
+		return -ENODEV;
+	}
+
+	emif_data->ti_emif_sram_virt =
+			gen_pool_alloc(emif_data->sram_pool_code,
+				       ti_emif_sram_sz);
+	if (!emif_data->ti_emif_sram_virt) {
+		dev_err(dev, "Unable to allocate code memory from ocmcram\n");
+		return -ENOMEM;
+	}
+
+	/* Save physical address to calculate resume offset during pm init */
+	emif_data->ti_emif_sram_phys =
+			gen_pool_virt_to_phys(emif_data->sram_pool_code,
+					      emif_data->ti_emif_sram_virt);
+
+	/* Get sram pool for data section and allocate space */
+	emif_data->sram_pool_data = of_gen_pool_get(np, "sram", 1);
+	if (!emif_data->sram_pool_data) {
+		dev_err(dev, "Unable to get sram pool for ocmcram data\n");
+		ret = -ENODEV;
+		goto err_free_sram_code;
+	}
+
+	emif_data->ti_emif_sram_data_virt =
+				gen_pool_alloc(emif_data->sram_pool_data,
+					       sizeof(struct emif_regs_amx3));
+	if (!emif_data->ti_emif_sram_data_virt) {
+		dev_err(dev, "Unable to allocate data memory from ocmcram\n");
+		ret = -ENOMEM;
+		goto err_free_sram_code;
+	}
+
+	/* Save physical address to calculate resume offset during pm init */
+	emif_data->ti_emif_sram_data_phys =
+		gen_pool_virt_to_phys(emif_data->sram_pool_data,
+				      emif_data->ti_emif_sram_data_virt);
+	/*
+	 * These functions are called during suspend path while MMU is
+	 * still on so add virtual base to offset for absolute address
+	 */
+	emif_data->pm_functions.save_context =
+		sram_suspend_address(emif_data,
+				     (unsigned long)ti_emif_save_context);
+	emif_data->pm_functions.enter_sr =
+		sram_suspend_address(emif_data,
+				     (unsigned long)ti_emif_enter_sr);
+	emif_data->pm_functions.abort_sr =
+		sram_suspend_address(emif_data,
+				     (unsigned long)ti_emif_abort_sr);
+
+	/*
+	 * These are called during resume path when MMU is not enabled
+	 * so physical address is used instead
+	 */
+	emif_data->pm_functions.restore_context =
+		sram_resume_address(emif_data,
+				    (unsigned long)ti_emif_restore_context);
+	emif_data->pm_functions.exit_sr =
+		sram_resume_address(emif_data,
+				    (unsigned long)ti_emif_exit_sr);
+
+	emif_data->pm_data.regs_virt =
+		(struct emif_regs_amx3 *)emif_data->ti_emif_sram_data_virt;
+	emif_data->pm_data.regs_phys = emif_data->ti_emif_sram_data_phys;
+
+	return 0;
+
+err_free_sram_code:
+	gen_pool_free(emif_data->sram_pool_code, emif_data->ti_emif_sram_virt,
+		      ti_emif_sram_sz);
+	return ret;
+}
+
+static int ti_emif_push_sram(struct device *dev, struct ti_emif_data *emif_data)
+{
+	void *copy_addr;
+	u32 data_addr;
+
+	copy_addr = sram_exec_copy(emif_data->sram_pool_code,
+				   (void *)emif_data->ti_emif_sram_virt,
+				   &ti_emif_sram, ti_emif_sram_sz);
+	if (!copy_addr) {
+		dev_err(dev, "Cannot copy emif code to sram\n");
+		return -ENODEV;
+	}
+
+	data_addr = sram_suspend_address(emif_data,
+					 (unsigned long)&ti_emif_pm_sram_data);
+	copy_addr = sram_exec_copy(emif_data->sram_pool_code,
+				   (void *)data_addr,
+				   &emif_data->pm_data,
+				   sizeof(emif_data->pm_data));
+	if (!copy_addr) {
+		dev_err(dev, "Cannot copy emif data to code sram\n");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+/*
+ * Due to Usage Note 3.1.2 "DDR3: JEDEC Compliance for Maximum
+ * Self-Refresh Command Limit" found in AM335x Silicon Errata
+ * (Document SPRZ360F Revised November 2013) we must configure
+ * the self refresh delay timer to 0xA (8192 cycles) to avoid
+ * generating too many refresh command from the EMIF.
+ */
+static void ti_emif_configure_sr_delay(struct ti_emif_data *emif_data)
+{
+	writel(EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES,
+	       (emif_data->pm_data.ti_emif_base_addr_virt +
+		EMIF_POWER_MANAGEMENT_CONTROL));
+
+	writel(EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES,
+	       (emif_data->pm_data.ti_emif_base_addr_virt +
+		EMIF_POWER_MANAGEMENT_CTRL_SHDW));
+}
+
+/**
+ * ti_emif_copy_pm_function_table - copy mapping of pm funcs in sram
+ * @sram_pool: pointer to struct gen_pool where dst resides
+ * @dst: void * to address that table should be copied
+ *
+ * Returns 0 if success other error code if table is not available
+ */
+int ti_emif_copy_pm_function_table(struct gen_pool *sram_pool, void *dst)
+{
+	void *copy_addr;
+
+	if (!emif_instance)
+		return -ENODEV;
+
+	copy_addr = sram_exec_copy(sram_pool, dst,
+				   &emif_instance->pm_functions,
+				   sizeof(emif_instance->pm_functions));
+	if (!copy_addr)
+		return -ENODEV;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ti_emif_copy_pm_function_table);
+
+/**
+ * ti_emif_get_mem_type - return type for memory type in use
+ *
+ * Returns memory type value read from EMIF or error code if fails
+ */
+int ti_emif_get_mem_type(void)
+{
+	unsigned long temp;
+
+	if (!emif_instance)
+		return -ENODEV;
+
+	temp = readl(emif_instance->pm_data.ti_emif_base_addr_virt +
+		     EMIF_SDRAM_CONFIG);
+
+	temp = (temp & SDRAM_TYPE_MASK) >> SDRAM_TYPE_SHIFT;
+	return temp;
+}
+EXPORT_SYMBOL_GPL(ti_emif_get_mem_type);
+
+static const struct of_device_id ti_emif_of_match[] = {
+	{ .compatible = "ti,emif-am3352", .data =
+					(void *)EMIF_SRAM_AM33_REG_LAYOUT, },
+	{ .compatible = "ti,emif-am4372", .data =
+					(void *)EMIF_SRAM_AM43_REG_LAYOUT, },
+	{},
+};
+MODULE_DEVICE_TABLE(of, ti_emif_of_match);
+
+static int ti_emif_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct resource *res;
+	struct device *dev = &pdev->dev;
+	const struct of_device_id *match;
+	struct ti_emif_data *emif_data;
+
+	emif_data = devm_kzalloc(dev, sizeof(*emif_data), GFP_KERNEL);
+	if (!emif_data)
+		return -ENOMEM;
+
+	match = of_match_device(ti_emif_of_match, &pdev->dev);
+	if (!match)
+		return -ENODEV;
+
+	emif_data->pm_data.ti_emif_sram_config = (unsigned long)match->data;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	emif_data->pm_data.ti_emif_base_addr_virt = devm_ioremap_resource(dev,
+									  res);
+	if (IS_ERR(emif_data->pm_data.ti_emif_base_addr_virt)) {
+		dev_err(dev, "could not ioremap emif mem\n");
+		ret = PTR_ERR(emif_data->pm_data.ti_emif_base_addr_virt);
+		return ret;
+	}
+
+	emif_data->pm_data.ti_emif_base_addr_phys = res->start;
+
+	ti_emif_configure_sr_delay(emif_data);
+
+	ret = ti_emif_alloc_sram(dev, emif_data);
+	if (ret)
+		return ret;
+
+	ret = ti_emif_push_sram(dev, emif_data);
+	if (ret)
+		goto fail_free_sram;
+
+	emif_instance = emif_data;
+
+	return 0;
+
+fail_free_sram:
+	ti_emif_free_sram(emif_data);
+
+	return ret;
+}
+
+static int ti_emif_remove(struct platform_device *pdev)
+{
+	struct ti_emif_data *emif_data = emif_instance;
+
+	emif_instance = NULL;
+
+	ti_emif_free_sram(emif_data);
+
+	return 0;
+}
+
+static struct platform_driver ti_emif_driver = {
+	.probe = ti_emif_probe,
+	.remove = ti_emif_remove,
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = of_match_ptr(ti_emif_of_match),
+	},
+};
+module_platform_driver(ti_emif_driver);
+
+MODULE_AUTHOR("Dave Gerlach <d-gerlach@ti.com>");
+MODULE_DESCRIPTION("Texas Instruments SRAM EMIF driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/memory/ti-emif-sram-pm.S b/drivers/memory/ti-emif-sram-pm.S
new file mode 100644
index 000000000000..a5369181e5c2
--- /dev/null
+++ b/drivers/memory/ti-emif-sram-pm.S
@@ -0,0 +1,334 @@
+/*
+ * Low level PM code for TI EMIF
+ *
+ * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
+ *	Dave Gerlach
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation version 2.
+ *
+ * This program is distributed "as is" WITHOUT ANY WARRANTY of any
+ * kind, whether express or implied; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <generated/ti-emif-asm-offsets.h>
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/memory.h>
+
+#include "emif.h"
+
+#define EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES	0x00a0
+#define EMIF_POWER_MGMT_SR_TIMER_MASK			0x00f0
+#define EMIF_POWER_MGMT_SELF_REFRESH_MODE		0x0200
+#define EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK		0x0700
+
+#define EMIF_SDCFG_TYPE_DDR2				0x2 << SDRAM_TYPE_SHIFT
+#define EMIF_STATUS_READY				0x4
+
+#define AM43XX_EMIF_PHY_CTRL_REG_COUNT                  0x120
+
+#define EMIF_AM437X_REGISTERS				0x1
+
+	.arm
+	.align 3
+
+ENTRY(ti_emif_sram)
+
+/*
+ * void ti_emif_save_context(void)
+ *
+ * Used during suspend to save the context of all required EMIF registers
+ * to local memory if the EMIF is going to lose context during the sleep
+ * transition. Operates on the VIRTUAL address of the EMIF.
+ */
+ENTRY(ti_emif_save_context)
+	stmfd   sp!, {r4 - r11, lr}     @ save registers on stack
+
+	adr	r4, ti_emif_pm_sram_data
+	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
+	ldr	r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
+
+	/* Save EMIF configuration */
+	ldr	r1, [r0, #EMIF_SDRAM_CONFIG]
+	str	r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
+	str	r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_SDRAM_TIMING_1]
+	str     r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_SDRAM_TIMING_2]
+	str     r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_SDRAM_TIMING_3]
+	str     r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
+	str     r1, [r2, #EMIF_PMCR_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
+	str     r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
+	str     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_DDR_PHY_CTRL_1]
+	str     r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
+
+	ldr	r1, [r0, #EMIF_COS_CONFIG]
+	str     r1, [r2, #EMIF_COS_CONFIG_OFFSET]
+
+	ldr	r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
+	str     r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
+
+	ldr	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
+	str     r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
+
+	ldr	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
+	str     r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
+
+	ldr	r1, [r0, #EMIF_OCP_CONFIG]
+	str     r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
+
+	ldr	r5, [r4, #EMIF_PM_CONFIG_OFFSET]
+	cmp	r5, #EMIF_SRAM_AM43_REG_LAYOUT
+	bne	emif_skip_save_extra_regs
+
+	ldr	r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
+	str     r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
+	str     r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
+
+	ldr	r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
+	str     r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
+
+	ldr	r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
+	str     r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
+
+	ldr	r1, [r0, #EMIF_DLL_CALIB_CTRL]
+	str     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
+
+	ldr	r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
+	str     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
+
+	/* Loop and save entire block of emif phy regs */
+	mov	r5, #0x0
+	add	r4, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
+	add	r3, r0, #EMIF_EXT_PHY_CTRL_1
+ddr_phy_ctrl_save:
+	ldr	r1, [r3, r5]
+	str	r1, [r4, r5]
+	add	r5, r5, #0x4
+	cmp	r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
+	bne	ddr_phy_ctrl_save
+
+emif_skip_save_extra_regs:
+	ldmfd	sp!, {r4 - r11, pc}	@ restore regs and return
+ENDPROC(ti_emif_save_context)
+
+/*
+ * void ti_emif_restore_context(void)
+ *
+ * Used during resume to restore the context of all required EMIF registers
+ * from local memory after the EMIF has lost context during a sleep transition.
+ * Operates on the PHYSICAL address of the EMIF.
+ */
+ENTRY(ti_emif_restore_context)
+	adr	r4, ti_emif_pm_sram_data
+	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
+	ldr	r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]
+
+	/* Config EMIF Timings */
+	ldr     r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
+	str	r1, [r0, #EMIF_DDR_PHY_CTRL_1]
+	str	r1, [r0, #EMIF_DDR_PHY_CTRL_1_SHDW]
+
+	ldr     r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
+	str	r1, [r0, #EMIF_SDRAM_TIMING_1]
+	str	r1, [r0, #EMIF_SDRAM_TIMING_1_SHDW]
+
+	ldr     r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
+	str	r1, [r0, #EMIF_SDRAM_TIMING_2]
+	str	r1, [r0, #EMIF_SDRAM_TIMING_2_SHDW]
+
+	ldr     r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
+	str	r1, [r0, #EMIF_SDRAM_TIMING_3]
+	str	r1, [r0, #EMIF_SDRAM_TIMING_3_SHDW]
+
+	ldr     r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
+	str	r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
+	str	r1, [r0, #EMIF_SDRAM_REFRESH_CTRL_SHDW]
+
+	ldr     r1, [r2, #EMIF_PMCR_VAL_OFFSET]
+	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
+
+	ldr     r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
+	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
+
+	ldr     r1, [r2, #EMIF_COS_CONFIG_OFFSET]
+	str	r1, [r0, #EMIF_COS_CONFIG]
+
+	ldr     r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
+	str	r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
+
+	ldr	r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
+	str	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
+
+	ldr     r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
+	str	r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
+
+	ldr     r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
+	str	r1, [r0, #EMIF_OCP_CONFIG]
+
+	ldr	r5, [r4, #EMIF_PM_CONFIG_OFFSET]
+	cmp	r5, #EMIF_SRAM_AM43_REG_LAYOUT
+	bne	emif_skip_restore_extra_regs
+
+	ldr     r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
+	str	r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
+
+	ldr     r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
+	str	r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
+
+	ldr     r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
+	str	r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
+
+	ldr     r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
+	str	r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
+
+	ldr     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
+	str	r1, [r0, #EMIF_DLL_CALIB_CTRL]
+
+	ldr     r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
+	str	r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
+
+	ldr     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
+	str	r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
+
+	/* Loop and restore entire block of emif phy regs */
+	mov	r5, #0x0
+	/* Load ti_emif_regs_amx3 + EMIF_EXT_PHY_CTRL_VALS_OFFSET for address
+	 * to phy register save space
+	 */
+	add	r3, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
+	add	r4, r0, #EMIF_EXT_PHY_CTRL_1
+ddr_phy_ctrl_restore:
+	ldr	r1, [r3, r5]
+	str	r1, [r4, r5]
+	add	r5, r5, #0x4
+	cmp	r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
+	bne	ddr_phy_ctrl_restore
+
+emif_skip_restore_extra_regs:
+	/*
+	 * Output impedence calib needed only for DDR3
+	 * but since the initial state of this will be
+	 * disabled for DDR2 no harm in restoring the
+	 * old configuration
+	 */
+	ldr     r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
+	str	r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
+
+	/* Write to sdcfg last for DDR2 only */
+	ldr	r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
+	and	r2, r1, #SDRAM_TYPE_MASK
+	cmp	r2, #EMIF_SDCFG_TYPE_DDR2
+	streq	r1, [r0, #EMIF_SDRAM_CONFIG]
+
+	mov	pc, lr
+ENDPROC(ti_emif_restore_context)
+
+/*
+ * void ti_emif_enter_sr(void)
+ *
+ * Programs the EMIF to tell the SDRAM to enter into self-refresh
+ * mode during a sleep transition. Operates on the VIRTUAL address
+ * of the EMIF.
+ */
+ENTRY(ti_emif_enter_sr)
+	stmfd   sp!, {r4 - r11, lr}     @ save registers on stack
+
+	adr	r4, ti_emif_pm_sram_data
+	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
+	ldr	r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
+
+	ldr	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
+	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
+	orr	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
+	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
+
+	ldmfd	sp!, {r4 - r11, pc}	@ restore regs and return
+ENDPROC(ti_emif_enter_sr)
+
+/*
+ * void ti_emif_exit_sr(void)
+ *
+ * Programs the EMIF to tell the SDRAM to exit self-refresh mode
+ * after a sleep transition. Operates on the PHYSICAL address of
+ * the EMIF.
+ */
+ENTRY(ti_emif_exit_sr)
+	adr	r4, ti_emif_pm_sram_data
+	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
+	ldr	r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]
+
+	/*
+	 * Toggle EMIF to exit refresh mode:
+	 * if EMIF lost context, PWR_MGT_CTRL is currently 0, writing disable
+	 *   (0x0), wont do diddly squat! so do a toggle from SR(0x2) to disable
+	 *   (0x0) here.
+	 * *If* EMIF did not lose context, nothing broken as we write the same
+	 *   value(0x2) to reg before we write a disable (0x0).
+	 */
+	ldr	r1, [r2, #EMIF_PMCR_VAL_OFFSET]
+	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
+	orr	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
+	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
+	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
+	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
+
+        /* Wait for EMIF to become ready */
+1:	ldr     r1, [r0, #EMIF_STATUS]
+	tst     r1, #EMIF_STATUS_READY
+	beq     1b
+
+	mov	pc, lr
+ENDPROC(ti_emif_exit_sr)
+
+/*
+ * void ti_emif_abort_sr(void)
+ *
+ * Disables self-refresh after a failed transition to a low-power
+ * state so the kernel can jump back to DDR and follow abort path.
+ * Operates on the VIRTUAL address of the EMIF.
+ */
+ENTRY(ti_emif_abort_sr)
+	stmfd   sp!, {r4 - r11, lr}     @ save registers on stack
+
+	adr	r4, ti_emif_pm_sram_data
+	ldr	r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
+	ldr	r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
+
+	ldr	r1, [r2, #EMIF_PMCR_VAL_OFFSET]
+	bic	r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
+	str	r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
+
+	/* Wait for EMIF to become ready */
+1:	ldr     r1, [r0, #EMIF_STATUS]
+	tst     r1, #EMIF_STATUS_READY
+	beq     1b
+
+	ldmfd	sp!, {r4 - r11, pc}	@ restore regs and return
+ENDPROC(ti_emif_abort_sr)
+
+	.align 3
+ENTRY(ti_emif_pm_sram_data)
+	.space EMIF_PM_DATA_SIZE
+ENTRY(ti_emif_sram_sz)
+        .word   . - ti_emif_save_context
diff --git a/drivers/memstick/host/Kconfig b/drivers/memstick/host/Kconfig
index 7310e32b5991..aa2b0786bbe9 100644
--- a/drivers/memstick/host/Kconfig
+++ b/drivers/memstick/host/Kconfig
@@ -45,7 +45,7 @@ config MEMSTICK_R592
 
 config MEMSTICK_REALTEK_PCI
 	tristate "Realtek PCI-E Memstick Card Interface Driver"
-	depends on MFD_RTSX_PCI
+	depends on MISC_RTSX_PCI
 	help
 	  Say Y here to include driver code to support Memstick card interface
 	  of Realtek PCI-E card reader
@@ -55,7 +55,7 @@ config MEMSTICK_REALTEK_PCI
 
 config MEMSTICK_REALTEK_USB
 	tristate "Realtek USB Memstick Card Interface Driver"
-	depends on MFD_RTSX_USB
+	depends on MISC_RTSX_USB
 	help
 	  Say Y here to include driver code to support Memstick card interface
 	  of Realtek RTS5129/39 series USB card reader
diff --git a/drivers/memstick/host/rtsx_pci_ms.c b/drivers/memstick/host/rtsx_pci_ms.c
index 818fa94354ae..a44b4578ba4d 100644
--- a/drivers/memstick/host/rtsx_pci_ms.c
+++ b/drivers/memstick/host/rtsx_pci_ms.c
@@ -24,7 +24,7 @@
 #include <linux/delay.h>
 #include <linux/platform_device.h>
 #include <linux/memstick.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 #include <asm/unaligned.h>
 
 struct realtek_pci_ms {
diff --git a/drivers/memstick/host/rtsx_usb_ms.c b/drivers/memstick/host/rtsx_usb_ms.c
index 2e3cf012ef48..4f64563df7de 100644
--- a/drivers/memstick/host/rtsx_usb_ms.c
+++ b/drivers/memstick/host/rtsx_usb_ms.c
@@ -25,7 +25,7 @@
 #include <linux/workqueue.h>
 #include <linux/memstick.h>
 #include <linux/kthread.h>
-#include <linux/mfd/rtsx_usb.h>
+#include <linux/rtsx_usb.h>
 #include <linux/pm_runtime.h>
 #include <linux/mutex.h>
 #include <linux/sched.h>
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index 7a93400eea2a..51eb1b027963 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -958,7 +958,7 @@ mpt_put_msg_frame(u8 cb_idx, MPT_ADAPTER *ioc, MPT_FRAME_HDR *mf)
 {
 	u32 mf_dma_addr;
 	int req_offset;
-	u16	 req_idx;	/* Request index */
+	u16 req_idx;	/* Request index */
 
 	/* ensure values are reset properly! */
 	mf->u.frame.hwhdr.msgctxu.fld.cb_idx = cb_idx;		/* byte */
@@ -994,7 +994,7 @@ mpt_put_msg_frame_hi_pri(u8 cb_idx, MPT_ADAPTER *ioc, MPT_FRAME_HDR *mf)
 {
 	u32 mf_dma_addr;
 	int req_offset;
-	u16	 req_idx;	/* Request index */
+	u16 req_idx;	/* Request index */
 
 	/* ensure values are reset properly! */
 	mf->u.frame.hwhdr.msgctxu.fld.cb_idx = cb_idx;
@@ -1128,11 +1128,12 @@ mpt_add_sge_64bit_1078(void *pAddr, u32 flagslength, dma_addr_t dma_addr)
 static void
 mpt_add_chain(void *pAddr, u8 next, u16 length, dma_addr_t dma_addr)
 {
-		SGEChain32_t *pChain = (SGEChain32_t *) pAddr;
-		pChain->Length = cpu_to_le16(length);
-		pChain->Flags = MPI_SGE_FLAGS_CHAIN_ELEMENT;
-		pChain->NextChainOffset = next;
-		pChain->Address = cpu_to_le32(dma_addr);
+	SGEChain32_t *pChain = (SGEChain32_t *) pAddr;
+
+	pChain->Length = cpu_to_le16(length);
+	pChain->Flags = MPI_SGE_FLAGS_CHAIN_ELEMENT;
+	pChain->NextChainOffset = next;
+	pChain->Address = cpu_to_le32(dma_addr);
 }
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
@@ -1147,18 +1148,18 @@ mpt_add_chain(void *pAddr, u8 next, u16 length, dma_addr_t dma_addr)
 static void
 mpt_add_chain_64bit(void *pAddr, u8 next, u16 length, dma_addr_t dma_addr)
 {
-		SGEChain64_t *pChain = (SGEChain64_t *) pAddr;
-		u32 tmp = dma_addr & 0xFFFFFFFF;
+	SGEChain64_t *pChain = (SGEChain64_t *) pAddr;
+	u32 tmp = dma_addr & 0xFFFFFFFF;
 
-		pChain->Length = cpu_to_le16(length);
-		pChain->Flags = (MPI_SGE_FLAGS_CHAIN_ELEMENT |
-				 MPI_SGE_FLAGS_64_BIT_ADDRESSING);
+	pChain->Length = cpu_to_le16(length);
+	pChain->Flags = (MPI_SGE_FLAGS_CHAIN_ELEMENT |
+			 MPI_SGE_FLAGS_64_BIT_ADDRESSING);
 
-		pChain->NextChainOffset = next;
+	pChain->NextChainOffset = next;
 
-		pChain->Address.Low = cpu_to_le32(tmp);
-		tmp = (u32)(upper_32_bits(dma_addr));
-		pChain->Address.High = cpu_to_le32(tmp);
+	pChain->Address.Low = cpu_to_le32(tmp);
+	tmp = (u32)(upper_32_bits(dma_addr));
+	pChain->Address.High = cpu_to_le32(tmp);
 }
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
@@ -1360,7 +1361,7 @@ mpt_host_page_alloc(MPT_ADAPTER *ioc, pIOCInit_t ioc_init)
 	ioc->add_sge(psge, flags_length, ioc->HostPageBuffer_dma);
 	ioc->facts.HostPageBufferSGE = ioc_init->HostPageBufferSGE;
 
-return 0;
+	return 0;
 }
 
 /*=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=*/
@@ -2152,7 +2153,7 @@ mpt_suspend(struct pci_dev *pdev, pm_message_t state)
 	    device_state);
 
 	/* put ioc into READY_STATE */
-	if(SendIocReset(ioc, MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET, CAN_SLEEP)) {
+	if (SendIocReset(ioc, MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET, CAN_SLEEP)) {
 		printk(MYIOC_s_ERR_FMT
 		"pci-suspend:  IOC msg unit reset failed!\n", ioc->name);
 	}
@@ -6348,7 +6349,7 @@ mpt_config(MPT_ADAPTER *ioc, CONFIGPARMS *pCfg)
 	u8		 page_type = 0, extend_page;
 	unsigned long 	 timeleft;
 	unsigned long	 flags;
-    int		 in_isr;
+	int		 in_isr;
 	u8		 issue_hard_reset = 0;
 	u8		 retry_count = 0;
 
@@ -7697,7 +7698,7 @@ mpt_display_event_info(MPT_ADAPTER *ioc, EventNotificationReply_t *pEventReply)
 		break;
 	}
 	if (ds)
-		strncpy(evStr, ds, EVENT_DESCR_STR_SZ);
+		strlcpy(evStr, ds, EVENT_DESCR_STR_SZ);
 
 
 	devtprintk(ioc, printk(MYIOC_s_DEBUG_FMT
@@ -8092,15 +8093,15 @@ mpt_spi_log_info(MPT_ADAPTER *ioc, u32 log_info)
 static void
 mpt_sas_log_info(MPT_ADAPTER *ioc, u32 log_info, u8 cb_idx)
 {
-union loginfo_type {
-	u32	loginfo;
-	struct {
-		u32	subcode:16;
-		u32	code:8;
-		u32	originator:4;
-		u32	bus_type:4;
-	}dw;
-};
+	union loginfo_type {
+		u32	loginfo;
+		struct {
+			u32	subcode:16;
+			u32	code:8;
+			u32	originator:4;
+			u32	bus_type:4;
+		} dw;
+	};
 	union loginfo_type sas_loginfo;
 	char *originator_desc = NULL;
 	char *code_desc = NULL;
diff --git a/drivers/message/fusion/mptctl.c b/drivers/message/fusion/mptctl.c
index 7b3b41368931..4470630dd545 100644
--- a/drivers/message/fusion/mptctl.c
+++ b/drivers/message/fusion/mptctl.c
@@ -2481,24 +2481,13 @@ mptctl_hp_hostinfo(unsigned long arg, unsigned int data_size)
 	else
 		karg.host_no =  -1;
 
-	/* Reformat the fw_version into a string
-	 */
-	karg.fw_version[0] = ioc->facts.FWVersion.Struct.Major >= 10 ?
-		((ioc->facts.FWVersion.Struct.Major / 10) + '0') : '0';
-	karg.fw_version[1] = (ioc->facts.FWVersion.Struct.Major % 10 ) + '0';
-	karg.fw_version[2] = '.';
-	karg.fw_version[3] = ioc->facts.FWVersion.Struct.Minor >= 10 ?
-		((ioc->facts.FWVersion.Struct.Minor / 10) + '0') : '0';
-	karg.fw_version[4] = (ioc->facts.FWVersion.Struct.Minor % 10 ) + '0';
-	karg.fw_version[5] = '.';
-	karg.fw_version[6] = ioc->facts.FWVersion.Struct.Unit >= 10 ?
-		((ioc->facts.FWVersion.Struct.Unit / 10) + '0') : '0';
-	karg.fw_version[7] = (ioc->facts.FWVersion.Struct.Unit % 10 ) + '0';
-	karg.fw_version[8] = '.';
-	karg.fw_version[9] = ioc->facts.FWVersion.Struct.Dev >= 10 ?
-		((ioc->facts.FWVersion.Struct.Dev / 10) + '0') : '0';
-	karg.fw_version[10] = (ioc->facts.FWVersion.Struct.Dev % 10 ) + '0';
-	karg.fw_version[11] = '\0';
+	/* Reformat the fw_version into a string */
+	snprintf(karg.fw_version, sizeof(karg.fw_version),
+		 "%.2hhu.%.2hhu.%.2hhu.%.2hhu",
+		 ioc->facts.FWVersion.Struct.Major,
+		 ioc->facts.FWVersion.Struct.Minor,
+		 ioc->facts.FWVersion.Struct.Unit,
+		 ioc->facts.FWVersion.Struct.Dev);
 
 	/* Issue a config request to get the device serial number
 	 */
@@ -2698,6 +2687,8 @@ mptctl_hp_targetinfo(unsigned long arg)
 				__FILE__, __LINE__, iocnum);
 		return -ENODEV;
 	}
+	if (karg.hdr.id >= MPT_MAX_FC_DEVICES)
+		return -EINVAL;
 	dctlprintk(ioc, printk(MYIOC_s_DEBUG_FMT "mptctl_hp_targetinfo called.\n",
 	    ioc->name));
 
diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c
index 345f6035599e..439ee9c5f535 100644
--- a/drivers/message/fusion/mptsas.c
+++ b/drivers/message/fusion/mptsas.c
@@ -1165,7 +1165,6 @@ mptsas_schedule_target_reset(void *iocp)
 	 * issue target reset to next device in the queue
 	 */
 
-	head = &hd->target_reset_list;
 	if (list_empty(head))
 		return;
 
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index 1d20a800e967..b860eb5aa194 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -222,6 +222,16 @@ config MFD_CROS_EC_SPI
 	  response time cannot be guaranteed, we support ignoring
 	  'pre-amble' bytes before the response actually starts.
 
+config MFD_CROS_EC_CHARDEV
+        tristate "Chrome OS Embedded Controller userspace device interface"
+        depends on MFD_CROS_EC
+        select CROS_EC_CTL
+        ---help---
+          This driver adds support to talk with the ChromeOS EC from userspace.
+
+          If you have a supported Chromebook, choose Y or M here.
+          The module will be called cros_ec_dev.
+
 config MFD_ASIC3
 	bool "Compaq ASIC3"
 	depends on GPIOLIB && ARM
@@ -877,7 +887,7 @@ config UCB1400_CORE
 
 config MFD_PM8XXX
 	tristate "Qualcomm PM8xxx PMIC chips driver"
-	depends on (ARM || HEXAGON)
+	depends on (ARM || HEXAGON || COMPILE_TEST)
 	select IRQ_DOMAIN
 	select MFD_CORE
 	select REGMAP
@@ -929,17 +939,6 @@ config MFD_RDC321X
 	  southbridge which provides access to GPIOs and Watchdog using the
 	  southbridge PCI device configuration space.
 
-config MFD_RTSX_PCI
-	tristate "Realtek PCI-E card reader"
-	depends on PCI
-	select MFD_CORE
-	help
-	  This supports for Realtek PCI-Express card reader including rts5209,
-	  rts5227, rts522A, rts5229, rts5249, rts524A, rts525A, rtl8411, etc.
-	  Realtek card reader supports access to many types of memory cards,
-	  such as Memory Stick, Memory Stick Pro, Secure Digital and
-	  MultiMediaCard.
-
 config MFD_RT5033
 	tristate "Richtek RT5033 Power Management IC"
 	depends on I2C
@@ -953,16 +952,6 @@ config MFD_RT5033
 	  sub-devices like charger, fuel gauge, flash LED, current source,
 	  LDO and Buck.
 
-config MFD_RTSX_USB
-	tristate "Realtek USB card reader"
-	depends on USB
-	select MFD_CORE
-	help
-	  Select this option to get support for Realtek USB 2.0 card readers
-	  including RTS5129, RTS5139, RTS5179 and RTS5170.
-	  Realtek card reader supports access to many types of memory cards,
-	  such as Memory Stick Pro, Secure Digital and MultiMediaCard.
-
 config MFD_RC5T583
 	bool "Ricoh RC5T583 Power Management system device"
 	depends on I2C=y
@@ -1859,5 +1848,13 @@ config MFD_VEXPRESS_SYSREG
 	  System Registers are the platform configuration block
 	  on the ARM Ltd. Versatile Express board.
 
+config RAVE_SP_CORE
+	tristate "RAVE SP MCU core driver"
+	depends on SERIAL_DEV_BUS
+	select CRC_CCITT
+	help
+	  Select this to get support for the Supervisory Processor
+	  device found on several devices in RAVE line of hardware.
+
 endmenu
 endif
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index d9474ade32e6..d9d2cf0d32ef 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -17,12 +17,9 @@ cros_ec_core-$(CONFIG_ACPI)	+= cros_ec_acpi_gpe.o
 obj-$(CONFIG_MFD_CROS_EC)	+= cros_ec_core.o
 obj-$(CONFIG_MFD_CROS_EC_I2C)	+= cros_ec_i2c.o
 obj-$(CONFIG_MFD_CROS_EC_SPI)	+= cros_ec_spi.o
+obj-$(CONFIG_MFD_CROS_EC_CHARDEV) += cros_ec_dev.o
 obj-$(CONFIG_MFD_EXYNOS_LPASS)	+= exynos-lpass.o
 
-rtsx_pci-objs			:= rtsx_pcr.o rts5209.o rts5229.o rtl8411.o rts5227.o rts5249.o
-obj-$(CONFIG_MFD_RTSX_PCI)	+= rtsx_pci.o
-obj-$(CONFIG_MFD_RTSX_USB)	+= rtsx_usb.o
-
 obj-$(CONFIG_HTC_PASIC3)	+= htc-pasic3.o
 obj-$(CONFIG_HTC_I2CPLD)	+= htc-i2cpld.o
 
@@ -230,3 +227,5 @@ obj-$(CONFIG_MFD_STM32_LPTIMER)	+= stm32-lptimer.o
 obj-$(CONFIG_MFD_STM32_TIMERS) 	+= stm32-timers.o
 obj-$(CONFIG_MFD_MXS_LRADC)     += mxs-lradc.o
 obj-$(CONFIG_MFD_SC27XX_PMIC)	+= sprd-sc27xx-spi.o
+obj-$(CONFIG_RAVE_SP_CORE)	+= rave-sp.o
+
diff --git a/drivers/mfd/ab8500-debugfs.c b/drivers/mfd/ab8500-debugfs.c
index c1c815241e02..8ba41073dd89 100644
--- a/drivers/mfd/ab8500-debugfs.c
+++ b/drivers/mfd/ab8500-debugfs.c
@@ -1258,7 +1258,6 @@ static struct ab8500_prcmu_ranges ab8540_debug_ranges[AB8500_NUM_BANKS] = {
 	},
 };
 
-
 static irqreturn_t ab8500_debug_handler(int irq, void *data)
 {
 	char buf[16];
@@ -1268,7 +1267,7 @@ static irqreturn_t ab8500_debug_handler(int irq, void *data)
 	if (irq_abb < num_irqs)
 		irq_count[irq_abb]++;
 	/*
-	 * This makes it possible to use poll for events (POLLPRI | POLLERR)
+	 * This makes it possible to use poll for events (EPOLLPRI | EPOLLERR)
 	 * from userspace on sysfs file named <irq-nr>
 	 */
 	sprintf(buf, "%d", irq);
@@ -1318,7 +1317,7 @@ static int ab8500_registers_print(struct device *dev, u32 bank,
 	return 0;
 }
 
-static int ab8500_print_bank_registers(struct seq_file *s, void *p)
+static int ab8500_bank_registers_show(struct seq_file *s, void *p)
 {
 	struct device *dev = s->private;
 	u32 bank = debug_bank;
@@ -1330,18 +1329,7 @@ static int ab8500_print_bank_registers(struct seq_file *s, void *p)
 	return ab8500_registers_print(dev, bank, s);
 }
 
-static int ab8500_registers_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_print_bank_registers, inode->i_private);
-}
-
-static const struct file_operations ab8500_registers_fops = {
-	.open = ab8500_registers_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_bank_registers);
 
 static int ab8500_print_all_banks(struct seq_file *s, void *p)
 {
@@ -1528,7 +1516,7 @@ void ab8500_debug_register_interrupt(int line)
 		num_interrupts[line]++;
 }
 
-static int ab8500_interrupts_print(struct seq_file *s, void *p)
+static int ab8500_interrupts_show(struct seq_file *s, void *p)
 {
 	int line;
 
@@ -1557,10 +1545,7 @@ static int ab8500_interrupts_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_interrupts_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_interrupts_print, inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8500_interrupts);
 
 /*
  * - HWREG DB8500 formated routines
@@ -1603,7 +1588,7 @@ static int ab8500_hwreg_open(struct inode *inode, struct file *file)
 #define AB8500_LAST_SIM_REG 0x8B
 #define AB8505_LAST_SIM_REG 0x8C
 
-static int ab8500_print_modem_registers(struct seq_file *s, void *p)
+static int ab8500_modem_show(struct seq_file *s, void *p)
 {
 	struct device *dev = s->private;
 	struct ab8500 *ab8500;
@@ -1620,18 +1605,15 @@ static int ab8500_print_modem_registers(struct seq_file *s, void *p)
 
 	err = abx500_get_register_interruptible(dev,
 		AB8500_REGU_CTRL1, AB8500_SUPPLY_CONTROL_REG, &orig_value);
-	if (err < 0) {
-		dev_err(dev, "ab->read fail %d\n", err);
-		return err;
-	}
+	if (err < 0)
+		goto report_read_failure;
+
 	/* Config 1 will allow APE side to read SIM registers */
 	err = abx500_set_register_interruptible(dev,
 		AB8500_REGU_CTRL1, AB8500_SUPPLY_CONTROL_REG,
 		AB8500_SUPPLY_CONTROL_CONFIG_1);
-	if (err < 0) {
-		dev_err(dev, "ab->write fail %d\n", err);
-		return err;
-	}
+	if (err < 0)
+		goto report_write_failure;
 
 	seq_printf(s, " bank 0x%02X:\n", bank);
 
@@ -1641,36 +1623,30 @@ static int ab8500_print_modem_registers(struct seq_file *s, void *p)
 	for (reg = AB8500_FIRST_SIM_REG; reg <= last_sim_reg; reg++) {
 		err = abx500_get_register_interruptible(dev,
 			bank, reg, &value);
-		if (err < 0) {
-			dev_err(dev, "ab->read fail %d\n", err);
-			return err;
-		}
+		if (err < 0)
+			goto report_read_failure;
+
 		seq_printf(s, "  [0x%02X/0x%02X]: 0x%02X\n", bank, reg, value);
 	}
 	err = abx500_set_register_interruptible(dev,
 		AB8500_REGU_CTRL1, AB8500_SUPPLY_CONTROL_REG, orig_value);
-	if (err < 0) {
-		dev_err(dev, "ab->write fail %d\n", err);
-		return err;
-	}
+	if (err < 0)
+		goto report_write_failure;
+
 	return 0;
-}
 
-static int ab8500_modem_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_print_modem_registers,
-			   inode->i_private);
+report_read_failure:
+	dev_err(dev, "ab->read fail %d\n", err);
+	return err;
+
+report_write_failure:
+	dev_err(dev, "ab->write fail %d\n", err);
+	return err;
 }
 
-static const struct file_operations ab8500_modem_fops = {
-	.open = ab8500_modem_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_modem);
 
-static int ab8500_gpadc_bat_ctrl_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_bat_ctrl_show(struct seq_file *s, void *p)
 {
 	int bat_ctrl_raw;
 	int bat_ctrl_convert;
@@ -1687,21 +1663,9 @@ static int ab8500_gpadc_bat_ctrl_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_bat_ctrl_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_gpadc_bat_ctrl_print,
-			   inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_bat_ctrl);
 
-static const struct file_operations ab8500_gpadc_bat_ctrl_fops = {
-	.open = ab8500_gpadc_bat_ctrl_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
-static int ab8500_gpadc_btemp_ball_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_btemp_ball_show(struct seq_file *s, void *p)
 {
 	int btemp_ball_raw;
 	int btemp_ball_convert;
@@ -1718,22 +1682,9 @@ static int ab8500_gpadc_btemp_ball_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_btemp_ball_open(struct inode *inode,
-					struct file *file)
-{
-	return single_open(file, ab8500_gpadc_btemp_ball_print,
-			   inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_btemp_ball);
 
-static const struct file_operations ab8500_gpadc_btemp_ball_fops = {
-	.open = ab8500_gpadc_btemp_ball_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
-static int ab8500_gpadc_main_charger_v_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_main_charger_v_show(struct seq_file *s, void *p)
 {
 	int main_charger_v_raw;
 	int main_charger_v_convert;
@@ -1750,22 +1701,9 @@ static int ab8500_gpadc_main_charger_v_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_main_charger_v_open(struct inode *inode,
-					    struct file *file)
-{
-	return single_open(file, ab8500_gpadc_main_charger_v_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_main_charger_v_fops = {
-	.open = ab8500_gpadc_main_charger_v_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_main_charger_v);
 
-static int ab8500_gpadc_acc_detect1_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_acc_detect1_show(struct seq_file *s, void *p)
 {
 	int acc_detect1_raw;
 	int acc_detect1_convert;
@@ -1782,22 +1720,9 @@ static int ab8500_gpadc_acc_detect1_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_acc_detect1_open(struct inode *inode,
-					 struct file *file)
-{
-	return single_open(file, ab8500_gpadc_acc_detect1_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_acc_detect1_fops = {
-	.open = ab8500_gpadc_acc_detect1_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_acc_detect1);
 
-static int ab8500_gpadc_acc_detect2_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_acc_detect2_show(struct seq_file *s, void *p)
 {
 	int acc_detect2_raw;
 	int acc_detect2_convert;
@@ -1814,22 +1739,9 @@ static int ab8500_gpadc_acc_detect2_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_acc_detect2_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8500_gpadc_acc_detect2_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_acc_detect2_fops = {
-	.open = ab8500_gpadc_acc_detect2_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_acc_detect2);
 
-static int ab8500_gpadc_aux1_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_aux1_show(struct seq_file *s, void *p)
 {
 	int aux1_raw;
 	int aux1_convert;
@@ -1846,20 +1758,9 @@ static int ab8500_gpadc_aux1_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_aux1_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_gpadc_aux1_print, inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_aux1);
 
-static const struct file_operations ab8500_gpadc_aux1_fops = {
-	.open = ab8500_gpadc_aux1_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
-static int ab8500_gpadc_aux2_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_aux2_show(struct seq_file *s, void *p)
 {
 	int aux2_raw;
 	int aux2_convert;
@@ -1876,20 +1777,9 @@ static int ab8500_gpadc_aux2_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_aux2_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_gpadc_aux2_print, inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_aux2_fops = {
-	.open = ab8500_gpadc_aux2_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_aux2);
 
-static int ab8500_gpadc_main_bat_v_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_main_bat_v_show(struct seq_file *s, void *p)
 {
 	int main_bat_v_raw;
 	int main_bat_v_convert;
@@ -1906,22 +1796,9 @@ static int ab8500_gpadc_main_bat_v_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_main_bat_v_open(struct inode *inode,
-					struct file *file)
-{
-	return single_open(file, ab8500_gpadc_main_bat_v_print,
-			   inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_main_bat_v);
 
-static const struct file_operations ab8500_gpadc_main_bat_v_fops = {
-	.open = ab8500_gpadc_main_bat_v_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
-static int ab8500_gpadc_vbus_v_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_vbus_v_show(struct seq_file *s, void *p)
 {
 	int vbus_v_raw;
 	int vbus_v_convert;
@@ -1938,20 +1815,9 @@ static int ab8500_gpadc_vbus_v_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_vbus_v_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_gpadc_vbus_v_print, inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_vbus_v_fops = {
-	.open = ab8500_gpadc_vbus_v_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_vbus_v);
 
-static int ab8500_gpadc_main_charger_c_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_main_charger_c_show(struct seq_file *s, void *p)
 {
 	int main_charger_c_raw;
 	int main_charger_c_convert;
@@ -1968,22 +1834,9 @@ static int ab8500_gpadc_main_charger_c_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_main_charger_c_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8500_gpadc_main_charger_c_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_main_charger_c_fops = {
-	.open = ab8500_gpadc_main_charger_c_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_main_charger_c);
 
-static int ab8500_gpadc_usb_charger_c_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_usb_charger_c_show(struct seq_file *s, void *p)
 {
 	int usb_charger_c_raw;
 	int usb_charger_c_convert;
@@ -2000,22 +1853,9 @@ static int ab8500_gpadc_usb_charger_c_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_usb_charger_c_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8500_gpadc_usb_charger_c_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_usb_charger_c_fops = {
-	.open = ab8500_gpadc_usb_charger_c_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_usb_charger_c);
 
-static int ab8500_gpadc_bk_bat_v_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_bk_bat_v_show(struct seq_file *s, void *p)
 {
 	int bk_bat_v_raw;
 	int bk_bat_v_convert;
@@ -2032,21 +1872,9 @@ static int ab8500_gpadc_bk_bat_v_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_bk_bat_v_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_gpadc_bk_bat_v_print,
-			   inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_bk_bat_v_fops = {
-	.open = ab8500_gpadc_bk_bat_v_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_bk_bat_v);
 
-static int ab8500_gpadc_die_temp_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_die_temp_show(struct seq_file *s, void *p)
 {
 	int die_temp_raw;
 	int die_temp_convert;
@@ -2063,21 +1891,9 @@ static int ab8500_gpadc_die_temp_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_die_temp_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_gpadc_die_temp_print,
-			   inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_die_temp);
 
-static const struct file_operations ab8500_gpadc_die_temp_fops = {
-	.open = ab8500_gpadc_die_temp_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
-static int ab8500_gpadc_usb_id_print(struct seq_file *s, void *p)
+static int ab8500_gpadc_usb_id_show(struct seq_file *s, void *p)
 {
 	int usb_id_raw;
 	int usb_id_convert;
@@ -2094,20 +1910,9 @@ static int ab8500_gpadc_usb_id_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8500_gpadc_usb_id_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8500_gpadc_usb_id_print, inode->i_private);
-}
-
-static const struct file_operations ab8500_gpadc_usb_id_fops = {
-	.open = ab8500_gpadc_usb_id_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8500_gpadc_usb_id);
 
-static int ab8540_gpadc_xtal_temp_print(struct seq_file *s, void *p)
+static int ab8540_gpadc_xtal_temp_show(struct seq_file *s, void *p)
 {
 	int xtal_temp_raw;
 	int xtal_temp_convert;
@@ -2124,21 +1929,9 @@ static int ab8540_gpadc_xtal_temp_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8540_gpadc_xtal_temp_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8540_gpadc_xtal_temp_print,
-		inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8540_gpadc_xtal_temp);
 
-static const struct file_operations ab8540_gpadc_xtal_temp_fops = {
-	.open = ab8540_gpadc_xtal_temp_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
-static int ab8540_gpadc_vbat_true_meas_print(struct seq_file *s, void *p)
+static int ab8540_gpadc_vbat_true_meas_show(struct seq_file *s, void *p)
 {
 	int vbat_true_meas_raw;
 	int vbat_true_meas_convert;
@@ -2156,22 +1949,9 @@ static int ab8540_gpadc_vbat_true_meas_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8540_gpadc_vbat_true_meas_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8540_gpadc_vbat_true_meas_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8540_gpadc_vbat_true_meas_fops = {
-	.open = ab8540_gpadc_vbat_true_meas_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8540_gpadc_vbat_true_meas);
 
-static int ab8540_gpadc_bat_ctrl_and_ibat_print(struct seq_file *s, void *p)
+static int ab8540_gpadc_bat_ctrl_and_ibat_show(struct seq_file *s, void *p)
 {
 	int bat_ctrl_raw;
 	int bat_ctrl_convert;
@@ -2197,22 +1977,9 @@ static int ab8540_gpadc_bat_ctrl_and_ibat_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8540_gpadc_bat_ctrl_and_ibat_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8540_gpadc_bat_ctrl_and_ibat_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8540_gpadc_bat_ctrl_and_ibat_fops = {
-	.open = ab8540_gpadc_bat_ctrl_and_ibat_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8540_gpadc_bat_ctrl_and_ibat);
 
-static int ab8540_gpadc_vbat_meas_and_ibat_print(struct seq_file *s, void *p)
+static int ab8540_gpadc_vbat_meas_and_ibat_show(struct seq_file *s, void *p)
 {
 	int vbat_meas_raw;
 	int vbat_meas_convert;
@@ -2237,23 +2004,9 @@ static int ab8540_gpadc_vbat_meas_and_ibat_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8540_gpadc_vbat_meas_and_ibat_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8540_gpadc_vbat_meas_and_ibat_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8540_gpadc_vbat_meas_and_ibat_fops = {
-	.open = ab8540_gpadc_vbat_meas_and_ibat_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8540_gpadc_vbat_meas_and_ibat);
 
-static int ab8540_gpadc_vbat_true_meas_and_ibat_print(struct seq_file *s,
-						      void *p)
+static int ab8540_gpadc_vbat_true_meas_and_ibat_show(struct seq_file *s, void *p)
 {
 	int vbat_true_meas_raw;
 	int vbat_true_meas_convert;
@@ -2279,23 +2032,9 @@ static int ab8540_gpadc_vbat_true_meas_and_ibat_print(struct seq_file *s,
 	return 0;
 }
 
-static int ab8540_gpadc_vbat_true_meas_and_ibat_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8540_gpadc_vbat_true_meas_and_ibat_print,
-		inode->i_private);
-}
+DEFINE_SHOW_ATTRIBUTE(ab8540_gpadc_vbat_true_meas_and_ibat);
 
-static const struct file_operations
-ab8540_gpadc_vbat_true_meas_and_ibat_fops = {
-	.open = ab8540_gpadc_vbat_true_meas_and_ibat_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
-static int ab8540_gpadc_bat_temp_and_ibat_print(struct seq_file *s, void *p)
+static int ab8540_gpadc_bat_temp_and_ibat_show(struct seq_file *s, void *p)
 {
 	int bat_temp_raw;
 	int bat_temp_convert;
@@ -2320,22 +2059,9 @@ static int ab8540_gpadc_bat_temp_and_ibat_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8540_gpadc_bat_temp_and_ibat_open(struct inode *inode,
-		struct file *file)
-{
-	return single_open(file, ab8540_gpadc_bat_temp_and_ibat_print,
-		inode->i_private);
-}
-
-static const struct file_operations ab8540_gpadc_bat_temp_and_ibat_fops = {
-	.open = ab8540_gpadc_bat_temp_and_ibat_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8540_gpadc_bat_temp_and_ibat);
 
-static int ab8540_gpadc_otp_cal_print(struct seq_file *s, void *p)
+static int ab8540_gpadc_otp_calib_show(struct seq_file *s, void *p)
 {
 	struct ab8500_gpadc *gpadc;
 	u16 vmain_l, vmain_h, btemp_l, btemp_h;
@@ -2359,18 +2085,7 @@ static int ab8540_gpadc_otp_cal_print(struct seq_file *s, void *p)
 	return 0;
 }
 
-static int ab8540_gpadc_otp_cal_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ab8540_gpadc_otp_cal_print, inode->i_private);
-}
-
-static const struct file_operations ab8540_gpadc_otp_calib_fops = {
-	.open = ab8540_gpadc_otp_cal_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
+DEFINE_SHOW_ATTRIBUTE(ab8540_gpadc_otp_calib);
 
 static int ab8500_gpadc_avg_sample_print(struct seq_file *s, void *p)
 {
@@ -2903,14 +2618,6 @@ static const struct file_operations ab8500_val_fops = {
 	.owner = THIS_MODULE,
 };
 
-static const struct file_operations ab8500_interrupts_fops = {
-	.open = ab8500_interrupts_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.owner = THIS_MODULE,
-};
-
 static const struct file_operations ab8500_subscribe_fops = {
 	.open = ab8500_subscribe_unsubscribe_open,
 	.write = ab8500_subscribe_write,
@@ -2997,7 +2704,7 @@ static int ab8500_debug_probe(struct platform_device *plf)
 		goto err;
 
 	file = debugfs_create_file("all-bank-registers", S_IRUGO, ab8500_dir,
-				   &plf->dev, &ab8500_registers_fops);
+				   &plf->dev, &ab8500_bank_registers_fops);
 	if (!file)
 		goto err;
 
diff --git a/drivers/mfd/atmel-flexcom.c b/drivers/mfd/atmel-flexcom.c
index 064bde9cff5a..f684a93a3340 100644
--- a/drivers/mfd/atmel-flexcom.c
+++ b/drivers/mfd/atmel-flexcom.c
@@ -39,34 +39,43 @@
 #define FLEX_MR_OPMODE(opmode)	(((opmode) << FLEX_MR_OPMODE_OFFSET) &	\
 				 FLEX_MR_OPMODE_MASK)
 
+struct atmel_flexcom {
+	void __iomem *base;
+	u32 opmode;
+	struct clk *clk;
+};
 
 static int atmel_flexcom_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
-	struct clk *clk;
 	struct resource *res;
-	void __iomem *base;
-	u32 opmode;
+	struct atmel_flexcom *ddata;
 	int err;
 
-	err = of_property_read_u32(np, "atmel,flexcom-mode", &opmode);
+	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, ddata);
+
+	err = of_property_read_u32(np, "atmel,flexcom-mode", &ddata->opmode);
 	if (err)
 		return err;
 
-	if (opmode < ATMEL_FLEXCOM_MODE_USART ||
-	    opmode > ATMEL_FLEXCOM_MODE_TWI)
+	if (ddata->opmode < ATMEL_FLEXCOM_MODE_USART ||
+	    ddata->opmode > ATMEL_FLEXCOM_MODE_TWI)
 		return -EINVAL;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(base))
-		return PTR_ERR(base);
+	ddata->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(ddata->base))
+		return PTR_ERR(ddata->base);
 
-	clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(clk))
-		return PTR_ERR(clk);
+	ddata->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(ddata->clk))
+		return PTR_ERR(ddata->clk);
 
-	err = clk_prepare_enable(clk);
+	err = clk_prepare_enable(ddata->clk);
 	if (err)
 		return err;
 
@@ -76,9 +85,9 @@ static int atmel_flexcom_probe(struct platform_device *pdev)
 	 * inaccessible and are read as zero. Also the external I/O lines of the
 	 * Flexcom are muxed to reach the selected device.
 	 */
-	writel(FLEX_MR_OPMODE(opmode), base + FLEX_MR);
+	writel(FLEX_MR_OPMODE(ddata->opmode), ddata->base + FLEX_MR);
 
-	clk_disable_unprepare(clk);
+	clk_disable_unprepare(ddata->clk);
 
 	return devm_of_platform_populate(&pdev->dev);
 }
@@ -89,10 +98,34 @@ static const struct of_device_id atmel_flexcom_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, atmel_flexcom_of_match);
 
+#ifdef CONFIG_PM_SLEEP
+static int atmel_flexcom_resume(struct device *dev)
+{
+	struct atmel_flexcom *ddata = dev_get_drvdata(dev);
+	int err;
+	u32 val;
+
+	err = clk_prepare_enable(ddata->clk);
+	if (err)
+		return err;
+
+	val = FLEX_MR_OPMODE(ddata->opmode),
+	writel(val, ddata->base + FLEX_MR);
+
+	clk_disable_unprepare(ddata->clk);
+
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(atmel_flexcom_pm_ops, NULL,
+			 atmel_flexcom_resume);
+
 static struct platform_driver atmel_flexcom_driver = {
 	.probe	= atmel_flexcom_probe,
 	.driver	= {
 		.name		= "atmel_flexcom",
+		.pm		= &atmel_flexcom_pm_ops,
 		.of_match_table	= atmel_flexcom_of_match,
 	},
 };
diff --git a/drivers/mfd/axp20x.c b/drivers/mfd/axp20x.c
index 2468b431bb22..e94c72c2faa2 100644
--- a/drivers/mfd/axp20x.c
+++ b/drivers/mfd/axp20x.c
@@ -129,6 +129,7 @@ static const struct regmap_range axp288_volatile_ranges[] = {
 	regmap_reg_range(AXP20X_PWR_INPUT_STATUS, AXP288_POWER_REASON),
 	regmap_reg_range(AXP288_BC_GLOBAL, AXP288_BC_GLOBAL),
 	regmap_reg_range(AXP288_BC_DET_STAT, AXP288_BC_DET_STAT),
+	regmap_reg_range(AXP20X_CHRG_BAK_CTRL, AXP20X_CHRG_BAK_CTRL),
 	regmap_reg_range(AXP20X_IRQ1_EN, AXP20X_IPSOUT_V_HIGH_L),
 	regmap_reg_range(AXP20X_TIMER_CTRL, AXP20X_TIMER_CTRL),
 	regmap_reg_range(AXP22X_GPIO_STATE, AXP22X_GPIO_STATE),
@@ -878,6 +879,9 @@ static struct mfd_cell axp813_cells[] = {
 		.resources		= axp803_pek_resources,
 	}, {
 		.name			= "axp20x-regulator",
+	}, {
+		.name			= "axp20x-gpio",
+		.of_compatible		= "x-powers,axp813-gpio",
 	}
 };
 
diff --git a/drivers/mfd/cros_ec.c b/drivers/mfd/cros_ec.c
index b0ca5a4c841e..d61024141e2b 100644
--- a/drivers/mfd/cros_ec.c
+++ b/drivers/mfd/cros_ec.c
@@ -40,13 +40,13 @@ static struct cros_ec_platform pd_p = {
 };
 
 static const struct mfd_cell ec_cell = {
-	.name = "cros-ec-ctl",
+	.name = "cros-ec-dev",
 	.platform_data = &ec_p,
 	.pdata_size = sizeof(ec_p),
 };
 
 static const struct mfd_cell ec_pd_cell = {
-	.name = "cros-ec-ctl",
+	.name = "cros-ec-dev",
 	.platform_data = &pd_p,
 	.pdata_size = sizeof(pd_p),
 };
diff --git a/drivers/platform/chrome/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c
similarity index 99%
rename from drivers/platform/chrome/cros_ec_dev.c
rename to drivers/mfd/cros_ec_dev.c
index cf6c4f0846b8..e4fafdd96e5e 100644
--- a/drivers/platform/chrome/cros_ec_dev.c
+++ b/drivers/mfd/cros_ec_dev.c
@@ -25,9 +25,10 @@
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 
-#include "cros_ec_debugfs.h"
 #include "cros_ec_dev.h"
 
+#define DRV_NAME "cros-ec-dev"
+
 /* Device variables */
 #define CROS_MAX_DEV 128
 static int ec_major;
@@ -461,7 +462,7 @@ static int ec_device_remove(struct platform_device *pdev)
 }
 
 static const struct platform_device_id cros_ec_id[] = {
-	{ "cros-ec-ctl", 0 },
+	{ DRV_NAME, 0 },
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(platform, cros_ec_id);
@@ -493,7 +494,7 @@ static const struct dev_pm_ops cros_ec_dev_pm_ops = {
 
 static struct platform_driver cros_ec_dev_driver = {
 	.driver = {
-		.name = "cros-ec-ctl",
+		.name = DRV_NAME,
 		.pm = &cros_ec_dev_pm_ops,
 	},
 	.probe = ec_device_probe,
@@ -544,6 +545,7 @@ static void __exit cros_ec_dev_exit(void)
 module_init(cros_ec_dev_init);
 module_exit(cros_ec_dev_exit);
 
+MODULE_ALIAS("platform:" DRV_NAME);
 MODULE_AUTHOR("Bill Richardson <wfrichar@chromium.org>");
 MODULE_DESCRIPTION("Userspace interface to the Chrome OS Embedded Controller");
 MODULE_VERSION("1.0");
diff --git a/drivers/platform/chrome/cros_ec_dev.h b/drivers/mfd/cros_ec_dev.h
similarity index 100%
rename from drivers/platform/chrome/cros_ec_dev.h
rename to drivers/mfd/cros_ec_dev.h
diff --git a/drivers/mfd/cros_ec_spi.c b/drivers/mfd/cros_ec_spi.c
index 59c82cdcf48d..1b52b8557034 100644
--- a/drivers/mfd/cros_ec_spi.c
+++ b/drivers/mfd/cros_ec_spi.c
@@ -72,8 +72,7 @@
  * struct cros_ec_spi - information about a SPI-connected EC
  *
  * @spi: SPI device we are connected to
- * @last_transfer_ns: time that we last finished a transfer, or 0 if there
- *	if no record
+ * @last_transfer_ns: time that we last finished a transfer.
  * @start_of_msg_delay: used to set the delay_usecs on the spi_transfer that
  *      is sent when we want to turn on CS at the start of a transaction.
  * @end_of_msg_delay: used to set the delay_usecs on the spi_transfer that
@@ -379,18 +378,15 @@ static int cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev,
 	u8 sum;
 	u8 rx_byte;
 	int ret = 0, final_ret;
+	unsigned long delay;
 
 	len = cros_ec_prepare_tx(ec_dev, ec_msg);
 	dev_dbg(ec_dev->dev, "prepared, len=%d\n", len);
 
 	/* If it's too soon to do another transaction, wait */
-	if (ec_spi->last_transfer_ns) {
-		unsigned long delay;	/* The delay completed so far */
-
-		delay = ktime_get_ns() - ec_spi->last_transfer_ns;
-		if (delay < EC_SPI_RECOVERY_TIME_NS)
-			ndelay(EC_SPI_RECOVERY_TIME_NS - delay);
-	}
+	delay = ktime_get_ns() - ec_spi->last_transfer_ns;
+	if (delay < EC_SPI_RECOVERY_TIME_NS)
+		ndelay(EC_SPI_RECOVERY_TIME_NS - delay);
 
 	rx_buf = kzalloc(len, GFP_KERNEL);
 	if (!rx_buf)
@@ -509,18 +505,15 @@ static int cros_ec_cmd_xfer_spi(struct cros_ec_device *ec_dev,
 	u8 rx_byte;
 	int sum;
 	int ret = 0, final_ret;
+	unsigned long delay;
 
 	len = cros_ec_prepare_tx(ec_dev, ec_msg);
 	dev_dbg(ec_dev->dev, "prepared, len=%d\n", len);
 
 	/* If it's too soon to do another transaction, wait */
-	if (ec_spi->last_transfer_ns) {
-		unsigned long delay;	/* The delay completed so far */
-
-		delay = ktime_get_ns() - ec_spi->last_transfer_ns;
-		if (delay < EC_SPI_RECOVERY_TIME_NS)
-			ndelay(EC_SPI_RECOVERY_TIME_NS - delay);
-	}
+	delay = ktime_get_ns() - ec_spi->last_transfer_ns;
+	if (delay < EC_SPI_RECOVERY_TIME_NS)
+		ndelay(EC_SPI_RECOVERY_TIME_NS - delay);
 
 	rx_buf = kzalloc(len, GFP_KERNEL);
 	if (!rx_buf)
diff --git a/drivers/mfd/intel-lpss.c b/drivers/mfd/intel-lpss.c
index 0e0ab9bb1530..9e545eb6e8b4 100644
--- a/drivers/mfd/intel-lpss.c
+++ b/drivers/mfd/intel-lpss.c
@@ -450,6 +450,8 @@ int intel_lpss_probe(struct device *dev,
 	if (ret)
 		goto err_remove_ltr;
 
+	dev_pm_set_driver_flags(dev, DPM_FLAG_SMART_SUSPEND);
+
 	return 0;
 
 err_remove_ltr:
@@ -478,7 +480,9 @@ EXPORT_SYMBOL_GPL(intel_lpss_remove);
 
 static int resume_lpss_device(struct device *dev, void *data)
 {
-	pm_runtime_resume(dev);
+	if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND))
+		pm_runtime_resume(dev);
+
 	return 0;
 }
 
diff --git a/drivers/mfd/intel_soc_pmic_core.c b/drivers/mfd/intel_soc_pmic_core.c
index 36adf9e8153e..274306d98ac1 100644
--- a/drivers/mfd/intel_soc_pmic_core.c
+++ b/drivers/mfd/intel_soc_pmic_core.c
@@ -16,7 +16,6 @@
  * Author: Zhu, Lejun <lejun.zhu@linux.intel.com>
  */
 
-#include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/mfd/core.h>
 #include <linux/i2c.h>
diff --git a/drivers/mfd/kempld-core.c b/drivers/mfd/kempld-core.c
index 55d824b3a808..390b27cb2c2e 100644
--- a/drivers/mfd/kempld-core.c
+++ b/drivers/mfd/kempld-core.c
@@ -458,7 +458,7 @@ static int kempld_probe(struct platform_device *pdev)
 		return -EINVAL;
 
 	pld->io_base = devm_ioport_map(dev, ioport->start,
-					ioport->end - ioport->start);
+					resource_size(ioport));
 	if (!pld->io_base)
 		return -ENOMEM;
 
diff --git a/drivers/mfd/lpc_ich.c b/drivers/mfd/lpc_ich.c
index cf1120abbf52..53dc1a43472c 100644
--- a/drivers/mfd/lpc_ich.c
+++ b/drivers/mfd/lpc_ich.c
@@ -1143,11 +1143,6 @@ static int lpc_ich_init_spi(struct pci_dev *dev)
 			res->end = res->start + SPIBASE_APL_SZ - 1;
 
 			pci_bus_read_config_dword(bus, spi, BCR, &bcr);
-			if (!(bcr & BCR_WPD)) {
-				bcr |= BCR_WPD;
-				pci_bus_write_config_dword(bus, spi, BCR, bcr);
-				pci_bus_read_config_dword(bus, spi, BCR, &bcr);
-			}
 			info->writeable = !!(bcr & BCR_WPD);
 		}
 
diff --git a/drivers/mfd/max77843.c b/drivers/mfd/max77843.c
index dc5caeaaa6a1..da9612dbb222 100644
--- a/drivers/mfd/max77843.c
+++ b/drivers/mfd/max77843.c
@@ -15,7 +15,6 @@
 #include <linux/i2c.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
-#include <linux/init.h>
 #include <linux/mfd/core.h>
 #include <linux/mfd/max77693-common.h>
 #include <linux/mfd/max77843-private.h>
diff --git a/drivers/mfd/palmas.c b/drivers/mfd/palmas.c
index 3922a93f9f92..663a2398b6b1 100644
--- a/drivers/mfd/palmas.c
+++ b/drivers/mfd/palmas.c
@@ -430,6 +430,7 @@ static void palmas_power_off(void)
 {
 	unsigned int addr;
 	int ret, slave;
+	u8 powerhold_mask;
 	struct device_node *np = palmas_dev->dev->of_node;
 
 	if (of_property_read_bool(np, "ti,palmas-override-powerhold")) {
@@ -437,8 +438,15 @@ static void palmas_power_off(void)
 					  PALMAS_PRIMARY_SECONDARY_PAD2);
 		slave = PALMAS_BASE_TO_SLAVE(PALMAS_PU_PD_OD_BASE);
 
+		if (of_device_is_compatible(np, "ti,tps65917"))
+			powerhold_mask =
+				TPS65917_PRIMARY_SECONDARY_PAD2_GPIO_5_MASK;
+		else
+			powerhold_mask =
+				PALMAS_PRIMARY_SECONDARY_PAD2_GPIO_7_MASK;
+
 		ret = regmap_update_bits(palmas_dev->regmap[slave], addr,
-				PALMAS_PRIMARY_SECONDARY_PAD2_GPIO_7_MASK, 0);
+					 powerhold_mask, 0);
 		if (ret)
 			dev_err(palmas_dev->dev,
 				"Unable to write PRIMARY_SECONDARY_PAD2 %d\n",
diff --git a/drivers/mfd/pcf50633-core.c b/drivers/mfd/pcf50633-core.c
index 6155d123a84e..f952dff6765f 100644
--- a/drivers/mfd/pcf50633-core.c
+++ b/drivers/mfd/pcf50633-core.c
@@ -149,7 +149,7 @@ pcf50633_client_dev_register(struct pcf50633 *pcf, const char *name,
 
 	*pdev = platform_device_alloc(name, -1);
 	if (!*pdev) {
-		dev_err(pcf->dev, "Falied to allocate %s\n", name);
+		dev_err(pcf->dev, "Failed to allocate %s\n", name);
 		return;
 	}
 
diff --git a/drivers/mfd/rave-sp.c b/drivers/mfd/rave-sp.c
new file mode 100644
index 000000000000..5c858e784a89
--- /dev/null
+++ b/drivers/mfd/rave-sp.c
@@ -0,0 +1,710 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/*
+ * Multifunction core driver for Zodiac Inflight Innovations RAVE
+ * Supervisory Processor(SP) MCU that is connected via dedicated UART
+ * port
+ *
+ * Copyright (C) 2017 Zodiac Inflight Innovations
+ */
+
+#include <linux/atomic.h>
+#include <linux/crc-ccitt.h>
+#include <linux/delay.h>
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/kernel.h>
+#include <linux/mfd/rave-sp.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/sched.h>
+#include <linux/serdev.h>
+#include <asm/unaligned.h>
+
+/*
+ * UART protocol using following entities:
+ *  - message to MCU => ACK response
+ *  - event from MCU => event ACK
+ *
+ * Frame structure:
+ * <STX> <DATA> <CHECKSUM> <ETX>
+ * Where:
+ * - STX - is start of transmission character
+ * - ETX - end of transmission
+ * - DATA - payload
+ * - CHECKSUM - checksum calculated on <DATA>
+ *
+ * If <DATA> or <CHECKSUM> contain one of control characters, then it is
+ * escaped using <DLE> control code. Added <DLE> does not participate in
+ * checksum calculation.
+ */
+#define RAVE_SP_STX			0x02
+#define RAVE_SP_ETX			0x03
+#define RAVE_SP_DLE			0x10
+
+#define RAVE_SP_MAX_DATA_SIZE		64
+#define RAVE_SP_CHECKSUM_SIZE		2  /* Worst case scenario on RDU2 */
+/*
+ * We don't store STX, ETX and unescaped bytes, so Rx is only
+ * DATA + CSUM
+ */
+#define RAVE_SP_RX_BUFFER_SIZE				\
+	(RAVE_SP_MAX_DATA_SIZE + RAVE_SP_CHECKSUM_SIZE)
+
+#define RAVE_SP_STX_ETX_SIZE		2
+/*
+ * For Tx we have to have space for everything, STX, EXT and
+ * potentially stuffed DATA + CSUM data + csum
+ */
+#define RAVE_SP_TX_BUFFER_SIZE				\
+	(RAVE_SP_STX_ETX_SIZE + 2 * RAVE_SP_RX_BUFFER_SIZE)
+
+#define RAVE_SP_BOOT_SOURCE_GET		0
+#define RAVE_SP_BOOT_SOURCE_SET		1
+
+#define RAVE_SP_RDU2_BOARD_TYPE_RMB	0
+#define RAVE_SP_RDU2_BOARD_TYPE_DEB	1
+
+#define RAVE_SP_BOOT_SOURCE_SD		0
+#define RAVE_SP_BOOT_SOURCE_EMMC	1
+#define RAVE_SP_BOOT_SOURCE_NOR		2
+
+/**
+ * enum rave_sp_deframer_state - Possible state for de-framer
+ *
+ * @RAVE_SP_EXPECT_SOF:		 Scanning input for start-of-frame marker
+ * @RAVE_SP_EXPECT_DATA:	 Got start of frame marker, collecting frame
+ * @RAVE_SP_EXPECT_ESCAPED_DATA: Got escape character, collecting escaped byte
+ */
+enum rave_sp_deframer_state {
+	RAVE_SP_EXPECT_SOF,
+	RAVE_SP_EXPECT_DATA,
+	RAVE_SP_EXPECT_ESCAPED_DATA,
+};
+
+/**
+ * struct rave_sp_deframer - Device protocol deframer
+ *
+ * @state:  Current state of the deframer
+ * @data:   Buffer used to collect deframed data
+ * @length: Number of bytes de-framed so far
+ */
+struct rave_sp_deframer {
+	enum rave_sp_deframer_state state;
+	unsigned char data[RAVE_SP_RX_BUFFER_SIZE];
+	size_t length;
+};
+
+/**
+ * struct rave_sp_reply - Reply as per RAVE device protocol
+ *
+ * @length:	Expected reply length
+ * @data:	Buffer to store reply payload in
+ * @code:	Expected reply code
+ * @ackid:	Expected reply ACK ID
+ * @completion: Successful reply reception completion
+ */
+struct rave_sp_reply {
+	size_t length;
+	void  *data;
+	u8     code;
+	u8     ackid;
+	struct completion received;
+};
+
+/**
+ * struct rave_sp_checksum - Variant specific checksum implementation details
+ *
+ * @length:	Caculated checksum length
+ * @subroutine:	Utilized checksum algorithm implementation
+ */
+struct rave_sp_checksum {
+	size_t length;
+	void (*subroutine)(const u8 *, size_t, u8 *);
+};
+
+/**
+ * struct rave_sp_variant_cmds - Variant specific command routines
+ *
+ * @translate:	Generic to variant specific command mapping routine
+ *
+ */
+struct rave_sp_variant_cmds {
+	int (*translate)(enum rave_sp_command);
+};
+
+/**
+ * struct rave_sp_variant - RAVE supervisory processor core variant
+ *
+ * @checksum:	Variant specific checksum implementation
+ * @cmd:	Variant specific command pointer table
+ *
+ */
+struct rave_sp_variant {
+	const struct rave_sp_checksum *checksum;
+	struct rave_sp_variant_cmds cmd;
+};
+
+/**
+ * struct rave_sp - RAVE supervisory processor core
+ *
+ * @serdev:			Pointer to underlying serdev
+ * @deframer:			Stored state of the protocol deframer
+ * @ackid:			ACK ID used in last reply sent to the device
+ * @bus_lock:			Lock to serialize access to the device
+ * @reply_lock:			Lock protecting @reply
+ * @reply:			Pointer to memory to store reply payload
+ *
+ * @variant:			Device variant specific information
+ * @event_notifier_list:	Input event notification chain
+ *
+ */
+struct rave_sp {
+	struct serdev_device *serdev;
+	struct rave_sp_deframer deframer;
+	atomic_t ackid;
+	struct mutex bus_lock;
+	struct mutex reply_lock;
+	struct rave_sp_reply *reply;
+
+	const struct rave_sp_variant *variant;
+	struct blocking_notifier_head event_notifier_list;
+};
+
+static bool rave_sp_id_is_event(u8 code)
+{
+	return (code & 0xF0) == RAVE_SP_EVNT_BASE;
+}
+
+static void rave_sp_unregister_event_notifier(struct device *dev, void *res)
+{
+	struct rave_sp *sp = dev_get_drvdata(dev->parent);
+	struct notifier_block *nb = *(struct notifier_block **)res;
+	struct blocking_notifier_head *bnh = &sp->event_notifier_list;
+
+	WARN_ON(blocking_notifier_chain_unregister(bnh, nb));
+}
+
+int devm_rave_sp_register_event_notifier(struct device *dev,
+					 struct notifier_block *nb)
+{
+	struct rave_sp *sp = dev_get_drvdata(dev->parent);
+	struct notifier_block **rcnb;
+	int ret;
+
+	rcnb = devres_alloc(rave_sp_unregister_event_notifier,
+			    sizeof(*rcnb), GFP_KERNEL);
+	if (!rcnb)
+		return -ENOMEM;
+
+	ret = blocking_notifier_chain_register(&sp->event_notifier_list, nb);
+	if (!ret) {
+		*rcnb = nb;
+		devres_add(dev, rcnb);
+	} else {
+		devres_free(rcnb);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(devm_rave_sp_register_event_notifier);
+
+static void csum_8b2c(const u8 *buf, size_t size, u8 *crc)
+{
+	*crc = *buf++;
+	size--;
+
+	while (size--)
+		*crc += *buf++;
+
+	*crc = 1 + ~(*crc);
+}
+
+static void csum_ccitt(const u8 *buf, size_t size, u8 *crc)
+{
+	const u16 calculated = crc_ccitt_false(0xffff, buf, size);
+
+	/*
+	 * While the rest of the wire protocol is little-endian,
+	 * CCITT-16 CRC in RDU2 device is sent out in big-endian order.
+	 */
+	put_unaligned_be16(calculated, crc);
+}
+
+static void *stuff(unsigned char *dest, const unsigned char *src, size_t n)
+{
+	while (n--) {
+		const unsigned char byte = *src++;
+
+		switch (byte) {
+		case RAVE_SP_STX:
+		case RAVE_SP_ETX:
+		case RAVE_SP_DLE:
+			*dest++ = RAVE_SP_DLE;
+			/* FALLTHROUGH */
+		default:
+			*dest++ = byte;
+		}
+	}
+
+	return dest;
+}
+
+static int rave_sp_write(struct rave_sp *sp, const u8 *data, u8 data_size)
+{
+	const size_t checksum_length = sp->variant->checksum->length;
+	unsigned char frame[RAVE_SP_TX_BUFFER_SIZE];
+	unsigned char crc[RAVE_SP_CHECKSUM_SIZE];
+	unsigned char *dest = frame;
+	size_t length;
+
+	if (WARN_ON(checksum_length > sizeof(crc)))
+		return -ENOMEM;
+
+	if (WARN_ON(data_size > sizeof(frame)))
+		return -ENOMEM;
+
+	sp->variant->checksum->subroutine(data, data_size, crc);
+
+	*dest++ = RAVE_SP_STX;
+	dest = stuff(dest, data, data_size);
+	dest = stuff(dest, crc, checksum_length);
+	*dest++ = RAVE_SP_ETX;
+
+	length = dest - frame;
+
+	print_hex_dump(KERN_DEBUG, "rave-sp tx: ", DUMP_PREFIX_NONE,
+		       16, 1, frame, length, false);
+
+	return serdev_device_write(sp->serdev, frame, length, HZ);
+}
+
+static u8 rave_sp_reply_code(u8 command)
+{
+	/*
+	 * There isn't a single rule that describes command code ->
+	 * ACK code transformation, but, going through various
+	 * versions of ICDs, there appear to be three distinct groups
+	 * that can be described by simple transformation.
+	 */
+	switch (command) {
+	case 0xA0 ... 0xBE:
+		/*
+		 * Commands implemented by firmware found in RDU1 and
+		 * older devices all seem to obey the following rule
+		 */
+		return command + 0x20;
+	case 0xE0 ... 0xEF:
+		/*
+		 * Events emitted by all versions of the firmare use
+		 * least significant bit to get an ACK code
+		 */
+		return command | 0x01;
+	default:
+		/*
+		 * Commands implemented by firmware found in RDU2 are
+		 * similar to "old" commands, but they use slightly
+		 * different offset
+		 */
+		return command + 0x40;
+	}
+}
+
+int rave_sp_exec(struct rave_sp *sp,
+		 void *__data,  size_t data_size,
+		 void *reply_data, size_t reply_data_size)
+{
+	struct rave_sp_reply reply = {
+		.data     = reply_data,
+		.length   = reply_data_size,
+		.received = COMPLETION_INITIALIZER_ONSTACK(reply.received),
+	};
+	unsigned char *data = __data;
+	int command, ret = 0;
+	u8 ackid;
+
+	command = sp->variant->cmd.translate(data[0]);
+	if (command < 0)
+		return command;
+
+	ackid       = atomic_inc_return(&sp->ackid);
+	reply.ackid = ackid;
+	reply.code  = rave_sp_reply_code((u8)command),
+
+	mutex_lock(&sp->bus_lock);
+
+	mutex_lock(&sp->reply_lock);
+	sp->reply = &reply;
+	mutex_unlock(&sp->reply_lock);
+
+	data[0] = command;
+	data[1] = ackid;
+
+	rave_sp_write(sp, data, data_size);
+
+	if (!wait_for_completion_timeout(&reply.received, HZ)) {
+		dev_err(&sp->serdev->dev, "Command timeout\n");
+		ret = -ETIMEDOUT;
+
+		mutex_lock(&sp->reply_lock);
+		sp->reply = NULL;
+		mutex_unlock(&sp->reply_lock);
+	}
+
+	mutex_unlock(&sp->bus_lock);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(rave_sp_exec);
+
+static void rave_sp_receive_event(struct rave_sp *sp,
+				  const unsigned char *data, size_t length)
+{
+	u8 cmd[] = {
+		[0] = rave_sp_reply_code(data[0]),
+		[1] = data[1],
+	};
+
+	rave_sp_write(sp, cmd, sizeof(cmd));
+
+	blocking_notifier_call_chain(&sp->event_notifier_list,
+				     rave_sp_action_pack(data[0], data[2]),
+				     NULL);
+}
+
+static void rave_sp_receive_reply(struct rave_sp *sp,
+				  const unsigned char *data, size_t length)
+{
+	struct device *dev = &sp->serdev->dev;
+	struct rave_sp_reply *reply;
+	const  size_t payload_length = length - 2;
+
+	mutex_lock(&sp->reply_lock);
+	reply = sp->reply;
+
+	if (reply) {
+		if (reply->code == data[0] && reply->ackid == data[1] &&
+		    payload_length >= reply->length) {
+			/*
+			 * We are relying on memcpy(dst, src, 0) to be a no-op
+			 * when handling commands that have a no-payload reply
+			 */
+			memcpy(reply->data, &data[2], reply->length);
+			complete(&reply->received);
+			sp->reply = NULL;
+		} else {
+			dev_err(dev, "Ignoring incorrect reply\n");
+			dev_dbg(dev, "Code:   expected = 0x%08x received = 0x%08x\n",
+				reply->code, data[0]);
+			dev_dbg(dev, "ACK ID: expected = 0x%08x received = 0x%08x\n",
+				reply->ackid, data[1]);
+			dev_dbg(dev, "Length: expected = %zu received = %zu\n",
+				reply->length, payload_length);
+		}
+	}
+
+	mutex_unlock(&sp->reply_lock);
+}
+
+static void rave_sp_receive_frame(struct rave_sp *sp,
+				  const unsigned char *data,
+				  size_t length)
+{
+	const size_t checksum_length = sp->variant->checksum->length;
+	const size_t payload_length  = length - checksum_length;
+	const u8 *crc_reported       = &data[payload_length];
+	struct device *dev           = &sp->serdev->dev;
+	u8 crc_calculated[checksum_length];
+
+	print_hex_dump(KERN_DEBUG, "rave-sp rx: ", DUMP_PREFIX_NONE,
+		       16, 1, data, length, false);
+
+	if (unlikely(length <= checksum_length)) {
+		dev_warn(dev, "Dropping short frame\n");
+		return;
+	}
+
+	sp->variant->checksum->subroutine(data, payload_length,
+					  crc_calculated);
+
+	if (memcmp(crc_calculated, crc_reported, checksum_length)) {
+		dev_warn(dev, "Dropping bad frame\n");
+		return;
+	}
+
+	if (rave_sp_id_is_event(data[0]))
+		rave_sp_receive_event(sp, data, length);
+	else
+		rave_sp_receive_reply(sp, data, length);
+}
+
+static int rave_sp_receive_buf(struct serdev_device *serdev,
+			       const unsigned char *buf, size_t size)
+{
+	struct device *dev = &serdev->dev;
+	struct rave_sp *sp = dev_get_drvdata(dev);
+	struct rave_sp_deframer *deframer = &sp->deframer;
+	const unsigned char *src = buf;
+	const unsigned char *end = buf + size;
+
+	while (src < end) {
+		const unsigned char byte = *src++;
+
+		switch (deframer->state) {
+		case RAVE_SP_EXPECT_SOF:
+			if (byte == RAVE_SP_STX)
+				deframer->state = RAVE_SP_EXPECT_DATA;
+			break;
+
+		case RAVE_SP_EXPECT_DATA:
+			/*
+			 * Treat special byte values first
+			 */
+			switch (byte) {
+			case RAVE_SP_ETX:
+				rave_sp_receive_frame(sp,
+						      deframer->data,
+						      deframer->length);
+				/*
+				 * Once we extracted a complete frame
+				 * out of a stream, we call it done
+				 * and proceed to bailing out while
+				 * resetting the framer to initial
+				 * state, regardless if we've consumed
+				 * all of the stream or not.
+				 */
+				goto reset_framer;
+			case RAVE_SP_STX:
+				dev_warn(dev, "Bad frame: STX before ETX\n");
+				/*
+				 * If we encounter second "start of
+				 * the frame" marker before seeing
+				 * corresponding "end of frame", we
+				 * reset the framer and ignore both:
+				 * frame started by first SOF and
+				 * frame started by current SOF.
+				 *
+				 * NOTE: The above means that only the
+				 * frame started by third SOF, sent
+				 * after this one will have a chance
+				 * to get throught.
+				 */
+				goto reset_framer;
+			case RAVE_SP_DLE:
+				deframer->state = RAVE_SP_EXPECT_ESCAPED_DATA;
+				/*
+				 * If we encounter escape sequence we
+				 * need to skip it and collect the
+				 * byte that follows. We do it by
+				 * forcing the next iteration of the
+				 * encompassing while loop.
+				 */
+				continue;
+			}
+			/*
+			 * For the rest of the bytes, that are not
+			 * speical snoflakes, we do the same thing
+			 * that we do to escaped data - collect it in
+			 * deframer buffer
+			 */
+
+			/* FALLTHROUGH */
+
+		case RAVE_SP_EXPECT_ESCAPED_DATA:
+			deframer->data[deframer->length++] = byte;
+
+			if (deframer->length == sizeof(deframer->data)) {
+				dev_warn(dev, "Bad frame: Too long\n");
+				/*
+				 * If the amount of data we've
+				 * accumulated for current frame so
+				 * far starts to exceed the capacity
+				 * of deframer's buffer, there's
+				 * nothing else we can do but to
+				 * discard that data and start
+				 * assemblying a new frame again
+				 */
+				goto reset_framer;
+			}
+
+			/*
+			 * We've extracted out special byte, now we
+			 * can go back to regular data collecting
+			 */
+			deframer->state = RAVE_SP_EXPECT_DATA;
+			break;
+		}
+	}
+
+	/*
+	 * The only way to get out of the above loop and end up here
+	 * is throught consuming all of the supplied data, so here we
+	 * report that we processed it all.
+	 */
+	return size;
+
+reset_framer:
+	/*
+	 * NOTE: A number of codepaths that will drop us here will do
+	 * so before consuming all 'size' bytes of the data passed by
+	 * serdev layer. We rely on the fact that serdev layer will
+	 * re-execute this handler with the remainder of the Rx bytes
+	 * once we report actual number of bytes that we processed.
+	 */
+	deframer->state  = RAVE_SP_EXPECT_SOF;
+	deframer->length = 0;
+
+	return src - buf;
+}
+
+static int rave_sp_rdu1_cmd_translate(enum rave_sp_command command)
+{
+	if (command >= RAVE_SP_CMD_STATUS &&
+	    command <= RAVE_SP_CMD_CONTROL_EVENTS)
+		return command;
+
+	return -EINVAL;
+}
+
+static int rave_sp_rdu2_cmd_translate(enum rave_sp_command command)
+{
+	if (command >= RAVE_SP_CMD_GET_FIRMWARE_VERSION &&
+	    command <= RAVE_SP_CMD_GET_GPIO_STATE)
+		return command;
+
+	if (command == RAVE_SP_CMD_REQ_COPPER_REV) {
+		/*
+		 * As per RDU2 ICD 3.4.47 CMD_GET_COPPER_REV code is
+		 * different from that for RDU1 and it is set to 0x28.
+		 */
+		return 0x28;
+	}
+
+	return rave_sp_rdu1_cmd_translate(command);
+}
+
+static int rave_sp_default_cmd_translate(enum rave_sp_command command)
+{
+	/*
+	 * All of the following command codes were taken from "Table :
+	 * Communications Protocol Message Types" in section 3.3
+	 * "MESSAGE TYPES" of Rave PIC24 ICD.
+	 */
+	switch (command) {
+	case RAVE_SP_CMD_GET_FIRMWARE_VERSION:
+		return 0x11;
+	case RAVE_SP_CMD_GET_BOOTLOADER_VERSION:
+		return 0x12;
+	case RAVE_SP_CMD_BOOT_SOURCE:
+		return 0x14;
+	case RAVE_SP_CMD_SW_WDT:
+		return 0x1C;
+	case RAVE_SP_CMD_RESET:
+		return 0x1E;
+	case RAVE_SP_CMD_RESET_REASON:
+		return 0x1F;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct rave_sp_checksum rave_sp_checksum_8b2c = {
+	.length     = 1,
+	.subroutine = csum_8b2c,
+};
+
+static const struct rave_sp_checksum rave_sp_checksum_ccitt = {
+	.length     = 2,
+	.subroutine = csum_ccitt,
+};
+
+static const struct rave_sp_variant rave_sp_legacy = {
+	.checksum = &rave_sp_checksum_8b2c,
+	.cmd = {
+		.translate = rave_sp_default_cmd_translate,
+	},
+};
+
+static const struct rave_sp_variant rave_sp_rdu1 = {
+	.checksum = &rave_sp_checksum_8b2c,
+	.cmd = {
+		.translate = rave_sp_rdu1_cmd_translate,
+	},
+};
+
+static const struct rave_sp_variant rave_sp_rdu2 = {
+	.checksum = &rave_sp_checksum_ccitt,
+	.cmd = {
+		.translate = rave_sp_rdu2_cmd_translate,
+	},
+};
+
+static const struct of_device_id rave_sp_dt_ids[] = {
+	{ .compatible = "zii,rave-sp-niu",  .data = &rave_sp_legacy },
+	{ .compatible = "zii,rave-sp-mezz", .data = &rave_sp_legacy },
+	{ .compatible = "zii,rave-sp-esb",  .data = &rave_sp_legacy },
+	{ .compatible = "zii,rave-sp-rdu1", .data = &rave_sp_rdu1   },
+	{ .compatible = "zii,rave-sp-rdu2", .data = &rave_sp_rdu2   },
+	{ /* sentinel */ }
+};
+
+static const struct serdev_device_ops rave_sp_serdev_device_ops = {
+	.receive_buf  = rave_sp_receive_buf,
+	.write_wakeup = serdev_device_write_wakeup,
+};
+
+static int rave_sp_probe(struct serdev_device *serdev)
+{
+	struct device *dev = &serdev->dev;
+	struct rave_sp *sp;
+	u32 baud;
+	int ret;
+
+	if (of_property_read_u32(dev->of_node, "current-speed", &baud)) {
+		dev_err(dev,
+			"'current-speed' is not specified in device node\n");
+		return -EINVAL;
+	}
+
+	sp = devm_kzalloc(dev, sizeof(*sp), GFP_KERNEL);
+	if (!sp)
+		return -ENOMEM;
+
+	sp->serdev = serdev;
+	dev_set_drvdata(dev, sp);
+
+	sp->variant = of_device_get_match_data(dev);
+	if (!sp->variant)
+		return -ENODEV;
+
+	mutex_init(&sp->bus_lock);
+	mutex_init(&sp->reply_lock);
+	BLOCKING_INIT_NOTIFIER_HEAD(&sp->event_notifier_list);
+
+	serdev_device_set_client_ops(serdev, &rave_sp_serdev_device_ops);
+	ret = devm_serdev_device_open(dev, serdev);
+	if (ret)
+		return ret;
+
+	serdev_device_set_baudrate(serdev, baud);
+
+	return devm_of_platform_populate(dev);
+}
+
+MODULE_DEVICE_TABLE(of, rave_sp_dt_ids);
+
+static struct serdev_device_driver rave_sp_drv = {
+	.probe			= rave_sp_probe,
+	.driver = {
+		.name		= "rave-sp",
+		.of_match_table	= rave_sp_dt_ids,
+	},
+};
+module_serdev_device_driver(rave_sp_drv);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Andrey Vostrikov <andrey.vostrikov@cogentembedded.com>");
+MODULE_AUTHOR("Nikita Yushchenko <nikita.yoush@cogentembedded.com>");
+MODULE_AUTHOR("Andrey Smirnov <andrew.smirnov@gmail.com>");
+MODULE_DESCRIPTION("RAVE SP core driver");
diff --git a/drivers/mfd/stm32-lptimer.c b/drivers/mfd/stm32-lptimer.c
index 075330a25f61..a00f99f36559 100644
--- a/drivers/mfd/stm32-lptimer.c
+++ b/drivers/mfd/stm32-lptimer.c
@@ -1,13 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * STM32 Low-Power Timer parent driver.
- *
  * Copyright (C) STMicroelectronics 2017
- *
  * Author: Fabrice Gasnier <fabrice.gasnier@st.com>
- *
  * Inspired by Benjamin Gaignard's stm32-timers driver
- *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/mfd/stm32-lptimer.h>
diff --git a/drivers/mfd/stm32-timers.c b/drivers/mfd/stm32-timers.c
index a6675a449409..1d347e5dfa79 100644
--- a/drivers/mfd/stm32-timers.c
+++ b/drivers/mfd/stm32-timers.c
@@ -1,9 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) STMicroelectronics 2016
- *
  * Author: Benjamin Gaignard <benjamin.gaignard@st.com>
- *
- * License terms:  GNU General Public License (GPL), version 2
  */
 
 #include <linux/mfd/stm32-timers.h>
diff --git a/drivers/mfd/syscon.c b/drivers/mfd/syscon.c
index b93fe4c4957a..7eaa40bc703f 100644
--- a/drivers/mfd/syscon.c
+++ b/drivers/mfd/syscon.c
@@ -13,6 +13,7 @@
  */
 
 #include <linux/err.h>
+#include <linux/hwspinlock.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/list.h>
@@ -87,6 +88,24 @@ static struct syscon *of_syscon_register(struct device_node *np)
 	if (ret)
 		reg_io_width = 4;
 
+	ret = of_hwspin_lock_get_id(np, 0);
+	if (ret > 0 || (IS_ENABLED(CONFIG_HWSPINLOCK) && ret == 0)) {
+		syscon_config.use_hwlock = true;
+		syscon_config.hwlock_id = ret;
+		syscon_config.hwlock_mode = HWLOCK_IRQSTATE;
+	} else if (ret < 0) {
+		switch (ret) {
+		case -ENOENT:
+			/* Ignore missing hwlock, it's optional. */
+			break;
+		default:
+			pr_err("Failed to retrieve valid hwlock: %d\n", ret);
+			/* fall-through */
+		case -EPROBE_DEFER:
+			goto err_regmap;
+		}
+	}
+
 	syscon_config.reg_stride = reg_io_width;
 	syscon_config.val_bits = reg_io_width * 8;
 	syscon_config.max_register = resource_size(&res) - reg_io_width;
diff --git a/drivers/mfd/ti_am335x_tscadc.c b/drivers/mfd/ti_am335x_tscadc.c
index 0f3fab47fe48..3cd958a31f36 100644
--- a/drivers/mfd/ti_am335x_tscadc.c
+++ b/drivers/mfd/ti_am335x_tscadc.c
@@ -124,7 +124,7 @@ static	int ti_tscadc_probe(struct platform_device *pdev)
 	struct ti_tscadc_dev	*tscadc;
 	struct resource		*res;
 	struct clk		*clk;
-	struct device_node	*node = pdev->dev.of_node;
+	struct device_node	*node;
 	struct mfd_cell		*cell;
 	struct property         *prop;
 	const __be32            *cur;
diff --git a/drivers/mfd/tmio_core.c b/drivers/mfd/tmio_core.c
index 83af78c1b0eb..ebf54cc28f7a 100644
--- a/drivers/mfd/tmio_core.c
+++ b/drivers/mfd/tmio_core.c
@@ -9,6 +9,26 @@
 #include <linux/export.h>
 #include <linux/mfd/tmio.h>
 
+#define CNF_CMD     0x04
+#define CNF_CTL_BASE   0x10
+#define CNF_INT_PIN  0x3d
+#define CNF_STOP_CLK_CTL 0x40
+#define CNF_GCLK_CTL 0x41
+#define CNF_SD_CLK_MODE 0x42
+#define CNF_PIN_STATUS 0x44
+#define CNF_PWR_CTL_1 0x48
+#define CNF_PWR_CTL_2 0x49
+#define CNF_PWR_CTL_3 0x4a
+#define CNF_CARD_DETECT_MODE 0x4c
+#define CNF_SD_SLOT 0x50
+#define CNF_EXT_GCLK_CTL_1 0xf0
+#define CNF_EXT_GCLK_CTL_2 0xf1
+#define CNF_EXT_GCLK_CTL_3 0xf9
+#define CNF_SD_LED_EN_1 0xfa
+#define CNF_SD_LED_EN_2 0xfe
+
+#define   SDCREN 0x2   /* Enable access to MMC CTL regs. (flag in COMMAND_REG)*/
+
 int tmio_core_mmc_enable(void __iomem *cnf, int shift, unsigned long base)
 {
 	/* Enable the MMC/SD Control registers */
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index f1a5c2357b14..03605f8fc0dc 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -53,7 +53,7 @@ config AD525X_DPOT_SPI
 
 config ATMEL_TCLIB
 	bool "Atmel AT32/AT91 Timer/Counter Library"
-	depends on (AVR32 || ARCH_AT91)
+	depends on ARCH_AT91
 	help
 	  Select this if you want a library to allocate the Timer/Counter
 	  blocks found on many Atmel processors.  This facilitates using
@@ -192,7 +192,7 @@ config ICS932S401
 
 config ATMEL_SSC
 	tristate "Device driver for Atmel SSC peripheral"
-	depends on HAS_IOMEM && (AVR32 || ARCH_AT91 || COMPILE_TEST)
+	depends on HAS_IOMEM && (ARCH_AT91 || COMPILE_TEST)
 	---help---
 	  This option enables device driver support for Atmel Synchronized
 	  Serial Communication peripheral (SSC).
@@ -496,6 +496,10 @@ config PCI_ENDPOINT_TEST
            Enable this configuration option to enable the host side test driver
            for PCI Endpoint.
 
+config MISC_RTSX
+	tristate
+	default MISC_RTSX_PCI || MISC_RTSX_USB
+
 source "drivers/misc/c2port/Kconfig"
 source "drivers/misc/eeprom/Kconfig"
 source "drivers/misc/cb710/Kconfig"
@@ -508,4 +512,6 @@ source "drivers/misc/mic/Kconfig"
 source "drivers/misc/genwqe/Kconfig"
 source "drivers/misc/echo/Kconfig"
 source "drivers/misc/cxl/Kconfig"
+source "drivers/misc/ocxl/Kconfig"
+source "drivers/misc/cardreader/Kconfig"
 endmenu
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index 5ca5f64df478..c3c8624f4d95 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -55,6 +55,8 @@ obj-$(CONFIG_CXL_BASE)		+= cxl/
 obj-$(CONFIG_ASPEED_LPC_CTRL)	+= aspeed-lpc-ctrl.o
 obj-$(CONFIG_ASPEED_LPC_SNOOP)	+= aspeed-lpc-snoop.o
 obj-$(CONFIG_PCI_ENDPOINT_TEST)	+= pci_endpoint_test.o
+obj-$(CONFIG_OCXL)		+= ocxl/
+obj-$(CONFIG_MISC_RTSX)		+= cardreader/
 
 lkdtm-$(CONFIG_LKDTM)		+= lkdtm_core.o
 lkdtm-$(CONFIG_LKDTM)		+= lkdtm_bugs.o
diff --git a/drivers/misc/ad525x_dpot.c b/drivers/misc/ad525x_dpot.c
index fe1672747bc1..bc591b7168db 100644
--- a/drivers/misc/ad525x_dpot.c
+++ b/drivers/misc/ad525x_dpot.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2009-2010 Analog Devices, Inc.
  * Author: Michael Hennerich <hennerich@blackfin.uclinux.org>
  *
- * DEVID		#Wipers		#Positions 	Resistor Options (kOhm)
+ * DEVID		#Wipers		#Positions	Resistor Options (kOhm)
  * AD5258		1		64		1, 10, 50, 100
  * AD5259		1		256		5, 10, 50, 100
  * AD5251		2		64		1, 10, 50, 100
@@ -84,12 +84,12 @@
 struct dpot_data {
 	struct ad_dpot_bus_data	bdata;
 	struct mutex update_lock;
-	unsigned rdac_mask;
-	unsigned max_pos;
+	unsigned int rdac_mask;
+	unsigned int max_pos;
 	unsigned long devid;
-	unsigned uid;
-	unsigned feat;
-	unsigned wipers;
+	unsigned int uid;
+	unsigned int feat;
+	unsigned int wipers;
 	u16 rdac_cache[MAX_RDACS];
 	DECLARE_BITMAP(otp_en_mask, MAX_RDACS);
 };
@@ -126,7 +126,7 @@ static inline int dpot_write_r8d16(struct dpot_data *dpot, u8 reg, u16 val)
 
 static s32 dpot_read_spi(struct dpot_data *dpot, u8 reg)
 {
-	unsigned ctrl = 0;
+	unsigned int ctrl = 0;
 	int value;
 
 	if (!(reg & (DPOT_ADDR_EEPROM | DPOT_ADDR_CMD))) {
@@ -175,7 +175,7 @@ static s32 dpot_read_spi(struct dpot_data *dpot, u8 reg)
 static s32 dpot_read_i2c(struct dpot_data *dpot, u8 reg)
 {
 	int value;
-	unsigned ctrl = 0;
+	unsigned int ctrl = 0;
 
 	switch (dpot->uid) {
 	case DPOT_UID(AD5246_ID):
@@ -238,7 +238,7 @@ static s32 dpot_read(struct dpot_data *dpot, u8 reg)
 
 static s32 dpot_write_spi(struct dpot_data *dpot, u8 reg, u16 value)
 {
-	unsigned val = 0;
+	unsigned int val = 0;
 
 	if (!(reg & (DPOT_ADDR_EEPROM | DPOT_ADDR_CMD | DPOT_ADDR_OTP))) {
 		if (dpot->feat & F_RDACS_WONLY)
@@ -328,7 +328,7 @@ static s32 dpot_write_spi(struct dpot_data *dpot, u8 reg, u16 value)
 static s32 dpot_write_i2c(struct dpot_data *dpot, u8 reg, u16 value)
 {
 	/* Only write the instruction byte for certain commands */
-	unsigned tmp = 0, ctrl = 0;
+	unsigned int tmp = 0, ctrl = 0;
 
 	switch (dpot->uid) {
 	case DPOT_UID(AD5246_ID):
@@ -515,11 +515,11 @@ set_##_name(struct device *dev, \
 #define DPOT_DEVICE_SHOW_SET(name, reg) \
 DPOT_DEVICE_SHOW(name, reg) \
 DPOT_DEVICE_SET(name, reg) \
-static DEVICE_ATTR(name, S_IWUSR | S_IRUGO, show_##name, set_##name);
+static DEVICE_ATTR(name, S_IWUSR | S_IRUGO, show_##name, set_##name)
 
 #define DPOT_DEVICE_SHOW_ONLY(name, reg) \
 DPOT_DEVICE_SHOW(name, reg) \
-static DEVICE_ATTR(name, S_IWUSR | S_IRUGO, show_##name, NULL);
+static DEVICE_ATTR(name, S_IWUSR | S_IRUGO, show_##name, NULL)
 
 DPOT_DEVICE_SHOW_SET(rdac0, DPOT_ADDR_RDAC | DPOT_RDAC0);
 DPOT_DEVICE_SHOW_SET(eeprom0, DPOT_ADDR_EEPROM | DPOT_RDAC0);
@@ -616,7 +616,7 @@ set_##_name(struct device *dev, \
 { \
 	return sysfs_do_cmd(dev, attr, buf, count, _cmd); \
 } \
-static DEVICE_ATTR(_name, S_IWUSR | S_IRUGO, NULL, set_##_name);
+static DEVICE_ATTR(_name, S_IWUSR | S_IRUGO, NULL, set_##_name)
 
 DPOT_DEVICE_DO_CMD(inc_all, DPOT_INC_ALL);
 DPOT_DEVICE_DO_CMD(dec_all, DPOT_DEC_ALL);
@@ -636,7 +636,7 @@ static const struct attribute_group ad525x_group_commands = {
 };
 
 static int ad_dpot_add_files(struct device *dev,
-		unsigned features, unsigned rdac)
+		unsigned int features, unsigned int rdac)
 {
 	int err = sysfs_create_file(&dev->kobj,
 		dpot_attrib_wipers[rdac]);
@@ -661,7 +661,7 @@ static int ad_dpot_add_files(struct device *dev,
 }
 
 static inline void ad_dpot_remove_files(struct device *dev,
-		unsigned features, unsigned rdac)
+		unsigned int features, unsigned int rdac)
 {
 	sysfs_remove_file(&dev->kobj,
 		dpot_attrib_wipers[rdac]);
diff --git a/drivers/misc/ad525x_dpot.h b/drivers/misc/ad525x_dpot.h
index 6bd1eba23bc0..443a51fd5680 100644
--- a/drivers/misc/ad525x_dpot.h
+++ b/drivers/misc/ad525x_dpot.h
@@ -195,12 +195,12 @@ enum dpot_devid {
 struct dpot_data;
 
 struct ad_dpot_bus_ops {
-	int (*read_d8) (void *client);
-	int (*read_r8d8) (void *client, u8 reg);
-	int (*read_r8d16) (void *client, u8 reg);
-	int (*write_d8) (void *client, u8 val);
-	int (*write_r8d8) (void *client, u8 reg, u8 val);
-	int (*write_r8d16) (void *client, u8 reg, u16 val);
+	int (*read_d8)(void *client);
+	int (*read_r8d8)(void *client, u8 reg);
+	int (*read_r8d16)(void *client, u8 reg);
+	int (*write_d8)(void *client, u8 val);
+	int (*write_r8d8)(void *client, u8 reg, u8 val);
+	int (*write_r8d16)(void *client, u8 reg, u16 val);
 };
 
 struct ad_dpot_bus_data {
diff --git a/drivers/misc/apds990x.c b/drivers/misc/apds990x.c
index c9f07032c2fc..ed9412d750b7 100644
--- a/drivers/misc/apds990x.c
+++ b/drivers/misc/apds990x.c
@@ -715,6 +715,7 @@ static ssize_t apds990x_rate_avail(struct device *dev,
 {
 	int i;
 	int pos = 0;
+
 	for (i = 0; i < ARRAY_SIZE(arates_hz); i++)
 		pos += sprintf(buf + pos, "%d ", arates_hz[i]);
 	sprintf(buf + pos - 1, "\n");
@@ -725,6 +726,7 @@ static ssize_t apds990x_rate_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	return sprintf(buf, "%d\n", chip->arate);
 }
 
@@ -784,6 +786,7 @@ static ssize_t apds990x_prox_show(struct device *dev,
 {
 	ssize_t ret;
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	if (pm_runtime_suspended(dev) || !chip->prox_en)
 		return -EIO;
 
@@ -807,6 +810,7 @@ static ssize_t apds990x_prox_enable_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	return sprintf(buf, "%d\n", chip->prox_en);
 }
 
@@ -847,6 +851,7 @@ static ssize_t apds990x_prox_reporting_mode_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	return sprintf(buf, "%s\n",
 		reporting_modes[!!chip->prox_continuous_mode]);
 }
@@ -884,6 +889,7 @@ static ssize_t apds990x_lux_thresh_above_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	return sprintf(buf, "%d\n", chip->lux_thres_hi);
 }
 
@@ -891,6 +897,7 @@ static ssize_t apds990x_lux_thresh_below_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	return sprintf(buf, "%d\n", chip->lux_thres_lo);
 }
 
@@ -926,6 +933,7 @@ static ssize_t apds990x_lux_thresh_above_store(struct device *dev,
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
 	int ret = apds990x_set_lux_thresh(chip, &chip->lux_thres_hi, buf);
+
 	if (ret < 0)
 		return ret;
 	return len;
@@ -937,6 +945,7 @@ static ssize_t apds990x_lux_thresh_below_store(struct device *dev,
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
 	int ret = apds990x_set_lux_thresh(chip, &chip->lux_thres_lo, buf);
+
 	if (ret < 0)
 		return ret;
 	return len;
@@ -954,6 +963,7 @@ static ssize_t apds990x_prox_threshold_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	return sprintf(buf, "%d\n", chip->prox_thres);
 }
 
@@ -1026,6 +1036,7 @@ static ssize_t apds990x_chip_id_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct apds990x_chip *chip =  dev_get_drvdata(dev);
+
 	return sprintf(buf, "%s %d\n", chip->chipname, chip->revision);
 }
 
diff --git a/drivers/misc/cardreader/Kconfig b/drivers/misc/cardreader/Kconfig
new file mode 100644
index 000000000000..69e815e32a8c
--- /dev/null
+++ b/drivers/misc/cardreader/Kconfig
@@ -0,0 +1,20 @@
+config MISC_RTSX_PCI
+	tristate "Realtek PCI-E card reader"
+	depends on PCI
+	select MFD_CORE
+	help
+	  This supports for Realtek PCI-Express card reader including rts5209,
+	  rts5227, rts522A, rts5229, rts5249, rts524A, rts525A, rtl8411, rts5260.
+	  Realtek card readers support access to many types of memory cards,
+	  such as Memory Stick, Memory Stick Pro, Secure Digital and
+	  MultiMediaCard.
+
+config MISC_RTSX_USB
+	tristate "Realtek USB card reader"
+	depends on USB
+	select MFD_CORE
+	help
+	  Select this option to get support for Realtek USB 2.0 card readers
+	  including RTS5129, RTS5139, RTS5179 and RTS5170.
+	  Realtek card reader supports access to many types of memory cards,
+	  such as Memory Stick Pro, Secure Digital and MultiMediaCard.
diff --git a/drivers/misc/cardreader/Makefile b/drivers/misc/cardreader/Makefile
new file mode 100644
index 000000000000..9fabfcc6fa7a
--- /dev/null
+++ b/drivers/misc/cardreader/Makefile
@@ -0,0 +1,4 @@
+rtsx_pci-objs := rtsx_pcr.o rts5209.o rts5229.o rtl8411.o rts5227.o rts5249.o rts5260.o
+
+obj-$(CONFIG_MISC_RTSX_PCI)	+= rtsx_pci.o
+obj-$(CONFIG_MISC_RTSX_USB)	+= rtsx_usb.o
diff --git a/drivers/mfd/rtl8411.c b/drivers/misc/cardreader/rtl8411.c
similarity index 99%
rename from drivers/mfd/rtl8411.c
rename to drivers/misc/cardreader/rtl8411.c
index b3ae6592014a..434fd070d3e3 100644
--- a/drivers/mfd/rtl8411.c
+++ b/drivers/misc/cardreader/rtl8411.c
@@ -23,7 +23,7 @@
 #include <linux/module.h>
 #include <linux/bitops.h>
 #include <linux/delay.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 
 #include "rtsx_pcr.h"
 
diff --git a/drivers/mfd/rts5209.c b/drivers/misc/cardreader/rts5209.c
similarity index 99%
rename from drivers/mfd/rts5209.c
rename to drivers/misc/cardreader/rts5209.c
index b95beecf767f..ce68c48d8ec9 100644
--- a/drivers/mfd/rts5209.c
+++ b/drivers/misc/cardreader/rts5209.c
@@ -21,7 +21,7 @@
 
 #include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 
 #include "rtsx_pcr.h"
 
diff --git a/drivers/mfd/rts5227.c b/drivers/misc/cardreader/rts5227.c
similarity index 99%
rename from drivers/mfd/rts5227.c
rename to drivers/misc/cardreader/rts5227.c
index ff296a4bf3d2..024dcba8d6c8 100644
--- a/drivers/mfd/rts5227.c
+++ b/drivers/misc/cardreader/rts5227.c
@@ -22,7 +22,7 @@
 
 #include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 
 #include "rtsx_pcr.h"
 
diff --git a/drivers/mfd/rts5229.c b/drivers/misc/cardreader/rts5229.c
similarity index 99%
rename from drivers/mfd/rts5229.c
rename to drivers/misc/cardreader/rts5229.c
index 9ed9dc84eac8..9119261337cc 100644
--- a/drivers/mfd/rts5229.c
+++ b/drivers/misc/cardreader/rts5229.c
@@ -21,7 +21,7 @@
 
 #include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 
 #include "rtsx_pcr.h"
 
diff --git a/drivers/mfd/rts5249.c b/drivers/misc/cardreader/rts5249.c
similarity index 99%
rename from drivers/mfd/rts5249.c
rename to drivers/misc/cardreader/rts5249.c
index 7fcf37ba922c..dbe013abdb83 100644
--- a/drivers/mfd/rts5249.c
+++ b/drivers/misc/cardreader/rts5249.c
@@ -21,7 +21,7 @@
 
 #include <linux/module.h>
 #include <linux/delay.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 
 #include "rtsx_pcr.h"
 
@@ -738,4 +738,3 @@ void rts525a_init_params(struct rtsx_pcr *pcr)
 	pcr->reg_pm_ctrl3 = RTS524A_PM_CTRL3;
 	pcr->ops = &rts525a_pcr_ops;
 }
-
diff --git a/drivers/misc/cardreader/rts5260.c b/drivers/misc/cardreader/rts5260.c
new file mode 100644
index 000000000000..07cb93abf685
--- /dev/null
+++ b/drivers/misc/cardreader/rts5260.c
@@ -0,0 +1,748 @@
+/* Driver for Realtek PCI-Express card reader
+ *
+ * Copyright(c) 2016-2017 Realtek Semiconductor Corp. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2, or (at your option) any
+ * later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Author:
+ *   Steven FENG <steven_feng@realsil.com.cn>
+ *   Rui FENG <rui_feng@realsil.com.cn>
+ *   Wei WANG <wei_wang@realsil.com.cn>
+ */
+
+#include <linux/module.h>
+#include <linux/delay.h>
+#include <linux/rtsx_pci.h>
+
+#include "rts5260.h"
+#include "rtsx_pcr.h"
+
+static u8 rts5260_get_ic_version(struct rtsx_pcr *pcr)
+{
+	u8 val;
+
+	rtsx_pci_read_register(pcr, DUMMY_REG_RESET_0, &val);
+	return val & IC_VERSION_MASK;
+}
+
+static void rts5260_fill_driving(struct rtsx_pcr *pcr, u8 voltage)
+{
+	u8 driving_3v3[6][3] = {
+		{0x94, 0x94, 0x94},
+		{0x11, 0x11, 0x18},
+		{0x55, 0x55, 0x5C},
+		{0x94, 0x94, 0x94},
+		{0x94, 0x94, 0x94},
+		{0xFF, 0xFF, 0xFF},
+	};
+	u8 driving_1v8[6][3] = {
+		{0x9A, 0x89, 0x89},
+		{0xC4, 0xC4, 0xC4},
+		{0x3C, 0x3C, 0x3C},
+		{0x9B, 0x99, 0x99},
+		{0x9A, 0x89, 0x89},
+		{0xFE, 0xFE, 0xFE},
+	};
+	u8 (*driving)[3], drive_sel;
+
+	if (voltage == OUTPUT_3V3) {
+		driving = driving_3v3;
+		drive_sel = pcr->sd30_drive_sel_3v3;
+	} else {
+		driving = driving_1v8;
+		drive_sel = pcr->sd30_drive_sel_1v8;
+	}
+
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD30_CLK_DRIVE_SEL,
+			 0xFF, driving[drive_sel][0]);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD30_CMD_DRIVE_SEL,
+			 0xFF, driving[drive_sel][1]);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, SD30_DAT_DRIVE_SEL,
+			 0xFF, driving[drive_sel][2]);
+}
+
+static void rtsx_base_fetch_vendor_settings(struct rtsx_pcr *pcr)
+{
+	u32 reg;
+
+	rtsx_pci_read_config_dword(pcr, PCR_SETTING_REG1, &reg);
+	pcr_dbg(pcr, "Cfg 0x%x: 0x%x\n", PCR_SETTING_REG1, reg);
+
+	if (!rtsx_vendor_setting_valid(reg)) {
+		pcr_dbg(pcr, "skip fetch vendor setting\n");
+		return;
+	}
+
+	pcr->aspm_en = rtsx_reg_to_aspm(reg);
+	pcr->sd30_drive_sel_1v8 = rtsx_reg_to_sd30_drive_sel_1v8(reg);
+	pcr->card_drive_sel &= 0x3F;
+	pcr->card_drive_sel |= rtsx_reg_to_card_drive_sel(reg);
+
+	rtsx_pci_read_config_dword(pcr, PCR_SETTING_REG2, &reg);
+	pcr_dbg(pcr, "Cfg 0x%x: 0x%x\n", PCR_SETTING_REG2, reg);
+	pcr->sd30_drive_sel_3v3 = rtsx_reg_to_sd30_drive_sel_3v3(reg);
+	if (rtsx_reg_check_reverse_socket(reg))
+		pcr->flags |= PCR_REVERSE_SOCKET;
+}
+
+static void rtsx_base_force_power_down(struct rtsx_pcr *pcr, u8 pm_state)
+{
+	/* Set relink_time to 0 */
+	rtsx_pci_write_register(pcr, AUTOLOAD_CFG_BASE + 1, MASK_8_BIT_DEF, 0);
+	rtsx_pci_write_register(pcr, AUTOLOAD_CFG_BASE + 2, MASK_8_BIT_DEF, 0);
+	rtsx_pci_write_register(pcr, AUTOLOAD_CFG_BASE + 3,
+				RELINK_TIME_MASK, 0);
+
+	if (pm_state == HOST_ENTER_S3)
+		rtsx_pci_write_register(pcr, pcr->reg_pm_ctrl3,
+					D3_DELINK_MODE_EN, D3_DELINK_MODE_EN);
+
+	rtsx_pci_write_register(pcr, FPDCTL, ALL_POWER_DOWN, ALL_POWER_DOWN);
+}
+
+static int rtsx_base_enable_auto_blink(struct rtsx_pcr *pcr)
+{
+	return rtsx_pci_write_register(pcr, OLT_LED_CTL,
+		LED_SHINE_MASK, LED_SHINE_EN);
+}
+
+static int rtsx_base_disable_auto_blink(struct rtsx_pcr *pcr)
+{
+	return rtsx_pci_write_register(pcr, OLT_LED_CTL,
+		LED_SHINE_MASK, LED_SHINE_DISABLE);
+}
+
+static int rts5260_turn_on_led(struct rtsx_pcr *pcr)
+{
+	return rtsx_pci_write_register(pcr, RTS5260_REG_GPIO_CTL0,
+		RTS5260_REG_GPIO_MASK, RTS5260_REG_GPIO_ON);
+}
+
+static int rts5260_turn_off_led(struct rtsx_pcr *pcr)
+{
+	return rtsx_pci_write_register(pcr, RTS5260_REG_GPIO_CTL0,
+		RTS5260_REG_GPIO_MASK, RTS5260_REG_GPIO_OFF);
+}
+
+/* SD Pull Control Enable:
+ *     SD_DAT[3:0] ==> pull up
+ *     SD_CD       ==> pull up
+ *     SD_WP       ==> pull up
+ *     SD_CMD      ==> pull up
+ *     SD_CLK      ==> pull down
+ */
+static const u32 rts5260_sd_pull_ctl_enable_tbl[] = {
+	RTSX_REG_PAIR(CARD_PULL_CTL1, 0x66),
+	RTSX_REG_PAIR(CARD_PULL_CTL2, 0xAA),
+	RTSX_REG_PAIR(CARD_PULL_CTL3, 0xE9),
+	RTSX_REG_PAIR(CARD_PULL_CTL4, 0xAA),
+	0,
+};
+
+/* SD Pull Control Disable:
+ *     SD_DAT[3:0] ==> pull down
+ *     SD_CD       ==> pull up
+ *     SD_WP       ==> pull down
+ *     SD_CMD      ==> pull down
+ *     SD_CLK      ==> pull down
+ */
+static const u32 rts5260_sd_pull_ctl_disable_tbl[] = {
+	RTSX_REG_PAIR(CARD_PULL_CTL1, 0x66),
+	RTSX_REG_PAIR(CARD_PULL_CTL2, 0x55),
+	RTSX_REG_PAIR(CARD_PULL_CTL3, 0xD5),
+	RTSX_REG_PAIR(CARD_PULL_CTL4, 0x55),
+	0,
+};
+
+/* MS Pull Control Enable:
+ *     MS CD       ==> pull up
+ *     others      ==> pull down
+ */
+static const u32 rts5260_ms_pull_ctl_enable_tbl[] = {
+	RTSX_REG_PAIR(CARD_PULL_CTL4, 0x55),
+	RTSX_REG_PAIR(CARD_PULL_CTL5, 0x55),
+	RTSX_REG_PAIR(CARD_PULL_CTL6, 0x15),
+	0,
+};
+
+/* MS Pull Control Disable:
+ *     MS CD       ==> pull up
+ *     others      ==> pull down
+ */
+static const u32 rts5260_ms_pull_ctl_disable_tbl[] = {
+	RTSX_REG_PAIR(CARD_PULL_CTL4, 0x55),
+	RTSX_REG_PAIR(CARD_PULL_CTL5, 0x55),
+	RTSX_REG_PAIR(CARD_PULL_CTL6, 0x15),
+	0,
+};
+
+static int sd_set_sample_push_timing_sd30(struct rtsx_pcr *pcr)
+{
+	rtsx_pci_write_register(pcr, SD_CFG1, SD_MODE_SELECT_MASK
+		| SD_ASYNC_FIFO_NOT_RST, SD_30_MODE | SD_ASYNC_FIFO_NOT_RST);
+	rtsx_pci_write_register(pcr, CLK_CTL, CLK_LOW_FREQ, CLK_LOW_FREQ);
+	rtsx_pci_write_register(pcr, CARD_CLK_SOURCE, 0xFF,
+				CRC_VAR_CLK0 | SD30_FIX_CLK | SAMPLE_VAR_CLK1);
+	rtsx_pci_write_register(pcr, CLK_CTL, CLK_LOW_FREQ, 0);
+
+	return 0;
+}
+
+static int rts5260_card_power_on(struct rtsx_pcr *pcr, int card)
+{
+	int err = 0;
+	struct rtsx_cr_option *option = &pcr->option;
+
+	if (option->ocp_en)
+		rtsx_pci_enable_ocp(pcr);
+
+	rtsx_pci_init_cmd(pcr);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, LDO_CONFIG2,
+			 DV331812_VDD1, DV331812_VDD1);
+	err = rtsx_pci_send_cmd(pcr, CMD_TIMEOUT_DEF);
+	if (err < 0)
+		return err;
+
+	rtsx_pci_init_cmd(pcr);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, LDO_VCC_CFG0,
+			 RTS5260_DVCC_TUNE_MASK, RTS5260_DVCC_33);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, LDO_VCC_CFG1,
+			 LDO_POW_SDVDD1_MASK, LDO_POW_SDVDD1_ON);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, LDO_CONFIG2,
+			 DV331812_POWERON, DV331812_POWERON);
+	err = rtsx_pci_send_cmd(pcr, CMD_TIMEOUT_DEF);
+
+	msleep(20);
+
+	if (pcr->extra_caps & EXTRA_CAPS_SD_SDR50 ||
+	    pcr->extra_caps & EXTRA_CAPS_SD_SDR104)
+		sd_set_sample_push_timing_sd30(pcr);
+
+	/* Initialize SD_CFG1 register */
+	rtsx_pci_write_register(pcr, SD_CFG1, 0xFF,
+				SD_CLK_DIVIDE_128 | SD_20_MODE);
+
+	rtsx_pci_write_register(pcr, SD_SAMPLE_POINT_CTL,
+				0xFF, SD20_RX_POS_EDGE);
+	rtsx_pci_write_register(pcr, SD_PUSH_POINT_CTL, 0xFF, 0);
+	rtsx_pci_write_register(pcr, CARD_STOP, SD_STOP | SD_CLR_ERR,
+				SD_STOP | SD_CLR_ERR);
+
+	/* Reset SD_CFG3 register */
+	rtsx_pci_write_register(pcr, SD_CFG3, SD30_CLK_END_EN, 0);
+	rtsx_pci_write_register(pcr, REG_SD_STOP_SDCLK_CFG,
+				SD30_CLK_STOP_CFG_EN | SD30_CLK_STOP_CFG1 |
+				SD30_CLK_STOP_CFG0, 0);
+
+	rtsx_pci_write_register(pcr, REG_PRE_RW_MODE, EN_INFINITE_MODE, 0);
+
+	return err;
+}
+
+static int rts5260_switch_output_voltage(struct rtsx_pcr *pcr, u8 voltage)
+{
+	switch (voltage) {
+	case OUTPUT_3V3:
+		rtsx_pci_write_register(pcr, LDO_CONFIG2,
+					DV331812_VDD1, DV331812_VDD1);
+		rtsx_pci_write_register(pcr, LDO_DV18_CFG,
+					DV331812_MASK, DV331812_33);
+		rtsx_pci_write_register(pcr, SD_PAD_CTL, SD_IO_USING_1V8, 0);
+		break;
+	case OUTPUT_1V8:
+		rtsx_pci_write_register(pcr, LDO_CONFIG2,
+					DV331812_VDD1, DV331812_VDD1);
+		rtsx_pci_write_register(pcr, LDO_DV18_CFG,
+					DV331812_MASK, DV331812_17);
+		rtsx_pci_write_register(pcr, SD_PAD_CTL, SD_IO_USING_1V8,
+					SD_IO_USING_1V8);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* set pad drive */
+	rtsx_pci_init_cmd(pcr);
+	rts5260_fill_driving(pcr, voltage);
+	return rtsx_pci_send_cmd(pcr, CMD_TIMEOUT_DEF);
+}
+
+static void rts5260_stop_cmd(struct rtsx_pcr *pcr)
+{
+	rtsx_pci_writel(pcr, RTSX_HCBCTLR, STOP_CMD);
+	rtsx_pci_writel(pcr, RTSX_HDBCTLR, STOP_DMA);
+	rtsx_pci_write_register(pcr, RTS5260_DMA_RST_CTL_0,
+				RTS5260_DMA_RST | RTS5260_ADMA3_RST,
+				RTS5260_DMA_RST | RTS5260_ADMA3_RST);
+	rtsx_pci_write_register(pcr, RBCTL, RB_FLUSH, RB_FLUSH);
+}
+
+static void rts5260_card_before_power_off(struct rtsx_pcr *pcr)
+{
+	struct rtsx_cr_option *option = &pcr->option;
+
+	rts5260_stop_cmd(pcr);
+	rts5260_switch_output_voltage(pcr, OUTPUT_3V3);
+
+	if (option->ocp_en)
+		rtsx_pci_disable_ocp(pcr);
+}
+
+static int rts5260_card_power_off(struct rtsx_pcr *pcr, int card)
+{
+	int err = 0;
+
+	rts5260_card_before_power_off(pcr);
+
+	rtsx_pci_init_cmd(pcr);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, LDO_VCC_CFG1,
+			 LDO_POW_SDVDD1_MASK, LDO_POW_SDVDD1_OFF);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, LDO_CONFIG2,
+			 DV331812_POWERON, DV331812_POWEROFF);
+	err = rtsx_pci_send_cmd(pcr, CMD_TIMEOUT_DEF);
+
+	return err;
+}
+
+static void rts5260_init_ocp(struct rtsx_pcr *pcr)
+{
+	struct rtsx_cr_option *option = &pcr->option;
+
+	if (option->ocp_en) {
+		u8 mask, val;
+
+		rtsx_pci_write_register(pcr, RTS5260_DVCC_CTRL,
+					RTS5260_DVCC_OCP_EN |
+					RTS5260_DVCC_OCP_CL_EN,
+					RTS5260_DVCC_OCP_EN |
+					RTS5260_DVCC_OCP_CL_EN);
+		rtsx_pci_write_register(pcr, RTS5260_DVIO_CTRL,
+					RTS5260_DVIO_OCP_EN |
+					RTS5260_DVIO_OCP_CL_EN,
+					RTS5260_DVIO_OCP_EN |
+					RTS5260_DVIO_OCP_CL_EN);
+
+		rtsx_pci_write_register(pcr, RTS5260_DVCC_CTRL,
+					RTS5260_DVCC_OCP_THD_MASK,
+					option->sd_400mA_ocp_thd);
+
+		rtsx_pci_write_register(pcr, RTS5260_DVIO_CTRL,
+					RTS5260_DVIO_OCP_THD_MASK,
+					RTS5260_DVIO_OCP_THD_350);
+
+		rtsx_pci_write_register(pcr, RTS5260_DV331812_CFG,
+					RTS5260_DV331812_OCP_THD_MASK,
+					RTS5260_DV331812_OCP_THD_210);
+
+		mask = SD_OCP_GLITCH_MASK | SDVIO_OCP_GLITCH_MASK;
+		val = pcr->hw_param.ocp_glitch;
+		rtsx_pci_write_register(pcr, REG_OCPGLITCH, mask, val);
+
+		rtsx_pci_enable_ocp(pcr);
+	} else {
+		rtsx_pci_write_register(pcr, RTS5260_DVCC_CTRL,
+					RTS5260_DVCC_OCP_EN |
+					RTS5260_DVCC_OCP_CL_EN, 0);
+		rtsx_pci_write_register(pcr, RTS5260_DVIO_CTRL,
+					RTS5260_DVIO_OCP_EN |
+					RTS5260_DVIO_OCP_CL_EN, 0);
+	}
+}
+
+static void rts5260_enable_ocp(struct rtsx_pcr *pcr)
+{
+	u8 val = 0;
+
+	rtsx_pci_write_register(pcr, FPDCTL, OC_POWER_DOWN, 0);
+
+	val = SD_OCP_INT_EN | SD_DETECT_EN;
+	val |= SDVIO_OCP_INT_EN | SDVIO_DETECT_EN;
+	rtsx_pci_write_register(pcr, REG_OCPCTL, 0xFF, val);
+	rtsx_pci_write_register(pcr, REG_DV3318_OCPCTL,
+				DV3318_DETECT_EN | DV3318_OCP_INT_EN,
+				DV3318_DETECT_EN | DV3318_OCP_INT_EN);
+}
+
+static void rts5260_disable_ocp(struct rtsx_pcr *pcr)
+{
+	u8 mask = 0;
+
+	mask = SD_OCP_INT_EN | SD_DETECT_EN;
+	mask |= SDVIO_OCP_INT_EN | SDVIO_DETECT_EN;
+	rtsx_pci_write_register(pcr, REG_OCPCTL, mask, 0);
+	rtsx_pci_write_register(pcr, REG_DV3318_OCPCTL,
+				DV3318_DETECT_EN | DV3318_OCP_INT_EN, 0);
+
+	rtsx_pci_write_register(pcr, FPDCTL, OC_POWER_DOWN,
+				OC_POWER_DOWN);
+}
+
+int rts5260_get_ocpstat(struct rtsx_pcr *pcr, u8 *val)
+{
+	return rtsx_pci_read_register(pcr, REG_OCPSTAT, val);
+}
+
+int rts5260_get_ocpstat2(struct rtsx_pcr *pcr, u8 *val)
+{
+	return rtsx_pci_read_register(pcr, REG_DV3318_OCPSTAT, val);
+}
+
+void rts5260_clear_ocpstat(struct rtsx_pcr *pcr)
+{
+	u8 mask = 0;
+	u8 val = 0;
+
+	mask = SD_OCP_INT_CLR | SD_OC_CLR;
+	mask |= SDVIO_OCP_INT_CLR | SDVIO_OC_CLR;
+	val = SD_OCP_INT_CLR | SD_OC_CLR;
+	val |= SDVIO_OCP_INT_CLR | SDVIO_OC_CLR;
+
+	rtsx_pci_write_register(pcr, REG_OCPCTL, mask, val);
+	rtsx_pci_write_register(pcr, REG_DV3318_OCPCTL,
+				DV3318_OCP_INT_CLR | DV3318_OCP_CLR,
+				DV3318_OCP_INT_CLR | DV3318_OCP_CLR);
+	udelay(10);
+	rtsx_pci_write_register(pcr, REG_OCPCTL, mask, 0);
+	rtsx_pci_write_register(pcr, REG_DV3318_OCPCTL,
+				DV3318_OCP_INT_CLR | DV3318_OCP_CLR, 0);
+}
+
+void rts5260_process_ocp(struct rtsx_pcr *pcr)
+{
+	if (!pcr->option.ocp_en)
+		return;
+
+	rtsx_pci_get_ocpstat(pcr, &pcr->ocp_stat);
+	rts5260_get_ocpstat2(pcr, &pcr->ocp_stat2);
+	if (pcr->card_exist & SD_EXIST)
+		rtsx_sd_power_off_card3v3(pcr);
+	else if (pcr->card_exist & MS_EXIST)
+		rtsx_ms_power_off_card3v3(pcr);
+
+	if (!(pcr->card_exist & MS_EXIST) && !(pcr->card_exist & SD_EXIST)) {
+		if ((pcr->ocp_stat & (SD_OC_NOW | SD_OC_EVER |
+			SDVIO_OC_NOW | SDVIO_OC_EVER)) ||
+			(pcr->ocp_stat2 & (DV3318_OCP_NOW | DV3318_OCP_EVER)))
+			rtsx_pci_clear_ocpstat(pcr);
+		pcr->ocp_stat = 0;
+		pcr->ocp_stat2 = 0;
+	}
+
+	if ((pcr->ocp_stat & (SD_OC_NOW | SD_OC_EVER |
+			SDVIO_OC_NOW | SDVIO_OC_EVER)) ||
+			(pcr->ocp_stat2 & (DV3318_OCP_NOW | DV3318_OCP_EVER))) {
+		if (pcr->card_exist & SD_EXIST)
+			rtsx_pci_write_register(pcr, CARD_OE, SD_OUTPUT_EN, 0);
+		else if (pcr->card_exist & MS_EXIST)
+			rtsx_pci_write_register(pcr, CARD_OE, MS_OUTPUT_EN, 0);
+	}
+}
+
+int rts5260_init_hw(struct rtsx_pcr *pcr)
+{
+	int err;
+
+	rtsx_pci_init_ocp(pcr);
+
+	rtsx_pci_init_cmd(pcr);
+
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, L1SUB_CONFIG1,
+			 AUX_CLK_ACTIVE_SEL_MASK, MAC_CKSW_DONE);
+	/* Rest L1SUB Config */
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, L1SUB_CONFIG3, 0xFF, 0x00);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, PM_CLK_FORCE_CTL,
+			 CLK_PM_EN, CLK_PM_EN);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, PWD_SUSPEND_EN, 0xFF, 0xFF);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, PWR_GATE_CTRL,
+			 PWR_GATE_EN, PWR_GATE_EN);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, REG_VREF,
+			 PWD_SUSPND_EN, PWD_SUSPND_EN);
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, RBCTL,
+			 U_AUTO_DMA_EN_MASK, U_AUTO_DMA_DISABLE);
+
+	if (pcr->flags & PCR_REVERSE_SOCKET)
+		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, PETXCFG, 0xB0, 0xB0);
+	else
+		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, PETXCFG, 0xB0, 0x80);
+
+	rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, OBFF_CFG,
+			 OBFF_EN_MASK, OBFF_DISABLE);
+
+	err = rtsx_pci_send_cmd(pcr, CMD_TIMEOUT_DEF);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static void rts5260_pwr_saving_setting(struct rtsx_pcr *pcr)
+{
+	int lss_l1_1, lss_l1_2;
+
+	lss_l1_1 = rtsx_check_dev_flag(pcr, ASPM_L1_1_EN)
+			| rtsx_check_dev_flag(pcr, PM_L1_1_EN);
+	lss_l1_2 = rtsx_check_dev_flag(pcr, ASPM_L1_2_EN)
+			| rtsx_check_dev_flag(pcr, PM_L1_2_EN);
+
+	if (lss_l1_2) {
+		pcr_dbg(pcr, "Set parameters for L1.2.");
+		rtsx_pci_write_register(pcr, PWR_GLOBAL_CTRL,
+					0xFF, PCIE_L1_2_EN);
+		rtsx_pci_write_register(pcr, PWR_FE_CTL,
+					0xFF, PCIE_L1_2_PD_FE_EN);
+	} else if (lss_l1_1) {
+		pcr_dbg(pcr, "Set parameters for L1.1.");
+		rtsx_pci_write_register(pcr, PWR_GLOBAL_CTRL,
+					0xFF, PCIE_L1_1_EN);
+		rtsx_pci_write_register(pcr, PWR_FE_CTL,
+					0xFF, PCIE_L1_1_PD_FE_EN);
+	} else {
+		pcr_dbg(pcr, "Set parameters for L1.");
+		rtsx_pci_write_register(pcr, PWR_GLOBAL_CTRL,
+					0xFF, PCIE_L1_0_EN);
+		rtsx_pci_write_register(pcr, PWR_FE_CTL,
+					0xFF, PCIE_L1_0_PD_FE_EN);
+	}
+
+	rtsx_pci_write_register(pcr, CFG_L1_0_PCIE_DPHY_RET_VALUE,
+				0xFF, CFG_L1_0_RET_VALUE_DEFAULT);
+	rtsx_pci_write_register(pcr, CFG_L1_0_PCIE_MAC_RET_VALUE,
+				0xFF, CFG_L1_0_RET_VALUE_DEFAULT);
+	rtsx_pci_write_register(pcr, CFG_L1_0_CRC_SD30_RET_VALUE,
+				0xFF, CFG_L1_0_RET_VALUE_DEFAULT);
+	rtsx_pci_write_register(pcr, CFG_L1_0_CRC_SD40_RET_VALUE,
+				0xFF, CFG_L1_0_RET_VALUE_DEFAULT);
+	rtsx_pci_write_register(pcr, CFG_L1_0_SYS_RET_VALUE,
+				0xFF, CFG_L1_0_RET_VALUE_DEFAULT);
+	/*Option cut APHY*/
+	rtsx_pci_write_register(pcr, CFG_PCIE_APHY_OFF_0,
+				0xFF, CFG_PCIE_APHY_OFF_0_DEFAULT);
+	rtsx_pci_write_register(pcr, CFG_PCIE_APHY_OFF_1,
+				0xFF, CFG_PCIE_APHY_OFF_1_DEFAULT);
+	rtsx_pci_write_register(pcr, CFG_PCIE_APHY_OFF_2,
+				0xFF, CFG_PCIE_APHY_OFF_2_DEFAULT);
+	rtsx_pci_write_register(pcr, CFG_PCIE_APHY_OFF_3,
+				0xFF, CFG_PCIE_APHY_OFF_3_DEFAULT);
+	/*CDR DEC*/
+	rtsx_pci_write_register(pcr, PWC_CDR, 0xFF, PWC_CDR_DEFAULT);
+	/*PWMPFM*/
+	rtsx_pci_write_register(pcr, CFG_LP_FPWM_VALUE,
+				0xFF, CFG_LP_FPWM_VALUE_DEFAULT);
+	/*No Power Saving WA*/
+	rtsx_pci_write_register(pcr, CFG_L1_0_CRC_MISC_RET_VALUE,
+				0xFF, CFG_L1_0_CRC_MISC_RET_VALUE_DEFAULT);
+}
+
+static void rts5260_init_from_cfg(struct rtsx_pcr *pcr)
+{
+	struct rtsx_cr_option *option = &pcr->option;
+	u32 lval;
+
+	rtsx_pci_read_config_dword(pcr, PCR_ASPM_SETTING_5260, &lval);
+
+	if (lval & ASPM_L1_1_EN_MASK)
+		rtsx_set_dev_flag(pcr, ASPM_L1_1_EN);
+
+	if (lval & ASPM_L1_2_EN_MASK)
+		rtsx_set_dev_flag(pcr, ASPM_L1_2_EN);
+
+	if (lval & PM_L1_1_EN_MASK)
+		rtsx_set_dev_flag(pcr, PM_L1_1_EN);
+
+	if (lval & PM_L1_2_EN_MASK)
+		rtsx_set_dev_flag(pcr, PM_L1_2_EN);
+
+	rts5260_pwr_saving_setting(pcr);
+
+	if (option->ltr_en) {
+		u16 val;
+
+		pcie_capability_read_word(pcr->pci, PCI_EXP_DEVCTL2, &val);
+		if (val & PCI_EXP_DEVCTL2_LTR_EN) {
+			option->ltr_enabled = true;
+			option->ltr_active = true;
+			rtsx_set_ltr_latency(pcr, option->ltr_active_latency);
+		} else {
+			option->ltr_enabled = false;
+		}
+	}
+
+	if (rtsx_check_dev_flag(pcr, ASPM_L1_1_EN | ASPM_L1_2_EN
+				| PM_L1_1_EN | PM_L1_2_EN))
+		option->force_clkreq_0 = false;
+	else
+		option->force_clkreq_0 = true;
+}
+
+static int rts5260_extra_init_hw(struct rtsx_pcr *pcr)
+{
+	struct rtsx_cr_option *option = &pcr->option;
+
+	/* Set mcu_cnt to 7 to ensure data can be sampled properly */
+	rtsx_pci_write_register(pcr, 0xFC03, 0x7F, 0x07);
+	rtsx_pci_write_register(pcr, SSC_DIV_N_0, 0xFF, 0x5D);
+
+	rts5260_init_from_cfg(pcr);
+
+	/* force no MDIO*/
+	rtsx_pci_write_register(pcr, RTS5260_AUTOLOAD_CFG4,
+				0xFF, RTS5260_MIMO_DISABLE);
+	/*Modify SDVCC Tune Default Parameters!*/
+	rtsx_pci_write_register(pcr, LDO_VCC_CFG0,
+				RTS5260_DVCC_TUNE_MASK, RTS5260_DVCC_33);
+
+	rtsx_pci_write_register(pcr, PCLK_CTL, PCLK_MODE_SEL, PCLK_MODE_SEL);
+
+	rts5260_init_hw(pcr);
+
+	/*
+	 * If u_force_clkreq_0 is enabled, CLKREQ# PIN will be forced
+	 * to drive low, and we forcibly request clock.
+	 */
+	if (option->force_clkreq_0)
+		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, PETXCFG,
+				 FORCE_CLKREQ_DELINK_MASK, FORCE_CLKREQ_LOW);
+	else
+		rtsx_pci_add_cmd(pcr, WRITE_REG_CMD, PETXCFG,
+				 FORCE_CLKREQ_DELINK_MASK, FORCE_CLKREQ_HIGH);
+
+	return 0;
+}
+
+void rts5260_set_aspm(struct rtsx_pcr *pcr, bool enable)
+{
+	struct rtsx_cr_option *option = &pcr->option;
+	u8 val = 0;
+
+	if (pcr->aspm_enabled == enable)
+		return;
+
+	if (option->dev_aspm_mode == DEV_ASPM_DYNAMIC) {
+		if (enable)
+			val = pcr->aspm_en;
+		rtsx_pci_update_cfg_byte(pcr, pcr->pcie_cap + PCI_EXP_LNKCTL,
+					 ASPM_MASK_NEG, val);
+	} else if (option->dev_aspm_mode == DEV_ASPM_BACKDOOR) {
+		u8 mask = FORCE_ASPM_VAL_MASK | FORCE_ASPM_CTL0;
+
+		if (!enable)
+			val = FORCE_ASPM_CTL0;
+		rtsx_pci_write_register(pcr, ASPM_FORCE_CTL, mask, val);
+	}
+
+	pcr->aspm_enabled = enable;
+}
+
+static void rts5260_set_l1off_cfg_sub_d0(struct rtsx_pcr *pcr, int active)
+{
+	struct rtsx_cr_option *option = &pcr->option;
+	u32 interrupt = rtsx_pci_readl(pcr, RTSX_BIPR);
+	int card_exist = (interrupt & SD_EXIST) | (interrupt & MS_EXIST);
+	int aspm_L1_1, aspm_L1_2;
+	u8 val = 0;
+
+	aspm_L1_1 = rtsx_check_dev_flag(pcr, ASPM_L1_1_EN);
+	aspm_L1_2 = rtsx_check_dev_flag(pcr, ASPM_L1_2_EN);
+
+	if (active) {
+		/* run, latency: 60us */
+		if (aspm_L1_1)
+			val = option->ltr_l1off_snooze_sspwrgate;
+	} else {
+		/* l1off, latency: 300us */
+		if (aspm_L1_2)
+			val = option->ltr_l1off_sspwrgate;
+	}
+
+	if (aspm_L1_1 || aspm_L1_2) {
+		if (rtsx_check_dev_flag(pcr,
+					LTR_L1SS_PWR_GATE_CHECK_CARD_EN)) {
+			if (card_exist)
+				val &= ~L1OFF_MBIAS2_EN_5250;
+			else
+				val |= L1OFF_MBIAS2_EN_5250;
+		}
+	}
+	rtsx_set_l1off_sub(pcr, val);
+}
+
+static const struct pcr_ops rts5260_pcr_ops = {
+	.fetch_vendor_settings = rtsx_base_fetch_vendor_settings,
+	.turn_on_led = rts5260_turn_on_led,
+	.turn_off_led = rts5260_turn_off_led,
+	.extra_init_hw = rts5260_extra_init_hw,
+	.enable_auto_blink = rtsx_base_enable_auto_blink,
+	.disable_auto_blink = rtsx_base_disable_auto_blink,
+	.card_power_on = rts5260_card_power_on,
+	.card_power_off = rts5260_card_power_off,
+	.switch_output_voltage = rts5260_switch_output_voltage,
+	.force_power_down = rtsx_base_force_power_down,
+	.stop_cmd = rts5260_stop_cmd,
+	.set_aspm = rts5260_set_aspm,
+	.set_l1off_cfg_sub_d0 = rts5260_set_l1off_cfg_sub_d0,
+	.enable_ocp = rts5260_enable_ocp,
+	.disable_ocp = rts5260_disable_ocp,
+	.init_ocp = rts5260_init_ocp,
+	.process_ocp = rts5260_process_ocp,
+	.get_ocpstat = rts5260_get_ocpstat,
+	.clear_ocpstat = rts5260_clear_ocpstat,
+};
+
+void rts5260_init_params(struct rtsx_pcr *pcr)
+{
+	struct rtsx_cr_option *option = &pcr->option;
+	struct rtsx_hw_param *hw_param = &pcr->hw_param;
+
+	pcr->extra_caps = EXTRA_CAPS_SD_SDR50 | EXTRA_CAPS_SD_SDR104;
+	pcr->num_slots = 2;
+
+	pcr->flags = 0;
+	pcr->card_drive_sel = RTSX_CARD_DRIVE_DEFAULT;
+	pcr->sd30_drive_sel_1v8 = CFG_DRIVER_TYPE_B;
+	pcr->sd30_drive_sel_3v3 = CFG_DRIVER_TYPE_B;
+	pcr->aspm_en = ASPM_L1_EN;
+	pcr->tx_initial_phase = SET_CLOCK_PHASE(1, 29, 16);
+	pcr->rx_initial_phase = SET_CLOCK_PHASE(24, 6, 5);
+
+	pcr->ic_version = rts5260_get_ic_version(pcr);
+	pcr->sd_pull_ctl_enable_tbl = rts5260_sd_pull_ctl_enable_tbl;
+	pcr->sd_pull_ctl_disable_tbl = rts5260_sd_pull_ctl_disable_tbl;
+	pcr->ms_pull_ctl_enable_tbl = rts5260_ms_pull_ctl_enable_tbl;
+	pcr->ms_pull_ctl_disable_tbl = rts5260_ms_pull_ctl_disable_tbl;
+
+	pcr->reg_pm_ctrl3 = RTS524A_PM_CTRL3;
+
+	pcr->ops = &rts5260_pcr_ops;
+
+	option->dev_flags = (LTR_L1SS_PWR_GATE_CHECK_CARD_EN
+				| LTR_L1SS_PWR_GATE_EN);
+	option->ltr_en = true;
+
+	/* init latency of active, idle, L1OFF to 60us, 300us, 3ms */
+	option->ltr_active_latency = LTR_ACTIVE_LATENCY_DEF;
+	option->ltr_idle_latency = LTR_IDLE_LATENCY_DEF;
+	option->ltr_l1off_latency = LTR_L1OFF_LATENCY_DEF;
+	option->dev_aspm_mode = DEV_ASPM_DYNAMIC;
+	option->l1_snooze_delay = L1_SNOOZE_DELAY_DEF;
+	option->ltr_l1off_sspwrgate = LTR_L1OFF_SSPWRGATE_5250_DEF;
+	option->ltr_l1off_snooze_sspwrgate =
+		LTR_L1OFF_SNOOZE_SSPWRGATE_5250_DEF;
+
+	option->ocp_en = 1;
+	if (option->ocp_en)
+		hw_param->interrupt_en |= SD_OC_INT_EN;
+	hw_param->ocp_glitch = SD_OCP_GLITCH_10M | SDVIO_OCP_GLITCH_800U;
+	option->sd_400mA_ocp_thd = RTS5260_DVCC_OCP_THD_550;
+	option->sd_800mA_ocp_thd = RTS5260_DVCC_OCP_THD_970;
+}
diff --git a/drivers/misc/cardreader/rts5260.h b/drivers/misc/cardreader/rts5260.h
new file mode 100644
index 000000000000..53a1411c8868
--- /dev/null
+++ b/drivers/misc/cardreader/rts5260.h
@@ -0,0 +1,45 @@
+#ifndef __RTS5260_H__
+#define __RTS5260_H__
+
+#define RTS5260_DVCC_CTRL		0xFF73
+#define RTS5260_DVCC_OCP_EN		(0x01 << 7)
+#define RTS5260_DVCC_OCP_THD_MASK	(0x07 << 4)
+#define RTS5260_DVCC_POWERON		(0x01 << 3)
+#define RTS5260_DVCC_OCP_CL_EN		(0x01 << 2)
+
+#define RTS5260_DVIO_CTRL		0xFF75
+#define RTS5260_DVIO_OCP_EN		(0x01 << 7)
+#define RTS5260_DVIO_OCP_THD_MASK	(0x07 << 4)
+#define RTS5260_DVIO_POWERON		(0x01 << 3)
+#define RTS5260_DVIO_OCP_CL_EN		(0x01 << 2)
+
+#define RTS5260_DV331812_CFG		0xFF71
+#define RTS5260_DV331812_OCP_EN		(0x01 << 7)
+#define RTS5260_DV331812_OCP_THD_MASK	(0x07 << 4)
+#define RTS5260_DV331812_POWERON	(0x01 << 3)
+#define RTS5260_DV331812_SEL		(0x01 << 2)
+#define RTS5260_DV331812_VDD1		(0x01 << 2)
+#define RTS5260_DV331812_VDD2		(0x00 << 2)
+
+#define RTS5260_DV331812_OCP_THD_120	(0x00 << 4)
+#define RTS5260_DV331812_OCP_THD_140	(0x01 << 4)
+#define RTS5260_DV331812_OCP_THD_160	(0x02 << 4)
+#define RTS5260_DV331812_OCP_THD_180	(0x03 << 4)
+#define RTS5260_DV331812_OCP_THD_210	(0x04 << 4)
+#define RTS5260_DV331812_OCP_THD_240	(0x05 << 4)
+#define RTS5260_DV331812_OCP_THD_270	(0x06 << 4)
+#define RTS5260_DV331812_OCP_THD_300	(0x07 << 4)
+
+#define RTS5260_DVIO_OCP_THD_250	(0x00 << 4)
+#define RTS5260_DVIO_OCP_THD_300	(0x01 << 4)
+#define RTS5260_DVIO_OCP_THD_350	(0x02 << 4)
+#define RTS5260_DVIO_OCP_THD_400	(0x03 << 4)
+#define RTS5260_DVIO_OCP_THD_450	(0x04 << 4)
+#define RTS5260_DVIO_OCP_THD_500	(0x05 << 4)
+#define RTS5260_DVIO_OCP_THD_550	(0x06 << 4)
+#define RTS5260_DVIO_OCP_THD_600	(0x07 << 4)
+
+#define RTS5260_DVCC_OCP_THD_550	(0x00 << 4)
+#define RTS5260_DVCC_OCP_THD_970	(0x05 << 4)
+
+#endif
diff --git a/drivers/mfd/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c
similarity index 92%
rename from drivers/mfd/rtsx_pcr.c
rename to drivers/misc/cardreader/rtsx_pcr.c
index c3ed885c155c..fd09b0960097 100644
--- a/drivers/mfd/rtsx_pcr.c
+++ b/drivers/misc/cardreader/rtsx_pcr.c
@@ -29,7 +29,7 @@
 #include <linux/idr.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/core.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 #include <linux/mmc/card.h>
 #include <asm/unaligned.h>
 
@@ -62,6 +62,7 @@ static const struct pci_device_id rtsx_pci_ids[] = {
 	{ PCI_DEVICE(0x10EC, 0x5286), PCI_CLASS_OTHERS << 16, 0xFF0000 },
 	{ PCI_DEVICE(0x10EC, 0x524A), PCI_CLASS_OTHERS << 16, 0xFF0000 },
 	{ PCI_DEVICE(0x10EC, 0x525A), PCI_CLASS_OTHERS << 16, 0xFF0000 },
+	{ PCI_DEVICE(0x10EC, 0x5260), PCI_CLASS_OTHERS << 16, 0xFF0000 },
 	{ 0, }
 };
 
@@ -334,6 +335,9 @@ EXPORT_SYMBOL_GPL(rtsx_pci_read_phy_register);
 
 void rtsx_pci_stop_cmd(struct rtsx_pcr *pcr)
 {
+	if (pcr->ops->stop_cmd)
+		return pcr->ops->stop_cmd(pcr);
+
 	rtsx_pci_writel(pcr, RTSX_HCBCTLR, STOP_CMD);
 	rtsx_pci_writel(pcr, RTSX_HDBCTLR, STOP_DMA);
 
@@ -826,7 +830,7 @@ int rtsx_pci_switch_clock(struct rtsx_pcr *pcr, unsigned int card_clock,
 		return err;
 
 	/* Wait SSC clock stable */
-	udelay(10);
+	udelay(SSC_CLOCK_STABLE_WAIT);
 	err = rtsx_pci_write_register(pcr, CLK_CTL, CLK_LOW_FREQ, 0);
 	if (err < 0)
 		return err;
@@ -963,6 +967,20 @@ static void rtsx_pci_card_detect(struct work_struct *work)
 				pcr->slots[RTSX_MS_CARD].p_dev);
 }
 
+void rtsx_pci_process_ocp(struct rtsx_pcr *pcr)
+{
+	if (pcr->ops->process_ocp)
+		pcr->ops->process_ocp(pcr);
+}
+
+int rtsx_pci_process_ocp_interrupt(struct rtsx_pcr *pcr)
+{
+	if (pcr->option.ocp_en)
+		rtsx_pci_process_ocp(pcr);
+
+	return 0;
+}
+
 static irqreturn_t rtsx_pci_isr(int irq, void *dev_id)
 {
 	struct rtsx_pcr *pcr = dev_id;
@@ -987,6 +1005,9 @@ static irqreturn_t rtsx_pci_isr(int irq, void *dev_id)
 
 	int_reg &= (pcr->bier | 0x7FFFFF);
 
+	if (int_reg & SD_OC_INT)
+		rtsx_pci_process_ocp_interrupt(pcr);
+
 	if (int_reg & SD_INT) {
 		if (int_reg & SD_EXIST) {
 			pcr->card_inserted |= SD_EXIST;
@@ -1119,6 +1140,102 @@ static void rtsx_pci_power_off(struct rtsx_pcr *pcr, u8 pm_state)
 }
 #endif
 
+void rtsx_pci_enable_ocp(struct rtsx_pcr *pcr)
+{
+	u8 val = SD_OCP_INT_EN | SD_DETECT_EN;
+
+	if (pcr->ops->enable_ocp)
+		pcr->ops->enable_ocp(pcr);
+	else
+		rtsx_pci_write_register(pcr, REG_OCPCTL, 0xFF, val);
+
+}
+
+void rtsx_pci_disable_ocp(struct rtsx_pcr *pcr)
+{
+	u8 mask = SD_OCP_INT_EN | SD_DETECT_EN;
+
+	if (pcr->ops->disable_ocp)
+		pcr->ops->disable_ocp(pcr);
+	else
+		rtsx_pci_write_register(pcr, REG_OCPCTL, mask, 0);
+}
+
+void rtsx_pci_init_ocp(struct rtsx_pcr *pcr)
+{
+	if (pcr->ops->init_ocp) {
+		pcr->ops->init_ocp(pcr);
+	} else {
+		struct rtsx_cr_option *option = &(pcr->option);
+
+		if (option->ocp_en) {
+			u8 val = option->sd_400mA_ocp_thd;
+
+			rtsx_pci_write_register(pcr, FPDCTL, OC_POWER_DOWN, 0);
+			rtsx_pci_write_register(pcr, REG_OCPPARA1,
+				SD_OCP_TIME_MASK, SD_OCP_TIME_800);
+			rtsx_pci_write_register(pcr, REG_OCPPARA2,
+				SD_OCP_THD_MASK, val);
+			rtsx_pci_write_register(pcr, REG_OCPGLITCH,
+				SD_OCP_GLITCH_MASK, pcr->hw_param.ocp_glitch);
+			rtsx_pci_enable_ocp(pcr);
+		} else {
+			/* OC power down */
+			rtsx_pci_write_register(pcr, FPDCTL, OC_POWER_DOWN,
+				OC_POWER_DOWN);
+		}
+	}
+}
+
+int rtsx_pci_get_ocpstat(struct rtsx_pcr *pcr, u8 *val)
+{
+	if (pcr->ops->get_ocpstat)
+		return pcr->ops->get_ocpstat(pcr, val);
+	else
+		return rtsx_pci_read_register(pcr, REG_OCPSTAT, val);
+}
+
+void rtsx_pci_clear_ocpstat(struct rtsx_pcr *pcr)
+{
+	if (pcr->ops->clear_ocpstat) {
+		pcr->ops->clear_ocpstat(pcr);
+	} else {
+		u8 mask = SD_OCP_INT_CLR | SD_OC_CLR;
+		u8 val = SD_OCP_INT_CLR | SD_OC_CLR;
+
+		rtsx_pci_write_register(pcr, REG_OCPCTL, mask, val);
+		rtsx_pci_write_register(pcr, REG_OCPCTL, mask, 0);
+	}
+}
+
+int rtsx_sd_power_off_card3v3(struct rtsx_pcr *pcr)
+{
+	rtsx_pci_write_register(pcr, CARD_CLK_EN, SD_CLK_EN |
+		MS_CLK_EN | SD40_CLK_EN, 0);
+	rtsx_pci_write_register(pcr, CARD_OE, SD_OUTPUT_EN, 0);
+
+	rtsx_pci_card_power_off(pcr, RTSX_SD_CARD);
+
+	msleep(50);
+
+	rtsx_pci_card_pull_ctl_disable(pcr, RTSX_SD_CARD);
+
+	return 0;
+}
+
+int rtsx_ms_power_off_card3v3(struct rtsx_pcr *pcr)
+{
+	rtsx_pci_write_register(pcr, CARD_CLK_EN, SD_CLK_EN |
+		MS_CLK_EN | SD40_CLK_EN, 0);
+
+	rtsx_pci_card_pull_ctl_disable(pcr, RTSX_MS_CARD);
+
+	rtsx_pci_write_register(pcr, CARD_OE, MS_OUTPUT_EN, 0);
+	rtsx_pci_card_power_off(pcr, RTSX_MS_CARD);
+
+	return 0;
+}
+
 static int rtsx_pci_init_hw(struct rtsx_pcr *pcr)
 {
 	int err;
@@ -1189,6 +1306,7 @@ static int rtsx_pci_init_hw(struct rtsx_pcr *pcr)
 	case PID_5250:
 	case PID_524A:
 	case PID_525A:
+	case PID_5260:
 		rtsx_pci_write_register(pcr, PM_CLK_FORCE_CTL, 1, 1);
 		break;
 	default:
@@ -1265,6 +1383,9 @@ static int rtsx_pci_init_chip(struct rtsx_pcr *pcr)
 	case 0x5286:
 		rtl8402_init_params(pcr);
 		break;
+	case 0x5260:
+		rts5260_init_params(pcr);
+		break;
 	}
 
 	pcr_dbg(pcr, "PID: 0x%04x, IC version: 0x%02x\n",
diff --git a/drivers/mfd/rtsx_pcr.h b/drivers/misc/cardreader/rtsx_pcr.h
similarity index 88%
rename from drivers/mfd/rtsx_pcr.h
rename to drivers/misc/cardreader/rtsx_pcr.h
index ec784e04fe20..6ea1655db0bb 100644
--- a/drivers/mfd/rtsx_pcr.h
+++ b/drivers/misc/cardreader/rtsx_pcr.h
@@ -22,7 +22,7 @@
 #ifndef __RTSX_PCR_H
 #define __RTSX_PCR_H
 
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 
 #define MIN_DIV_N_PCR		80
 #define MAX_DIV_N_PCR		208
@@ -44,6 +44,8 @@
 #define ASPM_MASK_NEG		0xFC
 #define MASK_8_BIT_DEF		0xFF
 
+#define SSC_CLOCK_STABLE_WAIT	130
+
 int __rtsx_pci_write_phy_register(struct rtsx_pcr *pcr, u8 addr, u16 val);
 int __rtsx_pci_read_phy_register(struct rtsx_pcr *pcr, u8 addr, u16 *val);
 
@@ -57,6 +59,7 @@ void rts5249_init_params(struct rtsx_pcr *pcr);
 void rts524a_init_params(struct rtsx_pcr *pcr);
 void rts525a_init_params(struct rtsx_pcr *pcr);
 void rtl8411b_init_params(struct rtsx_pcr *pcr);
+void rts5260_init_params(struct rtsx_pcr *pcr);
 
 static inline u8 map_sd_drive(int idx)
 {
@@ -99,5 +102,12 @@ do {									\
 int rtsx_gops_pm_reset(struct rtsx_pcr *pcr);
 int rtsx_set_ltr_latency(struct rtsx_pcr *pcr, u32 latency);
 int rtsx_set_l1off_sub(struct rtsx_pcr *pcr, u8 val);
+void rtsx_pci_init_ocp(struct rtsx_pcr *pcr);
+void rtsx_pci_disable_ocp(struct rtsx_pcr *pcr);
+void rtsx_pci_enable_ocp(struct rtsx_pcr *pcr);
+int rtsx_pci_get_ocpstat(struct rtsx_pcr *pcr, u8 *val);
+void rtsx_pci_clear_ocpstat(struct rtsx_pcr *pcr);
+int rtsx_sd_power_off_card3v3(struct rtsx_pcr *pcr);
+int rtsx_ms_power_off_card3v3(struct rtsx_pcr *pcr);
 
 #endif
diff --git a/drivers/mfd/rtsx_usb.c b/drivers/misc/cardreader/rtsx_usb.c
similarity index 99%
rename from drivers/mfd/rtsx_usb.c
rename to drivers/misc/cardreader/rtsx_usb.c
index 59d61b04c197..b97903ff1a72 100644
--- a/drivers/mfd/rtsx_usb.c
+++ b/drivers/misc/cardreader/rtsx_usb.c
@@ -23,7 +23,7 @@
 #include <linux/usb.h>
 #include <linux/platform_device.h>
 #include <linux/mfd/core.h>
-#include <linux/mfd/rtsx_usb.h>
+#include <linux/rtsx_usb.h>
 
 static int polling_pipe = 1;
 module_param(polling_pipe, int, S_IRUGO | S_IWUSR);
diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c
index 7c11bad5cded..753b1a698fc4 100644
--- a/drivers/misc/cxl/api.c
+++ b/drivers/misc/cxl/api.c
@@ -427,7 +427,7 @@ int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm)
 	return afu_mmap(file, vm);
 }
 EXPORT_SYMBOL_GPL(cxl_fd_mmap);
-unsigned int cxl_fd_poll(struct file *file, struct poll_table_struct *poll)
+__poll_t cxl_fd_poll(struct file *file, struct poll_table_struct *poll)
 {
 	return afu_poll(file, poll);
 }
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
index 12a41b2753f0..7ff315ad3692 100644
--- a/drivers/misc/cxl/context.c
+++ b/drivers/misc/cxl/context.c
@@ -45,6 +45,8 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
 	ctx->pid = NULL; /* Set in start work ioctl */
 	mutex_init(&ctx->mapping_lock);
 	ctx->mapping = NULL;
+	ctx->tidr = 0;
+	ctx->assign_tidr = false;
 
 	if (cxl_is_power8()) {
 		spin_lock_init(&ctx->sste_lock);
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
index e46a4062904a..4f015da78f28 100644
--- a/drivers/misc/cxl/cxl.h
+++ b/drivers/misc/cxl/cxl.h
@@ -630,6 +630,9 @@ struct cxl_context {
 	struct list_head extra_irq_contexts;
 
 	struct mm_struct *mm;
+
+	u16 tidr;
+	bool assign_tidr;
 };
 
 struct cxl_irq_info;
@@ -1081,7 +1084,7 @@ int afu_open(struct inode *inode, struct file *file);
 int afu_release(struct inode *inode, struct file *file);
 long afu_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
 int afu_mmap(struct file *file, struct vm_area_struct *vm);
-unsigned int afu_poll(struct file *file, struct poll_table_struct *poll);
+__poll_t afu_poll(struct file *file, struct poll_table_struct *poll);
 ssize_t afu_read(struct file *file, char __user *buf, size_t count, loff_t *off);
 extern const struct file_operations afu_fops;
 
diff --git a/drivers/misc/cxl/cxllib.c b/drivers/misc/cxl/cxllib.c
index dc9bc1807fdf..30ccba436b3b 100644
--- a/drivers/misc/cxl/cxllib.c
+++ b/drivers/misc/cxl/cxllib.c
@@ -199,10 +199,11 @@ int cxllib_get_PE_attributes(struct task_struct *task,
 		 */
 		attr->pid = mm->context.id;
 		mmput(mm);
+		attr->tid = task->thread.tidr;
 	} else {
 		attr->pid = 0;
+		attr->tid = 0;
 	}
-	attr->tid = 0;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(cxllib_get_PE_attributes);
diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
index 76c0b0ca9388..bd6ddbdb5cd1 100644
--- a/drivers/misc/cxl/file.c
+++ b/drivers/misc/cxl/file.c
@@ -173,7 +173,7 @@ static long afu_ioctl_start_work(struct cxl_context *ctx,
 	 * flags are set it's invalid
 	 */
 	if (work.reserved1 || work.reserved2 || work.reserved3 ||
-	    work.reserved4 || work.reserved5 || work.reserved6 ||
+	    work.reserved4 || work.reserved5 ||
 	    (work.flags & ~CXL_START_WORK_ALL)) {
 		rc = -EINVAL;
 		goto out;
@@ -186,12 +186,16 @@ static long afu_ioctl_start_work(struct cxl_context *ctx,
 		rc =  -EINVAL;
 		goto out;
 	}
+
 	if ((rc = afu_register_irqs(ctx, work.num_interrupts)))
 		goto out;
 
 	if (work.flags & CXL_START_WORK_AMR)
 		amr = work.amr & mfspr(SPRN_UAMOR);
 
+	if (work.flags & CXL_START_WORK_TID)
+		ctx->assign_tidr = true;
+
 	ctx->mmio_err_ff = !!(work.flags & CXL_START_WORK_ERR_FF);
 
 	/*
@@ -263,8 +267,15 @@ static long afu_ioctl_start_work(struct cxl_context *ctx,
 		goto out;
 	}
 
-	ctx->status = STARTED;
 	rc = 0;
+	if (work.flags & CXL_START_WORK_TID) {
+		work.tid = ctx->tidr;
+		if (copy_to_user(uwork, &work, sizeof(work)))
+			rc = -EFAULT;
+	}
+
+	ctx->status = STARTED;
+
 out:
 	mutex_unlock(&ctx->status_mutex);
 	return rc;
@@ -354,10 +365,10 @@ static inline bool ctx_event_pending(struct cxl_context *ctx)
 	return false;
 }
 
-unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
+__poll_t afu_poll(struct file *file, struct poll_table_struct *poll)
 {
 	struct cxl_context *ctx = file->private_data;
-	int mask = 0;
+	__poll_t mask = 0;
 	unsigned long flags;
 
 
@@ -367,11 +378,11 @@ unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
 
 	spin_lock_irqsave(&ctx->lock, flags);
 	if (ctx_event_pending(ctx))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	else if (ctx->status == CLOSED)
 		/* Only error on closed when there are no futher events pending
 		 */
-		mask |= POLLERR;
+		mask |= EPOLLERR;
 	spin_unlock_irqrestore(&ctx->lock, flags);
 
 	pr_devel("afu_poll pe: %i returning %#x\n", ctx->pe, mask);
diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
index 02b6b45b4c20..1b3d7c65ea3f 100644
--- a/drivers/misc/cxl/native.c
+++ b/drivers/misc/cxl/native.c
@@ -16,6 +16,7 @@
 #include <linux/uaccess.h>
 #include <linux/delay.h>
 #include <asm/synch.h>
+#include <asm/switch_to.h>
 #include <misc/cxl-base.h>
 
 #include "cxl.h"
@@ -655,6 +656,7 @@ static void update_ivtes_directed(struct cxl_context *ctx)
 static int process_element_entry_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
 {
 	u32 pid;
+	int rc;
 
 	cxl_assign_psn_space(ctx);
 
@@ -673,7 +675,16 @@ static int process_element_entry_psl9(struct cxl_context *ctx, u64 wed, u64 amr)
 		pid = ctx->mm->context.id;
 	}
 
-	ctx->elem->common.tid = 0;
+	/* Assign a unique TIDR (thread id) for the current thread */
+	if (!(ctx->tidr) && (ctx->assign_tidr)) {
+		rc = set_thread_tidr(current);
+		if (rc)
+			return -ENODEV;
+		ctx->tidr = current->thread.tidr;
+		pr_devel("%s: current tidr: %d\n", __func__, ctx->tidr);
+	}
+
+	ctx->elem->common.tid = cpu_to_be32(ctx->tidr);
 	ctx->elem->common.pid = cpu_to_be32(pid);
 
 	ctx->elem->sr = cpu_to_be64(calculate_sr(ctx));
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
index 19969ee86d6f..758842f65a1b 100644
--- a/drivers/misc/cxl/pci.c
+++ b/drivers/misc/cxl/pci.c
@@ -125,8 +125,6 @@ static const struct pci_device_id cxl_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0601), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0623), },
 	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0628), },
-	{ PCI_DEVICE_CLASS(0x120000, ~0), },
-
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, cxl_pci_tbl);
diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c
index 512a4897dbf6..7fd0bdc1436a 100644
--- a/drivers/misc/cxl/vphb.c
+++ b/drivers/misc/cxl/vphb.c
@@ -54,7 +54,7 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev)
 		return false;
 	}
 
-	set_dma_ops(&dev->dev, &dma_direct_ops);
+	set_dma_ops(&dev->dev, &dma_nommu_ops);
 	set_dma_offset(&dev->dev, PAGE_OFFSET);
 
 	return _cxl_pci_associate_default_context(dev, afu);
diff --git a/drivers/misc/ds1682.c b/drivers/misc/ds1682.c
index 7231260ac287..98a921ea9ee8 100644
--- a/drivers/misc/ds1682.c
+++ b/drivers/misc/ds1682.c
@@ -59,25 +59,42 @@ static ssize_t ds1682_show(struct device *dev, struct device_attribute *attr,
 {
 	struct sensor_device_attribute_2 *sattr = to_sensor_dev_attr_2(attr);
 	struct i2c_client *client = to_i2c_client(dev);
-	__le32 val = 0;
+	unsigned long long val, check;
+	__le32 val_le = 0;
 	int rc;
 
 	dev_dbg(dev, "ds1682_show() called on %s\n", attr->attr.name);
 
 	/* Read the register */
 	rc = i2c_smbus_read_i2c_block_data(client, sattr->index, sattr->nr,
-					   (u8 *) & val);
+					   (u8 *)&val_le);
 	if (rc < 0)
 		return -EIO;
 
-	/* Special case: the 32 bit regs are time values with 1/4s
-	 * resolution, scale them up to milliseconds */
-	if (sattr->nr == 4)
-		return sprintf(buf, "%llu\n",
-			((unsigned long long)le32_to_cpu(val)) * 250);
+	val = le32_to_cpu(val_le);
+
+	if (sattr->index == DS1682_REG_ELAPSED) {
+		int retries = 5;
+
+		/* Detect and retry when a tick occurs mid-read */
+		do {
+			rc = i2c_smbus_read_i2c_block_data(client, sattr->index,
+							   sattr->nr,
+							   (u8 *)&val_le);
+			if (rc < 0 || retries <= 0)
+				return -EIO;
+
+			check = val;
+			val = le32_to_cpu(val_le);
+			retries--;
+		} while (val != check && val != (check + 1));
+	}
 
-	/* Format the output string and return # of bytes */
-	return sprintf(buf, "%li\n", (long)le32_to_cpu(val));
+	/* Format the output string and return # of bytes
+	 * Special case: the 32 bit regs are time values with 1/4s
+	 * resolution, scale them up to milliseconds
+	 */
+	return sprintf(buf, "%llu\n", (sattr->nr == 4) ? (val * 250) : val);
 }
 
 static ssize_t ds1682_store(struct device *dev, struct device_attribute *attr,
diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index de58762097c4..68a1ac929917 100644
--- a/drivers/misc/eeprom/Kconfig
+++ b/drivers/misc/eeprom/Kconfig
@@ -4,6 +4,7 @@ config EEPROM_AT24
 	tristate "I2C EEPROMs / RAMs / ROMs from most vendors"
 	depends on I2C && SYSFS
 	select NVMEM
+	select REGMAP_I2C
 	help
 	  Enable this driver to get read/write support to most I2C EEPROMs
 	  and compatible devices like FRAMs, SRAMs, ROMs etc. After you
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
index 4d63ac8a82e0..01f9c4921c50 100644
--- a/drivers/misc/eeprom/at24.c
+++ b/drivers/misc/eeprom/at24.c
@@ -24,8 +24,10 @@
 #include <linux/acpi.h>
 #include <linux/i2c.h>
 #include <linux/nvmem-provider.h>
+#include <linux/regmap.h>
 #include <linux/platform_data/at24.h>
 #include <linux/pm_runtime.h>
+#include <linux/gpio/consumer.h>
 
 /*
  * I2C EEPROMs from most vendors are inexpensive and mostly interchangeable.
@@ -55,14 +57,13 @@
  * which won't work on pure SMBus systems.
  */
 
+struct at24_client {
+	struct i2c_client *client;
+	struct regmap *regmap;
+};
+
 struct at24_data {
 	struct at24_platform_data chip;
-	int use_smbus;
-	int use_smbus_write;
-
-	ssize_t (*read_func)(struct at24_data *, char *, unsigned int, size_t);
-	ssize_t (*write_func)(struct at24_data *,
-			      const char *, unsigned int, size_t);
 
 	/*
 	 * Lock protects against activities from other Linux tasks,
@@ -70,18 +71,20 @@ struct at24_data {
 	 */
 	struct mutex lock;
 
-	u8 *writebuf;
-	unsigned write_max;
-	unsigned num_addresses;
+	unsigned int write_max;
+	unsigned int num_addresses;
+	unsigned int offset_adj;
 
 	struct nvmem_config nvmem_config;
 	struct nvmem_device *nvmem;
 
+	struct gpio_desc *wp_gpio;
+
 	/*
 	 * Some chips tie up multiple I2C addresses; dummy devices reserve
 	 * them for us, and we'll use them with SMBus calls.
 	 */
-	struct i2c_client *client[];
+	struct at24_client client[];
 };
 
 /*
@@ -93,27 +96,17 @@ struct at24_data {
  *
  * This value is forced to be a power of two so that writes align on pages.
  */
-static unsigned io_limit = 128;
-module_param(io_limit, uint, 0);
-MODULE_PARM_DESC(io_limit, "Maximum bytes per I/O (default 128)");
+static unsigned int at24_io_limit = 128;
+module_param_named(io_limit, at24_io_limit, uint, 0);
+MODULE_PARM_DESC(at24_io_limit, "Maximum bytes per I/O (default 128)");
 
 /*
  * Specs often allow 5 msec for a page write, sometimes 20 msec;
  * it's important to recover from write timeouts.
  */
-static unsigned write_timeout = 25;
-module_param(write_timeout, uint, 0);
-MODULE_PARM_DESC(write_timeout, "Time (in ms) to try writes (default 25)");
-
-#define AT24_SIZE_BYTELEN 5
-#define AT24_SIZE_FLAGS 8
-
-#define AT24_BITMASK(x) (BIT(x) - 1)
-
-/* create non-zero magic value for given eeprom parameters */
-#define AT24_DEVICE_MAGIC(_len, _flags) 		\
-	((1 << AT24_SIZE_FLAGS | (_flags)) 		\
-	    << AT24_SIZE_BYTELEN | ilog2(_len))
+static unsigned int at24_write_timeout = 25;
+module_param_named(write_timeout, at24_write_timeout, uint, 0);
+MODULE_PARM_DESC(at24_write_timeout, "Time (in ms) to try writes (default 25)");
 
 /*
  * Both reads and writes fail if the previous write didn't complete yet. This
@@ -126,118 +119,123 @@ MODULE_PARM_DESC(write_timeout, "Time (in ms) to try writes (default 25)");
  * iteration of processing the request. Both should be unsigned integers
  * holding at least 32 bits.
  */
-#define loop_until_timeout(tout, op_time)				\
-	for (tout = jiffies + msecs_to_jiffies(write_timeout), op_time = 0; \
+#define at24_loop_until_timeout(tout, op_time)				\
+	for (tout = jiffies + msecs_to_jiffies(at24_write_timeout),	\
+	     op_time = 0;						\
 	     op_time ? time_before(op_time, tout) : true;		\
 	     usleep_range(1000, 1500), op_time = jiffies)
 
+struct at24_chip_data {
+	/*
+	 * these fields mirror their equivalents in
+	 * struct at24_platform_data
+	 */
+	u32 byte_len;
+	u8 flags;
+};
+
+#define AT24_CHIP_DATA(_name, _len, _flags)				\
+	static const struct at24_chip_data _name = {			\
+		.byte_len = _len, .flags = _flags,			\
+	}
+
+/* needs 8 addresses as A0-A2 are ignored */
+AT24_CHIP_DATA(at24_data_24c00, 128 / 8, AT24_FLAG_TAKE8ADDR);
+/* old variants can't be handled with this generic entry! */
+AT24_CHIP_DATA(at24_data_24c01, 1024 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs01, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c02, 2048 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs02, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24mac402, 48 / 8,
+	AT24_FLAG_MAC | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24mac602, 64 / 8,
+	AT24_FLAG_MAC | AT24_FLAG_READONLY);
+/* spd is a 24c02 in memory DIMMs */
+AT24_CHIP_DATA(at24_data_spd, 2048 / 8,
+	AT24_FLAG_READONLY | AT24_FLAG_IRUGO);
+AT24_CHIP_DATA(at24_data_24c04, 4096 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs04, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+/* 24rf08 quirk is handled at i2c-core */
+AT24_CHIP_DATA(at24_data_24c08, 8192 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs08, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c16, 16384 / 8, 0);
+AT24_CHIP_DATA(at24_data_24cs16, 16,
+	AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c32, 32768 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24cs32, 16,
+	AT24_FLAG_ADDR16 | AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c64, 65536 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24cs64, 16,
+	AT24_FLAG_ADDR16 | AT24_FLAG_SERIAL | AT24_FLAG_READONLY);
+AT24_CHIP_DATA(at24_data_24c128, 131072 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24c256, 262144 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24c512, 524288 / 8, AT24_FLAG_ADDR16);
+AT24_CHIP_DATA(at24_data_24c1024, 1048576 / 8, AT24_FLAG_ADDR16);
+/* identical to 24c08 ? */
+AT24_CHIP_DATA(at24_data_INT3499, 8192 / 8, 0);
+
 static const struct i2c_device_id at24_ids[] = {
-	/* needs 8 addresses as A0-A2 are ignored */
-	{ "24c00",	AT24_DEVICE_MAGIC(128 / 8,	AT24_FLAG_TAKE8ADDR) },
-	/* old variants can't be handled with this generic entry! */
-	{ "24c01",	AT24_DEVICE_MAGIC(1024 / 8,	0) },
-	{ "24cs01",	AT24_DEVICE_MAGIC(16,
-				AT24_FLAG_SERIAL | AT24_FLAG_READONLY) },
-	{ "24c02",	AT24_DEVICE_MAGIC(2048 / 8,	0) },
-	{ "24cs02",	AT24_DEVICE_MAGIC(16,
-				AT24_FLAG_SERIAL | AT24_FLAG_READONLY) },
-	{ "24mac402",	AT24_DEVICE_MAGIC(48 / 8,
-				AT24_FLAG_MAC | AT24_FLAG_READONLY) },
-	{ "24mac602",	AT24_DEVICE_MAGIC(64 / 8,
-				AT24_FLAG_MAC | AT24_FLAG_READONLY) },
-	/* spd is a 24c02 in memory DIMMs */
-	{ "spd",	AT24_DEVICE_MAGIC(2048 / 8,
-				AT24_FLAG_READONLY | AT24_FLAG_IRUGO) },
-	{ "24c04",	AT24_DEVICE_MAGIC(4096 / 8,	0) },
-	{ "24cs04",	AT24_DEVICE_MAGIC(16,
-				AT24_FLAG_SERIAL | AT24_FLAG_READONLY) },
-	/* 24rf08 quirk is handled at i2c-core */
-	{ "24c08",	AT24_DEVICE_MAGIC(8192 / 8,	0) },
-	{ "24cs08",	AT24_DEVICE_MAGIC(16,
-				AT24_FLAG_SERIAL | AT24_FLAG_READONLY) },
-	{ "24c16",	AT24_DEVICE_MAGIC(16384 / 8,	0) },
-	{ "24cs16",	AT24_DEVICE_MAGIC(16,
-				AT24_FLAG_SERIAL | AT24_FLAG_READONLY) },
-	{ "24c32",	AT24_DEVICE_MAGIC(32768 / 8,	AT24_FLAG_ADDR16) },
-	{ "24cs32",	AT24_DEVICE_MAGIC(16,
-				AT24_FLAG_ADDR16 |
-				AT24_FLAG_SERIAL |
-				AT24_FLAG_READONLY) },
-	{ "24c64",	AT24_DEVICE_MAGIC(65536 / 8,	AT24_FLAG_ADDR16) },
-	{ "24cs64",	AT24_DEVICE_MAGIC(16,
-				AT24_FLAG_ADDR16 |
-				AT24_FLAG_SERIAL |
-				AT24_FLAG_READONLY) },
-	{ "24c128",	AT24_DEVICE_MAGIC(131072 / 8,	AT24_FLAG_ADDR16) },
-	{ "24c256",	AT24_DEVICE_MAGIC(262144 / 8,	AT24_FLAG_ADDR16) },
-	{ "24c512",	AT24_DEVICE_MAGIC(524288 / 8,	AT24_FLAG_ADDR16) },
-	{ "24c1024",	AT24_DEVICE_MAGIC(1048576 / 8,	AT24_FLAG_ADDR16) },
-	{ "at24", 0 },
+	{ "24c00",	(kernel_ulong_t)&at24_data_24c00 },
+	{ "24c01",	(kernel_ulong_t)&at24_data_24c01 },
+	{ "24cs01",	(kernel_ulong_t)&at24_data_24cs01 },
+	{ "24c02",	(kernel_ulong_t)&at24_data_24c02 },
+	{ "24cs02",	(kernel_ulong_t)&at24_data_24cs02 },
+	{ "24mac402",	(kernel_ulong_t)&at24_data_24mac402 },
+	{ "24mac602",	(kernel_ulong_t)&at24_data_24mac602 },
+	{ "spd",	(kernel_ulong_t)&at24_data_spd },
+	{ "24c04",	(kernel_ulong_t)&at24_data_24c04 },
+	{ "24cs04",	(kernel_ulong_t)&at24_data_24cs04 },
+	{ "24c08",	(kernel_ulong_t)&at24_data_24c08 },
+	{ "24cs08",	(kernel_ulong_t)&at24_data_24cs08 },
+	{ "24c16",	(kernel_ulong_t)&at24_data_24c16 },
+	{ "24cs16",	(kernel_ulong_t)&at24_data_24cs16 },
+	{ "24c32",	(kernel_ulong_t)&at24_data_24c32 },
+	{ "24cs32",	(kernel_ulong_t)&at24_data_24cs32 },
+	{ "24c64",	(kernel_ulong_t)&at24_data_24c64 },
+	{ "24cs64",	(kernel_ulong_t)&at24_data_24cs64 },
+	{ "24c128",	(kernel_ulong_t)&at24_data_24c128 },
+	{ "24c256",	(kernel_ulong_t)&at24_data_24c256 },
+	{ "24c512",	(kernel_ulong_t)&at24_data_24c512 },
+	{ "24c1024",	(kernel_ulong_t)&at24_data_24c1024 },
+	{ "at24",	0 },
 	{ /* END OF LIST */ }
 };
 MODULE_DEVICE_TABLE(i2c, at24_ids);
 
 static const struct of_device_id at24_of_match[] = {
-	{
-		.compatible = "atmel,24c00",
-		.data = (void *)AT24_DEVICE_MAGIC(128 / 8, AT24_FLAG_TAKE8ADDR)
-	},
-	{
-		.compatible = "atmel,24c01",
-		.data = (void *)AT24_DEVICE_MAGIC(1024 / 8, 0)
-	},
-	{
-		.compatible = "atmel,24c02",
-		.data = (void *)AT24_DEVICE_MAGIC(2048 / 8, 0)
-	},
-	{
-		.compatible = "atmel,spd",
-		.data = (void *)AT24_DEVICE_MAGIC(2048 / 8,
-				AT24_FLAG_READONLY | AT24_FLAG_IRUGO)
-	},
-	{
-		.compatible = "atmel,24c04",
-		.data = (void *)AT24_DEVICE_MAGIC(4096 / 8, 0)
-	},
-	{
-		.compatible = "atmel,24c08",
-		.data = (void *)AT24_DEVICE_MAGIC(8192 / 8, 0)
-	},
-	{
-		.compatible = "atmel,24c16",
-		.data = (void *)AT24_DEVICE_MAGIC(16384 / 8, 0)
-	},
-	{
-		.compatible = "atmel,24c32",
-		.data = (void *)AT24_DEVICE_MAGIC(32768 / 8, AT24_FLAG_ADDR16)
-	},
-	{
-		.compatible = "atmel,24c64",
-		.data = (void *)AT24_DEVICE_MAGIC(65536 / 8, AT24_FLAG_ADDR16)
-	},
-	{
-		.compatible = "atmel,24c128",
-		.data = (void *)AT24_DEVICE_MAGIC(131072 / 8, AT24_FLAG_ADDR16)
-	},
-	{
-		.compatible = "atmel,24c256",
-		.data = (void *)AT24_DEVICE_MAGIC(262144 / 8, AT24_FLAG_ADDR16)
-	},
-	{
-		.compatible = "atmel,24c512",
-		.data = (void *)AT24_DEVICE_MAGIC(524288 / 8, AT24_FLAG_ADDR16)
-	},
-	{
-		.compatible = "atmel,24c1024",
-		.data = (void *)AT24_DEVICE_MAGIC(1048576 / 8, AT24_FLAG_ADDR16)
-	},
-	{ },
+	{ .compatible = "atmel,24c00",		.data = &at24_data_24c00 },
+	{ .compatible = "atmel,24c01",		.data = &at24_data_24c01 },
+	{ .compatible = "atmel,24cs01",		.data = &at24_data_24cs01 },
+	{ .compatible = "atmel,24c02",		.data = &at24_data_24c02 },
+	{ .compatible = "atmel,24cs02",		.data = &at24_data_24cs02 },
+	{ .compatible = "atmel,24mac402",	.data = &at24_data_24mac402 },
+	{ .compatible = "atmel,24mac602",	.data = &at24_data_24mac602 },
+	{ .compatible = "atmel,spd",		.data = &at24_data_spd },
+	{ .compatible = "atmel,24c04",		.data = &at24_data_24c04 },
+	{ .compatible = "atmel,24cs04",		.data = &at24_data_24cs04 },
+	{ .compatible = "atmel,24c08",		.data = &at24_data_24c08 },
+	{ .compatible = "atmel,24cs08",		.data = &at24_data_24cs08 },
+	{ .compatible = "atmel,24c16",		.data = &at24_data_24c16 },
+	{ .compatible = "atmel,24cs16",		.data = &at24_data_24cs16 },
+	{ .compatible = "atmel,24c32",		.data = &at24_data_24c32 },
+	{ .compatible = "atmel,24cs32",		.data = &at24_data_24cs32 },
+	{ .compatible = "atmel,24c64",		.data = &at24_data_24c64 },
+	{ .compatible = "atmel,24cs64",		.data = &at24_data_24cs64 },
+	{ .compatible = "atmel,24c128",		.data = &at24_data_24c128 },
+	{ .compatible = "atmel,24c256",		.data = &at24_data_24c256 },
+	{ .compatible = "atmel,24c512",		.data = &at24_data_24c512 },
+	{ .compatible = "atmel,24c1024",	.data = &at24_data_24c1024 },
+	{ /* END OF LIST */ },
 };
 MODULE_DEVICE_TABLE(of, at24_of_match);
 
 static const struct acpi_device_id at24_acpi_ids[] = {
-	{ "INT3499", AT24_DEVICE_MAGIC(8192 / 8, 0) },
-	{ }
+	{ "INT3499",	(kernel_ulong_t)&at24_data_INT3499 },
+	{ /* END OF LIST */ }
 };
 MODULE_DEVICE_TABLE(acpi, at24_acpi_ids);
 
@@ -251,20 +249,11 @@ MODULE_DEVICE_TABLE(acpi, at24_acpi_ids);
  * Slave address and byte offset derive from the offset. Always
  * set the byte address; on a multi-master board, another master
  * may have changed the chip's "current" address pointer.
- *
- * REVISIT some multi-address chips don't rollover page reads to
- * the next slave address, so we may need to truncate the count.
- * Those chips might need another quirk flag.
- *
- * If the real hardware used four adjacent 24c02 chips and that
- * were misconfigured as one 24c08, that would be a similar effect:
- * one "eeprom" file not four, but larger reads would fail when
- * they crossed certain pages.
  */
-static struct i2c_client *at24_translate_offset(struct at24_data *at24,
-						unsigned int *offset)
+static struct at24_client *at24_translate_offset(struct at24_data *at24,
+						 unsigned int *offset)
 {
-	unsigned i;
+	unsigned int i;
 
 	if (at24->chip.flags & AT24_FLAG_ADDR16) {
 		i = *offset >> 16;
@@ -274,168 +263,55 @@ static struct i2c_client *at24_translate_offset(struct at24_data *at24,
 		*offset &= 0xff;
 	}
 
-	return at24->client[i];
+	return &at24->client[i];
 }
 
-static ssize_t at24_eeprom_read_smbus(struct at24_data *at24, char *buf,
+static size_t at24_adjust_read_count(struct at24_data *at24,
 				      unsigned int offset, size_t count)
 {
-	unsigned long timeout, read_time;
-	struct i2c_client *client;
-	int status;
-
-	client = at24_translate_offset(at24, &offset);
-
-	if (count > io_limit)
-		count = io_limit;
-
-	/* Smaller eeproms can work given some SMBus extension calls */
-	if (count > I2C_SMBUS_BLOCK_MAX)
-		count = I2C_SMBUS_BLOCK_MAX;
-
-	loop_until_timeout(timeout, read_time) {
-		status = i2c_smbus_read_i2c_block_data_or_emulated(client,
-								   offset,
-								   count, buf);
-
-		dev_dbg(&client->dev, "read %zu@%d --> %d (%ld)\n",
-				count, offset, status, jiffies);
-
-		if (status == count)
-			return count;
-	}
-
-	return -ETIMEDOUT;
-}
-
-static ssize_t at24_eeprom_read_i2c(struct at24_data *at24, char *buf,
-				    unsigned int offset, size_t count)
-{
-	unsigned long timeout, read_time;
-	struct i2c_client *client;
-	struct i2c_msg msg[2];
-	int status, i;
-	u8 msgbuf[2];
-
-	memset(msg, 0, sizeof(msg));
-	client = at24_translate_offset(at24, &offset);
-
-	if (count > io_limit)
-		count = io_limit;
+	unsigned int bits;
+	size_t remainder;
 
 	/*
-	 * When we have a better choice than SMBus calls, use a combined I2C
-	 * message. Write address; then read up to io_limit data bytes. Note
-	 * that read page rollover helps us here (unlike writes). msgbuf is
-	 * u8 and will cast to our needs.
+	 * In case of multi-address chips that don't rollover reads to
+	 * the next slave address: truncate the count to the slave boundary,
+	 * so that the read never straddles slaves.
 	 */
-	i = 0;
-	if (at24->chip.flags & AT24_FLAG_ADDR16)
-		msgbuf[i++] = offset >> 8;
-	msgbuf[i++] = offset;
-
-	msg[0].addr = client->addr;
-	msg[0].buf = msgbuf;
-	msg[0].len = i;
-
-	msg[1].addr = client->addr;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = buf;
-	msg[1].len = count;
-
-	loop_until_timeout(timeout, read_time) {
-		status = i2c_transfer(client->adapter, msg, 2);
-		if (status == 2)
-			status = count;
-
-		dev_dbg(&client->dev, "read %zu@%d --> %d (%ld)\n",
-				count, offset, status, jiffies);
-
-		if (status == count)
-			return count;
+	if (at24->chip.flags & AT24_FLAG_NO_RDROL) {
+		bits = (at24->chip.flags & AT24_FLAG_ADDR16) ? 16 : 8;
+		remainder = BIT(bits) - offset;
+		if (count > remainder)
+			count = remainder;
 	}
 
-	return -ETIMEDOUT;
+	if (count > at24_io_limit)
+		count = at24_io_limit;
+
+	return count;
 }
 
-static ssize_t at24_eeprom_read_serial(struct at24_data *at24, char *buf,
-				       unsigned int offset, size_t count)
+static ssize_t at24_regmap_read(struct at24_data *at24, char *buf,
+				unsigned int offset, size_t count)
 {
 	unsigned long timeout, read_time;
+	struct at24_client *at24_client;
 	struct i2c_client *client;
-	struct i2c_msg msg[2];
-	u8 addrbuf[2];
-	int status;
-
-	client = at24_translate_offset(at24, &offset);
-
-	memset(msg, 0, sizeof(msg));
-	msg[0].addr = client->addr;
-	msg[0].buf = addrbuf;
-
-	/*
-	 * The address pointer of the device is shared between the regular
-	 * EEPROM array and the serial number block. The dummy write (part of
-	 * the sequential read protocol) ensures the address pointer is reset
-	 * to the desired position.
-	 */
-	if (at24->chip.flags & AT24_FLAG_ADDR16) {
-		/*
-		 * For 16 bit address pointers, the word address must contain
-		 * a '10' sequence in bits 11 and 10 regardless of the
-		 * intended position of the address pointer.
-		 */
-		addrbuf[0] = 0x08;
-		addrbuf[1] = offset;
-		msg[0].len = 2;
-	} else {
-		/*
-		 * Otherwise the word address must begin with a '10' sequence,
-		 * regardless of the intended address.
-		 */
-		addrbuf[0] = 0x80 + offset;
-		msg[0].len = 1;
-	}
-
-	msg[1].addr = client->addr;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = buf;
-	msg[1].len = count;
+	struct regmap *regmap;
+	int ret;
 
-	loop_until_timeout(timeout, read_time) {
-		status = i2c_transfer(client->adapter, msg, 2);
-		if (status == 2)
-			return count;
-	}
+	at24_client = at24_translate_offset(at24, &offset);
+	regmap = at24_client->regmap;
+	client = at24_client->client;
+	count = at24_adjust_read_count(at24, offset, count);
 
-	return -ETIMEDOUT;
-}
+	/* adjust offset for mac and serial read ops */
+	offset += at24->offset_adj;
 
-static ssize_t at24_eeprom_read_mac(struct at24_data *at24, char *buf,
-				    unsigned int offset, size_t count)
-{
-	unsigned long timeout, read_time;
-	struct i2c_client *client;
-	struct i2c_msg msg[2];
-	u8 addrbuf[2];
-	int status;
-
-	client = at24_translate_offset(at24, &offset);
-
-	memset(msg, 0, sizeof(msg));
-	msg[0].addr = client->addr;
-	msg[0].buf = addrbuf;
-	/* EUI-48 starts from 0x9a, EUI-64 from 0x98 */
-	addrbuf[0] = 0xa0 - at24->chip.byte_len + offset;
-	msg[0].len = 1;
-	msg[1].addr = client->addr;
-	msg[1].flags = I2C_M_RD;
-	msg[1].buf = buf;
-	msg[1].len = count;
-
-	loop_until_timeout(timeout, read_time) {
-		status = i2c_transfer(client->adapter, msg, 2);
-		if (status == 2)
+	at24_loop_until_timeout(timeout, read_time) {
+		ret = regmap_bulk_read(regmap, offset, buf, count);
+		dev_dbg(&client->dev, "read %zu@%d --> %d (%ld)\n",
+			count, offset, ret, jiffies);
+		if (!ret)
 			return count;
 	}
 
@@ -454,7 +330,7 @@ static ssize_t at24_eeprom_read_mac(struct at24_data *at24, char *buf,
 static size_t at24_adjust_write_count(struct at24_data *at24,
 				      unsigned int offset, size_t count)
 {
-	unsigned next_page;
+	unsigned int next_page;
 
 	/* write_max is at most a page */
 	if (count > at24->write_max)
@@ -468,91 +344,25 @@ static size_t at24_adjust_write_count(struct at24_data *at24,
 	return count;
 }
 
-static ssize_t at24_eeprom_write_smbus_block(struct at24_data *at24,
-					     const char *buf,
-					     unsigned int offset, size_t count)
+static ssize_t at24_regmap_write(struct at24_data *at24, const char *buf,
+				 unsigned int offset, size_t count)
 {
 	unsigned long timeout, write_time;
+	struct at24_client *at24_client;
 	struct i2c_client *client;
-	ssize_t status = 0;
-
-	client = at24_translate_offset(at24, &offset);
-	count = at24_adjust_write_count(at24, offset, count);
-
-	loop_until_timeout(timeout, write_time) {
-		status = i2c_smbus_write_i2c_block_data(client,
-							offset, count, buf);
-		if (status == 0)
-			status = count;
-
-		dev_dbg(&client->dev, "write %zu@%d --> %zd (%ld)\n",
-				count, offset, status, jiffies);
-
-		if (status == count)
-			return count;
-	}
-
-	return -ETIMEDOUT;
-}
-
-static ssize_t at24_eeprom_write_smbus_byte(struct at24_data *at24,
-					    const char *buf,
-					    unsigned int offset, size_t count)
-{
-	unsigned long timeout, write_time;
-	struct i2c_client *client;
-	ssize_t status = 0;
-
-	client = at24_translate_offset(at24, &offset);
-
-	loop_until_timeout(timeout, write_time) {
-		status = i2c_smbus_write_byte_data(client, offset, buf[0]);
-		if (status == 0)
-			status = count;
-
-		dev_dbg(&client->dev, "write %zu@%d --> %zd (%ld)\n",
-				count, offset, status, jiffies);
-
-		if (status == count)
-			return count;
-	}
-
-	return -ETIMEDOUT;
-}
-
-static ssize_t at24_eeprom_write_i2c(struct at24_data *at24, const char *buf,
-				     unsigned int offset, size_t count)
-{
-	unsigned long timeout, write_time;
-	struct i2c_client *client;
-	struct i2c_msg msg;
-	ssize_t status = 0;
-	int i = 0;
+	struct regmap *regmap;
+	int ret;
 
-	client = at24_translate_offset(at24, &offset);
+	at24_client = at24_translate_offset(at24, &offset);
+	regmap = at24_client->regmap;
+	client = at24_client->client;
 	count = at24_adjust_write_count(at24, offset, count);
 
-	msg.addr = client->addr;
-	msg.flags = 0;
-
-	/* msg.buf is u8 and casts will mask the values */
-	msg.buf = at24->writebuf;
-	if (at24->chip.flags & AT24_FLAG_ADDR16)
-		msg.buf[i++] = offset >> 8;
-
-	msg.buf[i++] = offset;
-	memcpy(&msg.buf[i], buf, count);
-	msg.len = i + count;
-
-	loop_until_timeout(timeout, write_time) {
-		status = i2c_transfer(client->adapter, &msg, 1);
-		if (status == 1)
-			status = count;
-
-		dev_dbg(&client->dev, "write %zu@%d --> %zd (%ld)\n",
-				count, offset, status, jiffies);
-
-		if (status == count)
+	at24_loop_until_timeout(timeout, write_time) {
+		ret = regmap_bulk_write(regmap, offset, buf, count);
+		dev_dbg(&client->dev, "write %zu@%d --> %d (%ld)\n",
+			count, offset, ret, jiffies);
+		if (!ret)
 			return count;
 	}
 
@@ -562,7 +372,7 @@ static ssize_t at24_eeprom_write_i2c(struct at24_data *at24, const char *buf,
 static int at24_read(void *priv, unsigned int off, void *val, size_t count)
 {
 	struct at24_data *at24 = priv;
-	struct device *dev = &at24->client[0]->dev;
+	struct device *dev = &at24->client[0].client->dev;
 	char *buf = val;
 	int ret;
 
@@ -587,7 +397,7 @@ static int at24_read(void *priv, unsigned int off, void *val, size_t count)
 	while (count) {
 		int	status;
 
-		status = at24->read_func(at24, buf, off, count);
+		status = at24_regmap_read(at24, buf, off, count);
 		if (status < 0) {
 			mutex_unlock(&at24->lock);
 			pm_runtime_put(dev);
@@ -608,7 +418,7 @@ static int at24_read(void *priv, unsigned int off, void *val, size_t count)
 static int at24_write(void *priv, unsigned int off, void *val, size_t count)
 {
 	struct at24_data *at24 = priv;
-	struct device *dev = &at24->client[0]->dev;
+	struct device *dev = &at24->client[0].client->dev;
 	char *buf = val;
 	int ret;
 
@@ -629,12 +439,14 @@ static int at24_write(void *priv, unsigned int off, void *val, size_t count)
 	 * from this host, but not from other I2C masters.
 	 */
 	mutex_lock(&at24->lock);
+	gpiod_set_value_cansleep(at24->wp_gpio, 0);
 
 	while (count) {
 		int status;
 
-		status = at24->write_func(at24, buf, off, count);
+		status = at24_regmap_write(at24, buf, off, count);
 		if (status < 0) {
+			gpiod_set_value_cansleep(at24->wp_gpio, 1);
 			mutex_unlock(&at24->lock);
 			pm_runtime_put(dev);
 			return status;
@@ -644,6 +456,7 @@ static int at24_write(void *priv, unsigned int off, void *val, size_t count)
 		count -= status;
 	}
 
+	gpiod_set_value_cansleep(at24->wp_gpio, 1);
 	mutex_unlock(&at24->lock);
 
 	pm_runtime_put(dev);
@@ -658,6 +471,8 @@ static void at24_get_pdata(struct device *dev, struct at24_platform_data *chip)
 
 	if (device_property_present(dev, "read-only"))
 		chip->flags |= AT24_FLAG_READONLY;
+	if (device_property_present(dev, "no-read-rollover"))
+		chip->flags |= AT24_FLAG_NO_RDROL;
 
 	err = device_property_read_u32(dev, "size", &val);
 	if (!err)
@@ -676,16 +491,38 @@ static void at24_get_pdata(struct device *dev, struct at24_platform_data *chip)
 	}
 }
 
+static unsigned int at24_get_offset_adj(u8 flags, unsigned int byte_len)
+{
+	if (flags & AT24_FLAG_MAC) {
+		/* EUI-48 starts from 0x9a, EUI-64 from 0x98 */
+		return 0xa0 - byte_len;
+	} else if (flags & AT24_FLAG_SERIAL && flags & AT24_FLAG_ADDR16) {
+		/*
+		 * For 16 bit address pointers, the word address must contain
+		 * a '10' sequence in bits 11 and 10 regardless of the
+		 * intended position of the address pointer.
+		 */
+		return 0x0800;
+	} else if (flags & AT24_FLAG_SERIAL) {
+		/*
+		 * Otherwise the word address must begin with a '10' sequence,
+		 * regardless of the intended address.
+		 */
+		return 0x0080;
+	} else {
+		return 0;
+	}
+}
+
 static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 {
-	struct at24_platform_data chip;
-	kernel_ulong_t magic = 0;
+	struct at24_platform_data chip = { 0 };
+	const struct at24_chip_data *cd = NULL;
 	bool writable;
-	int use_smbus = 0;
-	int use_smbus_write = 0;
 	struct at24_data *at24;
 	int err;
-	unsigned i, num_addresses;
+	unsigned int i, num_addresses;
+	struct regmap_config regmap_config = { };
 	u8 test_byte;
 
 	if (client->dev.platform_data) {
@@ -698,28 +535,22 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		 */
 		if (client->dev.of_node &&
 		    of_match_device(at24_of_match, &client->dev)) {
-			magic = (kernel_ulong_t)
-				of_device_get_match_data(&client->dev);
+			cd = of_device_get_match_data(&client->dev);
 		} else if (id) {
-			magic = id->driver_data;
+			cd = (void *)id->driver_data;
 		} else {
 			const struct acpi_device_id *aid;
 
 			aid = acpi_match_device(at24_acpi_ids, &client->dev);
 			if (aid)
-				magic = aid->driver_data;
+				cd = (void *)aid->driver_data;
 		}
-		if (!magic)
+		if (!cd)
 			return -ENODEV;
 
-		chip.byte_len = BIT(magic & AT24_BITMASK(AT24_SIZE_BYTELEN));
-		magic >>= AT24_SIZE_BYTELEN;
-		chip.flags = magic & AT24_BITMASK(AT24_SIZE_FLAGS);
-
+		chip.byte_len = cd->byte_len;
+		chip.flags = cd->flags;
 		at24_get_pdata(&client->dev, &chip);
-
-		chip.setup = NULL;
-		chip.context = NULL;
 	}
 
 	if (!is_power_of_2(chip.byte_len))
@@ -733,43 +564,10 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		dev_warn(&client->dev,
 			"page_size looks suspicious (no power of 2)!\n");
 
-	/*
-	 * REVISIT: the size of the EUI-48 byte array is 6 in at24mac402, while
-	 * the call to ilog2() in AT24_DEVICE_MAGIC() rounds it down to 4.
-	 *
-	 * Eventually we'll get rid of the magic values altoghether in favor of
-	 * real structs, but for now just manually set the right size.
-	 */
-	if (chip.flags & AT24_FLAG_MAC && chip.byte_len == 4)
-		chip.byte_len = 6;
-
-	/* Use I2C operations unless we're stuck with SMBus extensions. */
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
-		if (chip.flags & AT24_FLAG_ADDR16)
-			return -EPFNOSUPPORT;
-
-		if (i2c_check_functionality(client->adapter,
-				I2C_FUNC_SMBUS_READ_I2C_BLOCK)) {
-			use_smbus = I2C_SMBUS_I2C_BLOCK_DATA;
-		} else if (i2c_check_functionality(client->adapter,
-				I2C_FUNC_SMBUS_READ_WORD_DATA)) {
-			use_smbus = I2C_SMBUS_WORD_DATA;
-		} else if (i2c_check_functionality(client->adapter,
-				I2C_FUNC_SMBUS_READ_BYTE_DATA)) {
-			use_smbus = I2C_SMBUS_BYTE_DATA;
-		} else {
-			return -EPFNOSUPPORT;
-		}
-
-		if (i2c_check_functionality(client->adapter,
-				I2C_FUNC_SMBUS_WRITE_I2C_BLOCK)) {
-			use_smbus_write = I2C_SMBUS_I2C_BLOCK_DATA;
-		} else if (i2c_check_functionality(client->adapter,
-				I2C_FUNC_SMBUS_WRITE_BYTE_DATA)) {
-			use_smbus_write = I2C_SMBUS_BYTE_DATA;
-			chip.page_size = 1;
-		}
-	}
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C) &&
+	    !i2c_check_functionality(client->adapter,
+				     I2C_FUNC_SMBUS_WRITE_I2C_BLOCK))
+		chip.page_size = 1;
 
 	if (chip.flags & AT24_FLAG_TAKE8ADDR)
 		num_addresses = 8;
@@ -777,16 +575,28 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		num_addresses =	DIV_ROUND_UP(chip.byte_len,
 			(chip.flags & AT24_FLAG_ADDR16) ? 65536 : 256);
 
+	regmap_config.val_bits = 8;
+	regmap_config.reg_bits = (chip.flags & AT24_FLAG_ADDR16) ? 16 : 8;
+
 	at24 = devm_kzalloc(&client->dev, sizeof(struct at24_data) +
-		num_addresses * sizeof(struct i2c_client *), GFP_KERNEL);
+		num_addresses * sizeof(struct at24_client), GFP_KERNEL);
 	if (!at24)
 		return -ENOMEM;
 
 	mutex_init(&at24->lock);
-	at24->use_smbus = use_smbus;
-	at24->use_smbus_write = use_smbus_write;
 	at24->chip = chip;
 	at24->num_addresses = num_addresses;
+	at24->offset_adj = at24_get_offset_adj(chip.flags, chip.byte_len);
+
+	at24->wp_gpio = devm_gpiod_get_optional(&client->dev,
+						"wp", GPIOD_OUT_HIGH);
+	if (IS_ERR(at24->wp_gpio))
+		return PTR_ERR(at24->wp_gpio);
+
+	at24->client[0].client = client;
+	at24->client[0].regmap = devm_regmap_init_i2c(client, &regmap_config);
+	if (IS_ERR(at24->client[0].regmap))
+		return PTR_ERR(at24->client[0].regmap);
 
 	if ((chip.flags & AT24_FLAG_SERIAL) && (chip.flags & AT24_FLAG_MAC)) {
 		dev_err(&client->dev,
@@ -794,59 +604,32 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 		return -EINVAL;
 	}
 
-	if (chip.flags & AT24_FLAG_SERIAL) {
-		at24->read_func = at24_eeprom_read_serial;
-	} else if (chip.flags & AT24_FLAG_MAC) {
-		at24->read_func = at24_eeprom_read_mac;
-	} else {
-		at24->read_func = at24->use_smbus ? at24_eeprom_read_smbus
-						  : at24_eeprom_read_i2c;
-	}
-
-	if (at24->use_smbus) {
-		if (at24->use_smbus_write == I2C_SMBUS_I2C_BLOCK_DATA)
-			at24->write_func = at24_eeprom_write_smbus_block;
-		else
-			at24->write_func = at24_eeprom_write_smbus_byte;
-	} else {
-		at24->write_func = at24_eeprom_write_i2c;
-	}
-
 	writable = !(chip.flags & AT24_FLAG_READONLY);
 	if (writable) {
-		if (!use_smbus || use_smbus_write) {
-
-			unsigned write_max = chip.page_size;
-
-			if (write_max > io_limit)
-				write_max = io_limit;
-			if (use_smbus && write_max > I2C_SMBUS_BLOCK_MAX)
-				write_max = I2C_SMBUS_BLOCK_MAX;
-			at24->write_max = write_max;
-
-			/* buffer (data + address at the beginning) */
-			at24->writebuf = devm_kzalloc(&client->dev,
-				write_max + 2, GFP_KERNEL);
-			if (!at24->writebuf)
-				return -ENOMEM;
-		} else {
-			dev_warn(&client->dev,
-				"cannot write due to controller restrictions.");
-		}
+		at24->write_max = min_t(unsigned int,
+					chip.page_size, at24_io_limit);
+		if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C) &&
+		    at24->write_max > I2C_SMBUS_BLOCK_MAX)
+			at24->write_max = I2C_SMBUS_BLOCK_MAX;
 	}
 
-	at24->client[0] = client;
-
 	/* use dummy devices for multiple-address chips */
 	for (i = 1; i < num_addresses; i++) {
-		at24->client[i] = i2c_new_dummy(client->adapter,
-					client->addr + i);
-		if (!at24->client[i]) {
+		at24->client[i].client = i2c_new_dummy(client->adapter,
+						       client->addr + i);
+		if (!at24->client[i].client) {
 			dev_err(&client->dev, "address 0x%02x unavailable\n",
 					client->addr + i);
 			err = -EADDRINUSE;
 			goto err_clients;
 		}
+		at24->client[i].regmap = devm_regmap_init_i2c(
+						at24->client[i].client,
+						&regmap_config);
+		if (IS_ERR(at24->client[i].regmap)) {
+			err = PTR_ERR(at24->client[i].regmap);
+			goto err_clients;
+		}
 	}
 
 	i2c_set_clientdata(client, at24);
@@ -890,12 +673,6 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	dev_info(&client->dev, "%u byte %s EEPROM, %s, %u bytes/write\n",
 		chip.byte_len, client->name,
 		writable ? "writable" : "read-only", at24->write_max);
-	if (use_smbus == I2C_SMBUS_WORD_DATA ||
-	    use_smbus == I2C_SMBUS_BYTE_DATA) {
-		dev_notice(&client->dev, "Falling back to %s reads, "
-			   "performance will suffer\n", use_smbus ==
-			   I2C_SMBUS_WORD_DATA ? "word" : "byte");
-	}
 
 	/* export data to kernel code */
 	if (chip.setup)
@@ -905,8 +682,8 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
 
 err_clients:
 	for (i = 1; i < num_addresses; i++)
-		if (at24->client[i])
-			i2c_unregister_device(at24->client[i]);
+		if (at24->client[i].client)
+			i2c_unregister_device(at24->client[i].client);
 
 	pm_runtime_disable(&client->dev);
 
@@ -923,7 +700,7 @@ static int at24_remove(struct i2c_client *client)
 	nvmem_unregister(at24->nvmem);
 
 	for (i = 1; i < at24->num_addresses; i++)
-		i2c_unregister_device(at24->client[i]);
+		i2c_unregister_device(at24->client[i].client);
 
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
@@ -946,12 +723,12 @@ static struct i2c_driver at24_driver = {
 
 static int __init at24_init(void)
 {
-	if (!io_limit) {
-		pr_err("at24: io_limit must not be 0!\n");
+	if (!at24_io_limit) {
+		pr_err("at24: at24_io_limit must not be 0!\n");
 		return -EINVAL;
 	}
 
-	io_limit = rounddown_pow_of_two(io_limit);
+	at24_io_limit = rounddown_pow_of_two(at24_io_limit);
 	return i2c_add_driver(&at24_driver);
 }
 module_init(at24_init);
diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c
index 5afe4cd16569..9282ffd607ff 100644
--- a/drivers/misc/eeprom/at25.c
+++ b/drivers/misc/eeprom/at25.c
@@ -276,6 +276,9 @@ static int at25_fw_to_chip(struct device *dev, struct spi_eeprom *chip)
 			return -ENODEV;
 		}
 		switch (val) {
+		case 9:
+			chip->flags |= EE_INSTR_BIT3_IS_ADDR;
+			/* fall through */
 		case 8:
 			chip->flags |= EE_ADDR1;
 			break;
diff --git a/drivers/misc/enclosure.c b/drivers/misc/enclosure.c
index eb29113e0bac..5a17bfeb80d3 100644
--- a/drivers/misc/enclosure.c
+++ b/drivers/misc/enclosure.c
@@ -468,7 +468,7 @@ static struct class enclosure_class = {
 	.dev_groups		= enclosure_class_groups,
 };
 
-static const char *const enclosure_status [] = {
+static const char *const enclosure_status[] = {
 	[ENCLOSURE_STATUS_UNSUPPORTED] = "unsupported",
 	[ENCLOSURE_STATUS_OK] = "OK",
 	[ENCLOSURE_STATUS_CRITICAL] = "critical",
@@ -480,7 +480,7 @@ static const char *const enclosure_status [] = {
 	[ENCLOSURE_STATUS_MAX] = NULL,
 };
 
-static const char *const enclosure_type [] = {
+static const char *const enclosure_type[] = {
 	[ENCLOSURE_COMPONENT_DEVICE] = "device",
 	[ENCLOSURE_COMPONENT_ARRAY_DEVICE] = "array device",
 };
@@ -680,13 +680,7 @@ ATTRIBUTE_GROUPS(enclosure_component);
 
 static int __init enclosure_init(void)
 {
-	int err;
-
-	err = class_register(&enclosure_class);
-	if (err)
-		return err;
-
-	return 0;
+	return class_register(&enclosure_class);
 }
 
 static void __exit enclosure_exit(void)
diff --git a/drivers/misc/fsa9480.c b/drivers/misc/fsa9480.c
index 71d2793b372c..607b489a6501 100644
--- a/drivers/misc/fsa9480.c
+++ b/drivers/misc/fsa9480.c
@@ -465,6 +465,7 @@ static int fsa9480_probe(struct i2c_client *client,
 static int fsa9480_remove(struct i2c_client *client)
 {
 	struct fsa9480_usbsw *usbsw = i2c_get_clientdata(client);
+
 	if (client->irq)
 		free_irq(client->irq, usbsw);
 
diff --git a/drivers/misc/genwqe/card_base.c b/drivers/misc/genwqe/card_base.c
index 4fd21e86ad56..c7cd3675bcd1 100644
--- a/drivers/misc/genwqe/card_base.c
+++ b/drivers/misc/genwqe/card_base.c
@@ -153,11 +153,11 @@ static struct genwqe_dev *genwqe_dev_alloc(void)
 	cd->card_state = GENWQE_CARD_UNUSED;
 	spin_lock_init(&cd->print_lock);
 
-	cd->ddcb_software_timeout = genwqe_ddcb_software_timeout;
-	cd->kill_timeout = genwqe_kill_timeout;
+	cd->ddcb_software_timeout = GENWQE_DDCB_SOFTWARE_TIMEOUT;
+	cd->kill_timeout = GENWQE_KILL_TIMEOUT;
 
 	for (j = 0; j < GENWQE_MAX_VFS; j++)
-		cd->vf_jobtimeout_msec[j] = genwqe_vf_jobtimeout_msec;
+		cd->vf_jobtimeout_msec[j] = GENWQE_VF_JOBTIMEOUT_MSEC;
 
 	genwqe_devices[i] = cd;
 	return cd;
@@ -324,11 +324,11 @@ static bool genwqe_setup_pf_jtimer(struct genwqe_dev *cd)
 	u32 T = genwqe_T_psec(cd);
 	u64 x;
 
-	if (genwqe_pf_jobtimeout_msec == 0)
+	if (GENWQE_PF_JOBTIMEOUT_MSEC == 0)
 		return false;
 
 	/* PF: large value needed, flash update 2sec per block */
-	x = ilog2(genwqe_pf_jobtimeout_msec *
+	x = ilog2(GENWQE_PF_JOBTIMEOUT_MSEC *
 		  16000000000uL/(T * 15)) - 10;
 
 	genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
@@ -904,7 +904,7 @@ static int genwqe_reload_bistream(struct genwqe_dev *cd)
  *   b) a critical GFIR occured
  *
  * Informational GFIRs are checked and potentially printed in
- * health_check_interval seconds.
+ * GENWQE_HEALTH_CHECK_INTERVAL seconds.
  */
 static int genwqe_health_thread(void *data)
 {
@@ -918,7 +918,7 @@ static int genwqe_health_thread(void *data)
 		rc = wait_event_interruptible_timeout(cd->health_waitq,
 			 (genwqe_health_check_cond(cd, &gfir) ||
 			  (should_stop = kthread_should_stop())),
-				genwqe_health_check_interval * HZ);
+				GENWQE_HEALTH_CHECK_INTERVAL * HZ);
 
 		if (should_stop)
 			break;
@@ -1028,7 +1028,7 @@ static int genwqe_health_check_start(struct genwqe_dev *cd)
 {
 	int rc;
 
-	if (genwqe_health_check_interval <= 0)
+	if (GENWQE_HEALTH_CHECK_INTERVAL <= 0)
 		return 0;	/* valid for disabling the service */
 
 	/* moved before request_irq() */
diff --git a/drivers/misc/genwqe/card_base.h b/drivers/misc/genwqe/card_base.h
index 3743c87f8ab9..1c3967f10f55 100644
--- a/drivers/misc/genwqe/card_base.h
+++ b/drivers/misc/genwqe/card_base.h
@@ -47,13 +47,13 @@
 #define GENWQE_CARD_NO_MAX		(16 * GENWQE_MAX_FUNCS)
 
 /* Compile parameters, some of them appear in debugfs for later adjustment */
-#define genwqe_ddcb_max			32 /* DDCBs on the work-queue */
-#define genwqe_polling_enabled		0  /* in case of irqs not working */
-#define genwqe_ddcb_software_timeout	10 /* timeout per DDCB in seconds */
-#define genwqe_kill_timeout		8  /* time until process gets killed */
-#define genwqe_vf_jobtimeout_msec	250  /* 250 msec */
-#define genwqe_pf_jobtimeout_msec	8000 /* 8 sec should be ok */
-#define genwqe_health_check_interval	4 /* <= 0: disabled */
+#define GENWQE_DDCB_MAX			32 /* DDCBs on the work-queue */
+#define GENWQE_POLLING_ENABLED		0  /* in case of irqs not working */
+#define GENWQE_DDCB_SOFTWARE_TIMEOUT	10 /* timeout per DDCB in seconds */
+#define GENWQE_KILL_TIMEOUT		8  /* time until process gets killed */
+#define GENWQE_VF_JOBTIMEOUT_MSEC	250  /* 250 msec */
+#define GENWQE_PF_JOBTIMEOUT_MSEC	8000 /* 8 sec should be ok */
+#define GENWQE_HEALTH_CHECK_INTERVAL	4 /* <= 0: disabled */
 
 /* Sysfs attribute groups used when we create the genwqe device */
 extern const struct attribute_group *genwqe_attribute_groups[];
@@ -490,11 +490,9 @@ int  genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len);
 
 /* Memory allocation/deallocation; dma address handling */
 int  genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m,
-		      void *uaddr, unsigned long size,
-		      struct ddcb_requ *req);
+		      void *uaddr, unsigned long size);
 
-int  genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m,
-			struct ddcb_requ *req);
+int  genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m);
 
 static inline bool dma_mapping_used(struct dma_mapping *m)
 {
diff --git a/drivers/misc/genwqe/card_ddcb.c b/drivers/misc/genwqe/card_ddcb.c
index ddfeefe39540..b7f8d35c17a9 100644
--- a/drivers/misc/genwqe/card_ddcb.c
+++ b/drivers/misc/genwqe/card_ddcb.c
@@ -500,7 +500,7 @@ int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 
 	rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
 				ddcb_requ_finished(cd, req),
-				genwqe_ddcb_software_timeout * HZ);
+				GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
 
 	/*
 	 * We need to distinguish 3 cases here:
@@ -633,7 +633,7 @@ int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 	__be32 old, new;
 
 	/* unsigned long flags; */
-	if (genwqe_ddcb_software_timeout <= 0) {
+	if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
 		dev_err(&pci_dev->dev,
 			"[%s] err: software timeout is not set!\n", __func__);
 		return -EFAULT;
@@ -641,7 +641,7 @@ int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 
 	pddcb = &queue->ddcb_vaddr[req->num];
 
-	for (t = 0; t < genwqe_ddcb_software_timeout * 10; t++) {
+	for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
 
 		spin_lock_irqsave(&queue->ddcb_lock, flags);
 
@@ -718,7 +718,7 @@ int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 
 	dev_err(&pci_dev->dev,
 		"[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
-		__func__, req->num, genwqe_ddcb_software_timeout,
+		__func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
 		queue_status);
 
 	print_ddcb_info(cd, req->queue);
@@ -778,7 +778,7 @@ int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
 	/* FIXME circumvention to improve performance when no irq is
 	 * there.
 	 */
-	if (genwqe_polling_enabled)
+	if (GENWQE_POLLING_ENABLED)
 		genwqe_check_ddcb_queue(cd, queue);
 
 	/*
@@ -878,7 +878,7 @@ int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
 	pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
 
 	/* enable DDCB completion irq */
-	if (!genwqe_polling_enabled)
+	if (!GENWQE_POLLING_ENABLED)
 		pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
 
 	dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
@@ -1028,10 +1028,10 @@ static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
 	unsigned int queue_size;
 	struct pci_dev *pci_dev = cd->pci_dev;
 
-	if (genwqe_ddcb_max < 2)
+	if (GENWQE_DDCB_MAX < 2)
 		return -EINVAL;
 
-	queue_size = roundup(genwqe_ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
+	queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
 
 	queue->ddcbs_in_flight = 0;  /* statistics */
 	queue->ddcbs_max_in_flight = 0;
@@ -1040,7 +1040,7 @@ static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
 	queue->wait_on_busy = 0;
 
 	queue->ddcb_seq	  = 0x100; /* start sequence number */
-	queue->ddcb_max	  = genwqe_ddcb_max; /* module parameter */
+	queue->ddcb_max	  = GENWQE_DDCB_MAX;
 	queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
 						&queue->ddcb_daddr);
 	if (queue->ddcb_vaddr == NULL) {
@@ -1194,7 +1194,7 @@ static int genwqe_card_thread(void *data)
 
 		genwqe_check_ddcb_queue(cd, &cd->queue);
 
-		if (genwqe_polling_enabled) {
+		if (GENWQE_POLLING_ENABLED) {
 			rc = wait_event_interruptible_timeout(
 				cd->queue_waitq,
 				genwqe_ddcbs_in_flight(cd) ||
@@ -1340,7 +1340,7 @@ static int queue_wake_up_all(struct genwqe_dev *cd)
 int genwqe_finish_queue(struct genwqe_dev *cd)
 {
 	int i, rc = 0, in_flight;
-	int waitmax = genwqe_ddcb_software_timeout;
+	int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
 	struct pci_dev *pci_dev = cd->pci_dev;
 	struct ddcb_queue *queue = &cd->queue;
 
diff --git a/drivers/misc/genwqe/card_debugfs.c b/drivers/misc/genwqe/card_debugfs.c
index c715534e7fe7..f921dd590271 100644
--- a/drivers/misc/genwqe/card_debugfs.c
+++ b/drivers/misc/genwqe/card_debugfs.c
@@ -198,7 +198,7 @@ static int genwqe_jtimer_show(struct seq_file *s, void *unused)
 
 	jtimer = genwqe_read_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT, 0);
 	seq_printf(s, "  PF   0x%016llx %d msec\n", jtimer,
-		   genwqe_pf_jobtimeout_msec);
+		   GENWQE_PF_JOBTIMEOUT_MSEC);
 
 	for (vf_num = 0; vf_num < cd->num_vfs; vf_num++) {
 		jtimer = genwqe_read_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
diff --git a/drivers/misc/genwqe/card_dev.c b/drivers/misc/genwqe/card_dev.c
index 3ecfa35457e0..0dd6b5ef314a 100644
--- a/drivers/misc/genwqe/card_dev.c
+++ b/drivers/misc/genwqe/card_dev.c
@@ -226,7 +226,7 @@ static void genwqe_remove_mappings(struct genwqe_file *cfile)
 			kfree(dma_map);
 		} else if (dma_map->type == GENWQE_MAPPING_SGL_TEMP) {
 			/* we use dma_map statically from the request */
-			genwqe_user_vunmap(cd, dma_map, NULL);
+			genwqe_user_vunmap(cd, dma_map);
 		}
 	}
 }
@@ -249,7 +249,7 @@ static void genwqe_remove_pinnings(struct genwqe_file *cfile)
 		 * deleted.
 		 */
 		list_del_init(&dma_map->pin_list);
-		genwqe_user_vunmap(cd, dma_map, NULL);
+		genwqe_user_vunmap(cd, dma_map);
 		kfree(dma_map);
 	}
 }
@@ -790,7 +790,7 @@ static int genwqe_pin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
 		return -ENOMEM;
 
 	genwqe_mapping_init(dma_map, GENWQE_MAPPING_SGL_PINNED);
-	rc = genwqe_user_vmap(cd, dma_map, (void *)map_addr, map_size, NULL);
+	rc = genwqe_user_vmap(cd, dma_map, (void *)map_addr, map_size);
 	if (rc != 0) {
 		dev_err(&pci_dev->dev,
 			"[%s] genwqe_user_vmap rc=%d\n", __func__, rc);
@@ -820,7 +820,7 @@ static int genwqe_unpin_mem(struct genwqe_file *cfile, struct genwqe_mem *m)
 		return -ENOENT;
 
 	genwqe_del_pin(cfile, dma_map);
-	genwqe_user_vunmap(cd, dma_map, NULL);
+	genwqe_user_vunmap(cd, dma_map);
 	kfree(dma_map);
 	return 0;
 }
@@ -841,7 +841,7 @@ static int ddcb_cmd_cleanup(struct genwqe_file *cfile, struct ddcb_requ *req)
 
 		if (dma_mapping_used(dma_map)) {
 			__genwqe_del_mapping(cfile, dma_map);
-			genwqe_user_vunmap(cd, dma_map, req);
+			genwqe_user_vunmap(cd, dma_map);
 		}
 		if (req->sgls[i].sgl != NULL)
 			genwqe_free_sync_sgl(cd, &req->sgls[i]);
@@ -947,7 +947,7 @@ static int ddcb_cmd_fixups(struct genwqe_file *cfile, struct ddcb_requ *req)
 					m->write = 0;
 
 				rc = genwqe_user_vmap(cd, m, (void *)u_addr,
-						      u_size, req);
+						      u_size);
 				if (rc != 0)
 					goto err_out;
 
@@ -1011,7 +1011,6 @@ static int do_execute_ddcb(struct genwqe_file *cfile,
 {
 	int rc;
 	struct genwqe_ddcb_cmd *cmd;
-	struct ddcb_requ *req;
 	struct genwqe_dev *cd = cfile->cd;
 	struct file *filp = cfile->filp;
 
@@ -1019,8 +1018,6 @@ static int do_execute_ddcb(struct genwqe_file *cfile,
 	if (cmd == NULL)
 		return -ENOMEM;
 
-	req = container_of(cmd, struct ddcb_requ, cmd);
-
 	if (copy_from_user(cmd, (void __user *)arg, sizeof(*cmd))) {
 		ddcb_requ_free(cmd);
 		return -EFAULT;
@@ -1345,7 +1342,7 @@ static int genwqe_inform_and_stop_processes(struct genwqe_dev *cd)
 	rc = genwqe_kill_fasync(cd, SIGIO);
 	if (rc > 0) {
 		/* give kill_timeout seconds to close file descriptors ... */
-		for (i = 0; (i < genwqe_kill_timeout) &&
+		for (i = 0; (i < GENWQE_KILL_TIMEOUT) &&
 			     genwqe_open_files(cd); i++) {
 			dev_info(&pci_dev->dev, "  %d sec ...", i);
 
@@ -1363,7 +1360,7 @@ static int genwqe_inform_and_stop_processes(struct genwqe_dev *cd)
 		rc = genwqe_force_sig(cd, SIGKILL); /* force terminate */
 		if (rc) {
 			/* Give kill_timout more seconds to end processes */
-			for (i = 0; (i < genwqe_kill_timeout) &&
+			for (i = 0; (i < GENWQE_KILL_TIMEOUT) &&
 				     genwqe_open_files(cd); i++) {
 				dev_warn(&pci_dev->dev, "  %d sec ...", i);
 
diff --git a/drivers/misc/genwqe/card_utils.c b/drivers/misc/genwqe/card_utils.c
index 5c0d917636f7..8f2e6442d88b 100644
--- a/drivers/misc/genwqe/card_utils.c
+++ b/drivers/misc/genwqe/card_utils.c
@@ -524,22 +524,16 @@ int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl)
 }
 
 /**
- * free_user_pages() - Give pinned pages back
+ * genwqe_free_user_pages() - Give pinned pages back
  *
- * Documentation of get_user_pages is in mm/memory.c:
+ * Documentation of get_user_pages is in mm/gup.c:
  *
  * If the page is written to, set_page_dirty (or set_page_dirty_lock,
  * as appropriate) must be called after the page is finished with, and
  * before put_page is called.
- *
- * FIXME Could be of use to others and might belong in the generic
- * code, if others agree. E.g.
- *    ll_free_user_pages in drivers/staging/lustre/lustre/llite/rw26.c
- *    ceph_put_page_vector in net/ceph/pagevec.c
- *    maybe more?
  */
-static int free_user_pages(struct page **page_list, unsigned int nr_pages,
-			   int dirty)
+static int genwqe_free_user_pages(struct page **page_list,
+			unsigned int nr_pages, int dirty)
 {
 	unsigned int i;
 
@@ -577,7 +571,7 @@ static int free_user_pages(struct page **page_list, unsigned int nr_pages,
  * Return: 0 if success
  */
 int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
-		     unsigned long size, struct ddcb_requ *req)
+		     unsigned long size)
 {
 	int rc = -EINVAL;
 	unsigned long data, offs;
@@ -617,7 +611,7 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
 
 	/* assumption: get_user_pages can be killed by signals. */
 	if (rc < m->nr_pages) {
-		free_user_pages(m->page_list, rc, m->write);
+		genwqe_free_user_pages(m->page_list, rc, m->write);
 		rc = -EFAULT;
 		goto fail_get_user_pages;
 	}
@@ -629,7 +623,7 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
 	return 0;
 
  fail_free_user_pages:
-	free_user_pages(m->page_list, m->nr_pages, m->write);
+	genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
 
  fail_get_user_pages:
 	kfree(m->page_list);
@@ -647,8 +641,7 @@ int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr,
  * @cd:         pointer to genwqe device
  * @m:          mapping params
  */
-int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m,
-		       struct ddcb_requ *req)
+int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m)
 {
 	struct pci_dev *pci_dev = cd->pci_dev;
 
@@ -662,7 +655,7 @@ int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m,
 		genwqe_unmap_pages(cd, m->dma_list, m->nr_pages);
 
 	if (m->page_list) {
-		free_user_pages(m->page_list, m->nr_pages, m->write);
+		genwqe_free_user_pages(m->page_list, m->nr_pages, m->write);
 
 		kfree(m->page_list);
 		m->page_list = NULL;
diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c
index 097e3092c158..e9c9ef52c76a 100644
--- a/drivers/misc/hpilo.c
+++ b/drivers/misc/hpilo.c
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Driver for the HP iLO management processor.
  *
  * Copyright (C) 2008 Hewlett-Packard Development Company, L.P.
  *	David Altobelli <david.altobelli@hpe.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 #include <linux/kernel.h>
 #include <linux/types.h>
@@ -514,7 +511,7 @@ static ssize_t ilo_write(struct file *fp, const char __user *buf,
 	return err ? -EFAULT : len;
 }
 
-static unsigned int ilo_poll(struct file *fp, poll_table *wait)
+static __poll_t ilo_poll(struct file *fp, poll_table *wait)
 {
 	struct ccb_data *data = fp->private_data;
 	struct ccb *driver_ccb = &data->driver_ccb;
@@ -522,9 +519,9 @@ static unsigned int ilo_poll(struct file *fp, poll_table *wait)
 	poll_wait(fp, &data->ccb_waitq, wait);
 
 	if (is_channel_reset(driver_ccb))
-		return POLLERR;
+		return EPOLLERR;
 	else if (ilo_pkt_recv(data->ilo_hw, driver_ccb))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 
 	return 0;
 }
diff --git a/drivers/misc/hpilo.h b/drivers/misc/hpilo.h
index b97672e0cf90..94dfb9e40e29 100644
--- a/drivers/misc/hpilo.h
+++ b/drivers/misc/hpilo.h
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * linux/drivers/char/hpilo.h
  *
  * Copyright (C) 2008 Hewlett-Packard Development Company, L.P.
  *	David Altobelli <david.altobelli@hp.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 #ifndef __HPILO_H
 #define __HPILO_H
diff --git a/drivers/misc/ics932s401.c b/drivers/misc/ics932s401.c
index 28f51e01fd2b..81a0541ef3ac 100644
--- a/drivers/misc/ics932s401.c
+++ b/drivers/misc/ics932s401.c
@@ -33,7 +33,7 @@ static const unsigned short normal_i2c[] = { 0x69, I2C_CLIENT_END };
 
 /* ICS932S401 registers */
 #define ICS932S401_REG_CFG2			0x01
-#define 	ICS932S401_CFG1_SPREAD		0x01
+#define		ICS932S401_CFG1_SPREAD		0x01
 #define ICS932S401_REG_CFG7			0x06
 #define		ICS932S401_FS_MASK		0x07
 #define	ICS932S401_REG_VENDOR_REV		0x07
@@ -58,7 +58,7 @@ static const unsigned short normal_i2c[] = { 0x69, I2C_CLIENT_END };
 #define	ICS932S401_REG_SRC_SPREAD1		0x11
 #define ICS932S401_REG_SRC_SPREAD2		0x12
 #define ICS932S401_REG_CPU_DIVISOR		0x13
-#define 	ICS932S401_CPU_DIVISOR_SHIFT	4
+#define		ICS932S401_CPU_DIVISOR_SHIFT	4
 #define ICS932S401_REG_PCISRC_DIVISOR		0x14
 #define		ICS932S401_SRC_DIVISOR_MASK	0x0F
 #define		ICS932S401_PCI_DIVISOR_SHIFT	4
@@ -225,6 +225,7 @@ static ssize_t show_cpu_clock_sel(struct device *dev,
 	else {
 		/* Freq is neatly wrapped up for us */
 		int fid = data->regs[ICS932S401_REG_CFG7] & ICS932S401_FS_MASK;
+
 		freq = fs_speeds[fid];
 		if (data->regs[ICS932S401_REG_CTRL] & ICS932S401_CPU_ALT) {
 			switch (freq) {
@@ -352,8 +353,7 @@ static DEVICE_ATTR(ref_clock, S_IRUGO, show_value, NULL);
 static DEVICE_ATTR(cpu_spread, S_IRUGO, show_spread, NULL);
 static DEVICE_ATTR(src_spread, S_IRUGO, show_spread, NULL);
 
-static struct attribute *ics932s401_attr[] =
-{
+static struct attribute *ics932s401_attr[] = {
 	&dev_attr_spread_enabled.attr,
 	&dev_attr_cpu_clock_selection.attr,
 	&dev_attr_cpu_clock.attr,
diff --git a/drivers/misc/isl29003.c b/drivers/misc/isl29003.c
index 976df0013633..b8032882c865 100644
--- a/drivers/misc/isl29003.c
+++ b/drivers/misc/isl29003.c
@@ -78,6 +78,7 @@ static int __isl29003_read_reg(struct i2c_client *client,
 			       u32 reg, u8 mask, u8 shift)
 {
 	struct isl29003_data *data = i2c_get_clientdata(client);
+
 	return (data->reg_cache[reg] & mask) >> shift;
 }
 
@@ -160,6 +161,7 @@ static int isl29003_get_power_state(struct i2c_client *client)
 {
 	struct isl29003_data *data = i2c_get_clientdata(client);
 	u8 cmdreg = data->reg_cache[ISL29003_REG_COMMAND];
+
 	return ~cmdreg & ISL29003_ADC_PD;
 }
 
@@ -196,6 +198,7 @@ static ssize_t isl29003_show_range(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct i2c_client *client = to_i2c_client(dev);
+
 	return sprintf(buf, "%i\n", isl29003_get_range(client));
 }
 
@@ -231,6 +234,7 @@ static ssize_t isl29003_show_resolution(struct device *dev,
 					char *buf)
 {
 	struct i2c_client *client = to_i2c_client(dev);
+
 	return sprintf(buf, "%d\n", isl29003_get_resolution(client));
 }
 
@@ -264,6 +268,7 @@ static ssize_t isl29003_show_mode(struct device *dev,
 				  struct device_attribute *attr, char *buf)
 {
 	struct i2c_client *client = to_i2c_client(dev);
+
 	return sprintf(buf, "%d\n", isl29003_get_mode(client));
 }
 
@@ -298,6 +303,7 @@ static ssize_t isl29003_show_power_state(struct device *dev,
 					 char *buf)
 {
 	struct i2c_client *client = to_i2c_client(dev);
+
 	return sprintf(buf, "%d\n", isl29003_get_power_state(client));
 }
 
@@ -361,6 +367,7 @@ static int isl29003_init_client(struct i2c_client *client)
 	 * if one of the reads fails, we consider the init failed */
 	for (i = 0; i < ARRAY_SIZE(data->reg_cache); i++) {
 		int v = i2c_smbus_read_byte_data(client, i);
+
 		if (v < 0)
 			return -ENODEV;
 
diff --git a/drivers/misc/lis3lv02d/lis3lv02d.c b/drivers/misc/lis3lv02d/lis3lv02d.c
index 8d53609861d8..e9bb1cfa6a7a 100644
--- a/drivers/misc/lis3lv02d/lis3lv02d.c
+++ b/drivers/misc/lis3lv02d/lis3lv02d.c
@@ -651,14 +651,14 @@ static ssize_t lis3lv02d_misc_read(struct file *file, char __user *buf,
 	return retval;
 }
 
-static unsigned int lis3lv02d_misc_poll(struct file *file, poll_table *wait)
+static __poll_t lis3lv02d_misc_poll(struct file *file, poll_table *wait)
 {
 	struct lis3lv02d *lis3 = container_of(file->private_data,
 					      struct lis3lv02d, miscdev);
 
 	poll_wait(file, &lis3->misc_wait, wait);
 	if (atomic_read(&lis3->count))
-		return POLLIN | POLLRDNORM;
+		return EPOLLIN | EPOLLRDNORM;
 	return 0;
 }
 
diff --git a/drivers/misc/lkdtm.h b/drivers/misc/lkdtm.h
index 687a0dbbe199..9e513dcfd809 100644
--- a/drivers/misc/lkdtm.h
+++ b/drivers/misc/lkdtm.h
@@ -76,8 +76,8 @@ void __init lkdtm_usercopy_init(void);
 void __exit lkdtm_usercopy_exit(void);
 void lkdtm_USERCOPY_HEAP_SIZE_TO(void);
 void lkdtm_USERCOPY_HEAP_SIZE_FROM(void);
-void lkdtm_USERCOPY_HEAP_FLAG_TO(void);
-void lkdtm_USERCOPY_HEAP_FLAG_FROM(void);
+void lkdtm_USERCOPY_HEAP_WHITELIST_TO(void);
+void lkdtm_USERCOPY_HEAP_WHITELIST_FROM(void);
 void lkdtm_USERCOPY_STACK_FRAME_TO(void);
 void lkdtm_USERCOPY_STACK_FRAME_FROM(void);
 void lkdtm_USERCOPY_STACK_BEYOND(void);
diff --git a/drivers/misc/lkdtm_core.c b/drivers/misc/lkdtm_core.c
index ba92291508dc..2154d1bfd18b 100644
--- a/drivers/misc/lkdtm_core.c
+++ b/drivers/misc/lkdtm_core.c
@@ -96,7 +96,7 @@ static struct crashpoint crashpoints[] = {
 	CRASHPOINT("DIRECT",		 NULL),
 #ifdef CONFIG_KPROBES
 	CRASHPOINT("INT_HARDWARE_ENTRY", "do_IRQ"),
-	CRASHPOINT("INT_HW_IRQ_EN",	 "handle_IRQ_event"),
+	CRASHPOINT("INT_HW_IRQ_EN",	 "handle_irq_event"),
 	CRASHPOINT("INT_TASKLET_ENTRY",	 "tasklet_action"),
 	CRASHPOINT("FS_DEVRW",		 "ll_rw_block"),
 	CRASHPOINT("MEM_SWAPOUT",	 "shrink_inactive_list"),
@@ -177,8 +177,8 @@ static const struct crashtype crashtypes[] = {
 	CRASHTYPE(ATOMIC_TIMING),
 	CRASHTYPE(USERCOPY_HEAP_SIZE_TO),
 	CRASHTYPE(USERCOPY_HEAP_SIZE_FROM),
-	CRASHTYPE(USERCOPY_HEAP_FLAG_TO),
-	CRASHTYPE(USERCOPY_HEAP_FLAG_FROM),
+	CRASHTYPE(USERCOPY_HEAP_WHITELIST_TO),
+	CRASHTYPE(USERCOPY_HEAP_WHITELIST_FROM),
 	CRASHTYPE(USERCOPY_STACK_FRAME_TO),
 	CRASHTYPE(USERCOPY_STACK_FRAME_FROM),
 	CRASHTYPE(USERCOPY_STACK_BEYOND),
diff --git a/drivers/misc/lkdtm_heap.c b/drivers/misc/lkdtm_heap.c
index f5494a6d4be5..65026d7de130 100644
--- a/drivers/misc/lkdtm_heap.c
+++ b/drivers/misc/lkdtm_heap.c
@@ -16,6 +16,8 @@ void lkdtm_OVERWRITE_ALLOCATION(void)
 {
 	size_t len = 1020;
 	u32 *data = kmalloc(len, GFP_KERNEL);
+	if (!data)
+		return;
 
 	data[1024 / sizeof(u32)] = 0x12345678;
 	kfree(data);
@@ -33,6 +35,8 @@ void lkdtm_WRITE_AFTER_FREE(void)
 	size_t offset = (len / sizeof(*base)) / 2;
 
 	base = kmalloc(len, GFP_KERNEL);
+	if (!base)
+		return;
 	pr_info("Allocated memory %p-%p\n", base, &base[offset * 2]);
 	pr_info("Attempting bad write to freed memory at %p\n",
 		&base[offset]);
diff --git a/drivers/misc/lkdtm_usercopy.c b/drivers/misc/lkdtm_usercopy.c
index a64372cc148d..9725aed305bb 100644
--- a/drivers/misc/lkdtm_usercopy.c
+++ b/drivers/misc/lkdtm_usercopy.c
@@ -20,7 +20,7 @@
  */
 static volatile size_t unconst = 0;
 static volatile size_t cache_size = 1024;
-static struct kmem_cache *bad_cache;
+static struct kmem_cache *whitelist_cache;
 
 static const unsigned char test_text[] = "This is a test.\n";
 
@@ -115,10 +115,16 @@ static noinline void do_usercopy_stack(bool to_user, bool bad_frame)
 	vm_munmap(user_addr, PAGE_SIZE);
 }
 
+/*
+ * This checks for whole-object size validation with hardened usercopy,
+ * with or without usercopy whitelisting.
+ */
 static void do_usercopy_heap_size(bool to_user)
 {
 	unsigned long user_addr;
 	unsigned char *one, *two;
+	void __user *test_user_addr;
+	void *test_kern_addr;
 	size_t size = unconst + 1024;
 
 	one = kmalloc(size, GFP_KERNEL);
@@ -139,27 +145,30 @@ static void do_usercopy_heap_size(bool to_user)
 	memset(one, 'A', size);
 	memset(two, 'B', size);
 
+	test_user_addr = (void __user *)(user_addr + 16);
+	test_kern_addr = one + 16;
+
 	if (to_user) {
 		pr_info("attempting good copy_to_user of correct size\n");
-		if (copy_to_user((void __user *)user_addr, one, size)) {
+		if (copy_to_user(test_user_addr, test_kern_addr, size / 2)) {
 			pr_warn("copy_to_user failed unexpectedly?!\n");
 			goto free_user;
 		}
 
 		pr_info("attempting bad copy_to_user of too large size\n");
-		if (copy_to_user((void __user *)user_addr, one, 2 * size)) {
+		if (copy_to_user(test_user_addr, test_kern_addr, size)) {
 			pr_warn("copy_to_user failed, but lacked Oops\n");
 			goto free_user;
 		}
 	} else {
 		pr_info("attempting good copy_from_user of correct size\n");
-		if (copy_from_user(one, (void __user *)user_addr, size)) {
+		if (copy_from_user(test_kern_addr, test_user_addr, size / 2)) {
 			pr_warn("copy_from_user failed unexpectedly?!\n");
 			goto free_user;
 		}
 
 		pr_info("attempting bad copy_from_user of too large size\n");
-		if (copy_from_user(one, (void __user *)user_addr, 2 * size)) {
+		if (copy_from_user(test_kern_addr, test_user_addr, size)) {
 			pr_warn("copy_from_user failed, but lacked Oops\n");
 			goto free_user;
 		}
@@ -172,77 +181,79 @@ static void do_usercopy_heap_size(bool to_user)
 	kfree(two);
 }
 
-static void do_usercopy_heap_flag(bool to_user)
+/*
+ * This checks for the specific whitelist window within an object. If this
+ * test passes, then do_usercopy_heap_size() tests will pass too.
+ */
+static void do_usercopy_heap_whitelist(bool to_user)
 {
-	unsigned long user_addr;
-	unsigned char *good_buf = NULL;
-	unsigned char *bad_buf = NULL;
+	unsigned long user_alloc;
+	unsigned char *buf = NULL;
+	unsigned char __user *user_addr;
+	size_t offset, size;
 
 	/* Make sure cache was prepared. */
-	if (!bad_cache) {
+	if (!whitelist_cache) {
 		pr_warn("Failed to allocate kernel cache\n");
 		return;
 	}
 
 	/*
-	 * Allocate one buffer from each cache (kmalloc will have the
-	 * SLAB_USERCOPY flag already, but "bad_cache" won't).
+	 * Allocate a buffer with a whitelisted window in the buffer.
 	 */
-	good_buf = kmalloc(cache_size, GFP_KERNEL);
-	bad_buf = kmem_cache_alloc(bad_cache, GFP_KERNEL);
-	if (!good_buf || !bad_buf) {
-		pr_warn("Failed to allocate buffers from caches\n");
+	buf = kmem_cache_alloc(whitelist_cache, GFP_KERNEL);
+	if (!buf) {
+		pr_warn("Failed to allocate buffer from whitelist cache\n");
 		goto free_alloc;
 	}
 
 	/* Allocate user memory we'll poke at. */
-	user_addr = vm_mmap(NULL, 0, PAGE_SIZE,
+	user_alloc = vm_mmap(NULL, 0, PAGE_SIZE,
 			    PROT_READ | PROT_WRITE | PROT_EXEC,
 			    MAP_ANONYMOUS | MAP_PRIVATE, 0);
-	if (user_addr >= TASK_SIZE) {
+	if (user_alloc >= TASK_SIZE) {
 		pr_warn("Failed to allocate user memory\n");
 		goto free_alloc;
 	}
+	user_addr = (void __user *)user_alloc;
 
-	memset(good_buf, 'A', cache_size);
-	memset(bad_buf, 'B', cache_size);
+	memset(buf, 'B', cache_size);
+
+	/* Whitelisted window in buffer, from kmem_cache_create_usercopy. */
+	offset = (cache_size / 4) + unconst;
+	size = (cache_size / 16) + unconst;
 
 	if (to_user) {
-		pr_info("attempting good copy_to_user with SLAB_USERCOPY\n");
-		if (copy_to_user((void __user *)user_addr, good_buf,
-				 cache_size)) {
+		pr_info("attempting good copy_to_user inside whitelist\n");
+		if (copy_to_user(user_addr, buf + offset, size)) {
 			pr_warn("copy_to_user failed unexpectedly?!\n");
 			goto free_user;
 		}
 
-		pr_info("attempting bad copy_to_user w/o SLAB_USERCOPY\n");
-		if (copy_to_user((void __user *)user_addr, bad_buf,
-				 cache_size)) {
+		pr_info("attempting bad copy_to_user outside whitelist\n");
+		if (copy_to_user(user_addr, buf + offset - 1, size)) {
 			pr_warn("copy_to_user failed, but lacked Oops\n");
 			goto free_user;
 		}
 	} else {
-		pr_info("attempting good copy_from_user with SLAB_USERCOPY\n");
-		if (copy_from_user(good_buf, (void __user *)user_addr,
-				   cache_size)) {
+		pr_info("attempting good copy_from_user inside whitelist\n");
+		if (copy_from_user(buf + offset, user_addr, size)) {
 			pr_warn("copy_from_user failed unexpectedly?!\n");
 			goto free_user;
 		}
 
-		pr_info("attempting bad copy_from_user w/o SLAB_USERCOPY\n");
-		if (copy_from_user(bad_buf, (void __user *)user_addr,
-				   cache_size)) {
+		pr_info("attempting bad copy_from_user outside whitelist\n");
+		if (copy_from_user(buf + offset - 1, user_addr, size)) {
 			pr_warn("copy_from_user failed, but lacked Oops\n");
 			goto free_user;
 		}
 	}
 
 free_user:
-	vm_munmap(user_addr, PAGE_SIZE);
+	vm_munmap(user_alloc, PAGE_SIZE);
 free_alloc:
-	if (bad_buf)
-		kmem_cache_free(bad_cache, bad_buf);
-	kfree(good_buf);
+	if (buf)
+		kmem_cache_free(whitelist_cache, buf);
 }
 
 /* Callable tests. */
@@ -256,14 +267,14 @@ void lkdtm_USERCOPY_HEAP_SIZE_FROM(void)
 	do_usercopy_heap_size(false);
 }
 
-void lkdtm_USERCOPY_HEAP_FLAG_TO(void)
+void lkdtm_USERCOPY_HEAP_WHITELIST_TO(void)
 {
-	do_usercopy_heap_flag(true);
+	do_usercopy_heap_whitelist(true);
 }
 
-void lkdtm_USERCOPY_HEAP_FLAG_FROM(void)
+void lkdtm_USERCOPY_HEAP_WHITELIST_FROM(void)
 {
-	do_usercopy_heap_flag(false);
+	do_usercopy_heap_whitelist(false);
 }
 
 void lkdtm_USERCOPY_STACK_FRAME_TO(void)
@@ -314,11 +325,15 @@ void lkdtm_USERCOPY_KERNEL(void)
 void __init lkdtm_usercopy_init(void)
 {
 	/* Prepare cache that lacks SLAB_USERCOPY flag. */
-	bad_cache = kmem_cache_create("lkdtm-no-usercopy", cache_size, 0,
-				      0, NULL);
+	whitelist_cache =
+		kmem_cache_create_usercopy("lkdtm-usercopy", cache_size,
+					   0, 0,
+					   cache_size / 4,
+					   cache_size / 16,
+					   NULL);
 }
 
 void __exit lkdtm_usercopy_exit(void)
 {
-	kmem_cache_destroy(bad_cache);
+	kmem_cache_destroy(whitelist_cache);
 }
diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c
index 1ac10cb64d6e..772d02922529 100644
--- a/drivers/misc/mei/bus.c
+++ b/drivers/misc/mei/bus.c
@@ -543,14 +543,14 @@ int mei_cldev_disable(struct mei_cl_device *cldev)
 	mutex_lock(&bus->device_lock);
 
 	if (!mei_cl_is_connected(cl)) {
-		dev_dbg(bus->dev, "Already disconnected");
+		dev_dbg(bus->dev, "Already disconnected\n");
 		err = 0;
 		goto out;
 	}
 
 	err = mei_cl_disconnect(cl);
 	if (err < 0)
-		dev_err(bus->dev, "Could not disconnect from the ME client");
+		dev_err(bus->dev, "Could not disconnect from the ME client\n");
 
 out:
 	/* Flush queues and remove any pending read */
diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c
index be64969d986a..7e60c1817c31 100644
--- a/drivers/misc/mei/client.c
+++ b/drivers/misc/mei/client.c
@@ -945,6 +945,12 @@ int mei_cl_disconnect(struct mei_cl *cl)
 		return 0;
 	}
 
+	if (dev->dev_state == MEI_DEV_POWER_DOWN) {
+		cl_dbg(dev, cl, "Device is powering down, don't bother with disconnection\n");
+		mei_cl_set_disconnected(cl);
+		return 0;
+	}
+
 	rets = pm_runtime_get(dev->dev);
 	if (rets < 0 && rets != -EINPROGRESS) {
 		pm_runtime_put_noidle(dev->dev);
diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h
index 0ccccbaf530d..e4b10b2d1a08 100644
--- a/drivers/misc/mei/hw-me-regs.h
+++ b/drivers/misc/mei/hw-me-regs.h
@@ -132,6 +132,11 @@
 #define MEI_DEV_ID_KBP        0xA2BA  /* Kaby Point */
 #define MEI_DEV_ID_KBP_2      0xA2BB  /* Kaby Point 2 */
 
+#define MEI_DEV_ID_CNP_LP     0x9DE0  /* Cannon Point LP */
+#define MEI_DEV_ID_CNP_LP_4   0x9DE4  /* Cannon Point LP 4 (iTouch) */
+#define MEI_DEV_ID_CNP_H      0xA360  /* Cannon Point H */
+#define MEI_DEV_ID_CNP_H_4    0xA364  /* Cannon Point H 4 (iTouch) */
+
 /*
  * MEI HW Section
  */
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
index 10dcf4ff99a5..334ab02e1de2 100644
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -1260,7 +1260,9 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id)
 		if (rets == -ENODATA)
 			break;
 
-		if (rets && dev->dev_state != MEI_DEV_RESETTING) {
+		if (rets &&
+		    (dev->dev_state != MEI_DEV_RESETTING &&
+		     dev->dev_state != MEI_DEV_POWER_DOWN)) {
 			dev_err(dev->dev, "mei_irq_read_handler ret = %d.\n",
 						rets);
 			schedule_work(&dev->reset_work);
diff --git a/drivers/misc/mei/hw-txe.c b/drivers/misc/mei/hw-txe.c
index 24e4a4c96606..c2c8993e2a51 100644
--- a/drivers/misc/mei/hw-txe.c
+++ b/drivers/misc/mei/hw-txe.c
@@ -1127,7 +1127,9 @@ irqreturn_t mei_txe_irq_thread_handler(int irq, void *dev_id)
 	if (test_and_clear_bit(TXE_INTR_OUT_DB_BIT, &hw->intr_cause)) {
 		/* Read from TXE */
 		rets = mei_irq_read_handler(dev, &cmpl_list, &slots);
-		if (rets && dev->dev_state != MEI_DEV_RESETTING) {
+		if (rets &&
+		    (dev->dev_state != MEI_DEV_RESETTING &&
+		     dev->dev_state != MEI_DEV_POWER_DOWN)) {
 			dev_err(dev->dev,
 				"mei_irq_read_handler ret = %d.\n", rets);
 
diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c
index d2f691424dd1..c46f6e99a55e 100644
--- a/drivers/misc/mei/init.c
+++ b/drivers/misc/mei/init.c
@@ -310,6 +310,9 @@ void mei_stop(struct mei_device *dev)
 {
 	dev_dbg(dev->dev, "stopping the device.\n");
 
+	mutex_lock(&dev->device_lock);
+	dev->dev_state = MEI_DEV_POWER_DOWN;
+	mutex_unlock(&dev->device_lock);
 	mei_cl_bus_remove_devices(dev);
 
 	mei_cancel_work(dev);
@@ -319,7 +322,6 @@ void mei_stop(struct mei_device *dev)
 
 	mutex_lock(&dev->device_lock);
 
-	dev->dev_state = MEI_DEV_POWER_DOWN;
 	mei_reset(dev);
 	/* move device to disabled state unconditionally */
 	dev->dev_state = MEI_DEV_DISABLED;
diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c
index e825f013e54e..758dc73602d5 100644
--- a/drivers/misc/mei/main.c
+++ b/drivers/misc/mei/main.c
@@ -542,40 +542,40 @@ static long mei_compat_ioctl(struct file *file,
  *
  * Return: poll mask
  */
-static unsigned int mei_poll(struct file *file, poll_table *wait)
+static __poll_t mei_poll(struct file *file, poll_table *wait)
 {
-	unsigned long req_events = poll_requested_events(wait);
+	__poll_t req_events = poll_requested_events(wait);
 	struct mei_cl *cl = file->private_data;
 	struct mei_device *dev;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 	bool notify_en;
 
 	if (WARN_ON(!cl || !cl->dev))
-		return POLLERR;
+		return EPOLLERR;
 
 	dev = cl->dev;
 
 	mutex_lock(&dev->device_lock);
 
-	notify_en = cl->notify_en && (req_events & POLLPRI);
+	notify_en = cl->notify_en && (req_events & EPOLLPRI);
 
 	if (dev->dev_state != MEI_DEV_ENABLED ||
 	    !mei_cl_is_connected(cl)) {
-		mask = POLLERR;
+		mask = EPOLLERR;
 		goto out;
 	}
 
 	if (notify_en) {
 		poll_wait(file, &cl->ev_wait, wait);
 		if (cl->notify_ev)
-			mask |= POLLPRI;
+			mask |= EPOLLPRI;
 	}
 
-	if (req_events & (POLLIN | POLLRDNORM)) {
+	if (req_events & (EPOLLIN | EPOLLRDNORM)) {
 		poll_wait(file, &cl->rx_wait, wait);
 
 		if (!list_empty(&cl->rd_completed))
-			mask |= POLLIN | POLLRDNORM;
+			mask |= EPOLLIN | EPOLLRDNORM;
 		else
 			mei_cl_read_start(cl, mei_cl_mtu(cl), file);
 	}
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index f4f17552c9b8..ea4e152270a3 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -98,6 +98,11 @@ static const struct pci_device_id mei_me_pci_tbl[] = {
 	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP, MEI_ME_PCH8_CFG)},
 	{MEI_PCI_DEVICE(MEI_DEV_ID_KBP_2, MEI_ME_PCH8_CFG)},
 
+	{MEI_PCI_DEVICE(MEI_DEV_ID_CNP_LP, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_CNP_LP_4, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_CNP_H, MEI_ME_PCH8_CFG)},
+	{MEI_PCI_DEVICE(MEI_DEV_ID_CNP_H_4, MEI_ME_PCH8_CFG)},
+
 	/* required last entry */
 	{0, }
 };
@@ -238,8 +243,11 @@ static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 */
 	mei_me_set_pm_domain(dev);
 
-	if (mei_pg_is_enabled(dev))
+	if (mei_pg_is_enabled(dev)) {
 		pm_runtime_put_noidle(&pdev->dev);
+		if (hw->d0i3_supported)
+			pm_runtime_allow(&pdev->dev);
+	}
 
 	dev_dbg(&pdev->dev, "initialization successful.\n");
 
diff --git a/drivers/misc/mic/cosm/cosm_scif_server.c b/drivers/misc/mic/cosm/cosm_scif_server.c
index 85f7d09cc65f..05a63286741c 100644
--- a/drivers/misc/mic/cosm/cosm_scif_server.c
+++ b/drivers/misc/mic/cosm/cosm_scif_server.c
@@ -55,7 +55,7 @@
  *    message being sent to host SCIF. SCIF_DISCNCT message processing on the
  *    host SCIF sets the host COSM SCIF endpoint state to DISCONNECTED and wakes
  *    up the host COSM thread blocked in scif_poll(..) resulting in
- *    scif_poll(..)  returning POLLHUP.
+ *    scif_poll(..)  returning EPOLLHUP.
  * 5. On the card, scif_peer_release_dev is next called which results in an
  *    SCIF_EXIT message being sent to the host and after receiving the
  *    SCIF_EXIT_ACK from the host the peer device teardown on the card is
@@ -79,7 +79,7 @@
  *    processing. This results in the COSM endpoint on the card being closed and
  *    the SCIF host peer device on the card getting unregistered similar to
  *    steps 3, 4 and 5 for the card shutdown case above. scif_poll(..) on the
- *    host returns POLLHUP as a result.
+ *    host returns EPOLLHUP as a result.
  * 4. On the host, card peer device unregister and SCIF HW remove(..) also
  *    subsequently complete.
  *
@@ -87,11 +87,11 @@
  * ----------
  * If a reset is issued after the card has crashed, there is no SCIF_DISCNT
  * message from the card which would result in scif_poll(..) returning
- * POLLHUP. In this case when the host SCIF driver sends a SCIF_REMOVE_NODE
+ * EPOLLHUP. In this case when the host SCIF driver sends a SCIF_REMOVE_NODE
  * message to itself resulting in the card SCIF peer device being unregistered,
  * this results in a scif_peer_release_dev -> scif_cleanup_scifdev->
  * scif_invalidate_ep call sequence which sets the endpoint state to
- * DISCONNECTED and results in scif_poll(..) returning POLLHUP.
+ * DISCONNECTED and results in scif_poll(..) returning EPOLLHUP.
  */
 
 #define COSM_SCIF_BACKLOG 16
@@ -190,7 +190,7 @@ static void cosm_send_time(struct cosm_device *cdev)
 
 /*
  * Close this cosm_device's endpoint after its peer endpoint on the card has
- * been closed. In all cases except MIC card crash POLLHUP on the host is
+ * been closed. In all cases except MIC card crash EPOLLHUP on the host is
  * triggered by the client's endpoint being closed.
  */
 static void cosm_scif_close(struct cosm_device *cdev)
@@ -252,7 +252,7 @@ void cosm_scif_work(struct work_struct *work)
 
 	while (1) {
 		pollepd.epd = cdev->epd;
-		pollepd.events = POLLIN;
+		pollepd.events = EPOLLIN;
 
 		/* Drop the mutex before blocking in scif_poll(..) */
 		mutex_unlock(&cdev->cosm_mutex);
@@ -266,11 +266,11 @@ void cosm_scif_work(struct work_struct *work)
 		}
 
 		/* There is a message from the card */
-		if (pollepd.revents & POLLIN)
+		if (pollepd.revents & EPOLLIN)
 			cosm_scif_recv(cdev);
 
 		/* The peer endpoint is closed or this endpoint disconnected */
-		if (pollepd.revents & POLLHUP) {
+		if (pollepd.revents & EPOLLHUP) {
 			cosm_scif_close(cdev);
 			break;
 		}
diff --git a/drivers/misc/mic/cosm_client/cosm_scif_client.c b/drivers/misc/mic/cosm_client/cosm_scif_client.c
index aa530fcceaa9..beafc0da4027 100644
--- a/drivers/misc/mic/cosm_client/cosm_scif_client.c
+++ b/drivers/misc/mic/cosm_client/cosm_scif_client.c
@@ -160,7 +160,7 @@ static int cosm_scif_client(void *unused)
 
 	while (!kthread_should_stop()) {
 		pollepd.epd = client_epd;
-		pollepd.events = POLLIN;
+		pollepd.events = EPOLLIN;
 
 		rc = scif_poll(&pollepd, 1, COSM_HEARTBEAT_SEND_MSEC);
 		if (rc < 0) {
@@ -171,7 +171,7 @@ static int cosm_scif_client(void *unused)
 			continue;
 		}
 
-		if (pollepd.revents & POLLIN)
+		if (pollepd.revents & EPOLLIN)
 			cosm_client_recv();
 
 		msg.id = COSM_MSG_HEARTBEAT;
diff --git a/drivers/misc/mic/scif/scif_api.c b/drivers/misc/mic/scif/scif_api.c
index ddc9e4b08b5c..7b2dddcdd46d 100644
--- a/drivers/misc/mic/scif/scif_api.c
+++ b/drivers/misc/mic/scif/scif_api.c
@@ -1311,10 +1311,10 @@ static inline void _scif_poll_wait(struct file *f, wait_queue_head_t *wq,
 	spin_lock(&ep->lock);
 }
 
-unsigned int
+__poll_t
 __scif_pollfd(struct file *f, poll_table *wait, struct scif_endpt *ep)
 {
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	dev_dbg(scif_info.mdev.this_device,
 		"SCIFAPI pollfd: ep %p %s\n", ep, scif_ep_states[ep->state]);
@@ -1328,7 +1328,7 @@ __scif_pollfd(struct file *f, poll_table *wait, struct scif_endpt *ep)
 			if (ep->state == SCIFEP_CONNECTED ||
 			    ep->state == SCIFEP_DISCONNECTED ||
 			    ep->conn_err)
-				mask |= POLLOUT;
+				mask |= EPOLLOUT;
 			goto exit;
 		}
 	}
@@ -1338,34 +1338,34 @@ __scif_pollfd(struct file *f, poll_table *wait, struct scif_endpt *ep)
 		_scif_poll_wait(f, &ep->conwq, wait, ep);
 		if (ep->state == SCIFEP_LISTENING) {
 			if (ep->conreqcnt)
-				mask |= POLLIN;
+				mask |= EPOLLIN;
 			goto exit;
 		}
 	}
 
 	/* Endpoint is connected or disconnected */
 	if (ep->state == SCIFEP_CONNECTED || ep->state == SCIFEP_DISCONNECTED) {
-		if (poll_requested_events(wait) & POLLIN)
+		if (poll_requested_events(wait) & EPOLLIN)
 			_scif_poll_wait(f, &ep->recvwq, wait, ep);
-		if (poll_requested_events(wait) & POLLOUT)
+		if (poll_requested_events(wait) & EPOLLOUT)
 			_scif_poll_wait(f, &ep->sendwq, wait, ep);
 		if (ep->state == SCIFEP_CONNECTED ||
 		    ep->state == SCIFEP_DISCONNECTED) {
 			/* Data can be read without blocking */
 			if (scif_rb_count(&ep->qp_info.qp->inbound_q, 1))
-				mask |= POLLIN;
+				mask |= EPOLLIN;
 			/* Data can be written without blocking */
 			if (scif_rb_space(&ep->qp_info.qp->outbound_q))
-				mask |= POLLOUT;
-			/* Return POLLHUP if endpoint is disconnected */
+				mask |= EPOLLOUT;
+			/* Return EPOLLHUP if endpoint is disconnected */
 			if (ep->state == SCIFEP_DISCONNECTED)
-				mask |= POLLHUP;
+				mask |= EPOLLHUP;
 			goto exit;
 		}
 	}
 
-	/* Return POLLERR if the endpoint is in none of the above states */
-	mask |= POLLERR;
+	/* Return EPOLLERR if the endpoint is in none of the above states */
+	mask |= EPOLLERR;
 exit:
 	spin_unlock(&ep->lock);
 	return mask;
@@ -1389,7 +1389,8 @@ scif_poll(struct scif_pollepd *ufds, unsigned int nfds, long timeout_msecs)
 {
 	struct poll_wqueues table;
 	poll_table *pt;
-	int i, mask, count = 0, timed_out = timeout_msecs == 0;
+	int i, count = 0, timed_out = timeout_msecs == 0;
+	__poll_t mask;
 	u64 timeout = timeout_msecs < 0 ? MAX_SCHEDULE_TIMEOUT
 		: msecs_to_jiffies(timeout_msecs);
 
@@ -1397,10 +1398,10 @@ scif_poll(struct scif_pollepd *ufds, unsigned int nfds, long timeout_msecs)
 	pt = &table.pt;
 	while (1) {
 		for (i = 0; i < nfds; i++) {
-			pt->_key = ufds[i].events | POLLERR | POLLHUP;
+			pt->_key = ufds[i].events | EPOLLERR | EPOLLHUP;
 			mask = __scif_pollfd(ufds[i].epd->anon,
 					     pt, ufds[i].epd);
-			mask &= ufds[i].events | POLLERR | POLLHUP;
+			mask &= ufds[i].events | EPOLLERR | EPOLLHUP;
 			if (mask) {
 				count++;
 				pt->_qproc = NULL;
diff --git a/drivers/misc/mic/scif/scif_epd.h b/drivers/misc/mic/scif/scif_epd.h
index 1771d7a9b8d0..f39b663da287 100644
--- a/drivers/misc/mic/scif/scif_epd.h
+++ b/drivers/misc/mic/scif/scif_epd.h
@@ -203,7 +203,7 @@ void scif_clientrcvd(struct scif_dev *scifdev, struct scifmsg *msg);
 int __scif_connect(scif_epd_t epd, struct scif_port_id *dst, bool non_block);
 int __scif_flush(scif_epd_t epd);
 int scif_mmap(struct vm_area_struct *vma, scif_epd_t epd);
-unsigned int __scif_pollfd(struct file *f, poll_table *wait,
+__poll_t __scif_pollfd(struct file *f, poll_table *wait,
 			   struct scif_endpt *ep);
 int __scif_pin_pages(void *addr, size_t len, int *out_prot,
 		     int map_flags, scif_pinned_pages_t *pages);
diff --git a/drivers/misc/mic/scif/scif_fd.c b/drivers/misc/mic/scif/scif_fd.c
index f7e826142a72..5c2a57ae4f85 100644
--- a/drivers/misc/mic/scif/scif_fd.c
+++ b/drivers/misc/mic/scif/scif_fd.c
@@ -41,7 +41,7 @@ static int scif_fdmmap(struct file *f, struct vm_area_struct *vma)
 	return scif_mmap(vma, priv);
 }
 
-static unsigned int scif_fdpoll(struct file *f, poll_table *wait)
+static __poll_t scif_fdpoll(struct file *f, poll_table *wait)
 {
 	struct scif_endpt *priv = f->private_data;
 
diff --git a/drivers/misc/mic/vop/vop_main.c b/drivers/misc/mic/vop/vop_main.c
index a341938c7e2c..3633202e18f4 100644
--- a/drivers/misc/mic/vop/vop_main.c
+++ b/drivers/misc/mic/vop/vop_main.c
@@ -452,10 +452,12 @@ static irqreturn_t vop_virtio_intr_handler(int irq, void *data)
 
 static void vop_virtio_release_dev(struct device *_d)
 {
-	/*
-	 * No need for a release method similar to virtio PCI.
-	 * Provide an empty one to avoid getting a warning from core.
-	 */
+	struct virtio_device *vdev =
+			container_of(_d, struct virtio_device, dev);
+	struct _vop_vdev *vop_vdev =
+			container_of(vdev, struct _vop_vdev, vdev);
+
+	kfree(vop_vdev);
 }
 
 /*
@@ -466,7 +468,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 			   unsigned int offset, struct vop_device *vpdev,
 			   int dnode)
 {
-	struct _vop_vdev *vdev;
+	struct _vop_vdev *vdev, *reg_dev = NULL;
 	int ret;
 	u8 type = ioread8(&d->type);
 
@@ -497,6 +499,7 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 	vdev->c2h_vdev_db = ioread8(&vdev->dc->c2h_vdev_db);
 
 	ret = register_virtio_device(&vdev->vdev);
+	reg_dev = vdev;
 	if (ret) {
 		dev_err(_vop_dev(vdev),
 			"Failed to register vop device %u type %u\n",
@@ -512,7 +515,10 @@ static int _vop_add_device(struct mic_device_desc __iomem *d,
 free_irq:
 	vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
 kfree:
-	kfree(vdev);
+	if (reg_dev)
+		put_device(&vdev->vdev.dev);
+	else
+		kfree(vdev);
 	return ret;
 }
 
@@ -568,7 +574,7 @@ static int _vop_remove_device(struct mic_device_desc __iomem *d,
 		iowrite8(-1, &dc->h2c_vdev_db);
 		if (status & VIRTIO_CONFIG_S_DRIVER_OK)
 			wait_for_completion(&vdev->reset_done);
-		kfree(vdev);
+		put_device(&vdev->vdev.dev);
 		iowrite8(1, &dc->guest_ack);
 		dev_dbg(&vpdev->dev, "%s %d guest_ack %d\n",
 			__func__, __LINE__, ioread8(&dc->guest_ack));
diff --git a/drivers/misc/mic/vop/vop_vringh.c b/drivers/misc/mic/vop/vop_vringh.c
index fed992e2c258..cbc8ebcff5cf 100644
--- a/drivers/misc/mic/vop/vop_vringh.c
+++ b/drivers/misc/mic/vop/vop_vringh.c
@@ -937,13 +937,10 @@ static long vop_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
 		    dd.num_vq > MIC_MAX_VRINGS)
 			return -EINVAL;
 
-		dd_config = kzalloc(mic_desc_size(&dd), GFP_KERNEL);
-		if (!dd_config)
-			return -ENOMEM;
-		if (copy_from_user(dd_config, argp, mic_desc_size(&dd))) {
-			ret = -EFAULT;
-			goto free_ret;
-		}
+		dd_config = memdup_user(argp, mic_desc_size(&dd));
+		if (IS_ERR(dd_config))
+			return PTR_ERR(dd_config);
+
 		/* Ensure desc has not changed between the two reads */
 		if (memcmp(&dd, dd_config, sizeof(dd))) {
 			ret = -EINVAL;
@@ -995,17 +992,12 @@ static long vop_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
 		ret = vop_vdev_inited(vdev);
 		if (ret)
 			goto __unlock_ret;
-		buf = kzalloc(vdev->dd->config_len, GFP_KERNEL);
-		if (!buf) {
-			ret = -ENOMEM;
+		buf = memdup_user(argp, vdev->dd->config_len);
+		if (IS_ERR(buf)) {
+			ret = PTR_ERR(buf);
 			goto __unlock_ret;
 		}
-		if (copy_from_user(buf, argp, vdev->dd->config_len)) {
-			ret = -EFAULT;
-			goto done;
-		}
 		ret = vop_virtio_config_change(vdev, buf);
-done:
 		kfree(buf);
 __unlock_ret:
 		mutex_unlock(&vdev->vdev_mutex);
@@ -1018,27 +1010,27 @@ static long vop_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
 }
 
 /*
- * We return POLLIN | POLLOUT from poll when new buffers are enqueued, and
+ * We return EPOLLIN | EPOLLOUT from poll when new buffers are enqueued, and
  * not when previously enqueued buffers may be available. This means that
  * in the card->host (TX) path, when userspace is unblocked by poll it
  * must drain all available descriptors or it can stall.
  */
-static unsigned int vop_poll(struct file *f, poll_table *wait)
+static __poll_t vop_poll(struct file *f, poll_table *wait)
 {
 	struct vop_vdev *vdev = f->private_data;
-	int mask = 0;
+	__poll_t mask = 0;
 
 	mutex_lock(&vdev->vdev_mutex);
 	if (vop_vdev_inited(vdev)) {
-		mask = POLLERR;
+		mask = EPOLLERR;
 		goto done;
 	}
 	poll_wait(f, &vdev->waitq, wait);
 	if (vop_vdev_inited(vdev)) {
-		mask = POLLERR;
+		mask = EPOLLERR;
 	} else if (vdev->poll_wake) {
 		vdev->poll_wake = 0;
-		mask = POLLIN | POLLOUT;
+		mask = EPOLLIN | EPOLLOUT;
 	}
 done:
 	mutex_unlock(&vdev->vdev_mutex);
diff --git a/drivers/misc/ocxl/Kconfig b/drivers/misc/ocxl/Kconfig
new file mode 100644
index 000000000000..4bbdb0d3c8ee
--- /dev/null
+++ b/drivers/misc/ocxl/Kconfig
@@ -0,0 +1,31 @@
+#
+# Open Coherent Accelerator (OCXL) compatible devices
+#
+
+config OCXL_BASE
+	bool
+	default n
+	select PPC_COPRO_BASE
+
+config OCXL
+	tristate "OpenCAPI coherent accelerator support"
+	depends on PPC_POWERNV && PCI && EEH
+	select OCXL_BASE
+	default m
+	help
+	  Select this option to enable the ocxl driver for Open
+	  Coherent Accelerator Processor Interface (OpenCAPI) devices.
+
+	  OpenCAPI allows FPGA and ASIC accelerators to be coherently
+	  attached to a CPU over an OpenCAPI link.
+
+	  The ocxl driver enables userspace programs to access these
+	  accelerators through devices in /dev/ocxl/.
+
+	  For more information, see http://opencapi.org.
+
+	  This is not to be confused with the support for IBM CAPI
+	  accelerators (CONFIG_CXL), which are PCI-based instead of a
+	  dedicated OpenCAPI link, and don't follow the same protocol.
+
+	  If unsure, say N.
diff --git a/drivers/misc/ocxl/Makefile b/drivers/misc/ocxl/Makefile
new file mode 100644
index 000000000000..5229dcda8297
--- /dev/null
+++ b/drivers/misc/ocxl/Makefile
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0+
+ccflags-$(CONFIG_PPC_WERROR)	+= -Werror
+
+ocxl-y				+= main.o pci.o config.o file.o pasid.o
+ocxl-y				+= link.o context.o afu_irq.o sysfs.o trace.o
+obj-$(CONFIG_OCXL)		+= ocxl.o
+
+# For tracepoints to include our trace.h from tracepoint infrastructure:
+CFLAGS_trace.o := -I$(src)
+
+# ccflags-y += -DDEBUG
diff --git a/drivers/misc/ocxl/afu_irq.c b/drivers/misc/ocxl/afu_irq.c
new file mode 100644
index 000000000000..e70cfa24577f
--- /dev/null
+++ b/drivers/misc/ocxl/afu_irq.c
@@ -0,0 +1,202 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/interrupt.h>
+#include <linux/eventfd.h>
+#include <asm/pnv-ocxl.h>
+#include "ocxl_internal.h"
+#include "trace.h"
+
+struct afu_irq {
+	int id;
+	int hw_irq;
+	unsigned int virq;
+	char *name;
+	u64 trigger_page;
+	struct eventfd_ctx *ev_ctx;
+};
+
+static int irq_offset_to_id(struct ocxl_context *ctx, u64 offset)
+{
+	return (offset - ctx->afu->irq_base_offset) >> PAGE_SHIFT;
+}
+
+static u64 irq_id_to_offset(struct ocxl_context *ctx, int id)
+{
+	return ctx->afu->irq_base_offset + (id << PAGE_SHIFT);
+}
+
+static irqreturn_t afu_irq_handler(int virq, void *data)
+{
+	struct afu_irq *irq = (struct afu_irq *) data;
+
+	trace_ocxl_afu_irq_receive(virq);
+	if (irq->ev_ctx)
+		eventfd_signal(irq->ev_ctx, 1);
+	return IRQ_HANDLED;
+}
+
+static int setup_afu_irq(struct ocxl_context *ctx, struct afu_irq *irq)
+{
+	int rc;
+
+	irq->virq = irq_create_mapping(NULL, irq->hw_irq);
+	if (!irq->virq) {
+		pr_err("irq_create_mapping failed\n");
+		return -ENOMEM;
+	}
+	pr_debug("hw_irq %d mapped to virq %u\n", irq->hw_irq, irq->virq);
+
+	irq->name = kasprintf(GFP_KERNEL, "ocxl-afu-%u", irq->virq);
+	if (!irq->name) {
+		irq_dispose_mapping(irq->virq);
+		return -ENOMEM;
+	}
+
+	rc = request_irq(irq->virq, afu_irq_handler, 0, irq->name, irq);
+	if (rc) {
+		kfree(irq->name);
+		irq->name = NULL;
+		irq_dispose_mapping(irq->virq);
+		pr_err("request_irq failed: %d\n", rc);
+		return rc;
+	}
+	return 0;
+}
+
+static void release_afu_irq(struct afu_irq *irq)
+{
+	free_irq(irq->virq, irq);
+	irq_dispose_mapping(irq->virq);
+	kfree(irq->name);
+}
+
+int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset)
+{
+	struct afu_irq *irq;
+	int rc;
+
+	irq = kzalloc(sizeof(struct afu_irq), GFP_KERNEL);
+	if (!irq)
+		return -ENOMEM;
+
+	/*
+	 * We limit the number of afu irqs per context and per link to
+	 * avoid a single process or user depleting the pool of IPIs
+	 */
+
+	mutex_lock(&ctx->irq_lock);
+
+	irq->id = idr_alloc(&ctx->irq_idr, irq, 0, MAX_IRQ_PER_CONTEXT,
+			GFP_KERNEL);
+	if (irq->id < 0) {
+		rc = -ENOSPC;
+		goto err_unlock;
+	}
+
+	rc = ocxl_link_irq_alloc(ctx->afu->fn->link, &irq->hw_irq,
+				&irq->trigger_page);
+	if (rc)
+		goto err_idr;
+
+	rc = setup_afu_irq(ctx, irq);
+	if (rc)
+		goto err_alloc;
+
+	*irq_offset = irq_id_to_offset(ctx, irq->id);
+
+	trace_ocxl_afu_irq_alloc(ctx->pasid, irq->id, irq->virq, irq->hw_irq,
+				*irq_offset);
+	mutex_unlock(&ctx->irq_lock);
+	return 0;
+
+err_alloc:
+	ocxl_link_free_irq(ctx->afu->fn->link, irq->hw_irq);
+err_idr:
+	idr_remove(&ctx->irq_idr, irq->id);
+err_unlock:
+	mutex_unlock(&ctx->irq_lock);
+	kfree(irq);
+	return rc;
+}
+
+static void afu_irq_free(struct afu_irq *irq, struct ocxl_context *ctx)
+{
+	trace_ocxl_afu_irq_free(ctx->pasid, irq->id);
+	if (ctx->mapping)
+		unmap_mapping_range(ctx->mapping,
+				irq_id_to_offset(ctx, irq->id),
+				1 << PAGE_SHIFT, 1);
+	release_afu_irq(irq);
+	if (irq->ev_ctx)
+		eventfd_ctx_put(irq->ev_ctx);
+	ocxl_link_free_irq(ctx->afu->fn->link, irq->hw_irq);
+	kfree(irq);
+}
+
+int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset)
+{
+	struct afu_irq *irq;
+	int id = irq_offset_to_id(ctx, irq_offset);
+
+	mutex_lock(&ctx->irq_lock);
+
+	irq = idr_find(&ctx->irq_idr, id);
+	if (!irq) {
+		mutex_unlock(&ctx->irq_lock);
+		return -EINVAL;
+	}
+	idr_remove(&ctx->irq_idr, irq->id);
+	afu_irq_free(irq, ctx);
+	mutex_unlock(&ctx->irq_lock);
+	return 0;
+}
+
+void ocxl_afu_irq_free_all(struct ocxl_context *ctx)
+{
+	struct afu_irq *irq;
+	int id;
+
+	mutex_lock(&ctx->irq_lock);
+	idr_for_each_entry(&ctx->irq_idr, irq, id)
+		afu_irq_free(irq, ctx);
+	mutex_unlock(&ctx->irq_lock);
+}
+
+int ocxl_afu_irq_set_fd(struct ocxl_context *ctx, u64 irq_offset, int eventfd)
+{
+	struct afu_irq *irq;
+	struct eventfd_ctx *ev_ctx;
+	int rc = 0, id = irq_offset_to_id(ctx, irq_offset);
+
+	mutex_lock(&ctx->irq_lock);
+	irq = idr_find(&ctx->irq_idr, id);
+	if (!irq) {
+		rc = -EINVAL;
+		goto unlock;
+	}
+
+	ev_ctx = eventfd_ctx_fdget(eventfd);
+	if (IS_ERR(ev_ctx)) {
+		rc = -EINVAL;
+		goto unlock;
+	}
+
+	irq->ev_ctx = ev_ctx;
+unlock:
+	mutex_unlock(&ctx->irq_lock);
+	return rc;
+}
+
+u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, u64 irq_offset)
+{
+	struct afu_irq *irq;
+	int id = irq_offset_to_id(ctx, irq_offset);
+	u64 addr = 0;
+
+	mutex_lock(&ctx->irq_lock);
+	irq = idr_find(&ctx->irq_idr, id);
+	if (irq)
+		addr = irq->trigger_page;
+	mutex_unlock(&ctx->irq_lock);
+	return addr;
+}
diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c
new file mode 100644
index 000000000000..2e30de9c694a
--- /dev/null
+++ b/drivers/misc/ocxl/config.c
@@ -0,0 +1,723 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/pci.h>
+#include <asm/pnv-ocxl.h>
+#include <misc/ocxl.h>
+#include <misc/ocxl-config.h>
+
+#define EXTRACT_BIT(val, bit) (!!(val & BIT(bit)))
+#define EXTRACT_BITS(val, s, e) ((val & GENMASK(e, s)) >> s)
+
+#define OCXL_DVSEC_AFU_IDX_MASK              GENMASK(5, 0)
+#define OCXL_DVSEC_ACTAG_MASK                GENMASK(11, 0)
+#define OCXL_DVSEC_PASID_MASK                GENMASK(19, 0)
+#define OCXL_DVSEC_PASID_LOG_MASK            GENMASK(4, 0)
+
+#define OCXL_DVSEC_TEMPL_VERSION         0x0
+#define OCXL_DVSEC_TEMPL_NAME            0x4
+#define OCXL_DVSEC_TEMPL_AFU_VERSION     0x1C
+#define OCXL_DVSEC_TEMPL_MMIO_GLOBAL     0x20
+#define OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ  0x28
+#define OCXL_DVSEC_TEMPL_MMIO_PP         0x30
+#define OCXL_DVSEC_TEMPL_MMIO_PP_SZ      0x38
+#define OCXL_DVSEC_TEMPL_MEM_SZ          0x3C
+#define OCXL_DVSEC_TEMPL_WWID            0x40
+
+#define OCXL_MAX_AFU_PER_FUNCTION 64
+#define OCXL_TEMPL_LEN            0x58
+#define OCXL_TEMPL_NAME_LEN       24
+#define OCXL_CFG_TIMEOUT     3
+
+static int find_dvsec(struct pci_dev *dev, int dvsec_id)
+{
+	int vsec = 0;
+	u16 vendor, id;
+
+	while ((vsec = pci_find_next_ext_capability(dev, vsec,
+						    OCXL_EXT_CAP_ID_DVSEC))) {
+		pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
+				&vendor);
+		pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
+		if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id)
+			return vsec;
+	}
+	return 0;
+}
+
+static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
+{
+	int vsec = 0;
+	u16 vendor, id;
+	u8 idx;
+
+	while ((vsec = pci_find_next_ext_capability(dev, vsec,
+						    OCXL_EXT_CAP_ID_DVSEC))) {
+		pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET,
+				&vendor);
+		pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id);
+
+		if (vendor == PCI_VENDOR_ID_IBM &&
+			id == OCXL_DVSEC_AFU_CTRL_ID) {
+			pci_read_config_byte(dev,
+					vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX,
+					&idx);
+			if (idx == afu_idx)
+				return vsec;
+		}
+	}
+	return 0;
+}
+
+static int read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+	u16 val;
+	int pos;
+
+	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PASID);
+	if (!pos) {
+		/*
+		 * PASID capability is not mandatory, but there
+		 * shouldn't be any AFU
+		 */
+		dev_dbg(&dev->dev, "Function doesn't require any PASID\n");
+		fn->max_pasid_log = -1;
+		goto out;
+	}
+	pci_read_config_word(dev, pos + PCI_PASID_CAP, &val);
+	fn->max_pasid_log = EXTRACT_BITS(val, 8, 12);
+
+out:
+	dev_dbg(&dev->dev, "PASID capability:\n");
+	dev_dbg(&dev->dev, "  Max PASID log = %d\n", fn->max_pasid_log);
+	return 0;
+}
+
+static int read_dvsec_tl(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+	int pos;
+
+	pos = find_dvsec(dev, OCXL_DVSEC_TL_ID);
+	if (!pos && PCI_FUNC(dev->devfn) == 0) {
+		dev_err(&dev->dev, "Can't find TL DVSEC\n");
+		return -ENODEV;
+	}
+	if (pos && PCI_FUNC(dev->devfn) != 0) {
+		dev_err(&dev->dev, "TL DVSEC is only allowed on function 0\n");
+		return -ENODEV;
+	}
+	fn->dvsec_tl_pos = pos;
+	return 0;
+}
+
+static int read_dvsec_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+	int pos, afu_present;
+	u32 val;
+
+	pos = find_dvsec(dev, OCXL_DVSEC_FUNC_ID);
+	if (!pos) {
+		dev_err(&dev->dev, "Can't find function DVSEC\n");
+		return -ENODEV;
+	}
+	fn->dvsec_function_pos = pos;
+
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val);
+	afu_present = EXTRACT_BIT(val, 31);
+	if (!afu_present) {
+		fn->max_afu_index = -1;
+		dev_dbg(&dev->dev, "Function doesn't define any AFU\n");
+		goto out;
+	}
+	fn->max_afu_index = EXTRACT_BITS(val, 24, 29);
+
+out:
+	dev_dbg(&dev->dev, "Function DVSEC:\n");
+	dev_dbg(&dev->dev, "  Max AFU index = %d\n", fn->max_afu_index);
+	return 0;
+}
+
+static int read_dvsec_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+	int pos;
+
+	if (fn->max_afu_index < 0) {
+		fn->dvsec_afu_info_pos = -1;
+		return 0;
+	}
+
+	pos = find_dvsec(dev, OCXL_DVSEC_AFU_INFO_ID);
+	if (!pos) {
+		dev_err(&dev->dev, "Can't find AFU information DVSEC\n");
+		return -ENODEV;
+	}
+	fn->dvsec_afu_info_pos = pos;
+	return 0;
+}
+
+static int read_dvsec_vendor(struct pci_dev *dev)
+{
+	int pos;
+	u32 cfg, tlx, dlx;
+
+	/*
+	 * vendor specific DVSEC is optional
+	 *
+	 * It's currently only used on function 0 to specify the
+	 * version of some logic blocks. Some older images may not
+	 * even have it so we ignore any errors
+	 */
+	if (PCI_FUNC(dev->devfn) != 0)
+		return 0;
+
+	pos = find_dvsec(dev, OCXL_DVSEC_VENDOR_ID);
+	if (!pos)
+		return 0;
+
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_CFG_VERS, &cfg);
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_TLX_VERS, &tlx);
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_VENDOR_DLX_VERS, &dlx);
+
+	dev_dbg(&dev->dev, "Vendor specific DVSEC:\n");
+	dev_dbg(&dev->dev, "  CFG version = 0x%x\n", cfg);
+	dev_dbg(&dev->dev, "  TLX version = 0x%x\n", tlx);
+	dev_dbg(&dev->dev, "  DLX version = 0x%x\n", dlx);
+	return 0;
+}
+
+static int validate_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+	if (fn->max_pasid_log == -1 && fn->max_afu_index >= 0) {
+		dev_err(&dev->dev,
+			"AFUs are defined but no PASIDs are requested\n");
+		return -EINVAL;
+	}
+
+	if (fn->max_afu_index > OCXL_MAX_AFU_PER_FUNCTION) {
+		dev_err(&dev->dev,
+			"Max AFU index out of architectural limit (%d vs %d)\n",
+			fn->max_afu_index, OCXL_MAX_AFU_PER_FUNCTION);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int ocxl_config_read_function(struct pci_dev *dev, struct ocxl_fn_config *fn)
+{
+	int rc;
+
+	rc = read_pasid(dev, fn);
+	if (rc) {
+		dev_err(&dev->dev, "Invalid PASID configuration: %d\n", rc);
+		return -ENODEV;
+	}
+
+	rc = read_dvsec_tl(dev, fn);
+	if (rc) {
+		dev_err(&dev->dev,
+			"Invalid Transaction Layer DVSEC configuration: %d\n",
+			rc);
+		return -ENODEV;
+	}
+
+	rc = read_dvsec_function(dev, fn);
+	if (rc) {
+		dev_err(&dev->dev,
+			"Invalid Function DVSEC configuration: %d\n", rc);
+		return -ENODEV;
+	}
+
+	rc = read_dvsec_afu_info(dev, fn);
+	if (rc) {
+		dev_err(&dev->dev, "Invalid AFU configuration: %d\n", rc);
+		return -ENODEV;
+	}
+
+	rc = read_dvsec_vendor(dev);
+	if (rc) {
+		dev_err(&dev->dev,
+			"Invalid vendor specific DVSEC configuration: %d\n",
+			rc);
+		return -ENODEV;
+	}
+
+	rc = validate_function(dev, fn);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(ocxl_config_read_function);
+
+static int read_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn,
+			int offset, u32 *data)
+{
+	u32 val;
+	unsigned long timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
+	int pos = fn->dvsec_afu_info_pos;
+
+	/* Protect 'data valid' bit */
+	if (EXTRACT_BIT(offset, 31)) {
+		dev_err(&dev->dev, "Invalid offset in AFU info DVSEC\n");
+		return -EINVAL;
+	}
+
+	pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, offset);
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
+	while (!EXTRACT_BIT(val, 31)) {
+		if (time_after_eq(jiffies, timeout)) {
+			dev_err(&dev->dev,
+				"Timeout while reading AFU info DVSEC (offset=%d)\n",
+				offset);
+			return -EBUSY;
+		}
+		cpu_relax();
+		pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_OFF, &val);
+	}
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_INFO_DATA, data);
+	return 0;
+}
+
+int ocxl_config_check_afu_index(struct pci_dev *dev,
+				struct ocxl_fn_config *fn, int afu_idx)
+{
+	u32 val;
+	int rc, templ_major, templ_minor, len;
+
+	pci_write_config_word(dev, fn->dvsec_afu_info_pos, afu_idx);
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_VERSION, &val);
+	if (rc)
+		return rc;
+
+	/* AFU index map can have holes */
+	if (!val)
+		return 0;
+
+	templ_major = EXTRACT_BITS(val, 8, 15);
+	templ_minor = EXTRACT_BITS(val, 0, 7);
+	dev_dbg(&dev->dev, "AFU descriptor template version %d.%d\n",
+		templ_major, templ_minor);
+
+	len = EXTRACT_BITS(val, 16, 31);
+	if (len != OCXL_TEMPL_LEN) {
+		dev_warn(&dev->dev,
+			"Unexpected template length in AFU information (%#x)\n",
+			len);
+	}
+	return 1;
+}
+EXPORT_SYMBOL_GPL(ocxl_config_check_afu_index);
+
+static int read_afu_name(struct pci_dev *dev, struct ocxl_fn_config *fn,
+			struct ocxl_afu_config *afu)
+{
+	int i, rc;
+	u32 val, *ptr;
+
+	BUILD_BUG_ON(OCXL_AFU_NAME_SZ < OCXL_TEMPL_NAME_LEN);
+	for (i = 0; i < OCXL_TEMPL_NAME_LEN; i += 4) {
+		rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_NAME + i, &val);
+		if (rc)
+			return rc;
+		ptr = (u32 *) &afu->name[i];
+		*ptr = val;
+	}
+	afu->name[OCXL_AFU_NAME_SZ - 1] = '\0'; /* play safe */
+	return 0;
+}
+
+static int read_afu_mmio(struct pci_dev *dev, struct ocxl_fn_config *fn,
+			struct ocxl_afu_config *afu)
+{
+	int rc;
+	u32 val;
+
+	/*
+	 * Global MMIO
+	 */
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL, &val);
+	if (rc)
+		return rc;
+	afu->global_mmio_bar = EXTRACT_BITS(val, 0, 2);
+	afu->global_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
+
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL + 4, &val);
+	if (rc)
+		return rc;
+	afu->global_mmio_offset += (u64) val << 32;
+
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_GLOBAL_SZ, &val);
+	if (rc)
+		return rc;
+	afu->global_mmio_size = val;
+
+	/*
+	 * Per-process MMIO
+	 */
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP, &val);
+	if (rc)
+		return rc;
+	afu->pp_mmio_bar = EXTRACT_BITS(val, 0, 2);
+	afu->pp_mmio_offset = EXTRACT_BITS(val, 16, 31) << 16;
+
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP + 4, &val);
+	if (rc)
+		return rc;
+	afu->pp_mmio_offset += (u64) val << 32;
+
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MMIO_PP_SZ, &val);
+	if (rc)
+		return rc;
+	afu->pp_mmio_stride = val;
+
+	return 0;
+}
+
+static int read_afu_control(struct pci_dev *dev, struct ocxl_afu_config *afu)
+{
+	int pos;
+	u8 val8;
+	u16 val16;
+
+	pos = find_dvsec_afu_ctrl(dev, afu->idx);
+	if (!pos) {
+		dev_err(&dev->dev, "Can't find AFU control DVSEC for AFU %d\n",
+			afu->idx);
+		return -ENODEV;
+	}
+	afu->dvsec_afu_control_pos = pos;
+
+	pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_SUP, &val8);
+	afu->pasid_supported_log = EXTRACT_BITS(val8, 0, 4);
+
+	pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, &val16);
+	afu->actag_supported = EXTRACT_BITS(val16, 0, 11);
+	return 0;
+}
+
+static bool char_allowed(int c)
+{
+	/*
+	 * Permitted Characters : Alphanumeric, hyphen, underscore, comma
+	 */
+	if ((c >= 0x30 && c <= 0x39) /* digits */ ||
+		(c >= 0x41 && c <= 0x5A) /* upper case */ ||
+		(c >= 0x61 && c <= 0x7A) /* lower case */ ||
+		c == 0 /* NULL */ ||
+		c == 0x2D /* - */ ||
+		c == 0x5F /* _ */ ||
+		c == 0x2C /* , */)
+		return true;
+	return false;
+}
+
+static int validate_afu(struct pci_dev *dev, struct ocxl_afu_config *afu)
+{
+	int i;
+
+	if (!afu->name[0]) {
+		dev_err(&dev->dev, "Empty AFU name\n");
+		return -EINVAL;
+	}
+	for (i = 0; i < OCXL_TEMPL_NAME_LEN; i++) {
+		if (!char_allowed(afu->name[i])) {
+			dev_err(&dev->dev,
+				"Invalid character in AFU name\n");
+			return -EINVAL;
+		}
+	}
+
+	if (afu->global_mmio_bar != 0 &&
+		afu->global_mmio_bar != 2 &&
+		afu->global_mmio_bar != 4) {
+		dev_err(&dev->dev, "Invalid global MMIO bar number\n");
+		return -EINVAL;
+	}
+	if (afu->pp_mmio_bar != 0 &&
+		afu->pp_mmio_bar != 2 &&
+		afu->pp_mmio_bar != 4) {
+		dev_err(&dev->dev, "Invalid per-process MMIO bar number\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int ocxl_config_read_afu(struct pci_dev *dev, struct ocxl_fn_config *fn,
+			struct ocxl_afu_config *afu, u8 afu_idx)
+{
+	int rc;
+	u32 val32;
+
+	/*
+	 * First, we need to write the AFU idx for the AFU we want to
+	 * access.
+	 */
+	WARN_ON((afu_idx & OCXL_DVSEC_AFU_IDX_MASK) != afu_idx);
+	afu->idx = afu_idx;
+	pci_write_config_byte(dev,
+			fn->dvsec_afu_info_pos + OCXL_DVSEC_AFU_INFO_AFU_IDX,
+			afu->idx);
+
+	rc = read_afu_name(dev, fn, afu);
+	if (rc)
+		return rc;
+
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_AFU_VERSION, &val32);
+	if (rc)
+		return rc;
+	afu->version_major = EXTRACT_BITS(val32, 24, 31);
+	afu->version_minor = EXTRACT_BITS(val32, 16, 23);
+	afu->afuc_type = EXTRACT_BITS(val32, 14, 15);
+	afu->afum_type = EXTRACT_BITS(val32, 12, 13);
+	afu->profile = EXTRACT_BITS(val32, 0, 7);
+
+	rc = read_afu_mmio(dev, fn, afu);
+	if (rc)
+		return rc;
+
+	rc = read_afu_info(dev, fn, OCXL_DVSEC_TEMPL_MEM_SZ, &val32);
+	if (rc)
+		return rc;
+	afu->log_mem_size = EXTRACT_BITS(val32, 0, 7);
+
+	rc = read_afu_control(dev, afu);
+	if (rc)
+		return rc;
+
+	dev_dbg(&dev->dev, "AFU configuration:\n");
+	dev_dbg(&dev->dev, "  name = %s\n", afu->name);
+	dev_dbg(&dev->dev, "  version = %d.%d\n", afu->version_major,
+		afu->version_minor);
+	dev_dbg(&dev->dev, "  global mmio bar = %hhu\n", afu->global_mmio_bar);
+	dev_dbg(&dev->dev, "  global mmio offset = %#llx\n",
+		afu->global_mmio_offset);
+	dev_dbg(&dev->dev, "  global mmio size = %#x\n", afu->global_mmio_size);
+	dev_dbg(&dev->dev, "  pp mmio bar = %hhu\n", afu->pp_mmio_bar);
+	dev_dbg(&dev->dev, "  pp mmio offset = %#llx\n", afu->pp_mmio_offset);
+	dev_dbg(&dev->dev, "  pp mmio stride = %#x\n", afu->pp_mmio_stride);
+	dev_dbg(&dev->dev, "  mem size (log) = %hhu\n", afu->log_mem_size);
+	dev_dbg(&dev->dev, "  pasid supported (log) = %u\n",
+		afu->pasid_supported_log);
+	dev_dbg(&dev->dev, "  actag supported = %u\n",
+		afu->actag_supported);
+
+	rc = validate_afu(dev, afu);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(ocxl_config_read_afu);
+
+int ocxl_config_get_actag_info(struct pci_dev *dev, u16 *base, u16 *enabled,
+			u16 *supported)
+{
+	int rc;
+
+	/*
+	 * This is really a simple wrapper for the kernel API, to
+	 * avoid an external driver using ocxl as a library to call
+	 * platform-dependent code
+	 */
+	rc = pnv_ocxl_get_actag(dev, base, enabled, supported);
+	if (rc) {
+		dev_err(&dev->dev, "Can't get actag for device: %d\n", rc);
+		return rc;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ocxl_config_get_actag_info);
+
+void ocxl_config_set_afu_actag(struct pci_dev *dev, int pos, int actag_base,
+			int actag_count)
+{
+	u16 val;
+
+	val = actag_count & OCXL_DVSEC_ACTAG_MASK;
+	pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_EN, val);
+
+	val = actag_base & OCXL_DVSEC_ACTAG_MASK;
+	pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_BASE, val);
+}
+EXPORT_SYMBOL_GPL(ocxl_config_set_afu_actag);
+
+int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count)
+{
+	return pnv_ocxl_get_pasid_count(dev, count);
+}
+EXPORT_SYMBOL_GPL(ocxl_config_get_pasid_info);
+
+void ocxl_config_set_afu_pasid(struct pci_dev *dev, int pos, int pasid_base,
+			u32 pasid_count_log)
+{
+	u8 val8;
+	u32 val32;
+
+	val8 = pasid_count_log & OCXL_DVSEC_PASID_LOG_MASK;
+	pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_EN, val8);
+
+	pci_read_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
+			&val32);
+	val32 &= ~OCXL_DVSEC_PASID_MASK;
+	val32 |= pasid_base & OCXL_DVSEC_PASID_MASK;
+	pci_write_config_dword(dev, pos + OCXL_DVSEC_AFU_CTRL_PASID_BASE,
+			val32);
+}
+EXPORT_SYMBOL_GPL(ocxl_config_set_afu_pasid);
+
+void ocxl_config_set_afu_state(struct pci_dev *dev, int pos, int enable)
+{
+	u8 val;
+
+	pci_read_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, &val);
+	if (enable)
+		val |= 1;
+	else
+		val &= 0xFE;
+	pci_write_config_byte(dev, pos + OCXL_DVSEC_AFU_CTRL_ENABLE, val);
+}
+EXPORT_SYMBOL_GPL(ocxl_config_set_afu_state);
+
+int ocxl_config_set_TL(struct pci_dev *dev, int tl_dvsec)
+{
+	u32 val;
+	__be32 *be32ptr;
+	u8 timers;
+	int i, rc;
+	long recv_cap;
+	char *recv_rate;
+
+	/*
+	 * Skip on function != 0, as the TL can only be defined on 0
+	 */
+	if (PCI_FUNC(dev->devfn) != 0)
+		return 0;
+
+	recv_rate = kzalloc(PNV_OCXL_TL_RATE_BUF_SIZE, GFP_KERNEL);
+	if (!recv_rate)
+		return -ENOMEM;
+	/*
+	 * The spec defines 64 templates for messages in the
+	 * Transaction Layer (TL).
+	 *
+	 * The host and device each support a subset, so we need to
+	 * configure the transmitters on each side to send only
+	 * templates the receiver understands, at a rate the receiver
+	 * can process.  Per the spec, template 0 must be supported by
+	 * everybody. That's the template which has been used by the
+	 * host and device so far.
+	 *
+	 * The sending rate limit must be set before the template is
+	 * enabled.
+	 */
+
+	/*
+	 * Device -> host
+	 */
+	rc = pnv_ocxl_get_tl_cap(dev, &recv_cap, recv_rate,
+				PNV_OCXL_TL_RATE_BUF_SIZE);
+	if (rc)
+		goto out;
+
+	for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
+		be32ptr = (__be32 *) &recv_rate[i];
+		pci_write_config_dword(dev,
+				tl_dvsec + OCXL_DVSEC_TL_SEND_RATE + i,
+				be32_to_cpu(*be32ptr));
+	}
+	val = recv_cap >> 32;
+	pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP, val);
+	val = recv_cap & GENMASK(31, 0);
+	pci_write_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_SEND_CAP + 4, val);
+
+	/*
+	 * Host -> device
+	 */
+	for (i = 0; i < PNV_OCXL_TL_RATE_BUF_SIZE; i += 4) {
+		pci_read_config_dword(dev,
+				tl_dvsec + OCXL_DVSEC_TL_RECV_RATE + i,
+				&val);
+		be32ptr = (__be32 *) &recv_rate[i];
+		*be32ptr = cpu_to_be32(val);
+	}
+	pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP, &val);
+	recv_cap = (long) val << 32;
+	pci_read_config_dword(dev, tl_dvsec + OCXL_DVSEC_TL_RECV_CAP + 4, &val);
+	recv_cap |= val;
+
+	rc = pnv_ocxl_set_tl_conf(dev, recv_cap, __pa(recv_rate),
+				PNV_OCXL_TL_RATE_BUF_SIZE);
+	if (rc)
+		goto out;
+
+	/*
+	 * Opencapi commands needing to be retried are classified per
+	 * the TL in 2 groups: short and long commands.
+	 *
+	 * The short back off timer it not used for now. It will be
+	 * for opencapi 4.0.
+	 *
+	 * The long back off timer is typically used when an AFU hits
+	 * a page fault but the NPU is already processing one. So the
+	 * AFU needs to wait before it can resubmit. Having a value
+	 * too low doesn't break anything, but can generate extra
+	 * traffic on the link.
+	 * We set it to 1.6 us for now. It's shorter than, but in the
+	 * same order of magnitude as the time spent to process a page
+	 * fault.
+	 */
+	timers = 0x2 << 4; /* long timer = 1.6 us */
+	pci_write_config_byte(dev, tl_dvsec + OCXL_DVSEC_TL_BACKOFF_TIMERS,
+			timers);
+
+	rc = 0;
+out:
+	kfree(recv_rate);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(ocxl_config_set_TL);
+
+int ocxl_config_terminate_pasid(struct pci_dev *dev, int afu_control, int pasid)
+{
+	u32 val;
+	unsigned long timeout;
+
+	pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
+			&val);
+	if (EXTRACT_BIT(val, 20)) {
+		dev_err(&dev->dev,
+			"Can't terminate PASID %#x, previous termination didn't complete\n",
+			pasid);
+		return -EBUSY;
+	}
+
+	val &= ~OCXL_DVSEC_PASID_MASK;
+	val |= pasid & OCXL_DVSEC_PASID_MASK;
+	val |= BIT(20);
+	pci_write_config_dword(dev,
+			afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
+			val);
+
+	timeout = jiffies + (HZ * OCXL_CFG_TIMEOUT);
+	pci_read_config_dword(dev, afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
+			&val);
+	while (EXTRACT_BIT(val, 20)) {
+		if (time_after_eq(jiffies, timeout)) {
+			dev_err(&dev->dev,
+				"Timeout while waiting for AFU to terminate PASID %#x\n",
+				pasid);
+			return -EBUSY;
+		}
+		cpu_relax();
+		pci_read_config_dword(dev,
+				afu_control + OCXL_DVSEC_AFU_CTRL_TERM_PASID,
+				&val);
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ocxl_config_terminate_pasid);
+
+void ocxl_config_set_actag(struct pci_dev *dev, int func_dvsec, u32 tag_first,
+			u32 tag_count)
+{
+	u32 val;
+
+	val = (tag_first & OCXL_DVSEC_ACTAG_MASK) << 16;
+	val |= tag_count & OCXL_DVSEC_ACTAG_MASK;
+	pci_write_config_dword(dev, func_dvsec + OCXL_DVSEC_FUNC_OFF_ACTAG,
+			val);
+}
+EXPORT_SYMBOL_GPL(ocxl_config_set_actag);
diff --git a/drivers/misc/ocxl/context.c b/drivers/misc/ocxl/context.c
new file mode 100644
index 000000000000..909e8807824a
--- /dev/null
+++ b/drivers/misc/ocxl/context.c
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/sched/mm.h>
+#include "trace.h"
+#include "ocxl_internal.h"
+
+struct ocxl_context *ocxl_context_alloc(void)
+{
+	return kzalloc(sizeof(struct ocxl_context), GFP_KERNEL);
+}
+
+int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu,
+		struct address_space *mapping)
+{
+	int pasid;
+
+	ctx->afu = afu;
+	mutex_lock(&afu->contexts_lock);
+	pasid = idr_alloc(&afu->contexts_idr, ctx, afu->pasid_base,
+			afu->pasid_base + afu->pasid_max, GFP_KERNEL);
+	if (pasid < 0) {
+		mutex_unlock(&afu->contexts_lock);
+		return pasid;
+	}
+	afu->pasid_count++;
+	mutex_unlock(&afu->contexts_lock);
+
+	ctx->pasid = pasid;
+	ctx->status = OPENED;
+	mutex_init(&ctx->status_mutex);
+	ctx->mapping = mapping;
+	mutex_init(&ctx->mapping_lock);
+	init_waitqueue_head(&ctx->events_wq);
+	mutex_init(&ctx->xsl_error_lock);
+	mutex_init(&ctx->irq_lock);
+	idr_init(&ctx->irq_idr);
+	/*
+	 * Keep a reference on the AFU to make sure it's valid for the
+	 * duration of the life of the context
+	 */
+	ocxl_afu_get(afu);
+	return 0;
+}
+
+/*
+ * Callback for when a translation fault triggers an error
+ * data:	a pointer to the context which triggered the fault
+ * addr:	the address that triggered the error
+ * dsisr:	the value of the PPC64 dsisr register
+ */
+static void xsl_fault_error(void *data, u64 addr, u64 dsisr)
+{
+	struct ocxl_context *ctx = (struct ocxl_context *) data;
+
+	mutex_lock(&ctx->xsl_error_lock);
+	ctx->xsl_error.addr = addr;
+	ctx->xsl_error.dsisr = dsisr;
+	ctx->xsl_error.count++;
+	mutex_unlock(&ctx->xsl_error_lock);
+
+	wake_up_all(&ctx->events_wq);
+}
+
+int ocxl_context_attach(struct ocxl_context *ctx, u64 amr)
+{
+	int rc;
+
+	mutex_lock(&ctx->status_mutex);
+	if (ctx->status != OPENED) {
+		rc = -EIO;
+		goto out;
+	}
+
+	rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid,
+			current->mm->context.id, 0, amr, current->mm,
+			xsl_fault_error, ctx);
+	if (rc)
+		goto out;
+
+	ctx->status = ATTACHED;
+out:
+	mutex_unlock(&ctx->status_mutex);
+	return rc;
+}
+
+static int map_afu_irq(struct vm_area_struct *vma, unsigned long address,
+		u64 offset, struct ocxl_context *ctx)
+{
+	u64 trigger_addr;
+
+	trigger_addr = ocxl_afu_irq_get_addr(ctx, offset);
+	if (!trigger_addr)
+		return VM_FAULT_SIGBUS;
+
+	vm_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT);
+	return VM_FAULT_NOPAGE;
+}
+
+static int map_pp_mmio(struct vm_area_struct *vma, unsigned long address,
+		u64 offset, struct ocxl_context *ctx)
+{
+	u64 pp_mmio_addr;
+	int pasid_off;
+
+	if (offset >= ctx->afu->config.pp_mmio_stride)
+		return VM_FAULT_SIGBUS;
+
+	mutex_lock(&ctx->status_mutex);
+	if (ctx->status != ATTACHED) {
+		mutex_unlock(&ctx->status_mutex);
+		pr_debug("%s: Context not attached, failing mmio mmap\n",
+			__func__);
+		return VM_FAULT_SIGBUS;
+	}
+
+	pasid_off = ctx->pasid - ctx->afu->pasid_base;
+	pp_mmio_addr = ctx->afu->pp_mmio_start +
+		pasid_off * ctx->afu->config.pp_mmio_stride +
+		offset;
+
+	vm_insert_pfn(vma, address, pp_mmio_addr >> PAGE_SHIFT);
+	mutex_unlock(&ctx->status_mutex);
+	return VM_FAULT_NOPAGE;
+}
+
+static int ocxl_mmap_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct ocxl_context *ctx = vma->vm_file->private_data;
+	u64 offset;
+	int rc;
+
+	offset = vmf->pgoff << PAGE_SHIFT;
+	pr_debug("%s: pasid %d address 0x%lx offset 0x%llx\n", __func__,
+		ctx->pasid, vmf->address, offset);
+
+	if (offset < ctx->afu->irq_base_offset)
+		rc = map_pp_mmio(vma, vmf->address, offset, ctx);
+	else
+		rc = map_afu_irq(vma, vmf->address, offset, ctx);
+	return rc;
+}
+
+static const struct vm_operations_struct ocxl_vmops = {
+	.fault = ocxl_mmap_fault,
+};
+
+static int check_mmap_afu_irq(struct ocxl_context *ctx,
+			struct vm_area_struct *vma)
+{
+	/* only one page */
+	if (vma_pages(vma) != 1)
+		return -EINVAL;
+
+	/* check offset validty */
+	if (!ocxl_afu_irq_get_addr(ctx, vma->vm_pgoff << PAGE_SHIFT))
+		return -EINVAL;
+
+	/*
+	 * trigger page should only be accessible in write mode.
+	 *
+	 * It's a bit theoretical, as a page mmaped with only
+	 * PROT_WRITE is currently readable, but it doesn't hurt.
+	 */
+	if ((vma->vm_flags & VM_READ) || (vma->vm_flags & VM_EXEC) ||
+		!(vma->vm_flags & VM_WRITE))
+		return -EINVAL;
+	vma->vm_flags &= ~(VM_MAYREAD | VM_MAYEXEC);
+	return 0;
+}
+
+static int check_mmap_mmio(struct ocxl_context *ctx,
+			struct vm_area_struct *vma)
+{
+	if ((vma_pages(vma) + vma->vm_pgoff) >
+		(ctx->afu->config.pp_mmio_stride >> PAGE_SHIFT))
+		return -EINVAL;
+	return 0;
+}
+
+int ocxl_context_mmap(struct ocxl_context *ctx, struct vm_area_struct *vma)
+{
+	int rc;
+
+	if ((vma->vm_pgoff << PAGE_SHIFT) < ctx->afu->irq_base_offset)
+		rc = check_mmap_mmio(ctx, vma);
+	else
+		rc = check_mmap_afu_irq(ctx, vma);
+	if (rc)
+		return rc;
+
+	vma->vm_flags |= VM_IO | VM_PFNMAP;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_ops = &ocxl_vmops;
+	return 0;
+}
+
+int ocxl_context_detach(struct ocxl_context *ctx)
+{
+	struct pci_dev *dev;
+	int afu_control_pos;
+	enum ocxl_context_status status;
+	int rc;
+
+	mutex_lock(&ctx->status_mutex);
+	status = ctx->status;
+	ctx->status = CLOSED;
+	mutex_unlock(&ctx->status_mutex);
+	if (status != ATTACHED)
+		return 0;
+
+	dev = to_pci_dev(ctx->afu->fn->dev.parent);
+	afu_control_pos = ctx->afu->config.dvsec_afu_control_pos;
+
+	mutex_lock(&ctx->afu->afu_control_lock);
+	rc = ocxl_config_terminate_pasid(dev, afu_control_pos, ctx->pasid);
+	mutex_unlock(&ctx->afu->afu_control_lock);
+	trace_ocxl_terminate_pasid(ctx->pasid, rc);
+	if (rc) {
+		/*
+		 * If we timeout waiting for the AFU to terminate the
+		 * pasid, then it's dangerous to clean up the Process
+		 * Element entry in the SPA, as it may be referenced
+		 * in the future by the AFU. In which case, we would
+		 * checkstop because of an invalid PE access (FIR
+		 * register 2, bit 42). So leave the PE
+		 * defined. Caller shouldn't free the context so that
+		 * PASID remains allocated.
+		 *
+		 * A link reset will be required to cleanup the AFU
+		 * and the SPA.
+		 */
+		if (rc == -EBUSY)
+			return rc;
+	}
+	rc = ocxl_link_remove_pe(ctx->afu->fn->link, ctx->pasid);
+	if (rc) {
+		dev_warn(&ctx->afu->dev,
+			"Couldn't remove PE entry cleanly: %d\n", rc);
+	}
+	return 0;
+}
+
+void ocxl_context_detach_all(struct ocxl_afu *afu)
+{
+	struct ocxl_context *ctx;
+	int tmp;
+
+	mutex_lock(&afu->contexts_lock);
+	idr_for_each_entry(&afu->contexts_idr, ctx, tmp) {
+		ocxl_context_detach(ctx);
+		/*
+		 * We are force detaching - remove any active mmio
+		 * mappings so userspace cannot interfere with the
+		 * card if it comes back.  Easiest way to exercise
+		 * this is to unbind and rebind the driver via sysfs
+		 * while it is in use.
+		 */
+		mutex_lock(&ctx->mapping_lock);
+		if (ctx->mapping)
+			unmap_mapping_range(ctx->mapping, 0, 0, 1);
+		mutex_unlock(&ctx->mapping_lock);
+	}
+	mutex_unlock(&afu->contexts_lock);
+}
+
+void ocxl_context_free(struct ocxl_context *ctx)
+{
+	mutex_lock(&ctx->afu->contexts_lock);
+	ctx->afu->pasid_count--;
+	idr_remove(&ctx->afu->contexts_idr, ctx->pasid);
+	mutex_unlock(&ctx->afu->contexts_lock);
+
+	ocxl_afu_irq_free_all(ctx);
+	idr_destroy(&ctx->irq_idr);
+	/* reference to the AFU taken in ocxl_context_init */
+	ocxl_afu_put(ctx->afu);
+	kfree(ctx);
+}
diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c
new file mode 100644
index 000000000000..038509e5d031
--- /dev/null
+++ b/drivers/misc/ocxl/file.c
@@ -0,0 +1,461 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/fs.h>
+#include <linux/poll.h>
+#include <linux/sched/signal.h>
+#include <linux/uaccess.h>
+#include <uapi/misc/ocxl.h>
+#include "ocxl_internal.h"
+
+
+#define OCXL_NUM_MINORS 256 /* Total to reserve */
+
+static dev_t ocxl_dev;
+static struct class *ocxl_class;
+static struct mutex minors_idr_lock;
+static struct idr minors_idr;
+
+static struct ocxl_afu *find_and_get_afu(dev_t devno)
+{
+	struct ocxl_afu *afu;
+	int afu_minor;
+
+	afu_minor = MINOR(devno);
+	/*
+	 * We don't declare an RCU critical section here, as our AFU
+	 * is protected by a reference counter on the device. By the time the
+	 * minor number of a device is removed from the idr, the ref count of
+	 * the device is already at 0, so no user API will access that AFU and
+	 * this function can't return it.
+	 */
+	afu = idr_find(&minors_idr, afu_minor);
+	if (afu)
+		ocxl_afu_get(afu);
+	return afu;
+}
+
+static int allocate_afu_minor(struct ocxl_afu *afu)
+{
+	int minor;
+
+	mutex_lock(&minors_idr_lock);
+	minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL);
+	mutex_unlock(&minors_idr_lock);
+	return minor;
+}
+
+static void free_afu_minor(struct ocxl_afu *afu)
+{
+	mutex_lock(&minors_idr_lock);
+	idr_remove(&minors_idr, MINOR(afu->dev.devt));
+	mutex_unlock(&minors_idr_lock);
+}
+
+static int afu_open(struct inode *inode, struct file *file)
+{
+	struct ocxl_afu *afu;
+	struct ocxl_context *ctx;
+	int rc;
+
+	pr_debug("%s for device %x\n", __func__, inode->i_rdev);
+
+	afu = find_and_get_afu(inode->i_rdev);
+	if (!afu)
+		return -ENODEV;
+
+	ctx = ocxl_context_alloc();
+	if (!ctx) {
+		rc = -ENOMEM;
+		goto put_afu;
+	}
+
+	rc = ocxl_context_init(ctx, afu, inode->i_mapping);
+	if (rc)
+		goto put_afu;
+	file->private_data = ctx;
+	ocxl_afu_put(afu);
+	return 0;
+
+put_afu:
+	ocxl_afu_put(afu);
+	return rc;
+}
+
+static long afu_ioctl_attach(struct ocxl_context *ctx,
+			struct ocxl_ioctl_attach __user *uarg)
+{
+	struct ocxl_ioctl_attach arg;
+	u64 amr = 0;
+	int rc;
+
+	pr_debug("%s for context %d\n", __func__, ctx->pasid);
+
+	if (copy_from_user(&arg, uarg, sizeof(arg)))
+		return -EFAULT;
+
+	/* Make sure reserved fields are not set for forward compatibility */
+	if (arg.reserved1 || arg.reserved2 || arg.reserved3)
+		return -EINVAL;
+
+	amr = arg.amr & mfspr(SPRN_UAMOR);
+	rc = ocxl_context_attach(ctx, amr);
+	return rc;
+}
+
+static long afu_ioctl_get_metadata(struct ocxl_context *ctx,
+		struct ocxl_ioctl_metadata __user *uarg)
+{
+	struct ocxl_ioctl_metadata arg;
+
+	memset(&arg, 0, sizeof(arg));
+
+	arg.version = 0;
+
+	arg.afu_version_major = ctx->afu->config.version_major;
+	arg.afu_version_minor = ctx->afu->config.version_minor;
+	arg.pasid = ctx->pasid;
+	arg.pp_mmio_size = ctx->afu->config.pp_mmio_stride;
+	arg.global_mmio_size = ctx->afu->config.global_mmio_size;
+
+	if (copy_to_user(uarg, &arg, sizeof(arg)))
+		return -EFAULT;
+
+	return 0;
+}
+
+#define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" :			\
+			x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" :	\
+			x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" :		\
+			x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" :	\
+			x == OCXL_IOCTL_GET_METADATA ? "GET_METADATA" :	\
+			"UNKNOWN")
+
+static long afu_ioctl(struct file *file, unsigned int cmd,
+		unsigned long args)
+{
+	struct ocxl_context *ctx = file->private_data;
+	struct ocxl_ioctl_irq_fd irq_fd;
+	u64 irq_offset;
+	long rc;
+
+	pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid,
+		CMD_STR(cmd));
+
+	if (ctx->status == CLOSED)
+		return -EIO;
+
+	switch (cmd) {
+	case OCXL_IOCTL_ATTACH:
+		rc = afu_ioctl_attach(ctx,
+				(struct ocxl_ioctl_attach __user *) args);
+		break;
+
+	case OCXL_IOCTL_IRQ_ALLOC:
+		rc = ocxl_afu_irq_alloc(ctx, &irq_offset);
+		if (!rc) {
+			rc = copy_to_user((u64 __user *) args, &irq_offset,
+					sizeof(irq_offset));
+			if (rc) {
+				ocxl_afu_irq_free(ctx, irq_offset);
+				return -EFAULT;
+			}
+		}
+		break;
+
+	case OCXL_IOCTL_IRQ_FREE:
+		rc = copy_from_user(&irq_offset, (u64 __user *) args,
+				sizeof(irq_offset));
+		if (rc)
+			return -EFAULT;
+		rc = ocxl_afu_irq_free(ctx, irq_offset);
+		break;
+
+	case OCXL_IOCTL_IRQ_SET_FD:
+		rc = copy_from_user(&irq_fd, (u64 __user *) args,
+				sizeof(irq_fd));
+		if (rc)
+			return -EFAULT;
+		if (irq_fd.reserved)
+			return -EINVAL;
+		rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset,
+					irq_fd.eventfd);
+		break;
+
+	case OCXL_IOCTL_GET_METADATA:
+		rc = afu_ioctl_get_metadata(ctx,
+				(struct ocxl_ioctl_metadata __user *) args);
+		break;
+
+	default:
+		rc = -EINVAL;
+	}
+	return rc;
+}
+
+static long afu_compat_ioctl(struct file *file, unsigned int cmd,
+			unsigned long args)
+{
+	return afu_ioctl(file, cmd, args);
+}
+
+static int afu_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct ocxl_context *ctx = file->private_data;
+
+	pr_debug("%s for context %d\n", __func__, ctx->pasid);
+	return ocxl_context_mmap(ctx, vma);
+}
+
+static bool has_xsl_error(struct ocxl_context *ctx)
+{
+	bool ret;
+
+	mutex_lock(&ctx->xsl_error_lock);
+	ret = !!ctx->xsl_error.addr;
+	mutex_unlock(&ctx->xsl_error_lock);
+
+	return ret;
+}
+
+/*
+ * Are there any events pending on the AFU
+ * ctx: The AFU context
+ * Returns: true if there are events pending
+ */
+static bool afu_events_pending(struct ocxl_context *ctx)
+{
+	if (has_xsl_error(ctx))
+		return true;
+	return false;
+}
+
+static unsigned int afu_poll(struct file *file, struct poll_table_struct *wait)
+{
+	struct ocxl_context *ctx = file->private_data;
+	unsigned int mask = 0;
+	bool closed;
+
+	pr_debug("%s for context %d\n", __func__, ctx->pasid);
+
+	poll_wait(file, &ctx->events_wq, wait);
+
+	mutex_lock(&ctx->status_mutex);
+	closed = (ctx->status == CLOSED);
+	mutex_unlock(&ctx->status_mutex);
+
+	if (afu_events_pending(ctx))
+		mask = EPOLLIN | EPOLLRDNORM;
+	else if (closed)
+		mask = EPOLLERR;
+
+	return mask;
+}
+
+/*
+ * Populate the supplied buffer with a single XSL error
+ * ctx:	The AFU context to report the error from
+ * header: the event header to populate
+ * buf: The buffer to write the body into (should be at least
+ *      AFU_EVENT_BODY_XSL_ERROR_SIZE)
+ * Return: the amount of buffer that was populated
+ */
+static ssize_t append_xsl_error(struct ocxl_context *ctx,
+				struct ocxl_kernel_event_header *header,
+				char __user *buf)
+{
+	struct ocxl_kernel_event_xsl_fault_error body;
+
+	memset(&body, 0, sizeof(body));
+
+	mutex_lock(&ctx->xsl_error_lock);
+	if (!ctx->xsl_error.addr) {
+		mutex_unlock(&ctx->xsl_error_lock);
+		return 0;
+	}
+
+	body.addr = ctx->xsl_error.addr;
+	body.dsisr = ctx->xsl_error.dsisr;
+	body.count = ctx->xsl_error.count;
+
+	ctx->xsl_error.addr = 0;
+	ctx->xsl_error.dsisr = 0;
+	ctx->xsl_error.count = 0;
+
+	mutex_unlock(&ctx->xsl_error_lock);
+
+	header->type = OCXL_AFU_EVENT_XSL_FAULT_ERROR;
+
+	if (copy_to_user(buf, &body, sizeof(body)))
+		return -EFAULT;
+
+	return sizeof(body);
+}
+
+#define AFU_EVENT_BODY_MAX_SIZE sizeof(struct ocxl_kernel_event_xsl_fault_error)
+
+/*
+ * Reports events on the AFU
+ * Format:
+ *	Header (struct ocxl_kernel_event_header)
+ *	Body (struct ocxl_kernel_event_*)
+ *	Header...
+ */
+static ssize_t afu_read(struct file *file, char __user *buf, size_t count,
+			loff_t *off)
+{
+	struct ocxl_context *ctx = file->private_data;
+	struct ocxl_kernel_event_header header;
+	ssize_t rc;
+	ssize_t used = 0;
+	DEFINE_WAIT(event_wait);
+
+	memset(&header, 0, sizeof(header));
+
+	/* Require offset to be 0 */
+	if (*off != 0)
+		return -EINVAL;
+
+	if (count < (sizeof(struct ocxl_kernel_event_header) +
+			AFU_EVENT_BODY_MAX_SIZE))
+		return -EINVAL;
+
+	for (;;) {
+		prepare_to_wait(&ctx->events_wq, &event_wait,
+				TASK_INTERRUPTIBLE);
+
+		if (afu_events_pending(ctx))
+			break;
+
+		if (ctx->status == CLOSED)
+			break;
+
+		if (file->f_flags & O_NONBLOCK) {
+			finish_wait(&ctx->events_wq, &event_wait);
+			return -EAGAIN;
+		}
+
+		if (signal_pending(current)) {
+			finish_wait(&ctx->events_wq, &event_wait);
+			return -ERESTARTSYS;
+		}
+
+		schedule();
+	}
+
+	finish_wait(&ctx->events_wq, &event_wait);
+
+	if (has_xsl_error(ctx)) {
+		used = append_xsl_error(ctx, &header, buf + sizeof(header));
+		if (used < 0)
+			return used;
+	}
+
+	if (!afu_events_pending(ctx))
+		header.flags |= OCXL_KERNEL_EVENT_FLAG_LAST;
+
+	if (copy_to_user(buf, &header, sizeof(header)))
+		return -EFAULT;
+
+	used += sizeof(header);
+
+	rc = used;
+	return rc;
+}
+
+static int afu_release(struct inode *inode, struct file *file)
+{
+	struct ocxl_context *ctx = file->private_data;
+	int rc;
+
+	pr_debug("%s for device %x\n", __func__, inode->i_rdev);
+	rc = ocxl_context_detach(ctx);
+	mutex_lock(&ctx->mapping_lock);
+	ctx->mapping = NULL;
+	mutex_unlock(&ctx->mapping_lock);
+	wake_up_all(&ctx->events_wq);
+	if (rc != -EBUSY)
+		ocxl_context_free(ctx);
+	return 0;
+}
+
+static const struct file_operations ocxl_afu_fops = {
+	.owner		= THIS_MODULE,
+	.open           = afu_open,
+	.unlocked_ioctl = afu_ioctl,
+	.compat_ioctl   = afu_compat_ioctl,
+	.mmap           = afu_mmap,
+	.poll           = afu_poll,
+	.read           = afu_read,
+	.release        = afu_release,
+};
+
+int ocxl_create_cdev(struct ocxl_afu *afu)
+{
+	int rc;
+
+	cdev_init(&afu->cdev, &ocxl_afu_fops);
+	rc = cdev_add(&afu->cdev, afu->dev.devt, 1);
+	if (rc) {
+		dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc);
+		return rc;
+	}
+	return 0;
+}
+
+void ocxl_destroy_cdev(struct ocxl_afu *afu)
+{
+	cdev_del(&afu->cdev);
+}
+
+int ocxl_register_afu(struct ocxl_afu *afu)
+{
+	int minor;
+
+	minor = allocate_afu_minor(afu);
+	if (minor < 0)
+		return minor;
+	afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor);
+	afu->dev.class = ocxl_class;
+	return device_register(&afu->dev);
+}
+
+void ocxl_unregister_afu(struct ocxl_afu *afu)
+{
+	free_afu_minor(afu);
+}
+
+static char *ocxl_devnode(struct device *dev, umode_t *mode)
+{
+	return kasprintf(GFP_KERNEL, "ocxl/%s", dev_name(dev));
+}
+
+int ocxl_file_init(void)
+{
+	int rc;
+
+	mutex_init(&minors_idr_lock);
+	idr_init(&minors_idr);
+
+	rc = alloc_chrdev_region(&ocxl_dev, 0, OCXL_NUM_MINORS, "ocxl");
+	if (rc) {
+		pr_err("Unable to allocate ocxl major number: %d\n", rc);
+		return rc;
+	}
+
+	ocxl_class = class_create(THIS_MODULE, "ocxl");
+	if (IS_ERR(ocxl_class)) {
+		pr_err("Unable to create ocxl class\n");
+		unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS);
+		return PTR_ERR(ocxl_class);
+	}
+
+	ocxl_class->devnode = ocxl_devnode;
+	return 0;
+}
+
+void ocxl_file_exit(void)
+{
+	class_destroy(ocxl_class);
+	unregister_chrdev_region(ocxl_dev, OCXL_NUM_MINORS);
+	idr_destroy(&minors_idr);
+}
diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c
new file mode 100644
index 000000000000..f30790582dc0
--- /dev/null
+++ b/drivers/misc/ocxl/link.c
@@ -0,0 +1,647 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/sched/mm.h>
+#include <linux/mutex.h>
+#include <linux/mmu_context.h>
+#include <asm/copro.h>
+#include <asm/pnv-ocxl.h>
+#include <misc/ocxl.h>
+#include "ocxl_internal.h"
+#include "trace.h"
+
+
+#define SPA_PASID_BITS		15
+#define SPA_PASID_MAX		((1 << SPA_PASID_BITS) - 1)
+#define SPA_PE_MASK		SPA_PASID_MAX
+#define SPA_SPA_SIZE_LOG	22 /* Each SPA is 4 Mb */
+
+#define SPA_CFG_SF		(1ull << (63-0))
+#define SPA_CFG_TA		(1ull << (63-1))
+#define SPA_CFG_HV		(1ull << (63-3))
+#define SPA_CFG_UV		(1ull << (63-4))
+#define SPA_CFG_XLAT_hpt	(0ull << (63-6)) /* Hashed page table (HPT) mode */
+#define SPA_CFG_XLAT_roh	(2ull << (63-6)) /* Radix on HPT mode */
+#define SPA_CFG_XLAT_ror	(3ull << (63-6)) /* Radix on Radix mode */
+#define SPA_CFG_PR		(1ull << (63-49))
+#define SPA_CFG_TC		(1ull << (63-54))
+#define SPA_CFG_DR		(1ull << (63-59))
+
+#define SPA_XSL_TF		(1ull << (63-3))  /* Translation fault */
+#define SPA_XSL_S		(1ull << (63-38)) /* Store operation */
+
+#define SPA_PE_VALID		0x80000000
+
+
+struct pe_data {
+	struct mm_struct *mm;
+	/* callback to trigger when a translation fault occurs */
+	void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr);
+	/* opaque pointer to be passed to the above callback */
+	void *xsl_err_data;
+	struct rcu_head rcu;
+};
+
+struct spa {
+	struct ocxl_process_element *spa_mem;
+	int spa_order;
+	struct mutex spa_lock;
+	struct radix_tree_root pe_tree; /* Maps PE handles to pe_data */
+	char *irq_name;
+	int virq;
+	void __iomem *reg_dsisr;
+	void __iomem *reg_dar;
+	void __iomem *reg_tfc;
+	void __iomem *reg_pe_handle;
+	/*
+	 * The following field are used by the memory fault
+	 * interrupt handler. We can only have one interrupt at a
+	 * time. The NPU won't raise another interrupt until the
+	 * previous one has been ack'd by writing to the TFC register
+	 */
+	struct xsl_fault {
+		struct work_struct fault_work;
+		u64 pe;
+		u64 dsisr;
+		u64 dar;
+		struct pe_data pe_data;
+	} xsl_fault;
+};
+
+/*
+ * A opencapi link can be used be by several PCI functions. We have
+ * one link per device slot.
+ *
+ * A linked list of opencapi links should suffice, as there's a
+ * limited number of opencapi slots on a system and lookup is only
+ * done when the device is probed
+ */
+struct link {
+	struct list_head list;
+	struct kref ref;
+	int domain;
+	int bus;
+	int dev;
+	atomic_t irq_available;
+	struct spa *spa;
+	void *platform_data;
+};
+static struct list_head links_list = LIST_HEAD_INIT(links_list);
+static DEFINE_MUTEX(links_list_lock);
+
+enum xsl_response {
+	CONTINUE,
+	ADDRESS_ERROR,
+	RESTART,
+};
+
+
+static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe)
+{
+	u64 reg;
+
+	*dsisr = in_be64(spa->reg_dsisr);
+	*dar = in_be64(spa->reg_dar);
+	reg = in_be64(spa->reg_pe_handle);
+	*pe = reg & SPA_PE_MASK;
+}
+
+static void ack_irq(struct spa *spa, enum xsl_response r)
+{
+	u64 reg = 0;
+
+	/* continue is not supported */
+	if (r == RESTART)
+		reg = PPC_BIT(31);
+	else if (r == ADDRESS_ERROR)
+		reg = PPC_BIT(30);
+	else
+		WARN(1, "Invalid irq response %d\n", r);
+
+	if (reg) {
+		trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe,
+				spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg);
+		out_be64(spa->reg_tfc, reg);
+	}
+}
+
+static void xsl_fault_handler_bh(struct work_struct *fault_work)
+{
+	unsigned int flt = 0;
+	unsigned long access, flags, inv_flags = 0;
+	enum xsl_response r;
+	struct xsl_fault *fault = container_of(fault_work, struct xsl_fault,
+					fault_work);
+	struct spa *spa = container_of(fault, struct spa, xsl_fault);
+
+	int rc;
+
+	/*
+	 * We need to release a reference on the mm whenever exiting this
+	 * function (taken in the memory fault interrupt handler)
+	 */
+	rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr,
+				&flt);
+	if (rc) {
+		pr_debug("copro_handle_mm_fault failed: %d\n", rc);
+		if (fault->pe_data.xsl_err_cb) {
+			fault->pe_data.xsl_err_cb(
+				fault->pe_data.xsl_err_data,
+				fault->dar, fault->dsisr);
+		}
+		r = ADDRESS_ERROR;
+		goto ack;
+	}
+
+	if (!radix_enabled()) {
+		/*
+		 * update_mmu_cache() will not have loaded the hash
+		 * since current->trap is not a 0x400 or 0x300, so
+		 * just call hash_page_mm() here.
+		 */
+		access = _PAGE_PRESENT | _PAGE_READ;
+		if (fault->dsisr & SPA_XSL_S)
+			access |= _PAGE_WRITE;
+
+		if (REGION_ID(fault->dar) != USER_REGION_ID)
+			access |= _PAGE_PRIVILEGED;
+
+		local_irq_save(flags);
+		hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300,
+			inv_flags);
+		local_irq_restore(flags);
+	}
+	r = RESTART;
+ack:
+	mmdrop(fault->pe_data.mm);
+	ack_irq(spa, r);
+}
+
+static irqreturn_t xsl_fault_handler(int irq, void *data)
+{
+	struct link *link = (struct link *) data;
+	struct spa *spa = link->spa;
+	u64 dsisr, dar, pe_handle;
+	struct pe_data *pe_data;
+	struct ocxl_process_element *pe;
+	int lpid, pid, tid;
+
+	read_irq(spa, &dsisr, &dar, &pe_handle);
+	trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1);
+
+	WARN_ON(pe_handle > SPA_PE_MASK);
+	pe = spa->spa_mem + pe_handle;
+	lpid = be32_to_cpu(pe->lpid);
+	pid = be32_to_cpu(pe->pid);
+	tid = be32_to_cpu(pe->tid);
+	/* We could be reading all null values here if the PE is being
+	 * removed while an interrupt kicks in. It's not supposed to
+	 * happen if the driver notified the AFU to terminate the
+	 * PASID, and the AFU waited for pending operations before
+	 * acknowledging. But even if it happens, we won't find a
+	 * memory context below and fail silently, so it should be ok.
+	 */
+	if (!(dsisr & SPA_XSL_TF)) {
+		WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr);
+		ack_irq(spa, ADDRESS_ERROR);
+		return IRQ_HANDLED;
+	}
+
+	rcu_read_lock();
+	pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle);
+	if (!pe_data) {
+		/*
+		 * Could only happen if the driver didn't notify the
+		 * AFU about PASID termination before removing the PE,
+		 * or the AFU didn't wait for all memory access to
+		 * have completed.
+		 *
+		 * Either way, we fail early, but we shouldn't log an
+		 * error message, as it is a valid (if unexpected)
+		 * scenario
+		 */
+		rcu_read_unlock();
+		pr_debug("Unknown mm context for xsl interrupt\n");
+		ack_irq(spa, ADDRESS_ERROR);
+		return IRQ_HANDLED;
+	}
+	WARN_ON(pe_data->mm->context.id != pid);
+
+	spa->xsl_fault.pe = pe_handle;
+	spa->xsl_fault.dar = dar;
+	spa->xsl_fault.dsisr = dsisr;
+	spa->xsl_fault.pe_data = *pe_data;
+	mmgrab(pe_data->mm); /* mm count is released by bottom half */
+
+	rcu_read_unlock();
+	schedule_work(&spa->xsl_fault.fault_work);
+	return IRQ_HANDLED;
+}
+
+static void unmap_irq_registers(struct spa *spa)
+{
+	pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc,
+				spa->reg_pe_handle);
+}
+
+static int map_irq_registers(struct pci_dev *dev, struct spa *spa)
+{
+	return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar,
+				&spa->reg_tfc, &spa->reg_pe_handle);
+}
+
+static int setup_xsl_irq(struct pci_dev *dev, struct link *link)
+{
+	struct spa *spa = link->spa;
+	int rc;
+	int hwirq;
+
+	rc = pnv_ocxl_get_xsl_irq(dev, &hwirq);
+	if (rc)
+		return rc;
+
+	rc = map_irq_registers(dev, spa);
+	if (rc)
+		return rc;
+
+	spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x",
+				link->domain, link->bus, link->dev);
+	if (!spa->irq_name) {
+		unmap_irq_registers(spa);
+		dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n");
+		return -ENOMEM;
+	}
+	/*
+	 * At some point, we'll need to look into allowing a higher
+	 * number of interrupts. Could we have an IRQ domain per link?
+	 */
+	spa->virq = irq_create_mapping(NULL, hwirq);
+	if (!spa->virq) {
+		kfree(spa->irq_name);
+		unmap_irq_registers(spa);
+		dev_err(&dev->dev,
+			"irq_create_mapping failed for translation interrupt\n");
+		return -EINVAL;
+	}
+
+	dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq);
+
+	rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name,
+			link);
+	if (rc) {
+		irq_dispose_mapping(spa->virq);
+		kfree(spa->irq_name);
+		unmap_irq_registers(spa);
+		dev_err(&dev->dev,
+			"request_irq failed for translation interrupt: %d\n",
+			rc);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static void release_xsl_irq(struct link *link)
+{
+	struct spa *spa = link->spa;
+
+	if (spa->virq) {
+		free_irq(spa->virq, link);
+		irq_dispose_mapping(spa->virq);
+	}
+	kfree(spa->irq_name);
+	unmap_irq_registers(spa);
+}
+
+static int alloc_spa(struct pci_dev *dev, struct link *link)
+{
+	struct spa *spa;
+
+	spa = kzalloc(sizeof(struct spa), GFP_KERNEL);
+	if (!spa)
+		return -ENOMEM;
+
+	mutex_init(&spa->spa_lock);
+	INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL);
+	INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh);
+
+	spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT;
+	spa->spa_mem = (struct ocxl_process_element *)
+		__get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order);
+	if (!spa->spa_mem) {
+		dev_err(&dev->dev, "Can't allocate Shared Process Area\n");
+		kfree(spa);
+		return -ENOMEM;
+	}
+	pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus,
+		link->dev, spa->spa_mem);
+
+	link->spa = spa;
+	return 0;
+}
+
+static void free_spa(struct link *link)
+{
+	struct spa *spa = link->spa;
+
+	pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus,
+		link->dev);
+
+	if (spa && spa->spa_mem) {
+		free_pages((unsigned long) spa->spa_mem, spa->spa_order);
+		kfree(spa);
+		link->spa = NULL;
+	}
+}
+
+static int alloc_link(struct pci_dev *dev, int PE_mask, struct link **out_link)
+{
+	struct link *link;
+	int rc;
+
+	link = kzalloc(sizeof(struct link), GFP_KERNEL);
+	if (!link)
+		return -ENOMEM;
+
+	kref_init(&link->ref);
+	link->domain = pci_domain_nr(dev->bus);
+	link->bus = dev->bus->number;
+	link->dev = PCI_SLOT(dev->devfn);
+	atomic_set(&link->irq_available, MAX_IRQ_PER_LINK);
+
+	rc = alloc_spa(dev, link);
+	if (rc)
+		goto err_free;
+
+	rc = setup_xsl_irq(dev, link);
+	if (rc)
+		goto err_spa;
+
+	/* platform specific hook */
+	rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
+				&link->platform_data);
+	if (rc)
+		goto err_xsl_irq;
+
+	*out_link = link;
+	return 0;
+
+err_xsl_irq:
+	release_xsl_irq(link);
+err_spa:
+	free_spa(link);
+err_free:
+	kfree(link);
+	return rc;
+}
+
+static void free_link(struct link *link)
+{
+	release_xsl_irq(link);
+	free_spa(link);
+	kfree(link);
+}
+
+int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle)
+{
+	int rc = 0;
+	struct link *link;
+
+	mutex_lock(&links_list_lock);
+	list_for_each_entry(link, &links_list, list) {
+		/* The functions of a device all share the same link */
+		if (link->domain == pci_domain_nr(dev->bus) &&
+			link->bus == dev->bus->number &&
+			link->dev == PCI_SLOT(dev->devfn)) {
+			kref_get(&link->ref);
+			*link_handle = link;
+			goto unlock;
+		}
+	}
+	rc = alloc_link(dev, PE_mask, &link);
+	if (rc)
+		goto unlock;
+
+	list_add(&link->list, &links_list);
+	*link_handle = link;
+unlock:
+	mutex_unlock(&links_list_lock);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(ocxl_link_setup);
+
+static void release_xsl(struct kref *ref)
+{
+	struct link *link = container_of(ref, struct link, ref);
+
+	list_del(&link->list);
+	/* call platform code before releasing data */
+	pnv_ocxl_spa_release(link->platform_data);
+	free_link(link);
+}
+
+void ocxl_link_release(struct pci_dev *dev, void *link_handle)
+{
+	struct link *link = (struct link *) link_handle;
+
+	mutex_lock(&links_list_lock);
+	kref_put(&link->ref, release_xsl);
+	mutex_unlock(&links_list_lock);
+}
+EXPORT_SYMBOL_GPL(ocxl_link_release);
+
+static u64 calculate_cfg_state(bool kernel)
+{
+	u64 state;
+
+	state = SPA_CFG_DR;
+	if (mfspr(SPRN_LPCR) & LPCR_TC)
+		state |= SPA_CFG_TC;
+	if (radix_enabled())
+		state |= SPA_CFG_XLAT_ror;
+	else
+		state |= SPA_CFG_XLAT_hpt;
+	state |= SPA_CFG_HV;
+	if (kernel) {
+		if (mfmsr() & MSR_SF)
+			state |= SPA_CFG_SF;
+	} else {
+		state |= SPA_CFG_PR;
+		if (!test_tsk_thread_flag(current, TIF_32BIT))
+			state |= SPA_CFG_SF;
+	}
+	return state;
+}
+
+int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr,
+		u64 amr, struct mm_struct *mm,
+		void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr),
+		void *xsl_err_data)
+{
+	struct link *link = (struct link *) link_handle;
+	struct spa *spa = link->spa;
+	struct ocxl_process_element *pe;
+	int pe_handle, rc = 0;
+	struct pe_data *pe_data;
+
+	BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128);
+	if (pasid > SPA_PASID_MAX)
+		return -EINVAL;
+
+	mutex_lock(&spa->spa_lock);
+	pe_handle = pasid & SPA_PE_MASK;
+	pe = spa->spa_mem + pe_handle;
+
+	if (pe->software_state) {
+		rc = -EBUSY;
+		goto unlock;
+	}
+
+	pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL);
+	if (!pe_data) {
+		rc = -ENOMEM;
+		goto unlock;
+	}
+
+	pe_data->mm = mm;
+	pe_data->xsl_err_cb = xsl_err_cb;
+	pe_data->xsl_err_data = xsl_err_data;
+
+	memset(pe, 0, sizeof(struct ocxl_process_element));
+	pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0));
+	pe->lpid = cpu_to_be32(mfspr(SPRN_LPID));
+	pe->pid = cpu_to_be32(pidr);
+	pe->tid = cpu_to_be32(tidr);
+	pe->amr = cpu_to_be64(amr);
+	pe->software_state = cpu_to_be32(SPA_PE_VALID);
+
+	mm_context_add_copro(mm);
+	/*
+	 * Barrier is to make sure PE is visible in the SPA before it
+	 * is used by the device. It also helps with the global TLBI
+	 * invalidation
+	 */
+	mb();
+	radix_tree_insert(&spa->pe_tree, pe_handle, pe_data);
+
+	/*
+	 * The mm must stay valid for as long as the device uses it. We
+	 * lower the count when the context is removed from the SPA.
+	 *
+	 * We grab mm_count (and not mm_users), as we don't want to
+	 * end up in a circular dependency if a process mmaps its
+	 * mmio, therefore incrementing the file ref count when
+	 * calling mmap(), and forgets to unmap before exiting. In
+	 * that scenario, when the kernel handles the death of the
+	 * process, the file is not cleaned because unmap was not
+	 * called, and the mm wouldn't be freed because we would still
+	 * have a reference on mm_users. Incrementing mm_count solves
+	 * the problem.
+	 */
+	mmgrab(mm);
+	trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr);
+unlock:
+	mutex_unlock(&spa->spa_lock);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(ocxl_link_add_pe);
+
+int ocxl_link_remove_pe(void *link_handle, int pasid)
+{
+	struct link *link = (struct link *) link_handle;
+	struct spa *spa = link->spa;
+	struct ocxl_process_element *pe;
+	struct pe_data *pe_data;
+	int pe_handle, rc;
+
+	if (pasid > SPA_PASID_MAX)
+		return -EINVAL;
+
+	/*
+	 * About synchronization with our memory fault handler:
+	 *
+	 * Before removing the PE, the driver is supposed to have
+	 * notified the AFU, which should have cleaned up and make
+	 * sure the PASID is no longer in use, including pending
+	 * interrupts. However, there's no way to be sure...
+	 *
+	 * We clear the PE and remove the context from our radix
+	 * tree. From that point on, any new interrupt for that
+	 * context will fail silently, which is ok. As mentioned
+	 * above, that's not expected, but it could happen if the
+	 * driver or AFU didn't do the right thing.
+	 *
+	 * There could still be a bottom half running, but we don't
+	 * need to wait/flush, as it is managing a reference count on
+	 * the mm it reads from the radix tree.
+	 */
+	pe_handle = pasid & SPA_PE_MASK;
+	pe = spa->spa_mem + pe_handle;
+
+	mutex_lock(&spa->spa_lock);
+
+	if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) {
+		rc = -EINVAL;
+		goto unlock;
+	}
+
+	trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid,
+				be32_to_cpu(pe->pid), be32_to_cpu(pe->tid));
+
+	memset(pe, 0, sizeof(struct ocxl_process_element));
+	/*
+	 * The barrier makes sure the PE is removed from the SPA
+	 * before we clear the NPU context cache below, so that the
+	 * old PE cannot be reloaded erroneously.
+	 */
+	mb();
+
+	/*
+	 * hook to platform code
+	 * On powerpc, the entry needs to be cleared from the context
+	 * cache of the NPU.
+	 */
+	rc = pnv_ocxl_spa_remove_pe(link->platform_data, pe_handle);
+	WARN_ON(rc);
+
+	pe_data = radix_tree_delete(&spa->pe_tree, pe_handle);
+	if (!pe_data) {
+		WARN(1, "Couldn't find pe data when removing PE\n");
+	} else {
+		mm_context_remove_copro(pe_data->mm);
+		mmdrop(pe_data->mm);
+		kfree_rcu(pe_data, rcu);
+	}
+unlock:
+	mutex_unlock(&spa->spa_lock);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
+
+int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr)
+{
+	struct link *link = (struct link *) link_handle;
+	int rc, irq;
+	u64 addr;
+
+	if (atomic_dec_if_positive(&link->irq_available) < 0)
+		return -ENOSPC;
+
+	rc = pnv_ocxl_alloc_xive_irq(&irq, &addr);
+	if (rc) {
+		atomic_inc(&link->irq_available);
+		return rc;
+	}
+
+	*hw_irq = irq;
+	*trigger_addr = addr;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
+
+void ocxl_link_free_irq(void *link_handle, int hw_irq)
+{
+	struct link *link = (struct link *) link_handle;
+
+	pnv_ocxl_free_xive_irq(hw_irq);
+	atomic_inc(&link->irq_available);
+}
+EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
diff --git a/drivers/misc/ocxl/main.c b/drivers/misc/ocxl/main.c
new file mode 100644
index 000000000000..7210d9e059be
--- /dev/null
+++ b/drivers/misc/ocxl/main.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/module.h>
+#include <linux/pci.h>
+#include "ocxl_internal.h"
+
+static int __init init_ocxl(void)
+{
+	int rc = 0;
+
+	rc = ocxl_file_init();
+	if (rc)
+		return rc;
+
+	rc = pci_register_driver(&ocxl_pci_driver);
+	if (rc) {
+		ocxl_file_exit();
+		return rc;
+	}
+	return 0;
+}
+
+static void exit_ocxl(void)
+{
+	pci_unregister_driver(&ocxl_pci_driver);
+	ocxl_file_exit();
+}
+
+module_init(init_ocxl);
+module_exit(exit_ocxl);
+
+MODULE_DESCRIPTION("Open Coherent Accelerator");
+MODULE_LICENSE("GPL");
diff --git a/drivers/misc/ocxl/ocxl_internal.h b/drivers/misc/ocxl/ocxl_internal.h
new file mode 100644
index 000000000000..5d421824afd9
--- /dev/null
+++ b/drivers/misc/ocxl/ocxl_internal.h
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#ifndef _OCXL_INTERNAL_H_
+#define _OCXL_INTERNAL_H_
+
+#include <linux/pci.h>
+#include <linux/cdev.h>
+#include <linux/list.h>
+#include <misc/ocxl.h>
+
+#define MAX_IRQ_PER_LINK	2000
+#define MAX_IRQ_PER_CONTEXT	MAX_IRQ_PER_LINK
+
+#define to_ocxl_function(d) container_of(d, struct ocxl_fn, dev)
+#define to_ocxl_afu(d) container_of(d, struct ocxl_afu, dev)
+
+extern struct pci_driver ocxl_pci_driver;
+
+
+struct ocxl_fn {
+	struct device dev;
+	int bar_used[3];
+	struct ocxl_fn_config config;
+	struct list_head afu_list;
+	int pasid_base;
+	int actag_base;
+	int actag_enabled;
+	int actag_supported;
+	struct list_head pasid_list;
+	struct list_head actag_list;
+	void *link;
+};
+
+struct ocxl_afu {
+	struct ocxl_fn *fn;
+	struct list_head list;
+	struct device dev;
+	struct cdev cdev;
+	struct ocxl_afu_config config;
+	int pasid_base;
+	int pasid_count; /* opened contexts */
+	int pasid_max; /* maximum number of contexts */
+	int actag_base;
+	int actag_enabled;
+	struct mutex contexts_lock;
+	struct idr contexts_idr;
+	struct mutex afu_control_lock;
+	u64 global_mmio_start;
+	u64 irq_base_offset;
+	void __iomem *global_mmio_ptr;
+	u64 pp_mmio_start;
+	struct bin_attribute attr_global_mmio;
+};
+
+enum ocxl_context_status {
+	CLOSED,
+	OPENED,
+	ATTACHED,
+};
+
+// Contains metadata about a translation fault
+struct ocxl_xsl_error {
+	u64 addr; // The address that triggered the fault
+	u64 dsisr; // the value of the dsisr register
+	u64 count; // The number of times this fault has been triggered
+};
+
+struct ocxl_context {
+	struct ocxl_afu *afu;
+	int pasid;
+	struct mutex status_mutex;
+	enum ocxl_context_status status;
+	struct address_space *mapping;
+	struct mutex mapping_lock;
+	wait_queue_head_t events_wq;
+	struct mutex xsl_error_lock;
+	struct ocxl_xsl_error xsl_error;
+	struct mutex irq_lock;
+	struct idr irq_idr;
+};
+
+struct ocxl_process_element {
+	__be64 config_state;
+	__be32 reserved1[11];
+	__be32 lpid;
+	__be32 tid;
+	__be32 pid;
+	__be32 reserved2[10];
+	__be64 amr;
+	__be32 reserved3[3];
+	__be32 software_state;
+};
+
+
+extern struct ocxl_afu *ocxl_afu_get(struct ocxl_afu *afu);
+extern void ocxl_afu_put(struct ocxl_afu *afu);
+
+extern int ocxl_create_cdev(struct ocxl_afu *afu);
+extern void ocxl_destroy_cdev(struct ocxl_afu *afu);
+extern int ocxl_register_afu(struct ocxl_afu *afu);
+extern void ocxl_unregister_afu(struct ocxl_afu *afu);
+
+extern int ocxl_file_init(void);
+extern void ocxl_file_exit(void);
+
+extern int ocxl_pasid_afu_alloc(struct ocxl_fn *fn, u32 size);
+extern void ocxl_pasid_afu_free(struct ocxl_fn *fn, u32 start, u32 size);
+extern int ocxl_actag_afu_alloc(struct ocxl_fn *fn, u32 size);
+extern void ocxl_actag_afu_free(struct ocxl_fn *fn, u32 start, u32 size);
+
+extern struct ocxl_context *ocxl_context_alloc(void);
+extern int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu,
+			struct address_space *mapping);
+extern int ocxl_context_attach(struct ocxl_context *ctx, u64 amr);
+extern int ocxl_context_mmap(struct ocxl_context *ctx,
+			struct vm_area_struct *vma);
+extern int ocxl_context_detach(struct ocxl_context *ctx);
+extern void ocxl_context_detach_all(struct ocxl_afu *afu);
+extern void ocxl_context_free(struct ocxl_context *ctx);
+
+extern int ocxl_sysfs_add_afu(struct ocxl_afu *afu);
+extern void ocxl_sysfs_remove_afu(struct ocxl_afu *afu);
+
+extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset);
+extern int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset);
+extern void ocxl_afu_irq_free_all(struct ocxl_context *ctx);
+extern int ocxl_afu_irq_set_fd(struct ocxl_context *ctx, u64 irq_offset,
+			int eventfd);
+extern u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, u64 irq_offset);
+
+#endif /* _OCXL_INTERNAL_H_ */
diff --git a/drivers/misc/ocxl/pasid.c b/drivers/misc/ocxl/pasid.c
new file mode 100644
index 000000000000..d14cb56e6920
--- /dev/null
+++ b/drivers/misc/ocxl/pasid.c
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include "ocxl_internal.h"
+
+
+struct id_range {
+	struct list_head list;
+	u32 start;
+	u32 end;
+};
+
+#ifdef DEBUG
+static void dump_list(struct list_head *head, char *type_str)
+{
+	struct id_range *cur;
+
+	pr_debug("%s ranges allocated:\n", type_str);
+	list_for_each_entry(cur, head, list) {
+		pr_debug("Range %d->%d\n", cur->start, cur->end);
+	}
+}
+#endif
+
+static int range_alloc(struct list_head *head, u32 size, int max_id,
+		char *type_str)
+{
+	struct list_head *pos;
+	struct id_range *cur, *new;
+	int rc, last_end;
+
+	new = kmalloc(sizeof(struct id_range), GFP_KERNEL);
+	if (!new)
+		return -ENOMEM;
+
+	pos = head;
+	last_end = -1;
+	list_for_each_entry(cur, head, list) {
+		if ((cur->start - last_end) > size)
+			break;
+		last_end = cur->end;
+		pos = &cur->list;
+	}
+
+	new->start = last_end + 1;
+	new->end = new->start + size - 1;
+
+	if (new->end > max_id) {
+		kfree(new);
+		rc = -ENOSPC;
+	} else {
+		list_add(&new->list, pos);
+		rc = new->start;
+	}
+
+#ifdef DEBUG
+	dump_list(head, type_str);
+#endif
+	return rc;
+}
+
+static void range_free(struct list_head *head, u32 start, u32 size,
+		char *type_str)
+{
+	bool found = false;
+	struct id_range *cur, *tmp;
+
+	list_for_each_entry_safe(cur, tmp, head, list) {
+		if (cur->start == start && cur->end == (start + size - 1)) {
+			found = true;
+			list_del(&cur->list);
+			kfree(cur);
+			break;
+		}
+	}
+	WARN_ON(!found);
+#ifdef DEBUG
+	dump_list(head, type_str);
+#endif
+}
+
+int ocxl_pasid_afu_alloc(struct ocxl_fn *fn, u32 size)
+{
+	int max_pasid;
+
+	if (fn->config.max_pasid_log < 0)
+		return -ENOSPC;
+	max_pasid = 1 << fn->config.max_pasid_log;
+	return range_alloc(&fn->pasid_list, size, max_pasid, "afu pasid");
+}
+
+void ocxl_pasid_afu_free(struct ocxl_fn *fn, u32 start, u32 size)
+{
+	return range_free(&fn->pasid_list, start, size, "afu pasid");
+}
+
+int ocxl_actag_afu_alloc(struct ocxl_fn *fn, u32 size)
+{
+	int max_actag;
+
+	max_actag = fn->actag_enabled;
+	return range_alloc(&fn->actag_list, size, max_actag, "afu actag");
+}
+
+void ocxl_actag_afu_free(struct ocxl_fn *fn, u32 start, u32 size)
+{
+	return range_free(&fn->actag_list, start, size, "afu actag");
+}
diff --git a/drivers/misc/ocxl/pci.c b/drivers/misc/ocxl/pci.c
new file mode 100644
index 000000000000..0051d9ec76cc
--- /dev/null
+++ b/drivers/misc/ocxl/pci.c
@@ -0,0 +1,585 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/idr.h>
+#include <asm/pnv-ocxl.h>
+#include "ocxl_internal.h"
+
+/*
+ * Any opencapi device which wants to use this 'generic' driver should
+ * use the 0x062B device ID. Vendors should define the subsystem
+ * vendor/device ID to help differentiate devices.
+ */
+static const struct pci_device_id ocxl_pci_tbl[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x062B), },
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, ocxl_pci_tbl);
+
+
+static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn)
+{
+	return (get_device(&fn->dev) == NULL) ? NULL : fn;
+}
+
+static void ocxl_fn_put(struct ocxl_fn *fn)
+{
+	put_device(&fn->dev);
+}
+
+struct ocxl_afu *ocxl_afu_get(struct ocxl_afu *afu)
+{
+	return (get_device(&afu->dev) == NULL) ? NULL : afu;
+}
+
+void ocxl_afu_put(struct ocxl_afu *afu)
+{
+	put_device(&afu->dev);
+}
+
+static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn)
+{
+	struct ocxl_afu *afu;
+
+	afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL);
+	if (!afu)
+		return NULL;
+
+	mutex_init(&afu->contexts_lock);
+	mutex_init(&afu->afu_control_lock);
+	idr_init(&afu->contexts_idr);
+	afu->fn = fn;
+	ocxl_fn_get(fn);
+	return afu;
+}
+
+static void free_afu(struct ocxl_afu *afu)
+{
+	idr_destroy(&afu->contexts_idr);
+	ocxl_fn_put(afu->fn);
+	kfree(afu);
+}
+
+static void free_afu_dev(struct device *dev)
+{
+	struct ocxl_afu *afu = to_ocxl_afu(dev);
+
+	ocxl_unregister_afu(afu);
+	free_afu(afu);
+}
+
+static int set_afu_device(struct ocxl_afu *afu, const char *location)
+{
+	struct ocxl_fn *fn = afu->fn;
+	int rc;
+
+	afu->dev.parent = &fn->dev;
+	afu->dev.release = free_afu_dev;
+	rc = dev_set_name(&afu->dev, "%s.%s.%hhu", afu->config.name, location,
+		afu->config.idx);
+	return rc;
+}
+
+static int assign_afu_actag(struct ocxl_afu *afu, struct pci_dev *dev)
+{
+	struct ocxl_fn *fn = afu->fn;
+	int actag_count, actag_offset;
+
+	/*
+	 * if there were not enough actags for the function, each afu
+	 * reduces its count as well
+	 */
+	actag_count = afu->config.actag_supported *
+		fn->actag_enabled / fn->actag_supported;
+	actag_offset = ocxl_actag_afu_alloc(fn, actag_count);
+	if (actag_offset < 0) {
+		dev_err(&afu->dev, "Can't allocate %d actags for AFU: %d\n",
+			actag_count, actag_offset);
+		return actag_offset;
+	}
+	afu->actag_base = fn->actag_base + actag_offset;
+	afu->actag_enabled = actag_count;
+
+	ocxl_config_set_afu_actag(dev, afu->config.dvsec_afu_control_pos,
+				afu->actag_base, afu->actag_enabled);
+	dev_dbg(&afu->dev, "actag base=%d enabled=%d\n",
+		afu->actag_base, afu->actag_enabled);
+	return 0;
+}
+
+static void reclaim_afu_actag(struct ocxl_afu *afu)
+{
+	struct ocxl_fn *fn = afu->fn;
+	int start_offset, size;
+
+	start_offset = afu->actag_base - fn->actag_base;
+	size = afu->actag_enabled;
+	ocxl_actag_afu_free(afu->fn, start_offset, size);
+}
+
+static int assign_afu_pasid(struct ocxl_afu *afu, struct pci_dev *dev)
+{
+	struct ocxl_fn *fn = afu->fn;
+	int pasid_count, pasid_offset;
+
+	/*
+	 * We only support the case where the function configuration
+	 * requested enough PASIDs to cover all AFUs.
+	 */
+	pasid_count = 1 << afu->config.pasid_supported_log;
+	pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count);
+	if (pasid_offset < 0) {
+		dev_err(&afu->dev, "Can't allocate %d PASIDs for AFU: %d\n",
+			pasid_count, pasid_offset);
+		return pasid_offset;
+	}
+	afu->pasid_base = fn->pasid_base + pasid_offset;
+	afu->pasid_count = 0;
+	afu->pasid_max = pasid_count;
+
+	ocxl_config_set_afu_pasid(dev, afu->config.dvsec_afu_control_pos,
+				afu->pasid_base,
+				afu->config.pasid_supported_log);
+	dev_dbg(&afu->dev, "PASID base=%d, enabled=%d\n",
+		afu->pasid_base, pasid_count);
+	return 0;
+}
+
+static void reclaim_afu_pasid(struct ocxl_afu *afu)
+{
+	struct ocxl_fn *fn = afu->fn;
+	int start_offset, size;
+
+	start_offset = afu->pasid_base - fn->pasid_base;
+	size = 1 << afu->config.pasid_supported_log;
+	ocxl_pasid_afu_free(afu->fn, start_offset, size);
+}
+
+static int reserve_fn_bar(struct ocxl_fn *fn, int bar)
+{
+	struct pci_dev *dev = to_pci_dev(fn->dev.parent);
+	int rc, idx;
+
+	if (bar != 0 && bar != 2 && bar != 4)
+		return -EINVAL;
+
+	idx = bar >> 1;
+	if (fn->bar_used[idx]++ == 0) {
+		rc = pci_request_region(dev, bar, "ocxl");
+		if (rc)
+			return rc;
+	}
+	return 0;
+}
+
+static void release_fn_bar(struct ocxl_fn *fn, int bar)
+{
+	struct pci_dev *dev = to_pci_dev(fn->dev.parent);
+	int idx;
+
+	if (bar != 0 && bar != 2 && bar != 4)
+		return;
+
+	idx = bar >> 1;
+	if (--fn->bar_used[idx] == 0)
+		pci_release_region(dev, bar);
+	WARN_ON(fn->bar_used[idx] < 0);
+}
+
+static int map_mmio_areas(struct ocxl_afu *afu, struct pci_dev *dev)
+{
+	int rc;
+
+	rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar);
+	if (rc)
+		return rc;
+
+	rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar);
+	if (rc) {
+		release_fn_bar(afu->fn, afu->config.global_mmio_bar);
+		return rc;
+	}
+
+	afu->global_mmio_start =
+		pci_resource_start(dev, afu->config.global_mmio_bar) +
+		afu->config.global_mmio_offset;
+	afu->pp_mmio_start =
+		pci_resource_start(dev, afu->config.pp_mmio_bar) +
+		afu->config.pp_mmio_offset;
+
+	afu->global_mmio_ptr = ioremap(afu->global_mmio_start,
+				afu->config.global_mmio_size);
+	if (!afu->global_mmio_ptr) {
+		release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
+		release_fn_bar(afu->fn, afu->config.global_mmio_bar);
+		dev_err(&dev->dev, "Error mapping global mmio area\n");
+		return -ENOMEM;
+	}
+
+	/*
+	 * Leave an empty page between the per-process mmio area and
+	 * the AFU interrupt mappings
+	 */
+	afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE;
+	return 0;
+}
+
+static void unmap_mmio_areas(struct ocxl_afu *afu)
+{
+	if (afu->global_mmio_ptr) {
+		iounmap(afu->global_mmio_ptr);
+		afu->global_mmio_ptr = NULL;
+	}
+	afu->global_mmio_start = 0;
+	afu->pp_mmio_start = 0;
+	release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
+	release_fn_bar(afu->fn, afu->config.global_mmio_bar);
+}
+
+static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
+{
+	int rc;
+
+	rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx);
+	if (rc)
+		return rc;
+
+	rc = set_afu_device(afu, dev_name(&dev->dev));
+	if (rc)
+		return rc;
+
+	rc = assign_afu_actag(afu, dev);
+	if (rc)
+		return rc;
+
+	rc = assign_afu_pasid(afu, dev);
+	if (rc) {
+		reclaim_afu_actag(afu);
+		return rc;
+	}
+
+	rc = map_mmio_areas(afu, dev);
+	if (rc) {
+		reclaim_afu_pasid(afu);
+		reclaim_afu_actag(afu);
+		return rc;
+	}
+	return 0;
+}
+
+static void deconfigure_afu(struct ocxl_afu *afu)
+{
+	unmap_mmio_areas(afu);
+	reclaim_afu_pasid(afu);
+	reclaim_afu_actag(afu);
+}
+
+static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu)
+{
+	int rc;
+
+	ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1);
+	/*
+	 * Char device creation is the last step, as processes can
+	 * call our driver immediately, so all our inits must be finished.
+	 */
+	rc = ocxl_create_cdev(afu);
+	if (rc)
+		return rc;
+	return 0;
+}
+
+static void deactivate_afu(struct ocxl_afu *afu)
+{
+	struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
+
+	ocxl_destroy_cdev(afu);
+	ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0);
+}
+
+static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx)
+{
+	int rc;
+	struct ocxl_afu *afu;
+
+	afu = alloc_afu(fn);
+	if (!afu)
+		return -ENOMEM;
+
+	rc = configure_afu(afu, afu_idx, dev);
+	if (rc) {
+		free_afu(afu);
+		return rc;
+	}
+
+	rc = ocxl_register_afu(afu);
+	if (rc)
+		goto err;
+
+	rc = ocxl_sysfs_add_afu(afu);
+	if (rc)
+		goto err;
+
+	rc = activate_afu(dev, afu);
+	if (rc)
+		goto err_sys;
+
+	list_add_tail(&afu->list, &fn->afu_list);
+	return 0;
+
+err_sys:
+	ocxl_sysfs_remove_afu(afu);
+err:
+	deconfigure_afu(afu);
+	device_unregister(&afu->dev);
+	return rc;
+}
+
+static void remove_afu(struct ocxl_afu *afu)
+{
+	list_del(&afu->list);
+	ocxl_context_detach_all(afu);
+	deactivate_afu(afu);
+	ocxl_sysfs_remove_afu(afu);
+	deconfigure_afu(afu);
+	device_unregister(&afu->dev);
+}
+
+static struct ocxl_fn *alloc_function(struct pci_dev *dev)
+{
+	struct ocxl_fn *fn;
+
+	fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL);
+	if (!fn)
+		return NULL;
+
+	INIT_LIST_HEAD(&fn->afu_list);
+	INIT_LIST_HEAD(&fn->pasid_list);
+	INIT_LIST_HEAD(&fn->actag_list);
+	return fn;
+}
+
+static void free_function(struct ocxl_fn *fn)
+{
+	WARN_ON(!list_empty(&fn->afu_list));
+	WARN_ON(!list_empty(&fn->pasid_list));
+	kfree(fn);
+}
+
+static void free_function_dev(struct device *dev)
+{
+	struct ocxl_fn *fn = to_ocxl_function(dev);
+
+	free_function(fn);
+}
+
+static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev)
+{
+	int rc;
+
+	fn->dev.parent = &dev->dev;
+	fn->dev.release = free_function_dev;
+	rc = dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev));
+	if (rc)
+		return rc;
+	pci_set_drvdata(dev, fn);
+	return 0;
+}
+
+static int assign_function_actag(struct ocxl_fn *fn)
+{
+	struct pci_dev *dev = to_pci_dev(fn->dev.parent);
+	u16 base, enabled, supported;
+	int rc;
+
+	rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported);
+	if (rc)
+		return rc;
+
+	fn->actag_base = base;
+	fn->actag_enabled = enabled;
+	fn->actag_supported = supported;
+
+	ocxl_config_set_actag(dev, fn->config.dvsec_function_pos,
+			fn->actag_base,	fn->actag_enabled);
+	dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n",
+		fn->actag_base, fn->actag_enabled);
+	return 0;
+}
+
+static int set_function_pasid(struct ocxl_fn *fn)
+{
+	struct pci_dev *dev = to_pci_dev(fn->dev.parent);
+	int rc, desired_count, max_count;
+
+	/* A function may not require any PASID */
+	if (fn->config.max_pasid_log < 0)
+		return 0;
+
+	rc = ocxl_config_get_pasid_info(dev, &max_count);
+	if (rc)
+		return rc;
+
+	desired_count = 1 << fn->config.max_pasid_log;
+
+	if (desired_count > max_count) {
+		dev_err(&fn->dev,
+			"Function requires more PASIDs than is available (%d vs. %d)\n",
+			desired_count, max_count);
+		return -ENOSPC;
+	}
+
+	fn->pasid_base = 0;
+	return 0;
+}
+
+static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev)
+{
+	int rc;
+
+	rc = pci_enable_device(dev);
+	if (rc) {
+		dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc);
+		return rc;
+	}
+
+	/*
+	 * Once it has been confirmed to work on our hardware, we
+	 * should reset the function, to force the adapter to restart
+	 * from scratch.
+	 * A function reset would also reset all its AFUs.
+	 *
+	 * Some hints for implementation:
+	 *
+	 * - there's not status bit to know when the reset is done. We
+	 *   should try reading the config space to know when it's
+	 *   done.
+	 * - probably something like:
+	 *	Reset
+	 *	wait 100ms
+	 *	issue config read
+	 *	allow device up to 1 sec to return success on config
+	 *	read before declaring it broken
+	 *
+	 * Some shared logic on the card (CFG, TLX) won't be reset, so
+	 * there's no guarantee that it will be enough.
+	 */
+	rc = ocxl_config_read_function(dev, &fn->config);
+	if (rc)
+		return rc;
+
+	rc = set_function_device(fn, dev);
+	if (rc)
+		return rc;
+
+	rc = assign_function_actag(fn);
+	if (rc)
+		return rc;
+
+	rc = set_function_pasid(fn);
+	if (rc)
+		return rc;
+
+	rc = ocxl_link_setup(dev, 0, &fn->link);
+	if (rc)
+		return rc;
+
+	rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos);
+	if (rc) {
+		ocxl_link_release(dev, fn->link);
+		return rc;
+	}
+	return 0;
+}
+
+static void deconfigure_function(struct ocxl_fn *fn)
+{
+	struct pci_dev *dev = to_pci_dev(fn->dev.parent);
+
+	ocxl_link_release(dev, fn->link);
+	pci_disable_device(dev);
+}
+
+static struct ocxl_fn *init_function(struct pci_dev *dev)
+{
+	struct ocxl_fn *fn;
+	int rc;
+
+	fn = alloc_function(dev);
+	if (!fn)
+		return ERR_PTR(-ENOMEM);
+
+	rc = configure_function(fn, dev);
+	if (rc) {
+		free_function(fn);
+		return ERR_PTR(rc);
+	}
+
+	rc = device_register(&fn->dev);
+	if (rc) {
+		deconfigure_function(fn);
+		device_unregister(&fn->dev);
+		return ERR_PTR(rc);
+	}
+	return fn;
+}
+
+static void remove_function(struct ocxl_fn *fn)
+{
+	deconfigure_function(fn);
+	device_unregister(&fn->dev);
+}
+
+static int ocxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+	int rc, afu_count = 0;
+	u8 afu;
+	struct ocxl_fn *fn;
+
+	if (!radix_enabled()) {
+		dev_err(&dev->dev, "Unsupported memory model (hash)\n");
+		return -ENODEV;
+	}
+
+	fn = init_function(dev);
+	if (IS_ERR(fn)) {
+		dev_err(&dev->dev, "function init failed: %li\n",
+			PTR_ERR(fn));
+		return PTR_ERR(fn);
+	}
+
+	for (afu = 0; afu <= fn->config.max_afu_index; afu++) {
+		rc = ocxl_config_check_afu_index(dev, &fn->config, afu);
+		if (rc > 0) {
+			rc = init_afu(dev, fn, afu);
+			if (rc) {
+				dev_err(&dev->dev,
+					"Can't initialize AFU index %d\n", afu);
+				continue;
+			}
+			afu_count++;
+		}
+	}
+	dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count);
+	return 0;
+}
+
+static void ocxl_remove(struct pci_dev *dev)
+{
+	struct ocxl_afu *afu, *tmp;
+	struct ocxl_fn *fn = pci_get_drvdata(dev);
+
+	list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) {
+		remove_afu(afu);
+	}
+	remove_function(fn);
+}
+
+struct pci_driver ocxl_pci_driver = {
+	.name = "ocxl",
+	.id_table = ocxl_pci_tbl,
+	.probe = ocxl_probe,
+	.remove = ocxl_remove,
+	.shutdown = ocxl_remove,
+};
diff --git a/drivers/misc/ocxl/sysfs.c b/drivers/misc/ocxl/sysfs.c
new file mode 100644
index 000000000000..d9753a1db14b
--- /dev/null
+++ b/drivers/misc/ocxl/sysfs.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#include <linux/sysfs.h>
+#include "ocxl_internal.h"
+
+static ssize_t global_mmio_size_show(struct device *device,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct ocxl_afu *afu = to_ocxl_afu(device);
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
+			afu->config.global_mmio_size);
+}
+
+static ssize_t pp_mmio_size_show(struct device *device,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct ocxl_afu *afu = to_ocxl_afu(device);
+
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
+			afu->config.pp_mmio_stride);
+}
+
+static ssize_t afu_version_show(struct device *device,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct ocxl_afu *afu = to_ocxl_afu(device);
+
+	return scnprintf(buf, PAGE_SIZE, "%hhu:%hhu\n",
+			afu->config.version_major,
+			afu->config.version_minor);
+}
+
+static ssize_t contexts_show(struct device *device,
+		struct device_attribute *attr,
+		char *buf)
+{
+	struct ocxl_afu *afu = to_ocxl_afu(device);
+
+	return scnprintf(buf, PAGE_SIZE, "%d/%d\n",
+			afu->pasid_count, afu->pasid_max);
+}
+
+static struct device_attribute afu_attrs[] = {
+	__ATTR_RO(global_mmio_size),
+	__ATTR_RO(pp_mmio_size),
+	__ATTR_RO(afu_version),
+	__ATTR_RO(contexts),
+};
+
+static ssize_t global_mmio_read(struct file *filp, struct kobject *kobj,
+				struct bin_attribute *bin_attr, char *buf,
+				loff_t off, size_t count)
+{
+	struct ocxl_afu *afu = to_ocxl_afu(kobj_to_dev(kobj));
+
+	if (count == 0 || off < 0 ||
+		off >= afu->config.global_mmio_size)
+		return 0;
+	memcpy_fromio(buf, afu->global_mmio_ptr + off, count);
+	return count;
+}
+
+static int global_mmio_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct ocxl_afu *afu = vma->vm_private_data;
+	unsigned long offset;
+
+	if (vmf->pgoff >= (afu->config.global_mmio_size >> PAGE_SHIFT))
+		return VM_FAULT_SIGBUS;
+
+	offset = vmf->pgoff;
+	offset += (afu->global_mmio_start >> PAGE_SHIFT);
+	vm_insert_pfn(vma, vmf->address, offset);
+	return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct global_mmio_vmops = {
+	.fault = global_mmio_fault,
+};
+
+static int global_mmio_mmap(struct file *filp, struct kobject *kobj,
+			struct bin_attribute *bin_attr,
+			struct vm_area_struct *vma)
+{
+	struct ocxl_afu *afu = to_ocxl_afu(kobj_to_dev(kobj));
+
+	if ((vma_pages(vma) + vma->vm_pgoff) >
+		(afu->config.global_mmio_size >> PAGE_SHIFT))
+		return -EINVAL;
+
+	vma->vm_flags |= VM_IO | VM_PFNMAP;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_ops = &global_mmio_vmops;
+	vma->vm_private_data = afu;
+	return 0;
+}
+
+int ocxl_sysfs_add_afu(struct ocxl_afu *afu)
+{
+	int i, rc;
+
+	for (i = 0; i < ARRAY_SIZE(afu_attrs); i++) {
+		rc = device_create_file(&afu->dev, &afu_attrs[i]);
+		if (rc)
+			goto err;
+	}
+
+	sysfs_attr_init(&afu->attr_global_mmio.attr);
+	afu->attr_global_mmio.attr.name = "global_mmio_area";
+	afu->attr_global_mmio.attr.mode = 0600;
+	afu->attr_global_mmio.size = afu->config.global_mmio_size;
+	afu->attr_global_mmio.read = global_mmio_read;
+	afu->attr_global_mmio.mmap = global_mmio_mmap;
+	rc = device_create_bin_file(&afu->dev, &afu->attr_global_mmio);
+	if (rc) {
+		dev_err(&afu->dev,
+			"Unable to create global mmio attr for afu: %d\n",
+			rc);
+		goto err;
+	}
+
+	return 0;
+
+err:
+	for (i--; i >= 0; i--)
+		device_remove_file(&afu->dev, &afu_attrs[i]);
+	return rc;
+}
+
+void ocxl_sysfs_remove_afu(struct ocxl_afu *afu)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(afu_attrs); i++)
+		device_remove_file(&afu->dev, &afu_attrs[i]);
+	device_remove_bin_file(&afu->dev, &afu->attr_global_mmio);
+}
diff --git a/drivers/misc/ocxl/trace.c b/drivers/misc/ocxl/trace.c
new file mode 100644
index 000000000000..1e6947049697
--- /dev/null
+++ b/drivers/misc/ocxl/trace.c
@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+#endif
diff --git a/drivers/misc/ocxl/trace.h b/drivers/misc/ocxl/trace.h
new file mode 100644
index 000000000000..bcb7ff330c1e
--- /dev/null
+++ b/drivers/misc/ocxl/trace.h
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2017 IBM Corp.
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ocxl
+
+#if !defined(_TRACE_OCXL_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_OCXL_H
+
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(ocxl_context,
+	TP_PROTO(pid_t pid, void *spa, int pasid, u32 pidr, u32 tidr),
+	TP_ARGS(pid, spa, pasid, pidr, tidr),
+
+	TP_STRUCT__entry(
+		__field(pid_t, pid)
+		__field(void*, spa)
+		__field(int, pasid)
+		__field(u32, pidr)
+		__field(u32, tidr)
+	),
+
+	TP_fast_assign(
+		__entry->pid = pid;
+		__entry->spa = spa;
+		__entry->pasid = pasid;
+		__entry->pidr = pidr;
+		__entry->tidr = tidr;
+	),
+
+	TP_printk("linux pid=%d spa=0x%p pasid=0x%x pidr=0x%x tidr=0x%x",
+		__entry->pid,
+		__entry->spa,
+		__entry->pasid,
+		__entry->pidr,
+		__entry->tidr
+	)
+);
+
+DEFINE_EVENT(ocxl_context, ocxl_context_add,
+	TP_PROTO(pid_t pid, void *spa, int pasid, u32 pidr, u32 tidr),
+	TP_ARGS(pid, spa, pasid, pidr, tidr)
+);
+
+DEFINE_EVENT(ocxl_context, ocxl_context_remove,
+	TP_PROTO(pid_t pid, void *spa, int pasid, u32 pidr, u32 tidr),
+	TP_ARGS(pid, spa, pasid, pidr, tidr)
+);
+
+TRACE_EVENT(ocxl_terminate_pasid,
+	TP_PROTO(int pasid, int rc),
+	TP_ARGS(pasid, rc),
+
+	TP_STRUCT__entry(
+		__field(int, pasid)
+		__field(int, rc)
+	),
+
+	TP_fast_assign(
+		__entry->pasid = pasid;
+		__entry->rc = rc;
+	),
+
+	TP_printk("pasid=0x%x rc=%d",
+		__entry->pasid,
+		__entry->rc
+	)
+);
+
+DECLARE_EVENT_CLASS(ocxl_fault_handler,
+	TP_PROTO(void *spa, u64 pe, u64 dsisr, u64 dar, u64 tfc),
+	TP_ARGS(spa, pe, dsisr, dar, tfc),
+
+	TP_STRUCT__entry(
+		__field(void *, spa)
+		__field(u64, pe)
+		__field(u64, dsisr)
+		__field(u64, dar)
+		__field(u64, tfc)
+	),
+
+	TP_fast_assign(
+		__entry->spa = spa;
+		__entry->pe = pe;
+		__entry->dsisr = dsisr;
+		__entry->dar = dar;
+		__entry->tfc = tfc;
+	),
+
+	TP_printk("spa=%p pe=0x%llx dsisr=0x%llx dar=0x%llx tfc=0x%llx",
+		__entry->spa,
+		__entry->pe,
+		__entry->dsisr,
+		__entry->dar,
+		__entry->tfc
+	)
+);
+
+DEFINE_EVENT(ocxl_fault_handler, ocxl_fault,
+	TP_PROTO(void *spa, u64 pe, u64 dsisr, u64 dar, u64 tfc),
+	TP_ARGS(spa, pe, dsisr, dar, tfc)
+);
+
+DEFINE_EVENT(ocxl_fault_handler, ocxl_fault_ack,
+	TP_PROTO(void *spa, u64 pe, u64 dsisr, u64 dar, u64 tfc),
+	TP_ARGS(spa, pe, dsisr, dar, tfc)
+);
+
+TRACE_EVENT(ocxl_afu_irq_alloc,
+	TP_PROTO(int pasid, int irq_id, unsigned int virq, int hw_irq,
+		u64 irq_offset),
+	TP_ARGS(pasid, irq_id, virq, hw_irq, irq_offset),
+
+	TP_STRUCT__entry(
+		__field(int, pasid)
+		__field(int, irq_id)
+		__field(unsigned int, virq)
+		__field(int, hw_irq)
+		__field(u64, irq_offset)
+	),
+
+	TP_fast_assign(
+		__entry->pasid = pasid;
+		__entry->irq_id = irq_id;
+		__entry->virq = virq;
+		__entry->hw_irq = hw_irq;
+		__entry->irq_offset = irq_offset;
+	),
+
+	TP_printk("pasid=0x%x irq_id=%d virq=%u hw_irq=%d irq_offset=0x%llx",
+		__entry->pasid,
+		__entry->irq_id,
+		__entry->virq,
+		__entry->hw_irq,
+		__entry->irq_offset
+	)
+);
+
+TRACE_EVENT(ocxl_afu_irq_free,
+	TP_PROTO(int pasid, int irq_id),
+	TP_ARGS(pasid, irq_id),
+
+	TP_STRUCT__entry(
+		__field(int, pasid)
+		__field(int, irq_id)
+	),
+
+	TP_fast_assign(
+		__entry->pasid = pasid;
+		__entry->irq_id = irq_id;
+	),
+
+	TP_printk("pasid=0x%x irq_id=%d",
+		__entry->pasid,
+		__entry->irq_id
+	)
+);
+
+TRACE_EVENT(ocxl_afu_irq_receive,
+	TP_PROTO(int virq),
+	TP_ARGS(virq),
+
+	TP_STRUCT__entry(
+		__field(int, virq)
+	),
+
+	TP_fast_assign(
+		__entry->virq = virq;
+	),
+
+	TP_printk("virq=%d",
+		__entry->virq
+	)
+);
+
+#endif /* _TRACE_OCXL_H */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE trace
+#include <trace/define_trace.h>
diff --git a/drivers/misc/phantom.c b/drivers/misc/phantom.c
index 30754927fd80..b084245f6238 100644
--- a/drivers/misc/phantom.c
+++ b/drivers/misc/phantom.c
@@ -256,18 +256,18 @@ static int phantom_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int phantom_poll(struct file *file, poll_table *wait)
+static __poll_t phantom_poll(struct file *file, poll_table *wait)
 {
 	struct phantom_device *dev = file->private_data;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	pr_debug("phantom_poll: %d\n", atomic_read(&dev->counter));
 	poll_wait(file, &dev->wait, wait);
 
 	if (!(dev->status & PHB_RUNNING))
-		mask = POLLERR;
+		mask = EPOLLERR;
 	else if (atomic_read(&dev->counter))
-		mask = POLLIN | POLLRDNORM;
+		mask = EPOLLIN | EPOLLRDNORM;
 
 	pr_debug("phantom_poll end: %x/%d\n", mask, atomic_read(&dev->counter));
 
diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c
index 9918eda0e05f..a3454eb56fbf 100644
--- a/drivers/misc/sgi-gru/grutlbpurge.c
+++ b/drivers/misc/sgi-gru/grutlbpurge.c
@@ -258,6 +258,7 @@ static void gru_release(struct mmu_notifier *mn, struct mm_struct *mm)
 
 
 static const struct mmu_notifier_ops gru_mmuops = {
+	.flags			= MMU_INVALIDATE_DOES_NOT_BLOCK,
 	.invalidate_range_start	= gru_invalidate_range_start,
 	.invalidate_range_end	= gru_invalidate_range_end,
 	.release		= gru_release,
diff --git a/drivers/misc/vexpress-syscfg.c b/drivers/misc/vexpress-syscfg.c
index 2cde80c7bb93..9eea30f54fd6 100644
--- a/drivers/misc/vexpress-syscfg.c
+++ b/drivers/misc/vexpress-syscfg.c
@@ -270,10 +270,8 @@ static int vexpress_syscfg_probe(struct platform_device *pdev)
 	/* Must use dev.parent (MFD), as that's where DT phandle points at... */
 	bridge = vexpress_config_bridge_register(pdev->dev.parent,
 			&vexpress_syscfg_bridge_ops, syscfg);
-	if (IS_ERR(bridge))
-		return PTR_ERR(bridge);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(bridge);
 }
 
 static const struct platform_device_id vexpress_syscfg_id_table[] = {
diff --git a/drivers/misc/vmw_vmci/vmci_host.c b/drivers/misc/vmw_vmci/vmci_host.c
index 8a16a26e9658..83e0c95d20a4 100644
--- a/drivers/misc/vmw_vmci/vmci_host.c
+++ b/drivers/misc/vmw_vmci/vmci_host.c
@@ -166,11 +166,11 @@ static int vmci_host_close(struct inode *inode, struct file *filp)
  * This is used to wake up the VMX when a VMCI call arrives, or
  * to wake up select() or poll() at the next clock tick.
  */
-static unsigned int vmci_host_poll(struct file *filp, poll_table *wait)
+static __poll_t vmci_host_poll(struct file *filp, poll_table *wait)
 {
 	struct vmci_host_dev *vmci_host_dev = filp->private_data;
 	struct vmci_ctx *context = vmci_host_dev->context;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	if (vmci_host_dev->ct_type == VMCIOBJ_CONTEXT) {
 		/* Check for VMCI calls to this VM context. */
@@ -182,7 +182,7 @@ static unsigned int vmci_host_poll(struct file *filp, poll_table *wait)
 		if (context->pending_datagrams > 0 ||
 		    vmci_handle_arr_get_size(
 				context->pending_doorbell_array) > 0) {
-			mask = POLLIN;
+			mask = EPOLLIN;
 		}
 		spin_unlock(&context->lock);
 	}
diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c
index 8af5c2672f71..0339538c182d 100644
--- a/drivers/misc/vmw_vmci/vmci_queue_pair.c
+++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c
@@ -129,23 +129,6 @@
  * *_MEM state, and vice versa.
  */
 
-/*
- * VMCIMemcpy{To,From}QueueFunc() prototypes.  Functions of these
- * types are passed around to enqueue and dequeue routines.  Note that
- * often the functions passed are simply wrappers around memcpy
- * itself.
- *
- * Note: In order for the memcpy typedefs to be compatible with the VMKernel,
- * there's an unused last parameter for the hosted side.  In
- * ESX, that parameter holds a buffer type.
- */
-typedef int vmci_memcpy_to_queue_func(struct vmci_queue *queue,
-				      u64 queue_offset, const void *src,
-				      size_t src_offset, size_t size);
-typedef int vmci_memcpy_from_queue_func(void *dest, size_t dest_offset,
-					const struct vmci_queue *queue,
-					u64 queue_offset, size_t size);
-
 /* The Kernel specific component of the struct vmci_queue structure. */
 struct vmci_queue_kern_if {
 	struct mutex __mutex;	/* Protects the queue. */
@@ -351,11 +334,10 @@ static void *qp_alloc_queue(u64 size, u32 flags)
  * by traversing the offset -> page translation structure for the queue.
  * Assumes that offset + size does not wrap around in the queue.
  */
-static int __qp_memcpy_to_queue(struct vmci_queue *queue,
-				u64 queue_offset,
-				const void *src,
-				size_t size,
-				bool is_iovec)
+static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
+				  u64 queue_offset,
+				  struct iov_iter *from,
+				  size_t size)
 {
 	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
 	size_t bytes_copied = 0;
@@ -380,23 +362,12 @@ static int __qp_memcpy_to_queue(struct vmci_queue *queue,
 		else
 			to_copy = size - bytes_copied;
 
-		if (is_iovec) {
-			struct msghdr *msg = (struct msghdr *)src;
-			int err;
-
-			/* The iovec will track bytes_copied internally. */
-			err = memcpy_from_msg((u8 *)va + page_offset,
-					      msg, to_copy);
-			if (err != 0) {
-				if (kernel_if->host)
-					kunmap(kernel_if->u.h.page[page_index]);
-				return VMCI_ERROR_INVALID_ARGS;
-			}
-		} else {
-			memcpy((u8 *)va + page_offset,
-			       (u8 *)src + bytes_copied, to_copy);
+		if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
+					 from)) {
+			if (kernel_if->host)
+				kunmap(kernel_if->u.h.page[page_index]);
+			return VMCI_ERROR_INVALID_ARGS;
 		}
-
 		bytes_copied += to_copy;
 		if (kernel_if->host)
 			kunmap(kernel_if->u.h.page[page_index]);
@@ -411,11 +382,9 @@ static int __qp_memcpy_to_queue(struct vmci_queue *queue,
  * by traversing the offset -> page translation structure for the queue.
  * Assumes that offset + size does not wrap around in the queue.
  */
-static int __qp_memcpy_from_queue(void *dest,
-				  const struct vmci_queue *queue,
-				  u64 queue_offset,
-				  size_t size,
-				  bool is_iovec)
+static int qp_memcpy_from_queue_iter(struct iov_iter *to,
+				    const struct vmci_queue *queue,
+				    u64 queue_offset, size_t size)
 {
 	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
 	size_t bytes_copied = 0;
@@ -427,6 +396,7 @@ static int __qp_memcpy_from_queue(void *dest,
 		    (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
 		void *va;
 		size_t to_copy;
+		int err;
 
 		if (kernel_if->host)
 			va = kmap(kernel_if->u.h.page[page_index]);
@@ -440,23 +410,12 @@ static int __qp_memcpy_from_queue(void *dest,
 		else
 			to_copy = size - bytes_copied;
 
-		if (is_iovec) {
-			struct msghdr *msg = dest;
-			int err;
-
-			/* The iovec will track bytes_copied internally. */
-			err = memcpy_to_msg(msg, (u8 *)va + page_offset,
-					     to_copy);
-			if (err != 0) {
-				if (kernel_if->host)
-					kunmap(kernel_if->u.h.page[page_index]);
-				return VMCI_ERROR_INVALID_ARGS;
-			}
-		} else {
-			memcpy((u8 *)dest + bytes_copied,
-			       (u8 *)va + page_offset, to_copy);
+		err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
+		if (err != to_copy) {
+			if (kernel_if->host)
+				kunmap(kernel_if->u.h.page[page_index]);
+			return VMCI_ERROR_INVALID_ARGS;
 		}
-
 		bytes_copied += to_copy;
 		if (kernel_if->host)
 			kunmap(kernel_if->u.h.page[page_index]);
@@ -569,54 +528,6 @@ static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
 	return VMCI_SUCCESS;
 }
 
-static int qp_memcpy_to_queue(struct vmci_queue *queue,
-			      u64 queue_offset,
-			      const void *src, size_t src_offset, size_t size)
-{
-	return __qp_memcpy_to_queue(queue, queue_offset,
-				    (u8 *)src + src_offset, size, false);
-}
-
-static int qp_memcpy_from_queue(void *dest,
-				size_t dest_offset,
-				const struct vmci_queue *queue,
-				u64 queue_offset, size_t size)
-{
-	return __qp_memcpy_from_queue((u8 *)dest + dest_offset,
-				      queue, queue_offset, size, false);
-}
-
-/*
- * Copies from a given iovec from a VMCI Queue.
- */
-static int qp_memcpy_to_queue_iov(struct vmci_queue *queue,
-				  u64 queue_offset,
-				  const void *msg,
-				  size_t src_offset, size_t size)
-{
-
-	/*
-	 * We ignore src_offset because src is really a struct iovec * and will
-	 * maintain offset internally.
-	 */
-	return __qp_memcpy_to_queue(queue, queue_offset, msg, size, true);
-}
-
-/*
- * Copies to a given iovec from a VMCI Queue.
- */
-static int qp_memcpy_from_queue_iov(void *dest,
-				    size_t dest_offset,
-				    const struct vmci_queue *queue,
-				    u64 queue_offset, size_t size)
-{
-	/*
-	 * We ignore dest_offset because dest is really a struct iovec * and
-	 * will maintain offset internally.
-	 */
-	return __qp_memcpy_from_queue(dest, queue, queue_offset, size, true);
-}
-
 /*
  * Allocates kernel VA space of specified size plus space for the queue
  * and kernel interface.  This is different from the guest queue allocator,
@@ -2629,12 +2540,11 @@ static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
 				 struct vmci_queue *consume_q,
 				 const u64 produce_q_size,
-				 const void *buf,
-				 size_t buf_size,
-				 vmci_memcpy_to_queue_func memcpy_to_queue)
+				 struct iov_iter *from)
 {
 	s64 free_space;
 	u64 tail;
+	size_t buf_size = iov_iter_count(from);
 	size_t written;
 	ssize_t result;
 
@@ -2654,15 +2564,15 @@ static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
 	written = (size_t) (free_space > buf_size ? buf_size : free_space);
 	tail = vmci_q_header_producer_tail(produce_q->q_header);
 	if (likely(tail + written < produce_q_size)) {
-		result = memcpy_to_queue(produce_q, tail, buf, 0, written);
+		result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
 	} else {
 		/* Tail pointer wraps around. */
 
 		const size_t tmp = (size_t) (produce_q_size - tail);
 
-		result = memcpy_to_queue(produce_q, tail, buf, 0, tmp);
+		result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
 		if (result >= VMCI_SUCCESS)
-			result = memcpy_to_queue(produce_q, 0, buf, tmp,
+			result = qp_memcpy_to_queue_iter(produce_q, 0, from,
 						 written - tmp);
 	}
 
@@ -2690,11 +2600,10 @@ static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
 				 struct vmci_queue *consume_q,
 				 const u64 consume_q_size,
-				 void *buf,
-				 size_t buf_size,
-				 vmci_memcpy_from_queue_func memcpy_from_queue,
+				 struct iov_iter *to,
 				 bool update_consumer)
 {
+	size_t buf_size = iov_iter_count(to);
 	s64 buf_ready;
 	u64 head;
 	size_t read;
@@ -2716,15 +2625,15 @@ static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
 	read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
 	head = vmci_q_header_consumer_head(produce_q->q_header);
 	if (likely(head + read < consume_q_size)) {
-		result = memcpy_from_queue(buf, 0, consume_q, head, read);
+		result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
 	} else {
 		/* Head pointer wraps around. */
 
 		const size_t tmp = (size_t) (consume_q_size - head);
 
-		result = memcpy_from_queue(buf, 0, consume_q, head, tmp);
+		result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
 		if (result >= VMCI_SUCCESS)
-			result = memcpy_from_queue(buf, tmp, consume_q, 0,
+			result = qp_memcpy_from_queue_iter(to, consume_q, 0,
 						   read - tmp);
 
 	}
@@ -3118,18 +3027,21 @@ ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
 			   int buf_type)
 {
 	ssize_t result;
+	struct iov_iter from;
+	struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
 
 	if (!qpair || !buf)
 		return VMCI_ERROR_INVALID_ARGS;
 
+	iov_iter_kvec(&from, WRITE | ITER_KVEC, &v, 1, buf_size);
+
 	qp_lock(qpair);
 
 	do {
 		result = qp_enqueue_locked(qpair->produce_q,
 					   qpair->consume_q,
 					   qpair->produce_q_size,
-					   buf, buf_size,
-					   qp_memcpy_to_queue);
+					   &from);
 
 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
 		    !qp_wait_for_ready_queue(qpair))
@@ -3159,18 +3071,21 @@ ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
 			   int buf_type)
 {
 	ssize_t result;
+	struct iov_iter to;
+	struct kvec v = {.iov_base = buf, .iov_len = buf_size};
 
 	if (!qpair || !buf)
 		return VMCI_ERROR_INVALID_ARGS;
 
+	iov_iter_kvec(&to, READ | ITER_KVEC, &v, 1, buf_size);
+
 	qp_lock(qpair);
 
 	do {
 		result = qp_dequeue_locked(qpair->produce_q,
 					   qpair->consume_q,
 					   qpair->consume_q_size,
-					   buf, buf_size,
-					   qp_memcpy_from_queue, true);
+					   &to, true);
 
 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
 		    !qp_wait_for_ready_queue(qpair))
@@ -3200,19 +3115,22 @@ ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
 			size_t buf_size,
 			int buf_type)
 {
+	struct iov_iter to;
+	struct kvec v = {.iov_base = buf, .iov_len = buf_size};
 	ssize_t result;
 
 	if (!qpair || !buf)
 		return VMCI_ERROR_INVALID_ARGS;
 
+	iov_iter_kvec(&to, READ | ITER_KVEC, &v, 1, buf_size);
+
 	qp_lock(qpair);
 
 	do {
 		result = qp_dequeue_locked(qpair->produce_q,
 					   qpair->consume_q,
 					   qpair->consume_q_size,
-					   buf, buf_size,
-					   qp_memcpy_from_queue, false);
+					   &to, false);
 
 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
 		    !qp_wait_for_ready_queue(qpair))
@@ -3253,8 +3171,7 @@ ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
 		result = qp_enqueue_locked(qpair->produce_q,
 					   qpair->consume_q,
 					   qpair->produce_q_size,
-					   msg, iov_size,
-					   qp_memcpy_to_queue_iov);
+					   &msg->msg_iter);
 
 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
 		    !qp_wait_for_ready_queue(qpair))
@@ -3295,9 +3212,7 @@ ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
 		result = qp_dequeue_locked(qpair->produce_q,
 					   qpair->consume_q,
 					   qpair->consume_q_size,
-					   msg, iov_size,
-					   qp_memcpy_from_queue_iov,
-					   true);
+					   &msg->msg_iter, true);
 
 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
 		    !qp_wait_for_ready_queue(qpair))
@@ -3339,9 +3254,7 @@ ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
 		result = qp_dequeue_locked(qpair->produce_q,
 					   qpair->consume_q,
 					   qpair->consume_q_size,
-					   msg, iov_size,
-					   qp_memcpy_from_queue_iov,
-					   false);
+					   &msg->msg_iter, false);
 
 		if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
 		    !qp_wait_for_ready_queue(qpair))
diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c
index ccfa98af1dd3..2cfb963d9f37 100644
--- a/drivers/mmc/core/block.c
+++ b/drivers/mmc/core/block.c
@@ -63,9 +63,16 @@ MODULE_ALIAS("mmc:block");
 #endif
 #define MODULE_PARAM_PREFIX "mmcblk."
 
-#define MMC_BLK_TIMEOUT_MS  (10 * 60 * 1000)        /* 10 minute timeout */
+/*
+ * Set a 10 second timeout for polling write request busy state. Note, mmc core
+ * is setting a 3 second timeout for SD cards, and SDHCI has long had a 10
+ * second software timer to timeout the whole request, so 10 seconds should be
+ * ample.
+ */
+#define MMC_BLK_TIMEOUT_MS  (10 * 1000)
 #define MMC_SANITIZE_REQ_TIMEOUT 240000
 #define MMC_EXTRACT_INDEX_FROM_ARG(x) ((x & 0x00FF0000) >> 16)
+#define MMC_EXTRACT_VALUE_FROM_ARG(x) ((x & 0x0000FF00) >> 8)
 
 #define mmc_req_rel_wr(req)	((req->cmd_flags & REQ_FUA) && \
 				  (rq_data_dir(req) == WRITE))
@@ -112,6 +119,7 @@ struct mmc_blk_data {
 #define MMC_BLK_WRITE		BIT(1)
 #define MMC_BLK_DISCARD		BIT(2)
 #define MMC_BLK_SECDISCARD	BIT(3)
+#define MMC_BLK_CQE_RECOVERY	BIT(4)
 
 	/*
 	 * Only set in main mmc_blk_data associated
@@ -189,7 +197,7 @@ static void mmc_blk_put(struct mmc_blk_data *md)
 	md->usage--;
 	if (md->usage == 0) {
 		int devidx = mmc_get_devidx(md->disk);
-		blk_cleanup_queue(md->queue.queue);
+		blk_put_queue(md->queue.queue);
 		ida_simple_remove(&mmc_blk_ida, devidx);
 		put_disk(md->disk);
 		kfree(md);
@@ -579,6 +587,24 @@ static int __mmc_blk_ioctl_cmd(struct mmc_card *card, struct mmc_blk_data *md,
 		return data.error;
 	}
 
+	/*
+	 * Make sure the cache of the PARTITION_CONFIG register and
+	 * PARTITION_ACCESS bits is updated in case the ioctl ext_csd write
+	 * changed it successfully.
+	 */
+	if ((MMC_EXTRACT_INDEX_FROM_ARG(cmd.arg) == EXT_CSD_PART_CONFIG) &&
+	    (cmd.opcode == MMC_SWITCH)) {
+		struct mmc_blk_data *main_md = dev_get_drvdata(&card->dev);
+		u8 value = MMC_EXTRACT_VALUE_FROM_ARG(cmd.arg);
+
+		/*
+		 * Update cache so the next mmc_blk_part_switch call operates
+		 * on up-to-date data.
+		 */
+		card->ext_csd.part_config = value;
+		main_md->part_curr = value & EXT_CSD_PART_CONFIG_ACC_MASK;
+	}
+
 	/*
 	 * According to the SD specs, some commands require a delay after
 	 * issuing the command.
@@ -921,14 +947,54 @@ static int mmc_sd_num_wr_blocks(struct mmc_card *card, u32 *written_blocks)
 	return 0;
 }
 
+static unsigned int mmc_blk_clock_khz(struct mmc_host *host)
+{
+	if (host->actual_clock)
+		return host->actual_clock / 1000;
+
+	/* Clock may be subject to a divisor, fudge it by a factor of 2. */
+	if (host->ios.clock)
+		return host->ios.clock / 2000;
+
+	/* How can there be no clock */
+	WARN_ON_ONCE(1);
+	return 100; /* 100 kHz is minimum possible value */
+}
+
+static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host,
+					    struct mmc_data *data)
+{
+	unsigned int ms = DIV_ROUND_UP(data->timeout_ns, 1000000);
+	unsigned int khz;
+
+	if (data->timeout_clks) {
+		khz = mmc_blk_clock_khz(host);
+		ms += DIV_ROUND_UP(data->timeout_clks, khz);
+	}
+
+	return ms;
+}
+
+static inline bool mmc_blk_in_tran_state(u32 status)
+{
+	/*
+	 * Some cards mishandle the status bits, so make sure to check both the
+	 * busy indication and the card state.
+	 */
+	return status & R1_READY_FOR_DATA &&
+	       (R1_CURRENT_STATE(status) == R1_STATE_TRAN);
+}
+
 static int card_busy_detect(struct mmc_card *card, unsigned int timeout_ms,
-		bool hw_busy_detect, struct request *req, bool *gen_err)
+			    struct request *req, u32 *resp_errs)
 {
 	unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 	int err = 0;
 	u32 status;
 
 	do {
+		bool done = time_after(jiffies, timeout);
+
 		err = __mmc_send_status(card, &status, 5);
 		if (err) {
 			pr_err("%s: error %d requesting status\n",
@@ -936,25 +1002,18 @@ static int card_busy_detect(struct mmc_card *card, unsigned int timeout_ms,
 			return err;
 		}
 
-		if (status & R1_ERROR) {
-			pr_err("%s: %s: error sending status cmd, status %#x\n",
-				req->rq_disk->disk_name, __func__, status);
-			*gen_err = true;
-		}
-
-		/* We may rely on the host hw to handle busy detection.*/
-		if ((card->host->caps & MMC_CAP_WAIT_WHILE_BUSY) &&
-			hw_busy_detect)
-			break;
+		/* Accumulate any response error bits seen */
+		if (resp_errs)
+			*resp_errs |= status;
 
 		/*
 		 * Timeout if the device never becomes ready for data and never
 		 * leaves the program state.
 		 */
-		if (time_after(jiffies, timeout)) {
-			pr_err("%s: Card stuck in programming state! %s %s\n",
+		if (done) {
+			pr_err("%s: Card stuck in wrong state! %s %s status: %#x\n",
 				mmc_hostname(card->host),
-				req->rq_disk->disk_name, __func__);
+				req->rq_disk->disk_name, __func__, status);
 			return -ETIMEDOUT;
 		}
 
@@ -963,229 +1022,11 @@ static int card_busy_detect(struct mmc_card *card, unsigned int timeout_ms,
 		 * so make sure to check both the busy
 		 * indication and the card state.
 		 */
-	} while (!(status & R1_READY_FOR_DATA) ||
-		 (R1_CURRENT_STATE(status) == R1_STATE_PRG));
+	} while (!mmc_blk_in_tran_state(status));
 
 	return err;
 }
 
-static int send_stop(struct mmc_card *card, unsigned int timeout_ms,
-		struct request *req, bool *gen_err, u32 *stop_status)
-{
-	struct mmc_host *host = card->host;
-	struct mmc_command cmd = {};
-	int err;
-	bool use_r1b_resp = rq_data_dir(req) == WRITE;
-
-	/*
-	 * Normally we use R1B responses for WRITE, but in cases where the host
-	 * has specified a max_busy_timeout we need to validate it. A failure
-	 * means we need to prevent the host from doing hw busy detection, which
-	 * is done by converting to a R1 response instead.
-	 */
-	if (host->max_busy_timeout && (timeout_ms > host->max_busy_timeout))
-		use_r1b_resp = false;
-
-	cmd.opcode = MMC_STOP_TRANSMISSION;
-	if (use_r1b_resp) {
-		cmd.flags = MMC_RSP_SPI_R1B | MMC_RSP_R1B | MMC_CMD_AC;
-		cmd.busy_timeout = timeout_ms;
-	} else {
-		cmd.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC;
-	}
-
-	err = mmc_wait_for_cmd(host, &cmd, 5);
-	if (err)
-		return err;
-
-	*stop_status = cmd.resp[0];
-
-	/* No need to check card status in case of READ. */
-	if (rq_data_dir(req) == READ)
-		return 0;
-
-	if (!mmc_host_is_spi(host) &&
-		(*stop_status & R1_ERROR)) {
-		pr_err("%s: %s: general error sending stop command, resp %#x\n",
-			req->rq_disk->disk_name, __func__, *stop_status);
-		*gen_err = true;
-	}
-
-	return card_busy_detect(card, timeout_ms, use_r1b_resp, req, gen_err);
-}
-
-#define ERR_NOMEDIUM	3
-#define ERR_RETRY	2
-#define ERR_ABORT	1
-#define ERR_CONTINUE	0
-
-static int mmc_blk_cmd_error(struct request *req, const char *name, int error,
-	bool status_valid, u32 status)
-{
-	switch (error) {
-	case -EILSEQ:
-		/* response crc error, retry the r/w cmd */
-		pr_err("%s: %s sending %s command, card status %#x\n",
-			req->rq_disk->disk_name, "response CRC error",
-			name, status);
-		return ERR_RETRY;
-
-	case -ETIMEDOUT:
-		pr_err("%s: %s sending %s command, card status %#x\n",
-			req->rq_disk->disk_name, "timed out", name, status);
-
-		/* If the status cmd initially failed, retry the r/w cmd */
-		if (!status_valid) {
-			pr_err("%s: status not valid, retrying timeout\n",
-				req->rq_disk->disk_name);
-			return ERR_RETRY;
-		}
-
-		/*
-		 * If it was a r/w cmd crc error, or illegal command
-		 * (eg, issued in wrong state) then retry - we should
-		 * have corrected the state problem above.
-		 */
-		if (status & (R1_COM_CRC_ERROR | R1_ILLEGAL_COMMAND)) {
-			pr_err("%s: command error, retrying timeout\n",
-				req->rq_disk->disk_name);
-			return ERR_RETRY;
-		}
-
-		/* Otherwise abort the command */
-		return ERR_ABORT;
-
-	default:
-		/* We don't understand the error code the driver gave us */
-		pr_err("%s: unknown error %d sending read/write command, card status %#x\n",
-		       req->rq_disk->disk_name, error, status);
-		return ERR_ABORT;
-	}
-}
-
-/*
- * Initial r/w and stop cmd error recovery.
- * We don't know whether the card received the r/w cmd or not, so try to
- * restore things back to a sane state.  Essentially, we do this as follows:
- * - Obtain card status.  If the first attempt to obtain card status fails,
- *   the status word will reflect the failed status cmd, not the failed
- *   r/w cmd.  If we fail to obtain card status, it suggests we can no
- *   longer communicate with the card.
- * - Check the card state.  If the card received the cmd but there was a
- *   transient problem with the response, it might still be in a data transfer
- *   mode.  Try to send it a stop command.  If this fails, we can't recover.
- * - If the r/w cmd failed due to a response CRC error, it was probably
- *   transient, so retry the cmd.
- * - If the r/w cmd timed out, but we didn't get the r/w cmd status, retry.
- * - If the r/w cmd timed out, and the r/w cmd failed due to CRC error or
- *   illegal cmd, retry.
- * Otherwise we don't understand what happened, so abort.
- */
-static int mmc_blk_cmd_recovery(struct mmc_card *card, struct request *req,
-	struct mmc_blk_request *brq, bool *ecc_err, bool *gen_err)
-{
-	bool prev_cmd_status_valid = true;
-	u32 status, stop_status = 0;
-	int err, retry;
-
-	if (mmc_card_removed(card))
-		return ERR_NOMEDIUM;
-
-	/*
-	 * Try to get card status which indicates both the card state
-	 * and why there was no response.  If the first attempt fails,
-	 * we can't be sure the returned status is for the r/w command.
-	 */
-	for (retry = 2; retry >= 0; retry--) {
-		err = __mmc_send_status(card, &status, 0);
-		if (!err)
-			break;
-
-		/* Re-tune if needed */
-		mmc_retune_recheck(card->host);
-
-		prev_cmd_status_valid = false;
-		pr_err("%s: error %d sending status command, %sing\n",
-		       req->rq_disk->disk_name, err, retry ? "retry" : "abort");
-	}
-
-	/* We couldn't get a response from the card.  Give up. */
-	if (err) {
-		/* Check if the card is removed */
-		if (mmc_detect_card_removed(card->host))
-			return ERR_NOMEDIUM;
-		return ERR_ABORT;
-	}
-
-	/* Flag ECC errors */
-	if ((status & R1_CARD_ECC_FAILED) ||
-	    (brq->stop.resp[0] & R1_CARD_ECC_FAILED) ||
-	    (brq->cmd.resp[0] & R1_CARD_ECC_FAILED))
-		*ecc_err = true;
-
-	/* Flag General errors */
-	if (!mmc_host_is_spi(card->host) && rq_data_dir(req) != READ)
-		if ((status & R1_ERROR) ||
-			(brq->stop.resp[0] & R1_ERROR)) {
-			pr_err("%s: %s: general error sending stop or status command, stop cmd response %#x, card status %#x\n",
-			       req->rq_disk->disk_name, __func__,
-			       brq->stop.resp[0], status);
-			*gen_err = true;
-		}
-
-	/*
-	 * Check the current card state.  If it is in some data transfer
-	 * mode, tell it to stop (and hopefully transition back to TRAN.)
-	 */
-	if (R1_CURRENT_STATE(status) == R1_STATE_DATA ||
-	    R1_CURRENT_STATE(status) == R1_STATE_RCV) {
-		err = send_stop(card,
-			DIV_ROUND_UP(brq->data.timeout_ns, 1000000),
-			req, gen_err, &stop_status);
-		if (err) {
-			pr_err("%s: error %d sending stop command\n",
-			       req->rq_disk->disk_name, err);
-			/*
-			 * If the stop cmd also timed out, the card is probably
-			 * not present, so abort. Other errors are bad news too.
-			 */
-			return ERR_ABORT;
-		}
-
-		if (stop_status & R1_CARD_ECC_FAILED)
-			*ecc_err = true;
-	}
-
-	/* Check for set block count errors */
-	if (brq->sbc.error)
-		return mmc_blk_cmd_error(req, "SET_BLOCK_COUNT", brq->sbc.error,
-				prev_cmd_status_valid, status);
-
-	/* Check for r/w command errors */
-	if (brq->cmd.error)
-		return mmc_blk_cmd_error(req, "r/w cmd", brq->cmd.error,
-				prev_cmd_status_valid, status);
-
-	/* Data errors */
-	if (!brq->stop.error)
-		return ERR_CONTINUE;
-
-	/* Now for stop errors.  These aren't fatal to the transfer. */
-	pr_info("%s: error %d sending stop command, original cmd response %#x, card status %#x\n",
-	       req->rq_disk->disk_name, brq->stop.error,
-	       brq->cmd.resp[0], status);
-
-	/*
-	 * Subsitute in our own stop status as this will give the error
-	 * state which happened during the execution of the r/w command.
-	 */
-	if (stop_status) {
-		brq->stop.resp[0] = stop_status;
-		brq->stop.error = 0;
-	}
-	return ERR_CONTINUE;
-}
-
 static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host,
 			 int type)
 {
@@ -1281,7 +1122,7 @@ static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)
 		break;
 	}
 	mq_rq->drv_op_result = ret;
-	blk_end_request_all(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
+	blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
 }
 
 static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
@@ -1324,7 +1165,7 @@ static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)
 	else
 		mmc_blk_reset_success(md, type);
 fail:
-	blk_end_request(req, status, blk_rq_bytes(req));
+	blk_mq_end_request(req, status);
 }
 
 static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
@@ -1394,7 +1235,7 @@ static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,
 	if (!err)
 		mmc_blk_reset_success(md, type);
 out:
-	blk_end_request(req, status, blk_rq_bytes(req));
+	blk_mq_end_request(req, status);
 }
 
 static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
@@ -1404,7 +1245,7 @@ static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)
 	int ret = 0;
 
 	ret = mmc_flush_cache(card);
-	blk_end_request_all(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
+	blk_mq_end_request(req, ret ? BLK_STS_IOERR : BLK_STS_OK);
 }
 
 /*
@@ -1430,15 +1271,18 @@ static inline void mmc_apply_rel_rw(struct mmc_blk_request *brq,
 	}
 }
 
-#define CMD_ERRORS							\
-	(R1_OUT_OF_RANGE |	/* Command argument out of range */	\
-	 R1_ADDRESS_ERROR |	/* Misaligned address */		\
+#define CMD_ERRORS_EXCL_OOR						\
+	(R1_ADDRESS_ERROR |	/* Misaligned address */		\
 	 R1_BLOCK_LEN_ERROR |	/* Transferred block length incorrect */\
 	 R1_WP_VIOLATION |	/* Tried to write to protected block */	\
 	 R1_CARD_ECC_FAILED |	/* Card ECC failed */			\
 	 R1_CC_ERROR |		/* Card controller error */		\
 	 R1_ERROR)		/* General/unknown error */
 
+#define CMD_ERRORS							\
+	(CMD_ERRORS_EXCL_OOR |						\
+	 R1_OUT_OF_RANGE)	/* Command argument out of range */	\
+
 static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
 {
 	u32 val;
@@ -1481,116 +1325,6 @@ static void mmc_blk_eval_resp_error(struct mmc_blk_request *brq)
 	}
 }
 
-static enum mmc_blk_status mmc_blk_err_check(struct mmc_card *card,
-					     struct mmc_async_req *areq)
-{
-	struct mmc_queue_req *mq_mrq = container_of(areq, struct mmc_queue_req,
-						    areq);
-	struct mmc_blk_request *brq = &mq_mrq->brq;
-	struct request *req = mmc_queue_req_to_req(mq_mrq);
-	int need_retune = card->host->need_retune;
-	bool ecc_err = false;
-	bool gen_err = false;
-
-	/*
-	 * sbc.error indicates a problem with the set block count
-	 * command.  No data will have been transferred.
-	 *
-	 * cmd.error indicates a problem with the r/w command.  No
-	 * data will have been transferred.
-	 *
-	 * stop.error indicates a problem with the stop command.  Data
-	 * may have been transferred, or may still be transferring.
-	 */
-
-	mmc_blk_eval_resp_error(brq);
-
-	if (brq->sbc.error || brq->cmd.error ||
-	    brq->stop.error || brq->data.error) {
-		switch (mmc_blk_cmd_recovery(card, req, brq, &ecc_err, &gen_err)) {
-		case ERR_RETRY:
-			return MMC_BLK_RETRY;
-		case ERR_ABORT:
-			return MMC_BLK_ABORT;
-		case ERR_NOMEDIUM:
-			return MMC_BLK_NOMEDIUM;
-		case ERR_CONTINUE:
-			break;
-		}
-	}
-
-	/*
-	 * Check for errors relating to the execution of the
-	 * initial command - such as address errors.  No data
-	 * has been transferred.
-	 */
-	if (brq->cmd.resp[0] & CMD_ERRORS) {
-		pr_err("%s: r/w command failed, status = %#x\n",
-		       req->rq_disk->disk_name, brq->cmd.resp[0]);
-		return MMC_BLK_ABORT;
-	}
-
-	/*
-	 * Everything else is either success, or a data error of some
-	 * kind.  If it was a write, we may have transitioned to
-	 * program mode, which we have to wait for it to complete.
-	 */
-	if (!mmc_host_is_spi(card->host) && rq_data_dir(req) != READ) {
-		int err;
-
-		/* Check stop command response */
-		if (brq->stop.resp[0] & R1_ERROR) {
-			pr_err("%s: %s: general error sending stop command, stop cmd response %#x\n",
-			       req->rq_disk->disk_name, __func__,
-			       brq->stop.resp[0]);
-			gen_err = true;
-		}
-
-		err = card_busy_detect(card, MMC_BLK_TIMEOUT_MS, false, req,
-					&gen_err);
-		if (err)
-			return MMC_BLK_CMD_ERR;
-	}
-
-	/* if general error occurs, retry the write operation. */
-	if (gen_err) {
-		pr_warn("%s: retrying write for general error\n",
-				req->rq_disk->disk_name);
-		return MMC_BLK_RETRY;
-	}
-
-	/* Some errors (ECC) are flagged on the next commmand, so check stop, too */
-	if (brq->data.error || brq->stop.error) {
-		if (need_retune && !brq->retune_retry_done) {
-			pr_debug("%s: retrying because a re-tune was needed\n",
-				 req->rq_disk->disk_name);
-			brq->retune_retry_done = 1;
-			return MMC_BLK_RETRY;
-		}
-		pr_err("%s: error %d transferring data, sector %u, nr %u, cmd response %#x, card status %#x\n",
-		       req->rq_disk->disk_name, brq->data.error ?: brq->stop.error,
-		       (unsigned)blk_rq_pos(req),
-		       (unsigned)blk_rq_sectors(req),
-		       brq->cmd.resp[0], brq->stop.resp[0]);
-
-		if (rq_data_dir(req) == READ) {
-			if (ecc_err)
-				return MMC_BLK_ECC_ERR;
-			return MMC_BLK_DATA_ERR;
-		} else {
-			return MMC_BLK_CMD_ERR;
-		}
-	}
-
-	if (!brq->data.bytes_xfered)
-		return MMC_BLK_RETRY;
-
-	if (blk_rq_bytes(req) != brq->data.bytes_xfered)
-		return MMC_BLK_PARTIAL;
-
-	return MMC_BLK_SUCCESS;
-}
-
 static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,
 			      int disable_multi, bool *do_rel_wr_p,
 			      bool *do_data_tag_p)
@@ -1706,8 +1440,6 @@ static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,
 		brq->data.sg_len = i;
 	}
 
-	mqrq->areq.mrq = &brq->mrq;
-
 	if (do_rel_wr_p)
 		*do_rel_wr_p = do_rel_wr;
 
@@ -1715,6 +1447,138 @@ static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,
 		*do_data_tag_p = do_data_tag;
 }
 
+#define MMC_CQE_RETRIES 2
+
+static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_request *mrq = &mqrq->brq.mrq;
+	struct request_queue *q = req->q;
+	struct mmc_host *host = mq->card->host;
+	unsigned long flags;
+	bool put_card;
+	int err;
+
+	mmc_cqe_post_req(host, mrq);
+
+	if (mrq->cmd && mrq->cmd->error)
+		err = mrq->cmd->error;
+	else if (mrq->data && mrq->data->error)
+		err = mrq->data->error;
+	else
+		err = 0;
+
+	if (err) {
+		if (mqrq->retries++ < MMC_CQE_RETRIES)
+			blk_mq_requeue_request(req, true);
+		else
+			blk_mq_end_request(req, BLK_STS_IOERR);
+	} else if (mrq->data) {
+		if (blk_update_request(req, BLK_STS_OK, mrq->data->bytes_xfered))
+			blk_mq_requeue_request(req, true);
+		else
+			__blk_mq_end_request(req, BLK_STS_OK);
+	} else {
+		blk_mq_end_request(req, BLK_STS_OK);
+	}
+
+	spin_lock_irqsave(q->queue_lock, flags);
+
+	mq->in_flight[mmc_issue_type(mq, req)] -= 1;
+
+	put_card = (mmc_tot_in_flight(mq) == 0);
+
+	mmc_cqe_check_busy(mq);
+
+	spin_unlock_irqrestore(q->queue_lock, flags);
+
+	if (!mq->cqe_busy)
+		blk_mq_run_hw_queues(q, true);
+
+	if (put_card)
+		mmc_put_card(mq->card, &mq->ctx);
+}
+
+void mmc_blk_cqe_recovery(struct mmc_queue *mq)
+{
+	struct mmc_card *card = mq->card;
+	struct mmc_host *host = card->host;
+	int err;
+
+	pr_debug("%s: CQE recovery start\n", mmc_hostname(host));
+
+	err = mmc_cqe_recovery(host);
+	if (err)
+		mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);
+	else
+		mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);
+
+	pr_debug("%s: CQE recovery done\n", mmc_hostname(host));
+}
+
+static void mmc_blk_cqe_req_done(struct mmc_request *mrq)
+{
+	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
+						  brq.mrq);
+	struct request *req = mmc_queue_req_to_req(mqrq);
+	struct request_queue *q = req->q;
+	struct mmc_queue *mq = q->queuedata;
+
+	/*
+	 * Block layer timeouts race with completions which means the normal
+	 * completion path cannot be used during recovery.
+	 */
+	if (mq->in_recovery)
+		mmc_blk_cqe_complete_rq(mq, req);
+	else
+		blk_mq_complete_request(req);
+}
+
+static int mmc_blk_cqe_start_req(struct mmc_host *host, struct mmc_request *mrq)
+{
+	mrq->done		= mmc_blk_cqe_req_done;
+	mrq->recovery_notifier	= mmc_cqe_recovery_notifier;
+
+	return mmc_cqe_start_req(host, mrq);
+}
+
+static struct mmc_request *mmc_blk_cqe_prep_dcmd(struct mmc_queue_req *mqrq,
+						 struct request *req)
+{
+	struct mmc_blk_request *brq = &mqrq->brq;
+
+	memset(brq, 0, sizeof(*brq));
+
+	brq->mrq.cmd = &brq->cmd;
+	brq->mrq.tag = req->tag;
+
+	return &brq->mrq;
+}
+
+static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_request *mrq = mmc_blk_cqe_prep_dcmd(mqrq, req);
+
+	mrq->cmd->opcode = MMC_SWITCH;
+	mrq->cmd->arg = (MMC_SWITCH_MODE_WRITE_BYTE << 24) |
+			(EXT_CSD_FLUSH_CACHE << 16) |
+			(1 << 8) |
+			EXT_CSD_CMD_SET_NORMAL;
+	mrq->cmd->flags = MMC_CMD_AC | MMC_RSP_R1B;
+
+	return mmc_blk_cqe_start_req(mq->card->host, mrq);
+}
+
+static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+
+	mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);
+
+	return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);
+}
+
 static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
 			       struct mmc_card *card,
 			       int disable_multi,
@@ -1779,318 +1643,637 @@ static void mmc_blk_rw_rq_prep(struct mmc_queue_req *mqrq,
 		brq->sbc.flags = MMC_RSP_R1 | MMC_CMD_AC;
 		brq->mrq.sbc = &brq->sbc;
 	}
+}
 
-	mqrq->areq.err_check = mmc_blk_err_check;
+#define MMC_MAX_RETRIES		5
+#define MMC_DATA_RETRIES	2
+#define MMC_NO_RETRIES		(MMC_MAX_RETRIES + 1)
+
+static int mmc_blk_send_stop(struct mmc_card *card, unsigned int timeout)
+{
+	struct mmc_command cmd = {
+		.opcode = MMC_STOP_TRANSMISSION,
+		.flags = MMC_RSP_SPI_R1 | MMC_RSP_R1 | MMC_CMD_AC,
+		/* Some hosts wait for busy anyway, so provide a busy timeout */
+		.busy_timeout = timeout,
+	};
+
+	return mmc_wait_for_cmd(card->host, &cmd, 5);
 }
 
-static bool mmc_blk_rw_cmd_err(struct mmc_blk_data *md, struct mmc_card *card,
-			       struct mmc_blk_request *brq, struct request *req,
-			       bool old_req_pending)
+static int mmc_blk_fix_state(struct mmc_card *card, struct request *req)
 {
-	bool req_pending;
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_blk_request *brq = &mqrq->brq;
+	unsigned int timeout = mmc_blk_data_timeout_ms(card->host, &brq->data);
+	int err;
 
-	/*
-	 * If this is an SD card and we're writing, we can first
-	 * mark the known good sectors as ok.
-	 *
-	 * If the card is not SD, we can still ok written sectors
-	 * as reported by the controller (which might be less than
-	 * the real number of written sectors, but never more).
-	 */
-	if (mmc_card_sd(card)) {
-		u32 blocks;
+	mmc_retune_hold_now(card->host);
+
+	mmc_blk_send_stop(card, timeout);
+
+	err = card_busy_detect(card, timeout, req, NULL);
+
+	mmc_retune_release(card->host);
+
+	return err;
+}
+
+#define MMC_READ_SINGLE_RETRIES	2
+
+/* Single sector read during recovery */
+static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_request *mrq = &mqrq->brq.mrq;
+	struct mmc_card *card = mq->card;
+	struct mmc_host *host = card->host;
+	blk_status_t error = BLK_STS_OK;
+	int retries = 0;
+
+	do {
+		u32 status;
 		int err;
 
-		err = mmc_sd_num_wr_blocks(card, &blocks);
+		mmc_blk_rw_rq_prep(mqrq, card, 1, mq);
+
+		mmc_wait_for_req(host, mrq);
+
+		err = mmc_send_status(card, &status);
 		if (err)
-			req_pending = old_req_pending;
+			goto error_exit;
+
+		if (!mmc_host_is_spi(host) &&
+		    !mmc_blk_in_tran_state(status)) {
+			err = mmc_blk_fix_state(card, req);
+			if (err)
+				goto error_exit;
+		}
+
+		if (mrq->cmd->error && retries++ < MMC_READ_SINGLE_RETRIES)
+			continue;
+
+		retries = 0;
+
+		if (mrq->cmd->error ||
+		    mrq->data->error ||
+		    (!mmc_host_is_spi(host) &&
+		     (mrq->cmd->resp[0] & CMD_ERRORS || status & CMD_ERRORS)))
+			error = BLK_STS_IOERR;
 		else
-			req_pending = blk_end_request(req, BLK_STS_OK, blocks << 9);
-	} else {
-		req_pending = blk_end_request(req, BLK_STS_OK, brq->data.bytes_xfered);
-	}
-	return req_pending;
+			error = BLK_STS_OK;
+
+	} while (blk_update_request(req, error, 512));
+
+	return;
+
+error_exit:
+	mrq->data->bytes_xfered = 0;
+	blk_update_request(req, BLK_STS_IOERR, 512);
+	/* Let it try the remaining request again */
+	if (mqrq->retries > MMC_MAX_RETRIES - 1)
+		mqrq->retries = MMC_MAX_RETRIES - 1;
 }
 
-static void mmc_blk_rw_cmd_abort(struct mmc_queue *mq, struct mmc_card *card,
-				 struct request *req,
-				 struct mmc_queue_req *mqrq)
+static inline bool mmc_blk_oor_valid(struct mmc_blk_request *brq)
 {
-	if (mmc_card_removed(card))
-		req->rq_flags |= RQF_QUIET;
-	while (blk_end_request(req, BLK_STS_IOERR, blk_rq_cur_bytes(req)));
-	mq->qcnt--;
+	return !!brq->mrq.sbc;
 }
 
-/**
- * mmc_blk_rw_try_restart() - tries to restart the current async request
- * @mq: the queue with the card and host to restart
- * @req: a new request that want to be started after the current one
+static inline u32 mmc_blk_stop_err_bits(struct mmc_blk_request *brq)
+{
+	return mmc_blk_oor_valid(brq) ? CMD_ERRORS : CMD_ERRORS_EXCL_OOR;
+}
+
+/*
+ * Check for errors the host controller driver might not have seen such as
+ * response mode errors or invalid card state.
+ */
+static bool mmc_blk_status_error(struct request *req, u32 status)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_blk_request *brq = &mqrq->brq;
+	struct mmc_queue *mq = req->q->queuedata;
+	u32 stop_err_bits;
+
+	if (mmc_host_is_spi(mq->card->host))
+		return false;
+
+	stop_err_bits = mmc_blk_stop_err_bits(brq);
+
+	return brq->cmd.resp[0]  & CMD_ERRORS    ||
+	       brq->stop.resp[0] & stop_err_bits ||
+	       status            & stop_err_bits ||
+	       (rq_data_dir(req) == WRITE && !mmc_blk_in_tran_state(status));
+}
+
+static inline bool mmc_blk_cmd_started(struct mmc_blk_request *brq)
+{
+	return !brq->sbc.error && !brq->cmd.error &&
+	       !(brq->cmd.resp[0] & CMD_ERRORS);
+}
+
+/*
+ * Requests are completed by mmc_blk_mq_complete_rq() which sets simple
+ * policy:
+ * 1. A request that has transferred at least some data is considered
+ * successful and will be requeued if there is remaining data to
+ * transfer.
+ * 2. Otherwise the number of retries is incremented and the request
+ * will be requeued if there are remaining retries.
+ * 3. Otherwise the request will be errored out.
+ * That means mmc_blk_mq_complete_rq() is controlled by bytes_xfered and
+ * mqrq->retries. So there are only 4 possible actions here:
+ *	1. do not accept the bytes_xfered value i.e. set it to zero
+ *	2. change mqrq->retries to determine the number of retries
+ *	3. try to reset the card
+ *	4. read one sector at a time
  */
-static void mmc_blk_rw_try_restart(struct mmc_queue *mq, struct request *req,
-				   struct mmc_queue_req *mqrq)
+static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req)
 {
-	if (!req)
+	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_blk_request *brq = &mqrq->brq;
+	struct mmc_blk_data *md = mq->blkdata;
+	struct mmc_card *card = mq->card;
+	u32 status;
+	u32 blocks;
+	int err;
+
+	/*
+	 * Some errors the host driver might not have seen. Set the number of
+	 * bytes transferred to zero in that case.
+	 */
+	err = __mmc_send_status(card, &status, 0);
+	if (err || mmc_blk_status_error(req, status))
+		brq->data.bytes_xfered = 0;
+
+	mmc_retune_release(card->host);
+
+	/*
+	 * Try again to get the status. This also provides an opportunity for
+	 * re-tuning.
+	 */
+	if (err)
+		err = __mmc_send_status(card, &status, 0);
+
+	/*
+	 * Nothing more to do after the number of bytes transferred has been
+	 * updated and there is no card.
+	 */
+	if (err && mmc_detect_card_removed(card->host))
 		return;
 
+	/* Try to get back to "tran" state */
+	if (!mmc_host_is_spi(mq->card->host) &&
+	    (err || !mmc_blk_in_tran_state(status)))
+		err = mmc_blk_fix_state(mq->card, req);
+
 	/*
-	 * If the card was removed, just cancel everything and return.
+	 * Special case for SD cards where the card might record the number of
+	 * blocks written.
 	 */
-	if (mmc_card_removed(mq->card)) {
-		req->rq_flags |= RQF_QUIET;
-		blk_end_request_all(req, BLK_STS_IOERR);
-		mq->qcnt--; /* FIXME: just set to 0? */
+	if (!err && mmc_blk_cmd_started(brq) && mmc_card_sd(card) &&
+	    rq_data_dir(req) == WRITE) {
+		if (mmc_sd_num_wr_blocks(card, &blocks))
+			brq->data.bytes_xfered = 0;
+		else
+			brq->data.bytes_xfered = blocks << 9;
+	}
+
+	/* Reset if the card is in a bad state */
+	if (!mmc_host_is_spi(mq->card->host) &&
+	    err && mmc_blk_reset(md, card->host, type)) {
+		pr_err("%s: recovery failed!\n", req->rq_disk->disk_name);
+		mqrq->retries = MMC_NO_RETRIES;
+		return;
+	}
+
+	/*
+	 * If anything was done, just return and if there is anything remaining
+	 * on the request it will get requeued.
+	 */
+	if (brq->data.bytes_xfered)
+		return;
+
+	/* Reset before last retry */
+	if (mqrq->retries + 1 == MMC_MAX_RETRIES)
+		mmc_blk_reset(md, card->host, type);
+
+	/* Command errors fail fast, so use all MMC_MAX_RETRIES */
+	if (brq->sbc.error || brq->cmd.error)
+		return;
+
+	/* Reduce the remaining retries for data errors */
+	if (mqrq->retries < MMC_MAX_RETRIES - MMC_DATA_RETRIES) {
+		mqrq->retries = MMC_MAX_RETRIES - MMC_DATA_RETRIES;
+		return;
+	}
+
+	/* FIXME: Missing single sector read for large sector size */
+	if (!mmc_large_sector(card) && rq_data_dir(req) == READ &&
+	    brq->data.blocks > 1) {
+		/* Read one sector at a time */
+		mmc_blk_read_single(mq, req);
 		return;
 	}
-	/* Else proceed and try to restart the current async request */
-	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
-	mmc_start_areq(mq->card->host, &mqrq->areq, NULL);
 }
 
-static void mmc_blk_issue_rw_rq(struct mmc_queue *mq, struct request *new_req)
+static inline bool mmc_blk_rq_error(struct mmc_blk_request *brq)
 {
-	struct mmc_blk_data *md = mq->blkdata;
-	struct mmc_card *card = md->queue.card;
-	struct mmc_blk_request *brq;
-	int disable_multi = 0, retry = 0, type, retune_retry_done = 0;
-	enum mmc_blk_status status;
-	struct mmc_queue_req *mqrq_cur = NULL;
-	struct mmc_queue_req *mq_rq;
-	struct request *old_req;
-	struct mmc_async_req *new_areq;
-	struct mmc_async_req *old_areq;
-	bool req_pending = true;
+	mmc_blk_eval_resp_error(brq);
+
+	return brq->sbc.error || brq->cmd.error || brq->stop.error ||
+	       brq->data.error || brq->cmd.resp[0] & CMD_ERRORS;
+}
+
+static int mmc_blk_card_busy(struct mmc_card *card, struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	u32 status = 0;
+	int err;
+
+	if (mmc_host_is_spi(card->host) || rq_data_dir(req) == READ)
+		return 0;
+
+	err = card_busy_detect(card, MMC_BLK_TIMEOUT_MS, req, &status);
+
+	/*
+	 * Do not assume data transferred correctly if there are any error bits
+	 * set.
+	 */
+	if (status & mmc_blk_stop_err_bits(&mqrq->brq)) {
+		mqrq->brq.data.bytes_xfered = 0;
+		err = err ? err : -EIO;
+	}
+
+	/* Copy the exception bit so it will be seen later on */
+	if (mmc_card_mmc(card) && status & R1_EXCEPTION_EVENT)
+		mqrq->brq.cmd.resp[0] |= R1_EXCEPTION_EVENT;
 
-	if (new_req) {
-		mqrq_cur = req_to_mmc_queue_req(new_req);
-		mq->qcnt++;
+	return err;
+}
+
+static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,
+					    struct request *req)
+{
+	int type = rq_data_dir(req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
+
+	mmc_blk_reset_success(mq->blkdata, type);
+}
+
+static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	unsigned int nr_bytes = mqrq->brq.data.bytes_xfered;
+
+	if (nr_bytes) {
+		if (blk_update_request(req, BLK_STS_OK, nr_bytes))
+			blk_mq_requeue_request(req, true);
+		else
+			__blk_mq_end_request(req, BLK_STS_OK);
+	} else if (!blk_rq_bytes(req)) {
+		__blk_mq_end_request(req, BLK_STS_IOERR);
+	} else if (mqrq->retries++ < MMC_MAX_RETRIES) {
+		blk_mq_requeue_request(req, true);
+	} else {
+		if (mmc_card_removed(mq->card))
+			req->rq_flags |= RQF_QUIET;
+		blk_mq_end_request(req, BLK_STS_IOERR);
+	}
+}
+
+static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,
+					struct mmc_queue_req *mqrq)
+{
+	return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&
+	       (mqrq->brq.cmd.resp[0] & R1_EXCEPTION_EVENT ||
+		mqrq->brq.stop.resp[0] & R1_EXCEPTION_EVENT);
+}
+
+static void mmc_blk_urgent_bkops(struct mmc_queue *mq,
+				 struct mmc_queue_req *mqrq)
+{
+	if (mmc_blk_urgent_bkops_needed(mq, mqrq))
+		mmc_start_bkops(mq->card, true);
+}
+
+void mmc_blk_mq_complete(struct request *req)
+{
+	struct mmc_queue *mq = req->q->queuedata;
+
+	if (mq->use_cqe)
+		mmc_blk_cqe_complete_rq(mq, req);
+	else
+		mmc_blk_mq_complete_rq(mq, req);
+}
+
+static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,
+				       struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_host *host = mq->card->host;
+
+	if (mmc_blk_rq_error(&mqrq->brq) ||
+	    mmc_blk_card_busy(mq->card, req)) {
+		mmc_blk_mq_rw_recovery(mq, req);
+	} else {
+		mmc_blk_rw_reset_success(mq, req);
+		mmc_retune_release(host);
 	}
 
-	if (!mq->qcnt)
+	mmc_blk_urgent_bkops(mq, mqrq);
+}
+
+static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, struct request *req)
+{
+	struct request_queue *q = req->q;
+	unsigned long flags;
+	bool put_card;
+
+	spin_lock_irqsave(q->queue_lock, flags);
+
+	mq->in_flight[mmc_issue_type(mq, req)] -= 1;
+
+	put_card = (mmc_tot_in_flight(mq) == 0);
+
+	spin_unlock_irqrestore(q->queue_lock, flags);
+
+	if (put_card)
+		mmc_put_card(mq->card, &mq->ctx);
+}
+
+static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_request *mrq = &mqrq->brq.mrq;
+	struct mmc_host *host = mq->card->host;
+
+	mmc_post_req(host, mrq, 0);
+
+	/*
+	 * Block layer timeouts race with completions which means the normal
+	 * completion path cannot be used during recovery.
+	 */
+	if (mq->in_recovery)
+		mmc_blk_mq_complete_rq(mq, req);
+	else
+		blk_mq_complete_request(req);
+
+	mmc_blk_mq_dec_in_flight(mq, req);
+}
+
+void mmc_blk_mq_recovery(struct mmc_queue *mq)
+{
+	struct request *req = mq->recovery_req;
+	struct mmc_host *host = mq->card->host;
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+
+	mq->recovery_req = NULL;
+	mq->rw_wait = false;
+
+	if (mmc_blk_rq_error(&mqrq->brq)) {
+		mmc_retune_hold_now(host);
+		mmc_blk_mq_rw_recovery(mq, req);
+	}
+
+	mmc_blk_urgent_bkops(mq, mqrq);
+
+	mmc_blk_mq_post_req(mq, req);
+}
+
+static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,
+					 struct request **prev_req)
+{
+	if (mmc_host_done_complete(mq->card->host))
 		return;
 
-	do {
-		if (new_req) {
-			/*
-			 * When 4KB native sector is enabled, only 8 blocks
-			 * multiple read or write is allowed
-			 */
-			if (mmc_large_sector(card) &&
-				!IS_ALIGNED(blk_rq_sectors(new_req), 8)) {
-				pr_err("%s: Transfer size is not 4KB sector size aligned\n",
-					new_req->rq_disk->disk_name);
-				mmc_blk_rw_cmd_abort(mq, card, new_req, mqrq_cur);
-				return;
-			}
+	mutex_lock(&mq->complete_lock);
 
-			mmc_blk_rw_rq_prep(mqrq_cur, card, 0, mq);
-			new_areq = &mqrq_cur->areq;
-		} else
-			new_areq = NULL;
+	if (!mq->complete_req)
+		goto out_unlock;
 
-		old_areq = mmc_start_areq(card->host, new_areq, &status);
-		if (!old_areq) {
-			/*
-			 * We have just put the first request into the pipeline
-			 * and there is nothing more to do until it is
-			 * complete.
-			 */
-			return;
-		}
+	mmc_blk_mq_poll_completion(mq, mq->complete_req);
+
+	if (prev_req)
+		*prev_req = mq->complete_req;
+	else
+		mmc_blk_mq_post_req(mq, mq->complete_req);
+
+	mq->complete_req = NULL;
+
+out_unlock:
+	mutex_unlock(&mq->complete_lock);
+}
+
+void mmc_blk_mq_complete_work(struct work_struct *work)
+{
+	struct mmc_queue *mq = container_of(work, struct mmc_queue,
+					    complete_work);
+
+	mmc_blk_mq_complete_prev_req(mq, NULL);
+}
+
+static void mmc_blk_mq_req_done(struct mmc_request *mrq)
+{
+	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
+						  brq.mrq);
+	struct request *req = mmc_queue_req_to_req(mqrq);
+	struct request_queue *q = req->q;
+	struct mmc_queue *mq = q->queuedata;
+	struct mmc_host *host = mq->card->host;
+	unsigned long flags;
+
+	if (!mmc_host_done_complete(host)) {
+		bool waiting;
 
 		/*
-		 * An asynchronous request has been completed and we proceed
-		 * to handle the result of it.
+		 * We cannot complete the request in this context, so record
+		 * that there is a request to complete, and that a following
+		 * request does not need to wait (although it does need to
+		 * complete complete_req first).
 		 */
-		mq_rq =	container_of(old_areq, struct mmc_queue_req, areq);
-		brq = &mq_rq->brq;
-		old_req = mmc_queue_req_to_req(mq_rq);
-		type = rq_data_dir(old_req) == READ ? MMC_BLK_READ : MMC_BLK_WRITE;
-
-		switch (status) {
-		case MMC_BLK_SUCCESS:
-		case MMC_BLK_PARTIAL:
-			/*
-			 * A block was successfully transferred.
-			 */
-			mmc_blk_reset_success(md, type);
+		spin_lock_irqsave(q->queue_lock, flags);
+		mq->complete_req = req;
+		mq->rw_wait = false;
+		waiting = mq->waiting;
+		spin_unlock_irqrestore(q->queue_lock, flags);
 
-			req_pending = blk_end_request(old_req, BLK_STS_OK,
-						      brq->data.bytes_xfered);
-			/*
-			 * If the blk_end_request function returns non-zero even
-			 * though all data has been transferred and no errors
-			 * were returned by the host controller, it's a bug.
-			 */
-			if (status == MMC_BLK_SUCCESS && req_pending) {
-				pr_err("%s BUG rq_tot %d d_xfer %d\n",
-				       __func__, blk_rq_bytes(old_req),
-				       brq->data.bytes_xfered);
-				mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
-				return;
-			}
-			break;
-		case MMC_BLK_CMD_ERR:
-			req_pending = mmc_blk_rw_cmd_err(md, card, brq, old_req, req_pending);
-			if (mmc_blk_reset(md, card->host, type)) {
-				if (req_pending)
-					mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
-				else
-					mq->qcnt--;
-				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
-				return;
-			}
-			if (!req_pending) {
-				mq->qcnt--;
-				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
-				return;
-			}
-			break;
-		case MMC_BLK_RETRY:
-			retune_retry_done = brq->retune_retry_done;
-			if (retry++ < 5)
-				break;
-			/* Fall through */
-		case MMC_BLK_ABORT:
-			if (!mmc_blk_reset(md, card->host, type))
-				break;
-			mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
-			mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
-			return;
-		case MMC_BLK_DATA_ERR: {
-			int err;
-
-			err = mmc_blk_reset(md, card->host, type);
-			if (!err)
-				break;
-			if (err == -ENODEV) {
-				mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
-				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
-				return;
-			}
-			/* Fall through */
-		}
-		case MMC_BLK_ECC_ERR:
-			if (brq->data.blocks > 1) {
-				/* Redo read one sector at a time */
-				pr_warn("%s: retrying using single block read\n",
-					old_req->rq_disk->disk_name);
-				disable_multi = 1;
-				break;
-			}
-			/*
-			 * After an error, we redo I/O one sector at a
-			 * time, so we only reach here after trying to
-			 * read a single sector.
-			 */
-			req_pending = blk_end_request(old_req, BLK_STS_IOERR,
-						      brq->data.blksz);
-			if (!req_pending) {
-				mq->qcnt--;
-				mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
-				return;
-			}
-			break;
-		case MMC_BLK_NOMEDIUM:
-			mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
-			mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
-			return;
-		default:
-			pr_err("%s: Unhandled return value (%d)",
-					old_req->rq_disk->disk_name, status);
-			mmc_blk_rw_cmd_abort(mq, card, old_req, mq_rq);
-			mmc_blk_rw_try_restart(mq, new_req, mqrq_cur);
-			return;
-		}
+		/*
+		 * If 'waiting' then the waiting task will complete this
+		 * request, otherwise queue a work to do it. Note that
+		 * complete_work may still race with the dispatch of a following
+		 * request.
+		 */
+		if (waiting)
+			wake_up(&mq->wait);
+		else
+			kblockd_schedule_work(&mq->complete_work);
 
-		if (req_pending) {
-			/*
-			 * In case of a incomplete request
-			 * prepare it again and resend.
-			 */
-			mmc_blk_rw_rq_prep(mq_rq, card,
-					disable_multi, mq);
-			mmc_start_areq(card->host,
-					&mq_rq->areq, NULL);
-			mq_rq->brq.retune_retry_done = retune_retry_done;
-		}
-	} while (req_pending);
+		return;
+	}
+
+	/* Take the recovery path for errors or urgent background operations */
+	if (mmc_blk_rq_error(&mqrq->brq) ||
+	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {
+		spin_lock_irqsave(q->queue_lock, flags);
+		mq->recovery_needed = true;
+		mq->recovery_req = req;
+		spin_unlock_irqrestore(q->queue_lock, flags);
+		wake_up(&mq->wait);
+		schedule_work(&mq->recovery_work);
+		return;
+	}
 
-	mq->qcnt--;
+	mmc_blk_rw_reset_success(mq, req);
+
+	mq->rw_wait = false;
+	wake_up(&mq->wait);
+
+	mmc_blk_mq_post_req(mq, req);
 }
 
-void mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req)
+static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err)
+{
+	struct request_queue *q = mq->queue;
+	unsigned long flags;
+	bool done;
+
+	/*
+	 * Wait while there is another request in progress, but not if recovery
+	 * is needed. Also indicate whether there is a request waiting to start.
+	 */
+	spin_lock_irqsave(q->queue_lock, flags);
+	if (mq->recovery_needed) {
+		*err = -EBUSY;
+		done = true;
+	} else {
+		done = !mq->rw_wait;
+	}
+	mq->waiting = !done;
+	spin_unlock_irqrestore(q->queue_lock, flags);
+
+	return done;
+}
+
+static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req)
+{
+	int err = 0;
+
+	wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));
+
+	/* Always complete the previous request if there is one */
+	mmc_blk_mq_complete_prev_req(mq, prev_req);
+
+	return err;
+}
+
+static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,
+				  struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_host *host = mq->card->host;
+	struct request *prev_req = NULL;
+	int err = 0;
+
+	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);
+
+	mqrq->brq.mrq.done = mmc_blk_mq_req_done;
+
+	mmc_pre_req(host, &mqrq->brq.mrq);
+
+	err = mmc_blk_rw_wait(mq, &prev_req);
+	if (err)
+		goto out_post_req;
+
+	mq->rw_wait = true;
+
+	err = mmc_start_request(host, &mqrq->brq.mrq);
+
+	if (prev_req)
+		mmc_blk_mq_post_req(mq, prev_req);
+
+	if (err)
+		mq->rw_wait = false;
+
+	/* Release re-tuning here where there is no synchronization required */
+	if (err || mmc_host_done_complete(host))
+		mmc_retune_release(host);
+
+out_post_req:
+	if (err)
+		mmc_post_req(host, &mqrq->brq.mrq, err);
+
+	return err;
+}
+
+static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host)
+{
+	if (mq->use_cqe)
+		return host->cqe_ops->cqe_wait_for_idle(host);
+
+	return mmc_blk_rw_wait(mq, NULL);
+}
+
+enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req)
 {
-	int ret;
 	struct mmc_blk_data *md = mq->blkdata;
 	struct mmc_card *card = md->queue.card;
-
-	if (req && !mq->qcnt)
-		/* claim host only for the first request */
-		mmc_get_card(card, NULL);
+	struct mmc_host *host = card->host;
+	int ret;
 
 	ret = mmc_blk_part_switch(card, md->part_type);
-	if (ret) {
-		if (req) {
-			blk_end_request_all(req, BLK_STS_IOERR);
-		}
-		goto out;
-	}
+	if (ret)
+		return MMC_REQ_FAILED_TO_START;
 
-	if (req) {
+	switch (mmc_issue_type(mq, req)) {
+	case MMC_ISSUE_SYNC:
+		ret = mmc_blk_wait_for_idle(mq, host);
+		if (ret)
+			return MMC_REQ_BUSY;
 		switch (req_op(req)) {
 		case REQ_OP_DRV_IN:
 		case REQ_OP_DRV_OUT:
-			/*
-			 * Complete ongoing async transfer before issuing
-			 * ioctl()s
-			 */
-			if (mq->qcnt)
-				mmc_blk_issue_rw_rq(mq, NULL);
 			mmc_blk_issue_drv_op(mq, req);
 			break;
 		case REQ_OP_DISCARD:
-			/*
-			 * Complete ongoing async transfer before issuing
-			 * discard.
-			 */
-			if (mq->qcnt)
-				mmc_blk_issue_rw_rq(mq, NULL);
 			mmc_blk_issue_discard_rq(mq, req);
 			break;
 		case REQ_OP_SECURE_ERASE:
-			/*
-			 * Complete ongoing async transfer before issuing
-			 * secure erase.
-			 */
-			if (mq->qcnt)
-				mmc_blk_issue_rw_rq(mq, NULL);
 			mmc_blk_issue_secdiscard_rq(mq, req);
 			break;
 		case REQ_OP_FLUSH:
-			/*
-			 * Complete ongoing async transfer before issuing
-			 * flush.
-			 */
-			if (mq->qcnt)
-				mmc_blk_issue_rw_rq(mq, NULL);
 			mmc_blk_issue_flush(mq, req);
 			break;
 		default:
-			/* Normal request, just issue it */
-			mmc_blk_issue_rw_rq(mq, req);
-			card->host->context_info.is_waiting_last_req = false;
+			WARN_ON_ONCE(1);
+			return MMC_REQ_FAILED_TO_START;
+		}
+		return MMC_REQ_FINISHED;
+	case MMC_ISSUE_DCMD:
+	case MMC_ISSUE_ASYNC:
+		switch (req_op(req)) {
+		case REQ_OP_FLUSH:
+			ret = mmc_blk_cqe_issue_flush(mq, req);
 			break;
+		case REQ_OP_READ:
+		case REQ_OP_WRITE:
+			if (mq->use_cqe)
+				ret = mmc_blk_cqe_issue_rw_rq(mq, req);
+			else
+				ret = mmc_blk_mq_issue_rw_rq(mq, req);
+			break;
+		default:
+			WARN_ON_ONCE(1);
+			ret = -EINVAL;
 		}
-	} else {
-		/* No request, flushing the pipeline with NULL */
-		mmc_blk_issue_rw_rq(mq, NULL);
-		card->host->context_info.is_waiting_last_req = false;
+		if (!ret)
+			return MMC_REQ_STARTED;
+		return ret == -EBUSY ? MMC_REQ_BUSY : MMC_REQ_FAILED_TO_START;
+	default:
+		WARN_ON_ONCE(1);
+		return MMC_REQ_FAILED_TO_START;
 	}
-
-out:
-	if (!mq->qcnt)
-		mmc_put_card(card, NULL);
 }
 
 static inline int mmc_blk_readonly(struct mmc_card *card)
@@ -2156,6 +2339,18 @@ static struct mmc_blk_data *mmc_blk_alloc_req(struct mmc_card *card,
 
 	md->queue.blkdata = md;
 
+	/*
+	 * Keep an extra reference to the queue so that we can shutdown the
+	 * queue (i.e. call blk_cleanup_queue()) while there are still
+	 * references to the 'md'. The corresponding blk_put_queue() is in
+	 * mmc_blk_put().
+	 */
+	if (!blk_get_queue(md->queue.queue)) {
+		mmc_cleanup_queue(&md->queue);
+		ret = -ENODEV;
+		goto err_putdisk;
+	}
+
 	md->disk->major	= MMC_BLOCK_MAJOR;
 	md->disk->first_minor = devidx * perdev_minors;
 	md->disk->fops = &mmc_bdops;
@@ -2471,10 +2666,6 @@ static void mmc_blk_remove_req(struct mmc_blk_data *md)
 		 * from being accepted.
 		 */
 		card = md->queue.card;
-		spin_lock_irq(md->queue.queue->queue_lock);
-		queue_flag_set(QUEUE_FLAG_BYPASS, md->queue.queue);
-		spin_unlock_irq(md->queue.queue->queue_lock);
-		blk_set_queue_dying(md->queue.queue);
 		mmc_cleanup_queue(&md->queue);
 		if (md->disk->flags & GENHD_FL_UP) {
 			device_remove_file(disk_to_dev(md->disk), &md->force_ro);
@@ -2623,6 +2814,7 @@ static int mmc_ext_csd_open(struct inode *inode, struct file *filp)
 
 	if (n != EXT_CSD_STR_LEN) {
 		err = -EINVAL;
+		kfree(ext_csd);
 		goto out_free;
 	}
 
diff --git a/drivers/mmc/core/block.h b/drivers/mmc/core/block.h
index 5946636101ef..31153f656f41 100644
--- a/drivers/mmc/core/block.h
+++ b/drivers/mmc/core/block.h
@@ -5,6 +5,16 @@
 struct mmc_queue;
 struct request;
 
-void mmc_blk_issue_rq(struct mmc_queue *mq, struct request *req);
+void mmc_blk_cqe_recovery(struct mmc_queue *mq);
+
+enum mmc_issued;
+
+enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req);
+void mmc_blk_mq_complete(struct request *req);
+void mmc_blk_mq_recovery(struct mmc_queue *mq);
+
+struct work_struct;
+
+void mmc_blk_mq_complete_work(struct work_struct *work);
 
 #endif
diff --git a/drivers/mmc/core/bus.c b/drivers/mmc/core/bus.c
index 7586ff2ad1f1..fc92c6c1c9a4 100644
--- a/drivers/mmc/core/bus.c
+++ b/drivers/mmc/core/bus.c
@@ -351,8 +351,6 @@ int mmc_add_card(struct mmc_card *card)
 #ifdef CONFIG_DEBUG_FS
 	mmc_add_card_debugfs(card);
 #endif
-	mmc_init_context_info(card->host);
-
 	card->dev.of_node = mmc_of_find_child_device(card->host, 0);
 
 	device_enable_async_suspend(&card->dev);
diff --git a/drivers/mmc/core/card.h b/drivers/mmc/core/card.h
index 79a5b985ccf5..9c821eedd156 100644
--- a/drivers/mmc/core/card.h
+++ b/drivers/mmc/core/card.h
@@ -82,6 +82,7 @@ struct mmc_fixup {
 #define CID_MANFID_APACER       0x27
 #define CID_MANFID_KINGSTON     0x70
 #define CID_MANFID_HYNIX	0x90
+#define CID_MANFID_NUMONYX	0xFE
 
 #define END_FIXUP { NULL }
 
diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c
index 1f0f44f4dd5f..c0ba6d8823b7 100644
--- a/drivers/mmc/core/core.c
+++ b/drivers/mmc/core/core.c
@@ -341,6 +341,8 @@ int mmc_start_request(struct mmc_host *host, struct mmc_request *mrq)
 {
 	int err;
 
+	init_completion(&mrq->cmd_completion);
+
 	mmc_retune_hold(host);
 
 	if (mmc_card_removed(host->card))
@@ -361,20 +363,6 @@ int mmc_start_request(struct mmc_host *host, struct mmc_request *mrq)
 }
 EXPORT_SYMBOL(mmc_start_request);
 
-/*
- * mmc_wait_data_done() - done callback for data request
- * @mrq: done data request
- *
- * Wakes up mmc context, passed as a callback to host controller driver
- */
-static void mmc_wait_data_done(struct mmc_request *mrq)
-{
-	struct mmc_context_info *context_info = &mrq->host->context_info;
-
-	context_info->is_done_rcv = true;
-	wake_up_interruptible(&context_info->wait);
-}
-
 static void mmc_wait_done(struct mmc_request *mrq)
 {
 	complete(&mrq->completion);
@@ -392,37 +380,6 @@ static inline void mmc_wait_ongoing_tfr_cmd(struct mmc_host *host)
 		wait_for_completion(&ongoing_mrq->cmd_completion);
 }
 
-/*
- *__mmc_start_data_req() - starts data request
- * @host: MMC host to start the request
- * @mrq: data request to start
- *
- * Sets the done callback to be called when request is completed by the card.
- * Starts data mmc request execution
- * If an ongoing transfer is already in progress, wait for the command line
- * to become available before sending another command.
- */
-static int __mmc_start_data_req(struct mmc_host *host, struct mmc_request *mrq)
-{
-	int err;
-
-	mmc_wait_ongoing_tfr_cmd(host);
-
-	mrq->done = mmc_wait_data_done;
-	mrq->host = host;
-
-	init_completion(&mrq->cmd_completion);
-
-	err = mmc_start_request(host, mrq);
-	if (err) {
-		mrq->cmd->error = err;
-		mmc_complete_cmd(mrq);
-		mmc_wait_data_done(mrq);
-	}
-
-	return err;
-}
-
 static int __mmc_start_req(struct mmc_host *host, struct mmc_request *mrq)
 {
 	int err;
@@ -432,8 +389,6 @@ static int __mmc_start_req(struct mmc_host *host, struct mmc_request *mrq)
 	init_completion(&mrq->completion);
 	mrq->done = mmc_wait_done;
 
-	init_completion(&mrq->cmd_completion);
-
 	err = mmc_start_request(host, mrq);
 	if (err) {
 		mrq->cmd->error = err;
@@ -650,163 +605,10 @@ EXPORT_SYMBOL(mmc_cqe_recovery);
  */
 bool mmc_is_req_done(struct mmc_host *host, struct mmc_request *mrq)
 {
-	if (host->areq)
-		return host->context_info.is_done_rcv;
-	else
-		return completion_done(&mrq->completion);
+	return completion_done(&mrq->completion);
 }
 EXPORT_SYMBOL(mmc_is_req_done);
 
-/**
- *	mmc_pre_req - Prepare for a new request
- *	@host: MMC host to prepare command
- *	@mrq: MMC request to prepare for
- *
- *	mmc_pre_req() is called in prior to mmc_start_req() to let
- *	host prepare for the new request. Preparation of a request may be
- *	performed while another request is running on the host.
- */
-static void mmc_pre_req(struct mmc_host *host, struct mmc_request *mrq)
-{
-	if (host->ops->pre_req)
-		host->ops->pre_req(host, mrq);
-}
-
-/**
- *	mmc_post_req - Post process a completed request
- *	@host: MMC host to post process command
- *	@mrq: MMC request to post process for
- *	@err: Error, if non zero, clean up any resources made in pre_req
- *
- *	Let the host post process a completed request. Post processing of
- *	a request may be performed while another reuqest is running.
- */
-static void mmc_post_req(struct mmc_host *host, struct mmc_request *mrq,
-			 int err)
-{
-	if (host->ops->post_req)
-		host->ops->post_req(host, mrq, err);
-}
-
-/**
- * mmc_finalize_areq() - finalize an asynchronous request
- * @host: MMC host to finalize any ongoing request on
- *
- * Returns the status of the ongoing asynchronous request, but
- * MMC_BLK_SUCCESS if no request was going on.
- */
-static enum mmc_blk_status mmc_finalize_areq(struct mmc_host *host)
-{
-	struct mmc_context_info *context_info = &host->context_info;
-	enum mmc_blk_status status;
-
-	if (!host->areq)
-		return MMC_BLK_SUCCESS;
-
-	while (1) {
-		wait_event_interruptible(context_info->wait,
-				(context_info->is_done_rcv ||
-				 context_info->is_new_req));
-
-		if (context_info->is_done_rcv) {
-			struct mmc_command *cmd;
-
-			context_info->is_done_rcv = false;
-			cmd = host->areq->mrq->cmd;
-
-			if (!cmd->error || !cmd->retries ||
-			    mmc_card_removed(host->card)) {
-				status = host->areq->err_check(host->card,
-							       host->areq);
-				break; /* return status */
-			} else {
-				mmc_retune_recheck(host);
-				pr_info("%s: req failed (CMD%u): %d, retrying...\n",
-					mmc_hostname(host),
-					cmd->opcode, cmd->error);
-				cmd->retries--;
-				cmd->error = 0;
-				__mmc_start_request(host, host->areq->mrq);
-				continue; /* wait for done/new event again */
-			}
-		}
-
-		return MMC_BLK_NEW_REQUEST;
-	}
-
-	mmc_retune_release(host);
-
-	/*
-	 * Check BKOPS urgency for each R1 response
-	 */
-	if (host->card && mmc_card_mmc(host->card) &&
-	    ((mmc_resp_type(host->areq->mrq->cmd) == MMC_RSP_R1) ||
-	     (mmc_resp_type(host->areq->mrq->cmd) == MMC_RSP_R1B)) &&
-	    (host->areq->mrq->cmd->resp[0] & R1_EXCEPTION_EVENT)) {
-		mmc_start_bkops(host->card, true);
-	}
-
-	return status;
-}
-
-/**
- *	mmc_start_areq - start an asynchronous request
- *	@host: MMC host to start command
- *	@areq: asynchronous request to start
- *	@ret_stat: out parameter for status
- *
- *	Start a new MMC custom command request for a host.
- *	If there is on ongoing async request wait for completion
- *	of that request and start the new one and return.
- *	Does not wait for the new request to complete.
- *
- *      Returns the completed request, NULL in case of none completed.
- *	Wait for the an ongoing request (previoulsy started) to complete and
- *	return the completed request. If there is no ongoing request, NULL
- *	is returned without waiting. NULL is not an error condition.
- */
-struct mmc_async_req *mmc_start_areq(struct mmc_host *host,
-				     struct mmc_async_req *areq,
-				     enum mmc_blk_status *ret_stat)
-{
-	enum mmc_blk_status status;
-	int start_err = 0;
-	struct mmc_async_req *previous = host->areq;
-
-	/* Prepare a new request */
-	if (areq)
-		mmc_pre_req(host, areq->mrq);
-
-	/* Finalize previous request */
-	status = mmc_finalize_areq(host);
-	if (ret_stat)
-		*ret_stat = status;
-
-	/* The previous request is still going on... */
-	if (status == MMC_BLK_NEW_REQUEST)
-		return NULL;
-
-	/* Fine so far, start the new request! */
-	if (status == MMC_BLK_SUCCESS && areq)
-		start_err = __mmc_start_data_req(host, areq->mrq);
-
-	/* Postprocess the old request at this point */
-	if (host->areq)
-		mmc_post_req(host, host->areq->mrq, 0);
-
-	/* Cancel a prepared request if it was not started. */
-	if ((status != MMC_BLK_SUCCESS || start_err) && areq)
-		mmc_post_req(host, areq->mrq, -EINVAL);
-
-	if (status != MMC_BLK_SUCCESS)
-		host->areq = NULL;
-	else
-		host->areq = areq;
-
-	return previous;
-}
-EXPORT_SYMBOL(mmc_start_areq);
-
 /**
  *	mmc_wait_for_req - start a request and wait for completion
  *	@host: MMC host to start command
@@ -2959,6 +2761,14 @@ static int mmc_pm_notify(struct notifier_block *notify_block,
 		if (!err)
 			break;
 
+		if (!mmc_card_is_removable(host)) {
+			dev_warn(mmc_dev(host),
+				 "pre_suspend failed for non-removable host: "
+				 "%d\n", err);
+			/* Avoid removing non-removable hosts */
+			break;
+		}
+
 		/* Calling bus_ops->remove() with a claimed host can deadlock */
 		host->bus_ops->remove(host);
 		mmc_claim_host(host);
@@ -2994,22 +2804,6 @@ void mmc_unregister_pm_notifier(struct mmc_host *host)
 }
 #endif
 
-/**
- * mmc_init_context_info() - init synchronization context
- * @host: mmc host
- *
- * Init struct context_info needed to implement asynchronous
- * request mechanism, used by mmc core, host driver and mmc requests
- * supplier.
- */
-void mmc_init_context_info(struct mmc_host *host)
-{
-	host->context_info.is_new_req = false;
-	host->context_info.is_done_rcv = false;
-	host->context_info.is_waiting_last_req = false;
-	init_waitqueue_head(&host->context_info.wait);
-}
-
 static int __init mmc_init(void)
 {
 	int ret;
diff --git a/drivers/mmc/core/core.h b/drivers/mmc/core/core.h
index 71e6c6d7ceb7..d6303d69071b 100644
--- a/drivers/mmc/core/core.h
+++ b/drivers/mmc/core/core.h
@@ -62,12 +62,10 @@ void mmc_set_initial_state(struct mmc_host *host);
 
 static inline void mmc_delay(unsigned int ms)
 {
-	if (ms < 1000 / HZ) {
-		cond_resched();
-		mdelay(ms);
-	} else {
+	if (ms <= 20)
+		usleep_range(ms * 1000, ms * 1250);
+	else
 		msleep(ms);
-	}
 }
 
 void mmc_rescan(struct work_struct *work);
@@ -91,8 +89,6 @@ void mmc_remove_host_debugfs(struct mmc_host *host);
 void mmc_add_card_debugfs(struct mmc_card *card);
 void mmc_remove_card_debugfs(struct mmc_card *card);
 
-void mmc_init_context_info(struct mmc_host *host);
-
 int mmc_execute_tuning(struct mmc_card *card);
 int mmc_hs200_to_hs400(struct mmc_card *card);
 int mmc_hs400_to_hs200(struct mmc_card *card);
@@ -110,12 +106,6 @@ bool mmc_is_req_done(struct mmc_host *host, struct mmc_request *mrq);
 
 int mmc_start_request(struct mmc_host *host, struct mmc_request *mrq);
 
-struct mmc_async_req;
-
-struct mmc_async_req *mmc_start_areq(struct mmc_host *host,
-				     struct mmc_async_req *areq,
-				     enum mmc_blk_status *ret_stat);
-
 int mmc_erase(struct mmc_card *card, unsigned int from, unsigned int nr,
 		unsigned int arg);
 int mmc_can_erase(struct mmc_card *card);
@@ -152,4 +142,35 @@ int mmc_cqe_start_req(struct mmc_host *host, struct mmc_request *mrq);
 void mmc_cqe_post_req(struct mmc_host *host, struct mmc_request *mrq);
 int mmc_cqe_recovery(struct mmc_host *host);
 
+/**
+ *	mmc_pre_req - Prepare for a new request
+ *	@host: MMC host to prepare command
+ *	@mrq: MMC request to prepare for
+ *
+ *	mmc_pre_req() is called in prior to mmc_start_req() to let
+ *	host prepare for the new request. Preparation of a request may be
+ *	performed while another request is running on the host.
+ */
+static inline void mmc_pre_req(struct mmc_host *host, struct mmc_request *mrq)
+{
+	if (host->ops->pre_req)
+		host->ops->pre_req(host, mrq);
+}
+
+/**
+ *	mmc_post_req - Post process a completed request
+ *	@host: MMC host to post process command
+ *	@mrq: MMC request to post process for
+ *	@err: Error, if non zero, clean up any resources made in pre_req
+ *
+ *	Let the host post process a completed request. Post processing of
+ *	a request may be performed while another request is running.
+ */
+static inline void mmc_post_req(struct mmc_host *host, struct mmc_request *mrq,
+				int err)
+{
+	if (host->ops->post_req)
+		host->ops->post_req(host, mrq, err);
+}
+
 #endif
diff --git a/drivers/mmc/core/host.h b/drivers/mmc/core/host.h
index fb689a1065ed..06ec19b5bf9f 100644
--- a/drivers/mmc/core/host.h
+++ b/drivers/mmc/core/host.h
@@ -41,6 +41,11 @@ static inline int mmc_host_cmd23(struct mmc_host *host)
 	return host->caps & MMC_CAP_CMD23;
 }
 
+static inline bool mmc_host_done_complete(struct mmc_host *host)
+{
+	return host->caps & MMC_CAP_DONE_COMPLETE;
+}
+
 static inline int mmc_boot_partition_access(struct mmc_host *host)
 {
 	return !(host->caps2 & MMC_CAP2_BOOTPART_NOACC);
@@ -74,6 +79,5 @@ static inline bool mmc_card_hs400es(struct mmc_card *card)
 	return card->host->ios.enhanced_strobe;
 }
 
-
 #endif
 
diff --git a/drivers/mmc/core/mmc_ops.c b/drivers/mmc/core/mmc_ops.c
index 908e4db03535..42d6aa89a48a 100644
--- a/drivers/mmc/core/mmc_ops.c
+++ b/drivers/mmc/core/mmc_ops.c
@@ -848,7 +848,6 @@ int mmc_interrupt_hpi(struct mmc_card *card)
 		return 1;
 	}
 
-	mmc_claim_host(card->host);
 	err = mmc_send_status(card, &status);
 	if (err) {
 		pr_err("%s: Get card status fail\n", mmc_hostname(card->host));
@@ -890,7 +889,6 @@ int mmc_interrupt_hpi(struct mmc_card *card)
 	} while (!err);
 
 out:
-	mmc_release_host(card->host);
 	return err;
 }
 
@@ -932,9 +930,7 @@ static int mmc_read_bkops_status(struct mmc_card *card)
 	int err;
 	u8 *ext_csd;
 
-	mmc_claim_host(card->host);
 	err = mmc_get_ext_csd(card, &ext_csd);
-	mmc_release_host(card->host);
 	if (err)
 		return err;
 
diff --git a/drivers/mmc/core/mmc_test.c b/drivers/mmc/core/mmc_test.c
index 478869805b96..ef18daeaa4cc 100644
--- a/drivers/mmc/core/mmc_test.c
+++ b/drivers/mmc/core/mmc_test.c
@@ -101,7 +101,7 @@ struct mmc_test_transfer_result {
 	struct list_head link;
 	unsigned int count;
 	unsigned int sectors;
-	struct timespec ts;
+	struct timespec64 ts;
 	unsigned int rate;
 	unsigned int iops;
 };
@@ -171,11 +171,6 @@ struct mmc_test_multiple_rw {
 	enum mmc_test_prep_media prepare;
 };
 
-struct mmc_test_async_req {
-	struct mmc_async_req areq;
-	struct mmc_test_card *test;
-};
-
 /*******************************************************************/
 /*  General helper functions                                       */
 /*******************************************************************/
@@ -515,14 +510,11 @@ static int mmc_test_map_sg_max_scatter(struct mmc_test_mem *mem,
 /*
  * Calculate transfer rate in bytes per second.
  */
-static unsigned int mmc_test_rate(uint64_t bytes, struct timespec *ts)
+static unsigned int mmc_test_rate(uint64_t bytes, struct timespec64 *ts)
 {
 	uint64_t ns;
 
-	ns = ts->tv_sec;
-	ns *= 1000000000;
-	ns += ts->tv_nsec;
-
+	ns = timespec64_to_ns(ts);
 	bytes *= 1000000000;
 
 	while (ns > UINT_MAX) {
@@ -542,7 +534,7 @@ static unsigned int mmc_test_rate(uint64_t bytes, struct timespec *ts)
  * Save transfer results for future usage
  */
 static void mmc_test_save_transfer_result(struct mmc_test_card *test,
-	unsigned int count, unsigned int sectors, struct timespec ts,
+	unsigned int count, unsigned int sectors, struct timespec64 ts,
 	unsigned int rate, unsigned int iops)
 {
 	struct mmc_test_transfer_result *tr;
@@ -567,21 +559,21 @@ static void mmc_test_save_transfer_result(struct mmc_test_card *test,
  * Print the transfer rate.
  */
 static void mmc_test_print_rate(struct mmc_test_card *test, uint64_t bytes,
-				struct timespec *ts1, struct timespec *ts2)
+				struct timespec64 *ts1, struct timespec64 *ts2)
 {
 	unsigned int rate, iops, sectors = bytes >> 9;
-	struct timespec ts;
+	struct timespec64 ts;
 
-	ts = timespec_sub(*ts2, *ts1);
+	ts = timespec64_sub(*ts2, *ts1);
 
 	rate = mmc_test_rate(bytes, &ts);
 	iops = mmc_test_rate(100, &ts); /* I/O ops per sec x 100 */
 
-	pr_info("%s: Transfer of %u sectors (%u%s KiB) took %lu.%09lu "
+	pr_info("%s: Transfer of %u sectors (%u%s KiB) took %llu.%09u "
 			 "seconds (%u kB/s, %u KiB/s, %u.%02u IOPS)\n",
 			 mmc_hostname(test->card->host), sectors, sectors >> 1,
-			 (sectors & 1 ? ".5" : ""), (unsigned long)ts.tv_sec,
-			 (unsigned long)ts.tv_nsec, rate / 1000, rate / 1024,
+			 (sectors & 1 ? ".5" : ""), (u64)ts.tv_sec,
+			 (u32)ts.tv_nsec, rate / 1000, rate / 1024,
 			 iops / 100, iops % 100);
 
 	mmc_test_save_transfer_result(test, 1, sectors, ts, rate, iops);
@@ -591,24 +583,24 @@ static void mmc_test_print_rate(struct mmc_test_card *test, uint64_t bytes,
  * Print the average transfer rate.
  */
 static void mmc_test_print_avg_rate(struct mmc_test_card *test, uint64_t bytes,
-				    unsigned int count, struct timespec *ts1,
-				    struct timespec *ts2)
+				    unsigned int count, struct timespec64 *ts1,
+				    struct timespec64 *ts2)
 {
 	unsigned int rate, iops, sectors = bytes >> 9;
 	uint64_t tot = bytes * count;
-	struct timespec ts;
+	struct timespec64 ts;
 
-	ts = timespec_sub(*ts2, *ts1);
+	ts = timespec64_sub(*ts2, *ts1);
 
 	rate = mmc_test_rate(tot, &ts);
 	iops = mmc_test_rate(count * 100, &ts); /* I/O ops per sec x 100 */
 
 	pr_info("%s: Transfer of %u x %u sectors (%u x %u%s KiB) took "
-			 "%lu.%09lu seconds (%u kB/s, %u KiB/s, "
+			 "%llu.%09u seconds (%u kB/s, %u KiB/s, "
 			 "%u.%02u IOPS, sg_len %d)\n",
 			 mmc_hostname(test->card->host), count, sectors, count,
 			 sectors >> 1, (sectors & 1 ? ".5" : ""),
-			 (unsigned long)ts.tv_sec, (unsigned long)ts.tv_nsec,
+			 (u64)ts.tv_sec, (u32)ts.tv_nsec,
 			 rate / 1000, rate / 1024, iops / 100, iops % 100,
 			 test->area.sg_len);
 
@@ -741,30 +733,6 @@ static int mmc_test_check_result(struct mmc_test_card *test,
 	return ret;
 }
 
-static enum mmc_blk_status mmc_test_check_result_async(struct mmc_card *card,
-				       struct mmc_async_req *areq)
-{
-	struct mmc_test_async_req *test_async =
-		container_of(areq, struct mmc_test_async_req, areq);
-	int ret;
-
-	mmc_test_wait_busy(test_async->test);
-
-	/*
-	 * FIXME: this would earlier just casts a regular error code,
-	 * either of the kernel type -ERRORCODE or the local test framework
-	 * RESULT_* errorcode, into an enum mmc_blk_status and return as
-	 * result check. Instead, convert it to some reasonable type by just
-	 * returning either MMC_BLK_SUCCESS or MMC_BLK_CMD_ERR.
-	 * If possible, a reasonable error code should be returned.
-	 */
-	ret = mmc_test_check_result(test_async->test, areq->mrq);
-	if (ret)
-		return MMC_BLK_CMD_ERR;
-
-	return MMC_BLK_SUCCESS;
-}
-
 /*
  * Checks that a "short transfer" behaved as expected
  */
@@ -831,6 +799,45 @@ static struct mmc_test_req *mmc_test_req_alloc(void)
 	return rq;
 }
 
+static void mmc_test_wait_done(struct mmc_request *mrq)
+{
+	complete(&mrq->completion);
+}
+
+static int mmc_test_start_areq(struct mmc_test_card *test,
+			       struct mmc_request *mrq,
+			       struct mmc_request *prev_mrq)
+{
+	struct mmc_host *host = test->card->host;
+	int err = 0;
+
+	if (mrq) {
+		init_completion(&mrq->completion);
+		mrq->done = mmc_test_wait_done;
+		mmc_pre_req(host, mrq);
+	}
+
+	if (prev_mrq) {
+		wait_for_completion(&prev_mrq->completion);
+		err = mmc_test_wait_busy(test);
+		if (!err)
+			err = mmc_test_check_result(test, prev_mrq);
+	}
+
+	if (!err && mrq) {
+		err = mmc_start_request(host, mrq);
+		if (err)
+			mmc_retune_release(host);
+	}
+
+	if (prev_mrq)
+		mmc_post_req(host, prev_mrq, 0);
+
+	if (err && mrq)
+		mmc_post_req(host, mrq, err);
+
+	return err;
+}
 
 static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 				      struct scatterlist *sg, unsigned sg_len,
@@ -838,17 +845,10 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 				      unsigned blksz, int write, int count)
 {
 	struct mmc_test_req *rq1, *rq2;
-	struct mmc_test_async_req test_areq[2];
-	struct mmc_async_req *done_areq;
-	struct mmc_async_req *cur_areq = &test_areq[0].areq;
-	struct mmc_async_req *other_areq = &test_areq[1].areq;
-	enum mmc_blk_status status;
+	struct mmc_request *mrq, *prev_mrq;
 	int i;
 	int ret = RESULT_OK;
 
-	test_areq[0].test = test;
-	test_areq[1].test = test;
-
 	rq1 = mmc_test_req_alloc();
 	rq2 = mmc_test_req_alloc();
 	if (!rq1 || !rq2) {
@@ -856,33 +856,25 @@ static int mmc_test_nonblock_transfer(struct mmc_test_card *test,
 		goto err;
 	}
 
-	cur_areq->mrq = &rq1->mrq;
-	cur_areq->err_check = mmc_test_check_result_async;
-	other_areq->mrq = &rq2->mrq;
-	other_areq->err_check = mmc_test_check_result_async;
+	mrq = &rq1->mrq;
+	prev_mrq = NULL;
 
 	for (i = 0; i < count; i++) {
-		mmc_test_prepare_mrq(test, cur_areq->mrq, sg, sg_len, dev_addr,
-				     blocks, blksz, write);
-		done_areq = mmc_start_areq(test->card->host, cur_areq, &status);
-
-		if (status != MMC_BLK_SUCCESS || (!done_areq && i > 0)) {
-			ret = RESULT_FAIL;
+		mmc_test_req_reset(container_of(mrq, struct mmc_test_req, mrq));
+		mmc_test_prepare_mrq(test, mrq, sg, sg_len, dev_addr, blocks,
+				     blksz, write);
+		ret = mmc_test_start_areq(test, mrq, prev_mrq);
+		if (ret)
 			goto err;
-		}
 
-		if (done_areq)
-			mmc_test_req_reset(container_of(done_areq->mrq,
-						struct mmc_test_req, mrq));
+		if (!prev_mrq)
+			prev_mrq = &rq2->mrq;
 
-		swap(cur_areq, other_areq);
+		swap(mrq, prev_mrq);
 		dev_addr += blocks;
 	}
 
-	done_areq = mmc_start_areq(test->card->host, NULL, &status);
-	if (status != MMC_BLK_SUCCESS)
-		ret = RESULT_FAIL;
-
+	ret = mmc_test_start_areq(test, NULL, prev_mrq);
 err:
 	kfree(rq1);
 	kfree(rq2);
@@ -1449,7 +1441,7 @@ static int mmc_test_area_io_seq(struct mmc_test_card *test, unsigned long sz,
 				int max_scatter, int timed, int count,
 				bool nonblock, int min_sg_len)
 {
-	struct timespec ts1, ts2;
+	struct timespec64 ts1, ts2;
 	int ret = 0;
 	int i;
 	struct mmc_test_area *t = &test->area;
@@ -1475,7 +1467,7 @@ static int mmc_test_area_io_seq(struct mmc_test_card *test, unsigned long sz,
 		return ret;
 
 	if (timed)
-		getnstimeofday(&ts1);
+		ktime_get_ts64(&ts1);
 	if (nonblock)
 		ret = mmc_test_nonblock_transfer(test, t->sg, t->sg_len,
 				 dev_addr, t->blocks, 512, write, count);
@@ -1489,7 +1481,7 @@ static int mmc_test_area_io_seq(struct mmc_test_card *test, unsigned long sz,
 		return ret;
 
 	if (timed)
-		getnstimeofday(&ts2);
+		ktime_get_ts64(&ts2);
 
 	if (timed)
 		mmc_test_print_avg_rate(test, sz, count, &ts1, &ts2);
@@ -1747,7 +1739,7 @@ static int mmc_test_profile_trim_perf(struct mmc_test_card *test)
 	struct mmc_test_area *t = &test->area;
 	unsigned long sz;
 	unsigned int dev_addr;
-	struct timespec ts1, ts2;
+	struct timespec64 ts1, ts2;
 	int ret;
 
 	if (!mmc_can_trim(test->card))
@@ -1758,19 +1750,19 @@ static int mmc_test_profile_trim_perf(struct mmc_test_card *test)
 
 	for (sz = 512; sz < t->max_sz; sz <<= 1) {
 		dev_addr = t->dev_addr + (sz >> 9);
-		getnstimeofday(&ts1);
+		ktime_get_ts64(&ts1);
 		ret = mmc_erase(test->card, dev_addr, sz >> 9, MMC_TRIM_ARG);
 		if (ret)
 			return ret;
-		getnstimeofday(&ts2);
+		ktime_get_ts64(&ts2);
 		mmc_test_print_rate(test, sz, &ts1, &ts2);
 	}
 	dev_addr = t->dev_addr;
-	getnstimeofday(&ts1);
+	ktime_get_ts64(&ts1);
 	ret = mmc_erase(test->card, dev_addr, sz >> 9, MMC_TRIM_ARG);
 	if (ret)
 		return ret;
-	getnstimeofday(&ts2);
+	ktime_get_ts64(&ts2);
 	mmc_test_print_rate(test, sz, &ts1, &ts2);
 	return 0;
 }
@@ -1779,19 +1771,19 @@ static int mmc_test_seq_read_perf(struct mmc_test_card *test, unsigned long sz)
 {
 	struct mmc_test_area *t = &test->area;
 	unsigned int dev_addr, i, cnt;
-	struct timespec ts1, ts2;
+	struct timespec64 ts1, ts2;
 	int ret;
 
 	cnt = t->max_sz / sz;
 	dev_addr = t->dev_addr;
-	getnstimeofday(&ts1);
+	ktime_get_ts64(&ts1);
 	for (i = 0; i < cnt; i++) {
 		ret = mmc_test_area_io(test, sz, dev_addr, 0, 0, 0);
 		if (ret)
 			return ret;
 		dev_addr += (sz >> 9);
 	}
-	getnstimeofday(&ts2);
+	ktime_get_ts64(&ts2);
 	mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
 	return 0;
 }
@@ -1818,7 +1810,7 @@ static int mmc_test_seq_write_perf(struct mmc_test_card *test, unsigned long sz)
 {
 	struct mmc_test_area *t = &test->area;
 	unsigned int dev_addr, i, cnt;
-	struct timespec ts1, ts2;
+	struct timespec64 ts1, ts2;
 	int ret;
 
 	ret = mmc_test_area_erase(test);
@@ -1826,14 +1818,14 @@ static int mmc_test_seq_write_perf(struct mmc_test_card *test, unsigned long sz)
 		return ret;
 	cnt = t->max_sz / sz;
 	dev_addr = t->dev_addr;
-	getnstimeofday(&ts1);
+	ktime_get_ts64(&ts1);
 	for (i = 0; i < cnt; i++) {
 		ret = mmc_test_area_io(test, sz, dev_addr, 1, 0, 0);
 		if (ret)
 			return ret;
 		dev_addr += (sz >> 9);
 	}
-	getnstimeofday(&ts2);
+	ktime_get_ts64(&ts2);
 	mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
 	return 0;
 }
@@ -1864,7 +1856,7 @@ static int mmc_test_profile_seq_trim_perf(struct mmc_test_card *test)
 	struct mmc_test_area *t = &test->area;
 	unsigned long sz;
 	unsigned int dev_addr, i, cnt;
-	struct timespec ts1, ts2;
+	struct timespec64 ts1, ts2;
 	int ret;
 
 	if (!mmc_can_trim(test->card))
@@ -1882,7 +1874,7 @@ static int mmc_test_profile_seq_trim_perf(struct mmc_test_card *test)
 			return ret;
 		cnt = t->max_sz / sz;
 		dev_addr = t->dev_addr;
-		getnstimeofday(&ts1);
+		ktime_get_ts64(&ts1);
 		for (i = 0; i < cnt; i++) {
 			ret = mmc_erase(test->card, dev_addr, sz >> 9,
 					MMC_TRIM_ARG);
@@ -1890,7 +1882,7 @@ static int mmc_test_profile_seq_trim_perf(struct mmc_test_card *test)
 				return ret;
 			dev_addr += (sz >> 9);
 		}
-		getnstimeofday(&ts2);
+		ktime_get_ts64(&ts2);
 		mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
 	}
 	return 0;
@@ -1912,7 +1904,7 @@ static int mmc_test_rnd_perf(struct mmc_test_card *test, int write, int print,
 {
 	unsigned int dev_addr, cnt, rnd_addr, range1, range2, last_ea = 0, ea;
 	unsigned int ssz;
-	struct timespec ts1, ts2, ts;
+	struct timespec64 ts1, ts2, ts;
 	int ret;
 
 	ssz = sz >> 9;
@@ -1921,10 +1913,10 @@ static int mmc_test_rnd_perf(struct mmc_test_card *test, int write, int print,
 	range1 = rnd_addr / test->card->pref_erase;
 	range2 = range1 / ssz;
 
-	getnstimeofday(&ts1);
+	ktime_get_ts64(&ts1);
 	for (cnt = 0; cnt < UINT_MAX; cnt++) {
-		getnstimeofday(&ts2);
-		ts = timespec_sub(ts2, ts1);
+		ktime_get_ts64(&ts2);
+		ts = timespec64_sub(ts2, ts1);
 		if (ts.tv_sec >= 10)
 			break;
 		ea = mmc_test_rnd_num(range1);
@@ -1998,7 +1990,7 @@ static int mmc_test_seq_perf(struct mmc_test_card *test, int write,
 {
 	struct mmc_test_area *t = &test->area;
 	unsigned int dev_addr, i, cnt, sz, ssz;
-	struct timespec ts1, ts2;
+	struct timespec64 ts1, ts2;
 	int ret;
 
 	sz = t->max_tfr;
@@ -2025,7 +2017,7 @@ static int mmc_test_seq_perf(struct mmc_test_card *test, int write,
 	cnt = tot_sz / sz;
 	dev_addr &= 0xffff0000; /* Round to 64MiB boundary */
 
-	getnstimeofday(&ts1);
+	ktime_get_ts64(&ts1);
 	for (i = 0; i < cnt; i++) {
 		ret = mmc_test_area_io(test, sz, dev_addr, write,
 				       max_scatter, 0);
@@ -2033,7 +2025,7 @@ static int mmc_test_seq_perf(struct mmc_test_card *test, int write,
 			return ret;
 		dev_addr += ssz;
 	}
-	getnstimeofday(&ts2);
+	ktime_get_ts64(&ts2);
 
 	mmc_test_print_avg_rate(test, sz, cnt, &ts1, &ts2);
 
@@ -2328,10 +2320,17 @@ static int mmc_test_reset(struct mmc_test_card *test)
 	int err;
 
 	err = mmc_hw_reset(host);
-	if (!err)
+	if (!err) {
+		/*
+		 * Reset will re-enable the card's command queue, but tests
+		 * expect it to be disabled.
+		 */
+		if (card->ext_csd.cmdq_en)
+			mmc_cmdq_disable(card);
 		return RESULT_OK;
-	else if (err == -EOPNOTSUPP)
+	} else if (err == -EOPNOTSUPP) {
 		return RESULT_UNSUP_HOST;
+	}
 
 	return RESULT_FAIL;
 }
@@ -2356,11 +2355,9 @@ static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
 	struct mmc_test_req *rq = mmc_test_req_alloc();
 	struct mmc_host *host = test->card->host;
 	struct mmc_test_area *t = &test->area;
-	struct mmc_test_async_req test_areq = { .test = test };
 	struct mmc_request *mrq;
 	unsigned long timeout;
 	bool expired = false;
-	enum mmc_blk_status blkstat = MMC_BLK_SUCCESS;
 	int ret = 0, cmd_ret;
 	u32 status = 0;
 	int count = 0;
@@ -2373,9 +2370,6 @@ static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
 		mrq->sbc = &rq->sbc;
 	mrq->cap_cmd_during_tfr = true;
 
-	test_areq.areq.mrq = mrq;
-	test_areq.areq.err_check = mmc_test_check_result_async;
-
 	mmc_test_prepare_mrq(test, mrq, t->sg, t->sg_len, dev_addr, t->blocks,
 			     512, write);
 
@@ -2388,11 +2382,9 @@ static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
 
 	/* Start ongoing data request */
 	if (use_areq) {
-		mmc_start_areq(host, &test_areq.areq, &blkstat);
-		if (blkstat != MMC_BLK_SUCCESS) {
-			ret = RESULT_FAIL;
+		ret = mmc_test_start_areq(test, mrq, NULL);
+		if (ret)
 			goto out_free;
-		}
 	} else {
 		mmc_wait_for_req(host, mrq);
 	}
@@ -2426,9 +2418,7 @@ static int mmc_test_ongoing_transfer(struct mmc_test_card *test,
 
 	/* Wait for data request to complete */
 	if (use_areq) {
-		mmc_start_areq(host, NULL, &blkstat);
-		if (blkstat != MMC_BLK_SUCCESS)
-			ret = RESULT_FAIL;
+		ret = mmc_test_start_areq(test, NULL, mrq);
 	} else {
 		mmc_wait_for_req_done(test->card->host, mrq);
 	}
@@ -3066,10 +3056,9 @@ static int mtf_test_show(struct seq_file *sf, void *data)
 		seq_printf(sf, "Test %d: %d\n", gr->testcase + 1, gr->result);
 
 		list_for_each_entry(tr, &gr->tr_lst, link) {
-			seq_printf(sf, "%u %d %lu.%09lu %u %u.%02u\n",
+			seq_printf(sf, "%u %d %llu.%09u %u %u.%02u\n",
 				tr->count, tr->sectors,
-				(unsigned long)tr->ts.tv_sec,
-				(unsigned long)tr->ts.tv_nsec,
+				(u64)tr->ts.tv_sec, (u32)tr->ts.tv_nsec,
 				tr->rate, tr->iops / 100, tr->iops % 100);
 		}
 	}
diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c
index 4f33d277b125..421fab7250ac 100644
--- a/drivers/mmc/core/queue.c
+++ b/drivers/mmc/core/queue.c
@@ -22,100 +22,147 @@
 #include "block.h"
 #include "core.h"
 #include "card.h"
+#include "host.h"
 
-/*
- * Prepare a MMC request. This just filters out odd stuff.
- */
-static int mmc_prep_request(struct request_queue *q, struct request *req)
+static inline bool mmc_cqe_dcmd_busy(struct mmc_queue *mq)
 {
-	struct mmc_queue *mq = q->queuedata;
+	/* Allow only 1 DCMD at a time */
+	return mq->in_flight[MMC_ISSUE_DCMD];
+}
 
-	if (mq && mmc_card_removed(mq->card))
-		return BLKPREP_KILL;
+void mmc_cqe_check_busy(struct mmc_queue *mq)
+{
+	if ((mq->cqe_busy & MMC_CQE_DCMD_BUSY) && !mmc_cqe_dcmd_busy(mq))
+		mq->cqe_busy &= ~MMC_CQE_DCMD_BUSY;
 
-	req->rq_flags |= RQF_DONTPREP;
+	mq->cqe_busy &= ~MMC_CQE_QUEUE_FULL;
+}
 
-	return BLKPREP_OK;
+static inline bool mmc_cqe_can_dcmd(struct mmc_host *host)
+{
+	return host->caps2 & MMC_CAP2_CQE_DCMD;
 }
 
-static int mmc_queue_thread(void *d)
+static enum mmc_issue_type mmc_cqe_issue_type(struct mmc_host *host,
+					      struct request *req)
 {
-	struct mmc_queue *mq = d;
-	struct request_queue *q = mq->queue;
-	struct mmc_context_info *cntx = &mq->card->host->context_info;
+	switch (req_op(req)) {
+	case REQ_OP_DRV_IN:
+	case REQ_OP_DRV_OUT:
+	case REQ_OP_DISCARD:
+	case REQ_OP_SECURE_ERASE:
+		return MMC_ISSUE_SYNC;
+	case REQ_OP_FLUSH:
+		return mmc_cqe_can_dcmd(host) ? MMC_ISSUE_DCMD : MMC_ISSUE_SYNC;
+	default:
+		return MMC_ISSUE_ASYNC;
+	}
+}
 
-	current->flags |= PF_MEMALLOC;
+enum mmc_issue_type mmc_issue_type(struct mmc_queue *mq, struct request *req)
+{
+	struct mmc_host *host = mq->card->host;
 
-	down(&mq->thread_sem);
-	do {
-		struct request *req;
+	if (mq->use_cqe)
+		return mmc_cqe_issue_type(host, req);
 
-		spin_lock_irq(q->queue_lock);
-		set_current_state(TASK_INTERRUPTIBLE);
-		req = blk_fetch_request(q);
-		mq->asleep = false;
-		cntx->is_waiting_last_req = false;
-		cntx->is_new_req = false;
-		if (!req) {
-			/*
-			 * Dispatch queue is empty so set flags for
-			 * mmc_request_fn() to wake us up.
-			 */
-			if (mq->qcnt)
-				cntx->is_waiting_last_req = true;
-			else
-				mq->asleep = true;
-		}
-		spin_unlock_irq(q->queue_lock);
+	if (req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_WRITE)
+		return MMC_ISSUE_ASYNC;
 
-		if (req || mq->qcnt) {
-			set_current_state(TASK_RUNNING);
-			mmc_blk_issue_rq(mq, req);
-			cond_resched();
-		} else {
-			if (kthread_should_stop()) {
-				set_current_state(TASK_RUNNING);
-				break;
-			}
-			up(&mq->thread_sem);
-			schedule();
-			down(&mq->thread_sem);
-		}
-	} while (1);
-	up(&mq->thread_sem);
+	return MMC_ISSUE_SYNC;
+}
 
-	return 0;
+static void __mmc_cqe_recovery_notifier(struct mmc_queue *mq)
+{
+	if (!mq->recovery_needed) {
+		mq->recovery_needed = true;
+		schedule_work(&mq->recovery_work);
+	}
 }
 
-/*
- * Generic MMC request handler.  This is called for any queue on a
- * particular host.  When the host is not busy, we look for a request
- * on any queue on this host, and attempt to issue it.  This may
- * not be the queue we were asked to process.
- */
-static void mmc_request_fn(struct request_queue *q)
+void mmc_cqe_recovery_notifier(struct mmc_request *mrq)
 {
+	struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
+						  brq.mrq);
+	struct request *req = mmc_queue_req_to_req(mqrq);
+	struct request_queue *q = req->q;
 	struct mmc_queue *mq = q->queuedata;
-	struct request *req;
-	struct mmc_context_info *cntx;
+	unsigned long flags;
+
+	spin_lock_irqsave(q->queue_lock, flags);
+	__mmc_cqe_recovery_notifier(mq);
+	spin_unlock_irqrestore(q->queue_lock, flags);
+}
 
-	if (!mq) {
-		while ((req = blk_fetch_request(q)) != NULL) {
-			req->rq_flags |= RQF_QUIET;
-			__blk_end_request_all(req, BLK_STS_IOERR);
+static enum blk_eh_timer_return mmc_cqe_timed_out(struct request *req)
+{
+	struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
+	struct mmc_request *mrq = &mqrq->brq.mrq;
+	struct mmc_queue *mq = req->q->queuedata;
+	struct mmc_host *host = mq->card->host;
+	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
+	bool recovery_needed = false;
+
+	switch (issue_type) {
+	case MMC_ISSUE_ASYNC:
+	case MMC_ISSUE_DCMD:
+		if (host->cqe_ops->cqe_timeout(host, mrq, &recovery_needed)) {
+			if (recovery_needed)
+				__mmc_cqe_recovery_notifier(mq);
+			return BLK_EH_RESET_TIMER;
 		}
-		return;
+		/* No timeout */
+		return BLK_EH_HANDLED;
+	default:
+		/* Timeout is handled by mmc core */
+		return BLK_EH_RESET_TIMER;
 	}
+}
+
+static enum blk_eh_timer_return mmc_mq_timed_out(struct request *req,
+						 bool reserved)
+{
+	struct request_queue *q = req->q;
+	struct mmc_queue *mq = q->queuedata;
+	unsigned long flags;
+	int ret;
 
-	cntx = &mq->card->host->context_info;
+	spin_lock_irqsave(q->queue_lock, flags);
 
-	if (cntx->is_waiting_last_req) {
-		cntx->is_new_req = true;
-		wake_up_interruptible(&cntx->wait);
-	}
+	if (mq->recovery_needed || !mq->use_cqe)
+		ret = BLK_EH_RESET_TIMER;
+	else
+		ret = mmc_cqe_timed_out(req);
 
-	if (mq->asleep)
-		wake_up_process(mq->thread);
+	spin_unlock_irqrestore(q->queue_lock, flags);
+
+	return ret;
+}
+
+static void mmc_mq_recovery_handler(struct work_struct *work)
+{
+	struct mmc_queue *mq = container_of(work, struct mmc_queue,
+					    recovery_work);
+	struct request_queue *q = mq->queue;
+
+	mmc_get_card(mq->card, &mq->ctx);
+
+	mq->in_recovery = true;
+
+	if (mq->use_cqe)
+		mmc_blk_cqe_recovery(mq);
+	else
+		mmc_blk_mq_recovery(mq);
+
+	mq->in_recovery = false;
+
+	spin_lock_irq(q->queue_lock);
+	mq->recovery_needed = false;
+	spin_unlock_irq(q->queue_lock);
+
+	mmc_put_card(mq->card, &mq->ctx);
+
+	blk_mq_run_hw_queues(q, true);
 }
 
 static struct scatterlist *mmc_alloc_sg(int sg_len, gfp_t gfp)
@@ -154,11 +201,10 @@ static void mmc_queue_setup_discard(struct request_queue *q,
  * @req: the request
  * @gfp: memory allocation policy
  */
-static int mmc_init_request(struct request_queue *q, struct request *req,
-			    gfp_t gfp)
+static int __mmc_init_request(struct mmc_queue *mq, struct request *req,
+			      gfp_t gfp)
 {
 	struct mmc_queue_req *mq_rq = req_to_mmc_queue_req(req);
-	struct mmc_queue *mq = q->queuedata;
 	struct mmc_card *card = mq->card;
 	struct mmc_host *host = card->host;
 
@@ -177,6 +223,131 @@ static void mmc_exit_request(struct request_queue *q, struct request *req)
 	mq_rq->sg = NULL;
 }
 
+static int mmc_mq_init_request(struct blk_mq_tag_set *set, struct request *req,
+			       unsigned int hctx_idx, unsigned int numa_node)
+{
+	return __mmc_init_request(set->driver_data, req, GFP_KERNEL);
+}
+
+static void mmc_mq_exit_request(struct blk_mq_tag_set *set, struct request *req,
+				unsigned int hctx_idx)
+{
+	struct mmc_queue *mq = set->driver_data;
+
+	mmc_exit_request(mq->queue, req);
+}
+
+/*
+ * We use BLK_MQ_F_BLOCKING and have only 1 hardware queue, which means requests
+ * will not be dispatched in parallel.
+ */
+static blk_status_t mmc_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
+				    const struct blk_mq_queue_data *bd)
+{
+	struct request *req = bd->rq;
+	struct request_queue *q = req->q;
+	struct mmc_queue *mq = q->queuedata;
+	struct mmc_card *card = mq->card;
+	struct mmc_host *host = card->host;
+	enum mmc_issue_type issue_type;
+	enum mmc_issued issued;
+	bool get_card, cqe_retune_ok;
+	int ret;
+
+	if (mmc_card_removed(mq->card)) {
+		req->rq_flags |= RQF_QUIET;
+		return BLK_STS_IOERR;
+	}
+
+	issue_type = mmc_issue_type(mq, req);
+
+	spin_lock_irq(q->queue_lock);
+
+	if (mq->recovery_needed) {
+		spin_unlock_irq(q->queue_lock);
+		return BLK_STS_RESOURCE;
+	}
+
+	switch (issue_type) {
+	case MMC_ISSUE_DCMD:
+		if (mmc_cqe_dcmd_busy(mq)) {
+			mq->cqe_busy |= MMC_CQE_DCMD_BUSY;
+			spin_unlock_irq(q->queue_lock);
+			return BLK_STS_RESOURCE;
+		}
+		break;
+	case MMC_ISSUE_ASYNC:
+		break;
+	default:
+		/*
+		 * Timeouts are handled by mmc core, and we don't have a host
+		 * API to abort requests, so we can't handle the timeout anyway.
+		 * However, when the timeout happens, blk_mq_complete_request()
+		 * no longer works (to stop the request disappearing under us).
+		 * To avoid racing with that, set a large timeout.
+		 */
+		req->timeout = 600 * HZ;
+		break;
+	}
+
+	mq->in_flight[issue_type] += 1;
+	get_card = (mmc_tot_in_flight(mq) == 1);
+	cqe_retune_ok = (mmc_cqe_qcnt(mq) == 1);
+
+	spin_unlock_irq(q->queue_lock);
+
+	if (!(req->rq_flags & RQF_DONTPREP)) {
+		req_to_mmc_queue_req(req)->retries = 0;
+		req->rq_flags |= RQF_DONTPREP;
+	}
+
+	if (get_card)
+		mmc_get_card(card, &mq->ctx);
+
+	if (mq->use_cqe) {
+		host->retune_now = host->need_retune && cqe_retune_ok &&
+				   !host->hold_retune;
+	}
+
+	blk_mq_start_request(req);
+
+	issued = mmc_blk_mq_issue_rq(mq, req);
+
+	switch (issued) {
+	case MMC_REQ_BUSY:
+		ret = BLK_STS_RESOURCE;
+		break;
+	case MMC_REQ_FAILED_TO_START:
+		ret = BLK_STS_IOERR;
+		break;
+	default:
+		ret = BLK_STS_OK;
+		break;
+	}
+
+	if (issued != MMC_REQ_STARTED) {
+		bool put_card = false;
+
+		spin_lock_irq(q->queue_lock);
+		mq->in_flight[issue_type] -= 1;
+		if (mmc_tot_in_flight(mq) == 0)
+			put_card = true;
+		spin_unlock_irq(q->queue_lock);
+		if (put_card)
+			mmc_put_card(card, &mq->ctx);
+	}
+
+	return ret;
+}
+
+static const struct blk_mq_ops mmc_mq_ops = {
+	.queue_rq	= mmc_mq_queue_rq,
+	.init_request	= mmc_mq_init_request,
+	.exit_request	= mmc_mq_exit_request,
+	.complete	= mmc_blk_mq_complete,
+	.timeout	= mmc_mq_timed_out,
+};
+
 static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card)
 {
 	struct mmc_host *host = card->host;
@@ -196,124 +367,139 @@ static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card)
 	blk_queue_max_segments(mq->queue, host->max_segs);
 	blk_queue_max_segment_size(mq->queue, host->max_seg_size);
 
-	/* Initialize thread_sem even if it is not used */
-	sema_init(&mq->thread_sem, 1);
+	INIT_WORK(&mq->recovery_work, mmc_mq_recovery_handler);
+	INIT_WORK(&mq->complete_work, mmc_blk_mq_complete_work);
+
+	mutex_init(&mq->complete_lock);
+
+	init_waitqueue_head(&mq->wait);
 }
 
-/**
- * mmc_init_queue - initialise a queue structure.
- * @mq: mmc queue
- * @card: mmc card to attach this queue
- * @lock: queue lock
- * @subname: partition subname
- *
- * Initialise a MMC card request queue.
- */
-int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
-		   spinlock_t *lock, const char *subname)
+static int mmc_mq_init_queue(struct mmc_queue *mq, int q_depth,
+			     const struct blk_mq_ops *mq_ops, spinlock_t *lock)
 {
-	struct mmc_host *host = card->host;
-	int ret = -ENOMEM;
-
-	mq->card = card;
-	mq->queue = blk_alloc_queue(GFP_KERNEL);
-	if (!mq->queue)
-		return -ENOMEM;
-	mq->queue->queue_lock = lock;
-	mq->queue->request_fn = mmc_request_fn;
-	mq->queue->init_rq_fn = mmc_init_request;
-	mq->queue->exit_rq_fn = mmc_exit_request;
-	mq->queue->cmd_size = sizeof(struct mmc_queue_req);
-	mq->queue->queuedata = mq;
-	mq->qcnt = 0;
-	ret = blk_init_allocated_queue(mq->queue);
-	if (ret) {
-		blk_cleanup_queue(mq->queue);
+	int ret;
+
+	memset(&mq->tag_set, 0, sizeof(mq->tag_set));
+	mq->tag_set.ops = mq_ops;
+	mq->tag_set.queue_depth = q_depth;
+	mq->tag_set.numa_node = NUMA_NO_NODE;
+	mq->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE |
+			    BLK_MQ_F_BLOCKING;
+	mq->tag_set.nr_hw_queues = 1;
+	mq->tag_set.cmd_size = sizeof(struct mmc_queue_req);
+	mq->tag_set.driver_data = mq;
+
+	ret = blk_mq_alloc_tag_set(&mq->tag_set);
+	if (ret)
 		return ret;
-	}
-
-	blk_queue_prep_rq(mq->queue, mmc_prep_request);
-
-	mmc_setup_queue(mq, card);
 
-	mq->thread = kthread_run(mmc_queue_thread, mq, "mmcqd/%d%s",
-		host->index, subname ? subname : "");
-
-	if (IS_ERR(mq->thread)) {
-		ret = PTR_ERR(mq->thread);
-		goto cleanup_queue;
+	mq->queue = blk_mq_init_queue(&mq->tag_set);
+	if (IS_ERR(mq->queue)) {
+		ret = PTR_ERR(mq->queue);
+		goto free_tag_set;
 	}
 
+	mq->queue->queue_lock = lock;
+	mq->queue->queuedata = mq;
+
 	return 0;
 
-cleanup_queue:
-	blk_cleanup_queue(mq->queue);
+free_tag_set:
+	blk_mq_free_tag_set(&mq->tag_set);
+
 	return ret;
 }
 
-void mmc_cleanup_queue(struct mmc_queue *mq)
-{
-	struct request_queue *q = mq->queue;
-	unsigned long flags;
+/* Set queue depth to get a reasonable value for q->nr_requests */
+#define MMC_QUEUE_DEPTH 64
 
-	/* Make sure the queue isn't suspended, as that will deadlock */
-	mmc_queue_resume(mq);
+static int mmc_mq_init(struct mmc_queue *mq, struct mmc_card *card,
+			 spinlock_t *lock)
+{
+	struct mmc_host *host = card->host;
+	int q_depth;
+	int ret;
+
+	/*
+	 * The queue depth for CQE must match the hardware because the request
+	 * tag is used to index the hardware queue.
+	 */
+	if (mq->use_cqe)
+		q_depth = min_t(int, card->ext_csd.cmdq_depth, host->cqe_qdepth);
+	else
+		q_depth = MMC_QUEUE_DEPTH;
+
+	ret = mmc_mq_init_queue(mq, q_depth, &mmc_mq_ops, lock);
+	if (ret)
+		return ret;
 
-	/* Then terminate our worker thread */
-	kthread_stop(mq->thread);
+	blk_queue_rq_timeout(mq->queue, 60 * HZ);
 
-	/* Empty the queue */
-	spin_lock_irqsave(q->queue_lock, flags);
-	q->queuedata = NULL;
-	blk_start_queue(q);
-	spin_unlock_irqrestore(q->queue_lock, flags);
+	mmc_setup_queue(mq, card);
 
-	mq->card = NULL;
+	return 0;
 }
-EXPORT_SYMBOL(mmc_cleanup_queue);
 
 /**
- * mmc_queue_suspend - suspend a MMC request queue
- * @mq: MMC queue to suspend
+ * mmc_init_queue - initialise a queue structure.
+ * @mq: mmc queue
+ * @card: mmc card to attach this queue
+ * @lock: queue lock
+ * @subname: partition subname
  *
- * Stop the block request queue, and wait for our thread to
- * complete any outstanding requests.  This ensures that we
- * won't suspend while a request is being processed.
+ * Initialise a MMC card request queue.
  */
-void mmc_queue_suspend(struct mmc_queue *mq)
+int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card,
+		   spinlock_t *lock, const char *subname)
 {
-	struct request_queue *q = mq->queue;
-	unsigned long flags;
+	struct mmc_host *host = card->host;
 
-	if (!mq->suspended) {
-		mq->suspended |= true;
+	mq->card = card;
 
-		spin_lock_irqsave(q->queue_lock, flags);
-		blk_stop_queue(q);
-		spin_unlock_irqrestore(q->queue_lock, flags);
+	mq->use_cqe = host->cqe_enabled;
 
-		down(&mq->thread_sem);
-	}
+	return mmc_mq_init(mq, card, lock);
+}
+
+void mmc_queue_suspend(struct mmc_queue *mq)
+{
+	blk_mq_quiesce_queue(mq->queue);
+
+	/*
+	 * The host remains claimed while there are outstanding requests, so
+	 * simply claiming and releasing here ensures there are none.
+	 */
+	mmc_claim_host(mq->card->host);
+	mmc_release_host(mq->card->host);
 }
 
-/**
- * mmc_queue_resume - resume a previously suspended MMC request queue
- * @mq: MMC queue to resume
- */
 void mmc_queue_resume(struct mmc_queue *mq)
+{
+	blk_mq_unquiesce_queue(mq->queue);
+}
+
+void mmc_cleanup_queue(struct mmc_queue *mq)
 {
 	struct request_queue *q = mq->queue;
-	unsigned long flags;
 
-	if (mq->suspended) {
-		mq->suspended = false;
+	/*
+	 * The legacy code handled the possibility of being suspended,
+	 * so do that here too.
+	 */
+	if (blk_queue_quiesced(q))
+		blk_mq_unquiesce_queue(q);
 
-		up(&mq->thread_sem);
+	blk_cleanup_queue(q);
 
-		spin_lock_irqsave(q->queue_lock, flags);
-		blk_start_queue(q);
-		spin_unlock_irqrestore(q->queue_lock, flags);
-	}
+	/*
+	 * A request can be completed before the next request, potentially
+	 * leaving a complete_work with nothing to do. Such a work item might
+	 * still be queued at this point. Flush it.
+	 */
+	flush_work(&mq->complete_work);
+
+	mq->card = NULL;
 }
 
 /*
diff --git a/drivers/mmc/core/queue.h b/drivers/mmc/core/queue.h
index 547b457c4251..17e59d50b496 100644
--- a/drivers/mmc/core/queue.h
+++ b/drivers/mmc/core/queue.h
@@ -8,6 +8,20 @@
 #include <linux/mmc/core.h>
 #include <linux/mmc/host.h>
 
+enum mmc_issued {
+	MMC_REQ_STARTED,
+	MMC_REQ_BUSY,
+	MMC_REQ_FAILED_TO_START,
+	MMC_REQ_FINISHED,
+};
+
+enum mmc_issue_type {
+	MMC_ISSUE_SYNC,
+	MMC_ISSUE_DCMD,
+	MMC_ISSUE_ASYNC,
+	MMC_ISSUE_MAX,
+};
+
 static inline struct mmc_queue_req *req_to_mmc_queue_req(struct request *rq)
 {
 	return blk_mq_rq_to_pdu(rq);
@@ -20,7 +34,6 @@ static inline struct request *mmc_queue_req_to_req(struct mmc_queue_req *mqr)
 	return blk_mq_rq_from_pdu(mqr);
 }
 
-struct task_struct;
 struct mmc_blk_data;
 struct mmc_blk_ioc_data;
 
@@ -30,7 +43,6 @@ struct mmc_blk_request {
 	struct mmc_command	cmd;
 	struct mmc_command	stop;
 	struct mmc_data		data;
-	int			retune_retry_done;
 };
 
 /**
@@ -52,28 +64,34 @@ enum mmc_drv_op {
 struct mmc_queue_req {
 	struct mmc_blk_request	brq;
 	struct scatterlist	*sg;
-	struct mmc_async_req	areq;
 	enum mmc_drv_op		drv_op;
 	int			drv_op_result;
 	void			*drv_op_data;
 	unsigned int		ioc_count;
+	int			retries;
 };
 
 struct mmc_queue {
 	struct mmc_card		*card;
-	struct task_struct	*thread;
-	struct semaphore	thread_sem;
-	bool			suspended;
-	bool			asleep;
+	struct mmc_ctx		ctx;
+	struct blk_mq_tag_set	tag_set;
 	struct mmc_blk_data	*blkdata;
 	struct request_queue	*queue;
-	/*
-	 * FIXME: this counter is not a very reliable way of keeping
-	 * track of how many requests that are ongoing. Switch to just
-	 * letting the block core keep track of requests and per-request
-	 * associated mmc_queue_req data.
-	 */
-	int			qcnt;
+	int			in_flight[MMC_ISSUE_MAX];
+	unsigned int		cqe_busy;
+#define MMC_CQE_DCMD_BUSY	BIT(0)
+#define MMC_CQE_QUEUE_FULL	BIT(1)
+	bool			use_cqe;
+	bool			recovery_needed;
+	bool			in_recovery;
+	bool			rw_wait;
+	bool			waiting;
+	struct work_struct	recovery_work;
+	wait_queue_head_t	wait;
+	struct request		*recovery_req;
+	struct request		*complete_req;
+	struct mutex		complete_lock;
+	struct work_struct	complete_work;
 };
 
 extern int mmc_init_queue(struct mmc_queue *, struct mmc_card *, spinlock_t *,
@@ -84,4 +102,22 @@ extern void mmc_queue_resume(struct mmc_queue *);
 extern unsigned int mmc_queue_map_sg(struct mmc_queue *,
 				     struct mmc_queue_req *);
 
+void mmc_cqe_check_busy(struct mmc_queue *mq);
+void mmc_cqe_recovery_notifier(struct mmc_request *mrq);
+
+enum mmc_issue_type mmc_issue_type(struct mmc_queue *mq, struct request *req);
+
+static inline int mmc_tot_in_flight(struct mmc_queue *mq)
+{
+	return mq->in_flight[MMC_ISSUE_SYNC] +
+	       mq->in_flight[MMC_ISSUE_DCMD] +
+	       mq->in_flight[MMC_ISSUE_ASYNC];
+}
+
+static inline int mmc_cqe_qcnt(struct mmc_queue *mq)
+{
+	return mq->in_flight[MMC_ISSUE_DCMD] +
+	       mq->in_flight[MMC_ISSUE_ASYNC];
+}
+
 #endif
diff --git a/drivers/mmc/core/quirks.h b/drivers/mmc/core/quirks.h
index 75d317623852..5153577754f0 100644
--- a/drivers/mmc/core/quirks.h
+++ b/drivers/mmc/core/quirks.h
@@ -109,6 +109,12 @@ static const struct mmc_fixup mmc_ext_csd_fixups[] = {
 	 */
 	MMC_FIXUP_EXT_CSD_REV(CID_NAME_ANY, CID_MANFID_HYNIX,
 			      0x014a, add_quirk, MMC_QUIRK_BROKEN_HPI, 5),
+	/*
+	 * Certain Micron (Numonyx) eMMC 4.5 cards might get broken when HPI
+	 * feature is used so disable the HPI feature for such buggy cards.
+	 */
+	MMC_FIXUP_EXT_CSD_REV(CID_NAME_ANY, CID_MANFID_NUMONYX,
+			      0x014e, add_quirk, MMC_QUIRK_BROKEN_HPI, 6),
 
 	END_FIXUP
 };
diff --git a/drivers/mmc/core/slot-gpio.c b/drivers/mmc/core/slot-gpio.c
index 863f1dbbfc1b..3698b0576009 100644
--- a/drivers/mmc/core/slot-gpio.c
+++ b/drivers/mmc/core/slot-gpio.c
@@ -121,20 +121,18 @@ EXPORT_SYMBOL(mmc_gpio_request_ro);
 void mmc_gpiod_request_cd_irq(struct mmc_host *host)
 {
 	struct mmc_gpio *ctx = host->slot.handler_priv;
-	int ret, irq;
+	int irq = -EINVAL;
+	int ret;
 
 	if (host->slot.cd_irq >= 0 || !ctx || !ctx->cd_gpio)
 		return;
 
-	irq = gpiod_to_irq(ctx->cd_gpio);
-
 	/*
-	 * Even if gpiod_to_irq() returns a valid IRQ number, the platform might
-	 * still prefer to poll, e.g., because that IRQ number is already used
-	 * by another unit and cannot be shared.
+	 * Do not use IRQ if the platform prefers to poll, e.g., because that
+	 * IRQ number is already used by another unit and cannot be shared.
 	 */
-	if (irq >= 0 && host->caps & MMC_CAP_NEEDS_POLL)
-		irq = -EINVAL;
+	if (!(host->caps & MMC_CAP_NEEDS_POLL))
+		irq = gpiod_to_irq(ctx->cd_gpio);
 
 	if (irq >= 0) {
 		if (!ctx->cd_gpio_isr)
@@ -307,3 +305,11 @@ int mmc_gpiod_request_ro(struct mmc_host *host, const char *con_id,
 	return 0;
 }
 EXPORT_SYMBOL(mmc_gpiod_request_ro);
+
+bool mmc_can_gpio_ro(struct mmc_host *host)
+{
+	struct mmc_gpio *ctx = host->slot.handler_priv;
+
+	return ctx->ro_gpio ? true : false;
+}
+EXPORT_SYMBOL(mmc_can_gpio_ro);
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 567028c9219a..620c2d90a646 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -81,6 +81,7 @@ config MMC_SDHCI_BIG_ENDIAN_32BIT_BYTE_SWAPPER
 config MMC_SDHCI_PCI
 	tristate "SDHCI support on PCI bus"
 	depends on MMC_SDHCI && PCI
+	select MMC_CQHCI
 	help
 	  This selects the PCI Secure Digital Host Controller Interface.
 	  Most controllers found today are PCI devices.
@@ -132,6 +133,7 @@ config MMC_SDHCI_OF_ARASAN
 	depends on MMC_SDHCI_PLTFM
 	depends on OF
 	depends on COMMON_CLK
+	select MMC_CQHCI
 	help
 	  This selects the Arasan Secure Digital Host Controller Interface
 	  (SDHCI). This hardware is found e.g. in Xilinx' Zynq SoC.
@@ -320,7 +322,7 @@ config MMC_SDHCI_BCM_KONA
 config MMC_SDHCI_F_SDH30
 	tristate "SDHCI support for Fujitsu Semiconductor F_SDH30"
 	depends on MMC_SDHCI_PLTFM
-	depends on OF
+	depends on OF || ACPI
 	help
 	  This selects the Secure Digital Host Controller Interface (SDHCI)
 	  Needed by some Fujitsu SoC for MMC / SD / SDIO support.
@@ -595,18 +597,16 @@ config MMC_TMIO
 
 config MMC_SDHI
 	tristate "Renesas SDHI SD/SDIO controller support"
-	depends on SUPERH || ARM || ARM64
 	depends on SUPERH || ARCH_RENESAS || COMPILE_TEST
 	select MMC_TMIO_CORE
-	select MMC_SDHI_SYS_DMAC if (SUPERH || ARM)
-	select MMC_SDHI_INTERNAL_DMAC if ARM64
 	help
 	  This provides support for the SDHI SD/SDIO controller found in
 	  Renesas SuperH, ARM and ARM64 based SoCs
 
 config MMC_SDHI_SYS_DMAC
 	tristate "DMA for SDHI SD/SDIO controllers using SYS-DMAC"
-	depends on MMC_SDHI
+	depends on MMC_SDHI && HAS_DMA
+	default MMC_SDHI if (SUPERH || ARM)
 	help
 	  This provides DMA support for SDHI SD/SDIO controllers
 	  using SYS-DMAC via DMA Engine. This supports the controllers
@@ -615,7 +615,8 @@ config MMC_SDHI_SYS_DMAC
 config MMC_SDHI_INTERNAL_DMAC
 	tristate "DMA for SDHI SD/SDIO controllers using on-chip bus mastering"
 	depends on ARM64 || COMPILE_TEST
-	depends on MMC_SDHI
+	depends on MMC_SDHI && HAS_DMA
+	default MMC_SDHI if ARM64
 	help
 	  This provides DMA support for SDHI SD/SDIO controllers
 	  using on-chip bus mastering. This supports the controllers
@@ -838,14 +839,14 @@ config MMC_USDHI6ROL0
 
 config MMC_REALTEK_PCI
 	tristate "Realtek PCI-E SD/MMC Card Interface Driver"
-	depends on MFD_RTSX_PCI
+	depends on MISC_RTSX_PCI
 	help
 	  Say Y here to include driver code to support SD/MMC card interface
 	  of Realtek PCI-E card reader
 
 config MMC_REALTEK_USB
 	tristate "Realtek USB SD/MMC Card Interface Driver"
-	depends on MFD_RTSX_USB
+	depends on MISC_RTSX_USB
 	help
 	  Say Y here to include driver code to support SD/MMC card interface
 	  of Realtek RTS5129/39 series card reader
@@ -857,10 +858,22 @@ config MMC_SUNXI
 	  This selects support for the SD/MMC Host Controller on
 	  Allwinner sunxi SoCs.
 
+config MMC_CQHCI
+	tristate "Command Queue Host Controller Interface support"
+	depends on HAS_DMA
+	help
+	  This selects the Command Queue Host Controller Interface (CQHCI)
+	  support present in host controllers of Qualcomm Technologies, Inc
+	  amongst others.
+	  This controller supports eMMC devices with command queue support.
+
+	  If you have a controller with this interface, say Y or M here.
+
+	  If unsure, say N.
+
 config MMC_TOSHIBA_PCI
 	tristate "Toshiba Type A SD/MMC Card Interface Driver"
 	depends on PCI
-	help
 
 config MMC_BCM2835
 	tristate "Broadcom BCM2835 SDHOST MMC Controller support"
diff --git a/drivers/mmc/host/Makefile b/drivers/mmc/host/Makefile
index a43cf0d5a5d3..84cd1388abc3 100644
--- a/drivers/mmc/host/Makefile
+++ b/drivers/mmc/host/Makefile
@@ -11,7 +11,7 @@ obj-$(CONFIG_MMC_MXC)		+= mxcmmc.o
 obj-$(CONFIG_MMC_MXS)		+= mxs-mmc.o
 obj-$(CONFIG_MMC_SDHCI)		+= sdhci.o
 obj-$(CONFIG_MMC_SDHCI_PCI)	+= sdhci-pci.o
-sdhci-pci-y			+= sdhci-pci-core.o sdhci-pci-o2micro.o
+sdhci-pci-y			+= sdhci-pci-core.o sdhci-pci-o2micro.o sdhci-pci-arasan.o
 obj-$(subst m,y,$(CONFIG_MMC_SDHCI_PCI))	+= sdhci-pci-data.o
 obj-$(CONFIG_MMC_SDHCI_ACPI)	+= sdhci-acpi.o
 obj-$(CONFIG_MMC_SDHCI_PXAV3)	+= sdhci-pxav3.o
@@ -39,12 +39,8 @@ obj-$(CONFIG_MMC_SDRICOH_CS)	+= sdricoh_cs.o
 obj-$(CONFIG_MMC_TMIO)		+= tmio_mmc.o
 obj-$(CONFIG_MMC_TMIO_CORE)	+= tmio_mmc_core.o
 obj-$(CONFIG_MMC_SDHI)		+= renesas_sdhi_core.o
-ifeq ($(subst m,y,$(CONFIG_MMC_SDHI_SYS_DMAC)),y)
-obj-$(CONFIG_MMC_SDHI)		+= renesas_sdhi_sys_dmac.o
-endif
-ifeq ($(subst m,y,$(CONFIG_MMC_SDHI_INTERNAL_DMAC)),y)
-obj-$(CONFIG_MMC_SDHI)		+= renesas_sdhi_internal_dmac.o
-endif
+obj-$(CONFIG_MMC_SDHI_SYS_DMAC)		+= renesas_sdhi_sys_dmac.o
+obj-$(CONFIG_MMC_SDHI_INTERNAL_DMAC)	+= renesas_sdhi_internal_dmac.o
 obj-$(CONFIG_MMC_CB710)		+= cb710-mmc.o
 obj-$(CONFIG_MMC_VIA_SDMMC)	+= via-sdmmc.o
 obj-$(CONFIG_SDH_BFIN)		+= bfin_sdh.o
@@ -92,6 +88,7 @@ obj-$(CONFIG_MMC_SDHCI_ST)		+= sdhci-st.o
 obj-$(CONFIG_MMC_SDHCI_MICROCHIP_PIC32)	+= sdhci-pic32.o
 obj-$(CONFIG_MMC_SDHCI_BRCMSTB)		+= sdhci-brcmstb.o
 obj-$(CONFIG_MMC_SDHCI_OMAP)		+= sdhci-omap.o
+obj-$(CONFIG_MMC_CQHCI)			+= cqhci.o
 
 ifeq ($(CONFIG_CB710_DEBUG),y)
 	CFLAGS-cb710-mmc	+= -DDEBUG
diff --git a/drivers/mmc/host/android-goldfish.c b/drivers/mmc/host/android-goldfish.c
index 63fe5091ca59..63d27589cd89 100644
--- a/drivers/mmc/host/android-goldfish.c
+++ b/drivers/mmc/host/android-goldfish.c
@@ -42,13 +42,11 @@
 #include <linux/spinlock.h>
 #include <linux/timer.h>
 #include <linux/clk.h>
-#include <linux/scatterlist.h>
 
 #include <asm/io.h>
 #include <asm/irq.h>
 
 #include <asm/types.h>
-#include <asm/io.h>
 #include <linux/uaccess.h>
 
 #define DRIVER_NAME "goldfish_mmc"
diff --git a/drivers/mmc/host/bcm2835.c b/drivers/mmc/host/bcm2835.c
index 229dc18f0581..768972af8b85 100644
--- a/drivers/mmc/host/bcm2835.c
+++ b/drivers/mmc/host/bcm2835.c
@@ -1265,7 +1265,8 @@ static int bcm2835_add_host(struct bcm2835_host *host)
 	char pio_limit_string[20];
 	int ret;
 
-	mmc->f_max = host->max_clk;
+	if (!mmc->f_max || mmc->f_max > host->max_clk)
+		mmc->f_max = host->max_clk;
 	mmc->f_min = host->max_clk / SDCDIV_MAX_CDIV;
 
 	mmc->max_busy_timeout = ~0 / (mmc->f_max / 1000);
diff --git a/drivers/mmc/host/cqhci.c b/drivers/mmc/host/cqhci.c
new file mode 100644
index 000000000000..159270e947cf
--- /dev/null
+++ b/drivers/mmc/host/cqhci.c
@@ -0,0 +1,1150 @@
+/* Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/delay.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/scatterlist.h>
+#include <linux/platform_device.h>
+#include <linux/ktime.h>
+
+#include <linux/mmc/mmc.h>
+#include <linux/mmc/host.h>
+#include <linux/mmc/card.h>
+
+#include "cqhci.h"
+
+#define DCMD_SLOT 31
+#define NUM_SLOTS 32
+
+struct cqhci_slot {
+	struct mmc_request *mrq;
+	unsigned int flags;
+#define CQHCI_EXTERNAL_TIMEOUT	BIT(0)
+#define CQHCI_COMPLETED		BIT(1)
+#define CQHCI_HOST_CRC		BIT(2)
+#define CQHCI_HOST_TIMEOUT	BIT(3)
+#define CQHCI_HOST_OTHER	BIT(4)
+};
+
+static inline u8 *get_desc(struct cqhci_host *cq_host, u8 tag)
+{
+	return cq_host->desc_base + (tag * cq_host->slot_sz);
+}
+
+static inline u8 *get_link_desc(struct cqhci_host *cq_host, u8 tag)
+{
+	u8 *desc = get_desc(cq_host, tag);
+
+	return desc + cq_host->task_desc_len;
+}
+
+static inline dma_addr_t get_trans_desc_dma(struct cqhci_host *cq_host, u8 tag)
+{
+	return cq_host->trans_desc_dma_base +
+		(cq_host->mmc->max_segs * tag *
+		 cq_host->trans_desc_len);
+}
+
+static inline u8 *get_trans_desc(struct cqhci_host *cq_host, u8 tag)
+{
+	return cq_host->trans_desc_base +
+		(cq_host->trans_desc_len * cq_host->mmc->max_segs * tag);
+}
+
+static void setup_trans_desc(struct cqhci_host *cq_host, u8 tag)
+{
+	u8 *link_temp;
+	dma_addr_t trans_temp;
+
+	link_temp = get_link_desc(cq_host, tag);
+	trans_temp = get_trans_desc_dma(cq_host, tag);
+
+	memset(link_temp, 0, cq_host->link_desc_len);
+	if (cq_host->link_desc_len > 8)
+		*(link_temp + 8) = 0;
+
+	if (tag == DCMD_SLOT && (cq_host->mmc->caps2 & MMC_CAP2_CQE_DCMD)) {
+		*link_temp = CQHCI_VALID(0) | CQHCI_ACT(0) | CQHCI_END(1);
+		return;
+	}
+
+	*link_temp = CQHCI_VALID(1) | CQHCI_ACT(0x6) | CQHCI_END(0);
+
+	if (cq_host->dma64) {
+		__le64 *data_addr = (__le64 __force *)(link_temp + 4);
+
+		data_addr[0] = cpu_to_le64(trans_temp);
+	} else {
+		__le32 *data_addr = (__le32 __force *)(link_temp + 4);
+
+		data_addr[0] = cpu_to_le32(trans_temp);
+	}
+}
+
+static void cqhci_set_irqs(struct cqhci_host *cq_host, u32 set)
+{
+	cqhci_writel(cq_host, set, CQHCI_ISTE);
+	cqhci_writel(cq_host, set, CQHCI_ISGE);
+}
+
+#define DRV_NAME "cqhci"
+
+#define CQHCI_DUMP(f, x...) \
+	pr_err("%s: " DRV_NAME ": " f, mmc_hostname(mmc), ## x)
+
+static void cqhci_dumpregs(struct cqhci_host *cq_host)
+{
+	struct mmc_host *mmc = cq_host->mmc;
+
+	CQHCI_DUMP("============ CQHCI REGISTER DUMP ===========\n");
+
+	CQHCI_DUMP("Caps:      0x%08x | Version:  0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_CAP),
+		   cqhci_readl(cq_host, CQHCI_VER));
+	CQHCI_DUMP("Config:    0x%08x | Control:  0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_CFG),
+		   cqhci_readl(cq_host, CQHCI_CTL));
+	CQHCI_DUMP("Int stat:  0x%08x | Int enab: 0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_IS),
+		   cqhci_readl(cq_host, CQHCI_ISTE));
+	CQHCI_DUMP("Int sig:   0x%08x | Int Coal: 0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_ISGE),
+		   cqhci_readl(cq_host, CQHCI_IC));
+	CQHCI_DUMP("TDL base:  0x%08x | TDL up32: 0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_TDLBA),
+		   cqhci_readl(cq_host, CQHCI_TDLBAU));
+	CQHCI_DUMP("Doorbell:  0x%08x | TCN:      0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_TDBR),
+		   cqhci_readl(cq_host, CQHCI_TCN));
+	CQHCI_DUMP("Dev queue: 0x%08x | Dev Pend: 0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_DQS),
+		   cqhci_readl(cq_host, CQHCI_DPT));
+	CQHCI_DUMP("Task clr:  0x%08x | SSC1:     0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_TCLR),
+		   cqhci_readl(cq_host, CQHCI_SSC1));
+	CQHCI_DUMP("SSC2:      0x%08x | DCMD rsp: 0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_SSC2),
+		   cqhci_readl(cq_host, CQHCI_CRDCT));
+	CQHCI_DUMP("RED mask:  0x%08x | TERRI:    0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_RMEM),
+		   cqhci_readl(cq_host, CQHCI_TERRI));
+	CQHCI_DUMP("Resp idx:  0x%08x | Resp arg: 0x%08x\n",
+		   cqhci_readl(cq_host, CQHCI_CRI),
+		   cqhci_readl(cq_host, CQHCI_CRA));
+
+	if (cq_host->ops->dumpregs)
+		cq_host->ops->dumpregs(mmc);
+	else
+		CQHCI_DUMP(": ===========================================\n");
+}
+
+/**
+ * The allocated descriptor table for task, link & transfer descritors
+ * looks like:
+ * |----------|
+ * |task desc |  |->|----------|
+ * |----------|  |  |trans desc|
+ * |link desc-|->|  |----------|
+ * |----------|          .
+ *      .                .
+ *  no. of slots      max-segs
+ *      .           |----------|
+ * |----------|
+ * The idea here is to create the [task+trans] table and mark & point the
+ * link desc to the transfer desc table on a per slot basis.
+ */
+static int cqhci_host_alloc_tdl(struct cqhci_host *cq_host)
+{
+	int i = 0;
+
+	/* task descriptor can be 64/128 bit irrespective of arch */
+	if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) {
+		cqhci_writel(cq_host, cqhci_readl(cq_host, CQHCI_CFG) |
+			       CQHCI_TASK_DESC_SZ, CQHCI_CFG);
+		cq_host->task_desc_len = 16;
+	} else {
+		cq_host->task_desc_len = 8;
+	}
+
+	/*
+	 * 96 bits length of transfer desc instead of 128 bits which means
+	 * ADMA would expect next valid descriptor at the 96th bit
+	 * or 128th bit
+	 */
+	if (cq_host->dma64) {
+		if (cq_host->quirks & CQHCI_QUIRK_SHORT_TXFR_DESC_SZ)
+			cq_host->trans_desc_len = 12;
+		else
+			cq_host->trans_desc_len = 16;
+		cq_host->link_desc_len = 16;
+	} else {
+		cq_host->trans_desc_len = 8;
+		cq_host->link_desc_len = 8;
+	}
+
+	/* total size of a slot: 1 task & 1 transfer (link) */
+	cq_host->slot_sz = cq_host->task_desc_len + cq_host->link_desc_len;
+
+	cq_host->desc_size = cq_host->slot_sz * cq_host->num_slots;
+
+	cq_host->data_size = cq_host->trans_desc_len * cq_host->mmc->max_segs *
+		(cq_host->num_slots - 1);
+
+	pr_debug("%s: cqhci: desc_size: %zu data_sz: %zu slot-sz: %d\n",
+		 mmc_hostname(cq_host->mmc), cq_host->desc_size, cq_host->data_size,
+		 cq_host->slot_sz);
+
+	/*
+	 * allocate a dma-mapped chunk of memory for the descriptors
+	 * allocate a dma-mapped chunk of memory for link descriptors
+	 * setup each link-desc memory offset per slot-number to
+	 * the descriptor table.
+	 */
+	cq_host->desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc),
+						 cq_host->desc_size,
+						 &cq_host->desc_dma_base,
+						 GFP_KERNEL);
+	cq_host->trans_desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc),
+					      cq_host->data_size,
+					      &cq_host->trans_desc_dma_base,
+					      GFP_KERNEL);
+	if (!cq_host->desc_base || !cq_host->trans_desc_base)
+		return -ENOMEM;
+
+	pr_debug("%s: cqhci: desc-base: 0x%p trans-base: 0x%p\n desc_dma 0x%llx trans_dma: 0x%llx\n",
+		 mmc_hostname(cq_host->mmc), cq_host->desc_base, cq_host->trans_desc_base,
+		(unsigned long long)cq_host->desc_dma_base,
+		(unsigned long long)cq_host->trans_desc_dma_base);
+
+	for (; i < (cq_host->num_slots); i++)
+		setup_trans_desc(cq_host, i);
+
+	return 0;
+}
+
+static void __cqhci_enable(struct cqhci_host *cq_host)
+{
+	struct mmc_host *mmc = cq_host->mmc;
+	u32 cqcfg;
+
+	cqcfg = cqhci_readl(cq_host, CQHCI_CFG);
+
+	/* Configuration must not be changed while enabled */
+	if (cqcfg & CQHCI_ENABLE) {
+		cqcfg &= ~CQHCI_ENABLE;
+		cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+	}
+
+	cqcfg &= ~(CQHCI_DCMD | CQHCI_TASK_DESC_SZ);
+
+	if (mmc->caps2 & MMC_CAP2_CQE_DCMD)
+		cqcfg |= CQHCI_DCMD;
+
+	if (cq_host->caps & CQHCI_TASK_DESC_SZ_128)
+		cqcfg |= CQHCI_TASK_DESC_SZ;
+
+	cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+
+	cqhci_writel(cq_host, lower_32_bits(cq_host->desc_dma_base),
+		     CQHCI_TDLBA);
+	cqhci_writel(cq_host, upper_32_bits(cq_host->desc_dma_base),
+		     CQHCI_TDLBAU);
+
+	cqhci_writel(cq_host, cq_host->rca, CQHCI_SSC2);
+
+	cqhci_set_irqs(cq_host, 0);
+
+	cqcfg |= CQHCI_ENABLE;
+
+	cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+
+	mmc->cqe_on = true;
+
+	if (cq_host->ops->enable)
+		cq_host->ops->enable(mmc);
+
+	/* Ensure all writes are done before interrupts are enabled */
+	wmb();
+
+	cqhci_set_irqs(cq_host, CQHCI_IS_MASK);
+
+	cq_host->activated = true;
+}
+
+static void __cqhci_disable(struct cqhci_host *cq_host)
+{
+	u32 cqcfg;
+
+	cqcfg = cqhci_readl(cq_host, CQHCI_CFG);
+	cqcfg &= ~CQHCI_ENABLE;
+	cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+
+	cq_host->mmc->cqe_on = false;
+
+	cq_host->activated = false;
+}
+
+int cqhci_suspend(struct mmc_host *mmc)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+
+	if (cq_host->enabled)
+		__cqhci_disable(cq_host);
+
+	return 0;
+}
+EXPORT_SYMBOL(cqhci_suspend);
+
+int cqhci_resume(struct mmc_host *mmc)
+{
+	/* Re-enable is done upon first request */
+	return 0;
+}
+EXPORT_SYMBOL(cqhci_resume);
+
+static int cqhci_enable(struct mmc_host *mmc, struct mmc_card *card)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	int err;
+
+	if (cq_host->enabled)
+		return 0;
+
+	cq_host->rca = card->rca;
+
+	err = cqhci_host_alloc_tdl(cq_host);
+	if (err)
+		return err;
+
+	__cqhci_enable(cq_host);
+
+	cq_host->enabled = true;
+
+#ifdef DEBUG
+	cqhci_dumpregs(cq_host);
+#endif
+	return 0;
+}
+
+/* CQHCI is idle and should halt immediately, so set a small timeout */
+#define CQHCI_OFF_TIMEOUT 100
+
+static void cqhci_off(struct mmc_host *mmc)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	ktime_t timeout;
+	bool timed_out;
+	u32 reg;
+
+	if (!cq_host->enabled || !mmc->cqe_on || cq_host->recovery_halt)
+		return;
+
+	if (cq_host->ops->disable)
+		cq_host->ops->disable(mmc, false);
+
+	cqhci_writel(cq_host, CQHCI_HALT, CQHCI_CTL);
+
+	timeout = ktime_add_us(ktime_get(), CQHCI_OFF_TIMEOUT);
+	while (1) {
+		timed_out = ktime_compare(ktime_get(), timeout) > 0;
+		reg = cqhci_readl(cq_host, CQHCI_CTL);
+		if ((reg & CQHCI_HALT) || timed_out)
+			break;
+	}
+
+	if (timed_out)
+		pr_err("%s: cqhci: CQE stuck on\n", mmc_hostname(mmc));
+	else
+		pr_debug("%s: cqhci: CQE off\n", mmc_hostname(mmc));
+
+	mmc->cqe_on = false;
+}
+
+static void cqhci_disable(struct mmc_host *mmc)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+
+	if (!cq_host->enabled)
+		return;
+
+	cqhci_off(mmc);
+
+	__cqhci_disable(cq_host);
+
+	dmam_free_coherent(mmc_dev(mmc), cq_host->data_size,
+			   cq_host->trans_desc_base,
+			   cq_host->trans_desc_dma_base);
+
+	dmam_free_coherent(mmc_dev(mmc), cq_host->desc_size,
+			   cq_host->desc_base,
+			   cq_host->desc_dma_base);
+
+	cq_host->trans_desc_base = NULL;
+	cq_host->desc_base = NULL;
+
+	cq_host->enabled = false;
+}
+
+static void cqhci_prep_task_desc(struct mmc_request *mrq,
+					u64 *data, bool intr)
+{
+	u32 req_flags = mrq->data->flags;
+
+	*data = CQHCI_VALID(1) |
+		CQHCI_END(1) |
+		CQHCI_INT(intr) |
+		CQHCI_ACT(0x5) |
+		CQHCI_FORCED_PROG(!!(req_flags & MMC_DATA_FORCED_PRG)) |
+		CQHCI_DATA_TAG(!!(req_flags & MMC_DATA_DAT_TAG)) |
+		CQHCI_DATA_DIR(!!(req_flags & MMC_DATA_READ)) |
+		CQHCI_PRIORITY(!!(req_flags & MMC_DATA_PRIO)) |
+		CQHCI_QBAR(!!(req_flags & MMC_DATA_QBR)) |
+		CQHCI_REL_WRITE(!!(req_flags & MMC_DATA_REL_WR)) |
+		CQHCI_BLK_COUNT(mrq->data->blocks) |
+		CQHCI_BLK_ADDR((u64)mrq->data->blk_addr);
+
+	pr_debug("%s: cqhci: tag %d task descriptor 0x016%llx\n",
+		 mmc_hostname(mrq->host), mrq->tag, (unsigned long long)*data);
+}
+
+static int cqhci_dma_map(struct mmc_host *host, struct mmc_request *mrq)
+{
+	int sg_count;
+	struct mmc_data *data = mrq->data;
+
+	if (!data)
+		return -EINVAL;
+
+	sg_count = dma_map_sg(mmc_dev(host), data->sg,
+			      data->sg_len,
+			      (data->flags & MMC_DATA_WRITE) ?
+			      DMA_TO_DEVICE : DMA_FROM_DEVICE);
+	if (!sg_count) {
+		pr_err("%s: sg-len: %d\n", __func__, data->sg_len);
+		return -ENOMEM;
+	}
+
+	return sg_count;
+}
+
+static void cqhci_set_tran_desc(u8 *desc, dma_addr_t addr, int len, bool end,
+				bool dma64)
+{
+	__le32 *attr = (__le32 __force *)desc;
+
+	*attr = (CQHCI_VALID(1) |
+		 CQHCI_END(end ? 1 : 0) |
+		 CQHCI_INT(0) |
+		 CQHCI_ACT(0x4) |
+		 CQHCI_DAT_LENGTH(len));
+
+	if (dma64) {
+		__le64 *dataddr = (__le64 __force *)(desc + 4);
+
+		dataddr[0] = cpu_to_le64(addr);
+	} else {
+		__le32 *dataddr = (__le32 __force *)(desc + 4);
+
+		dataddr[0] = cpu_to_le32(addr);
+	}
+}
+
+static int cqhci_prep_tran_desc(struct mmc_request *mrq,
+			       struct cqhci_host *cq_host, int tag)
+{
+	struct mmc_data *data = mrq->data;
+	int i, sg_count, len;
+	bool end = false;
+	bool dma64 = cq_host->dma64;
+	dma_addr_t addr;
+	u8 *desc;
+	struct scatterlist *sg;
+
+	sg_count = cqhci_dma_map(mrq->host, mrq);
+	if (sg_count < 0) {
+		pr_err("%s: %s: unable to map sg lists, %d\n",
+				mmc_hostname(mrq->host), __func__, sg_count);
+		return sg_count;
+	}
+
+	desc = get_trans_desc(cq_host, tag);
+
+	for_each_sg(data->sg, sg, sg_count, i) {
+		addr = sg_dma_address(sg);
+		len = sg_dma_len(sg);
+
+		if ((i+1) == sg_count)
+			end = true;
+		cqhci_set_tran_desc(desc, addr, len, end, dma64);
+		desc += cq_host->trans_desc_len;
+	}
+
+	return 0;
+}
+
+static void cqhci_prep_dcmd_desc(struct mmc_host *mmc,
+				   struct mmc_request *mrq)
+{
+	u64 *task_desc = NULL;
+	u64 data = 0;
+	u8 resp_type;
+	u8 *desc;
+	__le64 *dataddr;
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	u8 timing;
+
+	if (!(mrq->cmd->flags & MMC_RSP_PRESENT)) {
+		resp_type = 0x0;
+		timing = 0x1;
+	} else {
+		if (mrq->cmd->flags & MMC_RSP_R1B) {
+			resp_type = 0x3;
+			timing = 0x0;
+		} else {
+			resp_type = 0x2;
+			timing = 0x1;
+		}
+	}
+
+	task_desc = (__le64 __force *)get_desc(cq_host, cq_host->dcmd_slot);
+	memset(task_desc, 0, cq_host->task_desc_len);
+	data |= (CQHCI_VALID(1) |
+		 CQHCI_END(1) |
+		 CQHCI_INT(1) |
+		 CQHCI_QBAR(1) |
+		 CQHCI_ACT(0x5) |
+		 CQHCI_CMD_INDEX(mrq->cmd->opcode) |
+		 CQHCI_CMD_TIMING(timing) | CQHCI_RESP_TYPE(resp_type));
+	*task_desc |= data;
+	desc = (u8 *)task_desc;
+	pr_debug("%s: cqhci: dcmd: cmd: %d timing: %d resp: %d\n",
+		 mmc_hostname(mmc), mrq->cmd->opcode, timing, resp_type);
+	dataddr = (__le64 __force *)(desc + 4);
+	dataddr[0] = cpu_to_le64((u64)mrq->cmd->arg);
+
+}
+
+static void cqhci_post_req(struct mmc_host *host, struct mmc_request *mrq)
+{
+	struct mmc_data *data = mrq->data;
+
+	if (data) {
+		dma_unmap_sg(mmc_dev(host), data->sg, data->sg_len,
+			     (data->flags & MMC_DATA_READ) ?
+			     DMA_FROM_DEVICE : DMA_TO_DEVICE);
+	}
+}
+
+static inline int cqhci_tag(struct mmc_request *mrq)
+{
+	return mrq->cmd ? DCMD_SLOT : mrq->tag;
+}
+
+static int cqhci_request(struct mmc_host *mmc, struct mmc_request *mrq)
+{
+	int err = 0;
+	u64 data = 0;
+	u64 *task_desc = NULL;
+	int tag = cqhci_tag(mrq);
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	unsigned long flags;
+
+	if (!cq_host->enabled) {
+		pr_err("%s: cqhci: not enabled\n", mmc_hostname(mmc));
+		return -EINVAL;
+	}
+
+	/* First request after resume has to re-enable */
+	if (!cq_host->activated)
+		__cqhci_enable(cq_host);
+
+	if (!mmc->cqe_on) {
+		cqhci_writel(cq_host, 0, CQHCI_CTL);
+		mmc->cqe_on = true;
+		pr_debug("%s: cqhci: CQE on\n", mmc_hostname(mmc));
+		if (cqhci_readl(cq_host, CQHCI_CTL) && CQHCI_HALT) {
+			pr_err("%s: cqhci: CQE failed to exit halt state\n",
+			       mmc_hostname(mmc));
+		}
+		if (cq_host->ops->enable)
+			cq_host->ops->enable(mmc);
+	}
+
+	if (mrq->data) {
+		task_desc = (__le64 __force *)get_desc(cq_host, tag);
+		cqhci_prep_task_desc(mrq, &data, 1);
+		*task_desc = cpu_to_le64(data);
+		err = cqhci_prep_tran_desc(mrq, cq_host, tag);
+		if (err) {
+			pr_err("%s: cqhci: failed to setup tx desc: %d\n",
+			       mmc_hostname(mmc), err);
+			return err;
+		}
+	} else {
+		cqhci_prep_dcmd_desc(mmc, mrq);
+	}
+
+	spin_lock_irqsave(&cq_host->lock, flags);
+
+	if (cq_host->recovery_halt) {
+		err = -EBUSY;
+		goto out_unlock;
+	}
+
+	cq_host->slot[tag].mrq = mrq;
+	cq_host->slot[tag].flags = 0;
+
+	cq_host->qcnt += 1;
+
+	cqhci_writel(cq_host, 1 << tag, CQHCI_TDBR);
+	if (!(cqhci_readl(cq_host, CQHCI_TDBR) & (1 << tag)))
+		pr_debug("%s: cqhci: doorbell not set for tag %d\n",
+			 mmc_hostname(mmc), tag);
+out_unlock:
+	spin_unlock_irqrestore(&cq_host->lock, flags);
+
+	if (err)
+		cqhci_post_req(mmc, mrq);
+
+	return err;
+}
+
+static void cqhci_recovery_needed(struct mmc_host *mmc, struct mmc_request *mrq,
+				  bool notify)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+
+	if (!cq_host->recovery_halt) {
+		cq_host->recovery_halt = true;
+		pr_debug("%s: cqhci: recovery needed\n", mmc_hostname(mmc));
+		wake_up(&cq_host->wait_queue);
+		if (notify && mrq->recovery_notifier)
+			mrq->recovery_notifier(mrq);
+	}
+}
+
+static unsigned int cqhci_error_flags(int error1, int error2)
+{
+	int error = error1 ? error1 : error2;
+
+	switch (error) {
+	case -EILSEQ:
+		return CQHCI_HOST_CRC;
+	case -ETIMEDOUT:
+		return CQHCI_HOST_TIMEOUT;
+	default:
+		return CQHCI_HOST_OTHER;
+	}
+}
+
+static void cqhci_error_irq(struct mmc_host *mmc, u32 status, int cmd_error,
+			    int data_error)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	struct cqhci_slot *slot;
+	u32 terri;
+	int tag;
+
+	spin_lock(&cq_host->lock);
+
+	terri = cqhci_readl(cq_host, CQHCI_TERRI);
+
+	pr_debug("%s: cqhci: error IRQ status: 0x%08x cmd error %d data error %d TERRI: 0x%08x\n",
+		 mmc_hostname(mmc), status, cmd_error, data_error, terri);
+
+	/* Forget about errors when recovery has already been triggered */
+	if (cq_host->recovery_halt)
+		goto out_unlock;
+
+	if (!cq_host->qcnt) {
+		WARN_ONCE(1, "%s: cqhci: error when idle. IRQ status: 0x%08x cmd error %d data error %d TERRI: 0x%08x\n",
+			  mmc_hostname(mmc), status, cmd_error, data_error,
+			  terri);
+		goto out_unlock;
+	}
+
+	if (CQHCI_TERRI_C_VALID(terri)) {
+		tag = CQHCI_TERRI_C_TASK(terri);
+		slot = &cq_host->slot[tag];
+		if (slot->mrq) {
+			slot->flags = cqhci_error_flags(cmd_error, data_error);
+			cqhci_recovery_needed(mmc, slot->mrq, true);
+		}
+	}
+
+	if (CQHCI_TERRI_D_VALID(terri)) {
+		tag = CQHCI_TERRI_D_TASK(terri);
+		slot = &cq_host->slot[tag];
+		if (slot->mrq) {
+			slot->flags = cqhci_error_flags(data_error, cmd_error);
+			cqhci_recovery_needed(mmc, slot->mrq, true);
+		}
+	}
+
+	if (!cq_host->recovery_halt) {
+		/*
+		 * The only way to guarantee forward progress is to mark at
+		 * least one task in error, so if none is indicated, pick one.
+		 */
+		for (tag = 0; tag < NUM_SLOTS; tag++) {
+			slot = &cq_host->slot[tag];
+			if (!slot->mrq)
+				continue;
+			slot->flags = cqhci_error_flags(data_error, cmd_error);
+			cqhci_recovery_needed(mmc, slot->mrq, true);
+			break;
+		}
+	}
+
+out_unlock:
+	spin_unlock(&cq_host->lock);
+}
+
+static void cqhci_finish_mrq(struct mmc_host *mmc, unsigned int tag)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	struct cqhci_slot *slot = &cq_host->slot[tag];
+	struct mmc_request *mrq = slot->mrq;
+	struct mmc_data *data;
+
+	if (!mrq) {
+		WARN_ONCE(1, "%s: cqhci: spurious TCN for tag %d\n",
+			  mmc_hostname(mmc), tag);
+		return;
+	}
+
+	/* No completions allowed during recovery */
+	if (cq_host->recovery_halt) {
+		slot->flags |= CQHCI_COMPLETED;
+		return;
+	}
+
+	slot->mrq = NULL;
+
+	cq_host->qcnt -= 1;
+
+	data = mrq->data;
+	if (data) {
+		if (data->error)
+			data->bytes_xfered = 0;
+		else
+			data->bytes_xfered = data->blksz * data->blocks;
+	}
+
+	mmc_cqe_request_done(mmc, mrq);
+}
+
+irqreturn_t cqhci_irq(struct mmc_host *mmc, u32 intmask, int cmd_error,
+		      int data_error)
+{
+	u32 status;
+	unsigned long tag = 0, comp_status;
+	struct cqhci_host *cq_host = mmc->cqe_private;
+
+	status = cqhci_readl(cq_host, CQHCI_IS);
+	cqhci_writel(cq_host, status, CQHCI_IS);
+
+	pr_debug("%s: cqhci: IRQ status: 0x%08x\n", mmc_hostname(mmc), status);
+
+	if ((status & CQHCI_IS_RED) || cmd_error || data_error)
+		cqhci_error_irq(mmc, status, cmd_error, data_error);
+
+	if (status & CQHCI_IS_TCC) {
+		/* read TCN and complete the request */
+		comp_status = cqhci_readl(cq_host, CQHCI_TCN);
+		cqhci_writel(cq_host, comp_status, CQHCI_TCN);
+		pr_debug("%s: cqhci: TCN: 0x%08lx\n",
+			 mmc_hostname(mmc), comp_status);
+
+		spin_lock(&cq_host->lock);
+
+		for_each_set_bit(tag, &comp_status, cq_host->num_slots) {
+			/* complete the corresponding mrq */
+			pr_debug("%s: cqhci: completing tag %lu\n",
+				 mmc_hostname(mmc), tag);
+			cqhci_finish_mrq(mmc, tag);
+		}
+
+		if (cq_host->waiting_for_idle && !cq_host->qcnt) {
+			cq_host->waiting_for_idle = false;
+			wake_up(&cq_host->wait_queue);
+		}
+
+		spin_unlock(&cq_host->lock);
+	}
+
+	if (status & CQHCI_IS_TCL)
+		wake_up(&cq_host->wait_queue);
+
+	if (status & CQHCI_IS_HAC)
+		wake_up(&cq_host->wait_queue);
+
+	return IRQ_HANDLED;
+}
+EXPORT_SYMBOL(cqhci_irq);
+
+static bool cqhci_is_idle(struct cqhci_host *cq_host, int *ret)
+{
+	unsigned long flags;
+	bool is_idle;
+
+	spin_lock_irqsave(&cq_host->lock, flags);
+	is_idle = !cq_host->qcnt || cq_host->recovery_halt;
+	*ret = cq_host->recovery_halt ? -EBUSY : 0;
+	cq_host->waiting_for_idle = !is_idle;
+	spin_unlock_irqrestore(&cq_host->lock, flags);
+
+	return is_idle;
+}
+
+static int cqhci_wait_for_idle(struct mmc_host *mmc)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	int ret;
+
+	wait_event(cq_host->wait_queue, cqhci_is_idle(cq_host, &ret));
+
+	return ret;
+}
+
+static bool cqhci_timeout(struct mmc_host *mmc, struct mmc_request *mrq,
+			  bool *recovery_needed)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	int tag = cqhci_tag(mrq);
+	struct cqhci_slot *slot = &cq_host->slot[tag];
+	unsigned long flags;
+	bool timed_out;
+
+	spin_lock_irqsave(&cq_host->lock, flags);
+	timed_out = slot->mrq == mrq;
+	if (timed_out) {
+		slot->flags |= CQHCI_EXTERNAL_TIMEOUT;
+		cqhci_recovery_needed(mmc, mrq, false);
+		*recovery_needed = cq_host->recovery_halt;
+	}
+	spin_unlock_irqrestore(&cq_host->lock, flags);
+
+	if (timed_out) {
+		pr_err("%s: cqhci: timeout for tag %d\n",
+		       mmc_hostname(mmc), tag);
+		cqhci_dumpregs(cq_host);
+	}
+
+	return timed_out;
+}
+
+static bool cqhci_tasks_cleared(struct cqhci_host *cq_host)
+{
+	return !(cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_CLEAR_ALL_TASKS);
+}
+
+static bool cqhci_clear_all_tasks(struct mmc_host *mmc, unsigned int timeout)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	bool ret;
+	u32 ctl;
+
+	cqhci_set_irqs(cq_host, CQHCI_IS_TCL);
+
+	ctl = cqhci_readl(cq_host, CQHCI_CTL);
+	ctl |= CQHCI_CLEAR_ALL_TASKS;
+	cqhci_writel(cq_host, ctl, CQHCI_CTL);
+
+	wait_event_timeout(cq_host->wait_queue, cqhci_tasks_cleared(cq_host),
+			   msecs_to_jiffies(timeout) + 1);
+
+	cqhci_set_irqs(cq_host, 0);
+
+	ret = cqhci_tasks_cleared(cq_host);
+
+	if (!ret)
+		pr_debug("%s: cqhci: Failed to clear tasks\n",
+			 mmc_hostname(mmc));
+
+	return ret;
+}
+
+static bool cqhci_halted(struct cqhci_host *cq_host)
+{
+	return cqhci_readl(cq_host, CQHCI_CTL) & CQHCI_HALT;
+}
+
+static bool cqhci_halt(struct mmc_host *mmc, unsigned int timeout)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	bool ret;
+	u32 ctl;
+
+	if (cqhci_halted(cq_host))
+		return true;
+
+	cqhci_set_irqs(cq_host, CQHCI_IS_HAC);
+
+	ctl = cqhci_readl(cq_host, CQHCI_CTL);
+	ctl |= CQHCI_HALT;
+	cqhci_writel(cq_host, ctl, CQHCI_CTL);
+
+	wait_event_timeout(cq_host->wait_queue, cqhci_halted(cq_host),
+			   msecs_to_jiffies(timeout) + 1);
+
+	cqhci_set_irqs(cq_host, 0);
+
+	ret = cqhci_halted(cq_host);
+
+	if (!ret)
+		pr_debug("%s: cqhci: Failed to halt\n", mmc_hostname(mmc));
+
+	return ret;
+}
+
+/*
+ * After halting we expect to be able to use the command line. We interpret the
+ * failure to halt to mean the data lines might still be in use (and the upper
+ * layers will need to send a STOP command), so we set the timeout based on a
+ * generous command timeout.
+ */
+#define CQHCI_START_HALT_TIMEOUT	5
+
+static void cqhci_recovery_start(struct mmc_host *mmc)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+
+	pr_debug("%s: cqhci: %s\n", mmc_hostname(mmc), __func__);
+
+	WARN_ON(!cq_host->recovery_halt);
+
+	cqhci_halt(mmc, CQHCI_START_HALT_TIMEOUT);
+
+	if (cq_host->ops->disable)
+		cq_host->ops->disable(mmc, true);
+
+	mmc->cqe_on = false;
+}
+
+static int cqhci_error_from_flags(unsigned int flags)
+{
+	if (!flags)
+		return 0;
+
+	/* CRC errors might indicate re-tuning so prefer to report that */
+	if (flags & CQHCI_HOST_CRC)
+		return -EILSEQ;
+
+	if (flags & (CQHCI_EXTERNAL_TIMEOUT | CQHCI_HOST_TIMEOUT))
+		return -ETIMEDOUT;
+
+	return -EIO;
+}
+
+static void cqhci_recover_mrq(struct cqhci_host *cq_host, unsigned int tag)
+{
+	struct cqhci_slot *slot = &cq_host->slot[tag];
+	struct mmc_request *mrq = slot->mrq;
+	struct mmc_data *data;
+
+	if (!mrq)
+		return;
+
+	slot->mrq = NULL;
+
+	cq_host->qcnt -= 1;
+
+	data = mrq->data;
+	if (data) {
+		data->bytes_xfered = 0;
+		data->error = cqhci_error_from_flags(slot->flags);
+	} else {
+		mrq->cmd->error = cqhci_error_from_flags(slot->flags);
+	}
+
+	mmc_cqe_request_done(cq_host->mmc, mrq);
+}
+
+static void cqhci_recover_mrqs(struct cqhci_host *cq_host)
+{
+	int i;
+
+	for (i = 0; i < cq_host->num_slots; i++)
+		cqhci_recover_mrq(cq_host, i);
+}
+
+/*
+ * By now the command and data lines should be unused so there is no reason for
+ * CQHCI to take a long time to halt, but if it doesn't halt there could be
+ * problems clearing tasks, so be generous.
+ */
+#define CQHCI_FINISH_HALT_TIMEOUT	20
+
+/* CQHCI could be expected to clear it's internal state pretty quickly */
+#define CQHCI_CLEAR_TIMEOUT		20
+
+static void cqhci_recovery_finish(struct mmc_host *mmc)
+{
+	struct cqhci_host *cq_host = mmc->cqe_private;
+	unsigned long flags;
+	u32 cqcfg;
+	bool ok;
+
+	pr_debug("%s: cqhci: %s\n", mmc_hostname(mmc), __func__);
+
+	WARN_ON(!cq_host->recovery_halt);
+
+	ok = cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT);
+
+	if (!cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT))
+		ok = false;
+
+	/*
+	 * The specification contradicts itself, by saying that tasks cannot be
+	 * cleared if CQHCI does not halt, but if CQHCI does not halt, it should
+	 * be disabled/re-enabled, but not to disable before clearing tasks.
+	 * Have a go anyway.
+	 */
+	if (!ok) {
+		pr_debug("%s: cqhci: disable / re-enable\n", mmc_hostname(mmc));
+		cqcfg = cqhci_readl(cq_host, CQHCI_CFG);
+		cqcfg &= ~CQHCI_ENABLE;
+		cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+		cqcfg |= CQHCI_ENABLE;
+		cqhci_writel(cq_host, cqcfg, CQHCI_CFG);
+		/* Be sure that there are no tasks */
+		ok = cqhci_halt(mmc, CQHCI_FINISH_HALT_TIMEOUT);
+		if (!cqhci_clear_all_tasks(mmc, CQHCI_CLEAR_TIMEOUT))
+			ok = false;
+		WARN_ON(!ok);
+	}
+
+	cqhci_recover_mrqs(cq_host);
+
+	WARN_ON(cq_host->qcnt);
+
+	spin_lock_irqsave(&cq_host->lock, flags);
+	cq_host->qcnt = 0;
+	cq_host->recovery_halt = false;
+	mmc->cqe_on = false;
+	spin_unlock_irqrestore(&cq_host->lock, flags);
+
+	/* Ensure all writes are done before interrupts are re-enabled */
+	wmb();
+
+	cqhci_writel(cq_host, CQHCI_IS_HAC | CQHCI_IS_TCL, CQHCI_IS);
+
+	cqhci_set_irqs(cq_host, CQHCI_IS_MASK);
+
+	pr_debug("%s: cqhci: recovery done\n", mmc_hostname(mmc));
+}
+
+static const struct mmc_cqe_ops cqhci_cqe_ops = {
+	.cqe_enable = cqhci_enable,
+	.cqe_disable = cqhci_disable,
+	.cqe_request = cqhci_request,
+	.cqe_post_req = cqhci_post_req,
+	.cqe_off = cqhci_off,
+	.cqe_wait_for_idle = cqhci_wait_for_idle,
+	.cqe_timeout = cqhci_timeout,
+	.cqe_recovery_start = cqhci_recovery_start,
+	.cqe_recovery_finish = cqhci_recovery_finish,
+};
+
+struct cqhci_host *cqhci_pltfm_init(struct platform_device *pdev)
+{
+	struct cqhci_host *cq_host;
+	struct resource *cqhci_memres = NULL;
+
+	/* check and setup CMDQ interface */
+	cqhci_memres = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+						   "cqhci_mem");
+	if (!cqhci_memres) {
+		dev_dbg(&pdev->dev, "CMDQ not supported\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	cq_host = devm_kzalloc(&pdev->dev, sizeof(*cq_host), GFP_KERNEL);
+	if (!cq_host)
+		return ERR_PTR(-ENOMEM);
+	cq_host->mmio = devm_ioremap(&pdev->dev,
+				     cqhci_memres->start,
+				     resource_size(cqhci_memres));
+	if (!cq_host->mmio) {
+		dev_err(&pdev->dev, "failed to remap cqhci regs\n");
+		return ERR_PTR(-EBUSY);
+	}
+	dev_dbg(&pdev->dev, "CMDQ ioremap: done\n");
+
+	return cq_host;
+}
+EXPORT_SYMBOL(cqhci_pltfm_init);
+
+static unsigned int cqhci_ver_major(struct cqhci_host *cq_host)
+{
+	return CQHCI_VER_MAJOR(cqhci_readl(cq_host, CQHCI_VER));
+}
+
+static unsigned int cqhci_ver_minor(struct cqhci_host *cq_host)
+{
+	u32 ver = cqhci_readl(cq_host, CQHCI_VER);
+
+	return CQHCI_VER_MINOR1(ver) * 10 + CQHCI_VER_MINOR2(ver);
+}
+
+int cqhci_init(struct cqhci_host *cq_host, struct mmc_host *mmc,
+	      bool dma64)
+{
+	int err;
+
+	cq_host->dma64 = dma64;
+	cq_host->mmc = mmc;
+	cq_host->mmc->cqe_private = cq_host;
+
+	cq_host->num_slots = NUM_SLOTS;
+	cq_host->dcmd_slot = DCMD_SLOT;
+
+	mmc->cqe_ops = &cqhci_cqe_ops;
+
+	mmc->cqe_qdepth = NUM_SLOTS;
+	if (mmc->caps2 & MMC_CAP2_CQE_DCMD)
+		mmc->cqe_qdepth -= 1;
+
+	cq_host->slot = devm_kcalloc(mmc_dev(mmc), cq_host->num_slots,
+				     sizeof(*cq_host->slot), GFP_KERNEL);
+	if (!cq_host->slot) {
+		err = -ENOMEM;
+		goto out_err;
+	}
+
+	spin_lock_init(&cq_host->lock);
+
+	init_completion(&cq_host->halt_comp);
+	init_waitqueue_head(&cq_host->wait_queue);
+
+	pr_info("%s: CQHCI version %u.%02u\n",
+		mmc_hostname(mmc), cqhci_ver_major(cq_host),
+		cqhci_ver_minor(cq_host));
+
+	return 0;
+
+out_err:
+	pr_err("%s: CQHCI version %u.%02u failed to initialize, error %d\n",
+	       mmc_hostname(mmc), cqhci_ver_major(cq_host),
+	       cqhci_ver_minor(cq_host), err);
+	return err;
+}
+EXPORT_SYMBOL(cqhci_init);
+
+MODULE_AUTHOR("Venkat Gopalakrishnan <venkatg@codeaurora.org>");
+MODULE_DESCRIPTION("Command Queue Host Controller Interface driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mmc/host/cqhci.h b/drivers/mmc/host/cqhci.h
new file mode 100644
index 000000000000..9e68286a07b4
--- /dev/null
+++ b/drivers/mmc/host/cqhci.h
@@ -0,0 +1,240 @@
+/* Copyright (c) 2015, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef LINUX_MMC_CQHCI_H
+#define LINUX_MMC_CQHCI_H
+
+#include <linux/compiler.h>
+#include <linux/bitops.h>
+#include <linux/spinlock_types.h>
+#include <linux/types.h>
+#include <linux/completion.h>
+#include <linux/wait.h>
+#include <linux/irqreturn.h>
+#include <asm/io.h>
+
+/* registers */
+/* version */
+#define CQHCI_VER			0x00
+#define CQHCI_VER_MAJOR(x)		(((x) & GENMASK(11, 8)) >> 8)
+#define CQHCI_VER_MINOR1(x)		(((x) & GENMASK(7, 4)) >> 4)
+#define CQHCI_VER_MINOR2(x)		((x) & GENMASK(3, 0))
+
+/* capabilities */
+#define CQHCI_CAP			0x04
+/* configuration */
+#define CQHCI_CFG			0x08
+#define CQHCI_DCMD			0x00001000
+#define CQHCI_TASK_DESC_SZ		0x00000100
+#define CQHCI_ENABLE			0x00000001
+
+/* control */
+#define CQHCI_CTL			0x0C
+#define CQHCI_CLEAR_ALL_TASKS		0x00000100
+#define CQHCI_HALT			0x00000001
+
+/* interrupt status */
+#define CQHCI_IS			0x10
+#define CQHCI_IS_HAC			BIT(0)
+#define CQHCI_IS_TCC			BIT(1)
+#define CQHCI_IS_RED			BIT(2)
+#define CQHCI_IS_TCL			BIT(3)
+
+#define CQHCI_IS_MASK (CQHCI_IS_TCC | CQHCI_IS_RED)
+
+/* interrupt status enable */
+#define CQHCI_ISTE			0x14
+
+/* interrupt signal enable */
+#define CQHCI_ISGE			0x18
+
+/* interrupt coalescing */
+#define CQHCI_IC			0x1C
+#define CQHCI_IC_ENABLE			BIT(31)
+#define CQHCI_IC_RESET			BIT(16)
+#define CQHCI_IC_ICCTHWEN		BIT(15)
+#define CQHCI_IC_ICCTH(x)		(((x) & 0x1F) << 8)
+#define CQHCI_IC_ICTOVALWEN		BIT(7)
+#define CQHCI_IC_ICTOVAL(x)		((x) & 0x7F)
+
+/* task list base address */
+#define CQHCI_TDLBA			0x20
+
+/* task list base address upper */
+#define CQHCI_TDLBAU			0x24
+
+/* door-bell */
+#define CQHCI_TDBR			0x28
+
+/* task completion notification */
+#define CQHCI_TCN			0x2C
+
+/* device queue status */
+#define CQHCI_DQS			0x30
+
+/* device pending tasks */
+#define CQHCI_DPT			0x34
+
+/* task clear */
+#define CQHCI_TCLR			0x38
+
+/* send status config 1 */
+#define CQHCI_SSC1			0x40
+
+/* send status config 2 */
+#define CQHCI_SSC2			0x44
+
+/* response for dcmd */
+#define CQHCI_CRDCT			0x48
+
+/* response mode error mask */
+#define CQHCI_RMEM			0x50
+
+/* task error info */
+#define CQHCI_TERRI			0x54
+
+#define CQHCI_TERRI_C_INDEX(x)		((x) & GENMASK(5, 0))
+#define CQHCI_TERRI_C_TASK(x)		(((x) & GENMASK(12, 8)) >> 8)
+#define CQHCI_TERRI_C_VALID(x)		((x) & BIT(15))
+#define CQHCI_TERRI_D_INDEX(x)		(((x) & GENMASK(21, 16)) >> 16)
+#define CQHCI_TERRI_D_TASK(x)		(((x) & GENMASK(28, 24)) >> 24)
+#define CQHCI_TERRI_D_VALID(x)		((x) & BIT(31))
+
+/* command response index */
+#define CQHCI_CRI			0x58
+
+/* command response argument */
+#define CQHCI_CRA			0x5C
+
+#define CQHCI_INT_ALL			0xF
+#define CQHCI_IC_DEFAULT_ICCTH		31
+#define CQHCI_IC_DEFAULT_ICTOVAL	1
+
+/* attribute fields */
+#define CQHCI_VALID(x)			(((x) & 1) << 0)
+#define CQHCI_END(x)			(((x) & 1) << 1)
+#define CQHCI_INT(x)			(((x) & 1) << 2)
+#define CQHCI_ACT(x)			(((x) & 0x7) << 3)
+
+/* data command task descriptor fields */
+#define CQHCI_FORCED_PROG(x)		(((x) & 1) << 6)
+#define CQHCI_CONTEXT(x)		(((x) & 0xF) << 7)
+#define CQHCI_DATA_TAG(x)		(((x) & 1) << 11)
+#define CQHCI_DATA_DIR(x)		(((x) & 1) << 12)
+#define CQHCI_PRIORITY(x)		(((x) & 1) << 13)
+#define CQHCI_QBAR(x)			(((x) & 1) << 14)
+#define CQHCI_REL_WRITE(x)		(((x) & 1) << 15)
+#define CQHCI_BLK_COUNT(x)		(((x) & 0xFFFF) << 16)
+#define CQHCI_BLK_ADDR(x)		(((x) & 0xFFFFFFFF) << 32)
+
+/* direct command task descriptor fields */
+#define CQHCI_CMD_INDEX(x)		(((x) & 0x3F) << 16)
+#define CQHCI_CMD_TIMING(x)		(((x) & 1) << 22)
+#define CQHCI_RESP_TYPE(x)		(((x) & 0x3) << 23)
+
+/* transfer descriptor fields */
+#define CQHCI_DAT_LENGTH(x)		(((x) & 0xFFFF) << 16)
+#define CQHCI_DAT_ADDR_LO(x)		(((x) & 0xFFFFFFFF) << 32)
+#define CQHCI_DAT_ADDR_HI(x)		(((x) & 0xFFFFFFFF) << 0)
+
+struct cqhci_host_ops;
+struct mmc_host;
+struct cqhci_slot;
+
+struct cqhci_host {
+	const struct cqhci_host_ops *ops;
+	void __iomem *mmio;
+	struct mmc_host *mmc;
+
+	spinlock_t lock;
+
+	/* relative card address of device */
+	unsigned int rca;
+
+	/* 64 bit DMA */
+	bool dma64;
+	int num_slots;
+	int qcnt;
+
+	u32 dcmd_slot;
+	u32 caps;
+#define CQHCI_TASK_DESC_SZ_128		0x1
+
+	u32 quirks;
+#define CQHCI_QUIRK_SHORT_TXFR_DESC_SZ	0x1
+
+	bool enabled;
+	bool halted;
+	bool init_done;
+	bool activated;
+	bool waiting_for_idle;
+	bool recovery_halt;
+
+	size_t desc_size;
+	size_t data_size;
+
+	u8 *desc_base;
+
+	/* total descriptor size */
+	u8 slot_sz;
+
+	/* 64/128 bit depends on CQHCI_CFG */
+	u8 task_desc_len;
+
+	/* 64 bit on 32-bit arch, 128 bit on 64-bit */
+	u8 link_desc_len;
+
+	u8 *trans_desc_base;
+	/* same length as transfer descriptor */
+	u8 trans_desc_len;
+
+	dma_addr_t desc_dma_base;
+	dma_addr_t trans_desc_dma_base;
+
+	struct completion halt_comp;
+	wait_queue_head_t wait_queue;
+	struct cqhci_slot *slot;
+};
+
+struct cqhci_host_ops {
+	void (*dumpregs)(struct mmc_host *mmc);
+	void (*write_l)(struct cqhci_host *host, u32 val, int reg);
+	u32 (*read_l)(struct cqhci_host *host, int reg);
+	void (*enable)(struct mmc_host *mmc);
+	void (*disable)(struct mmc_host *mmc, bool recovery);
+};
+
+static inline void cqhci_writel(struct cqhci_host *host, u32 val, int reg)
+{
+	if (unlikely(host->ops->write_l))
+		host->ops->write_l(host, val, reg);
+	else
+		writel_relaxed(val, host->mmio + reg);
+}
+
+static inline u32 cqhci_readl(struct cqhci_host *host, int reg)
+{
+	if (unlikely(host->ops->read_l))
+		return host->ops->read_l(host, reg);
+	else
+		return readl_relaxed(host->mmio + reg);
+}
+
+struct platform_device;
+
+irqreturn_t cqhci_irq(struct mmc_host *mmc, u32 intmask, int cmd_error,
+		      int data_error);
+int cqhci_init(struct cqhci_host *cq_host, struct mmc_host *mmc, bool dma64);
+struct cqhci_host *cqhci_pltfm_init(struct platform_device *pdev);
+int cqhci_suspend(struct mmc_host *mmc);
+int cqhci_resume(struct mmc_host *mmc);
+
+#endif
diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
index 351330dfb954..8e363174f9d6 100644
--- a/drivers/mmc/host/davinci_mmc.c
+++ b/drivers/mmc/host/davinci_mmc.c
@@ -174,7 +174,7 @@ module_param(poll_loopcount, uint, S_IRUGO);
 MODULE_PARM_DESC(poll_loopcount,
 		 "Maximum polling loop count. Default = 32");
 
-static unsigned __initdata use_dma = 1;
+static unsigned use_dma = 1;
 module_param(use_dma, uint, 0);
 MODULE_PARM_DESC(use_dma, "Whether to use DMA or not. Default = 1");
 
@@ -496,8 +496,7 @@ static int mmc_davinci_start_dma_transfer(struct mmc_davinci_host *host,
 	return ret;
 }
 
-static void __init_or_module
-davinci_release_dma_channels(struct mmc_davinci_host *host)
+static void davinci_release_dma_channels(struct mmc_davinci_host *host)
 {
 	if (!host->use_dma)
 		return;
@@ -506,7 +505,7 @@ davinci_release_dma_channels(struct mmc_davinci_host *host)
 	dma_release_channel(host->dma_rx);
 }
 
-static int __init davinci_acquire_dma_channels(struct mmc_davinci_host *host)
+static int davinci_acquire_dma_channels(struct mmc_davinci_host *host)
 {
 	host->dma_tx = dma_request_chan(mmc_dev(host->mmc), "tx");
 	if (IS_ERR(host->dma_tx)) {
@@ -1201,7 +1200,7 @@ static int mmc_davinci_parse_pdata(struct mmc_host *mmc)
 	return 0;
 }
 
-static int __init davinci_mmcsd_probe(struct platform_device *pdev)
+static int davinci_mmcsd_probe(struct platform_device *pdev)
 {
 	const struct of_device_id *match;
 	struct mmc_davinci_host *host = NULL;
@@ -1254,8 +1253,9 @@ static int __init davinci_mmcsd_probe(struct platform_device *pdev)
 		pdev->id_entry = match->data;
 		ret = mmc_of_parse(mmc);
 		if (ret) {
-			dev_err(&pdev->dev,
-				"could not parse of data: %d\n", ret);
+			if (ret != -EPROBE_DEFER)
+				dev_err(&pdev->dev,
+					"could not parse of data: %d\n", ret);
 			goto parse_fail;
 		}
 	} else {
@@ -1414,11 +1414,12 @@ static struct platform_driver davinci_mmcsd_driver = {
 		.pm	= davinci_mmcsd_pm_ops,
 		.of_match_table = davinci_mmc_dt_ids,
 	},
+	.probe		= davinci_mmcsd_probe,
 	.remove		= __exit_p(davinci_mmcsd_remove),
 	.id_table	= davinci_mmc_devtype,
 };
 
-module_platform_driver_probe(davinci_mmcsd_driver, davinci_mmcsd_probe);
+module_platform_driver(davinci_mmcsd_driver);
 
 MODULE_AUTHOR("Texas Instruments India");
 MODULE_LICENSE("GPL");
diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c
index 35026795be28..a84aa3f1ae85 100644
--- a/drivers/mmc/host/dw_mmc-exynos.c
+++ b/drivers/mmc/host/dw_mmc-exynos.c
@@ -165,9 +165,15 @@ static void dw_mci_exynos_set_clksel_timing(struct dw_mci *host, u32 timing)
 static int dw_mci_exynos_runtime_resume(struct device *dev)
 {
 	struct dw_mci *host = dev_get_drvdata(dev);
+	int ret;
+
+	ret = dw_mci_runtime_resume(dev);
+	if (ret)
+		return ret;
 
 	dw_mci_exynos_config_smu(host);
-	return dw_mci_runtime_resume(dev);
+
+	return ret;
 }
 
 /**
@@ -487,6 +493,7 @@ static unsigned long exynos_dwmmc_caps[4] = {
 
 static const struct dw_mci_drv_data exynos_drv_data = {
 	.caps			= exynos_dwmmc_caps,
+	.num_caps		= ARRAY_SIZE(exynos_dwmmc_caps),
 	.init			= dw_mci_exynos_priv_init,
 	.set_ios		= dw_mci_exynos_set_ios,
 	.parse_dt		= dw_mci_exynos_parse_dt,
diff --git a/drivers/mmc/host/dw_mmc-k3.c b/drivers/mmc/host/dw_mmc-k3.c
index 73fd75c3c824..89cdb3d533bb 100644
--- a/drivers/mmc/host/dw_mmc-k3.c
+++ b/drivers/mmc/host/dw_mmc-k3.c
@@ -135,6 +135,9 @@ static int dw_mci_hi6220_parse_dt(struct dw_mci *host)
 	if (priv->ctrl_id < 0)
 		priv->ctrl_id = 0;
 
+	if (priv->ctrl_id >= TIMING_MODE)
+		return -EINVAL;
+
 	host->priv = priv;
 	return 0;
 }
@@ -207,6 +210,7 @@ static int dw_mci_hi6220_execute_tuning(struct dw_mci_slot *slot, u32 opcode)
 
 static const struct dw_mci_drv_data hi6220_data = {
 	.caps			= dw_mci_hi6220_caps,
+	.num_caps		= ARRAY_SIZE(dw_mci_hi6220_caps),
 	.switch_voltage		= dw_mci_hi6220_switch_voltage,
 	.set_ios		= dw_mci_hi6220_set_ios,
 	.parse_dt		= dw_mci_hi6220_parse_dt,
diff --git a/drivers/mmc/host/dw_mmc-rockchip.c b/drivers/mmc/host/dw_mmc-rockchip.c
index a3f1c2b30145..339295212935 100644
--- a/drivers/mmc/host/dw_mmc-rockchip.c
+++ b/drivers/mmc/host/dw_mmc-rockchip.c
@@ -319,6 +319,7 @@ static const struct dw_mci_drv_data rk2928_drv_data = {
 
 static const struct dw_mci_drv_data rk3288_drv_data = {
 	.caps			= dw_mci_rk3288_dwmmc_caps,
+	.num_caps		= ARRAY_SIZE(dw_mci_rk3288_dwmmc_caps),
 	.set_ios		= dw_mci_rk3288_set_ios,
 	.execute_tuning		= dw_mci_rk3288_execute_tuning,
 	.parse_dt		= dw_mci_rk3288_parse_dt,
diff --git a/drivers/mmc/host/dw_mmc-zx.c b/drivers/mmc/host/dw_mmc-zx.c
index d38e94ae2b85..c06b5393312f 100644
--- a/drivers/mmc/host/dw_mmc-zx.c
+++ b/drivers/mmc/host/dw_mmc-zx.c
@@ -195,6 +195,7 @@ static unsigned long zx_dwmmc_caps[3] = {
 
 static const struct dw_mci_drv_data zx_drv_data = {
 	.caps			= zx_dwmmc_caps,
+	.num_caps		= ARRAY_SIZE(zx_dwmmc_caps),
 	.execute_tuning		= dw_mci_zx_execute_tuning,
 	.prepare_hs400_tuning	= dw_mci_zx_prepare_hs400_tuning,
 	.parse_dt               = dw_mci_zx_parse_dt,
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index 0aa39975f33b..06d47414d0c1 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -165,6 +165,8 @@ static int dw_mci_regs_show(struct seq_file *s, void *v)
 {
 	struct dw_mci *host = s->private;
 
+	pm_runtime_get_sync(host->dev);
+
 	seq_printf(s, "STATUS:\t0x%08x\n", mci_readl(host, STATUS));
 	seq_printf(s, "RINTSTS:\t0x%08x\n", mci_readl(host, RINTSTS));
 	seq_printf(s, "CMD:\t0x%08x\n", mci_readl(host, CMD));
@@ -172,6 +174,8 @@ static int dw_mci_regs_show(struct seq_file *s, void *v)
 	seq_printf(s, "INTMASK:\t0x%08x\n", mci_readl(host, INTMASK));
 	seq_printf(s, "CLKENA:\t0x%08x\n", mci_readl(host, CLKENA));
 
+	pm_runtime_put_autosuspend(host->dev);
+
 	return 0;
 }
 
@@ -409,7 +413,9 @@ static inline void dw_mci_set_cto(struct dw_mci *host)
 	cto_div = (mci_readl(host, CLKDIV) & 0xff) * 2;
 	if (cto_div == 0)
 		cto_div = 1;
-	cto_ms = DIV_ROUND_UP(MSEC_PER_SEC * cto_clks * cto_div, host->bus_hz);
+
+	cto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * cto_clks * cto_div,
+				  host->bus_hz);
 
 	/* add a bit spare time */
 	cto_ms += 10;
@@ -558,6 +564,7 @@ static int dw_mci_idmac_init(struct dw_mci *host)
 					(sizeof(struct idmac_desc_64addr) *
 							(i + 1))) >> 32;
 			/* Initialize reserved and buffer size fields to "0" */
+			p->des0 = 0;
 			p->des1 = 0;
 			p->des2 = 0;
 			p->des3 = 0;
@@ -580,6 +587,7 @@ static int dw_mci_idmac_init(struct dw_mci *host)
 		     i++, p++) {
 			p->des3 = cpu_to_le32(host->sg_dma +
 					(sizeof(struct idmac_desc) * (i + 1)));
+			p->des0 = 0;
 			p->des1 = 0;
 		}
 
@@ -1795,8 +1803,8 @@ static bool dw_mci_reset(struct dw_mci *host)
 	}
 
 	if (host->use_dma == TRANS_MODE_IDMAC)
-		/* It is also recommended that we reset and reprogram idmac */
-		dw_mci_idmac_reset(host);
+		/* It is also required that we reinit idmac */
+		dw_mci_idmac_init(host);
 
 	ret = true;
 
@@ -1944,8 +1952,9 @@ static void dw_mci_set_drto(struct dw_mci *host)
 	drto_div = (mci_readl(host, CLKDIV) & 0xff) * 2;
 	if (drto_div == 0)
 		drto_div = 1;
-	drto_ms = DIV_ROUND_UP(MSEC_PER_SEC * drto_clks * drto_div,
-			       host->bus_hz);
+
+	drto_ms = DIV_ROUND_UP_ULL((u64)MSEC_PER_SEC * drto_clks * drto_div,
+				   host->bus_hz);
 
 	/* add a bit spare time */
 	drto_ms += 10;
@@ -2778,12 +2787,57 @@ static irqreturn_t dw_mci_interrupt(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static int dw_mci_init_slot_caps(struct dw_mci_slot *slot)
+{
+	struct dw_mci *host = slot->host;
+	const struct dw_mci_drv_data *drv_data = host->drv_data;
+	struct mmc_host *mmc = slot->mmc;
+	int ctrl_id;
+
+	if (host->pdata->caps)
+		mmc->caps = host->pdata->caps;
+
+	/*
+	 * Support MMC_CAP_ERASE by default.
+	 * It needs to use trim/discard/erase commands.
+	 */
+	mmc->caps |= MMC_CAP_ERASE;
+
+	if (host->pdata->pm_caps)
+		mmc->pm_caps = host->pdata->pm_caps;
+
+	if (host->dev->of_node) {
+		ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");
+		if (ctrl_id < 0)
+			ctrl_id = 0;
+	} else {
+		ctrl_id = to_platform_device(host->dev)->id;
+	}
+
+	if (drv_data && drv_data->caps) {
+		if (ctrl_id >= drv_data->num_caps) {
+			dev_err(host->dev, "invalid controller id %d\n",
+				ctrl_id);
+			return -EINVAL;
+		}
+		mmc->caps |= drv_data->caps[ctrl_id];
+	}
+
+	if (host->pdata->caps2)
+		mmc->caps2 = host->pdata->caps2;
+
+	/* Process SDIO IRQs through the sdio_irq_work. */
+	if (mmc->caps & MMC_CAP_SDIO_IRQ)
+		mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
+
+	return 0;
+}
+
 static int dw_mci_init_slot(struct dw_mci *host)
 {
 	struct mmc_host *mmc;
 	struct dw_mci_slot *slot;
-	const struct dw_mci_drv_data *drv_data = host->drv_data;
-	int ctrl_id, ret;
+	int ret;
 	u32 freq[2];
 
 	mmc = mmc_alloc_host(sizeof(struct dw_mci_slot), host->dev);
@@ -2817,38 +2871,13 @@ static int dw_mci_init_slot(struct dw_mci *host)
 	if (!mmc->ocr_avail)
 		mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
 
-	if (host->pdata->caps)
-		mmc->caps = host->pdata->caps;
-
-	/*
-	 * Support MMC_CAP_ERASE by default.
-	 * It needs to use trim/discard/erase commands.
-	 */
-	mmc->caps |= MMC_CAP_ERASE;
-
-	if (host->pdata->pm_caps)
-		mmc->pm_caps = host->pdata->pm_caps;
-
-	if (host->dev->of_node) {
-		ctrl_id = of_alias_get_id(host->dev->of_node, "mshc");
-		if (ctrl_id < 0)
-			ctrl_id = 0;
-	} else {
-		ctrl_id = to_platform_device(host->dev)->id;
-	}
-	if (drv_data && drv_data->caps)
-		mmc->caps |= drv_data->caps[ctrl_id];
-
-	if (host->pdata->caps2)
-		mmc->caps2 = host->pdata->caps2;
-
 	ret = mmc_of_parse(mmc);
 	if (ret)
 		goto err_host_allocated;
 
-	/* Process SDIO IRQs through the sdio_irq_work. */
-	if (mmc->caps & MMC_CAP_SDIO_IRQ)
-		mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
+	ret = dw_mci_init_slot_caps(slot);
+	if (ret)
+		goto err_host_allocated;
 
 	/* Useful defaults if platform data is unset. */
 	if (host->use_dma == TRANS_MODE_IDMAC) {
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index e3124f06a47e..1424bd490dd1 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -543,6 +543,7 @@ struct dw_mci_slot {
 /**
  * dw_mci driver data - dw-mshc implementation specific driver data.
  * @caps: mmc subsystem specified capabilities of the controller(s).
+ * @num_caps: number of capabilities specified by @caps.
  * @init: early implementation specific initialization.
  * @set_ios: handle bus specific extensions.
  * @parse_dt: parse implementation specific device tree properties.
@@ -554,6 +555,7 @@ struct dw_mci_slot {
  */
 struct dw_mci_drv_data {
 	unsigned long	*caps;
+	u32		num_caps;
 	int		(*init)(struct dw_mci *host);
 	void		(*set_ios)(struct dw_mci *host, struct mmc_ios *ios);
 	int		(*parse_dt)(struct dw_mci *host);
diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
index e0862d3f65b3..4f972b879fe6 100644
--- a/drivers/mmc/host/meson-gx-mmc.c
+++ b/drivers/mmc/host/meson-gx-mmc.c
@@ -37,6 +37,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/bitfield.h>
+#include <linux/pinctrl/consumer.h>
 
 #define DRIVER_NAME "meson-gx-mmc"
 
@@ -716,22 +717,6 @@ static int meson_mmc_clk_phase_tuning(struct mmc_host *mmc, u32 opcode,
 static int meson_mmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
 {
 	struct meson_host *host = mmc_priv(mmc);
-	int ret;
-
-	/*
-	 * If this is the initial tuning, try to get a sane Rx starting
-	 * phase before doing the actual tuning.
-	 */
-	if (!mmc->doing_retune) {
-		ret = meson_mmc_clk_phase_tuning(mmc, opcode, host->rx_clk);
-
-		if (ret)
-			return ret;
-	}
-
-	ret = meson_mmc_clk_phase_tuning(mmc, opcode, host->tx_clk);
-	if (ret)
-		return ret;
 
 	return meson_mmc_clk_phase_tuning(mmc, opcode, host->rx_clk);
 }
@@ -762,9 +747,8 @@ static void meson_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 		if (!IS_ERR(mmc->supply.vmmc))
 			mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, ios->vdd);
 
-		/* Reset phases */
+		/* Reset rx phase */
 		clk_set_phase(host->rx_clk, 0);
-		clk_set_phase(host->tx_clk, 270);
 
 		break;
 
@@ -1208,7 +1192,7 @@ static int meson_mmc_probe(struct platform_device *pdev)
 	}
 
 	irq = platform_get_irq(pdev, 0);
-	if (!irq) {
+	if (irq <= 0) {
 		dev_err(&pdev->dev, "failed to get interrupt resource.\n");
 		ret = -EINVAL;
 		goto free_host;
diff --git a/drivers/mmc/host/mmci.c b/drivers/mmc/host/mmci.c
index e8a1bb1ae694..70b0df8b9c78 100644
--- a/drivers/mmc/host/mmci.c
+++ b/drivers/mmc/host/mmci.c
@@ -82,6 +82,10 @@ static unsigned int fmax = 515633;
  * @qcom_fifo: enables qcom specific fifo pio read logic.
  * @qcom_dml: enables qcom specific dma glue for dma transfers.
  * @reversed_irq_handling: handle data irq before cmd irq.
+ * @mmcimask1: true if variant have a MMCIMASK1 register.
+ * @start_err: bitmask identifying the STARTBITERR bit inside MMCISTATUS
+ *	       register.
+ * @opendrain: bitmask identifying the OPENDRAIN bit inside MMCIPOWER register
  */
 struct variant_data {
 	unsigned int		clkreg;
@@ -111,6 +115,9 @@ struct variant_data {
 	bool			qcom_fifo;
 	bool			qcom_dml;
 	bool			reversed_irq_handling;
+	bool			mmcimask1;
+	u32			start_err;
+	u32			opendrain;
 };
 
 static struct variant_data variant_arm = {
@@ -120,6 +127,9 @@ static struct variant_data variant_arm = {
 	.pwrreg_powerup		= MCI_PWR_UP,
 	.f_max			= 100000000,
 	.reversed_irq_handling	= true,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_ROD,
 };
 
 static struct variant_data variant_arm_extended_fifo = {
@@ -128,6 +138,9 @@ static struct variant_data variant_arm_extended_fifo = {
 	.datalength_bits	= 16,
 	.pwrreg_powerup		= MCI_PWR_UP,
 	.f_max			= 100000000,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_ROD,
 };
 
 static struct variant_data variant_arm_extended_fifo_hwfc = {
@@ -137,6 +150,9 @@ static struct variant_data variant_arm_extended_fifo_hwfc = {
 	.datalength_bits	= 16,
 	.pwrreg_powerup		= MCI_PWR_UP,
 	.f_max			= 100000000,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_ROD,
 };
 
 static struct variant_data variant_u300 = {
@@ -152,6 +168,9 @@ static struct variant_data variant_u300 = {
 	.signal_direction	= true,
 	.pwrreg_clkgate		= true,
 	.pwrreg_nopower		= true,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_OD,
 };
 
 static struct variant_data variant_nomadik = {
@@ -168,6 +187,9 @@ static struct variant_data variant_nomadik = {
 	.signal_direction	= true,
 	.pwrreg_clkgate		= true,
 	.pwrreg_nopower		= true,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_OD,
 };
 
 static struct variant_data variant_ux500 = {
@@ -190,6 +212,9 @@ static struct variant_data variant_ux500 = {
 	.busy_detect_flag	= MCI_ST_CARDBUSY,
 	.busy_detect_mask	= MCI_ST_BUSYENDMASK,
 	.pwrreg_nopower		= true,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_OD,
 };
 
 static struct variant_data variant_ux500v2 = {
@@ -214,6 +239,26 @@ static struct variant_data variant_ux500v2 = {
 	.busy_detect_flag	= MCI_ST_CARDBUSY,
 	.busy_detect_mask	= MCI_ST_BUSYENDMASK,
 	.pwrreg_nopower		= true,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_OD,
+};
+
+static struct variant_data variant_stm32 = {
+	.fifosize		= 32 * 4,
+	.fifohalfsize		= 8 * 4,
+	.clkreg			= MCI_CLK_ENABLE,
+	.clkreg_enable		= MCI_ST_UX500_HWFCEN,
+	.clkreg_8bit_bus_enable = MCI_ST_8BIT_BUS,
+	.clkreg_neg_edge_enable	= MCI_ST_UX500_NEG_EDGE,
+	.datalength_bits	= 24,
+	.datactrl_mask_sdio	= MCI_DPSM_ST_SDIOEN,
+	.st_sdio		= true,
+	.st_clkdiv		= true,
+	.pwrreg_powerup		= MCI_PWR_ON,
+	.f_max			= 48000000,
+	.pwrreg_clkgate		= true,
+	.pwrreg_nopower		= true,
 };
 
 static struct variant_data variant_qcom = {
@@ -232,6 +277,9 @@ static struct variant_data variant_qcom = {
 	.explicit_mclk_control	= true,
 	.qcom_fifo		= true,
 	.qcom_dml		= true,
+	.mmcimask1		= true,
+	.start_err		= MCI_STARTBITERR,
+	.opendrain		= MCI_ROD,
 };
 
 /* Busy detection for the ST Micro variant */
@@ -396,6 +444,7 @@ mmci_request_end(struct mmci_host *host, struct mmc_request *mrq)
 static void mmci_set_mask1(struct mmci_host *host, unsigned int mask)
 {
 	void __iomem *base = host->base;
+	struct variant_data *variant = host->variant;
 
 	if (host->singleirq) {
 		unsigned int mask0 = readl(base + MMCIMASK0);
@@ -406,7 +455,10 @@ static void mmci_set_mask1(struct mmci_host *host, unsigned int mask)
 		writel(mask0, base + MMCIMASK0);
 	}
 
-	writel(mask, base + MMCIMASK1);
+	if (variant->mmcimask1)
+		writel(mask, base + MMCIMASK1);
+
+	host->mask1_reg = mask;
 }
 
 static void mmci_stop_data(struct mmci_host *host)
@@ -921,8 +973,9 @@ mmci_data_irq(struct mmci_host *host, struct mmc_data *data,
 		return;
 
 	/* First check for errors */
-	if (status & (MCI_DATACRCFAIL|MCI_DATATIMEOUT|MCI_STARTBITERR|
-		      MCI_TXUNDERRUN|MCI_RXOVERRUN)) {
+	if (status & (MCI_DATACRCFAIL | MCI_DATATIMEOUT |
+		      host->variant->start_err |
+		      MCI_TXUNDERRUN | MCI_RXOVERRUN)) {
 		u32 remain, success;
 
 		/* Terminate the DMA transfer */
@@ -1286,7 +1339,7 @@ static irqreturn_t mmci_irq(int irq, void *dev_id)
 		status = readl(host->base + MMCISTATUS);
 
 		if (host->singleirq) {
-			if (status & readl(host->base + MMCIMASK1))
+			if (status & host->mask1_reg)
 				mmci_pio_irq(irq, dev_id);
 
 			status &= ~MCI_IRQ1MASK;
@@ -1429,16 +1482,18 @@ static void mmci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 				~MCI_ST_DATA2DIREN);
 	}
 
-	if (ios->bus_mode == MMC_BUSMODE_OPENDRAIN) {
-		if (host->hw_designer != AMBA_VENDOR_ST)
-			pwr |= MCI_ROD;
-		else {
-			/*
-			 * The ST Micro variant use the ROD bit for something
-			 * else and only has OD (Open Drain).
-			 */
-			pwr |= MCI_OD;
-		}
+	if (variant->opendrain) {
+		if (ios->bus_mode == MMC_BUSMODE_OPENDRAIN)
+			pwr |= variant->opendrain;
+	} else {
+		/*
+		 * If the variant cannot configure the pads by its own, then we
+		 * expect the pinctrl to be able to do that for us
+		 */
+		if (ios->bus_mode == MMC_BUSMODE_OPENDRAIN)
+			pinctrl_select_state(host->pinctrl, host->pins_opendrain);
+		else
+			pinctrl_select_state(host->pinctrl, host->pins_default);
 	}
 
 	/*
@@ -1583,6 +1638,35 @@ static int mmci_probe(struct amba_device *dev,
 	host = mmc_priv(mmc);
 	host->mmc = mmc;
 
+	/*
+	 * Some variant (STM32) doesn't have opendrain bit, nevertheless
+	 * pins can be set accordingly using pinctrl
+	 */
+	if (!variant->opendrain) {
+		host->pinctrl = devm_pinctrl_get(&dev->dev);
+		if (IS_ERR(host->pinctrl)) {
+			dev_err(&dev->dev, "failed to get pinctrl");
+			ret = PTR_ERR(host->pinctrl);
+			goto host_free;
+		}
+
+		host->pins_default = pinctrl_lookup_state(host->pinctrl,
+							  PINCTRL_STATE_DEFAULT);
+		if (IS_ERR(host->pins_default)) {
+			dev_err(mmc_dev(mmc), "Can't select default pins\n");
+			ret = PTR_ERR(host->pins_default);
+			goto host_free;
+		}
+
+		host->pins_opendrain = pinctrl_lookup_state(host->pinctrl,
+							    MMCI_PINCTRL_STATE_OPENDRAIN);
+		if (IS_ERR(host->pins_opendrain)) {
+			dev_err(mmc_dev(mmc), "Can't select opendrain pins\n");
+			ret = PTR_ERR(host->pins_opendrain);
+			goto host_free;
+		}
+	}
+
 	host->hw_designer = amba_manf(dev);
 	host->hw_revision = amba_rev(dev);
 	dev_dbg(mmc_dev(mmc), "designer ID = 0x%02x\n", host->hw_designer);
@@ -1729,7 +1813,10 @@ static int mmci_probe(struct amba_device *dev,
 	spin_lock_init(&host->lock);
 
 	writel(0, host->base + MMCIMASK0);
-	writel(0, host->base + MMCIMASK1);
+
+	if (variant->mmcimask1)
+		writel(0, host->base + MMCIMASK1);
+
 	writel(0xfff, host->base + MMCICLEAR);
 
 	/*
@@ -1809,6 +1896,7 @@ static int mmci_remove(struct amba_device *dev)
 
 	if (mmc) {
 		struct mmci_host *host = mmc_priv(mmc);
+		struct variant_data *variant = host->variant;
 
 		/*
 		 * Undo pm_runtime_put() in probe.  We use the _sync
@@ -1819,7 +1907,9 @@ static int mmci_remove(struct amba_device *dev)
 		mmc_remove_host(mmc);
 
 		writel(0, host->base + MMCIMASK0);
-		writel(0, host->base + MMCIMASK1);
+
+		if (variant->mmcimask1)
+			writel(0, host->base + MMCIMASK1);
 
 		writel(0, host->base + MMCICOMMAND);
 		writel(0, host->base + MMCIDATACTRL);
@@ -1951,6 +2041,11 @@ static const struct amba_id mmci_ids[] = {
 		.mask   = 0xf0ffffff,
 		.data	= &variant_ux500v2,
 	},
+	{
+		.id     = 0x00880180,
+		.mask   = 0x00ffffff,
+		.data	= &variant_stm32,
+	},
 	/* Qualcomm variants */
 	{
 		.id     = 0x00051180,
diff --git a/drivers/mmc/host/mmci.h b/drivers/mmc/host/mmci.h
index 4a8bef1aac8f..f91cdf7f6dae 100644
--- a/drivers/mmc/host/mmci.h
+++ b/drivers/mmc/host/mmci.h
@@ -192,6 +192,8 @@
 
 #define NR_SG		128
 
+#define MMCI_PINCTRL_STATE_OPENDRAIN "opendrain"
+
 struct clk;
 struct variant_data;
 struct dma_chan;
@@ -223,9 +225,13 @@ struct mmci_host {
 	u32			clk_reg;
 	u32			datactrl_reg;
 	u32			busy_status;
+	u32			mask1_reg;
 	bool			vqmmc_enabled;
 	struct mmci_platform_data *plat;
 	struct variant_data	*variant;
+	struct pinctrl		*pinctrl;
+	struct pinctrl_state	*pins_default;
+	struct pinctrl_state	*pins_opendrain;
 
 	u8			hw_designer;
 	u8			hw_revision:4;
diff --git a/drivers/mmc/host/renesas_sdhi.h b/drivers/mmc/host/renesas_sdhi.h
index b9dfea5d8193..f13f798d8506 100644
--- a/drivers/mmc/host/renesas_sdhi.h
+++ b/drivers/mmc/host/renesas_sdhi.h
@@ -35,6 +35,28 @@ struct renesas_sdhi_of_data {
 	unsigned short max_segs;
 };
 
+struct tmio_mmc_dma {
+	enum dma_slave_buswidth dma_buswidth;
+	bool (*filter)(struct dma_chan *chan, void *arg);
+	void (*enable)(struct tmio_mmc_host *host, bool enable);
+	struct completion	dma_dataend;
+	struct tasklet_struct	dma_complete;
+};
+
+struct renesas_sdhi {
+	struct clk *clk;
+	struct clk *clk_cd;
+	struct tmio_mmc_data mmc_data;
+	struct tmio_mmc_dma dma_priv;
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *pins_default, *pins_uhs;
+	void __iomem *scc_ctl;
+	u32 scc_tappos;
+};
+
+#define host_to_priv(host) \
+	container_of((host)->pdata, struct renesas_sdhi, mmc_data)
+
 int renesas_sdhi_probe(struct platform_device *pdev,
 		       const struct tmio_mmc_dma_ops *dma_ops);
 int renesas_sdhi_remove(struct platform_device *pdev);
diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c
index 157e1d9e7725..80943fa07db6 100644
--- a/drivers/mmc/host/renesas_sdhi_core.c
+++ b/drivers/mmc/host/renesas_sdhi_core.c
@@ -47,19 +47,6 @@
 #define SDHI_VER_GEN3_SD	0xcc10
 #define SDHI_VER_GEN3_SDMMC	0xcd10
 
-#define host_to_priv(host) \
-	container_of((host)->pdata, struct renesas_sdhi, mmc_data)
-
-struct renesas_sdhi {
-	struct clk *clk;
-	struct clk *clk_cd;
-	struct tmio_mmc_data mmc_data;
-	struct tmio_mmc_dma dma_priv;
-	struct pinctrl *pinctrl;
-	struct pinctrl_state *pins_default, *pins_uhs;
-	void __iomem *scc_ctl;
-};
-
 static void renesas_sdhi_sdbuf_width(struct tmio_mmc_host *host, int width)
 {
 	u32 val;
@@ -281,7 +268,7 @@ static unsigned int renesas_sdhi_init_tuning(struct tmio_mmc_host *host)
 		       ~SH_MOBILE_SDHI_SCC_RVSCNTL_RVSEN &
 		       sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_RVSCNTL));
 
-	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_DT2FF, host->scc_tappos);
+	sd_scc_write32(host, priv, SH_MOBILE_SDHI_SCC_DT2FF, priv->scc_tappos);
 
 	/* Read TAPNUM */
 	return (sd_scc_read32(host, priv, SH_MOBILE_SDHI_SCC_DTCNTL) >>
@@ -498,7 +485,7 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 	if (IS_ERR(priv->clk)) {
 		ret = PTR_ERR(priv->clk);
 		dev_err(&pdev->dev, "cannot get clock: %d\n", ret);
-		goto eprobe;
+		return ret;
 	}
 
 	/*
@@ -524,11 +511,9 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 						"state_uhs");
 	}
 
-	host = tmio_mmc_host_alloc(pdev);
-	if (!host) {
-		ret = -ENOMEM;
-		goto eprobe;
-	}
+	host = tmio_mmc_host_alloc(pdev, mmc_data);
+	if (IS_ERR(host))
+		return PTR_ERR(host);
 
 	if (of_data) {
 		mmc_data->flags |= of_data->tmio_flags;
@@ -542,18 +527,18 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 		host->bus_shift = of_data->bus_shift;
 	}
 
-	host->dma		= dma_priv;
 	host->write16_hook	= renesas_sdhi_write16_hook;
 	host->clk_enable	= renesas_sdhi_clk_enable;
 	host->clk_update	= renesas_sdhi_clk_update;
 	host->clk_disable	= renesas_sdhi_clk_disable;
 	host->multi_io_quirk	= renesas_sdhi_multi_io_quirk;
+	host->dma_ops		= dma_ops;
 
 	/* SDR speeds are only available on Gen2+ */
 	if (mmc_data->flags & TMIO_MMC_MIN_RCAR2) {
 		/* card_busy caused issues on r8a73a4 (pre-Gen2) CD-less SDHI */
-		host->card_busy	= renesas_sdhi_card_busy;
-		host->start_signal_voltage_switch =
+		host->ops.card_busy = renesas_sdhi_card_busy;
+		host->ops.start_signal_voltage_switch =
 			renesas_sdhi_start_signal_voltage_switch;
 	}
 
@@ -587,10 +572,14 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 	/* All SDHI have SDIO status bits which must be 1 */
 	mmc_data->flags |= TMIO_MMC_SDIO_STATUS_SETBITS;
 
-	ret = tmio_mmc_host_probe(host, mmc_data, dma_ops);
-	if (ret < 0)
+	ret = renesas_sdhi_clk_enable(host);
+	if (ret)
 		goto efree;
 
+	ret = tmio_mmc_host_probe(host);
+	if (ret < 0)
+		goto edisclk;
+
 	/* One Gen2 SDHI incarnation does NOT have a CBSY bit */
 	if (sd_ctrl_read16(host, CTL_VERSION) == SDHI_VER_GEN2_SDR50)
 		mmc_data->flags &= ~TMIO_MMC_HAVE_CBSY;
@@ -607,7 +596,7 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 		for (i = 0; i < of_data->taps_num; i++) {
 			if (taps[i].clk_rate == 0 ||
 			    taps[i].clk_rate == host->mmc->f_max) {
-				host->scc_tappos = taps->tap;
+				priv->scc_tappos = taps->tap;
 				hit = true;
 				break;
 			}
@@ -651,19 +640,21 @@ int renesas_sdhi_probe(struct platform_device *pdev,
 
 eirq:
 	tmio_mmc_host_remove(host);
+edisclk:
+	renesas_sdhi_clk_disable(host);
 efree:
 	tmio_mmc_host_free(host);
-eprobe:
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(renesas_sdhi_probe);
 
 int renesas_sdhi_remove(struct platform_device *pdev)
 {
-	struct mmc_host *mmc = platform_get_drvdata(pdev);
-	struct tmio_mmc_host *host = mmc_priv(mmc);
+	struct tmio_mmc_host *host = platform_get_drvdata(pdev);
 
 	tmio_mmc_host_remove(host);
+	renesas_sdhi_clk_disable(host);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
index 41cbe84c1d18..7c03cfead6f9 100644
--- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
@@ -103,6 +103,8 @@ renesas_sdhi_internal_dmac_dm_write(struct tmio_mmc_host *host,
 static void
 renesas_sdhi_internal_dmac_enable_dma(struct tmio_mmc_host *host, bool enable)
 {
+	struct renesas_sdhi *priv = host_to_priv(host);
+
 	if (!host->chan_tx || !host->chan_rx)
 		return;
 
@@ -110,8 +112,8 @@ renesas_sdhi_internal_dmac_enable_dma(struct tmio_mmc_host *host, bool enable)
 		renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO1,
 						    INFO1_CLEAR);
 
-	if (host->dma->enable)
-		host->dma->enable(host, enable);
+	if (priv->dma_priv.enable)
+		priv->dma_priv.enable(host, enable);
 }
 
 static void
@@ -130,7 +132,9 @@ renesas_sdhi_internal_dmac_abort_dma(struct tmio_mmc_host *host) {
 
 static void
 renesas_sdhi_internal_dmac_dataend_dma(struct tmio_mmc_host *host) {
-	tasklet_schedule(&host->dma_complete);
+	struct renesas_sdhi *priv = host_to_priv(host);
+
+	tasklet_schedule(&priv->dma_priv.dma_complete);
 }
 
 static void
@@ -220,10 +224,12 @@ static void
 renesas_sdhi_internal_dmac_request_dma(struct tmio_mmc_host *host,
 				       struct tmio_mmc_data *pdata)
 {
+	struct renesas_sdhi *priv = host_to_priv(host);
+
 	/* Each value is set to non-zero to assume "enabling" each DMA */
 	host->chan_rx = host->chan_tx = (void *)0xdeadbeaf;
 
-	tasklet_init(&host->dma_complete,
+	tasklet_init(&priv->dma_priv.dma_complete,
 		     renesas_sdhi_internal_dmac_complete_tasklet_fn,
 		     (unsigned long)host);
 	tasklet_init(&host->dma_issue,
@@ -255,6 +261,7 @@ static const struct soc_device_attribute gen3_soc_whitelist[] = {
         { .soc_id = "r8a7795", .revision = "ES1.*" },
         { .soc_id = "r8a7795", .revision = "ES2.0" },
         { .soc_id = "r8a7796", .revision = "ES1.0" },
+        { .soc_id = "r8a77995", .revision = "ES1.0" },
         { /* sentinel */ }
 };
 
diff --git a/drivers/mmc/host/renesas_sdhi_sys_dmac.c b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
index 9ab10436e4b8..82d757c480b2 100644
--- a/drivers/mmc/host/renesas_sdhi_sys_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_sys_dmac.c
@@ -117,11 +117,13 @@ MODULE_DEVICE_TABLE(of, renesas_sdhi_sys_dmac_of_match);
 static void renesas_sdhi_sys_dmac_enable_dma(struct tmio_mmc_host *host,
 					     bool enable)
 {
+	struct renesas_sdhi *priv = host_to_priv(host);
+
 	if (!host->chan_tx || !host->chan_rx)
 		return;
 
-	if (host->dma->enable)
-		host->dma->enable(host, enable);
+	if (priv->dma_priv.enable)
+		priv->dma_priv.enable(host, enable);
 }
 
 static void renesas_sdhi_sys_dmac_abort_dma(struct tmio_mmc_host *host)
@@ -138,12 +140,15 @@ static void renesas_sdhi_sys_dmac_abort_dma(struct tmio_mmc_host *host)
 
 static void renesas_sdhi_sys_dmac_dataend_dma(struct tmio_mmc_host *host)
 {
-	complete(&host->dma_dataend);
+	struct renesas_sdhi *priv = host_to_priv(host);
+
+	complete(&priv->dma_priv.dma_dataend);
 }
 
 static void renesas_sdhi_sys_dmac_dma_callback(void *arg)
 {
 	struct tmio_mmc_host *host = arg;
+	struct renesas_sdhi *priv = host_to_priv(host);
 
 	spin_lock_irq(&host->lock);
 
@@ -161,7 +166,7 @@ static void renesas_sdhi_sys_dmac_dma_callback(void *arg)
 
 	spin_unlock_irq(&host->lock);
 
-	wait_for_completion(&host->dma_dataend);
+	wait_for_completion(&priv->dma_priv.dma_dataend);
 
 	spin_lock_irq(&host->lock);
 	tmio_mmc_do_data_irq(host);
@@ -171,6 +176,7 @@ static void renesas_sdhi_sys_dmac_dma_callback(void *arg)
 
 static void renesas_sdhi_sys_dmac_start_dma_rx(struct tmio_mmc_host *host)
 {
+	struct renesas_sdhi *priv = host_to_priv(host);
 	struct scatterlist *sg = host->sg_ptr, *sg_tmp;
 	struct dma_async_tx_descriptor *desc = NULL;
 	struct dma_chan *chan = host->chan_rx;
@@ -214,7 +220,7 @@ static void renesas_sdhi_sys_dmac_start_dma_rx(struct tmio_mmc_host *host)
 					       DMA_CTRL_ACK);
 
 	if (desc) {
-		reinit_completion(&host->dma_dataend);
+		reinit_completion(&priv->dma_priv.dma_dataend);
 		desc->callback = renesas_sdhi_sys_dmac_dma_callback;
 		desc->callback_param = host;
 
@@ -245,6 +251,7 @@ static void renesas_sdhi_sys_dmac_start_dma_rx(struct tmio_mmc_host *host)
 
 static void renesas_sdhi_sys_dmac_start_dma_tx(struct tmio_mmc_host *host)
 {
+	struct renesas_sdhi *priv = host_to_priv(host);
 	struct scatterlist *sg = host->sg_ptr, *sg_tmp;
 	struct dma_async_tx_descriptor *desc = NULL;
 	struct dma_chan *chan = host->chan_tx;
@@ -293,7 +300,7 @@ static void renesas_sdhi_sys_dmac_start_dma_tx(struct tmio_mmc_host *host)
 					       DMA_CTRL_ACK);
 
 	if (desc) {
-		reinit_completion(&host->dma_dataend);
+		reinit_completion(&priv->dma_priv.dma_dataend);
 		desc->callback = renesas_sdhi_sys_dmac_dma_callback;
 		desc->callback_param = host;
 
@@ -341,7 +348,7 @@ static void renesas_sdhi_sys_dmac_issue_tasklet_fn(unsigned long priv)
 
 	spin_lock_irq(&host->lock);
 
-	if (host && host->data) {
+	if (host->data) {
 		if (host->data->flags & MMC_DATA_READ)
 			chan = host->chan_rx;
 		else
@@ -359,9 +366,11 @@ static void renesas_sdhi_sys_dmac_issue_tasklet_fn(unsigned long priv)
 static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host,
 					      struct tmio_mmc_data *pdata)
 {
+	struct renesas_sdhi *priv = host_to_priv(host);
+
 	/* We can only either use DMA for both Tx and Rx or not use it at all */
-	if (!host->dma || (!host->pdev->dev.of_node &&
-			   (!pdata->chan_priv_tx || !pdata->chan_priv_rx)))
+	if (!host->pdev->dev.of_node &&
+	    (!pdata->chan_priv_tx || !pdata->chan_priv_rx))
 		return;
 
 	if (!host->chan_tx && !host->chan_rx) {
@@ -378,7 +387,7 @@ static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host,
 		dma_cap_set(DMA_SLAVE, mask);
 
 		host->chan_tx = dma_request_slave_channel_compat(mask,
-					host->dma->filter, pdata->chan_priv_tx,
+					priv->dma_priv.filter, pdata->chan_priv_tx,
 					&host->pdev->dev, "tx");
 		dev_dbg(&host->pdev->dev, "%s: TX: got channel %p\n", __func__,
 			host->chan_tx);
@@ -389,7 +398,7 @@ static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host,
 		cfg.direction = DMA_MEM_TO_DEV;
 		cfg.dst_addr = res->start +
 			(CTL_SD_DATA_PORT << host->bus_shift);
-		cfg.dst_addr_width = host->dma->dma_buswidth;
+		cfg.dst_addr_width = priv->dma_priv.dma_buswidth;
 		if (!cfg.dst_addr_width)
 			cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 		cfg.src_addr = 0;
@@ -398,7 +407,7 @@ static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host,
 			goto ecfgtx;
 
 		host->chan_rx = dma_request_slave_channel_compat(mask,
-					host->dma->filter, pdata->chan_priv_rx,
+					priv->dma_priv.filter, pdata->chan_priv_rx,
 					&host->pdev->dev, "rx");
 		dev_dbg(&host->pdev->dev, "%s: RX: got channel %p\n", __func__,
 			host->chan_rx);
@@ -408,7 +417,7 @@ static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host,
 
 		cfg.direction = DMA_DEV_TO_MEM;
 		cfg.src_addr = cfg.dst_addr + host->pdata->dma_rx_offset;
-		cfg.src_addr_width = host->dma->dma_buswidth;
+		cfg.src_addr_width = priv->dma_priv.dma_buswidth;
 		if (!cfg.src_addr_width)
 			cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
 		cfg.dst_addr = 0;
@@ -420,7 +429,7 @@ static void renesas_sdhi_sys_dmac_request_dma(struct tmio_mmc_host *host,
 		if (!host->bounce_buf)
 			goto ebouncebuf;
 
-		init_completion(&host->dma_dataend);
+		init_completion(&priv->dma_priv.dma_dataend);
 		tasklet_init(&host->dma_issue,
 			     renesas_sdhi_sys_dmac_issue_tasklet_fn,
 			     (unsigned long)host);
diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c
index 0848dc0f882e..30bd8081307e 100644
--- a/drivers/mmc/host/rtsx_pci_sdmmc.c
+++ b/drivers/mmc/host/rtsx_pci_sdmmc.c
@@ -30,7 +30,7 @@
 #include <linux/mmc/sd.h>
 #include <linux/mmc/sdio.h>
 #include <linux/mmc/card.h>
-#include <linux/mfd/rtsx_pci.h>
+#include <linux/rtsx_pci.h>
 #include <asm/unaligned.h>
 
 struct realtek_pci_sdmmc {
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index 76da1687ab37..78422079ecfa 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -31,7 +31,7 @@
 #include <linux/scatterlist.h>
 #include <linux/pm_runtime.h>
 
-#include <linux/mfd/rtsx_usb.h>
+#include <linux/rtsx_usb.h>
 #include <asm/unaligned.h>
 
 #if defined(CONFIG_LEDS_CLASS) || (defined(CONFIG_LEDS_CLASS_MODULE) && \
diff --git a/drivers/mmc/host/s3cmci.c b/drivers/mmc/host/s3cmci.c
index 555c7f133eb8..f77493604312 100644
--- a/drivers/mmc/host/s3cmci.c
+++ b/drivers/mmc/host/s3cmci.c
@@ -1660,7 +1660,7 @@ static int s3cmci_probe(struct platform_device *pdev)
 	}
 
 	host->irq = platform_get_irq(pdev, 0);
-	if (host->irq == 0) {
+	if (host->irq <= 0) {
 		dev_err(&pdev->dev, "failed to get interrupt resource.\n");
 		ret = -EINVAL;
 		goto probe_iounmap;
diff --git a/drivers/mmc/host/sdhci-acpi.c b/drivers/mmc/host/sdhci-acpi.c
index b988997a1e80..32321bd596d8 100644
--- a/drivers/mmc/host/sdhci-acpi.c
+++ b/drivers/mmc/host/sdhci-acpi.c
@@ -76,6 +76,7 @@ struct sdhci_acpi_slot {
 	size_t		priv_size;
 	int (*probe_slot)(struct platform_device *, const char *, const char *);
 	int (*remove_slot)(struct platform_device *);
+	int (*setup_host)(struct platform_device *pdev);
 };
 
 struct sdhci_acpi_host {
@@ -96,14 +97,21 @@ static inline bool sdhci_acpi_flag(struct sdhci_acpi_host *c, unsigned int flag)
 	return c->slot && (c->slot->flags & flag);
 }
 
+#define INTEL_DSM_HS_CAPS_SDR25		BIT(0)
+#define INTEL_DSM_HS_CAPS_DDR50		BIT(1)
+#define INTEL_DSM_HS_CAPS_SDR50		BIT(2)
+#define INTEL_DSM_HS_CAPS_SDR104	BIT(3)
+
 enum {
 	INTEL_DSM_FNS		=  0,
 	INTEL_DSM_V18_SWITCH	=  3,
 	INTEL_DSM_V33_SWITCH	=  4,
+	INTEL_DSM_HS_CAPS	=  8,
 };
 
 struct intel_host {
 	u32	dsm_fns;
+	u32	hs_caps;
 };
 
 static const guid_t intel_dsm_guid =
@@ -152,6 +160,8 @@ static void intel_dsm_init(struct intel_host *intel_host, struct device *dev,
 {
 	int err;
 
+	intel_host->hs_caps = ~0;
+
 	err = __intel_dsm(intel_host, dev, INTEL_DSM_FNS, &intel_host->dsm_fns);
 	if (err) {
 		pr_debug("%s: DSM not supported, error %d\n",
@@ -161,6 +171,8 @@ static void intel_dsm_init(struct intel_host *intel_host, struct device *dev,
 
 	pr_debug("%s: DSM function mask %#x\n",
 		 mmc_hostname(mmc), intel_host->dsm_fns);
+
+	intel_dsm(intel_host, dev, INTEL_DSM_HS_CAPS, &intel_host->hs_caps);
 }
 
 static int intel_start_signal_voltage_switch(struct mmc_host *mmc,
@@ -398,6 +410,26 @@ static int intel_probe_slot(struct platform_device *pdev, const char *hid,
 	return 0;
 }
 
+static int intel_setup_host(struct platform_device *pdev)
+{
+	struct sdhci_acpi_host *c = platform_get_drvdata(pdev);
+	struct intel_host *intel_host = sdhci_acpi_priv(c);
+
+	if (!(intel_host->hs_caps & INTEL_DSM_HS_CAPS_SDR25))
+		c->host->mmc->caps &= ~MMC_CAP_UHS_SDR25;
+
+	if (!(intel_host->hs_caps & INTEL_DSM_HS_CAPS_SDR50))
+		c->host->mmc->caps &= ~MMC_CAP_UHS_SDR50;
+
+	if (!(intel_host->hs_caps & INTEL_DSM_HS_CAPS_DDR50))
+		c->host->mmc->caps &= ~MMC_CAP_UHS_DDR50;
+
+	if (!(intel_host->hs_caps & INTEL_DSM_HS_CAPS_SDR104))
+		c->host->mmc->caps &= ~MMC_CAP_UHS_SDR104;
+
+	return 0;
+}
+
 static const struct sdhci_acpi_slot sdhci_acpi_slot_int_emmc = {
 	.chip    = &sdhci_acpi_chip_int,
 	.caps    = MMC_CAP_8_BIT_DATA | MMC_CAP_NONREMOVABLE |
@@ -409,6 +441,7 @@ static const struct sdhci_acpi_slot sdhci_acpi_slot_int_emmc = {
 		   SDHCI_QUIRK2_STOP_WITH_TC |
 		   SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400,
 	.probe_slot	= intel_probe_slot,
+	.setup_host	= intel_setup_host,
 	.priv_size	= sizeof(struct intel_host),
 };
 
@@ -421,6 +454,7 @@ static const struct sdhci_acpi_slot sdhci_acpi_slot_int_sdio = {
 	.flags   = SDHCI_ACPI_RUNTIME_PM,
 	.pm_caps = MMC_PM_KEEP_POWER,
 	.probe_slot	= intel_probe_slot,
+	.setup_host	= intel_setup_host,
 	.priv_size	= sizeof(struct intel_host),
 };
 
@@ -432,6 +466,7 @@ static const struct sdhci_acpi_slot sdhci_acpi_slot_int_sd = {
 		   SDHCI_QUIRK2_STOP_WITH_TC,
 	.caps    = MMC_CAP_WAIT_WHILE_BUSY | MMC_CAP_AGGRESSIVE_PM,
 	.probe_slot	= intel_probe_slot,
+	.setup_host	= intel_setup_host,
 	.priv_size	= sizeof(struct intel_host),
 };
 
@@ -446,6 +481,83 @@ static const struct sdhci_acpi_slot sdhci_acpi_slot_qcom_sd = {
 	.caps    = MMC_CAP_NONREMOVABLE,
 };
 
+/* AMD sdhci reset dll register. */
+#define SDHCI_AMD_RESET_DLL_REGISTER    0x908
+
+static int amd_select_drive_strength(struct mmc_card *card,
+				     unsigned int max_dtr, int host_drv,
+				     int card_drv, int *drv_type)
+{
+	return MMC_SET_DRIVER_TYPE_A;
+}
+
+static void sdhci_acpi_amd_hs400_dll(struct sdhci_host *host)
+{
+	/* AMD Platform requires dll setting */
+	sdhci_writel(host, 0x40003210, SDHCI_AMD_RESET_DLL_REGISTER);
+	usleep_range(10, 20);
+	sdhci_writel(host, 0x40033210, SDHCI_AMD_RESET_DLL_REGISTER);
+}
+
+/*
+ * For AMD Platform it is required to disable the tuning
+ * bit first controller to bring to HS Mode from HS200
+ * mode, later enable to tune to HS400 mode.
+ */
+static void amd_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	unsigned int old_timing = host->timing;
+
+	sdhci_set_ios(mmc, ios);
+	if (old_timing == MMC_TIMING_MMC_HS200 &&
+	    ios->timing == MMC_TIMING_MMC_HS)
+		sdhci_writew(host, 0x9, SDHCI_HOST_CONTROL2);
+	if (old_timing != MMC_TIMING_MMC_HS400 &&
+	    ios->timing == MMC_TIMING_MMC_HS400) {
+		sdhci_writew(host, 0x80, SDHCI_HOST_CONTROL2);
+		sdhci_acpi_amd_hs400_dll(host);
+	}
+}
+
+static const struct sdhci_ops sdhci_acpi_ops_amd = {
+	.set_clock	= sdhci_set_clock,
+	.set_bus_width	= sdhci_set_bus_width,
+	.reset		= sdhci_reset,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+};
+
+static const struct sdhci_acpi_chip sdhci_acpi_chip_amd = {
+	.ops = &sdhci_acpi_ops_amd,
+};
+
+static int sdhci_acpi_emmc_amd_probe_slot(struct platform_device *pdev,
+					  const char *hid, const char *uid)
+{
+	struct sdhci_acpi_host *c = platform_get_drvdata(pdev);
+	struct sdhci_host *host   = c->host;
+
+	sdhci_read_caps(host);
+	if (host->caps1 & SDHCI_SUPPORT_DDR50)
+		host->mmc->caps = MMC_CAP_1_8V_DDR;
+
+	if ((host->caps1 & SDHCI_SUPPORT_SDR104) &&
+	    (host->mmc->caps & MMC_CAP_1_8V_DDR))
+		host->mmc->caps2 = MMC_CAP2_HS400_1_8V;
+
+	host->mmc_host_ops.select_drive_strength = amd_select_drive_strength;
+	host->mmc_host_ops.set_ios = amd_set_ios;
+	return 0;
+}
+
+static const struct sdhci_acpi_slot sdhci_acpi_slot_amd_emmc = {
+	.chip   = &sdhci_acpi_chip_amd,
+	.caps   = MMC_CAP_8_BIT_DATA | MMC_CAP_NONREMOVABLE,
+	.quirks = SDHCI_QUIRK_32BIT_DMA_ADDR | SDHCI_QUIRK_32BIT_DMA_SIZE |
+			SDHCI_QUIRK_32BIT_ADMA_SIZE,
+	.probe_slot     = sdhci_acpi_emmc_amd_probe_slot,
+};
+
 struct sdhci_acpi_uid_slot {
 	const char *hid;
 	const char *uid;
@@ -469,6 +581,7 @@ static const struct sdhci_acpi_uid_slot sdhci_acpi_uids[] = {
 	{ "PNP0D40"  },
 	{ "QCOM8051", NULL, &sdhci_acpi_slot_qcom_sd_3v },
 	{ "QCOM8052", NULL, &sdhci_acpi_slot_qcom_sd },
+	{ "AMDI0040", NULL, &sdhci_acpi_slot_amd_emmc },
 	{ },
 };
 
@@ -485,6 +598,7 @@ static const struct acpi_device_id sdhci_acpi_ids[] = {
 	{ "PNP0D40"  },
 	{ "QCOM8051" },
 	{ "QCOM8052" },
+	{ "AMDI0040" },
 	{ },
 };
 MODULE_DEVICE_TABLE(acpi, sdhci_acpi_ids);
@@ -566,6 +680,10 @@ static int sdhci_acpi_probe(struct platform_device *pdev)
 	host->hw_name	= "ACPI";
 	host->ops	= &sdhci_acpi_ops_dflt;
 	host->irq	= platform_get_irq(pdev, 0);
+	if (host->irq < 0) {
+		err = -EINVAL;
+		goto err_free;
+	}
 
 	host->ioaddr = devm_ioremap_nocache(dev, iomem->start,
 					    resource_size(iomem));
@@ -609,10 +727,20 @@ static int sdhci_acpi_probe(struct platform_device *pdev)
 		}
 	}
 
-	err = sdhci_add_host(host);
+	err = sdhci_setup_host(host);
 	if (err)
 		goto err_free;
 
+	if (c->slot && c->slot->setup_host) {
+		err = c->slot->setup_host(pdev);
+		if (err)
+			goto err_cleanup;
+	}
+
+	err = __sdhci_add_host(host);
+	if (err)
+		goto err_cleanup;
+
 	if (c->use_runtime_pm) {
 		pm_runtime_set_active(dev);
 		pm_suspend_ignore_children(dev, 1);
@@ -625,6 +753,8 @@ static int sdhci_acpi_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_cleanup:
+	sdhci_cleanup_host(c->host);
 err_free:
 	sdhci_free_host(c->host);
 	return err;
diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c
index 8b941f814472..cd2b5f643a15 100644
--- a/drivers/mmc/host/sdhci-esdhc-imx.c
+++ b/drivers/mmc/host/sdhci-esdhc-imx.c
@@ -193,6 +193,7 @@ struct pltfm_imx_data {
 	struct clk *clk_ipg;
 	struct clk *clk_ahb;
 	struct clk *clk_per;
+	unsigned int actual_clock;
 	enum {
 		NO_CMD_PENDING,      /* no multiblock command pending */
 		MULTIBLK_IN_PROCESS, /* exact multiblock cmd in process */
@@ -1403,11 +1404,15 @@ static int sdhci_esdhc_runtime_suspend(struct device *dev)
 	int ret;
 
 	ret = sdhci_runtime_suspend_host(host);
+	if (ret)
+		return ret;
 
 	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
 		mmc_retune_needed(host->mmc);
 
 	if (!sdhci_sdio_irq_enabled(host)) {
+		imx_data->actual_clock = host->mmc->actual_clock;
+		esdhc_pltfm_set_clock(host, 0);
 		clk_disable_unprepare(imx_data->clk_per);
 		clk_disable_unprepare(imx_data->clk_ipg);
 	}
@@ -1423,31 +1428,34 @@ static int sdhci_esdhc_runtime_resume(struct device *dev)
 	struct pltfm_imx_data *imx_data = sdhci_pltfm_priv(pltfm_host);
 	int err;
 
+	err = clk_prepare_enable(imx_data->clk_ahb);
+	if (err)
+		return err;
+
 	if (!sdhci_sdio_irq_enabled(host)) {
 		err = clk_prepare_enable(imx_data->clk_per);
 		if (err)
-			return err;
+			goto disable_ahb_clk;
 		err = clk_prepare_enable(imx_data->clk_ipg);
 		if (err)
 			goto disable_per_clk;
+		esdhc_pltfm_set_clock(host, imx_data->actual_clock);
 	}
-	err = clk_prepare_enable(imx_data->clk_ahb);
-	if (err)
-		goto disable_ipg_clk;
+
 	err = sdhci_runtime_resume_host(host);
 	if (err)
-		goto disable_ahb_clk;
+		goto disable_ipg_clk;
 
 	return 0;
 
-disable_ahb_clk:
-	clk_disable_unprepare(imx_data->clk_ahb);
 disable_ipg_clk:
 	if (!sdhci_sdio_irq_enabled(host))
 		clk_disable_unprepare(imx_data->clk_ipg);
 disable_per_clk:
 	if (!sdhci_sdio_irq_enabled(host))
 		clk_disable_unprepare(imx_data->clk_per);
+disable_ahb_clk:
+	clk_disable_unprepare(imx_data->clk_ahb);
 	return err;
 }
 #endif
diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c
index 0720ea717011..c33a5f7393bd 100644
--- a/drivers/mmc/host/sdhci-of-arasan.c
+++ b/drivers/mmc/host/sdhci-of-arasan.c
@@ -25,11 +25,13 @@
 #include <linux/of_device.h>
 #include <linux/phy/phy.h>
 #include <linux/regmap.h>
-#include "sdhci-pltfm.h"
 #include <linux/of.h>
 
-#define SDHCI_ARASAN_VENDOR_REGISTER	0x78
+#include "cqhci.h"
+#include "sdhci-pltfm.h"
 
+#define SDHCI_ARASAN_VENDOR_REGISTER	0x78
+#define SDHCI_ARASAN_CQE_BASE_ADDR	0x200
 #define VENDOR_ENHANCED_STROBE		BIT(0)
 
 #define PHY_CLK_TOO_SLOW_HZ		400000
@@ -90,6 +92,7 @@ struct sdhci_arasan_data {
 	struct phy	*phy;
 	bool		is_phy_on;
 
+	bool		has_cqe;
 	struct clk_hw	sdcardclk_hw;
 	struct clk      *sdcardclk;
 
@@ -262,6 +265,17 @@ static int sdhci_arasan_voltage_switch(struct mmc_host *mmc,
 	return -EINVAL;
 }
 
+static void sdhci_arasan_set_power(struct sdhci_host *host, unsigned char mode,
+		     unsigned short vdd)
+{
+	if (!IS_ERR(host->mmc->supply.vmmc)) {
+		struct mmc_host *mmc = host->mmc;
+
+		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
+	}
+	sdhci_set_power_noreg(host, mode, vdd);
+}
+
 static const struct sdhci_ops sdhci_arasan_ops = {
 	.set_clock = sdhci_arasan_set_clock,
 	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
@@ -269,6 +283,7 @@ static const struct sdhci_ops sdhci_arasan_ops = {
 	.set_bus_width = sdhci_set_bus_width,
 	.reset = sdhci_arasan_reset,
 	.set_uhs_signaling = sdhci_set_uhs_signaling,
+	.set_power = sdhci_arasan_set_power,
 };
 
 static const struct sdhci_pltfm_data sdhci_arasan_pdata = {
@@ -278,6 +293,62 @@ static const struct sdhci_pltfm_data sdhci_arasan_pdata = {
 			SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
 };
 
+static u32 sdhci_arasan_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+	int cmd_error = 0;
+	int data_error = 0;
+
+	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+		return intmask;
+
+	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+	return 0;
+}
+
+static void sdhci_arasan_dumpregs(struct mmc_host *mmc)
+{
+	sdhci_dumpregs(mmc_priv(mmc));
+}
+
+static void sdhci_arasan_cqe_enable(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	u32 reg;
+
+	reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	while (reg & SDHCI_DATA_AVAILABLE) {
+		sdhci_readl(host, SDHCI_BUFFER);
+		reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	}
+
+	sdhci_cqe_enable(mmc);
+}
+
+static const struct cqhci_host_ops sdhci_arasan_cqhci_ops = {
+	.enable         = sdhci_arasan_cqe_enable,
+	.disable        = sdhci_cqe_disable,
+	.dumpregs       = sdhci_arasan_dumpregs,
+};
+
+static const struct sdhci_ops sdhci_arasan_cqe_ops = {
+	.set_clock = sdhci_arasan_set_clock,
+	.get_max_clock = sdhci_pltfm_clk_get_max_clock,
+	.get_timeout_clock = sdhci_pltfm_clk_get_max_clock,
+	.set_bus_width = sdhci_set_bus_width,
+	.reset = sdhci_arasan_reset,
+	.set_uhs_signaling = sdhci_set_uhs_signaling,
+	.set_power = sdhci_arasan_set_power,
+	.irq = sdhci_arasan_cqhci_irq,
+};
+
+static const struct sdhci_pltfm_data sdhci_arasan_cqe_pdata = {
+	.ops = &sdhci_arasan_cqe_ops,
+	.quirks = SDHCI_QUIRK_CAP_CLOCK_BASE_BROKEN,
+	.quirks2 = SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
+			SDHCI_QUIRK2_CLOCK_DIV_ZERO_BROKEN,
+};
+
 #ifdef CONFIG_PM_SLEEP
 /**
  * sdhci_arasan_suspend - Suspend method for the driver
@@ -297,6 +368,12 @@ static int sdhci_arasan_suspend(struct device *dev)
 	if (host->tuning_mode != SDHCI_TUNING_MODE_3)
 		mmc_retune_needed(host->mmc);
 
+	if (sdhci_arasan->has_cqe) {
+		ret = cqhci_suspend(host->mmc);
+		if (ret)
+			return ret;
+	}
+
 	ret = sdhci_suspend_host(host);
 	if (ret)
 		return ret;
@@ -353,7 +430,16 @@ static int sdhci_arasan_resume(struct device *dev)
 		sdhci_arasan->is_phy_on = true;
 	}
 
-	return sdhci_resume_host(host);
+	ret = sdhci_resume_host(host);
+	if (ret) {
+		dev_err(dev, "Cannot resume host.\n");
+		return ret;
+	}
+
+	if (sdhci_arasan->has_cqe)
+		return cqhci_resume(host->mmc);
+
+	return 0;
 }
 #endif /* ! CONFIG_PM_SLEEP */
 
@@ -556,6 +642,49 @@ static void sdhci_arasan_unregister_sdclk(struct device *dev)
 	of_clk_del_provider(dev->of_node);
 }
 
+static int sdhci_arasan_add_host(struct sdhci_arasan_data *sdhci_arasan)
+{
+	struct sdhci_host *host = sdhci_arasan->host;
+	struct cqhci_host *cq_host;
+	bool dma64;
+	int ret;
+
+	if (!sdhci_arasan->has_cqe)
+		return sdhci_add_host(host);
+
+	ret = sdhci_setup_host(host);
+	if (ret)
+		return ret;
+
+	cq_host = devm_kzalloc(host->mmc->parent,
+			       sizeof(*cq_host), GFP_KERNEL);
+	if (!cq_host) {
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	cq_host->mmio = host->ioaddr + SDHCI_ARASAN_CQE_BASE_ADDR;
+	cq_host->ops = &sdhci_arasan_cqhci_ops;
+
+	dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
+	if (dma64)
+		cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+
+	ret = cqhci_init(cq_host, host->mmc, dma64);
+	if (ret)
+		goto cleanup;
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto cleanup;
+
+	return 0;
+
+cleanup:
+	sdhci_cleanup_host(host);
+	return ret;
+}
+
 static int sdhci_arasan_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -566,9 +695,15 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 	struct sdhci_pltfm_host *pltfm_host;
 	struct sdhci_arasan_data *sdhci_arasan;
 	struct device_node *np = pdev->dev.of_node;
+	const struct sdhci_pltfm_data *pdata;
+
+	if (of_device_is_compatible(pdev->dev.of_node, "arasan,sdhci-5.1"))
+		pdata = &sdhci_arasan_cqe_pdata;
+	else
+		pdata = &sdhci_arasan_pdata;
+
+	host = sdhci_pltfm_init(pdev, pdata, sizeof(*sdhci_arasan));
 
-	host = sdhci_pltfm_init(pdev, &sdhci_arasan_pdata,
-				sizeof(*sdhci_arasan));
 	if (IS_ERR(host))
 		return PTR_ERR(host);
 
@@ -663,9 +798,11 @@ static int sdhci_arasan_probe(struct platform_device *pdev)
 					sdhci_arasan_hs400_enhanced_strobe;
 		host->mmc_host_ops.start_signal_voltage_switch =
 					sdhci_arasan_voltage_switch;
+		sdhci_arasan->has_cqe = true;
+		host->mmc->caps2 |= MMC_CAP2_CQE | MMC_CAP2_CQE_DCMD;
 	}
 
-	ret = sdhci_add_host(host);
+	ret = sdhci_arasan_add_host(sdhci_arasan);
 	if (ret)
 		goto err_add_host;
 
diff --git a/drivers/mmc/host/sdhci-of-esdhc.c b/drivers/mmc/host/sdhci-of-esdhc.c
index 1f424374bbbb..4ffa6b173a21 100644
--- a/drivers/mmc/host/sdhci-of-esdhc.c
+++ b/drivers/mmc/host/sdhci-of-esdhc.c
@@ -589,10 +589,18 @@ static void esdhc_pltfm_set_bus_width(struct sdhci_host *host, int width)
 
 static void esdhc_reset(struct sdhci_host *host, u8 mask)
 {
+	u32 val;
+
 	sdhci_reset(host, mask);
 
 	sdhci_writel(host, host->ier, SDHCI_INT_ENABLE);
 	sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE);
+
+	if (mask & SDHCI_RESET_ALL) {
+		val = sdhci_readl(host, ESDHC_TBCTL);
+		val &= ~ESDHC_TB_EN;
+		sdhci_writel(host, val, ESDHC_TBCTL);
+	}
 }
 
 /* The SCFG, Supplemental Configuration Unit, provides SoC specific
diff --git a/drivers/mmc/host/sdhci-pci-arasan.c b/drivers/mmc/host/sdhci-pci-arasan.c
new file mode 100644
index 000000000000..499f3205ec5c
--- /dev/null
+++ b/drivers/mmc/host/sdhci-pci-arasan.c
@@ -0,0 +1,331 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * sdhci-pci-arasan.c - Driver for Arasan PCI Controller with
+ * integrated phy.
+ *
+ * Copyright (C) 2017 Arasan Chip Systems Inc.
+ *
+ * Author: Atul Garg <agarg@arasan.com>
+ */
+
+#include <linux/pci.h>
+#include <linux/delay.h>
+
+#include "sdhci.h"
+#include "sdhci-pci.h"
+
+/* Extra registers for Arasan SD/SDIO/MMC Host Controller with PHY */
+#define PHY_ADDR_REG	0x300
+#define PHY_DAT_REG	0x304
+
+#define PHY_WRITE	BIT(8)
+#define PHY_BUSY	BIT(9)
+#define DATA_MASK	0xFF
+
+/* PHY Specific Registers */
+#define DLL_STATUS	0x00
+#define IPAD_CTRL1	0x01
+#define IPAD_CTRL2	0x02
+#define IPAD_STS	0x03
+#define IOREN_CTRL1	0x06
+#define IOREN_CTRL2	0x07
+#define IOPU_CTRL1	0x08
+#define IOPU_CTRL2	0x09
+#define ITAP_DELAY	0x0C
+#define OTAP_DELAY	0x0D
+#define STRB_SEL	0x0E
+#define CLKBUF_SEL	0x0F
+#define MODE_CTRL	0x11
+#define DLL_TRIM	0x12
+#define CMD_CTRL	0x20
+#define DATA_CTRL	0x21
+#define STRB_CTRL	0x22
+#define CLK_CTRL	0x23
+#define PHY_CTRL	0x24
+
+#define DLL_ENBL	BIT(3)
+#define RTRIM_EN	BIT(1)
+#define PDB_ENBL	BIT(1)
+#define RETB_ENBL	BIT(6)
+#define ODEN_CMD	BIT(1)
+#define ODEN_DAT	0xFF
+#define REN_STRB	BIT(0)
+#define REN_CMND	BIT(1)
+#define REN_DATA	0xFF
+#define PU_CMD		BIT(1)
+#define PU_DAT		0xFF
+#define ITAPDLY_EN	BIT(0)
+#define OTAPDLY_EN	BIT(0)
+#define OD_REL_CMD	BIT(1)
+#define OD_REL_DAT	0xFF
+#define DLLTRM_ICP	0x8
+#define PDB_CMND	BIT(0)
+#define PDB_DATA	0xFF
+#define PDB_STRB	BIT(0)
+#define PDB_CLOCK	BIT(0)
+#define CALDONE_MASK	0x10
+#define DLL_RDY_MASK	0x10
+#define MAX_CLK_BUF	0x7
+
+/* Mode Controls */
+#define ENHSTRB_MODE	BIT(0)
+#define HS400_MODE	BIT(1)
+#define LEGACY_MODE	BIT(2)
+#define DDR50_MODE	BIT(3)
+
+/*
+ * Controller has no specific bits for HS200/HS.
+ * Used BIT(4), BIT(5) for software programming.
+ */
+#define HS200_MODE	BIT(4)
+#define HISPD_MODE	BIT(5)
+
+#define OTAPDLY(x)	(((x) << 1) | OTAPDLY_EN)
+#define ITAPDLY(x)	(((x) << 1) | ITAPDLY_EN)
+#define FREQSEL(x)	(((x) << 5) | DLL_ENBL)
+#define IOPAD(x, y)	((x) | ((y) << 2))
+
+/* Arasan private data */
+struct arasan_host {
+	u32 chg_clk;
+};
+
+static int arasan_phy_addr_poll(struct sdhci_host *host, u32 offset, u32 mask)
+{
+	ktime_t timeout = ktime_add_us(ktime_get(), 100);
+	bool failed;
+	u8 val = 0;
+
+	while (1) {
+		failed = ktime_after(ktime_get(), timeout);
+		val = sdhci_readw(host, PHY_ADDR_REG);
+		if (!(val & mask))
+			return 0;
+		if (failed)
+			return -EBUSY;
+	}
+}
+
+static int arasan_phy_write(struct sdhci_host *host, u8 data, u8 offset)
+{
+	sdhci_writew(host, data, PHY_DAT_REG);
+	sdhci_writew(host, (PHY_WRITE | offset), PHY_ADDR_REG);
+	return arasan_phy_addr_poll(host, PHY_ADDR_REG, PHY_BUSY);
+}
+
+static int arasan_phy_read(struct sdhci_host *host, u8 offset, u8 *data)
+{
+	int ret;
+
+	sdhci_writew(host, 0, PHY_DAT_REG);
+	sdhci_writew(host, offset, PHY_ADDR_REG);
+	ret = arasan_phy_addr_poll(host, PHY_ADDR_REG, PHY_BUSY);
+
+	/* Masking valid data bits */
+	*data = sdhci_readw(host, PHY_DAT_REG) & DATA_MASK;
+	return ret;
+}
+
+static int arasan_phy_sts_poll(struct sdhci_host *host, u32 offset, u32 mask)
+{
+	int ret;
+	ktime_t timeout = ktime_add_us(ktime_get(), 100);
+	bool failed;
+	u8 val = 0;
+
+	while (1) {
+		failed = ktime_after(ktime_get(), timeout);
+		ret = arasan_phy_read(host, offset, &val);
+		if (ret)
+			return -EBUSY;
+		else if (val & mask)
+			return 0;
+		if (failed)
+			return -EBUSY;
+	}
+}
+
+/* Initialize the Arasan PHY */
+static int arasan_phy_init(struct sdhci_host *host)
+{
+	int ret;
+	u8 val;
+
+	/* Program IOPADs and wait for calibration to be done */
+	if (arasan_phy_read(host, IPAD_CTRL1, &val) ||
+	    arasan_phy_write(host, val | RETB_ENBL | PDB_ENBL, IPAD_CTRL1) ||
+	    arasan_phy_read(host, IPAD_CTRL2, &val) ||
+	    arasan_phy_write(host, val | RTRIM_EN, IPAD_CTRL2))
+		return -EBUSY;
+	ret = arasan_phy_sts_poll(host, IPAD_STS, CALDONE_MASK);
+	if (ret)
+		return -EBUSY;
+
+	/* Program CMD/Data lines */
+	if (arasan_phy_read(host, IOREN_CTRL1, &val) ||
+	    arasan_phy_write(host, val | REN_CMND | REN_STRB, IOREN_CTRL1) ||
+	    arasan_phy_read(host, IOPU_CTRL1, &val) ||
+	    arasan_phy_write(host, val | PU_CMD, IOPU_CTRL1) ||
+	    arasan_phy_read(host, CMD_CTRL, &val) ||
+	    arasan_phy_write(host, val | PDB_CMND, CMD_CTRL) ||
+	    arasan_phy_read(host, IOREN_CTRL2, &val) ||
+	    arasan_phy_write(host, val | REN_DATA, IOREN_CTRL2) ||
+	    arasan_phy_read(host, IOPU_CTRL2, &val) ||
+	    arasan_phy_write(host, val | PU_DAT, IOPU_CTRL2) ||
+	    arasan_phy_read(host, DATA_CTRL, &val) ||
+	    arasan_phy_write(host, val | PDB_DATA, DATA_CTRL) ||
+	    arasan_phy_read(host, STRB_CTRL, &val) ||
+	    arasan_phy_write(host, val | PDB_STRB, STRB_CTRL) ||
+	    arasan_phy_read(host, CLK_CTRL, &val) ||
+	    arasan_phy_write(host, val | PDB_CLOCK, CLK_CTRL) ||
+	    arasan_phy_read(host, CLKBUF_SEL, &val) ||
+	    arasan_phy_write(host, val | MAX_CLK_BUF, CLKBUF_SEL) ||
+	    arasan_phy_write(host, LEGACY_MODE, MODE_CTRL))
+		return -EBUSY;
+	return 0;
+}
+
+/* Set Arasan PHY for different modes */
+static int arasan_phy_set(struct sdhci_host *host, u8 mode, u8 otap,
+			  u8 drv_type, u8 itap, u8 trim, u8 clk)
+{
+	u8 val;
+	int ret;
+
+	if (mode == HISPD_MODE || mode == HS200_MODE)
+		ret = arasan_phy_write(host, 0x0, MODE_CTRL);
+	else
+		ret = arasan_phy_write(host, mode, MODE_CTRL);
+	if (ret)
+		return ret;
+	if (mode == HS400_MODE || mode == HS200_MODE) {
+		ret = arasan_phy_read(host, IPAD_CTRL1, &val);
+		if (ret)
+			return ret;
+		ret = arasan_phy_write(host, IOPAD(val, drv_type), IPAD_CTRL1);
+		if (ret)
+			return ret;
+	}
+	if (mode == LEGACY_MODE) {
+		ret = arasan_phy_write(host, 0x0, OTAP_DELAY);
+		if (ret)
+			return ret;
+		ret = arasan_phy_write(host, 0x0, ITAP_DELAY);
+	} else {
+		ret = arasan_phy_write(host, OTAPDLY(otap), OTAP_DELAY);
+		if (ret)
+			return ret;
+		if (mode != HS200_MODE)
+			ret = arasan_phy_write(host, ITAPDLY(itap), ITAP_DELAY);
+		else
+			ret = arasan_phy_write(host, 0x0, ITAP_DELAY);
+	}
+	if (ret)
+		return ret;
+	if (mode != LEGACY_MODE) {
+		ret = arasan_phy_write(host, trim, DLL_TRIM);
+		if (ret)
+			return ret;
+	}
+	ret = arasan_phy_write(host, 0, DLL_STATUS);
+	if (ret)
+		return ret;
+	if (mode != LEGACY_MODE) {
+		ret = arasan_phy_write(host, FREQSEL(clk), DLL_STATUS);
+		if (ret)
+			return ret;
+		ret = arasan_phy_sts_poll(host, DLL_STATUS, DLL_RDY_MASK);
+		if (ret)
+			return -EBUSY;
+	}
+	return 0;
+}
+
+static int arasan_select_phy_clock(struct sdhci_host *host)
+{
+	struct sdhci_pci_slot *slot = sdhci_priv(host);
+	struct arasan_host *arasan_host = sdhci_pci_priv(slot);
+	u8 clk;
+
+	if (arasan_host->chg_clk == host->mmc->ios.clock)
+		return 0;
+
+	arasan_host->chg_clk = host->mmc->ios.clock;
+	if (host->mmc->ios.clock == 200000000)
+		clk = 0x0;
+	else if (host->mmc->ios.clock == 100000000)
+		clk = 0x2;
+	else if (host->mmc->ios.clock == 50000000)
+		clk = 0x1;
+	else
+		clk = 0x0;
+
+	if (host->mmc_host_ops.hs400_enhanced_strobe) {
+		arasan_phy_set(host, ENHSTRB_MODE, 1, 0x0, 0x0,
+			       DLLTRM_ICP, clk);
+	} else {
+		switch (host->mmc->ios.timing) {
+		case MMC_TIMING_LEGACY:
+			arasan_phy_set(host, LEGACY_MODE, 0x0, 0x0, 0x0,
+				       0x0, 0x0);
+			break;
+		case MMC_TIMING_MMC_HS:
+		case MMC_TIMING_SD_HS:
+			arasan_phy_set(host, HISPD_MODE, 0x3, 0x0, 0x2,
+				       DLLTRM_ICP, clk);
+			break;
+		case MMC_TIMING_MMC_HS200:
+		case MMC_TIMING_UHS_SDR104:
+			arasan_phy_set(host, HS200_MODE, 0x2,
+				       host->mmc->ios.drv_type, 0x0,
+				       DLLTRM_ICP, clk);
+			break;
+		case MMC_TIMING_MMC_DDR52:
+		case MMC_TIMING_UHS_DDR50:
+			arasan_phy_set(host, DDR50_MODE, 0x1, 0x0,
+				       0x0, DLLTRM_ICP, clk);
+			break;
+		case MMC_TIMING_MMC_HS400:
+			arasan_phy_set(host, HS400_MODE, 0x1,
+				       host->mmc->ios.drv_type, 0xa,
+				       DLLTRM_ICP, clk);
+			break;
+		default:
+			break;
+		}
+	}
+	return 0;
+}
+
+static int arasan_pci_probe_slot(struct sdhci_pci_slot *slot)
+{
+	int err;
+
+	slot->host->mmc->caps |= MMC_CAP_NONREMOVABLE | MMC_CAP_8_BIT_DATA;
+	err = arasan_phy_init(slot->host);
+	if (err)
+		return -ENODEV;
+	return 0;
+}
+
+static void arasan_sdhci_set_clock(struct sdhci_host *host, unsigned int clock)
+{
+	sdhci_set_clock(host, clock);
+
+	/* Change phy settings for the new clock */
+	arasan_select_phy_clock(host);
+}
+
+static const struct sdhci_ops arasan_sdhci_pci_ops = {
+	.set_clock	= arasan_sdhci_set_clock,
+	.enable_dma	= sdhci_pci_enable_dma,
+	.set_bus_width	= sdhci_set_bus_width,
+	.reset		= sdhci_reset,
+	.set_uhs_signaling	= sdhci_set_uhs_signaling,
+};
+
+const struct sdhci_pci_fixes sdhci_arasan = {
+	.probe_slot = arasan_pci_probe_slot,
+	.ops        = &arasan_sdhci_pci_ops,
+	.priv_size  = sizeof(struct arasan_host),
+};
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index 3e4f04fd5175..82c4f05f91d8 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -30,17 +30,37 @@
 #include <linux/mmc/sdhci-pci-data.h>
 #include <linux/acpi.h>
 
+#include "cqhci.h"
+
 #include "sdhci.h"
 #include "sdhci-pci.h"
 
-static int sdhci_pci_enable_dma(struct sdhci_host *host);
 static void sdhci_pci_hw_reset(struct sdhci_host *host);
 
 #ifdef CONFIG_PM_SLEEP
-static int __sdhci_pci_suspend_host(struct sdhci_pci_chip *chip)
+static int sdhci_pci_init_wakeup(struct sdhci_pci_chip *chip)
+{
+	mmc_pm_flag_t pm_flags = 0;
+	int i;
+
+	for (i = 0; i < chip->num_slots; i++) {
+		struct sdhci_pci_slot *slot = chip->slots[i];
+
+		if (slot)
+			pm_flags |= slot->host->mmc->pm_flags;
+	}
+
+	return device_set_wakeup_enable(&chip->pdev->dev,
+					(pm_flags & MMC_PM_KEEP_POWER) &&
+					(pm_flags & MMC_PM_WAKE_SDIO_IRQ));
+}
+
+static int sdhci_pci_suspend_host(struct sdhci_pci_chip *chip)
 {
 	int i, ret;
 
+	sdhci_pci_init_wakeup(chip);
+
 	for (i = 0; i < chip->num_slots; i++) {
 		struct sdhci_pci_slot *slot = chip->slots[i];
 		struct sdhci_host *host;
@@ -56,9 +76,6 @@ static int __sdhci_pci_suspend_host(struct sdhci_pci_chip *chip)
 		ret = sdhci_suspend_host(host);
 		if (ret)
 			goto err_pci_suspend;
-
-		if (host->mmc->pm_flags & MMC_PM_WAKE_SDIO_IRQ)
-			sdhci_enable_irq_wakeups(host);
 	}
 
 	return 0;
@@ -69,52 +86,44 @@ static int __sdhci_pci_suspend_host(struct sdhci_pci_chip *chip)
 	return ret;
 }
 
-static int sdhci_pci_init_wakeup(struct sdhci_pci_chip *chip)
+int sdhci_pci_resume_host(struct sdhci_pci_chip *chip)
 {
-	mmc_pm_flag_t pm_flags = 0;
-	int i;
+	struct sdhci_pci_slot *slot;
+	int i, ret;
 
 	for (i = 0; i < chip->num_slots; i++) {
-		struct sdhci_pci_slot *slot = chip->slots[i];
+		slot = chip->slots[i];
+		if (!slot)
+			continue;
 
-		if (slot)
-			pm_flags |= slot->host->mmc->pm_flags;
+		ret = sdhci_resume_host(slot->host);
+		if (ret)
+			return ret;
 	}
 
-	return device_init_wakeup(&chip->pdev->dev,
-				  (pm_flags & MMC_PM_KEEP_POWER) &&
-				  (pm_flags & MMC_PM_WAKE_SDIO_IRQ));
+	return 0;
 }
 
-static int sdhci_pci_suspend_host(struct sdhci_pci_chip *chip)
+static int sdhci_cqhci_suspend(struct sdhci_pci_chip *chip)
 {
 	int ret;
 
-	ret = __sdhci_pci_suspend_host(chip);
+	ret = cqhci_suspend(chip->slots[0]->host->mmc);
 	if (ret)
 		return ret;
 
-	sdhci_pci_init_wakeup(chip);
-
-	return 0;
+	return sdhci_pci_suspend_host(chip);
 }
 
-int sdhci_pci_resume_host(struct sdhci_pci_chip *chip)
+static int sdhci_cqhci_resume(struct sdhci_pci_chip *chip)
 {
-	struct sdhci_pci_slot *slot;
-	int i, ret;
-
-	for (i = 0; i < chip->num_slots; i++) {
-		slot = chip->slots[i];
-		if (!slot)
-			continue;
+	int ret;
 
-		ret = sdhci_resume_host(slot->host);
-		if (ret)
-			return ret;
-	}
+	ret = sdhci_pci_resume_host(chip);
+	if (ret)
+		return ret;
 
-	return 0;
+	return cqhci_resume(chip->slots[0]->host->mmc);
 }
 #endif
 
@@ -166,8 +175,48 @@ static int sdhci_pci_runtime_resume_host(struct sdhci_pci_chip *chip)
 
 	return 0;
 }
+
+static int sdhci_cqhci_runtime_suspend(struct sdhci_pci_chip *chip)
+{
+	int ret;
+
+	ret = cqhci_suspend(chip->slots[0]->host->mmc);
+	if (ret)
+		return ret;
+
+	return sdhci_pci_runtime_suspend_host(chip);
+}
+
+static int sdhci_cqhci_runtime_resume(struct sdhci_pci_chip *chip)
+{
+	int ret;
+
+	ret = sdhci_pci_runtime_resume_host(chip);
+	if (ret)
+		return ret;
+
+	return cqhci_resume(chip->slots[0]->host->mmc);
+}
 #endif
 
+static u32 sdhci_cqhci_irq(struct sdhci_host *host, u32 intmask)
+{
+	int cmd_error = 0;
+	int data_error = 0;
+
+	if (!sdhci_cqe_irq(host, intmask, &cmd_error, &data_error))
+		return intmask;
+
+	cqhci_irq(host->mmc, intmask, cmd_error, data_error);
+
+	return 0;
+}
+
+static void sdhci_pci_dumpregs(struct mmc_host *mmc)
+{
+	sdhci_dumpregs(mmc_priv(mmc));
+}
+
 /*****************************************************************************\
  *                                                                           *
  * Hardware specific quirk handling                                          *
@@ -583,6 +632,18 @@ static const struct sdhci_ops sdhci_intel_byt_ops = {
 	.voltage_switch		= sdhci_intel_voltage_switch,
 };
 
+static const struct sdhci_ops sdhci_intel_glk_ops = {
+	.set_clock		= sdhci_set_clock,
+	.set_power		= sdhci_intel_set_power,
+	.enable_dma		= sdhci_pci_enable_dma,
+	.set_bus_width		= sdhci_set_bus_width,
+	.reset			= sdhci_reset,
+	.set_uhs_signaling	= sdhci_set_uhs_signaling,
+	.hw_reset		= sdhci_pci_hw_reset,
+	.voltage_switch		= sdhci_intel_voltage_switch,
+	.irq			= sdhci_cqhci_irq,
+};
+
 static void byt_read_dsm(struct sdhci_pci_slot *slot)
 {
 	struct intel_host *intel_host = sdhci_pci_priv(slot);
@@ -593,9 +654,36 @@ static void byt_read_dsm(struct sdhci_pci_slot *slot)
 	slot->chip->rpm_retune = intel_host->d3_retune;
 }
 
-static int byt_emmc_probe_slot(struct sdhci_pci_slot *slot)
+static int intel_execute_tuning(struct mmc_host *mmc, u32 opcode)
+{
+	int err = sdhci_execute_tuning(mmc, opcode);
+	struct sdhci_host *host = mmc_priv(mmc);
+
+	if (err)
+		return err;
+
+	/*
+	 * Tuning can leave the IP in an active state (Buffer Read Enable bit
+	 * set) which prevents the entry to low power states (i.e. S0i3). Data
+	 * reset will clear it.
+	 */
+	sdhci_reset(host, SDHCI_RESET_DATA);
+
+	return 0;
+}
+
+static void byt_probe_slot(struct sdhci_pci_slot *slot)
 {
+	struct mmc_host_ops *ops = &slot->host->mmc_host_ops;
+
 	byt_read_dsm(slot);
+
+	ops->execute_tuning = intel_execute_tuning;
+}
+
+static int byt_emmc_probe_slot(struct sdhci_pci_slot *slot)
+{
+	byt_probe_slot(slot);
 	slot->host->mmc->caps |= MMC_CAP_8_BIT_DATA | MMC_CAP_NONREMOVABLE |
 				 MMC_CAP_HW_RESET | MMC_CAP_1_8V_DDR |
 				 MMC_CAP_CMD_DURING_TFR |
@@ -612,15 +700,83 @@ static int glk_emmc_probe_slot(struct sdhci_pci_slot *slot)
 {
 	int ret = byt_emmc_probe_slot(slot);
 
+	slot->host->mmc->caps2 |= MMC_CAP2_CQE;
+
 	if (slot->chip->pdev->device != PCI_DEVICE_ID_INTEL_GLK_EMMC) {
 		slot->host->mmc->caps2 |= MMC_CAP2_HS400_ES,
 		slot->host->mmc_host_ops.hs400_enhanced_strobe =
 						intel_hs400_enhanced_strobe;
+		slot->host->mmc->caps2 |= MMC_CAP2_CQE_DCMD;
 	}
 
 	return ret;
 }
 
+static void glk_cqe_enable(struct mmc_host *mmc)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	u32 reg;
+
+	/*
+	 * CQE gets stuck if it sees Buffer Read Enable bit set, which can be
+	 * the case after tuning, so ensure the buffer is drained.
+	 */
+	reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	while (reg & SDHCI_DATA_AVAILABLE) {
+		sdhci_readl(host, SDHCI_BUFFER);
+		reg = sdhci_readl(host, SDHCI_PRESENT_STATE);
+	}
+
+	sdhci_cqe_enable(mmc);
+}
+
+static const struct cqhci_host_ops glk_cqhci_ops = {
+	.enable		= glk_cqe_enable,
+	.disable	= sdhci_cqe_disable,
+	.dumpregs	= sdhci_pci_dumpregs,
+};
+
+static int glk_emmc_add_host(struct sdhci_pci_slot *slot)
+{
+	struct device *dev = &slot->chip->pdev->dev;
+	struct sdhci_host *host = slot->host;
+	struct cqhci_host *cq_host;
+	bool dma64;
+	int ret;
+
+	ret = sdhci_setup_host(host);
+	if (ret)
+		return ret;
+
+	cq_host = devm_kzalloc(dev, sizeof(*cq_host), GFP_KERNEL);
+	if (!cq_host) {
+		ret = -ENOMEM;
+		goto cleanup;
+	}
+
+	cq_host->mmio = host->ioaddr + 0x200;
+	cq_host->quirks |= CQHCI_QUIRK_SHORT_TXFR_DESC_SZ;
+	cq_host->ops = &glk_cqhci_ops;
+
+	dma64 = host->flags & SDHCI_USE_64_BIT_DMA;
+	if (dma64)
+		cq_host->caps |= CQHCI_TASK_DESC_SZ_128;
+
+	ret = cqhci_init(cq_host, host->mmc, dma64);
+	if (ret)
+		goto cleanup;
+
+	ret = __sdhci_add_host(host);
+	if (ret)
+		goto cleanup;
+
+	return 0;
+
+cleanup:
+	sdhci_cleanup_host(host);
+	return ret;
+}
+
 #ifdef CONFIG_ACPI
 static int ni_set_max_freq(struct sdhci_pci_slot *slot)
 {
@@ -650,7 +806,7 @@ static int ni_byt_sdio_probe_slot(struct sdhci_pci_slot *slot)
 {
 	int err;
 
-	byt_read_dsm(slot);
+	byt_probe_slot(slot);
 
 	err = ni_set_max_freq(slot);
 	if (err)
@@ -663,7 +819,7 @@ static int ni_byt_sdio_probe_slot(struct sdhci_pci_slot *slot)
 
 static int byt_sdio_probe_slot(struct sdhci_pci_slot *slot)
 {
-	byt_read_dsm(slot);
+	byt_probe_slot(slot);
 	slot->host->mmc->caps |= MMC_CAP_POWER_OFF_CARD | MMC_CAP_NONREMOVABLE |
 				 MMC_CAP_WAIT_WHILE_BUSY;
 	return 0;
@@ -671,7 +827,7 @@ static int byt_sdio_probe_slot(struct sdhci_pci_slot *slot)
 
 static int byt_sd_probe_slot(struct sdhci_pci_slot *slot)
 {
-	byt_read_dsm(slot);
+	byt_probe_slot(slot);
 	slot->host->mmc->caps |= MMC_CAP_WAIT_WHILE_BUSY |
 				 MMC_CAP_AGGRESSIVE_PM | MMC_CAP_CD_WAKE;
 	slot->cd_idx = 0;
@@ -699,11 +855,20 @@ static const struct sdhci_pci_fixes sdhci_intel_byt_emmc = {
 static const struct sdhci_pci_fixes sdhci_intel_glk_emmc = {
 	.allow_runtime_pm	= true,
 	.probe_slot		= glk_emmc_probe_slot,
+	.add_host		= glk_emmc_add_host,
+#ifdef CONFIG_PM_SLEEP
+	.suspend		= sdhci_cqhci_suspend,
+	.resume			= sdhci_cqhci_resume,
+#endif
+#ifdef CONFIG_PM
+	.runtime_suspend	= sdhci_cqhci_runtime_suspend,
+	.runtime_resume		= sdhci_cqhci_runtime_resume,
+#endif
 	.quirks			= SDHCI_QUIRK_NO_ENDATTR_IN_NOPDESC,
 	.quirks2		= SDHCI_QUIRK2_PRESET_VALUE_BROKEN |
 				  SDHCI_QUIRK2_CAPS_BIT63_FOR_HS400 |
 				  SDHCI_QUIRK2_STOP_WITH_TC,
-	.ops			= &sdhci_intel_byt_ops,
+	.ops			= &sdhci_intel_glk_ops,
 	.priv_size		= sizeof(struct intel_host),
 };
 
@@ -778,6 +943,8 @@ static int intel_mrfld_mmc_probe_slot(struct sdhci_pci_slot *slot)
 		slot->host->quirks2 |= SDHCI_QUIRK2_NO_1_8_V;
 		break;
 	case INTEL_MRFLD_SDIO:
+		/* Advertise 2.0v for compatibility with the SDIO card's OCR */
+		slot->host->ocr_mask = MMC_VDD_20_21 | MMC_VDD_165_195;
 		slot->host->mmc->caps |= MMC_CAP_NONREMOVABLE |
 					 MMC_CAP_POWER_OFF_CARD;
 		break;
@@ -955,7 +1122,7 @@ static int jmicron_suspend(struct sdhci_pci_chip *chip)
 {
 	int i, ret;
 
-	ret = __sdhci_pci_suspend_host(chip);
+	ret = sdhci_pci_suspend_host(chip);
 	if (ret)
 		return ret;
 
@@ -965,8 +1132,6 @@ static int jmicron_suspend(struct sdhci_pci_chip *chip)
 			jmicron_enable_mmc(chip->slots[i]->host, 0);
 	}
 
-	sdhci_pci_init_wakeup(chip);
-
 	return 0;
 }
 
@@ -1306,6 +1471,7 @@ static const struct pci_device_id pci_ids[] = {
 	SDHCI_PCI_DEVICE(O2, SDS1,     o2),
 	SDHCI_PCI_DEVICE(O2, SEABIRD0, o2),
 	SDHCI_PCI_DEVICE(O2, SEABIRD1, o2),
+	SDHCI_PCI_DEVICE(ARASAN, PHY_EMMC, arasan),
 	SDHCI_PCI_DEVICE_CLASS(AMD, SYSTEM_SDHCI, PCI_CLASS_MASK, amd),
 	/* Generic SD host controller */
 	{PCI_DEVICE_CLASS(SYSTEM_SDHCI, PCI_CLASS_MASK)},
@@ -1320,7 +1486,7 @@ MODULE_DEVICE_TABLE(pci, pci_ids);
  *                                                                           *
 \*****************************************************************************/
 
-static int sdhci_pci_enable_dma(struct sdhci_host *host)
+int sdhci_pci_enable_dma(struct sdhci_host *host)
 {
 	struct sdhci_pci_slot *slot;
 	struct pci_dev *pdev;
@@ -1543,10 +1709,13 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot(
 		}
 	}
 
-	host->mmc->pm_caps = MMC_PM_KEEP_POWER | MMC_PM_WAKE_SDIO_IRQ;
+	host->mmc->pm_caps = MMC_PM_KEEP_POWER;
 	host->mmc->slotno = slotno;
 	host->mmc->caps2 |= MMC_CAP2_NO_PRESCAN_POWERUP;
 
+	if (device_can_wakeup(&pdev->dev))
+		host->mmc->pm_caps |= MMC_PM_WAKE_SDIO_IRQ;
+
 	if (slot->cd_idx >= 0) {
 		ret = mmc_gpiod_request_cd(host->mmc, NULL, slot->cd_idx,
 					   slot->cd_override_level, 0, NULL);
diff --git a/drivers/mmc/host/sdhci-pci.h b/drivers/mmc/host/sdhci-pci.h
index 0056f08a29cc..5cbcdc448f98 100644
--- a/drivers/mmc/host/sdhci-pci.h
+++ b/drivers/mmc/host/sdhci-pci.h
@@ -55,6 +55,9 @@
 
 #define PCI_SUBDEVICE_ID_NI_7884	0x7884
 
+#define PCI_VENDOR_ID_ARASAN		0x16e6
+#define PCI_DEVICE_ID_ARASAN_PHY_EMMC	0x0670
+
 /*
  * PCI device class and mask
  */
@@ -170,11 +173,13 @@ static inline void *sdhci_pci_priv(struct sdhci_pci_slot *slot)
 #ifdef CONFIG_PM_SLEEP
 int sdhci_pci_resume_host(struct sdhci_pci_chip *chip);
 #endif
-
+int sdhci_pci_enable_dma(struct sdhci_host *host);
 int sdhci_pci_o2_probe_slot(struct sdhci_pci_slot *slot);
 int sdhci_pci_o2_probe(struct sdhci_pci_chip *chip);
 #ifdef CONFIG_PM_SLEEP
 int sdhci_pci_o2_resume(struct sdhci_pci_chip *chip);
 #endif
 
+extern const struct sdhci_pci_fixes sdhci_arasan;
+
 #endif /* __SDHCI_PCI_H */
diff --git a/drivers/mmc/host/sdhci-spear.c b/drivers/mmc/host/sdhci-spear.c
index 8c0f88428556..14511526a3a8 100644
--- a/drivers/mmc/host/sdhci-spear.c
+++ b/drivers/mmc/host/sdhci-spear.c
@@ -82,6 +82,10 @@ static int sdhci_probe(struct platform_device *pdev)
 	host->hw_name = "sdhci";
 	host->ops = &sdhci_pltfm_ops;
 	host->irq = platform_get_irq(pdev, 0);
+	if (host->irq <= 0) {
+		ret = -EINVAL;
+		goto err_host;
+	}
 	host->quirks = SDHCI_QUIRK_BROKEN_ADMA;
 
 	sdhci = sdhci_priv(host);
diff --git a/drivers/mmc/host/sdhci-xenon.c b/drivers/mmc/host/sdhci-xenon.c
index 0842bbc2d7ad..4d0791f6ec23 100644
--- a/drivers/mmc/host/sdhci-xenon.c
+++ b/drivers/mmc/host/sdhci-xenon.c
@@ -230,7 +230,14 @@ static void xenon_set_power(struct sdhci_host *host, unsigned char mode,
 		mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd);
 }
 
+static void xenon_voltage_switch(struct sdhci_host *host)
+{
+	/* Wait for 5ms after set 1.8V signal enable bit */
+	usleep_range(5000, 5500);
+}
+
 static const struct sdhci_ops sdhci_xenon_ops = {
+	.voltage_switch		= xenon_voltage_switch,
 	.set_clock		= sdhci_set_clock,
 	.set_power		= xenon_set_power,
 	.set_bus_width		= sdhci_set_bus_width,
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index e9290a3439d5..2020e57ffa7e 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -21,6 +21,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/slab.h>
 #include <linux/scatterlist.h>
+#include <linux/sizes.h>
 #include <linux/swiotlb.h>
 #include <linux/regulator/consumer.h>
 #include <linux/pm_runtime.h>
@@ -502,8 +503,35 @@ static int sdhci_pre_dma_transfer(struct sdhci_host *host,
 	if (data->host_cookie == COOKIE_PRE_MAPPED)
 		return data->sg_count;
 
-	sg_count = dma_map_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
-			      mmc_get_dma_dir(data));
+	/* Bounce write requests to the bounce buffer */
+	if (host->bounce_buffer) {
+		unsigned int length = data->blksz * data->blocks;
+
+		if (length > host->bounce_buffer_size) {
+			pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n",
+			       mmc_hostname(host->mmc), length,
+			       host->bounce_buffer_size);
+			return -EIO;
+		}
+		if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) {
+			/* Copy the data to the bounce buffer */
+			sg_copy_to_buffer(data->sg, data->sg_len,
+					  host->bounce_buffer,
+					  length);
+		}
+		/* Switch ownership to the DMA */
+		dma_sync_single_for_device(host->mmc->parent,
+					   host->bounce_addr,
+					   host->bounce_buffer_size,
+					   mmc_get_dma_dir(data));
+		/* Just a dummy value */
+		sg_count = 1;
+	} else {
+		/* Just access the data directly from memory */
+		sg_count = dma_map_sg(mmc_dev(host->mmc),
+				      data->sg, data->sg_len,
+				      mmc_get_dma_dir(data));
+	}
 
 	if (sg_count == 0)
 		return -ENOSPC;
@@ -673,6 +701,14 @@ static void sdhci_adma_table_post(struct sdhci_host *host,
 	}
 }
 
+static u32 sdhci_sdma_address(struct sdhci_host *host)
+{
+	if (host->bounce_buffer)
+		return host->bounce_addr;
+	else
+		return sg_dma_address(host->data->sg);
+}
+
 static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd)
 {
 	u8 count;
@@ -858,8 +894,8 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd)
 					     SDHCI_ADMA_ADDRESS_HI);
 		} else {
 			WARN_ON(sg_cnt != 1);
-			sdhci_writel(host, sg_dma_address(data->sg),
-				SDHCI_DMA_ADDRESS);
+			sdhci_writel(host, sdhci_sdma_address(host),
+				     SDHCI_DMA_ADDRESS);
 		}
 	}
 
@@ -1434,6 +1470,13 @@ void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode,
 	if (mode != MMC_POWER_OFF) {
 		switch (1 << vdd) {
 		case MMC_VDD_165_195:
+		/*
+		 * Without a regulator, SDHCI does not support 2.0v
+		 * so we only get here if the driver deliberately
+		 * added the 2.0v range to ocr_avail. Map it to 1.8v
+		 * for the purpose of turning on the power.
+		 */
+		case MMC_VDD_20_21:
 			pwr = SDHCI_POWER_180;
 			break;
 		case MMC_VDD_29_30:
@@ -2248,7 +2291,12 @@ static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq)
 
 	mrq->data->host_cookie = COOKIE_UNMAPPED;
 
-	if (host->flags & SDHCI_REQ_USE_DMA)
+	/*
+	 * No pre-mapping in the pre hook if we're using the bounce buffer,
+	 * for that we would need two bounce buffers since one buffer is
+	 * in flight when this is getting called.
+	 */
+	if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer)
 		sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED);
 }
 
@@ -2352,8 +2400,45 @@ static bool sdhci_request_done(struct sdhci_host *host)
 		struct mmc_data *data = mrq->data;
 
 		if (data && data->host_cookie == COOKIE_MAPPED) {
-			dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len,
-				     mmc_get_dma_dir(data));
+			if (host->bounce_buffer) {
+				/*
+				 * On reads, copy the bounced data into the
+				 * sglist
+				 */
+				if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) {
+					unsigned int length = data->bytes_xfered;
+
+					if (length > host->bounce_buffer_size) {
+						pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n",
+						       mmc_hostname(host->mmc),
+						       host->bounce_buffer_size,
+						       data->bytes_xfered);
+						/* Cap it down and continue */
+						length = host->bounce_buffer_size;
+					}
+					dma_sync_single_for_cpu(
+						host->mmc->parent,
+						host->bounce_addr,
+						host->bounce_buffer_size,
+						DMA_FROM_DEVICE);
+					sg_copy_from_buffer(data->sg,
+						data->sg_len,
+						host->bounce_buffer,
+						length);
+				} else {
+					/* No copying, just switch ownership */
+					dma_sync_single_for_cpu(
+						host->mmc->parent,
+						host->bounce_addr,
+						host->bounce_buffer_size,
+						mmc_get_dma_dir(data));
+				}
+			} else {
+				/* Unmap the raw data */
+				dma_unmap_sg(mmc_dev(host->mmc), data->sg,
+					     data->sg_len,
+					     mmc_get_dma_dir(data));
+			}
 			data->host_cookie = COOKIE_UNMAPPED;
 		}
 	}
@@ -2636,7 +2721,8 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask)
 		 */
 		if (intmask & SDHCI_INT_DMA_END) {
 			u32 dmastart, dmanow;
-			dmastart = sg_dma_address(host->data->sg);
+
+			dmastart = sdhci_sdma_address(host);
 			dmanow = dmastart + host->data->bytes_xfered;
 			/*
 			 * Force update to the next DMA block boundary.
@@ -2821,25 +2907,33 @@ static irqreturn_t sdhci_thread_irq(int irq, void *dev_id)
  * sdhci_disable_irq_wakeups() since it will be set by
  * sdhci_enable_card_detection() or sdhci_init().
  */
-void sdhci_enable_irq_wakeups(struct sdhci_host *host)
+static bool sdhci_enable_irq_wakeups(struct sdhci_host *host)
 {
+	u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE |
+		  SDHCI_WAKE_ON_INT;
+	u32 irq_val = 0;
+	u8 wake_val = 0;
 	u8 val;
-	u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE
-			| SDHCI_WAKE_ON_INT;
-	u32 irq_val = SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE |
-		      SDHCI_INT_CARD_INT;
 
-	val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
-	val |= mask ;
-	/* Avoid fake wake up */
-	if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) {
-		val &= ~(SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE);
-		irq_val &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE);
+	if (!(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION)) {
+		wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE;
+		irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE;
 	}
+
+	wake_val |= SDHCI_WAKE_ON_INT;
+	irq_val |= SDHCI_INT_CARD_INT;
+
+	val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
+	val &= ~mask;
+	val |= wake_val;
 	sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
+
 	sdhci_writel(host, irq_val, SDHCI_INT_ENABLE);
+
+	host->irq_wake_enabled = !enable_irq_wake(host->irq);
+
+	return host->irq_wake_enabled;
 }
-EXPORT_SYMBOL_GPL(sdhci_enable_irq_wakeups);
 
 static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
 {
@@ -2850,6 +2944,10 @@ static void sdhci_disable_irq_wakeups(struct sdhci_host *host)
 	val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL);
 	val &= ~mask;
 	sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL);
+
+	disable_irq_wake(host->irq);
+
+	host->irq_wake_enabled = false;
 }
 
 int sdhci_suspend_host(struct sdhci_host *host)
@@ -2858,15 +2956,14 @@ int sdhci_suspend_host(struct sdhci_host *host)
 
 	mmc_retune_timer_stop(host->mmc);
 
-	if (!device_may_wakeup(mmc_dev(host->mmc))) {
+	if (!device_may_wakeup(mmc_dev(host->mmc)) ||
+	    !sdhci_enable_irq_wakeups(host)) {
 		host->ier = 0;
 		sdhci_writel(host, 0, SDHCI_INT_ENABLE);
 		sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
 		free_irq(host->irq, host);
-	} else {
-		sdhci_enable_irq_wakeups(host);
-		enable_irq_wake(host->irq);
 	}
+
 	return 0;
 }
 
@@ -2894,15 +2991,14 @@ int sdhci_resume_host(struct sdhci_host *host)
 		mmiowb();
 	}
 
-	if (!device_may_wakeup(mmc_dev(host->mmc))) {
+	if (host->irq_wake_enabled) {
+		sdhci_disable_irq_wakeups(host);
+	} else {
 		ret = request_threaded_irq(host->irq, sdhci_irq,
 					   sdhci_thread_irq, IRQF_SHARED,
 					   mmc_hostname(host->mmc), host);
 		if (ret)
 			return ret;
-	} else {
-		sdhci_disable_irq_wakeups(host);
-		disable_irq_wake(host->irq);
 	}
 
 	sdhci_enable_card_detection(host);
@@ -3217,6 +3313,68 @@ void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1)
 }
 EXPORT_SYMBOL_GPL(__sdhci_read_caps);
 
+static int sdhci_allocate_bounce_buffer(struct sdhci_host *host)
+{
+	struct mmc_host *mmc = host->mmc;
+	unsigned int max_blocks;
+	unsigned int bounce_size;
+	int ret;
+
+	/*
+	 * Cap the bounce buffer at 64KB. Using a bigger bounce buffer
+	 * has diminishing returns, this is probably because SD/MMC
+	 * cards are usually optimized to handle this size of requests.
+	 */
+	bounce_size = SZ_64K;
+	/*
+	 * Adjust downwards to maximum request size if this is less
+	 * than our segment size, else hammer down the maximum
+	 * request size to the maximum buffer size.
+	 */
+	if (mmc->max_req_size < bounce_size)
+		bounce_size = mmc->max_req_size;
+	max_blocks = bounce_size / 512;
+
+	/*
+	 * When we just support one segment, we can get significant
+	 * speedups by the help of a bounce buffer to group scattered
+	 * reads/writes together.
+	 */
+	host->bounce_buffer = devm_kmalloc(mmc->parent,
+					   bounce_size,
+					   GFP_KERNEL);
+	if (!host->bounce_buffer) {
+		pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n",
+		       mmc_hostname(mmc),
+		       bounce_size);
+		/*
+		 * Exiting with zero here makes sure we proceed with
+		 * mmc->max_segs == 1.
+		 */
+		return 0;
+	}
+
+	host->bounce_addr = dma_map_single(mmc->parent,
+					   host->bounce_buffer,
+					   bounce_size,
+					   DMA_BIDIRECTIONAL);
+	ret = dma_mapping_error(mmc->parent, host->bounce_addr);
+	if (ret)
+		/* Again fall back to max_segs == 1 */
+		return 0;
+	host->bounce_buffer_size = bounce_size;
+
+	/* Lie about this since we're bouncing */
+	mmc->max_segs = max_blocks;
+	mmc->max_seg_size = bounce_size;
+	mmc->max_req_size = bounce_size;
+
+	pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n",
+		mmc_hostname(mmc), max_blocks, bounce_size);
+
+	return 0;
+}
+
 int sdhci_setup_host(struct sdhci_host *host)
 {
 	struct mmc_host *mmc;
@@ -3713,6 +3871,13 @@ int sdhci_setup_host(struct sdhci_host *host)
 	 */
 	mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535;
 
+	if (mmc->max_segs == 1) {
+		/* This may alter mmc->*_blk_* parameters */
+		ret = sdhci_allocate_bounce_buffer(host);
+		if (ret)
+			return ret;
+	}
+
 	return 0;
 
 unreg:
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index 54bc444c317f..c95b0a4a7594 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -440,6 +440,9 @@ struct sdhci_host {
 
 	int irq;		/* Device IRQ */
 	void __iomem *ioaddr;	/* Mapped address */
+	char *bounce_buffer;	/* For packing SDMA reads/writes */
+	dma_addr_t bounce_addr;
+	unsigned int bounce_buffer_size;
 
 	const struct sdhci_ops *ops;	/* Low level hw interface */
 
@@ -484,6 +487,7 @@ struct sdhci_host {
 	bool bus_on;		/* Bus power prevents runtime suspend */
 	bool preset_enabled;	/* Preset is enabled */
 	bool pending_reset;	/* Cmd/data reset is pending */
+	bool irq_wake_enabled;	/* IRQ wakeup is enabled */
 
 	struct mmc_request *mrqs_done[SDHCI_MAX_MRQS];	/* Requests done */
 	struct mmc_command *cmd;	/* Current command */
@@ -718,7 +722,6 @@ void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable);
 #ifdef CONFIG_PM
 int sdhci_suspend_host(struct sdhci_host *host);
 int sdhci_resume_host(struct sdhci_host *host);
-void sdhci_enable_irq_wakeups(struct sdhci_host *host);
 int sdhci_runtime_suspend_host(struct sdhci_host *host);
 int sdhci_runtime_resume_host(struct sdhci_host *host);
 #endif
diff --git a/drivers/mmc/host/sdhci_f_sdh30.c b/drivers/mmc/host/sdhci_f_sdh30.c
index 04ca0d33a521..485f7591fae4 100644
--- a/drivers/mmc/host/sdhci_f_sdh30.c
+++ b/drivers/mmc/host/sdhci_f_sdh30.c
@@ -10,9 +10,11 @@
  * the Free Software Foundation, version 2 of the License.
  */
 
+#include <linux/acpi.h>
 #include <linux/err.h>
 #include <linux/delay.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/property.h>
 #include <linux/clk.h>
 
@@ -146,7 +148,6 @@ static int sdhci_f_sdh30_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, host);
 
-	sdhci_get_of_property(pdev);
 	host->hw_name = "f_sdh30";
 	host->ops = &sdhci_f_sdh30_ops;
 	host->irq = irq;
@@ -158,25 +159,29 @@ static int sdhci_f_sdh30_probe(struct platform_device *pdev)
 		goto err;
 	}
 
-	priv->clk_iface = devm_clk_get(&pdev->dev, "iface");
-	if (IS_ERR(priv->clk_iface)) {
-		ret = PTR_ERR(priv->clk_iface);
-		goto err;
-	}
+	if (dev_of_node(dev)) {
+		sdhci_get_of_property(pdev);
 
-	ret = clk_prepare_enable(priv->clk_iface);
-	if (ret)
-		goto err;
+		priv->clk_iface = devm_clk_get(&pdev->dev, "iface");
+		if (IS_ERR(priv->clk_iface)) {
+			ret = PTR_ERR(priv->clk_iface);
+			goto err;
+		}
 
-	priv->clk = devm_clk_get(&pdev->dev, "core");
-	if (IS_ERR(priv->clk)) {
-		ret = PTR_ERR(priv->clk);
-		goto err_clk;
-	}
+		ret = clk_prepare_enable(priv->clk_iface);
+		if (ret)
+			goto err;
 
-	ret = clk_prepare_enable(priv->clk);
-	if (ret)
-		goto err_clk;
+		priv->clk = devm_clk_get(&pdev->dev, "core");
+		if (IS_ERR(priv->clk)) {
+			ret = PTR_ERR(priv->clk);
+			goto err_clk;
+		}
+
+		ret = clk_prepare_enable(priv->clk);
+		if (ret)
+			goto err_clk;
+	}
 
 	/* init vendor specific regs */
 	ctrl = sdhci_readw(host, F_SDH30_AHB_CONFIG);
@@ -226,16 +231,27 @@ static int sdhci_f_sdh30_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_OF
 static const struct of_device_id f_sdh30_dt_ids[] = {
 	{ .compatible = "fujitsu,mb86s70-sdhci-3.0" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, f_sdh30_dt_ids);
+#endif
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id f_sdh30_acpi_ids[] = {
+	{ "SCX0002" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(acpi, f_sdh30_acpi_ids);
+#endif
 
 static struct platform_driver sdhci_f_sdh30_driver = {
 	.driver = {
 		.name = "f_sdh30",
-		.of_match_table = f_sdh30_dt_ids,
+		.of_match_table = of_match_ptr(f_sdh30_dt_ids),
+		.acpi_match_table = ACPI_PTR(f_sdh30_acpi_ids),
 		.pm	= &sdhci_pltfm_pmops,
 	},
 	.probe	= sdhci_f_sdh30_probe,
diff --git a/drivers/mmc/host/sh_mmcif.c b/drivers/mmc/host/sh_mmcif.c
index 53fb18bb7bee..7bb00c68a756 100644
--- a/drivers/mmc/host/sh_mmcif.c
+++ b/drivers/mmc/host/sh_mmcif.c
@@ -916,7 +916,7 @@ static void sh_mmcif_start_cmd(struct sh_mmcif_host *host,
 			       struct mmc_request *mrq)
 {
 	struct mmc_command *cmd = mrq->cmd;
-	u32 opc = cmd->opcode;
+	u32 opc;
 	u32 mask = 0;
 	unsigned long flags;
 
diff --git a/drivers/mmc/host/sunxi-mmc.c b/drivers/mmc/host/sunxi-mmc.c
index cc98355dbdb9..bad612d6f879 100644
--- a/drivers/mmc/host/sunxi-mmc.c
+++ b/drivers/mmc/host/sunxi-mmc.c
@@ -3,7 +3,7 @@
  * (C) Copyright 2007-2011 Reuuimlla Technology Co., Ltd.
  * (C) Copyright 2007-2011 Aaron Maoye <leafy.myeh@reuuimllatech.com>
  * (C) Copyright 2013-2014 O2S GmbH <www.o2s.ch>
- * (C) Copyright 2013-2014 David Lanzend�rfer <david.lanzendoerfer@o2s.ch>
+ * (C) Copyright 2013-2014 David Lanzendörfer <david.lanzendoerfer@o2s.ch>
  * (C) Copyright 2013-2014 Hans de Goede <hdegoede@redhat.com>
  * (C) Copyright 2017 Sootech SA
  *
@@ -1255,6 +1255,11 @@ static int sunxi_mmc_resource_request(struct sunxi_mmc_host *host,
 		goto error_assert_reset;
 
 	host->irq = platform_get_irq(pdev, 0);
+	if (host->irq <= 0) {
+		ret = -EINVAL;
+		goto error_assert_reset;
+	}
+
 	return devm_request_threaded_irq(&pdev->dev, host->irq, sunxi_mmc_irq,
 			sunxi_mmc_handle_manual_stop, 0, "sunxi-mmc", host);
 
@@ -1393,5 +1398,5 @@ module_platform_driver(sunxi_mmc_driver);
 
 MODULE_DESCRIPTION("Allwinner's SD/MMC Card Controller Driver");
 MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("David Lanzend�rfer <david.lanzendoerfer@o2s.ch>");
+MODULE_AUTHOR("David Lanzendörfer <david.lanzendoerfer@o2s.ch>");
 MODULE_ALIAS("platform:sunxi-mmc");
diff --git a/drivers/mmc/host/tmio_mmc.c b/drivers/mmc/host/tmio_mmc.c
index 64b7e9f18361..43a2ea5cff24 100644
--- a/drivers/mmc/host/tmio_mmc.c
+++ b/drivers/mmc/host/tmio_mmc.c
@@ -92,14 +92,19 @@ static int tmio_mmc_probe(struct platform_device *pdev)
 
 	pdata->flags |= TMIO_MMC_HAVE_HIGH_REG;
 
-	host = tmio_mmc_host_alloc(pdev);
-	if (!host)
+	host = tmio_mmc_host_alloc(pdev, pdata);
+	if (IS_ERR(host)) {
+		ret = PTR_ERR(host);
 		goto cell_disable;
+	}
 
 	/* SD control register space size is 0x200, 0x400 for bus_shift=1 */
 	host->bus_shift = resource_size(res) >> 10;
 
-	ret = tmio_mmc_host_probe(host, pdata, NULL);
+	host->mmc->f_max = pdata->hclk;
+	host->mmc->f_min = pdata->hclk / 512;
+
+	ret = tmio_mmc_host_probe(host);
 	if (ret)
 		goto host_free;
 
@@ -128,15 +133,11 @@ static int tmio_mmc_probe(struct platform_device *pdev)
 static int tmio_mmc_remove(struct platform_device *pdev)
 {
 	const struct mfd_cell *cell = mfd_get_cell(pdev);
-	struct mmc_host *mmc = platform_get_drvdata(pdev);
+	struct tmio_mmc_host *host = platform_get_drvdata(pdev);
 
-	if (mmc) {
-		struct tmio_mmc_host *host = mmc_priv(mmc);
-
-		tmio_mmc_host_remove(host);
-		if (cell->disable)
-			cell->disable(pdev);
-	}
+	tmio_mmc_host_remove(host);
+	if (cell->disable)
+		cell->disable(pdev);
 
 	return 0;
 }
diff --git a/drivers/mmc/host/tmio_mmc.h b/drivers/mmc/host/tmio_mmc.h
index 3e6ff8921440..e7d651352dc9 100644
--- a/drivers/mmc/host/tmio_mmc.h
+++ b/drivers/mmc/host/tmio_mmc.h
@@ -112,12 +112,6 @@
 struct tmio_mmc_data;
 struct tmio_mmc_host;
 
-struct tmio_mmc_dma {
-	enum dma_slave_buswidth dma_buswidth;
-	bool (*filter)(struct dma_chan *chan, void *arg);
-	void (*enable)(struct tmio_mmc_host *host, bool enable);
-};
-
 struct tmio_mmc_dma_ops {
 	void (*start)(struct tmio_mmc_host *host, struct mmc_data *data);
 	void (*enable)(struct tmio_mmc_host *host, bool enable);
@@ -134,6 +128,7 @@ struct tmio_mmc_host {
 	struct mmc_request      *mrq;
 	struct mmc_data         *data;
 	struct mmc_host         *mmc;
+	struct mmc_host_ops     ops;
 
 	/* Callbacks for clock / power control */
 	void (*set_pwr)(struct platform_device *host, int state);
@@ -144,18 +139,15 @@ struct tmio_mmc_host {
 	struct scatterlist      *sg_orig;
 	unsigned int            sg_len;
 	unsigned int            sg_off;
-	unsigned long		bus_shift;
+	unsigned int		bus_shift;
 
 	struct platform_device *pdev;
 	struct tmio_mmc_data *pdata;
-	struct tmio_mmc_dma	*dma;
 
 	/* DMA support */
 	bool			force_pio;
 	struct dma_chan		*chan_rx;
 	struct dma_chan		*chan_tx;
-	struct completion	dma_dataend;
-	struct tasklet_struct	dma_complete;
 	struct tasklet_struct	dma_issue;
 	struct scatterlist	bounce_sg;
 	u8			*bounce_buf;
@@ -174,7 +166,6 @@ struct tmio_mmc_host {
 	struct mutex		ios_lock;	/* protect set_ios() context */
 	bool			native_hotplug;
 	bool			sdio_irq_enabled;
-	u32			scc_tappos;
 
 	/* Mandatory callback */
 	int (*clk_enable)(struct tmio_mmc_host *host);
@@ -185,9 +176,6 @@ struct tmio_mmc_host {
 	void (*clk_disable)(struct tmio_mmc_host *host);
 	int (*multi_io_quirk)(struct mmc_card *card,
 			      unsigned int direction, int blk_size);
-	int (*card_busy)(struct mmc_host *mmc);
-	int (*start_signal_voltage_switch)(struct mmc_host *mmc,
-					   struct mmc_ios *ios);
 	int (*write16_hook)(struct tmio_mmc_host *host, int addr);
 	void (*hw_reset)(struct tmio_mmc_host *host);
 	void (*prepare_tuning)(struct tmio_mmc_host *host, unsigned long tap);
@@ -207,11 +195,10 @@ struct tmio_mmc_host {
 	const struct tmio_mmc_dma_ops *dma_ops;
 };
 
-struct tmio_mmc_host *tmio_mmc_host_alloc(struct platform_device *pdev);
+struct tmio_mmc_host *tmio_mmc_host_alloc(struct platform_device *pdev,
+					  struct tmio_mmc_data *pdata);
 void tmio_mmc_host_free(struct tmio_mmc_host *host);
-int tmio_mmc_host_probe(struct tmio_mmc_host *host,
-			struct tmio_mmc_data *pdata,
-			const struct tmio_mmc_dma_ops *dma_ops);
+int tmio_mmc_host_probe(struct tmio_mmc_host *host);
 void tmio_mmc_host_remove(struct tmio_mmc_host *host);
 void tmio_mmc_do_data_irq(struct tmio_mmc_host *host);
 
@@ -240,26 +227,26 @@ int tmio_mmc_host_runtime_resume(struct device *dev);
 
 static inline u16 sd_ctrl_read16(struct tmio_mmc_host *host, int addr)
 {
-	return readw(host->ctl + (addr << host->bus_shift));
+	return ioread16(host->ctl + (addr << host->bus_shift));
 }
 
 static inline void sd_ctrl_read16_rep(struct tmio_mmc_host *host, int addr,
 				      u16 *buf, int count)
 {
-	readsw(host->ctl + (addr << host->bus_shift), buf, count);
+	ioread16_rep(host->ctl + (addr << host->bus_shift), buf, count);
 }
 
 static inline u32 sd_ctrl_read16_and_16_as_32(struct tmio_mmc_host *host,
 					      int addr)
 {
-	return readw(host->ctl + (addr << host->bus_shift)) |
-	       readw(host->ctl + ((addr + 2) << host->bus_shift)) << 16;
+	return ioread16(host->ctl + (addr << host->bus_shift)) |
+	       ioread16(host->ctl + ((addr + 2) << host->bus_shift)) << 16;
 }
 
 static inline void sd_ctrl_read32_rep(struct tmio_mmc_host *host, int addr,
 				      u32 *buf, int count)
 {
-	readsl(host->ctl + (addr << host->bus_shift), buf, count);
+	ioread32_rep(host->ctl + (addr << host->bus_shift), buf, count);
 }
 
 static inline void sd_ctrl_write16(struct tmio_mmc_host *host, int addr,
@@ -270,26 +257,26 @@ static inline void sd_ctrl_write16(struct tmio_mmc_host *host, int addr,
 	 */
 	if (host->write16_hook && host->write16_hook(host, addr))
 		return;
-	writew(val, host->ctl + (addr << host->bus_shift));
+	iowrite16(val, host->ctl + (addr << host->bus_shift));
 }
 
 static inline void sd_ctrl_write16_rep(struct tmio_mmc_host *host, int addr,
 				       u16 *buf, int count)
 {
-	writesw(host->ctl + (addr << host->bus_shift), buf, count);
+	iowrite16_rep(host->ctl + (addr << host->bus_shift), buf, count);
 }
 
 static inline void sd_ctrl_write32_as_16_and_16(struct tmio_mmc_host *host,
 						int addr, u32 val)
 {
-	writew(val & 0xffff, host->ctl + (addr << host->bus_shift));
-	writew(val >> 16, host->ctl + ((addr + 2) << host->bus_shift));
+	iowrite16(val & 0xffff, host->ctl + (addr << host->bus_shift));
+	iowrite16(val >> 16, host->ctl + ((addr + 2) << host->bus_shift));
 }
 
 static inline void sd_ctrl_write32_rep(struct tmio_mmc_host *host, int addr,
 				       const u32 *buf, int count)
 {
-	writesl(host->ctl + (addr << host->bus_shift), buf, count);
+	iowrite32_rep(host->ctl + (addr << host->bus_shift), buf, count);
 }
 
 #endif
diff --git a/drivers/mmc/host/tmio_mmc_core.c b/drivers/mmc/host/tmio_mmc_core.c
index 583bf3262df5..33494241245a 100644
--- a/drivers/mmc/host/tmio_mmc_core.c
+++ b/drivers/mmc/host/tmio_mmc_core.c
@@ -806,7 +806,7 @@ static int tmio_mmc_execute_tuning(struct mmc_host *mmc, u32 opcode)
 		if (ret == 0)
 			set_bit(i, host->taps);
 
-		mdelay(1);
+		usleep_range(1000, 1200);
 	}
 
 	ret = host->select_tuning(host);
@@ -926,20 +926,6 @@ static void tmio_mmc_done_work(struct work_struct *work)
 	tmio_mmc_finish_request(host);
 }
 
-static int tmio_mmc_clk_enable(struct tmio_mmc_host *host)
-{
-	if (!host->clk_enable)
-		return -ENOTSUPP;
-
-	return host->clk_enable(host);
-}
-
-static void tmio_mmc_clk_disable(struct tmio_mmc_host *host)
-{
-	if (host->clk_disable)
-		host->clk_disable(host);
-}
-
 static void tmio_mmc_power_on(struct tmio_mmc_host *host, unsigned short vdd)
 {
 	struct mmc_host *mmc = host->mmc;
@@ -958,7 +944,7 @@ static void tmio_mmc_power_on(struct tmio_mmc_host *host, unsigned short vdd)
 		 * 100us were not enough. Is this the same 140us delay, as in
 		 * tmio_mmc_set_ios()?
 		 */
-		udelay(200);
+		usleep_range(200, 300);
 	}
 	/*
 	 * It seems, VccQ should be switched on after Vcc, this is also what the
@@ -966,7 +952,7 @@ static void tmio_mmc_power_on(struct tmio_mmc_host *host, unsigned short vdd)
 	 */
 	if (!IS_ERR(mmc->supply.vqmmc) && !ret) {
 		ret = regulator_enable(mmc->supply.vqmmc);
-		udelay(200);
+		usleep_range(200, 300);
 	}
 
 	if (ret < 0)
@@ -1059,7 +1045,7 @@ static void tmio_mmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	}
 
 	/* Let things settle. delay taken from winCE driver */
-	udelay(140);
+	usleep_range(140, 200);
 	if (PTR_ERR(host->mrq) == -EINTR)
 		dev_dbg(&host->pdev->dev,
 			"%s.%d: IOS interrupted: clk %u, mode %u",
@@ -1076,15 +1062,9 @@ static int tmio_mmc_get_ro(struct mmc_host *mmc)
 {
 	struct tmio_mmc_host *host = mmc_priv(mmc);
 	struct tmio_mmc_data *pdata = host->pdata;
-	int ret = mmc_gpio_get_ro(mmc);
-
-	if (ret >= 0)
-		return ret;
-
-	ret = !((pdata->flags & TMIO_MMC_WRPROTECT_DISABLE) ||
-		(sd_ctrl_read16_and_16_as_32(host, CTL_STATUS) & TMIO_STAT_WRPROTECT));
 
-	return ret;
+	return !((pdata->flags & TMIO_MMC_WRPROTECT_DISABLE) ||
+		 (sd_ctrl_read16_and_16_as_32(host, CTL_STATUS) & TMIO_STAT_WRPROTECT));
 }
 
 static int tmio_multi_io_quirk(struct mmc_card *card,
@@ -1098,7 +1078,7 @@ static int tmio_multi_io_quirk(struct mmc_card *card,
 	return blk_size;
 }
 
-static struct mmc_host_ops tmio_mmc_ops = {
+static const struct mmc_host_ops tmio_mmc_ops = {
 	.request	= tmio_mmc_request,
 	.set_ios	= tmio_mmc_set_ios,
 	.get_ro         = tmio_mmc_get_ro,
@@ -1145,19 +1125,45 @@ static void tmio_mmc_of_parse(struct platform_device *pdev,
 		pdata->flags |= TMIO_MMC_WRPROTECT_DISABLE;
 }
 
-struct tmio_mmc_host*
-tmio_mmc_host_alloc(struct platform_device *pdev)
+struct tmio_mmc_host *tmio_mmc_host_alloc(struct platform_device *pdev,
+					  struct tmio_mmc_data *pdata)
 {
 	struct tmio_mmc_host *host;
 	struct mmc_host *mmc;
+	struct resource *res;
+	void __iomem *ctl;
+	int ret;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	ctl = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(ctl))
+		return ERR_CAST(ctl);
 
 	mmc = mmc_alloc_host(sizeof(struct tmio_mmc_host), &pdev->dev);
 	if (!mmc)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	host = mmc_priv(mmc);
+	host->ctl = ctl;
 	host->mmc = mmc;
 	host->pdev = pdev;
+	host->pdata = pdata;
+	host->ops = tmio_mmc_ops;
+	mmc->ops = &host->ops;
+
+	ret = mmc_of_parse(host->mmc);
+	if (ret) {
+		host = ERR_PTR(ret);
+		goto free;
+	}
+
+	tmio_mmc_of_parse(pdev, pdata);
+
+	platform_set_drvdata(pdev, host);
+
+	return host;
+free:
+	mmc_free_host(mmc);
 
 	return host;
 }
@@ -1169,32 +1175,24 @@ void tmio_mmc_host_free(struct tmio_mmc_host *host)
 }
 EXPORT_SYMBOL_GPL(tmio_mmc_host_free);
 
-int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
-			struct tmio_mmc_data *pdata,
-			const struct tmio_mmc_dma_ops *dma_ops)
+int tmio_mmc_host_probe(struct tmio_mmc_host *_host)
 {
 	struct platform_device *pdev = _host->pdev;
+	struct tmio_mmc_data *pdata = _host->pdata;
 	struct mmc_host *mmc = _host->mmc;
-	struct resource *res_ctl;
 	int ret;
 	u32 irq_mask = TMIO_MASK_CMD;
 
-	tmio_mmc_of_parse(pdev, pdata);
+	/*
+	 * Check the sanity of mmc->f_min to prevent tmio_mmc_set_clock() from
+	 * looping forever...
+	 */
+	if (mmc->f_min == 0)
+		return -EINVAL;
 
 	if (!(pdata->flags & TMIO_MMC_HAS_IDLE_WAIT))
 		_host->write16_hook = NULL;
 
-	res_ctl = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res_ctl)
-		return -EINVAL;
-
-	ret = mmc_of_parse(mmc);
-	if (ret < 0)
-		return ret;
-
-	_host->pdata = pdata;
-	platform_set_drvdata(pdev, mmc);
-
 	_host->set_pwr = pdata->set_pwr;
 	_host->set_clk_div = pdata->set_clk_div;
 
@@ -1202,15 +1200,11 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 	if (ret < 0)
 		return ret;
 
-	_host->ctl = devm_ioremap(&pdev->dev,
-				  res_ctl->start, resource_size(res_ctl));
-	if (!_host->ctl)
-		return -ENOMEM;
-
-	tmio_mmc_ops.card_busy = _host->card_busy;
-	tmio_mmc_ops.start_signal_voltage_switch =
-		_host->start_signal_voltage_switch;
-	mmc->ops = &tmio_mmc_ops;
+	if (pdata->flags & TMIO_MMC_USE_GPIO_CD) {
+		ret = mmc_gpio_request_cd(mmc, pdata->cd_gpio, 0);
+		if (ret)
+			return ret;
+	}
 
 	mmc->caps |= MMC_CAP_4_BIT_DATA | pdata->capabilities;
 	mmc->caps2 |= pdata->capabilities2;
@@ -1233,7 +1227,10 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 	}
 	mmc->max_seg_size = mmc->max_req_size;
 
-	_host->native_hotplug = !(pdata->flags & TMIO_MMC_USE_GPIO_CD ||
+	if (mmc_can_gpio_ro(mmc))
+		_host->ops.get_ro = mmc_gpio_get_ro;
+
+	_host->native_hotplug = !(mmc_can_gpio_cd(mmc) ||
 				  mmc->caps & MMC_CAP_NEEDS_POLL ||
 				  !mmc_card_is_removable(mmc));
 
@@ -1246,18 +1243,6 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 	if (pdata->flags & TMIO_MMC_MIN_RCAR2)
 		_host->native_hotplug = true;
 
-	if (tmio_mmc_clk_enable(_host) < 0) {
-		mmc->f_max = pdata->hclk;
-		mmc->f_min = mmc->f_max / 512;
-	}
-
-	/*
-	 * Check the sanity of mmc->f_min to prevent tmio_mmc_set_clock() from
-	 * looping forever...
-	 */
-	if (mmc->f_min == 0)
-		return -EINVAL;
-
 	/*
 	 * While using internal tmio hardware logic for card detection, we need
 	 * to ensure it stays powered for it to work.
@@ -1293,7 +1278,6 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 	INIT_WORK(&_host->done, tmio_mmc_done_work);
 
 	/* See if we also get DMA */
-	_host->dma_ops = dma_ops;
 	tmio_mmc_request_dma(_host, pdata);
 
 	pm_runtime_set_active(&pdev->dev);
@@ -1307,14 +1291,6 @@ int tmio_mmc_host_probe(struct tmio_mmc_host *_host,
 
 	dev_pm_qos_expose_latency_limit(&pdev->dev, 100);
 
-	if (pdata->flags & TMIO_MMC_USE_GPIO_CD) {
-		ret = mmc_gpio_request_cd(mmc, pdata->cd_gpio, 0);
-		if (ret)
-			goto remove_host;
-
-		mmc_gpiod_request_cd_irq(mmc);
-	}
-
 	return 0;
 
 remove_host:
@@ -1343,16 +1319,27 @@ void tmio_mmc_host_remove(struct tmio_mmc_host *host)
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	tmio_mmc_clk_disable(host);
 }
 EXPORT_SYMBOL_GPL(tmio_mmc_host_remove);
 
 #ifdef CONFIG_PM
+static int tmio_mmc_clk_enable(struct tmio_mmc_host *host)
+{
+	if (!host->clk_enable)
+		return -ENOTSUPP;
+
+	return host->clk_enable(host);
+}
+
+static void tmio_mmc_clk_disable(struct tmio_mmc_host *host)
+{
+	if (host->clk_disable)
+		host->clk_disable(host);
+}
+
 int tmio_mmc_host_runtime_suspend(struct device *dev)
 {
-	struct mmc_host *mmc = dev_get_drvdata(dev);
-	struct tmio_mmc_host *host = mmc_priv(mmc);
+	struct tmio_mmc_host *host = dev_get_drvdata(dev);
 
 	tmio_mmc_disable_mmc_irqs(host, TMIO_MASK_ALL);
 
@@ -1372,8 +1359,7 @@ static bool tmio_mmc_can_retune(struct tmio_mmc_host *host)
 
 int tmio_mmc_host_runtime_resume(struct device *dev)
 {
-	struct mmc_host *mmc = dev_get_drvdata(dev);
-	struct tmio_mmc_host *host = mmc_priv(mmc);
+	struct tmio_mmc_host *host = dev_get_drvdata(dev);
 
 	tmio_mmc_reset(host);
 	tmio_mmc_clk_enable(host);
diff --git a/drivers/mtd/chips/jedec_probe.c b/drivers/mtd/chips/jedec_probe.c
index 7c0b27d132b1..b479bd81120b 100644
--- a/drivers/mtd/chips/jedec_probe.c
+++ b/drivers/mtd/chips/jedec_probe.c
@@ -1889,6 +1889,8 @@ static inline u32 jedec_read_mfr(struct map_info *map, uint32_t base,
 	do {
 		uint32_t ofs = cfi_build_cmd_addr(0 + (bank << 8), map, cfi);
 		mask = (1 << (cfi->device_type * 8)) - 1;
+		if (ofs >= map->size)
+			return 0;
 		result = map_read(map, base + ofs);
 		bank++;
 	} while ((result.x[0] & mask) == CFI_MFR_CONTINUATION);
diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c
index 0806f72102c0..a85af236b44d 100644
--- a/drivers/mtd/devices/docg3.c
+++ b/drivers/mtd/devices/docg3.c
@@ -904,9 +904,6 @@ static int doc_read_oob(struct mtd_info *mtd, loff_t from,
 	if (ooblen % DOC_LAYOUT_OOB_SIZE)
 		return -EINVAL;
 
-	if (from + len > mtd->size)
-		return -EINVAL;
-
 	ops->oobretlen = 0;
 	ops->retlen = 0;
 	ret = 0;
@@ -990,36 +987,6 @@ static int doc_read_oob(struct mtd_info *mtd, loff_t from,
 	goto out;
 }
 
-/**
- * doc_read - Read bytes from flash
- * @mtd: the device
- * @from: the offset from first block and first page, in bytes, aligned on page
- *        size
- * @len: the number of bytes to read (must be a multiple of 4)
- * @retlen: the number of bytes actually read
- * @buf: the filled in buffer
- *
- * Reads flash memory pages. This function does not read the OOB chunk, but only
- * the page data.
- *
- * Returns 0 if read successful, of -EIO, -EINVAL if an error occurred
- */
-static int doc_read(struct mtd_info *mtd, loff_t from, size_t len,
-	     size_t *retlen, u_char *buf)
-{
-	struct mtd_oob_ops ops;
-	size_t ret;
-
-	memset(&ops, 0, sizeof(ops));
-	ops.datbuf = buf;
-	ops.len = len;
-	ops.mode = MTD_OPS_AUTO_OOB;
-
-	ret = doc_read_oob(mtd, from, &ops);
-	*retlen = ops.retlen;
-	return ret;
-}
-
 static int doc_reload_bbt(struct docg3 *docg3)
 {
 	int block = DOC_LAYOUT_BLOCK_BBT;
@@ -1471,8 +1438,6 @@ static int doc_write_oob(struct mtd_info *mtd, loff_t ofs,
 	if (len && ooblen &&
 	    (len / DOC_LAYOUT_PAGE_SIZE) != (ooblen / oobdelta))
 		return -EINVAL;
-	if (ofs + len > mtd->size)
-		return -EINVAL;
 
 	ops->oobretlen = 0;
 	ops->retlen = 0;
@@ -1513,39 +1478,6 @@ static int doc_write_oob(struct mtd_info *mtd, loff_t ofs,
 	return ret;
 }
 
-/**
- * doc_write - Write a buffer to the chip
- * @mtd: the device
- * @to: the offset from first block and first page, in bytes, aligned on page
- *      size
- * @len: the number of bytes to write (must be a full page size, ie. 512)
- * @retlen: the number of bytes actually written (0 or 512)
- * @buf: the buffer to get bytes from
- *
- * Writes data to the chip.
- *
- * Returns 0 if write successful, -EIO if write error
- */
-static int doc_write(struct mtd_info *mtd, loff_t to, size_t len,
-		     size_t *retlen, const u_char *buf)
-{
-	struct docg3 *docg3 = mtd->priv;
-	int ret;
-	struct mtd_oob_ops ops;
-
-	doc_dbg("doc_write(to=%lld, len=%zu)\n", to, len);
-	ops.datbuf = (char *)buf;
-	ops.len = len;
-	ops.mode = MTD_OPS_PLACE_OOB;
-	ops.oobbuf = NULL;
-	ops.ooblen = 0;
-	ops.ooboffs = 0;
-
-	ret = doc_write_oob(mtd, to, &ops);
-	*retlen = ops.retlen;
-	return ret;
-}
-
 static struct docg3 *sysfs_dev2docg3(struct device *dev,
 				     struct device_attribute *attr)
 {
@@ -1866,8 +1798,6 @@ static int __init doc_set_driver_info(int chip_id, struct mtd_info *mtd)
 	mtd->writebufsize = mtd->writesize = DOC_LAYOUT_PAGE_SIZE;
 	mtd->oobsize = DOC_LAYOUT_OOB_SIZE;
 	mtd->_erase = doc_erase;
-	mtd->_read = doc_read;
-	mtd->_write = doc_write;
 	mtd->_read_oob = doc_read_oob;
 	mtd->_write_oob = doc_write_oob;
 	mtd->_block_isbad = doc_block_isbad;
diff --git a/drivers/mtd/devices/m25p80.c b/drivers/mtd/devices/m25p80.c
index dbe6a1de2bb8..a4e18f6aaa33 100644
--- a/drivers/mtd/devices/m25p80.c
+++ b/drivers/mtd/devices/m25p80.c
@@ -307,10 +307,18 @@ static int m25p_remove(struct spi_device *spi)
 {
 	struct m25p	*flash = spi_get_drvdata(spi);
 
+	spi_nor_restore(&flash->spi_nor);
+
 	/* Clean up MTD stuff. */
 	return mtd_device_unregister(&flash->spi_nor.mtd);
 }
 
+static void m25p_shutdown(struct spi_device *spi)
+{
+	struct m25p *flash = spi_get_drvdata(spi);
+
+	spi_nor_restore(&flash->spi_nor);
+}
 /*
  * Do NOT add to this array without reading the following:
  *
@@ -386,6 +394,7 @@ static struct spi_driver m25p80_driver = {
 	.id_table	= m25p_ids,
 	.probe	= m25p_probe,
 	.remove	= m25p_remove,
+	.shutdown	= m25p_shutdown,
 
 	/* REVISIT: many of these chips have deep power-down modes, which
 	 * should clearly be entered on suspend() to minimize power use.
diff --git a/drivers/mtd/devices/mchp23k256.c b/drivers/mtd/devices/mchp23k256.c
index 8956b7dcc984..75f71d166fd6 100644
--- a/drivers/mtd/devices/mchp23k256.c
+++ b/drivers/mtd/devices/mchp23k256.c
@@ -68,6 +68,7 @@ static int mchp23k256_write(struct mtd_info *mtd, loff_t to, size_t len,
 	struct spi_transfer transfer[2] = {};
 	struct spi_message message;
 	unsigned char command[MAX_CMD_SIZE];
+	int ret;
 
 	spi_message_init(&message);
 
@@ -84,12 +85,16 @@ static int mchp23k256_write(struct mtd_info *mtd, loff_t to, size_t len,
 
 	mutex_lock(&flash->lock);
 
-	spi_sync(flash->spi, &message);
+	ret = spi_sync(flash->spi, &message);
+
+	mutex_unlock(&flash->lock);
+
+	if (ret)
+		return ret;
 
 	if (retlen && message.actual_length > sizeof(command))
 		*retlen += message.actual_length - sizeof(command);
 
-	mutex_unlock(&flash->lock);
 	return 0;
 }
 
@@ -100,6 +105,7 @@ static int mchp23k256_read(struct mtd_info *mtd, loff_t from, size_t len,
 	struct spi_transfer transfer[2] = {};
 	struct spi_message message;
 	unsigned char command[MAX_CMD_SIZE];
+	int ret;
 
 	spi_message_init(&message);
 
@@ -117,12 +123,16 @@ static int mchp23k256_read(struct mtd_info *mtd, loff_t from, size_t len,
 
 	mutex_lock(&flash->lock);
 
-	spi_sync(flash->spi, &message);
+	ret = spi_sync(flash->spi, &message);
+
+	mutex_unlock(&flash->lock);
+
+	if (ret)
+		return ret;
 
 	if (retlen && message.actual_length > sizeof(command))
 		*retlen += message.actual_length - sizeof(command);
 
-	mutex_unlock(&flash->lock);
 	return 0;
 }
 
diff --git a/drivers/mtd/mtdchar.c b/drivers/mtd/mtdchar.c
index de8c902059b8..7d80a8bb96fe 100644
--- a/drivers/mtd/mtdchar.c
+++ b/drivers/mtd/mtdchar.c
@@ -479,7 +479,7 @@ static int shrink_ecclayout(struct mtd_info *mtd,
 	for (i = 0; i < MTD_MAX_ECCPOS_ENTRIES;) {
 		u32 eccpos;
 
-		ret = mtd_ooblayout_ecc(mtd, section, &oobregion);
+		ret = mtd_ooblayout_ecc(mtd, section++, &oobregion);
 		if (ret < 0) {
 			if (ret != -ERANGE)
 				return ret;
@@ -526,7 +526,7 @@ static int get_oobinfo(struct mtd_info *mtd, struct nand_oobinfo *to)
 	for (i = 0; i < ARRAY_SIZE(to->eccpos);) {
 		u32 eccpos;
 
-		ret = mtd_ooblayout_ecc(mtd, section, &oobregion);
+		ret = mtd_ooblayout_ecc(mtd, section++, &oobregion);
 		if (ret < 0) {
 			if (ret != -ERANGE)
 				return ret;
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 73b605577447..28553c840d32 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -503,6 +503,11 @@ int add_mtd_device(struct mtd_info *mtd)
 		return -EEXIST;
 
 	BUG_ON(mtd->writesize == 0);
+
+	if (WARN_ON((!mtd->erasesize || !mtd->_erase) &&
+		    !(mtd->flags & MTD_NO_ERASE)))
+		return -EINVAL;
+
 	mutex_lock(&mtd_table_mutex);
 
 	i = idr_alloc(&mtd_idr, mtd, 0, 0, GFP_KERNEL);
@@ -1053,7 +1058,20 @@ int mtd_read(struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen,
 	 * representing the maximum number of bitflips that were corrected on
 	 * any one ecc region (if applicable; zero otherwise).
 	 */
-	ret_code = mtd->_read(mtd, from, len, retlen, buf);
+	if (mtd->_read) {
+		ret_code = mtd->_read(mtd, from, len, retlen, buf);
+	} else if (mtd->_read_oob) {
+		struct mtd_oob_ops ops = {
+			.len = len,
+			.datbuf = buf,
+		};
+
+		ret_code = mtd->_read_oob(mtd, from, &ops);
+		*retlen = ops.retlen;
+	} else {
+		return -ENOTSUPP;
+	}
+
 	if (unlikely(ret_code < 0))
 		return ret_code;
 	if (mtd->ecc_strength == 0)
@@ -1068,11 +1086,25 @@ int mtd_write(struct mtd_info *mtd, loff_t to, size_t len, size_t *retlen,
 	*retlen = 0;
 	if (to < 0 || to >= mtd->size || len > mtd->size - to)
 		return -EINVAL;
-	if (!mtd->_write || !(mtd->flags & MTD_WRITEABLE))
+	if ((!mtd->_write && !mtd->_write_oob) ||
+	    !(mtd->flags & MTD_WRITEABLE))
 		return -EROFS;
 	if (!len)
 		return 0;
 	ledtrig_mtd_activity();
+
+	if (!mtd->_write) {
+		struct mtd_oob_ops ops = {
+			.len = len,
+			.datbuf = (u8 *)buf,
+		};
+		int ret;
+
+		ret = mtd->_write_oob(mtd, to, &ops);
+		*retlen = ops.retlen;
+		return ret;
+	}
+
 	return mtd->_write(mtd, to, len, retlen, buf);
 }
 EXPORT_SYMBOL_GPL(mtd_write);
diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c
index be088bccd593..76cd21d1171b 100644
--- a/drivers/mtd/mtdpart.c
+++ b/drivers/mtd/mtdpart.c
@@ -105,34 +105,17 @@ static int part_read_oob(struct mtd_info *mtd, loff_t from,
 		struct mtd_oob_ops *ops)
 {
 	struct mtd_part *part = mtd_to_part(mtd);
+	struct mtd_ecc_stats stats;
 	int res;
 
-	if (from >= mtd->size)
-		return -EINVAL;
-	if (ops->datbuf && from + ops->len > mtd->size)
-		return -EINVAL;
-
-	/*
-	 * If OOB is also requested, make sure that we do not read past the end
-	 * of this partition.
-	 */
-	if (ops->oobbuf) {
-		size_t len, pages;
-
-		len = mtd_oobavail(mtd, ops);
-		pages = mtd_div_by_ws(mtd->size, mtd);
-		pages -= mtd_div_by_ws(from, mtd);
-		if (ops->ooboffs + ops->ooblen > pages * len)
-			return -EINVAL;
-	}
-
+	stats = part->parent->ecc_stats;
 	res = part->parent->_read_oob(part->parent, from + part->offset, ops);
-	if (unlikely(res)) {
-		if (mtd_is_bitflip(res))
-			mtd->ecc_stats.corrected++;
-		if (mtd_is_eccerr(res))
-			mtd->ecc_stats.failed++;
-	}
+	if (unlikely(mtd_is_eccerr(res)))
+		mtd->ecc_stats.failed +=
+			part->parent->ecc_stats.failed - stats.failed;
+	else
+		mtd->ecc_stats.corrected +=
+			part->parent->ecc_stats.corrected - stats.corrected;
 	return res;
 }
 
@@ -189,10 +172,6 @@ static int part_write_oob(struct mtd_info *mtd, loff_t to,
 {
 	struct mtd_part *part = mtd_to_part(mtd);
 
-	if (to >= mtd->size)
-		return -EINVAL;
-	if (ops->datbuf && to + ops->len > mtd->size)
-		return -EINVAL;
 	return part->parent->_write_oob(part->parent, to + part->offset, ops);
 }
 
@@ -435,8 +414,10 @@ static struct mtd_part *allocate_partition(struct mtd_info *parent,
 				parent->dev.parent;
 	slave->mtd.dev.of_node = part->of_node;
 
-	slave->mtd._read = part_read;
-	slave->mtd._write = part_write;
+	if (parent->_read)
+		slave->mtd._read = part_read;
+	if (parent->_write)
+		slave->mtd._write = part_write;
 
 	if (parent->_panic_write)
 		slave->mtd._panic_write = part_panic_write;
diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c
index f07492c6f4b2..7eb0e1f4f980 100644
--- a/drivers/mtd/mtdswap.c
+++ b/drivers/mtd/mtdswap.c
@@ -1223,8 +1223,9 @@ static int mtdswap_show(struct seq_file *s, void *data)
 	unsigned int max[MTDSWAP_TREE_CNT];
 	unsigned int i, cw = 0, cwp = 0, cwecount = 0, bb_cnt, mapped, pages;
 	uint64_t use_size;
-	char *name[] = {"clean", "used", "low", "high", "dirty", "bitflip",
-			"failing"};
+	static const char * const name[] = {
+		"clean", "used", "low", "high", "dirty", "bitflip", "failing"
+	};
 
 	mutex_lock(&d->mbd_dev->lock);
 
diff --git a/drivers/mtd/nand/Kconfig b/drivers/mtd/nand/Kconfig
index bb48aafed9a2..736ac887303c 100644
--- a/drivers/mtd/nand/Kconfig
+++ b/drivers/mtd/nand/Kconfig
@@ -315,6 +315,7 @@ config MTD_NAND_ATMEL
 
 config MTD_NAND_PXA3xx
 	tristate "NAND support on PXA3xx and Armada 370/XP"
+	depends on !MTD_NAND_MARVELL
 	depends on PXA3xx || ARCH_MMP || PLAT_ORION || ARCH_MVEBU
 	help
 
@@ -323,6 +324,18 @@ config MTD_NAND_PXA3xx
 	  platforms (XP, 370, 375, 38x, 39x) and 64-bit Armada
 	  platforms (7K, 8K) (NFCv2).
 
+config MTD_NAND_MARVELL
+	tristate "NAND controller support on Marvell boards"
+	depends on PXA3xx || ARCH_MMP || PLAT_ORION || ARCH_MVEBU || \
+		   COMPILE_TEST
+	depends on HAS_IOMEM && HAS_DMA
+	help
+	  This enables the NAND flash controller driver for Marvell boards,
+	  including:
+	  - PXA3xx processors (NFCv1)
+	  - 32-bit Armada platforms (XP, 37x, 38x, 39x) (NFCv2)
+	  - 64-bit Aramda platforms (7k, 8k) (NFCv2)
+
 config MTD_NAND_SLC_LPC32XX
 	tristate "NXP LPC32xx SLC Controller"
 	depends on ARCH_LPC32XX
@@ -376,9 +389,7 @@ config MTD_NAND_GPMI_NAND
 	 Enables NAND Flash support for IMX23, IMX28 or IMX6.
 	 The GPMI controller is very powerful, with the help of BCH
 	 module, it can do the hardware ECC. The GPMI supports several
-	 NAND flashs at the same time. The GPMI may conflicts with other
-	 block, such as SD card. So pay attention to it when you enable
-	 the GPMI.
+	 NAND flashs at the same time.
 
 config MTD_NAND_BRCMNAND
 	tristate "Broadcom STB NAND controller"
diff --git a/drivers/mtd/nand/Makefile b/drivers/mtd/nand/Makefile
index 118a1349aad3..921634ba400c 100644
--- a/drivers/mtd/nand/Makefile
+++ b/drivers/mtd/nand/Makefile
@@ -32,6 +32,7 @@ obj-$(CONFIG_MTD_NAND_OMAP2) 		+= omap2_nand.o
 obj-$(CONFIG_MTD_NAND_OMAP_BCH_BUILD)	+= omap_elm.o
 obj-$(CONFIG_MTD_NAND_CM_X270)		+= cmx270_nand.o
 obj-$(CONFIG_MTD_NAND_PXA3xx)		+= pxa3xx_nand.o
+obj-$(CONFIG_MTD_NAND_MARVELL)		+= marvell_nand.o
 obj-$(CONFIG_MTD_NAND_TMIO)		+= tmio_nand.o
 obj-$(CONFIG_MTD_NAND_PLATFORM)		+= plat_nand.o
 obj-$(CONFIG_MTD_NAND_PASEMI)		+= pasemi_nand.o
diff --git a/drivers/mtd/nand/atmel/nand-controller.c b/drivers/mtd/nand/atmel/nand-controller.c
index 90a71a56bc23..b2f00b398490 100644
--- a/drivers/mtd/nand/atmel/nand-controller.c
+++ b/drivers/mtd/nand/atmel/nand-controller.c
@@ -841,6 +841,8 @@ static int atmel_nand_pmecc_write_pg(struct nand_chip *chip, const u8 *buf,
 	struct atmel_nand *nand = to_atmel_nand(chip);
 	int ret;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	ret = atmel_nand_pmecc_enable(chip, NAND_ECC_WRITE, raw);
 	if (ret)
 		return ret;
@@ -857,7 +859,7 @@ static int atmel_nand_pmecc_write_pg(struct nand_chip *chip, const u8 *buf,
 
 	atmel_nand_write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int atmel_nand_pmecc_write_page(struct mtd_info *mtd,
@@ -881,6 +883,8 @@ static int atmel_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf,
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	int ret;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	ret = atmel_nand_pmecc_enable(chip, NAND_ECC_READ, raw);
 	if (ret)
 		return ret;
@@ -1000,7 +1004,7 @@ static int atmel_hsmc_nand_pmecc_read_pg(struct nand_chip *chip, u8 *buf,
 	 * to the non-optimized one.
 	 */
 	if (nand->activecs->rb.type != ATMEL_NAND_NATIVE_RB) {
-		chip->cmdfunc(mtd, NAND_CMD_READ0, 0x00, page);
+		nand_read_page_op(chip, page, 0, NULL, 0);
 
 		return atmel_nand_pmecc_read_pg(chip, buf, oob_required, page,
 						raw);
@@ -1178,7 +1182,6 @@ static int atmel_hsmc_nand_ecc_init(struct atmel_nand *nand)
 	chip->ecc.write_page = atmel_hsmc_nand_pmecc_write_page;
 	chip->ecc.read_page_raw = atmel_hsmc_nand_pmecc_read_page_raw;
 	chip->ecc.write_page_raw = atmel_hsmc_nand_pmecc_write_page_raw;
-	chip->ecc.options |= NAND_ECC_CUSTOM_PAGE_ACCESS;
 
 	return 0;
 }
diff --git a/drivers/mtd/nand/atmel/pmecc.c b/drivers/mtd/nand/atmel/pmecc.c
index fcbe4fd6e684..ca0a70389ba9 100644
--- a/drivers/mtd/nand/atmel/pmecc.c
+++ b/drivers/mtd/nand/atmel/pmecc.c
@@ -426,7 +426,7 @@ static int get_strength(struct atmel_pmecc_user *user)
 
 static int get_sectorsize(struct atmel_pmecc_user *user)
 {
-	return user->cache.cfg & PMECC_LOOKUP_TABLE_SIZE_1024 ? 1024 : 512;
+	return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;
 }
 
 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
diff --git a/drivers/mtd/nand/bf5xx_nand.c b/drivers/mtd/nand/bf5xx_nand.c
index 5655dca6ce43..87bbd177b3e5 100644
--- a/drivers/mtd/nand/bf5xx_nand.c
+++ b/drivers/mtd/nand/bf5xx_nand.c
@@ -572,6 +572,8 @@ static void bf5xx_nand_dma_write_buf(struct mtd_info *mtd,
 static int bf5xx_nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 		uint8_t *buf, int oob_required, int page)
 {
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	bf5xx_nand_read_buf(mtd, buf, mtd->writesize);
 	bf5xx_nand_read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
@@ -582,10 +584,10 @@ static int bf5xx_nand_write_page_raw(struct mtd_info *mtd,
 		struct nand_chip *chip,	const uint8_t *buf, int oob_required,
 		int page)
 {
-	bf5xx_nand_write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	bf5xx_nand_write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 /*
diff --git a/drivers/mtd/nand/brcmnand/brcmnand.c b/drivers/mtd/nand/brcmnand/brcmnand.c
index dd56a671ea42..c28fd2bc1a84 100644
--- a/drivers/mtd/nand/brcmnand/brcmnand.c
+++ b/drivers/mtd/nand/brcmnand/brcmnand.c
@@ -1071,7 +1071,7 @@ static void brcmnand_wp(struct mtd_info *mtd, int wp)
 			return;
 
 		brcmnand_set_wp(ctrl, wp);
-		chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
+		nand_status_op(chip, NULL);
 		/* NAND_STATUS_WP 0x00 = protected, 0x80 = not protected */
 		ret = bcmnand_ctrl_poll_status(ctrl,
 					       NAND_CTRL_RDY |
@@ -1453,7 +1453,7 @@ static uint8_t brcmnand_read_byte(struct mtd_info *mtd)
 
 		/* At FC_BYTES boundary, switch to next column */
 		if (host->last_byte > 0 && offs == 0)
-			chip->cmdfunc(mtd, NAND_CMD_RNDOUT, addr, -1);
+			nand_change_read_column_op(chip, addr, NULL, 0, false);
 
 		ret = ctrl->flash_cache[offs];
 		break;
@@ -1681,7 +1681,7 @@ static int brcmstb_nand_verify_erased_page(struct mtd_info *mtd,
 	int ret;
 
 	if (!buf) {
-		buf = chip->buffers->databuf;
+		buf = chip->data_buf;
 		/* Invalidate page cache */
 		chip->pagebuf = -1;
 	}
@@ -1689,7 +1689,6 @@ static int brcmstb_nand_verify_erased_page(struct mtd_info *mtd,
 	sas = mtd->oobsize / chip->ecc.steps;
 
 	/* read without ecc for verification */
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0x00, page);
 	ret = chip->ecc.read_page_raw(mtd, chip, buf, true, page);
 	if (ret)
 		return ret;
@@ -1793,6 +1792,8 @@ static int brcmnand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	struct brcmnand_host *host = nand_get_controller_data(chip);
 	u8 *oob = oob_required ? (u8 *)chip->oob_poi : NULL;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	return brcmnand_read(mtd, chip, host->last_addr,
 			mtd->writesize >> FC_SHIFT, (u32 *)buf, oob);
 }
@@ -1804,6 +1805,8 @@ static int brcmnand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 	u8 *oob = oob_required ? (u8 *)chip->oob_poi : NULL;
 	int ret;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	brcmnand_set_ecc_enabled(host, 0);
 	ret = brcmnand_read(mtd, chip, host->last_addr,
 			mtd->writesize >> FC_SHIFT, (u32 *)buf, oob);
@@ -1909,8 +1912,10 @@ static int brcmnand_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	struct brcmnand_host *host = nand_get_controller_data(chip);
 	void *oob = oob_required ? chip->oob_poi : NULL;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 	brcmnand_write(mtd, chip, host->last_addr, (const u32 *)buf, oob);
-	return 0;
+
+	return nand_prog_page_end_op(chip);
 }
 
 static int brcmnand_write_page_raw(struct mtd_info *mtd,
@@ -1920,10 +1925,12 @@ static int brcmnand_write_page_raw(struct mtd_info *mtd,
 	struct brcmnand_host *host = nand_get_controller_data(chip);
 	void *oob = oob_required ? chip->oob_poi : NULL;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 	brcmnand_set_ecc_enabled(host, 0);
 	brcmnand_write(mtd, chip, host->last_addr, (const u32 *)buf, oob);
 	brcmnand_set_ecc_enabled(host, 1);
-	return 0;
+
+	return nand_prog_page_end_op(chip);
 }
 
 static int brcmnand_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
@@ -2193,16 +2200,9 @@ static int brcmnand_setup_dev(struct brcmnand_host *host)
 	if (ctrl->nand_version >= 0x0702)
 		tmp |= ACC_CONTROL_RD_ERASED;
 	tmp &= ~ACC_CONTROL_FAST_PGM_RDIN;
-	if (ctrl->features & BRCMNAND_HAS_PREFETCH) {
-		/*
-		 * FIXME: Flash DMA + prefetch may see spurious erased-page ECC
-		 * errors
-		 */
-		if (has_flash_dma(ctrl))
-			tmp &= ~ACC_CONTROL_PREFETCH;
-		else
-			tmp |= ACC_CONTROL_PREFETCH;
-	}
+	if (ctrl->features & BRCMNAND_HAS_PREFETCH)
+		tmp &= ~ACC_CONTROL_PREFETCH;
+
 	nand_writereg(ctrl, offs, tmp);
 
 	return 0;
@@ -2230,6 +2230,9 @@ static int brcmnand_init_cs(struct brcmnand_host *host, struct device_node *dn)
 	nand_set_controller_data(chip, host);
 	mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "brcmnand.%d",
 				   host->cs);
+	if (!mtd->name)
+		return -ENOMEM;
+
 	mtd->owner = THIS_MODULE;
 	mtd->dev.parent = &pdev->dev;
 
@@ -2369,12 +2372,11 @@ static int brcmnand_resume(struct device *dev)
 
 	list_for_each_entry(host, &ctrl->host_list, node) {
 		struct nand_chip *chip = &host->chip;
-		struct mtd_info *mtd = nand_to_mtd(chip);
 
 		brcmnand_save_restore_cs_config(host, 1);
 
 		/* Reset the chip, required by some chips after power-up */
-		chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+		nand_reset_op(chip);
 	}
 
 	return 0;
diff --git a/drivers/mtd/nand/cafe_nand.c b/drivers/mtd/nand/cafe_nand.c
index bc558c438a57..567ff972d5fc 100644
--- a/drivers/mtd/nand/cafe_nand.c
+++ b/drivers/mtd/nand/cafe_nand.c
@@ -353,23 +353,15 @@ static void cafe_nand_bug(struct mtd_info *mtd)
 static int cafe_nand_write_oob(struct mtd_info *mtd,
 			       struct nand_chip *chip, int page)
 {
-	int status = 0;
-
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize, page);
-	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,
+				 mtd->oobsize);
 }
 
 /* Don't use -- use nand_read_oob_std for now */
 static int cafe_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 			      int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
-	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
-	return 0;
+	return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
 }
 /**
  * cafe_nand_read_page_syndrome - [REPLACEABLE] hardware ecc syndrome based page read
@@ -391,7 +383,7 @@ static int cafe_nand_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 		     cafe_readl(cafe, NAND_ECC_RESULT),
 		     cafe_readl(cafe, NAND_ECC_SYN01));
 
-	chip->read_buf(mtd, buf, mtd->writesize);
+	nand_read_page_op(chip, page, 0, buf, mtd->writesize);
 	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
 	if (checkecc && cafe_readl(cafe, NAND_ECC_RESULT) & (1<<18)) {
@@ -549,13 +541,13 @@ static int cafe_nand_write_page_lowlevel(struct mtd_info *mtd,
 {
 	struct cafe_priv *cafe = nand_get_controller_data(chip);
 
-	chip->write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
 	/* Set up ECC autogeneration */
 	cafe->ctl2 |= (1<<30);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int cafe_nand_block_bad(struct mtd_info *mtd, loff_t ofs)
@@ -613,7 +605,6 @@ static int cafe_nand_probe(struct pci_dev *pdev,
 	uint32_t ctrl;
 	int err = 0;
 	int old_dma;
-	struct nand_buffers *nbuf;
 
 	/* Very old versions shared the same PCI ident for all three
 	   functions on the chip. Verify the class too... */
@@ -661,7 +652,6 @@ static int cafe_nand_probe(struct pci_dev *pdev,
 
 	/* Enable the following for a flash based bad block table */
 	cafe->nand.bbt_options = NAND_BBT_USE_FLASH;
-	cafe->nand.options = NAND_OWN_BUFFERS;
 
 	if (skipbbt) {
 		cafe->nand.options |= NAND_SKIP_BBTSCAN;
@@ -731,32 +721,20 @@ static int cafe_nand_probe(struct pci_dev *pdev,
 	if (err)
 		goto out_irq;
 
-	cafe->dmabuf = dma_alloc_coherent(&cafe->pdev->dev,
-				2112 + sizeof(struct nand_buffers) +
-				mtd->writesize + mtd->oobsize,
-				&cafe->dmaaddr, GFP_KERNEL);
+	cafe->dmabuf = dma_alloc_coherent(&cafe->pdev->dev, 2112,
+					  &cafe->dmaaddr, GFP_KERNEL);
 	if (!cafe->dmabuf) {
 		err = -ENOMEM;
 		goto out_irq;
 	}
-	cafe->nand.buffers = nbuf = (void *)cafe->dmabuf + 2112;
 
 	/* Set up DMA address */
-	cafe_writel(cafe, cafe->dmaaddr & 0xffffffff, NAND_DMA_ADDR0);
-	if (sizeof(cafe->dmaaddr) > 4)
-		/* Shift in two parts to shut the compiler up */
-		cafe_writel(cafe, (cafe->dmaaddr >> 16) >> 16, NAND_DMA_ADDR1);
-	else
-		cafe_writel(cafe, 0, NAND_DMA_ADDR1);
+	cafe_writel(cafe, lower_32_bits(cafe->dmaaddr), NAND_DMA_ADDR0);
+	cafe_writel(cafe, upper_32_bits(cafe->dmaaddr), NAND_DMA_ADDR1);
 
 	cafe_dev_dbg(&cafe->pdev->dev, "Set DMA address to %x (virt %p)\n",
 		cafe_readl(cafe, NAND_DMA_ADDR0), cafe->dmabuf);
 
-	/* this driver does not need the @ecccalc and @ecccode */
-	nbuf->ecccalc = NULL;
-	nbuf->ecccode = NULL;
-	nbuf->databuf = (uint8_t *)(nbuf + 1);
-
 	/* Restore the DMA flag */
 	usedma = old_dma;
 
@@ -801,10 +779,7 @@ static int cafe_nand_probe(struct pci_dev *pdev,
 	goto out;
 
  out_free_dma:
-	dma_free_coherent(&cafe->pdev->dev,
-			2112 + sizeof(struct nand_buffers) +
-			mtd->writesize + mtd->oobsize,
-			cafe->dmabuf, cafe->dmaaddr);
+	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);
  out_irq:
 	/* Disable NAND IRQ in global IRQ mask register */
 	cafe_writel(cafe, ~1 & cafe_readl(cafe, GLOBAL_IRQ_MASK), GLOBAL_IRQ_MASK);
@@ -829,10 +804,7 @@ static void cafe_nand_remove(struct pci_dev *pdev)
 	nand_release(mtd);
 	free_rs(cafe->rs);
 	pci_iounmap(pdev, cafe->mmio);
-	dma_free_coherent(&cafe->pdev->dev,
-			2112 + sizeof(struct nand_buffers) +
-			mtd->writesize + mtd->oobsize,
-			cafe->dmabuf, cafe->dmaaddr);
+	dma_free_coherent(&cafe->pdev->dev, 2112, cafe->dmabuf, cafe->dmaaddr);
 	kfree(cafe);
 }
 
diff --git a/drivers/mtd/nand/denali.c b/drivers/mtd/nand/denali.c
index 5124f8ae8c04..313c7f50621b 100644
--- a/drivers/mtd/nand/denali.c
+++ b/drivers/mtd/nand/denali.c
@@ -330,16 +330,12 @@ static int denali_check_erased_page(struct mtd_info *mtd,
 				    unsigned long uncor_ecc_flags,
 				    unsigned int max_bitflips)
 {
-	uint8_t *ecc_code = chip->buffers->ecccode;
+	struct denali_nand_info *denali = mtd_to_denali(mtd);
+	uint8_t *ecc_code = chip->oob_poi + denali->oob_skip_bytes;
 	int ecc_steps = chip->ecc.steps;
 	int ecc_size = chip->ecc.size;
 	int ecc_bytes = chip->ecc.bytes;
-	int i, ret, stat;
-
-	ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
-					 chip->ecc.total);
-	if (ret)
-		return ret;
+	int i, stat;
 
 	for (i = 0; i < ecc_steps; i++) {
 		if (!(uncor_ecc_flags & BIT(i)))
@@ -645,8 +641,6 @@ static void denali_oob_xfer(struct mtd_info *mtd, struct nand_chip *chip,
 			    int page, int write)
 {
 	struct denali_nand_info *denali = mtd_to_denali(mtd);
-	unsigned int start_cmd = write ? NAND_CMD_SEQIN : NAND_CMD_READ0;
-	unsigned int rnd_cmd = write ? NAND_CMD_RNDIN : NAND_CMD_RNDOUT;
 	int writesize = mtd->writesize;
 	int oobsize = mtd->oobsize;
 	uint8_t *bufpoi = chip->oob_poi;
@@ -658,11 +652,11 @@ static void denali_oob_xfer(struct mtd_info *mtd, struct nand_chip *chip,
 	int i, pos, len;
 
 	/* BBM at the beginning of the OOB area */
-	chip->cmdfunc(mtd, start_cmd, writesize, page);
 	if (write)
-		chip->write_buf(mtd, bufpoi, oob_skip);
+		nand_prog_page_begin_op(chip, page, writesize, bufpoi,
+					oob_skip);
 	else
-		chip->read_buf(mtd, bufpoi, oob_skip);
+		nand_read_page_op(chip, page, writesize, bufpoi, oob_skip);
 	bufpoi += oob_skip;
 
 	/* OOB ECC */
@@ -675,30 +669,35 @@ static void denali_oob_xfer(struct mtd_info *mtd, struct nand_chip *chip,
 		else if (pos + len > writesize)
 			len = writesize - pos;
 
-		chip->cmdfunc(mtd, rnd_cmd, pos, -1);
 		if (write)
-			chip->write_buf(mtd, bufpoi, len);
+			nand_change_write_column_op(chip, pos, bufpoi, len,
+						    false);
 		else
-			chip->read_buf(mtd, bufpoi, len);
+			nand_change_read_column_op(chip, pos, bufpoi, len,
+						   false);
 		bufpoi += len;
 		if (len < ecc_bytes) {
 			len = ecc_bytes - len;
-			chip->cmdfunc(mtd, rnd_cmd, writesize + oob_skip, -1);
 			if (write)
-				chip->write_buf(mtd, bufpoi, len);
+				nand_change_write_column_op(chip, writesize +
+							    oob_skip, bufpoi,
+							    len, false);
 			else
-				chip->read_buf(mtd, bufpoi, len);
+				nand_change_read_column_op(chip, writesize +
+							   oob_skip, bufpoi,
+							   len, false);
 			bufpoi += len;
 		}
 	}
 
 	/* OOB free */
 	len = oobsize - (bufpoi - chip->oob_poi);
-	chip->cmdfunc(mtd, rnd_cmd, size - len, -1);
 	if (write)
-		chip->write_buf(mtd, bufpoi, len);
+		nand_change_write_column_op(chip, size - len, bufpoi, len,
+					    false);
 	else
-		chip->read_buf(mtd, bufpoi, len);
+		nand_change_read_column_op(chip, size - len, bufpoi, len,
+					   false);
 }
 
 static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
@@ -710,12 +709,12 @@ static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 	int ecc_steps = chip->ecc.steps;
 	int ecc_size = chip->ecc.size;
 	int ecc_bytes = chip->ecc.bytes;
-	void *dma_buf = denali->buf;
+	void *tmp_buf = denali->buf;
 	int oob_skip = denali->oob_skip_bytes;
 	size_t size = writesize + oobsize;
 	int ret, i, pos, len;
 
-	ret = denali_data_xfer(denali, dma_buf, size, page, 1, 0);
+	ret = denali_data_xfer(denali, tmp_buf, size, page, 1, 0);
 	if (ret)
 		return ret;
 
@@ -730,11 +729,11 @@ static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 			else if (pos + len > writesize)
 				len = writesize - pos;
 
-			memcpy(buf, dma_buf + pos, len);
+			memcpy(buf, tmp_buf + pos, len);
 			buf += len;
 			if (len < ecc_size) {
 				len = ecc_size - len;
-				memcpy(buf, dma_buf + writesize + oob_skip,
+				memcpy(buf, tmp_buf + writesize + oob_skip,
 				       len);
 				buf += len;
 			}
@@ -745,7 +744,7 @@ static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 		uint8_t *oob = chip->oob_poi;
 
 		/* BBM at the beginning of the OOB area */
-		memcpy(oob, dma_buf + writesize, oob_skip);
+		memcpy(oob, tmp_buf + writesize, oob_skip);
 		oob += oob_skip;
 
 		/* OOB ECC */
@@ -758,11 +757,11 @@ static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 			else if (pos + len > writesize)
 				len = writesize - pos;
 
-			memcpy(oob, dma_buf + pos, len);
+			memcpy(oob, tmp_buf + pos, len);
 			oob += len;
 			if (len < ecc_bytes) {
 				len = ecc_bytes - len;
-				memcpy(oob, dma_buf + writesize + oob_skip,
+				memcpy(oob, tmp_buf + writesize + oob_skip,
 				       len);
 				oob += len;
 			}
@@ -770,7 +769,7 @@ static int denali_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 
 		/* OOB free */
 		len = oobsize - (oob - chip->oob_poi);
-		memcpy(oob, dma_buf + size - len, len);
+		memcpy(oob, tmp_buf + size - len, len);
 	}
 
 	return 0;
@@ -788,16 +787,12 @@ static int denali_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
 			    int page)
 {
 	struct denali_nand_info *denali = mtd_to_denali(mtd);
-	int status;
 
 	denali_reset_irq(denali);
 
 	denali_oob_xfer(mtd, chip, page, 1);
 
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int denali_read_page(struct mtd_info *mtd, struct nand_chip *chip,
@@ -841,7 +836,7 @@ static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 	int ecc_steps = chip->ecc.steps;
 	int ecc_size = chip->ecc.size;
 	int ecc_bytes = chip->ecc.bytes;
-	void *dma_buf = denali->buf;
+	void *tmp_buf = denali->buf;
 	int oob_skip = denali->oob_skip_bytes;
 	size_t size = writesize + oobsize;
 	int i, pos, len;
@@ -851,7 +846,7 @@ static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 	 * This simplifies the logic.
 	 */
 	if (!buf || !oob_required)
-		memset(dma_buf, 0xff, size);
+		memset(tmp_buf, 0xff, size);
 
 	/* Arrange the buffer for syndrome payload/ecc layout */
 	if (buf) {
@@ -864,11 +859,11 @@ static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 			else if (pos + len > writesize)
 				len = writesize - pos;
 
-			memcpy(dma_buf + pos, buf, len);
+			memcpy(tmp_buf + pos, buf, len);
 			buf += len;
 			if (len < ecc_size) {
 				len = ecc_size - len;
-				memcpy(dma_buf + writesize + oob_skip, buf,
+				memcpy(tmp_buf + writesize + oob_skip, buf,
 				       len);
 				buf += len;
 			}
@@ -879,7 +874,7 @@ static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 		const uint8_t *oob = chip->oob_poi;
 
 		/* BBM at the beginning of the OOB area */
-		memcpy(dma_buf + writesize, oob, oob_skip);
+		memcpy(tmp_buf + writesize, oob, oob_skip);
 		oob += oob_skip;
 
 		/* OOB ECC */
@@ -892,11 +887,11 @@ static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 			else if (pos + len > writesize)
 				len = writesize - pos;
 
-			memcpy(dma_buf + pos, oob, len);
+			memcpy(tmp_buf + pos, oob, len);
 			oob += len;
 			if (len < ecc_bytes) {
 				len = ecc_bytes - len;
-				memcpy(dma_buf + writesize + oob_skip, oob,
+				memcpy(tmp_buf + writesize + oob_skip, oob,
 				       len);
 				oob += len;
 			}
@@ -904,10 +899,10 @@ static int denali_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 
 		/* OOB free */
 		len = oobsize - (oob - chip->oob_poi);
-		memcpy(dma_buf + size - len, oob, len);
+		memcpy(tmp_buf + size - len, oob, len);
 	}
 
-	return denali_data_xfer(denali, dma_buf, size, page, 1, 1);
+	return denali_data_xfer(denali, tmp_buf, size, page, 1, 1);
 }
 
 static int denali_write_page(struct mtd_info *mtd, struct nand_chip *chip,
@@ -951,7 +946,7 @@ static int denali_erase(struct mtd_info *mtd, int page)
 	irq_status = denali_wait_for_irq(denali,
 					 INTR__ERASE_COMP | INTR__ERASE_FAIL);
 
-	return irq_status & INTR__ERASE_COMP ? 0 : NAND_STATUS_FAIL;
+	return irq_status & INTR__ERASE_COMP ? 0 : -EIO;
 }
 
 static int denali_setup_data_interface(struct mtd_info *mtd, int chipnr,
@@ -1359,7 +1354,6 @@ int denali_init(struct denali_nand_info *denali)
 		chip->read_buf = denali_read_buf;
 		chip->write_buf = denali_write_buf;
 	}
-	chip->ecc.options |= NAND_ECC_CUSTOM_PAGE_ACCESS;
 	chip->ecc.read_page = denali_read_page;
 	chip->ecc.read_page_raw = denali_read_page_raw;
 	chip->ecc.write_page = denali_write_page;
diff --git a/drivers/mtd/nand/denali.h b/drivers/mtd/nand/denali.h
index 2911066dacac..9ad33d237378 100644
--- a/drivers/mtd/nand/denali.h
+++ b/drivers/mtd/nand/denali.h
@@ -329,7 +329,7 @@ struct denali_nand_info {
 #define DENALI_CAP_DMA_64BIT			BIT(1)
 
 int denali_calc_ecc_bytes(int step_size, int strength);
-extern int denali_init(struct denali_nand_info *denali);
-extern void denali_remove(struct denali_nand_info *denali);
+int denali_init(struct denali_nand_info *denali);
+void denali_remove(struct denali_nand_info *denali);
 
 #endif /* __DENALI_H__ */
diff --git a/drivers/mtd/nand/denali_pci.c b/drivers/mtd/nand/denali_pci.c
index 57fb7ae31412..49cb3e1f8bd0 100644
--- a/drivers/mtd/nand/denali_pci.c
+++ b/drivers/mtd/nand/denali_pci.c
@@ -125,3 +125,7 @@ static struct pci_driver denali_pci_driver = {
 	.remove = denali_pci_remove,
 };
 module_pci_driver(denali_pci_driver);
+
+MODULE_DESCRIPTION("PCI driver for Denali NAND controller");
+MODULE_AUTHOR("Intel Corporation and its suppliers");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/mtd/nand/diskonchip.c b/drivers/mtd/nand/diskonchip.c
index 72671dc52e2e..6bc93ea66f50 100644
--- a/drivers/mtd/nand/diskonchip.c
+++ b/drivers/mtd/nand/diskonchip.c
@@ -448,7 +448,7 @@ static int doc200x_wait(struct mtd_info *mtd, struct nand_chip *this)
 	int status;
 
 	DoC_WaitReady(doc);
-	this->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
+	nand_status_op(this, NULL);
 	DoC_WaitReady(doc);
 	status = (int)this->read_byte(mtd);
 
@@ -595,7 +595,7 @@ static void doc2001plus_select_chip(struct mtd_info *mtd, int chip)
 
 	/* Assert ChipEnable and deassert WriteProtect */
 	WriteDOC((DOC_FLASH_CE), docptr, Mplus_FlashSelect);
-	this->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+	nand_reset_op(this);
 
 	doc->curchip = chip;
 	doc->curfloor = floor;
diff --git a/drivers/mtd/nand/docg4.c b/drivers/mtd/nand/docg4.c
index 2436cbc71662..72f1327c4430 100644
--- a/drivers/mtd/nand/docg4.c
+++ b/drivers/mtd/nand/docg4.c
@@ -785,6 +785,8 @@ static int read_page(struct mtd_info *mtd, struct nand_chip *nand,
 
 	dev_dbg(doc->dev, "%s: page %08x\n", __func__, page);
 
+	nand_read_page_op(nand, page, 0, NULL, 0);
+
 	writew(DOC_ECCCONF0_READ_MODE |
 	       DOC_ECCCONF0_ECC_ENABLE |
 	       DOC_ECCCONF0_UNKNOWN |
@@ -864,7 +866,7 @@ static int docg4_read_oob(struct mtd_info *mtd, struct nand_chip *nand,
 
 	dev_dbg(doc->dev, "%s: page %x\n", __func__, page);
 
-	docg4_command(mtd, NAND_CMD_READ0, nand->ecc.size, page);
+	nand_read_page_op(nand, page, nand->ecc.size, NULL, 0);
 
 	writew(DOC_ECCCONF0_READ_MODE | DOCG4_OOB_SIZE, docptr + DOC_ECCCONF0);
 	write_nop(docptr);
@@ -900,6 +902,7 @@ static int docg4_erase_block(struct mtd_info *mtd, int page)
 	struct docg4_priv *doc = nand_get_controller_data(nand);
 	void __iomem *docptr = doc->virtadr;
 	uint16_t g4_page;
+	int status;
 
 	dev_dbg(doc->dev, "%s: page %04x\n", __func__, page);
 
@@ -939,11 +942,15 @@ static int docg4_erase_block(struct mtd_info *mtd, int page)
 	poll_status(doc);
 	write_nop(docptr);
 
-	return nand->waitfunc(mtd, nand);
+	status = nand->waitfunc(mtd, nand);
+	if (status < 0)
+		return status;
+
+	return status & NAND_STATUS_FAIL ? -EIO : 0;
 }
 
 static int write_page(struct mtd_info *mtd, struct nand_chip *nand,
-		       const uint8_t *buf, bool use_ecc)
+		      const uint8_t *buf, int page, bool use_ecc)
 {
 	struct docg4_priv *doc = nand_get_controller_data(nand);
 	void __iomem *docptr = doc->virtadr;
@@ -951,6 +958,8 @@ static int write_page(struct mtd_info *mtd, struct nand_chip *nand,
 
 	dev_dbg(doc->dev, "%s...\n", __func__);
 
+	nand_prog_page_begin_op(nand, page, 0, NULL, 0);
+
 	writew(DOC_ECCCONF0_ECC_ENABLE |
 	       DOC_ECCCONF0_UNKNOWN |
 	       DOCG4_BCH_SIZE,
@@ -995,19 +1004,19 @@ static int write_page(struct mtd_info *mtd, struct nand_chip *nand,
 	writew(0, docptr + DOC_DATAEND);
 	write_nop(docptr);
 
-	return 0;
+	return nand_prog_page_end_op(nand);
 }
 
 static int docg4_write_page_raw(struct mtd_info *mtd, struct nand_chip *nand,
 				const uint8_t *buf, int oob_required, int page)
 {
-	return write_page(mtd, nand, buf, false);
+	return write_page(mtd, nand, buf, page, false);
 }
 
 static int docg4_write_page(struct mtd_info *mtd, struct nand_chip *nand,
 			     const uint8_t *buf, int oob_required, int page)
 {
-	return write_page(mtd, nand, buf, true);
+	return write_page(mtd, nand, buf, page, true);
 }
 
 static int docg4_write_oob(struct mtd_info *mtd, struct nand_chip *nand,
diff --git a/drivers/mtd/nand/fsl_elbc_nand.c b/drivers/mtd/nand/fsl_elbc_nand.c
index 17db2f90aa2c..8b6dcd739ecb 100644
--- a/drivers/mtd/nand/fsl_elbc_nand.c
+++ b/drivers/mtd/nand/fsl_elbc_nand.c
@@ -713,7 +713,7 @@ static int fsl_elbc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	struct fsl_lbc_ctrl *ctrl = priv->ctrl;
 	struct fsl_elbc_fcm_ctrl *elbc_fcm_ctrl = ctrl->nand;
 
-	fsl_elbc_read_buf(mtd, buf, mtd->writesize);
+	nand_read_page_op(chip, page, 0, buf, mtd->writesize);
 	if (oob_required)
 		fsl_elbc_read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
@@ -729,10 +729,10 @@ static int fsl_elbc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 static int fsl_elbc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 				const uint8_t *buf, int oob_required, int page)
 {
-	fsl_elbc_write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	fsl_elbc_write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 /* ECC will be calculated automatically, and errors will be detected in
@@ -742,10 +742,10 @@ static int fsl_elbc_write_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 				uint32_t offset, uint32_t data_len,
 				const uint8_t *buf, int oob_required, int page)
 {
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 	fsl_elbc_write_buf(mtd, buf, mtd->writesize);
 	fsl_elbc_write_buf(mtd, chip->oob_poi, mtd->oobsize);
-
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int fsl_elbc_chip_init(struct fsl_elbc_mtd *priv)
diff --git a/drivers/mtd/nand/fsl_ifc_nand.c b/drivers/mtd/nand/fsl_ifc_nand.c
index 9e03bac7f34c..5a9c2f0020c2 100644
--- a/drivers/mtd/nand/fsl_ifc_nand.c
+++ b/drivers/mtd/nand/fsl_ifc_nand.c
@@ -173,14 +173,9 @@ static void set_addr(struct mtd_info *mtd, int column, int page_addr, int oob)
 
 /* returns nonzero if entire page is blank */
 static int check_read_ecc(struct mtd_info *mtd, struct fsl_ifc_ctrl *ctrl,
-			  u32 *eccstat, unsigned int bufnum)
+			  u32 eccstat, unsigned int bufnum)
 {
-	u32 reg = eccstat[bufnum / 4];
-	int errors;
-
-	errors = (reg >> ((3 - bufnum % 4) * 8)) & 15;
-
-	return errors;
+	return  (eccstat >> ((3 - bufnum % 4) * 8)) & 15;
 }
 
 /*
@@ -193,7 +188,7 @@ static void fsl_ifc_run_command(struct mtd_info *mtd)
 	struct fsl_ifc_ctrl *ctrl = priv->ctrl;
 	struct fsl_ifc_nand_ctrl *nctrl = ifc_nand_ctrl;
 	struct fsl_ifc_runtime __iomem *ifc = ctrl->rregs;
-	u32 eccstat[4];
+	u32 eccstat;
 	int i;
 
 	/* set the chip select for NAND Transaction */
@@ -228,19 +223,17 @@ static void fsl_ifc_run_command(struct mtd_info *mtd)
 	if (nctrl->eccread) {
 		int errors;
 		int bufnum = nctrl->page & priv->bufnum_mask;
-		int sector = bufnum * chip->ecc.steps;
-		int sector_end = sector + chip->ecc.steps - 1;
+		int sector_start = bufnum * chip->ecc.steps;
+		int sector_end = sector_start + chip->ecc.steps - 1;
 		__be32 *eccstat_regs;
 
-		if (ctrl->version >= FSL_IFC_VERSION_2_0_0)
-			eccstat_regs = ifc->ifc_nand.v2_nand_eccstat;
-		else
-			eccstat_regs = ifc->ifc_nand.v1_nand_eccstat;
+		eccstat_regs = ifc->ifc_nand.nand_eccstat;
+		eccstat = ifc_in32(&eccstat_regs[sector_start / 4]);
 
-		for (i = sector / 4; i <= sector_end / 4; i++)
-			eccstat[i] = ifc_in32(&eccstat_regs[i]);
+		for (i = sector_start; i <= sector_end; i++) {
+			if (i != sector_start && !(i % 4))
+				eccstat = ifc_in32(&eccstat_regs[i / 4]);
 
-		for (i = sector; i <= sector_end; i++) {
 			errors = check_read_ecc(mtd, ctrl, eccstat, i);
 
 			if (errors == 15) {
@@ -626,6 +619,7 @@ static int fsl_ifc_wait(struct mtd_info *mtd, struct nand_chip *chip)
 	struct fsl_ifc_ctrl *ctrl = priv->ctrl;
 	struct fsl_ifc_runtime __iomem *ifc = ctrl->rregs;
 	u32 nand_fsr;
+	int status;
 
 	/* Use READ_STATUS command, but wait for the device to be ready */
 	ifc_out32((IFC_FIR_OP_CW0 << IFC_NAND_FIR0_OP0_SHIFT) |
@@ -640,12 +634,12 @@ static int fsl_ifc_wait(struct mtd_info *mtd, struct nand_chip *chip)
 	fsl_ifc_run_command(mtd);
 
 	nand_fsr = ifc_in32(&ifc->ifc_nand.nand_fsr);
-
+	status = nand_fsr >> 24;
 	/*
 	 * The chip always seems to report that it is
 	 * write-protected, even when it is not.
 	 */
-	return nand_fsr | NAND_STATUS_WP;
+	return status | NAND_STATUS_WP;
 }
 
 /*
@@ -688,7 +682,7 @@ static int fsl_ifc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	struct fsl_ifc_ctrl *ctrl = priv->ctrl;
 	struct fsl_ifc_nand_ctrl *nctrl = ifc_nand_ctrl;
 
-	fsl_ifc_read_buf(mtd, buf, mtd->writesize);
+	nand_read_page_op(chip, page, 0, buf, mtd->writesize);
 	if (oob_required)
 		fsl_ifc_read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
@@ -711,10 +705,10 @@ static int fsl_ifc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 static int fsl_ifc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 			       const uint8_t *buf, int oob_required, int page)
 {
-	fsl_ifc_write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	fsl_ifc_write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int fsl_ifc_chip_init_tail(struct mtd_info *mtd)
@@ -916,6 +910,13 @@ static int fsl_ifc_chip_init(struct fsl_ifc_mtd *priv)
 	if (ctrl->version >= FSL_IFC_VERSION_1_1_0)
 		fsl_ifc_sram_init(priv);
 
+	/*
+	 * As IFC version 2.0.0 has 16KB of internal SRAM as compared to older
+	 * versions which had 8KB. Hence bufnum mask needs to be updated.
+	 */
+	if (ctrl->version >= FSL_IFC_VERSION_2_0_0)
+		priv->bufnum_mask = (priv->bufnum_mask * 2) + 1;
+
 	return 0;
 }
 
diff --git a/drivers/mtd/nand/fsmc_nand.c b/drivers/mtd/nand/fsmc_nand.c
index eac15d9bf49e..f49ed46fa770 100644
--- a/drivers/mtd/nand/fsmc_nand.c
+++ b/drivers/mtd/nand/fsmc_nand.c
@@ -684,8 +684,8 @@ static int fsmc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
 	uint8_t *p = buf;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
-	uint8_t *ecc_code = chip->buffers->ecccode;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
+	uint8_t *ecc_code = chip->ecc.code_buf;
 	int off, len, group = 0;
 	/*
 	 * ecc_oob is intentionally taken as uint16_t. In 16bit devices, we
@@ -697,7 +697,7 @@ static int fsmc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 	unsigned int max_bitflips = 0;
 
 	for (i = 0, s = 0; s < eccsteps; s++, i += eccbytes, p += eccsize) {
-		chip->cmdfunc(mtd, NAND_CMD_READ0, s * eccsize, page);
+		nand_read_page_op(chip, page, s * eccsize, NULL, 0);
 		chip->ecc.hwctl(mtd, NAND_ECC_READ);
 		chip->read_buf(mtd, p, eccsize);
 
@@ -720,8 +720,7 @@ static int fsmc_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 			if (chip->options & NAND_BUSWIDTH_16)
 				len = roundup(len, 2);
 
-			chip->cmdfunc(mtd, NAND_CMD_READOOB, off, page);
-			chip->read_buf(mtd, oob + j, len);
+			nand_read_oob_op(chip, page, off, oob + j, len);
 			j += len;
 		}
 
diff --git a/drivers/mtd/nand/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/gpmi-nand/gpmi-nand.c
index d4d824ef64e9..61fdd733492f 100644
--- a/drivers/mtd/nand/gpmi-nand/gpmi-nand.c
+++ b/drivers/mtd/nand/gpmi-nand/gpmi-nand.c
@@ -1029,11 +1029,13 @@ static void block_mark_swapping(struct gpmi_nand_data *this,
 	p[1] = (p[1] & mask) | (from_oob >> (8 - bit));
 }
 
-static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
-				uint8_t *buf, int oob_required, int page)
+static int gpmi_ecc_read_page_data(struct nand_chip *chip,
+				   uint8_t *buf, int oob_required,
+				   int page)
 {
 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
 	struct bch_geometry *nfc_geo = &this->bch_geometry;
+	struct mtd_info *mtd = nand_to_mtd(chip);
 	void          *payload_virt;
 	dma_addr_t    payload_phys;
 	void          *auxiliary_virt;
@@ -1094,8 +1096,8 @@ static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 			eccbytes = DIV_ROUND_UP(offset + eccbits, 8);
 			offset /= 8;
 			eccbytes -= offset;
-			chip->cmdfunc(mtd, NAND_CMD_RNDOUT, offset, -1);
-			chip->read_buf(mtd, eccbuf, eccbytes);
+			nand_change_read_column_op(chip, offset, eccbuf,
+						   eccbytes, false);
 
 			/*
 			 * ECC data are not byte aligned and we may have
@@ -1176,6 +1178,14 @@ static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	return max_bitflips;
 }
 
+static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
+			      uint8_t *buf, int oob_required, int page)
+{
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
+	return gpmi_ecc_read_page_data(chip, buf, oob_required, page);
+}
+
 /* Fake a virtual small page for the subpage read */
 static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 			uint32_t offs, uint32_t len, uint8_t *buf, int page)
@@ -1220,12 +1230,12 @@ static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 	meta = geo->metadata_size;
 	if (first) {
 		col = meta + (size + ecc_parity_size) * first;
-		chip->cmdfunc(mtd, NAND_CMD_RNDOUT, col, -1);
-
 		meta = 0;
 		buf = buf + first * size;
 	}
 
+	nand_read_page_op(chip, page, col, NULL, 0);
+
 	/* Save the old environment */
 	r1_old = r1_new = readl(bch_regs + HW_BCH_FLASH0LAYOUT0);
 	r2_old = r2_new = readl(bch_regs + HW_BCH_FLASH0LAYOUT1);
@@ -1254,7 +1264,7 @@ static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 
 	/* Read the subpage now */
 	this->swap_block_mark = false;
-	max_bitflips = gpmi_ecc_read_page(mtd, chip, buf, 0, page);
+	max_bitflips = gpmi_ecc_read_page_data(chip, buf, 0, page);
 
 	/* Restore */
 	writel(r1_old, bch_regs + HW_BCH_FLASH0LAYOUT0);
@@ -1277,6 +1287,9 @@ static int gpmi_ecc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	int        ret;
 
 	dev_dbg(this->dev, "ecc write page.\n");
+
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	if (this->swap_block_mark) {
 		/*
 		 * If control arrives here, we're doing block mark swapping.
@@ -1338,7 +1351,10 @@ static int gpmi_ecc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 				payload_virt, payload_phys);
 	}
 
-	return 0;
+	if (ret)
+		return ret;
+
+	return nand_prog_page_end_op(chip);
 }
 
 /*
@@ -1411,7 +1427,7 @@ static int gpmi_ecc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 	memset(chip->oob_poi, ~0, mtd->oobsize);
 
 	/* Read out the conventional OOB. */
-	chip->cmdfunc(mtd, NAND_CMD_READ0, mtd->writesize, page);
+	nand_read_page_op(chip, page, mtd->writesize, NULL, 0);
 	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
 	/*
@@ -1421,7 +1437,7 @@ static int gpmi_ecc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 	 */
 	if (GPMI_IS_MX23(this)) {
 		/* Read the block mark into the first byte of the OOB buffer. */
-		chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
+		nand_read_page_op(chip, page, 0, NULL, 0);
 		chip->oob_poi[0] = chip->read_byte(mtd);
 	}
 
@@ -1432,7 +1448,6 @@ static int
 gpmi_ecc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, int page)
 {
 	struct mtd_oob_region of = { };
-	int status = 0;
 
 	/* Do we have available oob area? */
 	mtd_ooblayout_free(mtd, 0, &of);
@@ -1442,12 +1457,8 @@ gpmi_ecc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, int page)
 	if (!nand_is_slc(chip))
 		return -EPERM;
 
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize + of.offset, page);
-	chip->write_buf(mtd, chip->oob_poi + of.offset, of.length);
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_op(chip, page, mtd->writesize + of.offset,
+				 chip->oob_poi + of.offset, of.length);
 }
 
 /*
@@ -1477,8 +1488,8 @@ static int gpmi_ecc_read_page_raw(struct mtd_info *mtd,
 	uint8_t *oob = chip->oob_poi;
 	int step;
 
-	chip->read_buf(mtd, tmp_buf,
-		       mtd->writesize + mtd->oobsize);
+	nand_read_page_op(chip, page, 0, tmp_buf,
+			  mtd->writesize + mtd->oobsize);
 
 	/*
 	 * If required, swap the bad block marker and the data stored in the
@@ -1487,12 +1498,8 @@ static int gpmi_ecc_read_page_raw(struct mtd_info *mtd,
 	 * See the layout description for a detailed explanation on why this
 	 * is needed.
 	 */
-	if (this->swap_block_mark) {
-		u8 swap = tmp_buf[0];
-
-		tmp_buf[0] = tmp_buf[mtd->writesize];
-		tmp_buf[mtd->writesize] = swap;
-	}
+	if (this->swap_block_mark)
+		swap(tmp_buf[0], tmp_buf[mtd->writesize]);
 
 	/*
 	 * Copy the metadata section into the oob buffer (this section is
@@ -1615,31 +1622,22 @@ static int gpmi_ecc_write_page_raw(struct mtd_info *mtd,
 	 * See the layout description for a detailed explanation on why this
 	 * is needed.
 	 */
-	if (this->swap_block_mark) {
-		u8 swap = tmp_buf[0];
-
-		tmp_buf[0] = tmp_buf[mtd->writesize];
-		tmp_buf[mtd->writesize] = swap;
-	}
+	if (this->swap_block_mark)
+		swap(tmp_buf[0], tmp_buf[mtd->writesize]);
 
-	chip->write_buf(mtd, tmp_buf, mtd->writesize + mtd->oobsize);
-
-	return 0;
+	return nand_prog_page_op(chip, page, 0, tmp_buf,
+				 mtd->writesize + mtd->oobsize);
 }
 
 static int gpmi_ecc_read_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
 				 int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
-
 	return gpmi_ecc_read_page_raw(mtd, chip, NULL, 1, page);
 }
 
 static int gpmi_ecc_write_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
 				 int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
-
 	return gpmi_ecc_write_page_raw(mtd, chip, NULL, 1, page);
 }
 
@@ -1649,7 +1647,7 @@ static int gpmi_block_markbad(struct mtd_info *mtd, loff_t ofs)
 	struct gpmi_nand_data *this = nand_get_controller_data(chip);
 	int ret = 0;
 	uint8_t *block_mark;
-	int column, page, status, chipnr;
+	int column, page, chipnr;
 
 	chipnr = (int)(ofs >> chip->chip_shift);
 	chip->select_chip(mtd, chipnr);
@@ -1663,13 +1661,7 @@ static int gpmi_block_markbad(struct mtd_info *mtd, loff_t ofs)
 	/* Shift to get page */
 	page = (int)(ofs >> chip->page_shift);
 
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, column, page);
-	chip->write_buf(mtd, block_mark, 1);
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-	if (status & NAND_STATUS_FAIL)
-		ret = -EIO;
+	ret = nand_prog_page_op(chip, page, column, block_mark, 1);
 
 	chip->select_chip(mtd, -1);
 
@@ -1712,7 +1704,7 @@ static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)
 	unsigned int search_area_size_in_strides;
 	unsigned int stride;
 	unsigned int page;
-	uint8_t *buffer = chip->buffers->databuf;
+	uint8_t *buffer = chip->data_buf;
 	int saved_chip_number;
 	int found_an_ncb_fingerprint = false;
 
@@ -1737,7 +1729,7 @@ static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)
 		 * Read the NCB fingerprint. The fingerprint is four bytes long
 		 * and starts in the 12th byte of the page.
 		 */
-		chip->cmdfunc(mtd, NAND_CMD_READ0, 12, page);
+		nand_read_page_op(chip, page, 12, NULL, 0);
 		chip->read_buf(mtd, buffer, strlen(fingerprint));
 
 		/* Look for the fingerprint. */
@@ -1771,7 +1763,7 @@ static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
 	unsigned int block;
 	unsigned int stride;
 	unsigned int page;
-	uint8_t      *buffer = chip->buffers->databuf;
+	uint8_t      *buffer = chip->data_buf;
 	int saved_chip_number;
 	int status;
 
@@ -1797,17 +1789,10 @@ static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
 	dev_dbg(dev, "Erasing the search area...\n");
 
 	for (block = 0; block < search_area_size_in_blocks; block++) {
-		/* Compute the page address. */
-		page = block * block_size_in_pages;
-
 		/* Erase this block. */
 		dev_dbg(dev, "\tErasing block 0x%x\n", block);
-		chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page);
-		chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1);
-
-		/* Wait for the erase to finish. */
-		status = chip->waitfunc(mtd, chip);
-		if (status & NAND_STATUS_FAIL)
+		status = nand_erase_op(chip, block);
+		if (status)
 			dev_err(dev, "[%s] Erase failed.\n", __func__);
 	}
 
@@ -1823,13 +1808,9 @@ static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
 
 		/* Write the first page of the current stride. */
 		dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page);
-		chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
-		chip->ecc.write_page_raw(mtd, chip, buffer, 0, page);
-		chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
 
-		/* Wait for the write to finish. */
-		status = chip->waitfunc(mtd, chip);
-		if (status & NAND_STATUS_FAIL)
+		status = chip->ecc.write_page_raw(mtd, chip, buffer, 0, page);
+		if (status)
 			dev_err(dev, "[%s] Write failed.\n", __func__);
 	}
 
@@ -1884,7 +1865,7 @@ static int mx23_boot_init(struct gpmi_nand_data  *this)
 
 		/* Send the command to read the conventional block mark. */
 		chip->select_chip(mtd, chipnr);
-		chip->cmdfunc(mtd, NAND_CMD_READ0, mtd->writesize, page);
+		nand_read_page_op(chip, page, mtd->writesize, NULL, 0);
 		block_mark = chip->read_byte(mtd);
 		chip->select_chip(mtd, -1);
 
diff --git a/drivers/mtd/nand/gpmi-nand/gpmi-nand.h b/drivers/mtd/nand/gpmi-nand/gpmi-nand.h
index a45e4ce13d10..06c1f993912c 100644
--- a/drivers/mtd/nand/gpmi-nand/gpmi-nand.h
+++ b/drivers/mtd/nand/gpmi-nand/gpmi-nand.h
@@ -268,31 +268,31 @@ struct timing_threshold {
 };
 
 /* Common Services */
-extern int common_nfc_set_geometry(struct gpmi_nand_data *);
-extern struct dma_chan *get_dma_chan(struct gpmi_nand_data *);
-extern void prepare_data_dma(struct gpmi_nand_data *,
-				enum dma_data_direction dr);
-extern int start_dma_without_bch_irq(struct gpmi_nand_data *,
-				struct dma_async_tx_descriptor *);
-extern int start_dma_with_bch_irq(struct gpmi_nand_data *,
-				struct dma_async_tx_descriptor *);
+int common_nfc_set_geometry(struct gpmi_nand_data *);
+struct dma_chan *get_dma_chan(struct gpmi_nand_data *);
+void prepare_data_dma(struct gpmi_nand_data *,
+		      enum dma_data_direction dr);
+int start_dma_without_bch_irq(struct gpmi_nand_data *,
+			      struct dma_async_tx_descriptor *);
+int start_dma_with_bch_irq(struct gpmi_nand_data *,
+			   struct dma_async_tx_descriptor *);
 
 /* GPMI-NAND helper function library */
-extern int gpmi_init(struct gpmi_nand_data *);
-extern int gpmi_extra_init(struct gpmi_nand_data *);
-extern void gpmi_clear_bch(struct gpmi_nand_data *);
-extern void gpmi_dump_info(struct gpmi_nand_data *);
-extern int bch_set_geometry(struct gpmi_nand_data *);
-extern int gpmi_is_ready(struct gpmi_nand_data *, unsigned chip);
-extern int gpmi_send_command(struct gpmi_nand_data *);
-extern void gpmi_begin(struct gpmi_nand_data *);
-extern void gpmi_end(struct gpmi_nand_data *);
-extern int gpmi_read_data(struct gpmi_nand_data *);
-extern int gpmi_send_data(struct gpmi_nand_data *);
-extern int gpmi_send_page(struct gpmi_nand_data *,
-			dma_addr_t payload, dma_addr_t auxiliary);
-extern int gpmi_read_page(struct gpmi_nand_data *,
-			dma_addr_t payload, dma_addr_t auxiliary);
+int gpmi_init(struct gpmi_nand_data *);
+int gpmi_extra_init(struct gpmi_nand_data *);
+void gpmi_clear_bch(struct gpmi_nand_data *);
+void gpmi_dump_info(struct gpmi_nand_data *);
+int bch_set_geometry(struct gpmi_nand_data *);
+int gpmi_is_ready(struct gpmi_nand_data *, unsigned chip);
+int gpmi_send_command(struct gpmi_nand_data *);
+void gpmi_begin(struct gpmi_nand_data *);
+void gpmi_end(struct gpmi_nand_data *);
+int gpmi_read_data(struct gpmi_nand_data *);
+int gpmi_send_data(struct gpmi_nand_data *);
+int gpmi_send_page(struct gpmi_nand_data *,
+		   dma_addr_t payload, dma_addr_t auxiliary);
+int gpmi_read_page(struct gpmi_nand_data *,
+		   dma_addr_t payload, dma_addr_t auxiliary);
 
 void gpmi_copy_bits(u8 *dst, size_t dst_bit_off,
 		    const u8 *src, size_t src_bit_off,
diff --git a/drivers/mtd/nand/hisi504_nand.c b/drivers/mtd/nand/hisi504_nand.c
index 0897261c3e17..cb862793ab6d 100644
--- a/drivers/mtd/nand/hisi504_nand.c
+++ b/drivers/mtd/nand/hisi504_nand.c
@@ -544,7 +544,7 @@ static int hisi_nand_read_page_hwecc(struct mtd_info *mtd,
 	int max_bitflips = 0, stat = 0, stat_max = 0, status_ecc;
 	int stat_1, stat_2;
 
-	chip->read_buf(mtd, buf, mtd->writesize);
+	nand_read_page_op(chip, page, 0, buf, mtd->writesize);
 	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
 	/* errors which can not be corrected by ECC */
@@ -574,8 +574,7 @@ static int hisi_nand_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 {
 	struct hinfc_host *host = nand_get_controller_data(chip);
 
-	chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
-	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
+	nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
 
 	if (host->irq_status & HINFC504_INTS_UE) {
 		host->irq_status = 0;
@@ -590,11 +589,11 @@ static int hisi_nand_write_page_hwecc(struct mtd_info *mtd,
 		struct nand_chip *chip, const uint8_t *buf, int oob_required,
 		int page)
 {
-	chip->write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	if (oob_required)
 		chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static void hisi_nfc_host_init(struct hinfc_host *host)
diff --git a/drivers/mtd/nand/jz4740_nand.c b/drivers/mtd/nand/jz4740_nand.c
index ad827d4af3e9..613b00a9604b 100644
--- a/drivers/mtd/nand/jz4740_nand.c
+++ b/drivers/mtd/nand/jz4740_nand.c
@@ -313,6 +313,7 @@ static int jz_nand_detect_bank(struct platform_device *pdev,
 	uint32_t ctrl;
 	struct nand_chip *chip = &nand->chip;
 	struct mtd_info *mtd = nand_to_mtd(chip);
+	u8 id[2];
 
 	/* Request I/O resource. */
 	sprintf(res_name, "bank%d", bank);
@@ -335,17 +336,16 @@ static int jz_nand_detect_bank(struct platform_device *pdev,
 
 		/* Retrieve the IDs from the first chip. */
 		chip->select_chip(mtd, 0);
-		chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
-		chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
-		*nand_maf_id = chip->read_byte(mtd);
-		*nand_dev_id = chip->read_byte(mtd);
+		nand_reset_op(chip);
+		nand_readid_op(chip, 0, id, sizeof(id));
+		*nand_maf_id = id[0];
+		*nand_dev_id = id[1];
 	} else {
 		/* Detect additional chip. */
 		chip->select_chip(mtd, chipnr);
-		chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
-		chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
-		if (*nand_maf_id != chip->read_byte(mtd)
-		 || *nand_dev_id != chip->read_byte(mtd)) {
+		nand_reset_op(chip);
+		nand_readid_op(chip, 0, id, sizeof(id));
+		if (*nand_maf_id != id[0] || *nand_dev_id != id[1]) {
 			ret = -ENODEV;
 			goto notfound_id;
 		}
diff --git a/drivers/mtd/nand/lpc32xx_mlc.c b/drivers/mtd/nand/lpc32xx_mlc.c
index 5796468db653..e357948a7505 100644
--- a/drivers/mtd/nand/lpc32xx_mlc.c
+++ b/drivers/mtd/nand/lpc32xx_mlc.c
@@ -461,7 +461,7 @@ static int lpc32xx_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	}
 
 	/* Writing Command and Address */
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
+	nand_read_page_op(chip, page, 0, NULL, 0);
 
 	/* For all sub-pages */
 	for (i = 0; i < host->mlcsubpages; i++) {
@@ -522,6 +522,8 @@ static int lpc32xx_write_page_lowlevel(struct mtd_info *mtd,
 		memcpy(dma_buf, buf, mtd->writesize);
 	}
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	for (i = 0; i < host->mlcsubpages; i++) {
 		/* Start Encode */
 		writeb(0x00, MLC_ECC_ENC_REG(host->io_base));
@@ -550,7 +552,8 @@ static int lpc32xx_write_page_lowlevel(struct mtd_info *mtd,
 		/* Wait for Controller Ready */
 		lpc32xx_waitfunc_controller(mtd, chip);
 	}
-	return 0;
+
+	return nand_prog_page_end_op(chip);
 }
 
 static int lpc32xx_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
diff --git a/drivers/mtd/nand/lpc32xx_slc.c b/drivers/mtd/nand/lpc32xx_slc.c
index b61f28a1554d..5f7cc6da0a7f 100644
--- a/drivers/mtd/nand/lpc32xx_slc.c
+++ b/drivers/mtd/nand/lpc32xx_slc.c
@@ -399,10 +399,7 @@ static void lpc32xx_nand_write_buf(struct mtd_info *mtd, const uint8_t *buf, int
 static int lpc32xx_nand_read_oob_syndrome(struct mtd_info *mtd,
 					  struct nand_chip *chip, int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
-	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
-
-	return 0;
+	return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
 }
 
 /*
@@ -411,17 +408,8 @@ static int lpc32xx_nand_read_oob_syndrome(struct mtd_info *mtd,
 static int lpc32xx_nand_write_oob_syndrome(struct mtd_info *mtd,
 	struct nand_chip *chip, int page)
 {
-	int status;
-
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize, page);
-	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
-
-	/* Send command to program the OOB data */
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,
+				 mtd->oobsize);
 }
 
 /*
@@ -632,7 +620,7 @@ static int lpc32xx_nand_read_page_syndrome(struct mtd_info *mtd,
 	uint8_t *oobecc, tmpecc[LPC32XX_ECC_SAVE_SIZE];
 
 	/* Issue read command */
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
+	nand_read_page_op(chip, page, 0, NULL, 0);
 
 	/* Read data and oob, calculate ECC */
 	status = lpc32xx_xfer(mtd, buf, chip->ecc.steps, 1);
@@ -675,7 +663,7 @@ static int lpc32xx_nand_read_page_raw_syndrome(struct mtd_info *mtd,
 					       int page)
 {
 	/* Issue read command */
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
+	nand_read_page_op(chip, page, 0, NULL, 0);
 
 	/* Raw reads can just use the FIFO interface */
 	chip->read_buf(mtd, buf, chip->ecc.size * chip->ecc.steps);
@@ -698,6 +686,8 @@ static int lpc32xx_nand_write_page_syndrome(struct mtd_info *mtd,
 	uint8_t *pb;
 	int error;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	/* Write data, calculate ECC on outbound data */
 	error = lpc32xx_xfer(mtd, (uint8_t *)buf, chip->ecc.steps, 0);
 	if (error)
@@ -716,7 +706,8 @@ static int lpc32xx_nand_write_page_syndrome(struct mtd_info *mtd,
 
 	/* Write ECC data to device */
 	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
-	return 0;
+
+	return nand_prog_page_end_op(chip);
 }
 
 /*
@@ -729,9 +720,11 @@ static int lpc32xx_nand_write_page_raw_syndrome(struct mtd_info *mtd,
 						int oob_required, int page)
 {
 	/* Raw writes can just use the FIFO interface */
-	chip->write_buf(mtd, buf, chip->ecc.size * chip->ecc.steps);
+	nand_prog_page_begin_op(chip, page, 0, buf,
+				chip->ecc.size * chip->ecc.steps);
 	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
-	return 0;
+
+	return nand_prog_page_end_op(chip);
 }
 
 static int lpc32xx_nand_dma_setup(struct lpc32xx_nand_host *host)
diff --git a/drivers/mtd/nand/marvell_nand.c b/drivers/mtd/nand/marvell_nand.c
new file mode 100644
index 000000000000..2196f2a233d6
--- /dev/null
+++ b/drivers/mtd/nand/marvell_nand.c
@@ -0,0 +1,2896 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Marvell NAND flash controller driver
+ *
+ * Copyright (C) 2017 Marvell
+ * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/clk.h>
+#include <linux/mtd/rawnand.h>
+#include <linux/of_platform.h>
+#include <linux/iopoll.h>
+#include <linux/interrupt.h>
+#include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+#include <asm/unaligned.h>
+
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma/pxa-dma.h>
+#include <linux/platform_data/mtd-nand-pxa3xx.h>
+
+/* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
+#define FIFO_DEPTH		8
+#define FIFO_REP(x)		(x / sizeof(u32))
+#define BCH_SEQ_READS		(32 / FIFO_DEPTH)
+/* NFC does not support transfers of larger chunks at a time */
+#define MAX_CHUNK_SIZE		2112
+/* NFCv1 cannot read more that 7 bytes of ID */
+#define NFCV1_READID_LEN	7
+/* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
+#define POLL_PERIOD		0
+#define POLL_TIMEOUT		100000
+/* Interrupt maximum wait period in ms */
+#define IRQ_TIMEOUT		1000
+/* Latency in clock cycles between SoC pins and NFC logic */
+#define MIN_RD_DEL_CNT		3
+/* Maximum number of contiguous address cycles */
+#define MAX_ADDRESS_CYC_NFCV1	5
+#define MAX_ADDRESS_CYC_NFCV2	7
+/* System control registers/bits to enable the NAND controller on some SoCs */
+#define GENCONF_SOC_DEVICE_MUX	0x208
+#define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
+#define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
+#define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
+#define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
+#define GENCONF_CLK_GATING_CTRL	0x220
+#define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
+#define GENCONF_ND_CLK_CTRL	0x700
+#define GENCONF_ND_CLK_CTRL_EN	BIT(0)
+
+/* NAND controller data flash control register */
+#define NDCR			0x00
+#define NDCR_ALL_INT		GENMASK(11, 0)
+#define NDCR_CS1_CMDDM		BIT(7)
+#define NDCR_CS0_CMDDM		BIT(8)
+#define NDCR_RDYM		BIT(11)
+#define NDCR_ND_ARB_EN		BIT(12)
+#define NDCR_RA_START		BIT(15)
+#define NDCR_RD_ID_CNT(x)	(min_t(unsigned int, x, 0x7) << 16)
+#define NDCR_PAGE_SZ(x)		(x >= 2048 ? BIT(24) : 0)
+#define NDCR_DWIDTH_M		BIT(26)
+#define NDCR_DWIDTH_C		BIT(27)
+#define NDCR_ND_RUN		BIT(28)
+#define NDCR_DMA_EN		BIT(29)
+#define NDCR_ECC_EN		BIT(30)
+#define NDCR_SPARE_EN		BIT(31)
+#define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
+				    NDCR_DWIDTH_M | NDCR_DWIDTH_C))
+
+/* NAND interface timing parameter 0 register */
+#define NDTR0			0x04
+#define NDTR0_TRP(x)		((min_t(unsigned int, x, 0xF) & 0x7) << 0)
+#define NDTR0_TRH(x)		(min_t(unsigned int, x, 0x7) << 3)
+#define NDTR0_ETRP(x)		((min_t(unsigned int, x, 0xF) & 0x8) << 3)
+#define NDTR0_SEL_NRE_EDGE	BIT(7)
+#define NDTR0_TWP(x)		(min_t(unsigned int, x, 0x7) << 8)
+#define NDTR0_TWH(x)		(min_t(unsigned int, x, 0x7) << 11)
+#define NDTR0_TCS(x)		(min_t(unsigned int, x, 0x7) << 16)
+#define NDTR0_TCH(x)		(min_t(unsigned int, x, 0x7) << 19)
+#define NDTR0_RD_CNT_DEL(x)	(min_t(unsigned int, x, 0xF) << 22)
+#define NDTR0_SELCNTR		BIT(26)
+#define NDTR0_TADL(x)		(min_t(unsigned int, x, 0x1F) << 27)
+
+/* NAND interface timing parameter 1 register */
+#define NDTR1			0x0C
+#define NDTR1_TAR(x)		(min_t(unsigned int, x, 0xF) << 0)
+#define NDTR1_TWHR(x)		(min_t(unsigned int, x, 0xF) << 4)
+#define NDTR1_TRHW(x)		(min_t(unsigned int, x / 16, 0x3) << 8)
+#define NDTR1_PRESCALE		BIT(14)
+#define NDTR1_WAIT_MODE		BIT(15)
+#define NDTR1_TR(x)		(min_t(unsigned int, x, 0xFFFF) << 16)
+
+/* NAND controller status register */
+#define NDSR			0x14
+#define NDSR_WRCMDREQ		BIT(0)
+#define NDSR_RDDREQ		BIT(1)
+#define NDSR_WRDREQ		BIT(2)
+#define NDSR_CORERR		BIT(3)
+#define NDSR_UNCERR		BIT(4)
+#define NDSR_CMDD(cs)		BIT(8 - cs)
+#define NDSR_RDY(rb)		BIT(11 + rb)
+#define NDSR_ERRCNT(x)		((x >> 16) & 0x1F)
+
+/* NAND ECC control register */
+#define NDECCCTRL		0x28
+#define NDECCCTRL_BCH_EN	BIT(0)
+
+/* NAND controller data buffer register */
+#define NDDB			0x40
+
+/* NAND controller command buffer 0 register */
+#define NDCB0			0x48
+#define NDCB0_CMD1(x)		((x & 0xFF) << 0)
+#define NDCB0_CMD2(x)		((x & 0xFF) << 8)
+#define NDCB0_ADDR_CYC(x)	((x & 0x7) << 16)
+#define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
+#define NDCB0_DBC		BIT(19)
+#define NDCB0_CMD_TYPE(x)	((x & 0x7) << 21)
+#define NDCB0_CSEL		BIT(24)
+#define NDCB0_RDY_BYP		BIT(27)
+#define NDCB0_LEN_OVRD		BIT(28)
+#define NDCB0_CMD_XTYPE(x)	((x & 0x7) << 29)
+
+/* NAND controller command buffer 1 register */
+#define NDCB1			0x4C
+#define NDCB1_COLS(x)		((x & 0xFFFF) << 0)
+#define NDCB1_ADDRS_PAGE(x)	(x << 16)
+
+/* NAND controller command buffer 2 register */
+#define NDCB2			0x50
+#define NDCB2_ADDR5_PAGE(x)	(((x >> 16) & 0xFF) << 0)
+#define NDCB2_ADDR5_CYC(x)	((x & 0xFF) << 0)
+
+/* NAND controller command buffer 3 register */
+#define NDCB3			0x54
+#define NDCB3_ADDR6_CYC(x)	((x & 0xFF) << 16)
+#define NDCB3_ADDR7_CYC(x)	((x & 0xFF) << 24)
+
+/* NAND controller command buffer 0 register 'type' and 'xtype' fields */
+#define TYPE_READ		0
+#define TYPE_WRITE		1
+#define TYPE_ERASE		2
+#define TYPE_READ_ID		3
+#define TYPE_STATUS		4
+#define TYPE_RESET		5
+#define TYPE_NAKED_CMD		6
+#define TYPE_NAKED_ADDR		7
+#define TYPE_MASK		7
+#define XTYPE_MONOLITHIC_RW	0
+#define XTYPE_LAST_NAKED_RW	1
+#define XTYPE_FINAL_COMMAND	3
+#define XTYPE_READ		4
+#define XTYPE_WRITE_DISPATCH	4
+#define XTYPE_NAKED_RW		5
+#define XTYPE_COMMAND_DISPATCH	6
+#define XTYPE_MASK		7
+
+/**
+ * Marvell ECC engine works differently than the others, in order to limit the
+ * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
+ * per subpage, and depending on a the desired strength needed by the NAND chip,
+ * a particular layout mixing data/spare/ecc is defined, with a possible last
+ * chunk smaller that the others.
+ *
+ * @writesize:		Full page size on which the layout applies
+ * @chunk:		Desired ECC chunk size on which the layout applies
+ * @strength:		Desired ECC strength (per chunk size bytes) on which the
+ *			layout applies
+ * @nchunks:		Total number of chunks
+ * @full_chunk_cnt:	Number of full-sized chunks, which is the number of
+ *			repetitions of the pattern:
+ *			(data_bytes + spare_bytes + ecc_bytes).
+ * @data_bytes:		Number of data bytes per chunk
+ * @spare_bytes:	Number of spare bytes per chunk
+ * @ecc_bytes:		Number of ecc bytes per chunk
+ * @last_data_bytes:	Number of data bytes in the last chunk
+ * @last_spare_bytes:	Number of spare bytes in the last chunk
+ * @last_ecc_bytes:	Number of ecc bytes in the last chunk
+ */
+struct marvell_hw_ecc_layout {
+	/* Constraints */
+	int writesize;
+	int chunk;
+	int strength;
+	/* Corresponding layout */
+	int nchunks;
+	int full_chunk_cnt;
+	int data_bytes;
+	int spare_bytes;
+	int ecc_bytes;
+	int last_data_bytes;
+	int last_spare_bytes;
+	int last_ecc_bytes;
+};
+
+#define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb)	\
+	{								\
+		.writesize = ws,					\
+		.chunk = dc,						\
+		.strength = ds,						\
+		.nchunks = nc,						\
+		.full_chunk_cnt = fcc,					\
+		.data_bytes = db,					\
+		.spare_bytes = sb,					\
+		.ecc_bytes = eb,					\
+		.last_data_bytes = ldb,					\
+		.last_spare_bytes = lsb,				\
+		.last_ecc_bytes = leb,					\
+	}
+
+/* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
+static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
+	MARVELL_LAYOUT(  512,   512,  1,  1,  1,  512,  8,  8,  0,  0,  0),
+	MARVELL_LAYOUT( 2048,   512,  1,  1,  1, 2048, 40, 24,  0,  0,  0),
+	MARVELL_LAYOUT( 2048,   512,  4,  1,  1, 2048, 32, 30,  0,  0,  0),
+	MARVELL_LAYOUT( 4096,   512,  4,  2,  2, 2048, 32, 30,  0,  0,  0),
+	MARVELL_LAYOUT( 4096,   512,  8,  5,  4, 1024,  0, 30,  0, 64, 30),
+};
+
+/**
+ * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
+ * is made by a field in NDCB0 register, and in another field in NDCB2 register.
+ * The datasheet describes the logic with an error: ADDR5 field is once
+ * declared at the beginning of NDCB2, and another time at its end. Because the
+ * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
+ * to use the last bit of this field instead of the first ones.
+ *
+ * @cs:			Wanted CE lane.
+ * @ndcb0_csel:		Value of the NDCB0 register with or without the flag
+ *			selecting the wanted CE lane. This is set once when
+ *			the Device Tree is probed.
+ * @rb:			Ready/Busy pin for the flash chip
+ */
+struct marvell_nand_chip_sel {
+	unsigned int cs;
+	u32 ndcb0_csel;
+	unsigned int rb;
+};
+
+/**
+ * NAND chip structure: stores NAND chip device related information
+ *
+ * @chip:		Base NAND chip structure
+ * @node:		Used to store NAND chips into a list
+ * @layout		NAND layout when using hardware ECC
+ * @ndcr:		Controller register value for this NAND chip
+ * @ndtr0:		Timing registers 0 value for this NAND chip
+ * @ndtr1:		Timing registers 1 value for this NAND chip
+ * @selected_die:	Current active CS
+ * @nsels:		Number of CS lines required by the NAND chip
+ * @sels:		Array of CS lines descriptions
+ */
+struct marvell_nand_chip {
+	struct nand_chip chip;
+	struct list_head node;
+	const struct marvell_hw_ecc_layout *layout;
+	u32 ndcr;
+	u32 ndtr0;
+	u32 ndtr1;
+	int addr_cyc;
+	int selected_die;
+	unsigned int nsels;
+	struct marvell_nand_chip_sel sels[0];
+};
+
+static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
+{
+	return container_of(chip, struct marvell_nand_chip, chip);
+}
+
+static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
+							*nand)
+{
+	return &nand->sels[nand->selected_die];
+}
+
+/**
+ * NAND controller capabilities for distinction between compatible strings
+ *
+ * @max_cs_nb:		Number of Chip Select lines available
+ * @max_rb_nb:		Number of Ready/Busy lines available
+ * @need_system_controller: Indicates if the SoC needs to have access to the
+ *                      system controller (ie. to enable the NAND controller)
+ * @legacy_of_bindings:	Indicates if DT parsing must be done using the old
+ *			fashion way
+ * @is_nfcv2:		NFCv2 has numerous enhancements compared to NFCv1, ie.
+ *			BCH error detection and correction algorithm,
+ *			NDCB3 register has been added
+ * @use_dma:		Use dma for data transfers
+ */
+struct marvell_nfc_caps {
+	unsigned int max_cs_nb;
+	unsigned int max_rb_nb;
+	bool need_system_controller;
+	bool legacy_of_bindings;
+	bool is_nfcv2;
+	bool use_dma;
+};
+
+/**
+ * NAND controller structure: stores Marvell NAND controller information
+ *
+ * @controller:		Base controller structure
+ * @dev:		Parent device (used to print error messages)
+ * @regs:		NAND controller registers
+ * @ecc_clk:		ECC block clock, two times the NAND controller clock
+ * @complete:		Completion object to wait for NAND controller events
+ * @assigned_cs:	Bitmask describing already assigned CS lines
+ * @chips:		List containing all the NAND chips attached to
+ *			this NAND controller
+ * @caps:		NAND controller capabilities for each compatible string
+ * @dma_chan:		DMA channel (NFCv1 only)
+ * @dma_buf:		32-bit aligned buffer for DMA transfers (NFCv1 only)
+ */
+struct marvell_nfc {
+	struct nand_hw_control controller;
+	struct device *dev;
+	void __iomem *regs;
+	struct clk *ecc_clk;
+	struct completion complete;
+	unsigned long assigned_cs;
+	struct list_head chips;
+	struct nand_chip *selected_chip;
+	const struct marvell_nfc_caps *caps;
+
+	/* DMA (NFCv1 only) */
+	bool use_dma;
+	struct dma_chan *dma_chan;
+	u8 *dma_buf;
+};
+
+static inline struct marvell_nfc *to_marvell_nfc(struct nand_hw_control *ctrl)
+{
+	return container_of(ctrl, struct marvell_nfc, controller);
+}
+
+/**
+ * NAND controller timings expressed in NAND Controller clock cycles
+ *
+ * @tRP:		ND_nRE pulse width
+ * @tRH:		ND_nRE high duration
+ * @tWP:		ND_nWE pulse time
+ * @tWH:		ND_nWE high duration
+ * @tCS:		Enable signal setup time
+ * @tCH:		Enable signal hold time
+ * @tADL:		Address to write data delay
+ * @tAR:		ND_ALE low to ND_nRE low delay
+ * @tWHR:		ND_nWE high to ND_nRE low for status read
+ * @tRHW:		ND_nRE high duration, read to write delay
+ * @tR:			ND_nWE high to ND_nRE low for read
+ */
+struct marvell_nfc_timings {
+	/* NDTR0 fields */
+	unsigned int tRP;
+	unsigned int tRH;
+	unsigned int tWP;
+	unsigned int tWH;
+	unsigned int tCS;
+	unsigned int tCH;
+	unsigned int tADL;
+	/* NDTR1 fields */
+	unsigned int tAR;
+	unsigned int tWHR;
+	unsigned int tRHW;
+	unsigned int tR;
+};
+
+/**
+ * Derives a duration in numbers of clock cycles.
+ *
+ * @ps: Duration in pico-seconds
+ * @period_ns:  Clock period in nano-seconds
+ *
+ * Convert the duration in nano-seconds, then divide by the period and
+ * return the number of clock periods.
+ */
+#define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
+
+/**
+ * NAND driver structure filled during the parsing of the ->exec_op() subop
+ * subset of instructions.
+ *
+ * @ndcb:		Array of values written to NDCBx registers
+ * @cle_ale_delay_ns:	Optional delay after the last CMD or ADDR cycle
+ * @rdy_timeout_ms:	Timeout for waits on Ready/Busy pin
+ * @rdy_delay_ns:	Optional delay after waiting for the RB pin
+ * @data_delay_ns:	Optional delay after the data xfer
+ * @data_instr_idx:	Index of the data instruction in the subop
+ * @data_instr:		Pointer to the data instruction in the subop
+ */
+struct marvell_nfc_op {
+	u32 ndcb[4];
+	unsigned int cle_ale_delay_ns;
+	unsigned int rdy_timeout_ms;
+	unsigned int rdy_delay_ns;
+	unsigned int data_delay_ns;
+	unsigned int data_instr_idx;
+	const struct nand_op_instr *data_instr;
+};
+
+/*
+ * Internal helper to conditionnally apply a delay (from the above structure,
+ * most of the time).
+ */
+static void cond_delay(unsigned int ns)
+{
+	if (!ns)
+		return;
+
+	if (ns < 10000)
+		ndelay(ns);
+	else
+		udelay(DIV_ROUND_UP(ns, 1000));
+}
+
+/*
+ * The controller has many flags that could generate interrupts, most of them
+ * are disabled and polling is used. For the very slow signals, using interrupts
+ * may relax the CPU charge.
+ */
+static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
+{
+	u32 reg;
+
+	/* Writing 1 disables the interrupt */
+	reg = readl_relaxed(nfc->regs + NDCR);
+	writel_relaxed(reg | int_mask, nfc->regs + NDCR);
+}
+
+static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
+{
+	u32 reg;
+
+	/* Writing 0 enables the interrupt */
+	reg = readl_relaxed(nfc->regs + NDCR);
+	writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
+}
+
+static void marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
+{
+	writel_relaxed(int_mask, nfc->regs + NDSR);
+}
+
+static void marvell_nfc_force_byte_access(struct nand_chip *chip,
+					  bool force_8bit)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	u32 ndcr;
+
+	/*
+	 * Callers of this function do not verify if the NAND is using a 16-bit
+	 * an 8-bit bus for normal operations, so we need to take care of that
+	 * here by leaving the configuration unchanged if the NAND does not have
+	 * the NAND_BUSWIDTH_16 flag set.
+	 */
+	if (!(chip->options & NAND_BUSWIDTH_16))
+		return;
+
+	ndcr = readl_relaxed(nfc->regs + NDCR);
+
+	if (force_8bit)
+		ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
+	else
+		ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
+
+	writel_relaxed(ndcr, nfc->regs + NDCR);
+}
+
+static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	u32 val;
+	int ret;
+
+	/*
+	 * The command is being processed, wait for the ND_RUN bit to be
+	 * cleared by the NFC. If not, we must clear it by hand.
+	 */
+	ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
+					 (val & NDCR_ND_RUN) == 0,
+					 POLL_PERIOD, POLL_TIMEOUT);
+	if (ret) {
+		dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
+		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
+			       nfc->regs + NDCR);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * Any time a command has to be sent to the controller, the following sequence
+ * has to be followed:
+ * - call marvell_nfc_prepare_cmd()
+ *      -> activate the ND_RUN bit that will kind of 'start a job'
+ *      -> wait the signal indicating the NFC is waiting for a command
+ * - send the command (cmd and address cycles)
+ * - enventually send or receive the data
+ * - call marvell_nfc_end_cmd() with the corresponding flag
+ *      -> wait the flag to be triggered or cancel the job with a timeout
+ *
+ * The following helpers are here to factorize the code a bit so that
+ * specialized functions responsible for executing the actual NAND
+ * operations do not have to replicate the same code blocks.
+ */
+static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	u32 ndcr, val;
+	int ret;
+
+	/* Poll ND_RUN and clear NDSR before issuing any command */
+	ret = marvell_nfc_wait_ndrun(chip);
+	if (ret) {
+		dev_err(nfc->dev, "Last operation did not succeed\n");
+		return ret;
+	}
+
+	ndcr = readl_relaxed(nfc->regs + NDCR);
+	writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
+
+	/* Assert ND_RUN bit and wait the NFC to be ready */
+	writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
+	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
+					 val & NDSR_WRCMDREQ,
+					 POLL_PERIOD, POLL_TIMEOUT);
+	if (ret) {
+		dev_err(nfc->dev, "Timeout on WRCMDRE\n");
+		return -ETIMEDOUT;
+	}
+
+	/* Command may be written, clear WRCMDREQ status bit */
+	writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
+
+	return 0;
+}
+
+static void marvell_nfc_send_cmd(struct nand_chip *chip,
+				 struct marvell_nfc_op *nfc_op)
+{
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+
+	dev_dbg(nfc->dev, "\nNDCR:  0x%08x\n"
+		"NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
+		(u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
+		nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
+
+	writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
+		       nfc->regs + NDCB0);
+	writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
+	writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
+
+	/*
+	 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
+	 * fields are used (only available on NFCv2).
+	 */
+	if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
+	    NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
+		if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
+			writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
+	}
+}
+
+static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
+			       const char *label)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	u32 val;
+	int ret;
+
+	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
+					 val & flag,
+					 POLL_PERIOD, POLL_TIMEOUT);
+
+	if (ret) {
+		dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
+			label, val);
+		if (nfc->dma_chan)
+			dmaengine_terminate_all(nfc->dma_chan);
+		return ret;
+	}
+
+	/*
+	 * DMA function uses this helper to poll on CMDD bits without wanting
+	 * them to be cleared.
+	 */
+	if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
+		return 0;
+
+	writel_relaxed(flag, nfc->regs + NDSR);
+
+	return 0;
+}
+
+static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
+{
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
+
+	return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
+}
+
+static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	int ret;
+
+	/* Timeout is expressed in ms */
+	if (!timeout_ms)
+		timeout_ms = IRQ_TIMEOUT;
+
+	init_completion(&nfc->complete);
+
+	marvell_nfc_enable_int(nfc, NDCR_RDYM);
+	ret = wait_for_completion_timeout(&nfc->complete,
+					  msecs_to_jiffies(timeout_ms));
+	marvell_nfc_disable_int(nfc, NDCR_RDYM);
+	marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
+	if (!ret) {
+		dev_err(nfc->dev, "Timeout waiting for RB signal\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void marvell_nfc_select_chip(struct mtd_info *mtd, int die_nr)
+{
+	struct nand_chip *chip = mtd_to_nand(mtd);
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	u32 ndcr_generic;
+
+	if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
+		return;
+
+	if (die_nr < 0 || die_nr >= marvell_nand->nsels) {
+		nfc->selected_chip = NULL;
+		marvell_nand->selected_die = -1;
+		return;
+	}
+
+	/*
+	 * Do not change the timing registers when using the DT property
+	 * marvell,nand-keep-config; in that case ->ndtr0 and ->ndtr1 from the
+	 * marvell_nand structure are supposedly empty.
+	 */
+	writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
+	writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
+
+	/*
+	 * Reset the NDCR register to a clean state for this particular chip,
+	 * also clear ND_RUN bit.
+	 */
+	ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
+		       NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
+	writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
+
+	/* Also reset the interrupt status register */
+	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
+
+	nfc->selected_chip = chip;
+	marvell_nand->selected_die = die_nr;
+}
+
+static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
+{
+	struct marvell_nfc *nfc = dev_id;
+	u32 st = readl_relaxed(nfc->regs + NDSR);
+	u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
+
+	/*
+	 * RDY interrupt mask is one bit in NDCR while there are two status
+	 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
+	 */
+	if (st & NDSR_RDY(1))
+		st |= NDSR_RDY(0);
+
+	if (!(st & ien))
+		return IRQ_NONE;
+
+	marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
+
+	if (!(st & (NDSR_RDDREQ | NDSR_WRDREQ | NDSR_WRCMDREQ)))
+		complete(&nfc->complete);
+
+	return IRQ_HANDLED;
+}
+
+/* HW ECC related functions */
+static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
+
+	if (!(ndcr & NDCR_ECC_EN)) {
+		writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
+
+		/*
+		 * When enabling BCH, set threshold to 0 to always know the
+		 * number of corrected bitflips.
+		 */
+		if (chip->ecc.algo == NAND_ECC_BCH)
+			writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
+	}
+}
+
+static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
+
+	if (ndcr & NDCR_ECC_EN) {
+		writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
+		if (chip->ecc.algo == NAND_ECC_BCH)
+			writel_relaxed(0, nfc->regs + NDECCCTRL);
+	}
+}
+
+/* DMA related helpers */
+static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
+{
+	u32 reg;
+
+	reg = readl_relaxed(nfc->regs + NDCR);
+	writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
+}
+
+static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
+{
+	u32 reg;
+
+	reg = readl_relaxed(nfc->regs + NDCR);
+	writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
+}
+
+/* Read/write PIO/DMA accessors */
+static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
+				     enum dma_data_direction direction,
+				     unsigned int len)
+{
+	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
+	struct dma_async_tx_descriptor *tx;
+	struct scatterlist sg;
+	dma_cookie_t cookie;
+	int ret;
+
+	marvell_nfc_enable_dma(nfc);
+	/* Prepare the DMA transfer */
+	sg_init_one(&sg, nfc->dma_buf, dma_len);
+	dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
+	tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
+				     direction == DMA_FROM_DEVICE ?
+				     DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
+				     DMA_PREP_INTERRUPT);
+	if (!tx) {
+		dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
+		return -ENXIO;
+	}
+
+	/* Do the task and wait for it to finish */
+	cookie = dmaengine_submit(tx);
+	ret = dma_submit_error(cookie);
+	if (ret)
+		return -EIO;
+
+	dma_async_issue_pending(nfc->dma_chan);
+	ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
+	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
+	marvell_nfc_disable_dma(nfc);
+	if (ret) {
+		dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
+			dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
+		dmaengine_terminate_all(nfc->dma_chan);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
+					unsigned int len)
+{
+	unsigned int last_len = len % FIFO_DEPTH;
+	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
+	int i;
+
+	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
+		ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
+
+	if (last_len) {
+		u8 tmp_buf[FIFO_DEPTH];
+
+		ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
+		memcpy(in + last_full_offset, tmp_buf, last_len);
+	}
+
+	return 0;
+}
+
+static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
+					 unsigned int len)
+{
+	unsigned int last_len = len % FIFO_DEPTH;
+	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
+	int i;
+
+	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
+		iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
+
+	if (last_len) {
+		u8 tmp_buf[FIFO_DEPTH];
+
+		memcpy(tmp_buf, out + last_full_offset, last_len);
+		iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
+	}
+
+	return 0;
+}
+
+static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
+					  u8 *data, int data_len,
+					  u8 *spare, int spare_len,
+					  u8 *ecc, int ecc_len,
+					  unsigned int *max_bitflips)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int bf;
+
+	/*
+	 * Blank pages (all 0xFF) that have not been written may be recognized
+	 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
+	 * check if the entire page (with ECC bytes) is actually blank or not.
+	 */
+	if (!data)
+		data_len = 0;
+	if (!spare)
+		spare_len = 0;
+	if (!ecc)
+		ecc_len = 0;
+
+	bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
+					 spare, spare_len, chip->ecc.strength);
+	if (bf < 0) {
+		mtd->ecc_stats.failed++;
+		return;
+	}
+
+	/* Update the stats and max_bitflips */
+	mtd->ecc_stats.corrected += bf;
+	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
+}
+
+/*
+ * Check a chunk is correct or not according to hardware ECC engine.
+ * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
+ * mtd->ecc_stats.failure is not, the function will instead return a non-zero
+ * value indicating that a check on the emptyness of the subpage must be
+ * performed before declaring the subpage corrupted.
+ */
+static int marvell_nfc_hw_ecc_correct(struct nand_chip *chip,
+				      unsigned int *max_bitflips)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	int bf = 0;
+	u32 ndsr;
+
+	ndsr = readl_relaxed(nfc->regs + NDSR);
+
+	/* Check uncorrectable error flag */
+	if (ndsr & NDSR_UNCERR) {
+		writel_relaxed(ndsr, nfc->regs + NDSR);
+
+		/*
+		 * Do not increment ->ecc_stats.failed now, instead, return a
+		 * non-zero value to indicate that this chunk was apparently
+		 * bad, and it should be check to see if it empty or not. If
+		 * the chunk (with ECC bytes) is not declared empty, the calling
+		 * function must increment the failure count.
+		 */
+		return -EBADMSG;
+	}
+
+	/* Check correctable error flag */
+	if (ndsr & NDSR_CORERR) {
+		writel_relaxed(ndsr, nfc->regs + NDSR);
+
+		if (chip->ecc.algo == NAND_ECC_BCH)
+			bf = NDSR_ERRCNT(ndsr);
+		else
+			bf = 1;
+	}
+
+	/* Update the stats and max_bitflips */
+	mtd->ecc_stats.corrected += bf;
+	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
+
+	return 0;
+}
+
+/* Hamming read helpers */
+static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
+					       u8 *data_buf, u8 *oob_buf,
+					       bool raw, int page)
+{
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	struct marvell_nfc_op nfc_op = {
+		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
+			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
+			   NDCB0_DBC |
+			   NDCB0_CMD1(NAND_CMD_READ0) |
+			   NDCB0_CMD2(NAND_CMD_READSTART),
+		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
+		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
+	};
+	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
+	int ret;
+
+	/* NFCv2 needs more information about the operation being executed */
+	if (nfc->caps->is_nfcv2)
+		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
+				  "RDDREQ while draining FIFO (data/oob)");
+	if (ret)
+		return ret;
+
+	/*
+	 * Read the page then the OOB area. Unlike what is shown in current
+	 * documentation, spare bytes are protected by the ECC engine, and must
+	 * be at the beginning of the OOB area or running this driver on legacy
+	 * systems will prevent the discovery of the BBM/BBT.
+	 */
+	if (nfc->use_dma) {
+		marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
+					  lt->data_bytes + oob_bytes);
+		memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
+		memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
+	} else {
+		marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
+		marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
+	}
+
+	ret = marvell_nfc_wait_cmdd(chip);
+
+	return ret;
+}
+
+static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct mtd_info *mtd,
+						struct nand_chip *chip, u8 *buf,
+						int oob_required, int page)
+{
+	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
+						   true, page);
+}
+
+static int marvell_nfc_hw_ecc_hmg_read_page(struct mtd_info *mtd,
+					    struct nand_chip *chip,
+					    u8 *buf, int oob_required,
+					    int page)
+{
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
+	int max_bitflips = 0, ret;
+	u8 *raw_buf;
+
+	marvell_nfc_enable_hw_ecc(chip);
+	marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
+					    page);
+	ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
+	marvell_nfc_disable_hw_ecc(chip);
+
+	if (!ret)
+		return max_bitflips;
+
+	/*
+	 * When ECC failures are detected, check if the full page has been
+	 * written or not. Ignore the failure if it is actually empty.
+	 */
+	raw_buf = kmalloc(full_sz, GFP_KERNEL);
+	if (!raw_buf)
+		return -ENOMEM;
+
+	marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
+					    lt->data_bytes, true, page);
+	marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
+				      &max_bitflips);
+	kfree(raw_buf);
+
+	return max_bitflips;
+}
+
+/*
+ * Spare area in Hamming layouts is not protected by the ECC engine (even if
+ * it appears before the ECC bytes when reading), the ->read_oob_raw() function
+ * also stands for ->read_oob().
+ */
+static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct mtd_info *mtd,
+					       struct nand_chip *chip, int page)
+{
+	/* Invalidate page cache */
+	chip->pagebuf = -1;
+
+	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, chip->data_buf,
+						   chip->oob_poi, true, page);
+}
+
+/* Hamming write helpers */
+static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
+						const u8 *data_buf,
+						const u8 *oob_buf, bool raw,
+						int page)
+{
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	struct marvell_nfc_op nfc_op = {
+		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
+			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
+			   NDCB0_CMD1(NAND_CMD_SEQIN) |
+			   NDCB0_CMD2(NAND_CMD_PAGEPROG) |
+			   NDCB0_DBC,
+		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
+		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
+	};
+	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
+	int ret;
+
+	/* NFCv2 needs more information about the operation being executed */
+	if (nfc->caps->is_nfcv2)
+		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
+				  "WRDREQ while loading FIFO (data)");
+	if (ret)
+		return ret;
+
+	/* Write the page then the OOB area */
+	if (nfc->use_dma) {
+		memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
+		memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
+		marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
+					  lt->ecc_bytes + lt->spare_bytes);
+	} else {
+		marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
+		marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
+	}
+
+	ret = marvell_nfc_wait_cmdd(chip);
+	if (ret)
+		return ret;
+
+	ret = marvell_nfc_wait_op(chip,
+				  chip->data_interface.timings.sdr.tPROG_max);
+	return ret;
+}
+
+static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct mtd_info *mtd,
+						 struct nand_chip *chip,
+						 const u8 *buf,
+						 int oob_required, int page)
+{
+	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
+						    true, page);
+}
+
+static int marvell_nfc_hw_ecc_hmg_write_page(struct mtd_info *mtd,
+					     struct nand_chip *chip,
+					     const u8 *buf,
+					     int oob_required, int page)
+{
+	int ret;
+
+	marvell_nfc_enable_hw_ecc(chip);
+	ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
+						   false, page);
+	marvell_nfc_disable_hw_ecc(chip);
+
+	return ret;
+}
+
+/*
+ * Spare area in Hamming layouts is not protected by the ECC engine (even if
+ * it appears before the ECC bytes when reading), the ->write_oob_raw() function
+ * also stands for ->write_oob().
+ */
+static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct mtd_info *mtd,
+						struct nand_chip *chip,
+						int page)
+{
+	/* Invalidate page cache */
+	chip->pagebuf = -1;
+
+	memset(chip->data_buf, 0xFF, mtd->writesize);
+
+	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, chip->data_buf,
+						    chip->oob_poi, true, page);
+}
+
+/* BCH read helpers */
+static int marvell_nfc_hw_ecc_bch_read_page_raw(struct mtd_info *mtd,
+						struct nand_chip *chip, u8 *buf,
+						int oob_required, int page)
+{
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	u8 *oob = chip->oob_poi;
+	int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
+	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
+		lt->last_spare_bytes;
+	int data_len = lt->data_bytes;
+	int spare_len = lt->spare_bytes;
+	int ecc_len = lt->ecc_bytes;
+	int chunk;
+
+	if (oob_required)
+		memset(chip->oob_poi, 0xFF, mtd->oobsize);
+
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
+	for (chunk = 0; chunk < lt->nchunks; chunk++) {
+		/* Update last chunk length */
+		if (chunk >= lt->full_chunk_cnt) {
+			data_len = lt->last_data_bytes;
+			spare_len = lt->last_spare_bytes;
+			ecc_len = lt->last_ecc_bytes;
+		}
+
+		/* Read data bytes*/
+		nand_change_read_column_op(chip, chunk * chunk_size,
+					   buf + (lt->data_bytes * chunk),
+					   data_len, false);
+
+		/* Read spare bytes */
+		nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
+				  spare_len, false);
+
+		/* Read ECC bytes */
+		nand_read_data_op(chip, oob + ecc_offset +
+				  (ALIGN(lt->ecc_bytes, 32) * chunk),
+				  ecc_len, false);
+	}
+
+	return 0;
+}
+
+static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
+					      u8 *data, unsigned int data_len,
+					      u8 *spare, unsigned int spare_len,
+					      int page)
+{
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	int i, ret;
+	struct marvell_nfc_op nfc_op = {
+		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
+			   NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
+			   NDCB0_LEN_OVRD,
+		.ndcb[1] = NDCB1_ADDRS_PAGE(page),
+		.ndcb[2] = NDCB2_ADDR5_PAGE(page),
+		.ndcb[3] = data_len + spare_len,
+	};
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return;
+
+	if (chunk == 0)
+		nfc_op.ndcb[0] |= NDCB0_DBC |
+				  NDCB0_CMD1(NAND_CMD_READ0) |
+				  NDCB0_CMD2(NAND_CMD_READSTART);
+
+	/*
+	 * Trigger the naked read operation only on the last chunk.
+	 * Otherwise, use monolithic read.
+	 */
+	if (lt->nchunks == 1 || (chunk < lt->nchunks - 1))
+		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
+	else
+		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+
+	/*
+	 * According to the datasheet, when reading from NDDB
+	 * with BCH enabled, after each 32 bytes reads, we
+	 * have to make sure that the NDSR.RDDREQ bit is set.
+	 *
+	 * Drain the FIFO, 8 32-bit reads at a time, and skip
+	 * the polling on the last read.
+	 *
+	 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
+	 */
+	for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
+		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
+				    "RDDREQ while draining FIFO (data)");
+		marvell_nfc_xfer_data_in_pio(nfc, data,
+					     FIFO_DEPTH * BCH_SEQ_READS);
+		data += FIFO_DEPTH * BCH_SEQ_READS;
+	}
+
+	for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
+		marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
+				    "RDDREQ while draining FIFO (OOB)");
+		marvell_nfc_xfer_data_in_pio(nfc, spare,
+					     FIFO_DEPTH * BCH_SEQ_READS);
+		spare += FIFO_DEPTH * BCH_SEQ_READS;
+	}
+}
+
+static int marvell_nfc_hw_ecc_bch_read_page(struct mtd_info *mtd,
+					    struct nand_chip *chip,
+					    u8 *buf, int oob_required,
+					    int page)
+{
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	int data_len = lt->data_bytes, spare_len = lt->spare_bytes, ecc_len;
+	u8 *data = buf, *spare = chip->oob_poi, *ecc;
+	int max_bitflips = 0;
+	u32 failure_mask = 0;
+	int chunk, ecc_offset_in_page, ret;
+
+	/*
+	 * With BCH, OOB is not fully used (and thus not read entirely), not
+	 * expected bytes could show up at the end of the OOB buffer if not
+	 * explicitly erased.
+	 */
+	if (oob_required)
+		memset(chip->oob_poi, 0xFF, mtd->oobsize);
+
+	marvell_nfc_enable_hw_ecc(chip);
+
+	for (chunk = 0; chunk < lt->nchunks; chunk++) {
+		/* Update length for the last chunk */
+		if (chunk >= lt->full_chunk_cnt) {
+			data_len = lt->last_data_bytes;
+			spare_len = lt->last_spare_bytes;
+		}
+
+		/* Read the chunk and detect number of bitflips */
+		marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
+						  spare, spare_len, page);
+		ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
+		if (ret)
+			failure_mask |= BIT(chunk);
+
+		data += data_len;
+		spare += spare_len;
+	}
+
+	marvell_nfc_disable_hw_ecc(chip);
+
+	if (!failure_mask)
+		return max_bitflips;
+
+	/*
+	 * Please note that dumping the ECC bytes during a normal read with OOB
+	 * area would add a significant overhead as ECC bytes are "consumed" by
+	 * the controller in normal mode and must be re-read in raw mode. To
+	 * avoid dropping the performances, we prefer not to include them. The
+	 * user should re-read the page in raw mode if ECC bytes are required.
+	 *
+	 * However, for any subpage read error reported by ->correct(), the ECC
+	 * bytes must be read in raw mode and the full subpage must be checked
+	 * to see if it is entirely empty of if there was an actual error.
+	 */
+	for (chunk = 0; chunk < lt->nchunks; chunk++) {
+		/* No failure reported for this chunk, move to the next one */
+		if (!(failure_mask & BIT(chunk)))
+			continue;
+
+		/* Derive ECC bytes positions (in page/buffer) and length */
+		ecc = chip->oob_poi +
+			(lt->full_chunk_cnt * lt->spare_bytes) +
+			lt->last_spare_bytes +
+			(chunk * ALIGN(lt->ecc_bytes, 32));
+		ecc_offset_in_page =
+			(chunk * (lt->data_bytes + lt->spare_bytes +
+				  lt->ecc_bytes)) +
+			(chunk < lt->full_chunk_cnt ?
+			 lt->data_bytes + lt->spare_bytes :
+			 lt->last_data_bytes + lt->last_spare_bytes);
+		ecc_len = chunk < lt->full_chunk_cnt ?
+			lt->ecc_bytes : lt->last_ecc_bytes;
+
+		/* Do the actual raw read of the ECC bytes */
+		nand_change_read_column_op(chip, ecc_offset_in_page,
+					   ecc, ecc_len, false);
+
+		/* Derive data/spare bytes positions (in buffer) and length */
+		data = buf + (chunk * lt->data_bytes);
+		data_len = chunk < lt->full_chunk_cnt ?
+			lt->data_bytes : lt->last_data_bytes;
+		spare = chip->oob_poi + (chunk * (lt->spare_bytes +
+						  lt->ecc_bytes));
+		spare_len = chunk < lt->full_chunk_cnt ?
+			lt->spare_bytes : lt->last_spare_bytes;
+
+		/* Check the entire chunk (data + spare + ecc) for emptyness */
+		marvell_nfc_check_empty_chunk(chip, data, data_len, spare,
+					      spare_len, ecc, ecc_len,
+					      &max_bitflips);
+	}
+
+	return max_bitflips;
+}
+
+static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct mtd_info *mtd,
+					       struct nand_chip *chip, int page)
+{
+	/* Invalidate page cache */
+	chip->pagebuf = -1;
+
+	return chip->ecc.read_page_raw(mtd, chip, chip->data_buf, true, page);
+}
+
+static int marvell_nfc_hw_ecc_bch_read_oob(struct mtd_info *mtd,
+					   struct nand_chip *chip, int page)
+{
+	/* Invalidate page cache */
+	chip->pagebuf = -1;
+
+	return chip->ecc.read_page(mtd, chip, chip->data_buf, true, page);
+}
+
+/* BCH write helpers */
+static int marvell_nfc_hw_ecc_bch_write_page_raw(struct mtd_info *mtd,
+						 struct nand_chip *chip,
+						 const u8 *buf,
+						 int oob_required, int page)
+{
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
+	int data_len = lt->data_bytes;
+	int spare_len = lt->spare_bytes;
+	int ecc_len = lt->ecc_bytes;
+	int spare_offset = 0;
+	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
+		lt->last_spare_bytes;
+	int chunk;
+
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
+	for (chunk = 0; chunk < lt->nchunks; chunk++) {
+		if (chunk >= lt->full_chunk_cnt) {
+			data_len = lt->last_data_bytes;
+			spare_len = lt->last_spare_bytes;
+			ecc_len = lt->last_ecc_bytes;
+		}
+
+		/* Point to the column of the next chunk */
+		nand_change_write_column_op(chip, chunk * full_chunk_size,
+					    NULL, 0, false);
+
+		/* Write the data */
+		nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
+				   data_len, false);
+
+		if (!oob_required)
+			continue;
+
+		/* Write the spare bytes */
+		if (spare_len)
+			nand_write_data_op(chip, chip->oob_poi + spare_offset,
+					   spare_len, false);
+
+		/* Write the ECC bytes */
+		if (ecc_len)
+			nand_write_data_op(chip, chip->oob_poi + ecc_offset,
+					   ecc_len, false);
+
+		spare_offset += spare_len;
+		ecc_offset += ALIGN(ecc_len, 32);
+	}
+
+	return nand_prog_page_end_op(chip);
+}
+
+static int
+marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
+				   const u8 *data, unsigned int data_len,
+				   const u8 *spare, unsigned int spare_len,
+				   int page)
+{
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	int ret;
+	struct marvell_nfc_op nfc_op = {
+		.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
+		.ndcb[3] = data_len + spare_len,
+	};
+
+	/*
+	 * First operation dispatches the CMD_SEQIN command, issue the address
+	 * cycles and asks for the first chunk of data.
+	 * All operations in the middle (if any) will issue a naked write and
+	 * also ask for data.
+	 * Last operation (if any) asks for the last chunk of data through a
+	 * last naked write.
+	 */
+	if (chunk == 0) {
+		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_WRITE_DISPATCH) |
+				  NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
+				  NDCB0_CMD1(NAND_CMD_SEQIN);
+		nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
+		nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
+	} else if (chunk < lt->nchunks - 1) {
+		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
+	} else {
+		nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
+	}
+
+	/* Always dispatch the PAGEPROG command on the last chunk */
+	if (chunk == lt->nchunks - 1)
+		nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
+				  "WRDREQ while loading FIFO (data)");
+	if (ret)
+		return ret;
+
+	/* Transfer the contents */
+	iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
+	iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
+
+	return 0;
+}
+
+static int marvell_nfc_hw_ecc_bch_write_page(struct mtd_info *mtd,
+					     struct nand_chip *chip,
+					     const u8 *buf,
+					     int oob_required, int page)
+{
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+	const u8 *data = buf;
+	const u8 *spare = chip->oob_poi;
+	int data_len = lt->data_bytes;
+	int spare_len = lt->spare_bytes;
+	int chunk, ret;
+
+	/* Spare data will be written anyway, so clear it to avoid garbage */
+	if (!oob_required)
+		memset(chip->oob_poi, 0xFF, mtd->oobsize);
+
+	marvell_nfc_enable_hw_ecc(chip);
+
+	for (chunk = 0; chunk < lt->nchunks; chunk++) {
+		if (chunk >= lt->full_chunk_cnt) {
+			data_len = lt->last_data_bytes;
+			spare_len = lt->last_spare_bytes;
+		}
+
+		marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
+						   spare, spare_len, page);
+		data += data_len;
+		spare += spare_len;
+
+		/*
+		 * Waiting only for CMDD or PAGED is not enough, ECC are
+		 * partially written. No flag is set once the operation is
+		 * really finished but the ND_RUN bit is cleared, so wait for it
+		 * before stepping into the next command.
+		 */
+		marvell_nfc_wait_ndrun(chip);
+	}
+
+	ret = marvell_nfc_wait_op(chip,
+				  chip->data_interface.timings.sdr.tPROG_max);
+
+	marvell_nfc_disable_hw_ecc(chip);
+
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct mtd_info *mtd,
+						struct nand_chip *chip,
+						int page)
+{
+	/* Invalidate page cache */
+	chip->pagebuf = -1;
+
+	memset(chip->data_buf, 0xFF, mtd->writesize);
+
+	return chip->ecc.write_page_raw(mtd, chip, chip->data_buf, true, page);
+}
+
+static int marvell_nfc_hw_ecc_bch_write_oob(struct mtd_info *mtd,
+					    struct nand_chip *chip, int page)
+{
+	/* Invalidate page cache */
+	chip->pagebuf = -1;
+
+	memset(chip->data_buf, 0xFF, mtd->writesize);
+
+	return chip->ecc.write_page(mtd, chip, chip->data_buf, true, page);
+}
+
+/* NAND framework ->exec_op() hooks and related helpers */
+static void marvell_nfc_parse_instructions(struct nand_chip *chip,
+					   const struct nand_subop *subop,
+					   struct marvell_nfc_op *nfc_op)
+{
+	const struct nand_op_instr *instr = NULL;
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	bool first_cmd = true;
+	unsigned int op_id;
+	int i;
+
+	/* Reset the input structure as most of its fields will be OR'ed */
+	memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
+
+	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
+		unsigned int offset, naddrs;
+		const u8 *addrs;
+		int len = nand_subop_get_data_len(subop, op_id);
+
+		instr = &subop->instrs[op_id];
+
+		switch (instr->type) {
+		case NAND_OP_CMD_INSTR:
+			if (first_cmd)
+				nfc_op->ndcb[0] |=
+					NDCB0_CMD1(instr->ctx.cmd.opcode);
+			else
+				nfc_op->ndcb[0] |=
+					NDCB0_CMD2(instr->ctx.cmd.opcode) |
+					NDCB0_DBC;
+
+			nfc_op->cle_ale_delay_ns = instr->delay_ns;
+			first_cmd = false;
+			break;
+
+		case NAND_OP_ADDR_INSTR:
+			offset = nand_subop_get_addr_start_off(subop, op_id);
+			naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
+			addrs = &instr->ctx.addr.addrs[offset];
+
+			nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
+
+			for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
+				nfc_op->ndcb[1] |= addrs[i] << (8 * i);
+
+			if (naddrs >= 5)
+				nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
+			if (naddrs >= 6)
+				nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
+			if (naddrs == 7)
+				nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
+
+			nfc_op->cle_ale_delay_ns = instr->delay_ns;
+			break;
+
+		case NAND_OP_DATA_IN_INSTR:
+			nfc_op->data_instr = instr;
+			nfc_op->data_instr_idx = op_id;
+			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
+			if (nfc->caps->is_nfcv2) {
+				nfc_op->ndcb[0] |=
+					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
+					NDCB0_LEN_OVRD;
+				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
+			}
+			nfc_op->data_delay_ns = instr->delay_ns;
+			break;
+
+		case NAND_OP_DATA_OUT_INSTR:
+			nfc_op->data_instr = instr;
+			nfc_op->data_instr_idx = op_id;
+			nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
+			if (nfc->caps->is_nfcv2) {
+				nfc_op->ndcb[0] |=
+					NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
+					NDCB0_LEN_OVRD;
+				nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
+			}
+			nfc_op->data_delay_ns = instr->delay_ns;
+			break;
+
+		case NAND_OP_WAITRDY_INSTR:
+			nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
+			nfc_op->rdy_delay_ns = instr->delay_ns;
+			break;
+		}
+	}
+}
+
+static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
+				     const struct nand_subop *subop,
+				     struct marvell_nfc_op *nfc_op)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	const struct nand_op_instr *instr = nfc_op->data_instr;
+	unsigned int op_id = nfc_op->data_instr_idx;
+	unsigned int len = nand_subop_get_data_len(subop, op_id);
+	unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
+	bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
+	int ret;
+
+	if (instr->ctx.data.force_8bit)
+		marvell_nfc_force_byte_access(chip, true);
+
+	if (reading) {
+		u8 *in = instr->ctx.data.buf.in + offset;
+
+		ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
+	} else {
+		const u8 *out = instr->ctx.data.buf.out + offset;
+
+		ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
+	}
+
+	if (instr->ctx.data.force_8bit)
+		marvell_nfc_force_byte_access(chip, false);
+
+	return ret;
+}
+
+static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
+					      const struct nand_subop *subop)
+{
+	struct marvell_nfc_op nfc_op;
+	bool reading;
+	int ret;
+
+	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
+	reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
+				  "RDDREQ/WRDREQ while draining raw data");
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.cle_ale_delay_ns);
+
+	if (reading) {
+		if (nfc_op.rdy_timeout_ms) {
+			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
+			if (ret)
+				return ret;
+		}
+
+		cond_delay(nfc_op.rdy_delay_ns);
+	}
+
+	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
+	ret = marvell_nfc_wait_cmdd(chip);
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.data_delay_ns);
+
+	if (!reading) {
+		if (nfc_op.rdy_timeout_ms) {
+			ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
+			if (ret)
+				return ret;
+		}
+
+		cond_delay(nfc_op.rdy_delay_ns);
+	}
+
+	/*
+	 * NDCR ND_RUN bit should be cleared automatically at the end of each
+	 * operation but experience shows that the behavior is buggy when it
+	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
+	 */
+	if (!reading) {
+		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+
+		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
+			       nfc->regs + NDCR);
+	}
+
+	return 0;
+}
+
+static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
+					 const struct nand_subop *subop)
+{
+	struct marvell_nfc_op nfc_op;
+	int ret;
+
+	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
+
+	/*
+	 * Naked access are different in that they need to be flagged as naked
+	 * by the controller. Reset the controller registers fields that inform
+	 * on the type and refill them according to the ongoing operation.
+	 */
+	nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
+			    NDCB0_CMD_XTYPE(XTYPE_MASK));
+	switch (subop->instrs[0].type) {
+	case NAND_OP_CMD_INSTR:
+		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
+		break;
+	case NAND_OP_ADDR_INSTR:
+		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
+		break;
+	case NAND_OP_DATA_IN_INSTR:
+		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
+				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
+		break;
+	case NAND_OP_DATA_OUT_INSTR:
+		nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
+				  NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
+		break;
+	default:
+		/* This should never happen */
+		break;
+	}
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+
+	if (!nfc_op.data_instr) {
+		ret = marvell_nfc_wait_cmdd(chip);
+		cond_delay(nfc_op.cle_ale_delay_ns);
+		return ret;
+	}
+
+	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
+				  "RDDREQ/WRDREQ while draining raw data");
+	if (ret)
+		return ret;
+
+	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
+	ret = marvell_nfc_wait_cmdd(chip);
+	if (ret)
+		return ret;
+
+	/*
+	 * NDCR ND_RUN bit should be cleared automatically at the end of each
+	 * operation but experience shows that the behavior is buggy when it
+	 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
+	 */
+	if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
+		struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+
+		writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
+			       nfc->regs + NDCR);
+	}
+
+	return 0;
+}
+
+static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
+					  const struct nand_subop *subop)
+{
+	struct marvell_nfc_op nfc_op;
+	int ret;
+
+	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
+
+	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
+	cond_delay(nfc_op.rdy_delay_ns);
+
+	return ret;
+}
+
+static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
+					 const struct nand_subop *subop)
+{
+	struct marvell_nfc_op nfc_op;
+	int ret;
+
+	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
+	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
+	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
+				  "RDDREQ while reading ID");
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.cle_ale_delay_ns);
+
+	if (nfc_op.rdy_timeout_ms) {
+		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
+		if (ret)
+			return ret;
+	}
+
+	cond_delay(nfc_op.rdy_delay_ns);
+
+	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
+	ret = marvell_nfc_wait_cmdd(chip);
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.data_delay_ns);
+
+	return 0;
+}
+
+static int marvell_nfc_read_status_exec(struct nand_chip *chip,
+					const struct nand_subop *subop)
+{
+	struct marvell_nfc_op nfc_op;
+	int ret;
+
+	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
+	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
+	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
+				  "RDDREQ while reading status");
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.cle_ale_delay_ns);
+
+	if (nfc_op.rdy_timeout_ms) {
+		ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
+		if (ret)
+			return ret;
+	}
+
+	cond_delay(nfc_op.rdy_delay_ns);
+
+	marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
+	ret = marvell_nfc_wait_cmdd(chip);
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.data_delay_ns);
+
+	return 0;
+}
+
+static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
+					   const struct nand_subop *subop)
+{
+	struct marvell_nfc_op nfc_op;
+	int ret;
+
+	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
+	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_wait_cmdd(chip);
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.cle_ale_delay_ns);
+
+	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.rdy_delay_ns);
+
+	return 0;
+}
+
+static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
+					   const struct nand_subop *subop)
+{
+	struct marvell_nfc_op nfc_op;
+	int ret;
+
+	marvell_nfc_parse_instructions(chip, subop, &nfc_op);
+	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
+
+	ret = marvell_nfc_prepare_cmd(chip);
+	if (ret)
+		return ret;
+
+	marvell_nfc_send_cmd(chip, &nfc_op);
+	ret = marvell_nfc_wait_cmdd(chip);
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.cle_ale_delay_ns);
+
+	ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
+	if (ret)
+		return ret;
+
+	cond_delay(nfc_op.rdy_delay_ns);
+
+	return 0;
+}
+
+static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
+	/* Monolithic reads/writes */
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_monolithic_access_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
+		NAND_OP_PARSER_PAT_CMD_ELEM(true),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
+		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_monolithic_access_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
+		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
+		NAND_OP_PARSER_PAT_CMD_ELEM(true),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
+	/* Naked commands */
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_naked_access_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_naked_access_exec,
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_naked_access_exec,
+		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_naked_access_exec,
+		NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_naked_waitrdy_exec,
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
+	);
+
+static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
+	/* Naked commands not supported, use a function for each pattern */
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_read_id_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
+		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_erase_cmd_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_read_status_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_reset_cmd_type_exec,
+		NAND_OP_PARSER_PAT_CMD_ELEM(false),
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
+	NAND_OP_PARSER_PATTERN(
+		marvell_nfc_naked_waitrdy_exec,
+		NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
+	);
+
+static int marvell_nfc_exec_op(struct nand_chip *chip,
+			       const struct nand_operation *op,
+			       bool check_only)
+{
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+
+	if (nfc->caps->is_nfcv2)
+		return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
+					      op, check_only);
+	else
+		return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
+					      op, check_only);
+}
+
+/*
+ * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
+ * usable.
+ */
+static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
+				      struct mtd_oob_region *oobregion)
+{
+	struct nand_chip *chip = mtd_to_nand(mtd);
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+
+	if (section)
+		return -ERANGE;
+
+	oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
+			    lt->last_ecc_bytes;
+	oobregion->offset = mtd->oobsize - oobregion->length;
+
+	return 0;
+}
+
+static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
+				       struct mtd_oob_region *oobregion)
+{
+	struct nand_chip *chip = mtd_to_nand(mtd);
+	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
+
+	if (section)
+		return -ERANGE;
+
+	/*
+	 * Bootrom looks in bytes 0 & 5 for bad blocks for the
+	 * 4KB page / 4bit BCH combination.
+	 */
+	if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
+		oobregion->offset = 6;
+	else
+		oobregion->offset = 2;
+
+	oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
+			    lt->last_spare_bytes - oobregion->offset;
+
+	return 0;
+}
+
+static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
+	.ecc = marvell_nand_ooblayout_ecc,
+	.free = marvell_nand_ooblayout_free,
+};
+
+static int marvell_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
+					 struct nand_ecc_ctrl *ecc)
+{
+	struct nand_chip *chip = mtd_to_nand(mtd);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	const struct marvell_hw_ecc_layout *l;
+	int i;
+
+	if (!nfc->caps->is_nfcv2 &&
+	    (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
+		dev_err(nfc->dev,
+			"NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
+			mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
+		return -ENOTSUPP;
+	}
+
+	to_marvell_nand(chip)->layout = NULL;
+	for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
+		l = &marvell_nfc_layouts[i];
+		if (mtd->writesize == l->writesize &&
+		    ecc->size == l->chunk && ecc->strength == l->strength) {
+			to_marvell_nand(chip)->layout = l;
+			break;
+		}
+	}
+
+	if (!to_marvell_nand(chip)->layout ||
+	    (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
+		dev_err(nfc->dev,
+			"ECC strength %d at page size %d is not supported\n",
+			ecc->strength, mtd->writesize);
+		return -ENOTSUPP;
+	}
+
+	mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
+	ecc->steps = l->nchunks;
+	ecc->size = l->data_bytes;
+
+	if (ecc->strength == 1) {
+		chip->ecc.algo = NAND_ECC_HAMMING;
+		ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
+		ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
+		ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
+		ecc->read_oob = ecc->read_oob_raw;
+		ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
+		ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
+		ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
+		ecc->write_oob = ecc->write_oob_raw;
+	} else {
+		chip->ecc.algo = NAND_ECC_BCH;
+		ecc->strength = 16;
+		ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
+		ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
+		ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
+		ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
+		ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
+		ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
+		ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
+		ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
+	}
+
+	return 0;
+}
+
+static int marvell_nand_ecc_init(struct mtd_info *mtd,
+				 struct nand_ecc_ctrl *ecc)
+{
+	struct nand_chip *chip = mtd_to_nand(mtd);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	int ret;
+
+	if (ecc->mode != NAND_ECC_NONE && (!ecc->size || !ecc->strength)) {
+		if (chip->ecc_step_ds && chip->ecc_strength_ds) {
+			ecc->size = chip->ecc_step_ds;
+			ecc->strength = chip->ecc_strength_ds;
+		} else {
+			dev_info(nfc->dev,
+				 "No minimum ECC strength, using 1b/512B\n");
+			ecc->size = 512;
+			ecc->strength = 1;
+		}
+	}
+
+	switch (ecc->mode) {
+	case NAND_ECC_HW:
+		ret = marvell_nand_hw_ecc_ctrl_init(mtd, ecc);
+		if (ret)
+			return ret;
+		break;
+	case NAND_ECC_NONE:
+	case NAND_ECC_SOFT:
+		if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
+		    mtd->writesize != SZ_2K) {
+			dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
+				mtd->writesize);
+			return -EINVAL;
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
+static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
+
+static struct nand_bbt_descr bbt_main_descr = {
+	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
+		   NAND_BBT_2BIT | NAND_BBT_VERSION,
+	.offs =	8,
+	.len = 6,
+	.veroffs = 14,
+	.maxblocks = 8,	/* Last 8 blocks in each chip */
+	.pattern = bbt_pattern
+};
+
+static struct nand_bbt_descr bbt_mirror_descr = {
+	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
+		   NAND_BBT_2BIT | NAND_BBT_VERSION,
+	.offs =	8,
+	.len = 6,
+	.veroffs = 14,
+	.maxblocks = 8,	/* Last 8 blocks in each chip */
+	.pattern = bbt_mirror_pattern
+};
+
+static int marvell_nfc_setup_data_interface(struct mtd_info *mtd, int chipnr,
+					    const struct nand_data_interface
+					    *conf)
+{
+	struct nand_chip *chip = mtd_to_nand(mtd);
+	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
+	struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
+	unsigned int period_ns = 1000000000 / clk_get_rate(nfc->ecc_clk) * 2;
+	const struct nand_sdr_timings *sdr;
+	struct marvell_nfc_timings nfc_tmg;
+	int read_delay;
+
+	sdr = nand_get_sdr_timings(conf);
+	if (IS_ERR(sdr))
+		return PTR_ERR(sdr);
+
+	/*
+	 * SDR timings are given in pico-seconds while NFC timings must be
+	 * expressed in NAND controller clock cycles, which is half of the
+	 * frequency of the accessible ECC clock retrieved by clk_get_rate().
+	 * This is not written anywhere in the datasheet but was observed
+	 * with an oscilloscope.
+	 *
+	 * NFC datasheet gives equations from which thoses calculations
+	 * are derived, they tend to be slightly more restrictives than the
+	 * given core timings and may improve the overall speed.
+	 */
+	nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
+	nfc_tmg.tRH = nfc_tmg.tRP;
+	nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
+	nfc_tmg.tWH = nfc_tmg.tWP;
+	nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
+	nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
+	nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
+	/*
+	 * Read delay is the time of propagation from SoC pins to NFC internal
+	 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
+	 * EDO mode, an additional delay of tRH must be taken into account so
+	 * the data is sampled on the falling edge instead of the rising edge.
+	 */
+	read_delay = sdr->tRC_min >= 30000 ?
+		MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
+
+	nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
+	/*
+	 * tWHR and tRHW are supposed to be read to write delays (and vice
+	 * versa) but in some cases, ie. when doing a change column, they must
+	 * be greater than that to be sure tCCS delay is respected.
+	 */
+	nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
+				 period_ns) - 2,
+	nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
+				 period_ns);
+
+	/* Use WAIT_MODE (wait for RB line) instead of only relying on delays */
+	nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
+
+	if (chipnr < 0)
+		return 0;
+
+	marvell_nand->ndtr0 =
+		NDTR0_TRP(nfc_tmg.tRP) |
+		NDTR0_TRH(nfc_tmg.tRH) |
+		NDTR0_ETRP(nfc_tmg.tRP) |
+		NDTR0_TWP(nfc_tmg.tWP) |
+		NDTR0_TWH(nfc_tmg.tWH) |
+		NDTR0_TCS(nfc_tmg.tCS) |
+		NDTR0_TCH(nfc_tmg.tCH) |
+		NDTR0_RD_CNT_DEL(read_delay) |
+		NDTR0_SELCNTR |
+		NDTR0_TADL(nfc_tmg.tADL);
+
+	marvell_nand->ndtr1 =
+		NDTR1_TAR(nfc_tmg.tAR) |
+		NDTR1_TWHR(nfc_tmg.tWHR) |
+		NDTR1_TRHW(nfc_tmg.tRHW) |
+		NDTR1_WAIT_MODE |
+		NDTR1_TR(nfc_tmg.tR);
+
+	return 0;
+}
+
+static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
+				  struct device_node *np)
+{
+	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
+	struct marvell_nand_chip *marvell_nand;
+	struct mtd_info *mtd;
+	struct nand_chip *chip;
+	int nsels, ret, i;
+	u32 cs, rb;
+
+	/*
+	 * The legacy "num-cs" property indicates the number of CS on the only
+	 * chip connected to the controller (legacy bindings does not support
+	 * more than one chip). CS are only incremented one by one while the RB
+	 * pin is always the #0.
+	 *
+	 * When not using legacy bindings, a couple of "reg" and "nand-rb"
+	 * properties must be filled. For each chip, expressed as a subnode,
+	 * "reg" points to the CS lines and "nand-rb" to the RB line.
+	 */
+	if (pdata) {
+		nsels = 1;
+	} else if (nfc->caps->legacy_of_bindings &&
+		   !of_get_property(np, "num-cs", &nsels)) {
+		dev_err(dev, "missing num-cs property\n");
+		return -EINVAL;
+	} else if (!of_get_property(np, "reg", &nsels)) {
+		dev_err(dev, "missing reg property\n");
+		return -EINVAL;
+	}
+
+	if (!pdata)
+		nsels /= sizeof(u32);
+	if (!nsels) {
+		dev_err(dev, "invalid reg property size\n");
+		return -EINVAL;
+	}
+
+	/* Alloc the nand chip structure */
+	marvell_nand = devm_kzalloc(dev, sizeof(*marvell_nand) +
+				    (nsels *
+				     sizeof(struct marvell_nand_chip_sel)),
+				    GFP_KERNEL);
+	if (!marvell_nand) {
+		dev_err(dev, "could not allocate chip structure\n");
+		return -ENOMEM;
+	}
+
+	marvell_nand->nsels = nsels;
+	marvell_nand->selected_die = -1;
+
+	for (i = 0; i < nsels; i++) {
+		if (pdata || nfc->caps->legacy_of_bindings) {
+			/*
+			 * Legacy bindings use the CS lines in natural
+			 * order (0, 1, ...)
+			 */
+			cs = i;
+		} else {
+			/* Retrieve CS id */
+			ret = of_property_read_u32_index(np, "reg", i, &cs);
+			if (ret) {
+				dev_err(dev, "could not retrieve reg property: %d\n",
+					ret);
+				return ret;
+			}
+		}
+
+		if (cs >= nfc->caps->max_cs_nb) {
+			dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
+				cs, nfc->caps->max_cs_nb);
+			return -EINVAL;
+		}
+
+		if (test_and_set_bit(cs, &nfc->assigned_cs)) {
+			dev_err(dev, "CS %d already assigned\n", cs);
+			return -EINVAL;
+		}
+
+		/*
+		 * The cs variable represents the chip select id, which must be
+		 * converted in bit fields for NDCB0 and NDCB2 to select the
+		 * right chip. Unfortunately, due to a lack of information on
+		 * the subject and incoherent documentation, the user should not
+		 * use CS1 and CS3 at all as asserting them is not supported in
+		 * a reliable way (due to multiplexing inside ADDR5 field).
+		 */
+		marvell_nand->sels[i].cs = cs;
+		switch (cs) {
+		case 0:
+		case 2:
+			marvell_nand->sels[i].ndcb0_csel = 0;
+			break;
+		case 1:
+		case 3:
+			marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
+			break;
+		default:
+			return -EINVAL;
+		}
+
+		/* Retrieve RB id */
+		if (pdata || nfc->caps->legacy_of_bindings) {
+			/* Legacy bindings always use RB #0 */
+			rb = 0;
+		} else {
+			ret = of_property_read_u32_index(np, "nand-rb", i,
+							 &rb);
+			if (ret) {
+				dev_err(dev,
+					"could not retrieve RB property: %d\n",
+					ret);
+				return ret;
+			}
+		}
+
+		if (rb >= nfc->caps->max_rb_nb) {
+			dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
+				rb, nfc->caps->max_rb_nb);
+			return -EINVAL;
+		}
+
+		marvell_nand->sels[i].rb = rb;
+	}
+
+	chip = &marvell_nand->chip;
+	chip->controller = &nfc->controller;
+	nand_set_flash_node(chip, np);
+
+	chip->exec_op = marvell_nfc_exec_op;
+	chip->select_chip = marvell_nfc_select_chip;
+	if (nfc->caps->is_nfcv2 &&
+	    !of_property_read_bool(np, "marvell,nand-keep-config"))
+		chip->setup_data_interface = marvell_nfc_setup_data_interface;
+
+	mtd = nand_to_mtd(chip);
+	mtd->dev.parent = dev;
+
+	/*
+	 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
+	 * in the DT node, this entry will be overwritten in nand_scan_ident().
+	 */
+	chip->ecc.mode = NAND_ECC_HW;
+
+	/*
+	 * Save a reference value for timing registers before
+	 * ->setup_data_interface() is called.
+	 */
+	marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
+	marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
+
+	chip->options |= NAND_BUSWIDTH_AUTO;
+	ret = nand_scan_ident(mtd, marvell_nand->nsels, NULL);
+	if (ret) {
+		dev_err(dev, "could not identify the nand chip\n");
+		return ret;
+	}
+
+	if (pdata && pdata->flash_bbt)
+		chip->bbt_options |= NAND_BBT_USE_FLASH;
+
+	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
+		/*
+		 * We'll use a bad block table stored in-flash and don't
+		 * allow writing the bad block marker to the flash.
+		 */
+		chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
+		chip->bbt_td = &bbt_main_descr;
+		chip->bbt_md = &bbt_mirror_descr;
+	}
+
+	/* Save the chip-specific fields of NDCR */
+	marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
+	if (chip->options & NAND_BUSWIDTH_16)
+		marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
+
+	/*
+	 * On small page NANDs, only one cycle is needed to pass the
+	 * column address.
+	 */
+	if (mtd->writesize <= 512) {
+		marvell_nand->addr_cyc = 1;
+	} else {
+		marvell_nand->addr_cyc = 2;
+		marvell_nand->ndcr |= NDCR_RA_START;
+	}
+
+	/*
+	 * Now add the number of cycles needed to pass the row
+	 * address.
+	 *
+	 * Addressing a chip using CS 2 or 3 should also need the third row
+	 * cycle but due to inconsistance in the documentation and lack of
+	 * hardware to test this situation, this case is not supported.
+	 */
+	if (chip->options & NAND_ROW_ADDR_3)
+		marvell_nand->addr_cyc += 3;
+	else
+		marvell_nand->addr_cyc += 2;
+
+	if (pdata) {
+		chip->ecc.size = pdata->ecc_step_size;
+		chip->ecc.strength = pdata->ecc_strength;
+	}
+
+	ret = marvell_nand_ecc_init(mtd, &chip->ecc);
+	if (ret) {
+		dev_err(dev, "ECC init failed: %d\n", ret);
+		return ret;
+	}
+
+	if (chip->ecc.mode == NAND_ECC_HW) {
+		/*
+		 * Subpage write not available with hardware ECC, prohibit also
+		 * subpage read as in userspace subpage access would still be
+		 * allowed and subpage write, if used, would lead to numerous
+		 * uncorrectable ECC errors.
+		 */
+		chip->options |= NAND_NO_SUBPAGE_WRITE;
+	}
+
+	if (pdata || nfc->caps->legacy_of_bindings) {
+		/*
+		 * We keep the MTD name unchanged to avoid breaking platforms
+		 * where the MTD cmdline parser is used and the bootloader
+		 * has not been updated to use the new naming scheme.
+		 */
+		mtd->name = "pxa3xx_nand-0";
+	} else if (!mtd->name) {
+		/*
+		 * If the new bindings are used and the bootloader has not been
+		 * updated to pass a new mtdparts parameter on the cmdline, you
+		 * should define the following property in your NAND node, ie:
+		 *
+		 *	label = "main-storage";
+		 *
+		 * This way, mtd->name will be set by the core when
+		 * nand_set_flash_node() is called.
+		 */
+		mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
+					   "%s:nand.%d", dev_name(nfc->dev),
+					   marvell_nand->sels[0].cs);
+		if (!mtd->name) {
+			dev_err(nfc->dev, "Failed to allocate mtd->name\n");
+			return -ENOMEM;
+		}
+	}
+
+	ret = nand_scan_tail(mtd);
+	if (ret) {
+		dev_err(dev, "nand_scan_tail failed: %d\n", ret);
+		return ret;
+	}
+
+	if (pdata)
+		/* Legacy bindings support only one chip */
+		ret = mtd_device_register(mtd, pdata->parts[0],
+					  pdata->nr_parts[0]);
+	else
+		ret = mtd_device_register(mtd, NULL, 0);
+	if (ret) {
+		dev_err(dev, "failed to register mtd device: %d\n", ret);
+		nand_release(mtd);
+		return ret;
+	}
+
+	list_add_tail(&marvell_nand->node, &nfc->chips);
+
+	return 0;
+}
+
+static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
+{
+	struct device_node *np = dev->of_node;
+	struct device_node *nand_np;
+	int max_cs = nfc->caps->max_cs_nb;
+	int nchips;
+	int ret;
+
+	if (!np)
+		nchips = 1;
+	else
+		nchips = of_get_child_count(np);
+
+	if (nchips > max_cs) {
+		dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
+			max_cs);
+		return -EINVAL;
+	}
+
+	/*
+	 * Legacy bindings do not use child nodes to exhibit NAND chip
+	 * properties and layout. Instead, NAND properties are mixed with the
+	 * controller ones, and partitions are defined as direct subnodes of the
+	 * NAND controller node.
+	 */
+	if (nfc->caps->legacy_of_bindings) {
+		ret = marvell_nand_chip_init(dev, nfc, np);
+		return ret;
+	}
+
+	for_each_child_of_node(np, nand_np) {
+		ret = marvell_nand_chip_init(dev, nfc, nand_np);
+		if (ret) {
+			of_node_put(nand_np);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
+{
+	struct marvell_nand_chip *entry, *temp;
+
+	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
+		nand_release(nand_to_mtd(&entry->chip));
+		list_del(&entry->node);
+	}
+}
+
+static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
+{
+	struct platform_device *pdev = container_of(nfc->dev,
+						    struct platform_device,
+						    dev);
+	struct dma_slave_config config = {};
+	struct resource *r;
+	dma_cap_mask_t mask;
+	struct pxad_param param;
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_PXA_DMA)) {
+		dev_warn(nfc->dev,
+			 "DMA not enabled in configuration\n");
+		return -ENOTSUPP;
+	}
+
+	ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
+	r = platform_get_resource(pdev, IORESOURCE_DMA, 0);
+	if (!r) {
+		dev_err(nfc->dev, "No resource defined for data DMA\n");
+		return -ENXIO;
+	}
+
+	param.drcmr = r->start;
+	param.prio = PXAD_PRIO_LOWEST;
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_SLAVE, mask);
+	nfc->dma_chan =
+		dma_request_slave_channel_compat(mask, pxad_filter_fn,
+						 &param, nfc->dev,
+						 "data");
+	if (!nfc->dma_chan) {
+		dev_err(nfc->dev,
+			"Unable to request data DMA channel\n");
+		return -ENODEV;
+	}
+
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!r)
+		return -ENXIO;
+
+	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+	config.src_addr = r->start + NDDB;
+	config.dst_addr = r->start + NDDB;
+	config.src_maxburst = 32;
+	config.dst_maxburst = 32;
+	ret = dmaengine_slave_config(nfc->dma_chan, &config);
+	if (ret < 0) {
+		dev_err(nfc->dev, "Failed to configure DMA channel\n");
+		return ret;
+	}
+
+	/*
+	 * DMA must act on length multiple of 32 and this length may be
+	 * bigger than the destination buffer. Use this buffer instead
+	 * for DMA transfers and then copy the desired amount of data to
+	 * the provided buffer.
+	 */
+	nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
+	if (!nfc->dma_buf)
+		return -ENOMEM;
+
+	nfc->use_dma = true;
+
+	return 0;
+}
+
+static int marvell_nfc_init(struct marvell_nfc *nfc)
+{
+	struct device_node *np = nfc->dev->of_node;
+
+	/*
+	 * Some SoCs like A7k/A8k need to enable manually the NAND
+	 * controller, gated clocks and reset bits to avoid being bootloader
+	 * dependent. This is done through the use of the System Functions
+	 * registers.
+	 */
+	if (nfc->caps->need_system_controller) {
+		struct regmap *sysctrl_base =
+			syscon_regmap_lookup_by_phandle(np,
+							"marvell,system-controller");
+		u32 reg;
+
+		if (IS_ERR(sysctrl_base))
+			return PTR_ERR(sysctrl_base);
+
+		reg = GENCONF_SOC_DEVICE_MUX_NFC_EN |
+		      GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
+		      GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
+		      GENCONF_SOC_DEVICE_MUX_NFC_INT_EN;
+		regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX, reg);
+
+		regmap_read(sysctrl_base, GENCONF_CLK_GATING_CTRL, &reg);
+		reg |= GENCONF_CLK_GATING_CTRL_ND_GATE;
+		regmap_write(sysctrl_base, GENCONF_CLK_GATING_CTRL, reg);
+
+		regmap_read(sysctrl_base, GENCONF_ND_CLK_CTRL, &reg);
+		reg |= GENCONF_ND_CLK_CTRL_EN;
+		regmap_write(sysctrl_base, GENCONF_ND_CLK_CTRL, reg);
+	}
+
+	/* Configure the DMA if appropriate */
+	if (!nfc->caps->is_nfcv2)
+		marvell_nfc_init_dma(nfc);
+
+	/*
+	 * ECC operations and interruptions are only enabled when specifically
+	 * needed. ECC shall not be activated in the early stages (fails probe).
+	 * Arbiter flag, even if marked as "reserved", must be set (empirical).
+	 * SPARE_EN bit must always be set or ECC bytes will not be at the same
+	 * offset in the read page and this will fail the protection.
+	 */
+	writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
+		       NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
+	writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
+	writel_relaxed(0, nfc->regs + NDECCCTRL);
+
+	return 0;
+}
+
+static int marvell_nfc_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *r;
+	struct marvell_nfc *nfc;
+	int ret;
+	int irq;
+
+	nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
+			   GFP_KERNEL);
+	if (!nfc)
+		return -ENOMEM;
+
+	nfc->dev = dev;
+	nand_hw_control_init(&nfc->controller);
+	INIT_LIST_HEAD(&nfc->chips);
+
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	nfc->regs = devm_ioremap_resource(dev, r);
+	if (IS_ERR(nfc->regs))
+		return PTR_ERR(nfc->regs);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "failed to retrieve irq\n");
+		return irq;
+	}
+
+	nfc->ecc_clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(nfc->ecc_clk))
+		return PTR_ERR(nfc->ecc_clk);
+
+	ret = clk_prepare_enable(nfc->ecc_clk);
+	if (ret)
+		return ret;
+
+	marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
+	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
+	ret = devm_request_irq(dev, irq, marvell_nfc_isr,
+			       0, "marvell-nfc", nfc);
+	if (ret)
+		goto unprepare_clk;
+
+	/* Get NAND controller capabilities */
+	if (pdev->id_entry)
+		nfc->caps = (void *)pdev->id_entry->driver_data;
+	else
+		nfc->caps = of_device_get_match_data(&pdev->dev);
+
+	if (!nfc->caps) {
+		dev_err(dev, "Could not retrieve NFC caps\n");
+		ret = -EINVAL;
+		goto unprepare_clk;
+	}
+
+	/* Init the controller and then probe the chips */
+	ret = marvell_nfc_init(nfc);
+	if (ret)
+		goto unprepare_clk;
+
+	platform_set_drvdata(pdev, nfc);
+
+	ret = marvell_nand_chips_init(dev, nfc);
+	if (ret)
+		goto unprepare_clk;
+
+	return 0;
+
+unprepare_clk:
+	clk_disable_unprepare(nfc->ecc_clk);
+
+	return ret;
+}
+
+static int marvell_nfc_remove(struct platform_device *pdev)
+{
+	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
+
+	marvell_nand_chips_cleanup(nfc);
+
+	if (nfc->use_dma) {
+		dmaengine_terminate_all(nfc->dma_chan);
+		dma_release_channel(nfc->dma_chan);
+	}
+
+	clk_disable_unprepare(nfc->ecc_clk);
+
+	return 0;
+}
+
+static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
+	.max_cs_nb = 4,
+	.max_rb_nb = 2,
+	.need_system_controller = true,
+	.is_nfcv2 = true,
+};
+
+static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
+	.max_cs_nb = 4,
+	.max_rb_nb = 2,
+	.is_nfcv2 = true,
+};
+
+static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
+	.max_cs_nb = 2,
+	.max_rb_nb = 1,
+	.use_dma = true,
+};
+
+static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
+	.max_cs_nb = 4,
+	.max_rb_nb = 2,
+	.need_system_controller = true,
+	.legacy_of_bindings = true,
+	.is_nfcv2 = true,
+};
+
+static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
+	.max_cs_nb = 4,
+	.max_rb_nb = 2,
+	.legacy_of_bindings = true,
+	.is_nfcv2 = true,
+};
+
+static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
+	.max_cs_nb = 2,
+	.max_rb_nb = 1,
+	.legacy_of_bindings = true,
+	.use_dma = true,
+};
+
+static const struct platform_device_id marvell_nfc_platform_ids[] = {
+	{
+		.name = "pxa3xx-nand",
+		.driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
+
+static const struct of_device_id marvell_nfc_of_ids[] = {
+	{
+		.compatible = "marvell,armada-8k-nand-controller",
+		.data = &marvell_armada_8k_nfc_caps,
+	},
+	{
+		.compatible = "marvell,armada370-nand-controller",
+		.data = &marvell_armada370_nfc_caps,
+	},
+	{
+		.compatible = "marvell,pxa3xx-nand-controller",
+		.data = &marvell_pxa3xx_nfc_caps,
+	},
+	/* Support for old/deprecated bindings: */
+	{
+		.compatible = "marvell,armada-8k-nand",
+		.data = &marvell_armada_8k_nfc_legacy_caps,
+	},
+	{
+		.compatible = "marvell,armada370-nand",
+		.data = &marvell_armada370_nfc_legacy_caps,
+	},
+	{
+		.compatible = "marvell,pxa3xx-nand",
+		.data = &marvell_pxa3xx_nfc_legacy_caps,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
+
+static struct platform_driver marvell_nfc_driver = {
+	.driver	= {
+		.name		= "marvell-nfc",
+		.of_match_table = marvell_nfc_of_ids,
+	},
+	.id_table = marvell_nfc_platform_ids,
+	.probe = marvell_nfc_probe,
+	.remove	= marvell_nfc_remove,
+};
+module_platform_driver(marvell_nfc_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Marvell NAND controller driver");
diff --git a/drivers/mtd/nand/mtk_ecc.c b/drivers/mtd/nand/mtk_ecc.c
index c51d214d169e..40d86a861a70 100644
--- a/drivers/mtd/nand/mtk_ecc.c
+++ b/drivers/mtd/nand/mtk_ecc.c
@@ -34,34 +34,28 @@
 
 #define ECC_ENCCON		(0x00)
 #define ECC_ENCCNFG		(0x04)
-#define		ECC_MODE_SHIFT		(5)
 #define		ECC_MS_SHIFT		(16)
 #define ECC_ENCDIADDR		(0x08)
 #define ECC_ENCIDLE		(0x0C)
-#define ECC_ENCIRQ_EN		(0x80)
-#define ECC_ENCIRQ_STA		(0x84)
 #define ECC_DECCON		(0x100)
 #define ECC_DECCNFG		(0x104)
 #define		DEC_EMPTY_EN		BIT(31)
 #define		DEC_CNFG_CORRECT	(0x3 << 12)
 #define ECC_DECIDLE		(0x10C)
 #define ECC_DECENUM0		(0x114)
-#define ECC_DECDONE		(0x124)
-#define ECC_DECIRQ_EN		(0x200)
-#define ECC_DECIRQ_STA		(0x204)
 
 #define ECC_TIMEOUT		(500000)
 
 #define ECC_IDLE_REG(op)	((op) == ECC_ENCODE ? ECC_ENCIDLE : ECC_DECIDLE)
 #define ECC_CTL_REG(op)		((op) == ECC_ENCODE ? ECC_ENCCON : ECC_DECCON)
-#define ECC_IRQ_REG(op)		((op) == ECC_ENCODE ? \
-					ECC_ENCIRQ_EN : ECC_DECIRQ_EN)
 
 struct mtk_ecc_caps {
 	u32 err_mask;
 	const u8 *ecc_strength;
+	const u32 *ecc_regs;
 	u8 num_ecc_strength;
-	u32 encode_parity_reg0;
+	u8 ecc_mode_shift;
+	u32 parity_bits;
 	int pg_irq_sel;
 };
 
@@ -89,6 +83,46 @@ static const u8 ecc_strength_mt2712[] = {
 	40, 44, 48, 52, 56, 60, 68, 72, 80
 };
 
+static const u8 ecc_strength_mt7622[] = {
+	4, 6, 8, 10, 12, 14, 16
+};
+
+enum mtk_ecc_regs {
+	ECC_ENCPAR00,
+	ECC_ENCIRQ_EN,
+	ECC_ENCIRQ_STA,
+	ECC_DECDONE,
+	ECC_DECIRQ_EN,
+	ECC_DECIRQ_STA,
+};
+
+static int mt2701_ecc_regs[] = {
+	[ECC_ENCPAR00] =        0x10,
+	[ECC_ENCIRQ_EN] =       0x80,
+	[ECC_ENCIRQ_STA] =      0x84,
+	[ECC_DECDONE] =         0x124,
+	[ECC_DECIRQ_EN] =       0x200,
+	[ECC_DECIRQ_STA] =      0x204,
+};
+
+static int mt2712_ecc_regs[] = {
+	[ECC_ENCPAR00] =        0x300,
+	[ECC_ENCIRQ_EN] =       0x80,
+	[ECC_ENCIRQ_STA] =      0x84,
+	[ECC_DECDONE] =         0x124,
+	[ECC_DECIRQ_EN] =       0x200,
+	[ECC_DECIRQ_STA] =      0x204,
+};
+
+static int mt7622_ecc_regs[] = {
+	[ECC_ENCPAR00] =        0x10,
+	[ECC_ENCIRQ_EN] =       0x30,
+	[ECC_ENCIRQ_STA] =      0x34,
+	[ECC_DECDONE] =         0x11c,
+	[ECC_DECIRQ_EN] =       0x140,
+	[ECC_DECIRQ_STA] =      0x144,
+};
+
 static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc,
 				     enum mtk_ecc_operation op)
 {
@@ -107,32 +141,30 @@ static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc,
 static irqreturn_t mtk_ecc_irq(int irq, void *id)
 {
 	struct mtk_ecc *ecc = id;
-	enum mtk_ecc_operation op;
 	u32 dec, enc;
 
-	dec = readw(ecc->regs + ECC_DECIRQ_STA) & ECC_IRQ_EN;
+	dec = readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECIRQ_STA])
+		    & ECC_IRQ_EN;
 	if (dec) {
-		op = ECC_DECODE;
-		dec = readw(ecc->regs + ECC_DECDONE);
+		dec = readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECDONE]);
 		if (dec & ecc->sectors) {
 			/*
 			 * Clear decode IRQ status once again to ensure that
 			 * there will be no extra IRQ.
 			 */
-			readw(ecc->regs + ECC_DECIRQ_STA);
+			readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECIRQ_STA]);
 			ecc->sectors = 0;
 			complete(&ecc->done);
 		} else {
 			return IRQ_HANDLED;
 		}
 	} else {
-		enc = readl(ecc->regs + ECC_ENCIRQ_STA) & ECC_IRQ_EN;
-		if (enc) {
-			op = ECC_ENCODE;
+		enc = readl(ecc->regs + ecc->caps->ecc_regs[ECC_ENCIRQ_STA])
+		      & ECC_IRQ_EN;
+		if (enc)
 			complete(&ecc->done);
-		} else {
+		else
 			return IRQ_NONE;
-		}
 	}
 
 	return IRQ_HANDLED;
@@ -160,7 +192,7 @@ static int mtk_ecc_config(struct mtk_ecc *ecc, struct mtk_ecc_config *config)
 		/* configure ECC encoder (in bits) */
 		enc_sz = config->len << 3;
 
-		reg = ecc_bit | (config->mode << ECC_MODE_SHIFT);
+		reg = ecc_bit | (config->mode << ecc->caps->ecc_mode_shift);
 		reg |= (enc_sz << ECC_MS_SHIFT);
 		writel(reg, ecc->regs + ECC_ENCCNFG);
 
@@ -171,9 +203,9 @@ static int mtk_ecc_config(struct mtk_ecc *ecc, struct mtk_ecc_config *config)
 	} else {
 		/* configure ECC decoder (in bits) */
 		dec_sz = (config->len << 3) +
-					config->strength * ECC_PARITY_BITS;
+			 config->strength * ecc->caps->parity_bits;
 
-		reg = ecc_bit | (config->mode << ECC_MODE_SHIFT);
+		reg = ecc_bit | (config->mode << ecc->caps->ecc_mode_shift);
 		reg |= (dec_sz << ECC_MS_SHIFT) | DEC_CNFG_CORRECT;
 		reg |= DEC_EMPTY_EN;
 		writel(reg, ecc->regs + ECC_DECCNFG);
@@ -291,7 +323,12 @@ int mtk_ecc_enable(struct mtk_ecc *ecc, struct mtk_ecc_config *config)
 		 */
 		if (ecc->caps->pg_irq_sel && config->mode == ECC_NFI_MODE)
 			reg_val |= ECC_PG_IRQ_SEL;
-		writew(reg_val, ecc->regs + ECC_IRQ_REG(op));
+		if (op == ECC_ENCODE)
+			writew(reg_val, ecc->regs +
+			       ecc->caps->ecc_regs[ECC_ENCIRQ_EN]);
+		else
+			writew(reg_val, ecc->regs +
+			       ecc->caps->ecc_regs[ECC_DECIRQ_EN]);
 	}
 
 	writew(ECC_OP_ENABLE, ecc->regs + ECC_CTL_REG(op));
@@ -310,13 +347,17 @@ void mtk_ecc_disable(struct mtk_ecc *ecc)
 
 	/* disable it */
 	mtk_ecc_wait_idle(ecc, op);
-	if (op == ECC_DECODE)
+	if (op == ECC_DECODE) {
 		/*
 		 * Clear decode IRQ status in case there is a timeout to wait
 		 * decode IRQ.
 		 */
-		readw(ecc->regs + ECC_DECIRQ_STA);
-	writew(0, ecc->regs + ECC_IRQ_REG(op));
+		readw(ecc->regs + ecc->caps->ecc_regs[ECC_DECDONE]);
+		writew(0, ecc->regs + ecc->caps->ecc_regs[ECC_DECIRQ_EN]);
+	} else {
+		writew(0, ecc->regs + ecc->caps->ecc_regs[ECC_ENCIRQ_EN]);
+	}
+
 	writew(ECC_OP_DISABLE, ecc->regs + ECC_CTL_REG(op));
 
 	mutex_unlock(&ecc->lock);
@@ -367,11 +408,11 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config,
 	mtk_ecc_wait_idle(ecc, ECC_ENCODE);
 
 	/* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */
-	len = (config->strength * ECC_PARITY_BITS + 7) >> 3;
+	len = (config->strength * ecc->caps->parity_bits + 7) >> 3;
 
 	/* write the parity bytes generated by the ECC back to temp buffer */
 	__ioread32_copy(ecc->eccdata,
-			ecc->regs + ecc->caps->encode_parity_reg0,
+			ecc->regs + ecc->caps->ecc_regs[ECC_ENCPAR00],
 			round_up(len, 4));
 
 	/* copy into possibly unaligned OOB region with actual length */
@@ -404,22 +445,42 @@ void mtk_ecc_adjust_strength(struct mtk_ecc *ecc, u32 *p)
 }
 EXPORT_SYMBOL(mtk_ecc_adjust_strength);
 
+unsigned int mtk_ecc_get_parity_bits(struct mtk_ecc *ecc)
+{
+	return ecc->caps->parity_bits;
+}
+EXPORT_SYMBOL(mtk_ecc_get_parity_bits);
+
 static const struct mtk_ecc_caps mtk_ecc_caps_mt2701 = {
 	.err_mask = 0x3f,
 	.ecc_strength = ecc_strength_mt2701,
+	.ecc_regs = mt2701_ecc_regs,
 	.num_ecc_strength = 20,
-	.encode_parity_reg0 = 0x10,
+	.ecc_mode_shift = 5,
+	.parity_bits = 14,
 	.pg_irq_sel = 0,
 };
 
 static const struct mtk_ecc_caps mtk_ecc_caps_mt2712 = {
 	.err_mask = 0x7f,
 	.ecc_strength = ecc_strength_mt2712,
+	.ecc_regs = mt2712_ecc_regs,
 	.num_ecc_strength = 23,
-	.encode_parity_reg0 = 0x300,
+	.ecc_mode_shift = 5,
+	.parity_bits = 14,
 	.pg_irq_sel = 1,
 };
 
+static const struct mtk_ecc_caps mtk_ecc_caps_mt7622 = {
+	.err_mask = 0x3f,
+	.ecc_strength = ecc_strength_mt7622,
+	.ecc_regs = mt7622_ecc_regs,
+	.num_ecc_strength = 7,
+	.ecc_mode_shift = 4,
+	.parity_bits = 13,
+	.pg_irq_sel = 0,
+};
+
 static const struct of_device_id mtk_ecc_dt_match[] = {
 	{
 		.compatible = "mediatek,mt2701-ecc",
@@ -427,6 +488,9 @@ static const struct of_device_id mtk_ecc_dt_match[] = {
 	}, {
 		.compatible = "mediatek,mt2712-ecc",
 		.data = &mtk_ecc_caps_mt2712,
+	}, {
+		.compatible = "mediatek,mt7622-ecc",
+		.data = &mtk_ecc_caps_mt7622,
 	},
 	{},
 };
@@ -452,7 +516,7 @@ static int mtk_ecc_probe(struct platform_device *pdev)
 
 	max_eccdata_size = ecc->caps->num_ecc_strength - 1;
 	max_eccdata_size = ecc->caps->ecc_strength[max_eccdata_size];
-	max_eccdata_size = (max_eccdata_size * ECC_PARITY_BITS + 7) >> 3;
+	max_eccdata_size = (max_eccdata_size * ecc->caps->parity_bits + 7) >> 3;
 	max_eccdata_size = round_up(max_eccdata_size, 4);
 	ecc->eccdata = devm_kzalloc(dev, max_eccdata_size, GFP_KERNEL);
 	if (!ecc->eccdata)
diff --git a/drivers/mtd/nand/mtk_ecc.h b/drivers/mtd/nand/mtk_ecc.h
index d245c14f1b80..a455df080952 100644
--- a/drivers/mtd/nand/mtk_ecc.h
+++ b/drivers/mtd/nand/mtk_ecc.h
@@ -14,8 +14,6 @@
 
 #include <linux/types.h>
 
-#define ECC_PARITY_BITS		(14)
-
 enum mtk_ecc_mode {ECC_DMA_MODE = 0, ECC_NFI_MODE = 1};
 enum mtk_ecc_operation {ECC_ENCODE, ECC_DECODE};
 
@@ -43,6 +41,7 @@ int mtk_ecc_wait_done(struct mtk_ecc *, enum mtk_ecc_operation);
 int mtk_ecc_enable(struct mtk_ecc *, struct mtk_ecc_config *);
 void mtk_ecc_disable(struct mtk_ecc *);
 void mtk_ecc_adjust_strength(struct mtk_ecc *ecc, u32 *p);
+unsigned int mtk_ecc_get_parity_bits(struct mtk_ecc *ecc);
 
 struct mtk_ecc *of_mtk_ecc_get(struct device_node *);
 void mtk_ecc_release(struct mtk_ecc *);
diff --git a/drivers/mtd/nand/mtk_nand.c b/drivers/mtd/nand/mtk_nand.c
index d86a7d131cc0..6977da3a26aa 100644
--- a/drivers/mtd/nand/mtk_nand.c
+++ b/drivers/mtd/nand/mtk_nand.c
@@ -97,7 +97,6 @@
 
 #define MTK_TIMEOUT		(500000)
 #define MTK_RESET_TIMEOUT	(1000000)
-#define MTK_MAX_SECTOR		(16)
 #define MTK_NAND_MAX_NSELS	(2)
 #define MTK_NFC_MIN_SPARE	(16)
 #define ACCTIMING(tpoecs, tprecs, tc2r, tw2r, twh, twst, trlt) \
@@ -109,6 +108,8 @@ struct mtk_nfc_caps {
 	u8 num_spare_size;
 	u8 pageformat_spare_shift;
 	u8 nfi_clk_div;
+	u8 max_sector;
+	u32 max_sector_size;
 };
 
 struct mtk_nfc_bad_mark_ctl {
@@ -173,6 +174,10 @@ static const u8 spare_size_mt2712[] = {
 	74
 };
 
+static const u8 spare_size_mt7622[] = {
+	16, 26, 27, 28
+};
+
 static inline struct mtk_nfc_nand_chip *to_mtk_nand(struct nand_chip *nand)
 {
 	return container_of(nand, struct mtk_nfc_nand_chip, nand);
@@ -450,7 +455,7 @@ static inline u8 mtk_nfc_read_byte(struct mtd_info *mtd)
 		 * set to max sector to allow the HW to continue reading over
 		 * unaligned accesses
 		 */
-		reg = (MTK_MAX_SECTOR << CON_SEC_SHIFT) | CON_BRD;
+		reg = (nfc->caps->max_sector << CON_SEC_SHIFT) | CON_BRD;
 		nfi_writel(nfc, reg, NFI_CON);
 
 		/* trigger to fetch data */
@@ -481,7 +486,7 @@ static void mtk_nfc_write_byte(struct mtd_info *mtd, u8 byte)
 		reg = nfi_readw(nfc, NFI_CNFG) | CNFG_BYTE_RW;
 		nfi_writew(nfc, reg, NFI_CNFG);
 
-		reg = MTK_MAX_SECTOR << CON_SEC_SHIFT | CON_BWR;
+		reg = nfc->caps->max_sector << CON_SEC_SHIFT | CON_BWR;
 		nfi_writel(nfc, reg, NFI_CON);
 
 		nfi_writew(nfc, STAR_EN, NFI_STRDATA);
@@ -761,6 +766,8 @@ static int mtk_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	u32 reg;
 	int ret;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	if (!raw) {
 		/* OOB => FDM: from register,  ECC: from HW */
 		reg = nfi_readw(nfc, NFI_CNFG) | CNFG_AUTO_FMT_EN;
@@ -794,7 +801,10 @@ static int mtk_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	if (!raw)
 		mtk_ecc_disable(nfc->ecc);
 
-	return ret;
+	if (ret)
+		return ret;
+
+	return nand_prog_page_end_op(chip);
 }
 
 static int mtk_nfc_write_page_hwecc(struct mtd_info *mtd,
@@ -832,18 +842,7 @@ static int mtk_nfc_write_subpage_hwecc(struct mtd_info *mtd,
 static int mtk_nfc_write_oob_std(struct mtd_info *mtd, struct nand_chip *chip,
 				 int page)
 {
-	int ret;
-
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
-
-	ret = mtk_nfc_write_page_raw(mtd, chip, NULL, 1, page);
-	if (ret < 0)
-		return -EIO;
-
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-	ret = chip->waitfunc(mtd, chip);
-
-	return ret & NAND_STATUS_FAIL ? -EIO : 0;
+	return mtk_nfc_write_page_raw(mtd, chip, NULL, 1, page);
 }
 
 static int mtk_nfc_update_ecc_stats(struct mtd_info *mtd, u8 *buf, u32 sectors)
@@ -892,8 +891,7 @@ static int mtk_nfc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 	len = sectors * chip->ecc.size + (raw ? sectors * spare : 0);
 	buf = bufpoi + start * chip->ecc.size;
 
-	if (column != 0)
-		chip->cmdfunc(mtd, NAND_CMD_RNDOUT, column, -1);
+	nand_read_page_op(chip, page, column, NULL, 0);
 
 	addr = dma_map_single(nfc->dev, buf, len, DMA_FROM_DEVICE);
 	rc = dma_mapping_error(nfc->dev, addr);
@@ -1016,8 +1014,6 @@ static int mtk_nfc_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 static int mtk_nfc_read_oob_std(struct mtd_info *mtd, struct nand_chip *chip,
 				int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
-
 	return mtk_nfc_read_page_raw(mtd, chip, NULL, 1, page);
 }
 
@@ -1126,9 +1122,11 @@ static void mtk_nfc_set_fdm(struct mtk_nfc_fdm *fdm, struct mtd_info *mtd)
 {
 	struct nand_chip *nand = mtd_to_nand(mtd);
 	struct mtk_nfc_nand_chip *chip = to_mtk_nand(nand);
+	struct mtk_nfc *nfc = nand_get_controller_data(nand);
 	u32 ecc_bytes;
 
-	ecc_bytes = DIV_ROUND_UP(nand->ecc.strength * ECC_PARITY_BITS, 8);
+	ecc_bytes = DIV_ROUND_UP(nand->ecc.strength *
+				 mtk_ecc_get_parity_bits(nfc->ecc), 8);
 
 	fdm->reg_size = chip->spare_per_sector - ecc_bytes;
 	if (fdm->reg_size > NFI_FDM_MAX_SIZE)
@@ -1208,7 +1206,8 @@ static int mtk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
 		 * this controller only supports 512 and 1024 sizes
 		 */
 		if (nand->ecc.size < 1024) {
-			if (mtd->writesize > 512) {
+			if (mtd->writesize > 512 &&
+			    nfc->caps->max_sector_size > 512) {
 				nand->ecc.size = 1024;
 				nand->ecc.strength <<= 1;
 			} else {
@@ -1223,7 +1222,8 @@ static int mtk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
 			return ret;
 
 		/* calculate oob bytes except ecc parity data */
-		free = ((nand->ecc.strength * ECC_PARITY_BITS) + 7) >> 3;
+		free = (nand->ecc.strength * mtk_ecc_get_parity_bits(nfc->ecc)
+			+ 7) >> 3;
 		free = spare - free;
 
 		/*
@@ -1233,10 +1233,12 @@ static int mtk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
 		 */
 		if (free > NFI_FDM_MAX_SIZE) {
 			spare -= NFI_FDM_MAX_SIZE;
-			nand->ecc.strength = (spare << 3) / ECC_PARITY_BITS;
+			nand->ecc.strength = (spare << 3) /
+					     mtk_ecc_get_parity_bits(nfc->ecc);
 		} else if (free < 0) {
 			spare -= NFI_FDM_MIN_SIZE;
-			nand->ecc.strength = (spare << 3) / ECC_PARITY_BITS;
+			nand->ecc.strength = (spare << 3) /
+					     mtk_ecc_get_parity_bits(nfc->ecc);
 		}
 	}
 
@@ -1389,6 +1391,8 @@ static const struct mtk_nfc_caps mtk_nfc_caps_mt2701 = {
 	.num_spare_size = 16,
 	.pageformat_spare_shift = 4,
 	.nfi_clk_div = 1,
+	.max_sector = 16,
+	.max_sector_size = 1024,
 };
 
 static const struct mtk_nfc_caps mtk_nfc_caps_mt2712 = {
@@ -1396,6 +1400,17 @@ static const struct mtk_nfc_caps mtk_nfc_caps_mt2712 = {
 	.num_spare_size = 19,
 	.pageformat_spare_shift = 16,
 	.nfi_clk_div = 2,
+	.max_sector = 16,
+	.max_sector_size = 1024,
+};
+
+static const struct mtk_nfc_caps mtk_nfc_caps_mt7622 = {
+	.spare_size = spare_size_mt7622,
+	.num_spare_size = 4,
+	.pageformat_spare_shift = 4,
+	.nfi_clk_div = 1,
+	.max_sector = 8,
+	.max_sector_size = 512,
 };
 
 static const struct of_device_id mtk_nfc_id_table[] = {
@@ -1405,6 +1420,9 @@ static const struct of_device_id mtk_nfc_id_table[] = {
 	}, {
 		.compatible = "mediatek,mt2712-nfc",
 		.data = &mtk_nfc_caps_mt2712,
+	}, {
+		.compatible = "mediatek,mt7622-nfc",
+		.data = &mtk_nfc_caps_mt7622,
 	},
 	{}
 };
@@ -1540,7 +1558,6 @@ static int mtk_nfc_resume(struct device *dev)
 	struct mtk_nfc *nfc = dev_get_drvdata(dev);
 	struct mtk_nfc_nand_chip *chip;
 	struct nand_chip *nand;
-	struct mtd_info *mtd;
 	int ret;
 	u32 i;
 
@@ -1553,11 +1570,8 @@ static int mtk_nfc_resume(struct device *dev)
 	/* reset NAND chip if VCC was powered off */
 	list_for_each_entry(chip, &nfc->chips, node) {
 		nand = &chip->nand;
-		mtd = nand_to_mtd(nand);
-		for (i = 0; i < chip->nsels; i++) {
-			nand->select_chip(mtd, i);
-			nand->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
-		}
+		for (i = 0; i < chip->nsels; i++)
+			nand_reset(nand, i);
 	}
 
 	return 0;
diff --git a/drivers/mtd/nand/nand_base.c b/drivers/mtd/nand/nand_base.c
index 6135d007a068..e70ca16a5118 100644
--- a/drivers/mtd/nand/nand_base.c
+++ b/drivers/mtd/nand/nand_base.c
@@ -561,14 +561,19 @@ static int nand_block_markbad_lowlevel(struct mtd_info *mtd, loff_t ofs)
 static int nand_check_wp(struct mtd_info *mtd)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
+	u8 status;
+	int ret;
 
 	/* Broken xD cards report WP despite being writable */
 	if (chip->options & NAND_BROKEN_XD)
 		return 0;
 
 	/* Check the WP bit */
-	chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
-	return (chip->read_byte(mtd) & NAND_STATUS_WP) ? 0 : 1;
+	ret = nand_status_op(chip, &status);
+	if (ret)
+		return ret;
+
+	return status & NAND_STATUS_WP ? 0 : 1;
 }
 
 /**
@@ -667,15 +672,82 @@ EXPORT_SYMBOL_GPL(nand_wait_ready);
 static void nand_wait_status_ready(struct mtd_info *mtd, unsigned long timeo)
 {
 	register struct nand_chip *chip = mtd_to_nand(mtd);
+	int ret;
 
 	timeo = jiffies + msecs_to_jiffies(timeo);
 	do {
-		if ((chip->read_byte(mtd) & NAND_STATUS_READY))
+		u8 status;
+
+		ret = nand_read_data_op(chip, &status, sizeof(status), true);
+		if (ret)
+			return;
+
+		if (status & NAND_STATUS_READY)
 			break;
 		touch_softlockup_watchdog();
 	} while (time_before(jiffies, timeo));
 };
 
+/**
+ * nand_soft_waitrdy - Poll STATUS reg until RDY bit is set to 1
+ * @chip: NAND chip structure
+ * @timeout_ms: Timeout in ms
+ *
+ * Poll the STATUS register using ->exec_op() until the RDY bit becomes 1.
+ * If that does not happen whitin the specified timeout, -ETIMEDOUT is
+ * returned.
+ *
+ * This helper is intended to be used when the controller does not have access
+ * to the NAND R/B pin.
+ *
+ * Be aware that calling this helper from an ->exec_op() implementation means
+ * ->exec_op() must be re-entrant.
+ *
+ * Return 0 if the NAND chip is ready, a negative error otherwise.
+ */
+int nand_soft_waitrdy(struct nand_chip *chip, unsigned long timeout_ms)
+{
+	u8 status = 0;
+	int ret;
+
+	if (!chip->exec_op)
+		return -ENOTSUPP;
+
+	ret = nand_status_op(chip, NULL);
+	if (ret)
+		return ret;
+
+	timeout_ms = jiffies + msecs_to_jiffies(timeout_ms);
+	do {
+		ret = nand_read_data_op(chip, &status, sizeof(status), true);
+		if (ret)
+			break;
+
+		if (status & NAND_STATUS_READY)
+			break;
+
+		/*
+		 * Typical lowest execution time for a tR on most NANDs is 10us,
+		 * use this as polling delay before doing something smarter (ie.
+		 * deriving a delay from the timeout value, timeout_ms/ratio).
+		 */
+		udelay(10);
+	} while	(time_before(jiffies, timeout_ms));
+
+	/*
+	 * We have to exit READ_STATUS mode in order to read real data on the
+	 * bus in case the WAITRDY instruction is preceding a DATA_IN
+	 * instruction.
+	 */
+	nand_exit_status_op(chip);
+
+	if (ret)
+		return ret;
+
+	return status & NAND_STATUS_READY ? 0 : -ETIMEDOUT;
+};
+EXPORT_SYMBOL_GPL(nand_soft_waitrdy);
+
 /**
  * nand_command - [DEFAULT] Send command to NAND device
  * @mtd: MTD device structure
@@ -710,7 +782,8 @@ static void nand_command(struct mtd_info *mtd, unsigned int command,
 		chip->cmd_ctrl(mtd, readcmd, ctrl);
 		ctrl &= ~NAND_CTRL_CHANGE;
 	}
-	chip->cmd_ctrl(mtd, command, ctrl);
+	if (command != NAND_CMD_NONE)
+		chip->cmd_ctrl(mtd, command, ctrl);
 
 	/* Address cycle, when necessary */
 	ctrl = NAND_CTRL_ALE | NAND_CTRL_CHANGE;
@@ -738,6 +811,7 @@ static void nand_command(struct mtd_info *mtd, unsigned int command,
 	 */
 	switch (command) {
 
+	case NAND_CMD_NONE:
 	case NAND_CMD_PAGEPROG:
 	case NAND_CMD_ERASE1:
 	case NAND_CMD_ERASE2:
@@ -802,8 +876,8 @@ static void nand_ccs_delay(struct nand_chip *chip)
 	 * Wait tCCS_min if it is correctly defined, otherwise wait 500ns
 	 * (which should be safe for all NANDs).
 	 */
-	if (chip->data_interface && chip->data_interface->timings.sdr.tCCS_min)
-		ndelay(chip->data_interface->timings.sdr.tCCS_min / 1000);
+	if (chip->setup_data_interface)
+		ndelay(chip->data_interface.timings.sdr.tCCS_min / 1000);
 	else
 		ndelay(500);
 }
@@ -831,7 +905,9 @@ static void nand_command_lp(struct mtd_info *mtd, unsigned int command,
 	}
 
 	/* Command latch cycle */
-	chip->cmd_ctrl(mtd, command, NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE);
+	if (command != NAND_CMD_NONE)
+		chip->cmd_ctrl(mtd, command,
+			       NAND_NCE | NAND_CLE | NAND_CTRL_CHANGE);
 
 	if (column != -1 || page_addr != -1) {
 		int ctrl = NAND_CTRL_CHANGE | NAND_NCE | NAND_ALE;
@@ -866,6 +942,7 @@ static void nand_command_lp(struct mtd_info *mtd, unsigned int command,
 	 */
 	switch (command) {
 
+	case NAND_CMD_NONE:
 	case NAND_CMD_CACHEDPROG:
 	case NAND_CMD_PAGEPROG:
 	case NAND_CMD_ERASE1:
@@ -1014,7 +1091,15 @@ static void panic_nand_wait(struct mtd_info *mtd, struct nand_chip *chip,
 			if (chip->dev_ready(mtd))
 				break;
 		} else {
-			if (chip->read_byte(mtd) & NAND_STATUS_READY)
+			int ret;
+			u8 status;
+
+			ret = nand_read_data_op(chip, &status, sizeof(status),
+						true);
+			if (ret)
+				return;
+
+			if (status & NAND_STATUS_READY)
 				break;
 		}
 		mdelay(1);
@@ -1031,8 +1116,9 @@ static void panic_nand_wait(struct mtd_info *mtd, struct nand_chip *chip,
 static int nand_wait(struct mtd_info *mtd, struct nand_chip *chip)
 {
 
-	int status;
 	unsigned long timeo = 400;
+	u8 status;
+	int ret;
 
 	/*
 	 * Apply this short delay always to ensure that we do wait tWB in any
@@ -1040,7 +1126,9 @@ static int nand_wait(struct mtd_info *mtd, struct nand_chip *chip)
 	 */
 	ndelay(100);
 
-	chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
+	ret = nand_status_op(chip, NULL);
+	if (ret)
+		return ret;
 
 	if (in_interrupt() || oops_in_progress)
 		panic_nand_wait(mtd, chip, timeo);
@@ -1051,14 +1139,22 @@ static int nand_wait(struct mtd_info *mtd, struct nand_chip *chip)
 				if (chip->dev_ready(mtd))
 					break;
 			} else {
-				if (chip->read_byte(mtd) & NAND_STATUS_READY)
+				ret = nand_read_data_op(chip, &status,
+							sizeof(status), true);
+				if (ret)
+					return ret;
+
+				if (status & NAND_STATUS_READY)
 					break;
 			}
 			cond_resched();
 		} while (time_before(jiffies, timeo));
 	}
 
-	status = (int)chip->read_byte(mtd);
+	ret = nand_read_data_op(chip, &status, sizeof(status), true);
+	if (ret)
+		return ret;
+
 	/* This can happen if in case of timeout or buggy dev_ready */
 	WARN_ON(!(status & NAND_STATUS_READY));
 	return status;
@@ -1076,7 +1172,6 @@ static int nand_wait(struct mtd_info *mtd, struct nand_chip *chip)
 static int nand_reset_data_interface(struct nand_chip *chip, int chipnr)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	const struct nand_data_interface *conf;
 	int ret;
 
 	if (!chip->setup_data_interface)
@@ -1096,8 +1191,8 @@ static int nand_reset_data_interface(struct nand_chip *chip, int chipnr)
 	 * timings to timing mode 0.
 	 */
 
-	conf = nand_get_default_data_interface();
-	ret = chip->setup_data_interface(mtd, chipnr, conf);
+	onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0);
+	ret = chip->setup_data_interface(mtd, chipnr, &chip->data_interface);
 	if (ret)
 		pr_err("Failed to configure data interface to SDR timing mode 0\n");
 
@@ -1122,7 +1217,7 @@ static int nand_setup_data_interface(struct nand_chip *chip, int chipnr)
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	int ret;
 
-	if (!chip->setup_data_interface || !chip->data_interface)
+	if (!chip->setup_data_interface)
 		return 0;
 
 	/*
@@ -1143,7 +1238,7 @@ static int nand_setup_data_interface(struct nand_chip *chip, int chipnr)
 			goto err;
 	}
 
-	ret = chip->setup_data_interface(mtd, chipnr, chip->data_interface);
+	ret = chip->setup_data_interface(mtd, chipnr, &chip->data_interface);
 err:
 	return ret;
 }
@@ -1183,21 +1278,19 @@ static int nand_init_data_interface(struct nand_chip *chip)
 		modes = GENMASK(chip->onfi_timing_mode_default, 0);
 	}
 
-	chip->data_interface = kzalloc(sizeof(*chip->data_interface),
-				       GFP_KERNEL);
-	if (!chip->data_interface)
-		return -ENOMEM;
 
 	for (mode = fls(modes) - 1; mode >= 0; mode--) {
-		ret = onfi_init_data_interface(chip, chip->data_interface,
-					       NAND_SDR_IFACE, mode);
+		ret = onfi_fill_data_interface(chip, NAND_SDR_IFACE, mode);
 		if (ret)
 			continue;
 
-		/* Pass -1 to only */
+		/*
+		 * Pass NAND_DATA_IFACE_CHECK_ONLY to only check if the
+		 * controller supports the requested timings.
+		 */
 		ret = chip->setup_data_interface(mtd,
 						 NAND_DATA_IFACE_CHECK_ONLY,
-						 chip->data_interface);
+						 &chip->data_interface);
 		if (!ret) {
 			chip->onfi_timing_mode_default = mode;
 			break;
@@ -1207,205 +1300,1626 @@ static int nand_init_data_interface(struct nand_chip *chip)
 	return 0;
 }
 
-static void nand_release_data_interface(struct nand_chip *chip)
-{
-	kfree(chip->data_interface);
-}
-
 /**
- * nand_reset - Reset and initialize a NAND device
+ * nand_fill_column_cycles - fill the column cycles of an address
  * @chip: The NAND chip
- * @chipnr: Internal die id
+ * @addrs: Array of address cycles to fill
+ * @offset_in_page: The offset in the page
+ *
+ * Fills the first or the first two bytes of the @addrs field depending
+ * on the NAND bus width and the page size.
  *
- * Returns 0 for success or negative error code otherwise
+ * Returns the number of cycles needed to encode the column, or a negative
+ * error code in case one of the arguments is invalid.
  */
-int nand_reset(struct nand_chip *chip, int chipnr)
+static int nand_fill_column_cycles(struct nand_chip *chip, u8 *addrs,
+				   unsigned int offset_in_page)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	int ret;
 
-	ret = nand_reset_data_interface(chip, chipnr);
-	if (ret)
-		return ret;
+	/* Make sure the offset is less than the actual page size. */
+	if (offset_in_page > mtd->writesize + mtd->oobsize)
+		return -EINVAL;
 
 	/*
-	 * The CS line has to be released before we can apply the new NAND
-	 * interface settings, hence this weird ->select_chip() dance.
+	 * On small page NANDs, there's a dedicated command to access the OOB
+	 * area, and the column address is relative to the start of the OOB
+	 * area, not the start of the page. Asjust the address accordingly.
 	 */
-	chip->select_chip(mtd, chipnr);
-	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
-	chip->select_chip(mtd, -1);
+	if (mtd->writesize <= 512 && offset_in_page >= mtd->writesize)
+		offset_in_page -= mtd->writesize;
 
-	chip->select_chip(mtd, chipnr);
-	ret = nand_setup_data_interface(chip, chipnr);
-	chip->select_chip(mtd, -1);
-	if (ret)
-		return ret;
+	/*
+	 * The offset in page is expressed in bytes, if the NAND bus is 16-bit
+	 * wide, then it must be divided by 2.
+	 */
+	if (chip->options & NAND_BUSWIDTH_16) {
+		if (WARN_ON(offset_in_page % 2))
+			return -EINVAL;
 
-	return 0;
+		offset_in_page /= 2;
+	}
+
+	addrs[0] = offset_in_page;
+
+	/*
+	 * Small page NANDs use 1 cycle for the columns, while large page NANDs
+	 * need 2
+	 */
+	if (mtd->writesize <= 512)
+		return 1;
+
+	addrs[1] = offset_in_page >> 8;
+
+	return 2;
 }
-EXPORT_SYMBOL_GPL(nand_reset);
 
-/**
- * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data
- * @buf: buffer to test
- * @len: buffer length
- * @bitflips_threshold: maximum number of bitflips
- *
- * Check if a buffer contains only 0xff, which means the underlying region
- * has been erased and is ready to be programmed.
- * The bitflips_threshold specify the maximum number of bitflips before
- * considering the region is not erased.
- * Note: The logic of this function has been extracted from the memweight
- * implementation, except that nand_check_erased_buf function exit before
- * testing the whole buffer if the number of bitflips exceed the
- * bitflips_threshold value.
- *
- * Returns a positive number of bitflips less than or equal to
- * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
- * threshold.
- */
-static int nand_check_erased_buf(void *buf, int len, int bitflips_threshold)
+static int nand_sp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
+				     unsigned int offset_in_page, void *buf,
+				     unsigned int len)
 {
-	const unsigned char *bitmap = buf;
-	int bitflips = 0;
-	int weight;
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	const struct nand_sdr_timings *sdr =
+		nand_get_sdr_timings(&chip->data_interface);
+	u8 addrs[4];
+	struct nand_op_instr instrs[] = {
+		NAND_OP_CMD(NAND_CMD_READ0, 0),
+		NAND_OP_ADDR(3, addrs, PSEC_TO_NSEC(sdr->tWB_max)),
+		NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max),
+				 PSEC_TO_NSEC(sdr->tRR_min)),
+		NAND_OP_DATA_IN(len, buf, 0),
+	};
+	struct nand_operation op = NAND_OPERATION(instrs);
+	int ret;
 
-	for (; len && ((uintptr_t)bitmap) % sizeof(long);
-	     len--, bitmap++) {
-		weight = hweight8(*bitmap);
-		bitflips += BITS_PER_BYTE - weight;
-		if (unlikely(bitflips > bitflips_threshold))
-			return -EBADMSG;
-	}
+	/* Drop the DATA_IN instruction if len is set to 0. */
+	if (!len)
+		op.ninstrs--;
 
-	for (; len >= sizeof(long);
-	     len -= sizeof(long), bitmap += sizeof(long)) {
-		unsigned long d = *((unsigned long *)bitmap);
-		if (d == ~0UL)
-			continue;
-		weight = hweight_long(d);
-		bitflips += BITS_PER_LONG - weight;
-		if (unlikely(bitflips > bitflips_threshold))
-			return -EBADMSG;
+	if (offset_in_page >= mtd->writesize)
+		instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
+	else if (offset_in_page >= 256 &&
+		 !(chip->options & NAND_BUSWIDTH_16))
+		instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
+
+	ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
+	if (ret < 0)
+		return ret;
+
+	addrs[1] = page;
+	addrs[2] = page >> 8;
+
+	if (chip->options & NAND_ROW_ADDR_3) {
+		addrs[3] = page >> 16;
+		instrs[1].ctx.addr.naddrs++;
 	}
 
-	for (; len > 0; len--, bitmap++) {
-		weight = hweight8(*bitmap);
-		bitflips += BITS_PER_BYTE - weight;
-		if (unlikely(bitflips > bitflips_threshold))
-			return -EBADMSG;
+	return nand_exec_op(chip, &op);
+}
+
+static int nand_lp_exec_read_page_op(struct nand_chip *chip, unsigned int page,
+				     unsigned int offset_in_page, void *buf,
+				     unsigned int len)
+{
+	const struct nand_sdr_timings *sdr =
+		nand_get_sdr_timings(&chip->data_interface);
+	u8 addrs[5];
+	struct nand_op_instr instrs[] = {
+		NAND_OP_CMD(NAND_CMD_READ0, 0),
+		NAND_OP_ADDR(4, addrs, 0),
+		NAND_OP_CMD(NAND_CMD_READSTART, PSEC_TO_NSEC(sdr->tWB_max)),
+		NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max),
+				 PSEC_TO_NSEC(sdr->tRR_min)),
+		NAND_OP_DATA_IN(len, buf, 0),
+	};
+	struct nand_operation op = NAND_OPERATION(instrs);
+	int ret;
+
+	/* Drop the DATA_IN instruction if len is set to 0. */
+	if (!len)
+		op.ninstrs--;
+
+	ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
+	if (ret < 0)
+		return ret;
+
+	addrs[2] = page;
+	addrs[3] = page >> 8;
+
+	if (chip->options & NAND_ROW_ADDR_3) {
+		addrs[4] = page >> 16;
+		instrs[1].ctx.addr.naddrs++;
 	}
 
-	return bitflips;
+	return nand_exec_op(chip, &op);
 }
 
 /**
- * nand_check_erased_ecc_chunk - check if an ECC chunk contains (almost) only
- *				 0xff data
- * @data: data buffer to test
- * @datalen: data length
- * @ecc: ECC buffer
- * @ecclen: ECC length
- * @extraoob: extra OOB buffer
- * @extraooblen: extra OOB length
- * @bitflips_threshold: maximum number of bitflips
- *
- * Check if a data buffer and its associated ECC and OOB data contains only
- * 0xff pattern, which means the underlying region has been erased and is
- * ready to be programmed.
- * The bitflips_threshold specify the maximum number of bitflips before
- * considering the region as not erased.
+ * nand_read_page_op - Do a READ PAGE operation
+ * @chip: The NAND chip
+ * @page: page to read
+ * @offset_in_page: offset within the page
+ * @buf: buffer used to store the data
+ * @len: length of the buffer
  *
- * Note:
- * 1/ ECC algorithms are working on pre-defined block sizes which are usually
- *    different from the NAND page size. When fixing bitflips, ECC engines will
- *    report the number of errors per chunk, and the NAND core infrastructure
- *    expect you to return the maximum number of bitflips for the whole page.
- *    This is why you should always use this function on a single chunk and
- *    not on the whole page. After checking each chunk you should update your
- *    max_bitflips value accordingly.
- * 2/ When checking for bitflips in erased pages you should not only check
- *    the payload data but also their associated ECC data, because a user might
- *    have programmed almost all bits to 1 but a few. In this case, we
- *    shouldn't consider the chunk as erased, and checking ECC bytes prevent
- *    this case.
- * 3/ The extraoob argument is optional, and should be used if some of your OOB
- *    data are protected by the ECC engine.
- *    It could also be used if you support subpages and want to attach some
- *    extra OOB data to an ECC chunk.
+ * This function issues a READ PAGE operation.
+ * This function does not select/unselect the CS line.
  *
- * Returns a positive number of bitflips less than or equal to
- * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
- * threshold. In case of success, the passed buffers are filled with 0xff.
+ * Returns 0 on success, a negative error code otherwise.
  */
-int nand_check_erased_ecc_chunk(void *data, int datalen,
-				void *ecc, int ecclen,
-				void *extraoob, int extraooblen,
-				int bitflips_threshold)
+int nand_read_page_op(struct nand_chip *chip, unsigned int page,
+		      unsigned int offset_in_page, void *buf, unsigned int len)
 {
-	int data_bitflips = 0, ecc_bitflips = 0, extraoob_bitflips = 0;
-
-	data_bitflips = nand_check_erased_buf(data, datalen,
-					      bitflips_threshold);
-	if (data_bitflips < 0)
-		return data_bitflips;
-
-	bitflips_threshold -= data_bitflips;
-
-	ecc_bitflips = nand_check_erased_buf(ecc, ecclen, bitflips_threshold);
-	if (ecc_bitflips < 0)
-		return ecc_bitflips;
+	struct mtd_info *mtd = nand_to_mtd(chip);
 
-	bitflips_threshold -= ecc_bitflips;
+	if (len && !buf)
+		return -EINVAL;
 
-	extraoob_bitflips = nand_check_erased_buf(extraoob, extraooblen,
-						  bitflips_threshold);
-	if (extraoob_bitflips < 0)
-		return extraoob_bitflips;
+	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
+		return -EINVAL;
 
-	if (data_bitflips)
-		memset(data, 0xff, datalen);
+	if (chip->exec_op) {
+		if (mtd->writesize > 512)
+			return nand_lp_exec_read_page_op(chip, page,
+							 offset_in_page, buf,
+							 len);
 
-	if (ecc_bitflips)
-		memset(ecc, 0xff, ecclen);
+		return nand_sp_exec_read_page_op(chip, page, offset_in_page,
+						 buf, len);
+	}
 
-	if (extraoob_bitflips)
-		memset(extraoob, 0xff, extraooblen);
+	chip->cmdfunc(mtd, NAND_CMD_READ0, offset_in_page, page);
+	if (len)
+		chip->read_buf(mtd, buf, len);
 
-	return data_bitflips + ecc_bitflips + extraoob_bitflips;
+	return 0;
 }
-EXPORT_SYMBOL(nand_check_erased_ecc_chunk);
+EXPORT_SYMBOL_GPL(nand_read_page_op);
 
 /**
- * nand_read_page_raw - [INTERN] read raw page data without ecc
- * @mtd: mtd info structure
- * @chip: nand chip info structure
- * @buf: buffer to store read data
- * @oob_required: caller requires OOB data read to chip->oob_poi
- * @page: page number to read
+ * nand_read_param_page_op - Do a READ PARAMETER PAGE operation
+ * @chip: The NAND chip
+ * @page: parameter page to read
+ * @buf: buffer used to store the data
+ * @len: length of the buffer
  *
- * Not for syndrome calculating ECC controllers, which use a special oob layout.
+ * This function issues a READ PARAMETER PAGE operation.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
  */
-int nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
-		       uint8_t *buf, int oob_required, int page)
+static int nand_read_param_page_op(struct nand_chip *chip, u8 page, void *buf,
+				   unsigned int len)
 {
-	chip->read_buf(mtd, buf, mtd->writesize);
-	if (oob_required)
-		chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	unsigned int i;
+	u8 *p = buf;
+
+	if (len && !buf)
+		return -EINVAL;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_PARAM, 0),
+			NAND_OP_ADDR(1, &page, PSEC_TO_NSEC(sdr->tWB_max)),
+			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tR_max),
+					 PSEC_TO_NSEC(sdr->tRR_min)),
+			NAND_OP_8BIT_DATA_IN(len, buf, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		/* Drop the DATA_IN instruction if len is set to 0. */
+		if (!len)
+			op.ninstrs--;
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_PARAM, page, -1);
+	for (i = 0; i < len; i++)
+		p[i] = chip->read_byte(mtd);
+
 	return 0;
 }
-EXPORT_SYMBOL(nand_read_page_raw);
 
 /**
- * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
- * @mtd: mtd info structure
- * @chip: nand chip info structure
- * @buf: buffer to store read data
- * @oob_required: caller requires OOB data read to chip->oob_poi
- * @page: page number to read
+ * nand_change_read_column_op - Do a CHANGE READ COLUMN operation
+ * @chip: The NAND chip
+ * @offset_in_page: offset within the page
+ * @buf: buffer used to store the data
+ * @len: length of the buffer
+ * @force_8bit: force 8-bit bus access
  *
- * We need a special oob layout and handling even when OOB isn't used.
+ * This function issues a CHANGE READ COLUMN operation.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_change_read_column_op(struct nand_chip *chip,
+			       unsigned int offset_in_page, void *buf,
+			       unsigned int len, bool force_8bit)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (len && !buf)
+		return -EINVAL;
+
+	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
+		return -EINVAL;
+
+	/* Small page NANDs do not support column change. */
+	if (mtd->writesize <= 512)
+		return -ENOTSUPP;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		u8 addrs[2] = {};
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_RNDOUT, 0),
+			NAND_OP_ADDR(2, addrs, 0),
+			NAND_OP_CMD(NAND_CMD_RNDOUTSTART,
+				    PSEC_TO_NSEC(sdr->tCCS_min)),
+			NAND_OP_DATA_IN(len, buf, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+		int ret;
+
+		ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
+		if (ret < 0)
+			return ret;
+
+		/* Drop the DATA_IN instruction if len is set to 0. */
+		if (!len)
+			op.ninstrs--;
+
+		instrs[3].ctx.data.force_8bit = force_8bit;
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_RNDOUT, offset_in_page, -1);
+	if (len)
+		chip->read_buf(mtd, buf, len);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_change_read_column_op);
+
+/**
+ * nand_read_oob_op - Do a READ OOB operation
+ * @chip: The NAND chip
+ * @page: page to read
+ * @offset_in_oob: offset within the OOB area
+ * @buf: buffer used to store the data
+ * @len: length of the buffer
+ *
+ * This function issues a READ OOB operation.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_read_oob_op(struct nand_chip *chip, unsigned int page,
+		     unsigned int offset_in_oob, void *buf, unsigned int len)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (len && !buf)
+		return -EINVAL;
+
+	if (offset_in_oob + len > mtd->oobsize)
+		return -EINVAL;
+
+	if (chip->exec_op)
+		return nand_read_page_op(chip, page,
+					 mtd->writesize + offset_in_oob,
+					 buf, len);
+
+	chip->cmdfunc(mtd, NAND_CMD_READOOB, offset_in_oob, page);
+	if (len)
+		chip->read_buf(mtd, buf, len);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_read_oob_op);
+
+static int nand_exec_prog_page_op(struct nand_chip *chip, unsigned int page,
+				  unsigned int offset_in_page, const void *buf,
+				  unsigned int len, bool prog)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	const struct nand_sdr_timings *sdr =
+		nand_get_sdr_timings(&chip->data_interface);
+	u8 addrs[5] = {};
+	struct nand_op_instr instrs[] = {
+		/*
+		 * The first instruction will be dropped if we're dealing
+		 * with a large page NAND and adjusted if we're dealing
+		 * with a small page NAND and the page offset is > 255.
+		 */
+		NAND_OP_CMD(NAND_CMD_READ0, 0),
+		NAND_OP_CMD(NAND_CMD_SEQIN, 0),
+		NAND_OP_ADDR(0, addrs, PSEC_TO_NSEC(sdr->tADL_min)),
+		NAND_OP_DATA_OUT(len, buf, 0),
+		NAND_OP_CMD(NAND_CMD_PAGEPROG, PSEC_TO_NSEC(sdr->tWB_max)),
+		NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tPROG_max), 0),
+	};
+	struct nand_operation op = NAND_OPERATION(instrs);
+	int naddrs = nand_fill_column_cycles(chip, addrs, offset_in_page);
+	int ret;
+	u8 status;
+
+	if (naddrs < 0)
+		return naddrs;
+
+	addrs[naddrs++] = page;
+	addrs[naddrs++] = page >> 8;
+	if (chip->options & NAND_ROW_ADDR_3)
+		addrs[naddrs++] = page >> 16;
+
+	instrs[2].ctx.addr.naddrs = naddrs;
+
+	/* Drop the last two instructions if we're not programming the page. */
+	if (!prog) {
+		op.ninstrs -= 2;
+		/* Also drop the DATA_OUT instruction if empty. */
+		if (!len)
+			op.ninstrs--;
+	}
+
+	if (mtd->writesize <= 512) {
+		/*
+		 * Small pages need some more tweaking: we have to adjust the
+		 * first instruction depending on the page offset we're trying
+		 * to access.
+		 */
+		if (offset_in_page >= mtd->writesize)
+			instrs[0].ctx.cmd.opcode = NAND_CMD_READOOB;
+		else if (offset_in_page >= 256 &&
+			 !(chip->options & NAND_BUSWIDTH_16))
+			instrs[0].ctx.cmd.opcode = NAND_CMD_READ1;
+	} else {
+		/*
+		 * Drop the first command if we're dealing with a large page
+		 * NAND.
+		 */
+		op.instrs++;
+		op.ninstrs--;
+	}
+
+	ret = nand_exec_op(chip, &op);
+	if (!prog || ret)
+		return ret;
+
+	ret = nand_status_op(chip, &status);
+	if (ret)
+		return ret;
+
+	return status;
+}
+
+/**
+ * nand_prog_page_begin_op - starts a PROG PAGE operation
+ * @chip: The NAND chip
+ * @page: page to write
+ * @offset_in_page: offset within the page
+ * @buf: buffer containing the data to write to the page
+ * @len: length of the buffer
+ *
+ * This function issues the first half of a PROG PAGE operation.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_prog_page_begin_op(struct nand_chip *chip, unsigned int page,
+			    unsigned int offset_in_page, const void *buf,
+			    unsigned int len)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (len && !buf)
+		return -EINVAL;
+
+	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
+		return -EINVAL;
+
+	if (chip->exec_op)
+		return nand_exec_prog_page_op(chip, page, offset_in_page, buf,
+					      len, false);
+
+	chip->cmdfunc(mtd, NAND_CMD_SEQIN, offset_in_page, page);
+
+	if (buf)
+		chip->write_buf(mtd, buf, len);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_prog_page_begin_op);
+
+/**
+ * nand_prog_page_end_op - ends a PROG PAGE operation
+ * @chip: The NAND chip
+ *
+ * This function issues the second half of a PROG PAGE operation.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_prog_page_end_op(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int ret;
+	u8 status;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_PAGEPROG,
+				    PSEC_TO_NSEC(sdr->tWB_max)),
+			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tPROG_max), 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		ret = nand_exec_op(chip, &op);
+		if (ret)
+			return ret;
+
+		ret = nand_status_op(chip, &status);
+		if (ret)
+			return ret;
+	} else {
+		chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
+		ret = chip->waitfunc(mtd, chip);
+		if (ret < 0)
+			return ret;
+
+		status = ret;
+	}
+
+	if (status & NAND_STATUS_FAIL)
+		return -EIO;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_prog_page_end_op);
+
+/**
+ * nand_prog_page_op - Do a full PROG PAGE operation
+ * @chip: The NAND chip
+ * @page: page to write
+ * @offset_in_page: offset within the page
+ * @buf: buffer containing the data to write to the page
+ * @len: length of the buffer
+ *
+ * This function issues a full PROG PAGE operation.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_prog_page_op(struct nand_chip *chip, unsigned int page,
+		      unsigned int offset_in_page, const void *buf,
+		      unsigned int len)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	int status;
+
+	if (!len || !buf)
+		return -EINVAL;
+
+	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
+		return -EINVAL;
+
+	if (chip->exec_op) {
+		status = nand_exec_prog_page_op(chip, page, offset_in_page, buf,
+						len, true);
+	} else {
+		chip->cmdfunc(mtd, NAND_CMD_SEQIN, offset_in_page, page);
+		chip->write_buf(mtd, buf, len);
+		chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
+		status = chip->waitfunc(mtd, chip);
+	}
+
+	if (status & NAND_STATUS_FAIL)
+		return -EIO;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_prog_page_op);
+
+/**
+ * nand_change_write_column_op - Do a CHANGE WRITE COLUMN operation
+ * @chip: The NAND chip
+ * @offset_in_page: offset within the page
+ * @buf: buffer containing the data to send to the NAND
+ * @len: length of the buffer
+ * @force_8bit: force 8-bit bus access
+ *
+ * This function issues a CHANGE WRITE COLUMN operation.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_change_write_column_op(struct nand_chip *chip,
+				unsigned int offset_in_page,
+				const void *buf, unsigned int len,
+				bool force_8bit)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (len && !buf)
+		return -EINVAL;
+
+	if (offset_in_page + len > mtd->writesize + mtd->oobsize)
+		return -EINVAL;
+
+	/* Small page NANDs do not support column change. */
+	if (mtd->writesize <= 512)
+		return -ENOTSUPP;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		u8 addrs[2];
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_RNDIN, 0),
+			NAND_OP_ADDR(2, addrs, PSEC_TO_NSEC(sdr->tCCS_min)),
+			NAND_OP_DATA_OUT(len, buf, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+		int ret;
+
+		ret = nand_fill_column_cycles(chip, addrs, offset_in_page);
+		if (ret < 0)
+			return ret;
+
+		instrs[2].ctx.data.force_8bit = force_8bit;
+
+		/* Drop the DATA_OUT instruction if len is set to 0. */
+		if (!len)
+			op.ninstrs--;
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_RNDIN, offset_in_page, -1);
+	if (len)
+		chip->write_buf(mtd, buf, len);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_change_write_column_op);
+
+/**
+ * nand_readid_op - Do a READID operation
+ * @chip: The NAND chip
+ * @addr: address cycle to pass after the READID command
+ * @buf: buffer used to store the ID
+ * @len: length of the buffer
+ *
+ * This function sends a READID command and reads back the ID returned by the
+ * NAND.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_readid_op(struct nand_chip *chip, u8 addr, void *buf,
+		   unsigned int len)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	unsigned int i;
+	u8 *id = buf;
+
+	if (len && !buf)
+		return -EINVAL;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_READID, 0),
+			NAND_OP_ADDR(1, &addr, PSEC_TO_NSEC(sdr->tADL_min)),
+			NAND_OP_8BIT_DATA_IN(len, buf, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		/* Drop the DATA_IN instruction if len is set to 0. */
+		if (!len)
+			op.ninstrs--;
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_READID, addr, -1);
+
+	for (i = 0; i < len; i++)
+		id[i] = chip->read_byte(mtd);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_readid_op);
+
+/**
+ * nand_status_op - Do a STATUS operation
+ * @chip: The NAND chip
+ * @status: out variable to store the NAND status
+ *
+ * This function sends a STATUS command and reads back the status returned by
+ * the NAND.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_status_op(struct nand_chip *chip, u8 *status)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_STATUS,
+				    PSEC_TO_NSEC(sdr->tADL_min)),
+			NAND_OP_8BIT_DATA_IN(1, status, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		if (!status)
+			op.ninstrs--;
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
+	if (status)
+		*status = chip->read_byte(mtd);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_status_op);
+
+/**
+ * nand_exit_status_op - Exit a STATUS operation
+ * @chip: The NAND chip
+ *
+ * This function sends a READ0 command to cancel the effect of the STATUS
+ * command to avoid reading only the status until a new read command is sent.
+ *
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_exit_status_op(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (chip->exec_op) {
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_READ0, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_READ0, -1, -1);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_exit_status_op);
+
+/**
+ * nand_erase_op - Do an erase operation
+ * @chip: The NAND chip
+ * @eraseblock: block to erase
+ *
+ * This function sends an ERASE command and waits for the NAND to be ready
+ * before returning.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_erase_op(struct nand_chip *chip, unsigned int eraseblock)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	unsigned int page = eraseblock <<
+			    (chip->phys_erase_shift - chip->page_shift);
+	int ret;
+	u8 status;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		u8 addrs[3] = {	page, page >> 8, page >> 16 };
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_ERASE1, 0),
+			NAND_OP_ADDR(2, addrs, 0),
+			NAND_OP_CMD(NAND_CMD_ERASE2,
+				    PSEC_TO_MSEC(sdr->tWB_max)),
+			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tBERS_max), 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		if (chip->options & NAND_ROW_ADDR_3)
+			instrs[1].ctx.addr.naddrs++;
+
+		ret = nand_exec_op(chip, &op);
+		if (ret)
+			return ret;
+
+		ret = nand_status_op(chip, &status);
+		if (ret)
+			return ret;
+	} else {
+		chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page);
+		chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1);
+
+		ret = chip->waitfunc(mtd, chip);
+		if (ret < 0)
+			return ret;
+
+		status = ret;
+	}
+
+	if (status & NAND_STATUS_FAIL)
+		return -EIO;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_erase_op);
+
+/**
+ * nand_set_features_op - Do a SET FEATURES operation
+ * @chip: The NAND chip
+ * @feature: feature id
+ * @data: 4 bytes of data
+ *
+ * This function sends a SET FEATURES command and waits for the NAND to be
+ * ready before returning.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+static int nand_set_features_op(struct nand_chip *chip, u8 feature,
+				const void *data)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	const u8 *params = data;
+	int i, ret;
+	u8 status;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_SET_FEATURES, 0),
+			NAND_OP_ADDR(1, &feature, PSEC_TO_NSEC(sdr->tADL_min)),
+			NAND_OP_8BIT_DATA_OUT(ONFI_SUBFEATURE_PARAM_LEN, data,
+					      PSEC_TO_NSEC(sdr->tWB_max)),
+			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tFEAT_max), 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		ret = nand_exec_op(chip, &op);
+		if (ret)
+			return ret;
+
+		ret = nand_status_op(chip, &status);
+		if (ret)
+			return ret;
+	} else {
+		chip->cmdfunc(mtd, NAND_CMD_SET_FEATURES, feature, -1);
+		for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
+			chip->write_byte(mtd, params[i]);
+
+		ret = chip->waitfunc(mtd, chip);
+		if (ret < 0)
+			return ret;
+
+		status = ret;
+	}
+
+	if (status & NAND_STATUS_FAIL)
+		return -EIO;
+
+	return 0;
+}
+
+/**
+ * nand_get_features_op - Do a GET FEATURES operation
+ * @chip: The NAND chip
+ * @feature: feature id
+ * @data: 4 bytes of data
+ *
+ * This function sends a GET FEATURES command and waits for the NAND to be
+ * ready before returning.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+static int nand_get_features_op(struct nand_chip *chip, u8 feature,
+				void *data)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	u8 *params = data;
+	int i;
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_GET_FEATURES, 0),
+			NAND_OP_ADDR(1, &feature, PSEC_TO_NSEC(sdr->tWB_max)),
+			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tFEAT_max),
+					 PSEC_TO_NSEC(sdr->tRR_min)),
+			NAND_OP_8BIT_DATA_IN(ONFI_SUBFEATURE_PARAM_LEN,
+					     data, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_GET_FEATURES, feature, -1);
+	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
+		params[i] = chip->read_byte(mtd);
+
+	return 0;
+}
+
+/**
+ * nand_reset_op - Do a reset operation
+ * @chip: The NAND chip
+ *
+ * This function sends a RESET command and waits for the NAND to be ready
+ * before returning.
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_reset_op(struct nand_chip *chip)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (chip->exec_op) {
+		const struct nand_sdr_timings *sdr =
+			nand_get_sdr_timings(&chip->data_interface);
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(NAND_CMD_RESET, PSEC_TO_NSEC(sdr->tWB_max)),
+			NAND_OP_WAIT_RDY(PSEC_TO_MSEC(sdr->tRST_max), 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_reset_op);
+
+/**
+ * nand_read_data_op - Read data from the NAND
+ * @chip: The NAND chip
+ * @buf: buffer used to store the data
+ * @len: length of the buffer
+ * @force_8bit: force 8-bit bus access
+ *
+ * This function does a raw data read on the bus. Usually used after launching
+ * another NAND operation like nand_read_page_op().
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_read_data_op(struct nand_chip *chip, void *buf, unsigned int len,
+		      bool force_8bit)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (!len || !buf)
+		return -EINVAL;
+
+	if (chip->exec_op) {
+		struct nand_op_instr instrs[] = {
+			NAND_OP_DATA_IN(len, buf, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		instrs[0].ctx.data.force_8bit = force_8bit;
+
+		return nand_exec_op(chip, &op);
+	}
+
+	if (force_8bit) {
+		u8 *p = buf;
+		unsigned int i;
+
+		for (i = 0; i < len; i++)
+			p[i] = chip->read_byte(mtd);
+	} else {
+		chip->read_buf(mtd, buf, len);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_read_data_op);
+
+/**
+ * nand_write_data_op - Write data from the NAND
+ * @chip: The NAND chip
+ * @buf: buffer containing the data to send on the bus
+ * @len: length of the buffer
+ * @force_8bit: force 8-bit bus access
+ *
+ * This function does a raw data write on the bus. Usually used after launching
+ * another NAND operation like nand_write_page_begin_op().
+ * This function does not select/unselect the CS line.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_write_data_op(struct nand_chip *chip, const void *buf,
+		       unsigned int len, bool force_8bit)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (!len || !buf)
+		return -EINVAL;
+
+	if (chip->exec_op) {
+		struct nand_op_instr instrs[] = {
+			NAND_OP_DATA_OUT(len, buf, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		instrs[0].ctx.data.force_8bit = force_8bit;
+
+		return nand_exec_op(chip, &op);
+	}
+
+	if (force_8bit) {
+		const u8 *p = buf;
+		unsigned int i;
+
+		for (i = 0; i < len; i++)
+			chip->write_byte(mtd, p[i]);
+	} else {
+		chip->write_buf(mtd, buf, len);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_write_data_op);
+
+/**
+ * struct nand_op_parser_ctx - Context used by the parser
+ * @instrs: array of all the instructions that must be addressed
+ * @ninstrs: length of the @instrs array
+ * @subop: Sub-operation to be passed to the NAND controller
+ *
+ * This structure is used by the core to split NAND operations into
+ * sub-operations that can be handled by the NAND controller.
+ */
+struct nand_op_parser_ctx {
+	const struct nand_op_instr *instrs;
+	unsigned int ninstrs;
+	struct nand_subop subop;
+};
+
+/**
+ * nand_op_parser_must_split_instr - Checks if an instruction must be split
+ * @pat: the parser pattern element that matches @instr
+ * @instr: pointer to the instruction to check
+ * @start_offset: this is an in/out parameter. If @instr has already been
+ *		  split, then @start_offset is the offset from which to start
+ *		  (either an address cycle or an offset in the data buffer).
+ *		  Conversely, if the function returns true (ie. instr must be
+ *		  split), this parameter is updated to point to the first
+ *		  data/address cycle that has not been taken care of.
+ *
+ * Some NAND controllers are limited and cannot send X address cycles with a
+ * unique operation, or cannot read/write more than Y bytes at the same time.
+ * In this case, split the instruction that does not fit in a single
+ * controller-operation into two or more chunks.
+ *
+ * Returns true if the instruction must be split, false otherwise.
+ * The @start_offset parameter is also updated to the offset at which the next
+ * bundle of instruction must start (if an address or a data instruction).
+ */
+static bool
+nand_op_parser_must_split_instr(const struct nand_op_parser_pattern_elem *pat,
+				const struct nand_op_instr *instr,
+				unsigned int *start_offset)
+{
+	switch (pat->type) {
+	case NAND_OP_ADDR_INSTR:
+		if (!pat->ctx.addr.maxcycles)
+			break;
+
+		if (instr->ctx.addr.naddrs - *start_offset >
+		    pat->ctx.addr.maxcycles) {
+			*start_offset += pat->ctx.addr.maxcycles;
+			return true;
+		}
+		break;
+
+	case NAND_OP_DATA_IN_INSTR:
+	case NAND_OP_DATA_OUT_INSTR:
+		if (!pat->ctx.data.maxlen)
+			break;
+
+		if (instr->ctx.data.len - *start_offset >
+		    pat->ctx.data.maxlen) {
+			*start_offset += pat->ctx.data.maxlen;
+			return true;
+		}
+		break;
+
+	default:
+		break;
+	}
+
+	return false;
+}
+
+/**
+ * nand_op_parser_match_pat - Checks if a pattern matches the instructions
+ *			      remaining in the parser context
+ * @pat: the pattern to test
+ * @ctx: the parser context structure to match with the pattern @pat
+ *
+ * Check if @pat matches the set or a sub-set of instructions remaining in @ctx.
+ * Returns true if this is the case, false ortherwise. When true is returned,
+ * @ctx->subop is updated with the set of instructions to be passed to the
+ * controller driver.
+ */
+static bool
+nand_op_parser_match_pat(const struct nand_op_parser_pattern *pat,
+			 struct nand_op_parser_ctx *ctx)
+{
+	unsigned int instr_offset = ctx->subop.first_instr_start_off;
+	const struct nand_op_instr *end = ctx->instrs + ctx->ninstrs;
+	const struct nand_op_instr *instr = ctx->subop.instrs;
+	unsigned int i, ninstrs;
+
+	for (i = 0, ninstrs = 0; i < pat->nelems && instr < end; i++) {
+		/*
+		 * The pattern instruction does not match the operation
+		 * instruction. If the instruction is marked optional in the
+		 * pattern definition, we skip the pattern element and continue
+		 * to the next one. If the element is mandatory, there's no
+		 * match and we can return false directly.
+		 */
+		if (instr->type != pat->elems[i].type) {
+			if (!pat->elems[i].optional)
+				return false;
+
+			continue;
+		}
+
+		/*
+		 * Now check the pattern element constraints. If the pattern is
+		 * not able to handle the whole instruction in a single step,
+		 * we have to split it.
+		 * The last_instr_end_off value comes back updated to point to
+		 * the position where we have to split the instruction (the
+		 * start of the next subop chunk).
+		 */
+		if (nand_op_parser_must_split_instr(&pat->elems[i], instr,
+						    &instr_offset)) {
+			ninstrs++;
+			i++;
+			break;
+		}
+
+		instr++;
+		ninstrs++;
+		instr_offset = 0;
+	}
+
+	/*
+	 * This can happen if all instructions of a pattern are optional.
+	 * Still, if there's not at least one instruction handled by this
+	 * pattern, this is not a match, and we should try the next one (if
+	 * any).
+	 */
+	if (!ninstrs)
+		return false;
+
+	/*
+	 * We had a match on the pattern head, but the pattern may be longer
+	 * than the instructions we're asked to execute. We need to make sure
+	 * there's no mandatory elements in the pattern tail.
+	 */
+	for (; i < pat->nelems; i++) {
+		if (!pat->elems[i].optional)
+			return false;
+	}
+
+	/*
+	 * We have a match: update the subop structure accordingly and return
+	 * true.
+	 */
+	ctx->subop.ninstrs = ninstrs;
+	ctx->subop.last_instr_end_off = instr_offset;
+
+	return true;
+}
+
+#if IS_ENABLED(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG)
+static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
+{
+	const struct nand_op_instr *instr;
+	char *prefix = "      ";
+	unsigned int i;
+
+	pr_debug("executing subop:\n");
+
+	for (i = 0; i < ctx->ninstrs; i++) {
+		instr = &ctx->instrs[i];
+
+		if (instr == &ctx->subop.instrs[0])
+			prefix = "    ->";
+
+		switch (instr->type) {
+		case NAND_OP_CMD_INSTR:
+			pr_debug("%sCMD      [0x%02x]\n", prefix,
+				 instr->ctx.cmd.opcode);
+			break;
+		case NAND_OP_ADDR_INSTR:
+			pr_debug("%sADDR     [%d cyc: %*ph]\n", prefix,
+				 instr->ctx.addr.naddrs,
+				 instr->ctx.addr.naddrs < 64 ?
+				 instr->ctx.addr.naddrs : 64,
+				 instr->ctx.addr.addrs);
+			break;
+		case NAND_OP_DATA_IN_INSTR:
+			pr_debug("%sDATA_IN  [%d B%s]\n", prefix,
+				 instr->ctx.data.len,
+				 instr->ctx.data.force_8bit ?
+				 ", force 8-bit" : "");
+			break;
+		case NAND_OP_DATA_OUT_INSTR:
+			pr_debug("%sDATA_OUT [%d B%s]\n", prefix,
+				 instr->ctx.data.len,
+				 instr->ctx.data.force_8bit ?
+				 ", force 8-bit" : "");
+			break;
+		case NAND_OP_WAITRDY_INSTR:
+			pr_debug("%sWAITRDY  [max %d ms]\n", prefix,
+				 instr->ctx.waitrdy.timeout_ms);
+			break;
+		}
+
+		if (instr == &ctx->subop.instrs[ctx->subop.ninstrs - 1])
+			prefix = "      ";
+	}
+}
+#else
+static void nand_op_parser_trace(const struct nand_op_parser_ctx *ctx)
+{
+	/* NOP */
+}
+#endif
+
+/**
+ * nand_op_parser_exec_op - exec_op parser
+ * @chip: the NAND chip
+ * @parser: patterns description provided by the controller driver
+ * @op: the NAND operation to address
+ * @check_only: when true, the function only checks if @op can be handled but
+ *		does not execute the operation
+ *
+ * Helper function designed to ease integration of NAND controller drivers that
+ * only support a limited set of instruction sequences. The supported sequences
+ * are described in @parser, and the framework takes care of splitting @op into
+ * multiple sub-operations (if required) and pass them back to the ->exec()
+ * callback of the matching pattern if @check_only is set to false.
+ *
+ * NAND controller drivers should call this function from their own ->exec_op()
+ * implementation.
+ *
+ * Returns 0 on success, a negative error code otherwise. A failure can be
+ * caused by an unsupported operation (none of the supported patterns is able
+ * to handle the requested operation), or an error returned by one of the
+ * matching pattern->exec() hook.
+ */
+int nand_op_parser_exec_op(struct nand_chip *chip,
+			   const struct nand_op_parser *parser,
+			   const struct nand_operation *op, bool check_only)
+{
+	struct nand_op_parser_ctx ctx = {
+		.subop.instrs = op->instrs,
+		.instrs = op->instrs,
+		.ninstrs = op->ninstrs,
+	};
+	unsigned int i;
+
+	while (ctx.subop.instrs < op->instrs + op->ninstrs) {
+		int ret;
+
+		for (i = 0; i < parser->npatterns; i++) {
+			const struct nand_op_parser_pattern *pattern;
+
+			pattern = &parser->patterns[i];
+			if (!nand_op_parser_match_pat(pattern, &ctx))
+				continue;
+
+			nand_op_parser_trace(&ctx);
+
+			if (check_only)
+				break;
+
+			ret = pattern->exec(chip, &ctx.subop);
+			if (ret)
+				return ret;
+
+			break;
+		}
+
+		if (i == parser->npatterns) {
+			pr_debug("->exec_op() parser: pattern not found!\n");
+			return -ENOTSUPP;
+		}
+
+		/*
+		 * Update the context structure by pointing to the start of the
+		 * next subop.
+		 */
+		ctx.subop.instrs = ctx.subop.instrs + ctx.subop.ninstrs;
+		if (ctx.subop.last_instr_end_off)
+			ctx.subop.instrs -= 1;
+
+		ctx.subop.first_instr_start_off = ctx.subop.last_instr_end_off;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_op_parser_exec_op);
+
+static bool nand_instr_is_data(const struct nand_op_instr *instr)
+{
+	return instr && (instr->type == NAND_OP_DATA_IN_INSTR ||
+			 instr->type == NAND_OP_DATA_OUT_INSTR);
+}
+
+static bool nand_subop_instr_is_valid(const struct nand_subop *subop,
+				      unsigned int instr_idx)
+{
+	return subop && instr_idx < subop->ninstrs;
+}
+
+static int nand_subop_get_start_off(const struct nand_subop *subop,
+				    unsigned int instr_idx)
+{
+	if (instr_idx)
+		return 0;
+
+	return subop->first_instr_start_off;
+}
+
+/**
+ * nand_subop_get_addr_start_off - Get the start offset in an address array
+ * @subop: The entire sub-operation
+ * @instr_idx: Index of the instruction inside the sub-operation
+ *
+ * During driver development, one could be tempted to directly use the
+ * ->addr.addrs field of address instructions. This is wrong as address
+ * instructions might be split.
+ *
+ * Given an address instruction, returns the offset of the first cycle to issue.
+ */
+int nand_subop_get_addr_start_off(const struct nand_subop *subop,
+				  unsigned int instr_idx)
+{
+	if (!nand_subop_instr_is_valid(subop, instr_idx) ||
+	    subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR)
+		return -EINVAL;
+
+	return nand_subop_get_start_off(subop, instr_idx);
+}
+EXPORT_SYMBOL_GPL(nand_subop_get_addr_start_off);
+
+/**
+ * nand_subop_get_num_addr_cyc - Get the remaining address cycles to assert
+ * @subop: The entire sub-operation
+ * @instr_idx: Index of the instruction inside the sub-operation
+ *
+ * During driver development, one could be tempted to directly use the
+ * ->addr->naddrs field of a data instruction. This is wrong as instructions
+ * might be split.
+ *
+ * Given an address instruction, returns the number of address cycle to issue.
+ */
+int nand_subop_get_num_addr_cyc(const struct nand_subop *subop,
+				unsigned int instr_idx)
+{
+	int start_off, end_off;
+
+	if (!nand_subop_instr_is_valid(subop, instr_idx) ||
+	    subop->instrs[instr_idx].type != NAND_OP_ADDR_INSTR)
+		return -EINVAL;
+
+	start_off = nand_subop_get_addr_start_off(subop, instr_idx);
+
+	if (instr_idx == subop->ninstrs - 1 &&
+	    subop->last_instr_end_off)
+		end_off = subop->last_instr_end_off;
+	else
+		end_off = subop->instrs[instr_idx].ctx.addr.naddrs;
+
+	return end_off - start_off;
+}
+EXPORT_SYMBOL_GPL(nand_subop_get_num_addr_cyc);
+
+/**
+ * nand_subop_get_data_start_off - Get the start offset in a data array
+ * @subop: The entire sub-operation
+ * @instr_idx: Index of the instruction inside the sub-operation
+ *
+ * During driver development, one could be tempted to directly use the
+ * ->data->buf.{in,out} field of data instructions. This is wrong as data
+ * instructions might be split.
+ *
+ * Given a data instruction, returns the offset to start from.
+ */
+int nand_subop_get_data_start_off(const struct nand_subop *subop,
+				  unsigned int instr_idx)
+{
+	if (!nand_subop_instr_is_valid(subop, instr_idx) ||
+	    !nand_instr_is_data(&subop->instrs[instr_idx]))
+		return -EINVAL;
+
+	return nand_subop_get_start_off(subop, instr_idx);
+}
+EXPORT_SYMBOL_GPL(nand_subop_get_data_start_off);
+
+/**
+ * nand_subop_get_data_len - Get the number of bytes to retrieve
+ * @subop: The entire sub-operation
+ * @instr_idx: Index of the instruction inside the sub-operation
+ *
+ * During driver development, one could be tempted to directly use the
+ * ->data->len field of a data instruction. This is wrong as data instructions
+ * might be split.
+ *
+ * Returns the length of the chunk of data to send/receive.
+ */
+int nand_subop_get_data_len(const struct nand_subop *subop,
+			    unsigned int instr_idx)
+{
+	int start_off = 0, end_off;
+
+	if (!nand_subop_instr_is_valid(subop, instr_idx) ||
+	    !nand_instr_is_data(&subop->instrs[instr_idx]))
+		return -EINVAL;
+
+	start_off = nand_subop_get_data_start_off(subop, instr_idx);
+
+	if (instr_idx == subop->ninstrs - 1 &&
+	    subop->last_instr_end_off)
+		end_off = subop->last_instr_end_off;
+	else
+		end_off = subop->instrs[instr_idx].ctx.data.len;
+
+	return end_off - start_off;
+}
+EXPORT_SYMBOL_GPL(nand_subop_get_data_len);
+
+/**
+ * nand_reset - Reset and initialize a NAND device
+ * @chip: The NAND chip
+ * @chipnr: Internal die id
+ *
+ * Save the timings data structure, then apply SDR timings mode 0 (see
+ * nand_reset_data_interface for details), do the reset operation, and
+ * apply back the previous timings.
+ *
+ * Returns 0 on success, a negative error code otherwise.
+ */
+int nand_reset(struct nand_chip *chip, int chipnr)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+	struct nand_data_interface saved_data_intf = chip->data_interface;
+	int ret;
+
+	ret = nand_reset_data_interface(chip, chipnr);
+	if (ret)
+		return ret;
+
+	/*
+	 * The CS line has to be released before we can apply the new NAND
+	 * interface settings, hence this weird ->select_chip() dance.
+	 */
+	chip->select_chip(mtd, chipnr);
+	ret = nand_reset_op(chip);
+	chip->select_chip(mtd, -1);
+	if (ret)
+		return ret;
+
+	chip->select_chip(mtd, chipnr);
+	chip->data_interface = saved_data_intf;
+	ret = nand_setup_data_interface(chip, chipnr);
+	chip->select_chip(mtd, -1);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nand_reset);
+
+/**
+ * nand_check_erased_buf - check if a buffer contains (almost) only 0xff data
+ * @buf: buffer to test
+ * @len: buffer length
+ * @bitflips_threshold: maximum number of bitflips
+ *
+ * Check if a buffer contains only 0xff, which means the underlying region
+ * has been erased and is ready to be programmed.
+ * The bitflips_threshold specify the maximum number of bitflips before
+ * considering the region is not erased.
+ * Note: The logic of this function has been extracted from the memweight
+ * implementation, except that nand_check_erased_buf function exit before
+ * testing the whole buffer if the number of bitflips exceed the
+ * bitflips_threshold value.
+ *
+ * Returns a positive number of bitflips less than or equal to
+ * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
+ * threshold.
+ */
+static int nand_check_erased_buf(void *buf, int len, int bitflips_threshold)
+{
+	const unsigned char *bitmap = buf;
+	int bitflips = 0;
+	int weight;
+
+	for (; len && ((uintptr_t)bitmap) % sizeof(long);
+	     len--, bitmap++) {
+		weight = hweight8(*bitmap);
+		bitflips += BITS_PER_BYTE - weight;
+		if (unlikely(bitflips > bitflips_threshold))
+			return -EBADMSG;
+	}
+
+	for (; len >= sizeof(long);
+	     len -= sizeof(long), bitmap += sizeof(long)) {
+		unsigned long d = *((unsigned long *)bitmap);
+		if (d == ~0UL)
+			continue;
+		weight = hweight_long(d);
+		bitflips += BITS_PER_LONG - weight;
+		if (unlikely(bitflips > bitflips_threshold))
+			return -EBADMSG;
+	}
+
+	for (; len > 0; len--, bitmap++) {
+		weight = hweight8(*bitmap);
+		bitflips += BITS_PER_BYTE - weight;
+		if (unlikely(bitflips > bitflips_threshold))
+			return -EBADMSG;
+	}
+
+	return bitflips;
+}
+
+/**
+ * nand_check_erased_ecc_chunk - check if an ECC chunk contains (almost) only
+ *				 0xff data
+ * @data: data buffer to test
+ * @datalen: data length
+ * @ecc: ECC buffer
+ * @ecclen: ECC length
+ * @extraoob: extra OOB buffer
+ * @extraooblen: extra OOB length
+ * @bitflips_threshold: maximum number of bitflips
+ *
+ * Check if a data buffer and its associated ECC and OOB data contains only
+ * 0xff pattern, which means the underlying region has been erased and is
+ * ready to be programmed.
+ * The bitflips_threshold specify the maximum number of bitflips before
+ * considering the region as not erased.
+ *
+ * Note:
+ * 1/ ECC algorithms are working on pre-defined block sizes which are usually
+ *    different from the NAND page size. When fixing bitflips, ECC engines will
+ *    report the number of errors per chunk, and the NAND core infrastructure
+ *    expect you to return the maximum number of bitflips for the whole page.
+ *    This is why you should always use this function on a single chunk and
+ *    not on the whole page. After checking each chunk you should update your
+ *    max_bitflips value accordingly.
+ * 2/ When checking for bitflips in erased pages you should not only check
+ *    the payload data but also their associated ECC data, because a user might
+ *    have programmed almost all bits to 1 but a few. In this case, we
+ *    shouldn't consider the chunk as erased, and checking ECC bytes prevent
+ *    this case.
+ * 3/ The extraoob argument is optional, and should be used if some of your OOB
+ *    data are protected by the ECC engine.
+ *    It could also be used if you support subpages and want to attach some
+ *    extra OOB data to an ECC chunk.
+ *
+ * Returns a positive number of bitflips less than or equal to
+ * bitflips_threshold, or -ERROR_CODE for bitflips in excess of the
+ * threshold. In case of success, the passed buffers are filled with 0xff.
+ */
+int nand_check_erased_ecc_chunk(void *data, int datalen,
+				void *ecc, int ecclen,
+				void *extraoob, int extraooblen,
+				int bitflips_threshold)
+{
+	int data_bitflips = 0, ecc_bitflips = 0, extraoob_bitflips = 0;
+
+	data_bitflips = nand_check_erased_buf(data, datalen,
+					      bitflips_threshold);
+	if (data_bitflips < 0)
+		return data_bitflips;
+
+	bitflips_threshold -= data_bitflips;
+
+	ecc_bitflips = nand_check_erased_buf(ecc, ecclen, bitflips_threshold);
+	if (ecc_bitflips < 0)
+		return ecc_bitflips;
+
+	bitflips_threshold -= ecc_bitflips;
+
+	extraoob_bitflips = nand_check_erased_buf(extraoob, extraooblen,
+						  bitflips_threshold);
+	if (extraoob_bitflips < 0)
+		return extraoob_bitflips;
+
+	if (data_bitflips)
+		memset(data, 0xff, datalen);
+
+	if (ecc_bitflips)
+		memset(ecc, 0xff, ecclen);
+
+	if (extraoob_bitflips)
+		memset(extraoob, 0xff, extraooblen);
+
+	return data_bitflips + ecc_bitflips + extraoob_bitflips;
+}
+EXPORT_SYMBOL(nand_check_erased_ecc_chunk);
+
+/**
+ * nand_read_page_raw - [INTERN] read raw page data without ecc
+ * @mtd: mtd info structure
+ * @chip: nand chip info structure
+ * @buf: buffer to store read data
+ * @oob_required: caller requires OOB data read to chip->oob_poi
+ * @page: page number to read
+ *
+ * Not for syndrome calculating ECC controllers, which use a special oob layout.
+ */
+int nand_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
+		       uint8_t *buf, int oob_required, int page)
+{
+	int ret;
+
+	ret = nand_read_page_op(chip, page, 0, buf, mtd->writesize);
+	if (ret)
+		return ret;
+
+	if (oob_required) {
+		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
+					false);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(nand_read_page_raw);
+
+/**
+ * nand_read_page_raw_syndrome - [INTERN] read raw page data without ecc
+ * @mtd: mtd info structure
+ * @chip: nand chip info structure
+ * @buf: buffer to store read data
+ * @oob_required: caller requires OOB data read to chip->oob_poi
+ * @page: page number to read
+ *
+ * We need a special oob layout and handling even when OOB isn't used.
  */
 static int nand_read_page_raw_syndrome(struct mtd_info *mtd,
 				       struct nand_chip *chip, uint8_t *buf,
@@ -1414,29 +2928,50 @@ static int nand_read_page_raw_syndrome(struct mtd_info *mtd,
 	int eccsize = chip->ecc.size;
 	int eccbytes = chip->ecc.bytes;
 	uint8_t *oob = chip->oob_poi;
-	int steps, size;
+	int steps, size, ret;
+
+	ret = nand_read_page_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
 
 	for (steps = chip->ecc.steps; steps > 0; steps--) {
-		chip->read_buf(mtd, buf, eccsize);
+		ret = nand_read_data_op(chip, buf, eccsize, false);
+		if (ret)
+			return ret;
+
 		buf += eccsize;
 
 		if (chip->ecc.prepad) {
-			chip->read_buf(mtd, oob, chip->ecc.prepad);
+			ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
+						false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.prepad;
 		}
 
-		chip->read_buf(mtd, oob, eccbytes);
+		ret = nand_read_data_op(chip, oob, eccbytes, false);
+		if (ret)
+			return ret;
+
 		oob += eccbytes;
 
 		if (chip->ecc.postpad) {
-			chip->read_buf(mtd, oob, chip->ecc.postpad);
+			ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
+						false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.postpad;
 		}
 	}
 
 	size = mtd->oobsize - (oob - chip->oob_poi);
-	if (size)
-		chip->read_buf(mtd, oob, size);
+	if (size) {
+		ret = nand_read_data_op(chip, oob, size, false);
+		if (ret)
+			return ret;
+	}
 
 	return 0;
 }
@@ -1456,8 +2991,8 @@ static int nand_read_page_swecc(struct mtd_info *mtd, struct nand_chip *chip,
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
 	uint8_t *p = buf;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
-	uint8_t *ecc_code = chip->buffers->ecccode;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
+	uint8_t *ecc_code = chip->ecc.code_buf;
 	unsigned int max_bitflips = 0;
 
 	chip->ecc.read_page_raw(mtd, chip, buf, 1, page);
@@ -1521,15 +3056,14 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 
 	data_col_addr = start_step * chip->ecc.size;
 	/* If we read not a page aligned data */
-	if (data_col_addr != 0)
-		chip->cmdfunc(mtd, NAND_CMD_RNDOUT, data_col_addr, -1);
-
 	p = bufpoi + data_col_addr;
-	chip->read_buf(mtd, p, datafrag_len);
+	ret = nand_read_page_op(chip, page, data_col_addr, p, datafrag_len);
+	if (ret)
+		return ret;
 
 	/* Calculate ECC */
 	for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size)
-		chip->ecc.calculate(mtd, p, &chip->buffers->ecccalc[i]);
+		chip->ecc.calculate(mtd, p, &chip->ecc.calc_buf[i]);
 
 	/*
 	 * The performance is faster if we position offsets according to
@@ -1543,8 +3077,11 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 		gaps = 1;
 
 	if (gaps) {
-		chip->cmdfunc(mtd, NAND_CMD_RNDOUT, mtd->writesize, -1);
-		chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
+		ret = nand_change_read_column_op(chip, mtd->writesize,
+						 chip->oob_poi, mtd->oobsize,
+						 false);
+		if (ret)
+			return ret;
 	} else {
 		/*
 		 * Send the command to read the particular ECC bytes take care
@@ -1558,12 +3095,15 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 		    (busw - 1))
 			aligned_len++;
 
-		chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
-			      mtd->writesize + aligned_pos, -1);
-		chip->read_buf(mtd, &chip->oob_poi[aligned_pos], aligned_len);
+		ret = nand_change_read_column_op(chip,
+						 mtd->writesize + aligned_pos,
+						 &chip->oob_poi[aligned_pos],
+						 aligned_len, false);
+		if (ret)
+			return ret;
 	}
 
-	ret = mtd_ooblayout_get_eccbytes(mtd, chip->buffers->ecccode,
+	ret = mtd_ooblayout_get_eccbytes(mtd, chip->ecc.code_buf,
 					 chip->oob_poi, index, eccfrag_len);
 	if (ret)
 		return ret;
@@ -1572,13 +3112,13 @@ static int nand_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
 	for (i = 0; i < eccfrag_len ; i += chip->ecc.bytes, p += chip->ecc.size) {
 		int stat;
 
-		stat = chip->ecc.correct(mtd, p,
-			&chip->buffers->ecccode[i], &chip->buffers->ecccalc[i]);
+		stat = chip->ecc.correct(mtd, p, &chip->ecc.code_buf[i],
+					 &chip->ecc.calc_buf[i]);
 		if (stat == -EBADMSG &&
 		    (chip->ecc.options & NAND_ECC_GENERIC_ERASED_CHECK)) {
 			/* check for empty pages with bitflips */
 			stat = nand_check_erased_ecc_chunk(p, chip->ecc.size,
-						&chip->buffers->ecccode[i],
+						&chip->ecc.code_buf[i],
 						chip->ecc.bytes,
 						NULL, 0,
 						chip->ecc.strength);
@@ -1611,16 +3151,27 @@ static int nand_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
 	uint8_t *p = buf;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
-	uint8_t *ecc_code = chip->buffers->ecccode;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
+	uint8_t *ecc_code = chip->ecc.code_buf;
 	unsigned int max_bitflips = 0;
 
+	ret = nand_read_page_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
+
 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
 		chip->ecc.hwctl(mtd, NAND_ECC_READ);
-		chip->read_buf(mtd, p, eccsize);
+
+		ret = nand_read_data_op(chip, p, eccsize, false);
+		if (ret)
+			return ret;
+
 		chip->ecc.calculate(mtd, p, &ecc_calc[i]);
 	}
-	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
+
+	ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false);
+	if (ret)
+		return ret;
 
 	ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
 					 chip->ecc.total);
@@ -1674,14 +3225,18 @@ static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd,
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
 	uint8_t *p = buf;
-	uint8_t *ecc_code = chip->buffers->ecccode;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
+	uint8_t *ecc_code = chip->ecc.code_buf;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
 	unsigned int max_bitflips = 0;
 
 	/* Read the OOB area first */
-	chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
-	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
+	ret = nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
+	if (ret)
+		return ret;
+
+	ret = nand_read_page_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
 
 	ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
 					 chip->ecc.total);
@@ -1692,7 +3247,11 @@ static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd,
 		int stat;
 
 		chip->ecc.hwctl(mtd, NAND_ECC_READ);
-		chip->read_buf(mtd, p, eccsize);
+
+		ret = nand_read_data_op(chip, p, eccsize, false);
+		if (ret)
+			return ret;
+
 		chip->ecc.calculate(mtd, p, &ecc_calc[i]);
 
 		stat = chip->ecc.correct(mtd, p, &ecc_code[i], NULL);
@@ -1729,7 +3288,7 @@ static int nand_read_page_hwecc_oob_first(struct mtd_info *mtd,
 static int nand_read_page_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
 				   uint8_t *buf, int oob_required, int page)
 {
-	int i, eccsize = chip->ecc.size;
+	int ret, i, eccsize = chip->ecc.size;
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
 	int eccpadbytes = eccbytes + chip->ecc.prepad + chip->ecc.postpad;
@@ -1737,25 +3296,44 @@ static int nand_read_page_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
 	uint8_t *oob = chip->oob_poi;
 	unsigned int max_bitflips = 0;
 
+	ret = nand_read_page_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
+
 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
 		int stat;
 
 		chip->ecc.hwctl(mtd, NAND_ECC_READ);
-		chip->read_buf(mtd, p, eccsize);
+
+		ret = nand_read_data_op(chip, p, eccsize, false);
+		if (ret)
+			return ret;
 
 		if (chip->ecc.prepad) {
-			chip->read_buf(mtd, oob, chip->ecc.prepad);
+			ret = nand_read_data_op(chip, oob, chip->ecc.prepad,
+						false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.prepad;
 		}
 
 		chip->ecc.hwctl(mtd, NAND_ECC_READSYN);
-		chip->read_buf(mtd, oob, eccbytes);
+
+		ret = nand_read_data_op(chip, oob, eccbytes, false);
+		if (ret)
+			return ret;
+
 		stat = chip->ecc.correct(mtd, p, oob, NULL);
 
 		oob += eccbytes;
 
 		if (chip->ecc.postpad) {
-			chip->read_buf(mtd, oob, chip->ecc.postpad);
+			ret = nand_read_data_op(chip, oob, chip->ecc.postpad,
+						false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.postpad;
 		}
 
@@ -1779,8 +3357,11 @@ static int nand_read_page_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
 
 	/* Calculate remaining oob bytes */
 	i = mtd->oobsize - (oob - chip->oob_poi);
-	if (i)
-		chip->read_buf(mtd, oob, i);
+	if (i) {
+		ret = nand_read_data_op(chip, oob, i, false);
+		if (ret)
+			return ret;
+	}
 
 	return max_bitflips;
 }
@@ -1894,16 +3475,13 @@ static int nand_do_read_ops(struct mtd_info *mtd, loff_t from,
 
 		/* Is the current page in the buffer? */
 		if (realpage != chip->pagebuf || oob) {
-			bufpoi = use_bufpoi ? chip->buffers->databuf : buf;
+			bufpoi = use_bufpoi ? chip->data_buf : buf;
 
 			if (use_bufpoi && aligned)
 				pr_debug("%s: using read bounce buffer for buf@%p\n",
 						 __func__, buf);
 
 read_retry:
-			if (nand_standard_page_accessors(&chip->ecc))
-				chip->cmdfunc(mtd, NAND_CMD_READ0, 0x00, page);
-
 			/*
 			 * Now read the page into the buffer.  Absent an error,
 			 * the read methods return max bitflips per ecc step.
@@ -1938,7 +3516,7 @@ static int nand_do_read_ops(struct mtd_info *mtd, loff_t from,
 					/* Invalidate page cache */
 					chip->pagebuf = -1;
 				}
-				memcpy(buf, chip->buffers->databuf + col, bytes);
+				memcpy(buf, chip->data_buf + col, bytes);
 			}
 
 			if (unlikely(oob)) {
@@ -1979,7 +3557,7 @@ static int nand_do_read_ops(struct mtd_info *mtd, loff_t from,
 			buf += bytes;
 			max_bitflips = max_t(unsigned int, max_bitflips, ret);
 		} else {
-			memcpy(buf, chip->buffers->databuf + col, bytes);
+			memcpy(buf, chip->data_buf + col, bytes);
 			buf += bytes;
 			max_bitflips = max_t(unsigned int, max_bitflips,
 					     chip->pagebuf_bitflips);
@@ -2026,33 +3604,6 @@ static int nand_do_read_ops(struct mtd_info *mtd, loff_t from,
 	return max_bitflips;
 }
 
-/**
- * nand_read - [MTD Interface] MTD compatibility function for nand_do_read_ecc
- * @mtd: MTD device structure
- * @from: offset to read from
- * @len: number of bytes to read
- * @retlen: pointer to variable to store the number of read bytes
- * @buf: the databuffer to put data
- *
- * Get hold of the chip and call nand_do_read.
- */
-static int nand_read(struct mtd_info *mtd, loff_t from, size_t len,
-		     size_t *retlen, uint8_t *buf)
-{
-	struct mtd_oob_ops ops;
-	int ret;
-
-	nand_get_device(mtd, FL_READING);
-	memset(&ops, 0, sizeof(ops));
-	ops.len = len;
-	ops.datbuf = buf;
-	ops.mode = MTD_OPS_PLACE_OOB;
-	ret = nand_do_read_ops(mtd, from, &ops);
-	*retlen = ops.retlen;
-	nand_release_device(mtd);
-	return ret;
-}
-
 /**
  * nand_read_oob_std - [REPLACEABLE] the most common OOB data read function
  * @mtd: mtd info structure
@@ -2061,9 +3612,7 @@ static int nand_read(struct mtd_info *mtd, loff_t from, size_t len,
  */
 int nand_read_oob_std(struct mtd_info *mtd, struct nand_chip *chip, int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
-	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
-	return 0;
+	return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
 }
 EXPORT_SYMBOL(nand_read_oob_std);
 
@@ -2081,25 +3630,43 @@ int nand_read_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
 	int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
 	int eccsize = chip->ecc.size;
 	uint8_t *bufpoi = chip->oob_poi;
-	int i, toread, sndrnd = 0, pos;
+	int i, toread, sndrnd = 0, pos, ret;
+
+	ret = nand_read_page_op(chip, page, chip->ecc.size, NULL, 0);
+	if (ret)
+		return ret;
 
-	chip->cmdfunc(mtd, NAND_CMD_READ0, chip->ecc.size, page);
 	for (i = 0; i < chip->ecc.steps; i++) {
 		if (sndrnd) {
+			int ret;
+
 			pos = eccsize + i * (eccsize + chunk);
 			if (mtd->writesize > 512)
-				chip->cmdfunc(mtd, NAND_CMD_RNDOUT, pos, -1);
+				ret = nand_change_read_column_op(chip, pos,
+								 NULL, 0,
+								 false);
 			else
-				chip->cmdfunc(mtd, NAND_CMD_READ0, pos, page);
+				ret = nand_read_page_op(chip, page, pos, NULL,
+							0);
+
+			if (ret)
+				return ret;
 		} else
 			sndrnd = 1;
 		toread = min_t(int, length, chunk);
-		chip->read_buf(mtd, bufpoi, toread);
+
+		ret = nand_read_data_op(chip, bufpoi, toread, false);
+		if (ret)
+			return ret;
+
 		bufpoi += toread;
 		length -= toread;
 	}
-	if (length > 0)
-		chip->read_buf(mtd, bufpoi, length);
+	if (length > 0) {
+		ret = nand_read_data_op(chip, bufpoi, length, false);
+		if (ret)
+			return ret;
+	}
 
 	return 0;
 }
@@ -2113,18 +3680,8 @@ EXPORT_SYMBOL(nand_read_oob_syndrome);
  */
 int nand_write_oob_std(struct mtd_info *mtd, struct nand_chip *chip, int page)
 {
-	int status = 0;
-	const uint8_t *buf = chip->oob_poi;
-	int length = mtd->oobsize;
-
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize, page);
-	chip->write_buf(mtd, buf, length);
-	/* Send command to program the OOB data */
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_op(chip, page, mtd->writesize, chip->oob_poi,
+				 mtd->oobsize);
 }
 EXPORT_SYMBOL(nand_write_oob_std);
 
@@ -2140,7 +3697,7 @@ int nand_write_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
 {
 	int chunk = chip->ecc.bytes + chip->ecc.prepad + chip->ecc.postpad;
 	int eccsize = chip->ecc.size, length = mtd->oobsize;
-	int i, len, pos, status = 0, sndcmd = 0, steps = chip->ecc.steps;
+	int ret, i, len, pos, sndcmd = 0, steps = chip->ecc.steps;
 	const uint8_t *bufpoi = chip->oob_poi;
 
 	/*
@@ -2154,7 +3711,10 @@ int nand_write_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
 	} else
 		pos = eccsize;
 
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, pos, page);
+	ret = nand_prog_page_begin_op(chip, page, pos, NULL, 0);
+	if (ret)
+		return ret;
+
 	for (i = 0; i < steps; i++) {
 		if (sndcmd) {
 			if (mtd->writesize <= 512) {
@@ -2163,28 +3723,40 @@ int nand_write_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip,
 				len = eccsize;
 				while (len > 0) {
 					int num = min_t(int, len, 4);
-					chip->write_buf(mtd, (uint8_t *)&fill,
-							num);
+
+					ret = nand_write_data_op(chip, &fill,
+								 num, false);
+					if (ret)
+						return ret;
+
 					len -= num;
 				}
 			} else {
 				pos = eccsize + i * (eccsize + chunk);
-				chip->cmdfunc(mtd, NAND_CMD_RNDIN, pos, -1);
+				ret = nand_change_write_column_op(chip, pos,
+								  NULL, 0,
+								  false);
+				if (ret)
+					return ret;
 			}
 		} else
 			sndcmd = 1;
 		len = min_t(int, length, chunk);
-		chip->write_buf(mtd, bufpoi, len);
+
+		ret = nand_write_data_op(chip, bufpoi, len, false);
+		if (ret)
+			return ret;
+
 		bufpoi += len;
 		length -= len;
 	}
-	if (length > 0)
-		chip->write_buf(mtd, bufpoi, length);
-
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-	status = chip->waitfunc(mtd, chip);
+	if (length > 0) {
+		ret = nand_write_data_op(chip, bufpoi, length, false);
+		if (ret)
+			return ret;
+	}
 
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_end_op(chip);
 }
 EXPORT_SYMBOL(nand_write_oob_syndrome);
 
@@ -2199,6 +3771,7 @@ EXPORT_SYMBOL(nand_write_oob_syndrome);
 static int nand_do_read_oob(struct mtd_info *mtd, loff_t from,
 			    struct mtd_oob_ops *ops)
 {
+	unsigned int max_bitflips = 0;
 	int page, realpage, chipnr;
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct mtd_ecc_stats stats;
@@ -2214,21 +3787,6 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from,
 
 	len = mtd_oobavail(mtd, ops);
 
-	if (unlikely(ops->ooboffs >= len)) {
-		pr_debug("%s: attempt to start read outside oob\n",
-				__func__);
-		return -EINVAL;
-	}
-
-	/* Do not allow reads past end of device */
-	if (unlikely(from >= mtd->size ||
-		     ops->ooboffs + readlen > ((mtd->size >> chip->page_shift) -
-					(from >> chip->page_shift)) * len)) {
-		pr_debug("%s: attempt to read beyond end of device\n",
-				__func__);
-		return -EINVAL;
-	}
-
 	chipnr = (int)(from >> chip->chip_shift);
 	chip->select_chip(mtd, chipnr);
 
@@ -2256,6 +3814,8 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from,
 				nand_wait_ready(mtd);
 		}
 
+		max_bitflips = max_t(unsigned int, max_bitflips, ret);
+
 		readlen -= len;
 		if (!readlen)
 			break;
@@ -2281,7 +3841,7 @@ static int nand_do_read_oob(struct mtd_info *mtd, loff_t from,
 	if (mtd->ecc_stats.failed - stats.failed)
 		return -EBADMSG;
 
-	return  mtd->ecc_stats.corrected - stats.corrected ? -EUCLEAN : 0;
+	return max_bitflips;
 }
 
 /**
@@ -2299,13 +3859,6 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from,
 
 	ops->retlen = 0;
 
-	/* Do not allow reads past end of device */
-	if (ops->datbuf && (from + ops->len) > mtd->size) {
-		pr_debug("%s: attempt to read beyond end of device\n",
-				__func__);
-		return -EINVAL;
-	}
-
 	if (ops->mode != MTD_OPS_PLACE_OOB &&
 	    ops->mode != MTD_OPS_AUTO_OOB &&
 	    ops->mode != MTD_OPS_RAW)
@@ -2336,11 +3889,20 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from,
 int nand_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 			const uint8_t *buf, int oob_required, int page)
 {
-	chip->write_buf(mtd, buf, mtd->writesize);
-	if (oob_required)
-		chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
+	int ret;
 
-	return 0;
+	ret = nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
+	if (ret)
+		return ret;
+
+	if (oob_required) {
+		ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize,
+					 false);
+		if (ret)
+			return ret;
+	}
+
+	return nand_prog_page_end_op(chip);
 }
 EXPORT_SYMBOL(nand_write_page_raw);
 
@@ -2362,31 +3924,52 @@ static int nand_write_page_raw_syndrome(struct mtd_info *mtd,
 	int eccsize = chip->ecc.size;
 	int eccbytes = chip->ecc.bytes;
 	uint8_t *oob = chip->oob_poi;
-	int steps, size;
+	int steps, size, ret;
+
+	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
 
 	for (steps = chip->ecc.steps; steps > 0; steps--) {
-		chip->write_buf(mtd, buf, eccsize);
+		ret = nand_write_data_op(chip, buf, eccsize, false);
+		if (ret)
+			return ret;
+
 		buf += eccsize;
 
 		if (chip->ecc.prepad) {
-			chip->write_buf(mtd, oob, chip->ecc.prepad);
+			ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
+						 false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.prepad;
 		}
 
-		chip->write_buf(mtd, oob, eccbytes);
+		ret = nand_write_data_op(chip, oob, eccbytes, false);
+		if (ret)
+			return ret;
+
 		oob += eccbytes;
 
 		if (chip->ecc.postpad) {
-			chip->write_buf(mtd, oob, chip->ecc.postpad);
+			ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
+						 false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.postpad;
 		}
 	}
 
 	size = mtd->oobsize - (oob - chip->oob_poi);
-	if (size)
-		chip->write_buf(mtd, oob, size);
+	if (size) {
+		ret = nand_write_data_op(chip, oob, size, false);
+		if (ret)
+			return ret;
+	}
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 /**
  * nand_write_page_swecc - [REPLACEABLE] software ECC based page write function
@@ -2403,7 +3986,7 @@ static int nand_write_page_swecc(struct mtd_info *mtd, struct nand_chip *chip,
 	int i, eccsize = chip->ecc.size, ret;
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
 	const uint8_t *p = buf;
 
 	/* Software ECC calculation */
@@ -2433,12 +4016,20 @@ static int nand_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 	int i, eccsize = chip->ecc.size, ret;
 	int eccbytes = chip->ecc.bytes;
 	int eccsteps = chip->ecc.steps;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
 	const uint8_t *p = buf;
 
+	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
+
 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
 		chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
-		chip->write_buf(mtd, p, eccsize);
+
+		ret = nand_write_data_op(chip, p, eccsize, false);
+		if (ret)
+			return ret;
+
 		chip->ecc.calculate(mtd, p, &ecc_calc[i]);
 	}
 
@@ -2447,9 +4038,11 @@ static int nand_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 	if (ret)
 		return ret;
 
-	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
+	ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
+	if (ret)
+		return ret;
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 
@@ -2469,7 +4062,7 @@ static int nand_write_subpage_hwecc(struct mtd_info *mtd,
 				int oob_required, int page)
 {
 	uint8_t *oob_buf  = chip->oob_poi;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
 	int ecc_size      = chip->ecc.size;
 	int ecc_bytes     = chip->ecc.bytes;
 	int ecc_steps     = chip->ecc.steps;
@@ -2478,12 +4071,18 @@ static int nand_write_subpage_hwecc(struct mtd_info *mtd,
 	int oob_bytes       = mtd->oobsize / ecc_steps;
 	int step, ret;
 
+	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
+
 	for (step = 0; step < ecc_steps; step++) {
 		/* configure controller for WRITE access */
 		chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
 
 		/* write data (untouched subpages already masked by 0xFF) */
-		chip->write_buf(mtd, buf, ecc_size);
+		ret = nand_write_data_op(chip, buf, ecc_size, false);
+		if (ret)
+			return ret;
 
 		/* mask ECC of un-touched subpages by padding 0xFF */
 		if ((step < start_step) || (step > end_step))
@@ -2503,16 +4102,18 @@ static int nand_write_subpage_hwecc(struct mtd_info *mtd,
 
 	/* copy calculated ECC for whole page to chip->buffer->oob */
 	/* this include masked-value(0xFF) for unwritten subpages */
-	ecc_calc = chip->buffers->ecccalc;
+	ecc_calc = chip->ecc.calc_buf;
 	ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
 					 chip->ecc.total);
 	if (ret)
 		return ret;
 
 	/* write OOB buffer to NAND device */
-	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
+	ret = nand_write_data_op(chip, chip->oob_poi, mtd->oobsize, false);
+	if (ret)
+		return ret;
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 
@@ -2537,33 +4138,55 @@ static int nand_write_page_syndrome(struct mtd_info *mtd,
 	int eccsteps = chip->ecc.steps;
 	const uint8_t *p = buf;
 	uint8_t *oob = chip->oob_poi;
+	int ret;
 
-	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
+	ret = nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+	if (ret)
+		return ret;
 
+	for (i = 0; eccsteps; eccsteps--, i += eccbytes, p += eccsize) {
 		chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
-		chip->write_buf(mtd, p, eccsize);
+
+		ret = nand_write_data_op(chip, p, eccsize, false);
+		if (ret)
+			return ret;
 
 		if (chip->ecc.prepad) {
-			chip->write_buf(mtd, oob, chip->ecc.prepad);
+			ret = nand_write_data_op(chip, oob, chip->ecc.prepad,
+						 false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.prepad;
 		}
 
 		chip->ecc.calculate(mtd, p, oob);
-		chip->write_buf(mtd, oob, eccbytes);
+
+		ret = nand_write_data_op(chip, oob, eccbytes, false);
+		if (ret)
+			return ret;
+
 		oob += eccbytes;
 
 		if (chip->ecc.postpad) {
-			chip->write_buf(mtd, oob, chip->ecc.postpad);
+			ret = nand_write_data_op(chip, oob, chip->ecc.postpad,
+						 false);
+			if (ret)
+				return ret;
+
 			oob += chip->ecc.postpad;
 		}
 	}
 
 	/* Calculate remaining oob bytes */
 	i = mtd->oobsize - (oob - chip->oob_poi);
-	if (i)
-		chip->write_buf(mtd, oob, i);
+	if (i) {
+		ret = nand_write_data_op(chip, oob, i, false);
+		if (ret)
+			return ret;
+	}
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 /**
@@ -2589,9 +4212,6 @@ static int nand_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	else
 		subpage = 0;
 
-	if (nand_standard_page_accessors(&chip->ecc))
-		chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
-
 	if (unlikely(raw))
 		status = chip->ecc.write_page_raw(mtd, chip, buf,
 						  oob_required, page);
@@ -2605,14 +4225,6 @@ static int nand_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	if (status < 0)
 		return status;
 
-	if (nand_standard_page_accessors(&chip->ecc)) {
-		chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-		status = chip->waitfunc(mtd, chip);
-		if (status & NAND_STATUS_FAIL)
-			return -EIO;
-	}
-
 	return 0;
 }
 
@@ -2737,9 +4349,9 @@ static int nand_do_write_ops(struct mtd_info *mtd, loff_t to,
 			if (part_pagewr)
 				bytes = min_t(int, bytes - column, writelen);
 			chip->pagebuf = -1;
-			memset(chip->buffers->databuf, 0xff, mtd->writesize);
-			memcpy(&chip->buffers->databuf[column], buf, bytes);
-			wbuf = chip->buffers->databuf;
+			memset(chip->data_buf, 0xff, mtd->writesize);
+			memcpy(&chip->data_buf[column], buf, bytes);
+			wbuf = chip->data_buf;
 		}
 
 		if (unlikely(oob)) {
@@ -2821,33 +4433,6 @@ static int panic_nand_write(struct mtd_info *mtd, loff_t to, size_t len,
 	return ret;
 }
 
-/**
- * nand_write - [MTD Interface] NAND write with ECC
- * @mtd: MTD device structure
- * @to: offset to write to
- * @len: number of bytes to write
- * @retlen: pointer to variable to store the number of written bytes
- * @buf: the data to write
- *
- * NAND write with ECC.
- */
-static int nand_write(struct mtd_info *mtd, loff_t to, size_t len,
-			  size_t *retlen, const uint8_t *buf)
-{
-	struct mtd_oob_ops ops;
-	int ret;
-
-	nand_get_device(mtd, FL_WRITING);
-	memset(&ops, 0, sizeof(ops));
-	ops.len = len;
-	ops.datbuf = (uint8_t *)buf;
-	ops.mode = MTD_OPS_PLACE_OOB;
-	ret = nand_do_write_ops(mtd, to, &ops);
-	*retlen = ops.retlen;
-	nand_release_device(mtd);
-	return ret;
-}
-
 /**
  * nand_do_write_oob - [MTD Interface] NAND write out-of-band
  * @mtd: MTD device structure
@@ -2874,22 +4459,6 @@ static int nand_do_write_oob(struct mtd_info *mtd, loff_t to,
 		return -EINVAL;
 	}
 
-	if (unlikely(ops->ooboffs >= len)) {
-		pr_debug("%s: attempt to start write outside oob\n",
-				__func__);
-		return -EINVAL;
-	}
-
-	/* Do not allow write past end of device */
-	if (unlikely(to >= mtd->size ||
-		     ops->ooboffs + ops->ooblen >
-			((mtd->size >> chip->page_shift) -
-			 (to >> chip->page_shift)) * len)) {
-		pr_debug("%s: attempt to write beyond end of device\n",
-				__func__);
-		return -EINVAL;
-	}
-
 	chipnr = (int)(to >> chip->chip_shift);
 
 	/*
@@ -2945,13 +4514,6 @@ static int nand_write_oob(struct mtd_info *mtd, loff_t to,
 
 	ops->retlen = 0;
 
-	/* Do not allow writes past end of device */
-	if (ops->datbuf && (to + ops->len) > mtd->size) {
-		pr_debug("%s: attempt to write beyond end of device\n",
-				__func__);
-		return -EINVAL;
-	}
-
 	nand_get_device(mtd, FL_WRITING);
 
 	switch (ops->mode) {
@@ -2984,11 +4546,12 @@ static int nand_write_oob(struct mtd_info *mtd, loff_t to,
 static int single_erase(struct mtd_info *mtd, int page)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
+	unsigned int eraseblock;
+
 	/* Send commands to erase a block */
-	chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page);
-	chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1);
+	eraseblock = page >> (chip->phys_erase_shift - chip->page_shift);
 
-	return chip->waitfunc(mtd, chip);
+	return nand_erase_op(chip, eraseblock);
 }
 
 /**
@@ -3072,7 +4635,7 @@ int nand_erase_nand(struct mtd_info *mtd, struct erase_info *instr,
 		status = chip->erase(mtd, page & chip->pagemask);
 
 		/* See if block erase succeeded */
-		if (status & NAND_STATUS_FAIL) {
+		if (status) {
 			pr_debug("%s: failed erase, page 0x%08x\n",
 					__func__, page);
 			instr->state = MTD_ERASE_FAILED;
@@ -3215,22 +4778,12 @@ static int nand_max_bad_blocks(struct mtd_info *mtd, loff_t ofs, size_t len)
 static int nand_onfi_set_features(struct mtd_info *mtd, struct nand_chip *chip,
 			int addr, uint8_t *subfeature_param)
 {
-	int status;
-	int i;
-
 	if (!chip->onfi_version ||
 	    !(le16_to_cpu(chip->onfi_params.opt_cmd)
 	      & ONFI_OPT_CMD_SET_GET_FEATURES))
 		return -EINVAL;
 
-	chip->cmdfunc(mtd, NAND_CMD_SET_FEATURES, addr, -1);
-	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
-		chip->write_byte(mtd, subfeature_param[i]);
-
-	status = chip->waitfunc(mtd, chip);
-	if (status & NAND_STATUS_FAIL)
-		return -EIO;
-	return 0;
+	return nand_set_features_op(chip, addr, subfeature_param);
 }
 
 /**
@@ -3243,17 +4796,12 @@ static int nand_onfi_set_features(struct mtd_info *mtd, struct nand_chip *chip,
 static int nand_onfi_get_features(struct mtd_info *mtd, struct nand_chip *chip,
 			int addr, uint8_t *subfeature_param)
 {
-	int i;
-
 	if (!chip->onfi_version ||
 	    !(le16_to_cpu(chip->onfi_params.opt_cmd)
 	      & ONFI_OPT_CMD_SET_GET_FEATURES))
 		return -EINVAL;
 
-	chip->cmdfunc(mtd, NAND_CMD_GET_FEATURES, addr, -1);
-	for (i = 0; i < ONFI_SUBFEATURE_PARAM_LEN; ++i)
-		*subfeature_param++ = chip->read_byte(mtd);
-	return 0;
+	return nand_get_features_op(chip, addr, subfeature_param);
 }
 
 /**
@@ -3319,7 +4867,7 @@ static void nand_set_defaults(struct nand_chip *chip)
 		chip->chip_delay = 20;
 
 	/* check, if a user supplied command function given */
-	if (chip->cmdfunc == NULL)
+	if (!chip->cmdfunc && !chip->exec_op)
 		chip->cmdfunc = nand_command;
 
 	/* check, if a user supplied wait function given */
@@ -3396,12 +4944,11 @@ static u16 onfi_crc16(u16 crc, u8 const *p, size_t len)
 static int nand_flash_detect_ext_param_page(struct nand_chip *chip,
 					    struct nand_onfi_params *p)
 {
-	struct mtd_info *mtd = nand_to_mtd(chip);
 	struct onfi_ext_param_page *ep;
 	struct onfi_ext_section *s;
 	struct onfi_ext_ecc_info *ecc;
 	uint8_t *cursor;
-	int ret = -EINVAL;
+	int ret;
 	int len;
 	int i;
 
@@ -3411,14 +4958,18 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip,
 		return -ENOMEM;
 
 	/* Send our own NAND_CMD_PARAM. */
-	chip->cmdfunc(mtd, NAND_CMD_PARAM, 0, -1);
+	ret = nand_read_param_page_op(chip, 0, NULL, 0);
+	if (ret)
+		goto ext_out;
 
 	/* Use the Change Read Column command to skip the ONFI param pages. */
-	chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
-			sizeof(*p) * p->num_of_param_pages , -1);
+	ret = nand_change_read_column_op(chip,
+					 sizeof(*p) * p->num_of_param_pages,
+					 ep, len, true);
+	if (ret)
+		goto ext_out;
 
-	/* Read out the Extended Parameter Page. */
-	chip->read_buf(mtd, (uint8_t *)ep, len);
+	ret = -EINVAL;
 	if ((onfi_crc16(ONFI_CRC_BASE, ((uint8_t *)ep) + 2, len - 2)
 		!= le16_to_cpu(ep->crc))) {
 		pr_debug("fail in the CRC.\n");
@@ -3471,19 +5022,23 @@ static int nand_flash_detect_onfi(struct nand_chip *chip)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	struct nand_onfi_params *p = &chip->onfi_params;
-	int i, j;
-	int val;
+	char id[4];
+	int i, ret, val;
 
 	/* Try ONFI for unknown chip or LP */
-	chip->cmdfunc(mtd, NAND_CMD_READID, 0x20, -1);
-	if (chip->read_byte(mtd) != 'O' || chip->read_byte(mtd) != 'N' ||
-		chip->read_byte(mtd) != 'F' || chip->read_byte(mtd) != 'I')
+	ret = nand_readid_op(chip, 0x20, id, sizeof(id));
+	if (ret || strncmp(id, "ONFI", 4))
+		return 0;
+
+	ret = nand_read_param_page_op(chip, 0, NULL, 0);
+	if (ret)
 		return 0;
 
-	chip->cmdfunc(mtd, NAND_CMD_PARAM, 0, -1);
 	for (i = 0; i < 3; i++) {
-		for (j = 0; j < sizeof(*p); j++)
-			((uint8_t *)p)[j] = chip->read_byte(mtd);
+		ret = nand_read_data_op(chip, p, sizeof(*p), true);
+		if (ret)
+			return 0;
+
 		if (onfi_crc16(ONFI_CRC_BASE, (uint8_t *)p, 254) ==
 				le16_to_cpu(p->crc)) {
 			break;
@@ -3574,20 +5129,22 @@ static int nand_flash_detect_jedec(struct nand_chip *chip)
 	struct mtd_info *mtd = nand_to_mtd(chip);
 	struct nand_jedec_params *p = &chip->jedec_params;
 	struct jedec_ecc_info *ecc;
-	int val;
-	int i, j;
+	char id[5];
+	int i, val, ret;
 
 	/* Try JEDEC for unknown chip or LP */
-	chip->cmdfunc(mtd, NAND_CMD_READID, 0x40, -1);
-	if (chip->read_byte(mtd) != 'J' || chip->read_byte(mtd) != 'E' ||
-		chip->read_byte(mtd) != 'D' || chip->read_byte(mtd) != 'E' ||
-		chip->read_byte(mtd) != 'C')
+	ret = nand_readid_op(chip, 0x40, id, sizeof(id));
+	if (ret || strncmp(id, "JEDEC", sizeof(id)))
+		return 0;
+
+	ret = nand_read_param_page_op(chip, 0x40, NULL, 0);
+	if (ret)
 		return 0;
 
-	chip->cmdfunc(mtd, NAND_CMD_PARAM, 0x40, -1);
 	for (i = 0; i < 3; i++) {
-		for (j = 0; j < sizeof(*p); j++)
-			((uint8_t *)p)[j] = chip->read_byte(mtd);
+		ret = nand_read_data_op(chip, p, sizeof(*p), true);
+		if (ret)
+			return 0;
 
 		if (onfi_crc16(ONFI_CRC_BASE, (uint8_t *)p, 510) ==
 				le16_to_cpu(p->crc))
@@ -3866,8 +5423,7 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
 {
 	const struct nand_manufacturer *manufacturer;
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	int busw;
-	int i;
+	int busw, ret;
 	u8 *id_data = chip->id.data;
 	u8 maf_id, dev_id;
 
@@ -3875,17 +5431,21 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
 	 * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx)
 	 * after power-up.
 	 */
-	nand_reset(chip, 0);
+	ret = nand_reset(chip, 0);
+	if (ret)
+		return ret;
 
 	/* Select the device */
 	chip->select_chip(mtd, 0);
 
 	/* Send the command for reading device ID */
-	chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
+	ret = nand_readid_op(chip, 0, id_data, 2);
+	if (ret)
+		return ret;
 
 	/* Read manufacturer and device IDs */
-	maf_id = chip->read_byte(mtd);
-	dev_id = chip->read_byte(mtd);
+	maf_id = id_data[0];
+	dev_id = id_data[1];
 
 	/*
 	 * Try again to make sure, as some systems the bus-hold or other
@@ -3894,11 +5454,10 @@ static int nand_detect(struct nand_chip *chip, struct nand_flash_dev *type)
 	 * not match, ignore the device completely.
 	 */
 
-	chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
-
 	/* Read entire ID string */
-	for (i = 0; i < ARRAY_SIZE(chip->id.data); i++)
-		id_data[i] = chip->read_byte(mtd);
+	ret = nand_readid_op(chip, 0, id_data, sizeof(chip->id.data));
+	if (ret)
+		return ret;
 
 	if (id_data[0] != maf_id || id_data[1] != dev_id) {
 		pr_info("second ID read did not match %02x,%02x against %02x,%02x\n",
@@ -4190,6 +5749,9 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips,
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	int ret;
 
+	/* Enforce the right timings for reset/detection */
+	onfi_fill_data_interface(chip, NAND_SDR_IFACE, 0);
+
 	ret = nand_dt_init(chip);
 	if (ret)
 		return ret;
@@ -4197,15 +5759,21 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips,
 	if (!mtd->name && mtd->dev.parent)
 		mtd->name = dev_name(mtd->dev.parent);
 
-	if ((!chip->cmdfunc || !chip->select_chip) && !chip->cmd_ctrl) {
+	/*
+	 * ->cmdfunc() is legacy and will only be used if ->exec_op() is not
+	 * populated.
+	 */
+	if (!chip->exec_op) {
 		/*
-		 * Default functions assigned for chip_select() and
-		 * cmdfunc() both expect cmd_ctrl() to be populated,
-		 * so we need to check that that's the case
+		 * Default functions assigned for ->cmdfunc() and
+		 * ->select_chip() both expect ->cmd_ctrl() to be populated.
 		 */
-		pr_err("chip.cmd_ctrl() callback is not provided");
-		return -EINVAL;
+		if ((!chip->cmdfunc || !chip->select_chip) && !chip->cmd_ctrl) {
+			pr_err("->cmd_ctrl() should be provided\n");
+			return -EINVAL;
+		}
 	}
+
 	/* Set the default functions */
 	nand_set_defaults(chip);
 
@@ -4225,15 +5793,16 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips,
 
 	/* Check for a chip array */
 	for (i = 1; i < maxchips; i++) {
+		u8 id[2];
+
 		/* See comment in nand_get_flash_type for reset */
 		nand_reset(chip, i);
 
 		chip->select_chip(mtd, i);
 		/* Send the command for reading device ID */
-		chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
+		nand_readid_op(chip, 0, id, sizeof(id));
 		/* Read manufacturer and device IDs */
-		if (nand_maf_id != chip->read_byte(mtd) ||
-		    nand_dev_id != chip->read_byte(mtd)) {
+		if (nand_maf_id != id[0] || nand_dev_id != id[1]) {
 			chip->select_chip(mtd, -1);
 			break;
 		}
@@ -4600,26 +6169,6 @@ static bool nand_ecc_strength_good(struct mtd_info *mtd)
 	return corr >= ds_corr && ecc->strength >= chip->ecc_strength_ds;
 }
 
-static bool invalid_ecc_page_accessors(struct nand_chip *chip)
-{
-	struct nand_ecc_ctrl *ecc = &chip->ecc;
-
-	if (nand_standard_page_accessors(ecc))
-		return false;
-
-	/*
-	 * NAND_ECC_CUSTOM_PAGE_ACCESS flag is set, make sure the NAND
-	 * controller driver implements all the page accessors because
-	 * default helpers are not suitable when the core does not
-	 * send the READ0/PAGEPROG commands.
-	 */
-	return (!ecc->read_page || !ecc->write_page ||
-		!ecc->read_page_raw || !ecc->write_page_raw ||
-		(NAND_HAS_SUBPAGE_READ(chip) && !ecc->read_subpage) ||
-		(NAND_HAS_SUBPAGE_WRITE(chip) && !ecc->write_subpage &&
-		 ecc->hwctl && ecc->calculate));
-}
-
 /**
  * nand_scan_tail - [NAND Interface] Scan for the NAND device
  * @mtd: MTD device structure
@@ -4632,7 +6181,6 @@ int nand_scan_tail(struct mtd_info *mtd)
 {
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
-	struct nand_buffers *nbuf = NULL;
 	int ret, i;
 
 	/* New bad blocks should be marked in OOB, flash-based BBT, or both */
@@ -4641,39 +6189,9 @@ int nand_scan_tail(struct mtd_info *mtd)
 		return -EINVAL;
 	}
 
-	if (invalid_ecc_page_accessors(chip)) {
-		pr_err("Invalid ECC page accessors setup\n");
-		return -EINVAL;
-	}
-
-	if (!(chip->options & NAND_OWN_BUFFERS)) {
-		nbuf = kzalloc(sizeof(*nbuf), GFP_KERNEL);
-		if (!nbuf)
-			return -ENOMEM;
-
-		nbuf->ecccalc = kmalloc(mtd->oobsize, GFP_KERNEL);
-		if (!nbuf->ecccalc) {
-			ret = -ENOMEM;
-			goto err_free_nbuf;
-		}
-
-		nbuf->ecccode = kmalloc(mtd->oobsize, GFP_KERNEL);
-		if (!nbuf->ecccode) {
-			ret = -ENOMEM;
-			goto err_free_nbuf;
-		}
-
-		nbuf->databuf = kmalloc(mtd->writesize + mtd->oobsize,
-					GFP_KERNEL);
-		if (!nbuf->databuf) {
-			ret = -ENOMEM;
-			goto err_free_nbuf;
-		}
-
-		chip->buffers = nbuf;
-	} else if (!chip->buffers) {
+	chip->data_buf = kmalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL);
+	if (!chip->data_buf)
 		return -ENOMEM;
-	}
 
 	/*
 	 * FIXME: some NAND manufacturer drivers expect the first die to be
@@ -4685,10 +6203,10 @@ int nand_scan_tail(struct mtd_info *mtd)
 	ret = nand_manufacturer_init(chip);
 	chip->select_chip(mtd, -1);
 	if (ret)
-		goto err_free_nbuf;
+		goto err_free_buf;
 
 	/* Set the internal oob buffer location, just after the page data */
-	chip->oob_poi = chip->buffers->databuf + mtd->writesize;
+	chip->oob_poi = chip->data_buf + mtd->writesize;
 
 	/*
 	 * If no default placement scheme is given, select an appropriate one.
@@ -4836,6 +6354,15 @@ int nand_scan_tail(struct mtd_info *mtd)
 		goto err_nand_manuf_cleanup;
 	}
 
+	if (ecc->correct || ecc->calculate) {
+		ecc->calc_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
+		ecc->code_buf = kmalloc(mtd->oobsize, GFP_KERNEL);
+		if (!ecc->calc_buf || !ecc->code_buf) {
+			ret = -ENOMEM;
+			goto err_nand_manuf_cleanup;
+		}
+	}
+
 	/* For many systems, the standard OOB write also works for raw */
 	if (!ecc->read_oob_raw)
 		ecc->read_oob_raw = ecc->read_oob;
@@ -4917,8 +6444,6 @@ int nand_scan_tail(struct mtd_info *mtd)
 	mtd->_erase = nand_erase;
 	mtd->_point = NULL;
 	mtd->_unpoint = NULL;
-	mtd->_read = nand_read;
-	mtd->_write = nand_write;
 	mtd->_panic_write = panic_nand_write;
 	mtd->_read_oob = nand_read_oob;
 	mtd->_write_oob = nand_write_oob;
@@ -4954,7 +6479,7 @@ int nand_scan_tail(struct mtd_info *mtd)
 		chip->select_chip(mtd, -1);
 
 		if (ret)
-			goto err_nand_data_iface_cleanup;
+			goto err_nand_manuf_cleanup;
 	}
 
 	/* Check, if we should skip the bad block table scan */
@@ -4964,23 +6489,18 @@ int nand_scan_tail(struct mtd_info *mtd)
 	/* Build bad block table */
 	ret = chip->scan_bbt(mtd);
 	if (ret)
-		goto err_nand_data_iface_cleanup;
+		goto err_nand_manuf_cleanup;
 
 	return 0;
 
-err_nand_data_iface_cleanup:
-	nand_release_data_interface(chip);
 
 err_nand_manuf_cleanup:
 	nand_manufacturer_cleanup(chip);
 
-err_free_nbuf:
-	if (nbuf) {
-		kfree(nbuf->databuf);
-		kfree(nbuf->ecccode);
-		kfree(nbuf->ecccalc);
-		kfree(nbuf);
-	}
+err_free_buf:
+	kfree(chip->data_buf);
+	kfree(ecc->code_buf);
+	kfree(ecc->calc_buf);
 
 	return ret;
 }
@@ -5028,16 +6548,11 @@ void nand_cleanup(struct nand_chip *chip)
 	    chip->ecc.algo == NAND_ECC_BCH)
 		nand_bch_free((struct nand_bch_control *)chip->ecc.priv);
 
-	nand_release_data_interface(chip);
-
 	/* Free bad block table memory */
 	kfree(chip->bbt);
-	if (!(chip->options & NAND_OWN_BUFFERS) && chip->buffers) {
-		kfree(chip->buffers->databuf);
-		kfree(chip->buffers->ecccode);
-		kfree(chip->buffers->ecccalc);
-		kfree(chip->buffers);
-	}
+	kfree(chip->data_buf);
+	kfree(chip->ecc.code_buf);
+	kfree(chip->ecc.calc_buf);
 
 	/* Free bad block descriptor memory */
 	if (chip->badblock_pattern && chip->badblock_pattern->options
diff --git a/drivers/mtd/nand/nand_bbt.c b/drivers/mtd/nand/nand_bbt.c
index 2915b6739bf8..36092850be2c 100644
--- a/drivers/mtd/nand/nand_bbt.c
+++ b/drivers/mtd/nand/nand_bbt.c
@@ -898,7 +898,7 @@ static inline int nand_memory_bbt(struct mtd_info *mtd, struct nand_bbt_descr *b
 {
 	struct nand_chip *this = mtd_to_nand(mtd);
 
-	return create_bbt(mtd, this->buffers->databuf, bd, -1);
+	return create_bbt(mtd, this->data_buf, bd, -1);
 }
 
 /**
diff --git a/drivers/mtd/nand/nand_hynix.c b/drivers/mtd/nand/nand_hynix.c
index 985751eda317..d542908a0ebb 100644
--- a/drivers/mtd/nand/nand_hynix.c
+++ b/drivers/mtd/nand/nand_hynix.c
@@ -66,16 +66,44 @@ struct hynix_read_retry_otp {
 };
 
 static bool hynix_nand_has_valid_jedecid(struct nand_chip *chip)
+{
+	u8 jedecid[5] = { };
+	int ret;
+
+	ret = nand_readid_op(chip, 0x40, jedecid, sizeof(jedecid));
+	if (ret)
+		return false;
+
+	return !strncmp("JEDEC", jedecid, sizeof(jedecid));
+}
+
+static int hynix_nand_cmd_op(struct nand_chip *chip, u8 cmd)
+{
+	struct mtd_info *mtd = nand_to_mtd(chip);
+
+	if (chip->exec_op) {
+		struct nand_op_instr instrs[] = {
+			NAND_OP_CMD(cmd, 0),
+		};
+		struct nand_operation op = NAND_OPERATION(instrs);
+
+		return nand_exec_op(chip, &op);
+	}
+
+	chip->cmdfunc(mtd, cmd, -1, -1);
+
+	return 0;
+}
+
+static int hynix_nand_reg_write_op(struct nand_chip *chip, u8 addr, u8 val)
 {
 	struct mtd_info *mtd = nand_to_mtd(chip);
-	u8 jedecid[6] = { };
-	int i = 0;
+	u16 column = ((u16)addr << 8) | addr;
 
-	chip->cmdfunc(mtd, NAND_CMD_READID, 0x40, -1);
-	for (i = 0; i < 5; i++)
-		jedecid[i] = chip->read_byte(mtd);
+	chip->cmdfunc(mtd, NAND_CMD_NONE, column, -1);
+	chip->write_byte(mtd, val);
 
-	return !strcmp("JEDEC", jedecid);
+	return 0;
 }
 
 static int hynix_nand_setup_read_retry(struct mtd_info *mtd, int retry_mode)
@@ -83,14 +111,15 @@ static int hynix_nand_setup_read_retry(struct mtd_info *mtd, int retry_mode)
 	struct nand_chip *chip = mtd_to_nand(mtd);
 	struct hynix_nand *hynix = nand_get_manufacturer_data(chip);
 	const u8 *values;
-	int status;
-	int i;
+	int i, ret;
 
 	values = hynix->read_retry->values +
 		 (retry_mode * hynix->read_retry->nregs);
 
 	/* Enter 'Set Hynix Parameters' mode */
-	chip->cmdfunc(mtd, NAND_HYNIX_CMD_SET_PARAMS, -1, -1);
+	ret = hynix_nand_cmd_op(chip, NAND_HYNIX_CMD_SET_PARAMS);
+	if (ret)
+		return ret;
 
 	/*
 	 * Configure the NAND in the requested read-retry mode.
@@ -102,21 +131,14 @@ static int hynix_nand_setup_read_retry(struct mtd_info *mtd, int retry_mode)
 	 * probably tweaked at production in this case).
 	 */
 	for (i = 0; i < hynix->read_retry->nregs; i++) {
-		int column = hynix->read_retry->regs[i];
-
-		column |= column << 8;
-		chip->cmdfunc(mtd, NAND_CMD_NONE, column, -1);
-		chip->write_byte(mtd, values[i]);
+		ret = hynix_nand_reg_write_op(chip, hynix->read_retry->regs[i],
+					      values[i]);
+		if (ret)
+			return ret;
 	}
 
 	/* Apply the new settings. */
-	chip->cmdfunc(mtd, NAND_HYNIX_CMD_APPLY_PARAMS, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-	if (status & NAND_STATUS_FAIL)
-		return -EIO;
-
-	return 0;
+	return hynix_nand_cmd_op(chip, NAND_HYNIX_CMD_APPLY_PARAMS);
 }
 
 /**
@@ -172,40 +194,63 @@ static int hynix_read_rr_otp(struct nand_chip *chip,
 			     const struct hynix_read_retry_otp *info,
 			     void *buf)
 {
-	struct mtd_info *mtd = nand_to_mtd(chip);
-	int i;
+	int i, ret;
 
-	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+	ret = nand_reset_op(chip);
+	if (ret)
+		return ret;
 
-	chip->cmdfunc(mtd, NAND_HYNIX_CMD_SET_PARAMS, -1, -1);
+	ret = hynix_nand_cmd_op(chip, NAND_HYNIX_CMD_SET_PARAMS);
+	if (ret)
+		return ret;
 
 	for (i = 0; i < info->nregs; i++) {
-		int column = info->regs[i];
-
-		column |= column << 8;
-		chip->cmdfunc(mtd, NAND_CMD_NONE, column, -1);
-		chip->write_byte(mtd, info->values[i]);
+		ret = hynix_nand_reg_write_op(chip, info->regs[i],
+					      info->values[i]);
+		if (ret)
+			return ret;
 	}
 
-	chip->cmdfunc(mtd, NAND_HYNIX_CMD_APPLY_PARAMS, -1, -1);
+	ret = hynix_nand_cmd_op(chip, NAND_HYNIX_CMD_APPLY_PARAMS);
+	if (ret)
+		return ret;
 
 	/* Sequence to enter OTP mode? */
-	chip->cmdfunc(mtd, 0x17, -1, -1);
-	chip->cmdfunc(mtd, 0x04, -1, -1);
-	chip->cmdfunc(mtd, 0x19, -1, -1);
+	ret = hynix_nand_cmd_op(chip, 0x17);
+	if (ret)
+		return ret;
+
+	ret = hynix_nand_cmd_op(chip, 0x4);
+	if (ret)
+		return ret;
+
+	ret = hynix_nand_cmd_op(chip, 0x19);
+	if (ret)
+		return ret;
 
 	/* Now read the page */
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0x0, info->page);
-	chip->read_buf(mtd, buf, info->size);
+	ret = nand_read_page_op(chip, info->page, 0, buf, info->size);
+	if (ret)
+		return ret;
 
 	/* Put everything back to normal */
-	chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
-	chip->cmdfunc(mtd, NAND_HYNIX_CMD_SET_PARAMS, 0x38, -1);
-	chip->write_byte(mtd, 0x0);
-	chip->cmdfunc(mtd, NAND_HYNIX_CMD_APPLY_PARAMS, -1, -1);
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0x0, -1);
+	ret = nand_reset_op(chip);
+	if (ret)
+		return ret;
 
-	return 0;
+	ret = hynix_nand_cmd_op(chip, NAND_HYNIX_CMD_SET_PARAMS);
+	if (ret)
+		return ret;
+
+	ret = hynix_nand_reg_write_op(chip, 0x38, 0);
+	if (ret)
+		return ret;
+
+	ret = hynix_nand_cmd_op(chip, NAND_HYNIX_CMD_APPLY_PARAMS);
+	if (ret)
+		return ret;
+
+	return nand_read_page_op(chip, 0, 0, NULL, 0);
 }
 
 #define NAND_HYNIX_1XNM_RR_COUNT_OFFS				0
diff --git a/drivers/mtd/nand/nand_micron.c b/drivers/mtd/nand/nand_micron.c
index abf6a3c376e8..02e109ae73f1 100644
--- a/drivers/mtd/nand/nand_micron.c
+++ b/drivers/mtd/nand/nand_micron.c
@@ -117,16 +117,28 @@ micron_nand_read_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip,
 				 uint8_t *buf, int oob_required,
 				 int page)
 {
-	int status;
-	int max_bitflips = 0;
+	u8 status;
+	int ret, max_bitflips = 0;
 
-	micron_nand_on_die_ecc_setup(chip, true);
+	ret = micron_nand_on_die_ecc_setup(chip, true);
+	if (ret)
+		return ret;
+
+	ret = nand_read_page_op(chip, page, 0, NULL, 0);
+	if (ret)
+		goto out;
+
+	ret = nand_status_op(chip, &status);
+	if (ret)
+		goto out;
+
+	ret = nand_exit_status_op(chip);
+	if (ret)
+		goto out;
 
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0x00, page);
-	chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
-	status = chip->read_byte(mtd);
 	if (status & NAND_STATUS_FAIL)
 		mtd->ecc_stats.failed++;
+
 	/*
 	 * The internal ECC doesn't tell us the number of bitflips
 	 * that have been corrected, but tells us if it recommends to
@@ -137,13 +149,15 @@ micron_nand_read_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip,
 	else if (status & NAND_STATUS_WRITE_RECOMMENDED)
 		max_bitflips = chip->ecc.strength;
 
-	chip->cmdfunc(mtd, NAND_CMD_READ0, -1, -1);
-
-	nand_read_page_raw(mtd, chip, buf, oob_required, page);
+	ret = nand_read_data_op(chip, buf, mtd->writesize, false);
+	if (!ret && oob_required)
+		ret = nand_read_data_op(chip, chip->oob_poi, mtd->oobsize,
+					false);
 
+out:
 	micron_nand_on_die_ecc_setup(chip, false);
 
-	return max_bitflips;
+	return ret ? ret : max_bitflips;
 }
 
 static int
@@ -151,46 +165,16 @@ micron_nand_write_page_on_die_ecc(struct mtd_info *mtd, struct nand_chip *chip,
 				  const uint8_t *buf, int oob_required,
 				  int page)
 {
-	int status;
-
-	micron_nand_on_die_ecc_setup(chip, true);
+	int ret;
 
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
-	nand_write_page_raw(mtd, chip, buf, oob_required, page);
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-	status = chip->waitfunc(mtd, chip);
+	ret = micron_nand_on_die_ecc_setup(chip, true);
+	if (ret)
+		return ret;
 
+	ret = nand_write_page_raw(mtd, chip, buf, oob_required, page);
 	micron_nand_on_die_ecc_setup(chip, false);
 
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
-}
-
-static int
-micron_nand_read_page_raw_on_die_ecc(struct mtd_info *mtd,
-				     struct nand_chip *chip,
-				     uint8_t *buf, int oob_required,
-				     int page)
-{
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0x00, page);
-	nand_read_page_raw(mtd, chip, buf, oob_required, page);
-
-	return 0;
-}
-
-static int
-micron_nand_write_page_raw_on_die_ecc(struct mtd_info *mtd,
-				      struct nand_chip *chip,
-				      const uint8_t *buf, int oob_required,
-				      int page)
-{
-	int status;
-
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
-	nand_write_page_raw(mtd, chip, buf, oob_required, page);
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return ret;
 }
 
 enum {
@@ -285,17 +269,14 @@ static int micron_nand_init(struct nand_chip *chip)
 			return -EINVAL;
 		}
 
-		chip->ecc.options = NAND_ECC_CUSTOM_PAGE_ACCESS;
 		chip->ecc.bytes = 8;
 		chip->ecc.size = 512;
 		chip->ecc.strength = 4;
 		chip->ecc.algo = NAND_ECC_BCH;
 		chip->ecc.read_page = micron_nand_read_page_on_die_ecc;
 		chip->ecc.write_page = micron_nand_write_page_on_die_ecc;
-		chip->ecc.read_page_raw =
-			micron_nand_read_page_raw_on_die_ecc;
-		chip->ecc.write_page_raw =
-			micron_nand_write_page_raw_on_die_ecc;
+		chip->ecc.read_page_raw = nand_read_page_raw;
+		chip->ecc.write_page_raw = nand_write_page_raw;
 
 		mtd_set_ooblayout(mtd, &micron_nand_on_die_ooblayout_ops);
 	}
diff --git a/drivers/mtd/nand/nand_samsung.c b/drivers/mtd/nand/nand_samsung.c
index d348f0129ae7..ef022f62f74c 100644
--- a/drivers/mtd/nand/nand_samsung.c
+++ b/drivers/mtd/nand/nand_samsung.c
@@ -91,6 +91,25 @@ static void samsung_nand_decode_id(struct nand_chip *chip)
 		}
 	} else {
 		nand_decode_ext_id(chip);
+
+		if (nand_is_slc(chip)) {
+			switch (chip->id.data[1]) {
+			/* K9F4G08U0D-S[I|C]B0(T00) */
+			case 0xDC:
+				chip->ecc_step_ds = 512;
+				chip->ecc_strength_ds = 1;
+				break;
+
+			/* K9F1G08U0E 21nm chips do not support subpage write */
+			case 0xF1:
+				if (chip->id.len > 4 &&
+				    (chip->id.data[4] & GENMASK(1, 0)) == 0x1)
+					chip->options |= NAND_NO_SUBPAGE_WRITE;
+				break;
+			default:
+				break;
+			}
+		}
 	}
 }
 
diff --git a/drivers/mtd/nand/nand_timings.c b/drivers/mtd/nand/nand_timings.c
index 5d1533bcc5bd..9400d039ddbd 100644
--- a/drivers/mtd/nand/nand_timings.c
+++ b/drivers/mtd/nand/nand_timings.c
@@ -283,16 +283,16 @@ const struct nand_sdr_timings *onfi_async_timing_mode_to_sdr_timings(int mode)
 EXPORT_SYMBOL(onfi_async_timing_mode_to_sdr_timings);
 
 /**
- * onfi_init_data_interface - [NAND Interface] Initialize a data interface from
+ * onfi_fill_data_interface - [NAND Interface] Initialize a data interface from
  * given ONFI mode
- * @iface: The data interface to be initialized
  * @mode: The ONFI timing mode
  */
-int onfi_init_data_interface(struct nand_chip *chip,
-			     struct nand_data_interface *iface,
+int onfi_fill_data_interface(struct nand_chip *chip,
 			     enum nand_data_interface_type type,
 			     int timing_mode)
 {
+	struct nand_data_interface *iface = &chip->data_interface;
+
 	if (type != NAND_SDR_IFACE)
 		return -EINVAL;
 
@@ -321,15 +321,4 @@ int onfi_init_data_interface(struct nand_chip *chip,
 
 	return 0;
 }
-EXPORT_SYMBOL(onfi_init_data_interface);
-
-/**
- * nand_get_default_data_interface - [NAND Interface] Retrieve NAND
- * data interface for mode 0. This is used as default timing after
- * reset.
- */
-const struct nand_data_interface *nand_get_default_data_interface(void)
-{
-	return &onfi_sdr_timings[0];
-}
-EXPORT_SYMBOL(nand_get_default_data_interface);
+EXPORT_SYMBOL(onfi_fill_data_interface);
diff --git a/drivers/mtd/nand/omap2.c b/drivers/mtd/nand/omap2.c
index dad438c4906a..8cdf7d3d8fa7 100644
--- a/drivers/mtd/nand/omap2.c
+++ b/drivers/mtd/nand/omap2.c
@@ -1530,7 +1530,9 @@ static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
 			       const uint8_t *buf, int oob_required, int page)
 {
 	int ret;
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
+
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 
 	/* Enable GPMC ecc engine */
 	chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
@@ -1548,7 +1550,8 @@ static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
 
 	/* Write ecc vector to OOB area */
 	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
-	return 0;
+
+	return nand_prog_page_end_op(chip);
 }
 
 /**
@@ -1568,7 +1571,7 @@ static int omap_write_subpage_bch(struct mtd_info *mtd,
 				  u32 data_len, const u8 *buf,
 				  int oob_required, int page)
 {
-	u8 *ecc_calc = chip->buffers->ecccalc;
+	u8 *ecc_calc = chip->ecc.calc_buf;
 	int ecc_size      = chip->ecc.size;
 	int ecc_bytes     = chip->ecc.bytes;
 	int ecc_steps     = chip->ecc.steps;
@@ -1582,6 +1585,7 @@ static int omap_write_subpage_bch(struct mtd_info *mtd,
 	 * ECC is calculated for all subpages but we choose
 	 * only what we want.
 	 */
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 
 	/* Enable GPMC ECC engine */
 	chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
@@ -1605,7 +1609,7 @@ static int omap_write_subpage_bch(struct mtd_info *mtd,
 
 	/* copy calculated ECC for whole page to chip->buffer->oob */
 	/* this include masked-value(0xFF) for unwritten subpages */
-	ecc_calc = chip->buffers->ecccalc;
+	ecc_calc = chip->ecc.calc_buf;
 	ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
 					 chip->ecc.total);
 	if (ret)
@@ -1614,7 +1618,7 @@ static int omap_write_subpage_bch(struct mtd_info *mtd,
 	/* write OOB buffer to NAND device */
 	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 /**
@@ -1635,11 +1639,13 @@ static int omap_write_subpage_bch(struct mtd_info *mtd,
 static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
 				uint8_t *buf, int oob_required, int page)
 {
-	uint8_t *ecc_calc = chip->buffers->ecccalc;
-	uint8_t *ecc_code = chip->buffers->ecccode;
+	uint8_t *ecc_calc = chip->ecc.calc_buf;
+	uint8_t *ecc_code = chip->ecc.code_buf;
 	int stat, ret;
 	unsigned int max_bitflips = 0;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	/* Enable GPMC ecc engine */
 	chip->ecc.hwctl(mtd, NAND_ECC_READ);
 
@@ -1647,10 +1653,10 @@ static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
 	chip->read_buf(mtd, buf, mtd->writesize);
 
 	/* Read oob bytes */
-	chip->cmdfunc(mtd, NAND_CMD_RNDOUT,
-		      mtd->writesize + BADBLOCK_MARKER_LENGTH, -1);
-	chip->read_buf(mtd, chip->oob_poi + BADBLOCK_MARKER_LENGTH,
-		       chip->ecc.total);
+	nand_change_read_column_op(chip,
+				   mtd->writesize + BADBLOCK_MARKER_LENGTH,
+				   chip->oob_poi + BADBLOCK_MARKER_LENGTH,
+				   chip->ecc.total, false);
 
 	/* Calculate ecc bytes */
 	omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc);
diff --git a/drivers/mtd/nand/pxa3xx_nand.c b/drivers/mtd/nand/pxa3xx_nand.c
index 9285f60e5783..d1979c7dbe7e 100644
--- a/drivers/mtd/nand/pxa3xx_nand.c
+++ b/drivers/mtd/nand/pxa3xx_nand.c
@@ -520,15 +520,13 @@ static int pxa3xx_nand_init_timings_compat(struct pxa3xx_nand_host *host,
 	struct nand_chip *chip = &host->chip;
 	struct pxa3xx_nand_info *info = host->info_data;
 	const struct pxa3xx_nand_flash *f = NULL;
-	struct mtd_info *mtd = nand_to_mtd(&host->chip);
 	int i, id, ntypes;
+	u8 idbuf[2];
 
 	ntypes = ARRAY_SIZE(builtin_flash_types);
 
-	chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1);
-
-	id = chip->read_byte(mtd);
-	id |= chip->read_byte(mtd) << 0x8;
+	nand_readid_op(chip, 0, idbuf, sizeof(idbuf));
+	id = idbuf[0] | (idbuf[1] << 8);
 
 	for (i = 0; i < ntypes; i++) {
 		f = &builtin_flash_types[i];
@@ -1351,10 +1349,10 @@ static int pxa3xx_nand_write_page_hwecc(struct mtd_info *mtd,
 		struct nand_chip *chip, const uint8_t *buf, int oob_required,
 		int page)
 {
-	chip->write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int pxa3xx_nand_read_page_hwecc(struct mtd_info *mtd,
@@ -1364,7 +1362,7 @@ static int pxa3xx_nand_read_page_hwecc(struct mtd_info *mtd,
 	struct pxa3xx_nand_host *host = nand_get_controller_data(chip);
 	struct pxa3xx_nand_info *info = host->info_data;
 
-	chip->read_buf(mtd, buf, mtd->writesize);
+	nand_read_page_op(chip, page, 0, buf, mtd->writesize);
 	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
 	if (info->retcode == ERR_CORERR && info->use_ecc) {
diff --git a/drivers/mtd/nand/qcom_nandc.c b/drivers/mtd/nand/qcom_nandc.c
index 2656c1ac5646..563b759ffca6 100644
--- a/drivers/mtd/nand/qcom_nandc.c
+++ b/drivers/mtd/nand/qcom_nandc.c
@@ -23,6 +23,7 @@
 #include <linux/of_device.h>
 #include <linux/delay.h>
 #include <linux/dma/qcom_bam_dma.h>
+#include <linux/dma-direct.h> /* XXX: drivers shall never use this directly! */
 
 /* NANDc reg offsets */
 #define	NAND_FLASH_CMD			0x00
@@ -1725,6 +1726,7 @@ static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	u8 *data_buf, *oob_buf = NULL;
 	int ret;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
 	data_buf = buf;
 	oob_buf = oob_required ? chip->oob_poi : NULL;
 
@@ -1750,6 +1752,7 @@ static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
 	int i, ret;
 	int read_loc;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
 	data_buf = buf;
 	oob_buf = chip->oob_poi;
 
@@ -1850,6 +1853,8 @@ static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	u8 *data_buf, *oob_buf;
 	int i, ret;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	clear_read_regs(nandc);
 	clear_bam_transaction(nandc);
 
@@ -1902,6 +1907,9 @@ static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 
 	free_descs(nandc);
 
+	if (!ret)
+		ret = nand_prog_page_end_op(chip);
+
 	return ret;
 }
 
@@ -1916,6 +1924,7 @@ static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
 	u8 *data_buf, *oob_buf;
 	int i, ret;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 	clear_read_regs(nandc);
 	clear_bam_transaction(nandc);
 
@@ -1970,6 +1979,9 @@ static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
 
 	free_descs(nandc);
 
+	if (!ret)
+		ret = nand_prog_page_end_op(chip);
+
 	return ret;
 }
 
@@ -1990,7 +2002,7 @@ static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
 	u8 *oob = chip->oob_poi;
 	int data_size, oob_size;
-	int ret, status = 0;
+	int ret;
 
 	host->use_ecc = true;
 
@@ -2027,11 +2039,7 @@ static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
 		return -EIO;
 	}
 
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
@@ -2081,7 +2089,7 @@ static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
 	struct qcom_nand_host *host = to_qcom_nand_host(chip);
 	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
-	int page, ret, status = 0;
+	int page, ret;
 
 	clear_read_regs(nandc);
 	clear_bam_transaction(nandc);
@@ -2114,11 +2122,7 @@ static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
 		return -EIO;
 	}
 
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_end_op(chip);
 }
 
 /*
@@ -2636,6 +2640,9 @@ static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
 
 	nand_set_flash_node(chip, dn);
 	mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
+	if (!mtd->name)
+		return -ENOMEM;
+
 	mtd->owner = THIS_MODULE;
 	mtd->dev.parent = dev;
 
diff --git a/drivers/mtd/nand/r852.c b/drivers/mtd/nand/r852.c
index fc9287af4614..595635b9e9de 100644
--- a/drivers/mtd/nand/r852.c
+++ b/drivers/mtd/nand/r852.c
@@ -364,7 +364,7 @@ static int r852_wait(struct mtd_info *mtd, struct nand_chip *chip)
 	struct r852_device *dev = nand_get_controller_data(chip);
 
 	unsigned long timeout;
-	int status;
+	u8 status;
 
 	timeout = jiffies + (chip->state == FL_ERASING ?
 		msecs_to_jiffies(400) : msecs_to_jiffies(20));
@@ -373,8 +373,7 @@ static int r852_wait(struct mtd_info *mtd, struct nand_chip *chip)
 		if (chip->dev_ready(mtd))
 			break;
 
-	chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
-	status = (int)chip->read_byte(mtd);
+	nand_status_op(chip, &status);
 
 	/* Unfortunelly, no way to send detailed error status... */
 	if (dev->dma_error) {
@@ -522,9 +521,7 @@ static int r852_ecc_correct(struct mtd_info *mtd, uint8_t *dat,
 static int r852_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 			     int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READOOB, 0, page);
-	chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
-	return 0;
+	return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize);
 }
 
 /*
@@ -1046,7 +1043,7 @@ static int r852_resume(struct device *device)
 	if (dev->card_registred) {
 		r852_engine_enable(dev);
 		dev->chip->select_chip(mtd, 0);
-		dev->chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1);
+		nand_reset_op(dev->chip);
 		dev->chip->select_chip(mtd, -1);
 	}
 
diff --git a/drivers/mtd/nand/sh_flctl.c b/drivers/mtd/nand/sh_flctl.c
index 3c5008a4f5f3..c4e7755448e6 100644
--- a/drivers/mtd/nand/sh_flctl.c
+++ b/drivers/mtd/nand/sh_flctl.c
@@ -614,7 +614,7 @@ static void set_cmd_regs(struct mtd_info *mtd, uint32_t cmd, uint32_t flcmcdr_va
 static int flctl_read_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 				uint8_t *buf, int oob_required, int page)
 {
-	chip->read_buf(mtd, buf, mtd->writesize);
+	nand_read_page_op(chip, page, 0, buf, mtd->writesize);
 	if (oob_required)
 		chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
 	return 0;
@@ -624,9 +624,9 @@ static int flctl_write_page_hwecc(struct mtd_info *mtd, struct nand_chip *chip,
 				  const uint8_t *buf, int oob_required,
 				  int page)
 {
-	chip->write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static void execmd_read_page_sector(struct mtd_info *mtd, int page_addr)
diff --git a/drivers/mtd/nand/sm_common.h b/drivers/mtd/nand/sm_common.h
index d3e028e58b0f..1581671b05ae 100644
--- a/drivers/mtd/nand/sm_common.h
+++ b/drivers/mtd/nand/sm_common.h
@@ -36,7 +36,7 @@ struct sm_oob {
 #define SM_SMALL_OOB_SIZE	8
 
 
-extern int sm_register_device(struct mtd_info *mtd, int smartmedia);
+int sm_register_device(struct mtd_info *mtd, int smartmedia);
 
 
 static inline int sm_sector_valid(struct sm_oob *oob)
diff --git a/drivers/mtd/nand/sunxi_nand.c b/drivers/mtd/nand/sunxi_nand.c
index 82244be3e766..f5a55c63935c 100644
--- a/drivers/mtd/nand/sunxi_nand.c
+++ b/drivers/mtd/nand/sunxi_nand.c
@@ -958,12 +958,12 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 	int ret;
 
 	if (*cur_off != data_off)
-		nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
+		nand_change_read_column_op(nand, data_off, NULL, 0, false);
 
 	sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
 
 	if (data_off + ecc->size != oob_off)
-		nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
+		nand_change_read_column_op(nand, oob_off, NULL, 0, false);
 
 	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 	if (ret)
@@ -991,16 +991,15 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 		 * Re-read the data with the randomizer disabled to identify
 		 * bitflips in erased pages.
 		 */
-		if (nand->options & NAND_NEED_SCRAMBLING) {
-			nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
-			nand->read_buf(mtd, data, ecc->size);
-		} else {
+		if (nand->options & NAND_NEED_SCRAMBLING)
+			nand_change_read_column_op(nand, data_off, data,
+						   ecc->size, false);
+		else
 			memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
 				      ecc->size);
-		}
 
-		nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
-		nand->read_buf(mtd, oob, ecc->bytes + 4);
+		nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
+					   false);
 
 		ret = nand_check_erased_ecc_chunk(data,	ecc->size,
 						  oob, ecc->bytes + 4,
@@ -1011,7 +1010,8 @@ static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
 		memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
 
 		if (oob_required) {
-			nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
+			nand_change_read_column_op(nand, oob_off, NULL, 0,
+						   false);
 			sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
 						      true, page);
 
@@ -1038,8 +1038,8 @@ static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
 		return;
 
 	if (!cur_off || *cur_off != offset)
-		nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
-			      offset + mtd->writesize, -1);
+		nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
+					   false);
 
 	if (!randomize)
 		sunxi_nfc_read_buf(mtd, oob + offset, len);
@@ -1116,9 +1116,9 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
 
 		if (oob_required && !erased) {
 			/* TODO: use DMA to retrieve OOB */
-			nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
-				      mtd->writesize + oob_off, -1);
-			nand->read_buf(mtd, oob, ecc->bytes + 4);
+			nand_change_read_column_op(nand,
+						   mtd->writesize + oob_off,
+						   oob, ecc->bytes + 4, false);
 
 			sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
 							    !i, page);
@@ -1143,18 +1143,17 @@ static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
 			/*
 			 * Re-read the data with the randomizer disabled to
 			 * identify bitflips in erased pages.
+			 * TODO: use DMA to read page in raw mode
 			 */
-			if (randomized) {
-				/* TODO: use DMA to read page in raw mode */
-				nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
-					      data_off, -1);
-				nand->read_buf(mtd, data, ecc->size);
-			}
+			if (randomized)
+				nand_change_read_column_op(nand, data_off,
+							   data, ecc->size,
+							   false);
 
 			/* TODO: use DMA to retrieve OOB */
-			nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
-				      mtd->writesize + oob_off, -1);
-			nand->read_buf(mtd, oob, ecc->bytes + 4);
+			nand_change_read_column_op(nand,
+						   mtd->writesize + oob_off,
+						   oob, ecc->bytes + 4, false);
 
 			ret = nand_check_erased_ecc_chunk(data,	ecc->size,
 							  oob, ecc->bytes + 4,
@@ -1187,12 +1186,12 @@ static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
 	int ret;
 
 	if (data_off != *cur_off)
-		nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
+		nand_change_write_column_op(nand, data_off, NULL, 0, false);
 
 	sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
 
 	if (data_off + ecc->size != oob_off)
-		nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
+		nand_change_write_column_op(nand, oob_off, NULL, 0, false);
 
 	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
 	if (ret)
@@ -1228,8 +1227,8 @@ static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
 		return;
 
 	if (!cur_off || *cur_off != offset)
-		nand->cmdfunc(mtd, NAND_CMD_RNDIN,
-			      offset + mtd->writesize, -1);
+		nand_change_write_column_op(nand, offset + mtd->writesize,
+					    NULL, 0, false);
 
 	sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
 
@@ -1246,6 +1245,8 @@ static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
 	int ret, i, cur_off = 0;
 	bool raw_mode = false;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	sunxi_nfc_hw_ecc_enable(mtd);
 
 	for (i = 0; i < ecc->steps; i++) {
@@ -1279,14 +1280,14 @@ static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
 {
 	int ret;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
 					       chip->ecc.steps);
 	if (ret >= 0)
 		return ret;
 
 	/* Fallback to PIO mode */
-	chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
-
 	return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
 }
 
@@ -1299,6 +1300,8 @@ static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
 	int ret, i, cur_off = 0;
 	unsigned int max_bitflips = 0;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	sunxi_nfc_hw_ecc_enable(mtd);
 
 	for (i = data_offs / ecc->size;
@@ -1330,13 +1333,13 @@ static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
 	int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
 	int ret;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
 	if (ret >= 0)
 		return ret;
 
 	/* Fallback to PIO mode */
-	chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
-
 	return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
 					     buf, page);
 }
@@ -1349,6 +1352,8 @@ static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
 	int ret, i, cur_off = 0;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	sunxi_nfc_hw_ecc_enable(mtd);
 
 	for (i = 0; i < ecc->steps; i++) {
@@ -1370,7 +1375,7 @@ static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
 
 	sunxi_nfc_hw_ecc_disable(mtd);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
@@ -1382,6 +1387,8 @@ static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
 	int ret, i, cur_off = 0;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	sunxi_nfc_hw_ecc_enable(mtd);
 
 	for (i = data_offs / ecc->size;
@@ -1400,7 +1407,7 @@ static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
 
 	sunxi_nfc_hw_ecc_disable(mtd);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
@@ -1430,6 +1437,8 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
 		sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
 	}
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	sunxi_nfc_hw_ecc_enable(mtd);
 	sunxi_nfc_randomizer_config(mtd, page, false);
 	sunxi_nfc_randomizer_enable(mtd);
@@ -1460,7 +1469,7 @@ static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
 		sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
 						 NULL, page);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 
 pio_fallback:
 	return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
@@ -1476,6 +1485,8 @@ static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
 	int ret, i, cur_off = 0;
 	bool raw_mode = false;
 
+	nand_read_page_op(chip, page, 0, NULL, 0);
+
 	sunxi_nfc_hw_ecc_enable(mtd);
 
 	for (i = 0; i < ecc->steps; i++) {
@@ -1512,6 +1523,8 @@ static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
 	struct nand_ecc_ctrl *ecc = &chip->ecc;
 	int ret, i, cur_off = 0;
 
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
+
 	sunxi_nfc_hw_ecc_enable(mtd);
 
 	for (i = 0; i < ecc->steps; i++) {
@@ -1533,41 +1546,33 @@ static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
 
 	sunxi_nfc_hw_ecc_disable(mtd);
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
 					    struct nand_chip *chip,
 					    int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
-
 	chip->pagebuf = -1;
 
-	return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
+	return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page);
 }
 
 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
 					     struct nand_chip *chip,
 					     int page)
 {
-	int ret, status;
-
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
+	int ret;
 
 	chip->pagebuf = -1;
 
-	memset(chip->buffers->databuf, 0xff, mtd->writesize);
-	ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
+	memset(chip->data_buf, 0xff, mtd->writesize);
+	ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page);
 	if (ret)
 		return ret;
 
 	/* Send command to program the OOB data */
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-
-	return status & NAND_STATUS_FAIL ? -EIO : 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static const s32 tWB_lut[] = {6, 12, 16, 20};
@@ -1853,8 +1858,14 @@ static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
 
 	/* Add ECC info retrieval from DT */
 	for (i = 0; i < ARRAY_SIZE(strengths); i++) {
-		if (ecc->strength <= strengths[i])
+		if (ecc->strength <= strengths[i]) {
+			/*
+			 * Update ecc->strength value with the actual strength
+			 * that will be used by the ECC engine.
+			 */
+			ecc->strength = strengths[i];
 			break;
+		}
 	}
 
 	if (i >= ARRAY_SIZE(strengths)) {
diff --git a/drivers/mtd/nand/tango_nand.c b/drivers/mtd/nand/tango_nand.c
index 766906f03943..c5bee00b7f5e 100644
--- a/drivers/mtd/nand/tango_nand.c
+++ b/drivers/mtd/nand/tango_nand.c
@@ -329,7 +329,7 @@ static void aux_read(struct nand_chip *chip, u8 **buf, int len, int *pos)
 
 	if (!*buf) {
 		/* skip over "len" bytes */
-		chip->cmdfunc(mtd, NAND_CMD_RNDOUT, *pos, -1);
+		nand_change_read_column_op(chip, *pos, NULL, 0, false);
 	} else {
 		tango_read_buf(mtd, *buf, len);
 		*buf += len;
@@ -344,7 +344,7 @@ static void aux_write(struct nand_chip *chip, const u8 **buf, int len, int *pos)
 
 	if (!*buf) {
 		/* skip over "len" bytes */
-		chip->cmdfunc(mtd, NAND_CMD_RNDIN, *pos, -1);
+		nand_change_write_column_op(chip, *pos, NULL, 0, false);
 	} else {
 		tango_write_buf(mtd, *buf, len);
 		*buf += len;
@@ -427,7 +427,7 @@ static void raw_write(struct nand_chip *chip, const u8 *buf, const u8 *oob)
 static int tango_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 			       u8 *buf, int oob_required, int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
+	nand_read_page_op(chip, page, 0, NULL, 0);
 	raw_read(chip, buf, chip->oob_poi);
 	return 0;
 }
@@ -435,23 +435,15 @@ static int tango_read_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 static int tango_write_page_raw(struct mtd_info *mtd, struct nand_chip *chip,
 				const u8 *buf, int oob_required, int page)
 {
-	int status;
-
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 	raw_write(chip, buf, chip->oob_poi);
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-
-	status = chip->waitfunc(mtd, chip);
-	if (status & NAND_STATUS_FAIL)
-		return -EIO;
-
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int tango_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 			  int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
+	nand_read_page_op(chip, page, 0, NULL, 0);
 	raw_read(chip, NULL, chip->oob_poi);
 	return 0;
 }
@@ -459,11 +451,9 @@ static int tango_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
 static int tango_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
 			   int page)
 {
-	chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
+	nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 	raw_write(chip, NULL, chip->oob_poi);
-	chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
-	chip->waitfunc(mtd, chip);
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static int oob_ecc(struct mtd_info *mtd, int idx, struct mtd_oob_region *res)
@@ -590,7 +580,6 @@ static int chip_init(struct device *dev, struct device_node *np)
 	ecc->write_page = tango_write_page;
 	ecc->read_oob = tango_read_oob;
 	ecc->write_oob = tango_write_oob;
-	ecc->options = NAND_ECC_CUSTOM_PAGE_ACCESS;
 
 	err = nand_scan_tail(mtd);
 	if (err)
diff --git a/drivers/mtd/nand/tmio_nand.c b/drivers/mtd/nand/tmio_nand.c
index 84dbf32332e1..dcaa924502de 100644
--- a/drivers/mtd/nand/tmio_nand.c
+++ b/drivers/mtd/nand/tmio_nand.c
@@ -192,6 +192,7 @@ tmio_nand_wait(struct mtd_info *mtd, struct nand_chip *nand_chip)
 {
 	struct tmio_nand *tmio = mtd_to_tmio(mtd);
 	long timeout;
+	u8 status;
 
 	/* enable RDYREQ interrupt */
 	tmio_iowrite8(0x0f, tmio->fcr + FCR_ISR);
@@ -212,8 +213,8 @@ tmio_nand_wait(struct mtd_info *mtd, struct nand_chip *nand_chip)
 		dev_warn(&tmio->dev->dev, "timeout waiting for interrupt\n");
 	}
 
-	nand_chip->cmdfunc(mtd, NAND_CMD_STATUS, -1, -1);
-	return nand_chip->read_byte(mtd);
+	nand_status_op(nand_chip, &status);
+	return status;
 }
 
 /*
diff --git a/drivers/mtd/nand/vf610_nfc.c b/drivers/mtd/nand/vf610_nfc.c
index 8037d4b48a05..f367144f3c6f 100644
--- a/drivers/mtd/nand/vf610_nfc.c
+++ b/drivers/mtd/nand/vf610_nfc.c
@@ -560,7 +560,7 @@ static int vf610_nfc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 	int eccsize = chip->ecc.size;
 	int stat;
 
-	vf610_nfc_read_buf(mtd, buf, eccsize);
+	nand_read_page_op(chip, page, 0, buf, eccsize);
 	if (oob_required)
 		vf610_nfc_read_buf(mtd, chip->oob_poi, mtd->oobsize);
 
@@ -580,7 +580,7 @@ static int vf610_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 {
 	struct vf610_nfc *nfc = mtd_to_nfc(mtd);
 
-	vf610_nfc_write_buf(mtd, buf, mtd->writesize);
+	nand_prog_page_begin_op(chip, page, 0, buf, mtd->writesize);
 	if (oob_required)
 		vf610_nfc_write_buf(mtd, chip->oob_poi, mtd->oobsize);
 
@@ -588,7 +588,7 @@ static int vf610_nfc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
 	nfc->use_hw_ecc = true;
 	nfc->write_sz = mtd->writesize + mtd->oobsize;
 
-	return 0;
+	return nand_prog_page_end_op(chip);
 }
 
 static const struct of_device_id vf610_nfc_dt_ids[] = {
@@ -752,10 +752,8 @@ static int vf610_nfc_probe(struct platform_device *pdev)
 		if (mtd->oobsize > 64)
 			mtd->oobsize = 64;
 
-		/*
-		 * mtd->ecclayout is not specified here because we're using the
-		 * default large page ECC layout defined in NAND core.
-		 */
+		/* Use default large page ECC layout defined in NAND core */
+		mtd_set_ooblayout(mtd, &nand_ooblayout_lp_ops);
 		if (chip->ecc.strength == 32) {
 			nfc->ecc_mode = ECC_60_BYTE;
 			chip->ecc.bytes = 60;
diff --git a/drivers/mtd/onenand/Kconfig b/drivers/mtd/onenand/Kconfig
index dcae2f6a2b11..9dc15748947b 100644
--- a/drivers/mtd/onenand/Kconfig
+++ b/drivers/mtd/onenand/Kconfig
@@ -4,8 +4,7 @@ menuconfig MTD_ONENAND
 	depends on HAS_IOMEM
 	help
 	  This enables support for accessing all type of OneNAND flash
-	  devices. For further information see
-	  <http://www.samsung.com/Products/Semiconductor/OneNAND/index.htm>
+	  devices.
 
 if MTD_ONENAND
 
@@ -26,9 +25,11 @@ config MTD_ONENAND_GENERIC
 config MTD_ONENAND_OMAP2
 	tristate "OneNAND on OMAP2/OMAP3 support"
 	depends on ARCH_OMAP2 || ARCH_OMAP3
+	depends on OF || COMPILE_TEST
 	help
-	  Support for a OneNAND flash device connected to an OMAP2/OMAP3 CPU
+	  Support for a OneNAND flash device connected to an OMAP2/OMAP3 SoC
 	  via the GPMC memory controller.
+	  Enable dmaengine and gpiolib for better performance.
 
 config MTD_ONENAND_SAMSUNG
         tristate "OneNAND on Samsung SOC controller support"
diff --git a/drivers/mtd/onenand/omap2.c b/drivers/mtd/onenand/omap2.c
index 24a1388d3031..87c34f607a75 100644
--- a/drivers/mtd/onenand/omap2.c
+++ b/drivers/mtd/onenand/omap2.c
@@ -28,19 +28,18 @@
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/onenand.h>
 #include <linux/mtd/partitions.h>
+#include <linux/of_device.h>
+#include <linux/omap-gpmc.h>
 #include <linux/platform_device.h>
 #include <linux/interrupt.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/io.h>
 #include <linux/slab.h>
-#include <linux/regulator/consumer.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 
 #include <asm/mach/flash.h>
-#include <linux/platform_data/mtd-onenand-omap2.h>
-
-#include <linux/omap-dma.h>
 
 #define DRIVER_NAME "omap2-onenand"
 
@@ -50,24 +49,17 @@ struct omap2_onenand {
 	struct platform_device *pdev;
 	int gpmc_cs;
 	unsigned long phys_base;
-	unsigned int mem_size;
-	int gpio_irq;
+	struct gpio_desc *int_gpiod;
 	struct mtd_info mtd;
 	struct onenand_chip onenand;
 	struct completion irq_done;
 	struct completion dma_done;
-	int dma_channel;
-	int freq;
-	int (*setup)(void __iomem *base, int *freq_ptr);
-	struct regulator *regulator;
-	u8 flags;
+	struct dma_chan *dma_chan;
 };
 
-static void omap2_onenand_dma_cb(int lch, u16 ch_status, void *data)
+static void omap2_onenand_dma_complete_func(void *completion)
 {
-	struct omap2_onenand *c = data;
-
-	complete(&c->dma_done);
+	complete(completion);
 }
 
 static irqreturn_t omap2_onenand_interrupt(int irq, void *dev_id)
@@ -90,6 +82,65 @@ static inline void write_reg(struct omap2_onenand *c, unsigned short value,
 	writew(value, c->onenand.base + reg);
 }
 
+static int omap2_onenand_set_cfg(struct omap2_onenand *c,
+				 bool sr, bool sw,
+				 int latency, int burst_len)
+{
+	unsigned short reg = ONENAND_SYS_CFG1_RDY | ONENAND_SYS_CFG1_INT;
+
+	reg |= latency << ONENAND_SYS_CFG1_BRL_SHIFT;
+
+	switch (burst_len) {
+	case 0:		/* continuous */
+		break;
+	case 4:
+		reg |= ONENAND_SYS_CFG1_BL_4;
+		break;
+	case 8:
+		reg |= ONENAND_SYS_CFG1_BL_8;
+		break;
+	case 16:
+		reg |= ONENAND_SYS_CFG1_BL_16;
+		break;
+	case 32:
+		reg |= ONENAND_SYS_CFG1_BL_32;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (latency > 5)
+		reg |= ONENAND_SYS_CFG1_HF;
+	if (latency > 7)
+		reg |= ONENAND_SYS_CFG1_VHF;
+	if (sr)
+		reg |= ONENAND_SYS_CFG1_SYNC_READ;
+	if (sw)
+		reg |= ONENAND_SYS_CFG1_SYNC_WRITE;
+
+	write_reg(c, reg, ONENAND_REG_SYS_CFG1);
+
+	return 0;
+}
+
+static int omap2_onenand_get_freq(int ver)
+{
+	switch ((ver >> 4) & 0xf) {
+	case 0:
+		return 40;
+	case 1:
+		return 54;
+	case 2:
+		return 66;
+	case 3:
+		return 83;
+	case 4:
+		return 104;
+	}
+
+	return -EINVAL;
+}
+
 static void wait_err(char *msg, int state, unsigned int ctrl, unsigned int intr)
 {
 	printk(KERN_ERR "onenand_wait: %s! state %d ctrl 0x%04x intr 0x%04x\n",
@@ -153,28 +204,22 @@ static int omap2_onenand_wait(struct mtd_info *mtd, int state)
 		if (!(syscfg & ONENAND_SYS_CFG1_IOBE)) {
 			syscfg |= ONENAND_SYS_CFG1_IOBE;
 			write_reg(c, syscfg, ONENAND_REG_SYS_CFG1);
-			if (c->flags & ONENAND_IN_OMAP34XX)
-				/* Add a delay to let GPIO settle */
-				syscfg = read_reg(c, ONENAND_REG_SYS_CFG1);
+			/* Add a delay to let GPIO settle */
+			syscfg = read_reg(c, ONENAND_REG_SYS_CFG1);
 		}
 
 		reinit_completion(&c->irq_done);
-		if (c->gpio_irq) {
-			result = gpio_get_value(c->gpio_irq);
-			if (result == -1) {
-				ctrl = read_reg(c, ONENAND_REG_CTRL_STATUS);
-				intr = read_reg(c, ONENAND_REG_INTERRUPT);
-				wait_err("gpio error", state, ctrl, intr);
-				return -EIO;
-			}
-		} else
-			result = 0;
-		if (result == 0) {
+		result = gpiod_get_value(c->int_gpiod);
+		if (result < 0) {
+			ctrl = read_reg(c, ONENAND_REG_CTRL_STATUS);
+			intr = read_reg(c, ONENAND_REG_INTERRUPT);
+			wait_err("gpio error", state, ctrl, intr);
+			return result;
+		} else if (result == 0) {
 			int retry_cnt = 0;
 retry:
-			result = wait_for_completion_timeout(&c->irq_done,
-						    msecs_to_jiffies(20));
-			if (result == 0) {
+			if (!wait_for_completion_io_timeout(&c->irq_done,
+						msecs_to_jiffies(20))) {
 				/* Timeout after 20ms */
 				ctrl = read_reg(c, ONENAND_REG_CTRL_STATUS);
 				if (ctrl & ONENAND_CTRL_ONGO &&
@@ -291,9 +336,42 @@ static inline int omap2_onenand_bufferram_offset(struct mtd_info *mtd, int area)
 	return 0;
 }
 
-#if defined(CONFIG_ARCH_OMAP3) || defined(MULTI_OMAP2)
+static inline int omap2_onenand_dma_transfer(struct omap2_onenand *c,
+					     dma_addr_t src, dma_addr_t dst,
+					     size_t count)
+{
+	struct dma_async_tx_descriptor *tx;
+	dma_cookie_t cookie;
+
+	tx = dmaengine_prep_dma_memcpy(c->dma_chan, dst, src, count, 0);
+	if (!tx) {
+		dev_err(&c->pdev->dev, "Failed to prepare DMA memcpy\n");
+		return -EIO;
+	}
+
+	reinit_completion(&c->dma_done);
+
+	tx->callback = omap2_onenand_dma_complete_func;
+	tx->callback_param = &c->dma_done;
+
+	cookie = tx->tx_submit(tx);
+	if (dma_submit_error(cookie)) {
+		dev_err(&c->pdev->dev, "Failed to do DMA tx_submit\n");
+		return -EIO;
+	}
+
+	dma_async_issue_pending(c->dma_chan);
+
+	if (!wait_for_completion_io_timeout(&c->dma_done,
+					    msecs_to_jiffies(20))) {
+		dmaengine_terminate_sync(c->dma_chan);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
 
-static int omap3_onenand_read_bufferram(struct mtd_info *mtd, int area,
+static int omap2_onenand_read_bufferram(struct mtd_info *mtd, int area,
 					unsigned char *buffer, int offset,
 					size_t count)
 {
@@ -301,10 +379,9 @@ static int omap3_onenand_read_bufferram(struct mtd_info *mtd, int area,
 	struct onenand_chip *this = mtd->priv;
 	dma_addr_t dma_src, dma_dst;
 	int bram_offset;
-	unsigned long timeout;
 	void *buf = (void *)buffer;
 	size_t xtra;
-	volatile unsigned *done;
+	int ret;
 
 	bram_offset = omap2_onenand_bufferram_offset(mtd, area) + area + offset;
 	if (bram_offset & 3 || (size_t)buf & 3 || count < 384)
@@ -341,25 +418,10 @@ static int omap3_onenand_read_bufferram(struct mtd_info *mtd, int area,
 		goto out_copy;
 	}
 
-	omap_set_dma_transfer_params(c->dma_channel, OMAP_DMA_DATA_TYPE_S32,
-				     count >> 2, 1, 0, 0, 0);
-	omap_set_dma_src_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				dma_src, 0, 0);
-	omap_set_dma_dest_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				 dma_dst, 0, 0);
-
-	reinit_completion(&c->dma_done);
-	omap_start_dma(c->dma_channel);
-
-	timeout = jiffies + msecs_to_jiffies(20);
-	done = &c->dma_done.done;
-	while (time_before(jiffies, timeout))
-		if (*done)
-			break;
-
+	ret = omap2_onenand_dma_transfer(c, dma_src, dma_dst, count);
 	dma_unmap_single(&c->pdev->dev, dma_dst, count, DMA_FROM_DEVICE);
 
-	if (!*done) {
+	if (ret) {
 		dev_err(&c->pdev->dev, "timeout waiting for DMA\n");
 		goto out_copy;
 	}
@@ -371,7 +433,7 @@ static int omap3_onenand_read_bufferram(struct mtd_info *mtd, int area,
 	return 0;
 }
 
-static int omap3_onenand_write_bufferram(struct mtd_info *mtd, int area,
+static int omap2_onenand_write_bufferram(struct mtd_info *mtd, int area,
 					 const unsigned char *buffer,
 					 int offset, size_t count)
 {
@@ -379,9 +441,8 @@ static int omap3_onenand_write_bufferram(struct mtd_info *mtd, int area,
 	struct onenand_chip *this = mtd->priv;
 	dma_addr_t dma_src, dma_dst;
 	int bram_offset;
-	unsigned long timeout;
 	void *buf = (void *)buffer;
-	volatile unsigned *done;
+	int ret;
 
 	bram_offset = omap2_onenand_bufferram_offset(mtd, area) + area + offset;
 	if (bram_offset & 3 || (size_t)buf & 3 || count < 384)
@@ -412,25 +473,10 @@ static int omap3_onenand_write_bufferram(struct mtd_info *mtd, int area,
 		return -1;
 	}
 
-	omap_set_dma_transfer_params(c->dma_channel, OMAP_DMA_DATA_TYPE_S32,
-				     count >> 2, 1, 0, 0, 0);
-	omap_set_dma_src_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				dma_src, 0, 0);
-	omap_set_dma_dest_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				 dma_dst, 0, 0);
-
-	reinit_completion(&c->dma_done);
-	omap_start_dma(c->dma_channel);
-
-	timeout = jiffies + msecs_to_jiffies(20);
-	done = &c->dma_done.done;
-	while (time_before(jiffies, timeout))
-		if (*done)
-			break;
-
+	ret = omap2_onenand_dma_transfer(c, dma_src, dma_dst, count);
 	dma_unmap_single(&c->pdev->dev, dma_src, count, DMA_TO_DEVICE);
 
-	if (!*done) {
+	if (ret) {
 		dev_err(&c->pdev->dev, "timeout waiting for DMA\n");
 		goto out_copy;
 	}
@@ -442,136 +488,6 @@ static int omap3_onenand_write_bufferram(struct mtd_info *mtd, int area,
 	return 0;
 }
 
-#else
-
-static int omap3_onenand_read_bufferram(struct mtd_info *mtd, int area,
-					unsigned char *buffer, int offset,
-					size_t count)
-{
-	return -ENOSYS;
-}
-
-static int omap3_onenand_write_bufferram(struct mtd_info *mtd, int area,
-					 const unsigned char *buffer,
-					 int offset, size_t count)
-{
-	return -ENOSYS;
-}
-
-#endif
-
-#if defined(CONFIG_ARCH_OMAP2) || defined(MULTI_OMAP2)
-
-static int omap2_onenand_read_bufferram(struct mtd_info *mtd, int area,
-					unsigned char *buffer, int offset,
-					size_t count)
-{
-	struct omap2_onenand *c = container_of(mtd, struct omap2_onenand, mtd);
-	struct onenand_chip *this = mtd->priv;
-	dma_addr_t dma_src, dma_dst;
-	int bram_offset;
-
-	bram_offset = omap2_onenand_bufferram_offset(mtd, area) + area + offset;
-	/* DMA is not used.  Revisit PM requirements before enabling it. */
-	if (1 || (c->dma_channel < 0) ||
-	    ((void *) buffer >= (void *) high_memory) || (bram_offset & 3) ||
-	    (((unsigned int) buffer) & 3) || (count < 1024) || (count & 3)) {
-		memcpy(buffer, (__force void *)(this->base + bram_offset),
-		       count);
-		return 0;
-	}
-
-	dma_src = c->phys_base + bram_offset;
-	dma_dst = dma_map_single(&c->pdev->dev, buffer, count,
-				 DMA_FROM_DEVICE);
-	if (dma_mapping_error(&c->pdev->dev, dma_dst)) {
-		dev_err(&c->pdev->dev,
-			"Couldn't DMA map a %d byte buffer\n",
-			count);
-		return -1;
-	}
-
-	omap_set_dma_transfer_params(c->dma_channel, OMAP_DMA_DATA_TYPE_S32,
-				     count / 4, 1, 0, 0, 0);
-	omap_set_dma_src_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				dma_src, 0, 0);
-	omap_set_dma_dest_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				 dma_dst, 0, 0);
-
-	reinit_completion(&c->dma_done);
-	omap_start_dma(c->dma_channel);
-	wait_for_completion(&c->dma_done);
-
-	dma_unmap_single(&c->pdev->dev, dma_dst, count, DMA_FROM_DEVICE);
-
-	return 0;
-}
-
-static int omap2_onenand_write_bufferram(struct mtd_info *mtd, int area,
-					 const unsigned char *buffer,
-					 int offset, size_t count)
-{
-	struct omap2_onenand *c = container_of(mtd, struct omap2_onenand, mtd);
-	struct onenand_chip *this = mtd->priv;
-	dma_addr_t dma_src, dma_dst;
-	int bram_offset;
-
-	bram_offset = omap2_onenand_bufferram_offset(mtd, area) + area + offset;
-	/* DMA is not used.  Revisit PM requirements before enabling it. */
-	if (1 || (c->dma_channel < 0) ||
-	    ((void *) buffer >= (void *) high_memory) || (bram_offset & 3) ||
-	    (((unsigned int) buffer) & 3) || (count < 1024) || (count & 3)) {
-		memcpy((__force void *)(this->base + bram_offset), buffer,
-		       count);
-		return 0;
-	}
-
-	dma_src = dma_map_single(&c->pdev->dev, (void *) buffer, count,
-				 DMA_TO_DEVICE);
-	dma_dst = c->phys_base + bram_offset;
-	if (dma_mapping_error(&c->pdev->dev, dma_src)) {
-		dev_err(&c->pdev->dev,
-			"Couldn't DMA map a %d byte buffer\n",
-			count);
-		return -1;
-	}
-
-	omap_set_dma_transfer_params(c->dma_channel, OMAP_DMA_DATA_TYPE_S16,
-				     count / 2, 1, 0, 0, 0);
-	omap_set_dma_src_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				dma_src, 0, 0);
-	omap_set_dma_dest_params(c->dma_channel, 0, OMAP_DMA_AMODE_POST_INC,
-				 dma_dst, 0, 0);
-
-	reinit_completion(&c->dma_done);
-	omap_start_dma(c->dma_channel);
-	wait_for_completion(&c->dma_done);
-
-	dma_unmap_single(&c->pdev->dev, dma_src, count, DMA_TO_DEVICE);
-
-	return 0;
-}
-
-#else
-
-static int omap2_onenand_read_bufferram(struct mtd_info *mtd, int area,
-					unsigned char *buffer, int offset,
-					size_t count)
-{
-	return -ENOSYS;
-}
-
-static int omap2_onenand_write_bufferram(struct mtd_info *mtd, int area,
-					 const unsigned char *buffer,
-					 int offset, size_t count)
-{
-	return -ENOSYS;
-}
-
-#endif
-
-static struct platform_driver omap2_onenand_driver;
-
 static void omap2_onenand_shutdown(struct platform_device *pdev)
 {
 	struct omap2_onenand *c = dev_get_drvdata(&pdev->dev);
@@ -583,168 +499,117 @@ static void omap2_onenand_shutdown(struct platform_device *pdev)
 	memset((__force void *)c->onenand.base, 0, ONENAND_BUFRAM_SIZE);
 }
 
-static int omap2_onenand_enable(struct mtd_info *mtd)
-{
-	int ret;
-	struct omap2_onenand *c = container_of(mtd, struct omap2_onenand, mtd);
-
-	ret = regulator_enable(c->regulator);
-	if (ret != 0)
-		dev_err(&c->pdev->dev, "can't enable regulator\n");
-
-	return ret;
-}
-
-static int omap2_onenand_disable(struct mtd_info *mtd)
-{
-	int ret;
-	struct omap2_onenand *c = container_of(mtd, struct omap2_onenand, mtd);
-
-	ret = regulator_disable(c->regulator);
-	if (ret != 0)
-		dev_err(&c->pdev->dev, "can't disable regulator\n");
-
-	return ret;
-}
-
 static int omap2_onenand_probe(struct platform_device *pdev)
 {
-	struct omap_onenand_platform_data *pdata;
-	struct omap2_onenand *c;
-	struct onenand_chip *this;
-	int r;
+	u32 val;
+	dma_cap_mask_t mask;
+	int freq, latency, r;
 	struct resource *res;
+	struct omap2_onenand *c;
+	struct gpmc_onenand_info info;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(dev, "error getting memory resource\n");
+		return -EINVAL;
+	}
 
-	pdata = dev_get_platdata(&pdev->dev);
-	if (pdata == NULL) {
-		dev_err(&pdev->dev, "platform data missing\n");
-		return -ENODEV;
+	r = of_property_read_u32(np, "reg", &val);
+	if (r) {
+		dev_err(dev, "reg not found in DT\n");
+		return r;
 	}
 
-	c = kzalloc(sizeof(struct omap2_onenand), GFP_KERNEL);
+	c = devm_kzalloc(dev, sizeof(struct omap2_onenand), GFP_KERNEL);
 	if (!c)
 		return -ENOMEM;
 
 	init_completion(&c->irq_done);
 	init_completion(&c->dma_done);
-	c->flags = pdata->flags;
-	c->gpmc_cs = pdata->cs;
-	c->gpio_irq = pdata->gpio_irq;
-	c->dma_channel = pdata->dma_channel;
-	if (c->dma_channel < 0) {
-		/* if -1, don't use DMA */
-		c->gpio_irq = 0;
-	}
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (res == NULL) {
-		r = -EINVAL;
-		dev_err(&pdev->dev, "error getting memory resource\n");
-		goto err_kfree;
-	}
-
+	c->gpmc_cs = val;
 	c->phys_base = res->start;
-	c->mem_size = resource_size(res);
-
-	if (request_mem_region(c->phys_base, c->mem_size,
-			       pdev->dev.driver->name) == NULL) {
-		dev_err(&pdev->dev, "Cannot reserve memory region at 0x%08lx, size: 0x%x\n",
-						c->phys_base, c->mem_size);
-		r = -EBUSY;
-		goto err_kfree;
-	}
-	c->onenand.base = ioremap(c->phys_base, c->mem_size);
-	if (c->onenand.base == NULL) {
-		r = -ENOMEM;
-		goto err_release_mem_region;
-	}
 
-	if (pdata->onenand_setup != NULL) {
-		r = pdata->onenand_setup(c->onenand.base, &c->freq);
-		if (r < 0) {
-			dev_err(&pdev->dev, "Onenand platform setup failed: "
-				"%d\n", r);
-			goto err_iounmap;
-		}
-		c->setup = pdata->onenand_setup;
+	c->onenand.base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(c->onenand.base))
+		return PTR_ERR(c->onenand.base);
+
+	c->int_gpiod = devm_gpiod_get_optional(dev, "int", GPIOD_IN);
+	if (IS_ERR(c->int_gpiod)) {
+		r = PTR_ERR(c->int_gpiod);
+		/* Just try again if this happens */
+		if (r != -EPROBE_DEFER)
+			dev_err(dev, "error getting gpio: %d\n", r);
+		return r;
 	}
 
-	if (c->gpio_irq) {
-		if ((r = gpio_request(c->gpio_irq, "OneNAND irq")) < 0) {
-			dev_err(&pdev->dev,  "Failed to request GPIO%d for "
-				"OneNAND\n", c->gpio_irq);
-			goto err_iounmap;
-	}
-	gpio_direction_input(c->gpio_irq);
+	if (c->int_gpiod) {
+		r = devm_request_irq(dev, gpiod_to_irq(c->int_gpiod),
+				     omap2_onenand_interrupt,
+				     IRQF_TRIGGER_RISING, "onenand", c);
+		if (r)
+			return r;
 
-	if ((r = request_irq(gpio_to_irq(c->gpio_irq),
-			     omap2_onenand_interrupt, IRQF_TRIGGER_RISING,
-			     pdev->dev.driver->name, c)) < 0)
-		goto err_release_gpio;
+		c->onenand.wait = omap2_onenand_wait;
 	}
 
-	if (c->dma_channel >= 0) {
-		r = omap_request_dma(0, pdev->dev.driver->name,
-				     omap2_onenand_dma_cb, (void *) c,
-				     &c->dma_channel);
-		if (r == 0) {
-			omap_set_dma_write_mode(c->dma_channel,
-						OMAP_DMA_WRITE_NON_POSTED);
-			omap_set_dma_src_data_pack(c->dma_channel, 1);
-			omap_set_dma_src_burst_mode(c->dma_channel,
-						    OMAP_DMA_DATA_BURST_8);
-			omap_set_dma_dest_data_pack(c->dma_channel, 1);
-			omap_set_dma_dest_burst_mode(c->dma_channel,
-						     OMAP_DMA_DATA_BURST_8);
-		} else {
-			dev_info(&pdev->dev,
-				 "failed to allocate DMA for OneNAND, "
-				 "using PIO instead\n");
-			c->dma_channel = -1;
-		}
-	}
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_MEMCPY, mask);
 
-	dev_info(&pdev->dev, "initializing on CS%d, phys base 0x%08lx, virtual "
-		 "base %p, freq %d MHz\n", c->gpmc_cs, c->phys_base,
-		 c->onenand.base, c->freq);
+	c->dma_chan = dma_request_channel(mask, NULL, NULL);
+	if (c->dma_chan) {
+		c->onenand.read_bufferram = omap2_onenand_read_bufferram;
+		c->onenand.write_bufferram = omap2_onenand_write_bufferram;
+	}
 
 	c->pdev = pdev;
 	c->mtd.priv = &c->onenand;
+	c->mtd.dev.parent = dev;
+	mtd_set_of_node(&c->mtd, dev->of_node);
 
-	c->mtd.dev.parent = &pdev->dev;
-	mtd_set_of_node(&c->mtd, pdata->of_node);
-
-	this = &c->onenand;
-	if (c->dma_channel >= 0) {
-		this->wait = omap2_onenand_wait;
-		if (c->flags & ONENAND_IN_OMAP34XX) {
-			this->read_bufferram = omap3_onenand_read_bufferram;
-			this->write_bufferram = omap3_onenand_write_bufferram;
-		} else {
-			this->read_bufferram = omap2_onenand_read_bufferram;
-			this->write_bufferram = omap2_onenand_write_bufferram;
-		}
-	}
+	dev_info(dev, "initializing on CS%d (0x%08lx), va %p, %s mode\n",
+		 c->gpmc_cs, c->phys_base, c->onenand.base,
+		 c->dma_chan ? "DMA" : "PIO");
 
-	if (pdata->regulator_can_sleep) {
-		c->regulator = regulator_get(&pdev->dev, "vonenand");
-		if (IS_ERR(c->regulator)) {
-			dev_err(&pdev->dev,  "Failed to get regulator\n");
-			r = PTR_ERR(c->regulator);
-			goto err_release_dma;
+	if ((r = onenand_scan(&c->mtd, 1)) < 0)
+		goto err_release_dma;
+
+	freq = omap2_onenand_get_freq(c->onenand.version_id);
+	if (freq > 0) {
+		switch (freq) {
+		case 104:
+			latency = 7;
+			break;
+		case 83:
+			latency = 6;
+			break;
+		case 66:
+			latency = 5;
+			break;
+		case 56:
+			latency = 4;
+			break;
+		default:	/* 40 MHz or lower */
+			latency = 3;
+			break;
 		}
-		c->onenand.enable = omap2_onenand_enable;
-		c->onenand.disable = omap2_onenand_disable;
-	}
 
-	if (pdata->skip_initial_unlocking)
-		this->options |= ONENAND_SKIP_INITIAL_UNLOCKING;
+		r = gpmc_omap_onenand_set_timings(dev, c->gpmc_cs,
+						  freq, latency, &info);
+		if (r)
+			goto err_release_onenand;
 
-	if ((r = onenand_scan(&c->mtd, 1)) < 0)
-		goto err_release_regulator;
+		r = omap2_onenand_set_cfg(c, info.sync_read, info.sync_write,
+					  latency, info.burst_len);
+		if (r)
+			goto err_release_onenand;
 
-	r = mtd_device_register(&c->mtd, pdata ? pdata->parts : NULL,
-				pdata ? pdata->nr_parts : 0);
+		if (info.sync_read || info.sync_write)
+			dev_info(dev, "optimized timings for %d MHz\n", freq);
+	}
+
+	r = mtd_device_register(&c->mtd, NULL, 0);
 	if (r)
 		goto err_release_onenand;
 
@@ -754,22 +619,9 @@ static int omap2_onenand_probe(struct platform_device *pdev)
 
 err_release_onenand:
 	onenand_release(&c->mtd);
-err_release_regulator:
-	regulator_put(c->regulator);
 err_release_dma:
-	if (c->dma_channel != -1)
-		omap_free_dma(c->dma_channel);
-	if (c->gpio_irq)
-		free_irq(gpio_to_irq(c->gpio_irq), c);
-err_release_gpio:
-	if (c->gpio_irq)
-		gpio_free(c->gpio_irq);
-err_iounmap:
-	iounmap(c->onenand.base);
-err_release_mem_region:
-	release_mem_region(c->phys_base, c->mem_size);
-err_kfree:
-	kfree(c);
+	if (c->dma_chan)
+		dma_release_channel(c->dma_chan);
 
 	return r;
 }
@@ -779,27 +631,26 @@ static int omap2_onenand_remove(struct platform_device *pdev)
 	struct omap2_onenand *c = dev_get_drvdata(&pdev->dev);
 
 	onenand_release(&c->mtd);
-	regulator_put(c->regulator);
-	if (c->dma_channel != -1)
-		omap_free_dma(c->dma_channel);
+	if (c->dma_chan)
+		dma_release_channel(c->dma_chan);
 	omap2_onenand_shutdown(pdev);
-	if (c->gpio_irq) {
-		free_irq(gpio_to_irq(c->gpio_irq), c);
-		gpio_free(c->gpio_irq);
-	}
-	iounmap(c->onenand.base);
-	release_mem_region(c->phys_base, c->mem_size);
-	kfree(c);
 
 	return 0;
 }
 
+static const struct of_device_id omap2_onenand_id_table[] = {
+	{ .compatible = "ti,omap2-onenand", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, omap2_onenand_id_table);
+
 static struct platform_driver omap2_onenand_driver = {
 	.probe		= omap2_onenand_probe,
 	.remove		= omap2_onenand_remove,
 	.shutdown	= omap2_onenand_shutdown,
 	.driver		= {
 		.name	= DRIVER_NAME,
+		.of_match_table = omap2_onenand_id_table,
 	},
 };
 
diff --git a/drivers/mtd/onenand/onenand_base.c b/drivers/mtd/onenand/onenand_base.c
index 1a6d0e367b89..979f4031f23c 100644
--- a/drivers/mtd/onenand/onenand_base.c
+++ b/drivers/mtd/onenand/onenand_base.c
@@ -1383,15 +1383,6 @@ static int onenand_read_oob_nolock(struct mtd_info *mtd, loff_t from,
 		return -EINVAL;
 	}
 
-	/* Do not allow reads past end of device */
-	if (unlikely(from >= mtd->size ||
-		     column + len > ((mtd->size >> this->page_shift) -
-				     (from >> this->page_shift)) * oobsize)) {
-		printk(KERN_ERR "%s: Attempted to read beyond end of device\n",
-			__func__);
-		return -EINVAL;
-	}
-
 	stats = mtd->ecc_stats;
 
 	readcmd = ONENAND_IS_4KB_PAGE(this) ? ONENAND_CMD_READ : ONENAND_CMD_READOOB;
@@ -1447,38 +1438,6 @@ static int onenand_read_oob_nolock(struct mtd_info *mtd, loff_t from,
 	return 0;
 }
 
-/**
- * onenand_read - [MTD Interface] Read data from flash
- * @param mtd		MTD device structure
- * @param from		offset to read from
- * @param len		number of bytes to read
- * @param retlen	pointer to variable to store the number of read bytes
- * @param buf		the databuffer to put data
- *
- * Read with ecc
-*/
-static int onenand_read(struct mtd_info *mtd, loff_t from, size_t len,
-	size_t *retlen, u_char *buf)
-{
-	struct onenand_chip *this = mtd->priv;
-	struct mtd_oob_ops ops = {
-		.len	= len,
-		.ooblen	= 0,
-		.datbuf	= buf,
-		.oobbuf	= NULL,
-	};
-	int ret;
-
-	onenand_get_device(mtd, FL_READING);
-	ret = ONENAND_IS_4KB_PAGE(this) ?
-		onenand_mlc_read_ops_nolock(mtd, from, &ops) :
-		onenand_read_ops_nolock(mtd, from, &ops);
-	onenand_release_device(mtd);
-
-	*retlen = ops.retlen;
-	return ret;
-}
-
 /**
  * onenand_read_oob - [MTD Interface] Read main and/or out-of-band
  * @param mtd:		MTD device structure
@@ -2056,15 +2015,6 @@ static int onenand_write_oob_nolock(struct mtd_info *mtd, loff_t to,
 		return -EINVAL;
 	}
 
-	/* Do not allow reads past end of device */
-	if (unlikely(to >= mtd->size ||
-		     column + len > ((mtd->size >> this->page_shift) -
-				     (to >> this->page_shift)) * oobsize)) {
-		printk(KERN_ERR "%s: Attempted to write past end of device\n",
-		       __func__);
-		return -EINVAL;
-	}
-
 	oobbuf = this->oob_buf;
 
 	oobcmd = ONENAND_IS_4KB_PAGE(this) ? ONENAND_CMD_PROG : ONENAND_CMD_PROGOOB;
@@ -2128,35 +2078,6 @@ static int onenand_write_oob_nolock(struct mtd_info *mtd, loff_t to,
 	return ret;
 }
 
-/**
- * onenand_write - [MTD Interface] write buffer to FLASH
- * @param mtd		MTD device structure
- * @param to		offset to write to
- * @param len		number of bytes to write
- * @param retlen	pointer to variable to store the number of written bytes
- * @param buf		the data to write
- *
- * Write with ECC
- */
-static int onenand_write(struct mtd_info *mtd, loff_t to, size_t len,
-	size_t *retlen, const u_char *buf)
-{
-	struct mtd_oob_ops ops = {
-		.len	= len,
-		.ooblen	= 0,
-		.datbuf	= (u_char *) buf,
-		.oobbuf	= NULL,
-	};
-	int ret;
-
-	onenand_get_device(mtd, FL_WRITING);
-	ret = onenand_write_ops_nolock(mtd, to, &ops);
-	onenand_release_device(mtd);
-
-	*retlen = ops.retlen;
-	return ret;
-}
-
 /**
  * onenand_write_oob - [MTD Interface] NAND write data and/or out-of-band
  * @param mtd:		MTD device structure
@@ -4038,8 +3959,6 @@ int onenand_scan(struct mtd_info *mtd, int maxchips)
 	mtd->_erase = onenand_erase;
 	mtd->_point = NULL;
 	mtd->_unpoint = NULL;
-	mtd->_read = onenand_read;
-	mtd->_write = onenand_write;
 	mtd->_read_oob = onenand_read_oob;
 	mtd->_write_oob = onenand_write_oob;
 	mtd->_panic_write = onenand_panic_write;
diff --git a/drivers/mtd/onenand/samsung.c b/drivers/mtd/onenand/samsung.c
index af0ac1a7bf8f..2e9d076e445a 100644
--- a/drivers/mtd/onenand/samsung.c
+++ b/drivers/mtd/onenand/samsung.c
@@ -25,8 +25,6 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 
-#include <asm/mach/flash.h>
-
 #include "samsung.h"
 
 enum soc_type {
@@ -129,16 +127,13 @@ struct s3c_onenand {
 	struct platform_device	*pdev;
 	enum soc_type	type;
 	void __iomem	*base;
-	struct resource *base_res;
 	void __iomem	*ahb_addr;
-	struct resource *ahb_res;
 	int		bootram_command;
-	void __iomem	*page_buf;
-	void __iomem	*oob_buf;
+	void		*page_buf;
+	void		*oob_buf;
 	unsigned int	(*mem_addr)(int fba, int fpa, int fsa);
 	unsigned int	(*cmd_map)(unsigned int type, unsigned int val);
 	void __iomem	*dma_addr;
-	struct resource *dma_res;
 	unsigned long	phys_base;
 	struct completion	complete;
 };
@@ -413,8 +408,8 @@ static int s3c_onenand_command(struct mtd_info *mtd, int cmd, loff_t addr,
 	/*
 	 * Emulate Two BufferRAMs and access with 4 bytes pointer
 	 */
-	m = (unsigned int *) onenand->page_buf;
-	s = (unsigned int *) onenand->oob_buf;
+	m = onenand->page_buf;
+	s = onenand->oob_buf;
 
 	if (index) {
 		m += (this->writesize >> 2);
@@ -486,11 +481,11 @@ static unsigned char *s3c_get_bufferram(struct mtd_info *mtd, int area)
 	unsigned char *p;
 
 	if (area == ONENAND_DATARAM) {
-		p = (unsigned char *) onenand->page_buf;
+		p = onenand->page_buf;
 		if (index == 1)
 			p += this->writesize;
 	} else {
-		p = (unsigned char *) onenand->oob_buf;
+		p = onenand->oob_buf;
 		if (index == 1)
 			p += mtd->oobsize;
 	}
@@ -851,15 +846,14 @@ static int s3c_onenand_probe(struct platform_device *pdev)
 	/* No need to check pdata. the platform data is optional */
 
 	size = sizeof(struct mtd_info) + sizeof(struct onenand_chip);
-	mtd = kzalloc(size, GFP_KERNEL);
+	mtd = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 	if (!mtd)
 		return -ENOMEM;
 
-	onenand = kzalloc(sizeof(struct s3c_onenand), GFP_KERNEL);
-	if (!onenand) {
-		err = -ENOMEM;
-		goto onenand_fail;
-	}
+	onenand = devm_kzalloc(&pdev->dev, sizeof(struct s3c_onenand),
+			       GFP_KERNEL);
+	if (!onenand)
+		return -ENOMEM;
 
 	this = (struct onenand_chip *) &mtd[1];
 	mtd->priv = this;
@@ -870,26 +864,12 @@ static int s3c_onenand_probe(struct platform_device *pdev)
 	s3c_onenand_setup(mtd);
 
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!r) {
-		dev_err(&pdev->dev, "no memory resource defined\n");
-		return -ENOENT;
-		goto ahb_resource_failed;
-	}
+	onenand->base = devm_ioremap_resource(&pdev->dev, r);
+	if (IS_ERR(onenand->base))
+		return PTR_ERR(onenand->base);
 
-	onenand->base_res = request_mem_region(r->start, resource_size(r),
-					       pdev->name);
-	if (!onenand->base_res) {
-		dev_err(&pdev->dev, "failed to request memory resource\n");
-		err = -EBUSY;
-		goto resource_failed;
-	}
+	onenand->phys_base = r->start;
 
-	onenand->base = ioremap(r->start, resource_size(r));
-	if (!onenand->base) {
-		dev_err(&pdev->dev, "failed to map memory resource\n");
-		err = -EFAULT;
-		goto ioremap_failed;
-	}
 	/* Set onenand_chip also */
 	this->base = onenand->base;
 
@@ -898,40 +878,20 @@ static int s3c_onenand_probe(struct platform_device *pdev)
 
 	if (onenand->type != TYPE_S5PC110) {
 		r = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-		if (!r) {
-			dev_err(&pdev->dev, "no buffer memory resource defined\n");
-			err = -ENOENT;
-			goto ahb_resource_failed;
-		}
-
-		onenand->ahb_res = request_mem_region(r->start, resource_size(r),
-						      pdev->name);
-		if (!onenand->ahb_res) {
-			dev_err(&pdev->dev, "failed to request buffer memory resource\n");
-			err = -EBUSY;
-			goto ahb_resource_failed;
-		}
-
-		onenand->ahb_addr = ioremap(r->start, resource_size(r));
-		if (!onenand->ahb_addr) {
-			dev_err(&pdev->dev, "failed to map buffer memory resource\n");
-			err = -EINVAL;
-			goto ahb_ioremap_failed;
-		}
+		onenand->ahb_addr = devm_ioremap_resource(&pdev->dev, r);
+		if (IS_ERR(onenand->ahb_addr))
+			return PTR_ERR(onenand->ahb_addr);
 
 		/* Allocate 4KiB BufferRAM */
-		onenand->page_buf = kzalloc(SZ_4K, GFP_KERNEL);
-		if (!onenand->page_buf) {
-			err = -ENOMEM;
-			goto page_buf_fail;
-		}
+		onenand->page_buf = devm_kzalloc(&pdev->dev, SZ_4K,
+						 GFP_KERNEL);
+		if (!onenand->page_buf)
+			return -ENOMEM;
 
 		/* Allocate 128 SpareRAM */
-		onenand->oob_buf = kzalloc(128, GFP_KERNEL);
-		if (!onenand->oob_buf) {
-			err = -ENOMEM;
-			goto oob_buf_fail;
-		}
+		onenand->oob_buf = devm_kzalloc(&pdev->dev, 128, GFP_KERNEL);
+		if (!onenand->oob_buf)
+			return -ENOMEM;
 
 		/* S3C doesn't handle subpage write */
 		mtd->subpage_sft = 0;
@@ -939,28 +899,9 @@ static int s3c_onenand_probe(struct platform_device *pdev)
 
 	} else { /* S5PC110 */
 		r = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-		if (!r) {
-			dev_err(&pdev->dev, "no dma memory resource defined\n");
-			err = -ENOENT;
-			goto dma_resource_failed;
-		}
-
-		onenand->dma_res = request_mem_region(r->start, resource_size(r),
-						      pdev->name);
-		if (!onenand->dma_res) {
-			dev_err(&pdev->dev, "failed to request dma memory resource\n");
-			err = -EBUSY;
-			goto dma_resource_failed;
-		}
-
-		onenand->dma_addr = ioremap(r->start, resource_size(r));
-		if (!onenand->dma_addr) {
-			dev_err(&pdev->dev, "failed to map dma memory resource\n");
-			err = -EINVAL;
-			goto dma_ioremap_failed;
-		}
-
-		onenand->phys_base = onenand->base_res->start;
+		onenand->dma_addr = devm_ioremap_resource(&pdev->dev, r);
+		if (IS_ERR(onenand->dma_addr))
+			return PTR_ERR(onenand->dma_addr);
 
 		s5pc110_dma_ops = s5pc110_dma_poll;
 		/* Interrupt support */
@@ -968,19 +909,20 @@ static int s3c_onenand_probe(struct platform_device *pdev)
 		if (r) {
 			init_completion(&onenand->complete);
 			s5pc110_dma_ops = s5pc110_dma_irq;
-			err = request_irq(r->start, s5pc110_onenand_irq,
-					IRQF_SHARED, "onenand", &onenand);
+			err = devm_request_irq(&pdev->dev, r->start,
+					       s5pc110_onenand_irq,
+					       IRQF_SHARED, "onenand",
+					       &onenand);
 			if (err) {
 				dev_err(&pdev->dev, "failed to get irq\n");
-				goto scan_failed;
+				return err;
 			}
 		}
 	}
 
-	if (onenand_scan(mtd, 1)) {
-		err = -EFAULT;
-		goto scan_failed;
-	}
+	err = onenand_scan(mtd, 1);
+	if (err)
+		return err;
 
 	if (onenand->type != TYPE_S5PC110) {
 		/* S3C doesn't handle subpage write */
@@ -994,40 +936,15 @@ static int s3c_onenand_probe(struct platform_device *pdev)
 	err = mtd_device_parse_register(mtd, NULL, NULL,
 					pdata ? pdata->parts : NULL,
 					pdata ? pdata->nr_parts : 0);
+	if (err) {
+		dev_err(&pdev->dev, "failed to parse partitions and register the MTD device\n");
+		onenand_release(mtd);
+		return err;
+	}
 
 	platform_set_drvdata(pdev, mtd);
 
 	return 0;
-
-scan_failed:
-	if (onenand->dma_addr)
-		iounmap(onenand->dma_addr);
-dma_ioremap_failed:
-	if (onenand->dma_res)
-		release_mem_region(onenand->dma_res->start,
-				   resource_size(onenand->dma_res));
-	kfree(onenand->oob_buf);
-oob_buf_fail:
-	kfree(onenand->page_buf);
-page_buf_fail:
-	if (onenand->ahb_addr)
-		iounmap(onenand->ahb_addr);
-ahb_ioremap_failed:
-	if (onenand->ahb_res)
-		release_mem_region(onenand->ahb_res->start,
-				   resource_size(onenand->ahb_res));
-dma_resource_failed:
-ahb_resource_failed:
-	iounmap(onenand->base);
-ioremap_failed:
-	if (onenand->base_res)
-		release_mem_region(onenand->base_res->start,
-				   resource_size(onenand->base_res));
-resource_failed:
-	kfree(onenand);
-onenand_fail:
-	kfree(mtd);
-	return err;
 }
 
 static int s3c_onenand_remove(struct platform_device *pdev)
@@ -1035,25 +952,7 @@ static int s3c_onenand_remove(struct platform_device *pdev)
 	struct mtd_info *mtd = platform_get_drvdata(pdev);
 
 	onenand_release(mtd);
-	if (onenand->ahb_addr)
-		iounmap(onenand->ahb_addr);
-	if (onenand->ahb_res)
-		release_mem_region(onenand->ahb_res->start,
-				   resource_size(onenand->ahb_res));
-	if (onenand->dma_addr)
-		iounmap(onenand->dma_addr);
-	if (onenand->dma_res)
-		release_mem_region(onenand->dma_res->start,
-				   resource_size(onenand->dma_res));
-
-	iounmap(onenand->base);
-	release_mem_region(onenand->base_res->start,
-			   resource_size(onenand->base_res));
-
-	kfree(onenand->oob_buf);
-	kfree(onenand->page_buf);
-	kfree(onenand);
-	kfree(mtd);
+
 	return 0;
 }
 
diff --git a/drivers/mtd/parsers/sharpslpart.c b/drivers/mtd/parsers/sharpslpart.c
index 5fe0079ea5ed..8893dc82a5c8 100644
--- a/drivers/mtd/parsers/sharpslpart.c
+++ b/drivers/mtd/parsers/sharpslpart.c
@@ -192,7 +192,7 @@ static int sharpsl_nand_init_ftl(struct mtd_info *mtd, struct sharpsl_ftl *ftl)
 
 	/* create physical-logical table */
 	for (block_num = 0; block_num < phymax; block_num++) {
-		block_adr = block_num * mtd->erasesize;
+		block_adr = (loff_t)block_num * mtd->erasesize;
 
 		if (mtd_block_isbad(mtd, block_adr))
 			continue;
@@ -219,7 +219,7 @@ static int sharpsl_nand_init_ftl(struct mtd_info *mtd, struct sharpsl_ftl *ftl)
 	return ret;
 }
 
-void sharpsl_nand_cleanup_ftl(struct sharpsl_ftl *ftl)
+static void sharpsl_nand_cleanup_ftl(struct sharpsl_ftl *ftl)
 {
 	kfree(ftl->log2phy);
 }
@@ -244,7 +244,7 @@ static int sharpsl_nand_read_laddr(struct mtd_info *mtd,
 		return -EINVAL;
 
 	block_num = ftl->log2phy[log_num];
-	block_adr = block_num * mtd->erasesize;
+	block_adr = (loff_t)block_num * mtd->erasesize;
 	block_ofs = mtd_mod_by_eb((u32)from, mtd);
 
 	err = mtd_read(mtd, block_adr + block_ofs, len, &retlen, buf);
diff --git a/drivers/mtd/spi-nor/cadence-quadspi.c b/drivers/mtd/spi-nor/cadence-quadspi.c
index 75a2bc447a99..4b8e9183489a 100644
--- a/drivers/mtd/spi-nor/cadence-quadspi.c
+++ b/drivers/mtd/spi-nor/cadence-quadspi.c
@@ -58,6 +58,7 @@ struct cqspi_flash_pdata {
 	u8		data_width;
 	u8		cs;
 	bool		registered;
+	bool		use_direct_mode;
 };
 
 struct cqspi_st {
@@ -68,6 +69,7 @@ struct cqspi_st {
 
 	void __iomem		*iobase;
 	void __iomem		*ahb_base;
+	resource_size_t		ahb_size;
 	struct completion	transfer_complete;
 	struct mutex		bus_mutex;
 
@@ -103,6 +105,7 @@ struct cqspi_st {
 /* Register map */
 #define CQSPI_REG_CONFIG			0x00
 #define CQSPI_REG_CONFIG_ENABLE_MASK		BIT(0)
+#define CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL	BIT(7)
 #define CQSPI_REG_CONFIG_DECODE_MASK		BIT(9)
 #define CQSPI_REG_CONFIG_CHIPSELECT_LSB		10
 #define CQSPI_REG_CONFIG_DMA_MASK		BIT(15)
@@ -450,8 +453,7 @@ static int cqspi_command_write_addr(struct spi_nor *nor,
 	return cqspi_exec_flash_cmd(cqspi, reg);
 }
 
-static int cqspi_indirect_read_setup(struct spi_nor *nor,
-				     const unsigned int from_addr)
+static int cqspi_read_setup(struct spi_nor *nor)
 {
 	struct cqspi_flash_pdata *f_pdata = nor->priv;
 	struct cqspi_st *cqspi = f_pdata->cqspi;
@@ -459,8 +461,6 @@ static int cqspi_indirect_read_setup(struct spi_nor *nor,
 	unsigned int dummy_clk = 0;
 	unsigned int reg;
 
-	writel(from_addr, reg_base + CQSPI_REG_INDIRECTRDSTARTADDR);
-
 	reg = nor->read_opcode << CQSPI_REG_RD_INSTR_OPCODE_LSB;
 	reg |= cqspi_calc_rdreg(nor, nor->read_opcode);
 
@@ -493,8 +493,8 @@ static int cqspi_indirect_read_setup(struct spi_nor *nor,
 	return 0;
 }
 
-static int cqspi_indirect_read_execute(struct spi_nor *nor,
-				       u8 *rxbuf, const unsigned n_rx)
+static int cqspi_indirect_read_execute(struct spi_nor *nor, u8 *rxbuf,
+				       loff_t from_addr, const size_t n_rx)
 {
 	struct cqspi_flash_pdata *f_pdata = nor->priv;
 	struct cqspi_st *cqspi = f_pdata->cqspi;
@@ -504,6 +504,7 @@ static int cqspi_indirect_read_execute(struct spi_nor *nor,
 	unsigned int bytes_to_read = 0;
 	int ret = 0;
 
+	writel(from_addr, reg_base + CQSPI_REG_INDIRECTRDSTARTADDR);
 	writel(remaining, reg_base + CQSPI_REG_INDIRECTRDBYTES);
 
 	/* Clear all interrupts. */
@@ -570,8 +571,7 @@ static int cqspi_indirect_read_execute(struct spi_nor *nor,
 	return ret;
 }
 
-static int cqspi_indirect_write_setup(struct spi_nor *nor,
-				      const unsigned int to_addr)
+static int cqspi_write_setup(struct spi_nor *nor)
 {
 	unsigned int reg;
 	struct cqspi_flash_pdata *f_pdata = nor->priv;
@@ -584,8 +584,6 @@ static int cqspi_indirect_write_setup(struct spi_nor *nor,
 	reg = cqspi_calc_rdreg(nor, nor->program_opcode);
 	writel(reg, reg_base + CQSPI_REG_RD_INSTR);
 
-	writel(to_addr, reg_base + CQSPI_REG_INDIRECTWRSTARTADDR);
-
 	reg = readl(reg_base + CQSPI_REG_SIZE);
 	reg &= ~CQSPI_REG_SIZE_ADDRESS_MASK;
 	reg |= (nor->addr_width - 1);
@@ -593,8 +591,8 @@ static int cqspi_indirect_write_setup(struct spi_nor *nor,
 	return 0;
 }
 
-static int cqspi_indirect_write_execute(struct spi_nor *nor,
-					const u8 *txbuf, const unsigned n_tx)
+static int cqspi_indirect_write_execute(struct spi_nor *nor, loff_t to_addr,
+					const u8 *txbuf, const size_t n_tx)
 {
 	const unsigned int page_size = nor->page_size;
 	struct cqspi_flash_pdata *f_pdata = nor->priv;
@@ -604,6 +602,7 @@ static int cqspi_indirect_write_execute(struct spi_nor *nor,
 	unsigned int write_bytes;
 	int ret;
 
+	writel(to_addr, reg_base + CQSPI_REG_INDIRECTWRSTARTADDR);
 	writel(remaining, reg_base + CQSPI_REG_INDIRECTWRBYTES);
 
 	/* Clear all interrupts. */
@@ -894,17 +893,22 @@ static int cqspi_set_protocol(struct spi_nor *nor, const int read)
 static ssize_t cqspi_write(struct spi_nor *nor, loff_t to,
 			   size_t len, const u_char *buf)
 {
+	struct cqspi_flash_pdata *f_pdata = nor->priv;
+	struct cqspi_st *cqspi = f_pdata->cqspi;
 	int ret;
 
 	ret = cqspi_set_protocol(nor, 0);
 	if (ret)
 		return ret;
 
-	ret = cqspi_indirect_write_setup(nor, to);
+	ret = cqspi_write_setup(nor);
 	if (ret)
 		return ret;
 
-	ret = cqspi_indirect_write_execute(nor, buf, len);
+	if (f_pdata->use_direct_mode)
+		memcpy_toio(cqspi->ahb_base + to, buf, len);
+	else
+		ret = cqspi_indirect_write_execute(nor, to, buf, len);
 	if (ret)
 		return ret;
 
@@ -914,17 +918,22 @@ static ssize_t cqspi_write(struct spi_nor *nor, loff_t to,
 static ssize_t cqspi_read(struct spi_nor *nor, loff_t from,
 			  size_t len, u_char *buf)
 {
+	struct cqspi_flash_pdata *f_pdata = nor->priv;
+	struct cqspi_st *cqspi = f_pdata->cqspi;
 	int ret;
 
 	ret = cqspi_set_protocol(nor, 1);
 	if (ret)
 		return ret;
 
-	ret = cqspi_indirect_read_setup(nor, from);
+	ret = cqspi_read_setup(nor);
 	if (ret)
 		return ret;
 
-	ret = cqspi_indirect_read_execute(nor, buf, len);
+	if (f_pdata->use_direct_mode)
+		memcpy_fromio(buf, cqspi->ahb_base + from, len);
+	else
+		ret = cqspi_indirect_read_execute(nor, buf, from, len);
 	if (ret)
 		return ret;
 
@@ -1059,6 +1068,8 @@ static int cqspi_of_get_pdata(struct platform_device *pdev)
 
 static void cqspi_controller_init(struct cqspi_st *cqspi)
 {
+	u32 reg;
+
 	cqspi_controller_enable(cqspi, 0);
 
 	/* Configure the remap address register, no remap */
@@ -1081,6 +1092,11 @@ static void cqspi_controller_init(struct cqspi_st *cqspi)
 	writel(cqspi->fifo_depth * cqspi->fifo_width / 8,
 	       cqspi->iobase + CQSPI_REG_INDIRECTWRWATERMARK);
 
+	/* Enable Direct Access Controller */
+	reg = readl(cqspi->iobase + CQSPI_REG_CONFIG);
+	reg |= CQSPI_REG_CONFIG_ENB_DIR_ACC_CTRL;
+	writel(reg, cqspi->iobase + CQSPI_REG_CONFIG);
+
 	cqspi_controller_enable(cqspi, 1);
 }
 
@@ -1156,6 +1172,12 @@ static int cqspi_setup_flash(struct cqspi_st *cqspi, struct device_node *np)
 			goto err;
 
 		f_pdata->registered = true;
+
+		if (mtd->size <= cqspi->ahb_size) {
+			f_pdata->use_direct_mode = true;
+			dev_dbg(nor->dev, "using direct mode for %s\n",
+				mtd->name);
+		}
 	}
 
 	return 0;
@@ -1215,6 +1237,7 @@ static int cqspi_probe(struct platform_device *pdev)
 		dev_err(dev, "Cannot remap AHB address.\n");
 		return PTR_ERR(cqspi->ahb_base);
 	}
+	cqspi->ahb_size = resource_size(res_ahb);
 
 	init_completion(&cqspi->transfer_complete);
 
diff --git a/drivers/mtd/spi-nor/fsl-quadspi.c b/drivers/mtd/spi-nor/fsl-quadspi.c
index f17d22435bfc..2901c7bd9e30 100644
--- a/drivers/mtd/spi-nor/fsl-quadspi.c
+++ b/drivers/mtd/spi-nor/fsl-quadspi.c
@@ -801,10 +801,10 @@ static int fsl_qspi_nor_setup_last(struct fsl_qspi *q)
 }
 
 static const struct of_device_id fsl_qspi_dt_ids[] = {
-	{ .compatible = "fsl,vf610-qspi", .data = (void *)&vybrid_data, },
-	{ .compatible = "fsl,imx6sx-qspi", .data = (void *)&imx6sx_data, },
-	{ .compatible = "fsl,imx7d-qspi", .data = (void *)&imx7d_data, },
-	{ .compatible = "fsl,imx6ul-qspi", .data = (void *)&imx6ul_data, },
+	{ .compatible = "fsl,vf610-qspi", .data = &vybrid_data, },
+	{ .compatible = "fsl,imx6sx-qspi", .data = &imx6sx_data, },
+	{ .compatible = "fsl,imx7d-qspi", .data = &imx7d_data, },
+	{ .compatible = "fsl,imx6ul-qspi", .data = &imx6ul_data, },
 	{ .compatible = "fsl,ls1021a-qspi", .data = (void *)&ls1021a_data, },
 	{ /* sentinel */ }
 };
diff --git a/drivers/mtd/spi-nor/intel-spi.c b/drivers/mtd/spi-nor/intel-spi.c
index ef034d898a23..699951523179 100644
--- a/drivers/mtd/spi-nor/intel-spi.c
+++ b/drivers/mtd/spi-nor/intel-spi.c
@@ -138,7 +138,6 @@
  * @erase_64k: 64k erase supported
  * @opcodes: Opcodes which are supported. This are programmed by BIOS
  *           before it locks down the controller.
- * @preopcodes: Preopcodes which are supported.
  */
 struct intel_spi {
 	struct device *dev;
@@ -155,7 +154,6 @@ struct intel_spi {
 	bool swseq_erase;
 	bool erase_64k;
 	u8 opcodes[8];
-	u8 preopcodes[2];
 };
 
 static bool writeable;
@@ -400,10 +398,6 @@ static int intel_spi_init(struct intel_spi *ispi)
 				ispi->opcodes[i] = opmenu0 >> i * 8;
 				ispi->opcodes[i + 4] = opmenu1 >> i * 8;
 			}
-
-			val = readl(ispi->sregs + PREOP_OPTYPE);
-			ispi->preopcodes[0] = val;
-			ispi->preopcodes[1] = val >> 8;
 		}
 	}
 
diff --git a/drivers/mtd/spi-nor/mtk-quadspi.c b/drivers/mtd/spi-nor/mtk-quadspi.c
index abe455ccd68b..5442993b71ff 100644
--- a/drivers/mtd/spi-nor/mtk-quadspi.c
+++ b/drivers/mtd/spi-nor/mtk-quadspi.c
@@ -110,7 +110,7 @@
 #define MTK_NOR_PRG_REG(n)		(MTK_NOR_PRGDATA0_REG + 4 * (n))
 #define MTK_NOR_SHREG(n)		(MTK_NOR_SHREG0_REG + 4 * (n))
 
-struct mt8173_nor {
+struct mtk_nor {
 	struct spi_nor nor;
 	struct device *dev;
 	void __iomem *base;	/* nor flash base address */
@@ -118,48 +118,48 @@ struct mt8173_nor {
 	struct clk *nor_clk;
 };
 
-static void mt8173_nor_set_read_mode(struct mt8173_nor *mt8173_nor)
+static void mtk_nor_set_read_mode(struct mtk_nor *mtk_nor)
 {
-	struct spi_nor *nor = &mt8173_nor->nor;
+	struct spi_nor *nor = &mtk_nor->nor;
 
 	switch (nor->read_proto) {
 	case SNOR_PROTO_1_1_1:
-		writeb(nor->read_opcode, mt8173_nor->base +
+		writeb(nor->read_opcode, mtk_nor->base +
 		       MTK_NOR_PRGDATA3_REG);
-		writeb(MTK_NOR_FAST_READ, mt8173_nor->base +
+		writeb(MTK_NOR_FAST_READ, mtk_nor->base +
 		       MTK_NOR_CFG1_REG);
 		break;
 	case SNOR_PROTO_1_1_2:
-		writeb(nor->read_opcode, mt8173_nor->base +
+		writeb(nor->read_opcode, mtk_nor->base +
 		       MTK_NOR_PRGDATA3_REG);
-		writeb(MTK_NOR_DUAL_READ_EN, mt8173_nor->base +
+		writeb(MTK_NOR_DUAL_READ_EN, mtk_nor->base +
 		       MTK_NOR_DUAL_REG);
 		break;
 	case SNOR_PROTO_1_1_4:
-		writeb(nor->read_opcode, mt8173_nor->base +
+		writeb(nor->read_opcode, mtk_nor->base +
 		       MTK_NOR_PRGDATA4_REG);
-		writeb(MTK_NOR_QUAD_READ_EN, mt8173_nor->base +
+		writeb(MTK_NOR_QUAD_READ_EN, mtk_nor->base +
 		       MTK_NOR_DUAL_REG);
 		break;
 	default:
-		writeb(MTK_NOR_DUAL_DISABLE, mt8173_nor->base +
+		writeb(MTK_NOR_DUAL_DISABLE, mtk_nor->base +
 		       MTK_NOR_DUAL_REG);
 		break;
 	}
 }
 
-static int mt8173_nor_execute_cmd(struct mt8173_nor *mt8173_nor, u8 cmdval)
+static int mtk_nor_execute_cmd(struct mtk_nor *mtk_nor, u8 cmdval)
 {
 	int reg;
 	u8 val = cmdval & 0x1f;
 
-	writeb(cmdval, mt8173_nor->base + MTK_NOR_CMD_REG);
-	return readl_poll_timeout(mt8173_nor->base + MTK_NOR_CMD_REG, reg,
+	writeb(cmdval, mtk_nor->base + MTK_NOR_CMD_REG);
+	return readl_poll_timeout(mtk_nor->base + MTK_NOR_CMD_REG, reg,
 				  !(reg & val), 100, 10000);
 }
 
-static int mt8173_nor_do_tx_rx(struct mt8173_nor *mt8173_nor, u8 op,
-			       u8 *tx, int txlen, u8 *rx, int rxlen)
+static int mtk_nor_do_tx_rx(struct mtk_nor *mtk_nor, u8 op,
+			    u8 *tx, int txlen, u8 *rx, int rxlen)
 {
 	int len = 1 + txlen + rxlen;
 	int i, ret, idx;
@@ -167,26 +167,26 @@ static int mt8173_nor_do_tx_rx(struct mt8173_nor *mt8173_nor, u8 op,
 	if (len > MTK_NOR_MAX_SHIFT)
 		return -EINVAL;
 
-	writeb(len * 8, mt8173_nor->base + MTK_NOR_CNT_REG);
+	writeb(len * 8, mtk_nor->base + MTK_NOR_CNT_REG);
 
 	/* start at PRGDATA5, go down to PRGDATA0 */
 	idx = MTK_NOR_MAX_RX_TX_SHIFT - 1;
 
 	/* opcode */
-	writeb(op, mt8173_nor->base + MTK_NOR_PRG_REG(idx));
+	writeb(op, mtk_nor->base + MTK_NOR_PRG_REG(idx));
 	idx--;
 
 	/* program TX data */
 	for (i = 0; i < txlen; i++, idx--)
-		writeb(tx[i], mt8173_nor->base + MTK_NOR_PRG_REG(idx));
+		writeb(tx[i], mtk_nor->base + MTK_NOR_PRG_REG(idx));
 
 	/* clear out rest of TX registers */
 	while (idx >= 0) {
-		writeb(0, mt8173_nor->base + MTK_NOR_PRG_REG(idx));
+		writeb(0, mtk_nor->base + MTK_NOR_PRG_REG(idx));
 		idx--;
 	}
 
-	ret = mt8173_nor_execute_cmd(mt8173_nor, MTK_NOR_PRG_CMD);
+	ret = mtk_nor_execute_cmd(mtk_nor, MTK_NOR_PRG_CMD);
 	if (ret)
 		return ret;
 
@@ -195,20 +195,20 @@ static int mt8173_nor_do_tx_rx(struct mt8173_nor *mt8173_nor, u8 op,
 
 	/* read out RX data */
 	for (i = 0; i < rxlen; i++, idx--)
-		rx[i] = readb(mt8173_nor->base + MTK_NOR_SHREG(idx));
+		rx[i] = readb(mtk_nor->base + MTK_NOR_SHREG(idx));
 
 	return 0;
 }
 
 /* Do a WRSR (Write Status Register) command */
-static int mt8173_nor_wr_sr(struct mt8173_nor *mt8173_nor, u8 sr)
+static int mtk_nor_wr_sr(struct mtk_nor *mtk_nor, u8 sr)
 {
-	writeb(sr, mt8173_nor->base + MTK_NOR_PRGDATA5_REG);
-	writeb(8, mt8173_nor->base + MTK_NOR_CNT_REG);
-	return mt8173_nor_execute_cmd(mt8173_nor, MTK_NOR_WRSR_CMD);
+	writeb(sr, mtk_nor->base + MTK_NOR_PRGDATA5_REG);
+	writeb(8, mtk_nor->base + MTK_NOR_CNT_REG);
+	return mtk_nor_execute_cmd(mtk_nor, MTK_NOR_WRSR_CMD);
 }
 
-static int mt8173_nor_write_buffer_enable(struct mt8173_nor *mt8173_nor)
+static int mtk_nor_write_buffer_enable(struct mtk_nor *mtk_nor)
 {
 	u8 reg;
 
@@ -216,27 +216,27 @@ static int mt8173_nor_write_buffer_enable(struct mt8173_nor *mt8173_nor)
 	 * 0: pre-fetch buffer use for read
 	 * 1: pre-fetch buffer use for page program
 	 */
-	writel(MTK_NOR_WR_BUF_ENABLE, mt8173_nor->base + MTK_NOR_CFG2_REG);
-	return readb_poll_timeout(mt8173_nor->base + MTK_NOR_CFG2_REG, reg,
+	writel(MTK_NOR_WR_BUF_ENABLE, mtk_nor->base + MTK_NOR_CFG2_REG);
+	return readb_poll_timeout(mtk_nor->base + MTK_NOR_CFG2_REG, reg,
 				  0x01 == (reg & 0x01), 100, 10000);
 }
 
-static int mt8173_nor_write_buffer_disable(struct mt8173_nor *mt8173_nor)
+static int mtk_nor_write_buffer_disable(struct mtk_nor *mtk_nor)
 {
 	u8 reg;
 
-	writel(MTK_NOR_WR_BUF_DISABLE, mt8173_nor->base + MTK_NOR_CFG2_REG);
-	return readb_poll_timeout(mt8173_nor->base + MTK_NOR_CFG2_REG, reg,
+	writel(MTK_NOR_WR_BUF_DISABLE, mtk_nor->base + MTK_NOR_CFG2_REG);
+	return readb_poll_timeout(mtk_nor->base + MTK_NOR_CFG2_REG, reg,
 				  MTK_NOR_WR_BUF_DISABLE == (reg & 0x1), 100,
 				  10000);
 }
 
-static void mt8173_nor_set_addr_width(struct mt8173_nor *mt8173_nor)
+static void mtk_nor_set_addr_width(struct mtk_nor *mtk_nor)
 {
 	u8 val;
-	struct spi_nor *nor = &mt8173_nor->nor;
+	struct spi_nor *nor = &mtk_nor->nor;
 
-	val = readb(mt8173_nor->base + MTK_NOR_DUAL_REG);
+	val = readb(mtk_nor->base + MTK_NOR_DUAL_REG);
 
 	switch (nor->addr_width) {
 	case 3:
@@ -246,115 +246,115 @@ static void mt8173_nor_set_addr_width(struct mt8173_nor *mt8173_nor)
 		val |= MTK_NOR_4B_ADDR_EN;
 		break;
 	default:
-		dev_warn(mt8173_nor->dev, "Unexpected address width %u.\n",
+		dev_warn(mtk_nor->dev, "Unexpected address width %u.\n",
 			 nor->addr_width);
 		break;
 	}
 
-	writeb(val, mt8173_nor->base + MTK_NOR_DUAL_REG);
+	writeb(val, mtk_nor->base + MTK_NOR_DUAL_REG);
 }
 
-static void mt8173_nor_set_addr(struct mt8173_nor *mt8173_nor, u32 addr)
+static void mtk_nor_set_addr(struct mtk_nor *mtk_nor, u32 addr)
 {
 	int i;
 
-	mt8173_nor_set_addr_width(mt8173_nor);
+	mtk_nor_set_addr_width(mtk_nor);
 
 	for (i = 0; i < 3; i++) {
-		writeb(addr & 0xff, mt8173_nor->base + MTK_NOR_RADR0_REG + i * 4);
+		writeb(addr & 0xff, mtk_nor->base + MTK_NOR_RADR0_REG + i * 4);
 		addr >>= 8;
 	}
 	/* Last register is non-contiguous */
-	writeb(addr & 0xff, mt8173_nor->base + MTK_NOR_RADR3_REG);
+	writeb(addr & 0xff, mtk_nor->base + MTK_NOR_RADR3_REG);
 }
 
-static ssize_t mt8173_nor_read(struct spi_nor *nor, loff_t from, size_t length,
-			       u_char *buffer)
+static ssize_t mtk_nor_read(struct spi_nor *nor, loff_t from, size_t length,
+			    u_char *buffer)
 {
 	int i, ret;
 	int addr = (int)from;
 	u8 *buf = (u8 *)buffer;
-	struct mt8173_nor *mt8173_nor = nor->priv;
+	struct mtk_nor *mtk_nor = nor->priv;
 
 	/* set mode for fast read mode ,dual mode or quad mode */
-	mt8173_nor_set_read_mode(mt8173_nor);
-	mt8173_nor_set_addr(mt8173_nor, addr);
+	mtk_nor_set_read_mode(mtk_nor);
+	mtk_nor_set_addr(mtk_nor, addr);
 
 	for (i = 0; i < length; i++) {
-		ret = mt8173_nor_execute_cmd(mt8173_nor, MTK_NOR_PIO_READ_CMD);
+		ret = mtk_nor_execute_cmd(mtk_nor, MTK_NOR_PIO_READ_CMD);
 		if (ret < 0)
 			return ret;
-		buf[i] = readb(mt8173_nor->base + MTK_NOR_RDATA_REG);
+		buf[i] = readb(mtk_nor->base + MTK_NOR_RDATA_REG);
 	}
 	return length;
 }
 
-static int mt8173_nor_write_single_byte(struct mt8173_nor *mt8173_nor,
-					int addr, int length, u8 *data)
+static int mtk_nor_write_single_byte(struct mtk_nor *mtk_nor,
+				     int addr, int length, u8 *data)
 {
 	int i, ret;
 
-	mt8173_nor_set_addr(mt8173_nor, addr);
+	mtk_nor_set_addr(mtk_nor, addr);
 
 	for (i = 0; i < length; i++) {
-		writeb(*data++, mt8173_nor->base + MTK_NOR_WDATA_REG);
-		ret = mt8173_nor_execute_cmd(mt8173_nor, MTK_NOR_PIO_WR_CMD);
+		writeb(*data++, mtk_nor->base + MTK_NOR_WDATA_REG);
+		ret = mtk_nor_execute_cmd(mtk_nor, MTK_NOR_PIO_WR_CMD);
 		if (ret < 0)
 			return ret;
 	}
 	return 0;
 }
 
-static int mt8173_nor_write_buffer(struct mt8173_nor *mt8173_nor, int addr,
-				   const u8 *buf)
+static int mtk_nor_write_buffer(struct mtk_nor *mtk_nor, int addr,
+				const u8 *buf)
 {
 	int i, bufidx, data;
 
-	mt8173_nor_set_addr(mt8173_nor, addr);
+	mtk_nor_set_addr(mtk_nor, addr);
 
 	bufidx = 0;
 	for (i = 0; i < SFLASH_WRBUF_SIZE; i += 4) {
 		data = buf[bufidx + 3]<<24 | buf[bufidx + 2]<<16 |
 		       buf[bufidx + 1]<<8 | buf[bufidx];
 		bufidx += 4;
-		writel(data, mt8173_nor->base + MTK_NOR_PP_DATA_REG);
+		writel(data, mtk_nor->base + MTK_NOR_PP_DATA_REG);
 	}
-	return mt8173_nor_execute_cmd(mt8173_nor, MTK_NOR_WR_CMD);
+	return mtk_nor_execute_cmd(mtk_nor, MTK_NOR_WR_CMD);
 }
 
-static ssize_t mt8173_nor_write(struct spi_nor *nor, loff_t to, size_t len,
-				const u_char *buf)
+static ssize_t mtk_nor_write(struct spi_nor *nor, loff_t to, size_t len,
+			     const u_char *buf)
 {
 	int ret;
-	struct mt8173_nor *mt8173_nor = nor->priv;
+	struct mtk_nor *mtk_nor = nor->priv;
 	size_t i;
 
-	ret = mt8173_nor_write_buffer_enable(mt8173_nor);
+	ret = mtk_nor_write_buffer_enable(mtk_nor);
 	if (ret < 0) {
-		dev_warn(mt8173_nor->dev, "write buffer enable failed!\n");
+		dev_warn(mtk_nor->dev, "write buffer enable failed!\n");
 		return ret;
 	}
 
 	for (i = 0; i + SFLASH_WRBUF_SIZE <= len; i += SFLASH_WRBUF_SIZE) {
-		ret = mt8173_nor_write_buffer(mt8173_nor, to, buf);
+		ret = mtk_nor_write_buffer(mtk_nor, to, buf);
 		if (ret < 0) {
-			dev_err(mt8173_nor->dev, "write buffer failed!\n");
+			dev_err(mtk_nor->dev, "write buffer failed!\n");
 			return ret;
 		}
 		to += SFLASH_WRBUF_SIZE;
 		buf += SFLASH_WRBUF_SIZE;
 	}
-	ret = mt8173_nor_write_buffer_disable(mt8173_nor);
+	ret = mtk_nor_write_buffer_disable(mtk_nor);
 	if (ret < 0) {
-		dev_warn(mt8173_nor->dev, "write buffer disable failed!\n");
+		dev_warn(mtk_nor->dev, "write buffer disable failed!\n");
 		return ret;
 	}
 
 	if (i < len) {
-		ret = mt8173_nor_write_single_byte(mt8173_nor, to,
-						   (int)(len - i), (u8 *)buf);
+		ret = mtk_nor_write_single_byte(mtk_nor, to,
+						(int)(len - i), (u8 *)buf);
 		if (ret < 0) {
-			dev_err(mt8173_nor->dev, "write single byte failed!\n");
+			dev_err(mtk_nor->dev, "write single byte failed!\n");
 			return ret;
 		}
 	}
@@ -362,72 +362,72 @@ static ssize_t mt8173_nor_write(struct spi_nor *nor, loff_t to, size_t len,
 	return len;
 }
 
-static int mt8173_nor_read_reg(struct spi_nor *nor, u8 opcode, u8 *buf, int len)
+static int mtk_nor_read_reg(struct spi_nor *nor, u8 opcode, u8 *buf, int len)
 {
 	int ret;
-	struct mt8173_nor *mt8173_nor = nor->priv;
+	struct mtk_nor *mtk_nor = nor->priv;
 
 	switch (opcode) {
 	case SPINOR_OP_RDSR:
-		ret = mt8173_nor_execute_cmd(mt8173_nor, MTK_NOR_RDSR_CMD);
+		ret = mtk_nor_execute_cmd(mtk_nor, MTK_NOR_RDSR_CMD);
 		if (ret < 0)
 			return ret;
 		if (len == 1)
-			*buf = readb(mt8173_nor->base + MTK_NOR_RDSR_REG);
+			*buf = readb(mtk_nor->base + MTK_NOR_RDSR_REG);
 		else
-			dev_err(mt8173_nor->dev, "len should be 1 for read status!\n");
+			dev_err(mtk_nor->dev, "len should be 1 for read status!\n");
 		break;
 	default:
-		ret = mt8173_nor_do_tx_rx(mt8173_nor, opcode, NULL, 0, buf, len);
+		ret = mtk_nor_do_tx_rx(mtk_nor, opcode, NULL, 0, buf, len);
 		break;
 	}
 	return ret;
 }
 
-static int mt8173_nor_write_reg(struct spi_nor *nor, u8 opcode, u8 *buf,
-				int len)
+static int mtk_nor_write_reg(struct spi_nor *nor, u8 opcode, u8 *buf,
+			     int len)
 {
 	int ret;
-	struct mt8173_nor *mt8173_nor = nor->priv;
+	struct mtk_nor *mtk_nor = nor->priv;
 
 	switch (opcode) {
 	case SPINOR_OP_WRSR:
 		/* We only handle 1 byte */
-		ret = mt8173_nor_wr_sr(mt8173_nor, *buf);
+		ret = mtk_nor_wr_sr(mtk_nor, *buf);
 		break;
 	default:
-		ret = mt8173_nor_do_tx_rx(mt8173_nor, opcode, buf, len, NULL, 0);
+		ret = mtk_nor_do_tx_rx(mtk_nor, opcode, buf, len, NULL, 0);
 		if (ret)
-			dev_warn(mt8173_nor->dev, "write reg failure!\n");
+			dev_warn(mtk_nor->dev, "write reg failure!\n");
 		break;
 	}
 	return ret;
 }
 
-static void mt8173_nor_disable_clk(struct mt8173_nor *mt8173_nor)
+static void mtk_nor_disable_clk(struct mtk_nor *mtk_nor)
 {
-	clk_disable_unprepare(mt8173_nor->spi_clk);
-	clk_disable_unprepare(mt8173_nor->nor_clk);
+	clk_disable_unprepare(mtk_nor->spi_clk);
+	clk_disable_unprepare(mtk_nor->nor_clk);
 }
 
-static int mt8173_nor_enable_clk(struct mt8173_nor *mt8173_nor)
+static int mtk_nor_enable_clk(struct mtk_nor *mtk_nor)
 {
 	int ret;
 
-	ret = clk_prepare_enable(mt8173_nor->spi_clk);
+	ret = clk_prepare_enable(mtk_nor->spi_clk);
 	if (ret)
 		return ret;
 
-	ret = clk_prepare_enable(mt8173_nor->nor_clk);
+	ret = clk_prepare_enable(mtk_nor->nor_clk);
 	if (ret) {
-		clk_disable_unprepare(mt8173_nor->spi_clk);
+		clk_disable_unprepare(mtk_nor->spi_clk);
 		return ret;
 	}
 
 	return 0;
 }
 
-static int mtk_nor_init(struct mt8173_nor *mt8173_nor,
+static int mtk_nor_init(struct mtk_nor *mtk_nor,
 			struct device_node *flash_node)
 {
 	const struct spi_nor_hwcaps hwcaps = {
@@ -439,18 +439,18 @@ static int mtk_nor_init(struct mt8173_nor *mt8173_nor,
 	struct spi_nor *nor;
 
 	/* initialize controller to accept commands */
-	writel(MTK_NOR_ENABLE_SF_CMD, mt8173_nor->base + MTK_NOR_WRPROT_REG);
+	writel(MTK_NOR_ENABLE_SF_CMD, mtk_nor->base + MTK_NOR_WRPROT_REG);
 
-	nor = &mt8173_nor->nor;
-	nor->dev = mt8173_nor->dev;
-	nor->priv = mt8173_nor;
+	nor = &mtk_nor->nor;
+	nor->dev = mtk_nor->dev;
+	nor->priv = mtk_nor;
 	spi_nor_set_flash_node(nor, flash_node);
 
 	/* fill the hooks to spi nor */
-	nor->read = mt8173_nor_read;
-	nor->read_reg = mt8173_nor_read_reg;
-	nor->write = mt8173_nor_write;
-	nor->write_reg = mt8173_nor_write_reg;
+	nor->read = mtk_nor_read;
+	nor->read_reg = mtk_nor_read_reg;
+	nor->write = mtk_nor_write;
+	nor->write_reg = mtk_nor_write_reg;
 	nor->mtd.name = "mtk_nor";
 	/* initialized with NULL */
 	ret = spi_nor_scan(nor, NULL, &hwcaps);
@@ -465,34 +465,34 @@ static int mtk_nor_drv_probe(struct platform_device *pdev)
 	struct device_node *flash_np;
 	struct resource *res;
 	int ret;
-	struct mt8173_nor *mt8173_nor;
+	struct mtk_nor *mtk_nor;
 
 	if (!pdev->dev.of_node) {
 		dev_err(&pdev->dev, "No DT found\n");
 		return -EINVAL;
 	}
 
-	mt8173_nor = devm_kzalloc(&pdev->dev, sizeof(*mt8173_nor), GFP_KERNEL);
-	if (!mt8173_nor)
+	mtk_nor = devm_kzalloc(&pdev->dev, sizeof(*mtk_nor), GFP_KERNEL);
+	if (!mtk_nor)
 		return -ENOMEM;
-	platform_set_drvdata(pdev, mt8173_nor);
+	platform_set_drvdata(pdev, mtk_nor);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	mt8173_nor->base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(mt8173_nor->base))
-		return PTR_ERR(mt8173_nor->base);
+	mtk_nor->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mtk_nor->base))
+		return PTR_ERR(mtk_nor->base);
 
-	mt8173_nor->spi_clk = devm_clk_get(&pdev->dev, "spi");
-	if (IS_ERR(mt8173_nor->spi_clk))
-		return PTR_ERR(mt8173_nor->spi_clk);
+	mtk_nor->spi_clk = devm_clk_get(&pdev->dev, "spi");
+	if (IS_ERR(mtk_nor->spi_clk))
+		return PTR_ERR(mtk_nor->spi_clk);
 
-	mt8173_nor->nor_clk = devm_clk_get(&pdev->dev, "sf");
-	if (IS_ERR(mt8173_nor->nor_clk))
-		return PTR_ERR(mt8173_nor->nor_clk);
+	mtk_nor->nor_clk = devm_clk_get(&pdev->dev, "sf");
+	if (IS_ERR(mtk_nor->nor_clk))
+		return PTR_ERR(mtk_nor->nor_clk);
 
-	mt8173_nor->dev = &pdev->dev;
+	mtk_nor->dev = &pdev->dev;
 
-	ret = mt8173_nor_enable_clk(mt8173_nor);
+	ret = mtk_nor_enable_clk(mtk_nor);
 	if (ret)
 		return ret;
 
@@ -503,20 +503,20 @@ static int mtk_nor_drv_probe(struct platform_device *pdev)
 		ret = -ENODEV;
 		goto nor_free;
 	}
-	ret = mtk_nor_init(mt8173_nor, flash_np);
+	ret = mtk_nor_init(mtk_nor, flash_np);
 
 nor_free:
 	if (ret)
-		mt8173_nor_disable_clk(mt8173_nor);
+		mtk_nor_disable_clk(mtk_nor);
 
 	return ret;
 }
 
 static int mtk_nor_drv_remove(struct platform_device *pdev)
 {
-	struct mt8173_nor *mt8173_nor = platform_get_drvdata(pdev);
+	struct mtk_nor *mtk_nor = platform_get_drvdata(pdev);
 
-	mt8173_nor_disable_clk(mt8173_nor);
+	mtk_nor_disable_clk(mtk_nor);
 
 	return 0;
 }
@@ -524,18 +524,18 @@ static int mtk_nor_drv_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int mtk_nor_suspend(struct device *dev)
 {
-	struct mt8173_nor *mt8173_nor = dev_get_drvdata(dev);
+	struct mtk_nor *mtk_nor = dev_get_drvdata(dev);
 
-	mt8173_nor_disable_clk(mt8173_nor);
+	mtk_nor_disable_clk(mtk_nor);
 
 	return 0;
 }
 
 static int mtk_nor_resume(struct device *dev)
 {
-	struct mt8173_nor *mt8173_nor = dev_get_drvdata(dev);
+	struct mtk_nor *mtk_nor = dev_get_drvdata(dev);
 
-	return mt8173_nor_enable_clk(mt8173_nor);
+	return mtk_nor_enable_clk(mtk_nor);
 }
 
 static const struct dev_pm_ops mtk_nor_dev_pm_ops = {
diff --git a/drivers/mtd/spi-nor/spi-nor.c b/drivers/mtd/spi-nor/spi-nor.c
index bc266f70a15b..d445a4d3b770 100644
--- a/drivers/mtd/spi-nor/spi-nor.c
+++ b/drivers/mtd/spi-nor/spi-nor.c
@@ -330,8 +330,22 @@ static inline int spi_nor_fsr_ready(struct spi_nor *nor)
 	int fsr = read_fsr(nor);
 	if (fsr < 0)
 		return fsr;
-	else
-		return fsr & FSR_READY;
+
+	if (fsr & (FSR_E_ERR | FSR_P_ERR)) {
+		if (fsr & FSR_E_ERR)
+			dev_err(nor->dev, "Erase operation failed.\n");
+		else
+			dev_err(nor->dev, "Program operation failed.\n");
+
+		if (fsr & FSR_PT_ERR)
+			dev_err(nor->dev,
+			"Attempted to modify a protected sector.\n");
+
+		nor->write_reg(nor, SPINOR_OP_CLFSR, NULL, 0);
+		return -EIO;
+	}
+
+	return fsr & FSR_READY;
 }
 
 static int spi_nor_ready(struct spi_nor *nor)
@@ -552,6 +566,27 @@ static int spi_nor_erase(struct mtd_info *mtd, struct erase_info *instr)
 	return ret;
 }
 
+/* Write status register and ensure bits in mask match written values */
+static int write_sr_and_check(struct spi_nor *nor, u8 status_new, u8 mask)
+{
+	int ret;
+
+	write_enable(nor);
+	ret = write_sr(nor, status_new);
+	if (ret)
+		return ret;
+
+	ret = spi_nor_wait_till_ready(nor);
+	if (ret)
+		return ret;
+
+	ret = read_sr(nor);
+	if (ret < 0)
+		return ret;
+
+	return ((ret & mask) != (status_new & mask)) ? -EIO : 0;
+}
+
 static void stm_get_locked_range(struct spi_nor *nor, u8 sr, loff_t *ofs,
 				 uint64_t *len)
 {
@@ -650,7 +685,6 @@ static int stm_lock(struct spi_nor *nor, loff_t ofs, uint64_t len)
 	loff_t lock_len;
 	bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB;
 	bool use_top;
-	int ret;
 
 	status_old = read_sr(nor);
 	if (status_old < 0)
@@ -714,11 +748,7 @@ static int stm_lock(struct spi_nor *nor, loff_t ofs, uint64_t len)
 	if ((status_new & mask) < (status_old & mask))
 		return -EINVAL;
 
-	write_enable(nor);
-	ret = write_sr(nor, status_new);
-	if (ret)
-		return ret;
-	return spi_nor_wait_till_ready(nor);
+	return write_sr_and_check(nor, status_new, mask);
 }
 
 /*
@@ -735,7 +765,6 @@ static int stm_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
 	loff_t lock_len;
 	bool can_be_top = true, can_be_bottom = nor->flags & SNOR_F_HAS_SR_TB;
 	bool use_top;
-	int ret;
 
 	status_old = read_sr(nor);
 	if (status_old < 0)
@@ -802,11 +831,7 @@ static int stm_unlock(struct spi_nor *nor, loff_t ofs, uint64_t len)
 	if ((status_new & mask) > (status_old & mask))
 		return -EINVAL;
 
-	write_enable(nor);
-	ret = write_sr(nor, status_new);
-	if (ret)
-		return ret;
-	return spi_nor_wait_till_ready(nor);
+	return write_sr_and_check(nor, status_new, mask);
 }
 
 /*
@@ -1020,7 +1045,13 @@ static const struct flash_info spi_nor_ids[] = {
 	{ "640s33b",  INFO(0x898913, 0, 64 * 1024, 128, 0) },
 
 	/* ISSI */
-	{ "is25cd512", INFO(0x7f9d20, 0, 32 * 1024,   2, SECT_4K) },
+	{ "is25cd512",  INFO(0x7f9d20, 0, 32 * 1024,   2, SECT_4K) },
+	{ "is25lq040b", INFO(0x9d4013, 0, 64 * 1024,   8,
+			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
+	{ "is25lp080d", INFO(0x9d6014, 0, 64 * 1024,  16,
+			SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
+	{ "is25lp128",  INFO(0x9d6018, 0, 64 * 1024, 256,
+			SECT_4K | SPI_NOR_DUAL_READ) },
 
 	/* Macronix */
 	{ "mx25l512e",   INFO(0xc22010, 0, 64 * 1024,   1, SECT_4K) },
@@ -1065,7 +1096,7 @@ static const struct flash_info spi_nor_ids[] = {
 	{ "pm25lv010",   INFO(0,        0, 32 * 1024,    4, SECT_4K_PMC) },
 	{ "pm25lq032",   INFO(0x7f9d46, 0, 64 * 1024,   64, SECT_4K) },
 
-	/* Spansion -- single (large) sector size only, at least
+	/* Spansion/Cypress -- single (large) sector size only, at least
 	 * for the chips listed here (without boot sectors).
 	 */
 	{ "s25sl032p",  INFO(0x010215, 0x4d00,  64 * 1024,  64, SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) },
@@ -1094,6 +1125,8 @@ static const struct flash_info spi_nor_ids[] = {
 	{ "s25fl204k",  INFO(0x014013,      0,  64 * 1024,   8, SECT_4K | SPI_NOR_DUAL_READ) },
 	{ "s25fl208k",  INFO(0x014014,      0,  64 * 1024,  16, SECT_4K | SPI_NOR_DUAL_READ) },
 	{ "s25fl064l",  INFO(0x016017,      0,  64 * 1024, 128, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) },
+	{ "s25fl128l",  INFO(0x016018,      0,  64 * 1024, 256, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) },
+	{ "s25fl256l",  INFO(0x016019,      0,  64 * 1024, 512, SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ | SPI_NOR_4B_OPCODES) },
 
 	/* SST -- large erase sizes are "overlays", "sectors" are 4K */
 	{ "sst25vf040b", INFO(0xbf258d, 0, 64 * 1024,  8, SECT_4K | SST_WRITE) },
@@ -2713,6 +2746,16 @@ static void spi_nor_resume(struct mtd_info *mtd)
 		dev_err(dev, "resume() failed\n");
 }
 
+void spi_nor_restore(struct spi_nor *nor)
+{
+	/* restore the addressing mode */
+	if ((nor->addr_width == 4) &&
+	    (JEDEC_MFR(nor->info) != SNOR_MFR_SPANSION) &&
+	    !(nor->info->flags & SPI_NOR_4B_OPCODES))
+		set_4byte(nor, nor->info, 0);
+}
+EXPORT_SYMBOL_GPL(spi_nor_restore);
+
 int spi_nor_scan(struct spi_nor *nor, const char *name,
 		 const struct spi_nor_hwcaps *hwcaps)
 {
diff --git a/drivers/mtd/tests/nandbiterrs.c b/drivers/mtd/tests/nandbiterrs.c
index 5f03b8c885a9..cde19c99e77b 100644
--- a/drivers/mtd/tests/nandbiterrs.c
+++ b/drivers/mtd/tests/nandbiterrs.c
@@ -151,7 +151,7 @@ static int read_page(int log)
 	memcpy(&oldstats, &mtd->ecc_stats, sizeof(oldstats));
 
 	err = mtd_read(mtd, offset, mtd->writesize, &read, rbuffer);
-	if (err == -EUCLEAN)
+	if (!err || err == -EUCLEAN)
 		err = mtd->ecc_stats.corrected - oldstats.corrected;
 
 	if (err < 0 || read != mtd->writesize) {
diff --git a/drivers/mtd/tests/oobtest.c b/drivers/mtd/tests/oobtest.c
index 1cb3f7758fb6..766b2c385682 100644
--- a/drivers/mtd/tests/oobtest.c
+++ b/drivers/mtd/tests/oobtest.c
@@ -193,6 +193,9 @@ static int verify_eraseblock(int ebnum)
 		ops.datbuf    = NULL;
 		ops.oobbuf    = readbuf;
 		err = mtd_read_oob(mtd, addr, &ops);
+		if (mtd_is_bitflip(err))
+			err = 0;
+
 		if (err || ops.oobretlen != use_len) {
 			pr_err("error: readoob failed at %#llx\n",
 			       (long long)addr);
@@ -227,6 +230,9 @@ static int verify_eraseblock(int ebnum)
 			ops.datbuf    = NULL;
 			ops.oobbuf    = readbuf;
 			err = mtd_read_oob(mtd, addr, &ops);
+			if (mtd_is_bitflip(err))
+				err = 0;
+
 			if (err || ops.oobretlen != mtd->oobavail) {
 				pr_err("error: readoob failed at %#llx\n",
 						(long long)addr);
@@ -286,6 +292,9 @@ static int verify_eraseblock_in_one_go(int ebnum)
 
 	/* read entire block's OOB at one go */
 	err = mtd_read_oob(mtd, addr, &ops);
+	if (mtd_is_bitflip(err))
+		err = 0;
+
 	if (err || ops.oobretlen != len) {
 		pr_err("error: readoob failed at %#llx\n",
 		       (long long)addr);
@@ -527,6 +536,9 @@ static int __init mtd_oobtest_init(void)
 	pr_info("attempting to start read past end of OOB\n");
 	pr_info("an error is expected...\n");
 	err = mtd_read_oob(mtd, addr0, &ops);
+	if (mtd_is_bitflip(err))
+		err = 0;
+
 	if (err) {
 		pr_info("error occurred as expected\n");
 		err = 0;
@@ -571,6 +583,9 @@ static int __init mtd_oobtest_init(void)
 		pr_info("attempting to read past end of device\n");
 		pr_info("an error is expected...\n");
 		err = mtd_read_oob(mtd, mtd->size - mtd->writesize, &ops);
+		if (mtd_is_bitflip(err))
+			err = 0;
+
 		if (err) {
 			pr_info("error occurred as expected\n");
 			err = 0;
@@ -615,6 +630,9 @@ static int __init mtd_oobtest_init(void)
 		pr_info("attempting to read past end of device\n");
 		pr_info("an error is expected...\n");
 		err = mtd_read_oob(mtd, mtd->size - mtd->writesize, &ops);
+		if (mtd_is_bitflip(err))
+			err = 0;
+
 		if (err) {
 			pr_info("error occurred as expected\n");
 			err = 0;
@@ -684,6 +702,9 @@ static int __init mtd_oobtest_init(void)
 		ops.datbuf    = NULL;
 		ops.oobbuf    = readbuf;
 		err = mtd_read_oob(mtd, addr, &ops);
+		if (mtd_is_bitflip(err))
+			err = 0;
+
 		if (err)
 			goto out;
 		if (memcmpshow(addr, readbuf, writebuf,
diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c
index b210fdb31c98..b1fc28f63882 100644
--- a/drivers/mtd/ubi/block.c
+++ b/drivers/mtd/ubi/block.c
@@ -99,6 +99,8 @@ struct ubiblock {
 
 /* Linked list of all ubiblock instances */
 static LIST_HEAD(ubiblock_devices);
+static DEFINE_IDR(ubiblock_minor_idr);
+/* Protects ubiblock_devices and ubiblock_minor_idr */
 static DEFINE_MUTEX(devices_mutex);
 static int ubiblock_major;
 
@@ -351,8 +353,6 @@ static const struct blk_mq_ops ubiblock_mq_ops = {
 	.init_request	= ubiblock_init_request,
 };
 
-static DEFINE_IDR(ubiblock_minor_idr);
-
 int ubiblock_create(struct ubi_volume_info *vi)
 {
 	struct ubiblock *dev;
@@ -365,14 +365,15 @@ int ubiblock_create(struct ubi_volume_info *vi)
 	/* Check that the volume isn't already handled */
 	mutex_lock(&devices_mutex);
 	if (find_dev_nolock(vi->ubi_num, vi->vol_id)) {
-		mutex_unlock(&devices_mutex);
-		return -EEXIST;
+		ret = -EEXIST;
+		goto out_unlock;
 	}
-	mutex_unlock(&devices_mutex);
 
 	dev = kzalloc(sizeof(struct ubiblock), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
+	if (!dev) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
 
 	mutex_init(&dev->dev_mutex);
 
@@ -437,14 +438,13 @@ int ubiblock_create(struct ubi_volume_info *vi)
 		goto out_free_queue;
 	}
 
-	mutex_lock(&devices_mutex);
 	list_add_tail(&dev->list, &ubiblock_devices);
-	mutex_unlock(&devices_mutex);
 
 	/* Must be the last step: anyone can call file ops from now on */
 	add_disk(dev->gd);
 	dev_info(disk_to_dev(dev->gd), "created from ubi%d:%d(%s)",
 		 dev->ubi_num, dev->vol_id, vi->name);
+	mutex_unlock(&devices_mutex);
 	return 0;
 
 out_free_queue:
@@ -457,6 +457,8 @@ int ubiblock_create(struct ubi_volume_info *vi)
 	put_disk(dev->gd);
 out_free_dev:
 	kfree(dev);
+out_unlock:
+	mutex_unlock(&devices_mutex);
 
 	return ret;
 }
@@ -478,30 +480,36 @@ static void ubiblock_cleanup(struct ubiblock *dev)
 int ubiblock_remove(struct ubi_volume_info *vi)
 {
 	struct ubiblock *dev;
+	int ret;
 
 	mutex_lock(&devices_mutex);
 	dev = find_dev_nolock(vi->ubi_num, vi->vol_id);
 	if (!dev) {
-		mutex_unlock(&devices_mutex);
-		return -ENODEV;
+		ret = -ENODEV;
+		goto out_unlock;
 	}
 
 	/* Found a device, let's lock it so we can check if it's busy */
 	mutex_lock(&dev->dev_mutex);
 	if (dev->refcnt > 0) {
-		mutex_unlock(&dev->dev_mutex);
-		mutex_unlock(&devices_mutex);
-		return -EBUSY;
+		ret = -EBUSY;
+		goto out_unlock_dev;
 	}
 
 	/* Remove from device list */
 	list_del(&dev->list);
-	mutex_unlock(&devices_mutex);
-
 	ubiblock_cleanup(dev);
 	mutex_unlock(&dev->dev_mutex);
+	mutex_unlock(&devices_mutex);
+
 	kfree(dev);
 	return 0;
+
+out_unlock_dev:
+	mutex_unlock(&dev->dev_mutex);
+out_unlock:
+	mutex_unlock(&devices_mutex);
+	return ret;
 }
 
 static int ubiblock_resize(struct ubi_volume_info *vi)
@@ -630,6 +638,7 @@ static void ubiblock_remove_all(void)
 	struct ubiblock *next;
 	struct ubiblock *dev;
 
+	mutex_lock(&devices_mutex);
 	list_for_each_entry_safe(dev, next, &ubiblock_devices, list) {
 		/* The module is being forcefully removed */
 		WARN_ON(dev->desc);
@@ -638,6 +647,7 @@ static void ubiblock_remove_all(void)
 		ubiblock_cleanup(dev);
 		kfree(dev);
 	}
+	mutex_unlock(&devices_mutex);
 }
 
 int __init ubiblock_init(void)
diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c
index 136ce05d2328..e941395de3ae 100644
--- a/drivers/mtd/ubi/build.c
+++ b/drivers/mtd/ubi/build.c
@@ -535,8 +535,17 @@ static int get_bad_peb_limit(const struct ubi_device *ubi, int max_beb_per1024)
 	int limit, device_pebs;
 	uint64_t device_size;
 
-	if (!max_beb_per1024)
-		return 0;
+	if (!max_beb_per1024) {
+		/*
+		 * Since max_beb_per1024 has not been set by the user in either
+		 * the cmdline or Kconfig, use mtd_max_bad_blocks to set the
+		 * limit if it is supported by the device.
+		 */
+		limit = mtd_max_bad_blocks(ubi->mtd, 0, ubi->mtd->size);
+		if (limit < 0)
+			return 0;
+		return limit;
+	}
 
 	/*
 	 * Here we are using size of the entire flash chip and
diff --git a/drivers/mtd/ubi/eba.c b/drivers/mtd/ubi/eba.c
index 388e46be6ad9..250e30fac61b 100644
--- a/drivers/mtd/ubi/eba.c
+++ b/drivers/mtd/ubi/eba.c
@@ -384,7 +384,7 @@ static int leb_write_lock(struct ubi_device *ubi, int vol_id, int lnum)
 }
 
 /**
- * leb_write_lock - lock logical eraseblock for writing.
+ * leb_write_trylock - try to lock logical eraseblock for writing.
  * @ubi: UBI device description object
  * @vol_id: volume ID
  * @lnum: logical eraseblock number
diff --git a/drivers/mtd/ubi/fastmap-wl.c b/drivers/mtd/ubi/fastmap-wl.c
index 4f0bd6b4422a..590d967011bb 100644
--- a/drivers/mtd/ubi/fastmap-wl.c
+++ b/drivers/mtd/ubi/fastmap-wl.c
@@ -66,7 +66,7 @@ static void return_unused_pool_pebs(struct ubi_device *ubi,
 	}
 }
 
-static int anchor_pebs_avalible(struct rb_root *root)
+static int anchor_pebs_available(struct rb_root *root)
 {
 	struct rb_node *p;
 	struct ubi_wl_entry *e;
diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c
index 5a832bc79b1b..91705962ba73 100644
--- a/drivers/mtd/ubi/fastmap.c
+++ b/drivers/mtd/ubi/fastmap.c
@@ -214,9 +214,8 @@ static void assign_aeb_to_av(struct ubi_attach_info *ai,
 			     struct ubi_ainf_volume *av)
 {
 	struct ubi_ainf_peb *tmp_aeb;
-	struct rb_node **p = &ai->volumes.rb_node, *parent = NULL;
+	struct rb_node **p = &av->root.rb_node, *parent = NULL;
 
-	p = &av->root.rb_node;
 	while (*p) {
 		parent = *p;
 
@@ -1063,7 +1062,7 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai,
 		e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
 		if (!e) {
 			while (i--)
-				kfree(fm->e[i]);
+				kmem_cache_free(ubi_wl_entry_slab, fm->e[i]);
 
 			ret = -ENOMEM;
 			goto free_hdr;
diff --git a/drivers/mtd/ubi/vmt.c b/drivers/mtd/ubi/vmt.c
index 85237cf661f9..3fd8d7ff7a02 100644
--- a/drivers/mtd/ubi/vmt.c
+++ b/drivers/mtd/ubi/vmt.c
@@ -270,6 +270,12 @@ int ubi_create_volume(struct ubi_device *ubi, struct ubi_mkvol_req *req)
 			vol->last_eb_bytes = vol->usable_leb_size;
 	}
 
+	/* Make volume "available" before it becomes accessible via sysfs */
+	spin_lock(&ubi->volumes_lock);
+	ubi->volumes[vol_id] = vol;
+	ubi->vol_count += 1;
+	spin_unlock(&ubi->volumes_lock);
+
 	/* Register character device for the volume */
 	cdev_init(&vol->cdev, &ubi_vol_cdev_operations);
 	vol->cdev.owner = THIS_MODULE;
@@ -298,11 +304,6 @@ int ubi_create_volume(struct ubi_device *ubi, struct ubi_mkvol_req *req)
 	if (err)
 		goto out_sysfs;
 
-	spin_lock(&ubi->volumes_lock);
-	ubi->volumes[vol_id] = vol;
-	ubi->vol_count += 1;
-	spin_unlock(&ubi->volumes_lock);
-
 	ubi_volume_notify(ubi, vol, UBI_VOLUME_ADDED);
 	self_check_volumes(ubi);
 	return err;
@@ -315,6 +316,10 @@ int ubi_create_volume(struct ubi_device *ubi, struct ubi_mkvol_req *req)
 	 */
 	cdev_device_del(&vol->cdev, &vol->dev);
 out_mapping:
+	spin_lock(&ubi->volumes_lock);
+	ubi->volumes[vol_id] = NULL;
+	ubi->vol_count -= 1;
+	spin_unlock(&ubi->volumes_lock);
 	ubi_eba_destroy_table(eba_tbl);
 out_acc:
 	spin_lock(&ubi->volumes_lock);
diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c
index b5b8cd6f481c..2052a647220e 100644
--- a/drivers/mtd/ubi/wl.c
+++ b/drivers/mtd/ubi/wl.c
@@ -692,7 +692,7 @@ static int wear_leveling_worker(struct ubi_device *ubi, struct ubi_work *wrk,
 #ifdef CONFIG_MTD_UBI_FASTMAP
 	/* Check whether we need to produce an anchor PEB */
 	if (!anchor)
-		anchor = !anchor_pebs_avalible(&ubi->free);
+		anchor = !anchor_pebs_available(&ubi->free);
 
 	if (anchor) {
 		e1 = find_anchor_wl_entry(&ubi->used);
@@ -1528,6 +1528,46 @@ static void shutdown_work(struct ubi_device *ubi)
 	}
 }
 
+/**
+ * erase_aeb - erase a PEB given in UBI attach info PEB
+ * @ubi: UBI device description object
+ * @aeb: UBI attach info PEB
+ * @sync: If true, erase synchronously. Otherwise schedule for erasure
+ */
+static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync)
+{
+	struct ubi_wl_entry *e;
+	int err;
+
+	e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
+	if (!e)
+		return -ENOMEM;
+
+	e->pnum = aeb->pnum;
+	e->ec = aeb->ec;
+	ubi->lookuptbl[e->pnum] = e;
+
+	if (sync) {
+		err = sync_erase(ubi, e, false);
+		if (err)
+			goto out_free;
+
+		wl_tree_add(e, &ubi->free);
+		ubi->free_count++;
+	} else {
+		err = schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0, false);
+		if (err)
+			goto out_free;
+	}
+
+	return 0;
+
+out_free:
+	wl_entry_destroy(ubi, e);
+
+	return err;
+}
+
 /**
  * ubi_wl_init - initialize the WL sub-system using attaching information.
  * @ubi: UBI device description object
@@ -1566,17 +1606,9 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 	list_for_each_entry_safe(aeb, tmp, &ai->erase, u.list) {
 		cond_resched();
 
-		e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
-		if (!e)
-			goto out_free;
-
-		e->pnum = aeb->pnum;
-		e->ec = aeb->ec;
-		ubi->lookuptbl[e->pnum] = e;
-		if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0, false)) {
-			wl_entry_destroy(ubi, e);
+		err = erase_aeb(ubi, aeb, false);
+		if (err)
 			goto out_free;
-		}
 
 		found_pebs++;
 	}
@@ -1585,8 +1617,10 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 		cond_resched();
 
 		e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
-		if (!e)
+		if (!e) {
+			err = -ENOMEM;
 			goto out_free;
+		}
 
 		e->pnum = aeb->pnum;
 		e->ec = aeb->ec;
@@ -1605,8 +1639,10 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 			cond_resched();
 
 			e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
-			if (!e)
+			if (!e) {
+				err = -ENOMEM;
 				goto out_free;
+			}
 
 			e->pnum = aeb->pnum;
 			e->ec = aeb->ec;
@@ -1635,6 +1671,8 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 			ubi_assert(!ubi->lookuptbl[e->pnum]);
 			ubi->lookuptbl[e->pnum] = e;
 		} else {
+			bool sync = false;
+
 			/*
 			 * Usually old Fastmap PEBs are scheduled for erasure
 			 * and we don't have to care about them but if we face
@@ -1644,18 +1682,21 @@ int ubi_wl_init(struct ubi_device *ubi, struct ubi_attach_info *ai)
 			if (ubi->lookuptbl[aeb->pnum])
 				continue;
 
-			e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL);
-			if (!e)
-				goto out_free;
+			/*
+			 * The fastmap update code might not find a free PEB for
+			 * writing the fastmap anchor to and then reuses the
+			 * current fastmap anchor PEB. When this PEB gets erased
+			 * and a power cut happens before it is written again we
+			 * must make sure that the fastmap attach code doesn't
+			 * find any outdated fastmap anchors, hence we erase the
+			 * outdated fastmap anchor PEBs synchronously here.
+			 */
+			if (aeb->vol_id == UBI_FM_SB_VOLUME_ID)
+				sync = true;
 
-			e->pnum = aeb->pnum;
-			e->ec = aeb->ec;
-			ubi_assert(!ubi->lookuptbl[e->pnum]);
-			ubi->lookuptbl[e->pnum] = e;
-			if (schedule_erase(ubi, e, aeb->vol_id, aeb->lnum, 0, false)) {
-				wl_entry_destroy(ubi, e);
+			err = erase_aeb(ubi, aeb, sync);
+			if (err)
 				goto out_free;
-			}
 		}
 
 		found_pebs++;
diff --git a/drivers/mtd/ubi/wl.h b/drivers/mtd/ubi/wl.h
index 2aaa3f7f2ba9..a9e2d669acd8 100644
--- a/drivers/mtd/ubi/wl.h
+++ b/drivers/mtd/ubi/wl.h
@@ -2,7 +2,7 @@
 #ifndef UBI_WL_H
 #define UBI_WL_H
 #ifdef CONFIG_MTD_UBI_FASTMAP
-static int anchor_pebs_avalible(struct rb_root *root);
+static int anchor_pebs_available(struct rb_root *root);
 static void update_fastmap_work_fn(struct work_struct *wrk);
 static struct ubi_wl_entry *find_anchor_wl_entry(struct rb_root *root);
 static struct ubi_wl_entry *get_peb_for_wl(struct ubi_device *ubi);
diff --git a/drivers/mux/Kconfig b/drivers/mux/Kconfig
index 19e4e904c9bf..6241678e99af 100644
--- a/drivers/mux/Kconfig
+++ b/drivers/mux/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Multiplexer devices
 #
diff --git a/drivers/mux/Makefile b/drivers/mux/Makefile
index 0e1e59760e3f..c3d883955fd5 100644
--- a/drivers/mux/Makefile
+++ b/drivers/mux/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Makefile for multiplexer devices.
 #
diff --git a/drivers/mux/adg792a.c b/drivers/mux/adg792a.c
index 12aa221ab90d..6a8725cf3d71 100644
--- a/drivers/mux/adg792a.c
+++ b/drivers/mux/adg792a.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Multiplexer driver for Analog Devices ADG792A/G Triple 4:1 mux
  *
  * Copyright (C) 2017 Axentia Technologies AB
  *
  * Author: Peter Rosin <peda@axentia.se>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/err.h>
diff --git a/drivers/mux/core.c b/drivers/mux/core.c
index 6e5cf9d9cd99..d1271c1ee23c 100644
--- a/drivers/mux/core.c
+++ b/drivers/mux/core.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Multiplexer subsystem
  *
  * Copyright (C) 2017 Axentia Technologies AB
  *
  * Author: Peter Rosin <peda@axentia.se>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #define pr_fmt(fmt) "mux-core: " fmt
diff --git a/drivers/mux/gpio.c b/drivers/mux/gpio.c
index 468bf1709606..6fdd9316db8b 100644
--- a/drivers/mux/gpio.c
+++ b/drivers/mux/gpio.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * GPIO-controlled multiplexer driver
  *
  * Copyright (C) 2017 Axentia Technologies AB
  *
  * Author: Peter Rosin <peda@axentia.se>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/err.h>
diff --git a/drivers/mux/mmio.c b/drivers/mux/mmio.c
index 37c1de359a70..935ac44aa209 100644
--- a/drivers/mux/mmio.c
+++ b/drivers/mux/mmio.c
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * MMIO register bitfield-controlled multiplexer driver
  *
  * Copyright (C) 2017 Pengutronix, Philipp Zabel <kernel@pengutronix.de>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/bitops.h>
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 0936da592e12..944ec3c9282c 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -497,4 +497,15 @@ config THUNDERBOLT_NET
 
 source "drivers/net/hyperv/Kconfig"
 
+config NETDEVSIM
+	tristate "Simulated networking device"
+	depends on DEBUG_FS
+	help
+	  This driver is a developer testing tool and software model that can
+	  be used to test various control path networking APIs, especially
+	  HW-offload related.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called netdevsim.
+
 endif # NETDEVICES
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 766f62d02a0b..04c3b747812c 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -78,3 +78,4 @@ obj-$(CONFIG_FUJITSU_ES) += fjes/
 
 thunderbolt-net-y += thunderbolt.o
 obj-$(CONFIG_THUNDERBOLT_NET) += thunderbolt-net.o
+obj-$(CONFIG_NETDEVSIM) += netdevsim/
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index c669554d70bb..b7b113018853 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -1528,39 +1528,6 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev,
 			goto err_close;
 	}
 
-	/* If the mode uses primary, then the following is handled by
-	 * bond_change_active_slave().
-	 */
-	if (!bond_uses_primary(bond)) {
-		/* set promiscuity level to new slave */
-		if (bond_dev->flags & IFF_PROMISC) {
-			res = dev_set_promiscuity(slave_dev, 1);
-			if (res)
-				goto err_close;
-		}
-
-		/* set allmulti level to new slave */
-		if (bond_dev->flags & IFF_ALLMULTI) {
-			res = dev_set_allmulti(slave_dev, 1);
-			if (res)
-				goto err_close;
-		}
-
-		netif_addr_lock_bh(bond_dev);
-
-		dev_mc_sync_multiple(slave_dev, bond_dev);
-		dev_uc_sync_multiple(slave_dev, bond_dev);
-
-		netif_addr_unlock_bh(bond_dev);
-	}
-
-	if (BOND_MODE(bond) == BOND_MODE_8023AD) {
-		/* add lacpdu mc addr to mc list */
-		u8 lacpdu_multicast[ETH_ALEN] = MULTICAST_LACPDU_ADDR;
-
-		dev_mc_add(slave_dev, lacpdu_multicast);
-	}
-
 	res = vlan_vids_add_by_dev(slave_dev, bond_dev);
 	if (res) {
 		netdev_err(bond_dev, "Couldn't add bond vlan ids to %s\n",
@@ -1725,6 +1692,40 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev,
 		goto err_upper_unlink;
 	}
 
+	/* If the mode uses primary, then the following is handled by
+	 * bond_change_active_slave().
+	 */
+	if (!bond_uses_primary(bond)) {
+		/* set promiscuity level to new slave */
+		if (bond_dev->flags & IFF_PROMISC) {
+			res = dev_set_promiscuity(slave_dev, 1);
+			if (res)
+				goto err_sysfs_del;
+		}
+
+		/* set allmulti level to new slave */
+		if (bond_dev->flags & IFF_ALLMULTI) {
+			res = dev_set_allmulti(slave_dev, 1);
+			if (res) {
+				if (bond_dev->flags & IFF_PROMISC)
+					dev_set_promiscuity(slave_dev, -1);
+				goto err_sysfs_del;
+			}
+		}
+
+		netif_addr_lock_bh(bond_dev);
+		dev_mc_sync_multiple(slave_dev, bond_dev);
+		dev_uc_sync_multiple(slave_dev, bond_dev);
+		netif_addr_unlock_bh(bond_dev);
+
+		if (BOND_MODE(bond) == BOND_MODE_8023AD) {
+			/* add lacpdu mc addr to mc list */
+			u8 lacpdu_multicast[ETH_ALEN] = MULTICAST_LACPDU_ADDR;
+
+			dev_mc_add(slave_dev, lacpdu_multicast);
+		}
+	}
+
 	bond->slave_cnt++;
 	bond_compute_features(bond);
 	bond_set_carrier(bond);
@@ -1748,6 +1749,9 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev,
 	return 0;
 
 /* Undo stages on error */
+err_sysfs_del:
+	bond_sysfs_slave_del(new_slave);
+
 err_upper_unlink:
 	bond_upper_dev_unlink(bond, new_slave);
 
@@ -1755,9 +1759,6 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev,
 	netdev_rx_handler_unregister(slave_dev);
 
 err_detach:
-	if (!bond_uses_primary(bond))
-		bond_hw_addr_flush(bond_dev, slave_dev);
-
 	vlan_vids_del_by_dev(slave_dev, bond_dev);
 	if (rcu_access_pointer(bond->primary_slave) == new_slave)
 		RCU_INIT_POINTER(bond->primary_slave, NULL);
diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c
index 8a9b085c2a98..58c705f24f96 100644
--- a/drivers/net/bonding/bond_options.c
+++ b/drivers/net/bonding/bond_options.c
@@ -1431,13 +1431,9 @@ static int bond_option_ad_actor_system_set(struct bonding *bond,
 {
 	u8 macaddr[ETH_ALEN];
 	u8 *mac;
-	int i;
 
 	if (newval->string) {
-		i = sscanf(newval->string, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
-			   &macaddr[0], &macaddr[1], &macaddr[2],
-			   &macaddr[3], &macaddr[4], &macaddr[5]);
-		if (i != ETH_ALEN)
+		if (!mac_pton(newval->string, macaddr))
 			goto err;
 		mac = macaddr;
 	} else {
diff --git a/drivers/net/caif/caif_hsi.c b/drivers/net/caif/caif_hsi.c
index b8029ea03307..433a14b9f731 100644
--- a/drivers/net/caif/caif_hsi.c
+++ b/drivers/net/caif/caif_hsi.c
@@ -264,7 +264,6 @@ static int cfhsi_tx_frm(struct cfhsi_desc *desc, struct cfhsi *cfhsi)
 	}
 
 	/* Create payload CAIF frames. */
-	pfrm = desc->emb_frm + CFHSI_MAX_EMB_FRM_SZ;
 	while (nfrms < CFHSI_MAX_PKTS) {
 		struct caif_payload_info *info;
 		int hpad;
diff --git a/drivers/net/can/c_can/c_can_pci.c b/drivers/net/can/c_can/c_can_pci.c
index d065c0e2d18e..406b4847e5dc 100644
--- a/drivers/net/can/c_can/c_can_pci.c
+++ b/drivers/net/can/c_can/c_can_pci.c
@@ -251,14 +251,14 @@ static void c_can_pci_remove(struct pci_dev *pdev)
 	pci_disable_device(pdev);
 }
 
-static struct c_can_pci_data c_can_sta2x11= {
+static const struct c_can_pci_data c_can_sta2x11= {
 	.type = BOSCH_C_CAN,
 	.reg_align = C_CAN_REG_ALIGN_32,
 	.freq = 52000000, /* 52 Mhz */
 	.bar = 0,
 };
 
-static struct c_can_pci_data c_can_pch = {
+static const struct c_can_pci_data c_can_pch = {
 	.type = BOSCH_C_CAN,
 	.reg_align = C_CAN_REG_32,
 	.freq = 50000000, /* 50 MHz */
diff --git a/drivers/net/can/cc770/cc770.c b/drivers/net/can/cc770/cc770.c
index 1e37313054f3..6da69af103e6 100644
--- a/drivers/net/can/cc770/cc770.c
+++ b/drivers/net/can/cc770/cc770.c
@@ -390,37 +390,23 @@ static int cc770_get_berr_counter(const struct net_device *dev,
 	return 0;
 }
 
-static netdev_tx_t cc770_start_xmit(struct sk_buff *skb, struct net_device *dev)
+static void cc770_tx(struct net_device *dev, int mo)
 {
 	struct cc770_priv *priv = netdev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
-	struct can_frame *cf = (struct can_frame *)skb->data;
-	unsigned int mo = obj2msgobj(CC770_OBJ_TX);
+	struct can_frame *cf = (struct can_frame *)priv->tx_skb->data;
 	u8 dlc, rtr;
 	u32 id;
 	int i;
 
-	if (can_dropped_invalid_skb(dev, skb))
-		return NETDEV_TX_OK;
-
-	if ((cc770_read_reg(priv,
-			    msgobj[mo].ctrl1) & TXRQST_UNC) == TXRQST_SET) {
-		netdev_err(dev, "TX register is still occupied!\n");
-		return NETDEV_TX_BUSY;
-	}
-
-	netif_stop_queue(dev);
-
 	dlc = cf->can_dlc;
 	id = cf->can_id;
-	if (cf->can_id & CAN_RTR_FLAG)
-		rtr = 0;
-	else
-		rtr = MSGCFG_DIR;
+	rtr = cf->can_id & CAN_RTR_FLAG ? 0 : MSGCFG_DIR;
+
+	cc770_write_reg(priv, msgobj[mo].ctrl0,
+			MSGVAL_RES | TXIE_RES | RXIE_RES | INTPND_RES);
 	cc770_write_reg(priv, msgobj[mo].ctrl1,
 			RMTPND_RES | TXRQST_RES | CPUUPD_SET | NEWDAT_RES);
-	cc770_write_reg(priv, msgobj[mo].ctrl0,
-			MSGVAL_SET | TXIE_SET | RXIE_RES | INTPND_RES);
+
 	if (id & CAN_EFF_FLAG) {
 		id &= CAN_EFF_MASK;
 		cc770_write_reg(priv, msgobj[mo].config,
@@ -439,22 +425,30 @@ static netdev_tx_t cc770_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	for (i = 0; i < dlc; i++)
 		cc770_write_reg(priv, msgobj[mo].data[i], cf->data[i]);
 
-	/* Store echo skb before starting the transfer */
-	can_put_echo_skb(skb, dev, 0);
-
 	cc770_write_reg(priv, msgobj[mo].ctrl1,
-			RMTPND_RES | TXRQST_SET | CPUUPD_RES | NEWDAT_UNC);
+			RMTPND_UNC | TXRQST_SET | CPUUPD_RES | NEWDAT_UNC);
+	cc770_write_reg(priv, msgobj[mo].ctrl0,
+			MSGVAL_SET | TXIE_SET | RXIE_SET | INTPND_UNC);
+}
 
-	stats->tx_bytes += dlc;
+static netdev_tx_t cc770_start_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct cc770_priv *priv = netdev_priv(dev);
+	unsigned int mo = obj2msgobj(CC770_OBJ_TX);
 
+	if (can_dropped_invalid_skb(dev, skb))
+		return NETDEV_TX_OK;
 
-	/*
-	 * HM: We had some cases of repeated IRQs so make sure the
-	 * INT is acknowledged I know it's already further up, but
-	 * doing again fixed the issue
-	 */
-	cc770_write_reg(priv, msgobj[mo].ctrl0,
-			MSGVAL_UNC | TXIE_UNC | RXIE_UNC | INTPND_RES);
+	netif_stop_queue(dev);
+
+	if ((cc770_read_reg(priv,
+			    msgobj[mo].ctrl1) & TXRQST_UNC) == TXRQST_SET) {
+		netdev_err(dev, "TX register is still occupied!\n");
+		return NETDEV_TX_BUSY;
+	}
+
+	priv->tx_skb = skb;
+	cc770_tx(dev, mo);
 
 	return NETDEV_TX_OK;
 }
@@ -680,19 +674,46 @@ static void cc770_tx_interrupt(struct net_device *dev, unsigned int o)
 	struct cc770_priv *priv = netdev_priv(dev);
 	struct net_device_stats *stats = &dev->stats;
 	unsigned int mo = obj2msgobj(o);
+	struct can_frame *cf;
+	u8 ctrl1;
+
+	ctrl1 = cc770_read_reg(priv, msgobj[mo].ctrl1);
 
-	/* Nothing more to send, switch off interrupts */
 	cc770_write_reg(priv, msgobj[mo].ctrl0,
 			MSGVAL_RES | TXIE_RES | RXIE_RES | INTPND_RES);
-	/*
-	 * We had some cases of repeated IRQ so make sure the
-	 * INT is acknowledged
+	cc770_write_reg(priv, msgobj[mo].ctrl1,
+			RMTPND_RES | TXRQST_RES | MSGLST_RES | NEWDAT_RES);
+
+	if (unlikely(!priv->tx_skb)) {
+		netdev_err(dev, "missing tx skb in tx interrupt\n");
+		return;
+	}
+
+	if (unlikely(ctrl1 & MSGLST_SET)) {
+		stats->rx_over_errors++;
+		stats->rx_errors++;
+	}
+
+	/* When the CC770 is sending an RTR message and it receives a regular
+	 * message that matches the id of the RTR message, it will overwrite the
+	 * outgoing message in the TX register. When this happens we must
+	 * process the received message and try to transmit the outgoing skb
+	 * again.
 	 */
-	cc770_write_reg(priv, msgobj[mo].ctrl0,
-			MSGVAL_UNC | TXIE_UNC | RXIE_UNC | INTPND_RES);
+	if (unlikely(ctrl1 & NEWDAT_SET)) {
+		cc770_rx(dev, mo, ctrl1);
+		cc770_tx(dev, mo);
+		return;
+	}
 
+	cf = (struct can_frame *)priv->tx_skb->data;
+	stats->tx_bytes += cf->can_dlc;
 	stats->tx_packets++;
+
+	can_put_echo_skb(priv->tx_skb, dev, 0);
 	can_get_echo_skb(dev, 0);
+	priv->tx_skb = NULL;
+
 	netif_wake_queue(dev);
 }
 
@@ -804,6 +825,7 @@ struct net_device *alloc_cc770dev(int sizeof_priv)
 	priv->can.do_set_bittiming = cc770_set_bittiming;
 	priv->can.do_set_mode = cc770_set_mode;
 	priv->can.ctrlmode_supported = CAN_CTRLMODE_3_SAMPLES;
+	priv->tx_skb = NULL;
 
 	memcpy(priv->obj_flags, cc770_obj_flags, sizeof(cc770_obj_flags));
 
diff --git a/drivers/net/can/cc770/cc770.h b/drivers/net/can/cc770/cc770.h
index a1739db98d91..95752e1d1283 100644
--- a/drivers/net/can/cc770/cc770.h
+++ b/drivers/net/can/cc770/cc770.h
@@ -193,6 +193,8 @@ struct cc770_priv {
 	u8 cpu_interface;	/* CPU interface register */
 	u8 clkout;		/* Clock out register */
 	u8 bus_config;		/* Bus conffiguration register */
+
+	struct sk_buff *tx_skb;
 };
 
 struct net_device *alloc_cc770dev(int sizeof_priv);
diff --git a/drivers/net/can/dev.c b/drivers/net/can/dev.c
index 365a8cc62405..b1779566c5bb 100644
--- a/drivers/net/can/dev.c
+++ b/drivers/net/can/dev.c
@@ -27,6 +27,7 @@
 #include <linux/can/skb.h>
 #include <linux/can/netlink.h>
 #include <linux/can/led.h>
+#include <linux/of.h>
 #include <net/rtnetlink.h>
 
 #define MOD_DESC "CAN device driver interface"
@@ -411,7 +412,7 @@ EXPORT_SYMBOL_GPL(can_change_state);
  * Local echo of CAN messages
  *
  * CAN network devices *should* support a local echo functionality
- * (see Documentation/networking/can.txt). To test the handling of CAN
+ * (see Documentation/networking/can.rst). To test the handling of CAN
  * interfaces that do not support the local echo both driver types are
  * implemented. In the case that the driver does not support the echo
  * the IFF_ECHO remains clear in dev->flags. This causes the PF_CAN core
@@ -814,6 +815,29 @@ int open_candev(struct net_device *dev)
 }
 EXPORT_SYMBOL_GPL(open_candev);
 
+#ifdef CONFIG_OF
+/* Common function that can be used to understand the limitation of
+ * a transceiver when it provides no means to determine these limitations
+ * at runtime.
+ */
+void of_can_transceiver(struct net_device *dev)
+{
+	struct device_node *dn;
+	struct can_priv *priv = netdev_priv(dev);
+	struct device_node *np = dev->dev.parent->of_node;
+	int ret;
+
+	dn = of_get_child_by_name(np, "can-transceiver");
+	if (!dn)
+		return;
+
+	ret = of_property_read_u32(dn, "max-bitrate", &priv->bitrate_max);
+	if ((ret && ret != -EINVAL) || (!ret && !priv->bitrate_max))
+		netdev_warn(dev, "Invalid value for transceiver max bitrate. Ignoring bitrate limit.\n");
+}
+EXPORT_SYMBOL_GPL(of_can_transceiver);
+#endif
+
 /*
  * Common close function for cleanup before the device gets closed.
  *
@@ -913,6 +937,13 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[],
 					priv->bitrate_const_cnt);
 		if (err)
 			return err;
+
+		if (priv->bitrate_max && bt.bitrate > priv->bitrate_max) {
+			netdev_err(dev, "arbitration bitrate surpasses transceiver capabilities of %d bps\n",
+				   priv->bitrate_max);
+			return -EINVAL;
+		}
+
 		memcpy(&priv->bittiming, &bt, sizeof(bt));
 
 		if (priv->do_set_bittiming) {
@@ -997,6 +1028,13 @@ static int can_changelink(struct net_device *dev, struct nlattr *tb[],
 					priv->data_bitrate_const_cnt);
 		if (err)
 			return err;
+
+		if (priv->bitrate_max && dbt.bitrate > priv->bitrate_max) {
+			netdev_err(dev, "canfd data bitrate surpasses transceiver capabilities of %d bps\n",
+				   priv->bitrate_max);
+			return -EINVAL;
+		}
+
 		memcpy(&priv->data_bittiming, &dbt, sizeof(dbt));
 
 		if (priv->do_set_data_bittiming) {
@@ -1064,6 +1102,7 @@ static size_t can_get_size(const struct net_device *dev)
 	if (priv->data_bitrate_const)				/* IFLA_CAN_DATA_BITRATE_CONST */
 		size += nla_total_size(sizeof(*priv->data_bitrate_const) *
 				       priv->data_bitrate_const_cnt);
+	size += sizeof(priv->bitrate_max);			/* IFLA_CAN_BITRATE_MAX */
 
 	return size;
 }
@@ -1121,7 +1160,11 @@ static int can_fill_info(struct sk_buff *skb, const struct net_device *dev)
 	     nla_put(skb, IFLA_CAN_DATA_BITRATE_CONST,
 		     sizeof(*priv->data_bitrate_const) *
 		     priv->data_bitrate_const_cnt,
-		     priv->data_bitrate_const))
+		     priv->data_bitrate_const)) ||
+
+	    (nla_put(skb, IFLA_CAN_BITRATE_MAX,
+		     sizeof(priv->bitrate_max),
+		     &priv->bitrate_max))
 	    )
 
 		return -EMSGSIZE;
diff --git a/drivers/net/can/flexcan.c b/drivers/net/can/flexcan.c
index 760d2c07e3a2..634c51e6b8ae 100644
--- a/drivers/net/can/flexcan.c
+++ b/drivers/net/can/flexcan.c
@@ -190,6 +190,7 @@
  *   MX53  FlexCAN2  03.00.00.00    yes        no        no       no        no
  *   MX6s  FlexCAN3  10.00.12.00    yes       yes        no       no       yes
  *   VF610 FlexCAN3  ?               no       yes        no      yes       yes?
+ * LS1021A FlexCAN2  03.00.04.00     no       yes        no       no       yes
  *
  * Some SOCs do not have the RX_WARN & TX_WARN interrupt line connected.
  */
@@ -279,6 +280,10 @@ struct flexcan_priv {
 	struct clk *clk_per;
 	const struct flexcan_devtype_data *devtype_data;
 	struct regulator *reg_xceiver;
+
+	/* Read and Write APIs */
+	u32 (*read)(void __iomem *addr);
+	void (*write)(u32 val, void __iomem *addr);
 };
 
 static const struct flexcan_devtype_data fsl_p1010_devtype_data = {
@@ -301,6 +306,12 @@ static const struct flexcan_devtype_data fsl_vf610_devtype_data = {
 		FLEXCAN_QUIRK_BROKEN_PERR_STATE,
 };
 
+static const struct flexcan_devtype_data fsl_ls1021a_r2_devtype_data = {
+	.quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
+		FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
+		FLEXCAN_QUIRK_USE_OFF_TIMESTAMP,
+};
+
 static const struct can_bittiming_const flexcan_bittiming_const = {
 	.name = DRV_NAME,
 	.tseg1_min = 4,
@@ -313,39 +324,45 @@ static const struct can_bittiming_const flexcan_bittiming_const = {
 	.brp_inc = 1,
 };
 
-/* Abstract off the read/write for arm versus ppc. This
- * assumes that PPC uses big-endian registers and everything
- * else uses little-endian registers, independent of CPU
- * endianness.
+/* FlexCAN module is essentially modelled as a little-endian IP in most
+ * SoCs, i.e the registers as well as the message buffer areas are
+ * implemented in a little-endian fashion.
+ *
+ * However there are some SoCs (e.g. LS1021A) which implement the FlexCAN
+ * module in a big-endian fashion (i.e the registers as well as the
+ * message buffer areas are implemented in a big-endian way).
+ *
+ * In addition, the FlexCAN module can be found on SoCs having ARM or
+ * PPC cores. So, we need to abstract off the register read/write
+ * functions, ensuring that these cater to all the combinations of module
+ * endianness and underlying CPU endianness.
  */
-#if defined(CONFIG_PPC)
-static inline u32 flexcan_read(void __iomem *addr)
+static inline u32 flexcan_read_be(void __iomem *addr)
 {
-	return in_be32(addr);
+	return ioread32be(addr);
 }
 
-static inline void flexcan_write(u32 val, void __iomem *addr)
+static inline void flexcan_write_be(u32 val, void __iomem *addr)
 {
-	out_be32(addr, val);
+	iowrite32be(val, addr);
 }
-#else
-static inline u32 flexcan_read(void __iomem *addr)
+
+static inline u32 flexcan_read_le(void __iomem *addr)
 {
-	return readl(addr);
+	return ioread32(addr);
 }
 
-static inline void flexcan_write(u32 val, void __iomem *addr)
+static inline void flexcan_write_le(u32 val, void __iomem *addr)
 {
-	writel(val, addr);
+	iowrite32(val, addr);
 }
-#endif
 
 static inline void flexcan_error_irq_enable(const struct flexcan_priv *priv)
 {
 	struct flexcan_regs __iomem *regs = priv->regs;
 	u32 reg_ctrl = (priv->reg_ctrl_default | FLEXCAN_CTRL_ERR_MSK);
 
-	flexcan_write(reg_ctrl, &regs->ctrl);
+	priv->write(reg_ctrl, &regs->ctrl);
 }
 
 static inline void flexcan_error_irq_disable(const struct flexcan_priv *priv)
@@ -353,7 +370,7 @@ static inline void flexcan_error_irq_disable(const struct flexcan_priv *priv)
 	struct flexcan_regs __iomem *regs = priv->regs;
 	u32 reg_ctrl = (priv->reg_ctrl_default & ~FLEXCAN_CTRL_ERR_MSK);
 
-	flexcan_write(reg_ctrl, &regs->ctrl);
+	priv->write(reg_ctrl, &regs->ctrl);
 }
 
 static inline int flexcan_transceiver_enable(const struct flexcan_priv *priv)
@@ -378,14 +395,14 @@ static int flexcan_chip_enable(struct flexcan_priv *priv)
 	unsigned int timeout = FLEXCAN_TIMEOUT_US / 10;
 	u32 reg;
 
-	reg = flexcan_read(&regs->mcr);
+	reg = priv->read(&regs->mcr);
 	reg &= ~FLEXCAN_MCR_MDIS;
-	flexcan_write(reg, &regs->mcr);
+	priv->write(reg, &regs->mcr);
 
-	while (timeout-- && (flexcan_read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK))
+	while (timeout-- && (priv->read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK))
 		udelay(10);
 
-	if (flexcan_read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK)
+	if (priv->read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK)
 		return -ETIMEDOUT;
 
 	return 0;
@@ -397,14 +414,14 @@ static int flexcan_chip_disable(struct flexcan_priv *priv)
 	unsigned int timeout = FLEXCAN_TIMEOUT_US / 10;
 	u32 reg;
 
-	reg = flexcan_read(&regs->mcr);
+	reg = priv->read(&regs->mcr);
 	reg |= FLEXCAN_MCR_MDIS;
-	flexcan_write(reg, &regs->mcr);
+	priv->write(reg, &regs->mcr);
 
-	while (timeout-- && !(flexcan_read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK))
+	while (timeout-- && !(priv->read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK))
 		udelay(10);
 
-	if (!(flexcan_read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK))
+	if (!(priv->read(&regs->mcr) & FLEXCAN_MCR_LPM_ACK))
 		return -ETIMEDOUT;
 
 	return 0;
@@ -416,14 +433,14 @@ static int flexcan_chip_freeze(struct flexcan_priv *priv)
 	unsigned int timeout = 1000 * 1000 * 10 / priv->can.bittiming.bitrate;
 	u32 reg;
 
-	reg = flexcan_read(&regs->mcr);
+	reg = priv->read(&regs->mcr);
 	reg |= FLEXCAN_MCR_HALT;
-	flexcan_write(reg, &regs->mcr);
+	priv->write(reg, &regs->mcr);
 
-	while (timeout-- && !(flexcan_read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK))
+	while (timeout-- && !(priv->read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK))
 		udelay(100);
 
-	if (!(flexcan_read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK))
+	if (!(priv->read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK))
 		return -ETIMEDOUT;
 
 	return 0;
@@ -435,14 +452,14 @@ static int flexcan_chip_unfreeze(struct flexcan_priv *priv)
 	unsigned int timeout = FLEXCAN_TIMEOUT_US / 10;
 	u32 reg;
 
-	reg = flexcan_read(&regs->mcr);
+	reg = priv->read(&regs->mcr);
 	reg &= ~FLEXCAN_MCR_HALT;
-	flexcan_write(reg, &regs->mcr);
+	priv->write(reg, &regs->mcr);
 
-	while (timeout-- && (flexcan_read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK))
+	while (timeout-- && (priv->read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK))
 		udelay(10);
 
-	if (flexcan_read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK)
+	if (priv->read(&regs->mcr) & FLEXCAN_MCR_FRZ_ACK)
 		return -ETIMEDOUT;
 
 	return 0;
@@ -453,11 +470,11 @@ static int flexcan_chip_softreset(struct flexcan_priv *priv)
 	struct flexcan_regs __iomem *regs = priv->regs;
 	unsigned int timeout = FLEXCAN_TIMEOUT_US / 10;
 
-	flexcan_write(FLEXCAN_MCR_SOFTRST, &regs->mcr);
-	while (timeout-- && (flexcan_read(&regs->mcr) & FLEXCAN_MCR_SOFTRST))
+	priv->write(FLEXCAN_MCR_SOFTRST, &regs->mcr);
+	while (timeout-- && (priv->read(&regs->mcr) & FLEXCAN_MCR_SOFTRST))
 		udelay(10);
 
-	if (flexcan_read(&regs->mcr) & FLEXCAN_MCR_SOFTRST)
+	if (priv->read(&regs->mcr) & FLEXCAN_MCR_SOFTRST)
 		return -ETIMEDOUT;
 
 	return 0;
@@ -468,7 +485,7 @@ static int __flexcan_get_berr_counter(const struct net_device *dev,
 {
 	const struct flexcan_priv *priv = netdev_priv(dev);
 	struct flexcan_regs __iomem *regs = priv->regs;
-	u32 reg = flexcan_read(&regs->ecr);
+	u32 reg = priv->read(&regs->ecr);
 
 	bec->txerr = (reg >> 0) & 0xff;
 	bec->rxerr = (reg >> 8) & 0xff;
@@ -524,24 +541,24 @@ static int flexcan_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	if (cf->can_dlc > 0) {
 		data = be32_to_cpup((__be32 *)&cf->data[0]);
-		flexcan_write(data, &priv->tx_mb->data[0]);
+		priv->write(data, &priv->tx_mb->data[0]);
 	}
 	if (cf->can_dlc > 4) {
 		data = be32_to_cpup((__be32 *)&cf->data[4]);
-		flexcan_write(data, &priv->tx_mb->data[1]);
+		priv->write(data, &priv->tx_mb->data[1]);
 	}
 
 	can_put_echo_skb(skb, dev, 0);
 
-	flexcan_write(can_id, &priv->tx_mb->can_id);
-	flexcan_write(ctrl, &priv->tx_mb->can_ctrl);
+	priv->write(can_id, &priv->tx_mb->can_id);
+	priv->write(ctrl, &priv->tx_mb->can_ctrl);
 
 	/* Errata ERR005829 step8:
 	 * Write twice INACTIVE(0x8) code to first MB.
 	 */
-	flexcan_write(FLEXCAN_MB_CODE_TX_INACTIVE,
+	priv->write(FLEXCAN_MB_CODE_TX_INACTIVE,
 		      &priv->tx_mb_reserved->can_ctrl);
-	flexcan_write(FLEXCAN_MB_CODE_TX_INACTIVE,
+	priv->write(FLEXCAN_MB_CODE_TX_INACTIVE,
 		      &priv->tx_mb_reserved->can_ctrl);
 
 	return NETDEV_TX_OK;
@@ -660,7 +677,7 @@ static unsigned int flexcan_mailbox_read(struct can_rx_offload *offload,
 		u32 code;
 
 		do {
-			reg_ctrl = flexcan_read(&mb->can_ctrl);
+			reg_ctrl = priv->read(&mb->can_ctrl);
 		} while (reg_ctrl & FLEXCAN_MB_CODE_RX_BUSY_BIT);
 
 		/* is this MB empty? */
@@ -675,17 +692,17 @@ static unsigned int flexcan_mailbox_read(struct can_rx_offload *offload,
 			offload->dev->stats.rx_errors++;
 		}
 	} else {
-		reg_iflag1 = flexcan_read(&regs->iflag1);
+		reg_iflag1 = priv->read(&regs->iflag1);
 		if (!(reg_iflag1 & FLEXCAN_IFLAG_RX_FIFO_AVAILABLE))
 			return 0;
 
-		reg_ctrl = flexcan_read(&mb->can_ctrl);
+		reg_ctrl = priv->read(&mb->can_ctrl);
 	}
 
 	/* increase timstamp to full 32 bit */
 	*timestamp = reg_ctrl << 16;
 
-	reg_id = flexcan_read(&mb->can_id);
+	reg_id = priv->read(&mb->can_id);
 	if (reg_ctrl & FLEXCAN_MB_CNT_IDE)
 		cf->can_id = ((reg_id >> 0) & CAN_EFF_MASK) | CAN_EFF_FLAG;
 	else
@@ -695,19 +712,19 @@ static unsigned int flexcan_mailbox_read(struct can_rx_offload *offload,
 		cf->can_id |= CAN_RTR_FLAG;
 	cf->can_dlc = get_can_dlc((reg_ctrl >> 16) & 0xf);
 
-	*(__be32 *)(cf->data + 0) = cpu_to_be32(flexcan_read(&mb->data[0]));
-	*(__be32 *)(cf->data + 4) = cpu_to_be32(flexcan_read(&mb->data[1]));
+	*(__be32 *)(cf->data + 0) = cpu_to_be32(priv->read(&mb->data[0]));
+	*(__be32 *)(cf->data + 4) = cpu_to_be32(priv->read(&mb->data[1]));
 
 	/* mark as read */
 	if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP) {
 		/* Clear IRQ */
 		if (n < 32)
-			flexcan_write(BIT(n), &regs->iflag1);
+			priv->write(BIT(n), &regs->iflag1);
 		else
-			flexcan_write(BIT(n - 32), &regs->iflag2);
+			priv->write(BIT(n - 32), &regs->iflag2);
 	} else {
-		flexcan_write(FLEXCAN_IFLAG_RX_FIFO_AVAILABLE, &regs->iflag1);
-		flexcan_read(&regs->timer);
+		priv->write(FLEXCAN_IFLAG_RX_FIFO_AVAILABLE, &regs->iflag1);
+		priv->read(&regs->timer);
 	}
 
 	return 1;
@@ -719,8 +736,8 @@ static inline u64 flexcan_read_reg_iflag_rx(struct flexcan_priv *priv)
 	struct flexcan_regs __iomem *regs = priv->regs;
 	u32 iflag1, iflag2;
 
-	iflag2 = flexcan_read(&regs->iflag2) & priv->reg_imask2_default;
-	iflag1 = flexcan_read(&regs->iflag1) & priv->reg_imask1_default &
+	iflag2 = priv->read(&regs->iflag2) & priv->reg_imask2_default;
+	iflag1 = priv->read(&regs->iflag1) & priv->reg_imask1_default &
 		~FLEXCAN_IFLAG_MB(priv->tx_mb_idx);
 
 	return (u64)iflag2 << 32 | iflag1;
@@ -736,7 +753,7 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id)
 	u32 reg_iflag1, reg_esr;
 	enum can_state last_state = priv->can.state;
 
-	reg_iflag1 = flexcan_read(&regs->iflag1);
+	reg_iflag1 = priv->read(&regs->iflag1);
 
 	/* reception interrupt */
 	if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP) {
@@ -759,7 +776,8 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id)
 		/* FIFO overflow interrupt */
 		if (reg_iflag1 & FLEXCAN_IFLAG_RX_FIFO_OVERFLOW) {
 			handled = IRQ_HANDLED;
-			flexcan_write(FLEXCAN_IFLAG_RX_FIFO_OVERFLOW, &regs->iflag1);
+			priv->write(FLEXCAN_IFLAG_RX_FIFO_OVERFLOW,
+				    &regs->iflag1);
 			dev->stats.rx_over_errors++;
 			dev->stats.rx_errors++;
 		}
@@ -773,18 +791,18 @@ static irqreturn_t flexcan_irq(int irq, void *dev_id)
 		can_led_event(dev, CAN_LED_EVENT_TX);
 
 		/* after sending a RTR frame MB is in RX mode */
-		flexcan_write(FLEXCAN_MB_CODE_TX_INACTIVE,
-			      &priv->tx_mb->can_ctrl);
-		flexcan_write(FLEXCAN_IFLAG_MB(priv->tx_mb_idx), &regs->iflag1);
+		priv->write(FLEXCAN_MB_CODE_TX_INACTIVE,
+			    &priv->tx_mb->can_ctrl);
+		priv->write(FLEXCAN_IFLAG_MB(priv->tx_mb_idx), &regs->iflag1);
 		netif_wake_queue(dev);
 	}
 
-	reg_esr = flexcan_read(&regs->esr);
+	reg_esr = priv->read(&regs->esr);
 
 	/* ACK all bus error and state change IRQ sources */
 	if (reg_esr & FLEXCAN_ESR_ALL_INT) {
 		handled = IRQ_HANDLED;
-		flexcan_write(reg_esr & FLEXCAN_ESR_ALL_INT, &regs->esr);
+		priv->write(reg_esr & FLEXCAN_ESR_ALL_INT, &regs->esr);
 	}
 
 	/* state change interrupt or broken error state quirk fix is enabled */
@@ -846,7 +864,7 @@ static void flexcan_set_bittiming(struct net_device *dev)
 	struct flexcan_regs __iomem *regs = priv->regs;
 	u32 reg;
 
-	reg = flexcan_read(&regs->ctrl);
+	reg = priv->read(&regs->ctrl);
 	reg &= ~(FLEXCAN_CTRL_PRESDIV(0xff) |
 		 FLEXCAN_CTRL_RJW(0x3) |
 		 FLEXCAN_CTRL_PSEG1(0x7) |
@@ -870,11 +888,11 @@ static void flexcan_set_bittiming(struct net_device *dev)
 		reg |= FLEXCAN_CTRL_SMP;
 
 	netdev_dbg(dev, "writing ctrl=0x%08x\n", reg);
-	flexcan_write(reg, &regs->ctrl);
+	priv->write(reg, &regs->ctrl);
 
 	/* print chip status */
 	netdev_dbg(dev, "%s: mcr=0x%08x ctrl=0x%08x\n", __func__,
-		   flexcan_read(&regs->mcr), flexcan_read(&regs->ctrl));
+		   priv->read(&regs->mcr), priv->read(&regs->ctrl));
 }
 
 /* flexcan_chip_start
@@ -913,7 +931,7 @@ static int flexcan_chip_start(struct net_device *dev)
 	 * choose format C
 	 * set max mailbox number
 	 */
-	reg_mcr = flexcan_read(&regs->mcr);
+	reg_mcr = priv->read(&regs->mcr);
 	reg_mcr &= ~FLEXCAN_MCR_MAXMB(0xff);
 	reg_mcr |= FLEXCAN_MCR_FRZ | FLEXCAN_MCR_HALT | FLEXCAN_MCR_SUPV |
 		FLEXCAN_MCR_WRN_EN | FLEXCAN_MCR_SRX_DIS | FLEXCAN_MCR_IRMQ |
@@ -927,7 +945,7 @@ static int flexcan_chip_start(struct net_device *dev)
 			FLEXCAN_MCR_MAXMB(priv->tx_mb_idx);
 	}
 	netdev_dbg(dev, "%s: writing mcr=0x%08x", __func__, reg_mcr);
-	flexcan_write(reg_mcr, &regs->mcr);
+	priv->write(reg_mcr, &regs->mcr);
 
 	/* CTRL
 	 *
@@ -940,7 +958,7 @@ static int flexcan_chip_start(struct net_device *dev)
 	 * enable bus off interrupt
 	 * (== FLEXCAN_CTRL_ERR_STATE)
 	 */
-	reg_ctrl = flexcan_read(&regs->ctrl);
+	reg_ctrl = priv->read(&regs->ctrl);
 	reg_ctrl &= ~FLEXCAN_CTRL_TSYN;
 	reg_ctrl |= FLEXCAN_CTRL_BOFF_REC | FLEXCAN_CTRL_LBUF |
 		FLEXCAN_CTRL_ERR_STATE;
@@ -960,45 +978,45 @@ static int flexcan_chip_start(struct net_device *dev)
 	/* leave interrupts disabled for now */
 	reg_ctrl &= ~FLEXCAN_CTRL_ERR_ALL;
 	netdev_dbg(dev, "%s: writing ctrl=0x%08x", __func__, reg_ctrl);
-	flexcan_write(reg_ctrl, &regs->ctrl);
+	priv->write(reg_ctrl, &regs->ctrl);
 
 	if ((priv->devtype_data->quirks & FLEXCAN_QUIRK_ENABLE_EACEN_RRS)) {
-		reg_ctrl2 = flexcan_read(&regs->ctrl2);
+		reg_ctrl2 = priv->read(&regs->ctrl2);
 		reg_ctrl2 |= FLEXCAN_CTRL2_EACEN | FLEXCAN_CTRL2_RRS;
-		flexcan_write(reg_ctrl2, &regs->ctrl2);
+		priv->write(reg_ctrl2, &regs->ctrl2);
 	}
 
 	/* clear and invalidate all mailboxes first */
 	for (i = priv->tx_mb_idx; i < ARRAY_SIZE(regs->mb); i++) {
-		flexcan_write(FLEXCAN_MB_CODE_RX_INACTIVE,
-			      &regs->mb[i].can_ctrl);
+		priv->write(FLEXCAN_MB_CODE_RX_INACTIVE,
+			    &regs->mb[i].can_ctrl);
 	}
 
 	if (priv->devtype_data->quirks & FLEXCAN_QUIRK_USE_OFF_TIMESTAMP) {
 		for (i = priv->offload.mb_first; i <= priv->offload.mb_last; i++)
-			flexcan_write(FLEXCAN_MB_CODE_RX_EMPTY,
-				      &regs->mb[i].can_ctrl);
+			priv->write(FLEXCAN_MB_CODE_RX_EMPTY,
+				    &regs->mb[i].can_ctrl);
 	}
 
 	/* Errata ERR005829: mark first TX mailbox as INACTIVE */
-	flexcan_write(FLEXCAN_MB_CODE_TX_INACTIVE,
-		      &priv->tx_mb_reserved->can_ctrl);
+	priv->write(FLEXCAN_MB_CODE_TX_INACTIVE,
+		    &priv->tx_mb_reserved->can_ctrl);
 
 	/* mark TX mailbox as INACTIVE */
-	flexcan_write(FLEXCAN_MB_CODE_TX_INACTIVE,
-		      &priv->tx_mb->can_ctrl);
+	priv->write(FLEXCAN_MB_CODE_TX_INACTIVE,
+		    &priv->tx_mb->can_ctrl);
 
 	/* acceptance mask/acceptance code (accept everything) */
-	flexcan_write(0x0, &regs->rxgmask);
-	flexcan_write(0x0, &regs->rx14mask);
-	flexcan_write(0x0, &regs->rx15mask);
+	priv->write(0x0, &regs->rxgmask);
+	priv->write(0x0, &regs->rx14mask);
+	priv->write(0x0, &regs->rx15mask);
 
 	if (priv->devtype_data->quirks & FLEXCAN_QUIRK_DISABLE_RXFG)
-		flexcan_write(0x0, &regs->rxfgmask);
+		priv->write(0x0, &regs->rxfgmask);
 
 	/* clear acceptance filters */
 	for (i = 0; i < ARRAY_SIZE(regs->mb); i++)
-		flexcan_write(0, &regs->rximr[i]);
+		priv->write(0, &regs->rximr[i]);
 
 	/* On Vybrid, disable memory error detection interrupts
 	 * and freeze mode.
@@ -1011,16 +1029,16 @@ static int flexcan_chip_start(struct net_device *dev)
 		 * and Correction of Memory Errors" to write to
 		 * MECR register
 		 */
-		reg_ctrl2 = flexcan_read(&regs->ctrl2);
+		reg_ctrl2 = priv->read(&regs->ctrl2);
 		reg_ctrl2 |= FLEXCAN_CTRL2_ECRWRE;
-		flexcan_write(reg_ctrl2, &regs->ctrl2);
+		priv->write(reg_ctrl2, &regs->ctrl2);
 
-		reg_mecr = flexcan_read(&regs->mecr);
+		reg_mecr = priv->read(&regs->mecr);
 		reg_mecr &= ~FLEXCAN_MECR_ECRWRDIS;
-		flexcan_write(reg_mecr, &regs->mecr);
+		priv->write(reg_mecr, &regs->mecr);
 		reg_mecr &= ~(FLEXCAN_MECR_NCEFAFRZ | FLEXCAN_MECR_HANCEI_MSK |
 			      FLEXCAN_MECR_FANCEI_MSK);
-		flexcan_write(reg_mecr, &regs->mecr);
+		priv->write(reg_mecr, &regs->mecr);
 	}
 
 	err = flexcan_transceiver_enable(priv);
@@ -1036,14 +1054,14 @@ static int flexcan_chip_start(struct net_device *dev)
 
 	/* enable interrupts atomically */
 	disable_irq(dev->irq);
-	flexcan_write(priv->reg_ctrl_default, &regs->ctrl);
-	flexcan_write(priv->reg_imask1_default, &regs->imask1);
-	flexcan_write(priv->reg_imask2_default, &regs->imask2);
+	priv->write(priv->reg_ctrl_default, &regs->ctrl);
+	priv->write(priv->reg_imask1_default, &regs->imask1);
+	priv->write(priv->reg_imask2_default, &regs->imask2);
 	enable_irq(dev->irq);
 
 	/* print chip status */
 	netdev_dbg(dev, "%s: reading mcr=0x%08x ctrl=0x%08x\n", __func__,
-		   flexcan_read(&regs->mcr), flexcan_read(&regs->ctrl));
+		   priv->read(&regs->mcr), priv->read(&regs->ctrl));
 
 	return 0;
 
@@ -1068,10 +1086,10 @@ static void flexcan_chip_stop(struct net_device *dev)
 	flexcan_chip_disable(priv);
 
 	/* Disable all interrupts */
-	flexcan_write(0, &regs->imask2);
-	flexcan_write(0, &regs->imask1);
-	flexcan_write(priv->reg_ctrl_default & ~FLEXCAN_CTRL_ERR_ALL,
-		      &regs->ctrl);
+	priv->write(0, &regs->imask2);
+	priv->write(0, &regs->imask1);
+	priv->write(priv->reg_ctrl_default & ~FLEXCAN_CTRL_ERR_ALL,
+		    &regs->ctrl);
 
 	flexcan_transceiver_disable(priv);
 	priv->can.state = CAN_STATE_STOPPED;
@@ -1186,26 +1204,26 @@ static int register_flexcandev(struct net_device *dev)
 	err = flexcan_chip_disable(priv);
 	if (err)
 		goto out_disable_per;
-	reg = flexcan_read(&regs->ctrl);
+	reg = priv->read(&regs->ctrl);
 	reg |= FLEXCAN_CTRL_CLK_SRC;
-	flexcan_write(reg, &regs->ctrl);
+	priv->write(reg, &regs->ctrl);
 
 	err = flexcan_chip_enable(priv);
 	if (err)
 		goto out_chip_disable;
 
 	/* set freeze, halt and activate FIFO, restrict register access */
-	reg = flexcan_read(&regs->mcr);
+	reg = priv->read(&regs->mcr);
 	reg |= FLEXCAN_MCR_FRZ | FLEXCAN_MCR_HALT |
 		FLEXCAN_MCR_FEN | FLEXCAN_MCR_SUPV;
-	flexcan_write(reg, &regs->mcr);
+	priv->write(reg, &regs->mcr);
 
 	/* Currently we only support newer versions of this core
 	 * featuring a RX hardware FIFO (although this driver doesn't
 	 * make use of it on some cores). Older cores, found on some
 	 * Coldfire derivates are not tested.
 	 */
-	reg = flexcan_read(&regs->mcr);
+	reg = priv->read(&regs->mcr);
 	if (!(reg & FLEXCAN_MCR_FEN)) {
 		netdev_err(dev, "Could not enable RX FIFO, unsupported core\n");
 		err = -ENODEV;
@@ -1233,8 +1251,12 @@ static void unregister_flexcandev(struct net_device *dev)
 static const struct of_device_id flexcan_of_match[] = {
 	{ .compatible = "fsl,imx6q-flexcan", .data = &fsl_imx6q_devtype_data, },
 	{ .compatible = "fsl,imx28-flexcan", .data = &fsl_imx28_devtype_data, },
+	{ .compatible = "fsl,imx53-flexcan", .data = &fsl_p1010_devtype_data, },
+	{ .compatible = "fsl,imx35-flexcan", .data = &fsl_p1010_devtype_data, },
+	{ .compatible = "fsl,imx25-flexcan", .data = &fsl_p1010_devtype_data, },
 	{ .compatible = "fsl,p1010-flexcan", .data = &fsl_p1010_devtype_data, },
 	{ .compatible = "fsl,vf610-flexcan", .data = &fsl_vf610_devtype_data, },
+	{ .compatible = "fsl,ls1021ar2-flexcan", .data = &fsl_ls1021a_r2_devtype_data, },
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, flexcan_of_match);
@@ -1314,6 +1336,21 @@ static int flexcan_probe(struct platform_device *pdev)
 	dev->flags |= IFF_ECHO;
 
 	priv = netdev_priv(dev);
+
+	if (of_property_read_bool(pdev->dev.of_node, "big-endian")) {
+		priv->read = flexcan_read_be;
+		priv->write = flexcan_write_be;
+	} else {
+		if (of_device_is_compatible(pdev->dev.of_node,
+					    "fsl,p1010-flexcan")) {
+			priv->read = flexcan_read_be;
+			priv->write = flexcan_write_be;
+		} else {
+			priv->read = flexcan_read_le;
+			priv->write = flexcan_write_le;
+		}
+	}
+
 	priv->can.clock.freq = clock_freq;
 	priv->can.bittiming_const = &flexcan_bittiming_const;
 	priv->can.do_set_mode = flexcan_set_mode;
diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c
index 2772d05ff11c..fedd927ba6ed 100644
--- a/drivers/net/can/ifi_canfd/ifi_canfd.c
+++ b/drivers/net/can/ifi_canfd/ifi_canfd.c
@@ -30,6 +30,7 @@
 #define IFI_CANFD_STCMD_ERROR_ACTIVE		BIT(2)
 #define IFI_CANFD_STCMD_ERROR_PASSIVE		BIT(3)
 #define IFI_CANFD_STCMD_BUSOFF			BIT(4)
+#define IFI_CANFD_STCMD_ERROR_WARNING		BIT(5)
 #define IFI_CANFD_STCMD_BUSMONITOR		BIT(16)
 #define IFI_CANFD_STCMD_LOOPBACK		BIT(18)
 #define IFI_CANFD_STCMD_DISABLE_CANFD		BIT(24)
@@ -52,7 +53,10 @@
 #define IFI_CANFD_TXSTCMD_OVERFLOW		BIT(13)
 
 #define IFI_CANFD_INTERRUPT			0xc
+#define IFI_CANFD_INTERRUPT_ERROR_BUSOFF	BIT(0)
 #define IFI_CANFD_INTERRUPT_ERROR_WARNING	BIT(1)
+#define IFI_CANFD_INTERRUPT_ERROR_STATE_CHG	BIT(2)
+#define IFI_CANFD_INTERRUPT_ERROR_REC_TEC_INC	BIT(3)
 #define IFI_CANFD_INTERRUPT_ERROR_COUNTER	BIT(10)
 #define IFI_CANFD_INTERRUPT_TXFIFO_EMPTY	BIT(16)
 #define IFI_CANFD_INTERRUPT_TXFIFO_REMOVE	BIT(22)
@@ -61,6 +65,10 @@
 #define IFI_CANFD_INTERRUPT_SET_IRQ		((u32)BIT(31))
 
 #define IFI_CANFD_IRQMASK			0x10
+#define IFI_CANFD_IRQMASK_ERROR_BUSOFF		BIT(0)
+#define IFI_CANFD_IRQMASK_ERROR_WARNING		BIT(1)
+#define IFI_CANFD_IRQMASK_ERROR_STATE_CHG	BIT(2)
+#define IFI_CANFD_IRQMASK_ERROR_REC_TEC_INC	BIT(3)
 #define IFI_CANFD_IRQMASK_SET_ERR		BIT(7)
 #define IFI_CANFD_IRQMASK_SET_TS		BIT(15)
 #define IFI_CANFD_IRQMASK_TXFIFO_EMPTY		BIT(16)
@@ -136,6 +144,8 @@
 #define IFI_CANFD_SYSCLOCK			0x50
 
 #define IFI_CANFD_VER				0x54
+#define IFI_CANFD_VER_REV_MASK			0xff
+#define IFI_CANFD_VER_REV_MIN_SUPPORTED		0x15
 
 #define IFI_CANFD_IP_ID				0x58
 #define IFI_CANFD_IP_ID_VALUE			0xD073CAFD
@@ -220,7 +230,10 @@ static void ifi_canfd_irq_enable(struct net_device *ndev, bool enable)
 
 	if (enable) {
 		enirq = IFI_CANFD_IRQMASK_TXFIFO_EMPTY |
-			IFI_CANFD_IRQMASK_RXFIFO_NEMPTY;
+			IFI_CANFD_IRQMASK_RXFIFO_NEMPTY |
+			IFI_CANFD_IRQMASK_ERROR_STATE_CHG |
+			IFI_CANFD_IRQMASK_ERROR_WARNING |
+			IFI_CANFD_IRQMASK_ERROR_BUSOFF;
 		if (priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING)
 			enirq |= IFI_CANFD_INTERRUPT_ERROR_COUNTER;
 	}
@@ -361,12 +374,13 @@ static int ifi_canfd_handle_lost_msg(struct net_device *ndev)
 	return 1;
 }
 
-static int ifi_canfd_handle_lec_err(struct net_device *ndev, const u32 errctr)
+static int ifi_canfd_handle_lec_err(struct net_device *ndev)
 {
 	struct ifi_canfd_priv *priv = netdev_priv(ndev);
 	struct net_device_stats *stats = &ndev->stats;
 	struct can_frame *cf;
 	struct sk_buff *skb;
+	u32 errctr = readl(priv->base + IFI_CANFD_ERROR_CTR);
 	const u32 errmask = IFI_CANFD_ERROR_CTR_OVERLOAD_FIRST |
 			    IFI_CANFD_ERROR_CTR_ACK_ERROR_FIRST |
 			    IFI_CANFD_ERROR_CTR_BIT0_ERROR_FIRST |
@@ -449,6 +463,11 @@ static int ifi_canfd_handle_state_change(struct net_device *ndev,
 
 	switch (new_state) {
 	case CAN_STATE_ERROR_ACTIVE:
+		/* error active state */
+		priv->can.can_stats.error_warning++;
+		priv->can.state = CAN_STATE_ERROR_ACTIVE;
+		break;
+	case CAN_STATE_ERROR_WARNING:
 		/* error warning state */
 		priv->can.can_stats.error_warning++;
 		priv->can.state = CAN_STATE_ERROR_WARNING;
@@ -477,7 +496,7 @@ static int ifi_canfd_handle_state_change(struct net_device *ndev,
 	ifi_canfd_get_berr_counter(ndev, &bec);
 
 	switch (new_state) {
-	case CAN_STATE_ERROR_ACTIVE:
+	case CAN_STATE_ERROR_WARNING:
 		/* error warning state */
 		cf->can_id |= CAN_ERR_CRTL;
 		cf->data[1] = (bec.txerr > bec.rxerr) ?
@@ -510,22 +529,21 @@ static int ifi_canfd_handle_state_change(struct net_device *ndev,
 	return 1;
 }
 
-static int ifi_canfd_handle_state_errors(struct net_device *ndev, u32 stcmd)
+static int ifi_canfd_handle_state_errors(struct net_device *ndev)
 {
 	struct ifi_canfd_priv *priv = netdev_priv(ndev);
+	u32 stcmd = readl(priv->base + IFI_CANFD_STCMD);
 	int work_done = 0;
-	u32 isr;
 
-	/*
-	 * The ErrWarn condition is a little special, since the bit is
-	 * located in the INTERRUPT register instead of STCMD register.
-	 */
-	isr = readl(priv->base + IFI_CANFD_INTERRUPT);
-	if ((isr & IFI_CANFD_INTERRUPT_ERROR_WARNING) &&
+	if ((stcmd & IFI_CANFD_STCMD_ERROR_ACTIVE) &&
+	    (priv->can.state != CAN_STATE_ERROR_ACTIVE)) {
+		netdev_dbg(ndev, "Error, entered active state\n");
+		work_done += ifi_canfd_handle_state_change(ndev,
+						CAN_STATE_ERROR_ACTIVE);
+	}
+
+	if ((stcmd & IFI_CANFD_STCMD_ERROR_WARNING) &&
 	    (priv->can.state != CAN_STATE_ERROR_WARNING)) {
-		/* Clear the interrupt */
-		writel(IFI_CANFD_INTERRUPT_ERROR_WARNING,
-		       priv->base + IFI_CANFD_INTERRUPT);
 		netdev_dbg(ndev, "Error, entered warning state\n");
 		work_done += ifi_canfd_handle_state_change(ndev,
 						CAN_STATE_ERROR_WARNING);
@@ -552,18 +570,11 @@ static int ifi_canfd_poll(struct napi_struct *napi, int quota)
 {
 	struct net_device *ndev = napi->dev;
 	struct ifi_canfd_priv *priv = netdev_priv(ndev);
-	const u32 stcmd_state_mask = IFI_CANFD_STCMD_ERROR_PASSIVE |
-				     IFI_CANFD_STCMD_BUSOFF;
-	int work_done = 0;
-
-	u32 stcmd = readl(priv->base + IFI_CANFD_STCMD);
 	u32 rxstcmd = readl(priv->base + IFI_CANFD_RXSTCMD);
-	u32 errctr = readl(priv->base + IFI_CANFD_ERROR_CTR);
+	int work_done = 0;
 
 	/* Handle bus state changes */
-	if ((stcmd & stcmd_state_mask) ||
-	    ((stcmd & IFI_CANFD_STCMD_ERROR_ACTIVE) == 0))
-		work_done += ifi_canfd_handle_state_errors(ndev, stcmd);
+	work_done += ifi_canfd_handle_state_errors(ndev);
 
 	/* Handle lost messages on RX */
 	if (rxstcmd & IFI_CANFD_RXSTCMD_OVERFLOW)
@@ -571,7 +582,7 @@ static int ifi_canfd_poll(struct napi_struct *napi, int quota)
 
 	/* Handle lec errors on the bus */
 	if (priv->can.ctrlmode & CAN_CTRLMODE_BERR_REPORTING)
-		work_done += ifi_canfd_handle_lec_err(ndev, errctr);
+		work_done += ifi_canfd_handle_lec_err(ndev);
 
 	/* Handle normal messages on RX */
 	if (!(rxstcmd & IFI_CANFD_RXSTCMD_EMPTY))
@@ -592,12 +603,13 @@ static irqreturn_t ifi_canfd_isr(int irq, void *dev_id)
 	struct net_device_stats *stats = &ndev->stats;
 	const u32 rx_irq_mask = IFI_CANFD_INTERRUPT_RXFIFO_NEMPTY |
 				IFI_CANFD_INTERRUPT_RXFIFO_NEMPTY_PER |
+				IFI_CANFD_INTERRUPT_ERROR_COUNTER |
+				IFI_CANFD_INTERRUPT_ERROR_STATE_CHG |
 				IFI_CANFD_INTERRUPT_ERROR_WARNING |
-				IFI_CANFD_INTERRUPT_ERROR_COUNTER;
+				IFI_CANFD_INTERRUPT_ERROR_BUSOFF;
 	const u32 tx_irq_mask = IFI_CANFD_INTERRUPT_TXFIFO_EMPTY |
 				IFI_CANFD_INTERRUPT_TXFIFO_REMOVE;
-	const u32 clr_irq_mask = ~((u32)(IFI_CANFD_INTERRUPT_SET_IRQ |
-					 IFI_CANFD_INTERRUPT_ERROR_WARNING));
+	const u32 clr_irq_mask = ~((u32)IFI_CANFD_INTERRUPT_SET_IRQ);
 	u32 isr;
 
 	isr = readl(priv->base + IFI_CANFD_INTERRUPT);
@@ -933,7 +945,7 @@ static int ifi_canfd_plat_probe(struct platform_device *pdev)
 	struct resource *res;
 	void __iomem *addr;
 	int irq, ret;
-	u32 id;
+	u32 id, rev;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	addr = devm_ioremap_resource(dev, res);
@@ -947,6 +959,13 @@ static int ifi_canfd_plat_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	rev = readl(addr + IFI_CANFD_VER) & IFI_CANFD_VER_REV_MASK;
+	if (rev < IFI_CANFD_VER_REV_MIN_SUPPORTED) {
+		dev_err(dev, "This block is too old (rev %i), minimum supported is rev %i\n",
+			rev, IFI_CANFD_VER_REV_MIN_SUPPORTED);
+		return -EINVAL;
+	}
+
 	ndev = alloc_candev(sizeof(*priv), 1);
 	if (!ndev)
 		return -ENOMEM;
diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c
index f4947a74b65f..b397a33f3d32 100644
--- a/drivers/net/can/m_can/m_can.c
+++ b/drivers/net/can/m_can/m_can.c
@@ -23,8 +23,10 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/iopoll.h>
 #include <linux/can/dev.h>
+#include <linux/pinctrl/consumer.h>
 
 /* napi related */
 #define M_CAN_NAPI_WEIGHT	64
@@ -126,6 +128,12 @@ enum m_can_mram_cfg {
 #define DBTP_DSJW_SHIFT		0
 #define DBTP_DSJW_MASK		(0xf << DBTP_DSJW_SHIFT)
 
+/* Transmitter Delay Compensation Register (TDCR) */
+#define TDCR_TDCO_SHIFT		8
+#define TDCR_TDCO_MASK		(0x7F << TDCR_TDCO_SHIFT)
+#define TDCR_TDCF_SHIFT		0
+#define TDCR_TDCF_MASK		(0x7F << TDCR_TDCF_SHIFT)
+
 /* Test Register (TEST) */
 #define TEST_LBCK		BIT(4)
 
@@ -246,7 +254,7 @@ enum m_can_mram_cfg {
 
 /* Rx FIFO 0/1 Configuration (RXF0C/RXF1C) */
 #define RXFC_FWM_SHIFT	24
-#define RXFC_FWM_MASK	(0x7f < RXFC_FWM_SHIFT)
+#define RXFC_FWM_MASK	(0x7f << RXFC_FWM_SHIFT)
 #define RXFC_FS_SHIFT	16
 #define RXFC_FS_MASK	(0x7f << RXFC_FS_SHIFT)
 
@@ -625,21 +633,16 @@ static int m_can_clk_start(struct m_can_priv *priv)
 {
 	int err;
 
-	err = clk_prepare_enable(priv->hclk);
+	err = pm_runtime_get_sync(priv->device);
 	if (err)
-		return err;
-
-	err = clk_prepare_enable(priv->cclk);
-	if (err)
-		clk_disable_unprepare(priv->hclk);
+		pm_runtime_put_noidle(priv->device);
 
 	return err;
 }
 
 static void m_can_clk_stop(struct m_can_priv *priv)
 {
-	clk_disable_unprepare(priv->cclk);
-	clk_disable_unprepare(priv->hclk);
+	pm_runtime_put_sync(priv->device);
 }
 
 static int m_can_get_berr_counter(const struct net_device *dev,
@@ -987,13 +990,47 @@ static int m_can_set_bittiming(struct net_device *dev)
 	m_can_write(priv, M_CAN_NBTP, reg_btp);
 
 	if (priv->can.ctrlmode & CAN_CTRLMODE_FD) {
+		reg_btp = 0;
 		brp = dbt->brp - 1;
 		sjw = dbt->sjw - 1;
 		tseg1 = dbt->prop_seg + dbt->phase_seg1 - 1;
 		tseg2 = dbt->phase_seg2 - 1;
-		reg_btp = (brp << DBTP_DBRP_SHIFT) | (sjw << DBTP_DSJW_SHIFT) |
-			(tseg1 << DBTP_DTSEG1_SHIFT) |
-			(tseg2 << DBTP_DTSEG2_SHIFT);
+
+		/* TDC is only needed for bitrates beyond 2.5 MBit/s.
+		 * This is mentioned in the "Bit Time Requirements for CAN FD"
+		 * paper presented at the International CAN Conference 2013
+		 */
+		if (dbt->bitrate > 2500000) {
+			u32 tdco, ssp;
+
+			/* Use the same value of secondary sampling point
+			 * as the data sampling point
+			 */
+			ssp = dbt->sample_point;
+
+			/* Equation based on Bosch's M_CAN User Manual's
+			 * Transmitter Delay Compensation Section
+			 */
+			tdco = (priv->can.clock.freq / 1000) *
+			       ssp / dbt->bitrate;
+
+			/* Max valid TDCO value is 127 */
+			if (tdco > 127) {
+				netdev_warn(dev, "TDCO value of %u is beyond maximum. Using maximum possible value\n",
+					    tdco);
+				tdco = 127;
+			}
+
+			reg_btp |= DBTP_TDC;
+			m_can_write(priv, M_CAN_TDCR,
+				    tdco << TDCR_TDCO_SHIFT);
+		}
+
+		reg_btp |= (brp << DBTP_DBRP_SHIFT) |
+			   (sjw << DBTP_DSJW_SHIFT) |
+			   (tseg1 << DBTP_DTSEG1_SHIFT) |
+			   (tseg2 << DBTP_DTSEG2_SHIFT);
+
 		m_can_write(priv, M_CAN_DBTP, reg_btp);
 	}
 
@@ -1143,11 +1180,6 @@ static int m_can_set_mode(struct net_device *dev, enum can_mode mode)
 	return 0;
 }
 
-static void free_m_can_dev(struct net_device *dev)
-{
-	free_candev(dev);
-}
-
 /* Checks core release number of M_CAN
  * returns 0 if an unsupported device is detected
  * else it returns the release and step coded as:
@@ -1207,31 +1239,20 @@ static bool m_can_niso_supported(const struct m_can_priv *priv)
 	return !niso_timeout;
 }
 
-static struct net_device *alloc_m_can_dev(struct platform_device *pdev,
-					  void __iomem *addr, u32 tx_fifo_size)
+static int m_can_dev_setup(struct platform_device *pdev, struct net_device *dev,
+			   void __iomem *addr)
 {
-	struct net_device *dev;
 	struct m_can_priv *priv;
 	int m_can_version;
-	unsigned int echo_buffer_count;
 
 	m_can_version = m_can_check_core_release(addr);
 	/* return if unsupported version */
 	if (!m_can_version) {
-		dev = NULL;
-		goto return_dev;
+		dev_err(&pdev->dev, "Unsupported version number: %2d",
+			m_can_version);
+		return -EINVAL;
 	}
 
-	/* If version < 3.1.x, then only one echo buffer is used */
-	echo_buffer_count = ((m_can_version == 30)
-				? 1U
-				: (unsigned int)tx_fifo_size);
-
-	dev = alloc_candev(sizeof(*priv), echo_buffer_count);
-	if (!dev) {
-		dev = NULL;
-		goto return_dev;
-	}
 	priv = netdev_priv(dev);
 	netif_napi_add(dev, &priv->napi, m_can_poll, M_CAN_NAPI_WEIGHT);
 
@@ -1273,16 +1294,12 @@ static struct net_device *alloc_m_can_dev(struct platform_device *pdev,
 						: 0);
 		break;
 	default:
-		/* Unsupported device: free candev */
-		free_m_can_dev(dev);
 		dev_err(&pdev->dev, "Unsupported version number: %2d",
 			priv->version);
-		dev = NULL;
-		break;
+		return -EINVAL;
 	}
 
-return_dev:
-	return dev;
+	return 0;
 }
 
 static int m_can_open(struct net_device *dev)
@@ -1574,37 +1591,26 @@ static int m_can_plat_probe(struct platform_device *pdev)
 		goto failed_ret;
 	}
 
-	/* Enable clocks. Necessary to read Core Release in order to determine
-	 * M_CAN version
-	 */
-	ret = clk_prepare_enable(hclk);
-	if (ret)
-		goto disable_hclk_ret;
-
-	ret = clk_prepare_enable(cclk);
-	if (ret)
-		goto disable_cclk_ret;
-
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "m_can");
 	addr = devm_ioremap_resource(&pdev->dev, res);
 	irq = platform_get_irq_byname(pdev, "int0");
 
 	if (IS_ERR(addr) || irq < 0) {
 		ret = -EINVAL;
-		goto disable_cclk_ret;
+		goto failed_ret;
 	}
 
 	/* message ram could be shared */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "message_ram");
 	if (!res) {
 		ret = -ENODEV;
-		goto disable_cclk_ret;
+		goto failed_ret;
 	}
 
 	mram_addr = devm_ioremap(&pdev->dev, res->start, resource_size(res));
 	if (!mram_addr) {
 		ret = -ENOMEM;
-		goto disable_cclk_ret;
+		goto failed_ret;
 	}
 
 	/* get message ram configuration */
@@ -1613,7 +1619,7 @@ static int m_can_plat_probe(struct platform_device *pdev)
 					 sizeof(mram_config_vals) / 4);
 	if (ret) {
 		dev_err(&pdev->dev, "Could not get Message RAM configuration.");
-		goto disable_cclk_ret;
+		goto failed_ret;
 	}
 
 	/* Get TX FIFO size
@@ -1622,11 +1628,12 @@ static int m_can_plat_probe(struct platform_device *pdev)
 	tx_fifo_size = mram_config_vals[7];
 
 	/* allocate the m_can device */
-	dev = alloc_m_can_dev(pdev, addr, tx_fifo_size);
+	dev = alloc_candev(sizeof(*priv), tx_fifo_size);
 	if (!dev) {
 		ret = -ENOMEM;
-		goto disable_cclk_ret;
+		goto failed_ret;
 	}
+
 	priv = netdev_priv(dev);
 	dev->irq = irq;
 	priv->device = &pdev->dev;
@@ -1640,30 +1647,42 @@ static int m_can_plat_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, dev);
 	SET_NETDEV_DEV(dev, &pdev->dev);
 
+	/* Enable clocks. Necessary to read Core Release in order to determine
+	 * M_CAN version
+	 */
+	pm_runtime_enable(&pdev->dev);
+	ret = m_can_clk_start(priv);
+	if (ret)
+		goto pm_runtime_fail;
+
+	ret = m_can_dev_setup(pdev, dev, addr);
+	if (ret)
+		goto clk_disable;
+
 	ret = register_m_can_dev(dev);
 	if (ret) {
 		dev_err(&pdev->dev, "registering %s failed (err=%d)\n",
 			KBUILD_MODNAME, ret);
-		goto failed_free_dev;
+		goto clk_disable;
 	}
 
 	devm_can_led_init(dev);
 
+	of_can_transceiver(dev);
+
 	dev_info(&pdev->dev, "%s device registered (irq=%d, version=%d)\n",
 		 KBUILD_MODNAME, dev->irq, priv->version);
 
 	/* Probe finished
 	 * Stop clocks. They will be reactivated once the M_CAN device is opened
 	 */
-
-	goto disable_cclk_ret;
-
-failed_free_dev:
-	free_m_can_dev(dev);
-disable_cclk_ret:
-	clk_disable_unprepare(cclk);
-disable_hclk_ret:
-	clk_disable_unprepare(hclk);
+clk_disable:
+	m_can_clk_stop(priv);
+pm_runtime_fail:
+	if (ret) {
+		pm_runtime_disable(&pdev->dev);
+		free_candev(dev);
+	}
 failed_ret:
 	return ret;
 }
@@ -1682,6 +1701,8 @@ static __maybe_unused int m_can_suspend(struct device *dev)
 		m_can_clk_stop(priv);
 	}
 
+	pinctrl_pm_select_sleep_state(dev);
+
 	priv->can.state = CAN_STATE_SLEEPING;
 
 	return 0;
@@ -1692,6 +1713,8 @@ static __maybe_unused int m_can_resume(struct device *dev)
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct m_can_priv *priv = netdev_priv(ndev);
 
+	pinctrl_pm_select_default_state(dev);
+
 	m_can_init_ram(priv);
 
 	priv->can.state = CAN_STATE_ERROR_ACTIVE;
@@ -1721,14 +1744,47 @@ static int m_can_plat_remove(struct platform_device *pdev)
 	struct net_device *dev = platform_get_drvdata(pdev);
 
 	unregister_m_can_dev(dev);
+
+	pm_runtime_disable(&pdev->dev);
+
 	platform_set_drvdata(pdev, NULL);
 
-	free_m_can_dev(dev);
+	free_candev(dev);
+
+	return 0;
+}
+
+static int __maybe_unused m_can_runtime_suspend(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct m_can_priv *priv = netdev_priv(ndev);
+
+	clk_disable_unprepare(priv->cclk);
+	clk_disable_unprepare(priv->hclk);
 
 	return 0;
 }
 
+static int __maybe_unused m_can_runtime_resume(struct device *dev)
+{
+	struct net_device *ndev = dev_get_drvdata(dev);
+	struct m_can_priv *priv = netdev_priv(ndev);
+	int err;
+
+	err = clk_prepare_enable(priv->hclk);
+	if (err)
+		return err;
+
+	err = clk_prepare_enable(priv->cclk);
+	if (err)
+		clk_disable_unprepare(priv->hclk);
+
+	return err;
+}
+
 static const struct dev_pm_ops m_can_pmops = {
+	SET_RUNTIME_PM_OPS(m_can_runtime_suspend,
+			   m_can_runtime_resume, NULL)
 	SET_SYSTEM_SLEEP_PM_OPS(m_can_suspend, m_can_resume)
 };
 
diff --git a/drivers/net/can/peak_canfd/peak_canfd.c b/drivers/net/can/peak_canfd/peak_canfd.c
index 55513411a82e..ed8561d4a90f 100644
--- a/drivers/net/can/peak_canfd/peak_canfd.c
+++ b/drivers/net/can/peak_canfd/peak_canfd.c
@@ -262,7 +262,6 @@ static int pucan_handle_can_rx(struct peak_canfd_priv *priv,
 
 		spin_lock_irqsave(&priv->echo_lock, flags);
 		can_get_echo_skb(priv->ndev, msg->client);
-		spin_unlock_irqrestore(&priv->echo_lock, flags);
 
 		/* count bytes of the echo instead of skb */
 		stats->tx_bytes += cf_len;
@@ -271,6 +270,7 @@ static int pucan_handle_can_rx(struct peak_canfd_priv *priv,
 		/* restart tx queue (a slot is free) */
 		netif_wake_queue(priv->ndev);
 
+		spin_unlock_irqrestore(&priv->echo_lock, flags);
 		return 0;
 	}
 
@@ -333,7 +333,6 @@ static int pucan_handle_status(struct peak_canfd_priv *priv,
 
 	/* this STATUS is the CNF of the RX_BARRIER: Tx path can be setup */
 	if (pucan_status_is_rx_barrier(msg)) {
-		unsigned long flags;
 
 		if (priv->enable_tx_path) {
 			int err = priv->enable_tx_path(priv);
@@ -342,16 +341,8 @@ static int pucan_handle_status(struct peak_canfd_priv *priv,
 				return err;
 		}
 
-		/* restart network queue only if echo skb array is free */
-		spin_lock_irqsave(&priv->echo_lock, flags);
-
-		if (!priv->can.echo_skb[priv->echo_idx]) {
-			spin_unlock_irqrestore(&priv->echo_lock, flags);
-
-			netif_wake_queue(ndev);
-		} else {
-			spin_unlock_irqrestore(&priv->echo_lock, flags);
-		}
+		/* start network queue (echo_skb array is empty) */
+		netif_start_queue(ndev);
 
 		return 0;
 	}
@@ -726,11 +717,6 @@ static netdev_tx_t peak_canfd_start_xmit(struct sk_buff *skb,
 	 */
 	should_stop_tx_queue = !!(priv->can.echo_skb[priv->echo_idx]);
 
-	spin_unlock_irqrestore(&priv->echo_lock, flags);
-
-	/* write the skb on the interface */
-	priv->write_tx_msg(priv, msg);
-
 	/* stop network tx queue if not enough room to save one more msg too */
 	if (priv->can.ctrlmode & CAN_CTRLMODE_FD)
 		should_stop_tx_queue |= (room_left <
@@ -742,6 +728,11 @@ static netdev_tx_t peak_canfd_start_xmit(struct sk_buff *skb,
 	if (should_stop_tx_queue)
 		netif_stop_queue(ndev);
 
+	spin_unlock_irqrestore(&priv->echo_lock, flags);
+
+	/* write the skb on the interface */
+	priv->write_tx_msg(priv, msg);
+
 	return NETDEV_TX_OK;
 }
 
diff --git a/drivers/net/can/peak_canfd/peak_pciefd_main.c b/drivers/net/can/peak_canfd/peak_pciefd_main.c
index 788c3464a3b0..3c51a884db87 100644
--- a/drivers/net/can/peak_canfd/peak_pciefd_main.c
+++ b/drivers/net/can/peak_canfd/peak_pciefd_main.c
@@ -349,8 +349,12 @@ static irqreturn_t pciefd_irq_handler(int irq, void *arg)
 		priv->tx_pages_free++;
 		spin_unlock_irqrestore(&priv->tx_lock, flags);
 
-		/* wake producer up */
-		netif_wake_queue(priv->ucan.ndev);
+		/* wake producer up (only if enough room in echo_skb array) */
+		spin_lock_irqsave(&priv->ucan.echo_lock, flags);
+		if (!priv->ucan.can.echo_skb[priv->ucan.echo_idx])
+			netif_wake_queue(priv->ucan.ndev);
+
+		spin_unlock_irqrestore(&priv->ucan.echo_lock, flags);
 	}
 
 	/* re-enable Rx DMA transfer for this CAN */
diff --git a/drivers/net/can/rx-offload.c b/drivers/net/can/rx-offload.c
index f394f77d7528..d94dae216820 100644
--- a/drivers/net/can/rx-offload.c
+++ b/drivers/net/can/rx-offload.c
@@ -256,7 +256,7 @@ int can_rx_offload_add_timestamp(struct net_device *dev, struct can_rx_offload *
 		weight = offload->mb_first - offload->mb_last;
 	}
 
-	return can_rx_offload_init_queue(dev, offload, weight);;
+	return can_rx_offload_init_queue(dev, offload, weight);
 }
 EXPORT_SYMBOL_GPL(can_rx_offload_add_timestamp);
 
diff --git a/drivers/net/can/slcan.c b/drivers/net/can/slcan.c
index 5d067c1b987f..89d60d8e467c 100644
--- a/drivers/net/can/slcan.c
+++ b/drivers/net/can/slcan.c
@@ -508,7 +508,7 @@ static void slc_sync(void)
 }
 
 /* Find a free SLCAN channel, and link in this `tty' line. */
-static struct slcan *slc_alloc(dev_t line)
+static struct slcan *slc_alloc(void)
 {
 	int i;
 	char name[IFNAMSIZ];
@@ -583,7 +583,7 @@ static int slcan_open(struct tty_struct *tty)
 
 	/* OK.  Find a free SLCAN channel to use. */
 	err = -ENFILE;
-	sl = slc_alloc(tty_devnum(tty));
+	sl = slc_alloc();
 	if (sl == NULL)
 		goto err_exit;
 
diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c
index f3f05fea8e1f..98d118b3aaf4 100644
--- a/drivers/net/can/spi/mcp251x.c
+++ b/drivers/net/can/spi/mcp251x.c
@@ -612,8 +612,7 @@ static int mcp251x_do_set_bittiming(struct net_device *net)
 	return 0;
 }
 
-static int mcp251x_setup(struct net_device *net, struct mcp251x_priv *priv,
-			 struct spi_device *spi)
+static int mcp251x_setup(struct net_device *net, struct spi_device *spi)
 {
 	mcp251x_do_set_bittiming(net);
 
@@ -775,7 +774,7 @@ static void mcp251x_restart_work_handler(struct work_struct *ws)
 	mutex_lock(&priv->mcp_lock);
 	if (priv->after_suspend) {
 		mcp251x_hw_reset(spi);
-		mcp251x_setup(net, priv, spi);
+		mcp251x_setup(net, spi);
 		if (priv->after_suspend & AFTER_SUSPEND_RESTART) {
 			mcp251x_set_normal_mode(spi);
 		} else if (priv->after_suspend & AFTER_SUSPEND_UP) {
@@ -971,7 +970,7 @@ static int mcp251x_open(struct net_device *net)
 		mcp251x_open_clean(net);
 		goto open_unlock;
 	}
-	ret = mcp251x_setup(net, priv, spi);
+	ret = mcp251x_setup(net, spi);
 	if (ret) {
 		mcp251x_open_clean(net);
 		goto open_unlock;
diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c
index 8bf80ad9dc44..17c21ad3b95e 100644
--- a/drivers/net/can/usb/gs_usb.c
+++ b/drivers/net/can/usb/gs_usb.c
@@ -243,7 +243,7 @@ static struct gs_tx_context *gs_get_tx_context(struct gs_can *dev,
 	return NULL;
 }
 
-static int gs_cmd_reset(struct gs_usb *gsusb, struct gs_can *gsdev)
+static int gs_cmd_reset(struct gs_can *gsdev)
 {
 	struct gs_device_mode *dm;
 	struct usb_interface *intf = gsdev->iface;
@@ -709,7 +709,7 @@ static int gs_can_close(struct net_device *netdev)
 	atomic_set(&dev->active_tx_urbs, 0);
 
 	/* reset the device */
-	rc = gs_cmd_reset(parent, dev);
+	rc = gs_cmd_reset(dev);
 	if (rc < 0)
 		netdev_warn(netdev, "Couldn't shutdown device (err=%d)", rc);
 
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c
index 25a9b79cc42d..f530a80f5051 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb.c
@@ -408,7 +408,6 @@ static int pcan_usb_decode_error(struct pcan_usb_msg_context *mc, u8 n,
 {
 	struct sk_buff *skb;
 	struct can_frame *cf;
-	struct timeval tv;
 	enum can_state new_state;
 
 	/* ignore this error until 1st ts received */
@@ -525,8 +524,8 @@ static int pcan_usb_decode_error(struct pcan_usb_msg_context *mc, u8 n,
 	if (status_len & PCAN_USB_STATUSLEN_TIMESTAMP) {
 		struct skb_shared_hwtstamps *hwts = skb_hwtstamps(skb);
 
-		peak_usb_get_ts_tv(&mc->pdev->time_ref, mc->ts16, &tv);
-		hwts->hwtstamp = timeval_to_ktime(tv);
+		peak_usb_get_ts_time(&mc->pdev->time_ref, mc->ts16,
+				     &hwts->hwtstamp);
 	}
 
 	mc->netdev->stats.rx_packets++;
@@ -610,7 +609,6 @@ static int pcan_usb_decode_data(struct pcan_usb_msg_context *mc, u8 status_len)
 	u8 rec_len = status_len & PCAN_USB_STATUSLEN_DLC;
 	struct sk_buff *skb;
 	struct can_frame *cf;
-	struct timeval tv;
 	struct skb_shared_hwtstamps *hwts;
 
 	skb = alloc_can_skb(mc->netdev, &cf);
@@ -658,9 +656,8 @@ static int pcan_usb_decode_data(struct pcan_usb_msg_context *mc, u8 status_len)
 	}
 
 	/* convert timestamp into kernel time */
-	peak_usb_get_ts_tv(&mc->pdev->time_ref, mc->ts16, &tv);
 	hwts = skb_hwtstamps(skb);
-	hwts->hwtstamp = timeval_to_ktime(tv);
+	peak_usb_get_ts_time(&mc->pdev->time_ref, mc->ts16, &hwts->hwtstamp);
 
 	/* update statistics */
 	mc->netdev->stats.rx_packets++;
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
index 1ca76e03e965..50e911428638 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
@@ -80,21 +80,6 @@ void peak_usb_init_time_ref(struct peak_time_ref *time_ref,
 	}
 }
 
-static void peak_usb_add_us(struct timeval *tv, u32 delta_us)
-{
-	/* number of s. to add to final time */
-	u32 delta_s = delta_us / 1000000;
-
-	delta_us -= delta_s * 1000000;
-
-	tv->tv_usec += delta_us;
-	if (tv->tv_usec >= 1000000) {
-		tv->tv_usec -= 1000000;
-		delta_s++;
-	}
-	tv->tv_sec += delta_s;
-}
-
 /*
  * sometimes, another now may be  more recent than current one...
  */
@@ -103,7 +88,7 @@ void peak_usb_update_ts_now(struct peak_time_ref *time_ref, u32 ts_now)
 	time_ref->ts_dev_2 = ts_now;
 
 	/* should wait at least two passes before computing */
-	if (time_ref->tv_host.tv_sec > 0) {
+	if (ktime_to_ns(time_ref->tv_host) > 0) {
 		u32 delta_ts = time_ref->ts_dev_2 - time_ref->ts_dev_1;
 
 		if (time_ref->ts_dev_2 < time_ref->ts_dev_1)
@@ -118,26 +103,26 @@ void peak_usb_update_ts_now(struct peak_time_ref *time_ref, u32 ts_now)
  */
 void peak_usb_set_ts_now(struct peak_time_ref *time_ref, u32 ts_now)
 {
-	if (time_ref->tv_host_0.tv_sec == 0) {
+	if (ktime_to_ns(time_ref->tv_host_0) == 0) {
 		/* use monotonic clock to correctly compute further deltas */
-		time_ref->tv_host_0 = ktime_to_timeval(ktime_get());
-		time_ref->tv_host.tv_sec = 0;
+		time_ref->tv_host_0 = ktime_get();
+		time_ref->tv_host = ktime_set(0, 0);
 	} else {
 		/*
-		 * delta_us should not be >= 2^32 => delta_s should be < 4294
+		 * delta_us should not be >= 2^32 => delta should be < 4294s
 		 * handle 32-bits wrapping here: if count of s. reaches 4200,
 		 * reset counters and change time base
 		 */
-		if (time_ref->tv_host.tv_sec != 0) {
-			u32 delta_s = time_ref->tv_host.tv_sec
-						- time_ref->tv_host_0.tv_sec;
-			if (delta_s > 4200) {
+		if (ktime_to_ns(time_ref->tv_host)) {
+			ktime_t delta = ktime_sub(time_ref->tv_host,
+						  time_ref->tv_host_0);
+			if (ktime_to_ns(delta) > (4200ull * NSEC_PER_SEC)) {
 				time_ref->tv_host_0 = time_ref->tv_host;
 				time_ref->ts_total = 0;
 			}
 		}
 
-		time_ref->tv_host = ktime_to_timeval(ktime_get());
+		time_ref->tv_host = ktime_get();
 		time_ref->tick_count++;
 	}
 
@@ -146,13 +131,12 @@ void peak_usb_set_ts_now(struct peak_time_ref *time_ref, u32 ts_now)
 }
 
 /*
- * compute timeval according to current ts and time_ref data
+ * compute time according to current ts and time_ref data
  */
-void peak_usb_get_ts_tv(struct peak_time_ref *time_ref, u32 ts,
-			struct timeval *tv)
+void peak_usb_get_ts_time(struct peak_time_ref *time_ref, u32 ts, ktime_t *time)
 {
-	/* protect from getting timeval before setting now */
-	if (time_ref->tv_host.tv_sec > 0) {
+	/* protect from getting time before setting now */
+	if (ktime_to_ns(time_ref->tv_host)) {
 		u64 delta_us;
 
 		delta_us = ts - time_ref->ts_dev_2;
@@ -164,10 +148,9 @@ void peak_usb_get_ts_tv(struct peak_time_ref *time_ref, u32 ts,
 		delta_us *= time_ref->adapter->us_per_ts_scale;
 		delta_us >>= time_ref->adapter->us_per_ts_shift;
 
-		*tv = time_ref->tv_host_0;
-		peak_usb_add_us(tv, (u32)delta_us);
+		*time = ktime_add_us(time_ref->tv_host_0, delta_us);
 	} else {
-		*tv = ktime_to_timeval(ktime_get());
+		*time = ktime_get();
 	}
 }
 
@@ -175,13 +158,11 @@ void peak_usb_get_ts_tv(struct peak_time_ref *time_ref, u32 ts,
  * post received skb after having set any hw timestamp
  */
 int peak_usb_netif_rx(struct sk_buff *skb,
-		      struct peak_time_ref *time_ref, u32 ts_low, u32 ts_high)
+		      struct peak_time_ref *time_ref, u32 ts_low)
 {
 	struct skb_shared_hwtstamps *hwts = skb_hwtstamps(skb);
-	struct timeval tv;
 
-	peak_usb_get_ts_tv(time_ref, ts_low, &tv);
-	hwts->hwtstamp = timeval_to_ktime(tv);
+	peak_usb_get_ts_time(time_ref, ts_low, &hwts->hwtstamp);
 
 	return netif_rx(skb);
 }
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.h b/drivers/net/can/usb/peak_usb/pcan_usb_core.h
index c01316cac354..fb23489e1cb8 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_core.h
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.h
@@ -96,7 +96,7 @@ extern const struct peak_usb_adapter pcan_usb_pro_fd;
 extern const struct peak_usb_adapter pcan_usb_x6;
 
 struct peak_time_ref {
-	struct timeval tv_host_0, tv_host;
+	ktime_t tv_host_0, tv_host;
 	u32 ts_dev_1, ts_dev_2;
 	u64 ts_total;
 	u32 tick_count;
@@ -151,10 +151,9 @@ void peak_usb_init_time_ref(struct peak_time_ref *time_ref,
 			    const struct peak_usb_adapter *adapter);
 void peak_usb_update_ts_now(struct peak_time_ref *time_ref, u32 ts_now);
 void peak_usb_set_ts_now(struct peak_time_ref *time_ref, u32 ts_now);
-void peak_usb_get_ts_tv(struct peak_time_ref *time_ref, u32 ts,
-			struct timeval *tv);
+void peak_usb_get_ts_time(struct peak_time_ref *time_ref, u32 ts, ktime_t *tv);
 int peak_usb_netif_rx(struct sk_buff *skb,
-		      struct peak_time_ref *time_ref, u32 ts_low, u32 ts_high);
+		      struct peak_time_ref *time_ref, u32 ts_low);
 void peak_usb_async_complete(struct urb *urb);
 void peak_usb_restart_complete(struct peak_usb_device *dev);
 
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
index 53d6bb045e9e..dd161c5eea8e 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c
@@ -514,8 +514,7 @@ static int pcan_usb_fd_decode_canmsg(struct pcan_usb_fd_if *usb_if,
 	else
 		memcpy(cfd->data, rm->d, cfd->len);
 
-	peak_usb_netif_rx(skb, &usb_if->time_ref,
-			  le32_to_cpu(rm->ts_low), le32_to_cpu(rm->ts_high));
+	peak_usb_netif_rx(skb, &usb_if->time_ref, le32_to_cpu(rm->ts_low));
 
 	netdev->stats.rx_packets++;
 	netdev->stats.rx_bytes += cfd->len;
@@ -575,8 +574,7 @@ static int pcan_usb_fd_decode_status(struct pcan_usb_fd_if *usb_if,
 	if (!skb)
 		return -ENOMEM;
 
-	peak_usb_netif_rx(skb, &usb_if->time_ref,
-			  le32_to_cpu(sm->ts_low), le32_to_cpu(sm->ts_high));
+	peak_usb_netif_rx(skb, &usb_if->time_ref, le32_to_cpu(sm->ts_low));
 
 	netdev->stats.rx_packets++;
 	netdev->stats.rx_bytes += cf->can_dlc;
@@ -618,8 +616,7 @@ static int pcan_usb_fd_decode_overrun(struct pcan_usb_fd_if *usb_if,
 	cf->can_id |= CAN_ERR_CRTL;
 	cf->data[1] |= CAN_ERR_CRTL_RX_OVERFLOW;
 
-	peak_usb_netif_rx(skb, &usb_if->time_ref,
-			  le32_to_cpu(ov->ts_low), le32_to_cpu(ov->ts_high));
+	peak_usb_netif_rx(skb, &usb_if->time_ref, le32_to_cpu(ov->ts_low));
 
 	netdev->stats.rx_over_errors++;
 	netdev->stats.rx_errors++;
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c
index bbdd6058cd2f..0105fbfea273 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c
@@ -531,7 +531,6 @@ static int pcan_usb_pro_handle_canmsg(struct pcan_usb_pro_interface *usb_if,
 	struct net_device *netdev = dev->netdev;
 	struct can_frame *can_frame;
 	struct sk_buff *skb;
-	struct timeval tv;
 	struct skb_shared_hwtstamps *hwts;
 
 	skb = alloc_can_skb(netdev, &can_frame);
@@ -549,9 +548,9 @@ static int pcan_usb_pro_handle_canmsg(struct pcan_usb_pro_interface *usb_if,
 	else
 		memcpy(can_frame->data, rx->data, can_frame->can_dlc);
 
-	peak_usb_get_ts_tv(&usb_if->time_ref, le32_to_cpu(rx->ts32), &tv);
 	hwts = skb_hwtstamps(skb);
-	hwts->hwtstamp = timeval_to_ktime(tv);
+	peak_usb_get_ts_time(&usb_if->time_ref, le32_to_cpu(rx->ts32),
+			     &hwts->hwtstamp);
 
 	netdev->stats.rx_packets++;
 	netdev->stats.rx_bytes += can_frame->can_dlc;
@@ -571,7 +570,6 @@ static int pcan_usb_pro_handle_error(struct pcan_usb_pro_interface *usb_if,
 	enum can_state new_state = CAN_STATE_ERROR_ACTIVE;
 	u8 err_mask = 0;
 	struct sk_buff *skb;
-	struct timeval tv;
 	struct skb_shared_hwtstamps *hwts;
 
 	/* nothing should be sent while in BUS_OFF state */
@@ -667,9 +665,8 @@ static int pcan_usb_pro_handle_error(struct pcan_usb_pro_interface *usb_if,
 
 	dev->can.state = new_state;
 
-	peak_usb_get_ts_tv(&usb_if->time_ref, le32_to_cpu(er->ts32), &tv);
 	hwts = skb_hwtstamps(skb);
-	hwts->hwtstamp = timeval_to_ktime(tv);
+	peak_usb_get_ts_time(&usb_if->time_ref, le32_to_cpu(er->ts32), &hwts->hwtstamp);
 	netdev->stats.rx_packets++;
 	netdev->stats.rx_bytes += can_frame->can_dlc;
 	netif_rx(skb);
diff --git a/drivers/net/can/vcan.c b/drivers/net/can/vcan.c
index a8cb33264ff1..c2b04f505e16 100644
--- a/drivers/net/can/vcan.c
+++ b/drivers/net/can/vcan.c
@@ -61,7 +61,7 @@ MODULE_ALIAS_RTNL_LINK(DRV_NAME);
 /*
  * CAN test feature:
  * Enable the echo on driver level for testing the CAN core echo modes.
- * See Documentation/networking/can.txt for details.
+ * See Documentation/networking/can.rst for details.
  */
 
 static bool echo; /* echo testing. Default: 0 (Off) */
diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c
index b4c4a2c76437..ed6828821fbd 100644
--- a/drivers/net/can/vxcan.c
+++ b/drivers/net/can/vxcan.c
@@ -227,10 +227,8 @@ static int vxcan_newlink(struct net *net, struct net_device *dev,
 	netif_carrier_off(peer);
 
 	err = rtnl_configure_link(peer, ifmp);
-	if (err < 0) {
-		unregister_netdevice(peer);
-		return err;
-	}
+	if (err < 0)
+		goto unregister_network_device;
 
 	/* register first device */
 	if (tb[IFLA_IFNAME])
@@ -239,10 +237,8 @@ static int vxcan_newlink(struct net *net, struct net_device *dev,
 		snprintf(dev->name, IFNAMSIZ, DRV_NAME "%%d");
 
 	err = register_netdevice(dev);
-	if (err < 0) {
-		unregister_netdevice(peer);
-		return err;
-	}
+	if (err < 0)
+		goto unregister_network_device;
 
 	netif_carrier_off(dev);
 
@@ -254,6 +250,10 @@ static int vxcan_newlink(struct net *net, struct net_device *dev,
 	rcu_assign_pointer(priv->peer, dev);
 
 	return 0;
+
+unregister_network_device:
+	unregister_netdevice(peer);
+	return err;
 }
 
 static void vxcan_dellink(struct net_device *dev, struct list_head *head)
diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
index 83a9bc892a3b..2b81b97e994f 100644
--- a/drivers/net/dsa/Kconfig
+++ b/drivers/net/dsa/Kconfig
@@ -33,7 +33,7 @@ config NET_DSA_MT7530
 
 config NET_DSA_MV88E6060
 	tristate "Marvell 88E6060 ethernet switch chip support"
-	depends on NET_DSA
+	depends on NET_DSA && NET_DSA_LEGACY
 	select NET_DSA_TAG_TRAILER
 	---help---
 	  This enables support for the Marvell 88E6060 ethernet switch
diff --git a/drivers/net/dsa/Makefile b/drivers/net/dsa/Makefile
index d040aeb45172..15c2a831edf1 100644
--- a/drivers/net/dsa/Makefile
+++ b/drivers/net/dsa/Makefile
@@ -1,7 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_NET_DSA_BCM_SF2)	+= bcm-sf2.o
 bcm-sf2-objs			:= bcm_sf2.o bcm_sf2_cfp.o
-obj-$(CONFIG_NET_DSA_LOOP)	+= dsa_loop.o dsa_loop_bdinfo.o
+obj-$(CONFIG_NET_DSA_LOOP)	+= dsa_loop.o
+ifdef CONFIG_NET_DSA_LOOP
+obj-$(CONFIG_FIXED_PHY)		+= dsa_loop_bdinfo.o
+endif
 obj-$(CONFIG_NET_DSA_MT7530)	+= mt7530.o
 obj-$(CONFIG_NET_DSA_MV88E6060) += mv88e6060.o
 obj-$(CONFIG_NET_DSA_QCA8K)	+= qca8k.o
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 4498ab897d94..63e02a54d537 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -814,8 +814,8 @@ void b53_get_strings(struct dsa_switch *ds, int port, uint8_t *data)
 	unsigned int i;
 
 	for (i = 0; i < mib_size; i++)
-		memcpy(data + i * ETH_GSTRING_LEN,
-		       mibs[i].name, ETH_GSTRING_LEN);
+		strlcpy(data + i * ETH_GSTRING_LEN,
+			mibs[i].name, ETH_GSTRING_LEN);
 }
 EXPORT_SYMBOL(b53_get_strings);
 
@@ -1029,8 +1029,7 @@ int b53_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering)
 EXPORT_SYMBOL(b53_vlan_filtering);
 
 int b53_vlan_prepare(struct dsa_switch *ds, int port,
-		     const struct switchdev_obj_port_vlan *vlan,
-		     struct switchdev_trans *trans)
+		     const struct switchdev_obj_port_vlan *vlan)
 {
 	struct b53_device *dev = ds->priv;
 
@@ -1047,8 +1046,7 @@ int b53_vlan_prepare(struct dsa_switch *ds, int port,
 EXPORT_SYMBOL(b53_vlan_prepare);
 
 void b53_vlan_add(struct dsa_switch *ds, int port,
-		  const struct switchdev_obj_port_vlan *vlan,
-		  struct switchdev_trans *trans)
+		  const struct switchdev_obj_port_vlan *vlan)
 {
 	struct b53_device *dev = ds->priv;
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
@@ -1495,8 +1493,7 @@ static bool b53_can_enable_brcm_tags(struct dsa_switch *ds, int port)
 	return false;
 }
 
-static enum dsa_tag_protocol b53_get_tag_protocol(struct dsa_switch *ds,
-						  int port)
+enum dsa_tag_protocol b53_get_tag_protocol(struct dsa_switch *ds, int port)
 {
 	struct b53_device *dev = ds->priv;
 
@@ -1517,6 +1514,7 @@ static enum dsa_tag_protocol b53_get_tag_protocol(struct dsa_switch *ds,
 
 	return DSA_TAG_PROTO_BRCM;
 }
+EXPORT_SYMBOL(b53_get_tag_protocol);
 
 int b53_mirror_add(struct dsa_switch *ds, int port,
 		   struct dsa_mall_mirror_tc_entry *mirror, bool ingress)
diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h
index daaaa1ecb996..d954cf36ecd8 100644
--- a/drivers/net/dsa/b53/b53_priv.h
+++ b/drivers/net/dsa/b53/b53_priv.h
@@ -295,11 +295,9 @@ void b53_br_set_stp_state(struct dsa_switch *ds, int port, u8 state);
 void b53_br_fast_age(struct dsa_switch *ds, int port);
 int b53_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering);
 int b53_vlan_prepare(struct dsa_switch *ds, int port,
-		     const struct switchdev_obj_port_vlan *vlan,
-		     struct switchdev_trans *trans);
+		     const struct switchdev_obj_port_vlan *vlan);
 void b53_vlan_add(struct dsa_switch *ds, int port,
-		  const struct switchdev_obj_port_vlan *vlan,
-		  struct switchdev_trans *trans);
+		  const struct switchdev_obj_port_vlan *vlan);
 int b53_vlan_del(struct dsa_switch *ds, int port,
 		 const struct switchdev_obj_port_vlan *vlan);
 int b53_fdb_add(struct dsa_switch *ds, int port,
@@ -310,6 +308,7 @@ int b53_fdb_dump(struct dsa_switch *ds, int port,
 		 dsa_fdb_dump_cb_t *cb, void *data);
 int b53_mirror_add(struct dsa_switch *ds, int port,
 		   struct dsa_mall_mirror_tc_entry *mirror, bool ingress);
+enum dsa_tag_protocol b53_get_tag_protocol(struct dsa_switch *ds, int port);
 void b53_mirror_del(struct dsa_switch *ds, int port,
 		    struct dsa_mall_mirror_tc_entry *mirror);
 int b53_enable_port(struct dsa_switch *ds, int port, struct phy_device *phy);
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index b62d47210db8..0378eded31f2 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -34,12 +34,6 @@
 #include "b53/b53_priv.h"
 #include "b53/b53_regs.h"
 
-static enum dsa_tag_protocol bcm_sf2_sw_get_tag_protocol(struct dsa_switch *ds,
-							 int port)
-{
-	return DSA_TAG_PROTO_BRCM;
-}
-
 static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
@@ -860,7 +854,7 @@ static const struct b53_io_ops bcm_sf2_io_ops = {
 };
 
 static const struct dsa_switch_ops bcm_sf2_ops = {
-	.get_tag_protocol	= bcm_sf2_sw_get_tag_protocol,
+	.get_tag_protocol	= b53_get_tag_protocol,
 	.setup			= bcm_sf2_sw_setup,
 	.get_strings		= b53_get_strings,
 	.get_ethtool_stats	= b53_get_ethtool_stats,
@@ -954,6 +948,9 @@ static const struct of_device_id bcm_sf2_of_match[] = {
 	{ .compatible = "brcm,bcm7278-switch-v4.0",
 	  .data = &bcm_sf2_7278_data
 	},
+	{ .compatible = "brcm,bcm7278-switch-v4.8",
+	  .data = &bcm_sf2_7278_data
+	},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, bcm_sf2_of_match);
diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c
index bb71d3d6f65b..7aa84ee4e771 100644
--- a/drivers/net/dsa/dsa_loop.c
+++ b/drivers/net/dsa/dsa_loop.c
@@ -174,9 +174,9 @@ static int dsa_loop_port_vlan_filtering(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int dsa_loop_port_vlan_prepare(struct dsa_switch *ds, int port,
-				      const struct switchdev_obj_port_vlan *vlan,
-				      struct switchdev_trans *trans)
+static int
+dsa_loop_port_vlan_prepare(struct dsa_switch *ds, int port,
+			   const struct switchdev_obj_port_vlan *vlan)
 {
 	struct dsa_loop_priv *ps = ds->priv;
 	struct mii_bus *bus = ps->bus;
@@ -193,8 +193,7 @@ static int dsa_loop_port_vlan_prepare(struct dsa_switch *ds, int port,
 }
 
 static void dsa_loop_port_vlan_add(struct dsa_switch *ds, int port,
-				   const struct switchdev_obj_port_vlan *vlan,
-				   struct switchdev_trans *trans)
+				   const struct switchdev_obj_port_vlan *vlan)
 {
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
 	bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c
index b24566bb74d2..6171c0853ff1 100644
--- a/drivers/net/dsa/lan9303-core.c
+++ b/drivers/net/dsa/lan9303-core.c
@@ -249,6 +249,28 @@ static int lan9303_read(struct regmap *regmap, unsigned int offset, u32 *reg)
 	return -EIO;
 }
 
+static int lan9303_read_wait(struct lan9303 *chip, int offset, u32 mask)
+{
+	int i;
+
+	for (i = 0; i < 25; i++) {
+		u32 reg;
+		int ret;
+
+		ret = lan9303_read(chip->regmap, offset, &reg);
+		if (ret) {
+			dev_err(chip->dev, "%s failed to read offset %d: %d\n",
+				__func__, offset, ret);
+			return ret;
+		}
+		if (!(reg & mask))
+			return 0;
+		usleep_range(1000, 2000);
+	}
+
+	return -ETIMEDOUT;
+}
+
 static int lan9303_virt_phy_reg_read(struct lan9303 *chip, int regnum)
 {
 	int ret;
@@ -274,22 +296,8 @@ static int lan9303_virt_phy_reg_write(struct lan9303 *chip, int regnum, u16 val)
 
 static int lan9303_indirect_phy_wait_for_completion(struct lan9303 *chip)
 {
-	int ret, i;
-	u32 reg;
-
-	for (i = 0; i < 25; i++) {
-		ret = lan9303_read(chip->regmap, LAN9303_PMI_ACCESS, &reg);
-		if (ret) {
-			dev_err(chip->dev,
-				"Failed to read pmi access status: %d\n", ret);
-			return ret;
-		}
-		if (!(reg & LAN9303_PMI_ACCESS_MII_BUSY))
-			return 0;
-		usleep_range(1000, 2000);
-	}
-
-	return -EIO;
+	return lan9303_read_wait(chip, LAN9303_PMI_ACCESS,
+				 LAN9303_PMI_ACCESS_MII_BUSY);
 }
 
 static int lan9303_indirect_phy_read(struct lan9303 *chip, int addr, int regnum)
@@ -366,22 +374,8 @@ EXPORT_SYMBOL_GPL(lan9303_indirect_phy_ops);
 
 static int lan9303_switch_wait_for_completion(struct lan9303 *chip)
 {
-	int ret, i;
-	u32 reg;
-
-	for (i = 0; i < 25; i++) {
-		ret = lan9303_read(chip->regmap, LAN9303_SWITCH_CSR_CMD, &reg);
-		if (ret) {
-			dev_err(chip->dev,
-				"Failed to read csr command status: %d\n", ret);
-			return ret;
-		}
-		if (!(reg & LAN9303_SWITCH_CSR_CMD_BUSY))
-			return 0;
-		usleep_range(1000, 2000);
-	}
-
-	return -EIO;
+	return lan9303_read_wait(chip, LAN9303_SWITCH_CSR_CMD,
+				 LAN9303_SWITCH_CSR_CMD_BUSY);
 }
 
 static int lan9303_write_switch_reg(struct lan9303 *chip, u16 regnum, u32 val)
@@ -485,7 +479,8 @@ static int lan9303_detect_phy_setup(struct lan9303 *chip)
 {
 	int reg;
 
-	/* depending on the 'phy_addr_sel_strap' setting, the three phys are
+	/* Calculate chip->phy_addr_base:
+	 * Depending on the 'phy_addr_sel_strap' setting, the three phys are
 	 * using IDs 0-1-2 or IDs 1-2-3. We cannot read back the
 	 * 'phy_addr_sel_strap' setting directly, so we need a test, which
 	 * configuration is active:
@@ -500,13 +495,10 @@ static int lan9303_detect_phy_setup(struct lan9303 *chip)
 		return reg;
 	}
 
-	if ((reg != 0) && (reg != 0xffff))
-		chip->phy_addr_sel_strap = 1;
-	else
-		chip->phy_addr_sel_strap = 0;
+	chip->phy_addr_base = reg != 0 && reg != 0xffff;
 
 	dev_dbg(chip->dev, "Phy setup '%s' detected\n",
-		chip->phy_addr_sel_strap ? "1-2-3" : "0-1-2");
+		chip->phy_addr_base ? "1-2-3" : "0-1-2");
 
 	return 0;
 }
@@ -546,20 +538,19 @@ lan9303_alr_cache_find_mac(struct lan9303 *chip, const u8 *mac_addr)
 	return NULL;
 }
 
-/* Wait a while until mask & reg == value. Otherwise return timeout. */
-static int lan9303_csr_reg_wait(struct lan9303 *chip, int regno,
-				int mask, char value)
+static int lan9303_csr_reg_wait(struct lan9303 *chip, int regno, u32 mask)
 {
 	int i;
 
-	for (i = 0; i < 0x1000; i++) {
+	for (i = 0; i < 25; i++) {
 		u32 reg;
 
 		lan9303_read_switch_reg(chip, regno, &reg);
-		if ((reg & mask) == value)
+		if (!(reg & mask))
 			return 0;
 		usleep_range(1000, 2000);
 	}
+
 	return -ETIMEDOUT;
 }
 
@@ -569,8 +560,7 @@ static int lan9303_alr_make_entry_raw(struct lan9303 *chip, u32 dat0, u32 dat1)
 	lan9303_write_switch_reg(chip, LAN9303_SWE_ALR_WR_DAT_1, dat1);
 	lan9303_write_switch_reg(chip, LAN9303_SWE_ALR_CMD,
 				 LAN9303_ALR_CMD_MAKE_ENTRY);
-	lan9303_csr_reg_wait(chip, LAN9303_SWE_ALR_CMD_STS, ALR_STS_MAKE_PEND,
-			     0);
+	lan9303_csr_reg_wait(chip, LAN9303_SWE_ALR_CMD_STS, ALR_STS_MAKE_PEND);
 	lan9303_write_switch_reg(chip, LAN9303_SWE_ALR_CMD, 0);
 
 	return 0;
@@ -583,6 +573,7 @@ static void lan9303_alr_loop(struct lan9303 *chip, alr_loop_cb_t *cb, void *ctx)
 {
 	int i;
 
+	mutex_lock(&chip->alr_mutex);
 	lan9303_write_switch_reg(chip, LAN9303_SWE_ALR_CMD,
 				 LAN9303_ALR_CMD_GET_FIRST);
 	lan9303_write_switch_reg(chip, LAN9303_SWE_ALR_CMD, 0);
@@ -606,6 +597,7 @@ static void lan9303_alr_loop(struct lan9303 *chip, alr_loop_cb_t *cb, void *ctx)
 					 LAN9303_ALR_CMD_GET_NEXT);
 		lan9303_write_switch_reg(chip, LAN9303_SWE_ALR_CMD, 0);
 	}
+	mutex_unlock(&chip->alr_mutex);
 }
 
 static void alr_reg_to_mac(u32 dat0, u32 dat1, u8 mac[6])
@@ -694,16 +686,20 @@ static int lan9303_alr_add_port(struct lan9303 *chip, const u8 *mac, int port,
 {
 	struct lan9303_alr_cache_entry *entr;
 
+	mutex_lock(&chip->alr_mutex);
 	entr = lan9303_alr_cache_find_mac(chip, mac);
 	if (!entr) { /*New entry */
 		entr = lan9303_alr_cache_find_free(chip);
-		if (!entr)
+		if (!entr) {
+			mutex_unlock(&chip->alr_mutex);
 			return -ENOSPC;
+		}
 		ether_addr_copy(entr->mac_addr, mac);
 	}
 	entr->port_map |= BIT(port);
 	entr->stp_override = stp_override;
 	lan9303_alr_set_entry(chip, mac, entr->port_map, stp_override);
+	mutex_unlock(&chip->alr_mutex);
 
 	return 0;
 }
@@ -713,15 +709,18 @@ static int lan9303_alr_del_port(struct lan9303 *chip, const u8 *mac, int port)
 {
 	struct lan9303_alr_cache_entry *entr;
 
+	mutex_lock(&chip->alr_mutex);
 	entr = lan9303_alr_cache_find_mac(chip, mac);
 	if (!entr)
-		return 0;  /* no static entry found */
+		goto out;  /* no static entry found */
 
 	entr->port_map &= ~BIT(port);
 	if (entr->port_map == 0) /* zero means its free again */
 		eth_zero_addr(entr->mac_addr);
 	lan9303_alr_set_entry(chip, mac, entr->port_map, entr->stp_override);
 
+out:
+	mutex_unlock(&chip->alr_mutex);
 	return 0;
 }
 
@@ -818,18 +817,16 @@ static void lan9303_bridge_ports(struct lan9303 *chip)
 	lan9303_alr_add_port(chip, eth_stp_addr, 0, true);
 }
 
-static int lan9303_handle_reset(struct lan9303 *chip)
+static void lan9303_handle_reset(struct lan9303 *chip)
 {
 	if (!chip->reset_gpio)
-		return 0;
+		return;
 
 	if (chip->reset_duration != 0)
 		msleep(chip->reset_duration);
 
 	/* release (deassert) reset and activate the device */
 	gpiod_set_value_cansleep(chip->reset_gpio, 0);
-
-	return 0;
 }
 
 /* stop processing packets for all ports */
@@ -866,7 +863,7 @@ static int lan9303_check_device(struct lan9303 *chip)
 	if ((reg >> 16) != LAN9303_CHIP_ID) {
 		dev_err(chip->dev, "expecting LAN9303 chip, but found: %X\n",
 			reg >> 16);
-		return ret;
+		return -ENODEV;
 	}
 
 	/* The default state of the LAN9303 device is to forward packets between
@@ -1018,7 +1015,7 @@ static int lan9303_get_sset_count(struct dsa_switch *ds)
 static int lan9303_phy_read(struct dsa_switch *ds, int phy, int regnum)
 {
 	struct lan9303 *chip = ds->priv;
-	int phy_base = chip->phy_addr_sel_strap;
+	int phy_base = chip->phy_addr_base;
 
 	if (phy == phy_base)
 		return lan9303_virt_phy_reg_read(chip, regnum);
@@ -1032,7 +1029,7 @@ static int lan9303_phy_write(struct dsa_switch *ds, int phy, int regnum,
 			     u16 val)
 {
 	struct lan9303 *chip = ds->priv;
-	int phy_base = chip->phy_addr_sel_strap;
+	int phy_base = chip->phy_addr_base;
 
 	if (phy == phy_base)
 		return lan9303_virt_phy_reg_write(chip, regnum, val);
@@ -1069,7 +1066,7 @@ static void lan9303_adjust_link(struct dsa_switch *ds, int port,
 
 	res =  lan9303_phy_write(ds, port, MII_BMCR, ctl);
 
-	if (port == chip->phy_addr_sel_strap) {
+	if (port == chip->phy_addr_base) {
 		/* Virtual Phy: Remove Turbo 200Mbit mode */
 		lan9303_read(chip->regmap, LAN9303_VIRT_SPECIAL_CTRL, &ctl);
 
@@ -1093,8 +1090,7 @@ static void lan9303_port_disable(struct dsa_switch *ds, int port,
 	struct lan9303 *chip = ds->priv;
 
 	lan9303_disable_processing_port(chip, port);
-	lan9303_phy_write(ds, chip->phy_addr_sel_strap + port,
-			  MII_BMCR, BMCR_PDOWN);
+	lan9303_phy_write(ds, chip->phy_addr_base + port, MII_BMCR, BMCR_PDOWN);
 }
 
 static int lan9303_port_bridge_join(struct dsa_switch *ds, int port,
@@ -1217,8 +1213,7 @@ static int lan9303_port_fdb_dump(struct dsa_switch *ds, int port,
 }
 
 static int lan9303_port_mdb_prepare(struct dsa_switch *ds, int port,
-				    const struct switchdev_obj_port_mdb *mdb,
-				    struct switchdev_trans *trans)
+				    const struct switchdev_obj_port_mdb *mdb)
 {
 	struct lan9303 *chip = ds->priv;
 
@@ -1235,8 +1230,7 @@ static int lan9303_port_mdb_prepare(struct dsa_switch *ds, int port,
 }
 
 static void lan9303_port_mdb_add(struct dsa_switch *ds, int port,
-				 const struct switchdev_obj_port_mdb *mdb,
-				 struct switchdev_trans *trans)
+				 const struct switchdev_obj_port_mdb *mdb)
 {
 	struct lan9303 *chip = ds->priv;
 
@@ -1284,26 +1278,31 @@ static const struct dsa_switch_ops lan9303_switch_ops = {
 
 static int lan9303_register_switch(struct lan9303 *chip)
 {
+	int base;
+
 	chip->ds = dsa_switch_alloc(chip->dev, LAN9303_NUM_PORTS);
 	if (!chip->ds)
 		return -ENOMEM;
 
 	chip->ds->priv = chip;
 	chip->ds->ops = &lan9303_switch_ops;
-	chip->ds->phys_mii_mask = chip->phy_addr_sel_strap ? 0xe : 0x7;
+	base = chip->phy_addr_base;
+	chip->ds->phys_mii_mask = GENMASK(LAN9303_NUM_PORTS - 1 + base, base);
 
 	return dsa_register_switch(chip->ds);
 }
 
-static void lan9303_probe_reset_gpio(struct lan9303 *chip,
+static int lan9303_probe_reset_gpio(struct lan9303 *chip,
 				     struct device_node *np)
 {
 	chip->reset_gpio = devm_gpiod_get_optional(chip->dev, "reset",
 						   GPIOD_OUT_LOW);
+	if (IS_ERR(chip->reset_gpio))
+		return PTR_ERR(chip->reset_gpio);
 
-	if (IS_ERR(chip->reset_gpio)) {
+	if (!chip->reset_gpio) {
 		dev_dbg(chip->dev, "No reset GPIO defined\n");
-		return;
+		return 0;
 	}
 
 	chip->reset_duration = 200;
@@ -1318,6 +1317,8 @@ static void lan9303_probe_reset_gpio(struct lan9303 *chip,
 	/* A sane reset duration should not be longer than 1s */
 	if (chip->reset_duration > 1000)
 		chip->reset_duration = 1000;
+
+	return 0;
 }
 
 int lan9303_probe(struct lan9303 *chip, struct device_node *np)
@@ -1325,13 +1326,14 @@ int lan9303_probe(struct lan9303 *chip, struct device_node *np)
 	int ret;
 
 	mutex_init(&chip->indirect_mutex);
+	mutex_init(&chip->alr_mutex);
 
-	lan9303_probe_reset_gpio(chip, np);
-
-	ret = lan9303_handle_reset(chip);
+	ret = lan9303_probe_reset_gpio(chip, np);
 	if (ret)
 		return ret;
 
+	lan9303_handle_reset(chip);
+
 	ret = lan9303_check_device(chip);
 	if (ret)
 		return ret;
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index b5be93a1e0df..663b0d5b982b 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -559,8 +559,7 @@ static int ksz_port_vlan_filtering(struct dsa_switch *ds, int port, bool flag)
 }
 
 static int ksz_port_vlan_prepare(struct dsa_switch *ds, int port,
-				 const struct switchdev_obj_port_vlan *vlan,
-				 struct switchdev_trans *trans)
+				 const struct switchdev_obj_port_vlan *vlan)
 {
 	/* nothing needed */
 
@@ -568,8 +567,7 @@ static int ksz_port_vlan_prepare(struct dsa_switch *ds, int port,
 }
 
 static void ksz_port_vlan_add(struct dsa_switch *ds, int port,
-			      const struct switchdev_obj_port_vlan *vlan,
-			      struct switchdev_trans *trans)
+			      const struct switchdev_obj_port_vlan *vlan)
 {
 	struct ksz_device *dev = ds->priv;
 	u32 vlan_table[3];
@@ -858,16 +856,14 @@ static int ksz_port_fdb_dump(struct dsa_switch *ds, int port,
 }
 
 static int ksz_port_mdb_prepare(struct dsa_switch *ds, int port,
-				const struct switchdev_obj_port_mdb *mdb,
-				struct switchdev_trans *trans)
+				const struct switchdev_obj_port_mdb *mdb)
 {
 	/* nothing to do */
 	return 0;
 }
 
 static void ksz_port_mdb_add(struct dsa_switch *ds, int port,
-			     const struct switchdev_obj_port_mdb *mdb,
-			     struct switchdev_trans *trans)
+			     const struct switchdev_obj_port_mdb *mdb)
 {
 	struct ksz_device *dev = ds->priv;
 	u32 static_table[4];
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 2820d69810b3..4e53c5ce23ff 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -804,6 +804,69 @@ mt7530_port_bridge_join(struct dsa_switch *ds, int port,
 	return 0;
 }
 
+static void
+mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
+{
+	struct mt7530_priv *priv = ds->priv;
+	bool all_user_ports_removed = true;
+	int i;
+
+	/* When a port is removed from the bridge, the port would be set up
+	 * back to the default as is at initial boot which is a VLAN-unaware
+	 * port.
+	 */
+	mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
+		   MT7530_PORT_MATRIX_MODE);
+	mt7530_rmw(priv, MT7530_PVC_P(port), VLAN_ATTR_MASK,
+		   VLAN_ATTR(MT7530_VLAN_TRANSPARENT));
+
+	priv->ports[port].vlan_filtering = false;
+
+	for (i = 0; i < MT7530_NUM_PORTS; i++) {
+		if (dsa_is_user_port(ds, i) &&
+		    priv->ports[i].vlan_filtering) {
+			all_user_ports_removed = false;
+			break;
+		}
+	}
+
+	/* CPU port also does the same thing until all user ports belonging to
+	 * the CPU port get out of VLAN filtering mode.
+	 */
+	if (all_user_ports_removed) {
+		mt7530_write(priv, MT7530_PCR_P(MT7530_CPU_PORT),
+			     PCR_MATRIX(dsa_user_ports(priv->ds)));
+		mt7530_write(priv, MT7530_PVC_P(MT7530_CPU_PORT),
+			     PORT_SPEC_TAG);
+	}
+}
+
+static void
+mt7530_port_set_vlan_aware(struct dsa_switch *ds, int port)
+{
+	struct mt7530_priv *priv = ds->priv;
+
+	/* The real fabric path would be decided on the membership in the
+	 * entry of VLAN table. PCR_MATRIX set up here with ALL_MEMBERS
+	 * means potential VLAN can be consisting of certain subset of all
+	 * ports.
+	 */
+	mt7530_rmw(priv, MT7530_PCR_P(port),
+		   PCR_MATRIX_MASK, PCR_MATRIX(MT7530_ALL_MEMBERS));
+
+	/* Trapped into security mode allows packet forwarding through VLAN
+	 * table lookup.
+	 */
+	mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK,
+		   MT7530_PORT_SECURITY_MODE);
+
+	/* Set the port as a user port which is to be able to recognize VID
+	 * from incoming packets before fetching entry within the VLAN table.
+	 */
+	mt7530_rmw(priv, MT7530_PVC_P(port), VLAN_ATTR_MASK,
+		   VLAN_ATTR(MT7530_VLAN_USER));
+}
+
 static void
 mt7530_port_bridge_leave(struct dsa_switch *ds, int port,
 			 struct net_device *bridge)
@@ -817,8 +880,11 @@ mt7530_port_bridge_leave(struct dsa_switch *ds, int port,
 		/* Remove this port from the port matrix of the other ports
 		 * in the same bridge. If the port is disabled, port matrix
 		 * is kept and not being setup until the port becomes enabled.
+		 * And the other port's port matrix cannot be broken when the
+		 * other port is still a VLAN-aware port.
 		 */
-		if (dsa_is_user_port(ds, i) && i != port) {
+		if (!priv->ports[i].vlan_filtering &&
+		    dsa_is_user_port(ds, i) && i != port) {
 			if (dsa_to_port(ds, i)->bridge_dev != bridge)
 				continue;
 			if (priv->ports[i].enable)
@@ -836,6 +902,8 @@ mt7530_port_bridge_leave(struct dsa_switch *ds, int port,
 			   PCR_MATRIX(BIT(MT7530_CPU_PORT)));
 	priv->ports[port].pm = PCR_MATRIX(BIT(MT7530_CPU_PORT));
 
+	mt7530_port_set_vlan_unaware(ds, port);
+
 	mutex_unlock(&priv->reg_mutex);
 }
 
@@ -906,6 +974,220 @@ mt7530_port_fdb_dump(struct dsa_switch *ds, int port,
 	return 0;
 }
 
+static int
+mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
+{
+	struct mt7530_dummy_poll p;
+	u32 val;
+	int ret;
+
+	val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
+	mt7530_write(priv, MT7530_VTCR, val);
+
+	INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
+	ret = readx_poll_timeout(_mt7530_read, &p, val,
+				 !(val & VTCR_BUSY), 20, 20000);
+	if (ret < 0) {
+		dev_err(priv->dev, "poll timeout\n");
+		return ret;
+	}
+
+	val = mt7530_read(priv, MT7530_VTCR);
+	if (val & VTCR_INVALID) {
+		dev_err(priv->dev, "read VTCR invalid\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+mt7530_port_vlan_filtering(struct dsa_switch *ds, int port,
+			   bool vlan_filtering)
+{
+	struct mt7530_priv *priv = ds->priv;
+
+	priv->ports[port].vlan_filtering = vlan_filtering;
+
+	if (vlan_filtering) {
+		/* The port is being kept as VLAN-unaware port when bridge is
+		 * set up with vlan_filtering not being set, Otherwise, the
+		 * port and the corresponding CPU port is required the setup
+		 * for becoming a VLAN-aware port.
+		 */
+		mt7530_port_set_vlan_aware(ds, port);
+		mt7530_port_set_vlan_aware(ds, MT7530_CPU_PORT);
+	}
+
+	return 0;
+}
+
+static int
+mt7530_port_vlan_prepare(struct dsa_switch *ds, int port,
+			 const struct switchdev_obj_port_vlan *vlan)
+{
+	/* nothing needed */
+
+	return 0;
+}
+
+static void
+mt7530_hw_vlan_add(struct mt7530_priv *priv,
+		   struct mt7530_hw_vlan_entry *entry)
+{
+	u8 new_members;
+	u32 val;
+
+	new_members = entry->old_members | BIT(entry->port) |
+		      BIT(MT7530_CPU_PORT);
+
+	/* Validate the entry with independent learning, create egress tag per
+	 * VLAN and joining the port as one of the port members.
+	 */
+	val = IVL_MAC | VTAG_EN | PORT_MEM(new_members) | VLAN_VALID;
+	mt7530_write(priv, MT7530_VAWD1, val);
+
+	/* Decide whether adding tag or not for those outgoing packets from the
+	 * port inside the VLAN.
+	 */
+	val = entry->untagged ? MT7530_VLAN_EGRESS_UNTAG :
+				MT7530_VLAN_EGRESS_TAG;
+	mt7530_rmw(priv, MT7530_VAWD2,
+		   ETAG_CTRL_P_MASK(entry->port),
+		   ETAG_CTRL_P(entry->port, val));
+
+	/* CPU port is always taken as a tagged port for serving more than one
+	 * VLANs across and also being applied with egress type stack mode for
+	 * that VLAN tags would be appended after hardware special tag used as
+	 * DSA tag.
+	 */
+	mt7530_rmw(priv, MT7530_VAWD2,
+		   ETAG_CTRL_P_MASK(MT7530_CPU_PORT),
+		   ETAG_CTRL_P(MT7530_CPU_PORT,
+			       MT7530_VLAN_EGRESS_STACK));
+}
+
+static void
+mt7530_hw_vlan_del(struct mt7530_priv *priv,
+		   struct mt7530_hw_vlan_entry *entry)
+{
+	u8 new_members;
+	u32 val;
+
+	new_members = entry->old_members & ~BIT(entry->port);
+
+	val = mt7530_read(priv, MT7530_VAWD1);
+	if (!(val & VLAN_VALID)) {
+		dev_err(priv->dev,
+			"Cannot be deleted due to invalid entry\n");
+		return;
+	}
+
+	/* If certain member apart from CPU port is still alive in the VLAN,
+	 * the entry would be kept valid. Otherwise, the entry is got to be
+	 * disabled.
+	 */
+	if (new_members && new_members != BIT(MT7530_CPU_PORT)) {
+		val = IVL_MAC | VTAG_EN | PORT_MEM(new_members) |
+		      VLAN_VALID;
+		mt7530_write(priv, MT7530_VAWD1, val);
+	} else {
+		mt7530_write(priv, MT7530_VAWD1, 0);
+		mt7530_write(priv, MT7530_VAWD2, 0);
+	}
+}
+
+static void
+mt7530_hw_vlan_update(struct mt7530_priv *priv, u16 vid,
+		      struct mt7530_hw_vlan_entry *entry,
+		      mt7530_vlan_op vlan_op)
+{
+	u32 val;
+
+	/* Fetch entry */
+	mt7530_vlan_cmd(priv, MT7530_VTCR_RD_VID, vid);
+
+	val = mt7530_read(priv, MT7530_VAWD1);
+
+	entry->old_members = (val >> PORT_MEM_SHFT) & PORT_MEM_MASK;
+
+	/* Manipulate entry */
+	vlan_op(priv, entry);
+
+	/* Flush result to hardware */
+	mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, vid);
+}
+
+static void
+mt7530_port_vlan_add(struct dsa_switch *ds, int port,
+		     const struct switchdev_obj_port_vlan *vlan)
+{
+	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+	bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID;
+	struct mt7530_hw_vlan_entry new_entry;
+	struct mt7530_priv *priv = ds->priv;
+	u16 vid;
+
+	/* The port is kept as VLAN-unaware if bridge with vlan_filtering not
+	 * being set.
+	 */
+	if (!priv->ports[port].vlan_filtering)
+		return;
+
+	mutex_lock(&priv->reg_mutex);
+
+	for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
+		mt7530_hw_vlan_entry_init(&new_entry, port, untagged);
+		mt7530_hw_vlan_update(priv, vid, &new_entry,
+				      mt7530_hw_vlan_add);
+	}
+
+	if (pvid) {
+		mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK,
+			   G0_PORT_VID(vlan->vid_end));
+		priv->ports[port].pvid = vlan->vid_end;
+	}
+
+	mutex_unlock(&priv->reg_mutex);
+}
+
+static int
+mt7530_port_vlan_del(struct dsa_switch *ds, int port,
+		     const struct switchdev_obj_port_vlan *vlan)
+{
+	struct mt7530_hw_vlan_entry target_entry;
+	struct mt7530_priv *priv = ds->priv;
+	u16 vid, pvid;
+
+	/* The port is kept as VLAN-unaware if bridge with vlan_filtering not
+	 * being set.
+	 */
+	if (!priv->ports[port].vlan_filtering)
+		return 0;
+
+	mutex_lock(&priv->reg_mutex);
+
+	pvid = priv->ports[port].pvid;
+	for (vid = vlan->vid_begin; vid <= vlan->vid_end; ++vid) {
+		mt7530_hw_vlan_entry_init(&target_entry, port, 0);
+		mt7530_hw_vlan_update(priv, vid, &target_entry,
+				      mt7530_hw_vlan_del);
+
+		/* PVID is being restored to the default whenever the PVID port
+		 * is being removed from the VLAN.
+		 */
+		if (pvid == vid)
+			pvid = G0_PORT_VID_DEF;
+	}
+
+	mt7530_rmw(priv, MT7530_PPBV1_P(port), G0_PORT_VID_MASK, pvid);
+	priv->ports[port].pvid = pvid;
+
+	mutex_unlock(&priv->reg_mutex);
+
+	return 0;
+}
+
 static enum dsa_tag_protocol
 mtk_get_tag_protocol(struct dsa_switch *ds, int port)
 {
@@ -1035,6 +1317,10 @@ static const struct dsa_switch_ops mt7530_switch_ops = {
 	.port_fdb_add		= mt7530_port_fdb_add,
 	.port_fdb_del		= mt7530_port_fdb_del,
 	.port_fdb_dump		= mt7530_port_fdb_dump,
+	.port_vlan_filtering	= mt7530_port_vlan_filtering,
+	.port_vlan_prepare	= mt7530_port_vlan_prepare,
+	.port_vlan_add		= mt7530_port_vlan_add,
+	.port_vlan_del		= mt7530_port_vlan_del,
 };
 
 static int
@@ -1123,6 +1409,7 @@ static const struct of_device_id mt7530_of_match[] = {
 	{ .compatible = "mediatek,mt7530" },
 	{ /* sentinel */ },
 };
+MODULE_DEVICE_TABLE(of, mt7530_of_match);
 
 static struct mdio_driver mt7530_mdio_driver = {
 	.probe  = mt7530_probe,
@@ -1138,4 +1425,3 @@ mdio_module_driver(mt7530_mdio_driver);
 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
 MODULE_DESCRIPTION("Driver for Mediatek MT7530 Switch");
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:mediatek-mt7530");
diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
index 74db9822eb40..d9b407a22a58 100644
--- a/drivers/net/dsa/mt7530.h
+++ b/drivers/net/dsa/mt7530.h
@@ -17,6 +17,7 @@
 #define MT7530_NUM_PORTS		7
 #define MT7530_CPU_PORT			6
 #define MT7530_NUM_FDB_RECORDS		2048
+#define MT7530_ALL_MEMBERS		0xff
 
 #define	NUM_TRGMII_CTRL			5
 
@@ -88,21 +89,42 @@ enum mt7530_fdb_cmd {
 /* Register for vlan table control */
 #define MT7530_VTCR			0x90
 #define  VTCR_BUSY			BIT(31)
-#define  VTCR_FUNC			(((x) & 0xf) << 12)
-#define  VTCR_FUNC_RD_VID		0x1
-#define  VTCR_FUNC_WR_VID		0x2
-#define  VTCR_FUNC_INV_VID		0x3
-#define  VTCR_FUNC_VAL_VID		0x4
+#define  VTCR_INVALID			BIT(16)
+#define  VTCR_FUNC(x)			(((x) & 0xf) << 12)
 #define  VTCR_VID			((x) & 0xfff)
 
+enum mt7530_vlan_cmd {
+	/* Read/Write the specified VID entry from VAWD register based
+	 * on VID.
+	 */
+	MT7530_VTCR_RD_VID = 0,
+	MT7530_VTCR_WR_VID = 1,
+};
+
 /* Register for setup vlan and acl write data */
 #define MT7530_VAWD1			0x94
 #define  PORT_STAG			BIT(31)
+/* Independent VLAN Learning */
 #define  IVL_MAC			BIT(30)
+/* Per VLAN Egress Tag Control */
+#define  VTAG_EN			BIT(28)
+/* VLAN Member Control */
 #define  PORT_MEM(x)			(((x) & 0xff) << 16)
-#define  VALID				BIT(1)
+/* VLAN Entry Valid */
+#define  VLAN_VALID			BIT(0)
+#define  PORT_MEM_SHFT			16
+#define  PORT_MEM_MASK			0xff
 
 #define MT7530_VAWD2			0x98
+/* Egress Tag Control */
+#define  ETAG_CTRL_P(p, x)		(((x) & 0x3) << ((p) << 1))
+#define  ETAG_CTRL_P_MASK(p)		ETAG_CTRL_P(p, 3)
+
+enum mt7530_vlan_egress_attr {
+	MT7530_VLAN_EGRESS_UNTAG = 0,
+	MT7530_VLAN_EGRESS_TAG = 2,
+	MT7530_VLAN_EGRESS_STACK = 3,
+};
 
 /* Register for port STP state control */
 #define MT7530_SSP_P(x)			(0x2000 + ((x) * 0x100))
@@ -120,11 +142,23 @@ enum mt7530_stp_state {
 /* Register for port control */
 #define MT7530_PCR_P(x)			(0x2004 + ((x) * 0x100))
 #define  PORT_VLAN(x)			((x) & 0x3)
+
+enum mt7530_port_mode {
+	/* Port Matrix Mode: Frames are forwarded by the PCR_MATRIX members. */
+	MT7530_PORT_MATRIX_MODE = PORT_VLAN(0),
+
+	/* Security Mode: Discard any frame due to ingress membership
+	 * violation or VID missed on the VLAN table.
+	 */
+	MT7530_PORT_SECURITY_MODE = PORT_VLAN(3),
+};
+
 #define  PCR_MATRIX(x)			(((x) & 0xff) << 16)
 #define  PORT_PRI(x)			(((x) & 0x7) << 24)
 #define  EG_TAG(x)			(((x) & 0x3) << 28)
 #define  PCR_MATRIX_MASK		PCR_MATRIX(0xff)
 #define  PCR_MATRIX_CLR			PCR_MATRIX(0)
+#define  PCR_PORT_VLAN_MASK		PORT_VLAN(3)
 
 /* Register for port security control */
 #define MT7530_PSC_P(x)			(0x200c + ((x) * 0x100))
@@ -134,10 +168,20 @@ enum mt7530_stp_state {
 #define MT7530_PVC_P(x)			(0x2010 + ((x) * 0x100))
 #define  PORT_SPEC_TAG			BIT(5)
 #define  VLAN_ATTR(x)			(((x) & 0x3) << 6)
+#define  VLAN_ATTR_MASK			VLAN_ATTR(3)
+
+enum mt7530_vlan_port_attr {
+	MT7530_VLAN_USER = 0,
+	MT7530_VLAN_TRANSPARENT = 3,
+};
+
 #define  STAG_VPID			(((x) & 0xffff) << 16)
 
 /* Register for port port-and-protocol based vlan 1 control */
 #define MT7530_PPBV1_P(x)		(0x2014 + ((x) * 0x100))
+#define  G0_PORT_VID(x)			(((x) & 0xfff) << 0)
+#define  G0_PORT_VID_MASK		G0_PORT_VID(0xfff)
+#define  G0_PORT_VID_DEF		G0_PORT_VID(1)
 
 /* Register for port MAC control register */
 #define MT7530_PMCR_P(x)		(0x3000 + ((x) * 0x100))
@@ -345,9 +389,20 @@ struct mt7530_fdb {
 	bool noarp;
 };
 
+/* struct mt7530_port -	This is the main data structure for holding the state
+ *			of the port.
+ * @enable:	The status used for show port is enabled or not.
+ * @pm:		The matrix used to show all connections with the port.
+ * @pvid:	The VLAN specified is to be considered a PVID at ingress.  Any
+ *		untagged frames will be assigned to the related VLAN.
+ * @vlan_filtering: The flags indicating whether the port that can recognize
+ *		    VLAN-tagged frames.
+ */
 struct mt7530_port {
 	bool enable;
 	u32 pm;
+	u16 pvid;
+	bool vlan_filtering;
 };
 
 /* struct mt7530_priv -	This is the main data structure for holding the state
@@ -382,6 +437,22 @@ struct mt7530_priv {
 	struct mutex reg_mutex;
 };
 
+struct mt7530_hw_vlan_entry {
+	int port;
+	u8  old_members;
+	bool untagged;
+};
+
+static inline void mt7530_hw_vlan_entry_init(struct mt7530_hw_vlan_entry *e,
+					     int port, bool untagged)
+{
+	e->port = port;
+	e->untagged = untagged;
+}
+
+typedef void (*mt7530_vlan_op)(struct mt7530_priv *,
+			       struct mt7530_hw_vlan_entry *);
+
 struct mt7530_hw_stats {
 	const char	*string;
 	u16		reg;
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 66d33e97cbc5..eb328bade225 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -1185,8 +1185,7 @@ static int mv88e6xxx_port_vlan_filtering(struct dsa_switch *ds, int port,
 
 static int
 mv88e6xxx_port_vlan_prepare(struct dsa_switch *ds, int port,
-			    const struct switchdev_obj_port_vlan *vlan,
-			    struct switchdev_trans *trans)
+			    const struct switchdev_obj_port_vlan *vlan)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
@@ -1295,8 +1294,7 @@ static int _mv88e6xxx_port_vlan_add(struct mv88e6xxx_chip *chip, int port,
 }
 
 static void mv88e6xxx_port_vlan_add(struct dsa_switch *ds, int port,
-				    const struct switchdev_obj_port_vlan *vlan,
-				    struct switchdev_trans *trans)
+				    const struct switchdev_obj_port_vlan *vlan)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
@@ -1725,9 +1723,11 @@ static int mv88e6xxx_setup_message_port(struct mv88e6xxx_chip *chip, int port)
 
 static int mv88e6xxx_setup_egress_floods(struct mv88e6xxx_chip *chip, int port)
 {
-	bool flood = port == dsa_upstream_port(chip->ds);
+	struct dsa_switch *ds = chip->ds;
+	bool flood;
 
 	/* Upstream ports flood frames with unknown unicast or multicast DA */
+	flood = dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port);
 	if (chip->info->ops->port_set_egress_floods)
 		return chip->info->ops->port_set_egress_floods(chip, port,
 							       flood, flood);
@@ -1744,6 +1744,39 @@ static int mv88e6xxx_serdes_power(struct mv88e6xxx_chip *chip, int port,
 	return 0;
 }
 
+static int mv88e6xxx_setup_upstream_port(struct mv88e6xxx_chip *chip, int port)
+{
+	struct dsa_switch *ds = chip->ds;
+	int upstream_port;
+	int err;
+
+	upstream_port = dsa_upstream_port(ds, port);
+	if (chip->info->ops->port_set_upstream_port) {
+		err = chip->info->ops->port_set_upstream_port(chip, port,
+							      upstream_port);
+		if (err)
+			return err;
+	}
+
+	if (port == upstream_port) {
+		if (chip->info->ops->set_cpu_port) {
+			err = chip->info->ops->set_cpu_port(chip,
+							    upstream_port);
+			if (err)
+				return err;
+		}
+
+		if (chip->info->ops->set_egress_port) {
+			err = chip->info->ops->set_egress_port(chip,
+							       upstream_port);
+			if (err)
+				return err;
+		}
+	}
+
+	return 0;
+}
+
 static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 {
 	struct dsa_switch *ds = chip->ds;
@@ -1814,13 +1847,9 @@ static int mv88e6xxx_setup_port(struct mv88e6xxx_chip *chip, int port)
 	if (err)
 		return err;
 
-	reg = 0;
-	if (chip->info->ops->port_set_upstream_port) {
-		err = chip->info->ops->port_set_upstream_port(
-			chip, port, dsa_upstream_port(ds));
-		if (err)
-			return err;
-	}
+	err = mv88e6xxx_setup_upstream_port(chip, port);
+	if (err)
+		return err;
 
 	err = mv88e6xxx_port_set_8021q_mode(chip, port,
 				MV88E6XXX_PORT_CTL2_8021Q_MODE_DISABLED);
@@ -1946,21 +1975,8 @@ static int mv88e6xxx_set_ageing_time(struct dsa_switch *ds,
 static int mv88e6xxx_g1_setup(struct mv88e6xxx_chip *chip)
 {
 	struct dsa_switch *ds = chip->ds;
-	u32 upstream_port = dsa_upstream_port(ds);
 	int err;
 
-	if (chip->info->ops->set_cpu_port) {
-		err = chip->info->ops->set_cpu_port(chip, upstream_port);
-		if (err)
-			return err;
-	}
-
-	if (chip->info->ops->set_egress_port) {
-		err = chip->info->ops->set_egress_port(chip, upstream_port);
-		if (err)
-			return err;
-	}
-
 	/* Disable remote management, and set the switch's DSA device number. */
 	err = mv88e6xxx_g1_write(chip, MV88E6XXX_G1_CTL2,
 				 MV88E6XXX_G1_CTL2_MULTIPLE_CASCADE |
@@ -3741,6 +3757,7 @@ static enum dsa_tag_protocol mv88e6xxx_get_tag_protocol(struct dsa_switch *ds,
 	return chip->info->tag_protocol;
 }
 
+#if IS_ENABLED(CONFIG_NET_DSA_LEGACY)
 static const char *mv88e6xxx_drv_probe(struct device *dsa_dev,
 				       struct device *host_dev, int sw_addr,
 				       void **priv)
@@ -3788,10 +3805,10 @@ static const char *mv88e6xxx_drv_probe(struct device *dsa_dev,
 
 	return NULL;
 }
+#endif
 
 static int mv88e6xxx_port_mdb_prepare(struct dsa_switch *ds, int port,
-				      const struct switchdev_obj_port_mdb *mdb,
-				      struct switchdev_trans *trans)
+				      const struct switchdev_obj_port_mdb *mdb)
 {
 	/* We don't need any dynamic resource from the kernel (yet),
 	 * so skip the prepare phase.
@@ -3801,8 +3818,7 @@ static int mv88e6xxx_port_mdb_prepare(struct dsa_switch *ds, int port,
 }
 
 static void mv88e6xxx_port_mdb_add(struct dsa_switch *ds, int port,
-				   const struct switchdev_obj_port_mdb *mdb,
-				   struct switchdev_trans *trans)
+				   const struct switchdev_obj_port_mdb *mdb)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 
@@ -3829,7 +3845,9 @@ static int mv88e6xxx_port_mdb_del(struct dsa_switch *ds, int port,
 }
 
 static const struct dsa_switch_ops mv88e6xxx_switch_ops = {
+#if IS_ENABLED(CONFIG_NET_DSA_LEGACY)
 	.probe			= mv88e6xxx_drv_probe,
+#endif
 	.get_tag_protocol	= mv88e6xxx_get_tag_protocol,
 	.setup			= mv88e6xxx_setup,
 	.adjust_link		= mv88e6xxx_adjust_link,
@@ -3958,11 +3976,19 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 			if (err)
 				goto out_g1_irq;
 		}
+
+		err = mv88e6xxx_g1_atu_prob_irq_setup(chip);
+		if (err)
+			goto out_g2_irq;
+
+		err = mv88e6xxx_g1_vtu_prob_irq_setup(chip);
+		if (err)
+			goto out_g1_atu_prob_irq;
 	}
 
 	err = mv88e6xxx_mdios_register(chip, np);
 	if (err)
-		goto out_g2_irq;
+		goto out_g1_vtu_prob_irq;
 
 	err = mv88e6xxx_register_switch(chip);
 	if (err)
@@ -3972,6 +3998,12 @@ static int mv88e6xxx_probe(struct mdio_device *mdiodev)
 
 out_mdio:
 	mv88e6xxx_mdios_unregister(chip);
+out_g1_vtu_prob_irq:
+	if (chip->irq > 0)
+		mv88e6xxx_g1_vtu_prob_irq_free(chip);
+out_g1_atu_prob_irq:
+	if (chip->irq > 0)
+		mv88e6xxx_g1_atu_prob_irq_free(chip);
 out_g2_irq:
 	if (chip->info->g2_irqs > 0 && chip->irq > 0)
 		mv88e6xxx_g2_irq_free(chip);
@@ -3995,6 +4027,8 @@ static void mv88e6xxx_remove(struct mdio_device *mdiodev)
 	mv88e6xxx_mdios_unregister(chip);
 
 	if (chip->irq > 0) {
+		mv88e6xxx_g1_vtu_prob_irq_free(chip);
+		mv88e6xxx_g1_atu_prob_irq_free(chip);
 		if (chip->info->g2_irqs > 0)
 			mv88e6xxx_g2_irq_free(chip);
 		mutex_lock(&chip->reg_lock);
diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h
index 334f6f7544ba..3dba6e90adcf 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.h
+++ b/drivers/net/dsa/mv88e6xxx/chip.h
@@ -207,6 +207,8 @@ struct mv88e6xxx_chip {
 	int irq;
 	int device_irq;
 	int watchdog_irq;
+	int atu_prob_irq;
+	int vtu_prob_irq;
 };
 
 struct mv88e6xxx_bus_ops {
diff --git a/drivers/net/dsa/mv88e6xxx/global1.h b/drivers/net/dsa/mv88e6xxx/global1.h
index b0dc7518b47f..6aee7316fea6 100644
--- a/drivers/net/dsa/mv88e6xxx/global1.h
+++ b/drivers/net/dsa/mv88e6xxx/global1.h
@@ -29,9 +29,9 @@
 #define MV88E6XXX_G1_STS_IRQ_AVB			8
 #define MV88E6XXX_G1_STS_IRQ_DEVICE			7
 #define MV88E6XXX_G1_STS_IRQ_STATS			6
-#define MV88E6XXX_G1_STS_IRQ_VTU_PROBLEM		5
+#define MV88E6XXX_G1_STS_IRQ_VTU_PROB			5
 #define MV88E6XXX_G1_STS_IRQ_VTU_DONE			4
-#define MV88E6XXX_G1_STS_IRQ_ATU_PROBLEM		3
+#define MV88E6XXX_G1_STS_IRQ_ATU_PROB			3
 #define MV88E6XXX_G1_STS_IRQ_ATU_DONE			2
 #define MV88E6XXX_G1_STS_IRQ_TCAM_DONE			1
 #define MV88E6XXX_G1_STS_IRQ_EEPROM_DONE		0
@@ -82,6 +82,10 @@
 #define MV88E6XXX_G1_VTU_OP_VTU_GET_NEXT	0x4000
 #define MV88E6XXX_G1_VTU_OP_STU_LOAD_PURGE	0x5000
 #define MV88E6XXX_G1_VTU_OP_STU_GET_NEXT	0x6000
+#define MV88E6XXX_G1_VTU_OP_GET_CLR_VIOLATION	0x7000
+#define MV88E6XXX_G1_VTU_OP_MEMBER_VIOLATION	BIT(6)
+#define MV88E6XXX_G1_VTU_OP_MISS_VIOLATION	BIT(5)
+#define MV88E6XXX_G1_VTU_OP_SPID_MASK		0xf
 
 /* Offset 0x06: VTU VID Register */
 #define MV88E6XXX_G1_VTU_VID		0x06
@@ -122,6 +126,10 @@
 #define MV88E6XXX_G1_ATU_OP_FLUSH_MOVE_ALL_DB		0x5000
 #define MV88E6XXX_G1_ATU_OP_FLUSH_MOVE_NON_STATIC_DB	0x6000
 #define MV88E6XXX_G1_ATU_OP_GET_CLR_VIOLATION		0x7000
+#define MV88E6XXX_G1_ATU_OP_AGE_OUT_VIOLATION		BIT(7)
+#define MV88E6XXX_G1_ATU_OP_MEMBER_VIOLATION		BIT(6)
+#define MV88E6XXX_G1_ATU_OP_MISS_VIOLTATION		BIT(5)
+#define MV88E6XXX_G1_ATU_OP_FULL_VIOLATION		BIT(4)
 
 /* Offset 0x0C: ATU Data Register */
 #define MV88E6XXX_G1_ATU_DATA				0x0c
@@ -255,6 +263,8 @@ int mv88e6xxx_g1_atu_loadpurge(struct mv88e6xxx_chip *chip, u16 fid,
 int mv88e6xxx_g1_atu_flush(struct mv88e6xxx_chip *chip, u16 fid, bool all);
 int mv88e6xxx_g1_atu_remove(struct mv88e6xxx_chip *chip, u16 fid, int port,
 			    bool all);
+int mv88e6xxx_g1_atu_prob_irq_setup(struct mv88e6xxx_chip *chip);
+void mv88e6xxx_g1_atu_prob_irq_free(struct mv88e6xxx_chip *chip);
 
 int mv88e6185_g1_vtu_getnext(struct mv88e6xxx_chip *chip,
 			     struct mv88e6xxx_vtu_entry *entry);
@@ -269,5 +279,7 @@ int mv88e6390_g1_vtu_getnext(struct mv88e6xxx_chip *chip,
 int mv88e6390_g1_vtu_loadpurge(struct mv88e6xxx_chip *chip,
 			       struct mv88e6xxx_vtu_entry *entry);
 int mv88e6xxx_g1_vtu_flush(struct mv88e6xxx_chip *chip);
+int mv88e6xxx_g1_vtu_prob_irq_setup(struct mv88e6xxx_chip *chip);
+void mv88e6xxx_g1_vtu_prob_irq_free(struct mv88e6xxx_chip *chip);
 
 #endif /* _MV88E6XXX_GLOBAL1_H */
diff --git a/drivers/net/dsa/mv88e6xxx/global1_atu.c b/drivers/net/dsa/mv88e6xxx/global1_atu.c
index efeef4b01442..20d941f4273b 100644
--- a/drivers/net/dsa/mv88e6xxx/global1_atu.c
+++ b/drivers/net/dsa/mv88e6xxx/global1_atu.c
@@ -9,6 +9,8 @@
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  */
+#include <linux/interrupt.h>
+#include <linux/irqdomain.h>
 
 #include "chip.h"
 #include "global1.h"
@@ -307,3 +309,88 @@ int mv88e6xxx_g1_atu_remove(struct mv88e6xxx_chip *chip, u16 fid, int port,
 
 	return mv88e6xxx_g1_atu_move(chip, fid, from_port, to_port, all);
 }
+
+static irqreturn_t mv88e6xxx_g1_atu_prob_irq_thread_fn(int irq, void *dev_id)
+{
+	struct mv88e6xxx_chip *chip = dev_id;
+	struct mv88e6xxx_atu_entry entry;
+	int err;
+	u16 val;
+
+	mutex_lock(&chip->reg_lock);
+
+	err = mv88e6xxx_g1_atu_op(chip, 0,
+				  MV88E6XXX_G1_ATU_OP_GET_CLR_VIOLATION);
+	if (err)
+		goto out;
+
+	err = mv88e6xxx_g1_read(chip, MV88E6XXX_G1_ATU_OP, &val);
+	if (err)
+		goto out;
+
+	err = mv88e6xxx_g1_atu_data_read(chip, &entry);
+	if (err)
+		goto out;
+
+	err = mv88e6xxx_g1_atu_mac_read(chip, &entry);
+	if (err)
+		goto out;
+
+	mutex_unlock(&chip->reg_lock);
+
+	if (val & MV88E6XXX_G1_ATU_OP_AGE_OUT_VIOLATION) {
+		dev_err_ratelimited(chip->dev,
+				    "ATU age out violation for %pM\n",
+				    entry.mac);
+	}
+
+	if (val & MV88E6XXX_G1_ATU_OP_MEMBER_VIOLATION) {
+		dev_err_ratelimited(chip->dev,
+				    "ATU member violation for %pM portvec %x\n",
+				    entry.mac, entry.portvec);
+	}
+
+	if (val & MV88E6XXX_G1_ATU_OP_MEMBER_VIOLATION)
+		dev_err_ratelimited(chip->dev,
+				    "ATU miss violation for %pM portvec %x\n",
+				    entry.mac, entry.portvec);
+
+	if (val & MV88E6XXX_G1_ATU_OP_FULL_VIOLATION)
+		dev_err_ratelimited(chip->dev,
+				    "ATU full violation for %pM portvec %x\n",
+				    entry.mac, entry.portvec);
+
+	return IRQ_HANDLED;
+
+out:
+	mutex_unlock(&chip->reg_lock);
+
+	dev_err(chip->dev, "ATU problem: error %d while handling interrupt\n",
+		err);
+	return IRQ_HANDLED;
+}
+
+int mv88e6xxx_g1_atu_prob_irq_setup(struct mv88e6xxx_chip *chip)
+{
+	int err;
+
+	chip->atu_prob_irq = irq_find_mapping(chip->g1_irq.domain,
+					      MV88E6XXX_G1_STS_IRQ_ATU_PROB);
+	if (chip->atu_prob_irq < 0)
+		return chip->atu_prob_irq;
+
+	err = request_threaded_irq(chip->atu_prob_irq, NULL,
+				   mv88e6xxx_g1_atu_prob_irq_thread_fn,
+				   IRQF_ONESHOT, "mv88e6xxx-g1-atu-prob",
+				   chip);
+	if (err)
+		irq_dispose_mapping(chip->atu_prob_irq);
+
+	return err;
+}
+
+void mv88e6xxx_g1_atu_prob_irq_free(struct mv88e6xxx_chip *chip)
+{
+	free_irq(chip->atu_prob_irq, chip);
+	irq_dispose_mapping(chip->atu_prob_irq);
+}
diff --git a/drivers/net/dsa/mv88e6xxx/global1_vtu.c b/drivers/net/dsa/mv88e6xxx/global1_vtu.c
index 8c8a0ec3d6e9..7997961647de 100644
--- a/drivers/net/dsa/mv88e6xxx/global1_vtu.c
+++ b/drivers/net/dsa/mv88e6xxx/global1_vtu.c
@@ -11,6 +11,9 @@
  * (at your option) any later version.
  */
 
+#include <linux/interrupt.h>
+#include <linux/irqdomain.h>
+
 #include "chip.h"
 #include "global1.h"
 
@@ -513,3 +516,74 @@ int mv88e6xxx_g1_vtu_flush(struct mv88e6xxx_chip *chip)
 
 	return mv88e6xxx_g1_vtu_op(chip, MV88E6XXX_G1_VTU_OP_FLUSH_ALL);
 }
+
+static irqreturn_t mv88e6xxx_g1_vtu_prob_irq_thread_fn(int irq, void *dev_id)
+{
+	struct mv88e6xxx_chip *chip = dev_id;
+	struct mv88e6xxx_vtu_entry entry;
+	int spid;
+	int err;
+	u16 val;
+
+	mutex_lock(&chip->reg_lock);
+
+	err = mv88e6xxx_g1_vtu_op(chip, MV88E6XXX_G1_VTU_OP_GET_CLR_VIOLATION);
+	if (err)
+		goto out;
+
+	err = mv88e6xxx_g1_read(chip, MV88E6XXX_G1_VTU_OP, &val);
+	if (err)
+		goto out;
+
+	err = mv88e6xxx_g1_vtu_vid_read(chip, &entry);
+	if (err)
+		goto out;
+
+	mutex_unlock(&chip->reg_lock);
+
+	spid = val & MV88E6XXX_G1_VTU_OP_SPID_MASK;
+
+	if (val & MV88E6XXX_G1_VTU_OP_MEMBER_VIOLATION) {
+		dev_err_ratelimited(chip->dev, "VTU member violation for vid %d, source port %d\n",
+				    entry.vid, spid);
+	}
+
+	if (val & MV88E6XXX_G1_VTU_OP_MISS_VIOLATION)
+		dev_err_ratelimited(chip->dev, "VTU miss violation for vid %d, source port %d\n",
+				    entry.vid, spid);
+
+	return IRQ_HANDLED;
+
+out:
+	mutex_unlock(&chip->reg_lock);
+
+	dev_err(chip->dev, "VTU problem: error %d while handling interrupt\n",
+		err);
+
+	return IRQ_HANDLED;
+}
+
+int mv88e6xxx_g1_vtu_prob_irq_setup(struct mv88e6xxx_chip *chip)
+{
+	int err;
+
+	chip->vtu_prob_irq = irq_find_mapping(chip->g1_irq.domain,
+					      MV88E6XXX_G1_STS_IRQ_VTU_PROB);
+	if (chip->vtu_prob_irq < 0)
+		return chip->vtu_prob_irq;
+
+	err = request_threaded_irq(chip->vtu_prob_irq, NULL,
+				   mv88e6xxx_g1_vtu_prob_irq_thread_fn,
+				   IRQF_ONESHOT, "mv88e6xxx-g1-vtu-prob",
+				   chip);
+	if (err)
+		irq_dispose_mapping(chip->vtu_prob_irq);
+
+	return err;
+}
+
+void mv88e6xxx_g1_vtu_prob_irq_free(struct mv88e6xxx_chip *chip)
+{
+	free_irq(chip->vtu_prob_irq, chip);
+	irq_dispose_mapping(chip->vtu_prob_irq);
+}
diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c
index 58483af80bdb..30b1c8512049 100644
--- a/drivers/net/dummy.c
+++ b/drivers/net/dummy.c
@@ -42,48 +42,7 @@
 #define DRV_NAME	"dummy"
 #define DRV_VERSION	"1.0"
 
-#undef pr_fmt
-#define pr_fmt(fmt) DRV_NAME ": " fmt
-
 static int numdummies = 1;
-static int num_vfs;
-
-struct vf_data_storage {
-	u8	vf_mac[ETH_ALEN];
-	u16	pf_vlan; /* When set, guest VLAN config not allowed. */
-	u16	pf_qos;
-	__be16	vlan_proto;
-	u16	min_tx_rate;
-	u16	max_tx_rate;
-	u8	spoofchk_enabled;
-	bool	rss_query_enabled;
-	u8	trusted;
-	int	link_state;
-};
-
-struct dummy_priv {
-	struct vf_data_storage	*vfinfo;
-};
-
-static int dummy_num_vf(struct device *dev)
-{
-	return num_vfs;
-}
-
-static struct bus_type dummy_bus = {
-	.name	= "dummy",
-	.num_vf	= dummy_num_vf,
-};
-
-static void release_dummy_parent(struct device *dev)
-{
-}
-
-static struct device dummy_parent = {
-	.init_name	= "dummy",
-	.bus		= &dummy_bus,
-	.release	= release_dummy_parent,
-};
 
 /* fake multicast ability */
 static void set_multicast_list(struct net_device *dev)
@@ -133,25 +92,10 @@ static netdev_tx_t dummy_xmit(struct sk_buff *skb, struct net_device *dev)
 
 static int dummy_dev_init(struct net_device *dev)
 {
-	struct dummy_priv *priv = netdev_priv(dev);
-
 	dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
 	if (!dev->dstats)
 		return -ENOMEM;
 
-	priv->vfinfo = NULL;
-
-	if (!num_vfs)
-		return 0;
-
-	dev->dev.parent = &dummy_parent;
-	priv->vfinfo = kcalloc(num_vfs, sizeof(struct vf_data_storage),
-			       GFP_KERNEL);
-	if (!priv->vfinfo) {
-		free_percpu(dev->dstats);
-		return -ENOMEM;
-	}
-
 	return 0;
 }
 
@@ -169,117 +113,6 @@ static int dummy_change_carrier(struct net_device *dev, bool new_carrier)
 	return 0;
 }
 
-static int dummy_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if (!is_valid_ether_addr(mac) || (vf >= num_vfs))
-		return -EINVAL;
-
-	memcpy(priv->vfinfo[vf].vf_mac, mac, ETH_ALEN);
-
-	return 0;
-}
-
-static int dummy_set_vf_vlan(struct net_device *dev, int vf,
-			     u16 vlan, u8 qos, __be16 vlan_proto)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if ((vf >= num_vfs) || (vlan > 4095) || (qos > 7))
-		return -EINVAL;
-
-	priv->vfinfo[vf].pf_vlan = vlan;
-	priv->vfinfo[vf].pf_qos = qos;
-	priv->vfinfo[vf].vlan_proto = vlan_proto;
-
-	return 0;
-}
-
-static int dummy_set_vf_rate(struct net_device *dev, int vf, int min, int max)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if (vf >= num_vfs)
-		return -EINVAL;
-
-	priv->vfinfo[vf].min_tx_rate = min;
-	priv->vfinfo[vf].max_tx_rate = max;
-
-	return 0;
-}
-
-static int dummy_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if (vf >= num_vfs)
-		return -EINVAL;
-
-	priv->vfinfo[vf].spoofchk_enabled = val;
-
-	return 0;
-}
-
-static int dummy_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if (vf >= num_vfs)
-		return -EINVAL;
-
-	priv->vfinfo[vf].rss_query_enabled = val;
-
-	return 0;
-}
-
-static int dummy_set_vf_trust(struct net_device *dev, int vf, bool val)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if (vf >= num_vfs)
-		return -EINVAL;
-
-	priv->vfinfo[vf].trusted = val;
-
-	return 0;
-}
-
-static int dummy_get_vf_config(struct net_device *dev,
-			       int vf, struct ifla_vf_info *ivi)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if (vf >= num_vfs)
-		return -EINVAL;
-
-	ivi->vf = vf;
-	memcpy(&ivi->mac, priv->vfinfo[vf].vf_mac, ETH_ALEN);
-	ivi->vlan = priv->vfinfo[vf].pf_vlan;
-	ivi->qos = priv->vfinfo[vf].pf_qos;
-	ivi->spoofchk = priv->vfinfo[vf].spoofchk_enabled;
-	ivi->linkstate = priv->vfinfo[vf].link_state;
-	ivi->min_tx_rate = priv->vfinfo[vf].min_tx_rate;
-	ivi->max_tx_rate = priv->vfinfo[vf].max_tx_rate;
-	ivi->rss_query_en = priv->vfinfo[vf].rss_query_enabled;
-	ivi->trusted = priv->vfinfo[vf].trusted;
-	ivi->vlan_proto = priv->vfinfo[vf].vlan_proto;
-
-	return 0;
-}
-
-static int dummy_set_vf_link_state(struct net_device *dev, int vf, int state)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	if (vf >= num_vfs)
-		return -EINVAL;
-
-	priv->vfinfo[vf].link_state = state;
-
-	return 0;
-}
-
 static const struct net_device_ops dummy_netdev_ops = {
 	.ndo_init		= dummy_dev_init,
 	.ndo_uninit		= dummy_dev_uninit,
@@ -289,14 +122,6 @@ static const struct net_device_ops dummy_netdev_ops = {
 	.ndo_set_mac_address	= eth_mac_addr,
 	.ndo_get_stats64	= dummy_get_stats64,
 	.ndo_change_carrier	= dummy_change_carrier,
-	.ndo_set_vf_mac		= dummy_set_vf_mac,
-	.ndo_set_vf_vlan	= dummy_set_vf_vlan,
-	.ndo_set_vf_rate	= dummy_set_vf_rate,
-	.ndo_set_vf_spoofchk	= dummy_set_vf_spoofchk,
-	.ndo_set_vf_trust	= dummy_set_vf_trust,
-	.ndo_get_vf_config	= dummy_get_vf_config,
-	.ndo_set_vf_link_state	= dummy_set_vf_link_state,
-	.ndo_set_vf_rss_query_en = dummy_set_vf_rss_query_en,
 };
 
 static void dummy_get_drvinfo(struct net_device *dev,
@@ -323,13 +148,6 @@ static const struct ethtool_ops dummy_ethtool_ops = {
 	.get_ts_info		= dummy_get_ts_info,
 };
 
-static void dummy_free_netdev(struct net_device *dev)
-{
-	struct dummy_priv *priv = netdev_priv(dev);
-
-	kfree(priv->vfinfo);
-}
-
 static void dummy_setup(struct net_device *dev)
 {
 	ether_setup(dev);
@@ -338,7 +156,6 @@ static void dummy_setup(struct net_device *dev)
 	dev->netdev_ops = &dummy_netdev_ops;
 	dev->ethtool_ops = &dummy_ethtool_ops;
 	dev->needs_free_netdev = true;
-	dev->priv_destructor = dummy_free_netdev;
 
 	/* Fill in device structure with ethernet-generic values. */
 	dev->flags |= IFF_NOARP;
@@ -370,7 +187,6 @@ static int dummy_validate(struct nlattr *tb[], struct nlattr *data[],
 
 static struct rtnl_link_ops dummy_link_ops __read_mostly = {
 	.kind		= DRV_NAME,
-	.priv_size	= sizeof(struct dummy_priv),
 	.setup		= dummy_setup,
 	.validate	= dummy_validate,
 };
@@ -379,16 +195,12 @@ static struct rtnl_link_ops dummy_link_ops __read_mostly = {
 module_param(numdummies, int, 0);
 MODULE_PARM_DESC(numdummies, "Number of dummy pseudo devices");
 
-module_param(num_vfs, int, 0);
-MODULE_PARM_DESC(num_vfs, "Number of dummy VFs per dummy device");
-
 static int __init dummy_init_one(void)
 {
 	struct net_device *dev_dummy;
 	int err;
 
-	dev_dummy = alloc_netdev(sizeof(struct dummy_priv),
-				 "dummy%d", NET_NAME_ENUM, dummy_setup);
+	dev_dummy = alloc_netdev(0, "dummy%d", NET_NAME_ENUM, dummy_setup);
 	if (!dev_dummy)
 		return -ENOMEM;
 
@@ -407,21 +219,6 @@ static int __init dummy_init_module(void)
 {
 	int i, err = 0;
 
-	if (num_vfs) {
-		err = bus_register(&dummy_bus);
-		if (err < 0) {
-			pr_err("registering dummy bus failed\n");
-			return err;
-		}
-
-		err = device_register(&dummy_parent);
-		if (err < 0) {
-			pr_err("registering dummy parent device failed\n");
-			bus_unregister(&dummy_bus);
-			return err;
-		}
-	}
-
 	rtnl_lock();
 	err = __rtnl_link_register(&dummy_link_ops);
 	if (err < 0)
@@ -437,22 +234,12 @@ static int __init dummy_init_module(void)
 out:
 	rtnl_unlock();
 
-	if (err && num_vfs) {
-		device_unregister(&dummy_parent);
-		bus_unregister(&dummy_bus);
-	}
-
 	return err;
 }
 
 static void __exit dummy_cleanup_module(void)
 {
 	rtnl_link_unregister(&dummy_link_ops);
-
-	if (num_vfs) {
-		device_unregister(&dummy_parent);
-		bus_unregister(&dummy_bus);
-	}
 }
 
 module_init(dummy_init_module);
diff --git a/drivers/net/ethernet/8390/Kconfig b/drivers/net/ethernet/8390/Kconfig
index 29c3075bfb05..fdc673484add 100644
--- a/drivers/net/ethernet/8390/Kconfig
+++ b/drivers/net/ethernet/8390/Kconfig
@@ -3,7 +3,7 @@
 #
 
 config NET_VENDOR_8390
-	bool "National Semi-conductor 8390 devices"
+	bool "National Semiconductor 8390 devices"
 	default y
 	depends on NET_VENDOR_NATSEMI
 	---help---
diff --git a/drivers/net/ethernet/8390/mac8390.c b/drivers/net/ethernet/8390/mac8390.c
index 9497f18eaba0..2f91ce8dc614 100644
--- a/drivers/net/ethernet/8390/mac8390.c
+++ b/drivers/net/ethernet/8390/mac8390.c
@@ -123,7 +123,8 @@ enum mac8390_access {
 };
 
 extern int mac8390_memtest(struct net_device *dev);
-static int mac8390_initdev(struct net_device *dev, struct nubus_dev *ndev,
+static int mac8390_initdev(struct net_device *dev,
+			   struct nubus_rsrc *ndev,
 			   enum mac8390_type type);
 
 static int mac8390_open(struct net_device *dev);
@@ -169,11 +170,11 @@ static void word_memcpy_tocard(unsigned long tp, const void *fp, int count);
 static void word_memcpy_fromcard(void *tp, unsigned long fp, int count);
 static u32 mac8390_msg_enable;
 
-static enum mac8390_type __init mac8390_ident(struct nubus_dev *dev)
+static enum mac8390_type __init mac8390_ident(struct nubus_rsrc *fres)
 {
-	switch (dev->dr_sw) {
+	switch (fres->dr_sw) {
 	case NUBUS_DRSW_3COM:
-		switch (dev->dr_hw) {
+		switch (fres->dr_hw) {
 		case NUBUS_DRHW_APPLE_SONIC_NB:
 		case NUBUS_DRHW_APPLE_SONIC_LC:
 		case NUBUS_DRHW_SONNET:
@@ -184,7 +185,7 @@ static enum mac8390_type __init mac8390_ident(struct nubus_dev *dev)
 		break;
 
 	case NUBUS_DRSW_APPLE:
-		switch (dev->dr_hw) {
+		switch (fres->dr_hw) {
 		case NUBUS_DRHW_ASANTE_LC:
 			return MAC8390_NONE;
 		case NUBUS_DRHW_CABLETRON:
@@ -201,7 +202,7 @@ static enum mac8390_type __init mac8390_ident(struct nubus_dev *dev)
 	case NUBUS_DRSW_TECHWORKS:
 	case NUBUS_DRSW_DAYNA2:
 	case NUBUS_DRSW_DAYNA_LC:
-		if (dev->dr_hw == NUBUS_DRHW_CABLETRON)
+		if (fres->dr_hw == NUBUS_DRHW_CABLETRON)
 			return MAC8390_CABLETRON;
 		else
 			return MAC8390_APPLE;
@@ -212,7 +213,7 @@ static enum mac8390_type __init mac8390_ident(struct nubus_dev *dev)
 		break;
 
 	case NUBUS_DRSW_KINETICS:
-		switch (dev->dr_hw) {
+		switch (fres->dr_hw) {
 		case NUBUS_DRHW_INTERLAN:
 			return MAC8390_INTERLAN;
 		default:
@@ -225,8 +226,8 @@ static enum mac8390_type __init mac8390_ident(struct nubus_dev *dev)
 		 * These correspond to Dayna Sonic cards
 		 * which use the macsonic driver
 		 */
-		if (dev->dr_hw == NUBUS_DRHW_SMC9194 ||
-		    dev->dr_hw == NUBUS_DRHW_INTERLAN)
+		if (fres->dr_hw == NUBUS_DRHW_SMC9194 ||
+		    fres->dr_hw == NUBUS_DRHW_INTERLAN)
 			return MAC8390_NONE;
 		else
 			return MAC8390_DAYNA;
@@ -289,7 +290,8 @@ static int __init mac8390_memsize(unsigned long membase)
 	return i * 0x1000;
 }
 
-static bool __init mac8390_init(struct net_device *dev, struct nubus_dev *ndev,
+static bool __init mac8390_init(struct net_device *dev,
+				struct nubus_rsrc *ndev,
 				enum mac8390_type cardtype)
 {
 	struct nubus_dir dir;
@@ -394,7 +396,7 @@ static bool __init mac8390_init(struct net_device *dev, struct nubus_dev *ndev,
 struct net_device * __init mac8390_probe(int unit)
 {
 	struct net_device *dev;
-	struct nubus_dev *ndev = NULL;
+	struct nubus_rsrc *ndev = NULL;
 	int err = -ENODEV;
 	struct ei_device *ei_local;
 
@@ -414,8 +416,11 @@ struct net_device * __init mac8390_probe(int unit)
 	if (unit >= 0)
 		sprintf(dev->name, "eth%d", unit);
 
-	while ((ndev = nubus_find_type(NUBUS_CAT_NETWORK, NUBUS_TYPE_ETHERNET,
-				       ndev))) {
+	for_each_func_rsrc(ndev) {
+		if (ndev->category != NUBUS_CAT_NETWORK ||
+		    ndev->type != NUBUS_TYPE_ETHERNET)
+			continue;
+
 		/* Have we seen it already? */
 		if (slots & (1 << ndev->board->slot))
 			continue;
@@ -489,7 +494,7 @@ static const struct net_device_ops mac8390_netdev_ops = {
 };
 
 static int __init mac8390_initdev(struct net_device *dev,
-				  struct nubus_dev *ndev,
+				  struct nubus_rsrc *ndev,
 				  enum mac8390_type type)
 {
 	static u32 fwrd4_offsets[16] = {
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index c60421339a98..b6cf4b6962f5 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -42,6 +42,7 @@ source "drivers/net/ethernet/cavium/Kconfig"
 source "drivers/net/ethernet/chelsio/Kconfig"
 source "drivers/net/ethernet/cirrus/Kconfig"
 source "drivers/net/ethernet/cisco/Kconfig"
+source "drivers/net/ethernet/cortina/Kconfig"
 
 config CX_ECAT
 	tristate "Beckhoff CX5020 EtherCAT master support"
@@ -170,6 +171,7 @@ source "drivers/net/ethernet/sis/Kconfig"
 source "drivers/net/ethernet/sfc/Kconfig"
 source "drivers/net/ethernet/sgi/Kconfig"
 source "drivers/net/ethernet/smsc/Kconfig"
+source "drivers/net/ethernet/socionext/Kconfig"
 source "drivers/net/ethernet/stmicro/Kconfig"
 source "drivers/net/ethernet/sun/Kconfig"
 source "drivers/net/ethernet/tehuti/Kconfig"
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index 39f6273358ed..3cdf01e96e0b 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_NET_VENDOR_CAVIUM) += cavium/
 obj-$(CONFIG_NET_VENDOR_CHELSIO) += chelsio/
 obj-$(CONFIG_NET_VENDOR_CIRRUS) += cirrus/
 obj-$(CONFIG_NET_VENDOR_CISCO) += cisco/
+obj-$(CONFIG_NET_VENDOR_CORTINA) += cortina/
 obj-$(CONFIG_CX_ECAT) += ec_bhf.o
 obj-$(CONFIG_DM9000) += davicom/
 obj-$(CONFIG_DNET) += dnet.o
@@ -82,6 +83,7 @@ obj-$(CONFIG_SFC) += sfc/
 obj-$(CONFIG_SFC_FALCON) += sfc/falcon/
 obj-$(CONFIG_NET_VENDOR_SGI) += sgi/
 obj-$(CONFIG_NET_VENDOR_SMSC) += smsc/
+obj-$(CONFIG_NET_VENDOR_SOCIONEXT) += socionext/
 obj-$(CONFIG_NET_VENDOR_STMICRO) += stmicro/
 obj-$(CONFIG_NET_VENDOR_SUN) += sun/
 obj-$(CONFIG_NET_VENDOR_TEHUTI) += tehuti/
diff --git a/drivers/net/ethernet/alteon/acenic.c b/drivers/net/ethernet/alteon/acenic.c
index a1a52eb53b14..8f71b79b4949 100644
--- a/drivers/net/ethernet/alteon/acenic.c
+++ b/drivers/net/ethernet/alteon/acenic.c
@@ -1436,13 +1436,13 @@ static int ace_init(struct net_device *dev)
 	ace_set_txprd(regs, ap, 0);
 	writel(0, &regs->RxRetCsm);
 
-       /*
-	* Enable DMA engine now.
-	* If we do this sooner, Mckinley box pukes.
-	* I assume it's because Tigon II DMA engine wants to check
-	* *something* even before the CPU is started.
-	*/
-       writel(1, &regs->AssistState);  /* enable DMA */
+	/*
+	 * Enable DMA engine now.
+	 * If we do this sooner, Mckinley box pukes.
+	 * I assume it's because Tigon II DMA engine wants to check
+	 * *something* even before the CPU is started.
+	 */
+	writel(1, &regs->AssistState);  /* enable DMA */
 
 	/*
 	 * Start the NIC CPU
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.c b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
index b11e573ad57a..ea149c134e15 100644
--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.c
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.c
@@ -504,3 +504,14 @@ int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)
 
 	return 0;
 }
+
+bool ena_com_cq_empty(struct ena_com_io_cq *io_cq)
+{
+	struct ena_eth_io_rx_cdesc_base *cdesc;
+
+	cdesc = ena_com_get_next_rx_cdesc(io_cq);
+	if (cdesc)
+		return false;
+	else
+		return true;
+}
diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.h b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
index bb53c3a4f8e9..2f7657227cfe 100644
--- a/drivers/net/ethernet/amazon/ena/ena_eth_com.h
+++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.h
@@ -88,6 +88,8 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
 
 int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id);
 
+bool ena_com_cq_empty(struct ena_com_io_cq *io_cq);
+
 static inline void ena_com_unmask_intr(struct ena_com_io_cq *io_cq,
 				       struct ena_eth_io_intr_reg *intr_reg)
 {
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index fbe21a817bd8..6975150d144e 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -161,6 +161,8 @@ static void ena_init_io_rings_common(struct ena_adapter *adapter,
 	ring->per_napi_packets = 0;
 	ring->per_napi_bytes = 0;
 	ring->cpu = 0;
+	ring->first_interrupt = false;
+	ring->no_interrupt_event_cnt = 0;
 	u64_stats_init(&ring->syncp);
 }
 
@@ -1277,6 +1279,9 @@ static irqreturn_t ena_intr_msix_io(int irq, void *data)
 {
 	struct ena_napi *ena_napi = data;
 
+	ena_napi->tx_ring->first_interrupt = true;
+	ena_napi->rx_ring->first_interrupt = true;
+
 	napi_schedule_irqoff(&ena_napi->napi);
 
 	return IRQ_HANDLED;
@@ -2663,8 +2668,32 @@ static void ena_fw_reset_device(struct work_struct *work)
 	rtnl_unlock();
 }
 
-static int check_missing_comp_in_queue(struct ena_adapter *adapter,
-				       struct ena_ring *tx_ring)
+static int check_for_rx_interrupt_queue(struct ena_adapter *adapter,
+					struct ena_ring *rx_ring)
+{
+	if (likely(rx_ring->first_interrupt))
+		return 0;
+
+	if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
+		return 0;
+
+	rx_ring->no_interrupt_event_cnt++;
+
+	if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {
+		netif_err(adapter, rx_err, adapter->netdev,
+			  "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
+			  rx_ring->qid);
+		adapter->reset_reason = ENA_REGS_RESET_MISS_INTERRUPT;
+		smp_mb__before_atomic();
+		set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
+					  struct ena_ring *tx_ring)
 {
 	struct ena_tx_buffer *tx_buf;
 	unsigned long last_jiffies;
@@ -2674,8 +2703,27 @@ static int check_missing_comp_in_queue(struct ena_adapter *adapter,
 	for (i = 0; i < tx_ring->ring_size; i++) {
 		tx_buf = &tx_ring->tx_buffer_info[i];
 		last_jiffies = tx_buf->last_jiffies;
-		if (unlikely(last_jiffies &&
-			     time_is_before_jiffies(last_jiffies + adapter->missing_tx_completion_to))) {
+
+		if (last_jiffies == 0)
+			/* no pending Tx at this location */
+			continue;
+
+		if (unlikely(!tx_ring->first_interrupt && time_is_before_jiffies(last_jiffies +
+			     2 * adapter->missing_tx_completion_to))) {
+			/* If after graceful period interrupt is still not
+			 * received, we schedule a reset
+			 */
+			netif_err(adapter, tx_err, adapter->netdev,
+				  "Potential MSIX issue on Tx side Queue = %d. Reset the device\n",
+				  tx_ring->qid);
+			adapter->reset_reason = ENA_REGS_RESET_MISS_INTERRUPT;
+			smp_mb__before_atomic();
+			set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
+			return -EIO;
+		}
+
+		if (unlikely(time_is_before_jiffies(last_jiffies +
+				adapter->missing_tx_completion_to))) {
 			if (!tx_buf->print_once)
 				netif_notice(adapter, tx_err, adapter->netdev,
 					     "Found a Tx that wasn't completed on time, qid %d, index %d.\n",
@@ -2704,9 +2752,10 @@ static int check_missing_comp_in_queue(struct ena_adapter *adapter,
 	return rc;
 }
 
-static void check_for_missing_tx_completions(struct ena_adapter *adapter)
+static void check_for_missing_completions(struct ena_adapter *adapter)
 {
 	struct ena_ring *tx_ring;
+	struct ena_ring *rx_ring;
 	int i, budget, rc;
 
 	/* Make sure the driver doesn't turn the device in other process */
@@ -2725,8 +2774,13 @@ static void check_for_missing_tx_completions(struct ena_adapter *adapter)
 
 	for (i = adapter->last_monitored_tx_qid; i < adapter->num_queues; i++) {
 		tx_ring = &adapter->tx_ring[i];
+		rx_ring = &adapter->rx_ring[i];
+
+		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
+		if (unlikely(rc))
+			return;
 
-		rc = check_missing_comp_in_queue(adapter, tx_ring);
+		rc = check_for_rx_interrupt_queue(adapter, rx_ring);
 		if (unlikely(rc))
 			return;
 
@@ -2885,7 +2939,7 @@ static void ena_timer_service(struct timer_list *t)
 
 	check_for_admin_com_state(adapter);
 
-	check_for_missing_tx_completions(adapter);
+	check_for_missing_completions(adapter);
 
 	check_for_empty_rx_ring(adapter);
 
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h
index 3bbc003871de..f1972b5ab650 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.h
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h
@@ -44,7 +44,7 @@
 #include "ena_eth_com.h"
 
 #define DRV_MODULE_VER_MAJOR	1
-#define DRV_MODULE_VER_MINOR	3
+#define DRV_MODULE_VER_MINOR	5
 #define DRV_MODULE_VER_SUBMINOR 0
 
 #define DRV_MODULE_NAME		"ena"
@@ -122,6 +122,7 @@
  * We wait for 6 sec just to be on the safe side.
  */
 #define ENA_DEVICE_KALIVE_TIMEOUT	(6 * HZ)
+#define ENA_MAX_NO_INTERRUPT_ITERATIONS 3
 
 #define ENA_MMIO_DISABLE_REG_READ	BIT(0)
 
@@ -236,6 +237,9 @@ struct ena_ring {
 	/* The maximum header length the device can handle */
 	u8 tx_max_header_size;
 
+	bool first_interrupt;
+	u16 no_interrupt_event_cnt;
+
 	/* cpu for TPH */
 	int cpu;
 	 /* number of tx/rx_buffer_info's entries */
diff --git a/drivers/net/ethernet/amazon/ena/ena_regs_defs.h b/drivers/net/ethernet/amazon/ena/ena_regs_defs.h
index 9aec43c5bba8..48ca97fbe7bc 100644
--- a/drivers/net/ethernet/amazon/ena/ena_regs_defs.h
+++ b/drivers/net/ethernet/amazon/ena/ena_regs_defs.h
@@ -60,6 +60,8 @@ enum ena_regs_reset_reason_types {
 	ENA_REGS_RESET_USER_TRIGGER		= 12,
 
 	ENA_REGS_RESET_GENERIC			= 13,
+
+	ENA_REGS_RESET_MISS_INTERRUPT		= 14,
 };
 
 /* ena_registers offsets */
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index a74a8fbad53a..100adee778df 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -595,7 +595,7 @@ static void xgbe_isr_task(unsigned long data)
 
 		reissue_mask = 1 << 0;
 		if (!pdata->per_channel_irq)
-			reissue_mask |= 0xffff < 4;
+			reissue_mask |= 0xffff << 4;
 
 		XP_IOWRITE(pdata, XP_INT_REISSUE_EN, reissue_mask);
 	}
@@ -2930,9 +2930,8 @@ void xgbe_dump_rx_desc(struct xgbe_prv_data *pdata, struct xgbe_ring *ring,
 void xgbe_print_pkt(struct net_device *netdev, struct sk_buff *skb, bool tx_rx)
 {
 	struct ethhdr *eth = (struct ethhdr *)skb->data;
-	unsigned char *buf = skb->data;
 	unsigned char buffer[128];
-	unsigned int i, j;
+	unsigned int i;
 
 	netdev_dbg(netdev, "\n************** SKB dump ****************\n");
 
@@ -2943,22 +2942,13 @@ void xgbe_print_pkt(struct net_device *netdev, struct sk_buff *skb, bool tx_rx)
 	netdev_dbg(netdev, "Src MAC addr: %pM\n", eth->h_source);
 	netdev_dbg(netdev, "Protocol: %#06hx\n", ntohs(eth->h_proto));
 
-	for (i = 0, j = 0; i < skb->len;) {
-		j += snprintf(buffer + j, sizeof(buffer) - j, "%02hhx",
-			      buf[i++]);
-
-		if ((i % 32) == 0) {
-			netdev_dbg(netdev, "  %#06x: %s\n", i - 32, buffer);
-			j = 0;
-		} else if ((i % 16) == 0) {
-			buffer[j++] = ' ';
-			buffer[j++] = ' ';
-		} else if ((i % 4) == 0) {
-			buffer[j++] = ' ';
-		}
+	for (i = 0; i < skb->len; i += 32) {
+		unsigned int len = min(skb->len - i, 32U);
+
+		hex_dump_to_buffer(&skb->data[i], len, 32, 1,
+				   buffer, sizeof(buffer), false);
+		netdev_dbg(netdev, "  %#06x: %s\n", i, buffer);
 	}
-	if (i % 32)
-		netdev_dbg(netdev, "  %#06x: %s\n", i - (i % 32), buffer);
 
 	netdev_dbg(netdev, "\n************** SKB dump ****************\n");
 }
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
index 3e5833cf1fab..eb23f9ba1a9a 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c
@@ -426,6 +426,8 @@ static int xgbe_pci_resume(struct pci_dev *pdev)
 	struct net_device *netdev = pdata->netdev;
 	int ret = 0;
 
+	XP_IOWRITE(pdata, XP_INT_EN, 0x1fffff);
+
 	pdata->lpm_ctrl &= ~MDIO_CTRL1_LPOWER;
 	XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl);
 
diff --git a/drivers/net/ethernet/aquantia/atlantic/Makefile b/drivers/net/ethernet/aquantia/atlantic/Makefile
index e4ae696920ef..686f6d8c9e79 100644
--- a/drivers/net/ethernet/aquantia/atlantic/Makefile
+++ b/drivers/net/ethernet/aquantia/atlantic/Makefile
@@ -39,4 +39,5 @@ atlantic-objs := aq_main.o \
 	hw_atl/hw_atl_a0.o \
 	hw_atl/hw_atl_b0.o \
 	hw_atl/hw_atl_utils.o \
+	hw_atl/hw_atl_utils_fw2x.o \
 	hw_atl/hw_atl_llh.o
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h b/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h
index 105fdb958cef..fc7383106946 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_cfg.h
@@ -36,6 +36,8 @@
 #define AQ_CFG_TX_FRAME_MAX  (16U * 1024U)
 #define AQ_CFG_RX_FRAME_MAX  (4U * 1024U)
 
+#define AQ_CFG_TX_CLEAN_BUDGET 256U
+
 /* LRO */
 #define AQ_CFG_IS_LRO_DEF           1U
 
@@ -65,7 +67,13 @@
 
 /*#define AQ_CFG_MAC_ADDR_PERMANENT {0x30, 0x0E, 0xE3, 0x12, 0x34, 0x56}*/
 
-#define AQ_CFG_FC_MODE 3U
+#define AQ_NIC_FC_OFF    0U
+#define AQ_NIC_FC_TX     1U
+#define AQ_NIC_FC_RX     2U
+#define AQ_NIC_FC_FULL   3U
+#define AQ_NIC_FC_AUTO   4U
+
+#define AQ_CFG_FC_MODE AQ_NIC_FC_FULL
 
 #define AQ_CFG_SPEED_MSK  0xFFFFU	/* 0xFFFFU==auto_neg */
 
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_common.h b/drivers/net/ethernet/aquantia/atlantic/aq_common.h
index 9eb5e222a234..d52b088ff8f0 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_common.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_common.h
@@ -16,8 +16,45 @@
 #include <linux/pci.h>
 
 #include "ver.h"
-#include "aq_nic.h"
 #include "aq_cfg.h"
 #include "aq_utils.h"
 
+#define PCI_VENDOR_ID_AQUANTIA  0x1D6A
+
+#define AQ_DEVICE_ID_0001	0x0001
+#define AQ_DEVICE_ID_D100	0xD100
+#define AQ_DEVICE_ID_D107	0xD107
+#define AQ_DEVICE_ID_D108	0xD108
+#define AQ_DEVICE_ID_D109	0xD109
+
+#define AQ_DEVICE_ID_AQC100	0x00B1
+#define AQ_DEVICE_ID_AQC107	0x07B1
+#define AQ_DEVICE_ID_AQC108	0x08B1
+#define AQ_DEVICE_ID_AQC109	0x09B1
+#define AQ_DEVICE_ID_AQC111	0x11B1
+#define AQ_DEVICE_ID_AQC112	0x12B1
+
+#define AQ_DEVICE_ID_AQC100S	0x80B1
+#define AQ_DEVICE_ID_AQC107S	0x87B1
+#define AQ_DEVICE_ID_AQC108S	0x88B1
+#define AQ_DEVICE_ID_AQC109S	0x89B1
+#define AQ_DEVICE_ID_AQC111S	0x91B1
+#define AQ_DEVICE_ID_AQC112S	0x92B1
+
+#define AQ_DEVICE_ID_AQC111E	0x51B1
+#define AQ_DEVICE_ID_AQC112E	0x52B1
+
+#define HW_ATL_NIC_NAME "aQuantia AQtion 10Gbit Network Adapter"
+
+#define AQ_HWREV_ANY	0
+#define AQ_HWREV_1	1
+#define AQ_HWREV_2	2
+
+#define AQ_NIC_RATE_10G        BIT(0)
+#define AQ_NIC_RATE_5G         BIT(1)
+#define AQ_NIC_RATE_5GSR       BIT(2)
+#define AQ_NIC_RATE_2GS        BIT(3)
+#define AQ_NIC_RATE_1G         BIT(4)
+#define AQ_NIC_RATE_100M       BIT(5)
+
 #endif /* AQ_COMMON_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
index b3825de6cdfb..a2d416b24ffc 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h
@@ -7,7 +7,7 @@
  * version 2, as published by the Free Software Foundation.
  */
 
-/* File aq_hw.h: Declaraion of abstract interface for NIC hardware specific
+/* File aq_hw.h: Declaration of abstract interface for NIC hardware specific
  * functions.
  */
 
@@ -15,12 +15,15 @@
 #define AQ_HW_H
 
 #include "aq_common.h"
+#include "aq_rss.h"
+#include "hw_atl/hw_atl_utils.h"
 
 /* NIC H/W capabilities */
 struct aq_hw_caps_s {
 	u64 hw_features;
 	u64 link_speed_msk;
 	unsigned int hw_priv_flags;
+	u32 media_type;
 	u32 rxds;
 	u32 txds;
 	u32 txhwb_alignment;
@@ -28,7 +31,7 @@ struct aq_hw_caps_s {
 	u32 vecs;
 	u32 mtu;
 	u32 mac_regs_count;
-	u8 ports;
+	u32 hw_alive_check_addr;
 	u8 msix_irqs;
 	u8 tcs;
 	u8 rxd_alignment;
@@ -39,7 +42,6 @@ struct aq_hw_caps_s {
 	u8 rx_rings;
 	bool flow_control;
 	bool is_64_dma;
-	u32 fw_ver_expected;
 };
 
 struct aq_hw_link_status_s {
@@ -86,30 +88,43 @@ struct aq_stats_s {
 
 #define AQ_HW_FLAG_ERRORS      (AQ_HW_FLAG_ERR_HW | AQ_HW_FLAG_ERR_UNPLUG)
 
+#define AQ_NIC_FLAGS_IS_NOT_READY (AQ_NIC_FLAG_STOPPING | \
+			AQ_NIC_FLAG_RESETTING | AQ_NIC_FLAG_CLOSING | \
+			AQ_NIC_FLAG_ERR_UNPLUG | AQ_NIC_FLAG_ERR_HW)
+
+#define AQ_NIC_FLAGS_IS_NOT_TX_READY (AQ_NIC_FLAGS_IS_NOT_READY | \
+					AQ_NIC_LINK_DOWN)
+
+#define AQ_HW_MEDIA_TYPE_TP    1U
+#define AQ_HW_MEDIA_TYPE_FIBRE 2U
+
 struct aq_hw_s {
-	struct aq_obj_s header;
+	atomic_t flags;
+	u8 rbl_enabled:1;
 	struct aq_nic_cfg_s *aq_nic_cfg;
-	struct aq_pci_func_s *aq_pci_func;
+	const struct aq_fw_ops *aq_fw_ops;
 	void __iomem *mmio;
-	unsigned int not_ff_addr;
 	struct aq_hw_link_status_s aq_link_status;
+	struct hw_aq_atl_utils_mbox mbox;
+	struct hw_atl_stats_s last_stats;
+	struct aq_stats_s curr_stats;
+	u64 speed;
+	u32 itr_tx;
+	u32 itr_rx;
+	unsigned int chip_features;
+	u32 fw_ver_actual;
+	atomic_t dpc;
+	u32 mbox_addr;
+	u32 rpc_addr;
+	u32 rpc_tid;
+	struct hw_aq_atl_utils_fw_rpc rpc;
 };
 
 struct aq_ring_s;
 struct aq_ring_param_s;
-struct aq_nic_cfg_s;
 struct sk_buff;
 
 struct aq_hw_ops {
-	struct aq_hw_s *(*create)(struct aq_pci_func_s *aq_pci_func,
-				  unsigned int port, struct aq_hw_ops *ops);
-
-	void (*destroy)(struct aq_hw_s *self);
-
-	int (*get_hw_caps)(struct aq_hw_s *self,
-			   struct aq_hw_caps_s *aq_hw_caps,
-			   unsigned short device,
-			   unsigned short subsystem_device);
 
 	int (*hw_ring_tx_xmit)(struct aq_hw_s *self, struct aq_ring_s *aq_ring,
 			       unsigned int frags);
@@ -123,20 +138,11 @@ struct aq_hw_ops {
 	int (*hw_ring_tx_head_update)(struct aq_hw_s *self,
 				      struct aq_ring_s *aq_ring);
 
-	int (*hw_get_mac_permanent)(struct aq_hw_s *self,
-				    struct aq_hw_caps_s *aq_hw_caps,
-				    u8 *mac);
-
 	int (*hw_set_mac_address)(struct aq_hw_s *self, u8 *mac_addr);
 
-	int (*hw_get_link_status)(struct aq_hw_s *self);
-
-	int (*hw_set_link_speed)(struct aq_hw_s *self, u32 speed);
-
 	int (*hw_reset)(struct aq_hw_s *self);
 
-	int (*hw_init)(struct aq_hw_s *self, struct aq_nic_cfg_s *aq_nic_cfg,
-		       u8 *mac_addr);
+	int (*hw_init)(struct aq_hw_s *self, u8 *mac_addr);
 
 	int (*hw_start)(struct aq_hw_s *self);
 
@@ -184,9 +190,8 @@ struct aq_hw_ops {
 			       struct aq_rss_parameters *rss_params);
 
 	int (*hw_get_regs)(struct aq_hw_s *self,
-			   struct aq_hw_caps_s *aq_hw_caps, u32 *regs_buff);
-
-	int (*hw_update_stats)(struct aq_hw_s *self);
+			   const struct aq_hw_caps_s *aq_hw_caps,
+			   u32 *regs_buff);
 
 	struct aq_stats_s *(*hw_get_hw_stats)(struct aq_hw_s *self);
 
@@ -197,4 +202,20 @@ struct aq_hw_ops {
 	int (*hw_set_power)(struct aq_hw_s *self, unsigned int power_state);
 };
 
+struct aq_fw_ops {
+	int (*init)(struct aq_hw_s *self);
+
+	int (*reset)(struct aq_hw_s *self);
+
+	int (*get_mac_permanent)(struct aq_hw_s *self, u8 *mac);
+
+	int (*set_link_speed)(struct aq_hw_s *self, u32 speed);
+
+	int (*set_state)(struct aq_hw_s *self, enum hal_atl_utils_fw_state_e state);
+
+	int (*update_link_status)(struct aq_hw_s *self);
+
+	int (*update_stats)(struct aq_hw_s *self);
+};
+
 #endif /* AQ_HW_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.c b/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.c
index 5f13465995f6..d526c4f19d34 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.c
@@ -13,6 +13,7 @@
 
 #include "aq_hw_utils.h"
 #include "aq_hw.h"
+#include "aq_nic.h"
 
 void aq_hw_write_reg_bit(struct aq_hw_s *aq_hw, u32 addr, u32 msk,
 			 u32 shift, u32 val)
@@ -39,8 +40,10 @@ u32 aq_hw_read_reg(struct aq_hw_s *hw, u32 reg)
 {
 	u32 value = readl(hw->mmio + reg);
 
-	if ((~0U) == value && (~0U) == readl(hw->mmio + hw->not_ff_addr))
-		aq_utils_obj_set(&hw->header.flags, AQ_HW_FLAG_ERR_UNPLUG);
+	if ((~0U) == value &&
+	    (~0U) == readl(hw->mmio +
+			   hw->aq_nic_cfg->aq_hw_caps->hw_alive_check_addr))
+		aq_utils_obj_set(&hw->flags, AQ_HW_FLAG_ERR_UNPLUG);
 
 	return value;
 }
@@ -54,11 +57,11 @@ int aq_hw_err_from_flags(struct aq_hw_s *hw)
 {
 	int err = 0;
 
-	if (aq_utils_obj_test(&hw->header.flags, AQ_HW_FLAG_ERR_UNPLUG)) {
+	if (aq_utils_obj_test(&hw->flags, AQ_HW_FLAG_ERR_UNPLUG)) {
 		err = -ENXIO;
 		goto err_exit;
 	}
-	if (aq_utils_obj_test(&hw->header.flags, AQ_HW_FLAG_ERR_HW)) {
+	if (aq_utils_obj_test(&hw->flags, AQ_HW_FLAG_ERR_HW)) {
 		err = -EIO;
 		goto err_exit;
 	}
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.h
index 03b72ddbffb9..dc88a1221f1d 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw_utils.h
@@ -35,6 +35,9 @@ do { \
 	} \
 } while (0)
 
+#define aq_pr_err(...) pr_err(AQ_CFG_DRV_NAME ": " __VA_ARGS__)
+#define aq_pr_trace(...) pr_info(AQ_CFG_DRV_NAME ": " __VA_ARGS__)
+
 struct aq_hw_s;
 
 void aq_hw_write_reg_bit(struct aq_hw_s *aq_hw, u32 addr, u32 msk,
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_main.c b/drivers/net/ethernet/aquantia/atlantic/aq_main.c
index 5d6c40d86775..ba5fe8c4125d 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_main.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_main.c
@@ -13,49 +13,39 @@
 #include "aq_nic.h"
 #include "aq_pci_func.h"
 #include "aq_ethtool.h"
-#include "hw_atl/hw_atl_a0.h"
-#include "hw_atl/hw_atl_b0.h"
 
 #include <linux/netdevice.h>
 #include <linux/module.h>
 
-static const struct pci_device_id aq_pci_tbl[] = {
-	{ PCI_VDEVICE(AQUANTIA, HW_ATL_DEVICE_ID_0001), },
-	{ PCI_VDEVICE(AQUANTIA, HW_ATL_DEVICE_ID_D100), },
-	{ PCI_VDEVICE(AQUANTIA, HW_ATL_DEVICE_ID_D107), },
-	{ PCI_VDEVICE(AQUANTIA, HW_ATL_DEVICE_ID_D108), },
-	{ PCI_VDEVICE(AQUANTIA, HW_ATL_DEVICE_ID_D109), },
-	{}
-};
-
-MODULE_DEVICE_TABLE(pci, aq_pci_tbl);
-
 MODULE_LICENSE("GPL v2");
 MODULE_VERSION(AQ_CFG_DRV_VERSION);
 MODULE_AUTHOR(AQ_CFG_DRV_AUTHOR);
 MODULE_DESCRIPTION(AQ_CFG_DRV_DESC);
 
-static struct aq_hw_ops *aq_pci_probe_get_hw_ops_by_id(struct pci_dev *pdev)
+static const struct net_device_ops aq_ndev_ops;
+
+struct net_device *aq_ndev_alloc(void)
 {
-	struct aq_hw_ops *ops = NULL;
+	struct net_device *ndev = NULL;
+	struct aq_nic_s *aq_nic = NULL;
+
+	ndev = alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_CFG_VECS_MAX);
+	if (!ndev)
+		return NULL;
 
-	ops = hw_atl_a0_get_ops_by_id(pdev);
-	if (!ops)
-		ops = hw_atl_b0_get_ops_by_id(pdev);
+	aq_nic = netdev_priv(ndev);
+	aq_nic->ndev = ndev;
+	ndev->netdev_ops = &aq_ndev_ops;
+	ndev->ethtool_ops = &aq_ethtool_ops;
 
-	return ops;
+	return ndev;
 }
 
 static int aq_ndev_open(struct net_device *ndev)
 {
-	struct aq_nic_s *aq_nic = NULL;
 	int err = 0;
+	struct aq_nic_s *aq_nic = netdev_priv(ndev);
 
-	aq_nic = aq_nic_alloc_hot(ndev);
-	if (!aq_nic) {
-		err = -ENOMEM;
-		goto err_exit;
-	}
 	err = aq_nic_init(aq_nic);
 	if (err < 0)
 		goto err_exit;
@@ -78,7 +68,6 @@ static int aq_ndev_close(struct net_device *ndev)
 	if (err < 0)
 		goto err_exit;
 	aq_nic_deinit(aq_nic);
-	aq_nic_free_hot_resources(aq_nic);
 
 err_exit:
 	return err;
@@ -150,15 +139,13 @@ static void aq_ndev_set_multicast_settings(struct net_device *ndev)
 
 	err = aq_nic_set_packet_filter(aq_nic, ndev->flags);
 	if (err < 0)
-		goto err_exit;
+		return;
 
 	if (netdev_mc_count(ndev)) {
 		err = aq_nic_set_multicast_list(aq_nic, ndev);
 		if (err < 0)
-			goto err_exit;
+			return;
 	}
-
-err_exit:;
 }
 
 static const struct net_device_ops aq_ndev_ops = {
@@ -170,66 +157,3 @@ static const struct net_device_ops aq_ndev_ops = {
 	.ndo_set_mac_address = aq_ndev_set_mac_address,
 	.ndo_set_features = aq_ndev_set_features
 };
-
-static int aq_pci_probe(struct pci_dev *pdev,
-			const struct pci_device_id *pci_id)
-{
-	struct aq_hw_ops *aq_hw_ops = NULL;
-	struct aq_pci_func_s *aq_pci_func = NULL;
-	int err = 0;
-
-	err = pci_enable_device(pdev);
-	if (err < 0)
-		goto err_exit;
-	aq_hw_ops = aq_pci_probe_get_hw_ops_by_id(pdev);
-	aq_pci_func = aq_pci_func_alloc(aq_hw_ops, pdev,
-					&aq_ndev_ops, &aq_ethtool_ops);
-	if (!aq_pci_func) {
-		err = -ENOMEM;
-		goto err_exit;
-	}
-	err = aq_pci_func_init(aq_pci_func);
-	if (err < 0)
-		goto err_exit;
-
-err_exit:
-	if (err < 0) {
-		if (aq_pci_func)
-			aq_pci_func_free(aq_pci_func);
-	}
-	return err;
-}
-
-static void aq_pci_remove(struct pci_dev *pdev)
-{
-	struct aq_pci_func_s *aq_pci_func = pci_get_drvdata(pdev);
-
-	aq_pci_func_deinit(aq_pci_func);
-	aq_pci_func_free(aq_pci_func);
-}
-
-static int aq_pci_suspend(struct pci_dev *pdev, pm_message_t pm_msg)
-{
-	struct aq_pci_func_s *aq_pci_func = pci_get_drvdata(pdev);
-
-	return aq_pci_func_change_pm_state(aq_pci_func, &pm_msg);
-}
-
-static int aq_pci_resume(struct pci_dev *pdev)
-{
-	struct aq_pci_func_s *aq_pci_func = pci_get_drvdata(pdev);
-	pm_message_t pm_msg = PMSG_RESTORE;
-
-	return aq_pci_func_change_pm_state(aq_pci_func, &pm_msg);
-}
-
-static struct pci_driver aq_pci_ops = {
-	.name = AQ_CFG_DRV_NAME,
-	.id_table = aq_pci_tbl,
-	.probe = aq_pci_probe,
-	.remove = aq_pci_remove,
-	.suspend = aq_pci_suspend,
-	.resume = aq_pci_resume,
-};
-
-module_pci_driver(aq_pci_ops);
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_main.h b/drivers/net/ethernet/aquantia/atlantic/aq_main.h
index 9748e7e575e0..ce92152eb43e 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_main.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_main.h
@@ -14,4 +14,6 @@
 
 #include "aq_common.h"
 
+struct net_device *aq_ndev_alloc(void);
+
 #endif /* AQ_MAIN_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
index 75a894a9251c..c96a92118b8b 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.c
@@ -14,7 +14,6 @@
 #include "aq_vec.h"
 #include "aq_hw.h"
 #include "aq_pci_func.h"
-#include "aq_nic_internal.h"
 
 #include <linux/moduleparam.h>
 #include <linux/netdevice.h>
@@ -61,19 +60,13 @@ static void aq_nic_rss_init(struct aq_nic_s *self, unsigned int num_rss_queues)
 		rss_params->indirection_table[i] = i & (num_rss_queues - 1);
 }
 
-/* Fills aq_nic_cfg with valid defaults */
-static void aq_nic_cfg_init_defaults(struct aq_nic_s *self)
+/* Checks hw_caps and 'corrects' aq_nic_cfg in runtime */
+void aq_nic_cfg_start(struct aq_nic_s *self)
 {
 	struct aq_nic_cfg_s *cfg = &self->aq_nic_cfg;
 
-	cfg->aq_hw_caps = &self->aq_hw_caps;
-
-	cfg->vecs = AQ_CFG_VECS_DEF;
 	cfg->tcs = AQ_CFG_TCS_DEF;
 
-	cfg->rxds = AQ_CFG_RXDS_DEF;
-	cfg->txds = AQ_CFG_TXDS_DEF;
-
 	cfg->is_polling = AQ_CFG_IS_POLLING_DEF;
 
 	cfg->itr = aq_itr;
@@ -94,19 +87,13 @@ static void aq_nic_cfg_init_defaults(struct aq_nic_s *self)
 	cfg->vlan_id = 0U;
 
 	aq_nic_rss_init(self, cfg->num_rss_queues);
-}
-
-/* Checks hw_caps and 'corrects' aq_nic_cfg in runtime */
-int aq_nic_cfg_start(struct aq_nic_s *self)
-{
-	struct aq_nic_cfg_s *cfg = &self->aq_nic_cfg;
 
 	/*descriptors */
-	cfg->rxds = min(cfg->rxds, cfg->aq_hw_caps->rxds);
-	cfg->txds = min(cfg->txds, cfg->aq_hw_caps->txds);
+	cfg->rxds = min(cfg->aq_hw_caps->rxds, AQ_CFG_RXDS_DEF);
+	cfg->txds = min(cfg->aq_hw_caps->txds, AQ_CFG_TXDS_DEF);
 
 	/*rss rings */
-	cfg->vecs = min(cfg->vecs, cfg->aq_hw_caps->vecs);
+	cfg->vecs = min(cfg->aq_hw_caps->vecs, AQ_CFG_VECS_DEF);
 	cfg->vecs = min(cfg->vecs, num_online_cpus());
 	/* cfg->vecs should be power of 2 for RSS */
 	if (cfg->vecs >= 8U)
@@ -120,23 +107,22 @@ int aq_nic_cfg_start(struct aq_nic_s *self)
 
 	cfg->num_rss_queues = min(cfg->vecs, AQ_CFG_NUM_RSS_QUEUES_DEF);
 
-	cfg->irq_type = aq_pci_func_get_irq_type(self->aq_pci_func);
+	cfg->irq_type = aq_pci_func_get_irq_type(self);
 
 	if ((cfg->irq_type == AQ_HW_IRQ_LEGACY) ||
-	    (self->aq_hw_caps.vecs == 1U) ||
+	    (cfg->aq_hw_caps->vecs == 1U) ||
 	    (cfg->vecs == 1U)) {
 		cfg->is_rss = 0U;
 		cfg->vecs = 1U;
 	}
 
-	cfg->link_speed_msk &= self->aq_hw_caps.link_speed_msk;
-	cfg->hw_features = self->aq_hw_caps.hw_features;
-	return 0;
+	cfg->link_speed_msk &= cfg->aq_hw_caps->link_speed_msk;
+	cfg->hw_features = cfg->aq_hw_caps->hw_features;
 }
 
 static int aq_nic_update_link_status(struct aq_nic_s *self)
 {
-	int err = self->aq_hw_ops.hw_get_link_status(self->aq_hw);
+	int err = self->aq_fw_ops->update_link_status(self->aq_hw);
 
 	if (err)
 		return err;
@@ -150,9 +136,9 @@ static int aq_nic_update_link_status(struct aq_nic_s *self)
 
 	self->link_status = self->aq_hw->aq_link_status;
 	if (!netif_carrier_ok(self->ndev) && self->link_status.mbps) {
-		aq_utils_obj_set(&self->header.flags,
+		aq_utils_obj_set(&self->flags,
 				 AQ_NIC_FLAG_STARTED);
-		aq_utils_obj_clear(&self->header.flags,
+		aq_utils_obj_clear(&self->flags,
 				   AQ_NIC_LINK_DOWN);
 		netif_carrier_on(self->ndev);
 		netif_tx_wake_all_queues(self->ndev);
@@ -160,7 +146,7 @@ static int aq_nic_update_link_status(struct aq_nic_s *self)
 	if (netif_carrier_ok(self->ndev) && !self->link_status.mbps) {
 		netif_carrier_off(self->ndev);
 		netif_tx_disable(self->ndev);
-		aq_utils_obj_set(&self->header.flags, AQ_NIC_LINK_DOWN);
+		aq_utils_obj_set(&self->flags, AQ_NIC_LINK_DOWN);
 	}
 	return 0;
 }
@@ -171,15 +157,15 @@ static void aq_nic_service_timer_cb(struct timer_list *t)
 	int ctimer = AQ_CFG_SERVICE_TIMER_INTERVAL;
 	int err = 0;
 
-	if (aq_utils_obj_test(&self->header.flags, AQ_NIC_FLAGS_IS_NOT_READY))
+	if (aq_utils_obj_test(&self->flags, AQ_NIC_FLAGS_IS_NOT_READY))
 		goto err_exit;
 
 	err = aq_nic_update_link_status(self);
 	if (err)
 		goto err_exit;
 
-	if (self->aq_hw_ops.hw_update_stats)
-		self->aq_hw_ops.hw_update_stats(self->aq_hw);
+	if (self->aq_fw_ops->update_stats)
+		self->aq_fw_ops->update_stats(self->aq_hw);
 
 	aq_nic_update_ndev_stats(self);
 
@@ -205,60 +191,6 @@ static void aq_nic_polling_timer_cb(struct timer_list *t)
 		AQ_CFG_POLLING_TIMER_INTERVAL);
 }
 
-static struct net_device *aq_nic_ndev_alloc(void)
-{
-	return alloc_etherdev_mq(sizeof(struct aq_nic_s), AQ_CFG_VECS_MAX);
-}
-
-struct aq_nic_s *aq_nic_alloc_cold(const struct net_device_ops *ndev_ops,
-				   const struct ethtool_ops *et_ops,
-				   struct pci_dev *pdev,
-				   struct aq_pci_func_s *aq_pci_func,
-				   unsigned int port,
-				   const struct aq_hw_ops *aq_hw_ops)
-{
-	struct net_device *ndev = NULL;
-	struct aq_nic_s *self = NULL;
-	int err = 0;
-
-	ndev = aq_nic_ndev_alloc();
-	if (!ndev) {
-		err = -ENOMEM;
-		goto err_exit;
-	}
-
-	self = netdev_priv(ndev);
-
-	ndev->netdev_ops = ndev_ops;
-	ndev->ethtool_ops = et_ops;
-
-	SET_NETDEV_DEV(ndev, &pdev->dev);
-
-	ndev->if_port = port;
-	self->ndev = ndev;
-
-	self->aq_pci_func = aq_pci_func;
-
-	self->aq_hw_ops = *aq_hw_ops;
-	self->port = (u8)port;
-
-	self->aq_hw = self->aq_hw_ops.create(aq_pci_func, self->port,
-						&self->aq_hw_ops);
-	err = self->aq_hw_ops.get_hw_caps(self->aq_hw, &self->aq_hw_caps,
-					  pdev->device, pdev->subsystem_device);
-	if (err < 0)
-		goto err_exit;
-
-	aq_nic_cfg_init_defaults(self);
-
-err_exit:
-	if (err < 0) {
-		aq_nic_free_hot_resources(self);
-		self = NULL;
-	}
-	return self;
-}
-
 int aq_nic_ndev_register(struct aq_nic_s *self)
 {
 	int err = 0;
@@ -267,10 +199,14 @@ int aq_nic_ndev_register(struct aq_nic_s *self)
 		err = -EINVAL;
 		goto err_exit;
 	}
-	err = self->aq_hw_ops.hw_get_mac_permanent(self->aq_hw,
-			    self->aq_nic_cfg.aq_hw_caps,
+
+	err = hw_atl_utils_initfw(self->aq_hw, &self->aq_fw_ops);
+	if (err)
+		goto err_exit;
+
+	err = self->aq_fw_ops->get_mac_permanent(self->aq_hw,
 			    self->ndev->dev_addr);
-	if (err < 0)
+	if (err)
 		goto err_exit;
 
 #if defined(AQ_CFG_MAC_ADDR_PERMANENT)
@@ -281,83 +217,41 @@ int aq_nic_ndev_register(struct aq_nic_s *self)
 	}
 #endif
 
+	for (self->aq_vecs = 0; self->aq_vecs < aq_nic_get_cfg(self)->vecs;
+	     self->aq_vecs++) {
+		self->aq_vec[self->aq_vecs] =
+		    aq_vec_alloc(self, self->aq_vecs, aq_nic_get_cfg(self));
+		if (!self->aq_vec[self->aq_vecs]) {
+			err = -ENOMEM;
+			goto err_exit;
+		}
+	}
+
 	netif_carrier_off(self->ndev);
 
 	netif_tx_disable(self->ndev);
 
 	err = register_netdev(self->ndev);
-	if (err < 0)
+	if (err)
 		goto err_exit;
 
 err_exit:
 	return err;
 }
 
-int aq_nic_ndev_init(struct aq_nic_s *self)
+void aq_nic_ndev_init(struct aq_nic_s *self)
 {
-	struct aq_hw_caps_s *aq_hw_caps = self->aq_nic_cfg.aq_hw_caps;
+	const struct aq_hw_caps_s *aq_hw_caps = self->aq_nic_cfg.aq_hw_caps;
 	struct aq_nic_cfg_s *aq_nic_cfg = &self->aq_nic_cfg;
 
 	self->ndev->hw_features |= aq_hw_caps->hw_features;
 	self->ndev->features = aq_hw_caps->hw_features;
 	self->ndev->priv_flags = aq_hw_caps->hw_priv_flags;
-	self->ndev->mtu = aq_nic_cfg->mtu - ETH_HLEN;
-	self->ndev->max_mtu = self->aq_hw_caps.mtu - ETH_FCS_LEN - ETH_HLEN;
-
-	return 0;
-}
+	self->ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
 
-void aq_nic_ndev_free(struct aq_nic_s *self)
-{
-	if (!self->ndev)
-		goto err_exit;
-
-	if (self->ndev->reg_state == NETREG_REGISTERED)
-		unregister_netdev(self->ndev);
-
-	if (self->aq_hw)
-		self->aq_hw_ops.destroy(self->aq_hw);
-
-	free_netdev(self->ndev);
-
-err_exit:;
-}
-
-struct aq_nic_s *aq_nic_alloc_hot(struct net_device *ndev)
-{
-	struct aq_nic_s *self = NULL;
-	int err = 0;
-
-	if (!ndev) {
-		err = -EINVAL;
-		goto err_exit;
-	}
-	self = netdev_priv(ndev);
-
-	if (!self) {
-		err = -EINVAL;
-		goto err_exit;
-	}
-	if (netif_running(ndev))
-		netif_tx_disable(ndev);
-	netif_carrier_off(self->ndev);
-
-	for (self->aq_vecs = 0; self->aq_vecs < self->aq_nic_cfg.vecs;
-		self->aq_vecs++) {
-		self->aq_vec[self->aq_vecs] =
-		    aq_vec_alloc(self, self->aq_vecs, &self->aq_nic_cfg);
-		if (!self->aq_vec[self->aq_vecs]) {
-			err = -ENOMEM;
-			goto err_exit;
-		}
-	}
+	self->ndev->mtu = aq_nic_cfg->mtu - ETH_HLEN;
+	self->ndev->max_mtu = aq_hw_caps->mtu - ETH_FCS_LEN - ETH_HLEN;
 
-err_exit:
-	if (err < 0) {
-		aq_nic_free_hot_resources(self);
-		self = NULL;
-	}
-	return self;
 }
 
 void aq_nic_set_tx_ring(struct aq_nic_s *self, unsigned int idx,
@@ -366,11 +260,6 @@ void aq_nic_set_tx_ring(struct aq_nic_s *self, unsigned int idx,
 	self->aq_ring_tx[idx] = ring;
 }
 
-struct device *aq_nic_get_dev(struct aq_nic_s *self)
-{
-	return self->ndev->dev.parent;
-}
-
 struct net_device *aq_nic_get_ndev(struct aq_nic_s *self)
 {
 	return self->ndev;
@@ -383,18 +272,20 @@ int aq_nic_init(struct aq_nic_s *self)
 	unsigned int i = 0U;
 
 	self->power_state = AQ_HW_POWER_STATE_D0;
-	err = self->aq_hw_ops.hw_reset(self->aq_hw);
+	err = self->aq_hw_ops->hw_reset(self->aq_hw);
 	if (err < 0)
 		goto err_exit;
 
-	err = self->aq_hw_ops.hw_init(self->aq_hw, &self->aq_nic_cfg,
-			    aq_nic_get_ndev(self)->dev_addr);
+	err = self->aq_hw_ops->hw_init(self->aq_hw,
+				       aq_nic_get_ndev(self)->dev_addr);
 	if (err < 0)
 		goto err_exit;
 
 	for (i = 0U, aq_vec = self->aq_vec[0];
 		self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i])
-		aq_vec_init(aq_vec, &self->aq_hw_ops, self->aq_hw);
+		aq_vec_init(aq_vec, self->aq_hw_ops, self->aq_hw);
+
+	netif_carrier_off(self->ndev);
 
 err_exit:
 	return err;
@@ -406,13 +297,13 @@ int aq_nic_start(struct aq_nic_s *self)
 	int err = 0;
 	unsigned int i = 0U;
 
-	err = self->aq_hw_ops.hw_multicast_list_set(self->aq_hw,
+	err = self->aq_hw_ops->hw_multicast_list_set(self->aq_hw,
 						    self->mc_list.ar,
 						    self->mc_list.count);
 	if (err < 0)
 		goto err_exit;
 
-	err = self->aq_hw_ops.hw_packet_filter_set(self->aq_hw,
+	err = self->aq_hw_ops->hw_packet_filter_set(self->aq_hw,
 						   self->packet_filter);
 	if (err < 0)
 		goto err_exit;
@@ -424,7 +315,7 @@ int aq_nic_start(struct aq_nic_s *self)
 			goto err_exit;
 	}
 
-	err = self->aq_hw_ops.hw_start(self->aq_hw);
+	err = self->aq_hw_ops->hw_start(self->aq_hw);
 	if (err < 0)
 		goto err_exit;
 
@@ -442,14 +333,14 @@ int aq_nic_start(struct aq_nic_s *self)
 	} else {
 		for (i = 0U, aq_vec = self->aq_vec[0];
 			self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) {
-			err = aq_pci_func_alloc_irq(self->aq_pci_func, i,
+			err = aq_pci_func_alloc_irq(self, i,
 						    self->ndev->name, aq_vec,
-					aq_vec_get_affinity_mask(aq_vec));
+						    aq_vec_get_affinity_mask(aq_vec));
 			if (err < 0)
 				goto err_exit;
 		}
 
-		err = self->aq_hw_ops.hw_irq_enable(self->aq_hw,
+		err = self->aq_hw_ops->hw_irq_enable(self->aq_hw,
 				    AQ_CFG_IRQ_MASK);
 		if (err < 0)
 			goto err_exit;
@@ -634,9 +525,8 @@ int aq_nic_xmit(struct aq_nic_s *self, struct sk_buff *skb)
 	frags = aq_nic_map_skb(self, skb, ring);
 
 	if (likely(frags)) {
-		err = self->aq_hw_ops.hw_ring_tx_xmit(self->aq_hw,
-						      ring,
-						      frags);
+		err = self->aq_hw_ops->hw_ring_tx_xmit(self->aq_hw,
+						       ring, frags);
 		if (err >= 0) {
 			++ring->stats.tx.packets;
 			ring->stats.tx.bytes += skb->len;
@@ -651,14 +541,14 @@ int aq_nic_xmit(struct aq_nic_s *self, struct sk_buff *skb)
 
 int aq_nic_update_interrupt_moderation_settings(struct aq_nic_s *self)
 {
-	return self->aq_hw_ops.hw_interrupt_moderation_set(self->aq_hw);
+	return self->aq_hw_ops->hw_interrupt_moderation_set(self->aq_hw);
 }
 
 int aq_nic_set_packet_filter(struct aq_nic_s *self, unsigned int flags)
 {
 	int err = 0;
 
-	err = self->aq_hw_ops.hw_packet_filter_set(self->aq_hw, flags);
+	err = self->aq_hw_ops->hw_packet_filter_set(self->aq_hw, flags);
 	if (err < 0)
 		goto err_exit;
 
@@ -690,11 +580,11 @@ int aq_nic_set_multicast_list(struct aq_nic_s *self, struct net_device *ndev)
 		 * multicast mask
 		 */
 		self->packet_filter |= IFF_ALLMULTI;
-		self->aq_hw->aq_nic_cfg->mc_list_count = 0;
-		return self->aq_hw_ops.hw_packet_filter_set(self->aq_hw,
-							self->packet_filter);
+		self->aq_nic_cfg.mc_list_count = 0;
+		return self->aq_hw_ops->hw_packet_filter_set(self->aq_hw,
+							     self->packet_filter);
 	} else {
-		return self->aq_hw_ops.hw_multicast_list_set(self->aq_hw,
+		return self->aq_hw_ops->hw_multicast_list_set(self->aq_hw,
 						    self->mc_list.ar,
 						    self->mc_list.count);
 	}
@@ -709,7 +599,7 @@ int aq_nic_set_mtu(struct aq_nic_s *self, int new_mtu)
 
 int aq_nic_set_mac(struct aq_nic_s *self, struct net_device *ndev)
 {
-	return self->aq_hw_ops.hw_set_mac_address(self->aq_hw, ndev->dev_addr);
+	return self->aq_hw_ops->hw_set_mac_address(self->aq_hw, ndev->dev_addr);
 }
 
 unsigned int aq_nic_get_link_speed(struct aq_nic_s *self)
@@ -724,8 +614,9 @@ int aq_nic_get_regs(struct aq_nic_s *self, struct ethtool_regs *regs, void *p)
 
 	regs->version = 1;
 
-	err = self->aq_hw_ops.hw_get_regs(self->aq_hw,
-					  &self->aq_hw_caps, regs_buff);
+	err = self->aq_hw_ops->hw_get_regs(self->aq_hw,
+					   self->aq_nic_cfg.aq_hw_caps,
+					   regs_buff);
 	if (err < 0)
 		goto err_exit;
 
@@ -735,7 +626,7 @@ int aq_nic_get_regs(struct aq_nic_s *self, struct ethtool_regs *regs, void *p)
 
 int aq_nic_get_regs_count(struct aq_nic_s *self)
 {
-	return self->aq_hw_caps.mac_regs_count;
+	return self->aq_nic_cfg.aq_hw_caps->mac_regs_count;
 }
 
 void aq_nic_get_stats(struct aq_nic_s *self, u64 *data)
@@ -743,7 +634,7 @@ void aq_nic_get_stats(struct aq_nic_s *self, u64 *data)
 	unsigned int i = 0U;
 	unsigned int count = 0U;
 	struct aq_vec_s *aq_vec = NULL;
-	struct aq_stats_s *stats = self->aq_hw_ops.hw_get_hw_stats(self->aq_hw);
+	struct aq_stats_s *stats = self->aq_hw_ops->hw_get_hw_stats(self->aq_hw);
 
 	if (!stats)
 		goto err_exit;
@@ -774,7 +665,6 @@ void aq_nic_get_stats(struct aq_nic_s *self, u64 *data)
 	i++;
 
 	data += i;
-	count = 0U;
 
 	for (i = 0U, aq_vec = self->aq_vec[0];
 		aq_vec && self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i]) {
@@ -788,7 +678,7 @@ err_exit:;
 static void aq_nic_update_ndev_stats(struct aq_nic_s *self)
 {
 	struct net_device *ndev = self->ndev;
-	struct aq_stats_s *stats = self->aq_hw_ops.hw_get_hw_stats(self->aq_hw);
+	struct aq_stats_s *stats = self->aq_hw_ops->hw_get_hw_stats(self->aq_hw);
 
 	ndev->stats.rx_packets = stats->uprc + stats->mprc + stats->bprc;
 	ndev->stats.rx_bytes = stats->ubrc + stats->mbrc + stats->bbrc;
@@ -802,39 +692,46 @@ static void aq_nic_update_ndev_stats(struct aq_nic_s *self)
 void aq_nic_get_link_ksettings(struct aq_nic_s *self,
 			       struct ethtool_link_ksettings *cmd)
 {
-	cmd->base.port = PORT_TP;
+	if (self->aq_nic_cfg.aq_hw_caps->media_type == AQ_HW_MEDIA_TYPE_FIBRE)
+		cmd->base.port = PORT_FIBRE;
+	else
+		cmd->base.port = PORT_TP;
 	/* This driver supports only 10G capable adapters, so DUPLEX_FULL */
 	cmd->base.duplex = DUPLEX_FULL;
 	cmd->base.autoneg = self->aq_nic_cfg.is_autoneg;
 
 	ethtool_link_ksettings_zero_link_mode(cmd, supported);
 
-	if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_10G)
+	if (self->aq_nic_cfg.aq_hw_caps->link_speed_msk & AQ_NIC_RATE_10G)
 		ethtool_link_ksettings_add_link_mode(cmd, supported,
 						     10000baseT_Full);
 
-	if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_5G)
+	if (self->aq_nic_cfg.aq_hw_caps->link_speed_msk & AQ_NIC_RATE_5G)
 		ethtool_link_ksettings_add_link_mode(cmd, supported,
 						     5000baseT_Full);
 
-	if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_2GS)
+	if (self->aq_nic_cfg.aq_hw_caps->link_speed_msk & AQ_NIC_RATE_2GS)
 		ethtool_link_ksettings_add_link_mode(cmd, supported,
 						     2500baseT_Full);
 
-	if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_1G)
+	if (self->aq_nic_cfg.aq_hw_caps->link_speed_msk & AQ_NIC_RATE_1G)
 		ethtool_link_ksettings_add_link_mode(cmd, supported,
 						     1000baseT_Full);
 
-	if (self->aq_hw_caps.link_speed_msk & AQ_NIC_RATE_100M)
+	if (self->aq_nic_cfg.aq_hw_caps->link_speed_msk & AQ_NIC_RATE_100M)
 		ethtool_link_ksettings_add_link_mode(cmd, supported,
 						     100baseT_Full);
 
-	if (self->aq_hw_caps.flow_control)
+	if (self->aq_nic_cfg.aq_hw_caps->flow_control)
 		ethtool_link_ksettings_add_link_mode(cmd, supported,
 						     Pause);
 
 	ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg);
-	ethtool_link_ksettings_add_link_mode(cmd, supported, TP);
+
+	if (self->aq_nic_cfg.aq_hw_caps->media_type == AQ_HW_MEDIA_TYPE_FIBRE)
+		ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE);
+	else
+		ethtool_link_ksettings_add_link_mode(cmd, supported, TP);
 
 	ethtool_link_ksettings_zero_link_mode(cmd, advertising);
 
@@ -865,7 +762,10 @@ void aq_nic_get_link_ksettings(struct aq_nic_s *self,
 		ethtool_link_ksettings_add_link_mode(cmd, advertising,
 						     Pause);
 
-	ethtool_link_ksettings_add_link_mode(cmd, advertising, TP);
+	if (self->aq_nic_cfg.aq_hw_caps->media_type == AQ_HW_MEDIA_TYPE_FIBRE)
+		ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE);
+	else
+		ethtool_link_ksettings_add_link_mode(cmd, advertising, TP);
 }
 
 int aq_nic_set_link_ksettings(struct aq_nic_s *self,
@@ -876,7 +776,7 @@ int aq_nic_set_link_ksettings(struct aq_nic_s *self,
 	int err = 0;
 
 	if (cmd->base.autoneg == AUTONEG_ENABLE) {
-		rate = self->aq_hw_caps.link_speed_msk;
+		rate = self->aq_nic_cfg.aq_hw_caps->link_speed_msk;
 		self->aq_nic_cfg.is_autoneg = true;
 	} else {
 		speed = cmd->base.speed;
@@ -907,7 +807,7 @@ int aq_nic_set_link_ksettings(struct aq_nic_s *self,
 			goto err_exit;
 		break;
 		}
-		if (!(self->aq_hw_caps.link_speed_msk & rate)) {
+		if (!(self->aq_nic_cfg.aq_hw_caps->link_speed_msk & rate)) {
 			err = -1;
 			goto err_exit;
 		}
@@ -915,7 +815,7 @@ int aq_nic_set_link_ksettings(struct aq_nic_s *self,
 		self->aq_nic_cfg.is_autoneg = false;
 	}
 
-	err = self->aq_hw_ops.hw_set_link_speed(self->aq_hw, rate);
+	err = self->aq_fw_ops->set_link_speed(self->aq_hw, rate);
 	if (err < 0)
 		goto err_exit;
 
@@ -934,7 +834,7 @@ u32 aq_nic_get_fw_version(struct aq_nic_s *self)
 {
 	u32 fw_version = 0U;
 
-	self->aq_hw_ops.hw_get_fw_version(self->aq_hw, &fw_version);
+	self->aq_hw_ops->hw_get_fw_version(self->aq_hw, &fw_version);
 
 	return fw_version;
 }
@@ -949,18 +849,18 @@ int aq_nic_stop(struct aq_nic_s *self)
 
 	del_timer_sync(&self->service_timer);
 
-	self->aq_hw_ops.hw_irq_disable(self->aq_hw, AQ_CFG_IRQ_MASK);
+	self->aq_hw_ops->hw_irq_disable(self->aq_hw, AQ_CFG_IRQ_MASK);
 
 	if (self->aq_nic_cfg.is_polling)
 		del_timer_sync(&self->polling_timer);
 	else
-		aq_pci_func_free_irqs(self->aq_pci_func);
+		aq_pci_func_free_irqs(self);
 
 	for (i = 0U, aq_vec = self->aq_vec[0];
 		self->aq_vecs > i; ++i, aq_vec = self->aq_vec[i])
 		aq_vec_stop(aq_vec);
 
-	return self->aq_hw_ops.hw_stop(self->aq_hw);
+	return self->aq_hw_ops->hw_stop(self->aq_hw);
 }
 
 void aq_nic_deinit(struct aq_nic_s *self)
@@ -976,23 +876,23 @@ void aq_nic_deinit(struct aq_nic_s *self)
 		aq_vec_deinit(aq_vec);
 
 	if (self->power_state == AQ_HW_POWER_STATE_D0) {
-		(void)self->aq_hw_ops.hw_deinit(self->aq_hw);
+		(void)self->aq_hw_ops->hw_deinit(self->aq_hw);
 	} else {
-		(void)self->aq_hw_ops.hw_set_power(self->aq_hw,
+		(void)self->aq_hw_ops->hw_set_power(self->aq_hw,
 						   self->power_state);
 	}
 
 err_exit:;
 }
 
-void aq_nic_free_hot_resources(struct aq_nic_s *self)
+void aq_nic_free_vectors(struct aq_nic_s *self)
 {
 	unsigned int i = 0U;
 
 	if (!self)
 		goto err_exit;
 
-	for (i = AQ_DIMOF(self->aq_vec); i--;) {
+	for (i = ARRAY_SIZE(self->aq_vec); i--;) {
 		if (self->aq_vec[i]) {
 			aq_vec_free(self->aq_vec[i]);
 			self->aq_vec[i] = NULL;
@@ -1039,3 +939,23 @@ int aq_nic_change_pm_state(struct aq_nic_s *self, pm_message_t *pm_msg)
 out:
 	return err;
 }
+
+void aq_nic_shutdown(struct aq_nic_s *self)
+{
+	int err = 0;
+
+	if (!self->ndev)
+		return;
+
+	rtnl_lock();
+
+	netif_device_detach(self->ndev);
+
+	err = aq_nic_stop(self);
+	if (err < 0)
+		goto err_exit;
+	aq_nic_deinit(self);
+
+err_exit:
+	rtnl_unlock();
+}
\ No newline at end of file
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.h b/drivers/net/ethernet/aquantia/atlantic/aq_nic.h
index 3c9f8db03d5f..219b550d1665 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.h
@@ -14,26 +14,15 @@
 
 #include "aq_common.h"
 #include "aq_rss.h"
+#include "aq_hw.h"
 
 struct aq_ring_s;
-struct aq_pci_func_s;
 struct aq_hw_ops;
-
-#define AQ_NIC_FC_OFF    0U
-#define AQ_NIC_FC_TX     1U
-#define AQ_NIC_FC_RX     2U
-#define AQ_NIC_FC_FULL   3U
-#define AQ_NIC_FC_AUTO   4U
-
-#define AQ_NIC_RATE_10G        BIT(0)
-#define AQ_NIC_RATE_5G         BIT(1)
-#define AQ_NIC_RATE_5GSR       BIT(2)
-#define AQ_NIC_RATE_2GS        BIT(3)
-#define AQ_NIC_RATE_1G         BIT(4)
-#define AQ_NIC_RATE_100M       BIT(5)
+struct aq_fw_s;
+struct aq_vec_s;
 
 struct aq_nic_cfg_s {
-	struct aq_hw_caps_s *aq_hw_caps;
+	const struct aq_hw_caps_s *aq_hw_caps;
 	u64 hw_features;
 	u32 rxds;		/* rx ring size, descriptors # */
 	u32 txds;		/* tx ring size, descriptors # */
@@ -44,7 +33,6 @@ struct aq_nic_cfg_s {
 	u16 tx_itr;
 	u32 num_rss_queues;
 	u32 mtu;
-	u32 ucp_0x364;
 	u32 flow_control;
 	u32 link_speed_msk;
 	u32 vlan_id;
@@ -69,20 +57,43 @@ struct aq_nic_cfg_s {
 #define AQ_NIC_TCVEC2RING(_NIC_, _TC_, _VEC_) \
 	((_TC_) * AQ_CFG_TCS_MAX + (_VEC_))
 
-struct aq_nic_s *aq_nic_alloc_cold(const struct net_device_ops *ndev_ops,
-				   const struct ethtool_ops *et_ops,
-				   struct pci_dev *pdev,
-				   struct aq_pci_func_s *aq_pci_func,
-				   unsigned int port,
-				   const struct aq_hw_ops *aq_hw_ops);
-int aq_nic_ndev_init(struct aq_nic_s *self);
+struct aq_nic_s {
+	atomic_t flags;
+	struct aq_vec_s *aq_vec[AQ_CFG_VECS_MAX];
+	struct aq_ring_s *aq_ring_tx[AQ_CFG_VECS_MAX * AQ_CFG_TCS_MAX];
+	struct aq_hw_s *aq_hw;
+	struct net_device *ndev;
+	unsigned int aq_vecs;
+	unsigned int packet_filter;
+	unsigned int power_state;
+	u8 port;
+	const struct aq_hw_ops *aq_hw_ops;
+	const struct aq_fw_ops *aq_fw_ops;
+	struct aq_nic_cfg_s aq_nic_cfg;
+	struct timer_list service_timer;
+	struct timer_list polling_timer;
+	struct aq_hw_link_status_s link_status;
+	struct {
+		u32 count;
+		u8 ar[AQ_CFG_MULTICAST_ADDRESS_MAX][ETH_ALEN];
+	} mc_list;
+
+	struct pci_dev *pdev;
+	unsigned int msix_entry_mask;
+};
+
+static inline struct device *aq_nic_get_dev(struct aq_nic_s *self)
+{
+	return self->ndev->dev.parent;
+}
+
+void aq_nic_ndev_init(struct aq_nic_s *self);
 struct aq_nic_s *aq_nic_alloc_hot(struct net_device *ndev);
 void aq_nic_set_tx_ring(struct aq_nic_s *self, unsigned int idx,
 			struct aq_ring_s *ring);
-struct device *aq_nic_get_dev(struct aq_nic_s *self);
 struct net_device *aq_nic_get_ndev(struct aq_nic_s *self);
 int aq_nic_init(struct aq_nic_s *self);
-int aq_nic_cfg_start(struct aq_nic_s *self);
+void aq_nic_cfg_start(struct aq_nic_s *self);
 int aq_nic_ndev_register(struct aq_nic_s *self);
 void aq_nic_ndev_free(struct aq_nic_s *self);
 int aq_nic_start(struct aq_nic_s *self);
@@ -93,6 +104,7 @@ void aq_nic_get_stats(struct aq_nic_s *self, u64 *data);
 int aq_nic_stop(struct aq_nic_s *self);
 void aq_nic_deinit(struct aq_nic_s *self);
 void aq_nic_free_hot_resources(struct aq_nic_s *self);
+void aq_nic_free_vectors(struct aq_nic_s *self);
 int aq_nic_set_mtu(struct aq_nic_s *self, int new_mtu);
 int aq_nic_set_mac(struct aq_nic_s *self, struct net_device *ndev);
 int aq_nic_set_packet_filter(struct aq_nic_s *self, unsigned int flags);
@@ -106,5 +118,6 @@ struct aq_nic_cfg_s *aq_nic_get_cfg(struct aq_nic_s *self);
 u32 aq_nic_get_fw_version(struct aq_nic_s *self);
 int aq_nic_change_pm_state(struct aq_nic_s *self, pm_message_t *pm_msg);
 int aq_nic_update_interrupt_moderation_settings(struct aq_nic_s *self);
+void aq_nic_shutdown(struct aq_nic_s *self);
 
 #endif /* AQ_NIC_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic_internal.h b/drivers/net/ethernet/aquantia/atlantic/aq_nic_internal.h
deleted file mode 100644
index e7d2711dc165..000000000000
--- a/drivers/net/ethernet/aquantia/atlantic/aq_nic_internal.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * aQuantia Corporation Network Driver
- * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- */
-
-/* File aq_nic_internal.h: Definition of private object structure. */
-
-#ifndef AQ_NIC_INTERNAL_H
-#define AQ_NIC_INTERNAL_H
-
-struct aq_nic_s {
-	struct aq_obj_s header;
-	struct aq_vec_s *aq_vec[AQ_CFG_VECS_MAX];
-	struct aq_ring_s *aq_ring_tx[AQ_CFG_VECS_MAX * AQ_CFG_TCS_MAX];
-	struct aq_hw_s *aq_hw;
-	struct net_device *ndev;
-	struct aq_pci_func_s *aq_pci_func;
-	unsigned int aq_vecs;
-	unsigned int packet_filter;
-	unsigned int power_state;
-	u8 port;
-	struct aq_hw_ops aq_hw_ops;
-	struct aq_hw_caps_s aq_hw_caps;
-	struct aq_nic_cfg_s aq_nic_cfg;
-	struct timer_list service_timer;
-	struct timer_list polling_timer;
-	struct aq_hw_link_status_s link_status;
-	struct {
-		u32 count;
-		u8 ar[AQ_CFG_MULTICAST_ADDRESS_MAX][ETH_ALEN];
-	} mc_list;
-};
-
-#define AQ_NIC_FLAGS_IS_NOT_READY (AQ_NIC_FLAG_STOPPING | \
-			AQ_NIC_FLAG_RESETTING | AQ_NIC_FLAG_CLOSING | \
-			AQ_NIC_FLAG_ERR_UNPLUG | AQ_NIC_FLAG_ERR_HW)
-
-#define AQ_NIC_FLAGS_IS_NOT_TX_READY (AQ_NIC_FLAGS_IS_NOT_READY | \
-					AQ_NIC_LINK_DOWN)
-
-#endif /* AQ_NIC_INTERNAL_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
index 58c29d04b186..ecc6306f940f 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c
@@ -9,181 +9,135 @@
 
 /* File aq_pci_func.c: Definition of PCI functions. */
 
-#include "aq_pci_func.h"
+#include <linux/interrupt.h>
+#include <linux/module.h>
+
+#include "aq_main.h"
 #include "aq_nic.h"
 #include "aq_vec.h"
 #include "aq_hw.h"
-#include <linux/interrupt.h>
-
-struct aq_pci_func_s {
-	struct pci_dev *pdev;
-	struct aq_nic_s *port[AQ_CFG_PCI_FUNC_PORTS];
-	void __iomem *mmio;
-	void *aq_vec[AQ_CFG_PCI_FUNC_MSIX_IRQS];
-	resource_size_t mmio_pa;
-	unsigned int msix_entry_mask;
-	unsigned int ports;
-	bool is_pci_enabled;
-	bool is_regions;
-	bool is_pci_using_dac;
-	struct aq_hw_caps_s aq_hw_caps;
+#include "aq_pci_func.h"
+#include "hw_atl/hw_atl_a0.h"
+#include "hw_atl/hw_atl_b0.h"
+
+static const struct pci_device_id aq_pci_tbl[] = {
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_0001), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_D100), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_D107), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_D108), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_D109), },
+
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC100), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC107), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC108), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC109), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC111), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC112), },
+
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC100S), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC107S), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC108S), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC109S), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC111S), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC112S), },
+
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC111E), },
+	{ PCI_VDEVICE(AQUANTIA, AQ_DEVICE_ID_AQC112E), },
+
+	{}
 };
 
-struct aq_pci_func_s *aq_pci_func_alloc(struct aq_hw_ops *aq_hw_ops,
-					struct pci_dev *pdev,
-					const struct net_device_ops *ndev_ops,
-					const struct ethtool_ops *eth_ops)
-{
-	struct aq_pci_func_s *self = NULL;
-	int err = 0;
-	unsigned int port = 0U;
-
-	if (!aq_hw_ops) {
-		err = -EFAULT;
-		goto err_exit;
-	}
-	self = kzalloc(sizeof(*self), GFP_KERNEL);
-	if (!self) {
-		err = -ENOMEM;
-		goto err_exit;
-	}
-
-	pci_set_drvdata(pdev, self);
-	self->pdev = pdev;
+static const struct aq_board_revision_s hw_atl_boards[] = {
+	{ AQ_DEVICE_ID_0001,	AQ_HWREV_1,	&hw_atl_ops_a0, &hw_atl_a0_caps_aqc107, },
+	{ AQ_DEVICE_ID_D100,	AQ_HWREV_1,	&hw_atl_ops_a0, &hw_atl_a0_caps_aqc100, },
+	{ AQ_DEVICE_ID_D107,	AQ_HWREV_1,	&hw_atl_ops_a0, &hw_atl_a0_caps_aqc107, },
+	{ AQ_DEVICE_ID_D108,	AQ_HWREV_1,	&hw_atl_ops_a0, &hw_atl_a0_caps_aqc108, },
+	{ AQ_DEVICE_ID_D109,	AQ_HWREV_1,	&hw_atl_ops_a0, &hw_atl_a0_caps_aqc109, },
+
+	{ AQ_DEVICE_ID_0001,	AQ_HWREV_2,	&hw_atl_ops_b0, &hw_atl_b0_caps_aqc107, },
+	{ AQ_DEVICE_ID_D100,	AQ_HWREV_2,	&hw_atl_ops_b0, &hw_atl_b0_caps_aqc100, },
+	{ AQ_DEVICE_ID_D107,	AQ_HWREV_2,	&hw_atl_ops_b0, &hw_atl_b0_caps_aqc107, },
+	{ AQ_DEVICE_ID_D108,	AQ_HWREV_2,	&hw_atl_ops_b0, &hw_atl_b0_caps_aqc108, },
+	{ AQ_DEVICE_ID_D109,	AQ_HWREV_2,	&hw_atl_ops_b0, &hw_atl_b0_caps_aqc109, },
+
+	{ AQ_DEVICE_ID_AQC100,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc107, },
+	{ AQ_DEVICE_ID_AQC107,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc107, },
+	{ AQ_DEVICE_ID_AQC108,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc108, },
+	{ AQ_DEVICE_ID_AQC109,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc109, },
+	{ AQ_DEVICE_ID_AQC111,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc111, },
+	{ AQ_DEVICE_ID_AQC112,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc112, },
+
+	{ AQ_DEVICE_ID_AQC100S,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc100s, },
+	{ AQ_DEVICE_ID_AQC107S,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc107s, },
+	{ AQ_DEVICE_ID_AQC108S,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc108s, },
+	{ AQ_DEVICE_ID_AQC109S,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc109s, },
+	{ AQ_DEVICE_ID_AQC111S,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc111s, },
+	{ AQ_DEVICE_ID_AQC112S,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc112s, },
+
+	{ AQ_DEVICE_ID_AQC111E,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc111e, },
+	{ AQ_DEVICE_ID_AQC112E,	AQ_HWREV_ANY,	&hw_atl_ops_b1, &hw_atl_b0_caps_aqc112e, },
+};
 
-	err = aq_hw_ops->get_hw_caps(NULL, &self->aq_hw_caps, pdev->device,
-				     pdev->subsystem_device);
-	if (err < 0)
-		goto err_exit;
+MODULE_DEVICE_TABLE(pci, aq_pci_tbl);
 
-	self->ports = self->aq_hw_caps.ports;
+static int aq_pci_probe_get_hw_by_id(struct pci_dev *pdev,
+				     const struct aq_hw_ops **ops,
+				     const struct aq_hw_caps_s **caps)
+{
+	int i = 0;
 
-	for (port = 0; port < self->ports; ++port) {
-		struct aq_nic_s *aq_nic = aq_nic_alloc_cold(ndev_ops, eth_ops,
-							    pdev, self,
-							    port, aq_hw_ops);
+	if (pdev->vendor != PCI_VENDOR_ID_AQUANTIA)
+		return -EINVAL;
 
-		if (!aq_nic) {
-			err = -ENOMEM;
-			goto err_exit;
+	for (i = 0; i < ARRAY_SIZE(hw_atl_boards); i++) {
+		if (hw_atl_boards[i].devid == pdev->device &&
+		    (hw_atl_boards[i].revision == AQ_HWREV_ANY ||
+		     hw_atl_boards[i].revision == pdev->revision)) {
+			*ops = hw_atl_boards[i].ops;
+			*caps = hw_atl_boards[i].caps;
+			break;
 		}
-		self->port[port] = aq_nic;
 	}
 
-err_exit:
-	if (err < 0) {
-		if (self)
-			aq_pci_func_free(self);
-		self = NULL;
-	}
+	if (i == ARRAY_SIZE(hw_atl_boards))
+		return -EINVAL;
 
-	(void)err;
-	return self;
+	return 0;
 }
 
-int aq_pci_func_init(struct aq_pci_func_s *self)
+int aq_pci_func_init(struct pci_dev *pdev)
 {
 	int err = 0;
-	unsigned int bar = 0U;
-	unsigned int port = 0U;
-	unsigned int numvecs = 0U;
 
-	err = pci_enable_device(self->pdev);
-	if (err < 0)
-		goto err_exit;
-
-	self->is_pci_enabled = true;
-
-	err = pci_set_dma_mask(self->pdev, DMA_BIT_MASK(64));
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 	if (!err) {
-		err = pci_set_consistent_dma_mask(self->pdev, DMA_BIT_MASK(64));
-		self->is_pci_using_dac = 1;
+		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+
 	}
 	if (err) {
-		err = pci_set_dma_mask(self->pdev, DMA_BIT_MASK(32));
+		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 		if (!err)
-			err = pci_set_consistent_dma_mask(self->pdev,
+			err = pci_set_consistent_dma_mask(pdev,
 							  DMA_BIT_MASK(32));
-		self->is_pci_using_dac = 0;
 	}
 	if (err != 0) {
 		err = -ENOSR;
 		goto err_exit;
 	}
 
-	err = pci_request_regions(self->pdev, AQ_CFG_DRV_NAME "_mmio");
+	err = pci_request_regions(pdev, AQ_CFG_DRV_NAME "_mmio");
 	if (err < 0)
 		goto err_exit;
 
-	self->is_regions = true;
+	pci_set_master(pdev);
 
-	pci_set_master(self->pdev);
-
-	for (bar = 0; bar < 4; ++bar) {
-		if (IORESOURCE_MEM & pci_resource_flags(self->pdev, bar)) {
-			resource_size_t reg_sz;
-
-			self->mmio_pa = pci_resource_start(self->pdev, bar);
-			if (self->mmio_pa == 0U) {
-				err = -EIO;
-				goto err_exit;
-			}
-
-			reg_sz = pci_resource_len(self->pdev, bar);
-			if ((reg_sz <= 24 /*ATL_REGS_SIZE*/)) {
-				err = -EIO;
-				goto err_exit;
-			}
-
-			self->mmio = ioremap_nocache(self->mmio_pa, reg_sz);
-			if (!self->mmio) {
-				err = -EIO;
-				goto err_exit;
-			}
-			break;
-		}
-	}
-
-	numvecs = min((u8)AQ_CFG_VECS_DEF, self->aq_hw_caps.msix_irqs);
-	numvecs = min(numvecs, num_online_cpus());
-
-	/* enable interrupts */
-#if !AQ_CFG_FORCE_LEGACY_INT
-	err = pci_alloc_irq_vectors(self->pdev, numvecs, numvecs, PCI_IRQ_MSIX);
-
-	if (err < 0) {
-		err = pci_alloc_irq_vectors(self->pdev, 1, 1,
-				PCI_IRQ_MSI | PCI_IRQ_LEGACY);
-		if (err < 0)
-			goto err_exit;
-	}
-#endif /* AQ_CFG_FORCE_LEGACY_INT */
-
-	/* net device init */
-	for (port = 0; port < self->ports; ++port) {
-		if (!self->port[port])
-			continue;
-
-		err = aq_nic_cfg_start(self->port[port]);
-		if (err < 0)
-			goto err_exit;
-
-		err = aq_nic_ndev_init(self->port[port]);
-		if (err < 0)
-			goto err_exit;
-
-		err = aq_nic_ndev_register(self->port[port]);
-		if (err < 0)
-			goto err_exit;
-	}
+	return 0;
 
 err_exit:
-	if (err < 0)
-		aq_pci_func_deinit(self);
 	return err;
 }
 
-int aq_pci_func_alloc_irq(struct aq_pci_func_s *self, unsigned int i,
+int aq_pci_func_alloc_irq(struct aq_nic_s *self, unsigned int i,
 			  char *name, void *aq_vec, cpumask_t *affinity_mask)
 {
 	struct pci_dev *pdev = self->pdev;
@@ -204,11 +158,10 @@ int aq_pci_func_alloc_irq(struct aq_pci_func_s *self, unsigned int i,
 			irq_set_affinity_hint(pci_irq_vector(pdev, i),
 					      affinity_mask);
 	}
-
 	return err;
 }
 
-void aq_pci_func_free_irqs(struct aq_pci_func_s *self)
+void aq_pci_func_free_irqs(struct aq_nic_s *self)
 {
 	struct pci_dev *pdev = self->pdev;
 	unsigned int i = 0U;
@@ -224,12 +177,7 @@ void aq_pci_func_free_irqs(struct aq_pci_func_s *self)
 	}
 }
 
-void __iomem *aq_pci_func_get_mmio(struct aq_pci_func_s *self)
-{
-	return self->mmio;
-}
-
-unsigned int aq_pci_func_get_irq_type(struct aq_pci_func_s *self)
+unsigned int aq_pci_func_get_irq_type(struct aq_nic_s *self)
 {
 	if (self->pdev->msix_enabled)
 		return AQ_HW_IRQ_MSIX;
@@ -238,62 +186,180 @@ unsigned int aq_pci_func_get_irq_type(struct aq_pci_func_s *self)
 	return AQ_HW_IRQ_LEGACY;
 }
 
-void aq_pci_func_deinit(struct aq_pci_func_s *self)
+static void aq_pci_free_irq_vectors(struct aq_nic_s *self)
 {
-	if (!self)
-		goto err_exit;
-
-	aq_pci_func_free_irqs(self);
 	pci_free_irq_vectors(self->pdev);
+}
 
-	if (self->is_regions)
-		pci_release_regions(self->pdev);
+static int aq_pci_probe(struct pci_dev *pdev,
+			const struct pci_device_id *pci_id)
+{
+	struct aq_nic_s *self = NULL;
+	int err = 0;
+	struct net_device *ndev;
+	resource_size_t mmio_pa;
+	u32 bar;
+	u32 numvecs;
 
-	if (self->is_pci_enabled)
-		pci_disable_device(self->pdev);
+	err = pci_enable_device(pdev);
+	if (err)
+		return err;
 
-err_exit:;
-}
+	err = aq_pci_func_init(pdev);
+	if (err)
+		goto err_pci_func;
 
-void aq_pci_func_free(struct aq_pci_func_s *self)
-{
-	unsigned int port = 0U;
+	ndev = aq_ndev_alloc();
+	if (!ndev) {
+		err = -ENOMEM;
+		goto err_ndev;
+	}
 
-	if (!self)
-		goto err_exit;
+	self = netdev_priv(ndev);
+	self->pdev = pdev;
+	SET_NETDEV_DEV(ndev, &pdev->dev);
+	pci_set_drvdata(pdev, self);
 
-	for (port = 0; port < self->ports; ++port) {
-		if (!self->port[port])
-			continue;
+	err = aq_pci_probe_get_hw_by_id(pdev, &self->aq_hw_ops,
+					&aq_nic_get_cfg(self)->aq_hw_caps);
+	if (err)
+		goto err_ioremap;
 
-		aq_nic_ndev_free(self->port[port]);
+	self->aq_hw = kzalloc(sizeof(*self->aq_hw), GFP_KERNEL);
+	if (!self->aq_hw) {
+		err = -ENOMEM;
+		goto err_ioremap;
 	}
+	self->aq_hw->aq_nic_cfg = aq_nic_get_cfg(self);
 
-	if (self->mmio)
-		iounmap(self->mmio);
+	for (bar = 0; bar < 4; ++bar) {
+		if (IORESOURCE_MEM & pci_resource_flags(pdev, bar)) {
+			resource_size_t reg_sz;
 
-	kfree(self);
+			mmio_pa = pci_resource_start(pdev, bar);
+			if (mmio_pa == 0U) {
+				err = -EIO;
+				goto err_free_aq_hw;
+			}
 
-err_exit:;
-}
+			reg_sz = pci_resource_len(pdev, bar);
+			if ((reg_sz <= 24 /*ATL_REGS_SIZE*/)) {
+				err = -EIO;
+				goto err_free_aq_hw;
+			}
 
-int aq_pci_func_change_pm_state(struct aq_pci_func_s *self,
-				pm_message_t *pm_msg)
-{
-	int err = 0;
-	unsigned int port = 0U;
+			self->aq_hw->mmio = ioremap_nocache(mmio_pa, reg_sz);
+			if (!self->aq_hw->mmio) {
+				err = -EIO;
+				goto err_free_aq_hw;
+			}
+			break;
+		}
+	}
 
-	if (!self) {
-		err = -EFAULT;
-		goto err_exit;
+	if (bar == 4) {
+		err = -EIO;
+		goto err_free_aq_hw;
 	}
-	for (port = 0; port < self->ports; ++port) {
-		if (!self->port[port])
-			continue;
 
-		(void)aq_nic_change_pm_state(self->port[port], pm_msg);
+	numvecs = min((u8)AQ_CFG_VECS_DEF,
+		      aq_nic_get_cfg(self)->aq_hw_caps->msix_irqs);
+	numvecs = min(numvecs, num_online_cpus());
+	/*enable interrupts */
+#if !AQ_CFG_FORCE_LEGACY_INT
+	err = pci_alloc_irq_vectors(self->pdev, numvecs, numvecs,
+				    PCI_IRQ_MSIX);
+
+	if (err < 0) {
+		err = pci_alloc_irq_vectors(self->pdev, 1, 1,
+					    PCI_IRQ_MSI | PCI_IRQ_LEGACY);
+		if (err < 0)
+			goto err_hwinit;
 	}
+#endif
 
-err_exit:
+	/* net device init */
+	aq_nic_cfg_start(self);
+
+	aq_nic_ndev_init(self);
+
+	err = aq_nic_ndev_register(self);
+	if (err < 0)
+		goto err_register;
+
+	return 0;
+
+err_register:
+	aq_nic_free_vectors(self);
+	aq_pci_free_irq_vectors(self);
+err_hwinit:
+	iounmap(self->aq_hw->mmio);
+err_free_aq_hw:
+	kfree(self->aq_hw);
+err_ioremap:
+	free_netdev(ndev);
+err_pci_func:
+	pci_release_regions(pdev);
+err_ndev:
+	pci_disable_device(pdev);
 	return err;
 }
+
+static void aq_pci_remove(struct pci_dev *pdev)
+{
+	struct aq_nic_s *self = pci_get_drvdata(pdev);
+
+	if (self->ndev) {
+		if (self->ndev->reg_state == NETREG_REGISTERED)
+			unregister_netdev(self->ndev);
+		aq_nic_free_vectors(self);
+		aq_pci_free_irq_vectors(self);
+		iounmap(self->aq_hw->mmio);
+		kfree(self->aq_hw);
+		pci_release_regions(pdev);
+		free_netdev(self->ndev);
+	}
+
+	pci_disable_device(pdev);
+}
+
+static void aq_pci_shutdown(struct pci_dev *pdev)
+{
+	struct aq_nic_s *self = pci_get_drvdata(pdev);
+
+	aq_nic_shutdown(self);
+
+	pci_disable_device(pdev);
+
+	if (system_state == SYSTEM_POWER_OFF) {
+		pci_wake_from_d3(pdev, false);
+		pci_set_power_state(pdev, PCI_D3hot);
+	}
+}
+
+static int aq_pci_suspend(struct pci_dev *pdev, pm_message_t pm_msg)
+{
+	struct aq_nic_s *self = pci_get_drvdata(pdev);
+
+	return aq_nic_change_pm_state(self, &pm_msg);
+}
+
+static int aq_pci_resume(struct pci_dev *pdev)
+{
+	struct aq_nic_s *self = pci_get_drvdata(pdev);
+	pm_message_t pm_msg = PMSG_RESTORE;
+
+	return aq_nic_change_pm_state(self, &pm_msg);
+}
+
+static struct pci_driver aq_pci_ops = {
+	.name = AQ_CFG_DRV_NAME,
+	.id_table = aq_pci_tbl,
+	.probe = aq_pci_probe,
+	.remove = aq_pci_remove,
+	.suspend = aq_pci_suspend,
+	.resume = aq_pci_resume,
+	.shutdown = aq_pci_shutdown,
+};
+
+module_pci_driver(aq_pci_ops);
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.h b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.h
index ecb033791203..aeee67bf69fa 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.h
@@ -13,22 +13,20 @@
 #define AQ_PCI_FUNC_H
 
 #include "aq_common.h"
+#include "aq_nic.h"
 
-struct aq_pci_func_s *aq_pci_func_alloc(struct aq_hw_ops *hw_ops,
-					struct pci_dev *pdev,
-					const struct net_device_ops *ndev_ops,
-					const struct ethtool_ops *eth_ops);
-int aq_pci_func_init(struct aq_pci_func_s *self);
-int aq_pci_func_alloc_irq(struct aq_pci_func_s *self, unsigned int i,
+struct aq_board_revision_s {
+	unsigned short devid;
+	unsigned short revision;
+	const struct aq_hw_ops *ops;
+	const struct aq_hw_caps_s *caps;
+};
+
+int aq_pci_func_init(struct pci_dev *pdev);
+int aq_pci_func_alloc_irq(struct aq_nic_s *self, unsigned int i,
 			  char *name, void *aq_vec,
 			  cpumask_t *affinity_mask);
-void aq_pci_func_free_irqs(struct aq_pci_func_s *self);
-int aq_pci_func_start(struct aq_pci_func_s *self);
-void __iomem *aq_pci_func_get_mmio(struct aq_pci_func_s *self);
-unsigned int aq_pci_func_get_irq_type(struct aq_pci_func_s *self);
-void aq_pci_func_deinit(struct aq_pci_func_s *self);
-void aq_pci_func_free(struct aq_pci_func_s *self);
-int aq_pci_func_change_pm_state(struct aq_pci_func_s *self,
-				pm_message_t *pm_msg);
+void aq_pci_func_free_irqs(struct aq_nic_s *self);
+unsigned int aq_pci_func_get_irq_type(struct aq_nic_s *self);
 
 #endif /* AQ_PCI_FUNC_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
index 519ca6534b85..b5f1f62e8e25 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c
@@ -136,11 +136,12 @@ void aq_ring_queue_stop(struct aq_ring_s *ring)
 		netif_stop_subqueue(ndev, ring->idx);
 }
 
-void aq_ring_tx_clean(struct aq_ring_s *self)
+bool aq_ring_tx_clean(struct aq_ring_s *self)
 {
 	struct device *dev = aq_nic_get_dev(self->aq_nic);
+	unsigned int budget = AQ_CFG_TX_CLEAN_BUDGET;
 
-	for (; self->sw_head != self->hw_head;
+	for (; self->sw_head != self->hw_head && budget--;
 		self->sw_head = aq_ring_next_dx(self, self->sw_head)) {
 		struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head];
 
@@ -167,6 +168,8 @@ void aq_ring_tx_clean(struct aq_ring_s *self)
 		buff->pa = 0U;
 		buff->eop_index = 0xffffU;
 	}
+
+	return !!budget;
 }
 
 #define AQ_SKB_ALIGN SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
@@ -279,10 +282,10 @@ int aq_ring_rx_clean(struct aq_ring_s *self,
 
 		skb_record_rx_queue(skb, self->idx);
 
-		napi_gro_receive(napi, skb);
-
 		++self->stats.rx.packets;
 		self->stats.rx.bytes += skb->len;
+
+		napi_gro_receive(napi, skb);
 	}
 
 err_exit:
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.h b/drivers/net/ethernet/aquantia/atlantic/aq_ring.h
index 5844078764bd..ac1329f4051d 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.h
@@ -15,6 +15,7 @@
 #include "aq_common.h"
 
 struct page;
+struct aq_nic_cfg_s;
 
 /*           TxC       SOP        DX         EOP
  *         +----------+----------+----------+-----------
@@ -105,7 +106,6 @@ union aq_ring_stats_s {
 };
 
 struct aq_ring_s {
-	struct aq_obj_s header;
 	struct aq_ring_buff_s *buff_ring;
 	u8 *dx_ring;		/* descriptors ring, dma shared mem */
 	struct aq_nic_s *aq_nic;
@@ -153,7 +153,7 @@ void aq_ring_free(struct aq_ring_s *self);
 void aq_ring_update_queue_state(struct aq_ring_s *ring);
 void aq_ring_queue_wake(struct aq_ring_s *ring);
 void aq_ring_queue_stop(struct aq_ring_s *ring);
-void aq_ring_tx_clean(struct aq_ring_s *self);
+bool aq_ring_tx_clean(struct aq_ring_s *self);
 int aq_ring_rx_clean(struct aq_ring_s *self,
 		     struct napi_struct *napi,
 		     int *work_done,
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_utils.h b/drivers/net/ethernet/aquantia/atlantic/aq_utils.h
index e12bcdfb874a..786ea8187c69 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_utils.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_utils.h
@@ -14,12 +14,6 @@
 
 #include "aq_common.h"
 
-#define AQ_DIMOF(_ARY_)  ARRAY_SIZE(_ARY_)
-
-struct aq_obj_s {
-	atomic_t flags;
-};
-
 static inline void aq_utils_obj_set(atomic_t *flags, u32 mask)
 {
 	unsigned long flags_old, flags_new;
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
index 5fecc9a099ef..d335c334fa56 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
@@ -19,8 +19,7 @@
 #include <linux/netdevice.h>
 
 struct aq_vec_s {
-	struct aq_obj_s header;
-	struct aq_hw_ops *aq_hw_ops;
+	const struct aq_hw_ops *aq_hw_ops;
 	struct aq_hw_s *aq_hw;
 	struct aq_nic_s *aq_nic;
 	unsigned int tx_rings;
@@ -36,12 +35,12 @@ struct aq_vec_s {
 static int aq_vec_poll(struct napi_struct *napi, int budget)
 {
 	struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi);
+	unsigned int sw_tail_old = 0U;
 	struct aq_ring_s *ring = NULL;
+	bool was_tx_cleaned = true;
+	unsigned int i = 0U;
 	int work_done = 0;
 	int err = 0;
-	unsigned int i = 0U;
-	unsigned int sw_tail_old = 0U;
-	bool was_tx_cleaned = false;
 
 	if (!self) {
 		err = -EINVAL;
@@ -58,9 +57,8 @@ static int aq_vec_poll(struct napi_struct *napi, int budget)
 
 			if (ring[AQ_VEC_TX_ID].sw_head !=
 			    ring[AQ_VEC_TX_ID].hw_head) {
-				aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
+				was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
 				aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]);
-				was_tx_cleaned = true;
 			}
 
 			err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw,
@@ -91,7 +89,7 @@ static int aq_vec_poll(struct napi_struct *napi, int budget)
 			}
 		}
 
-		if (was_tx_cleaned)
+		if (!was_tx_cleaned)
 			work_done = budget;
 
 		if (work_done < budget) {
@@ -166,7 +164,7 @@ struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
 	return self;
 }
 
-int aq_vec_init(struct aq_vec_s *self, struct aq_hw_ops *aq_hw_ops,
+int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
 		struct aq_hw_s *aq_hw)
 {
 	struct aq_ring_s *ring = NULL;
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_vec.h b/drivers/net/ethernet/aquantia/atlantic/aq_vec.h
index 6c68b184236c..8bdf60bb3f63 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_vec.h
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_vec.h
@@ -19,6 +19,8 @@
 
 struct aq_hw_s;
 struct aq_hw_ops;
+struct aq_nic_s;
+struct aq_nic_cfg_s;
 struct aq_ring_stats_rx_s;
 struct aq_ring_stats_tx_s;
 
@@ -26,7 +28,7 @@ irqreturn_t aq_vec_isr(int irq, void *private);
 irqreturn_t aq_vec_isr_legacy(int irq, void *private);
 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
 			      struct aq_nic_cfg_s *aq_nic_cfg);
-int aq_vec_init(struct aq_vec_s *self, struct aq_hw_ops *aq_hw_ops,
+int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
 		struct aq_hw_s *aq_hw);
 void aq_vec_deinit(struct aq_vec_s *self);
 void aq_vec_free(struct aq_vec_s *self);
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c
index f18dce14c93c..67e2f9fb9402 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.c
@@ -12,78 +12,100 @@
 #include "../aq_hw.h"
 #include "../aq_hw_utils.h"
 #include "../aq_ring.h"
+#include "../aq_nic.h"
 #include "hw_atl_a0.h"
 #include "hw_atl_utils.h"
 #include "hw_atl_llh.h"
 #include "hw_atl_a0_internal.h"
 
-static int hw_atl_a0_get_hw_caps(struct aq_hw_s *self,
-				 struct aq_hw_caps_s *aq_hw_caps,
-				 unsigned short device,
-				 unsigned short subsystem_device)
-{
-	memcpy(aq_hw_caps, &hw_atl_a0_hw_caps_, sizeof(*aq_hw_caps));
-
-	if (device == HW_ATL_DEVICE_ID_D108 && subsystem_device == 0x0001)
-		aq_hw_caps->link_speed_msk &= ~HW_ATL_A0_RATE_10G;
-
-	if (device == HW_ATL_DEVICE_ID_D109 && subsystem_device == 0x0001) {
-		aq_hw_caps->link_speed_msk &= ~HW_ATL_A0_RATE_10G;
-		aq_hw_caps->link_speed_msk &= ~HW_ATL_A0_RATE_5G;
-	}
-
-	return 0;
-}
-
-static struct aq_hw_s *hw_atl_a0_create(struct aq_pci_func_s *aq_pci_func,
-					unsigned int port,
-					struct aq_hw_ops *ops)
-{
-	struct hw_atl_s *self = NULL;
-
-	self = kzalloc(sizeof(*self), GFP_KERNEL);
-	if (!self)
-		goto err_exit;
-
-	self->base.aq_pci_func = aq_pci_func;
+#define DEFAULT_A0_BOARD_BASIC_CAPABILITIES \
+	.is_64_dma = true, \
+	.msix_irqs = 4U, \
+	.irq_mask = ~0U, \
+	.vecs = HW_ATL_A0_RSS_MAX, \
+	.tcs = HW_ATL_A0_TC_MAX, \
+	.rxd_alignment = 1U, \
+	.rxd_size = HW_ATL_A0_RXD_SIZE, \
+	.rxds = 248U, \
+	.txd_alignment = 1U, \
+	.txd_size = HW_ATL_A0_TXD_SIZE, \
+	.txds = 8U * 1024U, \
+	.txhwb_alignment = 4096U, \
+	.tx_rings = HW_ATL_A0_TX_RINGS, \
+	.rx_rings = HW_ATL_A0_RX_RINGS, \
+	.hw_features = NETIF_F_HW_CSUM | \
+			NETIF_F_RXHASH | \
+			NETIF_F_RXCSUM | \
+			NETIF_F_SG | \
+			NETIF_F_TSO, \
+	.hw_priv_flags = IFF_UNICAST_FLT, \
+	.flow_control = true, \
+	.mtu = HW_ATL_A0_MTU_JUMBO, \
+	.mac_regs_count = 88, \
+	.hw_alive_check_addr = 0x10U
+
+const struct aq_hw_caps_s hw_atl_a0_caps_aqc100 = {
+	DEFAULT_A0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_FIBRE,
+	.link_speed_msk = HW_ATL_A0_RATE_5G  |
+			  HW_ATL_A0_RATE_2G5 |
+			  HW_ATL_A0_RATE_1G  |
+			  HW_ATL_A0_RATE_100M,
+};
 
-	self->base.not_ff_addr = 0x10U;
+const struct aq_hw_caps_s hw_atl_a0_caps_aqc107 = {
+	DEFAULT_A0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_TP,
+	.link_speed_msk = HW_ATL_A0_RATE_10G |
+			  HW_ATL_A0_RATE_5G  |
+			  HW_ATL_A0_RATE_2G5 |
+			  HW_ATL_A0_RATE_1G  |
+			  HW_ATL_A0_RATE_100M,
+};
 
-err_exit:
-	return (struct aq_hw_s *)self;
-}
+const struct aq_hw_caps_s hw_atl_a0_caps_aqc108 = {
+	DEFAULT_A0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_TP,
+	.link_speed_msk = HW_ATL_A0_RATE_5G  |
+			  HW_ATL_A0_RATE_2G5 |
+			  HW_ATL_A0_RATE_1G  |
+			  HW_ATL_A0_RATE_100M,
+};
 
-static void hw_atl_a0_destroy(struct aq_hw_s *self)
-{
-	kfree(self);
-}
+const struct aq_hw_caps_s hw_atl_a0_caps_aqc109 = {
+	DEFAULT_A0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_TP,
+	.link_speed_msk = HW_ATL_A0_RATE_2G5 |
+			  HW_ATL_A0_RATE_1G  |
+			  HW_ATL_A0_RATE_100M,
+};
 
 static int hw_atl_a0_hw_reset(struct aq_hw_s *self)
 {
 	int err = 0;
 
-	glb_glb_reg_res_dis_set(self, 1U);
-	pci_pci_reg_res_dis_set(self, 0U);
-	rx_rx_reg_res_dis_set(self, 0U);
-	tx_tx_reg_res_dis_set(self, 0U);
+	hw_atl_glb_glb_reg_res_dis_set(self, 1U);
+	hw_atl_pci_pci_reg_res_dis_set(self, 0U);
+	hw_atl_rx_rx_reg_res_dis_set(self, 0U);
+	hw_atl_tx_tx_reg_res_dis_set(self, 0U);
 
 	HW_ATL_FLUSH();
-	glb_soft_res_set(self, 1);
+	hw_atl_glb_soft_res_set(self, 1);
 
 	/* check 10 times by 1ms */
-	AQ_HW_WAIT_FOR(glb_soft_res_get(self) == 0, 1000U, 10U);
+	AQ_HW_WAIT_FOR(hw_atl_glb_soft_res_get(self) == 0, 1000U, 10U);
 	if (err < 0)
 		goto err_exit;
 
-	itr_irq_reg_res_dis_set(self, 0U);
-	itr_res_irq_set(self, 1U);
+	hw_atl_itr_irq_reg_res_dis_set(self, 0U);
+	hw_atl_itr_res_irq_set(self, 1U);
 
 	/* check 10 times by 1ms */
-	AQ_HW_WAIT_FOR(itr_res_irq_get(self) == 0, 1000U, 10U);
+	AQ_HW_WAIT_FOR(hw_atl_itr_res_irq_get(self) == 0, 1000U, 10U);
 	if (err < 0)
 		goto err_exit;
 
-	hw_atl_utils_mpi_set(self, MPI_RESET, 0x0U);
+	self->aq_fw_ops->set_state(self, MPI_RESET);
 
 	err = aq_hw_err_from_flags(self);
 
@@ -99,51 +121,53 @@ static int hw_atl_a0_hw_qos_set(struct aq_hw_s *self)
 	bool is_rx_flow_control = false;
 
 	/* TPS Descriptor rate init */
-	tps_tx_pkt_shed_desc_rate_curr_time_res_set(self, 0x0U);
-	tps_tx_pkt_shed_desc_rate_lim_set(self, 0xA);
+	hw_atl_tps_tx_pkt_shed_desc_rate_curr_time_res_set(self, 0x0U);
+	hw_atl_tps_tx_pkt_shed_desc_rate_lim_set(self, 0xA);
 
 	/* TPS VM init */
-	tps_tx_pkt_shed_desc_vm_arb_mode_set(self, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_vm_arb_mode_set(self, 0U);
 
 	/* TPS TC credits init */
-	tps_tx_pkt_shed_desc_tc_arb_mode_set(self, 0U);
-	tps_tx_pkt_shed_data_arb_mode_set(self, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_arb_mode_set(self, 0U);
+	hw_atl_tps_tx_pkt_shed_data_arb_mode_set(self, 0U);
 
-	tps_tx_pkt_shed_tc_data_max_credit_set(self, 0xFFF, 0U);
-	tps_tx_pkt_shed_tc_data_weight_set(self, 0x64, 0U);
-	tps_tx_pkt_shed_desc_tc_max_credit_set(self, 0x50, 0U);
-	tps_tx_pkt_shed_desc_tc_weight_set(self, 0x1E, 0U);
+	hw_atl_tps_tx_pkt_shed_tc_data_max_credit_set(self, 0xFFF, 0U);
+	hw_atl_tps_tx_pkt_shed_tc_data_weight_set(self, 0x64, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_max_credit_set(self, 0x50, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_weight_set(self, 0x1E, 0U);
 
 	/* Tx buf size */
 	buff_size = HW_ATL_A0_TXBUF_MAX;
 
-	tpb_tx_pkt_buff_size_per_tc_set(self, buff_size, tc);
-	tpb_tx_buff_hi_threshold_per_tc_set(self,
-					    (buff_size * (1024 / 32U) * 66U) /
-					    100U, tc);
-	tpb_tx_buff_lo_threshold_per_tc_set(self,
-					    (buff_size * (1024 / 32U) * 50U) /
-					    100U, tc);
+	hw_atl_tpb_tx_pkt_buff_size_per_tc_set(self, buff_size, tc);
+	hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024 / 32U) * 66U) /
+						   100U, tc);
+	hw_atl_tpb_tx_buff_lo_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024 / 32U) * 50U) /
+						   100U, tc);
 
 	/* QoS Rx buf size per TC */
 	tc = 0;
 	is_rx_flow_control = (AQ_NIC_FC_RX & self->aq_nic_cfg->flow_control);
 	buff_size = HW_ATL_A0_RXBUF_MAX;
 
-	rpb_rx_pkt_buff_size_per_tc_set(self, buff_size, tc);
-	rpb_rx_buff_hi_threshold_per_tc_set(self,
-					    (buff_size *
-					    (1024U / 32U) * 66U) /
-					    100U, tc);
-	rpb_rx_buff_lo_threshold_per_tc_set(self,
-					    (buff_size *
-					    (1024U / 32U) * 50U) /
-					    100U, tc);
-	rpb_rx_xoff_en_per_tc_set(self, is_rx_flow_control ? 1U : 0U, tc);
+	hw_atl_rpb_rx_pkt_buff_size_per_tc_set(self, buff_size, tc);
+	hw_atl_rpb_rx_buff_hi_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024U / 32U) * 66U) /
+						   100U, tc);
+	hw_atl_rpb_rx_buff_lo_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024U / 32U) * 50U) /
+						   100U, tc);
+	hw_atl_rpb_rx_xoff_en_per_tc_set(self, is_rx_flow_control ? 1U : 0U, tc);
 
 	/* QoS 802.1p priority -> TC mapping */
 	for (i_priority = 8U; i_priority--;)
-		rpf_rpb_user_priority_tc_map_set(self, i_priority, 0U);
+		hw_atl_rpf_rpb_user_priority_tc_map_set(self, i_priority, 0U);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -151,20 +175,19 @@ static int hw_atl_a0_hw_qos_set(struct aq_hw_s *self)
 static int hw_atl_a0_hw_rss_hash_set(struct aq_hw_s *self,
 				     struct aq_rss_parameters *rss_params)
 {
-	struct aq_nic_cfg_s *cfg = NULL;
+	struct aq_nic_cfg_s *cfg = self->aq_nic_cfg;
 	int err = 0;
 	unsigned int i = 0U;
 	unsigned int addr = 0U;
 
-	cfg = self->aq_nic_cfg;
-
 	for (i = 10, addr = 0U; i--; ++addr) {
 		u32 key_data = cfg->is_rss ?
 			__swab32(rss_params->hash_secret_key[i]) : 0U;
-		rpf_rss_key_wr_data_set(self, key_data);
-		rpf_rss_key_addr_set(self, addr);
-		rpf_rss_key_wr_en_set(self, 1U);
-		AQ_HW_WAIT_FOR(rpf_rss_key_wr_en_get(self) == 0, 1000U, 10U);
+		hw_atl_rpf_rss_key_wr_data_set(self, key_data);
+		hw_atl_rpf_rss_key_addr_set(self, addr);
+		hw_atl_rpf_rss_key_wr_en_set(self, 1U);
+		AQ_HW_WAIT_FOR(hw_atl_rpf_rss_key_wr_en_get(self) == 0,
+			       1000U, 10U);
 		if (err < 0)
 			goto err_exit;
 	}
@@ -193,11 +216,12 @@ static int hw_atl_a0_hw_rss_set(struct aq_hw_s *self,
 			((i * 3U) & 0xFU));
 	}
 
-	for (i = AQ_DIMOF(bitary); i--;) {
-		rpf_rss_redir_tbl_wr_data_set(self, bitary[i]);
-		rpf_rss_redir_tbl_addr_set(self, i);
-		rpf_rss_redir_wr_en_set(self, 1U);
-		AQ_HW_WAIT_FOR(rpf_rss_redir_wr_en_get(self) == 0, 1000U, 10U);
+	for (i = ARRAY_SIZE(bitary); i--;) {
+		hw_atl_rpf_rss_redir_tbl_wr_data_set(self, bitary[i]);
+		hw_atl_rpf_rss_redir_tbl_addr_set(self, i);
+		hw_atl_rpf_rss_redir_wr_en_set(self, 1U);
+		AQ_HW_WAIT_FOR(hw_atl_rpf_rss_redir_wr_en_get(self) == 0,
+			       1000U, 10U);
 		if (err < 0)
 			goto err_exit;
 	}
@@ -212,35 +236,35 @@ static int hw_atl_a0_hw_offload_set(struct aq_hw_s *self,
 				    struct aq_nic_cfg_s *aq_nic_cfg)
 {
 	/* TX checksums offloads*/
-	tpo_ipv4header_crc_offload_en_set(self, 1);
-	tpo_tcp_udp_crc_offload_en_set(self, 1);
+	hw_atl_tpo_ipv4header_crc_offload_en_set(self, 1);
+	hw_atl_tpo_tcp_udp_crc_offload_en_set(self, 1);
 
 	/* RX checksums offloads*/
-	rpo_ipv4header_crc_offload_en_set(self, 1);
-	rpo_tcp_udp_crc_offload_en_set(self, 1);
+	hw_atl_rpo_ipv4header_crc_offload_en_set(self, 1);
+	hw_atl_rpo_tcp_udp_crc_offload_en_set(self, 1);
 
 	/* LSO offloads*/
-	tdm_large_send_offload_en_set(self, 0xFFFFFFFFU);
+	hw_atl_tdm_large_send_offload_en_set(self, 0xFFFFFFFFU);
 
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_a0_hw_init_tx_path(struct aq_hw_s *self)
 {
-	thm_lso_tcp_flag_of_first_pkt_set(self, 0x0FF6U);
-	thm_lso_tcp_flag_of_middle_pkt_set(self, 0x0FF6U);
-	thm_lso_tcp_flag_of_last_pkt_set(self, 0x0F7FU);
+	hw_atl_thm_lso_tcp_flag_of_first_pkt_set(self, 0x0FF6U);
+	hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(self, 0x0FF6U);
+	hw_atl_thm_lso_tcp_flag_of_last_pkt_set(self, 0x0F7FU);
 
 	/* Tx interrupts */
-	tdm_tx_desc_wr_wb_irq_en_set(self, 1U);
+	hw_atl_tdm_tx_desc_wr_wb_irq_en_set(self, 1U);
 
 	/* misc */
 	aq_hw_write_reg(self, 0x00007040U, IS_CHIP_FEATURE(TPO2) ?
 			0x00010000U : 0x00000000U);
-	tdm_tx_dca_en_set(self, 0U);
-	tdm_tx_dca_mode_set(self, 0U);
+	hw_atl_tdm_tx_dca_en_set(self, 0U);
+	hw_atl_tdm_tx_dca_mode_set(self, 0U);
 
-	tpb_tx_path_scp_ins_en_set(self, 1U);
+	hw_atl_tpb_tx_path_scp_ins_en_set(self, 1U);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -251,38 +275,38 @@ static int hw_atl_a0_hw_init_rx_path(struct aq_hw_s *self)
 	int i;
 
 	/* Rx TC/RSS number config */
-	rpb_rpf_rx_traf_class_mode_set(self, 1U);
+	hw_atl_rpb_rpf_rx_traf_class_mode_set(self, 1U);
 
 	/* Rx flow control */
-	rpb_rx_flow_ctl_mode_set(self, 1U);
+	hw_atl_rpb_rx_flow_ctl_mode_set(self, 1U);
 
 	/* RSS Ring selection */
-	reg_rx_flr_rss_control1set(self, cfg->is_rss ?
+	hw_atl_reg_rx_flr_rss_control1set(self, cfg->is_rss ?
 					0xB3333333U : 0x00000000U);
 
 	/* Multicast filters */
 	for (i = HW_ATL_A0_MAC_MAX; i--;) {
-		rpfl2_uc_flr_en_set(self, (i == 0U) ? 1U : 0U, i);
-		rpfl2unicast_flr_act_set(self, 1U, i);
+		hw_atl_rpfl2_uc_flr_en_set(self, (i == 0U) ? 1U : 0U, i);
+		hw_atl_rpfl2unicast_flr_act_set(self, 1U, i);
 	}
 
-	reg_rx_flr_mcst_flr_msk_set(self, 0x00000000U);
-	reg_rx_flr_mcst_flr_set(self, 0x00010FFFU, 0U);
+	hw_atl_reg_rx_flr_mcst_flr_msk_set(self, 0x00000000U);
+	hw_atl_reg_rx_flr_mcst_flr_set(self, 0x00010FFFU, 0U);
 
 	/* Vlan filters */
-	rpf_vlan_outer_etht_set(self, 0x88A8U);
-	rpf_vlan_inner_etht_set(self, 0x8100U);
-	rpf_vlan_prom_mode_en_set(self, 1);
+	hw_atl_rpf_vlan_outer_etht_set(self, 0x88A8U);
+	hw_atl_rpf_vlan_inner_etht_set(self, 0x8100U);
+	hw_atl_rpf_vlan_prom_mode_en_set(self, 1);
 
 	/* Rx Interrupts */
-	rdm_rx_desc_wr_wb_irq_en_set(self, 1U);
+	hw_atl_rdm_rx_desc_wr_wb_irq_en_set(self, 1U);
 
 	/* misc */
-	rpfl2broadcast_flr_act_set(self, 1U);
-	rpfl2broadcast_count_threshold_set(self, 0xFFFFU & (~0U / 256U));
+	hw_atl_rpfl2broadcast_flr_act_set(self, 1U);
+	hw_atl_rpfl2broadcast_count_threshold_set(self, 0xFFFFU & (~0U / 256U));
 
-	rdm_rx_dca_en_set(self, 0U);
-	rdm_rx_dca_mode_set(self, 0U);
+	hw_atl_rdm_rx_dca_en_set(self, 0U);
+	hw_atl_rdm_rx_dca_mode_set(self, 0U);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -301,10 +325,10 @@ static int hw_atl_a0_hw_mac_addr_set(struct aq_hw_s *self, u8 *mac_addr)
 	l = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
 		(mac_addr[4] << 8) | mac_addr[5];
 
-	rpfl2_uc_flr_en_set(self, 0U, HW_ATL_A0_MAC);
-	rpfl2unicast_dest_addresslsw_set(self, l, HW_ATL_A0_MAC);
-	rpfl2unicast_dest_addressmsw_set(self, h, HW_ATL_A0_MAC);
-	rpfl2_uc_flr_en_set(self, 1U, HW_ATL_A0_MAC);
+	hw_atl_rpfl2_uc_flr_en_set(self, 0U, HW_ATL_A0_MAC);
+	hw_atl_rpfl2unicast_dest_addresslsw_set(self, l, HW_ATL_A0_MAC);
+	hw_atl_rpfl2unicast_dest_addressmsw_set(self, h, HW_ATL_A0_MAC);
+	hw_atl_rpfl2_uc_flr_en_set(self, 1U, HW_ATL_A0_MAC);
 
 	err = aq_hw_err_from_flags(self);
 
@@ -312,9 +336,7 @@ static int hw_atl_a0_hw_mac_addr_set(struct aq_hw_s *self, u8 *mac_addr)
 	return err;
 }
 
-static int hw_atl_a0_hw_init(struct aq_hw_s *self,
-			     struct aq_nic_cfg_s *aq_nic_cfg,
-			     u8 *mac_addr)
+static int hw_atl_a0_hw_init(struct aq_hw_s *self, u8 *mac_addr)
 {
 	static u32 aq_hw_atl_igcr_table_[4][2] = {
 		{ 0x20000000U, 0x20000000U }, /* AQ_IRQ_INVALID */
@@ -325,20 +347,18 @@ static int hw_atl_a0_hw_init(struct aq_hw_s *self,
 
 	int err = 0;
 
-	self->aq_nic_cfg = aq_nic_cfg;
-
-	hw_atl_utils_hw_chip_features_init(self,
-					   &PHAL_ATLANTIC_A0->chip_features);
+	struct aq_nic_cfg_s *aq_nic_cfg = self->aq_nic_cfg;
 
 	hw_atl_a0_hw_init_tx_path(self);
 	hw_atl_a0_hw_init_rx_path(self);
 
 	hw_atl_a0_hw_mac_addr_set(self, mac_addr);
 
-	hw_atl_utils_mpi_set(self, MPI_INIT, aq_nic_cfg->link_speed_msk);
+	self->aq_fw_ops->set_link_speed(self, aq_nic_cfg->link_speed_msk);
+	self->aq_fw_ops->set_state(self, MPI_INIT);
 
-	reg_tx_dma_debug_ctl_set(self, 0x800000b8U);
-	reg_tx_dma_debug_ctl_set(self, 0x000000b8U);
+	hw_atl_reg_tx_dma_debug_ctl_set(self, 0x800000b8U);
+	hw_atl_reg_tx_dma_debug_ctl_set(self, 0x000000b8U);
 
 	hw_atl_a0_hw_qos_set(self);
 	hw_atl_a0_hw_rss_set(self, &aq_nic_cfg->aq_rss);
@@ -346,26 +366,25 @@ static int hw_atl_a0_hw_init(struct aq_hw_s *self,
 
 	/* Reset link status and read out initial hardware counters */
 	self->aq_link_status.mbps = 0;
-	hw_atl_utils_update_stats(self);
+	self->aq_fw_ops->update_stats(self);
 
 	err = aq_hw_err_from_flags(self);
 	if (err < 0)
 		goto err_exit;
 
 	/* Interrupts */
-	reg_irq_glb_ctl_set(self,
-			    aq_hw_atl_igcr_table_[aq_nic_cfg->irq_type]
-						 [(aq_nic_cfg->vecs > 1U) ?
-						 1 : 0]);
+	hw_atl_reg_irq_glb_ctl_set(self,
+				   aq_hw_atl_igcr_table_[aq_nic_cfg->irq_type]
+					[(aq_nic_cfg->vecs > 1U) ? 1 : 0]);
 
-	itr_irq_auto_masklsw_set(self, aq_nic_cfg->aq_hw_caps->irq_mask);
+	hw_atl_itr_irq_auto_masklsw_set(self, aq_nic_cfg->aq_hw_caps->irq_mask);
 
 	/* Interrupts */
-	reg_gen_irq_map_set(self,
-			    ((HW_ATL_A0_ERR_INT << 0x18) | (1U << 0x1F)) |
-			    ((HW_ATL_A0_ERR_INT << 0x10) | (1U << 0x17)) |
-			    ((HW_ATL_A0_ERR_INT << 8) | (1U << 0xF)) |
-			    ((HW_ATL_A0_ERR_INT) | (1U << 0x7)), 0U);
+	hw_atl_reg_gen_irq_map_set(self,
+				   ((HW_ATL_A0_ERR_INT << 0x18) | (1U << 0x1F)) |
+				   ((HW_ATL_A0_ERR_INT << 0x10) | (1U << 0x17)) |
+				   ((HW_ATL_A0_ERR_INT << 8) | (1U << 0xF)) |
+				   ((HW_ATL_A0_ERR_INT) | (1U << 0x7)), 0U);
 
 	hw_atl_a0_hw_offload_set(self, aq_nic_cfg);
 
@@ -376,28 +395,28 @@ static int hw_atl_a0_hw_init(struct aq_hw_s *self,
 static int hw_atl_a0_hw_ring_tx_start(struct aq_hw_s *self,
 				      struct aq_ring_s *ring)
 {
-	tdm_tx_desc_en_set(self, 1, ring->idx);
+	hw_atl_tdm_tx_desc_en_set(self, 1, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_a0_hw_ring_rx_start(struct aq_hw_s *self,
 				      struct aq_ring_s *ring)
 {
-	rdm_rx_desc_en_set(self, 1, ring->idx);
+	hw_atl_rdm_rx_desc_en_set(self, 1, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_a0_hw_start(struct aq_hw_s *self)
 {
-	tpb_tx_buff_en_set(self, 1);
-	rpb_rx_buff_en_set(self, 1);
+	hw_atl_tpb_tx_buff_en_set(self, 1);
+	hw_atl_rpb_rx_buff_en_set(self, 1);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_a0_hw_tx_ring_tail_update(struct aq_hw_s *self,
 					    struct aq_ring_s *ring)
 {
-	reg_tx_dma_desc_tail_ptr_set(self, ring->sw_tail, ring->idx);
+	hw_atl_reg_tx_dma_desc_tail_ptr_set(self, ring->sw_tail, ring->idx);
 	return 0;
 }
 
@@ -483,36 +502,37 @@ static int hw_atl_a0_hw_ring_rx_init(struct aq_hw_s *self,
 	u32 dma_desc_addr_lsw = (u32)aq_ring->dx_ring_pa;
 	u32 dma_desc_addr_msw = (u32)(((u64)aq_ring->dx_ring_pa) >> 32);
 
-	rdm_rx_desc_en_set(self, false, aq_ring->idx);
+	hw_atl_rdm_rx_desc_en_set(self, false, aq_ring->idx);
 
-	rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
 
-	reg_rx_dma_desc_base_addresslswset(self, dma_desc_addr_lsw,
-					   aq_ring->idx);
+	hw_atl_reg_rx_dma_desc_base_addresslswset(self, dma_desc_addr_lsw,
+						  aq_ring->idx);
 
-	reg_rx_dma_desc_base_addressmswset(self,
-					   dma_desc_addr_msw, aq_ring->idx);
+	hw_atl_reg_rx_dma_desc_base_addressmswset(self,
+						  dma_desc_addr_msw,
+						  aq_ring->idx);
 
-	rdm_rx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
 
-	rdm_rx_desc_data_buff_size_set(self,
-				       AQ_CFG_RX_FRAME_MAX / 1024U,
+	hw_atl_rdm_rx_desc_data_buff_size_set(self,
+					      AQ_CFG_RX_FRAME_MAX / 1024U,
 				       aq_ring->idx);
 
-	rdm_rx_desc_head_buff_size_set(self, 0U, aq_ring->idx);
-	rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
-	rpo_rx_desc_vlan_stripping_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_head_buff_size_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
+	hw_atl_rpo_rx_desc_vlan_stripping_set(self, 0U, aq_ring->idx);
 
 	/* Rx ring set mode */
 
 	/* Mapping interrupt vector */
-	itr_irq_map_rx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
-	itr_irq_map_en_rx_set(self, true, aq_ring->idx);
+	hw_atl_itr_irq_map_rx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
+	hw_atl_itr_irq_map_en_rx_set(self, true, aq_ring->idx);
 
-	rdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
-	rdm_rx_desc_dca_en_set(self, 0U, aq_ring->idx);
-	rdm_rx_head_dca_en_set(self, 0U, aq_ring->idx);
-	rdm_rx_pld_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
+	hw_atl_rdm_rx_desc_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_head_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_pld_dca_en_set(self, 0U, aq_ring->idx);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -524,25 +544,25 @@ static int hw_atl_a0_hw_ring_tx_init(struct aq_hw_s *self,
 	u32 dma_desc_lsw_addr = (u32)aq_ring->dx_ring_pa;
 	u32 dma_desc_msw_addr = (u32)(((u64)aq_ring->dx_ring_pa) >> 32);
 
-	reg_tx_dma_desc_base_addresslswset(self, dma_desc_lsw_addr,
-					   aq_ring->idx);
+	hw_atl_reg_tx_dma_desc_base_addresslswset(self, dma_desc_lsw_addr,
+						  aq_ring->idx);
 
-	reg_tx_dma_desc_base_addressmswset(self, dma_desc_msw_addr,
-					   aq_ring->idx);
+	hw_atl_reg_tx_dma_desc_base_addressmswset(self, dma_desc_msw_addr,
+						  aq_ring->idx);
 
-	tdm_tx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
+	hw_atl_tdm_tx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
 
 	hw_atl_a0_hw_tx_ring_tail_update(self, aq_ring);
 
 	/* Set Tx threshold */
-	tdm_tx_desc_wr_wb_threshold_set(self, 0U, aq_ring->idx);
+	hw_atl_tdm_tx_desc_wr_wb_threshold_set(self, 0U, aq_ring->idx);
 
 	/* Mapping interrupt vector */
-	itr_irq_map_tx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
-	itr_irq_map_en_tx_set(self, true, aq_ring->idx);
+	hw_atl_itr_irq_map_tx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
+	hw_atl_itr_irq_map_en_tx_set(self, true, aq_ring->idx);
 
-	tdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
-	tdm_tx_desc_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_tdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
+	hw_atl_tdm_tx_desc_dca_en_set(self, 0U, aq_ring->idx);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -563,7 +583,7 @@ static int hw_atl_a0_hw_ring_rx_fill(struct aq_hw_s *self,
 		rxd->hdr_addr = 0U;
 	}
 
-	reg_rx_dma_desc_tail_ptr_set(self, sw_tail_old, ring->idx);
+	hw_atl_reg_rx_dma_desc_tail_ptr_set(self, sw_tail_old, ring->idx);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -572,13 +592,13 @@ static int hw_atl_a0_hw_ring_tx_head_update(struct aq_hw_s *self,
 					    struct aq_ring_s *ring)
 {
 	int err = 0;
-	unsigned int hw_head_ = tdm_tx_desc_head_ptr_get(self, ring->idx);
+	unsigned int hw_head = hw_atl_tdm_tx_desc_head_ptr_get(self, ring->idx);
 
-	if (aq_utils_obj_test(&self->header.flags, AQ_HW_FLAG_ERR_UNPLUG)) {
+	if (aq_utils_obj_test(&self->flags, AQ_HW_FLAG_ERR_UNPLUG)) {
 		err = -ENXIO;
 		goto err_exit;
 	}
-	ring->hw_head = hw_head_;
+	ring->hw_head = hw_head;
 	err = aq_hw_err_from_flags(self);
 
 err_exit:
@@ -602,15 +622,16 @@ static int hw_atl_a0_hw_ring_rx_receive(struct aq_hw_s *self,
 
 		if (!(rxd_wb->status & 0x5U)) { /* RxD is not done */
 			if ((1U << 4) &
-				reg_rx_dma_desc_status_get(self, ring->idx)) {
-			rdm_rx_desc_en_set(self, false, ring->idx);
-			rdm_rx_desc_res_set(self, true, ring->idx);
-			rdm_rx_desc_res_set(self, false, ring->idx);
-			rdm_rx_desc_en_set(self, true, ring->idx);
+			hw_atl_reg_rx_dma_desc_status_get(self, ring->idx)) {
+				hw_atl_rdm_rx_desc_en_set(self, false, ring->idx);
+				hw_atl_rdm_rx_desc_res_set(self, true, ring->idx);
+				hw_atl_rdm_rx_desc_res_set(self, false, ring->idx);
+				hw_atl_rdm_rx_desc_en_set(self, true, ring->idx);
 			}
 
 			if (ring->hw_head ||
-			    (rdm_rx_desc_head_ptr_get(self, ring->idx) < 2U)) {
+			    (hw_atl_rdm_rx_desc_head_ptr_get(self,
+							     ring->idx) < 2U)) {
 				break;
 			} else if (!(rxd_wb->status & 0x1U)) {
 				struct hw_atl_rxd_wb_s *rxd_wb1 =
@@ -693,26 +714,25 @@ static int hw_atl_a0_hw_ring_rx_receive(struct aq_hw_s *self,
 
 static int hw_atl_a0_hw_irq_enable(struct aq_hw_s *self, u64 mask)
 {
-	itr_irq_msk_setlsw_set(self, LODWORD(mask) |
+	hw_atl_itr_irq_msk_setlsw_set(self, LODWORD(mask) |
 			       (1U << HW_ATL_A0_ERR_INT));
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_a0_hw_irq_disable(struct aq_hw_s *self, u64 mask)
 {
-	itr_irq_msk_clearlsw_set(self, LODWORD(mask));
-	itr_irq_status_clearlsw_set(self, LODWORD(mask));
+	hw_atl_itr_irq_msk_clearlsw_set(self, LODWORD(mask));
+	hw_atl_itr_irq_status_clearlsw_set(self, LODWORD(mask));
 
-	if ((1U << 16) & reg_gen_irq_status_get(self))
-
-		atomic_inc(&PHAL_ATLANTIC_A0->dpc);
+	if ((1U << 16) & hw_atl_reg_gen_irq_status_get(self))
+		atomic_inc(&self->dpc);
 
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_a0_hw_irq_read(struct aq_hw_s *self, u64 *mask)
 {
-	*mask = itr_irq_statuslsw_get(self);
+	*mask = hw_atl_itr_irq_statuslsw_get(self);
 	return aq_hw_err_from_flags(self);
 }
 
@@ -723,18 +743,20 @@ static int hw_atl_a0_hw_packet_filter_set(struct aq_hw_s *self,
 {
 	unsigned int i = 0U;
 
-	rpfl2promiscuous_mode_en_set(self, IS_FILTER_ENABLED(IFF_PROMISC));
-	rpfl2multicast_flr_en_set(self, IS_FILTER_ENABLED(IFF_MULTICAST), 0);
-	rpfl2broadcast_en_set(self, IS_FILTER_ENABLED(IFF_BROADCAST));
+	hw_atl_rpfl2promiscuous_mode_en_set(self,
+					    IS_FILTER_ENABLED(IFF_PROMISC));
+	hw_atl_rpfl2multicast_flr_en_set(self,
+					 IS_FILTER_ENABLED(IFF_MULTICAST), 0);
+	hw_atl_rpfl2broadcast_en_set(self, IS_FILTER_ENABLED(IFF_BROADCAST));
 
 	self->aq_nic_cfg->is_mc_list_enabled =
 			IS_FILTER_ENABLED(IFF_MULTICAST);
 
 	for (i = HW_ATL_A0_MAC_MIN; i < HW_ATL_A0_MAC_MAX; ++i)
-		rpfl2_uc_flr_en_set(self,
-				    (self->aq_nic_cfg->is_mc_list_enabled &&
-				    (i <= self->aq_nic_cfg->mc_list_count)) ?
-				    1U : 0U, i);
+		hw_atl_rpfl2_uc_flr_en_set(self,
+					   (self->aq_nic_cfg->is_mc_list_enabled &&
+					   (i <= self->aq_nic_cfg->mc_list_count)) ?
+					    1U : 0U, i);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -761,17 +783,19 @@ static int hw_atl_a0_hw_multicast_list_set(struct aq_hw_s *self,
 		u32 l = (ar_mac[i][2] << 24) | (ar_mac[i][3] << 16) |
 					(ar_mac[i][4] << 8) | ar_mac[i][5];
 
-		rpfl2_uc_flr_en_set(self, 0U, HW_ATL_A0_MAC_MIN + i);
+		hw_atl_rpfl2_uc_flr_en_set(self, 0U, HW_ATL_A0_MAC_MIN + i);
 
-		rpfl2unicast_dest_addresslsw_set(self,
-						 l, HW_ATL_A0_MAC_MIN + i);
+		hw_atl_rpfl2unicast_dest_addresslsw_set(self,
+							l,
+							HW_ATL_A0_MAC_MIN + i);
 
-		rpfl2unicast_dest_addressmsw_set(self,
-						 h, HW_ATL_A0_MAC_MIN + i);
+		hw_atl_rpfl2unicast_dest_addressmsw_set(self,
+							h,
+							HW_ATL_A0_MAC_MIN + i);
 
-		rpfl2_uc_flr_en_set(self,
-				    (self->aq_nic_cfg->is_mc_list_enabled),
-				    HW_ATL_A0_MAC_MIN + i);
+		hw_atl_rpfl2_uc_flr_en_set(self,
+					   (self->aq_nic_cfg->is_mc_list_enabled),
+					   HW_ATL_A0_MAC_MIN + i);
 	}
 
 	err = aq_hw_err_from_flags(self);
@@ -823,7 +847,7 @@ static int hw_atl_a0_hw_interrupt_moderation_set(struct aq_hw_s *self)
 	}
 
 	for (i = HW_ATL_A0_RINGS_MAX; i--;)
-		reg_irq_thr_set(self, itr_rx, i);
+		hw_atl_reg_irq_thr_set(self, itr_rx, i);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -837,38 +861,19 @@ static int hw_atl_a0_hw_stop(struct aq_hw_s *self)
 static int hw_atl_a0_hw_ring_tx_stop(struct aq_hw_s *self,
 				     struct aq_ring_s *ring)
 {
-	tdm_tx_desc_en_set(self, 0U, ring->idx);
+	hw_atl_tdm_tx_desc_en_set(self, 0U, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_a0_hw_ring_rx_stop(struct aq_hw_s *self,
 				     struct aq_ring_s *ring)
 {
-	rdm_rx_desc_en_set(self, 0U, ring->idx);
+	hw_atl_rdm_rx_desc_en_set(self, 0U, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
-static int hw_atl_a0_hw_set_speed(struct aq_hw_s *self, u32 speed)
-{
-	int err = 0;
-
-	err = hw_atl_utils_mpi_set_speed(self, speed, MPI_INIT);
-	if (err < 0)
-		goto err_exit;
-
-err_exit:
-	return err;
-}
-
-static struct aq_hw_ops hw_atl_ops_ = {
-	.create               = hw_atl_a0_create,
-	.destroy              = hw_atl_a0_destroy,
-	.get_hw_caps          = hw_atl_a0_get_hw_caps,
-
-	.hw_get_mac_permanent = hw_atl_utils_get_mac_permanent,
+const struct aq_hw_ops hw_atl_ops_a0 = {
 	.hw_set_mac_address   = hw_atl_a0_hw_mac_addr_set,
-	.hw_get_link_status   = hw_atl_utils_mpi_get_link_status,
-	.hw_set_link_speed    = hw_atl_a0_hw_set_speed,
 	.hw_init              = hw_atl_a0_hw_init,
 	.hw_deinit            = hw_atl_utils_hw_deinit,
 	.hw_set_power         = hw_atl_utils_hw_set_power,
@@ -898,21 +903,6 @@ static struct aq_hw_ops hw_atl_ops_ = {
 	.hw_rss_set                  = hw_atl_a0_hw_rss_set,
 	.hw_rss_hash_set             = hw_atl_a0_hw_rss_hash_set,
 	.hw_get_regs                 = hw_atl_utils_hw_get_regs,
-	.hw_update_stats             = hw_atl_utils_update_stats,
 	.hw_get_hw_stats             = hw_atl_utils_get_hw_stats,
 	.hw_get_fw_version           = hw_atl_utils_get_fw_version,
 };
-
-struct aq_hw_ops *hw_atl_a0_get_ops_by_id(struct pci_dev *pdev)
-{
-	bool is_vid_ok = (pdev->vendor == PCI_VENDOR_ID_AQUANTIA);
-	bool is_did_ok = ((pdev->device == HW_ATL_DEVICE_ID_0001) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D100) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D107) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D108) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D109));
-
-	bool is_rev_ok = (pdev->revision == 1U);
-
-	return (is_vid_ok && is_did_ok && is_rev_ok) ? &hw_atl_ops_ : NULL;
-}
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.h
index 6e1d527954c9..25fe954def03 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0.h
@@ -16,19 +16,11 @@
 
 #include "../aq_common.h"
 
-#ifndef PCI_VENDOR_ID_AQUANTIA
+extern const struct aq_hw_caps_s hw_atl_a0_caps_aqc100;
+extern const struct aq_hw_caps_s hw_atl_a0_caps_aqc107;
+extern const struct aq_hw_caps_s hw_atl_a0_caps_aqc108;
+extern const struct aq_hw_caps_s hw_atl_a0_caps_aqc109;
 
-#define PCI_VENDOR_ID_AQUANTIA  0x1D6A
-#define HW_ATL_DEVICE_ID_0001   0x0001
-#define HW_ATL_DEVICE_ID_D100   0xD100
-#define HW_ATL_DEVICE_ID_D107   0xD107
-#define HW_ATL_DEVICE_ID_D108   0xD108
-#define HW_ATL_DEVICE_ID_D109   0xD109
-
-#define HW_ATL_NIC_NAME "aQuantia AQtion 5Gbit Network Adapter"
-
-#endif
-
-struct aq_hw_ops *hw_atl_a0_get_ops_by_id(struct pci_dev *pdev);
+extern const struct aq_hw_ops hw_atl_ops_a0;
 
 #endif /* HW_ATL_A0_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0_internal.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0_internal.h
index 0592a0330cf0..1d8855558d74 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0_internal.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_a0_internal.h
@@ -88,69 +88,4 @@
 
 #define HW_ATL_A0_FW_VER_EXPECTED 0x01050006U
 
-/* Hardware tx descriptor */
-struct __packed hw_atl_txd_s {
-	u64 buf_addr;
-	u32 ctl;
-	u32 ctl2; /* 63..46 - payload length, 45 - ctx enable, 44 - ctx index */
-};
-
-/* Hardware tx context descriptor */
-struct __packed hw_atl_txc_s {
-	u32 rsvd;
-	u32 len;
-	u32 ctl;
-	u32 len2;
-};
-
-/* Hardware rx descriptor */
-struct __packed hw_atl_rxd_s {
-	u64 buf_addr;
-	u64 hdr_addr;
-};
-
-/* Hardware rx descriptor writeback */
-struct __packed hw_atl_rxd_wb_s {
-	u32 type;
-	u32 rss_hash;
-	u16 status;
-	u16 pkt_len;
-	u16 next_desc_ptr;
-	u16 vlan;
-};
-
-/* HW layer capabilities */
-static struct aq_hw_caps_s hw_atl_a0_hw_caps_ = {
-	.ports = 1U,
-	.is_64_dma = true,
-	.msix_irqs = 4U,
-	.irq_mask = ~0U,
-	.vecs = HW_ATL_A0_RSS_MAX,
-	.tcs = HW_ATL_A0_TC_MAX,
-	.rxd_alignment = 1U,
-	.rxd_size = HW_ATL_A0_RXD_SIZE,
-	.rxds = 248U,
-	.txd_alignment = 1U,
-	.txd_size = HW_ATL_A0_TXD_SIZE,
-	.txds = 8U * 1024U,
-	.txhwb_alignment = 4096U,
-	.tx_rings = HW_ATL_A0_TX_RINGS,
-	.rx_rings = HW_ATL_A0_RX_RINGS,
-	.hw_features = NETIF_F_HW_CSUM |
-			NETIF_F_RXCSUM |
-			NETIF_F_RXHASH |
-			NETIF_F_SG |
-			NETIF_F_TSO,
-	.hw_priv_flags = IFF_UNICAST_FLT,
-	.link_speed_msk = (HW_ATL_A0_RATE_10G |
-			HW_ATL_A0_RATE_5G |
-			HW_ATL_A0_RATE_2G5 |
-			HW_ATL_A0_RATE_1G |
-			HW_ATL_A0_RATE_100M),
-	.flow_control = true,
-	.mtu = HW_ATL_A0_MTU_JUMBO,
-	.mac_regs_count = 88,
-	.fw_ver_expected = HW_ATL_A0_FW_VER_EXPECTED,
-};
-
 #endif /* HW_ATL_A0_INTERNAL_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
index e4a22ce7bf09..819f6bcf9b4e 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.c
@@ -12,83 +12,89 @@
 #include "../aq_hw.h"
 #include "../aq_hw_utils.h"
 #include "../aq_ring.h"
+#include "../aq_nic.h"
 #include "hw_atl_b0.h"
 #include "hw_atl_utils.h"
 #include "hw_atl_llh.h"
 #include "hw_atl_b0_internal.h"
 #include "hw_atl_llh_internal.h"
 
-static int hw_atl_b0_get_hw_caps(struct aq_hw_s *self,
-				 struct aq_hw_caps_s *aq_hw_caps,
-				 unsigned short device,
-				 unsigned short subsystem_device)
-{
-	memcpy(aq_hw_caps, &hw_atl_b0_hw_caps_, sizeof(*aq_hw_caps));
-
-	if (device == HW_ATL_DEVICE_ID_D108 && subsystem_device == 0x0001)
-		aq_hw_caps->link_speed_msk &= ~HW_ATL_B0_RATE_10G;
-
-	if (device == HW_ATL_DEVICE_ID_D109 && subsystem_device == 0x0001) {
-		aq_hw_caps->link_speed_msk &= ~HW_ATL_B0_RATE_10G;
-		aq_hw_caps->link_speed_msk &= ~HW_ATL_B0_RATE_5G;
-	}
-
-	return 0;
-}
-
-static struct aq_hw_s *hw_atl_b0_create(struct aq_pci_func_s *aq_pci_func,
-					unsigned int port,
-					struct aq_hw_ops *ops)
-{
-	struct hw_atl_s *self = NULL;
-
-	self = kzalloc(sizeof(*self), GFP_KERNEL);
-	if (!self)
-		goto err_exit;
-
-	self->base.aq_pci_func = aq_pci_func;
+#define DEFAULT_B0_BOARD_BASIC_CAPABILITIES \
+	.is_64_dma = true,	\
+	.msix_irqs = 4U,	\
+	.irq_mask = ~0U,	\
+	.vecs = HW_ATL_B0_RSS_MAX,	\
+	.tcs = HW_ATL_B0_TC_MAX,	\
+	.rxd_alignment = 1U,		\
+	.rxd_size = HW_ATL_B0_RXD_SIZE, \
+	.rxds = 4U * 1024U,		\
+	.txd_alignment = 1U,		\
+	.txd_size = HW_ATL_B0_TXD_SIZE, \
+	.txds = 8U * 1024U,		\
+	.txhwb_alignment = 4096U,	\
+	.tx_rings = HW_ATL_B0_TX_RINGS, \
+	.rx_rings = HW_ATL_B0_RX_RINGS, \
+	.hw_features = NETIF_F_HW_CSUM | \
+			NETIF_F_RXCSUM | \
+			NETIF_F_RXHASH | \
+			NETIF_F_SG |  \
+			NETIF_F_TSO | \
+			NETIF_F_LRO,  \
+	.hw_priv_flags = IFF_UNICAST_FLT,   \
+	.flow_control = true,		\
+	.mtu = HW_ATL_B0_MTU_JUMBO,	\
+	.mac_regs_count = 88,		\
+	.hw_alive_check_addr = 0x10U
+
+const struct aq_hw_caps_s hw_atl_b0_caps_aqc100 = {
+	DEFAULT_B0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_FIBRE,
+	.link_speed_msk = HW_ATL_B0_RATE_10G |
+			  HW_ATL_B0_RATE_5G  |
+			  HW_ATL_B0_RATE_2G5 |
+			  HW_ATL_B0_RATE_1G  |
+			  HW_ATL_B0_RATE_100M,
+};
 
-	self->base.not_ff_addr = 0x10U;
+const struct aq_hw_caps_s hw_atl_b0_caps_aqc107 = {
+	DEFAULT_B0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_TP,
+	.link_speed_msk = HW_ATL_B0_RATE_10G |
+			  HW_ATL_B0_RATE_5G  |
+			  HW_ATL_B0_RATE_2G5 |
+			  HW_ATL_B0_RATE_1G  |
+			  HW_ATL_B0_RATE_100M,
+};
 
-err_exit:
-	return (struct aq_hw_s *)self;
-}
+const struct aq_hw_caps_s hw_atl_b0_caps_aqc108 = {
+	DEFAULT_B0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_TP,
+	.link_speed_msk = HW_ATL_B0_RATE_5G  |
+			  HW_ATL_B0_RATE_2G5 |
+			  HW_ATL_B0_RATE_1G  |
+			  HW_ATL_B0_RATE_100M,
+};
 
-static void hw_atl_b0_destroy(struct aq_hw_s *self)
-{
-	kfree(self);
-}
+const struct aq_hw_caps_s hw_atl_b0_caps_aqc109 = {
+	DEFAULT_B0_BOARD_BASIC_CAPABILITIES,
+	.media_type = AQ_HW_MEDIA_TYPE_TP,
+	.link_speed_msk = HW_ATL_B0_RATE_2G5 |
+			  HW_ATL_B0_RATE_1G  |
+			  HW_ATL_B0_RATE_100M,
+};
 
 static int hw_atl_b0_hw_reset(struct aq_hw_s *self)
 {
 	int err = 0;
 
-	glb_glb_reg_res_dis_set(self, 1U);
-	pci_pci_reg_res_dis_set(self, 0U);
-	rx_rx_reg_res_dis_set(self, 0U);
-	tx_tx_reg_res_dis_set(self, 0U);
-
-	HW_ATL_FLUSH();
-	glb_soft_res_set(self, 1);
-
-	/* check 10 times by 1ms */
-	AQ_HW_WAIT_FOR(glb_soft_res_get(self) == 0, 1000U, 10U);
-	if (err < 0)
-		goto err_exit;
-
-	itr_irq_reg_res_dis_set(self, 0U);
-	itr_res_irq_set(self, 1U);
-
-	/* check 10 times by 1ms */
-	AQ_HW_WAIT_FOR(itr_res_irq_get(self) == 0, 1000U, 10U);
-	if (err < 0)
-		goto err_exit;
+	err = hw_atl_utils_soft_reset(self);
+	if (err)
+		return err;
 
-	hw_atl_utils_mpi_set(self, MPI_RESET, 0x0U);
+	self->aq_fw_ops->set_state(self, MPI_RESET);
 
 	err = aq_hw_err_from_flags(self);
 
-err_exit:
 	return err;
 }
 
@@ -100,51 +106,53 @@ static int hw_atl_b0_hw_qos_set(struct aq_hw_s *self)
 	bool is_rx_flow_control = false;
 
 	/* TPS Descriptor rate init */
-	tps_tx_pkt_shed_desc_rate_curr_time_res_set(self, 0x0U);
-	tps_tx_pkt_shed_desc_rate_lim_set(self, 0xA);
+	hw_atl_tps_tx_pkt_shed_desc_rate_curr_time_res_set(self, 0x0U);
+	hw_atl_tps_tx_pkt_shed_desc_rate_lim_set(self, 0xA);
 
 	/* TPS VM init */
-	tps_tx_pkt_shed_desc_vm_arb_mode_set(self, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_vm_arb_mode_set(self, 0U);
 
 	/* TPS TC credits init */
-	tps_tx_pkt_shed_desc_tc_arb_mode_set(self, 0U);
-	tps_tx_pkt_shed_data_arb_mode_set(self, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_arb_mode_set(self, 0U);
+	hw_atl_tps_tx_pkt_shed_data_arb_mode_set(self, 0U);
 
-	tps_tx_pkt_shed_tc_data_max_credit_set(self, 0xFFF, 0U);
-	tps_tx_pkt_shed_tc_data_weight_set(self, 0x64, 0U);
-	tps_tx_pkt_shed_desc_tc_max_credit_set(self, 0x50, 0U);
-	tps_tx_pkt_shed_desc_tc_weight_set(self, 0x1E, 0U);
+	hw_atl_tps_tx_pkt_shed_tc_data_max_credit_set(self, 0xFFF, 0U);
+	hw_atl_tps_tx_pkt_shed_tc_data_weight_set(self, 0x64, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_max_credit_set(self, 0x50, 0U);
+	hw_atl_tps_tx_pkt_shed_desc_tc_weight_set(self, 0x1E, 0U);
 
 	/* Tx buf size */
 	buff_size = HW_ATL_B0_TXBUF_MAX;
 
-	tpb_tx_pkt_buff_size_per_tc_set(self, buff_size, tc);
-	tpb_tx_buff_hi_threshold_per_tc_set(self,
-					    (buff_size * (1024 / 32U) * 66U) /
-					    100U, tc);
-	tpb_tx_buff_lo_threshold_per_tc_set(self,
-					    (buff_size * (1024 / 32U) * 50U) /
-					    100U, tc);
+	hw_atl_tpb_tx_pkt_buff_size_per_tc_set(self, buff_size, tc);
+	hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024 / 32U) * 66U) /
+						   100U, tc);
+	hw_atl_tpb_tx_buff_lo_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024 / 32U) * 50U) /
+						   100U, tc);
 
 	/* QoS Rx buf size per TC */
 	tc = 0;
 	is_rx_flow_control = (AQ_NIC_FC_RX & self->aq_nic_cfg->flow_control);
 	buff_size = HW_ATL_B0_RXBUF_MAX;
 
-	rpb_rx_pkt_buff_size_per_tc_set(self, buff_size, tc);
-	rpb_rx_buff_hi_threshold_per_tc_set(self,
-					    (buff_size *
-					    (1024U / 32U) * 66U) /
-					    100U, tc);
-	rpb_rx_buff_lo_threshold_per_tc_set(self,
-					    (buff_size *
-					    (1024U / 32U) * 50U) /
-					    100U, tc);
-	rpb_rx_xoff_en_per_tc_set(self, is_rx_flow_control ? 1U : 0U, tc);
+	hw_atl_rpb_rx_pkt_buff_size_per_tc_set(self, buff_size, tc);
+	hw_atl_rpb_rx_buff_hi_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024U / 32U) * 66U) /
+						   100U, tc);
+	hw_atl_rpb_rx_buff_lo_threshold_per_tc_set(self,
+						   (buff_size *
+						   (1024U / 32U) * 50U) /
+						   100U, tc);
+	hw_atl_rpb_rx_xoff_en_per_tc_set(self, is_rx_flow_control ? 1U : 0U, tc);
 
 	/* QoS 802.1p priority -> TC mapping */
 	for (i_priority = 8U; i_priority--;)
-		rpf_rpb_user_priority_tc_map_set(self, i_priority, 0U);
+		hw_atl_rpf_rpb_user_priority_tc_map_set(self, i_priority, 0U);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -152,20 +160,19 @@ static int hw_atl_b0_hw_qos_set(struct aq_hw_s *self)
 static int hw_atl_b0_hw_rss_hash_set(struct aq_hw_s *self,
 				     struct aq_rss_parameters *rss_params)
 {
-	struct aq_nic_cfg_s *cfg = NULL;
+	struct aq_nic_cfg_s *cfg = self->aq_nic_cfg;
 	int err = 0;
 	unsigned int i = 0U;
 	unsigned int addr = 0U;
 
-	cfg = self->aq_nic_cfg;
-
 	for (i = 10, addr = 0U; i--; ++addr) {
 		u32 key_data = cfg->is_rss ?
 			__swab32(rss_params->hash_secret_key[i]) : 0U;
-		rpf_rss_key_wr_data_set(self, key_data);
-		rpf_rss_key_addr_set(self, addr);
-		rpf_rss_key_wr_en_set(self, 1U);
-		AQ_HW_WAIT_FOR(rpf_rss_key_wr_en_get(self) == 0, 1000U, 10U);
+		hw_atl_rpf_rss_key_wr_data_set(self, key_data);
+		hw_atl_rpf_rss_key_addr_set(self, addr);
+		hw_atl_rpf_rss_key_wr_en_set(self, 1U);
+		AQ_HW_WAIT_FOR(hw_atl_rpf_rss_key_wr_en_get(self) == 0,
+			       1000U, 10U);
 		if (err < 0)
 			goto err_exit;
 	}
@@ -194,11 +201,12 @@ static int hw_atl_b0_hw_rss_set(struct aq_hw_s *self,
 			((i * 3U) & 0xFU));
 	}
 
-	for (i = AQ_DIMOF(bitary); i--;) {
-		rpf_rss_redir_tbl_wr_data_set(self, bitary[i]);
-		rpf_rss_redir_tbl_addr_set(self, i);
-		rpf_rss_redir_wr_en_set(self, 1U);
-		AQ_HW_WAIT_FOR(rpf_rss_redir_wr_en_get(self) == 0, 1000U, 10U);
+	for (i = ARRAY_SIZE(bitary); i--;) {
+		hw_atl_rpf_rss_redir_tbl_wr_data_set(self, bitary[i]);
+		hw_atl_rpf_rss_redir_tbl_addr_set(self, i);
+		hw_atl_rpf_rss_redir_wr_en_set(self, 1U);
+		AQ_HW_WAIT_FOR(hw_atl_rpf_rss_redir_wr_en_get(self) == 0,
+			       1000U, 10U);
 		if (err < 0)
 			goto err_exit;
 	}
@@ -215,15 +223,15 @@ static int hw_atl_b0_hw_offload_set(struct aq_hw_s *self,
 	unsigned int i;
 
 	/* TX checksums offloads*/
-	tpo_ipv4header_crc_offload_en_set(self, 1);
-	tpo_tcp_udp_crc_offload_en_set(self, 1);
+	hw_atl_tpo_ipv4header_crc_offload_en_set(self, 1);
+	hw_atl_tpo_tcp_udp_crc_offload_en_set(self, 1);
 
 	/* RX checksums offloads*/
-	rpo_ipv4header_crc_offload_en_set(self, 1);
-	rpo_tcp_udp_crc_offload_en_set(self, 1);
+	hw_atl_rpo_ipv4header_crc_offload_en_set(self, 1);
+	hw_atl_rpo_tcp_udp_crc_offload_en_set(self, 1);
 
 	/* LSO offloads*/
-	tdm_large_send_offload_en_set(self, 0xFFFFFFFFU);
+	hw_atl_tdm_large_send_offload_en_set(self, 0xFFFFFFFFU);
 
 /* LRO offloads */
 	{
@@ -232,43 +240,44 @@ static int hw_atl_b0_hw_offload_set(struct aq_hw_s *self,
 			((2U < HW_ATL_B0_LRO_RXD_MAX) ? 0x1U : 0x0));
 
 		for (i = 0; i < HW_ATL_B0_RINGS_MAX; i++)
-			rpo_lro_max_num_of_descriptors_set(self, val, i);
+			hw_atl_rpo_lro_max_num_of_descriptors_set(self, val, i);
 
-		rpo_lro_time_base_divider_set(self, 0x61AU);
-		rpo_lro_inactive_interval_set(self, 0);
-		rpo_lro_max_coalescing_interval_set(self, 2);
+		hw_atl_rpo_lro_time_base_divider_set(self, 0x61AU);
+		hw_atl_rpo_lro_inactive_interval_set(self, 0);
+		hw_atl_rpo_lro_max_coalescing_interval_set(self, 2);
 
-		rpo_lro_qsessions_lim_set(self, 1U);
+		hw_atl_rpo_lro_qsessions_lim_set(self, 1U);
 
-		rpo_lro_total_desc_lim_set(self, 2U);
+		hw_atl_rpo_lro_total_desc_lim_set(self, 2U);
 
-		rpo_lro_patch_optimization_en_set(self, 0U);
+		hw_atl_rpo_lro_patch_optimization_en_set(self, 0U);
 
-		rpo_lro_min_pay_of_first_pkt_set(self, 10U);
+		hw_atl_rpo_lro_min_pay_of_first_pkt_set(self, 10U);
 
-		rpo_lro_pkt_lim_set(self, 1U);
+		hw_atl_rpo_lro_pkt_lim_set(self, 1U);
 
-		rpo_lro_en_set(self, aq_nic_cfg->is_lro ? 0xFFFFFFFFU : 0U);
+		hw_atl_rpo_lro_en_set(self,
+				      aq_nic_cfg->is_lro ? 0xFFFFFFFFU : 0U);
 	}
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_b0_hw_init_tx_path(struct aq_hw_s *self)
 {
-	thm_lso_tcp_flag_of_first_pkt_set(self, 0x0FF6U);
-	thm_lso_tcp_flag_of_middle_pkt_set(self, 0x0FF6U);
-	thm_lso_tcp_flag_of_last_pkt_set(self, 0x0F7FU);
+	hw_atl_thm_lso_tcp_flag_of_first_pkt_set(self, 0x0FF6U);
+	hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(self, 0x0FF6U);
+	hw_atl_thm_lso_tcp_flag_of_last_pkt_set(self, 0x0F7FU);
 
 	/* Tx interrupts */
-	tdm_tx_desc_wr_wb_irq_en_set(self, 1U);
+	hw_atl_tdm_tx_desc_wr_wb_irq_en_set(self, 1U);
 
 	/* misc */
 	aq_hw_write_reg(self, 0x00007040U, IS_CHIP_FEATURE(TPO2) ?
 			0x00010000U : 0x00000000U);
-	tdm_tx_dca_en_set(self, 0U);
-	tdm_tx_dca_mode_set(self, 0U);
+	hw_atl_tdm_tx_dca_en_set(self, 0U);
+	hw_atl_tdm_tx_dca_mode_set(self, 0U);
 
-	tpb_tx_path_scp_ins_en_set(self, 1U);
+	hw_atl_tpb_tx_path_scp_ins_en_set(self, 1U);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -279,55 +288,55 @@ static int hw_atl_b0_hw_init_rx_path(struct aq_hw_s *self)
 	int i;
 
 	/* Rx TC/RSS number config */
-	rpb_rpf_rx_traf_class_mode_set(self, 1U);
+	hw_atl_rpb_rpf_rx_traf_class_mode_set(self, 1U);
 
 	/* Rx flow control */
-	rpb_rx_flow_ctl_mode_set(self, 1U);
+	hw_atl_rpb_rx_flow_ctl_mode_set(self, 1U);
 
 	/* RSS Ring selection */
-	reg_rx_flr_rss_control1set(self, cfg->is_rss ?
+	hw_atl_reg_rx_flr_rss_control1set(self, cfg->is_rss ?
 					0xB3333333U : 0x00000000U);
 
 	/* Multicast filters */
 	for (i = HW_ATL_B0_MAC_MAX; i--;) {
-		rpfl2_uc_flr_en_set(self, (i == 0U) ? 1U : 0U, i);
-		rpfl2unicast_flr_act_set(self, 1U, i);
+		hw_atl_rpfl2_uc_flr_en_set(self, (i == 0U) ? 1U : 0U, i);
+		hw_atl_rpfl2unicast_flr_act_set(self, 1U, i);
 	}
 
-	reg_rx_flr_mcst_flr_msk_set(self, 0x00000000U);
-	reg_rx_flr_mcst_flr_set(self, 0x00010FFFU, 0U);
+	hw_atl_reg_rx_flr_mcst_flr_msk_set(self, 0x00000000U);
+	hw_atl_reg_rx_flr_mcst_flr_set(self, 0x00010FFFU, 0U);
 
 	/* Vlan filters */
-	rpf_vlan_outer_etht_set(self, 0x88A8U);
-	rpf_vlan_inner_etht_set(self, 0x8100U);
+	hw_atl_rpf_vlan_outer_etht_set(self, 0x88A8U);
+	hw_atl_rpf_vlan_inner_etht_set(self, 0x8100U);
 
 	if (cfg->vlan_id) {
-		rpf_vlan_flr_act_set(self, 1U, 0U);
-		rpf_vlan_id_flr_set(self, 0U, 0U);
-		rpf_vlan_flr_en_set(self, 0U, 0U);
+		hw_atl_rpf_vlan_flr_act_set(self, 1U, 0U);
+		hw_atl_rpf_vlan_id_flr_set(self, 0U, 0U);
+		hw_atl_rpf_vlan_flr_en_set(self, 0U, 0U);
 
-		rpf_vlan_accept_untagged_packets_set(self, 1U);
-		rpf_vlan_untagged_act_set(self, 1U);
+		hw_atl_rpf_vlan_accept_untagged_packets_set(self, 1U);
+		hw_atl_rpf_vlan_untagged_act_set(self, 1U);
 
-		rpf_vlan_flr_act_set(self, 1U, 1U);
-		rpf_vlan_id_flr_set(self, cfg->vlan_id, 0U);
-		rpf_vlan_flr_en_set(self, 1U, 1U);
+		hw_atl_rpf_vlan_flr_act_set(self, 1U, 1U);
+		hw_atl_rpf_vlan_id_flr_set(self, cfg->vlan_id, 0U);
+		hw_atl_rpf_vlan_flr_en_set(self, 1U, 1U);
 	} else {
-		rpf_vlan_prom_mode_en_set(self, 1);
+		hw_atl_rpf_vlan_prom_mode_en_set(self, 1);
 	}
 
 	/* Rx Interrupts */
-	rdm_rx_desc_wr_wb_irq_en_set(self, 1U);
+	hw_atl_rdm_rx_desc_wr_wb_irq_en_set(self, 1U);
 
 	/* misc */
 	aq_hw_write_reg(self, 0x00005040U,
 			IS_CHIP_FEATURE(RPF2) ? 0x000F0000U : 0x00000000U);
 
-	rpfl2broadcast_flr_act_set(self, 1U);
-	rpfl2broadcast_count_threshold_set(self, 0xFFFFU & (~0U / 256U));
+	hw_atl_rpfl2broadcast_flr_act_set(self, 1U);
+	hw_atl_rpfl2broadcast_count_threshold_set(self, 0xFFFFU & (~0U / 256U));
 
-	rdm_rx_dca_en_set(self, 0U);
-	rdm_rx_dca_mode_set(self, 0U);
+	hw_atl_rdm_rx_dca_en_set(self, 0U);
+	hw_atl_rdm_rx_dca_mode_set(self, 0U);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -346,10 +355,10 @@ static int hw_atl_b0_hw_mac_addr_set(struct aq_hw_s *self, u8 *mac_addr)
 	l = (mac_addr[2] << 24) | (mac_addr[3] << 16) |
 		(mac_addr[4] << 8) | mac_addr[5];
 
-	rpfl2_uc_flr_en_set(self, 0U, HW_ATL_B0_MAC);
-	rpfl2unicast_dest_addresslsw_set(self, l, HW_ATL_B0_MAC);
-	rpfl2unicast_dest_addressmsw_set(self, h, HW_ATL_B0_MAC);
-	rpfl2_uc_flr_en_set(self, 1U, HW_ATL_B0_MAC);
+	hw_atl_rpfl2_uc_flr_en_set(self, 0U, HW_ATL_B0_MAC);
+	hw_atl_rpfl2unicast_dest_addresslsw_set(self, l, HW_ATL_B0_MAC);
+	hw_atl_rpfl2unicast_dest_addressmsw_set(self, h, HW_ATL_B0_MAC);
+	hw_atl_rpfl2_uc_flr_en_set(self, 1U, HW_ATL_B0_MAC);
 
 	err = aq_hw_err_from_flags(self);
 
@@ -357,9 +366,7 @@ static int hw_atl_b0_hw_mac_addr_set(struct aq_hw_s *self, u8 *mac_addr)
 	return err;
 }
 
-static int hw_atl_b0_hw_init(struct aq_hw_s *self,
-			     struct aq_nic_cfg_s *aq_nic_cfg,
-			     u8 *mac_addr)
+static int hw_atl_b0_hw_init(struct aq_hw_s *self, u8 *mac_addr)
 {
 	static u32 aq_hw_atl_igcr_table_[4][2] = {
 		{ 0x20000000U, 0x20000000U }, /* AQ_IRQ_INVALID */
@@ -371,51 +378,50 @@ static int hw_atl_b0_hw_init(struct aq_hw_s *self,
 	int err = 0;
 	u32 val;
 
-	self->aq_nic_cfg = aq_nic_cfg;
-
-	hw_atl_utils_hw_chip_features_init(self,
-					   &PHAL_ATLANTIC_B0->chip_features);
+	struct aq_nic_cfg_s *aq_nic_cfg = self->aq_nic_cfg;
 
 	hw_atl_b0_hw_init_tx_path(self);
 	hw_atl_b0_hw_init_rx_path(self);
 
 	hw_atl_b0_hw_mac_addr_set(self, mac_addr);
 
-	hw_atl_utils_mpi_set(self, MPI_INIT, aq_nic_cfg->link_speed_msk);
+	self->aq_fw_ops->set_link_speed(self, aq_nic_cfg->link_speed_msk);
+	self->aq_fw_ops->set_state(self, MPI_INIT);
 
 	hw_atl_b0_hw_qos_set(self);
 	hw_atl_b0_hw_rss_set(self, &aq_nic_cfg->aq_rss);
 	hw_atl_b0_hw_rss_hash_set(self, &aq_nic_cfg->aq_rss);
 
 	/* Force limit MRRS on RDM/TDM to 2K */
-	val = aq_hw_read_reg(self, pci_reg_control6_adr);
-	aq_hw_write_reg(self, pci_reg_control6_adr, (val & ~0x707) | 0x404);
+	val = aq_hw_read_reg(self, HW_ATL_PCI_REG_CONTROL6_ADR);
+	aq_hw_write_reg(self, HW_ATL_PCI_REG_CONTROL6_ADR,
+			(val & ~0x707) | 0x404);
 
 	/* TX DMA total request limit. B0 hardware is not capable to
 	 * handle more than (8K-MRRS) incoming DMA data.
 	 * Value 24 in 256byte units
 	 */
-	aq_hw_write_reg(self, tx_dma_total_req_limit_adr, 24);
+	aq_hw_write_reg(self, HW_ATL_TX_DMA_TOTAL_REQ_LIMIT_ADR, 24);
 
 	/* Reset link status and read out initial hardware counters */
 	self->aq_link_status.mbps = 0;
-	hw_atl_utils_update_stats(self);
+	self->aq_fw_ops->update_stats(self);
 
 	err = aq_hw_err_from_flags(self);
 	if (err < 0)
 		goto err_exit;
 
 	/* Interrupts */
-	reg_irq_glb_ctl_set(self,
-			    aq_hw_atl_igcr_table_[aq_nic_cfg->irq_type]
+	hw_atl_reg_irq_glb_ctl_set(self,
+				   aq_hw_atl_igcr_table_[aq_nic_cfg->irq_type]
 						 [(aq_nic_cfg->vecs > 1U) ?
 						 1 : 0]);
 
-	itr_irq_auto_masklsw_set(self, aq_nic_cfg->aq_hw_caps->irq_mask);
+	hw_atl_itr_irq_auto_masklsw_set(self, aq_nic_cfg->aq_hw_caps->irq_mask);
 
 	/* Interrupts */
-	reg_gen_irq_map_set(self,
-			    ((HW_ATL_B0_ERR_INT << 0x18) | (1U << 0x1F)) |
+	hw_atl_reg_gen_irq_map_set(self,
+				   ((HW_ATL_B0_ERR_INT << 0x18) | (1U << 0x1F)) |
 			    ((HW_ATL_B0_ERR_INT << 0x10) | (1U << 0x17)), 0U);
 
 	hw_atl_b0_hw_offload_set(self, aq_nic_cfg);
@@ -427,28 +433,28 @@ static int hw_atl_b0_hw_init(struct aq_hw_s *self,
 static int hw_atl_b0_hw_ring_tx_start(struct aq_hw_s *self,
 				      struct aq_ring_s *ring)
 {
-	tdm_tx_desc_en_set(self, 1, ring->idx);
+	hw_atl_tdm_tx_desc_en_set(self, 1, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_b0_hw_ring_rx_start(struct aq_hw_s *self,
 				      struct aq_ring_s *ring)
 {
-	rdm_rx_desc_en_set(self, 1, ring->idx);
+	hw_atl_rdm_rx_desc_en_set(self, 1, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_b0_hw_start(struct aq_hw_s *self)
 {
-	tpb_tx_buff_en_set(self, 1);
-	rpb_rx_buff_en_set(self, 1);
+	hw_atl_tpb_tx_buff_en_set(self, 1);
+	hw_atl_rpb_rx_buff_en_set(self, 1);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_b0_hw_tx_ring_tail_update(struct aq_hw_s *self,
 					    struct aq_ring_s *ring)
 {
-	reg_tx_dma_desc_tail_ptr_set(self, ring->sw_tail, ring->idx);
+	hw_atl_reg_tx_dma_desc_tail_ptr_set(self, ring->sw_tail, ring->idx);
 	return 0;
 }
 
@@ -534,36 +540,36 @@ static int hw_atl_b0_hw_ring_rx_init(struct aq_hw_s *self,
 	u32 dma_desc_addr_lsw = (u32)aq_ring->dx_ring_pa;
 	u32 dma_desc_addr_msw = (u32)(((u64)aq_ring->dx_ring_pa) >> 32);
 
-	rdm_rx_desc_en_set(self, false, aq_ring->idx);
+	hw_atl_rdm_rx_desc_en_set(self, false, aq_ring->idx);
 
-	rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
 
-	reg_rx_dma_desc_base_addresslswset(self, dma_desc_addr_lsw,
-					   aq_ring->idx);
+	hw_atl_reg_rx_dma_desc_base_addresslswset(self, dma_desc_addr_lsw,
+						  aq_ring->idx);
 
-	reg_rx_dma_desc_base_addressmswset(self,
-					   dma_desc_addr_msw, aq_ring->idx);
+	hw_atl_reg_rx_dma_desc_base_addressmswset(self,
+						  dma_desc_addr_msw, aq_ring->idx);
 
-	rdm_rx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
 
-	rdm_rx_desc_data_buff_size_set(self,
-				       AQ_CFG_RX_FRAME_MAX / 1024U,
+	hw_atl_rdm_rx_desc_data_buff_size_set(self,
+					      AQ_CFG_RX_FRAME_MAX / 1024U,
 				       aq_ring->idx);
 
-	rdm_rx_desc_head_buff_size_set(self, 0U, aq_ring->idx);
-	rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
-	rpo_rx_desc_vlan_stripping_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_head_buff_size_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_desc_head_splitting_set(self, 0U, aq_ring->idx);
+	hw_atl_rpo_rx_desc_vlan_stripping_set(self, 0U, aq_ring->idx);
 
 	/* Rx ring set mode */
 
 	/* Mapping interrupt vector */
-	itr_irq_map_rx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
-	itr_irq_map_en_rx_set(self, true, aq_ring->idx);
+	hw_atl_itr_irq_map_rx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
+	hw_atl_itr_irq_map_en_rx_set(self, true, aq_ring->idx);
 
-	rdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
-	rdm_rx_desc_dca_en_set(self, 0U, aq_ring->idx);
-	rdm_rx_head_dca_en_set(self, 0U, aq_ring->idx);
-	rdm_rx_pld_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
+	hw_atl_rdm_rx_desc_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_head_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_rdm_rx_pld_dca_en_set(self, 0U, aq_ring->idx);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -575,25 +581,25 @@ static int hw_atl_b0_hw_ring_tx_init(struct aq_hw_s *self,
 	u32 dma_desc_lsw_addr = (u32)aq_ring->dx_ring_pa;
 	u32 dma_desc_msw_addr = (u32)(((u64)aq_ring->dx_ring_pa) >> 32);
 
-	reg_tx_dma_desc_base_addresslswset(self, dma_desc_lsw_addr,
-					   aq_ring->idx);
+	hw_atl_reg_tx_dma_desc_base_addresslswset(self, dma_desc_lsw_addr,
+						  aq_ring->idx);
 
-	reg_tx_dma_desc_base_addressmswset(self, dma_desc_msw_addr,
-					   aq_ring->idx);
+	hw_atl_reg_tx_dma_desc_base_addressmswset(self, dma_desc_msw_addr,
+						  aq_ring->idx);
 
-	tdm_tx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
+	hw_atl_tdm_tx_desc_len_set(self, aq_ring->size / 8U, aq_ring->idx);
 
 	hw_atl_b0_hw_tx_ring_tail_update(self, aq_ring);
 
 	/* Set Tx threshold */
-	tdm_tx_desc_wr_wb_threshold_set(self, 0U, aq_ring->idx);
+	hw_atl_tdm_tx_desc_wr_wb_threshold_set(self, 0U, aq_ring->idx);
 
 	/* Mapping interrupt vector */
-	itr_irq_map_tx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
-	itr_irq_map_en_tx_set(self, true, aq_ring->idx);
+	hw_atl_itr_irq_map_tx_set(self, aq_ring_param->vec_idx, aq_ring->idx);
+	hw_atl_itr_irq_map_en_tx_set(self, true, aq_ring->idx);
 
-	tdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
-	tdm_tx_desc_dca_en_set(self, 0U, aq_ring->idx);
+	hw_atl_tdm_cpu_id_set(self, aq_ring_param->cpu, aq_ring->idx);
+	hw_atl_tdm_tx_desc_dca_en_set(self, 0U, aq_ring->idx);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -614,7 +620,7 @@ static int hw_atl_b0_hw_ring_rx_fill(struct aq_hw_s *self,
 		rxd->hdr_addr = 0U;
 	}
 
-	reg_rx_dma_desc_tail_ptr_set(self, sw_tail_old, ring->idx);
+	hw_atl_reg_rx_dma_desc_tail_ptr_set(self, sw_tail_old, ring->idx);
 
 	return aq_hw_err_from_flags(self);
 }
@@ -623,9 +629,9 @@ static int hw_atl_b0_hw_ring_tx_head_update(struct aq_hw_s *self,
 					    struct aq_ring_s *ring)
 {
 	int err = 0;
-	unsigned int hw_head_ = tdm_tx_desc_head_ptr_get(self, ring->idx);
+	unsigned int hw_head_ = hw_atl_tdm_tx_desc_head_ptr_get(self, ring->idx);
 
-	if (aq_utils_obj_test(&self->header.flags, AQ_HW_FLAG_ERR_UNPLUG)) {
+	if (aq_utils_obj_test(&self->flags, AQ_HW_FLAG_ERR_UNPLUG)) {
 		err = -ENXIO;
 		goto err_exit;
 	}
@@ -728,22 +734,22 @@ static int hw_atl_b0_hw_ring_rx_receive(struct aq_hw_s *self,
 
 static int hw_atl_b0_hw_irq_enable(struct aq_hw_s *self, u64 mask)
 {
-	itr_irq_msk_setlsw_set(self, LODWORD(mask));
+	hw_atl_itr_irq_msk_setlsw_set(self, LODWORD(mask));
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_b0_hw_irq_disable(struct aq_hw_s *self, u64 mask)
 {
-	itr_irq_msk_clearlsw_set(self, LODWORD(mask));
-	itr_irq_status_clearlsw_set(self, LODWORD(mask));
+	hw_atl_itr_irq_msk_clearlsw_set(self, LODWORD(mask));
+	hw_atl_itr_irq_status_clearlsw_set(self, LODWORD(mask));
 
-	atomic_inc(&PHAL_ATLANTIC_B0->dpc);
+	atomic_inc(&self->dpc);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_b0_hw_irq_read(struct aq_hw_s *self, u64 *mask)
 {
-	*mask = itr_irq_statuslsw_get(self);
+	*mask = hw_atl_itr_irq_statuslsw_get(self);
 	return aq_hw_err_from_flags(self);
 }
 
@@ -754,20 +760,20 @@ static int hw_atl_b0_hw_packet_filter_set(struct aq_hw_s *self,
 {
 	unsigned int i = 0U;
 
-	rpfl2promiscuous_mode_en_set(self, IS_FILTER_ENABLED(IFF_PROMISC));
-	rpfl2multicast_flr_en_set(self,
-				  IS_FILTER_ENABLED(IFF_MULTICAST), 0);
+	hw_atl_rpfl2promiscuous_mode_en_set(self, IS_FILTER_ENABLED(IFF_PROMISC));
+	hw_atl_rpfl2multicast_flr_en_set(self,
+					 IS_FILTER_ENABLED(IFF_MULTICAST), 0);
 
-	rpfl2_accept_all_mc_packets_set(self,
-					IS_FILTER_ENABLED(IFF_ALLMULTI));
+	hw_atl_rpfl2_accept_all_mc_packets_set(self,
+					       IS_FILTER_ENABLED(IFF_ALLMULTI));
 
-	rpfl2broadcast_en_set(self, IS_FILTER_ENABLED(IFF_BROADCAST));
+	hw_atl_rpfl2broadcast_en_set(self, IS_FILTER_ENABLED(IFF_BROADCAST));
 
 	self->aq_nic_cfg->is_mc_list_enabled = IS_FILTER_ENABLED(IFF_MULTICAST);
 
 	for (i = HW_ATL_B0_MAC_MIN; i < HW_ATL_B0_MAC_MAX; ++i)
-		rpfl2_uc_flr_en_set(self,
-				    (self->aq_nic_cfg->is_mc_list_enabled &&
+		hw_atl_rpfl2_uc_flr_en_set(self,
+					   (self->aq_nic_cfg->is_mc_list_enabled &&
 				    (i <= self->aq_nic_cfg->mc_list_count)) ?
 				    1U : 0U, i);
 
@@ -796,16 +802,16 @@ static int hw_atl_b0_hw_multicast_list_set(struct aq_hw_s *self,
 		u32 l = (ar_mac[i][2] << 24) | (ar_mac[i][3] << 16) |
 					(ar_mac[i][4] << 8) | ar_mac[i][5];
 
-		rpfl2_uc_flr_en_set(self, 0U, HW_ATL_B0_MAC_MIN + i);
+		hw_atl_rpfl2_uc_flr_en_set(self, 0U, HW_ATL_B0_MAC_MIN + i);
 
-		rpfl2unicast_dest_addresslsw_set(self,
-						 l, HW_ATL_B0_MAC_MIN + i);
+		hw_atl_rpfl2unicast_dest_addresslsw_set(self,
+							l, HW_ATL_B0_MAC_MIN + i);
 
-		rpfl2unicast_dest_addressmsw_set(self,
-						 h, HW_ATL_B0_MAC_MIN + i);
+		hw_atl_rpfl2unicast_dest_addressmsw_set(self,
+							h, HW_ATL_B0_MAC_MIN + i);
 
-		rpfl2_uc_flr_en_set(self,
-				    (self->aq_nic_cfg->is_mc_list_enabled),
+		hw_atl_rpfl2_uc_flr_en_set(self,
+					   (self->aq_nic_cfg->is_mc_list_enabled),
 				    HW_ATL_B0_MAC_MIN + i);
 	}
 
@@ -824,10 +830,10 @@ static int hw_atl_b0_hw_interrupt_moderation_set(struct aq_hw_s *self)
 	switch (self->aq_nic_cfg->itr) {
 	case  AQ_CFG_INTERRUPT_MODERATION_ON:
 	case  AQ_CFG_INTERRUPT_MODERATION_AUTO:
-		tdm_tx_desc_wr_wb_irq_en_set(self, 0U);
-		tdm_tdm_intr_moder_en_set(self, 1U);
-		rdm_rx_desc_wr_wb_irq_en_set(self, 0U);
-		rdm_rdm_intr_moder_en_set(self, 1U);
+		hw_atl_tdm_tx_desc_wr_wb_irq_en_set(self, 0U);
+		hw_atl_tdm_tdm_intr_moder_en_set(self, 1U);
+		hw_atl_rdm_rx_desc_wr_wb_irq_en_set(self, 0U);
+		hw_atl_rdm_rdm_intr_moder_en_set(self, 1U);
 
 		if (self->aq_nic_cfg->itr == AQ_CFG_INTERRUPT_MODERATION_ON) {
 			/* HW timers are in 2us units */
@@ -887,18 +893,18 @@ static int hw_atl_b0_hw_interrupt_moderation_set(struct aq_hw_s *self)
 		}
 		break;
 	case AQ_CFG_INTERRUPT_MODERATION_OFF:
-		tdm_tx_desc_wr_wb_irq_en_set(self, 1U);
-		tdm_tdm_intr_moder_en_set(self, 0U);
-		rdm_rx_desc_wr_wb_irq_en_set(self, 1U);
-		rdm_rdm_intr_moder_en_set(self, 0U);
+		hw_atl_tdm_tx_desc_wr_wb_irq_en_set(self, 1U);
+		hw_atl_tdm_tdm_intr_moder_en_set(self, 0U);
+		hw_atl_rdm_rx_desc_wr_wb_irq_en_set(self, 1U);
+		hw_atl_rdm_rdm_intr_moder_en_set(self, 0U);
 		itr_tx = 0U;
 		itr_rx = 0U;
 		break;
 	}
 
 	for (i = HW_ATL_B0_RINGS_MAX; i--;) {
-		reg_tx_intr_moder_ctrl_set(self, itr_tx, i);
-		reg_rx_intr_moder_ctrl_set(self, itr_rx, i);
+		hw_atl_reg_tx_intr_moder_ctrl_set(self, itr_tx, i);
+		hw_atl_reg_rx_intr_moder_ctrl_set(self, itr_rx, i);
 	}
 
 	return aq_hw_err_from_flags(self);
@@ -913,38 +919,19 @@ static int hw_atl_b0_hw_stop(struct aq_hw_s *self)
 static int hw_atl_b0_hw_ring_tx_stop(struct aq_hw_s *self,
 				     struct aq_ring_s *ring)
 {
-	tdm_tx_desc_en_set(self, 0U, ring->idx);
+	hw_atl_tdm_tx_desc_en_set(self, 0U, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
 static int hw_atl_b0_hw_ring_rx_stop(struct aq_hw_s *self,
 				     struct aq_ring_s *ring)
 {
-	rdm_rx_desc_en_set(self, 0U, ring->idx);
+	hw_atl_rdm_rx_desc_en_set(self, 0U, ring->idx);
 	return aq_hw_err_from_flags(self);
 }
 
-static int hw_atl_b0_hw_set_speed(struct aq_hw_s *self, u32 speed)
-{
-	int err = 0;
-
-	err = hw_atl_utils_mpi_set_speed(self, speed, MPI_INIT);
-	if (err < 0)
-		goto err_exit;
-
-err_exit:
-	return err;
-}
-
-static struct aq_hw_ops hw_atl_ops_ = {
-	.create               = hw_atl_b0_create,
-	.destroy              = hw_atl_b0_destroy,
-	.get_hw_caps          = hw_atl_b0_get_hw_caps,
-
-	.hw_get_mac_permanent = hw_atl_utils_get_mac_permanent,
+const struct aq_hw_ops hw_atl_ops_b0 = {
 	.hw_set_mac_address   = hw_atl_b0_hw_mac_addr_set,
-	.hw_get_link_status   = hw_atl_utils_mpi_get_link_status,
-	.hw_set_link_speed    = hw_atl_b0_hw_set_speed,
 	.hw_init              = hw_atl_b0_hw_init,
 	.hw_deinit            = hw_atl_utils_hw_deinit,
 	.hw_set_power         = hw_atl_utils_hw_set_power,
@@ -974,21 +961,6 @@ static struct aq_hw_ops hw_atl_ops_ = {
 	.hw_rss_set                  = hw_atl_b0_hw_rss_set,
 	.hw_rss_hash_set             = hw_atl_b0_hw_rss_hash_set,
 	.hw_get_regs                 = hw_atl_utils_hw_get_regs,
-	.hw_update_stats             = hw_atl_utils_update_stats,
 	.hw_get_hw_stats             = hw_atl_utils_get_hw_stats,
 	.hw_get_fw_version           = hw_atl_utils_get_fw_version,
 };
-
-struct aq_hw_ops *hw_atl_b0_get_ops_by_id(struct pci_dev *pdev)
-{
-	bool is_vid_ok = (pdev->vendor == PCI_VENDOR_ID_AQUANTIA);
-	bool is_did_ok = ((pdev->device == HW_ATL_DEVICE_ID_0001) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D100) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D107) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D108) ||
-			(pdev->device == HW_ATL_DEVICE_ID_D109));
-
-	bool is_rev_ok = (pdev->revision == 2U);
-
-	return (is_vid_ok && is_did_ok && is_rev_ok) ? &hw_atl_ops_ : NULL;
-}
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.h
index a1e1bce6c1f3..2cc8dacfdc27 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0.h
@@ -16,19 +16,27 @@
 
 #include "../aq_common.h"
 
-#ifndef PCI_VENDOR_ID_AQUANTIA
+extern const struct aq_hw_caps_s hw_atl_b0_caps_aqc100;
+extern const struct aq_hw_caps_s hw_atl_b0_caps_aqc107;
+extern const struct aq_hw_caps_s hw_atl_b0_caps_aqc108;
+extern const struct aq_hw_caps_s hw_atl_b0_caps_aqc109;
 
-#define PCI_VENDOR_ID_AQUANTIA  0x1D6A
-#define HW_ATL_DEVICE_ID_0001   0x0001
-#define HW_ATL_DEVICE_ID_D100   0xD100
-#define HW_ATL_DEVICE_ID_D107   0xD107
-#define HW_ATL_DEVICE_ID_D108   0xD108
-#define HW_ATL_DEVICE_ID_D109   0xD109
+#define hw_atl_b0_caps_aqc111 hw_atl_b0_caps_aqc108
+#define hw_atl_b0_caps_aqc112 hw_atl_b0_caps_aqc109
 
-#define HW_ATL_NIC_NAME "aQuantia AQtion 5Gbit Network Adapter"
+#define hw_atl_b0_caps_aqc100s hw_atl_b0_caps_aqc100
+#define hw_atl_b0_caps_aqc107s hw_atl_b0_caps_aqc107
+#define hw_atl_b0_caps_aqc108s hw_atl_b0_caps_aqc108
+#define hw_atl_b0_caps_aqc109s hw_atl_b0_caps_aqc109
 
-#endif
+#define hw_atl_b0_caps_aqc111s hw_atl_b0_caps_aqc108
+#define hw_atl_b0_caps_aqc112s hw_atl_b0_caps_aqc109
 
-struct aq_hw_ops *hw_atl_b0_get_ops_by_id(struct pci_dev *pdev);
+#define hw_atl_b0_caps_aqc111e hw_atl_b0_caps_aqc108
+#define hw_atl_b0_caps_aqc112e hw_atl_b0_caps_aqc109
+
+extern const struct aq_hw_ops hw_atl_ops_b0;
+
+#define hw_atl_ops_b1 hw_atl_ops_b0
 
 #endif /* HW_ATL_B0_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0_internal.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0_internal.h
index 9aa2c6edfca2..405d1455c222 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0_internal.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_b0_internal.h
@@ -142,70 +142,6 @@
 #define HW_ATL_INTR_MODER_MAX  0x1FF
 #define HW_ATL_INTR_MODER_MIN  0xFF
 
-/* Hardware tx descriptor */
-struct __packed hw_atl_txd_s {
-	u64 buf_addr;
-	u32 ctl;
-	u32 ctl2; /* 63..46 - payload length, 45 - ctx enable, 44 - ctx index */
-};
-
-/* Hardware tx context descriptor */
-struct __packed hw_atl_txc_s {
-	u32 rsvd;
-	u32 len;
-	u32 ctl;
-	u32 len2;
-};
-
-/* Hardware rx descriptor */
-struct __packed hw_atl_rxd_s {
-	u64 buf_addr;
-	u64 hdr_addr;
-};
-
-/* Hardware rx descriptor writeback */
-struct __packed hw_atl_rxd_wb_s {
-	u32 type;
-	u32 rss_hash;
-	u16 status;
-	u16 pkt_len;
-	u16 next_desc_ptr;
-	u16 vlan;
-};
-
 /* HW layer capabilities */
-static struct aq_hw_caps_s hw_atl_b0_hw_caps_ = {
-	.ports = 1U,
-	.is_64_dma = true,
-	.msix_irqs = 4U,
-	.irq_mask = ~0U,
-	.vecs = HW_ATL_B0_RSS_MAX,
-	.tcs = HW_ATL_B0_TC_MAX,
-	.rxd_alignment = 1U,
-	.rxd_size = HW_ATL_B0_RXD_SIZE,
-	.rxds = 8U * 1024U,
-	.txd_alignment = 1U,
-	.txd_size = HW_ATL_B0_TXD_SIZE,
-	.txds = 8U * 1024U,
-	.txhwb_alignment = 4096U,
-	.tx_rings = HW_ATL_B0_TX_RINGS,
-	.rx_rings = HW_ATL_B0_RX_RINGS,
-	.hw_features = NETIF_F_HW_CSUM |
-			NETIF_F_RXCSUM |
-			NETIF_F_RXHASH |
-			NETIF_F_SG |
-			NETIF_F_TSO |
-			NETIF_F_LRO,
-	.hw_priv_flags = IFF_UNICAST_FLT,
-	.link_speed_msk = (HW_ATL_B0_RATE_10G |
-			HW_ATL_B0_RATE_5G |
-			HW_ATL_B0_RATE_2G5 |
-			HW_ATL_B0_RATE_1G |
-			HW_ATL_B0_RATE_100M),
-	.flow_control = true,
-	.mtu = HW_ATL_B0_MTU_JUMBO,
-	.mac_regs_count = 88,
-	.fw_ver_expected = HW_ATL_B0_FW_VER_EXPECTED,
-};
 
 #endif /* HW_ATL_B0_INTERNAL_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c
index 3de651afa8c7..10ba035dadb1 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.c
@@ -16,111 +16,115 @@
 #include "../aq_hw_utils.h"
 
 /* global */
-void reg_glb_cpu_sem_set(struct aq_hw_s *aq_hw, u32 glb_cpu_sem, u32 semaphore)
+void hw_atl_reg_glb_cpu_sem_set(struct aq_hw_s *aq_hw, u32 glb_cpu_sem,
+				u32 semaphore)
 {
-	aq_hw_write_reg(aq_hw, glb_cpu_sem_adr(semaphore), glb_cpu_sem);
+	aq_hw_write_reg(aq_hw, HW_ATL_GLB_CPU_SEM_ADR(semaphore), glb_cpu_sem);
 }
 
-u32 reg_glb_cpu_sem_get(struct aq_hw_s *aq_hw, u32 semaphore)
+u32 hw_atl_reg_glb_cpu_sem_get(struct aq_hw_s *aq_hw, u32 semaphore)
 {
-	return aq_hw_read_reg(aq_hw, glb_cpu_sem_adr(semaphore));
+	return aq_hw_read_reg(aq_hw, HW_ATL_GLB_CPU_SEM_ADR(semaphore));
 }
 
-void glb_glb_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 glb_reg_res_dis)
+void hw_atl_glb_glb_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 glb_reg_res_dis)
 {
-	aq_hw_write_reg_bit(aq_hw, glb_reg_res_dis_adr,
-			    glb_reg_res_dis_msk,
-			    glb_reg_res_dis_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_GLB_REG_RES_DIS_ADR,
+			    HW_ATL_GLB_REG_RES_DIS_MSK,
+			    HW_ATL_GLB_REG_RES_DIS_SHIFT,
 			    glb_reg_res_dis);
 }
 
-void glb_soft_res_set(struct aq_hw_s *aq_hw, u32 soft_res)
+void hw_atl_glb_soft_res_set(struct aq_hw_s *aq_hw, u32 soft_res)
 {
-	aq_hw_write_reg_bit(aq_hw, glb_soft_res_adr, glb_soft_res_msk,
-			    glb_soft_res_shift, soft_res);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_GLB_SOFT_RES_ADR,
+			    HW_ATL_GLB_SOFT_RES_MSK,
+			    HW_ATL_GLB_SOFT_RES_SHIFT, soft_res);
 }
 
-u32 glb_soft_res_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_glb_soft_res_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg_bit(aq_hw, glb_soft_res_adr,
-				  glb_soft_res_msk,
-				  glb_soft_res_shift);
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_GLB_SOFT_RES_ADR,
+				  HW_ATL_GLB_SOFT_RES_MSK,
+				  HW_ATL_GLB_SOFT_RES_SHIFT);
 }
 
-u32 reg_rx_dma_stat_counter7get(struct aq_hw_s *aq_hw)
+u32 hw_atl_reg_rx_dma_stat_counter7get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, rx_dma_stat_counter7_adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_RX_DMA_STAT_COUNTER7_ADR);
 }
 
-u32 reg_glb_mif_id_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_reg_glb_mif_id_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, glb_mif_id_adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_GLB_MIF_ID_ADR);
 }
 
 /* stats */
-u32 rpb_rx_dma_drop_pkt_cnt_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_rpb_rx_dma_drop_pkt_cnt_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, rpb_rx_dma_drop_pkt_cnt_adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_RPB_RX_DMA_DROP_PKT_CNT_ADR);
 }
 
-u32 stats_rx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_rx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_rx_dma_good_octet_counterlsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERLSW);
 }
 
-u32 stats_rx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_rx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_rx_dma_good_pkt_counterlsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERLSW);
 }
 
-u32 stats_tx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_tx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_tx_dma_good_octet_counterlsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERLSW);
 }
 
-u32 stats_tx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_tx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_tx_dma_good_pkt_counterlsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERLSW);
 }
 
-u32 stats_rx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_rx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_rx_dma_good_octet_countermsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERMSW);
 }
 
-u32 stats_rx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_rx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_rx_dma_good_pkt_countermsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERMSW);
 }
 
-u32 stats_tx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_tx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_tx_dma_good_octet_countermsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERMSW);
 }
 
-u32 stats_tx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_stats_tx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, stats_tx_dma_good_pkt_countermsw__adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERMSW);
 }
 
 /* interrupt */
-void itr_irq_auto_masklsw_set(struct aq_hw_s *aq_hw, u32 irq_auto_masklsw)
+void hw_atl_itr_irq_auto_masklsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_auto_masklsw)
 {
-	aq_hw_write_reg(aq_hw, itr_iamrlsw_adr, irq_auto_masklsw);
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_IAMRLSW_ADR, irq_auto_masklsw);
 }
 
-void itr_irq_map_en_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_rx, u32 rx)
+void hw_atl_itr_irq_map_en_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_rx,
+				  u32 rx)
 {
 /* register address for bitfield imr_rx{r}_en */
 	static u32 itr_imr_rxren_adr[32] = {
 			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
-			0x00002108U, 0x00002108U, 0x0000210cU, 0x0000210cU,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
 			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
-			0x00002118U, 0x00002118U, 0x0000211cU, 0x0000211cU,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
 			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
-			0x00002128U, 0x00002128U, 0x0000212cU, 0x0000212cU,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
 			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
-			0x00002138U, 0x00002138U, 0x0000213cU, 0x0000213cU
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
 		};
 
 /* bitmask for bitfield imr_rx{r}_en */
@@ -149,18 +153,19 @@ void itr_irq_map_en_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_rx, u32 rx)
 			    irq_map_en_rx);
 }
 
-void itr_irq_map_en_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_tx, u32 tx)
+void hw_atl_itr_irq_map_en_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_tx,
+				  u32 tx)
 {
 /* register address for bitfield imr_tx{t}_en */
 	static u32 itr_imr_txten_adr[32] = {
 			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
-			0x00002108U, 0x00002108U, 0x0000210cU, 0x0000210cU,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
 			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
-			0x00002118U, 0x00002118U, 0x0000211cU, 0x0000211cU,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
 			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
-			0x00002128U, 0x00002128U, 0x0000212cU, 0x0000212cU,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
 			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
-			0x00002138U, 0x00002138U, 0x0000213cU, 0x0000213cU
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
 		};
 
 /* bitmask for bitfield imr_tx{t}_en */
@@ -189,30 +194,30 @@ void itr_irq_map_en_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_tx, u32 tx)
 			    irq_map_en_tx);
 }
 
-void itr_irq_map_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_rx, u32 rx)
+void hw_atl_itr_irq_map_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_rx, u32 rx)
 {
 /* register address for bitfield imr_rx{r}[4:0] */
 	static u32 itr_imr_rxr_adr[32] = {
 			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
-			0x00002108U, 0x00002108U, 0x0000210cU, 0x0000210cU,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
 			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
-			0x00002118U, 0x00002118U, 0x0000211cU, 0x0000211cU,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
 			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
-			0x00002128U, 0x00002128U, 0x0000212cU, 0x0000212cU,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
 			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
-			0x00002138U, 0x00002138U, 0x0000213cU, 0x0000213cU
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
 		};
 
 /* bitmask for bitfield imr_rx{r}[4:0] */
 	static u32 itr_imr_rxr_msk[32] = {
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU,
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU,
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU,
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU,
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU,
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU,
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU,
-			0x00001f00U, 0x0000001fU, 0x00001f00U, 0x0000001fU
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU,
+			0x00001f00U, 0x0000001FU, 0x00001F00U, 0x0000001FU
 		};
 
 /* lower bit position of bitfield imr_rx{r}[4:0] */
@@ -229,30 +234,30 @@ void itr_irq_map_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_rx, u32 rx)
 			    irq_map_rx);
 }
 
-void itr_irq_map_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_tx, u32 tx)
+void hw_atl_itr_irq_map_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_tx, u32 tx)
 {
 /* register address for bitfield imr_tx{t}[4:0] */
 	static u32 itr_imr_txt_adr[32] = {
 			0x00002100U, 0x00002100U, 0x00002104U, 0x00002104U,
-			0x00002108U, 0x00002108U, 0x0000210cU, 0x0000210cU,
+			0x00002108U, 0x00002108U, 0x0000210CU, 0x0000210CU,
 			0x00002110U, 0x00002110U, 0x00002114U, 0x00002114U,
-			0x00002118U, 0x00002118U, 0x0000211cU, 0x0000211cU,
+			0x00002118U, 0x00002118U, 0x0000211CU, 0x0000211CU,
 			0x00002120U, 0x00002120U, 0x00002124U, 0x00002124U,
-			0x00002128U, 0x00002128U, 0x0000212cU, 0x0000212cU,
+			0x00002128U, 0x00002128U, 0x0000212CU, 0x0000212CU,
 			0x00002130U, 0x00002130U, 0x00002134U, 0x00002134U,
-			0x00002138U, 0x00002138U, 0x0000213cU, 0x0000213cU
+			0x00002138U, 0x00002138U, 0x0000213CU, 0x0000213CU
 		};
 
 /* bitmask for bitfield imr_tx{t}[4:0] */
 	static u32 itr_imr_txt_msk[32] = {
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U,
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U,
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U,
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U,
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U,
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U,
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U,
-			0x1f000000U, 0x001f0000U, 0x1f000000U, 0x001f0000U
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U,
+			0x1f000000U, 0x001F0000U, 0x1F000000U, 0x001F0000U
 		};
 
 /* lower bit position of bitfield imr_tx{t}[4:0] */
@@ -269,429 +274,463 @@ void itr_irq_map_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_tx, u32 tx)
 			    irq_map_tx);
 }
 
-void itr_irq_msk_clearlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_clearlsw)
+void hw_atl_itr_irq_msk_clearlsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_msk_clearlsw)
 {
-	aq_hw_write_reg(aq_hw, itr_imcrlsw_adr, irq_msk_clearlsw);
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_IMCRLSW_ADR, irq_msk_clearlsw);
 }
 
-void itr_irq_msk_setlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_setlsw)
+void hw_atl_itr_irq_msk_setlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_setlsw)
 {
-	aq_hw_write_reg(aq_hw, itr_imsrlsw_adr, irq_msk_setlsw);
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_IMSRLSW_ADR, irq_msk_setlsw);
 }
 
-void itr_irq_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 irq_reg_res_dis)
+void hw_atl_itr_irq_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 irq_reg_res_dis)
 {
-	aq_hw_write_reg_bit(aq_hw, itr_reg_res_dsbl_adr,
-			    itr_reg_res_dsbl_msk,
-			    itr_reg_res_dsbl_shift, irq_reg_res_dis);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_ITR_REG_RES_DSBL_ADR,
+			    HW_ATL_ITR_REG_RES_DSBL_MSK,
+			    HW_ATL_ITR_REG_RES_DSBL_SHIFT, irq_reg_res_dis);
 }
 
-void itr_irq_status_clearlsw_set(struct aq_hw_s *aq_hw,
-				 u32 irq_status_clearlsw)
+void hw_atl_itr_irq_status_clearlsw_set(struct aq_hw_s *aq_hw,
+					u32 irq_status_clearlsw)
 {
-	aq_hw_write_reg(aq_hw, itr_iscrlsw_adr, irq_status_clearlsw);
+	aq_hw_write_reg(aq_hw, HW_ATL_ITR_ISCRLSW_ADR, irq_status_clearlsw);
 }
 
-u32 itr_irq_statuslsw_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_itr_irq_statuslsw_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, itr_isrlsw_adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_ITR_ISRLSW_ADR);
 }
 
-u32 itr_res_irq_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_itr_res_irq_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg_bit(aq_hw, itr_res_adr, itr_res_msk,
-				  itr_res_shift);
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_ITR_RES_ADR, HW_ATL_ITR_RES_MSK,
+				  HW_ATL_ITR_RES_SHIFT);
 }
 
-void itr_res_irq_set(struct aq_hw_s *aq_hw, u32 res_irq)
+void hw_atl_itr_res_irq_set(struct aq_hw_s *aq_hw, u32 res_irq)
 {
-	aq_hw_write_reg_bit(aq_hw, itr_res_adr, itr_res_msk,
-			    itr_res_shift, res_irq);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_ITR_RES_ADR, HW_ATL_ITR_RES_MSK,
+			    HW_ATL_ITR_RES_SHIFT, res_irq);
 }
 
 /* rdm */
-void rdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca)
+void hw_atl_rdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_dcadcpuid_adr(dca),
-			    rdm_dcadcpuid_msk,
-			    rdm_dcadcpuid_shift, cpuid);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADCPUID_ADR(dca),
+			    HW_ATL_RDM_DCADCPUID_MSK,
+			    HW_ATL_RDM_DCADCPUID_SHIFT, cpuid);
 }
 
-void rdm_rx_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_dca_en)
+void hw_atl_rdm_rx_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_dca_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_dca_en_adr, rdm_dca_en_msk,
-			    rdm_dca_en_shift, rx_dca_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCA_EN_ADR, HW_ATL_RDM_DCA_EN_MSK,
+			    HW_ATL_RDM_DCA_EN_SHIFT, rx_dca_en);
 }
 
-void rdm_rx_dca_mode_set(struct aq_hw_s *aq_hw, u32 rx_dca_mode)
+void hw_atl_rdm_rx_dca_mode_set(struct aq_hw_s *aq_hw, u32 rx_dca_mode)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_dca_mode_adr, rdm_dca_mode_msk,
-			    rdm_dca_mode_shift, rx_dca_mode);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCA_MODE_ADR,
+			    HW_ATL_RDM_DCA_MODE_MSK,
+			    HW_ATL_RDM_DCA_MODE_SHIFT, rx_dca_mode);
 }
 
-void rdm_rx_desc_data_buff_size_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_data_buff_size, u32 descriptor)
+void hw_atl_rdm_rx_desc_data_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_data_buff_size,
+					   u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_descddata_size_adr(descriptor),
-			    rdm_descddata_size_msk,
-			    rdm_descddata_size_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDDATA_SIZE_ADR(descriptor),
+			    HW_ATL_RDM_DESCDDATA_SIZE_MSK,
+			    HW_ATL_RDM_DESCDDATA_SIZE_SHIFT,
 			    rx_desc_data_buff_size);
 }
 
-void rdm_rx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_dca_en, u32 dca)
+void hw_atl_rdm_rx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_dca_en,
+				   u32 dca)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_dcaddesc_en_adr(dca),
-			    rdm_dcaddesc_en_msk,
-			    rdm_dcaddesc_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADDESC_EN_ADR(dca),
+			    HW_ATL_RDM_DCADDESC_EN_MSK,
+			    HW_ATL_RDM_DCADDESC_EN_SHIFT,
 			    rx_desc_dca_en);
 }
 
-void rdm_rx_desc_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_en, u32 descriptor)
+void hw_atl_rdm_rx_desc_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_en,
+			       u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_descden_adr(descriptor),
-			    rdm_descden_msk,
-			    rdm_descden_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDEN_ADR(descriptor),
+			    HW_ATL_RDM_DESCDEN_MSK,
+			    HW_ATL_RDM_DESCDEN_SHIFT,
 			    rx_desc_en);
 }
 
-void rdm_rx_desc_head_buff_size_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_head_buff_size, u32 descriptor)
+void hw_atl_rdm_rx_desc_head_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_buff_size,
+					   u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_descdhdr_size_adr(descriptor),
-			    rdm_descdhdr_size_msk,
-			    rdm_descdhdr_size_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDHDR_SIZE_ADR(descriptor),
+			    HW_ATL_RDM_DESCDHDR_SIZE_MSK,
+			    HW_ATL_RDM_DESCDHDR_SIZE_SHIFT,
 			    rx_desc_head_buff_size);
 }
 
-void rdm_rx_desc_head_splitting_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_head_splitting, u32 descriptor)
+void hw_atl_rdm_rx_desc_head_splitting_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_splitting,
+					   u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_descdhdr_split_adr(descriptor),
-			    rdm_descdhdr_split_msk,
-			    rdm_descdhdr_split_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDHDR_SPLIT_ADR(descriptor),
+			    HW_ATL_RDM_DESCDHDR_SPLIT_MSK,
+			    HW_ATL_RDM_DESCDHDR_SPLIT_SHIFT,
 			    rx_desc_head_splitting);
 }
 
-u32 rdm_rx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor)
+u32 hw_atl_rdm_rx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor)
 {
-	return aq_hw_read_reg_bit(aq_hw, rdm_descdhd_adr(descriptor),
-				  rdm_descdhd_msk, rdm_descdhd_shift);
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_RDM_DESCDHD_ADR(descriptor),
+				  HW_ATL_RDM_DESCDHD_MSK,
+				  HW_ATL_RDM_DESCDHD_SHIFT);
 }
 
-void rdm_rx_desc_len_set(struct aq_hw_s *aq_hw, u32 rx_desc_len, u32 descriptor)
+void hw_atl_rdm_rx_desc_len_set(struct aq_hw_s *aq_hw, u32 rx_desc_len,
+				u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_descdlen_adr(descriptor),
-			    rdm_descdlen_msk, rdm_descdlen_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDLEN_ADR(descriptor),
+			    HW_ATL_RDM_DESCDLEN_MSK, HW_ATL_RDM_DESCDLEN_SHIFT,
 			    rx_desc_len);
 }
 
-void rdm_rx_desc_res_set(struct aq_hw_s *aq_hw, u32 rx_desc_res, u32 descriptor)
+void hw_atl_rdm_rx_desc_res_set(struct aq_hw_s *aq_hw, u32 rx_desc_res,
+				u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_descdreset_adr(descriptor),
-			    rdm_descdreset_msk, rdm_descdreset_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DESCDRESET_ADR(descriptor),
+			    HW_ATL_RDM_DESCDRESET_MSK,
+			    HW_ATL_RDM_DESCDRESET_SHIFT,
 			    rx_desc_res);
 }
 
-void rdm_rx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
-				  u32 rx_desc_wr_wb_irq_en)
+void hw_atl_rdm_rx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 rx_desc_wr_wb_irq_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_int_desc_wrb_en_adr,
-			    rdm_int_desc_wrb_en_msk,
-			    rdm_int_desc_wrb_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_INT_DESC_WRB_EN_ADR,
+			    HW_ATL_RDM_INT_DESC_WRB_EN_MSK,
+			    HW_ATL_RDM_INT_DESC_WRB_EN_SHIFT,
 			    rx_desc_wr_wb_irq_en);
 }
 
-void rdm_rx_head_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_head_dca_en, u32 dca)
+void hw_atl_rdm_rx_head_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_head_dca_en,
+				   u32 dca)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_dcadhdr_en_adr(dca),
-			    rdm_dcadhdr_en_msk,
-			    rdm_dcadhdr_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADHDR_EN_ADR(dca),
+			    HW_ATL_RDM_DCADHDR_EN_MSK,
+			    HW_ATL_RDM_DCADHDR_EN_SHIFT,
 			    rx_head_dca_en);
 }
 
-void rdm_rx_pld_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_pld_dca_en, u32 dca)
+void hw_atl_rdm_rx_pld_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_pld_dca_en,
+				  u32 dca)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_dcadpay_en_adr(dca),
-			    rdm_dcadpay_en_msk, rdm_dcadpay_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_DCADPAY_EN_ADR(dca),
+			    HW_ATL_RDM_DCADPAY_EN_MSK,
+			    HW_ATL_RDM_DCADPAY_EN_SHIFT,
 			    rx_pld_dca_en);
 }
 
-void rdm_rdm_intr_moder_en_set(struct aq_hw_s *aq_hw, u32 rdm_intr_moder_en)
+void hw_atl_rdm_rdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 rdm_intr_moder_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rdm_int_rim_en_adr,
-			    rdm_int_rim_en_msk,
-			    rdm_int_rim_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RDM_INT_RIM_EN_ADR,
+			    HW_ATL_RDM_INT_RIM_EN_MSK,
+			    HW_ATL_RDM_INT_RIM_EN_SHIFT,
 			    rdm_intr_moder_en);
 }
 
 /* reg */
-void reg_gen_irq_map_set(struct aq_hw_s *aq_hw, u32 gen_intr_map, u32 regidx)
+void hw_atl_reg_gen_irq_map_set(struct aq_hw_s *aq_hw, u32 gen_intr_map,
+				u32 regidx)
 {
-	aq_hw_write_reg(aq_hw, gen_intr_map_adr(regidx), gen_intr_map);
+	aq_hw_write_reg(aq_hw, HW_ATL_GEN_INTR_MAP_ADR(regidx), gen_intr_map);
 }
 
-u32 reg_gen_irq_status_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_reg_gen_irq_status_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, gen_intr_stat_adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_GEN_INTR_STAT_ADR);
 }
 
-void reg_irq_glb_ctl_set(struct aq_hw_s *aq_hw, u32 intr_glb_ctl)
+void hw_atl_reg_irq_glb_ctl_set(struct aq_hw_s *aq_hw, u32 intr_glb_ctl)
 {
-	aq_hw_write_reg(aq_hw, intr_glb_ctl_adr, intr_glb_ctl);
+	aq_hw_write_reg(aq_hw, HW_ATL_INTR_GLB_CTL_ADR, intr_glb_ctl);
 }
 
-void reg_irq_thr_set(struct aq_hw_s *aq_hw, u32 intr_thr, u32 throttle)
+void hw_atl_reg_irq_thr_set(struct aq_hw_s *aq_hw, u32 intr_thr, u32 throttle)
 {
-	aq_hw_write_reg(aq_hw, intr_thr_adr(throttle), intr_thr);
+	aq_hw_write_reg(aq_hw, HW_ATL_INTR_THR_ADR(throttle), intr_thr);
 }
 
-void reg_rx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
-					u32 rx_dma_desc_base_addrlsw,
-					u32 descriptor)
+void hw_atl_reg_rx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrlsw,
+					       u32 descriptor)
 {
-	aq_hw_write_reg(aq_hw, rx_dma_desc_base_addrlsw_adr(descriptor),
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor),
 			rx_dma_desc_base_addrlsw);
 }
 
-void reg_rx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
-					u32 rx_dma_desc_base_addrmsw,
-					u32 descriptor)
+void hw_atl_reg_rx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrmsw,
+					       u32 descriptor)
 {
-	aq_hw_write_reg(aq_hw, rx_dma_desc_base_addrmsw_adr(descriptor),
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor),
 			rx_dma_desc_base_addrmsw);
 }
 
-u32 reg_rx_dma_desc_status_get(struct aq_hw_s *aq_hw, u32 descriptor)
+u32 hw_atl_reg_rx_dma_desc_status_get(struct aq_hw_s *aq_hw, u32 descriptor)
 {
-	return aq_hw_read_reg(aq_hw, rx_dma_desc_stat_adr(descriptor));
+	return aq_hw_read_reg(aq_hw, HW_ATL_RX_DMA_DESC_STAT_ADR(descriptor));
 }
 
-void reg_rx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
-				  u32 rx_dma_desc_tail_ptr, u32 descriptor)
+void hw_atl_reg_rx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 rx_dma_desc_tail_ptr,
+					 u32 descriptor)
 {
-	aq_hw_write_reg(aq_hw, rx_dma_desc_tail_ptr_adr(descriptor),
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_DMA_DESC_TAIL_PTR_ADR(descriptor),
 			rx_dma_desc_tail_ptr);
 }
 
-void reg_rx_flr_mcst_flr_msk_set(struct aq_hw_s *aq_hw, u32 rx_flr_mcst_flr_msk)
+void hw_atl_reg_rx_flr_mcst_flr_msk_set(struct aq_hw_s *aq_hw,
+					u32 rx_flr_mcst_flr_msk)
 {
-	aq_hw_write_reg(aq_hw, rx_flr_mcst_flr_msk_adr, rx_flr_mcst_flr_msk);
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_MCST_FLR_MSK_ADR,
+			rx_flr_mcst_flr_msk);
 }
 
-void reg_rx_flr_mcst_flr_set(struct aq_hw_s *aq_hw, u32 rx_flr_mcst_flr,
-			     u32 filter)
+void hw_atl_reg_rx_flr_mcst_flr_set(struct aq_hw_s *aq_hw, u32 rx_flr_mcst_flr,
+				    u32 filter)
 {
-	aq_hw_write_reg(aq_hw, rx_flr_mcst_flr_adr(filter), rx_flr_mcst_flr);
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_MCST_FLR_ADR(filter),
+			rx_flr_mcst_flr);
 }
 
-void reg_rx_flr_rss_control1set(struct aq_hw_s *aq_hw, u32 rx_flr_rss_control1)
+void hw_atl_reg_rx_flr_rss_control1set(struct aq_hw_s *aq_hw,
+				       u32 rx_flr_rss_control1)
 {
-	aq_hw_write_reg(aq_hw, rx_flr_rss_control1_adr, rx_flr_rss_control1);
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_RSS_CONTROL1_ADR,
+			rx_flr_rss_control1);
 }
 
-void reg_rx_flr_control2_set(struct aq_hw_s *aq_hw, u32 rx_filter_control2)
+void hw_atl_reg_rx_flr_control2_set(struct aq_hw_s *aq_hw,
+				    u32 rx_filter_control2)
 {
-	aq_hw_write_reg(aq_hw, rx_flr_control2_adr, rx_filter_control2);
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_FLR_CONTROL2_ADR, rx_filter_control2);
 }
 
-void reg_rx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
-				u32 rx_intr_moderation_ctl,
-				u32 queue)
+void hw_atl_reg_rx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 rx_intr_moderation_ctl,
+				       u32 queue)
 {
-	aq_hw_write_reg(aq_hw, rx_intr_moderation_ctl_adr(queue),
+	aq_hw_write_reg(aq_hw, HW_ATL_RX_INTR_MODERATION_CTL_ADR(queue),
 			rx_intr_moderation_ctl);
 }
 
-void reg_tx_dma_debug_ctl_set(struct aq_hw_s *aq_hw, u32 tx_dma_debug_ctl)
+void hw_atl_reg_tx_dma_debug_ctl_set(struct aq_hw_s *aq_hw,
+				     u32 tx_dma_debug_ctl)
 {
-	aq_hw_write_reg(aq_hw, tx_dma_debug_ctl_adr, tx_dma_debug_ctl);
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DEBUG_CTL_ADR, tx_dma_debug_ctl);
 }
 
-void reg_tx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
-					u32 tx_dma_desc_base_addrlsw,
-					u32 descriptor)
+void hw_atl_reg_tx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrlsw,
+					       u32 descriptor)
 {
-	aq_hw_write_reg(aq_hw, tx_dma_desc_base_addrlsw_adr(descriptor),
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor),
 			tx_dma_desc_base_addrlsw);
 }
 
-void reg_tx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
-					u32 tx_dma_desc_base_addrmsw,
-					u32 descriptor)
+void hw_atl_reg_tx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrmsw,
+					       u32 descriptor)
 {
-	aq_hw_write_reg(aq_hw, tx_dma_desc_base_addrmsw_adr(descriptor),
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor),
 			tx_dma_desc_base_addrmsw);
 }
 
-void reg_tx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
-				  u32 tx_dma_desc_tail_ptr, u32 descriptor)
+void hw_atl_reg_tx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 tx_dma_desc_tail_ptr,
+					 u32 descriptor)
 {
-	aq_hw_write_reg(aq_hw, tx_dma_desc_tail_ptr_adr(descriptor),
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_DMA_DESC_TAIL_PTR_ADR(descriptor),
 			tx_dma_desc_tail_ptr);
 }
 
-void reg_tx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
-				u32 tx_intr_moderation_ctl,
-				u32 queue)
+void hw_atl_reg_tx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 tx_intr_moderation_ctl,
+				       u32 queue)
 {
-	aq_hw_write_reg(aq_hw, tx_intr_moderation_ctl_adr(queue),
+	aq_hw_write_reg(aq_hw, HW_ATL_TX_INTR_MODERATION_CTL_ADR(queue),
 			tx_intr_moderation_ctl);
 }
 
 /* RPB: rx packet buffer */
-void rpb_dma_sys_lbk_set(struct aq_hw_s *aq_hw, u32 dma_sys_lbk)
+void hw_atl_rpb_dma_sys_lbk_set(struct aq_hw_s *aq_hw, u32 dma_sys_lbk)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_dma_sys_lbk_adr,
-			    rpb_dma_sys_lbk_msk,
-			    rpb_dma_sys_lbk_shift, dma_sys_lbk);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_DMA_SYS_LBK_ADR,
+			    HW_ATL_RPB_DMA_SYS_LBK_MSK,
+			    HW_ATL_RPB_DMA_SYS_LBK_SHIFT, dma_sys_lbk);
 }
 
-void rpb_rpf_rx_traf_class_mode_set(struct aq_hw_s *aq_hw,
-				    u32 rx_traf_class_mode)
+void hw_atl_rpb_rpf_rx_traf_class_mode_set(struct aq_hw_s *aq_hw,
+					   u32 rx_traf_class_mode)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_rpf_rx_tc_mode_adr,
-			    rpb_rpf_rx_tc_mode_msk,
-			    rpb_rpf_rx_tc_mode_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RPF_RX_TC_MODE_ADR,
+			    HW_ATL_RPB_RPF_RX_TC_MODE_MSK,
+			    HW_ATL_RPB_RPF_RX_TC_MODE_SHIFT,
 			    rx_traf_class_mode);
 }
 
-void rpb_rx_buff_en_set(struct aq_hw_s *aq_hw, u32 rx_buff_en)
+void hw_atl_rpb_rx_buff_en_set(struct aq_hw_s *aq_hw, u32 rx_buff_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_rx_buf_en_adr, rpb_rx_buf_en_msk,
-			    rpb_rx_buf_en_shift, rx_buff_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RX_BUF_EN_ADR,
+			    HW_ATL_RPB_RX_BUF_EN_MSK,
+			    HW_ATL_RPB_RX_BUF_EN_SHIFT, rx_buff_en);
 }
 
-void rpb_rx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 rx_buff_hi_threshold_per_tc,
-					 u32 buffer)
+void hw_atl_rpb_rx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_hi_threshold_per_tc,
+						u32 buffer)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_rxbhi_thresh_adr(buffer),
-			    rpb_rxbhi_thresh_msk, rpb_rxbhi_thresh_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBHI_THRESH_ADR(buffer),
+			    HW_ATL_RPB_RXBHI_THRESH_MSK,
+			    HW_ATL_RPB_RXBHI_THRESH_SHIFT,
 			    rx_buff_hi_threshold_per_tc);
 }
 
-void rpb_rx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 rx_buff_lo_threshold_per_tc,
-					 u32 buffer)
+void hw_atl_rpb_rx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_lo_threshold_per_tc,
+						u32 buffer)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_rxblo_thresh_adr(buffer),
-			    rpb_rxblo_thresh_msk,
-			    rpb_rxblo_thresh_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBLO_THRESH_ADR(buffer),
+			    HW_ATL_RPB_RXBLO_THRESH_MSK,
+			    HW_ATL_RPB_RXBLO_THRESH_SHIFT,
 			    rx_buff_lo_threshold_per_tc);
 }
 
-void rpb_rx_flow_ctl_mode_set(struct aq_hw_s *aq_hw, u32 rx_flow_ctl_mode)
+void hw_atl_rpb_rx_flow_ctl_mode_set(struct aq_hw_s *aq_hw, u32 rx_flow_ctl_mode)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_rx_fc_mode_adr,
-			    rpb_rx_fc_mode_msk,
-			    rpb_rx_fc_mode_shift, rx_flow_ctl_mode);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RX_FC_MODE_ADR,
+			    HW_ATL_RPB_RX_FC_MODE_MSK,
+			    HW_ATL_RPB_RX_FC_MODE_SHIFT, rx_flow_ctl_mode);
 }
 
-void rpb_rx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
-				     u32 rx_pkt_buff_size_per_tc, u32 buffer)
+void hw_atl_rpb_rx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 rx_pkt_buff_size_per_tc, u32 buffer)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_rxbbuf_size_adr(buffer),
-			    rpb_rxbbuf_size_msk, rpb_rxbbuf_size_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBBUF_SIZE_ADR(buffer),
+			    HW_ATL_RPB_RXBBUF_SIZE_MSK,
+			    HW_ATL_RPB_RXBBUF_SIZE_SHIFT,
 			    rx_pkt_buff_size_per_tc);
 }
 
-void rpb_rx_xoff_en_per_tc_set(struct aq_hw_s *aq_hw, u32 rx_xoff_en_per_tc,
-			       u32 buffer)
+void hw_atl_rpb_rx_xoff_en_per_tc_set(struct aq_hw_s *aq_hw, u32 rx_xoff_en_per_tc,
+				      u32 buffer)
 {
-	aq_hw_write_reg_bit(aq_hw, rpb_rxbxoff_en_adr(buffer),
-			    rpb_rxbxoff_en_msk, rpb_rxbxoff_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPB_RXBXOFF_EN_ADR(buffer),
+			    HW_ATL_RPB_RXBXOFF_EN_MSK,
+			    HW_ATL_RPB_RXBXOFF_EN_SHIFT,
 			    rx_xoff_en_per_tc);
 }
 
 /* rpf */
 
-void rpfl2broadcast_count_threshold_set(struct aq_hw_s *aq_hw,
-					u32 l2broadcast_count_threshold)
+void hw_atl_rpfl2broadcast_count_threshold_set(struct aq_hw_s *aq_hw,
+					       u32 l2broadcast_count_threshold)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2bc_thresh_adr,
-			    rpfl2bc_thresh_msk,
-			    rpfl2bc_thresh_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2BC_THRESH_ADR,
+			    HW_ATL_RPFL2BC_THRESH_MSK,
+			    HW_ATL_RPFL2BC_THRESH_SHIFT,
 			    l2broadcast_count_threshold);
 }
 
-void rpfl2broadcast_en_set(struct aq_hw_s *aq_hw, u32 l2broadcast_en)
+void hw_atl_rpfl2broadcast_en_set(struct aq_hw_s *aq_hw, u32 l2broadcast_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2bc_en_adr, rpfl2bc_en_msk,
-			    rpfl2bc_en_shift, l2broadcast_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2BC_EN_ADR, HW_ATL_RPFL2BC_EN_MSK,
+			    HW_ATL_RPFL2BC_EN_SHIFT, l2broadcast_en);
 }
 
-void rpfl2broadcast_flr_act_set(struct aq_hw_s *aq_hw, u32 l2broadcast_flr_act)
+void hw_atl_rpfl2broadcast_flr_act_set(struct aq_hw_s *aq_hw,
+				       u32 l2broadcast_flr_act)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2bc_act_adr, rpfl2bc_act_msk,
-			    rpfl2bc_act_shift, l2broadcast_flr_act);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2BC_ACT_ADR,
+			    HW_ATL_RPFL2BC_ACT_MSK,
+			    HW_ATL_RPFL2BC_ACT_SHIFT, l2broadcast_flr_act);
 }
 
-void rpfl2multicast_flr_en_set(struct aq_hw_s *aq_hw, u32 l2multicast_flr_en,
-			       u32 filter)
+void hw_atl_rpfl2multicast_flr_en_set(struct aq_hw_s *aq_hw,
+				      u32 l2multicast_flr_en,
+				      u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2mc_enf_adr(filter),
-			    rpfl2mc_enf_msk,
-			    rpfl2mc_enf_shift, l2multicast_flr_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2MC_ENF_ADR(filter),
+			    HW_ATL_RPFL2MC_ENF_MSK,
+			    HW_ATL_RPFL2MC_ENF_SHIFT, l2multicast_flr_en);
 }
 
-void rpfl2promiscuous_mode_en_set(struct aq_hw_s *aq_hw,
-				  u32 l2promiscuous_mode_en)
+void hw_atl_rpfl2promiscuous_mode_en_set(struct aq_hw_s *aq_hw,
+					 u32 l2promiscuous_mode_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2promis_mode_adr,
-			    rpfl2promis_mode_msk,
-			    rpfl2promis_mode_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2PROMIS_MODE_ADR,
+			    HW_ATL_RPFL2PROMIS_MODE_MSK,
+			    HW_ATL_RPFL2PROMIS_MODE_SHIFT,
 			    l2promiscuous_mode_en);
 }
 
-void rpfl2unicast_flr_act_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_act,
-			      u32 filter)
+void hw_atl_rpfl2unicast_flr_act_set(struct aq_hw_s *aq_hw,
+				     u32 l2unicast_flr_act,
+				     u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2uc_actf_adr(filter),
-			    rpfl2uc_actf_msk, rpfl2uc_actf_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2UC_ACTF_ADR(filter),
+			    HW_ATL_RPFL2UC_ACTF_MSK, HW_ATL_RPFL2UC_ACTF_SHIFT,
 			    l2unicast_flr_act);
 }
 
-void rpfl2_uc_flr_en_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_en,
-			 u32 filter)
+void hw_atl_rpfl2_uc_flr_en_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_en,
+				u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2uc_enf_adr(filter),
-			    rpfl2uc_enf_msk,
-			    rpfl2uc_enf_shift, l2unicast_flr_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2UC_ENF_ADR(filter),
+			    HW_ATL_RPFL2UC_ENF_MSK,
+			    HW_ATL_RPFL2UC_ENF_SHIFT, l2unicast_flr_en);
 }
 
-void rpfl2unicast_dest_addresslsw_set(struct aq_hw_s *aq_hw,
-				      u32 l2unicast_dest_addresslsw,
-				      u32 filter)
+void hw_atl_rpfl2unicast_dest_addresslsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addresslsw,
+					     u32 filter)
 {
-	aq_hw_write_reg(aq_hw, rpfl2uc_daflsw_adr(filter),
+	aq_hw_write_reg(aq_hw, HW_ATL_RPFL2UC_DAFLSW_ADR(filter),
 			l2unicast_dest_addresslsw);
 }
 
-void rpfl2unicast_dest_addressmsw_set(struct aq_hw_s *aq_hw,
-				      u32 l2unicast_dest_addressmsw,
-				      u32 filter)
+void hw_atl_rpfl2unicast_dest_addressmsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addressmsw,
+					     u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2uc_dafmsw_adr(filter),
-			    rpfl2uc_dafmsw_msk, rpfl2uc_dafmsw_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2UC_DAFMSW_ADR(filter),
+			    HW_ATL_RPFL2UC_DAFMSW_MSK,
+			    HW_ATL_RPFL2UC_DAFMSW_SHIFT,
 			    l2unicast_dest_addressmsw);
 }
 
-void rpfl2_accept_all_mc_packets_set(struct aq_hw_s *aq_hw,
-				     u32 l2_accept_all_mc_packets)
+void hw_atl_rpfl2_accept_all_mc_packets_set(struct aq_hw_s *aq_hw,
+					    u32 l2_accept_all_mc_packets)
 {
-	aq_hw_write_reg_bit(aq_hw, rpfl2mc_accept_all_adr,
-			    rpfl2mc_accept_all_msk,
-			    rpfl2mc_accept_all_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPFL2MC_ACCEPT_ALL_ADR,
+			    HW_ATL_RPFL2MC_ACCEPT_ALL_MSK,
+			    HW_ATL_RPFL2MC_ACCEPT_ALL_SHIFT,
 			    l2_accept_all_mc_packets);
 }
 
-void rpf_rpb_user_priority_tc_map_set(struct aq_hw_s *aq_hw,
-				      u32 user_priority_tc_map, u32 tc)
+void hw_atl_rpf_rpb_user_priority_tc_map_set(struct aq_hw_s *aq_hw,
+					     u32 user_priority_tc_map, u32 tc)
 {
 /* register address for bitfield rx_tc_up{t}[2:0] */
 	static u32 rpf_rpb_rx_tc_upt_adr[8] = {
-			0x000054c4U, 0x000054c4U, 0x000054c4U, 0x000054c4U,
-			0x000054c4U, 0x000054c4U, 0x000054c4U, 0x000054c4U
+			0x000054c4U, 0x000054C4U, 0x000054C4U, 0x000054C4U,
+			0x000054c4U, 0x000054C4U, 0x000054C4U, 0x000054C4U
 		};
 
 /* bitmask for bitfield rx_tc_up{t}[2:0] */
@@ -711,273 +750,290 @@ void rpf_rpb_user_priority_tc_map_set(struct aq_hw_s *aq_hw,
 			    user_priority_tc_map);
 }
 
-void rpf_rss_key_addr_set(struct aq_hw_s *aq_hw, u32 rss_key_addr)
+void hw_atl_rpf_rss_key_addr_set(struct aq_hw_s *aq_hw, u32 rss_key_addr)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_rss_key_addr_adr,
-			    rpf_rss_key_addr_msk,
-			    rpf_rss_key_addr_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_KEY_ADDR_ADR,
+			    HW_ATL_RPF_RSS_KEY_ADDR_MSK,
+			    HW_ATL_RPF_RSS_KEY_ADDR_SHIFT,
 			    rss_key_addr);
 }
 
-void rpf_rss_key_wr_data_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_data)
+void hw_atl_rpf_rss_key_wr_data_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_data)
 {
-	aq_hw_write_reg(aq_hw, rpf_rss_key_wr_data_adr,
+	aq_hw_write_reg(aq_hw, HW_ATL_RPF_RSS_KEY_WR_DATA_ADR,
 			rss_key_wr_data);
 }
 
-u32 rpf_rss_key_wr_en_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_rpf_rss_key_wr_en_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg_bit(aq_hw, rpf_rss_key_wr_eni_adr,
-				  rpf_rss_key_wr_eni_msk,
-				  rpf_rss_key_wr_eni_shift);
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_RPF_RSS_KEY_WR_ENI_ADR,
+				  HW_ATL_RPF_RSS_KEY_WR_ENI_MSK,
+				  HW_ATL_RPF_RSS_KEY_WR_ENI_SHIFT);
 }
 
-void rpf_rss_key_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_en)
+void hw_atl_rpf_rss_key_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_rss_key_wr_eni_adr,
-			    rpf_rss_key_wr_eni_msk,
-			    rpf_rss_key_wr_eni_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_KEY_WR_ENI_ADR,
+			    HW_ATL_RPF_RSS_KEY_WR_ENI_MSK,
+			    HW_ATL_RPF_RSS_KEY_WR_ENI_SHIFT,
 			    rss_key_wr_en);
 }
 
-void rpf_rss_redir_tbl_addr_set(struct aq_hw_s *aq_hw, u32 rss_redir_tbl_addr)
+void hw_atl_rpf_rss_redir_tbl_addr_set(struct aq_hw_s *aq_hw,
+				       u32 rss_redir_tbl_addr)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_rss_redir_addr_adr,
-			    rpf_rss_redir_addr_msk,
-			    rpf_rss_redir_addr_shift, rss_redir_tbl_addr);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_ADDR_ADR,
+			    HW_ATL_RPF_RSS_REDIR_ADDR_MSK,
+			    HW_ATL_RPF_RSS_REDIR_ADDR_SHIFT,
+			    rss_redir_tbl_addr);
 }
 
-void rpf_rss_redir_tbl_wr_data_set(struct aq_hw_s *aq_hw,
-				   u32 rss_redir_tbl_wr_data)
+void hw_atl_rpf_rss_redir_tbl_wr_data_set(struct aq_hw_s *aq_hw,
+					  u32 rss_redir_tbl_wr_data)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_rss_redir_wr_data_adr,
-			    rpf_rss_redir_wr_data_msk,
-			    rpf_rss_redir_wr_data_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_WR_DATA_ADR,
+			    HW_ATL_RPF_RSS_REDIR_WR_DATA_MSK,
+			    HW_ATL_RPF_RSS_REDIR_WR_DATA_SHIFT,
 			    rss_redir_tbl_wr_data);
 }
 
-u32 rpf_rss_redir_wr_en_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_rpf_rss_redir_wr_en_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg_bit(aq_hw, rpf_rss_redir_wr_eni_adr,
-				  rpf_rss_redir_wr_eni_msk,
-				  rpf_rss_redir_wr_eni_shift);
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_WR_ENI_ADR,
+				  HW_ATL_RPF_RSS_REDIR_WR_ENI_MSK,
+				  HW_ATL_RPF_RSS_REDIR_WR_ENI_SHIFT);
 }
 
-void rpf_rss_redir_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_redir_wr_en)
+void hw_atl_rpf_rss_redir_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_redir_wr_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_rss_redir_wr_eni_adr,
-			    rpf_rss_redir_wr_eni_msk,
-			    rpf_rss_redir_wr_eni_shift, rss_redir_wr_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_RSS_REDIR_WR_ENI_ADR,
+			    HW_ATL_RPF_RSS_REDIR_WR_ENI_MSK,
+			    HW_ATL_RPF_RSS_REDIR_WR_ENI_SHIFT, rss_redir_wr_en);
 }
 
-void rpf_tpo_to_rpf_sys_lbk_set(struct aq_hw_s *aq_hw, u32 tpo_to_rpf_sys_lbk)
+void hw_atl_rpf_tpo_to_rpf_sys_lbk_set(struct aq_hw_s *aq_hw,
+				       u32 tpo_to_rpf_sys_lbk)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_tpo_rpf_sys_lbk_adr,
-			    rpf_tpo_rpf_sys_lbk_msk,
-			    rpf_tpo_rpf_sys_lbk_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_TPO_RPF_SYS_LBK_ADR,
+			    HW_ATL_RPF_TPO_RPF_SYS_LBK_MSK,
+			    HW_ATL_RPF_TPO_RPF_SYS_LBK_SHIFT,
 			    tpo_to_rpf_sys_lbk);
 }
 
-void rpf_vlan_inner_etht_set(struct aq_hw_s *aq_hw, u32 vlan_inner_etht)
+void hw_atl_rpf_vlan_inner_etht_set(struct aq_hw_s *aq_hw, u32 vlan_inner_etht)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_inner_tpid_adr,
-			    rpf_vl_inner_tpid_msk,
-			    rpf_vl_inner_tpid_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_INNER_TPID_ADR,
+			    HW_ATL_RPF_VL_INNER_TPID_MSK,
+			    HW_ATL_RPF_VL_INNER_TPID_SHIFT,
 			    vlan_inner_etht);
 }
 
-void rpf_vlan_outer_etht_set(struct aq_hw_s *aq_hw, u32 vlan_outer_etht)
+void hw_atl_rpf_vlan_outer_etht_set(struct aq_hw_s *aq_hw, u32 vlan_outer_etht)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_outer_tpid_adr,
-			    rpf_vl_outer_tpid_msk,
-			    rpf_vl_outer_tpid_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_OUTER_TPID_ADR,
+			    HW_ATL_RPF_VL_OUTER_TPID_MSK,
+			    HW_ATL_RPF_VL_OUTER_TPID_SHIFT,
 			    vlan_outer_etht);
 }
 
-void rpf_vlan_prom_mode_en_set(struct aq_hw_s *aq_hw, u32 vlan_prom_mode_en)
+void hw_atl_rpf_vlan_prom_mode_en_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_prom_mode_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_promis_mode_adr,
-			    rpf_vl_promis_mode_msk,
-			    rpf_vl_promis_mode_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_PROMIS_MODE_ADR,
+			    HW_ATL_RPF_VL_PROMIS_MODE_MSK,
+			    HW_ATL_RPF_VL_PROMIS_MODE_SHIFT,
 			    vlan_prom_mode_en);
 }
 
-void rpf_vlan_accept_untagged_packets_set(struct aq_hw_s *aq_hw,
-					  u32 vlan_accept_untagged_packets)
+void hw_atl_rpf_vlan_accept_untagged_packets_set(struct aq_hw_s *aq_hw,
+						 u32 vlan_acc_untagged_packets)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_accept_untagged_mode_adr,
-			    rpf_vl_accept_untagged_mode_msk,
-			    rpf_vl_accept_untagged_mode_shift,
-			    vlan_accept_untagged_packets);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_ADR,
+			    HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_MSK,
+			    HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_SHIFT,
+			    vlan_acc_untagged_packets);
 }
 
-void rpf_vlan_untagged_act_set(struct aq_hw_s *aq_hw, u32 vlan_untagged_act)
+void hw_atl_rpf_vlan_untagged_act_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_untagged_act)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_untagged_act_adr,
-			    rpf_vl_untagged_act_msk,
-			    rpf_vl_untagged_act_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_UNTAGGED_ACT_ADR,
+			    HW_ATL_RPF_VL_UNTAGGED_ACT_MSK,
+			    HW_ATL_RPF_VL_UNTAGGED_ACT_SHIFT,
 			    vlan_untagged_act);
 }
 
-void rpf_vlan_flr_en_set(struct aq_hw_s *aq_hw, u32 vlan_flr_en, u32 filter)
+void hw_atl_rpf_vlan_flr_en_set(struct aq_hw_s *aq_hw, u32 vlan_flr_en,
+				u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_en_f_adr(filter),
-			    rpf_vl_en_f_msk,
-			    rpf_vl_en_f_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_EN_F_ADR(filter),
+			    HW_ATL_RPF_VL_EN_F_MSK,
+			    HW_ATL_RPF_VL_EN_F_SHIFT,
 			    vlan_flr_en);
 }
 
-void rpf_vlan_flr_act_set(struct aq_hw_s *aq_hw, u32 vlan_flr_act, u32 filter)
+void hw_atl_rpf_vlan_flr_act_set(struct aq_hw_s *aq_hw, u32 vlan_flr_act,
+				 u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_act_f_adr(filter),
-			    rpf_vl_act_f_msk,
-			    rpf_vl_act_f_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_ACT_F_ADR(filter),
+			    HW_ATL_RPF_VL_ACT_F_MSK,
+			    HW_ATL_RPF_VL_ACT_F_SHIFT,
 			    vlan_flr_act);
 }
 
-void rpf_vlan_id_flr_set(struct aq_hw_s *aq_hw, u32 vlan_id_flr, u32 filter)
+void hw_atl_rpf_vlan_id_flr_set(struct aq_hw_s *aq_hw, u32 vlan_id_flr,
+				u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_vl_id_f_adr(filter),
-			    rpf_vl_id_f_msk,
-			    rpf_vl_id_f_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_VL_ID_F_ADR(filter),
+			    HW_ATL_RPF_VL_ID_F_MSK,
+			    HW_ATL_RPF_VL_ID_F_SHIFT,
 			    vlan_id_flr);
 }
 
-void rpf_etht_flr_en_set(struct aq_hw_s *aq_hw, u32 etht_flr_en, u32 filter)
+void hw_atl_rpf_etht_flr_en_set(struct aq_hw_s *aq_hw, u32 etht_flr_en,
+				u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_enf_adr(filter),
-			    rpf_et_enf_msk,
-			    rpf_et_enf_shift, etht_flr_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_ENF_ADR(filter),
+			    HW_ATL_RPF_ET_ENF_MSK,
+			    HW_ATL_RPF_ET_ENF_SHIFT, etht_flr_en);
 }
 
-void rpf_etht_user_priority_en_set(struct aq_hw_s *aq_hw,
-				   u32 etht_user_priority_en, u32 filter)
+void hw_atl_rpf_etht_user_priority_en_set(struct aq_hw_s *aq_hw,
+					  u32 etht_user_priority_en, u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_upfen_adr(filter),
-			    rpf_et_upfen_msk, rpf_et_upfen_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_UPFEN_ADR(filter),
+			    HW_ATL_RPF_ET_UPFEN_MSK, HW_ATL_RPF_ET_UPFEN_SHIFT,
 			    etht_user_priority_en);
 }
 
-void rpf_etht_rx_queue_en_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue_en,
-			      u32 filter)
+void hw_atl_rpf_etht_rx_queue_en_set(struct aq_hw_s *aq_hw,
+				     u32 etht_rx_queue_en,
+				     u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_rxqfen_adr(filter),
-			    rpf_et_rxqfen_msk, rpf_et_rxqfen_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_RXQFEN_ADR(filter),
+			    HW_ATL_RPF_ET_RXQFEN_MSK,
+			    HW_ATL_RPF_ET_RXQFEN_SHIFT,
 			    etht_rx_queue_en);
 }
 
-void rpf_etht_user_priority_set(struct aq_hw_s *aq_hw, u32 etht_user_priority,
-				u32 filter)
+void hw_atl_rpf_etht_user_priority_set(struct aq_hw_s *aq_hw,
+				       u32 etht_user_priority,
+				       u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_upf_adr(filter),
-			    rpf_et_upf_msk,
-			    rpf_et_upf_shift, etht_user_priority);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_UPF_ADR(filter),
+			    HW_ATL_RPF_ET_UPF_MSK,
+			    HW_ATL_RPF_ET_UPF_SHIFT, etht_user_priority);
 }
 
-void rpf_etht_rx_queue_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue,
-			   u32 filter)
+void hw_atl_rpf_etht_rx_queue_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue,
+				  u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_rxqf_adr(filter),
-			    rpf_et_rxqf_msk,
-			    rpf_et_rxqf_shift, etht_rx_queue);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_RXQF_ADR(filter),
+			    HW_ATL_RPF_ET_RXQF_MSK,
+			    HW_ATL_RPF_ET_RXQF_SHIFT, etht_rx_queue);
 }
 
-void rpf_etht_mgt_queue_set(struct aq_hw_s *aq_hw, u32 etht_mgt_queue,
-			    u32 filter)
+void hw_atl_rpf_etht_mgt_queue_set(struct aq_hw_s *aq_hw, u32 etht_mgt_queue,
+				   u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_mng_rxqf_adr(filter),
-			    rpf_et_mng_rxqf_msk, rpf_et_mng_rxqf_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_MNG_RXQF_ADR(filter),
+			    HW_ATL_RPF_ET_MNG_RXQF_MSK,
+			    HW_ATL_RPF_ET_MNG_RXQF_SHIFT,
 			    etht_mgt_queue);
 }
 
-void rpf_etht_flr_act_set(struct aq_hw_s *aq_hw, u32 etht_flr_act, u32 filter)
+void hw_atl_rpf_etht_flr_act_set(struct aq_hw_s *aq_hw, u32 etht_flr_act,
+				 u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_actf_adr(filter),
-			    rpf_et_actf_msk,
-			    rpf_et_actf_shift, etht_flr_act);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_ACTF_ADR(filter),
+			    HW_ATL_RPF_ET_ACTF_MSK,
+			    HW_ATL_RPF_ET_ACTF_SHIFT, etht_flr_act);
 }
 
-void rpf_etht_flr_set(struct aq_hw_s *aq_hw, u32 etht_flr, u32 filter)
+void hw_atl_rpf_etht_flr_set(struct aq_hw_s *aq_hw, u32 etht_flr, u32 filter)
 {
-	aq_hw_write_reg_bit(aq_hw, rpf_et_valf_adr(filter),
-			    rpf_et_valf_msk,
-			    rpf_et_valf_shift, etht_flr);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPF_ET_VALF_ADR(filter),
+			    HW_ATL_RPF_ET_VALF_MSK,
+			    HW_ATL_RPF_ET_VALF_SHIFT, etht_flr);
 }
 
 /* RPO: rx packet offload */
-void rpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				       u32 ipv4header_crc_offload_en)
+void hw_atl_rpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_ipv4chk_en_adr,
-			    rpo_ipv4chk_en_msk,
-			    rpo_ipv4chk_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_IPV4CHK_EN_ADR,
+			    HW_ATL_RPO_IPV4CHK_EN_MSK,
+			    HW_ATL_RPO_IPV4CHK_EN_SHIFT,
 			    ipv4header_crc_offload_en);
 }
 
-void rpo_rx_desc_vlan_stripping_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_vlan_stripping, u32 descriptor)
+void hw_atl_rpo_rx_desc_vlan_stripping_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_vlan_stripping,
+					   u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_descdvl_strip_adr(descriptor),
-			    rpo_descdvl_strip_msk,
-			    rpo_descdvl_strip_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_DESCDVL_STRIP_ADR(descriptor),
+			    HW_ATL_RPO_DESCDVL_STRIP_MSK,
+			    HW_ATL_RPO_DESCDVL_STRIP_SHIFT,
 			    rx_desc_vlan_stripping);
 }
 
-void rpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				    u32 tcp_udp_crc_offload_en)
+void hw_atl_rpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpol4chk_en_adr, rpol4chk_en_msk,
-			    rpol4chk_en_shift, tcp_udp_crc_offload_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPOL4CHK_EN_ADR,
+			    HW_ATL_RPOL4CHK_EN_MSK,
+			    HW_ATL_RPOL4CHK_EN_SHIFT, tcp_udp_crc_offload_en);
 }
 
-void rpo_lro_en_set(struct aq_hw_s *aq_hw, u32 lro_en)
+void hw_atl_rpo_lro_en_set(struct aq_hw_s *aq_hw, u32 lro_en)
 {
-	aq_hw_write_reg(aq_hw, rpo_lro_en_adr, lro_en);
+	aq_hw_write_reg(aq_hw, HW_ATL_RPO_LRO_EN_ADR, lro_en);
 }
 
-void rpo_lro_patch_optimization_en_set(struct aq_hw_s *aq_hw,
-				       u32 lro_patch_optimization_en)
+void hw_atl_rpo_lro_patch_optimization_en_set(struct aq_hw_s *aq_hw,
+					      u32 lro_patch_optimization_en)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_lro_ptopt_en_adr,
-			    rpo_lro_ptopt_en_msk,
-			    rpo_lro_ptopt_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_PTOPT_EN_ADR,
+			    HW_ATL_RPO_LRO_PTOPT_EN_MSK,
+			    HW_ATL_RPO_LRO_PTOPT_EN_SHIFT,
 			    lro_patch_optimization_en);
 }
 
-void rpo_lro_qsessions_lim_set(struct aq_hw_s *aq_hw,
-			       u32 lro_qsessions_lim)
+void hw_atl_rpo_lro_qsessions_lim_set(struct aq_hw_s *aq_hw,
+				      u32 lro_qsessions_lim)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_lro_qses_lmt_adr,
-			    rpo_lro_qses_lmt_msk,
-			    rpo_lro_qses_lmt_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_QSES_LMT_ADR,
+			    HW_ATL_RPO_LRO_QSES_LMT_MSK,
+			    HW_ATL_RPO_LRO_QSES_LMT_SHIFT,
 			    lro_qsessions_lim);
 }
 
-void rpo_lro_total_desc_lim_set(struct aq_hw_s *aq_hw, u32 lro_total_desc_lim)
+void hw_atl_rpo_lro_total_desc_lim_set(struct aq_hw_s *aq_hw,
+				       u32 lro_total_desc_lim)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_lro_tot_dsc_lmt_adr,
-			    rpo_lro_tot_dsc_lmt_msk,
-			    rpo_lro_tot_dsc_lmt_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_TOT_DSC_LMT_ADR,
+			    HW_ATL_RPO_LRO_TOT_DSC_LMT_MSK,
+			    HW_ATL_RPO_LRO_TOT_DSC_LMT_SHIFT,
 			    lro_total_desc_lim);
 }
 
-void rpo_lro_min_pay_of_first_pkt_set(struct aq_hw_s *aq_hw,
-				      u32 lro_min_pld_of_first_pkt)
+void hw_atl_rpo_lro_min_pay_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lro_min_pld_of_first_pkt)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_lro_pkt_min_adr,
-			    rpo_lro_pkt_min_msk,
-			    rpo_lro_pkt_min_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_PKT_MIN_ADR,
+			    HW_ATL_RPO_LRO_PKT_MIN_MSK,
+			    HW_ATL_RPO_LRO_PKT_MIN_SHIFT,
 			    lro_min_pld_of_first_pkt);
 }
 
-void rpo_lro_pkt_lim_set(struct aq_hw_s *aq_hw, u32 lro_pkt_lim)
+void hw_atl_rpo_lro_pkt_lim_set(struct aq_hw_s *aq_hw, u32 lro_pkt_lim)
 {
-	aq_hw_write_reg(aq_hw, rpo_lro_rsc_max_adr, lro_pkt_lim);
+	aq_hw_write_reg(aq_hw, HW_ATL_RPO_LRO_RSC_MAX_ADR, lro_pkt_lim);
 }
 
-void rpo_lro_max_num_of_descriptors_set(struct aq_hw_s *aq_hw,
-					u32 lro_max_number_of_descriptors,
-					u32 lro)
+void hw_atl_rpo_lro_max_num_of_descriptors_set(struct aq_hw_s *aq_hw,
+					       u32 lro_max_number_of_descriptors,
+					       u32 lro)
 {
 /* Register address for bitfield lro{L}_des_max[1:0] */
 	static u32 rpo_lro_ldes_max_adr[32] = {
@@ -1017,378 +1073,390 @@ void rpo_lro_max_num_of_descriptors_set(struct aq_hw_s *aq_hw,
 			    lro_max_number_of_descriptors);
 }
 
-void rpo_lro_time_base_divider_set(struct aq_hw_s *aq_hw,
-				   u32 lro_time_base_divider)
+void hw_atl_rpo_lro_time_base_divider_set(struct aq_hw_s *aq_hw,
+					  u32 lro_time_base_divider)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_lro_tb_div_adr,
-			    rpo_lro_tb_div_msk,
-			    rpo_lro_tb_div_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_TB_DIV_ADR,
+			    HW_ATL_RPO_LRO_TB_DIV_MSK,
+			    HW_ATL_RPO_LRO_TB_DIV_SHIFT,
 			    lro_time_base_divider);
 }
 
-void rpo_lro_inactive_interval_set(struct aq_hw_s *aq_hw,
-				   u32 lro_inactive_interval)
+void hw_atl_rpo_lro_inactive_interval_set(struct aq_hw_s *aq_hw,
+					  u32 lro_inactive_interval)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_lro_ina_ival_adr,
-			    rpo_lro_ina_ival_msk,
-			    rpo_lro_ina_ival_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_INA_IVAL_ADR,
+			    HW_ATL_RPO_LRO_INA_IVAL_MSK,
+			    HW_ATL_RPO_LRO_INA_IVAL_SHIFT,
 			    lro_inactive_interval);
 }
 
-void rpo_lro_max_coalescing_interval_set(struct aq_hw_s *aq_hw,
-					 u32 lro_max_coalescing_interval)
+void hw_atl_rpo_lro_max_coalescing_interval_set(struct aq_hw_s *aq_hw,
+						u32 lro_max_coal_interval)
 {
-	aq_hw_write_reg_bit(aq_hw, rpo_lro_max_ival_adr,
-			    rpo_lro_max_ival_msk,
-			    rpo_lro_max_ival_shift,
-			    lro_max_coalescing_interval);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RPO_LRO_MAX_IVAL_ADR,
+			    HW_ATL_RPO_LRO_MAX_IVAL_MSK,
+			    HW_ATL_RPO_LRO_MAX_IVAL_SHIFT,
+			    lro_max_coal_interval);
 }
 
 /* rx */
-void rx_rx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 rx_reg_res_dis)
+void hw_atl_rx_rx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 rx_reg_res_dis)
 {
-	aq_hw_write_reg_bit(aq_hw, rx_reg_res_dsbl_adr,
-			    rx_reg_res_dsbl_msk,
-			    rx_reg_res_dsbl_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_RX_REG_RES_DSBL_ADR,
+			    HW_ATL_RX_REG_RES_DSBL_MSK,
+			    HW_ATL_RX_REG_RES_DSBL_SHIFT,
 			    rx_reg_res_dis);
 }
 
 /* tdm */
-void tdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca)
+void hw_atl_tdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_dcadcpuid_adr(dca),
-			    tdm_dcadcpuid_msk,
-			    tdm_dcadcpuid_shift, cpuid);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCADCPUID_ADR(dca),
+			    HW_ATL_TDM_DCADCPUID_MSK,
+			    HW_ATL_TDM_DCADCPUID_SHIFT, cpuid);
 }
 
-void tdm_large_send_offload_en_set(struct aq_hw_s *aq_hw,
-				   u32 large_send_offload_en)
+void hw_atl_tdm_large_send_offload_en_set(struct aq_hw_s *aq_hw,
+					  u32 large_send_offload_en)
 {
-	aq_hw_write_reg(aq_hw, tdm_lso_en_adr, large_send_offload_en);
+	aq_hw_write_reg(aq_hw, HW_ATL_TDM_LSO_EN_ADR, large_send_offload_en);
 }
 
-void tdm_tx_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_dca_en)
+void hw_atl_tdm_tx_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_dca_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_dca_en_adr, tdm_dca_en_msk,
-			    tdm_dca_en_shift, tx_dca_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCA_EN_ADR, HW_ATL_TDM_DCA_EN_MSK,
+			    HW_ATL_TDM_DCA_EN_SHIFT, tx_dca_en);
 }
 
-void tdm_tx_dca_mode_set(struct aq_hw_s *aq_hw, u32 tx_dca_mode)
+void hw_atl_tdm_tx_dca_mode_set(struct aq_hw_s *aq_hw, u32 tx_dca_mode)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_dca_mode_adr, tdm_dca_mode_msk,
-			    tdm_dca_mode_shift, tx_dca_mode);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCA_MODE_ADR,
+			    HW_ATL_TDM_DCA_MODE_MSK,
+			    HW_ATL_TDM_DCA_MODE_SHIFT, tx_dca_mode);
 }
 
-void tdm_tx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_dca_en, u32 dca)
+void hw_atl_tdm_tx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_dca_en,
+				   u32 dca)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_dcaddesc_en_adr(dca),
-			    tdm_dcaddesc_en_msk, tdm_dcaddesc_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DCADDESC_EN_ADR(dca),
+			    HW_ATL_TDM_DCADDESC_EN_MSK,
+			    HW_ATL_TDM_DCADDESC_EN_SHIFT,
 			    tx_desc_dca_en);
 }
 
-void tdm_tx_desc_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_en, u32 descriptor)
+void hw_atl_tdm_tx_desc_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_en,
+			       u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_descden_adr(descriptor),
-			    tdm_descden_msk,
-			    tdm_descden_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DESCDEN_ADR(descriptor),
+			    HW_ATL_TDM_DESCDEN_MSK,
+			    HW_ATL_TDM_DESCDEN_SHIFT,
 			    tx_desc_en);
 }
 
-u32 tdm_tx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor)
+u32 hw_atl_tdm_tx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor)
 {
-	return aq_hw_read_reg_bit(aq_hw, tdm_descdhd_adr(descriptor),
-				  tdm_descdhd_msk, tdm_descdhd_shift);
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_TDM_DESCDHD_ADR(descriptor),
+				  HW_ATL_TDM_DESCDHD_MSK,
+				  HW_ATL_TDM_DESCDHD_SHIFT);
 }
 
-void tdm_tx_desc_len_set(struct aq_hw_s *aq_hw, u32 tx_desc_len,
-			 u32 descriptor)
+void hw_atl_tdm_tx_desc_len_set(struct aq_hw_s *aq_hw, u32 tx_desc_len,
+				u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_descdlen_adr(descriptor),
-			    tdm_descdlen_msk,
-			    tdm_descdlen_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DESCDLEN_ADR(descriptor),
+			    HW_ATL_TDM_DESCDLEN_MSK,
+			    HW_ATL_TDM_DESCDLEN_SHIFT,
 			    tx_desc_len);
 }
 
-void tdm_tx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
-				  u32 tx_desc_wr_wb_irq_en)
+void hw_atl_tdm_tx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 tx_desc_wr_wb_irq_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_int_desc_wrb_en_adr,
-			    tdm_int_desc_wrb_en_msk,
-			    tdm_int_desc_wrb_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_INT_DESC_WRB_EN_ADR,
+			    HW_ATL_TDM_INT_DESC_WRB_EN_MSK,
+			    HW_ATL_TDM_INT_DESC_WRB_EN_SHIFT,
 			    tx_desc_wr_wb_irq_en);
 }
 
-void tdm_tx_desc_wr_wb_threshold_set(struct aq_hw_s *aq_hw,
-				     u32 tx_desc_wr_wb_threshold,
-				     u32 descriptor)
+void hw_atl_tdm_tx_desc_wr_wb_threshold_set(struct aq_hw_s *aq_hw,
+					    u32 tx_desc_wr_wb_threshold,
+					    u32 descriptor)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_descdwrb_thresh_adr(descriptor),
-			    tdm_descdwrb_thresh_msk,
-			    tdm_descdwrb_thresh_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_DESCDWRB_THRESH_ADR(descriptor),
+			    HW_ATL_TDM_DESCDWRB_THRESH_MSK,
+			    HW_ATL_TDM_DESCDWRB_THRESH_SHIFT,
 			    tx_desc_wr_wb_threshold);
 }
 
-void tdm_tdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
-			       u32 tdm_irq_moderation_en)
+void hw_atl_tdm_tdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 tdm_irq_moderation_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tdm_int_mod_en_adr,
-			    tdm_int_mod_en_msk,
-			    tdm_int_mod_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TDM_INT_MOD_EN_ADR,
+			    HW_ATL_TDM_INT_MOD_EN_MSK,
+			    HW_ATL_TDM_INT_MOD_EN_SHIFT,
 			    tdm_irq_moderation_en);
 }
 
 /* thm */
-void thm_lso_tcp_flag_of_first_pkt_set(struct aq_hw_s *aq_hw,
-				       u32 lso_tcp_flag_of_first_pkt)
+void hw_atl_thm_lso_tcp_flag_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					      u32 lso_tcp_flag_of_first_pkt)
 {
-	aq_hw_write_reg_bit(aq_hw, thm_lso_tcp_flag_first_adr,
-			    thm_lso_tcp_flag_first_msk,
-			    thm_lso_tcp_flag_first_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_THM_LSO_TCP_FLAG_FIRST_ADR,
+			    HW_ATL_THM_LSO_TCP_FLAG_FIRST_MSK,
+			    HW_ATL_THM_LSO_TCP_FLAG_FIRST_SHIFT,
 			    lso_tcp_flag_of_first_pkt);
 }
 
-void thm_lso_tcp_flag_of_last_pkt_set(struct aq_hw_s *aq_hw,
-				      u32 lso_tcp_flag_of_last_pkt)
+void hw_atl_thm_lso_tcp_flag_of_last_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lso_tcp_flag_of_last_pkt)
 {
-	aq_hw_write_reg_bit(aq_hw, thm_lso_tcp_flag_last_adr,
-			    thm_lso_tcp_flag_last_msk,
-			    thm_lso_tcp_flag_last_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_THM_LSO_TCP_FLAG_LAST_ADR,
+			    HW_ATL_THM_LSO_TCP_FLAG_LAST_MSK,
+			    HW_ATL_THM_LSO_TCP_FLAG_LAST_SHIFT,
 			    lso_tcp_flag_of_last_pkt);
 }
 
-void thm_lso_tcp_flag_of_middle_pkt_set(struct aq_hw_s *aq_hw,
-					u32 lso_tcp_flag_of_middle_pkt)
+void hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(struct aq_hw_s *aq_hw,
+					       u32 lso_tcp_flag_of_middle_pkt)
 {
-	aq_hw_write_reg_bit(aq_hw, thm_lso_tcp_flag_mid_adr,
-			    thm_lso_tcp_flag_mid_msk,
-			    thm_lso_tcp_flag_mid_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_THM_LSO_TCP_FLAG_MID_ADR,
+			    HW_ATL_THM_LSO_TCP_FLAG_MID_MSK,
+			    HW_ATL_THM_LSO_TCP_FLAG_MID_SHIFT,
 			    lso_tcp_flag_of_middle_pkt);
 }
 
 /* TPB: tx packet buffer */
-void tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en)
+void hw_atl_tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tpb_tx_buf_en_adr, tpb_tx_buf_en_msk,
-			    tpb_tx_buf_en_shift, tx_buff_en);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TX_BUF_EN_ADR,
+			    HW_ATL_TPB_TX_BUF_EN_MSK,
+			    HW_ATL_TPB_TX_BUF_EN_SHIFT, tx_buff_en);
 }
 
-void tpb_tx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 tx_buff_hi_threshold_per_tc,
+void hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_hi_threshold_per_tc,
 					 u32 buffer)
 {
-	aq_hw_write_reg_bit(aq_hw, tpb_txbhi_thresh_adr(buffer),
-			    tpb_txbhi_thresh_msk, tpb_txbhi_thresh_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TXBHI_THRESH_ADR(buffer),
+			    HW_ATL_TPB_TXBHI_THRESH_MSK,
+			    HW_ATL_TPB_TXBHI_THRESH_SHIFT,
 			    tx_buff_hi_threshold_per_tc);
 }
 
-void tpb_tx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 tx_buff_lo_threshold_per_tc,
+void hw_atl_tpb_tx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_lo_threshold_per_tc,
 					 u32 buffer)
 {
-	aq_hw_write_reg_bit(aq_hw, tpb_txblo_thresh_adr(buffer),
-			    tpb_txblo_thresh_msk, tpb_txblo_thresh_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TXBLO_THRESH_ADR(buffer),
+			    HW_ATL_TPB_TXBLO_THRESH_MSK,
+			    HW_ATL_TPB_TXBLO_THRESH_SHIFT,
 			    tx_buff_lo_threshold_per_tc);
 }
 
-void tpb_tx_dma_sys_lbk_en_set(struct aq_hw_s *aq_hw, u32 tx_dma_sys_lbk_en)
+void hw_atl_tpb_tx_dma_sys_lbk_en_set(struct aq_hw_s *aq_hw, u32 tx_dma_sys_lbk_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tpb_dma_sys_lbk_adr,
-			    tpb_dma_sys_lbk_msk,
-			    tpb_dma_sys_lbk_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_DMA_SYS_LBK_ADR,
+			    HW_ATL_TPB_DMA_SYS_LBK_MSK,
+			    HW_ATL_TPB_DMA_SYS_LBK_SHIFT,
 			    tx_dma_sys_lbk_en);
 }
 
-void tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
-				     u32 tx_pkt_buff_size_per_tc, u32 buffer)
+void hw_atl_tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 tx_pkt_buff_size_per_tc, u32 buffer)
 {
-	aq_hw_write_reg_bit(aq_hw, tpb_txbbuf_size_adr(buffer),
-			    tpb_txbbuf_size_msk,
-			    tpb_txbbuf_size_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TXBBUF_SIZE_ADR(buffer),
+			    HW_ATL_TPB_TXBBUF_SIZE_MSK,
+			    HW_ATL_TPB_TXBBUF_SIZE_SHIFT,
 			    tx_pkt_buff_size_per_tc);
 }
 
-void tpb_tx_path_scp_ins_en_set(struct aq_hw_s *aq_hw, u32 tx_path_scp_ins_en)
+void hw_atl_tpb_tx_path_scp_ins_en_set(struct aq_hw_s *aq_hw, u32 tx_path_scp_ins_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tpb_tx_scp_ins_en_adr,
-			    tpb_tx_scp_ins_en_msk,
-			    tpb_tx_scp_ins_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPB_TX_SCP_INS_EN_ADR,
+			    HW_ATL_TPB_TX_SCP_INS_EN_MSK,
+			    HW_ATL_TPB_TX_SCP_INS_EN_SHIFT,
 			    tx_path_scp_ins_en);
 }
 
 /* TPO: tx packet offload */
-void tpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				       u32 ipv4header_crc_offload_en)
+void hw_atl_tpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tpo_ipv4chk_en_adr,
-			    tpo_ipv4chk_en_msk,
-			    tpo_ipv4chk_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPO_IPV4CHK_EN_ADR,
+			    HW_ATL_TPO_IPV4CHK_EN_MSK,
+			    HW_ATL_TPO_IPV4CHK_EN_SHIFT,
 			    ipv4header_crc_offload_en);
 }
 
-void tpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				    u32 tcp_udp_crc_offload_en)
+void hw_atl_tpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tpol4chk_en_adr,
-			    tpol4chk_en_msk,
-			    tpol4chk_en_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPOL4CHK_EN_ADR,
+			    HW_ATL_TPOL4CHK_EN_MSK,
+			    HW_ATL_TPOL4CHK_EN_SHIFT,
 			    tcp_udp_crc_offload_en);
 }
 
-void tpo_tx_pkt_sys_lbk_en_set(struct aq_hw_s *aq_hw, u32 tx_pkt_sys_lbk_en)
+void hw_atl_tpo_tx_pkt_sys_lbk_en_set(struct aq_hw_s *aq_hw,
+				      u32 tx_pkt_sys_lbk_en)
 {
-	aq_hw_write_reg_bit(aq_hw, tpo_pkt_sys_lbk_adr,
-			    tpo_pkt_sys_lbk_msk,
-			    tpo_pkt_sys_lbk_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPO_PKT_SYS_LBK_ADR,
+			    HW_ATL_TPO_PKT_SYS_LBK_MSK,
+			    HW_ATL_TPO_PKT_SYS_LBK_SHIFT,
 			    tx_pkt_sys_lbk_en);
 }
 
 /* TPS: tx packet scheduler */
-void tps_tx_pkt_shed_data_arb_mode_set(struct aq_hw_s *aq_hw,
-				       u32 tx_pkt_shed_data_arb_mode)
+void hw_atl_tps_tx_pkt_shed_data_arb_mode_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_data_arb_mode)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_data_tc_arb_mode_adr,
-			    tps_data_tc_arb_mode_msk,
-			    tps_data_tc_arb_mode_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DATA_TC_ARB_MODE_ADR,
+			    HW_ATL_TPS_DATA_TC_ARB_MODE_MSK,
+			    HW_ATL_TPS_DATA_TC_ARB_MODE_SHIFT,
 			    tx_pkt_shed_data_arb_mode);
 }
 
-void tps_tx_pkt_shed_desc_rate_curr_time_res_set(struct aq_hw_s *aq_hw,
-						 u32 curr_time_res)
+void hw_atl_tps_tx_pkt_shed_desc_rate_curr_time_res_set(struct aq_hw_s *aq_hw,
+							u32 curr_time_res)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_desc_rate_ta_rst_adr,
-			    tps_desc_rate_ta_rst_msk,
-			    tps_desc_rate_ta_rst_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_RATE_TA_RST_ADR,
+			    HW_ATL_TPS_DESC_RATE_TA_RST_MSK,
+			    HW_ATL_TPS_DESC_RATE_TA_RST_SHIFT,
 			    curr_time_res);
 }
 
-void tps_tx_pkt_shed_desc_rate_lim_set(struct aq_hw_s *aq_hw,
-				       u32 tx_pkt_shed_desc_rate_lim)
+void hw_atl_tps_tx_pkt_shed_desc_rate_lim_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_desc_rate_lim)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_desc_rate_lim_adr,
-			    tps_desc_rate_lim_msk,
-			    tps_desc_rate_lim_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_RATE_LIM_ADR,
+			    HW_ATL_TPS_DESC_RATE_LIM_MSK,
+			    HW_ATL_TPS_DESC_RATE_LIM_SHIFT,
 			    tx_pkt_shed_desc_rate_lim);
 }
 
-void tps_tx_pkt_shed_desc_tc_arb_mode_set(struct aq_hw_s *aq_hw,
-					  u32 tx_pkt_shed_desc_tc_arb_mode)
+void hw_atl_tps_tx_pkt_shed_desc_tc_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_desc_tc_arb_mode_adr,
-			    tps_desc_tc_arb_mode_msk,
-			    tps_desc_tc_arb_mode_shift,
-			    tx_pkt_shed_desc_tc_arb_mode);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_TC_ARB_MODE_ADR,
+			    HW_ATL_TPS_DESC_TC_ARB_MODE_MSK,
+			    HW_ATL_TPS_DESC_TC_ARB_MODE_SHIFT,
+			    arb_mode);
 }
 
-void tps_tx_pkt_shed_desc_tc_max_credit_set(struct aq_hw_s *aq_hw,
-					    u32 tx_pkt_shed_desc_tc_max_credit,
-					    u32 tc)
+void hw_atl_tps_tx_pkt_shed_desc_tc_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
+						   u32 tc)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_desc_tctcredit_max_adr(tc),
-			    tps_desc_tctcredit_max_msk,
-			    tps_desc_tctcredit_max_shift,
-			    tx_pkt_shed_desc_tc_max_credit);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_TCTCREDIT_MAX_ADR(tc),
+			    HW_ATL_TPS_DESC_TCTCREDIT_MAX_MSK,
+			    HW_ATL_TPS_DESC_TCTCREDIT_MAX_SHIFT,
+			    max_credit);
 }
 
-void tps_tx_pkt_shed_desc_tc_weight_set(struct aq_hw_s *aq_hw,
-					u32 tx_pkt_shed_desc_tc_weight, u32 tc)
+void hw_atl_tps_tx_pkt_shed_desc_tc_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_desc_tc_weight,
+					       u32 tc)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_desc_tctweight_adr(tc),
-			    tps_desc_tctweight_msk,
-			    tps_desc_tctweight_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_TCTWEIGHT_ADR(tc),
+			    HW_ATL_TPS_DESC_TCTWEIGHT_MSK,
+			    HW_ATL_TPS_DESC_TCTWEIGHT_SHIFT,
 			    tx_pkt_shed_desc_tc_weight);
 }
 
-void tps_tx_pkt_shed_desc_vm_arb_mode_set(struct aq_hw_s *aq_hw,
-					  u32 tx_pkt_shed_desc_vm_arb_mode)
+void hw_atl_tps_tx_pkt_shed_desc_vm_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_desc_vm_arb_mode_adr,
-			    tps_desc_vm_arb_mode_msk,
-			    tps_desc_vm_arb_mode_shift,
-			    tx_pkt_shed_desc_vm_arb_mode);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DESC_VM_ARB_MODE_ADR,
+			    HW_ATL_TPS_DESC_VM_ARB_MODE_MSK,
+			    HW_ATL_TPS_DESC_VM_ARB_MODE_SHIFT,
+			    arb_mode);
 }
 
-void tps_tx_pkt_shed_tc_data_max_credit_set(struct aq_hw_s *aq_hw,
-					    u32 tx_pkt_shed_tc_data_max_credit,
-					    u32 tc)
+void hw_atl_tps_tx_pkt_shed_tc_data_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
+						   u32 tc)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_data_tctcredit_max_adr(tc),
-			    tps_data_tctcredit_max_msk,
-			    tps_data_tctcredit_max_shift,
-			    tx_pkt_shed_tc_data_max_credit);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DATA_TCTCREDIT_MAX_ADR(tc),
+			    HW_ATL_TPS_DATA_TCTCREDIT_MAX_MSK,
+			    HW_ATL_TPS_DATA_TCTCREDIT_MAX_SHIFT,
+			    max_credit);
 }
 
-void tps_tx_pkt_shed_tc_data_weight_set(struct aq_hw_s *aq_hw,
-					u32 tx_pkt_shed_tc_data_weight, u32 tc)
+void hw_atl_tps_tx_pkt_shed_tc_data_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_tc_data_weight,
+					       u32 tc)
 {
-	aq_hw_write_reg_bit(aq_hw, tps_data_tctweight_adr(tc),
-			    tps_data_tctweight_msk,
-			    tps_data_tctweight_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TPS_DATA_TCTWEIGHT_ADR(tc),
+			    HW_ATL_TPS_DATA_TCTWEIGHT_MSK,
+			    HW_ATL_TPS_DATA_TCTWEIGHT_SHIFT,
 			    tx_pkt_shed_tc_data_weight);
 }
 
 /* tx */
-void tx_tx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 tx_reg_res_dis)
+void hw_atl_tx_tx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 tx_reg_res_dis)
 {
-	aq_hw_write_reg_bit(aq_hw, tx_reg_res_dsbl_adr,
-			    tx_reg_res_dsbl_msk,
-			    tx_reg_res_dsbl_shift, tx_reg_res_dis);
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_TX_REG_RES_DSBL_ADR,
+			    HW_ATL_TX_REG_RES_DSBL_MSK,
+			    HW_ATL_TX_REG_RES_DSBL_SHIFT, tx_reg_res_dis);
 }
 
 /* msm */
-u32 msm_reg_access_status_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_msm_reg_access_status_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg_bit(aq_hw, msm_reg_access_busy_adr,
-				  msm_reg_access_busy_msk,
-				  msm_reg_access_busy_shift);
+	return aq_hw_read_reg_bit(aq_hw, HW_ATL_MSM_REG_ACCESS_BUSY_ADR,
+				  HW_ATL_MSM_REG_ACCESS_BUSY_MSK,
+				  HW_ATL_MSM_REG_ACCESS_BUSY_SHIFT);
 }
 
-void msm_reg_addr_for_indirect_addr_set(struct aq_hw_s *aq_hw,
-					u32 reg_addr_for_indirect_addr)
+void hw_atl_msm_reg_addr_for_indirect_addr_set(struct aq_hw_s *aq_hw,
+					       u32 reg_addr_for_indirect_addr)
 {
-	aq_hw_write_reg_bit(aq_hw, msm_reg_addr_adr,
-			    msm_reg_addr_msk,
-			    msm_reg_addr_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_MSM_REG_ADDR_ADR,
+			    HW_ATL_MSM_REG_ADDR_MSK,
+			    HW_ATL_MSM_REG_ADDR_SHIFT,
 			    reg_addr_for_indirect_addr);
 }
 
-void msm_reg_rd_strobe_set(struct aq_hw_s *aq_hw, u32 reg_rd_strobe)
+void hw_atl_msm_reg_rd_strobe_set(struct aq_hw_s *aq_hw, u32 reg_rd_strobe)
 {
-	aq_hw_write_reg_bit(aq_hw, msm_reg_rd_strobe_adr,
-			    msm_reg_rd_strobe_msk,
-			    msm_reg_rd_strobe_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_MSM_REG_RD_STROBE_ADR,
+			    HW_ATL_MSM_REG_RD_STROBE_MSK,
+			    HW_ATL_MSM_REG_RD_STROBE_SHIFT,
 			    reg_rd_strobe);
 }
 
-u32 msm_reg_rd_data_get(struct aq_hw_s *aq_hw)
+u32 hw_atl_msm_reg_rd_data_get(struct aq_hw_s *aq_hw)
 {
-	return aq_hw_read_reg(aq_hw, msm_reg_rd_data_adr);
+	return aq_hw_read_reg(aq_hw, HW_ATL_MSM_REG_RD_DATA_ADR);
 }
 
-void msm_reg_wr_data_set(struct aq_hw_s *aq_hw, u32 reg_wr_data)
+void hw_atl_msm_reg_wr_data_set(struct aq_hw_s *aq_hw, u32 reg_wr_data)
 {
-	aq_hw_write_reg(aq_hw, msm_reg_wr_data_adr, reg_wr_data);
+	aq_hw_write_reg(aq_hw, HW_ATL_MSM_REG_WR_DATA_ADR, reg_wr_data);
 }
 
-void msm_reg_wr_strobe_set(struct aq_hw_s *aq_hw, u32 reg_wr_strobe)
+void hw_atl_msm_reg_wr_strobe_set(struct aq_hw_s *aq_hw, u32 reg_wr_strobe)
 {
-	aq_hw_write_reg_bit(aq_hw, msm_reg_wr_strobe_adr,
-			    msm_reg_wr_strobe_msk,
-			    msm_reg_wr_strobe_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_MSM_REG_WR_STROBE_ADR,
+			    HW_ATL_MSM_REG_WR_STROBE_MSK,
+			    HW_ATL_MSM_REG_WR_STROBE_SHIFT,
 			    reg_wr_strobe);
 }
 
 /* pci */
-void pci_pci_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 pci_reg_res_dis)
+void hw_atl_pci_pci_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 pci_reg_res_dis)
 {
-	aq_hw_write_reg_bit(aq_hw, pci_reg_res_dsbl_adr,
-			    pci_reg_res_dsbl_msk,
-			    pci_reg_res_dsbl_shift,
+	aq_hw_write_reg_bit(aq_hw, HW_ATL_PCI_REG_RES_DSBL_ADR,
+			    HW_ATL_PCI_REG_RES_DSBL_MSK,
+			    HW_ATL_PCI_REG_RES_DSBL_SHIFT,
 			    pci_reg_res_dis);
 }
 
-void reg_glb_cpu_scratch_scp_set(struct aq_hw_s *aq_hw, u32 glb_cpu_scratch_scp,
-				 u32 scratch_scp)
+void hw_atl_reg_glb_cpu_scratch_scp_set(struct aq_hw_s *aq_hw,
+					u32 glb_cpu_scratch_scp,
+					u32 scratch_scp)
 {
-	aq_hw_write_reg(aq_hw, glb_cpu_scratch_scp_adr(scratch_scp),
+	aq_hw_write_reg(aq_hw, HW_ATL_GLB_CPU_SCRATCH_SCP_ADR(scratch_scp),
 			glb_cpu_scratch_scp);
 }
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h
index ed1085b95adb..dfb426f2dc2c 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh.h
@@ -21,657 +21,681 @@ struct aq_hw_s;
 /* global */
 
 /* set global microprocessor semaphore */
-void reg_glb_cpu_sem_set(struct aq_hw_s *aq_hw,	u32 glb_cpu_sem,
-			 u32 semaphore);
+void hw_atl_reg_glb_cpu_sem_set(struct aq_hw_s *aq_hw,	u32 glb_cpu_sem,
+				u32 semaphore);
 
 /* get global microprocessor semaphore */
-u32 reg_glb_cpu_sem_get(struct aq_hw_s *aq_hw, u32 semaphore);
+u32 hw_atl_reg_glb_cpu_sem_get(struct aq_hw_s *aq_hw, u32 semaphore);
 
 /* set global register reset disable */
-void glb_glb_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 glb_reg_res_dis);
+void hw_atl_glb_glb_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 glb_reg_res_dis);
 
 /* set soft reset */
-void glb_soft_res_set(struct aq_hw_s *aq_hw, u32 soft_res);
+void hw_atl_glb_soft_res_set(struct aq_hw_s *aq_hw, u32 soft_res);
 
 /* get soft reset */
-u32 glb_soft_res_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_glb_soft_res_get(struct aq_hw_s *aq_hw);
 
 /* stats */
 
-u32 rpb_rx_dma_drop_pkt_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_rpb_rx_dma_drop_pkt_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get rx dma good octet counter lsw */
-u32 stats_rx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_rx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw);
 
 /* get rx dma good packet counter lsw */
-u32 stats_rx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_rx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw);
 
 /* get tx dma good octet counter lsw */
-u32 stats_tx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_tx_dma_good_octet_counterlsw_get(struct aq_hw_s *aq_hw);
 
 /* get tx dma good packet counter lsw */
-u32 stats_tx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_tx_dma_good_pkt_counterlsw_get(struct aq_hw_s *aq_hw);
 
 /* get rx dma good octet counter msw */
-u32 stats_rx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_rx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw);
 
 /* get rx dma good packet counter msw */
-u32 stats_rx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_rx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw);
 
 /* get tx dma good octet counter msw */
-u32 stats_tx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_tx_dma_good_octet_countermsw_get(struct aq_hw_s *aq_hw);
 
 /* get tx dma good packet counter msw */
-u32 stats_tx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_stats_tx_dma_good_pkt_countermsw_get(struct aq_hw_s *aq_hw);
 
 /* get msm rx errors counter register */
-u32 reg_mac_msm_rx_errs_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_rx_errs_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm rx unicast frames counter register */
-u32 reg_mac_msm_rx_ucst_frm_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_rx_ucst_frm_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm rx multicast frames counter register */
-u32 reg_mac_msm_rx_mcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_rx_mcst_frm_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm rx broadcast frames counter register */
-u32 reg_mac_msm_rx_bcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_rx_bcst_frm_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm rx broadcast octets counter register 1 */
-u32 reg_mac_msm_rx_bcst_octets_counter1get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_rx_bcst_octets_counter1get(struct aq_hw_s *aq_hw);
 
 /* get msm rx unicast octets counter register 0 */
-u32 reg_mac_msm_rx_ucst_octets_counter0get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_rx_ucst_octets_counter0get(struct aq_hw_s *aq_hw);
 
 /* get rx dma statistics counter 7 */
-u32 reg_rx_dma_stat_counter7get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_rx_dma_stat_counter7get(struct aq_hw_s *aq_hw);
 
 /* get msm tx errors counter register */
-u32 reg_mac_msm_tx_errs_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_tx_errs_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm tx unicast frames counter register */
-u32 reg_mac_msm_tx_ucst_frm_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_tx_ucst_frm_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm tx multicast frames counter register */
-u32 reg_mac_msm_tx_mcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_tx_mcst_frm_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm tx broadcast frames counter register */
-u32 reg_mac_msm_tx_bcst_frm_cnt_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_tx_bcst_frm_cnt_get(struct aq_hw_s *aq_hw);
 
 /* get msm tx multicast octets counter register 1 */
-u32 reg_mac_msm_tx_mcst_octets_counter1get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_tx_mcst_octets_counter1get(struct aq_hw_s *aq_hw);
 
 /* get msm tx broadcast octets counter register 1 */
-u32 reg_mac_msm_tx_bcst_octets_counter1get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_tx_bcst_octets_counter1get(struct aq_hw_s *aq_hw);
 
 /* get msm tx unicast octets counter register 0 */
-u32 reg_mac_msm_tx_ucst_octets_counter0get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_mac_msm_tx_ucst_octets_counter0get(struct aq_hw_s *aq_hw);
 
 /* get global mif identification */
-u32 reg_glb_mif_id_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_glb_mif_id_get(struct aq_hw_s *aq_hw);
 
 /* interrupt */
 
 /* set interrupt auto mask lsw */
-void itr_irq_auto_masklsw_set(struct aq_hw_s *aq_hw, u32 irq_auto_masklsw);
+void hw_atl_itr_irq_auto_masklsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_auto_masklsw);
 
 /* set interrupt mapping enable rx */
-void itr_irq_map_en_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_rx, u32 rx);
+void hw_atl_itr_irq_map_en_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_rx,
+				  u32 rx);
 
 /* set interrupt mapping enable tx */
-void itr_irq_map_en_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_tx, u32 tx);
+void hw_atl_itr_irq_map_en_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_en_tx,
+				  u32 tx);
 
 /* set interrupt mapping rx */
-void itr_irq_map_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_rx, u32 rx);
+void hw_atl_itr_irq_map_rx_set(struct aq_hw_s *aq_hw, u32 irq_map_rx, u32 rx);
 
 /* set interrupt mapping tx */
-void itr_irq_map_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_tx, u32 tx);
+void hw_atl_itr_irq_map_tx_set(struct aq_hw_s *aq_hw, u32 irq_map_tx, u32 tx);
 
 /* set interrupt mask clear lsw */
-void itr_irq_msk_clearlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_clearlsw);
+void hw_atl_itr_irq_msk_clearlsw_set(struct aq_hw_s *aq_hw,
+				     u32 irq_msk_clearlsw);
 
 /* set interrupt mask set lsw */
-void itr_irq_msk_setlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_setlsw);
+void hw_atl_itr_irq_msk_setlsw_set(struct aq_hw_s *aq_hw, u32 irq_msk_setlsw);
 
 /* set interrupt register reset disable */
-void itr_irq_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 irq_reg_res_dis);
+void hw_atl_itr_irq_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 irq_reg_res_dis);
 
 /* set interrupt status clear lsw */
-void itr_irq_status_clearlsw_set(struct aq_hw_s *aq_hw,
-				 u32 irq_status_clearlsw);
+void hw_atl_itr_irq_status_clearlsw_set(struct aq_hw_s *aq_hw,
+					u32 irq_status_clearlsw);
 
 /* get interrupt status lsw */
-u32 itr_irq_statuslsw_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_itr_irq_statuslsw_get(struct aq_hw_s *aq_hw);
 
 /* get reset interrupt */
-u32 itr_res_irq_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_itr_res_irq_get(struct aq_hw_s *aq_hw);
 
 /* set reset interrupt */
-void itr_res_irq_set(struct aq_hw_s *aq_hw, u32 res_irq);
+void hw_atl_itr_res_irq_set(struct aq_hw_s *aq_hw, u32 res_irq);
 
 /* rdm */
 
 /* set cpu id */
-void rdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca);
+void hw_atl_rdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca);
 
 /* set rx dca enable */
-void rdm_rx_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_dca_en);
+void hw_atl_rdm_rx_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_dca_en);
 
 /* set rx dca mode */
-void rdm_rx_dca_mode_set(struct aq_hw_s *aq_hw, u32 rx_dca_mode);
+void hw_atl_rdm_rx_dca_mode_set(struct aq_hw_s *aq_hw, u32 rx_dca_mode);
 
 /* set rx descriptor data buffer size */
-void rdm_rx_desc_data_buff_size_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_data_buff_size,
+void hw_atl_rdm_rx_desc_data_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_data_buff_size,
 				    u32 descriptor);
 
 /* set rx descriptor dca enable */
-void rdm_rx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_dca_en,
-			    u32 dca);
+void hw_atl_rdm_rx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_dca_en,
+				   u32 dca);
 
 /* set rx descriptor enable */
-void rdm_rx_desc_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_en,
-			u32 descriptor);
+void hw_atl_rdm_rx_desc_en_set(struct aq_hw_s *aq_hw, u32 rx_desc_en,
+			       u32 descriptor);
 
 /* set rx descriptor header splitting */
-void rdm_rx_desc_head_splitting_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_head_splitting,
+void hw_atl_rdm_rx_desc_head_splitting_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_splitting,
 				    u32 descriptor);
 
 /* get rx descriptor head pointer */
-u32 rdm_rx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor);
+u32 hw_atl_rdm_rx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor);
 
 /* set rx descriptor length */
-void rdm_rx_desc_len_set(struct aq_hw_s *aq_hw, u32 rx_desc_len,
-			 u32 descriptor);
+void hw_atl_rdm_rx_desc_len_set(struct aq_hw_s *aq_hw, u32 rx_desc_len,
+				u32 descriptor);
 
 /* set rx descriptor write-back interrupt enable */
-void rdm_rx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
-				  u32 rx_desc_wr_wb_irq_en);
+void hw_atl_rdm_rx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 rx_desc_wr_wb_irq_en);
 
 /* set rx header dca enable */
-void rdm_rx_head_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_head_dca_en,
-			    u32 dca);
+void hw_atl_rdm_rx_head_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_head_dca_en,
+				   u32 dca);
 
 /* set rx payload dca enable */
-void rdm_rx_pld_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_pld_dca_en, u32 dca);
+void hw_atl_rdm_rx_pld_dca_en_set(struct aq_hw_s *aq_hw, u32 rx_pld_dca_en,
+				  u32 dca);
 
 /* set rx descriptor header buffer size */
-void rdm_rx_desc_head_buff_size_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_head_buff_size,
-				    u32 descriptor);
+void hw_atl_rdm_rx_desc_head_buff_size_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_head_buff_size,
+					   u32 descriptor);
 
 /* set rx descriptor reset */
-void rdm_rx_desc_res_set(struct aq_hw_s *aq_hw, u32 rx_desc_res,
-			 u32 descriptor);
+void hw_atl_rdm_rx_desc_res_set(struct aq_hw_s *aq_hw, u32 rx_desc_res,
+				u32 descriptor);
 
 /* Set RDM Interrupt Moderation Enable */
-void rdm_rdm_intr_moder_en_set(struct aq_hw_s *aq_hw, u32 rdm_intr_moder_en);
+void hw_atl_rdm_rdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 rdm_intr_moder_en);
 
 /* reg */
 
 /* set general interrupt mapping register */
-void reg_gen_irq_map_set(struct aq_hw_s *aq_hw, u32 gen_intr_map, u32 regidx);
+void hw_atl_reg_gen_irq_map_set(struct aq_hw_s *aq_hw, u32 gen_intr_map,
+				u32 regidx);
 
 /* get general interrupt status register */
-u32 reg_gen_irq_status_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_reg_gen_irq_status_get(struct aq_hw_s *aq_hw);
 
 /* set interrupt global control register */
-void reg_irq_glb_ctl_set(struct aq_hw_s *aq_hw, u32 intr_glb_ctl);
+void hw_atl_reg_irq_glb_ctl_set(struct aq_hw_s *aq_hw, u32 intr_glb_ctl);
 
 /* set interrupt throttle register */
-void reg_irq_thr_set(struct aq_hw_s *aq_hw, u32 intr_thr, u32 throttle);
+void hw_atl_reg_irq_thr_set(struct aq_hw_s *aq_hw, u32 intr_thr, u32 throttle);
 
 /* set rx dma descriptor base address lsw */
-void reg_rx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
-					u32 rx_dma_desc_base_addrlsw,
+void hw_atl_reg_rx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrlsw,
 					u32 descriptor);
 
 /* set rx dma descriptor base address msw */
-void reg_rx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
-					u32 rx_dma_desc_base_addrmsw,
+void hw_atl_reg_rx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 rx_dma_desc_base_addrmsw,
 					u32 descriptor);
 
 /* get rx dma descriptor status register */
-u32 reg_rx_dma_desc_status_get(struct aq_hw_s *aq_hw, u32 descriptor);
+u32 hw_atl_reg_rx_dma_desc_status_get(struct aq_hw_s *aq_hw, u32 descriptor);
 
 /* set rx dma descriptor tail pointer register */
-void reg_rx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
-				  u32 rx_dma_desc_tail_ptr,
+void hw_atl_reg_rx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 rx_dma_desc_tail_ptr,
 				  u32 descriptor);
 
 /* set rx filter multicast filter mask register */
-void reg_rx_flr_mcst_flr_msk_set(struct aq_hw_s *aq_hw,
-				 u32 rx_flr_mcst_flr_msk);
+void hw_atl_reg_rx_flr_mcst_flr_msk_set(struct aq_hw_s *aq_hw,
+					u32 rx_flr_mcst_flr_msk);
 
 /* set rx filter multicast filter register */
-void reg_rx_flr_mcst_flr_set(struct aq_hw_s *aq_hw, u32 rx_flr_mcst_flr,
-			     u32 filter);
+void hw_atl_reg_rx_flr_mcst_flr_set(struct aq_hw_s *aq_hw, u32 rx_flr_mcst_flr,
+				    u32 filter);
 
 /* set rx filter rss control register 1 */
-void reg_rx_flr_rss_control1set(struct aq_hw_s *aq_hw,
-				u32 rx_flr_rss_control1);
+void hw_atl_reg_rx_flr_rss_control1set(struct aq_hw_s *aq_hw,
+				       u32 rx_flr_rss_control1);
 
 /* Set RX Filter Control Register 2 */
-void reg_rx_flr_control2_set(struct aq_hw_s *aq_hw, u32 rx_flr_control2);
+void hw_atl_reg_rx_flr_control2_set(struct aq_hw_s *aq_hw, u32 rx_flr_control2);
 
 /* Set RX Interrupt Moderation Control Register */
-void reg_rx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
-				u32 rx_intr_moderation_ctl,
+void hw_atl_reg_rx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 rx_intr_moderation_ctl,
 				u32 queue);
 
 /* set tx dma debug control */
-void reg_tx_dma_debug_ctl_set(struct aq_hw_s *aq_hw, u32 tx_dma_debug_ctl);
+void hw_atl_reg_tx_dma_debug_ctl_set(struct aq_hw_s *aq_hw,
+				     u32 tx_dma_debug_ctl);
 
 /* set tx dma descriptor base address lsw */
-void reg_tx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
-					u32 tx_dma_desc_base_addrlsw,
+void hw_atl_reg_tx_dma_desc_base_addresslswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrlsw,
 					u32 descriptor);
 
 /* set tx dma descriptor base address msw */
-void reg_tx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
-					u32 tx_dma_desc_base_addrmsw,
+void hw_atl_reg_tx_dma_desc_base_addressmswset(struct aq_hw_s *aq_hw,
+					       u32 tx_dma_desc_base_addrmsw,
 					u32 descriptor);
 
 /* set tx dma descriptor tail pointer register */
-void reg_tx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
-				  u32 tx_dma_desc_tail_ptr,
-				  u32 descriptor);
+void hw_atl_reg_tx_dma_desc_tail_ptr_set(struct aq_hw_s *aq_hw,
+					 u32 tx_dma_desc_tail_ptr,
+					 u32 descriptor);
 
 /* Set TX Interrupt Moderation Control Register */
-void reg_tx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
-				u32 tx_intr_moderation_ctl,
-				u32 queue);
+void hw_atl_reg_tx_intr_moder_ctrl_set(struct aq_hw_s *aq_hw,
+				       u32 tx_intr_moderation_ctl,
+				       u32 queue);
 
 /* set global microprocessor scratch pad */
-void reg_glb_cpu_scratch_scp_set(struct aq_hw_s *aq_hw,
-				 u32 glb_cpu_scratch_scp, u32 scratch_scp);
+void hw_atl_reg_glb_cpu_scratch_scp_set(struct aq_hw_s *aq_hw,
+					u32 glb_cpu_scratch_scp,
+					u32 scratch_scp);
 
 /* rpb */
 
 /* set dma system loopback */
-void rpb_dma_sys_lbk_set(struct aq_hw_s *aq_hw, u32 dma_sys_lbk);
+void hw_atl_rpb_dma_sys_lbk_set(struct aq_hw_s *aq_hw, u32 dma_sys_lbk);
 
 /* set rx traffic class mode */
-void rpb_rpf_rx_traf_class_mode_set(struct aq_hw_s *aq_hw,
-				    u32 rx_traf_class_mode);
+void hw_atl_rpb_rpf_rx_traf_class_mode_set(struct aq_hw_s *aq_hw,
+					   u32 rx_traf_class_mode);
 
 /* set rx buffer enable */
-void rpb_rx_buff_en_set(struct aq_hw_s *aq_hw, u32 rx_buff_en);
+void hw_atl_rpb_rx_buff_en_set(struct aq_hw_s *aq_hw, u32 rx_buff_en);
 
 /* set rx buffer high threshold (per tc) */
-void rpb_rx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 rx_buff_hi_threshold_per_tc,
-					 u32 buffer);
+void hw_atl_rpb_rx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_hi_threshold_per_tc,
+						u32 buffer);
 
 /* set rx buffer low threshold (per tc) */
-void rpb_rx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 rx_buff_lo_threshold_per_tc,
+void hw_atl_rpb_rx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 rx_buff_lo_threshold_per_tc,
 					 u32 buffer);
 
 /* set rx flow control mode */
-void rpb_rx_flow_ctl_mode_set(struct aq_hw_s *aq_hw, u32 rx_flow_ctl_mode);
+void hw_atl_rpb_rx_flow_ctl_mode_set(struct aq_hw_s *aq_hw, u32 rx_flow_ctl_mode);
 
 /* set rx packet buffer size (per tc) */
-void rpb_rx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
-				     u32 rx_pkt_buff_size_per_tc,
-				     u32 buffer);
+void hw_atl_rpb_rx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 rx_pkt_buff_size_per_tc,
+					    u32 buffer);
 
 /* set rx xoff enable (per tc) */
-void rpb_rx_xoff_en_per_tc_set(struct aq_hw_s *aq_hw, u32 rx_xoff_en_per_tc,
-			       u32 buffer);
+void hw_atl_rpb_rx_xoff_en_per_tc_set(struct aq_hw_s *aq_hw, u32 rx_xoff_en_per_tc,
+				      u32 buffer);
 
 /* rpf */
 
 /* set l2 broadcast count threshold */
-void rpfl2broadcast_count_threshold_set(struct aq_hw_s *aq_hw,
-					u32 l2broadcast_count_threshold);
+void hw_atl_rpfl2broadcast_count_threshold_set(struct aq_hw_s *aq_hw,
+					       u32 l2broadcast_count_threshold);
 
 /* set l2 broadcast enable */
-void rpfl2broadcast_en_set(struct aq_hw_s *aq_hw, u32 l2broadcast_en);
+void hw_atl_rpfl2broadcast_en_set(struct aq_hw_s *aq_hw, u32 l2broadcast_en);
 
 /* set l2 broadcast filter action */
-void rpfl2broadcast_flr_act_set(struct aq_hw_s *aq_hw,
-				u32 l2broadcast_flr_act);
+void hw_atl_rpfl2broadcast_flr_act_set(struct aq_hw_s *aq_hw,
+				       u32 l2broadcast_flr_act);
 
 /* set l2 multicast filter enable */
-void rpfl2multicast_flr_en_set(struct aq_hw_s *aq_hw, u32 l2multicast_flr_en,
-			       u32 filter);
+void hw_atl_rpfl2multicast_flr_en_set(struct aq_hw_s *aq_hw,
+				      u32 l2multicast_flr_en,
+				      u32 filter);
 
 /* set l2 promiscuous mode enable */
-void rpfl2promiscuous_mode_en_set(struct aq_hw_s *aq_hw,
-				  u32 l2promiscuous_mode_en);
+void hw_atl_rpfl2promiscuous_mode_en_set(struct aq_hw_s *aq_hw,
+					 u32 l2promiscuous_mode_en);
 
 /* set l2 unicast filter action */
-void rpfl2unicast_flr_act_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_act,
-			      u32 filter);
+void hw_atl_rpfl2unicast_flr_act_set(struct aq_hw_s *aq_hw,
+				     u32 l2unicast_flr_act,
+				     u32 filter);
 
 /* set l2 unicast filter enable */
-void rpfl2_uc_flr_en_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_en,
-			 u32 filter);
+void hw_atl_rpfl2_uc_flr_en_set(struct aq_hw_s *aq_hw, u32 l2unicast_flr_en,
+				u32 filter);
 
 /* set l2 unicast destination address lsw */
-void rpfl2unicast_dest_addresslsw_set(struct aq_hw_s *aq_hw,
-				      u32 l2unicast_dest_addresslsw,
+void hw_atl_rpfl2unicast_dest_addresslsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addresslsw,
 				      u32 filter);
 
 /* set l2 unicast destination address msw */
-void rpfl2unicast_dest_addressmsw_set(struct aq_hw_s *aq_hw,
-				      u32 l2unicast_dest_addressmsw,
+void hw_atl_rpfl2unicast_dest_addressmsw_set(struct aq_hw_s *aq_hw,
+					     u32 l2unicast_dest_addressmsw,
 				      u32 filter);
 
 /* Set L2 Accept all Multicast packets */
-void rpfl2_accept_all_mc_packets_set(struct aq_hw_s *aq_hw,
-				     u32 l2_accept_all_mc_packets);
+void hw_atl_rpfl2_accept_all_mc_packets_set(struct aq_hw_s *aq_hw,
+					    u32 l2_accept_all_mc_packets);
 
 /* set user-priority tc mapping */
-void rpf_rpb_user_priority_tc_map_set(struct aq_hw_s *aq_hw,
-				      u32 user_priority_tc_map, u32 tc);
+void hw_atl_rpf_rpb_user_priority_tc_map_set(struct aq_hw_s *aq_hw,
+					     u32 user_priority_tc_map, u32 tc);
 
 /* set rss key address */
-void rpf_rss_key_addr_set(struct aq_hw_s *aq_hw, u32 rss_key_addr);
+void hw_atl_rpf_rss_key_addr_set(struct aq_hw_s *aq_hw, u32 rss_key_addr);
 
 /* set rss key write data */
-void rpf_rss_key_wr_data_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_data);
+void hw_atl_rpf_rss_key_wr_data_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_data);
 
 /* get rss key write enable */
-u32 rpf_rss_key_wr_en_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_rpf_rss_key_wr_en_get(struct aq_hw_s *aq_hw);
 
 /* set rss key write enable */
-void rpf_rss_key_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_en);
+void hw_atl_rpf_rss_key_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_key_wr_en);
 
 /* set rss redirection table address */
-void rpf_rss_redir_tbl_addr_set(struct aq_hw_s *aq_hw,
-				u32 rss_redir_tbl_addr);
+void hw_atl_rpf_rss_redir_tbl_addr_set(struct aq_hw_s *aq_hw,
+				       u32 rss_redir_tbl_addr);
 
 /* set rss redirection table write data */
-void rpf_rss_redir_tbl_wr_data_set(struct aq_hw_s *aq_hw,
-				   u32 rss_redir_tbl_wr_data);
+void hw_atl_rpf_rss_redir_tbl_wr_data_set(struct aq_hw_s *aq_hw,
+					  u32 rss_redir_tbl_wr_data);
 
 /* get rss redirection write enable */
-u32 rpf_rss_redir_wr_en_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_rpf_rss_redir_wr_en_get(struct aq_hw_s *aq_hw);
 
 /* set rss redirection write enable */
-void rpf_rss_redir_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_redir_wr_en);
+void hw_atl_rpf_rss_redir_wr_en_set(struct aq_hw_s *aq_hw, u32 rss_redir_wr_en);
 
 /* set tpo to rpf system loopback */
-void rpf_tpo_to_rpf_sys_lbk_set(struct aq_hw_s *aq_hw,
-				u32 tpo_to_rpf_sys_lbk);
+void hw_atl_rpf_tpo_to_rpf_sys_lbk_set(struct aq_hw_s *aq_hw,
+				       u32 tpo_to_rpf_sys_lbk);
 
 /* set vlan inner ethertype */
-void rpf_vlan_inner_etht_set(struct aq_hw_s *aq_hw, u32 vlan_inner_etht);
+void hw_atl_rpf_vlan_inner_etht_set(struct aq_hw_s *aq_hw, u32 vlan_inner_etht);
 
 /* set vlan outer ethertype */
-void rpf_vlan_outer_etht_set(struct aq_hw_s *aq_hw, u32 vlan_outer_etht);
+void hw_atl_rpf_vlan_outer_etht_set(struct aq_hw_s *aq_hw, u32 vlan_outer_etht);
 
 /* set vlan promiscuous mode enable */
-void rpf_vlan_prom_mode_en_set(struct aq_hw_s *aq_hw, u32 vlan_prom_mode_en);
+void hw_atl_rpf_vlan_prom_mode_en_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_prom_mode_en);
 
 /* Set VLAN untagged action */
-void rpf_vlan_untagged_act_set(struct aq_hw_s *aq_hw, u32 vlan_untagged_act);
+void hw_atl_rpf_vlan_untagged_act_set(struct aq_hw_s *aq_hw,
+				      u32 vlan_untagged_act);
 
 /* Set VLAN accept untagged packets */
-void rpf_vlan_accept_untagged_packets_set(struct aq_hw_s *aq_hw,
-					  u32 vlan_accept_untagged_packets);
+void hw_atl_rpf_vlan_accept_untagged_packets_set(struct aq_hw_s *aq_hw,
+						 u32 vlan_acc_untagged_packets);
 
 /* Set VLAN filter enable */
-void rpf_vlan_flr_en_set(struct aq_hw_s *aq_hw, u32 vlan_flr_en, u32 filter);
+void hw_atl_rpf_vlan_flr_en_set(struct aq_hw_s *aq_hw, u32 vlan_flr_en,
+				u32 filter);
 
 /* Set VLAN Filter Action */
-void rpf_vlan_flr_act_set(struct aq_hw_s *aq_hw, u32 vlan_filter_act,
-			  u32 filter);
+void hw_atl_rpf_vlan_flr_act_set(struct aq_hw_s *aq_hw, u32 vlan_filter_act,
+				 u32 filter);
 
 /* Set VLAN ID Filter */
-void rpf_vlan_id_flr_set(struct aq_hw_s *aq_hw, u32 vlan_id_flr, u32 filter);
+void hw_atl_rpf_vlan_id_flr_set(struct aq_hw_s *aq_hw, u32 vlan_id_flr,
+				u32 filter);
 
 /* set ethertype filter enable */
-void rpf_etht_flr_en_set(struct aq_hw_s *aq_hw, u32 etht_flr_en, u32 filter);
+void hw_atl_rpf_etht_flr_en_set(struct aq_hw_s *aq_hw, u32 etht_flr_en,
+				u32 filter);
 
 /* set  ethertype user-priority enable */
-void rpf_etht_user_priority_en_set(struct aq_hw_s *aq_hw,
-				   u32 etht_user_priority_en, u32 filter);
+void hw_atl_rpf_etht_user_priority_en_set(struct aq_hw_s *aq_hw,
+					  u32 etht_user_priority_en,
+					  u32 filter);
 
 /* set  ethertype rx queue enable */
-void rpf_etht_rx_queue_en_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue_en,
-			      u32 filter);
+void hw_atl_rpf_etht_rx_queue_en_set(struct aq_hw_s *aq_hw,
+				     u32 etht_rx_queue_en,
+				     u32 filter);
 
 /* set ethertype rx queue */
-void rpf_etht_rx_queue_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue,
-			   u32 filter);
+void hw_atl_rpf_etht_rx_queue_set(struct aq_hw_s *aq_hw, u32 etht_rx_queue,
+				  u32 filter);
 
 /* set ethertype user-priority */
-void rpf_etht_user_priority_set(struct aq_hw_s *aq_hw, u32 etht_user_priority,
-				u32 filter);
+void hw_atl_rpf_etht_user_priority_set(struct aq_hw_s *aq_hw,
+				       u32 etht_user_priority,
+				       u32 filter);
 
 /* set ethertype management queue */
-void rpf_etht_mgt_queue_set(struct aq_hw_s *aq_hw, u32 etht_mgt_queue,
-			    u32 filter);
+void hw_atl_rpf_etht_mgt_queue_set(struct aq_hw_s *aq_hw, u32 etht_mgt_queue,
+				   u32 filter);
 
 /* set ethertype filter action */
-void rpf_etht_flr_act_set(struct aq_hw_s *aq_hw, u32 etht_flr_act,
-			  u32 filter);
+void hw_atl_rpf_etht_flr_act_set(struct aq_hw_s *aq_hw, u32 etht_flr_act,
+				 u32 filter);
 
 /* set ethertype filter */
-void rpf_etht_flr_set(struct aq_hw_s *aq_hw, u32 etht_flr, u32 filter);
+void hw_atl_rpf_etht_flr_set(struct aq_hw_s *aq_hw, u32 etht_flr, u32 filter);
 
 /* rpo */
 
 /* set ipv4 header checksum offload enable */
-void rpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				       u32 ipv4header_crc_offload_en);
+void hw_atl_rpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en);
 
 /* set rx descriptor vlan stripping */
-void rpo_rx_desc_vlan_stripping_set(struct aq_hw_s *aq_hw,
-				    u32 rx_desc_vlan_stripping,
-				    u32 descriptor);
+void hw_atl_rpo_rx_desc_vlan_stripping_set(struct aq_hw_s *aq_hw,
+					   u32 rx_desc_vlan_stripping,
+					   u32 descriptor);
 
 /* set tcp/udp checksum offload enable */
-void rpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				    u32 tcp_udp_crc_offload_en);
+void hw_atl_rpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en);
 
 /* Set LRO Patch Optimization Enable. */
-void rpo_lro_patch_optimization_en_set(struct aq_hw_s *aq_hw,
-				       u32 lro_patch_optimization_en);
+void hw_atl_rpo_lro_patch_optimization_en_set(struct aq_hw_s *aq_hw,
+					      u32 lro_patch_optimization_en);
 
 /* Set Large Receive Offload Enable */
-void rpo_lro_en_set(struct aq_hw_s *aq_hw, u32 lro_en);
+void hw_atl_rpo_lro_en_set(struct aq_hw_s *aq_hw, u32 lro_en);
 
 /* Set LRO Q Sessions Limit */
-void rpo_lro_qsessions_lim_set(struct aq_hw_s *aq_hw, u32 lro_qsessions_lim);
+void hw_atl_rpo_lro_qsessions_lim_set(struct aq_hw_s *aq_hw,
+				      u32 lro_qsessions_lim);
 
 /* Set LRO Total Descriptor Limit */
-void rpo_lro_total_desc_lim_set(struct aq_hw_s *aq_hw, u32 lro_total_desc_lim);
+void hw_atl_rpo_lro_total_desc_lim_set(struct aq_hw_s *aq_hw,
+				       u32 lro_total_desc_lim);
 
 /* Set LRO Min Payload of First Packet */
-void rpo_lro_min_pay_of_first_pkt_set(struct aq_hw_s *aq_hw,
-				      u32 lro_min_pld_of_first_pkt);
+void hw_atl_rpo_lro_min_pay_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lro_min_pld_of_first_pkt);
 
 /* Set LRO Packet Limit */
-void rpo_lro_pkt_lim_set(struct aq_hw_s *aq_hw, u32 lro_packet_lim);
+void hw_atl_rpo_lro_pkt_lim_set(struct aq_hw_s *aq_hw, u32 lro_packet_lim);
 
 /* Set LRO Max Number of Descriptors */
-void rpo_lro_max_num_of_descriptors_set(struct aq_hw_s *aq_hw,
-					u32 lro_max_desc_num, u32 lro);
+void hw_atl_rpo_lro_max_num_of_descriptors_set(struct aq_hw_s *aq_hw,
+					       u32 lro_max_desc_num, u32 lro);
 
 /* Set LRO Time Base Divider */
-void rpo_lro_time_base_divider_set(struct aq_hw_s *aq_hw,
-				   u32 lro_time_base_divider);
+void hw_atl_rpo_lro_time_base_divider_set(struct aq_hw_s *aq_hw,
+					  u32 lro_time_base_divider);
 
 /*Set LRO Inactive Interval */
-void rpo_lro_inactive_interval_set(struct aq_hw_s *aq_hw,
-				   u32 lro_inactive_interval);
+void hw_atl_rpo_lro_inactive_interval_set(struct aq_hw_s *aq_hw,
+					  u32 lro_inactive_interval);
 
 /*Set LRO Max Coalescing Interval */
-void rpo_lro_max_coalescing_interval_set(struct aq_hw_s *aq_hw,
-					 u32 lro_max_coalescing_interval);
+void hw_atl_rpo_lro_max_coalescing_interval_set(struct aq_hw_s *aq_hw,
+						u32 lro_max_coal_interval);
 
 /* rx */
 
 /* set rx register reset disable */
-void rx_rx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 rx_reg_res_dis);
+void hw_atl_rx_rx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 rx_reg_res_dis);
 
 /* tdm */
 
 /* set cpu id */
-void tdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca);
+void hw_atl_tdm_cpu_id_set(struct aq_hw_s *aq_hw, u32 cpuid, u32 dca);
 
 /* set large send offload enable */
-void tdm_large_send_offload_en_set(struct aq_hw_s *aq_hw,
-				   u32 large_send_offload_en);
+void hw_atl_tdm_large_send_offload_en_set(struct aq_hw_s *aq_hw,
+					  u32 large_send_offload_en);
 
 /* set tx descriptor enable */
-void tdm_tx_desc_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_en, u32 descriptor);
+void hw_atl_tdm_tx_desc_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_en,
+			       u32 descriptor);
 
 /* set tx dca enable */
-void tdm_tx_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_dca_en);
+void hw_atl_tdm_tx_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_dca_en);
 
 /* set tx dca mode */
-void tdm_tx_dca_mode_set(struct aq_hw_s *aq_hw, u32 tx_dca_mode);
+void hw_atl_tdm_tx_dca_mode_set(struct aq_hw_s *aq_hw, u32 tx_dca_mode);
 
 /* set tx descriptor dca enable */
-void tdm_tx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_dca_en, u32 dca);
+void hw_atl_tdm_tx_desc_dca_en_set(struct aq_hw_s *aq_hw, u32 tx_desc_dca_en,
+				   u32 dca);
 
 /* get tx descriptor head pointer */
-u32 tdm_tx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor);
+u32 hw_atl_tdm_tx_desc_head_ptr_get(struct aq_hw_s *aq_hw, u32 descriptor);
 
 /* set tx descriptor length */
-void tdm_tx_desc_len_set(struct aq_hw_s *aq_hw, u32 tx_desc_len,
-			 u32 descriptor);
+void hw_atl_tdm_tx_desc_len_set(struct aq_hw_s *aq_hw, u32 tx_desc_len,
+				u32 descriptor);
 
 /* set tx descriptor write-back interrupt enable */
-void tdm_tx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
-				  u32 tx_desc_wr_wb_irq_en);
+void hw_atl_tdm_tx_desc_wr_wb_irq_en_set(struct aq_hw_s *aq_hw,
+					 u32 tx_desc_wr_wb_irq_en);
 
 /* set tx descriptor write-back threshold */
-void tdm_tx_desc_wr_wb_threshold_set(struct aq_hw_s *aq_hw,
-				     u32 tx_desc_wr_wb_threshold,
+void hw_atl_tdm_tx_desc_wr_wb_threshold_set(struct aq_hw_s *aq_hw,
+					    u32 tx_desc_wr_wb_threshold,
 				     u32 descriptor);
 
 /* Set TDM Interrupt Moderation Enable */
-void tdm_tdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
-			       u32 tdm_irq_moderation_en);
+void hw_atl_tdm_tdm_intr_moder_en_set(struct aq_hw_s *aq_hw,
+				      u32 tdm_irq_moderation_en);
 /* thm */
 
 /* set lso tcp flag of first packet */
-void thm_lso_tcp_flag_of_first_pkt_set(struct aq_hw_s *aq_hw,
-				       u32 lso_tcp_flag_of_first_pkt);
+void hw_atl_thm_lso_tcp_flag_of_first_pkt_set(struct aq_hw_s *aq_hw,
+					      u32 lso_tcp_flag_of_first_pkt);
 
 /* set lso tcp flag of last packet */
-void thm_lso_tcp_flag_of_last_pkt_set(struct aq_hw_s *aq_hw,
-				      u32 lso_tcp_flag_of_last_pkt);
+void hw_atl_thm_lso_tcp_flag_of_last_pkt_set(struct aq_hw_s *aq_hw,
+					     u32 lso_tcp_flag_of_last_pkt);
 
 /* set lso tcp flag of middle packet */
-void thm_lso_tcp_flag_of_middle_pkt_set(struct aq_hw_s *aq_hw,
-					u32 lso_tcp_flag_of_middle_pkt);
+void hw_atl_thm_lso_tcp_flag_of_middle_pkt_set(struct aq_hw_s *aq_hw,
+					       u32 lso_tcp_flag_of_middle_pkt);
 
 /* tpb */
 
 /* set tx buffer enable */
-void tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en);
+void hw_atl_tpb_tx_buff_en_set(struct aq_hw_s *aq_hw, u32 tx_buff_en);
 
 /* set tx buffer high threshold (per tc) */
-void tpb_tx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 tx_buff_hi_threshold_per_tc,
+void hw_atl_tpb_tx_buff_hi_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_hi_threshold_per_tc,
 					 u32 buffer);
 
 /* set tx buffer low threshold (per tc) */
-void tpb_tx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
-					 u32 tx_buff_lo_threshold_per_tc,
+void hw_atl_tpb_tx_buff_lo_threshold_per_tc_set(struct aq_hw_s *aq_hw,
+						u32 tx_buff_lo_threshold_per_tc,
 					 u32 buffer);
 
 /* set tx dma system loopback enable */
-void tpb_tx_dma_sys_lbk_en_set(struct aq_hw_s *aq_hw, u32 tx_dma_sys_lbk_en);
+void hw_atl_tpb_tx_dma_sys_lbk_en_set(struct aq_hw_s *aq_hw, u32 tx_dma_sys_lbk_en);
 
 /* set tx packet buffer size (per tc) */
-void tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
-				     u32 tx_pkt_buff_size_per_tc, u32 buffer);
+void hw_atl_tpb_tx_pkt_buff_size_per_tc_set(struct aq_hw_s *aq_hw,
+					    u32 tx_pkt_buff_size_per_tc, u32 buffer);
 
 /* set tx path pad insert enable */
-void tpb_tx_path_scp_ins_en_set(struct aq_hw_s *aq_hw, u32 tx_path_scp_ins_en);
+void hw_atl_tpb_tx_path_scp_ins_en_set(struct aq_hw_s *aq_hw, u32 tx_path_scp_ins_en);
 
 /* tpo */
 
 /* set ipv4 header checksum offload enable */
-void tpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				       u32 ipv4header_crc_offload_en);
+void hw_atl_tpo_ipv4header_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					      u32 ipv4header_crc_offload_en);
 
 /* set tcp/udp checksum offload enable */
-void tpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
-				    u32 tcp_udp_crc_offload_en);
+void hw_atl_tpo_tcp_udp_crc_offload_en_set(struct aq_hw_s *aq_hw,
+					   u32 tcp_udp_crc_offload_en);
 
 /* set tx pkt system loopback enable */
-void tpo_tx_pkt_sys_lbk_en_set(struct aq_hw_s *aq_hw, u32 tx_pkt_sys_lbk_en);
+void hw_atl_tpo_tx_pkt_sys_lbk_en_set(struct aq_hw_s *aq_hw,
+				      u32 tx_pkt_sys_lbk_en);
 
 /* tps */
 
 /* set tx packet scheduler data arbitration mode */
-void tps_tx_pkt_shed_data_arb_mode_set(struct aq_hw_s *aq_hw,
-				       u32 tx_pkt_shed_data_arb_mode);
+void hw_atl_tps_tx_pkt_shed_data_arb_mode_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_data_arb_mode);
 
 /* set tx packet scheduler descriptor rate current time reset */
-void tps_tx_pkt_shed_desc_rate_curr_time_res_set(struct aq_hw_s *aq_hw,
-						 u32 curr_time_res);
+void hw_atl_tps_tx_pkt_shed_desc_rate_curr_time_res_set(struct aq_hw_s *aq_hw,
+							u32 curr_time_res);
 
 /* set tx packet scheduler descriptor rate limit */
-void tps_tx_pkt_shed_desc_rate_lim_set(struct aq_hw_s *aq_hw,
-				       u32 tx_pkt_shed_desc_rate_lim);
+void hw_atl_tps_tx_pkt_shed_desc_rate_lim_set(struct aq_hw_s *aq_hw,
+					      u32 tx_pkt_shed_desc_rate_lim);
 
 /* set tx packet scheduler descriptor tc arbitration mode */
-void tps_tx_pkt_shed_desc_tc_arb_mode_set(struct aq_hw_s *aq_hw,
-					  u32 tx_pkt_shed_desc_tc_arb_mode);
+void hw_atl_tps_tx_pkt_shed_desc_tc_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode);
 
 /* set tx packet scheduler descriptor tc max credit */
-void tps_tx_pkt_shed_desc_tc_max_credit_set(struct aq_hw_s *aq_hw,
-					    u32 tx_pkt_shed_desc_tc_max_credit,
+void hw_atl_tps_tx_pkt_shed_desc_tc_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
 					    u32 tc);
 
 /* set tx packet scheduler descriptor tc weight */
-void tps_tx_pkt_shed_desc_tc_weight_set(struct aq_hw_s *aq_hw,
-					u32 tx_pkt_shed_desc_tc_weight,
+void hw_atl_tps_tx_pkt_shed_desc_tc_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_desc_tc_weight,
 					u32 tc);
 
 /* set tx packet scheduler descriptor vm arbitration mode */
-void tps_tx_pkt_shed_desc_vm_arb_mode_set(struct aq_hw_s *aq_hw,
-					  u32 tx_pkt_shed_desc_vm_arb_mode);
+void hw_atl_tps_tx_pkt_shed_desc_vm_arb_mode_set(struct aq_hw_s *aq_hw,
+						 u32 arb_mode);
 
 /* set tx packet scheduler tc data max credit */
-void tps_tx_pkt_shed_tc_data_max_credit_set(struct aq_hw_s *aq_hw,
-					    u32 tx_pkt_shed_tc_data_max_credit,
+void hw_atl_tps_tx_pkt_shed_tc_data_max_credit_set(struct aq_hw_s *aq_hw,
+						   u32 max_credit,
 					    u32 tc);
 
 /* set tx packet scheduler tc data weight */
-void tps_tx_pkt_shed_tc_data_weight_set(struct aq_hw_s *aq_hw,
-					u32 tx_pkt_shed_tc_data_weight,
+void hw_atl_tps_tx_pkt_shed_tc_data_weight_set(struct aq_hw_s *aq_hw,
+					       u32 tx_pkt_shed_tc_data_weight,
 					u32 tc);
 
 /* tx */
 
 /* set tx register reset disable */
-void tx_tx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 tx_reg_res_dis);
+void hw_atl_tx_tx_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 tx_reg_res_dis);
 
 /* msm */
 
 /* get register access status */
-u32 msm_reg_access_status_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_msm_reg_access_status_get(struct aq_hw_s *aq_hw);
 
 /* set  register address for indirect address */
-void msm_reg_addr_for_indirect_addr_set(struct aq_hw_s *aq_hw,
-					u32 reg_addr_for_indirect_addr);
+void hw_atl_msm_reg_addr_for_indirect_addr_set(struct aq_hw_s *aq_hw,
+					       u32 reg_addr_for_indirect_addr);
 
 /* set register read strobe */
-void msm_reg_rd_strobe_set(struct aq_hw_s *aq_hw, u32 reg_rd_strobe);
+void hw_atl_msm_reg_rd_strobe_set(struct aq_hw_s *aq_hw, u32 reg_rd_strobe);
 
 /* get  register read data */
-u32 msm_reg_rd_data_get(struct aq_hw_s *aq_hw);
+u32 hw_atl_msm_reg_rd_data_get(struct aq_hw_s *aq_hw);
 
 /* set  register write data */
-void msm_reg_wr_data_set(struct aq_hw_s *aq_hw, u32 reg_wr_data);
+void hw_atl_msm_reg_wr_data_set(struct aq_hw_s *aq_hw, u32 reg_wr_data);
 
 /* set register write strobe */
-void msm_reg_wr_strobe_set(struct aq_hw_s *aq_hw, u32 reg_wr_strobe);
+void hw_atl_msm_reg_wr_strobe_set(struct aq_hw_s *aq_hw, u32 reg_wr_strobe);
 
 /* pci */
 
 /* set pci register reset disable */
-void pci_pci_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 pci_reg_res_dis);
+void hw_atl_pci_pci_reg_res_dis_set(struct aq_hw_s *aq_hw, u32 pci_reg_res_dis);
 
 #endif /* HW_ATL_LLH_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h
index 93450ec930e8..e0cf70120f1d 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_llh_internal.h
@@ -18,91 +18,91 @@
  * base address: 0x000003a0
  * parameter: semaphore {s} | stride size 0x4 | range [0, 15]
  */
-#define glb_cpu_sem_adr(semaphore)  (0x000003a0u + (semaphore) * 0x4)
+#define HW_ATL_GLB_CPU_SEM_ADR(semaphore)  (0x000003a0u + (semaphore) * 0x4)
 /* register address for bitfield rx dma good octet counter lsw [1f:0] */
-#define stats_rx_dma_good_octet_counterlsw__adr 0x00006808
+#define HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERLSW 0x00006808
 /* register address for bitfield rx dma good packet counter lsw [1f:0] */
-#define stats_rx_dma_good_pkt_counterlsw__adr 0x00006800
+#define HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERLSW 0x00006800
 /* register address for bitfield tx dma good octet counter lsw [1f:0] */
-#define stats_tx_dma_good_octet_counterlsw__adr 0x00008808
+#define HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERLSW 0x00008808
 /* register address for bitfield tx dma good packet counter lsw [1f:0] */
-#define stats_tx_dma_good_pkt_counterlsw__adr 0x00008800
+#define HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERLSW 0x00008800
 
 /* register address for bitfield rx dma good octet counter msw [3f:20] */
-#define stats_rx_dma_good_octet_countermsw__adr 0x0000680c
+#define HW_ATL_STATS_RX_DMA_GOOD_OCTET_COUNTERMSW 0x0000680c
 /* register address for bitfield rx dma good packet counter msw [3f:20] */
-#define stats_rx_dma_good_pkt_countermsw__adr 0x00006804
+#define HW_ATL_STATS_RX_DMA_GOOD_PKT_COUNTERMSW 0x00006804
 /* register address for bitfield tx dma good octet counter msw [3f:20] */
-#define stats_tx_dma_good_octet_countermsw__adr 0x0000880c
+#define HW_ATL_STATS_TX_DMA_GOOD_OCTET_COUNTERMSW 0x0000880c
 /* register address for bitfield tx dma good packet counter msw [3f:20] */
-#define stats_tx_dma_good_pkt_countermsw__adr 0x00008804
+#define HW_ATL_STATS_TX_DMA_GOOD_PKT_COUNTERMSW 0x00008804
 
 /* preprocessor definitions for msm rx errors counter register */
-#define mac_msm_rx_errs_cnt_adr 0x00000120u
+#define HW_ATL_MAC_MSM_RX_ERRS_CNT_ADR 0x00000120u
 
 /* preprocessor definitions for msm rx unicast frames counter register */
-#define mac_msm_rx_ucst_frm_cnt_adr 0x000000e0u
+#define HW_ATL_MAC_MSM_RX_UCST_FRM_CNT_ADR 0x000000e0u
 
 /* preprocessor definitions for msm rx multicast frames counter register */
-#define mac_msm_rx_mcst_frm_cnt_adr 0x000000e8u
+#define HW_ATL_MAC_MSM_RX_MCST_FRM_CNT_ADR 0x000000e8u
 
 /* preprocessor definitions for msm rx broadcast frames counter register */
-#define mac_msm_rx_bcst_frm_cnt_adr 0x000000f0u
+#define HW_ATL_MAC_MSM_RX_BCST_FRM_CNT_ADR 0x000000f0u
 
 /* preprocessor definitions for msm rx broadcast octets counter register 1 */
-#define mac_msm_rx_bcst_octets_counter1_adr 0x000001b0u
+#define HW_ATL_MAC_MSM_RX_BCST_OCTETS_COUNTER1_ADR 0x000001b0u
 
 /* preprocessor definitions for msm rx broadcast octets counter register 2 */
-#define mac_msm_rx_bcst_octets_counter2_adr 0x000001b4u
+#define HW_ATL_MAC_MSM_RX_BCST_OCTETS_COUNTER2_ADR 0x000001b4u
 
 /* preprocessor definitions for msm rx unicast octets counter register 0 */
-#define mac_msm_rx_ucst_octets_counter0_adr 0x000001b8u
+#define HW_ATL_MAC_MSM_RX_UCST_OCTETS_COUNTER0_ADR 0x000001b8u
 
 /* preprocessor definitions for rx dma statistics counter 7 */
-#define rx_dma_stat_counter7_adr 0x00006818u
+#define HW_ATL_RX_DMA_STAT_COUNTER7_ADR 0x00006818u
 
 /* preprocessor definitions for msm tx unicast frames counter register */
-#define mac_msm_tx_ucst_frm_cnt_adr 0x00000108u
+#define HW_ATL_MAC_MSM_TX_UCST_FRM_CNT_ADR 0x00000108u
 
 /* preprocessor definitions for msm tx multicast frames counter register */
-#define mac_msm_tx_mcst_frm_cnt_adr 0x00000110u
+#define HW_ATL_MAC_MSM_TX_MCST_FRM_CNT_ADR 0x00000110u
 
 /* preprocessor definitions for global mif identification */
-#define glb_mif_id_adr 0x0000001cu
+#define HW_ATL_GLB_MIF_ID_ADR 0x0000001cu
 
 /* register address for bitfield iamr_lsw[1f:0] */
-#define itr_iamrlsw_adr 0x00002090
+#define HW_ATL_ITR_IAMRLSW_ADR 0x00002090
 /* register address for bitfield rx dma drop packet counter [1f:0] */
-#define rpb_rx_dma_drop_pkt_cnt_adr 0x00006818
+#define HW_ATL_RPB_RX_DMA_DROP_PKT_CNT_ADR 0x00006818
 
 /* register address for bitfield imcr_lsw[1f:0] */
-#define itr_imcrlsw_adr 0x00002070
+#define HW_ATL_ITR_IMCRLSW_ADR 0x00002070
 /* register address for bitfield imsr_lsw[1f:0] */
-#define itr_imsrlsw_adr 0x00002060
+#define HW_ATL_ITR_IMSRLSW_ADR 0x00002060
 /* register address for bitfield itr_reg_res_dsbl */
-#define itr_reg_res_dsbl_adr 0x00002300
+#define HW_ATL_ITR_REG_RES_DSBL_ADR 0x00002300
 /* bitmask for bitfield itr_reg_res_dsbl */
-#define itr_reg_res_dsbl_msk 0x20000000
+#define HW_ATL_ITR_REG_RES_DSBL_MSK 0x20000000
 /* lower bit position of bitfield itr_reg_res_dsbl */
-#define itr_reg_res_dsbl_shift 29
+#define HW_ATL_ITR_REG_RES_DSBL_SHIFT 29
 /* register address for bitfield iscr_lsw[1f:0] */
-#define itr_iscrlsw_adr 0x00002050
+#define HW_ATL_ITR_ISCRLSW_ADR 0x00002050
 /* register address for bitfield isr_lsw[1f:0] */
-#define itr_isrlsw_adr 0x00002000
+#define HW_ATL_ITR_ISRLSW_ADR 0x00002000
 /* register address for bitfield itr_reset */
-#define itr_res_adr 0x00002300
+#define HW_ATL_ITR_RES_ADR 0x00002300
 /* bitmask for bitfield itr_reset */
-#define itr_res_msk 0x80000000
+#define HW_ATL_ITR_RES_MSK 0x80000000
 /* lower bit position of bitfield itr_reset */
-#define itr_res_shift 31
+#define HW_ATL_ITR_RES_SHIFT 31
 /* register address for bitfield dca{d}_cpuid[7:0] */
-#define rdm_dcadcpuid_adr(dca) (0x00006100 + (dca) * 0x4)
+#define HW_ATL_RDM_DCADCPUID_ADR(dca) (0x00006100 + (dca) * 0x4)
 /* bitmask for bitfield dca{d}_cpuid[7:0] */
-#define rdm_dcadcpuid_msk 0x000000ff
+#define HW_ATL_RDM_DCADCPUID_MSK 0x000000ff
 /* lower bit position of bitfield dca{d}_cpuid[7:0] */
-#define rdm_dcadcpuid_shift 0
+#define HW_ATL_RDM_DCADCPUID_SHIFT 0
 /* register address for bitfield dca_en */
-#define rdm_dca_en_adr 0x00006180
+#define HW_ATL_RDM_DCA_EN_ADR 0x00006180
 
 /* rx dca_en bitfield definitions
  * preprocessor definitions for the bitfield "dca_en".
@@ -110,17 +110,17 @@
  */
 
 /* register address for bitfield dca_en */
-#define rdm_dca_en_adr 0x00006180
+#define HW_ATL_RDM_DCA_EN_ADR 0x00006180
 /* bitmask for bitfield dca_en */
-#define rdm_dca_en_msk 0x80000000
+#define HW_ATL_RDM_DCA_EN_MSK 0x80000000
 /* inverted bitmask for bitfield dca_en */
-#define rdm_dca_en_mskn 0x7fffffff
+#define HW_ATL_RDM_DCA_EN_MSKN 0x7fffffff
 /* lower bit position of bitfield dca_en */
-#define rdm_dca_en_shift 31
+#define HW_ATL_RDM_DCA_EN_SHIFT 31
 /* width of bitfield dca_en */
-#define rdm_dca_en_width 1
+#define HW_ATL_RDM_DCA_EN_WIDTH 1
 /* default value of bitfield dca_en */
-#define rdm_dca_en_default 0x1
+#define HW_ATL_RDM_DCA_EN_DEFAULT 0x1
 
 /* rx dca_mode[3:0] bitfield definitions
  * preprocessor definitions for the bitfield "dca_mode[3:0]".
@@ -128,17 +128,17 @@
  */
 
 /* register address for bitfield dca_mode[3:0] */
-#define rdm_dca_mode_adr 0x00006180
+#define HW_ATL_RDM_DCA_MODE_ADR 0x00006180
 /* bitmask for bitfield dca_mode[3:0] */
-#define rdm_dca_mode_msk 0x0000000f
+#define HW_ATL_RDM_DCA_MODE_MSK 0x0000000f
 /* inverted bitmask for bitfield dca_mode[3:0] */
-#define rdm_dca_mode_mskn 0xfffffff0
+#define HW_ATL_RDM_DCA_MODE_MSKN 0xfffffff0
 /* lower bit position of bitfield dca_mode[3:0] */
-#define rdm_dca_mode_shift 0
+#define HW_ATL_RDM_DCA_MODE_SHIFT 0
 /* width of bitfield dca_mode[3:0] */
-#define rdm_dca_mode_width 4
+#define HW_ATL_RDM_DCA_MODE_WIDTH 4
 /* default value of bitfield dca_mode[3:0] */
-#define rdm_dca_mode_default 0x0
+#define HW_ATL_RDM_DCA_MODE_DEFAULT 0x0
 
 /* rx desc{d}_data_size[4:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_data_size[4:0]".
@@ -147,17 +147,18 @@
  */
 
 /* register address for bitfield desc{d}_data_size[4:0] */
-#define rdm_descddata_size_adr(descriptor) (0x00005b18 + (descriptor) * 0x20)
+#define HW_ATL_RDM_DESCDDATA_SIZE_ADR(descriptor) \
+	(0x00005b18 + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_data_size[4:0] */
-#define rdm_descddata_size_msk 0x0000001f
+#define HW_ATL_RDM_DESCDDATA_SIZE_MSK 0x0000001f
 /* inverted bitmask for bitfield desc{d}_data_size[4:0] */
-#define rdm_descddata_size_mskn 0xffffffe0
+#define HW_ATL_RDM_DESCDDATA_SIZE_MSKN 0xffffffe0
 /* lower bit position of bitfield desc{d}_data_size[4:0] */
-#define rdm_descddata_size_shift 0
+#define HW_ATL_RDM_DESCDDATA_SIZE_SHIFT 0
 /* width of bitfield desc{d}_data_size[4:0] */
-#define rdm_descddata_size_width 5
+#define HW_ATL_RDM_DESCDDATA_SIZE_WIDTH 5
 /* default value of bitfield desc{d}_data_size[4:0] */
-#define rdm_descddata_size_default 0x0
+#define HW_ATL_RDM_DESCDDATA_SIZE_DEFAULT 0x0
 
 /* rx dca{d}_desc_en bitfield definitions
  * preprocessor definitions for the bitfield "dca{d}_desc_en".
@@ -166,17 +167,17 @@
  */
 
 /* register address for bitfield dca{d}_desc_en */
-#define rdm_dcaddesc_en_adr(dca) (0x00006100 + (dca) * 0x4)
+#define HW_ATL_RDM_DCADDESC_EN_ADR(dca) (0x00006100 + (dca) * 0x4)
 /* bitmask for bitfield dca{d}_desc_en */
-#define rdm_dcaddesc_en_msk 0x80000000
+#define HW_ATL_RDM_DCADDESC_EN_MSK 0x80000000
 /* inverted bitmask for bitfield dca{d}_desc_en */
-#define rdm_dcaddesc_en_mskn 0x7fffffff
+#define HW_ATL_RDM_DCADDESC_EN_MSKN 0x7fffffff
 /* lower bit position of bitfield dca{d}_desc_en */
-#define rdm_dcaddesc_en_shift 31
+#define HW_ATL_RDM_DCADDESC_EN_SHIFT 31
 /* width of bitfield dca{d}_desc_en */
-#define rdm_dcaddesc_en_width 1
+#define HW_ATL_RDM_DCADDESC_EN_WIDTH 1
 /* default value of bitfield dca{d}_desc_en */
-#define rdm_dcaddesc_en_default 0x0
+#define HW_ATL_RDM_DCADDESC_EN_DEFAULT 0x0
 
 /* rx desc{d}_en bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_en".
@@ -185,17 +186,17 @@
  */
 
 /* register address for bitfield desc{d}_en */
-#define rdm_descden_adr(descriptor) (0x00005b08 + (descriptor) * 0x20)
+#define HW_ATL_RDM_DESCDEN_ADR(descriptor) (0x00005b08 + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_en */
-#define rdm_descden_msk 0x80000000
+#define HW_ATL_RDM_DESCDEN_MSK 0x80000000
 /* inverted bitmask for bitfield desc{d}_en */
-#define rdm_descden_mskn 0x7fffffff
+#define HW_ATL_RDM_DESCDEN_MSKN 0x7fffffff
 /* lower bit position of bitfield desc{d}_en */
-#define rdm_descden_shift 31
+#define HW_ATL_RDM_DESCDEN_SHIFT 31
 /* width of bitfield desc{d}_en */
-#define rdm_descden_width 1
+#define HW_ATL_RDM_DESCDEN_WIDTH 1
 /* default value of bitfield desc{d}_en */
-#define rdm_descden_default 0x0
+#define HW_ATL_RDM_DESCDEN_DEFAULT 0x0
 
 /* rx desc{d}_hdr_size[4:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_hdr_size[4:0]".
@@ -204,17 +205,18 @@
  */
 
 /* register address for bitfield desc{d}_hdr_size[4:0] */
-#define rdm_descdhdr_size_adr(descriptor) (0x00005b18 + (descriptor) * 0x20)
+#define HW_ATL_RDM_DESCDHDR_SIZE_ADR(descriptor) \
+	(0x00005b18 + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_hdr_size[4:0] */
-#define rdm_descdhdr_size_msk 0x00001f00
+#define HW_ATL_RDM_DESCDHDR_SIZE_MSK 0x00001f00
 /* inverted bitmask for bitfield desc{d}_hdr_size[4:0] */
-#define rdm_descdhdr_size_mskn 0xffffe0ff
+#define HW_ATL_RDM_DESCDHDR_SIZE_MSKN 0xffffe0ff
 /* lower bit position of bitfield desc{d}_hdr_size[4:0] */
-#define rdm_descdhdr_size_shift 8
+#define HW_ATL_RDM_DESCDHDR_SIZE_SHIFT 8
 /* width of bitfield desc{d}_hdr_size[4:0] */
-#define rdm_descdhdr_size_width 5
+#define HW_ATL_RDM_DESCDHDR_SIZE_WIDTH 5
 /* default value of bitfield desc{d}_hdr_size[4:0] */
-#define rdm_descdhdr_size_default 0x0
+#define HW_ATL_RDM_DESCDHDR_SIZE_DEFAULT 0x0
 
 /* rx desc{d}_hdr_split bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_hdr_split".
@@ -223,17 +225,18 @@
  */
 
 /* register address for bitfield desc{d}_hdr_split */
-#define rdm_descdhdr_split_adr(descriptor) (0x00005b08 + (descriptor) * 0x20)
+#define HW_ATL_RDM_DESCDHDR_SPLIT_ADR(descriptor) \
+	(0x00005b08 + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_hdr_split */
-#define rdm_descdhdr_split_msk 0x10000000
+#define HW_ATL_RDM_DESCDHDR_SPLIT_MSK 0x10000000
 /* inverted bitmask for bitfield desc{d}_hdr_split */
-#define rdm_descdhdr_split_mskn 0xefffffff
+#define HW_ATL_RDM_DESCDHDR_SPLIT_MSKN 0xefffffff
 /* lower bit position of bitfield desc{d}_hdr_split */
-#define rdm_descdhdr_split_shift 28
+#define HW_ATL_RDM_DESCDHDR_SPLIT_SHIFT 28
 /* width of bitfield desc{d}_hdr_split */
-#define rdm_descdhdr_split_width 1
+#define HW_ATL_RDM_DESCDHDR_SPLIT_WIDTH 1
 /* default value of bitfield desc{d}_hdr_split */
-#define rdm_descdhdr_split_default 0x0
+#define HW_ATL_RDM_DESCDHDR_SPLIT_DEFAULT 0x0
 
 /* rx desc{d}_hd[c:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_hd[c:0]".
@@ -242,15 +245,15 @@
  */
 
 /* register address for bitfield desc{d}_hd[c:0] */
-#define rdm_descdhd_adr(descriptor) (0x00005b0c + (descriptor) * 0x20)
+#define HW_ATL_RDM_DESCDHD_ADR(descriptor) (0x00005b0c + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_hd[c:0] */
-#define rdm_descdhd_msk 0x00001fff
+#define HW_ATL_RDM_DESCDHD_MSK 0x00001fff
 /* inverted bitmask for bitfield desc{d}_hd[c:0] */
-#define rdm_descdhd_mskn 0xffffe000
+#define HW_ATL_RDM_DESCDHD_MSKN 0xffffe000
 /* lower bit position of bitfield desc{d}_hd[c:0] */
-#define rdm_descdhd_shift 0
+#define HW_ATL_RDM_DESCDHD_SHIFT 0
 /* width of bitfield desc{d}_hd[c:0] */
-#define rdm_descdhd_width 13
+#define HW_ATL_RDM_DESCDHD_WIDTH 13
 
 /* rx desc{d}_len[9:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_len[9:0]".
@@ -259,17 +262,17 @@
  */
 
 /* register address for bitfield desc{d}_len[9:0] */
-#define rdm_descdlen_adr(descriptor) (0x00005b08 + (descriptor) * 0x20)
+#define HW_ATL_RDM_DESCDLEN_ADR(descriptor) (0x00005b08 + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_len[9:0] */
-#define rdm_descdlen_msk 0x00001ff8
+#define HW_ATL_RDM_DESCDLEN_MSK 0x00001ff8
 /* inverted bitmask for bitfield desc{d}_len[9:0] */
-#define rdm_descdlen_mskn 0xffffe007
+#define HW_ATL_RDM_DESCDLEN_MSKN 0xffffe007
 /* lower bit position of bitfield desc{d}_len[9:0] */
-#define rdm_descdlen_shift 3
+#define HW_ATL_RDM_DESCDLEN_SHIFT 3
 /* width of bitfield desc{d}_len[9:0] */
-#define rdm_descdlen_width 10
+#define HW_ATL_RDM_DESCDLEN_WIDTH 10
 /* default value of bitfield desc{d}_len[9:0] */
-#define rdm_descdlen_default 0x0
+#define HW_ATL_RDM_DESCDLEN_DEFAULT 0x0
 
 /* rx desc{d}_reset bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_reset".
@@ -278,17 +281,17 @@
  */
 
 /* register address for bitfield desc{d}_reset */
-#define rdm_descdreset_adr(descriptor) (0x00005b08 + (descriptor) * 0x20)
+#define HW_ATL_RDM_DESCDRESET_ADR(descriptor) (0x00005b08 + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_reset */
-#define rdm_descdreset_msk 0x02000000
+#define HW_ATL_RDM_DESCDRESET_MSK 0x02000000
 /* inverted bitmask for bitfield desc{d}_reset */
-#define rdm_descdreset_mskn 0xfdffffff
+#define HW_ATL_RDM_DESCDRESET_MSKN 0xfdffffff
 /* lower bit position of bitfield desc{d}_reset */
-#define rdm_descdreset_shift 25
+#define HW_ATL_RDM_DESCDRESET_SHIFT 25
 /* width of bitfield desc{d}_reset */
-#define rdm_descdreset_width 1
+#define HW_ATL_RDM_DESCDRESET_WIDTH 1
 /* default value of bitfield desc{d}_reset */
-#define rdm_descdreset_default 0x0
+#define HW_ATL_RDM_DESCDRESET_DEFAULT 0x0
 
 /* rx int_desc_wrb_en bitfield definitions
  * preprocessor definitions for the bitfield "int_desc_wrb_en".
@@ -296,17 +299,17 @@
  */
 
 /* register address for bitfield int_desc_wrb_en */
-#define rdm_int_desc_wrb_en_adr 0x00005a30
+#define HW_ATL_RDM_INT_DESC_WRB_EN_ADR 0x00005a30
 /* bitmask for bitfield int_desc_wrb_en */
-#define rdm_int_desc_wrb_en_msk 0x00000004
+#define HW_ATL_RDM_INT_DESC_WRB_EN_MSK 0x00000004
 /* inverted bitmask for bitfield int_desc_wrb_en */
-#define rdm_int_desc_wrb_en_mskn 0xfffffffb
+#define HW_ATL_RDM_INT_DESC_WRB_EN_MSKN 0xfffffffb
 /* lower bit position of bitfield int_desc_wrb_en */
-#define rdm_int_desc_wrb_en_shift 2
+#define HW_ATL_RDM_INT_DESC_WRB_EN_SHIFT 2
 /* width of bitfield int_desc_wrb_en */
-#define rdm_int_desc_wrb_en_width 1
+#define HW_ATL_RDM_INT_DESC_WRB_EN_WIDTH 1
 /* default value of bitfield int_desc_wrb_en */
-#define rdm_int_desc_wrb_en_default 0x0
+#define HW_ATL_RDM_INT_DESC_WRB_EN_DEFAULT 0x0
 
 /* rx dca{d}_hdr_en bitfield definitions
  * preprocessor definitions for the bitfield "dca{d}_hdr_en".
@@ -315,17 +318,17 @@
  */
 
 /* register address for bitfield dca{d}_hdr_en */
-#define rdm_dcadhdr_en_adr(dca) (0x00006100 + (dca) * 0x4)
+#define HW_ATL_RDM_DCADHDR_EN_ADR(dca) (0x00006100 + (dca) * 0x4)
 /* bitmask for bitfield dca{d}_hdr_en */
-#define rdm_dcadhdr_en_msk 0x40000000
+#define HW_ATL_RDM_DCADHDR_EN_MSK 0x40000000
 /* inverted bitmask for bitfield dca{d}_hdr_en */
-#define rdm_dcadhdr_en_mskn 0xbfffffff
+#define HW_ATL_RDM_DCADHDR_EN_MSKN 0xbfffffff
 /* lower bit position of bitfield dca{d}_hdr_en */
-#define rdm_dcadhdr_en_shift 30
+#define HW_ATL_RDM_DCADHDR_EN_SHIFT 30
 /* width of bitfield dca{d}_hdr_en */
-#define rdm_dcadhdr_en_width 1
+#define HW_ATL_RDM_DCADHDR_EN_WIDTH 1
 /* default value of bitfield dca{d}_hdr_en */
-#define rdm_dcadhdr_en_default 0x0
+#define HW_ATL_RDM_DCADHDR_EN_DEFAULT 0x0
 
 /* rx dca{d}_pay_en bitfield definitions
  * preprocessor definitions for the bitfield "dca{d}_pay_en".
@@ -334,17 +337,17 @@
  */
 
 /* register address for bitfield dca{d}_pay_en */
-#define rdm_dcadpay_en_adr(dca) (0x00006100 + (dca) * 0x4)
+#define HW_ATL_RDM_DCADPAY_EN_ADR(dca) (0x00006100 + (dca) * 0x4)
 /* bitmask for bitfield dca{d}_pay_en */
-#define rdm_dcadpay_en_msk 0x20000000
+#define HW_ATL_RDM_DCADPAY_EN_MSK 0x20000000
 /* inverted bitmask for bitfield dca{d}_pay_en */
-#define rdm_dcadpay_en_mskn 0xdfffffff
+#define HW_ATL_RDM_DCADPAY_EN_MSKN 0xdfffffff
 /* lower bit position of bitfield dca{d}_pay_en */
-#define rdm_dcadpay_en_shift 29
+#define HW_ATL_RDM_DCADPAY_EN_SHIFT 29
 /* width of bitfield dca{d}_pay_en */
-#define rdm_dcadpay_en_width 1
+#define HW_ATL_RDM_DCADPAY_EN_WIDTH 1
 /* default value of bitfield dca{d}_pay_en */
-#define rdm_dcadpay_en_default 0x0
+#define HW_ATL_RDM_DCADPAY_EN_DEFAULT 0x0
 
 /* RX rdm_int_rim_en Bitfield Definitions
  * Preprocessor definitions for the bitfield "rdm_int_rim_en".
@@ -352,51 +355,51 @@
  */
 
 /* Register address for bitfield rdm_int_rim_en */
-#define rdm_int_rim_en_adr 0x00005A30
+#define HW_ATL_RDM_INT_RIM_EN_ADR 0x00005A30
 /* Bitmask for bitfield rdm_int_rim_en */
-#define rdm_int_rim_en_msk 0x00000008
+#define HW_ATL_RDM_INT_RIM_EN_MSK 0x00000008
 /* Inverted bitmask for bitfield rdm_int_rim_en */
-#define rdm_int_rim_en_mskn 0xFFFFFFF7
+#define HW_ATL_RDM_INT_RIM_EN_MSKN 0xFFFFFFF7
 /* Lower bit position of bitfield rdm_int_rim_en */
-#define rdm_int_rim_en_shift 3
+#define HW_ATL_RDM_INT_RIM_EN_SHIFT 3
 /* Width of bitfield rdm_int_rim_en */
-#define rdm_int_rim_en_width 1
+#define HW_ATL_RDM_INT_RIM_EN_WIDTH 1
 /* Default value of bitfield rdm_int_rim_en */
-#define rdm_int_rim_en_default 0x0
+#define HW_ATL_RDM_INT_RIM_EN_DEFAULT 0x0
 
 /* general interrupt mapping register definitions
  * preprocessor definitions for general interrupt mapping register
  * base address: 0x00002180
  * parameter: regidx {f} | stride size 0x4 | range [0, 3]
  */
-#define gen_intr_map_adr(regidx) (0x00002180u + (regidx) * 0x4)
+#define HW_ATL_GEN_INTR_MAP_ADR(regidx) (0x00002180u + (regidx) * 0x4)
 
 /* general interrupt status register definitions
  * preprocessor definitions for general interrupt status register
  * address: 0x000021A0
  */
 
-#define gen_intr_stat_adr 0x000021A4U
+#define HW_ATL_GEN_INTR_STAT_ADR 0x000021A4U
 
 /* interrupt global control register  definitions
  * preprocessor definitions for interrupt global control register
  * address: 0x00002300
  */
-#define intr_glb_ctl_adr 0x00002300u
+#define HW_ATL_INTR_GLB_CTL_ADR 0x00002300u
 
 /* interrupt throttle register definitions
  * preprocessor definitions for interrupt throttle register
  * base address: 0x00002800
  * parameter: throttle {t} | stride size 0x4 | range [0, 31]
  */
-#define intr_thr_adr(throttle) (0x00002800u + (throttle) * 0x4)
+#define HW_ATL_INTR_THR_ADR(throttle) (0x00002800u + (throttle) * 0x4)
 
 /* rx dma descriptor base address lsw definitions
  * preprocessor definitions for rx dma descriptor base address lsw
  * base address: 0x00005b00
  * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
  */
-#define rx_dma_desc_base_addrlsw_adr(descriptor) \
+#define HW_ATL_RX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor) \
 (0x00005b00u + (descriptor) * 0x20)
 
 /* rx dma descriptor base address msw definitions
@@ -404,7 +407,7 @@
  * base address: 0x00005b04
  * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
  */
-#define rx_dma_desc_base_addrmsw_adr(descriptor) \
+#define HW_ATL_RX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor) \
 (0x00005b04u + (descriptor) * 0x20)
 
 /* rx dma descriptor status register definitions
@@ -412,46 +415,48 @@
  * base address: 0x00005b14
  * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
  */
-#define rx_dma_desc_stat_adr(descriptor) (0x00005b14u + (descriptor) * 0x20)
+#define HW_ATL_RX_DMA_DESC_STAT_ADR(descriptor) \
+	(0x00005b14u + (descriptor) * 0x20)
 
 /* rx dma descriptor tail pointer register definitions
  * preprocessor definitions for rx dma descriptor tail pointer register
  * base address: 0x00005b10
  * parameter: descriptor {d} | stride size 0x20 | range [0, 31]
  */
-#define rx_dma_desc_tail_ptr_adr(descriptor) (0x00005b10u + (descriptor) * 0x20)
+#define HW_ATL_RX_DMA_DESC_TAIL_PTR_ADR(descriptor) \
+	(0x00005b10u + (descriptor) * 0x20)
 
 /* rx interrupt moderation control register definitions
  * Preprocessor definitions for RX Interrupt Moderation Control Register
  * Base Address: 0x00005A40
  * Parameter: RIM {R} | stride size 0x4 | range [0, 31]
  */
-#define rx_intr_moderation_ctl_adr(rim) (0x00005A40u + (rim) * 0x4)
+#define HW_ATL_RX_INTR_MODERATION_CTL_ADR(rim) (0x00005A40u + (rim) * 0x4)
 
 /* rx filter multicast filter mask register definitions
  * preprocessor definitions for rx filter multicast filter mask register
  * address: 0x00005270
  */
-#define rx_flr_mcst_flr_msk_adr 0x00005270u
+#define HW_ATL_RX_FLR_MCST_FLR_MSK_ADR 0x00005270u
 
 /* rx filter multicast filter register definitions
  * preprocessor definitions for rx filter multicast filter register
  * base address: 0x00005250
  * parameter: filter {f} | stride size 0x4 | range [0, 7]
  */
-#define rx_flr_mcst_flr_adr(filter) (0x00005250u + (filter) * 0x4)
+#define HW_ATL_RX_FLR_MCST_FLR_ADR(filter) (0x00005250u + (filter) * 0x4)
 
 /* RX Filter RSS Control Register 1 Definitions
  * Preprocessor definitions for RX Filter RSS Control Register 1
  * Address: 0x000054C0
  */
-#define rx_flr_rss_control1_adr 0x000054C0u
+#define HW_ATL_RX_FLR_RSS_CONTROL1_ADR 0x000054C0u
 
 /* RX Filter Control Register 2 Definitions
  * Preprocessor definitions for RX Filter Control Register 2
  * Address: 0x00005104
  */
-#define rx_flr_control2_adr 0x00005104u
+#define HW_ATL_RX_FLR_CONTROL2_ADR 0x00005104u
 
 /* tx tx dma debug control [1f:0] bitfield definitions
  * preprocessor definitions for the bitfield "tx dma debug control [1f:0]".
@@ -459,24 +464,24 @@
  */
 
 /* register address for bitfield tx dma debug control [1f:0] */
-#define tdm_tx_dma_debug_ctl_adr 0x00008920
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_ADR 0x00008920
 /* bitmask for bitfield tx dma debug control [1f:0] */
-#define tdm_tx_dma_debug_ctl_msk 0xffffffff
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_MSK 0xffffffff
 /* inverted bitmask for bitfield tx dma debug control [1f:0] */
-#define tdm_tx_dma_debug_ctl_mskn 0x00000000
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_MSKN 0x00000000
 /* lower bit position of bitfield tx dma debug control [1f:0] */
-#define tdm_tx_dma_debug_ctl_shift 0
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_SHIFT 0
 /* width of bitfield tx dma debug control [1f:0] */
-#define tdm_tx_dma_debug_ctl_width 32
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_WIDTH 32
 /* default value of bitfield tx dma debug control [1f:0] */
-#define tdm_tx_dma_debug_ctl_default 0x0
+#define HW_ATL_TDM_TX_DMA_DEBUG_CTL_DEFAULT 0x0
 
 /* tx dma descriptor base address lsw definitions
  * preprocessor definitions for tx dma descriptor base address lsw
  * base address: 0x00007c00
  * parameter: descriptor {d} | stride size 0x40 | range [0, 31]
  */
-#define tx_dma_desc_base_addrlsw_adr(descriptor) \
+#define HW_ATL_TX_DMA_DESC_BASE_ADDRLSW_ADR(descriptor) \
 	(0x00007c00u + (descriptor) * 0x40)
 
 /* tx dma descriptor tail pointer register definitions
@@ -484,7 +489,8 @@
  * base address: 0x00007c10
  *  parameter: descriptor {d} | stride size 0x40 | range [0, 31]
  */
-#define tx_dma_desc_tail_ptr_adr(descriptor) (0x00007c10u + (descriptor) * 0x40)
+#define HW_ATL_TX_DMA_DESC_TAIL_PTR_ADR(descriptor) \
+	(0x00007c10u + (descriptor) * 0x40)
 
 /* rx dma_sys_loopback bitfield definitions
  * preprocessor definitions for the bitfield "dma_sys_loopback".
@@ -492,17 +498,17 @@
  */
 
 /* register address for bitfield dma_sys_loopback */
-#define rpb_dma_sys_lbk_adr 0x00005000
+#define HW_ATL_RPB_DMA_SYS_LBK_ADR 0x00005000
 /* bitmask for bitfield dma_sys_loopback */
-#define rpb_dma_sys_lbk_msk 0x00000040
+#define HW_ATL_RPB_DMA_SYS_LBK_MSK 0x00000040
 /* inverted bitmask for bitfield dma_sys_loopback */
-#define rpb_dma_sys_lbk_mskn 0xffffffbf
+#define HW_ATL_RPB_DMA_SYS_LBK_MSKN 0xffffffbf
 /* lower bit position of bitfield dma_sys_loopback */
-#define rpb_dma_sys_lbk_shift 6
+#define HW_ATL_RPB_DMA_SYS_LBK_SHIFT 6
 /* width of bitfield dma_sys_loopback */
-#define rpb_dma_sys_lbk_width 1
+#define HW_ATL_RPB_DMA_SYS_LBK_WIDTH 1
 /* default value of bitfield dma_sys_loopback */
-#define rpb_dma_sys_lbk_default 0x0
+#define HW_ATL_RPB_DMA_SYS_LBK_DEFAULT 0x0
 
 /* rx rx_tc_mode bitfield definitions
  * preprocessor definitions for the bitfield "rx_tc_mode".
@@ -510,17 +516,17 @@
  */
 
 /* register address for bitfield rx_tc_mode */
-#define rpb_rpf_rx_tc_mode_adr 0x00005700
+#define HW_ATL_RPB_RPF_RX_TC_MODE_ADR 0x00005700
 /* bitmask for bitfield rx_tc_mode */
-#define rpb_rpf_rx_tc_mode_msk 0x00000100
+#define HW_ATL_RPB_RPF_RX_TC_MODE_MSK 0x00000100
 /* inverted bitmask for bitfield rx_tc_mode */
-#define rpb_rpf_rx_tc_mode_mskn 0xfffffeff
+#define HW_ATL_RPB_RPF_RX_TC_MODE_MSKN 0xfffffeff
 /* lower bit position of bitfield rx_tc_mode */
-#define rpb_rpf_rx_tc_mode_shift 8
+#define HW_ATL_RPB_RPF_RX_TC_MODE_SHIFT 8
 /* width of bitfield rx_tc_mode */
-#define rpb_rpf_rx_tc_mode_width 1
+#define HW_ATL_RPB_RPF_RX_TC_MODE_WIDTH 1
 /* default value of bitfield rx_tc_mode */
-#define rpb_rpf_rx_tc_mode_default 0x0
+#define HW_ATL_RPB_RPF_RX_TC_MODE_DEFAULT 0x0
 
 /* rx rx_buf_en bitfield definitions
  * preprocessor definitions for the bitfield "rx_buf_en".
@@ -528,17 +534,17 @@
  */
 
 /* register address for bitfield rx_buf_en */
-#define rpb_rx_buf_en_adr 0x00005700
+#define HW_ATL_RPB_RX_BUF_EN_ADR 0x00005700
 /* bitmask for bitfield rx_buf_en */
-#define rpb_rx_buf_en_msk 0x00000001
+#define HW_ATL_RPB_RX_BUF_EN_MSK 0x00000001
 /* inverted bitmask for bitfield rx_buf_en */
-#define rpb_rx_buf_en_mskn 0xfffffffe
+#define HW_ATL_RPB_RX_BUF_EN_MSKN 0xfffffffe
 /* lower bit position of bitfield rx_buf_en */
-#define rpb_rx_buf_en_shift 0
+#define HW_ATL_RPB_RX_BUF_EN_SHIFT 0
 /* width of bitfield rx_buf_en */
-#define rpb_rx_buf_en_width 1
+#define HW_ATL_RPB_RX_BUF_EN_WIDTH 1
 /* default value of bitfield rx_buf_en */
-#define rpb_rx_buf_en_default 0x0
+#define HW_ATL_RPB_RX_BUF_EN_DEFAULT 0x0
 
 /* rx rx{b}_hi_thresh[d:0] bitfield definitions
  * preprocessor definitions for the bitfield "rx{b}_hi_thresh[d:0]".
@@ -547,17 +553,17 @@
  */
 
 /* register address for bitfield rx{b}_hi_thresh[d:0] */
-#define rpb_rxbhi_thresh_adr(buffer) (0x00005714 + (buffer) * 0x10)
+#define HW_ATL_RPB_RXBHI_THRESH_ADR(buffer) (0x00005714 + (buffer) * 0x10)
 /* bitmask for bitfield rx{b}_hi_thresh[d:0] */
-#define rpb_rxbhi_thresh_msk 0x3fff0000
+#define HW_ATL_RPB_RXBHI_THRESH_MSK 0x3fff0000
 /* inverted bitmask for bitfield rx{b}_hi_thresh[d:0] */
-#define rpb_rxbhi_thresh_mskn 0xc000ffff
+#define HW_ATL_RPB_RXBHI_THRESH_MSKN 0xc000ffff
 /* lower bit position of bitfield rx{b}_hi_thresh[d:0] */
-#define rpb_rxbhi_thresh_shift 16
+#define HW_ATL_RPB_RXBHI_THRESH_SHIFT 16
 /* width of bitfield rx{b}_hi_thresh[d:0] */
-#define rpb_rxbhi_thresh_width 14
+#define HW_ATL_RPB_RXBHI_THRESH_WIDTH 14
 /* default value of bitfield rx{b}_hi_thresh[d:0] */
-#define rpb_rxbhi_thresh_default 0x0
+#define HW_ATL_RPB_RXBHI_THRESH_DEFAULT 0x0
 
 /* rx rx{b}_lo_thresh[d:0] bitfield definitions
  * preprocessor definitions for the bitfield "rx{b}_lo_thresh[d:0]".
@@ -566,17 +572,17 @@
  */
 
 /* register address for bitfield rx{b}_lo_thresh[d:0] */
-#define rpb_rxblo_thresh_adr(buffer) (0x00005714 + (buffer) * 0x10)
+#define HW_ATL_RPB_RXBLO_THRESH_ADR(buffer) (0x00005714 + (buffer) * 0x10)
 /* bitmask for bitfield rx{b}_lo_thresh[d:0] */
-#define rpb_rxblo_thresh_msk 0x00003fff
+#define HW_ATL_RPB_RXBLO_THRESH_MSK 0x00003fff
 /* inverted bitmask for bitfield rx{b}_lo_thresh[d:0] */
-#define rpb_rxblo_thresh_mskn 0xffffc000
+#define HW_ATL_RPB_RXBLO_THRESH_MSKN 0xffffc000
 /* lower bit position of bitfield rx{b}_lo_thresh[d:0] */
-#define rpb_rxblo_thresh_shift 0
+#define HW_ATL_RPB_RXBLO_THRESH_SHIFT 0
 /* width of bitfield rx{b}_lo_thresh[d:0] */
-#define rpb_rxblo_thresh_width 14
+#define HW_ATL_RPB_RXBLO_THRESH_WIDTH 14
 /* default value of bitfield rx{b}_lo_thresh[d:0] */
-#define rpb_rxblo_thresh_default 0x0
+#define HW_ATL_RPB_RXBLO_THRESH_DEFAULT 0x0
 
 /* rx rx_fc_mode[1:0] bitfield definitions
  * preprocessor definitions for the bitfield "rx_fc_mode[1:0]".
@@ -584,17 +590,17 @@
  */
 
 /* register address for bitfield rx_fc_mode[1:0] */
-#define rpb_rx_fc_mode_adr 0x00005700
+#define HW_ATL_RPB_RX_FC_MODE_ADR 0x00005700
 /* bitmask for bitfield rx_fc_mode[1:0] */
-#define rpb_rx_fc_mode_msk 0x00000030
+#define HW_ATL_RPB_RX_FC_MODE_MSK 0x00000030
 /* inverted bitmask for bitfield rx_fc_mode[1:0] */
-#define rpb_rx_fc_mode_mskn 0xffffffcf
+#define HW_ATL_RPB_RX_FC_MODE_MSKN 0xffffffcf
 /* lower bit position of bitfield rx_fc_mode[1:0] */
-#define rpb_rx_fc_mode_shift 4
+#define HW_ATL_RPB_RX_FC_MODE_SHIFT 4
 /* width of bitfield rx_fc_mode[1:0] */
-#define rpb_rx_fc_mode_width 2
+#define HW_ATL_RPB_RX_FC_MODE_WIDTH 2
 /* default value of bitfield rx_fc_mode[1:0] */
-#define rpb_rx_fc_mode_default 0x0
+#define HW_ATL_RPB_RX_FC_MODE_DEFAULT 0x0
 
 /* rx rx{b}_buf_size[8:0] bitfield definitions
  * preprocessor definitions for the bitfield "rx{b}_buf_size[8:0]".
@@ -603,17 +609,17 @@
  */
 
 /* register address for bitfield rx{b}_buf_size[8:0] */
-#define rpb_rxbbuf_size_adr(buffer) (0x00005710 + (buffer) * 0x10)
+#define HW_ATL_RPB_RXBBUF_SIZE_ADR(buffer) (0x00005710 + (buffer) * 0x10)
 /* bitmask for bitfield rx{b}_buf_size[8:0] */
-#define rpb_rxbbuf_size_msk 0x000001ff
+#define HW_ATL_RPB_RXBBUF_SIZE_MSK 0x000001ff
 /* inverted bitmask for bitfield rx{b}_buf_size[8:0] */
-#define rpb_rxbbuf_size_mskn 0xfffffe00
+#define HW_ATL_RPB_RXBBUF_SIZE_MSKN 0xfffffe00
 /* lower bit position of bitfield rx{b}_buf_size[8:0] */
-#define rpb_rxbbuf_size_shift 0
+#define HW_ATL_RPB_RXBBUF_SIZE_SHIFT 0
 /* width of bitfield rx{b}_buf_size[8:0] */
-#define rpb_rxbbuf_size_width 9
+#define HW_ATL_RPB_RXBBUF_SIZE_WIDTH 9
 /* default value of bitfield rx{b}_buf_size[8:0] */
-#define rpb_rxbbuf_size_default 0x0
+#define HW_ATL_RPB_RXBBUF_SIZE_DEFAULT 0x0
 
 /* rx rx{b}_xoff_en bitfield definitions
  * preprocessor definitions for the bitfield "rx{b}_xoff_en".
@@ -622,17 +628,17 @@
  */
 
 /* register address for bitfield rx{b}_xoff_en */
-#define rpb_rxbxoff_en_adr(buffer) (0x00005714 + (buffer) * 0x10)
+#define HW_ATL_RPB_RXBXOFF_EN_ADR(buffer) (0x00005714 + (buffer) * 0x10)
 /* bitmask for bitfield rx{b}_xoff_en */
-#define rpb_rxbxoff_en_msk 0x80000000
+#define HW_ATL_RPB_RXBXOFF_EN_MSK 0x80000000
 /* inverted bitmask for bitfield rx{b}_xoff_en */
-#define rpb_rxbxoff_en_mskn 0x7fffffff
+#define HW_ATL_RPB_RXBXOFF_EN_MSKN 0x7fffffff
 /* lower bit position of bitfield rx{b}_xoff_en */
-#define rpb_rxbxoff_en_shift 31
+#define HW_ATL_RPB_RXBXOFF_EN_SHIFT 31
 /* width of bitfield rx{b}_xoff_en */
-#define rpb_rxbxoff_en_width 1
+#define HW_ATL_RPB_RXBXOFF_EN_WIDTH 1
 /* default value of bitfield rx{b}_xoff_en */
-#define rpb_rxbxoff_en_default 0x0
+#define HW_ATL_RPB_RXBXOFF_EN_DEFAULT 0x0
 
 /* rx l2_bc_thresh[f:0] bitfield definitions
  * preprocessor definitions for the bitfield "l2_bc_thresh[f:0]".
@@ -640,17 +646,17 @@
  */
 
 /* register address for bitfield l2_bc_thresh[f:0] */
-#define rpfl2bc_thresh_adr 0x00005100
+#define HW_ATL_RPFL2BC_THRESH_ADR 0x00005100
 /* bitmask for bitfield l2_bc_thresh[f:0] */
-#define rpfl2bc_thresh_msk 0xffff0000
+#define HW_ATL_RPFL2BC_THRESH_MSK 0xffff0000
 /* inverted bitmask for bitfield l2_bc_thresh[f:0] */
-#define rpfl2bc_thresh_mskn 0x0000ffff
+#define HW_ATL_RPFL2BC_THRESH_MSKN 0x0000ffff
 /* lower bit position of bitfield l2_bc_thresh[f:0] */
-#define rpfl2bc_thresh_shift 16
+#define HW_ATL_RPFL2BC_THRESH_SHIFT 16
 /* width of bitfield l2_bc_thresh[f:0] */
-#define rpfl2bc_thresh_width 16
+#define HW_ATL_RPFL2BC_THRESH_WIDTH 16
 /* default value of bitfield l2_bc_thresh[f:0] */
-#define rpfl2bc_thresh_default 0x0
+#define HW_ATL_RPFL2BC_THRESH_DEFAULT 0x0
 
 /* rx l2_bc_en bitfield definitions
  * preprocessor definitions for the bitfield "l2_bc_en".
@@ -658,17 +664,17 @@
  */
 
 /* register address for bitfield l2_bc_en */
-#define rpfl2bc_en_adr 0x00005100
+#define HW_ATL_RPFL2BC_EN_ADR 0x00005100
 /* bitmask for bitfield l2_bc_en */
-#define rpfl2bc_en_msk 0x00000001
+#define HW_ATL_RPFL2BC_EN_MSK 0x00000001
 /* inverted bitmask for bitfield l2_bc_en */
-#define rpfl2bc_en_mskn 0xfffffffe
+#define HW_ATL_RPFL2BC_EN_MSKN 0xfffffffe
 /* lower bit position of bitfield l2_bc_en */
-#define rpfl2bc_en_shift 0
+#define HW_ATL_RPFL2BC_EN_SHIFT 0
 /* width of bitfield l2_bc_en */
-#define rpfl2bc_en_width 1
+#define HW_ATL_RPFL2BC_EN_WIDTH 1
 /* default value of bitfield l2_bc_en */
-#define rpfl2bc_en_default 0x0
+#define HW_ATL_RPFL2BC_EN_DEFAULT 0x0
 
 /* rx l2_bc_act[2:0] bitfield definitions
  * preprocessor definitions for the bitfield "l2_bc_act[2:0]".
@@ -676,17 +682,17 @@
  */
 
 /* register address for bitfield l2_bc_act[2:0] */
-#define rpfl2bc_act_adr 0x00005100
+#define HW_ATL_RPFL2BC_ACT_ADR 0x00005100
 /* bitmask for bitfield l2_bc_act[2:0] */
-#define rpfl2bc_act_msk 0x00007000
+#define HW_ATL_RPFL2BC_ACT_MSK 0x00007000
 /* inverted bitmask for bitfield l2_bc_act[2:0] */
-#define rpfl2bc_act_mskn 0xffff8fff
+#define HW_ATL_RPFL2BC_ACT_MSKN 0xffff8fff
 /* lower bit position of bitfield l2_bc_act[2:0] */
-#define rpfl2bc_act_shift 12
+#define HW_ATL_RPFL2BC_ACT_SHIFT 12
 /* width of bitfield l2_bc_act[2:0] */
-#define rpfl2bc_act_width 3
+#define HW_ATL_RPFL2BC_ACT_WIDTH 3
 /* default value of bitfield l2_bc_act[2:0] */
-#define rpfl2bc_act_default 0x0
+#define HW_ATL_RPFL2BC_ACT_DEFAULT 0x0
 
 /* rx l2_mc_en{f} bitfield definitions
  * preprocessor definitions for the bitfield "l2_mc_en{f}".
@@ -695,17 +701,17 @@
  */
 
 /* register address for bitfield l2_mc_en{f} */
-#define rpfl2mc_enf_adr(filter) (0x00005250 + (filter) * 0x4)
+#define HW_ATL_RPFL2MC_ENF_ADR(filter) (0x00005250 + (filter) * 0x4)
 /* bitmask for bitfield l2_mc_en{f} */
-#define rpfl2mc_enf_msk 0x80000000
+#define HW_ATL_RPFL2MC_ENF_MSK 0x80000000
 /* inverted bitmask for bitfield l2_mc_en{f} */
-#define rpfl2mc_enf_mskn 0x7fffffff
+#define HW_ATL_RPFL2MC_ENF_MSKN 0x7fffffff
 /* lower bit position of bitfield l2_mc_en{f} */
-#define rpfl2mc_enf_shift 31
+#define HW_ATL_RPFL2MC_ENF_SHIFT 31
 /* width of bitfield l2_mc_en{f} */
-#define rpfl2mc_enf_width 1
+#define HW_ATL_RPFL2MC_ENF_WIDTH 1
 /* default value of bitfield l2_mc_en{f} */
-#define rpfl2mc_enf_default 0x0
+#define HW_ATL_RPFL2MC_ENF_DEFAULT 0x0
 
 /* rx l2_promis_mode bitfield definitions
  * preprocessor definitions for the bitfield "l2_promis_mode".
@@ -713,17 +719,17 @@
  */
 
 /* register address for bitfield l2_promis_mode */
-#define rpfl2promis_mode_adr 0x00005100
+#define HW_ATL_RPFL2PROMIS_MODE_ADR 0x00005100
 /* bitmask for bitfield l2_promis_mode */
-#define rpfl2promis_mode_msk 0x00000008
+#define HW_ATL_RPFL2PROMIS_MODE_MSK 0x00000008
 /* inverted bitmask for bitfield l2_promis_mode */
-#define rpfl2promis_mode_mskn 0xfffffff7
+#define HW_ATL_RPFL2PROMIS_MODE_MSKN 0xfffffff7
 /* lower bit position of bitfield l2_promis_mode */
-#define rpfl2promis_mode_shift 3
+#define HW_ATL_RPFL2PROMIS_MODE_SHIFT 3
 /* width of bitfield l2_promis_mode */
-#define rpfl2promis_mode_width 1
+#define HW_ATL_RPFL2PROMIS_MODE_WIDTH 1
 /* default value of bitfield l2_promis_mode */
-#define rpfl2promis_mode_default 0x0
+#define HW_ATL_RPFL2PROMIS_MODE_DEFAULT 0x0
 
 /* rx l2_uc_act{f}[2:0] bitfield definitions
  * preprocessor definitions for the bitfield "l2_uc_act{f}[2:0]".
@@ -732,17 +738,17 @@
  */
 
 /* register address for bitfield l2_uc_act{f}[2:0] */
-#define rpfl2uc_actf_adr(filter) (0x00005114 + (filter) * 0x8)
+#define HW_ATL_RPFL2UC_ACTF_ADR(filter) (0x00005114 + (filter) * 0x8)
 /* bitmask for bitfield l2_uc_act{f}[2:0] */
-#define rpfl2uc_actf_msk 0x00070000
+#define HW_ATL_RPFL2UC_ACTF_MSK 0x00070000
 /* inverted bitmask for bitfield l2_uc_act{f}[2:0] */
-#define rpfl2uc_actf_mskn 0xfff8ffff
+#define HW_ATL_RPFL2UC_ACTF_MSKN 0xfff8ffff
 /* lower bit position of bitfield l2_uc_act{f}[2:0] */
-#define rpfl2uc_actf_shift 16
+#define HW_ATL_RPFL2UC_ACTF_SHIFT 16
 /* width of bitfield l2_uc_act{f}[2:0] */
-#define rpfl2uc_actf_width 3
+#define HW_ATL_RPFL2UC_ACTF_WIDTH 3
 /* default value of bitfield l2_uc_act{f}[2:0] */
-#define rpfl2uc_actf_default 0x0
+#define HW_ATL_RPFL2UC_ACTF_DEFAULT 0x0
 
 /* rx l2_uc_en{f} bitfield definitions
  * preprocessor definitions for the bitfield "l2_uc_en{f}".
@@ -751,26 +757,26 @@
  */
 
 /* register address for bitfield l2_uc_en{f} */
-#define rpfl2uc_enf_adr(filter) (0x00005114 + (filter) * 0x8)
+#define HW_ATL_RPFL2UC_ENF_ADR(filter) (0x00005114 + (filter) * 0x8)
 /* bitmask for bitfield l2_uc_en{f} */
-#define rpfl2uc_enf_msk 0x80000000
+#define HW_ATL_RPFL2UC_ENF_MSK 0x80000000
 /* inverted bitmask for bitfield l2_uc_en{f} */
-#define rpfl2uc_enf_mskn 0x7fffffff
+#define HW_ATL_RPFL2UC_ENF_MSKN 0x7fffffff
 /* lower bit position of bitfield l2_uc_en{f} */
-#define rpfl2uc_enf_shift 31
+#define HW_ATL_RPFL2UC_ENF_SHIFT 31
 /* width of bitfield l2_uc_en{f} */
-#define rpfl2uc_enf_width 1
+#define HW_ATL_RPFL2UC_ENF_WIDTH 1
 /* default value of bitfield l2_uc_en{f} */
-#define rpfl2uc_enf_default 0x0
+#define HW_ATL_RPFL2UC_ENF_DEFAULT 0x0
 
 /* register address for bitfield l2_uc_da{f}_lsw[1f:0] */
-#define rpfl2uc_daflsw_adr(filter) (0x00005110 + (filter) * 0x8)
+#define HW_ATL_RPFL2UC_DAFLSW_ADR(filter) (0x00005110 + (filter) * 0x8)
 /* register address for bitfield l2_uc_da{f}_msw[f:0] */
-#define rpfl2uc_dafmsw_adr(filter) (0x00005114 + (filter) * 0x8)
+#define HW_ATL_RPFL2UC_DAFMSW_ADR(filter) (0x00005114 + (filter) * 0x8)
 /* bitmask for bitfield l2_uc_da{f}_msw[f:0] */
-#define rpfl2uc_dafmsw_msk 0x0000ffff
+#define HW_ATL_RPFL2UC_DAFMSW_MSK 0x0000ffff
 /* lower bit position of bitfield l2_uc_da{f}_msw[f:0] */
-#define rpfl2uc_dafmsw_shift 0
+#define HW_ATL_RPFL2UC_DAFMSW_SHIFT 0
 
 /* rx l2_mc_accept_all bitfield definitions
  * Preprocessor definitions for the bitfield "l2_mc_accept_all".
@@ -778,22 +784,22 @@
  */
 
 /* Register address for bitfield l2_mc_accept_all */
-#define rpfl2mc_accept_all_adr 0x00005270
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_ADR 0x00005270
 /* Bitmask for bitfield l2_mc_accept_all */
-#define rpfl2mc_accept_all_msk 0x00004000
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_MSK 0x00004000
 /* Inverted bitmask for bitfield l2_mc_accept_all */
-#define rpfl2mc_accept_all_mskn 0xFFFFBFFF
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_MSKN 0xFFFFBFFF
 /* Lower bit position of bitfield l2_mc_accept_all */
-#define rpfl2mc_accept_all_shift 14
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_SHIFT 14
 /* Width of bitfield l2_mc_accept_all */
-#define rpfl2mc_accept_all_width 1
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_WIDTH 1
 /* Default value of bitfield l2_mc_accept_all */
-#define rpfl2mc_accept_all_default 0x0
+#define HW_ATL_RPFL2MC_ACCEPT_ALL_DEFAULT 0x0
 
 /* width of bitfield rx_tc_up{t}[2:0] */
-#define rpf_rpb_rx_tc_upt_width 3
+#define HW_ATL_RPF_RPB_RX_TC_UPT_WIDTH 3
 /* default value of bitfield rx_tc_up{t}[2:0] */
-#define rpf_rpb_rx_tc_upt_default 0x0
+#define HW_ATL_RPF_RPB_RX_TC_UPT_DEFAULT 0x0
 
 /* rx rss_key_addr[4:0] bitfield definitions
  * preprocessor definitions for the bitfield "rss_key_addr[4:0]".
@@ -801,17 +807,17 @@
  */
 
 /* register address for bitfield rss_key_addr[4:0] */
-#define rpf_rss_key_addr_adr 0x000054d0
+#define HW_ATL_RPF_RSS_KEY_ADDR_ADR 0x000054d0
 /* bitmask for bitfield rss_key_addr[4:0] */
-#define rpf_rss_key_addr_msk 0x0000001f
+#define HW_ATL_RPF_RSS_KEY_ADDR_MSK 0x0000001f
 /* inverted bitmask for bitfield rss_key_addr[4:0] */
-#define rpf_rss_key_addr_mskn 0xffffffe0
+#define HW_ATL_RPF_RSS_KEY_ADDR_MSKN 0xffffffe0
 /* lower bit position of bitfield rss_key_addr[4:0] */
-#define rpf_rss_key_addr_shift 0
+#define HW_ATL_RPF_RSS_KEY_ADDR_SHIFT 0
 /* width of bitfield rss_key_addr[4:0] */
-#define rpf_rss_key_addr_width 5
+#define HW_ATL_RPF_RSS_KEY_ADDR_WIDTH 5
 /* default value of bitfield rss_key_addr[4:0] */
-#define rpf_rss_key_addr_default 0x0
+#define HW_ATL_RPF_RSS_KEY_ADDR_DEFAULT 0x0
 
 /* rx rss_key_wr_data[1f:0] bitfield definitions
  * preprocessor definitions for the bitfield "rss_key_wr_data[1f:0]".
@@ -819,17 +825,17 @@
  */
 
 /* register address for bitfield rss_key_wr_data[1f:0] */
-#define rpf_rss_key_wr_data_adr 0x000054d4
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_ADR 0x000054d4
 /* bitmask for bitfield rss_key_wr_data[1f:0] */
-#define rpf_rss_key_wr_data_msk 0xffffffff
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_MSK 0xffffffff
 /* inverted bitmask for bitfield rss_key_wr_data[1f:0] */
-#define rpf_rss_key_wr_data_mskn 0x00000000
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_MSKN 0x00000000
 /* lower bit position of bitfield rss_key_wr_data[1f:0] */
-#define rpf_rss_key_wr_data_shift 0
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_SHIFT 0
 /* width of bitfield rss_key_wr_data[1f:0] */
-#define rpf_rss_key_wr_data_width 32
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_WIDTH 32
 /* default value of bitfield rss_key_wr_data[1f:0] */
-#define rpf_rss_key_wr_data_default 0x0
+#define HW_ATL_RPF_RSS_KEY_WR_DATA_DEFAULT 0x0
 
 /* rx rss_key_wr_en_i bitfield definitions
  * preprocessor definitions for the bitfield "rss_key_wr_en_i".
@@ -837,17 +843,17 @@
  */
 
 /* register address for bitfield rss_key_wr_en_i */
-#define rpf_rss_key_wr_eni_adr 0x000054d0
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_ADR 0x000054d0
 /* bitmask for bitfield rss_key_wr_en_i */
-#define rpf_rss_key_wr_eni_msk 0x00000020
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_MSK 0x00000020
 /* inverted bitmask for bitfield rss_key_wr_en_i */
-#define rpf_rss_key_wr_eni_mskn 0xffffffdf
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_MSKN 0xffffffdf
 /* lower bit position of bitfield rss_key_wr_en_i */
-#define rpf_rss_key_wr_eni_shift 5
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_SHIFT 5
 /* width of bitfield rss_key_wr_en_i */
-#define rpf_rss_key_wr_eni_width 1
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_WIDTH 1
 /* default value of bitfield rss_key_wr_en_i */
-#define rpf_rss_key_wr_eni_default 0x0
+#define HW_ATL_RPF_RSS_KEY_WR_ENI_DEFAULT 0x0
 
 /* rx rss_redir_addr[3:0] bitfield definitions
  * preprocessor definitions for the bitfield "rss_redir_addr[3:0]".
@@ -855,17 +861,17 @@
  */
 
 /* register address for bitfield rss_redir_addr[3:0] */
-#define rpf_rss_redir_addr_adr 0x000054e0
+#define HW_ATL_RPF_RSS_REDIR_ADDR_ADR 0x000054e0
 /* bitmask for bitfield rss_redir_addr[3:0] */
-#define rpf_rss_redir_addr_msk 0x0000000f
+#define HW_ATL_RPF_RSS_REDIR_ADDR_MSK 0x0000000f
 /* inverted bitmask for bitfield rss_redir_addr[3:0] */
-#define rpf_rss_redir_addr_mskn 0xfffffff0
+#define HW_ATL_RPF_RSS_REDIR_ADDR_MSKN 0xfffffff0
 /* lower bit position of bitfield rss_redir_addr[3:0] */
-#define rpf_rss_redir_addr_shift 0
+#define HW_ATL_RPF_RSS_REDIR_ADDR_SHIFT 0
 /* width of bitfield rss_redir_addr[3:0] */
-#define rpf_rss_redir_addr_width 4
+#define HW_ATL_RPF_RSS_REDIR_ADDR_WIDTH 4
 /* default value of bitfield rss_redir_addr[3:0] */
-#define rpf_rss_redir_addr_default 0x0
+#define HW_ATL_RPF_RSS_REDIR_ADDR_DEFAULT 0x0
 
 /* rx rss_redir_wr_data[f:0] bitfield definitions
  * preprocessor definitions for the bitfield "rss_redir_wr_data[f:0]".
@@ -873,17 +879,17 @@
  */
 
 /* register address for bitfield rss_redir_wr_data[f:0] */
-#define rpf_rss_redir_wr_data_adr 0x000054e4
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_ADR 0x000054e4
 /* bitmask for bitfield rss_redir_wr_data[f:0] */
-#define rpf_rss_redir_wr_data_msk 0x0000ffff
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_MSK 0x0000ffff
 /* inverted bitmask for bitfield rss_redir_wr_data[f:0] */
-#define rpf_rss_redir_wr_data_mskn 0xffff0000
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_MSKN 0xffff0000
 /* lower bit position of bitfield rss_redir_wr_data[f:0] */
-#define rpf_rss_redir_wr_data_shift 0
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_SHIFT 0
 /* width of bitfield rss_redir_wr_data[f:0] */
-#define rpf_rss_redir_wr_data_width 16
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_WIDTH 16
 /* default value of bitfield rss_redir_wr_data[f:0] */
-#define rpf_rss_redir_wr_data_default 0x0
+#define HW_ATL_RPF_RSS_REDIR_WR_DATA_DEFAULT 0x0
 
 /* rx rss_redir_wr_en_i bitfield definitions
  * preprocessor definitions for the bitfield "rss_redir_wr_en_i".
@@ -891,17 +897,17 @@
  */
 
 /* register address for bitfield rss_redir_wr_en_i */
-#define rpf_rss_redir_wr_eni_adr 0x000054e0
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_ADR 0x000054e0
 /* bitmask for bitfield rss_redir_wr_en_i */
-#define rpf_rss_redir_wr_eni_msk 0x00000010
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_MSK 0x00000010
 /* inverted bitmask for bitfield rss_redir_wr_en_i */
-#define rpf_rss_redir_wr_eni_mskn 0xffffffef
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_MSKN 0xffffffef
 /* lower bit position of bitfield rss_redir_wr_en_i */
-#define rpf_rss_redir_wr_eni_shift 4
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_SHIFT 4
 /* width of bitfield rss_redir_wr_en_i */
-#define rpf_rss_redir_wr_eni_width 1
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_WIDTH 1
 /* default value of bitfield rss_redir_wr_en_i */
-#define rpf_rss_redir_wr_eni_default 0x0
+#define HW_ATL_RPF_RSS_REDIR_WR_ENI_DEFAULT 0x0
 
 /* rx tpo_rpf_sys_loopback bitfield definitions
  * preprocessor definitions for the bitfield "tpo_rpf_sys_loopback".
@@ -909,17 +915,17 @@
  */
 
 /* register address for bitfield tpo_rpf_sys_loopback */
-#define rpf_tpo_rpf_sys_lbk_adr 0x00005000
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_ADR 0x00005000
 /* bitmask for bitfield tpo_rpf_sys_loopback */
-#define rpf_tpo_rpf_sys_lbk_msk 0x00000100
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_MSK 0x00000100
 /* inverted bitmask for bitfield tpo_rpf_sys_loopback */
-#define rpf_tpo_rpf_sys_lbk_mskn 0xfffffeff
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_MSKN 0xfffffeff
 /* lower bit position of bitfield tpo_rpf_sys_loopback */
-#define rpf_tpo_rpf_sys_lbk_shift 8
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_SHIFT 8
 /* width of bitfield tpo_rpf_sys_loopback */
-#define rpf_tpo_rpf_sys_lbk_width 1
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_WIDTH 1
 /* default value of bitfield tpo_rpf_sys_loopback */
-#define rpf_tpo_rpf_sys_lbk_default 0x0
+#define HW_ATL_RPF_TPO_RPF_SYS_LBK_DEFAULT 0x0
 
 /* rx vl_inner_tpid[f:0] bitfield definitions
  * preprocessor definitions for the bitfield "vl_inner_tpid[f:0]".
@@ -927,17 +933,17 @@
  */
 
 /* register address for bitfield vl_inner_tpid[f:0] */
-#define rpf_vl_inner_tpid_adr 0x00005284
+#define HW_ATL_RPF_VL_INNER_TPID_ADR 0x00005284
 /* bitmask for bitfield vl_inner_tpid[f:0] */
-#define rpf_vl_inner_tpid_msk 0x0000ffff
+#define HW_ATL_RPF_VL_INNER_TPID_MSK 0x0000ffff
 /* inverted bitmask for bitfield vl_inner_tpid[f:0] */
-#define rpf_vl_inner_tpid_mskn 0xffff0000
+#define HW_ATL_RPF_VL_INNER_TPID_MSKN 0xffff0000
 /* lower bit position of bitfield vl_inner_tpid[f:0] */
-#define rpf_vl_inner_tpid_shift 0
+#define HW_ATL_RPF_VL_INNER_TPID_SHIFT 0
 /* width of bitfield vl_inner_tpid[f:0] */
-#define rpf_vl_inner_tpid_width 16
+#define HW_ATL_RPF_VL_INNER_TPID_WIDTH 16
 /* default value of bitfield vl_inner_tpid[f:0] */
-#define rpf_vl_inner_tpid_default 0x8100
+#define HW_ATL_RPF_VL_INNER_TPID_DEFAULT 0x8100
 
 /* rx vl_outer_tpid[f:0] bitfield definitions
  * preprocessor definitions for the bitfield "vl_outer_tpid[f:0]".
@@ -945,17 +951,17 @@
  */
 
 /* register address for bitfield vl_outer_tpid[f:0] */
-#define rpf_vl_outer_tpid_adr 0x00005284
+#define HW_ATL_RPF_VL_OUTER_TPID_ADR 0x00005284
 /* bitmask for bitfield vl_outer_tpid[f:0] */
-#define rpf_vl_outer_tpid_msk 0xffff0000
+#define HW_ATL_RPF_VL_OUTER_TPID_MSK 0xffff0000
 /* inverted bitmask for bitfield vl_outer_tpid[f:0] */
-#define rpf_vl_outer_tpid_mskn 0x0000ffff
+#define HW_ATL_RPF_VL_OUTER_TPID_MSKN 0x0000ffff
 /* lower bit position of bitfield vl_outer_tpid[f:0] */
-#define rpf_vl_outer_tpid_shift 16
+#define HW_ATL_RPF_VL_OUTER_TPID_SHIFT 16
 /* width of bitfield vl_outer_tpid[f:0] */
-#define rpf_vl_outer_tpid_width 16
+#define HW_ATL_RPF_VL_OUTER_TPID_WIDTH 16
 /* default value of bitfield vl_outer_tpid[f:0] */
-#define rpf_vl_outer_tpid_default 0x88a8
+#define HW_ATL_RPF_VL_OUTER_TPID_DEFAULT 0x88a8
 
 /* rx vl_promis_mode bitfield definitions
  * preprocessor definitions for the bitfield "vl_promis_mode".
@@ -963,17 +969,17 @@
  */
 
 /* register address for bitfield vl_promis_mode */
-#define rpf_vl_promis_mode_adr 0x00005280
+#define HW_ATL_RPF_VL_PROMIS_MODE_ADR 0x00005280
 /* bitmask for bitfield vl_promis_mode */
-#define rpf_vl_promis_mode_msk 0x00000002
+#define HW_ATL_RPF_VL_PROMIS_MODE_MSK 0x00000002
 /* inverted bitmask for bitfield vl_promis_mode */
-#define rpf_vl_promis_mode_mskn 0xfffffffd
+#define HW_ATL_RPF_VL_PROMIS_MODE_MSKN 0xfffffffd
 /* lower bit position of bitfield vl_promis_mode */
-#define rpf_vl_promis_mode_shift 1
+#define HW_ATL_RPF_VL_PROMIS_MODE_SHIFT 1
 /* width of bitfield vl_promis_mode */
-#define rpf_vl_promis_mode_width 1
+#define HW_ATL_RPF_VL_PROMIS_MODE_WIDTH 1
 /* default value of bitfield vl_promis_mode */
-#define rpf_vl_promis_mode_default 0x0
+#define HW_ATL_RPF_VL_PROMIS_MODE_DEFAULT 0x0
 
 /* RX vl_accept_untagged_mode Bitfield Definitions
  * Preprocessor definitions for the bitfield "vl_accept_untagged_mode".
@@ -981,17 +987,17 @@
  */
 
 /* Register address for bitfield vl_accept_untagged_mode */
-#define rpf_vl_accept_untagged_mode_adr 0x00005280
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_ADR 0x00005280
 /* Bitmask for bitfield vl_accept_untagged_mode */
-#define rpf_vl_accept_untagged_mode_msk 0x00000004
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_MSK 0x00000004
 /* Inverted bitmask for bitfield vl_accept_untagged_mode */
-#define rpf_vl_accept_untagged_mode_mskn 0xFFFFFFFB
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_MSKN 0xFFFFFFFB
 /* Lower bit position of bitfield vl_accept_untagged_mode */
-#define rpf_vl_accept_untagged_mode_shift 2
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_SHIFT 2
 /* Width of bitfield vl_accept_untagged_mode */
-#define rpf_vl_accept_untagged_mode_width 1
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_WIDTH 1
 /* Default value of bitfield vl_accept_untagged_mode */
-#define rpf_vl_accept_untagged_mode_default 0x0
+#define HW_ATL_RPF_VL_ACCEPT_UNTAGGED_MODE_DEFAULT 0x0
 
 /* rX vl_untagged_act[2:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "vl_untagged_act[2:0]".
@@ -999,17 +1005,17 @@
  */
 
 /* Register address for bitfield vl_untagged_act[2:0] */
-#define rpf_vl_untagged_act_adr 0x00005280
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_ADR 0x00005280
 /* Bitmask for bitfield vl_untagged_act[2:0] */
-#define rpf_vl_untagged_act_msk 0x00000038
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_MSK 0x00000038
 /* Inverted bitmask for bitfield vl_untagged_act[2:0] */
-#define rpf_vl_untagged_act_mskn 0xFFFFFFC7
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_MSKN 0xFFFFFFC7
 /* Lower bit position of bitfield vl_untagged_act[2:0] */
-#define rpf_vl_untagged_act_shift 3
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_SHIFT 3
 /* Width of bitfield vl_untagged_act[2:0] */
-#define rpf_vl_untagged_act_width 3
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_WIDTH 3
 /* Default value of bitfield vl_untagged_act[2:0] */
-#define rpf_vl_untagged_act_default 0x0
+#define HW_ATL_RPF_VL_UNTAGGED_ACT_DEFAULT 0x0
 
 /* RX vl_en{F} Bitfield Definitions
  * Preprocessor definitions for the bitfield "vl_en{F}".
@@ -1018,17 +1024,17 @@
  */
 
 /* Register address for bitfield vl_en{F} */
-#define rpf_vl_en_f_adr(filter) (0x00005290 + (filter) * 0x4)
+#define HW_ATL_RPF_VL_EN_F_ADR(filter) (0x00005290 + (filter) * 0x4)
 /* Bitmask for bitfield vl_en{F} */
-#define rpf_vl_en_f_msk 0x80000000
+#define HW_ATL_RPF_VL_EN_F_MSK 0x80000000
 /* Inverted bitmask for bitfield vl_en{F} */
-#define rpf_vl_en_f_mskn 0x7FFFFFFF
+#define HW_ATL_RPF_VL_EN_F_MSKN 0x7FFFFFFF
 /* Lower bit position of bitfield vl_en{F} */
-#define rpf_vl_en_f_shift 31
+#define HW_ATL_RPF_VL_EN_F_SHIFT 31
 /* Width of bitfield vl_en{F} */
-#define rpf_vl_en_f_width 1
+#define HW_ATL_RPF_VL_EN_F_WIDTH 1
 /* Default value of bitfield vl_en{F} */
-#define rpf_vl_en_f_default 0x0
+#define HW_ATL_RPF_VL_EN_F_DEFAULT 0x0
 
 /* RX vl_act{F}[2:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "vl_act{F}[2:0]".
@@ -1037,17 +1043,17 @@
  */
 
 /* Register address for bitfield vl_act{F}[2:0] */
-#define rpf_vl_act_f_adr(filter) (0x00005290 + (filter) * 0x4)
+#define HW_ATL_RPF_VL_ACT_F_ADR(filter) (0x00005290 + (filter) * 0x4)
 /* Bitmask for bitfield vl_act{F}[2:0] */
-#define rpf_vl_act_f_msk 0x00070000
+#define HW_ATL_RPF_VL_ACT_F_MSK 0x00070000
 /* Inverted bitmask for bitfield vl_act{F}[2:0] */
-#define rpf_vl_act_f_mskn 0xFFF8FFFF
+#define HW_ATL_RPF_VL_ACT_F_MSKN 0xFFF8FFFF
 /* Lower bit position of bitfield vl_act{F}[2:0] */
-#define rpf_vl_act_f_shift 16
+#define HW_ATL_RPF_VL_ACT_F_SHIFT 16
 /* Width of bitfield vl_act{F}[2:0] */
-#define rpf_vl_act_f_width 3
+#define HW_ATL_RPF_VL_ACT_F_WIDTH 3
 /* Default value of bitfield vl_act{F}[2:0] */
-#define rpf_vl_act_f_default 0x0
+#define HW_ATL_RPF_VL_ACT_F_DEFAULT 0x0
 
 /* RX vl_id{F}[B:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "vl_id{F}[B:0]".
@@ -1056,17 +1062,17 @@
  */
 
 /* Register address for bitfield vl_id{F}[B:0] */
-#define rpf_vl_id_f_adr(filter) (0x00005290 + (filter) * 0x4)
+#define HW_ATL_RPF_VL_ID_F_ADR(filter) (0x00005290 + (filter) * 0x4)
 /* Bitmask for bitfield vl_id{F}[B:0] */
-#define rpf_vl_id_f_msk 0x00000FFF
+#define HW_ATL_RPF_VL_ID_F_MSK 0x00000FFF
 /* Inverted bitmask for bitfield vl_id{F}[B:0] */
-#define rpf_vl_id_f_mskn 0xFFFFF000
+#define HW_ATL_RPF_VL_ID_F_MSKN 0xFFFFF000
 /* Lower bit position of bitfield vl_id{F}[B:0] */
-#define rpf_vl_id_f_shift 0
+#define HW_ATL_RPF_VL_ID_F_SHIFT 0
 /* Width of bitfield vl_id{F}[B:0] */
-#define rpf_vl_id_f_width 12
+#define HW_ATL_RPF_VL_ID_F_WIDTH 12
 /* Default value of bitfield vl_id{F}[B:0] */
-#define rpf_vl_id_f_default 0x0
+#define HW_ATL_RPF_VL_ID_F_DEFAULT 0x0
 
 /* RX et_en{F} Bitfield Definitions
  * Preprocessor definitions for the bitfield "et_en{F}".
@@ -1075,17 +1081,17 @@
  */
 
 /* Register address for bitfield et_en{F} */
-#define rpf_et_en_f_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_EN_F_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* Bitmask for bitfield et_en{F} */
-#define rpf_et_en_f_msk 0x80000000
+#define HW_ATL_RPF_ET_EN_F_MSK 0x80000000
 /* Inverted bitmask for bitfield et_en{F} */
-#define rpf_et_en_f_mskn 0x7FFFFFFF
+#define HW_ATL_RPF_ET_EN_F_MSKN 0x7FFFFFFF
 /* Lower bit position of bitfield et_en{F} */
-#define rpf_et_en_f_shift 31
+#define HW_ATL_RPF_ET_EN_F_SHIFT 31
 /* Width of bitfield et_en{F} */
-#define rpf_et_en_f_width 1
+#define HW_ATL_RPF_ET_EN_F_WIDTH 1
 /* Default value of bitfield et_en{F} */
-#define rpf_et_en_f_default 0x0
+#define HW_ATL_RPF_ET_EN_F_DEFAULT 0x0
 
 /* rx et_en{f} bitfield definitions
  * preprocessor definitions for the bitfield "et_en{f}".
@@ -1094,17 +1100,17 @@
  */
 
 /* register address for bitfield et_en{f} */
-#define rpf_et_enf_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_ENF_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_en{f} */
-#define rpf_et_enf_msk 0x80000000
+#define HW_ATL_RPF_ET_ENF_MSK 0x80000000
 /* inverted bitmask for bitfield et_en{f} */
-#define rpf_et_enf_mskn 0x7fffffff
+#define HW_ATL_RPF_ET_ENF_MSKN 0x7fffffff
 /* lower bit position of bitfield et_en{f} */
-#define rpf_et_enf_shift 31
+#define HW_ATL_RPF_ET_ENF_SHIFT 31
 /* width of bitfield et_en{f} */
-#define rpf_et_enf_width 1
+#define HW_ATL_RPF_ET_ENF_WIDTH 1
 /* default value of bitfield et_en{f} */
-#define rpf_et_enf_default 0x0
+#define HW_ATL_RPF_ET_ENF_DEFAULT 0x0
 
 /* rx et_up{f}_en bitfield definitions
  * preprocessor definitions for the bitfield "et_up{f}_en".
@@ -1113,17 +1119,17 @@
  */
 
 /* register address for bitfield et_up{f}_en */
-#define rpf_et_upfen_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_UPFEN_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_up{f}_en */
-#define rpf_et_upfen_msk 0x40000000
+#define HW_ATL_RPF_ET_UPFEN_MSK 0x40000000
 /* inverted bitmask for bitfield et_up{f}_en */
-#define rpf_et_upfen_mskn 0xbfffffff
+#define HW_ATL_RPF_ET_UPFEN_MSKN 0xbfffffff
 /* lower bit position of bitfield et_up{f}_en */
-#define rpf_et_upfen_shift 30
+#define HW_ATL_RPF_ET_UPFEN_SHIFT 30
 /* width of bitfield et_up{f}_en */
-#define rpf_et_upfen_width 1
+#define HW_ATL_RPF_ET_UPFEN_WIDTH 1
 /* default value of bitfield et_up{f}_en */
-#define rpf_et_upfen_default 0x0
+#define HW_ATL_RPF_ET_UPFEN_DEFAULT 0x0
 
 /* rx et_rxq{f}_en bitfield definitions
  * preprocessor definitions for the bitfield "et_rxq{f}_en".
@@ -1132,17 +1138,17 @@
  */
 
 /* register address for bitfield et_rxq{f}_en */
-#define rpf_et_rxqfen_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_RXQFEN_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_rxq{f}_en */
-#define rpf_et_rxqfen_msk 0x20000000
+#define HW_ATL_RPF_ET_RXQFEN_MSK 0x20000000
 /* inverted bitmask for bitfield et_rxq{f}_en */
-#define rpf_et_rxqfen_mskn 0xdfffffff
+#define HW_ATL_RPF_ET_RXQFEN_MSKN 0xdfffffff
 /* lower bit position of bitfield et_rxq{f}_en */
-#define rpf_et_rxqfen_shift 29
+#define HW_ATL_RPF_ET_RXQFEN_SHIFT 29
 /* width of bitfield et_rxq{f}_en */
-#define rpf_et_rxqfen_width 1
+#define HW_ATL_RPF_ET_RXQFEN_WIDTH 1
 /* default value of bitfield et_rxq{f}_en */
-#define rpf_et_rxqfen_default 0x0
+#define HW_ATL_RPF_ET_RXQFEN_DEFAULT 0x0
 
 /* rx et_up{f}[2:0] bitfield definitions
  * preprocessor definitions for the bitfield "et_up{f}[2:0]".
@@ -1151,17 +1157,17 @@
  */
 
 /* register address for bitfield et_up{f}[2:0] */
-#define rpf_et_upf_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_UPF_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_up{f}[2:0] */
-#define rpf_et_upf_msk 0x1c000000
+#define HW_ATL_RPF_ET_UPF_MSK 0x1c000000
 /* inverted bitmask for bitfield et_up{f}[2:0] */
-#define rpf_et_upf_mskn 0xe3ffffff
+#define HW_ATL_RPF_ET_UPF_MSKN 0xe3ffffff
 /* lower bit position of bitfield et_up{f}[2:0] */
-#define rpf_et_upf_shift 26
+#define HW_ATL_RPF_ET_UPF_SHIFT 26
 /* width of bitfield et_up{f}[2:0] */
-#define rpf_et_upf_width 3
+#define HW_ATL_RPF_ET_UPF_WIDTH 3
 /* default value of bitfield et_up{f}[2:0] */
-#define rpf_et_upf_default 0x0
+#define HW_ATL_RPF_ET_UPF_DEFAULT 0x0
 
 /* rx et_rxq{f}[4:0] bitfield definitions
  * preprocessor definitions for the bitfield "et_rxq{f}[4:0]".
@@ -1170,17 +1176,17 @@
  */
 
 /* register address for bitfield et_rxq{f}[4:0] */
-#define rpf_et_rxqf_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_RXQF_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_rxq{f}[4:0] */
-#define rpf_et_rxqf_msk 0x01f00000
+#define HW_ATL_RPF_ET_RXQF_MSK 0x01f00000
 /* inverted bitmask for bitfield et_rxq{f}[4:0] */
-#define rpf_et_rxqf_mskn 0xfe0fffff
+#define HW_ATL_RPF_ET_RXQF_MSKN 0xfe0fffff
 /* lower bit position of bitfield et_rxq{f}[4:0] */
-#define rpf_et_rxqf_shift 20
+#define HW_ATL_RPF_ET_RXQF_SHIFT 20
 /* width of bitfield et_rxq{f}[4:0] */
-#define rpf_et_rxqf_width 5
+#define HW_ATL_RPF_ET_RXQF_WIDTH 5
 /* default value of bitfield et_rxq{f}[4:0] */
-#define rpf_et_rxqf_default 0x0
+#define HW_ATL_RPF_ET_RXQF_DEFAULT 0x0
 
 /* rx et_mng_rxq{f} bitfield definitions
  * preprocessor definitions for the bitfield "et_mng_rxq{f}".
@@ -1189,17 +1195,17 @@
  */
 
 /* register address for bitfield et_mng_rxq{f} */
-#define rpf_et_mng_rxqf_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_MNG_RXQF_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_mng_rxq{f} */
-#define rpf_et_mng_rxqf_msk 0x00080000
+#define HW_ATL_RPF_ET_MNG_RXQF_MSK 0x00080000
 /* inverted bitmask for bitfield et_mng_rxq{f} */
-#define rpf_et_mng_rxqf_mskn 0xfff7ffff
+#define HW_ATL_RPF_ET_MNG_RXQF_MSKN 0xfff7ffff
 /* lower bit position of bitfield et_mng_rxq{f} */
-#define rpf_et_mng_rxqf_shift 19
+#define HW_ATL_RPF_ET_MNG_RXQF_SHIFT 19
 /* width of bitfield et_mng_rxq{f} */
-#define rpf_et_mng_rxqf_width 1
+#define HW_ATL_RPF_ET_MNG_RXQF_WIDTH 1
 /* default value of bitfield et_mng_rxq{f} */
-#define rpf_et_mng_rxqf_default 0x0
+#define HW_ATL_RPF_ET_MNG_RXQF_DEFAULT 0x0
 
 /* rx et_act{f}[2:0] bitfield definitions
  * preprocessor definitions for the bitfield "et_act{f}[2:0]".
@@ -1208,17 +1214,17 @@
  */
 
 /* register address for bitfield et_act{f}[2:0] */
-#define rpf_et_actf_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_ACTF_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_act{f}[2:0] */
-#define rpf_et_actf_msk 0x00070000
+#define HW_ATL_RPF_ET_ACTF_MSK 0x00070000
 /* inverted bitmask for bitfield et_act{f}[2:0] */
-#define rpf_et_actf_mskn 0xfff8ffff
+#define HW_ATL_RPF_ET_ACTF_MSKN 0xfff8ffff
 /* lower bit position of bitfield et_act{f}[2:0] */
-#define rpf_et_actf_shift 16
+#define HW_ATL_RPF_ET_ACTF_SHIFT 16
 /* width of bitfield et_act{f}[2:0] */
-#define rpf_et_actf_width 3
+#define HW_ATL_RPF_ET_ACTF_WIDTH 3
 /* default value of bitfield et_act{f}[2:0] */
-#define rpf_et_actf_default 0x0
+#define HW_ATL_RPF_ET_ACTF_DEFAULT 0x0
 
 /* rx et_val{f}[f:0] bitfield definitions
  * preprocessor definitions for the bitfield "et_val{f}[f:0]".
@@ -1227,17 +1233,17 @@
  */
 
 /* register address for bitfield et_val{f}[f:0] */
-#define rpf_et_valf_adr(filter) (0x00005300 + (filter) * 0x4)
+#define HW_ATL_RPF_ET_VALF_ADR(filter) (0x00005300 + (filter) * 0x4)
 /* bitmask for bitfield et_val{f}[f:0] */
-#define rpf_et_valf_msk 0x0000ffff
+#define HW_ATL_RPF_ET_VALF_MSK 0x0000ffff
 /* inverted bitmask for bitfield et_val{f}[f:0] */
-#define rpf_et_valf_mskn 0xffff0000
+#define HW_ATL_RPF_ET_VALF_MSKN 0xffff0000
 /* lower bit position of bitfield et_val{f}[f:0] */
-#define rpf_et_valf_shift 0
+#define HW_ATL_RPF_ET_VALF_SHIFT 0
 /* width of bitfield et_val{f}[f:0] */
-#define rpf_et_valf_width 16
+#define HW_ATL_RPF_ET_VALF_WIDTH 16
 /* default value of bitfield et_val{f}[f:0] */
-#define rpf_et_valf_default 0x0
+#define HW_ATL_RPF_ET_VALF_DEFAULT 0x0
 
 /* rx ipv4_chk_en bitfield definitions
  * preprocessor definitions for the bitfield "ipv4_chk_en".
@@ -1245,17 +1251,17 @@
  */
 
 /* register address for bitfield ipv4_chk_en */
-#define rpo_ipv4chk_en_adr 0x00005580
+#define HW_ATL_RPO_IPV4CHK_EN_ADR 0x00005580
 /* bitmask for bitfield ipv4_chk_en */
-#define rpo_ipv4chk_en_msk 0x00000002
+#define HW_ATL_RPO_IPV4CHK_EN_MSK 0x00000002
 /* inverted bitmask for bitfield ipv4_chk_en */
-#define rpo_ipv4chk_en_mskn 0xfffffffd
+#define HW_ATL_RPO_IPV4CHK_EN_MSKN 0xfffffffd
 /* lower bit position of bitfield ipv4_chk_en */
-#define rpo_ipv4chk_en_shift 1
+#define HW_ATL_RPO_IPV4CHK_EN_SHIFT 1
 /* width of bitfield ipv4_chk_en */
-#define rpo_ipv4chk_en_width 1
+#define HW_ATL_RPO_IPV4CHK_EN_WIDTH 1
 /* default value of bitfield ipv4_chk_en */
-#define rpo_ipv4chk_en_default 0x0
+#define HW_ATL_RPO_IPV4CHK_EN_DEFAULT 0x0
 
 /* rx desc{d}_vl_strip bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_vl_strip".
@@ -1264,17 +1270,18 @@
  */
 
 /* register address for bitfield desc{d}_vl_strip */
-#define rpo_descdvl_strip_adr(descriptor) (0x00005b08 + (descriptor) * 0x20)
+#define HW_ATL_RPO_DESCDVL_STRIP_ADR(descriptor) \
+	(0x00005b08 + (descriptor) * 0x20)
 /* bitmask for bitfield desc{d}_vl_strip */
-#define rpo_descdvl_strip_msk 0x20000000
+#define HW_ATL_RPO_DESCDVL_STRIP_MSK 0x20000000
 /* inverted bitmask for bitfield desc{d}_vl_strip */
-#define rpo_descdvl_strip_mskn 0xdfffffff
+#define HW_ATL_RPO_DESCDVL_STRIP_MSKN 0xdfffffff
 /* lower bit position of bitfield desc{d}_vl_strip */
-#define rpo_descdvl_strip_shift 29
+#define HW_ATL_RPO_DESCDVL_STRIP_SHIFT 29
 /* width of bitfield desc{d}_vl_strip */
-#define rpo_descdvl_strip_width 1
+#define HW_ATL_RPO_DESCDVL_STRIP_WIDTH 1
 /* default value of bitfield desc{d}_vl_strip */
-#define rpo_descdvl_strip_default 0x0
+#define HW_ATL_RPO_DESCDVL_STRIP_DEFAULT 0x0
 
 /* rx l4_chk_en bitfield definitions
  * preprocessor definitions for the bitfield "l4_chk_en".
@@ -1282,17 +1289,17 @@
  */
 
 /* register address for bitfield l4_chk_en */
-#define rpol4chk_en_adr 0x00005580
+#define HW_ATL_RPOL4CHK_EN_ADR 0x00005580
 /* bitmask for bitfield l4_chk_en */
-#define rpol4chk_en_msk 0x00000001
+#define HW_ATL_RPOL4CHK_EN_MSK 0x00000001
 /* inverted bitmask for bitfield l4_chk_en */
-#define rpol4chk_en_mskn 0xfffffffe
+#define HW_ATL_RPOL4CHK_EN_MSKN 0xfffffffe
 /* lower bit position of bitfield l4_chk_en */
-#define rpol4chk_en_shift 0
+#define HW_ATL_RPOL4CHK_EN_SHIFT 0
 /* width of bitfield l4_chk_en */
-#define rpol4chk_en_width 1
+#define HW_ATL_RPOL4CHK_EN_WIDTH 1
 /* default value of bitfield l4_chk_en */
-#define rpol4chk_en_default 0x0
+#define HW_ATL_RPOL4CHK_EN_DEFAULT 0x0
 
 /* rx reg_res_dsbl bitfield definitions
  * preprocessor definitions for the bitfield "reg_res_dsbl".
@@ -1300,17 +1307,17 @@
  */
 
 /* register address for bitfield reg_res_dsbl */
-#define rx_reg_res_dsbl_adr 0x00005000
+#define HW_ATL_RX_REG_RES_DSBL_ADR 0x00005000
 /* bitmask for bitfield reg_res_dsbl */
-#define rx_reg_res_dsbl_msk 0x20000000
+#define HW_ATL_RX_REG_RES_DSBL_MSK 0x20000000
 /* inverted bitmask for bitfield reg_res_dsbl */
-#define rx_reg_res_dsbl_mskn 0xdfffffff
+#define HW_ATL_RX_REG_RES_DSBL_MSKN 0xdfffffff
 /* lower bit position of bitfield reg_res_dsbl */
-#define rx_reg_res_dsbl_shift 29
+#define HW_ATL_RX_REG_RES_DSBL_SHIFT 29
 /* width of bitfield reg_res_dsbl */
-#define rx_reg_res_dsbl_width 1
+#define HW_ATL_RX_REG_RES_DSBL_WIDTH 1
 /* default value of bitfield reg_res_dsbl */
-#define rx_reg_res_dsbl_default 0x1
+#define HW_ATL_RX_REG_RES_DSBL_DEFAULT 0x1
 
 /* tx dca{d}_cpuid[7:0] bitfield definitions
  * preprocessor definitions for the bitfield "dca{d}_cpuid[7:0]".
@@ -1319,17 +1326,17 @@
  */
 
 /* register address for bitfield dca{d}_cpuid[7:0] */
-#define tdm_dcadcpuid_adr(dca) (0x00008400 + (dca) * 0x4)
+#define HW_ATL_TDM_DCADCPUID_ADR(dca) (0x00008400 + (dca) * 0x4)
 /* bitmask for bitfield dca{d}_cpuid[7:0] */
-#define tdm_dcadcpuid_msk 0x000000ff
+#define HW_ATL_TDM_DCADCPUID_MSK 0x000000ff
 /* inverted bitmask for bitfield dca{d}_cpuid[7:0] */
-#define tdm_dcadcpuid_mskn 0xffffff00
+#define HW_ATL_TDM_DCADCPUID_MSKN 0xffffff00
 /* lower bit position of bitfield dca{d}_cpuid[7:0] */
-#define tdm_dcadcpuid_shift 0
+#define HW_ATL_TDM_DCADCPUID_SHIFT 0
 /* width of bitfield dca{d}_cpuid[7:0] */
-#define tdm_dcadcpuid_width 8
+#define HW_ATL_TDM_DCADCPUID_WIDTH 8
 /* default value of bitfield dca{d}_cpuid[7:0] */
-#define tdm_dcadcpuid_default 0x0
+#define HW_ATL_TDM_DCADCPUID_DEFAULT 0x0
 
 /* tx lso_en[1f:0] bitfield definitions
  * preprocessor definitions for the bitfield "lso_en[1f:0]".
@@ -1337,17 +1344,17 @@
  */
 
 /* register address for bitfield lso_en[1f:0] */
-#define tdm_lso_en_adr 0x00007810
+#define HW_ATL_TDM_LSO_EN_ADR 0x00007810
 /* bitmask for bitfield lso_en[1f:0] */
-#define tdm_lso_en_msk 0xffffffff
+#define HW_ATL_TDM_LSO_EN_MSK 0xffffffff
 /* inverted bitmask for bitfield lso_en[1f:0] */
-#define tdm_lso_en_mskn 0x00000000
+#define HW_ATL_TDM_LSO_EN_MSKN 0x00000000
 /* lower bit position of bitfield lso_en[1f:0] */
-#define tdm_lso_en_shift 0
+#define HW_ATL_TDM_LSO_EN_SHIFT 0
 /* width of bitfield lso_en[1f:0] */
-#define tdm_lso_en_width 32
+#define HW_ATL_TDM_LSO_EN_WIDTH 32
 /* default value of bitfield lso_en[1f:0] */
-#define tdm_lso_en_default 0x0
+#define HW_ATL_TDM_LSO_EN_DEFAULT 0x0
 
 /* tx dca_en bitfield definitions
  * preprocessor definitions for the bitfield "dca_en".
@@ -1355,17 +1362,17 @@
  */
 
 /* register address for bitfield dca_en */
-#define tdm_dca_en_adr 0x00008480
+#define HW_ATL_TDM_DCA_EN_ADR 0x00008480
 /* bitmask for bitfield dca_en */
-#define tdm_dca_en_msk 0x80000000
+#define HW_ATL_TDM_DCA_EN_MSK 0x80000000
 /* inverted bitmask for bitfield dca_en */
-#define tdm_dca_en_mskn 0x7fffffff
+#define HW_ATL_TDM_DCA_EN_MSKN 0x7fffffff
 /* lower bit position of bitfield dca_en */
-#define tdm_dca_en_shift 31
+#define HW_ATL_TDM_DCA_EN_SHIFT 31
 /* width of bitfield dca_en */
-#define tdm_dca_en_width 1
+#define HW_ATL_TDM_DCA_EN_WIDTH 1
 /* default value of bitfield dca_en */
-#define tdm_dca_en_default 0x1
+#define HW_ATL_TDM_DCA_EN_DEFAULT 0x1
 
 /* tx dca_mode[3:0] bitfield definitions
  * preprocessor definitions for the bitfield "dca_mode[3:0]".
@@ -1373,17 +1380,17 @@
  */
 
 /* register address for bitfield dca_mode[3:0] */
-#define tdm_dca_mode_adr 0x00008480
+#define HW_ATL_TDM_DCA_MODE_ADR 0x00008480
 /* bitmask for bitfield dca_mode[3:0] */
-#define tdm_dca_mode_msk 0x0000000f
+#define HW_ATL_TDM_DCA_MODE_MSK 0x0000000f
 /* inverted bitmask for bitfield dca_mode[3:0] */
-#define tdm_dca_mode_mskn 0xfffffff0
+#define HW_ATL_TDM_DCA_MODE_MSKN 0xfffffff0
 /* lower bit position of bitfield dca_mode[3:0] */
-#define tdm_dca_mode_shift 0
+#define HW_ATL_TDM_DCA_MODE_SHIFT 0
 /* width of bitfield dca_mode[3:0] */
-#define tdm_dca_mode_width 4
+#define HW_ATL_TDM_DCA_MODE_WIDTH 4
 /* default value of bitfield dca_mode[3:0] */
-#define tdm_dca_mode_default 0x0
+#define HW_ATL_TDM_DCA_MODE_DEFAULT 0x0
 
 /* tx dca{d}_desc_en bitfield definitions
  * preprocessor definitions for the bitfield "dca{d}_desc_en".
@@ -1392,17 +1399,17 @@
  */
 
 /* register address for bitfield dca{d}_desc_en */
-#define tdm_dcaddesc_en_adr(dca) (0x00008400 + (dca) * 0x4)
+#define HW_ATL_TDM_DCADDESC_EN_ADR(dca) (0x00008400 + (dca) * 0x4)
 /* bitmask for bitfield dca{d}_desc_en */
-#define tdm_dcaddesc_en_msk 0x80000000
+#define HW_ATL_TDM_DCADDESC_EN_MSK 0x80000000
 /* inverted bitmask for bitfield dca{d}_desc_en */
-#define tdm_dcaddesc_en_mskn 0x7fffffff
+#define HW_ATL_TDM_DCADDESC_EN_MSKN 0x7fffffff
 /* lower bit position of bitfield dca{d}_desc_en */
-#define tdm_dcaddesc_en_shift 31
+#define HW_ATL_TDM_DCADDESC_EN_SHIFT 31
 /* width of bitfield dca{d}_desc_en */
-#define tdm_dcaddesc_en_width 1
+#define HW_ATL_TDM_DCADDESC_EN_WIDTH 1
 /* default value of bitfield dca{d}_desc_en */
-#define tdm_dcaddesc_en_default 0x0
+#define HW_ATL_TDM_DCADDESC_EN_DEFAULT 0x0
 
 /* tx desc{d}_en bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_en".
@@ -1411,17 +1418,17 @@
  */
 
 /* register address for bitfield desc{d}_en */
-#define tdm_descden_adr(descriptor) (0x00007c08 + (descriptor) * 0x40)
+#define HW_ATL_TDM_DESCDEN_ADR(descriptor) (0x00007c08 + (descriptor) * 0x40)
 /* bitmask for bitfield desc{d}_en */
-#define tdm_descden_msk 0x80000000
+#define HW_ATL_TDM_DESCDEN_MSK 0x80000000
 /* inverted bitmask for bitfield desc{d}_en */
-#define tdm_descden_mskn 0x7fffffff
+#define HW_ATL_TDM_DESCDEN_MSKN 0x7fffffff
 /* lower bit position of bitfield desc{d}_en */
-#define tdm_descden_shift 31
+#define HW_ATL_TDM_DESCDEN_SHIFT 31
 /* width of bitfield desc{d}_en */
-#define tdm_descden_width 1
+#define HW_ATL_TDM_DESCDEN_WIDTH 1
 /* default value of bitfield desc{d}_en */
-#define tdm_descden_default 0x0
+#define HW_ATL_TDM_DESCDEN_DEFAULT 0x0
 
 /* tx desc{d}_hd[c:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_hd[c:0]".
@@ -1430,15 +1437,15 @@
  */
 
 /* register address for bitfield desc{d}_hd[c:0] */
-#define tdm_descdhd_adr(descriptor) (0x00007c0c + (descriptor) * 0x40)
+#define HW_ATL_TDM_DESCDHD_ADR(descriptor) (0x00007c0c + (descriptor) * 0x40)
 /* bitmask for bitfield desc{d}_hd[c:0] */
-#define tdm_descdhd_msk 0x00001fff
+#define HW_ATL_TDM_DESCDHD_MSK 0x00001fff
 /* inverted bitmask for bitfield desc{d}_hd[c:0] */
-#define tdm_descdhd_mskn 0xffffe000
+#define HW_ATL_TDM_DESCDHD_MSKN 0xffffe000
 /* lower bit position of bitfield desc{d}_hd[c:0] */
-#define tdm_descdhd_shift 0
+#define HW_ATL_TDM_DESCDHD_SHIFT 0
 /* width of bitfield desc{d}_hd[c:0] */
-#define tdm_descdhd_width 13
+#define HW_ATL_TDM_DESCDHD_WIDTH 13
 
 /* tx desc{d}_len[9:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_len[9:0]".
@@ -1447,17 +1454,17 @@
  */
 
 /* register address for bitfield desc{d}_len[9:0] */
-#define tdm_descdlen_adr(descriptor) (0x00007c08 + (descriptor) * 0x40)
+#define HW_ATL_TDM_DESCDLEN_ADR(descriptor) (0x00007c08 + (descriptor) * 0x40)
 /* bitmask for bitfield desc{d}_len[9:0] */
-#define tdm_descdlen_msk 0x00001ff8
+#define HW_ATL_TDM_DESCDLEN_MSK 0x00001ff8
 /* inverted bitmask for bitfield desc{d}_len[9:0] */
-#define tdm_descdlen_mskn 0xffffe007
+#define HW_ATL_TDM_DESCDLEN_MSKN 0xffffe007
 /* lower bit position of bitfield desc{d}_len[9:0] */
-#define tdm_descdlen_shift 3
+#define HW_ATL_TDM_DESCDLEN_SHIFT 3
 /* width of bitfield desc{d}_len[9:0] */
-#define tdm_descdlen_width 10
+#define HW_ATL_TDM_DESCDLEN_WIDTH 10
 /* default value of bitfield desc{d}_len[9:0] */
-#define tdm_descdlen_default 0x0
+#define HW_ATL_TDM_DESCDLEN_DEFAULT 0x0
 
 /* tx int_desc_wrb_en bitfield definitions
  * preprocessor definitions for the bitfield "int_desc_wrb_en".
@@ -1465,17 +1472,17 @@
  */
 
 /* register address for bitfield int_desc_wrb_en */
-#define tdm_int_desc_wrb_en_adr 0x00007b40
+#define HW_ATL_TDM_INT_DESC_WRB_EN_ADR 0x00007b40
 /* bitmask for bitfield int_desc_wrb_en */
-#define tdm_int_desc_wrb_en_msk 0x00000002
+#define HW_ATL_TDM_INT_DESC_WRB_EN_MSK 0x00000002
 /* inverted bitmask for bitfield int_desc_wrb_en */
-#define tdm_int_desc_wrb_en_mskn 0xfffffffd
+#define HW_ATL_TDM_INT_DESC_WRB_EN_MSKN 0xfffffffd
 /* lower bit position of bitfield int_desc_wrb_en */
-#define tdm_int_desc_wrb_en_shift 1
+#define HW_ATL_TDM_INT_DESC_WRB_EN_SHIFT 1
 /* width of bitfield int_desc_wrb_en */
-#define tdm_int_desc_wrb_en_width 1
+#define HW_ATL_TDM_INT_DESC_WRB_EN_WIDTH 1
 /* default value of bitfield int_desc_wrb_en */
-#define tdm_int_desc_wrb_en_default 0x0
+#define HW_ATL_TDM_INT_DESC_WRB_EN_DEFAULT 0x0
 
 /* tx desc{d}_wrb_thresh[6:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc{d}_wrb_thresh[6:0]".
@@ -1484,17 +1491,18 @@
  */
 
 /* register address for bitfield desc{d}_wrb_thresh[6:0] */
-#define tdm_descdwrb_thresh_adr(descriptor) (0x00007c18 + (descriptor) * 0x40)
+#define HW_ATL_TDM_DESCDWRB_THRESH_ADR(descriptor) \
+	(0x00007c18 + (descriptor) * 0x40)
 /* bitmask for bitfield desc{d}_wrb_thresh[6:0] */
-#define tdm_descdwrb_thresh_msk 0x00007f00
+#define HW_ATL_TDM_DESCDWRB_THRESH_MSK 0x00007f00
 /* inverted bitmask for bitfield desc{d}_wrb_thresh[6:0] */
-#define tdm_descdwrb_thresh_mskn 0xffff80ff
+#define HW_ATL_TDM_DESCDWRB_THRESH_MSKN 0xffff80ff
 /* lower bit position of bitfield desc{d}_wrb_thresh[6:0] */
-#define tdm_descdwrb_thresh_shift 8
+#define HW_ATL_TDM_DESCDWRB_THRESH_SHIFT 8
 /* width of bitfield desc{d}_wrb_thresh[6:0] */
-#define tdm_descdwrb_thresh_width 7
+#define HW_ATL_TDM_DESCDWRB_THRESH_WIDTH 7
 /* default value of bitfield desc{d}_wrb_thresh[6:0] */
-#define tdm_descdwrb_thresh_default 0x0
+#define HW_ATL_TDM_DESCDWRB_THRESH_DEFAULT 0x0
 
 /* tx lso_tcp_flag_first[b:0] bitfield definitions
  * preprocessor definitions for the bitfield "lso_tcp_flag_first[b:0]".
@@ -1502,17 +1510,17 @@
  */
 
 /* register address for bitfield lso_tcp_flag_first[b:0] */
-#define thm_lso_tcp_flag_first_adr 0x00007820
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_ADR 0x00007820
 /* bitmask for bitfield lso_tcp_flag_first[b:0] */
-#define thm_lso_tcp_flag_first_msk 0x00000fff
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_MSK 0x00000fff
 /* inverted bitmask for bitfield lso_tcp_flag_first[b:0] */
-#define thm_lso_tcp_flag_first_mskn 0xfffff000
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_MSKN 0xfffff000
 /* lower bit position of bitfield lso_tcp_flag_first[b:0] */
-#define thm_lso_tcp_flag_first_shift 0
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_SHIFT 0
 /* width of bitfield lso_tcp_flag_first[b:0] */
-#define thm_lso_tcp_flag_first_width 12
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_WIDTH 12
 /* default value of bitfield lso_tcp_flag_first[b:0] */
-#define thm_lso_tcp_flag_first_default 0x0
+#define HW_ATL_THM_LSO_TCP_FLAG_FIRST_DEFAULT 0x0
 
 /* tx lso_tcp_flag_last[b:0] bitfield definitions
  * preprocessor definitions for the bitfield "lso_tcp_flag_last[b:0]".
@@ -1520,17 +1528,17 @@
  */
 
 /* register address for bitfield lso_tcp_flag_last[b:0] */
-#define thm_lso_tcp_flag_last_adr 0x00007824
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_ADR 0x00007824
 /* bitmask for bitfield lso_tcp_flag_last[b:0] */
-#define thm_lso_tcp_flag_last_msk 0x00000fff
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_MSK 0x00000fff
 /* inverted bitmask for bitfield lso_tcp_flag_last[b:0] */
-#define thm_lso_tcp_flag_last_mskn 0xfffff000
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_MSKN 0xfffff000
 /* lower bit position of bitfield lso_tcp_flag_last[b:0] */
-#define thm_lso_tcp_flag_last_shift 0
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_SHIFT 0
 /* width of bitfield lso_tcp_flag_last[b:0] */
-#define thm_lso_tcp_flag_last_width 12
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_WIDTH 12
 /* default value of bitfield lso_tcp_flag_last[b:0] */
-#define thm_lso_tcp_flag_last_default 0x0
+#define HW_ATL_THM_LSO_TCP_FLAG_LAST_DEFAULT 0x0
 
 /* tx lso_tcp_flag_mid[b:0] bitfield definitions
  * preprocessor definitions for the bitfield "lso_tcp_flag_mid[b:0]".
@@ -1538,17 +1546,17 @@
  */
 
 /* Register address for bitfield lro_rsc_max[1F:0] */
-#define rpo_lro_rsc_max_adr 0x00005598
+#define HW_ATL_RPO_LRO_RSC_MAX_ADR 0x00005598
 /* Bitmask for bitfield lro_rsc_max[1F:0] */
-#define rpo_lro_rsc_max_msk 0xFFFFFFFF
+#define HW_ATL_RPO_LRO_RSC_MAX_MSK 0xFFFFFFFF
 /* Inverted bitmask for bitfield lro_rsc_max[1F:0] */
-#define rpo_lro_rsc_max_mskn 0x00000000
+#define HW_ATL_RPO_LRO_RSC_MAX_MSKN 0x00000000
 /* Lower bit position of bitfield lro_rsc_max[1F:0] */
-#define rpo_lro_rsc_max_shift 0
+#define HW_ATL_RPO_LRO_RSC_MAX_SHIFT 0
 /* Width of bitfield lro_rsc_max[1F:0] */
-#define rpo_lro_rsc_max_width 32
+#define HW_ATL_RPO_LRO_RSC_MAX_WIDTH 32
 /* Default value of bitfield lro_rsc_max[1F:0] */
-#define rpo_lro_rsc_max_default 0x0
+#define HW_ATL_RPO_LRO_RSC_MAX_DEFAULT 0x0
 
 /* RX lro_en[1F:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "lro_en[1F:0]".
@@ -1556,17 +1564,17 @@
  */
 
 /* Register address for bitfield lro_en[1F:0] */
-#define rpo_lro_en_adr 0x00005590
+#define HW_ATL_RPO_LRO_EN_ADR 0x00005590
 /* Bitmask for bitfield lro_en[1F:0] */
-#define rpo_lro_en_msk 0xFFFFFFFF
+#define HW_ATL_RPO_LRO_EN_MSK 0xFFFFFFFF
 /* Inverted bitmask for bitfield lro_en[1F:0] */
-#define rpo_lro_en_mskn 0x00000000
+#define HW_ATL_RPO_LRO_EN_MSKN 0x00000000
 /* Lower bit position of bitfield lro_en[1F:0] */
-#define rpo_lro_en_shift 0
+#define HW_ATL_RPO_LRO_EN_SHIFT 0
 /* Width of bitfield lro_en[1F:0] */
-#define rpo_lro_en_width 32
+#define HW_ATL_RPO_LRO_EN_WIDTH 32
 /* Default value of bitfield lro_en[1F:0] */
-#define rpo_lro_en_default 0x0
+#define HW_ATL_RPO_LRO_EN_DEFAULT 0x0
 
 /* RX lro_ptopt_en Bitfield Definitions
  * Preprocessor definitions for the bitfield "lro_ptopt_en".
@@ -1574,17 +1582,17 @@
  */
 
 /* Register address for bitfield lro_ptopt_en */
-#define rpo_lro_ptopt_en_adr 0x00005594
+#define HW_ATL_RPO_LRO_PTOPT_EN_ADR 0x00005594
 /* Bitmask for bitfield lro_ptopt_en */
-#define rpo_lro_ptopt_en_msk 0x00008000
+#define HW_ATL_RPO_LRO_PTOPT_EN_MSK 0x00008000
 /* Inverted bitmask for bitfield lro_ptopt_en */
-#define rpo_lro_ptopt_en_mskn 0xFFFF7FFF
+#define HW_ATL_RPO_LRO_PTOPT_EN_MSKN 0xFFFF7FFF
 /* Lower bit position of bitfield lro_ptopt_en */
-#define rpo_lro_ptopt_en_shift 15
+#define HW_ATL_RPO_LRO_PTOPT_EN_SHIFT 15
 /* Width of bitfield lro_ptopt_en */
-#define rpo_lro_ptopt_en_width 1
+#define HW_ATL_RPO_LRO_PTOPT_EN_WIDTH 1
 /* Default value of bitfield lro_ptopt_en */
-#define rpo_lro_ptopt_en_defalt 0x1
+#define HW_ATL_RPO_LRO_PTOPT_EN_DEFALT 0x1
 
 /* RX lro_q_ses_lmt Bitfield Definitions
  * Preprocessor definitions for the bitfield "lro_q_ses_lmt".
@@ -1592,17 +1600,17 @@
  */
 
 /* Register address for bitfield lro_q_ses_lmt */
-#define rpo_lro_qses_lmt_adr 0x00005594
+#define HW_ATL_RPO_LRO_QSES_LMT_ADR 0x00005594
 /* Bitmask for bitfield lro_q_ses_lmt */
-#define rpo_lro_qses_lmt_msk 0x00003000
+#define HW_ATL_RPO_LRO_QSES_LMT_MSK 0x00003000
 /* Inverted bitmask for bitfield lro_q_ses_lmt */
-#define rpo_lro_qses_lmt_mskn 0xFFFFCFFF
+#define HW_ATL_RPO_LRO_QSES_LMT_MSKN 0xFFFFCFFF
 /* Lower bit position of bitfield lro_q_ses_lmt */
-#define rpo_lro_qses_lmt_shift 12
+#define HW_ATL_RPO_LRO_QSES_LMT_SHIFT 12
 /* Width of bitfield lro_q_ses_lmt */
-#define rpo_lro_qses_lmt_width 2
+#define HW_ATL_RPO_LRO_QSES_LMT_WIDTH 2
 /* Default value of bitfield lro_q_ses_lmt */
-#define rpo_lro_qses_lmt_default 0x1
+#define HW_ATL_RPO_LRO_QSES_LMT_DEFAULT 0x1
 
 /* RX lro_tot_dsc_lmt[1:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "lro_tot_dsc_lmt[1:0]".
@@ -1610,17 +1618,17 @@
  */
 
 /* Register address for bitfield lro_tot_dsc_lmt[1:0] */
-#define rpo_lro_tot_dsc_lmt_adr 0x00005594
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_ADR 0x00005594
 /* Bitmask for bitfield lro_tot_dsc_lmt[1:0] */
-#define rpo_lro_tot_dsc_lmt_msk 0x00000060
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_MSK 0x00000060
 /* Inverted bitmask for bitfield lro_tot_dsc_lmt[1:0] */
-#define rpo_lro_tot_dsc_lmt_mskn 0xFFFFFF9F
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_MSKN 0xFFFFFF9F
 /* Lower bit position of bitfield lro_tot_dsc_lmt[1:0] */
-#define rpo_lro_tot_dsc_lmt_shift 5
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_SHIFT 5
 /* Width of bitfield lro_tot_dsc_lmt[1:0] */
-#define rpo_lro_tot_dsc_lmt_width 2
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_WIDTH 2
 /* Default value of bitfield lro_tot_dsc_lmt[1:0] */
-#define rpo_lro_tot_dsc_lmt_defalt 0x1
+#define HW_ATL_RPO_LRO_TOT_DSC_LMT_DEFALT 0x1
 
 /* RX lro_pkt_min[4:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "lro_pkt_min[4:0]".
@@ -1628,22 +1636,22 @@
  */
 
 /* Register address for bitfield lro_pkt_min[4:0] */
-#define rpo_lro_pkt_min_adr 0x00005594
+#define HW_ATL_RPO_LRO_PKT_MIN_ADR 0x00005594
 /* Bitmask for bitfield lro_pkt_min[4:0] */
-#define rpo_lro_pkt_min_msk 0x0000001F
+#define HW_ATL_RPO_LRO_PKT_MIN_MSK 0x0000001F
 /* Inverted bitmask for bitfield lro_pkt_min[4:0] */
-#define rpo_lro_pkt_min_mskn 0xFFFFFFE0
+#define HW_ATL_RPO_LRO_PKT_MIN_MSKN 0xFFFFFFE0
 /* Lower bit position of bitfield lro_pkt_min[4:0] */
-#define rpo_lro_pkt_min_shift 0
+#define HW_ATL_RPO_LRO_PKT_MIN_SHIFT 0
 /* Width of bitfield lro_pkt_min[4:0] */
-#define rpo_lro_pkt_min_width 5
+#define HW_ATL_RPO_LRO_PKT_MIN_WIDTH 5
 /* Default value of bitfield lro_pkt_min[4:0] */
-#define rpo_lro_pkt_min_default 0x8
+#define HW_ATL_RPO_LRO_PKT_MIN_DEFAULT 0x8
 
 /* Width of bitfield lro{L}_des_max[1:0] */
-#define rpo_lro_ldes_max_width 2
+#define HW_ATL_RPO_LRO_LDES_MAX_WIDTH 2
 /* Default value of bitfield lro{L}_des_max[1:0] */
-#define rpo_lro_ldes_max_default 0x0
+#define HW_ATL_RPO_LRO_LDES_MAX_DEFAULT 0x0
 
 /* RX lro_tb_div[11:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "lro_tb_div[11:0]".
@@ -1651,17 +1659,17 @@
  */
 
 /* Register address for bitfield lro_tb_div[11:0] */
-#define rpo_lro_tb_div_adr 0x00005620
+#define HW_ATL_RPO_LRO_TB_DIV_ADR 0x00005620
 /* Bitmask for bitfield lro_tb_div[11:0] */
-#define rpo_lro_tb_div_msk 0xFFF00000
+#define HW_ATL_RPO_LRO_TB_DIV_MSK 0xFFF00000
 /* Inverted bitmask for bitfield lro_tb_div[11:0] */
-#define rpo_lro_tb_div_mskn 0x000FFFFF
+#define HW_ATL_RPO_LRO_TB_DIV_MSKN 0x000FFFFF
 /* Lower bit position of bitfield lro_tb_div[11:0] */
-#define rpo_lro_tb_div_shift 20
+#define HW_ATL_RPO_LRO_TB_DIV_SHIFT 20
 /* Width of bitfield lro_tb_div[11:0] */
-#define rpo_lro_tb_div_width 12
+#define HW_ATL_RPO_LRO_TB_DIV_WIDTH 12
 /* Default value of bitfield lro_tb_div[11:0] */
-#define rpo_lro_tb_div_default 0xC35
+#define HW_ATL_RPO_LRO_TB_DIV_DEFAULT 0xC35
 
 /* RX lro_ina_ival[9:0] Bitfield Definitions
  *   Preprocessor definitions for the bitfield "lro_ina_ival[9:0]".
@@ -1669,17 +1677,17 @@
  */
 
 /* Register address for bitfield lro_ina_ival[9:0] */
-#define rpo_lro_ina_ival_adr 0x00005620
+#define HW_ATL_RPO_LRO_INA_IVAL_ADR 0x00005620
 /* Bitmask for bitfield lro_ina_ival[9:0] */
-#define rpo_lro_ina_ival_msk 0x000FFC00
+#define HW_ATL_RPO_LRO_INA_IVAL_MSK 0x000FFC00
 /* Inverted bitmask for bitfield lro_ina_ival[9:0] */
-#define rpo_lro_ina_ival_mskn 0xFFF003FF
+#define HW_ATL_RPO_LRO_INA_IVAL_MSKN 0xFFF003FF
 /* Lower bit position of bitfield lro_ina_ival[9:0] */
-#define rpo_lro_ina_ival_shift 10
+#define HW_ATL_RPO_LRO_INA_IVAL_SHIFT 10
 /* Width of bitfield lro_ina_ival[9:0] */
-#define rpo_lro_ina_ival_width 10
+#define HW_ATL_RPO_LRO_INA_IVAL_WIDTH 10
 /* Default value of bitfield lro_ina_ival[9:0] */
-#define rpo_lro_ina_ival_default 0xA
+#define HW_ATL_RPO_LRO_INA_IVAL_DEFAULT 0xA
 
 /* RX lro_max_ival[9:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "lro_max_ival[9:0]".
@@ -1687,17 +1695,17 @@
  */
 
 /* Register address for bitfield lro_max_ival[9:0] */
-#define rpo_lro_max_ival_adr 0x00005620
+#define HW_ATL_RPO_LRO_MAX_IVAL_ADR 0x00005620
 /* Bitmask for bitfield lro_max_ival[9:0] */
-#define rpo_lro_max_ival_msk 0x000003FF
+#define HW_ATL_RPO_LRO_MAX_IVAL_MSK 0x000003FF
 /* Inverted bitmask for bitfield lro_max_ival[9:0] */
-#define rpo_lro_max_ival_mskn 0xFFFFFC00
+#define HW_ATL_RPO_LRO_MAX_IVAL_MSKN 0xFFFFFC00
 /* Lower bit position of bitfield lro_max_ival[9:0] */
-#define rpo_lro_max_ival_shift 0
+#define HW_ATL_RPO_LRO_MAX_IVAL_SHIFT 0
 /* Width of bitfield lro_max_ival[9:0] */
-#define rpo_lro_max_ival_width 10
+#define HW_ATL_RPO_LRO_MAX_IVAL_WIDTH 10
 /* Default value of bitfield lro_max_ival[9:0] */
-#define rpo_lro_max_ival_default 0x19
+#define HW_ATL_RPO_LRO_MAX_IVAL_DEFAULT 0x19
 
 /* TX dca{D}_cpuid[7:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "dca{D}_cpuid[7:0]".
@@ -1706,17 +1714,17 @@
  */
 
 /* Register address for bitfield dca{D}_cpuid[7:0] */
-#define tdm_dca_dcpuid_adr(dca) (0x00008400 + (dca) * 0x4)
+#define HW_ATL_TDM_DCA_DCPUID_ADR(dca) (0x00008400 + (dca) * 0x4)
 /* Bitmask for bitfield dca{D}_cpuid[7:0] */
-#define tdm_dca_dcpuid_msk 0x000000FF
+#define HW_ATL_TDM_DCA_DCPUID_MSK 0x000000FF
 /* Inverted bitmask for bitfield dca{D}_cpuid[7:0] */
-#define tdm_dca_dcpuid_mskn 0xFFFFFF00
+#define HW_ATL_TDM_DCA_DCPUID_MSKN 0xFFFFFF00
 /* Lower bit position of bitfield dca{D}_cpuid[7:0] */
-#define tdm_dca_dcpuid_shift 0
+#define HW_ATL_TDM_DCA_DCPUID_SHIFT 0
 /* Width of bitfield dca{D}_cpuid[7:0] */
-#define tdm_dca_dcpuid_width 8
+#define HW_ATL_TDM_DCA_DCPUID_WIDTH 8
 /* Default value of bitfield dca{D}_cpuid[7:0] */
-#define tdm_dca_dcpuid_default 0x0
+#define HW_ATL_TDM_DCA_DCPUID_DEFAULT 0x0
 
 /* TX dca{D}_desc_en Bitfield Definitions
  * Preprocessor definitions for the bitfield "dca{D}_desc_en".
@@ -1725,17 +1733,17 @@
  */
 
 /* Register address for bitfield dca{D}_desc_en */
-#define tdm_dca_ddesc_en_adr(dca) (0x00008400 + (dca) * 0x4)
+#define HW_ATL_TDM_DCA_DDESC_EN_ADR(dca) (0x00008400 + (dca) * 0x4)
 /* Bitmask for bitfield dca{D}_desc_en */
-#define tdm_dca_ddesc_en_msk 0x80000000
+#define HW_ATL_TDM_DCA_DDESC_EN_MSK 0x80000000
 /* Inverted bitmask for bitfield dca{D}_desc_en */
-#define tdm_dca_ddesc_en_mskn 0x7FFFFFFF
+#define HW_ATL_TDM_DCA_DDESC_EN_MSKN 0x7FFFFFFF
 /* Lower bit position of bitfield dca{D}_desc_en */
-#define tdm_dca_ddesc_en_shift 31
+#define HW_ATL_TDM_DCA_DDESC_EN_SHIFT 31
 /* Width of bitfield dca{D}_desc_en */
-#define tdm_dca_ddesc_en_width 1
+#define HW_ATL_TDM_DCA_DDESC_EN_WIDTH 1
 /* Default value of bitfield dca{D}_desc_en */
-#define tdm_dca_ddesc_en_default 0x0
+#define HW_ATL_TDM_DCA_DDESC_EN_DEFAULT 0x0
 
 /* TX desc{D}_en Bitfield Definitions
  * Preprocessor definitions for the bitfield "desc{D}_en".
@@ -1744,17 +1752,17 @@
  */
 
 /* Register address for bitfield desc{D}_en */
-#define tdm_desc_den_adr(descriptor) (0x00007C08 + (descriptor) * 0x40)
+#define HW_ATL_TDM_DESC_DEN_ADR(descriptor) (0x00007C08 + (descriptor) * 0x40)
 /* Bitmask for bitfield desc{D}_en */
-#define tdm_desc_den_msk 0x80000000
+#define HW_ATL_TDM_DESC_DEN_MSK 0x80000000
 /* Inverted bitmask for bitfield desc{D}_en */
-#define tdm_desc_den_mskn 0x7FFFFFFF
+#define HW_ATL_TDM_DESC_DEN_MSKN 0x7FFFFFFF
 /* Lower bit position of bitfield desc{D}_en */
-#define tdm_desc_den_shift 31
+#define HW_ATL_TDM_DESC_DEN_SHIFT 31
 /* Width of bitfield desc{D}_en */
-#define tdm_desc_den_width 1
+#define HW_ATL_TDM_DESC_DEN_WIDTH 1
 /* Default value of bitfield desc{D}_en */
-#define tdm_desc_den_default 0x0
+#define HW_ATL_TDM_DESC_DEN_DEFAULT 0x0
 
 /* TX desc{D}_hd[C:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "desc{D}_hd[C:0]".
@@ -1763,15 +1771,15 @@
  */
 
 /* Register address for bitfield desc{D}_hd[C:0] */
-#define tdm_desc_dhd_adr(descriptor) (0x00007C0C + (descriptor) * 0x40)
+#define HW_ATL_TDM_DESC_DHD_ADR(descriptor) (0x00007C0C + (descriptor) * 0x40)
 /* Bitmask for bitfield desc{D}_hd[C:0] */
-#define tdm_desc_dhd_msk 0x00001FFF
+#define HW_ATL_TDM_DESC_DHD_MSK 0x00001FFF
 /* Inverted bitmask for bitfield desc{D}_hd[C:0] */
-#define tdm_desc_dhd_mskn 0xFFFFE000
+#define HW_ATL_TDM_DESC_DHD_MSKN 0xFFFFE000
 /* Lower bit position of bitfield desc{D}_hd[C:0] */
-#define tdm_desc_dhd_shift 0
+#define HW_ATL_TDM_DESC_DHD_SHIFT 0
 /* Width of bitfield desc{D}_hd[C:0] */
-#define tdm_desc_dhd_width 13
+#define HW_ATL_TDM_DESC_DHD_WIDTH 13
 
 /* TX desc{D}_len[9:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "desc{D}_len[9:0]".
@@ -1780,17 +1788,17 @@
  */
 
 /* Register address for bitfield desc{D}_len[9:0] */
-#define tdm_desc_dlen_adr(descriptor) (0x00007C08 + (descriptor) * 0x40)
+#define HW_ATL_TDM_DESC_DLEN_ADR(descriptor) (0x00007C08 + (descriptor) * 0x40)
 /* Bitmask for bitfield desc{D}_len[9:0] */
-#define tdm_desc_dlen_msk 0x00001FF8
+#define HW_ATL_TDM_DESC_DLEN_MSK 0x00001FF8
 /* Inverted bitmask for bitfield desc{D}_len[9:0] */
-#define tdm_desc_dlen_mskn 0xFFFFE007
+#define HW_ATL_TDM_DESC_DLEN_MSKN 0xFFFFE007
 /* Lower bit position of bitfield desc{D}_len[9:0] */
-#define tdm_desc_dlen_shift 3
+#define HW_ATL_TDM_DESC_DLEN_SHIFT 3
 /* Width of bitfield desc{D}_len[9:0] */
-#define tdm_desc_dlen_width 10
+#define HW_ATL_TDM_DESC_DLEN_WIDTH 10
 /* Default value of bitfield desc{D}_len[9:0] */
-#define tdm_desc_dlen_default 0x0
+#define HW_ATL_TDM_DESC_DLEN_DEFAULT 0x0
 
 /* TX desc{D}_wrb_thresh[6:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "desc{D}_wrb_thresh[6:0]".
@@ -1799,18 +1807,18 @@
  */
 
 /* Register address for bitfield desc{D}_wrb_thresh[6:0] */
-#define tdm_desc_dwrb_thresh_adr(descriptor) \
+#define HW_ATL_TDM_DESC_DWRB_THRESH_ADR(descriptor) \
 	(0x00007C18 + (descriptor) * 0x40)
 /* Bitmask for bitfield desc{D}_wrb_thresh[6:0] */
-#define tdm_desc_dwrb_thresh_msk 0x00007F00
+#define HW_ATL_TDM_DESC_DWRB_THRESH_MSK 0x00007F00
 /* Inverted bitmask for bitfield desc{D}_wrb_thresh[6:0] */
-#define tdm_desc_dwrb_thresh_mskn 0xFFFF80FF
+#define HW_ATL_TDM_DESC_DWRB_THRESH_MSKN 0xFFFF80FF
 /* Lower bit position of bitfield desc{D}_wrb_thresh[6:0] */
-#define tdm_desc_dwrb_thresh_shift 8
+#define HW_ATL_TDM_DESC_DWRB_THRESH_SHIFT 8
 /* Width of bitfield desc{D}_wrb_thresh[6:0] */
-#define tdm_desc_dwrb_thresh_width 7
+#define HW_ATL_TDM_DESC_DWRB_THRESH_WIDTH 7
 /* Default value of bitfield desc{D}_wrb_thresh[6:0] */
-#define tdm_desc_dwrb_thresh_default 0x0
+#define HW_ATL_TDM_DESC_DWRB_THRESH_DEFAULT 0x0
 
 /* TX tdm_int_mod_en Bitfield Definitions
  * Preprocessor definitions for the bitfield "tdm_int_mod_en".
@@ -1818,34 +1826,34 @@
  */
 
 /* Register address for bitfield tdm_int_mod_en */
-#define tdm_int_mod_en_adr 0x00007B40
+#define HW_ATL_TDM_INT_MOD_EN_ADR 0x00007B40
 /* Bitmask for bitfield tdm_int_mod_en */
-#define tdm_int_mod_en_msk 0x00000010
+#define HW_ATL_TDM_INT_MOD_EN_MSK 0x00000010
 /* Inverted bitmask for bitfield tdm_int_mod_en */
-#define tdm_int_mod_en_mskn 0xFFFFFFEF
+#define HW_ATL_TDM_INT_MOD_EN_MSKN 0xFFFFFFEF
 /* Lower bit position of bitfield tdm_int_mod_en */
-#define tdm_int_mod_en_shift 4
+#define HW_ATL_TDM_INT_MOD_EN_SHIFT 4
 /* Width of bitfield tdm_int_mod_en */
-#define tdm_int_mod_en_width 1
+#define HW_ATL_TDM_INT_MOD_EN_WIDTH 1
 /* Default value of bitfield tdm_int_mod_en */
-#define tdm_int_mod_en_default 0x0
+#define HW_ATL_TDM_INT_MOD_EN_DEFAULT 0x0
 
 /* TX lso_tcp_flag_mid[B:0] Bitfield Definitions
  * Preprocessor definitions for the bitfield "lso_tcp_flag_mid[B:0]".
  * PORT="pif_thm_lso_tcp_flag_mid_i[11:0]"
  */
 /* register address for bitfield lso_tcp_flag_mid[b:0] */
-#define thm_lso_tcp_flag_mid_adr 0x00007820
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_ADR 0x00007820
 /* bitmask for bitfield lso_tcp_flag_mid[b:0] */
-#define thm_lso_tcp_flag_mid_msk 0x0fff0000
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_MSK 0x0fff0000
 /* inverted bitmask for bitfield lso_tcp_flag_mid[b:0] */
-#define thm_lso_tcp_flag_mid_mskn 0xf000ffff
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_MSKN 0xf000ffff
 /* lower bit position of bitfield lso_tcp_flag_mid[b:0] */
-#define thm_lso_tcp_flag_mid_shift 16
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_SHIFT 16
 /* width of bitfield lso_tcp_flag_mid[b:0] */
-#define thm_lso_tcp_flag_mid_width 12
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_WIDTH 12
 /* default value of bitfield lso_tcp_flag_mid[b:0] */
-#define thm_lso_tcp_flag_mid_default 0x0
+#define HW_ATL_THM_LSO_TCP_FLAG_MID_DEFAULT 0x0
 
 /* tx tx_buf_en bitfield definitions
  * preprocessor definitions for the bitfield "tx_buf_en".
@@ -1853,17 +1861,17 @@
  */
 
 /* register address for bitfield tx_buf_en */
-#define tpb_tx_buf_en_adr 0x00007900
+#define HW_ATL_TPB_TX_BUF_EN_ADR 0x00007900
 /* bitmask for bitfield tx_buf_en */
-#define tpb_tx_buf_en_msk 0x00000001
+#define HW_ATL_TPB_TX_BUF_EN_MSK 0x00000001
 /* inverted bitmask for bitfield tx_buf_en */
-#define tpb_tx_buf_en_mskn 0xfffffffe
+#define HW_ATL_TPB_TX_BUF_EN_MSKN 0xfffffffe
 /* lower bit position of bitfield tx_buf_en */
-#define tpb_tx_buf_en_shift 0
+#define HW_ATL_TPB_TX_BUF_EN_SHIFT 0
 /* width of bitfield tx_buf_en */
-#define tpb_tx_buf_en_width 1
+#define HW_ATL_TPB_TX_BUF_EN_WIDTH 1
 /* default value of bitfield tx_buf_en */
-#define tpb_tx_buf_en_default 0x0
+#define HW_ATL_TPB_TX_BUF_EN_DEFAULT 0x0
 
 /* tx tx{b}_hi_thresh[c:0] bitfield definitions
  * preprocessor definitions for the bitfield "tx{b}_hi_thresh[c:0]".
@@ -1872,17 +1880,17 @@
  */
 
 /* register address for bitfield tx{b}_hi_thresh[c:0] */
-#define tpb_txbhi_thresh_adr(buffer) (0x00007914 + (buffer) * 0x10)
+#define HW_ATL_TPB_TXBHI_THRESH_ADR(buffer) (0x00007914 + (buffer) * 0x10)
 /* bitmask for bitfield tx{b}_hi_thresh[c:0] */
-#define tpb_txbhi_thresh_msk 0x1fff0000
+#define HW_ATL_TPB_TXBHI_THRESH_MSK 0x1fff0000
 /* inverted bitmask for bitfield tx{b}_hi_thresh[c:0] */
-#define tpb_txbhi_thresh_mskn 0xe000ffff
+#define HW_ATL_TPB_TXBHI_THRESH_MSKN 0xe000ffff
 /* lower bit position of bitfield tx{b}_hi_thresh[c:0] */
-#define tpb_txbhi_thresh_shift 16
+#define HW_ATL_TPB_TXBHI_THRESH_SHIFT 16
 /* width of bitfield tx{b}_hi_thresh[c:0] */
-#define tpb_txbhi_thresh_width 13
+#define HW_ATL_TPB_TXBHI_THRESH_WIDTH 13
 /* default value of bitfield tx{b}_hi_thresh[c:0] */
-#define tpb_txbhi_thresh_default 0x0
+#define HW_ATL_TPB_TXBHI_THRESH_DEFAULT 0x0
 
 /* tx tx{b}_lo_thresh[c:0] bitfield definitions
  * preprocessor definitions for the bitfield "tx{b}_lo_thresh[c:0]".
@@ -1891,17 +1899,17 @@
  */
 
 /* register address for bitfield tx{b}_lo_thresh[c:0] */
-#define tpb_txblo_thresh_adr(buffer) (0x00007914 + (buffer) * 0x10)
+#define HW_ATL_TPB_TXBLO_THRESH_ADR(buffer) (0x00007914 + (buffer) * 0x10)
 /* bitmask for bitfield tx{b}_lo_thresh[c:0] */
-#define tpb_txblo_thresh_msk 0x00001fff
+#define HW_ATL_TPB_TXBLO_THRESH_MSK 0x00001fff
 /* inverted bitmask for bitfield tx{b}_lo_thresh[c:0] */
-#define tpb_txblo_thresh_mskn 0xffffe000
+#define HW_ATL_TPB_TXBLO_THRESH_MSKN 0xffffe000
 /* lower bit position of bitfield tx{b}_lo_thresh[c:0] */
-#define tpb_txblo_thresh_shift 0
+#define HW_ATL_TPB_TXBLO_THRESH_SHIFT 0
 /* width of bitfield tx{b}_lo_thresh[c:0] */
-#define tpb_txblo_thresh_width 13
+#define HW_ATL_TPB_TXBLO_THRESH_WIDTH 13
 /* default value of bitfield tx{b}_lo_thresh[c:0] */
-#define tpb_txblo_thresh_default 0x0
+#define HW_ATL_TPB_TXBLO_THRESH_DEFAULT 0x0
 
 /* tx dma_sys_loopback bitfield definitions
  * preprocessor definitions for the bitfield "dma_sys_loopback".
@@ -1909,17 +1917,17 @@
  */
 
 /* register address for bitfield dma_sys_loopback */
-#define tpb_dma_sys_lbk_adr 0x00007000
+#define HW_ATL_TPB_DMA_SYS_LBK_ADR 0x00007000
 /* bitmask for bitfield dma_sys_loopback */
-#define tpb_dma_sys_lbk_msk 0x00000040
+#define HW_ATL_TPB_DMA_SYS_LBK_MSK 0x00000040
 /* inverted bitmask for bitfield dma_sys_loopback */
-#define tpb_dma_sys_lbk_mskn 0xffffffbf
+#define HW_ATL_TPB_DMA_SYS_LBK_MSKN 0xffffffbf
 /* lower bit position of bitfield dma_sys_loopback */
-#define tpb_dma_sys_lbk_shift 6
+#define HW_ATL_TPB_DMA_SYS_LBK_SHIFT 6
 /* width of bitfield dma_sys_loopback */
-#define tpb_dma_sys_lbk_width 1
+#define HW_ATL_TPB_DMA_SYS_LBK_WIDTH 1
 /* default value of bitfield dma_sys_loopback */
-#define tpb_dma_sys_lbk_default 0x0
+#define HW_ATL_TPB_DMA_SYS_LBK_DEFAULT 0x0
 
 /* tx tx{b}_buf_size[7:0] bitfield definitions
  * preprocessor definitions for the bitfield "tx{b}_buf_size[7:0]".
@@ -1928,17 +1936,17 @@
  */
 
 /* register address for bitfield tx{b}_buf_size[7:0] */
-#define tpb_txbbuf_size_adr(buffer) (0x00007910 + (buffer) * 0x10)
+#define HW_ATL_TPB_TXBBUF_SIZE_ADR(buffer) (0x00007910 + (buffer) * 0x10)
 /* bitmask for bitfield tx{b}_buf_size[7:0] */
-#define tpb_txbbuf_size_msk 0x000000ff
+#define HW_ATL_TPB_TXBBUF_SIZE_MSK 0x000000ff
 /* inverted bitmask for bitfield tx{b}_buf_size[7:0] */
-#define tpb_txbbuf_size_mskn 0xffffff00
+#define HW_ATL_TPB_TXBBUF_SIZE_MSKN 0xffffff00
 /* lower bit position of bitfield tx{b}_buf_size[7:0] */
-#define tpb_txbbuf_size_shift 0
+#define HW_ATL_TPB_TXBBUF_SIZE_SHIFT 0
 /* width of bitfield tx{b}_buf_size[7:0] */
-#define tpb_txbbuf_size_width 8
+#define HW_ATL_TPB_TXBBUF_SIZE_WIDTH 8
 /* default value of bitfield tx{b}_buf_size[7:0] */
-#define tpb_txbbuf_size_default 0x0
+#define HW_ATL_TPB_TXBBUF_SIZE_DEFAULT 0x0
 
 /* tx tx_scp_ins_en bitfield definitions
  * preprocessor definitions for the bitfield "tx_scp_ins_en".
@@ -1946,17 +1954,17 @@
  */
 
 /* register address for bitfield tx_scp_ins_en */
-#define tpb_tx_scp_ins_en_adr 0x00007900
+#define HW_ATL_TPB_TX_SCP_INS_EN_ADR 0x00007900
 /* bitmask for bitfield tx_scp_ins_en */
-#define tpb_tx_scp_ins_en_msk 0x00000004
+#define HW_ATL_TPB_TX_SCP_INS_EN_MSK 0x00000004
 /* inverted bitmask for bitfield tx_scp_ins_en */
-#define tpb_tx_scp_ins_en_mskn 0xfffffffb
+#define HW_ATL_TPB_TX_SCP_INS_EN_MSKN 0xfffffffb
 /* lower bit position of bitfield tx_scp_ins_en */
-#define tpb_tx_scp_ins_en_shift 2
+#define HW_ATL_TPB_TX_SCP_INS_EN_SHIFT 2
 /* width of bitfield tx_scp_ins_en */
-#define tpb_tx_scp_ins_en_width 1
+#define HW_ATL_TPB_TX_SCP_INS_EN_WIDTH 1
 /* default value of bitfield tx_scp_ins_en */
-#define tpb_tx_scp_ins_en_default 0x0
+#define HW_ATL_TPB_TX_SCP_INS_EN_DEFAULT 0x0
 
 /* tx ipv4_chk_en bitfield definitions
  * preprocessor definitions for the bitfield "ipv4_chk_en".
@@ -1964,17 +1972,17 @@
  */
 
 /* register address for bitfield ipv4_chk_en */
-#define tpo_ipv4chk_en_adr 0x00007800
+#define HW_ATL_TPO_IPV4CHK_EN_ADR 0x00007800
 /* bitmask for bitfield ipv4_chk_en */
-#define tpo_ipv4chk_en_msk 0x00000002
+#define HW_ATL_TPO_IPV4CHK_EN_MSK 0x00000002
 /* inverted bitmask for bitfield ipv4_chk_en */
-#define tpo_ipv4chk_en_mskn 0xfffffffd
+#define HW_ATL_TPO_IPV4CHK_EN_MSKN 0xfffffffd
 /* lower bit position of bitfield ipv4_chk_en */
-#define tpo_ipv4chk_en_shift 1
+#define HW_ATL_TPO_IPV4CHK_EN_SHIFT 1
 /* width of bitfield ipv4_chk_en */
-#define tpo_ipv4chk_en_width 1
+#define HW_ATL_TPO_IPV4CHK_EN_WIDTH 1
 /* default value of bitfield ipv4_chk_en */
-#define tpo_ipv4chk_en_default 0x0
+#define HW_ATL_TPO_IPV4CHK_EN_DEFAULT 0x0
 
 /* tx l4_chk_en bitfield definitions
  * preprocessor definitions for the bitfield "l4_chk_en".
@@ -1982,17 +1990,17 @@
  */
 
 /* register address for bitfield l4_chk_en */
-#define tpol4chk_en_adr 0x00007800
+#define HW_ATL_TPOL4CHK_EN_ADR 0x00007800
 /* bitmask for bitfield l4_chk_en */
-#define tpol4chk_en_msk 0x00000001
+#define HW_ATL_TPOL4CHK_EN_MSK 0x00000001
 /* inverted bitmask for bitfield l4_chk_en */
-#define tpol4chk_en_mskn 0xfffffffe
+#define HW_ATL_TPOL4CHK_EN_MSKN 0xfffffffe
 /* lower bit position of bitfield l4_chk_en */
-#define tpol4chk_en_shift 0
+#define HW_ATL_TPOL4CHK_EN_SHIFT 0
 /* width of bitfield l4_chk_en */
-#define tpol4chk_en_width 1
+#define HW_ATL_TPOL4CHK_EN_WIDTH 1
 /* default value of bitfield l4_chk_en */
-#define tpol4chk_en_default 0x0
+#define HW_ATL_TPOL4CHK_EN_DEFAULT 0x0
 
 /* tx pkt_sys_loopback bitfield definitions
  * preprocessor definitions for the bitfield "pkt_sys_loopback".
@@ -2000,17 +2008,17 @@
  */
 
 /* register address for bitfield pkt_sys_loopback */
-#define tpo_pkt_sys_lbk_adr 0x00007000
+#define HW_ATL_TPO_PKT_SYS_LBK_ADR 0x00007000
 /* bitmask for bitfield pkt_sys_loopback */
-#define tpo_pkt_sys_lbk_msk 0x00000080
+#define HW_ATL_TPO_PKT_SYS_LBK_MSK 0x00000080
 /* inverted bitmask for bitfield pkt_sys_loopback */
-#define tpo_pkt_sys_lbk_mskn 0xffffff7f
+#define HW_ATL_TPO_PKT_SYS_LBK_MSKN 0xffffff7f
 /* lower bit position of bitfield pkt_sys_loopback */
-#define tpo_pkt_sys_lbk_shift 7
+#define HW_ATL_TPO_PKT_SYS_LBK_SHIFT 7
 /* width of bitfield pkt_sys_loopback */
-#define tpo_pkt_sys_lbk_width 1
+#define HW_ATL_TPO_PKT_SYS_LBK_WIDTH 1
 /* default value of bitfield pkt_sys_loopback */
-#define tpo_pkt_sys_lbk_default 0x0
+#define HW_ATL_TPO_PKT_SYS_LBK_DEFAULT 0x0
 
 /* tx data_tc_arb_mode bitfield definitions
  * preprocessor definitions for the bitfield "data_tc_arb_mode".
@@ -2018,17 +2026,17 @@
  */
 
 /* register address for bitfield data_tc_arb_mode */
-#define tps_data_tc_arb_mode_adr 0x00007100
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_ADR 0x00007100
 /* bitmask for bitfield data_tc_arb_mode */
-#define tps_data_tc_arb_mode_msk 0x00000001
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_MSK 0x00000001
 /* inverted bitmask for bitfield data_tc_arb_mode */
-#define tps_data_tc_arb_mode_mskn 0xfffffffe
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_MSKN 0xfffffffe
 /* lower bit position of bitfield data_tc_arb_mode */
-#define tps_data_tc_arb_mode_shift 0
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_SHIFT 0
 /* width of bitfield data_tc_arb_mode */
-#define tps_data_tc_arb_mode_width 1
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_WIDTH 1
 /* default value of bitfield data_tc_arb_mode */
-#define tps_data_tc_arb_mode_default 0x0
+#define HW_ATL_TPS_DATA_TC_ARB_MODE_DEFAULT 0x0
 
 /* tx desc_rate_ta_rst bitfield definitions
  * preprocessor definitions for the bitfield "desc_rate_ta_rst".
@@ -2036,17 +2044,17 @@
  */
 
 /* register address for bitfield desc_rate_ta_rst */
-#define tps_desc_rate_ta_rst_adr 0x00007310
+#define HW_ATL_TPS_DESC_RATE_TA_RST_ADR 0x00007310
 /* bitmask for bitfield desc_rate_ta_rst */
-#define tps_desc_rate_ta_rst_msk 0x80000000
+#define HW_ATL_TPS_DESC_RATE_TA_RST_MSK 0x80000000
 /* inverted bitmask for bitfield desc_rate_ta_rst */
-#define tps_desc_rate_ta_rst_mskn 0x7fffffff
+#define HW_ATL_TPS_DESC_RATE_TA_RST_MSKN 0x7fffffff
 /* lower bit position of bitfield desc_rate_ta_rst */
-#define tps_desc_rate_ta_rst_shift 31
+#define HW_ATL_TPS_DESC_RATE_TA_RST_SHIFT 31
 /* width of bitfield desc_rate_ta_rst */
-#define tps_desc_rate_ta_rst_width 1
+#define HW_ATL_TPS_DESC_RATE_TA_RST_WIDTH 1
 /* default value of bitfield desc_rate_ta_rst */
-#define tps_desc_rate_ta_rst_default 0x0
+#define HW_ATL_TPS_DESC_RATE_TA_RST_DEFAULT 0x0
 
 /* tx desc_rate_limit[a:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc_rate_limit[a:0]".
@@ -2054,17 +2062,17 @@
  */
 
 /* register address for bitfield desc_rate_limit[a:0] */
-#define tps_desc_rate_lim_adr 0x00007310
+#define HW_ATL_TPS_DESC_RATE_LIM_ADR 0x00007310
 /* bitmask for bitfield desc_rate_limit[a:0] */
-#define tps_desc_rate_lim_msk 0x000007ff
+#define HW_ATL_TPS_DESC_RATE_LIM_MSK 0x000007ff
 /* inverted bitmask for bitfield desc_rate_limit[a:0] */
-#define tps_desc_rate_lim_mskn 0xfffff800
+#define HW_ATL_TPS_DESC_RATE_LIM_MSKN 0xfffff800
 /* lower bit position of bitfield desc_rate_limit[a:0] */
-#define tps_desc_rate_lim_shift 0
+#define HW_ATL_TPS_DESC_RATE_LIM_SHIFT 0
 /* width of bitfield desc_rate_limit[a:0] */
-#define tps_desc_rate_lim_width 11
+#define HW_ATL_TPS_DESC_RATE_LIM_WIDTH 11
 /* default value of bitfield desc_rate_limit[a:0] */
-#define tps_desc_rate_lim_default 0x0
+#define HW_ATL_TPS_DESC_RATE_LIM_DEFAULT 0x0
 
 /* tx desc_tc_arb_mode[1:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc_tc_arb_mode[1:0]".
@@ -2072,17 +2080,17 @@
  */
 
 /* register address for bitfield desc_tc_arb_mode[1:0] */
-#define tps_desc_tc_arb_mode_adr 0x00007200
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_ADR 0x00007200
 /* bitmask for bitfield desc_tc_arb_mode[1:0] */
-#define tps_desc_tc_arb_mode_msk 0x00000003
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_MSK 0x00000003
 /* inverted bitmask for bitfield desc_tc_arb_mode[1:0] */
-#define tps_desc_tc_arb_mode_mskn 0xfffffffc
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_MSKN 0xfffffffc
 /* lower bit position of bitfield desc_tc_arb_mode[1:0] */
-#define tps_desc_tc_arb_mode_shift 0
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_SHIFT 0
 /* width of bitfield desc_tc_arb_mode[1:0] */
-#define tps_desc_tc_arb_mode_width 2
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_WIDTH 2
 /* default value of bitfield desc_tc_arb_mode[1:0] */
-#define tps_desc_tc_arb_mode_default 0x0
+#define HW_ATL_TPS_DESC_TC_ARB_MODE_DEFAULT 0x0
 
 /* tx desc_tc{t}_credit_max[b:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc_tc{t}_credit_max[b:0]".
@@ -2091,17 +2099,17 @@
  */
 
 /* register address for bitfield desc_tc{t}_credit_max[b:0] */
-#define tps_desc_tctcredit_max_adr(tc) (0x00007210 + (tc) * 0x4)
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_ADR(tc) (0x00007210 + (tc) * 0x4)
 /* bitmask for bitfield desc_tc{t}_credit_max[b:0] */
-#define tps_desc_tctcredit_max_msk 0x0fff0000
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_MSK 0x0fff0000
 /* inverted bitmask for bitfield desc_tc{t}_credit_max[b:0] */
-#define tps_desc_tctcredit_max_mskn 0xf000ffff
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_MSKN 0xf000ffff
 /* lower bit position of bitfield desc_tc{t}_credit_max[b:0] */
-#define tps_desc_tctcredit_max_shift 16
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_SHIFT 16
 /* width of bitfield desc_tc{t}_credit_max[b:0] */
-#define tps_desc_tctcredit_max_width 12
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_WIDTH 12
 /* default value of bitfield desc_tc{t}_credit_max[b:0] */
-#define tps_desc_tctcredit_max_default 0x0
+#define HW_ATL_TPS_DESC_TCTCREDIT_MAX_DEFAULT 0x0
 
 /* tx desc_tc{t}_weight[8:0] bitfield definitions
  * preprocessor definitions for the bitfield "desc_tc{t}_weight[8:0]".
@@ -2110,17 +2118,17 @@
  */
 
 /* register address for bitfield desc_tc{t}_weight[8:0] */
-#define tps_desc_tctweight_adr(tc) (0x00007210 + (tc) * 0x4)
+#define HW_ATL_TPS_DESC_TCTWEIGHT_ADR(tc) (0x00007210 + (tc) * 0x4)
 /* bitmask for bitfield desc_tc{t}_weight[8:0] */
-#define tps_desc_tctweight_msk 0x000001ff
+#define HW_ATL_TPS_DESC_TCTWEIGHT_MSK 0x000001ff
 /* inverted bitmask for bitfield desc_tc{t}_weight[8:0] */
-#define tps_desc_tctweight_mskn 0xfffffe00
+#define HW_ATL_TPS_DESC_TCTWEIGHT_MSKN 0xfffffe00
 /* lower bit position of bitfield desc_tc{t}_weight[8:0] */
-#define tps_desc_tctweight_shift 0
+#define HW_ATL_TPS_DESC_TCTWEIGHT_SHIFT 0
 /* width of bitfield desc_tc{t}_weight[8:0] */
-#define tps_desc_tctweight_width 9
+#define HW_ATL_TPS_DESC_TCTWEIGHT_WIDTH 9
 /* default value of bitfield desc_tc{t}_weight[8:0] */
-#define tps_desc_tctweight_default 0x0
+#define HW_ATL_TPS_DESC_TCTWEIGHT_DEFAULT 0x0
 
 /* tx desc_vm_arb_mode bitfield definitions
  * preprocessor definitions for the bitfield "desc_vm_arb_mode".
@@ -2128,17 +2136,17 @@
  */
 
 /* register address for bitfield desc_vm_arb_mode */
-#define tps_desc_vm_arb_mode_adr 0x00007300
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_ADR 0x00007300
 /* bitmask for bitfield desc_vm_arb_mode */
-#define tps_desc_vm_arb_mode_msk 0x00000001
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_MSK 0x00000001
 /* inverted bitmask for bitfield desc_vm_arb_mode */
-#define tps_desc_vm_arb_mode_mskn 0xfffffffe
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_MSKN 0xfffffffe
 /* lower bit position of bitfield desc_vm_arb_mode */
-#define tps_desc_vm_arb_mode_shift 0
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_SHIFT 0
 /* width of bitfield desc_vm_arb_mode */
-#define tps_desc_vm_arb_mode_width 1
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_WIDTH 1
 /* default value of bitfield desc_vm_arb_mode */
-#define tps_desc_vm_arb_mode_default 0x0
+#define HW_ATL_TPS_DESC_VM_ARB_MODE_DEFAULT 0x0
 
 /* tx data_tc{t}_credit_max[b:0] bitfield definitions
  * preprocessor definitions for the bitfield "data_tc{t}_credit_max[b:0]".
@@ -2147,17 +2155,17 @@
  */
 
 /* register address for bitfield data_tc{t}_credit_max[b:0] */
-#define tps_data_tctcredit_max_adr(tc) (0x00007110 + (tc) * 0x4)
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_ADR(tc) (0x00007110 + (tc) * 0x4)
 /* bitmask for bitfield data_tc{t}_credit_max[b:0] */
-#define tps_data_tctcredit_max_msk 0x0fff0000
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_MSK 0x0fff0000
 /* inverted bitmask for bitfield data_tc{t}_credit_max[b:0] */
-#define tps_data_tctcredit_max_mskn 0xf000ffff
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_MSKN 0xf000ffff
 /* lower bit position of bitfield data_tc{t}_credit_max[b:0] */
-#define tps_data_tctcredit_max_shift 16
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_SHIFT 16
 /* width of bitfield data_tc{t}_credit_max[b:0] */
-#define tps_data_tctcredit_max_width 12
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_WIDTH 12
 /* default value of bitfield data_tc{t}_credit_max[b:0] */
-#define tps_data_tctcredit_max_default 0x0
+#define HW_ATL_TPS_DATA_TCTCREDIT_MAX_DEFAULT 0x0
 
 /* tx data_tc{t}_weight[8:0] bitfield definitions
  * preprocessor definitions for the bitfield "data_tc{t}_weight[8:0]".
@@ -2166,17 +2174,17 @@
  */
 
 /* register address for bitfield data_tc{t}_weight[8:0] */
-#define tps_data_tctweight_adr(tc) (0x00007110 + (tc) * 0x4)
+#define HW_ATL_TPS_DATA_TCTWEIGHT_ADR(tc) (0x00007110 + (tc) * 0x4)
 /* bitmask for bitfield data_tc{t}_weight[8:0] */
-#define tps_data_tctweight_msk 0x000001ff
+#define HW_ATL_TPS_DATA_TCTWEIGHT_MSK 0x000001ff
 /* inverted bitmask for bitfield data_tc{t}_weight[8:0] */
-#define tps_data_tctweight_mskn 0xfffffe00
+#define HW_ATL_TPS_DATA_TCTWEIGHT_MSKN 0xfffffe00
 /* lower bit position of bitfield data_tc{t}_weight[8:0] */
-#define tps_data_tctweight_shift 0
+#define HW_ATL_TPS_DATA_TCTWEIGHT_SHIFT 0
 /* width of bitfield data_tc{t}_weight[8:0] */
-#define tps_data_tctweight_width 9
+#define HW_ATL_TPS_DATA_TCTWEIGHT_WIDTH 9
 /* default value of bitfield data_tc{t}_weight[8:0] */
-#define tps_data_tctweight_default 0x0
+#define HW_ATL_TPS_DATA_TCTWEIGHT_DEFAULT 0x0
 
 /* tx reg_res_dsbl bitfield definitions
  * preprocessor definitions for the bitfield "reg_res_dsbl".
@@ -2184,17 +2192,17 @@
  */
 
 /* register address for bitfield reg_res_dsbl */
-#define tx_reg_res_dsbl_adr 0x00007000
+#define HW_ATL_TX_REG_RES_DSBL_ADR 0x00007000
 /* bitmask for bitfield reg_res_dsbl */
-#define tx_reg_res_dsbl_msk 0x20000000
+#define HW_ATL_TX_REG_RES_DSBL_MSK 0x20000000
 /* inverted bitmask for bitfield reg_res_dsbl */
-#define tx_reg_res_dsbl_mskn 0xdfffffff
+#define HW_ATL_TX_REG_RES_DSBL_MSKN 0xdfffffff
 /* lower bit position of bitfield reg_res_dsbl */
-#define tx_reg_res_dsbl_shift 29
+#define HW_ATL_TX_REG_RES_DSBL_SHIFT 29
 /* width of bitfield reg_res_dsbl */
-#define tx_reg_res_dsbl_width 1
+#define HW_ATL_TX_REG_RES_DSBL_WIDTH 1
 /* default value of bitfield reg_res_dsbl */
-#define tx_reg_res_dsbl_default 0x1
+#define HW_ATL_TX_REG_RES_DSBL_DEFAULT 0x1
 
 /* mac_phy register access busy bitfield definitions
  * preprocessor definitions for the bitfield "register access busy".
@@ -2202,15 +2210,15 @@
  */
 
 /* register address for bitfield register access busy */
-#define msm_reg_access_busy_adr 0x00004400
+#define HW_ATL_MSM_REG_ACCESS_BUSY_ADR 0x00004400
 /* bitmask for bitfield register access busy */
-#define msm_reg_access_busy_msk 0x00001000
+#define HW_ATL_MSM_REG_ACCESS_BUSY_MSK 0x00001000
 /* inverted bitmask for bitfield register access busy */
-#define msm_reg_access_busy_mskn 0xffffefff
+#define HW_ATL_MSM_REG_ACCESS_BUSY_MSKN 0xffffefff
 /* lower bit position of bitfield register access busy */
-#define msm_reg_access_busy_shift 12
+#define HW_ATL_MSM_REG_ACCESS_BUSY_SHIFT 12
 /* width of bitfield register access busy */
-#define msm_reg_access_busy_width 1
+#define HW_ATL_MSM_REG_ACCESS_BUSY_WIDTH 1
 
 /* mac_phy msm register address[7:0] bitfield definitions
  * preprocessor definitions for the bitfield "msm register address[7:0]".
@@ -2218,17 +2226,17 @@
  */
 
 /* register address for bitfield msm register address[7:0] */
-#define msm_reg_addr_adr 0x00004400
+#define HW_ATL_MSM_REG_ADDR_ADR 0x00004400
 /* bitmask for bitfield msm register address[7:0] */
-#define msm_reg_addr_msk 0x000000ff
+#define HW_ATL_MSM_REG_ADDR_MSK 0x000000ff
 /* inverted bitmask for bitfield msm register address[7:0] */
-#define msm_reg_addr_mskn 0xffffff00
+#define HW_ATL_MSM_REG_ADDR_MSKN 0xffffff00
 /* lower bit position of bitfield msm register address[7:0] */
-#define msm_reg_addr_shift 0
+#define HW_ATL_MSM_REG_ADDR_SHIFT 0
 /* width of bitfield msm register address[7:0] */
-#define msm_reg_addr_width 8
+#define HW_ATL_MSM_REG_ADDR_WIDTH 8
 /* default value of bitfield msm register address[7:0] */
-#define msm_reg_addr_default 0x0
+#define HW_ATL_MSM_REG_ADDR_DEFAULT 0x0
 
 /* mac_phy register read strobe bitfield definitions
  * preprocessor definitions for the bitfield "register read strobe".
@@ -2236,17 +2244,17 @@
  */
 
 /* register address for bitfield register read strobe */
-#define msm_reg_rd_strobe_adr 0x00004400
+#define HW_ATL_MSM_REG_RD_STROBE_ADR 0x00004400
 /* bitmask for bitfield register read strobe */
-#define msm_reg_rd_strobe_msk 0x00000200
+#define HW_ATL_MSM_REG_RD_STROBE_MSK 0x00000200
 /* inverted bitmask for bitfield register read strobe */
-#define msm_reg_rd_strobe_mskn 0xfffffdff
+#define HW_ATL_MSM_REG_RD_STROBE_MSKN 0xfffffdff
 /* lower bit position of bitfield register read strobe */
-#define msm_reg_rd_strobe_shift 9
+#define HW_ATL_MSM_REG_RD_STROBE_SHIFT 9
 /* width of bitfield register read strobe */
-#define msm_reg_rd_strobe_width 1
+#define HW_ATL_MSM_REG_RD_STROBE_WIDTH 1
 /* default value of bitfield register read strobe */
-#define msm_reg_rd_strobe_default 0x0
+#define HW_ATL_MSM_REG_RD_STROBE_DEFAULT 0x0
 
 /* mac_phy msm register read data[31:0] bitfield definitions
  * preprocessor definitions for the bitfield "msm register read data[31:0]".
@@ -2254,15 +2262,15 @@
  */
 
 /* register address for bitfield msm register read data[31:0] */
-#define msm_reg_rd_data_adr 0x00004408
+#define HW_ATL_MSM_REG_RD_DATA_ADR 0x00004408
 /* bitmask for bitfield msm register read data[31:0] */
-#define msm_reg_rd_data_msk 0xffffffff
+#define HW_ATL_MSM_REG_RD_DATA_MSK 0xffffffff
 /* inverted bitmask for bitfield msm register read data[31:0] */
-#define msm_reg_rd_data_mskn 0x00000000
+#define HW_ATL_MSM_REG_RD_DATA_MSKN 0x00000000
 /* lower bit position of bitfield msm register read data[31:0] */
-#define msm_reg_rd_data_shift 0
+#define HW_ATL_MSM_REG_RD_DATA_SHIFT 0
 /* width of bitfield msm register read data[31:0] */
-#define msm_reg_rd_data_width 32
+#define HW_ATL_MSM_REG_RD_DATA_WIDTH 32
 
 /* mac_phy msm register write data[31:0] bitfield definitions
  * preprocessor definitions for the bitfield "msm register write data[31:0]".
@@ -2270,17 +2278,17 @@
  */
 
 /* register address for bitfield msm register write data[31:0] */
-#define msm_reg_wr_data_adr 0x00004404
+#define HW_ATL_MSM_REG_WR_DATA_ADR 0x00004404
 /* bitmask for bitfield msm register write data[31:0] */
-#define msm_reg_wr_data_msk 0xffffffff
+#define HW_ATL_MSM_REG_WR_DATA_MSK 0xffffffff
 /* inverted bitmask for bitfield msm register write data[31:0] */
-#define msm_reg_wr_data_mskn 0x00000000
+#define HW_ATL_MSM_REG_WR_DATA_MSKN 0x00000000
 /* lower bit position of bitfield msm register write data[31:0] */
-#define msm_reg_wr_data_shift 0
+#define HW_ATL_MSM_REG_WR_DATA_SHIFT 0
 /* width of bitfield msm register write data[31:0] */
-#define msm_reg_wr_data_width 32
+#define HW_ATL_MSM_REG_WR_DATA_WIDTH 32
 /* default value of bitfield msm register write data[31:0] */
-#define msm_reg_wr_data_default 0x0
+#define HW_ATL_MSM_REG_WR_DATA_DEFAULT 0x0
 
 /* mac_phy register write strobe bitfield definitions
  * preprocessor definitions for the bitfield "register write strobe".
@@ -2288,17 +2296,17 @@
  */
 
 /* register address for bitfield register write strobe */
-#define msm_reg_wr_strobe_adr 0x00004400
+#define HW_ATL_MSM_REG_WR_STROBE_ADR 0x00004400
 /* bitmask for bitfield register write strobe */
-#define msm_reg_wr_strobe_msk 0x00000100
+#define HW_ATL_MSM_REG_WR_STROBE_MSK 0x00000100
 /* inverted bitmask for bitfield register write strobe */
-#define msm_reg_wr_strobe_mskn 0xfffffeff
+#define HW_ATL_MSM_REG_WR_STROBE_MSKN 0xfffffeff
 /* lower bit position of bitfield register write strobe */
-#define msm_reg_wr_strobe_shift 8
+#define HW_ATL_MSM_REG_WR_STROBE_SHIFT 8
 /* width of bitfield register write strobe */
-#define msm_reg_wr_strobe_width 1
+#define HW_ATL_MSM_REG_WR_STROBE_WIDTH 1
 /* default value of bitfield register write strobe */
-#define msm_reg_wr_strobe_default 0x0
+#define HW_ATL_MSM_REG_WR_STROBE_DEFAULT 0x0
 
 /* mif soft reset bitfield definitions
  * preprocessor definitions for the bitfield "soft reset".
@@ -2306,17 +2314,17 @@
  */
 
 /* register address for bitfield soft reset */
-#define glb_soft_res_adr 0x00000000
+#define HW_ATL_GLB_SOFT_RES_ADR 0x00000000
 /* bitmask for bitfield soft reset */
-#define glb_soft_res_msk 0x00008000
+#define HW_ATL_GLB_SOFT_RES_MSK 0x00008000
 /* inverted bitmask for bitfield soft reset */
-#define glb_soft_res_mskn 0xffff7fff
+#define HW_ATL_GLB_SOFT_RES_MSKN 0xffff7fff
 /* lower bit position of bitfield soft reset */
-#define glb_soft_res_shift 15
+#define HW_ATL_GLB_SOFT_RES_SHIFT 15
 /* width of bitfield soft reset */
-#define glb_soft_res_width 1
+#define HW_ATL_GLB_SOFT_RES_WIDTH 1
 /* default value of bitfield soft reset */
-#define glb_soft_res_default 0x0
+#define HW_ATL_GLB_SOFT_RES_DEFAULT 0x0
 
 /* mif register reset disable bitfield definitions
  * preprocessor definitions for the bitfield "register reset disable".
@@ -2324,27 +2332,27 @@
  */
 
 /* register address for bitfield register reset disable */
-#define glb_reg_res_dis_adr 0x00000000
+#define HW_ATL_GLB_REG_RES_DIS_ADR 0x00000000
 /* bitmask for bitfield register reset disable */
-#define glb_reg_res_dis_msk 0x00004000
+#define HW_ATL_GLB_REG_RES_DIS_MSK 0x00004000
 /* inverted bitmask for bitfield register reset disable */
-#define glb_reg_res_dis_mskn 0xffffbfff
+#define HW_ATL_GLB_REG_RES_DIS_MSKN 0xffffbfff
 /* lower bit position of bitfield register reset disable */
-#define glb_reg_res_dis_shift 14
+#define HW_ATL_GLB_REG_RES_DIS_SHIFT 14
 /* width of bitfield register reset disable */
-#define glb_reg_res_dis_width 1
+#define HW_ATL_GLB_REG_RES_DIS_WIDTH 1
 /* default value of bitfield register reset disable */
-#define glb_reg_res_dis_default 0x1
+#define HW_ATL_GLB_REG_RES_DIS_DEFAULT 0x1
 
 /* tx dma debug control definitions */
-#define tx_dma_debug_ctl_adr 0x00008920u
+#define HW_ATL_TX_DMA_DEBUG_CTL_ADR 0x00008920u
 
 /* tx dma descriptor base address msw definitions */
-#define tx_dma_desc_base_addrmsw_adr(descriptor) \
+#define HW_ATL_TX_DMA_DESC_BASE_ADDRMSW_ADR(descriptor) \
 			(0x00007c04u + (descriptor) * 0x40)
 
 /* tx dma total request limit */
-#define tx_dma_total_req_limit_adr 0x00007b20u
+#define HW_ATL_TX_DMA_TOTAL_REQ_LIMIT_ADR 0x00007b20u
 
 /* tx interrupt moderation control register definitions
  * Preprocessor definitions for TX Interrupt Moderation Control Register
@@ -2352,7 +2360,7 @@
  * Parameter: queue {Q} | stride size 0x4 | range [0, 31]
  */
 
-#define tx_intr_moderation_ctl_adr(queue) (0x00008980u + (queue) * 0x4)
+#define HW_ATL_TX_INTR_MODERATION_CTL_ADR(queue) (0x00008980u + (queue) * 0x4)
 
 /* pcie reg_res_dsbl bitfield definitions
  * preprocessor definitions for the bitfield "reg_res_dsbl".
@@ -2360,22 +2368,23 @@
  */
 
 /* register address for bitfield reg_res_dsbl */
-#define pci_reg_res_dsbl_adr 0x00001000
+#define HW_ATL_PCI_REG_RES_DSBL_ADR 0x00001000
 /* bitmask for bitfield reg_res_dsbl */
-#define pci_reg_res_dsbl_msk 0x20000000
+#define HW_ATL_PCI_REG_RES_DSBL_MSK 0x20000000
 /* inverted bitmask for bitfield reg_res_dsbl */
-#define pci_reg_res_dsbl_mskn 0xdfffffff
+#define HW_ATL_PCI_REG_RES_DSBL_MSKN 0xdfffffff
 /* lower bit position of bitfield reg_res_dsbl */
-#define pci_reg_res_dsbl_shift 29
+#define HW_ATL_PCI_REG_RES_DSBL_SHIFT 29
 /* width of bitfield reg_res_dsbl */
-#define pci_reg_res_dsbl_width 1
+#define HW_ATL_PCI_REG_RES_DSBL_WIDTH 1
 /* default value of bitfield reg_res_dsbl */
-#define pci_reg_res_dsbl_default 0x1
+#define HW_ATL_PCI_REG_RES_DSBL_DEFAULT 0x1
 
 /* PCI core control register */
-#define pci_reg_control6_adr 0x1014u
+#define HW_ATL_PCI_REG_CONTROL6_ADR 0x1014u
 
 /* global microprocessor scratch pad definitions */
-#define glb_cpu_scratch_scp_adr(scratch_scp) (0x00000300u + (scratch_scp) * 0x4)
+#define HW_ATL_GLB_CPU_SCRATCH_SCP_ADR(scratch_scp) \
+	(0x00000300u + (scratch_scp) * 0x4)
 
 #endif /* HW_ATL_LLH_INTERNAL_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
index f2ce12ed4218..d3b847ec7465 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.c
@@ -11,62 +11,287 @@
  * abstraction layer.
  */
 
-#include "../aq_hw.h"
+#include "../aq_nic.h"
 #include "../aq_hw_utils.h"
-#include "../aq_pci_func.h"
-#include "../aq_ring.h"
-#include "../aq_vec.h"
 #include "hw_atl_utils.h"
 #include "hw_atl_llh.h"
+#include "hw_atl_llh_internal.h"
 
 #include <linux/random.h>
 
 #define HW_ATL_UCP_0X370_REG    0x0370U
 
+#define HW_ATL_MIF_CMD          0x0200U
+#define HW_ATL_MIF_ADDR         0x0208U
+#define HW_ATL_MIF_VAL          0x020CU
+
 #define HW_ATL_FW_SM_RAM        0x2U
+#define HW_ATL_MPI_FW_VERSION	0x18
 #define HW_ATL_MPI_CONTROL_ADR  0x0368U
 #define HW_ATL_MPI_STATE_ADR    0x036CU
 
 #define HW_ATL_MPI_STATE_MSK    0x00FFU
 #define HW_ATL_MPI_STATE_SHIFT  0U
-#define HW_ATL_MPI_SPEED_MSK    0xFFFFU
+#define HW_ATL_MPI_SPEED_MSK    0xFFFF0000U
 #define HW_ATL_MPI_SPEED_SHIFT  16U
 
-static int hw_atl_utils_fw_downld_dwords(struct aq_hw_s *self, u32 a,
-					 u32 *p, u32 cnt)
+#define HW_ATL_MPI_DAISY_CHAIN_STATUS	0x704
+#define HW_ATL_MPI_BOOT_EXIT_CODE	0x388
+
+#define HW_ATL_MAC_PHY_CONTROL	0x4000
+#define HW_ATL_MAC_PHY_MPI_RESET_BIT 0x1D
+
+#define HW_ATL_FW_VER_1X 0x01050006U
+#define HW_ATL_FW_VER_2X 0x02000000U
+#define HW_ATL_FW_VER_3X 0x03000000U
+
+#define FORCE_FLASHLESS 0
+
+static int hw_atl_utils_ver_match(u32 ver_expected, u32 ver_actual);
+
+int hw_atl_utils_initfw(struct aq_hw_s *self, const struct aq_fw_ops **fw_ops)
 {
 	int err = 0;
 
-	AQ_HW_WAIT_FOR(reg_glb_cpu_sem_get(self,
-					   HW_ATL_FW_SM_RAM) == 1U,
-					   1U, 10000U);
+	err = hw_atl_utils_soft_reset(self);
+	if (err)
+		return err;
+
+	hw_atl_utils_hw_chip_features_init(self,
+					   &self->chip_features);
+
+	hw_atl_utils_get_fw_version(self, &self->fw_ver_actual);
+
+	if (hw_atl_utils_ver_match(HW_ATL_FW_VER_1X,
+				   self->fw_ver_actual) == 0) {
+		*fw_ops = &aq_fw_1x_ops;
+	} else if (hw_atl_utils_ver_match(HW_ATL_FW_VER_2X,
+					self->fw_ver_actual) == 0) {
+		*fw_ops = &aq_fw_2x_ops;
+	} else if (hw_atl_utils_ver_match(HW_ATL_FW_VER_3X,
+					self->fw_ver_actual) == 0) {
+		*fw_ops = &aq_fw_2x_ops;
+	} else {
+		aq_pr_err("Bad FW version detected: %x\n",
+			  self->fw_ver_actual);
+		return -EOPNOTSUPP;
+	}
+	self->aq_fw_ops = *fw_ops;
+	err = self->aq_fw_ops->init(self);
+	return err;
+}
+
+static int hw_atl_utils_soft_reset_flb(struct aq_hw_s *self)
+{
+	u32 gsr, val;
+	int k = 0;
+
+	aq_hw_write_reg(self, 0x404, 0x40e1);
+	AQ_HW_SLEEP(50);
+
+	/* Cleanup SPI */
+	val = aq_hw_read_reg(self, 0x53C);
+	aq_hw_write_reg(self, 0x53C, val | 0x10);
+
+	gsr = aq_hw_read_reg(self, HW_ATL_GLB_SOFT_RES_ADR);
+	aq_hw_write_reg(self, HW_ATL_GLB_SOFT_RES_ADR, (gsr & 0xBFFF) | 0x8000);
+
+	/* Kickstart MAC */
+	aq_hw_write_reg(self, 0x404, 0x80e0);
+	aq_hw_write_reg(self, 0x32a8, 0x0);
+	aq_hw_write_reg(self, 0x520, 0x1);
+
+	/* Reset SPI again because of possible interrupted SPI burst */
+	val = aq_hw_read_reg(self, 0x53C);
+	aq_hw_write_reg(self, 0x53C, val | 0x10);
+	AQ_HW_SLEEP(10);
+	/* Clear SPI reset state */
+	aq_hw_write_reg(self, 0x53C, val & ~0x10);
+
+	aq_hw_write_reg(self, 0x404, 0x180e0);
+
+	for (k = 0; k < 1000; k++) {
+		u32 flb_status = aq_hw_read_reg(self,
+						HW_ATL_MPI_DAISY_CHAIN_STATUS);
+
+		flb_status = flb_status & 0x10;
+		if (flb_status)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (k == 1000) {
+		aq_pr_err("MAC kickstart failed\n");
+		return -EIO;
+	}
+
+	/* FW reset */
+	aq_hw_write_reg(self, 0x404, 0x80e0);
+	AQ_HW_SLEEP(50);
+	aq_hw_write_reg(self, 0x3a0, 0x1);
+
+	/* Kickstart PHY - skipped */
+
+	/* Global software reset*/
+	hw_atl_rx_rx_reg_res_dis_set(self, 0U);
+	hw_atl_tx_tx_reg_res_dis_set(self, 0U);
+	aq_hw_write_reg_bit(self, HW_ATL_MAC_PHY_CONTROL,
+			    BIT(HW_ATL_MAC_PHY_MPI_RESET_BIT),
+			    HW_ATL_MAC_PHY_MPI_RESET_BIT, 0x0);
+	gsr = aq_hw_read_reg(self, HW_ATL_GLB_SOFT_RES_ADR);
+	aq_hw_write_reg(self, HW_ATL_GLB_SOFT_RES_ADR, (gsr & 0xBFFF) | 0x8000);
+
+	for (k = 0; k < 1000; k++) {
+		u32 fw_state = aq_hw_read_reg(self, HW_ATL_MPI_FW_VERSION);
+
+		if (fw_state)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (k == 1000) {
+		aq_pr_err("FW kickstart failed\n");
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	AQ_HW_SLEEP(15);
+
+	return 0;
+}
+
+static int hw_atl_utils_soft_reset_rbl(struct aq_hw_s *self)
+{
+	u32 gsr, val, rbl_status;
+	int k;
+
+	aq_hw_write_reg(self, 0x404, 0x40e1);
+	aq_hw_write_reg(self, 0x3a0, 0x1);
+	aq_hw_write_reg(self, 0x32a8, 0x0);
+
+	/* Alter RBL status */
+	aq_hw_write_reg(self, 0x388, 0xDEAD);
+
+	/* Cleanup SPI */
+	val = aq_hw_read_reg(self, 0x53C);
+	aq_hw_write_reg(self, 0x53C, val | 0x10);
+
+	/* Global software reset*/
+	hw_atl_rx_rx_reg_res_dis_set(self, 0U);
+	hw_atl_tx_tx_reg_res_dis_set(self, 0U);
+	aq_hw_write_reg_bit(self, HW_ATL_MAC_PHY_CONTROL,
+			    BIT(HW_ATL_MAC_PHY_MPI_RESET_BIT),
+			    HW_ATL_MAC_PHY_MPI_RESET_BIT, 0x0);
+	gsr = aq_hw_read_reg(self, HW_ATL_GLB_SOFT_RES_ADR);
+	aq_hw_write_reg(self, HW_ATL_GLB_SOFT_RES_ADR,
+			(gsr & 0xFFFFBFFF) | 0x8000);
+
+	if (FORCE_FLASHLESS)
+		aq_hw_write_reg(self, 0x534, 0x0);
+
+	aq_hw_write_reg(self, 0x404, 0x40e0);
+
+	/* Wait for RBL boot */
+	for (k = 0; k < 1000; k++) {
+		rbl_status = aq_hw_read_reg(self, 0x388) & 0xFFFF;
+		if (rbl_status && rbl_status != 0xDEAD)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (!rbl_status || rbl_status == 0xDEAD) {
+		aq_pr_err("RBL Restart failed");
+		return -EIO;
+	}
+
+	/* Restore NVR */
+	if (FORCE_FLASHLESS)
+		aq_hw_write_reg(self, 0x534, 0xA0);
+
+	if (rbl_status == 0xF1A7) {
+		aq_pr_err("No FW detected. Dynamic FW load not implemented\n");
+		return -ENOTSUPP;
+	}
+
+	for (k = 0; k < 1000; k++) {
+		u32 fw_state = aq_hw_read_reg(self, HW_ATL_MPI_FW_VERSION);
+
+		if (fw_state)
+			break;
+		AQ_HW_SLEEP(10);
+	}
+	if (k == 1000) {
+		aq_pr_err("FW kickstart failed\n");
+		return -EIO;
+	}
+	/* Old FW requires fixed delay after init */
+	AQ_HW_SLEEP(15);
+
+	return 0;
+}
+
+int hw_atl_utils_soft_reset(struct aq_hw_s *self)
+{
+	int k;
+	u32 boot_exit_code = 0;
+
+	for (k = 0; k < 1000; ++k) {
+		u32 flb_status = aq_hw_read_reg(self,
+						HW_ATL_MPI_DAISY_CHAIN_STATUS);
+		boot_exit_code = aq_hw_read_reg(self,
+						HW_ATL_MPI_BOOT_EXIT_CODE);
+		if (flb_status != 0x06000000 || boot_exit_code != 0)
+			break;
+	}
+
+	if (k == 1000) {
+		aq_pr_err("Neither RBL nor FLB firmware started\n");
+		return -EOPNOTSUPP;
+	}
+
+	self->rbl_enabled = (boot_exit_code != 0);
+
+	if (self->rbl_enabled)
+		return hw_atl_utils_soft_reset_rbl(self);
+	else
+		return hw_atl_utils_soft_reset_flb(self);
+}
+
+int hw_atl_utils_fw_downld_dwords(struct aq_hw_s *self, u32 a,
+				  u32 *p, u32 cnt)
+{
+	int err = 0;
+
+	AQ_HW_WAIT_FOR(hw_atl_reg_glb_cpu_sem_get(self,
+						  HW_ATL_FW_SM_RAM) == 1U,
+						  1U, 10000U);
 
 	if (err < 0) {
 		bool is_locked;
 
-		reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
-		is_locked = reg_glb_cpu_sem_get(self, HW_ATL_FW_SM_RAM);
+		hw_atl_reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
+		is_locked = hw_atl_reg_glb_cpu_sem_get(self, HW_ATL_FW_SM_RAM);
 		if (!is_locked) {
 			err = -ETIME;
 			goto err_exit;
 		}
 	}
 
-	aq_hw_write_reg(self, 0x00000208U, a);
+	aq_hw_write_reg(self, HW_ATL_MIF_ADDR, a);
 
-	for (++cnt; --cnt;) {
-		u32 i = 0U;
+	for (++cnt; --cnt && !err;) {
+		aq_hw_write_reg(self, HW_ATL_MIF_CMD, 0x00008000U);
 
-		aq_hw_write_reg(self, 0x00000200U, 0x00008000U);
-
-		for (i = 1024U;
-			(0x100U & aq_hw_read_reg(self, 0x00000200U)) && --i;) {
-		}
+		if (IS_CHIP_FEATURE(REVISION_B1))
+			AQ_HW_WAIT_FOR(a != aq_hw_read_reg(self,
+							   HW_ATL_MIF_ADDR),
+				       1, 1000U);
+		else
+			AQ_HW_WAIT_FOR(!(0x100 & aq_hw_read_reg(self,
+							   HW_ATL_MIF_CMD)),
+				       1, 1000U);
 
-		*(p++) = aq_hw_read_reg(self, 0x0000020CU);
+		*(p++) = aq_hw_read_reg(self, HW_ATL_MIF_VAL);
+		a += 4;
 	}
 
-	reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
+	hw_atl_reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
 
 err_exit:
 	return err;
@@ -78,7 +303,7 @@ static int hw_atl_utils_fw_upload_dwords(struct aq_hw_s *self, u32 a, u32 *p,
 	int err = 0;
 	bool is_locked;
 
-	is_locked = reg_glb_cpu_sem_get(self, HW_ATL_FW_SM_RAM);
+	is_locked = hw_atl_reg_glb_cpu_sem_get(self, HW_ATL_FW_SM_RAM);
 	if (!is_locked) {
 		err = -ETIME;
 		goto err_exit;
@@ -97,7 +322,7 @@ static int hw_atl_utils_fw_upload_dwords(struct aq_hw_s *self, u32 a, u32 *p,
 		}
 	}
 
-	reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
+	hw_atl_reg_glb_cpu_sem_set(self, 1U, HW_ATL_FW_SM_RAM);
 
 err_exit:
 	return err;
@@ -119,7 +344,7 @@ static int hw_atl_utils_ver_match(u32 ver_expected, u32 ver_actual)
 }
 
 static int hw_atl_utils_init_ucp(struct aq_hw_s *self,
-				 struct aq_hw_caps_s *aq_hw_caps)
+				 const struct aq_hw_caps_s *aq_hw_caps)
 {
 	int err = 0;
 
@@ -133,20 +358,12 @@ static int hw_atl_utils_init_ucp(struct aq_hw_s *self,
 		aq_hw_write_reg(self, HW_ATL_UCP_0X370_REG, ucp_0x370);
 	}
 
-	reg_glb_cpu_scratch_scp_set(self, 0x00000000U, 25U);
+	hw_atl_reg_glb_cpu_scratch_scp_set(self, 0x00000000U, 25U);
 
 	/* check 10 times by 1ms */
-	AQ_HW_WAIT_FOR(0U != (PHAL_ATLANTIC_A0->mbox_addr =
+	AQ_HW_WAIT_FOR(0U != (self->mbox_addr =
 			aq_hw_read_reg(self, 0x360U)), 1000U, 10U);
 
-	err = hw_atl_utils_ver_match(aq_hw_caps->fw_ver_expected,
-				     aq_hw_read_reg(self, 0x18U));
-
-	if (err < 0)
-		pr_err("%s: Bad FW version detected: expected=%x, actual=%x\n",
-		       AQ_CFG_DRV_NAME,
-		       aq_hw_caps->fw_ver_expected,
-		       aq_hw_read_reg(self, 0x18U));
 	return err;
 }
 
@@ -174,14 +391,14 @@ static int hw_atl_utils_fw_rpc_call(struct aq_hw_s *self, unsigned int rpc_size)
 		err = -1;
 		goto err_exit;
 	}
-	err = hw_atl_utils_fw_upload_dwords(self, PHAL_ATLANTIC->rpc_addr,
-					    (u32 *)(void *)&PHAL_ATLANTIC->rpc,
+	err = hw_atl_utils_fw_upload_dwords(self, self->rpc_addr,
+					    (u32 *)(void *)&self->rpc,
 					    (rpc_size + sizeof(u32) -
 					    sizeof(u8)) / sizeof(u32));
 	if (err < 0)
 		goto err_exit;
 
-	sw.tid = 0xFFFFU & (++PHAL_ATLANTIC->rpc_tid);
+	sw.tid = 0xFFFFU & (++self->rpc_tid);
 	sw.len = (u16)rpc_size;
 	aq_hw_write_reg(self, HW_ATL_RPC_CONTROL_ADR, sw.val);
 
@@ -199,7 +416,7 @@ static int hw_atl_utils_fw_rpc_wait(struct aq_hw_s *self,
 	do {
 		sw.val = aq_hw_read_reg(self, HW_ATL_RPC_CONTROL_ADR);
 
-		PHAL_ATLANTIC->rpc_tid = sw.tid;
+		self->rpc_tid = sw.tid;
 
 		AQ_HW_WAIT_FOR(sw.tid ==
 				(fw.val =
@@ -221,9 +438,9 @@ static int hw_atl_utils_fw_rpc_wait(struct aq_hw_s *self,
 		if (fw.len) {
 			err =
 			hw_atl_utils_fw_downld_dwords(self,
-						      PHAL_ATLANTIC->rpc_addr,
+						      self->rpc_addr,
 						      (u32 *)(void *)
-						      &PHAL_ATLANTIC->rpc,
+						      &self->rpc,
 						      (fw.len + sizeof(u32) -
 						      sizeof(u8)) /
 						      sizeof(u32));
@@ -231,19 +448,18 @@ static int hw_atl_utils_fw_rpc_wait(struct aq_hw_s *self,
 				goto err_exit;
 		}
 
-		*rpc = &PHAL_ATLANTIC->rpc;
+		*rpc = &self->rpc;
 	}
 
 err_exit:
 	return err;
 }
 
-static int hw_atl_utils_mpi_create(struct aq_hw_s *self,
-				   struct aq_hw_caps_s *aq_hw_caps)
+static int hw_atl_utils_mpi_create(struct aq_hw_s *self)
 {
 	int err = 0;
 
-	err = hw_atl_utils_init_ucp(self, aq_hw_caps);
+	err = hw_atl_utils_init_ucp(self, self->aq_nic_cfg->aq_hw_caps);
 	if (err < 0)
 		goto err_exit;
 
@@ -259,7 +475,7 @@ int hw_atl_utils_mpi_read_mbox(struct aq_hw_s *self,
 			       struct hw_aq_atl_utils_mbox_header *pmbox)
 {
 	return hw_atl_utils_fw_downld_dwords(self,
-				      PHAL_ATLANTIC->mbox_addr,
+				      self->mbox_addr,
 				      (u32 *)(void *)pmbox,
 				      sizeof(*pmbox) / sizeof(u32));
 }
@@ -270,7 +486,7 @@ void hw_atl_utils_mpi_read_stats(struct aq_hw_s *self,
 	int err = 0;
 
 	err = hw_atl_utils_fw_downld_dwords(self,
-					    PHAL_ATLANTIC->mbox_addr,
+					    self->mbox_addr,
 					    (u32 *)(void *)pmbox,
 					    sizeof(*pmbox) / sizeof(u32));
 	if (err < 0)
@@ -281,27 +497,27 @@ void hw_atl_utils_mpi_read_stats(struct aq_hw_s *self,
 					self->aq_nic_cfg->mtu : 1514U;
 		pmbox->stats.ubrc = pmbox->stats.uprc * mtu;
 		pmbox->stats.ubtc = pmbox->stats.uptc * mtu;
-		pmbox->stats.dpc = atomic_read(&PHAL_ATLANTIC_A0->dpc);
+		pmbox->stats.dpc = atomic_read(&self->dpc);
 	} else {
-		pmbox->stats.dpc = reg_rx_dma_stat_counter7get(self);
+		pmbox->stats.dpc = hw_atl_reg_rx_dma_stat_counter7get(self);
 	}
 
 err_exit:;
 }
 
-int hw_atl_utils_mpi_set_speed(struct aq_hw_s *self, u32 speed,
-			       enum hal_atl_utils_fw_state_e state)
+int hw_atl_utils_mpi_set_speed(struct aq_hw_s *self, u32 speed)
 {
-	u32 ucp_0x368 = 0;
+	u32 val = aq_hw_read_reg(self, HW_ATL_MPI_CONTROL_ADR);
 
-	ucp_0x368 = (speed << HW_ATL_MPI_SPEED_SHIFT) | state;
-	aq_hw_write_reg(self, HW_ATL_MPI_CONTROL_ADR, ucp_0x368);
+	val = (val & HW_ATL_MPI_STATE_MSK) | (speed << HW_ATL_MPI_SPEED_SHIFT);
+	aq_hw_write_reg(self, HW_ATL_MPI_CONTROL_ADR, val);
 
 	return 0;
 }
 
 void hw_atl_utils_mpi_set(struct aq_hw_s *self,
-			  enum hal_atl_utils_fw_state_e state, u32 speed)
+			  enum hal_atl_utils_fw_state_e state,
+			  u32 speed)
 {
 	int err = 0;
 	u32 transaction_id = 0;
@@ -320,11 +536,22 @@ void hw_atl_utils_mpi_set(struct aq_hw_s *self,
 			goto err_exit;
 	}
 
-	err = hw_atl_utils_mpi_set_speed(self, speed, state);
+	aq_hw_write_reg(self, HW_ATL_MPI_CONTROL_ADR,
+			(speed << HW_ATL_MPI_SPEED_SHIFT) | state);
 
 err_exit:;
 }
 
+static int hw_atl_utils_mpi_set_state(struct aq_hw_s *self,
+				      enum hal_atl_utils_fw_state_e state)
+{
+	u32 val = aq_hw_read_reg(self, HW_ATL_MPI_CONTROL_ADR);
+
+	val = state | (val & HW_ATL_MPI_SPEED_MSK);
+	aq_hw_write_reg(self, HW_ATL_MPI_CONTROL_ADR, val);
+	return 0;
+}
+
 int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self)
 {
 	u32 cp0x036C = aq_hw_read_reg(self, HW_ATL_MPI_STATE_ADR);
@@ -365,7 +592,6 @@ int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self)
 }
 
 int hw_atl_utils_get_mac_permanent(struct aq_hw_s *self,
-				   struct aq_hw_caps_s *aq_hw_caps,
 				   u8 *mac)
 {
 	int err = 0;
@@ -373,15 +599,6 @@ int hw_atl_utils_get_mac_permanent(struct aq_hw_s *self,
 	u32 l = 0U;
 	u32 mac_addr[2];
 
-	self->mmio = aq_pci_func_get_mmio(self->aq_pci_func);
-
-	hw_atl_utils_hw_chip_features_init(self,
-					   &PHAL_ATLANTIC_A0->chip_features);
-
-	err = hw_atl_utils_mpi_create(self, aq_hw_caps);
-	if (err < 0)
-		goto err_exit;
-
 	if (!aq_hw_read_reg(self, HW_ATL_UCP_0X370_REG)) {
 		unsigned int rnd = 0;
 		unsigned int ucp_0x370 = 0;
@@ -396,7 +613,7 @@ int hw_atl_utils_get_mac_permanent(struct aq_hw_s *self,
 					    aq_hw_read_reg(self, 0x00000374U) +
 					    (40U * 4U),
 					    mac_addr,
-					    AQ_DIMOF(mac_addr));
+					    ARRAY_SIZE(mac_addr));
 	if (err < 0) {
 		mac_addr[0] = 0U;
 		mac_addr[1] = 0U;
@@ -427,7 +644,6 @@ int hw_atl_utils_get_mac_permanent(struct aq_hw_s *self,
 		mac[0] = (u8)(0xFFU & h);
 	}
 
-err_exit:
 	return err;
 }
 
@@ -465,17 +681,21 @@ unsigned int hw_atl_utils_mbps_2_speed_index(unsigned int mbps)
 void hw_atl_utils_hw_chip_features_init(struct aq_hw_s *self, u32 *p)
 {
 	u32 chip_features = 0U;
-	u32 val = reg_glb_mif_id_get(self);
+	u32 val = hw_atl_reg_glb_mif_id_get(self);
 	u32 mif_rev = val & 0xFFU;
 
-	if ((3U & mif_rev) == 1U) {
-		chip_features |=
-			HAL_ATLANTIC_UTILS_CHIP_REVISION_A0 |
+	if ((0xFU & mif_rev) == 1U) {
+		chip_features |= HAL_ATLANTIC_UTILS_CHIP_REVISION_A0 |
 			HAL_ATLANTIC_UTILS_CHIP_MPI_AQ |
 			HAL_ATLANTIC_UTILS_CHIP_MIPS;
-	} else if ((3U & mif_rev) == 2U) {
-		chip_features |=
-			HAL_ATLANTIC_UTILS_CHIP_REVISION_B0 |
+	} else if ((0xFU & mif_rev) == 2U) {
+		chip_features |= HAL_ATLANTIC_UTILS_CHIP_REVISION_B0 |
+			HAL_ATLANTIC_UTILS_CHIP_MPI_AQ |
+			HAL_ATLANTIC_UTILS_CHIP_MIPS |
+			HAL_ATLANTIC_UTILS_CHIP_TPO2 |
+			HAL_ATLANTIC_UTILS_CHIP_RPF2;
+	} else if ((0xFU & mif_rev) == 0xAU) {
+		chip_features |= HAL_ATLANTIC_UTILS_CHIP_REVISION_B1 |
 			HAL_ATLANTIC_UTILS_CHIP_MPI_AQ |
 			HAL_ATLANTIC_UTILS_CHIP_MIPS |
 			HAL_ATLANTIC_UTILS_CHIP_TPO2 |
@@ -500,13 +720,13 @@ int hw_atl_utils_hw_set_power(struct aq_hw_s *self,
 
 int hw_atl_utils_update_stats(struct aq_hw_s *self)
 {
-	struct hw_atl_s *hw_self = PHAL_ATLANTIC;
 	struct hw_aq_atl_utils_mbox mbox;
 
 	hw_atl_utils_mpi_read_stats(self, &mbox);
 
-#define AQ_SDELTA(_N_) (hw_self->curr_stats._N_ += \
-			mbox.stats._N_ - hw_self->last_stats._N_)
+#define AQ_SDELTA(_N_) (self->curr_stats._N_ += \
+			mbox.stats._N_ - self->last_stats._N_)
+
 	if (self->aq_link_status.mbps) {
 		AQ_SDELTA(uprc);
 		AQ_SDELTA(mprc);
@@ -527,19 +747,19 @@ int hw_atl_utils_update_stats(struct aq_hw_s *self)
 		AQ_SDELTA(dpc);
 	}
 #undef AQ_SDELTA
-	hw_self->curr_stats.dma_pkt_rc = stats_rx_dma_good_pkt_counterlsw_get(self);
-	hw_self->curr_stats.dma_pkt_tc = stats_tx_dma_good_pkt_counterlsw_get(self);
-	hw_self->curr_stats.dma_oct_rc = stats_rx_dma_good_octet_counterlsw_get(self);
-	hw_self->curr_stats.dma_oct_tc = stats_tx_dma_good_octet_counterlsw_get(self);
+	self->curr_stats.dma_pkt_rc = hw_atl_stats_rx_dma_good_pkt_counterlsw_get(self);
+	self->curr_stats.dma_pkt_tc = hw_atl_stats_tx_dma_good_pkt_counterlsw_get(self);
+	self->curr_stats.dma_oct_rc = hw_atl_stats_rx_dma_good_octet_counterlsw_get(self);
+	self->curr_stats.dma_oct_tc = hw_atl_stats_tx_dma_good_octet_counterlsw_get(self);
 
-	memcpy(&hw_self->last_stats, &mbox.stats, sizeof(mbox.stats));
+	memcpy(&self->last_stats, &mbox.stats, sizeof(mbox.stats));
 
 	return 0;
 }
 
 struct aq_stats_s *hw_atl_utils_get_hw_stats(struct aq_hw_s *self)
 {
-	return &PHAL_ATLANTIC->curr_stats;
+	return &self->curr_stats;
 }
 
 static const u32 hw_atl_utils_hw_mac_regs[] = {
@@ -568,14 +788,14 @@ static const u32 hw_atl_utils_hw_mac_regs[] = {
 };
 
 int hw_atl_utils_hw_get_regs(struct aq_hw_s *self,
-			     struct aq_hw_caps_s *aq_hw_caps,
+			     const struct aq_hw_caps_s *aq_hw_caps,
 			     u32 *regs_buff)
 {
 	unsigned int i = 0U;
 
 	for (i = 0; i < aq_hw_caps->mac_regs_count; i++)
 		regs_buff[i] = aq_hw_read_reg(self,
-			hw_atl_utils_hw_mac_regs[i]);
+					      hw_atl_utils_hw_mac_regs[i]);
 	return 0;
 }
 
@@ -584,3 +804,13 @@ int hw_atl_utils_get_fw_version(struct aq_hw_s *self, u32 *fw_version)
 	*fw_version = aq_hw_read_reg(self, 0x18U);
 	return 0;
 }
+
+const struct aq_fw_ops aq_fw_1x_ops = {
+	.init = hw_atl_utils_mpi_create,
+	.reset = NULL,
+	.get_mac_permanent = hw_atl_utils_get_mac_permanent,
+	.set_link_speed = hw_atl_utils_mpi_set_speed,
+	.set_state = hw_atl_utils_mpi_set_state,
+	.update_link_status = hw_atl_utils_mpi_get_link_status,
+	.update_stats = hw_atl_utils_update_stats,
+};
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
index 21aeca6908d3..cd8f18f39c61 100644
--- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h
@@ -14,10 +14,39 @@
 #ifndef HW_ATL_UTILS_H
 #define HW_ATL_UTILS_H
 
-#include "../aq_common.h"
-
 #define HW_ATL_FLUSH() { (void)aq_hw_read_reg(self, 0x10); }
 
+/* Hardware tx descriptor */
+struct __packed hw_atl_txd_s {
+	u64 buf_addr;
+	u32 ctl;
+	u32 ctl2; /* 63..46 - payload length, 45 - ctx enable, 44 - ctx index */
+};
+
+/* Hardware tx context descriptor */
+struct __packed hw_atl_txc_s {
+	u32 rsvd;
+	u32 len;
+	u32 ctl;
+	u32 len2;
+};
+
+/* Hardware rx descriptor */
+struct __packed hw_atl_rxd_s {
+	u64 buf_addr;
+	u64 hdr_addr;
+};
+
+/* Hardware rx descriptor writeback */
+struct __packed hw_atl_rxd_wb_s {
+	u32 type;
+	u32 rss_hash;
+	u16 status;
+	u16 pkt_len;
+	u16 next_desc_ptr;
+	u16 vlan;
+};
+
 struct __packed hw_atl_stats_s {
 	u32 uprc;
 	u32 mprc;
@@ -126,35 +155,16 @@ struct __packed hw_aq_atl_utils_mbox {
 	struct hw_atl_stats_s stats;
 };
 
-struct __packed hw_atl_s {
-	struct aq_hw_s base;
-	struct hw_atl_stats_s last_stats;
-	struct aq_stats_s curr_stats;
-	u64 speed;
-	unsigned int chip_features;
-	u32 fw_ver_actual;
-	atomic_t dpc;
-	u32 mbox_addr;
-	u32 rpc_addr;
-	u32 rpc_tid;
-	struct hw_aq_atl_utils_fw_rpc rpc;
-};
-
-#define SELF ((struct hw_atl_s *)self)
-
-#define PHAL_ATLANTIC ((struct hw_atl_s *)((void *)(self)))
-#define PHAL_ATLANTIC_A0 ((struct hw_atl_s *)((void *)(self)))
-#define PHAL_ATLANTIC_B0 ((struct hw_atl_s *)((void *)(self)))
-
 #define HAL_ATLANTIC_UTILS_CHIP_MIPS         0x00000001U
 #define HAL_ATLANTIC_UTILS_CHIP_TPO2         0x00000002U
 #define HAL_ATLANTIC_UTILS_CHIP_RPF2         0x00000004U
 #define HAL_ATLANTIC_UTILS_CHIP_MPI_AQ       0x00000010U
 #define HAL_ATLANTIC_UTILS_CHIP_REVISION_A0  0x01000000U
 #define HAL_ATLANTIC_UTILS_CHIP_REVISION_B0  0x02000000U
+#define HAL_ATLANTIC_UTILS_CHIP_REVISION_B1  0x04000000U
 
 #define IS_CHIP_FEATURE(_F_) (HAL_ATLANTIC_UTILS_CHIP_##_F_ & \
-				PHAL_ATLANTIC->chip_features)
+	self->chip_features)
 
 enum hal_atl_utils_fw_state_e {
 	MPI_DEINIT = 0,
@@ -171,6 +181,73 @@ enum hal_atl_utils_fw_state_e {
 #define HAL_ATLANTIC_RATE_100M       BIT(5)
 #define HAL_ATLANTIC_RATE_INVALID    BIT(6)
 
+enum hw_atl_fw2x_rate {
+	FW2X_RATE_100M    = 0x20,
+	FW2X_RATE_1G      = 0x100,
+	FW2X_RATE_2G5     = 0x200,
+	FW2X_RATE_5G      = 0x400,
+	FW2X_RATE_10G     = 0x800,
+};
+
+enum hw_atl_fw2x_caps_lo {
+	CAPS_LO_10BASET_HD = 0x00,
+	CAPS_LO_10BASET_FD,
+	CAPS_LO_100BASETX_HD,
+	CAPS_LO_100BASET4_HD,
+	CAPS_LO_100BASET2_HD,
+	CAPS_LO_100BASETX_FD,
+	CAPS_LO_100BASET2_FD,
+	CAPS_LO_1000BASET_HD,
+	CAPS_LO_1000BASET_FD,
+	CAPS_LO_2P5GBASET_FD,
+	CAPS_LO_5GBASET_FD,
+	CAPS_LO_10GBASET_FD,
+};
+
+enum hw_atl_fw2x_caps_hi {
+	CAPS_HI_RESERVED1 = 0x00,
+	CAPS_HI_10BASET_EEE,
+	CAPS_HI_RESERVED2,
+	CAPS_HI_PAUSE,
+	CAPS_HI_ASYMMETRIC_PAUSE,
+	CAPS_HI_100BASETX_EEE,
+	CAPS_HI_RESERVED3,
+	CAPS_HI_RESERVED4,
+	CAPS_HI_1000BASET_FD_EEE,
+	CAPS_HI_2P5GBASET_FD_EEE,
+	CAPS_HI_5GBASET_FD_EEE,
+	CAPS_HI_10GBASET_FD_EEE,
+	CAPS_HI_RESERVED5,
+	CAPS_HI_RESERVED6,
+	CAPS_HI_RESERVED7,
+	CAPS_HI_RESERVED8,
+	CAPS_HI_RESERVED9,
+	CAPS_HI_CABLE_DIAG,
+	CAPS_HI_TEMPERATURE,
+	CAPS_HI_DOWNSHIFT,
+	CAPS_HI_PTP_AVB_EN,
+	CAPS_HI_MEDIA_DETECT,
+	CAPS_HI_LINK_DROP,
+	CAPS_HI_SLEEP_PROXY,
+	CAPS_HI_WOL,
+	CAPS_HI_MAC_STOP,
+	CAPS_HI_EXT_LOOPBACK,
+	CAPS_HI_INT_LOOPBACK,
+	CAPS_HI_EFUSE_AGENT,
+	CAPS_HI_WOL_TIMER,
+	CAPS_HI_STATISTICS,
+	CAPS_HI_TRANSACTION_ID,
+};
+
+struct aq_hw_s;
+struct aq_fw_ops;
+struct aq_hw_caps_s;
+struct aq_hw_link_status_s;
+
+int hw_atl_utils_initfw(struct aq_hw_s *self, const struct aq_fw_ops **fw_ops);
+
+int hw_atl_utils_soft_reset(struct aq_hw_s *self);
+
 void hw_atl_utils_hw_chip_features_init(struct aq_hw_s *self, u32 *p);
 
 int hw_atl_utils_mpi_read_mbox(struct aq_hw_s *self,
@@ -183,19 +260,15 @@ void hw_atl_utils_mpi_set(struct aq_hw_s *self,
 			  enum hal_atl_utils_fw_state_e state,
 			  u32 speed);
 
-int hw_atl_utils_mpi_set_speed(struct aq_hw_s *self, u32 speed,
-			       enum hal_atl_utils_fw_state_e state);
-
 int hw_atl_utils_mpi_get_link_status(struct aq_hw_s *self);
 
 int hw_atl_utils_get_mac_permanent(struct aq_hw_s *self,
-				   struct aq_hw_caps_s *aq_hw_caps,
 				   u8 *mac);
 
 unsigned int hw_atl_utils_mbps_2_speed_index(unsigned int mbps);
 
 int hw_atl_utils_hw_get_regs(struct aq_hw_s *self,
-			     struct aq_hw_caps_s *aq_hw_caps,
+			     const struct aq_hw_caps_s *aq_hw_caps,
 			     u32 *regs_buff);
 
 int hw_atl_utils_hw_set_power(struct aq_hw_s *self,
@@ -208,5 +281,10 @@ int hw_atl_utils_get_fw_version(struct aq_hw_s *self, u32 *fw_version);
 int hw_atl_utils_update_stats(struct aq_hw_s *self);
 
 struct aq_stats_s *hw_atl_utils_get_hw_stats(struct aq_hw_s *self);
+int hw_atl_utils_fw_downld_dwords(struct aq_hw_s *self, u32 a,
+				  u32 *p, u32 cnt);
+
+extern const struct aq_fw_ops aq_fw_1x_ops;
+extern const struct aq_fw_ops aq_fw_2x_ops;
 
 #endif /* HW_ATL_UTILS_H */
diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
new file mode 100644
index 000000000000..8cfce95c82fc
--- /dev/null
+++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c
@@ -0,0 +1,184 @@
+/*
+ * aQuantia Corporation Network Driver
+ * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ */
+
+/* File hw_atl_utils_fw2x.c: Definition of firmware 2.x functions for
+ * Atlantic hardware abstraction layer.
+ */
+
+#include "../aq_hw.h"
+#include "../aq_hw_utils.h"
+#include "../aq_pci_func.h"
+#include "../aq_ring.h"
+#include "../aq_vec.h"
+#include "hw_atl_utils.h"
+#include "hw_atl_llh.h"
+
+#define HW_ATL_FW2X_MPI_EFUSE_ADDR	0x364
+#define HW_ATL_FW2X_MPI_MBOX_ADDR	0x360
+
+#define HW_ATL_FW2X_MPI_CONTROL_ADDR	0x368
+#define HW_ATL_FW2X_MPI_CONTROL2_ADDR	0x36C
+
+#define HW_ATL_FW2X_MPI_STATE_ADDR	0x370
+#define HW_ATL_FW2X_MPI_STATE2_ADDR	0x374
+
+static int aq_fw2x_init(struct aq_hw_s *self)
+{
+	int err = 0;
+
+	/* check 10 times by 1ms */
+	AQ_HW_WAIT_FOR(0U != (self->mbox_addr =
+			aq_hw_read_reg(self, HW_ATL_FW2X_MPI_MBOX_ADDR)),
+		       1000U, 10U);
+	return err;
+}
+
+static enum hw_atl_fw2x_rate link_speed_mask_2fw2x_ratemask(u32 speed)
+{
+	enum hw_atl_fw2x_rate rate = 0;
+
+	if (speed & AQ_NIC_RATE_10G)
+		rate |= FW2X_RATE_10G;
+
+	if (speed & AQ_NIC_RATE_5G)
+		rate |= FW2X_RATE_5G;
+
+	if (speed & AQ_NIC_RATE_5GSR)
+		rate |= FW2X_RATE_5G;
+
+	if (speed & AQ_NIC_RATE_2GS)
+		rate |= FW2X_RATE_2G5;
+
+	if (speed & AQ_NIC_RATE_1G)
+		rate |= FW2X_RATE_1G;
+
+	if (speed & AQ_NIC_RATE_100M)
+		rate |= FW2X_RATE_100M;
+
+	return rate;
+}
+
+static int aq_fw2x_set_link_speed(struct aq_hw_s *self, u32 speed)
+{
+	u32 val = link_speed_mask_2fw2x_ratemask(speed);
+
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR, val);
+
+	return 0;
+}
+
+static int aq_fw2x_set_state(struct aq_hw_s *self,
+			     enum hal_atl_utils_fw_state_e state)
+{
+	/* No explicit state in 2x fw */
+	return 0;
+}
+
+static int aq_fw2x_update_link_status(struct aq_hw_s *self)
+{
+	u32 mpi_state = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE_ADDR);
+	u32 speed = mpi_state & (FW2X_RATE_100M | FW2X_RATE_1G |
+				FW2X_RATE_2G5 | FW2X_RATE_5G | FW2X_RATE_10G);
+	struct aq_hw_link_status_s *link_status = &self->aq_link_status;
+
+	if (speed) {
+		if (speed & FW2X_RATE_10G)
+			link_status->mbps = 10000;
+		else if (speed & FW2X_RATE_5G)
+			link_status->mbps = 5000;
+		else if (speed & FW2X_RATE_2G5)
+			link_status->mbps = 2500;
+		else if (speed & FW2X_RATE_1G)
+			link_status->mbps = 1000;
+		else if (speed & FW2X_RATE_100M)
+			link_status->mbps = 100;
+		else
+			link_status->mbps = 10000;
+	} else {
+		link_status->mbps = 0;
+	}
+
+	return 0;
+}
+
+int aq_fw2x_get_mac_permanent(struct aq_hw_s *self, u8 *mac)
+{
+	int err = 0;
+	u32 h = 0U;
+	u32 l = 0U;
+	u32 mac_addr[2] = { 0 };
+	u32 efuse_addr = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_EFUSE_ADDR);
+
+	if (efuse_addr != 0) {
+		err = hw_atl_utils_fw_downld_dwords(self,
+						    efuse_addr + (40U * 4U),
+						    mac_addr,
+						    ARRAY_SIZE(mac_addr));
+		if (err)
+			return err;
+		mac_addr[0] = __swab32(mac_addr[0]);
+		mac_addr[1] = __swab32(mac_addr[1]);
+	}
+
+	ether_addr_copy(mac, (u8 *)mac_addr);
+
+	if ((mac[0] & 0x01U) || ((mac[0] | mac[1] | mac[2]) == 0x00U)) {
+		unsigned int rnd = 0;
+
+		get_random_bytes(&rnd, sizeof(unsigned int));
+
+		l = 0xE3000000U
+			| (0xFFFFU & rnd)
+			| (0x00 << 16);
+		h = 0x8001300EU;
+
+		mac[5] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[4] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[3] = (u8)(0xFFU & l);
+		l >>= 8;
+		mac[2] = (u8)(0xFFU & l);
+		mac[1] = (u8)(0xFFU & h);
+		h >>= 8;
+		mac[0] = (u8)(0xFFU & h);
+	}
+	return err;
+}
+
+static int aq_fw2x_update_stats(struct aq_hw_s *self)
+{
+	int err = 0;
+	u32 mpi_opts = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR);
+	u32 orig_stats_val = mpi_opts & BIT(CAPS_HI_STATISTICS);
+
+	/* Toggle statistics bit for FW to update */
+	mpi_opts = mpi_opts ^ BIT(CAPS_HI_STATISTICS);
+	aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL2_ADDR, mpi_opts);
+
+	/* Wait FW to report back */
+	AQ_HW_WAIT_FOR(orig_stats_val !=
+		       (aq_hw_read_reg(self, HW_ATL_FW2X_MPI_STATE2_ADDR) &
+				       BIT(CAPS_HI_STATISTICS)),
+		       1U, 10000U);
+	if (err)
+		return err;
+
+	return hw_atl_utils_update_stats(self);
+}
+
+const struct aq_fw_ops aq_fw_2x_ops = {
+	.init = aq_fw2x_init,
+	.reset = NULL,
+	.get_mac_permanent = aq_fw2x_get_mac_permanent,
+	.set_link_speed = aq_fw2x_set_link_speed,
+	.set_state = aq_fw2x_set_state,
+	.update_link_status = aq_fw2x_update_link_status,
+	.update_stats = aq_fw2x_update_stats,
+};
diff --git a/drivers/net/ethernet/aquantia/atlantic/ver.h b/drivers/net/ethernet/aquantia/atlantic/ver.h
index 9009f2651e70..a445de6837a6 100644
--- a/drivers/net/ethernet/aquantia/atlantic/ver.h
+++ b/drivers/net/ethernet/aquantia/atlantic/ver.h
@@ -10,10 +10,10 @@
 #ifndef VER_H
 #define VER_H
 
-#define NIC_MAJOR_DRIVER_VERSION           1
-#define NIC_MINOR_DRIVER_VERSION           6
-#define NIC_BUILD_DRIVER_VERSION           13
-#define NIC_REVISION_DRIVER_VERSION        0
+#define NIC_MAJOR_DRIVER_VERSION           2
+#define NIC_MINOR_DRIVER_VERSION           0
+#define NIC_BUILD_DRIVER_VERSION           2
+#define NIC_REVISION_DRIVER_VERSION        1
 
 #define AQ_CFG_DRV_VERSION_SUFFIX "-kern"
 
diff --git a/drivers/net/ethernet/arc/emac_rockchip.c b/drivers/net/ethernet/arc/emac_rockchip.c
index 16f9bee992fe..0f6576802607 100644
--- a/drivers/net/ethernet/arc/emac_rockchip.c
+++ b/drivers/net/ethernet/arc/emac_rockchip.c
@@ -169,8 +169,10 @@ static int emac_rockchip_probe(struct platform_device *pdev)
 	/* Optional regulator for PHY */
 	priv->regulator = devm_regulator_get_optional(dev, "phy");
 	if (IS_ERR(priv->regulator)) {
-		if (PTR_ERR(priv->regulator) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
+		if (PTR_ERR(priv->regulator) == -EPROBE_DEFER) {
+			err = -EPROBE_DEFER;
+			goto out_clk_disable;
+		}
 		dev_err(dev, "no regulator found\n");
 		priv->regulator = NULL;
 	}
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
index d9346e2ac720..14a59e51db67 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
@@ -1716,7 +1716,6 @@ static int bcm_enet_probe(struct platform_device *pdev)
 	struct bcm63xx_enet_platform_data *pd;
 	struct resource *res_mem, *res_irq, *res_irq_rx, *res_irq_tx;
 	struct mii_bus *bus;
-	const char *clk_name;
 	int i, ret;
 
 	if (!bcm_enet_shared_base[0])
@@ -1751,20 +1750,8 @@ static int bcm_enet_probe(struct platform_device *pdev)
 	dev->irq = priv->irq = res_irq->start;
 	priv->irq_rx = res_irq_rx->start;
 	priv->irq_tx = res_irq_tx->start;
-	priv->mac_id = pdev->id;
 
-	/* get rx & tx dma channel id for this mac */
-	if (priv->mac_id == 0) {
-		priv->rx_chan = 0;
-		priv->tx_chan = 1;
-		clk_name = "enet0";
-	} else {
-		priv->rx_chan = 2;
-		priv->tx_chan = 3;
-		clk_name = "enet1";
-	}
-
-	priv->mac_clk = devm_clk_get(&pdev->dev, clk_name);
+	priv->mac_clk = devm_clk_get(&pdev->dev, "enet");
 	if (IS_ERR(priv->mac_clk)) {
 		ret = PTR_ERR(priv->mac_clk);
 		goto out;
@@ -1795,9 +1782,11 @@ static int bcm_enet_probe(struct platform_device *pdev)
 		priv->dma_chan_width = pd->dma_chan_width;
 		priv->dma_has_sram = pd->dma_has_sram;
 		priv->dma_desc_shift = pd->dma_desc_shift;
+		priv->rx_chan = pd->rx_chan;
+		priv->tx_chan = pd->tx_chan;
 	}
 
-	if (priv->mac_id == 0 && priv->has_phy && !priv->use_external_mii) {
+	if (priv->has_phy && !priv->use_external_mii) {
 		/* using internal PHY, enable clock */
 		priv->phy_clk = devm_clk_get(&pdev->dev, "ephy");
 		if (IS_ERR(priv->phy_clk)) {
@@ -1828,7 +1817,7 @@ static int bcm_enet_probe(struct platform_device *pdev)
 		bus->priv = priv;
 		bus->read = bcm_enet_mdio_read_phylib;
 		bus->write = bcm_enet_mdio_write_phylib;
-		sprintf(bus->id, "%s-%d", pdev->name, priv->mac_id);
+		sprintf(bus->id, "%s-%d", pdev->name, pdev->id);
 
 		/* only probe bus where we think the PHY is, because
 		 * the mdio read operation return 0 instead of 0xffff
@@ -2139,27 +2128,25 @@ static int bcm_enetsw_open(struct net_device *dev)
 
 	/* allocate rx dma ring */
 	size = priv->rx_ring_size * sizeof(struct bcm_enet_desc);
-	p = dma_alloc_coherent(kdev, size, &priv->rx_desc_dma, GFP_KERNEL);
+	p = dma_zalloc_coherent(kdev, size, &priv->rx_desc_dma, GFP_KERNEL);
 	if (!p) {
 		dev_err(kdev, "cannot allocate rx ring %u\n", size);
 		ret = -ENOMEM;
 		goto out_freeirq_tx;
 	}
 
-	memset(p, 0, size);
 	priv->rx_desc_alloc_size = size;
 	priv->rx_desc_cpu = p;
 
 	/* allocate tx dma ring */
 	size = priv->tx_ring_size * sizeof(struct bcm_enet_desc);
-	p = dma_alloc_coherent(kdev, size, &priv->tx_desc_dma, GFP_KERNEL);
+	p = dma_zalloc_coherent(kdev, size, &priv->tx_desc_dma, GFP_KERNEL);
 	if (!p) {
 		dev_err(kdev, "cannot allocate tx ring\n");
 		ret = -ENOMEM;
 		goto out_free_rx_ring;
 	}
 
-	memset(p, 0, size);
 	priv->tx_desc_alloc_size = size;
 	priv->tx_desc_cpu = p;
 
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.h b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
index 5a66728d4776..1d3c917eb830 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.h
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.h
@@ -193,9 +193,6 @@ struct bcm_enet_mib_counters {
 
 struct bcm_enet_priv {
 
-	/* mac id (from platform device id) */
-	int mac_id;
-
 	/* base remapped address of device */
 	void __iomem *base;
 
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index 087f01b4dc3a..3fc549b88c43 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -855,10 +855,12 @@ static void bcm_sysport_tx_reclaim_one(struct bcm_sysport_tx_ring *ring,
 static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
 					     struct bcm_sysport_tx_ring *ring)
 {
-	unsigned int c_index, last_c_index, last_tx_cn, num_tx_cbs;
 	unsigned int pkts_compl = 0, bytes_compl = 0;
 	struct net_device *ndev = priv->netdev;
+	unsigned int txbds_processed = 0;
 	struct bcm_sysport_cb *cb;
+	unsigned int txbds_ready;
+	unsigned int c_index;
 	u32 hw_ind;
 
 	/* Clear status before servicing to reduce spurious interrupts */
@@ -871,29 +873,23 @@ static unsigned int __bcm_sysport_tx_reclaim(struct bcm_sysport_priv *priv,
 	/* Compute how many descriptors have been processed since last call */
 	hw_ind = tdma_readl(priv, TDMA_DESC_RING_PROD_CONS_INDEX(ring->index));
 	c_index = (hw_ind >> RING_CONS_INDEX_SHIFT) & RING_CONS_INDEX_MASK;
-	ring->p_index = (hw_ind & RING_PROD_INDEX_MASK);
-
-	last_c_index = ring->c_index;
-	num_tx_cbs = ring->size;
-
-	c_index &= (num_tx_cbs - 1);
-
-	if (c_index >= last_c_index)
-		last_tx_cn = c_index - last_c_index;
-	else
-		last_tx_cn = num_tx_cbs - last_c_index + c_index;
+	txbds_ready = (c_index - ring->c_index) & RING_CONS_INDEX_MASK;
 
 	netif_dbg(priv, tx_done, ndev,
-		  "ring=%d c_index=%d last_tx_cn=%d last_c_index=%d\n",
-		  ring->index, c_index, last_tx_cn, last_c_index);
+		  "ring=%d old_c_index=%u c_index=%u txbds_ready=%u\n",
+		  ring->index, ring->c_index, c_index, txbds_ready);
 
-	while (last_tx_cn-- > 0) {
-		cb = ring->cbs + last_c_index;
+	while (txbds_processed < txbds_ready) {
+		cb = &ring->cbs[ring->clean_index];
 		bcm_sysport_tx_reclaim_one(ring, cb, &bytes_compl, &pkts_compl);
 
 		ring->desc_count++;
-		last_c_index++;
-		last_c_index &= (num_tx_cbs - 1);
+		txbds_processed++;
+
+		if (likely(ring->clean_index < ring->size - 1))
+			ring->clean_index++;
+		else
+			ring->clean_index = 0;
 	}
 
 	u64_stats_update_begin(&priv->syncp);
@@ -1216,18 +1212,6 @@ static netdev_tx_t bcm_sysport_xmit(struct sk_buff *skb,
 		goto out;
 	}
 
-	/* The Ethernet switch we are interfaced with needs packets to be at
-	 * least 64 bytes (including FCS) otherwise they will be discarded when
-	 * they enter the switch port logic. When Broadcom tags are enabled, we
-	 * need to make sure that packets are at least 68 bytes
-	 * (including FCS and tag) because the length verification is done after
-	 * the Broadcom tag is stripped off the ingress packet.
-	 */
-	if (skb_put_padto(skb, ETH_ZLEN + ENET_BRCM_TAG_LEN)) {
-		ret = NETDEV_TX_OK;
-		goto out;
-	}
-
 	/* Insert TSB and checksum infos */
 	if (priv->tsb_en) {
 		skb = bcm_sysport_insert_tsb(skb, dev);
@@ -1406,6 +1390,7 @@ static int bcm_sysport_init_tx_ring(struct bcm_sysport_priv *priv,
 	netif_tx_napi_add(priv->netdev, &ring->napi, bcm_sysport_tx_poll, 64);
 	ring->index = index;
 	ring->size = size;
+	ring->clean_index = 0;
 	ring->alloc_size = ring->size;
 	ring->desc_cpu = p;
 	ring->desc_count = ring->size;
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.h b/drivers/net/ethernet/broadcom/bcmsysport.h
index f5a984c1c986..19c91c76e327 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.h
+++ b/drivers/net/ethernet/broadcom/bcmsysport.h
@@ -706,7 +706,7 @@ struct bcm_sysport_tx_ring {
 	unsigned int	desc_count;	/* Number of descriptors */
 	unsigned int	curr_desc;	/* Current descriptor */
 	unsigned int	c_index;	/* Last consumer index */
-	unsigned int	p_index;	/* Current producer index */
+	unsigned int	clean_index;	/* Current clean index */
 	struct bcm_sysport_cb *cbs;	/* Transmit control blocks */
 	struct dma_desc	*desc_cpu;	/* CPU view of the descriptor */
 	struct bcm_sysport_priv *priv;	/* private context backpointer */
diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
index 1d96cd594ade..8eef9fb6b1fe 100644
--- a/drivers/net/ethernet/broadcom/bgmac.c
+++ b/drivers/net/ethernet/broadcom/bgmac.c
@@ -128,8 +128,6 @@ bgmac_dma_tx_add_buf(struct bgmac *bgmac, struct bgmac_dma_ring *ring,
 	dma_desc->ctl1 = cpu_to_le32(ctl1);
 }
 
-#define ENET_BRCM_TAG_LEN	4
-
 static netdev_tx_t bgmac_dma_tx_add(struct bgmac *bgmac,
 				    struct bgmac_dma_ring *ring,
 				    struct sk_buff *skb)
@@ -142,18 +140,6 @@ static netdev_tx_t bgmac_dma_tx_add(struct bgmac *bgmac,
 	u32 flags;
 	int i;
 
-	/* The Ethernet switch we are interfaced with needs packets to be at
-	 * least 64 bytes (including FCS) otherwise they will be discarded when
-	 * they enter the switch port logic. When Broadcom tags are enabled, we
-	 * need to make sure that packets are at least 68 bytes
-	 * (including FCS and tag) because the length verification is done after
-	 * the Broadcom tag is stripped off the ingress packet.
-	 */
-	if (netdev_uses_dsa(net_dev)) {
-		if (skb_put_padto(skb, ETH_ZLEN + ENET_BRCM_TAG_LEN))
-			goto err_stats;
-	}
-
 	if (skb->len > BGMAC_DESC_CTL1_LEN) {
 		netdev_err(bgmac->net_dev, "Too long skb (%d)\n", skb->len);
 		goto err_drop;
@@ -240,7 +226,6 @@ static netdev_tx_t bgmac_dma_tx_add(struct bgmac *bgmac,
 
 err_drop:
 	dev_kfree_skb(skb);
-err_stats:
 	net_dev->stats.tx_dropped++;
 	net_dev->stats.tx_errors++;
 	return NETDEV_TX_OK;
diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
index 7919f6112ecf..5e34b34f7740 100644
--- a/drivers/net/ethernet/broadcom/bnx2.c
+++ b/drivers/net/ethernet/broadcom/bnx2.c
@@ -5818,8 +5818,8 @@ bnx2_run_loopback(struct bnx2 *bp, int loopback_mode)
 	struct l2_fhdr *rx_hdr;
 	int ret = -ENODEV;
 	struct bnx2_napi *bnapi = &bp->bnx2_napi[0], *tx_napi;
-	struct bnx2_tx_ring_info *txr = &bnapi->tx_ring;
-	struct bnx2_rx_ring_info *rxr = &bnapi->rx_ring;
+	struct bnx2_tx_ring_info *txr;
+	struct bnx2_rx_ring_info *rxr;
 
 	tx_napi = bnapi;
 
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
index 8ae269ec17a1..d7c98e807ca8 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
@@ -738,8 +738,9 @@ static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 			bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
 			break;
 		default:
-			WARN_ONCE(1, "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
-				  be16_to_cpu(skb->protocol));
+			netdev_WARN_ONCE(bp->dev,
+					 "Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
+					 be16_to_cpu(skb->protocol));
 		}
 	}
 #endif
@@ -2482,8 +2483,7 @@ static void bnx2x_bz_fp(struct bnx2x *bp, int index)
 	 */
 	if (bp->dev->features & NETIF_F_LRO)
 		fp->mode = TPA_MODE_LRO;
-	else if (bp->dev->features & NETIF_F_GRO &&
-		 bnx2x_mtu_allows_gro(bp->dev->mtu))
+	else if (bp->dev->features & NETIF_F_GRO_HW)
 		fp->mode = TPA_MODE_GRO;
 	else
 		fp->mode = TPA_MODE_DISABLED;
@@ -4874,6 +4874,9 @@ int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
 	 */
 	dev->mtu = new_mtu;
 
+	if (!bnx2x_mtu_allows_gro(new_mtu))
+		dev->features &= ~NETIF_F_GRO_HW;
+
 	if (IS_PF(bp) && SHMEM2_HAS(bp, curr_cfg))
 		SHMEM2_WR(bp, curr_cfg, CURR_CFG_MET_OS);
 
@@ -4903,10 +4906,13 @@ netdev_features_t bnx2x_fix_features(struct net_device *dev,
 	}
 
 	/* TPA requires Rx CSUM offloading */
-	if (!(features & NETIF_F_RXCSUM)) {
+	if (!(features & NETIF_F_RXCSUM))
+		features &= ~NETIF_F_LRO;
+
+	if (!(features & NETIF_F_GRO) || !bnx2x_mtu_allows_gro(dev->mtu))
+		features &= ~NETIF_F_GRO_HW;
+	if (features & NETIF_F_GRO_HW)
 		features &= ~NETIF_F_LRO;
-		features &= ~NETIF_F_GRO;
-	}
 
 	return features;
 }
@@ -4933,13 +4939,8 @@ int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
 		}
 	}
 
-	/* if GRO is changed while LRO is enabled, don't force a reload */
-	if ((changes & NETIF_F_GRO) && (features & NETIF_F_LRO))
-		changes &= ~NETIF_F_GRO;
-
-	/* if GRO is changed while HW TPA is off, don't force a reload */
-	if ((changes & NETIF_F_GRO) && bp->disable_tpa)
-		changes &= ~NETIF_F_GRO;
+	/* Don't care about GRO changes */
+	changes &= ~NETIF_F_GRO;
 
 	if (changes)
 		bnx2x_reload = true;
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index ddd5d3ebd201..b8388e93520a 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -12409,8 +12409,8 @@ static int bnx2x_init_bp(struct bnx2x *bp)
 
 	/* Set TPA flags */
 	if (bp->disable_tpa) {
-		bp->dev->hw_features &= ~NETIF_F_LRO;
-		bp->dev->features &= ~NETIF_F_LRO;
+		bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
+		bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
 	}
 
 	if (CHIP_IS_E1(bp))
@@ -12934,6 +12934,24 @@ static netdev_features_t bnx2x_features_check(struct sk_buff *skb,
 					      struct net_device *dev,
 					      netdev_features_t features)
 {
+	/*
+	 * A skb with gso_size + header length > 9700 will cause a
+	 * firmware panic. Drop GSO support.
+	 *
+	 * Eventually the upper layer should not pass these packets down.
+	 *
+	 * For speed, if the gso_size is <= 9000, assume there will
+	 * not be 700 bytes of headers and pass it through. Only do a
+	 * full (slow) validation if the gso_size is > 9000.
+	 *
+	 * (Due to the way SKB_BY_FRAGS works this will also do a full
+	 * validation in that case.)
+	 */
+	if (unlikely(skb_is_gso(skb) &&
+		     (skb_shinfo(skb)->gso_size > 9000) &&
+		     !skb_gso_validate_mac_len(skb, 9700)))
+		features &= ~NETIF_F_GSO_MASK;
+
 	features = vlan_features_check(skb, features);
 	return vxlan_features_check(skb, features);
 }
@@ -13282,7 +13300,7 @@ static int bnx2x_init_dev(struct bnx2x *bp, struct pci_dev *pdev,
 
 	dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 		NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6 |
-		NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO |
+		NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO | NETIF_F_GRO_HW |
 		NETIF_F_RXHASH | NETIF_F_HW_VLAN_CTAG_TX;
 	if (!chip_is_e1x) {
 		dev->hw_features |= NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM |
@@ -13318,6 +13336,8 @@ static int bnx2x_init_dev(struct bnx2x *bp, struct pci_dev *pdev,
 
 	dev->features |= dev->hw_features | NETIF_F_HW_VLAN_CTAG_RX;
 	dev->features |= NETIF_F_HIGHDMA;
+	if (dev->features & NETIF_F_LRO)
+		dev->features &= ~NETIF_F_GRO_HW;
 
 	/* Add Loopback capability to the device */
 	dev->hw_features |= NETIF_F_LOOPBACK;
@@ -13893,7 +13913,7 @@ static void bnx2x_register_phc(struct bnx2x *bp)
 	bp->ptp_clock = ptp_clock_register(&bp->ptp_clock_info, &bp->pdev->dev);
 	if (IS_ERR(bp->ptp_clock)) {
 		bp->ptp_clock = NULL;
-		BNX2X_ERR("PTP clock registeration failed\n");
+		BNX2X_ERR("PTP clock registration failed\n");
 	}
 }
 
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
index 3591077a5f6b..ffa7959f6b31 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.c
@@ -812,7 +812,7 @@ static u8 bnx2x_vf_is_pcie_pending(struct bnx2x *bp, u8 abs_vfid)
 	if (!vf)
 		return false;
 
-	dev = pci_get_bus_and_slot(vf->bus, vf->devfn);
+	dev = pci_get_domain_bus_and_slot(vf->domain, vf->bus, vf->devfn);
 	if (dev)
 		return bnx2x_is_pcie_pending(dev);
 	return false;
@@ -1041,6 +1041,13 @@ void bnx2x_iov_init_dmae(struct bnx2x *bp)
 		REG_WR(bp, DMAE_REG_BACKWARD_COMP_EN, 0);
 }
 
+static int bnx2x_vf_domain(struct bnx2x *bp, int vfid)
+{
+	struct pci_dev *dev = bp->pdev;
+
+	return pci_domain_nr(dev->bus);
+}
+
 static int bnx2x_vf_bus(struct bnx2x *bp, int vfid)
 {
 	struct pci_dev *dev = bp->pdev;
@@ -1606,6 +1613,7 @@ int bnx2x_iov_nic_init(struct bnx2x *bp)
 		struct bnx2x_virtf *vf = BP_VF(bp, vfid);
 
 		/* fill in the BDF and bars */
+		vf->domain = bnx2x_vf_domain(bp, vfid);
 		vf->bus = bnx2x_vf_bus(bp, vfid);
 		vf->devfn = bnx2x_vf_devfn(bp, vfid);
 		bnx2x_vf_set_bars(bp, vf);
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
index 53466f6cebab..eb814c65152f 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
@@ -182,6 +182,7 @@ struct bnx2x_virtf {
 	u32 error;	/* 0 means all's-well */
 
 	/* BDF */
+	unsigned int domain;
 	unsigned int bus;
 	unsigned int devfn;
 
diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile
index 59c8ec9c1cad..7c560d545c03 100644
--- a/drivers/net/ethernet/broadcom/bnxt/Makefile
+++ b/drivers/net/ethernet/broadcom/bnxt/Makefile
@@ -1,4 +1,4 @@
 obj-$(CONFIG_BNXT) += bnxt_en.o
 
-bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o
+bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o
 bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 61ca4eb7c6fa..c7e5e6f09647 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -1,7 +1,7 @@
 /* Broadcom NetXtreme-C/E network driver.
  *
  * Copyright (c) 2014-2016 Broadcom Corporation
- * Copyright (c) 2016-2017 Broadcom Limited
+ * Copyright (c) 2016-2018 Broadcom Limited
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -107,6 +107,7 @@ enum board_idx {
 	BCM57416_NPAR,
 	BCM57452,
 	BCM57454,
+	BCM5745x_NPAR,
 	BCM58802,
 	BCM58804,
 	BCM58808,
@@ -147,6 +148,7 @@ static const struct {
 	[BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
 	[BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
 	[BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
+	[BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
 	[BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
 	[BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
 	[BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
@@ -156,6 +158,8 @@ static const struct {
 };
 
 static const struct pci_device_id bnxt_pci_tbl[] = {
+	{ PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
+	{ PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
 	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
@@ -209,6 +213,7 @@ MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 
 static const u16 bnxt_vf_req_snif[] = {
 	HWRM_FUNC_CFG,
+	HWRM_FUNC_VF_CFG,
 	HWRM_PORT_PHY_QCFG,
 	HWRM_CFA_L2_FILTER_ALLOC,
 };
@@ -1434,7 +1439,7 @@ static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
 	    (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
 		u16 vlan_proto = tpa_info->metadata >>
 			RX_CMP_FLAGS2_METADATA_TPID_SFT;
-		u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_VID_MASK;
+		u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
 
 		__vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
 	}
@@ -1510,7 +1515,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 			       (struct rx_tpa_start_cmp_ext *)rxcmp1);
 
 		*event |= BNXT_RX_EVENT;
-		goto next_rx_no_prod;
+		goto next_rx_no_prod_no_len;
 
 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
 		skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
@@ -1526,7 +1531,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 			rc = 1;
 		}
 		*event |= BNXT_RX_EVENT;
-		goto next_rx_no_prod;
+		goto next_rx_no_prod_no_len;
 	}
 
 	cons = rxcmp->rx_cmp_opaque;
@@ -1618,7 +1623,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 	     cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
 	    (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
 		u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
-		u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_VID_MASK;
+		u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
 		u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
 
 		__vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
@@ -1644,7 +1649,10 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
 	rxr->rx_prod = NEXT_RX(prod);
 	rxr->rx_next_cons = NEXT_RX(cons);
 
-next_rx_no_prod:
+	cpr->rx_packets += 1;
+	cpr->rx_bytes += len;
+
+next_rx_no_prod_no_len:
 	*raw_cons = tmp_raw_cons;
 
 	return rc;
@@ -1706,12 +1714,16 @@ static int bnxt_async_event_process(struct bnxt *bp,
 
 		if (BNXT_VF(bp))
 			goto async_event_process_exit;
-		if (data1 & 0x20000) {
+
+		/* print unsupported speed warning in forced speed mode only */
+		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
+		    (data1 & 0x20000)) {
 			u16 fw_speed = link_info->force_link_speed;
 			u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
 
-			netdev_warn(bp->dev, "Link speed %d no longer supported\n",
-				    speed);
+			if (speed != SPEED_UNKNOWN)
+				netdev_warn(bp->dev, "Link speed %d no longer supported\n",
+					    speed);
 		}
 		set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
 		/* fall thru */
@@ -1798,6 +1810,7 @@ static irqreturn_t bnxt_msix(int irq, void *dev_instance)
 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
 	u32 cons = RING_CMP(cpr->cp_raw_cons);
 
+	cpr->event_ctr++;
 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
 	napi_schedule(&bnapi->napi);
 	return IRQ_HANDLED;
@@ -2021,6 +2034,15 @@ static int bnxt_poll(struct napi_struct *napi, int budget)
 			break;
 		}
 	}
+	if (bp->flags & BNXT_FLAG_DIM) {
+		struct net_dim_sample dim_sample;
+
+		net_dim_sample(cpr->event_ctr,
+			       cpr->rx_packets,
+			       cpr->rx_bytes,
+			       &dim_sample);
+		net_dim(&cpr->dim, dim_sample);
+	}
 	mmiowb();
 	return work_done;
 }
@@ -2243,6 +2265,9 @@ static void bnxt_free_rx_rings(struct bnxt *bp)
 		if (rxr->xdp_prog)
 			bpf_prog_put(rxr->xdp_prog);
 
+		if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
+			xdp_rxq_info_unreg(&rxr->xdp_rxq);
+
 		kfree(rxr->rx_tpa);
 		rxr->rx_tpa = NULL;
 
@@ -2276,6 +2301,10 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp)
 
 		ring = &rxr->rx_ring_struct;
 
+		rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
+		if (rc < 0)
+			return rc;
+
 		rc = bnxt_alloc_ring(bp, ring);
 		if (rc)
 			return rc;
@@ -2606,6 +2635,8 @@ static void bnxt_init_cp_rings(struct bnxt *bp)
 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
 
 		ring->fw_ring_id = INVALID_HW_RING_ID;
+		cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
+		cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
 	}
 }
 
@@ -2751,7 +2782,7 @@ void bnxt_set_tpa_flags(struct bnxt *bp)
 		return;
 	if (bp->dev->features & NETIF_F_LRO)
 		bp->flags |= BNXT_FLAG_LRO;
-	if (bp->dev->features & NETIF_F_GRO)
+	else if (bp->dev->features & NETIF_F_GRO_HW)
 		bp->flags |= BNXT_FLAG_GRO;
 }
 
@@ -2830,6 +2861,9 @@ void bnxt_set_ring_params(struct bnxt *bp)
 	bp->cp_ring_mask = bp->cp_bit - 1;
 }
 
+/* Changing allocation mode of RX rings.
+ * TODO: Update when extending xdp_rxq_info to support allocation modes.
+ */
 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
 {
 	if (page_mode) {
@@ -2839,10 +2873,10 @@ int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
 			min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
-		bp->dev->hw_features &= ~NETIF_F_LRO;
-		bp->dev->features &= ~NETIF_F_LRO;
 		bp->rx_dir = DMA_BIDIRECTIONAL;
 		bp->rx_skb_func = bnxt_rx_page_skb;
+		/* Disable LRO or GRO_HW */
+		netdev_update_features(bp->dev);
 	} else {
 		bp->dev->max_mtu = bp->max_mtu;
 		bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
@@ -3813,6 +3847,9 @@ static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
 	struct hwrm_vnic_tpa_cfg_input req = {0};
 
+	if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
+		return 0;
+
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
 
 	if (tpa_flags) {
@@ -4469,6 +4506,42 @@ static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
 	}
 }
 
+static int bnxt_hwrm_get_rings(struct bnxt *bp)
+{
+	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+	struct hwrm_func_qcfg_input req = {0};
+	int rc;
+
+	if (bp->hwrm_spec_code < 0x10601)
+		return 0;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
+	req.fid = cpu_to_le16(0xffff);
+	mutex_lock(&bp->hwrm_cmd_lock);
+	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc) {
+		mutex_unlock(&bp->hwrm_cmd_lock);
+		return -EIO;
+	}
+
+	hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
+	if (bp->flags & BNXT_FLAG_NEW_RM) {
+		u16 cp, stats;
+
+		hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
+		hw_resc->resv_hw_ring_grps =
+			le32_to_cpu(resp->alloc_hw_ring_grps);
+		hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
+		cp = le16_to_cpu(resp->alloc_cmpl_rings);
+		stats = le16_to_cpu(resp->alloc_stat_ctx);
+		cp = min_t(u16, cp, stats);
+		hw_resc->resv_cp_rings = cp;
+	}
+	mutex_unlock(&bp->hwrm_cmd_lock);
+	return 0;
+}
+
 /* Caller must hold bp->hwrm_cmd_lock */
 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
 {
@@ -4488,55 +4561,275 @@ int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
 	return rc;
 }
 
-static int bnxt_hwrm_reserve_tx_rings(struct bnxt *bp, int *tx_rings)
+static void
+__bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
+			     int tx_rings, int rx_rings, int ring_grps,
+			     int cp_rings, int vnics)
+{
+	u32 enables = 0;
+
+	bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
+	req->fid = cpu_to_le16(0xffff);
+	enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
+	req->num_tx_rings = cpu_to_le16(tx_rings);
+	if (bp->flags & BNXT_FLAG_NEW_RM) {
+		enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
+		enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
+				      FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
+		enables |= ring_grps ?
+			   FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
+		enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
+
+		req->num_rx_rings = cpu_to_le16(rx_rings);
+		req->num_hw_ring_grps = cpu_to_le16(ring_grps);
+		req->num_cmpl_rings = cpu_to_le16(cp_rings);
+		req->num_stat_ctxs = req->num_cmpl_rings;
+		req->num_vnics = cpu_to_le16(vnics);
+	}
+	req->enables = cpu_to_le32(enables);
+}
+
+static void
+__bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
+			     struct hwrm_func_vf_cfg_input *req, int tx_rings,
+			     int rx_rings, int ring_grps, int cp_rings,
+			     int vnics)
+{
+	u32 enables = 0;
+
+	bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
+	enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
+	enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
+	enables |= cp_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS |
+			      FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
+	enables |= ring_grps ? FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
+	enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
+
+	req->num_tx_rings = cpu_to_le16(tx_rings);
+	req->num_rx_rings = cpu_to_le16(rx_rings);
+	req->num_hw_ring_grps = cpu_to_le16(ring_grps);
+	req->num_cmpl_rings = cpu_to_le16(cp_rings);
+	req->num_stat_ctxs = req->num_cmpl_rings;
+	req->num_vnics = cpu_to_le16(vnics);
+
+	req->enables = cpu_to_le32(enables);
+}
+
+static int
+bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
+			   int ring_grps, int cp_rings, int vnics)
 {
 	struct hwrm_func_cfg_input req = {0};
 	int rc;
 
-	if (bp->hwrm_spec_code < 0x10601)
+	__bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
+				     cp_rings, vnics);
+	if (!req.enables)
 		return 0;
 
-	if (BNXT_VF(bp))
+	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		return -ENOMEM;
+
+	if (bp->hwrm_spec_code < 0x10601)
+		bp->hw_resc.resv_tx_rings = tx_rings;
+
+	rc = bnxt_hwrm_get_rings(bp);
+	return rc;
+}
+
+static int
+bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
+			   int ring_grps, int cp_rings, int vnics)
+{
+	struct hwrm_func_vf_cfg_input req = {0};
+	int rc;
+
+	if (!(bp->flags & BNXT_FLAG_NEW_RM)) {
+		bp->hw_resc.resv_tx_rings = tx_rings;
 		return 0;
+	}
 
-	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
-	req.fid = cpu_to_le16(0xffff);
-	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
-	req.num_tx_rings = cpu_to_le16(*tx_rings);
+	__bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
+				     cp_rings, vnics);
 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		return -ENOMEM;
+
+	rc = bnxt_hwrm_get_rings(bp);
+	return rc;
+}
+
+static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
+				   int cp, int vnic)
+{
+	if (BNXT_PF(bp))
+		return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, vnic);
+	else
+		return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, vnic);
+}
+
+static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
+			   bool shared);
+
+static int __bnxt_reserve_rings(struct bnxt *bp)
+{
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+	int tx = bp->tx_nr_rings;
+	int rx = bp->rx_nr_rings;
+	int cp = bp->cp_nr_rings;
+	int grp, rx_rings, rc;
+	bool sh = false;
+	int vnic = 1;
+
+	if (bp->hwrm_spec_code < 0x10601)
+		return 0;
+
+	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
+		sh = true;
+	if (bp->flags & BNXT_FLAG_RFS)
+		vnic = rx + 1;
+	if (bp->flags & BNXT_FLAG_AGG_RINGS)
+		rx <<= 1;
+
+	grp = bp->rx_nr_rings;
+	if (tx == hw_resc->resv_tx_rings &&
+	    (!(bp->flags & BNXT_FLAG_NEW_RM) ||
+	      (rx == hw_resc->resv_rx_rings &&
+	       grp == hw_resc->resv_hw_ring_grps &&
+	       cp == hw_resc->resv_cp_rings && vnic == hw_resc->resv_vnics)))
+		return 0;
+
+	rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, vnic);
 	if (rc)
 		return rc;
 
-	mutex_lock(&bp->hwrm_cmd_lock);
-	rc = __bnxt_hwrm_get_tx_rings(bp, 0xffff, tx_rings);
-	mutex_unlock(&bp->hwrm_cmd_lock);
-	if (!rc)
-		bp->tx_reserved_rings = *tx_rings;
+	tx = hw_resc->resv_tx_rings;
+	if (bp->flags & BNXT_FLAG_NEW_RM) {
+		rx = hw_resc->resv_rx_rings;
+		cp = hw_resc->resv_cp_rings;
+		grp = hw_resc->resv_hw_ring_grps;
+		vnic = hw_resc->resv_vnics;
+	}
+
+	rx_rings = rx;
+	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
+		if (rx >= 2) {
+			rx_rings = rx >> 1;
+		} else {
+			if (netif_running(bp->dev))
+				return -ENOMEM;
+
+			bp->flags &= ~BNXT_FLAG_AGG_RINGS;
+			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
+			bp->dev->hw_features &= ~NETIF_F_LRO;
+			bp->dev->features &= ~NETIF_F_LRO;
+			bnxt_set_ring_params(bp);
+		}
+	}
+	rx_rings = min_t(int, rx_rings, grp);
+	rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
+	if (bp->flags & BNXT_FLAG_AGG_RINGS)
+		rx = rx_rings << 1;
+	cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
+	bp->tx_nr_rings = tx;
+	bp->rx_nr_rings = rx_rings;
+	bp->cp_nr_rings = cp;
+
+	if (!tx || !rx || !cp || !grp || !vnic)
+		return -ENOMEM;
+
 	return rc;
 }
 
-static int bnxt_hwrm_check_tx_rings(struct bnxt *bp, int tx_rings)
+static bool bnxt_need_reserve_rings(struct bnxt *bp)
 {
-	struct hwrm_func_cfg_input req = {0};
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+	int rx = bp->rx_nr_rings;
+	int vnic = 1;
+
+	if (bp->hwrm_spec_code < 0x10601)
+		return false;
+
+	if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
+		return true;
+
+	if (bp->flags & BNXT_FLAG_RFS)
+		vnic = rx + 1;
+	if (bp->flags & BNXT_FLAG_AGG_RINGS)
+		rx <<= 1;
+	if ((bp->flags & BNXT_FLAG_NEW_RM) &&
+	    (hw_resc->resv_rx_rings != rx ||
+	     hw_resc->resv_cp_rings != bp->cp_nr_rings ||
+	     hw_resc->resv_vnics != vnic))
+		return true;
+	return false;
+}
+
+static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
+				    int ring_grps, int cp_rings, int vnics)
+{
+	struct hwrm_func_vf_cfg_input req = {0};
+	u32 flags;
 	int rc;
 
-	if (bp->hwrm_spec_code < 0x10801)
+	if (!(bp->flags & BNXT_FLAG_NEW_RM))
 		return 0;
 
-	if (BNXT_VF(bp))
-		return 0;
+	__bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
+				     cp_rings, vnics);
+	flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
+		FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
+		FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
+		FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST |
+		FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
+		FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
 
-	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
-	req.fid = cpu_to_le16(0xffff);
-	req.flags = cpu_to_le32(FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST);
-	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS);
-	req.num_tx_rings = cpu_to_le16(tx_rings);
+	req.flags = cpu_to_le32(flags);
 	rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 	if (rc)
 		return -ENOMEM;
 	return 0;
 }
 
+static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
+				    int ring_grps, int cp_rings, int vnics)
+{
+	struct hwrm_func_cfg_input req = {0};
+	u32 flags;
+	int rc;
+
+	__bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
+				     cp_rings, vnics);
+	flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
+	if (bp->flags & BNXT_FLAG_NEW_RM)
+		flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
+			 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
+			 FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST |
+			 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
+			 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
+
+	req.flags = cpu_to_le32(flags);
+	rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		return -ENOMEM;
+	return 0;
+}
+
+static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
+				 int ring_grps, int cp_rings, int vnics)
+{
+	if (bp->hwrm_spec_code < 0x10801)
+		return 0;
+
+	if (BNXT_PF(bp))
+		return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
+						ring_grps, cp_rings, vnics);
+
+	return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
+					cp_rings, vnics);
+}
+
 static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal,
 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
 {
@@ -4579,6 +4872,36 @@ static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal,
 	req->flags = cpu_to_le16(flags);
 }
 
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
+{
+	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
+	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
+	struct bnxt_coal coal;
+	unsigned int grp_idx;
+
+	/* Tick values in micro seconds.
+	 * 1 coal_buf x bufs_per_record = 1 completion record.
+	 */
+	memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
+
+	coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
+	coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
+
+	if (!bnapi->rx_ring)
+		return -ENODEV;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
+			       HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
+
+	bnxt_hwrm_set_coal_params(&coal, &req_rx);
+
+	grp_idx = bnapi->index;
+	req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
+
+	return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
+				 HWRM_CMD_TIMEOUT);
+}
+
 int bnxt_hwrm_set_coal(struct bnxt *bp)
 {
 	int i, rc = 0;
@@ -4732,11 +5055,60 @@ static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
 	return rc;
 }
 
-static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
+static int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp)
+{
+	struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
+	struct hwrm_func_resource_qcaps_input req = {0};
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+	int rc;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
+	req.fid = cpu_to_le16(0xffff);
+
+	mutex_lock(&bp->hwrm_cmd_lock);
+	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc) {
+		rc = -EIO;
+		goto hwrm_func_resc_qcaps_exit;
+	}
+
+	hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
+	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
+	hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
+	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
+	hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
+	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
+	hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
+	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
+	hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
+	hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
+	hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
+	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
+	hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
+	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
+	hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
+	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
+
+	if (BNXT_PF(bp)) {
+		struct bnxt_pf_info *pf = &bp->pf;
+
+		pf->vf_resv_strategy =
+			le16_to_cpu(resp->vf_reservation_strategy);
+		if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL)
+			pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
+	}
+hwrm_func_resc_qcaps_exit:
+	mutex_unlock(&bp->hwrm_cmd_lock);
+	return rc;
+}
+
+static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
 {
 	int rc = 0;
 	struct hwrm_func_qcaps_input req = {0};
 	struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+	u32 flags;
 
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
 	req.fid = cpu_to_le16(0xffff);
@@ -4746,16 +5118,27 @@ static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 	if (rc)
 		goto hwrm_func_qcaps_exit;
 
-	if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED))
+	flags = le32_to_cpu(resp->flags);
+	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
 		bp->flags |= BNXT_FLAG_ROCEV1_CAP;
-	if (resp->flags & cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED))
+	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
 		bp->flags |= BNXT_FLAG_ROCEV2_CAP;
 
 	bp->tx_push_thresh = 0;
-	if (resp->flags &
-	    cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
+	if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
 		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
 
+	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
+	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
+	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
+	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
+	hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
+	if (!hw_resc->max_hw_ring_grps)
+		hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
+	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
+	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
+	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
+
 	if (BNXT_PF(bp)) {
 		struct bnxt_pf_info *pf = &bp->pf;
 
@@ -4763,16 +5146,6 @@ static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 		pf->port_id = le16_to_cpu(resp->port_id);
 		bp->dev->dev_port = pf->port_id;
 		memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
-		pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
-		pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
-		pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
-		pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
-		pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
-		if (!pf->max_hw_ring_grps)
-			pf->max_hw_ring_grps = pf->max_tx_rings;
-		pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
-		pf->max_vnics = le16_to_cpu(resp->max_vnics);
-		pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
 		pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
 		pf->max_vfs = le16_to_cpu(resp->max_vfs);
 		pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
@@ -4781,26 +5154,13 @@ static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 		pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
 		pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
 		pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
-		if (resp->flags &
-		    cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED))
+		if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
 			bp->flags |= BNXT_FLAG_WOL_CAP;
 	} else {
 #ifdef CONFIG_BNXT_SRIOV
 		struct bnxt_vf_info *vf = &bp->vf;
 
 		vf->fw_fid = le16_to_cpu(resp->fid);
-
-		vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
-		vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
-		vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
-		vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
-		vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
-		if (!vf->max_hw_ring_grps)
-			vf->max_hw_ring_grps = vf->max_tx_rings;
-		vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
-		vf->max_vnics = le16_to_cpu(resp->max_vnics);
-		vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
-
 		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
 #endif
 	}
@@ -4810,6 +5170,21 @@ static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
 	return rc;
 }
 
+static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
+{
+	int rc;
+
+	rc = __bnxt_hwrm_func_qcaps(bp);
+	if (rc)
+		return rc;
+	if (bp->hwrm_spec_code >= 0x10803) {
+		rc = bnxt_hwrm_func_resc_qcaps(bp);
+		if (!rc)
+			bp->flags |= BNXT_FLAG_NEW_RM;
+	}
+	return 0;
+}
+
 static int bnxt_hwrm_func_reset(struct bnxt *bp)
 {
 	struct hwrm_func_reset_input req = {0};
@@ -4879,23 +5254,24 @@ static int bnxt_hwrm_ver_get(struct bnxt *bp)
 
 	memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
 
-	bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
-			     resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
-	if (resp->hwrm_intf_maj < 1) {
+	bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
+			     resp->hwrm_intf_min_8b << 8 |
+			     resp->hwrm_intf_upd_8b;
+	if (resp->hwrm_intf_maj_8b < 1) {
 		netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
-			    resp->hwrm_intf_maj, resp->hwrm_intf_min,
-			    resp->hwrm_intf_upd);
+			    resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
+			    resp->hwrm_intf_upd_8b);
 		netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
 	}
 	snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
-		 resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
-		 resp->hwrm_fw_rsvd);
+		 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
+		 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
 
 	bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
 	if (!bp->hwrm_cmd_timeout)
 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
 
-	if (resp->hwrm_intf_maj >= 1)
+	if (resp->hwrm_intf_maj_8b >= 1)
 		bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
 
 	bp->chip_num = le16_to_cpu(resp->chip_num);
@@ -5031,6 +5407,28 @@ static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
 	return rc;
 }
 
+static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
+{
+	struct hwrm_func_cfg_input req = {0};
+	int rc;
+
+	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
+		return 0;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
+	req.fid = cpu_to_le16(0xffff);
+	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
+	req.cache_linesize = FUNC_QCFG_RESP_CACHE_LINESIZE_CACHE_LINESIZE_64;
+	if (size == 128)
+		req.cache_linesize =
+			FUNC_QCFG_RESP_CACHE_LINESIZE_CACHE_LINESIZE_128;
+
+	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
+	if (rc)
+		rc = -EIO;
+	return rc;
+}
+
 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
 {
 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
@@ -5166,15 +5564,6 @@ static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
 				   rc);
 			goto err_out;
 		}
-		if (bp->tx_reserved_rings != bp->tx_nr_rings) {
-			int tx = bp->tx_nr_rings;
-
-			if (bnxt_hwrm_reserve_tx_rings(bp, &tx) ||
-			    tx < bp->tx_nr_rings) {
-				rc = -ENOMEM;
-				goto err_out;
-			}
-		}
 	}
 
 	rc = bnxt_hwrm_ring_alloc(bp);
@@ -5394,79 +5783,45 @@ static int bnxt_setup_int_mode(struct bnxt *bp)
 #ifdef CONFIG_RFS_ACCEL
 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		return bp->vf.max_rsscos_ctxs;
-#endif
-	return bp->pf.max_rsscos_ctxs;
+	return bp->hw_resc.max_rsscos_ctxs;
 }
 
 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		return bp->vf.max_vnics;
-#endif
-	return bp->pf.max_vnics;
+	return bp->hw_resc.max_vnics;
 }
 #endif
 
 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		return bp->vf.max_stat_ctxs;
-#endif
-	return bp->pf.max_stat_ctxs;
+	return bp->hw_resc.max_stat_ctxs;
 }
 
 void bnxt_set_max_func_stat_ctxs(struct bnxt *bp, unsigned int max)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		bp->vf.max_stat_ctxs = max;
-	else
-#endif
-		bp->pf.max_stat_ctxs = max;
+	bp->hw_resc.max_stat_ctxs = max;
 }
 
 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		return bp->vf.max_cp_rings;
-#endif
-	return bp->pf.max_cp_rings;
+	return bp->hw_resc.max_cp_rings;
 }
 
 void bnxt_set_max_func_cp_rings(struct bnxt *bp, unsigned int max)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		bp->vf.max_cp_rings = max;
-	else
-#endif
-		bp->pf.max_cp_rings = max;
+	bp->hw_resc.max_cp_rings = max;
 }
 
 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		return min_t(unsigned int, bp->vf.max_irqs,
-			     bp->vf.max_cp_rings);
-#endif
-	return min_t(unsigned int, bp->pf.max_irqs, bp->pf.max_cp_rings);
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+
+	return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
 }
 
 void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
 {
-#if defined(CONFIG_BNXT_SRIOV)
-	if (BNXT_VF(bp))
-		bp->vf.max_irqs = max_irqs;
-	else
-#endif
-		bp->pf.max_irqs = max_irqs;
+	bp->hw_resc.max_irqs = max_irqs;
 }
 
 static int bnxt_init_msix(struct bnxt *bp)
@@ -5505,7 +5860,6 @@ static int bnxt_init_msix(struct bnxt *bp)
 		if (rc)
 			goto msix_setup_exit;
 
-		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
 		bp->cp_nr_rings = (min == 1) ?
 				  max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
 				  bp->tx_nr_rings + bp->rx_nr_rings;
@@ -5537,7 +5891,6 @@ static int bnxt_init_inta(struct bnxt *bp)
 	bp->rx_nr_rings = 1;
 	bp->tx_nr_rings = 1;
 	bp->cp_nr_rings = 1;
-	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
 	bp->flags |= BNXT_FLAG_SHARED_RINGS;
 	bp->irq_tbl[0].vector = bp->pdev->irq;
 	return 0;
@@ -5567,6 +5920,36 @@ static void bnxt_clear_int_mode(struct bnxt *bp)
 	bp->flags &= ~BNXT_FLAG_USING_MSIX;
 }
 
+static int bnxt_reserve_rings(struct bnxt *bp)
+{
+	int orig_cp = bp->hw_resc.resv_cp_rings;
+	int tcs = netdev_get_num_tc(bp->dev);
+	int rc;
+
+	if (!bnxt_need_reserve_rings(bp))
+		return 0;
+
+	rc = __bnxt_reserve_rings(bp);
+	if (rc) {
+		netdev_err(bp->dev, "ring reservation failure rc: %d\n", rc);
+		return rc;
+	}
+	if ((bp->flags & BNXT_FLAG_NEW_RM) && bp->cp_nr_rings > orig_cp) {
+		bnxt_clear_int_mode(bp);
+		rc = bnxt_init_int_mode(bp);
+		if (rc)
+			return rc;
+	}
+	if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
+		netdev_err(bp->dev, "tx ring reservation failure\n");
+		netdev_reset_tc(bp->dev);
+		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
+		return -ENOMEM;
+	}
+	bp->num_stat_ctxs = bp->cp_nr_rings;
+	return 0;
+}
+
 static void bnxt_free_irq(struct bnxt *bp)
 {
 	struct bnxt_irq *irq;
@@ -5692,8 +6075,14 @@ static void bnxt_disable_napi(struct bnxt *bp)
 	if (!bp->bnapi)
 		return;
 
-	for (i = 0; i < bp->cp_nr_rings; i++)
+	for (i = 0; i < bp->cp_nr_rings; i++) {
+		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
+
+		if (bp->bnapi[i]->rx_ring)
+			cancel_work_sync(&cpr->dim.work);
+
 		napi_disable(&bp->bnapi[i]->napi);
+	}
 }
 
 static void bnxt_enable_napi(struct bnxt *bp)
@@ -5701,7 +6090,13 @@ static void bnxt_enable_napi(struct bnxt *bp)
 	int i;
 
 	for (i = 0; i < bp->cp_nr_rings; i++) {
+		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
 		bp->bnapi[i]->in_reset = false;
+
+		if (bp->bnapi[i]->rx_ring) {
+			INIT_WORK(&cpr->dim.work, bnxt_dim_work);
+			cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+		}
 		napi_enable(&bp->bnapi[i]->napi);
 	}
 }
@@ -6311,6 +6706,10 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 	bnxt_preset_reg_win(bp);
 	netif_carrier_off(bp->dev);
 	if (irq_re_init) {
+		rc = bnxt_reserve_rings(bp);
+		if (rc)
+			return rc;
+
 		rc = bnxt_setup_int_mode(bp);
 		if (rc) {
 			netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
@@ -6446,23 +6845,13 @@ static bool bnxt_drv_busy(struct bnxt *bp)
 		test_bit(BNXT_STATE_READ_STATS, &bp->state));
 }
 
-int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
+static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
+			     bool link_re_init)
 {
-	int rc = 0;
-
-#ifdef CONFIG_BNXT_SRIOV
-	if (bp->sriov_cfg) {
-		rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
-						      !bp->sriov_cfg,
-						      BNXT_SRIOV_CFG_WAIT_TMO);
-		if (rc)
-			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
-	}
-
 	/* Close the VF-reps before closing PF */
 	if (BNXT_PF(bp))
 		bnxt_vf_reps_close(bp);
-#endif
+
 	/* Change device state to avoid TX queue wake up's */
 	bnxt_tx_disable(bp);
 
@@ -6485,6 +6874,22 @@ int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
 		bnxt_del_napi(bp);
 	}
 	bnxt_free_mem(bp, irq_re_init);
+}
+
+int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
+{
+	int rc = 0;
+
+#ifdef CONFIG_BNXT_SRIOV
+	if (bp->sriov_cfg) {
+		rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
+						      !bp->sriov_cfg,
+						      BNXT_SRIOV_CFG_WAIT_TMO);
+		if (rc)
+			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
+	}
+#endif
+	__bnxt_close_nic(bp, irq_re_init, link_re_init);
 	return rc;
 }
 
@@ -6764,13 +7169,26 @@ static bool bnxt_rfs_capable(struct bnxt *bp)
 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
 		max_rss_ctxs = max_vnics;
 	if (vnics > max_vnics || vnics > max_rss_ctxs) {
-		netdev_warn(bp->dev,
-			    "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
-			    min(max_rss_ctxs - 1, max_vnics - 1));
+		if (bp->rx_nr_rings > 1)
+			netdev_warn(bp->dev,
+				    "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
+				    min(max_rss_ctxs - 1, max_vnics - 1));
 		return false;
 	}
 
-	return true;
+	if (!(bp->flags & BNXT_FLAG_NEW_RM))
+		return true;
+
+	if (vnics == bp->hw_resc.resv_vnics)
+		return true;
+
+	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, vnics);
+	if (vnics <= bp->hw_resc.resv_vnics)
+		return true;
+
+	netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
+	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 1);
+	return false;
 #else
 	return false;
 #endif
@@ -6784,6 +7202,15 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev,
 	if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
 		features &= ~NETIF_F_NTUPLE;
 
+	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
+		features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
+
+	if (!(features & NETIF_F_GRO))
+		features &= ~NETIF_F_GRO_HW;
+
+	if (features & NETIF_F_GRO_HW)
+		features &= ~NETIF_F_LRO;
+
 	/* Both CTAG and STAG VLAN accelaration on the RX side have to be
 	 * turned on or off together.
 	 */
@@ -6817,9 +7244,9 @@ static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
 	bool update_tpa = false;
 
 	flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
-	if ((features & NETIF_F_GRO) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
+	if (features & NETIF_F_GRO_HW)
 		flags |= BNXT_FLAG_GRO;
-	if (features & NETIF_F_LRO)
+	else if (features & NETIF_F_LRO)
 		flags |= BNXT_FLAG_LRO;
 
 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
@@ -7096,7 +7523,8 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
 {
 	int max_rx, max_tx, tx_sets = 1;
 	int tx_rings_needed;
-	int rc;
+	int rx_rings = rx;
+	int cp, vnics, rc;
 
 	if (tcs)
 		tx_sets = tcs;
@@ -7112,7 +7540,15 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
 	if (max_tx < tx_rings_needed)
 		return -ENOMEM;
 
-	return bnxt_hwrm_check_tx_rings(bp, tx_rings_needed);
+	vnics = 1;
+	if (bp->flags & BNXT_FLAG_RFS)
+		vnics += rx_rings;
+
+	if (bp->flags & BNXT_FLAG_AGG_RINGS)
+		rx_rings <<= 1;
+	cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
+	return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
+				     vnics);
 }
 
 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
@@ -7346,7 +7782,8 @@ static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 {
 	struct bnxt *bp = cb_priv;
 
-	if (!bnxt_tc_flower_enabled(bp) || !tc_can_offload(bp->dev))
+	if (!bnxt_tc_flower_enabled(bp) ||
+	    !tc_cls_can_offload_and_chain0(bp->dev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
@@ -7717,12 +8154,8 @@ int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr)
 
 	switch (attr->id) {
 	case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
-		/* In SRIOV each PF-pool (PF + child VFs) serves as a
-		 * switching domain, the PF's perm mac-addr can be used
-		 * as the unique parent-id
-		 */
-		attr->u.ppid.id_len = ETH_ALEN;
-		ether_addr_copy(attr->u.ppid.id, bp->pf.mac_addr);
+		attr->u.ppid.id_len = sizeof(bp->switch_id);
+		memcpy(attr->u.ppid.id, bp->switch_id, attr->u.ppid.id_len);
 		break;
 	default:
 		return -EOPNOTSUPP;
@@ -7800,8 +8233,6 @@ static void bnxt_remove_one(struct pci_dev *pdev)
 	bnxt_dcb_free(bp);
 	kfree(bp->edev);
 	bp->edev = NULL;
-	if (bp->xdp_prog)
-		bpf_prog_put(bp->xdp_prog);
 	bnxt_cleanup_pci(bp);
 	free_netdev(dev);
 }
@@ -7869,24 +8300,14 @@ static int bnxt_get_max_irq(struct pci_dev *pdev)
 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
 				int *max_cp)
 {
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
 	int max_ring_grps = 0;
 
-#ifdef CONFIG_BNXT_SRIOV
-	if (!BNXT_PF(bp)) {
-		*max_tx = bp->vf.max_tx_rings;
-		*max_rx = bp->vf.max_rx_rings;
-		*max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
-		*max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
-		max_ring_grps = bp->vf.max_hw_ring_grps;
-	} else
-#endif
-	{
-		*max_tx = bp->pf.max_tx_rings;
-		*max_rx = bp->pf.max_rx_rings;
-		*max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
-		*max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
-		max_ring_grps = bp->pf.max_hw_ring_grps;
-	}
+	*max_tx = hw_resc->max_tx_rings;
+	*max_rx = hw_resc->max_rx_rings;
+	*max_cp = min_t(int, hw_resc->max_irqs, hw_resc->max_cp_rings);
+	*max_cp = min_t(int, *max_cp, hw_resc->max_stat_ctxs);
+	max_ring_grps = hw_resc->max_hw_ring_grps;
 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
 		*max_cp -= 1;
 		*max_rx -= 2;
@@ -7922,8 +8343,8 @@ static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
 		if (rc)
 			return rc;
 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
-		bp->dev->hw_features &= ~NETIF_F_LRO;
-		bp->dev->features &= ~NETIF_F_LRO;
+		bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
+		bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
 		bnxt_set_ring_params(bp);
 	}
 
@@ -7951,6 +8372,17 @@ static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
 	return rc;
 }
 
+/* In initial default shared ring setting, each shared ring must have a
+ * RX/TX ring pair.
+ */
+static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
+{
+	bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
+	bp->rx_nr_rings = bp->cp_nr_rings;
+	bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
+	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
+}
+
 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
 {
 	int dflt_rings, max_rx_rings, max_tx_rings, rc;
@@ -7966,14 +8398,26 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
 		return rc;
 	bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
 	bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
+	if (sh)
+		bnxt_trim_dflt_sh_rings(bp);
+	else
+		bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
+	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
 
-	rc = bnxt_hwrm_reserve_tx_rings(bp, &bp->tx_nr_rings_per_tc);
+	rc = __bnxt_reserve_rings(bp);
 	if (rc)
 		netdev_warn(bp->dev, "Unable to reserve tx rings\n");
+	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
+	if (sh)
+		bnxt_trim_dflt_sh_rings(bp);
 
-	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
-	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
-			       bp->tx_nr_rings + bp->rx_nr_rings;
+	/* Rings may have been trimmed, re-reserve the trimmed rings. */
+	if (bnxt_need_reserve_rings(bp)) {
+		rc = __bnxt_reserve_rings(bp);
+		if (rc)
+			netdev_warn(bp->dev, "2nd rings reservation failed.\n");
+		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
+	}
 	bp->num_stat_ctxs = bp->cp_nr_rings;
 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
 		bp->rx_nr_rings++;
@@ -7982,11 +8426,30 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
 	return rc;
 }
 
-void bnxt_restore_pf_fw_resources(struct bnxt *bp)
+int bnxt_restore_pf_fw_resources(struct bnxt *bp)
 {
+	int rc;
+
 	ASSERT_RTNL();
+	if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
+		return 0;
+
 	bnxt_hwrm_func_qcaps(bp);
-	bnxt_subtract_ulp_resources(bp, BNXT_ROCE_ULP);
+
+	if (netif_running(bp->dev))
+		__bnxt_close_nic(bp, true, false);
+
+	bnxt_clear_int_mode(bp);
+	rc = bnxt_init_int_mode(bp);
+
+	if (netif_running(bp->dev)) {
+		if (rc)
+			dev_close(bp->dev);
+		else
+			rc = bnxt_open_nic(bp, true, false);
+	}
+
+	return rc;
 }
 
 static int bnxt_init_mac_addr(struct bnxt *bp)
@@ -8000,7 +8463,7 @@ static int bnxt_init_mac_addr(struct bnxt *bp)
 		struct bnxt_vf_info *vf = &bp->vf;
 
 		if (is_valid_ether_addr(vf->mac_addr)) {
-			/* overwrite netdev dev_adr with admin VF MAC */
+			/* overwrite netdev dev_addr with admin VF MAC */
 			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
 		} else {
 			eth_hw_addr_random(bp->dev);
@@ -8106,7 +8569,11 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
 	dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
 			    NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
+	if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
+		dev->hw_features |= NETIF_F_GRO_HW;
 	dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
+	if (dev->features & NETIF_F_GRO_HW)
+		dev->features &= ~NETIF_F_LRO;
 	dev->priv_flags |= IFF_UNICAST_FLT;
 
 #ifdef CONFIG_BNXT_SRIOV
@@ -8202,12 +8669,19 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (rc)
 		goto init_err_pci_clean;
 
+	/* No TC has been set yet and rings may have been trimmed due to
+	 * limited MSIX, so we re-initialize the TX rings per TC.
+	 */
+	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
+
 	bnxt_get_wol_settings(bp);
 	if (bp->flags & BNXT_FLAG_WOL_CAP)
 		device_set_wakeup_enable(&pdev->dev, bp->wol);
 	else
 		device_set_wakeup_capable(&pdev->dev, false);
 
+	bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
+
 	if (BNXT_PF(bp)) {
 		if (!bnxt_pf_wq) {
 			bnxt_pf_wq =
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 5359a1f0045f..5e3d62189cab 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -1,7 +1,7 @@
 /* Broadcom NetXtreme-C/E network driver.
  *
  * Copyright (c) 2014-2016 Broadcom Corporation
- * Copyright (c) 2016-2017 Broadcom Limited
+ * Copyright (c) 2016-2018 Broadcom Limited
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -12,10 +12,10 @@
 #define BNXT_H
 
 #define DRV_MODULE_NAME		"bnxt_en"
-#define DRV_MODULE_VERSION	"1.8.0"
+#define DRV_MODULE_VERSION	"1.9.0"
 
 #define DRV_VER_MAJ	1
-#define DRV_VER_MIN	8
+#define DRV_VER_MIN	9
 #define DRV_VER_UPD	0
 
 #include <linux/interrupt.h>
@@ -23,6 +23,8 @@
 #include <net/devlink.h>
 #include <net/dst_metadata.h>
 #include <net/switchdev.h>
+#include <net/xdp.h>
+#include <linux/net_dim.h>
 
 struct tx_bd {
 	__le32 tx_bd_len_flags_type;
@@ -187,6 +189,7 @@ struct rx_cmp_ext {
 	#define RX_CMP_FLAGS2_T_L4_CS_CALC			(0x1 << 3)
 	#define RX_CMP_FLAGS2_META_FORMAT_VLAN			(0x1 << 4)
 	__le32 rx_cmp_meta_data;
+	#define RX_CMP_FLAGS2_METADATA_TCI_MASK			0xffff
 	#define RX_CMP_FLAGS2_METADATA_VID_MASK			0xfff
 	#define RX_CMP_FLAGS2_METADATA_TPID_MASK		0xffff0000
 	 #define RX_CMP_FLAGS2_METADATA_TPID_SFT		 16
@@ -607,6 +610,17 @@ struct bnxt_tx_ring_info {
 	struct bnxt_ring_struct	tx_ring_struct;
 };
 
+struct bnxt_coal {
+	u16			coal_ticks;
+	u16			coal_ticks_irq;
+	u16			coal_bufs;
+	u16			coal_bufs_irq;
+			/* RING_IDLE enabled when coal ticks < idle_thresh  */
+	u16			idle_thresh;
+	u8			bufs_per_record;
+	u8			budget;
+};
+
 struct bnxt_tpa_info {
 	void			*data;
 	u8			*data_ptr;
@@ -664,12 +678,20 @@ struct bnxt_rx_ring_info {
 
 	struct bnxt_ring_struct	rx_ring_struct;
 	struct bnxt_ring_struct	rx_agg_ring_struct;
+	struct xdp_rxq_info	xdp_rxq;
 };
 
 struct bnxt_cp_ring_info {
 	u32			cp_raw_cons;
 	void __iomem		*cp_doorbell;
 
+	struct bnxt_coal	rx_ring_coal;
+	u64			rx_packets;
+	u64			rx_bytes;
+	u64			event_ctr;
+
+	struct net_dim		dim;
+
 	struct tx_cmp		*cp_desc_ring[MAX_CP_PAGES];
 
 	dma_addr_t		cp_desc_mapping[MAX_CP_PAGES];
@@ -755,19 +777,38 @@ struct bnxt_vnic_info {
 #define BNXT_VNIC_RFS_NEW_RSS_FLAG	0x10
 };
 
-#if defined(CONFIG_BNXT_SRIOV)
-struct bnxt_vf_info {
-	u16	fw_fid;
-	u8	mac_addr[ETH_ALEN];
+struct bnxt_hw_resc {
+	u16	min_rsscos_ctxs;
 	u16	max_rsscos_ctxs;
+	u16	min_cp_rings;
 	u16	max_cp_rings;
+	u16	resv_cp_rings;
+	u16	min_tx_rings;
 	u16	max_tx_rings;
+	u16	resv_tx_rings;
+	u16	min_rx_rings;
 	u16	max_rx_rings;
+	u16	resv_rx_rings;
+	u16	min_hw_ring_grps;
 	u16	max_hw_ring_grps;
+	u16	resv_hw_ring_grps;
+	u16	min_l2_ctxs;
 	u16	max_l2_ctxs;
-	u16	max_irqs;
+	u16	min_vnics;
 	u16	max_vnics;
+	u16	resv_vnics;
+	u16	min_stat_ctxs;
 	u16	max_stat_ctxs;
+	u16	max_irqs;
+};
+
+#if defined(CONFIG_BNXT_SRIOV)
+struct bnxt_vf_info {
+	u16	fw_fid;
+	u8	mac_addr[ETH_ALEN];	/* PF assigned MAC Address */
+	u8	vf_mac_addr[ETH_ALEN];	/* VF assigned MAC address, only
+					 * stored by PF.
+					 */
 	u16	vlan;
 	u32	flags;
 #define BNXT_VF_QOS		0x1
@@ -788,15 +829,6 @@ struct bnxt_pf_info {
 	u16	fw_fid;
 	u16	port_id;
 	u8	mac_addr[ETH_ALEN];
-	u16	max_rsscos_ctxs;
-	u16	max_cp_rings;
-	u16	max_tx_rings; /* HW assigned max tx rings for this PF */
-	u16	max_rx_rings; /* HW assigned max rx rings for this PF */
-	u16	max_hw_ring_grps;
-	u16	max_irqs;
-	u16	max_l2_ctxs;
-	u16	max_vnics;
-	u16	max_stat_ctxs;
 	u32	first_vf_id;
 	u16	active_vfs;
 	u16	max_vfs;
@@ -808,6 +840,9 @@ struct bnxt_pf_info {
 	u32	max_rx_wm_flows;
 	unsigned long	*vf_event_bmap;
 	u16	hwrm_cmd_req_pages;
+	u8	vf_resv_strategy;
+#define BNXT_VF_RESV_STRATEGY_MAXIMAL	0
+#define BNXT_VF_RESV_STRATEGY_MINIMAL	1
 	void			*hwrm_cmd_req_addr[4];
 	dma_addr_t		hwrm_cmd_req_dma_addr[4];
 	struct bnxt_vf_info	*vf;
@@ -944,17 +979,6 @@ struct bnxt_test_info {
 #define BNXT_CAG_REG_LEGACY_INT_STATUS	0x4014
 #define BNXT_CAG_REG_BASE		0x300000
 
-struct bnxt_coal {
-	u16			coal_ticks;
-	u16			coal_ticks_irq;
-	u16			coal_bufs;
-	u16			coal_bufs_irq;
-			/* RING_IDLE enabled when coal ticks < idle_thresh  */
-	u16			idle_thresh;
-	u8			bufs_per_record;
-	u8			budget;
-};
-
 struct bnxt_tc_flow_stats {
 	u64		packets;
 	u64		bytes;
@@ -1126,6 +1150,8 @@ struct bnxt {
 	#define BNXT_FLAG_DOUBLE_DB	0x400000
 	#define BNXT_FLAG_FW_DCBX_AGENT	0x800000
 	#define BNXT_FLAG_CHIP_NITRO_A0	0x1000000
+	#define BNXT_FLAG_DIM		0x2000000
+	#define BNXT_FLAG_NEW_RM	0x8000000
 
 	#define BNXT_FLAG_ALL_CONFIG_FEATS (BNXT_FLAG_TPA |		\
 					    BNXT_FLAG_RFS |		\
@@ -1185,7 +1211,6 @@ struct bnxt {
 	int			tx_nr_rings;
 	int			tx_nr_rings_per_tc;
 	int			tx_nr_rings_xdp;
-	int			tx_reserved_rings;
 
 	int			tx_wake_thresh;
 	int			tx_push_thresh;
@@ -1297,6 +1322,7 @@ struct bnxt {
 #define BNXT_LINK_SPEED_CHNG_SP_EVENT	14
 #define BNXT_FLOW_STATS_SP_EVENT	15
 
+	struct bnxt_hw_resc	hw_resc;
 	struct bnxt_pf_info	pf;
 #ifdef CONFIG_BNXT_SRIOV
 	int			nr_vfs;
@@ -1346,6 +1372,7 @@ struct bnxt {
 	enum devlink_eswitch_mode eswitch_mode;
 	struct bnxt_vf_rep	**vf_reps; /* array of vf-rep ptrs */
 	u16			*cfa_code_map; /* cfa_code -> vf_idx map */
+	u8			switch_id[8];
 	struct bnxt_tc_info	*tc_info;
 };
 
@@ -1421,6 +1448,9 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
 		     int tx_xdp);
 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc);
 int bnxt_get_max_rings(struct bnxt *, int *, int *, bool);
-void bnxt_restore_pf_fw_resources(struct bnxt *bp);
+int bnxt_restore_pf_fw_resources(struct bnxt *bp);
 int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr);
+void bnxt_dim_work(struct work_struct *work);
+int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi);
+
 #endif
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
index fed37cd9ae1d..3c746f2d9ed8 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c
@@ -278,12 +278,11 @@ static int bnxt_hwrm_set_dcbx_app(struct bnxt *bp, struct dcb_app *app,
 
 	n = IEEE_8021QAZ_MAX_TCS;
 	data_len = sizeof(*data) + sizeof(*fw_app) * n;
-	data = dma_alloc_coherent(&bp->pdev->dev, data_len, &mapping,
-				  GFP_KERNEL);
+	data = dma_zalloc_coherent(&bp->pdev->dev, data_len, &mapping,
+				   GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
 
-	memset(data, 0, data_len);
 	bnxt_hwrm_cmd_hdr_init(bp, &get, HWRM_FW_GET_STRUCTURED_DATA, -1, -1);
 	get.dest_data_addr = cpu_to_le64(mapping);
 	get.structure_id = cpu_to_le16(STRUCT_HDR_STRUCT_ID_DCBX_APP);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c
new file mode 100644
index 000000000000..408dd190331e
--- /dev/null
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c
@@ -0,0 +1,32 @@
+/* Broadcom NetXtreme-C/E network driver.
+ *
+ * Copyright (c) 2017-2018 Broadcom Limited
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/net_dim.h>
+#include "bnxt_hsi.h"
+#include "bnxt.h"
+
+void bnxt_dim_work(struct work_struct *work)
+{
+	struct net_dim *dim = container_of(work, struct net_dim,
+					   work);
+	struct bnxt_cp_ring_info *cpr = container_of(dim,
+						     struct bnxt_cp_ring_info,
+						     dim);
+	struct bnxt_napi *bnapi = container_of(cpr,
+					       struct bnxt_napi,
+					       cp_ring);
+	struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode,
+								  dim->profile_ix);
+
+	cpr->rx_ring_coal.coal_ticks = cur_profile.usec;
+	cpr->rx_ring_coal.coal_bufs = cur_profile.pkts;
+
+	bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi);
+	dim->state = NET_DIM_START_MEASURE;
+}
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index b13ce5ebde8d..1801582076be 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -49,6 +49,8 @@ static int bnxt_get_coalesce(struct net_device *dev,
 
 	memset(coal, 0, sizeof(*coal));
 
+	coal->use_adaptive_rx_coalesce = bp->flags & BNXT_FLAG_DIM;
+
 	hw_coal = &bp->rx_coal;
 	mult = hw_coal->bufs_per_record;
 	coal->rx_coalesce_usecs = hw_coal->coal_ticks;
@@ -77,6 +79,15 @@ static int bnxt_set_coalesce(struct net_device *dev,
 	int rc = 0;
 	u16 mult;
 
+	if (coal->use_adaptive_rx_coalesce) {
+		bp->flags |= BNXT_FLAG_DIM;
+	} else {
+		if (bp->flags & BNXT_FLAG_DIM) {
+			bp->flags &= ~(BNXT_FLAG_DIM);
+			goto reset_coalesce;
+		}
+	}
+
 	hw_coal = &bp->rx_coal;
 	mult = hw_coal->bufs_per_record;
 	hw_coal->coal_ticks = coal->rx_coalesce_usecs;
@@ -104,6 +115,7 @@ static int bnxt_set_coalesce(struct net_device *dev,
 		update_stats = true;
 	}
 
+reset_coalesce:
 	if (netif_running(dev)) {
 		if (update_stats) {
 			rc = bnxt_close_nic(bp, true, false);
@@ -1376,6 +1388,9 @@ static int bnxt_firmware_reset(struct net_device *dev,
 		req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
 		req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
 		break;
+	case BNXT_FW_RESET_AP:
+		req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_AP;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -2522,6 +2537,14 @@ static int bnxt_reset(struct net_device *dev, u32 *flags)
 		rc = bnxt_firmware_reset(dev, BNXT_FW_RESET_CHIP);
 		if (!rc)
 			netdev_info(dev, "Reset request successful. Reload driver to complete reset\n");
+	} else if (*flags == ETH_RESET_AP) {
+		/* This feature is not supported in older firmware versions */
+		if (bp->hwrm_spec_code < 0x10803)
+			return -EOPNOTSUPP;
+
+		rc = bnxt_firmware_reset(dev, BNXT_FW_RESET_AP);
+		if (!rc)
+			netdev_info(dev, "Reset Application Processor request successful.\n");
 	} else {
 		rc = -EINVAL;
 	}
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h
index ff601b42fcc8..836ef682f24c 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.h
@@ -34,6 +34,7 @@ struct bnxt_led_cfg {
 #define BNXT_LED_DFLT_ENABLES(x)			\
 	cpu_to_le32(BNXT_LED_DFLT_ENA << (BNXT_LED_DFLT_ENA_SHIFT * (x)))
 
+#define BNXT_FW_RESET_AP	0xfffe
 #define BNXT_FW_RESET_CHIP	0xffff
 
 extern const struct ethtool_ops bnxt_ethtool_ops;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
index c99f4d0880e4..82d17f8cc0db 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hsi.h
@@ -1,2437 +1,2700 @@
 /* Broadcom NetXtreme-C/E network driver.
  *
  * Copyright (c) 2014-2016 Broadcom Corporation
- * Copyright (c) 2016-2017 Broadcom Limited
+ * Copyright (c) 2016-2018 Broadcom Limited
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation.
+ *
+ * DO NOT MODIFY!!! This file is automatically generated.
  */
 
-#ifndef BNXT_HSI_H
-#define BNXT_HSI_H
+#ifndef _BNXT_HSI_H_
+#define _BNXT_HSI_H_
+
+/* hwrm_cmd_hdr (size:128b/16B) */
+struct hwrm_cmd_hdr {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+};
+
+/* hwrm_resp_hdr (size:64b/8B) */
+struct hwrm_resp_hdr {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+};
+
+#define CMD_DISCR_TLV_ENCAP 0x8000UL
+#define CMD_DISCR_LAST     CMD_DISCR_TLV_ENCAP
+
+
+#define TLV_TYPE_HWRM_REQUEST                    0x1UL
+#define TLV_TYPE_HWRM_RESPONSE                   0x2UL
+#define TLV_TYPE_ROCE_SP_COMMAND                 0x3UL
+#define TLV_TYPE_ENGINE_CKV_DEVICE_SERIAL_NUMBER 0x8001UL
+#define TLV_TYPE_ENGINE_CKV_NONCE                0x8002UL
+#define TLV_TYPE_ENGINE_CKV_IV                   0x8003UL
+#define TLV_TYPE_ENGINE_CKV_AUTH_TAG             0x8004UL
+#define TLV_TYPE_ENGINE_CKV_CIPHERTEXT           0x8005UL
+#define TLV_TYPE_ENGINE_CKV_ALGORITHMS           0x8006UL
+#define TLV_TYPE_ENGINE_CKV_ECC_PUBLIC_KEY       0x8007UL
+#define TLV_TYPE_ENGINE_CKV_ECDSA_SIGNATURE      0x8008UL
+#define TLV_TYPE_LAST                           TLV_TYPE_ENGINE_CKV_ECDSA_SIGNATURE
+
+
+/* tlv (size:64b/8B) */
+struct tlv {
+	__le16	cmd_discr;
+	u8	reserved_8b;
+	u8	flags;
+	#define TLV_FLAGS_MORE         0x1UL
+	#define TLV_FLAGS_MORE_LAST      0x0UL
+	#define TLV_FLAGS_MORE_NOT_LAST  0x1UL
+	#define TLV_FLAGS_REQUIRED     0x2UL
+	#define TLV_FLAGS_REQUIRED_NO    (0x0UL << 1)
+	#define TLV_FLAGS_REQUIRED_YES   (0x1UL << 1)
+	#define TLV_FLAGS_REQUIRED_LAST TLV_FLAGS_REQUIRED_YES
+	__le16	tlv_type;
+	__le16	length;
+};
+
+/* input (size:128b/16B) */
+struct input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+};
 
-/* HSI and HWRM Specification 1.8.3 */
-#define HWRM_VERSION_MAJOR	1
-#define HWRM_VERSION_MINOR	8
-#define HWRM_VERSION_UPDATE	3
+/* output (size:64b/8B) */
+struct output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+};
 
-#define HWRM_VERSION_RSVD	1 /* non-zero means beta version */
+/* hwrm_short_input (size:128b/16B) */
+struct hwrm_short_input {
+	__le16	req_type;
+	__le16	signature;
+	#define SHORT_REQ_SIGNATURE_SHORT_CMD 0x4321UL
+	#define SHORT_REQ_SIGNATURE_LAST     SHORT_REQ_SIGNATURE_SHORT_CMD
+	__le16	unused_0;
+	__le16	size;
+	__le64	req_addr;
+};
 
-#define HWRM_VERSION_STR	"1.8.3.1"
-/*
- * Following is the signature for HWRM message field that indicates not
- * applicable (All F's). Need to cast it the size of the field if needed.
- */
-#define HWRM_NA_SIGNATURE	((__le32)(-1))
-#define HWRM_MAX_REQ_LEN    (128)  /* hwrm_func_buf_rgtr */
-#define HWRM_MAX_RESP_LEN    (280)  /* hwrm_selftest_qlist */
-#define HW_HASH_INDEX_SIZE      0x80    /* 7 bit indirection table index. */
-#define HW_HASH_KEY_SIZE	40
-#define HWRM_RESP_VALID_KEY      1 /* valid key for HWRM response */
-
-/* Statistics Ejection Buffer Completion Record (16 bytes) */
+/* cmd_nums (size:64b/8B) */
+struct cmd_nums {
+	__le16	req_type;
+	#define HWRM_VER_GET                              0x0UL
+	#define HWRM_FUNC_BUF_UNRGTR                      0xeUL
+	#define HWRM_FUNC_VF_CFG                          0xfUL
+	#define HWRM_RESERVED1                            0x10UL
+	#define HWRM_FUNC_RESET                           0x11UL
+	#define HWRM_FUNC_GETFID                          0x12UL
+	#define HWRM_FUNC_VF_ALLOC                        0x13UL
+	#define HWRM_FUNC_VF_FREE                         0x14UL
+	#define HWRM_FUNC_QCAPS                           0x15UL
+	#define HWRM_FUNC_QCFG                            0x16UL
+	#define HWRM_FUNC_CFG                             0x17UL
+	#define HWRM_FUNC_QSTATS                          0x18UL
+	#define HWRM_FUNC_CLR_STATS                       0x19UL
+	#define HWRM_FUNC_DRV_UNRGTR                      0x1aUL
+	#define HWRM_FUNC_VF_RESC_FREE                    0x1bUL
+	#define HWRM_FUNC_VF_VNIC_IDS_QUERY               0x1cUL
+	#define HWRM_FUNC_DRV_RGTR                        0x1dUL
+	#define HWRM_FUNC_DRV_QVER                        0x1eUL
+	#define HWRM_FUNC_BUF_RGTR                        0x1fUL
+	#define HWRM_PORT_PHY_CFG                         0x20UL
+	#define HWRM_PORT_MAC_CFG                         0x21UL
+	#define HWRM_PORT_TS_QUERY                        0x22UL
+	#define HWRM_PORT_QSTATS                          0x23UL
+	#define HWRM_PORT_LPBK_QSTATS                     0x24UL
+	#define HWRM_PORT_CLR_STATS                       0x25UL
+	#define HWRM_PORT_LPBK_CLR_STATS                  0x26UL
+	#define HWRM_PORT_PHY_QCFG                        0x27UL
+	#define HWRM_PORT_MAC_QCFG                        0x28UL
+	#define HWRM_PORT_MAC_PTP_QCFG                    0x29UL
+	#define HWRM_PORT_PHY_QCAPS                       0x2aUL
+	#define HWRM_PORT_PHY_I2C_WRITE                   0x2bUL
+	#define HWRM_PORT_PHY_I2C_READ                    0x2cUL
+	#define HWRM_PORT_LED_CFG                         0x2dUL
+	#define HWRM_PORT_LED_QCFG                        0x2eUL
+	#define HWRM_PORT_LED_QCAPS                       0x2fUL
+	#define HWRM_QUEUE_QPORTCFG                       0x30UL
+	#define HWRM_QUEUE_QCFG                           0x31UL
+	#define HWRM_QUEUE_CFG                            0x32UL
+	#define HWRM_FUNC_VLAN_CFG                        0x33UL
+	#define HWRM_FUNC_VLAN_QCFG                       0x34UL
+	#define HWRM_QUEUE_PFCENABLE_QCFG                 0x35UL
+	#define HWRM_QUEUE_PFCENABLE_CFG                  0x36UL
+	#define HWRM_QUEUE_PRI2COS_QCFG                   0x37UL
+	#define HWRM_QUEUE_PRI2COS_CFG                    0x38UL
+	#define HWRM_QUEUE_COS2BW_QCFG                    0x39UL
+	#define HWRM_QUEUE_COS2BW_CFG                     0x3aUL
+	#define HWRM_QUEUE_DSCP_QCAPS                     0x3bUL
+	#define HWRM_QUEUE_DSCP2PRI_QCFG                  0x3cUL
+	#define HWRM_QUEUE_DSCP2PRI_CFG                   0x3dUL
+	#define HWRM_VNIC_ALLOC                           0x40UL
+	#define HWRM_VNIC_FREE                            0x41UL
+	#define HWRM_VNIC_CFG                             0x42UL
+	#define HWRM_VNIC_QCFG                            0x43UL
+	#define HWRM_VNIC_TPA_CFG                         0x44UL
+	#define HWRM_VNIC_TPA_QCFG                        0x45UL
+	#define HWRM_VNIC_RSS_CFG                         0x46UL
+	#define HWRM_VNIC_RSS_QCFG                        0x47UL
+	#define HWRM_VNIC_PLCMODES_CFG                    0x48UL
+	#define HWRM_VNIC_PLCMODES_QCFG                   0x49UL
+	#define HWRM_VNIC_QCAPS                           0x4aUL
+	#define HWRM_RING_ALLOC                           0x50UL
+	#define HWRM_RING_FREE                            0x51UL
+	#define HWRM_RING_CMPL_RING_QAGGINT_PARAMS        0x52UL
+	#define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS     0x53UL
+	#define HWRM_RING_RESET                           0x5eUL
+	#define HWRM_RING_GRP_ALLOC                       0x60UL
+	#define HWRM_RING_GRP_FREE                        0x61UL
+	#define HWRM_RESERVED5                            0x64UL
+	#define HWRM_RESERVED6                            0x65UL
+	#define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC            0x70UL
+	#define HWRM_VNIC_RSS_COS_LB_CTX_FREE             0x71UL
+	#define HWRM_CFA_L2_FILTER_ALLOC                  0x90UL
+	#define HWRM_CFA_L2_FILTER_FREE                   0x91UL
+	#define HWRM_CFA_L2_FILTER_CFG                    0x92UL
+	#define HWRM_CFA_L2_SET_RX_MASK                   0x93UL
+	#define HWRM_CFA_VLAN_ANTISPOOF_CFG               0x94UL
+	#define HWRM_CFA_TUNNEL_FILTER_ALLOC              0x95UL
+	#define HWRM_CFA_TUNNEL_FILTER_FREE               0x96UL
+	#define HWRM_CFA_ENCAP_RECORD_ALLOC               0x97UL
+	#define HWRM_CFA_ENCAP_RECORD_FREE                0x98UL
+	#define HWRM_CFA_NTUPLE_FILTER_ALLOC              0x99UL
+	#define HWRM_CFA_NTUPLE_FILTER_FREE               0x9aUL
+	#define HWRM_CFA_NTUPLE_FILTER_CFG                0x9bUL
+	#define HWRM_CFA_EM_FLOW_ALLOC                    0x9cUL
+	#define HWRM_CFA_EM_FLOW_FREE                     0x9dUL
+	#define HWRM_CFA_EM_FLOW_CFG                      0x9eUL
+	#define HWRM_TUNNEL_DST_PORT_QUERY                0xa0UL
+	#define HWRM_TUNNEL_DST_PORT_ALLOC                0xa1UL
+	#define HWRM_TUNNEL_DST_PORT_FREE                 0xa2UL
+	#define HWRM_STAT_CTX_ALLOC                       0xb0UL
+	#define HWRM_STAT_CTX_FREE                        0xb1UL
+	#define HWRM_STAT_CTX_QUERY                       0xb2UL
+	#define HWRM_STAT_CTX_CLR_STATS                   0xb3UL
+	#define HWRM_FW_RESET                             0xc0UL
+	#define HWRM_FW_QSTATUS                           0xc1UL
+	#define HWRM_FW_SET_TIME                          0xc8UL
+	#define HWRM_FW_GET_TIME                          0xc9UL
+	#define HWRM_FW_SET_STRUCTURED_DATA               0xcaUL
+	#define HWRM_FW_GET_STRUCTURED_DATA               0xcbUL
+	#define HWRM_FW_IPC_MAILBOX                       0xccUL
+	#define HWRM_EXEC_FWD_RESP                        0xd0UL
+	#define HWRM_REJECT_FWD_RESP                      0xd1UL
+	#define HWRM_FWD_RESP                             0xd2UL
+	#define HWRM_FWD_ASYNC_EVENT_CMPL                 0xd3UL
+	#define HWRM_TEMP_MONITOR_QUERY                   0xe0UL
+	#define HWRM_WOL_FILTER_ALLOC                     0xf0UL
+	#define HWRM_WOL_FILTER_FREE                      0xf1UL
+	#define HWRM_WOL_FILTER_QCFG                      0xf2UL
+	#define HWRM_WOL_REASON_QCFG                      0xf3UL
+	#define HWRM_CFA_METER_PROFILE_ALLOC              0xf5UL
+	#define HWRM_CFA_METER_PROFILE_FREE               0xf6UL
+	#define HWRM_CFA_METER_PROFILE_CFG                0xf7UL
+	#define HWRM_CFA_METER_INSTANCE_ALLOC             0xf8UL
+	#define HWRM_CFA_METER_INSTANCE_FREE              0xf9UL
+	#define HWRM_CFA_VFR_ALLOC                        0xfdUL
+	#define HWRM_CFA_VFR_FREE                         0xfeUL
+	#define HWRM_CFA_VF_PAIR_ALLOC                    0x100UL
+	#define HWRM_CFA_VF_PAIR_FREE                     0x101UL
+	#define HWRM_CFA_VF_PAIR_INFO                     0x102UL
+	#define HWRM_CFA_FLOW_ALLOC                       0x103UL
+	#define HWRM_CFA_FLOW_FREE                        0x104UL
+	#define HWRM_CFA_FLOW_FLUSH                       0x105UL
+	#define HWRM_CFA_FLOW_STATS                       0x106UL
+	#define HWRM_CFA_FLOW_INFO                        0x107UL
+	#define HWRM_CFA_DECAP_FILTER_ALLOC               0x108UL
+	#define HWRM_CFA_DECAP_FILTER_FREE                0x109UL
+	#define HWRM_CFA_VLAN_ANTISPOOF_QCFG              0x10aUL
+	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_ALLOC       0x10bUL
+	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_FREE        0x10cUL
+	#define HWRM_CFA_PAIR_ALLOC                       0x10dUL
+	#define HWRM_CFA_PAIR_FREE                        0x10eUL
+	#define HWRM_CFA_PAIR_INFO                        0x10fUL
+	#define HWRM_FW_IPC_MSG                           0x110UL
+	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_INFO        0x111UL
+	#define HWRM_ENGINE_CKV_HELLO                     0x12dUL
+	#define HWRM_ENGINE_CKV_STATUS                    0x12eUL
+	#define HWRM_ENGINE_CKV_CKEK_ADD                  0x12fUL
+	#define HWRM_ENGINE_CKV_CKEK_DELETE               0x130UL
+	#define HWRM_ENGINE_CKV_KEY_ADD                   0x131UL
+	#define HWRM_ENGINE_CKV_KEY_DELETE                0x132UL
+	#define HWRM_ENGINE_CKV_FLUSH                     0x133UL
+	#define HWRM_ENGINE_CKV_RNG_GET                   0x134UL
+	#define HWRM_ENGINE_CKV_KEY_GEN                   0x135UL
+	#define HWRM_ENGINE_QG_CONFIG_QUERY               0x13cUL
+	#define HWRM_ENGINE_QG_QUERY                      0x13dUL
+	#define HWRM_ENGINE_QG_METER_PROFILE_CONFIG_QUERY 0x13eUL
+	#define HWRM_ENGINE_QG_METER_PROFILE_QUERY        0x13fUL
+	#define HWRM_ENGINE_QG_METER_PROFILE_ALLOC        0x140UL
+	#define HWRM_ENGINE_QG_METER_PROFILE_FREE         0x141UL
+	#define HWRM_ENGINE_QG_METER_QUERY                0x142UL
+	#define HWRM_ENGINE_QG_METER_BIND                 0x143UL
+	#define HWRM_ENGINE_QG_METER_UNBIND               0x144UL
+	#define HWRM_ENGINE_QG_FUNC_BIND                  0x145UL
+	#define HWRM_ENGINE_SG_CONFIG_QUERY               0x146UL
+	#define HWRM_ENGINE_SG_QUERY                      0x147UL
+	#define HWRM_ENGINE_SG_METER_QUERY                0x148UL
+	#define HWRM_ENGINE_SG_METER_CONFIG               0x149UL
+	#define HWRM_ENGINE_SG_QG_BIND                    0x14aUL
+	#define HWRM_ENGINE_QG_SG_UNBIND                  0x14bUL
+	#define HWRM_ENGINE_CONFIG_QUERY                  0x154UL
+	#define HWRM_ENGINE_STATS_CONFIG                  0x155UL
+	#define HWRM_ENGINE_STATS_CLEAR                   0x156UL
+	#define HWRM_ENGINE_STATS_QUERY                   0x157UL
+	#define HWRM_ENGINE_RQ_ALLOC                      0x15eUL
+	#define HWRM_ENGINE_RQ_FREE                       0x15fUL
+	#define HWRM_ENGINE_CQ_ALLOC                      0x160UL
+	#define HWRM_ENGINE_CQ_FREE                       0x161UL
+	#define HWRM_ENGINE_NQ_ALLOC                      0x162UL
+	#define HWRM_ENGINE_NQ_FREE                       0x163UL
+	#define HWRM_ENGINE_ON_DIE_RQE_CREDITS            0x164UL
+	#define HWRM_FUNC_RESOURCE_QCAPS                  0x190UL
+	#define HWRM_FUNC_VF_RESOURCE_CFG                 0x191UL
+	#define HWRM_SELFTEST_QLIST                       0x200UL
+	#define HWRM_SELFTEST_EXEC                        0x201UL
+	#define HWRM_SELFTEST_IRQ                         0x202UL
+	#define HWRM_SELFTEST_RETRIEVE_SERDES_DATA        0x203UL
+	#define HWRM_DBG_READ_DIRECT                      0xff10UL
+	#define HWRM_DBG_READ_INDIRECT                    0xff11UL
+	#define HWRM_DBG_WRITE_DIRECT                     0xff12UL
+	#define HWRM_DBG_WRITE_INDIRECT                   0xff13UL
+	#define HWRM_DBG_DUMP                             0xff14UL
+	#define HWRM_DBG_ERASE_NVM                        0xff15UL
+	#define HWRM_DBG_CFG                              0xff16UL
+	#define HWRM_DBG_COREDUMP_LIST                    0xff17UL
+	#define HWRM_DBG_COREDUMP_INITIATE                0xff18UL
+	#define HWRM_DBG_COREDUMP_RETRIEVE                0xff19UL
+	#define HWRM_NVM_FACTORY_DEFAULTS                 0xffeeUL
+	#define HWRM_NVM_VALIDATE_OPTION                  0xffefUL
+	#define HWRM_NVM_FLUSH                            0xfff0UL
+	#define HWRM_NVM_GET_VARIABLE                     0xfff1UL
+	#define HWRM_NVM_SET_VARIABLE                     0xfff2UL
+	#define HWRM_NVM_INSTALL_UPDATE                   0xfff3UL
+	#define HWRM_NVM_MODIFY                           0xfff4UL
+	#define HWRM_NVM_VERIFY_UPDATE                    0xfff5UL
+	#define HWRM_NVM_GET_DEV_INFO                     0xfff6UL
+	#define HWRM_NVM_ERASE_DIR_ENTRY                  0xfff7UL
+	#define HWRM_NVM_MOD_DIR_ENTRY                    0xfff8UL
+	#define HWRM_NVM_FIND_DIR_ENTRY                   0xfff9UL
+	#define HWRM_NVM_GET_DIR_ENTRIES                  0xfffaUL
+	#define HWRM_NVM_GET_DIR_INFO                     0xfffbUL
+	#define HWRM_NVM_RAW_DUMP                         0xfffcUL
+	#define HWRM_NVM_READ                             0xfffdUL
+	#define HWRM_NVM_WRITE                            0xfffeUL
+	#define HWRM_NVM_RAW_WRITE_BLK                    0xffffUL
+	#define HWRM_LAST                                HWRM_NVM_RAW_WRITE_BLK
+	__le16	unused_0[3];
+};
+
+/* ret_codes (size:64b/8B) */
+struct ret_codes {
+	__le16	error_code;
+	#define HWRM_ERR_CODE_SUCCESS                0x0UL
+	#define HWRM_ERR_CODE_FAIL                   0x1UL
+	#define HWRM_ERR_CODE_INVALID_PARAMS         0x2UL
+	#define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED 0x3UL
+	#define HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR   0x4UL
+	#define HWRM_ERR_CODE_INVALID_FLAGS          0x5UL
+	#define HWRM_ERR_CODE_INVALID_ENABLES        0x6UL
+	#define HWRM_ERR_CODE_UNSUPPORTED_TLV        0x7UL
+	#define HWRM_ERR_CODE_NO_BUFFER              0x8UL
+	#define HWRM_ERR_CODE_HWRM_ERROR             0xfUL
+	#define HWRM_ERR_CODE_UNKNOWN_ERR            0xfffeUL
+	#define HWRM_ERR_CODE_CMD_NOT_SUPPORTED      0xffffUL
+	#define HWRM_ERR_CODE_LAST                  HWRM_ERR_CODE_CMD_NOT_SUPPORTED
+	__le16	unused_0[3];
+};
+
+/* hwrm_err_output (size:128b/16B) */
+struct hwrm_err_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	opaque_0;
+	__le16	opaque_1;
+	u8	cmd_err;
+	u8	valid;
+};
+#define HWRM_NA_SIGNATURE ((__le32)(-1))
+#define HWRM_MAX_REQ_LEN 128
+#define HWRM_MAX_RESP_LEN 280
+#define HW_HASH_INDEX_SIZE 0x80
+#define HW_HASH_KEY_SIZE 40
+#define HWRM_RESP_VALID_KEY 1
+#define HWRM_VERSION_MAJOR 1
+#define HWRM_VERSION_MINOR 9
+#define HWRM_VERSION_UPDATE 0
+#define HWRM_VERSION_RSVD 0
+#define HWRM_VERSION_STR "1.9.0.0"
+
+/* hwrm_ver_get_input (size:192b/24B) */
+struct hwrm_ver_get_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	hwrm_intf_maj;
+	u8	hwrm_intf_min;
+	u8	hwrm_intf_upd;
+	u8	unused_0[5];
+};
+
+/* hwrm_ver_get_output (size:1408b/176B) */
+struct hwrm_ver_get_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	hwrm_intf_maj_8b;
+	u8	hwrm_intf_min_8b;
+	u8	hwrm_intf_upd_8b;
+	u8	hwrm_intf_rsvd_8b;
+	u8	hwrm_fw_maj_8b;
+	u8	hwrm_fw_min_8b;
+	u8	hwrm_fw_bld_8b;
+	u8	hwrm_fw_rsvd_8b;
+	u8	mgmt_fw_maj_8b;
+	u8	mgmt_fw_min_8b;
+	u8	mgmt_fw_bld_8b;
+	u8	mgmt_fw_rsvd_8b;
+	u8	netctrl_fw_maj_8b;
+	u8	netctrl_fw_min_8b;
+	u8	netctrl_fw_bld_8b;
+	u8	netctrl_fw_rsvd_8b;
+	__le32	dev_caps_cfg;
+	#define VER_GET_RESP_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED     0x1UL
+	#define VER_GET_RESP_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED     0x2UL
+	#define VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED         0x4UL
+	#define VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED          0x8UL
+	u8	roce_fw_maj_8b;
+	u8	roce_fw_min_8b;
+	u8	roce_fw_bld_8b;
+	u8	roce_fw_rsvd_8b;
+	char	hwrm_fw_name[16];
+	char	mgmt_fw_name[16];
+	char	netctrl_fw_name[16];
+	u8	reserved2[16];
+	char	roce_fw_name[16];
+	__le16	chip_num;
+	u8	chip_rev;
+	u8	chip_metal;
+	u8	chip_bond_id;
+	u8	chip_platform_type;
+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_ASIC      0x0UL
+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_FPGA      0x1UL
+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_PALLADIUM 0x2UL
+	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_LAST     VER_GET_RESP_CHIP_PLATFORM_TYPE_PALLADIUM
+	__le16	max_req_win_len;
+	__le16	max_resp_len;
+	__le16	def_req_timeout;
+	u8	flags;
+	#define VER_GET_RESP_FLAGS_DEV_NOT_RDY       0x1UL
+	#define VER_GET_RESP_FLAGS_EXT_VER_AVAIL     0x2UL
+	u8	unused_0[2];
+	u8	always_1;
+	__le16	hwrm_intf_major;
+	__le16	hwrm_intf_minor;
+	__le16	hwrm_intf_build;
+	__le16	hwrm_intf_patch;
+	__le16	hwrm_fw_major;
+	__le16	hwrm_fw_minor;
+	__le16	hwrm_fw_build;
+	__le16	hwrm_fw_patch;
+	__le16	mgmt_fw_major;
+	__le16	mgmt_fw_minor;
+	__le16	mgmt_fw_build;
+	__le16	mgmt_fw_patch;
+	__le16	netctrl_fw_major;
+	__le16	netctrl_fw_minor;
+	__le16	netctrl_fw_build;
+	__le16	netctrl_fw_patch;
+	__le16	roce_fw_major;
+	__le16	roce_fw_minor;
+	__le16	roce_fw_build;
+	__le16	roce_fw_patch;
+	__le16	max_ext_req_len;
+	u8	unused_1[5];
+	u8	valid;
+};
+
+/* eject_cmpl (size:128b/16B) */
 struct eject_cmpl {
-	__le16 type;
-	#define EJECT_CMPL_TYPE_MASK				    0x3fUL
-	#define EJECT_CMPL_TYPE_SFT				    0
-	#define EJECT_CMPL_TYPE_STAT_EJECT			   0x1aUL
-	__le16 len;
-	__le32 opaque;
-	__le32 v;
-	#define EJECT_CMPL_V					    0x1UL
-	__le32 unused_2;
-};
-
-/* HWRM Completion Record (16 bytes) */
+	__le16	type;
+	#define EJECT_CMPL_TYPE_MASK      0x3fUL
+	#define EJECT_CMPL_TYPE_SFT       0
+	#define EJECT_CMPL_TYPE_STAT_EJECT  0x1aUL
+	#define EJECT_CMPL_TYPE_LAST       EJECT_CMPL_TYPE_STAT_EJECT
+	__le16	len;
+	__le32	opaque;
+	__le32	v;
+	#define EJECT_CMPL_V     0x1UL
+	__le32	unused_2;
+};
+
+/* hwrm_cmpl (size:128b/16B) */
 struct hwrm_cmpl {
-	__le16 type;
-	#define CMPL_TYPE_MASK					    0x3fUL
-	#define CMPL_TYPE_SFT					    0
-	#define CMPL_TYPE_HWRM_DONE				   0x20UL
-	__le16 sequence_id;
-	__le32 unused_1;
-	__le32 v;
-	#define CMPL_V						    0x1UL
-	__le32 unused_3;
-};
-
-/* HWRM Forwarded Request (16 bytes) */
+	__le16	type;
+	#define CMPL_TYPE_MASK     0x3fUL
+	#define CMPL_TYPE_SFT      0
+	#define CMPL_TYPE_HWRM_DONE  0x20UL
+	#define CMPL_TYPE_LAST      CMPL_TYPE_HWRM_DONE
+	__le16	sequence_id;
+	__le32	unused_1;
+	__le32	v;
+	#define CMPL_V     0x1UL
+	__le32	unused_3;
+};
+
+/* hwrm_fwd_req_cmpl (size:128b/16B) */
 struct hwrm_fwd_req_cmpl {
-	__le16 req_len_type;
-	#define FWD_REQ_CMPL_TYPE_MASK				    0x3fUL
-	#define FWD_REQ_CMPL_TYPE_SFT				    0
-	#define FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ			   0x22UL
-	#define FWD_REQ_CMPL_REQ_LEN_MASK			    0xffc0UL
-	#define FWD_REQ_CMPL_REQ_LEN_SFT			    6
-	__le16 source_id;
-	__le32 unused_0;
-	__le32 req_buf_addr_v[2];
-	#define FWD_REQ_CMPL_V					    0x1UL
-	#define FWD_REQ_CMPL_REQ_BUF_ADDR_MASK			    0xfffffffeUL
-	#define FWD_REQ_CMPL_REQ_BUF_ADDR_SFT			    1
-};
-
-/* HWRM Forwarded Response (16 bytes) */
+	__le16	req_len_type;
+	#define FWD_REQ_CMPL_TYPE_MASK        0x3fUL
+	#define FWD_REQ_CMPL_TYPE_SFT         0
+	#define FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ  0x22UL
+	#define FWD_REQ_CMPL_TYPE_LAST         FWD_REQ_CMPL_TYPE_HWRM_FWD_REQ
+	#define FWD_REQ_CMPL_REQ_LEN_MASK     0xffc0UL
+	#define FWD_REQ_CMPL_REQ_LEN_SFT      6
+	__le16	source_id;
+	__le32	unused0;
+	__le32	req_buf_addr_v[2];
+	#define FWD_REQ_CMPL_V                0x1UL
+	#define FWD_REQ_CMPL_REQ_BUF_ADDR_MASK 0xfffffffeUL
+	#define FWD_REQ_CMPL_REQ_BUF_ADDR_SFT 1
+};
+
+/* hwrm_fwd_resp_cmpl (size:128b/16B) */
 struct hwrm_fwd_resp_cmpl {
-	__le16 type;
-	#define FWD_RESP_CMPL_TYPE_MASK			    0x3fUL
-	#define FWD_RESP_CMPL_TYPE_SFT				    0
-	#define FWD_RESP_CMPL_TYPE_HWRM_FWD_RESP		   0x24UL
-	__le16 source_id;
-	__le16 resp_len;
-	__le16 unused_1;
-	__le32 resp_buf_addr_v[2];
-	#define FWD_RESP_CMPL_V				    0x1UL
-	#define FWD_RESP_CMPL_RESP_BUF_ADDR_MASK		    0xfffffffeUL
-	#define FWD_RESP_CMPL_RESP_BUF_ADDR_SFT		    1
-};
-
-/* HWRM Asynchronous Event Completion Record (16 bytes) */
+	__le16	type;
+	#define FWD_RESP_CMPL_TYPE_MASK         0x3fUL
+	#define FWD_RESP_CMPL_TYPE_SFT          0
+	#define FWD_RESP_CMPL_TYPE_HWRM_FWD_RESP  0x24UL
+	#define FWD_RESP_CMPL_TYPE_LAST          FWD_RESP_CMPL_TYPE_HWRM_FWD_RESP
+	__le16	source_id;
+	__le16	resp_len;
+	__le16	unused_1;
+	__le32	resp_buf_addr_v[2];
+	#define FWD_RESP_CMPL_V                 0x1UL
+	#define FWD_RESP_CMPL_RESP_BUF_ADDR_MASK 0xfffffffeUL
+	#define FWD_RESP_CMPL_RESP_BUF_ADDR_SFT 1
+};
+
+/* hwrm_async_event_cmpl (size:128b/16B) */
 struct hwrm_async_event_cmpl {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_TYPE_MASK			    0x3fUL
-	#define ASYNC_EVENT_CMPL_TYPE_SFT			    0
-	#define ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT		   0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE      0x0UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE	   0x1UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE       0x2UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE       0x3UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED   0x4UL
+	__le16	type;
+	#define ASYNC_EVENT_CMPL_TYPE_MASK            0x3fUL
+	#define ASYNC_EVENT_CMPL_TYPE_SFT             0
+	#define ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT  0x2eUL
+	#define ASYNC_EVENT_CMPL_TYPE_LAST             ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT
+	__le16	event_id;
+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE         0x0UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE            0x1UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE          0x2UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE          0x3UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED      0x4UL
 	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED 0x5UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE   0x6UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE     0x7UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD	   0x10UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD	   0x11UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_FLR_PROC_CMPLT     0x12UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD	   0x20UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD		   0x21UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR		   0x30UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE      0x31UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE 0x32UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE	   0x33UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_LLFC_PFC_CHANGE	   0x34UL
-	#define ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR		   0xffUL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_V				    0x1UL
-	#define ASYNC_EVENT_CMPL_OPAQUE_MASK			    0xfeUL
-	#define ASYNC_EVENT_CMPL_OPAQUE_SFT			    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-};
-
-/* HWRM Asynchronous Event Completion Record for link status change (16 bytes) */
+	#define ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE      0x6UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE        0x7UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD           0x10UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD             0x11UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_FUNC_FLR_PROC_CMPLT        0x12UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD             0x20UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD               0x21UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR                     0x30UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE         0x31UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE   0x32UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE              0x33UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_LLFC_PFC_CHANGE            0x34UL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR                 0xffUL
+	#define ASYNC_EVENT_CMPL_EVENT_ID_LAST                      ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR
+	__le32	event_data2;
+	u8	opaque_v;
+	#define ASYNC_EVENT_CMPL_V          0x1UL
+	#define ASYNC_EVENT_CMPL_OPAQUE_MASK 0xfeUL
+	#define ASYNC_EVENT_CMPL_OPAQUE_SFT 1
+	u8	timestamp_lo;
+	__le16	timestamp_hi;
+	__le32	event_data1;
+};
+
+/* hwrm_async_event_cmpl_link_status_change (size:128b/16B) */
 struct hwrm_async_event_cmpl_link_status_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_MASK      0x3fUL
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_SFT       0
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
+	__le16	type;
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_MASK            0x3fUL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_SFT             0
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_LAST             ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT
+	__le16	event_id;
 	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LINK_STATUS_CHANGE 0x0UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_OPAQUE_MASK    0xfeUL
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_OPAQUE_SFT     1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE 0x1UL
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_DOWN (0x0UL << 0)
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_UP (0x1UL << 0)
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_LAST    ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_UP
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_MASK 0xeUL
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_SFT 1
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_ID_MASK 0xffff0UL
-	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_ID_SFT 4
-};
-
-/* HWRM Asynchronous Event Completion Record for link MTU change (16 bytes) */
-struct hwrm_async_event_cmpl_link_mtu_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_TYPE_MASK	    0x3fUL
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_TYPE_SFT	    0
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_EVENT_ID_LINK_MTU_CHANGE 0x1UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_OPAQUE_MASK       0xfeUL
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_OPAQUE_SFT	    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_EVENT_DATA1_NEW_MTU_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_LINK_MTU_CHANGE_EVENT_DATA1_NEW_MTU_SFT 0
-};
-
-/* HWRM Asynchronous Event Completion Record for link speed change (16 bytes) */
-struct hwrm_async_event_cmpl_link_speed_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_TYPE_MASK       0x3fUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_TYPE_SFT	    0
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_ID_LINK_SPEED_CHANGE 0x2UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_OPAQUE_MASK     0xfeUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_OPAQUE_SFT      1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_FORCE 0x1UL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_MASK 0xfffeUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_SFT 1
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_100MB (0x1UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_1GB (0xaUL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_2GB (0x14UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_2_5GB (0x19UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_10GB (0x64UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_20GB (0xc8UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_25GB (0xfaUL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_40GB (0x190UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_50GB (0x1f4UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_100GB (0x3e8UL << 1)
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_LAST    ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_NEW_LINK_SPEED_100MBPS_100GB
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_PORT_ID_MASK 0xffff0000UL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CHANGE_EVENT_DATA1_PORT_ID_SFT 16
-};
-
-/* HWRM Asynchronous Event Completion Record for DCB Config change (16 bytes) */
-struct hwrm_async_event_cmpl_dcb_config_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_TYPE_MASK       0x3fUL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_TYPE_SFT	    0
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_ID_DCB_CONFIG_CHANGE 0x3UL
-	__le32 event_data2;
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA2_ETS 0x1UL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA2_PFC 0x2UL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA2_APP 0x4UL
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_OPAQUE_MASK     0xfeUL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_OPAQUE_SFT      1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_PORT_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_PORT_ID_SFT 0
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_MASK 0xff0000UL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_SFT 16
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_NONE (0xffUL << 16)
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_LAST    ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_ROCE_PRIORITY_NONE
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_MASK 0xff000000UL
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_SFT 24
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_NONE (0xffUL << 24)
-	#define ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_LAST    ASYNC_EVENT_CMPL_DCB_CONFIG_CHANGE_EVENT_DATA1_RECOMMEND_L2_PRIORITY_NONE
-};
-
-/* HWRM Asynchronous Event Completion Record for port connection not allowed (16 bytes) */
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LAST              ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_ID_LINK_STATUS_CHANGE
+	__le32	event_data2;
+	u8	opaque_v;
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_V          0x1UL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_OPAQUE_MASK 0xfeUL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_OPAQUE_SFT 1
+	u8	timestamp_lo;
+	__le16	timestamp_hi;
+	__le32	event_data1;
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE     0x1UL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_DOWN  0x0UL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_UP    0x1UL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_LAST ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_LINK_CHANGE_UP
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_MASK       0xeUL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_SFT        1
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_ID_MASK    0xffff0UL
+	#define ASYNC_EVENT_CMPL_LINK_STATUS_CHANGE_EVENT_DATA1_PORT_ID_SFT     4
+};
+
+/* hwrm_async_event_cmpl_port_conn_not_allowed (size:128b/16B) */
 struct hwrm_async_event_cmpl_port_conn_not_allowed {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_MASK   0x3fUL
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_SFT    0
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
+	__le16	type;
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_MASK            0x3fUL
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_SFT             0
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT  0x2eUL
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_LAST             ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT
+	__le16	event_id;
 	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_PORT_CONN_NOT_ALLOWED 0x4UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_V	    0x1UL
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_LAST                 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_ID_PORT_CONN_NOT_ALLOWED
+	__le32	event_data2;
+	u8	opaque_v;
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_V          0x1UL
 	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_OPAQUE_MASK 0xfeUL
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_OPAQUE_SFT  1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_SFT 0
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_MASK 0xff0000UL
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_SFT 16
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_NONE (0x0UL << 16)
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_DISABLETX (0x1UL << 16)
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_WARNINGMSG (0x2UL << 16)
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_PWRDOWN (0x3UL << 16)
-	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_LAST    ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_PWRDOWN
-};
-
-/* HWRM Asynchronous Event Completion Record for link speed config not allowed (16 bytes) */
-struct hwrm_async_event_cmpl_link_speed_cfg_not_allowed {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_TYPE_MASK 0x3fUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_TYPE_SFT 0
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED 0x5UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_V      0x1UL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_OPAQUE_MASK 0xfeUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_OPAQUE_SFT 1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_NOT_ALLOWED_EVENT_DATA1_PORT_ID_SFT 0
-};
-
-/* HWRM Asynchronous Event Completion Record for link speed configuration change (16 bytes) */
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_OPAQUE_SFT 1
+	u8	timestamp_lo;
+	__le16	timestamp_hi;
+	__le32	event_data1;
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK                 0xffffUL
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_SFT                  0
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_MASK      0xff0000UL
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_SFT       16
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_NONE        (0x0UL << 16)
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_DISABLETX   (0x1UL << 16)
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_WARNINGMSG  (0x2UL << 16)
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_PWRDOWN     (0x3UL << 16)
+	#define ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_LAST       ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_ENFORCEMENT_POLICY_PWRDOWN
+};
+
+/* hwrm_async_event_cmpl_link_speed_cfg_change (size:128b/16B) */
 struct hwrm_async_event_cmpl_link_speed_cfg_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_MASK   0x3fUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_SFT    0
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
+	__le16	type;
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_MASK            0x3fUL
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_SFT             0
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_LAST             ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT
+	__le16	event_id;
 	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LINK_SPEED_CFG_CHANGE 0x6UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_V	    0x1UL
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LAST                 ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_ID_LINK_SPEED_CFG_CHANGE
+	__le32	event_data2;
+	u8	opaque_v;
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_V          0x1UL
 	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_OPAQUE_MASK 0xfeUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_OPAQUE_SFT  1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_PORT_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_PORT_ID_SFT 0
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_SUPPORTED_LINK_SPEEDS_CHANGE 0x10000UL
-	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_ILLEGAL_LINK_SPEED_CFG 0x20000UL
-};
-
-/* HWRM Asynchronous Event Completion Record for Function Driver Unload (16 bytes) */
-struct hwrm_async_event_cmpl_func_drvr_unload {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_TYPE_MASK	    0x3fUL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_TYPE_SFT	    0
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_EVENT_ID_FUNC_DRVR_UNLOAD 0x10UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_OPAQUE_MASK      0xfeUL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_OPAQUE_SFT       1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_EVENT_DATA1_FUNC_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_UNLOAD_EVENT_DATA1_FUNC_ID_SFT 0
-};
-
-/* HWRM Asynchronous Event Completion Record for Function Driver load (16 bytes) */
-struct hwrm_async_event_cmpl_func_drvr_load {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_TYPE_MASK	    0x3fUL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_TYPE_SFT	    0
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_EVENT_ID_FUNC_DRVR_LOAD 0x11UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_OPAQUE_MASK	    0xfeUL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_OPAQUE_SFT	    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_EVENT_DATA1_FUNC_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_FUNC_DRVR_LOAD_EVENT_DATA1_FUNC_ID_SFT 0
-};
-
-/* HWRM Asynchronous Event Completion Record to indicate completion of FLR related processing (16 bytes) */
-struct hwrm_async_event_cmpl_func_flr_proc_cmplt {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_TYPE_MASK     0x3fUL
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_TYPE_SFT      0
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_EVENT_ID_FUNC_FLR_PROC_CMPLT 0x12UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_OPAQUE_MASK   0xfeUL
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_OPAQUE_SFT    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_EVENT_DATA1_FUNC_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_FUNC_FLR_PROC_CMPLT_EVENT_DATA1_FUNC_ID_SFT 0
-};
-
-/* HWRM Asynchronous Event Completion Record for PF Driver Unload (16 bytes) */
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_OPAQUE_SFT 1
+	u8	timestamp_lo;
+	__le16	timestamp_hi;
+	__le32	event_data1;
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_PORT_ID_MASK                     0xffffUL
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_PORT_ID_SFT                      0
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_SUPPORTED_LINK_SPEEDS_CHANGE     0x10000UL
+	#define ASYNC_EVENT_CMPL_LINK_SPEED_CFG_CHANGE_EVENT_DATA1_ILLEGAL_LINK_SPEED_CFG           0x20000UL
+};
+
+/* hwrm_async_event_cmpl_pf_drvr_unload (size:128b/16B) */
 struct hwrm_async_event_cmpl_pf_drvr_unload {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_MASK	    0x3fUL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_SFT	    0
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
+	__le16	type;
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_MASK            0x3fUL
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_SFT             0
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT  0x2eUL
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_LAST             ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_TYPE_HWRM_ASYNC_EVENT
+	__le16	event_id;
 	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_ID_PF_DRVR_UNLOAD 0x20UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_OPAQUE_MASK	    0xfeUL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_OPAQUE_SFT	    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_ID_LAST          ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_ID_PF_DRVR_UNLOAD
+	__le32	event_data2;
+	u8	opaque_v;
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_V          0x1UL
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_OPAQUE_MASK 0xfeUL
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_OPAQUE_SFT 1
+	u8	timestamp_lo;
+	__le16	timestamp_hi;
+	__le32	event_data1;
 	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_DATA1_FUNC_ID_MASK 0xffffUL
 	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_DATA1_FUNC_ID_SFT 0
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_DATA1_PORT_MASK 0x70000UL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_DATA1_PORT_SFT 16
-};
-
-/* HWRM Asynchronous Event Completion Record for PF Driver load (16 bytes) */
-struct hwrm_async_event_cmpl_pf_drvr_load {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_TYPE_MASK	    0x3fUL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_TYPE_SFT		    0
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_EVENT_ID_PF_DRVR_LOAD 0x21UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_OPAQUE_MASK	    0xfeUL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_OPAQUE_SFT	    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_EVENT_DATA1_FUNC_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_EVENT_DATA1_FUNC_ID_SFT 0
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_EVENT_DATA1_PORT_MASK 0x70000UL
-	#define ASYNC_EVENT_CMPL_PF_DRVR_LOAD_EVENT_DATA1_PORT_SFT 16
-};
-
-/* HWRM Asynchronous Event Completion Record for VF FLR (16 bytes) */
-struct hwrm_async_event_cmpl_vf_flr {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_VF_FLR_TYPE_MASK		    0x3fUL
-	#define ASYNC_EVENT_CMPL_VF_FLR_TYPE_SFT		    0
-	#define ASYNC_EVENT_CMPL_VF_FLR_TYPE_HWRM_ASYNC_EVENT     0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_VF_FLR_EVENT_ID_VF_FLR	   0x30UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_VF_FLR_V			    0x1UL
-	#define ASYNC_EVENT_CMPL_VF_FLR_OPAQUE_MASK		    0xfeUL
-	#define ASYNC_EVENT_CMPL_VF_FLR_OPAQUE_SFT		    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_VF_FLR_EVENT_DATA1_VF_ID_MASK     0xffffUL
-	#define ASYNC_EVENT_CMPL_VF_FLR_EVENT_DATA1_VF_ID_SFT      0
-};
-
-/* HWRM Asynchronous Event Completion Record for VF MAC Addr change (16 bytes) */
-struct hwrm_async_event_cmpl_vf_mac_addr_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_TYPE_MASK      0x3fUL
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_TYPE_SFT       0
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_EVENT_ID_VF_MAC_ADDR_CHANGE 0x31UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_OPAQUE_MASK    0xfeUL
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_OPAQUE_SFT     1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_EVENT_DATA1_VF_ID_MASK 0xffffUL
-	#define ASYNC_EVENT_CMPL_VF_MAC_ADDR_CHANGE_EVENT_DATA1_VF_ID_SFT 0
-};
-
-/* HWRM Asynchronous Event Completion Record for PF-VF communication status change (16 bytes) */
-struct hwrm_async_event_cmpl_pf_vf_comm_status_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_TYPE_MASK 0x3fUL
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_TYPE_SFT 0
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_EVENT_ID_PF_VF_COMM_STATUS_CHANGE 0x32UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_V	    0x1UL
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_OPAQUE_MASK 0xfeUL
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_OPAQUE_SFT 1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_PF_VF_COMM_STATUS_CHANGE_EVENT_DATA1_COMM_ESTABLISHED 0x1UL
-};
-
-/* HWRM Asynchronous Event Completion Record for VF configuration change (16 bytes) */
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_DATA1_PORT_MASK   0x70000UL
+	#define ASYNC_EVENT_CMPL_PF_DRVR_UNLOAD_EVENT_DATA1_PORT_SFT    16
+};
+
+/* hwrm_async_event_cmpl_vf_cfg_change (size:128b/16B) */
 struct hwrm_async_event_cmpl_vf_cfg_change {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_MASK	    0x3fUL
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_SFT	    0
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
+	__le16	type;
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_MASK            0x3fUL
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_SFT             0
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT  0x2eUL
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_LAST             ASYNC_EVENT_CMPL_VF_CFG_CHANGE_TYPE_HWRM_ASYNC_EVENT
+	__le16	event_id;
 	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE 0x33UL
-	__le32 event_data2;
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_V		    0x1UL
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_OPAQUE_MASK	    0xfeUL
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_OPAQUE_SFT	    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_MTU_CHANGE 0x1UL
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_MRU_CHANGE 0x2UL
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_MAC_ADDR_CHANGE 0x4UL
-	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_VLAN_CHANGE 0x8UL
-};
-
-/* HWRM Asynchronous Event Completion Record for HWRM Error (16 bytes) */
-struct hwrm_async_event_cmpl_hwrm_error {
-	__le16 type;
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_MASK		    0x3fUL
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_SFT		    0
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_TYPE_HWRM_ASYNC_EVENT 0x2eUL
-	__le16 event_id;
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_ID_HWRM_ERROR   0xffUL
-	__le32 event_data2;
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_MASK 0xffUL
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_SFT 0
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_WARNING 0x0UL
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_NONFATAL 0x1UL
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL 0x2UL
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_LAST    ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA2_SEVERITY_FATAL
-	u8 opaque_v;
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_V			    0x1UL
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_OPAQUE_MASK	    0xfeUL
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_OPAQUE_SFT		    1
-	u8 timestamp_lo;
-	__le16 timestamp_hi;
-	__le32 event_data1;
-	#define ASYNC_EVENT_CMPL_HWRM_ERROR_EVENT_DATA1_TIMESTAMP  0x1UL
-};
-
-/* hwrm_ver_get */
-/* Input (24 bytes) */
-struct hwrm_ver_get_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 hwrm_intf_maj;
-	u8 hwrm_intf_min;
-	u8 hwrm_intf_upd;
-	u8 unused_0[5];
-};
-
-/* Output (128 bytes) */
-struct hwrm_ver_get_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 hwrm_intf_maj;
-	u8 hwrm_intf_min;
-	u8 hwrm_intf_upd;
-	u8 hwrm_intf_rsvd;
-	u8 hwrm_fw_maj;
-	u8 hwrm_fw_min;
-	u8 hwrm_fw_bld;
-	u8 hwrm_fw_rsvd;
-	u8 mgmt_fw_maj;
-	u8 mgmt_fw_min;
-	u8 mgmt_fw_bld;
-	u8 mgmt_fw_rsvd;
-	u8 netctrl_fw_maj;
-	u8 netctrl_fw_min;
-	u8 netctrl_fw_bld;
-	u8 netctrl_fw_rsvd;
-	__le32 dev_caps_cfg;
-	#define VER_GET_RESP_DEV_CAPS_CFG_SECURE_FW_UPD_SUPPORTED  0x1UL
-	#define VER_GET_RESP_DEV_CAPS_CFG_FW_DCBX_AGENT_SUPPORTED  0x2UL
-	#define VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED      0x4UL
-	#define VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED       0x8UL
-	u8 roce_fw_maj;
-	u8 roce_fw_min;
-	u8 roce_fw_bld;
-	u8 roce_fw_rsvd;
-	char hwrm_fw_name[16];
-	char mgmt_fw_name[16];
-	char netctrl_fw_name[16];
-	__le32 reserved2[4];
-	char roce_fw_name[16];
-	__le16 chip_num;
-	u8 chip_rev;
-	u8 chip_metal;
-	u8 chip_bond_id;
-	u8 chip_platform_type;
-	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_ASIC		   0x0UL
-	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_FPGA		   0x1UL
-	#define VER_GET_RESP_CHIP_PLATFORM_TYPE_PALLADIUM	   0x2UL
-	__le16 max_req_win_len;
-	__le16 max_resp_len;
-	__le16 def_req_timeout;
-	u8 init_pending;
-	#define VER_GET_RESP_INIT_PENDING_DEV_NOT_RDY		    0x1UL
-	u8 unused_0;
-	u8 unused_1;
-	u8 valid;
-};
-
-/* hwrm_func_reset */
-/* Input (24 bytes) */
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_LAST         ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_ID_VF_CFG_CHANGE
+	__le32	event_data2;
+	u8	opaque_v;
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_V          0x1UL
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_OPAQUE_MASK 0xfeUL
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_OPAQUE_SFT 1
+	u8	timestamp_lo;
+	__le16	timestamp_hi;
+	__le32	event_data1;
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_MTU_CHANGE               0x1UL
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_MRU_CHANGE               0x2UL
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_MAC_ADDR_CHANGE     0x4UL
+	#define ASYNC_EVENT_CMPL_VF_CFG_CHANGE_EVENT_DATA1_DFLT_VLAN_CHANGE         0x8UL
+};
+
+/* hwrm_func_reset_input (size:192b/24B) */
 struct hwrm_func_reset_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define FUNC_RESET_REQ_ENABLES_VF_ID_VALID		    0x1UL
-	__le16 vf_id;
-	u8 func_reset_level;
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETALL	   0x0UL
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETME	   0x1UL
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETCHILDREN     0x2UL
-	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF	   0x3UL
-	u8 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define FUNC_RESET_REQ_ENABLES_VF_ID_VALID     0x1UL
+	__le16	vf_id;
+	u8	func_reset_level;
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETALL      0x0UL
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETME       0x1UL
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETCHILDREN 0x2UL
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF       0x3UL
+	#define FUNC_RESET_REQ_FUNC_RESET_LEVEL_LAST         FUNC_RESET_REQ_FUNC_RESET_LEVEL_RESETVF
+	u8	unused_0;
+};
+
+/* hwrm_func_reset_output (size:128b/16B) */
 struct hwrm_func_reset_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_getfid */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_getfid_input (size:192b/24B) */
 struct hwrm_func_getfid_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define FUNC_GETFID_REQ_ENABLES_PCI_ID			    0x1UL
-	__le16 pci_id;
-	__le16 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define FUNC_GETFID_REQ_ENABLES_PCI_ID     0x1UL
+	__le16	pci_id;
+	u8	unused_0[2];
+};
+
+/* hwrm_func_getfid_output (size:128b/16B) */
 struct hwrm_func_getfid_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 fid;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_func_vf_alloc */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	fid;
+	u8	unused_0[5];
+	u8	valid;
+};
+
+/* hwrm_func_vf_alloc_input (size:192b/24B) */
 struct hwrm_func_vf_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define FUNC_VF_ALLOC_REQ_ENABLES_FIRST_VF_ID		    0x1UL
-	__le16 first_vf_id;
-	__le16 num_vfs;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define FUNC_VF_ALLOC_REQ_ENABLES_FIRST_VF_ID     0x1UL
+	__le16	first_vf_id;
+	__le16	num_vfs;
+};
+
+/* hwrm_func_vf_alloc_output (size:128b/16B) */
 struct hwrm_func_vf_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 first_vf_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_func_vf_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	first_vf_id;
+	u8	unused_0[5];
+	u8	valid;
+};
+
+/* hwrm_func_vf_free_input (size:192b/24B) */
 struct hwrm_func_vf_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define FUNC_VF_FREE_REQ_ENABLES_FIRST_VF_ID		    0x1UL
-	__le16 first_vf_id;
-	__le16 num_vfs;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define FUNC_VF_FREE_REQ_ENABLES_FIRST_VF_ID     0x1UL
+	__le16	first_vf_id;
+	__le16	num_vfs;
+};
+
+/* hwrm_func_vf_free_output (size:128b/16B) */
 struct hwrm_func_vf_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_vf_cfg */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_vf_cfg_input (size:448b/56B) */
 struct hwrm_func_vf_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define FUNC_VF_CFG_REQ_ENABLES_MTU			    0x1UL
-	#define FUNC_VF_CFG_REQ_ENABLES_GUEST_VLAN		    0x2UL
-	#define FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR		    0x4UL
-	#define FUNC_VF_CFG_REQ_ENABLES_DFLT_MAC_ADDR		    0x8UL
-	__le16 mtu;
-	__le16 guest_vlan;
-	__le16 async_event_cr;
-	u8 dflt_mac_addr[6];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define FUNC_VF_CFG_REQ_ENABLES_MTU                  0x1UL
+	#define FUNC_VF_CFG_REQ_ENABLES_GUEST_VLAN           0x2UL
+	#define FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR       0x4UL
+	#define FUNC_VF_CFG_REQ_ENABLES_DFLT_MAC_ADDR        0x8UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS      0x10UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS       0x20UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS         0x40UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS         0x80UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS          0x100UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS            0x200UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS        0x400UL
+	#define FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS     0x800UL
+	__le16	mtu;
+	__le16	guest_vlan;
+	__le16	async_event_cr;
+	u8	dflt_mac_addr[6];
+	__le32	flags;
+	#define FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST             0x1UL
+	#define FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST             0x2UL
+	#define FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST           0x4UL
+	#define FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST     0x8UL
+	#define FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST       0x10UL
+	#define FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST       0x20UL
+	#define FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST           0x40UL
+	#define FUNC_VF_CFG_REQ_FLAGS_L2_CTX_ASSETS_TEST         0x80UL
+	__le16	num_rsscos_ctxs;
+	__le16	num_cmpl_rings;
+	__le16	num_tx_rings;
+	__le16	num_rx_rings;
+	__le16	num_l2_ctxs;
+	__le16	num_vnics;
+	__le16	num_stat_ctxs;
+	__le16	num_hw_ring_grps;
+	u8	unused_0[4];
+};
+
+/* hwrm_func_vf_cfg_output (size:128b/16B) */
 struct hwrm_func_vf_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_qcaps */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_qcaps_input (size:192b/24B) */
 struct hwrm_func_qcaps_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 fid;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	fid;
+	u8	unused_0[6];
 };
 
-/* Output (80 bytes) */
+/* hwrm_func_qcaps_output (size:640b/80B) */
 struct hwrm_func_qcaps_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 fid;
-	__le16 port_id;
-	__le32 flags;
-	#define FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED	    0x1UL
-	#define FUNC_QCAPS_RESP_FLAGS_GLOBAL_MSIX_AUTOMASKING      0x2UL
-	#define FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED		    0x4UL
-	#define FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED	    0x8UL
-	#define FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED	    0x10UL
-	#define FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED       0x20UL
-	#define FUNC_QCAPS_RESP_FLAGS_WOL_BMP_SUPPORTED	    0x40UL
-	#define FUNC_QCAPS_RESP_FLAGS_TX_RING_RL_SUPPORTED	    0x80UL
-	#define FUNC_QCAPS_RESP_FLAGS_TX_BW_CFG_SUPPORTED	    0x100UL
-	#define FUNC_QCAPS_RESP_FLAGS_VF_TX_RING_RL_SUPPORTED      0x200UL
-	#define FUNC_QCAPS_RESP_FLAGS_VF_BW_CFG_SUPPORTED	    0x400UL
-	#define FUNC_QCAPS_RESP_FLAGS_STD_TX_RING_MODE_SUPPORTED   0x800UL
-	u8 mac_address[6];
-	__le16 max_rsscos_ctx;
-	__le16 max_cmpl_rings;
-	__le16 max_tx_rings;
-	__le16 max_rx_rings;
-	__le16 max_l2_ctxs;
-	__le16 max_vnics;
-	__le16 first_vf_id;
-	__le16 max_vfs;
-	__le16 max_stat_ctx;
-	__le32 max_encap_records;
-	__le32 max_decap_records;
-	__le32 max_tx_em_flows;
-	__le32 max_tx_wm_flows;
-	__le32 max_rx_em_flows;
-	__le32 max_rx_wm_flows;
-	__le32 max_mcast_filters;
-	__le32 max_flow_id;
-	__le32 max_hw_ring_grps;
-	__le16 max_sp_tx_rings;
-	u8 unused_0;
-	u8 valid;
-};
-
-/* hwrm_func_qcfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	fid;
+	__le16	port_id;
+	__le32	flags;
+	#define FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED            0x1UL
+	#define FUNC_QCAPS_RESP_FLAGS_GLOBAL_MSIX_AUTOMASKING        0x2UL
+	#define FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED                  0x4UL
+	#define FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED              0x8UL
+	#define FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED              0x10UL
+	#define FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED         0x20UL
+	#define FUNC_QCAPS_RESP_FLAGS_WOL_BMP_SUPPORTED              0x40UL
+	#define FUNC_QCAPS_RESP_FLAGS_TX_RING_RL_SUPPORTED           0x80UL
+	#define FUNC_QCAPS_RESP_FLAGS_TX_BW_CFG_SUPPORTED            0x100UL
+	#define FUNC_QCAPS_RESP_FLAGS_VF_TX_RING_RL_SUPPORTED        0x200UL
+	#define FUNC_QCAPS_RESP_FLAGS_VF_BW_CFG_SUPPORTED            0x400UL
+	#define FUNC_QCAPS_RESP_FLAGS_STD_TX_RING_MODE_SUPPORTED     0x800UL
+	#define FUNC_QCAPS_RESP_FLAGS_GENEVE_TUN_FLAGS_SUPPORTED     0x1000UL
+	#define FUNC_QCAPS_RESP_FLAGS_NVGRE_TUN_FLAGS_SUPPORTED      0x2000UL
+	#define FUNC_QCAPS_RESP_FLAGS_GRE_TUN_FLAGS_SUPPORTED        0x4000UL
+	#define FUNC_QCAPS_RESP_FLAGS_MPLS_TUN_FLAGS_SUPPORTED       0x8000UL
+	u8	mac_address[6];
+	__le16	max_rsscos_ctx;
+	__le16	max_cmpl_rings;
+	__le16	max_tx_rings;
+	__le16	max_rx_rings;
+	__le16	max_l2_ctxs;
+	__le16	max_vnics;
+	__le16	first_vf_id;
+	__le16	max_vfs;
+	__le16	max_stat_ctx;
+	__le32	max_encap_records;
+	__le32	max_decap_records;
+	__le32	max_tx_em_flows;
+	__le32	max_tx_wm_flows;
+	__le32	max_rx_em_flows;
+	__le32	max_rx_wm_flows;
+	__le32	max_mcast_filters;
+	__le32	max_flow_id;
+	__le32	max_hw_ring_grps;
+	__le16	max_sp_tx_rings;
+	u8	unused_0;
+	u8	valid;
+};
+
+/* hwrm_func_qcfg_input (size:192b/24B) */
 struct hwrm_func_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 fid;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	fid;
+	u8	unused_0[6];
 };
 
-/* Output (72 bytes) */
+/* hwrm_func_qcfg_output (size:640b/80B) */
 struct hwrm_func_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 fid;
-	__le16 port_id;
-	__le16 vlan;
-	__le16 flags;
-	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_MAGICPKT_ENABLED      0x1UL
-	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_BMP_ENABLED	    0x2UL
-	#define FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED	    0x4UL
-	#define FUNC_QCFG_RESP_FLAGS_STD_TX_RING_MODE_ENABLED      0x8UL
-	#define FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED	    0x10UL
-	#define FUNC_QCFG_RESP_FLAGS_MULTI_HOST		    0x20UL
-	u8 mac_address[6];
-	__le16 pci_id;
-	__le16 alloc_rsscos_ctx;
-	__le16 alloc_cmpl_rings;
-	__le16 alloc_tx_rings;
-	__le16 alloc_rx_rings;
-	__le16 alloc_l2_ctx;
-	__le16 alloc_vnics;
-	__le16 mtu;
-	__le16 mru;
-	__le16 stat_ctx_id;
-	u8 port_partition_type;
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_SPF		   0x0UL
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_MPFS	   0x1UL
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0	   0x2UL
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5	   0x3UL
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0	   0x4UL
-	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN	   0xffUL
-	u8 port_pf_cnt;
-	#define FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL		   0x0UL
-	__le16 dflt_vnic_id;
-	__le16 max_mtu_configured;
-	__le32 min_bw;
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_MASK		    0xfffffffUL
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_SFT		    0
-	#define FUNC_QCFG_RESP_MIN_BW_SCALE			    0x10000000UL
-	#define FUNC_QCFG_RESP_MIN_BW_SCALE_BITS		   (0x0UL << 28)
-	#define FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES		   (0x1UL << 28)
-	#define FUNC_QCFG_RESP_MIN_BW_SCALE_LAST    FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MASK	    0xe0000000UL
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_SFT	    29
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MEGA	   (0x0UL << 29)
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_KILO	   (0x2UL << 29)
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_BASE	   (0x4UL << 29)
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_GIGA	   (0x6UL << 29)
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	fid;
+	__le16	port_id;
+	__le16	vlan;
+	__le16	flags;
+	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_MAGICPKT_ENABLED     0x1UL
+	#define FUNC_QCFG_RESP_FLAGS_OOB_WOL_BMP_ENABLED          0x2UL
+	#define FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED        0x4UL
+	#define FUNC_QCFG_RESP_FLAGS_STD_TX_RING_MODE_ENABLED     0x8UL
+	#define FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED        0x10UL
+	#define FUNC_QCFG_RESP_FLAGS_MULTI_HOST                   0x20UL
+	u8	mac_address[6];
+	__le16	pci_id;
+	__le16	alloc_rsscos_ctx;
+	__le16	alloc_cmpl_rings;
+	__le16	alloc_tx_rings;
+	__le16	alloc_rx_rings;
+	__le16	alloc_l2_ctx;
+	__le16	alloc_vnics;
+	__le16	mtu;
+	__le16	mru;
+	__le16	stat_ctx_id;
+	u8	port_partition_type;
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_SPF     0x0UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_MPFS    0x1UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0 0x2UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5 0x3UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0 0x4UL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN 0xffUL
+	#define FUNC_QCFG_RESP_PORT_PARTITION_TYPE_LAST   FUNC_QCFG_RESP_PORT_PARTITION_TYPE_UNKNOWN
+	u8	port_pf_cnt;
+	#define FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL 0x0UL
+	#define FUNC_QCFG_RESP_PORT_PF_CNT_LAST   FUNC_QCFG_RESP_PORT_PF_CNT_UNAVAIL
+	__le16	dflt_vnic_id;
+	__le16	max_mtu_configured;
+	__le32	min_bw;
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_SFT              0
+	#define FUNC_QCFG_RESP_MIN_BW_SCALE                     0x10000000UL
+	#define FUNC_QCFG_RESP_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define FUNC_QCFG_RESP_MIN_BW_SCALE_LAST                 FUNC_QCFG_RESP_MIN_BW_SCALE_BYTES
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
 	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
 	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_LAST    FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 max_bw;
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_MASK		    0xfffffffUL
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_SFT		    0
-	#define FUNC_QCFG_RESP_MAX_BW_SCALE			    0x10000000UL
-	#define FUNC_QCFG_RESP_MAX_BW_SCALE_BITS		   (0x0UL << 28)
-	#define FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES		   (0x1UL << 28)
-	#define FUNC_QCFG_RESP_MAX_BW_SCALE_LAST    FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MASK	    0xe0000000UL
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_SFT	    29
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MEGA	   (0x0UL << 29)
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_KILO	   (0x2UL << 29)
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_BASE	   (0x4UL << 29)
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_GIGA	   (0x6UL << 29)
+	#define FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_LAST         FUNC_QCFG_RESP_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	max_bw;
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_SFT              0
+	#define FUNC_QCFG_RESP_MAX_BW_SCALE                     0x10000000UL
+	#define FUNC_QCFG_RESP_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define FUNC_QCFG_RESP_MAX_BW_SCALE_LAST                 FUNC_QCFG_RESP_MAX_BW_SCALE_BYTES
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
 	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
 	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_LAST    FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 evb_mode;
-	#define FUNC_QCFG_RESP_EVB_MODE_NO_EVB			   0x0UL
-	#define FUNC_QCFG_RESP_EVB_MODE_VEB			   0x1UL
-	#define FUNC_QCFG_RESP_EVB_MODE_VEPA			   0x2UL
-	u8 unused_0;
-	__le16 alloc_vfs;
-	__le32 alloc_mcast_filters;
-	__le32 alloc_hw_ring_grps;
-	__le16 alloc_sp_tx_rings;
-	u8 unused_1;
-	u8 valid;
-};
-
-/* hwrm_func_vlan_cfg */
-/* Input (48 bytes) */
+	#define FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_LAST         FUNC_QCFG_RESP_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	evb_mode;
+	#define FUNC_QCFG_RESP_EVB_MODE_NO_EVB 0x0UL
+	#define FUNC_QCFG_RESP_EVB_MODE_VEB    0x1UL
+	#define FUNC_QCFG_RESP_EVB_MODE_VEPA   0x2UL
+	#define FUNC_QCFG_RESP_EVB_MODE_LAST  FUNC_QCFG_RESP_EVB_MODE_VEPA
+	u8	cache_linesize;
+	#define FUNC_QCFG_RESP_CACHE_LINESIZE_CACHE_LINESIZE_64  0x0UL
+	#define FUNC_QCFG_RESP_CACHE_LINESIZE_CACHE_LINESIZE_128 0x1UL
+	#define FUNC_QCFG_RESP_CACHE_LINESIZE_LAST              FUNC_QCFG_RESP_CACHE_LINESIZE_CACHE_LINESIZE_128
+	__le16	alloc_vfs;
+	__le32	alloc_mcast_filters;
+	__le32	alloc_hw_ring_grps;
+	__le16	alloc_sp_tx_rings;
+	__le16	alloc_stat_ctx;
+	u8	unused_2[7];
+	u8	valid;
+};
+
+/* hwrm_func_vlan_cfg_input (size:384b/48B) */
 struct hwrm_func_vlan_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 fid;
-	u8 unused_0;
-	u8 unused_1;
-	__le32 enables;
-	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_VID		    0x1UL
-	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_VID		    0x2UL
-	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_PCP		    0x4UL
-	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_PCP		    0x8UL
-	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_TPID		    0x10UL
-	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_TPID		    0x20UL
-	__le16 stag_vid;
-	u8 stag_pcp;
-	u8 unused_2;
-	__be16 stag_tpid;
-	__le16 ctag_vid;
-	u8 ctag_pcp;
-	u8 unused_3;
-	__be16 ctag_tpid;
-	__le32 rsvd1;
-	__le32 rsvd2;
-	__le32 unused_4;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	fid;
+	u8	unused_0[2];
+	__le32	enables;
+	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_VID      0x1UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_VID      0x2UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_PCP      0x4UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_PCP      0x8UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_STAG_TPID     0x10UL
+	#define FUNC_VLAN_CFG_REQ_ENABLES_CTAG_TPID     0x20UL
+	__le16	stag_vid;
+	u8	stag_pcp;
+	u8	unused_1;
+	__be16	stag_tpid;
+	__le16	ctag_vid;
+	u8	ctag_pcp;
+	u8	unused_2;
+	__be16	ctag_tpid;
+	__le32	rsvd1;
+	__le32	rsvd2;
+	u8	unused_3[4];
+};
+
+/* hwrm_func_vlan_cfg_output (size:128b/16B) */
 struct hwrm_func_vlan_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_cfg */
-/* Input (88 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_cfg_input (size:704b/88B) */
 struct hwrm_func_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 fid;
-	u8 unused_0;
-	u8 unused_1;
-	__le32 flags;
-	#define FUNC_CFG_REQ_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE      0x1UL
-	#define FUNC_CFG_REQ_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE       0x2UL
-	#define FUNC_CFG_REQ_FLAGS_RSVD_MASK			    0x1fcUL
-	#define FUNC_CFG_REQ_FLAGS_RSVD_SFT			    2
-	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_ENABLE	    0x200UL
-	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_DISABLE	    0x400UL
-	#define FUNC_CFG_REQ_FLAGS_VIRT_MAC_PERSIST		    0x800UL
-	#define FUNC_CFG_REQ_FLAGS_NO_AUTOCLEAR_STATISTIC	    0x1000UL
-	#define FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST		    0x2000UL
-	__le32 enables;
-	#define FUNC_CFG_REQ_ENABLES_MTU			    0x1UL
-	#define FUNC_CFG_REQ_ENABLES_MRU			    0x2UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS		    0x4UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS		    0x8UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS		    0x10UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS		    0x20UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_L2_CTXS		    0x40UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_VNICS			    0x80UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS		    0x100UL
-	#define FUNC_CFG_REQ_ENABLES_DFLT_MAC_ADDR		    0x200UL
-	#define FUNC_CFG_REQ_ENABLES_DFLT_VLAN			    0x400UL
-	#define FUNC_CFG_REQ_ENABLES_DFLT_IP_ADDR		    0x800UL
-	#define FUNC_CFG_REQ_ENABLES_MIN_BW			    0x1000UL
-	#define FUNC_CFG_REQ_ENABLES_MAX_BW			    0x2000UL
-	#define FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR		    0x4000UL
-	#define FUNC_CFG_REQ_ENABLES_VLAN_ANTISPOOF_MODE	    0x8000UL
-	#define FUNC_CFG_REQ_ENABLES_ALLOWED_VLAN_PRIS		    0x10000UL
-	#define FUNC_CFG_REQ_ENABLES_EVB_MODE			    0x20000UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_MCAST_FILTERS		    0x40000UL
-	#define FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS		    0x80000UL
-	__le16 mtu;
-	__le16 mru;
-	__le16 num_rsscos_ctxs;
-	__le16 num_cmpl_rings;
-	__le16 num_tx_rings;
-	__le16 num_rx_rings;
-	__le16 num_l2_ctxs;
-	__le16 num_vnics;
-	__le16 num_stat_ctxs;
-	__le16 num_hw_ring_grps;
-	u8 dflt_mac_addr[6];
-	__le16 dflt_vlan;
-	__be32 dflt_ip_addr[4];
-	__le32 min_bw;
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_MASK		    0xfffffffUL
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_SFT		    0
-	#define FUNC_CFG_REQ_MIN_BW_SCALE			    0x10000000UL
-	#define FUNC_CFG_REQ_MIN_BW_SCALE_BITS			   (0x0UL << 28)
-	#define FUNC_CFG_REQ_MIN_BW_SCALE_BYTES		   (0x1UL << 28)
-	#define FUNC_CFG_REQ_MIN_BW_SCALE_LAST    FUNC_CFG_REQ_MIN_BW_SCALE_BYTES
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MASK		    0xe0000000UL
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_SFT		    29
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MEGA		   (0x0UL << 29)
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_KILO		   (0x2UL << 29)
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_BASE		   (0x4UL << 29)
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_GIGA		   (0x6UL << 29)
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_PERCENT1_100    (0x1UL << 29)
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID	   (0x7UL << 29)
-	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_LAST    FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 max_bw;
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_MASK		    0xfffffffUL
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_SFT		    0
-	#define FUNC_CFG_REQ_MAX_BW_SCALE			    0x10000000UL
-	#define FUNC_CFG_REQ_MAX_BW_SCALE_BITS			   (0x0UL << 28)
-	#define FUNC_CFG_REQ_MAX_BW_SCALE_BYTES		   (0x1UL << 28)
-	#define FUNC_CFG_REQ_MAX_BW_SCALE_LAST    FUNC_CFG_REQ_MAX_BW_SCALE_BYTES
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MASK		    0xe0000000UL
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_SFT		    29
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MEGA		   (0x0UL << 29)
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_KILO		   (0x2UL << 29)
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_BASE		   (0x4UL << 29)
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_GIGA		   (0x6UL << 29)
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100    (0x1UL << 29)
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID	   (0x7UL << 29)
-	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_LAST    FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
-	__le16 async_event_cr;
-	u8 vlan_antispoof_mode;
-	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_NOCHECK	   0x0UL
-	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN    0x1UL
-	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_IF_VLANDNE 0x2UL
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	fid;
+	u8	unused_0[2];
+	__le32	flags;
+	#define FUNC_CFG_REQ_FLAGS_SRC_MAC_ADDR_CHECK_DISABLE     0x1UL
+	#define FUNC_CFG_REQ_FLAGS_SRC_MAC_ADDR_CHECK_ENABLE      0x2UL
+	#define FUNC_CFG_REQ_FLAGS_RSVD_MASK                      0x1fcUL
+	#define FUNC_CFG_REQ_FLAGS_RSVD_SFT                       2
+	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_ENABLE        0x200UL
+	#define FUNC_CFG_REQ_FLAGS_STD_TX_RING_MODE_DISABLE       0x400UL
+	#define FUNC_CFG_REQ_FLAGS_VIRT_MAC_PERSIST               0x800UL
+	#define FUNC_CFG_REQ_FLAGS_NO_AUTOCLEAR_STATISTIC         0x1000UL
+	#define FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST                 0x2000UL
+	#define FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST                 0x4000UL
+	#define FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST               0x8000UL
+	#define FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST         0x10000UL
+	#define FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST           0x20000UL
+	#define FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST           0x40000UL
+	#define FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST               0x80000UL
+	#define FUNC_CFG_REQ_FLAGS_L2_CTX_ASSETS_TEST             0x100000UL
+	__le32	enables;
+	#define FUNC_CFG_REQ_ENABLES_MTU                     0x1UL
+	#define FUNC_CFG_REQ_ENABLES_MRU                     0x2UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS         0x4UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS          0x8UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS            0x10UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS            0x20UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_L2_CTXS             0x40UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_VNICS               0x80UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS           0x100UL
+	#define FUNC_CFG_REQ_ENABLES_DFLT_MAC_ADDR           0x200UL
+	#define FUNC_CFG_REQ_ENABLES_DFLT_VLAN               0x400UL
+	#define FUNC_CFG_REQ_ENABLES_DFLT_IP_ADDR            0x800UL
+	#define FUNC_CFG_REQ_ENABLES_MIN_BW                  0x1000UL
+	#define FUNC_CFG_REQ_ENABLES_MAX_BW                  0x2000UL
+	#define FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR          0x4000UL
+	#define FUNC_CFG_REQ_ENABLES_VLAN_ANTISPOOF_MODE     0x8000UL
+	#define FUNC_CFG_REQ_ENABLES_ALLOWED_VLAN_PRIS       0x10000UL
+	#define FUNC_CFG_REQ_ENABLES_EVB_MODE                0x20000UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_MCAST_FILTERS       0x40000UL
+	#define FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS        0x80000UL
+	#define FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE          0x100000UL
+	__le16	mtu;
+	__le16	mru;
+	__le16	num_rsscos_ctxs;
+	__le16	num_cmpl_rings;
+	__le16	num_tx_rings;
+	__le16	num_rx_rings;
+	__le16	num_l2_ctxs;
+	__le16	num_vnics;
+	__le16	num_stat_ctxs;
+	__le16	num_hw_ring_grps;
+	u8	dflt_mac_addr[6];
+	__le16	dflt_vlan;
+	__be32	dflt_ip_addr[4];
+	__le32	min_bw;
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_SFT              0
+	#define FUNC_CFG_REQ_MIN_BW_SCALE                     0x10000000UL
+	#define FUNC_CFG_REQ_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define FUNC_CFG_REQ_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define FUNC_CFG_REQ_MIN_BW_SCALE_LAST                 FUNC_CFG_REQ_MIN_BW_SCALE_BYTES
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_LAST         FUNC_CFG_REQ_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	max_bw;
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_SFT              0
+	#define FUNC_CFG_REQ_MAX_BW_SCALE                     0x10000000UL
+	#define FUNC_CFG_REQ_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define FUNC_CFG_REQ_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define FUNC_CFG_REQ_MAX_BW_SCALE_LAST                 FUNC_CFG_REQ_MAX_BW_SCALE_BYTES
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_LAST         FUNC_CFG_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
+	__le16	async_event_cr;
+	u8	vlan_antispoof_mode;
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_NOCHECK                 0x0UL
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_VALIDATE_VLAN           0x1UL
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_IF_VLANDNE       0x2UL
 	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN 0x3UL
-	u8 allowed_vlan_pris;
-	u8 evb_mode;
-	#define FUNC_CFG_REQ_EVB_MODE_NO_EVB			   0x0UL
-	#define FUNC_CFG_REQ_EVB_MODE_VEB			   0x1UL
-	#define FUNC_CFG_REQ_EVB_MODE_VEPA			   0x2UL
-	u8 unused_2;
-	__le16 num_mcast_filters;
+	#define FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_LAST                   FUNC_CFG_REQ_VLAN_ANTISPOOF_MODE_INSERT_OR_OVERRIDE_VLAN
+	u8	allowed_vlan_pris;
+	u8	evb_mode;
+	#define FUNC_CFG_REQ_EVB_MODE_NO_EVB 0x0UL
+	#define FUNC_CFG_REQ_EVB_MODE_VEB    0x1UL
+	#define FUNC_CFG_REQ_EVB_MODE_VEPA   0x2UL
+	#define FUNC_CFG_REQ_EVB_MODE_LAST  FUNC_CFG_REQ_EVB_MODE_VEPA
+	u8	cache_linesize;
+	#define FUNC_CFG_REQ_CACHE_LINESIZE_CACHE_LINESIZE_64  0x0UL
+	#define FUNC_CFG_REQ_CACHE_LINESIZE_CACHE_LINESIZE_128 0x1UL
+	#define FUNC_CFG_REQ_CACHE_LINESIZE_LAST              FUNC_CFG_REQ_CACHE_LINESIZE_CACHE_LINESIZE_128
+	__le16	num_mcast_filters;
+};
+
+/* hwrm_func_cfg_output (size:128b/16B) */
+struct hwrm_func_cfg_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
 };
 
-/* Output (16 bytes) */
-struct hwrm_func_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_qstats */
-/* Input (24 bytes) */
+/* hwrm_func_qstats_input (size:192b/24B) */
 struct hwrm_func_qstats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 fid;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	fid;
+	u8	unused_0[6];
 };
 
-/* Output (176 bytes) */
+/* hwrm_func_qstats_output (size:1408b/176B) */
 struct hwrm_func_qstats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 tx_ucast_pkts;
-	__le64 tx_mcast_pkts;
-	__le64 tx_bcast_pkts;
-	__le64 tx_discard_pkts;
-	__le64 tx_drop_pkts;
-	__le64 tx_ucast_bytes;
-	__le64 tx_mcast_bytes;
-	__le64 tx_bcast_bytes;
-	__le64 rx_ucast_pkts;
-	__le64 rx_mcast_pkts;
-	__le64 rx_bcast_pkts;
-	__le64 rx_discard_pkts;
-	__le64 rx_drop_pkts;
-	__le64 rx_ucast_bytes;
-	__le64 rx_mcast_bytes;
-	__le64 rx_bcast_bytes;
-	__le64 rx_agg_pkts;
-	__le64 rx_agg_bytes;
-	__le64 rx_agg_events;
-	__le64 rx_agg_aborts;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_clr_stats */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	tx_ucast_pkts;
+	__le64	tx_mcast_pkts;
+	__le64	tx_bcast_pkts;
+	__le64	tx_discard_pkts;
+	__le64	tx_drop_pkts;
+	__le64	tx_ucast_bytes;
+	__le64	tx_mcast_bytes;
+	__le64	tx_bcast_bytes;
+	__le64	rx_ucast_pkts;
+	__le64	rx_mcast_pkts;
+	__le64	rx_bcast_pkts;
+	__le64	rx_discard_pkts;
+	__le64	rx_drop_pkts;
+	__le64	rx_ucast_bytes;
+	__le64	rx_mcast_bytes;
+	__le64	rx_bcast_bytes;
+	__le64	rx_agg_pkts;
+	__le64	rx_agg_bytes;
+	__le64	rx_agg_events;
+	__le64	rx_agg_aborts;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_clr_stats_input (size:192b/24B) */
 struct hwrm_func_clr_stats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 fid;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	fid;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_func_clr_stats_output (size:128b/16B) */
 struct hwrm_func_clr_stats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_vf_resc_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_vf_resc_free_input (size:192b/24B) */
 struct hwrm_func_vf_resc_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 vf_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	vf_id;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_func_vf_resc_free_output (size:128b/16B) */
 struct hwrm_func_vf_resc_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_vf_vnic_ids_query */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_vf_vnic_ids_query_input (size:256b/32B) */
 struct hwrm_func_vf_vnic_ids_query_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 vf_id;
-	u8 unused_0;
-	u8 unused_1;
-	__le32 max_vnic_id_cnt;
-	__le64 vnic_id_tbl_addr;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	vf_id;
+	u8	unused_0[2];
+	__le32	max_vnic_id_cnt;
+	__le64	vnic_id_tbl_addr;
+};
+
+/* hwrm_func_vf_vnic_ids_query_output (size:128b/16B) */
 struct hwrm_func_vf_vnic_ids_query_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 vnic_id_cnt;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_func_drv_rgtr */
-/* Input (80 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	vnic_id_cnt;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_func_drv_rgtr_input (size:832b/104B) */
 struct hwrm_func_drv_rgtr_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define FUNC_DRV_RGTR_REQ_FLAGS_FWD_ALL_MODE		    0x1UL
-	#define FUNC_DRV_RGTR_REQ_FLAGS_FWD_NONE_MODE		    0x2UL
-	__le32 enables;
-	#define FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE		    0x1UL
-	#define FUNC_DRV_RGTR_REQ_ENABLES_VER			    0x2UL
-	#define FUNC_DRV_RGTR_REQ_ENABLES_TIMESTAMP		    0x4UL
-	#define FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD		    0x8UL
-	#define FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD	    0x10UL
-	__le16 os_type;
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UNKNOWN		   0x0UL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_OTHER		   0x1UL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_MSDOS		   0xeUL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WINDOWS		   0x12UL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_SOLARIS		   0x1dUL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX		   0x24UL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_FREEBSD		   0x2aUL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_ESXI			   0x68UL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN864		   0x73UL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN2012R2		   0x74UL
-	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UEFI			   0x8000UL
-	u8 ver_maj;
-	u8 ver_min;
-	u8 ver_upd;
-	u8 unused_0;
-	__le16 unused_1;
-	__le32 timestamp;
-	__le32 unused_2;
-	__le32 vf_req_fwd[8];
-	__le32 async_event_fwd[8];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define FUNC_DRV_RGTR_REQ_FLAGS_FWD_ALL_MODE      0x1UL
+	#define FUNC_DRV_RGTR_REQ_FLAGS_FWD_NONE_MODE     0x2UL
+	__le32	enables;
+	#define FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE             0x1UL
+	#define FUNC_DRV_RGTR_REQ_ENABLES_VER                 0x2UL
+	#define FUNC_DRV_RGTR_REQ_ENABLES_TIMESTAMP           0x4UL
+	#define FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD          0x8UL
+	#define FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD     0x10UL
+	__le16	os_type;
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UNKNOWN   0x0UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_OTHER     0x1UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_MSDOS     0xeUL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WINDOWS   0x12UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_SOLARIS   0x1dUL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX     0x24UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_FREEBSD   0x2aUL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_ESXI      0x68UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN864    0x73UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_WIN2012R2 0x74UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_UEFI      0x8000UL
+	#define FUNC_DRV_RGTR_REQ_OS_TYPE_LAST     FUNC_DRV_RGTR_REQ_OS_TYPE_UEFI
+	u8	ver_maj;
+	u8	ver_min;
+	u8	ver_upd;
+	u8	unused_0[3];
+	__le32	timestamp;
+	u8	unused_1[4];
+	__le32	vf_req_fwd[8];
+	__le32	async_event_fwd[8];
+};
+
+/* hwrm_func_drv_rgtr_output (size:128b/16B) */
 struct hwrm_func_drv_rgtr_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_drv_unrgtr */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_drv_unrgtr_input (size:192b/24B) */
 struct hwrm_func_drv_unrgtr_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
 	#define FUNC_DRV_UNRGTR_REQ_FLAGS_PREPARE_FOR_SHUTDOWN     0x1UL
-	__le32 unused_0;
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_func_drv_unrgtr_output (size:128b/16B) */
 struct hwrm_func_drv_unrgtr_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_buf_rgtr */
-/* Input (128 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_buf_rgtr_input (size:1024b/128B) */
 struct hwrm_func_buf_rgtr_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define FUNC_BUF_RGTR_REQ_ENABLES_VF_ID		    0x1UL
-	#define FUNC_BUF_RGTR_REQ_ENABLES_ERR_BUF_ADDR		    0x2UL
-	__le16 vf_id;
-	__le16 req_buf_num_pages;
-	__le16 req_buf_page_size;
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_16B	   0x4UL
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4K		   0xcUL
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_8K		   0xdUL
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_64K	   0x10UL
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_2M		   0x15UL
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4M		   0x16UL
-	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_1G		   0x1eUL
-	__le16 req_buf_len;
-	__le16 resp_buf_len;
-	u8 unused_0;
-	u8 unused_1;
-	__le64 req_buf_page_addr0;
-	__le64 req_buf_page_addr1;
-	__le64 req_buf_page_addr2;
-	__le64 req_buf_page_addr3;
-	__le64 req_buf_page_addr4;
-	__le64 req_buf_page_addr5;
-	__le64 req_buf_page_addr6;
-	__le64 req_buf_page_addr7;
-	__le64 req_buf_page_addr8;
-	__le64 req_buf_page_addr9;
-	__le64 error_buf_addr;
-	__le64 resp_buf_addr;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define FUNC_BUF_RGTR_REQ_ENABLES_VF_ID            0x1UL
+	#define FUNC_BUF_RGTR_REQ_ENABLES_ERR_BUF_ADDR     0x2UL
+	__le16	vf_id;
+	__le16	req_buf_num_pages;
+	__le16	req_buf_page_size;
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_16B 0x4UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4K  0xcUL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_8K  0xdUL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_64K 0x10UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_2M  0x15UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_4M  0x16UL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_1G  0x1eUL
+	#define FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_LAST FUNC_BUF_RGTR_REQ_REQ_BUF_PAGE_SIZE_1G
+	__le16	req_buf_len;
+	__le16	resp_buf_len;
+	u8	unused_0[2];
+	__le64	req_buf_page_addr0;
+	__le64	req_buf_page_addr1;
+	__le64	req_buf_page_addr2;
+	__le64	req_buf_page_addr3;
+	__le64	req_buf_page_addr4;
+	__le64	req_buf_page_addr5;
+	__le64	req_buf_page_addr6;
+	__le64	req_buf_page_addr7;
+	__le64	req_buf_page_addr8;
+	__le64	req_buf_page_addr9;
+	__le64	error_buf_addr;
+	__le64	resp_buf_addr;
+};
+
+/* hwrm_func_buf_rgtr_output (size:128b/16B) */
 struct hwrm_func_buf_rgtr_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_func_drv_qver */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_func_drv_qver_input (size:192b/24B) */
 struct hwrm_func_drv_qver_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 reserved;
-	__le16 fid;
-	__le16 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	reserved;
+	__le16	fid;
+	u8	unused_0[2];
 };
 
-/* Output (16 bytes) */
+/* hwrm_func_drv_qver_output (size:128b/16B) */
 struct hwrm_func_drv_qver_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 os_type;
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_UNKNOWN		   0x0UL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_OTHER		   0x1UL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_MSDOS		   0xeUL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_WINDOWS		   0x12UL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_SOLARIS		   0x1dUL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_LINUX		   0x24UL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_FREEBSD		   0x2aUL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_ESXI		   0x68UL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN864		   0x73UL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN2012R2		   0x74UL
-	#define FUNC_DRV_QVER_RESP_OS_TYPE_UEFI		   0x8000UL
-	u8 ver_maj;
-	u8 ver_min;
-	u8 ver_upd;
-	u8 unused_0;
-	u8 unused_1;
-	u8 valid;
-};
-
-/* hwrm_port_phy_cfg */
-/* Input (56 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	os_type;
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_UNKNOWN   0x0UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_OTHER     0x1UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_MSDOS     0xeUL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WINDOWS   0x12UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_SOLARIS   0x1dUL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_LINUX     0x24UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_FREEBSD   0x2aUL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_ESXI      0x68UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN864    0x73UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_WIN2012R2 0x74UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_UEFI      0x8000UL
+	#define FUNC_DRV_QVER_RESP_OS_TYPE_LAST     FUNC_DRV_QVER_RESP_OS_TYPE_UEFI
+	u8	ver_maj;
+	u8	ver_min;
+	u8	ver_upd;
+	u8	unused_0[2];
+	u8	valid;
+};
+
+/* hwrm_func_resource_qcaps_input (size:192b/24B) */
+struct hwrm_func_resource_qcaps_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	fid;
+	u8	unused_0[6];
+};
+
+/* hwrm_func_resource_qcaps_output (size:384b/48B) */
+struct hwrm_func_resource_qcaps_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	max_vfs;
+	__le16	max_msix;
+	__le16	vf_reservation_strategy;
+	#define FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_MAXIMAL 0x0UL
+	#define FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_MINIMAL 0x1UL
+	#define FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_LAST   FUNC_RESOURCE_QCAPS_RESP_VF_RESERVATION_STRATEGY_MINIMAL
+	__le16	min_rsscos_ctx;
+	__le16	max_rsscos_ctx;
+	__le16	min_cmpl_rings;
+	__le16	max_cmpl_rings;
+	__le16	min_tx_rings;
+	__le16	max_tx_rings;
+	__le16	min_rx_rings;
+	__le16	max_rx_rings;
+	__le16	min_l2_ctxs;
+	__le16	max_l2_ctxs;
+	__le16	min_vnics;
+	__le16	max_vnics;
+	__le16	min_stat_ctx;
+	__le16	max_stat_ctx;
+	__le16	min_hw_ring_grps;
+	__le16	max_hw_ring_grps;
+	u8	unused_0;
+	u8	valid;
+};
+
+/* hwrm_func_vf_resource_cfg_input (size:448b/56B) */
+struct hwrm_func_vf_resource_cfg_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	vf_id;
+	__le16	max_msix;
+	__le16	min_rsscos_ctx;
+	__le16	max_rsscos_ctx;
+	__le16	min_cmpl_rings;
+	__le16	max_cmpl_rings;
+	__le16	min_tx_rings;
+	__le16	max_tx_rings;
+	__le16	min_rx_rings;
+	__le16	max_rx_rings;
+	__le16	min_l2_ctxs;
+	__le16	max_l2_ctxs;
+	__le16	min_vnics;
+	__le16	max_vnics;
+	__le16	min_stat_ctx;
+	__le16	max_stat_ctx;
+	__le16	min_hw_ring_grps;
+	__le16	max_hw_ring_grps;
+	u8	unused_0[4];
+};
+
+/* hwrm_func_vf_resource_cfg_output (size:256b/32B) */
+struct hwrm_func_vf_resource_cfg_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	reserved_rsscos_ctx;
+	__le16	reserved_cmpl_rings;
+	__le16	reserved_tx_rings;
+	__le16	reserved_rx_rings;
+	__le16	reserved_l2_ctxs;
+	__le16	reserved_vnics;
+	__le16	reserved_stat_ctx;
+	__le16	reserved_hw_ring_grps;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_port_phy_cfg_input (size:448b/56B) */
 struct hwrm_port_phy_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define PORT_PHY_CFG_REQ_FLAGS_RESET_PHY		    0x1UL
-	#define PORT_PHY_CFG_REQ_FLAGS_DEPRECATED		    0x2UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FORCE			    0x4UL
-	#define PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG		    0x8UL
-	#define PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE		    0x10UL
-	#define PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE		    0x20UL
-	#define PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE	    0x40UL
-	#define PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE	    0x80UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_ENABLE	    0x100UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_DISABLE	    0x200UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_ENABLE	    0x400UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_DISABLE	    0x800UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_ENABLE	    0x1000UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_DISABLE	    0x2000UL
-	#define PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN		    0x4000UL
-	__le32 enables;
-	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE		    0x1UL
-	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX		    0x2UL
-	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE		    0x4UL
-	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED	    0x8UL
-	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK      0x10UL
-	#define PORT_PHY_CFG_REQ_ENABLES_WIRESPEED		    0x20UL
-	#define PORT_PHY_CFG_REQ_ENABLES_LPBK			    0x40UL
-	#define PORT_PHY_CFG_REQ_ENABLES_PREEMPHASIS		    0x80UL
-	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE		    0x100UL
-	#define PORT_PHY_CFG_REQ_ENABLES_EEE_LINK_SPEED_MASK       0x200UL
-	#define PORT_PHY_CFG_REQ_ENABLES_TX_LPI_TIMER		    0x400UL
-	__le16 port_id;
-	__le16 force_link_speed;
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB	   0x1UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB		   0xaUL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2GB		   0x14UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2_5GB	   0x19UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB		   0x64UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_20GB		   0xc8UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB		   0xfaUL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB		   0x190UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB		   0x1f4UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB	   0x3e8UL
-	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB		   0xffffUL
-	u8 auto_mode;
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_NONE		   0x0UL
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_ALL_SPEEDS		   0x1UL
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_SPEED		   0x2UL
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_OR_BELOW	   0x3UL
-	#define PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK		   0x4UL
-	u8 auto_duplex;
-	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_HALF		   0x0UL
-	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_FULL		   0x1UL
-	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH		   0x2UL
-	u8 auto_pause;
-	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_TX			    0x1UL
-	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_RX			    0x2UL
-	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE	    0x4UL
-	u8 unused_0;
-	__le16 auto_link_speed;
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100MB		   0x1UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_1GB		   0xaUL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2GB		   0x14UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2_5GB		   0x19UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10GB		   0x64UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_20GB		   0xc8UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_25GB		   0xfaUL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_40GB		   0x190UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_50GB		   0x1f4UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100GB		   0x3e8UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB		   0xffffUL
-	__le16 auto_link_speed_mask;
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MBHD      0x1UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MB	    0x2UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GBHD	    0x4UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GB	    0x8UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2GB	    0x10UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2_5GB	    0x20UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10GB	    0x40UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_20GB	    0x80UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_25GB	    0x100UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_40GB	    0x200UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_50GB	    0x400UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100GB	    0x800UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MBHD       0x1000UL
-	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MB	    0x2000UL
-	u8 wirespeed;
-	#define PORT_PHY_CFG_REQ_WIRESPEED_OFF			   0x0UL
-	#define PORT_PHY_CFG_REQ_WIRESPEED_ON			   0x1UL
-	u8 lpbk;
-	#define PORT_PHY_CFG_REQ_LPBK_NONE			   0x0UL
-	#define PORT_PHY_CFG_REQ_LPBK_LOCAL			   0x1UL
-	#define PORT_PHY_CFG_REQ_LPBK_REMOTE			   0x2UL
-	u8 force_pause;
-	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_TX		    0x1UL
-	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_RX		    0x2UL
-	u8 unused_1;
-	__le32 preemphasis;
-	__le16 eee_link_speed_mask;
-	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD1	    0x1UL
-	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_100MB	    0x2UL
-	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD2	    0x4UL
-	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_1GB	    0x8UL
-	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD3	    0x10UL
-	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD4	    0x20UL
-	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_10GB	    0x40UL
-	u8 unused_2;
-	u8 unused_3;
-	__le32 tx_lpi_timer;
-	__le32 unused_4;
-	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_MASK		    0xffffffUL
-	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_SFT		    0
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define PORT_PHY_CFG_REQ_FLAGS_RESET_PHY                0x1UL
+	#define PORT_PHY_CFG_REQ_FLAGS_DEPRECATED               0x2UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FORCE                    0x4UL
+	#define PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG          0x8UL
+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE               0x10UL
+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE              0x20UL
+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE        0x40UL
+	#define PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE       0x80UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_ENABLE       0x100UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_AUTONEG_DISABLE      0x200UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_ENABLE      0x400UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE74_DISABLE     0x800UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_ENABLE      0x1000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FEC_CLAUSE91_DISABLE     0x2000UL
+	#define PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN           0x4000UL
+	__le32	enables;
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE                0x1UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_DUPLEX              0x2UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE               0x4UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED          0x8UL
+	#define PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK     0x10UL
+	#define PORT_PHY_CFG_REQ_ENABLES_WIRESPEED                0x20UL
+	#define PORT_PHY_CFG_REQ_ENABLES_LPBK                     0x40UL
+	#define PORT_PHY_CFG_REQ_ENABLES_PREEMPHASIS              0x80UL
+	#define PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE              0x100UL
+	#define PORT_PHY_CFG_REQ_ENABLES_EEE_LINK_SPEED_MASK      0x200UL
+	#define PORT_PHY_CFG_REQ_ENABLES_TX_LPI_TIMER             0x400UL
+	__le16	port_id;
+	__le16	force_link_speed;
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100MB 0x1UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_1GB   0xaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2GB   0x14UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_2_5GB 0x19UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10GB  0x64UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_20GB  0xc8UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_25GB  0xfaUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_40GB  0x190UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB  0xffffUL
+	#define PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_LAST PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_10MB
+	u8	auto_mode;
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_NONE         0x0UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ALL_SPEEDS   0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_SPEED    0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_ONE_OR_BELOW 0x3UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK   0x4UL
+	#define PORT_PHY_CFG_REQ_AUTO_MODE_LAST        PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK
+	u8	auto_duplex;
+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_HALF 0x0UL
+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_FULL 0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH 0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_DUPLEX_LAST PORT_PHY_CFG_REQ_AUTO_DUPLEX_BOTH
+	u8	auto_pause;
+	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_TX                0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_RX                0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE     0x4UL
+	u8	unused_0;
+	__le16	auto_link_speed;
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100MB 0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_1GB   0xaUL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2GB   0x14UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_2_5GB 0x19UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10GB  0x64UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_20GB  0xc8UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_25GB  0xfaUL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_40GB  0x190UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB  0xffffUL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_LAST PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_10MB
+	__le16	auto_link_speed_mask;
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MBHD     0x1UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100MB       0x2UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GBHD       0x4UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_1GB         0x8UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2GB         0x10UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_2_5GB       0x20UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10GB        0x40UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_20GB        0x80UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_25GB        0x100UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_40GB        0x200UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_50GB        0x400UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_100GB       0x800UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MBHD      0x1000UL
+	#define PORT_PHY_CFG_REQ_AUTO_LINK_SPEED_MASK_10MB        0x2000UL
+	u8	wirespeed;
+	#define PORT_PHY_CFG_REQ_WIRESPEED_OFF 0x0UL
+	#define PORT_PHY_CFG_REQ_WIRESPEED_ON  0x1UL
+	#define PORT_PHY_CFG_REQ_WIRESPEED_LAST PORT_PHY_CFG_REQ_WIRESPEED_ON
+	u8	lpbk;
+	#define PORT_PHY_CFG_REQ_LPBK_NONE   0x0UL
+	#define PORT_PHY_CFG_REQ_LPBK_LOCAL  0x1UL
+	#define PORT_PHY_CFG_REQ_LPBK_REMOTE 0x2UL
+	#define PORT_PHY_CFG_REQ_LPBK_LAST  PORT_PHY_CFG_REQ_LPBK_REMOTE
+	u8	force_pause;
+	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_TX     0x1UL
+	#define PORT_PHY_CFG_REQ_FORCE_PAUSE_RX     0x2UL
+	u8	unused_1;
+	__le32	preemphasis;
+	__le16	eee_link_speed_mask;
+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD1     0x1UL
+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_100MB     0x2UL
+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD2     0x4UL
+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_1GB       0x8UL
+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
+	#define PORT_PHY_CFG_REQ_EEE_LINK_SPEED_MASK_10GB      0x40UL
+	u8	unused_2[2];
+	__le32	tx_lpi_timer;
+	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_MASK 0xffffffUL
+	#define PORT_PHY_CFG_REQ_TX_LPI_TIMER_SFT 0
+	__le32	unused_3;
+};
+
+/* hwrm_port_phy_cfg_output (size:128b/16B) */
 struct hwrm_port_phy_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_port_phy_qcfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_port_phy_qcfg_input (size:192b/24B) */
 struct hwrm_port_phy_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
 };
 
-/* Output (96 bytes) */
+/* hwrm_port_phy_qcfg_output (size:768b/96B) */
 struct hwrm_port_phy_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 link;
-	#define PORT_PHY_QCFG_RESP_LINK_NO_LINK		   0x0UL
-	#define PORT_PHY_QCFG_RESP_LINK_SIGNAL			   0x1UL
-	#define PORT_PHY_QCFG_RESP_LINK_LINK			   0x2UL
-	u8 unused_0;
-	__le16 link_speed;
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100MB		   0x1UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_1GB		   0xaUL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2GB		   0x14UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2_5GB		   0x19UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10GB		   0x64UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_20GB		   0xc8UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_25GB		   0xfaUL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_40GB		   0x190UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB		   0x1f4UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB		   0x3e8UL
-	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB		   0xffffUL
-	u8 duplex_cfg;
-	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_HALF		   0x0UL
-	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_FULL		   0x1UL
-	u8 pause;
-	#define PORT_PHY_QCFG_RESP_PAUSE_TX			    0x1UL
-	#define PORT_PHY_QCFG_RESP_PAUSE_RX			    0x2UL
-	__le16 support_speeds;
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100MBHD	    0x1UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100MB	    0x2UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_1GBHD	    0x4UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_1GB		    0x8UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_2GB		    0x10UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_2_5GB	    0x20UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10GB		    0x40UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_20GB		    0x80UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_25GB		    0x100UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_40GB		    0x200UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_50GB		    0x400UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100GB	    0x800UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MBHD	    0x1000UL
-	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MB		    0x2000UL
-	__le16 force_link_speed;
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100MB	   0x1UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_1GB	   0xaUL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2GB	   0x14UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2_5GB	   0x19UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10GB	   0x64UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_20GB	   0xc8UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_25GB	   0xfaUL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_40GB	   0x190UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_50GB	   0x1f4UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100GB	   0x3e8UL
-	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10MB	   0xffffUL
-	u8 auto_mode;
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_NONE		   0x0UL
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ALL_SPEEDS	   0x1UL
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_SPEED		   0x2UL
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_OR_BELOW	   0x3UL
-	#define PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK	   0x4UL
-	u8 auto_pause;
-	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_TX		    0x1UL
-	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_RX		    0x2UL
-	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_AUTONEG_PAUSE	    0x4UL
-	__le16 auto_link_speed;
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100MB	   0x1UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_1GB		   0xaUL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2GB		   0x14UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2_5GB	   0x19UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10GB	   0x64UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_20GB	   0xc8UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_25GB	   0xfaUL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_40GB	   0x190UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_50GB	   0x1f4UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100GB	   0x3e8UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10MB	   0xffffUL
-	__le16 auto_link_speed_mask;
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MBHD    0x1UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MB      0x2UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_1GBHD      0x4UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_1GB	    0x8UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_2GB	    0x10UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_2_5GB      0x20UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10GB       0x40UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_20GB       0x80UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_25GB       0x100UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_40GB       0x200UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_50GB       0x400UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100GB      0x800UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MBHD     0x1000UL
-	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MB       0x2000UL
-	u8 wirespeed;
-	#define PORT_PHY_QCFG_RESP_WIRESPEED_OFF		   0x0UL
-	#define PORT_PHY_QCFG_RESP_WIRESPEED_ON		   0x1UL
-	u8 lpbk;
-	#define PORT_PHY_QCFG_RESP_LPBK_NONE			   0x0UL
-	#define PORT_PHY_QCFG_RESP_LPBK_LOCAL			   0x1UL
-	#define PORT_PHY_QCFG_RESP_LPBK_REMOTE			   0x2UL
-	u8 force_pause;
-	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_TX		    0x1UL
-	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_RX		    0x2UL
-	u8 module_status;
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NONE		   0x0UL
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX	   0x1UL
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG       0x2UL
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN	   0x3UL
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTINSERTED      0x4UL
-	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTAPPLICABLE    0xffUL
-	__le32 preemphasis;
-	u8 phy_maj;
-	u8 phy_min;
-	u8 phy_bld;
-	u8 phy_type;
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_UNKNOWN		   0x0UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASECR		   0x1UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR4		   0x2UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASELR		   0x3UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASESR		   0x4UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR2		   0x5UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKX		   0x6UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR		   0x7UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASET		   0x8UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASETE		   0x9UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_SGMIIEXTPHY	   0xaUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_L       0xbUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_S       0xcUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_N       0xdUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASESR		   0xeUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR4	   0xfUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR4	   0x10UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR4	   0x11UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER4	   0x12UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR10	   0x13UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASECR4	   0x14UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASESR4	   0x15UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASELR4	   0x16UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASEER4	   0x17UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_ACTIVE_CABLE      0x18UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASET		   0x19UL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASESX		   0x1aUL
-	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASECX		   0x1bUL
-	u8 media_type;
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN		   0x0UL
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP		   0x1UL
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC		   0x2UL
-	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_FIBRE		   0x3UL
-	u8 xcvr_pkg_type;
-	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_INTERNAL    0x1UL
-	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_EXTERNAL    0x2UL
-	u8 eee_config_phy_addr;
-	#define PORT_PHY_QCFG_RESP_PHY_ADDR_MASK		    0x1fUL
-	#define PORT_PHY_QCFG_RESP_PHY_ADDR_SFT		    0
-	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED	    0x20UL
-	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE	    0x40UL
-	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI	    0x80UL
-	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_MASK		    0xe0UL
-	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_SFT		    5
-	u8 parallel_detect;
-	#define PORT_PHY_QCFG_RESP_PARALLEL_DETECT		    0x1UL
-	#define PORT_PHY_QCFG_RESP_RESERVED_MASK		    0xfeUL
-	#define PORT_PHY_QCFG_RESP_RESERVED_SFT		    1
-	__le16 link_partner_adv_speeds;
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100MBHD 0x1UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100MB   0x2UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_1GBHD   0x4UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_1GB     0x8UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_2GB     0x10UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_2_5GB   0x20UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10GB    0x40UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_20GB    0x80UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_25GB    0x100UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_40GB    0x200UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_50GB    0x400UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100GB   0x800UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MBHD  0x1000UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MB    0x2000UL
-	u8 link_partner_adv_auto_mode;
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_NONE 0x0UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS 0x1UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED 0x2UL
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	link;
+	#define PORT_PHY_QCFG_RESP_LINK_NO_LINK 0x0UL
+	#define PORT_PHY_QCFG_RESP_LINK_SIGNAL  0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_LINK    0x2UL
+	#define PORT_PHY_QCFG_RESP_LINK_LAST   PORT_PHY_QCFG_RESP_LINK_LINK
+	u8	unused_0;
+	__le16	link_speed;
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100MB 0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_1GB   0xaUL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2GB   0x14UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_2_5GB 0x19UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10GB  0x64UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_20GB  0xc8UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_25GB  0xfaUL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_40GB  0x190UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_10MB  0xffffUL
+	#define PORT_PHY_QCFG_RESP_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_LINK_SPEED_10MB
+	u8	duplex_cfg;
+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_HALF 0x0UL
+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_FULL 0x1UL
+	#define PORT_PHY_QCFG_RESP_DUPLEX_CFG_LAST PORT_PHY_QCFG_RESP_DUPLEX_CFG_FULL
+	u8	pause;
+	#define PORT_PHY_QCFG_RESP_PAUSE_TX     0x1UL
+	#define PORT_PHY_QCFG_RESP_PAUSE_RX     0x2UL
+	__le16	support_speeds;
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100MBHD     0x1UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100MB       0x2UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_1GBHD       0x4UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_1GB         0x8UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_2GB         0x10UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_2_5GB       0x20UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10GB        0x40UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_20GB        0x80UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_25GB        0x100UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_40GB        0x200UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_50GB        0x400UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_100GB       0x800UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MBHD      0x1000UL
+	#define PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS_10MB        0x2000UL
+	__le16	force_link_speed;
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100MB 0x1UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_1GB   0xaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2GB   0x14UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_2_5GB 0x19UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10GB  0x64UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_20GB  0xc8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_25GB  0xfaUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_40GB  0x190UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10MB  0xffffUL
+	#define PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_FORCE_LINK_SPEED_10MB
+	u8	auto_mode;
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_NONE         0x0UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ALL_SPEEDS   0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_SPEED    0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_ONE_OR_BELOW 0x3UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK   0x4UL
+	#define PORT_PHY_QCFG_RESP_AUTO_MODE_LAST        PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK
+	u8	auto_pause;
+	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_TX                0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_RX                0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_PAUSE_AUTONEG_PAUSE     0x4UL
+	__le16	auto_link_speed;
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100MB 0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_1GB   0xaUL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2GB   0x14UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_2_5GB 0x19UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10GB  0x64UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_20GB  0xc8UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_25GB  0xfaUL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_40GB  0x190UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_50GB  0x1f4UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_100GB 0x3e8UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10MB  0xffffUL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_LAST PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_10MB
+	__le16	auto_link_speed_mask;
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MBHD     0x1UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100MB       0x2UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_1GBHD       0x4UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_1GB         0x8UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_2GB         0x10UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_2_5GB       0x20UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10GB        0x40UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_20GB        0x80UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_25GB        0x100UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_40GB        0x200UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_50GB        0x400UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_100GB       0x800UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MBHD      0x1000UL
+	#define PORT_PHY_QCFG_RESP_AUTO_LINK_SPEED_MASK_10MB        0x2000UL
+	u8	wirespeed;
+	#define PORT_PHY_QCFG_RESP_WIRESPEED_OFF 0x0UL
+	#define PORT_PHY_QCFG_RESP_WIRESPEED_ON  0x1UL
+	#define PORT_PHY_QCFG_RESP_WIRESPEED_LAST PORT_PHY_QCFG_RESP_WIRESPEED_ON
+	u8	lpbk;
+	#define PORT_PHY_QCFG_RESP_LPBK_NONE   0x0UL
+	#define PORT_PHY_QCFG_RESP_LPBK_LOCAL  0x1UL
+	#define PORT_PHY_QCFG_RESP_LPBK_REMOTE 0x2UL
+	#define PORT_PHY_QCFG_RESP_LPBK_LAST  PORT_PHY_QCFG_RESP_LPBK_REMOTE
+	u8	force_pause;
+	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_TX     0x1UL
+	#define PORT_PHY_QCFG_RESP_FORCE_PAUSE_RX     0x2UL
+	u8	module_status;
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NONE          0x0UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX     0x1UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG    0x2UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN       0x3UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTINSERTED   0x4UL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTAPPLICABLE 0xffUL
+	#define PORT_PHY_QCFG_RESP_MODULE_STATUS_LAST         PORT_PHY_QCFG_RESP_MODULE_STATUS_NOTAPPLICABLE
+	__le32	preemphasis;
+	u8	phy_maj;
+	u8	phy_min;
+	u8	phy_bld;
+	u8	phy_type;
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_UNKNOWN          0x0UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASECR           0x1UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR4          0x2UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASELR           0x3UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASESR           0x4UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR2          0x5UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKX           0x6UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASEKR           0x7UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASET            0x8UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_BASETE           0x9UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_SGMIIEXTPHY      0xaUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_L  0xbUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_S  0xcUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASECR_CA_N  0xdUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_25G_BASESR       0xeUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR4     0xfUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR4     0x10UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR4     0x11UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER4     0x12UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR10    0x13UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASECR4      0x14UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASESR4      0x15UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASELR4      0x16UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_BASEER4      0x17UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_40G_ACTIVE_CABLE 0x18UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASET         0x19UL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASESX        0x1aUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASECX        0x1bUL
+	#define PORT_PHY_QCFG_RESP_PHY_TYPE_LAST            PORT_PHY_QCFG_RESP_PHY_TYPE_1G_BASECX
+	u8	media_type;
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_UNKNOWN 0x0UL
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_TP      0x1UL
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC     0x2UL
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_FIBRE   0x3UL
+	#define PORT_PHY_QCFG_RESP_MEDIA_TYPE_LAST   PORT_PHY_QCFG_RESP_MEDIA_TYPE_FIBRE
+	u8	xcvr_pkg_type;
+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_INTERNAL 0x1UL
+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_EXTERNAL 0x2UL
+	#define PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_LAST         PORT_PHY_QCFG_RESP_XCVR_PKG_TYPE_XCVR_EXTERNAL
+	u8	eee_config_phy_addr;
+	#define PORT_PHY_QCFG_RESP_PHY_ADDR_MASK              0x1fUL
+	#define PORT_PHY_QCFG_RESP_PHY_ADDR_SFT               0
+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_MASK            0xe0UL
+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_SFT             5
+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED      0x20UL
+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE       0x40UL
+	#define PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI       0x80UL
+	u8	parallel_detect;
+	#define PORT_PHY_QCFG_RESP_PARALLEL_DETECT     0x1UL
+	__le16	link_partner_adv_speeds;
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100MBHD     0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100MB       0x2UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_1GBHD       0x4UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_1GB         0x8UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_2GB         0x10UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_2_5GB       0x20UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10GB        0x40UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_20GB        0x80UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_25GB        0x100UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_40GB        0x200UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_50GB        0x400UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_100GB       0x800UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MBHD      0x1000UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_SPEEDS_10MB        0x2000UL
+	u8	link_partner_adv_auto_mode;
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_NONE         0x0UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ALL_SPEEDS   0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_SPEED    0x2UL
 	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_ONE_OR_BELOW 0x3UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK 0x4UL
-	u8 link_partner_adv_pause;
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_TX       0x1UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_RX       0x2UL
-	__le16 adv_eee_link_speed_mask;
-	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD1   0x1UL
-	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_100MB   0x2UL
-	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD2   0x4UL
-	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_1GB     0x8UL
-	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD3   0x10UL
-	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD4   0x20UL
-	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_10GB    0x40UL
-	__le16 link_partner_adv_eee_link_speed_mask;
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD1 0x1UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_100MB 0x2UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD2 0x4UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_1GB 0x8UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD3 0x10UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD4 0x20UL
-	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_10GB 0x40UL
-	__le32 xcvr_identifier_type_tx_lpi_timer;
-	#define PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK		    0xffffffUL
-	#define PORT_PHY_QCFG_RESP_TX_LPI_TIMER_SFT		    0
-	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_MASK       0xff000000UL
-	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_SFT	    24
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK   0x4UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_LAST        PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_AUTO_MODE_SPEED_MASK
+	u8	link_partner_adv_pause;
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_TX     0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_PAUSE_RX     0x2UL
+	__le16	adv_eee_link_speed_mask;
+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD1     0x1UL
+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_100MB     0x2UL
+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD2     0x4UL
+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_1GB       0x8UL
+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
+	#define PORT_PHY_QCFG_RESP_ADV_EEE_LINK_SPEED_MASK_10GB      0x40UL
+	__le16	link_partner_adv_eee_link_speed_mask;
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD1     0x1UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_100MB     0x2UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD2     0x4UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_1GB       0x8UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD3     0x10UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_RSVD4     0x20UL
+	#define PORT_PHY_QCFG_RESP_LINK_PARTNER_ADV_EEE_LINK_SPEED_MASK_10GB      0x40UL
+	__le32	xcvr_identifier_type_tx_lpi_timer;
+	#define PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK            0xffffffUL
+	#define PORT_PHY_QCFG_RESP_TX_LPI_TIMER_SFT             0
+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_MASK    0xff000000UL
+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_SFT     24
 	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_UNKNOWN   (0x0UL << 24)
 	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_SFP       (0x3UL << 24)
 	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFP      (0xcUL << 24)
 	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFPPLUS  (0xdUL << 24)
 	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFP28    (0x11UL << 24)
-	__le16 fec_cfg;
-	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED      0x1UL
-	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_AUTONEG_SUPPORTED   0x2UL
-	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_AUTONEG_ENABLED     0x4UL
-	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_SUPPORTED  0x8UL
-	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_ENABLED    0x10UL
-	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_SUPPORTED  0x20UL
-	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_ENABLED    0x40UL
-	u8 duplex_state;
-	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF		   0x0UL
-	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL		   0x1UL
-	u8 unused_1;
-	char phy_vendor_name[16];
-	char phy_vendor_partnumber[16];
-	__le32 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 unused_5;
-	u8 valid;
-};
-
-/* hwrm_port_mac_cfg */
-/* Input (40 bytes) */
+	#define PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_LAST     PORT_PHY_QCFG_RESP_XCVR_IDENTIFIER_TYPE_QSFP28
+	__le16	fec_cfg;
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED         0x1UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_AUTONEG_SUPPORTED      0x2UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_AUTONEG_ENABLED        0x4UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_SUPPORTED     0x8UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_ENABLED       0x10UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_SUPPORTED     0x20UL
+	#define PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_ENABLED       0x40UL
+	u8	duplex_state;
+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_HALF 0x0UL
+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL 0x1UL
+	#define PORT_PHY_QCFG_RESP_DUPLEX_STATE_LAST PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL
+	u8	option_flags;
+	#define PORT_PHY_QCFG_RESP_OPTION_FLAGS_MEDIA_AUTO_DETECT     0x1UL
+	char	phy_vendor_name[16];
+	char	phy_vendor_partnumber[16];
+	u8	unused_2[7];
+	u8	valid;
+};
+
+/* hwrm_port_mac_cfg_input (size:320b/40B) */
 struct hwrm_port_mac_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define PORT_MAC_CFG_REQ_FLAGS_MATCH_LINK		    0x1UL
-	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_ENABLE	    0x2UL
-	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_ENABLE       0x4UL
-	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_ENABLE	    0x8UL
-	#define PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_ENABLE    0x10UL
-	#define PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_DISABLE   0x20UL
-	#define PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_ENABLE    0x40UL
-	#define PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_DISABLE   0x80UL
-	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_ENABLE		    0x100UL
-	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_DISABLE		    0x200UL
-	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_DISABLE	    0x400UL
-	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_DISABLE      0x800UL
-	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_DISABLE	    0x1000UL
-	__le32 enables;
-	#define PORT_MAC_CFG_REQ_ENABLES_IPG			    0x1UL
-	#define PORT_MAC_CFG_REQ_ENABLES_LPBK			    0x2UL
-	#define PORT_MAC_CFG_REQ_ENABLES_VLAN_PRI2COS_MAP_PRI      0x4UL
-	#define PORT_MAC_CFG_REQ_ENABLES_RESERVED1		    0x8UL
-	#define PORT_MAC_CFG_REQ_ENABLES_TUNNEL_PRI2COS_MAP_PRI    0x10UL
-	#define PORT_MAC_CFG_REQ_ENABLES_DSCP2COS_MAP_PRI	    0x20UL
-	#define PORT_MAC_CFG_REQ_ENABLES_RX_TS_CAPTURE_PTP_MSG_TYPE 0x40UL
-	#define PORT_MAC_CFG_REQ_ENABLES_TX_TS_CAPTURE_PTP_MSG_TYPE 0x80UL
-	#define PORT_MAC_CFG_REQ_ENABLES_COS_FIELD_CFG		    0x100UL
-	__le16 port_id;
-	u8 ipg;
-	u8 lpbk;
-	#define PORT_MAC_CFG_REQ_LPBK_NONE			   0x0UL
-	#define PORT_MAC_CFG_REQ_LPBK_LOCAL			   0x1UL
-	#define PORT_MAC_CFG_REQ_LPBK_REMOTE			   0x2UL
-	u8 vlan_pri2cos_map_pri;
-	u8 reserved1;
-	u8 tunnel_pri2cos_map_pri;
-	u8 dscp2pri_map_pri;
-	__le16 rx_ts_capture_ptp_msg_type;
-	__le16 tx_ts_capture_ptp_msg_type;
-	u8 cos_field_cfg;
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_RSVD1		    0x1UL
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_MASK   0x6UL
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_SFT    1
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_INNERMOST (0x0UL << 1)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTER (0x1UL << 1)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTERMOST (0x2UL << 1)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED (0x3UL << 1)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_LAST    PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_MASK 0x18UL
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_SFT  3
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_INNERMOST (0x0UL << 3)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTER (0x1UL << 3)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTERMOST (0x2UL << 3)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED (0x3UL << 3)
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_LAST    PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_MASK    0xe0UL
-	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_SFT     5
-	u8 unused_0[3];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define PORT_MAC_CFG_REQ_FLAGS_MATCH_LINK                    0x1UL
+	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_ENABLE           0x2UL
+	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_ENABLE         0x4UL
+	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_ENABLE            0x8UL
+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_ENABLE      0x10UL
+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_DISABLE     0x20UL
+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_ENABLE      0x40UL
+	#define PORT_MAC_CFG_REQ_FLAGS_PTP_TX_TS_CAPTURE_DISABLE     0x80UL
+	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_ENABLE                0x100UL
+	#define PORT_MAC_CFG_REQ_FLAGS_OOB_WOL_DISABLE               0x200UL
+	#define PORT_MAC_CFG_REQ_FLAGS_VLAN_PRI2COS_DISABLE          0x400UL
+	#define PORT_MAC_CFG_REQ_FLAGS_TUNNEL_PRI2COS_DISABLE        0x800UL
+	#define PORT_MAC_CFG_REQ_FLAGS_IP_DSCP2COS_DISABLE           0x1000UL
+	__le32	enables;
+	#define PORT_MAC_CFG_REQ_ENABLES_IPG                            0x1UL
+	#define PORT_MAC_CFG_REQ_ENABLES_LPBK                           0x2UL
+	#define PORT_MAC_CFG_REQ_ENABLES_VLAN_PRI2COS_MAP_PRI           0x4UL
+	#define PORT_MAC_CFG_REQ_ENABLES_TUNNEL_PRI2COS_MAP_PRI         0x10UL
+	#define PORT_MAC_CFG_REQ_ENABLES_DSCP2COS_MAP_PRI               0x20UL
+	#define PORT_MAC_CFG_REQ_ENABLES_RX_TS_CAPTURE_PTP_MSG_TYPE     0x40UL
+	#define PORT_MAC_CFG_REQ_ENABLES_TX_TS_CAPTURE_PTP_MSG_TYPE     0x80UL
+	#define PORT_MAC_CFG_REQ_ENABLES_COS_FIELD_CFG                  0x100UL
+	__le16	port_id;
+	u8	ipg;
+	u8	lpbk;
+	#define PORT_MAC_CFG_REQ_LPBK_NONE   0x0UL
+	#define PORT_MAC_CFG_REQ_LPBK_LOCAL  0x1UL
+	#define PORT_MAC_CFG_REQ_LPBK_REMOTE 0x2UL
+	#define PORT_MAC_CFG_REQ_LPBK_LAST  PORT_MAC_CFG_REQ_LPBK_REMOTE
+	u8	vlan_pri2cos_map_pri;
+	u8	reserved1;
+	u8	tunnel_pri2cos_map_pri;
+	u8	dscp2pri_map_pri;
+	__le16	rx_ts_capture_ptp_msg_type;
+	__le16	tx_ts_capture_ptp_msg_type;
+	u8	cos_field_cfg;
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_RSVD1                     0x1UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_MASK         0x6UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_SFT          1
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_INNERMOST      (0x0UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTER          (0x1UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_OUTERMOST      (0x2UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED    (0x3UL << 1)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_LAST          PORT_MAC_CFG_REQ_COS_FIELD_CFG_VLAN_PRI_SEL_UNSPECIFIED
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_MASK       0x18UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_SFT        3
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_INNERMOST    (0x0UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTER        (0x1UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_OUTERMOST    (0x2UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED  (0x3UL << 3)
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_LAST        PORT_MAC_CFG_REQ_COS_FIELD_CFG_T_VLAN_PRI_SEL_UNSPECIFIED
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_MASK          0xe0UL
+	#define PORT_MAC_CFG_REQ_COS_FIELD_CFG_DEFAULT_COS_SFT           5
+	u8	unused_0[3];
+};
+
+/* hwrm_port_mac_cfg_output (size:128b/16B) */
 struct hwrm_port_mac_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 mru;
-	__le16 mtu;
-	u8 ipg;
-	u8 lpbk;
-	#define PORT_MAC_CFG_RESP_LPBK_NONE			   0x0UL
-	#define PORT_MAC_CFG_RESP_LPBK_LOCAL			   0x1UL
-	#define PORT_MAC_CFG_RESP_LPBK_REMOTE			   0x2UL
-	u8 unused_0;
-	u8 valid;
-};
-
-/* hwrm_port_mac_ptp_qcfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	mru;
+	__le16	mtu;
+	u8	ipg;
+	u8	lpbk;
+	#define PORT_MAC_CFG_RESP_LPBK_NONE   0x0UL
+	#define PORT_MAC_CFG_RESP_LPBK_LOCAL  0x1UL
+	#define PORT_MAC_CFG_RESP_LPBK_REMOTE 0x2UL
+	#define PORT_MAC_CFG_RESP_LPBK_LAST  PORT_MAC_CFG_RESP_LPBK_REMOTE
+	u8	unused_0;
+	u8	valid;
+};
+
+/* hwrm_port_mac_ptp_qcfg_input (size:192b/24B) */
 struct hwrm_port_mac_ptp_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
 };
 
-/* Output (80 bytes) */
+/* hwrm_port_mac_ptp_qcfg_output (size:640b/80B) */
 struct hwrm_port_mac_ptp_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 flags;
-	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_DIRECT_ACCESS	    0x1UL
-	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS	    0x2UL
-	u8 unused_0;
-	__le16 unused_1;
-	__le32 rx_ts_reg_off_lower;
-	__le32 rx_ts_reg_off_upper;
-	__le32 rx_ts_reg_off_seq_id;
-	__le32 rx_ts_reg_off_src_id_0;
-	__le32 rx_ts_reg_off_src_id_1;
-	__le32 rx_ts_reg_off_src_id_2;
-	__le32 rx_ts_reg_off_domain_id;
-	__le32 rx_ts_reg_off_fifo;
-	__le32 rx_ts_reg_off_fifo_adv;
-	__le32 rx_ts_reg_off_granularity;
-	__le32 tx_ts_reg_off_lower;
-	__le32 tx_ts_reg_off_upper;
-	__le32 tx_ts_reg_off_seq_id;
-	__le32 tx_ts_reg_off_fifo;
-	__le32 tx_ts_reg_off_granularity;
-	__le32 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 unused_5;
-	u8 valid;
-};
-
-/* hwrm_port_qstats */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	flags;
+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_DIRECT_ACCESS     0x1UL
+	#define PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS       0x2UL
+	u8	unused_0[3];
+	__le32	rx_ts_reg_off_lower;
+	__le32	rx_ts_reg_off_upper;
+	__le32	rx_ts_reg_off_seq_id;
+	__le32	rx_ts_reg_off_src_id_0;
+	__le32	rx_ts_reg_off_src_id_1;
+	__le32	rx_ts_reg_off_src_id_2;
+	__le32	rx_ts_reg_off_domain_id;
+	__le32	rx_ts_reg_off_fifo;
+	__le32	rx_ts_reg_off_fifo_adv;
+	__le32	rx_ts_reg_off_granularity;
+	__le32	tx_ts_reg_off_lower;
+	__le32	tx_ts_reg_off_upper;
+	__le32	tx_ts_reg_off_seq_id;
+	__le32	tx_ts_reg_off_fifo;
+	__le32	tx_ts_reg_off_granularity;
+	u8	unused_1[7];
+	u8	valid;
+};
+
+/* hwrm_port_qstats_input (size:320b/40B) */
 struct hwrm_port_qstats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2[3];
-	u8 unused_3;
-	__le64 tx_stat_host_addr;
-	__le64 rx_stat_host_addr;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
+	__le64	tx_stat_host_addr;
+	__le64	rx_stat_host_addr;
+};
+
+/* hwrm_port_qstats_output (size:128b/16B) */
 struct hwrm_port_qstats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 tx_stat_size;
-	__le16 rx_stat_size;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_port_lpbk_qstats */
-/* Input (16 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	tx_stat_size;
+	__le16	rx_stat_size;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_port_lpbk_qstats_input (size:128b/16B) */
 struct hwrm_port_lpbk_qstats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (96 bytes) */
+/* hwrm_port_lpbk_qstats_output (size:768b/96B) */
 struct hwrm_port_lpbk_qstats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 lpbk_ucast_frames;
-	__le64 lpbk_mcast_frames;
-	__le64 lpbk_bcast_frames;
-	__le64 lpbk_ucast_bytes;
-	__le64 lpbk_mcast_bytes;
-	__le64 lpbk_bcast_bytes;
-	__le64 tx_stat_discard;
-	__le64 tx_stat_error;
-	__le64 rx_stat_discard;
-	__le64 rx_stat_error;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_port_clr_stats */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	lpbk_ucast_frames;
+	__le64	lpbk_mcast_frames;
+	__le64	lpbk_bcast_frames;
+	__le64	lpbk_ucast_bytes;
+	__le64	lpbk_mcast_bytes;
+	__le64	lpbk_bcast_bytes;
+	__le64	tx_stat_discard;
+	__le64	tx_stat_error;
+	__le64	rx_stat_discard;
+	__le64	rx_stat_error;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_port_clr_stats_input (size:192b/24B) */
 struct hwrm_port_clr_stats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_port_clr_stats_output (size:128b/16B) */
 struct hwrm_port_clr_stats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_port_lpbk_clr_stats */
-/* Input (16 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_port_lpbk_clr_stats_input (size:128b/16B) */
 struct hwrm_port_lpbk_clr_stats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (16 bytes) */
+/* hwrm_port_lpbk_clr_stats_output (size:128b/16B) */
 struct hwrm_port_lpbk_clr_stats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_port_phy_qcaps */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_port_phy_qcaps_input (size:192b/24B) */
 struct hwrm_port_phy_qcaps_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
 };
 
-/* Output (24 bytes) */
+/* hwrm_port_phy_qcaps_output (size:192b/24B) */
 struct hwrm_port_phy_qcaps_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 flags;
-	#define PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED	    0x1UL
-	#define PORT_PHY_QCAPS_RESP_FLAGS_RSVD1_MASK		    0xfeUL
-	#define PORT_PHY_QCAPS_RESP_FLAGS_RSVD1_SFT		    1
-	u8 port_cnt;
-	#define PORT_PHY_QCAPS_RESP_PORT_CNT_UNKNOWN		   0x0UL
-	#define PORT_PHY_QCAPS_RESP_PORT_CNT_1			   0x1UL
-	#define PORT_PHY_QCAPS_RESP_PORT_CNT_2			   0x2UL
-	#define PORT_PHY_QCAPS_RESP_PORT_CNT_3			   0x3UL
-	#define PORT_PHY_QCAPS_RESP_PORT_CNT_4			   0x4UL
-	__le16 supported_speeds_force_mode;
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MBHD 0x1UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MB 0x2UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_1GBHD 0x4UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_1GB 0x8UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_2GB 0x10UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_2_5GB 0x20UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10GB 0x40UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_20GB 0x80UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_25GB 0x100UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_40GB 0x200UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_50GB 0x400UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100GB 0x800UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10MBHD 0x1000UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10MB 0x2000UL
-	__le16 supported_speeds_auto_mode;
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100MBHD 0x1UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100MB 0x2UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_1GBHD 0x4UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_1GB 0x8UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_2GB 0x10UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_2_5GB 0x20UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10GB 0x40UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_20GB 0x80UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_25GB 0x100UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_40GB 0x200UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_50GB 0x400UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100GB 0x800UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10MBHD 0x1000UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10MB 0x2000UL
-	__le16 supported_speeds_eee_mode;
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD1 0x1UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_100MB 0x2UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD2 0x4UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_1GB  0x8UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD3 0x10UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD4 0x20UL
-	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_10GB 0x40UL
-	__le32 tx_lpi_timer_low;
-	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK	    0xffffffUL
-	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_SFT	    0
-	#define PORT_PHY_QCAPS_RESP_RSVD2_MASK			    0xff000000UL
-	#define PORT_PHY_QCAPS_RESP_RSVD2_SFT			    24
-	__le32 valid_tx_lpi_timer_high;
-	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK	    0xffffffUL
-	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_SFT	    0
-	#define PORT_PHY_QCAPS_RESP_VALID_MASK			    0xff000000UL
-	#define PORT_PHY_QCAPS_RESP_VALID_SFT			    24
-};
-
-/* hwrm_port_phy_i2c_read */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	flags;
+	#define PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED     0x1UL
+	#define PORT_PHY_QCAPS_RESP_FLAGS_RSVD1_MASK        0xfeUL
+	#define PORT_PHY_QCAPS_RESP_FLAGS_RSVD1_SFT         1
+	u8	port_cnt;
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_UNKNOWN 0x0UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_1       0x1UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_2       0x2UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_3       0x3UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_4       0x4UL
+	#define PORT_PHY_QCAPS_RESP_PORT_CNT_LAST   PORT_PHY_QCAPS_RESP_PORT_CNT_4
+	__le16	supported_speeds_force_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MBHD     0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100MB       0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_1GBHD       0x4UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_1GB         0x8UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_2GB         0x10UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_2_5GB       0x20UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10GB        0x40UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_20GB        0x80UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_25GB        0x100UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_40GB        0x200UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_50GB        0x400UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_100GB       0x800UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10MBHD      0x1000UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_FORCE_MODE_10MB        0x2000UL
+	__le16	supported_speeds_auto_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100MBHD     0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100MB       0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_1GBHD       0x4UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_1GB         0x8UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_2GB         0x10UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_2_5GB       0x20UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10GB        0x40UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_20GB        0x80UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_25GB        0x100UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_40GB        0x200UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_50GB        0x400UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_100GB       0x800UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10MBHD      0x1000UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_AUTO_MODE_10MB        0x2000UL
+	__le16	supported_speeds_eee_mode;
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD1     0x1UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_100MB     0x2UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD2     0x4UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_1GB       0x8UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD3     0x10UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_RSVD4     0x20UL
+	#define PORT_PHY_QCAPS_RESP_SUPPORTED_SPEEDS_EEE_MODE_10GB      0x40UL
+	__le32	tx_lpi_timer_low;
+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK 0xffffffUL
+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_SFT 0
+	#define PORT_PHY_QCAPS_RESP_RSVD2_MASK           0xff000000UL
+	#define PORT_PHY_QCAPS_RESP_RSVD2_SFT            24
+	__le32	valid_tx_lpi_timer_high;
+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK 0xffffffUL
+	#define PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_SFT 0
+	#define PORT_PHY_QCAPS_RESP_VALID_MASK            0xff000000UL
+	#define PORT_PHY_QCAPS_RESP_VALID_SFT             24
+};
+
+/* hwrm_port_phy_i2c_read_input (size:320b/40B) */
 struct hwrm_port_phy_i2c_read_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	__le32 enables;
-	#define PORT_PHY_I2C_READ_REQ_ENABLES_PAGE_OFFSET	    0x1UL
-	__le16 port_id;
-	u8 i2c_slave_addr;
-	u8 unused_0;
-	__le16 page_number;
-	__le16 page_offset;
-	u8 data_length;
-	u8 unused_1[7];
-};
-
-/* Output (80 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	__le32	enables;
+	#define PORT_PHY_I2C_READ_REQ_ENABLES_PAGE_OFFSET     0x1UL
+	__le16	port_id;
+	u8	i2c_slave_addr;
+	u8	unused_0;
+	__le16	page_number;
+	__le16	page_offset;
+	u8	data_length;
+	u8	unused_1[7];
+};
+
+/* hwrm_port_phy_i2c_read_output (size:640b/80B) */
 struct hwrm_port_phy_i2c_read_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 data[16];
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_port_led_cfg */
-/* Input (64 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	data[16];
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_port_led_cfg_input (size:512b/64B) */
 struct hwrm_port_led_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define PORT_LED_CFG_REQ_ENABLES_LED0_ID		    0x1UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED0_STATE		    0x2UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED0_COLOR		    0x4UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED0_BLINK_ON		    0x8UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED0_BLINK_OFF	    0x10UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED0_GROUP_ID		    0x20UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED1_ID		    0x40UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED1_STATE		    0x80UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED1_COLOR		    0x100UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED1_BLINK_ON		    0x200UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED1_BLINK_OFF	    0x400UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED1_GROUP_ID		    0x800UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED2_ID		    0x1000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED2_STATE		    0x2000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED2_COLOR		    0x4000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED2_BLINK_ON		    0x8000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED2_BLINK_OFF	    0x10000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED2_GROUP_ID		    0x20000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED3_ID		    0x40000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED3_STATE		    0x80000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED3_COLOR		    0x100000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED3_BLINK_ON		    0x200000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED3_BLINK_OFF	    0x400000UL
-	#define PORT_LED_CFG_REQ_ENABLES_LED3_GROUP_ID		    0x800000UL
-	__le16 port_id;
-	u8 num_leds;
-	u8 rsvd;
-	u8 led0_id;
-	u8 led0_state;
-	#define PORT_LED_CFG_REQ_LED0_STATE_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED0_STATE_OFF		   0x1UL
-	#define PORT_LED_CFG_REQ_LED0_STATE_ON			   0x2UL
-	#define PORT_LED_CFG_REQ_LED0_STATE_BLINK		   0x3UL
-	#define PORT_LED_CFG_REQ_LED0_STATE_BLINKALT		   0x4UL
-	u8 led0_color;
-	#define PORT_LED_CFG_REQ_LED0_COLOR_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED0_COLOR_AMBER		   0x1UL
-	#define PORT_LED_CFG_REQ_LED0_COLOR_GREEN		   0x2UL
-	#define PORT_LED_CFG_REQ_LED0_COLOR_GREENAMBER		   0x3UL
-	u8 unused_0;
-	__le16 led0_blink_on;
-	__le16 led0_blink_off;
-	u8 led0_group_id;
-	u8 rsvd0;
-	u8 led1_id;
-	u8 led1_state;
-	#define PORT_LED_CFG_REQ_LED1_STATE_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED1_STATE_OFF		   0x1UL
-	#define PORT_LED_CFG_REQ_LED1_STATE_ON			   0x2UL
-	#define PORT_LED_CFG_REQ_LED1_STATE_BLINK		   0x3UL
-	#define PORT_LED_CFG_REQ_LED1_STATE_BLINKALT		   0x4UL
-	u8 led1_color;
-	#define PORT_LED_CFG_REQ_LED1_COLOR_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED1_COLOR_AMBER		   0x1UL
-	#define PORT_LED_CFG_REQ_LED1_COLOR_GREEN		   0x2UL
-	#define PORT_LED_CFG_REQ_LED1_COLOR_GREENAMBER		   0x3UL
-	u8 unused_1;
-	__le16 led1_blink_on;
-	__le16 led1_blink_off;
-	u8 led1_group_id;
-	u8 rsvd1;
-	u8 led2_id;
-	u8 led2_state;
-	#define PORT_LED_CFG_REQ_LED2_STATE_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED2_STATE_OFF		   0x1UL
-	#define PORT_LED_CFG_REQ_LED2_STATE_ON			   0x2UL
-	#define PORT_LED_CFG_REQ_LED2_STATE_BLINK		   0x3UL
-	#define PORT_LED_CFG_REQ_LED2_STATE_BLINKALT		   0x4UL
-	u8 led2_color;
-	#define PORT_LED_CFG_REQ_LED2_COLOR_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED2_COLOR_AMBER		   0x1UL
-	#define PORT_LED_CFG_REQ_LED2_COLOR_GREEN		   0x2UL
-	#define PORT_LED_CFG_REQ_LED2_COLOR_GREENAMBER		   0x3UL
-	u8 unused_2;
-	__le16 led2_blink_on;
-	__le16 led2_blink_off;
-	u8 led2_group_id;
-	u8 rsvd2;
-	u8 led3_id;
-	u8 led3_state;
-	#define PORT_LED_CFG_REQ_LED3_STATE_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED3_STATE_OFF		   0x1UL
-	#define PORT_LED_CFG_REQ_LED3_STATE_ON			   0x2UL
-	#define PORT_LED_CFG_REQ_LED3_STATE_BLINK		   0x3UL
-	#define PORT_LED_CFG_REQ_LED3_STATE_BLINKALT		   0x4UL
-	u8 led3_color;
-	#define PORT_LED_CFG_REQ_LED3_COLOR_DEFAULT		   0x0UL
-	#define PORT_LED_CFG_REQ_LED3_COLOR_AMBER		   0x1UL
-	#define PORT_LED_CFG_REQ_LED3_COLOR_GREEN		   0x2UL
-	#define PORT_LED_CFG_REQ_LED3_COLOR_GREENAMBER		   0x3UL
-	u8 unused_3;
-	__le16 led3_blink_on;
-	__le16 led3_blink_off;
-	u8 led3_group_id;
-	u8 rsvd3;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define PORT_LED_CFG_REQ_ENABLES_LED0_ID            0x1UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED0_STATE         0x2UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED0_COLOR         0x4UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED0_BLINK_ON      0x8UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED0_BLINK_OFF     0x10UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED0_GROUP_ID      0x20UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED1_ID            0x40UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED1_STATE         0x80UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED1_COLOR         0x100UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED1_BLINK_ON      0x200UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED1_BLINK_OFF     0x400UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED1_GROUP_ID      0x800UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED2_ID            0x1000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED2_STATE         0x2000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED2_COLOR         0x4000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED2_BLINK_ON      0x8000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED2_BLINK_OFF     0x10000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED2_GROUP_ID      0x20000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED3_ID            0x40000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED3_STATE         0x80000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED3_COLOR         0x100000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED3_BLINK_ON      0x200000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED3_BLINK_OFF     0x400000UL
+	#define PORT_LED_CFG_REQ_ENABLES_LED3_GROUP_ID      0x800000UL
+	__le16	port_id;
+	u8	num_leds;
+	u8	rsvd;
+	u8	led0_id;
+	u8	led0_state;
+	#define PORT_LED_CFG_REQ_LED0_STATE_DEFAULT  0x0UL
+	#define PORT_LED_CFG_REQ_LED0_STATE_OFF      0x1UL
+	#define PORT_LED_CFG_REQ_LED0_STATE_ON       0x2UL
+	#define PORT_LED_CFG_REQ_LED0_STATE_BLINK    0x3UL
+	#define PORT_LED_CFG_REQ_LED0_STATE_BLINKALT 0x4UL
+	#define PORT_LED_CFG_REQ_LED0_STATE_LAST    PORT_LED_CFG_REQ_LED0_STATE_BLINKALT
+	u8	led0_color;
+	#define PORT_LED_CFG_REQ_LED0_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_CFG_REQ_LED0_COLOR_AMBER      0x1UL
+	#define PORT_LED_CFG_REQ_LED0_COLOR_GREEN      0x2UL
+	#define PORT_LED_CFG_REQ_LED0_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_CFG_REQ_LED0_COLOR_LAST      PORT_LED_CFG_REQ_LED0_COLOR_GREENAMBER
+	u8	unused_0;
+	__le16	led0_blink_on;
+	__le16	led0_blink_off;
+	u8	led0_group_id;
+	u8	rsvd0;
+	u8	led1_id;
+	u8	led1_state;
+	#define PORT_LED_CFG_REQ_LED1_STATE_DEFAULT  0x0UL
+	#define PORT_LED_CFG_REQ_LED1_STATE_OFF      0x1UL
+	#define PORT_LED_CFG_REQ_LED1_STATE_ON       0x2UL
+	#define PORT_LED_CFG_REQ_LED1_STATE_BLINK    0x3UL
+	#define PORT_LED_CFG_REQ_LED1_STATE_BLINKALT 0x4UL
+	#define PORT_LED_CFG_REQ_LED1_STATE_LAST    PORT_LED_CFG_REQ_LED1_STATE_BLINKALT
+	u8	led1_color;
+	#define PORT_LED_CFG_REQ_LED1_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_CFG_REQ_LED1_COLOR_AMBER      0x1UL
+	#define PORT_LED_CFG_REQ_LED1_COLOR_GREEN      0x2UL
+	#define PORT_LED_CFG_REQ_LED1_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_CFG_REQ_LED1_COLOR_LAST      PORT_LED_CFG_REQ_LED1_COLOR_GREENAMBER
+	u8	unused_1;
+	__le16	led1_blink_on;
+	__le16	led1_blink_off;
+	u8	led1_group_id;
+	u8	rsvd1;
+	u8	led2_id;
+	u8	led2_state;
+	#define PORT_LED_CFG_REQ_LED2_STATE_DEFAULT  0x0UL
+	#define PORT_LED_CFG_REQ_LED2_STATE_OFF      0x1UL
+	#define PORT_LED_CFG_REQ_LED2_STATE_ON       0x2UL
+	#define PORT_LED_CFG_REQ_LED2_STATE_BLINK    0x3UL
+	#define PORT_LED_CFG_REQ_LED2_STATE_BLINKALT 0x4UL
+	#define PORT_LED_CFG_REQ_LED2_STATE_LAST    PORT_LED_CFG_REQ_LED2_STATE_BLINKALT
+	u8	led2_color;
+	#define PORT_LED_CFG_REQ_LED2_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_CFG_REQ_LED2_COLOR_AMBER      0x1UL
+	#define PORT_LED_CFG_REQ_LED2_COLOR_GREEN      0x2UL
+	#define PORT_LED_CFG_REQ_LED2_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_CFG_REQ_LED2_COLOR_LAST      PORT_LED_CFG_REQ_LED2_COLOR_GREENAMBER
+	u8	unused_2;
+	__le16	led2_blink_on;
+	__le16	led2_blink_off;
+	u8	led2_group_id;
+	u8	rsvd2;
+	u8	led3_id;
+	u8	led3_state;
+	#define PORT_LED_CFG_REQ_LED3_STATE_DEFAULT  0x0UL
+	#define PORT_LED_CFG_REQ_LED3_STATE_OFF      0x1UL
+	#define PORT_LED_CFG_REQ_LED3_STATE_ON       0x2UL
+	#define PORT_LED_CFG_REQ_LED3_STATE_BLINK    0x3UL
+	#define PORT_LED_CFG_REQ_LED3_STATE_BLINKALT 0x4UL
+	#define PORT_LED_CFG_REQ_LED3_STATE_LAST    PORT_LED_CFG_REQ_LED3_STATE_BLINKALT
+	u8	led3_color;
+	#define PORT_LED_CFG_REQ_LED3_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_CFG_REQ_LED3_COLOR_AMBER      0x1UL
+	#define PORT_LED_CFG_REQ_LED3_COLOR_GREEN      0x2UL
+	#define PORT_LED_CFG_REQ_LED3_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_CFG_REQ_LED3_COLOR_LAST      PORT_LED_CFG_REQ_LED3_COLOR_GREENAMBER
+	u8	unused_3;
+	__le16	led3_blink_on;
+	__le16	led3_blink_off;
+	u8	led3_group_id;
+	u8	rsvd3;
+};
+
+/* hwrm_port_led_cfg_output (size:128b/16B) */
 struct hwrm_port_led_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_port_led_qcaps */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_port_led_qcfg_input (size:192b/24B) */
+struct hwrm_port_led_qcfg_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
+};
+
+/* hwrm_port_led_qcfg_output (size:448b/56B) */
+struct hwrm_port_led_qcfg_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	num_leds;
+	u8	led0_id;
+	u8	led0_type;
+	#define PORT_LED_QCFG_RESP_LED0_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCFG_RESP_LED0_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCFG_RESP_LED0_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCFG_RESP_LED0_TYPE_LAST    PORT_LED_QCFG_RESP_LED0_TYPE_INVALID
+	u8	led0_state;
+	#define PORT_LED_QCFG_RESP_LED0_STATE_DEFAULT  0x0UL
+	#define PORT_LED_QCFG_RESP_LED0_STATE_OFF      0x1UL
+	#define PORT_LED_QCFG_RESP_LED0_STATE_ON       0x2UL
+	#define PORT_LED_QCFG_RESP_LED0_STATE_BLINK    0x3UL
+	#define PORT_LED_QCFG_RESP_LED0_STATE_BLINKALT 0x4UL
+	#define PORT_LED_QCFG_RESP_LED0_STATE_LAST    PORT_LED_QCFG_RESP_LED0_STATE_BLINKALT
+	u8	led0_color;
+	#define PORT_LED_QCFG_RESP_LED0_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_QCFG_RESP_LED0_COLOR_AMBER      0x1UL
+	#define PORT_LED_QCFG_RESP_LED0_COLOR_GREEN      0x2UL
+	#define PORT_LED_QCFG_RESP_LED0_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_QCFG_RESP_LED0_COLOR_LAST      PORT_LED_QCFG_RESP_LED0_COLOR_GREENAMBER
+	u8	unused_0;
+	__le16	led0_blink_on;
+	__le16	led0_blink_off;
+	u8	led0_group_id;
+	u8	led1_id;
+	u8	led1_type;
+	#define PORT_LED_QCFG_RESP_LED1_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCFG_RESP_LED1_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCFG_RESP_LED1_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCFG_RESP_LED1_TYPE_LAST    PORT_LED_QCFG_RESP_LED1_TYPE_INVALID
+	u8	led1_state;
+	#define PORT_LED_QCFG_RESP_LED1_STATE_DEFAULT  0x0UL
+	#define PORT_LED_QCFG_RESP_LED1_STATE_OFF      0x1UL
+	#define PORT_LED_QCFG_RESP_LED1_STATE_ON       0x2UL
+	#define PORT_LED_QCFG_RESP_LED1_STATE_BLINK    0x3UL
+	#define PORT_LED_QCFG_RESP_LED1_STATE_BLINKALT 0x4UL
+	#define PORT_LED_QCFG_RESP_LED1_STATE_LAST    PORT_LED_QCFG_RESP_LED1_STATE_BLINKALT
+	u8	led1_color;
+	#define PORT_LED_QCFG_RESP_LED1_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_QCFG_RESP_LED1_COLOR_AMBER      0x1UL
+	#define PORT_LED_QCFG_RESP_LED1_COLOR_GREEN      0x2UL
+	#define PORT_LED_QCFG_RESP_LED1_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_QCFG_RESP_LED1_COLOR_LAST      PORT_LED_QCFG_RESP_LED1_COLOR_GREENAMBER
+	u8	unused_1;
+	__le16	led1_blink_on;
+	__le16	led1_blink_off;
+	u8	led1_group_id;
+	u8	led2_id;
+	u8	led2_type;
+	#define PORT_LED_QCFG_RESP_LED2_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCFG_RESP_LED2_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCFG_RESP_LED2_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCFG_RESP_LED2_TYPE_LAST    PORT_LED_QCFG_RESP_LED2_TYPE_INVALID
+	u8	led2_state;
+	#define PORT_LED_QCFG_RESP_LED2_STATE_DEFAULT  0x0UL
+	#define PORT_LED_QCFG_RESP_LED2_STATE_OFF      0x1UL
+	#define PORT_LED_QCFG_RESP_LED2_STATE_ON       0x2UL
+	#define PORT_LED_QCFG_RESP_LED2_STATE_BLINK    0x3UL
+	#define PORT_LED_QCFG_RESP_LED2_STATE_BLINKALT 0x4UL
+	#define PORT_LED_QCFG_RESP_LED2_STATE_LAST    PORT_LED_QCFG_RESP_LED2_STATE_BLINKALT
+	u8	led2_color;
+	#define PORT_LED_QCFG_RESP_LED2_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_QCFG_RESP_LED2_COLOR_AMBER      0x1UL
+	#define PORT_LED_QCFG_RESP_LED2_COLOR_GREEN      0x2UL
+	#define PORT_LED_QCFG_RESP_LED2_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_QCFG_RESP_LED2_COLOR_LAST      PORT_LED_QCFG_RESP_LED2_COLOR_GREENAMBER
+	u8	unused_2;
+	__le16	led2_blink_on;
+	__le16	led2_blink_off;
+	u8	led2_group_id;
+	u8	led3_id;
+	u8	led3_type;
+	#define PORT_LED_QCFG_RESP_LED3_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCFG_RESP_LED3_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCFG_RESP_LED3_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCFG_RESP_LED3_TYPE_LAST    PORT_LED_QCFG_RESP_LED3_TYPE_INVALID
+	u8	led3_state;
+	#define PORT_LED_QCFG_RESP_LED3_STATE_DEFAULT  0x0UL
+	#define PORT_LED_QCFG_RESP_LED3_STATE_OFF      0x1UL
+	#define PORT_LED_QCFG_RESP_LED3_STATE_ON       0x2UL
+	#define PORT_LED_QCFG_RESP_LED3_STATE_BLINK    0x3UL
+	#define PORT_LED_QCFG_RESP_LED3_STATE_BLINKALT 0x4UL
+	#define PORT_LED_QCFG_RESP_LED3_STATE_LAST    PORT_LED_QCFG_RESP_LED3_STATE_BLINKALT
+	u8	led3_color;
+	#define PORT_LED_QCFG_RESP_LED3_COLOR_DEFAULT    0x0UL
+	#define PORT_LED_QCFG_RESP_LED3_COLOR_AMBER      0x1UL
+	#define PORT_LED_QCFG_RESP_LED3_COLOR_GREEN      0x2UL
+	#define PORT_LED_QCFG_RESP_LED3_COLOR_GREENAMBER 0x3UL
+	#define PORT_LED_QCFG_RESP_LED3_COLOR_LAST      PORT_LED_QCFG_RESP_LED3_COLOR_GREENAMBER
+	u8	unused_3;
+	__le16	led3_blink_on;
+	__le16	led3_blink_off;
+	u8	led3_group_id;
+	u8	unused_4[6];
+	u8	valid;
+};
+
+/* hwrm_port_led_qcaps_input (size:192b/24B) */
 struct hwrm_port_led_qcaps_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
 };
 
-/* Output (48 bytes) */
+/* hwrm_port_led_qcaps_output (size:384b/48B) */
 struct hwrm_port_led_qcaps_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 num_leds;
-	u8 unused_0[3];
-	u8 led0_id;
-	u8 led0_type;
-	#define PORT_LED_QCAPS_RESP_LED0_TYPE_SPEED		   0x0UL
-	#define PORT_LED_QCAPS_RESP_LED0_TYPE_ACTIVITY		   0x1UL
-	#define PORT_LED_QCAPS_RESP_LED0_TYPE_INVALID		   0xffUL
-	u8 led0_group_id;
-	u8 unused_1;
-	__le16 led0_state_caps;
-	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_ENABLED	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_OFF_SUPPORTED  0x2UL
-	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_ON_SUPPORTED   0x4UL
-	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_BLINK_SUPPORTED 0x8UL
-	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_BLINK_ALT_SUPPORTED 0x10UL
-	__le16 led0_color_caps;
-	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_RSVD	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_AMBER_SUPPORTED 0x2UL
-	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_GREEN_SUPPORTED 0x4UL
-	u8 led1_id;
-	u8 led1_type;
-	#define PORT_LED_QCAPS_RESP_LED1_TYPE_SPEED		   0x0UL
-	#define PORT_LED_QCAPS_RESP_LED1_TYPE_ACTIVITY		   0x1UL
-	#define PORT_LED_QCAPS_RESP_LED1_TYPE_INVALID		   0xffUL
-	u8 led1_group_id;
-	u8 unused_2;
-	__le16 led1_state_caps;
-	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_ENABLED	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_OFF_SUPPORTED  0x2UL
-	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_ON_SUPPORTED   0x4UL
-	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_BLINK_SUPPORTED 0x8UL
-	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_BLINK_ALT_SUPPORTED 0x10UL
-	__le16 led1_color_caps;
-	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_RSVD	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_AMBER_SUPPORTED 0x2UL
-	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_GREEN_SUPPORTED 0x4UL
-	u8 led2_id;
-	u8 led2_type;
-	#define PORT_LED_QCAPS_RESP_LED2_TYPE_SPEED		   0x0UL
-	#define PORT_LED_QCAPS_RESP_LED2_TYPE_ACTIVITY		   0x1UL
-	#define PORT_LED_QCAPS_RESP_LED2_TYPE_INVALID		   0xffUL
-	u8 led2_group_id;
-	u8 unused_3;
-	__le16 led2_state_caps;
-	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_ENABLED	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_OFF_SUPPORTED  0x2UL
-	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_ON_SUPPORTED   0x4UL
-	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_BLINK_SUPPORTED 0x8UL
-	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_BLINK_ALT_SUPPORTED 0x10UL
-	__le16 led2_color_caps;
-	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_RSVD	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_AMBER_SUPPORTED 0x2UL
-	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_GREEN_SUPPORTED 0x4UL
-	u8 led3_id;
-	u8 led3_type;
-	#define PORT_LED_QCAPS_RESP_LED3_TYPE_SPEED		   0x0UL
-	#define PORT_LED_QCAPS_RESP_LED3_TYPE_ACTIVITY		   0x1UL
-	#define PORT_LED_QCAPS_RESP_LED3_TYPE_INVALID		   0xffUL
-	u8 led3_group_id;
-	u8 unused_4;
-	__le16 led3_state_caps;
-	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_ENABLED	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_OFF_SUPPORTED  0x2UL
-	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_ON_SUPPORTED   0x4UL
-	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_BLINK_SUPPORTED 0x8UL
-	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_BLINK_ALT_SUPPORTED 0x10UL
-	__le16 led3_color_caps;
-	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_RSVD	    0x1UL
-	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_AMBER_SUPPORTED 0x2UL
-	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_GREEN_SUPPORTED 0x4UL
-	u8 unused_5;
-	u8 unused_6;
-	u8 unused_7;
-	u8 valid;
-};
-
-/* hwrm_queue_qportcfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	num_leds;
+	u8	unused[3];
+	u8	led0_id;
+	u8	led0_type;
+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCAPS_RESP_LED0_TYPE_LAST    PORT_LED_QCAPS_RESP_LED0_TYPE_INVALID
+	u8	led0_group_id;
+	u8	unused_0;
+	__le16	led0_state_caps;
+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_ENABLED                 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_OFF_SUPPORTED           0x2UL
+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_ON_SUPPORTED            0x4UL
+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_BLINK_SUPPORTED         0x8UL
+	#define PORT_LED_QCAPS_RESP_LED0_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
+	__le16	led0_color_caps;
+	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_RSVD                0x1UL
+	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_AMBER_SUPPORTED     0x2UL
+	#define PORT_LED_QCAPS_RESP_LED0_COLOR_CAPS_GREEN_SUPPORTED     0x4UL
+	u8	led1_id;
+	u8	led1_type;
+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCAPS_RESP_LED1_TYPE_LAST    PORT_LED_QCAPS_RESP_LED1_TYPE_INVALID
+	u8	led1_group_id;
+	u8	unused_1;
+	__le16	led1_state_caps;
+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_ENABLED                 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_OFF_SUPPORTED           0x2UL
+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_ON_SUPPORTED            0x4UL
+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_BLINK_SUPPORTED         0x8UL
+	#define PORT_LED_QCAPS_RESP_LED1_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
+	__le16	led1_color_caps;
+	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_RSVD                0x1UL
+	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_AMBER_SUPPORTED     0x2UL
+	#define PORT_LED_QCAPS_RESP_LED1_COLOR_CAPS_GREEN_SUPPORTED     0x4UL
+	u8	led2_id;
+	u8	led2_type;
+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCAPS_RESP_LED2_TYPE_LAST    PORT_LED_QCAPS_RESP_LED2_TYPE_INVALID
+	u8	led2_group_id;
+	u8	unused_2;
+	__le16	led2_state_caps;
+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_ENABLED                 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_OFF_SUPPORTED           0x2UL
+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_ON_SUPPORTED            0x4UL
+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_BLINK_SUPPORTED         0x8UL
+	#define PORT_LED_QCAPS_RESP_LED2_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
+	__le16	led2_color_caps;
+	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_RSVD                0x1UL
+	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_AMBER_SUPPORTED     0x2UL
+	#define PORT_LED_QCAPS_RESP_LED2_COLOR_CAPS_GREEN_SUPPORTED     0x4UL
+	u8	led3_id;
+	u8	led3_type;
+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_SPEED    0x0UL
+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_ACTIVITY 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_INVALID  0xffUL
+	#define PORT_LED_QCAPS_RESP_LED3_TYPE_LAST    PORT_LED_QCAPS_RESP_LED3_TYPE_INVALID
+	u8	led3_group_id;
+	u8	unused_3;
+	__le16	led3_state_caps;
+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_ENABLED                 0x1UL
+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_OFF_SUPPORTED           0x2UL
+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_ON_SUPPORTED            0x4UL
+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_BLINK_SUPPORTED         0x8UL
+	#define PORT_LED_QCAPS_RESP_LED3_STATE_CAPS_BLINK_ALT_SUPPORTED     0x10UL
+	__le16	led3_color_caps;
+	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_RSVD                0x1UL
+	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_AMBER_SUPPORTED     0x2UL
+	#define PORT_LED_QCAPS_RESP_LED3_COLOR_CAPS_GREEN_SUPPORTED     0x4UL
+	u8	unused_4[3];
+	u8	valid;
+};
+
+/* hwrm_queue_qportcfg_input (size:192b/24B) */
 struct hwrm_queue_qportcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH			    0x1UL
-	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_TX		   0x0UL
-	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX		   0x1UL
-	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_LAST    QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX
-	__le16 port_id;
-	__le16 unused_0;
-};
-
-/* Output (32 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH     0x1UL
+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_TX    0x0UL
+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX    0x1UL
+	#define QUEUE_QPORTCFG_REQ_FLAGS_PATH_LAST QUEUE_QPORTCFG_REQ_FLAGS_PATH_RX
+	__le16	port_id;
+	u8	unused_0[2];
+};
+
+/* hwrm_queue_qportcfg_output (size:256b/32B) */
 struct hwrm_queue_qportcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 max_configurable_queues;
-	u8 max_configurable_lossless_queues;
-	u8 queue_cfg_allowed;
-	u8 queue_cfg_info;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG	    0x1UL
-	u8 queue_pfcenable_cfg_allowed;
-	u8 queue_pri2cos_cfg_allowed;
-	u8 queue_cos2bw_cfg_allowed;
-	u8 queue_id0;
-	u8 queue_id0_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSY 0x0UL
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	max_configurable_queues;
+	u8	max_configurable_lossless_queues;
+	u8	queue_cfg_allowed;
+	u8	queue_cfg_info;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG     0x1UL
+	u8	queue_pfcenable_cfg_allowed;
+	u8	queue_pri2cos_cfg_allowed;
+	u8	queue_cos2bw_cfg_allowed;
+	u8	queue_id0;
+	u8	queue_id0_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 queue_id1;
-	u8 queue_id1_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID0_SERVICE_PROFILE_UNKNOWN
+	u8	queue_id1;
+	u8	queue_id1_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 queue_id2;
-	u8 queue_id2_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID1_SERVICE_PROFILE_UNKNOWN
+	u8	queue_id2;
+	u8	queue_id2_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 queue_id3;
-	u8 queue_id3_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID2_SERVICE_PROFILE_UNKNOWN
+	u8	queue_id3;
+	u8	queue_id3_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 queue_id4;
-	u8 queue_id4_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID3_SERVICE_PROFILE_UNKNOWN
+	u8	queue_id4;
+	u8	queue_id4_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 queue_id5;
-	u8 queue_id5_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID4_SERVICE_PROFILE_UNKNOWN
+	u8	queue_id5;
+	u8	queue_id5_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 queue_id6;
-	u8 queue_id6_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID5_SERVICE_PROFILE_UNKNOWN
+	u8	queue_id6;
+	u8	queue_id6_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 queue_id7;
-	u8 queue_id7_service_profile;
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSY 0x0UL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID6_SERVICE_PROFILE_UNKNOWN
+	u8	queue_id7;
+	u8	queue_id7_service_profile;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSY    0x0UL
 	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LOSSLESS 0x1UL
-	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN 0xffUL
-	u8 valid;
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_LAST    QUEUE_QPORTCFG_RESP_QUEUE_ID7_SERVICE_PROFILE_UNKNOWN
+	u8	valid;
 };
 
-/* hwrm_queue_cfg */
-/* Input (40 bytes) */
+/* hwrm_queue_cfg_input (size:320b/40B) */
 struct hwrm_queue_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define QUEUE_CFG_REQ_FLAGS_PATH_MASK			    0x3UL
-	#define QUEUE_CFG_REQ_FLAGS_PATH_SFT			    0
-	#define QUEUE_CFG_REQ_FLAGS_PATH_TX			   0x0UL
-	#define QUEUE_CFG_REQ_FLAGS_PATH_RX			   0x1UL
-	#define QUEUE_CFG_REQ_FLAGS_PATH_BIDIR			   0x2UL
-	#define QUEUE_CFG_REQ_FLAGS_PATH_LAST    QUEUE_CFG_REQ_FLAGS_PATH_BIDIR
-	__le32 enables;
-	#define QUEUE_CFG_REQ_ENABLES_DFLT_LEN			    0x1UL
-	#define QUEUE_CFG_REQ_ENABLES_SERVICE_PROFILE		    0x2UL
-	__le32 queue_id;
-	__le32 dflt_len;
-	u8 service_profile;
-	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSY		   0x0UL
-	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSLESS		   0x1UL
-	#define QUEUE_CFG_REQ_SERVICE_PROFILE_UNKNOWN		   0xffUL
-	u8 unused_0[7];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define QUEUE_CFG_REQ_FLAGS_PATH_MASK 0x3UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_SFT  0
+	#define QUEUE_CFG_REQ_FLAGS_PATH_TX     0x0UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_RX     0x1UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_BIDIR  0x2UL
+	#define QUEUE_CFG_REQ_FLAGS_PATH_LAST  QUEUE_CFG_REQ_FLAGS_PATH_BIDIR
+	__le32	enables;
+	#define QUEUE_CFG_REQ_ENABLES_DFLT_LEN            0x1UL
+	#define QUEUE_CFG_REQ_ENABLES_SERVICE_PROFILE     0x2UL
+	__le32	queue_id;
+	__le32	dflt_len;
+	u8	service_profile;
+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSY    0x0UL
+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LOSSLESS 0x1UL
+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_UNKNOWN  0xffUL
+	#define QUEUE_CFG_REQ_SERVICE_PROFILE_LAST    QUEUE_CFG_REQ_SERVICE_PROFILE_UNKNOWN
+	u8	unused_0[7];
+};
+
+/* hwrm_queue_cfg_output (size:128b/16B) */
 struct hwrm_queue_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_queue_pfcenable_qcfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_queue_pfcenable_qcfg_input (size:192b/24B) */
 struct hwrm_queue_pfcenable_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_queue_pfcenable_qcfg_output (size:128b/16B) */
 struct hwrm_queue_pfcenable_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 flags;
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI0_PFC_ENABLED   0x1UL
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI1_PFC_ENABLED   0x2UL
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI2_PFC_ENABLED   0x4UL
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI3_PFC_ENABLED   0x8UL
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI4_PFC_ENABLED   0x10UL
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI5_PFC_ENABLED   0x20UL
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI6_PFC_ENABLED   0x40UL
-	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI7_PFC_ENABLED   0x80UL
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_queue_pfcenable_cfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	flags;
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI0_PFC_ENABLED     0x1UL
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI1_PFC_ENABLED     0x2UL
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI2_PFC_ENABLED     0x4UL
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI3_PFC_ENABLED     0x8UL
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI4_PFC_ENABLED     0x10UL
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI5_PFC_ENABLED     0x20UL
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI6_PFC_ENABLED     0x40UL
+	#define QUEUE_PFCENABLE_QCFG_RESP_FLAGS_PRI7_PFC_ENABLED     0x80UL
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_queue_pfcenable_cfg_input (size:192b/24B) */
 struct hwrm_queue_pfcenable_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
 	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI0_PFC_ENABLED     0x1UL
 	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI1_PFC_ENABLED     0x2UL
 	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI2_PFC_ENABLED     0x4UL
@@ -2440,1729 +2703,1664 @@ struct hwrm_queue_pfcenable_cfg_input {
 	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI5_PFC_ENABLED     0x20UL
 	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI6_PFC_ENABLED     0x40UL
 	#define QUEUE_PFCENABLE_CFG_REQ_FLAGS_PRI7_PFC_ENABLED     0x80UL
-	__le16 port_id;
-	__le16 unused_0;
+	__le16	port_id;
+	u8	unused_0[2];
 };
 
-/* Output (16 bytes) */
+/* hwrm_queue_pfcenable_cfg_output (size:128b/16B) */
 struct hwrm_queue_pfcenable_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_queue_pri2cos_qcfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_queue_pri2cos_qcfg_input (size:192b/24B) */
 struct hwrm_queue_pri2cos_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH		    0x1UL
-	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_TX		   (0x0UL << 0)
-	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_RX		   (0x1UL << 0)
-	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_LAST    QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_RX
-	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN		    0x2UL
-	u8 port_id;
-	u8 unused_0[3];
-};
-
-/* Output (24 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH      0x1UL
+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_TX     0x0UL
+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_RX     0x1UL
+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_LAST  QUEUE_PRI2COS_QCFG_REQ_FLAGS_PATH_RX
+	#define QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN     0x2UL
+	u8	port_id;
+	u8	unused_0[3];
+};
+
+/* hwrm_queue_pri2cos_qcfg_output (size:192b/24B) */
 struct hwrm_queue_pri2cos_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 pri0_cos_queue_id;
-	u8 pri1_cos_queue_id;
-	u8 pri2_cos_queue_id;
-	u8 pri3_cos_queue_id;
-	u8 pri4_cos_queue_id;
-	u8 pri5_cos_queue_id;
-	u8 pri6_cos_queue_id;
-	u8 pri7_cos_queue_id;
-	u8 queue_cfg_info;
-	#define QUEUE_PRI2COS_QCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG    0x1UL
-	u8 unused_0;
-	__le16 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_queue_pri2cos_cfg */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	pri0_cos_queue_id;
+	u8	pri1_cos_queue_id;
+	u8	pri2_cos_queue_id;
+	u8	pri3_cos_queue_id;
+	u8	pri4_cos_queue_id;
+	u8	pri5_cos_queue_id;
+	u8	pri6_cos_queue_id;
+	u8	pri7_cos_queue_id;
+	u8	queue_cfg_info;
+	#define QUEUE_PRI2COS_QCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG     0x1UL
+	u8	unused_0[6];
+	u8	valid;
+};
+
+/* hwrm_queue_pri2cos_cfg_input (size:320b/40B) */
 struct hwrm_queue_pri2cos_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_MASK		    0x3UL
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_SFT		    0
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_TX		   (0x0UL << 0)
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_RX		   (0x1UL << 0)
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR		   (0x2UL << 0)
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_LAST    QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR
-	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_IVLAN		    0x4UL
-	__le32 enables;
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI0_COS_QUEUE_ID    0x1UL
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI1_COS_QUEUE_ID    0x2UL
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI2_COS_QUEUE_ID    0x4UL
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI3_COS_QUEUE_ID    0x8UL
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI4_COS_QUEUE_ID    0x10UL
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI5_COS_QUEUE_ID    0x20UL
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI6_COS_QUEUE_ID    0x40UL
-	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI7_COS_QUEUE_ID    0x80UL
-	u8 port_id;
-	u8 pri0_cos_queue_id;
-	u8 pri1_cos_queue_id;
-	u8 pri2_cos_queue_id;
-	u8 pri3_cos_queue_id;
-	u8 pri4_cos_queue_id;
-	u8 pri5_cos_queue_id;
-	u8 pri6_cos_queue_id;
-	u8 pri7_cos_queue_id;
-	u8 unused_0[7];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_MASK 0x3UL
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_SFT  0
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_TX     0x0UL
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_RX     0x1UL
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR  0x2UL
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_LAST  QUEUE_PRI2COS_CFG_REQ_FLAGS_PATH_BIDIR
+	#define QUEUE_PRI2COS_CFG_REQ_FLAGS_IVLAN     0x4UL
+	__le32	enables;
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI0_COS_QUEUE_ID     0x1UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI1_COS_QUEUE_ID     0x2UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI2_COS_QUEUE_ID     0x4UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI3_COS_QUEUE_ID     0x8UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI4_COS_QUEUE_ID     0x10UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI5_COS_QUEUE_ID     0x20UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI6_COS_QUEUE_ID     0x40UL
+	#define QUEUE_PRI2COS_CFG_REQ_ENABLES_PRI7_COS_QUEUE_ID     0x80UL
+	u8	port_id;
+	u8	pri0_cos_queue_id;
+	u8	pri1_cos_queue_id;
+	u8	pri2_cos_queue_id;
+	u8	pri3_cos_queue_id;
+	u8	pri4_cos_queue_id;
+	u8	pri5_cos_queue_id;
+	u8	pri6_cos_queue_id;
+	u8	pri7_cos_queue_id;
+	u8	unused_0[7];
+};
+
+/* hwrm_queue_pri2cos_cfg_output (size:128b/16B) */
 struct hwrm_queue_pri2cos_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_queue_cos2bw_qcfg */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_queue_cos2bw_qcfg_input (size:192b/24B) */
 struct hwrm_queue_cos2bw_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
 };
 
-/* Output (112 bytes) */
+/* hwrm_queue_cos2bw_qcfg_output (size:896b/112B) */
 struct hwrm_queue_cos2bw_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 queue_id0;
-	u8 unused_0;
-	__le16 unused_1;
-	__le32 queue_id0_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id0_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id0_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_ETS   0x1UL
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	queue_id0;
+	u8	unused_0;
+	__le16	unused_1;
+	__le32	queue_id0_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id0_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id0_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id0_pri_lvl;
-	u8 queue_id0_bw_weight;
-	u8 queue_id1;
-	__le32 queue_id1_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id1_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id1_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_TSA_ASSIGN_ETS   0x1UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id0_pri_lvl;
+	u8	queue_id0_bw_weight;
+	u8	queue_id1;
+	__le32	queue_id1_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id1_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id1_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id1_pri_lvl;
-	u8 queue_id1_bw_weight;
-	u8 queue_id2;
-	__le32 queue_id2_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id2_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id2_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_TSA_ASSIGN_ETS   0x1UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID1_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id1_pri_lvl;
+	u8	queue_id1_bw_weight;
+	u8	queue_id2;
+	__le32	queue_id2_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id2_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id2_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id2_pri_lvl;
-	u8 queue_id2_bw_weight;
-	u8 queue_id3;
-	__le32 queue_id3_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id3_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id3_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_TSA_ASSIGN_ETS   0x1UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID2_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id2_pri_lvl;
+	u8	queue_id2_bw_weight;
+	u8	queue_id3;
+	__le32	queue_id3_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id3_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id3_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id3_pri_lvl;
-	u8 queue_id3_bw_weight;
-	u8 queue_id4;
-	__le32 queue_id4_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id4_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id4_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_TSA_ASSIGN_ETS   0x1UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID3_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id3_pri_lvl;
+	u8	queue_id3_bw_weight;
+	u8	queue_id4;
+	__le32	queue_id4_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id4_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id4_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id4_pri_lvl;
-	u8 queue_id4_bw_weight;
-	u8 queue_id5;
-	__le32 queue_id5_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id5_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id5_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_TSA_ASSIGN_ETS   0x1UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID4_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id4_pri_lvl;
+	u8	queue_id4_bw_weight;
+	u8	queue_id5;
+	__le32	queue_id5_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id5_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id5_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id5_pri_lvl;
-	u8 queue_id5_bw_weight;
-	u8 queue_id6;
-	__le32 queue_id6_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id6_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id6_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_TSA_ASSIGN_ETS   0x1UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID5_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id5_pri_lvl;
+	u8	queue_id5_bw_weight;
+	u8	queue_id6;
+	__le32	queue_id6_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id6_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id6_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id6_pri_lvl;
-	u8 queue_id6_bw_weight;
-	u8 queue_id7;
-	__le32 queue_id7_min_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id7_max_bw;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE      0x10000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_BITS (0x0UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id7_tsa_assign;
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_TSA_ASSIGN_SP    0x0UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_TSA_ASSIGN_ETS   0x1UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID6_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id6_pri_lvl;
+	u8	queue_id6_bw_weight;
+	u8	queue_id7;
+	__le32	queue_id7_min_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id7_max_bw;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id7_tsa_assign;
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id7_pri_lvl;
-	u8 queue_id7_bw_weight;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 unused_5;
-	u8 valid;
-};
-
-/* hwrm_queue_cos2bw_cfg */
-/* Input (128 bytes) */
+	#define QUEUE_COS2BW_QCFG_RESP_QUEUE_ID7_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id7_pri_lvl;
+	u8	queue_id7_bw_weight;
+	u8	unused_2[4];
+	u8	valid;
+};
+
+/* hwrm_queue_cos2bw_cfg_input (size:1024b/128B) */
 struct hwrm_queue_cos2bw_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	__le32 enables;
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID0_VALID   0x1UL
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID1_VALID   0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID2_VALID   0x4UL
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID3_VALID   0x8UL
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID4_VALID   0x10UL
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID5_VALID   0x20UL
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID6_VALID   0x40UL
-	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID7_VALID   0x80UL
-	__le16 port_id;
-	u8 queue_id0;
-	u8 unused_0;
-	__le32 queue_id0_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id0_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id0_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_ETS     0x1UL
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	__le32	enables;
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID0_VALID     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID1_VALID     0x2UL
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID2_VALID     0x4UL
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID3_VALID     0x8UL
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID4_VALID     0x10UL
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID5_VALID     0x20UL
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID6_VALID     0x40UL
+	#define QUEUE_COS2BW_CFG_REQ_ENABLES_COS_QUEUE_ID7_VALID     0x80UL
+	__le16	port_id;
+	u8	queue_id0;
+	u8	unused_0;
+	__le32	queue_id0_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id0_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id0_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id0_pri_lvl;
-	u8 queue_id0_bw_weight;
-	u8 queue_id1;
-	__le32 queue_id1_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id1_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id1_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID0_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id0_pri_lvl;
+	u8	queue_id0_bw_weight;
+	u8	queue_id1;
+	__le32	queue_id1_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id1_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id1_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id1_pri_lvl;
-	u8 queue_id1_bw_weight;
-	u8 queue_id2;
-	__le32 queue_id2_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id2_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id2_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID1_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id1_pri_lvl;
+	u8	queue_id1_bw_weight;
+	u8	queue_id2;
+	__le32	queue_id2_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id2_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id2_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id2_pri_lvl;
-	u8 queue_id2_bw_weight;
-	u8 queue_id3;
-	__le32 queue_id3_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id3_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id3_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID2_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id2_pri_lvl;
+	u8	queue_id2_bw_weight;
+	u8	queue_id3;
+	__le32	queue_id3_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id3_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id3_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id3_pri_lvl;
-	u8 queue_id3_bw_weight;
-	u8 queue_id4;
-	__le32 queue_id4_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id4_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id4_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID3_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id3_pri_lvl;
+	u8	queue_id3_bw_weight;
+	u8	queue_id4;
+	__le32	queue_id4_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id4_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id4_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id4_pri_lvl;
-	u8 queue_id4_bw_weight;
-	u8 queue_id5;
-	__le32 queue_id5_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id5_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id5_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID4_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id4_pri_lvl;
+	u8	queue_id4_bw_weight;
+	u8	queue_id5;
+	__le32	queue_id5_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id5_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id5_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id5_pri_lvl;
-	u8 queue_id5_bw_weight;
-	u8 queue_id6;
-	__le32 queue_id6_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id6_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id6_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID5_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id5_pri_lvl;
+	u8	queue_id5_bw_weight;
+	u8	queue_id6;
+	__le32	queue_id6_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id6_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id6_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id6_pri_lvl;
-	u8 queue_id6_bw_weight;
-	u8 queue_id7;
-	__le32 queue_id7_min_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID
-	__le32 queue_id7_max_bw;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_MASK 0xfffffffUL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_SFT 0
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE	    0x10000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_BITS  (0x0UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_BYTES (0x1UL << 28)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_BYTES
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MASK 0xe0000000UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_SFT 29
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MEGA (0x0UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_KILO (0x2UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_BASE (0x4UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_GIGA (0x6UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_PERCENT1_100 (0x1UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID (0x7UL << 29)
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_LAST    QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 queue_id7_tsa_assign;
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_SP      0x0UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_ETS     0x1UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID6_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id6_pri_lvl;
+	u8	queue_id6_bw_weight;
+	u8	queue_id7;
+	__le32	queue_id7_min_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MIN_BW_BW_VALUE_UNIT_INVALID
+	__le32	queue_id7_max_bw;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_SFT              0
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE                     0x10000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_LAST                 QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_SCALE_BYTES
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_LAST         QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	queue_id7_tsa_assign;
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_SP             0x0UL
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_ETS            0x1UL
 	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_RESERVED_FIRST 0x2UL
-	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_RESERVED_LAST 0xffUL
-	u8 queue_id7_pri_lvl;
-	u8 queue_id7_bw_weight;
-	u8 unused_1[5];
+	#define QUEUE_COS2BW_CFG_REQ_QUEUE_ID7_TSA_ASSIGN_RESERVED_LAST  0xffUL
+	u8	queue_id7_pri_lvl;
+	u8	queue_id7_bw_weight;
+	u8	unused_1[5];
 };
 
-/* Output (16 bytes) */
+/* hwrm_queue_cos2bw_cfg_output (size:128b/16B) */
 struct hwrm_queue_cos2bw_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_queue_dscp_qcaps */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_queue_dscp_qcaps_input (size:192b/24B) */
 struct hwrm_queue_dscp_qcaps_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 port_id;
-	u8 unused_0[7];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	port_id;
+	u8	unused_0[7];
 };
 
-/* Output (16 bytes) */
+/* hwrm_queue_dscp_qcaps_output (size:128b/16B) */
 struct hwrm_queue_dscp_qcaps_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 num_dscp_bits;
-	u8 unused_0;
-	__le16 max_entries;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_queue_dscp2pri_qcfg */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	num_dscp_bits;
+	u8	unused_0;
+	__le16	max_entries;
+	u8	unused_1[3];
+	u8	valid;
+};
+
+/* hwrm_queue_dscp2pri_qcfg_input (size:256b/32B) */
 struct hwrm_queue_dscp2pri_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 dest_data_addr;
-	u8 port_id;
-	u8 unused_0;
-	__le16 dest_data_buffer_size;
-	__le32 unused_1;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	dest_data_addr;
+	u8	port_id;
+	u8	unused_0;
+	__le16	dest_data_buffer_size;
+	u8	unused_1[4];
+};
+
+/* hwrm_queue_dscp2pri_qcfg_output (size:128b/16B) */
 struct hwrm_queue_dscp2pri_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 entry_cnt;
-	u8 default_pri;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_queue_dscp2pri_cfg */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	entry_cnt;
+	u8	default_pri;
+	u8	unused_0[4];
+	u8	valid;
+};
+
+/* hwrm_queue_dscp2pri_cfg_input (size:320b/40B) */
 struct hwrm_queue_dscp2pri_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 src_data_addr;
-	__le32 flags;
-	#define QUEUE_DSCP2PRI_CFG_REQ_FLAGS_USE_HW_DEFAULT_PRI    0x1UL
-	__le32 enables;
-	#define QUEUE_DSCP2PRI_CFG_REQ_ENABLES_DEFAULT_PRI	    0x1UL
-	u8 port_id;
-	u8 default_pri;
-	__le16 entry_cnt;
-	__le32 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	src_data_addr;
+	__le32	flags;
+	#define QUEUE_DSCP2PRI_CFG_REQ_FLAGS_USE_HW_DEFAULT_PRI     0x1UL
+	__le32	enables;
+	#define QUEUE_DSCP2PRI_CFG_REQ_ENABLES_DEFAULT_PRI     0x1UL
+	u8	port_id;
+	u8	default_pri;
+	__le16	entry_cnt;
+	u8	unused_0[4];
+};
+
+/* hwrm_queue_dscp2pri_cfg_output (size:128b/16B) */
 struct hwrm_queue_dscp2pri_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_vnic_alloc */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_alloc_input (size:192b/24B) */
 struct hwrm_vnic_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define VNIC_ALLOC_REQ_FLAGS_DEFAULT			    0x1UL
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define VNIC_ALLOC_REQ_FLAGS_DEFAULT     0x1UL
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_vnic_alloc_output (size:128b/16B) */
 struct hwrm_vnic_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 vnic_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_vnic_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	vnic_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_vnic_free_input (size:192b/24B) */
 struct hwrm_vnic_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 vnic_id;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	vnic_id;
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_vnic_free_output (size:128b/16B) */
 struct hwrm_vnic_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_vnic_cfg */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_cfg_input (size:320b/40B) */
 struct hwrm_vnic_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define VNIC_CFG_REQ_FLAGS_DEFAULT			    0x1UL
-	#define VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE		    0x2UL
-	#define VNIC_CFG_REQ_FLAGS_BD_STALL_MODE		    0x4UL
-	#define VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE		    0x8UL
-	#define VNIC_CFG_REQ_FLAGS_ROCE_ONLY_VNIC_MODE		    0x10UL
-	#define VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE		    0x20UL
-	#define VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE 0x40UL
-	__le32 enables;
-	#define VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP		    0x1UL
-	#define VNIC_CFG_REQ_ENABLES_RSS_RULE			    0x2UL
-	#define VNIC_CFG_REQ_ENABLES_COS_RULE			    0x4UL
-	#define VNIC_CFG_REQ_ENABLES_LB_RULE			    0x8UL
-	#define VNIC_CFG_REQ_ENABLES_MRU			    0x10UL
-	__le16 vnic_id;
-	__le16 dflt_ring_grp;
-	__le16 rss_rule;
-	__le16 cos_rule;
-	__le16 lb_rule;
-	__le16 mru;
-	__le32 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define VNIC_CFG_REQ_FLAGS_DEFAULT                              0x1UL
+	#define VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE                      0x2UL
+	#define VNIC_CFG_REQ_FLAGS_BD_STALL_MODE                        0x4UL
+	#define VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE                  0x8UL
+	#define VNIC_CFG_REQ_FLAGS_ROCE_ONLY_VNIC_MODE                  0x10UL
+	#define VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE                     0x20UL
+	#define VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE     0x40UL
+	__le32	enables;
+	#define VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP     0x1UL
+	#define VNIC_CFG_REQ_ENABLES_RSS_RULE          0x2UL
+	#define VNIC_CFG_REQ_ENABLES_COS_RULE          0x4UL
+	#define VNIC_CFG_REQ_ENABLES_LB_RULE           0x8UL
+	#define VNIC_CFG_REQ_ENABLES_MRU               0x10UL
+	__le16	vnic_id;
+	__le16	dflt_ring_grp;
+	__le16	rss_rule;
+	__le16	cos_rule;
+	__le16	lb_rule;
+	__le16	mru;
+	u8	unused_0[4];
+};
+
+/* hwrm_vnic_cfg_output (size:128b/16B) */
 struct hwrm_vnic_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_vnic_qcaps */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_qcaps_input (size:192b/24B) */
 struct hwrm_vnic_qcaps_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	u8	unused_0[4];
 };
 
-/* Output (24 bytes) */
+/* hwrm_vnic_qcaps_output (size:192b/24B) */
 struct hwrm_vnic_qcaps_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 mru;
-	u8 unused_0;
-	u8 unused_1;
-	__le32 flags;
-	#define VNIC_QCAPS_RESP_FLAGS_UNUSED			    0x1UL
-	#define VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP		    0x2UL
-	#define VNIC_QCAPS_RESP_FLAGS_BD_STALL_CAP		    0x4UL
-	#define VNIC_QCAPS_RESP_FLAGS_ROCE_DUAL_VNIC_CAP	    0x8UL
-	#define VNIC_QCAPS_RESP_FLAGS_ROCE_ONLY_VNIC_CAP	    0x10UL
-	#define VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP		    0x20UL
-	#define VNIC_QCAPS_RESP_FLAGS_ROCE_MIRROING_CAPABLE_VNIC_CAP 0x40UL
-	__le32 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 unused_5;
-	u8 valid;
-};
-
-/* hwrm_vnic_tpa_cfg */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	mru;
+	u8	unused_0[2];
+	__le32	flags;
+	#define VNIC_QCAPS_RESP_FLAGS_UNUSED                              0x1UL
+	#define VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP                      0x2UL
+	#define VNIC_QCAPS_RESP_FLAGS_BD_STALL_CAP                        0x4UL
+	#define VNIC_QCAPS_RESP_FLAGS_ROCE_DUAL_VNIC_CAP                  0x8UL
+	#define VNIC_QCAPS_RESP_FLAGS_ROCE_ONLY_VNIC_CAP                  0x10UL
+	#define VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP                     0x20UL
+	#define VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP     0x40UL
+	u8	unused_1[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_tpa_cfg_input (size:320b/40B) */
 struct hwrm_vnic_tpa_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define VNIC_TPA_CFG_REQ_FLAGS_TPA			    0x1UL
-	#define VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA		    0x2UL
-	#define VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE		    0x4UL
-	#define VNIC_TPA_CFG_REQ_FLAGS_GRO			    0x8UL
-	#define VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN		    0x10UL
-	#define VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ       0x20UL
-	#define VNIC_TPA_CFG_REQ_FLAGS_GRO_IPID_CHECK		    0x40UL
-	#define VNIC_TPA_CFG_REQ_FLAGS_GRO_TTL_CHECK		    0x80UL
-	__le32 enables;
-	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS		    0x1UL
-	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS		    0x2UL
-	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_TIMER		    0x4UL
-	#define VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN		    0x8UL
-	__le16 vnic_id;
-	__le16 max_agg_segs;
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_1		   0x0UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_2		   0x1UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_4		   0x2UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_8		   0x3UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX		   0x1fUL
-	__le16 max_aggs;
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_1			   0x0UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_2			   0x1UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_4			   0x2UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_8			   0x3UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_16			   0x4UL
-	#define VNIC_TPA_CFG_REQ_MAX_AGGS_MAX			   0x7UL
-	u8 unused_0;
-	u8 unused_1;
-	__le32 max_agg_timer;
-	__le32 min_agg_len;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define VNIC_TPA_CFG_REQ_FLAGS_TPA                       0x1UL
+	#define VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA                 0x2UL
+	#define VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE            0x4UL
+	#define VNIC_TPA_CFG_REQ_FLAGS_GRO                       0x8UL
+	#define VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN              0x10UL
+	#define VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ     0x20UL
+	#define VNIC_TPA_CFG_REQ_FLAGS_GRO_IPID_CHECK            0x40UL
+	#define VNIC_TPA_CFG_REQ_FLAGS_GRO_TTL_CHECK             0x80UL
+	__le32	enables;
+	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS      0x1UL
+	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS          0x2UL
+	#define VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_TIMER     0x4UL
+	#define VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN       0x8UL
+	__le16	vnic_id;
+	__le16	max_agg_segs;
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_1   0x0UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_2   0x1UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_4   0x2UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_8   0x3UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX 0x1fUL
+	#define VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_LAST VNIC_TPA_CFG_REQ_MAX_AGG_SEGS_MAX
+	__le16	max_aggs;
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_1   0x0UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_2   0x1UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_4   0x2UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_8   0x3UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_16  0x4UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_MAX 0x7UL
+	#define VNIC_TPA_CFG_REQ_MAX_AGGS_LAST VNIC_TPA_CFG_REQ_MAX_AGGS_MAX
+	u8	unused_0[2];
+	__le32	max_agg_timer;
+	__le32	min_agg_len;
+};
+
+/* hwrm_vnic_tpa_cfg_output (size:128b/16B) */
 struct hwrm_vnic_tpa_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_vnic_rss_cfg */
-/* Input (48 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_tpa_qcfg_input (size:192b/24B) */
+struct hwrm_vnic_tpa_qcfg_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	vnic_id;
+	u8	unused_0[6];
+};
+
+/* hwrm_vnic_tpa_qcfg_output (size:256b/32B) */
+struct hwrm_vnic_tpa_qcfg_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	flags;
+	#define VNIC_TPA_QCFG_RESP_FLAGS_TPA                       0x1UL
+	#define VNIC_TPA_QCFG_RESP_FLAGS_ENCAP_TPA                 0x2UL
+	#define VNIC_TPA_QCFG_RESP_FLAGS_RSC_WND_UPDATE            0x4UL
+	#define VNIC_TPA_QCFG_RESP_FLAGS_GRO                       0x8UL
+	#define VNIC_TPA_QCFG_RESP_FLAGS_AGG_WITH_ECN              0x10UL
+	#define VNIC_TPA_QCFG_RESP_FLAGS_AGG_WITH_SAME_GRE_SEQ     0x20UL
+	#define VNIC_TPA_QCFG_RESP_FLAGS_GRO_IPID_CHECK            0x40UL
+	#define VNIC_TPA_QCFG_RESP_FLAGS_GRO_TTL_CHECK             0x80UL
+	__le16	max_agg_segs;
+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_1   0x0UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_2   0x1UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_4   0x2UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_8   0x3UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_MAX 0x1fUL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_LAST VNIC_TPA_QCFG_RESP_MAX_AGG_SEGS_MAX
+	__le16	max_aggs;
+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_1   0x0UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_2   0x1UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_4   0x2UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_8   0x3UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_16  0x4UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_MAX 0x7UL
+	#define VNIC_TPA_QCFG_RESP_MAX_AGGS_LAST VNIC_TPA_QCFG_RESP_MAX_AGGS_MAX
+	__le32	max_agg_timer;
+	__le32	min_agg_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_rss_cfg_input (size:384b/48B) */
 struct hwrm_vnic_rss_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 hash_type;
-	#define VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4		    0x1UL
-	#define VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4		    0x2UL
-	#define VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4		    0x4UL
-	#define VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6		    0x8UL
-	#define VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6		    0x10UL
-	#define VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6		    0x20UL
-	__le32 unused_0;
-	__le64 ring_grp_tbl_addr;
-	__le64 hash_key_tbl_addr;
-	__le16 rss_ctx_idx;
-	__le16 unused_1[3];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	hash_type;
+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4         0x1UL
+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4     0x2UL
+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4     0x4UL
+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6         0x8UL
+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6     0x10UL
+	#define VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6     0x20UL
+	u8	unused_0[4];
+	__le64	ring_grp_tbl_addr;
+	__le64	hash_key_tbl_addr;
+	__le16	rss_ctx_idx;
+	u8	unused_1[6];
+};
+
+/* hwrm_vnic_rss_cfg_output (size:128b/16B) */
 struct hwrm_vnic_rss_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_vnic_plcmodes_cfg */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_plcmodes_cfg_input (size:320b/40B) */
 struct hwrm_vnic_plcmodes_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define VNIC_PLCMODES_CFG_REQ_FLAGS_REGULAR_PLACEMENT      0x1UL
-	#define VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT	    0x2UL
-	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4		    0x4UL
-	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6		    0x8UL
-	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_FCOE		    0x10UL
-	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_ROCE		    0x20UL
-	__le32 enables;
-	#define VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID   0x1UL
-	#define VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_OFFSET_VALID     0x2UL
-	#define VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID  0x4UL
-	__le32 vnic_id;
-	__le16 jumbo_thresh;
-	__le16 hds_offset;
-	__le16 hds_threshold;
-	__le16 unused_0[3];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_REGULAR_PLACEMENT     0x1UL
+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT       0x2UL
+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4              0x4UL
+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6              0x8UL
+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_FCOE              0x10UL
+	#define VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_ROCE              0x20UL
+	__le32	enables;
+	#define VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID      0x1UL
+	#define VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_OFFSET_VALID        0x2UL
+	#define VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID     0x4UL
+	__le32	vnic_id;
+	__le16	jumbo_thresh;
+	__le16	hds_offset;
+	__le16	hds_threshold;
+	u8	unused_0[6];
+};
+
+/* hwrm_vnic_plcmodes_cfg_output (size:128b/16B) */
 struct hwrm_vnic_plcmodes_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_vnic_rss_cos_lb_ctx_alloc */
-/* Input (16 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vnic_rss_cos_lb_ctx_alloc_input (size:128b/16B) */
 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (16 bytes) */
+/* hwrm_vnic_rss_cos_lb_ctx_alloc_output (size:128b/16B) */
 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 rss_cos_lb_ctx_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_vnic_rss_cos_lb_ctx_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	rss_cos_lb_ctx_id;
+	u8	unused_0[5];
+	u8	valid;
+};
+
+/* hwrm_vnic_rss_cos_lb_ctx_free_input (size:192b/24B) */
 struct hwrm_vnic_rss_cos_lb_ctx_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 rss_cos_lb_ctx_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	rss_cos_lb_ctx_id;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_vnic_rss_cos_lb_ctx_free_output (size:128b/16B) */
 struct hwrm_vnic_rss_cos_lb_ctx_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_ring_alloc */
-/* Input (80 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_ring_alloc_input (size:640b/80B) */
 struct hwrm_ring_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define RING_ALLOC_REQ_ENABLES_RESERVED1		    0x1UL
-	#define RING_ALLOC_REQ_ENABLES_RING_ARB_CFG		    0x2UL
-	#define RING_ALLOC_REQ_ENABLES_RESERVED3		    0x4UL
-	#define RING_ALLOC_REQ_ENABLES_STAT_CTX_ID_VALID	    0x8UL
-	#define RING_ALLOC_REQ_ENABLES_RESERVED4		    0x10UL
-	#define RING_ALLOC_REQ_ENABLES_MAX_BW_VALID		    0x20UL
-	u8 ring_type;
-	#define RING_ALLOC_REQ_RING_TYPE_L2_CMPL		   0x0UL
-	#define RING_ALLOC_REQ_RING_TYPE_TX			   0x1UL
-	#define RING_ALLOC_REQ_RING_TYPE_RX			   0x2UL
-	#define RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL		   0x3UL
-	u8 unused_0;
-	__le16 unused_1;
-	__le64 page_tbl_addr;
-	__le32 fbo;
-	u8 page_size;
-	u8 page_tbl_depth;
-	u8 unused_2;
-	u8 unused_3;
-	__le32 length;
-	__le16 logical_id;
-	__le16 cmpl_ring_id;
-	__le16 queue_id;
-	u8 unused_4;
-	u8 unused_5;
-	__le32 reserved1;
-	__le16 ring_arb_cfg;
-	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_MASK	    0xfUL
-	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SFT	    0
-	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SP	   (0x1UL << 0)
-	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ	   (0x2UL << 0)
-	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_LAST    RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ
-	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_MASK		    0xf0UL
-	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_SFT		    4
-	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_MASK  0xff00UL
-	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_SFT   8
-	u8 unused_6;
-	u8 unused_7;
-	__le32 reserved3;
-	__le32 stat_ctx_id;
-	__le32 reserved4;
-	__le32 max_bw;
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_MASK		    0xfffffffUL
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_SFT		    0
-	#define RING_ALLOC_REQ_MAX_BW_SCALE			    0x10000000UL
-	#define RING_ALLOC_REQ_MAX_BW_SCALE_BITS		   (0x0UL << 28)
-	#define RING_ALLOC_REQ_MAX_BW_SCALE_BYTES		   (0x1UL << 28)
-	#define RING_ALLOC_REQ_MAX_BW_SCALE_LAST    RING_ALLOC_REQ_MAX_BW_SCALE_BYTES
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MASK	    0xe0000000UL
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_SFT	    29
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MEGA	   (0x0UL << 29)
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_KILO	   (0x2UL << 29)
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_BASE	   (0x4UL << 29)
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_GIGA	   (0x6UL << 29)
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define RING_ALLOC_REQ_ENABLES_RING_ARB_CFG          0x2UL
+	#define RING_ALLOC_REQ_ENABLES_STAT_CTX_ID_VALID     0x8UL
+	#define RING_ALLOC_REQ_ENABLES_MAX_BW_VALID          0x20UL
+	u8	ring_type;
+	#define RING_ALLOC_REQ_RING_TYPE_L2_CMPL   0x0UL
+	#define RING_ALLOC_REQ_RING_TYPE_TX        0x1UL
+	#define RING_ALLOC_REQ_RING_TYPE_RX        0x2UL
+	#define RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL 0x3UL
+	#define RING_ALLOC_REQ_RING_TYPE_LAST     RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL
+	u8	unused_0[3];
+	__le64	page_tbl_addr;
+	__le32	fbo;
+	u8	page_size;
+	u8	page_tbl_depth;
+	u8	unused_1[2];
+	__le32	length;
+	__le16	logical_id;
+	__le16	cmpl_ring_id;
+	__le16	queue_id;
+	u8	unused_2[2];
+	__le32	reserved1;
+	__le16	ring_arb_cfg;
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_MASK      0xfUL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SFT       0
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_SP          0x1UL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ         0x2UL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_LAST       RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_WFQ
+	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_MASK            0xf0UL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_RSVD_SFT             4
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_MASK 0xff00UL
+	#define RING_ALLOC_REQ_RING_ARB_CFG_ARB_POLICY_PARAM_SFT 8
+	__le16	unused_3;
+	__le32	reserved3;
+	__le32	stat_ctx_id;
+	__le32	reserved4;
+	__le32	max_bw;
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_MASK             0xfffffffUL
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_SFT              0
+	#define RING_ALLOC_REQ_MAX_BW_SCALE                     0x10000000UL
+	#define RING_ALLOC_REQ_MAX_BW_SCALE_BITS                  (0x0UL << 28)
+	#define RING_ALLOC_REQ_MAX_BW_SCALE_BYTES                 (0x1UL << 28)
+	#define RING_ALLOC_REQ_MAX_BW_SCALE_LAST                 RING_ALLOC_REQ_MAX_BW_SCALE_BYTES
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MASK        0xe0000000UL
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_SFT         29
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_MEGA          (0x0UL << 29)
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_KILO          (0x2UL << 29)
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_BASE          (0x4UL << 29)
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_GIGA          (0x6UL << 29)
 	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_PERCENT1_100  (0x1UL << 29)
 	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID       (0x7UL << 29)
-	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_LAST    RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
-	u8 int_mode;
-	#define RING_ALLOC_REQ_INT_MODE_LEGACY			   0x0UL
-	#define RING_ALLOC_REQ_INT_MODE_RSVD			   0x1UL
-	#define RING_ALLOC_REQ_INT_MODE_MSIX			   0x2UL
-	#define RING_ALLOC_REQ_INT_MODE_POLL			   0x3UL
-	u8 unused_8[3];
+	#define RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_LAST         RING_ALLOC_REQ_MAX_BW_BW_VALUE_UNIT_INVALID
+	u8	int_mode;
+	#define RING_ALLOC_REQ_INT_MODE_LEGACY 0x0UL
+	#define RING_ALLOC_REQ_INT_MODE_RSVD   0x1UL
+	#define RING_ALLOC_REQ_INT_MODE_MSIX   0x2UL
+	#define RING_ALLOC_REQ_INT_MODE_POLL   0x3UL
+	#define RING_ALLOC_REQ_INT_MODE_LAST  RING_ALLOC_REQ_INT_MODE_POLL
+	u8	unused_4[3];
 };
 
-/* Output (16 bytes) */
+/* hwrm_ring_alloc_output (size:128b/16B) */
 struct hwrm_ring_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 ring_id;
-	__le16 logical_ring_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_ring_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	ring_id;
+	__le16	logical_ring_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_ring_free_input (size:192b/24B) */
 struct hwrm_ring_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 ring_type;
-	#define RING_FREE_REQ_RING_TYPE_L2_CMPL		   0x0UL
-	#define RING_FREE_REQ_RING_TYPE_TX			   0x1UL
-	#define RING_FREE_REQ_RING_TYPE_RX			   0x2UL
-	#define RING_FREE_REQ_RING_TYPE_ROCE_CMPL		   0x3UL
-	u8 unused_0;
-	__le16 ring_id;
-	__le32 unused_1;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	ring_type;
+	#define RING_FREE_REQ_RING_TYPE_L2_CMPL   0x0UL
+	#define RING_FREE_REQ_RING_TYPE_TX        0x1UL
+	#define RING_FREE_REQ_RING_TYPE_RX        0x2UL
+	#define RING_FREE_REQ_RING_TYPE_ROCE_CMPL 0x3UL
+	#define RING_FREE_REQ_RING_TYPE_LAST     RING_FREE_REQ_RING_TYPE_ROCE_CMPL
+	u8	unused_0;
+	__le16	ring_id;
+	u8	unused_1[4];
+};
+
+/* hwrm_ring_free_output (size:128b/16B) */
 struct hwrm_ring_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_ring_cmpl_ring_qaggint_params */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_ring_cmpl_ring_qaggint_params_input (size:192b/24B) */
 struct hwrm_ring_cmpl_ring_qaggint_params_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 ring_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	ring_id;
+	u8	unused_0[6];
 };
 
-/* Output (32 bytes) */
+/* hwrm_ring_cmpl_ring_qaggint_params_output (size:256b/32B) */
 struct hwrm_ring_cmpl_ring_qaggint_params_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 flags;
-	#define RING_CMPL_RING_QAGGINT_PARAMS_RESP_FLAGS_TIMER_RESET 0x1UL
-	#define RING_CMPL_RING_QAGGINT_PARAMS_RESP_FLAGS_RING_IDLE 0x2UL
-	__le16 num_cmpl_dma_aggr;
-	__le16 num_cmpl_dma_aggr_during_int;
-	__le16 cmpl_aggr_dma_tmr;
-	__le16 cmpl_aggr_dma_tmr_during_int;
-	__le16 int_lat_tmr_min;
-	__le16 int_lat_tmr_max;
-	__le16 num_cmpl_aggr_int;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_ring_cmpl_ring_cfg_aggint_params */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	flags;
+	#define RING_CMPL_RING_QAGGINT_PARAMS_RESP_FLAGS_TIMER_RESET     0x1UL
+	#define RING_CMPL_RING_QAGGINT_PARAMS_RESP_FLAGS_RING_IDLE       0x2UL
+	__le16	num_cmpl_dma_aggr;
+	__le16	num_cmpl_dma_aggr_during_int;
+	__le16	cmpl_aggr_dma_tmr;
+	__le16	cmpl_aggr_dma_tmr_during_int;
+	__le16	int_lat_tmr_min;
+	__le16	int_lat_tmr_max;
+	__le16	num_cmpl_aggr_int;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_ring_cmpl_ring_cfg_aggint_params_input (size:320b/40B) */
 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 ring_id;
-	__le16 flags;
-	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET 0x1UL
-	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE 0x2UL
-	__le16 num_cmpl_dma_aggr;
-	__le16 num_cmpl_dma_aggr_during_int;
-	__le16 cmpl_aggr_dma_tmr;
-	__le16 cmpl_aggr_dma_tmr_during_int;
-	__le16 int_lat_tmr_min;
-	__le16 int_lat_tmr_max;
-	__le16 num_cmpl_aggr_int;
-	__le16 unused_0[3];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	ring_id;
+	__le16	flags;
+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET     0x1UL
+	#define RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE       0x2UL
+	__le16	num_cmpl_dma_aggr;
+	__le16	num_cmpl_dma_aggr_during_int;
+	__le16	cmpl_aggr_dma_tmr;
+	__le16	cmpl_aggr_dma_tmr_during_int;
+	__le16	int_lat_tmr_min;
+	__le16	int_lat_tmr_max;
+	__le16	num_cmpl_aggr_int;
+	u8	unused_0[6];
+};
+
+/* hwrm_ring_cmpl_ring_cfg_aggint_params_output (size:128b/16B) */
 struct hwrm_ring_cmpl_ring_cfg_aggint_params_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_ring_reset */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_ring_reset_input (size:192b/24B) */
 struct hwrm_ring_reset_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 ring_type;
-	#define RING_RESET_REQ_RING_TYPE_L2_CMPL		   0x0UL
-	#define RING_RESET_REQ_RING_TYPE_TX			   0x1UL
-	#define RING_RESET_REQ_RING_TYPE_RX			   0x2UL
-	#define RING_RESET_REQ_RING_TYPE_ROCE_CMPL		   0x3UL
-	u8 unused_0;
-	__le16 ring_id;
-	__le32 unused_1;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	ring_type;
+	#define RING_RESET_REQ_RING_TYPE_L2_CMPL   0x0UL
+	#define RING_RESET_REQ_RING_TYPE_TX        0x1UL
+	#define RING_RESET_REQ_RING_TYPE_RX        0x2UL
+	#define RING_RESET_REQ_RING_TYPE_ROCE_CMPL 0x3UL
+	#define RING_RESET_REQ_RING_TYPE_LAST     RING_RESET_REQ_RING_TYPE_ROCE_CMPL
+	u8	unused_0;
+	__le16	ring_id;
+	u8	unused_1[4];
+};
+
+/* hwrm_ring_reset_output (size:128b/16B) */
 struct hwrm_ring_reset_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_ring_grp_alloc */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_ring_grp_alloc_input (size:192b/24B) */
 struct hwrm_ring_grp_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 cr;
-	__le16 rr;
-	__le16 ar;
-	__le16 sc;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	cr;
+	__le16	rr;
+	__le16	ar;
+	__le16	sc;
+};
+
+/* hwrm_ring_grp_alloc_output (size:128b/16B) */
 struct hwrm_ring_grp_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 ring_group_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_ring_grp_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	ring_group_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_ring_grp_free_input (size:192b/24B) */
 struct hwrm_ring_grp_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 ring_group_id;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	ring_group_id;
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_ring_grp_free_output (size:128b/16B) */
 struct hwrm_ring_grp_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_l2_filter_alloc */
-/* Input (96 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_l2_filter_alloc_input (size:768b/96B) */
 struct hwrm_cfa_l2_filter_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH		    0x1UL
-	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_TX		   (0x0UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX		   (0x1UL << 0)
-	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_LAST    CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX
-	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_LOOPBACK		    0x2UL
-	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_DROP		    0x4UL
-	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST	    0x8UL
-	__le32 enables;
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR	    0x1UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK       0x2UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN	    0x4UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN_MASK      0x8UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN	    0x10UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK      0x20UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR	    0x40UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR_MASK     0x80UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN	    0x100UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN_MASK    0x200UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN	    0x400UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN_MASK    0x800UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_TYPE	    0x1000UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_ID		    0x2000UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE	    0x4000UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID		    0x8000UL
-	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID     0x10000UL
-	u8 l2_addr[6];
-	u8 unused_0;
-	u8 unused_1;
-	u8 l2_addr_mask[6];
-	__le16 l2_ovlan;
-	__le16 l2_ovlan_mask;
-	__le16 l2_ivlan;
-	__le16 l2_ivlan_mask;
-	u8 unused_2;
-	u8 unused_3;
-	u8 t_l2_addr[6];
-	u8 unused_4;
-	u8 unused_5;
-	u8 t_l2_addr_mask[6];
-	__le16 t_l2_ovlan;
-	__le16 t_l2_ovlan_mask;
-	__le16 t_l2_ivlan;
-	__le16 t_l2_ivlan_mask;
-	u8 src_type;
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_NPORT		   0x0UL
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_PF		   0x1UL
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VF		   0x2UL
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VNIC		   0x3UL
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_KONG		   0x4UL
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_APE		   0x5UL
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_BONO		   0x6UL
-	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG		   0x7UL
-	u8 unused_6;
-	__le32 src_id;
-	u8 tunnel_type;
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL     0x0UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN	   0x1UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE	   0x2UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE	   0x3UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP	   0x4UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE	   0x5UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS	   0x6UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT	   0x7UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE	   0x8UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4      0x9UL
-	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL     0xffUL
-	u8 unused_7;
-	__le16 dst_id;
-	__le16 mirror_vnic_id;
-	u8 pri_hint;
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER	   0x0UL
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_ABOVE_FILTER     0x1UL
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_BELOW_FILTER     0x2UL
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MAX		   0x3UL
-	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN		   0x4UL
-	u8 unused_8;
-	__le32 unused_9;
-	__le64 l2_filter_id_hint;
-};
-
-/* Output (24 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH          0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_TX         0x0UL
+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX         0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_LAST      CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX
+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_LOOPBACK      0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_DROP          0x4UL
+	#define CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST     0x8UL
+	__le32	enables;
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR             0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK        0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN            0x4UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_OVLAN_MASK       0x8UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN            0x10UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK       0x20UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR           0x40UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_ADDR_MASK      0x80UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN          0x100UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_OVLAN_MASK     0x200UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN          0x400UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_L2_IVLAN_MASK     0x800UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_TYPE            0x1000UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_SRC_ID              0x2000UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE         0x4000UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID              0x8000UL
+	#define CFA_L2_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID      0x10000UL
+	u8	l2_addr[6];
+	u8	unused_0[2];
+	u8	l2_addr_mask[6];
+	__le16	l2_ovlan;
+	__le16	l2_ovlan_mask;
+	__le16	l2_ivlan;
+	__le16	l2_ivlan_mask;
+	u8	unused_1[2];
+	u8	t_l2_addr[6];
+	u8	unused_2[2];
+	u8	t_l2_addr_mask[6];
+	__le16	t_l2_ovlan;
+	__le16	t_l2_ovlan_mask;
+	__le16	t_l2_ivlan;
+	__le16	t_l2_ivlan_mask;
+	u8	src_type;
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_NPORT 0x0UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_PF    0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VF    0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_VNIC  0x3UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_KONG  0x4UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_APE   0x5UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_BONO  0x6UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG  0x7UL
+	#define CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_LAST CFA_L2_FILTER_ALLOC_REQ_SRC_TYPE_TANG
+	u8	unused_3;
+	__le32	src_id;
+	u8	tunnel_type;
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL 0x0UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE     0x3UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP      0x4UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE    0x5UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS      0x6UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT       0x7UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     0x8UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4  0x9UL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL 0xffUL
+	#define CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST     CFA_L2_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
+	u8	unused_4;
+	__le16	dst_id;
+	__le16	mirror_vnic_id;
+	u8	pri_hint;
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER    0x0UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_ABOVE_FILTER 0x1UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_BELOW_FILTER 0x2UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MAX          0x3UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN          0x4UL
+	#define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_LAST        CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN
+	u8	unused_5;
+	__le32	unused_6;
+	__le64	l2_filter_id_hint;
+};
+
+/* hwrm_cfa_l2_filter_alloc_output (size:192b/24B) */
 struct hwrm_cfa_l2_filter_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 l2_filter_id;
-	__le32 flow_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_cfa_l2_filter_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	l2_filter_id;
+	__le32	flow_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_cfa_l2_filter_free_input (size:192b/24B) */
 struct hwrm_cfa_l2_filter_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 l2_filter_id;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	l2_filter_id;
 };
 
-/* Output (16 bytes) */
+/* hwrm_cfa_l2_filter_free_output (size:128b/16B) */
 struct hwrm_cfa_l2_filter_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_l2_filter_cfg */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_l2_filter_cfg_input (size:320b/40B) */
 struct hwrm_cfa_l2_filter_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH		    0x1UL
-	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_TX		   (0x0UL << 0)
-	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX		   (0x1UL << 0)
-	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_LAST    CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX
-	#define CFA_L2_FILTER_CFG_REQ_FLAGS_DROP		    0x2UL
-	__le32 enables;
-	#define CFA_L2_FILTER_CFG_REQ_ENABLES_DST_ID		    0x1UL
-	#define CFA_L2_FILTER_CFG_REQ_ENABLES_NEW_MIRROR_VNIC_ID   0x2UL
-	__le64 l2_filter_id;
-	__le32 dst_id;
-	__le32 new_mirror_vnic_id;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH     0x1UL
+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_TX    0x0UL
+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX    0x1UL
+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_LAST CFA_L2_FILTER_CFG_REQ_FLAGS_PATH_RX
+	#define CFA_L2_FILTER_CFG_REQ_FLAGS_DROP     0x2UL
+	__le32	enables;
+	#define CFA_L2_FILTER_CFG_REQ_ENABLES_DST_ID                 0x1UL
+	#define CFA_L2_FILTER_CFG_REQ_ENABLES_NEW_MIRROR_VNIC_ID     0x2UL
+	__le64	l2_filter_id;
+	__le32	dst_id;
+	__le32	new_mirror_vnic_id;
+};
+
+/* hwrm_cfa_l2_filter_cfg_output (size:128b/16B) */
 struct hwrm_cfa_l2_filter_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_l2_set_rx_mask */
-/* Input (56 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_l2_set_rx_mask_input (size:448b/56B) */
 struct hwrm_cfa_l2_set_rx_mask_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 vnic_id;
-	__le32 mask;
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_RESERVED		    0x1UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_MCAST		    0x2UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST		    0x4UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_BCAST		    0x8UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS	    0x10UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_OUTERMOST		    0x20UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_VLANONLY		    0x40UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_VLAN_NONVLAN	    0x80UL
-	#define CFA_L2_SET_RX_MASK_REQ_MASK_ANYVLAN_NONVLAN	    0x100UL
-	__le64 mc_tbl_addr;
-	__le32 num_mc_entries;
-	__le32 unused_0;
-	__le64 vlan_tag_tbl_addr;
-	__le32 num_vlan_tags;
-	__le32 unused_1;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	vnic_id;
+	__le32	mask;
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_MCAST               0x2UL
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST           0x4UL
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_BCAST               0x8UL
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS         0x10UL
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_OUTERMOST           0x20UL
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_VLANONLY            0x40UL
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_VLAN_NONVLAN        0x80UL
+	#define CFA_L2_SET_RX_MASK_REQ_MASK_ANYVLAN_NONVLAN     0x100UL
+	__le64	mc_tbl_addr;
+	__le32	num_mc_entries;
+	u8	unused_0[4];
+	__le64	vlan_tag_tbl_addr;
+	__le32	num_vlan_tags;
+	u8	unused_1[4];
+};
+
+/* hwrm_cfa_l2_set_rx_mask_output (size:128b/16B) */
 struct hwrm_cfa_l2_set_rx_mask_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* Command specific Error Codes (8 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_l2_set_rx_mask_cmd_err (size:64b/8B) */
 struct hwrm_cfa_l2_set_rx_mask_cmd_err {
-	u8 code;
-	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_UNKNOWN	   0x0UL
+	u8	code;
+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_UNKNOWN                    0x0UL
 	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_NTUPLE_FILTER_CONFLICT_ERR 0x1UL
-	u8 unused_0[7];
+	#define CFA_L2_SET_RX_MASK_CMD_ERR_CODE_LAST                      CFA_L2_SET_RX_MASK_CMD_ERR_CODE_NTUPLE_FILTER_CONFLICT_ERR
+	u8	unused_0[7];
 };
 
-/* hwrm_cfa_tunnel_filter_alloc */
-/* Input (88 bytes) */
+/* hwrm_cfa_tunnel_filter_alloc_input (size:704b/88B) */
 struct hwrm_cfa_tunnel_filter_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_FLAGS_LOOPBACK	    0x1UL
-	__le32 enables;
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID   0x1UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_ADDR	    0x2UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN       0x4UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L3_ADDR	    0x8UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L3_ADDR_TYPE   0x10UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_T_L3_ADDR_TYPE 0x20UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_T_L3_ADDR      0x40UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE    0x80UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_VNI	    0x100UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_DST_VNIC_ID    0x200UL
-	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID 0x400UL
-	__le64 l2_filter_id;
-	u8 l2_addr[6];
-	__le16 l2_ivlan;
-	__le32 l3_addr[4];
-	__le32 t_l3_addr[4];
-	u8 l3_addr_type;
-	u8 t_l3_addr_type;
-	u8 tunnel_type;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_FLAGS_LOOPBACK     0x1UL
+	__le32	enables;
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID       0x1UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_ADDR            0x2UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN           0x4UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L3_ADDR            0x8UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_L3_ADDR_TYPE       0x10UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_T_L3_ADDR_TYPE     0x20UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_T_L3_ADDR          0x40UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE        0x80UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_VNI                0x100UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_DST_VNIC_ID        0x200UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID     0x400UL
+	__le64	l2_filter_id;
+	u8	l2_addr[6];
+	__le16	l2_ivlan;
+	__le32	l3_addr[4];
+	__le32	t_l3_addr[4];
+	u8	l3_addr_type;
+	u8	t_l3_addr_type;
+	u8	tunnel_type;
 	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL 0x0UL
 	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     0x1UL
 	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     0x2UL
@@ -4174,158 +4372,204 @@ struct hwrm_cfa_tunnel_filter_alloc_input {
 	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     0x8UL
 	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4  0x9UL
 	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL 0xffUL
-	u8 unused_0;
-	__le32 vni;
-	__le32 dst_vnic_id;
-	__le32 mirror_vnic_id;
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST     CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
+	u8	tunnel_flags;
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_FLAGS_TUN_FLAGS_OAM_CHECKSUM_EXPLHDR     0x1UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_FLAGS_TUN_FLAGS_CRITICAL_OPT_S1          0x2UL
+	#define CFA_TUNNEL_FILTER_ALLOC_REQ_TUNNEL_FLAGS_TUN_FLAGS_EXTHDR_SEQNUM_S0         0x4UL
+	__le32	vni;
+	__le32	dst_vnic_id;
+	__le32	mirror_vnic_id;
 };
 
-/* Output (24 bytes) */
+/* hwrm_cfa_tunnel_filter_alloc_output (size:192b/24B) */
 struct hwrm_cfa_tunnel_filter_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 tunnel_filter_id;
-	__le32 flow_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_cfa_tunnel_filter_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	tunnel_filter_id;
+	__le32	flow_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_cfa_tunnel_filter_free_input (size:192b/24B) */
 struct hwrm_cfa_tunnel_filter_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 tunnel_filter_id;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	tunnel_filter_id;
 };
 
-/* Output (16 bytes) */
+/* hwrm_cfa_tunnel_filter_free_output (size:128b/16B) */
 struct hwrm_cfa_tunnel_filter_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_encap_record_alloc */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_vxlan_ipv4_hdr (size:128b/16B) */
+struct hwrm_vxlan_ipv4_hdr {
+	u8	ver_hlen;
+	#define VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_MASK 0xfUL
+	#define VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT 0
+	#define VXLAN_IPV4_HDR_VER_HLEN_VERSION_MASK      0xf0UL
+	#define VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT       4
+	u8	tos;
+	__be16	ip_id;
+	__be16	flags_frag_offset;
+	u8	ttl;
+	u8	protocol;
+	__be32	src_ip_addr;
+	__be32	dest_ip_addr;
+};
+
+/* hwrm_vxlan_ipv6_hdr (size:320b/40B) */
+struct hwrm_vxlan_ipv6_hdr {
+	__be32	ver_tc_flow_label;
+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_VER_SFT         0x1cUL
+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_VER_MASK        0xf0000000UL
+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_TC_SFT          0x14UL
+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_TC_MASK         0xff00000UL
+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_SFT  0x0UL
+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_MASK 0xfffffUL
+	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_LAST           VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_MASK
+	__be16	payload_len;
+	u8	next_hdr;
+	u8	ttl;
+	__be32	src_ip_addr[4];
+	__be32	dest_ip_addr[4];
+};
+
+/* hwrm_cfa_encap_data_vxlan (size:576b/72B) */
+struct hwrm_cfa_encap_data_vxlan {
+	u8	src_mac_addr[6];
+	__le16	unused_0;
+	u8	dst_mac_addr[6];
+	u8	num_vlan_tags;
+	u8	unused_1;
+	__be16	ovlan_tpid;
+	__be16	ovlan_tci;
+	__be16	ivlan_tpid;
+	__be16	ivlan_tci;
+	__le32	l3[10];
+	#define CFA_ENCAP_DATA_VXLAN_L3_VER_MASK 0xfUL
+	#define CFA_ENCAP_DATA_VXLAN_L3_VER_IPV4 0x4UL
+	#define CFA_ENCAP_DATA_VXLAN_L3_VER_IPV6 0x6UL
+	#define CFA_ENCAP_DATA_VXLAN_L3_LAST    CFA_ENCAP_DATA_VXLAN_L3_VER_IPV6
+	__be16	src_port;
+	__be16	dst_port;
+	__be32	vni;
+};
+
+/* hwrm_cfa_encap_record_alloc_input (size:832b/104B) */
 struct hwrm_cfa_encap_record_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_FLAGS_LOOPBACK	    0x1UL
-	u8 encap_type;
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN       0x1UL
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_NVGRE       0x2UL
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_L2GRE       0x3UL
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPIP	   0x4UL
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_GENEVE      0x5UL
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_MPLS	   0x6UL
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VLAN	   0x7UL
-	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE       0x8UL
-	u8 unused_0;
-	__le16 unused_1;
-	__le32 encap_data[20];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_FLAGS_LOOPBACK     0x1UL
+	u8	encap_type;
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN  0x1UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_NVGRE  0x2UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_L2GRE  0x3UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPIP   0x4UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_GENEVE 0x5UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_MPLS   0x6UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VLAN   0x7UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE  0x8UL
+	#define CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_LAST  CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_IPGRE
+	u8	unused_0[3];
+	__le32	encap_data[20];
+};
+
+/* hwrm_cfa_encap_record_alloc_output (size:128b/16B) */
 struct hwrm_cfa_encap_record_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 encap_record_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_cfa_encap_record_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	encap_record_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_cfa_encap_record_free_input (size:192b/24B) */
 struct hwrm_cfa_encap_record_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 encap_record_id;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	encap_record_id;
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_cfa_encap_record_free_output (size:128b/16B) */
 struct hwrm_cfa_encap_record_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_ntuple_filter_alloc */
-/* Input (128 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_ntuple_filter_alloc_input (size:1024b/128B) */
 struct hwrm_cfa_ntuple_filter_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_LOOPBACK	    0x1UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP		    0x2UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_METER	    0x4UL
-	__le32 enables;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID   0x1UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE      0x2UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE    0x4UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR    0x8UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE    0x10UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR     0x20UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK 0x40UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR     0x80UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK 0x100UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL    0x200UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT       0x400UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK  0x800UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT       0x1000UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK  0x2000UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_PRI_HINT       0x4000UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_NTUPLE_FILTER_ID 0x8000UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID	    0x10000UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID 0x20000UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR    0x40000UL
-	__le64 l2_filter_id;
-	u8 src_macaddr[6];
-	__be16 ethertype;
-	u8 ip_addr_type;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN  0x0UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4     0x4UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6     0x6UL
-	u8 ip_protocol;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN   0x0UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP       0x6UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP       0x11UL
-	__le16 dst_id;
-	__le16 mirror_vnic_id;
-	u8 tunnel_type;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_LOOPBACK     0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP         0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_METER        0x4UL
+	__le32	enables;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID         0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE            0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE          0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR          0x8UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE          0x10UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR           0x20UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK      0x40UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR           0x80UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK      0x100UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL          0x200UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT             0x400UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK        0x800UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT             0x1000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK        0x2000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_PRI_HINT             0x4000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_NTUPLE_FILTER_ID     0x8000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID               0x10000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID       0x20000UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR          0x40000UL
+	__le64	l2_filter_id;
+	u8	src_macaddr[6];
+	__be16	ethertype;
+	u8	ip_addr_type;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN 0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4    0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6    0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_LAST   CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6
+	u8	ip_protocol;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN 0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP     0x6UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP     0x11UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_LAST   CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP
+	__le16	dst_id;
+	__le16	mirror_vnic_id;
+	u8	tunnel_type;
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL 0x0UL
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     0x1UL
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     0x2UL
@@ -4337,2221 +4581,1723 @@ struct hwrm_cfa_ntuple_filter_alloc_input {
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     0x8UL
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4  0x9UL
 	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL 0xffUL
-	u8 pri_hint;
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER    0x0UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_ABOVE	   0x1UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_BELOW	   0x2UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_HIGHEST      0x3UL
-	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST       0x4UL
-	__be32 src_ipaddr[4];
-	__be32 src_ipaddr_mask[4];
-	__be32 dst_ipaddr[4];
-	__be32 dst_ipaddr_mask[4];
-	__be16 src_port;
-	__be16 src_port_mask;
-	__be16 dst_port;
-	__be16 dst_port_mask;
-	__le64 ntuple_filter_id_hint;
-};
-
-/* Output (24 bytes) */
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST     CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
+	u8	pri_hint;
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER 0x0UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_ABOVE     0x1UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_BELOW     0x2UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_HIGHEST   0x3UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST    0x4UL
+	#define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LAST     CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST
+	__be32	src_ipaddr[4];
+	__be32	src_ipaddr_mask[4];
+	__be32	dst_ipaddr[4];
+	__be32	dst_ipaddr_mask[4];
+	__be16	src_port;
+	__be16	src_port_mask;
+	__be16	dst_port;
+	__be16	dst_port_mask;
+	__le64	ntuple_filter_id_hint;
+};
+
+/* hwrm_cfa_ntuple_filter_alloc_output (size:192b/24B) */
 struct hwrm_cfa_ntuple_filter_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 ntuple_filter_id;
-	__le32 flow_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* Command specific Error Codes (8 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	ntuple_filter_id;
+	__le32	flow_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_cfa_ntuple_filter_alloc_cmd_err (size:64b/8B) */
 struct hwrm_cfa_ntuple_filter_alloc_cmd_err {
-	u8 code;
-	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_UNKNOWN      0x0UL
+	u8	code;
+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_UNKNOWN                   0x0UL
 	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_RX_MASK_VLAN_CONFLICT_ERR 0x1UL
-	u8 unused_0[7];
+	#define CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_LAST                     CFA_NTUPLE_FILTER_ALLOC_CMD_ERR_CODE_RX_MASK_VLAN_CONFLICT_ERR
+	u8	unused_0[7];
 };
 
-/* hwrm_cfa_ntuple_filter_free */
-/* Input (24 bytes) */
+/* hwrm_cfa_ntuple_filter_free_input (size:192b/24B) */
 struct hwrm_cfa_ntuple_filter_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 ntuple_filter_id;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	ntuple_filter_id;
 };
 
-/* Output (16 bytes) */
+/* hwrm_cfa_ntuple_filter_free_output (size:128b/16B) */
 struct hwrm_cfa_ntuple_filter_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_ntuple_filter_cfg */
-/* Input (48 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_ntuple_filter_cfg_input (size:384b/48B) */
 struct hwrm_cfa_ntuple_filter_cfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_DST_ID       0x1UL
-	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_MIRROR_VNIC_ID 0x2UL
-	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_METER_INSTANCE_ID 0x4UL
-	__le32 unused_0;
-	__le64 ntuple_filter_id;
-	__le32 new_dst_id;
-	__le32 new_mirror_vnic_id;
-	__le16 new_meter_instance_id;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_DST_ID                0x1UL
+	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_MIRROR_VNIC_ID        0x2UL
+	#define CFA_NTUPLE_FILTER_CFG_REQ_ENABLES_NEW_METER_INSTANCE_ID     0x4UL
+	u8	unused_0[4];
+	__le64	ntuple_filter_id;
+	__le32	new_dst_id;
+	__le32	new_mirror_vnic_id;
+	__le16	new_meter_instance_id;
 	#define CFA_NTUPLE_FILTER_CFG_REQ_NEW_METER_INSTANCE_ID_INVALID 0xffffUL
-	__le16 unused_1[3];
+	#define CFA_NTUPLE_FILTER_CFG_REQ_NEW_METER_INSTANCE_ID_LAST   CFA_NTUPLE_FILTER_CFG_REQ_NEW_METER_INSTANCE_ID_INVALID
+	u8	unused_1[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_cfa_ntuple_filter_cfg_output (size:128b/16B) */
 struct hwrm_cfa_ntuple_filter_cfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_decap_filter_alloc */
-/* Input (104 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_decap_filter_alloc_input (size:832b/104B) */
 struct hwrm_cfa_decap_filter_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	#define CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL	    0x1UL
-	__le32 enables;
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE     0x1UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID       0x2UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR     0x4UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR     0x8UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_OVLAN_VID       0x10UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IVLAN_VID       0x20UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_OVLAN_VID     0x40UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID     0x80UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE       0x100UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR      0x200UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR      0x400UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE     0x800UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL     0x1000UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_PORT	    0x2000UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT	    0x4000UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_ID	    0x8000UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID  0x10000UL
-	__be32 tunnel_id;
-	u8 tunnel_type;
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL  0x0UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN      0x1UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE      0x2UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE      0x3UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP       0x4UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE     0x5UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS       0x6UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT	   0x7UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE      0x8UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4   0x9UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL  0xffUL
-	u8 unused_0;
-	__le16 unused_1;
-	u8 src_macaddr[6];
-	u8 unused_2;
-	u8 unused_3;
-	u8 dst_macaddr[6];
-	__be16 ovlan_vid;
-	__be16 ivlan_vid;
-	__be16 t_ovlan_vid;
-	__be16 t_ivlan_vid;
-	__be16 ethertype;
-	u8 ip_addr_type;
-	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN   0x0UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4      0x4UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6      0x6UL
-	u8 ip_protocol;
-	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN    0x0UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP	   0x6UL
-	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP	   0x11UL
-	u8 unused_4;
-	u8 unused_5;
-	u8 unused_6[3];
-	u8 unused_7;
-	__be32 src_ipaddr[4];
-	__be32 dst_ipaddr[4];
-	__be16 src_port;
-	__be16 dst_port;
-	__le16 dst_id;
-	__le16 l2_ctxt_ref_id;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	#define CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL     0x1UL
+	__le32	enables;
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE        0x1UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID          0x2UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR        0x4UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR        0x8UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_OVLAN_VID          0x10UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IVLAN_VID          0x20UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_OVLAN_VID        0x40UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID        0x80UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE          0x100UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR         0x200UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR         0x400UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE        0x800UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL        0x1000UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_PORT           0x2000UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT           0x4000UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_ID             0x8000UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID     0x10000UL
+	__be32	tunnel_id;
+	u8	tunnel_type;
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL 0x0UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN     0x1UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE     0x2UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE     0x3UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP      0x4UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE    0x5UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS      0x6UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT       0x7UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE     0x8UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4  0x9UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL 0xffUL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST     CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL
+	u8	unused_0;
+	__le16	unused_1;
+	u8	src_macaddr[6];
+	u8	unused_2[2];
+	u8	dst_macaddr[6];
+	__be16	ovlan_vid;
+	__be16	ivlan_vid;
+	__be16	t_ovlan_vid;
+	__be16	t_ivlan_vid;
+	__be16	ethertype;
+	u8	ip_addr_type;
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN 0x0UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4    0x4UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6    0x6UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_LAST   CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6
+	u8	ip_protocol;
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN 0x0UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP     0x6UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP     0x11UL
+	#define CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_LAST   CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP
+	__le16	unused_3;
+	__le32	unused_4;
+	__be32	src_ipaddr[4];
+	__be32	dst_ipaddr[4];
+	__be16	src_port;
+	__be16	dst_port;
+	__le16	dst_id;
+	__le16	l2_ctxt_ref_id;
+};
+
+/* hwrm_cfa_decap_filter_alloc_output (size:128b/16B) */
 struct hwrm_cfa_decap_filter_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 decap_filter_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_cfa_decap_filter_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	decap_filter_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_cfa_decap_filter_free_input (size:192b/24B) */
 struct hwrm_cfa_decap_filter_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 decap_filter_id;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	decap_filter_id;
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_cfa_decap_filter_free_output (size:128b/16B) */
 struct hwrm_cfa_decap_filter_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_flow_alloc */
-/* Input (128 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_flow_alloc_input (size:1024b/128B) */
 struct hwrm_cfa_flow_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 flags;
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL		    0x1UL
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_MASK		    0x6UL
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_SFT		    1
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_NONE		   (0x0UL << 1)
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE		   (0x1UL << 1)
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO		   (0x2UL << 1)
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_LAST    CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_MASK		    0x38UL
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_SFT		    3
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2		   (0x0UL << 3)
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4		   (0x1UL << 3)
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6		   (0x2UL << 3)
-	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_LAST    CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6
-	__le16 src_fid;
-	__le32 tunnel_handle;
-	__le16 action_flags;
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD		    0x1UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_RECYCLE	    0x2UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP		    0x4UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_METER		    0x8UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL		    0x10UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_SRC	    0x20UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_DEST	    0x40UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_IPV4_ADDRESS   0x80UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE  0x100UL
-	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TTL_DECREMENT      0x200UL
-	__le16 dst_fid;
-	__be16 l2_rewrite_vlan_tpid;
-	__be16 l2_rewrite_vlan_tci;
-	__le16 act_meter_id;
-	__le16 ref_flow_handle;
-	__be16 ethertype;
-	__be16 outer_vlan_tci;
-	__be16 dmac[3];
-	__be16 inner_vlan_tci;
-	__be16 smac[3];
-	u8 ip_dst_mask_len;
-	u8 ip_src_mask_len;
-	__be32 ip_dst[4];
-	__be32 ip_src[4];
-	__be16 l4_src_port;
-	__be16 l4_src_port_mask;
-	__be16 l4_dst_port;
-	__be16 l4_dst_port_mask;
-	__be32 nat_ip_address[4];
-	__be16 l2_rewrite_dmac[3];
-	__be16 nat_port;
-	__be16 l2_rewrite_smac[3];
-	u8 ip_proto;
-	u8 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	flags;
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL       0x1UL
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_MASK 0x6UL
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_SFT 1
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_NONE  (0x0UL << 1)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE   (0x1UL << 1)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO   (0x2UL << 1)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_LAST CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_TWO
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_MASK 0x38UL
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_SFT 3
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2    (0x0UL << 3)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4  (0x1UL << 3)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6  (0x2UL << 3)
+	#define CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_LAST CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6
+	__le16	src_fid;
+	__le32	tunnel_handle;
+	__le16	action_flags;
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD                   0x1UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_RECYCLE               0x2UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP                  0x4UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_METER                 0x8UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL                0x10UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_SRC               0x20UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_DEST              0x40UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_NAT_IPV4_ADDRESS      0x80UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE     0x100UL
+	#define CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TTL_DECREMENT         0x200UL
+	__le16	dst_fid;
+	__be16	l2_rewrite_vlan_tpid;
+	__be16	l2_rewrite_vlan_tci;
+	__le16	act_meter_id;
+	__le16	ref_flow_handle;
+	__be16	ethertype;
+	__be16	outer_vlan_tci;
+	__be16	dmac[3];
+	__be16	inner_vlan_tci;
+	__be16	smac[3];
+	u8	ip_dst_mask_len;
+	u8	ip_src_mask_len;
+	__be32	ip_dst[4];
+	__be32	ip_src[4];
+	__be16	l4_src_port;
+	__be16	l4_src_port_mask;
+	__be16	l4_dst_port;
+	__be16	l4_dst_port_mask;
+	__be32	nat_ip_address[4];
+	__be16	l2_rewrite_dmac[3];
+	__be16	nat_port;
+	__be16	l2_rewrite_smac[3];
+	u8	ip_proto;
+	u8	unused_0;
+};
+
+/* hwrm_cfa_flow_alloc_output (size:128b/16B) */
 struct hwrm_cfa_flow_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 flow_handle;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_cfa_flow_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	flow_handle;
+	u8	unused_0[5];
+	u8	valid;
+};
+
+/* hwrm_cfa_flow_free_input (size:192b/24B) */
 struct hwrm_cfa_flow_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 flow_handle;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	flow_handle;
+	u8	unused_0[6];
 };
 
-/* Output (32 bytes) */
+/* hwrm_cfa_flow_free_output (size:256b/32B) */
 struct hwrm_cfa_flow_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 packet;
-	__le64 byte;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_flow_stats */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	packet;
+	__le64	byte;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_flow_stats_input (size:320b/40B) */
 struct hwrm_cfa_flow_stats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 num_flows;
-	__le16 flow_handle_0;
-	__le16 flow_handle_1;
-	__le16 flow_handle_2;
-	__le16 flow_handle_3;
-	__le16 flow_handle_4;
-	__le16 flow_handle_5;
-	__le16 flow_handle_6;
-	__le16 flow_handle_7;
-	__le16 flow_handle_8;
-	__le16 flow_handle_9;
-	__le16 unused_0;
-};
-
-/* Output (176 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	num_flows;
+	__le16	flow_handle_0;
+	__le16	flow_handle_1;
+	__le16	flow_handle_2;
+	__le16	flow_handle_3;
+	__le16	flow_handle_4;
+	__le16	flow_handle_5;
+	__le16	flow_handle_6;
+	__le16	flow_handle_7;
+	__le16	flow_handle_8;
+	__le16	flow_handle_9;
+	u8	unused_0[2];
+};
+
+/* hwrm_cfa_flow_stats_output (size:1408b/176B) */
 struct hwrm_cfa_flow_stats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 packet_0;
-	__le64 packet_1;
-	__le64 packet_2;
-	__le64 packet_3;
-	__le64 packet_4;
-	__le64 packet_5;
-	__le64 packet_6;
-	__le64 packet_7;
-	__le64 packet_8;
-	__le64 packet_9;
-	__le64 byte_0;
-	__le64 byte_1;
-	__le64 byte_2;
-	__le64 byte_3;
-	__le64 byte_4;
-	__le64 byte_5;
-	__le64 byte_6;
-	__le64 byte_7;
-	__le64 byte_8;
-	__le64 byte_9;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_cfa_vfr_alloc */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	packet_0;
+	__le64	packet_1;
+	__le64	packet_2;
+	__le64	packet_3;
+	__le64	packet_4;
+	__le64	packet_5;
+	__le64	packet_6;
+	__le64	packet_7;
+	__le64	packet_8;
+	__le64	packet_9;
+	__le64	byte_0;
+	__le64	byte_1;
+	__le64	byte_2;
+	__le64	byte_3;
+	__le64	byte_4;
+	__le64	byte_5;
+	__le64	byte_6;
+	__le64	byte_7;
+	__le64	byte_8;
+	__le64	byte_9;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_cfa_vfr_alloc_input (size:448b/56B) */
 struct hwrm_cfa_vfr_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 vf_id;
-	__le16 reserved;
-	__le32 unused_0;
-	char vfr_name[32];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	vf_id;
+	__le16	reserved;
+	u8	unused_0[4];
+	char	vfr_name[32];
+};
+
+/* hwrm_cfa_vfr_alloc_output (size:128b/16B) */
 struct hwrm_cfa_vfr_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 rx_cfa_code;
-	__le16 tx_cfa_action;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_cfa_vfr_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	rx_cfa_code;
+	__le16	tx_cfa_action;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_cfa_vfr_free_input (size:384b/48B) */
 struct hwrm_cfa_vfr_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	char vfr_name[32];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	char	vfr_name[32];
 };
 
-/* Output (16 bytes) */
+/* hwrm_cfa_vfr_free_output (size:128b/16B) */
 struct hwrm_cfa_vfr_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_tunnel_dst_port_query */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_tunnel_dst_port_query_input (size:192b/24B) */
 struct hwrm_tunnel_dst_port_query_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 tunnel_type;
-	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN       0x1UL
-	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_GENEVE      0x5UL
-	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN_V4    0x9UL
-	u8 unused_0[7];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	tunnel_type;
+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN    0x1UL
+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_GENEVE   0x5UL
+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN_V4 0x9UL
+	#define TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_LAST    TUNNEL_DST_PORT_QUERY_REQ_TUNNEL_TYPE_VXLAN_V4
+	u8	unused_0[7];
+};
+
+/* hwrm_tunnel_dst_port_query_output (size:128b/16B) */
 struct hwrm_tunnel_dst_port_query_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 tunnel_dst_port_id;
-	__be16 tunnel_dst_port_val;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_tunnel_dst_port_alloc */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	tunnel_dst_port_id;
+	__be16	tunnel_dst_port_val;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_tunnel_dst_port_alloc_input (size:192b/24B) */
 struct hwrm_tunnel_dst_port_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 tunnel_type;
-	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN       0x1UL
-	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE      0x5UL
-	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4    0x9UL
-	u8 unused_0;
-	__be16 tunnel_dst_port_val;
-	__be32 unused_1;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	tunnel_type;
+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN    0x1UL
+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE   0x5UL
+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4 0x9UL
+	#define TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_LAST    TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4
+	u8	unused_0;
+	__be16	tunnel_dst_port_val;
+	u8	unused_1[4];
+};
+
+/* hwrm_tunnel_dst_port_alloc_output (size:128b/16B) */
 struct hwrm_tunnel_dst_port_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 tunnel_dst_port_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_tunnel_dst_port_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	tunnel_dst_port_id;
+	u8	unused_0[5];
+	u8	valid;
+};
+
+/* hwrm_tunnel_dst_port_free_input (size:192b/24B) */
 struct hwrm_tunnel_dst_port_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 tunnel_type;
-	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN	   0x1UL
-	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE       0x5UL
-	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_V4     0x9UL
-	u8 unused_0;
-	__le16 tunnel_dst_port_id;
-	__le32 unused_1;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	tunnel_type;
+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN    0x1UL
+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE   0x5UL
+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_V4 0x9UL
+	#define TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_LAST    TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_V4
+	u8	unused_0;
+	__le16	tunnel_dst_port_id;
+	u8	unused_1[4];
+};
+
+/* hwrm_tunnel_dst_port_free_output (size:128b/16B) */
 struct hwrm_tunnel_dst_port_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_stat_ctx_alloc */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_1[7];
+	u8	valid;
+};
+
+/* ctx_hw_stats (size:1280b/160B) */
+struct ctx_hw_stats {
+	__le64	rx_ucast_pkts;
+	__le64	rx_mcast_pkts;
+	__le64	rx_bcast_pkts;
+	__le64	rx_discard_pkts;
+	__le64	rx_drop_pkts;
+	__le64	rx_ucast_bytes;
+	__le64	rx_mcast_bytes;
+	__le64	rx_bcast_bytes;
+	__le64	tx_ucast_pkts;
+	__le64	tx_mcast_pkts;
+	__le64	tx_bcast_pkts;
+	__le64	tx_discard_pkts;
+	__le64	tx_drop_pkts;
+	__le64	tx_ucast_bytes;
+	__le64	tx_mcast_bytes;
+	__le64	tx_bcast_bytes;
+	__le64	tpa_pkts;
+	__le64	tpa_bytes;
+	__le64	tpa_events;
+	__le64	tpa_aborts;
+};
+
+/* hwrm_stat_ctx_alloc_input (size:256b/32B) */
 struct hwrm_stat_ctx_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 stats_dma_addr;
-	__le32 update_period_ms;
-	u8 stat_ctx_flags;
-	#define STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE		    0x1UL
-	u8 unused_0[3];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	stats_dma_addr;
+	__le32	update_period_ms;
+	u8	stat_ctx_flags;
+	#define STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE     0x1UL
+	u8	unused_0[3];
+};
+
+/* hwrm_stat_ctx_alloc_output (size:128b/16B) */
 struct hwrm_stat_ctx_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 stat_ctx_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_stat_ctx_free */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	stat_ctx_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_stat_ctx_free_input (size:192b/24B) */
 struct hwrm_stat_ctx_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 stat_ctx_id;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	stat_ctx_id;
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_stat_ctx_free_output (size:128b/16B) */
 struct hwrm_stat_ctx_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 stat_ctx_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_stat_ctx_query */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	stat_ctx_id;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_stat_ctx_query_input (size:192b/24B) */
 struct hwrm_stat_ctx_query_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 stat_ctx_id;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	stat_ctx_id;
+	u8	unused_0[4];
 };
 
-/* Output (176 bytes) */
+/* hwrm_stat_ctx_query_output (size:1408b/176B) */
 struct hwrm_stat_ctx_query_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 tx_ucast_pkts;
-	__le64 tx_mcast_pkts;
-	__le64 tx_bcast_pkts;
-	__le64 tx_err_pkts;
-	__le64 tx_drop_pkts;
-	__le64 tx_ucast_bytes;
-	__le64 tx_mcast_bytes;
-	__le64 tx_bcast_bytes;
-	__le64 rx_ucast_pkts;
-	__le64 rx_mcast_pkts;
-	__le64 rx_bcast_pkts;
-	__le64 rx_err_pkts;
-	__le64 rx_drop_pkts;
-	__le64 rx_ucast_bytes;
-	__le64 rx_mcast_bytes;
-	__le64 rx_bcast_bytes;
-	__le64 rx_agg_pkts;
-	__le64 rx_agg_bytes;
-	__le64 rx_agg_events;
-	__le64 rx_agg_aborts;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_stat_ctx_clr_stats */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	tx_ucast_pkts;
+	__le64	tx_mcast_pkts;
+	__le64	tx_bcast_pkts;
+	__le64	tx_err_pkts;
+	__le64	tx_drop_pkts;
+	__le64	tx_ucast_bytes;
+	__le64	tx_mcast_bytes;
+	__le64	tx_bcast_bytes;
+	__le64	rx_ucast_pkts;
+	__le64	rx_mcast_pkts;
+	__le64	rx_bcast_pkts;
+	__le64	rx_err_pkts;
+	__le64	rx_drop_pkts;
+	__le64	rx_ucast_bytes;
+	__le64	rx_mcast_bytes;
+	__le64	rx_bcast_bytes;
+	__le64	rx_agg_pkts;
+	__le64	rx_agg_bytes;
+	__le64	rx_agg_events;
+	__le64	rx_agg_aborts;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_stat_ctx_clr_stats_input (size:192b/24B) */
 struct hwrm_stat_ctx_clr_stats_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 stat_ctx_id;
-	__le32 unused_0;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	stat_ctx_id;
+	u8	unused_0[4];
 };
 
-/* Output (16 bytes) */
+/* hwrm_stat_ctx_clr_stats_output (size:128b/16B) */
 struct hwrm_stat_ctx_clr_stats_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_fw_reset */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* tx_port_stats (size:3264b/408B) */
+struct tx_port_stats {
+	__le64	tx_64b_frames;
+	__le64	tx_65b_127b_frames;
+	__le64	tx_128b_255b_frames;
+	__le64	tx_256b_511b_frames;
+	__le64	tx_512b_1023b_frames;
+	__le64	tx_1024b_1518_frames;
+	__le64	tx_good_vlan_frames;
+	__le64	tx_1519b_2047_frames;
+	__le64	tx_2048b_4095b_frames;
+	__le64	tx_4096b_9216b_frames;
+	__le64	tx_9217b_16383b_frames;
+	__le64	tx_good_frames;
+	__le64	tx_total_frames;
+	__le64	tx_ucast_frames;
+	__le64	tx_mcast_frames;
+	__le64	tx_bcast_frames;
+	__le64	tx_pause_frames;
+	__le64	tx_pfc_frames;
+	__le64	tx_jabber_frames;
+	__le64	tx_fcs_err_frames;
+	__le64	tx_control_frames;
+	__le64	tx_oversz_frames;
+	__le64	tx_single_dfrl_frames;
+	__le64	tx_multi_dfrl_frames;
+	__le64	tx_single_coll_frames;
+	__le64	tx_multi_coll_frames;
+	__le64	tx_late_coll_frames;
+	__le64	tx_excessive_coll_frames;
+	__le64	tx_frag_frames;
+	__le64	tx_err;
+	__le64	tx_tagged_frames;
+	__le64	tx_dbl_tagged_frames;
+	__le64	tx_runt_frames;
+	__le64	tx_fifo_underruns;
+	__le64	tx_pfc_ena_frames_pri0;
+	__le64	tx_pfc_ena_frames_pri1;
+	__le64	tx_pfc_ena_frames_pri2;
+	__le64	tx_pfc_ena_frames_pri3;
+	__le64	tx_pfc_ena_frames_pri4;
+	__le64	tx_pfc_ena_frames_pri5;
+	__le64	tx_pfc_ena_frames_pri6;
+	__le64	tx_pfc_ena_frames_pri7;
+	__le64	tx_eee_lpi_events;
+	__le64	tx_eee_lpi_duration;
+	__le64	tx_llfc_logical_msgs;
+	__le64	tx_hcfc_msgs;
+	__le64	tx_total_collisions;
+	__le64	tx_bytes;
+	__le64	tx_xthol_frames;
+	__le64	tx_stat_discard;
+	__le64	tx_stat_error;
+};
+
+/* rx_port_stats (size:4224b/528B) */
+struct rx_port_stats {
+	__le64	rx_64b_frames;
+	__le64	rx_65b_127b_frames;
+	__le64	rx_128b_255b_frames;
+	__le64	rx_256b_511b_frames;
+	__le64	rx_512b_1023b_frames;
+	__le64	rx_1024b_1518_frames;
+	__le64	rx_good_vlan_frames;
+	__le64	rx_1519b_2047b_frames;
+	__le64	rx_2048b_4095b_frames;
+	__le64	rx_4096b_9216b_frames;
+	__le64	rx_9217b_16383b_frames;
+	__le64	rx_total_frames;
+	__le64	rx_ucast_frames;
+	__le64	rx_mcast_frames;
+	__le64	rx_bcast_frames;
+	__le64	rx_fcs_err_frames;
+	__le64	rx_ctrl_frames;
+	__le64	rx_pause_frames;
+	__le64	rx_pfc_frames;
+	__le64	rx_unsupported_opcode_frames;
+	__le64	rx_unsupported_da_pausepfc_frames;
+	__le64	rx_wrong_sa_frames;
+	__le64	rx_align_err_frames;
+	__le64	rx_oor_len_frames;
+	__le64	rx_code_err_frames;
+	__le64	rx_false_carrier_frames;
+	__le64	rx_ovrsz_frames;
+	__le64	rx_jbr_frames;
+	__le64	rx_mtu_err_frames;
+	__le64	rx_match_crc_frames;
+	__le64	rx_promiscuous_frames;
+	__le64	rx_tagged_frames;
+	__le64	rx_double_tagged_frames;
+	__le64	rx_trunc_frames;
+	__le64	rx_good_frames;
+	__le64	rx_pfc_xon2xoff_frames_pri0;
+	__le64	rx_pfc_xon2xoff_frames_pri1;
+	__le64	rx_pfc_xon2xoff_frames_pri2;
+	__le64	rx_pfc_xon2xoff_frames_pri3;
+	__le64	rx_pfc_xon2xoff_frames_pri4;
+	__le64	rx_pfc_xon2xoff_frames_pri5;
+	__le64	rx_pfc_xon2xoff_frames_pri6;
+	__le64	rx_pfc_xon2xoff_frames_pri7;
+	__le64	rx_pfc_ena_frames_pri0;
+	__le64	rx_pfc_ena_frames_pri1;
+	__le64	rx_pfc_ena_frames_pri2;
+	__le64	rx_pfc_ena_frames_pri3;
+	__le64	rx_pfc_ena_frames_pri4;
+	__le64	rx_pfc_ena_frames_pri5;
+	__le64	rx_pfc_ena_frames_pri6;
+	__le64	rx_pfc_ena_frames_pri7;
+	__le64	rx_sch_crc_err_frames;
+	__le64	rx_undrsz_frames;
+	__le64	rx_frag_frames;
+	__le64	rx_eee_lpi_events;
+	__le64	rx_eee_lpi_duration;
+	__le64	rx_llfc_physical_msgs;
+	__le64	rx_llfc_logical_msgs;
+	__le64	rx_llfc_msgs_with_crc_err;
+	__le64	rx_hcfc_msgs;
+	__le64	rx_hcfc_msgs_with_crc_err;
+	__le64	rx_bytes;
+	__le64	rx_runt_bytes;
+	__le64	rx_runt_frames;
+	__le64	rx_stat_discard;
+	__le64	rx_stat_err;
+};
+
+/* hwrm_fw_reset_input (size:192b/24B) */
 struct hwrm_fw_reset_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 embedded_proc_type;
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_BOOT		   0x0UL
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_MGMT		   0x1UL
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   0x2UL
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_ROCE		   0x3UL
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_HOST		   0x4UL
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_AP		   0x5UL
-	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP		   0x6UL
-	u8 selfrst_status;
-	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTNONE	   0x0UL
-	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP	   0x1UL
-	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTPCIERST	   0x2UL
-	u8 host_idx;
-	u8 unused_0[5];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	embedded_proc_type;
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_BOOT    0x0UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_MGMT    0x1UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_NETCTRL 0x2UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_ROCE    0x3UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_HOST    0x4UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_AP      0x5UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP    0x6UL
+	#define FW_RESET_REQ_EMBEDDED_PROC_TYPE_LAST   FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP
+	u8	selfrst_status;
+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTNONE    0x0UL
+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP    0x1UL
+	#define FW_RESET_REQ_SELFRST_STATUS_SELFRSTPCIERST 0x2UL
+	#define FW_RESET_REQ_SELFRST_STATUS_LAST          FW_RESET_REQ_SELFRST_STATUS_SELFRSTPCIERST
+	u8	host_idx;
+	u8	unused_0[5];
+};
+
+/* hwrm_fw_reset_output (size:128b/16B) */
 struct hwrm_fw_reset_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 selfrst_status;
-	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTNONE	   0x0UL
-	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTASAP	   0x1UL
-	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTPCIERST       0x2UL
-	u8 unused_0;
-	__le16 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_fw_qstatus */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	selfrst_status;
+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTNONE    0x0UL
+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTASAP    0x1UL
+	#define FW_RESET_RESP_SELFRST_STATUS_SELFRSTPCIERST 0x2UL
+	#define FW_RESET_RESP_SELFRST_STATUS_LAST          FW_RESET_RESP_SELFRST_STATUS_SELFRSTPCIERST
+	u8	unused_0[6];
+	u8	valid;
+};
+
+/* hwrm_fw_qstatus_input (size:192b/24B) */
 struct hwrm_fw_qstatus_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 embedded_proc_type;
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_BOOT		   0x0UL
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_MGMT		   0x1UL
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_NETCTRL	   0x2UL
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_ROCE		   0x3UL
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_HOST		   0x4UL
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_AP		   0x5UL
-	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_CHIP		   0x6UL
-	u8 unused_0[7];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	embedded_proc_type;
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_BOOT    0x0UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_MGMT    0x1UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_NETCTRL 0x2UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_ROCE    0x3UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_HOST    0x4UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_AP      0x5UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_CHIP    0x6UL
+	#define FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_LAST   FW_QSTATUS_REQ_EMBEDDED_PROC_TYPE_CHIP
+	u8	unused_0[7];
+};
+
+/* hwrm_fw_qstatus_output (size:128b/16B) */
 struct hwrm_fw_qstatus_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 selfrst_status;
-	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTNONE	   0x0UL
-	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTASAP	   0x1UL
-	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPCIERST     0x2UL
-	u8 unused_0;
-	__le16 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_fw_set_time */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	selfrst_status;
+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTNONE    0x0UL
+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTASAP    0x1UL
+	#define FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPCIERST 0x2UL
+	#define FW_QSTATUS_RESP_SELFRST_STATUS_LAST          FW_QSTATUS_RESP_SELFRST_STATUS_SELFRSTPCIERST
+	u8	unused_0[6];
+	u8	valid;
+};
+
+/* hwrm_fw_set_time_input (size:256b/32B) */
 struct hwrm_fw_set_time_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 year;
-	#define FW_SET_TIME_REQ_YEAR_UNKNOWN			   0x0UL
-	u8 month;
-	u8 day;
-	u8 hour;
-	u8 minute;
-	u8 second;
-	u8 unused_0;
-	__le16 millisecond;
-	__le16 zone;
-	#define FW_SET_TIME_REQ_ZONE_UTC			   0x0UL
-	#define FW_SET_TIME_REQ_ZONE_UNKNOWN			   0xffffUL
-	__le32 unused_1;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	year;
+	#define FW_SET_TIME_REQ_YEAR_UNKNOWN 0x0UL
+	#define FW_SET_TIME_REQ_YEAR_LAST   FW_SET_TIME_REQ_YEAR_UNKNOWN
+	u8	month;
+	u8	day;
+	u8	hour;
+	u8	minute;
+	u8	second;
+	u8	unused_0;
+	__le16	millisecond;
+	__le16	zone;
+	#define FW_SET_TIME_REQ_ZONE_UTC     0x0UL
+	#define FW_SET_TIME_REQ_ZONE_UNKNOWN 0xffffUL
+	#define FW_SET_TIME_REQ_ZONE_LAST   FW_SET_TIME_REQ_ZONE_UNKNOWN
+	u8	unused_1[4];
+};
+
+/* hwrm_fw_set_time_output (size:128b/16B) */
 struct hwrm_fw_set_time_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_fw_set_structured_data */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_struct_hdr (size:128b/16B) */
+struct hwrm_struct_hdr {
+	__le16	struct_id;
+	#define STRUCT_HDR_STRUCT_ID_LLDP_CFG           0x41bUL
+	#define STRUCT_HDR_STRUCT_ID_DCBX_ETS           0x41dUL
+	#define STRUCT_HDR_STRUCT_ID_DCBX_PFC           0x41fUL
+	#define STRUCT_HDR_STRUCT_ID_DCBX_APP           0x421UL
+	#define STRUCT_HDR_STRUCT_ID_DCBX_FEATURE_STATE 0x422UL
+	#define STRUCT_HDR_STRUCT_ID_LLDP_GENERIC       0x424UL
+	#define STRUCT_HDR_STRUCT_ID_LLDP_DEVICE        0x426UL
+	#define STRUCT_HDR_STRUCT_ID_AFM_OPAQUE         0x1UL
+	#define STRUCT_HDR_STRUCT_ID_PORT_DESCRIPTION   0xaUL
+	#define STRUCT_HDR_STRUCT_ID_RSS_V2             0x64UL
+	#define STRUCT_HDR_STRUCT_ID_LAST              STRUCT_HDR_STRUCT_ID_RSS_V2
+	__le16	len;
+	u8	version;
+	u8	count;
+	__le16	subtype;
+	__le16	next_offset;
+	#define STRUCT_HDR_NEXT_OFFSET_LAST 0x0UL
+	u8	unused_0[6];
+};
+
+/* hwrm_struct_data_dcbx_app (size:64b/8B) */
+struct hwrm_struct_data_dcbx_app {
+	__be16	protocol_id;
+	u8	protocol_selector;
+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_ETHER_TYPE   0x1UL
+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_PORT     0x2UL
+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_UDP_PORT     0x3UL
+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_UDP_PORT 0x4UL
+	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_LAST        STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_UDP_PORT
+	u8	priority;
+	u8	valid;
+	u8	unused_0[3];
+};
+
+/* hwrm_fw_set_structured_data_input (size:256b/32B) */
 struct hwrm_fw_set_structured_data_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 src_data_addr;
-	__le16 data_len;
-	u8 hdr_cnt;
-	u8 unused_0[5];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	src_data_addr;
+	__le16	data_len;
+	u8	hdr_cnt;
+	u8	unused_0[5];
+};
+
+/* hwrm_fw_set_structured_data_output (size:128b/16B) */
 struct hwrm_fw_set_structured_data_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* Command specific Error Codes (8 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_fw_set_structured_data_cmd_err (size:64b/8B) */
 struct hwrm_fw_set_structured_data_cmd_err {
-	u8 code;
-	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN       0x0UL
-	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_HDR_CNT   0x1UL
-	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_FMT       0x2UL
-	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID	   0x3UL
-	u8 unused_0[7];
+	u8	code;
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN     0x0UL
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_HDR_CNT 0x1UL
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_FMT     0x2UL
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID      0x3UL
+	#define FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_LAST       FW_SET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID
+	u8	unused_0[7];
 };
 
-/* hwrm_fw_get_structured_data */
-/* Input (32 bytes) */
+/* hwrm_fw_get_structured_data_input (size:256b/32B) */
 struct hwrm_fw_get_structured_data_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 dest_data_addr;
-	__le16 data_len;
-	__le16 structure_id;
-	__le16 subtype;
-	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_ALL		   0xffffUL
-	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_ADMIN 0x100UL
-	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_PEER 0x101UL
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	dest_data_addr;
+	__le16	data_len;
+	__le16	structure_id;
+	__le16	subtype;
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_UNUSED                  0x0UL
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_ALL                     0xffffUL
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_ADMIN       0x100UL
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_PEER        0x101UL
 	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NEAR_BRIDGE_OPERATIONAL 0x102UL
-	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_ADMIN 0x200UL
-	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_PEER  0x201UL
-	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_OPERATIONAL 0x202UL
-	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_HOST_OPERATIONAL 0x300UL
-	u8 count;
-	u8 unused_0;
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_ADMIN          0x200UL
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_PEER           0x201UL
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_NON_TPMR_OPERATIONAL    0x202UL
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_HOST_OPERATIONAL        0x300UL
+	#define FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_LAST                   FW_GET_STRUCTURED_DATA_REQ_SUBTYPE_HOST_OPERATIONAL
+	u8	count;
+	u8	unused_0;
 };
 
-/* Output (16 bytes) */
+/* hwrm_fw_get_structured_data_output (size:128b/16B) */
 struct hwrm_fw_get_structured_data_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 hdr_cnt;
-	u8 unused_0;
-	__le16 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* Command specific Error Codes (8 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	hdr_cnt;
+	u8	unused_0[6];
+	u8	valid;
+};
+
+/* hwrm_fw_get_structured_data_cmd_err (size:64b/8B) */
 struct hwrm_fw_get_structured_data_cmd_err {
-	u8 code;
-	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN       0x0UL
-	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID	   0x3UL
-	u8 unused_0[7];
+	u8	code;
+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_UNKNOWN 0x0UL
+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID  0x3UL
+	#define FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_LAST   FW_GET_STRUCTURED_DATA_CMD_ERR_CODE_BAD_ID
+	u8	unused_0[7];
 };
 
-/* hwrm_exec_fwd_resp */
-/* Input (128 bytes) */
+/* hwrm_exec_fwd_resp_input (size:1024b/128B) */
 struct hwrm_exec_fwd_resp_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 encap_request[26];
-	__le16 encap_resp_target_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	encap_request[26];
+	__le16	encap_resp_target_id;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_exec_fwd_resp_output (size:128b/16B) */
 struct hwrm_exec_fwd_resp_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_reject_fwd_resp */
-/* Input (128 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_reject_fwd_resp_input (size:1024b/128B) */
 struct hwrm_reject_fwd_resp_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 encap_request[26];
-	__le16 encap_resp_target_id;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	encap_request[26];
+	__le16	encap_resp_target_id;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_reject_fwd_resp_output (size:128b/16B) */
 struct hwrm_reject_fwd_resp_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_fwd_resp */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_fwd_resp_input (size:1024b/128B) */
 struct hwrm_fwd_resp_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 encap_resp_target_id;
-	__le16 encap_resp_cmpl_ring;
-	__le16 encap_resp_len;
-	u8 unused_0;
-	u8 unused_1;
-	__le64 encap_resp_addr;
-	__le32 encap_resp[24];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	encap_resp_target_id;
+	__le16	encap_resp_cmpl_ring;
+	__le16	encap_resp_len;
+	u8	unused_0;
+	u8	unused_1;
+	__le64	encap_resp_addr;
+	__le32	encap_resp[24];
+};
+
+/* hwrm_fwd_resp_output (size:128b/16B) */
 struct hwrm_fwd_resp_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_fwd_async_event_cmpl */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_fwd_async_event_cmpl_input (size:320b/40B) */
 struct hwrm_fwd_async_event_cmpl_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 encap_async_event_target_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2[3];
-	u8 unused_3;
-	__le32 encap_async_event_cmpl[4];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	encap_async_event_target_id;
+	u8	unused_0[6];
+	__le32	encap_async_event_cmpl[4];
+};
+
+/* hwrm_fwd_async_event_cmpl_output (size:128b/16B) */
 struct hwrm_fwd_async_event_cmpl_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_temp_monitor_query */
-/* Input (16 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_temp_monitor_query_input (size:128b/16B) */
 struct hwrm_temp_monitor_query_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (16 bytes) */
+/* hwrm_temp_monitor_query_output (size:128b/16B) */
 struct hwrm_temp_monitor_query_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 temp;
-	u8 unused_0;
-	__le16 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_wol_filter_alloc */
-/* Input (64 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	temp;
+	u8	unused_0[6];
+	u8	valid;
+};
+
+/* hwrm_wol_filter_alloc_input (size:512b/64B) */
 struct hwrm_wol_filter_alloc_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
-	__le32 enables;
-	#define WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS	    0x1UL
-	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_OFFSET	    0x2UL
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
+	__le32	enables;
+	#define WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS           0x1UL
+	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_OFFSET        0x2UL
 	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_BUF_SIZE      0x4UL
 	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_BUF_ADDR      0x8UL
 	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_MASK_ADDR     0x10UL
 	#define WOL_FILTER_ALLOC_REQ_ENABLES_PATTERN_MASK_SIZE     0x20UL
-	__le16 port_id;
-	u8 wol_type;
-	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT		   0x0UL
-	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_BMP		   0x1UL
-	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_INVALID		   0xffUL
-	u8 unused_0;
-	__le32 unused_1;
-	u8 mac_address[6];
-	__le16 pattern_offset;
-	__le16 pattern_buf_size;
-	__le16 pattern_mask_size;
-	__le32 unused_2;
-	__le64 pattern_buf_addr;
-	__le64 pattern_mask_addr;
-};
-
-/* Output (16 bytes) */
+	__le16	port_id;
+	u8	wol_type;
+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT 0x0UL
+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_BMP      0x1UL
+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_INVALID  0xffUL
+	#define WOL_FILTER_ALLOC_REQ_WOL_TYPE_LAST    WOL_FILTER_ALLOC_REQ_WOL_TYPE_INVALID
+	u8	unused_0[5];
+	u8	mac_address[6];
+	__le16	pattern_offset;
+	__le16	pattern_buf_size;
+	__le16	pattern_mask_size;
+	u8	unused_1[4];
+	__le64	pattern_buf_addr;
+	__le64	pattern_mask_addr;
+};
+
+/* hwrm_wol_filter_alloc_output (size:128b/16B) */
 struct hwrm_wol_filter_alloc_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 wol_filter_id;
-	u8 unused_0;
-	__le16 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_wol_filter_free */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	wol_filter_id;
+	u8	unused_0[6];
+	u8	valid;
+};
+
+/* hwrm_wol_filter_free_input (size:256b/32B) */
 struct hwrm_wol_filter_free_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 flags;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	flags;
 	#define WOL_FILTER_FREE_REQ_FLAGS_FREE_ALL_WOL_FILTERS     0x1UL
-	__le32 enables;
-	#define WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID	    0x1UL
-	__le16 port_id;
-	u8 wol_filter_id;
-	u8 unused_0[5];
+	__le32	enables;
+	#define WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID     0x1UL
+	__le16	port_id;
+	u8	wol_filter_id;
+	u8	unused_0[5];
 };
 
-/* Output (16 bytes) */
+/* hwrm_wol_filter_free_output (size:128b/16B) */
 struct hwrm_wol_filter_free_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_wol_filter_qcfg */
-/* Input (56 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_wol_filter_qcfg_input (size:448b/56B) */
 struct hwrm_wol_filter_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	__le16 handle;
-	__le32 unused_0;
-	__le64 pattern_buf_addr;
-	__le16 pattern_buf_size;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3[3];
-	u8 unused_4;
-	__le64 pattern_mask_addr;
-	__le16 pattern_mask_size;
-	__le16 unused_5[3];
-};
-
-/* Output (32 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	__le16	handle;
+	u8	unused_0[4];
+	__le64	pattern_buf_addr;
+	__le16	pattern_buf_size;
+	u8	unused_1[6];
+	__le64	pattern_mask_addr;
+	__le16	pattern_mask_size;
+	u8	unused_2[6];
+};
+
+/* hwrm_wol_filter_qcfg_output (size:256b/32B) */
 struct hwrm_wol_filter_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 next_handle;
-	u8 wol_filter_id;
-	u8 wol_type;
-	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_MAGICPKT		   0x0UL
-	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_BMP		   0x1UL
-	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_INVALID		   0xffUL
-	__le32 unused_0;
-	u8 mac_address[6];
-	__le16 pattern_offset;
-	__le16 pattern_size;
-	__le16 pattern_mask_size;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_wol_reason_qcfg */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	next_handle;
+	u8	wol_filter_id;
+	u8	wol_type;
+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_MAGICPKT 0x0UL
+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_BMP      0x1UL
+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_INVALID  0xffUL
+	#define WOL_FILTER_QCFG_RESP_WOL_TYPE_LAST    WOL_FILTER_QCFG_RESP_WOL_TYPE_INVALID
+	__le32	unused_0;
+	u8	mac_address[6];
+	__le16	pattern_offset;
+	__le16	pattern_size;
+	__le16	pattern_mask_size;
+	u8	unused_1[3];
+	u8	valid;
+};
+
+/* hwrm_wol_reason_qcfg_input (size:320b/40B) */
 struct hwrm_wol_reason_qcfg_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 port_id;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2[3];
-	u8 unused_3;
-	__le64 wol_pkt_buf_addr;
-	__le16 wol_pkt_buf_size;
-	__le16 unused_4[3];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	u8	unused_0[6];
+	__le64	wol_pkt_buf_addr;
+	__le16	wol_pkt_buf_size;
+	u8	unused_1[6];
+};
+
+/* hwrm_wol_reason_qcfg_output (size:128b/16B) */
 struct hwrm_wol_reason_qcfg_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 wol_filter_id;
-	u8 wol_reason;
-	#define WOL_REASON_QCFG_RESP_WOL_REASON_MAGICPKT	   0x0UL
-	#define WOL_REASON_QCFG_RESP_WOL_REASON_BMP		   0x1UL
-	#define WOL_REASON_QCFG_RESP_WOL_REASON_INVALID	   0xffUL
-	u8 wol_pkt_len;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_dbg_read_direct */
-/* Input (32 bytes) */
-struct hwrm_dbg_read_direct_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 host_dest_addr;
-	__le32 read_addr;
-	__le32 read_len32;
-};
-
-/* Output (16 bytes) */
-struct hwrm_dbg_read_direct_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_read */
-/* Input (40 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	wol_filter_id;
+	u8	wol_reason;
+	#define WOL_REASON_QCFG_RESP_WOL_REASON_MAGICPKT 0x0UL
+	#define WOL_REASON_QCFG_RESP_WOL_REASON_BMP      0x1UL
+	#define WOL_REASON_QCFG_RESP_WOL_REASON_INVALID  0xffUL
+	#define WOL_REASON_QCFG_RESP_WOL_REASON_LAST    WOL_REASON_QCFG_RESP_WOL_REASON_INVALID
+	u8	wol_pkt_len;
+	u8	unused_0[4];
+	u8	valid;
+};
+
+/* hwrm_nvm_read_input (size:320b/40B) */
 struct hwrm_nvm_read_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 host_dest_addr;
-	__le16 dir_idx;
-	u8 unused_0;
-	u8 unused_1;
-	__le32 offset;
-	__le32 len;
-	__le32 unused_2;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_dest_addr;
+	__le16	dir_idx;
+	u8	unused_0[2];
+	__le32	offset;
+	__le32	len;
+	u8	unused_1[4];
+};
+
+/* hwrm_nvm_read_output (size:128b/16B) */
 struct hwrm_nvm_read_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_get_dir_entries */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_get_dir_entries_input (size:192b/24B) */
 struct hwrm_nvm_get_dir_entries_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 host_dest_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_dest_addr;
 };
 
-/* Output (16 bytes) */
+/* hwrm_nvm_get_dir_entries_output (size:128b/16B) */
 struct hwrm_nvm_get_dir_entries_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_get_dir_info */
-/* Input (16 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_get_dir_info_input (size:128b/16B) */
 struct hwrm_nvm_get_dir_info_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (24 bytes) */
+/* hwrm_nvm_get_dir_info_output (size:192b/24B) */
 struct hwrm_nvm_get_dir_info_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 entries;
-	__le32 entry_length;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_write */
-/* Input (48 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	entries;
+	__le32	entry_length;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_write_input (size:384b/48B) */
 struct hwrm_nvm_write_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 host_src_addr;
-	__le16 dir_type;
-	__le16 dir_ordinal;
-	__le16 dir_ext;
-	__le16 dir_attr;
-	__le32 dir_data_length;
-	__le16 option;
-	__le16 flags;
-	#define NVM_WRITE_REQ_FLAGS_KEEP_ORIG_ACTIVE_IMG	    0x1UL
-	__le32 dir_item_length;
-	__le32 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_src_addr;
+	__le16	dir_type;
+	__le16	dir_ordinal;
+	__le16	dir_ext;
+	__le16	dir_attr;
+	__le32	dir_data_length;
+	__le16	option;
+	__le16	flags;
+	#define NVM_WRITE_REQ_FLAGS_KEEP_ORIG_ACTIVE_IMG     0x1UL
+	__le32	dir_item_length;
+	__le32	unused_0;
+};
+
+/* hwrm_nvm_write_output (size:128b/16B) */
 struct hwrm_nvm_write_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 dir_item_length;
-	__le16 dir_idx;
-	u8 unused_0;
-	u8 valid;
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	dir_item_length;
+	__le16	dir_idx;
+	u8	unused_0;
+	u8	valid;
 };
 
-/* Command specific Error Codes (8 bytes) */
+/* hwrm_nvm_write_cmd_err (size:64b/8B) */
 struct hwrm_nvm_write_cmd_err {
-	u8 code;
-	#define NVM_WRITE_CMD_ERR_CODE_UNKNOWN			   0x0UL
-	#define NVM_WRITE_CMD_ERR_CODE_FRAG_ERR		   0x1UL
-	#define NVM_WRITE_CMD_ERR_CODE_NO_SPACE		   0x2UL
-	u8 unused_0[7];
+	u8	code;
+	#define NVM_WRITE_CMD_ERR_CODE_UNKNOWN  0x0UL
+	#define NVM_WRITE_CMD_ERR_CODE_FRAG_ERR 0x1UL
+	#define NVM_WRITE_CMD_ERR_CODE_NO_SPACE 0x2UL
+	#define NVM_WRITE_CMD_ERR_CODE_LAST    NVM_WRITE_CMD_ERR_CODE_NO_SPACE
+	u8	unused_0[7];
 };
 
-/* hwrm_nvm_modify */
-/* Input (40 bytes) */
+/* hwrm_nvm_modify_input (size:320b/40B) */
 struct hwrm_nvm_modify_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 host_src_addr;
-	__le16 dir_idx;
-	u8 unused_0;
-	u8 unused_1;
-	__le32 offset;
-	__le32 len;
-	__le32 unused_2;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_src_addr;
+	__le16	dir_idx;
+	u8	unused_0[2];
+	__le32	offset;
+	__le32	len;
+	u8	unused_1[4];
+};
+
+/* hwrm_nvm_modify_output (size:128b/16B) */
 struct hwrm_nvm_modify_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_find_dir_entry */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_find_dir_entry_input (size:256b/32B) */
 struct hwrm_nvm_find_dir_entry_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define NVM_FIND_DIR_ENTRY_REQ_ENABLES_DIR_IDX_VALID       0x1UL
-	__le16 dir_idx;
-	__le16 dir_type;
-	__le16 dir_ordinal;
-	__le16 dir_ext;
-	u8 opt_ordinal;
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_MASK	    0x3UL
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_SFT		    0
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_EQ		   0x0UL
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GE		   0x1UL
-	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GT		   0x2UL
-	u8 unused_1[3];
-};
-
-/* Output (32 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define NVM_FIND_DIR_ENTRY_REQ_ENABLES_DIR_IDX_VALID     0x1UL
+	__le16	dir_idx;
+	__le16	dir_type;
+	__le16	dir_ordinal;
+	__le16	dir_ext;
+	u8	opt_ordinal;
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_MASK 0x3UL
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_SFT 0
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_EQ    0x0UL
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GE    0x1UL
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GT    0x2UL
+	#define NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_LAST NVM_FIND_DIR_ENTRY_REQ_OPT_ORDINAL_GT
+	u8	unused_0[3];
+};
+
+/* hwrm_nvm_find_dir_entry_output (size:256b/32B) */
 struct hwrm_nvm_find_dir_entry_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 dir_item_length;
-	__le32 dir_data_length;
-	__le32 fw_ver;
-	__le16 dir_ordinal;
-	__le16 dir_idx;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_erase_dir_entry */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	dir_item_length;
+	__le32	dir_data_length;
+	__le32	fw_ver;
+	__le16	dir_ordinal;
+	__le16	dir_idx;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_erase_dir_entry_input (size:192b/24B) */
 struct hwrm_nvm_erase_dir_entry_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 dir_idx;
-	__le16 unused_0[3];
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	dir_idx;
+	u8	unused_0[6];
 };
 
-/* Output (16 bytes) */
+/* hwrm_nvm_erase_dir_entry_output (size:128b/16B) */
 struct hwrm_nvm_erase_dir_entry_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_get_dev_info */
-/* Input (16 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_get_dev_info_input (size:128b/16B) */
 struct hwrm_nvm_get_dev_info_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (32 bytes) */
+/* hwrm_nvm_get_dev_info_output (size:256b/32B) */
 struct hwrm_nvm_get_dev_info_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 manufacturer_id;
-	__le16 device_id;
-	__le32 sector_size;
-	__le32 nvram_size;
-	__le32 reserved_size;
-	__le32 available_size;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* hwrm_nvm_mod_dir_entry */
-/* Input (32 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	manufacturer_id;
+	__le16	device_id;
+	__le32	sector_size;
+	__le32	nvram_size;
+	__le32	reserved_size;
+	__le32	available_size;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_nvm_mod_dir_entry_input (size:256b/32B) */
 struct hwrm_nvm_mod_dir_entry_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 enables;
-	#define NVM_MOD_DIR_ENTRY_REQ_ENABLES_CHECKSUM		    0x1UL
-	__le16 dir_idx;
-	__le16 dir_ordinal;
-	__le16 dir_ext;
-	__le16 dir_attr;
-	__le32 checksum;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	enables;
+	#define NVM_MOD_DIR_ENTRY_REQ_ENABLES_CHECKSUM     0x1UL
+	__le16	dir_idx;
+	__le16	dir_ordinal;
+	__le16	dir_ext;
+	__le16	dir_attr;
+	__le32	checksum;
+};
+
+/* hwrm_nvm_mod_dir_entry_output (size:128b/16B) */
 struct hwrm_nvm_mod_dir_entry_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_verify_update */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_verify_update_input (size:192b/24B) */
 struct hwrm_nvm_verify_update_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le16 dir_type;
-	__le16 dir_ordinal;
-	__le16 dir_ext;
-	__le16 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	dir_type;
+	__le16	dir_ordinal;
+	__le16	dir_ext;
+	u8	unused_0[2];
+};
+
+/* hwrm_nvm_verify_update_output (size:128b/16B) */
 struct hwrm_nvm_verify_update_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_nvm_install_update */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_install_update_input (size:192b/24B) */
 struct hwrm_nvm_install_update_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le32 install_type;
-	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_NORMAL	   0x0UL
-	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_ALL	   0xffffffffUL
-	__le16 flags;
-	#define NVM_INSTALL_UPDATE_REQ_FLAGS_ERASE_UNUSED_SPACE    0x1UL
-	#define NVM_INSTALL_UPDATE_REQ_FLAGS_REMOVE_UNUSED_PKG     0x2UL
-	#define NVM_INSTALL_UPDATE_REQ_FLAGS_ALLOWED_TO_DEFRAG     0x4UL
-	__le16 unused_0;
-};
-
-/* Output (24 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	install_type;
+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_NORMAL 0x0UL
+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_ALL    0xffffffffUL
+	#define NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_LAST  NVM_INSTALL_UPDATE_REQ_INSTALL_TYPE_ALL
+	__le16	flags;
+	#define NVM_INSTALL_UPDATE_REQ_FLAGS_ERASE_UNUSED_SPACE     0x1UL
+	#define NVM_INSTALL_UPDATE_REQ_FLAGS_REMOVE_UNUSED_PKG      0x2UL
+	#define NVM_INSTALL_UPDATE_REQ_FLAGS_ALLOWED_TO_DEFRAG      0x4UL
+	u8	unused_0[2];
+};
+
+/* hwrm_nvm_install_update_output (size:192b/24B) */
 struct hwrm_nvm_install_update_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le64 installed_items;
-	u8 result;
-	#define NVM_INSTALL_UPDATE_RESP_RESULT_SUCCESS		   0x0UL
-	u8 problem_item;
-	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_NONE	   0x0UL
-	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_PACKAGE      0xffUL
-	u8 reset_required;
-	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_NONE       0x0UL
-	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_PCI	   0x1UL
-	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_POWER      0x2UL
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* Command specific Error Codes (8 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le64	installed_items;
+	u8	result;
+	#define NVM_INSTALL_UPDATE_RESP_RESULT_SUCCESS 0x0UL
+	#define NVM_INSTALL_UPDATE_RESP_RESULT_LAST   NVM_INSTALL_UPDATE_RESP_RESULT_SUCCESS
+	u8	problem_item;
+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_NONE    0x0UL
+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_PACKAGE 0xffUL
+	#define NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_LAST   NVM_INSTALL_UPDATE_RESP_PROBLEM_ITEM_PACKAGE
+	u8	reset_required;
+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_NONE  0x0UL
+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_PCI   0x1UL
+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_POWER 0x2UL
+	#define NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_LAST NVM_INSTALL_UPDATE_RESP_RESET_REQUIRED_POWER
+	u8	unused_0[4];
+	u8	valid;
+};
+
+/* hwrm_nvm_install_update_cmd_err (size:64b/8B) */
 struct hwrm_nvm_install_update_cmd_err {
-	u8 code;
-	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN	   0x0UL
-	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_FRAG_ERR	   0x1UL
-	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE	   0x2UL
-	u8 unused_0[7];
+	u8	code;
+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_UNKNOWN  0x0UL
+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_FRAG_ERR 0x1UL
+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE 0x2UL
+	#define NVM_INSTALL_UPDATE_CMD_ERR_CODE_LAST    NVM_INSTALL_UPDATE_CMD_ERR_CODE_NO_SPACE
+	u8	unused_0[7];
 };
 
-/* hwrm_nvm_get_variable */
-/* Input (40 bytes) */
+/* hwrm_nvm_get_variable_input (size:320b/40B) */
 struct hwrm_nvm_get_variable_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 dest_data_addr;
-	__le16 data_len;
-	__le16 option_num;
-	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_0		   0x0UL
-	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF	   0xffffUL
-	__le16 dimensions;
-	__le16 index_0;
-	__le16 index_1;
-	__le16 index_2;
-	__le16 index_3;
-	u8 flags;
-	#define NVM_GET_VARIABLE_REQ_FLAGS_FACTORY_DFLT	    0x1UL
-	u8 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	dest_data_addr;
+	__le16	data_len;
+	__le16	option_num;
+	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_0    0x0UL
+	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF 0xffffUL
+	#define NVM_GET_VARIABLE_REQ_OPTION_NUM_LAST     NVM_GET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF
+	__le16	dimensions;
+	__le16	index_0;
+	__le16	index_1;
+	__le16	index_2;
+	__le16	index_3;
+	u8	flags;
+	#define NVM_GET_VARIABLE_REQ_FLAGS_FACTORY_DFLT     0x1UL
+	u8	unused_0;
+};
+
+/* hwrm_nvm_get_variable_output (size:128b/16B) */
 struct hwrm_nvm_get_variable_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 data_len;
-	__le16 option_num;
-	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_0	   0x0UL
-	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_FFFF	   0xffffUL
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* Command specific Error Codes (8 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	data_len;
+	__le16	option_num;
+	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_0    0x0UL
+	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_FFFF 0xffffUL
+	#define NVM_GET_VARIABLE_RESP_OPTION_NUM_LAST     NVM_GET_VARIABLE_RESP_OPTION_NUM_RSVD_FFFF
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_nvm_get_variable_cmd_err (size:64b/8B) */
 struct hwrm_nvm_get_variable_cmd_err {
-	u8 code;
-	#define NVM_GET_VARIABLE_CMD_ERR_CODE_UNKNOWN		   0x0UL
-	#define NVM_GET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST       0x1UL
-	#define NVM_GET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR	   0x2UL
-	#define NVM_GET_VARIABLE_CMD_ERR_CODE_LEN_TOO_SHORT       0x3UL
-	u8 unused_0[7];
+	u8	code;
+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_UNKNOWN       0x0UL
+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST 0x1UL
+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR   0x2UL
+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_LEN_TOO_SHORT 0x3UL
+	#define NVM_GET_VARIABLE_CMD_ERR_CODE_LAST         NVM_GET_VARIABLE_CMD_ERR_CODE_LEN_TOO_SHORT
+	u8	unused_0[7];
 };
 
-/* hwrm_nvm_set_variable */
-/* Input (40 bytes) */
+/* hwrm_nvm_set_variable_input (size:320b/40B) */
 struct hwrm_nvm_set_variable_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 src_data_addr;
-	__le16 data_len;
-	__le16 option_num;
-	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_0		   0x0UL
-	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF	   0xffffUL
-	__le16 dimensions;
-	__le16 index_0;
-	__le16 index_1;
-	__le16 index_2;
-	__le16 index_3;
-	u8 flags;
-	#define NVM_SET_VARIABLE_REQ_FLAGS_FORCE_FLUSH		    0x1UL
-	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_MASK       0xeUL
-	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_SFT	    1
-	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_NONE      (0x0UL << 1)
-	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_HMAC_SHA1 (0x1UL << 1)
-	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_LAST    NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_HMAC_SHA1
-	u8 unused_0;
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	src_data_addr;
+	__le16	data_len;
+	__le16	option_num;
+	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_0    0x0UL
+	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF 0xffffUL
+	#define NVM_SET_VARIABLE_REQ_OPTION_NUM_LAST     NVM_SET_VARIABLE_REQ_OPTION_NUM_RSVD_FFFF
+	__le16	dimensions;
+	__le16	index_0;
+	__le16	index_1;
+	__le16	index_2;
+	__le16	index_3;
+	u8	flags;
+	#define NVM_SET_VARIABLE_REQ_FLAGS_FORCE_FLUSH           0x1UL
+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_MASK     0xeUL
+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_SFT      1
+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_NONE       (0x0UL << 1)
+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_HMAC_SHA1  (0x1UL << 1)
+	#define NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_LAST      NVM_SET_VARIABLE_REQ_FLAGS_ENCRYPT_MODE_HMAC_SHA1
+	u8	unused_0;
+};
+
+/* hwrm_nvm_set_variable_output (size:128b/16B) */
 struct hwrm_nvm_set_variable_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* Command specific Error Codes (8 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_set_variable_cmd_err (size:64b/8B) */
 struct hwrm_nvm_set_variable_cmd_err {
-	u8 code;
-	#define NVM_SET_VARIABLE_CMD_ERR_CODE_UNKNOWN		   0x0UL
-	#define NVM_SET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST       0x1UL
-	#define NVM_SET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR	   0x2UL
-	u8 unused_0[7];
+	u8	code;
+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_UNKNOWN       0x0UL
+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_VAR_NOT_EXIST 0x1UL
+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR   0x2UL
+	#define NVM_SET_VARIABLE_CMD_ERR_CODE_LAST         NVM_SET_VARIABLE_CMD_ERR_CODE_CORRUPT_VAR
+	u8	unused_0[7];
 };
 
-/* hwrm_selftest_qlist */
-/* Input (16 bytes) */
+/* hwrm_selftest_qlist_input (size:128b/16B) */
 struct hwrm_selftest_qlist_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (280 bytes) */
+/* hwrm_selftest_qlist_output (size:2240b/280B) */
 struct hwrm_selftest_qlist_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 num_tests;
-	u8 available_tests;
-	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_NVM_TEST       0x1UL
-	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_LINK_TEST      0x2UL
-	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_REGISTER_TEST  0x4UL
-	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_MEMORY_TEST    0x8UL
-	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_PCIE_SERDES_TEST 0x10UL
-	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_ETHERNET_SERDES_TEST 0x20UL
-	u8 offline_tests;
-	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_NVM_TEST	    0x1UL
-	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_LINK_TEST	    0x2UL
-	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_REGISTER_TEST    0x4UL
-	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_MEMORY_TEST      0x8UL
-	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_PCIE_SERDES_TEST 0x10UL
-	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_ETHERNET_SERDES_TEST 0x20UL
-	u8 unused_0;
-	__le16 test_timeout;
-	u8 unused_1;
-	u8 unused_2;
-	char test0_name[32];
-	char test1_name[32];
-	char test2_name[32];
-	char test3_name[32];
-	char test4_name[32];
-	char test5_name[32];
-	char test6_name[32];
-	char test7_name[32];
-	__le32 unused_3;
-	u8 unused_4;
-	u8 unused_5;
-	u8 unused_6;
-	u8 valid;
-};
-
-/* hwrm_selftest_exec */
-/* Input (24 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	num_tests;
+	u8	available_tests;
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_NVM_TEST                 0x1UL
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_LINK_TEST                0x2UL
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_REGISTER_TEST            0x4UL
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_MEMORY_TEST              0x8UL
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_PCIE_SERDES_TEST         0x10UL
+	#define SELFTEST_QLIST_RESP_AVAILABLE_TESTS_ETHERNET_SERDES_TEST     0x20UL
+	u8	offline_tests;
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_NVM_TEST                 0x1UL
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_LINK_TEST                0x2UL
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_REGISTER_TEST            0x4UL
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_MEMORY_TEST              0x8UL
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_PCIE_SERDES_TEST         0x10UL
+	#define SELFTEST_QLIST_RESP_OFFLINE_TESTS_ETHERNET_SERDES_TEST     0x20UL
+	u8	unused_0;
+	__le16	test_timeout;
+	u8	unused_1[2];
+	char	test0_name[32];
+	char	test1_name[32];
+	char	test2_name[32];
+	char	test3_name[32];
+	char	test4_name[32];
+	char	test5_name[32];
+	char	test6_name[32];
+	char	test7_name[32];
+	u8	unused_2[7];
+	u8	valid;
+};
+
+/* hwrm_selftest_exec_input (size:192b/24B) */
 struct hwrm_selftest_exec_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	u8 flags;
-	#define SELFTEST_EXEC_REQ_FLAGS_NVM_TEST		    0x1UL
-	#define SELFTEST_EXEC_REQ_FLAGS_LINK_TEST		    0x2UL
-	#define SELFTEST_EXEC_REQ_FLAGS_REGISTER_TEST		    0x4UL
-	#define SELFTEST_EXEC_REQ_FLAGS_MEMORY_TEST		    0x8UL
-	#define SELFTEST_EXEC_REQ_FLAGS_PCIE_SERDES_TEST	    0x10UL
-	#define SELFTEST_EXEC_REQ_FLAGS_ETHERNET_SERDES_TEST       0x20UL
-	u8 unused_0[7];
-};
-
-/* Output (16 bytes) */
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	flags;
+	#define SELFTEST_EXEC_REQ_FLAGS_NVM_TEST                 0x1UL
+	#define SELFTEST_EXEC_REQ_FLAGS_LINK_TEST                0x2UL
+	#define SELFTEST_EXEC_REQ_FLAGS_REGISTER_TEST            0x4UL
+	#define SELFTEST_EXEC_REQ_FLAGS_MEMORY_TEST              0x8UL
+	#define SELFTEST_EXEC_REQ_FLAGS_PCIE_SERDES_TEST         0x10UL
+	#define SELFTEST_EXEC_REQ_FLAGS_ETHERNET_SERDES_TEST     0x20UL
+	u8	pcie_lane_num;
+	u8	unused_0[6];
+};
+
+/* hwrm_selftest_exec_output (size:128b/16B) */
 struct hwrm_selftest_exec_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	u8 requested_tests;
-	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_NVM_TEST	    0x1UL
-	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_LINK_TEST       0x2UL
-	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_REGISTER_TEST   0x4UL
-	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_MEMORY_TEST     0x8UL
-	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_PCIE_SERDES_TEST 0x10UL
-	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_ETHERNET_SERDES_TEST 0x20UL
-	u8 test_success;
-	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_NVM_TEST	    0x1UL
-	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_LINK_TEST	    0x2UL
-	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_REGISTER_TEST      0x4UL
-	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_MEMORY_TEST	    0x8UL
-	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_PCIE_SERDES_TEST   0x10UL
-	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_ETHERNET_SERDES_TEST 0x20UL
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 unused_4;
-	u8 valid;
-};
-
-/* hwrm_selftest_irq */
-/* Input (16 bytes) */
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	requested_tests;
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_NVM_TEST                 0x1UL
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_LINK_TEST                0x2UL
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_REGISTER_TEST            0x4UL
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_MEMORY_TEST              0x8UL
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_PCIE_SERDES_TEST         0x10UL
+	#define SELFTEST_EXEC_RESP_REQUESTED_TESTS_ETHERNET_SERDES_TEST     0x20UL
+	u8	test_success;
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_NVM_TEST                 0x1UL
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_LINK_TEST                0x2UL
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_REGISTER_TEST            0x4UL
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_MEMORY_TEST              0x8UL
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_PCIE_SERDES_TEST         0x10UL
+	#define SELFTEST_EXEC_RESP_TEST_SUCCESS_ETHERNET_SERDES_TEST     0x20UL
+	u8	unused_0[5];
+	u8	valid;
+};
+
+/* hwrm_selftest_irq_input (size:128b/16B) */
 struct hwrm_selftest_irq_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
 };
 
-/* Output (16 bytes) */
+/* hwrm_selftest_irq_output (size:128b/16B) */
 struct hwrm_selftest_irq_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 unused_3;
-	u8 valid;
-};
-
-/* hwrm_selftest_retrieve_serdes_data */
-/* Input (32 bytes) */
-struct hwrm_selftest_retrieve_serdes_data_input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-	__le64 resp_data_addr;
-	__le32 resp_data_offset;
-	__le16 data_len;
-	u8 flags;
-	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_UNUSED_TEST_MASK 0xfUL
-	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_UNUSED_TEST_SFT 0
-	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_PCIE_SERDES_TEST 0x10UL
-	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_ETHERNET_SERDES_TEST 0x20UL
-	u8 unused_0;
-};
-
-/* Output (16 bytes) */
-struct hwrm_selftest_retrieve_serdes_data_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le16 total_data_len;
-	__le16 copied_data_len;
-	u8 unused_0;
-	u8 unused_1;
-	u8 unused_2;
-	u8 valid;
-};
-
-/* Hardware Resource Manager Specification */
-/* Input (16 bytes) */
-struct input {
-	__le16 req_type;
-	__le16 cmpl_ring;
-	__le16 seq_id;
-	__le16 target_id;
-	__le64 resp_addr;
-};
-
-/* Output (8 bytes) */
-struct output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-};
-
-/* Short Command Structure (16 bytes) */
-struct hwrm_short_input {
-	__le16 req_type;
-	__le16 signature;
-	#define SHORT_REQ_SIGNATURE_SHORT_CMD			   0x4321UL
-	__le16 unused_0;
-	__le16 size;
-	__le64 req_addr;
-};
-
-/* Command numbering (8 bytes) */
-struct cmd_nums {
-	__le16 req_type;
-	#define HWRM_VER_GET					   (0x0UL)
-	#define HWRM_FUNC_BUF_UNRGTR				   (0xeUL)
-	#define HWRM_FUNC_VF_CFG				   (0xfUL)
-	#define RESERVED1					   (0x10UL)
-	#define HWRM_FUNC_RESET				   (0x11UL)
-	#define HWRM_FUNC_GETFID				   (0x12UL)
-	#define HWRM_FUNC_VF_ALLOC				   (0x13UL)
-	#define HWRM_FUNC_VF_FREE				   (0x14UL)
-	#define HWRM_FUNC_QCAPS				   (0x15UL)
-	#define HWRM_FUNC_QCFG					   (0x16UL)
-	#define HWRM_FUNC_CFG					   (0x17UL)
-	#define HWRM_FUNC_QSTATS				   (0x18UL)
-	#define HWRM_FUNC_CLR_STATS				   (0x19UL)
-	#define HWRM_FUNC_DRV_UNRGTR				   (0x1aUL)
-	#define HWRM_FUNC_VF_RESC_FREE				   (0x1bUL)
-	#define HWRM_FUNC_VF_VNIC_IDS_QUERY			   (0x1cUL)
-	#define HWRM_FUNC_DRV_RGTR				   (0x1dUL)
-	#define HWRM_FUNC_DRV_QVER				   (0x1eUL)
-	#define HWRM_FUNC_BUF_RGTR				   (0x1fUL)
-	#define HWRM_PORT_PHY_CFG				   (0x20UL)
-	#define HWRM_PORT_MAC_CFG				   (0x21UL)
-	#define HWRM_PORT_TS_QUERY				   (0x22UL)
-	#define HWRM_PORT_QSTATS				   (0x23UL)
-	#define HWRM_PORT_LPBK_QSTATS				   (0x24UL)
-	#define HWRM_PORT_CLR_STATS				   (0x25UL)
-	#define HWRM_PORT_LPBK_CLR_STATS			   (0x26UL)
-	#define HWRM_PORT_PHY_QCFG				   (0x27UL)
-	#define HWRM_PORT_MAC_QCFG				   (0x28UL)
-	#define HWRM_PORT_MAC_PTP_QCFG				   (0x29UL)
-	#define HWRM_PORT_PHY_QCAPS				   (0x2aUL)
-	#define HWRM_PORT_PHY_I2C_WRITE			   (0x2bUL)
-	#define HWRM_PORT_PHY_I2C_READ				   (0x2cUL)
-	#define HWRM_PORT_LED_CFG				   (0x2dUL)
-	#define HWRM_PORT_LED_QCFG				   (0x2eUL)
-	#define HWRM_PORT_LED_QCAPS				   (0x2fUL)
-	#define HWRM_QUEUE_QPORTCFG				   (0x30UL)
-	#define HWRM_QUEUE_QCFG				   (0x31UL)
-	#define HWRM_QUEUE_CFG					   (0x32UL)
-	#define HWRM_FUNC_VLAN_CFG				   (0x33UL)
-	#define HWRM_FUNC_VLAN_QCFG				   (0x34UL)
-	#define HWRM_QUEUE_PFCENABLE_QCFG			   (0x35UL)
-	#define HWRM_QUEUE_PFCENABLE_CFG			   (0x36UL)
-	#define HWRM_QUEUE_PRI2COS_QCFG			   (0x37UL)
-	#define HWRM_QUEUE_PRI2COS_CFG				   (0x38UL)
-	#define HWRM_QUEUE_COS2BW_QCFG				   (0x39UL)
-	#define HWRM_QUEUE_COS2BW_CFG				   (0x3aUL)
-	#define HWRM_QUEUE_DSCP_QCAPS				   (0x3bUL)
-	#define HWRM_QUEUE_DSCP2PRI_QCFG			   (0x3cUL)
-	#define HWRM_QUEUE_DSCP2PRI_CFG			   (0x3dUL)
-	#define HWRM_VNIC_ALLOC				   (0x40UL)
-	#define HWRM_VNIC_FREE					   (0x41UL)
-	#define HWRM_VNIC_CFG					   (0x42UL)
-	#define HWRM_VNIC_QCFG					   (0x43UL)
-	#define HWRM_VNIC_TPA_CFG				   (0x44UL)
-	#define HWRM_VNIC_TPA_QCFG				   (0x45UL)
-	#define HWRM_VNIC_RSS_CFG				   (0x46UL)
-	#define HWRM_VNIC_RSS_QCFG				   (0x47UL)
-	#define HWRM_VNIC_PLCMODES_CFG				   (0x48UL)
-	#define HWRM_VNIC_PLCMODES_QCFG			   (0x49UL)
-	#define HWRM_VNIC_QCAPS				   (0x4aUL)
-	#define HWRM_RING_ALLOC				   (0x50UL)
-	#define HWRM_RING_FREE					   (0x51UL)
-	#define HWRM_RING_CMPL_RING_QAGGINT_PARAMS		   (0x52UL)
-	#define HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS		   (0x53UL)
-	#define HWRM_RING_RESET				   (0x5eUL)
-	#define HWRM_RING_GRP_ALLOC				   (0x60UL)
-	#define HWRM_RING_GRP_FREE				   (0x61UL)
-	#define RESERVED5					   (0x64UL)
-	#define RESERVED6					   (0x65UL)
-	#define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC			   (0x70UL)
-	#define HWRM_VNIC_RSS_COS_LB_CTX_FREE			   (0x71UL)
-	#define HWRM_CFA_L2_FILTER_ALLOC			   (0x90UL)
-	#define HWRM_CFA_L2_FILTER_FREE			   (0x91UL)
-	#define HWRM_CFA_L2_FILTER_CFG				   (0x92UL)
-	#define HWRM_CFA_L2_SET_RX_MASK			   (0x93UL)
-	#define HWRM_CFA_VLAN_ANTISPOOF_CFG			   (0x94UL)
-	#define HWRM_CFA_TUNNEL_FILTER_ALLOC			   (0x95UL)
-	#define HWRM_CFA_TUNNEL_FILTER_FREE			   (0x96UL)
-	#define HWRM_CFA_ENCAP_RECORD_ALLOC			   (0x97UL)
-	#define HWRM_CFA_ENCAP_RECORD_FREE			   (0x98UL)
-	#define HWRM_CFA_NTUPLE_FILTER_ALLOC			   (0x99UL)
-	#define HWRM_CFA_NTUPLE_FILTER_FREE			   (0x9aUL)
-	#define HWRM_CFA_NTUPLE_FILTER_CFG			   (0x9bUL)
-	#define HWRM_CFA_EM_FLOW_ALLOC				   (0x9cUL)
-	#define HWRM_CFA_EM_FLOW_FREE				   (0x9dUL)
-	#define HWRM_CFA_EM_FLOW_CFG				   (0x9eUL)
-	#define HWRM_TUNNEL_DST_PORT_QUERY			   (0xa0UL)
-	#define HWRM_TUNNEL_DST_PORT_ALLOC			   (0xa1UL)
-	#define HWRM_TUNNEL_DST_PORT_FREE			   (0xa2UL)
-	#define HWRM_STAT_CTX_ALLOC				   (0xb0UL)
-	#define HWRM_STAT_CTX_FREE				   (0xb1UL)
-	#define HWRM_STAT_CTX_QUERY				   (0xb2UL)
-	#define HWRM_STAT_CTX_CLR_STATS			   (0xb3UL)
-	#define HWRM_FW_RESET					   (0xc0UL)
-	#define HWRM_FW_QSTATUS				   (0xc1UL)
-	#define HWRM_FW_SET_TIME				   (0xc8UL)
-	#define HWRM_FW_GET_TIME				   (0xc9UL)
-	#define HWRM_FW_SET_STRUCTURED_DATA			   (0xcaUL)
-	#define HWRM_FW_GET_STRUCTURED_DATA			   (0xcbUL)
-	#define HWRM_FW_IPC_MAILBOX				   (0xccUL)
-	#define HWRM_EXEC_FWD_RESP				   (0xd0UL)
-	#define HWRM_REJECT_FWD_RESP				   (0xd1UL)
-	#define HWRM_FWD_RESP					   (0xd2UL)
-	#define HWRM_FWD_ASYNC_EVENT_CMPL			   (0xd3UL)
-	#define HWRM_TEMP_MONITOR_QUERY			   (0xe0UL)
-	#define HWRM_WOL_FILTER_ALLOC				   (0xf0UL)
-	#define HWRM_WOL_FILTER_FREE				   (0xf1UL)
-	#define HWRM_WOL_FILTER_QCFG				   (0xf2UL)
-	#define HWRM_WOL_REASON_QCFG				   (0xf3UL)
-	#define HWRM_CFA_METER_PROFILE_ALLOC			   (0xf5UL)
-	#define HWRM_CFA_METER_PROFILE_FREE			   (0xf6UL)
-	#define HWRM_CFA_METER_PROFILE_CFG			   (0xf7UL)
-	#define HWRM_CFA_METER_INSTANCE_ALLOC			   (0xf8UL)
-	#define HWRM_CFA_METER_INSTANCE_FREE			   (0xf9UL)
-	#define HWRM_CFA_VFR_ALLOC				   (0xfdUL)
-	#define HWRM_CFA_VFR_FREE				   (0xfeUL)
-	#define HWRM_CFA_VF_PAIR_ALLOC				   (0x100UL)
-	#define HWRM_CFA_VF_PAIR_FREE				   (0x101UL)
-	#define HWRM_CFA_VF_PAIR_INFO				   (0x102UL)
-	#define HWRM_CFA_FLOW_ALLOC				   (0x103UL)
-	#define HWRM_CFA_FLOW_FREE				   (0x104UL)
-	#define HWRM_CFA_FLOW_FLUSH				   (0x105UL)
-	#define HWRM_CFA_FLOW_STATS				   (0x106UL)
-	#define HWRM_CFA_FLOW_INFO				   (0x107UL)
-	#define HWRM_CFA_DECAP_FILTER_ALLOC			   (0x108UL)
-	#define HWRM_CFA_DECAP_FILTER_FREE			   (0x109UL)
-	#define HWRM_CFA_VLAN_ANTISPOOF_QCFG			   (0x10aUL)
-	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_ALLOC		   (0x10bUL)
-	#define HWRM_CFA_REDIRECT_TUNNEL_TYPE_FREE		   (0x10cUL)
-	#define HWRM_CFA_PAIR_ALLOC				   (0x10dUL)
-	#define HWRM_CFA_PAIR_FREE				   (0x10eUL)
-	#define HWRM_CFA_PAIR_INFO				   (0x10fUL)
-	#define HWRM_FW_IPC_MSG				   (0x110UL)
-	#define HWRM_SELFTEST_QLIST				   (0x200UL)
-	#define HWRM_SELFTEST_EXEC				   (0x201UL)
-	#define HWRM_SELFTEST_IRQ				   (0x202UL)
-	#define HWRM_SELFTEST_RETRIEVE_SERDES_DATA		   (0x203UL)
-	#define HWRM_DBG_READ_DIRECT				   (0xff10UL)
-	#define HWRM_DBG_READ_INDIRECT				   (0xff11UL)
-	#define HWRM_DBG_WRITE_DIRECT				   (0xff12UL)
-	#define HWRM_DBG_WRITE_INDIRECT			   (0xff13UL)
-	#define HWRM_DBG_DUMP					   (0xff14UL)
-	#define HWRM_DBG_ERASE_NVM				   (0xff15UL)
-	#define HWRM_DBG_CFG					   (0xff16UL)
-	#define HWRM_DBG_COREDUMP_LIST				   (0xff17UL)
-	#define HWRM_DBG_COREDUMP_INITIATE			   (0xff18UL)
-	#define HWRM_DBG_COREDUMP_RETRIEVE			   (0xff19UL)
-	#define HWRM_NVM_FACTORY_DEFAULTS			   (0xffeeUL)
-	#define HWRM_NVM_VALIDATE_OPTION			   (0xffefUL)
-	#define HWRM_NVM_FLUSH					   (0xfff0UL)
-	#define HWRM_NVM_GET_VARIABLE				   (0xfff1UL)
-	#define HWRM_NVM_SET_VARIABLE				   (0xfff2UL)
-	#define HWRM_NVM_INSTALL_UPDATE			   (0xfff3UL)
-	#define HWRM_NVM_MODIFY				   (0xfff4UL)
-	#define HWRM_NVM_VERIFY_UPDATE				   (0xfff5UL)
-	#define HWRM_NVM_GET_DEV_INFO				   (0xfff6UL)
-	#define HWRM_NVM_ERASE_DIR_ENTRY			   (0xfff7UL)
-	#define HWRM_NVM_MOD_DIR_ENTRY				   (0xfff8UL)
-	#define HWRM_NVM_FIND_DIR_ENTRY			   (0xfff9UL)
-	#define HWRM_NVM_GET_DIR_ENTRIES			   (0xfffaUL)
-	#define HWRM_NVM_GET_DIR_INFO				   (0xfffbUL)
-	#define HWRM_NVM_RAW_DUMP				   (0xfffcUL)
-	#define HWRM_NVM_READ					   (0xfffdUL)
-	#define HWRM_NVM_WRITE					   (0xfffeUL)
-	#define HWRM_NVM_RAW_WRITE_BLK				   (0xffffUL)
-	__le16 unused_0[3];
-};
-
-/* Return Codes (8 bytes) */
-struct ret_codes {
-	__le16 error_code;
-	#define HWRM_ERR_CODE_SUCCESS				   (0x0UL)
-	#define HWRM_ERR_CODE_FAIL				   (0x1UL)
-	#define HWRM_ERR_CODE_INVALID_PARAMS			   (0x2UL)
-	#define HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED		   (0x3UL)
-	#define HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR		   (0x4UL)
-	#define HWRM_ERR_CODE_INVALID_FLAGS			   (0x5UL)
-	#define HWRM_ERR_CODE_INVALID_ENABLES			   (0x6UL)
-	#define HWRM_ERR_CODE_HWRM_ERROR			   (0xfUL)
-	#define HWRM_ERR_CODE_UNKNOWN_ERR			   (0xfffeUL)
-	#define HWRM_ERR_CODE_CMD_NOT_SUPPORTED		   (0xffffUL)
-	__le16 unused_0[3];
-};
-
-/* Output (16 bytes) */
-struct hwrm_err_output {
-	__le16 error_code;
-	__le16 req_type;
-	__le16 seq_id;
-	__le16 resp_len;
-	__le32 opaque_0;
-	__le16 opaque_1;
-	u8 cmd_err;
-	u8 valid;
-};
-
-/* Port Tx Statistics Formats (408 bytes) */
-struct tx_port_stats {
-	__le64 tx_64b_frames;
-	__le64 tx_65b_127b_frames;
-	__le64 tx_128b_255b_frames;
-	__le64 tx_256b_511b_frames;
-	__le64 tx_512b_1023b_frames;
-	__le64 tx_1024b_1518_frames;
-	__le64 tx_good_vlan_frames;
-	__le64 tx_1519b_2047_frames;
-	__le64 tx_2048b_4095b_frames;
-	__le64 tx_4096b_9216b_frames;
-	__le64 tx_9217b_16383b_frames;
-	__le64 tx_good_frames;
-	__le64 tx_total_frames;
-	__le64 tx_ucast_frames;
-	__le64 tx_mcast_frames;
-	__le64 tx_bcast_frames;
-	__le64 tx_pause_frames;
-	__le64 tx_pfc_frames;
-	__le64 tx_jabber_frames;
-	__le64 tx_fcs_err_frames;
-	__le64 tx_control_frames;
-	__le64 tx_oversz_frames;
-	__le64 tx_single_dfrl_frames;
-	__le64 tx_multi_dfrl_frames;
-	__le64 tx_single_coll_frames;
-	__le64 tx_multi_coll_frames;
-	__le64 tx_late_coll_frames;
-	__le64 tx_excessive_coll_frames;
-	__le64 tx_frag_frames;
-	__le64 tx_err;
-	__le64 tx_tagged_frames;
-	__le64 tx_dbl_tagged_frames;
-	__le64 tx_runt_frames;
-	__le64 tx_fifo_underruns;
-	__le64 tx_pfc_ena_frames_pri0;
-	__le64 tx_pfc_ena_frames_pri1;
-	__le64 tx_pfc_ena_frames_pri2;
-	__le64 tx_pfc_ena_frames_pri3;
-	__le64 tx_pfc_ena_frames_pri4;
-	__le64 tx_pfc_ena_frames_pri5;
-	__le64 tx_pfc_ena_frames_pri6;
-	__le64 tx_pfc_ena_frames_pri7;
-	__le64 tx_eee_lpi_events;
-	__le64 tx_eee_lpi_duration;
-	__le64 tx_llfc_logical_msgs;
-	__le64 tx_hcfc_msgs;
-	__le64 tx_total_collisions;
-	__le64 tx_bytes;
-	__le64 tx_xthol_frames;
-	__le64 tx_stat_discard;
-	__le64 tx_stat_error;
-};
-
-/* Port Rx Statistics Formats (528 bytes) */
-struct rx_port_stats {
-	__le64 rx_64b_frames;
-	__le64 rx_65b_127b_frames;
-	__le64 rx_128b_255b_frames;
-	__le64 rx_256b_511b_frames;
-	__le64 rx_512b_1023b_frames;
-	__le64 rx_1024b_1518_frames;
-	__le64 rx_good_vlan_frames;
-	__le64 rx_1519b_2047b_frames;
-	__le64 rx_2048b_4095b_frames;
-	__le64 rx_4096b_9216b_frames;
-	__le64 rx_9217b_16383b_frames;
-	__le64 rx_total_frames;
-	__le64 rx_ucast_frames;
-	__le64 rx_mcast_frames;
-	__le64 rx_bcast_frames;
-	__le64 rx_fcs_err_frames;
-	__le64 rx_ctrl_frames;
-	__le64 rx_pause_frames;
-	__le64 rx_pfc_frames;
-	__le64 rx_unsupported_opcode_frames;
-	__le64 rx_unsupported_da_pausepfc_frames;
-	__le64 rx_wrong_sa_frames;
-	__le64 rx_align_err_frames;
-	__le64 rx_oor_len_frames;
-	__le64 rx_code_err_frames;
-	__le64 rx_false_carrier_frames;
-	__le64 rx_ovrsz_frames;
-	__le64 rx_jbr_frames;
-	__le64 rx_mtu_err_frames;
-	__le64 rx_match_crc_frames;
-	__le64 rx_promiscuous_frames;
-	__le64 rx_tagged_frames;
-	__le64 rx_double_tagged_frames;
-	__le64 rx_trunc_frames;
-	__le64 rx_good_frames;
-	__le64 rx_pfc_xon2xoff_frames_pri0;
-	__le64 rx_pfc_xon2xoff_frames_pri1;
-	__le64 rx_pfc_xon2xoff_frames_pri2;
-	__le64 rx_pfc_xon2xoff_frames_pri3;
-	__le64 rx_pfc_xon2xoff_frames_pri4;
-	__le64 rx_pfc_xon2xoff_frames_pri5;
-	__le64 rx_pfc_xon2xoff_frames_pri6;
-	__le64 rx_pfc_xon2xoff_frames_pri7;
-	__le64 rx_pfc_ena_frames_pri0;
-	__le64 rx_pfc_ena_frames_pri1;
-	__le64 rx_pfc_ena_frames_pri2;
-	__le64 rx_pfc_ena_frames_pri3;
-	__le64 rx_pfc_ena_frames_pri4;
-	__le64 rx_pfc_ena_frames_pri5;
-	__le64 rx_pfc_ena_frames_pri6;
-	__le64 rx_pfc_ena_frames_pri7;
-	__le64 rx_sch_crc_err_frames;
-	__le64 rx_undrsz_frames;
-	__le64 rx_frag_frames;
-	__le64 rx_eee_lpi_events;
-	__le64 rx_eee_lpi_duration;
-	__le64 rx_llfc_physical_msgs;
-	__le64 rx_llfc_logical_msgs;
-	__le64 rx_llfc_msgs_with_crc_err;
-	__le64 rx_hcfc_msgs;
-	__le64 rx_hcfc_msgs_with_crc_err;
-	__le64 rx_bytes;
-	__le64 rx_runt_bytes;
-	__le64 rx_runt_frames;
-	__le64 rx_stat_discard;
-	__le64 rx_stat_err;
-};
-
-/* VXLAN IPv4 encapsulation structure (16 bytes) */
-struct hwrm_vxlan_ipv4_hdr {
-	u8 ver_hlen;
-	#define VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_MASK	    0xfUL
-	#define VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT	    0
-	#define VXLAN_IPV4_HDR_VER_HLEN_VERSION_MASK		    0xf0UL
-	#define VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT		    4
-	u8 tos;
-	__be16 ip_id;
-	__be16 flags_frag_offset;
-	u8 ttl;
-	u8 protocol;
-	__be32 src_ip_addr;
-	__be32 dest_ip_addr;
-};
-
-/* VXLAN IPv6 encapsulation structure (32 bytes) */
-struct hwrm_vxlan_ipv6_hdr {
-	__be32 ver_tc_flow_label;
-	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_VER_SFT	   0x1cUL
-	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_VER_MASK	   0xf0000000UL
-	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_TC_SFT	   0x14UL
-	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_TC_MASK	   0xff00000UL
-	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_SFT   0x0UL
-	#define VXLAN_IPV6_HDR_VER_TC_FLOW_LABEL_FLOW_LABEL_MASK  0xfffffUL
-	__be16 payload_len;
-	u8 next_hdr;
-	u8 ttl;
-	__be32 src_ip_addr[4];
-	__be32 dest_ip_addr[4];
-};
-
-/* VXLAN encapsulation structure (72 bytes) */
-struct hwrm_cfa_encap_data_vxlan {
-	u8 src_mac_addr[6];
-	__le16 unused_0;
-	u8 dst_mac_addr[6];
-	u8 num_vlan_tags;
-	u8 unused_1;
-	__be16 ovlan_tpid;
-	__be16 ovlan_tci;
-	__be16 ivlan_tpid;
-	__be16 ivlan_tci;
-	__le32 l3[10];
-	#define CFA_ENCAP_DATA_VXLAN_L3_VER_MASK		   0xfUL
-	#define CFA_ENCAP_DATA_VXLAN_L3_VER_IPV4		   0x4UL
-	#define CFA_ENCAP_DATA_VXLAN_L3_VER_IPV6		   0x6UL
-	__be16 src_port;
-	__be16 dst_port;
-	__be32 vni;
-};
-
-/* Periodic Statistics Context DMA to host (160 bytes) */
-struct ctx_hw_stats {
-	__le64 rx_ucast_pkts;
-	__le64 rx_mcast_pkts;
-	__le64 rx_bcast_pkts;
-	__le64 rx_discard_pkts;
-	__le64 rx_drop_pkts;
-	__le64 rx_ucast_bytes;
-	__le64 rx_mcast_bytes;
-	__le64 rx_bcast_bytes;
-	__le64 tx_ucast_pkts;
-	__le64 tx_mcast_pkts;
-	__le64 tx_bcast_pkts;
-	__le64 tx_discard_pkts;
-	__le64 tx_drop_pkts;
-	__le64 tx_ucast_bytes;
-	__le64 tx_mcast_bytes;
-	__le64 tx_bcast_bytes;
-	__le64 tpa_pkts;
-	__le64 tpa_bytes;
-	__le64 tpa_events;
-	__le64 tpa_aborts;
-};
-
-/* Structure data header (16 bytes) */
-struct hwrm_struct_hdr {
-	__le16 struct_id;
-	#define STRUCT_HDR_STRUCT_ID_LLDP_CFG			   0x41bUL
-	#define STRUCT_HDR_STRUCT_ID_DCBX_ETS			   0x41dUL
-	#define STRUCT_HDR_STRUCT_ID_DCBX_PFC			   0x41fUL
-	#define STRUCT_HDR_STRUCT_ID_DCBX_APP			   0x421UL
-	#define STRUCT_HDR_STRUCT_ID_DCBX_FEATURE_STATE	   0x422UL
-	#define STRUCT_HDR_STRUCT_ID_LLDP_GENERIC		   0x424UL
-	#define STRUCT_HDR_STRUCT_ID_LLDP_DEVICE		   0x426UL
-	#define STRUCT_HDR_STRUCT_ID_AFM_OPAQUE		   0x1UL
-	#define STRUCT_HDR_STRUCT_ID_PORT_DESCRIPTION		   0xaUL
-	#define STRUCT_HDR_STRUCT_ID_RSS_V2			   0x64UL
-	__le16 len;
-	u8 version;
-	u8 count;
-	__le16 subtype;
-	__le16 next_offset;
-	#define STRUCT_HDR_NEXT_OFFSET_LAST			   0x0UL
-	__le16 unused_0[3];
-};
-
-/* DCBX Application configuration structure (1057) (8 bytes) */
-struct hwrm_struct_data_dcbx_app {
-	__be16 protocol_id;
-	u8 protocol_selector;
-	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_ETHER_TYPE 0x1UL
-	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_PORT   0x2UL
-	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_UDP_PORT   0x3UL
-	#define STRUCT_DATA_DCBX_APP_PROTOCOL_SELECTOR_TCP_UDP_PORT 0x4UL
-	u8 priority;
-	u8 valid;
-	u8 unused_0[3];
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
 };
 
-#endif
+#endif /* _BNXT_HSI_H_ */
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
index c9617675f934..d87faad901fe 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c
@@ -135,7 +135,10 @@ int bnxt_get_vf_config(struct net_device *dev, int vf_id,
 	ivi->vf = vf_id;
 	vf = &bp->pf.vf[vf_id];
 
-	memcpy(&ivi->mac, vf->mac_addr, ETH_ALEN);
+	if (is_valid_ether_addr(vf->mac_addr))
+		memcpy(&ivi->mac, vf->mac_addr, ETH_ALEN);
+	else
+		memcpy(&ivi->mac, vf->vf_mac_addr, ETH_ALEN);
 	ivi->max_tx_rate = vf->max_tx_rate;
 	ivi->min_tx_rate = vf->min_tx_rate;
 	ivi->vlan = vf->vlan;
@@ -416,29 +419,126 @@ static int bnxt_hwrm_func_buf_rgtr(struct bnxt *bp)
 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 }
 
-/* only call by PF to reserve resources for VF */
+/* Only called by PF to reserve resources for VFs, returns actual number of
+ * VFs configured, or < 0 on error.
+ */
+static int bnxt_hwrm_func_vf_resc_cfg(struct bnxt *bp, int num_vfs)
+{
+	struct hwrm_func_vf_resource_cfg_input req = {0};
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+	u16 vf_tx_rings, vf_rx_rings, vf_cp_rings;
+	u16 vf_stat_ctx, vf_vnics, vf_ring_grps;
+	struct bnxt_pf_info *pf = &bp->pf;
+	int i, rc = 0;
+
+	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_RESOURCE_CFG, -1, -1);
+
+	vf_cp_rings = hw_resc->max_cp_rings - bp->cp_nr_rings;
+	vf_stat_ctx = hw_resc->max_stat_ctxs - bp->num_stat_ctxs;
+	if (bp->flags & BNXT_FLAG_AGG_RINGS)
+		vf_rx_rings = hw_resc->max_rx_rings - bp->rx_nr_rings * 2;
+	else
+		vf_rx_rings = hw_resc->max_rx_rings - bp->rx_nr_rings;
+	vf_ring_grps = hw_resc->max_hw_ring_grps - bp->rx_nr_rings;
+	vf_tx_rings = hw_resc->max_tx_rings - bp->tx_nr_rings;
+	vf_vnics = hw_resc->max_vnics - bp->nr_vnics;
+	vf_vnics = min_t(u16, vf_vnics, vf_rx_rings);
+
+	req.min_rsscos_ctx = cpu_to_le16(1);
+	req.max_rsscos_ctx = cpu_to_le16(1);
+	if (pf->vf_resv_strategy == BNXT_VF_RESV_STRATEGY_MINIMAL) {
+		req.min_cmpl_rings = cpu_to_le16(1);
+		req.min_tx_rings = cpu_to_le16(1);
+		req.min_rx_rings = cpu_to_le16(1);
+		req.min_l2_ctxs = cpu_to_le16(1);
+		req.min_vnics = cpu_to_le16(1);
+		req.min_stat_ctx = cpu_to_le16(1);
+		req.min_hw_ring_grps = cpu_to_le16(1);
+	} else {
+		vf_cp_rings /= num_vfs;
+		vf_tx_rings /= num_vfs;
+		vf_rx_rings /= num_vfs;
+		vf_vnics /= num_vfs;
+		vf_stat_ctx /= num_vfs;
+		vf_ring_grps /= num_vfs;
+
+		req.min_cmpl_rings = cpu_to_le16(vf_cp_rings);
+		req.min_tx_rings = cpu_to_le16(vf_tx_rings);
+		req.min_rx_rings = cpu_to_le16(vf_rx_rings);
+		req.min_l2_ctxs = cpu_to_le16(4);
+		req.min_vnics = cpu_to_le16(vf_vnics);
+		req.min_stat_ctx = cpu_to_le16(vf_stat_ctx);
+		req.min_hw_ring_grps = cpu_to_le16(vf_ring_grps);
+	}
+	req.max_cmpl_rings = cpu_to_le16(vf_cp_rings);
+	req.max_tx_rings = cpu_to_le16(vf_tx_rings);
+	req.max_rx_rings = cpu_to_le16(vf_rx_rings);
+	req.max_l2_ctxs = cpu_to_le16(4);
+	req.max_vnics = cpu_to_le16(vf_vnics);
+	req.max_stat_ctx = cpu_to_le16(vf_stat_ctx);
+	req.max_hw_ring_grps = cpu_to_le16(vf_ring_grps);
+
+	mutex_lock(&bp->hwrm_cmd_lock);
+	for (i = 0; i < num_vfs; i++) {
+		req.vf_id = cpu_to_le16(pf->first_vf_id + i);
+		rc = _hwrm_send_message(bp, &req, sizeof(req),
+					HWRM_CMD_TIMEOUT);
+		if (rc) {
+			rc = -ENOMEM;
+			break;
+		}
+		pf->active_vfs = i + 1;
+		pf->vf[i].fw_fid = pf->first_vf_id + i;
+	}
+	mutex_unlock(&bp->hwrm_cmd_lock);
+	if (pf->active_vfs) {
+		u16 n = 1;
+
+		if (pf->vf_resv_strategy != BNXT_VF_RESV_STRATEGY_MINIMAL)
+			n = pf->active_vfs;
+
+		hw_resc->max_tx_rings -= vf_tx_rings * n;
+		hw_resc->max_rx_rings -= vf_rx_rings * n;
+		hw_resc->max_hw_ring_grps -= vf_ring_grps * n;
+		hw_resc->max_cp_rings -= vf_cp_rings * n;
+		hw_resc->max_rsscos_ctxs -= pf->active_vfs;
+		hw_resc->max_stat_ctxs -= vf_stat_ctx * n;
+		hw_resc->max_vnics -= vf_vnics * n;
+
+		rc = pf->active_vfs;
+	}
+	return rc;
+}
+
+/* Only called by PF to reserve resources for VFs, returns actual number of
+ * VFs configured, or < 0 on error.
+ */
 static int bnxt_hwrm_func_cfg(struct bnxt *bp, int num_vfs)
 {
 	u32 rc = 0, mtu, i;
 	u16 vf_tx_rings, vf_rx_rings, vf_cp_rings, vf_stat_ctx, vf_vnics;
-	u16 vf_ring_grps;
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
+	u16 vf_ring_grps, max_stat_ctxs;
 	struct hwrm_func_cfg_input req = {0};
 	struct bnxt_pf_info *pf = &bp->pf;
 	int total_vf_tx_rings = 0;
 
 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
 
+	max_stat_ctxs = hw_resc->max_stat_ctxs;
+
 	/* Remaining rings are distributed equally amongs VF's for now */
-	vf_cp_rings = (pf->max_cp_rings - bp->cp_nr_rings) / num_vfs;
-	vf_stat_ctx = (pf->max_stat_ctxs - bp->num_stat_ctxs) / num_vfs;
+	vf_cp_rings = (hw_resc->max_cp_rings - bp->cp_nr_rings) / num_vfs;
+	vf_stat_ctx = (max_stat_ctxs - bp->num_stat_ctxs) / num_vfs;
 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
-		vf_rx_rings = (pf->max_rx_rings - bp->rx_nr_rings * 2) /
+		vf_rx_rings = (hw_resc->max_rx_rings - bp->rx_nr_rings * 2) /
 			      num_vfs;
 	else
-		vf_rx_rings = (pf->max_rx_rings - bp->rx_nr_rings) / num_vfs;
-	vf_ring_grps = (bp->pf.max_hw_ring_grps - bp->rx_nr_rings) / num_vfs;
-	vf_tx_rings = (pf->max_tx_rings - bp->tx_nr_rings) / num_vfs;
-	vf_vnics = (pf->max_vnics - bp->nr_vnics) / num_vfs;
+		vf_rx_rings = (hw_resc->max_rx_rings - bp->rx_nr_rings) /
+			      num_vfs;
+	vf_ring_grps = (hw_resc->max_hw_ring_grps - bp->rx_nr_rings) / num_vfs;
+	vf_tx_rings = (hw_resc->max_tx_rings - bp->tx_nr_rings) / num_vfs;
+	vf_vnics = (hw_resc->max_vnics - bp->nr_vnics) / num_vfs;
 	vf_vnics = min_t(u16, vf_vnics, vf_rx_rings);
 
 	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_MTU |
@@ -485,22 +585,34 @@ static int bnxt_hwrm_func_cfg(struct bnxt *bp, int num_vfs)
 		total_vf_tx_rings += vf_tx_rsvd;
 	}
 	mutex_unlock(&bp->hwrm_cmd_lock);
-	if (!rc) {
-		pf->max_tx_rings -= total_vf_tx_rings;
-		pf->max_rx_rings -= vf_rx_rings * num_vfs;
-		pf->max_hw_ring_grps -= vf_ring_grps * num_vfs;
-		pf->max_cp_rings -= vf_cp_rings * num_vfs;
-		pf->max_rsscos_ctxs -= num_vfs;
-		pf->max_stat_ctxs -= vf_stat_ctx * num_vfs;
-		pf->max_vnics -= vf_vnics * num_vfs;
+	if (rc)
+		rc = -ENOMEM;
+	if (pf->active_vfs) {
+		hw_resc->max_tx_rings -= total_vf_tx_rings;
+		hw_resc->max_rx_rings -= vf_rx_rings * num_vfs;
+		hw_resc->max_hw_ring_grps -= vf_ring_grps * num_vfs;
+		hw_resc->max_cp_rings -= vf_cp_rings * num_vfs;
+		hw_resc->max_rsscos_ctxs -= num_vfs;
+		hw_resc->max_stat_ctxs -= vf_stat_ctx * num_vfs;
+		hw_resc->max_vnics -= vf_vnics * num_vfs;
+		rc = pf->active_vfs;
 	}
 	return rc;
 }
 
+static int bnxt_func_cfg(struct bnxt *bp, int num_vfs)
+{
+	if (bp->flags & BNXT_FLAG_NEW_RM)
+		return bnxt_hwrm_func_vf_resc_cfg(bp, num_vfs);
+	else
+		return bnxt_hwrm_func_cfg(bp, num_vfs);
+}
+
 static int bnxt_sriov_enable(struct bnxt *bp, int *num_vfs)
 {
 	int rc = 0, vfs_supported;
 	int min_rx_rings, min_tx_rings, min_rss_ctxs;
+	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
 	int tx_ok = 0, rx_ok = 0, rss_ok = 0;
 	int avail_cp, avail_stat;
 
@@ -510,8 +622,8 @@ static int bnxt_sriov_enable(struct bnxt *bp, int *num_vfs)
 	 */
 	vfs_supported = *num_vfs;
 
-	avail_cp = bp->pf.max_cp_rings - bp->cp_nr_rings;
-	avail_stat = bp->pf.max_stat_ctxs - bp->num_stat_ctxs;
+	avail_cp = hw_resc->max_cp_rings - bp->cp_nr_rings;
+	avail_stat = hw_resc->max_stat_ctxs - bp->num_stat_ctxs;
 	avail_cp = min_t(int, avail_cp, avail_stat);
 
 	while (vfs_supported) {
@@ -520,23 +632,24 @@ static int bnxt_sriov_enable(struct bnxt *bp, int *num_vfs)
 		min_rss_ctxs = vfs_supported;
 
 		if (bp->flags & BNXT_FLAG_AGG_RINGS) {
-			if (bp->pf.max_rx_rings - bp->rx_nr_rings * 2 >=
+			if (hw_resc->max_rx_rings - bp->rx_nr_rings * 2 >=
 			    min_rx_rings)
 				rx_ok = 1;
 		} else {
-			if (bp->pf.max_rx_rings - bp->rx_nr_rings >=
+			if (hw_resc->max_rx_rings - bp->rx_nr_rings >=
 			    min_rx_rings)
 				rx_ok = 1;
 		}
-		if (bp->pf.max_vnics - bp->nr_vnics < min_rx_rings ||
+		if (hw_resc->max_vnics - bp->nr_vnics < min_rx_rings ||
 		    avail_cp < min_rx_rings)
 			rx_ok = 0;
 
-		if (bp->pf.max_tx_rings - bp->tx_nr_rings >= min_tx_rings &&
+		if (hw_resc->max_tx_rings - bp->tx_nr_rings >= min_tx_rings &&
 		    avail_cp >= min_tx_rings)
 			tx_ok = 1;
 
-		if (bp->pf.max_rsscos_ctxs - bp->rsscos_nr_ctxs >= min_rss_ctxs)
+		if (hw_resc->max_rsscos_ctxs - bp->rsscos_nr_ctxs >=
+		    min_rss_ctxs)
 			rss_ok = 1;
 
 		if (tx_ok && rx_ok && rss_ok)
@@ -561,9 +674,16 @@ static int bnxt_sriov_enable(struct bnxt *bp, int *num_vfs)
 		goto err_out1;
 
 	/* Reserve resources for VFs */
-	rc = bnxt_hwrm_func_cfg(bp, *num_vfs);
-	if (rc)
-		goto err_out2;
+	rc = bnxt_func_cfg(bp, *num_vfs);
+	if (rc != *num_vfs) {
+		if (rc <= 0) {
+			netdev_warn(bp->dev, "Unable to reserve resources for SRIOV.\n");
+			*num_vfs = 0;
+			goto err_out2;
+		}
+		netdev_warn(bp->dev, "Only able to reserve resources for %d VFs.\n", rc);
+		*num_vfs = rc;
+	}
 
 	/* Register buffers for VFs */
 	rc = bnxt_hwrm_func_buf_rgtr(bp);
@@ -766,17 +886,51 @@ static int bnxt_hwrm_exec_fwd_resp(struct bnxt *bp, struct bnxt_vf_info *vf,
 	return rc;
 }
 
+static int bnxt_vf_store_mac(struct bnxt *bp, struct bnxt_vf_info *vf)
+{
+	u32 msg_size = sizeof(struct hwrm_func_vf_cfg_input);
+	struct hwrm_func_vf_cfg_input *req =
+		(struct hwrm_func_vf_cfg_input *)vf->hwrm_cmd_req_addr;
+
+	/* Only allow VF to set a valid MAC address if the PF assigned MAC
+	 * address is zero
+	 */
+	if (req->enables & cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_DFLT_MAC_ADDR)) {
+		if (is_valid_ether_addr(req->dflt_mac_addr) &&
+		    !is_valid_ether_addr(vf->mac_addr)) {
+			ether_addr_copy(vf->vf_mac_addr, req->dflt_mac_addr);
+			return bnxt_hwrm_exec_fwd_resp(bp, vf, msg_size);
+		}
+		return bnxt_hwrm_fwd_err_resp(bp, vf, msg_size);
+	}
+	return bnxt_hwrm_exec_fwd_resp(bp, vf, msg_size);
+}
+
 static int bnxt_vf_validate_set_mac(struct bnxt *bp, struct bnxt_vf_info *vf)
 {
 	u32 msg_size = sizeof(struct hwrm_cfa_l2_filter_alloc_input);
 	struct hwrm_cfa_l2_filter_alloc_input *req =
 		(struct hwrm_cfa_l2_filter_alloc_input *)vf->hwrm_cmd_req_addr;
+	bool mac_ok = false;
 
-	if (!is_valid_ether_addr(vf->mac_addr) ||
-	    ether_addr_equal((const u8 *)req->l2_addr, vf->mac_addr))
+	/* VF MAC address must first match PF MAC address, if it is valid.
+	 * Otherwise, it must match the VF MAC address if firmware spec >=
+	 * 1.2.2
+	 */
+	if (is_valid_ether_addr(vf->mac_addr)) {
+		if (ether_addr_equal((const u8 *)req->l2_addr, vf->mac_addr))
+			mac_ok = true;
+	} else if (is_valid_ether_addr(vf->vf_mac_addr)) {
+		if (ether_addr_equal((const u8 *)req->l2_addr, vf->vf_mac_addr))
+			mac_ok = true;
+	} else if (bp->hwrm_spec_code < 0x10202) {
+		mac_ok = true;
+	} else {
+		mac_ok = true;
+	}
+	if (mac_ok)
 		return bnxt_hwrm_exec_fwd_resp(bp, vf, msg_size);
-	else
-		return bnxt_hwrm_fwd_err_resp(bp, vf, msg_size);
+	return bnxt_hwrm_fwd_err_resp(bp, vf, msg_size);
 }
 
 static int bnxt_vf_set_link(struct bnxt *bp, struct bnxt_vf_info *vf)
@@ -838,6 +992,9 @@ static int bnxt_vf_req_validate_snd(struct bnxt *bp, struct bnxt_vf_info *vf)
 	u32 req_type = le16_to_cpu(encap_req->req_type);
 
 	switch (req_type) {
+	case HWRM_FUNC_VF_CFG:
+		rc = bnxt_vf_store_mac(bp, vf);
+		break;
 	case HWRM_CFA_L2_FILTER_ALLOC:
 		rc = bnxt_vf_validate_set_mac(bp, vf);
 		break;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
index d8fee26cd45e..65c2cee35766 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c
@@ -43,7 +43,7 @@ static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev)
 	}
 
 	/* Is dev a VF-rep? */
-	if (dev != pf_bp->dev)
+	if (bnxt_dev_is_vf_rep(dev))
 		return bnxt_vf_rep_get_fid(dev);
 
 	bp = netdev_priv(dev);
@@ -54,12 +54,10 @@ static int bnxt_tc_parse_redir(struct bnxt *bp,
 			       struct bnxt_tc_actions *actions,
 			       const struct tc_action *tc_act)
 {
-	int ifindex = tcf_mirred_ifindex(tc_act);
-	struct net_device *dev;
+	struct net_device *dev = tcf_mirred_dev(tc_act);
 
-	dev = __dev_get_by_index(dev_net(bp->dev), ifindex);
 	if (!dev) {
-		netdev_info(bp->dev, "no dev for ifindex=%d", ifindex);
+		netdev_info(bp->dev, "no dev in mirred action");
 		return -EINVAL;
 	}
 
@@ -148,9 +146,6 @@ static int bnxt_tc_parse_actions(struct bnxt *bp,
 		}
 	}
 
-	if (rc)
-		return rc;
-
 	if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) {
 		if (actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) {
 			/* dst_fid is PF's fid */
@@ -164,7 +159,7 @@ static int bnxt_tc_parse_actions(struct bnxt *bp,
 		}
 	}
 
-	return rc;
+	return 0;
 }
 
 #define GET_KEY(flow_cmd, key_type)					\
@@ -354,6 +349,9 @@ static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle)
 	if (rc)
 		netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d",
 			    __func__, flow_handle, rc);
+
+	if (rc)
+		rc = -EIO;
 	return rc;
 }
 
@@ -489,13 +487,15 @@ static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow,
 	req.action_flags = cpu_to_le16(action_flags);
 
 	mutex_lock(&bp->hwrm_cmd_lock);
-
 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 	if (!rc)
 		*flow_handle = resp->flow_handle;
-
 	mutex_unlock(&bp->hwrm_cmd_lock);
 
+	if (rc == HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR)
+		rc = -ENOSPC;
+	else if (rc)
+		rc = -EIO;
 	return rc;
 }
 
@@ -566,6 +566,8 @@ static int hwrm_cfa_decap_filter_alloc(struct bnxt *bp,
 		netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
 	mutex_unlock(&bp->hwrm_cmd_lock);
 
+	if (rc)
+		rc = -EIO;
 	return rc;
 }
 
@@ -581,6 +583,9 @@ static int hwrm_cfa_decap_filter_free(struct bnxt *bp,
 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 	if (rc)
 		netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
+
+	if (rc)
+		rc = -EIO;
 	return rc;
 }
 
@@ -629,6 +634,8 @@ static int hwrm_cfa_encap_record_alloc(struct bnxt *bp,
 		netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
 	mutex_unlock(&bp->hwrm_cmd_lock);
 
+	if (rc)
+		rc = -EIO;
 	return rc;
 }
 
@@ -644,6 +651,9 @@ static int hwrm_cfa_encap_record_free(struct bnxt *bp,
 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 	if (rc)
 		netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
+
+	if (rc)
+		rc = -EIO;
 	return rc;
 }
 
@@ -1274,11 +1284,8 @@ static int bnxt_tc_del_flow(struct bnxt *bp,
 	flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
 					   &tc_flow_cmd->cookie,
 					   tc_info->flow_ht_params);
-	if (!flow_node) {
-		netdev_info(bp->dev, "ERROR: no flow_node for cookie %lx",
-			    tc_flow_cmd->cookie);
+	if (!flow_node)
 		return -EINVAL;
-	}
 
 	return __bnxt_tc_del_flow(bp, flow_node);
 }
@@ -1295,11 +1302,8 @@ static int bnxt_tc_get_flow_stats(struct bnxt *bp,
 	flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
 					   &tc_flow_cmd->cookie,
 					   tc_info->flow_ht_params);
-	if (!flow_node) {
-		netdev_info(bp->dev, "Error: no flow_node for cookie %lx",
-			    tc_flow_cmd->cookie);
+	if (!flow_node)
 		return -1;
-	}
 
 	flow = &flow_node->flow;
 	curr_stats = &flow->stats;
@@ -1349,8 +1353,10 @@ bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp, int num_flows,
 	} else {
 		netdev_info(bp->dev, "error rc=%d", rc);
 	}
-
 	mutex_unlock(&bp->hwrm_cmd_lock);
+
+	if (rc)
+		rc = -EIO;
 	return rc;
 }
 
@@ -1417,11 +1423,7 @@ bnxt_tc_flow_stats_batch_prep(struct bnxt *bp,
 	void *flow_node;
 	int rc, i;
 
-	rc = rhashtable_walk_start(iter);
-	if (rc && rc != -EAGAIN) {
-		i = 0;
-		goto done;
-	}
+	rhashtable_walk_start(iter);
 
 	rc = 0;
 	for (i = 0; i < BNXT_FLOW_STATS_BATCH_MAX; i++) {
@@ -1483,9 +1485,6 @@ int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid,
 {
 	int rc = 0;
 
-	if (cls_flower->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (cls_flower->command) {
 	case TC_CLSFLOWER_REPLACE:
 		rc = bnxt_tc_add_flow(bp, src_fid, cls_flower);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
index 69186d188c43..26290403f38f 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
@@ -124,7 +124,8 @@ static int bnxt_vf_rep_setup_tc_block_cb(enum tc_setup_type type,
 	struct bnxt *bp = vf_rep->bp;
 	int vf_fid = bp->pf.vf[vf_rep->vf_idx].fw_fid;
 
-	if (!bnxt_tc_flower_enabled(vf_rep->bp) || !tc_can_offload(bp->dev))
+	if (!bnxt_tc_flower_enabled(vf_rep->bp) ||
+	    !tc_cls_can_offload_and_chain0(bp->dev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
@@ -241,6 +242,11 @@ static const struct net_device_ops bnxt_vf_rep_netdev_ops = {
 	.ndo_get_phys_port_name = bnxt_vf_rep_get_phys_port_name
 };
 
+bool bnxt_dev_is_vf_rep(struct net_device *dev)
+{
+	return dev->netdev_ops == &bnxt_vf_rep_netdev_ops;
+}
+
 /* Called when the parent PF interface is closed:
  * As the mode transition from SWITCHDEV to LEGACY
  * happens under the rtnl_lock() this routine is safe
@@ -376,6 +382,26 @@ static void bnxt_vf_rep_netdev_init(struct bnxt *bp, struct bnxt_vf_rep *vf_rep,
 	ether_addr_copy(dev->dev_addr, dev->perm_addr);
 }
 
+static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
+{
+	struct pci_dev *pdev = bp->pdev;
+	int pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
+	u32 dw;
+
+	if (!pos) {
+		netdev_info(bp->dev, "Unable do read adapter's DSN");
+		return -EOPNOTSUPP;
+	}
+
+	/* DSN (two dw) is at an offset of 4 from the cap pos */
+	pos += 4;
+	pci_read_config_dword(pdev, pos, &dw);
+	put_unaligned_le32(dw, &dsn[0]);
+	pci_read_config_dword(pdev, pos + 4, &dw);
+	put_unaligned_le32(dw, &dsn[4]);
+	return 0;
+}
+
 static int bnxt_vf_reps_create(struct bnxt *bp)
 {
 	u16 *cfa_code_map = NULL, num_vfs = pci_num_vf(bp->pdev);
@@ -440,6 +466,11 @@ static int bnxt_vf_reps_create(struct bnxt *bp)
 		}
 	}
 
+	/* Read the adapter's DSN to use as the eswitch switch_id */
+	rc = bnxt_pcie_dsn_get(bp, bp->switch_id);
+	if (rc)
+		goto err;
+
 	/* publish cfa_code_map only after all VF-reps have been initialized */
 	bp->cfa_code_map = cfa_code_map;
 	bp->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
index fb06bbe70e42..38b9a75ad724 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.h
@@ -28,6 +28,7 @@ static inline u16 bnxt_vf_rep_get_fid(struct net_device *dev)
 	return bp->pf.vf[vf_rep->vf_idx].fw_fid;
 }
 
+bool bnxt_dev_is_vf_rep(struct net_device *dev);
 int bnxt_dl_eswitch_mode_get(struct devlink *devlink, u16 *mode);
 int bnxt_dl_eswitch_mode_set(struct devlink *devlink, u16 mode);
 
@@ -54,5 +55,10 @@ static inline u16 bnxt_vf_rep_get_fid(struct net_device *dev)
 {
 	return 0;
 }
+
+static inline bool bnxt_dev_is_vf_rep(struct net_device *dev)
+{
+	return false;
+}
 #endif /* CONFIG_BNXT_SRIOV */
 #endif /* BNXT_VFR_H */
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
index 261e5847557a..1389ab5e05df 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c
@@ -96,6 +96,7 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 	xdp.data = *data_ptr;
 	xdp_set_data_meta_invalid(&xdp);
 	xdp.data_end = *data_ptr + *len;
+	xdp.rxq = &rxr->xdp_rxq;
 	orig_data = xdp.data;
 	mapping = rx_buf->mapping - bp->rx_dma_offset;
 
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
index 24b4f4ceceef..b1e35a9accf1 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
@@ -2527,9 +2527,10 @@ static void bcmgenet_irq_task(struct work_struct *work)
 	spin_unlock_irq(&priv->lock);
 
 	/* Link UP/DOWN event */
-	if (status & UMAC_IRQ_LINK_EVENT)
-		phy_mac_interrupt(priv->dev->phydev,
-				  !!(status & UMAC_IRQ_LINK_UP));
+	if (status & UMAC_IRQ_LINK_EVENT) {
+		priv->dev->phydev->link = !!(status & UMAC_IRQ_LINK_UP);
+		phy_mac_interrupt(priv->dev->phydev);
+	}
 }
 
 /* bcmgenet_isr1: handle Rx and Tx priority queues */
diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c
index 8995cfefbfcf..f2593978ae75 100644
--- a/drivers/net/ethernet/broadcom/tg3.c
+++ b/drivers/net/ethernet/broadcom/tg3.c
@@ -922,8 +922,8 @@ static int tg3_ape_send_event(struct tg3 *tp, u32 event)
 	if (!(apedata & APE_FW_STATUS_READY))
 		return -EAGAIN;
 
-	/* Wait for up to 1 millisecond for APE to service previous event. */
-	err = tg3_ape_event_lock(tp, 1000);
+	/* Wait for up to 20 millisecond for APE to service previous event. */
+	err = tg3_ape_event_lock(tp, 20000);
 	if (err)
 		return err;
 
@@ -946,6 +946,7 @@ static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
 
 	switch (kind) {
 	case RESET_KIND_INIT:
+		tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_COUNT, tp->ape_hb++);
 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG,
 				APE_HOST_SEG_SIG_MAGIC);
 		tg3_ape_write32(tp, TG3_APE_HOST_SEG_LEN,
@@ -962,13 +963,6 @@ static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
 		event = APE_EVENT_STATUS_STATE_START;
 		break;
 	case RESET_KIND_SHUTDOWN:
-		/* With the interface we are currently using,
-		 * APE does not track driver state.  Wiping
-		 * out the HOST SEGMENT SIGNATURE forces
-		 * the APE to assume OS absent status.
-		 */
-		tg3_ape_write32(tp, TG3_APE_HOST_SEG_SIG, 0x0);
-
 		if (device_may_wakeup(&tp->pdev->dev) &&
 		    tg3_flag(tp, WOL_ENABLE)) {
 			tg3_ape_write32(tp, TG3_APE_HOST_WOL_SPEED,
@@ -990,6 +984,18 @@ static void tg3_ape_driver_state_change(struct tg3 *tp, int kind)
 	tg3_ape_send_event(tp, event);
 }
 
+static void tg3_send_ape_heartbeat(struct tg3 *tp,
+				   unsigned long interval)
+{
+	/* Check if hb interval has exceeded */
+	if (!tg3_flag(tp, ENABLE_APE) ||
+	    time_before(jiffies, tp->ape_hb_jiffies + interval))
+		return;
+
+	tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_COUNT, tp->ape_hb++);
+	tp->ape_hb_jiffies = jiffies;
+}
+
 static void tg3_disable_ints(struct tg3 *tp)
 {
 	int i;
@@ -3227,7 +3233,7 @@ static int tg3_nvram_read_using_eeprom(struct tg3 *tp,
 	return 0;
 }
 
-#define NVRAM_CMD_TIMEOUT 5000
+#define NVRAM_CMD_TIMEOUT 10000
 
 static int tg3_nvram_exec_cmd(struct tg3 *tp, u32 nvram_cmd)
 {
@@ -7262,6 +7268,7 @@ static int tg3_poll_msix(struct napi_struct *napi, int budget)
 		}
 	}
 
+	tg3_send_ape_heartbeat(tp, TG3_APE_HB_INTERVAL << 1);
 	return work_done;
 
 tx_recovery:
@@ -7344,6 +7351,7 @@ static int tg3_poll(struct napi_struct *napi, int budget)
 		}
 	}
 
+	tg3_send_ape_heartbeat(tp, TG3_APE_HB_INTERVAL << 1);
 	return work_done;
 
 tx_recovery:
@@ -10732,7 +10740,7 @@ static int tg3_reset_hw(struct tg3 *tp, bool reset_phy)
 	if (tg3_flag(tp, ENABLE_APE))
 		/* Write our heartbeat update interval to APE. */
 		tg3_ape_write32(tp, TG3_APE_HOST_HEARTBEAT_INT_MS,
-				APE_HOST_HEARTBEAT_INT_DISABLE);
+				APE_HOST_HEARTBEAT_INT_5SEC);
 
 	tg3_write_sig_post_reset(tp, RESET_KIND_INIT);
 
@@ -11077,6 +11085,9 @@ static void tg3_timer(struct timer_list *t)
 		tp->asf_counter = tp->asf_multiplier;
 	}
 
+	/* Update the APE heartbeat every 5 seconds.*/
+	tg3_send_ape_heartbeat(tp, TG3_APE_HB_INTERVAL);
+
 	spin_unlock(&tp->lock);
 
 restart_timer:
@@ -14789,7 +14800,7 @@ static void tg3_get_5717_nvram_info(struct tg3 *tp)
 
 static void tg3_get_5720_nvram_info(struct tg3 *tp)
 {
-	u32 nvcfg1, nvmpinstrp;
+	u32 nvcfg1, nvmpinstrp, nv_status;
 
 	nvcfg1 = tr32(NVRAM_CFG1);
 	nvmpinstrp = nvcfg1 & NVRAM_CFG1_5752VENDOR_MASK;
@@ -14801,6 +14812,23 @@ static void tg3_get_5720_nvram_info(struct tg3 *tp)
 		}
 
 		switch (nvmpinstrp) {
+		case FLASH_5762_MX25L_100:
+		case FLASH_5762_MX25L_200:
+		case FLASH_5762_MX25L_400:
+		case FLASH_5762_MX25L_800:
+		case FLASH_5762_MX25L_160_320:
+			tp->nvram_pagesize = 4096;
+			tp->nvram_jedecnum = JEDEC_MACRONIX;
+			tg3_flag_set(tp, NVRAM_BUFFERED);
+			tg3_flag_set(tp, NO_NVRAM_ADDR_TRANS);
+			tg3_flag_set(tp, FLASH);
+			nv_status = tr32(NVRAM_AUTOSENSE_STATUS);
+			tp->nvram_size =
+				(1 << (nv_status >> AUTOSENSE_DEVID &
+						AUTOSENSE_DEVID_MASK)
+					<< AUTOSENSE_SIZE_IN_MB);
+			return;
+
 		case FLASH_5762_EEPROM_HD:
 			nvmpinstrp = FLASH_5720_EEPROM_HD;
 			break;
@@ -16636,6 +16664,8 @@ static int tg3_get_invariants(struct tg3 *tp, const struct pci_device_id *ent)
 				       pci_state_reg);
 
 		tg3_ape_lock_init(tp);
+		tp->ape_hb_interval =
+			msecs_to_jiffies(APE_HOST_HEARTBEAT_INT_5SEC);
 	}
 
 	/* Set up tp->grc_local_ctrl before calling
diff --git a/drivers/net/ethernet/broadcom/tg3.h b/drivers/net/ethernet/broadcom/tg3.h
index 1f0271fa7c74..1d61aa3efda1 100644
--- a/drivers/net/ethernet/broadcom/tg3.h
+++ b/drivers/net/ethernet/broadcom/tg3.h
@@ -1863,7 +1863,7 @@
 #define NVRAM_STAT			0x00007004
 #define NVRAM_WRDATA			0x00007008
 #define NVRAM_ADDR			0x0000700c
-#define  NVRAM_ADDR_MSK			0x00ffffff
+#define  NVRAM_ADDR_MSK			0x07ffffff
 #define NVRAM_RDDATA			0x00007010
 #define NVRAM_CFG1			0x00007014
 #define  NVRAM_CFG1_FLASHIF_ENAB	 0x00000001
@@ -1945,6 +1945,11 @@
 #define  FLASH_5720_EEPROM_LD		 0x00000003
 #define  FLASH_5762_EEPROM_HD		 0x02000001
 #define  FLASH_5762_EEPROM_LD		 0x02000003
+#define  FLASH_5762_MX25L_100           0x00800000
+#define  FLASH_5762_MX25L_200           0x00800002
+#define  FLASH_5762_MX25L_400           0x00800001
+#define  FLASH_5762_MX25L_800           0x00800003
+#define  FLASH_5762_MX25L_160_320       0x03800002
 #define  FLASH_5720VENDOR_M_ATMEL_DB011D 0x01000000
 #define  FLASH_5720VENDOR_M_ATMEL_DB021D 0x01000002
 #define  FLASH_5720VENDOR_M_ATMEL_DB041D 0x01000001
@@ -2009,7 +2014,11 @@
 /* 0x702c unused */
 
 #define NVRAM_ADDR_LOCKOUT		0x00007030
-/* 0x7034 --> 0x7500 unused */
+#define NVRAM_AUTOSENSE_STATUS         0x00007038
+#define AUTOSENSE_DEVID                        0x00000010
+#define AUTOSENSE_DEVID_MASK           0x00000007
+#define AUTOSENSE_SIZE_IN_MB           17
+/* 0x703c --> 0x7500 unused */
 
 #define OTP_MODE			0x00007500
 #define OTP_MODE_OTP_THRU_GRC		 0x00000001
@@ -2499,6 +2508,7 @@
 #define TG3_APE_LOCK_PHY3		5
 #define TG3_APE_LOCK_GPIO		7
 
+#define TG3_APE_HB_INTERVAL             (tp->ape_hb_interval)
 #define TG3_EEPROM_SB_F1R2_MBA_OFF	0x10
 
 
@@ -3378,6 +3388,7 @@ struct tg3 {
 #define JEDEC_ST			0x20
 #define JEDEC_SAIFUN			0x4f
 #define JEDEC_SST			0xbf
+#define JEDEC_MACRONIX                 0xc2
 
 #define ATMEL_AT24C02_CHIP_SIZE		TG3_NVRAM_SIZE_2KB
 #define ATMEL_AT24C02_PAGE_SIZE		(8)
@@ -3413,6 +3424,10 @@ struct tg3 {
 	struct device			*hwmon_dev;
 	bool				link_up;
 	bool				pcierr_recovery;
+
+	u32                             ape_hb;
+	unsigned long                   ape_hb_interval;
+	unsigned long                   ape_hb_jiffies;
 };
 
 /* Accessor macros for chip and asic attributes
diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h
index c93f3a2dc6c1..86659823b259 100644
--- a/drivers/net/ethernet/cadence/macb.h
+++ b/drivers/net/ethernet/cadence/macb.h
@@ -13,6 +13,7 @@
 #include <linux/phy.h>
 #include <linux/ptp_clock_kernel.h>
 #include <linux/net_tstamp.h>
+#include <linux/interrupt.h>
 
 #if defined(CONFIG_ARCH_DMA_ADDR_T_64BIT) || defined(CONFIG_MACB_USE_HWSTAMP)
 #define MACB_EXT_DESC
@@ -164,14 +165,38 @@
 #define GEM_DCFG5		0x0290 /* Design Config 5 */
 #define GEM_DCFG6		0x0294 /* Design Config 6 */
 #define GEM_DCFG7		0x0298 /* Design Config 7 */
+#define GEM_DCFG8		0x029C /* Design Config 8 */
 
 #define GEM_TXBDCTRL	0x04cc /* TX Buffer Descriptor control register */
 #define GEM_RXBDCTRL	0x04d0 /* RX Buffer Descriptor control register */
 
+/* Screener Type 2 match registers */
+#define GEM_SCRT2		0x540
+
+/* EtherType registers */
+#define GEM_ETHT		0x06E0
+
+/* Type 2 compare registers */
+#define GEM_T2CMPW0		0x0700
+#define GEM_T2CMPW1		0x0704
+#define T2CMP_OFST(t2idx)	(t2idx * 2)
+
+/* type 2 compare registers
+ * each location requires 3 compare regs
+ */
+#define GEM_IP4SRC_CMP(idx)		(idx * 3)
+#define GEM_IP4DST_CMP(idx)		(idx * 3 + 1)
+#define GEM_PORT_CMP(idx)		(idx * 3 + 2)
+
+/* Which screening type 2 EtherType register will be used (0 - 7) */
+#define SCRT2_ETHT		0
+
 #define GEM_ISR(hw_q)		(0x0400 + ((hw_q) << 2))
 #define GEM_TBQP(hw_q)		(0x0440 + ((hw_q) << 2))
 #define GEM_TBQPH(hw_q)		(0x04C8)
 #define GEM_RBQP(hw_q)		(0x0480 + ((hw_q) << 2))
+#define GEM_RBQS(hw_q)		(0x04A0 + ((hw_q) << 2))
+#define GEM_RBQPH(hw_q)		(0x04D4)
 #define GEM_IER(hw_q)		(0x0600 + ((hw_q) << 2))
 #define GEM_IDR(hw_q)		(0x0620 + ((hw_q) << 2))
 #define GEM_IMR(hw_q)		(0x0640 + ((hw_q) << 2))
@@ -455,6 +480,16 @@
 #define GEM_DAW64_OFFSET			23
 #define GEM_DAW64_SIZE				1
 
+/* Bitfields in DCFG8. */
+#define GEM_T1SCR_OFFSET			24
+#define GEM_T1SCR_SIZE				8
+#define GEM_T2SCR_OFFSET			16
+#define GEM_T2SCR_SIZE				8
+#define GEM_SCR2ETH_OFFSET			8
+#define GEM_SCR2ETH_SIZE			8
+#define GEM_SCR2CMP_OFFSET			0
+#define GEM_SCR2CMP_SIZE			8
+
 /* Bitfields in TISUBN */
 #define GEM_SUBNSINCR_OFFSET			0
 #define GEM_SUBNSINCR_SIZE			16
@@ -483,6 +518,66 @@
 #define GEM_RXTSMODE_OFFSET			4 /* RX Descriptor Timestamp Insertion mode */
 #define GEM_RXTSMODE_SIZE			2
 
+/* Bitfields in SCRT2 */
+#define GEM_QUEUE_OFFSET			0 /* Queue Number */
+#define GEM_QUEUE_SIZE				4
+#define GEM_VLANPR_OFFSET			4 /* VLAN Priority */
+#define GEM_VLANPR_SIZE				3
+#define GEM_VLANEN_OFFSET			8 /* VLAN Enable */
+#define GEM_VLANEN_SIZE				1
+#define GEM_ETHT2IDX_OFFSET			9 /* Index to screener type 2 EtherType register */
+#define GEM_ETHT2IDX_SIZE			3
+#define GEM_ETHTEN_OFFSET			12 /* EtherType Enable */
+#define GEM_ETHTEN_SIZE				1
+#define GEM_CMPA_OFFSET				13 /* Compare A - Index to screener type 2 Compare register */
+#define GEM_CMPA_SIZE				5
+#define GEM_CMPAEN_OFFSET			18 /* Compare A Enable */
+#define GEM_CMPAEN_SIZE				1
+#define GEM_CMPB_OFFSET				19 /* Compare B - Index to screener type 2 Compare register */
+#define GEM_CMPB_SIZE				5
+#define GEM_CMPBEN_OFFSET			24 /* Compare B Enable */
+#define GEM_CMPBEN_SIZE				1
+#define GEM_CMPC_OFFSET				25 /* Compare C - Index to screener type 2 Compare register */
+#define GEM_CMPC_SIZE				5
+#define GEM_CMPCEN_OFFSET			30 /* Compare C Enable */
+#define GEM_CMPCEN_SIZE				1
+
+/* Bitfields in ETHT */
+#define GEM_ETHTCMP_OFFSET			0 /* EtherType compare value */
+#define GEM_ETHTCMP_SIZE			16
+
+/* Bitfields in T2CMPW0 */
+#define GEM_T2CMP_OFFSET			16 /* 0xFFFF0000 compare value */
+#define GEM_T2CMP_SIZE				16
+#define GEM_T2MASK_OFFSET			0 /* 0x0000FFFF compare value or mask */
+#define GEM_T2MASK_SIZE				16
+
+/* Bitfields in T2CMPW1 */
+#define GEM_T2DISMSK_OFFSET			9 /* disable mask */
+#define GEM_T2DISMSK_SIZE			1
+#define GEM_T2CMPOFST_OFFSET			7 /* compare offset */
+#define GEM_T2CMPOFST_SIZE			2
+#define GEM_T2OFST_OFFSET			0 /* offset value */
+#define GEM_T2OFST_SIZE				7
+
+/* Offset for screener type 2 compare values (T2CMPOFST).
+ * Note the offset is applied after the specified point,
+ * e.g. GEM_T2COMPOFST_ETYPE denotes the EtherType field, so an offset
+ * of 12 bytes from this would be the source IP address in an IP header
+ */
+#define GEM_T2COMPOFST_SOF		0
+#define GEM_T2COMPOFST_ETYPE	1
+#define GEM_T2COMPOFST_IPHDR	2
+#define GEM_T2COMPOFST_TCPUDP	3
+
+/* offset from EtherType to IP address */
+#define ETYPE_SRCIP_OFFSET			12
+#define ETYPE_DSTIP_OFFSET			16
+
+/* offset from IP header to port */
+#define IPHDR_SRCPORT_OFFSET		0
+#define IPHDR_DSTPORT_OFFSET		2
+
 /* Transmit DMA buffer descriptor Word 1 */
 #define GEM_DMA_TXVALID_OFFSET		23 /* timestamp has been captured in the Buffer Descriptor */
 #define GEM_DMA_TXVALID_SIZE		1
@@ -583,6 +678,8 @@
 #define gem_writel(port, reg, value)	(port)->macb_reg_writel((port), GEM_##reg, (value))
 #define queue_readl(queue, reg)		(queue)->bp->macb_reg_readl((queue)->bp, (queue)->reg)
 #define queue_writel(queue, reg, value)	(queue)->bp->macb_reg_writel((queue)->bp, (queue)->reg, (value))
+#define gem_readl_n(port, reg, idx)		(port)->macb_reg_readl((port), GEM_##reg + idx * 4)
+#define gem_writel_n(port, reg, idx, value)	(port)->macb_reg_writel((port), GEM_##reg + idx * 4, (value))
 
 #define PTP_TS_BUFFER_SIZE		128 /* must be power of 2 */
 
@@ -920,13 +1017,42 @@ static const struct gem_statistic gem_statistics[] = {
 
 #define GEM_STATS_LEN ARRAY_SIZE(gem_statistics)
 
+#define QUEUE_STAT_TITLE(title) {	\
+	.stat_string = title,			\
+}
+
+/* per queue statistics, each should be unsigned long type */
+struct queue_stats {
+	union {
+		unsigned long first;
+		unsigned long rx_packets;
+	};
+	unsigned long rx_bytes;
+	unsigned long rx_dropped;
+	unsigned long tx_packets;
+	unsigned long tx_bytes;
+	unsigned long tx_dropped;
+};
+
+static const struct gem_statistic queue_statistics[] = {
+		QUEUE_STAT_TITLE("rx_packets"),
+		QUEUE_STAT_TITLE("rx_bytes"),
+		QUEUE_STAT_TITLE("rx_dropped"),
+		QUEUE_STAT_TITLE("tx_packets"),
+		QUEUE_STAT_TITLE("tx_bytes"),
+		QUEUE_STAT_TITLE("tx_dropped"),
+};
+
+#define QUEUE_STATS_LEN ARRAY_SIZE(queue_statistics)
+
 struct macb;
+struct macb_queue;
 
 struct macb_or_gem_ops {
 	int	(*mog_alloc_rx_buffers)(struct macb *bp);
 	void	(*mog_free_rx_buffers)(struct macb *bp);
 	void	(*mog_init_rings)(struct macb *bp);
-	int	(*mog_rx)(struct macb *bp, int budget);
+	int	(*mog_rx)(struct macb_queue *queue, int budget);
 };
 
 /* MACB-PTP interface: adapt to platform needs. */
@@ -968,6 +1094,9 @@ struct macb_queue {
 	unsigned int		IMR;
 	unsigned int		TBQP;
 	unsigned int		TBQPH;
+	unsigned int		RBQS;
+	unsigned int		RBQP;
+	unsigned int		RBQPH;
 
 	unsigned int		tx_head, tx_tail;
 	struct macb_dma_desc	*tx_ring;
@@ -975,6 +1104,16 @@ struct macb_queue {
 	dma_addr_t		tx_ring_dma;
 	struct work_struct	tx_error_task;
 
+	dma_addr_t		rx_ring_dma;
+	dma_addr_t		rx_buffers_dma;
+	unsigned int		rx_tail;
+	unsigned int		rx_prepared_head;
+	struct macb_dma_desc	*rx_ring;
+	struct sk_buff		**rx_skbuff;
+	void			*rx_buffers;
+	struct napi_struct	napi;
+	struct queue_stats stats;
+
 #ifdef CONFIG_MACB_USE_HWSTAMP
 	struct work_struct	tx_ts_task;
 	unsigned int		tx_ts_head, tx_ts_tail;
@@ -982,6 +1121,16 @@ struct macb_queue {
 #endif
 };
 
+struct ethtool_rx_fs_item {
+	struct ethtool_rx_flow_spec fs;
+	struct list_head list;
+};
+
+struct ethtool_rx_fs_list {
+	struct list_head list;
+	unsigned int count;
+};
+
 struct macb {
 	void __iomem		*regs;
 	bool			native_io;
@@ -990,11 +1139,6 @@ struct macb {
 	u32	(*macb_reg_readl)(struct macb *bp, int offset);
 	void	(*macb_reg_writel)(struct macb *bp, int offset, u32 value);
 
-	unsigned int		rx_tail;
-	unsigned int		rx_prepared_head;
-	struct macb_dma_desc	*rx_ring;
-	struct sk_buff		**rx_skbuff;
-	void			*rx_buffers;
 	size_t			rx_buffer_size;
 
 	unsigned int		rx_ring_size;
@@ -1011,15 +1155,11 @@ struct macb {
 	struct clk		*tx_clk;
 	struct clk		*rx_clk;
 	struct net_device	*dev;
-	struct napi_struct	napi;
 	union {
 		struct macb_stats	macb;
 		struct gem_stats	gem;
 	}			hw_stats;
 
-	dma_addr_t		rx_ring_dma;
-	dma_addr_t		rx_buffers_dma;
-
 	struct macb_or_gem_ops	macbgem_ops;
 
 	struct mii_bus		*mii_bus;
@@ -1032,7 +1172,6 @@ struct macb {
 	unsigned int		dma_burst_length;
 
 	phy_interface_t		phy_interface;
-	struct gpio_desc	*reset_gpio;
 
 	/* AT91RM9200 transmit */
 	struct sk_buff *skb;			/* holds skb until xmit interrupt completes */
@@ -1040,7 +1179,7 @@ struct macb {
 	int skb_length;				/* saved skb length for pci_unmap_single */
 	unsigned int		max_tx_length;
 
-	u64			ethtool_stats[GEM_STATS_LEN];
+	u64			ethtool_stats[GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES];
 
 	unsigned int		rx_frm_len_mask;
 	unsigned int		jumbo_max_len;
@@ -1057,6 +1196,13 @@ struct macb {
 	struct ptp_clock_info ptp_clock_info;
 	struct tsu_incr tsu_incr;
 	struct hwtstamp_config tstamp_config;
+
+	/* RX queue filer rule set*/
+	struct ethtool_rx_fs_list rx_fs_list;
+	spinlock_t rx_fs_lock;
+	unsigned int max_tuples;
+
+	struct tasklet_struct	hresp_err_tasklet;
 };
 
 #ifdef CONFIG_MACB_USE_HWSTAMP
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index 72a67f74b97b..e84afcf1ecb5 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -194,17 +194,17 @@ static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index)
 	return index & (bp->rx_ring_size - 1);
 }
 
-static struct macb_dma_desc *macb_rx_desc(struct macb *bp, unsigned int index)
+static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index)
 {
-	index = macb_rx_ring_wrap(bp, index);
-	index = macb_adj_dma_desc_idx(bp, index);
-	return &bp->rx_ring[index];
+	index = macb_rx_ring_wrap(queue->bp, index);
+	index = macb_adj_dma_desc_idx(queue->bp, index);
+	return &queue->rx_ring[index];
 }
 
-static void *macb_rx_buffer(struct macb *bp, unsigned int index)
+static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index)
 {
-	return bp->rx_buffers + bp->rx_buffer_size *
-	       macb_rx_ring_wrap(bp, index);
+	return queue->rx_buffers + queue->bp->rx_buffer_size *
+	       macb_rx_ring_wrap(queue->bp, index);
 }
 
 /* I/O accessors */
@@ -759,7 +759,9 @@ static void macb_tx_error_task(struct work_struct *work)
 					    macb_tx_ring_wrap(bp, tail),
 					    skb->data);
 				bp->dev->stats.tx_packets++;
+				queue->stats.tx_packets++;
 				bp->dev->stats.tx_bytes += skb->len;
+				queue->stats.tx_bytes += skb->len;
 			}
 		} else {
 			/* "Buffers exhausted mid-frame" errors may only happen
@@ -859,7 +861,9 @@ static void macb_tx_interrupt(struct macb_queue *queue)
 					    macb_tx_ring_wrap(bp, tail),
 					    skb->data);
 				bp->dev->stats.tx_packets++;
+				queue->stats.tx_packets++;
 				bp->dev->stats.tx_bytes += skb->len;
+				queue->stats.tx_bytes += skb->len;
 			}
 
 			/* Now we can safely release resources */
@@ -881,24 +885,25 @@ static void macb_tx_interrupt(struct macb_queue *queue)
 		netif_wake_subqueue(bp->dev, queue_index);
 }
 
-static void gem_rx_refill(struct macb *bp)
+static void gem_rx_refill(struct macb_queue *queue)
 {
 	unsigned int		entry;
 	struct sk_buff		*skb;
 	dma_addr_t		paddr;
+	struct macb *bp = queue->bp;
 	struct macb_dma_desc *desc;
 
-	while (CIRC_SPACE(bp->rx_prepared_head, bp->rx_tail,
-			  bp->rx_ring_size) > 0) {
-		entry = macb_rx_ring_wrap(bp, bp->rx_prepared_head);
+	while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,
+			bp->rx_ring_size) > 0) {
+		entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);
 
 		/* Make hw descriptor updates visible to CPU */
 		rmb();
 
-		bp->rx_prepared_head++;
-		desc = macb_rx_desc(bp, entry);
+		queue->rx_prepared_head++;
+		desc = macb_rx_desc(queue, entry);
 
-		if (!bp->rx_skbuff[entry]) {
+		if (!queue->rx_skbuff[entry]) {
 			/* allocate sk_buff for this free entry in ring */
 			skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);
 			if (unlikely(!skb)) {
@@ -916,7 +921,7 @@ static void gem_rx_refill(struct macb *bp)
 				break;
 			}
 
-			bp->rx_skbuff[entry] = skb;
+			queue->rx_skbuff[entry] = skb;
 
 			if (entry == bp->rx_ring_size - 1)
 				paddr |= MACB_BIT(RX_WRAP);
@@ -934,18 +939,18 @@ static void gem_rx_refill(struct macb *bp)
 	/* Make descriptor updates visible to hardware */
 	wmb();
 
-	netdev_vdbg(bp->dev, "rx ring: prepared head %d, tail %d\n",
-		    bp->rx_prepared_head, bp->rx_tail);
+	netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",
+			queue, queue->rx_prepared_head, queue->rx_tail);
 }
 
 /* Mark DMA descriptors from begin up to and not including end as unused */
-static void discard_partial_frame(struct macb *bp, unsigned int begin,
+static void discard_partial_frame(struct macb_queue *queue, unsigned int begin,
 				  unsigned int end)
 {
 	unsigned int frag;
 
 	for (frag = begin; frag != end; frag++) {
-		struct macb_dma_desc *desc = macb_rx_desc(bp, frag);
+		struct macb_dma_desc *desc = macb_rx_desc(queue, frag);
 
 		desc->addr &= ~MACB_BIT(RX_USED);
 	}
@@ -959,8 +964,9 @@ static void discard_partial_frame(struct macb *bp, unsigned int begin,
 	 */
 }
 
-static int gem_rx(struct macb *bp, int budget)
+static int gem_rx(struct macb_queue *queue, int budget)
 {
+	struct macb *bp = queue->bp;
 	unsigned int		len;
 	unsigned int		entry;
 	struct sk_buff		*skb;
@@ -972,8 +978,8 @@ static int gem_rx(struct macb *bp, int budget)
 		dma_addr_t addr;
 		bool rxused;
 
-		entry = macb_rx_ring_wrap(bp, bp->rx_tail);
-		desc = macb_rx_desc(bp, entry);
+		entry = macb_rx_ring_wrap(bp, queue->rx_tail);
+		desc = macb_rx_desc(queue, entry);
 
 		/* Make hw descriptor updates visible to CPU */
 		rmb();
@@ -985,24 +991,26 @@ static int gem_rx(struct macb *bp, int budget)
 		if (!rxused)
 			break;
 
-		bp->rx_tail++;
+		queue->rx_tail++;
 		count++;
 
 		if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) {
 			netdev_err(bp->dev,
 				   "not whole frame pointed by descriptor\n");
 			bp->dev->stats.rx_dropped++;
+			queue->stats.rx_dropped++;
 			break;
 		}
-		skb = bp->rx_skbuff[entry];
+		skb = queue->rx_skbuff[entry];
 		if (unlikely(!skb)) {
 			netdev_err(bp->dev,
 				   "inconsistent Rx descriptor chain\n");
 			bp->dev->stats.rx_dropped++;
+			queue->stats.rx_dropped++;
 			break;
 		}
 		/* now everything is ready for receiving packet */
-		bp->rx_skbuff[entry] = NULL;
+		queue->rx_skbuff[entry] = NULL;
 		len = ctrl & bp->rx_frm_len_mask;
 
 		netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);
@@ -1019,7 +1027,9 @@ static int gem_rx(struct macb *bp, int budget)
 			skb->ip_summed = CHECKSUM_UNNECESSARY;
 
 		bp->dev->stats.rx_packets++;
+		queue->stats.rx_packets++;
 		bp->dev->stats.rx_bytes += skb->len;
+		queue->stats.rx_bytes += skb->len;
 
 		gem_ptp_do_rxstamp(bp, skb, desc);
 
@@ -1035,12 +1045,12 @@ static int gem_rx(struct macb *bp, int budget)
 		netif_receive_skb(skb);
 	}
 
-	gem_rx_refill(bp);
+	gem_rx_refill(queue);
 
 	return count;
 }
 
-static int macb_rx_frame(struct macb *bp, unsigned int first_frag,
+static int macb_rx_frame(struct macb_queue *queue, unsigned int first_frag,
 			 unsigned int last_frag)
 {
 	unsigned int len;
@@ -1048,8 +1058,9 @@ static int macb_rx_frame(struct macb *bp, unsigned int first_frag,
 	unsigned int offset;
 	struct sk_buff *skb;
 	struct macb_dma_desc *desc;
+	struct macb *bp = queue->bp;
 
-	desc = macb_rx_desc(bp, last_frag);
+	desc = macb_rx_desc(queue, last_frag);
 	len = desc->ctrl & bp->rx_frm_len_mask;
 
 	netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",
@@ -1068,7 +1079,7 @@ static int macb_rx_frame(struct macb *bp, unsigned int first_frag,
 	if (!skb) {
 		bp->dev->stats.rx_dropped++;
 		for (frag = first_frag; ; frag++) {
-			desc = macb_rx_desc(bp, frag);
+			desc = macb_rx_desc(queue, frag);
 			desc->addr &= ~MACB_BIT(RX_USED);
 			if (frag == last_frag)
 				break;
@@ -1096,10 +1107,10 @@ static int macb_rx_frame(struct macb *bp, unsigned int first_frag,
 			frag_len = len - offset;
 		}
 		skb_copy_to_linear_data_offset(skb, offset,
-					       macb_rx_buffer(bp, frag),
+					       macb_rx_buffer(queue, frag),
 					       frag_len);
 		offset += bp->rx_buffer_size;
-		desc = macb_rx_desc(bp, frag);
+		desc = macb_rx_desc(queue, frag);
 		desc->addr &= ~MACB_BIT(RX_USED);
 
 		if (frag == last_frag)
@@ -1121,32 +1132,34 @@ static int macb_rx_frame(struct macb *bp, unsigned int first_frag,
 	return 0;
 }
 
-static inline void macb_init_rx_ring(struct macb *bp)
+static inline void macb_init_rx_ring(struct macb_queue *queue)
 {
+	struct macb *bp = queue->bp;
 	dma_addr_t addr;
 	struct macb_dma_desc *desc = NULL;
 	int i;
 
-	addr = bp->rx_buffers_dma;
+	addr = queue->rx_buffers_dma;
 	for (i = 0; i < bp->rx_ring_size; i++) {
-		desc = macb_rx_desc(bp, i);
+		desc = macb_rx_desc(queue, i);
 		macb_set_addr(bp, desc, addr);
 		desc->ctrl = 0;
 		addr += bp->rx_buffer_size;
 	}
 	desc->addr |= MACB_BIT(RX_WRAP);
-	bp->rx_tail = 0;
+	queue->rx_tail = 0;
 }
 
-static int macb_rx(struct macb *bp, int budget)
+static int macb_rx(struct macb_queue *queue, int budget)
 {
+	struct macb *bp = queue->bp;
 	bool reset_rx_queue = false;
 	int received = 0;
 	unsigned int tail;
 	int first_frag = -1;
 
-	for (tail = bp->rx_tail; budget > 0; tail++) {
-		struct macb_dma_desc *desc = macb_rx_desc(bp, tail);
+	for (tail = queue->rx_tail; budget > 0; tail++) {
+		struct macb_dma_desc *desc = macb_rx_desc(queue, tail);
 		u32 ctrl;
 
 		/* Make hw descriptor updates visible to CPU */
@@ -1159,7 +1172,7 @@ static int macb_rx(struct macb *bp, int budget)
 
 		if (ctrl & MACB_BIT(RX_SOF)) {
 			if (first_frag != -1)
-				discard_partial_frame(bp, first_frag, tail);
+				discard_partial_frame(queue, first_frag, tail);
 			first_frag = tail;
 		}
 
@@ -1171,7 +1184,7 @@ static int macb_rx(struct macb *bp, int budget)
 				continue;
 			}
 
-			dropped = macb_rx_frame(bp, first_frag, tail);
+			dropped = macb_rx_frame(queue, first_frag, tail);
 			first_frag = -1;
 			if (unlikely(dropped < 0)) {
 				reset_rx_queue = true;
@@ -1195,8 +1208,8 @@ static int macb_rx(struct macb *bp, int budget)
 		ctrl = macb_readl(bp, NCR);
 		macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
 
-		macb_init_rx_ring(bp);
-		macb_writel(bp, RBQP, bp->rx_ring_dma);
+		macb_init_rx_ring(queue);
+		queue_writel(queue, RBQP, queue->rx_ring_dma);
 
 		macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
 
@@ -1205,16 +1218,17 @@ static int macb_rx(struct macb *bp, int budget)
 	}
 
 	if (first_frag != -1)
-		bp->rx_tail = first_frag;
+		queue->rx_tail = first_frag;
 	else
-		bp->rx_tail = tail;
+		queue->rx_tail = tail;
 
 	return received;
 }
 
 static int macb_poll(struct napi_struct *napi, int budget)
 {
-	struct macb *bp = container_of(napi, struct macb, napi);
+	struct macb_queue *queue = container_of(napi, struct macb_queue, napi);
+	struct macb *bp = queue->bp;
 	int work_done;
 	u32 status;
 
@@ -1224,7 +1238,7 @@ static int macb_poll(struct napi_struct *napi, int budget)
 	netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n",
 		    (unsigned long)status, budget);
 
-	work_done = bp->macbgem_ops.mog_rx(bp, budget);
+	work_done = bp->macbgem_ops.mog_rx(queue, budget);
 	if (work_done < budget) {
 		napi_complete_done(napi, work_done);
 
@@ -1232,10 +1246,10 @@ static int macb_poll(struct napi_struct *napi, int budget)
 		status = macb_readl(bp, RSR);
 		if (status) {
 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
-				macb_writel(bp, ISR, MACB_BIT(RCOMP));
+				queue_writel(queue, ISR, MACB_BIT(RCOMP));
 			napi_reschedule(napi);
 		} else {
-			macb_writel(bp, IER, MACB_RX_INT_FLAGS);
+			queue_writel(queue, IER, MACB_RX_INT_FLAGS);
 		}
 	}
 
@@ -1244,6 +1258,57 @@ static int macb_poll(struct napi_struct *napi, int budget)
 	return work_done;
 }
 
+static void macb_hresp_error_task(unsigned long data)
+{
+	struct macb *bp = (struct macb *)data;
+	struct net_device *dev = bp->dev;
+	struct macb_queue *queue = bp->queues;
+	unsigned int q;
+	u32 ctrl;
+
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		queue_writel(queue, IDR, MACB_RX_INT_FLAGS |
+					 MACB_TX_INT_FLAGS |
+					 MACB_BIT(HRESP));
+	}
+	ctrl = macb_readl(bp, NCR);
+	ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
+	macb_writel(bp, NCR, ctrl);
+
+	netif_tx_stop_all_queues(dev);
+	netif_carrier_off(dev);
+
+	bp->macbgem_ops.mog_init_rings(bp);
+
+	/* Initialize TX and RX buffers */
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+		if (bp->hw_dma_cap & HW_DMA_CAP_64B)
+			queue_writel(queue, RBQPH,
+				     upper_32_bits(queue->rx_ring_dma));
+#endif
+		queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+		if (bp->hw_dma_cap & HW_DMA_CAP_64B)
+			queue_writel(queue, TBQPH,
+				     upper_32_bits(queue->tx_ring_dma));
+#endif
+
+		/* Enable interrupts */
+		queue_writel(queue, IER,
+			     MACB_RX_INT_FLAGS |
+			     MACB_TX_INT_FLAGS |
+			     MACB_BIT(HRESP));
+	}
+
+	ctrl |= MACB_BIT(RE) | MACB_BIT(TE);
+	macb_writel(bp, NCR, ctrl);
+
+	netif_carrier_on(dev);
+	netif_tx_start_all_queues(dev);
+}
+
 static irqreturn_t macb_interrupt(int irq, void *dev_id)
 {
 	struct macb_queue *queue = dev_id;
@@ -1282,9 +1347,9 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id)
 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
 				queue_writel(queue, ISR, MACB_BIT(RCOMP));
 
-			if (napi_schedule_prep(&bp->napi)) {
+			if (napi_schedule_prep(&queue->napi)) {
 				netdev_vdbg(bp->dev, "scheduling RX softirq\n");
-				__napi_schedule(&bp->napi);
+				__napi_schedule(&queue->napi);
 			}
 		}
 
@@ -1333,10 +1398,7 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id)
 		}
 
 		if (status & MACB_BIT(HRESP)) {
-			/* TODO: Reset the hardware, and maybe move the
-			 * netdev_err to a lower-priority context as well
-			 * (work queue?)
-			 */
+			tasklet_schedule(&bp->hresp_err_tasklet);
 			netdev_err(dev, "DMA bus error: HRESP not OK\n");
 
 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
@@ -1708,38 +1770,44 @@ static void gem_free_rx_buffers(struct macb *bp)
 {
 	struct sk_buff		*skb;
 	struct macb_dma_desc	*desc;
+	struct macb_queue *queue;
 	dma_addr_t		addr;
+	unsigned int q;
 	int i;
 
-	if (!bp->rx_skbuff)
-		return;
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		if (!queue->rx_skbuff)
+			continue;
 
-	for (i = 0; i < bp->rx_ring_size; i++) {
-		skb = bp->rx_skbuff[i];
+		for (i = 0; i < bp->rx_ring_size; i++) {
+			skb = queue->rx_skbuff[i];
 
-		if (!skb)
-			continue;
+			if (!skb)
+				continue;
 
-		desc = macb_rx_desc(bp, i);
-		addr = macb_get_addr(bp, desc);
+			desc = macb_rx_desc(queue, i);
+			addr = macb_get_addr(bp, desc);
 
-		dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,
-				 DMA_FROM_DEVICE);
-		dev_kfree_skb_any(skb);
-		skb = NULL;
-	}
+			dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,
+					DMA_FROM_DEVICE);
+			dev_kfree_skb_any(skb);
+			skb = NULL;
+		}
 
-	kfree(bp->rx_skbuff);
-	bp->rx_skbuff = NULL;
+		kfree(queue->rx_skbuff);
+		queue->rx_skbuff = NULL;
+	}
 }
 
 static void macb_free_rx_buffers(struct macb *bp)
 {
-	if (bp->rx_buffers) {
+	struct macb_queue *queue = &bp->queues[0];
+
+	if (queue->rx_buffers) {
 		dma_free_coherent(&bp->pdev->dev,
 				  bp->rx_ring_size * bp->rx_buffer_size,
-				  bp->rx_buffers, bp->rx_buffers_dma);
-		bp->rx_buffers = NULL;
+				  queue->rx_buffers, queue->rx_buffers_dma);
+		queue->rx_buffers = NULL;
 	}
 }
 
@@ -1748,11 +1816,12 @@ static void macb_free_consistent(struct macb *bp)
 	struct macb_queue *queue;
 	unsigned int q;
 
+	queue = &bp->queues[0];
 	bp->macbgem_ops.mog_free_rx_buffers(bp);
-	if (bp->rx_ring) {
+	if (queue->rx_ring) {
 		dma_free_coherent(&bp->pdev->dev, RX_RING_BYTES(bp),
-				  bp->rx_ring, bp->rx_ring_dma);
-		bp->rx_ring = NULL;
+				queue->rx_ring, queue->rx_ring_dma);
+		queue->rx_ring = NULL;
 	}
 
 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
@@ -1768,32 +1837,37 @@ static void macb_free_consistent(struct macb *bp)
 
 static int gem_alloc_rx_buffers(struct macb *bp)
 {
+	struct macb_queue *queue;
+	unsigned int q;
 	int size;
 
-	size = bp->rx_ring_size * sizeof(struct sk_buff *);
-	bp->rx_skbuff = kzalloc(size, GFP_KERNEL);
-	if (!bp->rx_skbuff)
-		return -ENOMEM;
-	else
-		netdev_dbg(bp->dev,
-			   "Allocated %d RX struct sk_buff entries at %p\n",
-			   bp->rx_ring_size, bp->rx_skbuff);
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		size = bp->rx_ring_size * sizeof(struct sk_buff *);
+		queue->rx_skbuff = kzalloc(size, GFP_KERNEL);
+		if (!queue->rx_skbuff)
+			return -ENOMEM;
+		else
+			netdev_dbg(bp->dev,
+				   "Allocated %d RX struct sk_buff entries at %p\n",
+				   bp->rx_ring_size, queue->rx_skbuff);
+	}
 	return 0;
 }
 
 static int macb_alloc_rx_buffers(struct macb *bp)
 {
+	struct macb_queue *queue = &bp->queues[0];
 	int size;
 
 	size = bp->rx_ring_size * bp->rx_buffer_size;
-	bp->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,
-					    &bp->rx_buffers_dma, GFP_KERNEL);
-	if (!bp->rx_buffers)
+	queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,
+					    &queue->rx_buffers_dma, GFP_KERNEL);
+	if (!queue->rx_buffers)
 		return -ENOMEM;
 
 	netdev_dbg(bp->dev,
 		   "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n",
-		   size, (unsigned long)bp->rx_buffers_dma, bp->rx_buffers);
+		   size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);
 	return 0;
 }
 
@@ -1819,17 +1893,16 @@ static int macb_alloc_consistent(struct macb *bp)
 		queue->tx_skb = kmalloc(size, GFP_KERNEL);
 		if (!queue->tx_skb)
 			goto out_err;
-	}
-
-	size = RX_RING_BYTES(bp);
-	bp->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
-					 &bp->rx_ring_dma, GFP_KERNEL);
-	if (!bp->rx_ring)
-		goto out_err;
-	netdev_dbg(bp->dev,
-		   "Allocated RX ring of %d bytes at %08lx (mapped %p)\n",
-		   size, (unsigned long)bp->rx_ring_dma, bp->rx_ring);
 
+		size = RX_RING_BYTES(bp);
+		queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
+						 &queue->rx_ring_dma, GFP_KERNEL);
+		if (!queue->rx_ring)
+			goto out_err;
+		netdev_dbg(bp->dev,
+			   "Allocated RX ring of %d bytes at %08lx (mapped %p)\n",
+			   size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);
+	}
 	if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))
 		goto out_err;
 
@@ -1856,12 +1929,13 @@ static void gem_init_rings(struct macb *bp)
 		desc->ctrl |= MACB_BIT(TX_WRAP);
 		queue->tx_head = 0;
 		queue->tx_tail = 0;
-	}
 
-	bp->rx_tail = 0;
-	bp->rx_prepared_head = 0;
+		queue->rx_tail = 0;
+		queue->rx_prepared_head = 0;
+
+		gem_rx_refill(queue);
+	}
 
-	gem_rx_refill(bp);
 }
 
 static void macb_init_rings(struct macb *bp)
@@ -1869,7 +1943,7 @@ static void macb_init_rings(struct macb *bp)
 	int i;
 	struct macb_dma_desc *desc = NULL;
 
-	macb_init_rx_ring(bp);
+	macb_init_rx_ring(&bp->queues[0]);
 
 	for (i = 0; i < bp->tx_ring_size; i++) {
 		desc = macb_tx_desc(&bp->queues[0], i);
@@ -1978,11 +2052,20 @@ static u32 macb_dbw(struct macb *bp)
  */
 static void macb_configure_dma(struct macb *bp)
 {
+	struct macb_queue *queue;
+	u32 buffer_size;
+	unsigned int q;
 	u32 dmacfg;
 
+	buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;
 	if (macb_is_gem(bp)) {
 		dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);
-		dmacfg |= GEM_BF(RXBS, bp->rx_buffer_size / RX_BUFFER_MULTIPLE);
+		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+			if (q)
+				queue_writel(queue, RBQS, buffer_size);
+			else
+				dmacfg |= GEM_BF(RXBS, buffer_size);
+		}
 		if (bp->dma_burst_length)
 			dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);
 		dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);
@@ -2051,12 +2134,12 @@ static void macb_init_hw(struct macb *bp)
 	macb_configure_dma(bp);
 
 	/* Initialize TX and RX buffers */
-	macb_writel(bp, RBQP, lower_32_bits(bp->rx_ring_dma));
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+		queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));
 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
-	if (bp->hw_dma_cap & HW_DMA_CAP_64B)
-		macb_writel(bp, RBQPH, upper_32_bits(bp->rx_ring_dma));
+		if (bp->hw_dma_cap & HW_DMA_CAP_64B)
+			queue_writel(queue, RBQPH, upper_32_bits(queue->rx_ring_dma));
 #endif
-	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
 		queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)
@@ -2197,6 +2280,8 @@ static int macb_open(struct net_device *dev)
 {
 	struct macb *bp = netdev_priv(dev);
 	size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;
+	struct macb_queue *queue;
+	unsigned int q;
 	int err;
 
 	netdev_dbg(bp->dev, "open\n");
@@ -2218,11 +2303,12 @@ static int macb_open(struct net_device *dev)
 		return err;
 	}
 
-	napi_enable(&bp->napi);
-
 	bp->macbgem_ops.mog_init_rings(bp);
 	macb_init_hw(bp);
 
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
+		napi_enable(&queue->napi);
+
 	/* schedule a link state check */
 	phy_start(dev->phydev);
 
@@ -2237,10 +2323,14 @@ static int macb_open(struct net_device *dev)
 static int macb_close(struct net_device *dev)
 {
 	struct macb *bp = netdev_priv(dev);
+	struct macb_queue *queue;
 	unsigned long flags;
+	unsigned int q;
 
 	netif_tx_stop_all_queues(dev);
-	napi_disable(&bp->napi);
+
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
+		napi_disable(&queue->napi);
 
 	if (dev->phydev)
 		phy_stop(dev->phydev);
@@ -2270,7 +2360,10 @@ static int macb_change_mtu(struct net_device *dev, int new_mtu)
 
 static void gem_update_stats(struct macb *bp)
 {
-	unsigned int i;
+	struct macb_queue *queue;
+	unsigned int i, q, idx;
+	unsigned long *stat;
+
 	u32 *p = &bp->hw_stats.gem.tx_octets_31_0;
 
 	for (i = 0; i < GEM_STATS_LEN; ++i, ++p) {
@@ -2287,6 +2380,11 @@ static void gem_update_stats(struct macb *bp)
 			*(++p) += val;
 		}
 	}
+
+	idx = GEM_STATS_LEN;
+	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
+		for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)
+			bp->ethtool_stats[idx++] = *stat;
 }
 
 static struct net_device_stats *gem_get_stats(struct macb *bp)
@@ -2334,14 +2432,17 @@ static void gem_get_ethtool_stats(struct net_device *dev,
 
 	bp = netdev_priv(dev);
 	gem_update_stats(bp);
-	memcpy(data, &bp->ethtool_stats, sizeof(u64) * GEM_STATS_LEN);
+	memcpy(data, &bp->ethtool_stats, sizeof(u64)
+			* (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES));
 }
 
 static int gem_get_sset_count(struct net_device *dev, int sset)
 {
+	struct macb *bp = netdev_priv(dev);
+
 	switch (sset) {
 	case ETH_SS_STATS:
-		return GEM_STATS_LEN;
+		return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -2349,13 +2450,25 @@ static int gem_get_sset_count(struct net_device *dev, int sset)
 
 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)
 {
+	char stat_string[ETH_GSTRING_LEN];
+	struct macb *bp = netdev_priv(dev);
+	struct macb_queue *queue;
 	unsigned int i;
+	unsigned int q;
 
 	switch (sset) {
 	case ETH_SS_STATS:
 		for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN)
 			memcpy(p, gem_statistics[i].stat_string,
 			       ETH_GSTRING_LEN);
+
+		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
+			for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) {
+				snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s",
+						q, queue_statistics[i].stat_string);
+				memcpy(p, stat_string, ETH_GSTRING_LEN);
+			}
+		}
 		break;
 	}
 }
@@ -2603,6 +2716,307 @@ static int macb_get_ts_info(struct net_device *netdev,
 	return ethtool_op_get_ts_info(netdev, info);
 }
 
+static void gem_enable_flow_filters(struct macb *bp, bool enable)
+{
+	struct ethtool_rx_fs_item *item;
+	u32 t2_scr;
+	int num_t2_scr;
+
+	num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8));
+
+	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
+		struct ethtool_rx_flow_spec *fs = &item->fs;
+		struct ethtool_tcpip4_spec *tp4sp_m;
+
+		if (fs->location >= num_t2_scr)
+			continue;
+
+		t2_scr = gem_readl_n(bp, SCRT2, fs->location);
+
+		/* enable/disable screener regs for the flow entry */
+		t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr);
+
+		/* only enable fields with no masking */
+		tp4sp_m = &(fs->m_u.tcp_ip4_spec);
+
+		if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))
+			t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr);
+		else
+			t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr);
+
+		if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))
+			t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr);
+		else
+			t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr);
+
+		if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))
+			t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr);
+		else
+			t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr);
+
+		gem_writel_n(bp, SCRT2, fs->location, t2_scr);
+	}
+}
+
+static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs)
+{
+	struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m;
+	uint16_t index = fs->location;
+	u32 w0, w1, t2_scr;
+	bool cmp_a = false;
+	bool cmp_b = false;
+	bool cmp_c = false;
+
+	tp4sp_v = &(fs->h_u.tcp_ip4_spec);
+	tp4sp_m = &(fs->m_u.tcp_ip4_spec);
+
+	/* ignore field if any masking set */
+	if (tp4sp_m->ip4src == 0xFFFFFFFF) {
+		/* 1st compare reg - IP source address */
+		w0 = 0;
+		w1 = 0;
+		w0 = tp4sp_v->ip4src;
+		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
+		w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
+		w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1);
+		gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0);
+		gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1);
+		cmp_a = true;
+	}
+
+	/* ignore field if any masking set */
+	if (tp4sp_m->ip4dst == 0xFFFFFFFF) {
+		/* 2nd compare reg - IP destination address */
+		w0 = 0;
+		w1 = 0;
+		w0 = tp4sp_v->ip4dst;
+		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
+		w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
+		w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1);
+		gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0);
+		gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1);
+		cmp_b = true;
+	}
+
+	/* ignore both port fields if masking set in both */
+	if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {
+		/* 3rd compare reg - source port, destination port */
+		w0 = 0;
+		w1 = 0;
+		w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1);
+		if (tp4sp_m->psrc == tp4sp_m->pdst) {
+			w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);
+			w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
+			w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
+			w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
+		} else {
+			/* only one port definition */
+			w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */
+			w0 = GEM_BFINS(T2MASK, 0xFFFF, w0);
+			if (tp4sp_m->psrc == 0xFFFF) { /* src port */
+				w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);
+				w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
+			} else { /* dst port */
+				w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
+				w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1);
+			}
+		}
+		gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0);
+		gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1);
+		cmp_c = true;
+	}
+
+	t2_scr = 0;
+	t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);
+	t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr);
+	if (cmp_a)
+		t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr);
+	if (cmp_b)
+		t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr);
+	if (cmp_c)
+		t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr);
+	gem_writel_n(bp, SCRT2, index, t2_scr);
+}
+
+static int gem_add_flow_filter(struct net_device *netdev,
+		struct ethtool_rxnfc *cmd)
+{
+	struct macb *bp = netdev_priv(netdev);
+	struct ethtool_rx_flow_spec *fs = &cmd->fs;
+	struct ethtool_rx_fs_item *item, *newfs;
+	unsigned long flags;
+	int ret = -EINVAL;
+	bool added = false;
+
+	newfs = kmalloc(sizeof(*newfs), GFP_KERNEL);
+	if (newfs == NULL)
+		return -ENOMEM;
+	memcpy(&newfs->fs, fs, sizeof(newfs->fs));
+
+	netdev_dbg(netdev,
+			"Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
+			fs->flow_type, (int)fs->ring_cookie, fs->location,
+			htonl(fs->h_u.tcp_ip4_spec.ip4src),
+			htonl(fs->h_u.tcp_ip4_spec.ip4dst),
+			htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst));
+
+	spin_lock_irqsave(&bp->rx_fs_lock, flags);
+
+	/* find correct place to add in list */
+	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
+		if (item->fs.location > newfs->fs.location) {
+			list_add_tail(&newfs->list, &item->list);
+			added = true;
+			break;
+		} else if (item->fs.location == fs->location) {
+			netdev_err(netdev, "Rule not added: location %d not free!\n",
+					fs->location);
+			ret = -EBUSY;
+			goto err;
+		}
+	}
+	if (!added)
+		list_add_tail(&newfs->list, &bp->rx_fs_list.list);
+
+	gem_prog_cmp_regs(bp, fs);
+	bp->rx_fs_list.count++;
+	/* enable filtering if NTUPLE on */
+	if (netdev->features & NETIF_F_NTUPLE)
+		gem_enable_flow_filters(bp, 1);
+
+	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
+	return 0;
+
+err:
+	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
+	kfree(newfs);
+	return ret;
+}
+
+static int gem_del_flow_filter(struct net_device *netdev,
+		struct ethtool_rxnfc *cmd)
+{
+	struct macb *bp = netdev_priv(netdev);
+	struct ethtool_rx_fs_item *item;
+	struct ethtool_rx_flow_spec *fs;
+	unsigned long flags;
+
+	spin_lock_irqsave(&bp->rx_fs_lock, flags);
+
+	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
+		if (item->fs.location == cmd->fs.location) {
+			/* disable screener regs for the flow entry */
+			fs = &(item->fs);
+			netdev_dbg(netdev,
+					"Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
+					fs->flow_type, (int)fs->ring_cookie, fs->location,
+					htonl(fs->h_u.tcp_ip4_spec.ip4src),
+					htonl(fs->h_u.tcp_ip4_spec.ip4dst),
+					htons(fs->h_u.tcp_ip4_spec.psrc),
+					htons(fs->h_u.tcp_ip4_spec.pdst));
+
+			gem_writel_n(bp, SCRT2, fs->location, 0);
+
+			list_del(&item->list);
+			bp->rx_fs_list.count--;
+			spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
+			kfree(item);
+			return 0;
+		}
+	}
+
+	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
+	return -EINVAL;
+}
+
+static int gem_get_flow_entry(struct net_device *netdev,
+		struct ethtool_rxnfc *cmd)
+{
+	struct macb *bp = netdev_priv(netdev);
+	struct ethtool_rx_fs_item *item;
+
+	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
+		if (item->fs.location == cmd->fs.location) {
+			memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));
+			return 0;
+		}
+	}
+	return -EINVAL;
+}
+
+static int gem_get_all_flow_entries(struct net_device *netdev,
+		struct ethtool_rxnfc *cmd, u32 *rule_locs)
+{
+	struct macb *bp = netdev_priv(netdev);
+	struct ethtool_rx_fs_item *item;
+	uint32_t cnt = 0;
+
+	list_for_each_entry(item, &bp->rx_fs_list.list, list) {
+		if (cnt == cmd->rule_cnt)
+			return -EMSGSIZE;
+		rule_locs[cnt] = item->fs.location;
+		cnt++;
+	}
+	cmd->data = bp->max_tuples;
+	cmd->rule_cnt = cnt;
+
+	return 0;
+}
+
+static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
+		u32 *rule_locs)
+{
+	struct macb *bp = netdev_priv(netdev);
+	int ret = 0;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_GRXRINGS:
+		cmd->data = bp->num_queues;
+		break;
+	case ETHTOOL_GRXCLSRLCNT:
+		cmd->rule_cnt = bp->rx_fs_list.count;
+		break;
+	case ETHTOOL_GRXCLSRULE:
+		ret = gem_get_flow_entry(netdev, cmd);
+		break;
+	case ETHTOOL_GRXCLSRLALL:
+		ret = gem_get_all_flow_entries(netdev, cmd, rule_locs);
+		break;
+	default:
+		netdev_err(netdev,
+			  "Command parameter %d is not supported\n", cmd->cmd);
+		ret = -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
+static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
+{
+	struct macb *bp = netdev_priv(netdev);
+	int ret;
+
+	switch (cmd->cmd) {
+	case ETHTOOL_SRXCLSRLINS:
+		if ((cmd->fs.location >= bp->max_tuples)
+				|| (cmd->fs.ring_cookie >= bp->num_queues)) {
+			ret = -EINVAL;
+			break;
+		}
+		ret = gem_add_flow_filter(netdev, cmd);
+		break;
+	case ETHTOOL_SRXCLSRLDEL:
+		ret = gem_del_flow_filter(netdev, cmd);
+		break;
+	default:
+		netdev_err(netdev,
+			  "Command parameter %d is not supported\n", cmd->cmd);
+		ret = -EOPNOTSUPP;
+	}
+
+	return ret;
+}
+
 static const struct ethtool_ops macb_ethtool_ops = {
 	.get_regs_len		= macb_get_regs_len,
 	.get_regs		= macb_get_regs,
@@ -2628,6 +3042,8 @@ static const struct ethtool_ops gem_ethtool_ops = {
 	.set_link_ksettings     = phy_ethtool_set_link_ksettings,
 	.get_ringparam		= macb_get_ringparam,
 	.set_ringparam		= macb_set_ringparam,
+	.get_rxnfc			= gem_get_rxnfc,
+	.set_rxnfc			= gem_set_rxnfc,
 };
 
 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
@@ -2685,6 +3101,12 @@ static int macb_set_features(struct net_device *netdev,
 		gem_writel(bp, NCFGR, netcfg);
 	}
 
+	/* RX Flow Filters */
+	if ((changed & NETIF_F_NTUPLE) && macb_is_gem(bp)) {
+		bool turn_on = features & NETIF_F_NTUPLE;
+
+		gem_enable_flow_filters(bp, turn_on);
+	}
 	return 0;
 }
 
@@ -2850,7 +3272,7 @@ static int macb_init(struct platform_device *pdev)
 	struct macb *bp = netdev_priv(dev);
 	struct macb_queue *queue;
 	int err;
-	u32 val;
+	u32 val, reg;
 
 	bp->tx_ring_size = DEFAULT_TX_RING_SIZE;
 	bp->rx_ring_size = DEFAULT_RX_RING_SIZE;
@@ -2865,15 +3287,20 @@ static int macb_init(struct platform_device *pdev)
 
 		queue = &bp->queues[q];
 		queue->bp = bp;
+		netif_napi_add(dev, &queue->napi, macb_poll, 64);
 		if (hw_q) {
 			queue->ISR  = GEM_ISR(hw_q - 1);
 			queue->IER  = GEM_IER(hw_q - 1);
 			queue->IDR  = GEM_IDR(hw_q - 1);
 			queue->IMR  = GEM_IMR(hw_q - 1);
 			queue->TBQP = GEM_TBQP(hw_q - 1);
+			queue->RBQP = GEM_RBQP(hw_q - 1);
+			queue->RBQS = GEM_RBQS(hw_q - 1);
 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
-			if (bp->hw_dma_cap & HW_DMA_CAP_64B)
+			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
 				queue->TBQPH = GEM_TBQPH(hw_q - 1);
+				queue->RBQPH = GEM_RBQPH(hw_q - 1);
+			}
 #endif
 		} else {
 			/* queue0 uses legacy registers */
@@ -2882,9 +3309,12 @@ static int macb_init(struct platform_device *pdev)
 			queue->IDR  = MACB_IDR;
 			queue->IMR  = MACB_IMR;
 			queue->TBQP = MACB_TBQP;
+			queue->RBQP = MACB_RBQP;
 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
-			if (bp->hw_dma_cap & HW_DMA_CAP_64B)
+			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
 				queue->TBQPH = MACB_TBQPH;
+				queue->RBQPH = MACB_RBQPH;
+			}
 #endif
 		}
 
@@ -2908,7 +3338,6 @@ static int macb_init(struct platform_device *pdev)
 	}
 
 	dev->netdev_ops = &macb_netdev_ops;
-	netif_napi_add(dev, &bp->napi, macb_poll, 64);
 
 	/* setup appropriated routines according to adapter type */
 	if (macb_is_gem(bp)) {
@@ -2941,6 +3370,30 @@ static int macb_init(struct platform_device *pdev)
 		dev->hw_features &= ~NETIF_F_SG;
 	dev->features = dev->hw_features;
 
+	/* Check RX Flow Filters support.
+	 * Max Rx flows set by availability of screeners & compare regs:
+	 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs
+	 */
+	reg = gem_readl(bp, DCFG8);
+	bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),
+			GEM_BFEXT(T2SCR, reg));
+	if (bp->max_tuples > 0) {
+		/* also needs one ethtype match to check IPv4 */
+		if (GEM_BFEXT(SCR2ETH, reg) > 0) {
+			/* program this reg now */
+			reg = 0;
+			reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg);
+			gem_writel_n(bp, ETHT, SCRT2_ETHT, reg);
+			/* Filtering is supported in hw but don't enable it in kernel now */
+			dev->hw_features |= NETIF_F_NTUPLE;
+			/* init Rx flow definitions */
+			INIT_LIST_HEAD(&bp->rx_fs_list.list);
+			bp->rx_fs_list.count = 0;
+			spin_lock_init(&bp->rx_fs_lock);
+		} else
+			bp->max_tuples = 0;
+	}
+
 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {
 		val = 0;
 		if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII)
@@ -2977,34 +3430,35 @@ static int macb_init(struct platform_device *pdev)
 static int at91ether_start(struct net_device *dev)
 {
 	struct macb *lp = netdev_priv(dev);
+	struct macb_queue *q = &lp->queues[0];
 	struct macb_dma_desc *desc;
 	dma_addr_t addr;
 	u32 ctl;
 	int i;
 
-	lp->rx_ring = dma_alloc_coherent(&lp->pdev->dev,
+	q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,
 					 (AT91ETHER_MAX_RX_DESCR *
 					  macb_dma_desc_get_size(lp)),
-					 &lp->rx_ring_dma, GFP_KERNEL);
-	if (!lp->rx_ring)
+					 &q->rx_ring_dma, GFP_KERNEL);
+	if (!q->rx_ring)
 		return -ENOMEM;
 
-	lp->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,
+	q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,
 					    AT91ETHER_MAX_RX_DESCR *
 					    AT91ETHER_MAX_RBUFF_SZ,
-					    &lp->rx_buffers_dma, GFP_KERNEL);
-	if (!lp->rx_buffers) {
+					    &q->rx_buffers_dma, GFP_KERNEL);
+	if (!q->rx_buffers) {
 		dma_free_coherent(&lp->pdev->dev,
 				  AT91ETHER_MAX_RX_DESCR *
 				  macb_dma_desc_get_size(lp),
-				  lp->rx_ring, lp->rx_ring_dma);
-		lp->rx_ring = NULL;
+				  q->rx_ring, q->rx_ring_dma);
+		q->rx_ring = NULL;
 		return -ENOMEM;
 	}
 
-	addr = lp->rx_buffers_dma;
+	addr = q->rx_buffers_dma;
 	for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) {
-		desc = macb_rx_desc(lp, i);
+		desc = macb_rx_desc(q, i);
 		macb_set_addr(lp, desc, addr);
 		desc->ctrl = 0;
 		addr += AT91ETHER_MAX_RBUFF_SZ;
@@ -3014,10 +3468,10 @@ static int at91ether_start(struct net_device *dev)
 	desc->addr |= MACB_BIT(RX_WRAP);
 
 	/* Reset buffer index */
-	lp->rx_tail = 0;
+	q->rx_tail = 0;
 
 	/* Program address of descriptor list in Rx Buffer Queue register */
-	macb_writel(lp, RBQP, lp->rx_ring_dma);
+	macb_writel(lp, RBQP, q->rx_ring_dma);
 
 	/* Enable Receive and Transmit */
 	ctl = macb_readl(lp, NCR);
@@ -3064,6 +3518,7 @@ static int at91ether_open(struct net_device *dev)
 static int at91ether_close(struct net_device *dev)
 {
 	struct macb *lp = netdev_priv(dev);
+	struct macb_queue *q = &lp->queues[0];
 	u32 ctl;
 
 	/* Disable Receiver and Transmitter */
@@ -3084,13 +3539,13 @@ static int at91ether_close(struct net_device *dev)
 	dma_free_coherent(&lp->pdev->dev,
 			  AT91ETHER_MAX_RX_DESCR *
 			  macb_dma_desc_get_size(lp),
-			  lp->rx_ring, lp->rx_ring_dma);
-	lp->rx_ring = NULL;
+			  q->rx_ring, q->rx_ring_dma);
+	q->rx_ring = NULL;
 
 	dma_free_coherent(&lp->pdev->dev,
 			  AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ,
-			  lp->rx_buffers, lp->rx_buffers_dma);
-	lp->rx_buffers = NULL;
+			  q->rx_buffers, q->rx_buffers_dma);
+	q->rx_buffers = NULL;
 
 	return 0;
 }
@@ -3134,14 +3589,15 @@ static int at91ether_start_xmit(struct sk_buff *skb, struct net_device *dev)
 static void at91ether_rx(struct net_device *dev)
 {
 	struct macb *lp = netdev_priv(dev);
+	struct macb_queue *q = &lp->queues[0];
 	struct macb_dma_desc *desc;
 	unsigned char *p_recv;
 	struct sk_buff *skb;
 	unsigned int pktlen;
 
-	desc = macb_rx_desc(lp, lp->rx_tail);
+	desc = macb_rx_desc(q, q->rx_tail);
 	while (desc->addr & MACB_BIT(RX_USED)) {
-		p_recv = lp->rx_buffers + lp->rx_tail * AT91ETHER_MAX_RBUFF_SZ;
+		p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;
 		pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);
 		skb = netdev_alloc_skb(dev, pktlen + 2);
 		if (skb) {
@@ -3163,12 +3619,12 @@ static void at91ether_rx(struct net_device *dev)
 		desc->addr &= ~MACB_BIT(RX_USED);
 
 		/* wrap after last buffer */
-		if (lp->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)
-			lp->rx_tail = 0;
+		if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)
+			q->rx_tail = 0;
 		else
-			lp->rx_tail++;
+			q->rx_tail++;
 
-		desc = macb_rx_desc(lp, lp->rx_tail);
+		desc = macb_rx_desc(q, q->rx_tail);
 	}
 }
 
@@ -3394,7 +3850,6 @@ static int macb_probe(struct platform_device *pdev)
 					      = macb_config->clk_init;
 	int (*init)(struct platform_device *) = macb_config->init;
 	struct device_node *np = pdev->dev.of_node;
-	struct device_node *phy_node;
 	struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL;
 	unsigned int queue_mask, num_queues;
 	struct macb_platform_data *pdata;
@@ -3500,18 +3955,6 @@ static int macb_probe(struct platform_device *pdev)
 	else
 		macb_get_hwaddr(bp);
 
-	/* Power up the PHY if there is a GPIO reset */
-	phy_node =  of_get_next_available_child(np, NULL);
-	if (phy_node) {
-		int gpio = of_get_named_gpio(phy_node, "reset-gpios", 0);
-
-		if (gpio_is_valid(gpio)) {
-			bp->reset_gpio = gpio_to_desc(gpio);
-			gpiod_direction_output(bp->reset_gpio, 1);
-		}
-	}
-	of_node_put(phy_node);
-
 	err = of_get_phy_mode(np);
 	if (err < 0) {
 		pdata = dev_get_platdata(&pdev->dev);
@@ -3542,6 +3985,9 @@ static int macb_probe(struct platform_device *pdev)
 		goto err_out_unregister_mdio;
 	}
 
+	tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task,
+		     (unsigned long)bp);
+
 	phy_attached_info(phydev);
 
 	netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n",
@@ -3558,10 +4004,6 @@ static int macb_probe(struct platform_device *pdev)
 		of_phy_deregister_fixed_link(np);
 	mdiobus_free(bp->mii_bus);
 
-	/* Shutdown the PHY if there is a GPIO reset */
-	if (bp->reset_gpio)
-		gpiod_set_value(bp->reset_gpio, 0);
-
 err_out_free_netdev:
 	free_netdev(dev);
 
@@ -3592,10 +4034,6 @@ static int macb_remove(struct platform_device *pdev)
 		dev->phydev = NULL;
 		mdiobus_free(bp->mii_bus);
 
-		/* Shutdown the PHY if there is a GPIO reset */
-		if (bp->reset_gpio)
-			gpiod_set_value(bp->reset_gpio, 0);
-
 		unregister_netdev(dev);
 		clk_disable_unprepare(bp->tx_clk);
 		clk_disable_unprepare(bp->hclk);
diff --git a/drivers/net/ethernet/cavium/Kconfig b/drivers/net/ethernet/cavium/Kconfig
index 63be75eb34d2..043e3c11c42b 100644
--- a/drivers/net/ethernet/cavium/Kconfig
+++ b/drivers/net/ethernet/cavium/Kconfig
@@ -27,6 +27,7 @@ config THUNDER_NIC_PF
 
 config THUNDER_NIC_VF
 	tristate "Thunder Virtual function driver"
+	imply CAVIUM_PTP
 	depends on 64BIT
 	---help---
 	  This driver supports Thunder's NIC virtual function
@@ -50,6 +51,18 @@ config	THUNDER_NIC_RGX
 	  This driver supports configuring XCV block of RGX interface
 	  present on CN81XX chip.
 
+config CAVIUM_PTP
+	tristate "Cavium PTP coprocessor as PTP clock"
+	depends on 64BIT
+	imply PTP_1588_CLOCK
+	default y
+	---help---
+	  This driver adds support for the Precision Time Protocol Clocks and
+	  Timestamping coprocessor (PTP) found on Cavium processors.
+	  PTP provides timestamping mechanism that is suitable for use in IEEE 1588
+	  Precision Time Protocol or other purposes.  Timestamps can be used in
+	  BGX, TNS, GTI, and NIC blocks.
+
 config LIQUIDIO
 	tristate "Cavium LiquidIO support"
 	depends on 64BIT
diff --git a/drivers/net/ethernet/cavium/Makefile b/drivers/net/ethernet/cavium/Makefile
index 872da9f7c31a..946bba84e81d 100644
--- a/drivers/net/ethernet/cavium/Makefile
+++ b/drivers/net/ethernet/cavium/Makefile
@@ -1,6 +1,7 @@
 #
 # Makefile for the Cavium ethernet device drivers.
 #
+obj-$(CONFIG_NET_VENDOR_CAVIUM) += common/
 obj-$(CONFIG_NET_VENDOR_CAVIUM) += thunder/
 obj-$(CONFIG_NET_VENDOR_CAVIUM) += liquidio/
 obj-$(CONFIG_NET_VENDOR_CAVIUM) += octeon/
diff --git a/drivers/net/ethernet/cavium/common/Makefile b/drivers/net/ethernet/cavium/common/Makefile
new file mode 100644
index 000000000000..dd8561b8060b
--- /dev/null
+++ b/drivers/net/ethernet/cavium/common/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_CAVIUM_PTP) += cavium_ptp.o
diff --git a/drivers/net/ethernet/cavium/common/cavium_ptp.c b/drivers/net/ethernet/cavium/common/cavium_ptp.c
new file mode 100644
index 000000000000..d59497a7bdce
--- /dev/null
+++ b/drivers/net/ethernet/cavium/common/cavium_ptp.c
@@ -0,0 +1,355 @@
+// SPDX-License-Identifier: GPL-2.0
+/* cavium_ptp.c - PTP 1588 clock on Cavium hardware
+ * Copyright (c) 2003-2015, 2017 Cavium, Inc.
+ */
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/timecounter.h>
+#include <linux/pci.h>
+
+#include "cavium_ptp.h"
+
+#define DRV_NAME	"Cavium PTP Driver"
+
+#define PCI_DEVICE_ID_CAVIUM_PTP	0xA00C
+#define PCI_DEVICE_ID_CAVIUM_RST	0xA00E
+
+#define PCI_PTP_BAR_NO	0
+#define PCI_RST_BAR_NO	0
+
+#define PTP_CLOCK_CFG		0xF00ULL
+#define  PTP_CLOCK_CFG_PTP_EN	BIT(0)
+#define PTP_CLOCK_LO		0xF08ULL
+#define PTP_CLOCK_HI		0xF10ULL
+#define PTP_CLOCK_COMP		0xF18ULL
+
+#define RST_BOOT	0x1600ULL
+#define CLOCK_BASE_RATE	50000000ULL
+
+static u64 ptp_cavium_clock_get(void)
+{
+	struct pci_dev *pdev;
+	void __iomem *base;
+	u64 ret = CLOCK_BASE_RATE * 16;
+
+	pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
+			      PCI_DEVICE_ID_CAVIUM_RST, NULL);
+	if (!pdev)
+		goto error;
+
+	base = pci_ioremap_bar(pdev, PCI_RST_BAR_NO);
+	if (!base)
+		goto error_put_pdev;
+
+	ret = CLOCK_BASE_RATE * ((readq(base + RST_BOOT) >> 33) & 0x3f);
+
+	iounmap(base);
+
+error_put_pdev:
+	pci_dev_put(pdev);
+
+error:
+	return ret;
+}
+
+struct cavium_ptp *cavium_ptp_get(void)
+{
+	struct cavium_ptp *ptp;
+	struct pci_dev *pdev;
+
+	pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
+			      PCI_DEVICE_ID_CAVIUM_PTP, NULL);
+	if (!pdev)
+		return ERR_PTR(-ENODEV);
+
+	ptp = pci_get_drvdata(pdev);
+	if (!ptp)
+		ptp = ERR_PTR(-EPROBE_DEFER);
+	if (IS_ERR(ptp))
+		pci_dev_put(pdev);
+
+	return ptp;
+}
+EXPORT_SYMBOL(cavium_ptp_get);
+
+void cavium_ptp_put(struct cavium_ptp *ptp)
+{
+	if (!ptp)
+		return;
+	pci_dev_put(ptp->pdev);
+}
+EXPORT_SYMBOL(cavium_ptp_put);
+
+/**
+ * cavium_ptp_adjfine() - Adjust ptp frequency
+ * @ptp: PTP clock info
+ * @scaled_ppm: how much to adjust by, in parts per million, but with a
+ *              16 bit binary fractional field
+ */
+static int cavium_ptp_adjfine(struct ptp_clock_info *ptp_info, long scaled_ppm)
+{
+	struct cavium_ptp *clock =
+		container_of(ptp_info, struct cavium_ptp, ptp_info);
+	unsigned long flags;
+	u64 comp;
+	u64 adj;
+	bool neg_adj = false;
+
+	if (scaled_ppm < 0) {
+		neg_adj = true;
+		scaled_ppm = -scaled_ppm;
+	}
+
+	/* The hardware adds the clock compensation value to the PTP clock
+	 * on every coprocessor clock cycle. Typical convention is that it
+	 * represent number of nanosecond betwen each cycle. In this
+	 * convention compensation value is in 64 bit fixed-point
+	 * representation where upper 32 bits are number of nanoseconds
+	 * and lower is fractions of nanosecond.
+	 * The scaled_ppm represent the ratio in "parts per bilion" by which the
+	 * compensation value should be corrected.
+	 * To calculate new compenstation value we use 64bit fixed point
+	 * arithmetic on following formula
+	 * comp = tbase + tbase * scaled_ppm / (1M * 2^16)
+	 * where tbase is the basic compensation value calculated initialy
+	 * in cavium_ptp_init() -> tbase = 1/Hz. Then we use endian
+	 * independent structure definition to write data to PTP register.
+	 */
+	comp = ((u64)1000000000ull << 32) / clock->clock_rate;
+	adj = comp * scaled_ppm;
+	adj >>= 16;
+	adj = div_u64(adj, 1000000ull);
+	comp = neg_adj ? comp - adj : comp + adj;
+
+	spin_lock_irqsave(&clock->spin_lock, flags);
+	writeq(comp, clock->reg_base + PTP_CLOCK_COMP);
+	spin_unlock_irqrestore(&clock->spin_lock, flags);
+
+	return 0;
+}
+
+/**
+ * cavium_ptp_adjtime() - Adjust ptp time
+ * @ptp:   PTP clock info
+ * @delta: how much to adjust by, in nanosecs
+ */
+static int cavium_ptp_adjtime(struct ptp_clock_info *ptp_info, s64 delta)
+{
+	struct cavium_ptp *clock =
+		container_of(ptp_info, struct cavium_ptp, ptp_info);
+	unsigned long flags;
+
+	spin_lock_irqsave(&clock->spin_lock, flags);
+	timecounter_adjtime(&clock->time_counter, delta);
+	spin_unlock_irqrestore(&clock->spin_lock, flags);
+
+	/* Sync, for network driver to get latest value */
+	smp_mb();
+
+	return 0;
+}
+
+/**
+ * cavium_ptp_gettime() - Get hardware clock time with adjustment
+ * @ptp: PTP clock info
+ * @ts:  timespec
+ */
+static int cavium_ptp_gettime(struct ptp_clock_info *ptp_info,
+			      struct timespec64 *ts)
+{
+	struct cavium_ptp *clock =
+		container_of(ptp_info, struct cavium_ptp, ptp_info);
+	unsigned long flags;
+	u64 nsec;
+
+	spin_lock_irqsave(&clock->spin_lock, flags);
+	nsec = timecounter_read(&clock->time_counter);
+	spin_unlock_irqrestore(&clock->spin_lock, flags);
+
+	*ts = ns_to_timespec64(nsec);
+
+	return 0;
+}
+
+/**
+ * cavium_ptp_settime() - Set hardware clock time. Reset adjustment
+ * @ptp: PTP clock info
+ * @ts:  timespec
+ */
+static int cavium_ptp_settime(struct ptp_clock_info *ptp_info,
+			      const struct timespec64 *ts)
+{
+	struct cavium_ptp *clock =
+		container_of(ptp_info, struct cavium_ptp, ptp_info);
+	unsigned long flags;
+	u64 nsec;
+
+	nsec = timespec64_to_ns(ts);
+
+	spin_lock_irqsave(&clock->spin_lock, flags);
+	timecounter_init(&clock->time_counter, &clock->cycle_counter, nsec);
+	spin_unlock_irqrestore(&clock->spin_lock, flags);
+
+	return 0;
+}
+
+/**
+ * cavium_ptp_enable() - Request to enable or disable an ancillary feature.
+ * @ptp: PTP clock info
+ * @rq:  request
+ * @on:  is it on
+ */
+static int cavium_ptp_enable(struct ptp_clock_info *ptp_info,
+			     struct ptp_clock_request *rq, int on)
+{
+	return -EOPNOTSUPP;
+}
+
+static u64 cavium_ptp_cc_read(const struct cyclecounter *cc)
+{
+	struct cavium_ptp *clock =
+		container_of(cc, struct cavium_ptp, cycle_counter);
+
+	return readq(clock->reg_base + PTP_CLOCK_HI);
+}
+
+static int cavium_ptp_probe(struct pci_dev *pdev,
+			    const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	struct cavium_ptp *clock;
+	struct cyclecounter *cc;
+	u64 clock_cfg;
+	u64 clock_comp;
+	int err;
+
+	clock = devm_kzalloc(dev, sizeof(*clock), GFP_KERNEL);
+	if (!clock) {
+		err = -ENOMEM;
+		goto error;
+	}
+
+	clock->pdev = pdev;
+
+	err = pcim_enable_device(pdev);
+	if (err)
+		goto error_free;
+
+	err = pcim_iomap_regions(pdev, 1 << PCI_PTP_BAR_NO, pci_name(pdev));
+	if (err)
+		goto error_free;
+
+	clock->reg_base = pcim_iomap_table(pdev)[PCI_PTP_BAR_NO];
+
+	spin_lock_init(&clock->spin_lock);
+
+	cc = &clock->cycle_counter;
+	cc->read = cavium_ptp_cc_read;
+	cc->mask = CYCLECOUNTER_MASK(64);
+	cc->mult = 1;
+	cc->shift = 0;
+
+	timecounter_init(&clock->time_counter, &clock->cycle_counter,
+			 ktime_to_ns(ktime_get_real()));
+
+	clock->clock_rate = ptp_cavium_clock_get();
+
+	clock->ptp_info = (struct ptp_clock_info) {
+		.owner		= THIS_MODULE,
+		.name		= "ThunderX PTP",
+		.max_adj	= 1000000000ull,
+		.n_ext_ts	= 0,
+		.n_pins		= 0,
+		.pps		= 0,
+		.adjfine	= cavium_ptp_adjfine,
+		.adjtime	= cavium_ptp_adjtime,
+		.gettime64	= cavium_ptp_gettime,
+		.settime64	= cavium_ptp_settime,
+		.enable		= cavium_ptp_enable,
+	};
+
+	clock_cfg = readq(clock->reg_base + PTP_CLOCK_CFG);
+	clock_cfg |= PTP_CLOCK_CFG_PTP_EN;
+	writeq(clock_cfg, clock->reg_base + PTP_CLOCK_CFG);
+
+	clock_comp = ((u64)1000000000ull << 32) / clock->clock_rate;
+	writeq(clock_comp, clock->reg_base + PTP_CLOCK_COMP);
+
+	clock->ptp_clock = ptp_clock_register(&clock->ptp_info, dev);
+	if (!clock->ptp_clock) {
+		err = -ENODEV;
+		goto error_stop;
+	}
+	if (IS_ERR(clock->ptp_clock)) {
+		err = PTR_ERR(clock->ptp_clock);
+		goto error_stop;
+	}
+
+	pci_set_drvdata(pdev, clock);
+	return 0;
+
+error_stop:
+	clock_cfg = readq(clock->reg_base + PTP_CLOCK_CFG);
+	clock_cfg &= ~PTP_CLOCK_CFG_PTP_EN;
+	writeq(clock_cfg, clock->reg_base + PTP_CLOCK_CFG);
+	pcim_iounmap_regions(pdev, 1 << PCI_PTP_BAR_NO);
+
+error_free:
+	devm_kfree(dev, clock);
+
+error:
+	/* For `cavium_ptp_get()` we need to differentiate between the case
+	 * when the core has not tried to probe this device and the case when
+	 * the probe failed.  In the later case we pretend that the
+	 * initialization was successful and keep the error in
+	 * `dev->driver_data`.
+	 */
+	pci_set_drvdata(pdev, ERR_PTR(err));
+	return 0;
+}
+
+static void cavium_ptp_remove(struct pci_dev *pdev)
+{
+	struct cavium_ptp *clock = pci_get_drvdata(pdev);
+	u64 clock_cfg;
+
+	if (IS_ERR_OR_NULL(clock))
+		return;
+
+	ptp_clock_unregister(clock->ptp_clock);
+
+	clock_cfg = readq(clock->reg_base + PTP_CLOCK_CFG);
+	clock_cfg &= ~PTP_CLOCK_CFG_PTP_EN;
+	writeq(clock_cfg, clock->reg_base + PTP_CLOCK_CFG);
+}
+
+static const struct pci_device_id cavium_ptp_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_PTP) },
+	{ 0, }
+};
+
+static struct pci_driver cavium_ptp_driver = {
+	.name = DRV_NAME,
+	.id_table = cavium_ptp_id_table,
+	.probe = cavium_ptp_probe,
+	.remove = cavium_ptp_remove,
+};
+
+static int __init cavium_ptp_init_module(void)
+{
+	return pci_register_driver(&cavium_ptp_driver);
+}
+
+static void __exit cavium_ptp_cleanup_module(void)
+{
+	pci_unregister_driver(&cavium_ptp_driver);
+}
+
+module_init(cavium_ptp_init_module);
+module_exit(cavium_ptp_cleanup_module);
+
+MODULE_DESCRIPTION(DRV_NAME);
+MODULE_AUTHOR("Cavium Networks <support@cavium.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DEVICE_TABLE(pci, cavium_ptp_id_table);
diff --git a/drivers/net/ethernet/cavium/common/cavium_ptp.h b/drivers/net/ethernet/cavium/common/cavium_ptp.h
new file mode 100644
index 000000000000..be2bafc7beeb
--- /dev/null
+++ b/drivers/net/ethernet/cavium/common/cavium_ptp.h
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0
+/* cavium_ptp.h - PTP 1588 clock on Cavium hardware
+ * Copyright (c) 2003-2015, 2017 Cavium, Inc.
+ */
+
+#ifndef CAVIUM_PTP_H
+#define CAVIUM_PTP_H
+
+#include <linux/ptp_clock_kernel.h>
+#include <linux/timecounter.h>
+
+struct cavium_ptp {
+	struct pci_dev *pdev;
+
+	/* Serialize access to cycle_counter, time_counter and hw_registers */
+	spinlock_t spin_lock;
+	struct cyclecounter cycle_counter;
+	struct timecounter time_counter;
+	void __iomem *reg_base;
+
+	u32 clock_rate;
+
+	struct ptp_clock_info ptp_info;
+	struct ptp_clock *ptp_clock;
+};
+
+#if IS_ENABLED(CONFIG_CAVIUM_PTP)
+
+struct cavium_ptp *cavium_ptp_get(void);
+void cavium_ptp_put(struct cavium_ptp *ptp);
+
+static inline u64 cavium_ptp_tstamp2time(struct cavium_ptp *ptp, u64 tstamp)
+{
+	unsigned long flags;
+	u64 ret;
+
+	spin_lock_irqsave(&ptp->spin_lock, flags);
+	ret = timecounter_cyc2time(&ptp->time_counter, tstamp);
+	spin_unlock_irqrestore(&ptp->spin_lock, flags);
+
+	return ret;
+}
+
+static inline int cavium_ptp_clock_index(struct cavium_ptp *clock)
+{
+	return ptp_clock_index(clock->ptp_clock);
+}
+
+#else
+
+static inline struct cavium_ptp *cavium_ptp_get(void)
+{
+	return ERR_PTR(-ENODEV);
+}
+
+static inline void cavium_ptp_put(struct cavium_ptp *ptp) {}
+
+static inline u64 cavium_ptp_tstamp2time(struct cavium_ptp *ptp, u64 tstamp)
+{
+	return 0;
+}
+
+static inline int cavium_ptp_clock_index(struct cavium_ptp *clock)
+{
+	return -1;
+}
+
+#endif
+
+#endif
diff --git a/drivers/net/ethernet/cavium/liquidio/octeon_device.c b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
index 2c615ab09e64..f38abf626412 100644
--- a/drivers/net/ethernet/cavium/liquidio/octeon_device.c
+++ b/drivers/net/ethernet/cavium/liquidio/octeon_device.c
@@ -702,12 +702,10 @@ static struct octeon_device *octeon_allocate_device_mem(u32 pci_id,
 	size = octdevsize + priv_size + configsize +
 		(sizeof(struct octeon_dispatch) * DISPATCH_LIST_SIZE);
 
-	buf = vmalloc(size);
+	buf = vzalloc(size);
 	if (!buf)
 		return NULL;
 
-	memset(buf, 0, size);
-
 	oct = (struct octeon_device *)buf;
 	oct->priv = (void *)(buf + octdevsize);
 	oct->chip = (void *)(buf + octdevsize + priv_size);
@@ -840,10 +838,9 @@ octeon_allocate_ioq_vector(struct octeon_device  *oct)
 
 	size = sizeof(struct octeon_ioq_vector) * num_ioqs;
 
-	oct->ioq_vector = vmalloc(size);
+	oct->ioq_vector = vzalloc(size);
 	if (!oct->ioq_vector)
 		return 1;
-	memset(oct->ioq_vector, 0, size);
 	for (i = 0; i < num_ioqs; i++) {
 		ioq_vector		= &oct->ioq_vector[i];
 		ioq_vector->oct_dev	= oct;
diff --git a/drivers/net/ethernet/cavium/thunder/nic.h b/drivers/net/ethernet/cavium/thunder/nic.h
index 4a02e618e318..4cacce5d2b16 100644
--- a/drivers/net/ethernet/cavium/thunder/nic.h
+++ b/drivers/net/ethernet/cavium/thunder/nic.h
@@ -263,6 +263,8 @@ struct nicvf_drv_stats {
 	struct u64_stats_sync   syncp;
 };
 
+struct cavium_ptp;
+
 struct nicvf {
 	struct nicvf		*pnicvf;
 	struct net_device	*netdev;
@@ -312,6 +314,33 @@ struct nicvf {
 	struct tasklet_struct	qs_err_task;
 	struct work_struct	reset_task;
 
+	/* PTP timestamp */
+	struct cavium_ptp	*ptp_clock;
+	/* Inbound timestamping is on */
+	bool			hw_rx_tstamp;
+	/* When the packet that requires timestamping is sent, hardware inserts
+	 * two entries to the completion queue.  First is the regular
+	 * CQE_TYPE_SEND entry that signals that the packet was sent.
+	 * The second is CQE_TYPE_SEND_PTP that contains the actual timestamp
+	 * for that packet.
+	 * `ptp_skb` is initialized in the handler for the CQE_TYPE_SEND
+	 * entry and is used and zeroed in the handler for the CQE_TYPE_SEND_PTP
+	 * entry.
+	 * So `ptp_skb` is used to hold the pointer to the packet between
+	 * the calls to CQE_TYPE_SEND and CQE_TYPE_SEND_PTP handlers.
+	 */
+	struct sk_buff		*ptp_skb;
+	/* `tx_ptp_skbs` is set when the hardware is sending a packet that
+	 * requires timestamping.  Cavium hardware can not process more than one
+	 * such packet at once so this is set each time the driver submits
+	 * a packet that requires timestamping to the send queue and clears
+	 * each time it receives the entry on the completion queue saying
+	 * that such packet was sent.
+	 * So `tx_ptp_skbs` prevents driver from submitting more than one
+	 * packet that requires timestamping to the hardware for transmitting.
+	 */
+	atomic_t		tx_ptp_skbs;
+
 	/* Interrupt coalescing settings */
 	u32			cq_coalesce_usecs;
 	u32			msg_enable;
@@ -371,6 +400,7 @@ struct nicvf {
 #define	NIC_MBOX_MSG_LOOPBACK		0x16	/* Set interface in loopback */
 #define	NIC_MBOX_MSG_RESET_STAT_COUNTER 0x17	/* Reset statistics counters */
 #define	NIC_MBOX_MSG_PFC		0x18	/* Pause frame control */
+#define	NIC_MBOX_MSG_PTP_CFG		0x19	/* HW packet timestamp */
 #define	NIC_MBOX_MSG_CFG_DONE		0xF0	/* VF configuration done */
 #define	NIC_MBOX_MSG_SHUTDOWN		0xF1	/* VF is being shutdown */
 
@@ -521,6 +551,11 @@ struct pfc {
 	u8    fc_tx;
 };
 
+struct set_ptp {
+	u8    msg;
+	bool  enable;
+};
+
 /* 128 bit shared memory between PF and each VF */
 union nic_mbx {
 	struct { u8 msg; }	msg;
@@ -540,6 +575,7 @@ union nic_mbx {
 	struct set_loopback	lbk;
 	struct reset_stat_cfg	reset_stat;
 	struct pfc		pfc;
+	struct set_ptp		ptp;
 };
 
 #define NIC_NODE_ID_MASK	0x03
diff --git a/drivers/net/ethernet/cavium/thunder/nic_main.c b/drivers/net/ethernet/cavium/thunder/nic_main.c
index 8f1dd55b3e08..7ff66a8194e2 100644
--- a/drivers/net/ethernet/cavium/thunder/nic_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nic_main.c
@@ -18,7 +18,7 @@
 #include "q_struct.h"
 #include "thunder_bgx.h"
 
-#define DRV_NAME	"thunder-nic"
+#define DRV_NAME	"nicpf"
 #define DRV_VERSION	"1.0"
 
 struct hw_info {
@@ -426,13 +426,22 @@ static void nic_init_hw(struct nicpf *nic)
 	/* Enable backpressure */
 	nic_reg_write(nic, NIC_PF_BP_CFG, (1ULL << 6) | 0x03);
 
-	/* TNS and TNS bypass modes are present only on 88xx */
+	/* TNS and TNS bypass modes are present only on 88xx
+	 * Also offset of this CSR has changed in 81xx and 83xx.
+	 */
 	if (nic->pdev->subsystem_device == PCI_SUBSYS_DEVID_88XX_NIC_PF) {
 		/* Disable TNS mode on both interfaces */
 		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG,
-			      (NIC_TNS_BYPASS_MODE << 7) | BGX0_BLOCK);
+			      (NIC_TNS_BYPASS_MODE << 7) |
+			      BGX0_BLOCK | (1ULL << 16));
 		nic_reg_write(nic, NIC_PF_INTF_0_1_SEND_CFG | (1 << 8),
-			      (NIC_TNS_BYPASS_MODE << 7) | BGX1_BLOCK);
+			      (NIC_TNS_BYPASS_MODE << 7) |
+			      BGX1_BLOCK | (1ULL << 16));
+	} else {
+		/* Configure timestamp generation timeout to 10us */
+		for (i = 0; i < nic->hw->bgx_cnt; i++)
+			nic_reg_write(nic, NIC_PF_INTFX_SEND_CFG | (i << 3),
+				      (1ULL << 16));
 	}
 
 	nic_reg_write(nic, NIC_PF_INTF_0_1_BP_CFG,
@@ -880,6 +889,44 @@ static void nic_pause_frame(struct nicpf *nic, int vf, struct pfc *cfg)
 	}
 }
 
+/* Enable or disable HW timestamping by BGX for pkts received on a LMAC */
+static void nic_config_timestamp(struct nicpf *nic, int vf, struct set_ptp *ptp)
+{
+	struct pkind_cfg *pkind;
+	u8 lmac, bgx_idx;
+	u64 pkind_val, pkind_idx;
+
+	if (vf >= nic->num_vf_en)
+		return;
+
+	bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
+	lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]);
+
+	pkind_idx = lmac + bgx_idx * MAX_LMAC_PER_BGX;
+	pkind_val = nic_reg_read(nic, NIC_PF_PKIND_0_15_CFG | (pkind_idx << 3));
+	pkind = (struct pkind_cfg *)&pkind_val;
+
+	if (ptp->enable && !pkind->hdr_sl) {
+		/* Skiplen to exclude 8byte timestamp while parsing pkt
+		 * If not configured, will result in L2 errors.
+		 */
+		pkind->hdr_sl = 4;
+		/* Adjust max packet length allowed */
+		pkind->maxlen += (pkind->hdr_sl * 2);
+		bgx_config_timestamping(nic->node, bgx_idx, lmac, true);
+		nic_reg_write(nic, NIC_PF_RX_ETYPE_0_7 | (1 << 3),
+			      (ETYPE_ALG_ENDPARSE << 16) | ETH_P_1588);
+	} else if (!ptp->enable && pkind->hdr_sl) {
+		pkind->maxlen -= (pkind->hdr_sl * 2);
+		pkind->hdr_sl = 0;
+		bgx_config_timestamping(nic->node, bgx_idx, lmac, false);
+		nic_reg_write(nic, NIC_PF_RX_ETYPE_0_7 | (1 << 3),
+			      (ETYPE_ALG_SKIP << 16) | ETH_P_8021Q);
+	}
+
+	nic_reg_write(nic, NIC_PF_PKIND_0_15_CFG | (pkind_idx << 3), pkind_val);
+}
+
 /* Interrupt handler to handle mailbox messages from VFs */
 static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 {
@@ -1022,6 +1069,9 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf)
 	case NIC_MBOX_MSG_PFC:
 		nic_pause_frame(nic, vf, &mbx.pfc);
 		goto unlock;
+	case NIC_MBOX_MSG_PTP_CFG:
+		nic_config_timestamp(nic, vf, &mbx.ptp);
+		break;
 	default:
 		dev_err(&nic->pdev->dev,
 			"Invalid msg from VF%d, msg 0x%x\n", vf, mbx.msg.msg);
diff --git a/drivers/net/ethernet/cavium/thunder/nic_reg.h b/drivers/net/ethernet/cavium/thunder/nic_reg.h
index 80d46337cf29..a16c48a1ebb2 100644
--- a/drivers/net/ethernet/cavium/thunder/nic_reg.h
+++ b/drivers/net/ethernet/cavium/thunder/nic_reg.h
@@ -99,6 +99,7 @@
 #define   NIC_PF_ECC3_DBE_INT_W1S		(0x2708)
 #define   NIC_PF_ECC3_DBE_ENA_W1C		(0x2710)
 #define   NIC_PF_ECC3_DBE_ENA_W1S		(0x2718)
+#define   NIC_PF_INTFX_SEND_CFG			(0x4000)
 #define   NIC_PF_MCAM_0_191_ENA			(0x100000)
 #define   NIC_PF_MCAM_0_191_M_0_5_DATA		(0x110000)
 #define   NIC_PF_MCAM_CTRL			(0x120000)
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
index b9ece9cbf98b..5603f5ab1fee 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
@@ -9,14 +9,16 @@
 /* ETHTOOL Support for VNIC_VF Device*/
 
 #include <linux/pci.h>
+#include <linux/net_tstamp.h>
 
 #include "nic_reg.h"
 #include "nic.h"
 #include "nicvf_queues.h"
 #include "q_struct.h"
 #include "thunder_bgx.h"
+#include "../common/cavium_ptp.h"
 
-#define DRV_NAME	"thunder-nicvf"
+#define DRV_NAME	"nicvf"
 #define DRV_VERSION     "1.0"
 
 struct nicvf_stat {
@@ -824,6 +826,31 @@ static int nicvf_set_pauseparam(struct net_device *dev,
 	return 0;
 }
 
+static int nicvf_get_ts_info(struct net_device *netdev,
+			     struct ethtool_ts_info *info)
+{
+	struct nicvf *nic = netdev_priv(netdev);
+
+	if (!nic->ptp_clock)
+		return ethtool_op_get_ts_info(netdev, info);
+
+	info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
+				SOF_TIMESTAMPING_RX_SOFTWARE |
+				SOF_TIMESTAMPING_SOFTWARE |
+				SOF_TIMESTAMPING_TX_HARDWARE |
+				SOF_TIMESTAMPING_RX_HARDWARE |
+				SOF_TIMESTAMPING_RAW_HARDWARE;
+
+	info->phc_index = cavium_ptp_clock_index(nic->ptp_clock);
+
+	info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
+
+	info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
+			   (1 << HWTSTAMP_FILTER_ALL);
+
+	return 0;
+}
+
 static const struct ethtool_ops nicvf_ethtool_ops = {
 	.get_link		= nicvf_get_link,
 	.get_drvinfo		= nicvf_get_drvinfo,
@@ -847,7 +874,7 @@ static const struct ethtool_ops nicvf_ethtool_ops = {
 	.set_channels		= nicvf_set_channels,
 	.get_pauseparam         = nicvf_get_pauseparam,
 	.set_pauseparam         = nicvf_set_pauseparam,
-	.get_ts_info		= ethtool_op_get_ts_info,
+	.get_ts_info		= nicvf_get_ts_info,
 	.get_link_ksettings	= nicvf_get_link_ksettings,
 };
 
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
index a063c36c4c58..7d9c5ffbd041 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
@@ -20,13 +20,15 @@
 #include <linux/bpf.h>
 #include <linux/bpf_trace.h>
 #include <linux/filter.h>
+#include <linux/net_tstamp.h>
 
 #include "nic_reg.h"
 #include "nic.h"
 #include "nicvf_queues.h"
 #include "thunder_bgx.h"
+#include "../common/cavium_ptp.h"
 
-#define DRV_NAME	"thunder-nicvf"
+#define DRV_NAME	"nicvf"
 #define DRV_VERSION	"1.0"
 
 /* Supported devices */
@@ -502,7 +504,7 @@ static int nicvf_init_resources(struct nicvf *nic)
 
 static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog,
 				struct cqe_rx_t *cqe_rx, struct snd_queue *sq,
-				struct sk_buff **skb)
+				struct rcv_queue *rq, struct sk_buff **skb)
 {
 	struct xdp_buff xdp;
 	struct page *page;
@@ -525,6 +527,7 @@ static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog,
 	xdp.data = (void *)cpu_addr;
 	xdp_set_data_meta_invalid(&xdp);
 	xdp.data_end = xdp.data + len;
+	xdp.rxq = &rq->xdp_rxq;
 	orig_data = xdp.data;
 
 	rcu_read_lock();
@@ -589,6 +592,44 @@ static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog,
 	return false;
 }
 
+static void nicvf_snd_ptp_handler(struct net_device *netdev,
+				  struct cqe_send_t *cqe_tx)
+{
+	struct nicvf *nic = netdev_priv(netdev);
+	struct skb_shared_hwtstamps ts;
+	u64 ns;
+
+	nic = nic->pnicvf;
+
+	/* Sync for 'ptp_skb' */
+	smp_rmb();
+
+	/* New timestamp request can be queued now */
+	atomic_set(&nic->tx_ptp_skbs, 0);
+
+	/* Check for timestamp requested skb */
+	if (!nic->ptp_skb)
+		return;
+
+	/* Check if timestamping is timedout, which is set to 10us */
+	if (cqe_tx->send_status == CQ_TX_ERROP_TSTMP_TIMEOUT ||
+	    cqe_tx->send_status == CQ_TX_ERROP_TSTMP_CONFLICT)
+		goto no_tstamp;
+
+	/* Get the timestamp */
+	memset(&ts, 0, sizeof(ts));
+	ns = cavium_ptp_tstamp2time(nic->ptp_clock, cqe_tx->ptp_timestamp);
+	ts.hwtstamp = ns_to_ktime(ns);
+	skb_tstamp_tx(nic->ptp_skb, &ts);
+
+no_tstamp:
+	/* Free the original skb */
+	dev_kfree_skb_any(nic->ptp_skb);
+	nic->ptp_skb = NULL;
+	/* Sync 'ptp_skb' */
+	smp_wmb();
+}
+
 static void nicvf_snd_pkt_handler(struct net_device *netdev,
 				  struct cqe_send_t *cqe_tx,
 				  int budget, int *subdesc_cnt,
@@ -645,7 +686,12 @@ static void nicvf_snd_pkt_handler(struct net_device *netdev,
 		prefetch(skb);
 		(*tx_pkts)++;
 		*tx_bytes += skb->len;
-		napi_consume_skb(skb, budget);
+		/* If timestamp is requested for this skb, don't free it */
+		if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS &&
+		    !nic->pnicvf->ptp_skb)
+			nic->pnicvf->ptp_skb = skb;
+		else
+			napi_consume_skb(skb, budget);
 		sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
 	} else {
 		/* In case of SW TSO on 88xx, only last segment will have
@@ -684,9 +730,25 @@ static inline void nicvf_set_rxhash(struct net_device *netdev,
 	skb_set_hash(skb, hash, hash_type);
 }
 
+static inline void nicvf_set_rxtstamp(struct nicvf *nic, struct sk_buff *skb)
+{
+	u64 ns;
+
+	if (!nic->ptp_clock || !nic->hw_rx_tstamp)
+		return;
+
+	/* The first 8 bytes is the timestamp */
+	ns = cavium_ptp_tstamp2time(nic->ptp_clock,
+				    be64_to_cpu(*(__be64 *)skb->data));
+	skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
+
+	__skb_pull(skb, 8);
+}
+
 static void nicvf_rcv_pkt_handler(struct net_device *netdev,
 				  struct napi_struct *napi,
-				  struct cqe_rx_t *cqe_rx, struct snd_queue *sq)
+				  struct cqe_rx_t *cqe_rx,
+				  struct snd_queue *sq, struct rcv_queue *rq)
 {
 	struct sk_buff *skb = NULL;
 	struct nicvf *nic = netdev_priv(netdev);
@@ -712,7 +774,7 @@ static void nicvf_rcv_pkt_handler(struct net_device *netdev,
 	/* For XDP, ignore pkts spanning multiple pages */
 	if (nic->xdp_prog && (cqe_rx->rb_cnt == 1)) {
 		/* Packet consumed by XDP */
-		if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, &skb))
+		if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, rq, &skb))
 			return;
 	} else {
 		skb = nicvf_get_rcv_skb(snic, cqe_rx,
@@ -734,6 +796,7 @@ static void nicvf_rcv_pkt_handler(struct net_device *netdev,
 		return;
 	}
 
+	nicvf_set_rxtstamp(nic, skb);
 	nicvf_set_rxhash(netdev, cqe_rx, skb);
 
 	skb_record_rx_queue(skb, rq_idx);
@@ -769,6 +832,7 @@ static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
 	struct cqe_rx_t *cq_desc;
 	struct netdev_queue *txq;
 	struct snd_queue *sq = &qs->sq[cq_idx];
+	struct rcv_queue *rq = &qs->rq[cq_idx];
 	unsigned int tx_pkts = 0, tx_bytes = 0, txq_idx;
 
 	spin_lock_bh(&cq->lock);
@@ -799,7 +863,7 @@ static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
 
 		switch (cq_desc->cqe_type) {
 		case CQE_TYPE_RX:
-			nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq);
+			nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq, rq);
 			work_done++;
 		break;
 		case CQE_TYPE_SEND:
@@ -808,10 +872,12 @@ static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
 					      &tx_pkts, &tx_bytes);
 			tx_done++;
 		break;
+		case CQE_TYPE_SEND_PTP:
+			nicvf_snd_ptp_handler(netdev, (void *)cq_desc);
+		break;
 		case CQE_TYPE_INVALID:
 		case CQE_TYPE_RX_SPLIT:
 		case CQE_TYPE_RX_TCP:
-		case CQE_TYPE_SEND_PTP:
 			/* Ignore for now */
 		break;
 		}
@@ -1307,12 +1373,28 @@ int nicvf_stop(struct net_device *netdev)
 
 	nicvf_free_cq_poll(nic);
 
+	/* Free any pending SKB saved to receive timestamp */
+	if (nic->ptp_skb) {
+		dev_kfree_skb_any(nic->ptp_skb);
+		nic->ptp_skb = NULL;
+	}
+
 	/* Clear multiqset info */
 	nic->pnicvf = nic;
 
 	return 0;
 }
 
+static int nicvf_config_hw_rx_tstamp(struct nicvf *nic, bool enable)
+{
+	union nic_mbx mbx = {};
+
+	mbx.ptp.msg = NIC_MBOX_MSG_PTP_CFG;
+	mbx.ptp.enable = enable;
+
+	return nicvf_send_msg_to_pf(nic, &mbx);
+}
+
 static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
 {
 	union nic_mbx mbx = {};
@@ -1382,6 +1464,12 @@ int nicvf_open(struct net_device *netdev)
 	if (nic->sqs_mode)
 		nicvf_get_primary_vf_struct(nic);
 
+	/* Configure PTP timestamp */
+	if (nic->ptp_clock)
+		nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp);
+	atomic_set(&nic->tx_ptp_skbs, 0);
+	nic->ptp_skb = NULL;
+
 	/* Configure receive side scaling and MTU */
 	if (!nic->sqs_mode) {
 		nicvf_rss_init(nic);
@@ -1764,6 +1852,73 @@ static int nicvf_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
 	}
 }
 
+static int nicvf_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr)
+{
+	struct hwtstamp_config config;
+	struct nicvf *nic = netdev_priv(netdev);
+
+	if (!nic->ptp_clock)
+		return -ENODEV;
+
+	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
+		return -EFAULT;
+
+	/* reserved for future extensions */
+	if (config.flags)
+		return -EINVAL;
+
+	switch (config.tx_type) {
+	case HWTSTAMP_TX_OFF:
+	case HWTSTAMP_TX_ON:
+		break;
+	default:
+		return -ERANGE;
+	}
+
+	switch (config.rx_filter) {
+	case HWTSTAMP_FILTER_NONE:
+		nic->hw_rx_tstamp = false;
+		break;
+	case HWTSTAMP_FILTER_ALL:
+	case HWTSTAMP_FILTER_SOME:
+	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
+	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
+	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
+	case HWTSTAMP_FILTER_PTP_V2_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
+		nic->hw_rx_tstamp = true;
+		config.rx_filter = HWTSTAMP_FILTER_ALL;
+		break;
+	default:
+		return -ERANGE;
+	}
+
+	if (netif_running(netdev))
+		nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp);
+
+	if (copy_to_user(ifr->ifr_data, &config, sizeof(config)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int nicvf_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
+{
+	switch (cmd) {
+	case SIOCSHWTSTAMP:
+		return nicvf_config_hwtstamp(netdev, req);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static const struct net_device_ops nicvf_netdev_ops = {
 	.ndo_open		= nicvf_open,
 	.ndo_stop		= nicvf_stop,
@@ -1775,6 +1930,7 @@ static const struct net_device_ops nicvf_netdev_ops = {
 	.ndo_fix_features       = nicvf_fix_features,
 	.ndo_set_features       = nicvf_set_features,
 	.ndo_bpf		= nicvf_xdp,
+	.ndo_do_ioctl           = nicvf_ioctl,
 };
 
 static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
@@ -1784,6 +1940,16 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct nicvf *nic;
 	int    err, qcount;
 	u16    sdevid;
+	struct cavium_ptp *ptp_clock;
+
+	ptp_clock = cavium_ptp_get();
+	if (IS_ERR(ptp_clock)) {
+		if (PTR_ERR(ptp_clock) == -ENODEV)
+			/* In virtualized environment we proceed without ptp */
+			ptp_clock = NULL;
+		else
+			return PTR_ERR(ptp_clock);
+	}
 
 	err = pci_enable_device(pdev);
 	if (err) {
@@ -1833,6 +1999,12 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	nic->pdev = pdev;
 	nic->pnicvf = nic;
 	nic->max_queues = qcount;
+	/* If no of CPUs are too low, there won't be any queues left
+	 * for XDP_TX, hence double it.
+	 */
+	if (!nic->t88)
+		nic->max_queues *= 2;
+	nic->ptp_clock = ptp_clock;
 
 	/* MAP VF's configuration registers */
 	nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
@@ -1946,6 +2118,7 @@ static void nicvf_remove(struct pci_dev *pdev)
 	pci_set_drvdata(pdev, NULL);
 	if (nic->drv_stats)
 		free_percpu(nic->drv_stats);
+	cavium_ptp_put(nic->ptp_clock);
 	free_netdev(netdev);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
index a3d12dbde95b..d42704d07484 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c
@@ -759,6 +759,7 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
 
 	if (!rq->enable) {
 		nicvf_reclaim_rcv_queue(nic, qs, qidx);
+		xdp_rxq_info_unreg(&rq->xdp_rxq);
 		return;
 	}
 
@@ -771,6 +772,9 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs,
 	/* all writes of RBDR data to be loaded into L2 Cache as well*/
 	rq->caching = 1;
 
+	/* Driver have no proper error path for failed XDP RX-queue info reg */
+	WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx) < 0);
+
 	/* Send a mailbox msg to PF to config RQ */
 	mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG;
 	mbx.rq.qs_num = qs->vnic_id;
@@ -977,6 +981,9 @@ void nicvf_qset_config(struct nicvf *nic, bool enable)
 		qs_cfg->be = 1;
 #endif
 		qs_cfg->vnic = qs->vnic_id;
+		/* Enable Tx timestamping capability */
+		if (nic->ptp_clock)
+			qs_cfg->send_tstmp_ena = 1;
 	}
 	nicvf_send_msg_to_pf(nic, &mbx);
 }
@@ -1384,6 +1391,29 @@ nicvf_sq_add_hdr_subdesc(struct nicvf *nic, struct snd_queue *sq, int qentry,
 		hdr->inner_l3_offset = skb_network_offset(skb) - 2;
 		this_cpu_inc(nic->pnicvf->drv_stats->tx_tso);
 	}
+
+	/* Check if timestamp is requested */
+	if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
+		skb_tx_timestamp(skb);
+		return;
+	}
+
+	/* Tx timestamping not supported along with TSO, so ignore request */
+	if (skb_shinfo(skb)->gso_size)
+		return;
+
+	/* HW supports only a single outstanding packet to timestamp */
+	if (!atomic_add_unless(&nic->pnicvf->tx_ptp_skbs, 1, 1))
+		return;
+
+	/* Mark the SKB for later reference */
+	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+
+	/* Finally enable timestamp generation
+	 * Since 'post_cqe' is also set, two CQEs will be posted
+	 * for this packet i.e CQE_TYPE_SEND and CQE_TYPE_SEND_PTP.
+	 */
+	hdr->tstmp = 1;
 }
 
 /* SQ GATHER subdescriptor
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
index 67d1a3230773..5e9a03cf1b4d 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.h
@@ -11,6 +11,7 @@
 
 #include <linux/netdevice.h>
 #include <linux/iommu.h>
+#include <net/xdp.h>
 #include "q_struct.h"
 
 #define MAX_QUEUE_SET			128
@@ -208,7 +209,7 @@ struct rx_tx_queue_stats {
 struct q_desc_mem {
 	dma_addr_t	dma;
 	u64		size;
-	u16		q_len;
+	u32		q_len;
 	dma_addr_t	phys_base;
 	void		*base;
 	void		*unalign_base;
@@ -251,6 +252,7 @@ struct rcv_queue {
 	u8		start_qs_rbdr_idx; /* RBDR idx in the above QS */
 	u8		caching;
 	struct		rx_tx_queue_stats stats;
+	struct xdp_rxq_info xdp_rxq;
 } ____cacheline_aligned_in_smp;
 
 struct cmp_queue {
diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
index 5e5c4d7796b8..91d34ea40e2c 100644
--- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
+++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.c
@@ -21,7 +21,7 @@
 #include "nic.h"
 #include "thunder_bgx.h"
 
-#define DRV_NAME	"thunder-BGX"
+#define DRV_NAME	"thunder_bgx"
 #define DRV_VERSION	"1.0"
 
 struct lmac {
@@ -245,6 +245,35 @@ void bgx_lmac_rx_tx_enable(int node, int bgx_idx, int lmacid, bool enable)
 }
 EXPORT_SYMBOL(bgx_lmac_rx_tx_enable);
 
+/* Enables or disables timestamp insertion by BGX for Rx packets */
+void bgx_config_timestamping(int node, int bgx_idx, int lmacid, bool enable)
+{
+	struct bgx *bgx = get_bgx(node, bgx_idx);
+	struct lmac *lmac;
+	u64 csr_offset, cfg;
+
+	if (!bgx)
+		return;
+
+	lmac = &bgx->lmac[lmacid];
+
+	if (lmac->lmac_type == BGX_MODE_SGMII ||
+	    lmac->lmac_type == BGX_MODE_QSGMII ||
+	    lmac->lmac_type == BGX_MODE_RGMII)
+		csr_offset = BGX_GMP_GMI_RXX_FRM_CTL;
+	else
+		csr_offset = BGX_SMUX_RX_FRM_CTL;
+
+	cfg = bgx_reg_read(bgx, lmacid, csr_offset);
+
+	if (enable)
+		cfg |= BGX_PKT_RX_PTP_EN;
+	else
+		cfg &= ~BGX_PKT_RX_PTP_EN;
+	bgx_reg_write(bgx, lmacid, csr_offset, cfg);
+}
+EXPORT_SYMBOL(bgx_config_timestamping);
+
 void bgx_lmac_get_pfc(int node, int bgx_idx, int lmacid, void *pause)
 {
 	struct pfc *pfc = (struct pfc *)pause;
diff --git a/drivers/net/ethernet/cavium/thunder/thunder_bgx.h b/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
index 23acdc5ab896..5a7567d31138 100644
--- a/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
+++ b/drivers/net/ethernet/cavium/thunder/thunder_bgx.h
@@ -122,6 +122,8 @@
 #define  SPU_DBG_CTL_AN_NONCE_MCT_DIS		BIT_ULL(29)
 
 #define BGX_SMUX_RX_INT			0x20000
+#define BGX_SMUX_RX_FRM_CTL		0x20020
+#define  BGX_PKT_RX_PTP_EN			BIT_ULL(12)
 #define BGX_SMUX_RX_JABBER		0x20030
 #define BGX_SMUX_RX_CTL			0x20048
 #define  SMU_RX_CTL_STATUS			(3ull << 0)
@@ -172,6 +174,7 @@
 #define  GMI_PORT_CFG_SPEED_MSB			BIT_ULL(8)
 #define  GMI_PORT_CFG_RX_IDLE			BIT_ULL(12)
 #define  GMI_PORT_CFG_TX_IDLE			BIT_ULL(13)
+#define BGX_GMP_GMI_RXX_FRM_CTL		0x38028
 #define BGX_GMP_GMI_RXX_JABBER		0x38038
 #define BGX_GMP_GMI_TXX_THRESH		0x38210
 #define BGX_GMP_GMI_TXX_APPEND		0x38218
@@ -223,6 +226,7 @@ void bgx_set_lmac_mac(int node, int bgx_idx, int lmacid, const u8 *mac);
 void bgx_get_lmac_link_state(int node, int bgx_idx, int lmacid, void *status);
 void bgx_lmac_internal_loopback(int node, int bgx_idx,
 				int lmac_idx, bool enable);
+void bgx_config_timestamping(int node, int bgx_idx, int lmacid, bool enable);
 void bgx_lmac_get_pfc(int node, int bgx_idx, int lmacid, void *pause);
 void bgx_lmac_set_pfc(int node, int bgx_idx, int lmacid, void *pause);
 
diff --git a/drivers/net/ethernet/cavium/thunder/thunder_xcv.c b/drivers/net/ethernet/cavium/thunder/thunder_xcv.c
index 578c7f8f11bf..2d5e8dab1f70 100644
--- a/drivers/net/ethernet/cavium/thunder/thunder_xcv.c
+++ b/drivers/net/ethernet/cavium/thunder/thunder_xcv.c
@@ -20,7 +20,7 @@
 #include "nic.h"
 #include "thunder_bgx.h"
 
-#define DRV_NAME	"thunder-xcv"
+#define DRV_NAME	"thunder_xcv"
 #define DRV_VERSION	"1.0"
 
 /* Register offsets */
diff --git a/drivers/net/ethernet/chelsio/Kconfig b/drivers/net/ethernet/chelsio/Kconfig
index 5713e83be08c..e2cdfa75673f 100644
--- a/drivers/net/ethernet/chelsio/Kconfig
+++ b/drivers/net/ethernet/chelsio/Kconfig
@@ -69,6 +69,7 @@ config CHELSIO_T4
 	depends on PCI && (IPV6 || IPV6=n)
 	select FW_LOADER
 	select MDIO
+	select ZLIB_DEFLATE
 	---help---
 	  This driver supports Chelsio T4, T5 & T6 based gigabit, 10Gb Ethernet
 	  adapter and T5/T6 based 40Gb and T6 based 25Gb, 50Gb and 100Gb
diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
index 6a015362c340..185fe8df7628 100644
--- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
@@ -3304,6 +3304,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		netdev->ethtool_ops = &cxgb_ethtool_ops;
 		netdev->min_mtu = 81;
 		netdev->max_mtu = ETH_MAX_MTU;
+		netdev->dev_port = pi->port_id;
 	}
 
 	pci_set_drvdata(pdev, adapter);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/Makefile b/drivers/net/ethernet/chelsio/cxgb4/Makefile
index 8c9c6b0d2e5d..53b6a02c778e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/Makefile
+++ b/drivers/net/ethernet/chelsio/cxgb4/Makefile
@@ -8,7 +8,7 @@ obj-$(CONFIG_CHELSIO_T4) += cxgb4.o
 cxgb4-objs := cxgb4_main.o l2t.o smt.o t4_hw.o sge.o clip_tbl.o cxgb4_ethtool.o \
 	      cxgb4_uld.o sched.o cxgb4_filter.o cxgb4_tc_u32.o \
 	      cxgb4_ptp.o cxgb4_tc_flower.o cxgb4_cudbg.o \
-	      cudbg_common.o cudbg_lib.o
+	      cudbg_common.o cudbg_lib.o cudbg_zlib.o
 cxgb4-$(CONFIG_CHELSIO_T4_DCB) +=  cxgb4_dcb.o
 cxgb4-$(CONFIG_CHELSIO_T4_FCOE) +=  cxgb4_fcoe.o
 cxgb4-$(CONFIG_DEBUG_FS) += cxgb4_debugfs.o
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c
index f78ba1743b5a..8edc49827af0 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_common.c
@@ -19,7 +19,8 @@
 #include "cudbg_if.h"
 #include "cudbg_lib_common.h"
 
-int cudbg_get_buff(struct cudbg_buffer *pdbg_buff, u32 size,
+int cudbg_get_buff(struct cudbg_init *pdbg_init,
+		   struct cudbg_buffer *pdbg_buff, u32 size,
 		   struct cudbg_buffer *pin_buff)
 {
 	u32 offset;
@@ -28,17 +29,30 @@ int cudbg_get_buff(struct cudbg_buffer *pdbg_buff, u32 size,
 	if (offset + size > pdbg_buff->size)
 		return CUDBG_STATUS_NO_MEM;
 
+	if (pdbg_init->compress_type != CUDBG_COMPRESSION_NONE) {
+		if (size > pdbg_init->compress_buff_size)
+			return CUDBG_STATUS_NO_MEM;
+
+		pin_buff->data = (char *)pdbg_init->compress_buff;
+		pin_buff->offset = 0;
+		pin_buff->size = size;
+		return 0;
+	}
+
 	pin_buff->data = (char *)pdbg_buff->data + offset;
 	pin_buff->offset = offset;
 	pin_buff->size = size;
-	pdbg_buff->size -= size;
 	return 0;
 }
 
-void cudbg_put_buff(struct cudbg_buffer *pin_buff,
-		    struct cudbg_buffer *pdbg_buff)
+void cudbg_put_buff(struct cudbg_init *pdbg_init,
+		    struct cudbg_buffer *pin_buff)
 {
-	pdbg_buff->size += pin_buff->size;
+	/* Clear compression buffer for re-use */
+	if (pdbg_init->compress_type != CUDBG_COMPRESSION_NONE)
+		memset(pdbg_init->compress_buff, 0,
+		       pdbg_init->compress_buff_size);
+
 	pin_buff->data = NULL;
 	pin_buff->offset = 0;
 	pin_buff->size = 0;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h
index 605689957496..b57acb8dc35b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_entity.h
@@ -18,17 +18,15 @@
 #ifndef __CUDBG_ENTITY_H__
 #define __CUDBG_ENTITY_H__
 
-#define EDC0_FLAG 3
-#define EDC1_FLAG 4
+#define EDC0_FLAG 0
+#define EDC1_FLAG 1
+#define MC_FLAG 2
+#define MC0_FLAG 3
+#define MC1_FLAG 4
+#define HMA_FLAG 5
 
 #define CUDBG_ENTITY_SIGNATURE 0xCCEDB001
 
-struct card_mem {
-	u16 size_edc0;
-	u16 size_edc1;
-	u16 mem_flag;
-};
-
 struct cudbg_mbox_log {
 	struct mbox_cmd entry;
 	u32 hi[MBOX_LEN / 8];
@@ -87,6 +85,48 @@ struct cudbg_tp_la {
 	u8 data[0];
 };
 
+static const char * const cudbg_region[] = {
+	"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
+	"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
+	"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
+	"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
+	"RQUDP region:", "PBL region:", "TXPBL region:",
+	"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
+	"On-chip queues:"
+};
+
+/* Memory region info relative to current memory (i.e. wrt 0). */
+struct cudbg_region_info {
+	bool exist; /* Does region exists in current memory? */
+	u32 start;  /* Start wrt 0 */
+	u32 end;    /* End wrt 0 */
+};
+
+struct cudbg_mem_desc {
+	u32 base;
+	u32 limit;
+	u32 idx;
+};
+
+struct cudbg_meminfo {
+	struct cudbg_mem_desc avail[4];
+	struct cudbg_mem_desc mem[ARRAY_SIZE(cudbg_region) + 3];
+	u32 avail_c;
+	u32 mem_c;
+	u32 up_ram_lo;
+	u32 up_ram_hi;
+	u32 up_extmem2_lo;
+	u32 up_extmem2_hi;
+	u32 rx_pages_data[3];
+	u32 tx_pages_data[4];
+	u32 p_structs;
+	u32 reserved[12];
+	u32 port_used[4];
+	u32 port_alloc[4];
+	u32 loopback_used[NCHAN];
+	u32 loopback_alloc[NCHAN];
+};
+
 struct cudbg_cim_pif_la {
 	int size;
 	u8 data[0];
@@ -145,6 +185,7 @@ struct cudbg_tid_info_region_rev1 {
 	u32 reserved[16];
 };
 
+#define CUDBG_LOWMEM_MAX_CTXT_QIDS 256
 #define CUDBG_MAX_FL_QIDS 1024
 
 struct cudbg_ch_cntxt {
@@ -334,6 +375,25 @@ static const u32 t5_pm_tx_array[][IREG_NUM_ELEM] = {
 	{0x8FF0, 0x8FF4, 0x10021, 0x1D}, /* t5_pm_tx_regs_10021_to_1003c */
 };
 
+#define CUDBG_NUM_PCIE_CONFIG_REGS 0x61
+
+static const u32 t5_pcie_config_array[][2] = {
+	{0x0, 0x34},
+	{0x3c, 0x40},
+	{0x50, 0x64},
+	{0x70, 0x80},
+	{0x94, 0xa0},
+	{0xb0, 0xb8},
+	{0xd0, 0xd4},
+	{0x100, 0x128},
+	{0x140, 0x148},
+	{0x150, 0x164},
+	{0x170, 0x178},
+	{0x180, 0x194},
+	{0x1a0, 0x1b8},
+	{0x1c0, 0x208},
+};
+
 static const u32 t6_ma_ireg_array[][IREG_NUM_ELEM] = {
 	{0x78f8, 0x78fc, 0xa000, 23}, /* t6_ma_regs_a000_to_a016 */
 	{0x78f8, 0x78fc, 0xa400, 30}, /* t6_ma_regs_a400_to_a41e */
@@ -345,37 +405,55 @@ static const u32 t6_ma_ireg_array2[][IREG_NUM_ELEM] = {
 	{0x78f8, 0x78fc, 0xe640, 13} /* t6_ma_regs_e640_to_e7c0 */
 };
 
-static const u32 t6_up_cim_reg_array[][IREG_NUM_ELEM] = {
-	{0x7b50, 0x7b54, 0x2000, 0x20}, /* up_cim_2000_to_207c */
-	{0x7b50, 0x7b54, 0x2080, 0x1d}, /* up_cim_2080_to_20fc */
-	{0x7b50, 0x7b54, 0x00, 0x20}, /* up_cim_00_to_7c */
-	{0x7b50, 0x7b54, 0x80, 0x20}, /* up_cim_80_to_fc */
-	{0x7b50, 0x7b54, 0x100, 0x11}, /* up_cim_100_to_14c */
-	{0x7b50, 0x7b54, 0x200, 0x10}, /* up_cim_200_to_23c */
-	{0x7b50, 0x7b54, 0x240, 0x2}, /* up_cim_240_to_244 */
-	{0x7b50, 0x7b54, 0x250, 0x2}, /* up_cim_250_to_254 */
-	{0x7b50, 0x7b54, 0x260, 0x2}, /* up_cim_260_to_264 */
-	{0x7b50, 0x7b54, 0x270, 0x2}, /* up_cim_270_to_274 */
-	{0x7b50, 0x7b54, 0x280, 0x20}, /* up_cim_280_to_2fc */
-	{0x7b50, 0x7b54, 0x300, 0x20}, /* up_cim_300_to_37c */
-	{0x7b50, 0x7b54, 0x380, 0x14}, /* up_cim_380_to_3cc */
-
-};
-
-static const u32 t5_up_cim_reg_array[][IREG_NUM_ELEM] = {
-	{0x7b50, 0x7b54, 0x2000, 0x20}, /* up_cim_2000_to_207c */
-	{0x7b50, 0x7b54, 0x2080, 0x19}, /* up_cim_2080_to_20ec */
-	{0x7b50, 0x7b54, 0x00, 0x20}, /* up_cim_00_to_7c */
-	{0x7b50, 0x7b54, 0x80, 0x20}, /* up_cim_80_to_fc */
-	{0x7b50, 0x7b54, 0x100, 0x11}, /* up_cim_100_to_14c */
-	{0x7b50, 0x7b54, 0x200, 0x10}, /* up_cim_200_to_23c */
-	{0x7b50, 0x7b54, 0x240, 0x2}, /* up_cim_240_to_244 */
-	{0x7b50, 0x7b54, 0x250, 0x2}, /* up_cim_250_to_254 */
-	{0x7b50, 0x7b54, 0x260, 0x2}, /* up_cim_260_to_264 */
-	{0x7b50, 0x7b54, 0x270, 0x2}, /* up_cim_270_to_274 */
-	{0x7b50, 0x7b54, 0x280, 0x20}, /* up_cim_280_to_2fc */
-	{0x7b50, 0x7b54, 0x300, 0x20}, /* up_cim_300_to_37c */
-	{0x7b50, 0x7b54, 0x380, 0x14}, /* up_cim_380_to_3cc */
+static const u32 t6_up_cim_reg_array[][IREG_NUM_ELEM + 1] = {
+	{0x7b50, 0x7b54, 0x2000, 0x20, 0}, /* up_cim_2000_to_207c */
+	{0x7b50, 0x7b54, 0x2080, 0x1d, 0}, /* up_cim_2080_to_20fc */
+	{0x7b50, 0x7b54, 0x00, 0x20, 0}, /* up_cim_00_to_7c */
+	{0x7b50, 0x7b54, 0x80, 0x20, 0}, /* up_cim_80_to_fc */
+	{0x7b50, 0x7b54, 0x100, 0x11, 0}, /* up_cim_100_to_14c */
+	{0x7b50, 0x7b54, 0x200, 0x10, 0}, /* up_cim_200_to_23c */
+	{0x7b50, 0x7b54, 0x240, 0x2, 0}, /* up_cim_240_to_244 */
+	{0x7b50, 0x7b54, 0x250, 0x2, 0}, /* up_cim_250_to_254 */
+	{0x7b50, 0x7b54, 0x260, 0x2, 0}, /* up_cim_260_to_264 */
+	{0x7b50, 0x7b54, 0x270, 0x2, 0}, /* up_cim_270_to_274 */
+	{0x7b50, 0x7b54, 0x280, 0x20, 0}, /* up_cim_280_to_2fc */
+	{0x7b50, 0x7b54, 0x300, 0x20, 0}, /* up_cim_300_to_37c */
+	{0x7b50, 0x7b54, 0x380, 0x14, 0}, /* up_cim_380_to_3cc */
+	{0x7b50, 0x7b54, 0x2900, 0x4, 0x4}, /* up_cim_2900_to_3d40 */
+	{0x7b50, 0x7b54, 0x2904, 0x4, 0x4}, /* up_cim_2904_to_3d44 */
+	{0x7b50, 0x7b54, 0x2908, 0x4, 0x4}, /* up_cim_2908_to_3d48 */
+	{0x7b50, 0x7b54, 0x2910, 0x4, 0x4}, /* up_cim_2910_to_3d4c */
+	{0x7b50, 0x7b54, 0x2914, 0x4, 0x4}, /* up_cim_2914_to_3d50 */
+	{0x7b50, 0x7b54, 0x2920, 0x10, 0x10}, /* up_cim_2920_to_2a10 */
+	{0x7b50, 0x7b54, 0x2924, 0x10, 0x10}, /* up_cim_2924_to_2a14 */
+	{0x7b50, 0x7b54, 0x2928, 0x10, 0x10}, /* up_cim_2928_to_2a18 */
+	{0x7b50, 0x7b54, 0x292c, 0x10, 0x10}, /* up_cim_292c_to_2a1c */
+};
+
+static const u32 t5_up_cim_reg_array[][IREG_NUM_ELEM + 1] = {
+	{0x7b50, 0x7b54, 0x2000, 0x20, 0}, /* up_cim_2000_to_207c */
+	{0x7b50, 0x7b54, 0x2080, 0x19, 0}, /* up_cim_2080_to_20ec */
+	{0x7b50, 0x7b54, 0x00, 0x20, 0}, /* up_cim_00_to_7c */
+	{0x7b50, 0x7b54, 0x80, 0x20, 0}, /* up_cim_80_to_fc */
+	{0x7b50, 0x7b54, 0x100, 0x11, 0}, /* up_cim_100_to_14c */
+	{0x7b50, 0x7b54, 0x200, 0x10, 0}, /* up_cim_200_to_23c */
+	{0x7b50, 0x7b54, 0x240, 0x2, 0}, /* up_cim_240_to_244 */
+	{0x7b50, 0x7b54, 0x250, 0x2, 0}, /* up_cim_250_to_254 */
+	{0x7b50, 0x7b54, 0x260, 0x2, 0}, /* up_cim_260_to_264 */
+	{0x7b50, 0x7b54, 0x270, 0x2, 0}, /* up_cim_270_to_274 */
+	{0x7b50, 0x7b54, 0x280, 0x20, 0}, /* up_cim_280_to_2fc */
+	{0x7b50, 0x7b54, 0x300, 0x20, 0}, /* up_cim_300_to_37c */
+	{0x7b50, 0x7b54, 0x380, 0x14, 0}, /* up_cim_380_to_3cc */
+	{0x7b50, 0x7b54, 0x2900, 0x4, 0x4}, /* up_cim_2900_to_3d40 */
+	{0x7b50, 0x7b54, 0x2904, 0x4, 0x4}, /* up_cim_2904_to_3d44 */
+	{0x7b50, 0x7b54, 0x2908, 0x4, 0x4}, /* up_cim_2908_to_3d48 */
+	{0x7b50, 0x7b54, 0x2910, 0x4, 0x4}, /* up_cim_2910_to_3d4c */
+	{0x7b50, 0x7b54, 0x2914, 0x4, 0x4}, /* up_cim_2914_to_3d50 */
+	{0x7b50, 0x7b54, 0x2918, 0x4, 0x4}, /* up_cim_2918_to_3d54 */
+	{0x7b50, 0x7b54, 0x291c, 0x4, 0x4}, /* up_cim_291c_to_3d58 */
+	{0x7b50, 0x7b54, 0x2924, 0x10, 0x10}, /* up_cim_2924_to_2914 */
+	{0x7b50, 0x7b54, 0x2928, 0x10, 0x10}, /* up_cim_2928_to_2a18 */
+	{0x7b50, 0x7b54, 0x292c, 0x10, 0x10}, /* up_cim_292c_to_2a1c */
 };
 
 static const u32 t6_hma_ireg_array[][IREG_NUM_ELEM] = {
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h
index e10ff1ee62c5..8568a51f6414 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_if.h
@@ -21,6 +21,7 @@
 /* Error codes */
 #define CUDBG_STATUS_NO_MEM -19
 #define CUDBG_STATUS_ENTITY_NOT_FOUND -24
+#define CUDBG_STATUS_NOT_IMPLEMENTED -28
 #define CUDBG_SYSTEM_ERROR -29
 #define CUDBG_STATUS_CCLK_NOT_DEFINED -32
 
@@ -47,6 +48,8 @@ enum cudbg_dbg_entity_type {
 	CUDBG_CIM_OBQ_NCSI = 17,
 	CUDBG_EDC0 = 18,
 	CUDBG_EDC1 = 19,
+	CUDBG_MC0 = 20,
+	CUDBG_MC1 = 21,
 	CUDBG_RSS = 22,
 	CUDBG_RSS_VF_CONF = 25,
 	CUDBG_PATH_MTU = 27,
@@ -56,6 +59,7 @@ enum cudbg_dbg_entity_type {
 	CUDBG_SGE_INDIRECT = 37,
 	CUDBG_ULPRX_LA = 41,
 	CUDBG_TP_LA = 43,
+	CUDBG_MEMINFO = 44,
 	CUDBG_CIM_PIF_LA = 45,
 	CUDBG_CLK = 46,
 	CUDBG_CIM_OBQ_RXQ0 = 47,
@@ -63,6 +67,7 @@ enum cudbg_dbg_entity_type {
 	CUDBG_PCIE_INDIRECT = 50,
 	CUDBG_PM_INDIRECT = 51,
 	CUDBG_TID_INFO = 54,
+	CUDBG_PCIE_CONFIG = 55,
 	CUDBG_DUMP_CONTEXT = 56,
 	CUDBG_MPS_TCAM = 57,
 	CUDBG_VPD_DATA = 58,
@@ -74,6 +79,7 @@ enum cudbg_dbg_entity_type {
 	CUDBG_PBT_TABLE = 65,
 	CUDBG_MBOX_LOG = 66,
 	CUDBG_HMA_INDIRECT = 67,
+	CUDBG_HMA = 68,
 	CUDBG_MAX_ENTITY = 70,
 };
 
@@ -81,6 +87,10 @@ struct cudbg_init {
 	struct adapter *adap; /* Pointer to adapter structure */
 	void *outbuf; /* Output buffer */
 	u32 outbuf_size;  /* Output buffer size */
+	u8 compress_type; /* Type of compression to use */
+	void *compress_buff; /* Compression buffer */
+	u32 compress_buff_size; /* Compression buffer size */
+	void *workspace; /* Workspace for zlib */
 };
 
 static inline unsigned int cudbg_mbytes_to_bytes(unsigned int size)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
index d699bf88d18f..00a1d2d13169 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
@@ -15,18 +15,65 @@
  *
  */
 
+#include <linux/sort.h>
+
 #include "t4_regs.h"
 #include "cxgb4.h"
 #include "cudbg_if.h"
 #include "cudbg_lib_common.h"
-#include "cudbg_lib.h"
 #include "cudbg_entity.h"
+#include "cudbg_lib.h"
+#include "cudbg_zlib.h"
 
-static void cudbg_write_and_release_buff(struct cudbg_buffer *pin_buff,
-					 struct cudbg_buffer *dbg_buff)
+static int cudbg_do_compression(struct cudbg_init *pdbg_init,
+				struct cudbg_buffer *pin_buff,
+				struct cudbg_buffer *dbg_buff)
 {
-	cudbg_update_buff(pin_buff, dbg_buff);
-	cudbg_put_buff(pin_buff, dbg_buff);
+	struct cudbg_buffer temp_in_buff = { 0 };
+	int bytes_left, bytes_read, bytes;
+	u32 offset = dbg_buff->offset;
+	int rc;
+
+	temp_in_buff.offset = pin_buff->offset;
+	temp_in_buff.data = pin_buff->data;
+	temp_in_buff.size = pin_buff->size;
+
+	bytes_left = pin_buff->size;
+	bytes_read = 0;
+	while (bytes_left > 0) {
+		/* Do compression in smaller chunks */
+		bytes = min_t(unsigned long, bytes_left,
+			      (unsigned long)CUDBG_CHUNK_SIZE);
+		temp_in_buff.data = (char *)pin_buff->data + bytes_read;
+		temp_in_buff.size = bytes;
+		rc = cudbg_compress_buff(pdbg_init, &temp_in_buff, dbg_buff);
+		if (rc)
+			return rc;
+		bytes_left -= bytes;
+		bytes_read += bytes;
+	}
+
+	pin_buff->size = dbg_buff->offset - offset;
+	return 0;
+}
+
+static int cudbg_write_and_release_buff(struct cudbg_init *pdbg_init,
+					struct cudbg_buffer *pin_buff,
+					struct cudbg_buffer *dbg_buff)
+{
+	int rc = 0;
+
+	if (pdbg_init->compress_type == CUDBG_COMPRESSION_NONE) {
+		cudbg_update_buff(pin_buff, dbg_buff);
+	} else {
+		rc = cudbg_do_compression(pdbg_init, pin_buff, dbg_buff);
+		if (rc)
+			goto out;
+	}
+
+out:
+	cudbg_put_buff(pdbg_init, pin_buff);
+	return rc;
 }
 
 static int is_fw_attached(struct cudbg_init *pdbg_init)
@@ -84,6 +131,277 @@ static int cudbg_read_vpd_reg(struct adapter *padap, u32 addr, u32 len,
 	return 0;
 }
 
+static int cudbg_mem_desc_cmp(const void *a, const void *b)
+{
+	return ((const struct cudbg_mem_desc *)a)->base -
+	       ((const struct cudbg_mem_desc *)b)->base;
+}
+
+int cudbg_fill_meminfo(struct adapter *padap,
+		       struct cudbg_meminfo *meminfo_buff)
+{
+	struct cudbg_mem_desc *md;
+	u32 lo, hi, used, alloc;
+	int n, i;
+
+	memset(meminfo_buff->avail, 0,
+	       ARRAY_SIZE(meminfo_buff->avail) *
+	       sizeof(struct cudbg_mem_desc));
+	memset(meminfo_buff->mem, 0,
+	       (ARRAY_SIZE(cudbg_region) + 3) * sizeof(struct cudbg_mem_desc));
+	md  = meminfo_buff->mem;
+
+	for (i = 0; i < ARRAY_SIZE(meminfo_buff->mem); i++) {
+		meminfo_buff->mem[i].limit = 0;
+		meminfo_buff->mem[i].idx = i;
+	}
+
+	/* Find and sort the populated memory ranges */
+	i = 0;
+	lo = t4_read_reg(padap, MA_TARGET_MEM_ENABLE_A);
+	if (lo & EDRAM0_ENABLE_F) {
+		hi = t4_read_reg(padap, MA_EDRAM0_BAR_A);
+		meminfo_buff->avail[i].base =
+			cudbg_mbytes_to_bytes(EDRAM0_BASE_G(hi));
+		meminfo_buff->avail[i].limit =
+			meminfo_buff->avail[i].base +
+			cudbg_mbytes_to_bytes(EDRAM0_SIZE_G(hi));
+		meminfo_buff->avail[i].idx = 0;
+		i++;
+	}
+
+	if (lo & EDRAM1_ENABLE_F) {
+		hi =  t4_read_reg(padap, MA_EDRAM1_BAR_A);
+		meminfo_buff->avail[i].base =
+			cudbg_mbytes_to_bytes(EDRAM1_BASE_G(hi));
+		meminfo_buff->avail[i].limit =
+			meminfo_buff->avail[i].base +
+			cudbg_mbytes_to_bytes(EDRAM1_SIZE_G(hi));
+		meminfo_buff->avail[i].idx = 1;
+		i++;
+	}
+
+	if (is_t5(padap->params.chip)) {
+		if (lo & EXT_MEM0_ENABLE_F) {
+			hi = t4_read_reg(padap, MA_EXT_MEMORY0_BAR_A);
+			meminfo_buff->avail[i].base =
+				cudbg_mbytes_to_bytes(EXT_MEM_BASE_G(hi));
+			meminfo_buff->avail[i].limit =
+				meminfo_buff->avail[i].base +
+				cudbg_mbytes_to_bytes(EXT_MEM_SIZE_G(hi));
+			meminfo_buff->avail[i].idx = 3;
+			i++;
+		}
+
+		if (lo & EXT_MEM1_ENABLE_F) {
+			hi = t4_read_reg(padap, MA_EXT_MEMORY1_BAR_A);
+			meminfo_buff->avail[i].base =
+				cudbg_mbytes_to_bytes(EXT_MEM1_BASE_G(hi));
+			meminfo_buff->avail[i].limit =
+				meminfo_buff->avail[i].base +
+				cudbg_mbytes_to_bytes(EXT_MEM1_SIZE_G(hi));
+			meminfo_buff->avail[i].idx = 4;
+			i++;
+		}
+	} else {
+		if (lo & EXT_MEM_ENABLE_F) {
+			hi = t4_read_reg(padap, MA_EXT_MEMORY_BAR_A);
+			meminfo_buff->avail[i].base =
+				cudbg_mbytes_to_bytes(EXT_MEM_BASE_G(hi));
+			meminfo_buff->avail[i].limit =
+				meminfo_buff->avail[i].base +
+				cudbg_mbytes_to_bytes(EXT_MEM_SIZE_G(hi));
+			meminfo_buff->avail[i].idx = 2;
+			i++;
+		}
+
+		if (lo & HMA_MUX_F) {
+			hi = t4_read_reg(padap, MA_EXT_MEMORY1_BAR_A);
+			meminfo_buff->avail[i].base =
+				cudbg_mbytes_to_bytes(EXT_MEM1_BASE_G(hi));
+			meminfo_buff->avail[i].limit =
+				meminfo_buff->avail[i].base +
+				cudbg_mbytes_to_bytes(EXT_MEM1_SIZE_G(hi));
+			meminfo_buff->avail[i].idx = 5;
+			i++;
+		}
+	}
+
+	if (!i) /* no memory available */
+		return CUDBG_STATUS_ENTITY_NOT_FOUND;
+
+	meminfo_buff->avail_c = i;
+	sort(meminfo_buff->avail, i, sizeof(struct cudbg_mem_desc),
+	     cudbg_mem_desc_cmp, NULL);
+	(md++)->base = t4_read_reg(padap, SGE_DBQ_CTXT_BADDR_A);
+	(md++)->base = t4_read_reg(padap, SGE_IMSG_CTXT_BADDR_A);
+	(md++)->base = t4_read_reg(padap, SGE_FLM_CACHE_BADDR_A);
+	(md++)->base = t4_read_reg(padap, TP_CMM_TCB_BASE_A);
+	(md++)->base = t4_read_reg(padap, TP_CMM_MM_BASE_A);
+	(md++)->base = t4_read_reg(padap, TP_CMM_TIMER_BASE_A);
+	(md++)->base = t4_read_reg(padap, TP_CMM_MM_RX_FLST_BASE_A);
+	(md++)->base = t4_read_reg(padap, TP_CMM_MM_TX_FLST_BASE_A);
+	(md++)->base = t4_read_reg(padap, TP_CMM_MM_PS_FLST_BASE_A);
+
+	/* the next few have explicit upper bounds */
+	md->base = t4_read_reg(padap, TP_PMM_TX_BASE_A);
+	md->limit = md->base - 1 +
+		    t4_read_reg(padap, TP_PMM_TX_PAGE_SIZE_A) *
+		    PMTXMAXPAGE_G(t4_read_reg(padap, TP_PMM_TX_MAX_PAGE_A));
+	md++;
+
+	md->base = t4_read_reg(padap, TP_PMM_RX_BASE_A);
+	md->limit = md->base - 1 +
+		    t4_read_reg(padap, TP_PMM_RX_PAGE_SIZE_A) *
+		    PMRXMAXPAGE_G(t4_read_reg(padap, TP_PMM_RX_MAX_PAGE_A));
+	md++;
+
+	if (t4_read_reg(padap, LE_DB_CONFIG_A) & HASHEN_F) {
+		if (CHELSIO_CHIP_VERSION(padap->params.chip) <= CHELSIO_T5) {
+			hi = t4_read_reg(padap, LE_DB_TID_HASHBASE_A) / 4;
+			md->base = t4_read_reg(padap, LE_DB_HASH_TID_BASE_A);
+		} else {
+			hi = t4_read_reg(padap, LE_DB_HASH_TID_BASE_A);
+			md->base = t4_read_reg(padap,
+					       LE_DB_HASH_TBL_BASE_ADDR_A);
+		}
+		md->limit = 0;
+	} else {
+		md->base = 0;
+		md->idx = ARRAY_SIZE(cudbg_region);  /* hide it */
+	}
+	md++;
+
+#define ulp_region(reg) do { \
+	md->base = t4_read_reg(padap, ULP_ ## reg ## _LLIMIT_A);\
+	(md++)->limit = t4_read_reg(padap, ULP_ ## reg ## _ULIMIT_A);\
+} while (0)
+
+	ulp_region(RX_ISCSI);
+	ulp_region(RX_TDDP);
+	ulp_region(TX_TPT);
+	ulp_region(RX_STAG);
+	ulp_region(RX_RQ);
+	ulp_region(RX_RQUDP);
+	ulp_region(RX_PBL);
+	ulp_region(TX_PBL);
+#undef ulp_region
+	md->base = 0;
+	md->idx = ARRAY_SIZE(cudbg_region);
+	if (!is_t4(padap->params.chip)) {
+		u32 fifo_size = t4_read_reg(padap, SGE_DBVFIFO_SIZE_A);
+		u32 sge_ctrl = t4_read_reg(padap, SGE_CONTROL2_A);
+		u32 size = 0;
+
+		if (is_t5(padap->params.chip)) {
+			if (sge_ctrl & VFIFO_ENABLE_F)
+				size = DBVFIFO_SIZE_G(fifo_size);
+		} else {
+			size = T6_DBVFIFO_SIZE_G(fifo_size);
+		}
+
+		if (size) {
+			md->base = BASEADDR_G(t4_read_reg(padap,
+							  SGE_DBVFIFO_BADDR_A));
+			md->limit = md->base + (size << 2) - 1;
+		}
+	}
+
+	md++;
+
+	md->base = t4_read_reg(padap, ULP_RX_CTX_BASE_A);
+	md->limit = 0;
+	md++;
+	md->base = t4_read_reg(padap, ULP_TX_ERR_TABLE_BASE_A);
+	md->limit = 0;
+	md++;
+
+	md->base = padap->vres.ocq.start;
+	if (padap->vres.ocq.size)
+		md->limit = md->base + padap->vres.ocq.size - 1;
+	else
+		md->idx = ARRAY_SIZE(cudbg_region);  /* hide it */
+	md++;
+
+	/* add any address-space holes, there can be up to 3 */
+	for (n = 0; n < i - 1; n++)
+		if (meminfo_buff->avail[n].limit <
+		    meminfo_buff->avail[n + 1].base)
+			(md++)->base = meminfo_buff->avail[n].limit;
+
+	if (meminfo_buff->avail[n].limit)
+		(md++)->base = meminfo_buff->avail[n].limit;
+
+	n = md - meminfo_buff->mem;
+	meminfo_buff->mem_c = n;
+
+	sort(meminfo_buff->mem, n, sizeof(struct cudbg_mem_desc),
+	     cudbg_mem_desc_cmp, NULL);
+
+	lo = t4_read_reg(padap, CIM_SDRAM_BASE_ADDR_A);
+	hi = t4_read_reg(padap, CIM_SDRAM_ADDR_SIZE_A) + lo - 1;
+	meminfo_buff->up_ram_lo = lo;
+	meminfo_buff->up_ram_hi = hi;
+
+	lo = t4_read_reg(padap, CIM_EXTMEM2_BASE_ADDR_A);
+	hi = t4_read_reg(padap, CIM_EXTMEM2_ADDR_SIZE_A) + lo - 1;
+	meminfo_buff->up_extmem2_lo = lo;
+	meminfo_buff->up_extmem2_hi = hi;
+
+	lo = t4_read_reg(padap, TP_PMM_RX_MAX_PAGE_A);
+	meminfo_buff->rx_pages_data[0] =  PMRXMAXPAGE_G(lo);
+	meminfo_buff->rx_pages_data[1] =
+		t4_read_reg(padap, TP_PMM_RX_PAGE_SIZE_A) >> 10;
+	meminfo_buff->rx_pages_data[2] = (lo & PMRXNUMCHN_F) ? 2 : 1;
+
+	lo = t4_read_reg(padap, TP_PMM_TX_MAX_PAGE_A);
+	hi = t4_read_reg(padap, TP_PMM_TX_PAGE_SIZE_A);
+	meminfo_buff->tx_pages_data[0] = PMTXMAXPAGE_G(lo);
+	meminfo_buff->tx_pages_data[1] =
+		hi >= (1 << 20) ? (hi >> 20) : (hi >> 10);
+	meminfo_buff->tx_pages_data[2] =
+		hi >= (1 << 20) ? 'M' : 'K';
+	meminfo_buff->tx_pages_data[3] = 1 << PMTXNUMCHN_G(lo);
+
+	meminfo_buff->p_structs = t4_read_reg(padap, TP_CMM_MM_MAX_PSTRUCT_A);
+
+	for (i = 0; i < 4; i++) {
+		if (CHELSIO_CHIP_VERSION(padap->params.chip) > CHELSIO_T5)
+			lo = t4_read_reg(padap,
+					 MPS_RX_MAC_BG_PG_CNT0_A + i * 4);
+		else
+			lo = t4_read_reg(padap, MPS_RX_PG_RSV0_A + i * 4);
+		if (is_t5(padap->params.chip)) {
+			used = T5_USED_G(lo);
+			alloc = T5_ALLOC_G(lo);
+		} else {
+			used = USED_G(lo);
+			alloc = ALLOC_G(lo);
+		}
+		meminfo_buff->port_used[i] = used;
+		meminfo_buff->port_alloc[i] = alloc;
+	}
+
+	for (i = 0; i < padap->params.arch.nchan; i++) {
+		if (CHELSIO_CHIP_VERSION(padap->params.chip) > CHELSIO_T5)
+			lo = t4_read_reg(padap,
+					 MPS_RX_LPBK_BG_PG_CNT0_A + i * 4);
+		else
+			lo = t4_read_reg(padap, MPS_RX_PG_RSV4_A + i * 4);
+		if (is_t5(padap->params.chip)) {
+			used = T5_USED_G(lo);
+			alloc = T5_ALLOC_G(lo);
+		} else {
+			used = USED_G(lo);
+			alloc = ALLOC_G(lo);
+		}
+		meminfo_buff->loopback_used[i] = used;
+		meminfo_buff->loopback_alloc[i] = alloc;
+	}
+
+	return 0;
+}
+
 int cudbg_collect_reg_dump(struct cudbg_init *pdbg_init,
 			   struct cudbg_buffer *dbg_buff,
 			   struct cudbg_error *cudbg_err)
@@ -98,12 +416,11 @@ int cudbg_collect_reg_dump(struct cudbg_init *pdbg_init,
 	else if (is_t5(padap->params.chip) || is_t6(padap->params.chip))
 		buf_size = T5_REGMAP_SIZE;
 
-	rc = cudbg_get_buff(dbg_buff, buf_size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, buf_size, &temp_buff);
 	if (rc)
 		return rc;
 	t4_get_regs(padap, (void *)temp_buff.data, temp_buff.size);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_fw_devlog(struct cudbg_init *pdbg_init,
@@ -122,7 +439,7 @@ int cudbg_collect_fw_devlog(struct cudbg_init *pdbg_init,
 	}
 
 	dparams = &padap->params.devlog;
-	rc = cudbg_get_buff(dbg_buff, dparams->size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, dparams->size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -137,12 +454,11 @@ int cudbg_collect_fw_devlog(struct cudbg_init *pdbg_init,
 		spin_unlock(&padap->win0_lock);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_cim_la(struct cudbg_init *pdbg_init,
@@ -156,21 +472,21 @@ int cudbg_collect_cim_la(struct cudbg_init *pdbg_init,
 
 	if (is_t6(padap->params.chip)) {
 		size = padap->params.cim_la_size / 10 + 1;
-		size *= 11 * sizeof(u32);
+		size *= 10 * sizeof(u32);
 	} else {
 		size = padap->params.cim_la_size / 8;
 		size *= 8 * sizeof(u32);
 	}
 
 	size += sizeof(cfg);
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
 	rc = t4_cim_read(padap, UP_UP_DBG_LA_CFG_A, 1, &cfg);
 	if (rc) {
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
 
@@ -180,11 +496,10 @@ int cudbg_collect_cim_la(struct cudbg_init *pdbg_init,
 			    NULL);
 	if (rc < 0) {
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_cim_ma_la(struct cudbg_init *pdbg_init,
@@ -196,7 +511,7 @@ int cudbg_collect_cim_ma_la(struct cudbg_init *pdbg_init,
 	int size, rc;
 
 	size = 2 * CIM_MALA_SIZE * 5 * sizeof(u32);
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -204,8 +519,7 @@ int cudbg_collect_cim_ma_la(struct cudbg_init *pdbg_init,
 			  (u32 *)temp_buff.data,
 			  (u32 *)((char *)temp_buff.data +
 				  5 * CIM_MALA_SIZE));
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_cim_qcfg(struct cudbg_init *pdbg_init,
@@ -217,7 +531,7 @@ int cudbg_collect_cim_qcfg(struct cudbg_init *pdbg_init,
 	struct cudbg_cim_qcfg *cim_qcfg_data;
 	int rc;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_cim_qcfg),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_cim_qcfg),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -228,7 +542,7 @@ int cudbg_collect_cim_qcfg(struct cudbg_init *pdbg_init,
 			 ARRAY_SIZE(cim_qcfg_data->stat), cim_qcfg_data->stat);
 	if (rc) {
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
 
@@ -237,14 +551,13 @@ int cudbg_collect_cim_qcfg(struct cudbg_init *pdbg_init,
 			 cim_qcfg_data->obq_wr);
 	if (rc) {
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
 
 	t4_read_cimq_cfg(padap, cim_qcfg_data->base, cim_qcfg_data->size,
 			 cim_qcfg_data->thres);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 static int cudbg_read_cim_ibq(struct cudbg_init *pdbg_init,
@@ -258,7 +571,7 @@ static int cudbg_read_cim_ibq(struct cudbg_init *pdbg_init,
 
 	/* collect CIM IBQ */
 	qsize = CIM_IBQ_SIZE * 4 * sizeof(u32);
-	rc = cudbg_get_buff(dbg_buff, qsize, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, qsize, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -272,11 +585,10 @@ static int cudbg_read_cim_ibq(struct cudbg_init *pdbg_init,
 		else
 			rc = no_of_read_words;
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_cim_ibq_tp0(struct cudbg_init *pdbg_init,
@@ -343,7 +655,7 @@ static int cudbg_read_cim_obq(struct cudbg_init *pdbg_init,
 
 	/* collect CIM OBQ */
 	qsize =  cudbg_cim_obq_size(padap, qid);
-	rc = cudbg_get_buff(dbg_buff, qsize, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, qsize, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -357,11 +669,10 @@ static int cudbg_read_cim_obq(struct cudbg_init *pdbg_init,
 		else
 			rc = no_of_read_words;
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_cim_obq_ulp0(struct cudbg_init *pdbg_init,
@@ -420,23 +731,211 @@ int cudbg_collect_obq_sge_rx_q1(struct cudbg_init *pdbg_init,
 	return cudbg_read_cim_obq(pdbg_init, dbg_buff, cudbg_err, 7);
 }
 
+static int cudbg_meminfo_get_mem_index(struct adapter *padap,
+				       struct cudbg_meminfo *mem_info,
+				       u8 mem_type, u8 *idx)
+{
+	u8 i, flag;
+
+	switch (mem_type) {
+	case MEM_EDC0:
+		flag = EDC0_FLAG;
+		break;
+	case MEM_EDC1:
+		flag = EDC1_FLAG;
+		break;
+	case MEM_MC0:
+		/* Some T5 cards have both MC0 and MC1. */
+		flag = is_t5(padap->params.chip) ? MC0_FLAG : MC_FLAG;
+		break;
+	case MEM_MC1:
+		flag = MC1_FLAG;
+		break;
+	case MEM_HMA:
+		flag = HMA_FLAG;
+		break;
+	default:
+		return CUDBG_STATUS_ENTITY_NOT_FOUND;
+	}
+
+	for (i = 0; i < mem_info->avail_c; i++) {
+		if (mem_info->avail[i].idx == flag) {
+			*idx = i;
+			return 0;
+		}
+	}
+
+	return CUDBG_STATUS_ENTITY_NOT_FOUND;
+}
+
+/* Fetch the @region_name's start and end from @meminfo. */
+static int cudbg_get_mem_region(struct adapter *padap,
+				struct cudbg_meminfo *meminfo,
+				u8 mem_type, const char *region_name,
+				struct cudbg_mem_desc *mem_desc)
+{
+	u8 mc, found = 0;
+	u32 i, idx = 0;
+	int rc;
+
+	rc = cudbg_meminfo_get_mem_index(padap, meminfo, mem_type, &mc);
+	if (rc)
+		return rc;
+
+	for (i = 0; i < ARRAY_SIZE(cudbg_region); i++) {
+		if (!strcmp(cudbg_region[i], region_name)) {
+			found = 1;
+			idx = i;
+			break;
+		}
+	}
+	if (!found)
+		return -EINVAL;
+
+	found = 0;
+	for (i = 0; i < meminfo->mem_c; i++) {
+		if (meminfo->mem[i].idx >= ARRAY_SIZE(cudbg_region))
+			continue; /* Skip holes */
+
+		if (!(meminfo->mem[i].limit))
+			meminfo->mem[i].limit =
+				i < meminfo->mem_c - 1 ?
+				meminfo->mem[i + 1].base - 1 : ~0;
+
+		if (meminfo->mem[i].idx == idx) {
+			/* Check if the region exists in @mem_type memory */
+			if (meminfo->mem[i].base < meminfo->avail[mc].base &&
+			    meminfo->mem[i].limit < meminfo->avail[mc].base)
+				return -EINVAL;
+
+			if (meminfo->mem[i].base > meminfo->avail[mc].limit)
+				return -EINVAL;
+
+			memcpy(mem_desc, &meminfo->mem[i],
+			       sizeof(struct cudbg_mem_desc));
+			found = 1;
+			break;
+		}
+	}
+	if (!found)
+		return -EINVAL;
+
+	return 0;
+}
+
+/* Fetch and update the start and end of the requested memory region w.r.t 0
+ * in the corresponding EDC/MC/HMA.
+ */
+static int cudbg_get_mem_relative(struct adapter *padap,
+				  struct cudbg_meminfo *meminfo,
+				  u8 mem_type, u32 *out_base, u32 *out_end)
+{
+	u8 mc_idx;
+	int rc;
+
+	rc = cudbg_meminfo_get_mem_index(padap, meminfo, mem_type, &mc_idx);
+	if (rc)
+		return rc;
+
+	if (*out_base < meminfo->avail[mc_idx].base)
+		*out_base = 0;
+	else
+		*out_base -= meminfo->avail[mc_idx].base;
+
+	if (*out_end > meminfo->avail[mc_idx].limit)
+		*out_end = meminfo->avail[mc_idx].limit;
+	else
+		*out_end -= meminfo->avail[mc_idx].base;
+
+	return 0;
+}
+
+/* Get TX and RX Payload region */
+static int cudbg_get_payload_range(struct adapter *padap, u8 mem_type,
+				   const char *region_name,
+				   struct cudbg_region_info *payload)
+{
+	struct cudbg_mem_desc mem_desc = { 0 };
+	struct cudbg_meminfo meminfo;
+	int rc;
+
+	rc = cudbg_fill_meminfo(padap, &meminfo);
+	if (rc)
+		return rc;
+
+	rc = cudbg_get_mem_region(padap, &meminfo, mem_type, region_name,
+				  &mem_desc);
+	if (rc) {
+		payload->exist = false;
+		return 0;
+	}
+
+	payload->exist = true;
+	payload->start = mem_desc.base;
+	payload->end = mem_desc.limit;
+
+	return cudbg_get_mem_relative(padap, &meminfo, mem_type,
+				      &payload->start, &payload->end);
+}
+
+#define CUDBG_YIELD_ITERATION 256
+
 static int cudbg_read_fw_mem(struct cudbg_init *pdbg_init,
 			     struct cudbg_buffer *dbg_buff, u8 mem_type,
 			     unsigned long tot_len,
 			     struct cudbg_error *cudbg_err)
 {
+	static const char * const region_name[] = { "Tx payload:",
+						    "Rx payload:" };
 	unsigned long bytes, bytes_left, bytes_read = 0;
 	struct adapter *padap = pdbg_init->adap;
 	struct cudbg_buffer temp_buff = { 0 };
+	struct cudbg_region_info payload[2];
+	u32 yield_count = 0;
 	int rc = 0;
+	u8 i;
+
+	/* Get TX/RX Payload region range if they exist */
+	memset(payload, 0, sizeof(payload));
+	for (i = 0; i < ARRAY_SIZE(region_name); i++) {
+		rc = cudbg_get_payload_range(padap, mem_type, region_name[i],
+					     &payload[i]);
+		if (rc)
+			return rc;
+
+		if (payload[i].exist) {
+			/* Align start and end to avoid wrap around */
+			payload[i].start = roundup(payload[i].start,
+						   CUDBG_CHUNK_SIZE);
+			payload[i].end = rounddown(payload[i].end,
+						   CUDBG_CHUNK_SIZE);
+		}
+	}
 
 	bytes_left = tot_len;
 	while (bytes_left > 0) {
+		/* As MC size is huge and read through PIO access, this
+		 * loop will hold cpu for a longer time. OS may think that
+		 * the process is hanged and will generate CPU stall traces.
+		 * So yield the cpu regularly.
+		 */
+		yield_count++;
+		if (!(yield_count % CUDBG_YIELD_ITERATION))
+			schedule();
+
 		bytes = min_t(unsigned long, bytes_left,
 			      (unsigned long)CUDBG_CHUNK_SIZE);
-		rc = cudbg_get_buff(dbg_buff, bytes, &temp_buff);
+		rc = cudbg_get_buff(pdbg_init, dbg_buff, bytes, &temp_buff);
 		if (rc)
 			return rc;
+
+		for (i = 0; i < ARRAY_SIZE(payload); i++)
+			if (payload[i].exist &&
+			    bytes_read >= payload[i].start &&
+			    bytes_read + bytes <= payload[i].end)
+				/* TX and RX Payload regions can't overlap */
+				goto skip_read;
+
 		spin_lock(&padap->win0_lock);
 		rc = t4_memory_rw(padap, MEMWIN_NIC, mem_type,
 				  bytes_read, bytes,
@@ -445,37 +944,23 @@ static int cudbg_read_fw_mem(struct cudbg_init *pdbg_init,
 		spin_unlock(&padap->win0_lock);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
+
+skip_read:
 		bytes_left -= bytes;
 		bytes_read += bytes;
-		cudbg_write_and_release_buff(&temp_buff, dbg_buff);
+		rc = cudbg_write_and_release_buff(pdbg_init, &temp_buff,
+						  dbg_buff);
+		if (rc) {
+			cudbg_put_buff(pdbg_init, &temp_buff);
+			return rc;
+		}
 	}
 	return rc;
 }
 
-static void cudbg_collect_mem_info(struct cudbg_init *pdbg_init,
-				   struct card_mem *mem_info)
-{
-	struct adapter *padap = pdbg_init->adap;
-	u32 value;
-
-	value = t4_read_reg(padap, MA_EDRAM0_BAR_A);
-	value = EDRAM0_SIZE_G(value);
-	mem_info->size_edc0 = (u16)value;
-
-	value = t4_read_reg(padap, MA_EDRAM1_BAR_A);
-	value = EDRAM1_SIZE_G(value);
-	mem_info->size_edc1 = (u16)value;
-
-	value = t4_read_reg(padap, MA_TARGET_MEM_ENABLE_A);
-	if (value & EDRAM0_ENABLE_F)
-		mem_info->mem_flag |= (1 << EDC0_FLAG);
-	if (value & EDRAM1_ENABLE_F)
-		mem_info->mem_flag |= (1 << EDC1_FLAG);
-}
-
 static void cudbg_t4_fwcache(struct cudbg_init *pdbg_init,
 			     struct cudbg_error *cudbg_err)
 {
@@ -495,37 +980,25 @@ static int cudbg_collect_mem_region(struct cudbg_init *pdbg_init,
 				    struct cudbg_error *cudbg_err,
 				    u8 mem_type)
 {
-	struct card_mem mem_info = {0};
-	unsigned long flag, size;
+	struct adapter *padap = pdbg_init->adap;
+	struct cudbg_meminfo mem_info;
+	unsigned long size;
+	u8 mc_idx;
 	int rc;
 
+	memset(&mem_info, 0, sizeof(struct cudbg_meminfo));
+	rc = cudbg_fill_meminfo(padap, &mem_info);
+	if (rc)
+		return rc;
+
 	cudbg_t4_fwcache(pdbg_init, cudbg_err);
-	cudbg_collect_mem_info(pdbg_init, &mem_info);
-	switch (mem_type) {
-	case MEM_EDC0:
-		flag = (1 << EDC0_FLAG);
-		size = cudbg_mbytes_to_bytes(mem_info.size_edc0);
-		break;
-	case MEM_EDC1:
-		flag = (1 << EDC1_FLAG);
-		size = cudbg_mbytes_to_bytes(mem_info.size_edc1);
-		break;
-	default:
-		rc = CUDBG_STATUS_ENTITY_NOT_FOUND;
-		goto err;
-	}
+	rc = cudbg_meminfo_get_mem_index(padap, &mem_info, mem_type, &mc_idx);
+	if (rc)
+		return rc;
 
-	if (mem_info.mem_flag & flag) {
-		rc = cudbg_read_fw_mem(pdbg_init, dbg_buff, mem_type,
-				       size, cudbg_err);
-		if (rc)
-			goto err;
-	} else {
-		rc = CUDBG_STATUS_ENTITY_NOT_FOUND;
-		goto err;
-	}
-err:
-	return rc;
+	size = mem_info.avail[mc_idx].limit - mem_info.avail[mc_idx].base;
+	return cudbg_read_fw_mem(pdbg_init, dbg_buff, mem_type, size,
+				 cudbg_err);
 }
 
 int cudbg_collect_edc0_meminfo(struct cudbg_init *pdbg_init,
@@ -544,26 +1017,51 @@ int cudbg_collect_edc1_meminfo(struct cudbg_init *pdbg_init,
 					MEM_EDC1);
 }
 
+int cudbg_collect_mc0_meminfo(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err)
+{
+	return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err,
+					MEM_MC0);
+}
+
+int cudbg_collect_mc1_meminfo(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err)
+{
+	return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err,
+					MEM_MC1);
+}
+
+int cudbg_collect_hma_meminfo(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err)
+{
+	return cudbg_collect_mem_region(pdbg_init, dbg_buff, cudbg_err,
+					MEM_HMA);
+}
+
 int cudbg_collect_rss(struct cudbg_init *pdbg_init,
 		      struct cudbg_buffer *dbg_buff,
 		      struct cudbg_error *cudbg_err)
 {
 	struct adapter *padap = pdbg_init->adap;
 	struct cudbg_buffer temp_buff = { 0 };
-	int rc;
+	int rc, nentries;
 
-	rc = cudbg_get_buff(dbg_buff, RSS_NENTRIES * sizeof(u16), &temp_buff);
+	nentries = t4_chip_rss_size(padap);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, nentries * sizeof(u16),
+			    &temp_buff);
 	if (rc)
 		return rc;
 
 	rc = t4_read_rss(padap, (u16 *)temp_buff.data);
 	if (rc) {
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_rss_vf_config(struct cudbg_init *pdbg_init,
@@ -576,7 +1074,7 @@ int cudbg_collect_rss_vf_config(struct cudbg_init *pdbg_init,
 	int vf, rc, vf_count;
 
 	vf_count = padap->params.arch.vfcount;
-	rc = cudbg_get_buff(dbg_buff,
+	rc = cudbg_get_buff(pdbg_init, dbg_buff,
 			    vf_count * sizeof(struct cudbg_rss_vf_conf),
 			    &temp_buff);
 	if (rc)
@@ -586,8 +1084,7 @@ int cudbg_collect_rss_vf_config(struct cudbg_init *pdbg_init,
 	for (vf = 0; vf < vf_count; vf++)
 		t4_read_rss_vf_config(padap, vf, &vfconf[vf].rss_vf_vfl,
 				      &vfconf[vf].rss_vf_vfh, true);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_path_mtu(struct cudbg_init *pdbg_init,
@@ -598,13 +1095,13 @@ int cudbg_collect_path_mtu(struct cudbg_init *pdbg_init,
 	struct cudbg_buffer temp_buff = { 0 };
 	int rc;
 
-	rc = cudbg_get_buff(dbg_buff, NMTUS * sizeof(u16), &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, NMTUS * sizeof(u16),
+			    &temp_buff);
 	if (rc)
 		return rc;
 
 	t4_read_mtu_tbl(padap, (u16 *)temp_buff.data, NULL);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_pm_stats(struct cudbg_init *pdbg_init,
@@ -616,7 +1113,7 @@ int cudbg_collect_pm_stats(struct cudbg_init *pdbg_init,
 	struct cudbg_pm_stats *pm_stats_buff;
 	int rc;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_pm_stats),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_pm_stats),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -624,8 +1121,7 @@ int cudbg_collect_pm_stats(struct cudbg_init *pdbg_init,
 	pm_stats_buff = (struct cudbg_pm_stats *)temp_buff.data;
 	t4_pmtx_get_stats(padap, pm_stats_buff->tx_cnt, pm_stats_buff->tx_cyc);
 	t4_pmrx_get_stats(padap, pm_stats_buff->rx_cnt, pm_stats_buff->rx_cyc);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_hw_sched(struct cudbg_init *pdbg_init,
@@ -640,7 +1136,7 @@ int cudbg_collect_hw_sched(struct cudbg_init *pdbg_init,
 	if (!padap->params.vpd.cclk)
 		return CUDBG_STATUS_CCLK_NOT_DEFINED;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_hw_sched),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_hw_sched),
 			    &temp_buff);
 	hw_sched_buff = (struct cudbg_hw_sched *)temp_buff.data;
 	hw_sched_buff->map = t4_read_reg(padap, TP_TX_MOD_QUEUE_REQ_MAP_A);
@@ -649,8 +1145,7 @@ int cudbg_collect_hw_sched(struct cudbg_init *pdbg_init,
 	for (i = 0; i < NTX_SCHED; ++i)
 		t4_get_tx_sched(padap, i, &hw_sched_buff->kbps[i],
 				&hw_sched_buff->ipg[i], true);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_tp_indirect(struct cudbg_init *pdbg_init,
@@ -674,7 +1169,7 @@ int cudbg_collect_tp_indirect(struct cudbg_init *pdbg_init,
 
 	n = n / (IREG_NUM_ELEM * sizeof(u32));
 	size = sizeof(struct ireg_buf) * n;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -763,8 +1258,7 @@ int cudbg_collect_tp_indirect(struct cudbg_init *pdbg_init,
 			       tp_pio->ireg_local_offset, true);
 		ch_tp_pio++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_sge_indirect(struct cudbg_init *pdbg_init,
@@ -776,7 +1270,8 @@ int cudbg_collect_sge_indirect(struct cudbg_init *pdbg_init,
 	struct ireg_buf *ch_sge_dbg;
 	int i, rc;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(*ch_sge_dbg) * 2, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(*ch_sge_dbg) * 2,
+			    &temp_buff);
 	if (rc)
 		return rc;
 
@@ -797,8 +1292,7 @@ int cudbg_collect_sge_indirect(struct cudbg_init *pdbg_init,
 				 sge_pio->ireg_local_offset);
 		ch_sge_dbg++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_ulprx_la(struct cudbg_init *pdbg_init,
@@ -810,7 +1304,7 @@ int cudbg_collect_ulprx_la(struct cudbg_init *pdbg_init,
 	struct cudbg_ulprx_la *ulprx_la_buff;
 	int rc;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_ulprx_la),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_ulprx_la),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -818,8 +1312,7 @@ int cudbg_collect_ulprx_la(struct cudbg_init *pdbg_init,
 	ulprx_la_buff = (struct cudbg_ulprx_la *)temp_buff.data;
 	t4_ulprx_read_la(padap, (u32 *)ulprx_la_buff->data);
 	ulprx_la_buff->size = ULPRX_LA_SIZE;
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_tp_la(struct cudbg_init *pdbg_init,
@@ -832,15 +1325,39 @@ int cudbg_collect_tp_la(struct cudbg_init *pdbg_init,
 	int size, rc;
 
 	size = sizeof(struct cudbg_tp_la) + TPLA_SIZE *  sizeof(u64);
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
 	tp_la_buff = (struct cudbg_tp_la *)temp_buff.data;
 	tp_la_buff->mode = DBGLAMODE_G(t4_read_reg(padap, TP_DBG_LA_CONFIG_A));
 	t4_tp_read_la(padap, (u64 *)tp_la_buff->data, NULL);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
+}
+
+int cudbg_collect_meminfo(struct cudbg_init *pdbg_init,
+			  struct cudbg_buffer *dbg_buff,
+			  struct cudbg_error *cudbg_err)
+{
+	struct adapter *padap = pdbg_init->adap;
+	struct cudbg_buffer temp_buff = { 0 };
+	struct cudbg_meminfo *meminfo_buff;
+	int rc;
+
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_meminfo),
+			    &temp_buff);
+	if (rc)
+		return rc;
+
+	meminfo_buff = (struct cudbg_meminfo *)temp_buff.data;
+	rc = cudbg_fill_meminfo(padap, meminfo_buff);
+	if (rc) {
+		cudbg_err->sys_err = rc;
+		cudbg_put_buff(pdbg_init, &temp_buff);
+		return rc;
+	}
+
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_cim_pif_la(struct cudbg_init *pdbg_init,
@@ -854,7 +1371,7 @@ int cudbg_collect_cim_pif_la(struct cudbg_init *pdbg_init,
 
 	size = sizeof(struct cudbg_cim_pif_la) +
 	       2 * CIM_PIFLA_SIZE * 6 * sizeof(u32);
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -863,8 +1380,7 @@ int cudbg_collect_cim_pif_la(struct cudbg_init *pdbg_init,
 	t4_cim_read_pif_la(padap, (u32 *)cim_pif_la_buff->data,
 			   (u32 *)cim_pif_la_buff->data + 6 * CIM_PIFLA_SIZE,
 			   NULL, NULL);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_clk_info(struct cudbg_init *pdbg_init,
@@ -880,7 +1396,7 @@ int cudbg_collect_clk_info(struct cudbg_init *pdbg_init,
 	if (!padap->params.vpd.cclk)
 		return CUDBG_STATUS_CCLK_NOT_DEFINED;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_clk_info),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_clk_info),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -912,8 +1428,7 @@ int cudbg_collect_clk_info(struct cudbg_init *pdbg_init,
 	clk_info_buff->finwait2_timer =
 		tp_tick_us * t4_read_reg(padap, TP_FINWAIT2_TIMER_A);
 
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_pcie_indirect(struct cudbg_init *pdbg_init,
@@ -928,7 +1443,7 @@ int cudbg_collect_pcie_indirect(struct cudbg_init *pdbg_init,
 
 	n = sizeof(t5_pcie_pdbg_array) / (IREG_NUM_ELEM * sizeof(u32));
 	size = sizeof(struct ireg_buf) * n * 2;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -969,8 +1484,7 @@ int cudbg_collect_pcie_indirect(struct cudbg_init *pdbg_init,
 				 pcie_pio->ireg_local_offset);
 		ch_pcie++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_pm_indirect(struct cudbg_init *pdbg_init,
@@ -985,7 +1499,7 @@ int cudbg_collect_pm_indirect(struct cudbg_init *pdbg_init,
 
 	n = sizeof(t5_pm_rx_array) / (IREG_NUM_ELEM * sizeof(u32));
 	size = sizeof(struct ireg_buf) * n * 2;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -1026,8 +1540,7 @@ int cudbg_collect_pm_indirect(struct cudbg_init *pdbg_init,
 				 pm_pio->ireg_local_offset);
 		ch_pm++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_tid(struct cudbg_init *pdbg_init,
@@ -1041,7 +1554,8 @@ int cudbg_collect_tid(struct cudbg_init *pdbg_init,
 	u32 para[2], val[2];
 	int rc;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_tid_info_region_rev1),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff,
+			    sizeof(struct cudbg_tid_info_region_rev1),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -1053,6 +1567,12 @@ int cudbg_collect_tid(struct cudbg_init *pdbg_init,
 	tid1->ver_hdr.size = sizeof(struct cudbg_tid_info_region_rev1) -
 			     sizeof(struct cudbg_ver_hdr);
 
+	/* If firmware is not attached/alive, use backdoor register
+	 * access to collect dump.
+	 */
+	if (!is_fw_attached(pdbg_init))
+		goto fill_tid;
+
 #define FW_PARAM_PFVF_A(param) \
 	(FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
 	 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param) | \
@@ -1064,7 +1584,7 @@ int cudbg_collect_tid(struct cudbg_init *pdbg_init,
 	rc = t4_query_params(padap, padap->mbox, padap->pf, 0, 2, para, val);
 	if (rc <  0) {
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
 	tid->uotid_base = val[0];
@@ -1083,13 +1603,16 @@ int cudbg_collect_tid(struct cudbg_init *pdbg_init,
 				     para, val);
 		if (rc < 0) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 		tid->hpftid_base = val[0];
 		tid->nhpftids = val[1] - val[0] + 1;
 	}
 
+#undef FW_PARAM_PFVF_A
+
+fill_tid:
 	tid->ntids = padap->tids.ntids;
 	tid->nstids = padap->tids.nstids;
 	tid->stid_base = padap->tids.stid_base;
@@ -1109,28 +1632,137 @@ int cudbg_collect_tid(struct cudbg_init *pdbg_init,
 	tid->ip_users = t4_read_reg(padap, LE_DB_ACT_CNT_IPV4_A);
 	tid->ipv6_users = t4_read_reg(padap, LE_DB_ACT_CNT_IPV6_A);
 
-#undef FW_PARAM_PFVF_A
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
+}
 
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+int cudbg_collect_pcie_config(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err)
+{
+	struct adapter *padap = pdbg_init->adap;
+	struct cudbg_buffer temp_buff = { 0 };
+	u32 size, *value, j;
+	int i, rc, n;
+
+	size = sizeof(u32) * CUDBG_NUM_PCIE_CONFIG_REGS;
+	n = sizeof(t5_pcie_config_array) / (2 * sizeof(u32));
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
+	if (rc)
+		return rc;
+
+	value = (u32 *)temp_buff.data;
+	for (i = 0; i < n; i++) {
+		for (j = t5_pcie_config_array[i][0];
+		     j <= t5_pcie_config_array[i][1]; j += 4) {
+			t4_hw_pci_read_cfg4(padap, j, value);
+			value++;
+		}
+	}
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
-int cudbg_dump_context_size(struct adapter *padap)
+static int cudbg_sge_ctxt_check_valid(u32 *buf, int type)
+{
+	int index, bit, bit_pos = 0;
+
+	switch (type) {
+	case CTXT_EGRESS:
+		bit_pos = 176;
+		break;
+	case CTXT_INGRESS:
+		bit_pos = 141;
+		break;
+	case CTXT_FLM:
+		bit_pos = 89;
+		break;
+	}
+	index = bit_pos / 32;
+	bit =  bit_pos % 32;
+	return buf[index] & (1U << bit);
+}
+
+static int cudbg_get_ctxt_region_info(struct adapter *padap,
+				      struct cudbg_region_info *ctx_info,
+				      u8 *mem_type)
 {
-	u32 value, size;
+	struct cudbg_mem_desc mem_desc;
+	struct cudbg_meminfo meminfo;
+	u32 i, j, value, found;
 	u8 flq;
+	int rc;
+
+	rc = cudbg_fill_meminfo(padap, &meminfo);
+	if (rc)
+		return rc;
+
+	/* Get EGRESS and INGRESS context region size */
+	for (i = CTXT_EGRESS; i <= CTXT_INGRESS; i++) {
+		found = 0;
+		memset(&mem_desc, 0, sizeof(struct cudbg_mem_desc));
+		for (j = 0; j < ARRAY_SIZE(meminfo.avail); j++) {
+			rc = cudbg_get_mem_region(padap, &meminfo, j,
+						  cudbg_region[i],
+						  &mem_desc);
+			if (!rc) {
+				found = 1;
+				rc = cudbg_get_mem_relative(padap, &meminfo, j,
+							    &mem_desc.base,
+							    &mem_desc.limit);
+				if (rc) {
+					ctx_info[i].exist = false;
+					break;
+				}
+				ctx_info[i].exist = true;
+				ctx_info[i].start = mem_desc.base;
+				ctx_info[i].end = mem_desc.limit;
+				mem_type[i] = j;
+				break;
+			}
+		}
+		if (!found)
+			ctx_info[i].exist = false;
+	}
 
+	/* Get FLM and CNM max qid. */
 	value = t4_read_reg(padap, SGE_FLM_CFG_A);
 
 	/* Get number of data freelist queues */
 	flq = HDRSTARTFLQ_G(value);
-	size = CUDBG_MAX_FL_QIDS >> flq;
+	ctx_info[CTXT_FLM].exist = true;
+	ctx_info[CTXT_FLM].end = (CUDBG_MAX_FL_QIDS >> flq) * SGE_CTXT_SIZE;
 
-	/* Add extra space for congestion manager contexts.
-	 * The number of CONM contexts are same as number of freelist
+	/* The number of CONM contexts are same as number of freelist
 	 * queues.
 	 */
-	size += size;
+	ctx_info[CTXT_CNM].exist = true;
+	ctx_info[CTXT_CNM].end = ctx_info[CTXT_FLM].end;
+
+	return 0;
+}
+
+int cudbg_dump_context_size(struct adapter *padap)
+{
+	struct cudbg_region_info region_info[CTXT_CNM + 1] = { {0} };
+	u8 mem_type[CTXT_INGRESS + 1] = { 0 };
+	u32 i, size = 0;
+	int rc;
+
+	/* Get max valid qid for each type of queue */
+	rc = cudbg_get_ctxt_region_info(padap, region_info, mem_type);
+	if (rc)
+		return rc;
+
+	for (i = 0; i < CTXT_CNM; i++) {
+		if (!region_info[i].exist) {
+			if (i == CTXT_EGRESS || i == CTXT_INGRESS)
+				size += CUDBG_LOWMEM_MAX_CTXT_QIDS *
+					SGE_CTXT_SIZE;
+			continue;
+		}
+
+		size += (region_info[i].end - region_info[i].start + 1) /
+			SGE_CTXT_SIZE;
+	}
 	return size * sizeof(struct cudbg_ch_cntxt);
 }
 
@@ -1153,44 +1785,144 @@ static void cudbg_read_sge_ctxt(struct cudbg_init *pdbg_init, u32 cid,
 		t4_sge_ctxt_rd_bd(padap, cid, ctype, data);
 }
 
+static void cudbg_get_sge_ctxt_fw(struct cudbg_init *pdbg_init, u32 max_qid,
+				  u8 ctxt_type,
+				  struct cudbg_ch_cntxt **out_buff)
+{
+	struct cudbg_ch_cntxt *buff = *out_buff;
+	int rc;
+	u32 j;
+
+	for (j = 0; j < max_qid; j++) {
+		cudbg_read_sge_ctxt(pdbg_init, j, ctxt_type, buff->data);
+		rc = cudbg_sge_ctxt_check_valid(buff->data, ctxt_type);
+		if (!rc)
+			continue;
+
+		buff->cntxt_type = ctxt_type;
+		buff->cntxt_id = j;
+		buff++;
+		if (ctxt_type == CTXT_FLM) {
+			cudbg_read_sge_ctxt(pdbg_init, j, CTXT_CNM, buff->data);
+			buff->cntxt_type = CTXT_CNM;
+			buff->cntxt_id = j;
+			buff++;
+		}
+	}
+
+	*out_buff = buff;
+}
+
 int cudbg_collect_dump_context(struct cudbg_init *pdbg_init,
 			       struct cudbg_buffer *dbg_buff,
 			       struct cudbg_error *cudbg_err)
 {
+	struct cudbg_region_info region_info[CTXT_CNM + 1] = { {0} };
 	struct adapter *padap = pdbg_init->adap;
+	u32 j, size, max_ctx_size, max_ctx_qid;
+	u8 mem_type[CTXT_INGRESS + 1] = { 0 };
 	struct cudbg_buffer temp_buff = { 0 };
 	struct cudbg_ch_cntxt *buff;
-	u32 size, i = 0;
+	u64 *dst_off, *src_off;
+	u8 *ctx_buf;
+	u8 i, k;
 	int rc;
 
+	/* Get max valid qid for each type of queue */
+	rc = cudbg_get_ctxt_region_info(padap, region_info, mem_type);
+	if (rc)
+		return rc;
+
 	rc = cudbg_dump_context_size(padap);
 	if (rc <= 0)
 		return CUDBG_STATUS_ENTITY_NOT_FOUND;
 
 	size = rc;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
+	/* Get buffer with enough space to read the biggest context
+	 * region in memory.
+	 */
+	max_ctx_size = max(region_info[CTXT_EGRESS].end -
+			   region_info[CTXT_EGRESS].start + 1,
+			   region_info[CTXT_INGRESS].end -
+			   region_info[CTXT_INGRESS].start + 1);
+
+	ctx_buf = kvzalloc(max_ctx_size, GFP_KERNEL);
+	if (!ctx_buf) {
+		cudbg_put_buff(pdbg_init, &temp_buff);
+		return -ENOMEM;
+	}
+
 	buff = (struct cudbg_ch_cntxt *)temp_buff.data;
-	while (size > 0) {
-		buff->cntxt_type = CTXT_FLM;
-		buff->cntxt_id = i;
-		cudbg_read_sge_ctxt(pdbg_init, i, CTXT_FLM, buff->data);
-		buff++;
-		size -= sizeof(struct cudbg_ch_cntxt);
 
-		buff->cntxt_type = CTXT_CNM;
-		buff->cntxt_id = i;
-		cudbg_read_sge_ctxt(pdbg_init, i, CTXT_CNM, buff->data);
-		buff++;
-		size -= sizeof(struct cudbg_ch_cntxt);
+	/* Collect EGRESS and INGRESS context data.
+	 * In case of failures, fallback to collecting via FW or
+	 * backdoor access.
+	 */
+	for (i = CTXT_EGRESS; i <= CTXT_INGRESS; i++) {
+		if (!region_info[i].exist) {
+			max_ctx_qid = CUDBG_LOWMEM_MAX_CTXT_QIDS;
+			cudbg_get_sge_ctxt_fw(pdbg_init, max_ctx_qid, i,
+					      &buff);
+			continue;
+		}
 
-		i++;
+		max_ctx_size = region_info[i].end - region_info[i].start + 1;
+		max_ctx_qid = max_ctx_size / SGE_CTXT_SIZE;
+
+		/* If firmware is not attached/alive, use backdoor register
+		 * access to collect dump.
+		 */
+		if (is_fw_attached(pdbg_init)) {
+			t4_sge_ctxt_flush(padap, padap->mbox, i);
+
+			rc = t4_memory_rw(padap, MEMWIN_NIC, mem_type[i],
+					  region_info[i].start, max_ctx_size,
+					  (__be32 *)ctx_buf, 1);
+		}
+
+		if (rc || !is_fw_attached(pdbg_init)) {
+			max_ctx_qid = CUDBG_LOWMEM_MAX_CTXT_QIDS;
+			cudbg_get_sge_ctxt_fw(pdbg_init, max_ctx_qid, i,
+					      &buff);
+			continue;
+		}
+
+		for (j = 0; j < max_ctx_qid; j++) {
+			src_off = (u64 *)(ctx_buf + j * SGE_CTXT_SIZE);
+			dst_off = (u64 *)buff->data;
+
+			/* The data is stored in 64-bit cpu order.  Convert it
+			 * to big endian before parsing.
+			 */
+			for (k = 0; k < SGE_CTXT_SIZE / sizeof(u64); k++)
+				dst_off[k] = cpu_to_be64(src_off[k]);
+
+			rc = cudbg_sge_ctxt_check_valid(buff->data, i);
+			if (!rc)
+				continue;
+
+			buff->cntxt_type = i;
+			buff->cntxt_id = j;
+			buff++;
+		}
 	}
 
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	kvfree(ctx_buf);
+
+	/* Collect FREELIST and CONGESTION MANAGER contexts */
+	max_ctx_size = region_info[CTXT_FLM].end -
+		       region_info[CTXT_FLM].start + 1;
+	max_ctx_qid = max_ctx_size / SGE_CTXT_SIZE;
+	/* Since FLM and CONM are 1-to-1 mapped, the below function
+	 * will fetch both FLM and CONM contexts.
+	 */
+	cudbg_get_sge_ctxt_fw(pdbg_init, max_ctx_qid, CTXT_FLM, &buff);
+
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 static inline void cudbg_tcamxy2valmask(u64 x, u64 y, u8 *addr, u64 *mask)
@@ -1228,9 +1960,10 @@ static void cudbg_mps_rpl_backdoor(struct adapter *padap,
 	mps_rplc->rplc31_0 = htonl(t4_read_reg(padap, MPS_VF_RPLCT_MAP0_A));
 }
 
-static int cudbg_collect_tcam_index(struct adapter *padap,
+static int cudbg_collect_tcam_index(struct cudbg_init *pdbg_init,
 				    struct cudbg_mps_tcam *tcam, u32 idx)
 {
+	struct adapter *padap = pdbg_init->adap;
 	u64 tcamy, tcamx, val;
 	u32 ctl, data2;
 	int rc = 0;
@@ -1315,12 +2048,22 @@ static int cudbg_collect_tcam_index(struct adapter *padap,
 			htons(FW_LDST_CMD_FID_V(FW_LDST_MPS_RPLC) |
 			      FW_LDST_CMD_IDX_V(idx));
 
-		rc = t4_wr_mbox(padap, padap->mbox, &ldst_cmd, sizeof(ldst_cmd),
-				&ldst_cmd);
-		if (rc)
+		/* If firmware is not attached/alive, use backdoor register
+		 * access to collect dump.
+		 */
+		if (is_fw_attached(pdbg_init))
+			rc = t4_wr_mbox(padap, padap->mbox, &ldst_cmd,
+					sizeof(ldst_cmd), &ldst_cmd);
+
+		if (rc || !is_fw_attached(pdbg_init)) {
 			cudbg_mps_rpl_backdoor(padap, &mps_rplc);
-		else
+			/* Ignore error since we collected directly from
+			 * reading registers.
+			 */
+			rc = 0;
+		} else {
 			mps_rplc = ldst_cmd.u.mps.rplc;
+		}
 
 		tcam->rplc[0] = ntohl(mps_rplc.rplc31_0);
 		tcam->rplc[1] = ntohl(mps_rplc.rplc63_32);
@@ -1351,16 +2094,16 @@ int cudbg_collect_mps_tcam(struct cudbg_init *pdbg_init,
 
 	n = padap->params.arch.mps_tcam_size;
 	size = sizeof(struct cudbg_mps_tcam) * n;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
 	tcam = (struct cudbg_mps_tcam *)temp_buff.data;
 	for (i = 0; i < n; i++) {
-		rc = cudbg_collect_tcam_index(padap, tcam, i);
+		rc = cudbg_collect_tcam_index(pdbg_init, tcam, i);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 		total_size += sizeof(struct cudbg_mps_tcam);
@@ -1370,11 +2113,10 @@ int cudbg_collect_mps_tcam(struct cudbg_init *pdbg_init,
 	if (!total_size) {
 		rc = CUDBG_SYSTEM_ERROR;
 		cudbg_err->sys_err = rc;
-		cudbg_put_buff(&temp_buff, dbg_buff);
+		cudbg_put_buff(pdbg_init, &temp_buff);
 		return rc;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_vpd_data(struct cudbg_init *pdbg_init,
@@ -1425,7 +2167,7 @@ int cudbg_collect_vpd_data(struct cudbg_init *pdbg_init,
 	if (rc)
 		return rc;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_vpd_data),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_vpd_data),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -1441,8 +2183,7 @@ int cudbg_collect_vpd_data(struct cudbg_init *pdbg_init,
 	vpd_data->fw_minor = FW_HDR_FW_VER_MINOR_G(fw_vers);
 	vpd_data->fw_micro = FW_HDR_FW_VER_MICRO_G(fw_vers);
 	vpd_data->fw_build = FW_HDR_FW_VER_BUILD_G(fw_vers);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 static int cudbg_read_tid(struct cudbg_init *pdbg_init, u32 tid,
@@ -1593,7 +2334,7 @@ int cudbg_collect_le_tcam(struct cudbg_init *pdbg_init,
 
 	size = sizeof(struct cudbg_tid_data) * tcam_region.max_tid;
 	size += sizeof(struct cudbg_tcam);
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -1605,7 +2346,7 @@ int cudbg_collect_le_tcam(struct cudbg_init *pdbg_init,
 		rc = cudbg_read_tid(pdbg_init, i, tid_data);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 
@@ -1616,8 +2357,7 @@ int cudbg_collect_le_tcam(struct cudbg_init *pdbg_init,
 		bytes += sizeof(struct cudbg_tid_data);
 	}
 
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_cctrl(struct cudbg_init *pdbg_init,
@@ -1630,13 +2370,12 @@ int cudbg_collect_cctrl(struct cudbg_init *pdbg_init,
 	int rc;
 
 	size = sizeof(u16) * NMTUS * NCCTRL_WIN;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
 	t4_read_cong_tbl(padap, (void *)temp_buff.data);
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_ma_indirect(struct cudbg_init *pdbg_init,
@@ -1654,7 +2393,7 @@ int cudbg_collect_ma_indirect(struct cudbg_init *pdbg_init,
 
 	n = sizeof(t6_ma_ireg_array) / (IREG_NUM_ELEM * sizeof(u32));
 	size = sizeof(struct ireg_buf) * n * 2;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -1690,8 +2429,7 @@ int cudbg_collect_ma_indirect(struct cudbg_init *pdbg_init,
 		}
 		ma_indr++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_ulptx_la(struct cudbg_init *pdbg_init,
@@ -1704,7 +2442,7 @@ int cudbg_collect_ulptx_la(struct cudbg_init *pdbg_init,
 	u32 i, j;
 	int rc;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_ulptx_la),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, sizeof(struct cudbg_ulptx_la),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -1725,8 +2463,7 @@ int cudbg_collect_ulptx_la(struct cudbg_init *pdbg_init,
 				t4_read_reg(padap,
 					    ULP_TX_LA_RDDATA_0_A + 0x10 * i);
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_up_cim_indirect(struct cudbg_init *pdbg_init,
@@ -1735,13 +2472,23 @@ int cudbg_collect_up_cim_indirect(struct cudbg_init *pdbg_init,
 {
 	struct adapter *padap = pdbg_init->adap;
 	struct cudbg_buffer temp_buff = { 0 };
+	u32 local_offset, local_range;
 	struct ireg_buf *up_cim;
+	u32 size, j, iter;
+	u32 instance = 0;
 	int i, rc, n;
-	u32 size;
 
-	n = sizeof(t5_up_cim_reg_array) / (IREG_NUM_ELEM * sizeof(u32));
+	if (is_t5(padap->params.chip))
+		n = sizeof(t5_up_cim_reg_array) /
+		    ((IREG_NUM_ELEM + 1) * sizeof(u32));
+	else if (is_t6(padap->params.chip))
+		n = sizeof(t6_up_cim_reg_array) /
+		    ((IREG_NUM_ELEM + 1) * sizeof(u32));
+	else
+		return CUDBG_STATUS_NOT_IMPLEMENTED;
+
 	size = sizeof(struct ireg_buf) * n;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -1757,6 +2504,7 @@ int cudbg_collect_up_cim_indirect(struct cudbg_init *pdbg_init,
 						t5_up_cim_reg_array[i][2];
 			up_cim_reg->ireg_offset_range =
 						t5_up_cim_reg_array[i][3];
+			instance = t5_up_cim_reg_array[i][4];
 		} else if (is_t6(padap->params.chip)) {
 			up_cim_reg->ireg_addr = t6_up_cim_reg_array[i][0];
 			up_cim_reg->ireg_data = t6_up_cim_reg_array[i][1];
@@ -1764,18 +2512,39 @@ int cudbg_collect_up_cim_indirect(struct cudbg_init *pdbg_init,
 						t6_up_cim_reg_array[i][2];
 			up_cim_reg->ireg_offset_range =
 						t6_up_cim_reg_array[i][3];
+			instance = t6_up_cim_reg_array[i][4];
 		}
 
-		rc = t4_cim_read(padap, up_cim_reg->ireg_local_offset,
-				 up_cim_reg->ireg_offset_range, buff);
-		if (rc) {
-			cudbg_put_buff(&temp_buff, dbg_buff);
-			return rc;
+		switch (instance) {
+		case NUM_CIM_CTL_TSCH_CHANNEL_INSTANCES:
+			iter = up_cim_reg->ireg_offset_range;
+			local_offset = 0x120;
+			local_range = 1;
+			break;
+		case NUM_CIM_CTL_TSCH_CHANNEL_TSCH_CLASS_INSTANCES:
+			iter = up_cim_reg->ireg_offset_range;
+			local_offset = 0x10;
+			local_range = 1;
+			break;
+		default:
+			iter = 1;
+			local_offset = 0;
+			local_range = up_cim_reg->ireg_offset_range;
+			break;
+		}
+
+		for (j = 0; j < iter; j++, buff++) {
+			rc = t4_cim_read(padap,
+					 up_cim_reg->ireg_local_offset +
+					 (j * local_offset), local_range, buff);
+			if (rc) {
+				cudbg_put_buff(pdbg_init, &temp_buff);
+				return rc;
+			}
 		}
 		up_cim++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init,
@@ -1788,7 +2557,8 @@ int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init,
 	int i, rc;
 	u32 addr;
 
-	rc = cudbg_get_buff(dbg_buff, sizeof(struct cudbg_pbt_tables),
+	rc = cudbg_get_buff(pdbg_init, dbg_buff,
+			    sizeof(struct cudbg_pbt_tables),
 			    &temp_buff);
 	if (rc)
 		return rc;
@@ -1801,7 +2571,7 @@ int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init,
 				 &pbt->pbt_dynamic[i]);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 	}
@@ -1814,7 +2584,7 @@ int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init,
 				 &pbt->pbt_static[i]);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 	}
@@ -1826,7 +2596,7 @@ int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init,
 				 &pbt->lrf_table[i]);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 	}
@@ -1838,12 +2608,11 @@ int cudbg_collect_pbt_tables(struct cudbg_init *pdbg_init,
 				 &pbt->pbt_data[i]);
 		if (rc) {
 			cudbg_err->sys_err = rc;
-			cudbg_put_buff(&temp_buff, dbg_buff);
+			cudbg_put_buff(pdbg_init, &temp_buff);
 			return rc;
 		}
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_mbox_log(struct cudbg_init *pdbg_init,
@@ -1864,7 +2633,7 @@ int cudbg_collect_mbox_log(struct cudbg_init *pdbg_init,
 	log = padap->mbox_log;
 	mbox_cmds = padap->mbox_log->size;
 	size = sizeof(struct cudbg_mbox_log) * mbox_cmds;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -1887,8 +2656,7 @@ int cudbg_collect_mbox_log(struct cudbg_init *pdbg_init,
 		}
 		mboxlog++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
 
 int cudbg_collect_hma_indirect(struct cudbg_init *pdbg_init,
@@ -1906,7 +2674,7 @@ int cudbg_collect_hma_indirect(struct cudbg_init *pdbg_init,
 
 	n = sizeof(t6_hma_ireg_array) / (IREG_NUM_ELEM * sizeof(u32));
 	size = sizeof(struct ireg_buf) * n;
-	rc = cudbg_get_buff(dbg_buff, size, &temp_buff);
+	rc = cudbg_get_buff(pdbg_init, dbg_buff, size, &temp_buff);
 	if (rc)
 		return rc;
 
@@ -1924,6 +2692,5 @@ int cudbg_collect_hma_indirect(struct cudbg_init *pdbg_init,
 				 hma_fli->ireg_local_offset);
 		hma_indr++;
 	}
-	cudbg_write_and_release_buff(&temp_buff, dbg_buff);
-	return rc;
+	return cudbg_write_and_release_buff(pdbg_init, &temp_buff, dbg_buff);
 }
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h
index caeee8e33e86..eebefe7cd18e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.h
@@ -75,6 +75,12 @@ int cudbg_collect_edc0_meminfo(struct cudbg_init *pdbg_init,
 int cudbg_collect_edc1_meminfo(struct cudbg_init *pdbg_init,
 			       struct cudbg_buffer *dbg_buff,
 			       struct cudbg_error *cudbg_err);
+int cudbg_collect_mc0_meminfo(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err);
+int cudbg_collect_mc1_meminfo(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err);
 int cudbg_collect_rss(struct cudbg_init *pdbg_init,
 		      struct cudbg_buffer *dbg_buff,
 		      struct cudbg_error *cudbg_err);
@@ -102,6 +108,9 @@ int cudbg_collect_ulprx_la(struct cudbg_init *pdbg_init,
 int cudbg_collect_tp_la(struct cudbg_init *pdbg_init,
 			struct cudbg_buffer *dbg_buff,
 			struct cudbg_error *cudbg_err);
+int cudbg_collect_meminfo(struct cudbg_init *pdbg_init,
+			  struct cudbg_buffer *dbg_buff,
+			  struct cudbg_error *cudbg_err);
 int cudbg_collect_cim_pif_la(struct cudbg_init *pdbg_init,
 			     struct cudbg_buffer *dbg_buff,
 			     struct cudbg_error *cudbg_err);
@@ -123,6 +132,9 @@ int cudbg_collect_pm_indirect(struct cudbg_init *pdbg_init,
 int cudbg_collect_tid(struct cudbg_init *pdbg_init,
 		      struct cudbg_buffer *dbg_buff,
 		      struct cudbg_error *cudbg_err);
+int cudbg_collect_pcie_config(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err);
 int cudbg_collect_dump_context(struct cudbg_init *pdbg_init,
 			       struct cudbg_buffer *dbg_buff,
 			       struct cudbg_error *cudbg_err);
@@ -156,6 +168,9 @@ int cudbg_collect_mbox_log(struct cudbg_init *pdbg_init,
 int cudbg_collect_hma_indirect(struct cudbg_init *pdbg_init,
 			       struct cudbg_buffer *dbg_buff,
 			       struct cudbg_error *cudbg_err);
+int cudbg_collect_hma_meminfo(struct cudbg_init *pdbg_init,
+			      struct cudbg_buffer *dbg_buff,
+			      struct cudbg_error *cudbg_err);
 
 struct cudbg_entity_hdr *cudbg_get_entity_hdr(void *outbuf, int i);
 void cudbg_align_debug_buffer(struct cudbg_buffer *dbg_buff,
@@ -163,7 +178,8 @@ void cudbg_align_debug_buffer(struct cudbg_buffer *dbg_buff,
 u32 cudbg_cim_obq_size(struct adapter *padap, int qid);
 int cudbg_dump_context_size(struct adapter *padap);
 
-struct cudbg_tcam;
+int cudbg_fill_meminfo(struct adapter *padap,
+		       struct cudbg_meminfo *meminfo_buff);
 void cudbg_fill_le_tcam_info(struct adapter *padap,
 			     struct cudbg_tcam *tcam_region);
 #endif /* __CUDBG_LIB_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib_common.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib_common.h
index 24b33f28e548..8150ea85d6a5 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib_common.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib_common.h
@@ -26,6 +26,7 @@ enum cudbg_dump_type {
 
 enum cudbg_compression_type {
 	CUDBG_COMPRESSION_NONE = 1,
+	CUDBG_COMPRESSION_ZLIB,
 };
 
 struct cudbg_hdr {
@@ -78,10 +79,11 @@ struct cudbg_error {
 #define CDUMP_MAX_COMP_BUF_SIZE ((64 * 1024) - 1)
 #define CUDBG_CHUNK_SIZE ((CDUMP_MAX_COMP_BUF_SIZE / 1024) * 1024)
 
-int cudbg_get_buff(struct cudbg_buffer *pdbg_buff, u32 size,
+int cudbg_get_buff(struct cudbg_init *pdbg_init,
+		   struct cudbg_buffer *pdbg_buff, u32 size,
 		   struct cudbg_buffer *pin_buff);
-void cudbg_put_buff(struct cudbg_buffer *pin_buff,
-		    struct cudbg_buffer *pdbg_buff);
+void cudbg_put_buff(struct cudbg_init *pdbg_init,
+		    struct cudbg_buffer *pin_buff);
 void cudbg_update_buff(struct cudbg_buffer *pin_buff,
 		       struct cudbg_buffer *pout_buff);
 #endif /* __CUDBG_LIB_COMMON_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c
new file mode 100644
index 000000000000..25cc06d75cff
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.c
@@ -0,0 +1,82 @@
+/*
+ *  Copyright (C) 2018 Chelsio Communications.  All rights reserved.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms and conditions of the GNU General Public License,
+ *  version 2, as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ *  more details.
+ *
+ *  The full GNU General Public License is included in this distribution in
+ *  the file called "COPYING".
+ *
+ */
+
+#include <linux/zlib.h>
+
+#include "cxgb4.h"
+#include "cudbg_if.h"
+#include "cudbg_lib_common.h"
+#include "cudbg_zlib.h"
+
+static int cudbg_get_compress_hdr(struct cudbg_buffer *pdbg_buff,
+				  struct cudbg_buffer *pin_buff)
+{
+	if (pdbg_buff->offset + sizeof(struct cudbg_compress_hdr) >
+	    pdbg_buff->size)
+		return CUDBG_STATUS_NO_MEM;
+
+	pin_buff->data = (char *)pdbg_buff->data + pdbg_buff->offset;
+	pin_buff->offset = 0;
+	pin_buff->size = sizeof(struct cudbg_compress_hdr);
+	pdbg_buff->offset += sizeof(struct cudbg_compress_hdr);
+	return 0;
+}
+
+int cudbg_compress_buff(struct cudbg_init *pdbg_init,
+			struct cudbg_buffer *pin_buff,
+			struct cudbg_buffer *pout_buff)
+{
+	struct cudbg_buffer temp_buff = { 0 };
+	struct z_stream_s compress_stream;
+	struct cudbg_compress_hdr *c_hdr;
+	int rc;
+
+	/* Write compression header to output buffer before compression */
+	rc = cudbg_get_compress_hdr(pout_buff, &temp_buff);
+	if (rc)
+		return rc;
+
+	c_hdr = (struct cudbg_compress_hdr *)temp_buff.data;
+	c_hdr->compress_id = CUDBG_ZLIB_COMPRESS_ID;
+
+	memset(&compress_stream, 0, sizeof(struct z_stream_s));
+	compress_stream.workspace = pdbg_init->workspace;
+	rc = zlib_deflateInit2(&compress_stream, Z_DEFAULT_COMPRESSION,
+			       Z_DEFLATED, CUDBG_ZLIB_WIN_BITS,
+			       CUDBG_ZLIB_MEM_LVL, Z_DEFAULT_STRATEGY);
+	if (rc != Z_OK)
+		return CUDBG_SYSTEM_ERROR;
+
+	compress_stream.next_in = pin_buff->data;
+	compress_stream.avail_in = pin_buff->size;
+	compress_stream.next_out = pout_buff->data + pout_buff->offset;
+	compress_stream.avail_out = pout_buff->size - pout_buff->offset;
+
+	rc = zlib_deflate(&compress_stream, Z_FINISH);
+	if (rc != Z_STREAM_END)
+		return CUDBG_SYSTEM_ERROR;
+
+	rc = zlib_deflateEnd(&compress_stream);
+	if (rc != Z_OK)
+		return CUDBG_SYSTEM_ERROR;
+
+	c_hdr->compress_size = compress_stream.total_out;
+	c_hdr->decompress_size = pin_buff->size;
+	pout_buff->offset += compress_stream.total_out;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h
new file mode 100644
index 000000000000..60d23805dfc3
--- /dev/null
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_zlib.h
@@ -0,0 +1,43 @@
+/*
+ *  Copyright (C) 2018 Chelsio Communications.  All rights reserved.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms and conditions of the GNU General Public License,
+ *  version 2, as published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ *  more details.
+ *
+ *  The full GNU General Public License is included in this distribution in
+ *  the file called "COPYING".
+ *
+ */
+
+#ifndef __CUDBG_ZLIB_H__
+#define __CUDBG_ZLIB_H__
+
+#include <linux/zlib.h>
+
+#define CUDBG_ZLIB_COMPRESS_ID 17
+#define CUDBG_ZLIB_WIN_BITS 12
+#define CUDBG_ZLIB_MEM_LVL 4
+
+struct cudbg_compress_hdr {
+	u32 compress_id;
+	u64 decompress_size;
+	u64 compress_size;
+	u64 rsvd[32];
+};
+
+static inline int cudbg_get_workspace_size(void)
+{
+	return zlib_deflate_workspacesize(CUDBG_ZLIB_WIN_BITS,
+					  CUDBG_ZLIB_MEM_LVL);
+}
+
+int cudbg_compress_buff(struct cudbg_init *pdbg_init,
+			struct cudbg_buffer *pin_buff,
+			struct cudbg_buffer *pout_buff);
+#endif /* __CUDBG_ZLIB_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
index d8424ed16c33..9040e13ce4b7 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h
@@ -58,6 +58,13 @@
 extern struct list_head adapter_list;
 extern struct mutex uld_mutex;
 
+/* Suspend an Ethernet Tx queue with fewer available descriptors than this.
+ * This is the same as calc_tx_descs() for a TSO packet with
+ * nr_frags == MAX_SKB_FRAGS.
+ */
+#define ETHTXQ_STOP_THRES \
+	(1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8))
+
 enum {
 	MAX_NPORTS	= 4,     /* max # of ports */
 	SERNUM_LEN	= 24,    /* Serial # length */
@@ -77,7 +84,8 @@ enum {
 	MEM_EDC1,
 	MEM_MC,
 	MEM_MC0 = MEM_MC,
-	MEM_MC1
+	MEM_MC1,
+	MEM_HMA,
 };
 
 enum {
@@ -311,6 +319,7 @@ struct vpd_params {
 };
 
 struct pci_params {
+	unsigned int vpd_cap_addr;
 	unsigned char speed;
 	unsigned char width;
 };
@@ -563,6 +572,7 @@ enum {                                 /* adapter flags */
 
 enum {
 	ULP_CRYPTO_LOOKASIDE = 1 << 0,
+	ULP_CRYPTO_IPSEC_INLINE = 1 << 1,
 };
 
 struct rx_sw_desc;
@@ -818,12 +828,17 @@ struct vf_info {
 	unsigned char vf_mac_addr[ETH_ALEN];
 	unsigned int tx_rate;
 	bool pf_set_mac;
+	u16 vlan;
 };
 
 struct mbox_list {
 	struct list_head list;
 };
 
+struct mps_encap_entry {
+	atomic_t refcnt;
+};
+
 struct adapter {
 	void __iomem *regs;
 	void __iomem *bar2;
@@ -838,6 +853,10 @@ struct adapter {
 	enum chip_type chip;
 
 	int msg_enable;
+	__be16 vxlan_port;
+	u8 vxlan_port_cnt;
+	__be16 geneve_port;
+	u8 geneve_port_cnt;
 
 	struct adapter_params params;
 	struct cxgb4_virt_res vres;
@@ -867,7 +886,10 @@ struct adapter {
 	unsigned int clipt_start;
 	unsigned int clipt_end;
 	struct clip_tbl *clipt;
+	unsigned int rawf_start;
+	unsigned int rawf_cnt;
 	struct smt_data *smt;
+	struct mps_encap_entry *mps_encap;
 	struct cxgb4_uld_info *uld;
 	void *uld_handle[CXGB4_ULD_MAX];
 	unsigned int num_uld;
@@ -967,6 +989,11 @@ enum {
 	SCHED_CLASS_RATEMODE_ABS = 1,   /* Kb/s */
 };
 
+struct tx_sw_desc {                /* SW state per Tx descriptor */
+	struct sk_buff *skb;
+	struct ulptx_sgl *sgl;
+};
+
 /* Support for "sched_queue" command to allow one or more NIC TX Queues
  * to be bound to a TX Scheduling Class.
  */
@@ -1304,6 +1331,7 @@ void t4_sge_start(struct adapter *adap);
 void t4_sge_stop(struct adapter *adap);
 void cxgb4_set_ethtool_ops(struct net_device *netdev);
 int cxgb4_write_rss(const struct port_info *pi, const u16 *queues);
+enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb);
 extern int dbfifo_int_thresh;
 
 #define for_each_port(adapter, iter) \
@@ -1422,6 +1450,21 @@ static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
 	q->size = size;
 }
 
+/**
+ *     t4_is_inserted_mod_type - is a plugged in Firmware Module Type
+ *     @fw_mod_type: the Firmware Mofule Type
+ *
+ *     Return whether the Firmware Module Type represents a real Transceiver
+ *     Module/Cable Module Type which has been inserted.
+ */
+static inline bool t4_is_inserted_mod_type(unsigned int fw_mod_type)
+{
+	return (fw_mod_type != FW_PORT_MOD_TYPE_NONE &&
+		fw_mod_type != FW_PORT_MOD_TYPE_NOTSUPPORTED &&
+		fw_mod_type != FW_PORT_MOD_TYPE_UNKNOWN &&
+		fw_mod_type != FW_PORT_MOD_TYPE_ERROR);
+}
+
 void t4_write_indirect(struct adapter *adap, unsigned int addr_reg,
 		       unsigned int data_reg, const u32 *vals,
 		       unsigned int nregs, unsigned int start_idx);
@@ -1511,6 +1554,7 @@ int t4_init_portinfo(struct port_info *pi, int mbox,
 		     int port, int pf, int vf, u8 mac[]);
 int t4_port_init(struct adapter *adap, int mbox, int pf, int vf);
 void t4_fatal_err(struct adapter *adapter);
+unsigned int t4_chip_rss_size(struct adapter *adapter);
 int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
 			int start, int n, const u16 *rspq, unsigned int nrspq);
 int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
@@ -1620,6 +1664,12 @@ int t4_free_vi(struct adapter *adap, unsigned int mbox,
 int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 		int mtu, int promisc, int all_multi, int bcast, int vlanex,
 		bool sleep_ok);
+int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			 const u8 *addr, const u8 *mask, unsigned int idx,
+			 u8 lookup_type, u8 port_id, bool sleep_ok);
+int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			  const u8 *addr, const u8 *mask, unsigned int idx,
+			  u8 lookup_type, u8 port_id, bool sleep_ok);
 int t4_alloc_mac_filt(struct adapter *adap, unsigned int mbox,
 		      unsigned int viid, bool free, unsigned int naddr,
 		      const u8 **addr, u16 *idx, u64 *hash, bool sleep_ok);
@@ -1652,7 +1702,7 @@ int t4_ctrl_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int eqid);
 int t4_ofld_eq_free(struct adapter *adap, unsigned int mbox, unsigned int pf,
 		    unsigned int vf, unsigned int eqid);
-int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox);
+int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox, int ctxt_type);
 void t4_handle_get_port_info(struct port_info *pi, const __be64 *rpl);
 int t4_update_port_info(struct port_info *pi);
 int t4_get_link_params(struct port_info *pi, unsigned int *link_okp,
@@ -1695,8 +1745,23 @@ void t4_uld_mem_free(struct adapter *adap);
 int t4_uld_mem_alloc(struct adapter *adap);
 void t4_uld_clean_up(struct adapter *adap);
 void t4_register_netevent_notifier(void);
+int t4_i2c_rd(struct adapter *adap, unsigned int mbox, int port,
+	      unsigned int devid, unsigned int offset,
+	      unsigned int len, u8 *buf);
 void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, struct sge_fl *fl);
 void free_tx_desc(struct adapter *adap, struct sge_txq *q,
 		  unsigned int n, bool unmap);
 void free_txq(struct adapter *adap, struct sge_txq *q);
+void cxgb4_reclaim_completed_tx(struct adapter *adap,
+				struct sge_txq *q, bool unmap);
+int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
+		  dma_addr_t *addr);
+void cxgb4_inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
+			 void *pos);
+void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
+		     struct ulptx_sgl *sgl, u64 *end, unsigned int start,
+		     const dma_addr_t *addr);
+void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n);
+int t4_set_vlan_acl(struct adapter *adap, unsigned int mbox, unsigned int vf,
+		    u16 vlan);
 #endif /* __CXGB4_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
index 29cc625e9833..143686c60234 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.c
@@ -18,11 +18,14 @@
 #include "t4_regs.h"
 #include "cxgb4.h"
 #include "cxgb4_cudbg.h"
-#include "cudbg_entity.h"
+#include "cudbg_zlib.h"
 
 static const struct cxgb4_collect_entity cxgb4_collect_mem_dump[] = {
 	{ CUDBG_EDC0, cudbg_collect_edc0_meminfo },
 	{ CUDBG_EDC1, cudbg_collect_edc1_meminfo },
+	{ CUDBG_MC0, cudbg_collect_mc0_meminfo },
+	{ CUDBG_MC1, cudbg_collect_mc1_meminfo },
+	{ CUDBG_HMA, cudbg_collect_hma_meminfo },
 };
 
 static const struct cxgb4_collect_entity cxgb4_collect_hw_dump[] = {
@@ -53,6 +56,7 @@ static const struct cxgb4_collect_entity cxgb4_collect_hw_dump[] = {
 	{ CUDBG_SGE_INDIRECT, cudbg_collect_sge_indirect },
 	{ CUDBG_ULPRX_LA, cudbg_collect_ulprx_la },
 	{ CUDBG_TP_LA, cudbg_collect_tp_la },
+	{ CUDBG_MEMINFO, cudbg_collect_meminfo },
 	{ CUDBG_CIM_PIF_LA, cudbg_collect_cim_pif_la },
 	{ CUDBG_CLK, cudbg_collect_clk_info },
 	{ CUDBG_CIM_OBQ_RXQ0, cudbg_collect_obq_sge_rx_q0 },
@@ -60,6 +64,7 @@ static const struct cxgb4_collect_entity cxgb4_collect_hw_dump[] = {
 	{ CUDBG_PCIE_INDIRECT, cudbg_collect_pcie_indirect },
 	{ CUDBG_PM_INDIRECT, cudbg_collect_pm_indirect },
 	{ CUDBG_TID_INFO, cudbg_collect_tid },
+	{ CUDBG_PCIE_CONFIG, cudbg_collect_pcie_config },
 	{ CUDBG_DUMP_CONTEXT, cudbg_collect_dump_context },
 	{ CUDBG_MPS_TCAM, cudbg_collect_mps_tcam },
 	{ CUDBG_VPD_DATA, cudbg_collect_vpd_data },
@@ -97,7 +102,7 @@ static u32 cxgb4_get_entity_length(struct adapter *adap, u32 entity)
 	case CUDBG_CIM_LA:
 		if (is_t6(adap->params.chip)) {
 			len = adap->params.cim_la_size / 10 + 1;
-			len *= 11 * sizeof(u32);
+			len *= 10 * sizeof(u32);
 		} else {
 			len = adap->params.cim_la_size / 8;
 			len *= 8 * sizeof(u32);
@@ -158,8 +163,24 @@ static u32 cxgb4_get_entity_length(struct adapter *adap, u32 entity)
 		}
 		len = cudbg_mbytes_to_bytes(len);
 		break;
+	case CUDBG_MC0:
+		value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A);
+		if (value & EXT_MEM0_ENABLE_F) {
+			value = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
+			len = EXT_MEM0_SIZE_G(value);
+		}
+		len = cudbg_mbytes_to_bytes(len);
+		break;
+	case CUDBG_MC1:
+		value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A);
+		if (value & EXT_MEM1_ENABLE_F) {
+			value = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
+			len = EXT_MEM1_SIZE_G(value);
+		}
+		len = cudbg_mbytes_to_bytes(len);
+		break;
 	case CUDBG_RSS:
-		len = RSS_NENTRIES * sizeof(u16);
+		len = t4_chip_rss_size(adap) * sizeof(u16);
 		break;
 	case CUDBG_RSS_VF_CONF:
 		len = adap->params.arch.vfcount *
@@ -201,6 +222,9 @@ static u32 cxgb4_get_entity_length(struct adapter *adap, u32 entity)
 	case CUDBG_TP_LA:
 		len = sizeof(struct cudbg_tp_la) + TPLA_SIZE * sizeof(u64);
 		break;
+	case CUDBG_MEMINFO:
+		len = sizeof(struct cudbg_meminfo);
+		break;
 	case CUDBG_CIM_PIF_LA:
 		len = sizeof(struct cudbg_cim_pif_la);
 		len += 2 * CIM_PIFLA_SIZE * 6 * sizeof(u32);
@@ -219,6 +243,9 @@ static u32 cxgb4_get_entity_length(struct adapter *adap, u32 entity)
 	case CUDBG_TID_INFO:
 		len = sizeof(struct cudbg_tid_info_region_rev1);
 		break;
+	case CUDBG_PCIE_CONFIG:
+		len = sizeof(u32) * CUDBG_NUM_PCIE_CONFIG_REGS;
+		break;
 	case CUDBG_DUMP_CONTEXT:
 		len = cudbg_dump_context_size(adap);
 		break;
@@ -248,7 +275,13 @@ static u32 cxgb4_get_entity_length(struct adapter *adap, u32 entity)
 		len = sizeof(struct cudbg_ulptx_la);
 		break;
 	case CUDBG_UP_CIM_INDIRECT:
-		n = sizeof(t5_up_cim_reg_array) / (IREG_NUM_ELEM * sizeof(u32));
+		n = 0;
+		if (is_t5(adap->params.chip))
+			n = sizeof(t5_up_cim_reg_array) /
+			    ((IREG_NUM_ELEM + 1) * sizeof(u32));
+		else if (is_t6(adap->params.chip))
+			n = sizeof(t6_up_cim_reg_array) /
+			    ((IREG_NUM_ELEM + 1) * sizeof(u32));
 		len = sizeof(struct ireg_buf) * n;
 		break;
 	case CUDBG_PBT_TABLE:
@@ -264,6 +297,17 @@ static u32 cxgb4_get_entity_length(struct adapter *adap, u32 entity)
 			len = sizeof(struct ireg_buf) * n;
 		}
 		break;
+	case CUDBG_HMA:
+		value = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A);
+		if (value & HMA_MUX_F) {
+			/* In T6, there's no MC1.  So, HMA shares MC1
+			 * address space.
+			 */
+			value = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
+			len = EXT_MEM1_SIZE_G(value);
+		}
+		len = cudbg_mbytes_to_bytes(len);
+		break;
 	default:
 		break;
 	}
@@ -275,6 +319,7 @@ u32 cxgb4_get_dump_length(struct adapter *adap, u32 flag)
 {
 	u32 i, entity;
 	u32 len = 0;
+	u32 wsize;
 
 	if (flag & CXGB4_ETH_DUMP_HW) {
 		for (i = 0; i < ARRAY_SIZE(cxgb4_collect_hw_dump); i++) {
@@ -290,6 +335,11 @@ u32 cxgb4_get_dump_length(struct adapter *adap, u32 flag)
 		}
 	}
 
+	/* If compression is enabled, a smaller destination buffer is enough */
+	wsize = cudbg_get_workspace_size();
+	if (wsize && len > CUDBG_DUMP_BUFF_SIZE)
+		len = CUDBG_DUMP_BUFF_SIZE;
+
 	return len;
 }
 
@@ -298,22 +348,14 @@ static void cxgb4_cudbg_collect_entity(struct cudbg_init *pdbg_init,
 				       const struct cxgb4_collect_entity *e_arr,
 				       u32 arr_size, void *buf, u32 *tot_size)
 {
-	struct adapter *adap = pdbg_init->adap;
 	struct cudbg_error cudbg_err = { 0 };
 	struct cudbg_entity_hdr *entity_hdr;
-	u32 entity_size, i;
-	u32 total_size = 0;
+	u32 i, total_size = 0;
 	int ret;
 
 	for (i = 0; i < arr_size; i++) {
 		const struct cxgb4_collect_entity *e = &e_arr[i];
 
-		/* Skip entities that won't fit in output buffer */
-		entity_size = cxgb4_get_entity_length(adap, e->entity);
-		if (entity_size >
-		    pdbg_init->outbuf_size - *tot_size - total_size)
-			continue;
-
 		entity_hdr = cudbg_get_entity_hdr(buf, e->entity);
 		entity_hdr->entity_type = e->entity;
 		entity_hdr->start_offset = dbg_buff->offset;
@@ -339,16 +381,40 @@ static void cxgb4_cudbg_collect_entity(struct cudbg_init *pdbg_init,
 	*tot_size += total_size;
 }
 
+static int cudbg_alloc_compress_buff(struct cudbg_init *pdbg_init)
+{
+	u32 workspace_size;
+
+	workspace_size = cudbg_get_workspace_size();
+	pdbg_init->compress_buff = vzalloc(CUDBG_COMPRESS_BUFF_SIZE +
+					   workspace_size);
+	if (!pdbg_init->compress_buff)
+		return -ENOMEM;
+
+	pdbg_init->compress_buff_size = CUDBG_COMPRESS_BUFF_SIZE;
+	pdbg_init->workspace = (u8 *)pdbg_init->compress_buff +
+			       CUDBG_COMPRESS_BUFF_SIZE - workspace_size;
+	return 0;
+}
+
+static void cudbg_free_compress_buff(struct cudbg_init *pdbg_init)
+{
+	if (pdbg_init->compress_buff)
+		vfree(pdbg_init->compress_buff);
+}
+
 int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size,
 			u32 flag)
 {
-	struct cudbg_init cudbg_init = { 0 };
 	struct cudbg_buffer dbg_buff = { 0 };
 	u32 size, min_size, total_size = 0;
+	struct cudbg_init cudbg_init;
 	struct cudbg_hdr *cudbg_hdr;
+	int rc;
 
 	size = *buf_size;
 
+	memset(&cudbg_init, 0, sizeof(struct cudbg_init));
 	cudbg_init.adap = adap;
 	cudbg_init.outbuf = buf;
 	cudbg_init.outbuf_size = size;
@@ -365,7 +431,6 @@ int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size,
 	cudbg_hdr->max_entities = CUDBG_MAX_ENTITY;
 	cudbg_hdr->chip_ver = adap->params.chip;
 	cudbg_hdr->dump_type = CUDBG_DUMP_TYPE_MINI;
-	cudbg_hdr->compress_type = CUDBG_COMPRESSION_NONE;
 
 	min_size = sizeof(struct cudbg_hdr) +
 		   sizeof(struct cudbg_entity_hdr) *
@@ -373,6 +438,24 @@ int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size,
 	if (size < min_size)
 		return -ENOMEM;
 
+	rc = cudbg_get_workspace_size();
+	if (rc) {
+		/* Zlib available.  So, use zlib deflate */
+		cudbg_init.compress_type = CUDBG_COMPRESSION_ZLIB;
+		rc = cudbg_alloc_compress_buff(&cudbg_init);
+		if (rc) {
+			/* Ignore error and continue without compression. */
+			dev_warn(adap->pdev_dev,
+				 "Fail allocating compression buffer ret: %d.  Continuing without compression.\n",
+				 rc);
+			cudbg_init.compress_type = CUDBG_COMPRESSION_NONE;
+			rc = 0;
+		}
+	} else {
+		cudbg_init.compress_type = CUDBG_COMPRESSION_NONE;
+	}
+
+	cudbg_hdr->compress_type = cudbg_init.compress_type;
 	dbg_buff.offset += min_size;
 	total_size = dbg_buff.offset;
 
@@ -390,8 +473,12 @@ int cxgb4_cudbg_collect(struct adapter *adap, void *buf, u32 *buf_size,
 					   buf,
 					   &total_size);
 
+	cudbg_free_compress_buff(&cudbg_init);
 	cudbg_hdr->data_len = total_size;
-	*buf_size = total_size;
+	if (cudbg_init.compress_type != CUDBG_COMPRESSION_NONE)
+		*buf_size = size;
+	else
+		*buf_size = total_size;
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
index c099b5aa2214..ce1ac9a1c878 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_cudbg.h
@@ -20,8 +20,12 @@
 
 #include "cudbg_if.h"
 #include "cudbg_lib_common.h"
+#include "cudbg_entity.h"
 #include "cudbg_lib.h"
 
+#define CUDBG_DUMP_BUFF_SIZE (32 * 1024 * 1024) /* 32 MB */
+#define CUDBG_COMPRESS_BUFF_SIZE (4 * 1024 * 1024) /* 4 MB */
+
 typedef int (*cudbg_collect_callback_t)(struct cudbg_init *pdbg_init,
 					struct cudbg_buffer *dbg_buff,
 					struct cudbg_error *cudbg_err);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
index 917663b35603..2822bbff73e8 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c
@@ -45,6 +45,10 @@
 #include "cxgb4_debugfs.h"
 #include "clip_tbl.h"
 #include "l2t.h"
+#include "cudbg_if.h"
+#include "cudbg_lib_common.h"
+#include "cudbg_entity.h"
+#include "cudbg_lib.h"
 
 /* generic seq_file support for showing a table of size rows x width. */
 static void *seq_tab_get_idx(struct seq_tab *tb, loff_t pos)
@@ -1739,7 +1743,7 @@ static int mps_tcam_show(struct seq_file *seq, void *v)
 			 */
 			if (lookup_type && (lookup_type != DATALKPTYPE_M)) {
 				/* Inner header VNI */
-				vniy = ((data2 & DATAVIDH2_F) << 23) |
+				vniy = (data2 & DATAVIDH2_F) |
 				       (DATAVIDH1_G(data2) << 16) | VIDL_G(val);
 				dip_hit = data2 & DATADIPHIT_F;
 			} else {
@@ -1749,6 +1753,7 @@ static int mps_tcam_show(struct seq_file *seq, void *v)
 			port_num = DATAPORTNUM_G(data2);
 
 			/* Read tcamx. Change the control param */
+			vnix = 0;
 			ctl |= CTLXYBITSEL_V(1);
 			t4_write_reg(adap, MPS_CLS_TCAM_DATA2_CTL_A, ctl);
 			val = t4_read_reg(adap, MPS_CLS_TCAM_DATA1_A);
@@ -1757,7 +1762,7 @@ static int mps_tcam_show(struct seq_file *seq, void *v)
 			data2 = t4_read_reg(adap, MPS_CLS_TCAM_DATA2_CTL_A);
 			if (lookup_type && (lookup_type != DATALKPTYPE_M)) {
 				/* Inner header VNI mask */
-				vnix = ((data2 & DATAVIDH2_F) << 23) |
+				vnix = (data2 & DATAVIDH2_F) |
 				       (DATAVIDH1_G(data2) << 16) | VIDL_G(val);
 			}
 		} else {
@@ -1830,7 +1835,8 @@ static int mps_tcam_show(struct seq_file *seq, void *v)
 					   addr[1], addr[2], addr[3],
 					   addr[4], addr[5],
 					   (unsigned long long)mask,
-					   vniy, vnix, dip_hit ? 'Y' : 'N',
+					   vniy, (vnix | vniy),
+					   dip_hit ? 'Y' : 'N',
 					   port_num,
 					   (cls_lo & T6_SRAM_VLD_F) ? 'Y' : 'N',
 					   PORTMAP_G(cls_hi),
@@ -2017,11 +2023,12 @@ static int rss_show(struct seq_file *seq, void *v, int idx)
 
 static int rss_open(struct inode *inode, struct file *file)
 {
-	int ret;
-	struct seq_tab *p;
 	struct adapter *adap = inode->i_private;
+	int ret, nentries;
+	struct seq_tab *p;
 
-	p = seq_open_tab(file, RSS_NENTRIES / 8, 8 * sizeof(u16), 0, rss_show);
+	nentries = t4_chip_rss_size(adap);
+	p = seq_open_tab(file, nentries / 8, 8 * sizeof(u16), 0, rss_show);
 	if (!p)
 		return -ENOMEM;
 
@@ -2664,10 +2671,14 @@ static const struct file_operations mem_debugfs_fops = {
 
 static int tid_info_show(struct seq_file *seq, void *v)
 {
+	unsigned int tid_start = 0;
 	struct adapter *adap = seq->private;
 	const struct tid_info *t = &adap->tids;
 	enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip);
 
+	if (chip > CHELSIO_T5)
+		tid_start = t4_read_reg(adap, LE_DB_ACTIVE_TABLE_START_INDEX_A);
+
 	if (t4_read_reg(adap, LE_DB_CONFIG_A) & HASHEN_F) {
 		unsigned int sb;
 		seq_printf(seq, "Connections in use: %u\n",
@@ -2679,8 +2690,8 @@ static int tid_info_show(struct seq_file *seq, void *v)
 			sb = t4_read_reg(adap, LE_DB_SRVR_START_INDEX_A);
 
 		if (sb) {
-			seq_printf(seq, "TID range: 0..%u/%u..%u", sb - 1,
-				   adap->tids.hash_base,
+			seq_printf(seq, "TID range: %u..%u/%u..%u", tid_start,
+				   sb - 1, adap->tids.hash_base,
 				   t->ntids - 1);
 			seq_printf(seq, ", in use: %u/%u\n",
 				   atomic_read(&t->tids_in_use),
@@ -2705,7 +2716,8 @@ static int tid_info_show(struct seq_file *seq, void *v)
 		seq_printf(seq, "Connections in use: %u\n",
 			   atomic_read(&t->conns_in_use));
 
-		seq_printf(seq, "TID range: 0..%u", t->ntids - 1);
+		seq_printf(seq, "TID range: %u..%u", tid_start,
+			   tid_start + t->ntids - 1);
 		seq_printf(seq, ", in use: %u\n",
 			   atomic_read(&t->tids_in_use));
 	}
@@ -2794,18 +2806,6 @@ static const struct file_operations blocked_fl_fops = {
 	.llseek  = generic_file_llseek,
 };
 
-struct mem_desc {
-	unsigned int base;
-	unsigned int limit;
-	unsigned int idx;
-};
-
-static int mem_desc_cmp(const void *a, const void *b)
-{
-	return ((const struct mem_desc *)a)->base -
-	       ((const struct mem_desc *)b)->base;
-}
-
 static void mem_region_show(struct seq_file *seq, const char *name,
 			    unsigned int from, unsigned int to)
 {
@@ -2819,250 +2819,60 @@ static void mem_region_show(struct seq_file *seq, const char *name,
 static int meminfo_show(struct seq_file *seq, void *v)
 {
 	static const char * const memory[] = { "EDC0:", "EDC1:", "MC:",
-					"MC0:", "MC1:"};
-	static const char * const region[] = {
-		"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
-		"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
-		"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
-		"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
-		"RQUDP region:", "PBL region:", "TXPBL region:",
-		"DBVFIFO region:", "ULPRX state:", "ULPTX state:",
-		"On-chip queues:"
-	};
-
-	int i, n;
-	u32 lo, hi, used, alloc;
-	struct mem_desc avail[4];
-	struct mem_desc mem[ARRAY_SIZE(region) + 3];      /* up to 3 holes */
-	struct mem_desc *md = mem;
+					       "MC0:", "MC1:", "HMA:"};
 	struct adapter *adap = seq->private;
+	struct cudbg_meminfo meminfo;
+	int i, rc;
 
-	for (i = 0; i < ARRAY_SIZE(mem); i++) {
-		mem[i].limit = 0;
-		mem[i].idx = i;
-	}
-
-	/* Find and sort the populated memory ranges */
-	i = 0;
-	lo = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A);
-	if (lo & EDRAM0_ENABLE_F) {
-		hi = t4_read_reg(adap, MA_EDRAM0_BAR_A);
-		avail[i].base = EDRAM0_BASE_G(hi) << 20;
-		avail[i].limit = avail[i].base + (EDRAM0_SIZE_G(hi) << 20);
-		avail[i].idx = 0;
-		i++;
-	}
-	if (lo & EDRAM1_ENABLE_F) {
-		hi = t4_read_reg(adap, MA_EDRAM1_BAR_A);
-		avail[i].base = EDRAM1_BASE_G(hi) << 20;
-		avail[i].limit = avail[i].base + (EDRAM1_SIZE_G(hi) << 20);
-		avail[i].idx = 1;
-		i++;
-	}
-
-	if (is_t5(adap->params.chip)) {
-		if (lo & EXT_MEM0_ENABLE_F) {
-			hi = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
-			avail[i].base = EXT_MEM0_BASE_G(hi) << 20;
-			avail[i].limit =
-				avail[i].base + (EXT_MEM0_SIZE_G(hi) << 20);
-			avail[i].idx = 3;
-			i++;
-		}
-		if (lo & EXT_MEM1_ENABLE_F) {
-			hi = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
-			avail[i].base = EXT_MEM1_BASE_G(hi) << 20;
-			avail[i].limit =
-				avail[i].base + (EXT_MEM1_SIZE_G(hi) << 20);
-			avail[i].idx = 4;
-			i++;
-		}
-	} else {
-		if (lo & EXT_MEM_ENABLE_F) {
-			hi = t4_read_reg(adap, MA_EXT_MEMORY_BAR_A);
-			avail[i].base = EXT_MEM_BASE_G(hi) << 20;
-			avail[i].limit =
-				avail[i].base + (EXT_MEM_SIZE_G(hi) << 20);
-			avail[i].idx = 2;
-			i++;
-		}
-	}
-	if (!i)                                    /* no memory available */
-		return 0;
-	sort(avail, i, sizeof(struct mem_desc), mem_desc_cmp, NULL);
-
-	(md++)->base = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A);
-	(md++)->base = t4_read_reg(adap, SGE_IMSG_CTXT_BADDR_A);
-	(md++)->base = t4_read_reg(adap, SGE_FLM_CACHE_BADDR_A);
-	(md++)->base = t4_read_reg(adap, TP_CMM_TCB_BASE_A);
-	(md++)->base = t4_read_reg(adap, TP_CMM_MM_BASE_A);
-	(md++)->base = t4_read_reg(adap, TP_CMM_TIMER_BASE_A);
-	(md++)->base = t4_read_reg(adap, TP_CMM_MM_RX_FLST_BASE_A);
-	(md++)->base = t4_read_reg(adap, TP_CMM_MM_TX_FLST_BASE_A);
-	(md++)->base = t4_read_reg(adap, TP_CMM_MM_PS_FLST_BASE_A);
-
-	/* the next few have explicit upper bounds */
-	md->base = t4_read_reg(adap, TP_PMM_TX_BASE_A);
-	md->limit = md->base - 1 +
-		    t4_read_reg(adap, TP_PMM_TX_PAGE_SIZE_A) *
-		    PMTXMAXPAGE_G(t4_read_reg(adap, TP_PMM_TX_MAX_PAGE_A));
-	md++;
-
-	md->base = t4_read_reg(adap, TP_PMM_RX_BASE_A);
-	md->limit = md->base - 1 +
-		    t4_read_reg(adap, TP_PMM_RX_PAGE_SIZE_A) *
-		    PMRXMAXPAGE_G(t4_read_reg(adap, TP_PMM_RX_MAX_PAGE_A));
-	md++;
-
-	if (t4_read_reg(adap, LE_DB_CONFIG_A) & HASHEN_F) {
-		if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) {
-			hi = t4_read_reg(adap, LE_DB_TID_HASHBASE_A) / 4;
-			md->base = t4_read_reg(adap, LE_DB_HASH_TID_BASE_A);
-		 } else {
-			hi = t4_read_reg(adap, LE_DB_HASH_TID_BASE_A);
-			md->base = t4_read_reg(adap,
-					       LE_DB_HASH_TBL_BASE_ADDR_A);
-		}
-		md->limit = 0;
-	} else {
-		md->base = 0;
-		md->idx = ARRAY_SIZE(region);  /* hide it */
-	}
-	md++;
-
-#define ulp_region(reg) do { \
-	md->base = t4_read_reg(adap, ULP_ ## reg ## _LLIMIT_A);\
-	(md++)->limit = t4_read_reg(adap, ULP_ ## reg ## _ULIMIT_A); \
-} while (0)
-
-	ulp_region(RX_ISCSI);
-	ulp_region(RX_TDDP);
-	ulp_region(TX_TPT);
-	ulp_region(RX_STAG);
-	ulp_region(RX_RQ);
-	ulp_region(RX_RQUDP);
-	ulp_region(RX_PBL);
-	ulp_region(TX_PBL);
-#undef ulp_region
-	md->base = 0;
-	md->idx = ARRAY_SIZE(region);
-	if (!is_t4(adap->params.chip)) {
-		u32 size = 0;
-		u32 sge_ctrl = t4_read_reg(adap, SGE_CONTROL2_A);
-		u32 fifo_size = t4_read_reg(adap, SGE_DBVFIFO_SIZE_A);
-
-		if (is_t5(adap->params.chip)) {
-			if (sge_ctrl & VFIFO_ENABLE_F)
-				size = DBVFIFO_SIZE_G(fifo_size);
-		} else {
-			size = T6_DBVFIFO_SIZE_G(fifo_size);
-		}
-
-		if (size) {
-			md->base = BASEADDR_G(t4_read_reg(adap,
-					SGE_DBVFIFO_BADDR_A));
-			md->limit = md->base + (size << 2) - 1;
-		}
-	}
-
-	md++;
-
-	md->base = t4_read_reg(adap, ULP_RX_CTX_BASE_A);
-	md->limit = 0;
-	md++;
-	md->base = t4_read_reg(adap, ULP_TX_ERR_TABLE_BASE_A);
-	md->limit = 0;
-	md++;
-
-	md->base = adap->vres.ocq.start;
-	if (adap->vres.ocq.size)
-		md->limit = md->base + adap->vres.ocq.size - 1;
-	else
-		md->idx = ARRAY_SIZE(region);  /* hide it */
-	md++;
-
-	/* add any address-space holes, there can be up to 3 */
-	for (n = 0; n < i - 1; n++)
-		if (avail[n].limit < avail[n + 1].base)
-			(md++)->base = avail[n].limit;
-	if (avail[n].limit)
-		(md++)->base = avail[n].limit;
-
-	n = md - mem;
-	sort(mem, n, sizeof(struct mem_desc), mem_desc_cmp, NULL);
+	memset(&meminfo, 0, sizeof(struct cudbg_meminfo));
+	rc = cudbg_fill_meminfo(adap, &meminfo);
+	if (rc)
+		return -ENXIO;
 
-	for (lo = 0; lo < i; lo++)
-		mem_region_show(seq, memory[avail[lo].idx], avail[lo].base,
-				avail[lo].limit - 1);
+	for (i = 0; i < meminfo.avail_c; i++)
+		mem_region_show(seq, memory[meminfo.avail[i].idx],
+				meminfo.avail[i].base,
+				meminfo.avail[i].limit - 1);
 
 	seq_putc(seq, '\n');
-	for (i = 0; i < n; i++) {
-		if (mem[i].idx >= ARRAY_SIZE(region))
+	for (i = 0; i < meminfo.mem_c; i++) {
+		if (meminfo.mem[i].idx >= ARRAY_SIZE(cudbg_region))
 			continue;                        /* skip holes */
-		if (!mem[i].limit)
-			mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
-		mem_region_show(seq, region[mem[i].idx], mem[i].base,
-				mem[i].limit);
+		if (!meminfo.mem[i].limit)
+			meminfo.mem[i].limit =
+				i < meminfo.mem_c - 1 ?
+				meminfo.mem[i + 1].base - 1 : ~0;
+		mem_region_show(seq, cudbg_region[meminfo.mem[i].idx],
+				meminfo.mem[i].base, meminfo.mem[i].limit);
 	}
 
 	seq_putc(seq, '\n');
-	lo = t4_read_reg(adap, CIM_SDRAM_BASE_ADDR_A);
-	hi = t4_read_reg(adap, CIM_SDRAM_ADDR_SIZE_A) + lo - 1;
-	mem_region_show(seq, "uP RAM:", lo, hi);
-
-	lo = t4_read_reg(adap, CIM_EXTMEM2_BASE_ADDR_A);
-	hi = t4_read_reg(adap, CIM_EXTMEM2_ADDR_SIZE_A) + lo - 1;
-	mem_region_show(seq, "uP Extmem2:", lo, hi);
+	mem_region_show(seq, "uP RAM:", meminfo.up_ram_lo, meminfo.up_ram_hi);
+	mem_region_show(seq, "uP Extmem2:", meminfo.up_extmem2_lo,
+			meminfo.up_extmem2_hi);
 
-	lo = t4_read_reg(adap, TP_PMM_RX_MAX_PAGE_A);
 	seq_printf(seq, "\n%u Rx pages of size %uKiB for %u channels\n",
-		   PMRXMAXPAGE_G(lo),
-		   t4_read_reg(adap, TP_PMM_RX_PAGE_SIZE_A) >> 10,
-		   (lo & PMRXNUMCHN_F) ? 2 : 1);
+		   meminfo.rx_pages_data[0], meminfo.rx_pages_data[1],
+		   meminfo.rx_pages_data[2]);
 
-	lo = t4_read_reg(adap, TP_PMM_TX_MAX_PAGE_A);
-	hi = t4_read_reg(adap, TP_PMM_TX_PAGE_SIZE_A);
 	seq_printf(seq, "%u Tx pages of size %u%ciB for %u channels\n",
-		   PMTXMAXPAGE_G(lo),
-		   hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
-		   hi >= (1 << 20) ? 'M' : 'K', 1 << PMTXNUMCHN_G(lo));
-	seq_printf(seq, "%u p-structs\n\n",
-		   t4_read_reg(adap, TP_CMM_MM_MAX_PSTRUCT_A));
-
-	for (i = 0; i < 4; i++) {
-		if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5)
-			lo = t4_read_reg(adap, MPS_RX_MAC_BG_PG_CNT0_A + i * 4);
-		else
-			lo = t4_read_reg(adap, MPS_RX_PG_RSV0_A + i * 4);
-		if (is_t5(adap->params.chip)) {
-			used = T5_USED_G(lo);
-			alloc = T5_ALLOC_G(lo);
-		} else {
-			used = USED_G(lo);
-			alloc = ALLOC_G(lo);
-		}
+		   meminfo.tx_pages_data[0], meminfo.tx_pages_data[1],
+		   meminfo.tx_pages_data[2], meminfo.tx_pages_data[3]);
+
+	seq_printf(seq, "%u p-structs\n\n", meminfo.p_structs);
+
+	for (i = 0; i < 4; i++)
 		/* For T6 these are MAC buffer groups */
 		seq_printf(seq, "Port %d using %u pages out of %u allocated\n",
-			   i, used, alloc);
-	}
-	for (i = 0; i < adap->params.arch.nchan; i++) {
-		if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5)
-			lo = t4_read_reg(adap,
-					 MPS_RX_LPBK_BG_PG_CNT0_A + i * 4);
-		else
-			lo = t4_read_reg(adap, MPS_RX_PG_RSV4_A + i * 4);
-		if (is_t5(adap->params.chip)) {
-			used = T5_USED_G(lo);
-			alloc = T5_ALLOC_G(lo);
-		} else {
-			used = USED_G(lo);
-			alloc = ALLOC_G(lo);
-		}
+			   i, meminfo.port_used[i], meminfo.port_alloc[i]);
+
+	for (i = 0; i < adap->params.arch.nchan; i++)
 		/* For T6 these are MAC buffer groups */
 		seq_printf(seq,
 			   "Loopback %d using %u pages out of %u allocated\n",
-			   i, used, alloc);
-	}
+			   i, meminfo.loopback_used[i],
+			   meminfo.loopback_alloc[i]);
+
 	return 0;
 }
 
@@ -3096,6 +2906,8 @@ static int chcr_show(struct seq_file *seq, void *v)
 		   atomic_read(&adap->chcr_stats.error));
 	seq_printf(seq, "Fallback: %10u \n",
 		   atomic_read(&adap->chcr_stats.fallback));
+	seq_printf(seq, "IPSec PDU: %10u\n",
+		   atomic_read(&adap->chcr_stats.ipsec_cnt));
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
index eb338212f5af..7852d98bad75 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
@@ -517,7 +517,8 @@ static int from_fw_port_mod_type(enum fw_port_type port_type,
 		else
 			return PORT_OTHER;
 	} else if (port_type == FW_PORT_TYPE_KR4_100G ||
-		   port_type == FW_PORT_TYPE_KR_SFP28) {
+		   port_type == FW_PORT_TYPE_KR_SFP28 ||
+		   port_type == FW_PORT_TYPE_KR_XLAUI) {
 		return PORT_NONE;
 	}
 
@@ -645,6 +646,13 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 		FW_CAPS_TO_LMM(SPEED_25G, 25000baseKR_Full);
 		break;
 
+	case FW_PORT_TYPE_KR_XLAUI:
+		SET_LMM(Backplane);
+		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
+		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
+		FW_CAPS_TO_LMM(SPEED_40G, 40000baseKR4_Full);
+		break;
+
 	case FW_PORT_TYPE_CR2_QSFP:
 		SET_LMM(FIBRE);
 		SET_LMM(50000baseSR2_Full);
@@ -1396,6 +1404,101 @@ static int get_dump_data(struct net_device *dev, struct ethtool_dump *eth_dump,
 	return 0;
 }
 
+static int cxgb4_get_module_info(struct net_device *dev,
+				 struct ethtool_modinfo *modinfo)
+{
+	struct port_info *pi = netdev_priv(dev);
+	u8 sff8472_comp, sff_diag_type, sff_rev;
+	struct adapter *adapter = pi->adapter;
+	int ret;
+
+	if (!t4_is_inserted_mod_type(pi->mod_type))
+		return -EINVAL;
+
+	switch (pi->port_type) {
+	case FW_PORT_TYPE_SFP:
+	case FW_PORT_TYPE_QSA:
+	case FW_PORT_TYPE_SFP28:
+		ret = t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan,
+				I2C_DEV_ADDR_A0, SFF_8472_COMP_ADDR,
+				SFF_8472_COMP_LEN, &sff8472_comp);
+		if (ret)
+			return ret;
+		ret = t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan,
+				I2C_DEV_ADDR_A0, SFP_DIAG_TYPE_ADDR,
+				SFP_DIAG_TYPE_LEN, &sff_diag_type);
+		if (ret)
+			return ret;
+
+		if (!sff8472_comp || (sff_diag_type & 4)) {
+			modinfo->type = ETH_MODULE_SFF_8079;
+			modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
+		} else {
+			modinfo->type = ETH_MODULE_SFF_8472;
+			modinfo->eeprom_len = ETH_MODULE_SFF_8472_LEN;
+		}
+		break;
+
+	case FW_PORT_TYPE_QSFP:
+	case FW_PORT_TYPE_QSFP_10G:
+	case FW_PORT_TYPE_CR_QSFP:
+	case FW_PORT_TYPE_CR2_QSFP:
+	case FW_PORT_TYPE_CR4_QSFP:
+		ret = t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan,
+				I2C_DEV_ADDR_A0, SFF_REV_ADDR,
+				SFF_REV_LEN, &sff_rev);
+		/* For QSFP type ports, revision value >= 3
+		 * means the SFP is 8636 compliant.
+		 */
+		if (ret)
+			return ret;
+		if (sff_rev >= 0x3) {
+			modinfo->type = ETH_MODULE_SFF_8636;
+			modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN;
+		} else {
+			modinfo->type = ETH_MODULE_SFF_8436;
+			modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN;
+		}
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int cxgb4_get_module_eeprom(struct net_device *dev,
+				   struct ethtool_eeprom *eprom, u8 *data)
+{
+	int ret = 0, offset = eprom->offset, len = eprom->len;
+	struct port_info *pi = netdev_priv(dev);
+	struct adapter *adapter = pi->adapter;
+
+	memset(data, 0, eprom->len);
+	if (offset + len <= I2C_PAGE_SIZE)
+		return t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan,
+				 I2C_DEV_ADDR_A0, offset, len, data);
+
+	/* offset + len spans 0xa0 and 0xa1 pages */
+	if (offset <= I2C_PAGE_SIZE) {
+		/* read 0xa0 page */
+		len = I2C_PAGE_SIZE - offset;
+		ret =  t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan,
+				 I2C_DEV_ADDR_A0, offset, len, data);
+		if (ret)
+			return ret;
+		offset = I2C_PAGE_SIZE;
+		/* Remaining bytes to be read from second page =
+		 * Total length - bytes read from first page
+		 */
+		len = eprom->len - len;
+	}
+	/* Read additional optical diagnostics from page 0xa2 if supported */
+	return t4_i2c_rd(adapter, adapter->mbox, pi->tx_chan, I2C_DEV_ADDR_A2,
+			 offset, len, &data[eprom->len - len]);
+}
+
 static const struct ethtool_ops cxgb_ethtool_ops = {
 	.get_link_ksettings = get_link_ksettings,
 	.set_link_ksettings = set_link_ksettings,
@@ -1430,6 +1533,8 @@ static const struct ethtool_ops cxgb_ethtool_ops = {
 	.set_dump          = set_dump,
 	.get_dump_flag     = get_dump_flag,
 	.get_dump_data     = get_dump_data,
+	.get_module_info   = cxgb4_get_module_info,
+	.get_module_eeprom = cxgb4_get_module_eeprom,
 };
 
 void cxgb4_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
index 5980f308a253..3177b0c9bd2d 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_filter.c
@@ -439,19 +439,32 @@ int cxgb4_get_free_ftid(struct net_device *dev, int family)
 		if (ftid >= t->nftids)
 			ftid = -1;
 	} else {
-		ftid = bitmap_find_free_region(t->ftid_bmap, t->nftids, 2);
-		if (ftid < 0)
-			goto out_unlock;
+		if (is_t6(adap->params.chip)) {
+			ftid = bitmap_find_free_region(t->ftid_bmap,
+						       t->nftids, 1);
+			if (ftid < 0)
+				goto out_unlock;
+
+			/* this is only a lookup, keep the found region
+			 * unallocated
+			 */
+			bitmap_release_region(t->ftid_bmap, ftid, 1);
+		} else {
+			ftid = bitmap_find_free_region(t->ftid_bmap,
+						       t->nftids, 2);
+			if (ftid < 0)
+				goto out_unlock;
 
-		/* this is only a lookup, keep the found region unallocated */
-		bitmap_release_region(t->ftid_bmap, ftid, 2);
+			bitmap_release_region(t->ftid_bmap, ftid, 2);
+		}
 	}
 out_unlock:
 	spin_unlock_bh(&t->ftid_lock);
 	return ftid;
 }
 
-static int cxgb4_set_ftid(struct tid_info *t, int fidx, int family)
+static int cxgb4_set_ftid(struct tid_info *t, int fidx, int family,
+			  unsigned int chip_ver)
 {
 	spin_lock_bh(&t->ftid_lock);
 
@@ -460,22 +473,31 @@ static int cxgb4_set_ftid(struct tid_info *t, int fidx, int family)
 		return -EBUSY;
 	}
 
-	if (family == PF_INET)
+	if (family == PF_INET) {
 		__set_bit(fidx, t->ftid_bmap);
-	else
-		bitmap_allocate_region(t->ftid_bmap, fidx, 2);
+	} else {
+		if (chip_ver < CHELSIO_T6)
+			bitmap_allocate_region(t->ftid_bmap, fidx, 2);
+		else
+			bitmap_allocate_region(t->ftid_bmap, fidx, 1);
+	}
 
 	spin_unlock_bh(&t->ftid_lock);
 	return 0;
 }
 
-static void cxgb4_clear_ftid(struct tid_info *t, int fidx, int family)
+static void cxgb4_clear_ftid(struct tid_info *t, int fidx, int family,
+			     unsigned int chip_ver)
 {
 	spin_lock_bh(&t->ftid_lock);
-	if (family == PF_INET)
+	if (family == PF_INET) {
 		__clear_bit(fidx, t->ftid_bmap);
-	else
-		bitmap_release_region(t->ftid_bmap, fidx, 2);
+	} else {
+		if (chip_ver < CHELSIO_T6)
+			bitmap_release_region(t->ftid_bmap, fidx, 2);
+		else
+			bitmap_release_region(t->ftid_bmap, fidx, 1);
+	}
 	spin_unlock_bh(&t->ftid_lock);
 }
 
@@ -694,7 +716,7 @@ void clear_filter(struct adapter *adap, struct filter_entry *f)
 	if (f->smt)
 		cxgb4_smt_release(f->smt);
 
-	if (f->fs.hash && f->fs.type)
+	if ((f->fs.hash || is_t6(adap->params.chip)) && f->fs.type)
 		cxgb4_clip_release(f->dev, (const u32 *)&f->fs.val.lip, 1);
 
 	/* The zeroing of the filter rule below clears the filter valid,
@@ -1189,6 +1211,7 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 		       struct filter_ctx *ctx)
 {
 	struct adapter *adapter = netdev2adap(dev);
+	unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
 	unsigned int max_fidx, fidx;
 	struct filter_entry *f;
 	u32 iconf;
@@ -1225,12 +1248,18 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 	 * insertion.
 	 */
 	if (fs->type == 0) { /* IPv4 */
-		/* If our IPv4 filter isn't being written to a
-		 * multiple of four filter index and there's an IPv6
-		 * filter at the multiple of 4 base slot, then we
-		 * prevent insertion.
+		/* For T6, If our IPv4 filter isn't being written to a
+		 * multiple of two filter index and there's an IPv6
+		 * filter at the multiple of 2 base slot, then we need
+		 * to delete that IPv6 filter ...
+		 * For adapters below T6, IPv6 filter occupies 4 entries.
+		 * Hence we need to delete the filter in multiple of 4 slot.
 		 */
-		fidx = filter_id & ~0x3;
+		if (chip_ver < CHELSIO_T6)
+			fidx = filter_id & ~0x3;
+		else
+			fidx = filter_id & ~0x1;
+
 		if (fidx != filter_id &&
 		    adapter->tids.ftid_tab[fidx].fs.type) {
 			f = &adapter->tids.ftid_tab[fidx];
@@ -1242,23 +1271,42 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 			}
 		}
 	} else { /* IPv6 */
-		/* Ensure that the IPv6 filter is aligned on a
-		 * multiple of 4 boundary.
-		 */
-		if (filter_id & 0x3) {
-			dev_err(adapter->pdev_dev,
-				"Invalid location. IPv6 must be aligned on a 4-slot boundary\n");
-			return -EINVAL;
-		}
+		if (chip_ver < CHELSIO_T6) {
+			/* Ensure that the IPv6 filter is aligned on a
+			 * multiple of 4 boundary.
+			 */
+			if (filter_id & 0x3) {
+				dev_err(adapter->pdev_dev,
+					"Invalid location. IPv6 must be aligned on a 4-slot boundary\n");
+				return -EINVAL;
+			}
 
-		/* Check all except the base overlapping IPv4 filter slots. */
-		for (fidx = filter_id + 1; fidx < filter_id + 4; fidx++) {
+			/* Check all except the base overlapping IPv4 filter
+			 * slots.
+			 */
+			for (fidx = filter_id + 1; fidx < filter_id + 4;
+			     fidx++) {
+				f = &adapter->tids.ftid_tab[fidx];
+				if (f->valid) {
+					dev_err(adapter->pdev_dev,
+						"Invalid location.  IPv6 requires 4 slots and an IPv4 filter exists at %u\n",
+						fidx);
+					return -EBUSY;
+				}
+			}
+		} else {
+			/* For T6, CLIP being enabled, IPv6 filter would occupy
+			 * 2 entries.
+			 */
+			if (filter_id & 0x1)
+				return -EINVAL;
+			/* Check overlapping IPv4 filter slot */
+			fidx = filter_id + 1;
 			f = &adapter->tids.ftid_tab[fidx];
 			if (f->valid) {
-				dev_err(adapter->pdev_dev,
-					"Invalid location.  IPv6 requires 4 slots and an IPv4 filter exists at %u\n",
-					fidx);
-				return -EINVAL;
+				pr_err("%s: IPv6 filter requires 2 indices. IPv4 filter already present at %d. Please remove IPv4 filter first.\n",
+				       __func__, fidx);
+				return -EBUSY;
 			}
 		}
 	}
@@ -1272,16 +1320,18 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 
 	fidx = filter_id + adapter->tids.ftid_base;
 	ret = cxgb4_set_ftid(&adapter->tids, filter_id,
-			     fs->type ? PF_INET6 : PF_INET);
+			     fs->type ? PF_INET6 : PF_INET,
+			     chip_ver);
 	if (ret)
 		return ret;
 
-	/* Check to make sure the filter requested is writable ... */
+	/* Check t  make sure the filter requested is writable ... */
 	ret = writable_filter(f);
 	if (ret) {
 		/* Clear the bits we have set above */
 		cxgb4_clear_ftid(&adapter->tids, filter_id,
-				 fs->type ? PF_INET6 : PF_INET);
+				 fs->type ? PF_INET6 : PF_INET,
+				 chip_ver);
 		return ret;
 	}
 
@@ -1291,6 +1341,17 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 	if (f->valid)
 		clear_filter(adapter, f);
 
+	if (is_t6(adapter->params.chip) && fs->type &&
+	    ipv6_addr_type((const struct in6_addr *)fs->val.lip) !=
+	    IPV6_ADDR_ANY) {
+		ret = cxgb4_clip_get(dev, (const u32 *)&fs->val.lip, 1);
+		if (ret) {
+			cxgb4_clear_ftid(&adapter->tids, filter_id, PF_INET6,
+					 chip_ver);
+			return ret;
+		}
+	}
+
 	/* Convert the filter specification into our internal format.
 	 * We copy the PF/VF specification into the Outer VLAN field
 	 * here so the rest of the code -- including the interface to
@@ -1316,7 +1377,8 @@ int __cxgb4_set_filter(struct net_device *dev, int filter_id,
 	ret = set_filter_wr(adapter, filter_id);
 	if (ret) {
 		cxgb4_clear_ftid(&adapter->tids, filter_id,
-				 fs->type ? PF_INET6 : PF_INET);
+				 fs->type ? PF_INET6 : PF_INET,
+				 chip_ver);
 		clear_filter(adapter, f);
 	}
 
@@ -1394,6 +1456,7 @@ int __cxgb4_del_filter(struct net_device *dev, int filter_id,
 		       struct filter_ctx *ctx)
 {
 	struct adapter *adapter = netdev2adap(dev);
+	unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
 	struct filter_entry *f;
 	unsigned int max_fidx;
 	int ret;
@@ -1419,7 +1482,8 @@ int __cxgb4_del_filter(struct net_device *dev, int filter_id,
 	if (f->valid) {
 		f->ctx = ctx;
 		cxgb4_clear_ftid(&adapter->tids, filter_id,
-				 f->fs.type ? PF_INET6 : PF_INET);
+				 f->fs.type ? PF_INET6 : PF_INET,
+				 chip_ver);
 		return del_filter_wr(adapter, filter_id);
 	}
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 6f900ffe25cc..61022b5f6743 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -65,6 +65,7 @@
 #include <net/addrconf.h>
 #include <linux/uaccess.h>
 #include <linux/crash_dump.h>
+#include <net/udp_tunnel.h>
 
 #include "cxgb4.h"
 #include "cxgb4_filter.h"
@@ -101,7 +102,9 @@ const char cxgb4_driver_version[] = DRV_VERSION;
  */
 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
 	static const struct pci_device_id cxgb4_pci_tbl[] = {
-#define CH_PCI_DEVICE_ID_FUNCTION 0x4
+#define CXGB4_UNIFIED_PF 0x4
+
+#define CH_PCI_DEVICE_ID_FUNCTION CXGB4_UNIFIED_PF
 
 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
  * called for both.
@@ -109,7 +112,7 @@ const char cxgb4_driver_version[] = DRV_VERSION;
 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0
 
 #define CH_PCI_ID_TABLE_ENTRY(devid) \
-		{PCI_VDEVICE(CHELSIO, (devid)), 4}
+		{PCI_VDEVICE(CHELSIO, (devid)), CXGB4_UNIFIED_PF}
 
 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
 		{ 0, } \
@@ -1673,7 +1676,7 @@ int cxgb4_flush_eq_cache(struct net_device *dev)
 {
 	struct adapter *adap = netdev2adap(dev);
 
-	return t4_sge_ctxt_flush(adap, adap->mbox);
+	return t4_sge_ctxt_flush(adap, adap->mbox, CTXT_EGRESS);
 }
 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
 
@@ -2604,7 +2607,7 @@ static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
 }
 
 #ifdef CONFIG_PCI_IOV
-static int dummy_open(struct net_device *dev)
+static int cxgb4_mgmt_open(struct net_device *dev)
 {
 	/* Turn carrier off since we don't have to transmit anything on this
 	 * interface.
@@ -2614,39 +2617,44 @@ static int dummy_open(struct net_device *dev)
 }
 
 /* Fill MAC address that will be assigned by the FW */
-static void fill_vf_station_mac_addr(struct adapter *adap)
+static void cxgb4_mgmt_fill_vf_station_mac_addr(struct adapter *adap)
 {
-	unsigned int i;
 	u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN];
+	unsigned int i, vf, nvfs;
+	u16 a, b;
 	int err;
 	u8 *na;
-	u16 a, b;
 
+	adap->params.pci.vpd_cap_addr = pci_find_capability(adap->pdev,
+							    PCI_CAP_ID_VPD);
 	err = t4_get_raw_vpd_params(adap, &adap->params.vpd);
-	if (!err) {
-		na = adap->params.vpd.na;
-		for (i = 0; i < ETH_ALEN; i++)
-			hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 +
-				      hex2val(na[2 * i + 1]));
-		a = (hw_addr[0] << 8) | hw_addr[1];
-		b = (hw_addr[1] << 8) | hw_addr[2];
-		a ^= b;
-		a |= 0x0200;    /* locally assigned Ethernet MAC address */
-		a &= ~0x0100;   /* not a multicast Ethernet MAC address */
-		macaddr[0] = a >> 8;
-		macaddr[1] = a & 0xff;
-
-		for (i = 2; i < 5; i++)
-			macaddr[i] = hw_addr[i + 1];
-
-		for (i = 0; i < adap->num_vfs; i++) {
-			macaddr[5] = adap->pf * 16 + i;
-			ether_addr_copy(adap->vfinfo[i].vf_mac_addr, macaddr);
-		}
+	if (err)
+		return;
+
+	na = adap->params.vpd.na;
+	for (i = 0; i < ETH_ALEN; i++)
+		hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 +
+			      hex2val(na[2 * i + 1]));
+
+	a = (hw_addr[0] << 8) | hw_addr[1];
+	b = (hw_addr[1] << 8) | hw_addr[2];
+	a ^= b;
+	a |= 0x0200;    /* locally assigned Ethernet MAC address */
+	a &= ~0x0100;   /* not a multicast Ethernet MAC address */
+	macaddr[0] = a >> 8;
+	macaddr[1] = a & 0xff;
+
+	for (i = 2; i < 5; i++)
+		macaddr[i] = hw_addr[i + 1];
+
+	for (vf = 0, nvfs = pci_sriov_get_totalvfs(adap->pdev);
+		vf < nvfs; vf++) {
+		macaddr[5] = adap->pf * 16 + vf;
+		ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, macaddr);
 	}
 }
 
-static int cxgb_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
+static int cxgb4_mgmt_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
 {
 	struct port_info *pi = netdev_priv(dev);
 	struct adapter *adap = pi->adapter;
@@ -2668,8 +2676,8 @@ static int cxgb_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
 	return ret;
 }
 
-static int cxgb_get_vf_config(struct net_device *dev,
-			      int vf, struct ifla_vf_info *ivi)
+static int cxgb4_mgmt_get_vf_config(struct net_device *dev,
+				    int vf, struct ifla_vf_info *ivi)
 {
 	struct port_info *pi = netdev_priv(dev);
 	struct adapter *adap = pi->adapter;
@@ -2683,8 +2691,8 @@ static int cxgb_get_vf_config(struct net_device *dev,
 	return 0;
 }
 
-static int cxgb_get_phys_port_id(struct net_device *dev,
-				 struct netdev_phys_item_id *ppid)
+static int cxgb4_mgmt_get_phys_port_id(struct net_device *dev,
+				       struct netdev_phys_item_id *ppid)
 {
 	struct port_info *pi = netdev_priv(dev);
 	unsigned int phy_port_id;
@@ -2695,8 +2703,8 @@ static int cxgb_get_phys_port_id(struct net_device *dev,
 	return 0;
 }
 
-static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
-			    int max_tx_rate)
+static int cxgb4_mgmt_set_vf_rate(struct net_device *dev, int vf,
+				  int min_tx_rate, int max_tx_rate)
 {
 	struct port_info *pi = netdev_priv(dev);
 	struct adapter *adap = pi->adapter;
@@ -2775,7 +2783,30 @@ static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
 	return 0;
 }
 
-#endif
+static int cxgb4_mgmt_set_vf_vlan(struct net_device *dev, int vf,
+				  u16 vlan, u8 qos, __be16 vlan_proto)
+{
+	struct port_info *pi = netdev_priv(dev);
+	struct adapter *adap = pi->adapter;
+	int ret;
+
+	if (vf >= adap->num_vfs || vlan > 4095 || qos > 7)
+		return -EINVAL;
+
+	if (vlan_proto != htons(ETH_P_8021Q) || qos != 0)
+		return -EPROTONOSUPPORT;
+
+	ret = t4_set_vlan_acl(adap, adap->mbox, vf + 1, vlan);
+	if (!ret) {
+		adap->vfinfo[vf].vlan = vlan;
+		return 0;
+	}
+
+	dev_err(adap->pdev_dev, "Err %d %s VLAN ACL for PF/VF %d/%d\n",
+		ret, (vlan ? "setting" : "clearing"), adap->pf, vf);
+	return ret;
+}
+#endif /* CONFIG_PCI_IOV */
 
 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
 {
@@ -2897,9 +2928,6 @@ static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
 static int cxgb_setup_tc_flower(struct net_device *dev,
 				struct tc_cls_flower_offload *cls_flower)
 {
-	if (cls_flower->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (cls_flower->command) {
 	case TC_CLSFLOWER_REPLACE:
 		return cxgb4_tc_flower_replace(dev, cls_flower);
@@ -2915,9 +2943,6 @@ static int cxgb_setup_tc_flower(struct net_device *dev,
 static int cxgb_setup_tc_cls_u32(struct net_device *dev,
 				 struct tc_cls_u32_offload *cls_u32)
 {
-	if (cls_u32->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (cls_u32->command) {
 	case TC_CLSU32_NEW_KNODE:
 	case TC_CLSU32_REPLACE_KNODE:
@@ -2943,7 +2968,7 @@ static int cxgb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 		return -EINVAL;
 	}
 
-	if (!tc_can_offload(dev))
+	if (!tc_cls_can_offload_and_chain0(dev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
@@ -2987,6 +3012,174 @@ static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type,
 	}
 }
 
+static void cxgb_del_udp_tunnel(struct net_device *netdev,
+				struct udp_tunnel_info *ti)
+{
+	struct port_info *pi = netdev_priv(netdev);
+	struct adapter *adapter = pi->adapter;
+	unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
+	u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
+	int ret = 0, i;
+
+	if (chip_ver < CHELSIO_T6)
+		return;
+
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		if (!adapter->vxlan_port_cnt ||
+		    adapter->vxlan_port != ti->port)
+			return; /* Invalid VxLAN destination port */
+
+		adapter->vxlan_port_cnt--;
+		if (adapter->vxlan_port_cnt)
+			return;
+
+		adapter->vxlan_port = 0;
+		t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 0);
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		if (!adapter->geneve_port_cnt ||
+		    adapter->geneve_port != ti->port)
+			return; /* Invalid GENEVE destination port */
+
+		adapter->geneve_port_cnt--;
+		if (adapter->geneve_port_cnt)
+			return;
+
+		adapter->geneve_port = 0;
+		t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 0);
+	default:
+		return;
+	}
+
+	/* Matchall mac entries can be deleted only after all tunnel ports
+	 * are brought down or removed.
+	 */
+	if (!adapter->rawf_cnt)
+		return;
+	for_each_port(adapter, i) {
+		pi = adap2pinfo(adapter, i);
+		ret = t4_free_raw_mac_filt(adapter, pi->viid,
+					   match_all_mac, match_all_mac,
+					   adapter->rawf_start +
+					    pi->port_id,
+					   1, pi->port_id, true);
+		if (ret < 0) {
+			netdev_info(netdev, "Failed to free mac filter entry, for port %d\n",
+				    i);
+			return;
+		}
+		atomic_dec(&adapter->mps_encap[adapter->rawf_start +
+			   pi->port_id].refcnt);
+	}
+}
+
+static void cxgb_add_udp_tunnel(struct net_device *netdev,
+				struct udp_tunnel_info *ti)
+{
+	struct port_info *pi = netdev_priv(netdev);
+	struct adapter *adapter = pi->adapter;
+	unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
+	u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
+	int i, ret;
+
+	if (chip_ver < CHELSIO_T6 || !adapter->rawf_cnt)
+		return;
+
+	switch (ti->type) {
+	case UDP_TUNNEL_TYPE_VXLAN:
+		/* Callback for adding vxlan port can be called with the same
+		 * port for both IPv4 and IPv6. We should not disable the
+		 * offloading when the same port for both protocols is added
+		 * and later one of them is removed.
+		 */
+		if (adapter->vxlan_port_cnt &&
+		    adapter->vxlan_port == ti->port) {
+			adapter->vxlan_port_cnt++;
+			return;
+		}
+
+		/* We will support only one VxLAN port */
+		if (adapter->vxlan_port_cnt) {
+			netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n",
+				    be16_to_cpu(adapter->vxlan_port),
+				    be16_to_cpu(ti->port));
+			return;
+		}
+
+		adapter->vxlan_port = ti->port;
+		adapter->vxlan_port_cnt = 1;
+
+		t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A,
+			     VXLAN_V(be16_to_cpu(ti->port)) | VXLAN_EN_F);
+		break;
+	case UDP_TUNNEL_TYPE_GENEVE:
+		if (adapter->geneve_port_cnt &&
+		    adapter->geneve_port == ti->port) {
+			adapter->geneve_port_cnt++;
+			return;
+		}
+
+		/* We will support only one GENEVE port */
+		if (adapter->geneve_port_cnt) {
+			netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n",
+				    be16_to_cpu(adapter->geneve_port),
+				    be16_to_cpu(ti->port));
+			return;
+		}
+
+		adapter->geneve_port = ti->port;
+		adapter->geneve_port_cnt = 1;
+
+		t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A,
+			     GENEVE_V(be16_to_cpu(ti->port)) | GENEVE_EN_F);
+	default:
+		return;
+	}
+
+	/* Create a 'match all' mac filter entry for inner mac,
+	 * if raw mac interface is supported. Once the linux kernel provides
+	 * driver entry points for adding/deleting the inner mac addresses,
+	 * we will remove this 'match all' entry and fallback to adding
+	 * exact match filters.
+	 */
+	for_each_port(adapter, i) {
+		pi = adap2pinfo(adapter, i);
+
+		ret = t4_alloc_raw_mac_filt(adapter, pi->viid,
+					    match_all_mac,
+					    match_all_mac,
+					    adapter->rawf_start +
+					    pi->port_id,
+					    1, pi->port_id, true);
+		if (ret < 0) {
+			netdev_info(netdev, "Failed to allocate a mac filter entry, not adding port %d\n",
+				    be16_to_cpu(ti->port));
+			cxgb_del_udp_tunnel(netdev, ti);
+			return;
+		}
+		atomic_inc(&adapter->mps_encap[ret].refcnt);
+	}
+}
+
+static netdev_features_t cxgb_features_check(struct sk_buff *skb,
+					     struct net_device *dev,
+					     netdev_features_t features)
+{
+	struct port_info *pi = netdev_priv(dev);
+	struct adapter *adapter = pi->adapter;
+
+	if (CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6)
+		return features;
+
+	/* Check if hw supports offload for this packet */
+	if (!skb->encapsulation || cxgb_encap_offload_supported(skb))
+		return features;
+
+	/* Offload is not supported for this encapsulated packet */
+	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
+}
+
 static netdev_features_t cxgb_fix_features(struct net_device *dev,
 					   netdev_features_t features)
 {
@@ -3018,20 +3211,25 @@ static const struct net_device_ops cxgb4_netdev_ops = {
 #endif /* CONFIG_CHELSIO_T4_FCOE */
 	.ndo_set_tx_maxrate   = cxgb_set_tx_maxrate,
 	.ndo_setup_tc         = cxgb_setup_tc,
+	.ndo_udp_tunnel_add   = cxgb_add_udp_tunnel,
+	.ndo_udp_tunnel_del   = cxgb_del_udp_tunnel,
+	.ndo_features_check   = cxgb_features_check,
 	.ndo_fix_features     = cxgb_fix_features,
 };
 
 #ifdef CONFIG_PCI_IOV
 static const struct net_device_ops cxgb4_mgmt_netdev_ops = {
-	.ndo_open             = dummy_open,
-	.ndo_set_vf_mac       = cxgb_set_vf_mac,
-	.ndo_get_vf_config    = cxgb_get_vf_config,
-	.ndo_set_vf_rate      = cxgb_set_vf_rate,
-	.ndo_get_phys_port_id = cxgb_get_phys_port_id,
+	.ndo_open             = cxgb4_mgmt_open,
+	.ndo_set_vf_mac       = cxgb4_mgmt_set_vf_mac,
+	.ndo_get_vf_config    = cxgb4_mgmt_get_vf_config,
+	.ndo_set_vf_rate      = cxgb4_mgmt_set_vf_rate,
+	.ndo_get_phys_port_id = cxgb4_mgmt_get_phys_port_id,
+	.ndo_set_vf_vlan      = cxgb4_mgmt_set_vf_vlan,
 };
 #endif
 
-static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
+static void cxgb4_mgmt_get_drvinfo(struct net_device *dev,
+				   struct ethtool_drvinfo *info)
 {
 	struct adapter *adapter = netdev2adap(dev);
 
@@ -3043,7 +3241,7 @@ static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 }
 
 static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = {
-	.get_drvinfo       = get_drvinfo,
+	.get_drvinfo       = cxgb4_mgmt_get_drvinfo,
 };
 
 void t4_fatal_err(struct adapter *adap)
@@ -4096,7 +4294,7 @@ static int adap_init0(struct adapter *adap)
 		} else {
 			adap->vres.ncrypto_fc = val[0];
 		}
-		adap->params.crypto |= ULP_CRYPTO_LOOKASIDE;
+		adap->params.crypto = ntohs(caps_cmd.cryptocaps);
 		adap->num_uld += 1;
 	}
 #undef FW_PARAM_PFVF
@@ -4759,7 +4957,7 @@ static int get_chip_type(struct pci_dev *pdev, u32 pl_rev)
 }
 
 #ifdef CONFIG_PCI_IOV
-static void dummy_setup(struct net_device *dev)
+static void cxgb4_mgmt_setup(struct net_device *dev)
 {
 	dev->type = ARPHRD_NONE;
 	dev->mtu = 0;
@@ -4772,39 +4970,6 @@ static void dummy_setup(struct net_device *dev)
 	/* Initialize the device structure. */
 	dev->netdev_ops = &cxgb4_mgmt_netdev_ops;
 	dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops;
-	dev->needs_free_netdev = true;
-}
-
-static int config_mgmt_dev(struct pci_dev *pdev)
-{
-	struct adapter *adap = pci_get_drvdata(pdev);
-	struct net_device *netdev;
-	struct port_info *pi;
-	char name[IFNAMSIZ];
-	int err;
-
-	snprintf(name, IFNAMSIZ, "mgmtpf%d%d", adap->adap_idx, adap->pf);
-	netdev = alloc_netdev(sizeof(struct port_info), name, NET_NAME_UNKNOWN,
-			      dummy_setup);
-	if (!netdev)
-		return -ENOMEM;
-
-	pi = netdev_priv(netdev);
-	pi->adapter = adap;
-	pi->tx_chan = adap->pf % adap->params.nports;
-	SET_NETDEV_DEV(netdev, &pdev->dev);
-
-	adap->port[0] = netdev;
-	pi->port_id = 0;
-
-	err = register_netdev(adap->port[0]);
-	if (err) {
-		pr_info("Unable to register VF mgmt netdev %s\n", name);
-		free_netdev(adap->port[0]);
-		adap->port[0] = NULL;
-		return err;
-	}
-	return 0;
 }
 
 static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
@@ -4816,9 +4981,10 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 
 	pcie_fw = readl(adap->regs + PCIE_FW_A);
 	/* Check if cxgb4 is the MASTER and fw is initialized */
-	if (!(pcie_fw & PCIE_FW_INIT_F) ||
+	if (num_vfs &&
+	    (!(pcie_fw & PCIE_FW_INIT_F) ||
 	    !(pcie_fw & PCIE_FW_MASTER_VLD_F) ||
-	    PCIE_FW_MASTER_G(pcie_fw) != 4) {
+	    PCIE_FW_MASTER_G(pcie_fw) != CXGB4_UNIFIED_PF)) {
 		dev_warn(&pdev->dev,
 			 "cxgb4 driver needs to be MASTER to support SRIOV\n");
 		return -EOPNOTSUPP;
@@ -4830,46 +4996,132 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 	if (current_vfs && pci_vfs_assigned(pdev)) {
 		dev_err(&pdev->dev,
 			"Cannot modify SR-IOV while VFs are assigned\n");
-		num_vfs = current_vfs;
-		return num_vfs;
+		return current_vfs;
 	}
-
-	/* Disable SRIOV when zero is passed.
-	 * One needs to disable SRIOV before modifying it, else
-	 * stack throws the below warning:
-	 * " 'n' VFs already enabled. Disable before enabling 'm' VFs."
+	/* Note that the upper-level code ensures that we're never called with
+	 * a non-zero "num_vfs" when we already have VFs instantiated.  But
+	 * it never hurts to code defensively.
 	 */
+	if (num_vfs != 0 && current_vfs != 0)
+		return -EBUSY;
+
+	/* Nothing to do for no change. */
+	if (num_vfs == current_vfs)
+		return num_vfs;
+
+	/* Disable SRIOV when zero is passed. */
 	if (!num_vfs) {
 		pci_disable_sriov(pdev);
-		if (adap->port[0]) {
-			unregister_netdev(adap->port[0]);
-			adap->port[0] = NULL;
-		}
+		/* free VF Management Interface */
+		unregister_netdev(adap->port[0]);
+		free_netdev(adap->port[0]);
+		adap->port[0] = NULL;
+
 		/* free VF resources */
+		adap->num_vfs = 0;
 		kfree(adap->vfinfo);
 		adap->vfinfo = NULL;
-		adap->num_vfs = 0;
-		return num_vfs;
+		return 0;
 	}
 
-	if (num_vfs != current_vfs) {
-		err = pci_enable_sriov(pdev, num_vfs);
+	if (!current_vfs) {
+		struct fw_pfvf_cmd port_cmd, port_rpl;
+		struct net_device *netdev;
+		unsigned int pmask, port;
+		struct pci_dev *pbridge;
+		struct port_info *pi;
+		char name[IFNAMSIZ];
+		u32 devcap2;
+		u16 flags;
+		int pos;
+
+		/* If we want to instantiate Virtual Functions, then our
+		 * parent bridge's PCI-E needs to support Alternative Routing
+		 * ID (ARI) because our VFs will show up at function offset 8
+		 * and above.
+		 */
+		pbridge = pdev->bus->self;
+		pos = pci_find_capability(pbridge, PCI_CAP_ID_EXP);
+		pci_read_config_word(pbridge, pos + PCI_EXP_FLAGS, &flags);
+		pci_read_config_dword(pbridge, pos + PCI_EXP_DEVCAP2, &devcap2);
+
+		if ((flags & PCI_EXP_FLAGS_VERS) < 2 ||
+		    !(devcap2 & PCI_EXP_DEVCAP2_ARI)) {
+			/* Our parent bridge does not support ARI so issue a
+			 * warning and skip instantiating the VFs.  They
+			 * won't be reachable.
+			 */
+			dev_warn(&pdev->dev, "Parent bridge %02x:%02x.%x doesn't support ARI; can't instantiate Virtual Functions\n",
+				 pbridge->bus->number, PCI_SLOT(pbridge->devfn),
+				 PCI_FUNC(pbridge->devfn));
+			return -ENOTSUPP;
+		}
+		memset(&port_cmd, 0, sizeof(port_cmd));
+		port_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) |
+						 FW_CMD_REQUEST_F |
+						 FW_CMD_READ_F |
+						 FW_PFVF_CMD_PFN_V(adap->pf) |
+						 FW_PFVF_CMD_VFN_V(0));
+		port_cmd.retval_len16 = cpu_to_be32(FW_LEN16(port_cmd));
+		err = t4_wr_mbox(adap, adap->mbox, &port_cmd, sizeof(port_cmd),
+				 &port_rpl);
 		if (err)
 			return err;
+		pmask = FW_PFVF_CMD_PMASK_G(be32_to_cpu(port_rpl.type_to_neq));
+		port = ffs(pmask) - 1;
+		/* Allocate VF Management Interface. */
+		snprintf(name, IFNAMSIZ, "mgmtpf%d,%d", adap->adap_idx,
+			 adap->pf);
+		netdev = alloc_netdev(sizeof(struct port_info),
+				      name, NET_NAME_UNKNOWN, cxgb4_mgmt_setup);
+		if (!netdev)
+			return -ENOMEM;
 
-		adap->num_vfs = num_vfs;
-		err = config_mgmt_dev(pdev);
-		if (err)
+		pi = netdev_priv(netdev);
+		pi->adapter = adap;
+		pi->lport = port;
+		pi->tx_chan = port;
+		SET_NETDEV_DEV(netdev, &pdev->dev);
+
+		adap->port[0] = netdev;
+		pi->port_id = 0;
+
+		err = register_netdev(adap->port[0]);
+		if (err) {
+			pr_info("Unable to register VF mgmt netdev %s\n", name);
+			free_netdev(adap->port[0]);
+			adap->port[0] = NULL;
 			return err;
+		}
+		/* Allocate and set up VF Information. */
+		adap->vfinfo = kcalloc(pci_sriov_get_totalvfs(pdev),
+				       sizeof(struct vf_info), GFP_KERNEL);
+		if (!adap->vfinfo) {
+			unregister_netdev(adap->port[0]);
+			free_netdev(adap->port[0]);
+			adap->port[0] = NULL;
+			return -ENOMEM;
+		}
+		cxgb4_mgmt_fill_vf_station_mac_addr(adap);
+	}
+	/* Instantiate the requested number of VFs. */
+	err = pci_enable_sriov(pdev, num_vfs);
+	if (err) {
+		pr_info("Unable to instantiate %d VFs\n", num_vfs);
+		if (!current_vfs) {
+			unregister_netdev(adap->port[0]);
+			free_netdev(adap->port[0]);
+			adap->port[0] = NULL;
+			kfree(adap->vfinfo);
+			adap->vfinfo = NULL;
+		}
+		return err;
 	}
 
-	adap->vfinfo = kcalloc(adap->num_vfs,
-			       sizeof(struct vf_info), GFP_KERNEL);
-	if (adap->vfinfo)
-		fill_vf_station_mac_addr(adap);
+	adap->num_vfs = num_vfs;
 	return num_vfs;
 }
-#endif
+#endif /* CONFIG_PCI_IOV */
 
 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
@@ -4882,9 +5134,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	u32 whoami, pl_rev;
 	enum chip_type chip;
 	static int adap_idx = 1;
-#ifdef CONFIG_PCI_IOV
-	u32 v, port_vec;
-#endif
 
 	printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
 
@@ -4908,9 +5157,16 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto out_disable_device;
 	}
 
+	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
+	if (!adapter) {
+		err = -ENOMEM;
+		goto out_unmap_bar0;
+	}
+
+	adapter->regs = regs;
 	err = t4_wait_dev_ready(regs);
 	if (err < 0)
-		goto out_unmap_bar0;
+		goto out_free_adapter;
 
 	/* We control everything through one PF */
 	whoami = readl(regs + PL_WHOAMI_A);
@@ -4918,13 +5174,31 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	chip = get_chip_type(pdev, pl_rev);
 	func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ?
 		SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami);
+
+	adapter->pdev = pdev;
+	adapter->pdev_dev = &pdev->dev;
+	adapter->name = pci_name(pdev);
+	adapter->mbox = func;
+	adapter->pf = func;
+	adapter->params.chip = chip;
+	adapter->adap_idx = adap_idx;
+	adapter->msg_enable = DFLT_MSG_ENABLE;
+	adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
+				    (sizeof(struct mbox_cmd) *
+				     T4_OS_LOG_MBOX_CMDS),
+				    GFP_KERNEL);
+	if (!adapter->mbox_log) {
+		err = -ENOMEM;
+		goto out_free_adapter;
+	}
+	spin_lock_init(&adapter->mbox_lock);
+	INIT_LIST_HEAD(&adapter->mlist.list);
+	pci_set_drvdata(pdev, adapter);
+
 	if (func != ent->driver_data) {
-#ifndef CONFIG_PCI_IOV
-		iounmap(regs);
-#endif
 		pci_disable_device(pdev);
 		pci_save_state(pdev);        /* to restore SR-IOV later */
-		goto sriov;
+		return 0;
 	}
 
 	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
@@ -4933,53 +5207,30 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		if (err) {
 			dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
 				"coherent allocations\n");
-			goto out_unmap_bar0;
+			goto out_free_adapter;
 		}
 	} else {
 		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 		if (err) {
 			dev_err(&pdev->dev, "no usable DMA configuration\n");
-			goto out_unmap_bar0;
+			goto out_free_adapter;
 		}
 	}
 
 	pci_enable_pcie_error_reporting(pdev);
 	pci_set_master(pdev);
 	pci_save_state(pdev);
-
-	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
-	if (!adapter) {
-		err = -ENOMEM;
-		goto out_unmap_bar0;
-	}
 	adap_idx++;
-
 	adapter->workq = create_singlethread_workqueue("cxgb4");
 	if (!adapter->workq) {
 		err = -ENOMEM;
 		goto out_free_adapter;
 	}
 
-	adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
-				    (sizeof(struct mbox_cmd) *
-				     T4_OS_LOG_MBOX_CMDS),
-				    GFP_KERNEL);
-	if (!adapter->mbox_log) {
-		err = -ENOMEM;
-		goto out_free_adapter;
-	}
 	adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS;
 
 	/* PCI device has been enabled */
 	adapter->flags |= DEV_ENABLED;
-
-	adapter->regs = regs;
-	adapter->pdev = pdev;
-	adapter->pdev_dev = &pdev->dev;
-	adapter->name = pci_name(pdev);
-	adapter->mbox = func;
-	adapter->pf = func;
-	adapter->msg_enable = DFLT_MSG_ENABLE;
 	memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
 
 	/* If possible, we use PCIe Relaxed Ordering Attribute to deliver
@@ -5002,9 +5253,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	spin_lock_init(&adapter->stats_lock);
 	spin_lock_init(&adapter->tid_release_lock);
 	spin_lock_init(&adapter->win0_lock);
-	spin_lock_init(&adapter->mbox_lock);
-
-	INIT_LIST_HEAD(&adapter->mlist.list);
 
 	INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
 	INIT_WORK(&adapter->db_full_task, process_db_full);
@@ -5080,6 +5328,10 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 			NETIF_F_RXCSUM | NETIF_F_RXHASH |
 			NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
 			NETIF_F_HW_TC;
+
+		if (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5)
+			netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
+
 		if (highdma)
 			netdev->hw_features |= NETIF_F_HIGHDMA;
 		netdev->features |= netdev->hw_features;
@@ -5273,58 +5525,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	setup_fw_sge_queues(adapter);
 	return 0;
 
-sriov:
-#ifdef CONFIG_PCI_IOV
-	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
-	if (!adapter) {
-		err = -ENOMEM;
-		goto free_pci_region;
-	}
-
-	adapter->pdev = pdev;
-	adapter->pdev_dev = &pdev->dev;
-	adapter->name = pci_name(pdev);
-	adapter->mbox = func;
-	adapter->pf = func;
-	adapter->regs = regs;
-	adapter->adap_idx = adap_idx;
-	adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
-				    (sizeof(struct mbox_cmd) *
-				     T4_OS_LOG_MBOX_CMDS),
-				    GFP_KERNEL);
-	if (!adapter->mbox_log) {
-		err = -ENOMEM;
-		goto free_adapter;
-	}
-	spin_lock_init(&adapter->mbox_lock);
-	INIT_LIST_HEAD(&adapter->mlist.list);
-
-	v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
-	    FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
-	err = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, 1,
-			      &v, &port_vec);
-	if (err < 0) {
-		dev_err(adapter->pdev_dev, "Could not fetch port params\n");
-		goto free_mbox_log;
-	}
-
-	adapter->params.nports = hweight32(port_vec);
-	pci_set_drvdata(pdev, adapter);
-	return 0;
-
-free_mbox_log:
-	kfree(adapter->mbox_log);
- free_adapter:
-	kfree(adapter);
- free_pci_region:
-	iounmap(regs);
-	pci_disable_sriov(pdev);
-	pci_release_regions(pdev);
-	return err;
-#else
-	return 0;
-#endif
-
  out_free_dev:
 	free_some_resources(adapter);
 	if (adapter->flags & USING_MSIX)
@@ -5401,31 +5601,24 @@ static void remove_one(struct pci_dev *pdev)
 #if IS_ENABLED(CONFIG_IPV6)
 		t4_cleanup_clip_tbl(adapter);
 #endif
-		iounmap(adapter->regs);
 		if (!is_t4(adapter->params.chip))
 			iounmap(adapter->bar2);
-		pci_disable_pcie_error_reporting(pdev);
-		if ((adapter->flags & DEV_ENABLED)) {
-			pci_disable_device(pdev);
-			adapter->flags &= ~DEV_ENABLED;
-		}
-		pci_release_regions(pdev);
-		kfree(adapter->mbox_log);
-		synchronize_rcu();
-		kfree(adapter);
 	}
 #ifdef CONFIG_PCI_IOV
 	else {
-		if (adapter->port[0])
-			unregister_netdev(adapter->port[0]);
-		iounmap(adapter->regs);
-		kfree(adapter->vfinfo);
-		kfree(adapter->mbox_log);
-		kfree(adapter);
-		pci_disable_sriov(pdev);
-		pci_release_regions(pdev);
+		cxgb4_iov_configure(adapter->pdev, 0);
 	}
 #endif
+	iounmap(adapter->regs);
+	pci_disable_pcie_error_reporting(pdev);
+	if ((adapter->flags & DEV_ENABLED)) {
+		pci_disable_device(pdev);
+		adapter->flags &= ~DEV_ENABLED;
+	}
+	pci_release_regions(pdev);
+	kfree(adapter->mbox_log);
+	synchronize_rcu();
+	kfree(adapter);
 }
 
 /* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt
@@ -5467,18 +5660,6 @@ static void shutdown_one(struct pci_dev *pdev)
 		if (adapter->flags & FW_OK)
 			t4_fw_bye(adapter, adapter->mbox);
 	}
-#ifdef CONFIG_PCI_IOV
-	else {
-		if (adapter->port[0])
-			unregister_netdev(adapter->port[0]);
-		iounmap(adapter->regs);
-		kfree(adapter->vfinfo);
-		kfree(adapter->mbox_log);
-		kfree(adapter);
-		pci_disable_sriov(pdev);
-		pci_release_regions(pdev);
-	}
-#endif
 }
 
 static struct pci_driver cxgb4_driver = {
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
index a452d5a1b0f3..36563364bae7 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c
@@ -43,7 +43,7 @@
 
 #define STATS_CHECK_PERIOD (HZ / 2)
 
-struct ch_tc_pedit_fields pedits[] = {
+static struct ch_tc_pedit_fields pedits[] = {
 	PEDIT_FIELDS(ETH_, DMAC_31_0, 4, dmac, 0),
 	PEDIT_FIELDS(ETH_, DMAC_47_32, 2, dmac, 4),
 	PEDIT_FIELDS(ETH_, SMAC_15_0, 2, smac, 0),
@@ -408,9 +408,7 @@ static void cxgb4_process_flow_actions(struct net_device *in,
 		} else if (is_tcf_gact_shot(a)) {
 			fs->action = FILTER_DROP;
 		} else if (is_tcf_mirred_egress_redirect(a)) {
-			int ifindex = tcf_mirred_ifindex(a);
-			struct net_device *out = __dev_get_by_index(dev_net(in),
-								    ifindex);
+			struct net_device *out = tcf_mirred_dev(a);
 			struct port_info *pi = netdev_priv(out);
 
 			fs->action = FILTER_SWITCH;
@@ -585,14 +583,14 @@ static int cxgb4_validate_flow_actions(struct net_device *dev,
 			/* Do nothing */
 		} else if (is_tcf_mirred_egress_redirect(a)) {
 			struct adapter *adap = netdev2adap(dev);
-			struct net_device *n_dev;
-			unsigned int i, ifindex;
+			struct net_device *n_dev, *target_dev;
+			unsigned int i;
 			bool found = false;
 
-			ifindex = tcf_mirred_ifindex(a);
+			target_dev = tcf_mirred_dev(a);
 			for_each_port(adap, i) {
 				n_dev = adap->port[i];
-				if (ifindex == n_dev->ifindex) {
+				if (target_dev == n_dev) {
 					found = true;
 					break;
 				}
@@ -768,9 +766,7 @@ static void ch_flower_stats_handler(struct work_struct *work)
 
 	rhashtable_walk_enter(&adap->flower_tbl, &iter);
 	do {
-		flower_entry = ERR_PTR(rhashtable_walk_start(&iter));
-		if (IS_ERR(flower_entry))
-			goto walk_stop;
+		rhashtable_walk_start(&iter);
 
 		while ((flower_entry = rhashtable_walk_next(&iter)) &&
 		       !IS_ERR(flower_entry)) {
@@ -789,8 +785,9 @@ static void ch_flower_stats_handler(struct work_struct *work)
 				spin_unlock(&flower_entry->lock);
 			}
 		}
-walk_stop:
+
 		rhashtable_walk_stop(&iter);
+
 	} while (flower_entry == ERR_PTR(-EAGAIN));
 	rhashtable_walk_exit(&iter);
 	mod_timer(&adap->flower_stats_timer, jiffies + STATS_CHECK_PERIOD);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
index cd0cd13a964d..ab174bcfbfb0 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_u32.c
@@ -114,14 +114,14 @@ static int fill_action_fields(struct adapter *adap,
 
 		/* Re-direct to specified port in hardware. */
 		if (is_tcf_mirred_egress_redirect(a)) {
-			struct net_device *n_dev;
-			unsigned int i, index;
+			struct net_device *n_dev, *target_dev;
 			bool found = false;
+			unsigned int i;
 
-			index = tcf_mirred_ifindex(a);
+			target_dev = tcf_mirred_dev(a);
 			for_each_port(adap, i) {
 				n_dev = adap->port[i];
-				if (index == n_dev->ifindex) {
+				if (target_dev == n_dev) {
 					fs->action = FILTER_SWITCH;
 					fs->eport = i;
 					found = true;
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
index 71a315bc1409..6b5fea4532f3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c
@@ -637,6 +637,7 @@ static void uld_init(struct adapter *adap, struct cxgb4_lld_info *lld)
 	lld->nchan = adap->params.nports;
 	lld->nports = adap->params.nports;
 	lld->wr_cred = adap->params.ofldq_wr_cred;
+	lld->crypto = adap->params.crypto;
 	lld->iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
 	lld->iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A);
 	lld->iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
index 08e709ab6dd4..a14e8db51cdc 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h
@@ -297,6 +297,7 @@ struct chcr_stats_debug {
 	atomic_t complete;
 	atomic_t error;
 	atomic_t fallback;
+	atomic_t ipsec_cnt;
 };
 
 #define OCQ_WIN_OFFSET(pdev, vres) \
@@ -322,6 +323,7 @@ struct cxgb4_lld_info {
 	unsigned char wr_cred;               /* WR 16-byte credits */
 	unsigned char adapter_type;          /* type of adapter */
 	unsigned char fw_api_ver;            /* FW API version */
+	unsigned char crypto;                /* crypto support */
 	unsigned int fw_vers;                /* FW version */
 	unsigned int iscsi_iolen;            /* iSCSI max I/O length */
 	unsigned int cclk_ps;                /* Core clock period in psec */
@@ -353,7 +355,7 @@ struct cxgb4_lld_info {
 };
 
 struct cxgb4_uld_info {
-	const char *name;
+	char name[IFNAMSIZ];
 	void *handle;
 	unsigned int nrxq;
 	unsigned int rxq_size;
@@ -370,6 +372,7 @@ struct cxgb4_uld_info {
 			      struct t4_lro_mgr *lro_mgr,
 			      struct napi_struct *napi);
 	void (*lro_flush)(struct t4_lro_mgr *);
+	int (*tx_handler)(struct sk_buff *skb, struct net_device *dev);
 };
 
 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index 922f2f937789..6e310a0da7c9 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -41,6 +41,7 @@
 #include <linux/jiffies.h>
 #include <linux/prefetch.h>
 #include <linux/export.h>
+#include <net/xfrm.h>
 #include <net/ipv6.h>
 #include <net/tcp.h>
 #include <net/busy_poll.h>
@@ -53,6 +54,7 @@
 #include "t4_msg.h"
 #include "t4fw_api.h"
 #include "cxgb4_ptp.h"
+#include "cxgb4_uld.h"
 
 /*
  * Rx buffer size.  We use largish buffers if possible but settle for single
@@ -109,14 +111,6 @@
  */
 #define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
 
-/*
- * Suspend an Ethernet Tx queue with fewer available descriptors than this.
- * This is the same as calc_tx_descs() for a TSO packet with
- * nr_frags == MAX_SKB_FRAGS.
- */
-#define ETHTXQ_STOP_THRES \
-	(1 + DIV_ROUND_UP((3 * MAX_SKB_FRAGS) / 2 + (MAX_SKB_FRAGS & 1), 8))
-
 /*
  * Suspension threshold for non-Ethernet Tx queues.  We require enough room
  * for a full sized WR.
@@ -134,11 +128,6 @@
  */
 #define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
 
-struct tx_sw_desc {                /* SW state per Tx descriptor */
-	struct sk_buff *skb;
-	struct ulptx_sgl *sgl;
-};
-
 struct rx_sw_desc {                /* SW state per Rx descriptor */
 	struct page *page;
 	dma_addr_t dma_addr;
@@ -248,8 +237,8 @@ static inline bool fl_starving(const struct adapter *adapter,
 	return fl->avail - fl->pend_cred <= s->fl_starve_thres;
 }
 
-static int map_skb(struct device *dev, const struct sk_buff *skb,
-		   dma_addr_t *addr)
+int cxgb4_map_skb(struct device *dev, const struct sk_buff *skb,
+		  dma_addr_t *addr)
 {
 	const skb_frag_t *fp, *end;
 	const struct skb_shared_info *si;
@@ -277,6 +266,7 @@ static int map_skb(struct device *dev, const struct sk_buff *skb,
 out_err:
 	return -ENOMEM;
 }
+EXPORT_SYMBOL(cxgb4_map_skb);
 
 #ifdef CONFIG_NEED_DMA_MAP_STATE
 static void unmap_skb(struct device *dev, const struct sk_buff *skb,
@@ -411,7 +401,7 @@ static inline int reclaimable(const struct sge_txq *q)
 }
 
 /**
- *	reclaim_completed_tx - reclaims completed Tx descriptors
+ *	cxgb4_reclaim_completed_tx - reclaims completed Tx descriptors
  *	@adap: the adapter
  *	@q: the Tx queue to reclaim completed descriptors from
  *	@unmap: whether the buffers should be unmapped for DMA
@@ -420,7 +410,7 @@ static inline int reclaimable(const struct sge_txq *q)
  *	and frees the associated buffers if possible.  Called with the Tx
  *	queue locked.
  */
-static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
+inline void cxgb4_reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
 					bool unmap)
 {
 	int avail = reclaimable(q);
@@ -437,6 +427,7 @@ static inline void reclaim_completed_tx(struct adapter *adap, struct sge_txq *q,
 		q->in_use -= avail;
 	}
 }
+EXPORT_SYMBOL(cxgb4_reclaim_completed_tx);
 
 static inline int get_buf_size(struct adapter *adapter,
 			       const struct rx_sw_desc *d)
@@ -770,12 +761,19 @@ static inline unsigned int flits_to_desc(unsigned int n)
  *	Returns whether an Ethernet packet is small enough to fit as
  *	immediate data. Return value corresponds to headroom required.
  */
-static inline int is_eth_imm(const struct sk_buff *skb)
+static inline int is_eth_imm(const struct sk_buff *skb, unsigned int chip_ver)
 {
-	int hdrlen = skb_shinfo(skb)->gso_size ?
-			sizeof(struct cpl_tx_pkt_lso_core) : 0;
+	int hdrlen = 0;
 
-	hdrlen += sizeof(struct cpl_tx_pkt);
+	if (skb->encapsulation && skb_shinfo(skb)->gso_size &&
+	    chip_ver > CHELSIO_T5) {
+		hdrlen = sizeof(struct cpl_tx_tnl_lso);
+		hdrlen += sizeof(struct cpl_tx_pkt_core);
+	} else {
+		hdrlen = skb_shinfo(skb)->gso_size ?
+			 sizeof(struct cpl_tx_pkt_lso_core) : 0;
+		hdrlen += sizeof(struct cpl_tx_pkt);
+	}
 	if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
 		return hdrlen;
 	return 0;
@@ -788,10 +786,11 @@ static inline int is_eth_imm(const struct sk_buff *skb)
  *	Returns the number of flits needed for a Tx WR for the given Ethernet
  *	packet, including the needed WR and CPL headers.
  */
-static inline unsigned int calc_tx_flits(const struct sk_buff *skb)
+static inline unsigned int calc_tx_flits(const struct sk_buff *skb,
+					 unsigned int chip_ver)
 {
 	unsigned int flits;
-	int hdrlen = is_eth_imm(skb);
+	int hdrlen = is_eth_imm(skb, chip_ver);
 
 	/* If the skb is small enough, we can pump it out as a work request
 	 * with only immediate data.  In that case we just have to have the
@@ -810,13 +809,20 @@ static inline unsigned int calc_tx_flits(const struct sk_buff *skb)
 	 * with an embedded TX Packet Write CPL message.
 	 */
 	flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
-	if (skb_shinfo(skb)->gso_size)
-		flits += (sizeof(struct fw_eth_tx_pkt_wr) +
-			  sizeof(struct cpl_tx_pkt_lso_core) +
-			  sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
-	else
+	if (skb_shinfo(skb)->gso_size) {
+		if (skb->encapsulation && chip_ver > CHELSIO_T5)
+			hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
+				 sizeof(struct cpl_tx_tnl_lso);
+		else
+			hdrlen = sizeof(struct fw_eth_tx_pkt_wr) +
+				 sizeof(struct cpl_tx_pkt_lso_core);
+
+		hdrlen += sizeof(struct cpl_tx_pkt_core);
+		flits += (hdrlen / sizeof(__be64));
+	} else {
 		flits += (sizeof(struct fw_eth_tx_pkt_wr) +
 			  sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
+	}
 	return flits;
 }
 
@@ -827,13 +833,14 @@ static inline unsigned int calc_tx_flits(const struct sk_buff *skb)
  *	Returns the number of Tx descriptors needed for the given Ethernet
  *	packet, including the needed WR and CPL headers.
  */
-static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
+static inline unsigned int calc_tx_descs(const struct sk_buff *skb,
+					 unsigned int chip_ver)
 {
-	return flits_to_desc(calc_tx_flits(skb));
+	return flits_to_desc(calc_tx_flits(skb, chip_ver));
 }
 
 /**
- *	write_sgl - populate a scatter/gather list for a packet
+ *	cxgb4_write_sgl - populate a scatter/gather list for a packet
  *	@skb: the packet
  *	@q: the Tx queue we are writing into
  *	@sgl: starting location for writing the SGL
@@ -849,9 +856,9 @@ static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
  *	right after the end of the SGL but does not account for any potential
  *	wrap around, i.e., @end > @sgl.
  */
-static void write_sgl(const struct sk_buff *skb, struct sge_txq *q,
-		      struct ulptx_sgl *sgl, u64 *end, unsigned int start,
-		      const dma_addr_t *addr)
+void cxgb4_write_sgl(const struct sk_buff *skb, struct sge_txq *q,
+		     struct ulptx_sgl *sgl, u64 *end, unsigned int start,
+		     const dma_addr_t *addr)
 {
 	unsigned int i, len;
 	struct ulptx_sge_pair *to;
@@ -903,6 +910,7 @@ static void write_sgl(const struct sk_buff *skb, struct sge_txq *q,
 	if ((uintptr_t)end & 8)           /* 0-pad to multiple of 16 */
 		*end = 0;
 }
+EXPORT_SYMBOL(cxgb4_write_sgl);
 
 /* This function copies 64 byte coalesced work request to
  * memory mapped BAR2 space. For coalesced WR SGE fetches
@@ -921,14 +929,14 @@ static void cxgb_pio_copy(u64 __iomem *dst, u64 *src)
 }
 
 /**
- *	ring_tx_db - check and potentially ring a Tx queue's doorbell
+ *	cxgb4_ring_tx_db - check and potentially ring a Tx queue's doorbell
  *	@adap: the adapter
  *	@q: the Tx queue
  *	@n: number of new descriptors to give to HW
  *
  *	Ring the doorbel for a Tx queue.
  */
-static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
+inline void cxgb4_ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
 {
 	/* Make sure that all writes to the TX Descriptors are committed
 	 * before we tell the hardware about them.
@@ -995,9 +1003,10 @@ static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
 		wmb();
 	}
 }
+EXPORT_SYMBOL(cxgb4_ring_tx_db);
 
 /**
- *	inline_tx_skb - inline a packet's data into Tx descriptors
+ *	cxgb4_inline_tx_skb - inline a packet's data into Tx descriptors
  *	@skb: the packet
  *	@q: the Tx queue where the packet will be inlined
  *	@pos: starting position in the Tx queue where to inline the packet
@@ -1007,8 +1016,8 @@ static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q, int n)
  *	Most of the complexity of this operation is dealing with wrap arounds
  *	in the middle of the packet we want to inline.
  */
-static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
-			  void *pos)
+void cxgb4_inline_tx_skb(const struct sk_buff *skb,
+			 const struct sge_txq *q, void *pos)
 {
 	u64 *p;
 	int left = (void *)q->stat - pos;
@@ -1030,6 +1039,7 @@ static void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *q,
 	if ((uintptr_t)p & 8)
 		*p = 0;
 }
+EXPORT_SYMBOL(cxgb4_inline_tx_skb);
 
 static void *inline_tx_skb_header(const struct sk_buff *skb,
 				  const struct sge_txq *q,  void *pos,
@@ -1154,6 +1164,105 @@ cxgb_fcoe_offload(struct sk_buff *skb, struct adapter *adap,
 }
 #endif /* CONFIG_CHELSIO_T4_FCOE */
 
+/* Returns tunnel type if hardware supports offloading of the same.
+ * It is called only for T5 and onwards.
+ */
+enum cpl_tx_tnl_lso_type cxgb_encap_offload_supported(struct sk_buff *skb)
+{
+	u8 l4_hdr = 0;
+	enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
+	struct port_info *pi = netdev_priv(skb->dev);
+	struct adapter *adapter = pi->adapter;
+
+	if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
+	    skb->inner_protocol != htons(ETH_P_TEB))
+		return tnl_type;
+
+	switch (vlan_get_protocol(skb)) {
+	case htons(ETH_P_IP):
+		l4_hdr = ip_hdr(skb)->protocol;
+		break;
+	case htons(ETH_P_IPV6):
+		l4_hdr = ipv6_hdr(skb)->nexthdr;
+		break;
+	default:
+		return tnl_type;
+	}
+
+	switch (l4_hdr) {
+	case IPPROTO_UDP:
+		if (adapter->vxlan_port == udp_hdr(skb)->dest)
+			tnl_type = TX_TNL_TYPE_VXLAN;
+		else if (adapter->geneve_port == udp_hdr(skb)->dest)
+			tnl_type = TX_TNL_TYPE_GENEVE;
+		break;
+	default:
+		return tnl_type;
+	}
+
+	return tnl_type;
+}
+
+static inline void t6_fill_tnl_lso(struct sk_buff *skb,
+				   struct cpl_tx_tnl_lso *tnl_lso,
+				   enum cpl_tx_tnl_lso_type tnl_type)
+{
+	u32 val;
+	int in_eth_xtra_len;
+	int l3hdr_len = skb_network_header_len(skb);
+	int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
+	const struct skb_shared_info *ssi = skb_shinfo(skb);
+	bool v6 = (ip_hdr(skb)->version == 6);
+
+	val = CPL_TX_TNL_LSO_OPCODE_V(CPL_TX_TNL_LSO) |
+	      CPL_TX_TNL_LSO_FIRST_F |
+	      CPL_TX_TNL_LSO_LAST_F |
+	      (v6 ? CPL_TX_TNL_LSO_IPV6OUT_F : 0) |
+	      CPL_TX_TNL_LSO_ETHHDRLENOUT_V(eth_xtra_len / 4) |
+	      CPL_TX_TNL_LSO_IPHDRLENOUT_V(l3hdr_len / 4) |
+	      (v6 ? 0 : CPL_TX_TNL_LSO_IPHDRCHKOUT_F) |
+	      CPL_TX_TNL_LSO_IPLENSETOUT_F |
+	      (v6 ? 0 : CPL_TX_TNL_LSO_IPIDINCOUT_F);
+	tnl_lso->op_to_IpIdSplitOut = htonl(val);
+
+	tnl_lso->IpIdOffsetOut = 0;
+
+	/* Get the tunnel header length */
+	val = skb_inner_mac_header(skb) - skb_mac_header(skb);
+	in_eth_xtra_len = skb_inner_network_header(skb) -
+			  skb_inner_mac_header(skb) - ETH_HLEN;
+
+	switch (tnl_type) {
+	case TX_TNL_TYPE_VXLAN:
+	case TX_TNL_TYPE_GENEVE:
+		tnl_lso->UdpLenSetOut_to_TnlHdrLen =
+			htons(CPL_TX_TNL_LSO_UDPCHKCLROUT_F |
+			CPL_TX_TNL_LSO_UDPLENSETOUT_F);
+		break;
+	default:
+		tnl_lso->UdpLenSetOut_to_TnlHdrLen = 0;
+		break;
+	}
+
+	tnl_lso->UdpLenSetOut_to_TnlHdrLen |=
+		 htons(CPL_TX_TNL_LSO_TNLHDRLEN_V(val) |
+		       CPL_TX_TNL_LSO_TNLTYPE_V(tnl_type));
+
+	tnl_lso->r1 = 0;
+
+	val = CPL_TX_TNL_LSO_ETHHDRLEN_V(in_eth_xtra_len / 4) |
+	      CPL_TX_TNL_LSO_IPV6_V(inner_ip_hdr(skb)->version == 6) |
+	      CPL_TX_TNL_LSO_IPHDRLEN_V(skb_inner_network_header_len(skb) / 4) |
+	      CPL_TX_TNL_LSO_TCPHDRLEN_V(inner_tcp_hdrlen(skb) / 4);
+	tnl_lso->Flow_to_TcpHdrLen = htonl(val);
+
+	tnl_lso->IpIdOffset = htons(0);
+
+	tnl_lso->IpIdSplit_to_Mss = htons(CPL_TX_TNL_LSO_MSS_V(ssi->gso_size));
+	tnl_lso->TCPSeqOffset = htonl(0);
+	tnl_lso->EthLenOffset_Size = htonl(CPL_TX_TNL_LSO_SIZE_V(skb->len));
+}
+
 /**
  *	t4_eth_xmit - add a packet to an Ethernet Tx queue
  *	@skb: the packet
@@ -1177,6 +1286,9 @@ netdev_tx_t t4_eth_xmit(struct sk_buff *skb, struct net_device *dev)
 	bool immediate = false;
 	int len, max_pkt_len;
 	bool ptp_enabled = is_ptp_enabled(skb, dev);
+	unsigned int chip_ver;
+	enum cpl_tx_tnl_lso_type tnl_type = TX_TNL_TYPE_OPAQUE;
+
 #ifdef CONFIG_CHELSIO_T4_FCOE
 	int err;
 #endif /* CONFIG_CHELSIO_T4_FCOE */
@@ -1199,6 +1311,12 @@ out_free:	dev_kfree_skb_any(skb);
 
 	pi = netdev_priv(dev);
 	adap = pi->adapter;
+	ssi = skb_shinfo(skb);
+#ifdef CONFIG_CHELSIO_IPSEC_INLINE
+	if (xfrm_offload(skb) && !ssi->gso_size)
+		return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev);
+#endif /* CHELSIO_IPSEC_INLINE */
+
 	qidx = skb_get_queue_mapping(skb);
 	if (ptp_enabled) {
 		spin_lock(&adap->ptp_lock);
@@ -1215,7 +1333,7 @@ out_free:	dev_kfree_skb_any(skb);
 	}
 	skb_tx_timestamp(skb);
 
-	reclaim_completed_tx(adap, &q->q, true);
+	cxgb4_reclaim_completed_tx(adap, &q->q, true);
 	cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
 
 #ifdef CONFIG_CHELSIO_T4_FCOE
@@ -1227,7 +1345,8 @@ out_free:	dev_kfree_skb_any(skb);
 	}
 #endif /* CONFIG_CHELSIO_T4_FCOE */
 
-	flits = calc_tx_flits(skb);
+	chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
+	flits = calc_tx_flits(skb, chip_ver);
 	ndesc = flits_to_desc(flits);
 	credits = txq_avail(&q->q) - ndesc;
 
@@ -1241,11 +1360,14 @@ out_free:	dev_kfree_skb_any(skb);
 		return NETDEV_TX_BUSY;
 	}
 
-	if (is_eth_imm(skb))
+	if (is_eth_imm(skb, chip_ver))
 		immediate = true;
 
+	if (skb->encapsulation && chip_ver > CHELSIO_T5)
+		tnl_type = cxgb_encap_offload_supported(skb);
+
 	if (!immediate &&
-	    unlikely(map_skb(adap->pdev_dev, skb, addr) < 0)) {
+	    unlikely(cxgb4_map_skb(adap->pdev_dev, skb, addr) < 0)) {
 		q->mapping_err++;
 		if (ptp_enabled)
 			spin_unlock(&adap->ptp_lock);
@@ -1264,39 +1386,63 @@ out_free:	dev_kfree_skb_any(skb);
 	end = (u64 *)wr + flits;
 
 	len = immediate ? skb->len : 0;
-	ssi = skb_shinfo(skb);
 	if (ssi->gso_size) {
 		struct cpl_tx_pkt_lso *lso = (void *)wr;
 		bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0;
 		int l3hdr_len = skb_network_header_len(skb);
 		int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN;
+		struct cpl_tx_tnl_lso *tnl_lso = (void *)(wr + 1);
+
+		if (tnl_type)
+			len += sizeof(*tnl_lso);
+		else
+			len += sizeof(*lso);
 
-		len += sizeof(*lso);
 		wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
 				       FW_WR_IMMDLEN_V(len));
-		lso->c.lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
-					LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
-					LSO_IPV6_V(v6) |
-					LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
-					LSO_IPHDR_LEN_V(l3hdr_len / 4) |
-					LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
-		lso->c.ipid_ofst = htons(0);
-		lso->c.mss = htons(ssi->gso_size);
-		lso->c.seqno_offset = htonl(0);
-		if (is_t4(adap->params.chip))
-			lso->c.len = htonl(skb->len);
-		else
-			lso->c.len = htonl(LSO_T5_XFER_SIZE_V(skb->len));
-		cpl = (void *)(lso + 1);
+		if (tnl_type) {
+			struct iphdr *iph = ip_hdr(skb);
 
-		if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
-			cntrl =	TXPKT_ETHHDR_LEN_V(eth_xtra_len);
-		else
-			cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
+			t6_fill_tnl_lso(skb, tnl_lso, tnl_type);
+			cpl = (void *)(tnl_lso + 1);
+			/* Driver is expected to compute partial checksum that
+			 * does not include the IP Total Length.
+			 */
+			if (iph->version == 4) {
+				iph->check = 0;
+				iph->tot_len = 0;
+				iph->check = (u16)(~ip_fast_csum((u8 *)iph,
+								 iph->ihl));
+			}
+			if (skb->ip_summed == CHECKSUM_PARTIAL)
+				cntrl = hwcsum(adap->params.chip, skb);
+		} else {
+			lso->c.lso_ctrl = htonl(LSO_OPCODE_V(CPL_TX_PKT_LSO) |
+					  LSO_FIRST_SLICE_F | LSO_LAST_SLICE_F |
+					  LSO_IPV6_V(v6) |
+					  LSO_ETHHDR_LEN_V(eth_xtra_len / 4) |
+					  LSO_IPHDR_LEN_V(l3hdr_len / 4) |
+					  LSO_TCPHDR_LEN_V(tcp_hdr(skb)->doff));
+			lso->c.ipid_ofst = htons(0);
+			lso->c.mss = htons(ssi->gso_size);
+			lso->c.seqno_offset = htonl(0);
+			if (is_t4(adap->params.chip))
+				lso->c.len = htonl(skb->len);
+			else
+				lso->c.len =
+					htonl(LSO_T5_XFER_SIZE_V(skb->len));
+			cpl = (void *)(lso + 1);
 
-		cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
-					   TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
-			 TXPKT_IPHDR_LEN_V(l3hdr_len);
+			if (CHELSIO_CHIP_VERSION(adap->params.chip)
+			    <= CHELSIO_T5)
+				cntrl =	TXPKT_ETHHDR_LEN_V(eth_xtra_len);
+			else
+				cntrl = T6_TXPKT_ETHHDR_LEN_V(eth_xtra_len);
+
+			cntrl |= TXPKT_CSUM_TYPE_V(v6 ?
+				 TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) |
+				 TXPKT_IPHDR_LEN_V(l3hdr_len);
+		}
 		q->tso++;
 		q->tx_cso += ssi->gso_segs;
 	} else {
@@ -1341,13 +1487,13 @@ out_free:	dev_kfree_skb_any(skb);
 	cpl->ctrl1 = cpu_to_be64(cntrl);
 
 	if (immediate) {
-		inline_tx_skb(skb, &q->q, cpl + 1);
+		cxgb4_inline_tx_skb(skb, &q->q, cpl + 1);
 		dev_consume_skb_any(skb);
 	} else {
 		int last_desc;
 
-		write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1), end, 0,
-			  addr);
+		cxgb4_write_sgl(skb, &q->q, (struct ulptx_sgl *)(cpl + 1),
+				end, 0, addr);
 		skb_orphan(skb);
 
 		last_desc = q->q.pidx + ndesc - 1;
@@ -1359,7 +1505,7 @@ out_free:	dev_kfree_skb_any(skb);
 
 	txq_advance(&q->q, ndesc);
 
-	ring_tx_db(adap, &q->q, ndesc);
+	cxgb4_ring_tx_db(adap, &q->q, ndesc);
 	if (ptp_enabled)
 		spin_unlock(&adap->ptp_lock);
 	return NETDEV_TX_OK;
@@ -1369,9 +1515,9 @@ out_free:	dev_kfree_skb_any(skb);
  *	reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
  *	@q: the SGE control Tx queue
  *
- *	This is a variant of reclaim_completed_tx() that is used for Tx queues
- *	that send only immediate data (presently just the control queues) and
- *	thus do not have any sk_buffs to release.
+ *	This is a variant of cxgb4_reclaim_completed_tx() that is used
+ *	for Tx queues that send only immediate data (presently just
+ *	the control queues) and	thus do not have any sk_buffs to release.
  */
 static inline void reclaim_completed_tx_imm(struct sge_txq *q)
 {
@@ -1446,13 +1592,13 @@ static int ctrl_xmit(struct sge_ctrl_txq *q, struct sk_buff *skb)
 	}
 
 	wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
-	inline_tx_skb(skb, &q->q, wr);
+	cxgb4_inline_tx_skb(skb, &q->q, wr);
 
 	txq_advance(&q->q, ndesc);
 	if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES))
 		ctrlq_check_stop(q, wr);
 
-	ring_tx_db(q->adap, &q->q, ndesc);
+	cxgb4_ring_tx_db(q->adap, &q->q, ndesc);
 	spin_unlock(&q->sendq.lock);
 
 	kfree_skb(skb);
@@ -1487,7 +1633,7 @@ static void restart_ctrlq(unsigned long data)
 		txq_advance(&q->q, ndesc);
 		spin_unlock(&q->sendq.lock);
 
-		inline_tx_skb(skb, &q->q, wr);
+		cxgb4_inline_tx_skb(skb, &q->q, wr);
 		kfree_skb(skb);
 
 		if (unlikely(txq_avail(&q->q) < TXQ_STOP_THRES)) {
@@ -1500,14 +1646,15 @@ static void restart_ctrlq(unsigned long data)
 			}
 		}
 		if (written > 16) {
-			ring_tx_db(q->adap, &q->q, written);
+			cxgb4_ring_tx_db(q->adap, &q->q, written);
 			written = 0;
 		}
 		spin_lock(&q->sendq.lock);
 	}
 	q->full = 0;
-ringdb: if (written)
-		ring_tx_db(q->adap, &q->q, written);
+ringdb:
+	if (written)
+		cxgb4_ring_tx_db(q->adap, &q->q, written);
 	spin_unlock(&q->sendq.lock);
 }
 
@@ -1650,7 +1797,7 @@ static void service_ofldq(struct sge_uld_txq *q)
 		 */
 		spin_unlock(&q->sendq.lock);
 
-		reclaim_completed_tx(q->adap, &q->q, false);
+		cxgb4_reclaim_completed_tx(q->adap, &q->q, false);
 
 		flits = skb->priority;                /* previously saved */
 		ndesc = flits_to_desc(flits);
@@ -1661,9 +1808,9 @@ static void service_ofldq(struct sge_uld_txq *q)
 
 		pos = (u64 *)&q->q.desc[q->q.pidx];
 		if (is_ofld_imm(skb))
-			inline_tx_skb(skb, &q->q, pos);
-		else if (map_skb(q->adap->pdev_dev, skb,
-				 (dma_addr_t *)skb->head)) {
+			cxgb4_inline_tx_skb(skb, &q->q, pos);
+		else if (cxgb4_map_skb(q->adap->pdev_dev, skb,
+				       (dma_addr_t *)skb->head)) {
 			txq_stop_maperr(q);
 			spin_lock(&q->sendq.lock);
 			break;
@@ -1694,9 +1841,9 @@ static void service_ofldq(struct sge_uld_txq *q)
 				pos = (void *)txq->desc;
 			}
 
-			write_sgl(skb, &q->q, (void *)pos,
-				  end, hdr_len,
-				  (dma_addr_t *)skb->head);
+			cxgb4_write_sgl(skb, &q->q, (void *)pos,
+					end, hdr_len,
+					(dma_addr_t *)skb->head);
 #ifdef CONFIG_NEED_DMA_MAP_STATE
 			skb->dev = q->adap->port[0];
 			skb->destructor = deferred_unmap_destructor;
@@ -1710,7 +1857,7 @@ static void service_ofldq(struct sge_uld_txq *q)
 		txq_advance(&q->q, ndesc);
 		written += ndesc;
 		if (unlikely(written > 32)) {
-			ring_tx_db(q->adap, &q->q, written);
+			cxgb4_ring_tx_db(q->adap, &q->q, written);
 			written = 0;
 		}
 
@@ -1725,7 +1872,7 @@ static void service_ofldq(struct sge_uld_txq *q)
 			kfree_skb(skb);
 	}
 	if (likely(written))
-		ring_tx_db(q->adap, &q->q, written);
+		cxgb4_ring_tx_db(q->adap, &q->q, written);
 
 	/*Indicate that no thread is processing the Pending Send Queue
 	 * currently.
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
index 375ef86a84da..920bccd6bc40 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c
@@ -195,9 +195,11 @@ static void t4_report_fw_error(struct adapter *adap)
 	u32 pcie_fw;
 
 	pcie_fw = t4_read_reg(adap, PCIE_FW_A);
-	if (pcie_fw & PCIE_FW_ERR_F)
+	if (pcie_fw & PCIE_FW_ERR_F) {
 		dev_err(adap->pdev_dev, "Firmware reports adapter error: %s\n",
 			reason[PCIE_FW_EVAL_G(pcie_fw)]);
+		adap->flags &= ~FW_OK;
+	}
 }
 
 /*
@@ -317,9 +319,9 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 	 * wait [for a while] till we're at the front [or bail out with an
 	 * EBUSY] ...
 	 */
-	spin_lock(&adap->mbox_lock);
+	spin_lock_bh(&adap->mbox_lock);
 	list_add_tail(&entry.list, &adap->mlist.list);
-	spin_unlock(&adap->mbox_lock);
+	spin_unlock_bh(&adap->mbox_lock);
 
 	delay_idx = 0;
 	ms = delay[0];
@@ -332,9 +334,9 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 		 */
 		pcie_fw = t4_read_reg(adap, PCIE_FW_A);
 		if (i > FW_CMD_MAX_TIMEOUT || (pcie_fw & PCIE_FW_ERR_F)) {
-			spin_lock(&adap->mbox_lock);
+			spin_lock_bh(&adap->mbox_lock);
 			list_del(&entry.list);
-			spin_unlock(&adap->mbox_lock);
+			spin_unlock_bh(&adap->mbox_lock);
 			ret = (pcie_fw & PCIE_FW_ERR_F) ? -ENXIO : -EBUSY;
 			t4_record_mbox(adap, cmd, size, access, ret);
 			return ret;
@@ -365,9 +367,9 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 	for (i = 0; v == MBOX_OWNER_NONE && i < 3; i++)
 		v = MBOWNER_G(t4_read_reg(adap, ctl_reg));
 	if (v != MBOX_OWNER_DRV) {
-		spin_lock(&adap->mbox_lock);
+		spin_lock_bh(&adap->mbox_lock);
 		list_del(&entry.list);
-		spin_unlock(&adap->mbox_lock);
+		spin_unlock_bh(&adap->mbox_lock);
 		ret = (v == MBOX_OWNER_FW) ? -EBUSY : -ETIMEDOUT;
 		t4_record_mbox(adap, cmd, size, access, ret);
 		return ret;
@@ -418,9 +420,9 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 			execute = i + ms;
 			t4_record_mbox(adap, cmd_rpl,
 				       MBOX_LEN, access, execute);
-			spin_lock(&adap->mbox_lock);
+			spin_lock_bh(&adap->mbox_lock);
 			list_del(&entry.list);
-			spin_unlock(&adap->mbox_lock);
+			spin_unlock_bh(&adap->mbox_lock);
 			return -FW_CMD_RETVAL_G((int)res);
 		}
 	}
@@ -430,9 +432,9 @@ int t4_wr_mbox_meat_timeout(struct adapter *adap, int mbox, const void *cmd,
 	dev_err(adap->pdev_dev, "command %#x in mailbox %d timed out\n",
 		*(const u8 *)cmd, mbox);
 	t4_report_fw_error(adap);
-	spin_lock(&adap->mbox_lock);
+	spin_lock_bh(&adap->mbox_lock);
 	list_del(&entry.list);
-	spin_unlock(&adap->mbox_lock);
+	spin_unlock_bh(&adap->mbox_lock);
 	t4_fatal_err(adap);
 	return ret;
 }
@@ -524,11 +526,14 @@ int t4_memory_rw(struct adapter *adap, int win, int mtype, u32 addr,
 	 * MEM_EDC1 = 1
 	 * MEM_MC   = 2 -- MEM_MC for chips with only 1 memory controller
 	 * MEM_MC1  = 3 -- for chips with 2 memory controllers (e.g. T5)
+	 * MEM_HMA  = 4
 	 */
 	edc_size  = EDRAM0_SIZE_G(t4_read_reg(adap, MA_EDRAM0_BAR_A));
-	if (mtype != MEM_MC1)
+	if (mtype == MEM_HMA) {
+		memoffset = 2 * (edc_size * 1024 * 1024);
+	} else if (mtype != MEM_MC1) {
 		memoffset = (mtype * (edc_size * 1024 * 1024));
-	else {
+	} else {
 		mc_size = EXT_MEM0_SIZE_G(t4_read_reg(adap,
 						      MA_EXT_MEMORY0_BAR_A));
 		memoffset = (MEM_MC0 * edc_size + mc_size) * 1024 * 1024;
@@ -2632,7 +2637,6 @@ void t4_get_regs(struct adapter *adap, void *buf, size_t buf_size)
 }
 
 #define EEPROM_STAT_ADDR   0x7bfc
-#define VPD_SIZE           0x800
 #define VPD_BASE           0x400
 #define VPD_BASE_OLD       0
 #define VPD_LEN            1024
@@ -2699,15 +2703,6 @@ int t4_get_raw_vpd_params(struct adapter *adapter, struct vpd_params *p)
 	if (!vpd)
 		return -ENOMEM;
 
-	/* We have two VPD data structures stored in the adapter VPD area.
-	 * By default, Linux calculates the size of the VPD area by traversing
-	 * the first VPD area at offset 0x0, so we need to tell the OS what
-	 * our real VPD size is.
-	 */
-	ret = pci_set_vpd_size(adapter->pdev, VPD_SIZE);
-	if (ret < 0)
-		goto out;
-
 	/* Card information normally starts at VPD_BASE but early cards had
 	 * it at 0.
 	 */
@@ -4923,6 +4918,14 @@ void t4_intr_disable(struct adapter *adapter)
 	t4_set_reg_field(adapter, PL_INT_MAP0_A, 1 << pf, 0);
 }
 
+unsigned int t4_chip_rss_size(struct adapter *adap)
+{
+	if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
+		return RSS_NENTRIES;
+	else
+		return T6_RSS_NENTRIES;
+}
+
 /**
  *	t4_config_rss_range - configure a portion of the RSS mapping table
  *	@adapter: the adapter
@@ -5061,10 +5064,11 @@ static int rd_rss_row(struct adapter *adap, int row, u32 *val)
  */
 int t4_read_rss(struct adapter *adapter, u16 *map)
 {
+	int i, ret, nentries;
 	u32 val;
-	int i, ret;
 
-	for (i = 0; i < RSS_NENTRIES / 2; ++i) {
+	nentries = t4_chip_rss_size(adapter);
+	for (i = 0; i < nentries / 2; ++i) {
 		ret = rd_rss_row(adapter, i, &val);
 		if (ret)
 			return ret;
@@ -5076,7 +5080,7 @@ int t4_read_rss(struct adapter *adapter, u16 *map)
 
 static unsigned int t4_use_ldst(struct adapter *adap)
 {
-	return (adap->flags & FW_OK) || !adap->use_bd;
+	return (adap->flags & FW_OK) && !adap->use_bd;
 }
 
 /**
@@ -6071,6 +6075,7 @@ const char *t4_get_port_type_description(enum fw_port_type port_type)
 		"CR2_QSFP",
 		"SFP28",
 		"KR_SFP28",
+		"KR_XLAUI"
 	};
 
 	if (port_type < ARRAY_SIZE(port_type_description))
@@ -6526,18 +6531,21 @@ void t4_sge_decode_idma_state(struct adapter *adapter, int state)
  *      t4_sge_ctxt_flush - flush the SGE context cache
  *      @adap: the adapter
  *      @mbox: mailbox to use for the FW command
+ *      @ctx_type: Egress or Ingress
  *
  *      Issues a FW command through the given mailbox to flush the
  *      SGE context cache.
  */
-int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox)
+int t4_sge_ctxt_flush(struct adapter *adap, unsigned int mbox, int ctxt_type)
 {
 	int ret;
 	u32 ldst_addrspace;
 	struct fw_ldst_cmd c;
 
 	memset(&c, 0, sizeof(c));
-	ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_SGE_EGRC);
+	ldst_addrspace = FW_LDST_CMD_ADDRSPACE_V(ctxt_type == CTXT_EGRESS ?
+						 FW_LDST_ADDRSPC_SGE_EGRC :
+						 FW_LDST_ADDRSPC_SGE_INGC);
 	c.op_to_addrspace = cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
 					FW_CMD_REQUEST_F | FW_CMD_READ_F |
 					ldst_addrspace);
@@ -7450,6 +7458,112 @@ int t4_set_rxmode(struct adapter *adap, unsigned int mbox, unsigned int viid,
 	return t4_wr_mbox_meat(adap, mbox, &c, sizeof(c), NULL, sleep_ok);
 }
 
+/**
+ *	t4_free_raw_mac_filt - Frees a raw mac entry in mps tcam
+ *	@adap: the adapter
+ *	@viid: the VI id
+ *	@addr: the MAC address
+ *	@mask: the mask
+ *	@idx: index of the entry in mps tcam
+ *	@lookup_type: MAC address for inner (1) or outer (0) header
+ *	@port_id: the port index
+ *	@sleep_ok: call is allowed to sleep
+ *
+ *	Removes the mac entry at the specified index using raw mac interface.
+ *
+ *	Returns a negative error number on failure.
+ */
+int t4_free_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			 const u8 *addr, const u8 *mask, unsigned int idx,
+			 u8 lookup_type, u8 port_id, bool sleep_ok)
+{
+	struct fw_vi_mac_cmd c;
+	struct fw_vi_mac_raw *p = &c.u.raw;
+	u32 val;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
+				   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
+				   FW_CMD_EXEC_V(0) |
+				   FW_VI_MAC_CMD_VIID_V(viid));
+	val = FW_CMD_LEN16_V(1) |
+	      FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_RAW);
+	c.freemacs_to_len16 = cpu_to_be32(FW_VI_MAC_CMD_FREEMACS_V(0) |
+					  FW_CMD_LEN16_V(val));
+
+	p->raw_idx_pkd = cpu_to_be32(FW_VI_MAC_CMD_RAW_IDX_V(idx) |
+				     FW_VI_MAC_ID_BASED_FREE);
+
+	/* Lookup Type. Outer header: 0, Inner header: 1 */
+	p->data0_pkd = cpu_to_be32(DATALKPTYPE_V(lookup_type) |
+				   DATAPORTNUM_V(port_id));
+	/* Lookup mask and port mask */
+	p->data0m_pkd = cpu_to_be64(DATALKPTYPE_V(DATALKPTYPE_M) |
+				    DATAPORTNUM_V(DATAPORTNUM_M));
+
+	/* Copy the address and the mask */
+	memcpy((u8 *)&p->data1[0] + 2, addr, ETH_ALEN);
+	memcpy((u8 *)&p->data1m[0] + 2, mask, ETH_ALEN);
+
+	return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+}
+
+/**
+ *	t4_alloc_raw_mac_filt - Adds a mac entry in mps tcam
+ *	@adap: the adapter
+ *	@viid: the VI id
+ *	@mac: the MAC address
+ *	@mask: the mask
+ *	@idx: index at which to add this entry
+ *	@port_id: the port index
+ *	@lookup_type: MAC address for inner (1) or outer (0) header
+ *	@sleep_ok: call is allowed to sleep
+ *
+ *	Adds the mac entry at the specified index using raw mac interface.
+ *
+ *	Returns a negative error number or the allocated index for this mac.
+ */
+int t4_alloc_raw_mac_filt(struct adapter *adap, unsigned int viid,
+			  const u8 *addr, const u8 *mask, unsigned int idx,
+			  u8 lookup_type, u8 port_id, bool sleep_ok)
+{
+	int ret = 0;
+	struct fw_vi_mac_cmd c;
+	struct fw_vi_mac_raw *p = &c.u.raw;
+	u32 val;
+
+	memset(&c, 0, sizeof(c));
+	c.op_to_viid = cpu_to_be32(FW_CMD_OP_V(FW_VI_MAC_CMD) |
+				   FW_CMD_REQUEST_F | FW_CMD_WRITE_F |
+				   FW_VI_MAC_CMD_VIID_V(viid));
+	val = FW_CMD_LEN16_V(1) |
+	      FW_VI_MAC_CMD_ENTRY_TYPE_V(FW_VI_MAC_TYPE_RAW);
+	c.freemacs_to_len16 = cpu_to_be32(val);
+
+	/* Specify that this is an inner mac address */
+	p->raw_idx_pkd = cpu_to_be32(FW_VI_MAC_CMD_RAW_IDX_V(idx));
+
+	/* Lookup Type. Outer header: 0, Inner header: 1 */
+	p->data0_pkd = cpu_to_be32(DATALKPTYPE_V(lookup_type) |
+				   DATAPORTNUM_V(port_id));
+	/* Lookup mask and port mask */
+	p->data0m_pkd = cpu_to_be64(DATALKPTYPE_V(DATALKPTYPE_M) |
+				    DATAPORTNUM_V(DATAPORTNUM_M));
+
+	/* Copy the address and the mask */
+	memcpy((u8 *)&p->data1[0] + 2, addr, ETH_ALEN);
+	memcpy((u8 *)&p->data1m[0] + 2, mask, ETH_ALEN);
+
+	ret = t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, sleep_ok);
+	if (ret == 0) {
+		ret = FW_VI_MAC_CMD_RAW_IDX_G(be32_to_cpu(p->raw_idx_pkd));
+		if (ret != idx)
+			ret = -ENOMEM;
+	}
+
+	return ret;
+}
+
 /**
  *	t4_alloc_mac_filt - allocates exact-match filters for MAC addresses
  *	@adap: the adapter
@@ -8491,22 +8605,6 @@ static int t4_get_flash_params(struct adapter *adap)
 	return 0;
 }
 
-static void set_pcie_completion_timeout(struct adapter *adapter, u8 range)
-{
-	u16 val;
-	u32 pcie_cap;
-
-	pcie_cap = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
-	if (pcie_cap) {
-		pci_read_config_word(adapter->pdev,
-				     pcie_cap + PCI_EXP_DEVCTL2, &val);
-		val &= ~PCI_EXP_DEVCTL2_COMP_TIMEOUT;
-		val |= range;
-		pci_write_config_word(adapter->pdev,
-				      pcie_cap + PCI_EXP_DEVCTL2, val);
-	}
-}
-
 /**
  *	t4_prep_adapter - prepare SW and HW for operation
  *	@adapter: the adapter
@@ -8592,8 +8690,9 @@ int t4_prep_adapter(struct adapter *adapter)
 	adapter->params.portvec = 1;
 	adapter->params.vpd.cclk = 50000;
 
-	/* Set pci completion timeout value to 4 seconds. */
-	set_pcie_completion_timeout(adapter, 0xd);
+	/* Set PCIe completion timeout to 4 seconds. */
+	pcie_capability_clear_and_set_word(adapter->pdev, PCI_EXP_DEVCTL2,
+					   PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0xd);
 	return 0;
 }
 
@@ -9736,3 +9835,91 @@ int t4_sched_params(struct adapter *adapter, int type, int level, int mode,
 	return t4_wr_mbox_meat(adapter, adapter->mbox, &cmd, sizeof(cmd),
 			       NULL, 1);
 }
+
+/**
+ *	t4_i2c_rd - read I2C data from adapter
+ *	@adap: the adapter
+ *	@port: Port number if per-port device; <0 if not
+ *	@devid: per-port device ID or absolute device ID
+ *	@offset: byte offset into device I2C space
+ *	@len: byte length of I2C space data
+ *	@buf: buffer in which to return I2C data
+ *
+ *	Reads the I2C data from the indicated device and location.
+ */
+int t4_i2c_rd(struct adapter *adap, unsigned int mbox, int port,
+	      unsigned int devid, unsigned int offset,
+	      unsigned int len, u8 *buf)
+{
+	struct fw_ldst_cmd ldst_cmd, ldst_rpl;
+	unsigned int i2c_max = sizeof(ldst_cmd.u.i2c.data);
+	int ret = 0;
+
+	if (len > I2C_PAGE_SIZE)
+		return -EINVAL;
+
+	/* Dont allow reads that spans multiple pages */
+	if (offset < I2C_PAGE_SIZE && offset + len > I2C_PAGE_SIZE)
+		return -EINVAL;
+
+	memset(&ldst_cmd, 0, sizeof(ldst_cmd));
+	ldst_cmd.op_to_addrspace =
+		cpu_to_be32(FW_CMD_OP_V(FW_LDST_CMD) |
+			    FW_CMD_REQUEST_F |
+			    FW_CMD_READ_F |
+			    FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_I2C));
+	ldst_cmd.cycles_to_len16 = cpu_to_be32(FW_LEN16(ldst_cmd));
+	ldst_cmd.u.i2c.pid = (port < 0 ? 0xff : port);
+	ldst_cmd.u.i2c.did = devid;
+
+	while (len > 0) {
+		unsigned int i2c_len = (len < i2c_max) ? len : i2c_max;
+
+		ldst_cmd.u.i2c.boffset = offset;
+		ldst_cmd.u.i2c.blen = i2c_len;
+
+		ret = t4_wr_mbox(adap, mbox, &ldst_cmd, sizeof(ldst_cmd),
+				 &ldst_rpl);
+		if (ret)
+			break;
+
+		memcpy(buf, ldst_rpl.u.i2c.data, i2c_len);
+		offset += i2c_len;
+		buf += i2c_len;
+		len -= i2c_len;
+	}
+
+	return ret;
+}
+
+/**
+ *      t4_set_vlan_acl - Set a VLAN id for the specified VF
+ *      @adapter: the adapter
+ *      @mbox: mailbox to use for the FW command
+ *      @vf: one of the VFs instantiated by the specified PF
+ *      @vlan: The vlanid to be set
+ */
+int t4_set_vlan_acl(struct adapter *adap, unsigned int mbox, unsigned int vf,
+		    u16 vlan)
+{
+	struct fw_acl_vlan_cmd vlan_cmd;
+	unsigned int enable;
+
+	enable = (vlan ? FW_ACL_VLAN_CMD_EN_F : 0);
+	memset(&vlan_cmd, 0, sizeof(vlan_cmd));
+	vlan_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_VLAN_CMD) |
+					 FW_CMD_REQUEST_F |
+					 FW_CMD_WRITE_F |
+					 FW_CMD_EXEC_F |
+					 FW_ACL_VLAN_CMD_PFN_V(adap->pf) |
+					 FW_ACL_VLAN_CMD_VFN_V(vf));
+	vlan_cmd.en_to_len16 = cpu_to_be32(enable | FW_LEN16(vlan_cmd));
+	/* Drop all packets that donot match vlan id */
+	vlan_cmd.dropnovlan_fm = FW_ACL_VLAN_CMD_FM_F;
+	if (enable != 0) {
+		vlan_cmd.nvlan = 1;
+		vlan_cmd.vlanid[0] = cpu_to_be16(vlan);
+	}
+
+	return t4_wr_mbox(adap, adap->mbox, &vlan_cmd, sizeof(vlan_cmd), NULL);
+}
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h
index a964ed184356..361d5032c288 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.h
@@ -38,21 +38,22 @@
 #include <linux/types.h>
 
 enum {
-	NCHAN          = 4,     /* # of HW channels */
-	MAX_MTU        = 9600,  /* max MAC MTU, excluding header + FCS */
-	EEPROMSIZE     = 17408, /* Serial EEPROM physical size */
-	EEPROMVSIZE    = 32768, /* Serial EEPROM virtual address space size */
-	EEPROMPFSIZE   = 1024,  /* EEPROM writable area size for PFn, n>0 */
-	RSS_NENTRIES   = 2048,  /* # of entries in RSS mapping table */
-	TCB_SIZE       = 128,   /* TCB size */
-	NMTUS          = 16,    /* size of MTU table */
-	NCCTRL_WIN     = 32,    /* # of congestion control windows */
-	NTX_SCHED      = 8,     /* # of HW Tx scheduling queues */
-	PM_NSTATS      = 5,     /* # of PM stats */
-	T6_PM_NSTATS   = 7,     /* # of PM stats in T6 */
-	MBOX_LEN       = 64,    /* mailbox size in bytes */
-	TRACE_LEN      = 112,   /* length of trace data and mask */
-	FILTER_OPT_LEN = 36,    /* filter tuple width for optional components */
+	NCHAN           = 4,    /* # of HW channels */
+	MAX_MTU         = 9600, /* max MAC MTU, excluding header + FCS */
+	EEPROMSIZE      = 17408,/* Serial EEPROM physical size */
+	EEPROMVSIZE     = 32768,/* Serial EEPROM virtual address space size */
+	EEPROMPFSIZE    = 1024, /* EEPROM writable area size for PFn, n>0 */
+	RSS_NENTRIES    = 2048, /* # of entries in RSS mapping table */
+	T6_RSS_NENTRIES = 4096, /* # of entries in RSS mapping table */
+	TCB_SIZE        = 128,  /* TCB size */
+	NMTUS           = 16,   /* size of MTU table */
+	NCCTRL_WIN      = 32,   /* # of congestion control windows */
+	NTX_SCHED       = 8,    /* # of HW Tx scheduling queues */
+	PM_NSTATS       = 5,    /* # of PM stats */
+	T6_PM_NSTATS    = 7,    /* # of PM stats in T6 */
+	MBOX_LEN        = 64,   /* mailbox size in bytes */
+	TRACE_LEN       = 112,  /* length of trace data and mask */
+	FILTER_OPT_LEN  = 36,   /* filter tuple width for optional components */
 };
 
 enum {
@@ -70,7 +71,9 @@ enum {
 
 /* SGE context types */
 enum ctxt_type {
-	CTXT_FLM = 2,
+	CTXT_EGRESS,
+	CTXT_INGRESS,
+	CTXT_FLM,
 	CTXT_CNM,
 };
 
@@ -284,4 +287,14 @@ enum {
 #define SGE_TIMESTAMP_V(x) ((__u64)(x) << SGE_TIMESTAMP_S)
 #define SGE_TIMESTAMP_G(x) (((__u64)(x) >> SGE_TIMESTAMP_S) & SGE_TIMESTAMP_M)
 
+#define I2C_DEV_ADDR_A0		0xa0
+#define I2C_DEV_ADDR_A2		0xa2
+#define I2C_PAGE_SIZE		0x100
+#define SFP_DIAG_TYPE_ADDR	0x5c
+#define SFP_DIAG_TYPE_LEN	0x1
+#define SFF_8472_COMP_ADDR	0x5e
+#define SFF_8472_COMP_LEN	0x1
+#define SFF_REV_ADDR		0x1
+#define SFF_REV_LEN		0x1
+
 #endif /* __T4_HW_H */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
index 7e12f241145b..d0db4427b77e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_msg.h
@@ -107,6 +107,7 @@ enum {
 
 	CPL_FW6_MSG           = 0xE0,
 	CPL_FW6_PLD           = 0xE1,
+	CPL_TX_TNL_LSO        = 0xEC,
 	CPL_TX_PKT_LSO        = 0xED,
 	CPL_TX_PKT_XT         = 0xEE,
 
@@ -1479,6 +1480,169 @@ struct ulp_txpkt {
 #define ULP_TXPKT_RO_V(x) ((x) << ULP_TXPKT_RO_S)
 #define ULP_TXPKT_RO_F ULP_TXPKT_RO_V(1U)
 
+enum cpl_tx_tnl_lso_type {
+	TX_TNL_TYPE_OPAQUE,
+	TX_TNL_TYPE_NVGRE,
+	TX_TNL_TYPE_VXLAN,
+	TX_TNL_TYPE_GENEVE,
+};
+
+struct cpl_tx_tnl_lso {
+	__be32 op_to_IpIdSplitOut;
+	__be16 IpIdOffsetOut;
+	__be16 UdpLenSetOut_to_TnlHdrLen;
+	__be64 r1;
+	__be32 Flow_to_TcpHdrLen;
+	__be16 IpIdOffset;
+	__be16 IpIdSplit_to_Mss;
+	__be32 TCPSeqOffset;
+	__be32 EthLenOffset_Size;
+	/* encapsulated CPL (TX_PKT_XT) follows here */
+};
+
+#define CPL_TX_TNL_LSO_OPCODE_S		24
+#define CPL_TX_TNL_LSO_OPCODE_M		0xff
+#define CPL_TX_TNL_LSO_OPCODE_V(x)      ((x) << CPL_TX_TNL_LSO_OPCODE_S)
+#define CPL_TX_TNL_LSO_OPCODE_G(x)      \
+	(((x) >> CPL_TX_TNL_LSO_OPCODE_S) & CPL_TX_TNL_LSO_OPCODE_M)
+
+#define CPL_TX_TNL_LSO_FIRST_S		23
+#define CPL_TX_TNL_LSO_FIRST_M		0x1
+#define CPL_TX_TNL_LSO_FIRST_V(x)	((x) << CPL_TX_TNL_LSO_FIRST_S)
+#define CPL_TX_TNL_LSO_FIRST_G(x)	\
+	(((x) >> CPL_TX_TNL_LSO_FIRST_S) & CPL_TX_TNL_LSO_FIRST_M)
+#define CPL_TX_TNL_LSO_FIRST_F		CPL_TX_TNL_LSO_FIRST_V(1U)
+
+#define CPL_TX_TNL_LSO_LAST_S		22
+#define CPL_TX_TNL_LSO_LAST_M		0x1
+#define CPL_TX_TNL_LSO_LAST_V(x)	((x) << CPL_TX_TNL_LSO_LAST_S)
+#define CPL_TX_TNL_LSO_LAST_G(x)	\
+	(((x) >> CPL_TX_TNL_LSO_LAST_S) & CPL_TX_TNL_LSO_LAST_M)
+#define CPL_TX_TNL_LSO_LAST_F		CPL_TX_TNL_LSO_LAST_V(1U)
+
+#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_S	21
+#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_M	0x1
+#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_V(x) \
+	((x) << CPL_TX_TNL_LSO_ETHHDRLENXOUT_S)
+#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_G(x) \
+	(((x) >> CPL_TX_TNL_LSO_ETHHDRLENXOUT_S) & \
+	 CPL_TX_TNL_LSO_ETHHDRLENXOUT_M)
+#define CPL_TX_TNL_LSO_ETHHDRLENXOUT_F CPL_TX_TNL_LSO_ETHHDRLENXOUT_V(1U)
+
+#define CPL_TX_TNL_LSO_IPV6OUT_S	20
+#define CPL_TX_TNL_LSO_IPV6OUT_M	0x1
+#define CPL_TX_TNL_LSO_IPV6OUT_V(x)	((x) << CPL_TX_TNL_LSO_IPV6OUT_S)
+#define CPL_TX_TNL_LSO_IPV6OUT_G(x)	\
+	(((x) >> CPL_TX_TNL_LSO_IPV6OUT_S) & CPL_TX_TNL_LSO_IPV6OUT_M)
+#define CPL_TX_TNL_LSO_IPV6OUT_F        CPL_TX_TNL_LSO_IPV6OUT_V(1U)
+
+#define CPL_TX_TNL_LSO_ETHHDRLEN_S	16
+#define CPL_TX_TNL_LSO_ETHHDRLEN_M	0xf
+#define CPL_TX_TNL_LSO_ETHHDRLEN_V(x)	((x) << CPL_TX_TNL_LSO_ETHHDRLEN_S)
+#define CPL_TX_TNL_LSO_ETHHDRLEN_G(x)	\
+	(((x) >> CPL_TX_TNL_LSO_ETHHDRLEN_S) & CPL_TX_TNL_LSO_ETHHDRLEN_M)
+
+#define CPL_TX_TNL_LSO_IPHDRLEN_S	4
+#define CPL_TX_TNL_LSO_IPHDRLEN_M	0xfff
+#define CPL_TX_TNL_LSO_IPHDRLEN_V(x)	((x) << CPL_TX_TNL_LSO_IPHDRLEN_S)
+#define CPL_TX_TNL_LSO_IPHDRLEN_G(x)    \
+	(((x) >> CPL_TX_TNL_LSO_IPHDRLEN_S) & CPL_TX_TNL_LSO_IPHDRLEN_M)
+
+#define CPL_TX_TNL_LSO_TCPHDRLEN_S	0
+#define CPL_TX_TNL_LSO_TCPHDRLEN_M	0xf
+#define CPL_TX_TNL_LSO_TCPHDRLEN_V(x)	((x) << CPL_TX_TNL_LSO_TCPHDRLEN_S)
+#define CPL_TX_TNL_LSO_TCPHDRLEN_G(x)   \
+	(((x) >> CPL_TX_TNL_LSO_TCPHDRLEN_S) & CPL_TX_TNL_LSO_TCPHDRLEN_M)
+
+#define CPL_TX_TNL_LSO_MSS_S            0
+#define CPL_TX_TNL_LSO_MSS_M            0x3fff
+#define CPL_TX_TNL_LSO_MSS_V(x)         ((x) << CPL_TX_TNL_LSO_MSS_S)
+#define CPL_TX_TNL_LSO_MSS_G(x)         \
+	(((x) >> CPL_TX_TNL_LSO_MSS_S) & CPL_TX_TNL_LSO_MSS_M)
+
+#define CPL_TX_TNL_LSO_SIZE_S		0
+#define CPL_TX_TNL_LSO_SIZE_M		0xfffffff
+#define CPL_TX_TNL_LSO_SIZE_V(x)	((x) << CPL_TX_TNL_LSO_SIZE_S)
+#define CPL_TX_TNL_LSO_SIZE_G(x)	\
+	(((x) >> CPL_TX_TNL_LSO_SIZE_S) & CPL_TX_TNL_LSO_SIZE_M)
+
+#define CPL_TX_TNL_LSO_ETHHDRLENOUT_S   16
+#define CPL_TX_TNL_LSO_ETHHDRLENOUT_M   0xf
+#define CPL_TX_TNL_LSO_ETHHDRLENOUT_V(x) \
+	((x) << CPL_TX_TNL_LSO_ETHHDRLENOUT_S)
+#define CPL_TX_TNL_LSO_ETHHDRLENOUT_G(x) \
+	(((x) >> CPL_TX_TNL_LSO_ETHHDRLENOUT_S) & CPL_TX_TNL_LSO_ETHHDRLENOUT_M)
+
+#define CPL_TX_TNL_LSO_IPHDRLENOUT_S    4
+#define CPL_TX_TNL_LSO_IPHDRLENOUT_M    0xfff
+#define CPL_TX_TNL_LSO_IPHDRLENOUT_V(x) ((x) << CPL_TX_TNL_LSO_IPHDRLENOUT_S)
+#define CPL_TX_TNL_LSO_IPHDRLENOUT_G(x) \
+	(((x) >> CPL_TX_TNL_LSO_IPHDRLENOUT_S) & CPL_TX_TNL_LSO_IPHDRLENOUT_M)
+
+#define CPL_TX_TNL_LSO_IPHDRCHKOUT_S    3
+#define CPL_TX_TNL_LSO_IPHDRCHKOUT_M    0x1
+#define CPL_TX_TNL_LSO_IPHDRCHKOUT_V(x) ((x) << CPL_TX_TNL_LSO_IPHDRCHKOUT_S)
+#define CPL_TX_TNL_LSO_IPHDRCHKOUT_G(x) \
+	(((x) >> CPL_TX_TNL_LSO_IPHDRCHKOUT_S) & CPL_TX_TNL_LSO_IPHDRCHKOUT_M)
+#define CPL_TX_TNL_LSO_IPHDRCHKOUT_F    CPL_TX_TNL_LSO_IPHDRCHKOUT_V(1U)
+
+#define CPL_TX_TNL_LSO_IPLENSETOUT_S	2
+#define CPL_TX_TNL_LSO_IPLENSETOUT_M	0x1
+#define CPL_TX_TNL_LSO_IPLENSETOUT_V(x) ((x) << CPL_TX_TNL_LSO_IPLENSETOUT_S)
+#define CPL_TX_TNL_LSO_IPLENSETOUT_G(x) \
+	(((x) >> CPL_TX_TNL_LSO_IPLENSETOUT_S) & CPL_TX_TNL_LSO_IPLENSETOUT_M)
+#define CPL_TX_TNL_LSO_IPLENSETOUT_F	CPL_TX_TNL_LSO_IPLENSETOUT_V(1U)
+
+#define CPL_TX_TNL_LSO_IPIDINCOUT_S	1
+#define CPL_TX_TNL_LSO_IPIDINCOUT_M	0x1
+#define CPL_TX_TNL_LSO_IPIDINCOUT_V(x)  ((x) << CPL_TX_TNL_LSO_IPIDINCOUT_S)
+#define CPL_TX_TNL_LSO_IPIDINCOUT_G(x)  \
+	(((x) >> CPL_TX_TNL_LSO_IPIDINCOUT_S) & CPL_TX_TNL_LSO_IPIDINCOUT_M)
+#define CPL_TX_TNL_LSO_IPIDINCOUT_F     CPL_TX_TNL_LSO_IPIDINCOUT_V(1U)
+
+#define CPL_TX_TNL_LSO_UDPCHKCLROUT_S   14
+#define CPL_TX_TNL_LSO_UDPCHKCLROUT_M   0x1
+#define CPL_TX_TNL_LSO_UDPCHKCLROUT_V(x) \
+	((x) << CPL_TX_TNL_LSO_UDPCHKCLROUT_S)
+#define CPL_TX_TNL_LSO_UDPCHKCLROUT_G(x) \
+	(((x) >> CPL_TX_TNL_LSO_UDPCHKCLROUT_S) & \
+	 CPL_TX_TNL_LSO_UDPCHKCLROUT_M)
+#define CPL_TX_TNL_LSO_UDPCHKCLROUT_F   CPL_TX_TNL_LSO_UDPCHKCLROUT_V(1U)
+
+#define CPL_TX_TNL_LSO_UDPLENSETOUT_S   15
+#define CPL_TX_TNL_LSO_UDPLENSETOUT_M   0x1
+#define CPL_TX_TNL_LSO_UDPLENSETOUT_V(x) \
+	((x) << CPL_TX_TNL_LSO_UDPLENSETOUT_S)
+#define CPL_TX_TNL_LSO_UDPLENSETOUT_G(x) \
+	(((x) >> CPL_TX_TNL_LSO_UDPLENSETOUT_S) & \
+	 CPL_TX_TNL_LSO_UDPLENSETOUT_M)
+#define CPL_TX_TNL_LSO_UDPLENSETOUT_F   CPL_TX_TNL_LSO_UDPLENSETOUT_V(1U)
+
+#define CPL_TX_TNL_LSO_TNLTYPE_S	12
+#define CPL_TX_TNL_LSO_TNLTYPE_M	0x3
+#define CPL_TX_TNL_LSO_TNLTYPE_V(x)	((x) << CPL_TX_TNL_LSO_TNLTYPE_S)
+#define CPL_TX_TNL_LSO_TNLTYPE_G(x)	\
+	(((x) >> CPL_TX_TNL_LSO_TNLTYPE_S) & CPL_TX_TNL_LSO_TNLTYPE_M)
+
+#define S_CPL_TX_TNL_LSO_ETHHDRLEN	16
+#define M_CPL_TX_TNL_LSO_ETHHDRLEN	0xf
+#define V_CPL_TX_TNL_LSO_ETHHDRLEN(x)	((x) << S_CPL_TX_TNL_LSO_ETHHDRLEN)
+#define G_CPL_TX_TNL_LSO_ETHHDRLEN(x)	\
+	(((x) >> S_CPL_TX_TNL_LSO_ETHHDRLEN) & M_CPL_TX_TNL_LSO_ETHHDRLEN)
+
+#define CPL_TX_TNL_LSO_TNLHDRLEN_S      0
+#define CPL_TX_TNL_LSO_TNLHDRLEN_M      0xfff
+#define CPL_TX_TNL_LSO_TNLHDRLEN_V(x)	((x) << CPL_TX_TNL_LSO_TNLHDRLEN_S)
+#define CPL_TX_TNL_LSO_TNLHDRLEN_G(x)   \
+	(((x) >> CPL_TX_TNL_LSO_TNLHDRLEN_S) & CPL_TX_TNL_LSO_TNLHDRLEN_M)
+
+#define CPL_TX_TNL_LSO_IPV6_S		20
+#define CPL_TX_TNL_LSO_IPV6_M		0x1
+#define CPL_TX_TNL_LSO_IPV6_V(x)	((x) << CPL_TX_TNL_LSO_IPV6_S)
+#define CPL_TX_TNL_LSO_IPV6_G(x)	\
+	(((x) >> CPL_TX_TNL_LSO_IPV6_S) & CPL_TX_TNL_LSO_IPV6_M)
+#define CPL_TX_TNL_LSO_IPV6_F		CPL_TX_TNL_LSO_IPV6_V(1U)
+
 #define ULP_TX_SC_MORE_S 23
 #define ULP_TX_SC_MORE_V(x) ((x) << ULP_TX_SC_MORE_S)
 #define ULP_TX_SC_MORE_F  ULP_TX_SC_MORE_V(1U)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
index 60cf9e02de5d..51b18035d691 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_pci_id_tbl.h
@@ -183,6 +183,7 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN
 	CH_PCI_ID_TABLE_FENTRY(0x50a9), /* Custom T580-KR */
 	CH_PCI_ID_TABLE_FENTRY(0x50aa), /* Custom T580-CR */
 	CH_PCI_ID_TABLE_FENTRY(0x50ab), /* Custom T520-CR */
+	CH_PCI_ID_TABLE_FENTRY(0x50ac), /* Custom T540-BT */
 
 	/* T6 adapters:
 	 */
@@ -206,6 +207,7 @@ CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN
 	CH_PCI_ID_TABLE_FENTRY(0x6084), /* Custom T64100-CR QSFP28 */
 	CH_PCI_ID_TABLE_FENTRY(0x6085), /* Custom T6240-SO */
 	CH_PCI_ID_TABLE_FENTRY(0x6086), /* Custom T6225-SO-CR */
+	CH_PCI_ID_TABLE_FENTRY(0x6087), /* Custom T6225-CR */
 CH_PCI_DEVICE_ID_TABLE_DEFINE_END;
 
 #endif /* __T4_PCI_ID_TBL_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
index a7cfece72828..a6df73398d17 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4_regs.h
@@ -45,6 +45,9 @@
 #define PF_BASE(idx) (PF0_BASE + (idx) * PF_STRIDE)
 #define PF_REG(idx, reg) (PF_BASE(idx) + (reg))
 
+#define NUM_CIM_CTL_TSCH_CHANNEL_INSTANCES 4
+#define NUM_CIM_CTL_TSCH_CHANNEL_TSCH_CLASS_INSTANCES 16
+
 #define MYPORT_BASE 0x1c000
 #define MYPORT_REG(reg_addr) (MYPORT_BASE + (reg_addr))
 
@@ -961,6 +964,10 @@
 
 #define MA_EXT_MEMORY1_BAR_A 0x7808
 
+#define HMA_MUX_S    5
+#define HMA_MUX_V(x) ((x) << HMA_MUX_S)
+#define HMA_MUX_F    HMA_MUX_V(1U)
+
 #define EXT_MEM1_BASE_S    16
 #define EXT_MEM1_BASE_M    0xfffU
 #define EXT_MEM1_BASE_G(x) (((x) >> EXT_MEM1_BASE_S) & EXT_MEM1_BASE_M)
@@ -2504,6 +2511,28 @@
 #define MPS_RX_MAC_BG_PG_CNT0_A 0x11208
 #define MPS_RX_LPBK_BG_PG_CNT0_A 0x11218
 
+#define MPS_RX_VXLAN_TYPE_A 0x11234
+
+#define VXLAN_EN_S    16
+#define VXLAN_EN_V(x) ((x) << VXLAN_EN_S)
+#define VXLAN_EN_F    VXLAN_EN_V(1U)
+
+#define VXLAN_S    0
+#define VXLAN_M    0xffffU
+#define VXLAN_V(x) ((x) << VXLAN_S)
+#define VXLAN_G(x) (((x) >> VXLAN_S) & VXLAN_M)
+
+#define MPS_RX_GENEVE_TYPE_A 0x11238
+
+#define GENEVE_EN_S    16
+#define GENEVE_EN_V(x) ((x) << GENEVE_EN_S)
+#define GENEVE_EN_F    GENEVE_EN_V(1U)
+
+#define GENEVE_S    0
+#define GENEVE_M    0xffffU
+#define GENEVE_V(x) ((x) << GENEVE_S)
+#define GENEVE_G(x) (((x) >> GENEVE_S) & GENEVE_M)
+
 #define MPS_CLS_TCAM_Y_L_A 0xf000
 #define MPS_CLS_TCAM_DATA0_A 0xf000
 #define MPS_CLS_TCAM_DATA1_A 0xf004
@@ -2530,8 +2559,14 @@
 
 #define DATAPORTNUM_S    12
 #define DATAPORTNUM_M    0xfU
+#define DATAPORTNUM_V(x) ((x) << DATAPORTNUM_S)
 #define DATAPORTNUM_G(x) (((x) >> DATAPORTNUM_S) & DATAPORTNUM_M)
 
+#define DATALKPTYPE_S    10
+#define DATALKPTYPE_M    0x3U
+#define DATALKPTYPE_V(x) ((x) << DATALKPTYPE_S)
+#define DATALKPTYPE_G(x) (((x) >> DATALKPTYPE_S) & DATALKPTYPE_M)
+
 #define DATADIPHIT_S    8
 #define DATADIPHIT_V(x) ((x) << DATADIPHIT_S)
 #define DATADIPHIT_F    DATADIPHIT_V(1U)
diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
index 57eb4ad3485d..0d83b4064a78 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
+++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h
@@ -513,6 +513,13 @@ struct fw_ulptx_wr {
 	u64 cookie;
 };
 
+#define FW_ULPTX_WR_DATA_S      28
+#define FW_ULPTX_WR_DATA_M      0x1
+#define FW_ULPTX_WR_DATA_V(x)   ((x) << FW_ULPTX_WR_DATA_S)
+#define FW_ULPTX_WR_DATA_G(x)   \
+	(((x) >> FW_ULPTX_WR_DATA_S) & FW_ULPTX_WR_DATA_M)
+#define FW_ULPTX_WR_DATA_F      FW_ULPTX_WR_DATA_V(1U)
+
 struct fw_tp_wr {
 	__be32 op_to_immdlen;
 	__be32 flowid_len16;
@@ -828,6 +835,7 @@ enum fw_ldst_addrspc {
 	FW_LDST_ADDRSPC_MPS       = 0x0020,
 	FW_LDST_ADDRSPC_FUNC      = 0x0028,
 	FW_LDST_ADDRSPC_FUNC_PCIE = 0x0029,
+	FW_LDST_ADDRSPC_I2C       = 0x0038,
 };
 
 enum fw_ldst_mps_fid {
@@ -2059,6 +2067,7 @@ struct fw_vi_cmd {
 #define FW_VI_MAC_ADD_MAC		0x3FF
 #define FW_VI_MAC_ADD_PERSIST_MAC	0x3FE
 #define FW_VI_MAC_MAC_BASED_FREE	0x3FD
+#define FW_VI_MAC_ID_BASED_FREE		0x3FC
 #define FW_CLS_TCAM_NUM_ENTRIES		336
 
 enum fw_vi_mac_smac {
@@ -2075,6 +2084,13 @@ enum fw_vi_mac_result {
 	FW_VI_MAC_R_F_ACL_CHECK
 };
 
+enum fw_vi_mac_entry_types {
+	FW_VI_MAC_TYPE_EXACTMAC,
+	FW_VI_MAC_TYPE_HASHVEC,
+	FW_VI_MAC_TYPE_RAW,
+	FW_VI_MAC_TYPE_EXACTMAC_VNI,
+};
+
 struct fw_vi_mac_cmd {
 	__be32 op_to_viid;
 	__be32 freemacs_to_len16;
@@ -2086,6 +2102,13 @@ struct fw_vi_mac_cmd {
 		struct fw_vi_mac_hash {
 			__be64 hashvec;
 		} hash;
+		struct fw_vi_mac_raw {
+			__be32 raw_idx_pkd;
+			__be32 data0_pkd;
+			__be32 data1[2];
+			__be64 data0m_pkd;
+			__be32 data1m[2];
+		} raw;
 	} u;
 };
 
@@ -2095,6 +2118,12 @@ struct fw_vi_mac_cmd {
 #define FW_VI_MAC_CMD_FREEMACS_S	31
 #define FW_VI_MAC_CMD_FREEMACS_V(x)	((x) << FW_VI_MAC_CMD_FREEMACS_S)
 
+#define FW_VI_MAC_CMD_ENTRY_TYPE_S      23
+#define FW_VI_MAC_CMD_ENTRY_TYPE_M      0x7
+#define FW_VI_MAC_CMD_ENTRY_TYPE_V(x)   ((x) << FW_VI_MAC_CMD_ENTRY_TYPE_S)
+#define FW_VI_MAC_CMD_ENTRY_TYPE_G(x)	\
+	(((x) >> FW_VI_MAC_CMD_ENTRY_TYPE_S) & FW_VI_MAC_CMD_ENTRY_TYPE_M)
+
 #define FW_VI_MAC_CMD_HASHVECEN_S	23
 #define FW_VI_MAC_CMD_HASHVECEN_V(x)	((x) << FW_VI_MAC_CMD_HASHVECEN_S)
 #define FW_VI_MAC_CMD_HASHVECEN_F	FW_VI_MAC_CMD_HASHVECEN_V(1U)
@@ -2121,6 +2150,12 @@ struct fw_vi_mac_cmd {
 #define FW_VI_MAC_CMD_IDX_G(x)	\
 	(((x) >> FW_VI_MAC_CMD_IDX_S) & FW_VI_MAC_CMD_IDX_M)
 
+#define FW_VI_MAC_CMD_RAW_IDX_S         16
+#define FW_VI_MAC_CMD_RAW_IDX_M         0xffff
+#define FW_VI_MAC_CMD_RAW_IDX_V(x)      ((x) << FW_VI_MAC_CMD_RAW_IDX_S)
+#define FW_VI_MAC_CMD_RAW_IDX_G(x)      \
+	(((x) >> FW_VI_MAC_CMD_RAW_IDX_S) & FW_VI_MAC_CMD_RAW_IDX_M)
+
 #define FW_RXMODE_MTU_NO_CHG	65535
 
 struct fw_vi_rxmode_cmd {
@@ -2325,14 +2360,22 @@ struct fw_acl_vlan_cmd {
 #define FW_ACL_VLAN_CMD_VFN_S		0
 #define FW_ACL_VLAN_CMD_VFN_V(x)	((x) << FW_ACL_VLAN_CMD_VFN_S)
 
-#define FW_ACL_VLAN_CMD_EN_S	31
-#define FW_ACL_VLAN_CMD_EN_V(x)	((x) << FW_ACL_VLAN_CMD_EN_S)
+#define FW_ACL_VLAN_CMD_EN_S		31
+#define FW_ACL_VLAN_CMD_EN_M		0x1
+#define FW_ACL_VLAN_CMD_EN_V(x)		((x) << FW_ACL_VLAN_CMD_EN_S)
+#define FW_ACL_VLAN_CMD_EN_G(x)         \
+	(((x) >> S_FW_ACL_VLAN_CMD_EN_S) & FW_ACL_VLAN_CMD_EN_M)
+#define FW_ACL_VLAN_CMD_EN_F            FW_ACL_VLAN_CMD_EN_V(1U)
 
 #define FW_ACL_VLAN_CMD_DROPNOVLAN_S	7
 #define FW_ACL_VLAN_CMD_DROPNOVLAN_V(x)	((x) << FW_ACL_VLAN_CMD_DROPNOVLAN_S)
 
-#define FW_ACL_VLAN_CMD_FM_S	6
-#define FW_ACL_VLAN_CMD_FM_V(x)	((x) << FW_ACL_VLAN_CMD_FM_S)
+#define FW_ACL_VLAN_CMD_FM_S		6
+#define FW_ACL_VLAN_CMD_FM_M		0x1
+#define FW_ACL_VLAN_CMD_FM_V(x)         ((x) << FW_ACL_VLAN_CMD_FM_S)
+#define FW_ACL_VLAN_CMD_FM_G(x)         \
+	(((x) >> FW_ACL_VLAN_CMD_FM_S) & FW_ACL_VLAN_CMD_FM_M)
+#define FW_ACL_VLAN_CMD_FM_F            FW_ACL_VLAN_CMD_FM_V(1U)
 
 /* old 16-bit port capabilities bitmap (fw_port_cap16_t) */
 enum fw_port_cap {
@@ -2828,6 +2871,7 @@ enum fw_port_type {
 	FW_PORT_TYPE_CR2_QSFP,
 	FW_PORT_TYPE_SFP28,
 	FW_PORT_TYPE_KR_SFP28,
+	FW_PORT_TYPE_KR_XLAUI,
 
 	FW_PORT_TYPE_NONE = FW_PORT_CMD_PTYPE_M
 };
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
index 08c6ddb84a04..5883f09e3804 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/adapter.h
@@ -92,6 +92,7 @@ struct sge_rspq;
  */
 struct port_info {
 	struct adapter *adapter;	/* our adapter */
+	u32 vlan_id;			/* vlan id for VST */
 	u16 viid;			/* virtual interface ID */
 	s16 xact_addr_filt;		/* index of our MAC address filter */
 	u16 rss_size;			/* size of VI's RSS table slice */
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index b48361cfdc78..b7e79e64d2ed 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -791,6 +791,8 @@ static int cxgb4vf_open(struct net_device *dev)
 	if (err)
 		goto err_unwind;
 
+	pi->vlan_id = t4vf_get_vf_vlan_acl(adapter);
+
 	netif_tx_start_all_queues(dev);
 	set_bit(pi->port_id, &adapter->open_device_map);
 	return 0;
@@ -1229,7 +1231,8 @@ static int from_fw_port_mod_type(enum fw_port_type port_type,
 		else
 			return PORT_OTHER;
 	} else if (port_type == FW_PORT_TYPE_KR4_100G ||
-		   port_type == FW_PORT_TYPE_KR_SFP28) {
+		   port_type == FW_PORT_TYPE_KR_SFP28 ||
+		   port_type == FW_PORT_TYPE_KR_XLAUI) {
 		return PORT_NONE;
 	}
 
@@ -1323,6 +1326,13 @@ static void fw_caps_to_lmm(enum fw_port_type port_type,
 		SET_LMM(25000baseKR_Full);
 		break;
 
+	case FW_PORT_TYPE_KR_XLAUI:
+		SET_LMM(Backplane);
+		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
+		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
+		FW_CAPS_TO_LMM(SPEED_40G, 40000baseKR4_Full);
+		break;
+
 	case FW_PORT_TYPE_CR2_QSFP:
 		SET_LMM(FIBRE);
 		SET_LMM(50000baseSR2_Full);
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
index 14d7e673c656..dfce5df7538e 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
@@ -1202,6 +1202,10 @@ int t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev)
 	BUG_ON(qidx >= pi->nqsets);
 	txq = &adapter->sge.ethtxq[pi->first_qset + qidx];
 
+	if (pi->vlan_id && !skb_vlan_tag_present(skb))
+		__vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
+				       pi->vlan_id);
+
 	/*
 	 * Take this opportunity to reclaim any TX Descriptors whose DMA
 	 * transfers have completed.
@@ -1570,6 +1574,7 @@ static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
 {
 	struct adapter *adapter = rxq->rspq.adapter;
 	struct sge *s = &adapter->sge;
+	struct port_info *pi;
 	int ret;
 	struct sk_buff *skb;
 
@@ -1586,8 +1591,9 @@ static void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl,
 	skb->truesize += skb->data_len;
 	skb->ip_summed = CHECKSUM_UNNECESSARY;
 	skb_record_rx_queue(skb, rxq->rspq.idx);
+	pi = netdev_priv(skb->dev);
 
-	if (pkt->vlan_ex) {
+	if (pkt->vlan_ex && !pi->vlan_id) {
 		__vlan_hwaccel_put_tag(skb, cpu_to_be16(ETH_P_8021Q),
 					be16_to_cpu(pkt->vlan));
 		rxq->stats.vlan_ex++;
@@ -1620,6 +1626,7 @@ int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp,
 	struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq);
 	struct adapter *adapter = rspq->adapter;
 	struct sge *s = &adapter->sge;
+	struct port_info *pi;
 
 	/*
 	 * If this is a good TCP packet and we have Generic Receive Offload
@@ -1644,6 +1651,7 @@ int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp,
 	__skb_pull(skb, s->pktshift);
 	skb->protocol = eth_type_trans(skb, rspq->netdev);
 	skb_record_rx_queue(skb, rspq->idx);
+	pi = netdev_priv(skb->dev);
 	rxq->stats.pkts++;
 
 	if (csum_ok && !pkt->err_vec &&
@@ -1660,9 +1668,10 @@ int t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp,
 	} else
 		skb_checksum_none_assert(skb);
 
-	if (pkt->vlan_ex) {
+	if (pkt->vlan_ex && !pi->vlan_id) {
 		rxq->stats.vlan_ex++;
-		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), be16_to_cpu(pkt->vlan));
+		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
+				       be16_to_cpu(pkt->vlan));
 	}
 
 	netif_receive_skb(skb);
@@ -2619,8 +2628,8 @@ void t4vf_sge_stop(struct adapter *adapter)
 int t4vf_sge_init(struct adapter *adapter)
 {
 	struct sge_params *sge_params = &adapter->params.sge;
-	u32 fl0 = sge_params->sge_fl_buffer_size[0];
-	u32 fl1 = sge_params->sge_fl_buffer_size[1];
+	u32 fl_small_pg = sge_params->sge_fl_buffer_size[0];
+	u32 fl_large_pg = sge_params->sge_fl_buffer_size[1];
 	struct sge *s = &adapter->sge;
 
 	/*
@@ -2628,9 +2637,20 @@ int t4vf_sge_init(struct adapter *adapter)
 	 * the Physical Function Driver.  Ideally we should be able to deal
 	 * with _any_ configuration.  Practice is different ...
 	 */
-	if (fl0 != PAGE_SIZE || (fl1 != 0 && fl1 <= fl0)) {
+
+	/* We only bother using the Large Page logic if the Large Page Buffer
+	 * is larger than our Page Size Buffer.
+	 */
+	if (fl_large_pg <= fl_small_pg)
+		fl_large_pg = 0;
+
+	/* The Page Size Buffer must be exactly equal to our Page Size and the
+	 * Large Page Size Buffer should be 0 (per above) or a power of 2.
+	 */
+	if (fl_small_pg != PAGE_SIZE ||
+	    (fl_large_pg & (fl_large_pg - 1)) != 0) {
 		dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n",
-			fl0, fl1);
+			fl_small_pg, fl_large_pg);
 		return -EINVAL;
 	}
 	if ((sge_params->sge_control & RXPKTCPLMODE_F) !=
@@ -2642,8 +2662,8 @@ int t4vf_sge_init(struct adapter *adapter)
 	/*
 	 * Now translate the adapter parameters into our internal forms.
 	 */
-	if (fl1)
-		s->fl_pg_order = ilog2(fl1) - PAGE_SHIFT;
+	if (fl_large_pg)
+		s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
 	s->stat_len = ((sge_params->sge_control & EGRSTATUSPAGESIZE_F)
 			? 128 : 64);
 	s->pktshift = PKTSHIFT_G(sge_params->sge_control);
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
index 9cf9c56b0f73..712e8f0c71b4 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h
@@ -413,5 +413,6 @@ int t4vf_handle_fw_rpl(struct adapter *, const __be64 *);
 int t4vf_prep_adapter(struct adapter *);
 int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int pf,
 			unsigned int *naddr, u8 *addr);
+int t4vf_get_vf_vlan_acl(struct adapter *adapter);
 
 #endif /* __T4VF_COMMON_H__ */
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
index 67aec59a14e6..798695bf8678 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c
@@ -2147,3 +2147,31 @@ int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int pf,
 
 	return ret;
 }
+
+/**
+ *	t4vf_get_vf_vlan_acl - Get the VLAN ID to be set to
+ *                             the VI of this VF.
+ *	@adapter: The adapter
+ *
+ *	Find the VLAN ID to be set to the VF's VI. The requested VLAN ID
+ *	is from the host OS via callback in the PF driver.
+ */
+int t4vf_get_vf_vlan_acl(struct adapter *adapter)
+{
+	struct fw_acl_vlan_cmd cmd;
+	int vlan = 0;
+	int ret = 0;
+
+	cmd.op_to_vfn = htonl(FW_CMD_OP_V(FW_ACL_VLAN_CMD) |
+			      FW_CMD_REQUEST_F | FW_CMD_READ_F);
+
+	/* Note: Do not enable the ACL */
+	cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd));
+
+	ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &cmd);
+
+	if (!ret)
+		vlan = be16_to_cpu(cmd.vlanid[0]);
+
+	return vlan;
+}
diff --git a/drivers/net/ethernet/cirrus/mac89x0.c b/drivers/net/ethernet/cirrus/mac89x0.c
index f910f0f386d6..977d4c2c759d 100644
--- a/drivers/net/ethernet/cirrus/mac89x0.c
+++ b/drivers/net/ethernet/cirrus/mac89x0.c
@@ -187,6 +187,7 @@ struct net_device * __init mac89x0_probe(int unit)
 	unsigned long ioaddr;
 	unsigned short sig;
 	int err = -ENODEV;
+	struct nubus_rsrc *fres;
 
 	if (!MACH_IS_MAC)
 		return ERR_PTR(-ENODEV);
@@ -207,8 +208,9 @@ struct net_device * __init mac89x0_probe(int unit)
 	/* We might have to parameterize this later */
 	slot = 0xE;
 	/* Get out now if there's a real NuBus card in slot E */
-	if (nubus_find_slot(slot, NULL) != NULL)
-		goto out;
+	for_each_func_rsrc(fres)
+		if (fres->board->slot == slot)
+			goto out;
 
 	/* The pseudo-ISA bits always live at offset 0x300 (gee,
            wonder why...) */
diff --git a/drivers/net/ethernet/cisco/enic/enic.h b/drivers/net/ethernet/cisco/enic/enic.h
index 6a9527004cb1..9b218f0e5a4c 100644
--- a/drivers/net/ethernet/cisco/enic/enic.h
+++ b/drivers/net/ethernet/cisco/enic/enic.h
@@ -43,6 +43,8 @@
 #define ENIC_CQ_MAX		(ENIC_WQ_MAX + ENIC_RQ_MAX)
 #define ENIC_INTR_MAX		(ENIC_CQ_MAX + 2)
 
+#define ENIC_WQ_NAPI_BUDGET	256
+
 #define ENIC_AIC_LARGE_PKT_DIFF	3
 
 struct enic_msix_entry {
diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
index 462d0ce51240..efb9333c7cf8 100644
--- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c
+++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
@@ -18,6 +18,7 @@
 
 #include <linux/netdevice.h>
 #include <linux/ethtool.h>
+#include <linux/net_tstamp.h>
 
 #include "enic_res.h"
 #include "enic.h"
@@ -578,6 +579,16 @@ static int enic_set_rxfh(struct net_device *netdev, const u32 *indir,
 	return __enic_set_rsskey(enic);
 }
 
+static int enic_get_ts_info(struct net_device *netdev,
+			    struct ethtool_ts_info *info)
+{
+	info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
+				SOF_TIMESTAMPING_RX_SOFTWARE |
+				SOF_TIMESTAMPING_SOFTWARE;
+
+	return 0;
+}
+
 static const struct ethtool_ops enic_ethtool_ops = {
 	.get_drvinfo = enic_get_drvinfo,
 	.get_msglevel = enic_get_msglevel,
@@ -597,6 +608,7 @@ static const struct ethtool_ops enic_ethtool_ops = {
 	.get_rxfh = enic_get_rxfh,
 	.set_rxfh = enic_set_rxfh,
 	.get_link_ksettings = enic_get_ksettings,
+	.get_ts_info = enic_get_ts_info,
 };
 
 void enic_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c
index e130fb757e7b..f202ba72a811 100644
--- a/drivers/net/ethernet/cisco/enic/enic_main.c
+++ b/drivers/net/ethernet/cisco/enic/enic_main.c
@@ -856,6 +856,7 @@ static netdev_tx_t enic_hard_start_xmit(struct sk_buff *skb,
 
 	if (vnic_wq_desc_avail(wq) < MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS)
 		netif_tx_stop_queue(txq);
+	skb_tx_timestamp(skb);
 	if (!skb->xmit_more || netif_xmit_stopped(txq))
 		vnic_wq_doorbell(wq);
 
@@ -1499,7 +1500,7 @@ static int enic_poll(struct napi_struct *napi, int budget)
 	unsigned int cq_wq = enic_cq_wq(enic, 0);
 	unsigned int intr = enic_legacy_io_intr();
 	unsigned int rq_work_to_do = budget;
-	unsigned int wq_work_to_do = -1; /* no limit */
+	unsigned int wq_work_to_do = ENIC_WQ_NAPI_BUDGET;
 	unsigned int  work_done, rq_work_done = 0, wq_work_done;
 	int err;
 
@@ -1597,7 +1598,7 @@ static int enic_poll_msix_wq(struct napi_struct *napi, int budget)
 	struct vnic_wq *wq = &enic->wq[wq_index];
 	unsigned int cq;
 	unsigned int intr;
-	unsigned int wq_work_to_do = -1; /* clean all desc possible */
+	unsigned int wq_work_to_do = ENIC_WQ_NAPI_BUDGET;
 	unsigned int wq_work_done;
 	unsigned int wq_irq;
 
diff --git a/drivers/net/ethernet/cortina/Kconfig b/drivers/net/ethernet/cortina/Kconfig
new file mode 100644
index 000000000000..89bc4579724d
--- /dev/null
+++ b/drivers/net/ethernet/cortina/Kconfig
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: GPL-2.0
+# Cortina ethernet devices
+
+config NET_VENDOR_CORTINA
+	bool "Cortina Gemini devices"
+	default y
+	---help---
+	  If you have a network (Ethernet) card belonging to this class, say Y
+	  and read the Ethernet-HOWTO, available from
+	  <http://www.tldp.org/docs.html#howto>.
+
+if NET_VENDOR_CORTINA
+
+config GEMINI_ETHERNET
+	tristate "Gemini Gigabit Ethernet support"
+	depends on OF
+	depends on HAS_IOMEM
+	select PHYLIB
+	select CRC32
+	---help---
+	  This driver supports StorLink SL351x (Gemini) dual Gigabit Ethernet.
+
+endif # NET_VENDOR_CORTINA
diff --git a/drivers/net/ethernet/cortina/Makefile b/drivers/net/ethernet/cortina/Makefile
new file mode 100644
index 000000000000..4e86d398a89c
--- /dev/null
+++ b/drivers/net/ethernet/cortina/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+# Makefile for the Cortina Gemini network device drivers.
+
+obj-$(CONFIG_GEMINI_ETHERNET) += gemini.o
diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c
new file mode 100644
index 000000000000..bd3f6e4d1341
--- /dev/null
+++ b/drivers/net/ethernet/cortina/gemini.c
@@ -0,0 +1,2594 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Ethernet device driver for Cortina Systems Gemini SoC
+ * Also known as the StorLink SL3512 and SL3516 (SL351x) or Lepus
+ * Net Engine and Gigabit Ethernet MAC (GMAC)
+ * This hardware contains a TCP Offload Engine (TOE) but currently the
+ * driver does not make use of it.
+ *
+ * Authors:
+ * Linus Walleij <linus.walleij@linaro.org>
+ * Tobias Waldvogel <tobias.waldvogel@gmail.com> (OpenWRT)
+ * Michał Mirosław <mirq-linux@rere.qmqm.pl>
+ * Paulius Zaleckas <paulius.zaleckas@gmail.com>
+ * Giuseppe De Robertis <Giuseppe.DeRobertis@ba.infn.it>
+ * Gary Chen & Ch Hsu Storlink Semiconductor
+ */
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+#include <linux/cache.h>
+#include <linux/interrupt.h>
+#include <linux/reset.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/of_mdio.h>
+#include <linux/of_net.h>
+#include <linux/of_platform.h>
+#include <linux/etherdevice.h>
+#include <linux/if_vlan.h>
+#include <linux/skbuff.h>
+#include <linux/phy.h>
+#include <linux/crc32.h>
+#include <linux/ethtool.h>
+#include <linux/tcp.h>
+#include <linux/u64_stats_sync.h>
+
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+
+#include "gemini.h"
+
+#define DRV_NAME		"gmac-gemini"
+#define DRV_VERSION		"1.0"
+
+#define HSIZE_8			0x00
+#define HSIZE_16		0x01
+#define HSIZE_32		0x02
+
+#define HBURST_SINGLE		0x00
+#define HBURST_INCR		0x01
+#define HBURST_INCR4		0x02
+#define HBURST_INCR8		0x03
+
+#define HPROT_DATA_CACHE	BIT(0)
+#define HPROT_PRIVILIGED	BIT(1)
+#define HPROT_BUFFERABLE	BIT(2)
+#define HPROT_CACHABLE		BIT(3)
+
+#define DEFAULT_RX_COALESCE_NSECS	0
+#define DEFAULT_GMAC_RXQ_ORDER		9
+#define DEFAULT_GMAC_TXQ_ORDER		8
+#define DEFAULT_RX_BUF_ORDER		11
+#define DEFAULT_NAPI_WEIGHT		64
+#define TX_MAX_FRAGS			16
+#define TX_QUEUE_NUM			1	/* max: 6 */
+#define RX_MAX_ALLOC_ORDER		2
+
+#define GMAC0_IRQ0_2 (GMAC0_TXDERR_INT_BIT | GMAC0_TXPERR_INT_BIT | \
+		      GMAC0_RXDERR_INT_BIT | GMAC0_RXPERR_INT_BIT)
+#define GMAC0_IRQ0_TXQ0_INTS (GMAC0_SWTQ00_EOF_INT_BIT | \
+			      GMAC0_SWTQ00_FIN_INT_BIT)
+#define GMAC0_IRQ4_8 (GMAC0_MIB_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT)
+
+#define GMAC_OFFLOAD_FEATURES (NETIF_F_SG | NETIF_F_IP_CSUM | \
+		NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | \
+		NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6)
+
+/**
+ * struct gmac_queue_page - page buffer per-page info
+ */
+struct gmac_queue_page {
+	struct page *page;
+	dma_addr_t mapping;
+};
+
+struct gmac_txq {
+	struct gmac_txdesc *ring;
+	struct sk_buff	**skb;
+	unsigned int	cptr;
+	unsigned int	noirq_packets;
+};
+
+struct gemini_ethernet;
+
+struct gemini_ethernet_port {
+	u8 id; /* 0 or 1 */
+
+	struct gemini_ethernet *geth;
+	struct net_device *netdev;
+	struct device *dev;
+	void __iomem *dma_base;
+	void __iomem *gmac_base;
+	struct clk *pclk;
+	struct reset_control *reset;
+	int irq;
+	__le32 mac_addr[3];
+
+	void __iomem		*rxq_rwptr;
+	struct gmac_rxdesc	*rxq_ring;
+	unsigned int		rxq_order;
+
+	struct napi_struct	napi;
+	struct hrtimer		rx_coalesce_timer;
+	unsigned int		rx_coalesce_nsecs;
+	unsigned int		freeq_refill;
+	struct gmac_txq		txq[TX_QUEUE_NUM];
+	unsigned int		txq_order;
+	unsigned int		irq_every_tx_packets;
+
+	dma_addr_t		rxq_dma_base;
+	dma_addr_t		txq_dma_base;
+
+	unsigned int		msg_enable;
+	spinlock_t		config_lock; /* Locks config register */
+
+	struct u64_stats_sync	tx_stats_syncp;
+	struct u64_stats_sync	rx_stats_syncp;
+	struct u64_stats_sync	ir_stats_syncp;
+
+	struct rtnl_link_stats64 stats;
+	u64			hw_stats[RX_STATS_NUM];
+	u64			rx_stats[RX_STATUS_NUM];
+	u64			rx_csum_stats[RX_CHKSUM_NUM];
+	u64			rx_napi_exits;
+	u64			tx_frag_stats[TX_MAX_FRAGS];
+	u64			tx_frags_linearized;
+	u64			tx_hw_csummed;
+};
+
+struct gemini_ethernet {
+	struct device *dev;
+	void __iomem *base;
+	struct gemini_ethernet_port *port0;
+	struct gemini_ethernet_port *port1;
+
+	spinlock_t	irq_lock; /* Locks IRQ-related registers */
+	unsigned int	freeq_order;
+	unsigned int	freeq_frag_order;
+	struct gmac_rxdesc *freeq_ring;
+	dma_addr_t	freeq_dma_base;
+	struct gmac_queue_page	*freeq_pages;
+	unsigned int	num_freeq_pages;
+	spinlock_t	freeq_lock; /* Locks queue from reentrance */
+};
+
+#define GMAC_STATS_NUM	( \
+	RX_STATS_NUM + RX_STATUS_NUM + RX_CHKSUM_NUM + 1 + \
+	TX_MAX_FRAGS + 2)
+
+static const char gmac_stats_strings[GMAC_STATS_NUM][ETH_GSTRING_LEN] = {
+	"GMAC_IN_DISCARDS",
+	"GMAC_IN_ERRORS",
+	"GMAC_IN_MCAST",
+	"GMAC_IN_BCAST",
+	"GMAC_IN_MAC1",
+	"GMAC_IN_MAC2",
+	"RX_STATUS_GOOD_FRAME",
+	"RX_STATUS_TOO_LONG_GOOD_CRC",
+	"RX_STATUS_RUNT_FRAME",
+	"RX_STATUS_SFD_NOT_FOUND",
+	"RX_STATUS_CRC_ERROR",
+	"RX_STATUS_TOO_LONG_BAD_CRC",
+	"RX_STATUS_ALIGNMENT_ERROR",
+	"RX_STATUS_TOO_LONG_BAD_ALIGN",
+	"RX_STATUS_RX_ERR",
+	"RX_STATUS_DA_FILTERED",
+	"RX_STATUS_BUFFER_FULL",
+	"RX_STATUS_11",
+	"RX_STATUS_12",
+	"RX_STATUS_13",
+	"RX_STATUS_14",
+	"RX_STATUS_15",
+	"RX_CHKSUM_IP_UDP_TCP_OK",
+	"RX_CHKSUM_IP_OK_ONLY",
+	"RX_CHKSUM_NONE",
+	"RX_CHKSUM_3",
+	"RX_CHKSUM_IP_ERR_UNKNOWN",
+	"RX_CHKSUM_IP_ERR",
+	"RX_CHKSUM_TCP_UDP_ERR",
+	"RX_CHKSUM_7",
+	"RX_NAPI_EXITS",
+	"TX_FRAGS[1]",
+	"TX_FRAGS[2]",
+	"TX_FRAGS[3]",
+	"TX_FRAGS[4]",
+	"TX_FRAGS[5]",
+	"TX_FRAGS[6]",
+	"TX_FRAGS[7]",
+	"TX_FRAGS[8]",
+	"TX_FRAGS[9]",
+	"TX_FRAGS[10]",
+	"TX_FRAGS[11]",
+	"TX_FRAGS[12]",
+	"TX_FRAGS[13]",
+	"TX_FRAGS[14]",
+	"TX_FRAGS[15]",
+	"TX_FRAGS[16+]",
+	"TX_FRAGS_LINEARIZED",
+	"TX_HW_CSUMMED",
+};
+
+static void gmac_dump_dma_state(struct net_device *netdev);
+
+static void gmac_update_config0_reg(struct net_device *netdev,
+				    u32 val, u32 vmask)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&port->config_lock, flags);
+
+	reg = readl(port->gmac_base + GMAC_CONFIG0);
+	reg = (reg & ~vmask) | val;
+	writel(reg, port->gmac_base + GMAC_CONFIG0);
+
+	spin_unlock_irqrestore(&port->config_lock, flags);
+}
+
+static void gmac_enable_tx_rx(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&port->config_lock, flags);
+
+	reg = readl(port->gmac_base + GMAC_CONFIG0);
+	reg &= ~CONFIG0_TX_RX_DISABLE;
+	writel(reg, port->gmac_base + GMAC_CONFIG0);
+
+	spin_unlock_irqrestore(&port->config_lock, flags);
+}
+
+static void gmac_disable_tx_rx(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned long flags;
+	u32 val;
+
+	spin_lock_irqsave(&port->config_lock, flags);
+
+	val = readl(port->gmac_base + GMAC_CONFIG0);
+	val |= CONFIG0_TX_RX_DISABLE;
+	writel(val, port->gmac_base + GMAC_CONFIG0);
+
+	spin_unlock_irqrestore(&port->config_lock, flags);
+
+	mdelay(10);	/* let GMAC consume packet */
+}
+
+static void gmac_set_flow_control(struct net_device *netdev, bool tx, bool rx)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned long flags;
+	u32 val;
+
+	spin_lock_irqsave(&port->config_lock, flags);
+
+	val = readl(port->gmac_base + GMAC_CONFIG0);
+	val &= ~CONFIG0_FLOW_CTL;
+	if (tx)
+		val |= CONFIG0_FLOW_TX;
+	if (rx)
+		val |= CONFIG0_FLOW_RX;
+	writel(val, port->gmac_base + GMAC_CONFIG0);
+
+	spin_unlock_irqrestore(&port->config_lock, flags);
+}
+
+static void gmac_speed_set(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct phy_device *phydev = netdev->phydev;
+	union gmac_status status, old_status;
+	int pause_tx = 0;
+	int pause_rx = 0;
+
+	status.bits32 = readl(port->gmac_base + GMAC_STATUS);
+	old_status.bits32 = status.bits32;
+	status.bits.link = phydev->link;
+	status.bits.duplex = phydev->duplex;
+
+	switch (phydev->speed) {
+	case 1000:
+		status.bits.speed = GMAC_SPEED_1000;
+		if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
+			status.bits.mii_rmii = GMAC_PHY_RGMII_1000;
+		netdev_info(netdev, "connect to RGMII @ 1Gbit\n");
+		break;
+	case 100:
+		status.bits.speed = GMAC_SPEED_100;
+		if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
+			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
+		netdev_info(netdev, "connect to RGMII @ 100 Mbit\n");
+		break;
+	case 10:
+		status.bits.speed = GMAC_SPEED_10;
+		if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
+			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
+		netdev_info(netdev, "connect to RGMII @ 10 Mbit\n");
+		break;
+	default:
+		netdev_warn(netdev, "Not supported PHY speed (%d)\n",
+			    phydev->speed);
+	}
+
+	if (phydev->duplex == DUPLEX_FULL) {
+		u16 lcladv = phy_read(phydev, MII_ADVERTISE);
+		u16 rmtadv = phy_read(phydev, MII_LPA);
+		u8 cap = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
+
+		if (cap & FLOW_CTRL_RX)
+			pause_rx = 1;
+		if (cap & FLOW_CTRL_TX)
+			pause_tx = 1;
+	}
+
+	gmac_set_flow_control(netdev, pause_tx, pause_rx);
+
+	if (old_status.bits32 == status.bits32)
+		return;
+
+	if (netif_msg_link(port)) {
+		phy_print_status(phydev);
+		netdev_info(netdev, "link flow control: %s\n",
+			    phydev->pause
+			    ? (phydev->asym_pause ? "tx" : "both")
+			    : (phydev->asym_pause ? "rx" : "none")
+		);
+	}
+
+	gmac_disable_tx_rx(netdev);
+	writel(status.bits32, port->gmac_base + GMAC_STATUS);
+	gmac_enable_tx_rx(netdev);
+}
+
+static int gmac_setup_phy(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	union gmac_status status = { .bits32 = 0 };
+	struct device *dev = port->dev;
+	struct phy_device *phy;
+
+	phy = of_phy_get_and_connect(netdev,
+				     dev->of_node,
+				     gmac_speed_set);
+	if (!phy)
+		return -ENODEV;
+	netdev->phydev = phy;
+
+	netdev_info(netdev, "connected to PHY \"%s\"\n",
+		    phydev_name(phy));
+	phy_attached_print(phy, "phy_id=0x%.8lx, phy_mode=%s\n",
+			   (unsigned long)phy->phy_id,
+			   phy_modes(phy->interface));
+
+	phy->supported &= PHY_GBIT_FEATURES;
+	phy->supported |= SUPPORTED_Asym_Pause | SUPPORTED_Pause;
+	phy->advertising = phy->supported;
+
+	/* set PHY interface type */
+	switch (phy->interface) {
+	case PHY_INTERFACE_MODE_MII:
+		netdev_info(netdev, "set GMAC0 to GMII mode, GMAC1 disabled\n");
+		status.bits.mii_rmii = GMAC_PHY_MII;
+		netdev_info(netdev, "connect to MII\n");
+		break;
+	case PHY_INTERFACE_MODE_GMII:
+		netdev_info(netdev, "set GMAC0 to GMII mode, GMAC1 disabled\n");
+		status.bits.mii_rmii = GMAC_PHY_GMII;
+		netdev_info(netdev, "connect to GMII\n");
+		break;
+	case PHY_INTERFACE_MODE_RGMII:
+		dev_info(dev, "set GMAC0 and GMAC1 to MII/RGMII mode\n");
+		status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
+		netdev_info(netdev, "connect to RGMII\n");
+		break;
+	default:
+		netdev_err(netdev, "Unsupported MII interface\n");
+		phy_disconnect(phy);
+		netdev->phydev = NULL;
+		return -EINVAL;
+	}
+	writel(status.bits32, port->gmac_base + GMAC_STATUS);
+
+	return 0;
+}
+
+static int gmac_pick_rx_max_len(int max_l3_len)
+{
+	/* index = CONFIG_MAXLEN_XXX values */
+	static const int max_len[8] = {
+		1536, 1518, 1522, 1542,
+		9212, 10236, 1518, 1518
+	};
+	int i, n = 5;
+
+	max_l3_len += ETH_HLEN + VLAN_HLEN;
+
+	if (max_l3_len > max_len[n])
+		return -1;
+
+	for (i = 0; i < 5; i++) {
+		if (max_len[i] >= max_l3_len && max_len[i] < max_len[n])
+			n = i;
+	}
+
+	return n;
+}
+
+static int gmac_init(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	union gmac_config0 config0 = { .bits = {
+		.dis_tx = 1,
+		.dis_rx = 1,
+		.ipv4_rx_chksum = 1,
+		.ipv6_rx_chksum = 1,
+		.rx_err_detect = 1,
+		.rgmm_edge = 1,
+		.port0_chk_hwq = 1,
+		.port1_chk_hwq = 1,
+		.port0_chk_toeq = 1,
+		.port1_chk_toeq = 1,
+		.port0_chk_classq = 1,
+		.port1_chk_classq = 1,
+	} };
+	union gmac_ahb_weight ahb_weight = { .bits = {
+		.rx_weight = 1,
+		.tx_weight = 1,
+		.hash_weight = 1,
+		.pre_req = 0x1f,
+		.tq_dv_threshold = 0,
+	} };
+	union gmac_tx_wcr0 hw_weigh = { .bits = {
+		.hw_tq3 = 1,
+		.hw_tq2 = 1,
+		.hw_tq1 = 1,
+		.hw_tq0 = 1,
+	} };
+	union gmac_tx_wcr1 sw_weigh = { .bits = {
+		.sw_tq5 = 1,
+		.sw_tq4 = 1,
+		.sw_tq3 = 1,
+		.sw_tq2 = 1,
+		.sw_tq1 = 1,
+		.sw_tq0 = 1,
+	} };
+	union gmac_config1 config1 = { .bits = {
+		.set_threshold = 16,
+		.rel_threshold = 24,
+	} };
+	union gmac_config2 config2 = { .bits = {
+		.set_threshold = 16,
+		.rel_threshold = 32,
+	} };
+	union gmac_config3 config3 = { .bits = {
+		.set_threshold = 0,
+		.rel_threshold = 0,
+	} };
+	union gmac_config0 tmp;
+	u32 val;
+
+	config0.bits.max_len = gmac_pick_rx_max_len(netdev->mtu);
+	tmp.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
+	config0.bits.reserved = tmp.bits.reserved;
+	writel(config0.bits32, port->gmac_base + GMAC_CONFIG0);
+	writel(config1.bits32, port->gmac_base + GMAC_CONFIG1);
+	writel(config2.bits32, port->gmac_base + GMAC_CONFIG2);
+	writel(config3.bits32, port->gmac_base + GMAC_CONFIG3);
+
+	val = readl(port->dma_base + GMAC_AHB_WEIGHT_REG);
+	writel(ahb_weight.bits32, port->dma_base + GMAC_AHB_WEIGHT_REG);
+
+	writel(hw_weigh.bits32,
+	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_0_REG);
+	writel(sw_weigh.bits32,
+	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_1_REG);
+
+	port->rxq_order = DEFAULT_GMAC_RXQ_ORDER;
+	port->txq_order = DEFAULT_GMAC_TXQ_ORDER;
+	port->rx_coalesce_nsecs = DEFAULT_RX_COALESCE_NSECS;
+
+	/* Mark every quarter of the queue a packet for interrupt
+	 * in order to be able to wake up the queue if it was stopped
+	 */
+	port->irq_every_tx_packets = 1 << (port->txq_order - 2);
+
+	return 0;
+}
+
+static void gmac_uninit(struct net_device *netdev)
+{
+	if (netdev->phydev)
+		phy_disconnect(netdev->phydev);
+}
+
+static int gmac_setup_txqs(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned int n_txq = netdev->num_tx_queues;
+	struct gemini_ethernet *geth = port->geth;
+	size_t entries = 1 << port->txq_order;
+	struct gmac_txq *txq = port->txq;
+	struct gmac_txdesc *desc_ring;
+	size_t len = n_txq * entries;
+	struct sk_buff **skb_tab;
+	void __iomem *rwptr_reg;
+	unsigned int r;
+	int i;
+
+	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
+
+	skb_tab = kcalloc(len, sizeof(*skb_tab), GFP_KERNEL);
+	if (!skb_tab)
+		return -ENOMEM;
+
+	desc_ring = dma_alloc_coherent(geth->dev, len * sizeof(*desc_ring),
+				       &port->txq_dma_base, GFP_KERNEL);
+
+	if (!desc_ring) {
+		kfree(skb_tab);
+		return -ENOMEM;
+	}
+
+	if (port->txq_dma_base & ~DMA_Q_BASE_MASK) {
+		dev_warn(geth->dev, "TX queue base it not aligned\n");
+		kfree(skb_tab);
+		return -ENOMEM;
+	}
+
+	writel(port->txq_dma_base | port->txq_order,
+	       port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
+
+	for (i = 0; i < n_txq; i++) {
+		txq->ring = desc_ring;
+		txq->skb = skb_tab;
+		txq->noirq_packets = 0;
+
+		r = readw(rwptr_reg);
+		rwptr_reg += 2;
+		writew(r, rwptr_reg);
+		rwptr_reg += 2;
+		txq->cptr = r;
+
+		txq++;
+		desc_ring += entries;
+		skb_tab += entries;
+	}
+
+	return 0;
+}
+
+static void gmac_clean_txq(struct net_device *netdev, struct gmac_txq *txq,
+			   unsigned int r)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned int m = (1 << port->txq_order) - 1;
+	struct gemini_ethernet *geth = port->geth;
+	unsigned int c = txq->cptr;
+	union gmac_txdesc_0 word0;
+	union gmac_txdesc_1 word1;
+	unsigned int hwchksum = 0;
+	unsigned long bytes = 0;
+	struct gmac_txdesc *txd;
+	unsigned short nfrags;
+	unsigned int errs = 0;
+	unsigned int pkts = 0;
+	unsigned int word3;
+	dma_addr_t mapping;
+
+	if (c == r)
+		return;
+
+	while (c != r) {
+		txd = txq->ring + c;
+		word0 = txd->word0;
+		word1 = txd->word1;
+		mapping = txd->word2.buf_adr;
+		word3 = txd->word3.bits32;
+
+		dma_unmap_single(geth->dev, mapping,
+				 word0.bits.buffer_size, DMA_TO_DEVICE);
+
+		if (word3 & EOF_BIT)
+			dev_kfree_skb(txq->skb[c]);
+
+		c++;
+		c &= m;
+
+		if (!(word3 & SOF_BIT))
+			continue;
+
+		if (!word0.bits.status_tx_ok) {
+			errs++;
+			continue;
+		}
+
+		pkts++;
+		bytes += txd->word1.bits.byte_count;
+
+		if (word1.bits32 & TSS_CHECKUM_ENABLE)
+			hwchksum++;
+
+		nfrags = word0.bits.desc_count - 1;
+		if (nfrags) {
+			if (nfrags >= TX_MAX_FRAGS)
+				nfrags = TX_MAX_FRAGS - 1;
+
+			u64_stats_update_begin(&port->tx_stats_syncp);
+			port->tx_frag_stats[nfrags]++;
+			u64_stats_update_end(&port->ir_stats_syncp);
+		}
+	}
+
+	u64_stats_update_begin(&port->ir_stats_syncp);
+	port->stats.tx_errors += errs;
+	port->stats.tx_packets += pkts;
+	port->stats.tx_bytes += bytes;
+	port->tx_hw_csummed += hwchksum;
+	u64_stats_update_end(&port->ir_stats_syncp);
+
+	txq->cptr = c;
+}
+
+static void gmac_cleanup_txqs(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned int n_txq = netdev->num_tx_queues;
+	struct gemini_ethernet *geth = port->geth;
+	void __iomem *rwptr_reg;
+	unsigned int r, i;
+
+	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
+
+	for (i = 0; i < n_txq; i++) {
+		r = readw(rwptr_reg);
+		rwptr_reg += 2;
+		writew(r, rwptr_reg);
+		rwptr_reg += 2;
+
+		gmac_clean_txq(netdev, port->txq + i, r);
+	}
+	writel(0, port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
+
+	kfree(port->txq->skb);
+	dma_free_coherent(geth->dev,
+			  n_txq * sizeof(*port->txq->ring) << port->txq_order,
+			  port->txq->ring, port->txq_dma_base);
+}
+
+static int gmac_setup_rxq(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct gemini_ethernet *geth = port->geth;
+	struct nontoe_qhdr __iomem *qhdr;
+
+	qhdr = geth->base + TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
+	port->rxq_rwptr = &qhdr->word1;
+
+	/* Remap a slew of memory to use for the RX queue */
+	port->rxq_ring = dma_alloc_coherent(geth->dev,
+				sizeof(*port->rxq_ring) << port->rxq_order,
+				&port->rxq_dma_base, GFP_KERNEL);
+	if (!port->rxq_ring)
+		return -ENOMEM;
+	if (port->rxq_dma_base & ~NONTOE_QHDR0_BASE_MASK) {
+		dev_warn(geth->dev, "RX queue base it not aligned\n");
+		return -ENOMEM;
+	}
+
+	writel(port->rxq_dma_base | port->rxq_order, &qhdr->word0);
+	writel(0, port->rxq_rwptr);
+	return 0;
+}
+
+static struct gmac_queue_page *
+gmac_get_queue_page(struct gemini_ethernet *geth,
+		    struct gemini_ethernet_port *port,
+		    dma_addr_t addr)
+{
+	struct gmac_queue_page *gpage;
+	dma_addr_t mapping;
+	int i;
+
+	/* Only look for even pages */
+	mapping = addr & PAGE_MASK;
+
+	if (!geth->freeq_pages) {
+		dev_err(geth->dev, "try to get page with no page list\n");
+		return NULL;
+	}
+
+	/* Look up a ring buffer page from virtual mapping */
+	for (i = 0; i < geth->num_freeq_pages; i++) {
+		gpage = &geth->freeq_pages[i];
+		if (gpage->mapping == mapping)
+			return gpage;
+	}
+
+	return NULL;
+}
+
+static void gmac_cleanup_rxq(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct gemini_ethernet *geth = port->geth;
+	struct gmac_rxdesc *rxd = port->rxq_ring;
+	static struct gmac_queue_page *gpage;
+	struct nontoe_qhdr __iomem *qhdr;
+	void __iomem *dma_reg;
+	void __iomem *ptr_reg;
+	dma_addr_t mapping;
+	union dma_rwptr rw;
+	unsigned int r, w;
+
+	qhdr = geth->base +
+		TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
+	dma_reg = &qhdr->word0;
+	ptr_reg = &qhdr->word1;
+
+	rw.bits32 = readl(ptr_reg);
+	r = rw.bits.rptr;
+	w = rw.bits.wptr;
+	writew(r, ptr_reg + 2);
+
+	writel(0, dma_reg);
+
+	/* Loop from read pointer to write pointer of the RX queue
+	 * and free up all pages by the queue.
+	 */
+	while (r != w) {
+		mapping = rxd[r].word2.buf_adr;
+		r++;
+		r &= ((1 << port->rxq_order) - 1);
+
+		if (!mapping)
+			continue;
+
+		/* Freeq pointers are one page off */
+		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
+		if (!gpage) {
+			dev_err(geth->dev, "could not find page\n");
+			continue;
+		}
+		/* Release the RX queue reference to the page */
+		put_page(gpage->page);
+	}
+
+	dma_free_coherent(geth->dev, sizeof(*port->rxq_ring) << port->rxq_order,
+			  port->rxq_ring, port->rxq_dma_base);
+}
+
+static struct page *geth_freeq_alloc_map_page(struct gemini_ethernet *geth,
+					      int pn)
+{
+	struct gmac_rxdesc *freeq_entry;
+	struct gmac_queue_page *gpage;
+	unsigned int fpp_order;
+	unsigned int frag_len;
+	dma_addr_t mapping;
+	struct page *page;
+	int i;
+
+	/* First allocate and DMA map a single page */
+	page = alloc_page(GFP_ATOMIC);
+	if (!page)
+		return NULL;
+
+	mapping = dma_map_single(geth->dev, page_address(page),
+				 PAGE_SIZE, DMA_FROM_DEVICE);
+	if (dma_mapping_error(geth->dev, mapping)) {
+		put_page(page);
+		return NULL;
+	}
+
+	/* The assign the page mapping (physical address) to the buffer address
+	 * in the hardware queue. PAGE_SHIFT on ARM is 12 (1 page is 4096 bytes,
+	 * 4k), and the default RX frag order is 11 (fragments are up 20 2048
+	 * bytes, 2k) so fpp_order (fragments per page order) is default 1. Thus
+	 * each page normally needs two entries in the queue.
+	 */
+	frag_len = 1 << geth->freeq_frag_order; /* Usually 2048 */
+	fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
+	freeq_entry = geth->freeq_ring + (pn << fpp_order);
+	dev_dbg(geth->dev, "allocate page %d fragment length %d fragments per page %d, freeq entry %p\n",
+		 pn, frag_len, (1 << fpp_order), freeq_entry);
+	for (i = (1 << fpp_order); i > 0; i--) {
+		freeq_entry->word2.buf_adr = mapping;
+		freeq_entry++;
+		mapping += frag_len;
+	}
+
+	/* If the freeq entry already has a page mapped, then unmap it. */
+	gpage = &geth->freeq_pages[pn];
+	if (gpage->page) {
+		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
+		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
+		/* This should be the last reference to the page so it gets
+		 * released
+		 */
+		put_page(gpage->page);
+	}
+
+	/* Then put our new mapping into the page table */
+	dev_dbg(geth->dev, "page %d, DMA addr: %08x, page %p\n",
+		pn, (unsigned int)mapping, page);
+	gpage->mapping = mapping;
+	gpage->page = page;
+
+	return page;
+}
+
+/**
+ * geth_fill_freeq() - Fill the freeq with empty fragments to use
+ * @geth: the ethernet adapter
+ * @refill: whether to reset the queue by filling in all freeq entries or
+ * just refill it, usually the interrupt to refill the queue happens when
+ * the queue is half empty.
+ */
+static unsigned int geth_fill_freeq(struct gemini_ethernet *geth, bool refill)
+{
+	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
+	unsigned int count = 0;
+	unsigned int pn, epn;
+	unsigned long flags;
+	union dma_rwptr rw;
+	unsigned int m_pn;
+
+	/* Mask for page */
+	m_pn = (1 << (geth->freeq_order - fpp_order)) - 1;
+
+	spin_lock_irqsave(&geth->freeq_lock, flags);
+
+	rw.bits32 = readl(geth->base + GLOBAL_SWFQ_RWPTR_REG);
+	pn = (refill ? rw.bits.wptr : rw.bits.rptr) >> fpp_order;
+	epn = (rw.bits.rptr >> fpp_order) - 1;
+	epn &= m_pn;
+
+	/* Loop over the freeq ring buffer entries */
+	while (pn != epn) {
+		struct gmac_queue_page *gpage;
+		struct page *page;
+
+		gpage = &geth->freeq_pages[pn];
+		page = gpage->page;
+
+		dev_dbg(geth->dev, "fill entry %d page ref count %d add %d refs\n",
+			pn, page_ref_count(page), 1 << fpp_order);
+
+		if (page_ref_count(page) > 1) {
+			unsigned int fl = (pn - epn) & m_pn;
+
+			if (fl > 64 >> fpp_order)
+				break;
+
+			page = geth_freeq_alloc_map_page(geth, pn);
+			if (!page)
+				break;
+		}
+
+		/* Add one reference per fragment in the page */
+		page_ref_add(page, 1 << fpp_order);
+		count += 1 << fpp_order;
+		pn++;
+		pn &= m_pn;
+	}
+
+	writew(pn << fpp_order, geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
+
+	spin_unlock_irqrestore(&geth->freeq_lock, flags);
+
+	return count;
+}
+
+static int geth_setup_freeq(struct gemini_ethernet *geth)
+{
+	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
+	unsigned int frag_len = 1 << geth->freeq_frag_order;
+	unsigned int len = 1 << geth->freeq_order;
+	unsigned int pages = len >> fpp_order;
+	union queue_threshold qt;
+	union dma_skb_size skbsz;
+	unsigned int filled;
+	unsigned int pn;
+
+	geth->freeq_ring = dma_alloc_coherent(geth->dev,
+		sizeof(*geth->freeq_ring) << geth->freeq_order,
+		&geth->freeq_dma_base, GFP_KERNEL);
+	if (!geth->freeq_ring)
+		return -ENOMEM;
+	if (geth->freeq_dma_base & ~DMA_Q_BASE_MASK) {
+		dev_warn(geth->dev, "queue ring base it not aligned\n");
+		goto err_freeq;
+	}
+
+	/* Allocate a mapping to page look-up index */
+	geth->freeq_pages = kzalloc(pages * sizeof(*geth->freeq_pages),
+				   GFP_KERNEL);
+	if (!geth->freeq_pages)
+		goto err_freeq;
+	geth->num_freeq_pages = pages;
+
+	dev_info(geth->dev, "allocate %d pages for queue\n", pages);
+	for (pn = 0; pn < pages; pn++)
+		if (!geth_freeq_alloc_map_page(geth, pn))
+			goto err_freeq_alloc;
+
+	filled = geth_fill_freeq(geth, false);
+	if (!filled)
+		goto err_freeq_alloc;
+
+	qt.bits32 = readl(geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
+	qt.bits.swfq_empty = 32;
+	writel(qt.bits32, geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
+
+	skbsz.bits.sw_skb_size = 1 << geth->freeq_frag_order;
+	writel(skbsz.bits32, geth->base + GLOBAL_DMA_SKB_SIZE_REG);
+	writel(geth->freeq_dma_base | geth->freeq_order,
+	       geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
+
+	return 0;
+
+err_freeq_alloc:
+	while (pn > 0) {
+		struct gmac_queue_page *gpage;
+		dma_addr_t mapping;
+
+		--pn;
+		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
+		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
+		gpage = &geth->freeq_pages[pn];
+		put_page(gpage->page);
+	}
+
+	kfree(geth->freeq_pages);
+err_freeq:
+	dma_free_coherent(geth->dev,
+			  sizeof(*geth->freeq_ring) << geth->freeq_order,
+			  geth->freeq_ring, geth->freeq_dma_base);
+	geth->freeq_ring = NULL;
+	return -ENOMEM;
+}
+
+/**
+ * geth_cleanup_freeq() - cleanup the DMA mappings and free the queue
+ * @geth: the Gemini global ethernet state
+ */
+static void geth_cleanup_freeq(struct gemini_ethernet *geth)
+{
+	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
+	unsigned int frag_len = 1 << geth->freeq_frag_order;
+	unsigned int len = 1 << geth->freeq_order;
+	unsigned int pages = len >> fpp_order;
+	unsigned int pn;
+
+	writew(readw(geth->base + GLOBAL_SWFQ_RWPTR_REG),
+	       geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
+	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
+
+	for (pn = 0; pn < pages; pn++) {
+		struct gmac_queue_page *gpage;
+		dma_addr_t mapping;
+
+		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
+		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
+
+		gpage = &geth->freeq_pages[pn];
+		while (page_ref_count(gpage->page) > 0)
+			put_page(gpage->page);
+	}
+
+	kfree(geth->freeq_pages);
+
+	dma_free_coherent(geth->dev,
+			  sizeof(*geth->freeq_ring) << geth->freeq_order,
+			  geth->freeq_ring, geth->freeq_dma_base);
+}
+
+/**
+ * geth_resize_freeq() - resize the software queue depth
+ * @port: the port requesting the change
+ *
+ * This gets called at least once during probe() so the device queue gets
+ * "resized" from the hardware defaults. Since both ports/net devices share
+ * the same hardware queue, some synchronization between the ports is
+ * needed.
+ */
+static int geth_resize_freeq(struct gemini_ethernet_port *port)
+{
+	struct gemini_ethernet *geth = port->geth;
+	struct net_device *netdev = port->netdev;
+	struct gemini_ethernet_port *other_port;
+	struct net_device *other_netdev;
+	unsigned int new_size = 0;
+	unsigned int new_order;
+	unsigned long flags;
+	u32 en;
+	int ret;
+
+	if (netdev->dev_id == 0)
+		other_netdev = geth->port1->netdev;
+	else
+		other_netdev = geth->port0->netdev;
+
+	if (other_netdev && netif_running(other_netdev))
+		return -EBUSY;
+
+	new_size = 1 << (port->rxq_order + 1);
+	netdev_dbg(netdev, "port %d size: %d order %d\n",
+		   netdev->dev_id,
+		   new_size,
+		   port->rxq_order);
+	if (other_netdev) {
+		other_port = netdev_priv(other_netdev);
+		new_size += 1 << (other_port->rxq_order + 1);
+		netdev_dbg(other_netdev, "port %d size: %d order %d\n",
+			   other_netdev->dev_id,
+			   (1 << (other_port->rxq_order + 1)),
+			   other_port->rxq_order);
+	}
+
+	new_order = min(15, ilog2(new_size - 1) + 1);
+	dev_dbg(geth->dev, "set shared queue to size %d order %d\n",
+		new_size, new_order);
+	if (geth->freeq_order == new_order)
+		return 0;
+
+	spin_lock_irqsave(&geth->irq_lock, flags);
+
+	/* Disable the software queue IRQs */
+	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+	en &= ~SWFQ_EMPTY_INT_BIT;
+	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+	spin_unlock_irqrestore(&geth->irq_lock, flags);
+
+	/* Drop the old queue */
+	if (geth->freeq_ring)
+		geth_cleanup_freeq(geth);
+
+	/* Allocate a new queue with the desired order */
+	geth->freeq_order = new_order;
+	ret = geth_setup_freeq(geth);
+
+	/* Restart the interrupts - NOTE if this is the first resize
+	 * after probe(), this is where the interrupts get turned on
+	 * in the first place.
+	 */
+	spin_lock_irqsave(&geth->irq_lock, flags);
+	en |= SWFQ_EMPTY_INT_BIT;
+	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+	spin_unlock_irqrestore(&geth->irq_lock, flags);
+
+	return ret;
+}
+
+static void gmac_tx_irq_enable(struct net_device *netdev,
+			       unsigned int txq, int en)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct gemini_ethernet *geth = port->geth;
+	u32 val, mask;
+
+	netdev_dbg(netdev, "%s device %d\n", __func__, netdev->dev_id);
+
+	mask = GMAC0_IRQ0_TXQ0_INTS << (6 * netdev->dev_id + txq);
+
+	if (en)
+		writel(mask, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
+
+	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
+	val = en ? val | mask : val & ~mask;
+	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
+}
+
+static void gmac_tx_irq(struct net_device *netdev, unsigned int txq_num)
+{
+	struct netdev_queue *ntxq = netdev_get_tx_queue(netdev, txq_num);
+
+	gmac_tx_irq_enable(netdev, txq_num, 0);
+	netif_tx_wake_queue(ntxq);
+}
+
+static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb,
+			    struct gmac_txq *txq, unsigned short *desc)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct skb_shared_info *skb_si =  skb_shinfo(skb);
+	unsigned short m = (1 << port->txq_order) - 1;
+	short frag, last_frag = skb_si->nr_frags - 1;
+	struct gemini_ethernet *geth = port->geth;
+	unsigned int word1, word3, buflen;
+	unsigned short w = *desc;
+	struct gmac_txdesc *txd;
+	skb_frag_t *skb_frag;
+	dma_addr_t mapping;
+	unsigned short mtu;
+	void *buffer;
+
+	mtu  = ETH_HLEN;
+	mtu += netdev->mtu;
+	if (skb->protocol == htons(ETH_P_8021Q))
+		mtu += VLAN_HLEN;
+
+	word1 = skb->len;
+	word3 = SOF_BIT;
+
+	if (word1 > mtu) {
+		word1 |= TSS_MTU_ENABLE_BIT;
+		word3 |= mtu;
+	}
+
+	if (skb->ip_summed != CHECKSUM_NONE) {
+		int tcp = 0;
+
+		if (skb->protocol == htons(ETH_P_IP)) {
+			word1 |= TSS_IP_CHKSUM_BIT;
+			tcp = ip_hdr(skb)->protocol == IPPROTO_TCP;
+		} else { /* IPv6 */
+			word1 |= TSS_IPV6_ENABLE_BIT;
+			tcp = ipv6_hdr(skb)->nexthdr == IPPROTO_TCP;
+		}
+
+		word1 |= tcp ? TSS_TCP_CHKSUM_BIT : TSS_UDP_CHKSUM_BIT;
+	}
+
+	frag = -1;
+	while (frag <= last_frag) {
+		if (frag == -1) {
+			buffer = skb->data;
+			buflen = skb_headlen(skb);
+		} else {
+			skb_frag = skb_si->frags + frag;
+			buffer = page_address(skb_frag_page(skb_frag)) +
+				 skb_frag->page_offset;
+			buflen = skb_frag->size;
+		}
+
+		if (frag == last_frag) {
+			word3 |= EOF_BIT;
+			txq->skb[w] = skb;
+		}
+
+		mapping = dma_map_single(geth->dev, buffer, buflen,
+					 DMA_TO_DEVICE);
+		if (dma_mapping_error(geth->dev, mapping))
+			goto map_error;
+
+		txd = txq->ring + w;
+		txd->word0.bits32 = buflen;
+		txd->word1.bits32 = word1;
+		txd->word2.buf_adr = mapping;
+		txd->word3.bits32 = word3;
+
+		word3 &= MTU_SIZE_BIT_MASK;
+		w++;
+		w &= m;
+		frag++;
+	}
+
+	*desc = w;
+	return 0;
+
+map_error:
+	while (w != *desc) {
+		w--;
+		w &= m;
+
+		dma_unmap_page(geth->dev, txq->ring[w].word2.buf_adr,
+			       txq->ring[w].word0.bits.buffer_size,
+			       DMA_TO_DEVICE);
+	}
+	return -ENOMEM;
+}
+
+static int gmac_start_xmit(struct sk_buff *skb, struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned short m = (1 << port->txq_order) - 1;
+	struct netdev_queue *ntxq;
+	unsigned short r, w, d;
+	void __iomem *ptr_reg;
+	struct gmac_txq *txq;
+	int txq_num, nfrags;
+	union dma_rwptr rw;
+
+	SKB_FRAG_ASSERT(skb);
+
+	if (skb->len >= 0x10000)
+		goto out_drop_free;
+
+	txq_num = skb_get_queue_mapping(skb);
+	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE_PTR_REG(txq_num);
+	txq = &port->txq[txq_num];
+	ntxq = netdev_get_tx_queue(netdev, txq_num);
+	nfrags = skb_shinfo(skb)->nr_frags;
+
+	rw.bits32 = readl(ptr_reg);
+	r = rw.bits.rptr;
+	w = rw.bits.wptr;
+
+	d = txq->cptr - w - 1;
+	d &= m;
+
+	if (d < nfrags + 2) {
+		gmac_clean_txq(netdev, txq, r);
+		d = txq->cptr - w - 1;
+		d &= m;
+
+		if (d < nfrags + 2) {
+			netif_tx_stop_queue(ntxq);
+
+			d = txq->cptr + nfrags + 16;
+			d &= m;
+			txq->ring[d].word3.bits.eofie = 1;
+			gmac_tx_irq_enable(netdev, txq_num, 1);
+
+			u64_stats_update_begin(&port->tx_stats_syncp);
+			netdev->stats.tx_fifo_errors++;
+			u64_stats_update_end(&port->tx_stats_syncp);
+			return NETDEV_TX_BUSY;
+		}
+	}
+
+	if (gmac_map_tx_bufs(netdev, skb, txq, &w)) {
+		if (skb_linearize(skb))
+			goto out_drop;
+
+		u64_stats_update_begin(&port->tx_stats_syncp);
+		port->tx_frags_linearized++;
+		u64_stats_update_end(&port->tx_stats_syncp);
+
+		if (gmac_map_tx_bufs(netdev, skb, txq, &w))
+			goto out_drop_free;
+	}
+
+	writew(w, ptr_reg + 2);
+
+	gmac_clean_txq(netdev, txq, r);
+	return NETDEV_TX_OK;
+
+out_drop_free:
+	dev_kfree_skb(skb);
+out_drop:
+	u64_stats_update_begin(&port->tx_stats_syncp);
+	port->stats.tx_dropped++;
+	u64_stats_update_end(&port->tx_stats_syncp);
+	return NETDEV_TX_OK;
+}
+
+static void gmac_tx_timeout(struct net_device *netdev)
+{
+	netdev_err(netdev, "Tx timeout\n");
+	gmac_dump_dma_state(netdev);
+}
+
+static void gmac_enable_irq(struct net_device *netdev, int enable)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct gemini_ethernet *geth = port->geth;
+	unsigned long flags;
+	u32 val, mask;
+
+	netdev_info(netdev, "%s device %d %s\n", __func__,
+		    netdev->dev_id, enable ? "enable" : "disable");
+	spin_lock_irqsave(&geth->irq_lock, flags);
+
+	mask = GMAC0_IRQ0_2 << (netdev->dev_id * 2);
+	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
+	val = enable ? (val | mask) : (val & ~mask);
+	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
+
+	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
+	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
+	val = enable ? (val | mask) : (val & ~mask);
+	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
+
+	mask = GMAC0_IRQ4_8 << (netdev->dev_id * 8);
+	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+	val = enable ? (val | mask) : (val & ~mask);
+	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+
+	spin_unlock_irqrestore(&geth->irq_lock, flags);
+}
+
+static void gmac_enable_rx_irq(struct net_device *netdev, int enable)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct gemini_ethernet *geth = port->geth;
+	unsigned long flags;
+	u32 val, mask;
+
+	netdev_dbg(netdev, "%s device %d %s\n", __func__, netdev->dev_id,
+		   enable ? "enable" : "disable");
+	spin_lock_irqsave(&geth->irq_lock, flags);
+	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
+
+	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
+	val = enable ? (val | mask) : (val & ~mask);
+	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
+
+	spin_unlock_irqrestore(&geth->irq_lock, flags);
+}
+
+static struct sk_buff *gmac_skb_if_good_frame(struct gemini_ethernet_port *port,
+					      union gmac_rxdesc_0 word0,
+					      unsigned int frame_len)
+{
+	unsigned int rx_csum = word0.bits.chksum_status;
+	unsigned int rx_status = word0.bits.status;
+	struct sk_buff *skb = NULL;
+
+	port->rx_stats[rx_status]++;
+	port->rx_csum_stats[rx_csum]++;
+
+	if (word0.bits.derr || word0.bits.perr ||
+	    rx_status || frame_len < ETH_ZLEN ||
+	    rx_csum >= RX_CHKSUM_IP_ERR_UNKNOWN) {
+		port->stats.rx_errors++;
+
+		if (frame_len < ETH_ZLEN || RX_ERROR_LENGTH(rx_status))
+			port->stats.rx_length_errors++;
+		if (RX_ERROR_OVER(rx_status))
+			port->stats.rx_over_errors++;
+		if (RX_ERROR_CRC(rx_status))
+			port->stats.rx_crc_errors++;
+		if (RX_ERROR_FRAME(rx_status))
+			port->stats.rx_frame_errors++;
+		return NULL;
+	}
+
+	skb = napi_get_frags(&port->napi);
+	if (!skb)
+		goto update_exit;
+
+	if (rx_csum == RX_CHKSUM_IP_UDP_TCP_OK)
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+update_exit:
+	port->stats.rx_bytes += frame_len;
+	port->stats.rx_packets++;
+	return skb;
+}
+
+static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned short m = (1 << port->rxq_order) - 1;
+	struct gemini_ethernet *geth = port->geth;
+	void __iomem *ptr_reg = port->rxq_rwptr;
+	unsigned int frame_len, frag_len;
+	struct gmac_rxdesc *rx = NULL;
+	struct gmac_queue_page *gpage;
+	static struct sk_buff *skb;
+	union gmac_rxdesc_0 word0;
+	union gmac_rxdesc_1 word1;
+	union gmac_rxdesc_3 word3;
+	struct page *page = NULL;
+	unsigned int page_offs;
+	unsigned short r, w;
+	union dma_rwptr rw;
+	dma_addr_t mapping;
+	int frag_nr = 0;
+
+	rw.bits32 = readl(ptr_reg);
+	/* Reset interrupt as all packages until here are taken into account */
+	writel(DEFAULT_Q0_INT_BIT << netdev->dev_id,
+	       geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
+	r = rw.bits.rptr;
+	w = rw.bits.wptr;
+
+	while (budget && w != r) {
+		rx = port->rxq_ring + r;
+		word0 = rx->word0;
+		word1 = rx->word1;
+		mapping = rx->word2.buf_adr;
+		word3 = rx->word3;
+
+		r++;
+		r &= m;
+
+		frag_len = word0.bits.buffer_size;
+		frame_len = word1.bits.byte_count;
+		page_offs = mapping & ~PAGE_MASK;
+
+		if (!mapping) {
+			netdev_err(netdev,
+				   "rxq[%u]: HW BUG: zero DMA desc\n", r);
+			goto err_drop;
+		}
+
+		/* Freeq pointers are one page off */
+		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
+		if (!gpage) {
+			dev_err(geth->dev, "could not find mapping\n");
+			continue;
+		}
+		page = gpage->page;
+
+		if (word3.bits32 & SOF_BIT) {
+			if (skb) {
+				napi_free_frags(&port->napi);
+				port->stats.rx_dropped++;
+			}
+
+			skb = gmac_skb_if_good_frame(port, word0, frame_len);
+			if (!skb)
+				goto err_drop;
+
+			page_offs += NET_IP_ALIGN;
+			frag_len -= NET_IP_ALIGN;
+			frag_nr = 0;
+
+		} else if (!skb) {
+			put_page(page);
+			continue;
+		}
+
+		if (word3.bits32 & EOF_BIT)
+			frag_len = frame_len - skb->len;
+
+		/* append page frag to skb */
+		if (frag_nr == MAX_SKB_FRAGS)
+			goto err_drop;
+
+		if (frag_len == 0)
+			netdev_err(netdev, "Received fragment with len = 0\n");
+
+		skb_fill_page_desc(skb, frag_nr, page, page_offs, frag_len);
+		skb->len += frag_len;
+		skb->data_len += frag_len;
+		skb->truesize += frag_len;
+		frag_nr++;
+
+		if (word3.bits32 & EOF_BIT) {
+			napi_gro_frags(&port->napi);
+			skb = NULL;
+			--budget;
+		}
+		continue;
+
+err_drop:
+		if (skb) {
+			napi_free_frags(&port->napi);
+			skb = NULL;
+		}
+
+		if (mapping)
+			put_page(page);
+
+		port->stats.rx_dropped++;
+	}
+
+	writew(r, ptr_reg);
+	return budget;
+}
+
+static int gmac_napi_poll(struct napi_struct *napi, int budget)
+{
+	struct gemini_ethernet_port *port = netdev_priv(napi->dev);
+	struct gemini_ethernet *geth = port->geth;
+	unsigned int freeq_threshold;
+	unsigned int received;
+
+	freeq_threshold = 1 << (geth->freeq_order - 1);
+	u64_stats_update_begin(&port->rx_stats_syncp);
+
+	received = gmac_rx(napi->dev, budget);
+	if (received < budget) {
+		napi_gro_flush(napi, false);
+		napi_complete_done(napi, received);
+		gmac_enable_rx_irq(napi->dev, 1);
+		++port->rx_napi_exits;
+	}
+
+	port->freeq_refill += (budget - received);
+	if (port->freeq_refill > freeq_threshold) {
+		port->freeq_refill -= freeq_threshold;
+		geth_fill_freeq(geth, true);
+	}
+
+	u64_stats_update_end(&port->rx_stats_syncp);
+	return received;
+}
+
+static void gmac_dump_dma_state(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct gemini_ethernet *geth = port->geth;
+	void __iomem *ptr_reg;
+	u32 reg[5];
+
+	/* Interrupt status */
+	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
+	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
+	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
+	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
+	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
+	netdev_err(netdev, "IRQ status: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
+		   reg[0], reg[1], reg[2], reg[3], reg[4]);
+
+	/* Interrupt enable */
+	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
+	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
+	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
+	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
+	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+	netdev_err(netdev, "IRQ enable: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
+		   reg[0], reg[1], reg[2], reg[3], reg[4]);
+
+	/* RX DMA status */
+	reg[0] = readl(port->dma_base + GMAC_DMA_RX_FIRST_DESC_REG);
+	reg[1] = readl(port->dma_base + GMAC_DMA_RX_CURR_DESC_REG);
+	reg[2] = GET_RPTR(port->rxq_rwptr);
+	reg[3] = GET_WPTR(port->rxq_rwptr);
+	netdev_err(netdev, "RX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
+		   reg[0], reg[1], reg[2], reg[3]);
+
+	reg[0] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD0_REG);
+	reg[1] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD1_REG);
+	reg[2] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD2_REG);
+	reg[3] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD3_REG);
+	netdev_err(netdev, "RX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+		   reg[0], reg[1], reg[2], reg[3]);
+
+	/* TX DMA status */
+	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
+
+	reg[0] = readl(port->dma_base + GMAC_DMA_TX_FIRST_DESC_REG);
+	reg[1] = readl(port->dma_base + GMAC_DMA_TX_CURR_DESC_REG);
+	reg[2] = GET_RPTR(ptr_reg);
+	reg[3] = GET_WPTR(ptr_reg);
+	netdev_err(netdev, "TX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
+		   reg[0], reg[1], reg[2], reg[3]);
+
+	reg[0] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD0_REG);
+	reg[1] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD1_REG);
+	reg[2] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD2_REG);
+	reg[3] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD3_REG);
+	netdev_err(netdev, "TX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
+		   reg[0], reg[1], reg[2], reg[3]);
+
+	/* FREE queues status */
+	ptr_reg = geth->base + GLOBAL_SWFQ_RWPTR_REG;
+
+	reg[0] = GET_RPTR(ptr_reg);
+	reg[1] = GET_WPTR(ptr_reg);
+
+	ptr_reg = geth->base + GLOBAL_HWFQ_RWPTR_REG;
+
+	reg[2] = GET_RPTR(ptr_reg);
+	reg[3] = GET_WPTR(ptr_reg);
+	netdev_err(netdev, "FQ SW ptr: %u %u, HW ptr: %u %u\n",
+		   reg[0], reg[1], reg[2], reg[3]);
+}
+
+static void gmac_update_hw_stats(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned int rx_discards, rx_mcast, rx_bcast;
+	struct gemini_ethernet *geth = port->geth;
+	unsigned long flags;
+
+	spin_lock_irqsave(&geth->irq_lock, flags);
+	u64_stats_update_begin(&port->ir_stats_syncp);
+
+	rx_discards = readl(port->gmac_base + GMAC_IN_DISCARDS);
+	port->hw_stats[0] += rx_discards;
+	port->hw_stats[1] += readl(port->gmac_base + GMAC_IN_ERRORS);
+	rx_mcast = readl(port->gmac_base + GMAC_IN_MCAST);
+	port->hw_stats[2] += rx_mcast;
+	rx_bcast = readl(port->gmac_base + GMAC_IN_BCAST);
+	port->hw_stats[3] += rx_bcast;
+	port->hw_stats[4] += readl(port->gmac_base + GMAC_IN_MAC1);
+	port->hw_stats[5] += readl(port->gmac_base + GMAC_IN_MAC2);
+
+	port->stats.rx_missed_errors += rx_discards;
+	port->stats.multicast += rx_mcast;
+	port->stats.multicast += rx_bcast;
+
+	writel(GMAC0_MIB_INT_BIT << (netdev->dev_id * 8),
+	       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
+
+	u64_stats_update_end(&port->ir_stats_syncp);
+	spin_unlock_irqrestore(&geth->irq_lock, flags);
+}
+
+/**
+ * gmac_get_intr_flags() - get interrupt status flags for a port from
+ * @netdev: the net device for the port to get flags from
+ * @i: the interrupt status register 0..4
+ */
+static u32 gmac_get_intr_flags(struct net_device *netdev, int i)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	struct gemini_ethernet *geth = port->geth;
+	void __iomem *irqif_reg, *irqen_reg;
+	unsigned int offs, val;
+
+	/* Calculate the offset using the stride of the status registers */
+	offs = i * (GLOBAL_INTERRUPT_STATUS_1_REG -
+		    GLOBAL_INTERRUPT_STATUS_0_REG);
+
+	irqif_reg = geth->base + GLOBAL_INTERRUPT_STATUS_0_REG + offs;
+	irqen_reg = geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG + offs;
+
+	val = readl(irqif_reg) & readl(irqen_reg);
+	return val;
+}
+
+static enum hrtimer_restart gmac_coalesce_delay_expired(struct hrtimer *timer)
+{
+	struct gemini_ethernet_port *port =
+		container_of(timer, struct gemini_ethernet_port,
+			     rx_coalesce_timer);
+
+	napi_schedule(&port->napi);
+	return HRTIMER_NORESTART;
+}
+
+static irqreturn_t gmac_irq(int irq, void *data)
+{
+	struct gemini_ethernet_port *port;
+	struct net_device *netdev = data;
+	struct gemini_ethernet *geth;
+	u32 val, orr = 0;
+
+	port = netdev_priv(netdev);
+	geth = port->geth;
+
+	val = gmac_get_intr_flags(netdev, 0);
+	orr |= val;
+
+	if (val & (GMAC0_IRQ0_2 << (netdev->dev_id * 2))) {
+		/* Oh, crap */
+		netdev_err(netdev, "hw failure/sw bug\n");
+		gmac_dump_dma_state(netdev);
+
+		/* don't know how to recover, just reduce losses */
+		gmac_enable_irq(netdev, 0);
+		return IRQ_HANDLED;
+	}
+
+	if (val & (GMAC0_IRQ0_TXQ0_INTS << (netdev->dev_id * 6)))
+		gmac_tx_irq(netdev, 0);
+
+	val = gmac_get_intr_flags(netdev, 1);
+	orr |= val;
+
+	if (val & (DEFAULT_Q0_INT_BIT << netdev->dev_id)) {
+		gmac_enable_rx_irq(netdev, 0);
+
+		if (!port->rx_coalesce_nsecs) {
+			napi_schedule(&port->napi);
+		} else {
+			ktime_t ktime;
+
+			ktime = ktime_set(0, port->rx_coalesce_nsecs);
+			hrtimer_start(&port->rx_coalesce_timer, ktime,
+				      HRTIMER_MODE_REL);
+		}
+	}
+
+	val = gmac_get_intr_flags(netdev, 4);
+	orr |= val;
+
+	if (val & (GMAC0_MIB_INT_BIT << (netdev->dev_id * 8)))
+		gmac_update_hw_stats(netdev);
+
+	if (val & (GMAC0_RX_OVERRUN_INT_BIT << (netdev->dev_id * 8))) {
+		writel(GMAC0_RXDERR_INT_BIT << (netdev->dev_id * 8),
+		       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
+
+		spin_lock(&geth->irq_lock);
+		u64_stats_update_begin(&port->ir_stats_syncp);
+		++port->stats.rx_fifo_errors;
+		u64_stats_update_end(&port->ir_stats_syncp);
+		spin_unlock(&geth->irq_lock);
+	}
+
+	return orr ? IRQ_HANDLED : IRQ_NONE;
+}
+
+static void gmac_start_dma(struct gemini_ethernet_port *port)
+{
+	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
+	union gmac_dma_ctrl dma_ctrl;
+
+	dma_ctrl.bits32 = readl(dma_ctrl_reg);
+	dma_ctrl.bits.rd_enable = 1;
+	dma_ctrl.bits.td_enable = 1;
+	dma_ctrl.bits.loopback = 0;
+	dma_ctrl.bits.drop_small_ack = 0;
+	dma_ctrl.bits.rd_insert_bytes = NET_IP_ALIGN;
+	dma_ctrl.bits.rd_prot = HPROT_DATA_CACHE | HPROT_PRIVILIGED;
+	dma_ctrl.bits.rd_burst_size = HBURST_INCR8;
+	dma_ctrl.bits.rd_bus = HSIZE_8;
+	dma_ctrl.bits.td_prot = HPROT_DATA_CACHE;
+	dma_ctrl.bits.td_burst_size = HBURST_INCR8;
+	dma_ctrl.bits.td_bus = HSIZE_8;
+
+	writel(dma_ctrl.bits32, dma_ctrl_reg);
+}
+
+static void gmac_stop_dma(struct gemini_ethernet_port *port)
+{
+	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
+	union gmac_dma_ctrl dma_ctrl;
+
+	dma_ctrl.bits32 = readl(dma_ctrl_reg);
+	dma_ctrl.bits.rd_enable = 0;
+	dma_ctrl.bits.td_enable = 0;
+	writel(dma_ctrl.bits32, dma_ctrl_reg);
+}
+
+static int gmac_open(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	int err;
+
+	if (!netdev->phydev) {
+		err = gmac_setup_phy(netdev);
+		if (err) {
+			netif_err(port, ifup, netdev,
+				  "PHY init failed: %d\n", err);
+			return err;
+		}
+	}
+
+	err = request_irq(netdev->irq, gmac_irq,
+			  IRQF_SHARED, netdev->name, netdev);
+	if (err) {
+		netdev_err(netdev, "no IRQ\n");
+		return err;
+	}
+
+	netif_carrier_off(netdev);
+	phy_start(netdev->phydev);
+
+	err = geth_resize_freeq(port);
+	if (err) {
+		netdev_err(netdev, "could not resize freeq\n");
+		goto err_stop_phy;
+	}
+
+	err = gmac_setup_rxq(netdev);
+	if (err) {
+		netdev_err(netdev, "could not setup RXQ\n");
+		goto err_stop_phy;
+	}
+
+	err = gmac_setup_txqs(netdev);
+	if (err) {
+		netdev_err(netdev, "could not setup TXQs\n");
+		gmac_cleanup_rxq(netdev);
+		goto err_stop_phy;
+	}
+
+	napi_enable(&port->napi);
+
+	gmac_start_dma(port);
+	gmac_enable_irq(netdev, 1);
+	gmac_enable_tx_rx(netdev);
+	netif_tx_start_all_queues(netdev);
+
+	hrtimer_init(&port->rx_coalesce_timer, CLOCK_MONOTONIC,
+		     HRTIMER_MODE_REL);
+	port->rx_coalesce_timer.function = &gmac_coalesce_delay_expired;
+
+	netdev_info(netdev, "opened\n");
+
+	return 0;
+
+err_stop_phy:
+	phy_stop(netdev->phydev);
+	free_irq(netdev->irq, netdev);
+	return err;
+}
+
+static int gmac_stop(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+
+	hrtimer_cancel(&port->rx_coalesce_timer);
+	netif_tx_stop_all_queues(netdev);
+	gmac_disable_tx_rx(netdev);
+	gmac_stop_dma(port);
+	napi_disable(&port->napi);
+
+	gmac_enable_irq(netdev, 0);
+	gmac_cleanup_rxq(netdev);
+	gmac_cleanup_txqs(netdev);
+
+	phy_stop(netdev->phydev);
+	free_irq(netdev->irq, netdev);
+
+	gmac_update_hw_stats(netdev);
+	return 0;
+}
+
+static void gmac_set_rx_mode(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	union gmac_rx_fltr filter = { .bits = {
+		.broadcast = 1,
+		.multicast = 1,
+		.unicast = 1,
+	} };
+	struct netdev_hw_addr *ha;
+	unsigned int bit_nr;
+	u32 mc_filter[2];
+
+	mc_filter[1] = 0;
+	mc_filter[0] = 0;
+
+	if (netdev->flags & IFF_PROMISC) {
+		filter.bits.error = 1;
+		filter.bits.promiscuous = 1;
+		mc_filter[1] = ~0;
+		mc_filter[0] = ~0;
+	} else if (netdev->flags & IFF_ALLMULTI) {
+		mc_filter[1] = ~0;
+		mc_filter[0] = ~0;
+	} else {
+		netdev_for_each_mc_addr(ha, netdev) {
+			bit_nr = ~crc32_le(~0, ha->addr, ETH_ALEN) & 0x3f;
+			mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 0x1f);
+		}
+	}
+
+	writel(mc_filter[0], port->gmac_base + GMAC_MCAST_FIL0);
+	writel(mc_filter[1], port->gmac_base + GMAC_MCAST_FIL1);
+	writel(filter.bits32, port->gmac_base + GMAC_RX_FLTR);
+}
+
+static void gmac_write_mac_address(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	__le32 addr[3];
+
+	memset(addr, 0, sizeof(addr));
+	memcpy(addr, netdev->dev_addr, ETH_ALEN);
+
+	writel(le32_to_cpu(addr[0]), port->gmac_base + GMAC_STA_ADD0);
+	writel(le32_to_cpu(addr[1]), port->gmac_base + GMAC_STA_ADD1);
+	writel(le32_to_cpu(addr[2]), port->gmac_base + GMAC_STA_ADD2);
+}
+
+static int gmac_set_mac_address(struct net_device *netdev, void *addr)
+{
+	struct sockaddr *sa = addr;
+
+	memcpy(netdev->dev_addr, sa->sa_data, ETH_ALEN);
+	gmac_write_mac_address(netdev);
+
+	return 0;
+}
+
+static void gmac_clear_hw_stats(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+
+	readl(port->gmac_base + GMAC_IN_DISCARDS);
+	readl(port->gmac_base + GMAC_IN_ERRORS);
+	readl(port->gmac_base + GMAC_IN_MCAST);
+	readl(port->gmac_base + GMAC_IN_BCAST);
+	readl(port->gmac_base + GMAC_IN_MAC1);
+	readl(port->gmac_base + GMAC_IN_MAC2);
+}
+
+static void gmac_get_stats64(struct net_device *netdev,
+			     struct rtnl_link_stats64 *stats)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned int start;
+
+	gmac_update_hw_stats(netdev);
+
+	/* Racing with RX NAPI */
+	do {
+		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
+
+		stats->rx_packets = port->stats.rx_packets;
+		stats->rx_bytes = port->stats.rx_bytes;
+		stats->rx_errors = port->stats.rx_errors;
+		stats->rx_dropped = port->stats.rx_dropped;
+
+		stats->rx_length_errors = port->stats.rx_length_errors;
+		stats->rx_over_errors = port->stats.rx_over_errors;
+		stats->rx_crc_errors = port->stats.rx_crc_errors;
+		stats->rx_frame_errors = port->stats.rx_frame_errors;
+
+	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
+
+	/* Racing with MIB and TX completion interrupts */
+	do {
+		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
+
+		stats->tx_errors = port->stats.tx_errors;
+		stats->tx_packets = port->stats.tx_packets;
+		stats->tx_bytes = port->stats.tx_bytes;
+
+		stats->multicast = port->stats.multicast;
+		stats->rx_missed_errors = port->stats.rx_missed_errors;
+		stats->rx_fifo_errors = port->stats.rx_fifo_errors;
+
+	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
+
+	/* Racing with hard_start_xmit */
+	do {
+		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
+
+		stats->tx_dropped = port->stats.tx_dropped;
+
+	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
+
+	stats->rx_dropped += stats->rx_missed_errors;
+}
+
+static int gmac_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	int max_len = gmac_pick_rx_max_len(new_mtu);
+
+	if (max_len < 0)
+		return -EINVAL;
+
+	gmac_disable_tx_rx(netdev);
+
+	netdev->mtu = new_mtu;
+	gmac_update_config0_reg(netdev, max_len << CONFIG0_MAXLEN_SHIFT,
+				CONFIG0_MAXLEN_MASK);
+
+	netdev_update_features(netdev);
+
+	gmac_enable_tx_rx(netdev);
+
+	return 0;
+}
+
+static netdev_features_t gmac_fix_features(struct net_device *netdev,
+					   netdev_features_t features)
+{
+	if (netdev->mtu + ETH_HLEN + VLAN_HLEN > MTU_SIZE_BIT_MASK)
+		features &= ~GMAC_OFFLOAD_FEATURES;
+
+	return features;
+}
+
+static int gmac_set_features(struct net_device *netdev,
+			     netdev_features_t features)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	int enable = features & NETIF_F_RXCSUM;
+	unsigned long flags;
+	u32 reg;
+
+	spin_lock_irqsave(&port->config_lock, flags);
+
+	reg = readl(port->gmac_base + GMAC_CONFIG0);
+	reg = enable ? reg | CONFIG0_RX_CHKSUM : reg & ~CONFIG0_RX_CHKSUM;
+	writel(reg, port->gmac_base + GMAC_CONFIG0);
+
+	spin_unlock_irqrestore(&port->config_lock, flags);
+	return 0;
+}
+
+static int gmac_get_sset_count(struct net_device *netdev, int sset)
+{
+	return sset == ETH_SS_STATS ? GMAC_STATS_NUM : 0;
+}
+
+static void gmac_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
+{
+	if (stringset != ETH_SS_STATS)
+		return;
+
+	memcpy(data, gmac_stats_strings, sizeof(gmac_stats_strings));
+}
+
+static void gmac_get_ethtool_stats(struct net_device *netdev,
+				   struct ethtool_stats *estats, u64 *values)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	unsigned int start;
+	u64 *p;
+	int i;
+
+	gmac_update_hw_stats(netdev);
+
+	/* Racing with MIB interrupt */
+	do {
+		p = values;
+		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
+
+		for (i = 0; i < RX_STATS_NUM; i++)
+			*p++ = port->hw_stats[i];
+
+	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
+	values = p;
+
+	/* Racing with RX NAPI */
+	do {
+		p = values;
+		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
+
+		for (i = 0; i < RX_STATUS_NUM; i++)
+			*p++ = port->rx_stats[i];
+		for (i = 0; i < RX_CHKSUM_NUM; i++)
+			*p++ = port->rx_csum_stats[i];
+		*p++ = port->rx_napi_exits;
+
+	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
+	values = p;
+
+	/* Racing with TX start_xmit */
+	do {
+		p = values;
+		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
+
+		for (i = 0; i < TX_MAX_FRAGS; i++) {
+			*values++ = port->tx_frag_stats[i];
+			port->tx_frag_stats[i] = 0;
+		}
+		*values++ = port->tx_frags_linearized;
+		*values++ = port->tx_hw_csummed;
+
+	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
+}
+
+static int gmac_get_ksettings(struct net_device *netdev,
+			      struct ethtool_link_ksettings *cmd)
+{
+	if (!netdev->phydev)
+		return -ENXIO;
+	phy_ethtool_ksettings_get(netdev->phydev, cmd);
+
+	return 0;
+}
+
+static int gmac_set_ksettings(struct net_device *netdev,
+			      const struct ethtool_link_ksettings *cmd)
+{
+	if (!netdev->phydev)
+		return -ENXIO;
+	return phy_ethtool_ksettings_set(netdev->phydev, cmd);
+}
+
+static int gmac_nway_reset(struct net_device *netdev)
+{
+	if (!netdev->phydev)
+		return -ENXIO;
+	return phy_start_aneg(netdev->phydev);
+}
+
+static void gmac_get_pauseparam(struct net_device *netdev,
+				struct ethtool_pauseparam *pparam)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	union gmac_config0 config0;
+
+	config0.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
+
+	pparam->rx_pause = config0.bits.rx_fc_en;
+	pparam->tx_pause = config0.bits.tx_fc_en;
+	pparam->autoneg = true;
+}
+
+static void gmac_get_ringparam(struct net_device *netdev,
+			       struct ethtool_ringparam *rp)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	union gmac_config0 config0;
+
+	config0.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
+
+	rp->rx_max_pending = 1 << 15;
+	rp->rx_mini_max_pending = 0;
+	rp->rx_jumbo_max_pending = 0;
+	rp->tx_max_pending = 1 << 15;
+
+	rp->rx_pending = 1 << port->rxq_order;
+	rp->rx_mini_pending = 0;
+	rp->rx_jumbo_pending = 0;
+	rp->tx_pending = 1 << port->txq_order;
+}
+
+static int gmac_set_ringparam(struct net_device *netdev,
+			      struct ethtool_ringparam *rp)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+	int err = 0;
+
+	if (netif_running(netdev))
+		return -EBUSY;
+
+	if (rp->rx_pending) {
+		port->rxq_order = min(15, ilog2(rp->rx_pending - 1) + 1);
+		err = geth_resize_freeq(port);
+	}
+	if (rp->tx_pending) {
+		port->txq_order = min(15, ilog2(rp->tx_pending - 1) + 1);
+		port->irq_every_tx_packets = 1 << (port->txq_order - 2);
+	}
+
+	return err;
+}
+
+static int gmac_get_coalesce(struct net_device *netdev,
+			     struct ethtool_coalesce *ecmd)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+
+	ecmd->rx_max_coalesced_frames = 1;
+	ecmd->tx_max_coalesced_frames = port->irq_every_tx_packets;
+	ecmd->rx_coalesce_usecs = port->rx_coalesce_nsecs / 1000;
+
+	return 0;
+}
+
+static int gmac_set_coalesce(struct net_device *netdev,
+			     struct ethtool_coalesce *ecmd)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+
+	if (ecmd->tx_max_coalesced_frames < 1)
+		return -EINVAL;
+	if (ecmd->tx_max_coalesced_frames >= 1 << port->txq_order)
+		return -EINVAL;
+
+	port->irq_every_tx_packets = ecmd->tx_max_coalesced_frames;
+	port->rx_coalesce_nsecs = ecmd->rx_coalesce_usecs * 1000;
+
+	return 0;
+}
+
+static u32 gmac_get_msglevel(struct net_device *netdev)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+
+	return port->msg_enable;
+}
+
+static void gmac_set_msglevel(struct net_device *netdev, u32 level)
+{
+	struct gemini_ethernet_port *port = netdev_priv(netdev);
+
+	port->msg_enable = level;
+}
+
+static void gmac_get_drvinfo(struct net_device *netdev,
+			     struct ethtool_drvinfo *info)
+{
+	strcpy(info->driver,  DRV_NAME);
+	strcpy(info->version, DRV_VERSION);
+	strcpy(info->bus_info, netdev->dev_id ? "1" : "0");
+}
+
+static const struct net_device_ops gmac_351x_ops = {
+	.ndo_init		= gmac_init,
+	.ndo_uninit		= gmac_uninit,
+	.ndo_open		= gmac_open,
+	.ndo_stop		= gmac_stop,
+	.ndo_start_xmit		= gmac_start_xmit,
+	.ndo_tx_timeout		= gmac_tx_timeout,
+	.ndo_set_rx_mode	= gmac_set_rx_mode,
+	.ndo_set_mac_address	= gmac_set_mac_address,
+	.ndo_get_stats64	= gmac_get_stats64,
+	.ndo_change_mtu		= gmac_change_mtu,
+	.ndo_fix_features	= gmac_fix_features,
+	.ndo_set_features	= gmac_set_features,
+};
+
+static const struct ethtool_ops gmac_351x_ethtool_ops = {
+	.get_sset_count	= gmac_get_sset_count,
+	.get_strings	= gmac_get_strings,
+	.get_ethtool_stats = gmac_get_ethtool_stats,
+	.get_link	= ethtool_op_get_link,
+	.get_link_ksettings = gmac_get_ksettings,
+	.set_link_ksettings = gmac_set_ksettings,
+	.nway_reset	= gmac_nway_reset,
+	.get_pauseparam	= gmac_get_pauseparam,
+	.get_ringparam	= gmac_get_ringparam,
+	.set_ringparam	= gmac_set_ringparam,
+	.get_coalesce	= gmac_get_coalesce,
+	.set_coalesce	= gmac_set_coalesce,
+	.get_msglevel	= gmac_get_msglevel,
+	.set_msglevel	= gmac_set_msglevel,
+	.get_drvinfo	= gmac_get_drvinfo,
+};
+
+static irqreturn_t gemini_port_irq_thread(int irq, void *data)
+{
+	unsigned long irqmask = SWFQ_EMPTY_INT_BIT;
+	struct gemini_ethernet_port *port = data;
+	struct gemini_ethernet *geth;
+	unsigned long flags;
+
+	geth = port->geth;
+	/* The queue is half empty so refill it */
+	geth_fill_freeq(geth, true);
+
+	spin_lock_irqsave(&geth->irq_lock, flags);
+	/* ACK queue interrupt */
+	writel(irqmask, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
+	/* Enable queue interrupt again */
+	irqmask |= readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+	writel(irqmask, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+	spin_unlock_irqrestore(&geth->irq_lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t gemini_port_irq(int irq, void *data)
+{
+	struct gemini_ethernet_port *port = data;
+	struct gemini_ethernet *geth;
+	irqreturn_t ret = IRQ_NONE;
+	u32 val, en;
+
+	geth = port->geth;
+	spin_lock(&geth->irq_lock);
+
+	val = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
+	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+
+	if (val & en & SWFQ_EMPTY_INT_BIT) {
+		/* Disable the queue empty interrupt while we work on
+		 * processing the queue. Also disable overrun interrupts
+		 * as there is not much we can do about it here.
+		 */
+		en &= ~(SWFQ_EMPTY_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT
+					   | GMAC1_RX_OVERRUN_INT_BIT);
+		writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	spin_unlock(&geth->irq_lock);
+
+	return ret;
+}
+
+static void gemini_port_remove(struct gemini_ethernet_port *port)
+{
+	if (port->netdev)
+		unregister_netdev(port->netdev);
+	clk_disable_unprepare(port->pclk);
+	geth_cleanup_freeq(port->geth);
+}
+
+static void gemini_ethernet_init(struct gemini_ethernet *geth)
+{
+	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
+	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
+	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
+	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
+	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
+
+	/* Interrupt config:
+	 *
+	 *	GMAC0 intr bits ------> int0 ----> eth0
+	 *	GMAC1 intr bits ------> int1 ----> eth1
+	 *	TOE intr -------------> int1 ----> eth1
+	 *	Classification Intr --> int0 ----> eth0
+	 *	Default Q0 -----------> int0 ----> eth0
+	 *	Default Q1 -----------> int1 ----> eth1
+	 *	FreeQ intr -----------> int1 ----> eth1
+	 */
+	writel(0xCCFC0FC0, geth->base + GLOBAL_INTERRUPT_SELECT_0_REG);
+	writel(0x00F00002, geth->base + GLOBAL_INTERRUPT_SELECT_1_REG);
+	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_2_REG);
+	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_3_REG);
+	writel(0xFF000003, geth->base + GLOBAL_INTERRUPT_SELECT_4_REG);
+
+	/* edge-triggered interrupts packed to level-triggered one... */
+	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
+	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
+	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
+	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
+	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
+
+	/* Set up queue */
+	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
+	writel(0, geth->base + GLOBAL_HW_FREEQ_BASE_SIZE_REG);
+	writel(0, geth->base + GLOBAL_SWFQ_RWPTR_REG);
+	writel(0, geth->base + GLOBAL_HWFQ_RWPTR_REG);
+
+	geth->freeq_frag_order = DEFAULT_RX_BUF_ORDER;
+	/* This makes the queue resize on probe() so that we
+	 * set up and enable the queue IRQ. FIXME: fragile.
+	 */
+	geth->freeq_order = 1;
+}
+
+static void gemini_port_save_mac_addr(struct gemini_ethernet_port *port)
+{
+	port->mac_addr[0] =
+		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD0));
+	port->mac_addr[1] =
+		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD1));
+	port->mac_addr[2] =
+		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD2));
+}
+
+static int gemini_ethernet_port_probe(struct platform_device *pdev)
+{
+	char *port_names[2] = { "ethernet0", "ethernet1" };
+	struct gemini_ethernet_port *port;
+	struct device *dev = &pdev->dev;
+	struct gemini_ethernet *geth;
+	struct net_device *netdev;
+	struct resource *gmacres;
+	struct resource *dmares;
+	struct device *parent;
+	unsigned int id;
+	int irq;
+	int ret;
+
+	parent = dev->parent;
+	geth = dev_get_drvdata(parent);
+
+	if (!strcmp(dev_name(dev), "60008000.ethernet-port"))
+		id = 0;
+	else if (!strcmp(dev_name(dev), "6000c000.ethernet-port"))
+		id = 1;
+	else
+		return -ENODEV;
+
+	dev_info(dev, "probe %s ID %d\n", dev_name(dev), id);
+
+	netdev = alloc_etherdev_mq(sizeof(*port), TX_QUEUE_NUM);
+	if (!netdev) {
+		dev_err(dev, "Can't allocate ethernet device #%d\n", id);
+		return -ENOMEM;
+	}
+
+	port = netdev_priv(netdev);
+	SET_NETDEV_DEV(netdev, dev);
+	port->netdev = netdev;
+	port->id = id;
+	port->geth = geth;
+	port->dev = dev;
+
+	/* DMA memory */
+	dmares = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!dmares) {
+		dev_err(dev, "no DMA resource\n");
+		return -ENODEV;
+	}
+	port->dma_base = devm_ioremap_resource(dev, dmares);
+	if (IS_ERR(port->dma_base))
+		return PTR_ERR(port->dma_base);
+
+	/* GMAC config memory */
+	gmacres = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (!gmacres) {
+		dev_err(dev, "no GMAC resource\n");
+		return -ENODEV;
+	}
+	port->gmac_base = devm_ioremap_resource(dev, gmacres);
+	if (IS_ERR(port->gmac_base))
+		return PTR_ERR(port->gmac_base);
+
+	/* Interrupt */
+	irq = platform_get_irq(pdev, 0);
+	if (irq <= 0) {
+		dev_err(dev, "no IRQ\n");
+		return irq ? irq : -ENODEV;
+	}
+	port->irq = irq;
+
+	/* Clock the port */
+	port->pclk = devm_clk_get(dev, "PCLK");
+	if (IS_ERR(port->pclk)) {
+		dev_err(dev, "no PCLK\n");
+		return PTR_ERR(port->pclk);
+	}
+	ret = clk_prepare_enable(port->pclk);
+	if (ret)
+		return ret;
+
+	/* Maybe there is a nice ethernet address we should use */
+	gemini_port_save_mac_addr(port);
+
+	/* Reset the port */
+	port->reset = devm_reset_control_get_exclusive(dev, NULL);
+	if (IS_ERR(port->reset)) {
+		dev_err(dev, "no reset\n");
+		return PTR_ERR(port->reset);
+	}
+	reset_control_reset(port->reset);
+	usleep_range(100, 500);
+
+	/* Assign pointer in the main state container */
+	if (!id)
+		geth->port0 = port;
+	else
+		geth->port1 = port;
+	platform_set_drvdata(pdev, port);
+
+	/* Set up and register the netdev */
+	netdev->dev_id = port->id;
+	netdev->irq = irq;
+	netdev->netdev_ops = &gmac_351x_ops;
+	netdev->ethtool_ops = &gmac_351x_ethtool_ops;
+
+	spin_lock_init(&port->config_lock);
+	gmac_clear_hw_stats(netdev);
+
+	netdev->hw_features = GMAC_OFFLOAD_FEATURES;
+	netdev->features |= GMAC_OFFLOAD_FEATURES | NETIF_F_GRO;
+
+	port->freeq_refill = 0;
+	netif_napi_add(netdev, &port->napi, gmac_napi_poll,
+		       DEFAULT_NAPI_WEIGHT);
+
+	if (is_valid_ether_addr((void *)port->mac_addr)) {
+		memcpy(netdev->dev_addr, port->mac_addr, ETH_ALEN);
+	} else {
+		dev_dbg(dev, "ethernet address 0x%08x%08x%08x invalid\n",
+			port->mac_addr[0], port->mac_addr[1],
+			port->mac_addr[2]);
+		dev_info(dev, "using a random ethernet address\n");
+		random_ether_addr(netdev->dev_addr);
+	}
+	gmac_write_mac_address(netdev);
+
+	ret = devm_request_threaded_irq(port->dev,
+					port->irq,
+					gemini_port_irq,
+					gemini_port_irq_thread,
+					IRQF_SHARED,
+					port_names[port->id],
+					port);
+	if (ret)
+		return ret;
+
+	ret = register_netdev(netdev);
+	if (!ret) {
+		netdev_info(netdev,
+			    "irq %d, DMA @ 0x%pap, GMAC @ 0x%pap\n",
+			    port->irq, &dmares->start,
+			    &gmacres->start);
+		ret = gmac_setup_phy(netdev);
+		if (ret)
+			netdev_info(netdev,
+				    "PHY init failed, deferring to ifup time\n");
+		return 0;
+	}
+
+	port->netdev = NULL;
+	free_netdev(netdev);
+	return ret;
+}
+
+static int gemini_ethernet_port_remove(struct platform_device *pdev)
+{
+	struct gemini_ethernet_port *port = platform_get_drvdata(pdev);
+
+	gemini_port_remove(port);
+	return 0;
+}
+
+static const struct of_device_id gemini_ethernet_port_of_match[] = {
+	{
+		.compatible = "cortina,gemini-ethernet-port",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, gemini_ethernet_port_of_match);
+
+static struct platform_driver gemini_ethernet_port_driver = {
+	.driver = {
+		.name = "gemini-ethernet-port",
+		.of_match_table = of_match_ptr(gemini_ethernet_port_of_match),
+	},
+	.probe = gemini_ethernet_port_probe,
+	.remove = gemini_ethernet_port_remove,
+};
+
+static int gemini_ethernet_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct gemini_ethernet *geth;
+	unsigned int retry = 5;
+	struct resource *res;
+	u32 val;
+
+	/* Global registers */
+	geth = devm_kzalloc(dev, sizeof(*geth), GFP_KERNEL);
+	if (!geth)
+		return -ENOMEM;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -ENODEV;
+	geth->base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(geth->base))
+		return PTR_ERR(geth->base);
+	geth->dev = dev;
+
+	/* Wait for ports to stabilize */
+	do {
+		udelay(2);
+		val = readl(geth->base + GLOBAL_TOE_VERSION_REG);
+		barrier();
+	} while (!val && --retry);
+	if (!retry) {
+		dev_err(dev, "failed to reset ethernet\n");
+		return -EIO;
+	}
+	dev_info(dev, "Ethernet device ID: 0x%03x, revision 0x%01x\n",
+		 (val >> 4) & 0xFFFU, val & 0xFU);
+
+	spin_lock_init(&geth->irq_lock);
+	spin_lock_init(&geth->freeq_lock);
+	gemini_ethernet_init(geth);
+
+	/* The children will use this */
+	platform_set_drvdata(pdev, geth);
+
+	/* Spawn child devices for the two ports */
+	return devm_of_platform_populate(dev);
+}
+
+static int gemini_ethernet_remove(struct platform_device *pdev)
+{
+	struct gemini_ethernet *geth = platform_get_drvdata(pdev);
+
+	gemini_ethernet_init(geth);
+	geth_cleanup_freeq(geth);
+
+	return 0;
+}
+
+static const struct of_device_id gemini_ethernet_of_match[] = {
+	{
+		.compatible = "cortina,gemini-ethernet",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, gemini_ethernet_of_match);
+
+static struct platform_driver gemini_ethernet_driver = {
+	.driver = {
+		.name = DRV_NAME,
+		.of_match_table = of_match_ptr(gemini_ethernet_of_match),
+	},
+	.probe = gemini_ethernet_probe,
+	.remove = gemini_ethernet_remove,
+};
+
+static int __init gemini_ethernet_module_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&gemini_ethernet_port_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&gemini_ethernet_driver);
+	if (ret) {
+		platform_driver_unregister(&gemini_ethernet_port_driver);
+		return ret;
+	}
+
+	return 0;
+}
+module_init(gemini_ethernet_module_init);
+
+static void __exit gemini_ethernet_module_exit(void)
+{
+	platform_driver_unregister(&gemini_ethernet_driver);
+	platform_driver_unregister(&gemini_ethernet_port_driver);
+}
+module_exit(gemini_ethernet_module_exit);
+
+MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
+MODULE_DESCRIPTION("StorLink SL351x (Gemini) ethernet driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/drivers/net/ethernet/cortina/gemini.h b/drivers/net/ethernet/cortina/gemini.h
new file mode 100644
index 000000000000..0b12f89bf89a
--- /dev/null
+++ b/drivers/net/ethernet/cortina/gemini.h
@@ -0,0 +1,958 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Register definitions for Gemini GMAC Ethernet device driver
+ *
+ * Copyright (C) 2006 Storlink, Corp.
+ * Copyright (C) 2008-2009 Paulius Zaleckas <paulius.zaleckas@teltonika.lt>
+ * Copyright (C) 2010 Michał Mirosław <mirq-linux@rere.qmqm.pl>
+ * Copytight (C) 2017 Linus Walleij <linus.walleij@linaro.org>
+ */
+#ifndef _GEMINI_ETHERNET_H
+#define _GEMINI_ETHERNET_H
+
+#include <linux/bitops.h>
+
+/* Base Registers */
+#define TOE_NONTOE_QUE_HDR_BASE		0x2000
+#define TOE_TOE_QUE_HDR_BASE		0x3000
+
+/* Queue ID */
+#define TOE_SW_FREE_QID			0x00
+#define TOE_HW_FREE_QID			0x01
+#define TOE_GMAC0_SW_TXQ0_QID		0x02
+#define TOE_GMAC0_SW_TXQ1_QID		0x03
+#define TOE_GMAC0_SW_TXQ2_QID		0x04
+#define TOE_GMAC0_SW_TXQ3_QID		0x05
+#define TOE_GMAC0_SW_TXQ4_QID		0x06
+#define TOE_GMAC0_SW_TXQ5_QID		0x07
+#define TOE_GMAC0_HW_TXQ0_QID		0x08
+#define TOE_GMAC0_HW_TXQ1_QID		0x09
+#define TOE_GMAC0_HW_TXQ2_QID		0x0A
+#define TOE_GMAC0_HW_TXQ3_QID		0x0B
+#define TOE_GMAC1_SW_TXQ0_QID		0x12
+#define TOE_GMAC1_SW_TXQ1_QID		0x13
+#define TOE_GMAC1_SW_TXQ2_QID		0x14
+#define TOE_GMAC1_SW_TXQ3_QID		0x15
+#define TOE_GMAC1_SW_TXQ4_QID		0x16
+#define TOE_GMAC1_SW_TXQ5_QID		0x17
+#define TOE_GMAC1_HW_TXQ0_QID		0x18
+#define TOE_GMAC1_HW_TXQ1_QID		0x19
+#define TOE_GMAC1_HW_TXQ2_QID		0x1A
+#define TOE_GMAC1_HW_TXQ3_QID		0x1B
+#define TOE_GMAC0_DEFAULT_QID		0x20
+#define TOE_GMAC1_DEFAULT_QID		0x21
+#define TOE_CLASSIFICATION_QID(x)	(0x22 + x)	/* 0x22 ~ 0x2F */
+#define TOE_TOE_QID(x)			(0x40 + x)	/* 0x40 ~ 0x7F */
+
+/* TOE DMA Queue Size should be 2^n, n = 6...12
+ * TOE DMA Queues are the following queue types:
+ *		SW Free Queue, HW Free Queue,
+ *		GMAC 0/1 SW TX Q0-5, and GMAC 0/1 HW TX Q0-5
+ * The base address and descriptor number are configured at
+ * DMA Queues Descriptor Ring Base Address/Size Register (offset 0x0004)
+ */
+#define GET_WPTR(addr)			readw((addr) + 2)
+#define GET_RPTR(addr)			readw((addr))
+#define SET_WPTR(addr, data)		writew((data), (addr) + 2)
+#define SET_RPTR(addr, data)		writew((data), (addr))
+#define __RWPTR_NEXT(x, mask)		(((unsigned int)(x) + 1) & (mask))
+#define __RWPTR_PREV(x, mask)		(((unsigned int)(x) - 1) & (mask))
+#define __RWPTR_DISTANCE(r, w, mask)	(((unsigned int)(w) - (r)) & (mask))
+#define __RWPTR_MASK(order)		((1 << (order)) - 1)
+#define RWPTR_NEXT(x, order)		__RWPTR_NEXT((x), __RWPTR_MASK((order)))
+#define RWPTR_PREV(x, order)		__RWPTR_PREV((x), __RWPTR_MASK((order)))
+#define RWPTR_DISTANCE(r, w, order)	__RWPTR_DISTANCE((r), (w), \
+						__RWPTR_MASK((order)))
+
+/* Global registers */
+#define GLOBAL_TOE_VERSION_REG		0x0000
+#define GLOBAL_SW_FREEQ_BASE_SIZE_REG	0x0004
+#define GLOBAL_HW_FREEQ_BASE_SIZE_REG	0x0008
+#define GLOBAL_DMA_SKB_SIZE_REG		0x0010
+#define GLOBAL_SWFQ_RWPTR_REG		0x0014
+#define GLOBAL_HWFQ_RWPTR_REG		0x0018
+#define GLOBAL_INTERRUPT_STATUS_0_REG	0x0020
+#define GLOBAL_INTERRUPT_ENABLE_0_REG	0x0024
+#define GLOBAL_INTERRUPT_SELECT_0_REG	0x0028
+#define GLOBAL_INTERRUPT_STATUS_1_REG	0x0030
+#define GLOBAL_INTERRUPT_ENABLE_1_REG	0x0034
+#define GLOBAL_INTERRUPT_SELECT_1_REG	0x0038
+#define GLOBAL_INTERRUPT_STATUS_2_REG	0x0040
+#define GLOBAL_INTERRUPT_ENABLE_2_REG	0x0044
+#define GLOBAL_INTERRUPT_SELECT_2_REG	0x0048
+#define GLOBAL_INTERRUPT_STATUS_3_REG	0x0050
+#define GLOBAL_INTERRUPT_ENABLE_3_REG	0x0054
+#define GLOBAL_INTERRUPT_SELECT_3_REG	0x0058
+#define GLOBAL_INTERRUPT_STATUS_4_REG	0x0060
+#define GLOBAL_INTERRUPT_ENABLE_4_REG	0x0064
+#define GLOBAL_INTERRUPT_SELECT_4_REG	0x0068
+#define GLOBAL_HASH_TABLE_BASE_REG	0x006C
+#define GLOBAL_QUEUE_THRESHOLD_REG	0x0070
+
+/* GMAC 0/1 DMA/TOE register */
+#define GMAC_DMA_CTRL_REG		0x0000
+#define GMAC_TX_WEIGHTING_CTRL_0_REG	0x0004
+#define GMAC_TX_WEIGHTING_CTRL_1_REG	0x0008
+#define GMAC_SW_TX_QUEUE0_PTR_REG	0x000C
+#define GMAC_SW_TX_QUEUE1_PTR_REG	0x0010
+#define GMAC_SW_TX_QUEUE2_PTR_REG	0x0014
+#define GMAC_SW_TX_QUEUE3_PTR_REG	0x0018
+#define GMAC_SW_TX_QUEUE4_PTR_REG	0x001C
+#define GMAC_SW_TX_QUEUE5_PTR_REG	0x0020
+#define GMAC_SW_TX_QUEUE_PTR_REG(i)	(GMAC_SW_TX_QUEUE0_PTR_REG + 4 * (i))
+#define GMAC_HW_TX_QUEUE0_PTR_REG	0x0024
+#define GMAC_HW_TX_QUEUE1_PTR_REG	0x0028
+#define GMAC_HW_TX_QUEUE2_PTR_REG	0x002C
+#define GMAC_HW_TX_QUEUE3_PTR_REG	0x0030
+#define GMAC_HW_TX_QUEUE_PTR_REG(i)	(GMAC_HW_TX_QUEUE0_PTR_REG + 4 * (i))
+#define GMAC_DMA_TX_FIRST_DESC_REG	0x0038
+#define GMAC_DMA_TX_CURR_DESC_REG	0x003C
+#define GMAC_DMA_TX_DESC_WORD0_REG	0x0040
+#define GMAC_DMA_TX_DESC_WORD1_REG	0x0044
+#define GMAC_DMA_TX_DESC_WORD2_REG	0x0048
+#define GMAC_DMA_TX_DESC_WORD3_REG	0x004C
+#define GMAC_SW_TX_QUEUE_BASE_REG	0x0050
+#define GMAC_HW_TX_QUEUE_BASE_REG	0x0054
+#define GMAC_DMA_RX_FIRST_DESC_REG	0x0058
+#define GMAC_DMA_RX_CURR_DESC_REG	0x005C
+#define GMAC_DMA_RX_DESC_WORD0_REG	0x0060
+#define GMAC_DMA_RX_DESC_WORD1_REG	0x0064
+#define GMAC_DMA_RX_DESC_WORD2_REG	0x0068
+#define GMAC_DMA_RX_DESC_WORD3_REG	0x006C
+#define GMAC_HASH_ENGINE_REG0		0x0070
+#define GMAC_HASH_ENGINE_REG1		0x0074
+/* matching rule 0 Control register 0 */
+#define GMAC_MR0CR0			0x0078
+#define GMAC_MR0CR1			0x007C
+#define GMAC_MR0CR2			0x0080
+#define GMAC_MR1CR0			0x0084
+#define GMAC_MR1CR1			0x0088
+#define GMAC_MR1CR2			0x008C
+#define GMAC_MR2CR0			0x0090
+#define GMAC_MR2CR1			0x0094
+#define GMAC_MR2CR2			0x0098
+#define GMAC_MR3CR0			0x009C
+#define GMAC_MR3CR1			0x00A0
+#define GMAC_MR3CR2			0x00A4
+/* Support Protocol Register 0 */
+#define GMAC_SPR0			0x00A8
+#define GMAC_SPR1			0x00AC
+#define GMAC_SPR2			0x00B0
+#define GMAC_SPR3			0x00B4
+#define GMAC_SPR4			0x00B8
+#define GMAC_SPR5			0x00BC
+#define GMAC_SPR6			0x00C0
+#define GMAC_SPR7			0x00C4
+/* GMAC Hash/Rx/Tx AHB Weighting register */
+#define GMAC_AHB_WEIGHT_REG		0x00C8
+
+/* TOE GMAC 0/1 register */
+#define GMAC_STA_ADD0			0x0000
+#define GMAC_STA_ADD1			0x0004
+#define GMAC_STA_ADD2			0x0008
+#define GMAC_RX_FLTR			0x000c
+#define GMAC_MCAST_FIL0			0x0010
+#define GMAC_MCAST_FIL1			0x0014
+#define GMAC_CONFIG0			0x0018
+#define GMAC_CONFIG1			0x001c
+#define GMAC_CONFIG2			0x0020
+#define GMAC_CONFIG3			0x0024
+#define GMAC_RESERVED			0x0028
+#define GMAC_STATUS			0x002c
+#define GMAC_IN_DISCARDS		0x0030
+#define GMAC_IN_ERRORS			0x0034
+#define GMAC_IN_MCAST			0x0038
+#define GMAC_IN_BCAST			0x003c
+#define GMAC_IN_MAC1			0x0040	/* for STA 1 MAC Address */
+#define GMAC_IN_MAC2			0x0044	/* for STA 2 MAC Address */
+
+#define RX_STATS_NUM	6
+
+/* DMA Queues description Ring Base Address/Size Register (offset 0x0004) */
+union dma_q_base_size {
+	unsigned int bits32;
+	unsigned int base_size;
+};
+
+#define DMA_Q_BASE_MASK		(~0x0f)
+
+/* DMA SKB Buffer register (offset 0x0008) */
+union dma_skb_size {
+	unsigned int bits32;
+	struct bit_0008 {
+		unsigned int sw_skb_size : 16;	/* SW Free poll SKB Size */
+		unsigned int hw_skb_size : 16;	/* HW Free poll SKB Size */
+	} bits;
+};
+
+/* DMA SW Free Queue Read/Write Pointer Register (offset 0x000c) */
+union dma_rwptr {
+	unsigned int bits32;
+	struct bit_000c {
+		unsigned int rptr	: 16;	/* Read Ptr, RO */
+		unsigned int wptr	: 16;	/* Write Ptr, RW */
+	} bits;
+};
+
+/* Interrupt Status Register 0	(offset 0x0020)
+ * Interrupt Mask Register 0	(offset 0x0024)
+ * Interrupt Select Register 0	(offset 0x0028)
+ */
+#define GMAC1_TXDERR_INT_BIT		BIT(31)
+#define GMAC1_TXPERR_INT_BIT		BIT(30)
+#define GMAC0_TXDERR_INT_BIT		BIT(29)
+#define GMAC0_TXPERR_INT_BIT		BIT(28)
+#define GMAC1_RXDERR_INT_BIT		BIT(27)
+#define GMAC1_RXPERR_INT_BIT		BIT(26)
+#define GMAC0_RXDERR_INT_BIT		BIT(25)
+#define GMAC0_RXPERR_INT_BIT		BIT(24)
+#define GMAC1_SWTQ15_FIN_INT_BIT	BIT(23)
+#define GMAC1_SWTQ14_FIN_INT_BIT	BIT(22)
+#define GMAC1_SWTQ13_FIN_INT_BIT	BIT(21)
+#define GMAC1_SWTQ12_FIN_INT_BIT	BIT(20)
+#define GMAC1_SWTQ11_FIN_INT_BIT	BIT(19)
+#define GMAC1_SWTQ10_FIN_INT_BIT	BIT(18)
+#define GMAC0_SWTQ05_FIN_INT_BIT	BIT(17)
+#define GMAC0_SWTQ04_FIN_INT_BIT	BIT(16)
+#define GMAC0_SWTQ03_FIN_INT_BIT	BIT(15)
+#define GMAC0_SWTQ02_FIN_INT_BIT	BIT(14)
+#define GMAC0_SWTQ01_FIN_INT_BIT	BIT(13)
+#define GMAC0_SWTQ00_FIN_INT_BIT	BIT(12)
+#define GMAC1_SWTQ15_EOF_INT_BIT	BIT(11)
+#define GMAC1_SWTQ14_EOF_INT_BIT	BIT(10)
+#define GMAC1_SWTQ13_EOF_INT_BIT	BIT(9)
+#define GMAC1_SWTQ12_EOF_INT_BIT	BIT(8)
+#define GMAC1_SWTQ11_EOF_INT_BIT	BIT(7)
+#define GMAC1_SWTQ10_EOF_INT_BIT	BIT(6)
+#define GMAC0_SWTQ05_EOF_INT_BIT	BIT(5)
+#define GMAC0_SWTQ04_EOF_INT_BIT	BIT(4)
+#define GMAC0_SWTQ03_EOF_INT_BIT	BIT(3)
+#define GMAC0_SWTQ02_EOF_INT_BIT	BIT(2)
+#define GMAC0_SWTQ01_EOF_INT_BIT	BIT(1)
+#define GMAC0_SWTQ00_EOF_INT_BIT	BIT(0)
+
+/* Interrupt Status Register 1	(offset 0x0030)
+ * Interrupt Mask Register 1	(offset 0x0034)
+ * Interrupt Select Register 1	(offset 0x0038)
+ */
+#define TOE_IQ3_FULL_INT_BIT		BIT(31)
+#define TOE_IQ2_FULL_INT_BIT		BIT(30)
+#define TOE_IQ1_FULL_INT_BIT		BIT(29)
+#define TOE_IQ0_FULL_INT_BIT		BIT(28)
+#define TOE_IQ3_INT_BIT			BIT(27)
+#define TOE_IQ2_INT_BIT			BIT(26)
+#define TOE_IQ1_INT_BIT			BIT(25)
+#define TOE_IQ0_INT_BIT			BIT(24)
+#define GMAC1_HWTQ13_EOF_INT_BIT	BIT(23)
+#define GMAC1_HWTQ12_EOF_INT_BIT	BIT(22)
+#define GMAC1_HWTQ11_EOF_INT_BIT	BIT(21)
+#define GMAC1_HWTQ10_EOF_INT_BIT	BIT(20)
+#define GMAC0_HWTQ03_EOF_INT_BIT	BIT(19)
+#define GMAC0_HWTQ02_EOF_INT_BIT	BIT(18)
+#define GMAC0_HWTQ01_EOF_INT_BIT	BIT(17)
+#define GMAC0_HWTQ00_EOF_INT_BIT	BIT(16)
+#define CLASS_RX_INT_BIT(x)		BIT((x + 2))
+#define DEFAULT_Q1_INT_BIT		BIT(1)
+#define DEFAULT_Q0_INT_BIT		BIT(0)
+
+#define TOE_IQ_INT_BITS		(TOE_IQ0_INT_BIT | TOE_IQ1_INT_BIT | \
+				 TOE_IQ2_INT_BIT | TOE_IQ3_INT_BIT)
+#define	TOE_IQ_FULL_BITS	(TOE_IQ0_FULL_INT_BIT | TOE_IQ1_FULL_INT_BIT | \
+				 TOE_IQ2_FULL_INT_BIT | TOE_IQ3_FULL_INT_BIT)
+#define	TOE_IQ_ALL_BITS		(TOE_IQ_INT_BITS | TOE_IQ_FULL_BITS)
+#define TOE_CLASS_RX_INT_BITS	0xfffc
+
+/* Interrupt Status Register 2	(offset 0x0040)
+ * Interrupt Mask Register 2	(offset 0x0044)
+ * Interrupt Select Register 2	(offset 0x0048)
+ */
+#define TOE_QL_FULL_INT_BIT(x)		BIT(x)
+
+/* Interrupt Status Register 3	(offset 0x0050)
+ * Interrupt Mask Register 3	(offset 0x0054)
+ * Interrupt Select Register 3	(offset 0x0058)
+ */
+#define TOE_QH_FULL_INT_BIT(x)		BIT(x - 32)
+
+/* Interrupt Status Register 4	(offset 0x0060)
+ * Interrupt Mask Register 4	(offset 0x0064)
+ * Interrupt Select Register 4	(offset 0x0068)
+ */
+#define GMAC1_RESERVED_INT_BIT		BIT(31)
+#define GMAC1_MIB_INT_BIT		BIT(30)
+#define GMAC1_RX_PAUSE_ON_INT_BIT	BIT(29)
+#define GMAC1_TX_PAUSE_ON_INT_BIT	BIT(28)
+#define GMAC1_RX_PAUSE_OFF_INT_BIT	BIT(27)
+#define GMAC1_TX_PAUSE_OFF_INT_BIT	BIT(26)
+#define GMAC1_RX_OVERRUN_INT_BIT	BIT(25)
+#define GMAC1_STATUS_CHANGE_INT_BIT	BIT(24)
+#define GMAC0_RESERVED_INT_BIT		BIT(23)
+#define GMAC0_MIB_INT_BIT		BIT(22)
+#define GMAC0_RX_PAUSE_ON_INT_BIT	BIT(21)
+#define GMAC0_TX_PAUSE_ON_INT_BIT	BIT(20)
+#define GMAC0_RX_PAUSE_OFF_INT_BIT	BIT(19)
+#define GMAC0_TX_PAUSE_OFF_INT_BIT	BIT(18)
+#define GMAC0_RX_OVERRUN_INT_BIT	BIT(17)
+#define GMAC0_STATUS_CHANGE_INT_BIT	BIT(16)
+#define CLASS_RX_FULL_INT_BIT(x)	BIT(x + 2)
+#define HWFQ_EMPTY_INT_BIT		BIT(1)
+#define SWFQ_EMPTY_INT_BIT		BIT(0)
+
+#define GMAC0_INT_BITS	(GMAC0_RESERVED_INT_BIT | GMAC0_MIB_INT_BIT | \
+			 GMAC0_RX_PAUSE_ON_INT_BIT | \
+			 GMAC0_TX_PAUSE_ON_INT_BIT | \
+			 GMAC0_RX_PAUSE_OFF_INT_BIT | \
+			 GMAC0_TX_PAUSE_OFF_INT_BIT | \
+			 GMAC0_RX_OVERRUN_INT_BIT | \
+			 GMAC0_STATUS_CHANGE_INT_BIT)
+#define GMAC1_INT_BITS	(GMAC1_RESERVED_INT_BIT | GMAC1_MIB_INT_BIT | \
+			 GMAC1_RX_PAUSE_ON_INT_BIT | \
+			 GMAC1_TX_PAUSE_ON_INT_BIT | \
+			 GMAC1_RX_PAUSE_OFF_INT_BIT | \
+			 GMAC1_TX_PAUSE_OFF_INT_BIT | \
+			 GMAC1_RX_OVERRUN_INT_BIT | \
+			 GMAC1_STATUS_CHANGE_INT_BIT)
+
+#define CLASS_RX_FULL_INT_BITS		0xfffc
+
+/* GLOBAL_QUEUE_THRESHOLD_REG	(offset 0x0070) */
+union queue_threshold {
+	unsigned int bits32;
+	struct bit_0070_2 {
+		/*  7:0 Software Free Queue Empty Threshold */
+		unsigned int swfq_empty:8;
+		/* 15:8 Hardware Free Queue Empty Threshold */
+		unsigned int hwfq_empty:8;
+		/* 23:16 */
+		unsigned int intrq:8;
+		/* 31:24 */
+		unsigned int toe_class:8;
+	} bits;
+};
+
+/* GMAC DMA Control Register
+ * GMAC0 offset 0x8000
+ * GMAC1 offset 0xC000
+ */
+union gmac_dma_ctrl {
+	unsigned int bits32;
+	struct bit_8000 {
+		/* bit 1:0 Peripheral Bus Width */
+		unsigned int td_bus:2;
+		/* bit 3:2 TxDMA max burst size for every AHB request */
+		unsigned int td_burst_size:2;
+		/* bit 7:4 TxDMA protection control */
+		unsigned int td_prot:4;
+		/* bit 9:8 Peripheral Bus Width */
+		unsigned int rd_bus:2;
+		/* bit 11:10 DMA max burst size for every AHB request */
+		unsigned int rd_burst_size:2;
+		/* bit 15:12 DMA Protection Control */
+		unsigned int rd_prot:4;
+		/* bit 17:16 */
+		unsigned int rd_insert_bytes:2;
+		/* bit 27:18 */
+		unsigned int reserved:10;
+		/* bit 28 1: Drop, 0: Accept */
+		unsigned int drop_small_ack:1;
+		/* bit 29 Loopback TxDMA to RxDMA */
+		unsigned int loopback:1;
+		/* bit 30 Tx DMA Enable */
+		unsigned int td_enable:1;
+		/* bit 31 Rx DMA Enable */
+		unsigned int rd_enable:1;
+	} bits;
+};
+
+/* GMAC Tx Weighting Control Register 0
+ * GMAC0 offset 0x8004
+ * GMAC1 offset 0xC004
+ */
+union gmac_tx_wcr0 {
+	unsigned int bits32;
+	struct bit_8004 {
+		/* bit 5:0 HW TX Queue 3 */
+		unsigned int hw_tq0:6;
+		/* bit 11:6 HW TX Queue 2 */
+		unsigned int hw_tq1:6;
+		/* bit 17:12 HW TX Queue 1 */
+		unsigned int hw_tq2:6;
+		/* bit 23:18 HW TX Queue 0 */
+		unsigned int hw_tq3:6;
+		/* bit 31:24 */
+		unsigned int reserved:8;
+	} bits;
+};
+
+/* GMAC Tx Weighting Control Register 1
+ * GMAC0 offset 0x8008
+ * GMAC1 offset 0xC008
+ */
+union gmac_tx_wcr1 {
+	unsigned int bits32;
+	struct bit_8008 {
+		/* bit 4:0 SW TX Queue 0 */
+		unsigned int sw_tq0:5;
+		/* bit 9:5 SW TX Queue 1 */
+		unsigned int sw_tq1:5;
+		/* bit 14:10 SW TX Queue 2 */
+		unsigned int sw_tq2:5;
+		/* bit 19:15 SW TX Queue 3 */
+		unsigned int sw_tq3:5;
+		/* bit 24:20 SW TX Queue 4 */
+		unsigned int sw_tq4:5;
+		/* bit 29:25 SW TX Queue 5 */
+		unsigned int sw_tq5:5;
+		/* bit 31:30 */
+		unsigned int reserved:2;
+	} bits;
+};
+
+/* GMAC DMA Tx Description Word 0 Register
+ * GMAC0 offset 0x8040
+ * GMAC1 offset 0xC040
+ */
+union gmac_txdesc_0 {
+	unsigned int bits32;
+	struct bit_8040 {
+		/* bit 15:0 Transfer size */
+		unsigned int buffer_size:16;
+		/* bit 21:16 number of descriptors used for the current frame */
+		unsigned int desc_count:6;
+		/* bit 22 Tx Status, 1: Successful 0: Failed */
+		unsigned int status_tx_ok:1;
+		/* bit 28:23 Tx Status, Reserved bits */
+		unsigned int status_rvd:6;
+		/* bit 29 protocol error during processing this descriptor */
+		unsigned int perr:1;
+		/* bit 30 data error during processing this descriptor */
+		unsigned int derr:1;
+		/* bit 31 */
+		unsigned int reserved:1;
+	} bits;
+};
+
+/* GMAC DMA Tx Description Word 1 Register
+ * GMAC0 offset 0x8044
+ * GMAC1 offset 0xC044
+ */
+union gmac_txdesc_1 {
+	unsigned int bits32;
+	struct txdesc_word1 {
+		/* bit 15: 0 Tx Frame Byte Count */
+		unsigned int byte_count:16;
+		/* bit 16 TSS segmentation use MTU setting */
+		unsigned int mtu_enable:1;
+		/* bit 17 IPV4 Header Checksum Enable */
+		unsigned int ip_chksum:1;
+		/* bit 18 IPV6 Tx Enable */
+		unsigned int ipv6_enable:1;
+		/* bit 19 TCP Checksum Enable */
+		unsigned int tcp_chksum:1;
+		/* bit 20 UDP Checksum Enable */
+		unsigned int udp_chksum:1;
+		/* bit 21 Bypass HW offload engine */
+		unsigned int bypass_tss:1;
+		/* bit 22 Don't update IP length field */
+		unsigned int ip_fixed_len:1;
+		/* bit 31:23 Tx Flag, Reserved */
+		unsigned int reserved:9;
+	} bits;
+};
+
+#define TSS_IP_FIXED_LEN_BIT	BIT(22)
+#define TSS_BYPASS_BIT		BIT(21)
+#define TSS_UDP_CHKSUM_BIT	BIT(20)
+#define TSS_TCP_CHKSUM_BIT	BIT(19)
+#define TSS_IPV6_ENABLE_BIT	BIT(18)
+#define TSS_IP_CHKSUM_BIT	BIT(17)
+#define TSS_MTU_ENABLE_BIT	BIT(16)
+
+#define TSS_CHECKUM_ENABLE	\
+	(TSS_IP_CHKSUM_BIT | TSS_IPV6_ENABLE_BIT | \
+	 TSS_TCP_CHKSUM_BIT | TSS_UDP_CHKSUM_BIT)
+
+/* GMAC DMA Tx Description Word 2 Register
+ * GMAC0 offset 0x8048
+ * GMAC1 offset 0xC048
+ */
+union gmac_txdesc_2 {
+	unsigned int	bits32;
+	unsigned int	buf_adr;
+};
+
+/* GMAC DMA Tx Description Word 3 Register
+ * GMAC0 offset 0x804C
+ * GMAC1 offset 0xC04C
+ */
+union gmac_txdesc_3 {
+	unsigned int bits32;
+	struct txdesc_word3 {
+		/* bit 12: 0 Tx Frame Byte Count */
+		unsigned int mtu_size:13;
+		/* bit 28:13 */
+		unsigned int reserved:16;
+		/* bit 29 End of frame interrupt enable */
+		unsigned int eofie:1;
+		/* bit 31:30 11: only one, 10: first, 01: last, 00: linking */
+		unsigned int sof_eof:2;
+	} bits;
+};
+
+#define SOF_EOF_BIT_MASK	0x3fffffff
+#define SOF_BIT			0x80000000
+#define EOF_BIT			0x40000000
+#define EOFIE_BIT		BIT(29)
+#define MTU_SIZE_BIT_MASK	0x1fff
+
+/* GMAC Tx Descriptor */
+struct gmac_txdesc {
+	union gmac_txdesc_0 word0;
+	union gmac_txdesc_1 word1;
+	union gmac_txdesc_2 word2;
+	union gmac_txdesc_3 word3;
+};
+
+/* GMAC DMA Rx Description Word 0 Register
+ * GMAC0 offset 0x8060
+ * GMAC1 offset 0xC060
+ */
+union gmac_rxdesc_0 {
+	unsigned int bits32;
+	struct bit_8060 {
+		/* bit 15:0 number of descriptors used for the current frame */
+		unsigned int buffer_size:16;
+		/* bit 21:16 number of descriptors used for the current frame */
+		unsigned int desc_count:6;
+		/* bit 24:22 Status of rx frame */
+		unsigned int status:4;
+		/* bit 28:26 Check Sum Status */
+		unsigned int chksum_status:3;
+		/* bit 29 protocol error during processing this descriptor */
+		unsigned int perr:1;
+		/* bit 30 data error during processing this descriptor */
+		unsigned int derr:1;
+		/* bit 31 TOE/CIS Queue Full dropped packet to default queue */
+		unsigned int drop:1;
+	} bits;
+};
+
+#define	GMAC_RXDESC_0_T_derr			BIT(30)
+#define	GMAC_RXDESC_0_T_perr			BIT(29)
+#define	GMAC_RXDESC_0_T_chksum_status(x)	BIT(x + 26)
+#define	GMAC_RXDESC_0_T_status(x)		BIT(x + 22)
+#define	GMAC_RXDESC_0_T_desc_count(x)		BIT(x + 16)
+
+#define	RX_CHKSUM_IP_UDP_TCP_OK			0
+#define	RX_CHKSUM_IP_OK_ONLY			1
+#define	RX_CHKSUM_NONE				2
+#define	RX_CHKSUM_IP_ERR_UNKNOWN		4
+#define	RX_CHKSUM_IP_ERR			5
+#define	RX_CHKSUM_TCP_UDP_ERR			6
+#define RX_CHKSUM_NUM				8
+
+#define RX_STATUS_GOOD_FRAME			0
+#define RX_STATUS_TOO_LONG_GOOD_CRC		1
+#define RX_STATUS_RUNT_FRAME			2
+#define RX_STATUS_SFD_NOT_FOUND			3
+#define RX_STATUS_CRC_ERROR			4
+#define RX_STATUS_TOO_LONG_BAD_CRC		5
+#define RX_STATUS_ALIGNMENT_ERROR		6
+#define RX_STATUS_TOO_LONG_BAD_ALIGN		7
+#define RX_STATUS_RX_ERR			8
+#define RX_STATUS_DA_FILTERED			9
+#define RX_STATUS_BUFFER_FULL			10
+#define RX_STATUS_NUM				16
+
+#define RX_ERROR_LENGTH(s) \
+	((s) == RX_STATUS_TOO_LONG_GOOD_CRC || \
+	 (s) == RX_STATUS_TOO_LONG_BAD_CRC || \
+	 (s) == RX_STATUS_TOO_LONG_BAD_ALIGN)
+#define RX_ERROR_OVER(s) \
+	((s) == RX_STATUS_BUFFER_FULL)
+#define RX_ERROR_CRC(s) \
+	((s) == RX_STATUS_CRC_ERROR || \
+	 (s) == RX_STATUS_TOO_LONG_BAD_CRC)
+#define RX_ERROR_FRAME(s) \
+	((s) == RX_STATUS_ALIGNMENT_ERROR || \
+	 (s) == RX_STATUS_TOO_LONG_BAD_ALIGN)
+#define RX_ERROR_FIFO(s) \
+	(0)
+
+/* GMAC DMA Rx Description Word 1 Register
+ * GMAC0 offset 0x8064
+ * GMAC1 offset 0xC064
+ */
+union gmac_rxdesc_1 {
+	unsigned int bits32;
+	struct rxdesc_word1 {
+		/* bit 15: 0 Rx Frame Byte Count */
+		unsigned int byte_count:16;
+		/* bit 31:16 Software ID */
+		unsigned int sw_id:16;
+	} bits;
+};
+
+/* GMAC DMA Rx Description Word 2 Register
+ * GMAC0 offset 0x8068
+ * GMAC1 offset 0xC068
+ */
+union gmac_rxdesc_2 {
+	unsigned int	bits32;
+	unsigned int	buf_adr;
+};
+
+#define RX_INSERT_NONE		0
+#define RX_INSERT_1_BYTE	1
+#define RX_INSERT_2_BYTE	2
+#define RX_INSERT_3_BYTE	3
+
+/* GMAC DMA Rx Description Word 3 Register
+ * GMAC0 offset 0x806C
+ * GMAC1 offset 0xC06C
+ */
+union gmac_rxdesc_3 {
+	unsigned int bits32;
+	struct rxdesc_word3 {
+		/* bit 7: 0 L3 data offset */
+		unsigned int l3_offset:8;
+		/* bit 15: 8 L4 data offset */
+		unsigned int l4_offset:8;
+		/* bit 23: 16 L7 data offset */
+		unsigned int l7_offset:8;
+		/* bit 24 Duplicated ACK detected */
+		unsigned int dup_ack:1;
+		/* bit 25 abnormal case found */
+		unsigned int abnormal:1;
+		/* bit 26 IPV4 option or IPV6 extension header */
+		unsigned int option:1;
+		/* bit 27 Out of Sequence packet */
+		unsigned int out_of_seq:1;
+		/* bit 28 Control Flag is present */
+		unsigned int ctrl_flag:1;
+		/* bit 29 End of frame interrupt enable */
+		unsigned int eofie:1;
+		/* bit 31:30 11: only one, 10: first, 01: last, 00: linking */
+		unsigned int sof_eof:2;
+	} bits;
+};
+
+/* GMAC Rx Descriptor, this is simply fitted over the queue registers */
+struct gmac_rxdesc {
+	union gmac_rxdesc_0 word0;
+	union gmac_rxdesc_1 word1;
+	union gmac_rxdesc_2 word2;
+	union gmac_rxdesc_3 word3;
+};
+
+/* GMAC Matching Rule Control Register 0
+ * GMAC0 offset 0x8078
+ * GMAC1 offset 0xC078
+ */
+#define MR_L2_BIT		BIT(31)
+#define MR_L3_BIT		BIT(30)
+#define MR_L4_BIT		BIT(29)
+#define MR_L7_BIT		BIT(28)
+#define MR_PORT_BIT		BIT(27)
+#define MR_PRIORITY_BIT		BIT(26)
+#define MR_DA_BIT		BIT(23)
+#define MR_SA_BIT		BIT(22)
+#define MR_ETHER_TYPE_BIT	BIT(21)
+#define MR_VLAN_BIT		BIT(20)
+#define MR_PPPOE_BIT		BIT(19)
+#define MR_IP_VER_BIT		BIT(15)
+#define MR_IP_HDR_LEN_BIT	BIT(14)
+#define MR_FLOW_LABLE_BIT	BIT(13)
+#define MR_TOS_TRAFFIC_BIT	BIT(12)
+#define MR_SPR_BIT(x)		BIT(x)
+#define MR_SPR_BITS		0xff
+
+/* GMAC_AHB_WEIGHT registers
+ * GMAC0 offset 0x80C8
+ * GMAC1 offset 0xC0C8
+ */
+union gmac_ahb_weight {
+	unsigned int bits32;
+	struct bit_80C8 {
+		/* 4:0 */
+		unsigned int hash_weight:5;
+		/* 9:5 */
+		unsigned int rx_weight:5;
+		/* 14:10 */
+		unsigned int tx_weight:5;
+		/* 19:15 Rx Data Pre Request FIFO Threshold */
+		unsigned int pre_req:5;
+		/* 24:20 DMA TqCtrl to Start tqDV FIFO Threshold */
+		unsigned int tq_dv_threshold:5;
+		/* 31:25 */
+		unsigned int reserved:7;
+	} bits;
+};
+
+/* GMAC RX FLTR
+ * GMAC0 Offset 0xA00C
+ * GMAC1 Offset 0xE00C
+ */
+union gmac_rx_fltr {
+	unsigned int bits32;
+	struct bit1_000c {
+		/* Enable receive of unicast frames that are sent to STA
+		 * address
+		 */
+		unsigned int unicast:1;
+		/* Enable receive of multicast frames that pass multicast
+		 * filter
+		 */
+		unsigned int multicast:1;
+		/* Enable receive of broadcast frames */
+		unsigned int broadcast:1;
+		/* Enable receive of all frames */
+		unsigned int promiscuous:1;
+		/* Enable receive of all error frames */
+		unsigned int error:1;
+		unsigned int reserved:27;
+	} bits;
+};
+
+/* GMAC Configuration 0
+ * GMAC0 Offset 0xA018
+ * GMAC1 Offset 0xE018
+ */
+union gmac_config0 {
+	unsigned int bits32;
+	struct bit1_0018 {
+		/* 0: disable transmit */
+		unsigned int dis_tx:1;
+		/* 1: disable receive */
+		unsigned int dis_rx:1;
+		/* 2: transmit data loopback enable */
+		unsigned int loop_back:1;
+		/* 3: flow control also trigged by Rx queues */
+		unsigned int flow_ctrl:1;
+		/* 4-7: adjust IFG from 96+/-56 */
+		unsigned int adj_ifg:4;
+		/* 8-10 maximum receive frame length allowed */
+		unsigned int max_len:3;
+		/* 11: disable back-off function */
+		unsigned int dis_bkoff:1;
+		/* 12: disable 16 collisions abort function */
+		unsigned int dis_col:1;
+		/* 13: speed up timers in simulation */
+		unsigned int sim_test:1;
+		/* 14: RX flow control enable */
+		unsigned int rx_fc_en:1;
+		/* 15: TX flow control enable */
+		unsigned int tx_fc_en:1;
+		/* 16: RGMII in-band status enable */
+		unsigned int rgmii_en:1;
+		/* 17: IPv4 RX Checksum enable */
+		unsigned int ipv4_rx_chksum:1;
+		/* 18: IPv6 RX Checksum enable */
+		unsigned int ipv6_rx_chksum:1;
+		/* 19: Remove Rx VLAN tag */
+		unsigned int rx_tag_remove:1;
+		/* 20 */
+		unsigned int rgmm_edge:1;
+		/* 21 */
+		unsigned int rxc_inv:1;
+		/* 22 */
+		unsigned int ipv6_exthdr_order:1;
+		/* 23 */
+		unsigned int rx_err_detect:1;
+		/* 24 */
+		unsigned int port0_chk_hwq:1;
+		/* 25 */
+		unsigned int port1_chk_hwq:1;
+		/* 26 */
+		unsigned int port0_chk_toeq:1;
+		/* 27 */
+		unsigned int port1_chk_toeq:1;
+		/* 28 */
+		unsigned int port0_chk_classq:1;
+		/* 29 */
+		unsigned int port1_chk_classq:1;
+		/* 30, 31 */
+		unsigned int reserved:2;
+	} bits;
+};
+
+#define CONFIG0_TX_RX_DISABLE	(BIT(1) | BIT(0))
+#define CONFIG0_RX_CHKSUM	(BIT(18) | BIT(17))
+#define CONFIG0_FLOW_RX		BIT(14)
+#define CONFIG0_FLOW_TX		BIT(15)
+#define CONFIG0_FLOW_TX_RX	(BIT(14) | BIT(15))
+#define CONFIG0_FLOW_CTL	(BIT(14) | BIT(15))
+
+#define CONFIG0_MAXLEN_SHIFT	8
+#define CONFIG0_MAXLEN_MASK	(7 << CONFIG0_MAXLEN_SHIFT)
+#define  CONFIG0_MAXLEN_1536	0
+#define  CONFIG0_MAXLEN_1518	1
+#define  CONFIG0_MAXLEN_1522	2
+#define  CONFIG0_MAXLEN_1542	3
+#define  CONFIG0_MAXLEN_9k	4	/* 9212 */
+#define  CONFIG0_MAXLEN_10k	5	/* 10236 */
+#define  CONFIG0_MAXLEN_1518__6	6
+#define  CONFIG0_MAXLEN_1518__7	7
+
+/* GMAC Configuration 1
+ * GMAC0 Offset 0xA01C
+ * GMAC1 Offset 0xE01C
+ */
+union gmac_config1 {
+	unsigned int bits32;
+	struct bit1_001c {
+		/* Flow control set threshold */
+		unsigned int set_threshold:8;
+		/* Flow control release threshold */
+		unsigned int rel_threshold:8;
+		unsigned int reserved:16;
+	} bits;
+};
+
+#define GMAC_FLOWCTRL_SET_MAX		32
+#define GMAC_FLOWCTRL_SET_MIN		0
+#define GMAC_FLOWCTRL_RELEASE_MAX	32
+#define GMAC_FLOWCTRL_RELEASE_MIN	0
+
+/* GMAC Configuration 2
+ * GMAC0 Offset 0xA020
+ * GMAC1 Offset 0xE020
+ */
+union gmac_config2 {
+	unsigned int bits32;
+	struct bit1_0020 {
+		/* Flow control set threshold */
+		unsigned int set_threshold:16;
+		/* Flow control release threshold */
+		unsigned int rel_threshold:16;
+	} bits;
+};
+
+/* GMAC Configuration 3
+ * GMAC0 Offset 0xA024
+ * GMAC1 Offset 0xE024
+ */
+union gmac_config3 {
+	unsigned int bits32;
+	struct bit1_0024 {
+		/* Flow control set threshold */
+		unsigned int set_threshold:16;
+		/* Flow control release threshold */
+		unsigned int rel_threshold:16;
+	} bits;
+};
+
+/* GMAC STATUS
+ * GMAC0 Offset 0xA02C
+ * GMAC1 Offset 0xE02C
+ */
+union gmac_status {
+	unsigned int bits32;
+	struct bit1_002c {
+		/* Link status */
+		unsigned int link:1;
+		/* Link speed(00->2.5M 01->25M 10->125M) */
+		unsigned int speed:2;
+		/* Duplex mode */
+		unsigned int duplex:1;
+		unsigned int reserved_1:1;
+		/* PHY interface type */
+		unsigned int mii_rmii:2;
+		unsigned int reserved_2:25;
+	} bits;
+};
+
+#define GMAC_SPEED_10			0
+#define GMAC_SPEED_100			1
+#define GMAC_SPEED_1000			2
+
+#define GMAC_PHY_MII			0
+#define GMAC_PHY_GMII			1
+#define GMAC_PHY_RGMII_100_10		2
+#define GMAC_PHY_RGMII_1000		3
+
+/* Queue Header
+ *	(1) TOE Queue Header
+ *	(2) Non-TOE Queue Header
+ *	(3) Interrupt Queue Header
+ *
+ * memory Layout
+ *	TOE Queue Header
+ *		     0x60003000 +---------------------------+ 0x0000
+ *				|     TOE Queue 0 Header    |
+ *				|         8 * 4 Bytes	    |
+ *				+---------------------------+ 0x0020
+ *				|     TOE Queue 1 Header    |
+ *				|         8 * 4 Bytes	    |
+ *				+---------------------------+ 0x0040
+ *				|          ......           |
+ *				|                           |
+ *				+---------------------------+
+ *
+ *	Non TOE Queue Header
+ *		     0x60002000 +---------------------------+ 0x0000
+ *				|   Default Queue 0 Header  |
+ *				|         2 * 4 Bytes       |
+ *				+---------------------------+ 0x0008
+ *				|   Default Queue 1 Header  |
+ *				|         2 * 4 Bytes       |
+ *				+---------------------------+ 0x0010
+ *				|   Classification Queue 0  |
+ *				|	  2 * 4 Bytes       |
+ *				+---------------------------+
+ *				|   Classification Queue 1  |
+ *				|	  2 * 4 Bytes       |
+ *				+---------------------------+ (n * 8 + 0x10)
+ *				|		...	    |
+ *				|	  2 * 4 Bytes	    |
+ *				+---------------------------+ (13 * 8 + 0x10)
+ *				|   Classification Queue 13 |
+ *				|	  2 * 4 Bytes	    |
+ *				+---------------------------+ 0x80
+ *				|      Interrupt Queue 0    |
+ *				|	  2 * 4 Bytes	    |
+ *				+---------------------------+
+ *				|      Interrupt Queue 1    |
+ *				|	  2 * 4 Bytes	    |
+ *				+---------------------------+
+ *				|      Interrupt Queue 2    |
+ *				|	  2 * 4 Bytes	    |
+ *				+---------------------------+
+ *				|      Interrupt Queue 3    |
+ *				|	  2 * 4 Bytes	    |
+ *				+---------------------------+
+ *
+ */
+#define TOE_QUEUE_HDR_ADDR(n)	(TOE_TOE_QUE_HDR_BASE + n * 32)
+#define TOE_Q_HDR_AREA_END	(TOE_QUEUE_HDR_ADDR(TOE_TOE_QUEUE_MAX + 1))
+#define TOE_DEFAULT_Q_HDR_BASE(x) (TOE_NONTOE_QUE_HDR_BASE + 0x08 * (x))
+#define TOE_CLASS_Q_HDR_BASE	(TOE_NONTOE_QUE_HDR_BASE + 0x10)
+#define TOE_INTR_Q_HDR_BASE	(TOE_NONTOE_QUE_HDR_BASE + 0x80)
+#define INTERRUPT_QUEUE_HDR_ADDR(n) (TOE_INTR_Q_HDR_BASE + n * 8)
+#define NONTOE_Q_HDR_AREA_END (INTERRUPT_QUEUE_HDR_ADDR(TOE_INTR_QUEUE_MAX + 1))
+
+/* NONTOE Queue Header Word 0 */
+union nontoe_qhdr0 {
+	unsigned int bits32;
+	unsigned int base_size;
+};
+
+#define NONTOE_QHDR0_BASE_MASK	(~0x0f)
+
+/* NONTOE Queue Header Word 1 */
+union nontoe_qhdr1 {
+	unsigned int bits32;
+	struct bit_nonqhdr1 {
+		/* bit 15:0 */
+		unsigned int rptr:16;
+		/* bit 31:16 */
+		unsigned int wptr:16;
+	} bits;
+};
+
+/* Non-TOE Queue Header */
+struct nontoe_qhdr {
+	union nontoe_qhdr0 word0;
+	union nontoe_qhdr1 word1;
+};
+
+#endif /* _GEMINI_ETHERNET_H */
diff --git a/drivers/net/ethernet/emulex/benet/be.h b/drivers/net/ethernet/emulex/benet/be.h
index 8984c4938881..382891f81e09 100644
--- a/drivers/net/ethernet/emulex/benet/be.h
+++ b/drivers/net/ethernet/emulex/benet/be.h
@@ -248,6 +248,7 @@ struct be_tx_stats {
 	u32 tx_spoof_check_err;
 	u32 tx_qinq_err;
 	u32 tx_internal_parity_err;
+	u32 tx_sge_err;
 	struct u64_stats_sync sync;
 	struct u64_stats_sync sync_compl;
 };
@@ -944,8 +945,10 @@ static inline bool is_ipv6_ext_hdr(struct sk_buff *skb)
 #define BE_ERROR_EEH		1
 #define BE_ERROR_UE		BIT(1)
 #define BE_ERROR_FW		BIT(2)
-#define BE_ERROR_HW		(BE_ERROR_EEH | BE_ERROR_UE)
-#define BE_ERROR_ANY		(BE_ERROR_EEH | BE_ERROR_UE | BE_ERROR_FW)
+#define BE_ERROR_TX		BIT(3)
+#define BE_ERROR_HW		(BE_ERROR_EEH | BE_ERROR_UE | BE_ERROR_TX)
+#define BE_ERROR_ANY		(BE_ERROR_EEH | BE_ERROR_UE | BE_ERROR_FW | \
+				 BE_ERROR_TX)
 #define BE_CLEAR_ALL		0xFF
 
 static inline u8 be_check_error(struct be_adapter *adapter, u32 err_type)
diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c
index 02dd5246dfae..1a49297224ed 100644
--- a/drivers/net/ethernet/emulex/benet/be_cmds.c
+++ b/drivers/net/ethernet/emulex/benet/be_cmds.c
@@ -103,7 +103,7 @@ static struct be_cmd_priv_map cmd_priv_map[] = {
 static bool be_cmd_allowed(struct be_adapter *adapter, u8 opcode, u8 subsystem)
 {
 	int i;
-	int num_entries = sizeof(cmd_priv_map)/sizeof(struct be_cmd_priv_map);
+	int num_entries = ARRAY_SIZE(cmd_priv_map);
 	u32 cmd_privileges = adapter->cmd_privileges;
 
 	for (i = 0; i < num_entries; i++)
diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c
index 7d1819c9e8cc..7f7e206f95f8 100644
--- a/drivers/net/ethernet/emulex/benet/be_ethtool.c
+++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c
@@ -189,6 +189,7 @@ static const struct be_ethtool_stat et_tx_stats[] = {
 	 * packet data. This counter is applicable only for Lancer adapters.
 	 */
 	{DRVSTAT_TX_INFO(tx_internal_parity_err)},
+	{DRVSTAT_TX_INFO(tx_sge_err)},
 	{DRVSTAT_TX_INFO(tx_bytes)},
 	{DRVSTAT_TX_INFO(tx_pkts)},
 	{DRVSTAT_TX_INFO(tx_vxlan_offload_pkts)},
diff --git a/drivers/net/ethernet/emulex/benet/be_hw.h b/drivers/net/ethernet/emulex/benet/be_hw.h
index c967f45705d9..db5f92fb87e0 100644
--- a/drivers/net/ethernet/emulex/benet/be_hw.h
+++ b/drivers/net/ethernet/emulex/benet/be_hw.h
@@ -261,6 +261,7 @@ struct be_eth_hdr_wrb {
 #define LANCER_TX_COMP_HSW_DROP_MAC_ERR		0x3
 #define LANCER_TX_COMP_HSW_DROP_VLAN_ERR	0x5
 #define LANCER_TX_COMP_QINQ_ERR			0x7
+#define LANCER_TX_COMP_SGE_ERR			0x9
 #define LANCER_TX_COMP_PARITY_ERR		0xb
 #define LANCER_TX_COMP_DMA_ERR			0xd
 
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index e180657a02ef..5774fb6f8aa0 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -991,9 +991,8 @@ static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 {
 	u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 	struct device *dev = &adapter->pdev->dev;
-	struct be_queue_info *txq = &txo->q;
 	bool map_single = false;
-	u32 head = txq->head;
+	u32 head;
 	dma_addr_t busaddr;
 	int len;
 
@@ -2584,7 +2583,48 @@ static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
 	}
 }
 
-static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo)
+static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)
+{
+	switch (status) {
+	case BE_TX_COMP_HDR_PARSE_ERR:
+		tx_stats(txo)->tx_hdr_parse_err++;
+		break;
+	case BE_TX_COMP_NDMA_ERR:
+		tx_stats(txo)->tx_dma_err++;
+		break;
+	case BE_TX_COMP_ACL_ERR:
+		tx_stats(txo)->tx_spoof_check_err++;
+		break;
+	}
+}
+
+static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)
+{
+	switch (status) {
+	case LANCER_TX_COMP_LSO_ERR:
+		tx_stats(txo)->tx_tso_err++;
+		break;
+	case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
+	case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
+		tx_stats(txo)->tx_spoof_check_err++;
+		break;
+	case LANCER_TX_COMP_QINQ_ERR:
+		tx_stats(txo)->tx_qinq_err++;
+		break;
+	case LANCER_TX_COMP_PARITY_ERR:
+		tx_stats(txo)->tx_internal_parity_err++;
+		break;
+	case LANCER_TX_COMP_DMA_ERR:
+		tx_stats(txo)->tx_dma_err++;
+		break;
+	case LANCER_TX_COMP_SGE_ERR:
+		tx_stats(txo)->tx_sge_err++;
+		break;
+	}
+}
+
+static struct be_tx_compl_info *be_tx_compl_get(struct be_adapter *adapter,
+						struct be_tx_obj *txo)
 {
 	struct be_queue_info *tx_cq = &txo->cq;
 	struct be_tx_compl_info *txcp = &txo->txcp;
@@ -2600,6 +2640,24 @@ static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo)
 	txcp->status = GET_TX_COMPL_BITS(status, compl);
 	txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl);
 
+	if (txcp->status) {
+		if (lancer_chip(adapter)) {
+			lancer_update_tx_err(txo, txcp->status);
+			/* Reset the adapter incase of TSO,
+			 * SGE or Parity error
+			 */
+			if (txcp->status == LANCER_TX_COMP_LSO_ERR ||
+			    txcp->status == LANCER_TX_COMP_PARITY_ERR ||
+			    txcp->status == LANCER_TX_COMP_SGE_ERR)
+				be_set_error(adapter, BE_ERROR_TX);
+		} else {
+			be_update_tx_err(txo, txcp->status);
+		}
+	}
+
+	if (be_check_error(adapter, BE_ERROR_TX))
+		return NULL;
+
 	compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
 	queue_tail_inc(tx_cq);
 	return txcp;
@@ -2742,7 +2800,7 @@ static void be_tx_compl_clean(struct be_adapter *adapter)
 			cmpl = 0;
 			num_wrbs = 0;
 			txq = &txo->q;
-			while ((txcp = be_tx_compl_get(txo))) {
+			while ((txcp = be_tx_compl_get(adapter, txo))) {
 				num_wrbs +=
 					be_tx_compl_process(adapter, txo,
 							    txcp->end_index);
@@ -3121,42 +3179,6 @@ static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
 	return work_done;
 }
 
-static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)
-{
-	switch (status) {
-	case BE_TX_COMP_HDR_PARSE_ERR:
-		tx_stats(txo)->tx_hdr_parse_err++;
-		break;
-	case BE_TX_COMP_NDMA_ERR:
-		tx_stats(txo)->tx_dma_err++;
-		break;
-	case BE_TX_COMP_ACL_ERR:
-		tx_stats(txo)->tx_spoof_check_err++;
-		break;
-	}
-}
-
-static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)
-{
-	switch (status) {
-	case LANCER_TX_COMP_LSO_ERR:
-		tx_stats(txo)->tx_tso_err++;
-		break;
-	case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
-	case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
-		tx_stats(txo)->tx_spoof_check_err++;
-		break;
-	case LANCER_TX_COMP_QINQ_ERR:
-		tx_stats(txo)->tx_qinq_err++;
-		break;
-	case LANCER_TX_COMP_PARITY_ERR:
-		tx_stats(txo)->tx_internal_parity_err++;
-		break;
-	case LANCER_TX_COMP_DMA_ERR:
-		tx_stats(txo)->tx_dma_err++;
-		break;
-	}
-}
 
 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
 			  int idx)
@@ -3164,16 +3186,9 @@ static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
 	int num_wrbs = 0, work_done = 0;
 	struct be_tx_compl_info *txcp;
 
-	while ((txcp = be_tx_compl_get(txo))) {
+	while ((txcp = be_tx_compl_get(adapter, txo))) {
 		num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);
 		work_done++;
-
-		if (txcp->status) {
-			if (lancer_chip(adapter))
-				lancer_update_tx_err(txo, txcp->status);
-			else
-				be_update_tx_err(txo, txcp->status);
-		}
 	}
 
 	if (work_done) {
@@ -4634,8 +4649,7 @@ int be_update_queues(struct be_adapter *adapter)
 
 	be_schedule_worker(adapter);
 
-	/*
-	 * The IF was destroyed and re-created. We need to clear
+	/* The IF was destroyed and re-created. We need to clear
 	 * all promiscuous flags valid for the destroyed IF.
 	 * Without this promisc mode is not restored during
 	 * be_open() because the driver thinks that it is
@@ -5106,9 +5120,12 @@ static netdev_features_t be_features_check(struct sk_buff *skb,
 			features &= ~NETIF_F_TSO6;
 
 		/* Lancer cannot handle the packet with MSS less than 256.
+		 * Also it can't handle a TSO packet with a single segment
 		 * Disable the GSO support in such cases
 		 */
-		if (lancer_chip(adapter) && skb_shinfo(skb)->gso_size < 256)
+		if (lancer_chip(adapter) &&
+		    (skb_shinfo(skb)->gso_size < 256 ||
+		     skb_shinfo(skb)->gso_segs == 1))
 			features &= ~NETIF_F_GSO_MASK;
 	}
 
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
index 7caa8da48421..e4ec32a9ca15 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
@@ -2008,7 +2008,6 @@ static inline int dpaa_xmit(struct dpaa_priv *priv,
 	}
 
 	if (unlikely(err < 0)) {
-		percpu_stats->tx_errors++;
 		percpu_stats->tx_fifo_errors++;
 		return err;
 	}
@@ -2278,7 +2277,6 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
 	vaddr = phys_to_virt(addr);
 	prefetch(vaddr + qm_fd_get_offset(fd));
 
-	fd_format = qm_fd_get_format(fd);
 	/* The only FD types that we may receive are contig and S/G */
 	WARN_ON((fd_format != qm_fd_contig) && (fd_format != qm_fd_sg));
 
@@ -2311,8 +2309,10 @@ static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
 
 	skb_len = skb->len;
 
-	if (unlikely(netif_receive_skb(skb) == NET_RX_DROP))
+	if (unlikely(netif_receive_skb(skb) == NET_RX_DROP)) {
+		percpu_stats->rx_dropped++;
 		return qman_cb_dqrr_consume;
+	}
 
 	percpu_stats->rx_packets++;
 	percpu_stats->rx_bytes += skb_len;
@@ -2860,7 +2860,7 @@ static int dpaa_remove(struct platform_device *pdev)
 	struct device *dev;
 	int err;
 
-	dev = &pdev->dev;
+	dev = pdev->dev.parent;
 	net_dev = dev_get_drvdata(dev);
 
 	priv = netdev_priv(net_dev);
diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h
index 5385074b3b7d..e7381f8ef89d 100644
--- a/drivers/net/ethernet/freescale/fec.h
+++ b/drivers/net/ethernet/freescale/fec.h
@@ -20,7 +20,8 @@
 #include <linux/timecounter.h>
 
 #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
-    defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM)
+    defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \
+    defined(CONFIG_ARM64)
 /*
  *	Just figures, Motorola would have to change the offsets for
  *	registers in the same peripheral device on different models
@@ -195,7 +196,7 @@
  *	Evidently, ARM SoCs have the FEC block generated in a
  *	little endian mode so adjust endianness accordingly.
  */
-#if defined(CONFIG_ARM)
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
 #define fec32_to_cpu le32_to_cpu
 #define fec16_to_cpu le16_to_cpu
 #define cpu_to_fec32 cpu_to_le32
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index a74300a4459c..d4604bc8eb5b 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -195,7 +195,8 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address");
  * account when setting it.
  */
 #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
-    defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM)
+    defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \
+    defined(CONFIG_ARM64)
 #define	OPT_FRAME_SIZE	(PKT_MAXBUF_SIZE << 16)
 #else
 #define	OPT_FRAME_SIZE	0
@@ -1868,6 +1869,8 @@ static int fec_enet_clk_enable(struct net_device *ndev, bool enable)
 		ret = clk_prepare_enable(fep->clk_ref);
 		if (ret)
 			goto failed_clk_ref;
+
+		phy_reset_after_clk_enable(ndev->phydev);
 	} else {
 		clk_disable_unprepare(fep->clk_ahb);
 		clk_disable_unprepare(fep->clk_enet_out);
@@ -2107,7 +2110,8 @@ static int fec_enet_get_regs_len(struct net_device *ndev)
 
 /* List of registers that can be safety be read to dump them with ethtool */
 #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
-	defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM)
+	defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \
+	defined(CONFIG_ARM64)
 static u32 fec_enet_register_offset[] = {
 	FEC_IEVENT, FEC_IMASK, FEC_R_DES_ACTIVE_0, FEC_X_DES_ACTIVE_0,
 	FEC_ECNTRL, FEC_MII_DATA, FEC_MII_SPEED, FEC_MIB_CTRLSTAT, FEC_R_CNTRL,
@@ -2840,6 +2844,7 @@ fec_enet_open(struct net_device *ndev)
 {
 	struct fec_enet_private *fep = netdev_priv(ndev);
 	int ret;
+	bool reset_again;
 
 	ret = pm_runtime_get_sync(&fep->pdev->dev);
 	if (ret < 0)
@@ -2850,6 +2855,17 @@ fec_enet_open(struct net_device *ndev)
 	if (ret)
 		goto clk_enable;
 
+	/* During the first fec_enet_open call the PHY isn't probed at this
+	 * point. Therefore the phy_reset_after_clk_enable() call within
+	 * fec_enet_clk_enable() fails. As we need this reset in order to be
+	 * sure the PHY is working correctly we check if we need to reset again
+	 * later when the PHY is probed
+	 */
+	if (ndev->phydev && ndev->phydev->drv)
+		reset_again = false;
+	else
+		reset_again = true;
+
 	/* I should reset the ring buffers here, but I don't yet know
 	 * a simple way to do that.
 	 */
@@ -2866,6 +2882,12 @@ fec_enet_open(struct net_device *ndev)
 	if (ret)
 		goto err_enet_mii_probe;
 
+	/* Call phy_reset_after_clk_enable() again if it failed during
+	 * phy_reset_after_clk_enable() before because the PHY wasn't probed.
+	 */
+	if (reset_again)
+		phy_reset_after_clk_enable(ndev->phydev);
+
 	if (fep->quirks & FEC_QUIRK_ERR006687)
 		imx6q_cpuidle_fec_irqs_used();
 
@@ -3119,7 +3141,7 @@ static int fec_enet_init(struct net_device *ndev)
 	unsigned dsize_log2 = __fls(dsize);
 
 	WARN_ON(dsize != (1 << dsize_log2));
-#if defined(CONFIG_ARM)
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
 	fep->rx_align = 0xf;
 	fep->tx_align = 0xf;
 #else
@@ -3578,6 +3600,8 @@ fec_drv_remove(struct platform_device *pdev)
 	fec_enet_mii_remove(fep);
 	if (fep->reg_phy)
 		regulator_disable(fep->reg_phy);
+	pm_runtime_put(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
 	if (of_phy_is_fixed_link(np))
 		of_phy_deregister_fixed_link(np);
 	of_node_put(fep->phy_node);
diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.c b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
index ea43b4974149..7af31ddd093f 100644
--- a/drivers/net/ethernet/freescale/fman/fman_dtsec.c
+++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
@@ -1100,7 +1100,7 @@ int dtsec_add_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr)
 	set_bucket(dtsec->regs, bucket, true);
 
 	/* Create element to be added to the driver hash table */
-	hash_entry = kmalloc(sizeof(*hash_entry), GFP_KERNEL);
+	hash_entry = kmalloc(sizeof(*hash_entry), GFP_ATOMIC);
 	if (!hash_entry)
 		return -ENOMEM;
 	hash_entry->addr = addr;
diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c
index 7f837006bb6a..f27f9bae1a4a 100644
--- a/drivers/net/ethernet/freescale/gianfar.c
+++ b/drivers/net/ethernet/freescale/gianfar.c
@@ -2932,26 +2932,19 @@ static irqreturn_t gfar_transmit(int irq, void *grp_id)
 static bool gfar_add_rx_frag(struct gfar_rx_buff *rxb, u32 lstatus,
 			     struct sk_buff *skb, bool first)
 {
-	unsigned int size = lstatus & BD_LENGTH_MASK;
+	int size = lstatus & BD_LENGTH_MASK;
 	struct page *page = rxb->page;
-	bool last = !!(lstatus & BD_LFLAG(RXBD_LAST));
-
-	/* Remove the FCS from the packet length */
-	if (last)
-		size -= ETH_FCS_LEN;
 
 	if (likely(first)) {
 		skb_put(skb, size);
 	} else {
 		/* the last fragments' length contains the full frame length */
-		if (last)
+		if (lstatus & BD_LFLAG(RXBD_LAST))
 			size -= skb->len;
 
-		/* in case the last fragment consisted only of the FCS */
-		if (size > 0)
-			skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
-					rxb->page_offset + RXBUF_ALIGNMENT,
-					size, GFAR_RXB_TRUESIZE);
+		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
+				rxb->page_offset + RXBUF_ALIGNMENT,
+				size, GFAR_RXB_TRUESIZE);
 	}
 
 	/* try reuse page */
@@ -3064,12 +3057,12 @@ static void gfar_process_frame(struct net_device *ndev, struct sk_buff *skb)
 	if (priv->padding)
 		skb_pull(skb, priv->padding);
 
+	/* Trim off the FCS */
+	pskb_trim(skb, skb->len - ETH_FCS_LEN);
+
 	if (ndev->features & NETIF_F_RXCSUM)
 		gfar_rx_checksum(skb, fcb);
 
-	/* Tell the skb what kind of packet this is */
-	skb->protocol = eth_type_trans(skb, ndev);
-
 	/* There's need to check for NETIF_F_HW_VLAN_CTAG_RX here.
 	 * Even if vlan rx accel is disabled, on some chips
 	 * RXFCB_VLN is pseudo randomly set.
@@ -3140,13 +3133,15 @@ int gfar_clean_rx_ring(struct gfar_priv_rx_q *rx_queue, int rx_work_limit)
 			continue;
 		}
 
+		gfar_process_frame(ndev, skb);
+
 		/* Increment the number of packets */
 		total_pkts++;
 		total_bytes += skb->len;
 
 		skb_record_rx_queue(skb, rx_queue->qindex);
 
-		gfar_process_frame(ndev, skb);
+		skb->protocol = eth_type_trans(skb, ndev);
 
 		/* Send the packet up the stack */
 		napi_gro_receive(&rx_queue->grp->napi_rx, skb);
diff --git a/drivers/net/ethernet/hisilicon/Kconfig b/drivers/net/ethernet/hisilicon/Kconfig
index 30000b6aa7b8..8bcf470ff5f3 100644
--- a/drivers/net/ethernet/hisilicon/Kconfig
+++ b/drivers/net/ethernet/hisilicon/Kconfig
@@ -94,15 +94,6 @@ config HNS3_HCLGE
 	  compatibility layer. The engine would be used in Hisilicon hip08 family of
 	  SoCs and further upcoming SoCs.
 
-config HNS3_ENET
-	tristate "Hisilicon HNS3 Ethernet Device Support"
-	depends on 64BIT && PCI
-	depends on HNS3 && HNS3_HCLGE
-	---help---
-	  This selects the Ethernet Driver for Hisilicon Network Subsystem 3 for hip08
-	  family of SoCs. This module depends upon HNAE3 driver to access the HNAE3
-	  devices and their associated operations.
-
 config HNS3_DCB
 	bool "Hisilicon HNS3 Data Center Bridge Support"
 	default n
@@ -112,4 +103,23 @@ config HNS3_DCB
 
 	  If unsure, say N.
 
+config HNS3_HCLGEVF
+    tristate "Hisilicon HNS3VF Acceleration Engine & Compatibility Layer Support"
+    depends on PCI_MSI
+    depends on HNS3
+	depends on HNS3_HCLGE
+    ---help---
+	  This selects the HNS3 VF drivers network acceleration engine & its hardware
+	  compatibility layer. The engine would be used in Hisilicon hip08 family of
+	  SoCs and further upcoming SoCs.
+
+config HNS3_ENET
+	tristate "Hisilicon HNS3 Ethernet Device Support"
+	depends on 64BIT && PCI
+	depends on HNS3
+	---help---
+	  This selects the Ethernet Driver for Hisilicon Network Subsystem 3 for hip08
+	  family of SoCs. This module depends upon HNAE3 driver to access the HNAE3
+	  devices and their associated operations.
+
 endif # NET_VENDOR_HISILICON
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
index 86944bc3b273..74bd260ca02a 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_gmac.c
@@ -666,7 +666,7 @@ static void hns_gmac_get_strings(u32 stringset, u8 *data)
 
 static int hns_gmac_get_sset_count(int stringset)
 {
-	if (stringset == ETH_SS_STATS || stringset == ETH_SS_PRIV_FLAGS)
+	if (stringset == ETH_SS_STATS)
 		return ARRAY_SIZE(g_gmac_stats_string);
 
 	return 0;
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
index 8b5cdf490850..cac86e9ae0dd 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_mac.c
@@ -1168,7 +1168,7 @@ void hns_set_led_opt(struct hns_mac_cb *mac_cb)
 int hns_cpld_led_set_id(struct hns_mac_cb *mac_cb,
 			enum hnae_led_state status)
 {
-	if (!mac_cb || !mac_cb->cpld_ctrl)
+	if (!mac_cb)
 		return 0;
 
 	return mac_cb->dsaf_dev->misc_op->cpld_set_led_id(mac_cb, status);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
index 408b63faf9a8..acf29633ec79 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_misc.c
@@ -18,6 +18,7 @@ enum _dsm_op_index {
 	HNS_OP_LED_SET_FUNC             = 0x3,
 	HNS_OP_GET_PORT_TYPE_FUNC       = 0x4,
 	HNS_OP_GET_SFP_STAT_FUNC        = 0x5,
+	HNS_OP_LOCATE_LED_SET_FUNC      = 0x6,
 };
 
 enum _dsm_rst_type {
@@ -43,12 +44,17 @@ static void dsaf_write_sub(struct dsaf_device *dsaf_dev, u32 reg, u32 val)
 
 static u32 dsaf_read_sub(struct dsaf_device *dsaf_dev, u32 reg)
 {
-	u32 ret;
-
-	if (dsaf_dev->sub_ctrl)
-		ret = dsaf_read_syscon(dsaf_dev->sub_ctrl, reg);
-	else
+	u32 ret = 0;
+	int err;
+
+	if (dsaf_dev->sub_ctrl) {
+		err = dsaf_read_syscon(dsaf_dev->sub_ctrl, reg, &ret);
+		if (err)
+			dev_err(dsaf_dev->dev, "dsaf_read_syscon error %d!\n",
+				err);
+	} else {
 		ret = dsaf_read_reg(dsaf_dev->sc_base, reg);
+	}
 
 	return ret;
 }
@@ -81,6 +87,33 @@ static void hns_dsaf_acpi_ledctrl_by_port(struct hns_mac_cb *mac_cb, u8 op_type,
        ACPI_FREE(obj);
 }
 
+static void hns_dsaf_acpi_locate_ledctrl_by_port(struct hns_mac_cb *mac_cb,
+						 u8 op_type, u32 locate,
+						 u32 port)
+{
+	union acpi_object obj_args[2], argv4;
+	union acpi_object *obj;
+
+	obj_args[0].integer.type = ACPI_TYPE_INTEGER;
+	obj_args[0].integer.value = locate;
+	obj_args[1].integer.type = ACPI_TYPE_INTEGER;
+	obj_args[1].integer.value = port;
+
+	argv4.type = ACPI_TYPE_PACKAGE;
+	argv4.package.count = 2;
+	argv4.package.elements = obj_args;
+
+	obj = acpi_evaluate_dsm(ACPI_HANDLE(mac_cb->dev),
+				&hns_dsaf_acpi_dsm_guid, 0, op_type, &argv4);
+	if (!obj) {
+		dev_err(mac_cb->dev, "ledctrl fail, locate:%d port:%d!\n",
+			locate, port);
+		return;
+	}
+
+	ACPI_FREE(obj);
+}
+
 static void hns_cpld_set_led(struct hns_mac_cb *mac_cb, int link_status,
 			     u16 speed, int data)
 {
@@ -160,15 +193,23 @@ static void cpld_led_reset_acpi(struct hns_mac_cb *mac_cb)
 static int cpld_set_led_id(struct hns_mac_cb *mac_cb,
 			   enum hnae_led_state status)
 {
+	u32 val = 0;
+	int ret;
+
+	if (!mac_cb->cpld_ctrl)
+		return 0;
+
 	switch (status) {
 	case HNAE_LED_ACTIVE:
-		mac_cb->cpld_led_value =
-			dsaf_read_syscon(mac_cb->cpld_ctrl,
-					 mac_cb->cpld_ctrl_reg);
-		dsaf_set_bit(mac_cb->cpld_led_value, DSAF_LED_ANCHOR_B,
-			     CPLD_LED_ON_VALUE);
+		ret = dsaf_read_syscon(mac_cb->cpld_ctrl, mac_cb->cpld_ctrl_reg,
+				       &val);
+		if (ret)
+			return ret;
+
+		dsaf_set_bit(val, DSAF_LED_ANCHOR_B, CPLD_LED_ON_VALUE);
 		dsaf_write_syscon(mac_cb->cpld_ctrl, mac_cb->cpld_ctrl_reg,
-				  mac_cb->cpld_led_value);
+				  val);
+		mac_cb->cpld_led_value = val;
 		break;
 	case HNAE_LED_INACTIVE:
 		dsaf_set_bit(mac_cb->cpld_led_value, DSAF_LED_ANCHOR_B,
@@ -184,6 +225,30 @@ static int cpld_set_led_id(struct hns_mac_cb *mac_cb,
 	return 0;
 }
 
+static int cpld_set_led_id_acpi(struct hns_mac_cb *mac_cb,
+				enum hnae_led_state status)
+{
+	switch (status) {
+	case HNAE_LED_ACTIVE:
+		hns_dsaf_acpi_locate_ledctrl_by_port(mac_cb,
+						     HNS_OP_LOCATE_LED_SET_FUNC,
+						     CPLD_LED_ON_VALUE,
+						     mac_cb->mac_id);
+		break;
+	case HNAE_LED_INACTIVE:
+		hns_dsaf_acpi_locate_ledctrl_by_port(mac_cb,
+						     HNS_OP_LOCATE_LED_SET_FUNC,
+						     CPLD_LED_DEFAULT_VALUE,
+						     mac_cb->mac_id);
+		break;
+	default:
+		dev_err(mac_cb->dev, "invalid led state: %d!", status);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 #define RESET_REQ_OR_DREQ 1
 
 static void hns_dsaf_acpi_srst_by_port(struct dsaf_device *dsaf_dev, u8 op_type,
@@ -505,12 +570,19 @@ static phy_interface_t hns_mac_get_phy_if_acpi(struct hns_mac_cb *mac_cb)
 
 int hns_mac_get_sfp_prsnt(struct hns_mac_cb *mac_cb, int *sfp_prsnt)
 {
+	u32 val = 0;
+	int ret;
+
 	if (!mac_cb->cpld_ctrl)
 		return -ENODEV;
 
-	*sfp_prsnt = !dsaf_read_syscon(mac_cb->cpld_ctrl, mac_cb->cpld_ctrl_reg
-					+ MAC_SFP_PORT_OFFSET);
+	ret = dsaf_read_syscon(mac_cb->cpld_ctrl,
+			       mac_cb->cpld_ctrl_reg + MAC_SFP_PORT_OFFSET,
+			       &val);
+	if (ret)
+		return ret;
 
+	*sfp_prsnt = !val;
 	return 0;
 }
 
@@ -560,7 +632,7 @@ static int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, bool en)
 #define RX_CSR(lane, reg) ((0x4080 + (reg) * 0x0002 + (lane) * 0x0200) * 2)
 	u64 reg_offset = RX_CSR(lane_id[mac_cb->mac_id], 0);
 
-	int sfp_prsnt;
+	int sfp_prsnt = 0;
 	int ret = hns_mac_get_sfp_prsnt(mac_cb, &sfp_prsnt);
 
 	if (!mac_cb->phy_dev) {
@@ -572,7 +644,7 @@ static int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, bool en)
 	}
 
 	if (mac_cb->serdes_ctrl) {
-		u32 origin;
+		u32 origin = 0;
 
 		if (!AE_IS_VER1(mac_cb->dsaf_dev->dsaf_ver)) {
 #define HILINK_ACCESS_SEL_CFG		0x40008
@@ -589,7 +661,10 @@ static int hns_mac_config_sds_loopback(struct hns_mac_cb *mac_cb, bool en)
 						  HILINK_ACCESS_SEL_CFG, 3);
 		}
 
-		origin = dsaf_read_syscon(mac_cb->serdes_ctrl, reg_offset);
+		ret = dsaf_read_syscon(mac_cb->serdes_ctrl, reg_offset,
+				       &origin);
+		if (ret)
+			return ret;
 
 		dsaf_set_field(origin, 1ull << 10, 10, en);
 		dsaf_write_syscon(mac_cb->serdes_ctrl, reg_offset, origin);
@@ -660,7 +735,7 @@ struct dsaf_misc_op *hns_misc_op_get(struct dsaf_device *dsaf_dev)
 	} else if (is_acpi_node(dsaf_dev->dev->fwnode)) {
 		misc_op->cpld_set_led = hns_cpld_set_led_acpi;
 		misc_op->cpld_reset_led = cpld_led_reset_acpi;
-		misc_op->cpld_set_led_id = cpld_set_led_id;
+		misc_op->cpld_set_led_id = cpld_set_led_id_acpi;
 
 		misc_op->dsaf_reset = hns_dsaf_rst_acpi;
 		misc_op->xge_srst = hns_dsaf_xge_srst_by_port_acpi;
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
index b62816c1574e..93e71e27401b 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_ppe.c
@@ -422,7 +422,7 @@ void hns_ppe_update_stats(struct hns_ppe_cb *ppe_cb)
 
 int hns_ppe_get_sset_count(int stringset)
 {
-	if (stringset == ETH_SS_STATS || stringset == ETH_SS_PRIV_FLAGS)
+	if (stringset == ETH_SS_STATS)
 		return ETH_PPE_STATIC_NUM;
 	return 0;
 }
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
index 6f3570cfb501..e2e28532e4dc 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_rcb.c
@@ -876,7 +876,7 @@ void hns_rcb_get_stats(struct hnae_queue *queue, u64 *data)
  */
 int hns_rcb_get_ring_sset_count(int stringset)
 {
-	if (stringset == ETH_SS_STATS || stringset == ETH_SS_PRIV_FLAGS)
+	if (stringset == ETH_SS_STATS)
 		return HNS_RING_STATIC_REG_NUM;
 
 	return 0;
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
index 46a52d9bb196..886cbbf25761 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
+++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_reg.h
@@ -1034,12 +1034,9 @@ static inline void dsaf_write_syscon(struct regmap *base, u32 reg, u32 value)
 	regmap_write(base, reg, value);
 }
 
-static inline u32 dsaf_read_syscon(struct regmap *base, u32 reg)
+static inline int dsaf_read_syscon(struct regmap *base, u32 reg, u32 *val)
 {
-	unsigned int val;
-
-	regmap_read(base, reg, &val);
-	return val;
+	return regmap_read(base, reg, val);
 }
 
 #define dsaf_read_dev(a, reg) \
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
index 7ea7f8a4aa2a..2e14a3ae1d8b 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c
@@ -993,8 +993,10 @@ int hns_get_sset_count(struct net_device *netdev, int stringset)
 			cnt--;
 
 		return cnt;
-	} else {
+	} else if (stringset == ETH_SS_STATS) {
 		return (HNS_NET_STATS_CNT + ops->get_sset_count(h, stringset));
+	} else {
+		return -EOPNOTSUPP;
 	}
 }
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/Makefile b/drivers/net/ethernet/hisilicon/hns3/Makefile
index a9349e1f3e51..002534f12b66 100644
--- a/drivers/net/ethernet/hisilicon/hns3/Makefile
+++ b/drivers/net/ethernet/hisilicon/hns3/Makefile
@@ -1,7 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0+
 #
 # Makefile for the HISILICON network device drivers.
 #
 
 obj-$(CONFIG_HNS3) += hns3pf/
+obj-$(CONFIG_HNS3) += hns3vf/
 
 obj-$(CONFIG_HNS3) += hnae3.o
+
+obj-$(CONFIG_HNS3_ENET) += hns3.o
+hns3-objs = hns3_enet.o hns3_ethtool.o
+
+hns3-$(CONFIG_HNS3_DCB) += hns3_dcbnl.o
diff --git a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
new file mode 100644
index 000000000000..3e9203ea42a6
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/* Copyright (c) 2016-2017 Hisilicon Limited. */
+
+#ifndef __HCLGE_MBX_H
+#define __HCLGE_MBX_H
+#include <linux/init.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+
+#define HCLGE_MBX_VF_MSG_DATA_NUM	16
+
+enum HCLGE_MBX_OPCODE {
+	HCLGE_MBX_RESET = 0x01,		/* (VF -> PF) assert reset */
+	HCLGE_MBX_SET_UNICAST,		/* (VF -> PF) set UC addr */
+	HCLGE_MBX_SET_MULTICAST,	/* (VF -> PF) set MC addr */
+	HCLGE_MBX_SET_VLAN,		/* (VF -> PF) set VLAN */
+	HCLGE_MBX_MAP_RING_TO_VECTOR,	/* (VF -> PF) map ring-to-vector */
+	HCLGE_MBX_UNMAP_RING_TO_VECTOR,	/* (VF -> PF) unamp ring-to-vector */
+	HCLGE_MBX_SET_PROMISC_MODE,	/* (VF -> PF) set promiscuous mode */
+	HCLGE_MBX_SET_MACVLAN,		/* (VF -> PF) set unicast filter */
+	HCLGE_MBX_API_NEGOTIATE,	/* (VF -> PF) negotiate API version */
+	HCLGE_MBX_GET_QINFO,		/* (VF -> PF) get queue config */
+	HCLGE_MBX_GET_TCINFO,		/* (VF -> PF) get TC config */
+	HCLGE_MBX_GET_RETA,		/* (VF -> PF) get RETA */
+	HCLGE_MBX_GET_RSS_KEY,		/* (VF -> PF) get RSS key */
+	HCLGE_MBX_GET_MAC_ADDR,		/* (VF -> PF) get MAC addr */
+	HCLGE_MBX_PF_VF_RESP,		/* (PF -> VF) generate respone to VF */
+	HCLGE_MBX_GET_BDNUM,		/* (VF -> PF) get BD num */
+	HCLGE_MBX_GET_BUFSIZE,		/* (VF -> PF) get buffer size */
+	HCLGE_MBX_GET_STREAMID,		/* (VF -> PF) get stream id */
+	HCLGE_MBX_SET_AESTART,		/* (VF -> PF) start ae */
+	HCLGE_MBX_SET_TSOSTATS,		/* (VF -> PF) get tso stats */
+	HCLGE_MBX_LINK_STAT_CHANGE,	/* (PF -> VF) link status has changed */
+	HCLGE_MBX_GET_BASE_CONFIG,	/* (VF -> PF) get config */
+	HCLGE_MBX_BIND_FUNC_QUEUE,	/* (VF -> PF) bind function and queue */
+	HCLGE_MBX_GET_LINK_STATUS,	/* (VF -> PF) get link status */
+	HCLGE_MBX_QUEUE_RESET,		/* (VF -> PF) reset queue */
+};
+
+/* below are per-VF mac-vlan subcodes */
+enum hclge_mbx_mac_vlan_subcode {
+	HCLGE_MBX_MAC_VLAN_UC_MODIFY = 0,	/* modify UC mac addr */
+	HCLGE_MBX_MAC_VLAN_UC_ADD,		/* add a new UC mac addr */
+	HCLGE_MBX_MAC_VLAN_UC_REMOVE,		/* remove a new UC mac addr */
+	HCLGE_MBX_MAC_VLAN_MC_MODIFY,		/* modify MC mac addr */
+	HCLGE_MBX_MAC_VLAN_MC_ADD,		/* add new MC mac addr */
+	HCLGE_MBX_MAC_VLAN_MC_REMOVE,		/* remove MC mac addr */
+	HCLGE_MBX_MAC_VLAN_MC_FUNC_MTA_ENABLE,	/* config func MTA enable */
+};
+
+/* below are per-VF vlan cfg subcodes */
+enum hclge_mbx_vlan_cfg_subcode {
+	HCLGE_MBX_VLAN_FILTER = 0,	/* set vlan filter */
+	HCLGE_MBX_VLAN_TX_OFF_CFG,	/* set tx side vlan offload */
+	HCLGE_MBX_VLAN_RX_OFF_CFG,	/* set rx side vlan offload */
+};
+
+#define HCLGE_MBX_MAX_MSG_SIZE	16
+#define HCLGE_MBX_MAX_RESP_DATA_SIZE	8
+
+struct hclgevf_mbx_resp_status {
+	struct mutex mbx_mutex; /* protects against contending sync cmd resp */
+	u32 origin_mbx_msg;
+	bool received_resp;
+	int resp_status;
+	u8 additional_info[HCLGE_MBX_MAX_RESP_DATA_SIZE];
+};
+
+struct hclge_mbx_vf_to_pf_cmd {
+	u8 rsv;
+	u8 mbx_src_vfid; /* Auto filled by IMP */
+	u8 rsv1[2];
+	u8 msg_len;
+	u8 rsv2[3];
+	u8 msg[HCLGE_MBX_MAX_MSG_SIZE];
+};
+
+struct hclge_mbx_pf_to_vf_cmd {
+	u8 dest_vfid;
+	u8 rsv[3];
+	u8 msg_len;
+	u8 rsv1[3];
+	u16 msg[8];
+};
+
+#define hclge_mbx_ring_ptr_move_crq(crq) \
+	(crq->next_to_use = (crq->next_to_use + 1) % crq->desc_num)
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.c b/drivers/net/ethernet/hisilicon/hns3/hnae3.c
index 5bcb2238acb2..02145f2de820 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.c
@@ -196,9 +196,18 @@ int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev)
 	const struct pci_device_id *id;
 	struct hnae3_ae_algo *ae_algo;
 	struct hnae3_client *client;
-	int ret = 0;
+	int ret = 0, lock_acquired;
+
+	/* we can get deadlocked if SRIOV is being enabled in context to probe
+	 * and probe gets called again in same context. This can happen when
+	 * pci_enable_sriov() is called to create VFs from PF probes context.
+	 * Therefore, for simplicity uniformly defering further probing in all
+	 * cases where we detect contention.
+	 */
+	lock_acquired = mutex_trylock(&hnae3_common_lock);
+	if (!lock_acquired)
+		return -EPROBE_DEFER;
 
-	mutex_lock(&hnae3_common_lock);
 	list_add_tail(&ae_dev->node, &hnae3_ae_dev_list);
 
 	/* Check if there are matched ae_algo */
@@ -211,6 +220,7 @@ int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev)
 
 		if (!ae_dev->ops) {
 			dev_err(&ae_dev->pdev->dev, "ae_dev ops are null\n");
+			ret = -EOPNOTSUPP;
 			goto out_err;
 		}
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
index 67c59e1039f2..fd06bc78c58e 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
@@ -133,11 +133,16 @@ struct hnae3_vector_info {
 #define HNAE3_RING_TYPE_B 0
 #define HNAE3_RING_TYPE_TX 0
 #define HNAE3_RING_TYPE_RX 1
+#define HNAE3_RING_GL_IDX_S 0
+#define HNAE3_RING_GL_IDX_M GENMASK(1, 0)
+#define HNAE3_RING_GL_RX 0
+#define HNAE3_RING_GL_TX 1
 
 struct hnae3_ring_chain_node {
 	struct hnae3_ring_chain_node *next;
 	u32 tqp_index;
 	u32 flag;
+	u32 int_gl_idx;
 };
 
 #define HNAE3_IS_TX_RING(node) \
@@ -274,10 +279,14 @@ struct hnae3_ae_dev {
  *   Get firmware version
  * get_mdix_mode()
  *   Get media typr of phy
+ * enable_vlan_filter()
+ *   Enable vlan filter
  * set_vlan_filter()
  *   Set vlan filter config of Ports
  * set_vf_vlan_filter()
  *   Set vlan filter config of vf
+ * enable_hw_strip_rxvtag()
+ *   Enable/disable hardware strip vlan tag of packets received
  */
 struct hnae3_ae_ops {
 	int (*init_ae_dev)(struct hnae3_ae_dev *ae_dev);
@@ -347,7 +356,8 @@ struct hnae3_ae_ops {
 			    u32 stringset, u8 *data);
 	int (*get_sset_count)(struct hnae3_handle *handle, int stringset);
 
-	void (*get_regs)(struct hnae3_handle *handle, void *data);
+	void (*get_regs)(struct hnae3_handle *handle, u32 *version,
+			 void *data);
 	int (*get_regs_len)(struct hnae3_handle *handle);
 
 	u32 (*get_rss_key_size)(struct hnae3_handle *handle);
@@ -380,12 +390,23 @@ struct hnae3_ae_ops {
 	void (*get_mdix_mode)(struct hnae3_handle *handle,
 			      u8 *tp_mdix_ctrl, u8 *tp_mdix);
 
+	void (*enable_vlan_filter)(struct hnae3_handle *handle, bool enable);
 	int (*set_vlan_filter)(struct hnae3_handle *handle, __be16 proto,
 			       u16 vlan_id, bool is_kill);
 	int (*set_vf_vlan_filter)(struct hnae3_handle *handle, int vfid,
 				  u16 vlan, u8 qos, __be16 proto);
+	int (*enable_hw_strip_rxvtag)(struct hnae3_handle *handle, bool enable);
 	void (*reset_event)(struct hnae3_handle *handle,
 			    enum hnae3_reset_type reset);
+	void (*get_channels)(struct hnae3_handle *handle,
+			     struct ethtool_channels *ch);
+	void (*get_tqps_and_rss_info)(struct hnae3_handle *h,
+				      u16 *free_tqps, u16 *max_rss_size);
+	int (*set_channels)(struct hnae3_handle *handle, u32 new_tqps_num);
+	void (*get_flowctrl_adv)(struct hnae3_handle *handle,
+				 u32 *flowctrl_adv);
+	int (*set_led_id)(struct hnae3_handle *handle,
+			  enum ethtool_phys_id_state status);
 };
 
 struct hnae3_dcb_ops {
@@ -435,6 +456,8 @@ struct hnae3_knic_private_info {
 	u16 num_tqps;		  /* total number of TQPs in this handle */
 	struct hnae3_queue **tqp;  /* array base of all TQPs in this instance */
 	const struct hnae3_dcb_ops *dcb_ops;
+
+	u16 int_rl_setting;
 };
 
 struct hnae3_roce_private_info {
@@ -452,9 +475,10 @@ struct hnae3_unic_private_info {
 	struct hnae3_queue **tqp;  /* array base of all TQPs of this instance */
 };
 
-#define HNAE3_SUPPORT_MAC_LOOPBACK    1
-#define HNAE3_SUPPORT_PHY_LOOPBACK    2
-#define HNAE3_SUPPORT_SERDES_LOOPBACK 4
+#define HNAE3_SUPPORT_MAC_LOOPBACK    BIT(0)
+#define HNAE3_SUPPORT_PHY_LOOPBACK    BIT(1)
+#define HNAE3_SUPPORT_SERDES_LOOPBACK BIT(2)
+#define HNAE3_SUPPORT_VF	      BIT(3)
 
 struct hnae3_handle {
 	struct hnae3_client *client;
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_dcbnl.c b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c
similarity index 97%
rename from drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_dcbnl.c
rename to drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c
index 925619a7c50a..eb82700da7d0 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_dcbnl.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c
@@ -93,7 +93,7 @@ void hns3_dcbnl_setup(struct hnae3_handle *handle)
 {
 	struct net_device *dev = handle->kinfo.netdev;
 
-	if (!handle->kinfo.dcb_ops)
+	if ((!handle->kinfo.dcb_ops) || (handle->flags & HNAE3_SUPPORT_VF))
 		return;
 
 	dev->dcbnl_ops = &hns3_dcbnl_ops;
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
similarity index 89%
rename from drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c
rename to drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
index 59415090ff0f..601b6295d3f8 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
@@ -52,6 +52,8 @@ static const struct pci_device_id hns3_pci_tbl[] = {
 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
 	/* required last entry */
 	{0, }
 };
@@ -156,43 +158,68 @@ static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
 	napi_disable(&tqp_vector->napi);
 }
 
-static void hns3_set_vector_coalesc_gl(struct hns3_enet_tqp_vector *tqp_vector,
-				       u32 gl_value)
+void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
+				 u32 rl_value)
 {
-	/* this defines the configuration for GL (Interrupt Gap Limiter)
-	 * GL defines inter interrupt gap.
-	 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
-	 */
-	writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
-	writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
-	writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL2_OFFSET);
-}
+	u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
 
-static void hns3_set_vector_coalesc_rl(struct hns3_enet_tqp_vector *tqp_vector,
-				       u32 rl_value)
-{
 	/* this defines the configuration for RL (Interrupt Rate Limiter).
 	 * Rl defines rate of interrupts i.e. number of interrupts-per-second
 	 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
 	 */
-	writel(rl_value, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
+
+	if (rl_reg > 0 && !tqp_vector->tx_group.gl_adapt_enable &&
+	    !tqp_vector->rx_group.gl_adapt_enable)
+		/* According to the hardware, the range of rl_reg is
+		 * 0-59 and the unit is 4.
+		 */
+		rl_reg |=  HNS3_INT_RL_ENABLE_MASK;
+
+	writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
+}
+
+void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
+				    u32 gl_value)
+{
+	u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
+
+	writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
+}
+
+void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
+				    u32 gl_value)
+{
+	u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
+
+	writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
 }
 
-static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector)
+static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
+				   struct hns3_nic_priv *priv)
 {
+	struct hnae3_handle *h = priv->ae_handle;
+
 	/* initialize the configuration for interrupt coalescing.
 	 * 1. GL (Interrupt Gap Limiter)
 	 * 2. RL (Interrupt Rate Limiter)
 	 */
 
-	/* Default :enable interrupt coalesce */
-	tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
+	/* Default: enable interrupt coalescing self-adaptive and GL */
+	tqp_vector->tx_group.gl_adapt_enable = 1;
+	tqp_vector->rx_group.gl_adapt_enable = 1;
+
 	tqp_vector->tx_group.int_gl = HNS3_INT_GL_50K;
-	hns3_set_vector_coalesc_gl(tqp_vector, HNS3_INT_GL_50K);
-	/* for now we are disabling Interrupt RL - we
-	 * will re-enable later
-	 */
-	hns3_set_vector_coalesc_rl(tqp_vector, 0);
+	tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
+
+	hns3_set_vector_coalesce_tx_gl(tqp_vector,
+				       tqp_vector->tx_group.int_gl);
+	hns3_set_vector_coalesce_rx_gl(tqp_vector,
+				       tqp_vector->rx_group.int_gl);
+
+	/* Default: disable RL */
+	h->kinfo.int_rl_setting = 0;
+	hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
+
 	tqp_vector->rx_group.flow_level = HNS3_FLOW_LOW;
 	tqp_vector->tx_group.flow_level = HNS3_FLOW_LOW;
 }
@@ -245,6 +272,8 @@ static int hns3_nic_net_up(struct net_device *netdev)
 	if (ret)
 		goto out_start_err;
 
+	clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
+
 	return 0;
 
 out_start_err:
@@ -284,6 +313,9 @@ static void hns3_nic_net_down(struct net_device *netdev)
 	const struct hnae3_ae_ops *ops;
 	int i;
 
+	if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
+		return;
+
 	/* stop ae_dev */
 	ops = priv->ae_handle->ae_algo->ops;
 	if (ops->stop)
@@ -721,6 +753,58 @@ static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
 	hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
 }
 
+static int hns3_fill_desc_vtags(struct sk_buff *skb,
+				struct hns3_enet_ring *tx_ring,
+				u32 *inner_vlan_flag,
+				u32 *out_vlan_flag,
+				u16 *inner_vtag,
+				u16 *out_vtag)
+{
+#define HNS3_TX_VLAN_PRIO_SHIFT 13
+
+	if (skb->protocol == htons(ETH_P_8021Q) &&
+	    !(tx_ring->tqp->handle->kinfo.netdev->features &
+	    NETIF_F_HW_VLAN_CTAG_TX)) {
+		/* When HW VLAN acceleration is turned off, and the stack
+		 * sets the protocol to 802.1q, the driver just need to
+		 * set the protocol to the encapsulated ethertype.
+		 */
+		skb->protocol = vlan_get_protocol(skb);
+		return 0;
+	}
+
+	if (skb_vlan_tag_present(skb)) {
+		u16 vlan_tag;
+
+		vlan_tag = skb_vlan_tag_get(skb);
+		vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
+
+		/* Based on hw strategy, use out_vtag in two layer tag case,
+		 * and use inner_vtag in one tag case.
+		 */
+		if (skb->protocol == htons(ETH_P_8021Q)) {
+			hnae_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
+			*out_vtag = vlan_tag;
+		} else {
+			hnae_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
+			*inner_vtag = vlan_tag;
+		}
+	} else if (skb->protocol == htons(ETH_P_8021Q)) {
+		struct vlan_ethhdr *vhdr;
+		int rc;
+
+		rc = skb_cow_head(skb, 0);
+		if (rc < 0)
+			return rc;
+		vhdr = (struct vlan_ethhdr *)skb->data;
+		vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
+					<< HNS3_TX_VLAN_PRIO_SHIFT);
+	}
+
+	skb->protocol = vlan_get_protocol(skb);
+	return 0;
+}
+
 static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 			  int size, dma_addr_t dma, int frag_end,
 			  enum hns_desc_type type)
@@ -731,6 +815,8 @@ static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 	u16 bdtp_fe_sc_vld_ra_ri = 0;
 	u32 type_cs_vlan_tso = 0;
 	struct sk_buff *skb;
+	u16 inner_vtag = 0;
+	u16 out_vtag = 0;
 	u32 paylen = 0;
 	u16 mss = 0;
 	__be16 protocol;
@@ -754,15 +840,16 @@ static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 		skb = (struct sk_buff *)priv;
 		paylen = skb->len;
 
+		ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
+					   &ol_type_vlan_len_msec,
+					   &inner_vtag, &out_vtag);
+		if (unlikely(ret))
+			return ret;
+
 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
 			skb_reset_mac_len(skb);
 			protocol = skb->protocol;
 
-			/* vlan packet*/
-			if (protocol == htons(ETH_P_8021Q)) {
-				protocol = vlan_get_protocol(skb);
-				skb->protocol = protocol;
-			}
 			ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
 			if (ret)
 				return ret;
@@ -788,6 +875,8 @@ static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
 			cpu_to_le32(type_cs_vlan_tso);
 		desc->tx.paylen = cpu_to_le32(paylen);
 		desc->tx.mss = cpu_to_le16(mss);
+		desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
+		desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
 	}
 
 	/* move ring pointer to next.*/
@@ -1029,25 +1118,50 @@ static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
 static int hns3_nic_set_features(struct net_device *netdev,
 				 netdev_features_t features)
 {
+	netdev_features_t changed = netdev->features ^ features;
 	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int ret;
 
-	if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
-		priv->ops.fill_desc = hns3_fill_desc_tso;
-		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
-	} else {
-		priv->ops.fill_desc = hns3_fill_desc;
-		priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
+	if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
+		if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
+			priv->ops.fill_desc = hns3_fill_desc_tso;
+			priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
+		} else {
+			priv->ops.fill_desc = hns3_fill_desc;
+			priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
+		}
+	}
+
+	if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
+	    h->ae_algo->ops->enable_vlan_filter) {
+		if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
+			h->ae_algo->ops->enable_vlan_filter(h, true);
+		else
+			h->ae_algo->ops->enable_vlan_filter(h, false);
+	}
+
+	if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
+	    h->ae_algo->ops->enable_hw_strip_rxvtag) {
+		if (features & NETIF_F_HW_VLAN_CTAG_RX)
+			ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
+		else
+			ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
+
+		if (ret)
+			return ret;
 	}
 
 	netdev->features = features;
 	return 0;
 }
 
-static void
-hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
+static void hns3_nic_get_stats64(struct net_device *netdev,
+				 struct rtnl_link_stats64 *stats)
 {
 	struct hns3_nic_priv *priv = netdev_priv(netdev);
 	int queue_num = priv->ae_handle->kinfo.num_tqps;
+	struct hnae3_handle *handle = priv->ae_handle;
 	struct hns3_enet_ring *ring;
 	unsigned int start;
 	unsigned int idx;
@@ -1055,6 +1169,13 @@ hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
 	u64 rx_bytes = 0;
 	u64 tx_pkts = 0;
 	u64 rx_pkts = 0;
+	u64 tx_drop = 0;
+	u64 rx_drop = 0;
+
+	if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
+		return;
+
+	handle->ae_algo->ops->update_stats(handle, &netdev->stats);
 
 	for (idx = 0; idx < queue_num; idx++) {
 		/* fetch the tx stats */
@@ -1063,6 +1184,8 @@ hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
 			start = u64_stats_fetch_begin_irq(&ring->syncp);
 			tx_bytes += ring->stats.tx_bytes;
 			tx_pkts += ring->stats.tx_pkts;
+			tx_drop += ring->stats.tx_busy;
+			tx_drop += ring->stats.sw_err_cnt;
 		} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 
 		/* fetch the rx stats */
@@ -1071,6 +1194,9 @@ hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
 			start = u64_stats_fetch_begin_irq(&ring->syncp);
 			rx_bytes += ring->stats.rx_bytes;
 			rx_pkts += ring->stats.rx_pkts;
+			rx_drop += ring->stats.non_vld_descs;
+			rx_drop += ring->stats.err_pkt_len;
+			rx_drop += ring->stats.l2_err;
 		} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 	}
 
@@ -1086,8 +1212,8 @@ hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
 	stats->rx_missed_errors = netdev->stats.rx_missed_errors;
 
 	stats->tx_errors = netdev->stats.tx_errors;
-	stats->rx_dropped = netdev->stats.rx_dropped;
-	stats->tx_dropped = netdev->stats.tx_dropped;
+	stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
+	stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
 	stats->collisions = netdev->stats.collisions;
 	stats->rx_over_errors = netdev->stats.rx_over_errors;
 	stats->rx_frame_errors = netdev->stats.rx_frame_errors;
@@ -1317,6 +1443,8 @@ static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
 		return ret;
 	}
 
+	netdev->mtu = new_mtu;
+
 	/* if the netdev was running earlier, bring it up again */
 	if (if_running && hns3_nic_net_open(netdev))
 		ret = -EINVAL;
@@ -1476,6 +1604,8 @@ static struct pci_driver hns3_driver = {
 /* set default feature to hns3 */
 static void hns3_set_default_feature(struct net_device *netdev)
 {
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+
 	netdev->priv_flags |= IFF_UNICAST_FLT;
 
 	netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
@@ -1490,6 +1620,7 @@ static void hns3_set_default_feature(struct net_device *netdev)
 
 	netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 		NETIF_F_HW_VLAN_CTAG_FILTER |
+		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
 		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
 		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
@@ -1503,11 +1634,15 @@ static void hns3_set_default_feature(struct net_device *netdev)
 		NETIF_F_GSO_UDP_TUNNEL_CSUM;
 
 	netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
-		NETIF_F_HW_VLAN_CTAG_FILTER |
+		NETIF_F_HW_VLAN_CTAG_TX |
 		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
 		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
 		NETIF_F_GSO_UDP_TUNNEL_CSUM;
+
+	if (!(h->flags & HNAE3_SUPPORT_VF))
+		netdev->hw_features |=
+			NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_RX;
 }
 
 static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
@@ -2083,6 +2218,22 @@ static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
 
 	prefetchw(skb->data);
 
+	/* Based on hw strategy, the tag offloaded will be stored at
+	 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
+	 * in one layer tag case.
+	 */
+	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
+		u16 vlan_tag;
+
+		vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
+		if (!(vlan_tag & VLAN_VID_MASK))
+			vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
+		if (vlan_tag & VLAN_VID_MASK)
+			__vlan_hwaccel_put_tag(skb,
+					       htons(ETH_P_8021Q),
+					       vlan_tag);
+	}
+
 	bnum = 1;
 	if (length <= HNS3_RX_HEAD_SIZE) {
 		memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
@@ -2301,25 +2452,22 @@ static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
 
 static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
 {
-	u16 rx_int_gl, tx_int_gl;
-	bool rx, tx;
-
-	rx = hns3_get_new_int_gl(&tqp_vector->rx_group);
-	tx = hns3_get_new_int_gl(&tqp_vector->tx_group);
-	rx_int_gl = tqp_vector->rx_group.int_gl;
-	tx_int_gl = tqp_vector->tx_group.int_gl;
-	if (rx && tx) {
-		if (rx_int_gl > tx_int_gl) {
-			tqp_vector->tx_group.int_gl = rx_int_gl;
-			tqp_vector->tx_group.flow_level =
-				tqp_vector->rx_group.flow_level;
-			hns3_set_vector_coalesc_gl(tqp_vector, rx_int_gl);
-		} else {
-			tqp_vector->rx_group.int_gl = tx_int_gl;
-			tqp_vector->rx_group.flow_level =
-				tqp_vector->tx_group.flow_level;
-			hns3_set_vector_coalesc_gl(tqp_vector, tx_int_gl);
-		}
+	struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
+	struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
+	bool rx_update, tx_update;
+
+	if (rx_group->gl_adapt_enable) {
+		rx_update = hns3_get_new_int_gl(rx_group);
+		if (rx_update)
+			hns3_set_vector_coalesce_rx_gl(tqp_vector,
+						       rx_group->int_gl);
+	}
+
+	if (tx_group->gl_adapt_enable) {
+		tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
+		if (tx_update)
+			hns3_set_vector_coalesce_tx_gl(tqp_vector,
+						       tx_group->int_gl);
 	}
 }
 
@@ -2380,6 +2528,8 @@ static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
 		cur_chain->tqp_index = tx_ring->tqp->tqp_index;
 		hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
 			     HNAE3_RING_TYPE_TX);
+		hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
+			       HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
 
 		cur_chain->next = NULL;
 
@@ -2395,6 +2545,10 @@ static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
 			chain->tqp_index = tx_ring->tqp->tqp_index;
 			hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
 				     HNAE3_RING_TYPE_TX);
+			hnae_set_field(chain->int_gl_idx,
+				       HNAE3_RING_GL_IDX_M,
+				       HNAE3_RING_GL_IDX_S,
+				       HNAE3_RING_GL_TX);
 
 			cur_chain = chain;
 		}
@@ -2406,6 +2560,8 @@ static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
 		cur_chain->tqp_index = rx_ring->tqp->tqp_index;
 		hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
 			     HNAE3_RING_TYPE_RX);
+		hnae_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
+			       HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
 
 		rx_ring = rx_ring->next;
 	}
@@ -2419,6 +2575,9 @@ static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
 		chain->tqp_index = rx_ring->tqp->tqp_index;
 		hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
 			     HNAE3_RING_TYPE_RX);
+		hnae_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
+			       HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
+
 		cur_chain = chain;
 
 		rx_ring = rx_ring->next;
@@ -2507,7 +2666,7 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
 		tqp_vector->rx_group.total_packets = 0;
 		tqp_vector->tx_group.total_bytes = 0;
 		tqp_vector->tx_group.total_packets = 0;
-		hns3_vector_gl_rl_init(tqp_vector);
+		hns3_vector_gl_rl_init(tqp_vector, priv);
 		tqp_vector->handle = h;
 
 		ret = hns3_get_vector_ring_chain(tqp_vector,
@@ -2649,6 +2808,19 @@ static int hns3_get_ring_config(struct hns3_nic_priv *priv)
 	return ret;
 }
 
+static void hns3_put_ring_config(struct hns3_nic_priv *priv)
+{
+	struct hnae3_handle *h = priv->ae_handle;
+	int i;
+
+	for (i = 0; i < h->kinfo.num_tqps; i++) {
+		devm_kfree(priv->dev, priv->ring_data[i].ring);
+		devm_kfree(priv->dev,
+			   priv->ring_data[i + h->kinfo.num_tqps].ring);
+	}
+	devm_kfree(priv->dev, priv->ring_data);
+}
+
 static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
 {
 	int ret;
@@ -2785,8 +2957,12 @@ int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
 			h->ae_algo->ops->reset_queue(h, i);
 
 		hns3_fini_ring(priv->ring_data[i].ring);
+		devm_kfree(priv->dev, priv->ring_data[i].ring);
 		hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
+		devm_kfree(priv->dev,
+			   priv->ring_data[i + h->kinfo.num_tqps].ring);
 	}
+	devm_kfree(priv->dev, priv->ring_data);
 
 	return 0;
 }
@@ -3142,8 +3318,8 @@ static int hns3_reset_notify(struct hnae3_handle *handle,
 
 	switch (type) {
 	case HNAE3_UP_CLIENT:
-                ret = hns3_reset_notify_up_enet(handle);
-                break;
+		ret = hns3_reset_notify_up_enet(handle);
+		break;
 	case HNAE3_DOWN_CLIENT:
 		ret = hns3_reset_notify_down_enet(handle);
 		break;
@@ -3160,6 +3336,115 @@ static int hns3_reset_notify(struct hnae3_handle *handle,
 	return ret;
 }
 
+static u16 hns3_get_max_available_channels(struct net_device *netdev)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+	u16 free_tqps, max_rss_size, max_tqps;
+
+	h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
+	max_tqps = h->kinfo.num_tc * max_rss_size;
+
+	return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
+}
+
+static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+	int ret;
+
+	ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
+	if (ret)
+		return ret;
+
+	ret = hns3_get_ring_config(priv);
+	if (ret)
+		return ret;
+
+	ret = hns3_nic_init_vector_data(priv);
+	if (ret)
+		goto err_uninit_vector;
+
+	ret = hns3_init_all_ring(priv);
+	if (ret)
+		goto err_put_ring;
+
+	return 0;
+
+err_put_ring:
+	hns3_put_ring_config(priv);
+err_uninit_vector:
+	hns3_nic_uninit_vector_data(priv);
+	return ret;
+}
+
+static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
+{
+	return (new_tqp_num / num_tc) * num_tc;
+}
+
+int hns3_set_channels(struct net_device *netdev,
+		      struct ethtool_channels *ch)
+{
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+	struct hnae3_knic_private_info *kinfo = &h->kinfo;
+	bool if_running = netif_running(netdev);
+	u32 new_tqp_num = ch->combined_count;
+	u16 org_tqp_num;
+	int ret;
+
+	if (ch->rx_count || ch->tx_count)
+		return -EINVAL;
+
+	if (new_tqp_num > hns3_get_max_available_channels(netdev) ||
+	    new_tqp_num < kinfo->num_tc) {
+		dev_err(&netdev->dev,
+			"Change tqps fail, the tqp range is from %d to %d",
+			kinfo->num_tc,
+			hns3_get_max_available_channels(netdev));
+		return -EINVAL;
+	}
+
+	new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
+	if (kinfo->num_tqps == new_tqp_num)
+		return 0;
+
+	if (if_running)
+		dev_close(netdev);
+
+	hns3_clear_all_ring(h);
+
+	ret = hns3_nic_uninit_vector_data(priv);
+	if (ret) {
+		dev_err(&netdev->dev,
+			"Unbind vector with tqp fail, nothing is changed");
+		goto open_netdev;
+	}
+
+	hns3_uninit_all_ring(priv);
+
+	org_tqp_num = h->kinfo.num_tqps;
+	ret = hns3_modify_tqp_num(netdev, new_tqp_num);
+	if (ret) {
+		ret = hns3_modify_tqp_num(netdev, org_tqp_num);
+		if (ret) {
+			/* If revert to old tqp failed, fatal error occurred */
+			dev_err(&netdev->dev,
+				"Revert to old tqp num fail, ret=%d", ret);
+			return ret;
+		}
+		dev_info(&netdev->dev,
+			 "Change tqp num fail, Revert to old tqp num");
+	}
+
+open_netdev:
+	if (if_running)
+		dev_open(netdev);
+
+	return ret;
+}
+
 static const struct hnae3_client_ops client_ops = {
 	.init_instance = hns3_client_init,
 	.uninit_instance = hns3_client_uninit,
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
similarity index 94%
rename from drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.h
rename to drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
index 8a9de759957b..213f501b30bb 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_enet.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
@@ -451,10 +451,14 @@ enum hns3_link_mode_bits {
 	HNS3_LM_COUNT = 15
 };
 
-#define HNS3_INT_GL_50K		0x000A
-#define HNS3_INT_GL_20K		0x0019
-#define HNS3_INT_GL_18K		0x001B
-#define HNS3_INT_GL_8K		0x003E
+#define HNS3_INT_GL_MAX			0x1FE0
+#define HNS3_INT_GL_50K			0x0014
+#define HNS3_INT_GL_20K			0x0032
+#define HNS3_INT_GL_18K			0x0036
+#define HNS3_INT_GL_8K			0x007C
+
+#define HNS3_INT_RL_MAX			0x00EC
+#define HNS3_INT_RL_ENABLE_MASK		0x40
 
 struct hns3_enet_ring_group {
 	/* array of pointers to rings */
@@ -464,6 +468,7 @@ struct hns3_enet_ring_group {
 	u16 count;
 	enum hns3_flow_level_range flow_level;
 	u16 int_gl;
+	u8 gl_adapt_enable;
 };
 
 struct hns3_enet_tqp_vector {
@@ -594,7 +599,15 @@ static inline void hns3_write_reg(void __iomem *base, u32 reg, u32 value)
 #define hns3_get_handle(ndev) \
 	(((struct hns3_nic_priv *)netdev_priv(ndev))->ae_handle)
 
+#define hns3_gl_usec_to_reg(int_gl) (int_gl >> 1)
+#define hns3_gl_round_down(int_gl) round_down(int_gl, 2)
+
+#define hns3_rl_usec_to_reg(int_rl) (int_rl >> 2)
+#define hns3_rl_round_down(int_rl) round_down(int_rl, 4)
+
 void hns3_ethtool_set_ops(struct net_device *netdev);
+int hns3_set_channels(struct net_device *netdev,
+		      struct ethtool_channels *ch);
 
 bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget);
 int hns3_init_all_ring(struct hns3_nic_priv *priv);
@@ -604,6 +617,13 @@ int hns3_clean_rx_ring(
 		struct hns3_enet_ring *ring, int budget,
 		void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *));
 
+void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
+				    u32 gl_value);
+void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
+				    u32 gl_value);
+void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
+				 u32 rl_value);
+
 #ifdef CONFIG_HNS3_DCB
 void hns3_dcbnl_setup(struct hnae3_handle *handle);
 #else
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
similarity index 70%
rename from drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_ethtool.c
rename to drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
index a21470c72da3..b034c7f24eda 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hns3_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
@@ -15,26 +15,25 @@
 
 struct hns3_stats {
 	char stats_string[ETH_GSTRING_LEN];
-	int stats_size;
 	int stats_offset;
 };
 
 /* tqp related stats */
 #define HNS3_TQP_STAT(_string, _member)	{			\
 	.stats_string = _string,				\
-	.stats_size = FIELD_SIZEOF(struct ring_stats, _member),	\
-	.stats_offset = offsetof(struct hns3_enet_ring, stats),	\
-}								\
+	.stats_offset = offsetof(struct hns3_enet_ring, stats) +\
+			offsetof(struct ring_stats, _member),   \
+}
 
 static const struct hns3_stats hns3_txq_stats[] = {
 	/* Tx per-queue statistics */
-	HNS3_TQP_STAT("tx_io_err_cnt", io_err_cnt),
-	HNS3_TQP_STAT("tx_sw_err_cnt", sw_err_cnt),
-	HNS3_TQP_STAT("tx_seg_pkt_cnt", seg_pkt_cnt),
-	HNS3_TQP_STAT("tx_pkts", tx_pkts),
-	HNS3_TQP_STAT("tx_bytes", tx_bytes),
-	HNS3_TQP_STAT("tx_err_cnt", tx_err_cnt),
-	HNS3_TQP_STAT("tx_restart_queue", restart_queue),
+	HNS3_TQP_STAT("io_err_cnt", io_err_cnt),
+	HNS3_TQP_STAT("tx_dropped", sw_err_cnt),
+	HNS3_TQP_STAT("seg_pkt_cnt", seg_pkt_cnt),
+	HNS3_TQP_STAT("packets", tx_pkts),
+	HNS3_TQP_STAT("bytes", tx_bytes),
+	HNS3_TQP_STAT("errors", tx_err_cnt),
+	HNS3_TQP_STAT("tx_wake", restart_queue),
 	HNS3_TQP_STAT("tx_busy", tx_busy),
 };
 
@@ -42,18 +41,18 @@ static const struct hns3_stats hns3_txq_stats[] = {
 
 static const struct hns3_stats hns3_rxq_stats[] = {
 	/* Rx per-queue statistics */
-	HNS3_TQP_STAT("rx_io_err_cnt", io_err_cnt),
-	HNS3_TQP_STAT("rx_sw_err_cnt", sw_err_cnt),
-	HNS3_TQP_STAT("rx_seg_pkt_cnt", seg_pkt_cnt),
-	HNS3_TQP_STAT("rx_pkts", rx_pkts),
-	HNS3_TQP_STAT("rx_bytes", rx_bytes),
-	HNS3_TQP_STAT("rx_err_cnt", rx_err_cnt),
-	HNS3_TQP_STAT("rx_reuse_pg_cnt", reuse_pg_cnt),
-	HNS3_TQP_STAT("rx_err_pkt_len", err_pkt_len),
-	HNS3_TQP_STAT("rx_non_vld_descs", non_vld_descs),
-	HNS3_TQP_STAT("rx_err_bd_num", err_bd_num),
-	HNS3_TQP_STAT("rx_l2_err", l2_err),
-	HNS3_TQP_STAT("rx_l3l4_csum_err", l3l4_csum_err),
+	HNS3_TQP_STAT("io_err_cnt", io_err_cnt),
+	HNS3_TQP_STAT("rx_dropped", sw_err_cnt),
+	HNS3_TQP_STAT("seg_pkt_cnt", seg_pkt_cnt),
+	HNS3_TQP_STAT("packets", rx_pkts),
+	HNS3_TQP_STAT("bytes", rx_bytes),
+	HNS3_TQP_STAT("errors", rx_err_cnt),
+	HNS3_TQP_STAT("reuse_pg_cnt", reuse_pg_cnt),
+	HNS3_TQP_STAT("err_pkt_len", err_pkt_len),
+	HNS3_TQP_STAT("non_vld_descs", non_vld_descs),
+	HNS3_TQP_STAT("err_bd_num", err_bd_num),
+	HNS3_TQP_STAT("l2_err", l2_err),
+	HNS3_TQP_STAT("l3l4_csum_err", l3l4_csum_err),
 };
 
 #define HNS3_RXQ_STATS_COUNT ARRAY_SIZE(hns3_rxq_stats)
@@ -389,9 +388,9 @@ static int hns3_get_sset_count(struct net_device *netdev, int stringset)
 }
 
 static void *hns3_update_strings(u8 *data, const struct hns3_stats *stats,
-				 u32 stat_count, u32 num_tqps)
+		u32 stat_count, u32 num_tqps, const char *prefix)
 {
-#define MAX_PREFIX_SIZE (8 + 4)
+#define MAX_PREFIX_SIZE (6 + 4)
 	u32 size_left;
 	u32 i, j;
 	u32 n1;
@@ -401,7 +400,8 @@ static void *hns3_update_strings(u8 *data, const struct hns3_stats *stats,
 			data[ETH_GSTRING_LEN - 1] = '\0';
 
 			/* first, prepend the prefix string */
-			n1 = snprintf(data, MAX_PREFIX_SIZE, "rcb_q%d_", i);
+			n1 = snprintf(data, MAX_PREFIX_SIZE, "%s#%d_",
+				      prefix, i);
 			n1 = min_t(uint, n1, MAX_PREFIX_SIZE - 1);
 			size_left = (ETH_GSTRING_LEN - 1) - n1;
 
@@ -417,14 +417,16 @@ static void *hns3_update_strings(u8 *data, const struct hns3_stats *stats,
 static u8 *hns3_get_strings_tqps(struct hnae3_handle *handle, u8 *data)
 {
 	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+	const char tx_prefix[] = "txq";
+	const char rx_prefix[] = "rxq";
 
 	/* get strings for Tx */
 	data = hns3_update_strings(data, hns3_txq_stats, HNS3_TXQ_STATS_COUNT,
-				   kinfo->num_tqps);
+				   kinfo->num_tqps, tx_prefix);
 
 	/* get strings for Rx */
 	data = hns3_update_strings(data, hns3_rxq_stats, HNS3_RXQ_STATS_COUNT,
-				   kinfo->num_tqps);
+				   kinfo->num_tqps, rx_prefix);
 
 	return data;
 }
@@ -455,13 +457,13 @@ static u64 *hns3_get_stats_tqps(struct hnae3_handle *handle, u64 *data)
 	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
 	struct hns3_enet_ring *ring;
 	u8 *stat;
-	u32 i;
+	int i, j;
 
 	/* get stats for Tx */
 	for (i = 0; i < kinfo->num_tqps; i++) {
 		ring = nic_priv->ring_data[i].ring;
-		for (i = 0; i < HNS3_TXQ_STATS_COUNT; i++) {
-			stat = (u8 *)ring + hns3_txq_stats[i].stats_offset;
+		for (j = 0; j < HNS3_TXQ_STATS_COUNT; j++) {
+			stat = (u8 *)ring + hns3_txq_stats[j].stats_offset;
 			*data++ = *(u64 *)stat;
 		}
 	}
@@ -469,8 +471,8 @@ static u64 *hns3_get_stats_tqps(struct hnae3_handle *handle, u64 *data)
 	/* get stats for Rx */
 	for (i = 0; i < kinfo->num_tqps; i++) {
 		ring = nic_priv->ring_data[i + kinfo->num_tqps].ring;
-		for (i = 0; i < HNS3_RXQ_STATS_COUNT; i++) {
-			stat = (u8 *)ring + hns3_rxq_stats[i].stats_offset;
+		for (j = 0; j < HNS3_RXQ_STATS_COUNT; j++) {
+			stat = (u8 *)ring + hns3_rxq_stats[j].stats_offset;
 			*data++ = *(u64 *)stat;
 		}
 	}
@@ -559,10 +561,23 @@ static void hns3_get_pauseparam(struct net_device *netdev,
 			&param->rx_pause, &param->tx_pause);
 }
 
+static int hns3_set_pauseparam(struct net_device *netdev,
+			       struct ethtool_pauseparam *param)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+
+	if (h->ae_algo->ops->set_pauseparam)
+		return h->ae_algo->ops->set_pauseparam(h, param->autoneg,
+						       param->rx_pause,
+						       param->tx_pause);
+	return -EOPNOTSUPP;
+}
+
 static int hns3_get_link_ksettings(struct net_device *netdev,
 				   struct ethtool_link_ksettings *cmd)
 {
 	struct hnae3_handle *h = hns3_get_handle(netdev);
+	u32 flowctrl_adv = 0;
 	u32 supported_caps;
 	u32 advertised_caps;
 	u8 media_type = HNAE3_MEDIA_TYPE_UNKNOWN;
@@ -638,6 +653,8 @@ static int hns3_get_link_ksettings(struct net_device *netdev,
 		if (!cmd->base.autoneg)
 			advertised_caps &= ~HNS3_LM_AUTONEG_BIT;
 
+		advertised_caps &= ~HNS3_LM_PAUSE_BIT;
+
 		/* now, map driver link modes to ethtool link modes */
 		hns3_driv_to_eth_caps(supported_caps, cmd, false);
 		hns3_driv_to_eth_caps(advertised_caps, cmd, true);
@@ -650,6 +667,18 @@ static int hns3_get_link_ksettings(struct net_device *netdev,
 	/* 4.mdio_support */
 	cmd->base.mdio_support = ETH_MDIO_SUPPORTS_C22;
 
+	/* 5.get flow control setttings */
+	if (h->ae_algo->ops->get_flowctrl_adv)
+		h->ae_algo->ops->get_flowctrl_adv(h, &flowctrl_adv);
+
+	if (flowctrl_adv & ADVERTISED_Pause)
+		ethtool_link_ksettings_add_link_mode(cmd, advertising,
+						     Pause);
+
+	if (flowctrl_adv & ADVERTISED_Asym_Pause)
+		ethtool_link_ksettings_add_link_mode(cmd, advertising,
+						     Asym_Pause);
+
 	return 0;
 }
 
@@ -730,7 +759,7 @@ static int hns3_get_rxnfc(struct net_device *netdev,
 
 	switch (cmd->cmd) {
 	case ETHTOOL_GRXRINGS:
-		cmd->data = h->kinfo.num_tc * h->kinfo.rss_size;
+		cmd->data = h->kinfo.rss_size;
 		break;
 	case ETHTOOL_GRXFH:
 		return h->ae_algo->ops->get_rss_tuple(h, cmd);
@@ -849,6 +878,241 @@ static int hns3_nway_reset(struct net_device *netdev)
 	return genphy_restart_aneg(phy);
 }
 
+static void hns3_get_channels(struct net_device *netdev,
+			      struct ethtool_channels *ch)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+
+	if (h->ae_algo->ops->get_channels)
+		h->ae_algo->ops->get_channels(h, ch);
+}
+
+static int hns3_get_coalesce_per_queue(struct net_device *netdev, u32 queue,
+				       struct ethtool_coalesce *cmd)
+{
+	struct hns3_enet_tqp_vector *tx_vector, *rx_vector;
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	u16 queue_num = h->kinfo.num_tqps;
+
+	if (queue >= queue_num) {
+		netdev_err(netdev,
+			   "Invalid queue value %d! Queue max id=%d\n",
+			   queue, queue_num - 1);
+		return -EINVAL;
+	}
+
+	tx_vector = priv->ring_data[queue].ring->tqp_vector;
+	rx_vector = priv->ring_data[queue_num + queue].ring->tqp_vector;
+
+	cmd->use_adaptive_tx_coalesce = tx_vector->tx_group.gl_adapt_enable;
+	cmd->use_adaptive_rx_coalesce = rx_vector->rx_group.gl_adapt_enable;
+
+	cmd->tx_coalesce_usecs = tx_vector->tx_group.int_gl;
+	cmd->rx_coalesce_usecs = rx_vector->rx_group.int_gl;
+
+	cmd->tx_coalesce_usecs_high = h->kinfo.int_rl_setting;
+	cmd->rx_coalesce_usecs_high = h->kinfo.int_rl_setting;
+
+	return 0;
+}
+
+static int hns3_get_coalesce(struct net_device *netdev,
+			     struct ethtool_coalesce *cmd)
+{
+	return hns3_get_coalesce_per_queue(netdev, 0, cmd);
+}
+
+static int hns3_check_gl_coalesce_para(struct net_device *netdev,
+				       struct ethtool_coalesce *cmd)
+{
+	u32 rx_gl, tx_gl;
+
+	if (cmd->rx_coalesce_usecs > HNS3_INT_GL_MAX) {
+		netdev_err(netdev,
+			   "Invalid rx-usecs value, rx-usecs range is 0-%d\n",
+			   HNS3_INT_GL_MAX);
+		return -EINVAL;
+	}
+
+	if (cmd->tx_coalesce_usecs > HNS3_INT_GL_MAX) {
+		netdev_err(netdev,
+			   "Invalid tx-usecs value, tx-usecs range is 0-%d\n",
+			   HNS3_INT_GL_MAX);
+		return -EINVAL;
+	}
+
+	rx_gl = hns3_gl_round_down(cmd->rx_coalesce_usecs);
+	if (rx_gl != cmd->rx_coalesce_usecs) {
+		netdev_info(netdev,
+			    "rx_usecs(%d) rounded down to %d, because it must be multiple of 2.\n",
+			    cmd->rx_coalesce_usecs, rx_gl);
+	}
+
+	tx_gl = hns3_gl_round_down(cmd->tx_coalesce_usecs);
+	if (tx_gl != cmd->tx_coalesce_usecs) {
+		netdev_info(netdev,
+			    "tx_usecs(%d) rounded down to %d, because it must be multiple of 2.\n",
+			    cmd->tx_coalesce_usecs, tx_gl);
+	}
+
+	return 0;
+}
+
+static int hns3_check_rl_coalesce_para(struct net_device *netdev,
+				       struct ethtool_coalesce *cmd)
+{
+	u32 rl;
+
+	if (cmd->tx_coalesce_usecs_high != cmd->rx_coalesce_usecs_high) {
+		netdev_err(netdev,
+			   "tx_usecs_high must be same as rx_usecs_high.\n");
+		return -EINVAL;
+	}
+
+	if (cmd->rx_coalesce_usecs_high > HNS3_INT_RL_MAX) {
+		netdev_err(netdev,
+			   "Invalid usecs_high value, usecs_high range is 0-%d\n",
+			   HNS3_INT_RL_MAX);
+		return -EINVAL;
+	}
+
+	rl = hns3_rl_round_down(cmd->rx_coalesce_usecs_high);
+	if (rl != cmd->rx_coalesce_usecs_high) {
+		netdev_info(netdev,
+			    "usecs_high(%d) rounded down to %d, because it must be multiple of 4.\n",
+			    cmd->rx_coalesce_usecs_high, rl);
+	}
+
+	return 0;
+}
+
+static int hns3_check_coalesce_para(struct net_device *netdev,
+				    struct ethtool_coalesce *cmd)
+{
+	int ret;
+
+	ret = hns3_check_gl_coalesce_para(netdev, cmd);
+	if (ret) {
+		netdev_err(netdev,
+			   "Check gl coalesce param fail. ret = %d\n", ret);
+		return ret;
+	}
+
+	ret = hns3_check_rl_coalesce_para(netdev, cmd);
+	if (ret) {
+		netdev_err(netdev,
+			   "Check rl coalesce param fail. ret = %d\n", ret);
+		return ret;
+	}
+
+	if (cmd->use_adaptive_tx_coalesce == 1 ||
+	    cmd->use_adaptive_rx_coalesce == 1) {
+		netdev_info(netdev,
+			    "adaptive-tx=%d and adaptive-rx=%d, tx_usecs or rx_usecs will changed dynamically.\n",
+			    cmd->use_adaptive_tx_coalesce,
+			    cmd->use_adaptive_rx_coalesce);
+	}
+
+	return 0;
+}
+
+static void hns3_set_coalesce_per_queue(struct net_device *netdev,
+					struct ethtool_coalesce *cmd,
+					u32 queue)
+{
+	struct hns3_enet_tqp_vector *tx_vector, *rx_vector;
+	struct hns3_nic_priv *priv = netdev_priv(netdev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int queue_num = h->kinfo.num_tqps;
+
+	tx_vector = priv->ring_data[queue].ring->tqp_vector;
+	rx_vector = priv->ring_data[queue_num + queue].ring->tqp_vector;
+
+	tx_vector->tx_group.gl_adapt_enable = cmd->use_adaptive_tx_coalesce;
+	rx_vector->rx_group.gl_adapt_enable = cmd->use_adaptive_rx_coalesce;
+
+	tx_vector->tx_group.int_gl = cmd->tx_coalesce_usecs;
+	rx_vector->rx_group.int_gl = cmd->rx_coalesce_usecs;
+
+	hns3_set_vector_coalesce_tx_gl(tx_vector, tx_vector->tx_group.int_gl);
+	hns3_set_vector_coalesce_rx_gl(rx_vector, rx_vector->rx_group.int_gl);
+
+	hns3_set_vector_coalesce_rl(tx_vector, h->kinfo.int_rl_setting);
+	hns3_set_vector_coalesce_rl(rx_vector, h->kinfo.int_rl_setting);
+}
+
+static int hns3_set_coalesce(struct net_device *netdev,
+			     struct ethtool_coalesce *cmd)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+	u16 queue_num = h->kinfo.num_tqps;
+	int ret;
+	int i;
+
+	ret = hns3_check_coalesce_para(netdev, cmd);
+	if (ret)
+		return ret;
+
+	h->kinfo.int_rl_setting =
+		hns3_rl_round_down(cmd->rx_coalesce_usecs_high);
+
+	for (i = 0; i < queue_num; i++)
+		hns3_set_coalesce_per_queue(netdev, cmd, i);
+
+	return 0;
+}
+
+static int hns3_get_regs_len(struct net_device *netdev)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+
+	if (!h->ae_algo->ops->get_regs_len)
+		return -EOPNOTSUPP;
+
+	return h->ae_algo->ops->get_regs_len(h);
+}
+
+static void hns3_get_regs(struct net_device *netdev,
+			  struct ethtool_regs *cmd, void *data)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+
+	if (!h->ae_algo->ops->get_regs)
+		return;
+
+	h->ae_algo->ops->get_regs(h, &cmd->version, data);
+}
+
+static int hns3_set_phys_id(struct net_device *netdev,
+			    enum ethtool_phys_id_state state)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+
+	if (!h->ae_algo || !h->ae_algo->ops || !h->ae_algo->ops->set_led_id)
+		return -EOPNOTSUPP;
+
+	return h->ae_algo->ops->set_led_id(h, state);
+}
+
+static const struct ethtool_ops hns3vf_ethtool_ops = {
+	.get_drvinfo = hns3_get_drvinfo,
+	.get_ringparam = hns3_get_ringparam,
+	.set_ringparam = hns3_set_ringparam,
+	.get_strings = hns3_get_strings,
+	.get_ethtool_stats = hns3_get_stats,
+	.get_sset_count = hns3_get_sset_count,
+	.get_rxnfc = hns3_get_rxnfc,
+	.get_rxfh_key_size = hns3_get_rss_key_size,
+	.get_rxfh_indir_size = hns3_get_rss_indir_size,
+	.get_rxfh = hns3_get_rss,
+	.set_rxfh = hns3_set_rss,
+	.get_link_ksettings = hns3_get_link_ksettings,
+	.get_channels = hns3_get_channels,
+	.get_coalesce = hns3_get_coalesce,
+	.set_coalesce = hns3_set_coalesce,
+};
+
 static const struct ethtool_ops hns3_ethtool_ops = {
 	.self_test = hns3_self_test,
 	.get_drvinfo = hns3_get_drvinfo,
@@ -856,6 +1120,7 @@ static const struct ethtool_ops hns3_ethtool_ops = {
 	.get_ringparam = hns3_get_ringparam,
 	.set_ringparam = hns3_set_ringparam,
 	.get_pauseparam = hns3_get_pauseparam,
+	.set_pauseparam = hns3_set_pauseparam,
 	.get_strings = hns3_get_strings,
 	.get_ethtool_stats = hns3_get_stats,
 	.get_sset_count = hns3_get_sset_count,
@@ -868,9 +1133,21 @@ static const struct ethtool_ops hns3_ethtool_ops = {
 	.get_link_ksettings = hns3_get_link_ksettings,
 	.set_link_ksettings = hns3_set_link_ksettings,
 	.nway_reset = hns3_nway_reset,
+	.get_channels = hns3_get_channels,
+	.set_channels = hns3_set_channels,
+	.get_coalesce = hns3_get_coalesce,
+	.set_coalesce = hns3_set_coalesce,
+	.get_regs_len = hns3_get_regs_len,
+	.get_regs = hns3_get_regs,
+	.set_phys_id = hns3_set_phys_id,
 };
 
 void hns3_ethtool_set_ops(struct net_device *netdev)
 {
-	netdev->ethtool_ops = &hns3_ethtool_ops;
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+
+	if (h->flags & HNAE3_SUPPORT_VF)
+		netdev->ethtool_ops = &hns3vf_ethtool_ops;
+	else
+		netdev->ethtool_ops = &hns3_ethtool_ops;
 }
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile b/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile
index d2b20d01a58c..cb8ddd043476 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0+
 #
 # Makefile for the HISILICON network device drivers.
 #
@@ -5,11 +6,6 @@
 ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3
 
 obj-$(CONFIG_HNS3_HCLGE) += hclge.o
-hclge-objs = hclge_main.o hclge_cmd.o hclge_mdio.o hclge_tm.o
+hclge-objs = hclge_main.o hclge_cmd.o hclge_mdio.o hclge_tm.o hclge_mbx.o
 
 hclge-$(CONFIG_HNS3_DCB) += hclge_dcb.o
-
-obj-$(CONFIG_HNS3_ENET) += hns3.o
-hns3-objs = hns3_enet.o hns3_ethtool.o
-
-hns3-$(CONFIG_HNS3_DCB) += hns3_dcbnl.o
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
index ce5ed8845042..3fd10a6bec53 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
@@ -102,6 +102,10 @@ enum hclge_opcode_type {
 	HCLGE_OPC_STATS_64_BIT		= 0x0030,
 	HCLGE_OPC_STATS_32_BIT		= 0x0031,
 	HCLGE_OPC_STATS_MAC		= 0x0032,
+
+	HCLGE_OPC_QUERY_REG_NUM		= 0x0040,
+	HCLGE_OPC_QUERY_32_BIT_REG	= 0x0041,
+	HCLGE_OPC_QUERY_64_BIT_REG	= 0x0042,
 	/* Device management command */
 
 	/* MAC commond */
@@ -111,6 +115,7 @@ enum hclge_opcode_type {
 	HCLGE_OPC_QUERY_LINK_STATUS	= 0x0307,
 	HCLGE_OPC_CONFIG_MAX_FRM_SIZE	= 0x0308,
 	HCLGE_OPC_CONFIG_SPEED_DUP	= 0x0309,
+	HCLGE_OPC_STATS_MAC_TRAFFIC	= 0x0314,
 	/* MACSEC command */
 
 	/* PFC/Pause CMD*/
@@ -180,6 +185,10 @@ enum hclge_opcode_type {
 	/* Promisuous mode command */
 	HCLGE_OPC_CFG_PROMISC_MODE	= 0x0E01,
 
+	/* Vlan offload command */
+	HCLGE_OPC_VLAN_PORT_TX_CFG	= 0x0F01,
+	HCLGE_OPC_VLAN_PORT_RX_CFG	= 0x0F02,
+
 	/* Interrupts cmd */
 	HCLGE_OPC_ADD_RING_TO_VECTOR	= 0x1503,
 	HCLGE_OPC_DEL_RING_TO_VECTOR	= 0x1504,
@@ -191,6 +200,7 @@ enum hclge_opcode_type {
 	HCLGE_OPC_MAC_VLAN_INSERT	    = 0x1003,
 	HCLGE_OPC_MAC_ETHTYPE_ADD	    = 0x1010,
 	HCLGE_OPC_MAC_ETHTYPE_REMOVE	= 0x1011,
+	HCLGE_OPC_MAC_VLAN_MASK_SET	= 0x1012,
 
 	/* Multicast linear table cmd */
 	HCLGE_OPC_MTA_MAC_MODE_CFG	    = 0x1020,
@@ -218,6 +228,9 @@ enum hclge_opcode_type {
 
 	/* Mailbox cmd */
 	HCLGEVF_OPC_MBX_PF_TO_VF	= 0x2000,
+
+	/* Led command */
+	HCLGE_OPC_LED_STATUS_CFG	= 0xB000,
 };
 
 #define HCLGE_TQP_REG_OFFSET		0x80000
@@ -399,6 +412,8 @@ struct hclge_pf_res_cmd {
 #define HCLGE_CFG_MAC_ADDR_H_M	GENMASK(15, 0)
 #define HCLGE_CFG_DEFAULT_SPEED_S	16
 #define HCLGE_CFG_DEFAULT_SPEED_M	GENMASK(23, 16)
+#define HCLGE_CFG_RSS_SIZE_S	24
+#define HCLGE_CFG_RSS_SIZE_M	GENMASK(31, 24)
 
 struct hclge_cfg_param_cmd {
 	__le32 offset;
@@ -549,8 +564,6 @@ struct hclge_config_auto_neg_cmd {
 	u8      rsv[20];
 };
 
-#define HCLGE_MAC_MIN_MTU		64
-#define HCLGE_MAC_MAX_MTU		9728
 #define HCLGE_MAC_UPLINK_PORT		0x100
 
 struct hclge_config_max_frm_size_cmd {
@@ -587,6 +600,37 @@ struct hclge_mac_vlan_tbl_entry_cmd {
 	u8      rsv2[6];
 };
 
+#define HCLGE_VLAN_MASK_EN_B		0x0
+struct hclge_mac_vlan_mask_entry_cmd {
+	u8 rsv0[2];
+	u8 vlan_mask;
+	u8 rsv1;
+	u8 mac_mask[6];
+	u8 rsv2[14];
+};
+
+#define HCLGE_MAC_MGR_MASK_VLAN_B		BIT(0)
+#define HCLGE_MAC_MGR_MASK_MAC_B		BIT(1)
+#define HCLGE_MAC_MGR_MASK_ETHERTYPE_B		BIT(2)
+#define HCLGE_MAC_ETHERTYPE_LLDP		0x88cc
+
+struct hclge_mac_mgr_tbl_entry_cmd {
+	u8      flags;
+	u8      resp_code;
+	__le16  vlan_tag;
+	__le32  mac_addr_hi32;
+	__le16  mac_addr_lo16;
+	__le16  rsv1;
+	__le16  ethter_type;
+	__le16  egress_port;
+	__le16  egress_queue;
+	u8      sw_port_id_aware;
+	u8      rsv2;
+	u8      i_port_bitmap;
+	u8      i_port_direction;
+	u8      rsv3[2];
+};
+
 #define HCLGE_CFG_MTA_MAC_SEL_S		0x0
 #define HCLGE_CFG_MTA_MAC_SEL_M		GENMASK(1, 0)
 #define HCLGE_CFG_MTA_MAC_EN_B		0x7
@@ -658,6 +702,47 @@ struct hclge_vlan_filter_vf_cfg_cmd {
 	u8  vf_bitmap[16];
 };
 
+#define HCLGE_ACCEPT_TAG_B		0
+#define HCLGE_ACCEPT_UNTAG_B		1
+#define HCLGE_PORT_INS_TAG1_EN_B	2
+#define HCLGE_PORT_INS_TAG2_EN_B	3
+#define HCLGE_CFG_NIC_ROCE_SEL_B	4
+struct hclge_vport_vtag_tx_cfg_cmd {
+	u8 vport_vlan_cfg;
+	u8 vf_offset;
+	u8 rsv1[2];
+	__le16 def_vlan_tag1;
+	__le16 def_vlan_tag2;
+	u8 vf_bitmap[8];
+	u8 rsv2[8];
+};
+
+#define HCLGE_REM_TAG1_EN_B		0
+#define HCLGE_REM_TAG2_EN_B		1
+#define HCLGE_SHOW_TAG1_EN_B		2
+#define HCLGE_SHOW_TAG2_EN_B		3
+struct hclge_vport_vtag_rx_cfg_cmd {
+	u8 vport_vlan_cfg;
+	u8 vf_offset;
+	u8 rsv1[6];
+	u8 vf_bitmap[8];
+	u8 rsv2[8];
+};
+
+struct hclge_tx_vlan_type_cfg_cmd {
+	__le16 ot_vlan_type;
+	__le16 in_vlan_type;
+	u8 rsv[20];
+};
+
+struct hclge_rx_vlan_type_cfg_cmd {
+	__le16 ot_fst_vlan_type;
+	__le16 ot_sec_vlan_type;
+	__le16 in_fst_vlan_type;
+	__le16 in_sec_vlan_type;
+	u8 rsv[16];
+};
+
 struct hclge_cfg_com_tqp_queue_cmd {
 	__le16 tqp_id;
 	__le16 stream_id;
@@ -726,6 +811,23 @@ struct hclge_reset_cmd {
 #define HCLGE_NIC_CMQ_DESC_NUM		1024
 #define HCLGE_NIC_CMQ_DESC_NUM_S	3
 
+#define HCLGE_LED_PORT_SPEED_STATE_S	0
+#define HCLGE_LED_PORT_SPEED_STATE_M	GENMASK(5, 0)
+#define HCLGE_LED_ACTIVITY_STATE_S	0
+#define HCLGE_LED_ACTIVITY_STATE_M	GENMASK(1, 0)
+#define HCLGE_LED_LINK_STATE_S		0
+#define HCLGE_LED_LINK_STATE_M		GENMASK(1, 0)
+#define HCLGE_LED_LOCATE_STATE_S	0
+#define HCLGE_LED_LOCATE_STATE_M	GENMASK(1, 0)
+
+struct hclge_set_led_state_cmd {
+	u8 port_speed_led_config;
+	u8 link_led_config;
+	u8 activity_led_config;
+	u8 locate_led_config;
+	u8 rsv[20];
+};
+
 int hclge_cmd_init(struct hclge_dev *hdev);
 static inline void hclge_write_reg(void __iomem *base, u32 reg, u32 value)
 {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index 59ed806a52c3..32bc6f68e297 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -17,10 +17,12 @@
 #include <linux/netdevice.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
-
+#include <linux/if_vlan.h>
+#include <net/rtnetlink.h>
 #include "hclge_cmd.h"
 #include "hclge_dcb.h"
 #include "hclge_main.h"
+#include "hclge_mbx.h"
 #include "hclge_mdio.h"
 #include "hclge_tm.h"
 #include "hnae3.h"
@@ -34,8 +36,10 @@
 static int hclge_set_mta_filter_mode(struct hclge_dev *hdev,
 				     enum hclge_mta_dmac_sel_type mta_mac_sel,
 				     bool enable);
+static int hclge_set_mtu(struct hnae3_handle *handle, int new_mtu);
 static int hclge_init_vlan_config(struct hclge_dev *hdev);
 static int hclge_reset_ae_dev(struct hnae3_ae_dev *ae_dev);
+static int hclge_update_led_status(struct hclge_dev *hdev);
 
 static struct hnae3_ae_algo ae_algo;
 
@@ -278,8 +282,8 @@ static const struct hclge_comm_stats_str g_mac_stats_string[] = {
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_broad_pkt_num)},
 	{"mac_tx_undersize_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_undersize_pkt_num)},
-	{"mac_tx_overrsize_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_overrsize_pkt_num)},
+	{"mac_tx_oversize_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_oversize_pkt_num)},
 	{"mac_tx_64_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_64_oct_pkt_num)},
 	{"mac_tx_65_127_oct_pkt_num",
@@ -292,8 +296,24 @@ static const struct hclge_comm_stats_str g_mac_stats_string[] = {
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_512_1023_oct_pkt_num)},
 	{"mac_tx_1024_1518_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_1024_1518_oct_pkt_num)},
-	{"mac_tx_1519_max_oct_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_1519_max_oct_pkt_num)},
+	{"mac_tx_1519_2047_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_1519_2047_oct_pkt_num)},
+	{"mac_tx_2048_4095_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_2048_4095_oct_pkt_num)},
+	{"mac_tx_4096_8191_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_4096_8191_oct_pkt_num)},
+	{"mac_tx_8192_12287_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_8192_12287_oct_pkt_num)},
+	{"mac_tx_8192_9216_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_8192_9216_oct_pkt_num)},
+	{"mac_tx_9217_12287_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_9217_12287_oct_pkt_num)},
+	{"mac_tx_12288_16383_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_12288_16383_oct_pkt_num)},
+	{"mac_tx_1519_max_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_1519_max_good_oct_pkt_num)},
+	{"mac_tx_1519_max_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_1519_max_bad_oct_pkt_num)},
 	{"mac_rx_total_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_total_pkt_num)},
 	{"mac_rx_total_oct_num",
@@ -314,8 +334,8 @@ static const struct hclge_comm_stats_str g_mac_stats_string[] = {
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_broad_pkt_num)},
 	{"mac_rx_undersize_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_undersize_pkt_num)},
-	{"mac_rx_overrsize_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_overrsize_pkt_num)},
+	{"mac_rx_oversize_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_oversize_pkt_num)},
 	{"mac_rx_64_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_64_oct_pkt_num)},
 	{"mac_rx_65_127_oct_pkt_num",
@@ -328,33 +348,59 @@ static const struct hclge_comm_stats_str g_mac_stats_string[] = {
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_512_1023_oct_pkt_num)},
 	{"mac_rx_1024_1518_oct_pkt_num",
 		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_1024_1518_oct_pkt_num)},
-	{"mac_rx_1519_max_oct_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_1519_max_oct_pkt_num)},
-
-	{"mac_trans_fragment_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_fragment_pkt_num)},
-	{"mac_trans_undermin_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_undermin_pkt_num)},
-	{"mac_trans_jabber_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_jabber_pkt_num)},
-	{"mac_trans_err_all_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_err_all_pkt_num)},
-	{"mac_trans_from_app_good_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_from_app_good_pkt_num)},
-	{"mac_trans_from_app_bad_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_trans_from_app_bad_pkt_num)},
-	{"mac_rcv_fragment_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_fragment_pkt_num)},
-	{"mac_rcv_undermin_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_undermin_pkt_num)},
-	{"mac_rcv_jabber_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_jabber_pkt_num)},
-	{"mac_rcv_fcs_err_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_fcs_err_pkt_num)},
-	{"mac_rcv_send_app_good_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_send_app_good_pkt_num)},
-	{"mac_rcv_send_app_bad_pkt_num",
-		HCLGE_MAC_STATS_FIELD_OFF(mac_rcv_send_app_bad_pkt_num)}
+	{"mac_rx_1519_2047_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_1519_2047_oct_pkt_num)},
+	{"mac_rx_2048_4095_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_2048_4095_oct_pkt_num)},
+	{"mac_rx_4096_8191_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_4096_8191_oct_pkt_num)},
+	{"mac_rx_8192_12287_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_8192_12287_oct_pkt_num)},
+	{"mac_rx_8192_9216_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_8192_9216_oct_pkt_num)},
+	{"mac_rx_9217_12287_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_9217_12287_oct_pkt_num)},
+	{"mac_rx_12288_16383_oct_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_12288_16383_oct_pkt_num)},
+	{"mac_rx_1519_max_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_1519_max_good_oct_pkt_num)},
+	{"mac_rx_1519_max_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_1519_max_bad_oct_pkt_num)},
+
+	{"mac_tx_fragment_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_fragment_pkt_num)},
+	{"mac_tx_undermin_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_undermin_pkt_num)},
+	{"mac_tx_jabber_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_jabber_pkt_num)},
+	{"mac_tx_err_all_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_err_all_pkt_num)},
+	{"mac_tx_from_app_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_from_app_good_pkt_num)},
+	{"mac_tx_from_app_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_tx_from_app_bad_pkt_num)},
+	{"mac_rx_fragment_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_fragment_pkt_num)},
+	{"mac_rx_undermin_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_undermin_pkt_num)},
+	{"mac_rx_jabber_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_jabber_pkt_num)},
+	{"mac_rx_fcs_err_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_fcs_err_pkt_num)},
+	{"mac_rx_send_app_good_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_send_app_good_pkt_num)},
+	{"mac_rx_send_app_bad_pkt_num",
+		HCLGE_MAC_STATS_FIELD_OFF(mac_rx_send_app_bad_pkt_num)}
+};
+
+static const struct hclge_mac_mgr_tbl_entry_cmd hclge_mgr_table[] = {
+	{
+		.flags = HCLGE_MAC_MGR_MASK_VLAN_B,
+		.ethter_type = cpu_to_le16(HCLGE_MAC_ETHERTYPE_LLDP),
+		.mac_addr_hi32 = cpu_to_le32(htonl(0x0180C200)),
+		.mac_addr_lo16 = cpu_to_le16(htons(0x000E)),
+		.i_port_bitmap = 0x1,
+	},
 };
 
 static int hclge_64_bit_update_stats(struct hclge_dev *hdev)
@@ -460,9 +506,41 @@ static int hclge_32_bit_update_stats(struct hclge_dev *hdev)
 	return 0;
 }
 
+static int hclge_mac_get_traffic_stats(struct hclge_dev *hdev)
+{
+	struct hclge_mac_stats *mac_stats = &hdev->hw_stats.mac_stats;
+	struct hclge_desc desc;
+	__le64 *desc_data;
+	int ret;
+
+	/* for fiber port, need to query the total rx/tx packets statstics,
+	 * used for data transferring checking.
+	 */
+	if (hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_FIBER)
+		return 0;
+
+	if (test_bit(HCLGE_STATE_STATISTICS_UPDATING, &hdev->state))
+		return 0;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_STATS_MAC_TRAFFIC, true);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get MAC total pkt stats fail, ret = %d\n", ret);
+
+		return ret;
+	}
+
+	desc_data = (__le64 *)(&desc.data[0]);
+	mac_stats->mac_tx_total_pkt_num += le64_to_cpu(*desc_data++);
+	mac_stats->mac_rx_total_pkt_num += le64_to_cpu(*desc_data);
+
+	return 0;
+}
+
 static int hclge_mac_update_stats(struct hclge_dev *hdev)
 {
-#define HCLGE_MAC_CMD_NUM 17
+#define HCLGE_MAC_CMD_NUM 21
 #define HCLGE_RTN_DATA_NUM 4
 
 	u64 *data = (u64 *)(&hdev->hw_stats.mac_stats);
@@ -524,7 +602,7 @@ static int hclge_tqps_update_stats(struct hnae3_handle *handle)
 			return ret;
 		}
 		tqp->tqp_stats.rcb_rx_ring_pktnum_rcd +=
-			le32_to_cpu(desc[0].data[4]);
+			le32_to_cpu(desc[0].data[1]);
 	}
 
 	for (i = 0; i < kinfo->num_tqps; i++) {
@@ -544,7 +622,7 @@ static int hclge_tqps_update_stats(struct hnae3_handle *handle)
 			return ret;
 		}
 		tqp->tqp_stats.rcb_tx_ring_pktnum_rcd +=
-			le32_to_cpu(desc[0].data[4]);
+			le32_to_cpu(desc[0].data[1]);
 	}
 
 	return 0;
@@ -586,7 +664,7 @@ static u8 *hclge_tqps_get_strings(struct hnae3_handle *handle, u8 *data)
 	for (i = 0; i < kinfo->num_tqps; i++) {
 		struct hclge_tqp *tqp = container_of(handle->kinfo.tqp[i],
 			struct hclge_tqp, q);
-		snprintf(buff, ETH_GSTRING_LEN, "rcb_q%d_tx_pktnum_rcd",
+		snprintf(buff, ETH_GSTRING_LEN, "txq#%d_pktnum_rcd",
 			 tqp->index);
 		buff = buff + ETH_GSTRING_LEN;
 	}
@@ -594,7 +672,7 @@ static u8 *hclge_tqps_get_strings(struct hnae3_handle *handle, u8 *data)
 	for (i = 0; i < kinfo->num_tqps; i++) {
 		struct hclge_tqp *tqp = container_of(kinfo->tqp[i],
 			struct hclge_tqp, q);
-		snprintf(buff, ETH_GSTRING_LEN, "rcb_q%d_rx_pktnum_rcd",
+		snprintf(buff, ETH_GSTRING_LEN, "rxq#%d_pktnum_rcd",
 			 tqp->index);
 		buff = buff + ETH_GSTRING_LEN;
 	}
@@ -642,23 +720,22 @@ static void hclge_update_netstat(struct hclge_hw_stats *hw_stats,
 	net_stats->rx_dropped += hw_stats->all_32_bit_stats.ppp_key_drop_num;
 	net_stats->rx_dropped += hw_stats->all_32_bit_stats.ssu_key_drop_num;
 
-	net_stats->rx_errors = hw_stats->mac_stats.mac_rx_overrsize_pkt_num;
+	net_stats->rx_errors = hw_stats->mac_stats.mac_rx_oversize_pkt_num;
 	net_stats->rx_errors += hw_stats->mac_stats.mac_rx_undersize_pkt_num;
-	net_stats->rx_errors += hw_stats->all_32_bit_stats.igu_rx_err_pkt;
 	net_stats->rx_errors += hw_stats->all_32_bit_stats.igu_rx_no_eof_pkt;
 	net_stats->rx_errors += hw_stats->all_32_bit_stats.igu_rx_no_sof_pkt;
-	net_stats->rx_errors += hw_stats->mac_stats.mac_rcv_fcs_err_pkt_num;
+	net_stats->rx_errors += hw_stats->mac_stats.mac_rx_fcs_err_pkt_num;
 
 	net_stats->multicast = hw_stats->mac_stats.mac_tx_multi_pkt_num;
 	net_stats->multicast += hw_stats->mac_stats.mac_rx_multi_pkt_num;
 
-	net_stats->rx_crc_errors = hw_stats->mac_stats.mac_rcv_fcs_err_pkt_num;
+	net_stats->rx_crc_errors = hw_stats->mac_stats.mac_rx_fcs_err_pkt_num;
 	net_stats->rx_length_errors =
 		hw_stats->mac_stats.mac_rx_undersize_pkt_num;
 	net_stats->rx_length_errors +=
-		hw_stats->mac_stats.mac_rx_overrsize_pkt_num;
+		hw_stats->mac_stats.mac_rx_oversize_pkt_num;
 	net_stats->rx_over_errors =
-		hw_stats->mac_stats.mac_rx_overrsize_pkt_num;
+		hw_stats->mac_stats.mac_rx_oversize_pkt_num;
 }
 
 static void hclge_update_stats_for_all(struct hclge_dev *hdev)
@@ -698,6 +775,9 @@ static void hclge_update_stats(struct hnae3_handle *handle,
 	struct hclge_hw_stats *hw_stats = &hdev->hw_stats;
 	int status;
 
+	if (test_and_set_bit(HCLGE_STATE_STATISTICS_UPDATING, &hdev->state))
+		return;
+
 	status = hclge_mac_update_stats(hdev);
 	if (status)
 		dev_err(&hdev->pdev->dev,
@@ -723,6 +803,8 @@ static void hclge_update_stats(struct hnae3_handle *handle,
 			status);
 
 	hclge_update_netstat(hw_stats, net_stats);
+
+	clear_bit(HCLGE_STATE_STATISTICS_UPDATING, &hdev->state);
 }
 
 static int hclge_get_sset_count(struct hnae3_handle *handle, int stringset)
@@ -981,6 +1063,10 @@ static void hclge_parse_cfg(struct hclge_cfg *cfg, struct hclge_desc *desc)
 	cfg->default_speed = hnae_get_field(__le32_to_cpu(req->param[3]),
 					    HCLGE_CFG_DEFAULT_SPEED_M,
 					    HCLGE_CFG_DEFAULT_SPEED_S);
+	cfg->rss_size_max = hnae_get_field(__le32_to_cpu(req->param[3]),
+					   HCLGE_CFG_RSS_SIZE_M,
+					   HCLGE_CFG_RSS_SIZE_S);
+
 	for (i = 0; i < ETH_ALEN; i++)
 		cfg->mac_addr[i] = (mac_addr_tmp >> (8 * i)) & 0xff;
 
@@ -1058,7 +1144,7 @@ static int hclge_configure(struct hclge_dev *hdev)
 
 	hdev->num_vmdq_vport = cfg.vmdq_vport_num;
 	hdev->base_tqp_pid = 0;
-	hdev->rss_size_max = 1;
+	hdev->rss_size_max = cfg.rss_size_max;
 	hdev->rx_buf_len = cfg.rx_buf_len;
 	ether_addr_copy(hdev->hw.mac.mac_addr, cfg.mac_addr);
 	hdev->hw.mac.media_type = cfg.media_type;
@@ -1095,10 +1181,7 @@ static int hclge_configure(struct hclge_dev *hdev)
 	for (i = 0; i < hdev->tm_info.num_tc; i++)
 		hnae_set_bit(hdev->hw_tc_map, i, 1);
 
-	if (!hdev->num_vmdq_vport && !hdev->num_req_vfs)
-		hdev->tx_sch_mode = HCLGE_FLAG_TC_BASE_SCH_MODE;
-	else
-		hdev->tx_sch_mode = HCLGE_FLAG_VNET_BASE_SCH_MODE;
+	hdev->tx_sch_mode = HCLGE_FLAG_TC_BASE_SCH_MODE;
 
 	return ret;
 }
@@ -2132,28 +2215,6 @@ static int hclge_query_mac_an_speed_dup(struct hclge_dev *hdev, int *speed,
 	return 0;
 }
 
-static int hclge_query_autoneg_result(struct hclge_dev *hdev)
-{
-	struct hclge_mac *mac = &hdev->hw.mac;
-	struct hclge_query_an_speed_dup_cmd *req;
-	struct hclge_desc desc;
-	int ret;
-
-	req = (struct hclge_query_an_speed_dup_cmd *)desc.data;
-
-	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_AN_RESULT, true);
-	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-	if (ret) {
-		dev_err(&hdev->pdev->dev,
-			"autoneg result query cmd failed %d.\n", ret);
-		return ret;
-	}
-
-	mac->autoneg = hnae_get_bit(req->an_syn_dup_speed, HCLGE_QUERY_AN_B);
-
-	return 0;
-}
-
 static int hclge_set_autoneg_en(struct hclge_dev *hdev, bool enable)
 {
 	struct hclge_config_auto_neg_cmd *req;
@@ -2189,15 +2250,45 @@ static int hclge_get_autoneg(struct hnae3_handle *handle)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
+	struct phy_device *phydev = hdev->hw.mac.phydev;
 
-	hclge_query_autoneg_result(hdev);
+	if (phydev)
+		return phydev->autoneg;
 
 	return hdev->hw.mac.autoneg;
 }
 
+static int hclge_set_default_mac_vlan_mask(struct hclge_dev *hdev,
+					   bool mask_vlan,
+					   u8 *mac_mask)
+{
+	struct hclge_mac_vlan_mask_entry_cmd *req;
+	struct hclge_desc desc;
+	int status;
+
+	req = (struct hclge_mac_vlan_mask_entry_cmd *)desc.data;
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MAC_VLAN_MASK_SET, false);
+
+	hnae_set_bit(req->vlan_mask, HCLGE_VLAN_MASK_EN_B,
+		     mask_vlan ? 1 : 0);
+	ether_addr_copy(req->mac_mask, mac_mask);
+
+	status = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Config mac_vlan_mask failed for cmd_send, ret =%d\n",
+			status);
+
+	return status;
+}
+
 static int hclge_mac_init(struct hclge_dev *hdev)
 {
+	struct hnae3_handle *handle = &hdev->vport[0].nic;
+	struct net_device *netdev = handle->kinfo.netdev;
 	struct hclge_mac *mac = &hdev->hw.mac;
+	u8 mac_mask[ETH_ALEN] = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+	int mtu;
 	int ret;
 
 	ret = hclge_cfg_mac_speed_dup(hdev, hdev->hw.mac.speed, HCLGE_MAC_FULL);
@@ -2223,7 +2314,45 @@ static int hclge_mac_init(struct hclge_dev *hdev)
 		return ret;
 	}
 
-	return hclge_cfg_func_mta_filter(hdev, 0, hdev->accept_mta_mc);
+	ret = hclge_cfg_func_mta_filter(hdev, 0, hdev->accept_mta_mc);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"set mta filter mode fail ret=%d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_set_default_mac_vlan_mask(hdev, true, mac_mask);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"set default mac_vlan_mask fail ret=%d\n", ret);
+		return ret;
+	}
+
+	if (netdev)
+		mtu = netdev->mtu;
+	else
+		mtu = ETH_DATA_LEN;
+
+	ret = hclge_set_mtu(handle, mtu);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"set mtu failed ret=%d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void hclge_mbx_task_schedule(struct hclge_dev *hdev)
+{
+	if (!test_and_set_bit(HCLGE_STATE_MBX_SERVICE_SCHED, &hdev->state))
+		schedule_work(&hdev->mbx_service_task);
+}
+
+static void hclge_reset_task_schedule(struct hclge_dev *hdev)
+{
+	if (!test_and_set_bit(HCLGE_STATE_RST_SERVICE_SCHED, &hdev->state))
+		schedule_work(&hdev->rst_service_task);
 }
 
 static void hclge_task_schedule(struct hclge_dev *hdev)
@@ -2350,6 +2479,7 @@ static void hclge_service_timer(struct timer_list *t)
 	struct hclge_dev *hdev = from_timer(hdev, t, service_timer);
 
 	mod_timer(&hdev->service_timer, jiffies + HZ);
+	hdev->hw_stats.stats_timer++;
 	hclge_task_schedule(hdev);
 }
 
@@ -2362,6 +2492,64 @@ static void hclge_service_complete(struct hclge_dev *hdev)
 	clear_bit(HCLGE_STATE_SERVICE_SCHED, &hdev->state);
 }
 
+static u32 hclge_check_event_cause(struct hclge_dev *hdev, u32 *clearval)
+{
+	u32 rst_src_reg;
+	u32 cmdq_src_reg;
+
+	/* fetch the events from their corresponding regs */
+	rst_src_reg = hclge_read_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG);
+	cmdq_src_reg = hclge_read_dev(&hdev->hw, HCLGE_VECTOR0_CMDQ_SRC_REG);
+
+	/* Assumption: If by any chance reset and mailbox events are reported
+	 * together then we will only process reset event in this go and will
+	 * defer the processing of the mailbox events. Since, we would have not
+	 * cleared RX CMDQ event this time we would receive again another
+	 * interrupt from H/W just for the mailbox.
+	 */
+
+	/* check for vector0 reset event sources */
+	if (BIT(HCLGE_VECTOR0_GLOBALRESET_INT_B) & rst_src_reg) {
+		set_bit(HNAE3_GLOBAL_RESET, &hdev->reset_pending);
+		*clearval = BIT(HCLGE_VECTOR0_GLOBALRESET_INT_B);
+		return HCLGE_VECTOR0_EVENT_RST;
+	}
+
+	if (BIT(HCLGE_VECTOR0_CORERESET_INT_B) & rst_src_reg) {
+		set_bit(HNAE3_CORE_RESET, &hdev->reset_pending);
+		*clearval = BIT(HCLGE_VECTOR0_CORERESET_INT_B);
+		return HCLGE_VECTOR0_EVENT_RST;
+	}
+
+	if (BIT(HCLGE_VECTOR0_IMPRESET_INT_B) & rst_src_reg) {
+		set_bit(HNAE3_IMP_RESET, &hdev->reset_pending);
+		*clearval = BIT(HCLGE_VECTOR0_IMPRESET_INT_B);
+		return HCLGE_VECTOR0_EVENT_RST;
+	}
+
+	/* check for vector0 mailbox(=CMDQ RX) event source */
+	if (BIT(HCLGE_VECTOR0_RX_CMDQ_INT_B) & cmdq_src_reg) {
+		cmdq_src_reg &= ~BIT(HCLGE_VECTOR0_RX_CMDQ_INT_B);
+		*clearval = cmdq_src_reg;
+		return HCLGE_VECTOR0_EVENT_MBX;
+	}
+
+	return HCLGE_VECTOR0_EVENT_OTHER;
+}
+
+static void hclge_clear_event_cause(struct hclge_dev *hdev, u32 event_type,
+				    u32 regclr)
+{
+	switch (event_type) {
+	case HCLGE_VECTOR0_EVENT_RST:
+		hclge_write_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG, regclr);
+		break;
+	case HCLGE_VECTOR0_EVENT_MBX:
+		hclge_write_dev(&hdev->hw, HCLGE_VECTOR0_CMDQ_SRC_REG, regclr);
+		break;
+	}
+}
+
 static void hclge_enable_vector(struct hclge_misc_vector *vector, bool enable)
 {
 	writel(enable ? 1 : 0, vector->addr);
@@ -2370,10 +2558,38 @@ static void hclge_enable_vector(struct hclge_misc_vector *vector, bool enable)
 static irqreturn_t hclge_misc_irq_handle(int irq, void *data)
 {
 	struct hclge_dev *hdev = data;
+	u32 event_cause;
+	u32 clearval;
 
 	hclge_enable_vector(&hdev->misc_vector, false);
-	if (!test_and_set_bit(HCLGE_STATE_SERVICE_SCHED, &hdev->state))
-		schedule_work(&hdev->service_task);
+	event_cause = hclge_check_event_cause(hdev, &clearval);
+
+	/* vector 0 interrupt is shared with reset and mailbox source events.*/
+	switch (event_cause) {
+	case HCLGE_VECTOR0_EVENT_RST:
+		hclge_reset_task_schedule(hdev);
+		break;
+	case HCLGE_VECTOR0_EVENT_MBX:
+		/* If we are here then,
+		 * 1. Either we are not handling any mbx task and we are not
+		 *    scheduled as well
+		 *                        OR
+		 * 2. We could be handling a mbx task but nothing more is
+		 *    scheduled.
+		 * In both cases, we should schedule mbx task as there are more
+		 * mbx messages reported by this interrupt.
+		 */
+		hclge_mbx_task_schedule(hdev);
+
+	default:
+		dev_dbg(&hdev->pdev->dev,
+			"received unknown or unhandled event of vector0\n");
+		break;
+	}
+
+	/* we should clear the source of interrupt */
+	hclge_clear_event_cause(hdev, event_cause, clearval);
+	hclge_enable_vector(&hdev->misc_vector, true);
 
 	return IRQ_HANDLED;
 }
@@ -2404,9 +2620,9 @@ static int hclge_misc_irq_init(struct hclge_dev *hdev)
 
 	hclge_get_misc_vector(hdev);
 
-	ret = devm_request_irq(&hdev->pdev->dev,
-			       hdev->misc_vector.vector_irq,
-			       hclge_misc_irq_handle, 0, "hclge_misc", hdev);
+	/* this would be explicitly freed in the end */
+	ret = request_irq(hdev->misc_vector.vector_irq, hclge_misc_irq_handle,
+			  0, "hclge_misc", hdev);
 	if (ret) {
 		hclge_free_vector(hdev, 0);
 		dev_err(&hdev->pdev->dev, "request misc irq(%d) fail\n",
@@ -2416,6 +2632,12 @@ static int hclge_misc_irq_init(struct hclge_dev *hdev)
 	return ret;
 }
 
+static void hclge_misc_irq_uninit(struct hclge_dev *hdev)
+{
+	free_irq(hdev->misc_vector.vector_irq, hdev);
+	hclge_free_vector(hdev, 0);
+}
+
 static int hclge_notify_client(struct hclge_dev *hdev,
 			       enum hnae3_reset_notify_type type)
 {
@@ -2471,12 +2693,6 @@ static int hclge_reset_wait(struct hclge_dev *hdev)
 		cnt++;
 	}
 
-	/* must clear reset status register to
-	 * prevent driver detect reset interrupt again
-	 */
-	reg = hclge_read_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG);
-	hclge_write_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG, reg);
-
 	if (cnt >= HCLGE_RESET_WAIT_CNT) {
 		dev_warn(&hdev->pdev->dev,
 			 "Wait for reset timeout: %d\n", hdev->reset_type);
@@ -2505,12 +2721,12 @@ static int hclge_func_reset_cmd(struct hclge_dev *hdev, int func_id)
 	return ret;
 }
 
-static void hclge_do_reset(struct hclge_dev *hdev, enum hnae3_reset_type type)
+static void hclge_do_reset(struct hclge_dev *hdev)
 {
 	struct pci_dev *pdev = hdev->pdev;
 	u32 val;
 
-	switch (type) {
+	switch (hdev->reset_type) {
 	case HNAE3_GLOBAL_RESET:
 		val = hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG);
 		hnae_set_bit(val, HCLGE_GLOBAL_RESET_BIT, 1);
@@ -2526,30 +2742,62 @@ static void hclge_do_reset(struct hclge_dev *hdev, enum hnae3_reset_type type)
 	case HNAE3_FUNC_RESET:
 		dev_info(&pdev->dev, "PF Reset requested\n");
 		hclge_func_reset_cmd(hdev, 0);
+		/* schedule again to check later */
+		set_bit(HNAE3_FUNC_RESET, &hdev->reset_pending);
+		hclge_reset_task_schedule(hdev);
 		break;
 	default:
 		dev_warn(&pdev->dev,
-			 "Unsupported reset type: %d\n", type);
+			 "Unsupported reset type: %d\n", hdev->reset_type);
 		break;
 	}
 }
 
-static enum hnae3_reset_type hclge_detected_reset_event(struct hclge_dev *hdev)
+static enum hnae3_reset_type hclge_get_reset_level(struct hclge_dev *hdev,
+						   unsigned long *addr)
 {
 	enum hnae3_reset_type rst_level = HNAE3_NONE_RESET;
-	u32 rst_reg_val;
 
-	rst_reg_val = hclge_read_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG);
-	if (BIT(HCLGE_VECTOR0_GLOBALRESET_INT_B) & rst_reg_val)
+	/* return the highest priority reset level amongst all */
+	if (test_bit(HNAE3_GLOBAL_RESET, addr))
 		rst_level = HNAE3_GLOBAL_RESET;
-	else if (BIT(HCLGE_VECTOR0_CORERESET_INT_B) & rst_reg_val)
+	else if (test_bit(HNAE3_CORE_RESET, addr))
 		rst_level = HNAE3_CORE_RESET;
-	else if (BIT(HCLGE_VECTOR0_IMPRESET_INT_B) & rst_reg_val)
+	else if (test_bit(HNAE3_IMP_RESET, addr))
 		rst_level = HNAE3_IMP_RESET;
+	else if (test_bit(HNAE3_FUNC_RESET, addr))
+		rst_level = HNAE3_FUNC_RESET;
+
+	/* now, clear all other resets */
+	clear_bit(HNAE3_GLOBAL_RESET, addr);
+	clear_bit(HNAE3_CORE_RESET, addr);
+	clear_bit(HNAE3_IMP_RESET, addr);
+	clear_bit(HNAE3_FUNC_RESET, addr);
 
 	return rst_level;
 }
 
+static void hclge_reset(struct hclge_dev *hdev)
+{
+	/* perform reset of the stack & ae device for a client */
+
+	hclge_notify_client(hdev, HNAE3_DOWN_CLIENT);
+
+	if (!hclge_reset_wait(hdev)) {
+		rtnl_lock();
+		hclge_notify_client(hdev, HNAE3_UNINIT_CLIENT);
+		hclge_reset_ae_dev(hdev->ae_dev);
+		hclge_notify_client(hdev, HNAE3_INIT_CLIENT);
+		rtnl_unlock();
+	} else {
+		/* schedule again to check pending resets later */
+		set_bit(hdev->reset_type, &hdev->reset_pending);
+		hclge_reset_task_schedule(hdev);
+	}
+
+	hclge_notify_client(hdev, HNAE3_UP_CLIENT);
+}
+
 static void hclge_reset_event(struct hnae3_handle *handle,
 			      enum hnae3_reset_type reset)
 {
@@ -2563,14 +2811,9 @@ static void hclge_reset_event(struct hnae3_handle *handle,
 	case HNAE3_FUNC_RESET:
 	case HNAE3_CORE_RESET:
 	case HNAE3_GLOBAL_RESET:
-		if (test_bit(HCLGE_STATE_RESET_INT, &hdev->state)) {
-			dev_err(&hdev->pdev->dev, "Already in reset state");
-			return;
-		}
-		hdev->reset_type = reset;
-		set_bit(HCLGE_STATE_RESET_INT, &hdev->state);
-		set_bit(HCLGE_STATE_SERVICE_SCHED, &hdev->state);
-		schedule_work(&hdev->service_task);
+		/* request reset & schedule reset task */
+		set_bit(reset, &hdev->reset_request);
+		hclge_reset_task_schedule(hdev);
 		break;
 	default:
 		dev_warn(&hdev->pdev->dev, "Unsupported reset event:%d", reset);
@@ -2580,49 +2823,55 @@ static void hclge_reset_event(struct hnae3_handle *handle,
 
 static void hclge_reset_subtask(struct hclge_dev *hdev)
 {
-	bool do_reset;
+	/* check if there is any ongoing reset in the hardware. This status can
+	 * be checked from reset_pending. If there is then, we need to wait for
+	 * hardware to complete reset.
+	 *    a. If we are able to figure out in reasonable time that hardware
+	 *       has fully resetted then, we can proceed with driver, client
+	 *       reset.
+	 *    b. else, we can come back later to check this status so re-sched
+	 *       now.
+	 */
+	hdev->reset_type = hclge_get_reset_level(hdev, &hdev->reset_pending);
+	if (hdev->reset_type != HNAE3_NONE_RESET)
+		hclge_reset(hdev);
 
-	do_reset = hdev->reset_type != HNAE3_NONE_RESET;
+	/* check if we got any *new* reset requests to be honored */
+	hdev->reset_type = hclge_get_reset_level(hdev, &hdev->reset_request);
+	if (hdev->reset_type != HNAE3_NONE_RESET)
+		hclge_do_reset(hdev);
 
-	/* Reset is detected by interrupt */
-	if (hdev->reset_type == HNAE3_NONE_RESET)
-		hdev->reset_type = hclge_detected_reset_event(hdev);
+	hdev->reset_type = HNAE3_NONE_RESET;
+}
+
+static void hclge_reset_service_task(struct work_struct *work)
+{
+	struct hclge_dev *hdev =
+		container_of(work, struct hclge_dev, rst_service_task);
 
-	if (hdev->reset_type == HNAE3_NONE_RESET)
+	if (test_and_set_bit(HCLGE_STATE_RST_HANDLING, &hdev->state))
 		return;
 
-	switch (hdev->reset_type) {
-	case HNAE3_FUNC_RESET:
-	case HNAE3_CORE_RESET:
-	case HNAE3_GLOBAL_RESET:
-	case HNAE3_IMP_RESET:
-		hclge_notify_client(hdev, HNAE3_DOWN_CLIENT);
+	clear_bit(HCLGE_STATE_RST_SERVICE_SCHED, &hdev->state);
 
-		if (do_reset)
-			hclge_do_reset(hdev, hdev->reset_type);
-		else
-			set_bit(HCLGE_STATE_RESET_INT, &hdev->state);
+	hclge_reset_subtask(hdev);
 
-		if (!hclge_reset_wait(hdev)) {
-			hclge_notify_client(hdev, HNAE3_UNINIT_CLIENT);
-			hclge_reset_ae_dev(hdev->ae_dev);
-			hclge_notify_client(hdev, HNAE3_INIT_CLIENT);
-			clear_bit(HCLGE_STATE_RESET_INT, &hdev->state);
-		}
-		hclge_notify_client(hdev, HNAE3_UP_CLIENT);
-		break;
-	default:
-		dev_err(&hdev->pdev->dev, "Unsupported reset type:%d\n",
-			hdev->reset_type);
-		break;
-	}
-	hdev->reset_type = HNAE3_NONE_RESET;
+	clear_bit(HCLGE_STATE_RST_HANDLING, &hdev->state);
 }
 
-static void hclge_misc_irq_service_task(struct hclge_dev *hdev)
+static void hclge_mailbox_service_task(struct work_struct *work)
 {
-	hclge_reset_subtask(hdev);
-	hclge_enable_vector(&hdev->misc_vector, true);
+	struct hclge_dev *hdev =
+		container_of(work, struct hclge_dev, mbx_service_task);
+
+	if (test_and_set_bit(HCLGE_STATE_MBX_HANDLING, &hdev->state))
+		return;
+
+	clear_bit(HCLGE_STATE_MBX_SERVICE_SCHED, &hdev->state);
+
+	hclge_mbx_handler(hdev);
+
+	clear_bit(HCLGE_STATE_MBX_HANDLING, &hdev->state);
 }
 
 static void hclge_service_task(struct work_struct *work)
@@ -2630,10 +2879,20 @@ static void hclge_service_task(struct work_struct *work)
 	struct hclge_dev *hdev =
 		container_of(work, struct hclge_dev, service_task);
 
-	hclge_misc_irq_service_task(hdev);
+	/* The total rx/tx packets statstics are wanted to be updated
+	 * per second. Both hclge_update_stats_for_all() and
+	 * hclge_mac_get_traffic_stats() can do it.
+	 */
+	if (hdev->hw_stats.stats_timer >= HCLGE_STATS_TIMER_INTERVAL) {
+		hclge_update_stats_for_all(hdev);
+		hdev->hw_stats.stats_timer = 0;
+	} else {
+		hclge_mac_get_traffic_stats(hdev);
+	}
+
 	hclge_update_speed_duplex(hdev);
 	hclge_update_link_status(hdev);
-	hclge_update_stats_for_all(hdev);
+	hclge_update_led_status(hdev);
 	hclge_service_complete(hdev);
 }
 
@@ -3174,49 +3433,53 @@ int hclge_rss_init_hw(struct hclge_dev *hdev)
 	return ret;
 }
 
-int hclge_map_vport_ring_to_vector(struct hclge_vport *vport, int vector_id,
-				   struct hnae3_ring_chain_node *ring_chain)
+int hclge_bind_ring_with_vector(struct hclge_vport *vport,
+				int vector_id, bool en,
+				struct hnae3_ring_chain_node *ring_chain)
 {
 	struct hclge_dev *hdev = vport->back;
-	struct hclge_ctrl_vector_chain_cmd *req;
 	struct hnae3_ring_chain_node *node;
 	struct hclge_desc desc;
-	int ret;
+	struct hclge_ctrl_vector_chain_cmd *req
+		= (struct hclge_ctrl_vector_chain_cmd *)desc.data;
+	enum hclge_cmd_status status;
+	enum hclge_opcode_type op;
+	u16 tqp_type_and_id;
 	int i;
 
-	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_ADD_RING_TO_VECTOR, false);
-
-	req = (struct hclge_ctrl_vector_chain_cmd *)desc.data;
+	op = en ? HCLGE_OPC_ADD_RING_TO_VECTOR : HCLGE_OPC_DEL_RING_TO_VECTOR;
+	hclge_cmd_setup_basic_desc(&desc, op, false);
 	req->int_vector_id = vector_id;
 
 	i = 0;
 	for (node = ring_chain; node; node = node->next) {
-		u16 type_and_id = 0;
-
-		hnae_set_field(type_and_id, HCLGE_INT_TYPE_M, HCLGE_INT_TYPE_S,
+		tqp_type_and_id = le16_to_cpu(req->tqp_type_and_id[i]);
+		hnae_set_field(tqp_type_and_id,  HCLGE_INT_TYPE_M,
+			       HCLGE_INT_TYPE_S,
 			       hnae_get_bit(node->flag, HNAE3_RING_TYPE_B));
-		hnae_set_field(type_and_id, HCLGE_TQP_ID_M, HCLGE_TQP_ID_S,
-			       node->tqp_index);
-		hnae_set_field(type_and_id, HCLGE_INT_GL_IDX_M,
+		hnae_set_field(tqp_type_and_id, HCLGE_TQP_ID_M,
+			       HCLGE_TQP_ID_S, node->tqp_index);
+		hnae_set_field(tqp_type_and_id, HCLGE_INT_GL_IDX_M,
 			       HCLGE_INT_GL_IDX_S,
-			       hnae_get_bit(node->flag, HNAE3_RING_TYPE_B));
-		req->tqp_type_and_id[i] = cpu_to_le16(type_and_id);
-		req->vfid = vport->vport_id;
-
+			       hnae_get_field(node->int_gl_idx,
+					      HNAE3_RING_GL_IDX_M,
+					      HNAE3_RING_GL_IDX_S));
+		req->tqp_type_and_id[i] = cpu_to_le16(tqp_type_and_id);
 		if (++i >= HCLGE_VECTOR_ELEMENTS_PER_CMD) {
 			req->int_cause_num = HCLGE_VECTOR_ELEMENTS_PER_CMD;
+			req->vfid = vport->vport_id;
 
-			ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-			if (ret) {
+			status = hclge_cmd_send(&hdev->hw, &desc, 1);
+			if (status) {
 				dev_err(&hdev->pdev->dev,
 					"Map TQP fail, status is %d.\n",
-					ret);
-				return ret;
+					status);
+				return -EIO;
 			}
 			i = 0;
 
 			hclge_cmd_setup_basic_desc(&desc,
-						   HCLGE_OPC_ADD_RING_TO_VECTOR,
+						   op,
 						   false);
 			req->int_vector_id = vector_id;
 		}
@@ -3224,21 +3487,21 @@ int hclge_map_vport_ring_to_vector(struct hclge_vport *vport, int vector_id,
 
 	if (i > 0) {
 		req->int_cause_num = i;
-
-		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-		if (ret) {
+		req->vfid = vport->vport_id;
+		status = hclge_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
 			dev_err(&hdev->pdev->dev,
-				"Map TQP fail, status is %d.\n", ret);
-			return ret;
+				"Map TQP fail, status is %d.\n", status);
+			return -EIO;
 		}
 	}
 
 	return 0;
 }
 
-static int hclge_map_handle_ring_to_vector(
-		struct hnae3_handle *handle, int vector,
-		struct hnae3_ring_chain_node *ring_chain)
+static int hclge_map_ring_to_vector(struct hnae3_handle *handle,
+				    int vector,
+				    struct hnae3_ring_chain_node *ring_chain)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
@@ -3247,24 +3510,20 @@ static int hclge_map_handle_ring_to_vector(
 	vector_id = hclge_get_vector_index(hdev, vector);
 	if (vector_id < 0) {
 		dev_err(&hdev->pdev->dev,
-			"Get vector index fail. ret =%d\n", vector_id);
+			"Get vector index fail. vector_id =%d\n", vector_id);
 		return vector_id;
 	}
 
-	return hclge_map_vport_ring_to_vector(vport, vector_id, ring_chain);
+	return hclge_bind_ring_with_vector(vport, vector_id, true, ring_chain);
 }
 
-static int hclge_unmap_ring_from_vector(
-	struct hnae3_handle *handle, int vector,
-	struct hnae3_ring_chain_node *ring_chain)
+static int hclge_unmap_ring_frm_vector(struct hnae3_handle *handle,
+				       int vector,
+				       struct hnae3_ring_chain_node *ring_chain)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
-	struct hclge_ctrl_vector_chain_cmd *req;
-	struct hnae3_ring_chain_node *node;
-	struct hclge_desc desc;
-	int i, vector_id;
-	int ret;
+	int vector_id, ret;
 
 	vector_id = hclge_get_vector_index(hdev, vector);
 	if (vector_id < 0) {
@@ -3273,54 +3532,17 @@ static int hclge_unmap_ring_from_vector(
 		return vector_id;
 	}
 
-	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_DEL_RING_TO_VECTOR, false);
-
-	req = (struct hclge_ctrl_vector_chain_cmd *)desc.data;
-	req->int_vector_id = vector_id;
-
-	i = 0;
-	for (node = ring_chain; node; node = node->next) {
-		u16 type_and_id = 0;
-
-		hnae_set_field(type_and_id, HCLGE_INT_TYPE_M, HCLGE_INT_TYPE_S,
-			       hnae_get_bit(node->flag, HNAE3_RING_TYPE_B));
-		hnae_set_field(type_and_id, HCLGE_TQP_ID_M, HCLGE_TQP_ID_S,
-			       node->tqp_index);
-		hnae_set_field(type_and_id, HCLGE_INT_GL_IDX_M,
-			       HCLGE_INT_GL_IDX_S,
-			       hnae_get_bit(node->flag, HNAE3_RING_TYPE_B));
-
-		req->tqp_type_and_id[i] = cpu_to_le16(type_and_id);
-		req->vfid = vport->vport_id;
-
-		if (++i >= HCLGE_VECTOR_ELEMENTS_PER_CMD) {
-			req->int_cause_num = HCLGE_VECTOR_ELEMENTS_PER_CMD;
-
-			ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-			if (ret) {
-				dev_err(&hdev->pdev->dev,
-					"Unmap TQP fail, status is %d.\n",
-					ret);
-				return ret;
-			}
-			i = 0;
-			hclge_cmd_setup_basic_desc(&desc,
-						   HCLGE_OPC_DEL_RING_TO_VECTOR,
-						   false);
-			req->int_vector_id = vector_id;
-		}
+	ret = hclge_bind_ring_with_vector(vport, vector_id, false, ring_chain);
+	if (ret) {
+		dev_err(&handle->pdev->dev,
+			"Unmap ring from vector fail. vectorid=%d, ret =%d\n",
+			vector_id,
+			ret);
+		return ret;
 	}
 
-	if (i > 0) {
-		req->int_cause_num = i;
-
-		ret = hclge_cmd_send(&hdev->hw, &desc, 1);
-		if (ret) {
-			dev_err(&hdev->pdev->dev,
-				"Unmap TQP fail, status is %d.\n", ret);
-			return ret;
-		}
-	}
+	/* Free this MSIX or MSI vector */
+	hclge_free_vector(hdev, vector_id);
 
 	return 0;
 }
@@ -4077,44 +4299,155 @@ int hclge_rm_mc_addr_common(struct hclge_vport *vport,
 	return status;
 }
 
-static void hclge_get_mac_addr(struct hnae3_handle *handle, u8 *p)
+static int hclge_get_mac_ethertype_cmd_status(struct hclge_dev *hdev,
+					      u16 cmdq_resp, u8 resp_code)
 {
-	struct hclge_vport *vport = hclge_get_vport(handle);
-	struct hclge_dev *hdev = vport->back;
-
-	ether_addr_copy(p, hdev->hw.mac.mac_addr);
-}
+#define HCLGE_ETHERTYPE_SUCCESS_ADD		0
+#define HCLGE_ETHERTYPE_ALREADY_ADD		1
+#define HCLGE_ETHERTYPE_MGR_TBL_OVERFLOW	2
+#define HCLGE_ETHERTYPE_KEY_CONFLICT		3
 
-static int hclge_set_mac_addr(struct hnae3_handle *handle, void *p)
-{
-	const unsigned char *new_addr = (const unsigned char *)p;
-	struct hclge_vport *vport = hclge_get_vport(handle);
-	struct hclge_dev *hdev = vport->back;
+	int return_status;
 
-	/* mac addr check */
-	if (is_zero_ether_addr(new_addr) ||
-	    is_broadcast_ether_addr(new_addr) ||
-	    is_multicast_ether_addr(new_addr)) {
+	if (cmdq_resp) {
 		dev_err(&hdev->pdev->dev,
-			"Change uc mac err! invalid mac:%p.\n",
-			 new_addr);
-		return -EINVAL;
+			"cmdq execute failed for get_mac_ethertype_cmd_status, status=%d.\n",
+			cmdq_resp);
+		return -EIO;
 	}
 
-	hclge_rm_uc_addr(handle, hdev->hw.mac.mac_addr);
-
-	if (!hclge_add_uc_addr(handle, new_addr)) {
-		ether_addr_copy(hdev->hw.mac.mac_addr, new_addr);
-		return 0;
+	switch (resp_code) {
+	case HCLGE_ETHERTYPE_SUCCESS_ADD:
+	case HCLGE_ETHERTYPE_ALREADY_ADD:
+		return_status = 0;
+		break;
+	case HCLGE_ETHERTYPE_MGR_TBL_OVERFLOW:
+		dev_err(&hdev->pdev->dev,
+			"add mac ethertype failed for manager table overflow.\n");
+		return_status = -EIO;
+		break;
+	case HCLGE_ETHERTYPE_KEY_CONFLICT:
+		dev_err(&hdev->pdev->dev,
+			"add mac ethertype failed for key conflict.\n");
+		return_status = -EIO;
+		break;
+	default:
+		dev_err(&hdev->pdev->dev,
+			"add mac ethertype failed for undefined, code=%d.\n",
+			resp_code);
+		return_status = -EIO;
 	}
 
-	return -EIO;
+	return return_status;
 }
 
-static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type,
-				      bool filter_en)
+static int hclge_add_mgr_tbl(struct hclge_dev *hdev,
+			     const struct hclge_mac_mgr_tbl_entry_cmd *req)
 {
-	struct hclge_vlan_filter_ctrl_cmd *req;
+	struct hclge_desc desc;
+	u8 resp_code;
+	u16 retval;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MAC_ETHTYPE_ADD, false);
+	memcpy(desc.data, req, sizeof(struct hclge_mac_mgr_tbl_entry_cmd));
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"add mac ethertype failed for cmd_send, ret =%d.\n",
+			ret);
+		return ret;
+	}
+
+	resp_code = (le32_to_cpu(desc.data[0]) >> 8) & 0xff;
+	retval = le16_to_cpu(desc.retval);
+
+	return hclge_get_mac_ethertype_cmd_status(hdev, retval, resp_code);
+}
+
+static int init_mgr_tbl(struct hclge_dev *hdev)
+{
+	int ret;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(hclge_mgr_table); i++) {
+		ret = hclge_add_mgr_tbl(hdev, &hclge_mgr_table[i]);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"add mac ethertype failed, ret =%d.\n",
+				ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void hclge_get_mac_addr(struct hnae3_handle *handle, u8 *p)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	ether_addr_copy(p, hdev->hw.mac.mac_addr);
+}
+
+static int hclge_set_mac_addr(struct hnae3_handle *handle, void *p)
+{
+	const unsigned char *new_addr = (const unsigned char *)p;
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	int ret;
+
+	/* mac addr check */
+	if (is_zero_ether_addr(new_addr) ||
+	    is_broadcast_ether_addr(new_addr) ||
+	    is_multicast_ether_addr(new_addr)) {
+		dev_err(&hdev->pdev->dev,
+			"Change uc mac err! invalid mac:%p.\n",
+			 new_addr);
+		return -EINVAL;
+	}
+
+	ret = hclge_rm_uc_addr(handle, hdev->hw.mac.mac_addr);
+	if (ret)
+		dev_warn(&hdev->pdev->dev,
+			 "remove old uc mac address fail, ret =%d.\n",
+			 ret);
+
+	ret = hclge_add_uc_addr(handle, new_addr);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"add uc mac address fail, ret =%d.\n",
+			ret);
+
+		ret = hclge_add_uc_addr(handle, hdev->hw.mac.mac_addr);
+		if (ret) {
+			dev_err(&hdev->pdev->dev,
+				"restore uc mac address fail, ret =%d.\n",
+				ret);
+		}
+
+		return -EIO;
+	}
+
+	ret = hclge_mac_pause_addr_cfg(hdev, new_addr);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"configure mac pause address fail, ret =%d.\n",
+			ret);
+		return -EIO;
+	}
+
+	ether_addr_copy(hdev->hw.mac.mac_addr, new_addr);
+
+	return 0;
+}
+
+static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type,
+				      bool filter_en)
+{
+	struct hclge_vlan_filter_ctrl_cmd *req;
 	struct hclge_desc desc;
 	int ret;
 
@@ -4134,6 +4467,17 @@ static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type,
 	return 0;
 }
 
+#define HCLGE_FILTER_TYPE_VF		0
+#define HCLGE_FILTER_TYPE_PORT		1
+
+static void hclge_enable_vlan_filter(struct hnae3_handle *handle, bool enable)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, enable);
+}
+
 int hclge_set_vf_vlan_common(struct hclge_dev *hdev, int vfid,
 			     bool is_kill, u16 vlan, u8 qos, __be16 proto)
 {
@@ -4250,43 +4594,204 @@ static int hclge_set_vf_vlan_filter(struct hnae3_handle *handle, int vfid,
 	return hclge_set_vf_vlan_common(hdev, vfid, false, vlan, qos, proto);
 }
 
+static int hclge_set_vlan_tx_offload_cfg(struct hclge_vport *vport)
+{
+	struct hclge_tx_vtag_cfg *vcfg = &vport->txvlan_cfg;
+	struct hclge_vport_vtag_tx_cfg_cmd *req;
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_desc desc;
+	int status;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_VLAN_PORT_TX_CFG, false);
+
+	req = (struct hclge_vport_vtag_tx_cfg_cmd *)desc.data;
+	req->def_vlan_tag1 = cpu_to_le16(vcfg->default_tag1);
+	req->def_vlan_tag2 = cpu_to_le16(vcfg->default_tag2);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_TAG_B,
+		     vcfg->accept_tag ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_ACCEPT_UNTAG_B,
+		     vcfg->accept_untag ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_PORT_INS_TAG1_EN_B,
+		     vcfg->insert_tag1_en ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_PORT_INS_TAG2_EN_B,
+		     vcfg->insert_tag2_en ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_CFG_NIC_ROCE_SEL_B, 0);
+
+	req->vf_offset = vport->vport_id / HCLGE_VF_NUM_PER_CMD;
+	req->vf_bitmap[req->vf_offset] =
+		1 << (vport->vport_id % HCLGE_VF_NUM_PER_BYTE);
+
+	status = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Send port txvlan cfg command fail, ret =%d\n",
+			status);
+
+	return status;
+}
+
+static int hclge_set_vlan_rx_offload_cfg(struct hclge_vport *vport)
+{
+	struct hclge_rx_vtag_cfg *vcfg = &vport->rxvlan_cfg;
+	struct hclge_vport_vtag_rx_cfg_cmd *req;
+	struct hclge_dev *hdev = vport->back;
+	struct hclge_desc desc;
+	int status;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_VLAN_PORT_RX_CFG, false);
+
+	req = (struct hclge_vport_vtag_rx_cfg_cmd *)desc.data;
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_REM_TAG1_EN_B,
+		     vcfg->strip_tag1_en ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_REM_TAG2_EN_B,
+		     vcfg->strip_tag2_en ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_SHOW_TAG1_EN_B,
+		     vcfg->vlan1_vlan_prionly ? 1 : 0);
+	hnae_set_bit(req->vport_vlan_cfg, HCLGE_SHOW_TAG2_EN_B,
+		     vcfg->vlan2_vlan_prionly ? 1 : 0);
+
+	req->vf_offset = vport->vport_id / HCLGE_VF_NUM_PER_CMD;
+	req->vf_bitmap[req->vf_offset] =
+		1 << (vport->vport_id % HCLGE_VF_NUM_PER_BYTE);
+
+	status = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Send port rxvlan cfg command fail, ret =%d\n",
+			status);
+
+	return status;
+}
+
+static int hclge_set_vlan_protocol_type(struct hclge_dev *hdev)
+{
+	struct hclge_rx_vlan_type_cfg_cmd *rx_req;
+	struct hclge_tx_vlan_type_cfg_cmd *tx_req;
+	struct hclge_desc desc;
+	int status;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MAC_VLAN_TYPE_ID, false);
+	rx_req = (struct hclge_rx_vlan_type_cfg_cmd *)desc.data;
+	rx_req->ot_fst_vlan_type =
+		cpu_to_le16(hdev->vlan_type_cfg.rx_ot_fst_vlan_type);
+	rx_req->ot_sec_vlan_type =
+		cpu_to_le16(hdev->vlan_type_cfg.rx_ot_sec_vlan_type);
+	rx_req->in_fst_vlan_type =
+		cpu_to_le16(hdev->vlan_type_cfg.rx_in_fst_vlan_type);
+	rx_req->in_sec_vlan_type =
+		cpu_to_le16(hdev->vlan_type_cfg.rx_in_sec_vlan_type);
+
+	status = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (status) {
+		dev_err(&hdev->pdev->dev,
+			"Send rxvlan protocol type command fail, ret =%d\n",
+			status);
+		return status;
+	}
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_MAC_VLAN_INSERT, false);
+
+	tx_req = (struct hclge_tx_vlan_type_cfg_cmd *)&desc.data;
+	tx_req->ot_vlan_type = cpu_to_le16(hdev->vlan_type_cfg.tx_ot_vlan_type);
+	tx_req->in_vlan_type = cpu_to_le16(hdev->vlan_type_cfg.tx_in_vlan_type);
+
+	status = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Send txvlan protocol type command fail, ret =%d\n",
+			status);
+
+	return status;
+}
+
 static int hclge_init_vlan_config(struct hclge_dev *hdev)
 {
-#define HCLGE_VLAN_TYPE_VF_TABLE   0
-#define HCLGE_VLAN_TYPE_PORT_TABLE 1
+#define HCLGE_DEF_VLAN_TYPE		0x8100
+
 	struct hnae3_handle *handle;
+	struct hclge_vport *vport;
 	int ret;
+	int i;
+
+	ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, true);
+	if (ret)
+		return ret;
 
-	ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_VLAN_TYPE_VF_TABLE,
-					 true);
+	ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_PORT, true);
 	if (ret)
 		return ret;
 
-	ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_VLAN_TYPE_PORT_TABLE,
-					 true);
+	hdev->vlan_type_cfg.rx_in_fst_vlan_type = HCLGE_DEF_VLAN_TYPE;
+	hdev->vlan_type_cfg.rx_in_sec_vlan_type = HCLGE_DEF_VLAN_TYPE;
+	hdev->vlan_type_cfg.rx_ot_fst_vlan_type = HCLGE_DEF_VLAN_TYPE;
+	hdev->vlan_type_cfg.rx_ot_sec_vlan_type = HCLGE_DEF_VLAN_TYPE;
+	hdev->vlan_type_cfg.tx_ot_vlan_type = HCLGE_DEF_VLAN_TYPE;
+	hdev->vlan_type_cfg.tx_in_vlan_type = HCLGE_DEF_VLAN_TYPE;
+
+	ret = hclge_set_vlan_protocol_type(hdev);
 	if (ret)
 		return ret;
 
+	for (i = 0; i < hdev->num_alloc_vport; i++) {
+		vport = &hdev->vport[i];
+		vport->txvlan_cfg.accept_tag = true;
+		vport->txvlan_cfg.accept_untag = true;
+		vport->txvlan_cfg.insert_tag1_en = false;
+		vport->txvlan_cfg.insert_tag2_en = false;
+		vport->txvlan_cfg.default_tag1 = 0;
+		vport->txvlan_cfg.default_tag2 = 0;
+
+		ret = hclge_set_vlan_tx_offload_cfg(vport);
+		if (ret)
+			return ret;
+
+		vport->rxvlan_cfg.strip_tag1_en = false;
+		vport->rxvlan_cfg.strip_tag2_en = true;
+		vport->rxvlan_cfg.vlan1_vlan_prionly = false;
+		vport->rxvlan_cfg.vlan2_vlan_prionly = false;
+
+		ret = hclge_set_vlan_rx_offload_cfg(vport);
+		if (ret)
+			return ret;
+	}
+
 	handle = &hdev->vport[0].nic;
 	return hclge_set_port_vlan_filter(handle, htons(ETH_P_8021Q), 0, false);
 }
 
+static int hclge_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+
+	vport->rxvlan_cfg.strip_tag1_en = false;
+	vport->rxvlan_cfg.strip_tag2_en = enable;
+	vport->rxvlan_cfg.vlan1_vlan_prionly = false;
+	vport->rxvlan_cfg.vlan2_vlan_prionly = false;
+
+	return hclge_set_vlan_rx_offload_cfg(vport);
+}
+
 static int hclge_set_mtu(struct hnae3_handle *handle, int new_mtu)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_config_max_frm_size_cmd *req;
 	struct hclge_dev *hdev = vport->back;
 	struct hclge_desc desc;
+	int max_frm_size;
 	int ret;
 
-	if ((new_mtu < HCLGE_MAC_MIN_MTU) || (new_mtu > HCLGE_MAC_MAX_MTU))
+	max_frm_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
+
+	if (max_frm_size < HCLGE_MAC_MIN_FRAME ||
+	    max_frm_size > HCLGE_MAC_MAX_FRAME)
 		return -EINVAL;
 
-	hdev->mps = new_mtu;
+	max_frm_size = max(max_frm_size, HCLGE_MAC_DEFAULT_FRAME);
+
 	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAX_FRM_SIZE, false);
 
 	req = (struct hclge_config_max_frm_size_cmd *)desc.data;
-	req->max_frm_size = cpu_to_le16(new_mtu);
+	req->max_frm_size = cpu_to_le16(max_frm_size);
 
 	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
 	if (ret) {
@@ -4294,6 +4799,8 @@ static int hclge_set_mtu(struct hnae3_handle *handle, int new_mtu)
 		return ret;
 	}
 
+	hdev->mps = max_frm_size;
+
 	return 0;
 }
 
@@ -4341,7 +4848,7 @@ static int hclge_get_reset_status(struct hclge_dev *hdev, u16 queue_id)
 	return hnae_get_bit(req->ready_to_reset, HCLGE_TQP_RESET_B);
 }
 
-static void hclge_reset_tqp(struct hnae3_handle *handle, u16 queue_id)
+void hclge_reset_tqp(struct hnae3_handle *handle, u16 queue_id)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
@@ -4392,6 +4899,100 @@ static u32 hclge_get_fw_version(struct hnae3_handle *handle)
 	return hdev->fw_version;
 }
 
+static void hclge_get_flowctrl_adv(struct hnae3_handle *handle,
+				   u32 *flowctrl_adv)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct phy_device *phydev = hdev->hw.mac.phydev;
+
+	if (!phydev)
+		return;
+
+	*flowctrl_adv |= (phydev->advertising & ADVERTISED_Pause) |
+			 (phydev->advertising & ADVERTISED_Asym_Pause);
+}
+
+static void hclge_set_flowctrl_adv(struct hclge_dev *hdev, u32 rx_en, u32 tx_en)
+{
+	struct phy_device *phydev = hdev->hw.mac.phydev;
+
+	if (!phydev)
+		return;
+
+	phydev->advertising &= ~(ADVERTISED_Pause | ADVERTISED_Asym_Pause);
+
+	if (rx_en)
+		phydev->advertising |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
+
+	if (tx_en)
+		phydev->advertising ^= ADVERTISED_Asym_Pause;
+}
+
+static int hclge_cfg_pauseparam(struct hclge_dev *hdev, u32 rx_en, u32 tx_en)
+{
+	int ret;
+
+	if (rx_en && tx_en)
+		hdev->fc_mode_last_time = HCLGE_FC_FULL;
+	else if (rx_en && !tx_en)
+		hdev->fc_mode_last_time = HCLGE_FC_RX_PAUSE;
+	else if (!rx_en && tx_en)
+		hdev->fc_mode_last_time = HCLGE_FC_TX_PAUSE;
+	else
+		hdev->fc_mode_last_time = HCLGE_FC_NONE;
+
+	if (hdev->tm_info.fc_mode == HCLGE_FC_PFC)
+		return 0;
+
+	ret = hclge_mac_pause_en_cfg(hdev, tx_en, rx_en);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "configure pauseparam error, ret = %d.\n",
+			ret);
+		return ret;
+	}
+
+	hdev->tm_info.fc_mode = hdev->fc_mode_last_time;
+
+	return 0;
+}
+
+int hclge_cfg_flowctrl(struct hclge_dev *hdev)
+{
+	struct phy_device *phydev = hdev->hw.mac.phydev;
+	u16 remote_advertising = 0;
+	u16 local_advertising = 0;
+	u32 rx_pause, tx_pause;
+	u8 flowctl;
+
+	if (!phydev->link || !phydev->autoneg)
+		return 0;
+
+	if (phydev->advertising & ADVERTISED_Pause)
+		local_advertising = ADVERTISE_PAUSE_CAP;
+
+	if (phydev->advertising & ADVERTISED_Asym_Pause)
+		local_advertising |= ADVERTISE_PAUSE_ASYM;
+
+	if (phydev->pause)
+		remote_advertising = LPA_PAUSE_CAP;
+
+	if (phydev->asym_pause)
+		remote_advertising |= LPA_PAUSE_ASYM;
+
+	flowctl = mii_resolve_flowctrl_fdx(local_advertising,
+					   remote_advertising);
+	tx_pause = flowctl & FLOW_CTRL_TX;
+	rx_pause = flowctl & FLOW_CTRL_RX;
+
+	if (phydev->duplex == HCLGE_MAC_HALF) {
+		tx_pause = 0;
+		rx_pause = 0;
+	}
+
+	return hclge_cfg_pauseparam(hdev, rx_pause, tx_pause);
+}
+
 static void hclge_get_pauseparam(struct hnae3_handle *handle, u32 *auto_neg,
 				 u32 *rx_en, u32 *tx_en)
 {
@@ -4421,6 +5022,41 @@ static void hclge_get_pauseparam(struct hnae3_handle *handle, u32 *auto_neg,
 	}
 }
 
+static int hclge_set_pauseparam(struct hnae3_handle *handle, u32 auto_neg,
+				u32 rx_en, u32 tx_en)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct phy_device *phydev = hdev->hw.mac.phydev;
+	u32 fc_autoneg;
+
+	/* Only support flow control negotiation for netdev with
+	 * phy attached for now.
+	 */
+	if (!phydev)
+		return -EOPNOTSUPP;
+
+	fc_autoneg = hclge_get_autoneg(handle);
+	if (auto_neg != fc_autoneg) {
+		dev_info(&hdev->pdev->dev,
+			 "To change autoneg please use: ethtool -s <dev> autoneg <on|off>\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (hdev->tm_info.fc_mode == HCLGE_FC_PFC) {
+		dev_info(&hdev->pdev->dev,
+			 "Priority flow control enabled. Cannot set link flow control.\n");
+		return -EOPNOTSUPP;
+	}
+
+	hclge_set_flowctrl_adv(hdev, rx_en, tx_en);
+
+	if (!fc_autoneg)
+		return hclge_cfg_pauseparam(hdev, rx_en, tx_en);
+
+	return phy_start_aneg(phydev);
+}
+
 static void hclge_get_ksettings_an_result(struct hnae3_handle *handle,
 					  u8 *auto_neg, u32 *speed, u8 *duplex)
 {
@@ -4661,6 +5297,8 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 	hdev->pdev = pdev;
 	hdev->ae_dev = ae_dev;
 	hdev->reset_type = HNAE3_NONE_RESET;
+	hdev->reset_request = 0;
+	hdev->reset_pending = 0;
 	ae_dev->priv = hdev;
 
 	ret = hclge_pci_init(hdev);
@@ -4768,16 +5406,28 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 		return ret;
 	}
 
+	ret = init_mgr_tbl(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "manager table init fail, ret =%d\n", ret);
+		return ret;
+	}
+
 	hclge_dcb_ops_set(hdev);
 
 	timer_setup(&hdev->service_timer, hclge_service_timer, 0);
 	INIT_WORK(&hdev->service_task, hclge_service_task);
+	INIT_WORK(&hdev->rst_service_task, hclge_reset_service_task);
+	INIT_WORK(&hdev->mbx_service_task, hclge_mailbox_service_task);
 
 	/* Enable MISC vector(vector0) */
 	hclge_enable_vector(&hdev->misc_vector, true);
 
 	set_bit(HCLGE_STATE_SERVICE_INITED, &hdev->state);
 	set_bit(HCLGE_STATE_DOWN, &hdev->state);
+	clear_bit(HCLGE_STATE_RST_SERVICE_SCHED, &hdev->state);
+	clear_bit(HCLGE_STATE_RST_HANDLING, &hdev->state);
+	clear_bit(HCLGE_STATE_MBX_SERVICE_SCHED, &hdev->state);
+	clear_bit(HCLGE_STATE_MBX_HANDLING, &hdev->state);
 
 	pr_info("%s driver initialization finished.\n", HCLGE_DRIVER_NAME);
 	return 0;
@@ -4889,25 +5539,471 @@ static void hclge_uninit_ae_dev(struct hnae3_ae_dev *ae_dev)
 		del_timer_sync(&hdev->service_timer);
 	if (hdev->service_task.func)
 		cancel_work_sync(&hdev->service_task);
+	if (hdev->rst_service_task.func)
+		cancel_work_sync(&hdev->rst_service_task);
+	if (hdev->mbx_service_task.func)
+		cancel_work_sync(&hdev->mbx_service_task);
 
 	if (mac->phydev)
 		mdiobus_unregister(mac->mdio_bus);
 
 	/* Disable MISC vector(vector0) */
 	hclge_enable_vector(&hdev->misc_vector, false);
-	hclge_free_vector(hdev, 0);
 	hclge_destroy_cmd_queue(&hdev->hw);
+	hclge_misc_irq_uninit(hdev);
 	hclge_pci_uninit(hdev);
 	ae_dev->priv = NULL;
 }
 
+static u32 hclge_get_max_channels(struct hnae3_handle *handle)
+{
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+
+	return min_t(u32, hdev->rss_size_max * kinfo->num_tc, hdev->num_tqps);
+}
+
+static void hclge_get_channels(struct hnae3_handle *handle,
+			       struct ethtool_channels *ch)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+
+	ch->max_combined = hclge_get_max_channels(handle);
+	ch->other_count = 1;
+	ch->max_other = 1;
+	ch->combined_count = vport->alloc_tqps;
+}
+
+static void hclge_get_tqps_and_rss_info(struct hnae3_handle *handle,
+					u16 *free_tqps, u16 *max_rss_size)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	u16 temp_tqps = 0;
+	int i;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		if (!hdev->htqp[i].alloced)
+			temp_tqps++;
+	}
+	*free_tqps = temp_tqps;
+	*max_rss_size = hdev->rss_size_max;
+}
+
+static void hclge_release_tqp(struct hclge_vport *vport)
+{
+	struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
+	struct hclge_dev *hdev = vport->back;
+	int i;
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		struct hclge_tqp *tqp =
+			container_of(kinfo->tqp[i], struct hclge_tqp, q);
+
+		tqp->q.handle = NULL;
+		tqp->q.tqp_index = 0;
+		tqp->alloced = false;
+	}
+
+	devm_kfree(&hdev->pdev->dev, kinfo->tqp);
+	kinfo->tqp = NULL;
+}
+
+static int hclge_set_channels(struct hnae3_handle *handle, u32 new_tqps_num)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
+	struct hclge_dev *hdev = vport->back;
+	int cur_rss_size = kinfo->rss_size;
+	int cur_tqps = kinfo->num_tqps;
+	u16 tc_offset[HCLGE_MAX_TC_NUM];
+	u16 tc_valid[HCLGE_MAX_TC_NUM];
+	u16 tc_size[HCLGE_MAX_TC_NUM];
+	u16 roundup_size;
+	u32 *rss_indir;
+	int ret, i;
+
+	hclge_release_tqp(vport);
+
+	ret = hclge_knic_setup(vport, new_tqps_num);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "setup nic fail, ret =%d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_map_tqp_to_vport(hdev, vport);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "map vport tqp fail, ret =%d\n", ret);
+		return ret;
+	}
+
+	ret = hclge_tm_schd_init(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "tm schd init fail, ret =%d\n", ret);
+		return ret;
+	}
+
+	roundup_size = roundup_pow_of_two(kinfo->rss_size);
+	roundup_size = ilog2(roundup_size);
+	/* Set the RSS TC mode according to the new RSS size */
+	for (i = 0; i < HCLGE_MAX_TC_NUM; i++) {
+		tc_valid[i] = 0;
+
+		if (!(hdev->hw_tc_map & BIT(i)))
+			continue;
+
+		tc_valid[i] = 1;
+		tc_size[i] = roundup_size;
+		tc_offset[i] = kinfo->rss_size * i;
+	}
+	ret = hclge_set_rss_tc_mode(hdev, tc_valid, tc_size, tc_offset);
+	if (ret)
+		return ret;
+
+	/* Reinitializes the rss indirect table according to the new RSS size */
+	rss_indir = kcalloc(HCLGE_RSS_IND_TBL_SIZE, sizeof(u32), GFP_KERNEL);
+	if (!rss_indir)
+		return -ENOMEM;
+
+	for (i = 0; i < HCLGE_RSS_IND_TBL_SIZE; i++)
+		rss_indir[i] = i % kinfo->rss_size;
+
+	ret = hclge_set_rss(handle, rss_indir, NULL, 0);
+	if (ret)
+		dev_err(&hdev->pdev->dev, "set rss indir table fail, ret=%d\n",
+			ret);
+
+	kfree(rss_indir);
+
+	if (!ret)
+		dev_info(&hdev->pdev->dev,
+			 "Channels changed, rss_size from %d to %d, tqps from %d to %d",
+			 cur_rss_size, kinfo->rss_size,
+			 cur_tqps, kinfo->rss_size * kinfo->num_tc);
+
+	return ret;
+}
+
+static int hclge_get_regs_num(struct hclge_dev *hdev, u32 *regs_num_32_bit,
+			      u32 *regs_num_64_bit)
+{
+	struct hclge_desc desc;
+	u32 total_num;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_QUERY_REG_NUM, true);
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Query register number cmd failed, ret = %d.\n", ret);
+		return ret;
+	}
+
+	*regs_num_32_bit = le32_to_cpu(desc.data[0]);
+	*regs_num_64_bit = le32_to_cpu(desc.data[1]);
+
+	total_num = *regs_num_32_bit + *regs_num_64_bit;
+	if (!total_num)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int hclge_get_32_bit_regs(struct hclge_dev *hdev, u32 regs_num,
+				 void *data)
+{
+#define HCLGE_32_BIT_REG_RTN_DATANUM 8
+
+	struct hclge_desc *desc;
+	u32 *reg_val = data;
+	__le32 *desc_data;
+	int cmd_num;
+	int i, k, n;
+	int ret;
+
+	if (regs_num == 0)
+		return 0;
+
+	cmd_num = DIV_ROUND_UP(regs_num + 2, HCLGE_32_BIT_REG_RTN_DATANUM);
+	desc = kcalloc(cmd_num, sizeof(struct hclge_desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_QUERY_32_BIT_REG, true);
+	ret = hclge_cmd_send(&hdev->hw, desc, cmd_num);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Query 32 bit register cmd failed, ret = %d.\n", ret);
+		kfree(desc);
+		return ret;
+	}
+
+	for (i = 0; i < cmd_num; i++) {
+		if (i == 0) {
+			desc_data = (__le32 *)(&desc[i].data[0]);
+			n = HCLGE_32_BIT_REG_RTN_DATANUM - 2;
+		} else {
+			desc_data = (__le32 *)(&desc[i]);
+			n = HCLGE_32_BIT_REG_RTN_DATANUM;
+		}
+		for (k = 0; k < n; k++) {
+			*reg_val++ = le32_to_cpu(*desc_data++);
+
+			regs_num--;
+			if (!regs_num)
+				break;
+		}
+	}
+
+	kfree(desc);
+	return 0;
+}
+
+static int hclge_get_64_bit_regs(struct hclge_dev *hdev, u32 regs_num,
+				 void *data)
+{
+#define HCLGE_64_BIT_REG_RTN_DATANUM 4
+
+	struct hclge_desc *desc;
+	u64 *reg_val = data;
+	__le64 *desc_data;
+	int cmd_num;
+	int i, k, n;
+	int ret;
+
+	if (regs_num == 0)
+		return 0;
+
+	cmd_num = DIV_ROUND_UP(regs_num + 1, HCLGE_64_BIT_REG_RTN_DATANUM);
+	desc = kcalloc(cmd_num, sizeof(struct hclge_desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_QUERY_64_BIT_REG, true);
+	ret = hclge_cmd_send(&hdev->hw, desc, cmd_num);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Query 64 bit register cmd failed, ret = %d.\n", ret);
+		kfree(desc);
+		return ret;
+	}
+
+	for (i = 0; i < cmd_num; i++) {
+		if (i == 0) {
+			desc_data = (__le64 *)(&desc[i].data[0]);
+			n = HCLGE_64_BIT_REG_RTN_DATANUM - 1;
+		} else {
+			desc_data = (__le64 *)(&desc[i]);
+			n = HCLGE_64_BIT_REG_RTN_DATANUM;
+		}
+		for (k = 0; k < n; k++) {
+			*reg_val++ = le64_to_cpu(*desc_data++);
+
+			regs_num--;
+			if (!regs_num)
+				break;
+		}
+	}
+
+	kfree(desc);
+	return 0;
+}
+
+static int hclge_get_regs_len(struct hnae3_handle *handle)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	u32 regs_num_32_bit, regs_num_64_bit;
+	int ret;
+
+	ret = hclge_get_regs_num(hdev, &regs_num_32_bit, &regs_num_64_bit);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get register number failed, ret = %d.\n", ret);
+		return -EOPNOTSUPP;
+	}
+
+	return regs_num_32_bit * sizeof(u32) + regs_num_64_bit * sizeof(u64);
+}
+
+static void hclge_get_regs(struct hnae3_handle *handle, u32 *version,
+			   void *data)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	u32 regs_num_32_bit, regs_num_64_bit;
+	int ret;
+
+	*version = hdev->fw_version;
+
+	ret = hclge_get_regs_num(hdev, &regs_num_32_bit, &regs_num_64_bit);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get register number failed, ret = %d.\n", ret);
+		return;
+	}
+
+	ret = hclge_get_32_bit_regs(hdev, regs_num_32_bit, data);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"Get 32 bit register failed, ret = %d.\n", ret);
+		return;
+	}
+
+	data = (u32 *)data + regs_num_32_bit;
+	ret = hclge_get_64_bit_regs(hdev, regs_num_64_bit,
+				    data);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"Get 64 bit register failed, ret = %d.\n", ret);
+}
+
+static int hclge_set_led_status_sfp(struct hclge_dev *hdev, u8 speed_led_status,
+				    u8 act_led_status, u8 link_led_status,
+				    u8 locate_led_status)
+{
+	struct hclge_set_led_state_cmd *req;
+	struct hclge_desc desc;
+	int ret;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_LED_STATUS_CFG, false);
+
+	req = (struct hclge_set_led_state_cmd *)desc.data;
+	hnae_set_field(req->port_speed_led_config, HCLGE_LED_PORT_SPEED_STATE_M,
+		       HCLGE_LED_PORT_SPEED_STATE_S, speed_led_status);
+	hnae_set_field(req->link_led_config, HCLGE_LED_ACTIVITY_STATE_M,
+		       HCLGE_LED_ACTIVITY_STATE_S, act_led_status);
+	hnae_set_field(req->activity_led_config, HCLGE_LED_LINK_STATE_M,
+		       HCLGE_LED_LINK_STATE_S, link_led_status);
+	hnae_set_field(req->locate_led_config, HCLGE_LED_LOCATE_STATE_M,
+		       HCLGE_LED_LOCATE_STATE_S, locate_led_status);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"Send set led state cmd error, ret =%d\n", ret);
+
+	return ret;
+}
+
+enum hclge_led_status {
+	HCLGE_LED_OFF,
+	HCLGE_LED_ON,
+	HCLGE_LED_NO_CHANGE = 0xFF,
+};
+
+static int hclge_set_led_id(struct hnae3_handle *handle,
+			    enum ethtool_phys_id_state status)
+{
+#define BLINK_FREQUENCY		2
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	struct phy_device *phydev = hdev->hw.mac.phydev;
+	int ret = 0;
+
+	if (phydev || hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_FIBER)
+		return -EOPNOTSUPP;
+
+	switch (status) {
+	case ETHTOOL_ID_ACTIVE:
+		ret = hclge_set_led_status_sfp(hdev,
+					       HCLGE_LED_NO_CHANGE,
+					       HCLGE_LED_NO_CHANGE,
+					       HCLGE_LED_NO_CHANGE,
+					       HCLGE_LED_ON);
+		break;
+	case ETHTOOL_ID_INACTIVE:
+		ret = hclge_set_led_status_sfp(hdev,
+					       HCLGE_LED_NO_CHANGE,
+					       HCLGE_LED_NO_CHANGE,
+					       HCLGE_LED_NO_CHANGE,
+					       HCLGE_LED_OFF);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+enum hclge_led_port_speed {
+	HCLGE_SPEED_LED_FOR_1G,
+	HCLGE_SPEED_LED_FOR_10G,
+	HCLGE_SPEED_LED_FOR_25G,
+	HCLGE_SPEED_LED_FOR_40G,
+	HCLGE_SPEED_LED_FOR_50G,
+	HCLGE_SPEED_LED_FOR_100G,
+};
+
+static u8 hclge_led_get_speed_status(u32 speed)
+{
+	u8 speed_led;
+
+	switch (speed) {
+	case HCLGE_MAC_SPEED_1G:
+		speed_led = HCLGE_SPEED_LED_FOR_1G;
+		break;
+	case HCLGE_MAC_SPEED_10G:
+		speed_led = HCLGE_SPEED_LED_FOR_10G;
+		break;
+	case HCLGE_MAC_SPEED_25G:
+		speed_led = HCLGE_SPEED_LED_FOR_25G;
+		break;
+	case HCLGE_MAC_SPEED_40G:
+		speed_led = HCLGE_SPEED_LED_FOR_40G;
+		break;
+	case HCLGE_MAC_SPEED_50G:
+		speed_led = HCLGE_SPEED_LED_FOR_50G;
+		break;
+	case HCLGE_MAC_SPEED_100G:
+		speed_led = HCLGE_SPEED_LED_FOR_100G;
+		break;
+	default:
+		speed_led = HCLGE_LED_NO_CHANGE;
+	}
+
+	return speed_led;
+}
+
+static int hclge_update_led_status(struct hclge_dev *hdev)
+{
+	u8 port_speed_status, link_status, activity_status;
+	u64 rx_pkts, tx_pkts;
+
+	if (hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_FIBER)
+		return 0;
+
+	port_speed_status = hclge_led_get_speed_status(hdev->hw.mac.speed);
+
+	rx_pkts = hdev->hw_stats.mac_stats.mac_rx_total_pkt_num;
+	tx_pkts = hdev->hw_stats.mac_stats.mac_tx_total_pkt_num;
+	if (rx_pkts != hdev->rx_pkts_for_led ||
+	    tx_pkts != hdev->tx_pkts_for_led)
+		activity_status = HCLGE_LED_ON;
+	else
+		activity_status = HCLGE_LED_OFF;
+	hdev->rx_pkts_for_led = rx_pkts;
+	hdev->tx_pkts_for_led = tx_pkts;
+
+	if (hdev->hw.mac.link)
+		link_status = HCLGE_LED_ON;
+	else
+		link_status = HCLGE_LED_OFF;
+
+	return hclge_set_led_status_sfp(hdev, port_speed_status,
+					activity_status, link_status,
+					HCLGE_LED_NO_CHANGE);
+}
+
 static const struct hnae3_ae_ops hclge_ops = {
 	.init_ae_dev = hclge_init_ae_dev,
 	.uninit_ae_dev = hclge_uninit_ae_dev,
 	.init_client_instance = hclge_init_client_instance,
 	.uninit_client_instance = hclge_uninit_client_instance,
-	.map_ring_to_vector = hclge_map_handle_ring_to_vector,
-	.unmap_ring_from_vector = hclge_unmap_ring_from_vector,
+	.map_ring_to_vector = hclge_map_ring_to_vector,
+	.unmap_ring_from_vector = hclge_unmap_ring_frm_vector,
 	.get_vector = hclge_get_vector,
 	.set_promisc_mode = hclge_set_promisc_mode,
 	.set_loopback = hclge_set_loopback,
@@ -4934,6 +6030,7 @@ static const struct hnae3_ae_ops hclge_ops = {
 	.set_autoneg = hclge_set_autoneg,
 	.get_autoneg = hclge_get_autoneg,
 	.get_pauseparam = hclge_get_pauseparam,
+	.set_pauseparam = hclge_set_pauseparam,
 	.set_mtu = hclge_set_mtu,
 	.reset_queue = hclge_reset_tqp,
 	.get_stats = hclge_get_stats,
@@ -4942,9 +6039,18 @@ static const struct hnae3_ae_ops hclge_ops = {
 	.get_sset_count = hclge_get_sset_count,
 	.get_fw_version = hclge_get_fw_version,
 	.get_mdix_mode = hclge_get_mdix_mode,
+	.enable_vlan_filter = hclge_enable_vlan_filter,
 	.set_vlan_filter = hclge_set_port_vlan_filter,
 	.set_vf_vlan_filter = hclge_set_vf_vlan_filter,
+	.enable_hw_strip_rxvtag = hclge_en_hw_strip_rxvtag,
 	.reset_event = hclge_reset_event,
+	.get_tqps_and_rss_info = hclge_get_tqps_and_rss_info,
+	.set_channels = hclge_set_channels,
+	.get_channels = hclge_get_channels,
+	.get_flowctrl_adv = hclge_get_flowctrl_adv,
+	.get_regs_len = hclge_get_regs_len,
+	.get_regs = hclge_get_regs,
+	.set_led_id = hclge_set_led_id,
 };
 
 static struct hnae3_ae_algo ae_algo = {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
index 7027814ea5d7..d99a76a9557c 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
@@ -79,6 +79,10 @@
 #define HCLGE_PHY_MDIX_STATUS_B	(6)
 #define HCLGE_PHY_SPEED_DUP_RESOLVE_B	(11)
 
+/* Factor used to calculate offset and bitmap of VF num */
+#define HCLGE_VF_NUM_PER_CMD           64
+#define HCLGE_VF_NUM_PER_BYTE          8
+
 /* Reset related Registers */
 #define HCLGE_MISC_RESET_STS_REG	0x20700
 #define HCLGE_GLOBAL_RESET_REG		0x20A00
@@ -92,6 +96,16 @@
 #define HCLGE_VECTOR0_CORERESET_INT_B	6
 #define HCLGE_VECTOR0_IMPRESET_INT_B	7
 
+/* Vector0 interrupt CMDQ event source register(RW) */
+#define HCLGE_VECTOR0_CMDQ_SRC_REG	0x27100
+/* CMDQ register bits for RX event(=MBX event) */
+#define HCLGE_VECTOR0_RX_CMDQ_INT_B	1
+
+#define HCLGE_MAC_DEFAULT_FRAME \
+	(ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN + ETH_DATA_LEN)
+#define HCLGE_MAC_MIN_FRAME		64
+#define HCLGE_MAC_MAX_FRAME		9728
+
 enum HCLGE_DEV_STATE {
 	HCLGE_STATE_REINITING,
 	HCLGE_STATE_DOWN,
@@ -99,12 +113,20 @@ enum HCLGE_DEV_STATE {
 	HCLGE_STATE_REMOVING,
 	HCLGE_STATE_SERVICE_INITED,
 	HCLGE_STATE_SERVICE_SCHED,
+	HCLGE_STATE_RST_SERVICE_SCHED,
+	HCLGE_STATE_RST_HANDLING,
+	HCLGE_STATE_MBX_SERVICE_SCHED,
 	HCLGE_STATE_MBX_HANDLING,
-	HCLGE_STATE_MBX_IRQ,
-	HCLGE_STATE_RESET_INT,
+	HCLGE_STATE_STATISTICS_UPDATING,
 	HCLGE_STATE_MAX
 };
 
+enum hclge_evt_cause {
+	HCLGE_VECTOR0_EVENT_RST,
+	HCLGE_VECTOR0_EVENT_MBX,
+	HCLGE_VECTOR0_EVENT_OTHER,
+};
+
 #define HCLGE_MPF_ENBALE 1
 struct hclge_caps {
 	u16 num_tqp;
@@ -208,6 +230,7 @@ struct hclge_cfg {
 	u8 tc_num;
 	u16 tqp_desc_num;
 	u16 rx_buf_len;
+	u16 rss_size_max;
 	u8 phy_addr;
 	u8 media_type;
 	u8 mac_addr[ETH_ALEN];
@@ -364,14 +387,23 @@ struct hclge_mac_stats {
 	u64 mac_tx_multi_pkt_num;
 	u64 mac_tx_broad_pkt_num;
 	u64 mac_tx_undersize_pkt_num;
-	u64 mac_tx_overrsize_pkt_num;
+	u64 mac_tx_oversize_pkt_num;
 	u64 mac_tx_64_oct_pkt_num;
 	u64 mac_tx_65_127_oct_pkt_num;
 	u64 mac_tx_128_255_oct_pkt_num;
 	u64 mac_tx_256_511_oct_pkt_num;
 	u64 mac_tx_512_1023_oct_pkt_num;
 	u64 mac_tx_1024_1518_oct_pkt_num;
-	u64 mac_tx_1519_max_oct_pkt_num;
+	u64 mac_tx_1519_2047_oct_pkt_num;
+	u64 mac_tx_2048_4095_oct_pkt_num;
+	u64 mac_tx_4096_8191_oct_pkt_num;
+	u64 mac_tx_8192_12287_oct_pkt_num; /* valid for GE MAC only */
+	u64 mac_tx_8192_9216_oct_pkt_num; /* valid for LGE & CGE MAC only */
+	u64 mac_tx_9217_12287_oct_pkt_num; /* valid for LGE & CGE MAC */
+	u64 mac_tx_12288_16383_oct_pkt_num;
+	u64 mac_tx_1519_max_good_oct_pkt_num;
+	u64 mac_tx_1519_max_bad_oct_pkt_num;
+
 	u64 mac_rx_total_pkt_num;
 	u64 mac_rx_total_oct_num;
 	u64 mac_rx_good_pkt_num;
@@ -382,33 +414,52 @@ struct hclge_mac_stats {
 	u64 mac_rx_multi_pkt_num;
 	u64 mac_rx_broad_pkt_num;
 	u64 mac_rx_undersize_pkt_num;
-	u64 mac_rx_overrsize_pkt_num;
+	u64 mac_rx_oversize_pkt_num;
 	u64 mac_rx_64_oct_pkt_num;
 	u64 mac_rx_65_127_oct_pkt_num;
 	u64 mac_rx_128_255_oct_pkt_num;
 	u64 mac_rx_256_511_oct_pkt_num;
 	u64 mac_rx_512_1023_oct_pkt_num;
 	u64 mac_rx_1024_1518_oct_pkt_num;
-	u64 mac_rx_1519_max_oct_pkt_num;
-
-	u64 mac_trans_fragment_pkt_num;
-	u64 mac_trans_undermin_pkt_num;
-	u64 mac_trans_jabber_pkt_num;
-	u64 mac_trans_err_all_pkt_num;
-	u64 mac_trans_from_app_good_pkt_num;
-	u64 mac_trans_from_app_bad_pkt_num;
-	u64 mac_rcv_fragment_pkt_num;
-	u64 mac_rcv_undermin_pkt_num;
-	u64 mac_rcv_jabber_pkt_num;
-	u64 mac_rcv_fcs_err_pkt_num;
-	u64 mac_rcv_send_app_good_pkt_num;
-	u64 mac_rcv_send_app_bad_pkt_num;
+	u64 mac_rx_1519_2047_oct_pkt_num;
+	u64 mac_rx_2048_4095_oct_pkt_num;
+	u64 mac_rx_4096_8191_oct_pkt_num;
+	u64 mac_rx_8192_12287_oct_pkt_num;/* valid for GE MAC only */
+	u64 mac_rx_8192_9216_oct_pkt_num; /* valid for LGE & CGE MAC only */
+	u64 mac_rx_9217_12287_oct_pkt_num; /* valid for LGE & CGE MAC only */
+	u64 mac_rx_12288_16383_oct_pkt_num;
+	u64 mac_rx_1519_max_good_oct_pkt_num;
+	u64 mac_rx_1519_max_bad_oct_pkt_num;
+
+	u64 mac_tx_fragment_pkt_num;
+	u64 mac_tx_undermin_pkt_num;
+	u64 mac_tx_jabber_pkt_num;
+	u64 mac_tx_err_all_pkt_num;
+	u64 mac_tx_from_app_good_pkt_num;
+	u64 mac_tx_from_app_bad_pkt_num;
+	u64 mac_rx_fragment_pkt_num;
+	u64 mac_rx_undermin_pkt_num;
+	u64 mac_rx_jabber_pkt_num;
+	u64 mac_rx_fcs_err_pkt_num;
+	u64 mac_rx_send_app_good_pkt_num;
+	u64 mac_rx_send_app_bad_pkt_num;
 };
 
+#define HCLGE_STATS_TIMER_INTERVAL	(60 * 5)
 struct hclge_hw_stats {
 	struct hclge_mac_stats      mac_stats;
 	struct hclge_64_bit_stats   all_64_bit_stats;
 	struct hclge_32_bit_stats   all_32_bit_stats;
+	u32 stats_timer;
+};
+
+struct hclge_vlan_type_cfg {
+	u16 rx_ot_fst_vlan_type;
+	u16 rx_ot_sec_vlan_type;
+	u16 rx_in_fst_vlan_type;
+	u16 rx_in_sec_vlan_type;
+	u16 tx_ot_vlan_type;
+	u16 tx_in_vlan_type;
 };
 
 struct hclge_dev {
@@ -420,6 +471,8 @@ struct hclge_dev {
 	unsigned long state;
 
 	enum hnae3_reset_type reset_type;
+	unsigned long reset_request;	/* reset has been requested */
+	unsigned long reset_pending;	/* client rst is pending to be served */
 	u32 fw_version;
 	u16 num_vmdq_vport;		/* Num vmdq vport this PF has set up */
 	u16 num_tqps;			/* Num task queue pairs of this PF */
@@ -469,6 +522,8 @@ struct hclge_dev {
 	unsigned long service_timer_previous;
 	struct timer_list service_timer;
 	struct work_struct service_task;
+	struct work_struct rst_service_task;
+	struct work_struct mbx_service_task;
 
 	bool cur_promisc;
 	int num_alloc_vfs;	/* Actual number of VFs allocated */
@@ -493,6 +548,29 @@ struct hclge_dev {
 	enum hclge_mta_dmac_sel_type mta_mac_sel_type;
 	bool enable_mta; /* Mutilcast filter enable */
 	bool accept_mta_mc; /* Whether accept mta filter multicast */
+
+	struct hclge_vlan_type_cfg vlan_type_cfg;
+
+	u64 rx_pkts_for_led;
+	u64 tx_pkts_for_led;
+};
+
+/* VPort level vlan tag configuration for TX direction */
+struct hclge_tx_vtag_cfg {
+	bool accept_tag;	/* Whether accept tagged packet from host */
+	bool accept_untag;	/* Whether accept untagged packet from host */
+	bool insert_tag1_en;	/* Whether insert inner vlan tag */
+	bool insert_tag2_en;	/* Whether insert outer vlan tag */
+	u16  default_tag1;	/* The default inner vlan tag to insert */
+	u16  default_tag2;	/* The default outer vlan tag to insert */
+};
+
+/* VPort level vlan tag configuration for RX direction */
+struct hclge_rx_vtag_cfg {
+	bool strip_tag1_en;	/* Whether strip inner vlan tag */
+	bool strip_tag2_en;	/* Whether strip outer vlan tag */
+	bool vlan1_vlan_prionly;/* Inner VLAN Tag up to descriptor Enable */
+	bool vlan2_vlan_prionly;/* Outer VLAN Tag up to descriptor Enable */
 };
 
 struct hclge_vport {
@@ -507,6 +585,9 @@ struct hclge_vport {
 	u16 bw_limit;		/* VSI BW Limit (0 = disabled) */
 	u8  dwrr;
 
+	struct hclge_tx_vtag_cfg  txvlan_cfg;
+	struct hclge_rx_vtag_cfg  rxvlan_cfg;
+
 	int vport_id;
 	struct hclge_dev *back;  /* Back reference to associated dev */
 	struct hnae3_handle nic;
@@ -529,8 +610,10 @@ int hclge_cfg_func_mta_filter(struct hclge_dev *hdev,
 			      u8 func_id,
 			      bool enable);
 struct hclge_vport *hclge_get_vport(struct hnae3_handle *handle);
-int hclge_map_vport_ring_to_vector(struct hclge_vport *vport, int vector,
-				   struct hnae3_ring_chain_node *ring_chain);
+int hclge_bind_ring_with_vector(struct hclge_vport *vport,
+				int vector_id, bool en,
+				struct hnae3_ring_chain_node *ring_chain);
+
 static inline int hclge_get_queue_id(struct hnae3_queue *queue)
 {
 	struct hclge_tqp *tqp = container_of(queue, struct hclge_tqp, q);
@@ -544,4 +627,8 @@ int hclge_set_vf_vlan_common(struct hclge_dev *vport, int vfid,
 
 int hclge_buffer_alloc(struct hclge_dev *hdev);
 int hclge_rss_init_hw(struct hclge_dev *hdev);
+
+void hclge_mbx_handler(struct hclge_dev *hdev);
+void hclge_reset_tqp(struct hnae3_handle *handle, u16 queue_id);
+int hclge_cfg_flowctrl(struct hclge_dev *hdev);
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
new file mode 100644
index 000000000000..f38fc5ce9f51
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
@@ -0,0 +1,418 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2016-2017 Hisilicon Limited.
+
+#include "hclge_main.h"
+#include "hclge_mbx.h"
+#include "hnae3.h"
+
+/* hclge_gen_resp_to_vf: used to generate a synchronous response to VF when PF
+ * receives a mailbox message from VF.
+ * @vport: pointer to struct hclge_vport
+ * @vf_to_pf_req: pointer to hclge_mbx_vf_to_pf_cmd of the original mailbox
+ *		  message
+ * @resp_status: indicate to VF whether its request success(0) or failed.
+ */
+static int hclge_gen_resp_to_vf(struct hclge_vport *vport,
+				struct hclge_mbx_vf_to_pf_cmd *vf_to_pf_req,
+				int resp_status,
+				u8 *resp_data, u16 resp_data_len)
+{
+	struct hclge_mbx_pf_to_vf_cmd *resp_pf_to_vf;
+	struct hclge_dev *hdev = vport->back;
+	enum hclge_cmd_status status;
+	struct hclge_desc desc;
+
+	resp_pf_to_vf = (struct hclge_mbx_pf_to_vf_cmd *)desc.data;
+
+	if (resp_data_len > HCLGE_MBX_MAX_RESP_DATA_SIZE) {
+		dev_err(&hdev->pdev->dev,
+			"PF fail to gen resp to VF len %d exceeds max len %d\n",
+			resp_data_len,
+			HCLGE_MBX_MAX_RESP_DATA_SIZE);
+	}
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_MBX_PF_TO_VF, false);
+
+	resp_pf_to_vf->dest_vfid = vf_to_pf_req->mbx_src_vfid;
+	resp_pf_to_vf->msg_len = vf_to_pf_req->msg_len;
+
+	resp_pf_to_vf->msg[0] = HCLGE_MBX_PF_VF_RESP;
+	resp_pf_to_vf->msg[1] = vf_to_pf_req->msg[0];
+	resp_pf_to_vf->msg[2] = vf_to_pf_req->msg[1];
+	resp_pf_to_vf->msg[3] = (resp_status == 0) ? 0 : 1;
+
+	if (resp_data && resp_data_len > 0)
+		memcpy(&resp_pf_to_vf->msg[4], resp_data, resp_data_len);
+
+	status = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"PF failed(=%d) to send response to VF\n", status);
+
+	return status;
+}
+
+static int hclge_send_mbx_msg(struct hclge_vport *vport, u8 *msg, u16 msg_len,
+			      u16 mbx_opcode, u8 dest_vfid)
+{
+	struct hclge_mbx_pf_to_vf_cmd *resp_pf_to_vf;
+	struct hclge_dev *hdev = vport->back;
+	enum hclge_cmd_status status;
+	struct hclge_desc desc;
+
+	resp_pf_to_vf = (struct hclge_mbx_pf_to_vf_cmd *)desc.data;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_MBX_PF_TO_VF, false);
+
+	resp_pf_to_vf->dest_vfid = dest_vfid;
+	resp_pf_to_vf->msg_len = msg_len;
+	resp_pf_to_vf->msg[0] = mbx_opcode;
+
+	memcpy(&resp_pf_to_vf->msg[1], msg, msg_len);
+
+	status = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"PF failed(=%d) to send mailbox message to VF\n",
+			status);
+
+	return status;
+}
+
+static void hclge_free_vector_ring_chain(struct hnae3_ring_chain_node *head)
+{
+	struct hnae3_ring_chain_node *chain_tmp, *chain;
+
+	chain = head->next;
+
+	while (chain) {
+		chain_tmp = chain->next;
+		kzfree(chain);
+		chain = chain_tmp;
+	}
+}
+
+/* hclge_get_ring_chain_from_mbx: get ring type & tqpid from mailbox message
+ * msg[0]: opcode
+ * msg[1]: <not relevant to this function>
+ * msg[2]: ring_num
+ * msg[3]: first ring type (TX|RX)
+ * msg[4]: first tqp id
+ * msg[5] ~ msg[14]: other ring type and tqp id
+ */
+static int hclge_get_ring_chain_from_mbx(
+			struct hclge_mbx_vf_to_pf_cmd *req,
+			struct hnae3_ring_chain_node *ring_chain,
+			struct hclge_vport *vport)
+{
+#define HCLGE_RING_NODE_VARIABLE_NUM		3
+#define HCLGE_RING_MAP_MBX_BASIC_MSG_NUM	3
+	struct hnae3_ring_chain_node *cur_chain, *new_chain;
+	int ring_num;
+	int i;
+
+	ring_num = req->msg[2];
+
+	hnae_set_bit(ring_chain->flag, HNAE3_RING_TYPE_B, req->msg[3]);
+	ring_chain->tqp_index =
+			hclge_get_queue_id(vport->nic.kinfo.tqp[req->msg[4]]);
+	hnae_set_field(ring_chain->int_gl_idx, HCLGE_INT_GL_IDX_M,
+		       HCLGE_INT_GL_IDX_S,
+		       req->msg[5]);
+
+	cur_chain = ring_chain;
+
+	for (i = 1; i < ring_num; i++) {
+		new_chain = kzalloc(sizeof(*new_chain), GFP_KERNEL);
+		if (!new_chain)
+			goto err;
+
+		hnae_set_bit(new_chain->flag, HNAE3_RING_TYPE_B,
+			     req->msg[HCLGE_RING_NODE_VARIABLE_NUM * i +
+			     HCLGE_RING_MAP_MBX_BASIC_MSG_NUM]);
+
+		new_chain->tqp_index =
+		hclge_get_queue_id(vport->nic.kinfo.tqp
+			[req->msg[HCLGE_RING_NODE_VARIABLE_NUM * i +
+			HCLGE_RING_MAP_MBX_BASIC_MSG_NUM + 1]]);
+
+		hnae_set_field(new_chain->int_gl_idx, HCLGE_INT_GL_IDX_M,
+			       HCLGE_INT_GL_IDX_S,
+			       req->msg[HCLGE_RING_NODE_VARIABLE_NUM * i +
+			       HCLGE_RING_MAP_MBX_BASIC_MSG_NUM + 2]);
+
+		cur_chain->next = new_chain;
+		cur_chain = new_chain;
+	}
+
+	return 0;
+err:
+	hclge_free_vector_ring_chain(ring_chain);
+	return -ENOMEM;
+}
+
+static int hclge_map_unmap_ring_to_vf_vector(struct hclge_vport *vport, bool en,
+					     struct hclge_mbx_vf_to_pf_cmd *req)
+{
+	struct hnae3_ring_chain_node ring_chain;
+	int vector_id = req->msg[1];
+	int ret;
+
+	memset(&ring_chain, 0, sizeof(ring_chain));
+	ret = hclge_get_ring_chain_from_mbx(req, &ring_chain, vport);
+	if (ret)
+		return ret;
+
+	ret = hclge_bind_ring_with_vector(vport, vector_id, en, &ring_chain);
+	if (ret)
+		return ret;
+
+	hclge_free_vector_ring_chain(&ring_chain);
+
+	return 0;
+}
+
+static int hclge_set_vf_promisc_mode(struct hclge_vport *vport,
+				     struct hclge_mbx_vf_to_pf_cmd *req)
+{
+	bool en = req->msg[1] ? true : false;
+	struct hclge_promisc_param param;
+
+	/* always enable broadcast promisc bit */
+	hclge_promisc_param_init(&param, en, en, true, vport->vport_id);
+	return hclge_cmd_set_promisc_mode(vport->back, &param);
+}
+
+static int hclge_set_vf_uc_mac_addr(struct hclge_vport *vport,
+				    struct hclge_mbx_vf_to_pf_cmd *mbx_req,
+				    bool gen_resp)
+{
+	const u8 *mac_addr = (const u8 *)(&mbx_req->msg[2]);
+	struct hclge_dev *hdev = vport->back;
+	int status;
+
+	if (mbx_req->msg[1] == HCLGE_MBX_MAC_VLAN_UC_MODIFY) {
+		const u8 *old_addr = (const u8 *)(&mbx_req->msg[8]);
+
+		hclge_rm_uc_addr_common(vport, old_addr);
+		status = hclge_add_uc_addr_common(vport, mac_addr);
+	} else if (mbx_req->msg[1] == HCLGE_MBX_MAC_VLAN_UC_ADD) {
+		status = hclge_add_uc_addr_common(vport, mac_addr);
+	} else if (mbx_req->msg[1] == HCLGE_MBX_MAC_VLAN_UC_REMOVE) {
+		status = hclge_rm_uc_addr_common(vport, mac_addr);
+	} else {
+		dev_err(&hdev->pdev->dev,
+			"failed to set unicast mac addr, unknown subcode %d\n",
+			mbx_req->msg[1]);
+		return -EIO;
+	}
+
+	if (gen_resp)
+		hclge_gen_resp_to_vf(vport, mbx_req, status, NULL, 0);
+
+	return 0;
+}
+
+static int hclge_set_vf_mc_mac_addr(struct hclge_vport *vport,
+				    struct hclge_mbx_vf_to_pf_cmd *mbx_req,
+				    bool gen_resp)
+{
+	const u8 *mac_addr = (const u8 *)(&mbx_req->msg[2]);
+	struct hclge_dev *hdev = vport->back;
+	int status;
+
+	if (mbx_req->msg[1] == HCLGE_MBX_MAC_VLAN_MC_ADD) {
+		status = hclge_add_mc_addr_common(vport, mac_addr);
+	} else if (mbx_req->msg[1] == HCLGE_MBX_MAC_VLAN_MC_REMOVE) {
+		status = hclge_rm_mc_addr_common(vport, mac_addr);
+	} else if (mbx_req->msg[1] == HCLGE_MBX_MAC_VLAN_MC_FUNC_MTA_ENABLE) {
+		u8 func_id = vport->vport_id;
+		bool enable = mbx_req->msg[2];
+
+		status = hclge_cfg_func_mta_filter(hdev, func_id, enable);
+	} else {
+		dev_err(&hdev->pdev->dev,
+			"failed to set mcast mac addr, unknown subcode %d\n",
+			mbx_req->msg[1]);
+		return -EIO;
+	}
+
+	if (gen_resp)
+		hclge_gen_resp_to_vf(vport, mbx_req, status, NULL, 0);
+
+	return 0;
+}
+
+static int hclge_set_vf_vlan_cfg(struct hclge_vport *vport,
+				 struct hclge_mbx_vf_to_pf_cmd *mbx_req,
+				 bool gen_resp)
+{
+	struct hclge_dev *hdev = vport->back;
+	int status = 0;
+
+	if (mbx_req->msg[1] == HCLGE_MBX_VLAN_FILTER) {
+		u16 vlan, proto;
+		bool is_kill;
+
+		is_kill = !!mbx_req->msg[2];
+		memcpy(&vlan, &mbx_req->msg[3], sizeof(vlan));
+		memcpy(&proto, &mbx_req->msg[5], sizeof(proto));
+		status = hclge_set_vf_vlan_common(hdev, vport->vport_id,
+						  is_kill, vlan, 0,
+						  cpu_to_be16(proto));
+	}
+
+	if (gen_resp)
+		status = hclge_gen_resp_to_vf(vport, mbx_req, status, NULL, 0);
+
+	return status;
+}
+
+static int hclge_get_vf_tcinfo(struct hclge_vport *vport,
+			       struct hclge_mbx_vf_to_pf_cmd *mbx_req,
+			       bool gen_resp)
+{
+	struct hclge_dev *hdev = vport->back;
+	int ret;
+
+	ret = hclge_gen_resp_to_vf(vport, mbx_req, 0, &hdev->hw_tc_map,
+				   sizeof(u8));
+
+	return ret;
+}
+
+static int hclge_get_vf_queue_info(struct hclge_vport *vport,
+				   struct hclge_mbx_vf_to_pf_cmd *mbx_req,
+				   bool gen_resp)
+{
+#define HCLGE_TQPS_RSS_INFO_LEN		8
+	u8 resp_data[HCLGE_TQPS_RSS_INFO_LEN];
+	struct hclge_dev *hdev = vport->back;
+
+	/* get the queue related info */
+	memcpy(&resp_data[0], &vport->alloc_tqps, sizeof(u16));
+	memcpy(&resp_data[2], &hdev->rss_size_max, sizeof(u16));
+	memcpy(&resp_data[4], &hdev->num_desc, sizeof(u16));
+	memcpy(&resp_data[6], &hdev->rx_buf_len, sizeof(u16));
+
+	return hclge_gen_resp_to_vf(vport, mbx_req, 0, resp_data,
+				    HCLGE_TQPS_RSS_INFO_LEN);
+}
+
+static int hclge_get_link_info(struct hclge_vport *vport,
+			       struct hclge_mbx_vf_to_pf_cmd *mbx_req)
+{
+	struct hclge_dev *hdev = vport->back;
+	u16 link_status;
+	u8 msg_data[2];
+	u8 dest_vfid;
+
+	/* mac.link can only be 0 or 1 */
+	link_status = (u16)hdev->hw.mac.link;
+	memcpy(&msg_data[0], &link_status, sizeof(u16));
+	dest_vfid = mbx_req->mbx_src_vfid;
+
+	/* send this requested info to VF */
+	return hclge_send_mbx_msg(vport, msg_data, sizeof(u8),
+				  HCLGE_MBX_LINK_STAT_CHANGE, dest_vfid);
+}
+
+static void hclge_reset_vf_queue(struct hclge_vport *vport,
+				 struct hclge_mbx_vf_to_pf_cmd *mbx_req)
+{
+	u16 queue_id;
+
+	memcpy(&queue_id, &mbx_req->msg[2], sizeof(queue_id));
+
+	hclge_reset_tqp(&vport->nic, queue_id);
+}
+
+void hclge_mbx_handler(struct hclge_dev *hdev)
+{
+	struct hclge_cmq_ring *crq = &hdev->hw.cmq.crq;
+	struct hclge_mbx_vf_to_pf_cmd *req;
+	struct hclge_vport *vport;
+	struct hclge_desc *desc;
+	int ret;
+
+	/* handle all the mailbox requests in the queue */
+	while (hnae_get_bit(crq->desc[crq->next_to_use].flag,
+			    HCLGE_CMDQ_RX_OUTVLD_B)) {
+		desc = &crq->desc[crq->next_to_use];
+		req = (struct hclge_mbx_vf_to_pf_cmd *)desc->data;
+
+		vport = &hdev->vport[req->mbx_src_vfid];
+
+		switch (req->msg[0]) {
+		case HCLGE_MBX_MAP_RING_TO_VECTOR:
+			ret = hclge_map_unmap_ring_to_vf_vector(vport, true,
+								req);
+			break;
+		case HCLGE_MBX_UNMAP_RING_TO_VECTOR:
+			ret = hclge_map_unmap_ring_to_vf_vector(vport, false,
+								req);
+			break;
+		case HCLGE_MBX_SET_PROMISC_MODE:
+			ret = hclge_set_vf_promisc_mode(vport, req);
+			if (ret)
+				dev_err(&hdev->pdev->dev,
+					"PF fail(%d) to set VF promisc mode\n",
+					ret);
+			break;
+		case HCLGE_MBX_SET_UNICAST:
+			ret = hclge_set_vf_uc_mac_addr(vport, req, false);
+			if (ret)
+				dev_err(&hdev->pdev->dev,
+					"PF fail(%d) to set VF UC MAC Addr\n",
+					ret);
+			break;
+		case HCLGE_MBX_SET_MULTICAST:
+			ret = hclge_set_vf_mc_mac_addr(vport, req, false);
+			if (ret)
+				dev_err(&hdev->pdev->dev,
+					"PF fail(%d) to set VF MC MAC Addr\n",
+					ret);
+			break;
+		case HCLGE_MBX_SET_VLAN:
+			ret = hclge_set_vf_vlan_cfg(vport, req, false);
+			if (ret)
+				dev_err(&hdev->pdev->dev,
+					"PF failed(%d) to config VF's VLAN\n",
+					ret);
+			break;
+		case HCLGE_MBX_GET_QINFO:
+			ret = hclge_get_vf_queue_info(vport, req, true);
+			if (ret)
+				dev_err(&hdev->pdev->dev,
+					"PF failed(%d) to get Q info for VF\n",
+					ret);
+			break;
+		case HCLGE_MBX_GET_TCINFO:
+			ret = hclge_get_vf_tcinfo(vport, req, true);
+			if (ret)
+				dev_err(&hdev->pdev->dev,
+					"PF failed(%d) to get TC info for VF\n",
+					ret);
+			break;
+		case HCLGE_MBX_GET_LINK_STATUS:
+			ret = hclge_get_link_info(vport, req);
+			if (ret)
+				dev_err(&hdev->pdev->dev,
+					"PF fail(%d) to get link stat for VF\n",
+					ret);
+			break;
+		case HCLGE_MBX_QUEUE_RESET:
+			hclge_reset_vf_queue(vport, req);
+			break;
+		default:
+			dev_err(&hdev->pdev->dev,
+				"un-supported mailbox message, code = %d\n",
+				req->msg[0]);
+			break;
+		}
+		hclge_mbx_ring_ptr_move_crq(crq);
+	}
+
+	/* Write back CMDQ_RQ header pointer, M7 need this pointer */
+	hclge_write_dev(&hdev->hw, HCLGE_NIC_CRQ_HEAD_REG, crq->next_to_use);
+}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
index 7069e9408d7d..c1dea3a47bdd 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
@@ -17,6 +17,7 @@
 #define HCLGE_PHY_SUPPORTED_FEATURES	(SUPPORTED_Autoneg | \
 					 SUPPORTED_TP | \
 					 SUPPORTED_Pause | \
+					 SUPPORTED_Asym_Pause | \
 					 PHY_10BT_FEATURES | \
 					 PHY_100BT_FEATURES | \
 					 PHY_1000BT_FEATURES)
@@ -183,6 +184,10 @@ static void hclge_mac_adjust_link(struct net_device *netdev)
 	ret = hclge_cfg_mac_speed_dup(hdev, speed, duplex);
 	if (ret)
 		netdev_err(netdev, "failed to adjust link.\n");
+
+	ret = hclge_cfg_flowctrl(hdev);
+	if (ret)
+		netdev_err(netdev, "failed to configure flow control.\n");
 }
 
 int hclge_mac_start_phy(struct hclge_dev *hdev)
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
index 7bfa2e5497cb..36bd79a77940 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
@@ -23,8 +23,8 @@ enum hclge_shaper_level {
 	HCLGE_SHAPER_LVL_PF	= 1,
 };
 
-#define HCLGE_SHAPER_BS_U_DEF	1
-#define HCLGE_SHAPER_BS_S_DEF	4
+#define HCLGE_SHAPER_BS_U_DEF	5
+#define HCLGE_SHAPER_BS_S_DEF	20
 
 #define HCLGE_ETHER_MAX_RATE	100000
 
@@ -112,7 +112,7 @@ static int hclge_shaper_para_calc(u32 ir, u8 shaper_level,
 	return 0;
 }
 
-static int hclge_mac_pause_en_cfg(struct hclge_dev *hdev, bool tx, bool rx)
+int hclge_mac_pause_en_cfg(struct hclge_dev *hdev, bool tx, bool rx)
 {
 	struct hclge_desc desc;
 
@@ -138,6 +138,46 @@ static int hclge_pfc_pause_en_cfg(struct hclge_dev *hdev, u8 tx_rx_bitmap,
 	return hclge_cmd_send(&hdev->hw, &desc, 1);
 }
 
+static int hclge_mac_pause_param_cfg(struct hclge_dev *hdev, const u8 *addr,
+				     u8 pause_trans_gap, u16 pause_trans_time)
+{
+	struct hclge_cfg_pause_param_cmd *pause_param;
+	struct hclge_desc desc;
+
+	pause_param = (struct hclge_cfg_pause_param_cmd *)&desc.data;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_MAC_PARA, false);
+
+	ether_addr_copy(pause_param->mac_addr, addr);
+	pause_param->pause_trans_gap = pause_trans_gap;
+	pause_param->pause_trans_time = cpu_to_le16(pause_trans_time);
+
+	return hclge_cmd_send(&hdev->hw, &desc, 1);
+}
+
+int hclge_mac_pause_addr_cfg(struct hclge_dev *hdev, const u8 *mac_addr)
+{
+	struct hclge_cfg_pause_param_cmd *pause_param;
+	struct hclge_desc desc;
+	u16 trans_time;
+	u8 trans_gap;
+	int ret;
+
+	pause_param = (struct hclge_cfg_pause_param_cmd *)&desc.data;
+
+	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CFG_MAC_PARA, true);
+
+	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
+	if (ret)
+		return ret;
+
+	trans_gap = pause_param->pause_trans_gap;
+	trans_time = le16_to_cpu(pause_param->pause_trans_time);
+
+	return hclge_mac_pause_param_cfg(hdev, mac_addr, trans_gap,
+					 trans_time);
+}
+
 static int hclge_fill_pri_array(struct hclge_dev *hdev, u8 *pri, u8 pri_id)
 {
 	u8 tc;
@@ -1056,6 +1096,15 @@ static int hclge_tm_schd_setup_hw(struct hclge_dev *hdev)
 	return hclge_tm_schd_mode_hw(hdev);
 }
 
+static int hclge_mac_pause_param_setup_hw(struct hclge_dev *hdev)
+{
+	struct hclge_mac *mac = &hdev->hw.mac;
+
+	return hclge_mac_pause_param_cfg(hdev, mac->mac_addr,
+					 HCLGE_DEFAULT_PAUSE_TRANS_GAP,
+					 HCLGE_DEFAULT_PAUSE_TRANS_TIME);
+}
+
 static int hclge_pfc_setup_hw(struct hclge_dev *hdev)
 {
 	u8 enable_bitmap = 0;
@@ -1102,8 +1151,13 @@ int hclge_pause_setup_hw(struct hclge_dev *hdev)
 	int ret;
 	u8 i;
 
-	if (hdev->tm_info.fc_mode != HCLGE_FC_PFC)
-		return hclge_mac_pause_setup_hw(hdev);
+	if (hdev->tm_info.fc_mode != HCLGE_FC_PFC) {
+		ret = hclge_mac_pause_setup_hw(hdev);
+		if (ret)
+			return ret;
+
+		return hclge_mac_pause_param_setup_hw(hdev);
+	}
 
 	/* Only DCB-supported dev supports qset back pressure and pfc cmd */
 	if (!hnae3_dev_dcb_supported(hdev))
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
index bf59961918ab..5401e7559437 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
@@ -18,6 +18,9 @@
 
 #define HCLGE_TM_PORT_BASE_MODE_MSK	BIT(0)
 
+#define HCLGE_DEFAULT_PAUSE_TRANS_GAP	0xFF
+#define HCLGE_DEFAULT_PAUSE_TRANS_TIME	0xFFFF
+
 /* SP or DWRR */
 #define HCLGE_TM_TX_SCHD_DWRR_MSK	BIT(0)
 #define HCLGE_TM_TX_SCHD_SP_MSK		(0xFE)
@@ -99,6 +102,13 @@ struct hclge_pfc_en_cmd {
 	u8 pri_en_bitmap;
 };
 
+struct hclge_cfg_pause_param_cmd {
+	u8 mac_addr[ETH_ALEN];
+	u8 pause_trans_gap;
+	u8 rsvd;
+	__le16 pause_trans_time;
+};
+
 struct hclge_port_shapping_cmd {
 	__le32 port_shapping_para;
 };
@@ -118,4 +128,6 @@ void hclge_tm_schd_info_update(struct hclge_dev *hdev, u8 num_tc);
 int hclge_tm_dwrr_cfg(struct hclge_dev *hdev);
 int hclge_tm_map_cfg(struct hclge_dev *hdev);
 int hclge_tm_init_hw(struct hclge_dev *hdev);
+int hclge_mac_pause_en_cfg(struct hclge_dev *hdev, bool tx, bool rx);
+int hclge_mac_pause_addr_cfg(struct hclge_dev *hdev, const u8 *mac_addr);
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/Makefile b/drivers/net/ethernet/hisilicon/hns3/hns3vf/Makefile
new file mode 100644
index 000000000000..fb93bbd35845
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0+
+#
+# Makefile for the HISILICON network device drivers.
+#
+
+ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3
+
+obj-$(CONFIG_HNS3_HCLGEVF) += hclgevf.o
+hclgevf-objs = hclgevf_main.o hclgevf_cmd.o hclgevf_mbx.o
\ No newline at end of file
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c
new file mode 100644
index 000000000000..85985e731311
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.c
@@ -0,0 +1,342 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2016-2017 Hisilicon Limited.
+
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include "hclgevf_cmd.h"
+#include "hclgevf_main.h"
+#include "hnae3.h"
+
+#define hclgevf_is_csq(ring) ((ring)->flag & HCLGEVF_TYPE_CSQ)
+#define hclgevf_ring_to_dma_dir(ring) (hclgevf_is_csq(ring) ? \
+					DMA_TO_DEVICE : DMA_FROM_DEVICE)
+#define cmq_ring_to_dev(ring)   (&(ring)->dev->pdev->dev)
+
+static int hclgevf_ring_space(struct hclgevf_cmq_ring *ring)
+{
+	int ntc = ring->next_to_clean;
+	int ntu = ring->next_to_use;
+	int used;
+
+	used = (ntu - ntc + ring->desc_num) % ring->desc_num;
+
+	return ring->desc_num - used - 1;
+}
+
+static int hclgevf_cmd_csq_clean(struct hclgevf_hw *hw)
+{
+	struct hclgevf_cmq_ring *csq = &hw->cmq.csq;
+	u16 ntc = csq->next_to_clean;
+	struct hclgevf_desc *desc;
+	int clean = 0;
+	u32 head;
+
+	desc = &csq->desc[ntc];
+	head = hclgevf_read_dev(hw, HCLGEVF_NIC_CSQ_HEAD_REG);
+	while (head != ntc) {
+		memset(desc, 0, sizeof(*desc));
+		ntc++;
+		if (ntc == csq->desc_num)
+			ntc = 0;
+		desc = &csq->desc[ntc];
+		clean++;
+	}
+	csq->next_to_clean = ntc;
+
+	return clean;
+}
+
+static bool hclgevf_cmd_csq_done(struct hclgevf_hw *hw)
+{
+	u32 head;
+
+	head = hclgevf_read_dev(hw, HCLGEVF_NIC_CSQ_HEAD_REG);
+
+	return head == hw->cmq.csq.next_to_use;
+}
+
+static bool hclgevf_is_special_opcode(u16 opcode)
+{
+	u16 spec_opcode[] = {0x30, 0x31, 0x32};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(spec_opcode); i++) {
+		if (spec_opcode[i] == opcode)
+			return true;
+	}
+
+	return false;
+}
+
+static int hclgevf_alloc_cmd_desc(struct hclgevf_cmq_ring *ring)
+{
+	int size = ring->desc_num * sizeof(struct hclgevf_desc);
+
+	ring->desc = kzalloc(size, GFP_KERNEL);
+	if (!ring->desc)
+		return -ENOMEM;
+
+	ring->desc_dma_addr = dma_map_single(cmq_ring_to_dev(ring), ring->desc,
+					     size, DMA_BIDIRECTIONAL);
+
+	if (dma_mapping_error(cmq_ring_to_dev(ring), ring->desc_dma_addr)) {
+		ring->desc_dma_addr = 0;
+		kfree(ring->desc);
+		ring->desc = NULL;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void hclgevf_free_cmd_desc(struct hclgevf_cmq_ring *ring)
+{
+	dma_unmap_single(cmq_ring_to_dev(ring), ring->desc_dma_addr,
+			 ring->desc_num * sizeof(ring->desc[0]),
+			 hclgevf_ring_to_dma_dir(ring));
+
+	ring->desc_dma_addr = 0;
+	kfree(ring->desc);
+	ring->desc = NULL;
+}
+
+static int hclgevf_init_cmd_queue(struct hclgevf_dev *hdev,
+				  struct hclgevf_cmq_ring *ring)
+{
+	struct hclgevf_hw *hw = &hdev->hw;
+	int ring_type = ring->flag;
+	u32 reg_val;
+	int ret;
+
+	ring->desc_num = HCLGEVF_NIC_CMQ_DESC_NUM;
+	spin_lock_init(&ring->lock);
+	ring->next_to_clean = 0;
+	ring->next_to_use = 0;
+	ring->dev = hdev;
+
+	/* allocate CSQ/CRQ descriptor */
+	ret = hclgevf_alloc_cmd_desc(ring);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "failed(%d) to alloc %s desc\n", ret,
+			(ring_type == HCLGEVF_TYPE_CSQ) ? "CSQ" : "CRQ");
+		return ret;
+	}
+
+	/* initialize the hardware registers with csq/crq dma-address,
+	 * descriptor number, head & tail pointers
+	 */
+	switch (ring_type) {
+	case HCLGEVF_TYPE_CSQ:
+		reg_val = (u32)ring->desc_dma_addr;
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_BASEADDR_L_REG, reg_val);
+		reg_val = (u32)((ring->desc_dma_addr >> 31) >> 1);
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_BASEADDR_H_REG, reg_val);
+
+		reg_val = (ring->desc_num >> HCLGEVF_NIC_CMQ_DESC_NUM_S);
+		reg_val |= HCLGEVF_NIC_CMQ_ENABLE;
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_DEPTH_REG, reg_val);
+
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_TAIL_REG, 0);
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_HEAD_REG, 0);
+		break;
+	case HCLGEVF_TYPE_CRQ:
+		reg_val = (u32)ring->desc_dma_addr;
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_BASEADDR_L_REG, reg_val);
+		reg_val = (u32)((ring->desc_dma_addr >> 31) >> 1);
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_BASEADDR_H_REG, reg_val);
+
+		reg_val = (ring->desc_num >> HCLGEVF_NIC_CMQ_DESC_NUM_S);
+		reg_val |= HCLGEVF_NIC_CMQ_ENABLE;
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_DEPTH_REG, reg_val);
+
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_TAIL_REG, 0);
+		hclgevf_write_dev(hw, HCLGEVF_NIC_CRQ_HEAD_REG, 0);
+		break;
+	}
+
+	return 0;
+}
+
+void hclgevf_cmd_setup_basic_desc(struct hclgevf_desc *desc,
+				  enum hclgevf_opcode_type opcode, bool is_read)
+{
+	memset(desc, 0, sizeof(struct hclgevf_desc));
+	desc->opcode = cpu_to_le16(opcode);
+	desc->flag = cpu_to_le16(HCLGEVF_CMD_FLAG_NO_INTR |
+				 HCLGEVF_CMD_FLAG_IN);
+	if (is_read)
+		desc->flag |= cpu_to_le16(HCLGEVF_CMD_FLAG_WR);
+	else
+		desc->flag &= cpu_to_le16(~HCLGEVF_CMD_FLAG_WR);
+}
+
+/* hclgevf_cmd_send - send command to command queue
+ * @hw: pointer to the hw struct
+ * @desc: prefilled descriptor for describing the command
+ * @num : the number of descriptors to be sent
+ *
+ * This is the main send command for command queue, it
+ * sends the queue, cleans the queue, etc
+ */
+int hclgevf_cmd_send(struct hclgevf_hw *hw, struct hclgevf_desc *desc, int num)
+{
+	struct hclgevf_dev *hdev = (struct hclgevf_dev *)hw->hdev;
+	struct hclgevf_desc *desc_to_use;
+	bool complete = false;
+	u32 timeout = 0;
+	int handle = 0;
+	int status = 0;
+	u16 retval;
+	u16 opcode;
+	int ntc;
+
+	spin_lock_bh(&hw->cmq.csq.lock);
+
+	if (num > hclgevf_ring_space(&hw->cmq.csq)) {
+		spin_unlock_bh(&hw->cmq.csq.lock);
+		return -EBUSY;
+	}
+
+	/* Record the location of desc in the ring for this time
+	 * which will be use for hardware to write back
+	 */
+	ntc = hw->cmq.csq.next_to_use;
+	opcode = le16_to_cpu(desc[0].opcode);
+	while (handle < num) {
+		desc_to_use = &hw->cmq.csq.desc[hw->cmq.csq.next_to_use];
+		*desc_to_use = desc[handle];
+		(hw->cmq.csq.next_to_use)++;
+		if (hw->cmq.csq.next_to_use == hw->cmq.csq.desc_num)
+			hw->cmq.csq.next_to_use = 0;
+		handle++;
+	}
+
+	/* Write to hardware */
+	hclgevf_write_dev(hw, HCLGEVF_NIC_CSQ_TAIL_REG,
+			  hw->cmq.csq.next_to_use);
+
+	/* If the command is sync, wait for the firmware to write back,
+	 * if multi descriptors to be sent, use the first one to check
+	 */
+	if (HCLGEVF_SEND_SYNC(le16_to_cpu(desc->flag))) {
+		do {
+			if (hclgevf_cmd_csq_done(hw))
+				break;
+			udelay(1);
+			timeout++;
+		} while (timeout < hw->cmq.tx_timeout);
+	}
+
+	if (hclgevf_cmd_csq_done(hw)) {
+		complete = true;
+		handle = 0;
+
+		while (handle < num) {
+			/* Get the result of hardware write back */
+			desc_to_use = &hw->cmq.csq.desc[ntc];
+			desc[handle] = *desc_to_use;
+
+			if (likely(!hclgevf_is_special_opcode(opcode)))
+				retval = le16_to_cpu(desc[handle].retval);
+			else
+				retval = le16_to_cpu(desc[0].retval);
+
+			if ((enum hclgevf_cmd_return_status)retval ==
+			    HCLGEVF_CMD_EXEC_SUCCESS)
+				status = 0;
+			else
+				status = -EIO;
+			hw->cmq.last_status = (enum hclgevf_cmd_status)retval;
+			ntc++;
+			handle++;
+			if (ntc == hw->cmq.csq.desc_num)
+				ntc = 0;
+		}
+	}
+
+	if (!complete)
+		status = -EAGAIN;
+
+	/* Clean the command send queue */
+	handle = hclgevf_cmd_csq_clean(hw);
+	if (handle != num) {
+		dev_warn(&hdev->pdev->dev,
+			 "cleaned %d, need to clean %d\n", handle, num);
+	}
+
+	spin_unlock_bh(&hw->cmq.csq.lock);
+
+	return status;
+}
+
+static int  hclgevf_cmd_query_firmware_version(struct hclgevf_hw *hw,
+					       u32 *version)
+{
+	struct hclgevf_query_version_cmd *resp;
+	struct hclgevf_desc desc;
+	int status;
+
+	resp = (struct hclgevf_query_version_cmd *)desc.data;
+
+	hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_QUERY_FW_VER, 1);
+	status = hclgevf_cmd_send(hw, &desc, 1);
+	if (!status)
+		*version = le32_to_cpu(resp->firmware);
+
+	return status;
+}
+
+int hclgevf_cmd_init(struct hclgevf_dev *hdev)
+{
+	u32 version;
+	int ret;
+
+	/* setup Tx write back timeout */
+	hdev->hw.cmq.tx_timeout = HCLGEVF_CMDQ_TX_TIMEOUT;
+
+	/* setup queue CSQ/CRQ rings */
+	hdev->hw.cmq.csq.flag = HCLGEVF_TYPE_CSQ;
+	ret = hclgevf_init_cmd_queue(hdev, &hdev->hw.cmq.csq);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed(%d) to initialize CSQ ring\n", ret);
+		return ret;
+	}
+
+	hdev->hw.cmq.crq.flag = HCLGEVF_TYPE_CRQ;
+	ret = hclgevf_init_cmd_queue(hdev, &hdev->hw.cmq.crq);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed(%d) to initialize CRQ ring\n", ret);
+		goto err_csq;
+	}
+
+	/* get firmware version */
+	ret = hclgevf_cmd_query_firmware_version(&hdev->hw, &version);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed(%d) to query firmware version\n", ret);
+		goto err_crq;
+	}
+	hdev->fw_version = version;
+
+	dev_info(&hdev->pdev->dev, "The firmware version is %08x\n", version);
+
+	return 0;
+err_crq:
+	hclgevf_free_cmd_desc(&hdev->hw.cmq.crq);
+err_csq:
+	hclgevf_free_cmd_desc(&hdev->hw.cmq.csq);
+
+	return ret;
+}
+
+void hclgevf_cmd_uninit(struct hclgevf_dev *hdev)
+{
+	hclgevf_free_cmd_desc(&hdev->hw.cmq.csq);
+	hclgevf_free_cmd_desc(&hdev->hw.cmq.crq);
+}
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h
new file mode 100644
index 000000000000..2caca9317f8c
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_cmd.h
@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/* Copyright (c) 2016-2017 Hisilicon Limited. */
+
+#ifndef __HCLGEVF_CMD_H
+#define __HCLGEVF_CMD_H
+#include <linux/io.h>
+#include <linux/types.h>
+#include "hnae3.h"
+
+#define HCLGEVF_CMDQ_TX_TIMEOUT		200
+#define HCLGEVF_CMDQ_RX_INVLD_B		0
+#define HCLGEVF_CMDQ_RX_OUTVLD_B	1
+
+struct hclgevf_hw;
+struct hclgevf_dev;
+
+struct hclgevf_desc {
+	__le16 opcode;
+	__le16 flag;
+	__le16 retval;
+	__le16 rsv;
+	__le32 data[6];
+};
+
+struct hclgevf_desc_cb {
+	dma_addr_t dma;
+	void *va;
+	u32 length;
+};
+
+struct hclgevf_cmq_ring {
+	dma_addr_t desc_dma_addr;
+	struct hclgevf_desc *desc;
+	struct hclgevf_desc_cb *desc_cb;
+	struct hclgevf_dev  *dev;
+	u32 head;
+	u32 tail;
+
+	u16 buf_size;
+	u16 desc_num;
+	int next_to_use;
+	int next_to_clean;
+	u8 flag;
+	spinlock_t lock; /* Command queue lock */
+};
+
+enum hclgevf_cmd_return_status {
+	HCLGEVF_CMD_EXEC_SUCCESS	= 0,
+	HCLGEVF_CMD_NO_AUTH	= 1,
+	HCLGEVF_CMD_NOT_EXEC	= 2,
+	HCLGEVF_CMD_QUEUE_FULL	= 3,
+};
+
+enum hclgevf_cmd_status {
+	HCLGEVF_STATUS_SUCCESS	= 0,
+	HCLGEVF_ERR_CSQ_FULL	= -1,
+	HCLGEVF_ERR_CSQ_TIMEOUT	= -2,
+	HCLGEVF_ERR_CSQ_ERROR	= -3
+};
+
+struct hclgevf_cmq {
+	struct hclgevf_cmq_ring csq;
+	struct hclgevf_cmq_ring crq;
+	u16 tx_timeout; /* Tx timeout */
+	enum hclgevf_cmd_status last_status;
+};
+
+#define HCLGEVF_CMD_FLAG_IN_VALID_SHIFT		0
+#define HCLGEVF_CMD_FLAG_OUT_VALID_SHIFT	1
+#define HCLGEVF_CMD_FLAG_NEXT_SHIFT		2
+#define HCLGEVF_CMD_FLAG_WR_OR_RD_SHIFT		3
+#define HCLGEVF_CMD_FLAG_NO_INTR_SHIFT		4
+#define HCLGEVF_CMD_FLAG_ERR_INTR_SHIFT		5
+
+#define HCLGEVF_CMD_FLAG_IN		BIT(HCLGEVF_CMD_FLAG_IN_VALID_SHIFT)
+#define HCLGEVF_CMD_FLAG_OUT		BIT(HCLGEVF_CMD_FLAG_OUT_VALID_SHIFT)
+#define HCLGEVF_CMD_FLAG_NEXT		BIT(HCLGEVF_CMD_FLAG_NEXT_SHIFT)
+#define HCLGEVF_CMD_FLAG_WR		BIT(HCLGEVF_CMD_FLAG_WR_OR_RD_SHIFT)
+#define HCLGEVF_CMD_FLAG_NO_INTR	BIT(HCLGEVF_CMD_FLAG_NO_INTR_SHIFT)
+#define HCLGEVF_CMD_FLAG_ERR_INTR	BIT(HCLGEVF_CMD_FLAG_ERR_INTR_SHIFT)
+
+enum hclgevf_opcode_type {
+	/* Generic command */
+	HCLGEVF_OPC_QUERY_FW_VER	= 0x0001,
+	/* TQP command */
+	HCLGEVF_OPC_QUERY_TX_STATUS	= 0x0B03,
+	HCLGEVF_OPC_QUERY_RX_STATUS	= 0x0B13,
+	HCLGEVF_OPC_CFG_COM_TQP_QUEUE	= 0x0B20,
+	/* RSS cmd */
+	HCLGEVF_OPC_RSS_GENERIC_CONFIG	= 0x0D01,
+	HCLGEVF_OPC_RSS_INDIR_TABLE	= 0x0D07,
+	HCLGEVF_OPC_RSS_TC_MODE		= 0x0D08,
+	/* Mailbox cmd */
+	HCLGEVF_OPC_MBX_VF_TO_PF	= 0x2001,
+};
+
+#define HCLGEVF_TQP_REG_OFFSET		0x80000
+#define HCLGEVF_TQP_REG_SIZE		0x200
+
+struct hclgevf_tqp_map {
+	__le16 tqp_id;	/* Absolute tqp id for in this pf */
+	u8 tqp_vf; /* VF id */
+#define HCLGEVF_TQP_MAP_TYPE_PF		0
+#define HCLGEVF_TQP_MAP_TYPE_VF		1
+#define HCLGEVF_TQP_MAP_TYPE_B		0
+#define HCLGEVF_TQP_MAP_EN_B		1
+	u8 tqp_flag;	/* Indicate it's pf or vf tqp */
+	__le16 tqp_vid; /* Virtual id in this pf/vf */
+	u8 rsv[18];
+};
+
+#define HCLGEVF_VECTOR_ELEMENTS_PER_CMD	10
+
+enum hclgevf_int_type {
+	HCLGEVF_INT_TX = 0,
+	HCLGEVF_INT_RX,
+	HCLGEVF_INT_EVENT,
+};
+
+struct hclgevf_ctrl_vector_chain {
+	u8 int_vector_id;
+	u8 int_cause_num;
+#define HCLGEVF_INT_TYPE_S	0
+#define HCLGEVF_INT_TYPE_M	0x3
+#define HCLGEVF_TQP_ID_S	2
+#define HCLGEVF_TQP_ID_M	(0x3fff << HCLGEVF_TQP_ID_S)
+	__le16 tqp_type_and_id[HCLGEVF_VECTOR_ELEMENTS_PER_CMD];
+	u8 vfid;
+	u8 resv;
+};
+
+struct hclgevf_query_version_cmd {
+	__le32 firmware;
+	__le32 firmware_rsv[5];
+};
+
+#define HCLGEVF_RSS_HASH_KEY_OFFSET	4
+#define HCLGEVF_RSS_HASH_KEY_NUM	16
+struct hclgevf_rss_config_cmd {
+	u8 hash_config;
+	u8 rsv[7];
+	u8 hash_key[HCLGEVF_RSS_HASH_KEY_NUM];
+};
+
+struct hclgevf_rss_input_tuple_cmd {
+	u8 ipv4_tcp_en;
+	u8 ipv4_udp_en;
+	u8 ipv4_stcp_en;
+	u8 ipv4_fragment_en;
+	u8 ipv6_tcp_en;
+	u8 ipv6_udp_en;
+	u8 ipv6_stcp_en;
+	u8 ipv6_fragment_en;
+	u8 rsv[16];
+};
+
+#define HCLGEVF_RSS_CFG_TBL_SIZE	16
+
+struct hclgevf_rss_indirection_table_cmd {
+	u16 start_table_index;
+	u16 rss_set_bitmap;
+	u8 rsv[4];
+	u8 rss_result[HCLGEVF_RSS_CFG_TBL_SIZE];
+};
+
+#define HCLGEVF_RSS_TC_OFFSET_S		0
+#define HCLGEVF_RSS_TC_OFFSET_M		(0x3ff << HCLGEVF_RSS_TC_OFFSET_S)
+#define HCLGEVF_RSS_TC_SIZE_S		12
+#define HCLGEVF_RSS_TC_SIZE_M		(0x7 << HCLGEVF_RSS_TC_SIZE_S)
+#define HCLGEVF_RSS_TC_VALID_B		15
+#define HCLGEVF_MAX_TC_NUM		8
+struct hclgevf_rss_tc_mode_cmd {
+	u16 rss_tc_mode[HCLGEVF_MAX_TC_NUM];
+	u8 rsv[8];
+};
+
+#define HCLGEVF_LINK_STS_B	0
+#define HCLGEVF_LINK_STATUS	BIT(HCLGEVF_LINK_STS_B)
+struct hclgevf_link_status_cmd {
+	u8 status;
+	u8 rsv[23];
+};
+
+#define HCLGEVF_RING_ID_MASK	0x3ff
+#define HCLGEVF_TQP_ENABLE_B	0
+
+struct hclgevf_cfg_com_tqp_queue_cmd {
+	__le16 tqp_id;
+	__le16 stream_id;
+	u8 enable;
+	u8 rsv[19];
+};
+
+struct hclgevf_cfg_tx_queue_pointer_cmd {
+	__le16 tqp_id;
+	__le16 tx_tail;
+	__le16 tx_head;
+	__le16 fbd_num;
+	__le16 ring_offset;
+	u8 rsv[14];
+};
+
+#define HCLGEVF_TYPE_CRQ		0
+#define HCLGEVF_TYPE_CSQ		1
+#define HCLGEVF_NIC_CSQ_BASEADDR_L_REG	0x27000
+#define HCLGEVF_NIC_CSQ_BASEADDR_H_REG	0x27004
+#define HCLGEVF_NIC_CSQ_DEPTH_REG	0x27008
+#define HCLGEVF_NIC_CSQ_TAIL_REG	0x27010
+#define HCLGEVF_NIC_CSQ_HEAD_REG	0x27014
+#define HCLGEVF_NIC_CRQ_BASEADDR_L_REG	0x27018
+#define HCLGEVF_NIC_CRQ_BASEADDR_H_REG	0x2701c
+#define HCLGEVF_NIC_CRQ_DEPTH_REG	0x27020
+#define HCLGEVF_NIC_CRQ_TAIL_REG	0x27024
+#define HCLGEVF_NIC_CRQ_HEAD_REG	0x27028
+#define HCLGEVF_NIC_CMQ_EN_B		16
+#define HCLGEVF_NIC_CMQ_ENABLE		BIT(HCLGEVF_NIC_CMQ_EN_B)
+#define HCLGEVF_NIC_CMQ_DESC_NUM	1024
+#define HCLGEVF_NIC_CMQ_DESC_NUM_S	3
+#define HCLGEVF_NIC_CMDQ_INT_SRC_REG	0x27100
+
+static inline void hclgevf_write_reg(void __iomem *base, u32 reg, u32 value)
+{
+	writel(value, base + reg);
+}
+
+static inline u32 hclgevf_read_reg(u8 __iomem *base, u32 reg)
+{
+	u8 __iomem *reg_addr = READ_ONCE(base);
+
+	return readl(reg_addr + reg);
+}
+
+#define hclgevf_write_dev(a, reg, value) \
+	hclgevf_write_reg((a)->io_base, (reg), (value))
+#define hclgevf_read_dev(a, reg) \
+	hclgevf_read_reg((a)->io_base, (reg))
+
+#define HCLGEVF_SEND_SYNC(flag) \
+	((flag) & HCLGEVF_CMD_FLAG_NO_INTR)
+
+int hclgevf_cmd_init(struct hclgevf_dev *hdev);
+void hclgevf_cmd_uninit(struct hclgevf_dev *hdev);
+
+int hclgevf_cmd_send(struct hclgevf_hw *hw, struct hclgevf_desc *desc, int num);
+void hclgevf_cmd_setup_basic_desc(struct hclgevf_desc *desc,
+				  enum hclgevf_opcode_type opcode,
+				  bool is_read);
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
new file mode 100644
index 000000000000..0d89965f7928
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
@@ -0,0 +1,1505 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2016-2017 Hisilicon Limited.
+
+#include <linux/etherdevice.h>
+#include "hclgevf_cmd.h"
+#include "hclgevf_main.h"
+#include "hclge_mbx.h"
+#include "hnae3.h"
+
+#define HCLGEVF_NAME	"hclgevf"
+
+static struct hnae3_ae_algo ae_algovf;
+
+static const struct pci_device_id ae_algovf_pci_tbl[] = {
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
+	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
+	/* required last entry */
+	{0, }
+};
+
+static inline struct hclgevf_dev *hclgevf_ae_get_hdev(
+	struct hnae3_handle *handle)
+{
+	return container_of(handle, struct hclgevf_dev, nic);
+}
+
+static int hclgevf_tqps_update_stats(struct hnae3_handle *handle)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hnae3_queue *queue;
+	struct hclgevf_desc desc;
+	struct hclgevf_tqp *tqp;
+	int status;
+	int i;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		queue = handle->kinfo.tqp[i];
+		tqp = container_of(queue, struct hclgevf_tqp, q);
+		hclgevf_cmd_setup_basic_desc(&desc,
+					     HCLGEVF_OPC_QUERY_RX_STATUS,
+					     true);
+
+		desc.data[0] = cpu_to_le32(tqp->index & 0x1ff);
+		status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"Query tqp stat fail, status = %d,queue = %d\n",
+				status,	i);
+			return status;
+		}
+		tqp->tqp_stats.rcb_rx_ring_pktnum_rcd +=
+			le32_to_cpu(desc.data[1]);
+
+		hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_QUERY_TX_STATUS,
+					     true);
+
+		desc.data[0] = cpu_to_le32(tqp->index & 0x1ff);
+		status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"Query tqp stat fail, status = %d,queue = %d\n",
+				status, i);
+			return status;
+		}
+		tqp->tqp_stats.rcb_tx_ring_pktnum_rcd +=
+			le32_to_cpu(desc.data[1]);
+	}
+
+	return 0;
+}
+
+static u64 *hclgevf_tqps_get_stats(struct hnae3_handle *handle, u64 *data)
+{
+	struct hnae3_knic_private_info *kinfo = &handle->kinfo;
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hclgevf_tqp *tqp;
+	u64 *buff = data;
+	int i;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		tqp = container_of(handle->kinfo.tqp[i], struct hclgevf_tqp, q);
+		*buff++ = tqp->tqp_stats.rcb_tx_ring_pktnum_rcd;
+	}
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		tqp = container_of(handle->kinfo.tqp[i], struct hclgevf_tqp, q);
+		*buff++ = tqp->tqp_stats.rcb_rx_ring_pktnum_rcd;
+	}
+
+	return buff;
+}
+
+static int hclgevf_tqps_get_sset_count(struct hnae3_handle *handle, int strset)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	return hdev->num_tqps * 2;
+}
+
+static u8 *hclgevf_tqps_get_strings(struct hnae3_handle *handle, u8 *data)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	u8 *buff = data;
+	int i = 0;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		struct hclgevf_tqp *tqp = container_of(handle->kinfo.tqp[i],
+			struct hclgevf_tqp, q);
+		snprintf(buff, ETH_GSTRING_LEN, "txq#%d_pktnum_rcd",
+			 tqp->index);
+		buff += ETH_GSTRING_LEN;
+	}
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		struct hclgevf_tqp *tqp = container_of(handle->kinfo.tqp[i],
+			struct hclgevf_tqp, q);
+		snprintf(buff, ETH_GSTRING_LEN, "rxq#%d_pktnum_rcd",
+			 tqp->index);
+		buff += ETH_GSTRING_LEN;
+	}
+
+	return buff;
+}
+
+static void hclgevf_update_stats(struct hnae3_handle *handle,
+				 struct net_device_stats *net_stats)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	int status;
+
+	status = hclgevf_tqps_update_stats(handle);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"VF update of TQPS stats fail, status = %d.\n",
+			status);
+}
+
+static int hclgevf_get_sset_count(struct hnae3_handle *handle, int strset)
+{
+	if (strset == ETH_SS_TEST)
+		return -EOPNOTSUPP;
+	else if (strset == ETH_SS_STATS)
+		return hclgevf_tqps_get_sset_count(handle, strset);
+
+	return 0;
+}
+
+static void hclgevf_get_strings(struct hnae3_handle *handle, u32 strset,
+				u8 *data)
+{
+	u8 *p = (char *)data;
+
+	if (strset == ETH_SS_STATS)
+		p = hclgevf_tqps_get_strings(handle, p);
+}
+
+static void hclgevf_get_stats(struct hnae3_handle *handle, u64 *data)
+{
+	hclgevf_tqps_get_stats(handle, data);
+}
+
+static int hclgevf_get_tc_info(struct hclgevf_dev *hdev)
+{
+	u8 resp_msg;
+	int status;
+
+	status = hclgevf_send_mbx_msg(hdev, HCLGE_MBX_GET_TCINFO, 0, NULL, 0,
+				      true, &resp_msg, sizeof(u8));
+	if (status) {
+		dev_err(&hdev->pdev->dev,
+			"VF request to get TC info from PF failed %d",
+			status);
+		return status;
+	}
+
+	hdev->hw_tc_map = resp_msg;
+
+	return 0;
+}
+
+static int hclge_get_queue_info(struct hclgevf_dev *hdev)
+{
+#define HCLGEVF_TQPS_RSS_INFO_LEN	8
+	u8 resp_msg[HCLGEVF_TQPS_RSS_INFO_LEN];
+	int status;
+
+	status = hclgevf_send_mbx_msg(hdev, HCLGE_MBX_GET_QINFO, 0, NULL, 0,
+				      true, resp_msg,
+				      HCLGEVF_TQPS_RSS_INFO_LEN);
+	if (status) {
+		dev_err(&hdev->pdev->dev,
+			"VF request to get tqp info from PF failed %d",
+			status);
+		return status;
+	}
+
+	memcpy(&hdev->num_tqps, &resp_msg[0], sizeof(u16));
+	memcpy(&hdev->rss_size_max, &resp_msg[2], sizeof(u16));
+	memcpy(&hdev->num_desc, &resp_msg[4], sizeof(u16));
+	memcpy(&hdev->rx_buf_len, &resp_msg[6], sizeof(u16));
+
+	return 0;
+}
+
+static int hclgevf_alloc_tqps(struct hclgevf_dev *hdev)
+{
+	struct hclgevf_tqp *tqp;
+	int i;
+
+	hdev->htqp = devm_kcalloc(&hdev->pdev->dev, hdev->num_tqps,
+				  sizeof(struct hclgevf_tqp), GFP_KERNEL);
+	if (!hdev->htqp)
+		return -ENOMEM;
+
+	tqp = hdev->htqp;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		tqp->dev = &hdev->pdev->dev;
+		tqp->index = i;
+
+		tqp->q.ae_algo = &ae_algovf;
+		tqp->q.buf_size = hdev->rx_buf_len;
+		tqp->q.desc_num = hdev->num_desc;
+		tqp->q.io_base = hdev->hw.io_base + HCLGEVF_TQP_REG_OFFSET +
+			i * HCLGEVF_TQP_REG_SIZE;
+
+		tqp++;
+	}
+
+	return 0;
+}
+
+static int hclgevf_knic_setup(struct hclgevf_dev *hdev)
+{
+	struct hnae3_handle *nic = &hdev->nic;
+	struct hnae3_knic_private_info *kinfo;
+	u16 new_tqps = hdev->num_tqps;
+	int i;
+
+	kinfo = &nic->kinfo;
+	kinfo->num_tc = 0;
+	kinfo->num_desc = hdev->num_desc;
+	kinfo->rx_buf_len = hdev->rx_buf_len;
+	for (i = 0; i < HCLGEVF_MAX_TC_NUM; i++)
+		if (hdev->hw_tc_map & BIT(i))
+			kinfo->num_tc++;
+
+	kinfo->rss_size
+		= min_t(u16, hdev->rss_size_max, new_tqps / kinfo->num_tc);
+	new_tqps = kinfo->rss_size * kinfo->num_tc;
+	kinfo->num_tqps = min(new_tqps, hdev->num_tqps);
+
+	kinfo->tqp = devm_kcalloc(&hdev->pdev->dev, kinfo->num_tqps,
+				  sizeof(struct hnae3_queue *), GFP_KERNEL);
+	if (!kinfo->tqp)
+		return -ENOMEM;
+
+	for (i = 0; i < kinfo->num_tqps; i++) {
+		hdev->htqp[i].q.handle = &hdev->nic;
+		hdev->htqp[i].q.tqp_index = i;
+		kinfo->tqp[i] = &hdev->htqp[i].q;
+	}
+
+	return 0;
+}
+
+static void hclgevf_request_link_info(struct hclgevf_dev *hdev)
+{
+	int status;
+	u8 resp_msg;
+
+	status = hclgevf_send_mbx_msg(hdev, HCLGE_MBX_GET_LINK_STATUS, 0, NULL,
+				      0, false, &resp_msg, sizeof(u8));
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"VF failed to fetch link status(%d) from PF", status);
+}
+
+void hclgevf_update_link_status(struct hclgevf_dev *hdev, int link_state)
+{
+	struct hnae3_handle *handle = &hdev->nic;
+	struct hnae3_client *client;
+
+	client = handle->client;
+
+	if (link_state != hdev->hw.mac.link) {
+		client->ops->link_status_change(handle, !!link_state);
+		hdev->hw.mac.link = link_state;
+	}
+}
+
+static int hclgevf_set_handle_info(struct hclgevf_dev *hdev)
+{
+	struct hnae3_handle *nic = &hdev->nic;
+	int ret;
+
+	nic->ae_algo = &ae_algovf;
+	nic->pdev = hdev->pdev;
+	nic->numa_node_mask = hdev->numa_node_mask;
+	nic->flags |= HNAE3_SUPPORT_VF;
+
+	if (hdev->ae_dev->dev_type != HNAE3_DEV_KNIC) {
+		dev_err(&hdev->pdev->dev, "unsupported device type %d\n",
+			hdev->ae_dev->dev_type);
+		return -EINVAL;
+	}
+
+	ret = hclgevf_knic_setup(hdev);
+	if (ret)
+		dev_err(&hdev->pdev->dev, "VF knic setup failed %d\n",
+			ret);
+	return ret;
+}
+
+static void hclgevf_free_vector(struct hclgevf_dev *hdev, int vector_id)
+{
+	hdev->vector_status[vector_id] = HCLGEVF_INVALID_VPORT;
+	hdev->num_msi_left += 1;
+	hdev->num_msi_used -= 1;
+}
+
+static int hclgevf_get_vector(struct hnae3_handle *handle, u16 vector_num,
+			      struct hnae3_vector_info *vector_info)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hnae3_vector_info *vector = vector_info;
+	int alloc = 0;
+	int i, j;
+
+	vector_num = min(hdev->num_msi_left, vector_num);
+
+	for (j = 0; j < vector_num; j++) {
+		for (i = HCLGEVF_MISC_VECTOR_NUM + 1; i < hdev->num_msi; i++) {
+			if (hdev->vector_status[i] == HCLGEVF_INVALID_VPORT) {
+				vector->vector = pci_irq_vector(hdev->pdev, i);
+				vector->io_addr = hdev->hw.io_base +
+					HCLGEVF_VECTOR_REG_BASE +
+					(i - 1) * HCLGEVF_VECTOR_REG_OFFSET;
+				hdev->vector_status[i] = 0;
+				hdev->vector_irq[i] = vector->vector;
+
+				vector++;
+				alloc++;
+
+				break;
+			}
+		}
+	}
+	hdev->num_msi_left -= alloc;
+	hdev->num_msi_used += alloc;
+
+	return alloc;
+}
+
+static int hclgevf_get_vector_index(struct hclgevf_dev *hdev, int vector)
+{
+	int i;
+
+	for (i = 0; i < hdev->num_msi; i++)
+		if (vector == hdev->vector_irq[i])
+			return i;
+
+	return -EINVAL;
+}
+
+static u32 hclgevf_get_rss_key_size(struct hnae3_handle *handle)
+{
+	return HCLGEVF_RSS_KEY_SIZE;
+}
+
+static u32 hclgevf_get_rss_indir_size(struct hnae3_handle *handle)
+{
+	return HCLGEVF_RSS_IND_TBL_SIZE;
+}
+
+static int hclgevf_set_rss_indir_table(struct hclgevf_dev *hdev)
+{
+	const u8 *indir = hdev->rss_cfg.rss_indirection_tbl;
+	struct hclgevf_rss_indirection_table_cmd *req;
+	struct hclgevf_desc desc;
+	int status;
+	int i, j;
+
+	req = (struct hclgevf_rss_indirection_table_cmd *)desc.data;
+
+	for (i = 0; i < HCLGEVF_RSS_CFG_TBL_NUM; i++) {
+		hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_RSS_INDIR_TABLE,
+					     false);
+		req->start_table_index = i * HCLGEVF_RSS_CFG_TBL_SIZE;
+		req->rss_set_bitmap = HCLGEVF_RSS_SET_BITMAP_MSK;
+		for (j = 0; j < HCLGEVF_RSS_CFG_TBL_SIZE; j++)
+			req->rss_result[j] =
+				indir[i * HCLGEVF_RSS_CFG_TBL_SIZE + j];
+
+		status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"VF failed(=%d) to set RSS indirection table\n",
+				status);
+			return status;
+		}
+	}
+
+	return 0;
+}
+
+static int hclgevf_set_rss_tc_mode(struct hclgevf_dev *hdev,  u16 rss_size)
+{
+	struct hclgevf_rss_tc_mode_cmd *req;
+	u16 tc_offset[HCLGEVF_MAX_TC_NUM];
+	u16 tc_valid[HCLGEVF_MAX_TC_NUM];
+	u16 tc_size[HCLGEVF_MAX_TC_NUM];
+	struct hclgevf_desc desc;
+	u16 roundup_size;
+	int status;
+	int i;
+
+	req = (struct hclgevf_rss_tc_mode_cmd *)desc.data;
+
+	roundup_size = roundup_pow_of_two(rss_size);
+	roundup_size = ilog2(roundup_size);
+
+	for (i = 0; i < HCLGEVF_MAX_TC_NUM; i++) {
+		tc_valid[i] = !!(hdev->hw_tc_map & BIT(i));
+		tc_size[i] = roundup_size;
+		tc_offset[i] = rss_size * i;
+	}
+
+	hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_RSS_TC_MODE, false);
+	for (i = 0; i < HCLGEVF_MAX_TC_NUM; i++) {
+		hnae_set_bit(req->rss_tc_mode[i], HCLGEVF_RSS_TC_VALID_B,
+			     (tc_valid[i] & 0x1));
+		hnae_set_field(req->rss_tc_mode[i], HCLGEVF_RSS_TC_SIZE_M,
+			       HCLGEVF_RSS_TC_SIZE_S, tc_size[i]);
+		hnae_set_field(req->rss_tc_mode[i], HCLGEVF_RSS_TC_OFFSET_M,
+			       HCLGEVF_RSS_TC_OFFSET_S, tc_offset[i]);
+	}
+	status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"VF failed(=%d) to set rss tc mode\n", status);
+
+	return status;
+}
+
+static int hclgevf_get_rss_hw_cfg(struct hnae3_handle *handle, u8 *hash,
+				  u8 *key)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hclgevf_rss_config_cmd *req;
+	int lkup_times = key ? 3 : 1;
+	struct hclgevf_desc desc;
+	int key_offset;
+	int key_size;
+	int status;
+
+	req = (struct hclgevf_rss_config_cmd *)desc.data;
+	lkup_times = (lkup_times == 3) ? 3 : ((hash) ? 1 : 0);
+
+	for (key_offset = 0; key_offset < lkup_times; key_offset++) {
+		hclgevf_cmd_setup_basic_desc(&desc,
+					     HCLGEVF_OPC_RSS_GENERIC_CONFIG,
+					     true);
+		req->hash_config |= (key_offset << HCLGEVF_RSS_HASH_KEY_OFFSET);
+
+		status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"failed to get hardware RSS cfg, status = %d\n",
+				status);
+			return status;
+		}
+
+		if (key_offset == 2)
+			key_size =
+			HCLGEVF_RSS_KEY_SIZE - HCLGEVF_RSS_HASH_KEY_NUM * 2;
+		else
+			key_size = HCLGEVF_RSS_HASH_KEY_NUM;
+
+		if (key)
+			memcpy(key + key_offset * HCLGEVF_RSS_HASH_KEY_NUM,
+			       req->hash_key,
+			       key_size);
+	}
+
+	if (hash) {
+		if ((req->hash_config & 0xf) == HCLGEVF_RSS_HASH_ALGO_TOEPLITZ)
+			*hash = ETH_RSS_HASH_TOP;
+		else
+			*hash = ETH_RSS_HASH_UNKNOWN;
+	}
+
+	return 0;
+}
+
+static int hclgevf_get_rss(struct hnae3_handle *handle, u32 *indir, u8 *key,
+			   u8 *hfunc)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hclgevf_rss_cfg *rss_cfg = &hdev->rss_cfg;
+	int i;
+
+	if (indir)
+		for (i = 0; i < HCLGEVF_RSS_IND_TBL_SIZE; i++)
+			indir[i] = rss_cfg->rss_indirection_tbl[i];
+
+	return hclgevf_get_rss_hw_cfg(handle, hfunc, key);
+}
+
+static int hclgevf_set_rss(struct hnae3_handle *handle, const u32 *indir,
+			   const  u8 *key, const  u8 hfunc)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hclgevf_rss_cfg *rss_cfg = &hdev->rss_cfg;
+	int i;
+
+	/* update the shadow RSS table with user specified qids */
+	for (i = 0; i < HCLGEVF_RSS_IND_TBL_SIZE; i++)
+		rss_cfg->rss_indirection_tbl[i] = indir[i];
+
+	/* update the hardware */
+	return hclgevf_set_rss_indir_table(hdev);
+}
+
+static int hclgevf_get_tc_size(struct hnae3_handle *handle)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hclgevf_rss_cfg *rss_cfg = &hdev->rss_cfg;
+
+	return rss_cfg->rss_size;
+}
+
+static int hclgevf_bind_ring_to_vector(struct hnae3_handle *handle, bool en,
+				       int vector,
+				       struct hnae3_ring_chain_node *ring_chain)
+{
+#define HCLGEVF_RING_NODE_VARIABLE_NUM		3
+#define HCLGEVF_RING_MAP_MBX_BASIC_MSG_NUM	3
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hnae3_ring_chain_node *node;
+	struct hclge_mbx_vf_to_pf_cmd *req;
+	struct hclgevf_desc desc;
+	int i, vector_id;
+	int status;
+	u8 type;
+
+	req = (struct hclge_mbx_vf_to_pf_cmd *)desc.data;
+	vector_id = hclgevf_get_vector_index(hdev, vector);
+	if (vector_id < 0) {
+		dev_err(&handle->pdev->dev,
+			"Get vector index fail. ret =%d\n", vector_id);
+		return vector_id;
+	}
+
+	hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_MBX_VF_TO_PF, false);
+	type = en ?
+		HCLGE_MBX_MAP_RING_TO_VECTOR : HCLGE_MBX_UNMAP_RING_TO_VECTOR;
+	req->msg[0] = type;
+	req->msg[1] = vector_id; /* vector_id should be id in VF */
+
+	i = 0;
+	for (node = ring_chain; node; node = node->next) {
+		i++;
+		/* msg[2] is cause num */
+		req->msg[HCLGEVF_RING_NODE_VARIABLE_NUM * i] =
+				hnae_get_bit(node->flag, HNAE3_RING_TYPE_B);
+		req->msg[HCLGEVF_RING_NODE_VARIABLE_NUM * i + 1] =
+				node->tqp_index;
+		req->msg[HCLGEVF_RING_NODE_VARIABLE_NUM * i + 2] =
+				hnae_get_field(node->int_gl_idx,
+					       HNAE3_RING_GL_IDX_M,
+					       HNAE3_RING_GL_IDX_S);
+
+		if (i == (HCLGE_MBX_VF_MSG_DATA_NUM -
+		    HCLGEVF_RING_MAP_MBX_BASIC_MSG_NUM) /
+		    HCLGEVF_RING_NODE_VARIABLE_NUM) {
+			req->msg[2] = i;
+
+			status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+			if (status) {
+				dev_err(&hdev->pdev->dev,
+					"Map TQP fail, status is %d.\n",
+					status);
+				return status;
+			}
+			i = 0;
+			hclgevf_cmd_setup_basic_desc(&desc,
+						     HCLGEVF_OPC_MBX_VF_TO_PF,
+						     false);
+			req->msg[0] = type;
+			req->msg[1] = vector_id;
+		}
+	}
+
+	if (i > 0) {
+		req->msg[2] = i;
+
+		status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"Map TQP fail, status is %d.\n", status);
+			return status;
+		}
+	}
+
+	return 0;
+}
+
+static int hclgevf_map_ring_to_vector(struct hnae3_handle *handle, int vector,
+				      struct hnae3_ring_chain_node *ring_chain)
+{
+	return hclgevf_bind_ring_to_vector(handle, true, vector, ring_chain);
+}
+
+static int hclgevf_unmap_ring_from_vector(
+				struct hnae3_handle *handle,
+				int vector,
+				struct hnae3_ring_chain_node *ring_chain)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	int ret, vector_id;
+
+	vector_id = hclgevf_get_vector_index(hdev, vector);
+	if (vector_id < 0) {
+		dev_err(&handle->pdev->dev,
+			"Get vector index fail. ret =%d\n", vector_id);
+		return vector_id;
+	}
+
+	ret = hclgevf_bind_ring_to_vector(handle, false, vector, ring_chain);
+	if (ret) {
+		dev_err(&handle->pdev->dev,
+			"Unmap ring from vector fail. vector=%d, ret =%d\n",
+			vector_id,
+			ret);
+		return ret;
+	}
+
+	hclgevf_free_vector(hdev, vector);
+
+	return 0;
+}
+
+static int hclgevf_cmd_set_promisc_mode(struct hclgevf_dev *hdev, u32 en)
+{
+	struct hclge_mbx_vf_to_pf_cmd *req;
+	struct hclgevf_desc desc;
+	int status;
+
+	req = (struct hclge_mbx_vf_to_pf_cmd *)desc.data;
+
+	hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_MBX_VF_TO_PF, false);
+	req->msg[0] = HCLGE_MBX_SET_PROMISC_MODE;
+	req->msg[1] = en;
+
+	status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"Set promisc mode fail, status is %d.\n", status);
+
+	return status;
+}
+
+static void hclgevf_set_promisc_mode(struct hnae3_handle *handle, u32 en)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	hclgevf_cmd_set_promisc_mode(hdev, en);
+}
+
+static int hclgevf_tqp_enable(struct hclgevf_dev *hdev, int tqp_id,
+			      int stream_id, bool enable)
+{
+	struct hclgevf_cfg_com_tqp_queue_cmd *req;
+	struct hclgevf_desc desc;
+	int status;
+
+	req = (struct hclgevf_cfg_com_tqp_queue_cmd *)desc.data;
+
+	hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_CFG_COM_TQP_QUEUE,
+				     false);
+	req->tqp_id = cpu_to_le16(tqp_id & HCLGEVF_RING_ID_MASK);
+	req->stream_id = cpu_to_le16(stream_id);
+	req->enable |= enable << HCLGEVF_TQP_ENABLE_B;
+
+	status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+	if (status)
+		dev_err(&hdev->pdev->dev,
+			"TQP enable fail, status =%d.\n", status);
+
+	return status;
+}
+
+static int hclgevf_get_queue_id(struct hnae3_queue *queue)
+{
+	struct hclgevf_tqp *tqp = container_of(queue, struct hclgevf_tqp, q);
+
+	return tqp->index;
+}
+
+static void hclgevf_reset_tqp_stats(struct hnae3_handle *handle)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	struct hnae3_queue *queue;
+	struct hclgevf_tqp *tqp;
+	int i;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		queue = handle->kinfo.tqp[i];
+		tqp = container_of(queue, struct hclgevf_tqp, q);
+		memset(&tqp->tqp_stats, 0, sizeof(tqp->tqp_stats));
+	}
+}
+
+static int hclgevf_cfg_func_mta_filter(struct hnae3_handle *handle, bool en)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	u8 msg[2] = {0};
+
+	msg[0] = en;
+	return hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_MULTICAST,
+				    HCLGE_MBX_MAC_VLAN_MC_FUNC_MTA_ENABLE,
+				    msg, 1, false, NULL, 0);
+}
+
+static void hclgevf_get_mac_addr(struct hnae3_handle *handle, u8 *p)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	ether_addr_copy(p, hdev->hw.mac.mac_addr);
+}
+
+static int hclgevf_set_mac_addr(struct hnae3_handle *handle, void *p)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	u8 *old_mac_addr = (u8 *)hdev->hw.mac.mac_addr;
+	u8 *new_mac_addr = (u8 *)p;
+	u8 msg_data[ETH_ALEN * 2];
+	int status;
+
+	ether_addr_copy(msg_data, new_mac_addr);
+	ether_addr_copy(&msg_data[ETH_ALEN], old_mac_addr);
+
+	status = hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_UNICAST,
+				      HCLGE_MBX_MAC_VLAN_UC_MODIFY,
+				      msg_data, ETH_ALEN * 2,
+				      false, NULL, 0);
+	if (!status)
+		ether_addr_copy(hdev->hw.mac.mac_addr, new_mac_addr);
+
+	return status;
+}
+
+static int hclgevf_add_uc_addr(struct hnae3_handle *handle,
+			       const unsigned char *addr)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	return hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_UNICAST,
+				    HCLGE_MBX_MAC_VLAN_UC_ADD,
+				    addr, ETH_ALEN, false, NULL, 0);
+}
+
+static int hclgevf_rm_uc_addr(struct hnae3_handle *handle,
+			      const unsigned char *addr)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	return hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_UNICAST,
+				    HCLGE_MBX_MAC_VLAN_UC_REMOVE,
+				    addr, ETH_ALEN, false, NULL, 0);
+}
+
+static int hclgevf_add_mc_addr(struct hnae3_handle *handle,
+			       const unsigned char *addr)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	return hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_MULTICAST,
+				    HCLGE_MBX_MAC_VLAN_MC_ADD,
+				    addr, ETH_ALEN, false, NULL, 0);
+}
+
+static int hclgevf_rm_mc_addr(struct hnae3_handle *handle,
+			      const unsigned char *addr)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	return hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_MULTICAST,
+				    HCLGE_MBX_MAC_VLAN_MC_REMOVE,
+				    addr, ETH_ALEN, false, NULL, 0);
+}
+
+static int hclgevf_set_vlan_filter(struct hnae3_handle *handle,
+				   __be16 proto, u16 vlan_id,
+				   bool is_kill)
+{
+#define HCLGEVF_VLAN_MBX_MSG_LEN 5
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	u8 msg_data[HCLGEVF_VLAN_MBX_MSG_LEN];
+
+	if (vlan_id > 4095)
+		return -EINVAL;
+
+	if (proto != htons(ETH_P_8021Q))
+		return -EPROTONOSUPPORT;
+
+	msg_data[0] = is_kill;
+	memcpy(&msg_data[1], &vlan_id, sizeof(vlan_id));
+	memcpy(&msg_data[3], &proto, sizeof(proto));
+	return hclgevf_send_mbx_msg(hdev, HCLGE_MBX_SET_VLAN,
+				    HCLGE_MBX_VLAN_FILTER, msg_data,
+				    HCLGEVF_VLAN_MBX_MSG_LEN, false, NULL, 0);
+}
+
+static void hclgevf_reset_tqp(struct hnae3_handle *handle, u16 queue_id)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	u8 msg_data[2];
+
+	memcpy(&msg_data[0], &queue_id, sizeof(queue_id));
+
+	hclgevf_send_mbx_msg(hdev, HCLGE_MBX_QUEUE_RESET, 0, msg_data, 2, false,
+			     NULL, 0);
+}
+
+static u32 hclgevf_get_fw_version(struct hnae3_handle *handle)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	return hdev->fw_version;
+}
+
+static void hclgevf_get_misc_vector(struct hclgevf_dev *hdev)
+{
+	struct hclgevf_misc_vector *vector = &hdev->misc_vector;
+
+	vector->vector_irq = pci_irq_vector(hdev->pdev,
+					    HCLGEVF_MISC_VECTOR_NUM);
+	vector->addr = hdev->hw.io_base + HCLGEVF_MISC_VECTOR_REG_BASE;
+	/* vector status always valid for Vector 0 */
+	hdev->vector_status[HCLGEVF_MISC_VECTOR_NUM] = 0;
+	hdev->vector_irq[HCLGEVF_MISC_VECTOR_NUM] = vector->vector_irq;
+
+	hdev->num_msi_left -= 1;
+	hdev->num_msi_used += 1;
+}
+
+static void hclgevf_mbx_task_schedule(struct hclgevf_dev *hdev)
+{
+	if (!test_and_set_bit(HCLGEVF_STATE_MBX_SERVICE_SCHED, &hdev->state))
+		schedule_work(&hdev->mbx_service_task);
+}
+
+static void hclgevf_task_schedule(struct hclgevf_dev *hdev)
+{
+	if (!test_bit(HCLGEVF_STATE_DOWN, &hdev->state)  &&
+	    !test_and_set_bit(HCLGEVF_STATE_SERVICE_SCHED, &hdev->state))
+		schedule_work(&hdev->service_task);
+}
+
+static void hclgevf_service_timer(struct timer_list *t)
+{
+	struct hclgevf_dev *hdev = from_timer(hdev, t, service_timer);
+
+	mod_timer(&hdev->service_timer, jiffies + 5 * HZ);
+
+	hclgevf_task_schedule(hdev);
+}
+
+static void hclgevf_mailbox_service_task(struct work_struct *work)
+{
+	struct hclgevf_dev *hdev;
+
+	hdev = container_of(work, struct hclgevf_dev, mbx_service_task);
+
+	if (test_and_set_bit(HCLGEVF_STATE_MBX_HANDLING, &hdev->state))
+		return;
+
+	clear_bit(HCLGEVF_STATE_MBX_SERVICE_SCHED, &hdev->state);
+
+	hclgevf_mbx_handler(hdev);
+
+	clear_bit(HCLGEVF_STATE_MBX_HANDLING, &hdev->state);
+}
+
+static void hclgevf_service_task(struct work_struct *work)
+{
+	struct hclgevf_dev *hdev;
+
+	hdev = container_of(work, struct hclgevf_dev, service_task);
+
+	/* request the link status from the PF. PF would be able to tell VF
+	 * about such updates in future so we might remove this later
+	 */
+	hclgevf_request_link_info(hdev);
+
+	clear_bit(HCLGEVF_STATE_SERVICE_SCHED, &hdev->state);
+}
+
+static void hclgevf_clear_event_cause(struct hclgevf_dev *hdev, u32 regclr)
+{
+	hclgevf_write_dev(&hdev->hw, HCLGEVF_VECTOR0_CMDQ_SRC_REG, regclr);
+}
+
+static bool hclgevf_check_event_cause(struct hclgevf_dev *hdev, u32 *clearval)
+{
+	u32 cmdq_src_reg;
+
+	/* fetch the events from their corresponding regs */
+	cmdq_src_reg = hclgevf_read_dev(&hdev->hw,
+					HCLGEVF_VECTOR0_CMDQ_SRC_REG);
+
+	/* check for vector0 mailbox(=CMDQ RX) event source */
+	if (BIT(HCLGEVF_VECTOR0_RX_CMDQ_INT_B) & cmdq_src_reg) {
+		cmdq_src_reg &= ~BIT(HCLGEVF_VECTOR0_RX_CMDQ_INT_B);
+		*clearval = cmdq_src_reg;
+		return true;
+	}
+
+	dev_dbg(&hdev->pdev->dev, "vector 0 interrupt from unknown source\n");
+
+	return false;
+}
+
+static void hclgevf_enable_vector(struct hclgevf_misc_vector *vector, bool en)
+{
+	writel(en ? 1 : 0, vector->addr);
+}
+
+static irqreturn_t hclgevf_misc_irq_handle(int irq, void *data)
+{
+	struct hclgevf_dev *hdev = data;
+	u32 clearval;
+
+	hclgevf_enable_vector(&hdev->misc_vector, false);
+	if (!hclgevf_check_event_cause(hdev, &clearval))
+		goto skip_sched;
+
+	/* schedule the VF mailbox service task, if not already scheduled */
+	hclgevf_mbx_task_schedule(hdev);
+
+	hclgevf_clear_event_cause(hdev, clearval);
+
+skip_sched:
+	hclgevf_enable_vector(&hdev->misc_vector, true);
+
+	return IRQ_HANDLED;
+}
+
+static int hclgevf_configure(struct hclgevf_dev *hdev)
+{
+	int ret;
+
+	/* get queue configuration from PF */
+	ret = hclge_get_queue_info(hdev);
+	if (ret)
+		return ret;
+	/* get tc configuration from PF */
+	return hclgevf_get_tc_info(hdev);
+}
+
+static int hclgevf_init_roce_base_info(struct hclgevf_dev *hdev)
+{
+	struct hnae3_handle *roce = &hdev->roce;
+	struct hnae3_handle *nic = &hdev->nic;
+
+	roce->rinfo.num_vectors = HCLGEVF_ROCEE_VECTOR_NUM;
+
+	if (hdev->num_msi_left < roce->rinfo.num_vectors ||
+	    hdev->num_msi_left == 0)
+		return -EINVAL;
+
+	roce->rinfo.base_vector =
+		hdev->vector_status[hdev->num_msi_used];
+
+	roce->rinfo.netdev = nic->kinfo.netdev;
+	roce->rinfo.roce_io_base = hdev->hw.io_base;
+
+	roce->pdev = nic->pdev;
+	roce->ae_algo = nic->ae_algo;
+	roce->numa_node_mask = nic->numa_node_mask;
+
+	return 0;
+}
+
+static int hclgevf_rss_init_hw(struct hclgevf_dev *hdev)
+{
+	struct hclgevf_rss_cfg *rss_cfg = &hdev->rss_cfg;
+	int i, ret;
+
+	rss_cfg->rss_size = hdev->rss_size_max;
+
+	/* Initialize RSS indirect table for each vport */
+	for (i = 0; i < HCLGEVF_RSS_IND_TBL_SIZE; i++)
+		rss_cfg->rss_indirection_tbl[i] = i % hdev->rss_size_max;
+
+	ret = hclgevf_set_rss_indir_table(hdev);
+	if (ret)
+		return ret;
+
+	return hclgevf_set_rss_tc_mode(hdev, hdev->rss_size_max);
+}
+
+static int hclgevf_init_vlan_config(struct hclgevf_dev *hdev)
+{
+	/* other vlan config(like, VLAN TX/RX offload) would also be added
+	 * here later
+	 */
+	return hclgevf_set_vlan_filter(&hdev->nic, htons(ETH_P_8021Q), 0,
+				       false);
+}
+
+static int hclgevf_ae_start(struct hnae3_handle *handle)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	int i, queue_id;
+
+	for (i = 0; i < handle->kinfo.num_tqps; i++) {
+		/* ring enable */
+		queue_id = hclgevf_get_queue_id(handle->kinfo.tqp[i]);
+		if (queue_id < 0) {
+			dev_warn(&hdev->pdev->dev,
+				 "Get invalid queue id, ignore it\n");
+			continue;
+		}
+
+		hclgevf_tqp_enable(hdev, queue_id, 0, true);
+	}
+
+	/* reset tqp stats */
+	hclgevf_reset_tqp_stats(handle);
+
+	hclgevf_request_link_info(hdev);
+
+	clear_bit(HCLGEVF_STATE_DOWN, &hdev->state);
+	mod_timer(&hdev->service_timer, jiffies + HZ);
+
+	return 0;
+}
+
+static void hclgevf_ae_stop(struct hnae3_handle *handle)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+	int i, queue_id;
+
+	for (i = 0; i < hdev->num_tqps; i++) {
+		/* Ring disable */
+		queue_id = hclgevf_get_queue_id(handle->kinfo.tqp[i]);
+		if (queue_id < 0) {
+			dev_warn(&hdev->pdev->dev,
+				 "Get invalid queue id, ignore it\n");
+			continue;
+		}
+
+		hclgevf_tqp_enable(hdev, queue_id, 0, false);
+	}
+
+	/* reset tqp stats */
+	hclgevf_reset_tqp_stats(handle);
+}
+
+static void hclgevf_state_init(struct hclgevf_dev *hdev)
+{
+	/* setup tasks for the MBX */
+	INIT_WORK(&hdev->mbx_service_task, hclgevf_mailbox_service_task);
+	clear_bit(HCLGEVF_STATE_MBX_SERVICE_SCHED, &hdev->state);
+	clear_bit(HCLGEVF_STATE_MBX_HANDLING, &hdev->state);
+
+	/* setup tasks for service timer */
+	timer_setup(&hdev->service_timer, hclgevf_service_timer, 0);
+
+	INIT_WORK(&hdev->service_task, hclgevf_service_task);
+	clear_bit(HCLGEVF_STATE_SERVICE_SCHED, &hdev->state);
+
+	mutex_init(&hdev->mbx_resp.mbx_mutex);
+
+	/* bring the device down */
+	set_bit(HCLGEVF_STATE_DOWN, &hdev->state);
+}
+
+static void hclgevf_state_uninit(struct hclgevf_dev *hdev)
+{
+	set_bit(HCLGEVF_STATE_DOWN, &hdev->state);
+
+	if (hdev->service_timer.function)
+		del_timer_sync(&hdev->service_timer);
+	if (hdev->service_task.func)
+		cancel_work_sync(&hdev->service_task);
+	if (hdev->mbx_service_task.func)
+		cancel_work_sync(&hdev->mbx_service_task);
+
+	mutex_destroy(&hdev->mbx_resp.mbx_mutex);
+}
+
+static int hclgevf_init_msi(struct hclgevf_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+	int vectors;
+	int i;
+
+	hdev->num_msi = HCLGEVF_MAX_VF_VECTOR_NUM;
+
+	vectors = pci_alloc_irq_vectors(pdev, 1, hdev->num_msi,
+					PCI_IRQ_MSI | PCI_IRQ_MSIX);
+	if (vectors < 0) {
+		dev_err(&pdev->dev,
+			"failed(%d) to allocate MSI/MSI-X vectors\n",
+			vectors);
+		return vectors;
+	}
+	if (vectors < hdev->num_msi)
+		dev_warn(&hdev->pdev->dev,
+			 "requested %d MSI/MSI-X, but allocated %d MSI/MSI-X\n",
+			 hdev->num_msi, vectors);
+
+	hdev->num_msi = vectors;
+	hdev->num_msi_left = vectors;
+	hdev->base_msi_vector = pdev->irq;
+
+	hdev->vector_status = devm_kcalloc(&pdev->dev, hdev->num_msi,
+					   sizeof(u16), GFP_KERNEL);
+	if (!hdev->vector_status) {
+		pci_free_irq_vectors(pdev);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < hdev->num_msi; i++)
+		hdev->vector_status[i] = HCLGEVF_INVALID_VPORT;
+
+	hdev->vector_irq = devm_kcalloc(&pdev->dev, hdev->num_msi,
+					sizeof(int), GFP_KERNEL);
+	if (!hdev->vector_irq) {
+		pci_free_irq_vectors(pdev);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void hclgevf_uninit_msi(struct hclgevf_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+
+	pci_free_irq_vectors(pdev);
+}
+
+static int hclgevf_misc_irq_init(struct hclgevf_dev *hdev)
+{
+	int ret = 0;
+
+	hclgevf_get_misc_vector(hdev);
+
+	ret = request_irq(hdev->misc_vector.vector_irq, hclgevf_misc_irq_handle,
+			  0, "hclgevf_cmd", hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev, "VF failed to request misc irq(%d)\n",
+			hdev->misc_vector.vector_irq);
+		return ret;
+	}
+
+	/* enable misc. vector(vector 0) */
+	hclgevf_enable_vector(&hdev->misc_vector, true);
+
+	return ret;
+}
+
+static void hclgevf_misc_irq_uninit(struct hclgevf_dev *hdev)
+{
+	/* disable misc vector(vector 0) */
+	hclgevf_enable_vector(&hdev->misc_vector, false);
+	free_irq(hdev->misc_vector.vector_irq, hdev);
+	hclgevf_free_vector(hdev, 0);
+}
+
+static int hclgevf_init_instance(struct hclgevf_dev *hdev,
+				 struct hnae3_client *client)
+{
+	int ret;
+
+	switch (client->type) {
+	case HNAE3_CLIENT_KNIC:
+		hdev->nic_client = client;
+		hdev->nic.client = client;
+
+		ret = client->ops->init_instance(&hdev->nic);
+		if (ret)
+			return ret;
+
+		if (hdev->roce_client && hnae3_dev_roce_supported(hdev)) {
+			struct hnae3_client *rc = hdev->roce_client;
+
+			ret = hclgevf_init_roce_base_info(hdev);
+			if (ret)
+				return ret;
+			ret = rc->ops->init_instance(&hdev->roce);
+			if (ret)
+				return ret;
+		}
+		break;
+	case HNAE3_CLIENT_UNIC:
+		hdev->nic_client = client;
+		hdev->nic.client = client;
+
+		ret = client->ops->init_instance(&hdev->nic);
+		if (ret)
+			return ret;
+		break;
+	case HNAE3_CLIENT_ROCE:
+		hdev->roce_client = client;
+		hdev->roce.client = client;
+
+		if (hdev->roce_client && hnae3_dev_roce_supported(hdev)) {
+			ret = hclgevf_init_roce_base_info(hdev);
+			if (ret)
+				return ret;
+
+			ret = client->ops->init_instance(&hdev->roce);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void hclgevf_uninit_instance(struct hclgevf_dev *hdev,
+				    struct hnae3_client *client)
+{
+	/* un-init roce, if it exists */
+	if (hdev->roce_client)
+		hdev->roce_client->ops->uninit_instance(&hdev->roce, 0);
+
+	/* un-init nic/unic, if this was not called by roce client */
+	if ((client->ops->uninit_instance) &&
+	    (client->type != HNAE3_CLIENT_ROCE))
+		client->ops->uninit_instance(&hdev->nic, 0);
+}
+
+static int hclgevf_register_client(struct hnae3_client *client,
+				   struct hnae3_ae_dev *ae_dev)
+{
+	struct hclgevf_dev *hdev = ae_dev->priv;
+
+	return hclgevf_init_instance(hdev, client);
+}
+
+static void hclgevf_unregister_client(struct hnae3_client *client,
+				      struct hnae3_ae_dev *ae_dev)
+{
+	struct hclgevf_dev *hdev = ae_dev->priv;
+
+	hclgevf_uninit_instance(hdev, client);
+}
+
+static int hclgevf_pci_init(struct hclgevf_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+	struct hclgevf_hw *hw;
+	int ret;
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to enable PCI device\n");
+		goto err_no_drvdata;
+	}
+
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		dev_err(&pdev->dev, "can't set consistent PCI DMA, exiting");
+		goto err_disable_device;
+	}
+
+	ret = pci_request_regions(pdev, HCLGEVF_DRIVER_NAME);
+	if (ret) {
+		dev_err(&pdev->dev, "PCI request regions failed %d\n", ret);
+		goto err_disable_device;
+	}
+
+	pci_set_master(pdev);
+	hw = &hdev->hw;
+	hw->hdev = hdev;
+	hw->io_base = pci_iomap(pdev, 2, 0);
+	if (!hw->io_base) {
+		dev_err(&pdev->dev, "can't map configuration register space\n");
+		ret = -ENOMEM;
+		goto err_clr_master;
+	}
+
+	return 0;
+
+err_clr_master:
+	pci_clear_master(pdev);
+	pci_release_regions(pdev);
+err_disable_device:
+	pci_disable_device(pdev);
+err_no_drvdata:
+	pci_set_drvdata(pdev, NULL);
+	return ret;
+}
+
+static void hclgevf_pci_uninit(struct hclgevf_dev *hdev)
+{
+	struct pci_dev *pdev = hdev->pdev;
+
+	pci_iounmap(pdev, hdev->hw.io_base);
+	pci_clear_master(pdev);
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static int hclgevf_init_ae_dev(struct hnae3_ae_dev *ae_dev)
+{
+	struct pci_dev *pdev = ae_dev->pdev;
+	struct hclgevf_dev *hdev;
+	int ret;
+
+	hdev = devm_kzalloc(&pdev->dev, sizeof(*hdev), GFP_KERNEL);
+	if (!hdev)
+		return -ENOMEM;
+
+	hdev->pdev = pdev;
+	hdev->ae_dev = ae_dev;
+	ae_dev->priv = hdev;
+
+	ret = hclgevf_pci_init(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "PCI initialization failed\n");
+		return ret;
+	}
+
+	ret = hclgevf_init_msi(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed(%d) to init MSI/MSI-X\n", ret);
+		goto err_irq_init;
+	}
+
+	hclgevf_state_init(hdev);
+
+	ret = hclgevf_misc_irq_init(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed(%d) to init Misc IRQ(vector0)\n",
+			ret);
+		goto err_misc_irq_init;
+	}
+
+	ret = hclgevf_cmd_init(hdev);
+	if (ret)
+		goto err_cmd_init;
+
+	ret = hclgevf_configure(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed(%d) to fetch configuration\n", ret);
+		goto err_config;
+	}
+
+	ret = hclgevf_alloc_tqps(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed(%d) to allocate TQPs\n", ret);
+		goto err_config;
+	}
+
+	ret = hclgevf_set_handle_info(hdev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed(%d) to set handle info\n", ret);
+		goto err_config;
+	}
+
+	/* Initialize VF's MTA */
+	hdev->accept_mta_mc = true;
+	ret = hclgevf_cfg_func_mta_filter(&hdev->nic, hdev->accept_mta_mc);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed(%d) to set mta filter mode\n", ret);
+		goto err_config;
+	}
+
+	/* Initialize RSS for this VF */
+	ret = hclgevf_rss_init_hw(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed(%d) to initialize RSS\n", ret);
+		goto err_config;
+	}
+
+	ret = hclgevf_init_vlan_config(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed(%d) to initialize VLAN config\n", ret);
+		goto err_config;
+	}
+
+	pr_info("finished initializing %s driver\n", HCLGEVF_DRIVER_NAME);
+
+	return 0;
+
+err_config:
+	hclgevf_cmd_uninit(hdev);
+err_cmd_init:
+	hclgevf_misc_irq_uninit(hdev);
+err_misc_irq_init:
+	hclgevf_state_uninit(hdev);
+	hclgevf_uninit_msi(hdev);
+err_irq_init:
+	hclgevf_pci_uninit(hdev);
+	return ret;
+}
+
+static void hclgevf_uninit_ae_dev(struct hnae3_ae_dev *ae_dev)
+{
+	struct hclgevf_dev *hdev = ae_dev->priv;
+
+	hclgevf_cmd_uninit(hdev);
+	hclgevf_misc_irq_uninit(hdev);
+	hclgevf_state_uninit(hdev);
+	hclgevf_uninit_msi(hdev);
+	hclgevf_pci_uninit(hdev);
+	ae_dev->priv = NULL;
+}
+
+static u32 hclgevf_get_max_channels(struct hclgevf_dev *hdev)
+{
+	struct hnae3_handle *nic = &hdev->nic;
+	struct hnae3_knic_private_info *kinfo = &nic->kinfo;
+
+	return min_t(u32, hdev->rss_size_max * kinfo->num_tc, hdev->num_tqps);
+}
+
+/**
+ * hclgevf_get_channels - Get the current channels enabled and max supported.
+ * @handle: hardware information for network interface
+ * @ch: ethtool channels structure
+ *
+ * We don't support separate tx and rx queues as channels. The other count
+ * represents how many queues are being used for control. max_combined counts
+ * how many queue pairs we can support. They may not be mapped 1 to 1 with
+ * q_vectors since we support a lot more queue pairs than q_vectors.
+ **/
+static void hclgevf_get_channels(struct hnae3_handle *handle,
+				 struct ethtool_channels *ch)
+{
+	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
+
+	ch->max_combined = hclgevf_get_max_channels(hdev);
+	ch->other_count = 0;
+	ch->max_other = 0;
+	ch->combined_count = hdev->num_tqps;
+}
+
+static const struct hnae3_ae_ops hclgevf_ops = {
+	.init_ae_dev = hclgevf_init_ae_dev,
+	.uninit_ae_dev = hclgevf_uninit_ae_dev,
+	.init_client_instance = hclgevf_register_client,
+	.uninit_client_instance = hclgevf_unregister_client,
+	.start = hclgevf_ae_start,
+	.stop = hclgevf_ae_stop,
+	.map_ring_to_vector = hclgevf_map_ring_to_vector,
+	.unmap_ring_from_vector = hclgevf_unmap_ring_from_vector,
+	.get_vector = hclgevf_get_vector,
+	.reset_queue = hclgevf_reset_tqp,
+	.set_promisc_mode = hclgevf_set_promisc_mode,
+	.get_mac_addr = hclgevf_get_mac_addr,
+	.set_mac_addr = hclgevf_set_mac_addr,
+	.add_uc_addr = hclgevf_add_uc_addr,
+	.rm_uc_addr = hclgevf_rm_uc_addr,
+	.add_mc_addr = hclgevf_add_mc_addr,
+	.rm_mc_addr = hclgevf_rm_mc_addr,
+	.get_stats = hclgevf_get_stats,
+	.update_stats = hclgevf_update_stats,
+	.get_strings = hclgevf_get_strings,
+	.get_sset_count = hclgevf_get_sset_count,
+	.get_rss_key_size = hclgevf_get_rss_key_size,
+	.get_rss_indir_size = hclgevf_get_rss_indir_size,
+	.get_rss = hclgevf_get_rss,
+	.set_rss = hclgevf_set_rss,
+	.get_tc_size = hclgevf_get_tc_size,
+	.get_fw_version = hclgevf_get_fw_version,
+	.set_vlan_filter = hclgevf_set_vlan_filter,
+	.get_channels = hclgevf_get_channels,
+};
+
+static struct hnae3_ae_algo ae_algovf = {
+	.ops = &hclgevf_ops,
+	.name = HCLGEVF_NAME,
+	.pdev_id_table = ae_algovf_pci_tbl,
+};
+
+static int hclgevf_init(void)
+{
+	pr_info("%s is initializing\n", HCLGEVF_NAME);
+
+	return hnae3_register_ae_algo(&ae_algovf);
+}
+
+static void hclgevf_exit(void)
+{
+	hnae3_unregister_ae_algo(&ae_algovf);
+}
+module_init(hclgevf_init);
+module_exit(hclgevf_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
+MODULE_DESCRIPTION("HCLGEVF Driver");
+MODULE_VERSION(HCLGEVF_MOD_VERSION);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
new file mode 100644
index 000000000000..a63bee4a3674
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.h
@@ -0,0 +1,164 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/* Copyright (c) 2016-2017 Hisilicon Limited. */
+
+#ifndef __HCLGEVF_MAIN_H
+#define __HCLGEVF_MAIN_H
+#include <linux/fs.h>
+#include <linux/types.h>
+#include "hclge_mbx.h"
+#include "hclgevf_cmd.h"
+#include "hnae3.h"
+
+#define HCLGEVF_MOD_VERSION "v1.0"
+#define HCLGEVF_DRIVER_NAME "hclgevf"
+
+#define HCLGEVF_ROCEE_VECTOR_NUM	0
+#define HCLGEVF_MISC_VECTOR_NUM		0
+
+#define HCLGEVF_INVALID_VPORT		0xffff
+
+/* This number in actual depends upon the total number of VFs
+ * created by physical function. But the maximum number of
+ * possible vector-per-VF is {VFn(1-32), VECTn(32 + 1)}.
+ */
+#define HCLGEVF_MAX_VF_VECTOR_NUM	(32 + 1)
+
+#define HCLGEVF_VECTOR_REG_BASE		0x20000
+#define HCLGEVF_MISC_VECTOR_REG_BASE	0x20400
+#define HCLGEVF_VECTOR_REG_OFFSET	0x4
+#define HCLGEVF_VECTOR_VF_OFFSET		0x100000
+
+/* Vector0 interrupt CMDQ event source register(RW) */
+#define HCLGEVF_VECTOR0_CMDQ_SRC_REG	0x27100
+/* CMDQ register bits for RX event(=MBX event) */
+#define HCLGEVF_VECTOR0_RX_CMDQ_INT_B	1
+
+#define HCLGEVF_TQP_RESET_TRY_TIMES	10
+
+#define HCLGEVF_RSS_IND_TBL_SIZE		512
+#define HCLGEVF_RSS_SET_BITMAP_MSK	0xffff
+#define HCLGEVF_RSS_KEY_SIZE		40
+#define HCLGEVF_RSS_HASH_ALGO_TOEPLITZ	0
+#define HCLGEVF_RSS_HASH_ALGO_SIMPLE	1
+#define HCLGEVF_RSS_HASH_ALGO_SYMMETRIC	2
+#define HCLGEVF_RSS_HASH_ALGO_MASK	0xf
+#define HCLGEVF_RSS_CFG_TBL_NUM \
+	(HCLGEVF_RSS_IND_TBL_SIZE / HCLGEVF_RSS_CFG_TBL_SIZE)
+
+/* states of hclgevf device & tasks */
+enum hclgevf_states {
+	/* device states */
+	HCLGEVF_STATE_DOWN,
+	HCLGEVF_STATE_DISABLED,
+	/* task states */
+	HCLGEVF_STATE_SERVICE_SCHED,
+	HCLGEVF_STATE_MBX_SERVICE_SCHED,
+	HCLGEVF_STATE_MBX_HANDLING,
+};
+
+#define HCLGEVF_MPF_ENBALE 1
+
+struct hclgevf_mac {
+	u8 mac_addr[ETH_ALEN];
+	int link;
+};
+
+struct hclgevf_hw {
+	void __iomem *io_base;
+	int num_vec;
+	struct hclgevf_cmq cmq;
+	struct hclgevf_mac mac;
+	void *hdev; /* hchgevf device it is part of */
+};
+
+/* TQP stats */
+struct hlcgevf_tqp_stats {
+	/* query_tqp_tx_queue_statistics ,opcode id:  0x0B03 */
+	u64 rcb_tx_ring_pktnum_rcd; /* 32bit */
+	/* query_tqp_rx_queue_statistics ,opcode id:  0x0B13 */
+	u64 rcb_rx_ring_pktnum_rcd; /* 32bit */
+};
+
+struct hclgevf_tqp {
+	struct device *dev;	/* device for DMA mapping */
+	struct hnae3_queue q;
+	struct hlcgevf_tqp_stats tqp_stats;
+	u16 index;		/* global index in a NIC controller */
+
+	bool alloced;
+};
+
+struct hclgevf_cfg {
+	u8 vmdq_vport_num;
+	u8 tc_num;
+	u16 tqp_desc_num;
+	u16 rx_buf_len;
+	u8 phy_addr;
+	u8 media_type;
+	u8 mac_addr[ETH_ALEN];
+	u32 numa_node_map;
+};
+
+struct hclgevf_rss_cfg {
+	u8  rss_hash_key[HCLGEVF_RSS_KEY_SIZE]; /* user configured hash keys */
+	u32 hash_algo;
+	u32 rss_size;
+	u8 hw_tc_map;
+	u8  rss_indirection_tbl[HCLGEVF_RSS_IND_TBL_SIZE]; /* shadow table */
+};
+
+struct hclgevf_misc_vector {
+	u8 __iomem *addr;
+	int vector_irq;
+};
+
+struct hclgevf_dev {
+	struct pci_dev *pdev;
+	struct hnae3_ae_dev *ae_dev;
+	struct hclgevf_hw hw;
+	struct hclgevf_misc_vector misc_vector;
+	struct hclgevf_rss_cfg rss_cfg;
+	unsigned long state;
+
+	u32 fw_version;
+	u16 num_tqps;		/* num task queue pairs of this PF */
+
+	u16 alloc_rss_size;	/* allocated RSS task queue */
+	u16 rss_size_max;	/* HW defined max RSS task queue */
+
+	u16 num_alloc_vport;	/* num vports this driver supports */
+	u32 numa_node_mask;
+	u16 rx_buf_len;
+	u16 num_desc;
+	u8 hw_tc_map;
+
+	u16 num_msi;
+	u16 num_msi_left;
+	u16 num_msi_used;
+	u32 base_msi_vector;
+	u16 *vector_status;
+	int *vector_irq;
+
+	bool accept_mta_mc; /* whether to accept mta filter multicast */
+	struct hclgevf_mbx_resp_status mbx_resp; /* mailbox response */
+
+	struct timer_list service_timer;
+	struct work_struct service_task;
+	struct work_struct mbx_service_task;
+
+	struct hclgevf_tqp *htqp;
+
+	struct hnae3_handle nic;
+	struct hnae3_handle roce;
+
+	struct hnae3_client *nic_client;
+	struct hnae3_client *roce_client;
+	u32 flag;
+};
+
+int hclgevf_send_mbx_msg(struct hclgevf_dev *hdev, u16 code, u16 subcode,
+			 const u8 *msg_data, u8 msg_len, bool need_resp,
+			 u8 *resp_data, u16 resp_len);
+void hclgevf_mbx_handler(struct hclgevf_dev *hdev);
+void hclgevf_update_link_status(struct hclgevf_dev *hdev, int link_state);
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c
new file mode 100644
index 000000000000..e39cad285fa9
--- /dev/null
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_mbx.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2016-2017 Hisilicon Limited.
+
+#include "hclge_mbx.h"
+#include "hclgevf_main.h"
+#include "hnae3.h"
+
+static void hclgevf_reset_mbx_resp_status(struct hclgevf_dev *hdev)
+{
+	/* this function should be called with mbx_resp.mbx_mutex held
+	 * to prtect the received_response from race condition
+	 */
+	hdev->mbx_resp.received_resp  = false;
+	hdev->mbx_resp.origin_mbx_msg = 0;
+	hdev->mbx_resp.resp_status    = 0;
+	memset(hdev->mbx_resp.additional_info, 0, HCLGE_MBX_MAX_RESP_DATA_SIZE);
+}
+
+/* hclgevf_get_mbx_resp: used to get a response from PF after VF sends a mailbox
+ * message to PF.
+ * @hdev: pointer to struct hclgevf_dev
+ * @resp_msg: pointer to store the original message type and response status
+ * @len: the resp_msg data array length.
+ */
+static int hclgevf_get_mbx_resp(struct hclgevf_dev *hdev, u16 code0, u16 code1,
+				u8 *resp_data, u16 resp_len)
+{
+#define HCLGEVF_MAX_TRY_TIMES	500
+#define HCLGEVF_SLEEP_USCOEND	1000
+	struct hclgevf_mbx_resp_status *mbx_resp;
+	u16 r_code0, r_code1;
+	int i = 0;
+
+	if (resp_len > HCLGE_MBX_MAX_RESP_DATA_SIZE) {
+		dev_err(&hdev->pdev->dev,
+			"VF mbx response len(=%d) exceeds maximum(=%d)\n",
+			resp_len,
+			HCLGE_MBX_MAX_RESP_DATA_SIZE);
+		return -EINVAL;
+	}
+
+	while ((!hdev->mbx_resp.received_resp) && (i < HCLGEVF_MAX_TRY_TIMES)) {
+		udelay(HCLGEVF_SLEEP_USCOEND);
+		i++;
+	}
+
+	if (i >= HCLGEVF_MAX_TRY_TIMES) {
+		dev_err(&hdev->pdev->dev,
+			"VF could not get mbx resp(=%d) from PF in %d tries\n",
+			hdev->mbx_resp.received_resp, i);
+		return -EIO;
+	}
+
+	mbx_resp = &hdev->mbx_resp;
+	r_code0 = (u16)(mbx_resp->origin_mbx_msg >> 16);
+	r_code1 = (u16)(mbx_resp->origin_mbx_msg & 0xff);
+	if (resp_data)
+		memcpy(resp_data, &mbx_resp->additional_info[0], resp_len);
+
+	hclgevf_reset_mbx_resp_status(hdev);
+
+	if (!(r_code0 == code0 && r_code1 == code1 && !mbx_resp->resp_status)) {
+		dev_err(&hdev->pdev->dev,
+			"VF could not match resp code(code0=%d,code1=%d), %d",
+			code0, code1, mbx_resp->resp_status);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int hclgevf_send_mbx_msg(struct hclgevf_dev *hdev, u16 code, u16 subcode,
+			 const u8 *msg_data, u8 msg_len, bool need_resp,
+			 u8 *resp_data, u16 resp_len)
+{
+	struct hclge_mbx_vf_to_pf_cmd *req;
+	struct hclgevf_desc desc;
+	int status;
+
+	req = (struct hclge_mbx_vf_to_pf_cmd *)desc.data;
+
+	/* first two bytes are reserved for code & subcode */
+	if (msg_len > (HCLGE_MBX_MAX_MSG_SIZE - 2)) {
+		dev_err(&hdev->pdev->dev,
+			"VF send mbx msg fail, msg len %d exceeds max len %d\n",
+			msg_len, HCLGE_MBX_MAX_MSG_SIZE);
+		return -EINVAL;
+	}
+
+	hclgevf_cmd_setup_basic_desc(&desc, HCLGEVF_OPC_MBX_VF_TO_PF, false);
+	req->msg[0] = code;
+	req->msg[1] = subcode;
+	memcpy(&req->msg[2], msg_data, msg_len);
+
+	/* synchronous send */
+	if (need_resp) {
+		mutex_lock(&hdev->mbx_resp.mbx_mutex);
+		hclgevf_reset_mbx_resp_status(hdev);
+		status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"VF failed(=%d) to send mbx message to PF\n",
+				status);
+			mutex_unlock(&hdev->mbx_resp.mbx_mutex);
+			return status;
+		}
+
+		status = hclgevf_get_mbx_resp(hdev, code, subcode, resp_data,
+					      resp_len);
+		mutex_unlock(&hdev->mbx_resp.mbx_mutex);
+	} else {
+		/* asynchronous send */
+		status = hclgevf_cmd_send(&hdev->hw, &desc, 1);
+		if (status) {
+			dev_err(&hdev->pdev->dev,
+				"VF failed(=%d) to send mbx message to PF\n",
+				status);
+			return status;
+		}
+	}
+
+	return status;
+}
+
+void hclgevf_mbx_handler(struct hclgevf_dev *hdev)
+{
+	struct hclgevf_mbx_resp_status *resp;
+	struct hclge_mbx_pf_to_vf_cmd *req;
+	struct hclgevf_cmq_ring *crq;
+	struct hclgevf_desc *desc;
+	u16 link_status, flag;
+	u8 *temp;
+	int i;
+
+	resp = &hdev->mbx_resp;
+	crq = &hdev->hw.cmq.crq;
+
+	flag = le16_to_cpu(crq->desc[crq->next_to_use].flag);
+	while (hnae_get_bit(flag, HCLGEVF_CMDQ_RX_OUTVLD_B)) {
+		desc = &crq->desc[crq->next_to_use];
+		req = (struct hclge_mbx_pf_to_vf_cmd *)desc->data;
+
+		switch (req->msg[0]) {
+		case HCLGE_MBX_PF_VF_RESP:
+			if (resp->received_resp)
+				dev_warn(&hdev->pdev->dev,
+					 "VF mbx resp flag not clear(%d)\n",
+					 req->msg[1]);
+			resp->received_resp = true;
+
+			resp->origin_mbx_msg = (req->msg[1] << 16);
+			resp->origin_mbx_msg |= req->msg[2];
+			resp->resp_status = req->msg[3];
+
+			temp = (u8 *)&req->msg[4];
+			for (i = 0; i < HCLGE_MBX_MAX_RESP_DATA_SIZE; i++) {
+				resp->additional_info[i] = *temp;
+				temp++;
+			}
+			break;
+		case HCLGE_MBX_LINK_STAT_CHANGE:
+			link_status = le16_to_cpu(req->msg[1]);
+
+			/* update upper layer with new link link status */
+			hclgevf_update_link_status(hdev, link_status);
+
+			break;
+		default:
+			dev_err(&hdev->pdev->dev,
+				"VF received unsupported(%d) mbx msg from PF\n",
+				req->msg[0]);
+			break;
+		}
+		hclge_mbx_ring_ptr_move_crq(crq);
+		flag = le16_to_cpu(crq->desc[crq->next_to_use].flag);
+	}
+
+	/* Write back CMDQ_RQ header pointer, M7 need this pointer */
+	hclgevf_write_dev(&hdev->hw, HCLGEVF_NIC_CRQ_HEAD_REG,
+			  crq->next_to_use);
+}
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
index 7d95f0866fb0..28a81ac97af5 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
@@ -143,7 +143,7 @@ int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs,
 	struct hinic_hwif *hwif = cmdqs->hwif;
 	struct pci_dev *pdev = hwif->pdev;
 
-	cmdq_buf->buf = pci_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL,
+	cmdq_buf->buf = dma_pool_alloc(cmdqs->cmdq_buf_pool, GFP_KERNEL,
 				       &cmdq_buf->dma_addr);
 	if (!cmdq_buf->buf) {
 		dev_err(&pdev->dev, "Failed to allocate cmd from the pool\n");
@@ -161,7 +161,7 @@ int hinic_alloc_cmdq_buf(struct hinic_cmdqs *cmdqs,
 void hinic_free_cmdq_buf(struct hinic_cmdqs *cmdqs,
 			 struct hinic_cmdq_buf *cmdq_buf)
 {
-	pci_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr);
+	dma_pool_free(cmdqs->cmdq_buf_pool, cmdq_buf->buf, cmdq_buf->dma_addr);
 }
 
 static unsigned int cmdq_wqe_size_from_bdlen(enum bufdesc_len len)
@@ -875,7 +875,7 @@ int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
 	int err;
 
 	cmdqs->hwif = hwif;
-	cmdqs->cmdq_buf_pool = pci_pool_create("hinic_cmdq", pdev,
+	cmdqs->cmdq_buf_pool = dma_pool_create("hinic_cmdq", &pdev->dev,
 					       HINIC_CMDQ_BUF_SIZE,
 					       HINIC_CMDQ_BUF_SIZE, 0);
 	if (!cmdqs->cmdq_buf_pool)
@@ -916,7 +916,7 @@ int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
 	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
 
 err_saved_wqs:
-	pci_pool_destroy(cmdqs->cmdq_buf_pool);
+	dma_pool_destroy(cmdqs->cmdq_buf_pool);
 	return err;
 }
 
@@ -942,5 +942,5 @@ void hinic_free_cmdqs(struct hinic_cmdqs *cmdqs)
 
 	devm_kfree(&pdev->dev, cmdqs->saved_wqs);
 
-	pci_pool_destroy(cmdqs->cmdq_buf_pool);
+	dma_pool_destroy(cmdqs->cmdq_buf_pool);
 }
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
index b35583400cb6..23f8d39eab68 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
@@ -157,7 +157,7 @@ struct hinic_cmdq {
 struct hinic_cmdqs {
 	struct hinic_hwif       *hwif;
 
-	struct pci_pool         *cmdq_buf_pool;
+	struct dma_pool         *cmdq_buf_pool;
 
 	struct hinic_wq         *saved_wqs;
 
diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c
index 241db3199b88..354c0982847b 100644
--- a/drivers/net/ethernet/ibm/emac/core.c
+++ b/drivers/net/ethernet/ibm/emac/core.c
@@ -199,18 +199,18 @@ static void __emac_set_multicast_list(struct emac_instance *dev);
 
 static inline int emac_phy_supports_gige(int phy_mode)
 {
-	return  phy_mode == PHY_MODE_GMII ||
-		phy_mode == PHY_MODE_RGMII ||
-		phy_mode == PHY_MODE_SGMII ||
-		phy_mode == PHY_MODE_TBI ||
-		phy_mode == PHY_MODE_RTBI;
+	return  phy_interface_mode_is_rgmii(phy_mode) ||
+		phy_mode == PHY_INTERFACE_MODE_GMII ||
+		phy_mode == PHY_INTERFACE_MODE_SGMII ||
+		phy_mode == PHY_INTERFACE_MODE_TBI ||
+		phy_mode == PHY_INTERFACE_MODE_RTBI;
 }
 
 static inline int emac_phy_gpcs(int phy_mode)
 {
-	return  phy_mode == PHY_MODE_SGMII ||
-		phy_mode == PHY_MODE_TBI ||
-		phy_mode == PHY_MODE_RTBI;
+	return  phy_mode == PHY_INTERFACE_MODE_SGMII ||
+		phy_mode == PHY_INTERFACE_MODE_TBI ||
+		phy_mode == PHY_INTERFACE_MODE_RTBI;
 }
 
 static inline void emac_tx_enable(struct emac_instance *dev)
@@ -2871,7 +2871,7 @@ static int emac_init_config(struct emac_instance *dev)
 	/* PHY mode needs some decoding */
 	dev->phy_mode = of_get_phy_mode(np);
 	if (dev->phy_mode < 0)
-		dev->phy_mode = PHY_MODE_NA;
+		dev->phy_mode = PHY_INTERFACE_MODE_NA;
 
 	/* Check EMAC version */
 	if (of_device_is_compatible(np, "ibm,emac4sync")) {
@@ -3174,7 +3174,7 @@ static int emac_probe(struct platform_device *ofdev)
 	printk(KERN_INFO "%s: EMAC-%d %pOF, MAC %pM\n",
 	       ndev->name, dev->cell_index, np, ndev->dev_addr);
 
-	if (dev->phy_mode == PHY_MODE_SGMII)
+	if (dev->phy_mode == PHY_INTERFACE_MODE_SGMII)
 		printk(KERN_NOTICE "%s: in SGMII mode\n", ndev->name);
 
 	if (dev->phy.address >= 0)
diff --git a/drivers/net/ethernet/ibm/emac/emac.h b/drivers/net/ethernet/ibm/emac/emac.h
index c26d2631ca30..e2f80cca9bed 100644
--- a/drivers/net/ethernet/ibm/emac/emac.h
+++ b/drivers/net/ethernet/ibm/emac/emac.h
@@ -104,19 +104,6 @@ struct emac_regs {
 	} u1;
 };
 
-/*
- * PHY mode settings (EMAC <-> ZMII/RGMII bridge <-> PHY)
- */
-#define PHY_MODE_NA	PHY_INTERFACE_MODE_NA
-#define PHY_MODE_MII	PHY_INTERFACE_MODE_MII
-#define PHY_MODE_RMII	PHY_INTERFACE_MODE_RMII
-#define PHY_MODE_SMII	PHY_INTERFACE_MODE_SMII
-#define PHY_MODE_RGMII	PHY_INTERFACE_MODE_RGMII
-#define PHY_MODE_TBI	PHY_INTERFACE_MODE_TBI
-#define PHY_MODE_GMII	PHY_INTERFACE_MODE_GMII
-#define PHY_MODE_RTBI	PHY_INTERFACE_MODE_RTBI
-#define PHY_MODE_SGMII	PHY_INTERFACE_MODE_SGMII
-
 /* EMACx_MR0 */
 #define EMAC_MR0_RXI			0x80000000
 #define EMAC_MR0_TXI			0x40000000
diff --git a/drivers/net/ethernet/ibm/emac/phy.c b/drivers/net/ethernet/ibm/emac/phy.c
index 35865d05fccd..aa070c063e48 100644
--- a/drivers/net/ethernet/ibm/emac/phy.c
+++ b/drivers/net/ethernet/ibm/emac/phy.c
@@ -96,7 +96,7 @@ int emac_mii_reset_gpcs(struct mii_phy *phy)
 	if ((val & BMCR_ISOLATE) && limit > 0)
 		gpcs_phy_write(phy, MII_BMCR, val & ~BMCR_ISOLATE);
 
-	if (limit > 0 && phy->mode == PHY_MODE_SGMII) {
+	if (limit > 0 && phy->mode == PHY_INTERFACE_MODE_SGMII) {
 		/* Configure GPCS interface to recommended setting for SGMII */
 		gpcs_phy_write(phy, 0x04, 0x8120); /* AsymPause, FDX */
 		gpcs_phy_write(phy, 0x07, 0x2801); /* msg_pg, toggle */
@@ -313,16 +313,16 @@ static int cis8201_init(struct mii_phy *phy)
 	epcr &= ~EPCR_MODE_MASK;
 
 	switch (phy->mode) {
-	case PHY_MODE_TBI:
+	case PHY_INTERFACE_MODE_TBI:
 		epcr |= EPCR_TBI_MODE;
 		break;
-	case PHY_MODE_RTBI:
+	case PHY_INTERFACE_MODE_RTBI:
 		epcr |= EPCR_RTBI_MODE;
 		break;
-	case PHY_MODE_GMII:
+	case PHY_INTERFACE_MODE_GMII:
 		epcr |= EPCR_GMII_MODE;
 		break;
-	case PHY_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII:
 	default:
 		epcr |= EPCR_RGMII_MODE;
 	}
diff --git a/drivers/net/ethernet/ibm/emac/rgmii.c b/drivers/net/ethernet/ibm/emac/rgmii.c
index c4a1ac38bba8..00f5999de3cf 100644
--- a/drivers/net/ethernet/ibm/emac/rgmii.c
+++ b/drivers/net/ethernet/ibm/emac/rgmii.c
@@ -52,43 +52,28 @@
 /* RGMII bridge supports only GMII/TBI and RGMII/RTBI PHYs */
 static inline int rgmii_valid_mode(int phy_mode)
 {
-	return  phy_mode == PHY_MODE_GMII ||
-		phy_mode == PHY_MODE_MII ||
-		phy_mode == PHY_MODE_RGMII ||
-		phy_mode == PHY_MODE_TBI ||
-		phy_mode == PHY_MODE_RTBI;
-}
-
-static inline const char *rgmii_mode_name(int mode)
-{
-	switch (mode) {
-	case PHY_MODE_RGMII:
-		return "RGMII";
-	case PHY_MODE_TBI:
-		return "TBI";
-	case PHY_MODE_GMII:
-		return "GMII";
-	case PHY_MODE_MII:
-		return "MII";
-	case PHY_MODE_RTBI:
-		return "RTBI";
-	default:
-		BUG();
-	}
+	return  phy_interface_mode_is_rgmii(phy_mode) ||
+		phy_mode == PHY_INTERFACE_MODE_GMII ||
+		phy_mode == PHY_INTERFACE_MODE_MII ||
+		phy_mode == PHY_INTERFACE_MODE_TBI ||
+		phy_mode == PHY_INTERFACE_MODE_RTBI;
 }
 
 static inline u32 rgmii_mode_mask(int mode, int input)
 {
 	switch (mode) {
-	case PHY_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
 		return RGMII_FER_RGMII(input);
-	case PHY_MODE_TBI:
+	case PHY_INTERFACE_MODE_TBI:
 		return RGMII_FER_TBI(input);
-	case PHY_MODE_GMII:
+	case PHY_INTERFACE_MODE_GMII:
 		return RGMII_FER_GMII(input);
-	case PHY_MODE_MII:
+	case PHY_INTERFACE_MODE_MII:
 		return RGMII_FER_MII(input);
-	case PHY_MODE_RTBI:
+	case PHY_INTERFACE_MODE_RTBI:
 		return RGMII_FER_RTBI(input);
 	default:
 		BUG();
@@ -115,7 +100,7 @@ int rgmii_attach(struct platform_device *ofdev, int input, int mode)
 	out_be32(&p->fer, in_be32(&p->fer) | rgmii_mode_mask(mode, input));
 
 	printk(KERN_NOTICE "%pOF: input %d in %s mode\n",
-	       ofdev->dev.of_node, input, rgmii_mode_name(mode));
+	       ofdev->dev.of_node, input, phy_modes(mode));
 
 	++dev->users;
 
diff --git a/drivers/net/ethernet/ibm/emac/zmii.c b/drivers/net/ethernet/ibm/emac/zmii.c
index 89c42d362292..fdcc734541fe 100644
--- a/drivers/net/ethernet/ibm/emac/zmii.c
+++ b/drivers/net/ethernet/ibm/emac/zmii.c
@@ -49,20 +49,20 @@
  */
 static inline int zmii_valid_mode(int mode)
 {
-	return  mode == PHY_MODE_MII ||
-		mode == PHY_MODE_RMII ||
-		mode == PHY_MODE_SMII ||
-		mode == PHY_MODE_NA;
+	return  mode == PHY_INTERFACE_MODE_MII ||
+		mode == PHY_INTERFACE_MODE_RMII ||
+		mode == PHY_INTERFACE_MODE_SMII ||
+		mode == PHY_INTERFACE_MODE_NA;
 }
 
 static inline const char *zmii_mode_name(int mode)
 {
 	switch (mode) {
-	case PHY_MODE_MII:
+	case PHY_INTERFACE_MODE_MII:
 		return "MII";
-	case PHY_MODE_RMII:
+	case PHY_INTERFACE_MODE_RMII:
 		return "RMII";
-	case PHY_MODE_SMII:
+	case PHY_INTERFACE_MODE_SMII:
 		return "SMII";
 	default:
 		BUG();
@@ -72,11 +72,11 @@ static inline const char *zmii_mode_name(int mode)
 static inline u32 zmii_mode_mask(int mode, int input)
 {
 	switch (mode) {
-	case PHY_MODE_MII:
+	case PHY_INTERFACE_MODE_MII:
 		return ZMII_FER_MII(input);
-	case PHY_MODE_RMII:
+	case PHY_INTERFACE_MODE_RMII:
 		return ZMII_FER_RMII(input);
-	case PHY_MODE_SMII:
+	case PHY_INTERFACE_MODE_SMII:
 		return ZMII_FER_SMII(input);
 	default:
 		return 0;
@@ -106,27 +106,27 @@ int zmii_attach(struct platform_device *ofdev, int input, int *mode)
 	 * Please, always specify PHY mode in your board port to avoid
 	 * any surprises.
 	 */
-	if (dev->mode == PHY_MODE_NA) {
-		if (*mode == PHY_MODE_NA) {
+	if (dev->mode == PHY_INTERFACE_MODE_NA) {
+		if (*mode == PHY_INTERFACE_MODE_NA) {
 			u32 r = dev->fer_save;
 
 			ZMII_DBG(dev, "autodetecting mode, FER = 0x%08x" NL, r);
 
 			if (r & (ZMII_FER_MII(0) | ZMII_FER_MII(1)))
-				dev->mode = PHY_MODE_MII;
+				dev->mode = PHY_INTERFACE_MODE_MII;
 			else if (r & (ZMII_FER_RMII(0) | ZMII_FER_RMII(1)))
-				dev->mode = PHY_MODE_RMII;
+				dev->mode = PHY_INTERFACE_MODE_RMII;
 			else
-				dev->mode = PHY_MODE_SMII;
-		} else
+				dev->mode = PHY_INTERFACE_MODE_SMII;
+		} else {
 			dev->mode = *mode;
-
+		}
 		printk(KERN_NOTICE "%pOF: bridge in %s mode\n",
 		       ofdev->dev.of_node,
 		       zmii_mode_name(dev->mode));
 	} else {
 		/* All inputs must use the same mode */
-		if (*mode != PHY_MODE_NA && *mode != dev->mode) {
+		if (*mode != PHY_INTERFACE_MODE_NA && *mode != dev->mode) {
 			printk(KERN_ERR
 			       "%pOF: invalid mode %d specified for input %d\n",
 			       ofdev->dev.of_node, *mode, input);
@@ -246,7 +246,7 @@ static int zmii_probe(struct platform_device *ofdev)
 
 	mutex_init(&dev->lock);
 	dev->ofdev = ofdev;
-	dev->mode = PHY_MODE_NA;
+	dev->mode = PHY_INTERFACE_MODE_NA;
 
 	rc = -ENXIO;
 	if (of_address_to_resource(np, 0, &regs)) {
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index b65f5f3ac034..1b3cc8bb0705 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -59,6 +59,7 @@
 #include <linux/mm.h>
 #include <linux/ethtool.h>
 #include <linux/proc_fs.h>
+#include <linux/if_arp.h>
 #include <linux/in.h>
 #include <linux/ip.h>
 #include <linux/ipv6.h>
@@ -353,6 +354,8 @@ static void release_stats_buffers(struct ibmvnic_adapter *adapter)
 {
 	kfree(adapter->tx_stats_buffers);
 	kfree(adapter->rx_stats_buffers);
+	adapter->tx_stats_buffers = NULL;
+	adapter->rx_stats_buffers = NULL;
 }
 
 static int init_stats_buffers(struct ibmvnic_adapter *adapter)
@@ -598,6 +601,8 @@ static void release_vpd_data(struct ibmvnic_adapter *adapter)
 
 	kfree(adapter->vpd->buff);
 	kfree(adapter->vpd);
+
+	adapter->vpd = NULL;
 }
 
 static void release_tx_pools(struct ibmvnic_adapter *adapter)
@@ -786,6 +791,18 @@ static int ibmvnic_login(struct net_device *netdev)
 	return 0;
 }
 
+static void release_login_buffer(struct ibmvnic_adapter *adapter)
+{
+	kfree(adapter->login_buf);
+	adapter->login_buf = NULL;
+}
+
+static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter)
+{
+	kfree(adapter->login_rsp_buf);
+	adapter->login_rsp_buf = NULL;
+}
+
 static void release_resources(struct ibmvnic_adapter *adapter)
 {
 	int i;
@@ -808,6 +825,10 @@ static void release_resources(struct ibmvnic_adapter *adapter)
 			}
 		}
 	}
+	kfree(adapter->napi);
+	adapter->napi = NULL;
+
+	release_login_rsp_buffer(adapter);
 }
 
 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)
@@ -908,6 +929,7 @@ static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)
 	if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {
 		dev_err(dev, "Could not map VPD buffer\n");
 		kfree(adapter->vpd->buff);
+		adapter->vpd->buff = NULL;
 		return -ENOMEM;
 	}
 
@@ -1051,6 +1073,35 @@ static int ibmvnic_open(struct net_device *netdev)
 	return rc;
 }
 
+static void clean_rx_pools(struct ibmvnic_adapter *adapter)
+{
+	struct ibmvnic_rx_pool *rx_pool;
+	u64 rx_entries;
+	int rx_scrqs;
+	int i, j;
+
+	if (!adapter->rx_pool)
+		return;
+
+	rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);
+	rx_entries = adapter->req_rx_add_entries_per_subcrq;
+
+	/* Free any remaining skbs in the rx buffer pools */
+	for (i = 0; i < rx_scrqs; i++) {
+		rx_pool = &adapter->rx_pool[i];
+		if (!rx_pool)
+			continue;
+
+		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);
+		for (j = 0; j < rx_entries; j++) {
+			if (rx_pool->rx_buff[j].skb) {
+				dev_kfree_skb_any(rx_pool->rx_buff[j].skb);
+				rx_pool->rx_buff[j].skb = NULL;
+			}
+		}
+	}
+}
+
 static void clean_tx_pools(struct ibmvnic_adapter *adapter)
 {
 	struct ibmvnic_tx_pool *tx_pool;
@@ -1128,7 +1179,7 @@ static int __ibmvnic_close(struct net_device *netdev)
 			}
 		}
 	}
-
+	clean_rx_pools(adapter);
 	clean_tx_pools(adapter);
 	adapter->state = VNIC_CLOSED;
 	return rc;
@@ -1177,6 +1228,9 @@ static int build_hdr_data(u8 hdr_field, struct sk_buff *skb,
 			hdr_len[2] = tcp_hdrlen(skb);
 		else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
 			hdr_len[2] = sizeof(struct udphdr);
+	} else if (skb->protocol == htons(ETH_P_ARP)) {
+		hdr_len[1] = arp_hdr_len(skb->dev);
+		hdr_len[2] = 0;
 	}
 
 	memset(hdr_data, 0, 120);
@@ -1410,9 +1464,7 @@ static int ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
 		hdrs += 2;
 	}
 	/* determine if l2/3/4 headers are sent to firmware */
-	if ((*hdrs >> 7) & 1 &&
-	    (skb->protocol == htons(ETH_P_IP) ||
-	     skb->protocol == htons(ETH_P_IPV6))) {
+	if ((*hdrs >> 7) & 1) {
 		build_hdr_descs_arr(tx_buff, &num_entries, *hdrs);
 		tx_crq.v1.n_crq_elem = num_entries;
 		tx_buff->indir_arr[0] = tx_crq;
@@ -1543,15 +1595,19 @@ static int __ibmvnic_set_mac(struct net_device *netdev, struct sockaddr *p)
 	crq.change_mac_addr.first = IBMVNIC_CRQ_CMD;
 	crq.change_mac_addr.cmd = CHANGE_MAC_ADDR;
 	ether_addr_copy(&crq.change_mac_addr.mac_addr[0], addr->sa_data);
+
+	init_completion(&adapter->fw_done);
 	ibmvnic_send_crq(adapter, &crq);
+	wait_for_completion(&adapter->fw_done);
 	/* netdev->dev_addr is changed in handle_change_mac_rsp function */
-	return 0;
+	return adapter->fw_done_rc ? -EIO : 0;
 }
 
 static int ibmvnic_set_mac(struct net_device *netdev, void *p)
 {
 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);
 	struct sockaddr *addr = p;
+	int rc;
 
 	if (adapter->state == VNIC_PROBED) {
 		memcpy(&adapter->desired.mac, addr, sizeof(struct sockaddr));
@@ -1559,9 +1615,9 @@ static int ibmvnic_set_mac(struct net_device *netdev, void *p)
 		return 0;
 	}
 
-	__ibmvnic_set_mac(netdev, addr);
+	rc = __ibmvnic_set_mac(netdev, addr);
 
-	return 0;
+	return rc;
 }
 
 /**
@@ -1630,6 +1686,7 @@ static int do_reset(struct ibmvnic_adapter *adapter,
 				return rc;
 		} else if (adapter->req_rx_queues != old_num_rx_queues ||
 			   adapter->req_tx_queues != old_num_tx_queues) {
+			adapter->map_id = 1;
 			release_rx_pools(adapter);
 			release_tx_pools(adapter);
 			init_rx_pools(netdev);
@@ -1658,8 +1715,6 @@ static int do_reset(struct ibmvnic_adapter *adapter,
 		return 0;
 	}
 
-	netif_carrier_on(netdev);
-
 	/* kick napi */
 	for (i = 0; i < adapter->req_rx_queues; i++)
 		napi_schedule(&adapter->napi[i]);
@@ -1667,6 +1722,8 @@ static int do_reset(struct ibmvnic_adapter *adapter,
 	if (adapter->reset_reason != VNIC_RESET_FAILOVER)
 		netdev_notify_peers(netdev);
 
+	netif_carrier_on(netdev);
+
 	return 0;
 }
 
@@ -1822,7 +1879,8 @@ static int ibmvnic_poll(struct napi_struct *napi, int budget)
 		u16 offset;
 		u8 flags = 0;
 
-		if (unlikely(adapter->resetting)) {
+		if (unlikely(adapter->resetting &&
+			     adapter->reset_reason != VNIC_RESET_NON_FATAL)) {
 			enable_scrq_irq(adapter, adapter->rx_scrq[scrq_num]);
 			napi_complete_done(napi, frames_processed);
 			return frames_processed;
@@ -1840,6 +1898,12 @@ static int ibmvnic_poll(struct napi_struct *napi, int budget)
 				   be16_to_cpu(next->rx_comp.rc));
 			/* free the entry */
 			next->rx_comp.first = 0;
+			dev_kfree_skb_any(rx_buff->skb);
+			remove_buff_from_pool(adapter, rx_buff);
+			continue;
+		} else if (!rx_buff->skb) {
+			/* free the entry */
+			next->rx_comp.first = 0;
 			remove_buff_from_pool(adapter, rx_buff);
 			continue;
 		}
@@ -2484,6 +2548,12 @@ static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance)
 	struct ibmvnic_sub_crq_queue *scrq = instance;
 	struct ibmvnic_adapter *adapter = scrq->adapter;
 
+	/* When booting a kdump kernel we can hit pending interrupts
+	 * prior to completing driver initialization.
+	 */
+	if (unlikely(adapter->state != VNIC_OPEN))
+		return IRQ_NONE;
+
 	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;
 
 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {
@@ -2893,8 +2963,12 @@ static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,
 				cpu_to_be64(u64_crq[1]));
 
 	if (rc) {
-		if (rc == H_CLOSED)
+		if (rc == H_CLOSED) {
 			dev_warn(dev, "CRQ Queue closed\n");
+			if (adapter->resetting)
+				ibmvnic_reset(adapter, VNIC_RESET_FATAL);
+		}
+
 		dev_warn(dev, "Send error (rc=%d)\n", rc);
 	}
 
@@ -2990,6 +3064,7 @@ static void send_login(struct ibmvnic_adapter *adapter)
 	struct vnic_login_client_data *vlcd;
 	int i;
 
+	release_login_rsp_buffer(adapter);
 	client_data_len = vnic_client_data_len(adapter);
 
 	buffer_size =
@@ -3271,7 +3346,7 @@ static void handle_vpd_rsp(union ibmvnic_crq *crq,
 			   struct ibmvnic_adapter *adapter)
 {
 	struct device *dev = &adapter->vdev->dev;
-	unsigned char *substr = NULL, *ptr = NULL;
+	unsigned char *substr = NULL;
 	u8 fw_level_len = 0;
 
 	memset(adapter->fw_version, 0, 32);
@@ -3290,7 +3365,7 @@ static void handle_vpd_rsp(union ibmvnic_crq *crq,
 	 */
 	substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);
 	if (!substr) {
-		dev_info(dev, "No FW level provided by VPD\n");
+		dev_info(dev, "Warning - No FW level has been provided in the VPD buffer by the VIOS Server\n");
 		goto complete;
 	}
 
@@ -3305,16 +3380,14 @@ static void handle_vpd_rsp(union ibmvnic_crq *crq,
 	/* copy firmware version string from vpd into adapter */
 	if ((substr + 3 + fw_level_len) <
 	    (adapter->vpd->buff + adapter->vpd->len)) {
-		ptr = strncpy((char *)adapter->fw_version,
-			      substr + 3, fw_level_len);
-
-		if (!ptr)
-			dev_err(dev, "Failed to isolate FW level string\n");
+		strncpy((char *)adapter->fw_version, substr + 3, fw_level_len);
 	} else {
 		dev_info(dev, "FW substr extrapolated VPD buff\n");
 	}
 
 complete:
+	if (adapter->fw_version[0] == '\0')
+		strncpy((char *)adapter->fw_version, "N/A", 3 * sizeof(char));
 	complete(&adapter->fw_done);
 }
 
@@ -3558,8 +3631,8 @@ static void handle_error_indication(union ibmvnic_crq *crq,
 		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);
 }
 
-static void handle_change_mac_rsp(union ibmvnic_crq *crq,
-				  struct ibmvnic_adapter *adapter)
+static int handle_change_mac_rsp(union ibmvnic_crq *crq,
+				 struct ibmvnic_adapter *adapter)
 {
 	struct net_device *netdev = adapter->netdev;
 	struct device *dev = &adapter->vdev->dev;
@@ -3568,10 +3641,13 @@ static void handle_change_mac_rsp(union ibmvnic_crq *crq,
 	rc = crq->change_mac_addr_rsp.rc.code;
 	if (rc) {
 		dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc);
-		return;
+		goto out;
 	}
 	memcpy(netdev->dev_addr, &crq->change_mac_addr_rsp.mac_addr[0],
 	       ETH_ALEN);
+out:
+	complete(&adapter->fw_done);
+	return rc;
 }
 
 static void handle_request_cap_rsp(union ibmvnic_crq *crq,
@@ -3714,6 +3790,7 @@ static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq,
 		ibmvnic_remove(adapter->vdev);
 		return -EIO;
 	}
+	release_login_buffer(adapter);
 	complete(&adapter->init_done);
 
 	return 0;
@@ -4031,7 +4108,7 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq,
 		break;
 	case CHANGE_MAC_ADDR_RSP:
 		netdev_dbg(netdev, "Got MAC address change Response\n");
-		handle_change_mac_rsp(crq, adapter);
+		adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter);
 		break;
 	case ERROR_INDICATION:
 		netdev_dbg(netdev, "Got Error Indication\n");
@@ -4335,7 +4412,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)
 	}
 
 	netdev = alloc_etherdev_mq(sizeof(struct ibmvnic_adapter),
-				   IBMVNIC_MAX_TX_QUEUES);
+				   IBMVNIC_MAX_QUEUES);
 	if (!netdev)
 		return -ENOMEM;
 
@@ -4461,7 +4538,7 @@ static ssize_t failover_store(struct device *dev, struct device_attribute *attr,
 	return count;
 }
 
-static DEVICE_ATTR(failover, 0200, NULL, failover_store);
+static DEVICE_ATTR_WO(failover);
 
 static unsigned long ibmvnic_get_desired_dma(struct vio_dev *vdev)
 {
diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index 3aec42118db2..fe21a6e2ddae 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -39,7 +39,7 @@
 #define IBMVNIC_RX_WEIGHT		16
 /* when changing this, update IBMVNIC_IO_ENTITLEMENT_DEFAULT */
 #define IBMVNIC_BUFFS_PER_POOL	100
-#define IBMVNIC_MAX_TX_QUEUES	5
+#define IBMVNIC_MAX_QUEUES	10
 
 #define IBMVNIC_TSO_BUF_SZ	65536
 #define IBMVNIC_TSO_BUFS	64
diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c
index 44b3937f7e81..29486478836e 100644
--- a/drivers/net/ethernet/intel/e100.c
+++ b/drivers/net/ethernet/intel/e100.c
@@ -607,7 +607,7 @@ struct nic {
 	struct mem *mem;
 	dma_addr_t dma_addr;
 
-	struct pci_pool *cbs_pool;
+	struct dma_pool *cbs_pool;
 	dma_addr_t cbs_dma_addr;
 	u8 adaptive_ifs;
 	u8 tx_threshold;
@@ -1892,7 +1892,7 @@ static void e100_clean_cbs(struct nic *nic)
 			nic->cb_to_clean = nic->cb_to_clean->next;
 			nic->cbs_avail++;
 		}
-		pci_pool_free(nic->cbs_pool, nic->cbs, nic->cbs_dma_addr);
+		dma_pool_free(nic->cbs_pool, nic->cbs, nic->cbs_dma_addr);
 		nic->cbs = NULL;
 		nic->cbs_avail = 0;
 	}
@@ -1910,7 +1910,7 @@ static int e100_alloc_cbs(struct nic *nic)
 	nic->cb_to_use = nic->cb_to_send = nic->cb_to_clean = NULL;
 	nic->cbs_avail = 0;
 
-	nic->cbs = pci_pool_zalloc(nic->cbs_pool, GFP_KERNEL,
+	nic->cbs = dma_pool_zalloc(nic->cbs_pool, GFP_KERNEL,
 				   &nic->cbs_dma_addr);
 	if (!nic->cbs)
 		return -ENOMEM;
@@ -2960,8 +2960,8 @@ static int e100_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		netif_err(nic, probe, nic->netdev, "Cannot register net device, aborting\n");
 		goto err_out_free;
 	}
-	nic->cbs_pool = pci_pool_create(netdev->name,
-			   nic->pdev,
+	nic->cbs_pool = dma_pool_create(netdev->name,
+			   &nic->pdev->dev,
 			   nic->params.cbs.max * sizeof(struct cb),
 			   sizeof(u32),
 			   0);
@@ -3001,7 +3001,7 @@ static void e100_remove(struct pci_dev *pdev)
 		unregister_netdev(netdev);
 		e100_free(nic);
 		pci_iounmap(pdev, nic->csr);
-		pci_pool_destroy(nic->cbs_pool);
+		dma_pool_destroy(nic->cbs_pool);
 		free_netdev(netdev);
 		pci_release_regions(pdev);
 		pci_disable_device(pdev);
diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
index 3b3983a1ffbb..dc71e87c3260 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
@@ -1838,8 +1838,8 @@ static void e1000_get_ethtool_stats(struct net_device *netdev,
 			p = (char *)adapter + stat->stat_offset;
 			break;
 		default:
-			WARN_ONCE(1, "Invalid E1000 stat type: %u index %d\n",
-				  stat->type, i);
+			netdev_WARN_ONCE(netdev, "Invalid E1000 stat type: %u index %d\n",
+					 stat->type, i);
 			continue;
 		}
 
diff --git a/drivers/net/ethernet/intel/e1000e/defines.h b/drivers/net/ethernet/intel/e1000e/defines.h
index afb7ebe20b24..824fd44e25f0 100644
--- a/drivers/net/ethernet/intel/e1000e/defines.h
+++ b/drivers/net/ethernet/intel/e1000e/defines.h
@@ -400,6 +400,10 @@
 #define E1000_ICR_RXDMT0        0x00000010 /* Rx desc min. threshold (0) */
 #define E1000_ICR_RXO           0x00000040 /* Receiver Overrun */
 #define E1000_ICR_RXT0          0x00000080 /* Rx timer intr (ring 0) */
+#define E1000_ICR_MDAC          0x00000200 /* MDIO Access Complete */
+#define E1000_ICR_SRPD          0x00010000 /* Small Receive Packet Detected */
+#define E1000_ICR_ACK           0x00020000 /* Receive ACK Frame Detected */
+#define E1000_ICR_MNG           0x00040000 /* Manageability Event Detected */
 #define E1000_ICR_ECCER         0x00400000 /* Uncorrectable ECC Error */
 /* If this bit asserted, the driver should claim the interrupt */
 #define E1000_ICR_INT_ASSERTED	0x80000000
@@ -407,7 +411,7 @@
 #define E1000_ICR_RXQ1          0x00200000 /* Rx Queue 1 Interrupt */
 #define E1000_ICR_TXQ0          0x00400000 /* Tx Queue 0 Interrupt */
 #define E1000_ICR_TXQ1          0x00800000 /* Tx Queue 1 Interrupt */
-#define E1000_ICR_OTHER         0x01000000 /* Other Interrupts */
+#define E1000_ICR_OTHER         0x01000000 /* Other Interrupt */
 
 /* PBA ECC Register */
 #define E1000_PBA_ECC_COUNTER_MASK  0xFFF00000 /* ECC counter mask */
@@ -431,12 +435,27 @@
 	E1000_IMS_RXSEQ  |    \
 	E1000_IMS_LSC)
 
+/* These are all of the events related to the OTHER interrupt.
+ */
+#define IMS_OTHER_MASK ( \
+	E1000_IMS_LSC  | \
+	E1000_IMS_RXO  | \
+	E1000_IMS_MDAC | \
+	E1000_IMS_SRPD | \
+	E1000_IMS_ACK  | \
+	E1000_IMS_MNG)
+
 /* Interrupt Mask Set */
 #define E1000_IMS_TXDW      E1000_ICR_TXDW      /* Transmit desc written back */
 #define E1000_IMS_LSC       E1000_ICR_LSC       /* Link Status Change */
 #define E1000_IMS_RXSEQ     E1000_ICR_RXSEQ     /* Rx sequence error */
 #define E1000_IMS_RXDMT0    E1000_ICR_RXDMT0    /* Rx desc min. threshold */
+#define E1000_IMS_RXO       E1000_ICR_RXO       /* Receiver Overrun */
 #define E1000_IMS_RXT0      E1000_ICR_RXT0      /* Rx timer intr */
+#define E1000_IMS_MDAC      E1000_ICR_MDAC      /* MDIO Access Complete */
+#define E1000_IMS_SRPD      E1000_ICR_SRPD      /* Small Receive Packet */
+#define E1000_IMS_ACK       E1000_ICR_ACK       /* Receive ACK Frame Detected */
+#define E1000_IMS_MNG       E1000_ICR_MNG       /* Manageability Event */
 #define E1000_IMS_ECCER     E1000_ICR_ECCER     /* Uncorrectable ECC Error */
 #define E1000_IMS_RXQ0      E1000_ICR_RXQ0      /* Rx Queue 0 Interrupt */
 #define E1000_IMS_RXQ1      E1000_ICR_RXQ1      /* Rx Queue 1 Interrupt */
diff --git a/drivers/net/ethernet/intel/e1000e/ich8lan.c b/drivers/net/ethernet/intel/e1000e/ich8lan.c
index 31277d3bb7dc..1dddfb7b2de6 100644
--- a/drivers/net/ethernet/intel/e1000e/ich8lan.c
+++ b/drivers/net/ethernet/intel/e1000e/ich8lan.c
@@ -1367,9 +1367,6 @@ static s32 e1000_disable_ulp_lpt_lp(struct e1000_hw *hw, bool force)
  *  Checks to see of the link status of the hardware has changed.  If a
  *  change in link status has been detected, then we read the PHY registers
  *  to get the current speed/duplex if link exists.
- *
- *  Returns a negative error code (-E1000_ERR_*) or 0 (link down) or 1 (link
- *  up).
  **/
 static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 {
@@ -1385,7 +1382,8 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 	 * Change or Rx Sequence Error interrupt.
 	 */
 	if (!mac->get_link_status)
-		return 1;
+		return 0;
+	mac->get_link_status = false;
 
 	/* First we want to see if the MII Status Register reports
 	 * link.  If so, then we want to get the current speed/duplex
@@ -1393,12 +1391,12 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 	 */
 	ret_val = e1000e_phy_has_link_generic(hw, 1, 0, &link);
 	if (ret_val)
-		return ret_val;
+		goto out;
 
 	if (hw->mac.type == e1000_pchlan) {
 		ret_val = e1000_k1_gig_workaround_hv(hw, link);
 		if (ret_val)
-			return ret_val;
+			goto out;
 	}
 
 	/* When connected at 10Mbps half-duplex, some parts are excessively
@@ -1431,7 +1429,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 
 		ret_val = hw->phy.ops.acquire(hw);
 		if (ret_val)
-			return ret_val;
+			goto out;
 
 		if (hw->mac.type == e1000_pch2lan)
 			emi_addr = I82579_RX_CONFIG;
@@ -1453,7 +1451,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 		hw->phy.ops.release(hw);
 
 		if (ret_val)
-			return ret_val;
+			goto out;
 
 		if (hw->mac.type >= e1000_pch_spt) {
 			u16 data;
@@ -1462,14 +1460,14 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 			if (speed == SPEED_1000) {
 				ret_val = hw->phy.ops.acquire(hw);
 				if (ret_val)
-					return ret_val;
+					goto out;
 
 				ret_val = e1e_rphy_locked(hw,
 							  PHY_REG(776, 20),
 							  &data);
 				if (ret_val) {
 					hw->phy.ops.release(hw);
-					return ret_val;
+					goto out;
 				}
 
 				ptr_gap = (data & (0x3FF << 2)) >> 2;
@@ -1483,18 +1481,18 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 				}
 				hw->phy.ops.release(hw);
 				if (ret_val)
-					return ret_val;
+					goto out;
 			} else {
 				ret_val = hw->phy.ops.acquire(hw);
 				if (ret_val)
-					return ret_val;
+					goto out;
 
 				ret_val = e1e_wphy_locked(hw,
 							  PHY_REG(776, 20),
 							  0xC023);
 				hw->phy.ops.release(hw);
 				if (ret_val)
-					return ret_val;
+					goto out;
 
 			}
 		}
@@ -1521,7 +1519,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 	    (hw->adapter->pdev->device == E1000_DEV_ID_PCH_I218_V3)) {
 		ret_val = e1000_k1_workaround_lpt_lp(hw, link);
 		if (ret_val)
-			return ret_val;
+			goto out;
 	}
 	if (hw->mac.type >= e1000_pch_lpt) {
 		/* Set platform power management values for
@@ -1529,7 +1527,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 		 */
 		ret_val = e1000_platform_pm_pch_lpt(hw, link);
 		if (ret_val)
-			return ret_val;
+			goto out;
 	}
 
 	/* Clear link partner's EEE ability */
@@ -1552,9 +1550,7 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 	}
 
 	if (!link)
-		return 0;	/* No link detected */
-
-	mac->get_link_status = false;
+		goto out;
 
 	switch (hw->mac.type) {
 	case e1000_pch2lan:
@@ -1616,12 +1612,14 @@ static s32 e1000_check_for_copper_link_ich8lan(struct e1000_hw *hw)
 	 * different link partner.
 	 */
 	ret_val = e1000e_config_fc_after_link_up(hw);
-	if (ret_val) {
+	if (ret_val)
 		e_dbg("Error configuring flow control\n");
-		return ret_val;
-	}
 
-	return 1;
+	return ret_val;
+
+out:
+	mac->get_link_status = true;
+	return ret_val;
 }
 
 static s32 e1000_get_variants_ich8lan(struct e1000_adapter *adapter)
diff --git a/drivers/net/ethernet/intel/e1000e/mac.c b/drivers/net/ethernet/intel/e1000e/mac.c
index f457c5703d0c..5bdc3a2d4fd7 100644
--- a/drivers/net/ethernet/intel/e1000e/mac.c
+++ b/drivers/net/ethernet/intel/e1000e/mac.c
@@ -410,9 +410,6 @@ void e1000e_clear_hw_cntrs_base(struct e1000_hw *hw)
  *  Checks to see of the link status of the hardware has changed.  If a
  *  change in link status has been detected, then we read the PHY registers
  *  to get the current speed/duplex if link exists.
- *
- *  Returns a negative error code (-E1000_ERR_*) or 0 (link down) or 1 (link
- *  up).
  **/
 s32 e1000e_check_for_copper_link(struct e1000_hw *hw)
 {
@@ -426,20 +423,16 @@ s32 e1000e_check_for_copper_link(struct e1000_hw *hw)
 	 * Change or Rx Sequence Error interrupt.
 	 */
 	if (!mac->get_link_status)
-		return 1;
+		return 0;
+	mac->get_link_status = false;
 
 	/* First we want to see if the MII Status Register reports
 	 * link.  If so, then we want to get the current speed/duplex
 	 * of the PHY.
 	 */
 	ret_val = e1000e_phy_has_link_generic(hw, 1, 0, &link);
-	if (ret_val)
-		return ret_val;
-
-	if (!link)
-		return 0;	/* No link detected */
-
-	mac->get_link_status = false;
+	if (ret_val || !link)
+		goto out;
 
 	/* Check if there was DownShift, must be checked
 	 * immediately after link-up
@@ -464,12 +457,14 @@ s32 e1000e_check_for_copper_link(struct e1000_hw *hw)
 	 * different link partner.
 	 */
 	ret_val = e1000e_config_fc_after_link_up(hw);
-	if (ret_val) {
+	if (ret_val)
 		e_dbg("Error configuring flow control\n");
-		return ret_val;
-	}
 
-	return 1;
+	return ret_val;
+
+out:
+	mac->get_link_status = true;
+	return ret_val;
 }
 
 /**
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index 9f18d39bdc8f..dc853b0863af 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -1914,30 +1914,20 @@ static irqreturn_t e1000_msix_other(int __always_unused irq, void *data)
 	struct net_device *netdev = data;
 	struct e1000_adapter *adapter = netdev_priv(netdev);
 	struct e1000_hw *hw = &adapter->hw;
-	u32 icr;
-	bool enable = true;
-
-	icr = er32(ICR);
-	if (icr & E1000_ICR_RXO) {
-		ew32(ICR, E1000_ICR_RXO);
-		enable = false;
-		/* napi poll will re-enable Other, make sure it runs */
-		if (napi_schedule_prep(&adapter->napi)) {
-			adapter->total_rx_bytes = 0;
-			adapter->total_rx_packets = 0;
-			__napi_schedule(&adapter->napi);
-		}
-	}
+	u32 icr = er32(ICR);
+
+	if (icr & adapter->eiac_mask)
+		ew32(ICS, (icr & adapter->eiac_mask));
+
 	if (icr & E1000_ICR_LSC) {
-		ew32(ICR, E1000_ICR_LSC);
 		hw->mac.get_link_status = true;
 		/* guard against interrupt when we're going down */
 		if (!test_bit(__E1000_DOWN, &adapter->state))
 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
 	}
 
-	if (enable && !test_bit(__E1000_DOWN, &adapter->state))
-		ew32(IMS, E1000_IMS_OTHER);
+	if (!test_bit(__E1000_DOWN, &adapter->state))
+		ew32(IMS, E1000_IMS_OTHER | IMS_OTHER_MASK);
 
 	return IRQ_HANDLED;
 }
@@ -2040,7 +2030,6 @@ static void e1000_configure_msix(struct e1000_adapter *adapter)
 		       hw->hw_addr + E1000_EITR_82574(vector));
 	else
 		writel(1, hw->hw_addr + E1000_EITR_82574(vector));
-	adapter->eiac_mask |= E1000_IMS_OTHER;
 
 	/* Cause Tx interrupts on every write back */
 	ivar |= BIT(31);
@@ -2265,7 +2254,8 @@ static void e1000_irq_enable(struct e1000_adapter *adapter)
 
 	if (adapter->msix_entries) {
 		ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
-		ew32(IMS, adapter->eiac_mask | E1000_IMS_LSC);
+		ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER |
+		     IMS_OTHER_MASK);
 	} else if (hw->mac.type >= e1000_pch_lpt) {
 		ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER);
 	} else {
@@ -2333,8 +2323,8 @@ static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
 {
 	struct pci_dev *pdev = adapter->pdev;
 
-	ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
-					GFP_KERNEL);
+	ring->desc = dma_zalloc_coherent(&pdev->dev, ring->size, &ring->dma,
+					 GFP_KERNEL);
 	if (!ring->desc)
 		return -ENOMEM;
 
@@ -2707,8 +2697,7 @@ static int e1000e_poll(struct napi_struct *napi, int weight)
 		napi_complete_done(napi, work_done);
 		if (!test_bit(__E1000_DOWN, &adapter->state)) {
 			if (adapter->msix_entries)
-				ew32(IMS, adapter->rx_ring->ims_val |
-				     E1000_IMS_OTHER);
+				ew32(IMS, adapter->rx_ring->ims_val);
 			else
 				e1000_irq_enable(adapter);
 		}
@@ -3303,9 +3292,11 @@ static void e1000_configure_rx(struct e1000_adapter *adapter)
 		if (adapter->flags & FLAG_IS_ICH) {
 			u32 rxdctl = er32(RXDCTL(0));
 
-			ew32(RXDCTL(0), rxdctl | 0x3);
+			ew32(RXDCTL(0), rxdctl | 0x3 | BIT(8));
 		}
 
+		dev_info(&adapter->pdev->dev,
+			 "Some CPU C-states have been disabled in order to enable jumbo frames\n");
 		pm_qos_update_request(&adapter->pm_qos_req, lat);
 	} else {
 		pm_qos_update_request(&adapter->pm_qos_req,
@@ -5099,7 +5090,7 @@ static bool e1000e_has_link(struct e1000_adapter *adapter)
 	case e1000_media_type_copper:
 		if (hw->mac.get_link_status) {
 			ret_val = hw->mac.ops.check_for_link(hw);
-			link_active = ret_val > 0;
+			link_active = !hw->mac.get_link_status;
 		} else {
 			link_active = true;
 		}
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_iov.c b/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
index ea3ab24265ee..760cfa52d02c 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_iov.c
@@ -353,7 +353,7 @@ int fm10k_iov_resume(struct pci_dev *pdev)
 		struct fm10k_vf_info *vf_info = &iov_data->vf_info[i];
 
 		/* allocate all but the last GLORT to the VFs */
-		if (i == ((~hw->mac.dglort_map) >> FM10K_DGLORTMAP_MASK_SHIFT))
+		if (i == (~hw->mac.dglort_map >> FM10K_DGLORTMAP_MASK_SHIFT))
 			break;
 
 		/* assign GLORT to VF, and restrict it to multicast */
@@ -511,7 +511,7 @@ int fm10k_iov_configure(struct pci_dev *pdev, int num_vfs)
 		return err;
 
 	/* allocate VFs if not already allocated */
-	if (num_vfs && (num_vfs != current_vfs)) {
+	if (num_vfs && num_vfs != current_vfs) {
 		/* Disable completer abort error reporting as
 		 * the VFs can trigger this any time they read a queue
 		 * that they don't own.
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
index 538b42d5c187..8e12aae065d8 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
@@ -446,13 +446,13 @@ static void fm10k_type_trans(struct fm10k_ring *rx_ring,
 
 	skb->protocol = eth_type_trans(skb, dev);
 
+	/* Record Rx queue, or update macvlan statistics */
 	if (!l2_accel)
-		return;
-
-	/* update MACVLAN statistics */
-	macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, 1,
-			 !!(rx_desc->w.hdr_info &
-			    cpu_to_le16(FM10K_RXD_HDR_INFO_XC_MASK)));
+		skb_record_rx_queue(skb, rx_ring->queue_index);
+	else
+		macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
+				 (skb->pkt_type == PACKET_BROADCAST) ||
+				 (skb->pkt_type == PACKET_MULTICAST));
 }
 
 /**
@@ -479,8 +479,6 @@ static unsigned int fm10k_process_skb_fields(struct fm10k_ring *rx_ring,
 
 	FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan;
 
-	skb_record_rx_queue(skb, rx_ring->queue_index);
-
 	FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort;
 
 	if (rx_desc->w.vlan) {
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
index adc62fb38c49..a38ae5c54da3 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_netdev.c
@@ -934,8 +934,12 @@ static int fm10k_update_vid(struct net_device *netdev, u16 vid, bool set)
 	if (vid >= VLAN_N_VID)
 		return -EINVAL;
 
-	/* Verify we have permission to add VLANs */
-	if (hw->mac.vlan_override)
+	/* Verify that we have permission to add VLANs. If this is a request
+	 * to remove a VLAN, we still want to allow the user to remove the
+	 * VLAN device. In that case, we need to clear the bit in the
+	 * active_vlans bitmask.
+	 */
+	if (set && hw->mac.vlan_override)
 		return -EACCES;
 
 	/* update active_vlans bitmask */
@@ -954,6 +958,12 @@ static int fm10k_update_vid(struct net_device *netdev, u16 vid, bool set)
 			rx_ring->vid &= ~FM10K_VLAN_CLEAR;
 	}
 
+	/* If our VLAN has been overridden, there is no reason to send VLAN
+	 * removal requests as they will be silently ignored.
+	 */
+	if (hw->mac.vlan_override)
+		return 0;
+
 	/* Do not remove default VLAN ID related entries from VLAN and MAC
 	 * tables
 	 */
@@ -1040,14 +1050,13 @@ static int __fm10k_uc_sync(struct net_device *dev,
 			   const unsigned char *addr, bool sync)
 {
 	struct fm10k_intfc *interface = netdev_priv(dev);
-	struct fm10k_hw *hw = &interface->hw;
 	u16 vid, glort = interface->glort;
 	s32 err;
 
 	if (!is_valid_ether_addr(addr))
 		return -EADDRNOTAVAIL;
 
-	for (vid = hw->mac.default_vid ? fm10k_find_next_vlan(interface, 0) : 1;
+	for (vid = fm10k_find_next_vlan(interface, 0);
 	     vid < VLAN_N_VID;
 	     vid = fm10k_find_next_vlan(interface, vid)) {
 		err = fm10k_queue_mac_request(interface, glort,
@@ -1106,14 +1115,13 @@ static int __fm10k_mc_sync(struct net_device *dev,
 			   const unsigned char *addr, bool sync)
 {
 	struct fm10k_intfc *interface = netdev_priv(dev);
-	struct fm10k_hw *hw = &interface->hw;
 	u16 vid, glort = interface->glort;
 	s32 err;
 
 	if (!is_multicast_ether_addr(addr))
 		return -EADDRNOTAVAIL;
 
-	for (vid = hw->mac.default_vid ? fm10k_find_next_vlan(interface, 0) : 1;
+	for (vid = fm10k_find_next_vlan(interface, 0);
 	     vid < VLAN_N_VID;
 	     vid = fm10k_find_next_vlan(interface, vid)) {
 		err = fm10k_queue_mac_request(interface, glort,
@@ -1157,10 +1165,12 @@ static void fm10k_set_rx_mode(struct net_device *dev)
 
 	/* update xcast mode first, but only if it changed */
 	if (interface->xcast_mode != xcast_mode) {
-		/* update VLAN table */
+		/* update VLAN table when entering promiscuous mode */
 		if (xcast_mode == FM10K_XCAST_MODE_PROMISC)
 			fm10k_queue_vlan_request(interface, FM10K_VLAN_ALL,
 						 0, true);
+
+		/* clear VLAN table when exiting promiscuous mode */
 		if (interface->xcast_mode == FM10K_XCAST_MODE_PROMISC)
 			fm10k_clear_unused_vlans(interface);
 
@@ -1182,9 +1192,10 @@ static void fm10k_set_rx_mode(struct net_device *dev)
 
 void fm10k_restore_rx_state(struct fm10k_intfc *interface)
 {
+	struct fm10k_l2_accel *l2_accel = interface->l2_accel;
 	struct net_device *netdev = interface->netdev;
 	struct fm10k_hw *hw = &interface->hw;
-	int xcast_mode;
+	int xcast_mode, i;
 	u16 vid, glort;
 
 	/* record glort for this interface */
@@ -1211,11 +1222,8 @@ void fm10k_restore_rx_state(struct fm10k_intfc *interface)
 	fm10k_queue_vlan_request(interface, FM10K_VLAN_ALL, 0,
 				 xcast_mode == FM10K_XCAST_MODE_PROMISC);
 
-	/* Add filter for VLAN 0 */
-	fm10k_queue_vlan_request(interface, 0, 0, true);
-
 	/* update table with current entries */
-	for (vid = hw->mac.default_vid ? fm10k_find_next_vlan(interface, 0) : 1;
+	for (vid = fm10k_find_next_vlan(interface, 0);
 	     vid < VLAN_N_VID;
 	     vid = fm10k_find_next_vlan(interface, vid)) {
 		fm10k_queue_vlan_request(interface, vid, 0, true);
@@ -1234,6 +1242,24 @@ void fm10k_restore_rx_state(struct fm10k_intfc *interface)
 	__dev_uc_sync(netdev, fm10k_uc_sync, fm10k_uc_unsync);
 	__dev_mc_sync(netdev, fm10k_mc_sync, fm10k_mc_unsync);
 
+	/* synchronize macvlan addresses */
+	if (l2_accel) {
+		for (i = 0; i < l2_accel->size; i++) {
+			struct net_device *sdev = l2_accel->macvlan[i];
+
+			if (!sdev)
+				continue;
+
+			glort = l2_accel->dglort + 1 + i;
+
+			hw->mac.ops.update_xcast_mode(hw, glort,
+						      FM10K_XCAST_MODE_MULTI);
+			fm10k_queue_mac_request(interface, glort,
+						sdev->dev_addr,
+						hw->mac.default_vid, true);
+		}
+	}
+
 	fm10k_mbx_unlock(interface);
 
 	/* record updated xcast mode state */
@@ -1490,7 +1516,7 @@ static void *fm10k_dfwd_add_station(struct net_device *dev,
 		hw->mac.ops.update_xcast_mode(hw, glort,
 					      FM10K_XCAST_MODE_MULTI);
 		fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
-					0, true);
+					hw->mac.default_vid, true);
 	}
 
 	fm10k_mbx_unlock(interface);
@@ -1530,7 +1556,7 @@ static void fm10k_dfwd_del_station(struct net_device *dev, void *priv)
 		hw->mac.ops.update_xcast_mode(hw, glort,
 					      FM10K_XCAST_MODE_NONE);
 		fm10k_queue_mac_request(interface, glort, sdev->dev_addr,
-					0, false);
+					hw->mac.default_vid, false);
 	}
 
 	fm10k_mbx_unlock(interface);
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
index 425d814aed4d..d6406fc31ffb 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c
@@ -866,7 +866,7 @@ static s32 fm10k_iov_assign_default_mac_vlan_pf(struct fm10k_hw *hw,
 	/* Determine correct default VLAN ID. The FM10K_VLAN_OVERRIDE bit is
 	 * used here to indicate to the VF that it will not have privilege to
 	 * write VLAN_TABLE. All policy is enforced on the PF but this allows
-	 * the VF to correctly report errors to userspace rqeuests.
+	 * the VF to correctly report errors to userspace requests.
 	 */
 	if (vf_info->pf_vid)
 		vf_vid = vf_info->pf_vid | FM10K_VLAN_OVERRIDE;
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index e019baa905c5..46e9f4e0a02c 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -508,39 +508,40 @@ struct i40e_pf {
 #define I40E_HW_PORT_ID_VALID			BIT(17)
 #define I40E_HW_RESTART_AUTONEG			BIT(18)
 
-	u32 flags;
-#define I40E_FLAG_RX_CSUM_ENABLED		BIT(0)
-#define I40E_FLAG_MSI_ENABLED			BIT(1)
-#define I40E_FLAG_MSIX_ENABLED			BIT(2)
-#define I40E_FLAG_RSS_ENABLED			BIT(3)
-#define I40E_FLAG_VMDQ_ENABLED			BIT(4)
-#define I40E_FLAG_FILTER_SYNC			BIT(5)
-#define I40E_FLAG_SRIOV_ENABLED			BIT(6)
-#define I40E_FLAG_DCB_CAPABLE			BIT(7)
-#define I40E_FLAG_DCB_ENABLED			BIT(8)
-#define I40E_FLAG_FD_SB_ENABLED			BIT(9)
-#define I40E_FLAG_FD_ATR_ENABLED		BIT(10)
-#define I40E_FLAG_FD_SB_AUTO_DISABLED		BIT(11)
-#define I40E_FLAG_FD_ATR_AUTO_DISABLED		BIT(12)
-#define I40E_FLAG_MFP_ENABLED			BIT(13)
-#define I40E_FLAG_UDP_FILTER_SYNC		BIT(14)
-#define I40E_FLAG_HW_ATR_EVICT_ENABLED		BIT(15)
-#define I40E_FLAG_VEB_MODE_ENABLED		BIT(16)
-#define I40E_FLAG_VEB_STATS_ENABLED		BIT(17)
-#define I40E_FLAG_LINK_POLLING_ENABLED		BIT(18)
-#define I40E_FLAG_TRUE_PROMISC_SUPPORT		BIT(19)
-#define I40E_FLAG_TEMP_LINK_POLLING		BIT(20)
-#define I40E_FLAG_LEGACY_RX			BIT(21)
-#define I40E_FLAG_PTP				BIT(22)
-#define I40E_FLAG_IWARP_ENABLED			BIT(23)
-#define I40E_FLAG_SERVICE_CLIENT_REQUESTED	BIT(24)
-#define I40E_FLAG_CLIENT_L2_CHANGE		BIT(25)
-#define I40E_FLAG_CLIENT_RESET			BIT(26)
-#define I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED	BIT(27)
-#define I40E_FLAG_SOURCE_PRUNING_DISABLED	BIT(28)
-#define I40E_FLAG_TC_MQPRIO			BIT(29)
-#define I40E_FLAG_FD_SB_INACTIVE		BIT(30)
-#define I40E_FLAG_FD_SB_TO_CLOUD_FILTER		BIT(31)
+	u64 flags;
+#define I40E_FLAG_RX_CSUM_ENABLED		BIT_ULL(0)
+#define I40E_FLAG_MSI_ENABLED			BIT_ULL(1)
+#define I40E_FLAG_MSIX_ENABLED			BIT_ULL(2)
+#define I40E_FLAG_RSS_ENABLED			BIT_ULL(3)
+#define I40E_FLAG_VMDQ_ENABLED			BIT_ULL(4)
+#define I40E_FLAG_FILTER_SYNC			BIT_ULL(5)
+#define I40E_FLAG_SRIOV_ENABLED			BIT_ULL(6)
+#define I40E_FLAG_DCB_CAPABLE			BIT_ULL(7)
+#define I40E_FLAG_DCB_ENABLED			BIT_ULL(8)
+#define I40E_FLAG_FD_SB_ENABLED			BIT_ULL(9)
+#define I40E_FLAG_FD_ATR_ENABLED		BIT_ULL(10)
+#define I40E_FLAG_FD_SB_AUTO_DISABLED		BIT_ULL(11)
+#define I40E_FLAG_FD_ATR_AUTO_DISABLED		BIT_ULL(12)
+#define I40E_FLAG_MFP_ENABLED			BIT_ULL(13)
+#define I40E_FLAG_UDP_FILTER_SYNC		BIT_ULL(14)
+#define I40E_FLAG_HW_ATR_EVICT_ENABLED		BIT_ULL(15)
+#define I40E_FLAG_VEB_MODE_ENABLED		BIT_ULL(16)
+#define I40E_FLAG_VEB_STATS_ENABLED		BIT_ULL(17)
+#define I40E_FLAG_LINK_POLLING_ENABLED		BIT_ULL(18)
+#define I40E_FLAG_TRUE_PROMISC_SUPPORT		BIT_ULL(19)
+#define I40E_FLAG_TEMP_LINK_POLLING		BIT_ULL(20)
+#define I40E_FLAG_LEGACY_RX			BIT_ULL(21)
+#define I40E_FLAG_PTP				BIT_ULL(22)
+#define I40E_FLAG_IWARP_ENABLED			BIT_ULL(23)
+#define I40E_FLAG_SERVICE_CLIENT_REQUESTED	BIT_ULL(24)
+#define I40E_FLAG_CLIENT_L2_CHANGE		BIT_ULL(25)
+#define I40E_FLAG_CLIENT_RESET			BIT_ULL(26)
+#define I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED	BIT_ULL(27)
+#define I40E_FLAG_SOURCE_PRUNING_DISABLED	BIT_ULL(28)
+#define I40E_FLAG_TC_MQPRIO			BIT_ULL(29)
+#define I40E_FLAG_FD_SB_INACTIVE		BIT_ULL(30)
+#define I40E_FLAG_FD_SB_TO_CLOUD_FILTER		BIT_ULL(31)
+#define I40E_FLAG_DISABLE_FW_LLDP		BIT_ULL(32)
 
 	struct i40e_client_instance *cinst;
 	bool stat_offsets_loaded;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq.c b/drivers/net/ethernet/intel/i40e/i40e_adminq.c
index 9af74253c3f7..e78971605e0b 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_adminq.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_adminq.c
@@ -907,10 +907,15 @@ i40e_status i40e_asq_send_command(struct i40e_hw *hw,
 	/* update the error if time out occurred */
 	if ((!cmd_completed) &&
 	    (!details->async && !details->postpone)) {
-		i40e_debug(hw,
-			   I40E_DEBUG_AQ_MESSAGE,
-			   "AQTX: Writeback timeout.\n");
-		status = I40E_ERR_ADMIN_QUEUE_TIMEOUT;
+		if (rd32(hw, hw->aq.asq.len) & I40E_GL_ATQLEN_ATQCRIT_MASK) {
+			i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
+				   "AQTX: AQ Critical error.\n");
+			status = I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR;
+		} else {
+			i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
+				   "AQTX: Writeback timeout.\n");
+			status = I40E_ERR_ADMIN_QUEUE_TIMEOUT;
+		}
 	}
 
 asq_send_command_error:
@@ -971,7 +976,7 @@ i40e_status i40e_clean_arq_element(struct i40e_hw *hw,
 	}
 
 	/* set next_to_use to head */
-	ntu = (rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK);
+	ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK;
 	if (ntu == ntc) {
 		/* nothing to do - shouldn't need to update ring's values */
 		ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK;
@@ -1027,7 +1032,7 @@ i40e_status i40e_clean_arq_element(struct i40e_hw *hw,
 	hw->aq.arq.next_to_clean = ntc;
 	hw->aq.arq.next_to_use = ntu;
 
-	i40e_nvmupd_check_wait_event(hw, le16_to_cpu(e->desc.opcode));
+	i40e_nvmupd_check_wait_event(hw, le16_to_cpu(e->desc.opcode), &e->desc);
 clean_arq_element_out:
 	/* Set pending if needed, unlock and return */
 	if (pending)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
index b0188b8f91ba..a852775d3059 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_adminq_cmd.h
@@ -198,13 +198,14 @@ enum i40e_admin_queue_opc {
 	i40e_aqc_opc_add_mirror_rule	= 0x0260,
 	i40e_aqc_opc_delete_mirror_rule	= 0x0261,
 
-	/* Pipeline Personalization Profile */
+	/* Dynamic Device Personalization */
 	i40e_aqc_opc_write_personalization_profile	= 0x0270,
 	i40e_aqc_opc_get_personalization_profile_list	= 0x0271,
 
 	/* DCB commands */
 	i40e_aqc_opc_dcb_ignore_pfc	= 0x0301,
 	i40e_aqc_opc_dcb_updated	= 0x0302,
+	i40e_aqc_opc_set_dcb_parameters = 0x0303,
 
 	/* TX scheduler */
 	i40e_aqc_opc_configure_vsi_bw_limit		= 0x0400,
@@ -1594,7 +1595,7 @@ struct i40e_aqc_add_delete_mirror_rule_completion {
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_add_delete_mirror_rule_completion);
 
-/* Pipeline Personalization Profile */
+/* Dynamic Device Personalization */
 struct i40e_aqc_write_personalization_profile {
 	u8      flags;
 	u8      reserved[3];
@@ -1605,7 +1606,7 @@ struct i40e_aqc_write_personalization_profile {
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_write_personalization_profile);
 
-struct i40e_aqc_write_ppp_resp {
+struct i40e_aqc_write_ddp_resp {
 	__le32 error_offset;
 	__le32 error_info;
 	__le32 addr_high;
@@ -1614,8 +1615,8 @@ struct i40e_aqc_write_ppp_resp {
 
 struct i40e_aqc_get_applied_profiles {
 	u8      flags;
-#define I40E_AQC_GET_PPP_GET_CONF	0x1
-#define I40E_AQC_GET_PPP_GET_RDPU_CONF	0x2
+#define I40E_AQC_GET_DDP_GET_CONF	0x1
+#define I40E_AQC_GET_DDP_GET_RDPU_CONF	0x2
 	u8      rsv[3];
 	__le32  reserved;
 	__le32  addr_high;
@@ -2231,8 +2232,12 @@ I40E_CHECK_CMD_LENGTH(i40e_aqc_phy_register_access);
  */
 struct i40e_aqc_nvm_update {
 	u8	command_flags;
-#define I40E_AQ_NVM_LAST_CMD	0x01
-#define I40E_AQ_NVM_FLASH_ONLY	0x80
+#define I40E_AQ_NVM_LAST_CMD			0x01
+#define I40E_AQ_NVM_FLASH_ONLY			0x80
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_SHIFT	1
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_MASK	0x03
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_SELECTED	0x03
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_ALL	0x01
 	u8	module_pointer;
 	__le16	length;
 	__le32	offset;
@@ -2492,6 +2497,17 @@ struct i40e_aqc_lldp_start {
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_lldp_start);
 
+/* Set DCB (direct 0x0303) */
+struct i40e_aqc_set_dcb_parameters {
+	u8 command;
+#define I40E_AQ_DCB_SET_AGENT	0x1
+#define I40E_DCB_VALID		0x1
+	u8 valid_flags;
+	u8 reserved[14];
+};
+
+I40E_CHECK_CMD_LENGTH(i40e_aqc_set_dcb_parameters);
+
 /* Get CEE DCBX Oper Config (0x0A07)
  * uses the generic descriptor struct
  * returns below as indirect response
diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c
index 1b1e2acbd07f..0de9610c1d8d 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_client.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_client.c
@@ -378,11 +378,11 @@ void i40e_client_subtask(struct i40e_pf *pf)
 	if (!client || !cdev)
 		return;
 
-	/* Here we handle client opens. If the client is down, but
-	 * the netdev is up, then open the client.
+	/* Here we handle client opens. If the client is down, and
+	 * the netdev is registered, then open the client.
 	 */
 	if (!test_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state)) {
-		if (!test_bit(__I40E_VSI_DOWN, vsi->state) &&
+		if (vsi->netdev_registered &&
 		    client->ops && client->ops->open) {
 			set_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state);
 			ret = client->ops->open(&cdev->lan_info, client);
@@ -393,17 +393,19 @@ void i40e_client_subtask(struct i40e_pf *pf)
 				i40e_client_del_instance(pf);
 			}
 		}
-	} else {
-	/* Likewise for client close. If the client is up, but the netdev
-	 * is down, then close the client.
-	 */
-		if (test_bit(__I40E_VSI_DOWN, vsi->state) &&
-		    client->ops && client->ops->close) {
-			clear_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state);
-			client->ops->close(&cdev->lan_info, client, false);
-			i40e_client_release_qvlist(&cdev->lan_info);
-		}
 	}
+
+	/* enable/disable PE TCP_ENA flag based on netdev down/up
+	 */
+	if (test_bit(__I40E_VSI_DOWN, vsi->state))
+		i40e_client_update_vsi_ctxt(&cdev->lan_info, client,
+					    0, 0, 0,
+					    I40E_CLIENT_VSI_FLAG_TCP_ENABLE);
+	else
+		i40e_client_update_vsi_ctxt(&cdev->lan_info, client,
+					    0, 0,
+					    I40E_CLIENT_VSI_FLAG_TCP_ENABLE,
+					    I40E_CLIENT_VSI_FLAG_TCP_ENABLE);
 }
 
 /**
@@ -717,13 +719,13 @@ static int i40e_client_update_vsi_ctxt(struct i40e_info *ldev,
 		return -ENOENT;
 	}
 
-	if ((valid_flag & I40E_CLIENT_VSI_FLAG_TCP_PACKET_ENABLE) &&
-	    (flag & I40E_CLIENT_VSI_FLAG_TCP_PACKET_ENABLE)) {
+	if ((valid_flag & I40E_CLIENT_VSI_FLAG_TCP_ENABLE) &&
+	    (flag & I40E_CLIENT_VSI_FLAG_TCP_ENABLE)) {
 		ctxt.info.valid_sections =
 			cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
 		ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
-	} else if ((valid_flag & I40E_CLIENT_VSI_FLAG_TCP_PACKET_ENABLE) &&
-		  !(flag & I40E_CLIENT_VSI_FLAG_TCP_PACKET_ENABLE)) {
+	} else if ((valid_flag & I40E_CLIENT_VSI_FLAG_TCP_ENABLE) &&
+		  !(flag & I40E_CLIENT_VSI_FLAG_TCP_ENABLE)) {
 		ctxt.info.valid_sections =
 			cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
 		ctxt.info.queueing_opt_flags &= ~I40E_AQ_VSI_QUE_OPT_TCP_ENA;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.h b/drivers/net/ethernet/intel/i40e/i40e_client.h
index 15b21a5315b5..ba55c889e4c5 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_client.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_client.h
@@ -132,7 +132,7 @@ struct i40e_info {
 
 #define I40E_CLIENT_RESET_LEVEL_PF   1
 #define I40E_CLIENT_RESET_LEVEL_CORE 2
-#define I40E_CLIENT_VSI_FLAG_TCP_PACKET_ENABLE  BIT(1)
+#define I40E_CLIENT_VSI_FLAG_TCP_ENABLE  BIT(1)
 
 struct i40e_ops {
 	/* setup_q_vector_list enables queues with a particular vector */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c
index 095965f268bd..ef5a868aae46 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_common.c
@@ -278,6 +278,8 @@ const char *i40e_stat_str(struct i40e_hw *hw, i40e_status stat_err)
 		return "I40E_NOT_SUPPORTED";
 	case I40E_ERR_FIRMWARE_API_VERSION:
 		return "I40E_ERR_FIRMWARE_API_VERSION";
+	case I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
+		return "I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR";
 	}
 
 	snprintf(hw->err_str, sizeof(hw->err_str), "%d", stat_err);
@@ -1486,6 +1488,7 @@ u32 i40e_led_get(struct i40e_hw *hw)
 		case I40E_COMBINED_ACTIVITY:
 		case I40E_FILTER_ACTIVITY:
 		case I40E_MAC_ACTIVITY:
+		case I40E_LINK_ACTIVITY:
 			continue;
 		default:
 			break;
@@ -1534,6 +1537,7 @@ void i40e_led_set(struct i40e_hw *hw, u32 mode, bool blink)
 		case I40E_COMBINED_ACTIVITY:
 		case I40E_FILTER_ACTIVITY:
 		case I40E_MAC_ACTIVITY:
+		case I40E_LINK_ACTIVITY:
 			continue;
 		default:
 			break;
@@ -1544,9 +1548,6 @@ void i40e_led_set(struct i40e_hw *hw, u32 mode, bool blink)
 		gpio_val |= ((mode << I40E_GLGEN_GPIO_CTL_LED_MODE_SHIFT) &
 			     I40E_GLGEN_GPIO_CTL_LED_MODE_MASK);
 
-		if (mode == I40E_LINK_ACTIVITY)
-			blink = false;
-
 		if (blink)
 			gpio_val |= BIT(I40E_GLGEN_GPIO_CTL_LED_BLINK_SHIFT);
 		else
@@ -3465,13 +3466,14 @@ i40e_status i40e_aq_discover_capabilities(struct i40e_hw *hw,
  * @length: length of the section to be written (in bytes from the offset)
  * @data: command buffer (size [bytes] = length)
  * @last_command: tells if this is the last command in a series
+ * @preservation_flags: Preservation mode flags
  * @cmd_details: pointer to command details structure or NULL
  *
  * Update the NVM using the admin queue commands
  **/
 i40e_status i40e_aq_update_nvm(struct i40e_hw *hw, u8 module_pointer,
 			       u32 offset, u16 length, void *data,
-			       bool last_command,
+				bool last_command, u8 preservation_flags,
 			       struct i40e_asq_cmd_details *cmd_details)
 {
 	struct i40e_aq_desc desc;
@@ -3490,6 +3492,16 @@ i40e_status i40e_aq_update_nvm(struct i40e_hw *hw, u8 module_pointer,
 	/* If this is the last command in a series, set the proper flag. */
 	if (last_command)
 		cmd->command_flags |= I40E_AQ_NVM_LAST_CMD;
+	if (hw->mac.type == I40E_MAC_X722) {
+		if (preservation_flags == I40E_NVM_PRESERVATION_FLAGS_SELECTED)
+			cmd->command_flags |=
+				(I40E_AQ_NVM_PRESERVATION_FLAGS_SELECTED <<
+				 I40E_AQ_NVM_PRESERVATION_FLAGS_SHIFT);
+		else if (preservation_flags == I40E_NVM_PRESERVATION_FLAGS_ALL)
+			cmd->command_flags |=
+				(I40E_AQ_NVM_PRESERVATION_FLAGS_ALL <<
+				 I40E_AQ_NVM_PRESERVATION_FLAGS_SHIFT);
+	}
 	cmd->module_pointer = module_pointer;
 	cmd->offset = cpu_to_le32(offset);
 	cmd->length = cpu_to_le16(length);
@@ -3629,7 +3641,34 @@ i40e_status i40e_aq_start_lldp(struct i40e_hw *hw,
 	i40e_fill_default_direct_cmd_desc(&desc, i40e_aqc_opc_lldp_start);
 
 	cmd->command = I40E_AQ_LLDP_AGENT_START;
+	status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
+
+	return status;
+}
 
+/**
+ * i40e_aq_set_dcb_parameters
+ * @hw: pointer to the hw struct
+ * @cmd_details: pointer to command details structure or NULL
+ * @dcb_enable: True if DCB configuration needs to be applied
+ *
+ **/
+enum i40e_status_code
+i40e_aq_set_dcb_parameters(struct i40e_hw *hw, bool dcb_enable,
+			   struct i40e_asq_cmd_details *cmd_details)
+{
+	struct i40e_aq_desc desc;
+	struct i40e_aqc_set_dcb_parameters *cmd =
+		(struct i40e_aqc_set_dcb_parameters *)&desc.params.raw;
+	i40e_status status;
+
+	i40e_fill_default_direct_cmd_desc(&desc,
+					  i40e_aqc_opc_set_dcb_parameters);
+
+	if (dcb_enable) {
+		cmd->valid_flags = I40E_DCB_VALID;
+		cmd->command = I40E_AQ_DCB_SET_AGENT;
+	}
 	status = i40e_asq_send_command(hw, &desc, NULL, 0, cmd_details);
 
 	return status;
@@ -5236,7 +5275,7 @@ i40e_status i40e_aq_get_phy_register(struct i40e_hw *hw,
 }
 
 /**
- * i40e_aq_write_ppp - Write pipeline personalization profile (ppp)
+ * i40e_aq_write_ddp - Write dynamic device personalization (ddp)
  * @hw: pointer to the hw struct
  * @buff: command buffer (size in bytes = buff_size)
  * @buff_size: buffer size in bytes
@@ -5246,7 +5285,7 @@ i40e_status i40e_aq_get_phy_register(struct i40e_hw *hw,
  * @cmd_details: pointer to command details structure or NULL
  **/
 enum
-i40e_status_code i40e_aq_write_ppp(struct i40e_hw *hw, void *buff,
+i40e_status_code i40e_aq_write_ddp(struct i40e_hw *hw, void *buff,
 				   u16 buff_size, u32 track_id,
 				   u32 *error_offset, u32 *error_info,
 				   struct i40e_asq_cmd_details *cmd_details)
@@ -5255,7 +5294,7 @@ i40e_status_code i40e_aq_write_ppp(struct i40e_hw *hw, void *buff,
 	struct i40e_aqc_write_personalization_profile *cmd =
 		(struct i40e_aqc_write_personalization_profile *)
 		&desc.params.raw;
-	struct i40e_aqc_write_ppp_resp *resp;
+	struct i40e_aqc_write_ddp_resp *resp;
 	i40e_status status;
 
 	i40e_fill_default_direct_cmd_desc(&desc,
@@ -5271,7 +5310,7 @@ i40e_status_code i40e_aq_write_ppp(struct i40e_hw *hw, void *buff,
 
 	status = i40e_asq_send_command(hw, &desc, buff, buff_size, cmd_details);
 	if (!status) {
-		resp = (struct i40e_aqc_write_ppp_resp *)&desc.params.raw;
+		resp = (struct i40e_aqc_write_ddp_resp *)&desc.params.raw;
 		if (error_offset)
 			*error_offset = le32_to_cpu(resp->error_offset);
 		if (error_info)
@@ -5282,14 +5321,14 @@ i40e_status_code i40e_aq_write_ppp(struct i40e_hw *hw, void *buff,
 }
 
 /**
- * i40e_aq_get_ppp_list - Read pipeline personalization profile (ppp)
+ * i40e_aq_get_ddp_list - Read dynamic device personalization (ddp)
  * @hw: pointer to the hw struct
  * @buff: command buffer (size in bytes = buff_size)
  * @buff_size: buffer size in bytes
  * @cmd_details: pointer to command details structure or NULL
  **/
 enum
-i40e_status_code i40e_aq_get_ppp_list(struct i40e_hw *hw, void *buff,
+i40e_status_code i40e_aq_get_ddp_list(struct i40e_hw *hw, void *buff,
 				      u16 buff_size, u8 flags,
 				      struct i40e_asq_cmd_details *cmd_details)
 {
@@ -5364,11 +5403,6 @@ i40e_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile,
 	u32 offset = 0, info = 0;
 	u32 i;
 
-	if (!track_id) {
-		i40e_debug(hw, I40E_DEBUG_PACKAGE, "Track_id can't be 0.");
-		return I40E_NOT_SUPPORTED;
-	}
-
 	dev_cnt = profile->device_table_count;
 
 	for (i = 0; i < dev_cnt; i++) {
@@ -5378,7 +5412,7 @@ i40e_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile,
 				break;
 	}
 	if (i == dev_cnt) {
-		i40e_debug(hw, I40E_DEBUG_PACKAGE, "Device doesn't support PPP");
+		i40e_debug(hw, I40E_DEBUG_PACKAGE, "Device doesn't support DDP");
 		return I40E_ERR_DEVICE_NOT_SUPPORTED;
 	}
 
@@ -5397,7 +5431,7 @@ i40e_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile,
 			sizeof(struct i40e_profile_section_header);
 
 		/* Write profile */
-		status = i40e_aq_write_ppp(hw, (void *)sec, (u16)section_size,
+		status = i40e_aq_write_ddp(hw, (void *)sec, (u16)section_size,
 					   track_id, &offset, &info, NULL);
 		if (status) {
 			i40e_debug(hw, I40E_DEBUG_PACKAGE,
@@ -5439,10 +5473,10 @@ i40e_add_pinfo_to_list(struct i40e_hw *hw,
 					     sec->section.offset);
 	pinfo->track_id = track_id;
 	pinfo->version = profile->version;
-	pinfo->op = I40E_PPP_ADD_TRACKID;
-	memcpy(pinfo->name, profile->name, I40E_PPP_NAME_SIZE);
+	pinfo->op = I40E_DDP_ADD_TRACKID;
+	memcpy(pinfo->name, profile->name, I40E_DDP_NAME_SIZE);
 
-	status = i40e_aq_write_ppp(hw, (void *)sec, sec->data_end,
+	status = i40e_aq_write_ddp(hw, (void *)sec, sec->data_end,
 				   track_id, &offset, &info, NULL);
 
 	return status;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index 5f6cf7212d4f..2f5bee713fef 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -126,6 +126,10 @@ static const struct i40e_stats i40e_gstrings_stats[] = {
 	I40E_PF_STAT("link_xoff_rx", stats.link_xoff_rx),
 	I40E_PF_STAT("link_xon_tx", stats.link_xon_tx),
 	I40E_PF_STAT("link_xoff_tx", stats.link_xoff_tx),
+	I40E_PF_STAT("priority_xon_rx", stats.priority_xon_rx),
+	I40E_PF_STAT("priority_xoff_rx", stats.priority_xoff_rx),
+	I40E_PF_STAT("priority_xon_tx", stats.priority_xon_tx),
+	I40E_PF_STAT("priority_xoff_tx", stats.priority_xoff_tx),
 	I40E_PF_STAT("rx_size_64", stats.rx_size_64),
 	I40E_PF_STAT("rx_size_127", stats.rx_size_127),
 	I40E_PF_STAT("rx_size_255", stats.rx_size_255),
@@ -229,6 +233,7 @@ static const struct i40e_priv_flags i40e_gstrings_priv_flags[] = {
 	I40E_PRIV_FLAG("legacy-rx", I40E_FLAG_LEGACY_RX, 0),
 	I40E_PRIV_FLAG("disable-source-pruning",
 		       I40E_FLAG_SOURCE_PRUNING_DISABLED, 0),
+	I40E_PRIV_FLAG("disable-fw-lldp", I40E_FLAG_DISABLE_FW_LLDP, 0),
 };
 
 #define I40E_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_gstrings_priv_flags)
@@ -1585,6 +1590,8 @@ static int i40e_set_ringparam(struct net_device *netdev,
 			 */
 			rx_rings[i].desc = NULL;
 			rx_rings[i].rx_bi = NULL;
+			/* Clear cloned XDP RX-queue info before setup call */
+			memset(&rx_rings[i].xdp_rxq, 0, sizeof(rx_rings[i].xdp_rxq));
 			/* this is to allow wr32 to have something to write to
 			 * during early allocation of Rx buffers
 			 */
@@ -2299,6 +2306,8 @@ static void i40e_set_itr_per_queue(struct i40e_vsi *vsi,
 				   struct ethtool_coalesce *ec,
 				   int queue)
 {
+	struct i40e_ring *rx_ring = vsi->rx_rings[queue];
+	struct i40e_ring *tx_ring = vsi->tx_rings[queue];
 	struct i40e_pf *pf = vsi->back;
 	struct i40e_hw *hw = &pf->hw;
 	struct i40e_q_vector *q_vector;
@@ -2306,26 +2315,26 @@ static void i40e_set_itr_per_queue(struct i40e_vsi *vsi,
 
 	intrl = i40e_intrl_usec_to_reg(vsi->int_rate_limit);
 
-	vsi->rx_rings[queue]->rx_itr_setting = ec->rx_coalesce_usecs;
-	vsi->tx_rings[queue]->tx_itr_setting = ec->tx_coalesce_usecs;
+	rx_ring->rx_itr_setting = ec->rx_coalesce_usecs;
+	tx_ring->tx_itr_setting = ec->tx_coalesce_usecs;
 
 	if (ec->use_adaptive_rx_coalesce)
-		vsi->rx_rings[queue]->rx_itr_setting |= I40E_ITR_DYNAMIC;
+		rx_ring->rx_itr_setting |= I40E_ITR_DYNAMIC;
 	else
-		vsi->rx_rings[queue]->rx_itr_setting &= ~I40E_ITR_DYNAMIC;
+		rx_ring->rx_itr_setting &= ~I40E_ITR_DYNAMIC;
 
 	if (ec->use_adaptive_tx_coalesce)
-		vsi->tx_rings[queue]->tx_itr_setting |= I40E_ITR_DYNAMIC;
+		tx_ring->tx_itr_setting |= I40E_ITR_DYNAMIC;
 	else
-		vsi->tx_rings[queue]->tx_itr_setting &= ~I40E_ITR_DYNAMIC;
+		tx_ring->tx_itr_setting &= ~I40E_ITR_DYNAMIC;
 
-	q_vector = vsi->rx_rings[queue]->q_vector;
-	q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[queue]->rx_itr_setting);
+	q_vector = rx_ring->q_vector;
+	q_vector->rx.itr = ITR_TO_REG(rx_ring->rx_itr_setting);
 	vector = vsi->base_vector + q_vector->v_idx;
 	wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), q_vector->rx.itr);
 
-	q_vector = vsi->tx_rings[queue]->q_vector;
-	q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[queue]->tx_itr_setting);
+	q_vector = tx_ring->q_vector;
+	q_vector->tx.itr = ITR_TO_REG(tx_ring->tx_itr_setting);
 	vector = vsi->base_vector + q_vector->v_idx;
 	wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), q_vector->tx.itr);
 
@@ -2740,16 +2749,16 @@ static int i40e_get_ethtool_fdir_entry(struct i40e_pf *pf,
 
 no_input_set:
 	if (input_set & I40E_L3_SRC_MASK)
-		fsp->m_u.tcp_ip4_spec.ip4src = htonl(0xFFFF);
+		fsp->m_u.tcp_ip4_spec.ip4src = htonl(0xFFFFFFFF);
 
 	if (input_set & I40E_L3_DST_MASK)
-		fsp->m_u.tcp_ip4_spec.ip4dst = htonl(0xFFFF);
+		fsp->m_u.tcp_ip4_spec.ip4dst = htonl(0xFFFFFFFF);
 
 	if (input_set & I40E_L4_SRC_MASK)
-		fsp->m_u.tcp_ip4_spec.psrc = htons(0xFFFFFFFF);
+		fsp->m_u.tcp_ip4_spec.psrc = htons(0xFFFF);
 
 	if (input_set & I40E_L4_DST_MASK)
-		fsp->m_u.tcp_ip4_spec.pdst = htons(0xFFFFFFFF);
+		fsp->m_u.tcp_ip4_spec.pdst = htons(0xFFFF);
 
 	if (rule->dest_ctl == I40E_FILTER_PROGRAM_DESC_DEST_DROP_PACKET)
 		fsp->ring_cookie = RX_CLS_FLOW_DISC;
@@ -3800,6 +3809,16 @@ static int i40e_check_fdir_input_set(struct i40e_vsi *vsi,
 
 	i40e_write_fd_input_set(pf, index, new_mask);
 
+	/* IP_USER_FLOW filters match both IPv4/Other and IPv4/Fragmented
+	 * frames. If we're programming the input set for IPv4/Other, we also
+	 * need to program the IPv4/Fragmented input set. Since we don't have
+	 * separate support, we'll always assume and enforce that the two flow
+	 * types must have matching input sets.
+	 */
+	if (index == I40E_FILTER_PCTYPE_NONF_IPV4_OTHER)
+		i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
+					new_mask);
+
 	/* Add the new offset and update table, if necessary */
 	if (new_flex_offset) {
 		err = i40e_add_flex_offset(&pf->l4_flex_pit_list, src_offset,
@@ -3821,6 +3840,87 @@ static int i40e_check_fdir_input_set(struct i40e_vsi *vsi,
 	return 0;
 }
 
+/**
+ * i40e_match_fdir_filter - Return true of two filters match
+ * @a: pointer to filter struct
+ * @b: pointer to filter struct
+ *
+ * Returns true if the two filters match exactly the same criteria. I.e. they
+ * match the same flow type and have the same parameters. We don't need to
+ * check any input-set since all filters of the same flow type must use the
+ * same input set.
+ **/
+static bool i40e_match_fdir_filter(struct i40e_fdir_filter *a,
+				   struct i40e_fdir_filter *b)
+{
+	/* The filters do not much if any of these criteria differ. */
+	if (a->dst_ip != b->dst_ip ||
+	    a->src_ip != b->src_ip ||
+	    a->dst_port != b->dst_port ||
+	    a->src_port != b->src_port ||
+	    a->flow_type != b->flow_type ||
+	    a->ip4_proto != b->ip4_proto)
+		return false;
+
+	return true;
+}
+
+/**
+ * i40e_disallow_matching_filters - Check that new filters differ
+ * @vsi: pointer to the targeted VSI
+ * @input: new filter to check
+ *
+ * Due to hardware limitations, it is not possible for two filters that match
+ * similar criteria to be programmed at the same time. This is true for a few
+ * reasons:
+ *
+ * (a) all filters matching a particular flow type must use the same input
+ * set, that is they must match the same criteria.
+ * (b) different flow types will never match the same packet, as the flow type
+ * is decided by hardware before checking which rules apply.
+ * (c) hardware has no way to distinguish which order filters apply in.
+ *
+ * Due to this, we can't really support using the location data to order
+ * filters in the hardware parsing. It is technically possible for the user to
+ * request two filters matching the same criteria but which select different
+ * queues. In this case, rather than keep both filters in the list, we reject
+ * the 2nd filter when the user requests adding it.
+ *
+ * This avoids needing to track location for programming the filter to
+ * hardware, and ensures that we avoid some strange scenarios involving
+ * deleting filters which match the same criteria.
+ **/
+static int i40e_disallow_matching_filters(struct i40e_vsi *vsi,
+					  struct i40e_fdir_filter *input)
+{
+	struct i40e_pf *pf = vsi->back;
+	struct i40e_fdir_filter *rule;
+	struct hlist_node *node2;
+
+	/* Loop through every filter, and check that it doesn't match */
+	hlist_for_each_entry_safe(rule, node2,
+				  &pf->fdir_filter_list, fdir_node) {
+		/* Don't check the filters match if they share the same fd_id,
+		 * since the new filter is actually just updating the target
+		 * of the old filter.
+		 */
+		if (rule->fd_id == input->fd_id)
+			continue;
+
+		/* If any filters match, then print a warning message to the
+		 * kernel message buffer and bail out.
+		 */
+		if (i40e_match_fdir_filter(rule, input)) {
+			dev_warn(&pf->pdev->dev,
+				 "Existing user defined filter %d already matches this flow.\n",
+				 rule->fd_id);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 /**
  * i40e_add_fdir_ethtool - Add/Remove Flow Director filters
  * @vsi: pointer to the targeted VSI
@@ -3933,19 +4033,25 @@ static int i40e_add_fdir_ethtool(struct i40e_vsi *vsi,
 		input->flex_offset = userdef.flex_offset;
 	}
 
-	ret = i40e_add_del_fdir(vsi, input, true);
+	/* Avoid programming two filters with identical match criteria. */
+	ret = i40e_disallow_matching_filters(vsi, input);
 	if (ret)
-		goto free_input;
+		goto free_filter_memory;
 
 	/* Add the input filter to the fdir_input_list, possibly replacing
 	 * a previous filter. Do not free the input structure after adding it
 	 * to the list as this would cause a use-after-free bug.
 	 */
 	i40e_update_ethtool_fdir_entry(vsi, input, fsp->location, NULL);
-
+	ret = i40e_add_del_fdir(vsi, input, true);
+	if (ret)
+		goto remove_sw_rule;
 	return 0;
 
-free_input:
+remove_sw_rule:
+	hlist_del(&input->fdir_node);
+	pf->fdir_pf_active_filters--;
+free_filter_memory:
 	kfree(input);
 	return ret;
 }
@@ -4258,7 +4364,7 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	struct i40e_netdev_priv *np = netdev_priv(dev);
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
-	u32 orig_flags, new_flags, changed_flags;
+	u64 orig_flags, new_flags, changed_flags;
 	u32 i, j;
 
 	orig_flags = READ_ONCE(pf->flags);
@@ -4309,13 +4415,32 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 	    !(pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE))
 		return -EOPNOTSUPP;
 
+	/* Disable FW LLDP not supported if NPAR active or if FW
+	 * API version < 1.7
+	 */
+	if (new_flags & I40E_FLAG_DISABLE_FW_LLDP) {
+		if (pf->hw.func_caps.npar_enable) {
+			dev_warn(&pf->pdev->dev,
+				 "Unable to stop FW LLDP if NPAR active\n");
+			return -EOPNOTSUPP;
+		}
+
+		if (pf->hw.aq.api_maj_ver < 1 ||
+		    (pf->hw.aq.api_maj_ver == 1 &&
+		     pf->hw.aq.api_min_ver < 7)) {
+			dev_warn(&pf->pdev->dev,
+				 "FW ver does not support stopping FW LLDP\n");
+			return -EOPNOTSUPP;
+		}
+	}
+
 	/* Compare and exchange the new flags into place. If we failed, that
 	 * is if cmpxchg returns anything but the old value, this means that
 	 * something else has modified the flags variable since we copied it
 	 * originally. We'll just punt with an error and log something in the
 	 * message buffer.
 	 */
-	if (cmpxchg(&pf->flags, orig_flags, new_flags) != orig_flags) {
+	if (cmpxchg64(&pf->flags, orig_flags, new_flags) != orig_flags) {
 		dev_warn(&pf->pdev->dev,
 			 "Unable to update pf->flags as it was modified by another thread...\n");
 		return -EAGAIN;
@@ -4354,12 +4479,37 @@ static int i40e_set_priv_flags(struct net_device *dev, u32 flags)
 		}
 	}
 
+	if (changed_flags & I40E_FLAG_DISABLE_FW_LLDP) {
+		if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
+			struct i40e_dcbx_config *dcbcfg;
+			int i;
+
+			i40e_aq_stop_lldp(&pf->hw, true, NULL);
+			i40e_aq_set_dcb_parameters(&pf->hw, true, NULL);
+			/* reset local_dcbx_config to default */
+			dcbcfg = &pf->hw.local_dcbx_config;
+			dcbcfg->etscfg.willing = 1;
+			dcbcfg->etscfg.maxtcs = 0;
+			dcbcfg->etscfg.tcbwtable[0] = 100;
+			for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++)
+				dcbcfg->etscfg.tcbwtable[i] = 0;
+			for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
+				dcbcfg->etscfg.prioritytable[i] = 0;
+			dcbcfg->etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
+			dcbcfg->pfc.willing = 1;
+			dcbcfg->pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
+		} else {
+			i40e_aq_start_lldp(&pf->hw, NULL);
+		}
+	}
+
 	/* Issue reset to cause things to take effect, as additional bits
 	 * are added we will need to create a mask of bits requiring reset
 	 */
 	if (changed_flags & (I40E_FLAG_VEB_STATS_ENABLED |
 			     I40E_FLAG_LEGACY_RX |
-			     I40E_FLAG_SOURCE_PRUNING_DISABLED))
+			     I40E_FLAG_SOURCE_PRUNING_DISABLED |
+			     I40E_FLAG_DISABLE_FW_LLDP))
 		i40e_do_reset(pf, BIT(__I40E_PF_RESET_REQUESTED), true);
 
 	return 0;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index af792112a2d3..e31adbc75f9c 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -47,8 +47,8 @@ static const char i40e_driver_string[] =
 #define DRV_KERN "-k"
 
 #define DRV_VERSION_MAJOR 2
-#define DRV_VERSION_MINOR 1
-#define DRV_VERSION_BUILD 14
+#define DRV_VERSION_MINOR 3
+#define DRV_VERSION_BUILD 2
 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
 	     __stringify(DRV_VERSION_MINOR) "." \
 	     __stringify(DRV_VERSION_BUILD)    DRV_KERN
@@ -1785,7 +1785,7 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 	struct i40e_pf *pf = vsi->back;
 	u16 sections = 0;
 	u8 netdev_tc = 0;
-	u16 numtc = 0;
+	u16 numtc = 1;
 	u16 qcount;
 	u8 offset;
 	u16 qmap;
@@ -1795,9 +1795,11 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
 	offset = 0;
 
+	/* Number of queues per enabled TC */
+	num_tc_qps = vsi->alloc_queue_pairs;
 	if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
 		/* Find numtc from enabled TC bitmap */
-		for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
+		for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 			if (enabled_tc & BIT(i)) /* TC is enabled */
 				numtc++;
 		}
@@ -1805,18 +1807,17 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 			dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
 			numtc = 1;
 		}
-	} else {
-		/* At least TC0 is enabled in non-DCB, non-MQPRIO case */
-		numtc = 1;
+		num_tc_qps = num_tc_qps / numtc;
+		num_tc_qps = min_t(int, num_tc_qps,
+				   i40e_pf_get_max_q_per_tc(pf));
 	}
 
 	vsi->tc_config.numtc = numtc;
 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
-	/* Number of queues per enabled TC */
-	qcount = vsi->alloc_queue_pairs;
 
-	num_tc_qps = qcount / numtc;
-	num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
+	/* Do not allow use more TC queue pairs than MSI-X vectors exist */
+	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
+		num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
 
 	/* Setup queue offset/count for all TCs for given VSI */
 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
@@ -1827,9 +1828,13 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
 
 			switch (vsi->type) {
 			case I40E_VSI_MAIN:
-				qcount = min_t(int, pf->alloc_rss_size,
-					       num_tc_qps);
-				break;
+				if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
+				    I40E_FLAG_FD_ATR_ENABLED)) ||
+				    vsi->tc_config.enabled_tc != 1) {
+					qcount = min_t(int, pf->alloc_rss_size,
+						       num_tc_qps);
+					break;
+				}
 			case I40E_VSI_FDIR:
 			case I40E_VSI_SRIOV:
 			case I40E_VSI_VMDQ2:
@@ -4122,6 +4127,7 @@ static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
 		num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
 
 		q_vector->num_ringpairs = num_ringpairs;
+		q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
 
 		q_vector->rx.count = 0;
 		q_vector->tx.count = 0;
@@ -4876,104 +4882,6 @@ static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
 
 #endif
 
-/**
- * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
- * @q_idx: TX queue number
- * @vsi: Pointer to VSI struct
- *
- * This function checks specified queue for given VSI. Detects hung condition.
- * We proactively detect hung TX queues by checking if interrupts are disabled
- * but there are pending descriptors.  If it appears hung, attempt to recover
- * by triggering a SW interrupt.
- **/
-static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
-{
-	struct i40e_ring *tx_ring = NULL;
-	struct i40e_pf	*pf;
-	u32 val, tx_pending;
-	int i;
-
-	pf = vsi->back;
-
-	/* now that we have an index, find the tx_ring struct */
-	for (i = 0; i < vsi->num_queue_pairs; i++) {
-		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
-			if (q_idx == vsi->tx_rings[i]->queue_index) {
-				tx_ring = vsi->tx_rings[i];
-				break;
-			}
-		}
-	}
-
-	if (!tx_ring)
-		return;
-
-	/* Read interrupt register */
-	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
-		val = rd32(&pf->hw,
-			   I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
-					       tx_ring->vsi->base_vector - 1));
-	else
-		val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
-
-	tx_pending = i40e_get_tx_pending(tx_ring);
-
-	/* Interrupts are disabled and TX pending is non-zero,
-	 * trigger the SW interrupt (don't wait). Worst case
-	 * there will be one extra interrupt which may result
-	 * into not cleaning any queues because queues are cleaned.
-	 */
-	if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK)))
-		i40e_force_wb(vsi, tx_ring->q_vector);
-}
-
-/**
- * i40e_detect_recover_hung - Function to detect and recover hung_queues
- * @pf:  pointer to PF struct
- *
- * LAN VSI has netdev and netdev has TX queues. This function is to check
- * each of those TX queues if they are hung, trigger recovery by issuing
- * SW interrupt.
- **/
-static void i40e_detect_recover_hung(struct i40e_pf *pf)
-{
-	struct net_device *netdev;
-	struct i40e_vsi *vsi;
-	unsigned int i;
-
-	/* Only for LAN VSI */
-	vsi = pf->vsi[pf->lan_vsi];
-
-	if (!vsi)
-		return;
-
-	/* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
-	if (test_bit(__I40E_VSI_DOWN, vsi->back->state) ||
-	    test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state))
-		return;
-
-	/* Make sure type is MAIN VSI */
-	if (vsi->type != I40E_VSI_MAIN)
-		return;
-
-	netdev = vsi->netdev;
-	if (!netdev)
-		return;
-
-	/* Bail out if netif_carrier is not OK */
-	if (!netif_carrier_ok(netdev))
-		return;
-
-	/* Go thru' TX queues for netdev */
-	for (i = 0; i < netdev->num_tx_queues; i++) {
-		struct netdev_queue *q;
-
-		q = netdev_get_tx_queue(netdev, i);
-		if (q)
-			i40e_detect_recover_hung_queue(i, vsi);
-	}
-}
-
 /**
  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
  * @pf: pointer to PF
@@ -5342,6 +5250,8 @@ static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
 {
 	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
+	struct i40e_pf *pf = vsi->back;
+	struct i40e_hw *hw = &pf->hw;
 	struct i40e_vsi_context ctxt;
 	int ret = 0;
 	int i;
@@ -5359,10 +5269,40 @@ static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
 
 	ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
 	if (ret) {
-		dev_info(&vsi->back->pdev->dev,
+		struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
+
+		dev_info(&pf->pdev->dev,
 			 "Failed configuring TC map %d for VSI %d\n",
 			 enabled_tc, vsi->seid);
-		goto out;
+		ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
+						  &bw_config, NULL);
+		if (ret) {
+			dev_info(&pf->pdev->dev,
+				 "Failed querying vsi bw info, err %s aq_err %s\n",
+				 i40e_stat_str(hw, ret),
+				 i40e_aq_str(hw, hw->aq.asq_last_status));
+			goto out;
+		}
+		if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
+			u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
+
+			if (!valid_tc)
+				valid_tc = bw_config.tc_valid_bits;
+			/* Always enable TC0, no matter what */
+			valid_tc |= 1;
+			dev_info(&pf->pdev->dev,
+				 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
+				 enabled_tc, bw_config.tc_valid_bits, valid_tc);
+			enabled_tc = valid_tc;
+		}
+
+		ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
+		if (ret) {
+			dev_err(&pf->pdev->dev,
+				"Unable to  configure TC map %d for VSI %d\n",
+				enabled_tc, vsi->seid);
+			goto out;
+		}
 	}
 
 	/* Update Queue Pairs Mapping for currently enabled UPs */
@@ -5402,13 +5342,12 @@ static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
 	/* Update the VSI after updating the VSI queue-mapping
 	 * information
 	 */
-	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
+	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
 	if (ret) {
-		dev_info(&vsi->back->pdev->dev,
+		dev_info(&pf->pdev->dev,
 			 "Update vsi tc config failed, err %s aq_err %s\n",
-			 i40e_stat_str(&vsi->back->hw, ret),
-			 i40e_aq_str(&vsi->back->hw,
-				     vsi->back->hw.aq.asq_last_status));
+			 i40e_stat_str(hw, ret),
+			 i40e_aq_str(hw, hw->aq.asq_last_status));
 		goto out;
 	}
 	/* update the local VSI info with updated queue map */
@@ -5418,11 +5357,10 @@ static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
 	/* Update current VSI BW information */
 	ret = i40e_vsi_get_bw_info(vsi);
 	if (ret) {
-		dev_info(&vsi->back->pdev->dev,
+		dev_info(&pf->pdev->dev,
 			 "Failed updating vsi bw info, err %s aq_err %s\n",
-			 i40e_stat_str(&vsi->back->hw, ret),
-			 i40e_aq_str(&vsi->back->hw,
-				     vsi->back->hw.aq.asq_last_status));
+			 i40e_stat_str(hw, ret),
+			 i40e_aq_str(hw, hw->aq.asq_last_status));
 		goto out;
 	}
 
@@ -6388,8 +6326,11 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 	struct i40e_hw *hw = &pf->hw;
 	int err = 0;
 
-	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
-	if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT)
+	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable
+	 * Also do not enable DCBx if FW LLDP agent is disabled
+	 */
+	if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) ||
+	    (pf->flags & I40E_FLAG_DISABLE_FW_LLDP))
 		goto out;
 
 	/* Get the initial DCB configuration */
@@ -6416,6 +6357,9 @@ static int i40e_init_pf_dcb(struct i40e_pf *pf)
 			dev_dbg(&pf->pdev->dev,
 				"DCBX offload is supported for this PF.\n");
 		}
+	} else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
+		dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
+		pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
 	} else {
 		dev_info(&pf->pdev->dev,
 			 "Query for DCB configuration failed, err %s aq_err %s\n",
@@ -7505,11 +7449,6 @@ static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
 {
 	struct i40e_vsi *vsi = np->vsi;
 
-	if (!tc_can_offload(vsi->netdev))
-		return -EOPNOTSUPP;
-	if (cls_flower->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (cls_flower->command) {
 	case TC_CLSFLOWER_REPLACE:
 		return i40e_configure_clsflower(vsi, cls_flower);
@@ -7527,6 +7466,9 @@ static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 {
 	struct i40e_netdev_priv *np = cb_priv;
 
+	if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
+		return -EOPNOTSUPP;
+
 	switch (type) {
 	case TC_SETUP_CLSFLOWER:
 		return i40e_setup_tc_cls_flower(np, type_data);
@@ -7744,6 +7686,9 @@ static void i40e_fdir_filter_exit(struct i40e_pf *pf)
 	/* Reprogram the default input set for Other/IPv4 */
 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
+
+	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
+				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
 }
 
 /**
@@ -9077,6 +9022,17 @@ static int i40e_rebuild_channels(struct i40e_vsi *vsi)
 				 vsi->uplink_seid);
 			return ret;
 		}
+		/* Reconfigure TX queues using QTX_CTL register */
+		ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
+		if (ret) {
+			dev_info(&vsi->back->pdev->dev,
+				 "failed to configure TX rings for channel %u\n",
+				 ch->seid);
+			return ret;
+		}
+		/* update 'next_base_queue' */
+		vsi->next_base_queue = vsi->next_base_queue +
+							ch->num_queue_pairs;
 		if (ch->max_tx_rate) {
 			u64 credits = ch->max_tx_rate;
 
@@ -9280,6 +9236,9 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
 		goto end_core_reset;
 	}
 
+	/* Enable FW to write a default DCB config on link-up */
+	i40e_aq_set_dcb_parameters(hw, true, NULL);
+
 #ifdef CONFIG_I40E_DCB
 	ret = i40e_init_pf_dcb(pf);
 	if (ret) {
@@ -9697,7 +9656,7 @@ static void i40e_service_task(struct work_struct *work)
 	if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
 		return;
 
-	i40e_detect_recover_hung(pf);
+	i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
 	i40e_sync_filters_subtask(pf);
 	i40e_reset_subtask(pf);
 	i40e_handle_mdd_event(pf);
@@ -10464,10 +10423,9 @@ static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
 	/* set up vector assignment tracking */
 	size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
 	pf->irq_pile = kzalloc(size, GFP_KERNEL);
-	if (!pf->irq_pile) {
-		dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
+	if (!pf->irq_pile)
 		return -ENOMEM;
-	}
+
 	pf->irq_pile->num_entries = vectors;
 	pf->irq_pile->search_hint = 0;
 
@@ -10785,8 +10743,13 @@ static int i40e_pf_config_rss(struct i40e_pf *pf)
 	/* Determine the RSS size of the VSI */
 	if (!vsi->rss_size) {
 		u16 qcount;
-
-		qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
+		/* If the firmware does something weird during VSI init, we
+		 * could end up with zero TCs. Check for that to avoid
+		 * divide-by-zero. It probably won't pass traffic, but it also
+		 * won't panic.
+		 */
+		qcount = vsi->num_queue_pairs /
+			 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
 	}
 	if (!vsi->rss_size)
@@ -10974,7 +10937,7 @@ i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
 	ret = i40e_aq_update_nvm(&pf->hw,
 				 I40E_SR_NVM_CONTROL_WORD,
 				 0x10, sizeof(nvm_word),
-				 &nvm_word, true, NULL);
+				 &nvm_word, true, 0, NULL);
 	/* Save off last admin queue command status before releasing
 	 * the NVM
 	 */
@@ -11115,13 +11078,13 @@ static int i40e_sw_init(struct i40e_pf *pf)
 	    pf->hw.aq.fw_maj_ver >= 6)
 		pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
 
-	if (pf->hw.func_caps.vmdq) {
+	if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
 		pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
 		pf->flags |= I40E_FLAG_VMDQ_ENABLED;
 		pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
 	}
 
-	if (pf->hw.func_caps.iwarp) {
+	if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
 		pf->flags |= I40E_FLAG_IWARP_ENABLED;
 		/* IWARP needs one extra vector for CQP just like MISC.*/
 		pf->num_iwarp_msix = (int)num_online_cpus() + 1;
@@ -13598,6 +13561,10 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	pci_set_drvdata(pdev, pf);
 	pci_save_state(pdev);
+
+	/* Enable FW to write default DCB config on link-up */
+	i40e_aq_set_dcb_parameters(hw, true, NULL);
+
 #ifdef CONFIG_I40E_DCB
 	err = i40e_init_pf_dcb(pf);
 	if (err) {
diff --git a/drivers/net/ethernet/intel/i40e/i40e_nvm.c b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
index 7689c2ee0d46..76a5cb04e4fe 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_nvm.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_nvm.c
@@ -239,8 +239,9 @@ static i40e_status i40e_read_nvm_word_srctl(struct i40e_hw *hw, u16 offset,
  *
  * Writes a 16 bit words buffer to the Shadow RAM using the admin command.
  **/
-static i40e_status i40e_read_nvm_aq(struct i40e_hw *hw, u8 module_pointer,
-				    u32 offset, u16 words, void *data,
+static i40e_status i40e_read_nvm_aq(struct i40e_hw *hw,
+				    u8 module_pointer, u32 offset,
+				    u16 words, void *data,
 				    bool last_command)
 {
 	i40e_status ret_code = I40E_ERR_NVM;
@@ -389,7 +390,7 @@ static i40e_status i40e_read_nvm_buffer_aq(struct i40e_hw *hw, u16 offset,
 					   u16 *words, u16 *data)
 {
 	i40e_status ret_code;
-	u16 read_size = *words;
+	u16 read_size;
 	bool last_cmd = false;
 	u16 words_read = 0;
 	u16 i = 0;
@@ -496,7 +497,8 @@ static i40e_status i40e_write_nvm_aq(struct i40e_hw *hw, u8 module_pointer,
 		ret_code = i40e_aq_update_nvm(hw, module_pointer,
 					      2 * offset,  /*bytes*/
 					      2 * words,   /*bytes*/
-					      data, last_command, &cmd_details);
+					      data, last_command, 0,
+					      &cmd_details);
 
 	return ret_code;
 }
@@ -677,6 +679,9 @@ static i40e_status i40e_nvmupd_exec_aq(struct i40e_hw *hw,
 static i40e_status i40e_nvmupd_get_aq_result(struct i40e_hw *hw,
 					     struct i40e_nvm_access *cmd,
 					     u8 *bytes, int *perrno);
+static i40e_status i40e_nvmupd_get_aq_event(struct i40e_hw *hw,
+					    struct i40e_nvm_access *cmd,
+					    u8 *bytes, int *perrno);
 static inline u8 i40e_nvmupd_get_module(u32 val)
 {
 	return (u8)(val & I40E_NVM_MOD_PNT_MASK);
@@ -686,6 +691,12 @@ static inline u8 i40e_nvmupd_get_transaction(u32 val)
 	return (u8)((val & I40E_NVM_TRANS_MASK) >> I40E_NVM_TRANS_SHIFT);
 }
 
+static inline u8 i40e_nvmupd_get_preservation_flags(u32 val)
+{
+	return (u8)((val & I40E_NVM_PRESERVATION_FLAGS_MASK) >>
+		    I40E_NVM_PRESERVATION_FLAGS_SHIFT);
+}
+
 static const char * const i40e_nvm_update_state_str[] = {
 	"I40E_NVMUPD_INVALID",
 	"I40E_NVMUPD_READ_CON",
@@ -703,6 +714,7 @@ static const char * const i40e_nvm_update_state_str[] = {
 	"I40E_NVMUPD_STATUS",
 	"I40E_NVMUPD_EXEC_AQ",
 	"I40E_NVMUPD_GET_AQ_RESULT",
+	"I40E_NVMUPD_GET_AQ_EVENT",
 };
 
 /**
@@ -798,9 +810,9 @@ i40e_status i40e_nvmupd_command(struct i40e_hw *hw,
 		 * the wait info and return before doing anything else
 		 */
 		if (cmd->offset == 0xffff) {
-			i40e_nvmupd_check_wait_event(hw, hw->nvm_wait_opcode);
+			i40e_nvmupd_clear_wait_state(hw);
 			status = 0;
-			goto exit;
+			break;
 		}
 
 		status = I40E_ERR_NOT_READY;
@@ -815,7 +827,7 @@ i40e_status i40e_nvmupd_command(struct i40e_hw *hw,
 		*perrno = -ESRCH;
 		break;
 	}
-exit:
+
 	mutex_unlock(&hw->aq.arq_mutex);
 	return status;
 }
@@ -944,6 +956,10 @@ static i40e_status i40e_nvmupd_state_init(struct i40e_hw *hw,
 		status = i40e_nvmupd_get_aq_result(hw, cmd, bytes, perrno);
 		break;
 
+	case I40E_NVMUPD_GET_AQ_EVENT:
+		status = i40e_nvmupd_get_aq_event(hw, cmd, bytes, perrno);
+		break;
+
 	default:
 		i40e_debug(hw, I40E_DEBUG_NVM,
 			   "NVMUPD: bad cmd %s in init state\n",
@@ -1118,38 +1134,53 @@ static i40e_status i40e_nvmupd_state_writing(struct i40e_hw *hw,
 }
 
 /**
- * i40e_nvmupd_check_wait_event - handle NVM update operation events
+ * i40e_nvmupd_clear_wait_state - clear wait state on hw
  * @hw: pointer to the hardware structure
- * @opcode: the event that just happened
  **/
-void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode)
+void i40e_nvmupd_clear_wait_state(struct i40e_hw *hw)
 {
-	if (opcode == hw->nvm_wait_opcode) {
-		i40e_debug(hw, I40E_DEBUG_NVM,
-			   "NVMUPD: clearing wait on opcode 0x%04x\n", opcode);
-		if (hw->nvm_release_on_done) {
-			i40e_release_nvm(hw);
-			hw->nvm_release_on_done = false;
-		}
-		hw->nvm_wait_opcode = 0;
+	i40e_debug(hw, I40E_DEBUG_NVM,
+		   "NVMUPD: clearing wait on opcode 0x%04x\n",
+		   hw->nvm_wait_opcode);
 
-		if (hw->aq.arq_last_status) {
-			hw->nvmupd_state = I40E_NVMUPD_STATE_ERROR;
-			return;
-		}
+	if (hw->nvm_release_on_done) {
+		i40e_release_nvm(hw);
+		hw->nvm_release_on_done = false;
+	}
+	hw->nvm_wait_opcode = 0;
 
-		switch (hw->nvmupd_state) {
-		case I40E_NVMUPD_STATE_INIT_WAIT:
-			hw->nvmupd_state = I40E_NVMUPD_STATE_INIT;
-			break;
+	if (hw->aq.arq_last_status) {
+		hw->nvmupd_state = I40E_NVMUPD_STATE_ERROR;
+		return;
+	}
 
-		case I40E_NVMUPD_STATE_WRITE_WAIT:
-			hw->nvmupd_state = I40E_NVMUPD_STATE_WRITING;
-			break;
+	switch (hw->nvmupd_state) {
+	case I40E_NVMUPD_STATE_INIT_WAIT:
+		hw->nvmupd_state = I40E_NVMUPD_STATE_INIT;
+		break;
 
-		default:
-			break;
-		}
+	case I40E_NVMUPD_STATE_WRITE_WAIT:
+		hw->nvmupd_state = I40E_NVMUPD_STATE_WRITING;
+		break;
+
+	default:
+		break;
+	}
+}
+
+/**
+ * i40e_nvmupd_check_wait_event - handle NVM update operation events
+ * @hw: pointer to the hardware structure
+ * @opcode: the event that just happened
+ **/
+void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode,
+				  struct i40e_aq_desc *desc)
+{
+	u32 aq_desc_len = sizeof(struct i40e_aq_desc);
+
+	if (opcode == hw->nvm_wait_opcode) {
+		memcpy(&hw->nvm_aq_event_desc, desc, aq_desc_len);
+		i40e_nvmupd_clear_wait_state(hw);
 	}
 }
 
@@ -1205,6 +1236,9 @@ static enum i40e_nvmupd_cmd i40e_nvmupd_validate_command(struct i40e_hw *hw,
 			else if (module == 0)
 				upd_cmd = I40E_NVMUPD_GET_AQ_RESULT;
 			break;
+		case I40E_NVM_AQE:
+			upd_cmd = I40E_NVMUPD_GET_AQ_EVENT;
+			break;
 		}
 		break;
 
@@ -1267,6 +1301,9 @@ static i40e_status i40e_nvmupd_exec_aq(struct i40e_hw *hw,
 	u32 aq_data_len;
 
 	i40e_debug(hw, I40E_DEBUG_NVM, "NVMUPD: %s\n", __func__);
+	if (cmd->offset == 0xffff)
+		return 0;
+
 	memset(&cmd_details, 0, sizeof(cmd_details));
 	cmd_details.wb_desc = &hw->nvm_wb_desc;
 
@@ -1302,6 +1339,9 @@ static i40e_status i40e_nvmupd_exec_aq(struct i40e_hw *hw,
 		}
 	}
 
+	if (cmd->offset)
+		memset(&hw->nvm_aq_event_desc, 0, aq_desc_len);
+
 	/* and away we go! */
 	status = i40e_asq_send_command(hw, aq_desc, buff,
 				       buff_size, &cmd_details);
@@ -1311,6 +1351,7 @@ static i40e_status i40e_nvmupd_exec_aq(struct i40e_hw *hw,
 			   i40e_stat_str(hw, status),
 			   i40e_aq_str(hw, hw->aq.asq_last_status));
 		*perrno = i40e_aq_rc_to_posix(status, hw->aq.asq_last_status);
+		return status;
 	}
 
 	/* should we wait for a followup event? */
@@ -1391,6 +1432,40 @@ static i40e_status i40e_nvmupd_get_aq_result(struct i40e_hw *hw,
 	return 0;
 }
 
+/**
+ * i40e_nvmupd_get_aq_event - Get the Admin Queue event from previous exec_aq
+ * @hw: pointer to hardware structure
+ * @cmd: pointer to nvm update command buffer
+ * @bytes: pointer to the data buffer
+ * @perrno: pointer to return error code
+ *
+ * cmd structure contains identifiers and data buffer
+ **/
+static i40e_status i40e_nvmupd_get_aq_event(struct i40e_hw *hw,
+					    struct i40e_nvm_access *cmd,
+					    u8 *bytes, int *perrno)
+{
+	u32 aq_total_len;
+	u32 aq_desc_len;
+
+	i40e_debug(hw, I40E_DEBUG_NVM, "NVMUPD: %s\n", __func__);
+
+	aq_desc_len = sizeof(struct i40e_aq_desc);
+	aq_total_len = aq_desc_len + le16_to_cpu(hw->nvm_aq_event_desc.datalen);
+
+	/* check copylength range */
+	if (cmd->data_size > aq_total_len) {
+		i40e_debug(hw, I40E_DEBUG_NVM,
+			   "%s: copy length %d too big, trimming to %d\n",
+			   __func__, cmd->data_size, aq_total_len);
+		cmd->data_size = aq_total_len;
+	}
+
+	memcpy(bytes, &hw->nvm_aq_event_desc, cmd->data_size);
+
+	return 0;
+}
+
 /**
  * i40e_nvmupd_nvm_read - Read NVM
  * @hw: pointer to hardware structure
@@ -1486,18 +1561,20 @@ static i40e_status i40e_nvmupd_nvm_write(struct i40e_hw *hw,
 	i40e_status status = 0;
 	struct i40e_asq_cmd_details cmd_details;
 	u8 module, transaction;
+	u8 preservation_flags;
 	bool last;
 
 	transaction = i40e_nvmupd_get_transaction(cmd->config);
 	module = i40e_nvmupd_get_module(cmd->config);
 	last = (transaction & I40E_NVM_LCB);
+	preservation_flags = i40e_nvmupd_get_preservation_flags(cmd->config);
 
 	memset(&cmd_details, 0, sizeof(cmd_details));
 	cmd_details.wb_desc = &hw->nvm_wb_desc;
 
 	status = i40e_aq_update_nvm(hw, module, cmd->offset,
 				    (u16)cmd->data_size, bytes, last,
-				    &cmd_details);
+				    preservation_flags, &cmd_details);
 	if (status) {
 		i40e_debug(hw, I40E_DEBUG_NVM,
 			   "i40e_nvmupd_nvm_write mod 0x%x off 0x%x len 0x%x\n",
diff --git a/drivers/net/ethernet/intel/i40e/i40e_prototype.h b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
index 3bb6659db822..83798b7841b9 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_prototype.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_prototype.h
@@ -214,7 +214,7 @@ i40e_status i40e_aq_discover_capabilities(struct i40e_hw *hw,
 				struct i40e_asq_cmd_details *cmd_details);
 i40e_status i40e_aq_update_nvm(struct i40e_hw *hw, u8 module_pointer,
 				u32 offset, u16 length, void *data,
-				bool last_command,
+				bool last_command, u8 preservation_flags,
 				struct i40e_asq_cmd_details *cmd_details);
 i40e_status i40e_aq_get_lldp_mib(struct i40e_hw *hw, u8 bridge_type,
 				u8 mib_type, void *buff, u16 buff_size,
@@ -225,6 +225,10 @@ i40e_status i40e_aq_cfg_lldp_mib_change_event(struct i40e_hw *hw,
 				struct i40e_asq_cmd_details *cmd_details);
 i40e_status i40e_aq_stop_lldp(struct i40e_hw *hw, bool shutdown_agent,
 				struct i40e_asq_cmd_details *cmd_details);
+i40e_status i40e_aq_set_dcb_parameters(struct i40e_hw *hw,
+				       bool dcb_enable,
+				       struct i40e_asq_cmd_details
+				       *cmd_details);
 i40e_status i40e_aq_start_lldp(struct i40e_hw *hw,
 				struct i40e_asq_cmd_details *cmd_details);
 i40e_status i40e_aq_get_cee_dcb_config(struct i40e_hw *hw,
@@ -333,7 +337,9 @@ i40e_status i40e_validate_nvm_checksum(struct i40e_hw *hw,
 i40e_status i40e_nvmupd_command(struct i40e_hw *hw,
 				struct i40e_nvm_access *cmd,
 				u8 *bytes, int *);
-void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode);
+void i40e_nvmupd_check_wait_event(struct i40e_hw *hw, u16 opcode,
+				  struct i40e_aq_desc *desc);
+void i40e_nvmupd_clear_wait_state(struct i40e_hw *hw);
 void i40e_set_pci_config_data(struct i40e_hw *hw, u16 link_status);
 
 extern struct i40e_rx_ptype_decoded i40e_ptype_lookup[];
@@ -343,6 +349,37 @@ static inline struct i40e_rx_ptype_decoded decode_rx_desc_ptype(u8 ptype)
 	return i40e_ptype_lookup[ptype];
 }
 
+/**
+ * i40e_virtchnl_link_speed - Convert AdminQ link_speed to virtchnl definition
+ * @link_speed: the speed to convert
+ *
+ * Returns the link_speed in terms of the virtchnl interface, for use in
+ * converting link_speed as reported by the AdminQ into the format used for
+ * talking to virtchnl devices. If we can't represent the link speed properly,
+ * report LINK_SPEED_UNKNOWN.
+ **/
+static inline enum virtchnl_link_speed
+i40e_virtchnl_link_speed(enum i40e_aq_link_speed link_speed)
+{
+	switch (link_speed) {
+	case I40E_LINK_SPEED_100MB:
+		return VIRTCHNL_LINK_SPEED_100MB;
+	case I40E_LINK_SPEED_1GB:
+		return VIRTCHNL_LINK_SPEED_1GB;
+	case I40E_LINK_SPEED_10GB:
+		return VIRTCHNL_LINK_SPEED_10GB;
+	case I40E_LINK_SPEED_40GB:
+		return VIRTCHNL_LINK_SPEED_40GB;
+	case I40E_LINK_SPEED_20GB:
+		return VIRTCHNL_LINK_SPEED_20GB;
+	case I40E_LINK_SPEED_25GB:
+		return VIRTCHNL_LINK_SPEED_25GB;
+	case I40E_LINK_SPEED_UNKNOWN:
+	default:
+		return VIRTCHNL_LINK_SPEED_UNKNOWN;
+	}
+}
+
 /* prototype for functions used for SW locks */
 
 /* i40e_common for VF drivers*/
@@ -400,13 +437,15 @@ i40e_status i40e_write_phy_register(struct i40e_hw *hw, u8 page, u16 reg,
 u8 i40e_get_phy_address(struct i40e_hw *hw, u8 dev_num);
 i40e_status i40e_blink_phy_link_led(struct i40e_hw *hw,
 				    u32 time, u32 interval);
-i40e_status i40e_aq_write_ppp(struct i40e_hw *hw, void *buff,
+i40e_status i40e_aq_write_ddp(struct i40e_hw *hw, void *buff,
 			      u16 buff_size, u32 track_id,
 			      u32 *error_offset, u32 *error_info,
-			      struct i40e_asq_cmd_details *cmd_details);
-i40e_status i40e_aq_get_ppp_list(struct i40e_hw *hw, void *buff,
+			      struct i40e_asq_cmd_details *
+			      cmd_details);
+i40e_status i40e_aq_get_ddp_list(struct i40e_hw *hw, void *buff,
 				 u16 buff_size, u8 flags,
-				 struct i40e_asq_cmd_details *cmd_details);
+				 struct i40e_asq_cmd_details *
+				 cmd_details);
 struct i40e_generic_seg_header *
 i40e_find_segment_in_package(u32 segment_type,
 			     struct i40e_package_header *pkg_header);
diff --git a/drivers/net/ethernet/intel/i40e/i40e_status.h b/drivers/net/ethernet/intel/i40e/i40e_status.h
index 5f9cac55aa55..afb72e711d43 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_status.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_status.h
@@ -95,6 +95,7 @@ enum i40e_status_code {
 	I40E_ERR_NOT_READY			= -63,
 	I40E_NOT_SUPPORTED			= -64,
 	I40E_ERR_FIRMWARE_API_VERSION		= -65,
+	I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR	= -66,
 };
 
 #endif /* _I40E_STATUS_H_ */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
index 5bc2748ac468..e554aa6cf070 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c
@@ -27,6 +27,7 @@
 #include <linux/prefetch.h>
 #include <net/busy_poll.h>
 #include <linux/bpf_trace.h>
+#include <net/xdp.h>
 #include "i40e.h"
 #include "i40e_trace.h"
 #include "i40e_prototype.h"
@@ -725,6 +726,59 @@ u32 i40e_get_tx_pending(struct i40e_ring *ring)
 	return 0;
 }
 
+/**
+ * i40e_detect_recover_hung - Function to detect and recover hung_queues
+ * @vsi:  pointer to vsi struct with tx queues
+ *
+ * VSI has netdev and netdev has TX queues. This function is to check each of
+ * those TX queues if they are hung, trigger recovery by issuing SW interrupt.
+ **/
+void i40e_detect_recover_hung(struct i40e_vsi *vsi)
+{
+	struct i40e_ring *tx_ring = NULL;
+	struct net_device *netdev;
+	unsigned int i;
+	int packets;
+
+	if (!vsi)
+		return;
+
+	if (test_bit(__I40E_VSI_DOWN, vsi->state))
+		return;
+
+	netdev = vsi->netdev;
+	if (!netdev)
+		return;
+
+	if (!netif_carrier_ok(netdev))
+		return;
+
+	for (i = 0; i < vsi->num_queue_pairs; i++) {
+		tx_ring = vsi->tx_rings[i];
+		if (tx_ring && tx_ring->desc) {
+			/* If packet counter has not changed the queue is
+			 * likely stalled, so force an interrupt for this
+			 * queue.
+			 *
+			 * prev_pkt_ctr would be negative if there was no
+			 * pending work.
+			 */
+			packets = tx_ring->stats.packets & INT_MAX;
+			if (tx_ring->tx_stats.prev_pkt_ctr == packets) {
+				i40e_force_wb(vsi, tx_ring->q_vector);
+				continue;
+			}
+
+			/* Memory barrier between read of packet count and call
+			 * to i40e_get_tx_pending()
+			 */
+			smp_rmb();
+			tx_ring->tx_stats.prev_pkt_ctr =
+			    i40e_get_tx_pending(tx_ring) ? packets : -1;
+		}
+	}
+}
+
 #define WB_STRIDE 4
 
 /**
@@ -902,7 +956,7 @@ static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
 		      I40E_PFINT_DYN_CTLN_ITR_INDX_MASK; /* set noitr */
 
 		wr32(&vsi->back->hw,
-		     I40E_PFINT_DYN_CTLN(q_vector->v_idx + vsi->base_vector - 1),
+		     I40E_PFINT_DYN_CTLN(q_vector->reg_idx),
 		     val);
 	} else {
 		val = I40E_PFINT_DYN_CTL0_WB_ON_ITR_MASK |
@@ -929,8 +983,7 @@ void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
 			  /* allow 00 to be written to the index */
 
 		wr32(&vsi->back->hw,
-		     I40E_PFINT_DYN_CTLN(q_vector->v_idx +
-					 vsi->base_vector - 1), val);
+		     I40E_PFINT_DYN_CTLN(q_vector->reg_idx), val);
 	} else {
 		u32 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
 			  I40E_PFINT_DYN_CTL0_ITR_INDX_MASK | /* set noitr */
@@ -1162,6 +1215,7 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
 
 	tx_ring->next_to_use = 0;
 	tx_ring->next_to_clean = 0;
+	tx_ring->tx_stats.prev_pkt_ctr = -1;
 	return 0;
 
 err:
@@ -1236,6 +1290,8 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
 void i40e_free_rx_resources(struct i40e_ring *rx_ring)
 {
 	i40e_clean_rx_ring(rx_ring);
+	if (rx_ring->vsi->type == I40E_VSI_MAIN)
+		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 	rx_ring->xdp_prog = NULL;
 	kfree(rx_ring->rx_bi);
 	rx_ring->rx_bi = NULL;
@@ -1256,6 +1312,7 @@ void i40e_free_rx_resources(struct i40e_ring *rx_ring)
 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
 {
 	struct device *dev = rx_ring->dev;
+	int err = -ENOMEM;
 	int bi_size;
 
 	/* warn if we are about to overwrite the pointer */
@@ -1283,13 +1340,21 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
 	rx_ring->next_to_clean = 0;
 	rx_ring->next_to_use = 0;
 
+	/* XDP RX-queue info only needed for RX rings exposed to XDP */
+	if (rx_ring->vsi->type == I40E_VSI_MAIN) {
+		err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
+				       rx_ring->queue_index);
+		if (err < 0)
+			goto err;
+	}
+
 	rx_ring->xdp_prog = rx_ring->vsi->xdp_prog;
 
 	return 0;
 err:
 	kfree(rx_ring->rx_bi);
 	rx_ring->rx_bi = NULL;
-	return -ENOMEM;
+	return err;
 }
 
 /**
@@ -2068,11 +2133,13 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
 	struct sk_buff *skb = rx_ring->skb;
 	u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
 	bool failure = false, xdp_xmit = false;
+	struct xdp_buff xdp;
+
+	xdp.rxq = &rx_ring->xdp_rxq;
 
 	while (likely(total_rx_packets < (unsigned int)budget)) {
 		struct i40e_rx_buffer *rx_buffer;
 		union i40e_rx_desc *rx_desc;
-		struct xdp_buff xdp;
 		unsigned int size;
 		u16 vlan_tag;
 		u8 rx_ptype;
@@ -2243,7 +2310,6 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	struct i40e_hw *hw = &vsi->back->hw;
 	bool rx = false, tx = false;
 	u32 rxval, txval;
-	int vector;
 	int idx = q_vector->v_idx;
 	int rx_itr_setting, tx_itr_setting;
 
@@ -2253,8 +2319,6 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 		return;
 	}
 
-	vector = (q_vector->v_idx + vsi->base_vector);
-
 	/* avoid dynamic calculation if in countdown mode OR if
 	 * all dynamic is disabled
 	 */
@@ -2303,12 +2367,12 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 		 */
 		rxval |= BIT(31);
 		/* don't check _DOWN because interrupt isn't being enabled */
-		wr32(hw, INTREG(vector - 1), rxval);
+		wr32(hw, INTREG(q_vector->reg_idx), rxval);
 	}
 
 enable_int:
 	if (!test_bit(__I40E_VSI_DOWN, vsi->state))
-		wr32(hw, INTREG(vector - 1), txval);
+		wr32(hw, INTREG(q_vector->reg_idx), txval);
 
 	if (q_vector->itr_countdown)
 		q_vector->itr_countdown--;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.h b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
index fbae1182e2ea..701b708628b0 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.h
@@ -27,6 +27,8 @@
 #ifndef _I40E_TXRX_H_
 #define _I40E_TXRX_H_
 
+#include <net/xdp.h>
+
 /* Interrupt Throttling and Rate Limiting Goodies */
 
 #define I40E_MAX_ITR               0x0FF0  /* reg uses 2 usec resolution */
@@ -277,7 +279,7 @@ static inline unsigned int i40e_txd_use_count(unsigned int size)
 }
 
 /* Tx Descriptors needed, worst case */
-#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
+#define DESC_NEEDED (MAX_SKB_FRAGS + 6)
 #define I40E_MIN_DESC_PENDING	4
 
 #define I40E_TX_FLAGS_HW_VLAN		BIT(1)
@@ -331,6 +333,7 @@ struct i40e_tx_queue_stats {
 	u64 tx_done_old;
 	u64 tx_linearize;
 	u64 tx_force_wb;
+	int prev_pkt_ctr;
 };
 
 struct i40e_rx_queue_stats {
@@ -428,6 +431,7 @@ struct i40e_ring {
 					 */
 
 	struct i40e_channel *ch;
+	struct xdp_rxq_info xdp_rxq;
 } ____cacheline_internodealigned_in_smp;
 
 static inline bool ring_uses_build_skb(struct i40e_ring *ring)
@@ -498,6 +502,7 @@ void i40e_free_rx_resources(struct i40e_ring *rx_ring);
 int i40e_napi_poll(struct napi_struct *napi, int budget);
 void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector);
 u32 i40e_get_tx_pending(struct i40e_ring *ring);
+void i40e_detect_recover_hung(struct i40e_vsi *vsi);
 int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
 bool __i40e_chk_linearize(struct sk_buff *skb);
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h
index 0e8568719b4e..cd294e6a8587 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_type.h
@@ -402,6 +402,7 @@ enum i40e_nvmupd_cmd {
 	I40E_NVMUPD_STATUS,
 	I40E_NVMUPD_EXEC_AQ,
 	I40E_NVMUPD_GET_AQ_RESULT,
+	I40E_NVMUPD_GET_AQ_EVENT,
 };
 
 enum i40e_nvmupd_state {
@@ -421,15 +422,21 @@ enum i40e_nvmupd_state {
 
 #define I40E_NVM_MOD_PNT_MASK 0xFF
 
-#define I40E_NVM_TRANS_SHIFT	8
-#define I40E_NVM_TRANS_MASK	(0xf << I40E_NVM_TRANS_SHIFT)
-#define I40E_NVM_CON		0x0
-#define I40E_NVM_SNT		0x1
-#define I40E_NVM_LCB		0x2
-#define I40E_NVM_SA		(I40E_NVM_SNT | I40E_NVM_LCB)
-#define I40E_NVM_ERA		0x4
-#define I40E_NVM_CSUM		0x8
-#define I40E_NVM_EXEC		0xf
+#define I40E_NVM_TRANS_SHIFT			8
+#define I40E_NVM_TRANS_MASK			(0xf << I40E_NVM_TRANS_SHIFT)
+#define I40E_NVM_PRESERVATION_FLAGS_SHIFT	12
+#define I40E_NVM_PRESERVATION_FLAGS_MASK \
+				(0x3 << I40E_NVM_PRESERVATION_FLAGS_SHIFT)
+#define I40E_NVM_PRESERVATION_FLAGS_SELECTED	0x01
+#define I40E_NVM_PRESERVATION_FLAGS_ALL		0x02
+#define I40E_NVM_CON				0x0
+#define I40E_NVM_SNT				0x1
+#define I40E_NVM_LCB				0x2
+#define I40E_NVM_SA				(I40E_NVM_SNT | I40E_NVM_LCB)
+#define I40E_NVM_ERA				0x4
+#define I40E_NVM_CSUM				0x8
+#define I40E_NVM_AQE				0xe
+#define I40E_NVM_EXEC				0xf
 
 #define I40E_NVM_ADAPT_SHIFT	16
 #define I40E_NVM_ADAPT_MASK	(0xffff << I40E_NVM_ADAPT_SHIFT)
@@ -611,6 +618,7 @@ struct i40e_hw {
 	/* state of nvm update process */
 	enum i40e_nvmupd_state nvmupd_state;
 	struct i40e_aq_desc nvm_wb_desc;
+	struct i40e_aq_desc nvm_aq_event_desc;
 	struct i40e_virt_mem nvm_buff;
 	bool nvm_release_on_done;
 	u16 nvm_wait_opcode;
@@ -1502,19 +1510,19 @@ struct i40e_lldp_variables {
 #define I40E_FLEX_57_SHIFT		6
 #define I40E_FLEX_57_MASK		(0x1ULL << I40E_FLEX_57_SHIFT)
 
-/* Version format for PPP */
-struct i40e_ppp_version {
+/* Version format for Dynamic Device Personalization(DDP) */
+struct i40e_ddp_version {
 	u8 major;
 	u8 minor;
 	u8 update;
 	u8 draft;
 };
 
-#define I40E_PPP_NAME_SIZE	32
+#define I40E_DDP_NAME_SIZE	32
 
 /* Package header */
 struct i40e_package_header {
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u32 segment_count;
 	u32 segment_offset[1];
 };
@@ -1526,16 +1534,16 @@ struct i40e_generic_seg_header {
 #define SEGMENT_TYPE_I40E	0x00000011
 #define SEGMENT_TYPE_X722	0x00000012
 	u32 type;
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u32 size;
-	char name[I40E_PPP_NAME_SIZE];
+	char name[I40E_DDP_NAME_SIZE];
 };
 
 struct i40e_metadata_segment {
 	struct i40e_generic_seg_header header;
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u32 track_id;
-	char name[I40E_PPP_NAME_SIZE];
+	char name[I40E_DDP_NAME_SIZE];
 };
 
 struct i40e_device_id_entry {
@@ -1545,8 +1553,8 @@ struct i40e_device_id_entry {
 
 struct i40e_profile_segment {
 	struct i40e_generic_seg_header header;
-	struct i40e_ppp_version version;
-	char name[I40E_PPP_NAME_SIZE];
+	struct i40e_ddp_version version;
+	char name[I40E_DDP_NAME_SIZE];
 	u32 device_table_count;
 	struct i40e_device_id_entry device_table[1];
 };
@@ -1573,11 +1581,11 @@ struct i40e_profile_section_header {
 
 struct i40e_profile_info {
 	u32 track_id;
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u8 op;
-#define I40E_PPP_ADD_TRACKID		0x01
-#define I40E_PPP_REMOVE_TRACKID	0x02
+#define I40E_DDP_ADD_TRACKID		0x01
+#define I40E_DDP_REMOVE_TRACKID	0x02
 	u8 reserved[7];
-	u8 name[I40E_PPP_NAME_SIZE];
+	u8 name[I40E_DDP_NAME_SIZE];
 };
 #endif /* _I40E_TYPE_H_ */
diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
index 36cb8e068e85..e9309fb9084b 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c
@@ -81,12 +81,12 @@ static void i40e_vc_notify_vf_link_state(struct i40e_vf *vf)
 	if (vf->link_forced) {
 		pfe.event_data.link_event.link_status = vf->link_up;
 		pfe.event_data.link_event.link_speed =
-			(vf->link_up ? I40E_LINK_SPEED_40GB : 0);
+			(vf->link_up ? VIRTCHNL_LINK_SPEED_40GB : 0);
 	} else {
 		pfe.event_data.link_event.link_status =
 			ls->link_info & I40E_AQ_LINK_UP;
 		pfe.event_data.link_event.link_speed =
-			(enum virtchnl_link_speed)ls->link_speed;
+			i40e_virtchnl_link_speed(ls->link_speed);
 	}
 	i40e_aq_send_msg_to_vf(hw, abs_vf_id, VIRTCHNL_OP_EVENT,
 			       0, (u8 *)&pfe, sizeof(pfe), NULL);
@@ -2749,6 +2749,7 @@ int i40e_vc_process_vf_msg(struct i40e_pf *pf, s16 vf_id, u32 v_opcode,
 		break;
 	case VIRTCHNL_OP_GET_VF_RESOURCES:
 		ret = i40e_vc_get_vf_resources_msg(vf, msg);
+		i40e_vc_notify_vf_link_state(vf);
 		break;
 	case VIRTCHNL_OP_RESET_VF:
 		i40e_vc_reset_vf_msg(vf);
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_adminq.c b/drivers/net/ethernet/intel/i40evf/i40e_adminq.c
index 8b0d4b255dea..d1aab6b8bfb1 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_adminq.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_adminq.c
@@ -837,10 +837,15 @@ i40e_status i40evf_asq_send_command(struct i40e_hw *hw,
 	/* update the error if time out occurred */
 	if ((!cmd_completed) &&
 	    (!details->async && !details->postpone)) {
-		i40e_debug(hw,
-			   I40E_DEBUG_AQ_MESSAGE,
-			   "AQTX: Writeback timeout.\n");
-		status = I40E_ERR_ADMIN_QUEUE_TIMEOUT;
+		if (rd32(hw, hw->aq.asq.len) & I40E_VF_ATQLEN1_ATQCRIT_MASK) {
+			i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
+				   "AQTX: AQ Critical error.\n");
+			status = I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR;
+		} else {
+			i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
+				   "AQTX: Writeback timeout.\n");
+			status = I40E_ERR_ADMIN_QUEUE_TIMEOUT;
+		}
 	}
 
 asq_send_command_error:
@@ -901,7 +906,7 @@ i40e_status i40evf_clean_arq_element(struct i40e_hw *hw,
 	}
 
 	/* set next_to_use to head */
-	ntu = (rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK);
+	ntu = rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK;
 	if (ntu == ntc) {
 		/* nothing to do - shouldn't need to update ring's values */
 		ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK;
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h b/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
index 06b04572c518..815de8d9c3fb 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_adminq_cmd.h
@@ -198,13 +198,14 @@ enum i40e_admin_queue_opc {
 	i40e_aqc_opc_add_mirror_rule	= 0x0260,
 	i40e_aqc_opc_delete_mirror_rule	= 0x0261,
 
-	/* Pipeline Personalization Profile */
+	/* Dynamic Device Personalization */
 	i40e_aqc_opc_write_personalization_profile	= 0x0270,
 	i40e_aqc_opc_get_personalization_profile_list	= 0x0271,
 
 	/* DCB commands */
 	i40e_aqc_opc_dcb_ignore_pfc	= 0x0301,
 	i40e_aqc_opc_dcb_updated	= 0x0302,
+	i40e_aqc_opc_set_dcb_parameters = 0x0303,
 
 	/* TX scheduler */
 	i40e_aqc_opc_configure_vsi_bw_limit		= 0x0400,
@@ -1562,7 +1563,7 @@ struct i40e_aqc_add_delete_mirror_rule_completion {
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_add_delete_mirror_rule_completion);
 
-/* Pipeline Personalization Profile */
+/* Dynamic Device Personalization */
 struct i40e_aqc_write_personalization_profile {
 	u8      flags;
 	u8      reserved[3];
@@ -1573,7 +1574,7 @@ struct i40e_aqc_write_personalization_profile {
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_write_personalization_profile);
 
-struct i40e_aqc_write_ppp_resp {
+struct i40e_aqc_write_ddp_resp {
 	__le32 error_offset;
 	__le32 error_info;
 	__le32 addr_high;
@@ -1582,8 +1583,8 @@ struct i40e_aqc_write_ppp_resp {
 
 struct i40e_aqc_get_applied_profiles {
 	u8      flags;
-#define I40E_AQC_GET_PPP_GET_CONF	0x1
-#define I40E_AQC_GET_PPP_GET_RDPU_CONF	0x2
+#define I40E_AQC_GET_DDP_GET_CONF	0x1
+#define I40E_AQC_GET_DDP_GET_RDPU_CONF	0x2
 	u8      rsv[3];
 	__le32  reserved;
 	__le32  addr_high;
@@ -2196,8 +2197,12 @@ I40E_CHECK_CMD_LENGTH(i40e_aqc_phy_register_access);
  */
 struct i40e_aqc_nvm_update {
 	u8	command_flags;
-#define I40E_AQ_NVM_LAST_CMD	0x01
-#define I40E_AQ_NVM_FLASH_ONLY	0x80
+#define I40E_AQ_NVM_LAST_CMD			0x01
+#define I40E_AQ_NVM_FLASH_ONLY			0x80
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_SHIFT	1
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_MASK	0x03
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_SELECTED	0x03
+#define I40E_AQ_NVM_PRESERVATION_FLAGS_ALL	0x01
 	u8	module_pointer;
 	__le16	length;
 	__le32	offset;
@@ -2457,6 +2462,17 @@ struct i40e_aqc_lldp_start {
 
 I40E_CHECK_CMD_LENGTH(i40e_aqc_lldp_start);
 
+/* Set DCB (direct 0x0303) */
+struct i40e_aqc_set_dcb_parameters {
+	u8 command;
+#define I40E_AQ_DCB_SET_AGENT	0x1
+#define I40E_DCB_VALID		0x1
+	u8 valid_flags;
+	u8 reserved[14];
+};
+
+I40E_CHECK_CMD_LENGTH(i40e_aqc_set_dcb_parameters);
+
 /* Apply MIB changes (0x0A07)
  * uses the generic struc as it contains no data
  */
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_common.c b/drivers/net/ethernet/intel/i40evf/i40e_common.c
index 7d70bf69b249..67bf5cebb76f 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_common.c
@@ -284,6 +284,8 @@ const char *i40evf_stat_str(struct i40e_hw *hw, i40e_status stat_err)
 		return "I40E_NOT_SUPPORTED";
 	case I40E_ERR_FIRMWARE_API_VERSION:
 		return "I40E_ERR_FIRMWARE_API_VERSION";
+	case I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
+		return "I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR";
 	}
 
 	snprintf(hw->err_str, sizeof(hw->err_str), "%d", stat_err);
@@ -1202,7 +1204,7 @@ i40e_status i40e_vf_reset(struct i40e_hw *hw)
 }
 
 /**
- * i40evf_aq_write_ppp - Write pipeline personalization profile (ppp)
+ * i40evf_aq_write_ddp - Write dynamic device personalization (ddp)
  * @hw: pointer to the hw struct
  * @buff: command buffer (size in bytes = buff_size)
  * @buff_size: buffer size in bytes
@@ -1212,7 +1214,7 @@ i40e_status i40e_vf_reset(struct i40e_hw *hw)
  * @cmd_details: pointer to command details structure or NULL
  **/
 enum
-i40e_status_code i40evf_aq_write_ppp(struct i40e_hw *hw, void *buff,
+i40e_status_code i40evf_aq_write_ddp(struct i40e_hw *hw, void *buff,
 				     u16 buff_size, u32 track_id,
 				     u32 *error_offset, u32 *error_info,
 				     struct i40e_asq_cmd_details *cmd_details)
@@ -1221,7 +1223,7 @@ i40e_status_code i40evf_aq_write_ppp(struct i40e_hw *hw, void *buff,
 	struct i40e_aqc_write_personalization_profile *cmd =
 		(struct i40e_aqc_write_personalization_profile *)
 		&desc.params.raw;
-	struct i40e_aqc_write_ppp_resp *resp;
+	struct i40e_aqc_write_ddp_resp *resp;
 	i40e_status status;
 
 	i40evf_fill_default_direct_cmd_desc(&desc,
@@ -1237,7 +1239,7 @@ i40e_status_code i40evf_aq_write_ppp(struct i40e_hw *hw, void *buff,
 
 	status = i40evf_asq_send_command(hw, &desc, buff, buff_size, cmd_details);
 	if (!status) {
-		resp = (struct i40e_aqc_write_ppp_resp *)&desc.params.raw;
+		resp = (struct i40e_aqc_write_ddp_resp *)&desc.params.raw;
 		if (error_offset)
 			*error_offset = le32_to_cpu(resp->error_offset);
 		if (error_info)
@@ -1248,16 +1250,16 @@ i40e_status_code i40evf_aq_write_ppp(struct i40e_hw *hw, void *buff,
 }
 
 /**
- * i40evf_aq_get_ppp_list - Read pipeline personalization profile (ppp)
+ * i40evf_aq_get_ddp_list - Read dynamic device personalization (ddp)
  * @hw: pointer to the hw struct
  * @buff: command buffer (size in bytes = buff_size)
  * @buff_size: buffer size in bytes
  * @cmd_details: pointer to command details structure or NULL
  **/
 enum
-i40e_status_code i40evf_aq_get_ppp_list(struct i40e_hw *hw, void *buff,
+i40e_status_code i40evf_aq_get_ddp_list(struct i40e_hw *hw, void *buff,
 					u16 buff_size, u8 flags,
-				      struct i40e_asq_cmd_details *cmd_details)
+				       struct i40e_asq_cmd_details *cmd_details)
 {
 	struct i40e_aq_desc desc;
 	struct i40e_aqc_get_applied_profiles *cmd =
@@ -1330,11 +1332,6 @@ i40evf_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile,
 	u32 offset = 0, info = 0;
 	u32 i;
 
-	if (!track_id) {
-		i40e_debug(hw, I40E_DEBUG_PACKAGE, "Track_id can't be 0.");
-		return I40E_NOT_SUPPORTED;
-	}
-
 	dev_cnt = profile->device_table_count;
 
 	for (i = 0; i < dev_cnt; i++) {
@@ -1344,7 +1341,7 @@ i40evf_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile,
 				break;
 	}
 	if (i == dev_cnt) {
-		i40e_debug(hw, I40E_DEBUG_PACKAGE, "Device doesn't support PPP");
+		i40e_debug(hw, I40E_DEBUG_PACKAGE, "Device doesn't support DDP");
 		return I40E_ERR_DEVICE_NOT_SUPPORTED;
 	}
 
@@ -1363,7 +1360,7 @@ i40evf_write_profile(struct i40e_hw *hw, struct i40e_profile_segment *profile,
 			sizeof(struct i40e_profile_section_header);
 
 		/* Write profile */
-		status = i40evf_aq_write_ppp(hw, (void *)sec, (u16)section_size,
+		status = i40evf_aq_write_ddp(hw, (void *)sec, (u16)section_size,
 					     track_id, &offset, &info, NULL);
 		if (status) {
 			i40e_debug(hw, I40E_DEBUG_PACKAGE,
@@ -1405,10 +1402,10 @@ i40evf_add_pinfo_to_list(struct i40e_hw *hw,
 					     sec->section.offset);
 	pinfo->track_id = track_id;
 	pinfo->version = profile->version;
-	pinfo->op = I40E_PPP_ADD_TRACKID;
-	memcpy(pinfo->name, profile->name, I40E_PPP_NAME_SIZE);
+	pinfo->op = I40E_DDP_ADD_TRACKID;
+	memcpy(pinfo->name, profile->name, I40E_DDP_NAME_SIZE);
 
-	status = i40evf_aq_write_ppp(hw, (void *)sec, sec->data_end,
+	status = i40evf_aq_write_ddp(hw, (void *)sec, sec->data_end,
 				     track_id, &offset, &info, NULL);
 	return status;
 }
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_prototype.h b/drivers/net/ethernet/intel/i40evf/i40e_prototype.h
index b624b5994075..47c429931a57 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_prototype.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_prototype.h
@@ -131,13 +131,15 @@ i40e_status i40e_write_phy_register(struct i40e_hw *hw, u8 page, u16 reg,
 u8 i40e_get_phy_address(struct i40e_hw *hw, u8 dev_num);
 i40e_status i40e_blink_phy_link_led(struct i40e_hw *hw,
 				    u32 time, u32 interval);
-i40e_status i40evf_aq_write_ppp(struct i40e_hw *hw, void *buff,
+i40e_status i40evf_aq_write_ddp(struct i40e_hw *hw, void *buff,
 				u16 buff_size, u32 track_id,
 				u32 *error_offset, u32 *error_info,
-				struct i40e_asq_cmd_details *cmd_details);
-i40e_status i40evf_aq_get_ppp_list(struct i40e_hw *hw, void *buff,
+				struct i40e_asq_cmd_details *
+				cmd_details);
+i40e_status i40evf_aq_get_ddp_list(struct i40e_hw *hw, void *buff,
 				   u16 buff_size, u8 flags,
-				   struct i40e_asq_cmd_details *cmd_details);
+				   struct i40e_asq_cmd_details *
+				   cmd_details);
 struct i40e_generic_seg_header *
 i40evf_find_segment_in_package(u32 segment_type,
 			       struct i40e_package_header *pkg_header);
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_status.h b/drivers/net/ethernet/intel/i40evf/i40e_status.h
index 7fa7a41915c1..5b222246e08b 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_status.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_status.h
@@ -95,6 +95,7 @@ enum i40e_status_code {
 	I40E_ERR_NOT_READY			= -63,
 	I40E_NOT_SUPPORTED			= -64,
 	I40E_ERR_FIRMWARE_API_VERSION		= -65,
+	I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR	= -66,
 };
 
 #endif /* _I40E_STATUS_H_ */
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
index 1ba29bb85b67..357d6051281f 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.c
@@ -148,6 +148,59 @@ u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw)
 	return 0;
 }
 
+/**
+ * i40evf_detect_recover_hung - Function to detect and recover hung_queues
+ * @vsi:  pointer to vsi struct with tx queues
+ *
+ * VSI has netdev and netdev has TX queues. This function is to check each of
+ * those TX queues if they are hung, trigger recovery by issuing SW interrupt.
+ **/
+void i40evf_detect_recover_hung(struct i40e_vsi *vsi)
+{
+	struct i40e_ring *tx_ring = NULL;
+	struct net_device *netdev;
+	unsigned int i;
+	int packets;
+
+	if (!vsi)
+		return;
+
+	if (test_bit(__I40E_VSI_DOWN, vsi->state))
+		return;
+
+	netdev = vsi->netdev;
+	if (!netdev)
+		return;
+
+	if (!netif_carrier_ok(netdev))
+		return;
+
+	for (i = 0; i < vsi->back->num_active_queues; i++) {
+		tx_ring = &vsi->back->tx_rings[i];
+		if (tx_ring && tx_ring->desc) {
+			/* If packet counter has not changed the queue is
+			 * likely stalled, so force an interrupt for this
+			 * queue.
+			 *
+			 * prev_pkt_ctr would be negative if there was no
+			 * pending work.
+			 */
+			packets = tx_ring->stats.packets & INT_MAX;
+			if (tx_ring->tx_stats.prev_pkt_ctr == packets) {
+				i40evf_force_wb(vsi, tx_ring->q_vector);
+				continue;
+			}
+
+			/* Memory barrier between read of packet count and call
+			 * to i40evf_get_tx_pending()
+			 */
+			smp_rmb();
+			tx_ring->tx_stats.prev_pkt_ctr =
+			  i40evf_get_tx_pending(tx_ring, false) ? packets : -1;
+		}
+	}
+}
+
 #define WB_STRIDE 4
 
 /**
@@ -316,8 +369,7 @@ static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
 	      I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK; /* set noitr */
 
 	wr32(&vsi->back->hw,
-	     I40E_VFINT_DYN_CTLN1(q_vector->v_idx +
-				  vsi->base_vector - 1), val);
+	     I40E_VFINT_DYN_CTLN1(q_vector->reg_idx), val);
 	q_vector->arm_wb_state = true;
 }
 
@@ -336,7 +388,7 @@ void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
 		  /* allow 00 to be written to the index */;
 
 	wr32(&vsi->back->hw,
-	     I40E_VFINT_DYN_CTLN1(q_vector->v_idx + vsi->base_vector - 1),
+	     I40E_VFINT_DYN_CTLN1(q_vector->reg_idx),
 	     val);
 }
 
@@ -469,6 +521,7 @@ int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring)
 
 	tx_ring->next_to_use = 0;
 	tx_ring->next_to_clean = 0;
+	tx_ring->tx_stats.prev_pkt_ctr = -1;
 	return 0;
 
 err:
@@ -1444,12 +1497,9 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 	struct i40e_hw *hw = &vsi->back->hw;
 	bool rx = false, tx = false;
 	u32 rxval, txval;
-	int vector;
 	int idx = q_vector->v_idx;
 	int rx_itr_setting, tx_itr_setting;
 
-	vector = (q_vector->v_idx + vsi->base_vector);
-
 	/* avoid dynamic calculation if in countdown mode OR if
 	 * all dynamic is disabled
 	 */
@@ -1498,12 +1548,12 @@ static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
 		 */
 		rxval |= BIT(31);
 		/* don't check _DOWN because interrupt isn't being enabled */
-		wr32(hw, INTREG(vector - 1), rxval);
+		wr32(hw, INTREG(q_vector->reg_idx), rxval);
 	}
 
 enable_int:
 	if (!test_bit(__I40E_VSI_DOWN, vsi->state))
-		wr32(hw, INTREG(vector - 1), txval);
+		wr32(hw, INTREG(q_vector->reg_idx), txval);
 
 	if (q_vector->itr_countdown)
 		q_vector->itr_countdown--;
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
index 8d26c85d12e1..7798a6645c3f 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_txrx.h
@@ -260,7 +260,7 @@ static inline unsigned int i40e_txd_use_count(unsigned int size)
 }
 
 /* Tx Descriptors needed, worst case */
-#define DESC_NEEDED (MAX_SKB_FRAGS + 4)
+#define DESC_NEEDED (MAX_SKB_FRAGS + 6)
 #define I40E_MIN_DESC_PENDING	4
 
 #define I40E_TX_FLAGS_HW_VLAN		BIT(1)
@@ -313,6 +313,7 @@ struct i40e_tx_queue_stats {
 	u64 tx_done_old;
 	u64 tx_linearize;
 	u64 tx_force_wb;
+	int prev_pkt_ctr;
 	u64 tx_lost_interrupt;
 };
 
@@ -467,6 +468,7 @@ void i40evf_free_rx_resources(struct i40e_ring *rx_ring);
 int i40evf_napi_poll(struct napi_struct *napi, int budget);
 void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector);
 u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw);
+void i40evf_detect_recover_hung(struct i40e_vsi *vsi);
 int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size);
 bool __i40evf_chk_linearize(struct sk_buff *skb);
 
diff --git a/drivers/net/ethernet/intel/i40evf/i40e_type.h b/drivers/net/ethernet/intel/i40evf/i40e_type.h
index 213b773dfad6..54951c84a481 100644
--- a/drivers/net/ethernet/intel/i40evf/i40e_type.h
+++ b/drivers/net/ethernet/intel/i40evf/i40e_type.h
@@ -361,6 +361,7 @@ enum i40e_nvmupd_cmd {
 	I40E_NVMUPD_STATUS,
 	I40E_NVMUPD_EXEC_AQ,
 	I40E_NVMUPD_GET_AQ_RESULT,
+	I40E_NVMUPD_GET_AQ_EVENT,
 };
 
 enum i40e_nvmupd_state {
@@ -380,15 +381,21 @@ enum i40e_nvmupd_state {
 
 #define I40E_NVM_MOD_PNT_MASK 0xFF
 
-#define I40E_NVM_TRANS_SHIFT	8
-#define I40E_NVM_TRANS_MASK	(0xf << I40E_NVM_TRANS_SHIFT)
-#define I40E_NVM_CON		0x0
-#define I40E_NVM_SNT		0x1
-#define I40E_NVM_LCB		0x2
-#define I40E_NVM_SA		(I40E_NVM_SNT | I40E_NVM_LCB)
-#define I40E_NVM_ERA		0x4
-#define I40E_NVM_CSUM		0x8
-#define I40E_NVM_EXEC		0xf
+#define I40E_NVM_TRANS_SHIFT			8
+#define I40E_NVM_TRANS_MASK			(0xf << I40E_NVM_TRANS_SHIFT)
+#define I40E_NVM_PRESERVATION_FLAGS_SHIFT	12
+#define I40E_NVM_PRESERVATION_FLAGS_MASK \
+				(0x3 << I40E_NVM_PRESERVATION_FLAGS_SHIFT)
+#define I40E_NVM_PRESERVATION_FLAGS_SELECTED	0x01
+#define I40E_NVM_PRESERVATION_FLAGS_ALL		0x02
+#define I40E_NVM_CON				0x0
+#define I40E_NVM_SNT				0x1
+#define I40E_NVM_LCB				0x2
+#define I40E_NVM_SA				(I40E_NVM_SNT | I40E_NVM_LCB)
+#define I40E_NVM_ERA				0x4
+#define I40E_NVM_CSUM				0x8
+#define I40E_NVM_AQE				0xe
+#define I40E_NVM_EXEC				0xf
 
 #define I40E_NVM_ADAPT_SHIFT	16
 #define I40E_NVM_ADAPT_MASK	(0xffff << I40E_NVM_ADAPT_SHIFT)
@@ -561,6 +568,7 @@ struct i40e_hw {
 	/* state of nvm update process */
 	enum i40e_nvmupd_state nvmupd_state;
 	struct i40e_aq_desc nvm_wb_desc;
+	struct i40e_aq_desc nvm_aq_event_desc;
 	struct i40e_virt_mem nvm_buff;
 	bool nvm_release_on_done;
 	u16 nvm_wait_opcode;
@@ -1422,19 +1430,19 @@ enum i40e_reset_type {
 #define I40E_FD_INSET_FLEX_WORD57_MASK		(0x1ULL << \
 					I40E_FD_INSET_FLEX_WORD57_SHIFT)
 
-/* Version format for PPP */
-struct i40e_ppp_version {
+/* Version format for Dynamic Device Personalization(DDP) */
+struct i40e_ddp_version {
 	u8 major;
 	u8 minor;
 	u8 update;
 	u8 draft;
 };
 
-#define I40E_PPP_NAME_SIZE	32
+#define I40E_DDP_NAME_SIZE	32
 
 /* Package header */
 struct i40e_package_header {
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u32 segment_count;
 	u32 segment_offset[1];
 };
@@ -1446,16 +1454,16 @@ struct i40e_generic_seg_header {
 #define SEGMENT_TYPE_I40E	0x00000011
 #define SEGMENT_TYPE_X722	0x00000012
 	u32 type;
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u32 size;
-	char name[I40E_PPP_NAME_SIZE];
+	char name[I40E_DDP_NAME_SIZE];
 };
 
 struct i40e_metadata_segment {
 	struct i40e_generic_seg_header header;
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u32 track_id;
-	char name[I40E_PPP_NAME_SIZE];
+	char name[I40E_DDP_NAME_SIZE];
 };
 
 struct i40e_device_id_entry {
@@ -1465,8 +1473,8 @@ struct i40e_device_id_entry {
 
 struct i40e_profile_segment {
 	struct i40e_generic_seg_header header;
-	struct i40e_ppp_version version;
-	char name[I40E_PPP_NAME_SIZE];
+	struct i40e_ddp_version version;
+	char name[I40E_DDP_NAME_SIZE];
 	u32 device_table_count;
 	struct i40e_device_id_entry device_table[1];
 };
@@ -1493,11 +1501,11 @@ struct i40e_profile_section_header {
 
 struct i40e_profile_info {
 	u32 track_id;
-	struct i40e_ppp_version version;
+	struct i40e_ddp_version version;
 	u8 op;
-#define I40E_PPP_ADD_TRACKID		0x01
-#define I40E_PPP_REMOVE_TRACKID	0x02
+#define I40E_DDP_ADD_TRACKID		0x01
+#define I40E_DDP_REMOVE_TRACKID	0x02
 	u8 reserved[7];
-	u8 name[I40E_PPP_NAME_SIZE];
+	u8 name[I40E_DDP_NAME_SIZE];
 };
 #endif /* _I40E_TYPE_H_ */
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf.h b/drivers/net/ethernet/intel/i40evf/i40evf.h
index de0af521d602..9690c1ea019e 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf.h
+++ b/drivers/net/ethernet/intel/i40evf/i40evf.h
@@ -114,14 +114,14 @@ struct i40e_q_vector {
 	struct i40evf_adapter *adapter;
 	struct i40e_vsi *vsi;
 	struct napi_struct napi;
-	unsigned long reg_idx;
 	struct i40e_ring_container rx;
 	struct i40e_ring_container tx;
 	u32 ring_mask;
 	u8 num_ringpairs;	/* total number of ring pairs in vector */
 #define ITR_COUNTDOWN_START 100
 	u8 itr_countdown;	/* when 0 or 1 update ITR */
-	int v_idx;	/* vector index in list */
+	u16 v_idx;		/* index in the vsi->q_vector array. */
+	u16 reg_idx;		/* register index of the interrupt */
 	char name[IFNAMSIZ + 15];
 	bool arm_wb_state;
 	cpumask_t affinity_mask;
@@ -187,6 +187,7 @@ enum i40evf_state_t {
 enum i40evf_critical_section_t {
 	__I40EVF_IN_CRITICAL_TASK,	/* cannot be interrupted */
 	__I40EVF_IN_CLIENT_TASK,
+	__I40EVF_IN_REMOVE_TASK,	/* device being removed */
 };
 
 /* board specific private data structure */
@@ -199,6 +200,9 @@ struct i40evf_adapter {
 	wait_queue_head_t down_waitqueue;
 	struct i40e_q_vector *q_vectors;
 	struct list_head vlan_filter_list;
+	struct list_head mac_filter_list;
+	/* Lock to protect accesses to MAC and VLAN lists */
+	spinlock_t mac_vlan_list_lock;
 	char misc_vector_name[IFNAMSIZ + 9];
 	int num_active_queues;
 	int num_req_queues;
@@ -206,7 +210,6 @@ struct i40evf_adapter {
 	/* TX */
 	struct i40e_ring *tx_rings;
 	u32 tx_timeout_count;
-	struct list_head mac_filter_list;
 	u32 tx_desc_count;
 
 	/* RX */
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
index da006fa3fec1..e2d8aa19d205 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_ethtool.c
@@ -512,31 +512,31 @@ static void i40evf_set_itr_per_queue(struct i40evf_adapter *adapter,
 				     struct ethtool_coalesce *ec,
 				     int queue)
 {
+	struct i40e_ring *rx_ring = &adapter->rx_rings[queue];
+	struct i40e_ring *tx_ring = &adapter->tx_rings[queue];
 	struct i40e_vsi *vsi = &adapter->vsi;
 	struct i40e_hw *hw = &adapter->hw;
 	struct i40e_q_vector *q_vector;
 	u16 vector;
 
-	adapter->rx_rings[queue].rx_itr_setting = ec->rx_coalesce_usecs;
-	adapter->tx_rings[queue].tx_itr_setting = ec->tx_coalesce_usecs;
+	rx_ring->rx_itr_setting = ec->rx_coalesce_usecs;
+	tx_ring->tx_itr_setting = ec->tx_coalesce_usecs;
 
-	if (ec->use_adaptive_rx_coalesce)
-		adapter->rx_rings[queue].rx_itr_setting |= I40E_ITR_DYNAMIC;
-	else
-		adapter->rx_rings[queue].rx_itr_setting &= ~I40E_ITR_DYNAMIC;
+	rx_ring->rx_itr_setting |= I40E_ITR_DYNAMIC;
+	if (!ec->use_adaptive_rx_coalesce)
+		rx_ring->rx_itr_setting ^= I40E_ITR_DYNAMIC;
 
-	if (ec->use_adaptive_tx_coalesce)
-		adapter->tx_rings[queue].tx_itr_setting |= I40E_ITR_DYNAMIC;
-	else
-		adapter->tx_rings[queue].tx_itr_setting &= ~I40E_ITR_DYNAMIC;
+	tx_ring->tx_itr_setting |= I40E_ITR_DYNAMIC;
+	if (!ec->use_adaptive_tx_coalesce)
+		tx_ring->tx_itr_setting ^= I40E_ITR_DYNAMIC;
 
-	q_vector = adapter->rx_rings[queue].q_vector;
-	q_vector->rx.itr = ITR_TO_REG(adapter->rx_rings[queue].rx_itr_setting);
+	q_vector = rx_ring->q_vector;
+	q_vector->rx.itr = ITR_TO_REG(rx_ring->rx_itr_setting);
 	vector = vsi->base_vector + q_vector->v_idx;
 	wr32(hw, I40E_VFINT_ITRN1(I40E_RX_ITR, vector - 1), q_vector->rx.itr);
 
-	q_vector = adapter->tx_rings[queue].q_vector;
-	q_vector->tx.itr = ITR_TO_REG(adapter->tx_rings[queue].tx_itr_setting);
+	q_vector = tx_ring->q_vector;
+	q_vector->tx.itr = ITR_TO_REG(tx_ring->tx_itr_setting);
 	vector = vsi->base_vector + q_vector->v_idx;
 	wr32(hw, I40E_VFINT_ITRN1(I40E_TX_ITR, vector - 1), q_vector->tx.itr);
 
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_main.c b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
index 7b2a4eba92e2..16989ad2ca90 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_main.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_main.c
@@ -45,8 +45,8 @@ static const char i40evf_driver_string[] =
 #define DRV_KERN "-k"
 
 #define DRV_VERSION_MAJOR 3
-#define DRV_VERSION_MINOR 0
-#define DRV_VERSION_BUILD 1
+#define DRV_VERSION_MINOR 2
+#define DRV_VERSION_BUILD 2
 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
 	     __stringify(DRV_VERSION_MINOR) "." \
 	     __stringify(DRV_VERSION_BUILD) \
@@ -276,37 +276,7 @@ void i40evf_irq_enable_queues(struct i40evf_adapter *adapter, u32 mask)
 		if (mask & BIT(i - 1)) {
 			wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1),
 			     I40E_VFINT_DYN_CTLN1_INTENA_MASK |
-			     I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK |
-			     I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK);
-		}
-	}
-}
-
-/**
- * i40evf_fire_sw_int - Generate SW interrupt for specified vectors
- * @adapter: board private structure
- * @mask: bitmap of vectors to trigger
- **/
-static void i40evf_fire_sw_int(struct i40evf_adapter *adapter, u32 mask)
-{
-	struct i40e_hw *hw = &adapter->hw;
-	int i;
-	u32 dyn_ctl;
-
-	if (mask & 1) {
-		dyn_ctl = rd32(hw, I40E_VFINT_DYN_CTL01);
-		dyn_ctl |= I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
-			   I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK |
-			   I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK;
-		wr32(hw, I40E_VFINT_DYN_CTL01, dyn_ctl);
-	}
-	for (i = 1; i < adapter->num_msix_vectors; i++) {
-		if (mask & BIT(i)) {
-			dyn_ctl = rd32(hw, I40E_VFINT_DYN_CTLN1(i - 1));
-			dyn_ctl |= I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
-				   I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK |
-				   I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK;
-			wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1), dyn_ctl);
+			     I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK);
 		}
 	}
 }
@@ -337,15 +307,10 @@ static irqreturn_t i40evf_msix_aq(int irq, void *data)
 	struct net_device *netdev = data;
 	struct i40evf_adapter *adapter = netdev_priv(netdev);
 	struct i40e_hw *hw = &adapter->hw;
-	u32 val;
 
 	/* handle non-queue interrupts, these reads clear the registers */
-	val = rd32(hw, I40E_VFINT_ICR01);
-	val = rd32(hw, I40E_VFINT_ICR0_ENA1);
-
-	val = rd32(hw, I40E_VFINT_DYN_CTL01) |
-	      I40E_VFINT_DYN_CTL01_CLEARPBA_MASK;
-	wr32(hw, I40E_VFINT_DYN_CTL01, val);
+	rd32(hw, I40E_VFINT_ICR01);
+	rd32(hw, I40E_VFINT_ICR0_ENA1);
 
 	/* schedule work on the private workqueue */
 	schedule_work(&adapter->adminq_task);
@@ -706,7 +671,8 @@ static void i40evf_configure_rx(struct i40evf_adapter *adapter)
  * @adapter: board private structure
  * @vlan: vlan tag
  *
- * Returns ptr to the filter object or NULL
+ * Returns ptr to the filter object or NULL. Must be called while holding the
+ * mac_vlan_list_lock.
  **/
 static struct
 i40evf_vlan_filter *i40evf_find_vlan(struct i40evf_adapter *adapter, u16 vlan)
@@ -731,14 +697,8 @@ static struct
 i40evf_vlan_filter *i40evf_add_vlan(struct i40evf_adapter *adapter, u16 vlan)
 {
 	struct i40evf_vlan_filter *f = NULL;
-	int count = 50;
 
-	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
-				&adapter->crit_section)) {
-		udelay(1);
-		if (--count == 0)
-			goto out;
-	}
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
 
 	f = i40evf_find_vlan(adapter, vlan);
 	if (!f) {
@@ -755,8 +715,7 @@ i40evf_vlan_filter *i40evf_add_vlan(struct i40evf_adapter *adapter, u16 vlan)
 	}
 
 clearout:
-	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
-out:
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
 	return f;
 }
 
@@ -768,21 +727,16 @@ i40evf_vlan_filter *i40evf_add_vlan(struct i40evf_adapter *adapter, u16 vlan)
 static void i40evf_del_vlan(struct i40evf_adapter *adapter, u16 vlan)
 {
 	struct i40evf_vlan_filter *f;
-	int count = 50;
 
-	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
-				&adapter->crit_section)) {
-		udelay(1);
-		if (--count == 0)
-			return;
-	}
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
 
 	f = i40evf_find_vlan(adapter, vlan);
 	if (f) {
 		f->remove = true;
 		adapter->aq_required |= I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
 	}
-	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
+
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
 }
 
 /**
@@ -824,7 +778,8 @@ static int i40evf_vlan_rx_kill_vid(struct net_device *netdev,
  * @adapter: board private structure
  * @macaddr: the MAC address
  *
- * Returns ptr to the filter object or NULL
+ * Returns ptr to the filter object or NULL. Must be called while holding the
+ * mac_vlan_list_lock.
  **/
 static struct
 i40evf_mac_filter *i40evf_find_filter(struct i40evf_adapter *adapter,
@@ -854,26 +809,17 @@ i40evf_mac_filter *i40evf_add_filter(struct i40evf_adapter *adapter,
 				     u8 *macaddr)
 {
 	struct i40evf_mac_filter *f;
-	int count = 50;
 
 	if (!macaddr)
 		return NULL;
 
-	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
-				&adapter->crit_section)) {
-		udelay(1);
-		if (--count == 0)
-			return NULL;
-	}
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
 
 	f = i40evf_find_filter(adapter, macaddr);
 	if (!f) {
 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
-		if (!f) {
-			clear_bit(__I40EVF_IN_CRITICAL_TASK,
-				  &adapter->crit_section);
-			return NULL;
-		}
+		if (!f)
+			goto clearout;
 
 		ether_addr_copy(f->macaddr, macaddr);
 
@@ -884,7 +830,8 @@ i40evf_mac_filter *i40evf_add_filter(struct i40evf_adapter *adapter,
 		f->remove = false;
 	}
 
-	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
+clearout:
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
 	return f;
 }
 
@@ -911,12 +858,16 @@ static int i40evf_set_mac(struct net_device *netdev, void *p)
 	if (adapter->flags & I40EVF_FLAG_ADDR_SET_BY_PF)
 		return -EPERM;
 
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	f = i40evf_find_filter(adapter, hw->mac.addr);
 	if (f) {
 		f->remove = true;
 		adapter->aq_required |= I40EVF_FLAG_AQ_DEL_MAC_FILTER;
 	}
 
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	f = i40evf_add_filter(adapter, addr->sa_data);
 	if (f) {
 		ether_addr_copy(hw->mac.addr, addr->sa_data);
@@ -937,7 +888,6 @@ static void i40evf_set_rx_mode(struct net_device *netdev)
 	struct netdev_hw_addr *uca;
 	struct netdev_hw_addr *mca;
 	struct netdev_hw_addr *ha;
-	int count = 50;
 
 	/* add addr if not already in the filter list */
 	netdev_for_each_uc_addr(uca, netdev) {
@@ -947,16 +897,8 @@ static void i40evf_set_rx_mode(struct net_device *netdev)
 		i40evf_add_filter(adapter, mca->addr);
 	}
 
-	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
-				&adapter->crit_section)) {
-		udelay(1);
-		if (--count == 0) {
-			dev_err(&adapter->pdev->dev,
-				"Failed to get lock in %s\n", __func__);
-			return;
-		}
-	}
-	/* remove filter if not in netdev list */
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
 		netdev_for_each_mc_addr(mca, netdev)
 			if (ether_addr_equal(mca->addr, f->macaddr))
@@ -995,7 +937,7 @@ static void i40evf_set_rx_mode(struct net_device *netdev)
 		 adapter->flags & I40EVF_FLAG_ALLMULTI_ON)
 		adapter->aq_required |= I40EVF_FLAG_AQ_RELEASE_ALLMULTI;
 
-	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
 }
 
 /**
@@ -1058,6 +1000,8 @@ static void i40evf_configure(struct i40evf_adapter *adapter)
 /**
  * i40evf_up_complete - Finish the last steps of bringing up a connection
  * @adapter: board private structure
+ *
+ * Expects to be called while holding the __I40EVF_IN_CRITICAL_TASK bit lock.
  **/
 static void i40evf_up_complete(struct i40evf_adapter *adapter)
 {
@@ -1075,6 +1019,8 @@ static void i40evf_up_complete(struct i40evf_adapter *adapter)
 /**
  * i40e_down - Shutdown the connection processing
  * @adapter: board private structure
+ *
+ * Expects to be called while holding the __I40EVF_IN_CRITICAL_TASK bit lock.
  **/
 void i40evf_down(struct i40evf_adapter *adapter)
 {
@@ -1084,16 +1030,14 @@ void i40evf_down(struct i40evf_adapter *adapter)
 	if (adapter->state <= __I40EVF_DOWN_PENDING)
 		return;
 
-	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
-				&adapter->crit_section))
-		usleep_range(500, 1000);
-
 	netif_carrier_off(netdev);
 	netif_tx_disable(netdev);
 	adapter->link_up = false;
 	i40evf_napi_disable_all(adapter);
 	i40evf_irq_disable(adapter);
 
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	/* remove all MAC filters */
 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
 		f->remove = true;
@@ -1102,6 +1046,9 @@ void i40evf_down(struct i40evf_adapter *adapter)
 	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 		f->remove = true;
 	}
+
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	if (!(adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) &&
 	    adapter->state != __I40EVF_RESETTING) {
 		/* cancel any current operation */
@@ -1115,7 +1062,6 @@ void i40evf_down(struct i40evf_adapter *adapter)
 		adapter->aq_required |= I40EVF_FLAG_AQ_DISABLE_QUEUES;
 	}
 
-	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 	mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
 }
 
@@ -1441,6 +1387,7 @@ static int i40evf_alloc_q_vectors(struct i40evf_adapter *adapter)
 		q_vector->adapter = adapter;
 		q_vector->vsi = &adapter->vsi;
 		q_vector->v_idx = q_idx;
+		q_vector->reg_idx = q_idx;
 		cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
 		netif_napi_add(adapter->netdev, &q_vector->napi,
 			       i40evf_napi_poll, NAPI_POLL_WEIGHT);
@@ -1770,13 +1717,8 @@ static void i40evf_watchdog_task(struct work_struct *work)
 	if (adapter->state == __I40EVF_RUNNING)
 		i40evf_request_stats(adapter);
 watchdog_done:
-	if (adapter->state == __I40EVF_RUNNING) {
-		i40evf_irq_enable_queues(adapter, ~0);
-		i40evf_fire_sw_int(adapter, 0xFF);
-	} else {
-		i40evf_fire_sw_int(adapter, 0x1);
-	}
-
+	if (adapter->state == __I40EVF_RUNNING)
+		i40evf_detect_recover_hung(&adapter->vsi);
 	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 restart_watchdog:
 	if (adapter->state == __I40EVF_REMOVE)
@@ -1796,7 +1738,11 @@ static void i40evf_disable_vf(struct i40evf_adapter *adapter)
 
 	adapter->flags |= I40EVF_FLAG_PF_COMMS_FAILED;
 
-	if (netif_running(adapter->netdev)) {
+	/* We don't use netif_running() because it may be true prior to
+	 * ndo_open() returning, so we can't assume it means all our open
+	 * tasks have finished, since we're not holding the rtnl_lock here.
+	 */
+	if (adapter->state == __I40EVF_RUNNING) {
 		set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
 		netif_carrier_off(adapter->netdev);
 		netif_tx_disable(adapter->netdev);
@@ -1808,6 +1754,8 @@ static void i40evf_disable_vf(struct i40evf_adapter *adapter)
 		i40evf_free_all_rx_resources(adapter);
 	}
 
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	/* Delete all of the filters, both MAC and VLAN. */
 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
 		list_del(&f->list);
@@ -1819,6 +1767,8 @@ static void i40evf_disable_vf(struct i40evf_adapter *adapter)
 		kfree(fv);
 	}
 
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	i40evf_free_misc_irq(adapter);
 	i40evf_reset_interrupt_capability(adapter);
 	i40evf_free_queues(adapter);
@@ -1854,6 +1804,13 @@ static void i40evf_reset_task(struct work_struct *work)
 	struct i40evf_mac_filter *f;
 	u32 reg_val;
 	int i = 0, err;
+	bool running;
+
+	/* When device is being removed it doesn't make sense to run the reset
+	 * task, just return in such a case.
+	 */
+	if (test_bit(__I40EVF_IN_REMOVE_TASK, &adapter->crit_section))
+		return;
 
 	while (test_and_set_bit(__I40EVF_IN_CLIENT_TASK,
 				&adapter->crit_section))
@@ -1913,7 +1870,13 @@ static void i40evf_reset_task(struct work_struct *work)
 	}
 
 continue_reset:
-	if (netif_running(netdev)) {
+	/* We don't use netif_running() because it may be true prior to
+	 * ndo_open() returning, so we can't assume it means all our open
+	 * tasks have finished, since we're not holding the rtnl_lock here.
+	 */
+	running = (adapter->state == __I40EVF_RUNNING);
+
+	if (running) {
 		netif_carrier_off(netdev);
 		netif_tx_stop_all_queues(netdev);
 		adapter->link_up = false;
@@ -1948,6 +1911,8 @@ static void i40evf_reset_task(struct work_struct *work)
 	adapter->aq_required |= I40EVF_FLAG_AQ_GET_CONFIG;
 	adapter->aq_required |= I40EVF_FLAG_AQ_MAP_VECTORS;
 
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	/* re-add all MAC filters */
 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
 		f->add = true;
@@ -1956,15 +1921,19 @@ static void i40evf_reset_task(struct work_struct *work)
 	list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
 		vlf->add = true;
 	}
+
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
 	adapter->aq_required |= I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
-	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
-	clear_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section);
 	i40evf_misc_irq_enable(adapter);
 
 	mod_timer(&adapter->watchdog_timer, jiffies + 2);
 
-	if (netif_running(adapter->netdev)) {
+	/* We were running when the reset started, so we need to restore some
+	 * state here.
+	 */
+	if (running) {
 		/* allocate transmit descriptors */
 		err = i40evf_setup_all_tx_resources(adapter);
 		if (err)
@@ -1993,9 +1962,13 @@ static void i40evf_reset_task(struct work_struct *work)
 		adapter->state = __I40EVF_DOWN;
 		wake_up(&adapter->down_waitqueue);
 	}
+	clear_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section);
+	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 
 	return;
 reset_err:
+	clear_bit(__I40EVF_IN_CLIENT_TASK, &adapter->crit_section);
+	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 	dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
 	i40evf_close(netdev);
 }
@@ -2239,8 +2212,14 @@ static int i40evf_open(struct net_device *netdev)
 		return -EIO;
 	}
 
-	if (adapter->state != __I40EVF_DOWN)
-		return -EBUSY;
+	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
+				&adapter->crit_section))
+		usleep_range(500, 1000);
+
+	if (adapter->state != __I40EVF_DOWN) {
+		err = -EBUSY;
+		goto err_unlock;
+	}
 
 	/* allocate transmit descriptors */
 	err = i40evf_setup_all_tx_resources(adapter);
@@ -2264,6 +2243,8 @@ static int i40evf_open(struct net_device *netdev)
 
 	i40evf_irq_enable(adapter, true);
 
+	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
+
 	return 0;
 
 err_req_irq:
@@ -2273,6 +2254,8 @@ static int i40evf_open(struct net_device *netdev)
 	i40evf_free_all_rx_resources(adapter);
 err_setup_tx:
 	i40evf_free_all_tx_resources(adapter);
+err_unlock:
+	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 
 	return err;
 }
@@ -2296,6 +2279,9 @@ static int i40evf_close(struct net_device *netdev)
 	if (adapter->state <= __I40EVF_DOWN_PENDING)
 		return 0;
 
+	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
+				&adapter->crit_section))
+		usleep_range(500, 1000);
 
 	set_bit(__I40E_VSI_DOWN, adapter->vsi.state);
 	if (CLIENT_ENABLED(adapter))
@@ -2305,6 +2291,8 @@ static int i40evf_close(struct net_device *netdev)
 	adapter->state = __I40EVF_DOWN_PENDING;
 	i40evf_free_traffic_irqs(adapter);
 
+	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
+
 	/* We explicitly don't free resources here because the hardware is
 	 * still active and can DMA into memory. Resources are cleared in
 	 * i40evf_virtchnl_completion() after we get confirmation from the PF
@@ -2357,13 +2345,19 @@ static int i40evf_set_features(struct net_device *netdev,
 {
 	struct i40evf_adapter *adapter = netdev_priv(netdev);
 
-	if (!VLAN_ALLOWED(adapter))
+	/* Don't allow changing VLAN_RX flag when VLAN is set for VF
+	 * and return an error in this case
+	 */
+	if (VLAN_ALLOWED(adapter)) {
+		if (features & NETIF_F_HW_VLAN_CTAG_RX)
+			adapter->aq_required |=
+				I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
+		else
+			adapter->aq_required |=
+				I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
+	} else if ((netdev->features ^ features) & NETIF_F_HW_VLAN_CTAG_RX) {
 		return -EINVAL;
-
-	if (features & NETIF_F_HW_VLAN_CTAG_RX)
-		adapter->aq_required |= I40EVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
-	else
-		adapter->aq_required |= I40EVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
+	}
 
 	return 0;
 }
@@ -2943,6 +2937,8 @@ static int i40evf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	mutex_init(&hw->aq.asq_mutex);
 	mutex_init(&hw->aq.arq_mutex);
 
+	spin_lock_init(&adapter->mac_vlan_list_lock);
+
 	INIT_LIST_HEAD(&adapter->mac_filter_list);
 	INIT_LIST_HEAD(&adapter->vlan_filter_list);
 
@@ -2985,6 +2981,10 @@ static int i40evf_suspend(struct pci_dev *pdev, pm_message_t state)
 
 	netif_device_detach(netdev);
 
+	while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
+				&adapter->crit_section))
+		usleep_range(500, 1000);
+
 	if (netif_running(netdev)) {
 		rtnl_lock();
 		i40evf_down(adapter);
@@ -2993,6 +2993,8 @@ static int i40evf_suspend(struct pci_dev *pdev, pm_message_t state)
 	i40evf_free_misc_irq(adapter);
 	i40evf_reset_interrupt_capability(adapter);
 
+	clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
+
 	retval = pci_save_state(pdev);
 	if (retval)
 		return retval;
@@ -3066,7 +3068,8 @@ static void i40evf_remove(struct pci_dev *pdev)
 	struct i40evf_mac_filter *f, *ftmp;
 	struct i40e_hw *hw = &adapter->hw;
 	int err;
-
+	/* Indicate we are in remove and not to run reset_task */
+	set_bit(__I40EVF_IN_REMOVE_TASK, &adapter->crit_section);
 	cancel_delayed_work_sync(&adapter->init_task);
 	cancel_work_sync(&adapter->reset_task);
 	cancel_delayed_work_sync(&adapter->client_task);
@@ -3101,8 +3104,6 @@ static void i40evf_remove(struct pci_dev *pdev)
 	if (adapter->watchdog_timer.function)
 		del_timer_sync(&adapter->watchdog_timer);
 
-	flush_scheduled_work();
-
 	i40evf_free_rss(adapter);
 
 	if (hw->aq.asq.count)
@@ -3118,6 +3119,7 @@ static void i40evf_remove(struct pci_dev *pdev)
 	i40evf_free_all_rx_resources(adapter);
 	i40evf_free_queues(adapter);
 	kfree(adapter->vf_res);
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
 	/* If we got removed before an up/down sequence, we've got a filter
 	 * hanging out there that we need to get rid of.
 	 */
@@ -3130,6 +3132,8 @@ static void i40evf_remove(struct pci_dev *pdev)
 		kfree(f);
 	}
 
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	free_netdev(netdev);
 
 	pci_disable_pcie_error_reporting(pdev);
diff --git a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
index 46c8b8a3907c..50ce0d6c09ef 100644
--- a/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
+++ b/drivers/net/ethernet/intel/i40evf/i40evf_virtchnl.c
@@ -433,12 +433,16 @@ void i40evf_add_ether_addrs(struct i40evf_adapter *adapter)
 			adapter->current_op);
 		return;
 	}
+
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
 		if (f->add)
 			count++;
 	}
 	if (!count) {
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_ADD_MAC_FILTER;
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
 	}
 	adapter->current_op = VIRTCHNL_OP_ADD_ETH_ADDR;
@@ -456,8 +460,10 @@ void i40evf_add_ether_addrs(struct i40evf_adapter *adapter)
 	}
 
 	veal = kzalloc(len, GFP_KERNEL);
-	if (!veal)
+	if (!veal) {
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
+	}
 
 	veal->vsi_id = adapter->vsi_res->vsi_id;
 	veal->num_elements = count;
@@ -472,6 +478,9 @@ void i40evf_add_ether_addrs(struct i40evf_adapter *adapter)
 	}
 	if (!more)
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_ADD_MAC_FILTER;
+
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	i40evf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_ETH_ADDR,
 			   (u8 *)veal, len);
 	kfree(veal);
@@ -498,12 +507,16 @@ void i40evf_del_ether_addrs(struct i40evf_adapter *adapter)
 			adapter->current_op);
 		return;
 	}
+
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
 		if (f->remove)
 			count++;
 	}
 	if (!count) {
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_DEL_MAC_FILTER;
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
 	}
 	adapter->current_op = VIRTCHNL_OP_DEL_ETH_ADDR;
@@ -520,8 +533,10 @@ void i40evf_del_ether_addrs(struct i40evf_adapter *adapter)
 		more = true;
 	}
 	veal = kzalloc(len, GFP_KERNEL);
-	if (!veal)
+	if (!veal) {
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
+	}
 
 	veal->vsi_id = adapter->vsi_res->vsi_id;
 	veal->num_elements = count;
@@ -537,6 +552,9 @@ void i40evf_del_ether_addrs(struct i40evf_adapter *adapter)
 	}
 	if (!more)
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_DEL_MAC_FILTER;
+
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	i40evf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_ETH_ADDR,
 			   (u8 *)veal, len);
 	kfree(veal);
@@ -564,12 +582,15 @@ void i40evf_add_vlans(struct i40evf_adapter *adapter)
 		return;
 	}
 
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 		if (f->add)
 			count++;
 	}
 	if (!count) {
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
 	}
 	adapter->current_op = VIRTCHNL_OP_ADD_VLAN;
@@ -586,8 +607,10 @@ void i40evf_add_vlans(struct i40evf_adapter *adapter)
 		more = true;
 	}
 	vvfl = kzalloc(len, GFP_KERNEL);
-	if (!vvfl)
+	if (!vvfl) {
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
+	}
 
 	vvfl->vsi_id = adapter->vsi_res->vsi_id;
 	vvfl->num_elements = count;
@@ -602,6 +625,9 @@ void i40evf_add_vlans(struct i40evf_adapter *adapter)
 	}
 	if (!more)
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
+
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	i40evf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_VLAN, (u8 *)vvfl, len);
 	kfree(vvfl);
 }
@@ -628,12 +654,15 @@ void i40evf_del_vlans(struct i40evf_adapter *adapter)
 		return;
 	}
 
+	spin_lock_bh(&adapter->mac_vlan_list_lock);
+
 	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 		if (f->remove)
 			count++;
 	}
 	if (!count) {
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
 	}
 	adapter->current_op = VIRTCHNL_OP_DEL_VLAN;
@@ -650,8 +679,10 @@ void i40evf_del_vlans(struct i40evf_adapter *adapter)
 		more = true;
 	}
 	vvfl = kzalloc(len, GFP_KERNEL);
-	if (!vvfl)
+	if (!vvfl) {
+		spin_unlock_bh(&adapter->mac_vlan_list_lock);
 		return;
+	}
 
 	vvfl->vsi_id = adapter->vsi_res->vsi_id;
 	vvfl->num_elements = count;
@@ -667,6 +698,9 @@ void i40evf_del_vlans(struct i40evf_adapter *adapter)
 	}
 	if (!more)
 		adapter->aq_required &= ~I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
+
+	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+
 	i40evf_send_pf_msg(adapter, VIRTCHNL_OP_DEL_VLAN, (u8 *)vvfl, len);
 	kfree(vvfl);
 }
@@ -705,8 +739,10 @@ void i40evf_set_promiscuous(struct i40evf_adapter *adapter, int flags)
 	}
 
 	if (!flags) {
-		adapter->flags &= ~I40EVF_FLAG_PROMISC_ON;
-		adapter->aq_required &= ~I40EVF_FLAG_AQ_RELEASE_PROMISC;
+		adapter->flags &= ~(I40EVF_FLAG_PROMISC_ON |
+				    I40EVF_FLAG_ALLMULTI_ON);
+		adapter->aq_required &= ~(I40EVF_FLAG_AQ_RELEASE_PROMISC |
+					  I40EVF_FLAG_AQ_RELEASE_ALLMULTI);
 		dev_info(&adapter->pdev->dev, "Leaving promiscuous mode\n");
 	}
 
@@ -965,23 +1001,34 @@ void i40evf_virtchnl_completion(struct i40evf_adapter *adapter,
 	if (v_opcode == VIRTCHNL_OP_EVENT) {
 		struct virtchnl_pf_event *vpe =
 			(struct virtchnl_pf_event *)msg;
+		bool link_up = vpe->event_data.link_event.link_status;
 		switch (vpe->event) {
 		case VIRTCHNL_EVENT_LINK_CHANGE:
 			adapter->link_speed =
 				vpe->event_data.link_event.link_speed;
-			if (adapter->link_up !=
-			    vpe->event_data.link_event.link_status) {
-				adapter->link_up =
-					vpe->event_data.link_event.link_status;
-				if (adapter->link_up) {
-					netif_tx_start_all_queues(netdev);
-					netif_carrier_on(netdev);
-				} else {
-					netif_tx_stop_all_queues(netdev);
-					netif_carrier_off(netdev);
-				}
-				i40evf_print_link_message(adapter);
+
+			/* we've already got the right link status, bail */
+			if (adapter->link_up == link_up)
+				break;
+
+			/* If we get link up message and start queues before
+			 * our queues are configured it will trigger a TX hang.
+			 * In that case, just ignore the link status message,
+			 * we'll get another one after we enable queues and
+			 * actually prepared to send traffic.
+			 */
+			if (link_up && adapter->state != __I40EVF_RUNNING)
+				break;
+
+			adapter->link_up = link_up;
+			if (link_up) {
+				netif_tx_start_all_queues(netdev);
+				netif_carrier_on(netdev);
+			} else {
+				netif_tx_stop_all_queues(netdev);
+				netif_carrier_off(netdev);
 			}
+			i40evf_print_link_message(adapter);
 			break;
 		case VIRTCHNL_EVENT_RESET_IMPENDING:
 			dev_info(&adapter->pdev->dev, "PF reset warning received\n");
diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h
index 92845692087a..1c6b8d9176a8 100644
--- a/drivers/net/ethernet/intel/igb/igb.h
+++ b/drivers/net/ethernet/intel/igb/igb.h
@@ -690,6 +690,7 @@ void igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va,
 int igb_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr);
 int igb_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr);
 void igb_set_flag_queue_pairs(struct igb_adapter *, const u32);
+unsigned int igb_get_max_rss_queues(struct igb_adapter *);
 #ifdef CONFIG_IGB_HWMON
 void igb_sysfs_exit(struct igb_adapter *adapter);
 int igb_sysfs_init(struct igb_adapter *adapter);
diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c
index d06a8db514d4..606e6761758f 100644
--- a/drivers/net/ethernet/intel/igb/igb_ethtool.c
+++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c
@@ -3338,37 +3338,7 @@ static int igb_set_rxfh(struct net_device *netdev, const u32 *indir,
 
 static unsigned int igb_max_channels(struct igb_adapter *adapter)
 {
-	struct e1000_hw *hw = &adapter->hw;
-	unsigned int max_combined = 0;
-
-	switch (hw->mac.type) {
-	case e1000_i211:
-		max_combined = IGB_MAX_RX_QUEUES_I211;
-		break;
-	case e1000_82575:
-	case e1000_i210:
-		max_combined = IGB_MAX_RX_QUEUES_82575;
-		break;
-	case e1000_i350:
-		if (!!adapter->vfs_allocated_count) {
-			max_combined = 1;
-			break;
-		}
-		/* fall through */
-	case e1000_82576:
-		if (!!adapter->vfs_allocated_count) {
-			max_combined = 2;
-			break;
-		}
-		/* fall through */
-	case e1000_82580:
-	case e1000_i354:
-	default:
-		max_combined = IGB_MAX_RX_QUEUES;
-		break;
-	}
-
-	return max_combined;
+	return igb_get_max_rss_queues(adapter);
 }
 
 static void igb_get_channels(struct net_device *netdev,
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index c208753ff5b7..b88fae785369 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -1744,6 +1744,20 @@ static void igb_configure_cbs(struct igb_adapter *adapter, int queue,
 		 *     value = idleSlope * 61034
 		 *             -----------------                          (E6)
 		 *                  1000000
+		 *
+		 * NOTE: For i210, given the above, we can see that idleslope
+		 *       is represented in 16.38431 kbps units by the value at
+		 *       the TQAVCC register (1Gbps / 61034), which reduces
+		 *       the granularity for idleslope increments.
+		 *       For instance, if you want to configure a 2576kbps
+		 *       idleslope, the value to be written on the register
+		 *       would have to be 157.23. If rounded down, you end
+		 *       up with less bandwidth available than originally
+		 *       required (~2572 kbps). If rounded up, you end up
+		 *       with a higher bandwidth (~2589 kbps). Below the
+		 *       approach we take is to always round up the
+		 *       calculated value, so the resulting bandwidth might
+		 *       be slightly higher for some configurations.
 		 */
 		value = DIV_ROUND_UP_ULL(idleslope * 61034ULL, 1000000);
 
@@ -3200,8 +3214,6 @@ static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs)
 	/* if allocation failed then we do not support SR-IOV */
 	if (!adapter->vf_data) {
 		adapter->vfs_allocated_count = 0;
-		dev_err(&pdev->dev,
-			"Unable to allocate memory for VF Data Storage\n");
 		err = -ENOMEM;
 		goto out;
 	}
@@ -3373,10 +3385,10 @@ static void igb_probe_vfs(struct igb_adapter *adapter)
 #endif /* CONFIG_PCI_IOV */
 }
 
-static void igb_init_queue_configuration(struct igb_adapter *adapter)
+unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter)
 {
 	struct e1000_hw *hw = &adapter->hw;
-	u32 max_rss_queues;
+	unsigned int max_rss_queues;
 
 	/* Determine the maximum number of RSS queues supported. */
 	switch (hw->mac.type) {
@@ -3407,6 +3419,14 @@ static void igb_init_queue_configuration(struct igb_adapter *adapter)
 		break;
 	}
 
+	return max_rss_queues;
+}
+
+static void igb_init_queue_configuration(struct igb_adapter *adapter)
+{
+	u32 max_rss_queues;
+
+	max_rss_queues = igb_get_max_rss_queues(adapter);
 	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
 
 	igb_set_flag_queue_pairs(adapter, max_rss_queues);
@@ -3676,7 +3696,7 @@ static int __igb_close(struct net_device *netdev, bool suspending)
 
 int igb_close(struct net_device *netdev)
 {
-	if (netif_device_present(netdev))
+	if (netif_device_present(netdev) || netdev->dismantle)
 		return __igb_close(netdev, false);
 	return 0;
 }
@@ -8718,7 +8738,8 @@ static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
 
 	/* Indicate to hardware the Address is Valid. */
 	if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
-		rar_high |= E1000_RAH_AV;
+		if (is_valid_ether_addr(addr))
+			rar_high |= E1000_RAH_AV;
 
 		if (hw->mac.type == e1000_82575)
 			rar_high |= E1000_RAH_POOL_1 *
@@ -8756,17 +8777,36 @@ static int igb_set_vf_mac(struct igb_adapter *adapter,
 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
 {
 	struct igb_adapter *adapter = netdev_priv(netdev);
-	if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
+
+	if (vf >= adapter->vfs_allocated_count)
+		return -EINVAL;
+
+	/* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
+	 * flag and allows to overwrite the MAC via VF netdev.  This
+	 * is necessary to allow libvirt a way to restore the original
+	 * MAC after unbinding vfio-pci and reloading igbvf after shutting
+	 * down a VM.
+	 */
+	if (is_zero_ether_addr(mac)) {
+		adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
+		dev_info(&adapter->pdev->dev,
+			 "remove administratively set MAC on VF %d\n",
+			 vf);
+	} else if (is_valid_ether_addr(mac)) {
+		adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
+		dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
+			 mac, vf);
+		dev_info(&adapter->pdev->dev,
+			 "Reload the VF driver to make this change effective.");
+		/* Generate additional warning if PF is down */
+		if (test_bit(__IGB_DOWN, &adapter->state)) {
+			dev_warn(&adapter->pdev->dev,
+				 "The VF MAC address has been set, but the PF device is not up.\n");
+			dev_warn(&adapter->pdev->dev,
+				 "Bring the PF device up before attempting to use the VF device.\n");
+		}
+	} else {
 		return -EINVAL;
-	adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
-	dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
-	dev_info(&adapter->pdev->dev,
-		 "Reload the VF driver to make this change effective.");
-	if (test_bit(__IGB_DOWN, &adapter->state)) {
-		dev_warn(&adapter->pdev->dev,
-			 "The VF MAC address has been set, but the PF device is not up.\n");
-		dev_warn(&adapter->pdev->dev,
-			 "Bring the PF device up before attempting to use the VF device.\n");
 	}
 	return igb_set_vf_mac(adapter, vf, mac);
 }
diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c
index 841c2a083349..0746b19ec6d3 100644
--- a/drivers/net/ethernet/intel/igb/igb_ptp.c
+++ b/drivers/net/ethernet/intel/igb/igb_ptp.c
@@ -643,6 +643,10 @@ static void igb_ptp_tx_work(struct work_struct *work)
 		adapter->ptp_tx_skb = NULL;
 		clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
 		adapter->tx_hwtstamp_timeouts++;
+		/* Clear the tx valid bit in TSYNCTXCTL register to enable
+		 * interrupt
+		 */
+		rd32(E1000_TXSTMPH);
 		dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
 		return;
 	}
@@ -717,6 +721,7 @@ void igb_ptp_rx_hang(struct igb_adapter *adapter)
  */
 void igb_ptp_tx_hang(struct igb_adapter *adapter)
 {
+	struct e1000_hw *hw = &adapter->hw;
 	bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
 					      IGB_PTP_TX_TIMEOUT);
 
@@ -736,6 +741,10 @@ void igb_ptp_tx_hang(struct igb_adapter *adapter)
 		adapter->ptp_tx_skb = NULL;
 		clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
 		adapter->tx_hwtstamp_timeouts++;
+		/* Clear the tx valid bit in TSYNCTXCTL register to enable
+		 * interrupt
+		 */
+		rd32(E1000_TXSTMPH);
 		dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
 	}
 }
diff --git a/drivers/net/ethernet/intel/ixgbe/Makefile b/drivers/net/ethernet/intel/ixgbe/Makefile
index 35e6fa643c7e..8319465eb38d 100644
--- a/drivers/net/ethernet/intel/ixgbe/Makefile
+++ b/drivers/net/ethernet/intel/ixgbe/Makefile
@@ -42,3 +42,4 @@ ixgbe-$(CONFIG_IXGBE_DCB) +=  ixgbe_dcb.o ixgbe_dcb_82598.o \
 ixgbe-$(CONFIG_IXGBE_HWMON) += ixgbe_sysfs.o
 ixgbe-$(CONFIG_DEBUG_FS) += ixgbe_debugfs.o
 ixgbe-$(CONFIG_FCOE:m=y) += ixgbe_fcoe.o
+ixgbe-$(CONFIG_XFRM_OFFLOAD) += ixgbe_ipsec.o
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index 468c3555a629..c1e3a0039ea5 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -52,7 +52,9 @@
 #ifdef CONFIG_IXGBE_DCA
 #include <linux/dca.h>
 #endif
+#include "ixgbe_ipsec.h"
 
+#include <net/xdp.h>
 #include <net/busy_poll.h>
 
 /* common prefix used by pr_<> macros */
@@ -170,10 +172,11 @@ enum ixgbe_tx_flags {
 	IXGBE_TX_FLAGS_CC	= 0x08,
 	IXGBE_TX_FLAGS_IPV4	= 0x10,
 	IXGBE_TX_FLAGS_CSUM	= 0x20,
+	IXGBE_TX_FLAGS_IPSEC	= 0x40,
 
 	/* software defined flags */
-	IXGBE_TX_FLAGS_SW_VLAN	= 0x40,
-	IXGBE_TX_FLAGS_FCOE	= 0x80,
+	IXGBE_TX_FLAGS_SW_VLAN	= 0x80,
+	IXGBE_TX_FLAGS_FCOE	= 0x100,
 };
 
 /* VLAN info */
@@ -332,7 +335,6 @@ struct ixgbe_ring {
 	struct net_device *netdev;	/* netdev ring belongs to */
 	struct bpf_prog *xdp_prog;
 	struct device *dev;		/* device for DMA mapping */
-	struct ixgbe_fwd_adapter *l2_accel_priv;
 	void *desc;			/* descriptor ring memory */
 	union {
 		struct ixgbe_tx_buffer *tx_buffer_info;
@@ -371,6 +373,7 @@ struct ixgbe_ring {
 		struct ixgbe_tx_queue_stats tx_stats;
 		struct ixgbe_rx_queue_stats rx_stats;
 	};
+	struct xdp_rxq_info xdp_rxq;
 } ____cacheline_internodealigned_in_smp;
 
 enum ixgbe_ring_f_enum {
@@ -395,8 +398,7 @@ enum ixgbe_ring_f_enum {
 #define MAX_XDP_QUEUES			(IXGBE_MAX_FDIR_INDICES + 1)
 #define IXGBE_MAX_L2A_QUEUES		4
 #define IXGBE_BAD_L2A_QUEUE		3
-#define IXGBE_MAX_MACVLANS		31
-#define IXGBE_MAX_DCBMACVLANS		8
+#define IXGBE_MAX_MACVLANS		63
 
 struct ixgbe_ring_feature {
 	u16 limit;	/* upper limit on feature indices */
@@ -629,15 +631,18 @@ struct ixgbe_adapter {
 #define IXGBE_FLAG2_EEE_CAPABLE			BIT(14)
 #define IXGBE_FLAG2_EEE_ENABLED			BIT(15)
 #define IXGBE_FLAG2_RX_LEGACY			BIT(16)
+#define IXGBE_FLAG2_IPSEC_ENABLED		BIT(17)
 
 	/* Tx fast path data */
 	int num_tx_queues;
 	u16 tx_itr_setting;
 	u16 tx_work_limit;
+	u64 tx_ipsec;
 
 	/* Rx fast path data */
 	int num_rx_queues;
 	u16 rx_itr_setting;
+	u64 rx_ipsec;
 
 	/* Port number used to identify VXLAN traffic */
 	__be16 vxlan_port;
@@ -674,6 +679,7 @@ struct ixgbe_adapter {
 	struct ieee_ets *ixgbe_ieee_ets;
 	struct ixgbe_dcb_config dcb_cfg;
 	struct ixgbe_dcb_config temp_dcb_cfg;
+	u8 hw_tcs;
 	u8 dcb_set_bitmap;
 	u8 dcbx_cap;
 	enum ixgbe_fc_mode last_lfc_mode;
@@ -721,8 +727,7 @@ struct ixgbe_adapter {
 
 	u16 bridge_mode;
 
-	u16 eeprom_verh;
-	u16 eeprom_verl;
+	char eeprom_id[NVM_VER_SIZE];
 	u16 eeprom_cap;
 
 	u32 interrupt_event;
@@ -766,7 +771,8 @@ struct ixgbe_adapter {
 #endif /*CONFIG_DEBUG_FS*/
 
 	u8 default_up;
-	unsigned long fwd_bitmask; /* Bitmask indicating in use pools */
+	/* Bitmask indicating in use pools */
+	DECLARE_BITMAP(fwd_bitmask, IXGBE_MAX_MACVLANS + 1);
 
 #define IXGBE_MAX_LINK_HANDLE 10
 	struct ixgbe_jump_table *jump_tables[IXGBE_MAX_LINK_HANDLE];
@@ -780,6 +786,10 @@ struct ixgbe_adapter {
 
 #define IXGBE_RSS_KEY_SIZE     40  /* size of RSS Hash Key in bytes */
 	u32 *rss_key;
+
+#ifdef CONFIG_XFRM
+	struct ixgbe_ipsec *ipsec;
+#endif /* CONFIG_XFRM */
 };
 
 static inline u8 ixgbe_max_rss_indices(struct ixgbe_adapter *adapter)
@@ -1010,4 +1020,24 @@ void ixgbe_store_key(struct ixgbe_adapter *adapter);
 void ixgbe_store_reta(struct ixgbe_adapter *adapter);
 s32 ixgbe_negotiate_fc(struct ixgbe_hw *hw, u32 adv_reg, u32 lp_reg,
 		       u32 adv_sym, u32 adv_asm, u32 lp_sym, u32 lp_asm);
+#ifdef CONFIG_XFRM_OFFLOAD
+void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter);
+void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter);
+void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter);
+void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring,
+		    union ixgbe_adv_rx_desc *rx_desc,
+		    struct sk_buff *skb);
+int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first,
+		   struct ixgbe_ipsec_tx_data *itd);
+#else
+static inline void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter) { };
+static inline void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter) { };
+static inline void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter) { };
+static inline void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring,
+				  union ixgbe_adv_rx_desc *rx_desc,
+				  struct sk_buff *skb) { };
+static inline int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring,
+				 struct ixgbe_tx_buffer *first,
+				 struct ixgbe_ipsec_tx_data *itd) { return 0; };
+#endif /* CONFIG_XFRM_OFFLOAD */
 #endif /* _IXGBE_H_ */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c
index 8a32eb7d47b9..a0ebd9ecf243 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_82598.c
@@ -431,6 +431,7 @@ static s32 ixgbe_fc_enable_82598(struct ixgbe_hw *hw)
 /**
  *  ixgbe_start_mac_link_82598 - Configures MAC link settings
  *  @hw: pointer to hardware structure
+ *  @autoneg_wait_to_complete: true when waiting for completion is needed
  *
  *  Configures link settings based on values in the ixgbe_hw struct.
  *  Restarts the link.  Performs autonegotiation if needed.
@@ -1054,7 +1055,7 @@ static s32 ixgbe_read_i2c_eeprom_82598(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_read_i2c_sff8472_82598 - Reads 8 bit word over I2C interface.
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset at address 0xA2
- *  @eeprom_data: value read
+ *  @sff8472_data: value read
  *
  *  Performs 8 byte read operation to SFP module's SFF-8472 data over I2C
  **/
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c
index d602637ccc40..4dfc81dbee4b 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_82599.c
@@ -221,7 +221,7 @@ static s32 prot_autoc_read_82599(struct ixgbe_hw *hw, bool *locked,
 /**
  * prot_autoc_write_82599 - Hides MAC differences needed for AUTOC write
  * @hw: pointer to hardware structure
- * @reg_val: value to write to AUTOC
+ * @autoc: value to write to AUTOC
  * @locked: bool to indicate whether the SW/FW lock was already taken by
  *	     previous proc_autoc_read_82599.
  *
@@ -1310,10 +1310,11 @@ do { \
 
 /**
  *  ixgbe_atr_compute_sig_hash_82599 - Compute the signature hash
- *  @stream: input bitstream to compute the hash on
+ *  @input: input bitstream to compute the hash on
+ *  @common: compressed common input dword
  *
  *  This function is almost identical to the function above but contains
- *  several optomizations such as unwinding all of the loops, letting the
+ *  several optimizations such as unwinding all of the loops, letting the
  *  compiler work out all of the conditional ifs since the keys are static
  *  defines, and computing two keys at once since the hashed dword stream
  *  will be the same for both keys.
@@ -1446,7 +1447,7 @@ do { \
 
 /**
  *  ixgbe_atr_compute_perfect_hash_82599 - Compute the perfect filter hash
- *  @atr_input: input bitstream to compute the hash on
+ *  @input: input bitstream to compute the hash on
  *  @input_mask: mask for the input bitstream
  *
  *  This function serves two main purposes.  First it applies the input_mask
@@ -2078,6 +2079,7 @@ static s32 ixgbe_reset_pipeline_82599(struct ixgbe_hw *hw)
  *  ixgbe_read_i2c_byte_82599 - Reads 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to read
+ *  @dev_addr: address to read from
  *  @data: value read
  *
  *  Performs byte read operation to SFP module's EEPROM over I2C interface at
@@ -2131,6 +2133,7 @@ static s32 ixgbe_read_i2c_byte_82599(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_write_i2c_byte_82599 - Writes 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to write
+ *  @dev_addr: address to write to
  *  @data: value to write
  *
  *  Performs byte write operation to SFP module's EEPROM over I2C interface at
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
index 9bef255f6a18..61188f343955 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c
@@ -1613,6 +1613,7 @@ static void ixgbe_shift_out_eeprom_bits(struct ixgbe_hw *hw, u16 data,
 /**
  *  ixgbe_shift_in_eeprom_bits - Shift data bits in from the EEPROM
  *  @hw: pointer to hardware structure
+ *  @count: number of bits to shift
  **/
 static u16 ixgbe_shift_in_eeprom_bits(struct ixgbe_hw *hw, u16 count)
 {
@@ -1667,7 +1668,7 @@ static void ixgbe_raise_eeprom_clk(struct ixgbe_hw *hw, u32 *eec)
 /**
  *  ixgbe_lower_eeprom_clk - Lowers the EEPROM's clock input.
  *  @hw: pointer to hardware structure
- *  @eecd: EECD's current value
+ *  @eec: EEC's current value
  **/
 static void ixgbe_lower_eeprom_clk(struct ixgbe_hw *hw, u32 *eec)
 {
@@ -2037,7 +2038,7 @@ static s32 ixgbe_mta_vector(struct ixgbe_hw *hw, u8 *mc_addr)
 /**
  *  ixgbe_set_mta - Set bit-vector in multicast table
  *  @hw: pointer to hardware structure
- *  @hash_value: Multicast address hash value
+ *  @mc_addr: Multicast address
  *
  *  Sets the bit-vector in the multicast table.
  **/
@@ -3086,6 +3087,8 @@ s32 ixgbe_init_uta_tables_generic(struct ixgbe_hw *hw)
  *  ixgbe_find_vlvf_slot - find the vlanid or the first empty slot
  *  @hw: pointer to hardware structure
  *  @vlan: VLAN id to write to VLAN filter
+ *  @vlvf_bypass: true to find vlanid only, false returns first empty slot if
+ *		  vlanid not found
  *
  *  return the VLVF index where this VLAN id should be placed
  *
@@ -3476,7 +3479,7 @@ void ixgbe_set_mac_anti_spoofing(struct ixgbe_hw *hw, bool enable, int vf)
  *  ixgbe_set_vlan_anti_spoofing - Enable/Disable VLAN anti-spoofing
  *  @hw: pointer to hardware structure
  *  @enable: enable or disable switch for VLAN anti-spoofing
- *  @pf: Virtual Function pool - VF Pool to set for VLAN anti-spoofing
+ *  @vf: Virtual Function pool - VF Pool to set for VLAN anti-spoofing
  *
  **/
 void ixgbe_set_vlan_anti_spoofing(struct ixgbe_hw *hw, bool enable, int vf)
@@ -4028,6 +4031,118 @@ s32 ixgbe_init_thermal_sensor_thresh_generic(struct ixgbe_hw *hw)
 	return 0;
 }
 
+/**
+ *  ixgbe_get_orom_version - Return option ROM from EEPROM
+ *
+ *  @hw: pointer to hardware structure
+ *  @nvm_ver: pointer to output structure
+ *
+ *  if valid option ROM version, nvm_ver->or_valid set to true
+ *  else nvm_ver->or_valid is false.
+ **/
+void ixgbe_get_orom_version(struct ixgbe_hw *hw,
+			    struct ixgbe_nvm_version *nvm_ver)
+{
+	u16 offset, eeprom_cfg_blkh, eeprom_cfg_blkl;
+
+	nvm_ver->or_valid = false;
+	/* Option Rom may or may not be present.  Start with pointer */
+	hw->eeprom.ops.read(hw, NVM_OROM_OFFSET, &offset);
+
+	/* make sure offset is valid */
+	if (offset == 0x0 || offset == NVM_INVALID_PTR)
+		return;
+
+	hw->eeprom.ops.read(hw, offset + NVM_OROM_BLK_HI, &eeprom_cfg_blkh);
+	hw->eeprom.ops.read(hw, offset + NVM_OROM_BLK_LOW, &eeprom_cfg_blkl);
+
+	/* option rom exists and is valid */
+	if ((eeprom_cfg_blkl | eeprom_cfg_blkh) == 0x0 ||
+	    eeprom_cfg_blkl == NVM_VER_INVALID ||
+	    eeprom_cfg_blkh == NVM_VER_INVALID)
+		return;
+
+	nvm_ver->or_valid = true;
+	nvm_ver->or_major = eeprom_cfg_blkl >> NVM_OROM_SHIFT;
+	nvm_ver->or_build = (eeprom_cfg_blkl << NVM_OROM_SHIFT) |
+			    (eeprom_cfg_blkh >> NVM_OROM_SHIFT);
+	nvm_ver->or_patch = eeprom_cfg_blkh & NVM_OROM_PATCH_MASK;
+}
+
+/**
+ *  ixgbe_get_oem_prod_version Etrack ID from EEPROM
+ *
+ *  @hw: pointer to hardware structure
+ *  @nvm_ver: pointer to output structure
+ *
+ *  if valid OEM product version, nvm_ver->oem_valid set to true
+ *  else nvm_ver->oem_valid is false.
+ **/
+void ixgbe_get_oem_prod_version(struct ixgbe_hw *hw,
+				struct ixgbe_nvm_version *nvm_ver)
+{
+	u16 rel_num, prod_ver, mod_len, cap, offset;
+
+	nvm_ver->oem_valid = false;
+	hw->eeprom.ops.read(hw, NVM_OEM_PROD_VER_PTR, &offset);
+
+	/* Return is offset to OEM Product Version block is invalid */
+	if (offset == 0x0 || offset == NVM_INVALID_PTR)
+		return;
+
+	/* Read product version block */
+	hw->eeprom.ops.read(hw, offset, &mod_len);
+	hw->eeprom.ops.read(hw, offset + NVM_OEM_PROD_VER_CAP_OFF, &cap);
+
+	/* Return if OEM product version block is invalid */
+	if (mod_len != NVM_OEM_PROD_VER_MOD_LEN ||
+	    (cap & NVM_OEM_PROD_VER_CAP_MASK) != 0x0)
+		return;
+
+	hw->eeprom.ops.read(hw, offset + NVM_OEM_PROD_VER_OFF_L, &prod_ver);
+	hw->eeprom.ops.read(hw, offset + NVM_OEM_PROD_VER_OFF_H, &rel_num);
+
+	/* Return if version is invalid */
+	if ((rel_num | prod_ver) == 0x0 ||
+	    rel_num == NVM_VER_INVALID || prod_ver == NVM_VER_INVALID)
+		return;
+
+	nvm_ver->oem_major = prod_ver >> NVM_VER_SHIFT;
+	nvm_ver->oem_minor = prod_ver & NVM_VER_MASK;
+	nvm_ver->oem_release = rel_num;
+	nvm_ver->oem_valid = true;
+}
+
+/**
+ *  ixgbe_get_etk_id - Return Etrack ID from EEPROM
+ *
+ *  @hw: pointer to hardware structure
+ *  @nvm_ver: pointer to output structure
+ *
+ *  word read errors will return 0xFFFF
+ **/
+void ixgbe_get_etk_id(struct ixgbe_hw *hw,
+		      struct ixgbe_nvm_version *nvm_ver)
+{
+	u16 etk_id_l, etk_id_h;
+
+	if (hw->eeprom.ops.read(hw, NVM_ETK_OFF_LOW, &etk_id_l))
+		etk_id_l = NVM_VER_INVALID;
+	if (hw->eeprom.ops.read(hw, NVM_ETK_OFF_HI, &etk_id_h))
+		etk_id_h = NVM_VER_INVALID;
+
+	/* The word order for the version format is determined by high order
+	 * word bit 15.
+	 */
+	if ((etk_id_h & NVM_ETK_VALID) == 0) {
+		nvm_ver->etk_id = etk_id_h;
+		nvm_ver->etk_id |= (etk_id_l << NVM_ETK_SHIFT);
+	} else {
+		nvm_ver->etk_id = etk_id_l;
+		nvm_ver->etk_id |= (etk_id_h << NVM_ETK_SHIFT);
+	}
+}
+
 void ixgbe_disable_rx_generic(struct ixgbe_hw *hw)
 {
 	u32 rxctrl;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h
index a01409e2e06c..4d4c02366cb3 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.h
@@ -139,6 +139,12 @@ extern const u32 ixgbe_mvals_8259X[IXGBE_MVALS_IDX_LIMIT];
 
 s32 ixgbe_get_thermal_sensor_data_generic(struct ixgbe_hw *hw);
 s32 ixgbe_init_thermal_sensor_thresh_generic(struct ixgbe_hw *hw);
+void ixgbe_get_etk_id(struct ixgbe_hw *hw,
+		      struct ixgbe_nvm_version *nvm_ver);
+void ixgbe_get_oem_prod_version(struct ixgbe_hw *hw,
+				struct ixgbe_nvm_version *nvm_ver);
+void ixgbe_get_orom_version(struct ixgbe_hw *hw,
+			    struct ixgbe_nvm_version *nvm_ver);
 void ixgbe_disable_rx_generic(struct ixgbe_hw *hw);
 void ixgbe_enable_rx_generic(struct ixgbe_hw *hw);
 s32 ixgbe_setup_mac_link_multispeed_fiber(struct ixgbe_hw *hw,
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c
index 072ef3b5fc61..aaea8282bfd2 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb.c
@@ -39,6 +39,10 @@
  * are the smallest unit programmable into the underlying
  * hardware. The IEEE 802.1Qaz specification do not use bandwidth
  * groups so this is much simplified from the CEE case.
+ * @bw: bandwidth index by traffic class
+ * @refill: refill credits index by traffic class
+ * @max: max credits by traffic class
+ * @max_frame: maximum frame size
  */
 static s32 ixgbe_ieee_credits(__u8 *bw, __u16 *refill,
 			      __u16 *max, int max_frame)
@@ -72,8 +76,10 @@ static s32 ixgbe_ieee_credits(__u8 *bw, __u16 *refill,
 
 /**
  * ixgbe_dcb_calculate_tc_credits - Calculates traffic class credits
- * @ixgbe_dcb_config: Struct containing DCB settings.
- * @direction: Configuring either Tx or Rx.
+ * @hw: pointer to hardware structure
+ * @dcb_config: Struct containing DCB settings
+ * @max_frame: Maximum frame size
+ * @direction: Configuring either Tx or Rx
  *
  * This function calculates the credits allocated to each traffic class.
  * It should be called only after the rules are checked by
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c
index b79e93a5b699..f94c7e82a30b 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82598.c
@@ -34,7 +34,9 @@
 /**
  * ixgbe_dcb_config_rx_arbiter_82598 - Config Rx data arbiter
  * @hw: pointer to hardware structure
- * @dcb_config: pointer to ixgbe_dcb_config structure
+ * @refill: refill credits index by traffic class
+ * @max: max credits index by traffic class
+ * @prio_type: priority type indexed by traffic class
  *
  * Configure Rx Data Arbiter and credits for each traffic class.
  */
@@ -91,7 +93,10 @@ s32 ixgbe_dcb_config_rx_arbiter_82598(struct ixgbe_hw *hw,
 /**
  * ixgbe_dcb_config_tx_desc_arbiter_82598 - Config Tx Desc. arbiter
  * @hw: pointer to hardware structure
- * @dcb_config: pointer to ixgbe_dcb_config structure
+ * @refill: refill credits index by traffic class
+ * @max: max credits index by traffic class
+ * @bwg_id: bandwidth grouping indexed by traffic class
+ * @prio_type: priority type indexed by traffic class
  *
  * Configure Tx Descriptor Arbiter and credits for each traffic class.
  */
@@ -137,7 +142,10 @@ s32 ixgbe_dcb_config_tx_desc_arbiter_82598(struct ixgbe_hw *hw,
 /**
  * ixgbe_dcb_config_tx_data_arbiter_82598 - Config Tx data arbiter
  * @hw: pointer to hardware structure
- * @dcb_config: pointer to ixgbe_dcb_config structure
+ * @refill: refill credits index by traffic class
+ * @max: max credits index by traffic class
+ * @bwg_id: bandwidth grouping indexed by traffic class
+ * @prio_type: priority type indexed by traffic class
  *
  * Configure Tx Data Arbiter and credits for each traffic class.
  */
@@ -184,7 +192,7 @@ s32 ixgbe_dcb_config_tx_data_arbiter_82598(struct ixgbe_hw *hw,
 /**
  * ixgbe_dcb_config_pfc_82598 - Config priority flow control
  * @hw: pointer to hardware structure
- * @dcb_config: pointer to ixgbe_dcb_config structure
+ * @pfc_en: enabled pfc bitmask
  *
  * Configure Priority Flow Control for each traffic class.
  */
@@ -269,7 +277,11 @@ static s32 ixgbe_dcb_config_tc_stats_82598(struct ixgbe_hw *hw)
 /**
  * ixgbe_dcb_hw_config_82598 - Config and enable DCB
  * @hw: pointer to hardware structure
- * @dcb_config: pointer to ixgbe_dcb_config structure
+ * @pfc_en: enabled pfc bitmask
+ * @refill: refill credits index by traffic class
+ * @max: max credits index by traffic class
+ * @bwg_id: bandwidth grouping indexed by traffic class
+ * @prio_type: priority type indexed by traffic class
  *
  * Configure dcb settings and enable dcb mode.
  */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c
index 1011d644978f..1eed6811e914 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_82599.c
@@ -38,6 +38,7 @@
  * @max: max credits index by traffic class
  * @bwg_id: bandwidth grouping indexed by traffic class
  * @prio_type: priority type indexed by traffic class
+ * @prio_tc: priority to tc assignments indexed by priority
  *
  * Configure Rx Packet Arbiter and credits for each traffic class.
  */
@@ -148,6 +149,7 @@ s32 ixgbe_dcb_config_tx_desc_arbiter_82599(struct ixgbe_hw *hw,
  * @max: max credits index by traffic class
  * @bwg_id: bandwidth grouping indexed by traffic class
  * @prio_type: priority type indexed by traffic class
+ * @prio_tc: priority to tc assignments indexed by priority
  *
  * Configure Tx Packet Arbiter and credits for each traffic class.
  */
@@ -344,11 +346,12 @@ static s32 ixgbe_dcb_config_tc_stats_82599(struct ixgbe_hw *hw)
 /**
  * ixgbe_dcb_hw_config_82599 - Configure and enable DCB
  * @hw: pointer to hardware structure
+ * @pfc_en: enabled pfc bitmask
  * @refill: refill credits index by traffic class
  * @max: max credits index by traffic class
  * @bwg_id: bandwidth grouping indexed by traffic class
  * @prio_type: priority type indexed by traffic class
- * @pfc_en: enabled pfc bitmask
+ * @prio_tc: priority to tc assignments indexed by priority
  *
  * Configure dcb settings and enable dcb mode.
  */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c
index 78c52375acc6..b33f3f87e4b1 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c
@@ -571,7 +571,7 @@ static int ixgbe_dcbnl_ieee_setets(struct net_device *dev,
 	if (max_tc > adapter->dcb_cfg.num_tcs.pg_tcs)
 		return -EINVAL;
 
-	if (max_tc != netdev_get_num_tc(dev)) {
+	if (max_tc != adapter->hw_tcs) {
 		err = ixgbe_setup_tc(dev, max_tc);
 		if (err)
 			return err;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c
index 5e2c1e35e517..ad54080488ee 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c
@@ -249,7 +249,7 @@ void ixgbe_dbg_adapter_init(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_dbg_adapter_exit - clear out the adapter's debugfs entries
- * @pf: the pf that is stopping
+ * @adapter: the adapter that is exiting
  **/
 void ixgbe_dbg_adapter_exit(struct ixgbe_adapter *adapter)
 {
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
index 0aad1c2a3667..221f15803480 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
@@ -115,6 +115,8 @@ static const struct ixgbe_stats ixgbe_gstrings_stats[] = {
 	{"tx_hwtstamp_timeouts", IXGBE_STAT(tx_hwtstamp_timeouts)},
 	{"tx_hwtstamp_skipped", IXGBE_STAT(tx_hwtstamp_skipped)},
 	{"rx_hwtstamp_cleared", IXGBE_STAT(rx_hwtstamp_cleared)},
+	{"tx_ipsec", IXGBE_STAT(tx_ipsec)},
+	{"rx_ipsec", IXGBE_STAT(rx_ipsec)},
 #ifdef IXGBE_FCOE
 	{"fcoe_bad_fccrc", IXGBE_STAT(stats.fccrc)},
 	{"rx_fcoe_dropped", IXGBE_STAT(stats.fcoerpdc)},
@@ -1014,16 +1016,13 @@ static void ixgbe_get_drvinfo(struct net_device *netdev,
 			      struct ethtool_drvinfo *drvinfo)
 {
 	struct ixgbe_adapter *adapter = netdev_priv(netdev);
-	u32 nvm_track_id;
 
 	strlcpy(drvinfo->driver, ixgbe_driver_name, sizeof(drvinfo->driver));
 	strlcpy(drvinfo->version, ixgbe_driver_version,
 		sizeof(drvinfo->version));
 
-	nvm_track_id = (adapter->eeprom_verh << 16) |
-			adapter->eeprom_verl;
-	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "0x%08x",
-		 nvm_track_id);
+	strlcpy(drvinfo->fw_version, adapter->eeprom_id,
+		sizeof(drvinfo->fw_version));
 
 	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
@@ -1156,6 +1155,10 @@ static int ixgbe_set_ringparam(struct net_device *netdev,
 			memcpy(&temp_ring[i], adapter->rx_ring[i],
 			       sizeof(struct ixgbe_ring));
 
+			/* Clear copied XDP RX-queue info */
+			memset(&temp_ring[i].xdp_rxq, 0,
+			       sizeof(temp_ring[i].xdp_rxq));
+
 			temp_ring[i].count = new_rx_count;
 			err = ixgbe_setup_rx_resources(adapter, &temp_ring[i]);
 			if (err) {
@@ -3082,26 +3085,9 @@ static int ixgbe_get_ts_info(struct net_device *dev,
 	case ixgbe_mac_X550EM_x:
 	case ixgbe_mac_x550em_a:
 		info->rx_filters |= BIT(HWTSTAMP_FILTER_ALL);
-		/* fallthrough */
+		break;
 	case ixgbe_mac_X540:
 	case ixgbe_mac_82599EB:
-		info->so_timestamping =
-			SOF_TIMESTAMPING_TX_SOFTWARE |
-			SOF_TIMESTAMPING_RX_SOFTWARE |
-			SOF_TIMESTAMPING_SOFTWARE |
-			SOF_TIMESTAMPING_TX_HARDWARE |
-			SOF_TIMESTAMPING_RX_HARDWARE |
-			SOF_TIMESTAMPING_RAW_HARDWARE;
-
-		if (adapter->ptp_clock)
-			info->phc_index = ptp_clock_index(adapter->ptp_clock);
-		else
-			info->phc_index = -1;
-
-		info->tx_types =
-			BIT(HWTSTAMP_TX_OFF) |
-			BIT(HWTSTAMP_TX_ON);
-
 		info->rx_filters |=
 			BIT(HWTSTAMP_FILTER_PTP_V1_L4_SYNC) |
 			BIT(HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ) |
@@ -3110,13 +3096,31 @@ static int ixgbe_get_ts_info(struct net_device *dev,
 	default:
 		return ethtool_op_get_ts_info(dev, info);
 	}
+
+	info->so_timestamping =
+		SOF_TIMESTAMPING_TX_SOFTWARE |
+		SOF_TIMESTAMPING_RX_SOFTWARE |
+		SOF_TIMESTAMPING_SOFTWARE |
+		SOF_TIMESTAMPING_TX_HARDWARE |
+		SOF_TIMESTAMPING_RX_HARDWARE |
+		SOF_TIMESTAMPING_RAW_HARDWARE;
+
+	if (adapter->ptp_clock)
+		info->phc_index = ptp_clock_index(adapter->ptp_clock);
+	else
+		info->phc_index = -1;
+
+	info->tx_types =
+		BIT(HWTSTAMP_TX_OFF) |
+		BIT(HWTSTAMP_TX_ON);
+
 	return 0;
 }
 
 static unsigned int ixgbe_max_channels(struct ixgbe_adapter *adapter)
 {
 	unsigned int max_combined;
-	u8 tcs = netdev_get_num_tc(adapter->netdev);
+	u8 tcs = adapter->hw_tcs;
 
 	if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
 		/* We only support one q_vector without MSI-X */
@@ -3173,7 +3177,7 @@ static void ixgbe_get_channels(struct net_device *dev,
 		return;
 
 	/* same thing goes for being DCB enabled */
-	if (netdev_get_num_tc(dev) > 1)
+	if (adapter->hw_tcs > 1)
 		return;
 
 	/* if ATR is disabled we can exit */
@@ -3219,7 +3223,7 @@ static int ixgbe_set_channels(struct net_device *dev,
 
 #endif
 	/* use setup TC to update any traffic class queue mapping */
-	return ixgbe_setup_tc(dev, netdev_get_num_tc(dev));
+	return ixgbe_setup_tc(dev, adapter->hw_tcs);
 }
 
 static int ixgbe_get_module_info(struct net_device *dev,
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
index a23c2b5411a0..7a09a40e4472 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
@@ -150,6 +150,7 @@ int ixgbe_fcoe_ddp_put(struct net_device *netdev, u16 xid)
  * @xid: the exchange id requesting ddp
  * @sgl: the scatter-gather list for this request
  * @sgc: the number of scatter-gather items
+ * @target_mode: 1 to setup target mode, 0 to setup initiator mode
  *
  * Returns : 1 for success and 0 for no ddp
  */
@@ -1034,11 +1035,8 @@ int ixgbe_fcoe_get_hbainfo(struct net_device *netdev,
 		 ixgbe_driver_name,
 		 ixgbe_driver_version);
 	/* Firmware Version */
-	snprintf(info->firmware_version,
-		 sizeof(info->firmware_version),
-		 "0x%08x",
-		 (adapter->eeprom_verh << 16) |
-		  adapter->eeprom_verl);
+	strlcpy(info->firmware_version, adapter->eeprom_id,
+		sizeof(info->firmware_version));
 
 	/* Model */
 	if (hw->mac.type == ixgbe_mac_82599EB) {
@@ -1066,7 +1064,7 @@ int ixgbe_fcoe_get_hbainfo(struct net_device *netdev,
 
 /**
  * ixgbe_fcoe_get_tc - get the current TC that fcoe is mapped to
- * @adapter - pointer to the device adapter structure
+ * @adapter: pointer to the device adapter structure
  *
  * Return : TC that FCoE is mapped to
  */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
new file mode 100644
index 000000000000..93eacddb6704
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c
@@ -0,0 +1,941 @@
+/*******************************************************************************
+ *
+ * Intel 10 Gigabit PCI Express Linux driver
+ * Copyright(c) 2017 Oracle and/or its affiliates. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * Contact Information:
+ * Linux NICS <linux.nics@intel.com>
+ * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ ******************************************************************************/
+
+#include "ixgbe.h"
+#include <net/xfrm.h>
+#include <crypto/aead.h>
+
+/**
+ * ixgbe_ipsec_set_tx_sa - set the Tx SA registers
+ * @hw: hw specific details
+ * @idx: register index to write
+ * @key: key byte array
+ * @salt: salt bytes
+ **/
+static void ixgbe_ipsec_set_tx_sa(struct ixgbe_hw *hw, u16 idx,
+				  u32 key[], u32 salt)
+{
+	u32 reg;
+	int i;
+
+	for (i = 0; i < 4; i++)
+		IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(i), cpu_to_be32(key[3 - i]));
+	IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, cpu_to_be32(salt));
+	IXGBE_WRITE_FLUSH(hw);
+
+	reg = IXGBE_READ_REG(hw, IXGBE_IPSTXIDX);
+	reg &= IXGBE_RXTXIDX_IPS_EN;
+	reg |= idx << IXGBE_RXTXIDX_IDX_SHIFT | IXGBE_RXTXIDX_WRITE;
+	IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, reg);
+	IXGBE_WRITE_FLUSH(hw);
+}
+
+/**
+ * ixgbe_ipsec_set_rx_item - set an Rx table item
+ * @hw: hw specific details
+ * @idx: register index to write
+ * @tbl: table selector
+ *
+ * Trigger the device to store into a particular Rx table the
+ * data that has already been loaded into the input register
+ **/
+static void ixgbe_ipsec_set_rx_item(struct ixgbe_hw *hw, u16 idx,
+				    enum ixgbe_ipsec_tbl_sel tbl)
+{
+	u32 reg;
+
+	reg = IXGBE_READ_REG(hw, IXGBE_IPSRXIDX);
+	reg &= IXGBE_RXTXIDX_IPS_EN;
+	reg |= tbl << IXGBE_RXIDX_TBL_SHIFT |
+	       idx << IXGBE_RXTXIDX_IDX_SHIFT |
+	       IXGBE_RXTXIDX_WRITE;
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, reg);
+	IXGBE_WRITE_FLUSH(hw);
+}
+
+/**
+ * ixgbe_ipsec_set_rx_sa - set up the register bits to save SA info
+ * @hw: hw specific details
+ * @idx: register index to write
+ * @spi: security parameter index
+ * @key: key byte array
+ * @salt: salt bytes
+ * @mode: rx decrypt control bits
+ * @ip_idx: index into IP table for related IP address
+ **/
+static void ixgbe_ipsec_set_rx_sa(struct ixgbe_hw *hw, u16 idx, __be32 spi,
+				  u32 key[], u32 salt, u32 mode, u32 ip_idx)
+{
+	int i;
+
+	/* store the SPI (in bigendian) and IPidx */
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, cpu_to_le32(spi));
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, ip_idx);
+	IXGBE_WRITE_FLUSH(hw);
+
+	ixgbe_ipsec_set_rx_item(hw, idx, ips_rx_spi_tbl);
+
+	/* store the key, salt, and mode */
+	for (i = 0; i < 4; i++)
+		IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(i), cpu_to_be32(key[3 - i]));
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, cpu_to_be32(salt));
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, mode);
+	IXGBE_WRITE_FLUSH(hw);
+
+	ixgbe_ipsec_set_rx_item(hw, idx, ips_rx_key_tbl);
+}
+
+/**
+ * ixgbe_ipsec_set_rx_ip - set up the register bits to save SA IP addr info
+ * @hw: hw specific details
+ * @idx: register index to write
+ * @addr: IP address byte array
+ **/
+static void ixgbe_ipsec_set_rx_ip(struct ixgbe_hw *hw, u16 idx, __be32 addr[])
+{
+	int i;
+
+	/* store the ip address */
+	for (i = 0; i < 4; i++)
+		IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(i), cpu_to_le32(addr[i]));
+	IXGBE_WRITE_FLUSH(hw);
+
+	ixgbe_ipsec_set_rx_item(hw, idx, ips_rx_ip_tbl);
+}
+
+/**
+ * ixgbe_ipsec_clear_hw_tables - because some tables don't get cleared on reset
+ * @adapter: board private structure
+ **/
+static void ixgbe_ipsec_clear_hw_tables(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_ipsec *ipsec = adapter->ipsec;
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32 buf[4] = {0, 0, 0, 0};
+	u16 idx;
+
+	/* disable Rx and Tx SA lookup */
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, 0);
+	IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, 0);
+
+	/* scrub the tables - split the loops for the max of the IP table */
+	for (idx = 0; idx < IXGBE_IPSEC_MAX_RX_IP_COUNT; idx++) {
+		ixgbe_ipsec_set_tx_sa(hw, idx, buf, 0);
+		ixgbe_ipsec_set_rx_sa(hw, idx, 0, buf, 0, 0, 0);
+		ixgbe_ipsec_set_rx_ip(hw, idx, (__be32 *)buf);
+	}
+	for (; idx < IXGBE_IPSEC_MAX_SA_COUNT; idx++) {
+		ixgbe_ipsec_set_tx_sa(hw, idx, buf, 0);
+		ixgbe_ipsec_set_rx_sa(hw, idx, 0, buf, 0, 0, 0);
+	}
+
+	ipsec->num_rx_sa = 0;
+	ipsec->num_tx_sa = 0;
+}
+
+/**
+ * ixgbe_ipsec_stop_data
+ * @adapter: board private structure
+ **/
+static void ixgbe_ipsec_stop_data(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_hw *hw = &adapter->hw;
+	bool link = adapter->link_up;
+	u32 t_rdy, r_rdy;
+	u32 limit;
+	u32 reg;
+
+	/* halt data paths */
+	reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL);
+	reg |= IXGBE_SECTXCTRL_TX_DIS;
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, reg);
+
+	reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
+	reg |= IXGBE_SECRXCTRL_RX_DIS;
+	IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg);
+
+	IXGBE_WRITE_FLUSH(hw);
+
+	/* If the tx fifo doesn't have link, but still has data,
+	 * we can't clear the tx sec block.  Set the MAC loopback
+	 * before block clear
+	 */
+	if (!link) {
+		reg = IXGBE_READ_REG(hw, IXGBE_MACC);
+		reg |= IXGBE_MACC_FLU;
+		IXGBE_WRITE_REG(hw, IXGBE_MACC, reg);
+
+		reg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
+		reg |= IXGBE_HLREG0_LPBK;
+		IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg);
+
+		IXGBE_WRITE_FLUSH(hw);
+		mdelay(3);
+	}
+
+	/* wait for the paths to empty */
+	limit = 20;
+	do {
+		mdelay(10);
+		t_rdy = IXGBE_READ_REG(hw, IXGBE_SECTXSTAT) &
+			IXGBE_SECTXSTAT_SECTX_RDY;
+		r_rdy = IXGBE_READ_REG(hw, IXGBE_SECRXSTAT) &
+			IXGBE_SECRXSTAT_SECRX_RDY;
+	} while (!t_rdy && !r_rdy && limit--);
+
+	/* undo loopback if we played with it earlier */
+	if (!link) {
+		reg = IXGBE_READ_REG(hw, IXGBE_MACC);
+		reg &= ~IXGBE_MACC_FLU;
+		IXGBE_WRITE_REG(hw, IXGBE_MACC, reg);
+
+		reg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
+		reg &= ~IXGBE_HLREG0_LPBK;
+		IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg);
+
+		IXGBE_WRITE_FLUSH(hw);
+	}
+}
+
+/**
+ * ixgbe_ipsec_stop_engine
+ * @adapter: board private structure
+ **/
+static void ixgbe_ipsec_stop_engine(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32 reg;
+
+	ixgbe_ipsec_stop_data(adapter);
+
+	/* disable Rx and Tx SA lookup */
+	IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, 0);
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, 0);
+
+	/* disable the Rx and Tx engines and full packet store-n-forward */
+	reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL);
+	reg |= IXGBE_SECTXCTRL_SECTX_DIS;
+	reg &= ~IXGBE_SECTXCTRL_STORE_FORWARD;
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, reg);
+
+	reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
+	reg |= IXGBE_SECRXCTRL_SECRX_DIS;
+	IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg);
+
+	/* restore the "tx security buffer almost full threshold" to 0x250 */
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x250);
+
+	/* Set minimum IFG between packets back to the default 0x1 */
+	reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
+	reg = (reg & 0xfffffff0) | 0x1;
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
+
+	/* final set for normal (no ipsec offload) processing */
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, IXGBE_SECTXCTRL_SECTX_DIS);
+	IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, IXGBE_SECRXCTRL_SECRX_DIS);
+
+	IXGBE_WRITE_FLUSH(hw);
+}
+
+/**
+ * ixgbe_ipsec_start_engine
+ * @adapter: board private structure
+ *
+ * NOTE: this increases power consumption whether being used or not
+ **/
+static void ixgbe_ipsec_start_engine(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32 reg;
+
+	ixgbe_ipsec_stop_data(adapter);
+
+	/* Set minimum IFG between packets to 3 */
+	reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
+	reg = (reg & 0xfffffff0) | 0x3;
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
+
+	/* Set "tx security buffer almost full threshold" to 0x15 so that the
+	 * almost full indication is generated only after buffer contains at
+	 * least an entire jumbo packet.
+	 */
+	reg = IXGBE_READ_REG(hw, IXGBE_SECTXBUFFAF);
+	reg = (reg & 0xfffffc00) | 0x15;
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, reg);
+
+	/* restart the data paths by clearing the DISABLE bits */
+	IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
+	IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL, IXGBE_SECTXCTRL_STORE_FORWARD);
+
+	/* enable Rx and Tx SA lookup */
+	IXGBE_WRITE_REG(hw, IXGBE_IPSTXIDX, IXGBE_RXTXIDX_IPS_EN);
+	IXGBE_WRITE_REG(hw, IXGBE_IPSRXIDX, IXGBE_RXTXIDX_IPS_EN);
+
+	IXGBE_WRITE_FLUSH(hw);
+}
+
+/**
+ * ixgbe_ipsec_restore - restore the ipsec HW settings after a reset
+ * @adapter: board private structure
+ **/
+void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_ipsec *ipsec = adapter->ipsec;
+	struct ixgbe_hw *hw = &adapter->hw;
+	int i;
+
+	if (!(adapter->flags2 & IXGBE_FLAG2_IPSEC_ENABLED))
+		return;
+
+	/* clean up and restart the engine */
+	ixgbe_ipsec_stop_engine(adapter);
+	ixgbe_ipsec_clear_hw_tables(adapter);
+	ixgbe_ipsec_start_engine(adapter);
+
+	/* reload the IP addrs */
+	for (i = 0; i < IXGBE_IPSEC_MAX_RX_IP_COUNT; i++) {
+		struct rx_ip_sa *ipsa = &ipsec->ip_tbl[i];
+
+		if (ipsa->used)
+			ixgbe_ipsec_set_rx_ip(hw, i, ipsa->ipaddr);
+	}
+
+	/* reload the Rx and Tx keys */
+	for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT; i++) {
+		struct rx_sa *rsa = &ipsec->rx_tbl[i];
+		struct tx_sa *tsa = &ipsec->tx_tbl[i];
+
+		if (rsa->used)
+			ixgbe_ipsec_set_rx_sa(hw, i, rsa->xs->id.spi,
+					      rsa->key, rsa->salt,
+					      rsa->mode, rsa->iptbl_ind);
+
+		if (tsa->used)
+			ixgbe_ipsec_set_tx_sa(hw, i, tsa->key, tsa->salt);
+	}
+}
+
+/**
+ * ixgbe_ipsec_find_empty_idx - find the first unused security parameter index
+ * @ipsec: pointer to ipsec struct
+ * @rxtable: true if we need to look in the Rx table
+ *
+ * Returns the first unused index in either the Rx or Tx SA table
+ **/
+static int ixgbe_ipsec_find_empty_idx(struct ixgbe_ipsec *ipsec, bool rxtable)
+{
+	u32 i;
+
+	if (rxtable) {
+		if (ipsec->num_rx_sa == IXGBE_IPSEC_MAX_SA_COUNT)
+			return -ENOSPC;
+
+		/* search rx sa table */
+		for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT; i++) {
+			if (!ipsec->rx_tbl[i].used)
+				return i;
+		}
+	} else {
+		if (ipsec->num_tx_sa == IXGBE_IPSEC_MAX_SA_COUNT)
+			return -ENOSPC;
+
+		/* search tx sa table */
+		for (i = 0; i < IXGBE_IPSEC_MAX_SA_COUNT; i++) {
+			if (!ipsec->tx_tbl[i].used)
+				return i;
+		}
+	}
+
+	return -ENOSPC;
+}
+
+/**
+ * ixgbe_ipsec_find_rx_state - find the state that matches
+ * @ipsec: pointer to ipsec struct
+ * @daddr: inbound address to match
+ * @proto: protocol to match
+ * @spi: SPI to match
+ * @ip4: true if using an ipv4 address
+ *
+ * Returns a pointer to the matching SA state information
+ **/
+static struct xfrm_state *ixgbe_ipsec_find_rx_state(struct ixgbe_ipsec *ipsec,
+						    __be32 *daddr, u8 proto,
+						    __be32 spi, bool ip4)
+{
+	struct rx_sa *rsa;
+	struct xfrm_state *ret = NULL;
+
+	rcu_read_lock();
+	hash_for_each_possible_rcu(ipsec->rx_sa_list, rsa, hlist, spi)
+		if (spi == rsa->xs->id.spi &&
+		    ((ip4 && *daddr == rsa->xs->id.daddr.a4) ||
+		      (!ip4 && !memcmp(daddr, &rsa->xs->id.daddr.a6,
+				       sizeof(rsa->xs->id.daddr.a6)))) &&
+		    proto == rsa->xs->id.proto) {
+			ret = rsa->xs;
+			xfrm_state_hold(ret);
+			break;
+		}
+	rcu_read_unlock();
+	return ret;
+}
+
+/**
+ * ixgbe_ipsec_parse_proto_keys - find the key and salt based on the protocol
+ * @xs: pointer to xfrm_state struct
+ * @mykey: pointer to key array to populate
+ * @mysalt: pointer to salt value to populate
+ *
+ * This copies the protocol keys and salt to our own data tables.  The
+ * 82599 family only supports the one algorithm.
+ **/
+static int ixgbe_ipsec_parse_proto_keys(struct xfrm_state *xs,
+					u32 *mykey, u32 *mysalt)
+{
+	struct net_device *dev = xs->xso.dev;
+	unsigned char *key_data;
+	char *alg_name = NULL;
+	const char aes_gcm_name[] = "rfc4106(gcm(aes))";
+	int key_len;
+
+	if (xs->aead) {
+		key_data = &xs->aead->alg_key[0];
+		key_len = xs->aead->alg_key_len;
+		alg_name = xs->aead->alg_name;
+	} else {
+		netdev_err(dev, "Unsupported IPsec algorithm\n");
+		return -EINVAL;
+	}
+
+	if (strcmp(alg_name, aes_gcm_name)) {
+		netdev_err(dev, "Unsupported IPsec algorithm - please use %s\n",
+			   aes_gcm_name);
+		return -EINVAL;
+	}
+
+	/* The key bytes come down in a bigendian array of bytes, so
+	 * we don't need to do any byteswapping.
+	 * 160 accounts for 16 byte key and 4 byte salt
+	 */
+	if (key_len == 160) {
+		*mysalt = ((u32 *)key_data)[4];
+	} else if (key_len != 128) {
+		netdev_err(dev, "IPsec hw offload only supports keys up to 128 bits with a 32 bit salt\n");
+		return -EINVAL;
+	} else {
+		netdev_info(dev, "IPsec hw offload parameters missing 32 bit salt value\n");
+		*mysalt = 0;
+	}
+	memcpy(mykey, key_data, 16);
+
+	return 0;
+}
+
+/**
+ * ixgbe_ipsec_add_sa - program device with a security association
+ * @xs: pointer to transformer state struct
+ **/
+static int ixgbe_ipsec_add_sa(struct xfrm_state *xs)
+{
+	struct net_device *dev = xs->xso.dev;
+	struct ixgbe_adapter *adapter = netdev_priv(dev);
+	struct ixgbe_ipsec *ipsec = adapter->ipsec;
+	struct ixgbe_hw *hw = &adapter->hw;
+	int checked, match, first;
+	u16 sa_idx;
+	int ret;
+	int i;
+
+	if (xs->id.proto != IPPROTO_ESP && xs->id.proto != IPPROTO_AH) {
+		netdev_err(dev, "Unsupported protocol 0x%04x for ipsec offload\n",
+			   xs->id.proto);
+		return -EINVAL;
+	}
+
+	if (xs->xso.flags & XFRM_OFFLOAD_INBOUND) {
+		struct rx_sa rsa;
+
+		if (xs->calg) {
+			netdev_err(dev, "Compression offload not supported\n");
+			return -EINVAL;
+		}
+
+		/* find the first unused index */
+		ret = ixgbe_ipsec_find_empty_idx(ipsec, true);
+		if (ret < 0) {
+			netdev_err(dev, "No space for SA in Rx table!\n");
+			return ret;
+		}
+		sa_idx = (u16)ret;
+
+		memset(&rsa, 0, sizeof(rsa));
+		rsa.used = true;
+		rsa.xs = xs;
+
+		if (rsa.xs->id.proto & IPPROTO_ESP)
+			rsa.decrypt = xs->ealg || xs->aead;
+
+		/* get the key and salt */
+		ret = ixgbe_ipsec_parse_proto_keys(xs, rsa.key, &rsa.salt);
+		if (ret) {
+			netdev_err(dev, "Failed to get key data for Rx SA table\n");
+			return ret;
+		}
+
+		/* get ip for rx sa table */
+		if (xs->props.family == AF_INET6)
+			memcpy(rsa.ipaddr, &xs->id.daddr.a6, 16);
+		else
+			memcpy(&rsa.ipaddr[3], &xs->id.daddr.a4, 4);
+
+		/* The HW does not have a 1:1 mapping from keys to IP addrs, so
+		 * check for a matching IP addr entry in the table.  If the addr
+		 * already exists, use it; else find an unused slot and add the
+		 * addr.  If one does not exist and there are no unused table
+		 * entries, fail the request.
+		 */
+
+		/* Find an existing match or first not used, and stop looking
+		 * after we've checked all we know we have.
+		 */
+		checked = 0;
+		match = -1;
+		first = -1;
+		for (i = 0;
+		     i < IXGBE_IPSEC_MAX_RX_IP_COUNT &&
+		     (checked < ipsec->num_rx_sa || first < 0);
+		     i++) {
+			if (ipsec->ip_tbl[i].used) {
+				if (!memcmp(ipsec->ip_tbl[i].ipaddr,
+					    rsa.ipaddr, sizeof(rsa.ipaddr))) {
+					match = i;
+					break;
+				}
+				checked++;
+			} else if (first < 0) {
+				first = i;  /* track the first empty seen */
+			}
+		}
+
+		if (ipsec->num_rx_sa == 0)
+			first = 0;
+
+		if (match >= 0) {
+			/* addrs are the same, we should use this one */
+			rsa.iptbl_ind = match;
+			ipsec->ip_tbl[match].ref_cnt++;
+
+		} else if (first >= 0) {
+			/* no matches, but here's an empty slot */
+			rsa.iptbl_ind = first;
+
+			memcpy(ipsec->ip_tbl[first].ipaddr,
+			       rsa.ipaddr, sizeof(rsa.ipaddr));
+			ipsec->ip_tbl[first].ref_cnt = 1;
+			ipsec->ip_tbl[first].used = true;
+
+			ixgbe_ipsec_set_rx_ip(hw, rsa.iptbl_ind, rsa.ipaddr);
+
+		} else {
+			/* no match and no empty slot */
+			netdev_err(dev, "No space for SA in Rx IP SA table\n");
+			memset(&rsa, 0, sizeof(rsa));
+			return -ENOSPC;
+		}
+
+		rsa.mode = IXGBE_RXMOD_VALID;
+		if (rsa.xs->id.proto & IPPROTO_ESP)
+			rsa.mode |= IXGBE_RXMOD_PROTO_ESP;
+		if (rsa.decrypt)
+			rsa.mode |= IXGBE_RXMOD_DECRYPT;
+		if (rsa.xs->props.family == AF_INET6)
+			rsa.mode |= IXGBE_RXMOD_IPV6;
+
+		/* the preparations worked, so save the info */
+		memcpy(&ipsec->rx_tbl[sa_idx], &rsa, sizeof(rsa));
+
+		ixgbe_ipsec_set_rx_sa(hw, sa_idx, rsa.xs->id.spi, rsa.key,
+				      rsa.salt, rsa.mode, rsa.iptbl_ind);
+		xs->xso.offload_handle = sa_idx + IXGBE_IPSEC_BASE_RX_INDEX;
+
+		ipsec->num_rx_sa++;
+
+		/* hash the new entry for faster search in Rx path */
+		hash_add_rcu(ipsec->rx_sa_list, &ipsec->rx_tbl[sa_idx].hlist,
+			     rsa.xs->id.spi);
+	} else {
+		struct tx_sa tsa;
+
+		/* find the first unused index */
+		ret = ixgbe_ipsec_find_empty_idx(ipsec, false);
+		if (ret < 0) {
+			netdev_err(dev, "No space for SA in Tx table\n");
+			return ret;
+		}
+		sa_idx = (u16)ret;
+
+		memset(&tsa, 0, sizeof(tsa));
+		tsa.used = true;
+		tsa.xs = xs;
+
+		if (xs->id.proto & IPPROTO_ESP)
+			tsa.encrypt = xs->ealg || xs->aead;
+
+		ret = ixgbe_ipsec_parse_proto_keys(xs, tsa.key, &tsa.salt);
+		if (ret) {
+			netdev_err(dev, "Failed to get key data for Tx SA table\n");
+			memset(&tsa, 0, sizeof(tsa));
+			return ret;
+		}
+
+		/* the preparations worked, so save the info */
+		memcpy(&ipsec->tx_tbl[sa_idx], &tsa, sizeof(tsa));
+
+		ixgbe_ipsec_set_tx_sa(hw, sa_idx, tsa.key, tsa.salt);
+
+		xs->xso.offload_handle = sa_idx + IXGBE_IPSEC_BASE_TX_INDEX;
+
+		ipsec->num_tx_sa++;
+	}
+
+	/* enable the engine if not already warmed up */
+	if (!(adapter->flags2 & IXGBE_FLAG2_IPSEC_ENABLED)) {
+		ixgbe_ipsec_start_engine(adapter);
+		adapter->flags2 |= IXGBE_FLAG2_IPSEC_ENABLED;
+	}
+
+	return 0;
+}
+
+/**
+ * ixgbe_ipsec_del_sa - clear out this specific SA
+ * @xs: pointer to transformer state struct
+ **/
+static void ixgbe_ipsec_del_sa(struct xfrm_state *xs)
+{
+	struct net_device *dev = xs->xso.dev;
+	struct ixgbe_adapter *adapter = netdev_priv(dev);
+	struct ixgbe_ipsec *ipsec = adapter->ipsec;
+	struct ixgbe_hw *hw = &adapter->hw;
+	u32 zerobuf[4] = {0, 0, 0, 0};
+	u16 sa_idx;
+
+	if (xs->xso.flags & XFRM_OFFLOAD_INBOUND) {
+		struct rx_sa *rsa;
+		u8 ipi;
+
+		sa_idx = xs->xso.offload_handle - IXGBE_IPSEC_BASE_RX_INDEX;
+		rsa = &ipsec->rx_tbl[sa_idx];
+
+		if (!rsa->used) {
+			netdev_err(dev, "Invalid Rx SA selected sa_idx=%d offload_handle=%lu\n",
+				   sa_idx, xs->xso.offload_handle);
+			return;
+		}
+
+		ixgbe_ipsec_set_rx_sa(hw, sa_idx, 0, zerobuf, 0, 0, 0);
+		hash_del_rcu(&rsa->hlist);
+
+		/* if the IP table entry is referenced by only this SA,
+		 * i.e. ref_cnt is only 1, clear the IP table entry as well
+		 */
+		ipi = rsa->iptbl_ind;
+		if (ipsec->ip_tbl[ipi].ref_cnt > 0) {
+			ipsec->ip_tbl[ipi].ref_cnt--;
+
+			if (!ipsec->ip_tbl[ipi].ref_cnt) {
+				memset(&ipsec->ip_tbl[ipi], 0,
+				       sizeof(struct rx_ip_sa));
+				ixgbe_ipsec_set_rx_ip(hw, ipi, zerobuf);
+			}
+		}
+
+		memset(rsa, 0, sizeof(struct rx_sa));
+		ipsec->num_rx_sa--;
+	} else {
+		sa_idx = xs->xso.offload_handle - IXGBE_IPSEC_BASE_TX_INDEX;
+
+		if (!ipsec->tx_tbl[sa_idx].used) {
+			netdev_err(dev, "Invalid Tx SA selected sa_idx=%d offload_handle=%lu\n",
+				   sa_idx, xs->xso.offload_handle);
+			return;
+		}
+
+		ixgbe_ipsec_set_tx_sa(hw, sa_idx, zerobuf, 0);
+		memset(&ipsec->tx_tbl[sa_idx], 0, sizeof(struct tx_sa));
+		ipsec->num_tx_sa--;
+	}
+
+	/* if there are no SAs left, stop the engine to save energy */
+	if (ipsec->num_rx_sa == 0 && ipsec->num_tx_sa == 0) {
+		adapter->flags2 &= ~IXGBE_FLAG2_IPSEC_ENABLED;
+		ixgbe_ipsec_stop_engine(adapter);
+	}
+}
+
+/**
+ * ixgbe_ipsec_offload_ok - can this packet use the xfrm hw offload
+ * @skb: current data packet
+ * @xs: pointer to transformer state struct
+ **/
+static bool ixgbe_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *xs)
+{
+	if (xs->props.family == AF_INET) {
+		/* Offload with IPv4 options is not supported yet */
+		if (ip_hdr(skb)->ihl != 5)
+			return false;
+	} else {
+		/* Offload with IPv6 extension headers is not support yet */
+		if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
+			return false;
+	}
+
+	return true;
+}
+
+/**
+ * ixgbe_ipsec_free - called by xfrm garbage collections
+ * @xs: pointer to transformer state struct
+ *
+ * We don't have any garbage to collect, so we shouldn't bother
+ * implementing this function, but the XFRM code doesn't check for
+ * existence before calling the API callback.
+ **/
+static void ixgbe_ipsec_free(struct xfrm_state *xs)
+{
+}
+
+static const struct xfrmdev_ops ixgbe_xfrmdev_ops = {
+	.xdo_dev_state_add = ixgbe_ipsec_add_sa,
+	.xdo_dev_state_delete = ixgbe_ipsec_del_sa,
+	.xdo_dev_offload_ok = ixgbe_ipsec_offload_ok,
+	.xdo_dev_state_free = ixgbe_ipsec_free,
+};
+
+/**
+ * ixgbe_ipsec_tx - setup Tx flags for ipsec offload
+ * @tx_ring: outgoing context
+ * @first: current data packet
+ * @itd: ipsec Tx data for later use in building context descriptor
+ **/
+int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring,
+		   struct ixgbe_tx_buffer *first,
+		   struct ixgbe_ipsec_tx_data *itd)
+{
+	struct ixgbe_adapter *adapter = netdev_priv(tx_ring->netdev);
+	struct ixgbe_ipsec *ipsec = adapter->ipsec;
+	struct xfrm_state *xs;
+	struct tx_sa *tsa;
+
+	if (unlikely(!first->skb->sp->len)) {
+		netdev_err(tx_ring->netdev, "%s: no xfrm state len = %d\n",
+			   __func__, first->skb->sp->len);
+		return 0;
+	}
+
+	xs = xfrm_input_state(first->skb);
+	if (unlikely(!xs)) {
+		netdev_err(tx_ring->netdev, "%s: no xfrm_input_state() xs = %p\n",
+			   __func__, xs);
+		return 0;
+	}
+
+	itd->sa_idx = xs->xso.offload_handle - IXGBE_IPSEC_BASE_TX_INDEX;
+	if (unlikely(itd->sa_idx > IXGBE_IPSEC_MAX_SA_COUNT)) {
+		netdev_err(tx_ring->netdev, "%s: bad sa_idx=%d handle=%lu\n",
+			   __func__, itd->sa_idx, xs->xso.offload_handle);
+		return 0;
+	}
+
+	tsa = &ipsec->tx_tbl[itd->sa_idx];
+	if (unlikely(!tsa->used)) {
+		netdev_err(tx_ring->netdev, "%s: unused sa_idx=%d\n",
+			   __func__, itd->sa_idx);
+		return 0;
+	}
+
+	first->tx_flags |= IXGBE_TX_FLAGS_IPSEC | IXGBE_TX_FLAGS_CC;
+
+	itd->flags = 0;
+	if (xs->id.proto == IPPROTO_ESP) {
+		itd->flags |= IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP |
+			      IXGBE_ADVTXD_TUCMD_L4T_TCP;
+		if (first->protocol == htons(ETH_P_IP))
+			itd->flags |= IXGBE_ADVTXD_TUCMD_IPV4;
+		itd->trailer_len = xs->props.trailer_len;
+	}
+	if (tsa->encrypt)
+		itd->flags |= IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN;
+
+	return 1;
+}
+
+/**
+ * ixgbe_ipsec_rx - decode ipsec bits from Rx descriptor
+ * @rx_ring: receiving ring
+ * @rx_desc: receive data descriptor
+ * @skb: current data packet
+ *
+ * Determine if there was an ipsec encapsulation noticed, and if so set up
+ * the resulting status for later in the receive stack.
+ **/
+void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring,
+		    union ixgbe_adv_rx_desc *rx_desc,
+		    struct sk_buff *skb)
+{
+	struct ixgbe_adapter *adapter = netdev_priv(rx_ring->netdev);
+	__le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
+	__le16 ipsec_pkt_types = cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPSEC_AH |
+					     IXGBE_RXDADV_PKTTYPE_IPSEC_ESP);
+	struct ixgbe_ipsec *ipsec = adapter->ipsec;
+	struct xfrm_offload *xo = NULL;
+	struct xfrm_state *xs = NULL;
+	struct ipv6hdr *ip6 = NULL;
+	struct iphdr *ip4 = NULL;
+	void *daddr;
+	__be32 spi;
+	u8 *c_hdr;
+	u8 proto;
+
+	/* Find the ip and crypto headers in the data.
+	 * We can assume no vlan header in the way, b/c the
+	 * hw won't recognize the IPsec packet and anyway the
+	 * currently vlan device doesn't support xfrm offload.
+	 */
+	if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPV4)) {
+		ip4 = (struct iphdr *)(skb->data + ETH_HLEN);
+		daddr = &ip4->daddr;
+		c_hdr = (u8 *)ip4 + ip4->ihl * 4;
+	} else if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPV6)) {
+		ip6 = (struct ipv6hdr *)(skb->data + ETH_HLEN);
+		daddr = &ip6->daddr;
+		c_hdr = (u8 *)ip6 + sizeof(struct ipv6hdr);
+	} else {
+		return;
+	}
+
+	switch (pkt_info & ipsec_pkt_types) {
+	case cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPSEC_AH):
+		spi = ((struct ip_auth_hdr *)c_hdr)->spi;
+		proto = IPPROTO_AH;
+		break;
+	case cpu_to_le16(IXGBE_RXDADV_PKTTYPE_IPSEC_ESP):
+		spi = ((struct ip_esp_hdr *)c_hdr)->spi;
+		proto = IPPROTO_ESP;
+		break;
+	default:
+		return;
+	}
+
+	xs = ixgbe_ipsec_find_rx_state(ipsec, daddr, proto, spi, !!ip4);
+	if (unlikely(!xs))
+		return;
+
+	skb->sp = secpath_dup(skb->sp);
+	if (unlikely(!skb->sp))
+		return;
+
+	skb->sp->xvec[skb->sp->len++] = xs;
+	skb->sp->olen++;
+	xo = xfrm_offload(skb);
+	xo->flags = CRYPTO_DONE;
+	xo->status = CRYPTO_SUCCESS;
+
+	adapter->rx_ipsec++;
+}
+
+/**
+ * ixgbe_init_ipsec_offload - initialize security registers for IPSec operation
+ * @adapter: board private structure
+ **/
+void ixgbe_init_ipsec_offload(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_ipsec *ipsec;
+	size_t size;
+
+	if (adapter->hw.mac.type == ixgbe_mac_82598EB)
+		return;
+
+	ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL);
+	if (!ipsec)
+		goto err1;
+	hash_init(ipsec->rx_sa_list);
+
+	size = sizeof(struct rx_sa) * IXGBE_IPSEC_MAX_SA_COUNT;
+	ipsec->rx_tbl = kzalloc(size, GFP_KERNEL);
+	if (!ipsec->rx_tbl)
+		goto err2;
+
+	size = sizeof(struct tx_sa) * IXGBE_IPSEC_MAX_SA_COUNT;
+	ipsec->tx_tbl = kzalloc(size, GFP_KERNEL);
+	if (!ipsec->tx_tbl)
+		goto err2;
+
+	size = sizeof(struct rx_ip_sa) * IXGBE_IPSEC_MAX_RX_IP_COUNT;
+	ipsec->ip_tbl = kzalloc(size, GFP_KERNEL);
+	if (!ipsec->ip_tbl)
+		goto err2;
+
+	ipsec->num_rx_sa = 0;
+	ipsec->num_tx_sa = 0;
+
+	adapter->ipsec = ipsec;
+	ixgbe_ipsec_stop_engine(adapter);
+	ixgbe_ipsec_clear_hw_tables(adapter);
+
+	adapter->netdev->xfrmdev_ops = &ixgbe_xfrmdev_ops;
+	adapter->netdev->features |= NETIF_F_HW_ESP;
+	adapter->netdev->hw_enc_features |= NETIF_F_HW_ESP;
+
+	return;
+
+err2:
+	kfree(ipsec->ip_tbl);
+	kfree(ipsec->rx_tbl);
+	kfree(ipsec->tx_tbl);
+err1:
+	kfree(adapter->ipsec);
+	netdev_err(adapter->netdev, "Unable to allocate memory for SA tables");
+}
+
+/**
+ * ixgbe_stop_ipsec_offload - tear down the ipsec offload
+ * @adapter: board private structure
+ **/
+void ixgbe_stop_ipsec_offload(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_ipsec *ipsec = adapter->ipsec;
+
+	adapter->ipsec = NULL;
+	if (ipsec) {
+		kfree(ipsec->ip_tbl);
+		kfree(ipsec->rx_tbl);
+		kfree(ipsec->tx_tbl);
+		kfree(ipsec);
+	}
+}
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h
new file mode 100644
index 000000000000..da3ce7849e85
--- /dev/null
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.h
@@ -0,0 +1,93 @@
+/*******************************************************************************
+
+  Intel 10 Gigabit PCI Express Linux driver
+  Copyright(c) 2017 Oracle and/or its affiliates. All rights reserved.
+
+  This program is free software; you can redistribute it and/or modify it
+  under the terms and conditions of the GNU General Public License,
+  version 2, as published by the Free Software Foundation.
+
+  This program is distributed in the hope it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+  more details.
+
+  You should have received a copy of the GNU General Public License along with
+  this program.  If not, see <http://www.gnu.org/licenses/>.
+
+  The full GNU General Public License is included in this distribution in
+  the file called "COPYING".
+
+  Contact Information:
+  Linux NICS <linux.nics@intel.com>
+  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
+  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+
+*******************************************************************************/
+
+#ifndef _IXGBE_IPSEC_H_
+#define _IXGBE_IPSEC_H_
+
+#define IXGBE_IPSEC_MAX_SA_COUNT	1024
+#define IXGBE_IPSEC_MAX_RX_IP_COUNT	128
+#define IXGBE_IPSEC_BASE_RX_INDEX	0
+#define IXGBE_IPSEC_BASE_TX_INDEX	IXGBE_IPSEC_MAX_SA_COUNT
+
+#define IXGBE_RXTXIDX_IPS_EN		0x00000001
+#define IXGBE_RXIDX_TBL_SHIFT		1
+enum ixgbe_ipsec_tbl_sel {
+	ips_rx_ip_tbl	=	0x01,
+	ips_rx_spi_tbl	=	0x02,
+	ips_rx_key_tbl	=	0x03,
+};
+
+#define IXGBE_RXTXIDX_IDX_SHIFT		3
+#define IXGBE_RXTXIDX_READ		0x40000000
+#define IXGBE_RXTXIDX_WRITE		0x80000000
+
+#define IXGBE_RXMOD_VALID		0x00000001
+#define IXGBE_RXMOD_PROTO_ESP		0x00000004
+#define IXGBE_RXMOD_DECRYPT		0x00000008
+#define IXGBE_RXMOD_IPV6		0x00000010
+
+struct rx_sa {
+	struct hlist_node hlist;
+	struct xfrm_state *xs;
+	__be32 ipaddr[4];
+	u32 key[4];
+	u32 salt;
+	u32 mode;
+	u8  iptbl_ind;
+	bool used;
+	bool decrypt;
+};
+
+struct rx_ip_sa {
+	__be32 ipaddr[4];
+	u32 ref_cnt;
+	bool used;
+};
+
+struct tx_sa {
+	struct xfrm_state *xs;
+	u32 key[4];
+	u32 salt;
+	bool encrypt;
+	bool used;
+};
+
+struct ixgbe_ipsec_tx_data {
+	u32 flags;
+	u16 trailer_len;
+	u16 sa_idx;
+};
+
+struct ixgbe_ipsec {
+	u16 num_rx_sa;
+	u16 num_tx_sa;
+	struct rx_ip_sa *ip_tbl;
+	struct rx_sa *rx_tbl;
+	struct tx_sa *tx_tbl;
+	DECLARE_HASHTABLE(rx_sa_list, 10);
+};
+#endif /* _IXGBE_IPSEC_H_ */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
index 8e2a957aca18..4242f0213e46 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
@@ -46,8 +46,8 @@ static bool ixgbe_cache_ring_dcb_sriov(struct ixgbe_adapter *adapter)
 #endif /* IXGBE_FCOE */
 	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
 	int i;
-	u16 reg_idx;
-	u8 tcs = netdev_get_num_tc(adapter->netdev);
+	u16 reg_idx, pool;
+	u8 tcs = adapter->hw_tcs;
 
 	/* verify we have DCB queueing enabled before proceeding */
 	if (tcs <= 1)
@@ -58,12 +58,16 @@ static bool ixgbe_cache_ring_dcb_sriov(struct ixgbe_adapter *adapter)
 		return false;
 
 	/* start at VMDq register offset for SR-IOV enabled setups */
+	pool = 0;
 	reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
-	for (i = 0; i < adapter->num_rx_queues; i++, reg_idx++) {
+	for (i = 0, pool = 0; i < adapter->num_rx_queues; i++, reg_idx++) {
 		/* If we are greater than indices move to next pool */
-		if ((reg_idx & ~vmdq->mask) >= tcs)
+		if ((reg_idx & ~vmdq->mask) >= tcs) {
+			pool++;
 			reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask);
+		}
 		adapter->rx_ring[i]->reg_idx = reg_idx;
+		adapter->rx_ring[i]->netdev = pool ? NULL : adapter->netdev;
 	}
 
 	reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
@@ -92,6 +96,7 @@ static bool ixgbe_cache_ring_dcb_sriov(struct ixgbe_adapter *adapter)
 		for (i = fcoe->offset; i < adapter->num_rx_queues; i++) {
 			reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask) + fcoe_tc;
 			adapter->rx_ring[i]->reg_idx = reg_idx;
+			adapter->rx_ring[i]->netdev = adapter->netdev;
 			reg_idx++;
 		}
 
@@ -111,9 +116,8 @@ static bool ixgbe_cache_ring_dcb_sriov(struct ixgbe_adapter *adapter)
 static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc,
 				    unsigned int *tx, unsigned int *rx)
 {
-	struct net_device *dev = adapter->netdev;
 	struct ixgbe_hw *hw = &adapter->hw;
-	u8 num_tcs = netdev_get_num_tc(dev);
+	u8 num_tcs = adapter->hw_tcs;
 
 	*tx = 0;
 	*rx = 0;
@@ -168,10 +172,9 @@ static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc,
  **/
 static bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter)
 {
-	struct net_device *dev = adapter->netdev;
+	u8 num_tcs = adapter->hw_tcs;
 	unsigned int tx_idx, rx_idx;
 	int tc, offset, rss_i, i;
-	u8 num_tcs = netdev_get_num_tc(dev);
 
 	/* verify we have DCB queueing enabled before proceeding */
 	if (num_tcs <= 1)
@@ -184,6 +187,7 @@ static bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter)
 		for (i = 0; i < rss_i; i++, tx_idx++, rx_idx++) {
 			adapter->tx_ring[offset + i]->reg_idx = tx_idx;
 			adapter->rx_ring[offset + i]->reg_idx = rx_idx;
+			adapter->rx_ring[offset + i]->netdev = adapter->netdev;
 			adapter->tx_ring[offset + i]->dcb_tc = tc;
 			adapter->rx_ring[offset + i]->dcb_tc = tc;
 		}
@@ -208,14 +212,15 @@ static bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter)
 #endif /* IXGBE_FCOE */
 	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
 	struct ixgbe_ring_feature *rss = &adapter->ring_feature[RING_F_RSS];
+	u16 reg_idx, pool;
 	int i;
-	u16 reg_idx;
 
 	/* only proceed if VMDq is enabled */
 	if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED))
 		return false;
 
 	/* start at VMDq register offset for SR-IOV enabled setups */
+	pool = 0;
 	reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
 	for (i = 0; i < adapter->num_rx_queues; i++, reg_idx++) {
 #ifdef IXGBE_FCOE
@@ -224,15 +229,20 @@ static bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter)
 			break;
 #endif
 		/* If we are greater than indices move to next pool */
-		if ((reg_idx & ~vmdq->mask) >= rss->indices)
+		if ((reg_idx & ~vmdq->mask) >= rss->indices) {
+			pool++;
 			reg_idx = __ALIGN_MASK(reg_idx, ~vmdq->mask);
+		}
 		adapter->rx_ring[i]->reg_idx = reg_idx;
+		adapter->rx_ring[i]->netdev = pool ? NULL : adapter->netdev;
 	}
 
 #ifdef IXGBE_FCOE
 	/* FCoE uses a linear block of queues so just assigning 1:1 */
-	for (; i < adapter->num_rx_queues; i++, reg_idx++)
+	for (; i < adapter->num_rx_queues; i++, reg_idx++) {
 		adapter->rx_ring[i]->reg_idx = reg_idx;
+		adapter->rx_ring[i]->netdev = adapter->netdev;
+	}
 
 #endif
 	reg_idx = vmdq->offset * __ALIGN_MASK(1, ~vmdq->mask);
@@ -269,8 +279,10 @@ static bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter)
 {
 	int i, reg_idx;
 
-	for (i = 0; i < adapter->num_rx_queues; i++)
+	for (i = 0; i < adapter->num_rx_queues; i++) {
 		adapter->rx_ring[i]->reg_idx = i;
+		adapter->rx_ring[i]->netdev = adapter->netdev;
+	}
 	for (i = 0, reg_idx = 0; i < adapter->num_tx_queues; i++, reg_idx++)
 		adapter->tx_ring[i]->reg_idx = reg_idx;
 	for (i = 0; i < adapter->num_xdp_queues; i++, reg_idx++)
@@ -340,7 +352,7 @@ static bool ixgbe_set_dcb_sriov_queues(struct ixgbe_adapter *adapter)
 #ifdef IXGBE_FCOE
 	u16 fcoe_i = 0;
 #endif
-	u8 tcs = netdev_get_num_tc(adapter->netdev);
+	u8 tcs = adapter->hw_tcs;
 
 	/* verify we have DCB queueing enabled before proceeding */
 	if (tcs <= 1)
@@ -350,6 +362,9 @@ static bool ixgbe_set_dcb_sriov_queues(struct ixgbe_adapter *adapter)
 	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
 		return false;
 
+	/* limit VMDq instances on the PF by number of Tx queues */
+	vmdq_i = min_t(u16, vmdq_i, MAX_TX_QUEUES / tcs);
+
 	/* Add starting offset to total pool count */
 	vmdq_i += adapter->ring_feature[RING_F_VMDQ].offset;
 
@@ -437,7 +452,7 @@ static bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter)
 	int tcs;
 
 	/* Map queue offset and counts onto allocated tx queues */
-	tcs = netdev_get_num_tc(dev);
+	tcs = adapter->hw_tcs;
 
 	/* verify we have DCB queueing enabled before proceeding */
 	if (tcs <= 1)
@@ -512,12 +527,14 @@ static bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
 #ifdef IXGBE_FCOE
 	u16 fcoe_i = 0;
 #endif
-	bool pools = (find_first_zero_bit(&adapter->fwd_bitmask, 32) > 1);
 
 	/* only proceed if SR-IOV is enabled */
 	if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
 		return false;
 
+	/* limit l2fwd RSS based on total Tx queue limit */
+	rss_i = min_t(u16, rss_i, MAX_TX_QUEUES / vmdq_i);
+
 	/* Add starting offset to total pool count */
 	vmdq_i += adapter->ring_feature[RING_F_VMDQ].offset;
 
@@ -525,7 +542,7 @@ static bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
 	vmdq_i = min_t(u16, IXGBE_MAX_VMDQ_INDICES, vmdq_i);
 
 	/* 64 pool mode with 2 queues per pool */
-	if ((vmdq_i > 32) || (vmdq_i > 16 && pools)) {
+	if (vmdq_i > 32) {
 		vmdq_m = IXGBE_82599_VMDQ_2Q_MASK;
 		rss_m = IXGBE_RSS_2Q_MASK;
 		rss_i = min_t(u16, rss_i, 2);
@@ -602,6 +619,10 @@ static bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
 	}
 
 #endif
+	/* populate TC0 for use by pool 0 */
+	netdev_set_tc_queue(adapter->netdev, 0,
+			    adapter->num_rx_queues_per_pool, 0);
+
 	return true;
 }
 
@@ -701,7 +722,7 @@ static void ixgbe_set_num_queues(struct ixgbe_adapter *adapter)
 	adapter->num_rx_queues = 1;
 	adapter->num_tx_queues = 1;
 	adapter->num_xdp_queues = 0;
-	adapter->num_rx_pools = adapter->num_rx_queues;
+	adapter->num_rx_pools = 1;
 	adapter->num_rx_queues_per_pool = 1;
 
 #ifdef CONFIG_IXGBE_DCB
@@ -834,7 +855,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
 	int node = NUMA_NO_NODE;
 	int cpu = -1;
 	int ring_count, size;
-	u8 tcs = netdev_get_num_tc(adapter->netdev);
+	u8 tcs = adapter->hw_tcs;
 
 	ring_count = txr_count + rxr_count + xdp_count;
 	size = sizeof(struct ixgbe_q_vector) +
@@ -917,11 +938,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
 
 		/* apply Tx specific ring traits */
 		ring->count = adapter->tx_ring_count;
-		if (adapter->num_rx_pools > 1)
-			ring->queue_index =
-				txr_idx % adapter->num_rx_queues_per_pool;
-		else
-			ring->queue_index = txr_idx;
+		ring->queue_index = txr_idx;
 
 		/* assign ring to adapter */
 		adapter->tx_ring[txr_idx] = ring;
@@ -991,11 +1008,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
 #endif /* IXGBE_FCOE */
 		/* apply Rx specific ring traits */
 		ring->count = adapter->rx_ring_count;
-		if (adapter->num_rx_pools > 1)
-			ring->queue_index =
-				rxr_idx % adapter->num_rx_queues_per_pool;
-		else
-			ring->queue_index = rxr_idx;
+		ring->queue_index = rxr_idx;
 
 		/* assign ring to adapter */
 		adapter->rx_ring[rxr_idx] = ring;
@@ -1171,7 +1184,7 @@ static void ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter)
 	 */
 
 	/* Disable DCB unless we only have a single traffic class */
-	if (netdev_get_num_tc(adapter->netdev) > 1) {
+	if (adapter->hw_tcs > 1) {
 		e_dev_warn("Number of DCB TCs exceeds number of available queues. Disabling DCB support.\n");
 		netdev_reset_tc(adapter->netdev);
 
@@ -1183,6 +1196,7 @@ static void ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter)
 		adapter->dcb_cfg.pfc_mode_enable = false;
 	}
 
+	adapter->hw_tcs = 0;
 	adapter->dcb_cfg.num_tcs.pg_tcs = 1;
 	adapter->dcb_cfg.num_tcs.pfc_tcs = 1;
 
@@ -1268,7 +1282,7 @@ void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter)
 }
 
 void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens,
-		       u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
+		       u32 fceof_saidx, u32 type_tucmd, u32 mss_l4len_idx)
 {
 	struct ixgbe_adv_tx_context_desc *context_desc;
 	u16 i = tx_ring->next_to_use;
@@ -1282,7 +1296,7 @@ void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens,
 	type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
 
 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
-	context_desc->seqnum_seed	= cpu_to_le32(fcoe_sof_eof);
+	context_desc->fceof_saidx	= cpu_to_le32(fceof_saidx);
 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
 }
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 62a18914f00f..9fc063af233c 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -192,6 +192,13 @@ static struct workqueue_struct *ixgbe_wq;
 static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);
 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);
 
+static const struct net_device_ops ixgbe_netdev_ops;
+
+static bool netif_is_ixgbe(struct net_device *dev)
+{
+	return dev && (dev->netdev_ops == &ixgbe_netdev_ops);
+}
+
 static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,
 					  u32 reg, u16 *value)
 {
@@ -1064,24 +1071,12 @@ static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
 
 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
 {
-	struct ixgbe_adapter *adapter;
-	struct ixgbe_hw *hw;
-	u32 head, tail;
+	unsigned int head, tail;
 
-	if (ring->l2_accel_priv)
-		adapter = ring->l2_accel_priv->real_adapter;
-	else
-		adapter = netdev_priv(ring->netdev);
+	head = ring->next_to_clean;
+	tail = ring->next_to_use;
 
-	hw = &adapter->hw;
-	head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
-	tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
-
-	if (head != tail)
-		return (head < tail) ?
-			tail - head : (tail + ring->count - head);
-
-	return 0;
+	return ((head <= tail) ? tail : tail + ring->count) - head;
 }
 
 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
@@ -1133,6 +1128,9 @@ static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate
+ * @netdev: network interface device structure
+ * @queue_index: Tx queue to set
+ * @maxrate: desired maximum transmit bitrate
  **/
 static int ixgbe_tx_maxrate(struct net_device *netdev,
 			    int queue_index, u32 maxrate)
@@ -1173,7 +1171,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 	struct ixgbe_adapter *adapter = q_vector->adapter;
 	struct ixgbe_tx_buffer *tx_buffer;
 	union ixgbe_adv_tx_desc *tx_desc;
-	unsigned int total_bytes = 0, total_packets = 0;
+	unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;
 	unsigned int budget = q_vector->tx.work_limit;
 	unsigned int i = tx_ring->next_to_clean;
 
@@ -1204,6 +1202,8 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 		/* update the statistics for this packet */
 		total_bytes += tx_buffer->bytecount;
 		total_packets += tx_buffer->gso_segs;
+		if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)
+			total_ipsec++;
 
 		/* free the skb */
 		if (ring_is_xdp(tx_ring))
@@ -1266,6 +1266,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 	u64_stats_update_end(&tx_ring->syncp);
 	q_vector->tx.total_bytes += total_bytes;
 	q_vector->tx.total_packets += total_packets;
+	adapter->tx_ipsec += total_ipsec;
 
 	if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
 		/* schedule immediate reset if we believe we hung */
@@ -1754,9 +1755,18 @@ static void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,
 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
 	}
 
-	skb_record_rx_queue(skb, rx_ring->queue_index);
+	if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))
+		ixgbe_ipsec_rx(rx_ring, rx_desc, skb);
 
 	skb->protocol = eth_type_trans(skb, dev);
+
+	/* record Rx queue, or update MACVLAN statistics */
+	if (netif_is_ixgbe(dev))
+		skb_record_rx_queue(skb, rx_ring->queue_index);
+	else
+		macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
+				 (skb->pkt_type == PACKET_BROADCAST) ||
+				 (skb->pkt_type == PACKET_MULTICAST));
 }
 
 static void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,
@@ -1878,6 +1888,14 @@ static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,
 				     ixgbe_rx_pg_size(rx_ring),
 				     DMA_FROM_DEVICE,
 				     IXGBE_RX_DMA_ATTR);
+	} else if (ring_uses_build_skb(rx_ring)) {
+		unsigned long offset = (unsigned long)(skb->data) & ~PAGE_MASK;
+
+		dma_sync_single_range_for_cpu(rx_ring->dev,
+					      IXGBE_CB(skb)->dma,
+					      offset,
+					      skb_headlen(skb),
+					      DMA_FROM_DEVICE);
 	} else {
 		struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
 
@@ -1921,10 +1939,13 @@ static bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,
 	if (IS_ERR(skb))
 		return true;
 
-	/* verify that the packet does not have any known errors */
-	if (unlikely(ixgbe_test_staterr(rx_desc,
-					IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
-	    !(netdev->features & NETIF_F_RXALL))) {
+	/* Verify netdev is present, and that packet does not have any
+	 * errors that would be unacceptable to the netdev.
+	 */
+	if (!netdev ||
+	    (unlikely(ixgbe_test_staterr(rx_desc,
+					 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&
+	     !(netdev->features & NETIF_F_RXALL)))) {
 		dev_kfree_skb_any(skb);
 		return true;
 	}
@@ -2021,8 +2042,8 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer)
  * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff
  * @rx_ring: rx descriptor ring to transact packets on
  * @rx_buffer: buffer containing page to add
- * @rx_desc: descriptor containing length of buffer written by hardware
  * @skb: sk_buff to place the data into
+ * @size: size of data in rx_buffer
  *
  * This function will add the data contained in rx_buffer->page to the skb.
  * This is done either through a direct copy if the data in the buffer is
@@ -2318,12 +2339,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
 #endif /* IXGBE_FCOE */
 	u16 cleaned_count = ixgbe_desc_unused(rx_ring);
 	bool xdp_xmit = false;
+	struct xdp_buff xdp;
+
+	xdp.rxq = &rx_ring->xdp_rxq;
 
 	while (likely(total_rx_packets < budget)) {
 		union ixgbe_adv_rx_desc *rx_desc;
 		struct ixgbe_rx_buffer *rx_buffer;
 		struct sk_buff *skb;
-		struct xdp_buff xdp;
 		unsigned int size;
 
 		/* return some buffers to hardware, one at a time is too slow */
@@ -2515,13 +2538,6 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
 	IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
 }
 
-enum latency_range {
-	lowest_latency = 0,
-	low_latency = 1,
-	bulk_latency = 2,
-	latency_invalid = 255
-};
-
 /**
  * ixgbe_update_itr - update the dynamic ITR value based on statistics
  * @q_vector: structure containing interrupt and ring information
@@ -2534,8 +2550,6 @@ enum latency_range {
  *      based on theoretical maximum wire speed and thresholds were set based
  *      on testing data as well as attempting to minimize response time
  *      while increasing bulk throughput.
- *      this functionality is controlled by the InterruptThrottleRate module
- *      parameter (see ixgbe_param.c)
  **/
 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
 			     struct ixgbe_ring_container *ring_container)
@@ -3020,6 +3034,8 @@ static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
 /**
  * ixgbe_irq_enable - Enable default interrupt generation settings
  * @adapter: board private structure
+ * @queues: enable irqs for queues
+ * @flush: flush register write
  **/
 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
 				    bool flush)
@@ -3475,6 +3491,7 @@ static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
+ * @adapter: board private structure
  *
  **/
 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
@@ -3586,7 +3603,7 @@ static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32 rttdcs, mtqc;
-	u8 tcs = netdev_get_num_tc(adapter->netdev);
+	u8 tcs = adapter->hw_tcs;
 
 	if (hw->mac.type == ixgbe_mac_82598EB)
 		return;
@@ -3853,16 +3870,20 @@ static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)
 	u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32 vfreta = 0;
-	unsigned int pf_pool = adapter->num_vfs;
 
 	/* Write redirection table to HW */
 	for (i = 0; i < reta_entries; i++) {
+		u16 pool = adapter->num_rx_pools;
+
 		vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;
-		if ((i & 3) == 3) {
-			IXGBE_WRITE_REG(hw, IXGBE_PFVFRETA(i >> 2, pf_pool),
+		if ((i & 3) != 3)
+			continue;
+
+		while (pool--)
+			IXGBE_WRITE_REG(hw,
+					IXGBE_PFVFRETA(i >> 2, VMDQ_P(pool)),
 					vfreta);
-			vfreta = 0;
-		}
+		vfreta = 0;
 	}
 }
 
@@ -3899,13 +3920,17 @@ static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;
-	unsigned int pf_pool = adapter->num_vfs;
 	int i, j;
 
 	/* Fill out hash function seeds */
-	for (i = 0; i < 10; i++)
-		IXGBE_WRITE_REG(hw, IXGBE_PFVFRSSRK(i, pf_pool),
-				*(adapter->rss_key + i));
+	for (i = 0; i < 10; i++) {
+		u16 pool = adapter->num_rx_pools;
+
+		while (pool--)
+			IXGBE_WRITE_REG(hw,
+					IXGBE_PFVFRSSRK(i, VMDQ_P(pool)),
+					*(adapter->rss_key + i));
+	}
 
 	/* Fill out the redirection table */
 	for (i = 0, j = 0; i < 64; i++, j++) {
@@ -3933,7 +3958,7 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
 		if (adapter->ring_feature[RING_F_RSS].mask)
 			mrqc = IXGBE_MRQC_RSSEN;
 	} else {
-		u8 tcs = netdev_get_num_tc(adapter->netdev);
+		u8 tcs = adapter->hw_tcs;
 
 		if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
 			if (tcs > 4)
@@ -3971,7 +3996,7 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
 
 	if ((hw->mac.type >= ixgbe_mac_X550) &&
 	    (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {
-		unsigned int pf_pool = adapter->num_vfs;
+		u16 pool = adapter->num_rx_pools;
 
 		/* Enable VF RSS mode */
 		mrqc |= IXGBE_MRQC_MULTIPLE_RSS;
@@ -3981,7 +4006,11 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
 		ixgbe_setup_vfreta(adapter);
 		vfmrqc = IXGBE_MRQC_RSSEN;
 		vfmrqc |= rss_field;
-		IXGBE_WRITE_REG(hw, IXGBE_PFVFMRQC(pf_pool), vfmrqc);
+
+		while (pool--)
+			IXGBE_WRITE_REG(hw,
+					IXGBE_PFVFMRQC(VMDQ_P(pool)),
+					vfmrqc);
 	} else {
 		ixgbe_setup_reta(adapter);
 		mrqc |= rss_field;
@@ -3991,8 +4020,8 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_configure_rscctl - enable RSC for the indicated ring
- * @adapter:    address of board private structure
- * @index:      index of ring to set
+ * @adapter: address of board private structure
+ * @ring: structure containing ring specific data
  **/
 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
 				   struct ixgbe_ring *ring)
@@ -4112,11 +4141,15 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
 		rxdctl &= ~0x3FFFFF;
 		rxdctl |=  0x080420;
 #if (PAGE_SIZE < 8192)
-	} else {
+	/* RXDCTL.RLPML does not work on 82599 */
+	} else if (hw->mac.type != ixgbe_mac_82599EB) {
 		rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |
 			    IXGBE_RXDCTL_RLPML_EN);
 
-		/* Limit the maximum frame size so we don't overrun the skb */
+		/* Limit the maximum frame size so we don't overrun the skb.
+		 * This can happen in SRIOV mode when the MTU of the VF is
+		 * higher than the MTU of the PF.
+		 */
 		if (ring_uses_build_skb(ring) &&
 		    !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))
 			rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB |
@@ -4144,7 +4177,7 @@ static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	int rss_i = adapter->ring_feature[RING_F_RSS].indices;
-	u16 pool;
+	u16 pool = adapter->num_rx_pools;
 
 	/* PSRTYPE must be initialized in non 82598 adapters */
 	u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
@@ -4161,7 +4194,7 @@ static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
 	else if (rss_i > 1)
 		psrtype |= 1u << 29;
 
-	for_each_set_bit(pool, &adapter->fwd_bitmask, 32)
+	while (pool--)
 		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
 }
 
@@ -4488,8 +4521,9 @@ static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			struct ixgbe_ring *ring = adapter->rx_ring[i];
 
-			if (ring->l2_accel_priv)
+			if (!netif_is_ixgbe(ring->netdev))
 				continue;
+
 			j = ring->reg_idx;
 			vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
 			vlnctrl &= ~IXGBE_RXDCTL_VME;
@@ -4525,8 +4559,9 @@ static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			struct ixgbe_ring *ring = adapter->rx_ring[i];
 
-			if (ring->l2_accel_priv)
+			if (!netif_is_ixgbe(ring->netdev))
 				continue;
+
 			j = ring->reg_idx;
 			vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
 			vlnctrl |= IXGBE_RXDCTL_VME;
@@ -4846,9 +4881,11 @@ int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,
 
 	return -ENOMEM;
 }
+
 /**
  * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
  * @netdev: network interface device structure
+ * @vfn: pool to associate with unicast addresses
  *
  * Writes unicast address list to the RAR table.
  * Returns: -ENOMEM on failure/insufficient address space
@@ -5195,7 +5232,7 @@ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb)
 static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
-	int num_tc = netdev_get_num_tc(adapter->netdev);
+	int num_tc = adapter->hw_tcs;
 	int i;
 
 	if (!num_tc)
@@ -5218,7 +5255,7 @@ static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	int hdrm;
-	u8 tc = netdev_get_num_tc(adapter->netdev);
+	u8 tc = adapter->hw_tcs;
 
 	if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
 	    adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
@@ -5277,29 +5314,6 @@ static void ixgbe_macvlan_set_rx_mode(struct net_device *dev, unsigned int pool,
 	IXGBE_WRITE_REG(hw, IXGBE_VMOLR(pool), vmolr);
 }
 
-static void ixgbe_fwd_psrtype(struct ixgbe_fwd_adapter *vadapter)
-{
-	struct ixgbe_adapter *adapter = vadapter->real_adapter;
-	int rss_i = adapter->num_rx_queues_per_pool;
-	struct ixgbe_hw *hw = &adapter->hw;
-	u16 pool = vadapter->pool;
-	u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
-		      IXGBE_PSRTYPE_UDPHDR |
-		      IXGBE_PSRTYPE_IPV4HDR |
-		      IXGBE_PSRTYPE_L2HDR |
-		      IXGBE_PSRTYPE_IPV6HDR;
-
-	if (hw->mac.type == ixgbe_mac_82598EB)
-		return;
-
-	if (rss_i > 3)
-		psrtype |= 2u << 29;
-	else if (rss_i > 1)
-		psrtype |= 1u << 29;
-
-	IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);
-}
-
 /**
  * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
  * @rx_ring: ring to free buffers from
@@ -5352,96 +5366,45 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
 	rx_ring->next_to_use = 0;
 }
 
-static void ixgbe_disable_fwd_ring(struct ixgbe_fwd_adapter *vadapter,
-				   struct ixgbe_ring *rx_ring)
-{
-	struct ixgbe_adapter *adapter = vadapter->real_adapter;
-	int index = rx_ring->queue_index + vadapter->rx_base_queue;
-
-	/* shutdown specific queue receive and wait for dma to settle */
-	ixgbe_disable_rx_queue(adapter, rx_ring);
-	usleep_range(10000, 20000);
-	ixgbe_irq_disable_queues(adapter, BIT_ULL(index));
-	ixgbe_clean_rx_ring(rx_ring);
-	rx_ring->l2_accel_priv = NULL;
-}
-
-static int ixgbe_fwd_ring_down(struct net_device *vdev,
-			       struct ixgbe_fwd_adapter *accel)
-{
-	struct ixgbe_adapter *adapter = accel->real_adapter;
-	unsigned int rxbase = accel->rx_base_queue;
-	unsigned int txbase = accel->tx_base_queue;
-	int i;
-
-	netif_tx_stop_all_queues(vdev);
-
-	for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
-		ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]);
-		adapter->rx_ring[rxbase + i]->netdev = adapter->netdev;
-	}
-
-	for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
-		adapter->tx_ring[txbase + i]->l2_accel_priv = NULL;
-		adapter->tx_ring[txbase + i]->netdev = adapter->netdev;
-	}
-
-
-	return 0;
-}
-
 static int ixgbe_fwd_ring_up(struct net_device *vdev,
 			     struct ixgbe_fwd_adapter *accel)
 {
 	struct ixgbe_adapter *adapter = accel->real_adapter;
-	unsigned int rxbase, txbase, queues;
-	int i, baseq, err = 0;
+	int i, baseq, err;
 
-	if (!test_bit(accel->pool, &adapter->fwd_bitmask))
+	if (!test_bit(accel->pool, adapter->fwd_bitmask))
 		return 0;
 
 	baseq = accel->pool * adapter->num_rx_queues_per_pool;
-	netdev_dbg(vdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n",
+	netdev_dbg(vdev, "pool %i:%i queues %i:%i\n",
 		   accel->pool, adapter->num_rx_pools,
-		   baseq, baseq + adapter->num_rx_queues_per_pool,
-		   adapter->fwd_bitmask);
+		   baseq, baseq + adapter->num_rx_queues_per_pool);
 
 	accel->netdev = vdev;
-	accel->rx_base_queue = rxbase = baseq;
-	accel->tx_base_queue = txbase = baseq;
+	accel->rx_base_queue = baseq;
+	accel->tx_base_queue = baseq;
 
 	for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
-		ixgbe_disable_fwd_ring(accel, adapter->rx_ring[rxbase + i]);
+		adapter->rx_ring[baseq + i]->netdev = vdev;
 
-	for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
-		adapter->rx_ring[rxbase + i]->netdev = vdev;
-		adapter->rx_ring[rxbase + i]->l2_accel_priv = accel;
-		ixgbe_configure_rx_ring(adapter, adapter->rx_ring[rxbase + i]);
-	}
+	/* Guarantee all rings are updated before we update the
+	 * MAC address filter.
+	 */
+	wmb();
 
-	for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
-		adapter->tx_ring[txbase + i]->netdev = vdev;
-		adapter->tx_ring[txbase + i]->l2_accel_priv = accel;
+	/* ixgbe_add_mac_filter will return an index if it succeeds, so we
+	 * need to only treat it as an error value if it is negative.
+	 */
+	err = ixgbe_add_mac_filter(adapter, vdev->dev_addr,
+				   VMDQ_P(accel->pool));
+	if (err >= 0) {
+		ixgbe_macvlan_set_rx_mode(vdev, accel->pool, adapter);
+		return 0;
 	}
 
-	queues = min_t(unsigned int,
-		       adapter->num_rx_queues_per_pool, vdev->num_tx_queues);
-	err = netif_set_real_num_tx_queues(vdev, queues);
-	if (err)
-		goto fwd_queue_err;
-
-	err = netif_set_real_num_rx_queues(vdev, queues);
-	if (err)
-		goto fwd_queue_err;
-
-	if (is_valid_ether_addr(vdev->dev_addr))
-		ixgbe_add_mac_filter(adapter, vdev->dev_addr, accel->pool);
+	for (i = 0; i < adapter->num_rx_queues_per_pool; i++)
+		adapter->rx_ring[baseq + i]->netdev = NULL;
 
-	ixgbe_fwd_psrtype(accel);
-	ixgbe_macvlan_set_rx_mode(vdev, accel->pool, adapter);
-	return err;
-fwd_queue_err:
-	ixgbe_fwd_ring_down(vdev, accel);
 	return err;
 }
 
@@ -5480,6 +5443,7 @@ static void ixgbe_configure(struct ixgbe_adapter *adapter)
 
 	ixgbe_set_rx_mode(adapter->netdev);
 	ixgbe_restore_vlan(adapter);
+	ixgbe_ipsec_restore(adapter);
 
 	switch (hw->mac.type) {
 	case ixgbe_mac_82599EB:
@@ -5917,21 +5881,6 @@ static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
 	spin_unlock(&adapter->fdir_perfect_lock);
 }
 
-static int ixgbe_disable_macvlan(struct net_device *upper, void *data)
-{
-	if (netif_is_macvlan(upper)) {
-		struct macvlan_dev *vlan = netdev_priv(upper);
-
-		if (vlan->fwd_priv) {
-			netif_tx_stop_all_queues(upper);
-			netif_carrier_off(upper);
-			netif_tx_disable(upper);
-		}
-	}
-
-	return 0;
-}
-
 void ixgbe_down(struct ixgbe_adapter *adapter)
 {
 	struct net_device *netdev = adapter->netdev;
@@ -5961,10 +5910,6 @@ void ixgbe_down(struct ixgbe_adapter *adapter)
 	netif_carrier_off(netdev);
 	netif_tx_disable(netdev);
 
-	/* disable any upper devices */
-	netdev_walk_all_upper_dev_rcu(adapter->netdev,
-				      ixgbe_disable_macvlan, NULL);
-
 	ixgbe_irq_disable(adapter);
 
 	ixgbe_napi_disable_all(adapter);
@@ -6119,6 +6064,7 @@ static void ixgbe_init_dcb(struct ixgbe_adapter *adapter)
 /**
  * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
  * @adapter: board private structure to initialize
+ * @ii: pointer to ixgbe_info for device
  *
  * ixgbe_sw_init initializes the Adapter private data structure.
  * Fields are initialized based on PCI device information and
@@ -6153,6 +6099,7 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter,
 	fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus());
 	adapter->ring_feature[RING_F_FDIR].limit = fdir;
 	adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
+	adapter->ring_feature[RING_F_VMDQ].limit = 1;
 #ifdef CONFIG_IXGBE_DCA
 	adapter->flags |= IXGBE_FLAG_DCA_CAPABLE;
 #endif
@@ -6302,7 +6249,7 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter,
 	}
 
 	/* PF holds first pool slot */
-	set_bit(0, &adapter->fwd_bitmask);
+	set_bit(0, adapter->fwd_bitmask);
 	set_bit(__IXGBE_DOWN, &adapter->state);
 
 	return 0;
@@ -6402,6 +6349,7 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
+ * @adapter: pointer to ixgbe_adapter
  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
  *
  * Returns 0 on success, negative on failure
@@ -6444,6 +6392,11 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter,
 	rx_ring->next_to_clean = 0;
 	rx_ring->next_to_use = 0;
 
+	/* XDP RX-queue info */
+	if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev,
+			     rx_ring->queue_index) < 0)
+		goto err;
+
 	rx_ring->xdp_prog = adapter->xdp_prog;
 
 	return 0;
@@ -6541,6 +6494,7 @@ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
 	ixgbe_clean_rx_ring(rx_ring);
 
 	rx_ring->xdp_prog = NULL;
+	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 	vfree(rx_ring->rx_buffer_info);
 	rx_ring->rx_buffer_info = NULL;
 
@@ -6646,20 +6600,12 @@ int ixgbe_open(struct net_device *netdev)
 		goto err_req_irq;
 
 	/* Notify the stack of the actual queue counts. */
-	if (adapter->num_rx_pools > 1)
-		queues = adapter->num_rx_queues_per_pool;
-	else
-		queues = adapter->num_tx_queues;
-
+	queues = adapter->num_tx_queues;
 	err = netif_set_real_num_tx_queues(netdev, queues);
 	if (err)
 		goto err_set_queues;
 
-	if (adapter->num_rx_pools > 1 &&
-	    adapter->num_rx_queues > IXGBE_MAX_L2A_QUEUES)
-		queues = IXGBE_MAX_L2A_QUEUES;
-	else
-		queues = adapter->num_rx_queues;
+	queues = adapter->num_rx_queues;
 	err = netif_set_real_num_rx_queues(netdev, queues);
 	if (err)
 		goto err_set_queues;
@@ -6783,7 +6729,7 @@ static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
 	struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
 	struct net_device *netdev = adapter->netdev;
 	struct ixgbe_hw *hw = &adapter->hw;
-	u32 ctrl, fctrl;
+	u32 ctrl;
 	u32 wufc = adapter->wol;
 #ifdef CONFIG_PM
 	int retval = 0;
@@ -6808,18 +6754,18 @@ static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
 		hw->mac.ops.stop_link_on_d3(hw);
 
 	if (wufc) {
+		u32 fctrl;
+
 		ixgbe_set_rx_mode(netdev);
 
 		/* enable the optics for 82599 SFP+ fiber as we can WoL */
 		if (hw->mac.ops.enable_tx_laser)
 			hw->mac.ops.enable_tx_laser(hw);
 
-		/* turn on all-multi mode if wake on multicast is enabled */
-		if (wufc & IXGBE_WUFC_MC) {
-			fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
-			fctrl |= IXGBE_FCTRL_MPE;
-			IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
-		}
+		/* enable the reception of multicast packets */
+		fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
+		fctrl |= IXGBE_FCTRL_MPE;
+		IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
 
 		ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
 		ctrl |= IXGBE_CTRL_GIO_DIS;
@@ -7216,7 +7162,6 @@ static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
 /**
  * ixgbe_watchdog_update_link - update the link status
  * @adapter: pointer to the device adapter structure
- * @link_speed: pointer to a u32 to store the link_speed
  **/
 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
 {
@@ -7273,18 +7218,6 @@ static void ixgbe_update_default_up(struct ixgbe_adapter *adapter)
 #endif
 }
 
-static int ixgbe_enable_macvlan(struct net_device *upper, void *data)
-{
-	if (netif_is_macvlan(upper)) {
-		struct macvlan_dev *vlan = netdev_priv(upper);
-
-		if (vlan->fwd_priv)
-			netif_tx_wake_all_queues(upper);
-	}
-
-	return 0;
-}
-
 /**
  * ixgbe_watchdog_link_is_up - update netif_carrier status and
  *                             print link up message
@@ -7338,6 +7271,9 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
 	case IXGBE_LINK_SPEED_10GB_FULL:
 		speed_str = "10 Gbps";
 		break;
+	case IXGBE_LINK_SPEED_5GB_FULL:
+		speed_str = "5 Gbps";
+		break;
 	case IXGBE_LINK_SPEED_2_5GB_FULL:
 		speed_str = "2.5 Gbps";
 		break;
@@ -7365,12 +7301,6 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
 	/* enable transmits */
 	netif_tx_wake_all_queues(adapter->netdev);
 
-	/* enable any upper devices */
-	rtnl_lock();
-	netdev_walk_all_upper_dev_rcu(adapter->netdev,
-				      ixgbe_enable_macvlan, NULL);
-	rtnl_unlock();
-
 	/* update the default user priority for VFs */
 	ixgbe_update_default_up(adapter);
 
@@ -7655,6 +7585,7 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
+	u32 cap_speed;
 	u32 speed;
 	bool autoneg = false;
 
@@ -7667,16 +7598,14 @@ static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
 
 	adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
 
-	speed = hw->phy.autoneg_advertised;
-	if ((!speed) && (hw->mac.ops.get_link_capabilities)) {
-		hw->mac.ops.get_link_capabilities(hw, &speed, &autoneg);
+	hw->mac.ops.get_link_capabilities(hw, &cap_speed, &autoneg);
 
-		/* setup the highest link when no autoneg */
-		if (!autoneg) {
-			if (speed & IXGBE_LINK_SPEED_10GB_FULL)
-				speed = IXGBE_LINK_SPEED_10GB_FULL;
-		}
-	}
+	/* advertise highest capable link speed */
+	if (!autoneg && (cap_speed & IXGBE_LINK_SPEED_10GB_FULL))
+		speed = IXGBE_LINK_SPEED_10GB_FULL;
+	else
+		speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL |
+				     IXGBE_LINK_SPEED_1GB_FULL);
 
 	if (hw->mac.ops.setup_link)
 		hw->mac.ops.setup_link(hw, speed, true);
@@ -7688,7 +7617,7 @@ static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_service_timer - Timer Call-back
- * @data: pointer to adapter cast into an unsigned long
+ * @t: pointer to timer_list structure
  **/
 static void ixgbe_service_timer(struct timer_list *t)
 {
@@ -7888,10 +7817,12 @@ static inline bool ixgbe_ipv6_csum_is_sctp(struct sk_buff *skb)
 }
 
 static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
-			  struct ixgbe_tx_buffer *first)
+			  struct ixgbe_tx_buffer *first,
+			  struct ixgbe_ipsec_tx_data *itd)
 {
 	struct sk_buff *skb = first->skb;
 	u32 vlan_macip_lens = 0;
+	u32 fceof_saidx = 0;
 	u32 type_tucmd = 0;
 
 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
@@ -7932,7 +7863,12 @@ static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
 	vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
 	vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
 
-	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd, 0);
+	if (first->tx_flags & IXGBE_TX_FLAGS_IPSEC) {
+		fceof_saidx |= itd->sa_idx;
+		type_tucmd |= itd->flags | itd->trailer_len;
+	}
+
+	ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd, 0);
 }
 
 #define IXGBE_SET_FLAG(_input, _flag, _result) \
@@ -7975,11 +7911,16 @@ static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
 					IXGBE_TX_FLAGS_CSUM,
 					IXGBE_ADVTXD_POPTS_TXSM);
 
-	/* enble IPv4 checksum for TSO */
+	/* enable IPv4 checksum for TSO */
 	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
 					IXGBE_TX_FLAGS_IPV4,
 					IXGBE_ADVTXD_POPTS_IXSM);
 
+	/* enable IPsec */
+	olinfo_status |= IXGBE_SET_FLAG(tx_flags,
+					IXGBE_TX_FLAGS_IPSEC,
+					IXGBE_ADVTXD_POPTS_IPSEC);
+
 	/*
 	 * Check Context must be set if Tx switch is enabled, which it
 	 * always is for case where virtual functions are running
@@ -8332,14 +8273,19 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,
 			      void *accel_priv, select_queue_fallback_t fallback)
 {
 	struct ixgbe_fwd_adapter *fwd_adapter = accel_priv;
-#ifdef IXGBE_FCOE
 	struct ixgbe_adapter *adapter;
-	struct ixgbe_ring_feature *f;
 	int txq;
+#ifdef IXGBE_FCOE
+	struct ixgbe_ring_feature *f;
 #endif
 
-	if (fwd_adapter)
-		return skb->queue_mapping + fwd_adapter->tx_base_queue;
+	if (fwd_adapter) {
+		adapter = netdev_priv(dev);
+		txq = reciprocal_scale(skb_get_hash(skb),
+				       adapter->num_rx_queues_per_pool);
+
+		return txq + fwd_adapter->tx_base_queue;
+	}
 
 #ifdef IXGBE_FCOE
 
@@ -8438,6 +8384,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
 	u32 tx_flags = 0;
 	unsigned short f;
 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
+	struct ixgbe_ipsec_tx_data ipsec_tx = { 0 };
 	__be16 protocol = skb->protocol;
 	u8 hdr_len = 0;
 
@@ -8542,11 +8489,16 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
 	}
 
 #endif /* IXGBE_FCOE */
+
+#ifdef CONFIG_XFRM_OFFLOAD
+	if (skb->sp && !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx))
+		goto out_drop;
+#endif
 	tso = ixgbe_tso(tx_ring, first, &hdr_len);
 	if (tso < 0)
 		goto out_drop;
 	else if (!tso)
-		ixgbe_tx_csum(tx_ring, first);
+		ixgbe_tx_csum(tx_ring, first, &ipsec_tx);
 
 	/* add the ATR filter if ATR is on */
 	if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
@@ -8671,7 +8623,7 @@ static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
 /**
  * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
  * netdev->dev_addrs
- * @netdev: network interface device structure
+ * @dev: network interface device structure
  *
  * Returns non-zero on failure
  **/
@@ -8695,7 +8647,7 @@ static int ixgbe_add_sanmac_netdev(struct net_device *dev)
 /**
  * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
  * netdev->dev_addrs
- * @netdev: network interface device structure
+ * @dev: network interface device structure
  *
  * Returns non-zero on failure
  **/
@@ -8862,14 +8814,13 @@ static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)
 /**
  * ixgbe_setup_tc - configure net_device for multiple traffic classes
  *
- * @netdev: net device to configure
+ * @dev: net device to configure
  * @tc: number of traffic classes to enable
  */
 int ixgbe_setup_tc(struct net_device *dev, u8 tc)
 {
 	struct ixgbe_adapter *adapter = netdev_priv(dev);
 	struct ixgbe_hw *hw = &adapter->hw;
-	bool pools;
 
 	/* Hardware supports up to 8 traffic classes */
 	if (tc > adapter->dcb_cfg.num_tcs.pg_tcs)
@@ -8878,10 +8829,6 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc)
 	if (hw->mac.type == ixgbe_mac_82598EB && tc && tc < MAX_TRAFFIC_CLASS)
 		return -EINVAL;
 
-	pools = (find_first_zero_bit(&adapter->fwd_bitmask, 32) > 1);
-	if (tc && pools && adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS)
-		return -EBUSY;
-
 	/* Hardware has to reinitialize queues and interrupts to
 	 * match packet buffer alignment. Unfortunately, the
 	 * hardware is not flexible enough to do this dynamically.
@@ -8898,6 +8845,7 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc)
 		netdev_set_num_tc(dev, tc);
 		ixgbe_set_prio_tc_map(adapter);
 
+		adapter->hw_tcs = tc;
 		adapter->flags |= IXGBE_FLAG_DCB_ENABLED;
 
 		if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
@@ -8907,10 +8855,19 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc)
 	} else {
 		netdev_reset_tc(dev);
 
+		/* To support macvlan offload we have to use num_tc to
+		 * restrict the queues that can be used by the device.
+		 * By doing this we can avoid reporting a false number of
+		 * queues.
+		 */
+		if (!tc && adapter->num_rx_pools > 1)
+			netdev_set_num_tc(dev, 1);
+
 		if (adapter->hw.mac.type == ixgbe_mac_82598EB)
 			adapter->hw.fc.requested_mode = adapter->last_lfc_mode;
 
 		adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
+		adapter->hw_tcs = tc;
 
 		adapter->temp_dcb_cfg.pfc_mode_enable = false;
 		adapter->dcb_cfg.pfc_mode_enable = false;
@@ -9044,6 +9001,7 @@ static int get_macvlan_queue(struct net_device *upper, void *_data)
 static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex,
 				  u8 *queue, u64 *action)
 {
+	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
 	unsigned int num_vfs = adapter->num_vfs, vf;
 	struct upper_walk_data data;
 	struct net_device *upper;
@@ -9052,11 +9010,7 @@ static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex,
 	for (vf = 0; vf < num_vfs; ++vf) {
 		upper = pci_get_drvdata(adapter->vfinfo[vf].vfdev);
 		if (upper->ifindex == ifindex) {
-			if (adapter->num_rx_pools > 1)
-				*queue = vf * 2;
-			else
-				*queue = vf * adapter->num_rx_queues_per_pool;
-
+			*queue = vf * __ALIGN_MASK(1, ~vmdq->mask);
 			*action = vf + 1;
 			*action <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF;
 			return 0;
@@ -9101,9 +9055,11 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter,
 
 		/* Redirect to a VF or a offloaded macvlan */
 		if (is_tcf_mirred_egress_redirect(a)) {
-			int ifindex = tcf_mirred_ifindex(a);
+			struct net_device *dev = tcf_mirred_dev(a);
 
-			err = handle_redirect_action(adapter, ifindex, queue,
+			if (!dev)
+				return -EINVAL;
+			err = handle_redirect_action(adapter, dev->ifindex, queue,
 						     action);
 			if (err == 0)
 				return err;
@@ -9362,9 +9318,6 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,
 static int ixgbe_setup_tc_cls_u32(struct ixgbe_adapter *adapter,
 				  struct tc_cls_u32_offload *cls_u32)
 {
-	if (cls_u32->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (cls_u32->command) {
 	case TC_CLSU32_NEW_KNODE:
 	case TC_CLSU32_REPLACE_KNODE:
@@ -9386,7 +9339,7 @@ static int ixgbe_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 {
 	struct ixgbe_adapter *adapter = cb_priv;
 
-	if (!tc_can_offload(adapter->netdev))
+	if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
@@ -9444,7 +9397,7 @@ void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)
 	struct net_device *netdev = adapter->netdev;
 
 	rtnl_lock();
-	ixgbe_setup_tc(netdev, netdev_get_num_tc(netdev));
+	ixgbe_setup_tc(netdev, adapter->hw_tcs);
 	rtnl_unlock();
 }
 
@@ -9520,7 +9473,7 @@ static int ixgbe_set_features(struct net_device *netdev,
 		/* We cannot enable ATR if SR-IOV is enabled */
 		if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED ||
 		    /* We cannot enable ATR if we have 2 or more tcs */
-		    (netdev_get_num_tc(netdev) > 1) ||
+		    (adapter->hw_tcs > 1) ||
 		    /* We cannot enable ATR if RSS is disabled */
 		    (adapter->ring_feature[RING_F_RSS].limit <= 1) ||
 		    /* A sample rate of 0 indicates ATR disabled */
@@ -9695,8 +9648,8 @@ static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
 
 /**
  * ixgbe_configure_bridge_mode - set various bridge modes
- * @adapter - the private structure
- * @mode - requested bridge mode
+ * @adapter: the private structure
+ * @mode: requested bridge mode
  *
  * Configure some settings require for various bridge modes.
  **/
@@ -9821,6 +9774,7 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
 	struct ixgbe_fwd_adapter *fwd_adapter = NULL;
 	struct ixgbe_adapter *adapter = netdev_priv(pdev);
 	int used_pools = adapter->num_vfs + adapter->num_rx_pools;
+	int tcs = adapter->hw_tcs ? : 1;
 	unsigned int limit;
 	int pool, err;
 
@@ -9831,24 +9785,8 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
 	if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)
 		return ERR_PTR(-EINVAL);
 
-#ifdef CONFIG_RPS
-	if (vdev->num_rx_queues != vdev->num_tx_queues) {
-		netdev_info(pdev, "%s: Only supports a single queue count for TX and RX\n",
-			    vdev->name);
-		return ERR_PTR(-EINVAL);
-	}
-#endif
-	/* Check for hardware restriction on number of rx/tx queues */
-	if (vdev->num_tx_queues > IXGBE_MAX_L2A_QUEUES ||
-	    vdev->num_tx_queues == IXGBE_BAD_L2A_QUEUE) {
-		netdev_info(pdev,
-			    "%s: Supports RX/TX Queue counts 1,2, and 4\n",
-			    pdev->name);
-		return ERR_PTR(-EINVAL);
-	}
-
 	if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
-	      adapter->num_rx_pools > IXGBE_MAX_DCBMACVLANS - 1) ||
+	      adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) ||
 	    (adapter->num_rx_pools > IXGBE_MAX_MACVLANS))
 		return ERR_PTR(-EBUSY);
 
@@ -9856,53 +9794,68 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)
 	if (!fwd_adapter)
 		return ERR_PTR(-ENOMEM);
 
-	pool = find_first_zero_bit(&adapter->fwd_bitmask, 32);
-	adapter->num_rx_pools++;
-	set_bit(pool, &adapter->fwd_bitmask);
-	limit = find_last_bit(&adapter->fwd_bitmask, 32);
+	pool = find_first_zero_bit(adapter->fwd_bitmask, adapter->num_rx_pools);
+	set_bit(pool, adapter->fwd_bitmask);
+	limit = find_last_bit(adapter->fwd_bitmask, adapter->num_rx_pools + 1);
 
 	/* Enable VMDq flag so device will be set in VM mode */
 	adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED | IXGBE_FLAG_SRIOV_ENABLED;
 	adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
-	adapter->ring_feature[RING_F_RSS].limit = vdev->num_tx_queues;
 
-	/* Force reinit of ring allocation with VMDQ enabled */
-	err = ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev));
-	if (err)
-		goto fwd_add_err;
 	fwd_adapter->pool = pool;
 	fwd_adapter->real_adapter = adapter;
 
-	if (netif_running(pdev)) {
+	/* Force reinit of ring allocation with VMDQ enabled */
+	err = ixgbe_setup_tc(pdev, adapter->hw_tcs);
+
+	if (!err && netif_running(pdev))
 		err = ixgbe_fwd_ring_up(vdev, fwd_adapter);
-		if (err)
-			goto fwd_add_err;
-		netif_tx_start_all_queues(vdev);
-	}
 
-	return fwd_adapter;
-fwd_add_err:
+	if (!err)
+		return fwd_adapter;
+
 	/* unwind counter and free adapter struct */
 	netdev_info(pdev,
 		    "%s: dfwd hardware acceleration failed\n", vdev->name);
-	clear_bit(pool, &adapter->fwd_bitmask);
-	adapter->num_rx_pools--;
+	clear_bit(pool, adapter->fwd_bitmask);
 	kfree(fwd_adapter);
 	return ERR_PTR(err);
 }
 
 static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
 {
-	struct ixgbe_fwd_adapter *fwd_adapter = priv;
-	struct ixgbe_adapter *adapter = fwd_adapter->real_adapter;
-	unsigned int limit;
+	struct ixgbe_fwd_adapter *accel = priv;
+	struct ixgbe_adapter *adapter = accel->real_adapter;
+	unsigned int rxbase = accel->rx_base_queue;
+	unsigned int limit, i;
+
+	/* delete unicast filter associated with offloaded interface */
+	ixgbe_del_mac_filter(adapter, accel->netdev->dev_addr,
+			     VMDQ_P(accel->pool));
+
+	/* disable ability to receive packets for this pool */
+	IXGBE_WRITE_REG(&adapter->hw, IXGBE_VMOLR(accel->pool), 0);
+
+	/* Allow remaining Rx packets to get flushed out of the
+	 * Rx FIFO before we drop the netdev for the ring.
+	 */
+	usleep_range(10000, 20000);
 
-	clear_bit(fwd_adapter->pool, &adapter->fwd_bitmask);
-	adapter->num_rx_pools--;
+	for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {
+		struct ixgbe_ring *ring = adapter->rx_ring[rxbase + i];
+		struct ixgbe_q_vector *qv = ring->q_vector;
 
-	limit = find_last_bit(&adapter->fwd_bitmask, 32);
+		/* Make sure we aren't processing any packets and clear
+		 * netdev to shut down the ring.
+		 */
+		if (netif_running(adapter->netdev))
+			napi_synchronize(&qv->napi);
+		ring->netdev = NULL;
+	}
+
+	clear_bit(accel->pool, adapter->fwd_bitmask);
+	limit = find_last_bit(adapter->fwd_bitmask, adapter->num_rx_pools);
 	adapter->ring_feature[RING_F_VMDQ].limit = limit + 1;
-	ixgbe_fwd_ring_down(fwd_adapter->netdev, fwd_adapter);
 
 	/* go back to full RSS if we're done with our VMQs */
 	if (adapter->ring_feature[RING_F_VMDQ].limit == 1) {
@@ -9914,13 +9867,13 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv)
 		adapter->ring_feature[RING_F_RSS].limit = rss;
 	}
 
-	ixgbe_setup_tc(pdev, netdev_get_num_tc(pdev));
-	netdev_dbg(pdev, "pool %i:%i queues %i:%i VSI bitmask %lx\n",
-		   fwd_adapter->pool, adapter->num_rx_pools,
-		   fwd_adapter->rx_base_queue,
-		   fwd_adapter->rx_base_queue + adapter->num_rx_queues_per_pool,
-		   adapter->fwd_bitmask);
-	kfree(fwd_adapter);
+	ixgbe_setup_tc(pdev, adapter->hw_tcs);
+	netdev_dbg(pdev, "pool %i:%i queues %i:%i\n",
+		   accel->pool, adapter->num_rx_pools,
+		   accel->rx_base_queue,
+		   accel->rx_base_queue +
+		   adapter->num_rx_queues_per_pool);
+	kfree(accel);
 }
 
 #define IXGBE_MAX_MAC_HDR_LEN		127
@@ -9954,6 +9907,12 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev,
 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
 		features &= ~NETIF_F_TSO;
 
+#ifdef CONFIG_XFRM_OFFLOAD
+	/* IPsec offload doesn't get along well with others *yet* */
+	if (skb->sp)
+		features &= ~(NETIF_F_TSO | NETIF_F_HW_CSUM);
+#endif
+
 	return features;
 }
 
@@ -9987,7 +9946,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog)
 
 	/* If transitioning XDP modes reconfigure rings */
 	if (!!prog != !!old_prog) {
-		int err = ixgbe_setup_tc(dev, netdev_get_num_tc(dev));
+		int err = ixgbe_setup_tc(dev, adapter->hw_tcs);
 
 		if (err) {
 			rcu_assign_pointer(adapter->xdp_prog, old_prog);
@@ -10164,7 +10123,7 @@ static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter)
  * ixgbe_wol_supported - Check whether device supports WoL
  * @adapter: the adapter private structure
  * @device_id: the device ID
- * @subdev_id: the subsystem device ID
+ * @subdevice_id: the subsystem device ID
  *
  * This function is used by probe and ethtool to determine
  * which devices have WoL support
@@ -10232,6 +10191,41 @@ bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,
 	return false;
 }
 
+/**
+ * ixgbe_set_fw_version - Set FW version
+ * @adapter: the adapter private structure
+ *
+ * This function is used by probe and ethtool to determine the FW version to
+ * format to display. The FW version is taken from the EEPROM/NVM.
+ */
+static void ixgbe_set_fw_version(struct ixgbe_adapter *adapter)
+{
+	struct ixgbe_hw *hw = &adapter->hw;
+	struct ixgbe_nvm_version nvm_ver;
+
+	ixgbe_get_oem_prod_version(hw, &nvm_ver);
+	if (nvm_ver.oem_valid) {
+		snprintf(adapter->eeprom_id, sizeof(adapter->eeprom_id),
+			 "%x.%x.%x", nvm_ver.oem_major, nvm_ver.oem_minor,
+			 nvm_ver.oem_release);
+		return;
+	}
+
+	ixgbe_get_etk_id(hw, &nvm_ver);
+	ixgbe_get_orom_version(hw, &nvm_ver);
+
+	if (nvm_ver.or_valid) {
+		snprintf(adapter->eeprom_id, sizeof(adapter->eeprom_id),
+			 "0x%08x, %d.%d.%d", nvm_ver.etk_id, nvm_ver.or_major,
+			 nvm_ver.or_build, nvm_ver.or_patch);
+		return;
+	}
+
+	/* Set ETrack ID format */
+	snprintf(adapter->eeprom_id, sizeof(adapter->eeprom_id),
+		 "0x%08x", nvm_ver.etk_id);
+}
+
 /**
  * ixgbe_probe - Device Initialization Routine
  * @pdev: PCI device information struct
@@ -10508,6 +10502,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 					 NETIF_F_FCOE_MTU;
 	}
 #endif /* IXGBE_FCOE */
+	ixgbe_init_ipsec_offload(adapter);
 
 	if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
 		netdev->hw_features |= NETIF_F_LRO;
@@ -10568,8 +10563,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
 
 	/* save off EEPROM version number */
-	hw->eeprom.ops.read(hw, 0x2e, &adapter->eeprom_verh);
-	hw->eeprom.ops.read(hw, 0x2d, &adapter->eeprom_verl);
+	ixgbe_set_fw_version(adapter);
 
 	/* pick up the PCI bus settings for reporting later */
 	if (ixgbe_pcie_from_parent(hw))
@@ -10744,6 +10738,7 @@ static void ixgbe_remove(struct pci_dev *pdev)
 	if (netdev->reg_state == NETREG_REGISTERED)
 		unregister_netdev(netdev);
 
+	ixgbe_stop_ipsec_offload(adapter);
 	ixgbe_clear_interrupt_scheme(adapter);
 
 	ixgbe_release_hw_control(adapter);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
index 654a402f0e9e..91bde90f9265 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_phy.c
@@ -378,7 +378,7 @@ static s32 ixgbe_get_phy_id(struct ixgbe_hw *hw)
 
 /**
  *  ixgbe_get_phy_type_from_id - Get the phy type
- *  @hw: pointer to hardware structure
+ *  @phy_id: hardware phy id
  *
  **/
 static enum ixgbe_phy_type ixgbe_get_phy_type_from_id(u32 phy_id)
@@ -489,6 +489,7 @@ s32 ixgbe_reset_phy_generic(struct ixgbe_hw *hw)
  *  the SWFW lock
  *  @hw: pointer to hardware structure
  *  @reg_addr: 32 bit address of PHY register to read
+ *  @device_type: 5 bit device type
  *  @phy_data: Pointer to read data from PHY register
  **/
 s32 ixgbe_read_phy_reg_mdi(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type,
@@ -564,6 +565,7 @@ s32 ixgbe_read_phy_reg_mdi(struct ixgbe_hw *hw, u32 reg_addr, u32 device_type,
  *  using the SWFW lock - this function is needed in most cases
  *  @hw: pointer to hardware structure
  *  @reg_addr: 32 bit address of PHY register to read
+ *  @device_type: 5 bit device type
  *  @phy_data: Pointer to read data from PHY register
  **/
 s32 ixgbe_read_phy_reg_generic(struct ixgbe_hw *hw, u32 reg_addr,
@@ -763,6 +765,7 @@ s32 ixgbe_setup_phy_link_generic(struct ixgbe_hw *hw)
  *  ixgbe_setup_phy_link_speed_generic - Sets the auto advertised capabilities
  *  @hw: pointer to hardware structure
  *  @speed: new link speed
+ *  @autoneg_wait_to_complete: unused
  **/
 s32 ixgbe_setup_phy_link_speed_generic(struct ixgbe_hw *hw,
 				       ixgbe_link_speed speed,
@@ -861,6 +864,8 @@ s32 ixgbe_get_copper_link_capabilities_generic(struct ixgbe_hw *hw,
 /**
  *  ixgbe_check_phy_link_tnx - Determine link and speed status
  *  @hw: pointer to hardware structure
+ *  @speed: link speed
+ *  @link_up: status of link
  *
  *  Reads the VS1 register to determine if link is up and the current speed for
  *  the PHY.
@@ -1667,7 +1672,7 @@ s32 ixgbe_read_i2c_eeprom_generic(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_read_i2c_sff8472_generic - Reads 8 bit word over I2C interface
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset at address 0xA2
- *  @eeprom_data: value read
+ *  @sff8472_data: value read
  *
  *  Performs byte read operation to SFP module's SFF-8472 data over I2C
  **/
@@ -1714,6 +1719,7 @@ static bool ixgbe_is_sfp_probe(struct ixgbe_hw *hw, u8 offset, u8 addr)
  *  ixgbe_read_i2c_byte_generic_int - Reads 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to read
+ *  @dev_addr: device address
  *  @data: value read
  *  @lock: true if to take and release semaphore
  *
@@ -1804,6 +1810,7 @@ static s32 ixgbe_read_i2c_byte_generic_int(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_read_i2c_byte_generic - Reads 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to read
+ *  @dev_addr: device address
  *  @data: value read
  *
  *  Performs byte read operation to SFP module's EEPROM over I2C interface at
@@ -1820,6 +1827,7 @@ s32 ixgbe_read_i2c_byte_generic(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_read_i2c_byte_generic_unlocked - Reads 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to read
+ *  @dev_addr: device address
  *  @data: value read
  *
  *  Performs byte read operation to SFP module's EEPROM over I2C interface at
@@ -1836,6 +1844,7 @@ s32 ixgbe_read_i2c_byte_generic_unlocked(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_write_i2c_byte_generic_int - Writes 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to write
+ *  @dev_addr: device address
  *  @data: value to write
  *  @lock: true if to take and release semaphore
  *
@@ -1904,6 +1913,7 @@ static s32 ixgbe_write_i2c_byte_generic_int(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_write_i2c_byte_generic - Writes 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to write
+ *  @dev_addr: device address
  *  @data: value to write
  *
  *  Performs byte write operation to SFP module's EEPROM over I2C interface at
@@ -1920,6 +1930,7 @@ s32 ixgbe_write_i2c_byte_generic(struct ixgbe_hw *hw, u8 byte_offset,
  *  ixgbe_write_i2c_byte_generic_unlocked - Writes 8 bit word over I2C
  *  @hw: pointer to hardware structure
  *  @byte_offset: byte offset to write
+ *  @dev_addr: device address
  *  @data: value to write
  *
  *  Performs byte write operation to SFP module's EEPROM over I2C interface at
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
index ae312c45696a..f6cc9166082a 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c
@@ -166,7 +166,7 @@
 
 /**
  * ixgbe_ptp_setup_sdp_x540
- * @hw: the hardware private structure
+ * @adapter: private adapter structure
  *
  * this function enables or disables the clock out feature on SDP0 for
  * the X540 device. It will create a 1second periodic output that can
@@ -299,7 +299,7 @@ static u64 ixgbe_ptp_read_82599(const struct cyclecounter *cc)
  * ixgbe_ptp_convert_to_hwtstamp - convert register value to hw timestamp
  * @adapter: private adapter structure
  * @hwtstamp: stack timestamp structure
- * @systim: unsigned 64bit system time value
+ * @timestamp: unsigned 64bit system time value
  *
  * We need to convert the adapter's RX/TXSTMP registers into a hwtstamp value
  * which can be used by the stack's ptp functions.
@@ -1015,7 +1015,7 @@ static int ixgbe_ptp_set_timestamp_mode(struct ixgbe_adapter *adapter,
 /**
  * ixgbe_ptp_set_ts_config - user entry point for timestamp mode
  * @adapter: pointer to adapter struct
- * @ifreq: ioctl data
+ * @ifr: ioctl data
  *
  * Set hardware to requested mode. If unsupported, return an error with no
  * changes. Otherwise, store the mode for future reference.
@@ -1338,7 +1338,7 @@ void ixgbe_ptp_init(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_ptp_suspend - stop PTP work items
- * @ adapter: pointer to adapter struct
+ * @adapter: pointer to adapter struct
  *
  * this function suspends PTP activity, and prevents more PTP work from being
  * generated, but does not destroy the PTP clock device.
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
index 112d24c6c9ce..27a70a52f3c9 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
@@ -78,12 +78,9 @@ static int __ixgbe_enable_sriov(struct ixgbe_adapter *adapter,
 	struct ixgbe_hw *hw = &adapter->hw;
 	int i;
 
-	adapter->flags |= IXGBE_FLAG_SRIOV_ENABLED;
-
 	/* Enable VMDq flag so device will be set in VM mode */
-	adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED;
-	if (!adapter->ring_feature[RING_F_VMDQ].limit)
-		adapter->ring_feature[RING_F_VMDQ].limit = 1;
+	adapter->flags |= IXGBE_FLAG_SRIOV_ENABLED |
+			  IXGBE_FLAG_VMDQ_ENABLED;
 
 	/* Allocate memory for per VF control structures */
 	adapter->vfinfo = kcalloc(num_vfs, sizeof(struct vf_data_storage),
@@ -227,9 +224,6 @@ void ixgbe_enable_sriov(struct ixgbe_adapter *adapter, unsigned int max_vfs)
 int ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 {
 	unsigned int num_vfs = adapter->num_vfs, vf;
-	struct ixgbe_hw *hw = &adapter->hw;
-	u32 gpie;
-	u32 vmdctl;
 	int rss;
 
 	/* set num VFs to 0 to prevent access to vfinfo */
@@ -271,18 +265,6 @@ int ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 	pci_disable_sriov(adapter->pdev);
 #endif
 
-	/* turn off device IOV mode */
-	IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, 0);
-	gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
-	gpie &= ~IXGBE_GPIE_VTMODE_MASK;
-	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
-
-	/* set default pool back to 0 */
-	vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
-	vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
-	IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
-	IXGBE_WRITE_FLUSH(hw);
-
 	/* Disable VMDq flag so device will be set in VM mode */
 	if (adapter->ring_feature[RING_F_VMDQ].limit == 1) {
 		adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
@@ -305,10 +287,9 @@ static int ixgbe_pci_sriov_enable(struct pci_dev *dev, int num_vfs)
 {
 #ifdef CONFIG_PCI_IOV
 	struct ixgbe_adapter *adapter = pci_get_drvdata(dev);
-	int err = 0;
-	u8 num_tc;
-	int i;
 	int pre_existing_vfs = pci_num_vf(dev);
+	int err = 0, num_rx_pools, i, limit;
+	u8 num_tc;
 
 	if (pre_existing_vfs && pre_existing_vfs != num_vfs)
 		err = ixgbe_disable_sriov(adapter);
@@ -330,23 +311,15 @@ static int ixgbe_pci_sriov_enable(struct pci_dev *dev, int num_vfs)
 	 * than we have available pools. The PCI bus driver already checks for
 	 * other values out of range.
 	 */
-	num_tc = netdev_get_num_tc(adapter->netdev);
-
-	if (num_tc > 4) {
-		if ((num_vfs + adapter->num_rx_pools) > IXGBE_MAX_VFS_8TC) {
-			e_dev_err("Currently the device is configured with %d TCs, Creating more than %d VFs is not allowed\n", num_tc, IXGBE_MAX_VFS_8TC);
-			return -EPERM;
-		}
-	} else if ((num_tc > 1) && (num_tc <= 4)) {
-		if ((num_vfs + adapter->num_rx_pools) > IXGBE_MAX_VFS_4TC) {
-			e_dev_err("Currently the device is configured with %d TCs, Creating more than %d VFs is not allowed\n", num_tc, IXGBE_MAX_VFS_4TC);
-			return -EPERM;
-		}
-	} else {
-		if ((num_vfs + adapter->num_rx_pools) > IXGBE_MAX_VFS_1TC) {
-			e_dev_err("Currently the device is configured with %d TCs, Creating more than %d VFs is not allowed\n", num_tc, IXGBE_MAX_VFS_1TC);
-			return -EPERM;
-		}
+	num_tc = adapter->hw_tcs;
+	num_rx_pools = adapter->num_rx_pools;
+	limit = (num_tc > 4) ? IXGBE_MAX_VFS_8TC :
+		(num_tc > 1) ? IXGBE_MAX_VFS_4TC : IXGBE_MAX_VFS_1TC;
+
+	if (num_vfs > (limit - num_rx_pools)) {
+		e_dev_err("Currently configured with %d TCs, and %d offloaded macvlans. Creating more than %d VFs is not allowed\n",
+			  num_tc, num_rx_pools - 1, limit - num_rx_pools);
+		return -EPERM;
 	}
 
 	err = __ixgbe_enable_sriov(adapter, num_vfs);
@@ -378,13 +351,15 @@ static int ixgbe_pci_sriov_disable(struct pci_dev *dev)
 	int err;
 #ifdef CONFIG_PCI_IOV
 	u32 current_flags = adapter->flags;
+	int prev_num_vf = pci_num_vf(dev);
 #endif
 
 	err = ixgbe_disable_sriov(adapter);
 
 	/* Only reinit if no error and state changed */
 #ifdef CONFIG_PCI_IOV
-	if (!err && current_flags != adapter->flags)
+	if (!err && (current_flags != adapter->flags ||
+		     prev_num_vf != pci_num_vf(dev)))
 		ixgbe_sriov_reinit(adapter);
 #endif
 
@@ -738,7 +713,7 @@ static inline void ixgbe_vf_reset_event(struct ixgbe_adapter *adapter, u32 vf)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
 	struct vf_data_storage *vfinfo = &adapter->vfinfo[vf];
-	u8 num_tcs = netdev_get_num_tc(adapter->netdev);
+	u8 num_tcs = adapter->hw_tcs;
 
 	/* remove VLAN filters beloning to this VF */
 	ixgbe_clear_vf_vlans(adapter, vf);
@@ -946,7 +921,7 @@ static int ixgbe_set_vf_vlan_msg(struct ixgbe_adapter *adapter,
 {
 	u32 add = (msgbuf[0] & IXGBE_VT_MSGINFO_MASK) >> IXGBE_VT_MSGINFO_SHIFT;
 	u32 vid = (msgbuf[1] & IXGBE_VLVF_VLANID_MASK);
-	u8 tcs = netdev_get_num_tc(adapter->netdev);
+	u8 tcs = adapter->hw_tcs;
 
 	if (adapter->vfinfo[vf].pf_vlan || tcs) {
 		e_warn(drv,
@@ -1034,7 +1009,7 @@ static int ixgbe_get_vf_queues(struct ixgbe_adapter *adapter,
 	struct net_device *dev = adapter->netdev;
 	struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];
 	unsigned int default_tc = 0;
-	u8 num_tcs = netdev_get_num_tc(dev);
+	u8 num_tcs = adapter->hw_tcs;
 
 	/* verify the PF is supporting the correct APIs */
 	switch (adapter->vfinfo[vf].vf_api) {
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
index ffa0ee5cd0f5..ca45359686d3 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
@@ -235,6 +235,45 @@ struct ixgbe_thermal_sensor_data {
 	struct ixgbe_thermal_diode_data sensor[IXGBE_MAX_SENSORS];
 };
 
+#define NVM_OROM_OFFSET		0x17
+#define NVM_OROM_BLK_LOW	0x83
+#define NVM_OROM_BLK_HI		0x84
+#define NVM_OROM_PATCH_MASK	0xFF
+#define NVM_OROM_SHIFT		8
+
+#define NVM_VER_MASK		0x00FF	/* version mask */
+#define NVM_VER_SHIFT		8	/* version bit shift */
+#define NVM_OEM_PROD_VER_PTR	0x1B /* OEM Product version block pointer */
+#define NVM_OEM_PROD_VER_CAP_OFF 0x1 /* OEM Product version format offset */
+#define NVM_OEM_PROD_VER_OFF_L	0x2  /* OEM Product version offset low */
+#define NVM_OEM_PROD_VER_OFF_H	0x3  /* OEM Product version offset high */
+#define NVM_OEM_PROD_VER_CAP_MASK 0xF /* OEM Product version cap mask */
+#define NVM_OEM_PROD_VER_MOD_LEN 0x3 /* OEM Product version module length */
+#define NVM_ETK_OFF_LOW		0x2D /* version low order word */
+#define NVM_ETK_OFF_HI		0x2E /* version high order word */
+#define NVM_ETK_SHIFT		16   /* high version word shift */
+#define NVM_VER_INVALID		0xFFFF
+#define NVM_ETK_VALID		0x8000
+#define NVM_INVALID_PTR		0xFFFF
+#define NVM_VER_SIZE		32   /* version sting size */
+
+struct ixgbe_nvm_version {
+	u32 etk_id;
+	u8  nvm_major;
+	u16 nvm_minor;
+	u8  nvm_id;
+
+	bool oem_valid;
+	u8   oem_major;
+	u8   oem_minor;
+	u16  oem_release;
+
+	bool or_valid;
+	u8  or_major;
+	u16 or_build;
+	u8  or_patch;
+};
+
 /* Interrupt Registers */
 #define IXGBE_EICR      0x00800
 #define IXGBE_EICS      0x00808
@@ -2321,11 +2360,6 @@ enum {
 #define IXGBE_TXD_CMD_VLE    0x40000000 /* Add VLAN tag */
 #define IXGBE_TXD_STAT_DD    0x00000001 /* Descriptor Done */
 
-#define IXGBE_RXDADV_IPSEC_STATUS_SECP                  0x00020000
-#define IXGBE_RXDADV_IPSEC_ERROR_INVALID_PROTOCOL       0x08000000
-#define IXGBE_RXDADV_IPSEC_ERROR_INVALID_LENGTH         0x10000000
-#define IXGBE_RXDADV_IPSEC_ERROR_AUTH_FAILED            0x18000000
-#define IXGBE_RXDADV_IPSEC_ERROR_BIT_MASK               0x18000000
 /* Multiple Transmit Queue Command Register */
 #define IXGBE_MTQC_RT_ENA       0x1 /* DCB Enable */
 #define IXGBE_MTQC_VT_ENA       0x2 /* VMDQ2 Enable */
@@ -2377,6 +2411,9 @@ enum {
 #define IXGBE_RXDADV_ERR_LE     0x02000000 /* Length Error */
 #define IXGBE_RXDADV_ERR_PE     0x08000000 /* Packet Error */
 #define IXGBE_RXDADV_ERR_OSE    0x10000000 /* Oversize Error */
+#define IXGBE_RXDADV_ERR_IPSEC_INV_PROTOCOL  0x08000000 /* overlap ERR_PE  */
+#define IXGBE_RXDADV_ERR_IPSEC_INV_LENGTH    0x10000000 /* overlap ERR_OSE */
+#define IXGBE_RXDADV_ERR_IPSEC_AUTH_FAILED   0x18000000
 #define IXGBE_RXDADV_ERR_USE    0x20000000 /* Undersize Error */
 #define IXGBE_RXDADV_ERR_TCPE   0x40000000 /* TCP/UDP Checksum Error */
 #define IXGBE_RXDADV_ERR_IPE    0x80000000 /* IP Checksum Error */
@@ -2398,6 +2435,7 @@ enum {
 #define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */
 #define IXGBE_RXDADV_STAT_FCSTAT_DDP    0x00000030 /* 11: Ctxt w/ DDP */
 #define IXGBE_RXDADV_STAT_TS		0x00010000 /* IEEE 1588 Time Stamp */
+#define IXGBE_RXDADV_STAT_SECP		0x00020000 /* IPsec/MACsec pkt found */
 
 /* PSRTYPE bit definitions */
 #define IXGBE_PSRTYPE_TCPHDR    0x00000010
@@ -2464,13 +2502,6 @@ enum {
 #define IXGBE_RXDADV_PKTTYPE_ETQF_MASK  0x00000070 /* ETQF has 8 indices */
 #define IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT 4          /* Right-shift 4 bits */
 
-/* Security Processing bit Indication */
-#define IXGBE_RXDADV_LNKSEC_STATUS_SECP         0x00020000
-#define IXGBE_RXDADV_LNKSEC_ERROR_NO_SA_MATCH   0x08000000
-#define IXGBE_RXDADV_LNKSEC_ERROR_REPLAY_ERROR  0x10000000
-#define IXGBE_RXDADV_LNKSEC_ERROR_BIT_MASK      0x18000000
-#define IXGBE_RXDADV_LNKSEC_ERROR_BAD_SIG       0x18000000
-
 /* Masks to determine if packets should be dropped due to frame errors */
 #define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \
 				      IXGBE_RXD_ERR_CE | \
@@ -2484,6 +2515,8 @@ enum {
 				      IXGBE_RXDADV_ERR_LE | \
 				      IXGBE_RXDADV_ERR_PE | \
 				      IXGBE_RXDADV_ERR_OSE | \
+				      IXGBE_RXDADV_ERR_IPSEC_INV_PROTOCOL | \
+				      IXGBE_RXDADV_ERR_IPSEC_INV_LENGTH | \
 				      IXGBE_RXDADV_ERR_USE)
 
 /* Multicast bit mask */
@@ -2862,7 +2895,7 @@ union ixgbe_adv_rx_desc {
 /* Context descriptors */
 struct ixgbe_adv_tx_context_desc {
 	__le32 vlan_macip_lens;
-	__le32 seqnum_seed;
+	__le32 fceof_saidx;
 	__le32 type_tucmd_mlhl;
 	__le32 mss_l4len_idx;
 };
@@ -2893,6 +2926,7 @@ struct ixgbe_adv_tx_context_desc {
 				 IXGBE_ADVTXD_POPTS_SHIFT)
 #define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \
 				 IXGBE_ADVTXD_POPTS_SHIFT)
+#define IXGBE_ADVTXD_POPTS_IPSEC     0x00000400 /* IPSec offload request */
 #define IXGBE_ADVTXD_POPTS_ISCO_1ST  0x00000000 /* 1st TSO of iSCSI PDU */
 #define IXGBE_ADVTXD_POPTS_ISCO_MDL  0x00000800 /* Middle TSO of iSCSI PDU */
 #define IXGBE_ADVTXD_POPTS_ISCO_LAST 0x00001000 /* Last TSO of iSCSI PDU */
@@ -2908,7 +2942,6 @@ struct ixgbe_adv_tx_context_desc {
 #define IXGBE_ADVTXD_TUCMD_L4T_SCTP  0x00001000  /* L4 Packet TYPE of SCTP */
 #define IXGBE_ADVTXD_TUCMD_L4T_RSV     0x00001800 /* RSV L4 Packet TYPE */
 #define IXGBE_ADVTXD_TUCMD_MKRREQ    0x00002000 /*Req requires Markers and CRC*/
-#define IXGBE_ADVTXD_POPTS_IPSEC      0x00000400 /* IPSec offload request */
 #define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */
 #define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000/* ESP Encrypt Enable */
 #define IXGBE_ADVTXT_TUCMD_FCOE      0x00008000       /* FCoE Frame Type */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
index cb7da5f9c4da..f470d0204771 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
@@ -949,7 +949,7 @@ static s32 ixgbe_checksum_ptr_x550(struct ixgbe_hw *hw, u16 ptr,
 	u16 length, bufsz, i, start;
 	u16 *local_buffer;
 
-	bufsz = sizeof(buf) / sizeof(buf[0]);
+	bufsz = ARRAY_SIZE(buf);
 
 	/* Read a chunk at the pointer location */
 	if (!buffer) {
@@ -1642,10 +1642,12 @@ static s32 ixgbe_supported_sfp_modules_X550em(struct ixgbe_hw *hw, bool *linear)
 }
 
 /**
- *  ixgbe_setup_mac_link_sfp_x550em - Configure the KR PHY for SFP.
- *  @hw: pointer to hardware structure
+ * ixgbe_setup_mac_link_sfp_x550em - Configure the KR PHY for SFP.
+ * @hw: pointer to hardware structure
+ * @speed: the link speed to force
+ * @autoneg_wait_to_complete: unused
  *
- *  Configures the extern PHY and the integrated KR PHY for SFP support.
+ * Configures the extern PHY and the integrated KR PHY for SFP support.
  */
 static s32
 ixgbe_setup_mac_link_sfp_x550em(struct ixgbe_hw *hw,
@@ -1737,6 +1739,8 @@ static s32 ixgbe_setup_sfi_x550a(struct ixgbe_hw *hw, ixgbe_link_speed *speed)
 /**
  * ixgbe_setup_mac_link_sfp_n - Setup internal PHY for native SFP
  * @hw: pointer to hardware structure
+ * @speed: link speed
+ * @autoneg_wait_to_complete: unused
  *
  * Configure the the integrated PHY for native SFP support.
  */
@@ -1784,6 +1788,8 @@ ixgbe_setup_mac_link_sfp_n(struct ixgbe_hw *hw, ixgbe_link_speed speed,
 /**
  * ixgbe_setup_mac_link_sfp_x550a - Setup internal PHY for SFP
  * @hw: pointer to hardware structure
+ * @speed: link speed
+ * @autoneg_wait_to_complete: unused
  *
  * Configure the the integrated PHY for SFP support.
  */
@@ -1859,7 +1865,7 @@ ixgbe_setup_mac_link_sfp_x550a(struct ixgbe_hw *hw, ixgbe_link_speed speed,
  * ixgbe_setup_mac_link_t_X550em - Sets the auto advertised link speed
  * @hw: pointer to hardware structure
  * @speed: new link speed
- * @autoneg_wait_to_complete: true when waiting for completion is needed
+ * @autoneg_wait: true when waiting for completion is needed
  *
  * Setup internal/external PHY link speed based on link speed, then set
  * external PHY auto advertised link speed.
@@ -1943,6 +1949,8 @@ static s32 ixgbe_check_link_t_X550em(struct ixgbe_hw *hw,
 /**
  * ixgbe_setup_sgmii - Set up link for sgmii
  * @hw: pointer to hardware structure
+ * @speed: unused
+ * @autoneg_wait_to_complete: unused
  */
 static s32
 ixgbe_setup_sgmii(struct ixgbe_hw *hw, __always_unused ixgbe_link_speed speed,
@@ -2014,6 +2022,8 @@ ixgbe_setup_sgmii(struct ixgbe_hw *hw, __always_unused ixgbe_link_speed speed,
 /**
  * ixgbe_setup_sgmii_fw - Set up link for sgmii with firmware-controlled PHYs
  * @hw: pointer to hardware structure
+ * @speed: the link speed to force
+ * @autoneg_wait: true when waiting for completion is needed
  */
 static s32 ixgbe_setup_sgmii_fw(struct ixgbe_hw *hw, ixgbe_link_speed speed,
 				bool autoneg_wait)
@@ -3735,6 +3745,7 @@ static void ixgbe_release_swfw_sync_x550em_a(struct ixgbe_hw *hw, u32 mask)
  * ixgbe_read_phy_reg_x550a - Reads specified PHY register
  * @hw: pointer to hardware structure
  * @reg_addr: 32 bit address of PHY register to read
+ * @device_type: 5 bit device type
  * @phy_data: Pointer to read data from PHY register
  *
  * Reads a value from a specified PHY register using the SWFW lock and PHY
diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
index ff9d05f308ee..4400e49090b4 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
@@ -75,6 +75,9 @@ static struct ixgbe_stats ixgbevf_gstrings_stats[] = {
 	IXGBEVF_STAT("tx_timeout_count", tx_timeout_count),
 	IXGBEVF_NETDEV_STAT(multicast),
 	IXGBEVF_STAT("rx_csum_offload_errors", hw_csum_rx_error),
+	IXGBEVF_STAT("alloc_rx_page", alloc_rx_page),
+	IXGBEVF_STAT("alloc_rx_page_failed", alloc_rx_page_failed),
+	IXGBEVF_STAT("alloc_rx_buff_failed", alloc_rx_buff_failed),
 };
 
 #define IXGBEVF_QUEUE_STATS_LEN ( \
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
index 581f44bbd7b3..f6952425c87d 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h
@@ -62,7 +62,12 @@ struct ixgbevf_tx_buffer {
 struct ixgbevf_rx_buffer {
 	dma_addr_t dma;
 	struct page *page;
-	unsigned int page_offset;
+#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536)
+	__u32 page_offset;
+#else
+	__u16 page_offset;
+#endif
+	__u16 pagecnt_bias;
 };
 
 struct ixgbevf_stats {
@@ -79,6 +84,7 @@ struct ixgbevf_tx_queue_stats {
 struct ixgbevf_rx_queue_stats {
 	u64 alloc_rx_page_failed;
 	u64 alloc_rx_buff_failed;
+	u64 alloc_rx_page;
 	u64 csum_err;
 };
 
@@ -260,6 +266,9 @@ static inline void ixgbevf_write_tail(struct ixgbevf_ring *ring, u32 value)
 #define MIN_MSIX_Q_VECTORS	1
 #define MIN_MSIX_COUNT		(MIN_MSIX_Q_VECTORS + NON_Q_VECTORS)
 
+#define IXGBEVF_RX_DMA_ATTR \
+	(DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
+
 /* board specific private data structure */
 struct ixgbevf_adapter {
 	/* this field must be first, see ixgbevf_process_skb_fields */
@@ -287,8 +296,9 @@ struct ixgbevf_adapter {
 	u64 hw_csum_rx_error;
 	u64 hw_rx_no_dma_resources;
 	int num_msix_vectors;
-	u32 alloc_rx_page_failed;
-	u32 alloc_rx_buff_failed;
+	u64 alloc_rx_page_failed;
+	u64 alloc_rx_buff_failed;
+	u64 alloc_rx_page;
 
 	struct msix_entry *msix_entries;
 
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 1f4a69134ade..9b3d43d28106 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -206,28 +206,6 @@ static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 	}
 }
 
-static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
-					struct ixgbevf_tx_buffer *tx_buffer)
-{
-	if (tx_buffer->skb) {
-		dev_kfree_skb_any(tx_buffer->skb);
-		if (dma_unmap_len(tx_buffer, len))
-			dma_unmap_single(tx_ring->dev,
-					 dma_unmap_addr(tx_buffer, dma),
-					 dma_unmap_len(tx_buffer, len),
-					 DMA_TO_DEVICE);
-	} else if (dma_unmap_len(tx_buffer, len)) {
-		dma_unmap_page(tx_ring->dev,
-			       dma_unmap_addr(tx_buffer, dma),
-			       dma_unmap_len(tx_buffer, len),
-			       DMA_TO_DEVICE);
-	}
-	tx_buffer->next_to_watch = NULL;
-	tx_buffer->skb = NULL;
-	dma_unmap_len_set(tx_buffer, len, 0);
-	/* tx_buffer must be completely set up in the transmit path */
-}
-
 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring)
 {
 	return ring->stats.packets;
@@ -349,7 +327,6 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 				 DMA_TO_DEVICE);
 
 		/* clear tx_buffer data */
-		tx_buffer->skb = NULL;
 		dma_unmap_len_set(tx_buffer, len, 0);
 
 		/* unmap remaining buffers */
@@ -554,7 +531,6 @@ static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
  * ixgbevf_is_non_eop - process handling of non-EOP buffers
  * @rx_ring: Rx ring being processed
  * @rx_desc: Rx descriptor for current buffer
- * @skb: current socket buffer containing buffer in progress
  *
  * This function updates next to clean.  If the buffer is an EOP buffer
  * this function exits returning false, otherwise it will place the
@@ -596,8 +572,8 @@ static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
 	}
 
 	/* map page for use */
-	dma = dma_map_page(rx_ring->dev, page, 0,
-			   PAGE_SIZE, DMA_FROM_DEVICE);
+	dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE,
+				 DMA_FROM_DEVICE, IXGBEVF_RX_DMA_ATTR);
 
 	/* if mapping failed free memory back to system since
 	 * there isn't much point in holding memory we can't use
@@ -605,13 +581,15 @@ static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
 	if (dma_mapping_error(rx_ring->dev, dma)) {
 		__free_page(page);
 
-		rx_ring->rx_stats.alloc_rx_buff_failed++;
+		rx_ring->rx_stats.alloc_rx_page_failed++;
 		return false;
 	}
 
 	bi->dma = dma;
 	bi->page = page;
 	bi->page_offset = 0;
+	bi->pagecnt_bias = 1;
+	rx_ring->rx_stats.alloc_rx_page++;
 
 	return true;
 }
@@ -640,6 +618,12 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
 		if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
 			break;
 
+		/* sync the buffer for use by the device */
+		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
+						 bi->page_offset,
+						 IXGBEVF_RX_BUFSZ,
+						 DMA_FROM_DEVICE);
+
 		/* Refresh the desc even if pkt_addr didn't change
 		 * because each write-back erases this info.
 		 */
@@ -654,8 +638,8 @@ static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
 			i -= rx_ring->count;
 		}
 
-		/* clear the hdr_addr for the next_to_use descriptor */
-		rx_desc->read.hdr_addr = 0;
+		/* clear the length for the next_to_use descriptor */
+		rx_desc->wb.upper.length = 0;
 
 		cleaned_count--;
 	} while (cleaned_count);
@@ -742,12 +726,7 @@ static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
 	new_buff->page = old_buff->page;
 	new_buff->dma = old_buff->dma;
 	new_buff->page_offset = old_buff->page_offset;
-
-	/* sync the buffer for use by the device */
-	dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
-					 new_buff->page_offset,
-					 IXGBEVF_RX_BUFSZ,
-					 DMA_FROM_DEVICE);
+	new_buff->pagecnt_bias = old_buff->pagecnt_bias;
 }
 
 static inline bool ixgbevf_page_is_reserved(struct page *page)
@@ -755,6 +734,45 @@ static inline bool ixgbevf_page_is_reserved(struct page *page)
 	return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
 }
 
+static bool ixgbevf_can_reuse_rx_page(struct ixgbevf_rx_buffer *rx_buffer,
+				      struct page *page,
+				      const unsigned int truesize)
+{
+	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias--;
+
+	/* avoid re-using remote pages */
+	if (unlikely(ixgbevf_page_is_reserved(page)))
+		return false;
+
+#if (PAGE_SIZE < 8192)
+	/* if we are only owner of page we can reuse it */
+	if (unlikely(page_ref_count(page) != pagecnt_bias))
+		return false;
+
+	/* flip page offset to other buffer */
+	rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ;
+
+#else
+	/* move offset up to the next cache line */
+	rx_buffer->page_offset += truesize;
+
+	if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ))
+		return false;
+
+#endif
+
+	/* If we have drained the page fragment pool we need to update
+	 * the pagecnt_bias and page count so that we fully restock the
+	 * number of references the driver holds.
+	 */
+	if (unlikely(pagecnt_bias == 1)) {
+		page_ref_add(page, USHRT_MAX);
+		rx_buffer->pagecnt_bias = USHRT_MAX;
+	}
+
+	return true;
+}
+
 /**
  * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
  * @rx_ring: rx descriptor ring to transact packets on
@@ -772,12 +790,12 @@ static inline bool ixgbevf_page_is_reserved(struct page *page)
  **/
 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
 				struct ixgbevf_rx_buffer *rx_buffer,
+				u16 size,
 				union ixgbe_adv_rx_desc *rx_desc,
 				struct sk_buff *skb)
 {
 	struct page *page = rx_buffer->page;
 	unsigned char *va = page_address(page) + rx_buffer->page_offset;
-	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
 #if (PAGE_SIZE < 8192)
 	unsigned int truesize = IXGBEVF_RX_BUFSZ;
 #else
@@ -796,7 +814,6 @@ static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
 			return true;
 
 		/* this page cannot be reused so discard it */
-		put_page(page);
 		return false;
 	}
 
@@ -816,32 +833,7 @@ static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
 			(unsigned long)va & ~PAGE_MASK, size, truesize);
 
-	/* avoid re-using remote pages */
-	if (unlikely(ixgbevf_page_is_reserved(page)))
-		return false;
-
-#if (PAGE_SIZE < 8192)
-	/* if we are only owner of page we can reuse it */
-	if (unlikely(page_count(page) != 1))
-		return false;
-
-	/* flip page offset to other buffer */
-	rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ;
-
-#else
-	/* move offset up to the next cache line */
-	rx_buffer->page_offset += truesize;
-
-	if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ))
-		return false;
-
-#endif
-	/* Even if we own the page, we are not allowed to use atomic_set()
-	 * This would break get_page_unless_zero() users.
-	 */
-	page_ref_inc(page);
-
-	return true;
+	return ixgbevf_can_reuse_rx_page(rx_buffer, page, truesize);
 }
 
 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,
@@ -850,11 +842,19 @@ static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,
 {
 	struct ixgbevf_rx_buffer *rx_buffer;
 	struct page *page;
+	u16 size = le16_to_cpu(rx_desc->wb.upper.length);
 
 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
 	page = rx_buffer->page;
 	prefetchw(page);
 
+	/* we are reusing so sync this buffer for CPU use */
+	dma_sync_single_range_for_cpu(rx_ring->dev,
+				      rx_buffer->dma,
+				      rx_buffer->page_offset,
+				      size,
+				      DMA_FROM_DEVICE);
+
 	if (likely(!skb)) {
 		void *page_addr = page_address(page) +
 				  rx_buffer->page_offset;
@@ -880,21 +880,18 @@ static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,
 		prefetchw(skb->data);
 	}
 
-	/* we are reusing so sync this buffer for CPU use */
-	dma_sync_single_range_for_cpu(rx_ring->dev,
-				      rx_buffer->dma,
-				      rx_buffer->page_offset,
-				      IXGBEVF_RX_BUFSZ,
-				      DMA_FROM_DEVICE);
-
 	/* pull page into skb */
-	if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
+	if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, size, rx_desc, skb)) {
 		/* hand second half of page back to the ring */
 		ixgbevf_reuse_rx_page(rx_ring, rx_buffer);
 	} else {
-		/* we are not reusing the buffer so unmap it */
-		dma_unmap_page(rx_ring->dev, rx_buffer->dma,
-			       PAGE_SIZE, DMA_FROM_DEVICE);
+		/* We are not reusing the buffer so unmap it and free
+		 * any references we are holding to it
+		 */
+		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
+				     PAGE_SIZE, DMA_FROM_DEVICE,
+				     IXGBEVF_RX_DMA_ATTR);
+		__page_frag_cache_drain(page, rx_buffer->pagecnt_bias);
 	}
 
 	/* clear contents of buffer_info */
@@ -931,7 +928,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 
 		rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
 
-		if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
+		if (!rx_desc->wb.upper.length)
 			break;
 
 		/* This memory barrier is needed to keep us from reading
@@ -944,8 +941,10 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 		skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb);
 
 		/* exit if we failed to retrieve a buffer */
-		if (!skb)
+		if (!skb) {
+			rx_ring->rx_stats.alloc_rx_buff_failed++;
 			break;
+		}
 
 		cleaned_count++;
 
@@ -1554,6 +1553,10 @@ static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
 	txdctl |= (1u << 8) |    /* HTHRESH = 1 */
 		   32;           /* PTHRESH = 32 */
 
+	/* reinitialize tx_buffer_info */
+	memset(ring->tx_buffer_info, 0,
+	       sizeof(struct ixgbevf_tx_buffer) * ring->count);
+
 	clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state);
 
 	IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
@@ -1722,6 +1725,7 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
 				      struct ixgbevf_ring *ring)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
+	union ixgbe_adv_rx_desc *rx_desc;
 	u64 rdba = ring->dma;
 	u32 rxdctl;
 	u8 reg_idx = ring->reg_idx;
@@ -1750,6 +1754,14 @@ static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
 	IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
 	ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
 
+	/* initialize rx_buffer_info */
+	memset(ring->rx_buffer_info, 0,
+	       sizeof(struct ixgbevf_rx_buffer) * ring->count);
+
+	/* initialize Rx descriptor 0 */
+	rx_desc = IXGBEVF_RX_DESC(ring, 0);
+	rx_desc->wb.upper.length = 0;
+
 	/* reset ntu and ntc to place SW in sync with hardwdare */
 	ring->next_to_clean = 0;
 	ring->next_to_use = 0;
@@ -1896,10 +1908,6 @@ static void ixgbevf_set_rx_mode(struct net_device *netdev)
 	unsigned int flags = netdev->flags;
 	int xcast_mode;
 
-	xcast_mode = (flags & IFF_ALLMULTI) ? IXGBEVF_XCAST_MODE_ALLMULTI :
-		     (flags & (IFF_BROADCAST | IFF_MULTICAST)) ?
-		     IXGBEVF_XCAST_MODE_MULTI : IXGBEVF_XCAST_MODE_NONE;
-
 	/* request the most inclusive mode we need */
 	if (flags & IFF_PROMISC)
 		xcast_mode = IXGBEVF_XCAST_MODE_PROMISC;
@@ -2108,9 +2116,7 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter)
  **/
 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
 {
-	struct device *dev = rx_ring->dev;
-	unsigned long size;
-	unsigned int i;
+	u16 i = rx_ring->next_to_clean;
 
 	/* Free Rx ring sk_buff */
 	if (rx_ring->skb) {
@@ -2118,29 +2124,39 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
 		rx_ring->skb = NULL;
 	}
 
-	/* ring already cleared, nothing to do */
-	if (!rx_ring->rx_buffer_info)
-		return;
-
 	/* Free all the Rx ring pages */
-	for (i = 0; i < rx_ring->count; i++) {
+	while (i != rx_ring->next_to_alloc) {
 		struct ixgbevf_rx_buffer *rx_buffer;
 
 		rx_buffer = &rx_ring->rx_buffer_info[i];
-		if (rx_buffer->dma)
-			dma_unmap_page(dev, rx_buffer->dma,
-				       PAGE_SIZE, DMA_FROM_DEVICE);
-		rx_buffer->dma = 0;
-		if (rx_buffer->page)
-			__free_page(rx_buffer->page);
-		rx_buffer->page = NULL;
-	}
 
-	size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
-	memset(rx_ring->rx_buffer_info, 0, size);
+		/* Invalidate cache lines that may have been written to by
+		 * device so that we avoid corrupting memory.
+		 */
+		dma_sync_single_range_for_cpu(rx_ring->dev,
+					      rx_buffer->dma,
+					      rx_buffer->page_offset,
+					      IXGBEVF_RX_BUFSZ,
+					      DMA_FROM_DEVICE);
+
+		/* free resources associated with mapping */
+		dma_unmap_page_attrs(rx_ring->dev,
+				     rx_buffer->dma,
+				     PAGE_SIZE,
+				     DMA_FROM_DEVICE,
+				     IXGBEVF_RX_DMA_ATTR);
+
+		__page_frag_cache_drain(rx_buffer->page,
+					rx_buffer->pagecnt_bias);
 
-	/* Zero out the descriptor ring */
-	memset(rx_ring->desc, 0, rx_ring->size);
+		i++;
+		if (i == rx_ring->count)
+			i = 0;
+	}
+
+	rx_ring->next_to_alloc = 0;
+	rx_ring->next_to_clean = 0;
+	rx_ring->next_to_use = 0;
 }
 
 /**
@@ -2149,23 +2165,57 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
  **/
 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
 {
-	struct ixgbevf_tx_buffer *tx_buffer_info;
-	unsigned long size;
-	unsigned int i;
+	u16 i = tx_ring->next_to_clean;
+	struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
 
-	if (!tx_ring->tx_buffer_info)
-		return;
+	while (i != tx_ring->next_to_use) {
+		union ixgbe_adv_tx_desc *eop_desc, *tx_desc;
 
-	/* Free all the Tx ring sk_buffs */
-	for (i = 0; i < tx_ring->count; i++) {
-		tx_buffer_info = &tx_ring->tx_buffer_info[i];
-		ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
+		/* Free all the Tx ring sk_buffs */
+		dev_kfree_skb_any(tx_buffer->skb);
+
+		/* unmap skb header data */
+		dma_unmap_single(tx_ring->dev,
+				 dma_unmap_addr(tx_buffer, dma),
+				 dma_unmap_len(tx_buffer, len),
+				 DMA_TO_DEVICE);
+
+		/* check for eop_desc to determine the end of the packet */
+		eop_desc = tx_buffer->next_to_watch;
+		tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
+
+		/* unmap remaining buffers */
+		while (tx_desc != eop_desc) {
+			tx_buffer++;
+			tx_desc++;
+			i++;
+			if (unlikely(i == tx_ring->count)) {
+				i = 0;
+				tx_buffer = tx_ring->tx_buffer_info;
+				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
+			}
+
+			/* unmap any remaining paged data */
+			if (dma_unmap_len(tx_buffer, len))
+				dma_unmap_page(tx_ring->dev,
+					       dma_unmap_addr(tx_buffer, dma),
+					       dma_unmap_len(tx_buffer, len),
+					       DMA_TO_DEVICE);
+		}
+
+		/* move us one more past the eop_desc for start of next pkt */
+		tx_buffer++;
+		i++;
+		if (unlikely(i == tx_ring->count)) {
+			i = 0;
+			tx_buffer = tx_ring->tx_buffer_info;
+		}
 	}
 
-	size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
-	memset(tx_ring->tx_buffer_info, 0, size);
+	/* reset next_to_use and next_to_clean */
+	tx_ring->next_to_use = 0;
+	tx_ring->next_to_clean = 0;
 
-	memset(tx_ring->desc, 0, tx_ring->size);
 }
 
 /**
@@ -2717,6 +2767,8 @@ static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
 {
 	struct ixgbe_hw *hw = &adapter->hw;
+	u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
+	u64 alloc_rx_page = 0, hw_csum_rx_error = 0;
 	int i;
 
 	if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
@@ -2737,15 +2789,23 @@ void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
 				adapter->stats.vfmprc);
 
 	for (i = 0;  i  < adapter->num_rx_queues;  i++) {
-		adapter->hw_csum_rx_error +=
-			adapter->rx_ring[i]->hw_csum_rx_error;
-		adapter->rx_ring[i]->hw_csum_rx_error = 0;
+		struct ixgbevf_ring *rx_ring = adapter->rx_ring[i];
+
+		hw_csum_rx_error += rx_ring->rx_stats.csum_err;
+		alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
+		alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
+		alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;
 	}
+
+	adapter->hw_csum_rx_error = hw_csum_rx_error;
+	adapter->alloc_rx_page_failed = alloc_rx_page_failed;
+	adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
+	adapter->alloc_rx_page = alloc_rx_page;
 }
 
 /**
  * ixgbevf_service_timer - Timer Call-back
- * @data: pointer to adapter cast into an unsigned long
+ * @t: pointer to timer_list struct
  **/
 static void ixgbevf_service_timer(struct timer_list *t)
 {
@@ -2888,7 +2948,7 @@ static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter)
 
 /**
  * ixgbevf_watchdog_subtask - worker thread to bring link up
- * @work: pointer to work_struct containing our data
+ * @adapter: board private structure
  **/
 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter)
 {
@@ -2985,7 +3045,7 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
 	int size;
 
 	size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
-	tx_ring->tx_buffer_info = vzalloc(size);
+	tx_ring->tx_buffer_info = vmalloc(size);
 	if (!tx_ring->tx_buffer_info)
 		goto err;
 
@@ -3045,7 +3105,7 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
 	int size;
 
 	size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
-	rx_ring->rx_buffer_info = vzalloc(size);
+	rx_ring->rx_buffer_info = vmalloc(size);
 	if (!rx_ring->rx_buffer_info)
 		goto err;
 
@@ -3487,34 +3547,37 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 			   struct ixgbevf_tx_buffer *first,
 			   const u8 hdr_len)
 {
-	dma_addr_t dma;
 	struct sk_buff *skb = first->skb;
 	struct ixgbevf_tx_buffer *tx_buffer;
 	union ixgbe_adv_tx_desc *tx_desc;
-	struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
-	unsigned int data_len = skb->data_len;
-	unsigned int size = skb_headlen(skb);
-	unsigned int paylen = skb->len - hdr_len;
+	struct skb_frag_struct *frag;
+	dma_addr_t dma;
+	unsigned int data_len, size;
 	u32 tx_flags = first->tx_flags;
-	__le32 cmd_type;
+	__le32 cmd_type = ixgbevf_tx_cmd_type(tx_flags);
 	u16 i = tx_ring->next_to_use;
 
 	tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
 
-	ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
-	cmd_type = ixgbevf_tx_cmd_type(tx_flags);
+	ixgbevf_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);
+
+	size = skb_headlen(skb);
+	data_len = skb->data_len;
 
 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
-	if (dma_mapping_error(tx_ring->dev, dma))
-		goto dma_error;
 
-	/* record length, and DMA address */
-	dma_unmap_len_set(first, len, size);
-	dma_unmap_addr_set(first, dma, dma);
+	tx_buffer = first;
 
-	tx_desc->read.buffer_addr = cpu_to_le64(dma);
+	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
+		if (dma_mapping_error(tx_ring->dev, dma))
+			goto dma_error;
+
+		/* record length, and DMA address */
+		dma_unmap_len_set(tx_buffer, len, size);
+		dma_unmap_addr_set(tx_buffer, dma, dma);
+
+		tx_desc->read.buffer_addr = cpu_to_le64(dma);
 
-	for (;;) {
 		while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
 			tx_desc->read.cmd_type_len =
 				cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
@@ -3525,12 +3588,12 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 				tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 				i = 0;
 			}
+			tx_desc->read.olinfo_status = 0;
 
 			dma += IXGBE_MAX_DATA_PER_TXD;
 			size -= IXGBE_MAX_DATA_PER_TXD;
 
 			tx_desc->read.buffer_addr = cpu_to_le64(dma);
-			tx_desc->read.olinfo_status = 0;
 		}
 
 		if (likely(!data_len))
@@ -3544,23 +3607,15 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 			tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 			i = 0;
 		}
+		tx_desc->read.olinfo_status = 0;
 
 		size = skb_frag_size(frag);
 		data_len -= size;
 
 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
 				       DMA_TO_DEVICE);
-		if (dma_mapping_error(tx_ring->dev, dma))
-			goto dma_error;
 
 		tx_buffer = &tx_ring->tx_buffer_info[i];
-		dma_unmap_len_set(tx_buffer, len, size);
-		dma_unmap_addr_set(tx_buffer, dma, dma);
-
-		tx_desc->read.buffer_addr = cpu_to_le64(dma);
-		tx_desc->read.olinfo_status = 0;
-
-		frag++;
 	}
 
 	/* write last descriptor with RS and EOP bits */
@@ -3594,18 +3649,32 @@ static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
 	return;
 dma_error:
 	dev_err(tx_ring->dev, "TX DMA map failed\n");
+	tx_buffer = &tx_ring->tx_buffer_info[i];
 
 	/* clear dma mappings for failed tx_buffer_info map */
-	for (;;) {
+	while (tx_buffer != first) {
+		if (dma_unmap_len(tx_buffer, len))
+			dma_unmap_page(tx_ring->dev,
+				       dma_unmap_addr(tx_buffer, dma),
+				       dma_unmap_len(tx_buffer, len),
+				       DMA_TO_DEVICE);
+		dma_unmap_len_set(tx_buffer, len, 0);
+
+		if (i-- == 0)
+			i += tx_ring->count;
 		tx_buffer = &tx_ring->tx_buffer_info[i];
-		ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
-		if (tx_buffer == first)
-			break;
-		if (i == 0)
-			i = tx_ring->count;
-		i--;
 	}
 
+	if (dma_unmap_len(tx_buffer, len))
+		dma_unmap_single(tx_ring->dev,
+				 dma_unmap_addr(tx_buffer, dma),
+				 dma_unmap_len(tx_buffer, len),
+				 DMA_TO_DEVICE);
+	dma_unmap_len_set(tx_buffer, len, 0);
+
+	dev_kfree_skb_any(tx_buffer->skb);
+	tx_buffer->skb = NULL;
+
 	tx_ring->next_to_use = i;
 }
 
@@ -4368,6 +4437,7 @@ static void __exit ixgbevf_exit_module(void)
 /**
  * ixgbevf_get_hw_dev_name - return device name string
  * used by hardware layer to print debugging information
+ * @hw: pointer to private hardware struct
  **/
 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
 {
diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.c b/drivers/net/ethernet/intel/ixgbevf/vf.c
index 0c25006ce9af..38d3a327c1bc 100644
--- a/drivers/net/ethernet/intel/ixgbevf/vf.c
+++ b/drivers/net/ethernet/intel/ixgbevf/vf.c
@@ -146,6 +146,7 @@ static s32 ixgbevf_reset_hw_vf(struct ixgbe_hw *hw)
 /**
  * Hyper-V variant; the VF/PF communication is through the PCI
  * config space.
+ * @hw: pointer to private hardware struct
  */
 static s32 ixgbevf_hv_reset_hw_vf(struct ixgbe_hw *hw)
 {
@@ -285,7 +286,7 @@ static s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr)
 		ether_addr_copy(msg_addr, addr);
 
 	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
-					     sizeof(msgbuf) / sizeof(u32));
+					     ARRAY_SIZE(msgbuf));
 	if (!ret_val) {
 		msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
@@ -303,7 +304,7 @@ static s32 ixgbevf_hv_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr)
 
 /**
  * ixgbevf_get_reta_locked - get the RSS redirection table (RETA) contents.
- * @adapter: pointer to the port handle
+ * @hw: pointer to hardware structure
  * @reta: buffer to fill with RETA contents.
  * @num_rx_queues: Number of Rx queues configured for this port
  *
@@ -455,8 +456,7 @@ static s32 ixgbevf_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr,
 	ether_addr_copy(msg_addr, addr);
 
 	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
-					     sizeof(msgbuf) / sizeof(u32));
-
+					     ARRAY_SIZE(msgbuf));
 	msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
 	/* if nacked the address was rejected, use "perm_addr" */
@@ -536,6 +536,8 @@ static s32 ixgbevf_update_mc_addr_list_vf(struct ixgbe_hw *hw,
 
 /**
  * Hyper-V variant - just a stub.
+ * @hw: unused
+ * @netdev: unused
  */
 static s32 ixgbevf_hv_update_mc_addr_list_vf(struct ixgbe_hw *hw,
 					     struct net_device *netdev)
@@ -571,7 +573,7 @@ static s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode)
 	msgbuf[1] = xcast_mode;
 
 	err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
-					 sizeof(msgbuf) / sizeof(u32));
+					 ARRAY_SIZE(msgbuf));
 	if (err)
 		return err;
 
@@ -584,6 +586,8 @@ static s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode)
 
 /**
  * Hyper-V variant - just a stub.
+ * @hw: unused
+ * @xcast_mode: unused
  */
 static s32 ixgbevf_hv_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode)
 {
@@ -609,7 +613,7 @@ static s32 ixgbevf_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind,
 	msgbuf[0] |= vlan_on << IXGBE_VT_MSGINFO_SHIFT;
 
 	err = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
-					 sizeof(msgbuf) / sizeof(u32));
+					 ARRAY_SIZE(msgbuf));
 	if (err)
 		goto mbx_err;
 
@@ -626,6 +630,10 @@ static s32 ixgbevf_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind,
 
 /**
  * Hyper-V variant - just a stub.
+ * @hw: unused
+ * @vlan: unused
+ * @vind: unused
+ * @vlan_on: unused
  */
 static s32 ixgbevf_hv_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind,
 				  bool vlan_on)
@@ -655,7 +663,7 @@ static s32 ixgbevf_setup_mac_link_vf(struct ixgbe_hw *hw,
  *  @hw: pointer to hardware structure
  *  @speed: pointer to link speed
  *  @link_up: true is link is up, false otherwise
- *  @autoneg_wait_to_complete: true when waiting for completion is needed
+ *  @autoneg_wait_to_complete: unused
  *
  *  Reads the links register to determine if link is up and the current speed
  **/
@@ -740,6 +748,10 @@ static s32 ixgbevf_check_mac_link_vf(struct ixgbe_hw *hw,
 
 /**
  * Hyper-V variant; there is no mailbox communication.
+ * @hw: pointer to private hardware struct
+ * @speed: pointer to link speed
+ * @link_up: true is link is up, false otherwise
+ * @autoneg_wait_to_complete: unused
  */
 static s32 ixgbevf_hv_check_mac_link_vf(struct ixgbe_hw *hw,
 					ixgbe_link_speed *speed,
@@ -813,7 +825,7 @@ static s32 ixgbevf_set_rlpml_vf(struct ixgbe_hw *hw, u16 max_size)
 	msgbuf[1] = max_size;
 
 	ret_val = ixgbevf_write_msg_read_ack(hw, msgbuf, msgbuf,
-					     sizeof(msgbuf) / sizeof(u32));
+					     ARRAY_SIZE(msgbuf));
 	if (ret_val)
 		return ret_val;
 	if ((msgbuf[0] & IXGBE_VF_SET_LPE) &&
@@ -859,8 +871,7 @@ static int ixgbevf_negotiate_api_version_vf(struct ixgbe_hw *hw, int api)
 	msg[1] = api;
 	msg[2] = 0;
 
-	err = ixgbevf_write_msg_read_ack(hw, msg, msg,
-					 sizeof(msg) / sizeof(u32));
+	err = ixgbevf_write_msg_read_ack(hw, msg, msg, ARRAY_SIZE(msg));
 	if (!err) {
 		msg[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
@@ -911,8 +922,7 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs,
 	msg[0] = IXGBE_VF_GET_QUEUE;
 	msg[1] = msg[2] = msg[3] = msg[4] = 0;
 
-	err = ixgbevf_write_msg_read_ack(hw, msg, msg,
-					 sizeof(msg) / sizeof(u32));
+	err = ixgbevf_write_msg_read_ack(hw, msg, msg, ARRAY_SIZE(msg));
 	if (!err) {
 		msg[0] &= ~IXGBE_VT_MSGTYPE_CTS;
 
diff --git a/drivers/net/ethernet/jme.c b/drivers/net/ethernet/jme.c
index 62d848df26ef..8a165842fa85 100644
--- a/drivers/net/ethernet/jme.c
+++ b/drivers/net/ethernet/jme.c
@@ -1071,7 +1071,7 @@ static int
 jme_process_receive(struct jme_adapter *jme, int limit)
 {
 	struct jme_ring *rxring = &(jme->rxring[0]);
-	struct rxdesc *rxdesc = rxring->desc;
+	struct rxdesc *rxdesc;
 	int i, j, ccnt, desccnt, mask = jme->rx_ring_mask;
 
 	if (unlikely(!atomic_dec_and_test(&jme->rx_cleaning)))
diff --git a/drivers/net/ethernet/marvell/Kconfig b/drivers/net/ethernet/marvell/Kconfig
index da6fb825afea..ebe5c9148935 100644
--- a/drivers/net/ethernet/marvell/Kconfig
+++ b/drivers/net/ethernet/marvell/Kconfig
@@ -60,7 +60,7 @@ config MVNETA
 	depends on ARCH_MVEBU || COMPILE_TEST
 	depends on HAS_DMA
 	select MVMDIO
-	select FIXED_PHY
+	select PHYLINK
 	---help---
 	  This driver supports the network interface units in the
 	  Marvell ARMADA XP, ARMADA 370, ARMADA 38x and
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index a539263cd79c..3f6fb635738c 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -28,7 +28,7 @@
 #include <linux/of_mdio.h>
 #include <linux/of_net.h>
 #include <linux/phy.h>
-#include <linux/phy_fixed.h>
+#include <linux/phylink.h>
 #include <linux/platform_device.h>
 #include <linux/skbuff.h>
 #include <net/hwbm.h>
@@ -189,6 +189,7 @@
 #define MVNETA_GMAC_CTRL_0                       0x2c00
 #define      MVNETA_GMAC_MAX_RX_SIZE_SHIFT       2
 #define      MVNETA_GMAC_MAX_RX_SIZE_MASK        0x7ffc
+#define      MVNETA_GMAC0_PORT_1000BASE_X        BIT(1)
 #define      MVNETA_GMAC0_PORT_ENABLE            BIT(0)
 #define MVNETA_GMAC_CTRL_2                       0x2c08
 #define      MVNETA_GMAC2_INBAND_AN_ENABLE       BIT(0)
@@ -204,13 +205,19 @@
 #define      MVNETA_GMAC_TX_FLOW_CTRL_ENABLE     BIT(5)
 #define      MVNETA_GMAC_RX_FLOW_CTRL_ACTIVE     BIT(6)
 #define      MVNETA_GMAC_TX_FLOW_CTRL_ACTIVE     BIT(7)
+#define      MVNETA_GMAC_AN_COMPLETE             BIT(11)
+#define      MVNETA_GMAC_SYNC_OK                 BIT(14)
 #define MVNETA_GMAC_AUTONEG_CONFIG               0x2c0c
 #define      MVNETA_GMAC_FORCE_LINK_DOWN         BIT(0)
 #define      MVNETA_GMAC_FORCE_LINK_PASS         BIT(1)
 #define      MVNETA_GMAC_INBAND_AN_ENABLE        BIT(2)
+#define      MVNETA_GMAC_AN_BYPASS_ENABLE        BIT(3)
+#define      MVNETA_GMAC_INBAND_RESTART_AN       BIT(4)
 #define      MVNETA_GMAC_CONFIG_MII_SPEED        BIT(5)
 #define      MVNETA_GMAC_CONFIG_GMII_SPEED       BIT(6)
 #define      MVNETA_GMAC_AN_SPEED_EN             BIT(7)
+#define      MVNETA_GMAC_CONFIG_FLOW_CTRL        BIT(8)
+#define      MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL    BIT(9)
 #define      MVNETA_GMAC_AN_FLOW_CTRL_EN         BIT(11)
 #define      MVNETA_GMAC_CONFIG_FULL_DUPLEX      BIT(12)
 #define      MVNETA_GMAC_AN_DUPLEX_EN            BIT(13)
@@ -237,6 +244,12 @@
 #define MVNETA_TXQ_TOKEN_SIZE_REG(q)             (0x3e40 + ((q) << 2))
 #define      MVNETA_TXQ_TOKEN_SIZE_MAX           0x7fffffff
 
+#define MVNETA_LPI_CTRL_0                        0x2cc0
+#define MVNETA_LPI_CTRL_1                        0x2cc4
+#define      MVNETA_LPI_REQUEST_ENABLE           BIT(0)
+#define MVNETA_LPI_CTRL_2                        0x2cc8
+#define MVNETA_LPI_STATUS                        0x2ccc
+
 #define MVNETA_CAUSE_TXQ_SENT_DESC_ALL_MASK	 0xff
 
 /* Descriptor ring Macros */
@@ -313,6 +326,11 @@
 #define MVNETA_RX_GET_BM_POOL_ID(rxd) \
 	(((rxd)->status & MVNETA_RXD_BM_POOL_MASK) >> MVNETA_RXD_BM_POOL_SHIFT)
 
+enum {
+	ETHTOOL_STAT_EEE_WAKEUP,
+	ETHTOOL_MAX_STATS,
+};
+
 struct mvneta_statistic {
 	unsigned short offset;
 	unsigned short type;
@@ -321,6 +339,7 @@ struct mvneta_statistic {
 
 #define T_REG_32	32
 #define T_REG_64	64
+#define T_SW		1
 
 static const struct mvneta_statistic mvneta_statistics[] = {
 	{ 0x3000, T_REG_64, "good_octets_received", },
@@ -355,6 +374,7 @@ static const struct mvneta_statistic mvneta_statistics[] = {
 	{ 0x304c, T_REG_32, "broadcast_frames_sent", },
 	{ 0x3054, T_REG_32, "fc_sent", },
 	{ 0x300c, T_REG_32, "internal_mac_transmit_err", },
+	{ ETHTOOL_STAT_EEE_WAKEUP, T_SW, "eee_wakeup_errors", },
 };
 
 struct mvneta_pcpu_stats {
@@ -407,20 +427,20 @@ struct mvneta_port {
 	u16 tx_ring_size;
 	u16 rx_ring_size;
 
-	struct mii_bus *mii_bus;
 	phy_interface_t phy_interface;
-	struct device_node *phy_node;
-	unsigned int link;
-	unsigned int duplex;
-	unsigned int speed;
+	struct device_node *dn;
 	unsigned int tx_csum_limit;
-	unsigned int use_inband_status:1;
+	struct phylink *phylink;
 
 	struct mvneta_bm *bm_priv;
 	struct mvneta_bm_pool *pool_long;
 	struct mvneta_bm_pool *pool_short;
 	int bm_win_id;
 
+	bool eee_enabled;
+	bool eee_active;
+	bool tx_lpi_enabled;
+
 	u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)];
 
 	u32 indir[MVNETA_RSS_LU_TABLE_SIZE];
@@ -1112,6 +1132,7 @@ static void mvneta_port_up(struct mvneta_port *pp)
 	}
 	mvreg_write(pp, MVNETA_TXQ_CMD, q_map);
 
+	q_map = 0;
 	/* Enable all initialized RXQs. */
 	for (queue = 0; queue < rxq_number; queue++) {
 		struct mvneta_rx_queue *rxq = &pp->rxqs[queue];
@@ -1214,10 +1235,6 @@ static void mvneta_port_disable(struct mvneta_port *pp)
 	val &= ~MVNETA_GMAC0_PORT_ENABLE;
 	mvreg_write(pp, MVNETA_GMAC_CTRL_0, val);
 
-	pp->link = 0;
-	pp->duplex = -1;
-	pp->speed = 0;
-
 	udelay(200);
 }
 
@@ -1277,44 +1294,6 @@ static void mvneta_set_other_mcast_table(struct mvneta_port *pp, int queue)
 		mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + offset, val);
 }
 
-static void mvneta_set_autoneg(struct mvneta_port *pp, int enable)
-{
-	u32 val;
-
-	if (enable) {
-		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
-		val &= ~(MVNETA_GMAC_FORCE_LINK_PASS |
-			 MVNETA_GMAC_FORCE_LINK_DOWN |
-			 MVNETA_GMAC_AN_FLOW_CTRL_EN);
-		val |= MVNETA_GMAC_INBAND_AN_ENABLE |
-		       MVNETA_GMAC_AN_SPEED_EN |
-		       MVNETA_GMAC_AN_DUPLEX_EN;
-		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
-
-		val = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
-		val |= MVNETA_GMAC_1MS_CLOCK_ENABLE;
-		mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, val);
-
-		val = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
-		val |= MVNETA_GMAC2_INBAND_AN_ENABLE;
-		mvreg_write(pp, MVNETA_GMAC_CTRL_2, val);
-	} else {
-		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
-		val &= ~(MVNETA_GMAC_INBAND_AN_ENABLE |
-		       MVNETA_GMAC_AN_SPEED_EN |
-		       MVNETA_GMAC_AN_DUPLEX_EN);
-		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
-
-		val = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
-		val &= ~MVNETA_GMAC_1MS_CLOCK_ENABLE;
-		mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, val);
-
-		val = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
-		val &= ~MVNETA_GMAC2_INBAND_AN_ENABLE;
-		mvreg_write(pp, MVNETA_GMAC_CTRL_2, val);
-	}
-}
-
 static void mvneta_percpu_unmask_interrupt(void *arg)
 {
 	struct mvneta_port *pp = arg;
@@ -1467,7 +1446,6 @@ static void mvneta_defaults_set(struct mvneta_port *pp)
 	val &= ~MVNETA_PHY_POLLING_ENABLE;
 	mvreg_write(pp, MVNETA_UNIT_CONTROL, val);
 
-	mvneta_set_autoneg(pp, pp->use_inband_status);
 	mvneta_set_ucast_table(pp, -1);
 	mvneta_set_special_mcast_table(pp, -1);
 	mvneta_set_other_mcast_table(pp, -1);
@@ -2692,26 +2670,11 @@ static irqreturn_t mvneta_percpu_isr(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static int mvneta_fixed_link_update(struct mvneta_port *pp,
-				    struct phy_device *phy)
+static void mvneta_link_change(struct mvneta_port *pp)
 {
-	struct fixed_phy_status status;
-	struct fixed_phy_status changed = {};
 	u32 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS);
 
-	status.link = !!(gmac_stat & MVNETA_GMAC_LINK_UP);
-	if (gmac_stat & MVNETA_GMAC_SPEED_1000)
-		status.speed = SPEED_1000;
-	else if (gmac_stat & MVNETA_GMAC_SPEED_100)
-		status.speed = SPEED_100;
-	else
-		status.speed = SPEED_10;
-	status.duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX);
-	changed.link = 1;
-	changed.speed = 1;
-	changed.duplex = 1;
-	fixed_phy_update_state(phy, &status, &changed);
-	return 0;
+	phylink_mac_change(pp->phylink, !!(gmac_stat & MVNETA_GMAC_LINK_UP));
 }
 
 /* NAPI handler
@@ -2727,7 +2690,6 @@ static int mvneta_poll(struct napi_struct *napi, int budget)
 	u32 cause_rx_tx;
 	int rx_queue;
 	struct mvneta_port *pp = netdev_priv(napi->dev);
-	struct net_device *ndev = pp->dev;
 	struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports);
 
 	if (!netif_running(pp->dev)) {
@@ -2741,12 +2703,10 @@ static int mvneta_poll(struct napi_struct *napi, int budget)
 		u32 cause_misc = mvreg_read(pp, MVNETA_INTR_MISC_CAUSE);
 
 		mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0);
-		if (pp->use_inband_status && (cause_misc &
-				(MVNETA_CAUSE_PHY_STATUS_CHANGE |
-				 MVNETA_CAUSE_LINK_CHANGE |
-				 MVNETA_CAUSE_PSC_SYNC_CHANGE))) {
-			mvneta_fixed_link_update(pp, ndev->phydev);
-		}
+
+		if (cause_misc & (MVNETA_CAUSE_PHY_STATUS_CHANGE |
+				  MVNETA_CAUSE_LINK_CHANGE))
+			mvneta_link_change(pp);
 	}
 
 	/* Release Tx descriptors */
@@ -3060,7 +3020,6 @@ static int mvneta_setup_txqs(struct mvneta_port *pp)
 static void mvneta_start_dev(struct mvneta_port *pp)
 {
 	int cpu;
-	struct net_device *ndev = pp->dev;
 
 	mvneta_max_rx_size_set(pp, pp->pkt_size);
 	mvneta_txq_max_tx_size_set(pp, pp->pkt_size);
@@ -3085,19 +3044,17 @@ static void mvneta_start_dev(struct mvneta_port *pp)
 
 	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
 		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
-		    MVNETA_CAUSE_LINK_CHANGE |
-		    MVNETA_CAUSE_PSC_SYNC_CHANGE);
+		    MVNETA_CAUSE_LINK_CHANGE);
 
-	phy_start(ndev->phydev);
+	phylink_start(pp->phylink);
 	netif_tx_start_all_queues(pp->dev);
 }
 
 static void mvneta_stop_dev(struct mvneta_port *pp)
 {
 	unsigned int cpu;
-	struct net_device *ndev = pp->dev;
 
-	phy_stop(ndev->phydev);
+	phylink_stop(pp->phylink);
 
 	if (!pp->neta_armada3700) {
 		for_each_online_cpu(cpu) {
@@ -3251,103 +3208,260 @@ static int mvneta_set_mac_addr(struct net_device *dev, void *addr)
 	return 0;
 }
 
-static void mvneta_adjust_link(struct net_device *ndev)
+static void mvneta_validate(struct net_device *ndev, unsigned long *supported,
+			    struct phylink_link_state *state)
+{
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
+
+	/* We only support QSGMII, SGMII, 802.3z and RGMII modes */
+	if (state->interface != PHY_INTERFACE_MODE_NA &&
+	    state->interface != PHY_INTERFACE_MODE_QSGMII &&
+	    state->interface != PHY_INTERFACE_MODE_SGMII &&
+	    !phy_interface_mode_is_8023z(state->interface) &&
+	    !phy_interface_mode_is_rgmii(state->interface)) {
+		bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
+		return;
+	}
+
+	/* Allow all the expected bits */
+	phylink_set(mask, Autoneg);
+	phylink_set_port_modes(mask);
+
+	/* Asymmetric pause is unsupported */
+	phylink_set(mask, Pause);
+	/* Half-duplex at speeds higher than 100Mbit is unsupported */
+	phylink_set(mask, 1000baseT_Full);
+	phylink_set(mask, 1000baseX_Full);
+
+	if (!phy_interface_mode_is_8023z(state->interface)) {
+		/* 10M and 100M are only supported in non-802.3z mode */
+		phylink_set(mask, 10baseT_Half);
+		phylink_set(mask, 10baseT_Full);
+		phylink_set(mask, 100baseT_Half);
+		phylink_set(mask, 100baseT_Full);
+	}
+
+	bitmap_and(supported, supported, mask,
+		   __ETHTOOL_LINK_MODE_MASK_NBITS);
+	bitmap_and(state->advertising, state->advertising, mask,
+		   __ETHTOOL_LINK_MODE_MASK_NBITS);
+}
+
+static int mvneta_mac_link_state(struct net_device *ndev,
+				 struct phylink_link_state *state)
 {
 	struct mvneta_port *pp = netdev_priv(ndev);
-	struct phy_device *phydev = ndev->phydev;
-	int status_change = 0;
+	u32 gmac_stat;
 
-	if (phydev->link) {
-		if ((pp->speed != phydev->speed) ||
-		    (pp->duplex != phydev->duplex)) {
-			u32 val;
+	gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS);
 
-			val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
-			val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED |
-				 MVNETA_GMAC_CONFIG_GMII_SPEED |
-				 MVNETA_GMAC_CONFIG_FULL_DUPLEX);
+	if (gmac_stat & MVNETA_GMAC_SPEED_1000)
+		state->speed = SPEED_1000;
+	else if (gmac_stat & MVNETA_GMAC_SPEED_100)
+		state->speed = SPEED_100;
+	else
+		state->speed = SPEED_10;
 
-			if (phydev->duplex)
-				val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
+	state->an_complete = !!(gmac_stat & MVNETA_GMAC_AN_COMPLETE);
+	state->link = !!(gmac_stat & MVNETA_GMAC_LINK_UP);
+	state->duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX);
 
-			if (phydev->speed == SPEED_1000)
-				val |= MVNETA_GMAC_CONFIG_GMII_SPEED;
-			else if (phydev->speed == SPEED_100)
-				val |= MVNETA_GMAC_CONFIG_MII_SPEED;
+	state->pause = 0;
+	if (gmac_stat & MVNETA_GMAC_RX_FLOW_CTRL_ENABLE)
+		state->pause |= MLO_PAUSE_RX;
+	if (gmac_stat & MVNETA_GMAC_TX_FLOW_CTRL_ENABLE)
+		state->pause |= MLO_PAUSE_TX;
 
-			mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
+	return 1;
+}
 
-			pp->duplex = phydev->duplex;
-			pp->speed  = phydev->speed;
-		}
+static void mvneta_mac_an_restart(struct net_device *ndev)
+{
+	struct mvneta_port *pp = netdev_priv(ndev);
+	u32 gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
+
+	mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
+		    gmac_an | MVNETA_GMAC_INBAND_RESTART_AN);
+	mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
+		    gmac_an & ~MVNETA_GMAC_INBAND_RESTART_AN);
+}
+
+static void mvneta_mac_config(struct net_device *ndev, unsigned int mode,
+	const struct phylink_link_state *state)
+{
+	struct mvneta_port *pp = netdev_priv(ndev);
+	u32 new_ctrl0, gmac_ctrl0 = mvreg_read(pp, MVNETA_GMAC_CTRL_0);
+	u32 new_ctrl2, gmac_ctrl2 = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
+	u32 new_clk, gmac_clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER);
+	u32 new_an, gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
+
+	new_ctrl0 = gmac_ctrl0 & ~MVNETA_GMAC0_PORT_1000BASE_X;
+	new_ctrl2 = gmac_ctrl2 & ~(MVNETA_GMAC2_INBAND_AN_ENABLE |
+				   MVNETA_GMAC2_PORT_RESET);
+	new_clk = gmac_clk & ~MVNETA_GMAC_1MS_CLOCK_ENABLE;
+	new_an = gmac_an & ~(MVNETA_GMAC_INBAND_AN_ENABLE |
+			     MVNETA_GMAC_INBAND_RESTART_AN |
+			     MVNETA_GMAC_CONFIG_MII_SPEED |
+			     MVNETA_GMAC_CONFIG_GMII_SPEED |
+			     MVNETA_GMAC_AN_SPEED_EN |
+			     MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL |
+			     MVNETA_GMAC_CONFIG_FLOW_CTRL |
+			     MVNETA_GMAC_AN_FLOW_CTRL_EN |
+			     MVNETA_GMAC_CONFIG_FULL_DUPLEX |
+			     MVNETA_GMAC_AN_DUPLEX_EN);
+
+	/* Even though it might look weird, when we're configured in
+	 * SGMII or QSGMII mode, the RGMII bit needs to be set.
+	 */
+	new_ctrl2 |= MVNETA_GMAC2_PORT_RGMII;
+
+	if (state->interface == PHY_INTERFACE_MODE_QSGMII ||
+	    state->interface == PHY_INTERFACE_MODE_SGMII ||
+	    phy_interface_mode_is_8023z(state->interface))
+		new_ctrl2 |= MVNETA_GMAC2_PCS_ENABLE;
+
+	if (phylink_test(state->advertising, Pause))
+		new_an |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL;
+	if (state->pause & MLO_PAUSE_TXRX_MASK)
+		new_an |= MVNETA_GMAC_CONFIG_FLOW_CTRL;
+
+	if (!phylink_autoneg_inband(mode)) {
+		/* Phy or fixed speed */
+		if (state->duplex)
+			new_an |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
+
+		if (state->speed == SPEED_1000)
+			new_an |= MVNETA_GMAC_CONFIG_GMII_SPEED;
+		else if (state->speed == SPEED_100)
+			new_an |= MVNETA_GMAC_CONFIG_MII_SPEED;
+	} else if (state->interface == PHY_INTERFACE_MODE_SGMII) {
+		/* SGMII mode receives the state from the PHY */
+		new_ctrl2 |= MVNETA_GMAC2_INBAND_AN_ENABLE;
+		new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE;
+		new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN |
+				     MVNETA_GMAC_FORCE_LINK_PASS)) |
+			 MVNETA_GMAC_INBAND_AN_ENABLE |
+			 MVNETA_GMAC_AN_SPEED_EN |
+			 MVNETA_GMAC_AN_DUPLEX_EN;
+	} else {
+		/* 802.3z negotiation - only 1000base-X */
+		new_ctrl0 |= MVNETA_GMAC0_PORT_1000BASE_X;
+		new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE;
+		new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN |
+				     MVNETA_GMAC_FORCE_LINK_PASS)) |
+			 MVNETA_GMAC_INBAND_AN_ENABLE |
+			 MVNETA_GMAC_CONFIG_GMII_SPEED |
+			 /* The MAC only supports FD mode */
+			 MVNETA_GMAC_CONFIG_FULL_DUPLEX;
+
+		if (state->pause & MLO_PAUSE_AN && state->an_enabled)
+			new_an |= MVNETA_GMAC_AN_FLOW_CTRL_EN;
 	}
 
-	if (phydev->link != pp->link) {
-		if (!phydev->link) {
-			pp->duplex = -1;
-			pp->speed = 0;
-		}
+	/* Armada 370 documentation says we can only change the port mode
+	 * and in-band enable when the link is down, so force it down
+	 * while making these changes. We also do this for GMAC_CTRL2 */
+	if ((new_ctrl0 ^ gmac_ctrl0) & MVNETA_GMAC0_PORT_1000BASE_X ||
+	    (new_ctrl2 ^ gmac_ctrl2) & MVNETA_GMAC2_INBAND_AN_ENABLE ||
+	    (new_an  ^ gmac_an) & MVNETA_GMAC_INBAND_AN_ENABLE) {
+		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
+			    (gmac_an & ~MVNETA_GMAC_FORCE_LINK_PASS) |
+			    MVNETA_GMAC_FORCE_LINK_DOWN);
+	}
 
-		pp->link = phydev->link;
-		status_change = 1;
+	if (new_ctrl0 != gmac_ctrl0)
+		mvreg_write(pp, MVNETA_GMAC_CTRL_0, new_ctrl0);
+	if (new_ctrl2 != gmac_ctrl2)
+		mvreg_write(pp, MVNETA_GMAC_CTRL_2, new_ctrl2);
+	if (new_clk != gmac_clk)
+		mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, new_clk);
+	if (new_an != gmac_an)
+		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, new_an);
+
+	if (gmac_ctrl2 & MVNETA_GMAC2_PORT_RESET) {
+		while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) &
+			MVNETA_GMAC2_PORT_RESET) != 0)
+			continue;
 	}
+}
 
-	if (status_change) {
-		if (phydev->link) {
-			if (!pp->use_inband_status) {
-				u32 val = mvreg_read(pp,
-						  MVNETA_GMAC_AUTONEG_CONFIG);
-				val &= ~MVNETA_GMAC_FORCE_LINK_DOWN;
-				val |= MVNETA_GMAC_FORCE_LINK_PASS;
-				mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
-					    val);
-			}
-			mvneta_port_up(pp);
-		} else {
-			if (!pp->use_inband_status) {
-				u32 val = mvreg_read(pp,
-						  MVNETA_GMAC_AUTONEG_CONFIG);
-				val &= ~MVNETA_GMAC_FORCE_LINK_PASS;
-				val |= MVNETA_GMAC_FORCE_LINK_DOWN;
-				mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG,
-					    val);
-			}
-			mvneta_port_down(pp);
-		}
-		phy_print_status(phydev);
+static void mvneta_set_eee(struct mvneta_port *pp, bool enable)
+{
+	u32 lpi_ctl1;
+
+	lpi_ctl1 = mvreg_read(pp, MVNETA_LPI_CTRL_1);
+	if (enable)
+		lpi_ctl1 |= MVNETA_LPI_REQUEST_ENABLE;
+	else
+		lpi_ctl1 &= ~MVNETA_LPI_REQUEST_ENABLE;
+	mvreg_write(pp, MVNETA_LPI_CTRL_1, lpi_ctl1);
+}
+
+static void mvneta_mac_link_down(struct net_device *ndev, unsigned int mode)
+{
+	struct mvneta_port *pp = netdev_priv(ndev);
+	u32 val;
+
+	mvneta_port_down(pp);
+
+	if (!phylink_autoneg_inband(mode)) {
+		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
+		val &= ~MVNETA_GMAC_FORCE_LINK_PASS;
+		val |= MVNETA_GMAC_FORCE_LINK_DOWN;
+		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
 	}
+
+	pp->eee_active = false;
+	mvneta_set_eee(pp, false);
 }
 
-static int mvneta_mdio_probe(struct mvneta_port *pp)
+static void mvneta_mac_link_up(struct net_device *ndev, unsigned int mode,
+			       struct phy_device *phy)
 {
-	struct phy_device *phy_dev;
-	struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
+	struct mvneta_port *pp = netdev_priv(ndev);
+	u32 val;
 
-	phy_dev = of_phy_connect(pp->dev, pp->phy_node, mvneta_adjust_link, 0,
-				 pp->phy_interface);
-	if (!phy_dev) {
-		netdev_err(pp->dev, "could not find the PHY\n");
-		return -ENODEV;
+	if (!phylink_autoneg_inband(mode)) {
+		val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
+		val &= ~MVNETA_GMAC_FORCE_LINK_DOWN;
+		val |= MVNETA_GMAC_FORCE_LINK_PASS;
+		mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
 	}
 
-	phy_ethtool_get_wol(phy_dev, &wol);
-	device_set_wakeup_capable(&pp->dev->dev, !!wol.supported);
+	mvneta_port_up(pp);
 
-	phy_dev->supported &= PHY_GBIT_FEATURES;
-	phy_dev->advertising = phy_dev->supported;
+	if (phy && pp->eee_enabled) {
+		pp->eee_active = phy_init_eee(phy, 0) >= 0;
+		mvneta_set_eee(pp, pp->eee_active && pp->tx_lpi_enabled);
+	}
+}
 
-	pp->link    = 0;
-	pp->duplex  = 0;
-	pp->speed   = 0;
+static const struct phylink_mac_ops mvneta_phylink_ops = {
+	.validate = mvneta_validate,
+	.mac_link_state = mvneta_mac_link_state,
+	.mac_an_restart = mvneta_mac_an_restart,
+	.mac_config = mvneta_mac_config,
+	.mac_link_down = mvneta_mac_link_down,
+	.mac_link_up = mvneta_mac_link_up,
+};
 
-	return 0;
+static int mvneta_mdio_probe(struct mvneta_port *pp)
+{
+	struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
+	int err = phylink_of_phy_connect(pp->phylink, pp->dn, 0);
+
+	if (err)
+		netdev_err(pp->dev, "could not attach PHY: %d\n", err);
+
+	phylink_ethtool_get_wol(pp->phylink, &wol);
+	device_set_wakeup_capable(&pp->dev->dev, !!wol.supported);
+
+	return err;
 }
 
 static void mvneta_mdio_remove(struct mvneta_port *pp)
 {
-	struct net_device *ndev = pp->dev;
-
-	phy_disconnect(ndev->phydev);
+	phylink_disconnect_phy(pp->phylink);
 }
 
 /* Electing a CPU must be done in an atomic way: it should be done
@@ -3455,8 +3569,7 @@ static int mvneta_cpu_online(unsigned int cpu, struct hlist_node *node)
 	on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
 	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
 		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
-		    MVNETA_CAUSE_LINK_CHANGE |
-		    MVNETA_CAUSE_PSC_SYNC_CHANGE);
+		    MVNETA_CAUSE_LINK_CHANGE);
 	netif_tx_start_all_queues(pp->dev);
 	spin_unlock(&pp->lock);
 	return 0;
@@ -3497,8 +3610,7 @@ static int mvneta_cpu_dead(unsigned int cpu, struct hlist_node *node)
 	on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true);
 	mvreg_write(pp, MVNETA_INTR_MISC_MASK,
 		    MVNETA_CAUSE_PHY_STATUS_CHANGE |
-		    MVNETA_CAUSE_LINK_CHANGE |
-		    MVNETA_CAUSE_PSC_SYNC_CHANGE);
+		    MVNETA_CAUSE_LINK_CHANGE);
 	netif_tx_start_all_queues(pp->dev);
 	return 0;
 }
@@ -3626,10 +3738,9 @@ static int mvneta_stop(struct net_device *dev)
 
 static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 {
-	if (!dev->phydev)
-		return -ENOTSUPP;
+	struct mvneta_port *pp = netdev_priv(dev);
 
-	return phy_mii_ioctl(dev->phydev, ifr, cmd);
+	return phylink_mii_ioctl(pp->phylink, ifr, cmd);
 }
 
 /* Ethtool methods */
@@ -3640,44 +3751,25 @@ mvneta_ethtool_set_link_ksettings(struct net_device *ndev,
 				  const struct ethtool_link_ksettings *cmd)
 {
 	struct mvneta_port *pp = netdev_priv(ndev);
-	struct phy_device *phydev = ndev->phydev;
-
-	if (!phydev)
-		return -ENODEV;
-
-	if ((cmd->base.autoneg == AUTONEG_ENABLE) != pp->use_inband_status) {
-		u32 val;
-
-		mvneta_set_autoneg(pp, cmd->base.autoneg == AUTONEG_ENABLE);
-
-		if (cmd->base.autoneg == AUTONEG_DISABLE) {
-			val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG);
-			val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED |
-				 MVNETA_GMAC_CONFIG_GMII_SPEED |
-				 MVNETA_GMAC_CONFIG_FULL_DUPLEX);
 
-			if (phydev->duplex)
-				val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX;
-
-			if (phydev->speed == SPEED_1000)
-				val |= MVNETA_GMAC_CONFIG_GMII_SPEED;
-			else if (phydev->speed == SPEED_100)
-				val |= MVNETA_GMAC_CONFIG_MII_SPEED;
+	return phylink_ethtool_ksettings_set(pp->phylink, cmd);
+}
 
-			mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val);
-		}
+/* Get link ksettings for ethtools */
+static int
+mvneta_ethtool_get_link_ksettings(struct net_device *ndev,
+				  struct ethtool_link_ksettings *cmd)
+{
+	struct mvneta_port *pp = netdev_priv(ndev);
 
-		pp->use_inband_status = (cmd->base.autoneg == AUTONEG_ENABLE);
-		netdev_info(pp->dev, "autoneg status set to %i\n",
-			    pp->use_inband_status);
+	return phylink_ethtool_ksettings_get(pp->phylink, cmd);
+}
 
-		if (netif_running(ndev)) {
-			mvneta_port_down(pp);
-			mvneta_port_up(pp);
-		}
-	}
+static int mvneta_ethtool_nway_reset(struct net_device *dev)
+{
+	struct mvneta_port *pp = netdev_priv(dev);
 
-	return phy_ethtool_ksettings_set(ndev->phydev, cmd);
+	return phylink_ethtool_nway_reset(pp->phylink);
 }
 
 /* Set interrupt coalescing for ethtools */
@@ -3769,6 +3861,22 @@ static int mvneta_ethtool_set_ringparam(struct net_device *dev,
 	return 0;
 }
 
+static void mvneta_ethtool_get_pauseparam(struct net_device *dev,
+					  struct ethtool_pauseparam *pause)
+{
+	struct mvneta_port *pp = netdev_priv(dev);
+
+	phylink_ethtool_get_pauseparam(pp->phylink, pause);
+}
+
+static int mvneta_ethtool_set_pauseparam(struct net_device *dev,
+					 struct ethtool_pauseparam *pause)
+{
+	struct mvneta_port *pp = netdev_priv(dev);
+
+	return phylink_ethtool_set_pauseparam(pp->phylink, pause);
+}
+
 static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset,
 				       u8 *data)
 {
@@ -3785,26 +3893,35 @@ static void mvneta_ethtool_update_stats(struct mvneta_port *pp)
 {
 	const struct mvneta_statistic *s;
 	void __iomem *base = pp->base;
-	u32 high, low, val;
-	u64 val64;
+	u32 high, low;
+	u64 val;
 	int i;
 
 	for (i = 0, s = mvneta_statistics;
 	     s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics);
 	     s++, i++) {
+		val = 0;
+
 		switch (s->type) {
 		case T_REG_32:
 			val = readl_relaxed(base + s->offset);
-			pp->ethtool_stats[i] += val;
 			break;
 		case T_REG_64:
 			/* Docs say to read low 32-bit then high */
 			low = readl_relaxed(base + s->offset);
 			high = readl_relaxed(base + s->offset + 4);
-			val64 = (u64)high << 32 | low;
-			pp->ethtool_stats[i] += val64;
+			val = (u64)high << 32 | low;
+			break;
+		case T_SW:
+			switch (s->offset) {
+			case ETHTOOL_STAT_EEE_WAKEUP:
+				val = phylink_get_eee_err(pp->phylink);
+				break;
+			}
 			break;
 		}
+
+		pp->ethtool_stats[i] += val;
 	}
 }
 
@@ -3939,28 +4056,81 @@ static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
 static void mvneta_ethtool_get_wol(struct net_device *dev,
 				   struct ethtool_wolinfo *wol)
 {
-	wol->supported = 0;
-	wol->wolopts = 0;
+	struct mvneta_port *pp = netdev_priv(dev);
 
-	if (dev->phydev)
-		phy_ethtool_get_wol(dev->phydev, wol);
+	phylink_ethtool_get_wol(pp->phylink, wol);
 }
 
 static int mvneta_ethtool_set_wol(struct net_device *dev,
 				  struct ethtool_wolinfo *wol)
 {
+	struct mvneta_port *pp = netdev_priv(dev);
 	int ret;
 
-	if (!dev->phydev)
-		return -EOPNOTSUPP;
-
-	ret = phy_ethtool_set_wol(dev->phydev, wol);
+	ret = phylink_ethtool_set_wol(pp->phylink, wol);
 	if (!ret)
 		device_set_wakeup_enable(&dev->dev, !!wol->wolopts);
 
 	return ret;
 }
 
+static int mvneta_ethtool_get_module_info(struct net_device *dev,
+					  struct ethtool_modinfo *modinfo)
+{
+	struct mvneta_port *pp = netdev_priv(dev);
+
+	return phylink_ethtool_get_module_info(pp->phylink, modinfo);
+}
+
+static int mvneta_ethtool_get_module_eeprom(struct net_device *dev,
+					    struct ethtool_eeprom *ee, u8 *buf)
+{
+	struct mvneta_port *pp = netdev_priv(dev);
+
+	return phylink_ethtool_get_module_eeprom(pp->phylink, ee, buf);
+}
+
+static int mvneta_ethtool_get_eee(struct net_device *dev,
+				  struct ethtool_eee *eee)
+{
+	struct mvneta_port *pp = netdev_priv(dev);
+	u32 lpi_ctl0;
+
+	lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0);
+
+	eee->eee_enabled = pp->eee_enabled;
+	eee->eee_active = pp->eee_active;
+	eee->tx_lpi_enabled = pp->tx_lpi_enabled;
+	eee->tx_lpi_timer = (lpi_ctl0) >> 8; // * scale;
+
+	return phylink_ethtool_get_eee(pp->phylink, eee);
+}
+
+static int mvneta_ethtool_set_eee(struct net_device *dev,
+				  struct ethtool_eee *eee)
+{
+	struct mvneta_port *pp = netdev_priv(dev);
+	u32 lpi_ctl0;
+
+	/* The Armada 37x documents do not give limits for this other than
+	 * it being an 8-bit register. */
+	if (eee->tx_lpi_enabled &&
+	    (eee->tx_lpi_timer < 0 || eee->tx_lpi_timer > 255))
+		return -EINVAL;
+
+	lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0);
+	lpi_ctl0 &= ~(0xff << 8);
+	lpi_ctl0 |= eee->tx_lpi_timer << 8;
+	mvreg_write(pp, MVNETA_LPI_CTRL_0, lpi_ctl0);
+
+	pp->eee_enabled = eee->eee_enabled;
+	pp->tx_lpi_enabled = eee->tx_lpi_enabled;
+
+	mvneta_set_eee(pp, eee->tx_lpi_enabled && eee->eee_enabled);
+
+	return phylink_ethtool_set_eee(pp->phylink, eee);
+}
+
 static const struct net_device_ops mvneta_netdev_ops = {
 	.ndo_open            = mvneta_open,
 	.ndo_stop            = mvneta_stop,
@@ -3974,13 +4144,15 @@ static const struct net_device_ops mvneta_netdev_ops = {
 };
 
 static const struct ethtool_ops mvneta_eth_tool_ops = {
-	.nway_reset	= phy_ethtool_nway_reset,
+	.nway_reset	= mvneta_ethtool_nway_reset,
 	.get_link       = ethtool_op_get_link,
 	.set_coalesce   = mvneta_ethtool_set_coalesce,
 	.get_coalesce   = mvneta_ethtool_get_coalesce,
 	.get_drvinfo    = mvneta_ethtool_get_drvinfo,
 	.get_ringparam  = mvneta_ethtool_get_ringparam,
 	.set_ringparam	= mvneta_ethtool_set_ringparam,
+	.get_pauseparam	= mvneta_ethtool_get_pauseparam,
+	.set_pauseparam	= mvneta_ethtool_set_pauseparam,
 	.get_strings	= mvneta_ethtool_get_strings,
 	.get_ethtool_stats = mvneta_ethtool_get_stats,
 	.get_sset_count	= mvneta_ethtool_get_sset_count,
@@ -3988,10 +4160,14 @@ static const struct ethtool_ops mvneta_eth_tool_ops = {
 	.get_rxnfc	= mvneta_ethtool_get_rxnfc,
 	.get_rxfh	= mvneta_ethtool_get_rxfh,
 	.set_rxfh	= mvneta_ethtool_set_rxfh,
-	.get_link_ksettings = phy_ethtool_get_link_ksettings,
+	.get_link_ksettings = mvneta_ethtool_get_link_ksettings,
 	.set_link_ksettings = mvneta_ethtool_set_link_ksettings,
 	.get_wol        = mvneta_ethtool_get_wol,
 	.set_wol        = mvneta_ethtool_set_wol,
+	.get_module_info = mvneta_ethtool_get_module_info,
+	.get_module_eeprom = mvneta_ethtool_get_module_eeprom,
+	.get_eee	= mvneta_ethtool_get_eee,
+	.set_eee	= mvneta_ethtool_set_eee,
 };
 
 /* Initialize hw */
@@ -4091,42 +4267,16 @@ static void mvneta_conf_mbus_windows(struct mvneta_port *pp,
 /* Power up the port */
 static int mvneta_port_power_up(struct mvneta_port *pp, int phy_mode)
 {
-	u32 ctrl;
-
 	/* MAC Cause register should be cleared */
 	mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0);
 
-	ctrl = mvreg_read(pp, MVNETA_GMAC_CTRL_2);
-
-	/* Even though it might look weird, when we're configured in
-	 * SGMII or QSGMII mode, the RGMII bit needs to be set.
-	 */
-	switch(phy_mode) {
-	case PHY_INTERFACE_MODE_QSGMII:
+	if (phy_mode == PHY_INTERFACE_MODE_QSGMII)
 		mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO);
-		ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII;
-		break;
-	case PHY_INTERFACE_MODE_SGMII:
+	else if (phy_mode == PHY_INTERFACE_MODE_SGMII ||
+		 phy_mode == PHY_INTERFACE_MODE_1000BASEX)
 		mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO);
-		ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII;
-		break;
-	case PHY_INTERFACE_MODE_RGMII:
-	case PHY_INTERFACE_MODE_RGMII_ID:
-	case PHY_INTERFACE_MODE_RGMII_RXID:
-	case PHY_INTERFACE_MODE_RGMII_TXID:
-		ctrl |= MVNETA_GMAC2_PORT_RGMII;
-		break;
-	default:
+	else if (!phy_interface_mode_is_rgmii(phy_mode))
 		return -EINVAL;
-	}
-
-	/* Cancel Port Reset */
-	ctrl &= ~MVNETA_GMAC2_PORT_RESET;
-	mvreg_write(pp, MVNETA_GMAC_CTRL_2, ctrl);
-
-	while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) &
-		MVNETA_GMAC2_PORT_RESET) != 0)
-		continue;
 
 	return 0;
 }
@@ -4136,14 +4286,13 @@ static int mvneta_probe(struct platform_device *pdev)
 {
 	struct resource *res;
 	struct device_node *dn = pdev->dev.of_node;
-	struct device_node *phy_node;
 	struct device_node *bm_node;
 	struct mvneta_port *pp;
 	struct net_device *dev;
+	struct phylink *phylink;
 	const char *dt_mac_addr;
 	char hw_mac_addr[ETH_ALEN];
 	const char *mac_from;
-	const char *managed;
 	int tx_csum_limit;
 	int phy_mode;
 	int err;
@@ -4159,31 +4308,18 @@ static int mvneta_probe(struct platform_device *pdev)
 		goto err_free_netdev;
 	}
 
-	phy_node = of_parse_phandle(dn, "phy", 0);
-	if (!phy_node) {
-		if (!of_phy_is_fixed_link(dn)) {
-			dev_err(&pdev->dev, "no PHY specified\n");
-			err = -ENODEV;
-			goto err_free_irq;
-		}
-
-		err = of_phy_register_fixed_link(dn);
-		if (err < 0) {
-			dev_err(&pdev->dev, "cannot register fixed PHY\n");
-			goto err_free_irq;
-		}
-
-		/* In the case of a fixed PHY, the DT node associated
-		 * to the PHY is the Ethernet MAC DT node.
-		 */
-		phy_node = of_node_get(dn);
-	}
-
 	phy_mode = of_get_phy_mode(dn);
 	if (phy_mode < 0) {
 		dev_err(&pdev->dev, "incorrect phy-mode\n");
 		err = -EINVAL;
-		goto err_put_phy_node;
+		goto err_free_irq;
+	}
+
+	phylink = phylink_create(dev, pdev->dev.fwnode, phy_mode,
+				 &mvneta_phylink_ops);
+	if (IS_ERR(phylink)) {
+		err = PTR_ERR(phylink);
+		goto err_free_irq;
 	}
 
 	dev->tx_queue_len = MVNETA_MAX_TXD;
@@ -4194,12 +4330,9 @@ static int mvneta_probe(struct platform_device *pdev)
 
 	pp = netdev_priv(dev);
 	spin_lock_init(&pp->lock);
-	pp->phy_node = phy_node;
+	pp->phylink = phylink;
 	pp->phy_interface = phy_mode;
-
-	err = of_property_read_string(dn, "managed", &managed);
-	pp->use_inband_status = (err == 0 &&
-				 strcmp(managed, "in-band-status") == 0);
+	pp->dn = dn;
 
 	pp->rxq_def = rxq_def;
 
@@ -4221,7 +4354,7 @@ static int mvneta_probe(struct platform_device *pdev)
 		pp->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(pp->clk)) {
 		err = PTR_ERR(pp->clk);
-		goto err_put_phy_node;
+		goto err_free_phylink;
 	}
 
 	clk_prepare_enable(pp->clk);
@@ -4358,14 +4491,6 @@ static int mvneta_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, pp->dev);
 
-	if (pp->use_inband_status) {
-		struct phy_device *phy = of_phy_find_device(dn);
-
-		mvneta_fixed_link_update(pp, phy);
-
-		put_device(&phy->mdio.dev);
-	}
-
 	return 0;
 
 err_netdev:
@@ -4382,10 +4507,9 @@ static int mvneta_probe(struct platform_device *pdev)
 err_clk:
 	clk_disable_unprepare(pp->clk_bus);
 	clk_disable_unprepare(pp->clk);
-err_put_phy_node:
-	of_node_put(phy_node);
-	if (of_phy_is_fixed_link(dn))
-		of_phy_deregister_fixed_link(dn);
+err_free_phylink:
+	if (pp->phylink)
+		phylink_destroy(pp->phylink);
 err_free_irq:
 	irq_dispose_mapping(dev->irq);
 err_free_netdev:
@@ -4397,7 +4521,6 @@ static int mvneta_probe(struct platform_device *pdev)
 static int mvneta_remove(struct platform_device *pdev)
 {
 	struct net_device  *dev = platform_get_drvdata(pdev);
-	struct device_node *dn = pdev->dev.of_node;
 	struct mvneta_port *pp = netdev_priv(dev);
 
 	unregister_netdev(dev);
@@ -4405,10 +4528,8 @@ static int mvneta_remove(struct platform_device *pdev)
 	clk_disable_unprepare(pp->clk);
 	free_percpu(pp->ports);
 	free_percpu(pp->stats);
-	if (of_phy_is_fixed_link(dn))
-		of_phy_deregister_fixed_link(dn);
 	irq_dispose_mapping(dev->irq);
-	of_node_put(pp->phy_node);
+	phylink_destroy(pp->phylink);
 	free_netdev(dev);
 
 	if (pp->bm_priv) {
@@ -4426,8 +4547,10 @@ static int mvneta_suspend(struct device *device)
 	struct net_device *dev = dev_get_drvdata(device);
 	struct mvneta_port *pp = netdev_priv(dev);
 
+	rtnl_lock();
 	if (netif_running(dev))
 		mvneta_stop(dev);
+	rtnl_unlock();
 	netif_device_detach(dev);
 	clk_disable_unprepare(pp->clk_bus);
 	clk_disable_unprepare(pp->clk);
@@ -4460,14 +4583,13 @@ static int mvneta_resume(struct device *device)
 		return err;
 	}
 
-	if (pp->use_inband_status)
-		mvneta_fixed_link_update(pp, dev->phydev);
-
 	netif_device_attach(dev);
+	rtnl_lock();
 	if (netif_running(dev)) {
 		mvneta_open(dev);
 		mvneta_set_rx_mode(dev);
 	}
+	rtnl_unlock();
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/marvell/mvpp2.c b/drivers/net/ethernet/marvell/mvpp2.c
index 634b2f41cc9e..5a1668cdb461 100644
--- a/drivers/net/ethernet/marvell/mvpp2.c
+++ b/drivers/net/ethernet/marvell/mvpp2.c
@@ -10,6 +10,7 @@
  * warranty of any kind, whether express or implied.
  */
 
+#include <linux/acpi.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
@@ -454,11 +455,11 @@
 /* Various constants */
 
 /* Coalescing */
-#define MVPP2_TXDONE_COAL_PKTS_THRESH	15
+#define MVPP2_TXDONE_COAL_PKTS_THRESH	64
 #define MVPP2_TXDONE_HRTIMER_PERIOD_NS	1000000UL
 #define MVPP2_TXDONE_COAL_USEC		1000
 #define MVPP2_RX_COAL_PKTS		32
-#define MVPP2_RX_COAL_USEC		100
+#define MVPP2_RX_COAL_USEC		64
 
 /* The two bytes Marvell header. Either contains a special value used
  * by Marvell switches when a specific hardware mode is enabled (not
@@ -504,10 +505,12 @@
 #define MVPP2_DEFAULT_RXQ		4
 
 /* Max number of Rx descriptors */
-#define MVPP2_MAX_RXD			128
+#define MVPP2_MAX_RXD_MAX		1024
+#define MVPP2_MAX_RXD_DFLT		128
 
 /* Max number of Tx descriptors */
-#define MVPP2_MAX_TXD			1024
+#define MVPP2_MAX_TXD_MAX		2048
+#define MVPP2_MAX_TXD_DFLT		1024
 
 /* Amount of Tx descriptors that can be reserved at once by CPU */
 #define MVPP2_CPU_DESC_CHUNK		64
@@ -863,7 +866,7 @@ struct mvpp2 {
 
 	/* List of pointers to port structures */
 	int port_count;
-	struct mvpp2_port **port_list;
+	struct mvpp2_port *port_list[MVPP2_MAX_PORTS];
 
 	/* Aggregated TXQs */
 	struct mvpp2_tx_queue *aggr_txqs;
@@ -930,6 +933,9 @@ struct mvpp2_port {
 
 	struct mvpp2 *priv;
 
+	/* Firmware node associated to the port */
+	struct fwnode_handle *fwnode;
+
 	/* Per-port registers' base address */
 	void __iomem *base;
 	void __iomem *stats_base;
@@ -5802,6 +5808,7 @@ static int mvpp2_txq_init(struct mvpp2_port *port,
 		txq_pcpu->reserved_num = 0;
 		txq_pcpu->txq_put_index = 0;
 		txq_pcpu->txq_get_index = 0;
+		txq_pcpu->tso_headers = NULL;
 
 		txq_pcpu->stop_threshold = txq->size - MVPP2_MAX_SKB_DESCS;
 		txq_pcpu->wake_threshold = txq_pcpu->stop_threshold / 2;
@@ -5829,10 +5836,13 @@ static void mvpp2_txq_deinit(struct mvpp2_port *port,
 		txq_pcpu = per_cpu_ptr(txq->pcpu, cpu);
 		kfree(txq_pcpu->buffs);
 
-		dma_free_coherent(port->dev->dev.parent,
-				  txq_pcpu->size * TSO_HEADER_SIZE,
-				  txq_pcpu->tso_headers,
-				  txq_pcpu->tso_headers_dma);
+		if (txq_pcpu->tso_headers)
+			dma_free_coherent(port->dev->dev.parent,
+					  txq_pcpu->size * TSO_HEADER_SIZE,
+					  txq_pcpu->tso_headers,
+					  txq_pcpu->tso_headers_dma);
+
+		txq_pcpu->tso_headers = NULL;
 	}
 
 	if (txq->descs)
@@ -6832,13 +6842,13 @@ static int mvpp2_check_ringparam_valid(struct net_device *dev,
 	if (ring->rx_pending == 0 || ring->tx_pending == 0)
 		return -EINVAL;
 
-	if (ring->rx_pending > MVPP2_MAX_RXD)
-		new_rx_pending = MVPP2_MAX_RXD;
+	if (ring->rx_pending > MVPP2_MAX_RXD_MAX)
+		new_rx_pending = MVPP2_MAX_RXD_MAX;
 	else if (!IS_ALIGNED(ring->rx_pending, 16))
 		new_rx_pending = ALIGN(ring->rx_pending, 16);
 
-	if (ring->tx_pending > MVPP2_MAX_TXD)
-		new_tx_pending = MVPP2_MAX_TXD;
+	if (ring->tx_pending > MVPP2_MAX_TXD_MAX)
+		new_tx_pending = MVPP2_MAX_TXD_MAX;
 	else if (!IS_ALIGNED(ring->tx_pending, 32))
 		new_tx_pending = ALIGN(ring->tx_pending, 32);
 
@@ -7127,6 +7137,7 @@ static void mvpp2_set_rx_mode(struct net_device *dev)
 	int id = port->id;
 	bool allmulti = dev->flags & IFF_ALLMULTI;
 
+retry:
 	mvpp2_prs_mac_promisc_set(priv, id, dev->flags & IFF_PROMISC);
 	mvpp2_prs_mac_multi_set(priv, id, MVPP2_PE_MAC_MC_ALL, allmulti);
 	mvpp2_prs_mac_multi_set(priv, id, MVPP2_PE_MAC_MC_IP6, allmulti);
@@ -7134,9 +7145,13 @@ static void mvpp2_set_rx_mode(struct net_device *dev)
 	/* Remove all port->id's mcast enries */
 	mvpp2_prs_mcast_del_all(priv, id);
 
-	if (allmulti && !netdev_mc_empty(dev)) {
-		netdev_for_each_mc_addr(ha, dev)
-			mvpp2_prs_mac_da_accept(priv, id, ha->addr, true);
+	if (!allmulti) {
+		netdev_for_each_mc_addr(ha, dev) {
+			if (mvpp2_prs_mac_da_accept(priv, id, ha->addr, true)) {
+				allmulti = true;
+				goto retry;
+			}
+		}
 	}
 }
 
@@ -7318,9 +7333,10 @@ static int mvpp2_ethtool_get_coalesce(struct net_device *dev,
 {
 	struct mvpp2_port *port = netdev_priv(dev);
 
-	c->rx_coalesce_usecs        = port->rxqs[0]->time_coal;
-	c->rx_max_coalesced_frames  = port->rxqs[0]->pkts_coal;
-	c->tx_max_coalesced_frames =  port->txqs[0]->done_pkts_coal;
+	c->rx_coalesce_usecs       = port->rxqs[0]->time_coal;
+	c->rx_max_coalesced_frames = port->rxqs[0]->pkts_coal;
+	c->tx_max_coalesced_frames = port->txqs[0]->done_pkts_coal;
+	c->tx_coalesce_usecs       = port->tx_time_coal;
 	return 0;
 }
 
@@ -7340,8 +7356,8 @@ static void mvpp2_ethtool_get_ringparam(struct net_device *dev,
 {
 	struct mvpp2_port *port = netdev_priv(dev);
 
-	ring->rx_max_pending = MVPP2_MAX_RXD;
-	ring->tx_max_pending = MVPP2_MAX_TXD;
+	ring->rx_max_pending = MVPP2_MAX_RXD_MAX;
+	ring->tx_max_pending = MVPP2_MAX_TXD_MAX;
 	ring->rx_pending = port->rx_ring_size;
 	ring->tx_pending = port->tx_ring_size;
 }
@@ -7492,7 +7508,10 @@ static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
 			strncpy(irqname, "rx-shared", sizeof(irqname));
 		}
 
-		v->irq = of_irq_get_byname(port_node, irqname);
+		if (port_node)
+			v->irq = of_irq_get_byname(port_node, irqname);
+		else
+			v->irq = fwnode_irq_get(port->fwnode, i);
 		if (v->irq <= 0) {
 			ret = -EINVAL;
 			goto err;
@@ -7704,17 +7723,16 @@ static bool mvpp2_port_has_tx_irqs(struct mvpp2 *priv,
 }
 
 static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
-				     struct device_node *port_node,
+				     struct fwnode_handle *fwnode,
 				     char **mac_from)
 {
 	struct mvpp2_port *port = netdev_priv(dev);
 	char hw_mac_addr[ETH_ALEN] = {0};
-	const char *dt_mac_addr;
+	char fw_mac_addr[ETH_ALEN];
 
-	dt_mac_addr = of_get_mac_address(port_node);
-	if (dt_mac_addr && is_valid_ether_addr(dt_mac_addr)) {
-		*mac_from = "device tree";
-		ether_addr_copy(dev->dev_addr, dt_mac_addr);
+	if (fwnode_get_mac_address(fwnode, fw_mac_addr, ETH_ALEN)) {
+		*mac_from = "firmware node";
+		ether_addr_copy(dev->dev_addr, fw_mac_addr);
 		return;
 	}
 
@@ -7733,13 +7751,14 @@ static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
 
 /* Ports initialization */
 static int mvpp2_port_probe(struct platform_device *pdev,
-			    struct device_node *port_node,
-			    struct mvpp2 *priv, int index)
+			    struct fwnode_handle *port_fwnode,
+			    struct mvpp2 *priv)
 {
 	struct device_node *phy_node;
-	struct phy *comphy;
+	struct phy *comphy = NULL;
 	struct mvpp2_port *port;
 	struct mvpp2_port_pcpu *port_pcpu;
+	struct device_node *port_node = to_of_node(port_fwnode);
 	struct net_device *dev;
 	struct resource *res;
 	char *mac_from = "";
@@ -7750,7 +7769,12 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	int phy_mode;
 	int err, i, cpu;
 
-	has_tx_irqs = mvpp2_port_has_tx_irqs(priv, port_node);
+	if (port_node) {
+		has_tx_irqs = mvpp2_port_has_tx_irqs(priv, port_node);
+	} else {
+		has_tx_irqs = true;
+		queue_mode = MVPP2_QDIST_MULTI_MODE;
+	}
 
 	if (!has_tx_irqs)
 		queue_mode = MVPP2_QDIST_SINGLE_MODE;
@@ -7765,36 +7789,43 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	if (!dev)
 		return -ENOMEM;
 
-	phy_node = of_parse_phandle(port_node, "phy", 0);
-	phy_mode = of_get_phy_mode(port_node);
+	if (port_node)
+		phy_node = of_parse_phandle(port_node, "phy", 0);
+	else
+		phy_node = NULL;
+
+	phy_mode = fwnode_get_phy_mode(port_fwnode);
 	if (phy_mode < 0) {
 		dev_err(&pdev->dev, "incorrect phy mode\n");
 		err = phy_mode;
 		goto err_free_netdev;
 	}
 
-	comphy = devm_of_phy_get(&pdev->dev, port_node, NULL);
-	if (IS_ERR(comphy)) {
-		if (PTR_ERR(comphy) == -EPROBE_DEFER) {
-			err = -EPROBE_DEFER;
-			goto err_free_netdev;
+	if (port_node) {
+		comphy = devm_of_phy_get(&pdev->dev, port_node, NULL);
+		if (IS_ERR(comphy)) {
+			if (PTR_ERR(comphy) == -EPROBE_DEFER) {
+				err = -EPROBE_DEFER;
+				goto err_free_netdev;
+			}
+			comphy = NULL;
 		}
-		comphy = NULL;
 	}
 
-	if (of_property_read_u32(port_node, "port-id", &id)) {
+	if (fwnode_property_read_u32(port_fwnode, "port-id", &id)) {
 		err = -EINVAL;
 		dev_err(&pdev->dev, "missing port-id value\n");
 		goto err_free_netdev;
 	}
 
-	dev->tx_queue_len = MVPP2_MAX_TXD;
+	dev->tx_queue_len = MVPP2_MAX_TXD_MAX;
 	dev->watchdog_timeo = 5 * HZ;
 	dev->netdev_ops = &mvpp2_netdev_ops;
 	dev->ethtool_ops = &mvpp2_eth_tool_ops;
 
 	port = netdev_priv(dev);
 	port->dev = dev;
+	port->fwnode = port_fwnode;
 	port->ntxqs = ntxqs;
 	port->nrxqs = nrxqs;
 	port->priv = priv;
@@ -7804,7 +7835,10 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	if (err)
 		goto err_free_netdev;
 
-	port->link_irq = of_irq_get_byname(port_node, "link");
+	if (port_node)
+		port->link_irq = of_irq_get_byname(port_node, "link");
+	else
+		port->link_irq = fwnode_irq_get(port_fwnode, port->nqvecs + 1);
 	if (port->link_irq == -EPROBE_DEFER) {
 		err = -EPROBE_DEFER;
 		goto err_deinit_qvecs;
@@ -7813,7 +7847,7 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 		/* the link irq is optional */
 		port->link_irq = 0;
 
-	if (of_property_read_bool(port_node, "marvell,loopback"))
+	if (fwnode_property_read_bool(port_fwnode, "marvell,loopback"))
 		port->flags |= MVPP2_F_LOOPBACK;
 
 	port->id = id;
@@ -7838,8 +7872,8 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 				   MVPP21_MIB_COUNTERS_OFFSET +
 				   port->gop_id * MVPP21_MIB_COUNTERS_PORT_SZ;
 	} else {
-		if (of_property_read_u32(port_node, "gop-port-id",
-					 &port->gop_id)) {
+		if (fwnode_property_read_u32(port_fwnode, "gop-port-id",
+					     &port->gop_id)) {
 			err = -EINVAL;
 			dev_err(&pdev->dev, "missing gop-port-id value\n");
 			goto err_deinit_qvecs;
@@ -7869,10 +7903,10 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	mutex_init(&port->gather_stats_lock);
 	INIT_DELAYED_WORK(&port->stats_work, mvpp2_gather_hw_statistics);
 
-	mvpp2_port_copy_mac_addr(dev, priv, port_node, &mac_from);
+	mvpp2_port_copy_mac_addr(dev, priv, port_fwnode, &mac_from);
 
-	port->tx_ring_size = MVPP2_MAX_TXD;
-	port->rx_ring_size = MVPP2_MAX_RXD;
+	port->tx_ring_size = MVPP2_MAX_TXD_DFLT;
+	port->rx_ring_size = MVPP2_MAX_RXD_DFLT;
 	SET_NETDEV_DEV(dev, &pdev->dev);
 
 	err = mvpp2_port_init(port);
@@ -7927,7 +7961,8 @@ static int mvpp2_port_probe(struct platform_device *pdev,
 	}
 	netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr);
 
-	priv->port_list[index] = port;
+	priv->port_list[priv->port_count++] = port;
+
 	return 0;
 
 err_free_port_pcpu:
@@ -8186,8 +8221,9 @@ static int mvpp2_init(struct platform_device *pdev, struct mvpp2 *priv)
 
 static int mvpp2_probe(struct platform_device *pdev)
 {
-	struct device_node *dn = pdev->dev.of_node;
-	struct device_node *port_node;
+	const struct acpi_device_id *acpi_id;
+	struct fwnode_handle *fwnode = pdev->dev.fwnode;
+	struct fwnode_handle *port_fwnode;
 	struct mvpp2 *priv;
 	struct resource *res;
 	void __iomem *base;
@@ -8198,8 +8234,14 @@ static int mvpp2_probe(struct platform_device *pdev)
 	if (!priv)
 		return -ENOMEM;
 
-	priv->hw_version =
-		(unsigned long)of_device_get_match_data(&pdev->dev);
+	if (has_acpi_companion(&pdev->dev)) {
+		acpi_id = acpi_match_device(pdev->dev.driver->acpi_match_table,
+					    &pdev->dev);
+		priv->hw_version = (unsigned long)acpi_id->driver_data;
+	} else {
+		priv->hw_version =
+			(unsigned long)of_device_get_match_data(&pdev->dev);
+	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(&pdev->dev, res);
@@ -8213,10 +8255,23 @@ static int mvpp2_probe(struct platform_device *pdev)
 			return PTR_ERR(priv->lms_base);
 	} else {
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+		if (has_acpi_companion(&pdev->dev)) {
+			/* In case the MDIO memory region is declared in
+			 * the ACPI, it can already appear as 'in-use'
+			 * in the OS. Because it is overlapped by second
+			 * region of the network controller, make
+			 * sure it is released, before requesting it again.
+			 * The care is taken by mvpp2 driver to avoid
+			 * concurrent access to this memory region.
+			 */
+			release_resource(res);
+		}
 		priv->iface_base = devm_ioremap_resource(&pdev->dev, res);
 		if (IS_ERR(priv->iface_base))
 			return PTR_ERR(priv->iface_base);
+	}
 
+	if (priv->hw_version == MVPP22 && dev_of_node(&pdev->dev)) {
 		priv->sysctrl_base =
 			syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 							"marvell,system-controller");
@@ -8242,32 +8297,34 @@ static int mvpp2_probe(struct platform_device *pdev)
 	else
 		priv->max_port_rxqs = 32;
 
-	priv->pp_clk = devm_clk_get(&pdev->dev, "pp_clk");
-	if (IS_ERR(priv->pp_clk))
-		return PTR_ERR(priv->pp_clk);
-	err = clk_prepare_enable(priv->pp_clk);
-	if (err < 0)
-		return err;
-
-	priv->gop_clk = devm_clk_get(&pdev->dev, "gop_clk");
-	if (IS_ERR(priv->gop_clk)) {
-		err = PTR_ERR(priv->gop_clk);
-		goto err_pp_clk;
-	}
-	err = clk_prepare_enable(priv->gop_clk);
-	if (err < 0)
-		goto err_pp_clk;
+	if (dev_of_node(&pdev->dev)) {
+		priv->pp_clk = devm_clk_get(&pdev->dev, "pp_clk");
+		if (IS_ERR(priv->pp_clk))
+			return PTR_ERR(priv->pp_clk);
+		err = clk_prepare_enable(priv->pp_clk);
+		if (err < 0)
+			return err;
 
-	if (priv->hw_version == MVPP22) {
-		priv->mg_clk = devm_clk_get(&pdev->dev, "mg_clk");
-		if (IS_ERR(priv->mg_clk)) {
-			err = PTR_ERR(priv->mg_clk);
-			goto err_gop_clk;
+		priv->gop_clk = devm_clk_get(&pdev->dev, "gop_clk");
+		if (IS_ERR(priv->gop_clk)) {
+			err = PTR_ERR(priv->gop_clk);
+			goto err_pp_clk;
 		}
-
-		err = clk_prepare_enable(priv->mg_clk);
+		err = clk_prepare_enable(priv->gop_clk);
 		if (err < 0)
-			goto err_gop_clk;
+			goto err_pp_clk;
+
+		if (priv->hw_version == MVPP22) {
+			priv->mg_clk = devm_clk_get(&pdev->dev, "mg_clk");
+			if (IS_ERR(priv->mg_clk)) {
+				err = PTR_ERR(priv->mg_clk);
+				goto err_gop_clk;
+			}
+
+			err = clk_prepare_enable(priv->mg_clk);
+			if (err < 0)
+				goto err_gop_clk;
+		}
 
 		priv->axi_clk = devm_clk_get(&pdev->dev, "axi_clk");
 		if (IS_ERR(priv->axi_clk)) {
@@ -8280,10 +8337,14 @@ static int mvpp2_probe(struct platform_device *pdev)
 			if (err < 0)
 				goto err_gop_clk;
 		}
-	}
 
-	/* Get system's tclk rate */
-	priv->tclk = clk_get_rate(priv->pp_clk);
+		/* Get system's tclk rate */
+		priv->tclk = clk_get_rate(priv->pp_clk);
+	} else if (device_property_read_u32(&pdev->dev, "clock-frequency",
+					    &priv->tclk)) {
+		dev_err(&pdev->dev, "missing clock-frequency value\n");
+		return -EINVAL;
+	}
 
 	if (priv->hw_version == MVPP22) {
 		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(40));
@@ -8306,30 +8367,19 @@ static int mvpp2_probe(struct platform_device *pdev)
 		goto err_mg_clk;
 	}
 
-	priv->port_count = of_get_available_child_count(dn);
+	/* Initialize ports */
+	fwnode_for_each_available_child_node(fwnode, port_fwnode) {
+		err = mvpp2_port_probe(pdev, port_fwnode, priv);
+		if (err < 0)
+			goto err_port_probe;
+	}
+
 	if (priv->port_count == 0) {
 		dev_err(&pdev->dev, "no ports enabled\n");
 		err = -ENODEV;
 		goto err_mg_clk;
 	}
 
-	priv->port_list = devm_kcalloc(&pdev->dev, priv->port_count,
-				       sizeof(*priv->port_list),
-				       GFP_KERNEL);
-	if (!priv->port_list) {
-		err = -ENOMEM;
-		goto err_mg_clk;
-	}
-
-	/* Initialize ports */
-	i = 0;
-	for_each_available_child_of_node(dn, port_node) {
-		err = mvpp2_port_probe(pdev, port_node, priv, i);
-		if (err < 0)
-			goto err_port_probe;
-		i++;
-	}
-
 	/* Statistics must be gathered regularly because some of them (like
 	 * packets counters) are 32-bit registers and could overflow quite
 	 * quickly. For instance, a 10Gb link used at full bandwidth with the
@@ -8350,7 +8400,7 @@ static int mvpp2_probe(struct platform_device *pdev)
 
 err_port_probe:
 	i = 0;
-	for_each_available_child_of_node(dn, port_node) {
+	fwnode_for_each_available_child_node(fwnode, port_fwnode) {
 		if (priv->port_list[i])
 			mvpp2_port_remove(priv->port_list[i]);
 		i++;
@@ -8369,14 +8419,14 @@ static int mvpp2_probe(struct platform_device *pdev)
 static int mvpp2_remove(struct platform_device *pdev)
 {
 	struct mvpp2 *priv = platform_get_drvdata(pdev);
-	struct device_node *dn = pdev->dev.of_node;
-	struct device_node *port_node;
+	struct fwnode_handle *fwnode = pdev->dev.fwnode;
+	struct fwnode_handle *port_fwnode;
 	int i = 0;
 
 	flush_workqueue(priv->stats_queue);
 	destroy_workqueue(priv->stats_queue);
 
-	for_each_available_child_of_node(dn, port_node) {
+	fwnode_for_each_available_child_node(fwnode, port_fwnode) {
 		if (priv->port_list[i]) {
 			mutex_destroy(&priv->port_list[i]->gather_stats_lock);
 			mvpp2_port_remove(priv->port_list[i]);
@@ -8399,6 +8449,9 @@ static int mvpp2_remove(struct platform_device *pdev)
 				  aggr_txq->descs_dma);
 	}
 
+	if (is_acpi_node(port_fwnode))
+		return 0;
+
 	clk_disable_unprepare(priv->axi_clk);
 	clk_disable_unprepare(priv->mg_clk);
 	clk_disable_unprepare(priv->pp_clk);
@@ -8420,12 +8473,19 @@ static const struct of_device_id mvpp2_match[] = {
 };
 MODULE_DEVICE_TABLE(of, mvpp2_match);
 
+static const struct acpi_device_id mvpp2_acpi_match[] = {
+	{ "MRVL0110", MVPP22 },
+	{ },
+};
+MODULE_DEVICE_TABLE(acpi, mvpp2_acpi_match);
+
 static struct platform_driver mvpp2_driver = {
 	.probe = mvpp2_probe,
 	.remove = mvpp2_remove,
 	.driver = {
 		.name = MVPP2_DRIVER_NAME,
 		.of_match_table = mvpp2_match,
+		.acpi_match_table = ACPI_PTR(mvpp2_acpi_match),
 	},
 };
 
diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c
index 7bbd86f08e5f..3a9730612a70 100644
--- a/drivers/net/ethernet/marvell/pxa168_eth.c
+++ b/drivers/net/ethernet/marvell/pxa168_eth.c
@@ -1362,6 +1362,15 @@ static int pxa168_eth_do_ioctl(struct net_device *dev, struct ifreq *ifr,
 	return -EOPNOTSUPP;
 }
 
+#ifdef CONFIG_NET_POLL_CONTROLLER
+static void pxa168_eth_netpoll(struct net_device *dev)
+{
+	disable_irq(dev->irq);
+	pxa168_eth_int_handler(dev->irq, dev);
+	enable_irq(dev->irq);
+}
+#endif
+
 static void pxa168_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
@@ -1390,6 +1399,9 @@ static const struct net_device_ops pxa168_eth_netdev_ops = {
 	.ndo_do_ioctl		= pxa168_eth_do_ioctl,
 	.ndo_change_mtu		= pxa168_eth_change_mtu,
 	.ndo_tx_timeout		= pxa168_eth_tx_timeout,
+#ifdef CONFIG_NET_POLL_CONTROLLER
+	.ndo_poll_controller    = pxa168_eth_netpoll,
+#endif
 };
 
 static int pxa168_eth_probe(struct platform_device *pdev)
diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c
index 9efe1771423c..9fe85300e7b6 100644
--- a/drivers/net/ethernet/marvell/sky2.c
+++ b/drivers/net/ethernet/marvell/sky2.c
@@ -4287,7 +4287,7 @@ static int sky2_vpd_wait(const struct sky2_hw *hw, int cap, u16 busy)
 			dev_err(&hw->pdev->dev, "VPD cycle timed out\n");
 			return -ETIMEDOUT;
 		}
-		mdelay(1);
+		msleep(1);
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index fc67e35b253e..e0b72bf428f7 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -23,6 +23,7 @@
 #include <linux/reset.h>
 #include <linux/tcp.h>
 #include <linux/interrupt.h>
+#include <linux/pinctrl/devinfo.h>
 
 #include "mtk_eth_soc.h"
 
@@ -1952,14 +1953,16 @@ static int mtk_hw_init(struct mtk_eth *eth)
 	}
 	regmap_write(eth->ethsys, ETHSYS_SYSCFG0, val);
 
-	/* Set GE2 driving and slew rate */
-	regmap_write(eth->pctl, GPIO_DRV_SEL10, 0xa00);
+	if (eth->pctl) {
+		/* Set GE2 driving and slew rate */
+		regmap_write(eth->pctl, GPIO_DRV_SEL10, 0xa00);
 
-	/* set GE2 TDSEL */
-	regmap_write(eth->pctl, GPIO_OD33_CTRL8, 0x5);
+		/* set GE2 TDSEL */
+		regmap_write(eth->pctl, GPIO_OD33_CTRL8, 0x5);
 
-	/* set GE2 TUNE */
-	regmap_write(eth->pctl, GPIO_BIAS_CTRL, 0x0);
+		/* set GE2 TUNE */
+		regmap_write(eth->pctl, GPIO_BIAS_CTRL, 0x0);
+	}
 
 	/* Set linkdown as the default for each GMAC. Its own MCR would be set
 	 * up with the more appropriate value when mtk_phy_link_adjust call is
@@ -2538,11 +2541,13 @@ static int mtk_probe(struct platform_device *pdev)
 		}
 	}
 
-	eth->pctl = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
-						    "mediatek,pctl");
-	if (IS_ERR(eth->pctl)) {
-		dev_err(&pdev->dev, "no pctl regmap found\n");
-		return PTR_ERR(eth->pctl);
+	if (eth->soc->required_pctl) {
+		eth->pctl = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
+							    "mediatek,pctl");
+		if (IS_ERR(eth->pctl)) {
+			dev_err(&pdev->dev, "no pctl regmap found\n");
+			return PTR_ERR(eth->pctl);
+		}
 	}
 
 	for (i = 0; i < 3; i++) {
@@ -2668,17 +2673,20 @@ static int mtk_remove(struct platform_device *pdev)
 
 static const struct mtk_soc_data mt2701_data = {
 	.caps = MTK_GMAC1_TRGMII,
-	.required_clks = MT7623_CLKS_BITMAP
+	.required_clks = MT7623_CLKS_BITMAP,
+	.required_pctl = true,
 };
 
 static const struct mtk_soc_data mt7622_data = {
 	.caps = MTK_DUAL_GMAC_SHARED_SGMII | MTK_GMAC1_ESW,
-	.required_clks = MT7622_CLKS_BITMAP
+	.required_clks = MT7622_CLKS_BITMAP,
+	.required_pctl = false,
 };
 
 static const struct mtk_soc_data mt7623_data = {
 	.caps = MTK_GMAC1_TRGMII,
-	.required_clks = MT7623_CLKS_BITMAP
+	.required_clks = MT7623_CLKS_BITMAP,
+	.required_pctl = true,
 };
 
 const struct of_device_id of_mtk_match[] = {
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
index a3af4660de81..672b8c353c47 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
@@ -573,10 +573,13 @@ struct mtk_rx_ring {
  * @caps			Flags shown the extra capability for the SoC
  * @required_clks		Flags shown the bitmap for required clocks on
  *				the target SoC
+ * @required_pctl		A bool value to show whether the SoC requires
+ *				the extra setup for those pins used by GMAC.
  */
 struct mtk_soc_data {
 	u32		caps;
 	u32		required_clks;
+	bool		required_pctl;
 };
 
 /* currently no SoC has more than 2 macs */
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c b/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
index 5f41dc92aa68..752a72499b4f 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_dcb_nl.c
@@ -156,57 +156,63 @@ static int mlx4_en_dcbnl_getnumtcs(struct net_device *netdev, int tcid, u8 *num)
 static u8 mlx4_en_dcbnl_set_all(struct net_device *netdev)
 {
 	struct mlx4_en_priv *priv = netdev_priv(netdev);
+	struct mlx4_en_port_profile *prof = priv->prof;
 	struct mlx4_en_dev *mdev = priv->mdev;
+	u8 tx_pause, tx_ppp, rx_pause, rx_ppp;
 
 	if (!(priv->dcbx_cap & DCB_CAP_DCBX_VER_CEE))
 		return 1;
 
 	if (priv->cee_config.pfc_state) {
 		int tc;
+		rx_ppp = prof->rx_ppp;
+		tx_ppp = prof->tx_ppp;
 
-		priv->prof->rx_pause = 0;
-		priv->prof->tx_pause = 0;
 		for (tc = 0; tc < CEE_DCBX_MAX_PRIO; tc++) {
 			u8 tc_mask = 1 << tc;
 
 			switch (priv->cee_config.dcb_pfc[tc]) {
 			case pfc_disabled:
-				priv->prof->tx_ppp &= ~tc_mask;
-				priv->prof->rx_ppp &= ~tc_mask;
+				tx_ppp &= ~tc_mask;
+				rx_ppp &= ~tc_mask;
 				break;
 			case pfc_enabled_full:
-				priv->prof->tx_ppp |= tc_mask;
-				priv->prof->rx_ppp |= tc_mask;
+				tx_ppp |= tc_mask;
+				rx_ppp |= tc_mask;
 				break;
 			case pfc_enabled_tx:
-				priv->prof->tx_ppp |= tc_mask;
-				priv->prof->rx_ppp &= ~tc_mask;
+				tx_ppp |= tc_mask;
+				rx_ppp &= ~tc_mask;
 				break;
 			case pfc_enabled_rx:
-				priv->prof->tx_ppp &= ~tc_mask;
-				priv->prof->rx_ppp |= tc_mask;
+				tx_ppp &= ~tc_mask;
+				rx_ppp |= tc_mask;
 				break;
 			default:
 				break;
 			}
 		}
-		en_dbg(DRV, priv, "Set pfc on\n");
+		rx_pause = !!(rx_ppp || tx_ppp) ? 0 : prof->rx_pause;
+		tx_pause = !!(rx_ppp || tx_ppp) ? 0 : prof->tx_pause;
 	} else {
-		priv->prof->rx_pause = 1;
-		priv->prof->tx_pause = 1;
-		en_dbg(DRV, priv, "Set pfc off\n");
+		rx_ppp = 0;
+		tx_ppp = 0;
+		rx_pause = prof->rx_pause;
+		tx_pause = prof->tx_pause;
 	}
 
 	if (mlx4_SET_PORT_general(mdev->dev, priv->port,
 				  priv->rx_skb_size + ETH_FCS_LEN,
-				  priv->prof->tx_pause,
-				  priv->prof->tx_ppp,
-				  priv->prof->rx_pause,
-				  priv->prof->rx_ppp)) {
+				  tx_pause, tx_ppp, rx_pause, rx_ppp)) {
 		en_err(priv, "Failed setting pause params\n");
 		return 1;
 	}
 
+	prof->tx_ppp = tx_ppp;
+	prof->rx_ppp = rx_ppp;
+	prof->tx_pause = tx_pause;
+	prof->rx_pause = rx_pause;
+
 	return 0;
 }
 
@@ -310,6 +316,7 @@ static int mlx4_en_ets_validate(struct mlx4_en_priv *priv, struct ieee_ets *ets)
 		}
 
 		switch (ets->tc_tsa[i]) {
+		case IEEE_8021QAZ_TSA_VENDOR:
 		case IEEE_8021QAZ_TSA_STRICT:
 			break;
 		case IEEE_8021QAZ_TSA_ETS:
@@ -347,6 +354,10 @@ static int mlx4_en_config_port_scheduler(struct mlx4_en_priv *priv,
 	/* higher TC means higher priority => lower pg */
 	for (i = IEEE_8021QAZ_MAX_TCS - 1; i >= 0; i--) {
 		switch (ets->tc_tsa[i]) {
+		case IEEE_8021QAZ_TSA_VENDOR:
+			pg[i] = MLX4_EN_TC_VENDOR;
+			tc_tx_bw[i] = MLX4_EN_BW_MAX;
+			break;
 		case IEEE_8021QAZ_TSA_STRICT:
 			pg[i] = num_strict++;
 			tc_tx_bw[i] = MLX4_EN_BW_MAX;
@@ -403,6 +414,7 @@ static int mlx4_en_dcbnl_ieee_setpfc(struct net_device *dev,
 	struct mlx4_en_priv *priv = netdev_priv(dev);
 	struct mlx4_en_port_profile *prof = priv->prof;
 	struct mlx4_en_dev *mdev = priv->mdev;
+	u32 tx_pause, tx_ppp, rx_pause, rx_ppp;
 	int err;
 
 	en_dbg(DRV, priv, "cap: 0x%x en: 0x%x mbc: 0x%x delay: %d\n",
@@ -411,23 +423,26 @@ static int mlx4_en_dcbnl_ieee_setpfc(struct net_device *dev,
 			pfc->mbc,
 			pfc->delay);
 
-	prof->rx_pause = !pfc->pfc_en;
-	prof->tx_pause = !pfc->pfc_en;
-	prof->rx_ppp = pfc->pfc_en;
-	prof->tx_ppp = pfc->pfc_en;
+	rx_pause = prof->rx_pause && !pfc->pfc_en;
+	tx_pause = prof->tx_pause && !pfc->pfc_en;
+	rx_ppp = pfc->pfc_en;
+	tx_ppp = pfc->pfc_en;
 
 	err = mlx4_SET_PORT_general(mdev->dev, priv->port,
 				    priv->rx_skb_size + ETH_FCS_LEN,
-				    prof->tx_pause,
-				    prof->tx_ppp,
-				    prof->rx_pause,
-				    prof->rx_ppp);
-	if (err)
+				    tx_pause, tx_ppp, rx_pause, rx_ppp);
+	if (err) {
 		en_err(priv, "Failed setting pause params\n");
-	else
-		mlx4_en_update_pfc_stats_bitmap(mdev->dev, &priv->stats_bitmap,
-						prof->rx_ppp, prof->rx_pause,
-						prof->tx_ppp, prof->tx_pause);
+		return err;
+	}
+
+	mlx4_en_update_pfc_stats_bitmap(mdev->dev, &priv->stats_bitmap,
+					rx_ppp, rx_pause, tx_ppp, tx_pause);
+
+	prof->tx_ppp = tx_ppp;
+	prof->rx_ppp = rx_ppp;
+	prof->rx_pause = rx_pause;
+	prof->tx_pause = tx_pause;
 
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
index bf1f04164885..f3302edba8b4 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
@@ -1046,27 +1046,32 @@ static int mlx4_en_set_pauseparam(struct net_device *dev,
 {
 	struct mlx4_en_priv *priv = netdev_priv(dev);
 	struct mlx4_en_dev *mdev = priv->mdev;
+	u8 tx_pause, tx_ppp, rx_pause, rx_ppp;
 	int err;
 
 	if (pause->autoneg)
 		return -EINVAL;
 
-	priv->prof->tx_pause = pause->tx_pause != 0;
-	priv->prof->rx_pause = pause->rx_pause != 0;
+	tx_pause = !!(pause->tx_pause);
+	rx_pause = !!(pause->rx_pause);
+	rx_ppp = priv->prof->rx_ppp && !(tx_pause || rx_pause);
+	tx_ppp = priv->prof->tx_ppp && !(tx_pause || rx_pause);
+
 	err = mlx4_SET_PORT_general(mdev->dev, priv->port,
 				    priv->rx_skb_size + ETH_FCS_LEN,
-				    priv->prof->tx_pause,
-				    priv->prof->tx_ppp,
-				    priv->prof->rx_pause,
-				    priv->prof->rx_ppp);
-	if (err)
-		en_err(priv, "Failed setting pause params\n");
-	else
-		mlx4_en_update_pfc_stats_bitmap(mdev->dev, &priv->stats_bitmap,
-						priv->prof->rx_ppp,
-						priv->prof->rx_pause,
-						priv->prof->tx_ppp,
-						priv->prof->tx_pause);
+				    tx_pause, tx_ppp, rx_pause, rx_ppp);
+	if (err) {
+		en_err(priv, "Failed setting pause params, err = %d\n", err);
+		return err;
+	}
+
+	mlx4_en_update_pfc_stats_bitmap(mdev->dev, &priv->stats_bitmap,
+					rx_ppp, rx_pause, tx_ppp, tx_pause);
+
+	priv->prof->tx_pause = tx_pause;
+	priv->prof->rx_pause = rx_pause;
+	priv->prof->tx_ppp = tx_ppp;
+	priv->prof->rx_ppp = rx_ppp;
 
 	return err;
 }
@@ -1094,12 +1099,21 @@ static int mlx4_en_set_ringparam(struct net_device *dev,
 	if (param->rx_jumbo_pending || param->rx_mini_pending)
 		return -EINVAL;
 
+	if (param->rx_pending < MLX4_EN_MIN_RX_SIZE) {
+		en_warn(priv, "%s: rx_pending (%d) < min (%d)\n",
+			__func__, param->rx_pending,
+			MLX4_EN_MIN_RX_SIZE);
+		return -EINVAL;
+	}
+	if (param->tx_pending < MLX4_EN_MIN_TX_SIZE) {
+		en_warn(priv, "%s: tx_pending (%d) < min (%lu)\n",
+			__func__, param->tx_pending,
+			MLX4_EN_MIN_TX_SIZE);
+		return -EINVAL;
+	}
+
 	rx_size = roundup_pow_of_two(param->rx_pending);
-	rx_size = max_t(u32, rx_size, MLX4_EN_MIN_RX_SIZE);
-	rx_size = min_t(u32, rx_size, MLX4_EN_MAX_RX_SIZE);
 	tx_size = roundup_pow_of_two(param->tx_pending);
-	tx_size = max_t(u32, tx_size, MLX4_EN_MIN_TX_SIZE);
-	tx_size = min_t(u32, tx_size, MLX4_EN_MAX_TX_SIZE);
 
 	if (rx_size == (priv->port_up ? priv->rx_ring[0]->actual_size :
 					priv->rx_ring[0]->size) &&
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_main.c b/drivers/net/ethernet/mellanox/mlx4/en_main.c
index 2c2965497ed3..d25e16d2c319 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_main.c
@@ -163,9 +163,9 @@ static void mlx4_en_get_profile(struct mlx4_en_dev *mdev)
 		params->udp_rss = 0;
 	}
 	for (i = 1; i <= MLX4_MAX_PORTS; i++) {
-		params->prof[i].rx_pause = 1;
+		params->prof[i].rx_pause = !(pfcrx || pfctx);
 		params->prof[i].rx_ppp = pfcrx;
-		params->prof[i].tx_pause = 1;
+		params->prof[i].tx_pause = !(pfcrx || pfctx);
 		params->prof[i].tx_ppp = pfctx;
 		params->prof[i].tx_ring_size = MLX4_EN_DEF_TX_RING_SIZE;
 		params->prof[i].rx_ring_size = MLX4_EN_DEF_RX_RING_SIZE;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 99051a294fa6..8fc51bc29003 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -2172,8 +2172,9 @@ static int mlx4_en_alloc_resources(struct mlx4_en_priv *priv)
 
 		if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i],
 					   prof->rx_ring_size, priv->stride,
-					   node))
+					   node, i))
 			goto err;
+
 	}
 
 #ifdef CONFIG_RFS_ACCEL
@@ -3336,6 +3337,13 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
 	priv->msg_enable = MLX4_EN_MSG_LEVEL;
 #ifdef CONFIG_MLX4_EN_DCB
 	if (!mlx4_is_slave(priv->mdev->dev)) {
+		u8 prio;
+
+		for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; ++prio) {
+			priv->ets.prio_tc[prio] = prio;
+			priv->ets.tc_tsa[prio]  = IEEE_8021QAZ_TSA_VENDOR;
+		}
+
 		priv->dcbx_cap = DCB_CAP_DCBX_VER_CEE | DCB_CAP_DCBX_HOST |
 			DCB_CAP_DCBX_VER_IEEE;
 		priv->flags |= MLX4_EN_DCB_ENABLED;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 85e28efcda33..b4d144e67514 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -262,7 +262,7 @@ void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev)
 
 int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
 			   struct mlx4_en_rx_ring **pring,
-			   u32 size, u16 stride, int node)
+			   u32 size, u16 stride, int node, int queue_index)
 {
 	struct mlx4_en_dev *mdev = priv->mdev;
 	struct mlx4_en_rx_ring *ring;
@@ -286,6 +286,9 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
 	ring->log_stride = ffs(ring->stride) - 1;
 	ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
 
+	if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index) < 0)
+		goto err_ring;
+
 	tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
 					sizeof(struct mlx4_en_rx_alloc));
 	ring->rx_info = vzalloc_node(tmp, node);
@@ -293,7 +296,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
 		ring->rx_info = vzalloc(tmp);
 		if (!ring->rx_info) {
 			err = -ENOMEM;
-			goto err_ring;
+			goto err_xdp_info;
 		}
 	}
 
@@ -317,6 +320,8 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
 err_info:
 	vfree(ring->rx_info);
 	ring->rx_info = NULL;
+err_xdp_info:
+	xdp_rxq_info_unreg(&ring->xdp_rxq);
 err_ring:
 	kfree(ring);
 	*pring = NULL;
@@ -440,6 +445,7 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
 					lockdep_is_held(&mdev->state_lock));
 	if (old_prog)
 		bpf_prog_put(old_prog);
+	xdp_rxq_info_unreg(&ring->xdp_rxq);
 	mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
 	vfree(ring->rx_info);
 	ring->rx_info = NULL;
@@ -617,6 +623,10 @@ static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb,
 	return 0;
 }
 #endif
+
+/* We reach this function only after checking that any of
+ * the (IPv4 | IPv6) bits are set in cqe->status.
+ */
 static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
 		      netdev_features_t dev_features)
 {
@@ -632,13 +642,11 @@ static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
 		hdr += sizeof(struct vlan_hdr);
 	}
 
-	if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4))
-		return get_fixed_ipv4_csum(hw_checksum, skb, hdr);
 #if IS_ENABLED(CONFIG_IPV6)
 	if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV6))
 		return get_fixed_ipv6_csum(hw_checksum, skb, hdr);
 #endif
-	return 0;
+	return get_fixed_ipv4_csum(hw_checksum, skb, hdr);
 }
 
 int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
@@ -650,6 +658,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 	int cq_ring = cq->ring;
 	bool doorbell_pending;
 	struct mlx4_cqe *cqe;
+	struct xdp_buff xdp;
 	int polled = 0;
 	int index;
 
@@ -664,6 +673,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 	/* Protect accesses to: ring->xdp_prog, priv->mac_hash list */
 	rcu_read_lock();
 	xdp_prog = rcu_dereference(ring->xdp_prog);
+	xdp.rxq = &ring->xdp_rxq;
 	doorbell_pending = 0;
 
 	/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
@@ -748,7 +758,6 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 		 * read bytes but not past the end of the frag.
 		 */
 		if (xdp_prog) {
-			struct xdp_buff xdp;
 			dma_addr_t dma;
 			void *orig_data;
 			u32 act;
@@ -814,33 +823,33 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int bud
 		if (likely(dev->features & NETIF_F_RXCSUM)) {
 			if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_TCP |
 						      MLX4_CQE_STATUS_UDP)) {
-				if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
-				    cqe->checksum == cpu_to_be16(0xffff)) {
-					bool l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
-						(cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
-
-					ip_summed = CHECKSUM_UNNECESSARY;
-					hash_type = PKT_HASH_TYPE_L4;
-					if (l2_tunnel)
-						skb->csum_level = 1;
-					ring->csum_ok++;
-				} else {
+				bool l2_tunnel;
+
+				if (!((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
+				      cqe->checksum == cpu_to_be16(0xffff)))
 					goto csum_none;
-				}
+
+				l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
+					(cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
+				ip_summed = CHECKSUM_UNNECESSARY;
+				hash_type = PKT_HASH_TYPE_L4;
+				if (l2_tunnel)
+					skb->csum_level = 1;
+				ring->csum_ok++;
 			} else {
-				if (priv->flags & MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP &&
-				    (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
-							       MLX4_CQE_STATUS_IPV6))) {
-					if (check_csum(cqe, skb, va, dev->features)) {
-						goto csum_none;
-					} else {
-						ip_summed = CHECKSUM_COMPLETE;
-						hash_type = PKT_HASH_TYPE_L3;
-						ring->csum_complete++;
-					}
-				} else {
+				if (!(priv->flags & MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP &&
+				      (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 |
+#if IS_ENABLED(CONFIG_IPV6)
+								 MLX4_CQE_STATUS_IPV6))))
+#else
+								 0))))
+#endif
 					goto csum_none;
-				}
+				if (check_csum(cqe, skb, va, dev->features))
+					goto csum_none;
+				ip_summed = CHECKSUM_COMPLETE;
+				hash_type = PKT_HASH_TYPE_L3;
+				ring->csum_complete++;
 			}
 		} else {
 csum_none:
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index 2b72677eccd4..f470ae37d937 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -46,6 +46,7 @@
 #endif
 #include <linux/cpu_rmap.h>
 #include <linux/ptp_clock_kernel.h>
+#include <net/xdp.h>
 
 #include <linux/mlx4/device.h>
 #include <linux/mlx4/qp.h>
@@ -356,6 +357,7 @@ struct mlx4_en_rx_ring {
 	unsigned long dropped;
 	int hwtstamp_rx_filter;
 	cpumask_var_t affinity_mask;
+	struct xdp_rxq_info xdp_rxq;
 };
 
 struct mlx4_en_cq {
@@ -479,6 +481,7 @@ struct mlx4_en_frag_info {
 #define MLX4_EN_BW_MIN 1
 #define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */
 
+#define MLX4_EN_TC_VENDOR 0
 #define MLX4_EN_TC_ETS 7
 
 enum dcb_pfc_type {
@@ -719,7 +722,7 @@ void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev);
 void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv);
 int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
 			   struct mlx4_en_rx_ring **pring,
-			   u32 size, u16 stride, int node);
+			   u32 size, u16 stride, int node, int queue_index);
 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
 			     struct mlx4_en_rx_ring **pring,
 			     u32 size, u16 stride);
diff --git a/drivers/net/ethernet/mellanox/mlx4/mr.c b/drivers/net/ethernet/mellanox/mlx4/mr.c
index c7c0764991c9..2e84f10f59ba 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mr.c
+++ b/drivers/net/ethernet/mellanox/mlx4/mr.c
@@ -1103,30 +1103,16 @@ EXPORT_SYMBOL_GPL(mlx4_fmr_enable);
 void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,
 		    u32 *lkey, u32 *rkey)
 {
-	struct mlx4_cmd_mailbox *mailbox;
-	int err;
-
 	if (!fmr->maps)
 		return;
 
-	fmr->maps = 0;
+	/* To unmap: it is sufficient to take back ownership from HW */
+	*(u8 *)fmr->mpt = MLX4_MPT_STATUS_SW;
 
-	mailbox = mlx4_alloc_cmd_mailbox(dev);
-	if (IS_ERR(mailbox)) {
-		err = PTR_ERR(mailbox);
-		pr_warn("mlx4_ib: mlx4_alloc_cmd_mailbox failed (%d)\n", err);
-		return;
-	}
+	/* Make sure MPT status is visible */
+	wmb();
 
-	err = mlx4_HW2SW_MPT(dev, NULL,
-			     key_to_hw_index(fmr->mr.key) &
-			     (dev->caps.num_mpts - 1));
-	mlx4_free_cmd_mailbox(dev, mailbox);
-	if (err) {
-		pr_warn("mlx4_ib: mlx4_HW2SW_MPT failed (%d)\n", err);
-		return;
-	}
-	fmr->mr.enabled = MLX4_MPT_EN_SW;
+	fmr->maps = 0;
 }
 EXPORT_SYMBOL_GPL(mlx4_fmr_unmap);
 
@@ -1136,6 +1122,22 @@ int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr)
 
 	if (fmr->maps)
 		return -EBUSY;
+	if (fmr->mr.enabled == MLX4_MPT_EN_HW) {
+		/* In case of FMR was enabled and unmapped
+		 * make sure to give ownership of MPT back to HW
+		 * so HW2SW_MPT command will success.
+		 */
+		*(u8 *)fmr->mpt = MLX4_MPT_STATUS_SW;
+		/* Make sure MPT status is visible before changing MPT fields */
+		wmb();
+		fmr->mpt->length = 0;
+		fmr->mpt->start  = 0;
+		/* Make sure MPT data is visible after changing MPT status */
+		wmb();
+		*(u8 *)fmr->mpt = MLX4_MPT_STATUS_HW;
+		/* make sure MPT status is visible */
+		wmb();
+	}
 
 	ret = mlx4_mr_free(dev, &fmr->mr);
 	if (ret)
diff --git a/drivers/net/ethernet/mellanox/mlx4/qp.c b/drivers/net/ethernet/mellanox/mlx4/qp.c
index 769598f7b6c8..3aaf4bad6c5a 100644
--- a/drivers/net/ethernet/mellanox/mlx4/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx4/qp.c
@@ -287,6 +287,9 @@ void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt)
 	u64 in_param = 0;
 	int err;
 
+	if (!cnt)
+		return;
+
 	if (mlx4_is_mfunc(dev)) {
 		set_param_l(&in_param, base_qpn);
 		set_param_h(&in_param, cnt);
diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
index 606a0e0beeae..29e50f787349 100644
--- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
+++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c
@@ -5088,6 +5088,7 @@ static void rem_slave_fs_rule(struct mlx4_dev *dev, int slave)
 						 &tracker->res_tree[RES_FS_RULE]);
 					list_del(&fs_rule->com.list);
 					spin_unlock_irq(mlx4_tlock(dev));
+					kfree(fs_rule->mirr_mbox);
 					kfree(fs_rule);
 					state = 0;
 					break;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index 25deaa5a534c..c032319f1cb9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -46,7 +46,7 @@ config MLX5_MPFS
 
 config MLX5_ESWITCH
 	bool "Mellanox Technologies MLX5 SRIOV E-Switch support"
-	depends on MLX5_CORE_EN
+	depends on MLX5_CORE_EN && NET_SWITCHDEV
 	default y
 	---help---
 	  Mellanox Technologies Ethernet SRIOV E-Switch support in ConnectX NIC.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 19b21b40ab07..c805769d92a9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -14,7 +14,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \
 		fpga/ipsec.o
 
 mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \
-		en_tx.o en_rx.o en_rx_am.o en_txrx.o en_stats.o vxlan.o \
+		en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \
 		en_arfs.o en_fs_ethtool.o en_selftest.o
 
 mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
index 0be4575b58a2..fd509160c8f6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
@@ -96,10 +96,10 @@ static void print_lyr_2_4_hdrs(struct trace_seq *p,
 					  "%pI4");
 		} else if (ethertype.v == ETH_P_IPV6) {
 			static const struct in6_addr full_ones = {
-				.in6_u.u6_addr32 = {htonl(0xffffffff),
-						    htonl(0xffffffff),
-						    htonl(0xffffffff),
-						    htonl(0xffffffff)},
+				.in6_u.u6_addr32 = {__constant_htonl(0xffffffff),
+						    __constant_htonl(0xffffffff),
+						    __constant_htonl(0xffffffff),
+						    __constant_htonl(0xffffffff)},
 			};
 			DECLARE_MASK_VAL(struct in6_addr, src_ipv6);
 			DECLARE_MASK_VAL(struct in6_addr, dst_ipv6);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index c2d89bfa1a70..4c9360b25532 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -44,8 +44,11 @@
 #include <linux/mlx5/port.h>
 #include <linux/mlx5/vport.h>
 #include <linux/mlx5/transobj.h>
+#include <linux/mlx5/fs.h>
 #include <linux/rhashtable.h>
 #include <net/switchdev.h>
+#include <net/xdp.h>
+#include <linux/net_dim.h>
 #include "wq.h"
 #include "mlx5_core.h"
 #include "en_stats.h"
@@ -226,12 +229,6 @@ enum mlx5e_priv_flag {
 #define MLX5E_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */
 #endif
 
-struct mlx5e_cq_moder {
-	u16 usec;
-	u16 pkts;
-	u8 cq_period_mode;
-};
-
 struct mlx5e_params {
 	u8  log_sq_size;
 	u8  rq_wq_type;
@@ -242,8 +239,8 @@ struct mlx5e_params {
 	u16 num_channels;
 	u8  num_tc;
 	bool rx_cqe_compress_def;
-	struct mlx5e_cq_moder rx_cq_moderation;
-	struct mlx5e_cq_moder tx_cq_moderation;
+	struct net_dim_cq_moder rx_cq_moderation;
+	struct net_dim_cq_moder tx_cq_moderation;
 	bool lro_en;
 	u32 lro_wqe_sz;
 	u16 tx_max_inline;
@@ -253,7 +250,7 @@ struct mlx5e_params {
 	u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE];
 	bool vlan_strip_disable;
 	bool scatter_fcs_en;
-	bool rx_am_enabled;
+	bool rx_dim_enabled;
 	u32 lro_timeout;
 	u32 pflags;
 	struct bpf_prog *xdp_prog;
@@ -472,32 +469,6 @@ struct mlx5e_mpw_info {
 	u16 skbs_frags[MLX5_MPWRQ_PAGES_PER_WQE];
 };
 
-struct mlx5e_rx_am_stats {
-	int ppms; /* packets per msec */
-	int bpms; /* bytes per msec */
-	int epms; /* events per msec */
-};
-
-struct mlx5e_rx_am_sample {
-	ktime_t	time;
-	u32	pkt_ctr;
-	u32	byte_ctr;
-	u16	event_ctr;
-};
-
-struct mlx5e_rx_am { /* Adaptive Moderation */
-	u8					state;
-	struct mlx5e_rx_am_stats		prev_stats;
-	struct mlx5e_rx_am_sample		start_sample;
-	struct work_struct			work;
-	u8					profile_ix;
-	u8					mode;
-	u8					tune_state;
-	u8					steps_right;
-	u8					steps_left;
-	u8					tired;
-};
-
 /* a single cache unit is capable to serve one napi call (for non-striding rq)
  * or a MPWQE (for striding rq).
  */
@@ -558,7 +529,7 @@ struct mlx5e_rq {
 	unsigned long          state;
 	int                    ix;
 
-	struct mlx5e_rx_am     am; /* Adaptive Moderation */
+	struct net_dim         dim; /* Dynamic Interrupt Moderation */
 
 	/* XDP */
 	struct bpf_prog       *xdp_prog;
@@ -571,6 +542,9 @@ struct mlx5e_rq {
 	u32                    rqn;
 	struct mlx5_core_dev  *mdev;
 	struct mlx5_core_mkey  umr_mkey;
+
+	/* XDP read-mostly */
+	struct xdp_rxq_info    xdp_rxq;
 } ____cacheline_aligned_in_smp;
 
 struct mlx5e_channel {
@@ -587,6 +561,7 @@ struct mlx5e_channel {
 
 	/* data path - accessed per napi poll */
 	struct irq_desc *irq_desc;
+	struct mlx5e_ch_stats      stats;
 
 	/* control */
 	struct mlx5e_priv         *priv;
@@ -655,6 +630,7 @@ struct mlx5e_tc_table {
 	struct rhashtable               ht;
 
 	DECLARE_HASHTABLE(mod_hdr_tbl, 8);
+	DECLARE_HASHTABLE(hairpin_tbl, 8);
 };
 
 struct mlx5e_vlan_table {
@@ -722,6 +698,11 @@ enum {
 	MLX5E_ARFS_FT_LEVEL
 };
 
+enum {
+	MLX5E_TC_FT_LEVEL = 0,
+	MLX5E_TC_TTC_FT_LEVEL,
+};
+
 struct mlx5e_ethtool_table {
 	struct mlx5_flow_table *ft;
 	int                    num_rules;
@@ -860,11 +841,7 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
 void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
 
-void mlx5e_rx_am(struct mlx5e_rq *rq);
-void mlx5e_rx_am_work(struct work_struct *work);
-struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode);
-
-void mlx5e_update_stats(struct mlx5e_priv *priv, bool full);
+void mlx5e_update_stats(struct mlx5e_priv *priv);
 
 int mlx5e_create_flow_steering(struct mlx5e_priv *priv);
 void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv);
@@ -1054,11 +1031,26 @@ int mlx5e_create_direct_tirs(struct mlx5e_priv *priv);
 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv);
 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt);
 
-int mlx5e_create_ttc_table(struct mlx5e_priv *priv);
-void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv);
+struct ttc_params {
+	struct mlx5_flow_table_attr ft_attr;
+	u32 any_tt_tirn;
+	u32 indir_tirn[MLX5E_NUM_INDIR_TIRS];
+	struct mlx5e_ttc_table *inner_ttc;
+};
+
+void mlx5e_set_ttc_basic_params(struct mlx5e_priv *priv, struct ttc_params *ttc_params);
+void mlx5e_set_ttc_ft_params(struct ttc_params *ttc_params);
+void mlx5e_set_inner_ttc_ft_params(struct ttc_params *ttc_params);
 
-int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv);
-void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv);
+int mlx5e_create_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
+			   struct mlx5e_ttc_table *ttc);
+void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv,
+			     struct mlx5e_ttc_table *ttc);
+
+int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
+				 struct mlx5e_ttc_table *ttc);
+void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv,
+				   struct mlx5e_ttc_table *ttc);
 
 int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
 		     u32 underlay_qpn, u32 *tisn);
@@ -1071,6 +1063,8 @@ int mlx5e_open(struct net_device *netdev);
 void mlx5e_update_stats_work(struct work_struct *work);
 u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout);
 
+int mlx5e_bits_invert(unsigned long a, int size);
+
 /* ethtool helpers */
 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
 			       struct ethtool_drvinfo *drvinfo);
@@ -1110,4 +1104,5 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
 			    struct mlx5e_params *params,
 			    u16 max_channels);
 u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev);
+void mlx5e_rx_dim_work(struct work_struct *work);
 #endif /* __MLX5_EN_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c
new file mode 100644
index 000000000000..602851ab5b14
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/net_dim.h>
+#include "en.h"
+
+void mlx5e_rx_dim_work(struct work_struct *work)
+{
+	struct net_dim *dim = container_of(work, struct net_dim,
+					   work);
+	struct mlx5e_rq *rq = container_of(dim, struct mlx5e_rq, dim);
+	struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode,
+								  dim->profile_ix);
+
+	mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq,
+				       cur_profile.usec, cur_profile.pkts);
+
+	dim->state = NET_DIM_START_MEASURE;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index ea5fff2c3143..59ebfdae6695 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -207,7 +207,7 @@ void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv,
 		return;
 
 	mutex_lock(&priv->state_lock);
-	mlx5e_update_stats(priv, true);
+	mlx5e_update_stats(priv);
 	mutex_unlock(&priv->state_lock);
 
 	for (i = 0; i < mlx5e_num_stats_grps; i++)
@@ -295,7 +295,6 @@ int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
 	struct mlx5e_channels new_channels = {};
 	u32 rx_pending_wqes;
 	u32 min_rq_size;
-	u32 max_rq_size;
 	u8 log_rq_size;
 	u8 log_sq_size;
 	u32 num_mtts;
@@ -314,8 +313,6 @@ int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
 
 	min_rq_size = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
 					       1 << mlx5_min_log_rq_size(rq_wq_type));
-	max_rq_size = mlx5e_rx_wqes_to_packets(priv, rq_wq_type,
-					       1 << mlx5_max_log_rq_size(rq_wq_type));
 	rx_pending_wqes = mlx5e_packets_to_rx_wqes(priv, rq_wq_type,
 						   param->rx_pending);
 
@@ -325,12 +322,6 @@ int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
 			    min_rq_size);
 		return -EINVAL;
 	}
-	if (param->rx_pending > max_rq_size) {
-		netdev_info(priv->netdev, "%s: rx_pending (%d) > max (%d)\n",
-			    __func__, param->rx_pending,
-			    max_rq_size);
-		return -EINVAL;
-	}
 
 	num_mtts = MLX5E_REQUIRED_MTTS(rx_pending_wqes);
 	if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
@@ -346,12 +337,6 @@ int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
 			    1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE);
 		return -EINVAL;
 	}
-	if (param->tx_pending > (1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE)) {
-		netdev_info(priv->netdev, "%s: tx_pending (%d) > max (%d)\n",
-			    __func__, param->tx_pending,
-			    1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE);
-		return -EINVAL;
-	}
 
 	log_rq_size = order_base_2(rx_pending_wqes);
 	log_sq_size = order_base_2(param->tx_pending);
@@ -479,7 +464,7 @@ int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv,
 	coal->rx_max_coalesced_frames = priv->channels.params.rx_cq_moderation.pkts;
 	coal->tx_coalesce_usecs       = priv->channels.params.tx_cq_moderation.usec;
 	coal->tx_max_coalesced_frames = priv->channels.params.tx_cq_moderation.pkts;
-	coal->use_adaptive_rx_coalesce = priv->channels.params.rx_am_enabled;
+	coal->use_adaptive_rx_coalesce = priv->channels.params.rx_dim_enabled;
 
 	return 0;
 }
@@ -492,6 +477,9 @@ static int mlx5e_get_coalesce(struct net_device *netdev,
 	return mlx5e_ethtool_get_coalesce(priv, coal);
 }
 
+#define MLX5E_MAX_COAL_TIME		MLX5_MAX_CQ_PERIOD
+#define MLX5E_MAX_COAL_FRAMES		MLX5_MAX_CQ_COUNT
+
 static void
 mlx5e_set_priv_channels_coalesce(struct mlx5e_priv *priv, struct ethtool_coalesce *coal)
 {
@@ -526,6 +514,20 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
 	if (!MLX5_CAP_GEN(mdev, cq_moderation))
 		return -EOPNOTSUPP;
 
+	if (coal->tx_coalesce_usecs > MLX5E_MAX_COAL_TIME ||
+	    coal->rx_coalesce_usecs > MLX5E_MAX_COAL_TIME) {
+		netdev_info(priv->netdev, "%s: maximum coalesce time supported is %lu usecs\n",
+			    __func__, MLX5E_MAX_COAL_TIME);
+		return -ERANGE;
+	}
+
+	if (coal->tx_max_coalesced_frames > MLX5E_MAX_COAL_FRAMES ||
+	    coal->rx_max_coalesced_frames > MLX5E_MAX_COAL_FRAMES) {
+		netdev_info(priv->netdev, "%s: maximum coalesced frames supported is %lu\n",
+			    __func__, MLX5E_MAX_COAL_FRAMES);
+		return -ERANGE;
+	}
+
 	mutex_lock(&priv->state_lock);
 	new_channels.params = priv->channels.params;
 
@@ -533,7 +535,7 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
 	new_channels.params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames;
 	new_channels.params.rx_cq_moderation.usec = coal->rx_coalesce_usecs;
 	new_channels.params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames;
-	new_channels.params.rx_am_enabled         = !!coal->use_adaptive_rx_coalesce;
+	new_channels.params.rx_dim_enabled        = !!coal->use_adaptive_rx_coalesce;
 
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 		priv->channels.params = new_channels.params;
@@ -541,7 +543,7 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
 	}
 	/* we are opened */
 
-	reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_am_enabled;
+	reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_dim_enabled;
 	if (!reset) {
 		mlx5e_set_priv_channels_coalesce(priv, coal);
 		priv->channels.params = new_channels.params;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index def513484845..f64dda2bed31 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -806,25 +806,25 @@ mlx5e_generate_ttc_rule(struct mlx5e_priv *priv,
 	return err ? ERR_PTR(err) : rule;
 }
 
-static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv)
+static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv,
+					  struct ttc_params *params,
+					  struct mlx5e_ttc_table *ttc)
 {
 	struct mlx5_flow_destination dest = {};
-	struct mlx5e_ttc_table *ttc;
 	struct mlx5_flow_handle **rules;
 	struct mlx5_flow_table *ft;
 	int tt;
 	int err;
 
-	ttc = &priv->fs.ttc;
 	ft = ttc->ft.t;
 	rules = ttc->rules;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 	for (tt = 0; tt < MLX5E_NUM_TT; tt++) {
 		if (tt == MLX5E_TT_ANY)
-			dest.tir_num = priv->direct_tir[0].tirn;
+			dest.tir_num = params->any_tt_tirn;
 		else
-			dest.tir_num = priv->indir_tir[tt].tirn;
+			dest.tir_num = params->indir_tirn[tt];
 		rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest,
 						    ttc_rules[tt].etype,
 						    ttc_rules[tt].proto);
@@ -832,12 +832,12 @@ static int mlx5e_generate_ttc_table_rules(struct mlx5e_priv *priv)
 			goto del_rules;
 	}
 
-	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
+	if (!params->inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev))
 		return 0;
 
 	rules     = ttc->tunnel_rules;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
-	dest.ft   = priv->fs.inner_ttc.ft.t;
+	dest.ft   = params->inner_ttc->ft.t;
 	for (tt = 0; tt < MLX5E_NUM_TUNNEL_TT; tt++) {
 		rules[tt] = mlx5e_generate_ttc_rule(priv, ft, &dest,
 						    ttc_tunnel_rules[tt].etype,
@@ -977,25 +977,25 @@ mlx5e_generate_inner_ttc_rule(struct mlx5e_priv *priv,
 	return err ? ERR_PTR(err) : rule;
 }
 
-static int mlx5e_generate_inner_ttc_table_rules(struct mlx5e_priv *priv)
+static int mlx5e_generate_inner_ttc_table_rules(struct mlx5e_priv *priv,
+						struct ttc_params *params,
+						struct mlx5e_ttc_table *ttc)
 {
 	struct mlx5_flow_destination dest = {};
 	struct mlx5_flow_handle **rules;
-	struct mlx5e_ttc_table *ttc;
 	struct mlx5_flow_table *ft;
 	int err;
 	int tt;
 
-	ttc =  &priv->fs.inner_ttc;
 	ft = ttc->ft.t;
 	rules = ttc->rules;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 	for (tt = 0; tt < MLX5E_NUM_TT; tt++) {
 		if (tt == MLX5E_TT_ANY)
-			dest.tir_num = priv->direct_tir[0].tirn;
+			dest.tir_num = params->any_tt_tirn;
 		else
-			dest.tir_num = priv->inner_indir_tir[tt].tirn;
+			dest.tir_num = params->indir_tirn[tt];
 
 		rules[tt] = mlx5e_generate_inner_ttc_rule(priv, ft, &dest,
 							  ttc_rules[tt].etype,
@@ -1075,21 +1075,42 @@ static int mlx5e_create_inner_ttc_table_groups(struct mlx5e_ttc_table *ttc)
 	return err;
 }
 
-int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv)
+void mlx5e_set_ttc_basic_params(struct mlx5e_priv *priv,
+				struct ttc_params *ttc_params)
+{
+	ttc_params->any_tt_tirn = priv->direct_tir[0].tirn;
+	ttc_params->inner_ttc = &priv->fs.inner_ttc;
+}
+
+void mlx5e_set_inner_ttc_ft_params(struct ttc_params *ttc_params)
+{
+	struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
+
+	ft_attr->max_fte = MLX5E_INNER_TTC_TABLE_SIZE;
+	ft_attr->level = MLX5E_INNER_TTC_FT_LEVEL;
+	ft_attr->prio = MLX5E_NIC_PRIO;
+}
+
+void mlx5e_set_ttc_ft_params(struct ttc_params *ttc_params)
+
+{
+	struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
+
+	ft_attr->max_fte = MLX5E_TTC_TABLE_SIZE;
+	ft_attr->level = MLX5E_TTC_FT_LEVEL;
+	ft_attr->prio = MLX5E_NIC_PRIO;
+}
+
+int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
+				 struct mlx5e_ttc_table *ttc)
 {
-	struct mlx5e_ttc_table *ttc = &priv->fs.inner_ttc;
-	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5e_flow_table *ft = &ttc->ft;
 	int err;
 
 	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
 		return 0;
 
-	ft_attr.max_fte = MLX5E_INNER_TTC_TABLE_SIZE;
-	ft_attr.level   = MLX5E_INNER_TTC_FT_LEVEL;
-	ft_attr.prio    = MLX5E_NIC_PRIO;
-
-	ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
+	ft->t = mlx5_create_flow_table(priv->fs.ns, &params->ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
@@ -1100,7 +1121,7 @@ int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv)
 	if (err)
 		goto err;
 
-	err = mlx5e_generate_inner_ttc_table_rules(priv);
+	err = mlx5e_generate_inner_ttc_table_rules(priv, params, ttc);
 	if (err)
 		goto err;
 
@@ -1111,10 +1132,9 @@ int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv)
 	return err;
 }
 
-void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv)
+void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv,
+				   struct mlx5e_ttc_table *ttc)
 {
-	struct mlx5e_ttc_table *ttc = &priv->fs.inner_ttc;
-
 	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
 		return;
 
@@ -1122,27 +1142,21 @@ void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv)
 	mlx5e_destroy_flow_table(&ttc->ft);
 }
 
-void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv)
+void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv,
+			     struct mlx5e_ttc_table *ttc)
 {
-	struct mlx5e_ttc_table *ttc = &priv->fs.ttc;
-
 	mlx5e_cleanup_ttc_rules(ttc);
 	mlx5e_destroy_flow_table(&ttc->ft);
 }
 
-int mlx5e_create_ttc_table(struct mlx5e_priv *priv)
+int mlx5e_create_ttc_table(struct mlx5e_priv *priv, struct ttc_params *params,
+			   struct mlx5e_ttc_table *ttc)
 {
 	bool match_ipv_outer = MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version);
-	struct mlx5e_ttc_table *ttc = &priv->fs.ttc;
-	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5e_flow_table *ft = &ttc->ft;
 	int err;
 
-	ft_attr.max_fte = MLX5E_TTC_TABLE_SIZE;
-	ft_attr.level = MLX5E_TTC_FT_LEVEL;
-	ft_attr.prio = MLX5E_NIC_PRIO;
-
-	ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
+	ft->t = mlx5_create_flow_table(priv->fs.ns, &params->ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
@@ -1153,7 +1167,7 @@ int mlx5e_create_ttc_table(struct mlx5e_priv *priv)
 	if (err)
 		goto err;
 
-	err = mlx5e_generate_ttc_table_rules(priv);
+	err = mlx5e_generate_ttc_table_rules(priv, params, ttc);
 	if (err)
 		goto err;
 
@@ -1474,7 +1488,8 @@ static void mlx5e_destroy_vlan_table(struct mlx5e_priv *priv)
 
 int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
 {
-	int err;
+	struct ttc_params ttc_params = {};
+	int tt, err;
 
 	priv->fs.ns = mlx5_get_flow_namespace(priv->mdev,
 					       MLX5_FLOW_NAMESPACE_KERNEL);
@@ -1489,14 +1504,23 @@ int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
 		priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
 	}
 
-	err = mlx5e_create_inner_ttc_table(priv);
+	mlx5e_set_ttc_basic_params(priv, &ttc_params);
+	mlx5e_set_inner_ttc_ft_params(&ttc_params);
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
+		ttc_params.indir_tirn[tt] = priv->inner_indir_tir[tt].tirn;
+
+	err = mlx5e_create_inner_ttc_table(priv, &ttc_params, &priv->fs.inner_ttc);
 	if (err) {
 		netdev_err(priv->netdev, "Failed to create inner ttc table, err=%d\n",
 			   err);
 		goto err_destroy_arfs_tables;
 	}
 
-	err = mlx5e_create_ttc_table(priv);
+	mlx5e_set_ttc_ft_params(&ttc_params);
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
+		ttc_params.indir_tirn[tt] = priv->indir_tir[tt].tirn;
+
+	err = mlx5e_create_ttc_table(priv, &ttc_params, &priv->fs.ttc);
 	if (err) {
 		netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n",
 			   err);
@@ -1524,9 +1548,9 @@ int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
 err_destroy_l2_table:
 	mlx5e_destroy_l2_table(priv);
 err_destroy_ttc_table:
-	mlx5e_destroy_ttc_table(priv);
+	mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
 err_destroy_inner_ttc_table:
-	mlx5e_destroy_inner_ttc_table(priv);
+	mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
 err_destroy_arfs_tables:
 	mlx5e_arfs_destroy_tables(priv);
 
@@ -1537,8 +1561,8 @@ void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv)
 {
 	mlx5e_destroy_vlan_table(priv);
 	mlx5e_destroy_l2_table(priv);
-	mlx5e_destroy_ttc_table(priv);
-	mlx5e_destroy_inner_ttc_table(priv);
+	mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
+	mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
 	mlx5e_arfs_destroy_tables(priv);
 	mlx5e_ethtool_cleanup_steering(priv);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index d8aefeed124d..9b4827d36e3e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -173,182 +173,23 @@ static void mlx5e_tx_timeout_work(struct work_struct *work)
 	rtnl_unlock();
 }
 
-static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
+void mlx5e_update_stats(struct mlx5e_priv *priv)
 {
-	struct mlx5e_sw_stats temp, *s = &temp;
-	struct mlx5e_rq_stats *rq_stats;
-	struct mlx5e_sq_stats *sq_stats;
-	int i, j;
-
-	memset(s, 0, sizeof(*s));
-	for (i = 0; i < priv->channels.num; i++) {
-		struct mlx5e_channel *c = priv->channels.c[i];
-
-		rq_stats = &c->rq.stats;
-
-		s->rx_packets	+= rq_stats->packets;
-		s->rx_bytes	+= rq_stats->bytes;
-		s->rx_lro_packets += rq_stats->lro_packets;
-		s->rx_lro_bytes	+= rq_stats->lro_bytes;
-		s->rx_removed_vlan_packets += rq_stats->removed_vlan_packets;
-		s->rx_csum_none	+= rq_stats->csum_none;
-		s->rx_csum_complete += rq_stats->csum_complete;
-		s->rx_csum_unnecessary += rq_stats->csum_unnecessary;
-		s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
-		s->rx_xdp_drop += rq_stats->xdp_drop;
-		s->rx_xdp_tx += rq_stats->xdp_tx;
-		s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
-		s->rx_wqe_err   += rq_stats->wqe_err;
-		s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
-		s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
-		s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
-		s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
-		s->rx_page_reuse  += rq_stats->page_reuse;
-		s->rx_cache_reuse += rq_stats->cache_reuse;
-		s->rx_cache_full  += rq_stats->cache_full;
-		s->rx_cache_empty += rq_stats->cache_empty;
-		s->rx_cache_busy  += rq_stats->cache_busy;
-		s->rx_cache_waive += rq_stats->cache_waive;
-
-		for (j = 0; j < priv->channels.params.num_tc; j++) {
-			sq_stats = &c->sq[j].stats;
-
-			s->tx_packets		+= sq_stats->packets;
-			s->tx_bytes		+= sq_stats->bytes;
-			s->tx_tso_packets	+= sq_stats->tso_packets;
-			s->tx_tso_bytes		+= sq_stats->tso_bytes;
-			s->tx_tso_inner_packets	+= sq_stats->tso_inner_packets;
-			s->tx_tso_inner_bytes	+= sq_stats->tso_inner_bytes;
-			s->tx_added_vlan_packets += sq_stats->added_vlan_packets;
-			s->tx_queue_stopped	+= sq_stats->stopped;
-			s->tx_queue_wake	+= sq_stats->wake;
-			s->tx_queue_dropped	+= sq_stats->dropped;
-			s->tx_xmit_more		+= sq_stats->xmit_more;
-			s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
-			s->tx_csum_none		+= sq_stats->csum_none;
-			s->tx_csum_partial	+= sq_stats->csum_partial;
-		}
-	}
-
-	s->link_down_events_phy = MLX5_GET(ppcnt_reg,
-				priv->stats.pport.phy_counters,
-				counter_set.phys_layer_cntrs.link_down_events);
-	memcpy(&priv->stats.sw, s, sizeof(*s));
-}
-
-static void mlx5e_update_vport_counters(struct mlx5e_priv *priv)
-{
-	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
-	u32 *out = (u32 *)priv->stats.vport.query_vport_out;
-	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
-	struct mlx5_core_dev *mdev = priv->mdev;
-
-	MLX5_SET(query_vport_counter_in, in, opcode,
-		 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
-	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
-	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
-
-	mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
-}
-
-static void mlx5e_update_pport_counters(struct mlx5e_priv *priv, bool full)
-{
-	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
-	struct mlx5_core_dev *mdev = priv->mdev;
-	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
-	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
-	int prio;
-	void *out;
-
-	MLX5_SET(ppcnt_reg, in, local_port, 1);
-
-	out = pstats->IEEE_802_3_counters;
-	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
-	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
-
-	if (!full)
-		return;
-
-	out = pstats->RFC_2863_counters;
-	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
-	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
-
-	out = pstats->RFC_2819_counters;
-	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
-	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
-
-	out = pstats->phy_counters;
-	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
-	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
-
-	if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) {
-		out = pstats->phy_statistical_counters;
-		MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
-		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
-	}
-
-	if (MLX5_CAP_PCAM_FEATURE(mdev, rx_buffer_fullness_counters)) {
-		out = pstats->eth_ext_counters;
-		MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
-		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
-	}
-
-	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
-	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
-		out = pstats->per_prio_counters[prio];
-		MLX5_SET(ppcnt_reg, in, prio_tc, prio);
-		mlx5_core_access_reg(mdev, in, sz, out, sz,
-				     MLX5_REG_PPCNT, 0, 0);
-	}
-}
-
-static void mlx5e_update_q_counter(struct mlx5e_priv *priv)
-{
-	struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
-	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)];
-	int err;
-
-	if (!priv->q_counter)
-		return;
-
-	err = mlx5_core_query_q_counter(priv->mdev, priv->q_counter, 0, out, sizeof(out));
-	if (err)
-		return;
-
-	qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer);
-}
-
-static void mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
-{
-	struct mlx5e_pcie_stats *pcie_stats = &priv->stats.pcie;
-	struct mlx5_core_dev *mdev = priv->mdev;
-	u32 in[MLX5_ST_SZ_DW(mpcnt_reg)] = {0};
-	int sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
-	void *out;
-
-	if (!MLX5_CAP_MCAM_FEATURE(mdev, pcie_performance_group))
-		return;
-
-	out = pcie_stats->pcie_perf_counters;
-	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
-	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
-}
+	int i;
 
-void mlx5e_update_stats(struct mlx5e_priv *priv, bool full)
-{
-	if (full) {
-		mlx5e_update_pcie_counters(priv);
-		mlx5e_ipsec_update_stats(priv);
-	}
-	mlx5e_update_pport_counters(priv, full);
-	mlx5e_update_vport_counters(priv);
-	mlx5e_update_q_counter(priv);
-	mlx5e_update_sw_counters(priv);
+	for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
+		if (mlx5e_stats_grps[i].update_stats)
+			mlx5e_stats_grps[i].update_stats(priv);
 }
 
 static void mlx5e_update_ndo_stats(struct mlx5e_priv *priv)
 {
-	mlx5e_update_stats(priv, false);
+	int i;
+
+	for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
+		if (mlx5e_stats_grps[i].update_stats_mask &
+		    MLX5E_NDO_UPDATE_STATS)
+			mlx5e_stats_grps[i].update_stats(priv);
 }
 
 void mlx5e_update_stats_work(struct work_struct *work)
@@ -582,6 +423,10 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 		goto err_rq_wq_destroy;
 	}
 
+	err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix);
+	if (err < 0)
+		goto err_rq_wq_destroy;
+
 	rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 	rq->buff.headroom = params->rq_headroom;
 
@@ -674,8 +519,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 		wqe->data.lkey = rq->mkey_be;
 	}
 
-	INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
-	rq->am.mode = params->rx_cq_moderation.cq_period_mode;
+	INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
+
+	switch (params->rx_cq_moderation.cq_period_mode) {
+	case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
+		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
+		break;
+	case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
+	default:
+		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
+	}
+
 	rq->page_cache.head = 0;
 	rq->page_cache.tail = 0;
 
@@ -687,6 +541,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
 err_rq_wq_destroy:
 	if (rq->xdp_prog)
 		bpf_prog_put(rq->xdp_prog);
+	xdp_rxq_info_unreg(&rq->xdp_rxq);
 	mlx5_wq_destroy(&rq->wq_ctrl);
 
 	return err;
@@ -699,6 +554,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
 	if (rq->xdp_prog)
 		bpf_prog_put(rq->xdp_prog);
 
+	xdp_rxq_info_unreg(&rq->xdp_rxq);
+
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 		mlx5e_rq_free_mpwqe_info(rq);
@@ -919,7 +776,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c,
 	if (err)
 		goto err_destroy_rq;
 
-	if (params->rx_am_enabled)
+	if (params->rx_dim_enabled)
 		c->rq.state |= BIT(MLX5E_RQ_STATE_AM);
 
 	return 0;
@@ -952,7 +809,7 @@ static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
 
 static void mlx5e_close_rq(struct mlx5e_rq *rq)
 {
-	cancel_work_sync(&rq->am.work);
+	cancel_work_sync(&rq->dim.work);
 	mlx5e_destroy_rq(rq);
 	mlx5e_free_rx_descs(rq);
 	mlx5e_free_rq(rq);
@@ -1565,7 +1422,7 @@ static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
 }
 
 static int mlx5e_open_cq(struct mlx5e_channel *c,
-			 struct mlx5e_cq_moder moder,
+			 struct net_dim_cq_moder moder,
 			 struct mlx5e_cq_param *param,
 			 struct mlx5e_cq *cq)
 {
@@ -1747,7 +1604,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 			      struct mlx5e_channel_param *cparam,
 			      struct mlx5e_channel **cp)
 {
-	struct mlx5e_cq_moder icocq_moder = {0, 0};
+	struct net_dim_cq_moder icocq_moder = {0, 0};
 	struct net_device *netdev = priv->netdev;
 	int cpu = mlx5e_get_cpu(priv, ix);
 	struct mlx5e_channel *c;
@@ -1911,13 +1768,16 @@ static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
 	param->wq.linear = 1;
 }
 
-static void mlx5e_build_drop_rq_param(struct mlx5e_rq_param *param)
+static void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
+				      struct mlx5e_rq_param *param)
 {
 	void *rqc = param->rqc;
 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
 
 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
 	MLX5_SET(wq, wq, log_wq_stride,    ilog2(sizeof(struct mlx5e_rx_wqe)));
+
+	param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
 }
 
 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
@@ -1999,7 +1859,7 @@ static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
 
 	mlx5e_build_common_cq_param(priv, param);
 
-	param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
+	param->cq_period_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 }
 
 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
@@ -2203,7 +2063,7 @@ static int mlx5e_rx_hash_fn(int hfunc)
 	       MLX5_RX_HASH_FN_INVERTED_XOR8;
 }
 
-static int mlx5e_bits_invert(unsigned long a, int size)
+int mlx5e_bits_invert(unsigned long a, int size)
 {
 	int inv = 0;
 	int i;
@@ -2712,6 +2572,9 @@ int mlx5e_open(struct net_device *netdev)
 		mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_UP);
 	mutex_unlock(&priv->state_lock);
 
+	if (mlx5e_vxlan_allowed(priv->mdev))
+		udp_tunnel_get_rx_info(netdev);
+
 	return err;
 }
 
@@ -2765,6 +2628,9 @@ static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
 	if (err)
 		return err;
 
+	/* Mark as unused given "Drop-RQ" packets never reach XDP */
+	xdp_rxq_info_unused(&rq->xdp_rxq);
+
 	rq->mdev = mdev;
 
 	return 0;
@@ -2774,6 +2640,9 @@ static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev,
 			       struct mlx5e_cq *cq,
 			       struct mlx5e_cq_param *param)
 {
+	param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
+	param->wq.db_numa_node  = dev_to_node(&mdev->pdev->dev);
+
 	return mlx5e_alloc_cq_common(mdev, param, cq);
 }
 
@@ -2785,7 +2654,7 @@ static int mlx5e_open_drop_rq(struct mlx5_core_dev *mdev,
 	struct mlx5e_cq *cq = &drop_rq->cq;
 	int err;
 
-	mlx5e_build_drop_rq_param(&rq_param);
+	mlx5e_build_drop_rq_param(mdev, &rq_param);
 
 	err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param);
 	if (err)
@@ -3084,9 +2953,6 @@ static int mlx5e_setup_tc_mqprio(struct net_device *netdev,
 static int mlx5e_setup_tc_cls_flower(struct mlx5e_priv *priv,
 				     struct tc_cls_flower_offload *cls_flower)
 {
-	if (cls_flower->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (cls_flower->command) {
 	case TC_CLSFLOWER_REPLACE:
 		return mlx5e_configure_flower(priv, cls_flower);
@@ -3104,7 +2970,7 @@ int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
 {
 	struct mlx5e_priv *priv = cb_priv;
 
-	if (!tc_can_offload(priv->netdev))
+	if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
@@ -3137,8 +3003,8 @@ static int mlx5e_setup_tc_block(struct net_device *dev,
 }
 #endif
 
-int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
-		   void *type_data)
+static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
+			  void *type_data)
 {
 	switch (type) {
 #ifdef CONFIG_MLX5_ESWITCH
@@ -3738,26 +3604,62 @@ static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
 	return features;
 }
 
+static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev,
+					struct mlx5e_txqsq *sq)
+{
+	struct mlx5e_priv *priv = netdev_priv(dev);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	int irqn_not_used, eqn;
+	struct mlx5_eq *eq;
+	u32 eqe_count;
+
+	if (mlx5_vector2eqn(mdev, sq->cq.mcq.vector, &eqn, &irqn_not_used))
+		return false;
+
+	eq = mlx5_eqn2eq(mdev, eqn);
+	if (IS_ERR(eq))
+		return false;
+
+	netdev_err(dev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n",
+		   eqn, eq->cons_index, eq->irqn);
+
+	eqe_count = mlx5_eq_poll_irq_disabled(eq);
+	if (!eqe_count)
+		return false;
+
+	netdev_err(dev, "Recover %d eqes on EQ 0x%x\n", eqe_count, eq->eqn);
+	sq->channel->stats.eq_rearm++;
+	return true;
+}
+
 static void mlx5e_tx_timeout(struct net_device *dev)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
-	bool sched_work = false;
+	bool reopen_channels = false;
 	int i;
 
 	netdev_err(dev, "TX timeout detected\n");
 
 	for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) {
+		struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, i);
 		struct mlx5e_txqsq *sq = priv->txq2sq[i];
 
-		if (!netif_xmit_stopped(netdev_get_tx_queue(dev, i)))
+		if (!netif_xmit_stopped(dev_queue))
 			continue;
-		sched_work = true;
-		clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
-		netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x\n",
-			   i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc);
+		netdev_err(dev, "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u\n",
+			   i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc,
+			   jiffies_to_usecs(jiffies - dev_queue->trans_start));
+
+		/* If we recover a lost interrupt, most likely TX timeout will
+		 * be resolved, skip reopening channels
+		 */
+		if (!mlx5e_tx_timeout_eq_recover(dev, sq)) {
+			clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
+			reopen_channels = true;
+		}
 	}
 
-	if (sched_work && test_bit(MLX5E_STATE_OPENED, &priv->state))
+	if (reopen_channels && test_bit(MLX5E_STATE_OPENED, &priv->state))
 		schedule_work(&priv->tx_timeout_work);
 }
 
@@ -4044,9 +3946,18 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
 		params->rx_cq_moderation.usec =
 			MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
 
-	if (params->rx_am_enabled)
-		params->rx_cq_moderation =
-			mlx5e_am_get_def_profile(cq_period_mode);
+	if (params->rx_dim_enabled) {
+		switch (cq_period_mode) {
+		case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
+			params->rx_cq_moderation =
+				net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE);
+			break;
+		case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
+		default:
+			params->rx_cq_moderation =
+				net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE);
+		}
+	}
 
 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
 			params->rx_cq_moderation.cq_period_mode ==
@@ -4108,7 +4019,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
 	cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
 			MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
 			MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
-	params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
+	params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
 	mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
 	mlx5e_set_tx_cq_mode_params(params, cq_period_mode);
 
@@ -4161,7 +4072,7 @@ static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
 	}
 }
 
-#if IS_ENABLED(CONFIG_NET_SWITCHDEV) && IS_ENABLED(CONFIG_MLX5_ESWITCH)
+#if IS_ENABLED(CONFIG_MLX5_ESWITCH)
 static const struct switchdev_ops mlx5e_switchdev_ops = {
 	.switchdev_port_attr_get	= mlx5e_attr_get,
 };
@@ -4267,7 +4178,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	mlx5e_set_netdev_dev_addr(netdev);
 
-#if IS_ENABLED(CONFIG_NET_SWITCHDEV) && IS_ENABLED(CONFIG_MLX5_ESWITCH)
+#if IS_ENABLED(CONFIG_MLX5_ESWITCH)
 	if (MLX5_VPORT_MANAGER(mdev))
 		netdev->switchdev_ops = &mlx5e_switchdev_ops;
 #endif
@@ -4315,9 +4226,6 @@ static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
 {
 	mlx5e_ipsec_cleanup(priv);
 	mlx5e_vxlan_cleanup(priv);
-
-	if (priv->channels.params.xdp_prog)
-		bpf_prog_put(priv->channels.params.xdp_prog);
 }
 
 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
@@ -4422,12 +4330,6 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
 #ifdef CONFIG_MLX5_CORE_EN_DCB
 	mlx5e_dcbnl_init_app(priv);
 #endif
-	/* Device already registered: sync netdev system state */
-	if (mlx5e_vxlan_allowed(mdev)) {
-		rtnl_lock();
-		udp_tunnel_get_rx_info(netdev);
-		rtnl_unlock();
-	}
 
 	queue_work(priv->wq, &priv->set_rx_mode_work);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 3409d86eb06b..500d817d2b0a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -44,6 +44,11 @@
 #include "en_tc.h"
 #include "fs_core.h"
 
+#define MLX5E_REP_PARAMS_LOG_SQ_SIZE \
+	max(0x6, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)
+#define MLX5E_REP_PARAMS_LOG_RQ_SIZE \
+	max(0x6, MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE)
+
 static const char mlx5e_rep_driver_name[] = "mlx5e_rep";
 
 static void mlx5e_rep_get_drvinfo(struct net_device *dev,
@@ -190,6 +195,63 @@ int mlx5e_attr_get(struct net_device *dev, struct switchdev_attr *attr)
 	return 0;
 }
 
+static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw,
+				 struct mlx5_eswitch_rep *rep)
+{
+	struct mlx5e_rep_sq *rep_sq, *tmp;
+	struct mlx5e_rep_priv *rpriv;
+
+	if (esw->mode != SRIOV_OFFLOADS)
+		return;
+
+	rpriv = mlx5e_rep_to_rep_priv(rep);
+	list_for_each_entry_safe(rep_sq, tmp, &rpriv->vport_sqs_list, list) {
+		mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule);
+		list_del(&rep_sq->list);
+		kfree(rep_sq);
+	}
+}
+
+static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw,
+				 struct mlx5_eswitch_rep *rep,
+				 u32 *sqns_array, int sqns_num)
+{
+	struct mlx5_flow_handle *flow_rule;
+	struct mlx5e_rep_priv *rpriv;
+	struct mlx5e_rep_sq *rep_sq;
+	int err;
+	int i;
+
+	if (esw->mode != SRIOV_OFFLOADS)
+		return 0;
+
+	rpriv = mlx5e_rep_to_rep_priv(rep);
+	for (i = 0; i < sqns_num; i++) {
+		rep_sq = kzalloc(sizeof(*rep_sq), GFP_KERNEL);
+		if (!rep_sq) {
+			err = -ENOMEM;
+			goto out_err;
+		}
+
+		/* Add re-inject rule to the PF/representor sqs */
+		flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw,
+								rep->vport,
+								sqns_array[i]);
+		if (IS_ERR(flow_rule)) {
+			err = PTR_ERR(flow_rule);
+			kfree(rep_sq);
+			goto out_err;
+		}
+		rep_sq->send_to_vport_rule = flow_rule;
+		list_add(&rep_sq->list, &rpriv->vport_sqs_list);
+	}
+	return 0;
+
+out_err:
+	mlx5e_sqs2vport_stop(esw, rep);
+	return err;
+}
+
 int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
@@ -198,9 +260,9 @@ int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
 	struct mlx5e_channel *c;
 	int n, tc, num_sqs = 0;
 	int err = -ENOMEM;
-	u16 *sqs;
+	u32 *sqs;
 
-	sqs = kcalloc(priv->channels.num * priv->channels.params.num_tc, sizeof(u16), GFP_KERNEL);
+	sqs = kcalloc(priv->channels.num * priv->channels.params.num_tc, sizeof(*sqs), GFP_KERNEL);
 	if (!sqs)
 		goto out;
 
@@ -210,7 +272,7 @@ int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
 			sqs[num_sqs++] = c->sq[tc].sqn;
 	}
 
-	err = mlx5_eswitch_sqs2vport_start(esw, rep, sqs, num_sqs);
+	err = mlx5e_sqs2vport_start(esw, rep, sqs, num_sqs);
 	kfree(sqs);
 
 out:
@@ -225,20 +287,20 @@ void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv)
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 	struct mlx5_eswitch_rep *rep = rpriv->rep;
 
-	mlx5_eswitch_sqs2vport_stop(esw, rep);
+	mlx5e_sqs2vport_stop(esw, rep);
 }
 
 static void mlx5e_rep_neigh_update_init_interval(struct mlx5e_rep_priv *rpriv)
 {
 #if IS_ENABLED(CONFIG_IPV6)
-	unsigned long ipv6_interval = NEIGH_VAR(&ipv6_stub->nd_tbl->parms,
+	unsigned long ipv6_interval = NEIGH_VAR(&nd_tbl.parms,
 						DELAY_PROBE_TIME);
 #else
 	unsigned long ipv6_interval = ~0UL;
 #endif
 	unsigned long ipv4_interval = NEIGH_VAR(&arp_tbl.parms,
 						DELAY_PROBE_TIME);
-	struct net_device *netdev = rpriv->rep->netdev;
+	struct net_device *netdev = rpriv->netdev;
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 
 	rpriv->neigh_update.min_interval = min_t(unsigned long, ipv6_interval, ipv4_interval);
@@ -259,7 +321,7 @@ static void mlx5e_rep_neigh_stats_work(struct work_struct *work)
 {
 	struct mlx5e_rep_priv *rpriv = container_of(work, struct mlx5e_rep_priv,
 						    neigh_update.neigh_stats_work.work);
-	struct net_device *netdev = rpriv->rep->netdev;
+	struct net_device *netdev = rpriv->netdev;
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 	struct mlx5e_neigh_hash_entry *nhe;
 
@@ -355,7 +417,7 @@ static int mlx5e_rep_netevent_event(struct notifier_block *nb,
 	struct mlx5e_rep_priv *rpriv = container_of(nb, struct mlx5e_rep_priv,
 						    neigh_update.netevent_nb);
 	struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
-	struct net_device *netdev = rpriv->rep->netdev;
+	struct net_device *netdev = rpriv->netdev;
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 	struct mlx5e_neigh_hash_entry *nhe = NULL;
 	struct mlx5e_neigh m_neigh = {};
@@ -367,7 +429,7 @@ static int mlx5e_rep_netevent_event(struct notifier_block *nb,
 	case NETEVENT_NEIGH_UPDATE:
 		n = ptr;
 #if IS_ENABLED(CONFIG_IPV6)
-		if (n->tbl != ipv6_stub->nd_tbl && n->tbl != &arp_tbl)
+		if (n->tbl != &nd_tbl && n->tbl != &arp_tbl)
 #else
 		if (n->tbl != &arp_tbl)
 #endif
@@ -415,7 +477,7 @@ static int mlx5e_rep_netevent_event(struct notifier_block *nb,
 		 * done per device delay prob time parameter.
 		 */
 #if IS_ENABLED(CONFIG_IPV6)
-		if (!p->dev || (p->tbl != ipv6_stub->nd_tbl && p->tbl != &arp_tbl))
+		if (!p->dev || (p->tbl != &nd_tbl && p->tbl != &arp_tbl))
 #else
 		if (!p->dev || p->tbl != &arp_tbl)
 #endif
@@ -483,7 +545,7 @@ static int mlx5e_rep_neigh_init(struct mlx5e_rep_priv *rpriv)
 static void mlx5e_rep_neigh_cleanup(struct mlx5e_rep_priv *rpriv)
 {
 	struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update;
-	struct mlx5e_priv *priv = netdev_priv(rpriv->rep->netdev);
+	struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 
 	unregister_netevent_notifier(&neigh_update->netevent_nb);
 
@@ -611,7 +673,6 @@ static int mlx5e_rep_open(struct net_device *dev)
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 	struct mlx5_eswitch_rep *rep = rpriv->rep;
-	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	int err;
 
 	mutex_lock(&priv->state_lock);
@@ -619,8 +680,9 @@ static int mlx5e_rep_open(struct net_device *dev)
 	if (err)
 		goto unlock;
 
-	if (!mlx5_eswitch_set_vport_state(esw, rep->vport,
-					  MLX5_ESW_VPORT_ADMIN_STATE_UP))
+	if (!mlx5_modify_vport_admin_state(priv->mdev,
+			MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
+			rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_UP))
 		netif_carrier_on(dev);
 
 unlock:
@@ -633,11 +695,12 @@ static int mlx5e_rep_close(struct net_device *dev)
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
 	struct mlx5_eswitch_rep *rep = rpriv->rep;
-	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	int ret;
 
 	mutex_lock(&priv->state_lock);
-	(void)mlx5_eswitch_set_vport_state(esw, rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
+	mlx5_modify_vport_admin_state(priv->mdev,
+			MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT,
+			rep->vport, MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
 	ret = mlx5e_close_locked(dev);
 	mutex_unlock(&priv->state_lock);
 	return ret;
@@ -662,9 +725,6 @@ static int
 mlx5e_rep_setup_tc_cls_flower(struct mlx5e_priv *priv,
 			      struct tc_cls_flower_offload *cls_flower)
 {
-	if (cls_flower->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (cls_flower->command) {
 	case TC_CLSFLOWER_REPLACE:
 		return mlx5e_configure_flower(priv, cls_flower);
@@ -682,7 +742,7 @@ static int mlx5e_rep_setup_tc_cb(enum tc_setup_type type, void *type_data,
 {
 	struct mlx5e_priv *priv = cb_priv;
 
-	if (!tc_can_offload(priv->netdev))
+	if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
@@ -823,11 +883,11 @@ static void mlx5e_build_rep_params(struct mlx5_core_dev *mdev,
 					 MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
 					 MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
 
-	params->log_sq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
+	params->log_sq_size = MLX5E_REP_PARAMS_LOG_SQ_SIZE;
 	params->rq_wq_type  = MLX5_WQ_TYPE_LINKED_LIST;
-	params->log_rq_size = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE;
+	params->log_rq_size = MLX5E_REP_PARAMS_LOG_RQ_SIZE;
 
-	params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
+	params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
 	mlx5e_set_rx_cq_mode_params(params, cq_period_mode);
 
 	params->tx_max_inline         = mlx5e_get_max_inline_cap(mdev);
@@ -845,9 +905,7 @@ static void mlx5e_build_rep_netdev(struct net_device *netdev)
 
 	netdev->ethtool_ops	  = &mlx5e_rep_ethtool_ops;
 
-#ifdef CONFIG_NET_SWITCHDEV
 	netdev->switchdev_ops = &mlx5e_rep_switchdev_ops;
-#endif
 
 	netdev->features	 |= NETIF_F_VLAN_CHALLENGED | NETIF_F_HW_TC | NETIF_F_NETNS_LOCAL;
 	netdev->hw_features      |= NETIF_F_HW_TC;
@@ -906,7 +964,7 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
 		err = PTR_ERR(flow_rule);
 		goto err_destroy_direct_tirs;
 	}
-	rep->vport_rx_rule = flow_rule;
+	rpriv->vport_rx_rule = flow_rule;
 
 	err = mlx5e_tc_init(priv);
 	if (err)
@@ -915,7 +973,7 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
 	return 0;
 
 err_del_flow_rule:
-	mlx5_del_flow_rules(rep->vport_rx_rule);
+	mlx5_del_flow_rules(rpriv->vport_rx_rule);
 err_destroy_direct_tirs:
 	mlx5e_destroy_direct_tirs(priv);
 err_destroy_direct_rqts:
@@ -926,10 +984,9 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
 static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv)
 {
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
-	struct mlx5_eswitch_rep *rep = rpriv->rep;
 
 	mlx5e_tc_cleanup(priv);
-	mlx5_del_flow_rules(rep->vport_rx_rule);
+	mlx5_del_flow_rules(rpriv->vport_rx_rule);
 	mlx5e_destroy_direct_tirs(priv);
 	mlx5e_destroy_direct_rqts(priv);
 }
@@ -969,10 +1026,10 @@ static const struct mlx5e_profile mlx5e_rep_profile = {
 /* e-Switch vport representors */
 
 static int
-mlx5e_nic_rep_load(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
+mlx5e_nic_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 {
-	struct mlx5e_priv *priv = netdev_priv(rep->netdev);
-	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+	struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
+	struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 
 	int err;
 
@@ -994,10 +1051,10 @@ mlx5e_nic_rep_load(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
 }
 
 static void
-mlx5e_nic_rep_unload(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
+mlx5e_nic_rep_unload(struct mlx5_eswitch_rep *rep)
 {
-	struct mlx5e_priv *priv = netdev_priv(rep->netdev);
-	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+	struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
+	struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 
 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
 		mlx5e_remove_sqs_fwd_rules(priv);
@@ -1010,8 +1067,9 @@ mlx5e_nic_rep_unload(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
 }
 
 static int
-mlx5e_vport_rep_load(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
+mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 {
+	struct mlx5e_rep_priv *uplink_rpriv;
 	struct mlx5e_rep_priv *rpriv;
 	struct net_device *netdev;
 	struct mlx5e_priv *upriv;
@@ -1021,7 +1079,7 @@ mlx5e_vport_rep_load(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
 	if (!rpriv)
 		return -ENOMEM;
 
-	netdev = mlx5e_create_netdev(esw->dev, &mlx5e_rep_profile, rpriv);
+	netdev = mlx5e_create_netdev(dev, &mlx5e_rep_profile, rpriv);
 	if (!netdev) {
 		pr_warn("Failed to create representor netdev for vport %d\n",
 			rep->vport);
@@ -1029,8 +1087,10 @@ mlx5e_vport_rep_load(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
 		return -EINVAL;
 	}
 
-	rep->netdev = netdev;
+	rpriv->netdev = netdev;
 	rpriv->rep = rep;
+	rep->rep_if[REP_ETH].priv = rpriv;
+	INIT_LIST_HEAD(&rpriv->vport_sqs_list);
 
 	err = mlx5e_attach_netdev(netdev_priv(netdev));
 	if (err) {
@@ -1046,7 +1106,8 @@ mlx5e_vport_rep_load(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
 		goto err_detach_netdev;
 	}
 
-	upriv = netdev_priv(mlx5_eswitch_get_uplink_netdev(esw));
+	uplink_rpriv = mlx5_eswitch_get_uplink_priv(dev->priv.eswitch, REP_ETH);
+	upriv = netdev_priv(uplink_rpriv->netdev);
 	err = tc_setup_cb_egdev_register(netdev, mlx5e_setup_tc_block_cb,
 					 upriv);
 	if (err)
@@ -1078,16 +1139,19 @@ mlx5e_vport_rep_load(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
 }
 
 static void
-mlx5e_vport_rep_unload(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep)
+mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep)
 {
-	struct net_device *netdev = rep->netdev;
+	struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep);
+	struct net_device *netdev = rpriv->netdev;
 	struct mlx5e_priv *priv = netdev_priv(netdev);
-	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+	struct mlx5e_rep_priv *uplink_rpriv;
 	void *ppriv = priv->ppriv;
 	struct mlx5e_priv *upriv;
 
-	unregister_netdev(rep->netdev);
-	upriv = netdev_priv(mlx5_eswitch_get_uplink_netdev(esw));
+	unregister_netdev(netdev);
+	uplink_rpriv = mlx5_eswitch_get_uplink_priv(priv->mdev->priv.eswitch,
+						    REP_ETH);
+	upriv = netdev_priv(uplink_rpriv->netdev);
 	tc_setup_cb_egdev_unregister(netdev, mlx5e_setup_tc_block_cb,
 				     upriv);
 	mlx5e_rep_neigh_cleanup(rpriv);
@@ -1102,18 +1166,13 @@ static void mlx5e_rep_register_vf_vports(struct mlx5e_priv *priv)
 	struct mlx5_eswitch *esw   = mdev->priv.eswitch;
 	int total_vfs = MLX5_TOTAL_VPORTS(mdev);
 	int vport;
-	u8 mac[ETH_ALEN];
-
-	mlx5_query_nic_vport_mac_address(mdev, 0, mac);
 
 	for (vport = 1; vport < total_vfs; vport++) {
-		struct mlx5_eswitch_rep rep;
+		struct mlx5_eswitch_rep_if rep_if = {};
 
-		rep.load = mlx5e_vport_rep_load;
-		rep.unload = mlx5e_vport_rep_unload;
-		rep.vport = vport;
-		ether_addr_copy(rep.hw_id, mac);
-		mlx5_eswitch_register_vport_rep(esw, vport, &rep);
+		rep_if.load = mlx5e_vport_rep_load;
+		rep_if.unload = mlx5e_vport_rep_unload;
+		mlx5_eswitch_register_vport_rep(esw, vport, &rep_if, REP_ETH);
 	}
 }
 
@@ -1125,21 +1184,24 @@ static void mlx5e_rep_unregister_vf_vports(struct mlx5e_priv *priv)
 	int vport;
 
 	for (vport = 1; vport < total_vfs; vport++)
-		mlx5_eswitch_unregister_vport_rep(esw, vport);
+		mlx5_eswitch_unregister_vport_rep(esw, vport, REP_ETH);
 }
 
 void mlx5e_register_vport_reps(struct mlx5e_priv *priv)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
 	struct mlx5_eswitch *esw   = mdev->priv.eswitch;
-	struct mlx5_eswitch_rep rep;
+	struct mlx5_eswitch_rep_if rep_if;
+	struct mlx5e_rep_priv *rpriv;
+
+	rpriv = priv->ppriv;
+	rpriv->netdev = priv->netdev;
 
-	mlx5_query_nic_vport_mac_address(mdev, 0, rep.hw_id);
-	rep.load = mlx5e_nic_rep_load;
-	rep.unload = mlx5e_nic_rep_unload;
-	rep.vport = FDB_UPLINK_VPORT;
-	rep.netdev = priv->netdev;
-	mlx5_eswitch_register_vport_rep(esw, 0, &rep); /* UPLINK PF vport*/
+	rep_if.load = mlx5e_nic_rep_load;
+	rep_if.unload = mlx5e_nic_rep_unload;
+	rep_if.priv = rpriv;
+	INIT_LIST_HEAD(&rpriv->vport_sqs_list);
+	mlx5_eswitch_register_vport_rep(esw, 0, &rep_if, REP_ETH); /* UPLINK PF vport*/
 
 	mlx5e_rep_register_vf_vports(priv); /* VFs vports */
 }
@@ -1150,7 +1212,7 @@ void mlx5e_unregister_vport_reps(struct mlx5e_priv *priv)
 	struct mlx5_eswitch *esw   = mdev->priv.eswitch;
 
 	mlx5e_rep_unregister_vf_vports(priv); /* VFs vports */
-	mlx5_eswitch_unregister_vport_rep(esw, 0); /* UPLINK PF*/
+	mlx5_eswitch_unregister_vport_rep(esw, 0, REP_ETH); /* UPLINK PF*/
 }
 
 void *mlx5e_alloc_nic_rep_priv(struct mlx5_core_dev *mdev)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index 5659ed9f51e6..b9b481f2833a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -56,8 +56,17 @@ struct mlx5e_neigh_update_table {
 struct mlx5e_rep_priv {
 	struct mlx5_eswitch_rep *rep;
 	struct mlx5e_neigh_update_table neigh_update;
+	struct net_device      *netdev;
+	struct mlx5_flow_handle *vport_rx_rule;
+	struct list_head       vport_sqs_list;
 };
 
+static inline
+struct mlx5e_rep_priv *mlx5e_rep_to_rep_priv(struct mlx5_eswitch_rep *rep)
+{
+	return (struct mlx5e_rep_priv *)rep->rep_if[REP_ETH].priv;
+}
+
 struct mlx5e_neigh {
 	struct net_device *dev;
 	union {
@@ -124,6 +133,11 @@ struct mlx5e_encap_entry {
 	int encap_size;
 };
 
+struct mlx5e_rep_sq {
+	struct mlx5_flow_handle	*send_to_vport_rule;
+	struct list_head	 list;
+};
+
 void *mlx5e_alloc_nic_rep_priv(struct mlx5_core_dev *mdev);
 void mlx5e_register_vport_reps(struct mlx5e_priv *priv);
 void mlx5e_unregister_vport_reps(struct mlx5e_priv *priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 5b499c7a698f..e5c3ab46a24a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -36,6 +36,7 @@
 #include <linux/tcp.h>
 #include <linux/bpf_trace.h>
 #include <net/busy_poll.h>
+#include <net/ip6_checksum.h>
 #include "en.h"
 #include "en_tc.h"
 #include "eswitch.h"
@@ -495,8 +496,8 @@ static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
 	mlx5_cqwq_pop(&cq->wq);
 
 	if (unlikely((cqe->op_own >> 4) != MLX5_CQE_REQ)) {
-		WARN_ONCE(true, "mlx5e: Bad OP in ICOSQ CQE: 0x%x\n",
-			  cqe->op_own);
+		netdev_WARN_ONCE(cq->channel->netdev,
+				 "Bad OP in ICOSQ CQE: 0x%x\n", cqe->op_own);
 		return;
 	}
 
@@ -506,9 +507,8 @@ static inline void mlx5e_poll_ico_single_cqe(struct mlx5e_cq *cq,
 	}
 
 	if (unlikely(icowi->opcode != MLX5_OPCODE_NOP))
-		WARN_ONCE(true,
-			  "mlx5e: Bad OPCODE in ICOSQ WQE info: 0x%x\n",
-			  icowi->opcode);
+		netdev_WARN_ONCE(cq->channel->netdev,
+				 "Bad OPCODE in ICOSQ WQE info: 0x%x\n", icowi->opcode);
 }
 
 static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq, struct mlx5e_rq *rq)
@@ -547,20 +547,33 @@ bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
 	return true;
 }
 
+static void mlx5e_lro_update_tcp_hdr(struct mlx5_cqe64 *cqe, struct tcphdr *tcp)
+{
+	u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
+	u8 tcp_ack     = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) ||
+			 (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);
+
+	tcp->check                      = 0;
+	tcp->psh                        = get_cqe_lro_tcppsh(cqe);
+
+	if (tcp_ack) {
+		tcp->ack                = 1;
+		tcp->ack_seq            = cqe->lro_ack_seq_num;
+		tcp->window             = cqe->lro_tcp_win;
+	}
+}
+
 static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
 				 u32 cqe_bcnt)
 {
 	struct ethhdr	*eth = (struct ethhdr *)(skb->data);
 	struct tcphdr	*tcp;
 	int network_depth = 0;
+	__wsum check;
 	__be16 proto;
 	u16 tot_len;
 	void *ip_p;
 
-	u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
-	u8 tcp_ack = (l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA) ||
-		(l4_hdr_type == CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA);
-
 	proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
 
 	tot_len = cqe_bcnt - network_depth;
@@ -577,23 +590,30 @@ static void mlx5e_lro_update_hdr(struct sk_buff *skb, struct mlx5_cqe64 *cqe,
 		ipv4->check             = 0;
 		ipv4->check             = ip_fast_csum((unsigned char *)ipv4,
 						       ipv4->ihl);
+
+		mlx5e_lro_update_tcp_hdr(cqe, tcp);
+		check = csum_partial(tcp, tcp->doff * 4,
+				     csum_unfold((__force __sum16)cqe->check_sum));
+		/* Almost done, don't forget the pseudo header */
+		tcp->check = csum_tcpudp_magic(ipv4->saddr, ipv4->daddr,
+					       tot_len - sizeof(struct iphdr),
+					       IPPROTO_TCP, check);
 	} else {
+		u16 payload_len = tot_len - sizeof(struct ipv6hdr);
 		struct ipv6hdr *ipv6 = ip_p;
 
 		tcp = ip_p + sizeof(struct ipv6hdr);
 		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
 
 		ipv6->hop_limit         = cqe->lro_min_ttl;
-		ipv6->payload_len       = cpu_to_be16(tot_len -
-						      sizeof(struct ipv6hdr));
-	}
-
-	tcp->psh = get_cqe_lro_tcppsh(cqe);
-
-	if (tcp_ack) {
-		tcp->ack                = 1;
-		tcp->ack_seq            = cqe->lro_ack_seq_num;
-		tcp->window             = cqe->lro_tcp_win;
+		ipv6->payload_len       = cpu_to_be16(payload_len);
+
+		mlx5e_lro_update_tcp_hdr(cqe, tcp);
+		check = csum_partial(tcp, tcp->doff * 4,
+				     csum_unfold((__force __sum16)cqe->check_sum));
+		/* Almost done, don't forget the pseudo header */
+		tcp->check = csum_ipv6_magic(&ipv6->saddr, &ipv6->daddr, payload_len,
+					     IPPROTO_TCP, check);
 	}
 }
 
@@ -632,7 +652,7 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
 		return;
 	}
 
-	if (is_last_ethertype_ip(skb, &network_depth)) {
+	if (likely(is_last_ethertype_ip(skb, &network_depth))) {
 		skb->ip_summed = CHECKSUM_COMPLETE;
 		skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
 		if (network_depth > ETH_HLEN)
@@ -812,6 +832,7 @@ static inline int mlx5e_xdp_handle(struct mlx5e_rq *rq,
 	xdp_set_data_meta_invalid(&xdp);
 	xdp.data_end = xdp.data + *len;
 	xdp.data_hard_start = va;
+	xdp.rxq = &rq->xdp_rxq;
 
 	act = bpf_prog_run_xdp(prog, &xdp);
 	switch (act) {
@@ -1175,7 +1196,9 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
 					 u32 cqe_bcnt,
 					 struct sk_buff *skb)
 {
+	struct hwtstamp_config *tstamp;
 	struct net_device *netdev;
+	struct mlx5e_priv *priv;
 	char *pseudo_header;
 	u32 qpn;
 	u8 *dgid;
@@ -1194,6 +1217,9 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
 		return;
 	}
 
+	priv = mlx5i_epriv(netdev);
+	tstamp = &priv->tstamp;
+
 	g = (be32_to_cpu(cqe->flags_rqpn) >> 28) & 3;
 	dgid = skb->data + MLX5_IB_GRH_DGID_OFFSET;
 	if ((!g) || dgid[0] != 0xff)
@@ -1214,7 +1240,7 @@ static inline void mlx5i_complete_rx_cqe(struct mlx5e_rq *rq,
 	skb->ip_summed = CHECKSUM_COMPLETE;
 	skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
 
-	if (unlikely(mlx5e_rx_hw_stamp(rq->tstamp)))
+	if (unlikely(mlx5e_rx_hw_stamp(tstamp)))
 		skb_hwtstamps(skb)->hwtstamp =
 				mlx5_timecounter_cyc2time(rq->clock, get_cqe_ts(cqe));
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c
deleted file mode 100644
index b69a705fd787..000000000000
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#include "en.h"
-
-/* Adaptive moderation profiles */
-#define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256
-#define MLX5E_RX_AM_DEF_PROFILE_CQE 1
-#define MLX5E_RX_AM_DEF_PROFILE_EQE 1
-#define MLX5E_PARAMS_AM_NUM_PROFILES 5
-
-/* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */
-#define MLX5_AM_EQE_PROFILES { \
-	{1,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
-	{8,   MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
-	{64,  MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
-	{128, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
-	{256, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \
-}
-
-#define MLX5_AM_CQE_PROFILES { \
-	{2,  256},             \
-	{8,  128},             \
-	{16, 64},              \
-	{32, 64},              \
-	{64, 64}               \
-}
-
-static const struct mlx5e_cq_moder
-profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = {
-	MLX5_AM_EQE_PROFILES,
-	MLX5_AM_CQE_PROFILES,
-};
-
-static inline struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix)
-{
-	struct mlx5e_cq_moder cq_moder;
-
-	cq_moder = profile[cq_period_mode][ix];
-	cq_moder.cq_period_mode = cq_period_mode;
-	return cq_moder;
-}
-
-struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode)
-{
-	int default_profile_ix;
-
-	if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
-		default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_CQE;
-	else /* MLX5_CQ_PERIOD_MODE_START_FROM_EQE */
-		default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_EQE;
-
-	return mlx5e_am_get_profile(rx_cq_period_mode, default_profile_ix);
-}
-
-/* Adaptive moderation logic */
-enum {
-	MLX5E_AM_START_MEASURE,
-	MLX5E_AM_MEASURE_IN_PROGRESS,
-	MLX5E_AM_APPLY_NEW_PROFILE,
-};
-
-enum {
-	MLX5E_AM_PARKING_ON_TOP,
-	MLX5E_AM_PARKING_TIRED,
-	MLX5E_AM_GOING_RIGHT,
-	MLX5E_AM_GOING_LEFT,
-};
-
-enum {
-	MLX5E_AM_STATS_WORSE,
-	MLX5E_AM_STATS_SAME,
-	MLX5E_AM_STATS_BETTER,
-};
-
-enum {
-	MLX5E_AM_STEPPED,
-	MLX5E_AM_TOO_TIRED,
-	MLX5E_AM_ON_EDGE,
-};
-
-static bool mlx5e_am_on_top(struct mlx5e_rx_am *am)
-{
-	switch (am->tune_state) {
-	case MLX5E_AM_PARKING_ON_TOP:
-	case MLX5E_AM_PARKING_TIRED:
-		return true;
-	case MLX5E_AM_GOING_RIGHT:
-		return (am->steps_left > 1) && (am->steps_right == 1);
-	default: /* MLX5E_AM_GOING_LEFT */
-		return (am->steps_right > 1) && (am->steps_left == 1);
-	}
-}
-
-static void mlx5e_am_turn(struct mlx5e_rx_am *am)
-{
-	switch (am->tune_state) {
-	case MLX5E_AM_PARKING_ON_TOP:
-	case MLX5E_AM_PARKING_TIRED:
-		break;
-	case MLX5E_AM_GOING_RIGHT:
-		am->tune_state = MLX5E_AM_GOING_LEFT;
-		am->steps_left = 0;
-		break;
-	case MLX5E_AM_GOING_LEFT:
-		am->tune_state = MLX5E_AM_GOING_RIGHT;
-		am->steps_right = 0;
-		break;
-	}
-}
-
-static int mlx5e_am_step(struct mlx5e_rx_am *am)
-{
-	if (am->tired == (MLX5E_PARAMS_AM_NUM_PROFILES * 2))
-		return MLX5E_AM_TOO_TIRED;
-
-	switch (am->tune_state) {
-	case MLX5E_AM_PARKING_ON_TOP:
-	case MLX5E_AM_PARKING_TIRED:
-		break;
-	case MLX5E_AM_GOING_RIGHT:
-		if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1))
-			return MLX5E_AM_ON_EDGE;
-		am->profile_ix++;
-		am->steps_right++;
-		break;
-	case MLX5E_AM_GOING_LEFT:
-		if (am->profile_ix == 0)
-			return MLX5E_AM_ON_EDGE;
-		am->profile_ix--;
-		am->steps_left++;
-		break;
-	}
-
-	am->tired++;
-	return MLX5E_AM_STEPPED;
-}
-
-static void mlx5e_am_park_on_top(struct mlx5e_rx_am *am)
-{
-	am->steps_right  = 0;
-	am->steps_left   = 0;
-	am->tired        = 0;
-	am->tune_state   = MLX5E_AM_PARKING_ON_TOP;
-}
-
-static void mlx5e_am_park_tired(struct mlx5e_rx_am *am)
-{
-	am->steps_right  = 0;
-	am->steps_left   = 0;
-	am->tune_state   = MLX5E_AM_PARKING_TIRED;
-}
-
-static void mlx5e_am_exit_parking(struct mlx5e_rx_am *am)
-{
-	am->tune_state = am->profile_ix ? MLX5E_AM_GOING_LEFT :
-					  MLX5E_AM_GOING_RIGHT;
-	mlx5e_am_step(am);
-}
-
-#define IS_SIGNIFICANT_DIFF(val, ref) \
-	(((100 * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */
-
-static int mlx5e_am_stats_compare(struct mlx5e_rx_am_stats *curr,
-				  struct mlx5e_rx_am_stats *prev)
-{
-	if (!prev->bpms)
-		return curr->bpms ? MLX5E_AM_STATS_BETTER :
-				    MLX5E_AM_STATS_SAME;
-
-	if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms))
-		return (curr->bpms > prev->bpms) ? MLX5E_AM_STATS_BETTER :
-						   MLX5E_AM_STATS_WORSE;
-
-	if (!prev->ppms)
-		return curr->ppms ? MLX5E_AM_STATS_BETTER :
-				    MLX5E_AM_STATS_SAME;
-
-	if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms))
-		return (curr->ppms > prev->ppms) ? MLX5E_AM_STATS_BETTER :
-						   MLX5E_AM_STATS_WORSE;
-	if (!prev->epms)
-		return MLX5E_AM_STATS_SAME;
-
-	if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms))
-		return (curr->epms < prev->epms) ? MLX5E_AM_STATS_BETTER :
-						   MLX5E_AM_STATS_WORSE;
-
-	return MLX5E_AM_STATS_SAME;
-}
-
-static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats,
-			      struct mlx5e_rx_am *am)
-{
-	int prev_state = am->tune_state;
-	int prev_ix = am->profile_ix;
-	int stats_res;
-	int step_res;
-
-	switch (am->tune_state) {
-	case MLX5E_AM_PARKING_ON_TOP:
-		stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
-		if (stats_res != MLX5E_AM_STATS_SAME)
-			mlx5e_am_exit_parking(am);
-		break;
-
-	case MLX5E_AM_PARKING_TIRED:
-		am->tired--;
-		if (!am->tired)
-			mlx5e_am_exit_parking(am);
-		break;
-
-	case MLX5E_AM_GOING_RIGHT:
-	case MLX5E_AM_GOING_LEFT:
-		stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats);
-		if (stats_res != MLX5E_AM_STATS_BETTER)
-			mlx5e_am_turn(am);
-
-		if (mlx5e_am_on_top(am)) {
-			mlx5e_am_park_on_top(am);
-			break;
-		}
-
-		step_res = mlx5e_am_step(am);
-		switch (step_res) {
-		case MLX5E_AM_ON_EDGE:
-			mlx5e_am_park_on_top(am);
-			break;
-		case MLX5E_AM_TOO_TIRED:
-			mlx5e_am_park_tired(am);
-			break;
-		}
-
-		break;
-	}
-
-	if ((prev_state     != MLX5E_AM_PARKING_ON_TOP) ||
-	    (am->tune_state != MLX5E_AM_PARKING_ON_TOP))
-		am->prev_stats = *curr_stats;
-
-	return am->profile_ix != prev_ix;
-}
-
-static void mlx5e_am_sample(struct mlx5e_rq *rq,
-			    struct mlx5e_rx_am_sample *s)
-{
-	s->time	     = ktime_get();
-	s->pkt_ctr   = rq->stats.packets;
-	s->byte_ctr  = rq->stats.bytes;
-	s->event_ctr = rq->cq.event_ctr;
-}
-
-#define MLX5E_AM_NEVENTS 64
-#define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE)
-#define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1))
-
-static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start,
-				struct mlx5e_rx_am_sample *end,
-				struct mlx5e_rx_am_stats *curr_stats)
-{
-	/* u32 holds up to 71 minutes, should be enough */
-	u32 delta_us = ktime_us_delta(end->time, start->time);
-	u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr);
-	u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr,
-			     start->byte_ctr);
-
-	if (!delta_us)
-		return;
-
-	curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us);
-	curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us);
-	curr_stats->epms = DIV_ROUND_UP(MLX5E_AM_NEVENTS * USEC_PER_MSEC,
-					delta_us);
-}
-
-void mlx5e_rx_am_work(struct work_struct *work)
-{
-	struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am,
-					      work);
-	struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am);
-	struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix];
-
-	mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq,
-				       cur_profile.usec, cur_profile.pkts);
-
-	am->state = MLX5E_AM_START_MEASURE;
-}
-
-void mlx5e_rx_am(struct mlx5e_rq *rq)
-{
-	struct mlx5e_rx_am *am = &rq->am;
-	struct mlx5e_rx_am_sample end_sample;
-	struct mlx5e_rx_am_stats curr_stats;
-	u16 nevents;
-
-	switch (am->state) {
-	case MLX5E_AM_MEASURE_IN_PROGRESS:
-		nevents = BIT_GAP(BITS_PER_TYPE(u16), rq->cq.event_ctr,
-				  am->start_sample.event_ctr);
-		if (nevents < MLX5E_AM_NEVENTS)
-			break;
-		mlx5e_am_sample(rq, &end_sample);
-		mlx5e_am_calc_stats(&am->start_sample, &end_sample,
-				    &curr_stats);
-		if (mlx5e_am_decision(&curr_stats, am)) {
-			am->state = MLX5E_AM_APPLY_NEW_PROFILE;
-			schedule_work(&am->work);
-			break;
-		}
-		/* fall through */
-	case MLX5E_AM_START_MEASURE:
-		mlx5e_am_sample(rq, &am->start_sample);
-		am->state = MLX5E_AM_MEASURE_IN_PROGRESS;
-		break;
-	case MLX5E_AM_APPLY_NEW_PROFILE:
-		break;
-	}
-}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
index 5a4608281f38..707976482c09 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
@@ -216,7 +216,8 @@ mlx5e_test_loopback_validate(struct sk_buff *skb,
 	if (iph->protocol != IPPROTO_UDP)
 		goto out;
 
-	udph = udp_hdr(skb);
+	/* Don't assume skb_transport_header() was set */
+	udph = (struct udphdr *)((u8 *)iph + 4 * iph->ihl);
 	if (udph->dest != htons(9))
 		goto out;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index b74ddc7984bc..5f0f3493d747 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -71,6 +71,7 @@ static const struct counter_desc sw_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_empty) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_busy) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cache_waive) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, ch_eq_rearm) },
 	{ MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, link_down_events_phy) },
 };
 
@@ -99,6 +100,72 @@ static int mlx5e_grp_sw_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
 	return idx;
 }
 
+static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_sw_stats temp, *s = &temp;
+	struct mlx5e_rq_stats *rq_stats;
+	struct mlx5e_sq_stats *sq_stats;
+	struct mlx5e_ch_stats *ch_stats;
+	int i, j;
+
+	memset(s, 0, sizeof(*s));
+	for (i = 0; i < priv->channels.num; i++) {
+		struct mlx5e_channel *c = priv->channels.c[i];
+
+		rq_stats = &c->rq.stats;
+		ch_stats = &c->stats;
+
+		s->rx_packets	+= rq_stats->packets;
+		s->rx_bytes	+= rq_stats->bytes;
+		s->rx_lro_packets += rq_stats->lro_packets;
+		s->rx_lro_bytes	+= rq_stats->lro_bytes;
+		s->rx_removed_vlan_packets += rq_stats->removed_vlan_packets;
+		s->rx_csum_none	+= rq_stats->csum_none;
+		s->rx_csum_complete += rq_stats->csum_complete;
+		s->rx_csum_unnecessary += rq_stats->csum_unnecessary;
+		s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
+		s->rx_xdp_drop += rq_stats->xdp_drop;
+		s->rx_xdp_tx += rq_stats->xdp_tx;
+		s->rx_xdp_tx_full += rq_stats->xdp_tx_full;
+		s->rx_wqe_err   += rq_stats->wqe_err;
+		s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
+		s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
+		s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
+		s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
+		s->rx_page_reuse  += rq_stats->page_reuse;
+		s->rx_cache_reuse += rq_stats->cache_reuse;
+		s->rx_cache_full  += rq_stats->cache_full;
+		s->rx_cache_empty += rq_stats->cache_empty;
+		s->rx_cache_busy  += rq_stats->cache_busy;
+		s->rx_cache_waive += rq_stats->cache_waive;
+		s->ch_eq_rearm += ch_stats->eq_rearm;
+
+		for (j = 0; j < priv->channels.params.num_tc; j++) {
+			sq_stats = &c->sq[j].stats;
+
+			s->tx_packets		+= sq_stats->packets;
+			s->tx_bytes		+= sq_stats->bytes;
+			s->tx_tso_packets	+= sq_stats->tso_packets;
+			s->tx_tso_bytes		+= sq_stats->tso_bytes;
+			s->tx_tso_inner_packets	+= sq_stats->tso_inner_packets;
+			s->tx_tso_inner_bytes	+= sq_stats->tso_inner_bytes;
+			s->tx_added_vlan_packets += sq_stats->added_vlan_packets;
+			s->tx_queue_stopped	+= sq_stats->stopped;
+			s->tx_queue_wake	+= sq_stats->wake;
+			s->tx_queue_dropped	+= sq_stats->dropped;
+			s->tx_xmit_more		+= sq_stats->xmit_more;
+			s->tx_csum_partial_inner += sq_stats->csum_partial_inner;
+			s->tx_csum_none		+= sq_stats->csum_none;
+			s->tx_csum_partial	+= sq_stats->csum_partial;
+		}
+	}
+
+	s->link_down_events_phy = MLX5_GET(ppcnt_reg,
+				priv->stats.pport.phy_counters,
+				counter_set.phys_layer_cntrs.link_down_events);
+	memcpy(&priv->stats.sw, s, sizeof(*s));
+}
+
 static const struct counter_desc q_stats_desc[] = {
 	{ MLX5E_DECLARE_STAT(struct mlx5e_qcounter_stats, rx_out_of_buffer) },
 };
@@ -128,6 +195,22 @@ static int mlx5e_grp_q_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
 	return idx;
 }
 
+static void mlx5e_grp_q_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_qcounter_stats *qcnt = &priv->stats.qcnt;
+	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)];
+	int err;
+
+	if (!priv->q_counter)
+		return;
+
+	err = mlx5_core_query_q_counter(priv->mdev, priv->q_counter, 0, out, sizeof(out));
+	if (err)
+		return;
+
+	qcnt->rx_out_of_buffer = MLX5_GET(query_q_counter_out, out, out_of_buffer);
+}
+
 #define VPORT_COUNTER_OFF(c) MLX5_BYTE_OFF(query_vport_counter_out, c)
 static const struct counter_desc vport_stats_desc[] = {
 	{ "rx_vport_unicast_packets",
@@ -200,6 +283,19 @@ static int mlx5e_grp_vport_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx;
 }
 
+static void mlx5e_grp_vport_update_stats(struct mlx5e_priv *priv)
+{
+	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
+	u32 *out = (u32 *)priv->stats.vport.query_vport_out;
+	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	MLX5_SET(query_vport_counter_in, in, opcode, MLX5_CMD_OP_QUERY_VPORT_COUNTER);
+	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
+	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
+	mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen);
+}
+
 #define PPORT_802_3_OFF(c) \
 	MLX5_BYTE_OFF(ppcnt_reg, \
 		      counter_set.eth_802_3_cntrs_grp_data_layout.c##_high)
@@ -252,6 +348,20 @@ static int mlx5e_grp_802_3_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx;
 }
 
+static void mlx5e_grp_802_3_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+	void *out;
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+	out = pstats->IEEE_802_3_counters;
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+}
+
 #define PPORT_2863_OFF(c) \
 	MLX5_BYTE_OFF(ppcnt_reg, \
 		      counter_set.eth_2863_cntrs_grp_data_layout.c##_high)
@@ -289,6 +399,20 @@ static int mlx5e_grp_2863_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx;
 }
 
+static void mlx5e_grp_2863_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+	void *out;
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+	out = pstats->RFC_2863_counters;
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+}
+
 #define PPORT_2819_OFF(c) \
 	MLX5_BYTE_OFF(ppcnt_reg, \
 		      counter_set.eth_2819_cntrs_grp_data_layout.c##_high)
@@ -336,6 +460,20 @@ static int mlx5e_grp_2819_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx;
 }
 
+static void mlx5e_grp_2819_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+	void *out;
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+	out = pstats->RFC_2819_counters;
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+}
+
 #define PPORT_PHY_STATISTICAL_OFF(c) \
 	MLX5_BYTE_OFF(ppcnt_reg, \
 		      counter_set.phys_layer_statistical_cntrs.c##_high)
@@ -376,6 +514,27 @@ static int mlx5e_grp_phy_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
 	return idx;
 }
 
+static void mlx5e_grp_phy_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+	void *out;
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+	out = pstats->phy_counters;
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+
+	if (!MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group))
+		return;
+
+	out = pstats->phy_statistical_counters;
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+}
+
 #define PPORT_ETH_EXT_OFF(c) \
 	MLX5_BYTE_OFF(ppcnt_reg, \
 		      counter_set.eth_extended_cntrs_grp_data_layout.c##_high)
@@ -418,6 +577,23 @@ static int mlx5e_grp_eth_ext_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx;
 }
 
+static void mlx5e_grp_eth_ext_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+	void *out;
+
+	if (!MLX5_CAP_PCAM_FEATURE(mdev, rx_buffer_fullness_counters))
+		return;
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+	out = pstats->eth_ext_counters;
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+}
+
 #define PCIE_PERF_OFF(c) \
 	MLX5_BYTE_OFF(mpcnt_reg, counter_set.pcie_perf_cntrs_grp_data_layout.c)
 static const struct counter_desc pcie_perf_stats_desc[] = {
@@ -505,6 +681,22 @@ static int mlx5e_grp_pcie_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx;
 }
 
+static void mlx5e_grp_pcie_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_pcie_stats *pcie_stats = &priv->stats.pcie;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(mpcnt_reg)] = {0};
+	int sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
+	void *out;
+
+	if (!MLX5_CAP_MCAM_FEATURE(mdev, pcie_performance_group))
+		return;
+
+	out = pcie_stats->pcie_perf_counters;
+	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
+}
+
 #define PPORT_PER_PRIO_OFF(c) \
 	MLX5_BYTE_OFF(ppcnt_reg, \
 		      counter_set.eth_per_prio_grp_data_layout.c##_high)
@@ -656,6 +848,47 @@ static int mlx5e_grp_per_prio_pfc_fill_stats(struct mlx5e_priv *priv,
 	return idx;
 }
 
+static int mlx5e_grp_per_prio_get_num_stats(struct mlx5e_priv *priv)
+{
+	return mlx5e_grp_per_prio_traffic_get_num_stats(priv) +
+		mlx5e_grp_per_prio_pfc_get_num_stats(priv);
+}
+
+static int mlx5e_grp_per_prio_fill_strings(struct mlx5e_priv *priv, u8 *data,
+					   int idx)
+{
+	idx = mlx5e_grp_per_prio_traffic_fill_strings(priv, data, idx);
+	idx = mlx5e_grp_per_prio_pfc_fill_strings(priv, data, idx);
+	return idx;
+}
+
+static int mlx5e_grp_per_prio_fill_stats(struct mlx5e_priv *priv, u64 *data,
+					 int idx)
+{
+	idx = mlx5e_grp_per_prio_traffic_fill_stats(priv, data, idx);
+	idx = mlx5e_grp_per_prio_pfc_fill_stats(priv, data, idx);
+	return idx;
+}
+
+static void mlx5e_grp_per_prio_update_stats(struct mlx5e_priv *priv)
+{
+	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {0};
+	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+	int prio;
+	void *out;
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
+	for (prio = 0; prio < NUM_PPORT_PRIO; prio++) {
+		out = pstats->per_prio_counters[prio];
+		MLX5_SET(ppcnt_reg, in, prio_tc, prio);
+		mlx5_core_access_reg(mdev, in, sz, out, sz,
+				     MLX5_REG_PPCNT, 0, 0);
+	}
+}
+
 static const struct counter_desc mlx5e_pme_status_desc[] = {
 	{ "module_unplug", 8 },
 };
@@ -723,6 +956,11 @@ static int mlx5e_grp_ipsec_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx + mlx5e_ipsec_get_stats(priv, data + idx);
 }
 
+static void mlx5e_grp_ipsec_update_stats(struct mlx5e_priv *priv)
+{
+	mlx5e_ipsec_update_stats(priv);
+}
+
 static const struct counter_desc rq_stats_desc[] = {
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, packets) },
 	{ MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, bytes) },
@@ -767,12 +1005,18 @@ static const struct counter_desc sq_stats_desc[] = {
 	{ MLX5E_DECLARE_TX_STAT(struct mlx5e_sq_stats, xmit_more) },
 };
 
+static const struct counter_desc ch_stats_desc[] = {
+	{ MLX5E_DECLARE_CH_STAT(struct mlx5e_ch_stats, eq_rearm) },
+};
+
 #define NUM_RQ_STATS			ARRAY_SIZE(rq_stats_desc)
 #define NUM_SQ_STATS			ARRAY_SIZE(sq_stats_desc)
+#define NUM_CH_STATS			ARRAY_SIZE(ch_stats_desc)
 
 static int mlx5e_grp_channels_get_num_stats(struct mlx5e_priv *priv)
 {
 	return (NUM_RQ_STATS * priv->channels.num) +
+		(NUM_CH_STATS * priv->channels.num) +
 		(NUM_SQ_STATS * priv->channels.num * priv->channels.params.num_tc);
 }
 
@@ -784,6 +1028,11 @@ static int mlx5e_grp_channels_fill_strings(struct mlx5e_priv *priv, u8 *data,
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 		return idx;
 
+	for (i = 0; i < priv->channels.num; i++)
+		for (j = 0; j < NUM_CH_STATS; j++)
+			sprintf(data + (idx++) * ETH_GSTRING_LEN,
+				ch_stats_desc[j].format, i);
+
 	for (i = 0; i < priv->channels.num; i++)
 		for (j = 0; j < NUM_RQ_STATS; j++)
 			sprintf(data + (idx++) * ETH_GSTRING_LEN, rq_stats_desc[j].format, i);
@@ -807,6 +1056,12 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 		return idx;
 
+	for (i = 0; i < channels->num; i++)
+		for (j = 0; j < NUM_CH_STATS; j++)
+			data[idx++] =
+				MLX5E_READ_CTR64_CPU(&channels->c[i]->stats,
+						     ch_stats_desc, j);
+
 	for (i = 0; i < channels->num; i++)
 		for (j = 0; j < NUM_RQ_STATS; j++)
 			data[idx++] =
@@ -823,61 +1078,71 @@ static int mlx5e_grp_channels_fill_stats(struct mlx5e_priv *priv, u64 *data,
 	return idx;
 }
 
+/* The stats groups order is opposite to the update_stats() order calls */
 const struct mlx5e_stats_grp mlx5e_stats_grps[] = {
 	{
 		.get_num_stats = mlx5e_grp_sw_get_num_stats,
 		.fill_strings = mlx5e_grp_sw_fill_strings,
 		.fill_stats = mlx5e_grp_sw_fill_stats,
+		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
+		.update_stats = mlx5e_grp_sw_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_q_get_num_stats,
 		.fill_strings = mlx5e_grp_q_fill_strings,
 		.fill_stats = mlx5e_grp_q_fill_stats,
+		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
+		.update_stats = mlx5e_grp_q_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_vport_get_num_stats,
 		.fill_strings = mlx5e_grp_vport_fill_strings,
 		.fill_stats = mlx5e_grp_vport_fill_stats,
+		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
+		.update_stats = mlx5e_grp_vport_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_802_3_get_num_stats,
 		.fill_strings = mlx5e_grp_802_3_fill_strings,
 		.fill_stats = mlx5e_grp_802_3_fill_stats,
+		.update_stats_mask = MLX5E_NDO_UPDATE_STATS,
+		.update_stats = mlx5e_grp_802_3_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_2863_get_num_stats,
 		.fill_strings = mlx5e_grp_2863_fill_strings,
 		.fill_stats = mlx5e_grp_2863_fill_stats,
+		.update_stats = mlx5e_grp_2863_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_2819_get_num_stats,
 		.fill_strings = mlx5e_grp_2819_fill_strings,
 		.fill_stats = mlx5e_grp_2819_fill_stats,
+		.update_stats = mlx5e_grp_2819_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_phy_get_num_stats,
 		.fill_strings = mlx5e_grp_phy_fill_strings,
 		.fill_stats = mlx5e_grp_phy_fill_stats,
+		.update_stats = mlx5e_grp_phy_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_eth_ext_get_num_stats,
 		.fill_strings = mlx5e_grp_eth_ext_fill_strings,
 		.fill_stats = mlx5e_grp_eth_ext_fill_stats,
+		.update_stats = mlx5e_grp_eth_ext_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_pcie_get_num_stats,
 		.fill_strings = mlx5e_grp_pcie_fill_strings,
 		.fill_stats = mlx5e_grp_pcie_fill_stats,
+		.update_stats = mlx5e_grp_pcie_update_stats,
 	},
 	{
-		.get_num_stats = mlx5e_grp_per_prio_traffic_get_num_stats,
-		.fill_strings = mlx5e_grp_per_prio_traffic_fill_strings,
-		.fill_stats = mlx5e_grp_per_prio_traffic_fill_stats,
-	},
-	{
-		.get_num_stats = mlx5e_grp_per_prio_pfc_get_num_stats,
-		.fill_strings = mlx5e_grp_per_prio_pfc_fill_strings,
-		.fill_stats = mlx5e_grp_per_prio_pfc_fill_stats,
+		.get_num_stats = mlx5e_grp_per_prio_get_num_stats,
+		.fill_strings = mlx5e_grp_per_prio_fill_strings,
+		.fill_stats = mlx5e_grp_per_prio_fill_stats,
+		.update_stats = mlx5e_grp_per_prio_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_pme_get_num_stats,
@@ -888,6 +1153,7 @@ const struct mlx5e_stats_grp mlx5e_stats_grps[] = {
 		.get_num_stats = mlx5e_grp_ipsec_get_num_stats,
 		.fill_strings = mlx5e_grp_ipsec_fill_strings,
 		.fill_stats = mlx5e_grp_ipsec_fill_stats,
+		.update_stats = mlx5e_grp_ipsec_update_stats,
 	},
 	{
 		.get_num_stats = mlx5e_grp_channels_get_num_stats,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index d679e21f686e..0b3320a2b072 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -44,6 +44,7 @@
 #define MLX5E_DECLARE_STAT(type, fld) #fld, offsetof(type, fld)
 #define MLX5E_DECLARE_RX_STAT(type, fld) "rx%d_"#fld, offsetof(type, fld)
 #define MLX5E_DECLARE_TX_STAT(type, fld) "tx%d_"#fld, offsetof(type, fld)
+#define MLX5E_DECLARE_CH_STAT(type, fld) "ch%d_"#fld, offsetof(type, fld)
 
 struct counter_desc {
 	char		format[ETH_GSTRING_LEN];
@@ -88,6 +89,7 @@ struct mlx5e_sw_stats {
 	u64 rx_cache_empty;
 	u64 rx_cache_busy;
 	u64 rx_cache_waive;
+	u64 ch_eq_rearm;
 
 	/* Special handling counters */
 	u64 link_down_events_phy;
@@ -192,6 +194,10 @@ struct mlx5e_sq_stats {
 	u64 dropped;
 };
 
+struct mlx5e_ch_stats {
+	u64 eq_rearm;
+};
+
 struct mlx5e_stats {
 	struct mlx5e_sw_stats sw;
 	struct mlx5e_qcounter_stats qcnt;
@@ -201,11 +207,17 @@ struct mlx5e_stats {
 	struct mlx5e_pcie_stats pcie;
 };
 
+enum {
+	MLX5E_NDO_UPDATE_STATS = BIT(0x1),
+};
+
 struct mlx5e_priv;
 struct mlx5e_stats_grp {
+	u16 update_stats_mask;
 	int (*get_num_stats)(struct mlx5e_priv *priv);
 	int (*fill_strings)(struct mlx5e_priv *priv, u8 *data, int idx);
 	int (*fill_stats)(struct mlx5e_priv *priv, u64 *data, int idx);
+	void (*update_stats)(struct mlx5e_priv *priv);
 };
 
 extern const struct mlx5e_stats_grp mlx5e_stats_grps[];
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index 55979ec2e88a..43234cabf444 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -51,17 +51,22 @@
 #include "en_tc.h"
 #include "eswitch.h"
 #include "vxlan.h"
+#include "fs_core.h"
 
 struct mlx5_nic_flow_attr {
 	u32 action;
 	u32 flow_tag;
 	u32 mod_hdr_id;
+	u32 hairpin_tirn;
+	struct mlx5_flow_table	*hairpin_ft;
 };
 
 enum {
 	MLX5E_TC_FLOW_ESWITCH	= BIT(0),
 	MLX5E_TC_FLOW_NIC	= BIT(1),
 	MLX5E_TC_FLOW_OFFLOADED	= BIT(2),
+	MLX5E_TC_FLOW_HAIRPIN	= BIT(3),
+	MLX5E_TC_FLOW_HAIRPIN_RSS = BIT(4),
 };
 
 struct mlx5e_tc_flow {
@@ -71,6 +76,7 @@ struct mlx5e_tc_flow {
 	struct mlx5_flow_handle *rule;
 	struct list_head	encap;   /* flows sharing the same encap ID */
 	struct list_head	mod_hdr; /* flows sharing the same mod hdr ID */
+	struct list_head	hairpin; /* flows sharing the same hairpin */
 	union {
 		struct mlx5_esw_flow_attr esw_attr[0];
 		struct mlx5_nic_flow_attr nic_attr[0];
@@ -93,6 +99,32 @@ enum {
 #define MLX5E_TC_TABLE_NUM_GROUPS 4
 #define MLX5E_TC_TABLE_MAX_GROUP_SIZE (1 << 16)
 
+struct mlx5e_hairpin {
+	struct mlx5_hairpin *pair;
+
+	struct mlx5_core_dev *func_mdev;
+	struct mlx5e_priv *func_priv;
+	u32 tdn;
+	u32 tirn;
+
+	int num_channels;
+	struct mlx5e_rqt indir_rqt;
+	u32 indir_tirn[MLX5E_NUM_INDIR_TIRS];
+	struct mlx5e_ttc_table ttc;
+};
+
+struct mlx5e_hairpin_entry {
+	/* a node of a hash table which keeps all the  hairpin entries */
+	struct hlist_node hairpin_hlist;
+
+	/* flows sharing the same hairpin */
+	struct list_head flows;
+
+	u16 peer_vhca_id;
+	u8 prio;
+	struct mlx5e_hairpin *hp;
+};
+
 struct mod_hdr_key {
 	int num_actions;
 	void *actions;
@@ -222,6 +254,417 @@ static void mlx5e_detach_mod_hdr(struct mlx5e_priv *priv,
 	}
 }
 
+static
+struct mlx5_core_dev *mlx5e_hairpin_get_mdev(struct net *net, int ifindex)
+{
+	struct net_device *netdev;
+	struct mlx5e_priv *priv;
+
+	netdev = __dev_get_by_index(net, ifindex);
+	priv = netdev_priv(netdev);
+	return priv->mdev;
+}
+
+static int mlx5e_hairpin_create_transport(struct mlx5e_hairpin *hp)
+{
+	u32 in[MLX5_ST_SZ_DW(create_tir_in)] = {0};
+	void *tirc;
+	int err;
+
+	err = mlx5_core_alloc_transport_domain(hp->func_mdev, &hp->tdn);
+	if (err)
+		goto alloc_tdn_err;
+
+	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
+
+	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
+	MLX5_SET(tirc, tirc, inline_rqn, hp->pair->rqn[0]);
+	MLX5_SET(tirc, tirc, transport_domain, hp->tdn);
+
+	err = mlx5_core_create_tir(hp->func_mdev, in, MLX5_ST_SZ_BYTES(create_tir_in), &hp->tirn);
+	if (err)
+		goto create_tir_err;
+
+	return 0;
+
+create_tir_err:
+	mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn);
+alloc_tdn_err:
+	return err;
+}
+
+static void mlx5e_hairpin_destroy_transport(struct mlx5e_hairpin *hp)
+{
+	mlx5_core_destroy_tir(hp->func_mdev, hp->tirn);
+	mlx5_core_dealloc_transport_domain(hp->func_mdev, hp->tdn);
+}
+
+static void mlx5e_hairpin_fill_rqt_rqns(struct mlx5e_hairpin *hp, void *rqtc)
+{
+	u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE], rqn;
+	struct mlx5e_priv *priv = hp->func_priv;
+	int i, ix, sz = MLX5E_INDIR_RQT_SIZE;
+
+	mlx5e_build_default_indir_rqt(indirection_rqt, sz,
+				      hp->num_channels);
+
+	for (i = 0; i < sz; i++) {
+		ix = i;
+		if (priv->channels.params.rss_hfunc == ETH_RSS_HASH_XOR)
+			ix = mlx5e_bits_invert(i, ilog2(sz));
+		ix = indirection_rqt[ix];
+		rqn = hp->pair->rqn[ix];
+		MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
+	}
+}
+
+static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp)
+{
+	int inlen, err, sz = MLX5E_INDIR_RQT_SIZE;
+	struct mlx5e_priv *priv = hp->func_priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	void *rqtc;
+	u32 *in;
+
+	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
+
+	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
+	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
+
+	mlx5e_hairpin_fill_rqt_rqns(hp, rqtc);
+
+	err = mlx5_core_create_rqt(mdev, in, inlen, &hp->indir_rqt.rqtn);
+	if (!err)
+		hp->indir_rqt.enabled = true;
+
+	kvfree(in);
+	return err;
+}
+
+static int mlx5e_hairpin_create_indirect_tirs(struct mlx5e_hairpin *hp)
+{
+	struct mlx5e_priv *priv = hp->func_priv;
+	u32 in[MLX5_ST_SZ_DW(create_tir_in)];
+	int tt, i, err;
+	void *tirc;
+
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
+		memset(in, 0, MLX5_ST_SZ_BYTES(create_tir_in));
+		tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
+
+		MLX5_SET(tirc, tirc, transport_domain, hp->tdn);
+		MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
+		MLX5_SET(tirc, tirc, indirect_table, hp->indir_rqt.rqtn);
+		mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, false);
+
+		err = mlx5_core_create_tir(hp->func_mdev, in,
+					   MLX5_ST_SZ_BYTES(create_tir_in), &hp->indir_tirn[tt]);
+		if (err) {
+			mlx5_core_warn(hp->func_mdev, "create indirect tirs failed, %d\n", err);
+			goto err_destroy_tirs;
+		}
+	}
+	return 0;
+
+err_destroy_tirs:
+	for (i = 0; i < tt; i++)
+		mlx5_core_destroy_tir(hp->func_mdev, hp->indir_tirn[i]);
+	return err;
+}
+
+static void mlx5e_hairpin_destroy_indirect_tirs(struct mlx5e_hairpin *hp)
+{
+	int tt;
+
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
+		mlx5_core_destroy_tir(hp->func_mdev, hp->indir_tirn[tt]);
+}
+
+static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin *hp,
+					 struct ttc_params *ttc_params)
+{
+	struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
+	int tt;
+
+	memset(ttc_params, 0, sizeof(*ttc_params));
+
+	ttc_params->any_tt_tirn = hp->tirn;
+
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
+		ttc_params->indir_tirn[tt] = hp->indir_tirn[tt];
+
+	ft_attr->max_fte = MLX5E_NUM_TT;
+	ft_attr->level = MLX5E_TC_TTC_FT_LEVEL;
+	ft_attr->prio = MLX5E_TC_PRIO;
+}
+
+static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp)
+{
+	struct mlx5e_priv *priv = hp->func_priv;
+	struct ttc_params ttc_params;
+	int err;
+
+	err = mlx5e_hairpin_create_indirect_rqt(hp);
+	if (err)
+		return err;
+
+	err = mlx5e_hairpin_create_indirect_tirs(hp);
+	if (err)
+		goto err_create_indirect_tirs;
+
+	mlx5e_hairpin_set_ttc_params(hp, &ttc_params);
+	err = mlx5e_create_ttc_table(priv, &ttc_params, &hp->ttc);
+	if (err)
+		goto err_create_ttc_table;
+
+	netdev_dbg(priv->netdev, "add hairpin: using %d channels rss ttc table id %x\n",
+		   hp->num_channels, hp->ttc.ft.t->id);
+
+	return 0;
+
+err_create_ttc_table:
+	mlx5e_hairpin_destroy_indirect_tirs(hp);
+err_create_indirect_tirs:
+	mlx5e_destroy_rqt(priv, &hp->indir_rqt);
+
+	return err;
+}
+
+static void mlx5e_hairpin_rss_cleanup(struct mlx5e_hairpin *hp)
+{
+	struct mlx5e_priv *priv = hp->func_priv;
+
+	mlx5e_destroy_ttc_table(priv, &hp->ttc);
+	mlx5e_hairpin_destroy_indirect_tirs(hp);
+	mlx5e_destroy_rqt(priv, &hp->indir_rqt);
+}
+
+static struct mlx5e_hairpin *
+mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params,
+		     int peer_ifindex)
+{
+	struct mlx5_core_dev *func_mdev, *peer_mdev;
+	struct mlx5e_hairpin *hp;
+	struct mlx5_hairpin *pair;
+	int err;
+
+	hp = kzalloc(sizeof(*hp), GFP_KERNEL);
+	if (!hp)
+		return ERR_PTR(-ENOMEM);
+
+	func_mdev = priv->mdev;
+	peer_mdev = mlx5e_hairpin_get_mdev(dev_net(priv->netdev), peer_ifindex);
+
+	pair = mlx5_core_hairpin_create(func_mdev, peer_mdev, params);
+	if (IS_ERR(pair)) {
+		err = PTR_ERR(pair);
+		goto create_pair_err;
+	}
+	hp->pair = pair;
+	hp->func_mdev = func_mdev;
+	hp->func_priv = priv;
+	hp->num_channels = params->num_channels;
+
+	err = mlx5e_hairpin_create_transport(hp);
+	if (err)
+		goto create_transport_err;
+
+	if (hp->num_channels > 1) {
+		err = mlx5e_hairpin_rss_init(hp);
+		if (err)
+			goto rss_init_err;
+	}
+
+	return hp;
+
+rss_init_err:
+	mlx5e_hairpin_destroy_transport(hp);
+create_transport_err:
+	mlx5_core_hairpin_destroy(hp->pair);
+create_pair_err:
+	kfree(hp);
+	return ERR_PTR(err);
+}
+
+static void mlx5e_hairpin_destroy(struct mlx5e_hairpin *hp)
+{
+	if (hp->num_channels > 1)
+		mlx5e_hairpin_rss_cleanup(hp);
+	mlx5e_hairpin_destroy_transport(hp);
+	mlx5_core_hairpin_destroy(hp->pair);
+	kvfree(hp);
+}
+
+static inline u32 hash_hairpin_info(u16 peer_vhca_id, u8 prio)
+{
+	return (peer_vhca_id << 16 | prio);
+}
+
+static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv,
+						     u16 peer_vhca_id, u8 prio)
+{
+	struct mlx5e_hairpin_entry *hpe;
+	u32 hash_key = hash_hairpin_info(peer_vhca_id, prio);
+
+	hash_for_each_possible(priv->fs.tc.hairpin_tbl, hpe,
+			       hairpin_hlist, hash_key) {
+		if (hpe->peer_vhca_id == peer_vhca_id && hpe->prio == prio)
+			return hpe;
+	}
+
+	return NULL;
+}
+
+#define UNKNOWN_MATCH_PRIO 8
+
+static int mlx5e_hairpin_get_prio(struct mlx5e_priv *priv,
+				  struct mlx5_flow_spec *spec, u8 *match_prio)
+{
+	void *headers_c, *headers_v;
+	u8 prio_val, prio_mask = 0;
+	bool vlan_present;
+
+#ifdef CONFIG_MLX5_CORE_EN_DCB
+	if (priv->dcbx_dp.trust_state != MLX5_QPTS_TRUST_PCP) {
+		netdev_warn(priv->netdev,
+			    "only PCP trust state supported for hairpin\n");
+		return -EOPNOTSUPP;
+	}
+#endif
+	headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers);
+	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers);
+
+	vlan_present = MLX5_GET(fte_match_set_lyr_2_4, headers_v, cvlan_tag);
+	if (vlan_present) {
+		prio_mask = MLX5_GET(fte_match_set_lyr_2_4, headers_c, first_prio);
+		prio_val = MLX5_GET(fte_match_set_lyr_2_4, headers_v, first_prio);
+	}
+
+	if (!vlan_present || !prio_mask) {
+		prio_val = UNKNOWN_MATCH_PRIO;
+	} else if (prio_mask != 0x7) {
+		netdev_warn(priv->netdev,
+			    "masked priority match not supported for hairpin\n");
+		return -EOPNOTSUPP;
+	}
+
+	*match_prio = prio_val;
+	return 0;
+}
+
+static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
+				  struct mlx5e_tc_flow *flow,
+				  struct mlx5e_tc_flow_parse_attr *parse_attr)
+{
+	int peer_ifindex = parse_attr->mirred_ifindex;
+	struct mlx5_hairpin_params params;
+	struct mlx5_core_dev *peer_mdev;
+	struct mlx5e_hairpin_entry *hpe;
+	struct mlx5e_hairpin *hp;
+	u64 link_speed64;
+	u32 link_speed;
+	u8 match_prio;
+	u16 peer_id;
+	int err;
+
+	peer_mdev = mlx5e_hairpin_get_mdev(dev_net(priv->netdev), peer_ifindex);
+	if (!MLX5_CAP_GEN(priv->mdev, hairpin) || !MLX5_CAP_GEN(peer_mdev, hairpin)) {
+		netdev_warn(priv->netdev, "hairpin is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	peer_id = MLX5_CAP_GEN(peer_mdev, vhca_id);
+	err = mlx5e_hairpin_get_prio(priv, &parse_attr->spec, &match_prio);
+	if (err)
+		return err;
+	hpe = mlx5e_hairpin_get(priv, peer_id, match_prio);
+	if (hpe)
+		goto attach_flow;
+
+	hpe = kzalloc(sizeof(*hpe), GFP_KERNEL);
+	if (!hpe)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&hpe->flows);
+	hpe->peer_vhca_id = peer_id;
+	hpe->prio = match_prio;
+
+	params.log_data_size = 15;
+	params.log_data_size = min_t(u8, params.log_data_size,
+				     MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz));
+	params.log_data_size = max_t(u8, params.log_data_size,
+				     MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz));
+
+	params.log_num_packets = params.log_data_size -
+				 MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(priv->mdev);
+	params.log_num_packets = min_t(u8, params.log_num_packets,
+				       MLX5_CAP_GEN(priv->mdev, log_max_hairpin_num_packets));
+
+	params.q_counter = priv->q_counter;
+	/* set hairpin pair per each 50Gbs share of the link */
+	mlx5e_get_max_linkspeed(priv->mdev, &link_speed);
+	link_speed = max_t(u32, link_speed, 50000);
+	link_speed64 = link_speed;
+	do_div(link_speed64, 50000);
+	params.num_channels = link_speed64;
+
+	hp = mlx5e_hairpin_create(priv, &params, peer_ifindex);
+	if (IS_ERR(hp)) {
+		err = PTR_ERR(hp);
+		goto create_hairpin_err;
+	}
+
+	netdev_dbg(priv->netdev, "add hairpin: tirn %x rqn %x peer %s sqn %x prio %d (log) data %d packets %d\n",
+		   hp->tirn, hp->pair->rqn[0], hp->pair->peer_mdev->priv.name,
+		   hp->pair->sqn[0], match_prio, params.log_data_size, params.log_num_packets);
+
+	hpe->hp = hp;
+	hash_add(priv->fs.tc.hairpin_tbl, &hpe->hairpin_hlist,
+		 hash_hairpin_info(peer_id, match_prio));
+
+attach_flow:
+	if (hpe->hp->num_channels > 1) {
+		flow->flags |= MLX5E_TC_FLOW_HAIRPIN_RSS;
+		flow->nic_attr->hairpin_ft = hpe->hp->ttc.ft.t;
+	} else {
+		flow->nic_attr->hairpin_tirn = hpe->hp->tirn;
+	}
+	list_add(&flow->hairpin, &hpe->flows);
+
+	return 0;
+
+create_hairpin_err:
+	kfree(hpe);
+	return err;
+}
+
+static void mlx5e_hairpin_flow_del(struct mlx5e_priv *priv,
+				   struct mlx5e_tc_flow *flow)
+{
+	struct list_head *next = flow->hairpin.next;
+
+	list_del(&flow->hairpin);
+
+	/* no more hairpin flows for us, release the hairpin pair */
+	if (list_empty(next)) {
+		struct mlx5e_hairpin_entry *hpe;
+
+		hpe = list_entry(next, struct mlx5e_hairpin_entry, flows);
+
+		netdev_dbg(priv->netdev, "del hairpin: peer %s\n",
+			   hpe->hp->pair->peer_mdev->priv.name);
+
+		mlx5e_hairpin_destroy(hpe->hp);
+		hash_del(&hpe->hairpin_hlist);
+		kfree(hpe);
+	}
+}
+
 static struct mlx5_flow_handle *
 mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 		      struct mlx5e_tc_flow_parse_attr *parse_attr,
@@ -229,7 +672,7 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 {
 	struct mlx5_nic_flow_attr *attr = flow->nic_attr;
 	struct mlx5_core_dev *dev = priv->mdev;
-	struct mlx5_flow_destination dest = {};
+	struct mlx5_flow_destination dest[2] = {};
 	struct mlx5_flow_act flow_act = {
 		.action = attr->action,
 		.flow_tag = attr->flow_tag,
@@ -238,18 +681,37 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 	struct mlx5_fc *counter = NULL;
 	struct mlx5_flow_handle *rule;
 	bool table_created = false;
-	int err;
+	int err, dest_ix = 0;
 
-	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
-		dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
-		dest.ft = priv->fs.vlan.ft.t;
-	} else if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
-		counter = mlx5_fc_create(dev, true);
-		if (IS_ERR(counter))
-			return ERR_CAST(counter);
+	if (flow->flags & MLX5E_TC_FLOW_HAIRPIN) {
+		err = mlx5e_hairpin_flow_add(priv, flow, parse_attr);
+		if (err) {
+			rule = ERR_PTR(err);
+			goto err_add_hairpin_flow;
+		}
+		if (flow->flags & MLX5E_TC_FLOW_HAIRPIN_RSS) {
+			dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+			dest[dest_ix].ft = attr->hairpin_ft;
+		} else {
+			dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
+			dest[dest_ix].tir_num = attr->hairpin_tirn;
+		}
+		dest_ix++;
+	} else if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
+		dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+		dest[dest_ix].ft = priv->fs.vlan.ft.t;
+		dest_ix++;
+	}
 
-		dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
-		dest.counter = counter;
+	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
+		counter = mlx5_fc_create(dev, true);
+		if (IS_ERR(counter)) {
+			rule = ERR_CAST(counter);
+			goto err_fc_create;
+		}
+		dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+		dest[dest_ix].counter = counter;
+		dest_ix++;
 	}
 
 	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
@@ -279,7 +741,7 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 							    MLX5E_TC_PRIO,
 							    tc_tbl_size,
 							    MLX5E_TC_TABLE_NUM_GROUPS,
-							    0, 0);
+							    MLX5E_TC_FT_LEVEL, 0);
 		if (IS_ERR(priv->fs.tc.t)) {
 			netdev_err(priv->netdev,
 				   "Failed to create tc offload table\n");
@@ -292,7 +754,7 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 
 	parse_attr->spec.match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 	rule = mlx5_add_flow_rules(priv->fs.tc.t, &parse_attr->spec,
-				   &flow_act, &dest, 1);
+				   &flow_act, dest, dest_ix);
 
 	if (IS_ERR(rule))
 		goto err_add_rule;
@@ -309,7 +771,10 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
 		mlx5e_detach_mod_hdr(priv, flow);
 err_create_mod_hdr_id:
 	mlx5_fc_destroy(dev, counter);
-
+err_fc_create:
+	if (flow->flags & MLX5E_TC_FLOW_HAIRPIN)
+		mlx5e_hairpin_flow_del(priv, flow);
+err_add_hairpin_flow:
 	return rule;
 }
 
@@ -330,6 +795,9 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
 
 	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
 		mlx5e_detach_mod_hdr(priv, flow);
+
+	if (flow->flags & MLX5E_TC_FLOW_HAIRPIN)
+		mlx5e_hairpin_flow_del(priv, flow);
 }
 
 static void mlx5e_detach_encap(struct mlx5e_priv *priv,
@@ -495,7 +963,7 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe)
 		tbl = &arp_tbl;
 #if IS_ENABLED(CONFIG_IPV6)
 	else if (m_neigh->family == AF_INET6)
-		tbl = ipv6_stub->nd_tbl;
+		tbl = &nd_tbl;
 #endif
 	else
 		return;
@@ -617,7 +1085,8 @@ static int parse_tunnel_attr(struct mlx5e_priv *priv,
 						  FLOW_DISSECTOR_KEY_ENC_PORTS,
 						  f->mask);
 		struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
-		struct net_device *up_dev = mlx5_eswitch_get_uplink_netdev(esw);
+		struct mlx5e_rep_priv *uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
+		struct net_device *up_dev = uplink_rpriv->netdev;
 		struct mlx5e_priv *up_priv = netdev_priv(up_dev);
 
 		/* Full udp dst port must be given */
@@ -1421,6 +1890,20 @@ static bool actions_match_supported(struct mlx5e_priv *priv,
 	return true;
 }
 
+static bool same_hw_devs(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv)
+{
+	struct mlx5_core_dev *fmdev, *pmdev;
+	u16 func_id, peer_id;
+
+	fmdev = priv->mdev;
+	pmdev = peer_priv->mdev;
+
+	func_id = (u16)((fmdev->pdev->bus->number << 8) | PCI_SLOT(fmdev->pdev->devfn));
+	peer_id = (u16)((pmdev->pdev->bus->number << 8) | PCI_SLOT(pmdev->pdev->devfn));
+
+	return (func_id == peer_id);
+}
+
 static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 				struct mlx5e_tc_flow_parse_attr *parse_attr,
 				struct mlx5e_tc_flow *flow)
@@ -1465,6 +1948,23 @@ static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 			return -EOPNOTSUPP;
 		}
 
+		if (is_tcf_mirred_egress_redirect(a)) {
+			struct net_device *peer_dev = tcf_mirred_dev(a);
+
+			if (priv->netdev->netdev_ops == peer_dev->netdev_ops &&
+			    same_hw_devs(priv, netdev_priv(peer_dev))) {
+				parse_attr->mirred_ifindex = peer_dev->ifindex;
+				flow->flags |= MLX5E_TC_FLOW_HAIRPIN;
+				attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+						MLX5_FLOW_CONTEXT_ACTION_COUNT;
+			} else {
+				netdev_warn(priv->netdev, "device %s not on same HW, can't offload\n",
+					    peer_dev->name);
+				return -EINVAL;
+			}
+			continue;
+		}
+
 		if (is_tcf_skbedit_mark(a)) {
 			u32 mark = tcf_skbedit_mark(a);
 
@@ -1507,6 +2007,7 @@ static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
 				   int *out_ttl)
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
+	struct mlx5e_rep_priv *uplink_rpriv;
 	struct rtable *rt;
 	struct neighbour *n = NULL;
 
@@ -1520,9 +2021,10 @@ static int mlx5e_route_lookup_ipv4(struct mlx5e_priv *priv,
 #else
 	return -EOPNOTSUPP;
 #endif
+	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 	/* if the egress device isn't on the same HW e-switch, we use the uplink */
 	if (!switchdev_port_same_parent_id(priv->netdev, rt->dst.dev))
-		*out_dev = mlx5_eswitch_get_uplink_netdev(esw);
+		*out_dev = uplink_rpriv->netdev;
 	else
 		*out_dev = rt->dst.dev;
 
@@ -1547,6 +2049,7 @@ static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv,
 	struct dst_entry *dst;
 
 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
+	struct mlx5e_rep_priv *uplink_rpriv;
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	int ret;
 
@@ -1557,9 +2060,10 @@ static int mlx5e_route_lookup_ipv6(struct mlx5e_priv *priv,
 
 	*out_ttl = ip6_dst_hoplimit(dst);
 
+	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 	/* if the egress device isn't on the same HW e-switch, we use the uplink */
 	if (!switchdev_port_same_parent_id(priv->netdev, dst->dev))
-		*out_dev = mlx5_eswitch_get_uplink_netdev(esw);
+		*out_dev = uplink_rpriv->netdev;
 	else
 		*out_dev = dst->dev;
 #else
@@ -1859,7 +2363,9 @@ static int mlx5e_attach_encap(struct mlx5e_priv *priv,
 			      struct mlx5e_tc_flow *flow)
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
-	struct net_device *up_dev = mlx5_eswitch_get_uplink_netdev(esw);
+	struct mlx5e_rep_priv *uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw,
+									   REP_ETH);
+	struct net_device *up_dev = uplink_rpriv->netdev;
 	unsigned short family = ip_tunnel_info_af(tun_info);
 	struct mlx5e_priv *up_priv = netdev_priv(up_dev);
 	struct mlx5_esw_flow_attr *attr = flow->esw_attr;
@@ -1982,11 +2488,10 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 		}
 
 		if (is_tcf_mirred_egress_redirect(a)) {
-			int ifindex = tcf_mirred_ifindex(a);
 			struct net_device *out_dev;
 			struct mlx5e_priv *out_priv;
 
-			out_dev = __dev_get_by_index(dev_net(priv->netdev), ifindex);
+			out_dev = tcf_mirred_dev(a);
 
 			if (switchdev_port_same_parent_id(priv->netdev,
 							  out_dev)) {
@@ -1996,7 +2501,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 				rpriv = out_priv->ppriv;
 				attr->out_rep = rpriv->rep;
 			} else if (encap) {
-				parse_attr->mirred_ifindex = ifindex;
+				parse_attr->mirred_ifindex = out_dev->ifindex;
 				parse_attr->tun_info = *info;
 				attr->parse_attr = parse_attr;
 				attr->action |= MLX5_FLOW_CONTEXT_ACTION_ENCAP |
@@ -2024,7 +2529,8 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, struct tcf_exts *exts,
 			if (tcf_vlan_action(a) == TCA_VLAN_ACT_POP) {
 				attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_POP;
 			} else if (tcf_vlan_action(a) == TCA_VLAN_ACT_PUSH) {
-				if (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q))
+				if (tcf_vlan_push_proto(a) != htons(ETH_P_8021Q) ||
+				    tcf_vlan_push_prio(a))
 					return -EOPNOTSUPP;
 
 				attr->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH;
@@ -2102,19 +2608,19 @@ int mlx5e_configure_flower(struct mlx5e_priv *priv,
 	if (err != -EAGAIN)
 		flow->flags |= MLX5E_TC_FLOW_OFFLOADED;
 
+	if (!(flow->flags & MLX5E_TC_FLOW_ESWITCH) ||
+	    !(flow->esw_attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP))
+		kvfree(parse_attr);
+
 	err = rhashtable_insert_fast(&tc->ht, &flow->node,
 				     tc->ht_params);
-	if (err)
-		goto err_del_rule;
+	if (err) {
+		mlx5e_tc_del_flow(priv, flow);
+		kfree(flow);
+	}
 
-	if (flow->flags & MLX5E_TC_FLOW_ESWITCH &&
-	    !(flow->esw_attr->action & MLX5_FLOW_CONTEXT_ACTION_ENCAP))
-		kvfree(parse_attr);
 	return err;
 
-err_del_rule:
-	mlx5e_tc_del_flow(priv, flow);
-
 err_free:
 	kvfree(parse_attr);
 	kfree(flow);
@@ -2182,6 +2688,7 @@ int mlx5e_tc_init(struct mlx5e_priv *priv)
 	struct mlx5e_tc_table *tc = &priv->fs.tc;
 
 	hash_init(tc->mod_hdr_tbl);
+	hash_init(tc->hairpin_tbl);
 
 	tc->ht_params = mlx5e_tc_flow_ht_params;
 	return rhashtable_init(&tc->ht, &tc->ht_params);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index 569b42a01026..11b4f1089d1c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -176,7 +176,7 @@ static inline u16 mlx5e_calc_min_inline(enum mlx5_inline_modes mode,
 	default:
 		hlen = mlx5e_skb_l2_header_offset(skb);
 	}
-	return min_t(u16, hlen, skb->len);
+	return min_t(u16, hlen, skb_headlen(skb));
 }
 
 static inline void mlx5e_tx_skb_pull_inline(unsigned char **skb_data,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index ab92298eafc3..f292bb346985 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -78,8 +78,14 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 	for (i = 0; i < c->num_tc; i++)
 		mlx5e_cq_arm(&c->sq[i].cq);
 
-	if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM))
-		mlx5e_rx_am(&c->rq);
+	if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) {
+		struct net_dim_sample dim_sample;
+		net_dim_sample(c->rq.cq.event_ctr,
+			       c->rq.stats.packets,
+			       c->rq.stats.bytes,
+			       &dim_sample);
+		net_dim(&c->rq.dim, dim_sample);
+	}
 
 	mlx5e_cq_arm(&c->rq.cq);
 	mlx5e_cq_arm(&c->icosq.cq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index e7e7cef2bde4..25106e996a96 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -51,7 +51,7 @@ enum {
 
 enum {
 	MLX5_NUM_SPARE_EQE	= 0x80,
-	MLX5_NUM_ASYNC_EQE	= 0x100,
+	MLX5_NUM_ASYNC_EQE	= 0x1000,
 	MLX5_NUM_CMD_EQE	= 32,
 	MLX5_NUM_PF_DRAIN	= 64,
 };
@@ -417,7 +417,11 @@ static irqreturn_t mlx5_eq_int(int irq, void *eq_ptr)
 			cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
 			mlx5_cq_completion(dev, cqn);
 			break;
-
+		case MLX5_EVENT_TYPE_DCT_DRAINED:
+			rsn = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
+			rsn |= (MLX5_RES_DCT << MLX5_USER_INDEX_LEN);
+			mlx5_rsc_event(dev, rsn, eqe->type);
+			break;
 		case MLX5_EVENT_TYPE_PATH_MIG:
 		case MLX5_EVENT_TYPE_COMM_EST:
 		case MLX5_EVENT_TYPE_SQ_DRAINED:
@@ -530,6 +534,24 @@ static irqreturn_t mlx5_eq_int(int irq, void *eq_ptr)
 	return IRQ_HANDLED;
 }
 
+/* Some architectures don't latch interrupts when they are disabled, so using
+ * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to
+ * avoid losing them.  It is not recommended to use it, unless this is the last
+ * resort.
+ */
+u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq *eq)
+{
+	u32 count_eqe;
+
+	disable_irq(eq->irqn);
+	count_eqe = eq->cons_index;
+	mlx5_eq_int(eq->irqn, eq);
+	count_eqe = eq->cons_index - count_eqe;
+	enable_irq(eq->irqn);
+
+	return count_eqe;
+}
+
 static void init_eq_buf(struct mlx5_eq *eq)
 {
 	struct mlx5_eqe *eqe;
@@ -715,6 +737,9 @@ int mlx5_start_eqs(struct mlx5_core_dev *dev)
 
 	if (MLX5_CAP_GEN(dev, fpga))
 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR);
+	if (MLX5_CAP_GEN_MAX(dev, dct))
+		async_event_mask |= (1ull << MLX5_EVENT_TYPE_DCT_DRAINED);
+
 
 	err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
 				 MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index bbb140f517c4..c2b1d7d351fc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -37,6 +37,7 @@
 #include <linux/mlx5/fs.h>
 #include "mlx5_core.h"
 #include "eswitch.h"
+#include "fs_core.h"
 
 #define UPLINK_VPORT 0xFFFF
 
@@ -867,9 +868,10 @@ static int esw_vport_enable_egress_acl(struct mlx5_eswitch *esw,
 	esw_debug(dev, "Create vport[%d] egress ACL log_max_size(%d)\n",
 		  vport->vport, MLX5_CAP_ESW_EGRESS_ACL(dev, log_max_ft_size));
 
-	root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_EGRESS);
+	root_ns = mlx5_get_flow_vport_acl_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_EGRESS,
+						    vport->vport);
 	if (!root_ns) {
-		esw_warn(dev, "Failed to get E-Switch egress flow namespace\n");
+		esw_warn(dev, "Failed to get E-Switch egress flow namespace for vport (%d)\n", vport->vport);
 		return -EOPNOTSUPP;
 	}
 
@@ -984,9 +986,10 @@ static int esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw,
 	esw_debug(dev, "Create vport[%d] ingress ACL log_max_size(%d)\n",
 		  vport->vport, MLX5_CAP_ESW_INGRESS_ACL(dev, log_max_ft_size));
 
-	root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_INGRESS);
+	root_ns = mlx5_get_flow_vport_acl_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_INGRESS,
+						    vport->vport);
 	if (!root_ns) {
-		esw_warn(dev, "Failed to get E-Switch ingress flow namespace\n");
+		esw_warn(dev, "Failed to get E-Switch ingress flow namespace for vport (%d)\n", vport->vport);
 		return -EOPNOTSUPP;
 	}
 
@@ -1121,8 +1124,12 @@ static void esw_vport_disable_ingress_acl(struct mlx5_eswitch *esw,
 static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 				    struct mlx5_vport *vport)
 {
+	struct mlx5_fc *counter = vport->ingress.drop_counter;
+	struct mlx5_flow_destination drop_ctr_dst = {0};
+	struct mlx5_flow_destination *dst = NULL;
 	struct mlx5_flow_act flow_act = {0};
 	struct mlx5_flow_spec *spec;
+	int dest_num = 0;
 	int err = 0;
 	u8 *smac_v;
 
@@ -1186,9 +1193,18 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 
 	memset(spec, 0, sizeof(*spec));
 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
+
+	/* Attach drop flow counter */
+	if (counter) {
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
+		drop_ctr_dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+		drop_ctr_dst.counter = counter;
+		dst = &drop_ctr_dst;
+		dest_num++;
+	}
 	vport->ingress.drop_rule =
 		mlx5_add_flow_rules(vport->ingress.acl, spec,
-				    &flow_act, NULL, 0);
+				    &flow_act, dst, dest_num);
 	if (IS_ERR(vport->ingress.drop_rule)) {
 		err = PTR_ERR(vport->ingress.drop_rule);
 		esw_warn(esw->dev,
@@ -1208,8 +1224,12 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw,
 static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 				   struct mlx5_vport *vport)
 {
+	struct mlx5_fc *counter = vport->egress.drop_counter;
+	struct mlx5_flow_destination drop_ctr_dst = {0};
+	struct mlx5_flow_destination *dst = NULL;
 	struct mlx5_flow_act flow_act = {0};
 	struct mlx5_flow_spec *spec;
+	int dest_num = 0;
 	int err = 0;
 
 	esw_vport_cleanup_egress_rules(esw, vport);
@@ -1260,9 +1280,18 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw,
 	/* Drop others rule (star rule) */
 	memset(spec, 0, sizeof(*spec));
 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
+
+	/* Attach egress drop flow counter */
+	if (counter) {
+		flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
+		drop_ctr_dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+		drop_ctr_dst.counter = counter;
+		dst = &drop_ctr_dst;
+		dest_num++;
+	}
 	vport->egress.drop_rule =
 		mlx5_add_flow_rules(vport->egress.acl, spec,
-				    &flow_act, NULL, 0);
+				    &flow_act, dst, dest_num);
 	if (IS_ERR(vport->egress.drop_rule)) {
 		err = PTR_ERR(vport->egress.drop_rule);
 		esw_warn(esw->dev,
@@ -1290,7 +1319,7 @@ static int esw_create_tsar(struct mlx5_eswitch *esw)
 
 	err = mlx5_create_scheduling_element_cmd(dev,
 						 SCHEDULING_HIERARCHY_E_SWITCH,
-						 &tsar_ctx,
+						 tsar_ctx,
 						 &esw->qos.root_tsar_id);
 	if (err) {
 		esw_warn(esw->dev, "E-Switch create TSAR failed (%d)\n", err);
@@ -1333,20 +1362,20 @@ static int esw_vport_enable_qos(struct mlx5_eswitch *esw, int vport_num,
 	if (vport->qos.enabled)
 		return -EEXIST;
 
-	MLX5_SET(scheduling_context, &sched_ctx, element_type,
+	MLX5_SET(scheduling_context, sched_ctx, element_type,
 		 SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT);
-	vport_elem = MLX5_ADDR_OF(scheduling_context, &sched_ctx,
+	vport_elem = MLX5_ADDR_OF(scheduling_context, sched_ctx,
 				  element_attributes);
 	MLX5_SET(vport_element, vport_elem, vport_number, vport_num);
-	MLX5_SET(scheduling_context, &sched_ctx, parent_element_id,
+	MLX5_SET(scheduling_context, sched_ctx, parent_element_id,
 		 esw->qos.root_tsar_id);
-	MLX5_SET(scheduling_context, &sched_ctx, max_average_bw,
+	MLX5_SET(scheduling_context, sched_ctx, max_average_bw,
 		 initial_max_rate);
-	MLX5_SET(scheduling_context, &sched_ctx, bw_share, initial_bw_share);
+	MLX5_SET(scheduling_context, sched_ctx, bw_share, initial_bw_share);
 
 	err = mlx5_create_scheduling_element_cmd(dev,
 						 SCHEDULING_HIERARCHY_E_SWITCH,
-						 &sched_ctx,
+						 sched_ctx,
 						 &vport->qos.esw_tsar_ix);
 	if (err) {
 		esw_warn(esw->dev, "E-Switch create TSAR vport element failed (vport=%d,err=%d)\n",
@@ -1392,22 +1421,22 @@ static int esw_vport_qos_config(struct mlx5_eswitch *esw, int vport_num,
 	if (!vport->qos.enabled)
 		return -EIO;
 
-	MLX5_SET(scheduling_context, &sched_ctx, element_type,
+	MLX5_SET(scheduling_context, sched_ctx, element_type,
 		 SCHEDULING_CONTEXT_ELEMENT_TYPE_VPORT);
-	vport_elem = MLX5_ADDR_OF(scheduling_context, &sched_ctx,
+	vport_elem = MLX5_ADDR_OF(scheduling_context, sched_ctx,
 				  element_attributes);
 	MLX5_SET(vport_element, vport_elem, vport_number, vport_num);
-	MLX5_SET(scheduling_context, &sched_ctx, parent_element_id,
+	MLX5_SET(scheduling_context, sched_ctx, parent_element_id,
 		 esw->qos.root_tsar_id);
-	MLX5_SET(scheduling_context, &sched_ctx, max_average_bw,
+	MLX5_SET(scheduling_context, sched_ctx, max_average_bw,
 		 max_rate);
-	MLX5_SET(scheduling_context, &sched_ctx, bw_share, bw_share);
+	MLX5_SET(scheduling_context, sched_ctx, bw_share, bw_share);
 	bitmask |= MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW;
 	bitmask |= MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_BW_SHARE;
 
 	err = mlx5_modify_scheduling_element_cmd(dev,
 						 SCHEDULING_HIERARCHY_E_SWITCH,
-						 &sched_ctx,
+						 sched_ctx,
 						 vport->qos.esw_tsar_ix,
 						 bitmask);
 	if (err) {
@@ -1455,6 +1484,41 @@ static void esw_apply_vport_conf(struct mlx5_eswitch *esw,
 	}
 }
 
+static void esw_vport_create_drop_counters(struct mlx5_vport *vport)
+{
+	struct mlx5_core_dev *dev = vport->dev;
+
+	if (MLX5_CAP_ESW_INGRESS_ACL(dev, flow_counter)) {
+		vport->ingress.drop_counter = mlx5_fc_create(dev, false);
+		if (IS_ERR(vport->ingress.drop_counter)) {
+			esw_warn(dev,
+				 "vport[%d] configure ingress drop rule counter failed\n",
+				 vport->vport);
+			vport->ingress.drop_counter = NULL;
+		}
+	}
+
+	if (MLX5_CAP_ESW_EGRESS_ACL(dev, flow_counter)) {
+		vport->egress.drop_counter = mlx5_fc_create(dev, false);
+		if (IS_ERR(vport->egress.drop_counter)) {
+			esw_warn(dev,
+				 "vport[%d] configure egress drop rule counter failed\n",
+				 vport->vport);
+			vport->egress.drop_counter = NULL;
+		}
+	}
+}
+
+static void esw_vport_destroy_drop_counters(struct mlx5_vport *vport)
+{
+	struct mlx5_core_dev *dev = vport->dev;
+
+	if (vport->ingress.drop_counter)
+		mlx5_fc_destroy(dev, vport->ingress.drop_counter);
+	if (vport->egress.drop_counter)
+		mlx5_fc_destroy(dev, vport->egress.drop_counter);
+}
+
 static void esw_enable_vport(struct mlx5_eswitch *esw, int vport_num,
 			     int enable_events)
 {
@@ -1465,6 +1529,10 @@ static void esw_enable_vport(struct mlx5_eswitch *esw, int vport_num,
 
 	esw_debug(esw->dev, "Enabling VPORT(%d)\n", vport_num);
 
+	/* Create steering drop counters for ingress and egress ACLs */
+	if (vport_num && esw->mode == SRIOV_LEGACY)
+		esw_vport_create_drop_counters(vport);
+
 	/* Restore old vport configuration */
 	esw_apply_vport_conf(esw, vport);
 
@@ -1519,6 +1587,7 @@ static void esw_disable_vport(struct mlx5_eswitch *esw, int vport_num)
 					      MLX5_ESW_VPORT_ADMIN_STATE_DOWN);
 		esw_vport_disable_egress_acl(esw, vport);
 		esw_vport_disable_ingress_acl(esw, vport);
+		esw_vport_destroy_drop_counters(vport);
 	}
 	esw->enabled_vports--;
 	mutex_unlock(&esw->state_lock);
@@ -1644,13 +1713,9 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 		goto abort;
 	}
 
-	esw->offloads.vport_reps =
-		kzalloc(total_vports * sizeof(struct mlx5_eswitch_rep),
-			GFP_KERNEL);
-	if (!esw->offloads.vport_reps) {
-		err = -ENOMEM;
+	err = esw_offloads_init_reps(esw);
+	if (err)
 		goto abort;
-	}
 
 	hash_init(esw->offloads.encap_tbl);
 	hash_init(esw->offloads.mod_hdr_tbl);
@@ -1681,8 +1746,8 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 abort:
 	if (esw->work_queue)
 		destroy_workqueue(esw->work_queue);
+	esw_offloads_cleanup_reps(esw);
 	kfree(esw->vports);
-	kfree(esw->offloads.vport_reps);
 	kfree(esw);
 	return err;
 }
@@ -1696,7 +1761,7 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
 
 	esw->dev->priv.eswitch = NULL;
 	destroy_workqueue(esw->work_queue);
-	kfree(esw->offloads.vport_reps);
+	esw_offloads_cleanup_reps(esw);
 	kfree(esw->vports);
 	kfree(esw);
 }
@@ -2018,12 +2083,36 @@ int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, int vport,
 	return err;
 }
 
+static void mlx5_eswitch_query_vport_drop_stats(struct mlx5_core_dev *dev,
+						int vport_idx,
+						struct mlx5_vport_drop_stats *stats)
+{
+	struct mlx5_eswitch *esw = dev->priv.eswitch;
+	struct mlx5_vport *vport = &esw->vports[vport_idx];
+	u64 bytes = 0;
+	u16 idx = 0;
+
+	if (!vport->enabled || esw->mode != SRIOV_LEGACY)
+		return;
+
+	if (vport->egress.drop_counter) {
+		idx = vport->egress.drop_counter->id;
+		mlx5_fc_query(dev, idx, &stats->rx_dropped, &bytes);
+	}
+
+	if (vport->ingress.drop_counter) {
+		idx = vport->ingress.drop_counter->id;
+		mlx5_fc_query(dev, idx, &stats->tx_dropped, &bytes);
+	}
+}
+
 int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw,
 				 int vport,
 				 struct ifla_vf_stats *vf_stats)
 {
 	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
 	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)] = {0};
+	struct mlx5_vport_drop_stats stats = {0};
 	int err = 0;
 	u32 *out;
 
@@ -2078,6 +2167,10 @@ int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw,
 	vf_stats->broadcast =
 		MLX5_GET_CTR(out, received_eth_broadcast.packets);
 
+	mlx5_eswitch_query_vport_drop_stats(esw->dev, vport, &stats);
+	vf_stats->rx_dropped = stats.rx_dropped;
+	vf_stats->tx_dropped = stats.tx_dropped;
+
 free_out:
 	kvfree(out);
 	return err;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 565c8b7a399a..2fa037066b2f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -45,6 +45,11 @@ enum {
 	SRIOV_OFFLOADS
 };
 
+enum {
+	REP_ETH,
+	NUM_REP_TYPES,
+};
+
 #ifdef CONFIG_MLX5_ESWITCH
 
 #define MLX5_MAX_UC_PER_VPORT(dev) \
@@ -68,6 +73,7 @@ struct vport_ingress {
 	struct mlx5_flow_group *drop_grp;
 	struct mlx5_flow_handle  *allow_rule;
 	struct mlx5_flow_handle  *drop_rule;
+	struct mlx5_fc           *drop_counter;
 };
 
 struct vport_egress {
@@ -76,6 +82,12 @@ struct vport_egress {
 	struct mlx5_flow_group *drop_grp;
 	struct mlx5_flow_handle  *allowed_vlan;
 	struct mlx5_flow_handle  *drop_rule;
+	struct mlx5_fc           *drop_counter;
+};
+
+struct mlx5_vport_drop_stats {
+	u64 rx_dropped;
+	u64 tx_dropped;
 };
 
 struct mlx5_vport_info {
@@ -133,25 +145,21 @@ struct mlx5_eswitch_fdb {
 	};
 };
 
-struct mlx5_esw_sq {
-	struct mlx5_flow_handle	*send_to_vport_rule;
-	struct list_head	 list;
+struct mlx5_eswitch_rep;
+struct mlx5_eswitch_rep_if {
+	int		       (*load)(struct mlx5_core_dev *dev,
+				       struct mlx5_eswitch_rep *rep);
+	void		       (*unload)(struct mlx5_eswitch_rep *rep);
+	void			*priv;
+	bool		       valid;
 };
 
 struct mlx5_eswitch_rep {
-	int		       (*load)(struct mlx5_eswitch *esw,
-				       struct mlx5_eswitch_rep *rep);
-	void		       (*unload)(struct mlx5_eswitch *esw,
-					 struct mlx5_eswitch_rep *rep);
+	struct mlx5_eswitch_rep_if rep_if[NUM_REP_TYPES];
 	u16		       vport;
 	u8		       hw_id[ETH_ALEN];
-	struct net_device      *netdev;
-
-	struct mlx5_flow_handle *vport_rx_rule;
-	struct list_head       vport_sqs_list;
 	u16		       vlan;
 	u32		       vlan_refcount;
-	bool		       valid;
 };
 
 struct mlx5_esw_offload {
@@ -197,6 +205,8 @@ struct mlx5_eswitch {
 
 void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports);
 int esw_offloads_init(struct mlx5_eswitch *esw, int nvports);
+void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw);
+int esw_offloads_init_reps(struct mlx5_eswitch *esw);
 
 /* E-Switch API */
 int mlx5_eswitch_init(struct mlx5_core_dev *dev);
@@ -221,6 +231,10 @@ int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw,
 int mlx5_eswitch_get_vport_stats(struct mlx5_eswitch *esw,
 				 int vport,
 				 struct ifla_vf_stats *vf_stats);
+struct mlx5_flow_handle *
+mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport,
+				    u32 sqn);
+void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule);
 
 struct mlx5_flow_spec;
 struct mlx5_esw_flow_attr;
@@ -257,12 +271,6 @@ struct mlx5_esw_flow_attr {
 	struct mlx5e_tc_flow_parse_attr *parse_attr;
 };
 
-int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch *esw,
-				 struct mlx5_eswitch_rep *rep,
-				 u16 *sqns_array, int sqns_num);
-void mlx5_eswitch_sqs2vport_stop(struct mlx5_eswitch *esw,
-				 struct mlx5_eswitch_rep *rep);
-
 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode);
 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode);
 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode);
@@ -272,10 +280,12 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, u8 encap);
 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, u8 *encap);
 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch *esw,
 				     int vport_index,
-				     struct mlx5_eswitch_rep *rep);
+				     struct mlx5_eswitch_rep_if *rep_if,
+				     u8 rep_type);
 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch *esw,
-				       int vport_index);
-struct net_device *mlx5_eswitch_get_uplink_netdev(struct mlx5_eswitch *esw);
+				       int vport_index,
+				       u8 rep_type);
+void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type);
 
 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
 				 struct mlx5_esw_flow_attr *attr);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index 1143d80119bd..99f583a15cc3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -130,7 +130,7 @@ static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
 	esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
 	for (vf_vport = 1; vf_vport < esw->enabled_vports; vf_vport++) {
 		rep = &esw->offloads.vport_reps[vf_vport];
-		if (!rep->valid)
+		if (!rep->rep_if[REP_ETH].valid)
 			continue;
 
 		err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
@@ -302,7 +302,7 @@ int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
 	return err;
 }
 
-static struct mlx5_flow_handle *
+struct mlx5_flow_handle *
 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport, u32 sqn)
 {
 	struct mlx5_flow_act flow_act = {0};
@@ -339,57 +339,9 @@ mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport, u32 sqn
 	return flow_rule;
 }
 
-void mlx5_eswitch_sqs2vport_stop(struct mlx5_eswitch *esw,
-				 struct mlx5_eswitch_rep *rep)
-{
-	struct mlx5_esw_sq *esw_sq, *tmp;
-
-	if (esw->mode != SRIOV_OFFLOADS)
-		return;
-
-	list_for_each_entry_safe(esw_sq, tmp, &rep->vport_sqs_list, list) {
-		mlx5_del_flow_rules(esw_sq->send_to_vport_rule);
-		list_del(&esw_sq->list);
-		kfree(esw_sq);
-	}
-}
-
-int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch *esw,
-				 struct mlx5_eswitch_rep *rep,
-				 u16 *sqns_array, int sqns_num)
+void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
 {
-	struct mlx5_flow_handle *flow_rule;
-	struct mlx5_esw_sq *esw_sq;
-	int err;
-	int i;
-
-	if (esw->mode != SRIOV_OFFLOADS)
-		return 0;
-
-	for (i = 0; i < sqns_num; i++) {
-		esw_sq = kzalloc(sizeof(*esw_sq), GFP_KERNEL);
-		if (!esw_sq) {
-			err = -ENOMEM;
-			goto out_err;
-		}
-
-		/* Add re-inject rule to the PF/representor sqs */
-		flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw,
-								rep->vport,
-								sqns_array[i]);
-		if (IS_ERR(flow_rule)) {
-			err = PTR_ERR(flow_rule);
-			kfree(esw_sq);
-			goto out_err;
-		}
-		esw_sq->send_to_vport_rule = flow_rule;
-		list_add(&esw_sq->list, &rep->vport_sqs_list);
-	}
-	return 0;
-
-out_err:
-	mlx5_eswitch_sqs2vport_stop(esw, rep);
-	return err;
+	mlx5_del_flow_rules(rule);
 }
 
 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
@@ -732,12 +684,111 @@ static int esw_offloads_start(struct mlx5_eswitch *esw)
 	return err;
 }
 
-int esw_offloads_init(struct mlx5_eswitch *esw, int nvports)
+void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
+{
+	kfree(esw->offloads.vport_reps);
+}
+
+int esw_offloads_init_reps(struct mlx5_eswitch *esw)
+{
+	int total_vfs = MLX5_TOTAL_VPORTS(esw->dev);
+	struct mlx5_core_dev *dev = esw->dev;
+	struct mlx5_esw_offload *offloads;
+	struct mlx5_eswitch_rep *rep;
+	u8 hw_id[ETH_ALEN];
+	int vport;
+
+	esw->offloads.vport_reps = kcalloc(total_vfs,
+					   sizeof(struct mlx5_eswitch_rep),
+					   GFP_KERNEL);
+	if (!esw->offloads.vport_reps)
+		return -ENOMEM;
+
+	offloads = &esw->offloads;
+	mlx5_query_nic_vport_mac_address(dev, 0, hw_id);
+
+	for (vport = 0; vport < total_vfs; vport++) {
+		rep = &offloads->vport_reps[vport];
+
+		rep->vport = vport;
+		ether_addr_copy(rep->hw_id, hw_id);
+	}
+
+	offloads->vport_reps[0].vport = FDB_UPLINK_VPORT;
+
+	return 0;
+}
+
+static void esw_offloads_unload_reps_type(struct mlx5_eswitch *esw, int nvports,
+					  u8 rep_type)
+{
+	struct mlx5_eswitch_rep *rep;
+	int vport;
+
+	for (vport = nvports - 1; vport >= 0; vport--) {
+		rep = &esw->offloads.vport_reps[vport];
+		if (!rep->rep_if[rep_type].valid)
+			continue;
+
+		rep->rep_if[rep_type].unload(rep);
+	}
+}
+
+static void esw_offloads_unload_reps(struct mlx5_eswitch *esw, int nvports)
+{
+	u8 rep_type = NUM_REP_TYPES;
+
+	while (rep_type-- > 0)
+		esw_offloads_unload_reps_type(esw, nvports, rep_type);
+}
+
+static int esw_offloads_load_reps_type(struct mlx5_eswitch *esw, int nvports,
+				       u8 rep_type)
 {
 	struct mlx5_eswitch_rep *rep;
 	int vport;
 	int err;
 
+	for (vport = 0; vport < nvports; vport++) {
+		rep = &esw->offloads.vport_reps[vport];
+		if (!rep->rep_if[rep_type].valid)
+			continue;
+
+		err = rep->rep_if[rep_type].load(esw->dev, rep);
+		if (err)
+			goto err_reps;
+	}
+
+	return 0;
+
+err_reps:
+	esw_offloads_unload_reps_type(esw, vport, rep_type);
+	return err;
+}
+
+static int esw_offloads_load_reps(struct mlx5_eswitch *esw, int nvports)
+{
+	u8 rep_type = 0;
+	int err;
+
+	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
+		err = esw_offloads_load_reps_type(esw, nvports, rep_type);
+		if (err)
+			goto err_reps;
+	}
+
+	return err;
+
+err_reps:
+	while (rep_type-- > 0)
+		esw_offloads_unload_reps_type(esw, nvports, rep_type);
+	return err;
+}
+
+int esw_offloads_init(struct mlx5_eswitch *esw, int nvports)
+{
+	int err;
+
 	/* disable PF RoCE so missed packets don't go through RoCE steering */
 	mlx5_dev_list_lock();
 	mlx5_remove_dev_by_protocol(esw->dev, MLX5_INTERFACE_PROTOCOL_IB);
@@ -755,25 +806,13 @@ int esw_offloads_init(struct mlx5_eswitch *esw, int nvports)
 	if (err)
 		goto create_fg_err;
 
-	for (vport = 0; vport < nvports; vport++) {
-		rep = &esw->offloads.vport_reps[vport];
-		if (!rep->valid)
-			continue;
-
-		err = rep->load(esw, rep);
-		if (err)
-			goto err_reps;
-	}
+	err = esw_offloads_load_reps(esw, nvports);
+	if (err)
+		goto err_reps;
 
 	return 0;
 
 err_reps:
-	for (vport--; vport >= 0; vport--) {
-		rep = &esw->offloads.vport_reps[vport];
-		if (!rep->valid)
-			continue;
-		rep->unload(esw, rep);
-	}
 	esw_destroy_vport_rx_group(esw);
 
 create_fg_err:
@@ -814,16 +853,7 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw)
 
 void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports)
 {
-	struct mlx5_eswitch_rep *rep;
-	int vport;
-
-	for (vport = nvports - 1; vport >= 0; vport--) {
-		rep = &esw->offloads.vport_reps[vport];
-		if (!rep->valid)
-			continue;
-		rep->unload(esw, rep);
-	}
-
+	esw_offloads_unload_reps(esw, nvports);
 	esw_destroy_vport_rx_group(esw);
 	esw_destroy_offloads_table(esw);
 	esw_destroy_offloads_fdb_tables(esw);
@@ -1120,27 +1150,23 @@ int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, u8 *encap)
 
 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch *esw,
 				     int vport_index,
-				     struct mlx5_eswitch_rep *__rep)
+				     struct mlx5_eswitch_rep_if *__rep_if,
+				     u8 rep_type)
 {
 	struct mlx5_esw_offload *offloads = &esw->offloads;
-	struct mlx5_eswitch_rep *rep;
-
-	rep = &offloads->vport_reps[vport_index];
+	struct mlx5_eswitch_rep_if *rep_if;
 
-	memset(rep, 0, sizeof(*rep));
+	rep_if = &offloads->vport_reps[vport_index].rep_if[rep_type];
 
-	rep->load   = __rep->load;
-	rep->unload = __rep->unload;
-	rep->vport  = __rep->vport;
-	rep->netdev = __rep->netdev;
-	ether_addr_copy(rep->hw_id, __rep->hw_id);
+	rep_if->load   = __rep_if->load;
+	rep_if->unload = __rep_if->unload;
+	rep_if->priv = __rep_if->priv;
 
-	INIT_LIST_HEAD(&rep->vport_sqs_list);
-	rep->valid = true;
+	rep_if->valid = true;
 }
 
 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch *esw,
-				       int vport_index)
+				       int vport_index, u8 rep_type)
 {
 	struct mlx5_esw_offload *offloads = &esw->offloads;
 	struct mlx5_eswitch_rep *rep;
@@ -1148,17 +1174,17 @@ void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch *esw,
 	rep = &offloads->vport_reps[vport_index];
 
 	if (esw->mode == SRIOV_OFFLOADS && esw->vports[vport_index].enabled)
-		rep->unload(esw, rep);
+		rep->rep_if[rep_type].unload(rep);
 
-	rep->valid = false;
+	rep->rep_if[rep_type].valid = false;
 }
 
-struct net_device *mlx5_eswitch_get_uplink_netdev(struct mlx5_eswitch *esw)
+void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
 {
 #define UPLINK_REP_INDEX 0
 	struct mlx5_esw_offload *offloads = &esw->offloads;
 	struct mlx5_eswitch_rep *rep;
 
 	rep = &offloads->vport_reps[UPLINK_REP_INDEX];
-	return rep->netdev;
+	return rep->rep_if[rep_type].priv;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
index c4392f741c5f..e6175f8ac0e4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/conn.c
@@ -688,7 +688,7 @@ static inline int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn)
 	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
 	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
 	MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
-	MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM);
+	MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
 	MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn);
 	MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
 	MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
@@ -727,7 +727,7 @@ static inline int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn)
 	MLX5_SET(qpc, qpc, next_rcv_psn,
 		 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn));
 	MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
-	MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM);
+	MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
 	ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32),
 			MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32));
 	MLX5_SET(qpc, qpc, primary_address_path.udp_sport,
@@ -888,7 +888,8 @@ struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev,
 	err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index,
 				     MLX5_ROCE_VERSION_2,
 				     MLX5_ROCE_L3_TYPE_IPV6,
-				     remote_ip, remote_mac, true, 0);
+				     remote_ip, remote_mac, true, 0,
+				     MLX5_FPGA_PORT_NUM);
 	if (err) {
 		mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err);
 		ret = ERR_PTR(err);
@@ -954,7 +955,7 @@ struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev,
 	mlx5_fpga_conn_destroy_cq(conn);
 err_gid:
 	mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL,
-			       NULL, false, 0);
+			       NULL, false, 0, MLX5_FPGA_PORT_NUM);
 err_rsvd_gid:
 	mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index);
 err:
@@ -982,7 +983,7 @@ void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn)
 	mlx5_fpga_conn_destroy_cq(conn);
 
 	mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0,
-			       NULL, NULL, false, 0);
+			       NULL, NULL, false, 0, MLX5_FPGA_PORT_NUM);
 	mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index);
 	kfree(conn);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index dfaad9ecb2b8..31fc2cfac3b3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -89,6 +89,9 @@
 /* One more level for tc */
 #define KERNEL_MIN_LEVEL (KERNEL_NIC_PRIO_NUM_LEVELS + 1)
 
+#define KERNEL_NIC_TC_NUM_PRIOS  1
+#define KERNEL_NIC_TC_NUM_LEVELS 2
+
 #define ANCHOR_NUM_LEVELS 1
 #define ANCHOR_NUM_PRIOS 1
 #define ANCHOR_MIN_LEVEL (BY_PASS_MIN_LEVEL + 1)
@@ -134,7 +137,7 @@ static struct init_tree_node {
 			 ADD_NS(ADD_MULTIPLE_PRIO(ETHTOOL_NUM_PRIOS,
 						  ETHTOOL_PRIO_NUM_LEVELS))),
 		ADD_PRIO(0, KERNEL_MIN_LEVEL, 0, {},
-			 ADD_NS(ADD_MULTIPLE_PRIO(1, 1),
+			 ADD_NS(ADD_MULTIPLE_PRIO(KERNEL_NIC_TC_NUM_PRIOS, KERNEL_NIC_TC_NUM_LEVELS),
 				ADD_MULTIPLE_PRIO(KERNEL_NIC_NUM_PRIOS,
 						  KERNEL_NIC_PRIO_NUM_LEVELS))),
 		ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0,
@@ -1426,7 +1429,8 @@ static bool check_conflicting_actions(u32 action1, u32 action2)
 
 	if (xored_actions & (MLX5_FLOW_CONTEXT_ACTION_DROP  |
 			     MLX5_FLOW_CONTEXT_ACTION_ENCAP |
-			     MLX5_FLOW_CONTEXT_ACTION_DECAP))
+			     MLX5_FLOW_CONTEXT_ACTION_DECAP |
+			     MLX5_FLOW_CONTEXT_ACTION_MOD_HDR))
 		return true;
 
 	return false;
@@ -1755,8 +1759,11 @@ _mlx5_add_flow_rules(struct mlx5_flow_table *ft,
 
 	/* Collect all fgs which has a matching match_criteria */
 	err = build_match_list(&match_head, ft, spec);
-	if (err)
+	if (err) {
+		if (take_write)
+			up_write_ref_node(&ft->node);
 		return ERR_PTR(err);
+	}
 
 	if (!take_write)
 		up_read_ref_node(&ft->node);
@@ -1765,8 +1772,11 @@ _mlx5_add_flow_rules(struct mlx5_flow_table *ft,
 				      dest_num, version);
 	free_match_list(&match_head);
 	if (!IS_ERR(rule) ||
-	    (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN))
+	    (PTR_ERR(rule) != -ENOENT && PTR_ERR(rule) != -EAGAIN)) {
+		if (take_write)
+			up_write_ref_node(&ft->node);
 		return rule;
+	}
 
 	if (!take_write) {
 		nested_down_write_ref_node(&ft->node, FS_LOCK_GRANDPARENT);
@@ -2026,16 +2036,6 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
 			return &steering->fdb_root_ns->ns;
 		else
 			return NULL;
-	case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
-		if (steering->esw_egress_root_ns)
-			return &steering->esw_egress_root_ns->ns;
-		else
-			return NULL;
-	case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
-		if (steering->esw_ingress_root_ns)
-			return &steering->esw_ingress_root_ns->ns;
-		else
-			return NULL;
 	case MLX5_FLOW_NAMESPACE_SNIFFER_RX:
 		if (steering->sniffer_rx_root_ns)
 			return &steering->sniffer_rx_root_ns->ns;
@@ -2066,6 +2066,33 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL(mlx5_get_flow_namespace);
 
+struct mlx5_flow_namespace *mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev,
+							      enum mlx5_flow_namespace_type type,
+							      int vport)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+
+	if (!steering || vport >= MLX5_TOTAL_VPORTS(dev))
+		return NULL;
+
+	switch (type) {
+	case MLX5_FLOW_NAMESPACE_ESW_EGRESS:
+		if (steering->esw_egress_root_ns &&
+		    steering->esw_egress_root_ns[vport])
+			return &steering->esw_egress_root_ns[vport]->ns;
+		else
+			return NULL;
+	case MLX5_FLOW_NAMESPACE_ESW_INGRESS:
+		if (steering->esw_ingress_root_ns &&
+		    steering->esw_ingress_root_ns[vport])
+			return &steering->esw_ingress_root_ns[vport]->ns;
+		else
+			return NULL;
+	default:
+		return NULL;
+	}
+}
+
 static struct fs_prio *fs_create_prio(struct mlx5_flow_namespace *ns,
 				      unsigned int prio, int num_levels)
 {
@@ -2343,13 +2370,41 @@ static void cleanup_root_ns(struct mlx5_flow_root_namespace *root_ns)
 	clean_tree(&root_ns->ns.node);
 }
 
+static void cleanup_egress_acls_root_ns(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int i;
+
+	if (!steering->esw_egress_root_ns)
+		return;
+
+	for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++)
+		cleanup_root_ns(steering->esw_egress_root_ns[i]);
+
+	kfree(steering->esw_egress_root_ns);
+}
+
+static void cleanup_ingress_acls_root_ns(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int i;
+
+	if (!steering->esw_ingress_root_ns)
+		return;
+
+	for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++)
+		cleanup_root_ns(steering->esw_ingress_root_ns[i]);
+
+	kfree(steering->esw_ingress_root_ns);
+}
+
 void mlx5_cleanup_fs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_flow_steering *steering = dev->priv.steering;
 
 	cleanup_root_ns(steering->root_ns);
-	cleanup_root_ns(steering->esw_egress_root_ns);
-	cleanup_root_ns(steering->esw_ingress_root_ns);
+	cleanup_egress_acls_root_ns(dev);
+	cleanup_ingress_acls_root_ns(dev);
 	cleanup_root_ns(steering->fdb_root_ns);
 	cleanup_root_ns(steering->sniffer_rx_root_ns);
 	cleanup_root_ns(steering->sniffer_tx_root_ns);
@@ -2418,34 +2473,86 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering)
 	return PTR_ERR(prio);
 }
 
-static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering)
+static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
 {
 	struct fs_prio *prio;
 
-	steering->esw_egress_root_ns = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL);
-	if (!steering->esw_egress_root_ns)
+	steering->esw_egress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_EGRESS_ACL);
+	if (!steering->esw_egress_root_ns[vport])
 		return -ENOMEM;
 
 	/* create 1 prio*/
-	prio = fs_create_prio(&steering->esw_egress_root_ns->ns, 0,
-			      MLX5_TOTAL_VPORTS(steering->dev));
+	prio = fs_create_prio(&steering->esw_egress_root_ns[vport]->ns, 0, 1);
 	return PTR_ERR_OR_ZERO(prio);
 }
 
-static int init_egress_acl_root_ns(struct mlx5_flow_steering *steering)
+static int init_ingress_acl_root_ns(struct mlx5_flow_steering *steering, int vport)
 {
 	struct fs_prio *prio;
 
-	steering->esw_ingress_root_ns = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL);
-	if (!steering->esw_ingress_root_ns)
+	steering->esw_ingress_root_ns[vport] = create_root_ns(steering, FS_FT_ESW_INGRESS_ACL);
+	if (!steering->esw_ingress_root_ns[vport])
 		return -ENOMEM;
 
 	/* create 1 prio*/
-	prio = fs_create_prio(&steering->esw_ingress_root_ns->ns, 0,
-			      MLX5_TOTAL_VPORTS(steering->dev));
+	prio = fs_create_prio(&steering->esw_ingress_root_ns[vport]->ns, 0, 1);
 	return PTR_ERR_OR_ZERO(prio);
 }
 
+static int init_egress_acls_root_ns(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int err;
+	int i;
+
+	steering->esw_egress_root_ns = kcalloc(MLX5_TOTAL_VPORTS(dev),
+					       sizeof(*steering->esw_egress_root_ns),
+					       GFP_KERNEL);
+	if (!steering->esw_egress_root_ns)
+		return -ENOMEM;
+
+	for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++) {
+		err = init_egress_acl_root_ns(steering, i);
+		if (err)
+			goto cleanup_root_ns;
+	}
+
+	return 0;
+
+cleanup_root_ns:
+	for (i--; i >= 0; i--)
+		cleanup_root_ns(steering->esw_egress_root_ns[i]);
+	kfree(steering->esw_egress_root_ns);
+	return err;
+}
+
+static int init_ingress_acls_root_ns(struct mlx5_core_dev *dev)
+{
+	struct mlx5_flow_steering *steering = dev->priv.steering;
+	int err;
+	int i;
+
+	steering->esw_ingress_root_ns = kcalloc(MLX5_TOTAL_VPORTS(dev),
+						sizeof(*steering->esw_ingress_root_ns),
+						GFP_KERNEL);
+	if (!steering->esw_ingress_root_ns)
+		return -ENOMEM;
+
+	for (i = 0; i < MLX5_TOTAL_VPORTS(dev); i++) {
+		err = init_ingress_acl_root_ns(steering, i);
+		if (err)
+			goto cleanup_root_ns;
+	}
+
+	return 0;
+
+cleanup_root_ns:
+	for (i--; i >= 0; i--)
+		cleanup_root_ns(steering->esw_ingress_root_ns[i]);
+	kfree(steering->esw_ingress_root_ns);
+	return err;
+}
+
 int mlx5_init_fs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_flow_steering *steering;
@@ -2488,12 +2595,12 @@ int mlx5_init_fs(struct mlx5_core_dev *dev)
 				goto err;
 		}
 		if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) {
-			err = init_egress_acl_root_ns(steering);
+			err = init_egress_acls_root_ns(dev);
 			if (err)
 				goto err;
 		}
 		if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) {
-			err = init_ingress_acl_root_ns(steering);
+			err = init_ingress_acls_root_ns(dev);
 			if (err)
 				goto err;
 		}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
index 397d24a621a4..05262708f14b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.h
@@ -71,8 +71,8 @@ struct mlx5_flow_steering {
 	struct kmem_cache               *ftes_cache;
 	struct mlx5_flow_root_namespace *root_ns;
 	struct mlx5_flow_root_namespace *fdb_root_ns;
-	struct mlx5_flow_root_namespace *esw_egress_root_ns;
-	struct mlx5_flow_root_namespace *esw_ingress_root_ns;
+	struct mlx5_flow_root_namespace **esw_egress_root_ns;
+	struct mlx5_flow_root_namespace **esw_ingress_root_ns;
 	struct mlx5_flow_root_namespace	*sniffer_tx_root_ns;
 	struct mlx5_flow_root_namespace	*sniffer_rx_root_ns;
 };
@@ -233,6 +233,8 @@ void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev,
 			      unsigned long delay);
 void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
 				      unsigned long interval);
+int mlx5_fc_query(struct mlx5_core_dev *dev, u16 id,
+		  u64 *packets, u64 *bytes);
 
 int mlx5_init_fs(struct mlx5_core_dev *dev);
 void mlx5_cleanup_fs(struct mlx5_core_dev *dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
index 89d1f8650033..b7ab929d5f8e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c
@@ -312,6 +312,12 @@ void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev)
 	}
 }
 
+int mlx5_fc_query(struct mlx5_core_dev *dev, u16 id,
+		  u64 *packets, u64 *bytes)
+{
+	return mlx5_cmd_fc_query(dev, id, packets, bytes);
+}
+
 void mlx5_fc_query_cached(struct mlx5_fc *counter,
 			  u64 *bytes, u64 *packets, u64 *lastuse)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index 5ef1b56b6a96..9d11e92fb541 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -195,12 +195,20 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
 	return 0;
 }
 
-int mlx5_cmd_init_hca(struct mlx5_core_dev *dev)
+int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id)
 {
 	u32 out[MLX5_ST_SZ_DW(init_hca_out)] = {0};
 	u32 in[MLX5_ST_SZ_DW(init_hca_in)]   = {0};
+	int i;
 
 	MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA);
+
+	if (MLX5_CAP_GEN(dev, sw_owner_id)) {
+		for (i = 0; i < 4; i++)
+			MLX5_ARRAY_SET(init_hca_in, in, sw_owner_id, i,
+				       sw_owner_id[i]);
+	}
+
 	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index 21d29f7936f6..d39b0b7011b2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -124,7 +124,7 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
 		trigger_cmd_completions(dev);
 	}
 
-	mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0);
+	mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 1);
 	mlx5_core_err(dev, "end\n");
 
 unlock:
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
index 6f338a9219c8..90cb50fe17fd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
@@ -254,4 +254,5 @@ const struct ethtool_ops mlx5i_ethtool_ops = {
 const struct ethtool_ops mlx5i_pkey_ethtool_ops = {
 	.get_drvinfo        = mlx5i_get_drvinfo,
 	.get_link           = ethtool_op_get_link,
+	.get_ts_info        = mlx5i_get_ts_info,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
index ee2f378c5030..f953378bd13d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
@@ -41,7 +41,6 @@
 static int mlx5i_open(struct net_device *netdev);
 static int mlx5i_close(struct net_device *netdev);
 static int mlx5i_change_mtu(struct net_device *netdev, int new_mtu);
-static int mlx5i_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd);
 
 static const struct net_device_ops mlx5i_netdev_ops = {
 	.ndo_open                = mlx5i_open,
@@ -189,7 +188,7 @@ int mlx5i_create_underlay_qp(struct mlx5_core_dev *mdev, struct mlx5_core_qp *qp
 		 MLX5_QP_ENHANCED_ULP_STATELESS_MODE);
 
 	addr_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
-	MLX5_SET(ads, addr_path, port, 1);
+	MLX5_SET(ads, addr_path, vhca_port_num, 1);
 	MLX5_SET(ads, addr_path, grh, 1);
 
 	ret = mlx5_core_create_qp(mdev, qp, in, inlen);
@@ -242,7 +241,8 @@ static void mlx5i_cleanup_tx(struct mlx5e_priv *priv)
 
 static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
 {
-	int err;
+	struct ttc_params ttc_params = {};
+	int tt, err;
 
 	priv->fs.ns = mlx5_get_flow_namespace(priv->mdev,
 					       MLX5_FLOW_NAMESPACE_KERNEL);
@@ -257,14 +257,23 @@ static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
 		priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
 	}
 
-	err = mlx5e_create_inner_ttc_table(priv);
+	mlx5e_set_ttc_basic_params(priv, &ttc_params);
+	mlx5e_set_inner_ttc_ft_params(&ttc_params);
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
+		ttc_params.indir_tirn[tt] = priv->inner_indir_tir[tt].tirn;
+
+	err = mlx5e_create_inner_ttc_table(priv, &ttc_params, &priv->fs.inner_ttc);
 	if (err) {
 		netdev_err(priv->netdev, "Failed to create inner ttc table, err=%d\n",
 			   err);
 		goto err_destroy_arfs_tables;
 	}
 
-	err = mlx5e_create_ttc_table(priv);
+	mlx5e_set_ttc_ft_params(&ttc_params);
+	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
+		ttc_params.indir_tirn[tt] = priv->indir_tir[tt].tirn;
+
+	err = mlx5e_create_ttc_table(priv, &ttc_params, &priv->fs.ttc);
 	if (err) {
 		netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n",
 			   err);
@@ -274,7 +283,7 @@ static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
 	return 0;
 
 err_destroy_inner_ttc_table:
-	mlx5e_destroy_inner_ttc_table(priv);
+	mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
 err_destroy_arfs_tables:
 	mlx5e_arfs_destroy_tables(priv);
 
@@ -283,8 +292,8 @@ static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
 
 static void mlx5i_destroy_flow_steering(struct mlx5e_priv *priv)
 {
-	mlx5e_destroy_ttc_table(priv);
-	mlx5e_destroy_inner_ttc_table(priv);
+	mlx5e_destroy_ttc_table(priv, &priv->fs.ttc);
+	mlx5e_destroy_inner_ttc_table(priv, &priv->fs.inner_ttc);
 	mlx5e_arfs_destroy_tables(priv);
 }
 
@@ -398,7 +407,7 @@ int mlx5i_dev_init(struct net_device *dev)
 	return 0;
 }
 
-static int mlx5i_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+int mlx5i_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 {
 	struct mlx5e_priv *priv = mlx5i_epriv(dev);
 
@@ -486,7 +495,7 @@ static int mlx5i_close(struct net_device *netdev)
 	mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qp.qpn);
 	mlx5i_uninit_underlay_qp(epriv);
 	mlx5e_deactivate_priv_channels(epriv);
-	mlx5e_close_channels(&epriv->channels);;
+	mlx5e_close_channels(&epriv->channels);
 unlock:
 	mutex_unlock(&epriv->state_lock);
 	return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h
index 49008022c306..6d9053bcbe95 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.h
@@ -76,9 +76,10 @@ int mlx5i_pkey_del_qpn(struct net_device *netdev, u32 qpn);
 /* Get the net-device corresponding to the given underlay QPN */
 struct net_device *mlx5i_pkey_get_netdev(struct net_device *netdev, u32 qpn);
 
-/* Shared ndo functionts */
+/* Shared ndo functions */
 int mlx5i_dev_init(struct net_device *dev);
 void mlx5i_dev_cleanup(struct net_device *dev);
+int mlx5i_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd);
 
 /* Parent profile functions */
 void mlx5i_init(struct mlx5_core_dev *mdev,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
index 531b02cc979b..b69e9d847a6b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
@@ -140,6 +140,7 @@ static int mlx5i_pkey_close(struct net_device *netdev);
 static int mlx5i_pkey_dev_init(struct net_device *dev);
 static void mlx5i_pkey_dev_cleanup(struct net_device *netdev);
 static int mlx5i_pkey_change_mtu(struct net_device *netdev, int new_mtu);
+static int mlx5i_pkey_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd);
 
 static const struct net_device_ops mlx5i_pkey_netdev_ops = {
 	.ndo_open                = mlx5i_pkey_open,
@@ -147,6 +148,7 @@ static const struct net_device_ops mlx5i_pkey_netdev_ops = {
 	.ndo_init                = mlx5i_pkey_dev_init,
 	.ndo_uninit              = mlx5i_pkey_dev_cleanup,
 	.ndo_change_mtu          = mlx5i_pkey_change_mtu,
+	.ndo_do_ioctl            = mlx5i_pkey_ioctl,
 };
 
 /* Child NDOs */
@@ -174,6 +176,11 @@ static int mlx5i_pkey_dev_init(struct net_device *dev)
 	return mlx5i_dev_init(dev);
 }
 
+static int mlx5i_pkey_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+{
+	return mlx5i_ioctl(dev, ifr, cmd);
+}
+
 static void mlx5i_pkey_dev_cleanup(struct net_device *netdev)
 {
 	return mlx5i_dev_cleanup(netdev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index 5701f125e99c..857035583ccd 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -31,7 +31,10 @@
  */
 
 #include <linux/clocksource.h>
+#include <linux/highmem.h>
+#include <rdma/mlx5-abi.h>
 #include "en.h"
+#include "clock.h"
 
 enum {
 	MLX5_CYCLES_SHIFT	= 23
@@ -71,6 +74,28 @@ static u64 read_internal_timer(const struct cyclecounter *cc)
 	return mlx5_read_internal_timer(mdev) & cc->mask;
 }
 
+static void mlx5_update_clock_info_page(struct mlx5_core_dev *mdev)
+{
+	struct mlx5_ib_clock_info *clock_info = mdev->clock_info;
+	struct mlx5_clock *clock = &mdev->clock;
+	u32 sign;
+
+	if (!clock_info)
+		return;
+
+	sign = smp_load_acquire(&clock_info->sign);
+	smp_store_mb(clock_info->sign,
+		     sign | MLX5_IB_CLOCK_INFO_KERNEL_UPDATING);
+
+	clock_info->cycles = clock->tc.cycle_last;
+	clock_info->mult   = clock->cycles.mult;
+	clock_info->nsec   = clock->tc.nsec;
+	clock_info->frac   = clock->tc.frac;
+
+	smp_store_release(&clock_info->sign,
+			  sign + MLX5_IB_CLOCK_INFO_KERNEL_UPDATING * 2);
+}
+
 static void mlx5_pps_out(struct work_struct *work)
 {
 	struct mlx5_pps *pps_info = container_of(work, struct mlx5_pps,
@@ -109,6 +134,7 @@ static void mlx5_timestamp_overflow(struct work_struct *work)
 
 	write_lock_irqsave(&clock->lock, flags);
 	timecounter_read(&clock->tc);
+	mlx5_update_clock_info_page(clock->mdev);
 	write_unlock_irqrestore(&clock->lock, flags);
 	schedule_delayed_work(&clock->overflow_work, clock->overflow_period);
 }
@@ -123,6 +149,7 @@ static int mlx5_ptp_settime(struct ptp_clock_info *ptp,
 
 	write_lock_irqsave(&clock->lock, flags);
 	timecounter_init(&clock->tc, &clock->cycles, ns);
+	mlx5_update_clock_info_page(clock->mdev);
 	write_unlock_irqrestore(&clock->lock, flags);
 
 	return 0;
@@ -152,6 +179,7 @@ static int mlx5_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 
 	write_lock_irqsave(&clock->lock, flags);
 	timecounter_adjtime(&clock->tc, delta);
+	mlx5_update_clock_info_page(clock->mdev);
 	write_unlock_irqrestore(&clock->lock, flags);
 
 	return 0;
@@ -179,6 +207,7 @@ static int mlx5_ptp_adjfreq(struct ptp_clock_info *ptp, s32 delta)
 	timecounter_read(&clock->tc);
 	clock->cycles.mult = neg_adj ? clock->nominal_c_mult - diff :
 				       clock->nominal_c_mult + diff;
+	mlx5_update_clock_info_page(clock->mdev);
 	write_unlock_irqrestore(&clock->lock, flags);
 
 	return 0;
@@ -474,6 +503,7 @@ void mlx5_init_clock(struct mlx5_core_dev *mdev)
 						  clock->cycles.shift);
 	clock->nominal_c_mult = clock->cycles.mult;
 	clock->cycles.mask = CLOCKSOURCE_MASK(41);
+	clock->mdev = mdev;
 
 	timecounter_init(&clock->tc, &clock->cycles,
 			 ktime_to_ns(ktime_get_real()));
@@ -486,6 +516,25 @@ void mlx5_init_clock(struct mlx5_core_dev *mdev)
 	do_div(ns, NSEC_PER_SEC / 2 / HZ);
 	clock->overflow_period = ns;
 
+	mdev->clock_info_page = alloc_page(GFP_KERNEL);
+	if (mdev->clock_info_page) {
+		mdev->clock_info = kmap(mdev->clock_info_page);
+		if (!mdev->clock_info) {
+			__free_page(mdev->clock_info_page);
+			mlx5_core_warn(mdev, "failed to map clock page\n");
+		} else {
+			mdev->clock_info->sign   = 0;
+			mdev->clock_info->nsec   = clock->tc.nsec;
+			mdev->clock_info->cycles = clock->tc.cycle_last;
+			mdev->clock_info->mask   = clock->cycles.mask;
+			mdev->clock_info->mult   = clock->nominal_c_mult;
+			mdev->clock_info->shift  = clock->cycles.shift;
+			mdev->clock_info->frac   = clock->tc.frac;
+			mdev->clock_info->overflow_period =
+						clock->overflow_period;
+		}
+	}
+
 	INIT_WORK(&clock->pps_info.out_work, mlx5_pps_out);
 	INIT_DELAYED_WORK(&clock->overflow_work, mlx5_timestamp_overflow);
 	if (clock->overflow_period)
@@ -525,5 +574,12 @@ void mlx5_cleanup_clock(struct mlx5_core_dev *mdev)
 
 	cancel_work_sync(&clock->pps_info.out_work);
 	cancel_delayed_work_sync(&clock->overflow_work);
+
+	if (mdev->clock_info) {
+		kunmap(mdev->clock_info_page);
+		__free_page(mdev->clock_info_page);
+		mdev->clock_info = NULL;
+	}
+
 	kfree(clock->ptp_info.pin_config);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/gid.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/gid.c
index 573f59f46d41..7722a3f9bb68 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/gid.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/gid.c
@@ -121,7 +121,7 @@ EXPORT_SYMBOL_GPL(mlx5_core_reserved_gids_count);
 
 int mlx5_core_roce_gid_set(struct mlx5_core_dev *dev, unsigned int index,
 			   u8 roce_version, u8 roce_l3_type, const u8 *gid,
-			   const u8 *mac, bool vlan, u16 vlan_id)
+			   const u8 *mac, bool vlan, u16 vlan_id, u8 port_num)
 {
 #define MLX5_SET_RA(p, f, v) MLX5_SET(roce_addr_layout, p, f, v)
 	u32  in[MLX5_ST_SZ_DW(set_roce_address_in)] = {0};
@@ -148,6 +148,9 @@ int mlx5_core_roce_gid_set(struct mlx5_core_dev *dev, unsigned int index,
 		memcpy(addr_l3_addr, gid, gidsz);
 	}
 
+	if (MLX5_CAP_GEN(dev, num_vhca_ports) > 0)
+		MLX5_SET(set_roce_address_in, in, vhca_port_num, port_num);
+
 	MLX5_SET(set_roce_address_in, in, roce_address_index, index);
 	MLX5_SET(set_roce_address_in, in, opcode, MLX5_CMD_OP_SET_ROCE_ADDRESS);
 	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 0f88fd30a09a..ae391e4b7070 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -75,6 +75,8 @@ static unsigned int prof_sel = MLX5_DEFAULT_PROF;
 module_param_named(prof_sel, prof_sel, uint, 0444);
 MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
 
+static u32 sw_owner_id[4];
+
 enum {
 	MLX5_ATOMIC_REQ_MODE_BE = 0x0,
 	MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
@@ -549,7 +551,16 @@ static int handle_hca_cap(struct mlx5_core_dev *dev)
 		MLX5_SET(cmd_hca_cap,
 			 set_hca_cap,
 			 cache_line_128byte,
-			 cache_line_size() == 128 ? 1 : 0);
+			 cache_line_size() >= 128 ? 1 : 0);
+
+	if (MLX5_CAP_GEN_MAX(dev, dct))
+		MLX5_SET(cmd_hca_cap, set_hca_cap, dct, 1);
+
+	if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports))
+		MLX5_SET(cmd_hca_cap,
+			 set_hca_cap,
+			 num_vhca_ports,
+			 MLX5_CAP_GEN_MAX(dev, num_vhca_ports));
 
 	err = set_caps(dev, set_ctx, set_sz,
 		       MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
@@ -1107,7 +1118,7 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
 		goto reclaim_boot_pages;
 	}
 
-	err = mlx5_cmd_init_hca(dev);
+	err = mlx5_cmd_init_hca(dev, sw_owner_id);
 	if (err) {
 		dev_err(&pdev->dev, "init hca failed\n");
 		goto err_pagealloc_stop;
@@ -1643,6 +1654,8 @@ static int __init init(void)
 {
 	int err;
 
+	get_random_bytes(&sw_owner_id, sizeof(sw_owner_id));
+
 	mlx5_core_verify_params();
 	mlx5_register_debugfs();
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index ff4a0b889a6f..394552f36fcf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -86,7 +86,7 @@ enum {
 
 int mlx5_query_hca_caps(struct mlx5_core_dev *dev);
 int mlx5_query_board_id(struct mlx5_core_dev *dev);
-int mlx5_cmd_init_hca(struct mlx5_core_dev *dev);
+int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id);
 int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev);
 int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev);
 void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
@@ -116,6 +116,7 @@ int mlx5_destroy_scheduling_element_cmd(struct mlx5_core_dev *dev, u8 hierarchy,
 int mlx5_wait_for_vf_pages(struct mlx5_core_dev *dev);
 u64 mlx5_read_internal_timer(struct mlx5_core_dev *dev);
 struct mlx5_eq *mlx5_eqn2eq(struct mlx5_core_dev *dev, int eqn);
+u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq *eq);
 void mlx5_cq_tasklet_cb(unsigned long data);
 
 int mlx5_query_pcam_reg(struct mlx5_core_dev *dev, u32 *pcam, u8 feature_group,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/qp.c b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
index 889130edb715..02d6c5b5d502 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/qp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/qp.c
@@ -98,6 +98,11 @@ static u64 sq_allowed_event_types(void)
 	return BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);
 }
 
+static u64 dct_allowed_event_types(void)
+{
+	return BIT(MLX5_EVENT_TYPE_DCT_DRAINED);
+}
+
 static bool is_event_type_allowed(int rsc_type, int event_type)
 {
 	switch (rsc_type) {
@@ -107,6 +112,8 @@ static bool is_event_type_allowed(int rsc_type, int event_type)
 		return BIT(event_type) & rq_allowed_event_types();
 	case MLX5_EVENT_QUEUE_TYPE_SQ:
 		return BIT(event_type) & sq_allowed_event_types();
+	case MLX5_EVENT_QUEUE_TYPE_DCT:
+		return BIT(event_type) & dct_allowed_event_types();
 	default:
 		WARN(1, "Event arrived for unknown resource type");
 		return false;
@@ -116,6 +123,7 @@ static bool is_event_type_allowed(int rsc_type, int event_type)
 void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type)
 {
 	struct mlx5_core_rsc_common *common = mlx5_get_rsc(dev, rsn);
+	struct mlx5_core_dct *dct;
 	struct mlx5_core_qp *qp;
 
 	if (!common)
@@ -134,7 +142,11 @@ void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type)
 		qp = (struct mlx5_core_qp *)common;
 		qp->event(qp, event_type);
 		break;
-
+	case MLX5_RES_DCT:
+		dct = (struct mlx5_core_dct *)common;
+		if (event_type == MLX5_EVENT_TYPE_DCT_DRAINED)
+			complete(&dct->drained);
+		break;
 	default:
 		mlx5_core_warn(dev, "invalid resource type for 0x%x\n", rsn);
 	}
@@ -142,9 +154,9 @@ void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type)
 	mlx5_core_put_rsc(common);
 }
 
-static int create_qprqsq_common(struct mlx5_core_dev *dev,
-				struct mlx5_core_qp *qp,
-				int rsc_type)
+static int create_resource_common(struct mlx5_core_dev *dev,
+				  struct mlx5_core_qp *qp,
+				  int rsc_type)
 {
 	struct mlx5_qp_table *table = &dev->priv.qp_table;
 	int err;
@@ -165,8 +177,8 @@ static int create_qprqsq_common(struct mlx5_core_dev *dev,
 	return 0;
 }
 
-static void destroy_qprqsq_common(struct mlx5_core_dev *dev,
-				  struct mlx5_core_qp *qp)
+static void destroy_resource_common(struct mlx5_core_dev *dev,
+				    struct mlx5_core_qp *qp)
 {
 	struct mlx5_qp_table *table = &dev->priv.qp_table;
 	unsigned long flags;
@@ -179,6 +191,40 @@ static void destroy_qprqsq_common(struct mlx5_core_dev *dev,
 	wait_for_completion(&qp->common.free);
 }
 
+int mlx5_core_create_dct(struct mlx5_core_dev *dev,
+			 struct mlx5_core_dct *dct,
+			 u32 *in, int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(create_dct_out)]   = {0};
+	u32 din[MLX5_ST_SZ_DW(destroy_dct_in)]   = {0};
+	u32 dout[MLX5_ST_SZ_DW(destroy_dct_out)] = {0};
+	struct mlx5_core_qp *qp = &dct->mqp;
+	int err;
+
+	init_completion(&dct->drained);
+	MLX5_SET(create_dct_in, in, opcode, MLX5_CMD_OP_CREATE_DCT);
+
+	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
+	if (err) {
+		mlx5_core_warn(dev, "create DCT failed, ret %d\n", err);
+		return err;
+	}
+
+	qp->qpn = MLX5_GET(create_dct_out, out, dctn);
+	err = create_resource_common(dev, qp, MLX5_RES_DCT);
+	if (err)
+		goto err_cmd;
+
+	return 0;
+err_cmd:
+	MLX5_SET(destroy_dct_in, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
+	MLX5_SET(destroy_dct_in, din, dctn, qp->qpn);
+	mlx5_cmd_exec(dev, (void *)&in, sizeof(din),
+		      (void *)&out, sizeof(dout));
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_create_dct);
+
 int mlx5_core_create_qp(struct mlx5_core_dev *dev,
 			struct mlx5_core_qp *qp,
 			u32 *in, int inlen)
@@ -197,7 +243,7 @@ int mlx5_core_create_qp(struct mlx5_core_dev *dev,
 	qp->qpn = MLX5_GET(create_qp_out, out, qpn);
 	mlx5_core_dbg(dev, "qpn = 0x%x\n", qp->qpn);
 
-	err = create_qprqsq_common(dev, qp, MLX5_RES_QP);
+	err = create_resource_common(dev, qp, MLX5_RES_QP);
 	if (err)
 		goto err_cmd;
 
@@ -220,6 +266,47 @@ int mlx5_core_create_qp(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL_GPL(mlx5_core_create_qp);
 
+static int mlx5_core_drain_dct(struct mlx5_core_dev *dev,
+			       struct mlx5_core_dct *dct)
+{
+	u32 out[MLX5_ST_SZ_DW(drain_dct_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(drain_dct_in)]   = {0};
+	struct mlx5_core_qp *qp = &dct->mqp;
+
+	MLX5_SET(drain_dct_in, in, opcode, MLX5_CMD_OP_DRAIN_DCT);
+	MLX5_SET(drain_dct_in, in, dctn, qp->qpn);
+	return mlx5_cmd_exec(dev, (void *)&in, sizeof(in),
+			     (void *)&out, sizeof(out));
+}
+
+int mlx5_core_destroy_dct(struct mlx5_core_dev *dev,
+			  struct mlx5_core_dct *dct)
+{
+	u32 out[MLX5_ST_SZ_DW(destroy_dct_out)] = {0};
+	u32 in[MLX5_ST_SZ_DW(destroy_dct_in)]   = {0};
+	struct mlx5_core_qp *qp = &dct->mqp;
+	int err;
+
+	err = mlx5_core_drain_dct(dev, dct);
+	if (err) {
+		if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
+			goto destroy;
+		} else {
+			mlx5_core_warn(dev, "failed drain DCT 0x%x with error 0x%x\n", qp->qpn, err);
+			return err;
+		}
+	}
+	wait_for_completion(&dct->drained);
+destroy:
+	destroy_resource_common(dev, &dct->mqp);
+	MLX5_SET(destroy_dct_in, in, opcode, MLX5_CMD_OP_DESTROY_DCT);
+	MLX5_SET(destroy_dct_in, in, dctn, qp->qpn);
+	err = mlx5_cmd_exec(dev, (void *)&in, sizeof(in),
+			    (void *)&out, sizeof(out));
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_destroy_dct);
+
 int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
 			 struct mlx5_core_qp *qp)
 {
@@ -229,7 +316,7 @@ int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
 
 	mlx5_debug_qp_remove(dev, qp);
 
-	destroy_qprqsq_common(dev, qp);
+	destroy_resource_common(dev, qp);
 
 	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
 	MLX5_SET(destroy_qp_in, in, qpn, qp->qpn);
@@ -405,6 +492,20 @@ int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
 }
 EXPORT_SYMBOL_GPL(mlx5_core_qp_query);
 
+int mlx5_core_dct_query(struct mlx5_core_dev *dev, struct mlx5_core_dct *dct,
+			u32 *out, int outlen)
+{
+	u32 in[MLX5_ST_SZ_DW(query_dct_in)] = {0};
+	struct mlx5_core_qp *qp = &dct->mqp;
+
+	MLX5_SET(query_dct_in, in, opcode, MLX5_CMD_OP_QUERY_DCT);
+	MLX5_SET(query_dct_in, in, dctn, qp->qpn);
+
+	return mlx5_cmd_exec(dev, (void *)&in, sizeof(in),
+			     (void *)out, outlen);
+}
+EXPORT_SYMBOL_GPL(mlx5_core_dct_query);
+
 int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn)
 {
 	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)] = {0};
@@ -441,7 +542,7 @@ int mlx5_core_create_rq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
 		return err;
 
 	rq->qpn = rqn;
-	err = create_qprqsq_common(dev, rq, MLX5_RES_RQ);
+	err = create_resource_common(dev, rq, MLX5_RES_RQ);
 	if (err)
 		goto err_destroy_rq;
 
@@ -457,7 +558,7 @@ EXPORT_SYMBOL(mlx5_core_create_rq_tracked);
 void mlx5_core_destroy_rq_tracked(struct mlx5_core_dev *dev,
 				  struct mlx5_core_qp *rq)
 {
-	destroy_qprqsq_common(dev, rq);
+	destroy_resource_common(dev, rq);
 	mlx5_core_destroy_rq(dev, rq->qpn);
 }
 EXPORT_SYMBOL(mlx5_core_destroy_rq_tracked);
@@ -473,7 +574,7 @@ int mlx5_core_create_sq_tracked(struct mlx5_core_dev *dev, u32 *in, int inlen,
 		return err;
 
 	sq->qpn = sqn;
-	err = create_qprqsq_common(dev, sq, MLX5_RES_SQ);
+	err = create_resource_common(dev, sq, MLX5_RES_SQ);
 	if (err)
 		goto err_destroy_sq;
 
@@ -489,7 +590,7 @@ EXPORT_SYMBOL(mlx5_core_create_sq_tracked);
 void mlx5_core_destroy_sq_tracked(struct mlx5_core_dev *dev,
 				  struct mlx5_core_qp *sq)
 {
-	destroy_qprqsq_common(dev, sq);
+	destroy_resource_common(dev, sq);
 	mlx5_core_destroy_sq(dev, sq->qpn);
 }
 EXPORT_SYMBOL(mlx5_core_destroy_sq_tracked);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
index 5e128d7a9ffd..9e38343a951f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/transobj.c
@@ -398,3 +398,217 @@ void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
 	mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 }
 EXPORT_SYMBOL(mlx5_core_destroy_rqt);
+
+static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
+				  struct mlx5_hairpin_params *params, u32 *rqn)
+{
+	u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
+	void *rqc, *wq;
+
+	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
+	wq  = MLX5_ADDR_OF(rqc, rqc, wq);
+
+	MLX5_SET(rqc, rqc, hairpin, 1);
+	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
+	MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);
+
+	MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
+	MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
+
+	return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
+}
+
+static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
+				  struct mlx5_hairpin_params *params, u32 *sqn)
+{
+	u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
+	void *sqc, *wq;
+
+	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
+	wq  = MLX5_ADDR_OF(sqc, sqc, wq);
+
+	MLX5_SET(sqc, sqc, hairpin, 1);
+	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
+
+	MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
+	MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
+
+	return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
+}
+
+static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
+				      struct mlx5_hairpin_params *params)
+{
+	int i, j, err;
+
+	for (i = 0; i < hp->num_channels; i++) {
+		err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
+		if (err)
+			goto out_err_rq;
+	}
+
+	for (i = 0; i < hp->num_channels; i++) {
+		err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
+		if (err)
+			goto out_err_sq;
+	}
+
+	return 0;
+
+out_err_sq:
+	for (j = 0; j < i; j++)
+		mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
+	i = hp->num_channels;
+out_err_rq:
+	for (j = 0; j < i; j++)
+		mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
+	return err;
+}
+
+static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
+{
+	int i;
+
+	for (i = 0; i < hp->num_channels; i++) {
+		mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
+		mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
+	}
+}
+
+static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
+				  int curr_state, int next_state,
+				  u16 peer_vhca, u32 peer_sq)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {0};
+	void *rqc;
+
+	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
+
+	if (next_state == MLX5_RQC_STATE_RDY) {
+		MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
+		MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
+	}
+
+	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
+	MLX5_SET(rqc, rqc, state, next_state);
+
+	return mlx5_core_modify_rq(func_mdev, rqn,
+				   in, MLX5_ST_SZ_BYTES(modify_rq_in));
+}
+
+static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
+				  int curr_state, int next_state,
+				  u16 peer_vhca, u32 peer_rq)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
+	void *sqc;
+
+	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
+
+	if (next_state == MLX5_RQC_STATE_RDY) {
+		MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
+		MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
+	}
+
+	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
+	MLX5_SET(sqc, sqc, state, next_state);
+
+	return mlx5_core_modify_sq(peer_mdev, sqn,
+				   in, MLX5_ST_SZ_BYTES(modify_sq_in));
+}
+
+static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
+{
+	int i, j, err;
+
+	/* set peer SQs */
+	for (i = 0; i < hp->num_channels; i++) {
+		err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
+					     MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
+					     MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
+		if (err)
+			goto err_modify_sq;
+	}
+
+	/* set func RQs */
+	for (i = 0; i < hp->num_channels; i++) {
+		err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
+					     MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
+					     MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
+		if (err)
+			goto err_modify_rq;
+	}
+
+	return 0;
+
+err_modify_rq:
+	for (j = 0; j < i; j++)
+		mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
+				       MLX5_RQC_STATE_RST, 0, 0);
+	i = hp->num_channels;
+err_modify_sq:
+	for (j = 0; j < i; j++)
+		mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
+				       MLX5_SQC_STATE_RST, 0, 0);
+	return err;
+}
+
+static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
+{
+	int i;
+
+	/* unset func RQs */
+	for (i = 0; i < hp->num_channels; i++)
+		mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
+				       MLX5_RQC_STATE_RST, 0, 0);
+
+	/* unset peer SQs */
+	for (i = 0; i < hp->num_channels; i++)
+		mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
+				       MLX5_SQC_STATE_RST, 0, 0);
+}
+
+struct mlx5_hairpin *
+mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
+			 struct mlx5_core_dev *peer_mdev,
+			 struct mlx5_hairpin_params *params)
+{
+	struct mlx5_hairpin *hp;
+	int size, err;
+
+	size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
+	hp = kzalloc(size, GFP_KERNEL);
+	if (!hp)
+		return ERR_PTR(-ENOMEM);
+
+	hp->func_mdev = func_mdev;
+	hp->peer_mdev = peer_mdev;
+	hp->num_channels = params->num_channels;
+
+	hp->rqn = (void *)hp + sizeof(*hp);
+	hp->sqn = hp->rqn + params->num_channels;
+
+	/* alloc and pair func --> peer hairpin */
+	err = mlx5_hairpin_create_queues(hp, params);
+	if (err)
+		goto err_create_queues;
+
+	err = mlx5_hairpin_pair_queues(hp);
+	if (err)
+		goto err_pair_queues;
+
+	return hp;
+
+err_pair_queues:
+	mlx5_hairpin_destroy_queues(hp);
+err_create_queues:
+	kfree(hp);
+	return ERR_PTR(err);
+}
+
+void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
+{
+	mlx5_hairpin_unpair_queues(hp);
+	mlx5_hairpin_destroy_queues(hp);
+	kfree(hp);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
index a1296a62497d..dfe36cf6fbea 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c
@@ -36,6 +36,9 @@
 #include <linux/mlx5/vport.h>
 #include "mlx5_core.h"
 
+/* Mutex to hold while enabling or disabling RoCE */
+static DEFINE_MUTEX(mlx5_roce_en_lock);
+
 static int _mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod,
 				   u16 vport, u32 *out, int outlen)
 {
@@ -998,17 +1001,35 @@ static int mlx5_nic_vport_update_roce_state(struct mlx5_core_dev *mdev,
 
 int mlx5_nic_vport_enable_roce(struct mlx5_core_dev *mdev)
 {
-	if (atomic_inc_return(&mdev->roce.roce_en) != 1)
-		return 0;
-	return mlx5_nic_vport_update_roce_state(mdev, MLX5_VPORT_ROCE_ENABLED);
+	int err = 0;
+
+	mutex_lock(&mlx5_roce_en_lock);
+	if (!mdev->roce.roce_en)
+		err = mlx5_nic_vport_update_roce_state(mdev, MLX5_VPORT_ROCE_ENABLED);
+
+	if (!err)
+		mdev->roce.roce_en++;
+	mutex_unlock(&mlx5_roce_en_lock);
+
+	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_nic_vport_enable_roce);
 
 int mlx5_nic_vport_disable_roce(struct mlx5_core_dev *mdev)
 {
-	if (atomic_dec_return(&mdev->roce.roce_en) != 0)
-		return 0;
-	return mlx5_nic_vport_update_roce_state(mdev, MLX5_VPORT_ROCE_DISABLED);
+	int err = 0;
+
+	mutex_lock(&mlx5_roce_en_lock);
+	if (mdev->roce.roce_en) {
+		mdev->roce.roce_en--;
+		if (mdev->roce.roce_en == 0)
+			err = mlx5_nic_vport_update_roce_state(mdev, MLX5_VPORT_ROCE_DISABLED);
+
+		if (err)
+			mdev->roce.roce_en++;
+	}
+	mutex_unlock(&mlx5_roce_en_lock);
+	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_nic_vport_disable_roce);
 
@@ -1110,3 +1131,61 @@ int mlx5_core_modify_hca_vport_context(struct mlx5_core_dev *dev,
 	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_core_modify_hca_vport_context);
+
+int mlx5_nic_vport_affiliate_multiport(struct mlx5_core_dev *master_mdev,
+				       struct mlx5_core_dev *port_mdev)
+{
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+	void *in;
+	int err;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	err = mlx5_nic_vport_enable_roce(port_mdev);
+	if (err)
+		goto free;
+
+	MLX5_SET(modify_nic_vport_context_in, in, field_select.affiliation, 1);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.affiliated_vhca_id,
+		 MLX5_CAP_GEN(master_mdev, vhca_id));
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.affiliation_criteria,
+		 MLX5_CAP_GEN(port_mdev, affiliate_nic_vport_criteria));
+
+	err = mlx5_modify_nic_vport_context(port_mdev, in, inlen);
+	if (err)
+		mlx5_nic_vport_disable_roce(port_mdev);
+
+free:
+	kvfree(in);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_affiliate_multiport);
+
+int mlx5_nic_vport_unaffiliate_multiport(struct mlx5_core_dev *port_mdev)
+{
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+	void *in;
+	int err;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(modify_nic_vport_context_in, in, field_select.affiliation, 1);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.affiliated_vhca_id, 0);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.affiliation_criteria, 0);
+
+	err = mlx5_modify_nic_vport_context(port_mdev, in, inlen);
+	if (!err)
+		mlx5_nic_vport_disable_roce(port_mdev);
+
+	kvfree(in);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_unaffiliate_multiport);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c
index f3315bc874ad..3529b545675d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.c
@@ -113,6 +113,7 @@ struct mlxsw_core {
 	struct mlxsw_thermal *thermal;
 	struct mlxsw_core_port *ports;
 	unsigned int max_ports;
+	bool reload_fail;
 	unsigned long driver_priv[0];
 	/* driver_priv has to be always the last item */
 };
@@ -962,7 +963,28 @@ mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
 						     pool_type, p_cur, p_max);
 }
 
+static int mlxsw_devlink_core_bus_device_reload(struct devlink *devlink)
+{
+	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
+	const struct mlxsw_bus *mlxsw_bus = mlxsw_core->bus;
+	int err;
+
+	if (!mlxsw_bus->reset)
+		return -EOPNOTSUPP;
+
+	mlxsw_core_bus_device_unregister(mlxsw_core, true);
+	mlxsw_bus->reset(mlxsw_core->bus_priv);
+	err = mlxsw_core_bus_device_register(mlxsw_core->bus_info,
+					     mlxsw_core->bus,
+					     mlxsw_core->bus_priv, true,
+					     devlink);
+	if (err)
+		mlxsw_core->reload_fail = true;
+	return err;
+}
+
 static const struct devlink_ops mlxsw_devlink_ops = {
+	.reload				= mlxsw_devlink_core_bus_device_reload,
 	.port_type_set			= mlxsw_devlink_port_type_set,
 	.port_split			= mlxsw_devlink_port_split,
 	.port_unsplit			= mlxsw_devlink_port_unsplit,
@@ -980,23 +1002,26 @@ static const struct devlink_ops mlxsw_devlink_ops = {
 
 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 				   const struct mlxsw_bus *mlxsw_bus,
-				   void *bus_priv)
+				   void *bus_priv, bool reload,
+				   struct devlink *devlink)
 {
 	const char *device_kind = mlxsw_bus_info->device_kind;
 	struct mlxsw_core *mlxsw_core;
 	struct mlxsw_driver *mlxsw_driver;
-	struct devlink *devlink;
 	size_t alloc_size;
 	int err;
 
 	mlxsw_driver = mlxsw_core_driver_get(device_kind);
 	if (!mlxsw_driver)
 		return -EINVAL;
-	alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
-	devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size);
-	if (!devlink) {
-		err = -ENOMEM;
-		goto err_devlink_alloc;
+
+	if (!reload) {
+		alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
+		devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size);
+		if (!devlink) {
+			err = -ENOMEM;
+			goto err_devlink_alloc;
+		}
 	}
 
 	mlxsw_core = devlink_priv(devlink);
@@ -1012,6 +1037,12 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 	if (err)
 		goto err_bus_init;
 
+	if (mlxsw_driver->resources_register && !reload) {
+		err = mlxsw_driver->resources_register(mlxsw_core);
+		if (err)
+			goto err_register_resources;
+	}
+
 	err = mlxsw_ports_init(mlxsw_core);
 	if (err)
 		goto err_ports_init;
@@ -1032,9 +1063,11 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 	if (err)
 		goto err_emad_init;
 
-	err = devlink_register(devlink, mlxsw_bus_info->dev);
-	if (err)
-		goto err_devlink_register;
+	if (!reload) {
+		err = devlink_register(devlink, mlxsw_bus_info->dev);
+		if (err)
+			goto err_devlink_register;
+	}
 
 	err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon);
 	if (err)
@@ -1057,7 +1090,8 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 	mlxsw_thermal_fini(mlxsw_core->thermal);
 err_thermal_init:
 err_hwmon_init:
-	devlink_unregister(devlink);
+	if (!reload)
+		devlink_unregister(devlink);
 err_devlink_register:
 	mlxsw_emad_fini(mlxsw_core);
 err_emad_init:
@@ -1067,26 +1101,40 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 err_ports_init:
 	mlxsw_bus->fini(bus_priv);
 err_bus_init:
-	devlink_free(devlink);
+	if (!reload)
+		devlink_resources_unregister(devlink, NULL);
+err_register_resources:
+	if (!reload)
+		devlink_free(devlink);
 err_devlink_alloc:
 	mlxsw_core_driver_put(device_kind);
 	return err;
 }
 EXPORT_SYMBOL(mlxsw_core_bus_device_register);
 
-void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core)
+void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
+				      bool reload)
 {
 	const char *device_kind = mlxsw_core->bus_info->device_kind;
 	struct devlink *devlink = priv_to_devlink(mlxsw_core);
 
+	if (mlxsw_core->reload_fail)
+		goto reload_fail;
+
 	if (mlxsw_core->driver->fini)
 		mlxsw_core->driver->fini(mlxsw_core);
 	mlxsw_thermal_fini(mlxsw_core->thermal);
-	devlink_unregister(devlink);
+	if (!reload)
+		devlink_unregister(devlink);
 	mlxsw_emad_fini(mlxsw_core);
 	kfree(mlxsw_core->lag.mapping);
 	mlxsw_ports_fini(mlxsw_core);
+	if (!reload)
+		devlink_resources_unregister(devlink, NULL);
 	mlxsw_core->bus->fini(mlxsw_core->bus_priv);
+	if (reload)
+		return;
+reload_fail:
 	devlink_free(devlink);
 	mlxsw_core_driver_put(device_kind);
 }
@@ -1791,6 +1839,22 @@ void mlxsw_core_flush_owq(void)
 }
 EXPORT_SYMBOL(mlxsw_core_flush_owq);
 
+int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
+			     const struct mlxsw_config_profile *profile,
+			     u64 *p_single_size, u64 *p_double_size,
+			     u64 *p_linear_size)
+{
+	struct mlxsw_driver *driver = mlxsw_core->driver;
+
+	if (!driver->kvd_sizes_get)
+		return -EINVAL;
+
+	return driver->kvd_sizes_get(mlxsw_core, profile,
+				     p_single_size, p_double_size,
+				     p_linear_size);
+}
+EXPORT_SYMBOL(mlxsw_core_kvd_sizes_get);
+
 static int __init mlxsw_core_module_init(void)
 {
 	int err;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h
index 6e966af72fc4..5ddafd74dc00 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.h
@@ -66,8 +66,9 @@ void mlxsw_core_driver_unregister(struct mlxsw_driver *mlxsw_driver);
 
 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 				   const struct mlxsw_bus *mlxsw_bus,
-				   void *bus_priv);
-void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core);
+				   void *bus_priv, bool reload,
+				   struct devlink *devlink);
+void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, bool reload);
 
 struct mlxsw_tx_info {
 	u8 local_port;
@@ -308,10 +309,20 @@ struct mlxsw_driver {
 				       u32 *p_cur, u32 *p_max);
 	void (*txhdr_construct)(struct sk_buff *skb,
 				const struct mlxsw_tx_info *tx_info);
+	int (*resources_register)(struct mlxsw_core *mlxsw_core);
+	int (*kvd_sizes_get)(struct mlxsw_core *mlxsw_core,
+			     const struct mlxsw_config_profile *profile,
+			     u64 *p_single_size, u64 *p_double_size,
+			     u64 *p_linear_size);
 	u8 txhdr_len;
 	const struct mlxsw_config_profile *profile;
 };
 
+int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
+			     const struct mlxsw_config_profile *profile,
+			     u64 *p_single_size, u64 *p_double_size,
+			     u64 *p_linear_size);
+
 bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core,
 			  enum mlxsw_res_id res_id);
 
@@ -332,6 +343,7 @@ struct mlxsw_bus {
 		    const struct mlxsw_config_profile *profile,
 		    struct mlxsw_res *res);
 	void (*fini)(void *bus_priv);
+	void (*reset)(void *bus_priv);
 	bool (*skb_transmit_busy)(void *bus_priv,
 				  const struct mlxsw_tx_info *tx_info);
 	int (*skb_transmit)(void *bus_priv, struct sk_buff *skb,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c
index 6a979a09ab72..996dc099cd58 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c
@@ -310,9 +310,33 @@ struct mlxsw_afa_block {
 	struct mlxsw_afa_set *first_set;
 	struct mlxsw_afa_set *cur_set;
 	unsigned int cur_act_index; /* In current set. */
-	struct list_head fwd_entry_ref_list;
+	struct list_head resource_list; /* List of resources held by actions
+					 * in this block.
+					 */
 };
 
+struct mlxsw_afa_resource {
+	struct list_head list;
+	void (*destructor)(struct mlxsw_afa_block *block,
+			   struct mlxsw_afa_resource *resource);
+};
+
+static void mlxsw_afa_resource_add(struct mlxsw_afa_block *block,
+				   struct mlxsw_afa_resource *resource)
+{
+	list_add(&resource->list, &block->resource_list);
+}
+
+static void mlxsw_afa_resources_destroy(struct mlxsw_afa_block *block)
+{
+	struct mlxsw_afa_resource *resource, *tmp;
+
+	list_for_each_entry_safe(resource, tmp, &block->resource_list, list) {
+		list_del(&resource->list);
+		resource->destructor(block, resource);
+	}
+}
+
 struct mlxsw_afa_block *mlxsw_afa_block_create(struct mlxsw_afa *mlxsw_afa)
 {
 	struct mlxsw_afa_block *block;
@@ -320,7 +344,7 @@ struct mlxsw_afa_block *mlxsw_afa_block_create(struct mlxsw_afa *mlxsw_afa)
 	block = kzalloc(sizeof(*block), GFP_KERNEL);
 	if (!block)
 		return NULL;
-	INIT_LIST_HEAD(&block->fwd_entry_ref_list);
+	INIT_LIST_HEAD(&block->resource_list);
 	block->afa = mlxsw_afa;
 
 	/* At least one action set is always present, so just create it here */
@@ -336,8 +360,6 @@ struct mlxsw_afa_block *mlxsw_afa_block_create(struct mlxsw_afa *mlxsw_afa)
 }
 EXPORT_SYMBOL(mlxsw_afa_block_create);
 
-static void mlxsw_afa_fwd_entry_refs_destroy(struct mlxsw_afa_block *block);
-
 void mlxsw_afa_block_destroy(struct mlxsw_afa_block *block)
 {
 	struct mlxsw_afa_set *set = block->first_set;
@@ -348,7 +370,7 @@ void mlxsw_afa_block_destroy(struct mlxsw_afa_block *block)
 		mlxsw_afa_set_put(block->afa, set);
 		set = next_set;
 	} while (set);
-	mlxsw_afa_fwd_entry_refs_destroy(block);
+	mlxsw_afa_resources_destroy(block);
 	kfree(block);
 }
 EXPORT_SYMBOL(mlxsw_afa_block_destroy);
@@ -421,6 +443,17 @@ int mlxsw_afa_block_jump(struct mlxsw_afa_block *block, u16 group_id)
 }
 EXPORT_SYMBOL(mlxsw_afa_block_jump);
 
+int mlxsw_afa_block_terminate(struct mlxsw_afa_block *block)
+{
+	if (block->finished)
+		return -EINVAL;
+	mlxsw_afa_set_goto_set(block->cur_set,
+			       MLXSW_AFA_SET_GOTO_BINDING_CMD_TERM, 0);
+	block->finished = true;
+	return 0;
+}
+EXPORT_SYMBOL(mlxsw_afa_block_terminate);
+
 static struct mlxsw_afa_fwd_entry *
 mlxsw_afa_fwd_entry_create(struct mlxsw_afa *mlxsw_afa, u8 local_port)
 {
@@ -489,10 +522,29 @@ static void mlxsw_afa_fwd_entry_put(struct mlxsw_afa *mlxsw_afa,
 }
 
 struct mlxsw_afa_fwd_entry_ref {
-	struct list_head list;
+	struct mlxsw_afa_resource resource;
 	struct mlxsw_afa_fwd_entry *fwd_entry;
 };
 
+static void
+mlxsw_afa_fwd_entry_ref_destroy(struct mlxsw_afa_block *block,
+				struct mlxsw_afa_fwd_entry_ref *fwd_entry_ref)
+{
+	mlxsw_afa_fwd_entry_put(block->afa, fwd_entry_ref->fwd_entry);
+	kfree(fwd_entry_ref);
+}
+
+static void
+mlxsw_afa_fwd_entry_ref_destructor(struct mlxsw_afa_block *block,
+				   struct mlxsw_afa_resource *resource)
+{
+	struct mlxsw_afa_fwd_entry_ref *fwd_entry_ref;
+
+	fwd_entry_ref = container_of(resource, struct mlxsw_afa_fwd_entry_ref,
+				     resource);
+	mlxsw_afa_fwd_entry_ref_destroy(block, fwd_entry_ref);
+}
+
 static struct mlxsw_afa_fwd_entry_ref *
 mlxsw_afa_fwd_entry_ref_create(struct mlxsw_afa_block *block, u8 local_port)
 {
@@ -509,7 +561,8 @@ mlxsw_afa_fwd_entry_ref_create(struct mlxsw_afa_block *block, u8 local_port)
 		goto err_fwd_entry_get;
 	}
 	fwd_entry_ref->fwd_entry = fwd_entry;
-	list_add(&fwd_entry_ref->list, &block->fwd_entry_ref_list);
+	fwd_entry_ref->resource.destructor = mlxsw_afa_fwd_entry_ref_destructor;
+	mlxsw_afa_resource_add(block, &fwd_entry_ref->resource);
 	return fwd_entry_ref;
 
 err_fwd_entry_get:
@@ -517,23 +570,51 @@ mlxsw_afa_fwd_entry_ref_create(struct mlxsw_afa_block *block, u8 local_port)
 	return ERR_PTR(err);
 }
 
+struct mlxsw_afa_counter {
+	struct mlxsw_afa_resource resource;
+	u32 counter_index;
+};
+
 static void
-mlxsw_afa_fwd_entry_ref_destroy(struct mlxsw_afa_block *block,
-				struct mlxsw_afa_fwd_entry_ref *fwd_entry_ref)
+mlxsw_afa_counter_destroy(struct mlxsw_afa_block *block,
+			  struct mlxsw_afa_counter *counter)
 {
-	list_del(&fwd_entry_ref->list);
-	mlxsw_afa_fwd_entry_put(block->afa, fwd_entry_ref->fwd_entry);
-	kfree(fwd_entry_ref);
+	block->afa->ops->counter_index_put(block->afa->ops_priv,
+					   counter->counter_index);
+	kfree(counter);
 }
 
-static void mlxsw_afa_fwd_entry_refs_destroy(struct mlxsw_afa_block *block)
+static void
+mlxsw_afa_counter_destructor(struct mlxsw_afa_block *block,
+			     struct mlxsw_afa_resource *resource)
 {
-	struct mlxsw_afa_fwd_entry_ref *fwd_entry_ref;
-	struct mlxsw_afa_fwd_entry_ref *tmp;
+	struct mlxsw_afa_counter *counter;
+
+	counter = container_of(resource, struct mlxsw_afa_counter, resource);
+	mlxsw_afa_counter_destroy(block, counter);
+}
+
+static struct mlxsw_afa_counter *
+mlxsw_afa_counter_create(struct mlxsw_afa_block *block)
+{
+	struct mlxsw_afa_counter *counter;
+	int err;
+
+	counter = kzalloc(sizeof(*counter), GFP_KERNEL);
+	if (!counter)
+		return ERR_PTR(-ENOMEM);
+
+	err = block->afa->ops->counter_index_get(block->afa->ops_priv,
+						 &counter->counter_index);
+	if (err)
+		goto err_counter_index_get;
+	counter->resource.destructor = mlxsw_afa_counter_destructor;
+	mlxsw_afa_resource_add(block, &counter->resource);
+	return counter;
 
-	list_for_each_entry_safe(fwd_entry_ref, tmp,
-				 &block->fwd_entry_ref_list, list)
-		mlxsw_afa_fwd_entry_ref_destroy(block, fwd_entry_ref);
+err_counter_index_get:
+	kfree(counter);
+	return ERR_PTR(err);
 }
 
 #define MLXSW_AFA_ONE_ACTION_LEN 32
@@ -690,6 +771,16 @@ MLXSW_ITEM32(afa, trapdisc, forward_action, 0x00, 0, 4);
  */
 MLXSW_ITEM32(afa, trapdisc, trap_id, 0x04, 0, 9);
 
+/* afa_trapdisc_mirror_agent
+ * Mirror agent.
+ */
+MLXSW_ITEM32(afa, trapdisc, mirror_agent, 0x08, 29, 3);
+
+/* afa_trapdisc_mirror_enable
+ * Mirror enable.
+ */
+MLXSW_ITEM32(afa, trapdisc, mirror_enable, 0x08, 24, 1);
+
 static inline void
 mlxsw_afa_trapdisc_pack(char *payload,
 			enum mlxsw_afa_trapdisc_trap_action trap_action,
@@ -701,6 +792,14 @@ mlxsw_afa_trapdisc_pack(char *payload,
 	mlxsw_afa_trapdisc_trap_id_set(payload, trap_id);
 }
 
+static inline void
+mlxsw_afa_trapdisc_mirror_pack(char *payload, bool mirror_enable,
+			       u8 mirror_agent)
+{
+	mlxsw_afa_trapdisc_mirror_enable_set(payload, mirror_enable);
+	mlxsw_afa_trapdisc_mirror_agent_set(payload, mirror_agent);
+}
+
 int mlxsw_afa_block_append_drop(struct mlxsw_afa_block *block)
 {
 	char *act = mlxsw_afa_block_append_action(block,
@@ -746,6 +845,104 @@ int mlxsw_afa_block_append_trap_and_forward(struct mlxsw_afa_block *block,
 }
 EXPORT_SYMBOL(mlxsw_afa_block_append_trap_and_forward);
 
+struct mlxsw_afa_mirror {
+	struct mlxsw_afa_resource resource;
+	int span_id;
+	u8 local_in_port;
+	u8 local_out_port;
+	bool ingress;
+};
+
+static void
+mlxsw_afa_mirror_destroy(struct mlxsw_afa_block *block,
+			 struct mlxsw_afa_mirror *mirror)
+{
+	block->afa->ops->mirror_del(block->afa->ops_priv,
+				    mirror->local_in_port,
+				    mirror->local_out_port,
+				    mirror->ingress);
+	kfree(mirror);
+}
+
+static void
+mlxsw_afa_mirror_destructor(struct mlxsw_afa_block *block,
+			    struct mlxsw_afa_resource *resource)
+{
+	struct mlxsw_afa_mirror *mirror;
+
+	mirror = container_of(resource, struct mlxsw_afa_mirror, resource);
+	mlxsw_afa_mirror_destroy(block, mirror);
+}
+
+static struct mlxsw_afa_mirror *
+mlxsw_afa_mirror_create(struct mlxsw_afa_block *block,
+			u8 local_in_port, u8 local_out_port,
+			bool ingress)
+{
+	struct mlxsw_afa_mirror *mirror;
+	int err;
+
+	mirror = kzalloc(sizeof(*mirror), GFP_KERNEL);
+	if (!mirror)
+		return ERR_PTR(-ENOMEM);
+
+	err = block->afa->ops->mirror_add(block->afa->ops_priv,
+					  local_in_port, local_out_port,
+					  ingress, &mirror->span_id);
+	if (err)
+		goto err_mirror_add;
+
+	mirror->ingress = ingress;
+	mirror->local_out_port = local_out_port;
+	mirror->local_in_port = local_in_port;
+	mirror->resource.destructor = mlxsw_afa_mirror_destructor;
+	mlxsw_afa_resource_add(block, &mirror->resource);
+	return mirror;
+
+err_mirror_add:
+	kfree(mirror);
+	return ERR_PTR(err);
+}
+
+static int
+mlxsw_afa_block_append_allocated_mirror(struct mlxsw_afa_block *block,
+					u8 mirror_agent)
+{
+	char *act = mlxsw_afa_block_append_action(block,
+						  MLXSW_AFA_TRAPDISC_CODE,
+						  MLXSW_AFA_TRAPDISC_SIZE);
+	if (!act)
+		return -ENOBUFS;
+	mlxsw_afa_trapdisc_pack(act, MLXSW_AFA_TRAPDISC_TRAP_ACTION_NOP,
+				MLXSW_AFA_TRAPDISC_FORWARD_ACTION_FORWARD, 0);
+	mlxsw_afa_trapdisc_mirror_pack(act, true, mirror_agent);
+	return 0;
+}
+
+int
+mlxsw_afa_block_append_mirror(struct mlxsw_afa_block *block,
+			      u8 local_in_port, u8 local_out_port, bool ingress)
+{
+	struct mlxsw_afa_mirror *mirror;
+	int err;
+
+	mirror = mlxsw_afa_mirror_create(block, local_in_port, local_out_port,
+					 ingress);
+	if (IS_ERR(mirror))
+		return PTR_ERR(mirror);
+
+	err = mlxsw_afa_block_append_allocated_mirror(block, mirror->span_id);
+	if (err)
+		goto err_append_allocated_mirror;
+
+	return 0;
+
+err_append_allocated_mirror:
+	mlxsw_afa_mirror_destroy(block, mirror);
+	return err;
+}
+EXPORT_SYMBOL(mlxsw_afa_block_append_mirror);
+
 /* Forwarding Action
  * -----------------
  * Forwarding Action can be used to implement Policy Based Switching (PBS)
@@ -853,11 +1050,10 @@ mlxsw_afa_polcnt_pack(char *payload,
 	mlxsw_afa_polcnt_counter_index_set(payload, counter_index);
 }
 
-int mlxsw_afa_block_append_counter(struct mlxsw_afa_block *block,
-				   u32 counter_index)
+int mlxsw_afa_block_append_allocated_counter(struct mlxsw_afa_block *block,
+					     u32 counter_index)
 {
-	char *act = mlxsw_afa_block_append_action(block,
-						  MLXSW_AFA_POLCNT_CODE,
+	char *act = mlxsw_afa_block_append_action(block, MLXSW_AFA_POLCNT_CODE,
 						  MLXSW_AFA_POLCNT_SIZE);
 	if (!act)
 		return -ENOBUFS;
@@ -865,6 +1061,32 @@ int mlxsw_afa_block_append_counter(struct mlxsw_afa_block *block,
 			      counter_index);
 	return 0;
 }
+EXPORT_SYMBOL(mlxsw_afa_block_append_allocated_counter);
+
+int mlxsw_afa_block_append_counter(struct mlxsw_afa_block *block,
+				   u32 *p_counter_index)
+{
+	struct mlxsw_afa_counter *counter;
+	u32 counter_index;
+	int err;
+
+	counter = mlxsw_afa_counter_create(block);
+	if (IS_ERR(counter))
+		return PTR_ERR(counter);
+	counter_index = counter->counter_index;
+
+	err = mlxsw_afa_block_append_allocated_counter(block, counter_index);
+	if (err)
+		goto err_append_allocated_counter;
+
+	if (p_counter_index)
+		*p_counter_index = counter_index;
+	return 0;
+
+err_append_allocated_counter:
+	mlxsw_afa_counter_destroy(block, counter);
+	return err;
+}
 EXPORT_SYMBOL(mlxsw_afa_block_append_counter);
 
 /* Virtual Router and Forwarding Domain Action
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h
index a8d3314c3a24..b91f2b0829b0 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h
@@ -46,6 +46,12 @@ struct mlxsw_afa_ops {
 	void (*kvdl_set_del)(void *priv, u32 kvdl_index, bool is_first);
 	int (*kvdl_fwd_entry_add)(void *priv, u32 *p_kvdl_index, u8 local_port);
 	void (*kvdl_fwd_entry_del)(void *priv, u32 kvdl_index);
+	int (*counter_index_get)(void *priv, unsigned int *p_counter_index);
+	void (*counter_index_put)(void *priv, unsigned int counter_index);
+	int (*mirror_add)(void *priv, u8 locol_in_port, u8 local_out_port,
+			  bool ingress, int *p_span_id);
+	void (*mirror_del)(void *priv, u8 locol_in_port, u8 local_out_port,
+			   bool ingress);
 };
 
 struct mlxsw_afa *mlxsw_afa_create(unsigned int max_acts_per_set,
@@ -59,16 +65,22 @@ char *mlxsw_afa_block_first_set(struct mlxsw_afa_block *block);
 u32 mlxsw_afa_block_first_set_kvdl_index(struct mlxsw_afa_block *block);
 int mlxsw_afa_block_continue(struct mlxsw_afa_block *block);
 int mlxsw_afa_block_jump(struct mlxsw_afa_block *block, u16 group_id);
+int mlxsw_afa_block_terminate(struct mlxsw_afa_block *block);
 int mlxsw_afa_block_append_drop(struct mlxsw_afa_block *block);
 int mlxsw_afa_block_append_trap(struct mlxsw_afa_block *block, u16 trap_id);
 int mlxsw_afa_block_append_trap_and_forward(struct mlxsw_afa_block *block,
 					    u16 trap_id);
+int mlxsw_afa_block_append_mirror(struct mlxsw_afa_block *block,
+				  u8 local_in_port, u8 local_out_port,
+				  bool ingress);
 int mlxsw_afa_block_append_fwd(struct mlxsw_afa_block *block,
 			       u8 local_port, bool in_port);
 int mlxsw_afa_block_append_vlan_modify(struct mlxsw_afa_block *block,
 				       u16 vid, u8 pcp, u8 et);
+int mlxsw_afa_block_append_allocated_counter(struct mlxsw_afa_block *block,
+					     u32 counter_index);
 int mlxsw_afa_block_append_counter(struct mlxsw_afa_block *block,
-				   u32 counter_index);
+				   u32 *p_counter_index);
 int mlxsw_afa_block_append_fid_set(struct mlxsw_afa_block *block, u16 fid);
 int mlxsw_afa_block_append_mcrouter(struct mlxsw_afa_block *block,
 				    u16 expected_irif, u16 min_mtu,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h
index f6963b0b4a55..122506daa586 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.h
@@ -107,20 +107,20 @@ static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
 	MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12),
 	MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3),
 	MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9),
-	MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x14, 0, 8),
-	MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x14, 9, 2),
-	MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x14, 11, 6),
-	MLXSW_AFK_ELEMENT_INFO_U32(SRC_IP4, 0x18, 0, 32),
-	MLXSW_AFK_ELEMENT_INFO_U32(DST_IP4, 0x1C, 0, 32),
-	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP6_HI, 0x18, 8),
-	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP6_LO, 0x20, 8),
-	MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP6_HI, 0x28, 8),
-	MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP6_LO, 0x30, 8),
 	MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16),
 	MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16),
+	MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8),
+	MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2),
+	MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6),
+	MLXSW_AFK_ELEMENT_INFO_U32(SRC_IP4, 0x20, 0, 32),
+	MLXSW_AFK_ELEMENT_INFO_U32(DST_IP4, 0x24, 0, 32),
+	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP6_HI, 0x20, 8),
+	MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP6_LO, 0x28, 8),
+	MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP6_HI, 0x30, 8),
+	MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP6_LO, 0x38, 8),
 };
 
-#define MLXSW_AFK_ELEMENT_STORAGE_SIZE 0x38
+#define MLXSW_AFK_ELEMENT_STORAGE_SIZE 0x40
 
 struct mlxsw_afk_element_inst { /* element instance in actual block */
 	const struct mlxsw_afk_element_info *info;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/i2c.c b/drivers/net/ethernet/mellanox/mlxsw/i2c.c
index c0dcfa05b077..25f9915ebd82 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/i2c.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/i2c.c
@@ -539,7 +539,8 @@ static int mlxsw_i2c_probe(struct i2c_client *client,
 	mlxsw_i2c->dev = &client->dev;
 
 	err = mlxsw_core_bus_device_register(&mlxsw_i2c->bus_info,
-					     &mlxsw_i2c_bus, mlxsw_i2c);
+					     &mlxsw_i2c_bus, mlxsw_i2c, false,
+					     NULL);
 	if (err) {
 		dev_err(&client->dev, "Fail to register core bus\n");
 		return err;
@@ -557,7 +558,7 @@ static int mlxsw_i2c_remove(struct i2c_client *client)
 {
 	struct mlxsw_i2c *mlxsw_i2c = i2c_get_clientdata(client);
 
-	mlxsw_core_bus_device_unregister(mlxsw_i2c->core);
+	mlxsw_core_bus_device_unregister(mlxsw_i2c->core, false);
 	mutex_destroy(&mlxsw_i2c->cmd.lock);
 
 	return 0;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/item.h b/drivers/net/ethernet/mellanox/mlxsw/item.h
index 28427f0758c7..31c886edc791 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/item.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/item.h
@@ -42,7 +42,7 @@
 
 struct mlxsw_item {
 	unsigned short	offset;		/* bytes in container */
-	unsigned short	step;		/* step in bytes for indexed items */
+	short		step;		/* step in bytes for indexed items */
 	unsigned short	in_step_offset; /* offset within one step */
 	unsigned char	shift;		/* shift in bits */
 	unsigned char	element_size;	/* size of element in bit array */
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
index 6ef20e5cc77d..85faa87bf42d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
@@ -154,6 +154,7 @@ struct mlxsw_pci {
 		} comp;
 	} cmd;
 	struct mlxsw_bus_info bus_info;
+	const struct pci_device_id *id;
 };
 
 static void mlxsw_pci_queue_tasklet_schedule(struct mlxsw_pci_queue *q)
@@ -1052,38 +1053,18 @@ static int mlxsw_pci_resources_query(struct mlxsw_pci *mlxsw_pci, char *mbox,
 }
 
 static int
-mlxsw_pci_profile_get_kvd_sizes(const struct mlxsw_config_profile *profile,
+mlxsw_pci_profile_get_kvd_sizes(const struct mlxsw_pci *mlxsw_pci,
+				const struct mlxsw_config_profile *profile,
 				struct mlxsw_res *res)
 {
-	u32 single_size, double_size, linear_size;
-
-	if (!MLXSW_RES_VALID(res, KVD_SINGLE_MIN_SIZE) ||
-	    !MLXSW_RES_VALID(res, KVD_DOUBLE_MIN_SIZE) ||
-	    !profile->used_kvd_split_data)
-		return -EIO;
-
-	linear_size = profile->kvd_linear_size;
+	u64 single_size, double_size, linear_size;
+	int err;
 
-	/* The hash part is what left of the kvd without the
-	 * linear part. It is split to the single size and
-	 * double size by the parts ratio from the profile.
-	 * Both sizes must be a multiplications of the
-	 * granularity from the profile.
-	 */
-	double_size = MLXSW_RES_GET(res, KVD_SIZE) - linear_size;
-	double_size *= profile->kvd_hash_double_parts;
-	double_size /= profile->kvd_hash_double_parts +
-		       profile->kvd_hash_single_parts;
-	double_size /= profile->kvd_hash_granularity;
-	double_size *= profile->kvd_hash_granularity;
-	single_size = MLXSW_RES_GET(res, KVD_SIZE) - double_size -
-		      linear_size;
-
-	/* Check results are legal. */
-	if (single_size < MLXSW_RES_GET(res, KVD_SINGLE_MIN_SIZE) ||
-	    double_size < MLXSW_RES_GET(res, KVD_DOUBLE_MIN_SIZE) ||
-	    MLXSW_RES_GET(res, KVD_SIZE) < linear_size)
-		return -EIO;
+	err = mlxsw_core_kvd_sizes_get(mlxsw_pci->core, profile,
+				       &single_size, &double_size,
+				       &linear_size);
+	if (err)
+		return err;
 
 	MLXSW_RES_SET(res, KVD_SINGLE_SIZE, single_size);
 	MLXSW_RES_SET(res, KVD_DOUBLE_SIZE, double_size);
@@ -1184,7 +1165,7 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
 			mbox, profile->adaptive_routing_group_cap);
 	}
 	if (MLXSW_RES_VALID(res, KVD_SIZE)) {
-		err = mlxsw_pci_profile_get_kvd_sizes(profile, res);
+		err = mlxsw_pci_profile_get_kvd_sizes(mlxsw_pci, profile, res);
 		if (err)
 			return err;
 
@@ -1622,16 +1603,6 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod,
 	return err;
 }
 
-static const struct mlxsw_bus mlxsw_pci_bus = {
-	.kind			= "pci",
-	.init			= mlxsw_pci_init,
-	.fini			= mlxsw_pci_fini,
-	.skb_transmit_busy	= mlxsw_pci_skb_transmit_busy,
-	.skb_transmit		= mlxsw_pci_skb_transmit,
-	.cmd_exec		= mlxsw_pci_cmd_exec,
-	.features		= MLXSW_BUS_F_TXRX,
-};
-
 static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,
 			      const struct pci_device_id *id)
 {
@@ -1660,6 +1631,41 @@ static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci,
 	return 0;
 }
 
+static void mlxsw_pci_free_irq_vectors(struct mlxsw_pci *mlxsw_pci)
+{
+	pci_free_irq_vectors(mlxsw_pci->pdev);
+}
+
+static int mlxsw_pci_alloc_irq_vectors(struct mlxsw_pci *mlxsw_pci)
+{
+	int err;
+
+	err = pci_alloc_irq_vectors(mlxsw_pci->pdev, 1, 1, PCI_IRQ_MSIX);
+	if (err < 0)
+		dev_err(&mlxsw_pci->pdev->dev, "MSI-X init failed\n");
+	return err;
+}
+
+static void mlxsw_pci_reset(void *bus_priv)
+{
+	struct mlxsw_pci *mlxsw_pci = bus_priv;
+
+	mlxsw_pci_free_irq_vectors(mlxsw_pci);
+	mlxsw_pci_sw_reset(mlxsw_pci, mlxsw_pci->id);
+	mlxsw_pci_alloc_irq_vectors(mlxsw_pci);
+}
+
+static const struct mlxsw_bus mlxsw_pci_bus = {
+	.kind			= "pci",
+	.init			= mlxsw_pci_init,
+	.fini			= mlxsw_pci_fini,
+	.skb_transmit_busy	= mlxsw_pci_skb_transmit_busy,
+	.skb_transmit		= mlxsw_pci_skb_transmit,
+	.cmd_exec		= mlxsw_pci_cmd_exec,
+	.features		= MLXSW_BUS_F_TXRX,
+	.reset			= mlxsw_pci_reset,
+};
+
 static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	const char *driver_name = pdev->driver->name;
@@ -1721,7 +1727,7 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_sw_reset;
 	}
 
-	err = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX);
+	err = mlxsw_pci_alloc_irq_vectors(mlxsw_pci);
 	if (err < 0) {
 		dev_err(&pdev->dev, "MSI-X init failed\n");
 		goto err_msix_init;
@@ -1730,9 +1736,11 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	mlxsw_pci->bus_info.device_kind = driver_name;
 	mlxsw_pci->bus_info.device_name = pci_name(mlxsw_pci->pdev);
 	mlxsw_pci->bus_info.dev = &pdev->dev;
+	mlxsw_pci->id = id;
 
 	err = mlxsw_core_bus_device_register(&mlxsw_pci->bus_info,
-					     &mlxsw_pci_bus, mlxsw_pci);
+					     &mlxsw_pci_bus, mlxsw_pci, false,
+					     NULL);
 	if (err) {
 		dev_err(&pdev->dev, "cannot register bus device\n");
 		goto err_bus_device_register;
@@ -1741,7 +1749,7 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return 0;
 
 err_bus_device_register:
-	pci_free_irq_vectors(mlxsw_pci->pdev);
+	mlxsw_pci_free_irq_vectors(mlxsw_pci);
 err_msix_init:
 err_sw_reset:
 	iounmap(mlxsw_pci->hw_addr);
@@ -1760,8 +1768,8 @@ static void mlxsw_pci_remove(struct pci_dev *pdev)
 {
 	struct mlxsw_pci *mlxsw_pci = pci_get_drvdata(pdev);
 
-	mlxsw_core_bus_device_unregister(mlxsw_pci->core);
-	pci_free_irq_vectors(mlxsw_pci->pdev);
+	mlxsw_core_bus_device_unregister(mlxsw_pci->core, false);
+	mlxsw_pci_free_irq_vectors(mlxsw_pci);
 	iounmap(mlxsw_pci->hw_addr);
 	pci_release_regions(mlxsw_pci->pdev);
 	pci_disable_device(mlxsw_pci->pdev);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
index 6c4e08b8058a..0e08be41c8e0 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
@@ -4827,6 +4827,42 @@ static inline void mlxsw_reg_ratr_counter_pack(char *payload, u64 counter_index,
 	mlxsw_reg_ratr_counter_set_type_set(payload, set_type);
 }
 
+/* RDPM - Router DSCP to Priority Mapping
+ * --------------------------------------
+ * Controls the mapping from DSCP field to switch priority on routed packets
+ */
+#define MLXSW_REG_RDPM_ID 0x8009
+#define MLXSW_REG_RDPM_BASE_LEN 0x00
+#define MLXSW_REG_RDPM_DSCP_ENTRY_REC_LEN 0x01
+#define MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT 64
+#define MLXSW_REG_RDPM_LEN 0x40
+#define MLXSW_REG_RDPM_LAST_ENTRY (MLXSW_REG_RDPM_BASE_LEN + \
+				   MLXSW_REG_RDPM_LEN - \
+				   MLXSW_REG_RDPM_DSCP_ENTRY_REC_LEN)
+
+MLXSW_REG_DEFINE(rdpm, MLXSW_REG_RDPM_ID, MLXSW_REG_RDPM_LEN);
+
+/* reg_dscp_entry_e
+ * Enable update of the specific entry
+ * Access: Index
+ */
+MLXSW_ITEM8_INDEXED(reg, rdpm, dscp_entry_e, MLXSW_REG_RDPM_LAST_ENTRY, 7, 1,
+		    -MLXSW_REG_RDPM_DSCP_ENTRY_REC_LEN, 0x00, false);
+
+/* reg_dscp_entry_prio
+ * Switch Priority
+ * Access: RW
+ */
+MLXSW_ITEM8_INDEXED(reg, rdpm, dscp_entry_prio, MLXSW_REG_RDPM_LAST_ENTRY, 0, 4,
+		    -MLXSW_REG_RDPM_DSCP_ENTRY_REC_LEN, 0x00, false);
+
+static inline void mlxsw_reg_rdpm_pack(char *payload, unsigned short index,
+				       u8 prio)
+{
+	mlxsw_reg_rdpm_dscp_entry_e_set(payload, index, 1);
+	mlxsw_reg_rdpm_dscp_entry_prio_set(payload, index, prio);
+}
+
 /* RICNT - Router Interface Counter Register
  * -----------------------------------------
  * The RICNT register retrieves per port performance counters
@@ -7640,6 +7676,7 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = {
 	MLXSW_REG(rtar),
 	MLXSW_REG(ratr),
 	MLXSW_REG(rtdp),
+	MLXSW_REG(rdpm),
 	MLXSW_REG(ricnt),
 	MLXSW_REG(rrcr),
 	MLXSW_REG(ralta),
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index c3837ca7a705..bf400c75fcc8 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -76,12 +76,7 @@
 #define MLXSW_FWREV_MAJOR 13
 #define MLXSW_FWREV_MINOR 1530
 #define MLXSW_FWREV_SUBMINOR 152
-
-static const struct mlxsw_fw_rev mlxsw_sp_supported_fw_rev = {
-	.major = MLXSW_FWREV_MAJOR,
-	.minor = MLXSW_FWREV_MINOR,
-	.subminor = MLXSW_FWREV_SUBMINOR
-};
+#define MLXSW_FWREV_MINOR_TO_BRANCH(minor) ((minor) / 100)
 
 #define MLXSW_SP_FW_FILENAME \
 	"mellanox/mlxsw_spectrum-" __stringify(MLXSW_FWREV_MAJOR) \
@@ -339,28 +334,25 @@ static int mlxsw_sp_firmware_flash(struct mlxsw_sp *mlxsw_sp,
 	return mlxfw_firmware_flash(&mlxsw_sp_mlxfw_dev.mlxfw_dev, firmware);
 }
 
-static bool mlxsw_sp_fw_rev_ge(const struct mlxsw_fw_rev *a,
-			       const struct mlxsw_fw_rev *b)
-{
-	if (a->major != b->major)
-		return a->major > b->major;
-	if (a->minor != b->minor)
-		return a->minor > b->minor;
-	return a->subminor >= b->subminor;
-}
-
 static int mlxsw_sp_fw_rev_validate(struct mlxsw_sp *mlxsw_sp)
 {
 	const struct mlxsw_fw_rev *rev = &mlxsw_sp->bus_info->fw_rev;
 	const struct firmware *firmware;
 	int err;
 
-	if (mlxsw_sp_fw_rev_ge(rev, &mlxsw_sp_supported_fw_rev))
+	/* Validate driver & FW are compatible */
+	if (rev->major != MLXSW_FWREV_MAJOR) {
+		WARN(1, "Mismatch in major FW version [%d:%d] is never expected; Please contact support\n",
+		     rev->major, MLXSW_FWREV_MAJOR);
+		return -EINVAL;
+	}
+	if (MLXSW_FWREV_MINOR_TO_BRANCH(rev->minor) ==
+	    MLXSW_FWREV_MINOR_TO_BRANCH(MLXSW_FWREV_MINOR))
 		return 0;
 
-	dev_info(mlxsw_sp->bus_info->dev, "The firmware version %d.%d.%d out of data\n",
+	dev_info(mlxsw_sp->bus_info->dev, "The firmware version %d.%d.%d is incompatible with the driver\n",
 		 rev->major, rev->minor, rev->subminor);
-	dev_info(mlxsw_sp->bus_info->dev, "Upgrading firmware using file %s\n",
+	dev_info(mlxsw_sp->bus_info->dev, "Flashing firmware using file %s\n",
 		 MLXSW_SP_FW_FILENAME);
 
 	err = request_firmware_direct(&firmware, MLXSW_SP_FW_FILENAME,
@@ -576,7 +568,7 @@ static void mlxsw_sp_span_entry_destroy(struct mlxsw_sp *mlxsw_sp,
 	span_entry->used = false;
 }
 
-static struct mlxsw_sp_span_entry *
+struct mlxsw_sp_span_entry *
 mlxsw_sp_span_entry_find(struct mlxsw_sp *mlxsw_sp, u8 local_port)
 {
 	int i;
@@ -663,13 +655,17 @@ static int mlxsw_sp_span_port_mtu_update(struct mlxsw_sp_port *port, u16 mtu)
 }
 
 static struct mlxsw_sp_span_inspected_port *
-mlxsw_sp_span_entry_bound_port_find(struct mlxsw_sp_port *port,
-				    struct mlxsw_sp_span_entry *span_entry)
+mlxsw_sp_span_entry_bound_port_find(struct mlxsw_sp_span_entry *span_entry,
+				    enum mlxsw_sp_span_type type,
+				    struct mlxsw_sp_port *port,
+				    bool bind)
 {
 	struct mlxsw_sp_span_inspected_port *p;
 
 	list_for_each_entry(p, &span_entry->bound_ports_list, list)
-		if (port->local_port == p->local_port)
+		if (type == p->type &&
+		    port->local_port == p->local_port &&
+		    bind == p->bound)
 			return p;
 	return NULL;
 }
@@ -677,15 +673,44 @@ mlxsw_sp_span_entry_bound_port_find(struct mlxsw_sp_port *port,
 static int
 mlxsw_sp_span_inspected_port_bind(struct mlxsw_sp_port *port,
 				  struct mlxsw_sp_span_entry *span_entry,
-				  enum mlxsw_sp_span_type type)
+				  enum mlxsw_sp_span_type type,
+				  bool bind)
 {
-	struct mlxsw_sp_span_inspected_port *inspected_port;
 	struct mlxsw_sp *mlxsw_sp = port->mlxsw_sp;
 	char mpar_pl[MLXSW_REG_MPAR_LEN];
-	char sbib_pl[MLXSW_REG_SBIB_LEN];
 	int pa_id = span_entry->id;
+
+	/* bind the port to the SPAN entry */
+	mlxsw_reg_mpar_pack(mpar_pl, port->local_port,
+			    (enum mlxsw_reg_mpar_i_e) type, bind, pa_id);
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpar), mpar_pl);
+}
+
+static int
+mlxsw_sp_span_inspected_port_add(struct mlxsw_sp_port *port,
+				 struct mlxsw_sp_span_entry *span_entry,
+				 enum mlxsw_sp_span_type type,
+				 bool bind)
+{
+	struct mlxsw_sp_span_inspected_port *inspected_port;
+	struct mlxsw_sp *mlxsw_sp = port->mlxsw_sp;
+	char sbib_pl[MLXSW_REG_SBIB_LEN];
+	int i;
 	int err;
 
+	/* A given (source port, direction) can only be bound to one analyzer,
+	 * so if a binding is requested, check for conflicts.
+	 */
+	if (bind)
+		for (i = 0; i < mlxsw_sp->span.entries_count; i++) {
+			struct mlxsw_sp_span_entry *curr =
+				&mlxsw_sp->span.entries[i];
+
+			if (mlxsw_sp_span_entry_bound_port_find(curr, type,
+								port, bind))
+				return -EEXIST;
+		}
+
 	/* if it is an egress SPAN, bind a shared buffer to it */
 	if (type == MLXSW_SP_SPAN_EGRESS) {
 		u32 buffsize = mlxsw_sp_span_mtu_to_buffsize(mlxsw_sp,
@@ -699,12 +724,12 @@ mlxsw_sp_span_inspected_port_bind(struct mlxsw_sp_port *port,
 		}
 	}
 
-	/* bind the port to the SPAN entry */
-	mlxsw_reg_mpar_pack(mpar_pl, port->local_port,
-			    (enum mlxsw_reg_mpar_i_e) type, true, pa_id);
-	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpar), mpar_pl);
-	if (err)
-		goto err_mpar_reg_write;
+	if (bind) {
+		err = mlxsw_sp_span_inspected_port_bind(port, span_entry, type,
+							true);
+		if (err)
+			goto err_port_bind;
+	}
 
 	inspected_port = kzalloc(sizeof(*inspected_port), GFP_KERNEL);
 	if (!inspected_port) {
@@ -713,12 +738,16 @@ mlxsw_sp_span_inspected_port_bind(struct mlxsw_sp_port *port,
 	}
 	inspected_port->local_port = port->local_port;
 	inspected_port->type = type;
+	inspected_port->bound = bind;
 	list_add_tail(&inspected_port->list, &span_entry->bound_ports_list);
 
 	return 0;
 
-err_mpar_reg_write:
 err_inspected_port_alloc:
+	if (bind)
+		mlxsw_sp_span_inspected_port_bind(port, span_entry, type,
+						  false);
+err_port_bind:
 	if (type == MLXSW_SP_SPAN_EGRESS) {
 		mlxsw_reg_sbib_pack(sbib_pl, port->local_port, 0);
 		mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbib), sbib_pl);
@@ -727,25 +756,23 @@ mlxsw_sp_span_inspected_port_bind(struct mlxsw_sp_port *port,
 }
 
 static void
-mlxsw_sp_span_inspected_port_unbind(struct mlxsw_sp_port *port,
-				    struct mlxsw_sp_span_entry *span_entry,
-				    enum mlxsw_sp_span_type type)
+mlxsw_sp_span_inspected_port_del(struct mlxsw_sp_port *port,
+				 struct mlxsw_sp_span_entry *span_entry,
+				 enum mlxsw_sp_span_type type,
+				 bool bind)
 {
 	struct mlxsw_sp_span_inspected_port *inspected_port;
 	struct mlxsw_sp *mlxsw_sp = port->mlxsw_sp;
-	char mpar_pl[MLXSW_REG_MPAR_LEN];
 	char sbib_pl[MLXSW_REG_SBIB_LEN];
-	int pa_id = span_entry->id;
 
-	inspected_port = mlxsw_sp_span_entry_bound_port_find(port, span_entry);
+	inspected_port = mlxsw_sp_span_entry_bound_port_find(span_entry, type,
+							     port, bind);
 	if (!inspected_port)
 		return;
 
-	/* remove the inspected port */
-	mlxsw_reg_mpar_pack(mpar_pl, port->local_port,
-			    (enum mlxsw_reg_mpar_i_e) type, false, pa_id);
-	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(mpar), mpar_pl);
-
+	if (bind)
+		mlxsw_sp_span_inspected_port_bind(port, span_entry, type,
+						  false);
 	/* remove the SBIB buffer if it was egress SPAN */
 	if (type == MLXSW_SP_SPAN_EGRESS) {
 		mlxsw_reg_sbib_pack(sbib_pl, port->local_port, 0);
@@ -758,9 +785,9 @@ mlxsw_sp_span_inspected_port_unbind(struct mlxsw_sp_port *port,
 	kfree(inspected_port);
 }
 
-static int mlxsw_sp_span_mirror_add(struct mlxsw_sp_port *from,
-				    struct mlxsw_sp_port *to,
-				    enum mlxsw_sp_span_type type)
+int mlxsw_sp_span_mirror_add(struct mlxsw_sp_port *from,
+			     struct mlxsw_sp_port *to,
+			     enum mlxsw_sp_span_type type, bool bind)
 {
 	struct mlxsw_sp *mlxsw_sp = from->mlxsw_sp;
 	struct mlxsw_sp_span_entry *span_entry;
@@ -773,7 +800,7 @@ static int mlxsw_sp_span_mirror_add(struct mlxsw_sp_port *from,
 	netdev_dbg(from->dev, "Adding inspected port to SPAN entry %d\n",
 		   span_entry->id);
 
-	err = mlxsw_sp_span_inspected_port_bind(from, span_entry, type);
+	err = mlxsw_sp_span_inspected_port_add(from, span_entry, type, bind);
 	if (err)
 		goto err_port_bind;
 
@@ -784,9 +811,8 @@ static int mlxsw_sp_span_mirror_add(struct mlxsw_sp_port *from,
 	return err;
 }
 
-static void mlxsw_sp_span_mirror_remove(struct mlxsw_sp_port *from,
-					u8 destination_port,
-					enum mlxsw_sp_span_type type)
+void mlxsw_sp_span_mirror_del(struct mlxsw_sp_port *from, u8 destination_port,
+			      enum mlxsw_sp_span_type type, bool bind)
 {
 	struct mlxsw_sp_span_entry *span_entry;
 
@@ -799,7 +825,7 @@ static void mlxsw_sp_span_mirror_remove(struct mlxsw_sp_port *from,
 
 	netdev_dbg(from->dev, "removing inspected port from SPAN entry %d\n",
 		   span_entry->id);
-	mlxsw_sp_span_inspected_port_unbind(from, span_entry, type);
+	mlxsw_sp_span_inspected_port_del(from, span_entry, type, bind);
 }
 
 static int mlxsw_sp_port_sample_set(struct mlxsw_sp_port *mlxsw_sp_port,
@@ -1453,6 +1479,7 @@ mlxsw_sp_port_vlan_create(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid)
 	}
 
 	mlxsw_sp_port_vlan->mlxsw_sp_port = mlxsw_sp_port;
+	mlxsw_sp_port_vlan->ref_count = 1;
 	mlxsw_sp_port_vlan->vid = vid;
 	list_add(&mlxsw_sp_port_vlan->list, &mlxsw_sp_port->vlans_list);
 
@@ -1480,8 +1507,10 @@ mlxsw_sp_port_vlan_get(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid)
 	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
 
 	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
-	if (mlxsw_sp_port_vlan)
+	if (mlxsw_sp_port_vlan) {
+		mlxsw_sp_port_vlan->ref_count++;
 		return mlxsw_sp_port_vlan;
+	}
 
 	return mlxsw_sp_port_vlan_create(mlxsw_sp_port, vid);
 }
@@ -1490,6 +1519,9 @@ void mlxsw_sp_port_vlan_put(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
 {
 	struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
 
+	if (--mlxsw_sp_port_vlan->ref_count != 0)
+		return;
+
 	if (mlxsw_sp_port_vlan->bridge_port)
 		mlxsw_sp_port_vlan_bridge_leave(mlxsw_sp_port_vlan);
 	else if (fid)
@@ -1571,14 +1603,11 @@ mlxsw_sp_port_add_cls_matchall_mirror(struct mlxsw_sp_port *mlxsw_sp_port,
 				      const struct tc_action *a,
 				      bool ingress)
 {
-	struct net *net = dev_net(mlxsw_sp_port->dev);
 	enum mlxsw_sp_span_type span_type;
 	struct mlxsw_sp_port *to_port;
 	struct net_device *to_dev;
-	int ifindex;
 
-	ifindex = tcf_mirred_ifindex(a);
-	to_dev = __dev_get_by_index(net, ifindex);
+	to_dev = tcf_mirred_dev(a);
 	if (!to_dev) {
 		netdev_err(mlxsw_sp_port->dev, "Could not find requested device\n");
 		return -EINVAL;
@@ -1593,7 +1622,8 @@ mlxsw_sp_port_add_cls_matchall_mirror(struct mlxsw_sp_port *mlxsw_sp_port,
 	mirror->to_local_port = to_port->local_port;
 	mirror->ingress = ingress;
 	span_type = ingress ? MLXSW_SP_SPAN_INGRESS : MLXSW_SP_SPAN_EGRESS;
-	return mlxsw_sp_span_mirror_add(mlxsw_sp_port, to_port, span_type);
+	return mlxsw_sp_span_mirror_add(mlxsw_sp_port, to_port, span_type,
+					true);
 }
 
 static void
@@ -1604,8 +1634,8 @@ mlxsw_sp_port_del_cls_matchall_mirror(struct mlxsw_sp_port *mlxsw_sp_port,
 
 	span_type = mirror->ingress ?
 			MLXSW_SP_SPAN_INGRESS : MLXSW_SP_SPAN_EGRESS;
-	mlxsw_sp_span_mirror_remove(mlxsw_sp_port, mirror->to_local_port,
-				    span_type);
+	mlxsw_sp_span_mirror_del(mlxsw_sp_port, mirror->to_local_port,
+				 span_type, true);
 }
 
 static int
@@ -1734,9 +1764,6 @@ static int mlxsw_sp_setup_tc_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
 					  struct tc_cls_matchall_offload *f,
 					  bool ingress)
 {
-	if (f->common.chain_index)
-		return -EOPNOTSUPP;
-
 	switch (f->command) {
 	case TC_CLSMATCHALL_REPLACE:
 		return mlxsw_sp_port_add_cls_matchall(mlxsw_sp_port, f,
@@ -1750,72 +1777,187 @@ static int mlxsw_sp_setup_tc_cls_matchall(struct mlxsw_sp_port *mlxsw_sp_port,
 }
 
 static int
-mlxsw_sp_setup_tc_cls_flower(struct mlxsw_sp_port *mlxsw_sp_port,
-			     struct tc_cls_flower_offload *f,
-			     bool ingress)
+mlxsw_sp_setup_tc_cls_flower(struct mlxsw_sp_acl_block *acl_block,
+			     struct tc_cls_flower_offload *f)
 {
+	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_acl_block_mlxsw_sp(acl_block);
+
 	switch (f->command) {
 	case TC_CLSFLOWER_REPLACE:
-		return mlxsw_sp_flower_replace(mlxsw_sp_port, ingress, f);
+		return mlxsw_sp_flower_replace(mlxsw_sp, acl_block, f);
 	case TC_CLSFLOWER_DESTROY:
-		mlxsw_sp_flower_destroy(mlxsw_sp_port, ingress, f);
+		mlxsw_sp_flower_destroy(mlxsw_sp, acl_block, f);
 		return 0;
 	case TC_CLSFLOWER_STATS:
-		return mlxsw_sp_flower_stats(mlxsw_sp_port, ingress, f);
+		return mlxsw_sp_flower_stats(mlxsw_sp, acl_block, f);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
-static int mlxsw_sp_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
-				      void *cb_priv, bool ingress)
+static int mlxsw_sp_setup_tc_block_cb_matchall(enum tc_setup_type type,
+					       void *type_data,
+					       void *cb_priv, bool ingress)
 {
 	struct mlxsw_sp_port *mlxsw_sp_port = cb_priv;
 
-	if (!tc_can_offload(mlxsw_sp_port->dev))
-		return -EOPNOTSUPP;
-
 	switch (type) {
 	case TC_SETUP_CLSMATCHALL:
+		if (!tc_cls_can_offload_and_chain0(mlxsw_sp_port->dev,
+						   type_data))
+			return -EOPNOTSUPP;
+
 		return mlxsw_sp_setup_tc_cls_matchall(mlxsw_sp_port, type_data,
 						      ingress);
 	case TC_SETUP_CLSFLOWER:
-		return mlxsw_sp_setup_tc_cls_flower(mlxsw_sp_port, type_data,
-						    ingress);
+		return 0;
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
-static int mlxsw_sp_setup_tc_block_cb_ig(enum tc_setup_type type,
-					 void *type_data, void *cb_priv)
+static int mlxsw_sp_setup_tc_block_cb_matchall_ig(enum tc_setup_type type,
+						  void *type_data,
+						  void *cb_priv)
 {
-	return mlxsw_sp_setup_tc_block_cb(type, type_data, cb_priv, true);
+	return mlxsw_sp_setup_tc_block_cb_matchall(type, type_data,
+						   cb_priv, true);
 }
 
-static int mlxsw_sp_setup_tc_block_cb_eg(enum tc_setup_type type,
-					 void *type_data, void *cb_priv)
+static int mlxsw_sp_setup_tc_block_cb_matchall_eg(enum tc_setup_type type,
+						  void *type_data,
+						  void *cb_priv)
 {
-	return mlxsw_sp_setup_tc_block_cb(type, type_data, cb_priv, false);
+	return mlxsw_sp_setup_tc_block_cb_matchall(type, type_data,
+						   cb_priv, false);
+}
+
+static int mlxsw_sp_setup_tc_block_cb_flower(enum tc_setup_type type,
+					     void *type_data, void *cb_priv)
+{
+	struct mlxsw_sp_acl_block *acl_block = cb_priv;
+
+	switch (type) {
+	case TC_SETUP_CLSMATCHALL:
+		return 0;
+	case TC_SETUP_CLSFLOWER:
+		if (mlxsw_sp_acl_block_disabled(acl_block))
+			return -EOPNOTSUPP;
+
+		return mlxsw_sp_setup_tc_cls_flower(acl_block, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int
+mlxsw_sp_setup_tc_block_flower_bind(struct mlxsw_sp_port *mlxsw_sp_port,
+				    struct tcf_block *block, bool ingress)
+{
+	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
+	struct mlxsw_sp_acl_block *acl_block;
+	struct tcf_block_cb *block_cb;
+	int err;
+
+	block_cb = tcf_block_cb_lookup(block, mlxsw_sp_setup_tc_block_cb_flower,
+				       mlxsw_sp);
+	if (!block_cb) {
+		acl_block = mlxsw_sp_acl_block_create(mlxsw_sp, block->net);
+		if (!acl_block)
+			return -ENOMEM;
+		block_cb = __tcf_block_cb_register(block,
+						   mlxsw_sp_setup_tc_block_cb_flower,
+						   mlxsw_sp, acl_block);
+		if (IS_ERR(block_cb)) {
+			err = PTR_ERR(block_cb);
+			goto err_cb_register;
+		}
+	} else {
+		acl_block = tcf_block_cb_priv(block_cb);
+	}
+	tcf_block_cb_incref(block_cb);
+	err = mlxsw_sp_acl_block_bind(mlxsw_sp, acl_block,
+				      mlxsw_sp_port, ingress);
+	if (err)
+		goto err_block_bind;
+
+	if (ingress)
+		mlxsw_sp_port->ing_acl_block = acl_block;
+	else
+		mlxsw_sp_port->eg_acl_block = acl_block;
+
+	return 0;
+
+err_block_bind:
+	if (!tcf_block_cb_decref(block_cb)) {
+		__tcf_block_cb_unregister(block_cb);
+err_cb_register:
+		mlxsw_sp_acl_block_destroy(acl_block);
+	}
+	return err;
+}
+
+static void
+mlxsw_sp_setup_tc_block_flower_unbind(struct mlxsw_sp_port *mlxsw_sp_port,
+				      struct tcf_block *block, bool ingress)
+{
+	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
+	struct mlxsw_sp_acl_block *acl_block;
+	struct tcf_block_cb *block_cb;
+	int err;
+
+	block_cb = tcf_block_cb_lookup(block, mlxsw_sp_setup_tc_block_cb_flower,
+				       mlxsw_sp);
+	if (!block_cb)
+		return;
+
+	if (ingress)
+		mlxsw_sp_port->ing_acl_block = NULL;
+	else
+		mlxsw_sp_port->eg_acl_block = NULL;
+
+	acl_block = tcf_block_cb_priv(block_cb);
+	err = mlxsw_sp_acl_block_unbind(mlxsw_sp, acl_block,
+					mlxsw_sp_port, ingress);
+	if (!err && !tcf_block_cb_decref(block_cb)) {
+		__tcf_block_cb_unregister(block_cb);
+		mlxsw_sp_acl_block_destroy(acl_block);
+	}
 }
 
 static int mlxsw_sp_setup_tc_block(struct mlxsw_sp_port *mlxsw_sp_port,
 				   struct tc_block_offload *f)
 {
 	tc_setup_cb_t *cb;
+	bool ingress;
+	int err;
 
-	if (f->binder_type == TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
-		cb = mlxsw_sp_setup_tc_block_cb_ig;
-	else if (f->binder_type == TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
-		cb = mlxsw_sp_setup_tc_block_cb_eg;
-	else
+	if (f->binder_type == TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) {
+		cb = mlxsw_sp_setup_tc_block_cb_matchall_ig;
+		ingress = true;
+	} else if (f->binder_type == TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
+		cb = mlxsw_sp_setup_tc_block_cb_matchall_eg;
+		ingress = false;
+	} else {
 		return -EOPNOTSUPP;
+	}
 
 	switch (f->command) {
 	case TC_BLOCK_BIND:
-		return tcf_block_cb_register(f->block, cb, mlxsw_sp_port,
-					     mlxsw_sp_port);
+		err = tcf_block_cb_register(f->block, cb, mlxsw_sp_port,
+					    mlxsw_sp_port);
+		if (err)
+			return err;
+		err = mlxsw_sp_setup_tc_block_flower_bind(mlxsw_sp_port,
+							  f->block, ingress);
+		if (err) {
+			tcf_block_cb_unregister(f->block, cb, mlxsw_sp_port);
+			return err;
+		}
+		return 0;
 	case TC_BLOCK_UNBIND:
+		mlxsw_sp_setup_tc_block_flower_unbind(mlxsw_sp_port,
+						      f->block, ingress);
 		tcf_block_cb_unregister(f->block, cb, mlxsw_sp_port);
 		return 0;
 	default:
@@ -1833,11 +1975,69 @@ static int mlxsw_sp_setup_tc(struct net_device *dev, enum tc_setup_type type,
 		return mlxsw_sp_setup_tc_block(mlxsw_sp_port, type_data);
 	case TC_SETUP_QDISC_RED:
 		return mlxsw_sp_setup_tc_red(mlxsw_sp_port, type_data);
+	case TC_SETUP_QDISC_PRIO:
+		return mlxsw_sp_setup_tc_prio(mlxsw_sp_port, type_data);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
+
+static int mlxsw_sp_feature_hw_tc(struct net_device *dev, bool enable)
+{
+	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
+
+	if (!enable) {
+		if (mlxsw_sp_acl_block_rule_count(mlxsw_sp_port->ing_acl_block) ||
+		    mlxsw_sp_acl_block_rule_count(mlxsw_sp_port->eg_acl_block) ||
+		    !list_empty(&mlxsw_sp_port->mall_tc_list)) {
+			netdev_err(dev, "Active offloaded tc filters, can't turn hw_tc_offload off\n");
+			return -EINVAL;
+		}
+		mlxsw_sp_acl_block_disable_inc(mlxsw_sp_port->ing_acl_block);
+		mlxsw_sp_acl_block_disable_inc(mlxsw_sp_port->eg_acl_block);
+	} else {
+		mlxsw_sp_acl_block_disable_dec(mlxsw_sp_port->ing_acl_block);
+		mlxsw_sp_acl_block_disable_dec(mlxsw_sp_port->eg_acl_block);
+	}
+	return 0;
+}
+
+typedef int (*mlxsw_sp_feature_handler)(struct net_device *dev, bool enable);
+
+static int mlxsw_sp_handle_feature(struct net_device *dev,
+				   netdev_features_t wanted_features,
+				   netdev_features_t feature,
+				   mlxsw_sp_feature_handler feature_handler)
+{
+	netdev_features_t changes = wanted_features ^ dev->features;
+	bool enable = !!(wanted_features & feature);
+	int err;
+
+	if (!(changes & feature))
+		return 0;
+
+	err = feature_handler(dev, enable);
+	if (err) {
+		netdev_err(dev, "%s feature %pNF failed, err %d\n",
+			   enable ? "Enable" : "Disable", &feature, err);
+		return err;
+	}
+
+	if (enable)
+		dev->features |= feature;
+	else
+		dev->features &= ~feature;
+
+	return 0;
+}
+static int mlxsw_sp_set_features(struct net_device *dev,
+				 netdev_features_t features)
+{
+	return mlxsw_sp_handle_feature(dev, features, NETIF_F_HW_TC,
+				       mlxsw_sp_feature_hw_tc);
+}
+
 static const struct net_device_ops mlxsw_sp_port_netdev_ops = {
 	.ndo_open		= mlxsw_sp_port_open,
 	.ndo_stop		= mlxsw_sp_port_stop,
@@ -1852,6 +2052,7 @@ static const struct net_device_ops mlxsw_sp_port_netdev_ops = {
 	.ndo_vlan_rx_add_vid	= mlxsw_sp_port_add_vid,
 	.ndo_vlan_rx_kill_vid	= mlxsw_sp_port_kill_vid,
 	.ndo_get_phys_port_name	= mlxsw_sp_port_get_phys_port_name,
+	.ndo_set_features	= mlxsw_sp_set_features,
 };
 
 static void mlxsw_sp_port_get_drvinfo(struct net_device *dev,
@@ -3039,6 +3240,13 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 		goto err_port_fids_init;
 	}
 
+	err = mlxsw_sp_tc_qdisc_init(mlxsw_sp_port);
+	if (err) {
+		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to initialize TC qdiscs\n",
+			mlxsw_sp_port->local_port);
+		goto err_port_qdiscs_init;
+	}
+
 	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_get(mlxsw_sp_port, 1);
 	if (IS_ERR(mlxsw_sp_port_vlan)) {
 		dev_err(mlxsw_sp->bus_info->dev, "Port %d: Failed to create VID 1\n",
@@ -3067,6 +3275,8 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 	mlxsw_sp_port_switchdev_fini(mlxsw_sp_port);
 	mlxsw_sp_port_vlan_put(mlxsw_sp_port_vlan);
 err_port_vlan_get:
+	mlxsw_sp_tc_qdisc_fini(mlxsw_sp_port);
+err_port_qdiscs_init:
 	mlxsw_sp_port_fids_fini(mlxsw_sp_port);
 err_port_fids_init:
 	mlxsw_sp_port_dcb_fini(mlxsw_sp_port);
@@ -3102,6 +3312,7 @@ static void mlxsw_sp_port_remove(struct mlxsw_sp *mlxsw_sp, u8 local_port)
 	mlxsw_sp->ports[local_port] = NULL;
 	mlxsw_sp_port_switchdev_fini(mlxsw_sp_port);
 	mlxsw_sp_port_vlan_flush(mlxsw_sp_port);
+	mlxsw_sp_tc_qdisc_fini(mlxsw_sp_port);
 	mlxsw_sp_port_fids_fini(mlxsw_sp_port);
 	mlxsw_sp_port_dcb_fini(mlxsw_sp_port);
 	mlxsw_sp_port_swid_set(mlxsw_sp_port, MLXSW_PORT_SWID_DISABLED_PORT);
@@ -3933,6 +4144,254 @@ static const struct mlxsw_config_profile mlxsw_sp_config_profile = {
 	.resource_query_enable		= 1,
 };
 
+static bool
+mlxsw_sp_resource_kvd_granularity_validate(struct netlink_ext_ack *extack,
+					   u64 size)
+{
+	const struct mlxsw_config_profile *profile;
+
+	profile = &mlxsw_sp_config_profile;
+	if (size % profile->kvd_hash_granularity) {
+		NL_SET_ERR_MSG_MOD(extack, "resource set with wrong granularity");
+		return false;
+	}
+	return true;
+}
+
+static int
+mlxsw_sp_resource_kvd_size_validate(struct devlink *devlink, u64 size,
+				    struct netlink_ext_ack *extack)
+{
+	NL_SET_ERR_MSG_MOD(extack, "kvd size cannot be changed");
+	return -EINVAL;
+}
+
+static int
+mlxsw_sp_resource_kvd_linear_size_validate(struct devlink *devlink, u64 size,
+					   struct netlink_ext_ack *extack)
+{
+	if (!mlxsw_sp_resource_kvd_granularity_validate(extack, size))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+mlxsw_sp_resource_kvd_hash_single_size_validate(struct devlink *devlink, u64 size,
+						struct netlink_ext_ack *extack)
+{
+	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
+
+	if (!mlxsw_sp_resource_kvd_granularity_validate(extack, size))
+		return -EINVAL;
+
+	if (size < MLXSW_CORE_RES_GET(mlxsw_core, KVD_SINGLE_MIN_SIZE)) {
+		NL_SET_ERR_MSG_MOD(extack, "hash single size is smaller than minimum");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int
+mlxsw_sp_resource_kvd_hash_double_size_validate(struct devlink *devlink, u64 size,
+						struct netlink_ext_ack *extack)
+{
+	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
+
+	if (!mlxsw_sp_resource_kvd_granularity_validate(extack, size))
+		return -EINVAL;
+
+	if (size < MLXSW_CORE_RES_GET(mlxsw_core, KVD_DOUBLE_MIN_SIZE)) {
+		NL_SET_ERR_MSG_MOD(extack, "hash double size is smaller than minimum");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static u64 mlxsw_sp_resource_kvd_linear_occ_get(struct devlink *devlink)
+{
+	struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
+	struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
+
+	return mlxsw_sp_kvdl_occ_get(mlxsw_sp);
+}
+
+static struct devlink_resource_ops mlxsw_sp_resource_kvd_ops = {
+	.size_validate = mlxsw_sp_resource_kvd_size_validate,
+};
+
+static struct devlink_resource_ops mlxsw_sp_resource_kvd_linear_ops = {
+	.size_validate = mlxsw_sp_resource_kvd_linear_size_validate,
+	.occ_get = mlxsw_sp_resource_kvd_linear_occ_get,
+};
+
+static struct devlink_resource_ops mlxsw_sp_resource_kvd_hash_single_ops = {
+	.size_validate = mlxsw_sp_resource_kvd_hash_single_size_validate,
+};
+
+static struct devlink_resource_ops mlxsw_sp_resource_kvd_hash_double_ops = {
+	.size_validate = mlxsw_sp_resource_kvd_hash_double_size_validate,
+};
+
+static void
+mlxsw_sp_resource_size_params_prepare(struct mlxsw_core *mlxsw_core,
+				      struct devlink_resource_size_params *kvd_size_params,
+				      struct devlink_resource_size_params *linear_size_params,
+				      struct devlink_resource_size_params *hash_double_size_params,
+				      struct devlink_resource_size_params *hash_single_size_params)
+{
+	u32 single_size_min = MLXSW_CORE_RES_GET(mlxsw_core,
+						 KVD_SINGLE_MIN_SIZE);
+	u32 double_size_min = MLXSW_CORE_RES_GET(mlxsw_core,
+						 KVD_DOUBLE_MIN_SIZE);
+	u32 kvd_size = MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE);
+	u32 linear_size_min = 0;
+
+	devlink_resource_size_params_init(kvd_size_params, kvd_size, kvd_size,
+					  MLXSW_SP_KVD_GRANULARITY,
+					  DEVLINK_RESOURCE_UNIT_ENTRY);
+	devlink_resource_size_params_init(linear_size_params, linear_size_min,
+					  kvd_size - single_size_min -
+					  double_size_min,
+					  MLXSW_SP_KVD_GRANULARITY,
+					  DEVLINK_RESOURCE_UNIT_ENTRY);
+	devlink_resource_size_params_init(hash_double_size_params,
+					  double_size_min,
+					  kvd_size - single_size_min -
+					  linear_size_min,
+					  MLXSW_SP_KVD_GRANULARITY,
+					  DEVLINK_RESOURCE_UNIT_ENTRY);
+	devlink_resource_size_params_init(hash_single_size_params,
+					  single_size_min,
+					  kvd_size - double_size_min -
+					  linear_size_min,
+					  MLXSW_SP_KVD_GRANULARITY,
+					  DEVLINK_RESOURCE_UNIT_ENTRY);
+}
+
+static int mlxsw_sp_resources_register(struct mlxsw_core *mlxsw_core)
+{
+	struct devlink *devlink = priv_to_devlink(mlxsw_core);
+	struct devlink_resource_size_params hash_single_size_params;
+	struct devlink_resource_size_params hash_double_size_params;
+	struct devlink_resource_size_params linear_size_params;
+	struct devlink_resource_size_params kvd_size_params;
+	u32 kvd_size, single_size, double_size, linear_size;
+	const struct mlxsw_config_profile *profile;
+	int err;
+
+	profile = &mlxsw_sp_config_profile;
+	if (!MLXSW_CORE_RES_VALID(mlxsw_core, KVD_SIZE))
+		return -EIO;
+
+	mlxsw_sp_resource_size_params_prepare(mlxsw_core, &kvd_size_params,
+					      &linear_size_params,
+					      &hash_double_size_params,
+					      &hash_single_size_params);
+
+	kvd_size = MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE);
+	err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD,
+					true, kvd_size,
+					MLXSW_SP_RESOURCE_KVD,
+					DEVLINK_RESOURCE_ID_PARENT_TOP,
+					&kvd_size_params,
+					&mlxsw_sp_resource_kvd_ops);
+	if (err)
+		return err;
+
+	linear_size = profile->kvd_linear_size;
+	err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_LINEAR,
+					false, linear_size,
+					MLXSW_SP_RESOURCE_KVD_LINEAR,
+					MLXSW_SP_RESOURCE_KVD,
+					&linear_size_params,
+					&mlxsw_sp_resource_kvd_linear_ops);
+	if (err)
+		return err;
+
+	double_size = kvd_size - linear_size;
+	double_size *= profile->kvd_hash_double_parts;
+	double_size /= profile->kvd_hash_double_parts +
+		       profile->kvd_hash_single_parts;
+	double_size = rounddown(double_size, profile->kvd_hash_granularity);
+	err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_HASH_DOUBLE,
+					false, double_size,
+					MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE,
+					MLXSW_SP_RESOURCE_KVD,
+					&hash_double_size_params,
+					&mlxsw_sp_resource_kvd_hash_double_ops);
+	if (err)
+		return err;
+
+	single_size = kvd_size - double_size - linear_size;
+	err = devlink_resource_register(devlink, MLXSW_SP_RESOURCE_NAME_KVD_HASH_SINGLE,
+					false, single_size,
+					MLXSW_SP_RESOURCE_KVD_HASH_SINGLE,
+					MLXSW_SP_RESOURCE_KVD,
+					&hash_single_size_params,
+					&mlxsw_sp_resource_kvd_hash_single_ops);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int mlxsw_sp_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
+				  const struct mlxsw_config_profile *profile,
+				  u64 *p_single_size, u64 *p_double_size,
+				  u64 *p_linear_size)
+{
+	struct devlink *devlink = priv_to_devlink(mlxsw_core);
+	u32 double_size;
+	int err;
+
+	if (!MLXSW_CORE_RES_VALID(mlxsw_core, KVD_SINGLE_MIN_SIZE) ||
+	    !MLXSW_CORE_RES_VALID(mlxsw_core, KVD_DOUBLE_MIN_SIZE) ||
+	    !profile->used_kvd_split_data)
+		return -EIO;
+
+	/* The hash part is what left of the kvd without the
+	 * linear part. It is split to the single size and
+	 * double size by the parts ratio from the profile.
+	 * Both sizes must be a multiplications of the
+	 * granularity from the profile. In case the user
+	 * provided the sizes they are obtained via devlink.
+	 */
+	err = devlink_resource_size_get(devlink,
+					MLXSW_SP_RESOURCE_KVD_LINEAR,
+					p_linear_size);
+	if (err)
+		*p_linear_size = profile->kvd_linear_size;
+
+	err = devlink_resource_size_get(devlink,
+					MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE,
+					p_double_size);
+	if (err) {
+		double_size = MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE) -
+			      *p_linear_size;
+		double_size *= profile->kvd_hash_double_parts;
+		double_size /= profile->kvd_hash_double_parts +
+			       profile->kvd_hash_single_parts;
+		*p_double_size = rounddown(double_size,
+					   profile->kvd_hash_granularity);
+	}
+
+	err = devlink_resource_size_get(devlink,
+					MLXSW_SP_RESOURCE_KVD_HASH_SINGLE,
+					p_single_size);
+	if (err)
+		*p_single_size = MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE) -
+				 *p_double_size - *p_linear_size;
+
+	/* Check results are legal. */
+	if (*p_single_size < MLXSW_CORE_RES_GET(mlxsw_core, KVD_SINGLE_MIN_SIZE) ||
+	    *p_double_size < MLXSW_CORE_RES_GET(mlxsw_core, KVD_DOUBLE_MIN_SIZE) ||
+	    MLXSW_CORE_RES_GET(mlxsw_core, KVD_SIZE) < *p_linear_size)
+		return -EIO;
+
+	return 0;
+}
+
 static struct mlxsw_driver mlxsw_sp_driver = {
 	.kind				= mlxsw_sp_driver_name,
 	.priv_size			= sizeof(struct mlxsw_sp),
@@ -3952,6 +4411,8 @@ static struct mlxsw_driver mlxsw_sp_driver = {
 	.sb_occ_port_pool_get		= mlxsw_sp_sb_occ_port_pool_get,
 	.sb_occ_tc_port_bind_get	= mlxsw_sp_sb_occ_tc_port_bind_get,
 	.txhdr_construct		= mlxsw_sp_txhdr_construct,
+	.resources_register		= mlxsw_sp_resources_register,
+	.kvd_sizes_get			= mlxsw_sp_kvd_sizes_get,
 	.txhdr_len			= MLXSW_TXHDR_LEN,
 	.profile			= &mlxsw_sp_config_profile,
 };
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index 05ce1befd9b3..92064db2ae44 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -66,6 +66,18 @@
 #define MLXSW_SP_KVD_LINEAR_SIZE 98304 /* entries */
 #define MLXSW_SP_KVD_GRANULARITY 128
 
+#define MLXSW_SP_RESOURCE_NAME_KVD "kvd"
+#define MLXSW_SP_RESOURCE_NAME_KVD_LINEAR "linear"
+#define MLXSW_SP_RESOURCE_NAME_KVD_HASH_SINGLE "hash_single"
+#define MLXSW_SP_RESOURCE_NAME_KVD_HASH_DOUBLE "hash_double"
+
+enum mlxsw_sp_resource_id {
+	MLXSW_SP_RESOURCE_KVD,
+	MLXSW_SP_RESOURCE_KVD_LINEAR,
+	MLXSW_SP_RESOURCE_KVD_HASH_SINGLE,
+	MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE,
+};
+
 struct mlxsw_sp_port;
 struct mlxsw_sp_rif;
 
@@ -108,6 +120,9 @@ struct mlxsw_sp_span_inspected_port {
 	struct list_head list;
 	enum mlxsw_sp_span_type type;
 	u8 local_port;
+
+	/* Whether this is a directly bound mirror (port-to-port) or an ACL. */
+	bool bound;
 };
 
 struct mlxsw_sp_span_entry {
@@ -199,34 +214,12 @@ struct mlxsw_sp_port_vlan {
 	struct list_head list;
 	struct mlxsw_sp_port *mlxsw_sp_port;
 	struct mlxsw_sp_fid *fid;
+	unsigned int ref_count;
 	u16 vid;
 	struct mlxsw_sp_bridge_port *bridge_port;
 	struct list_head bridge_vlan_node;
 };
 
-enum mlxsw_sp_qdisc_type {
-	MLXSW_SP_QDISC_NO_QDISC,
-	MLXSW_SP_QDISC_RED,
-};
-
-struct mlxsw_sp_qdisc {
-	u32 handle;
-	enum mlxsw_sp_qdisc_type type;
-	struct red_stats xstats_base;
-	union {
-		struct {
-			u64 tail_drop_base;
-			u64 ecn_base;
-			u64 wred_drop_base;
-		} red;
-	} xstats;
-
-	u64 tx_bytes;
-	u64 tx_packets;
-	u64 drops;
-	u64 overlimits;
-};
-
 /* No need an internal lock; At worse - miss a single periodic iteration */
 struct mlxsw_sp_port_xstats {
 	u64 ecn;
@@ -269,7 +262,10 @@ struct mlxsw_sp_port {
 	} periodic_hw_stats;
 	struct mlxsw_sp_port_sample *sample;
 	struct list_head vlans_list;
-	struct mlxsw_sp_qdisc root_qdisc;
+	struct mlxsw_sp_qdisc *root_qdisc;
+	unsigned acl_rule_count;
+	struct mlxsw_sp_acl_block *ing_acl_block;
+	struct mlxsw_sp_acl_block *eg_acl_block;
 };
 
 static inline bool
@@ -404,6 +400,16 @@ struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find(struct net_device *dev);
 struct mlxsw_sp_port *mlxsw_sp_port_lower_dev_hold(struct net_device *dev);
 void mlxsw_sp_port_dev_put(struct mlxsw_sp_port *mlxsw_sp_port);
 struct mlxsw_sp_port *mlxsw_sp_port_dev_lower_find_rcu(struct net_device *dev);
+int mlxsw_sp_span_mirror_add(struct mlxsw_sp_port *from,
+			     struct mlxsw_sp_port *to,
+			     enum mlxsw_sp_span_type type,
+			     bool bind);
+void mlxsw_sp_span_mirror_del(struct mlxsw_sp_port *from,
+			      u8 destination_port,
+			      enum mlxsw_sp_span_type type,
+			      bool bind);
+struct mlxsw_sp_span_entry *
+mlxsw_sp_span_entry_find(struct mlxsw_sp *mlxsw_sp, u8 local_port);
 
 /* spectrum_dcb.c */
 #ifdef CONFIG_MLXSW_SPECTRUM_DCB
@@ -458,13 +464,13 @@ void mlxsw_sp_kvdl_free(struct mlxsw_sp *mlxsw_sp, int entry_index);
 int mlxsw_sp_kvdl_alloc_size_query(struct mlxsw_sp *mlxsw_sp,
 				   unsigned int entry_count,
 				   unsigned int *p_alloc_size);
+u64 mlxsw_sp_kvdl_occ_get(const struct mlxsw_sp *mlxsw_sp);
 
 struct mlxsw_sp_acl_rule_info {
 	unsigned int priority;
 	struct mlxsw_afk_element_values values;
 	struct mlxsw_afa_block *act_block;
 	unsigned int counter_index;
-	bool counter_valid;
 };
 
 enum mlxsw_sp_acl_profile {
@@ -477,8 +483,11 @@ struct mlxsw_sp_acl_profile_ops {
 			   void *priv, void *ruleset_priv);
 	void (*ruleset_del)(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv);
 	int (*ruleset_bind)(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
-			    struct net_device *dev, bool ingress);
-	void (*ruleset_unbind)(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv);
+			    struct mlxsw_sp_port *mlxsw_sp_port,
+			    bool ingress);
+	void (*ruleset_unbind)(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
+			       struct mlxsw_sp_port *mlxsw_sp_port,
+			       bool ingress);
 	u16 (*ruleset_group_id)(void *ruleset_priv);
 	size_t rule_priv_size;
 	int (*rule_add)(struct mlxsw_sp *mlxsw_sp,
@@ -498,17 +507,34 @@ struct mlxsw_sp_acl_ops {
 				       enum mlxsw_sp_acl_profile profile);
 };
 
+struct mlxsw_sp_acl_block;
 struct mlxsw_sp_acl_ruleset;
 
 /* spectrum_acl.c */
 struct mlxsw_afk *mlxsw_sp_acl_afk(struct mlxsw_sp_acl *acl);
+struct mlxsw_sp *mlxsw_sp_acl_block_mlxsw_sp(struct mlxsw_sp_acl_block *block);
+unsigned int mlxsw_sp_acl_block_rule_count(struct mlxsw_sp_acl_block *block);
+void mlxsw_sp_acl_block_disable_inc(struct mlxsw_sp_acl_block *block);
+void mlxsw_sp_acl_block_disable_dec(struct mlxsw_sp_acl_block *block);
+bool mlxsw_sp_acl_block_disabled(struct mlxsw_sp_acl_block *block);
+struct mlxsw_sp_acl_block *mlxsw_sp_acl_block_create(struct mlxsw_sp *mlxsw_sp,
+						     struct net *net);
+void mlxsw_sp_acl_block_destroy(struct mlxsw_sp_acl_block *block);
+int mlxsw_sp_acl_block_bind(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block,
+			    struct mlxsw_sp_port *mlxsw_sp_port,
+			    bool ingress);
+int mlxsw_sp_acl_block_unbind(struct mlxsw_sp *mlxsw_sp,
+			      struct mlxsw_sp_acl_block *block,
+			      struct mlxsw_sp_port *mlxsw_sp_port,
+			      bool ingress);
 struct mlxsw_sp_acl_ruleset *
-mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
-			    bool ingress, u32 chain_index,
+mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block, u32 chain_index,
 			    enum mlxsw_sp_acl_profile profile);
 struct mlxsw_sp_acl_ruleset *
-mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
-			 bool ingress, u32 chain_index,
+mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp,
+			 struct mlxsw_sp_acl_block *block, u32 chain_index,
 			 enum mlxsw_sp_acl_profile profile);
 void mlxsw_sp_acl_ruleset_put(struct mlxsw_sp *mlxsw_sp,
 			      struct mlxsw_sp_acl_ruleset *ruleset);
@@ -530,8 +556,13 @@ void mlxsw_sp_acl_rulei_keymask_buf(struct mlxsw_sp_acl_rule_info *rulei,
 int mlxsw_sp_acl_rulei_act_continue(struct mlxsw_sp_acl_rule_info *rulei);
 int mlxsw_sp_acl_rulei_act_jump(struct mlxsw_sp_acl_rule_info *rulei,
 				u16 group_id);
+int mlxsw_sp_acl_rulei_act_terminate(struct mlxsw_sp_acl_rule_info *rulei);
 int mlxsw_sp_acl_rulei_act_drop(struct mlxsw_sp_acl_rule_info *rulei);
 int mlxsw_sp_acl_rulei_act_trap(struct mlxsw_sp_acl_rule_info *rulei);
+int mlxsw_sp_acl_rulei_act_mirror(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_acl_rule_info *rulei,
+				  struct mlxsw_sp_acl_block *block,
+				  struct net_device *out_dev);
 int mlxsw_sp_acl_rulei_act_fwd(struct mlxsw_sp *mlxsw_sp,
 			       struct mlxsw_sp_acl_rule_info *rulei,
 			       struct net_device *out_dev);
@@ -575,16 +606,23 @@ void mlxsw_sp_acl_fini(struct mlxsw_sp *mlxsw_sp);
 extern const struct mlxsw_sp_acl_ops mlxsw_sp_acl_tcam_ops;
 
 /* spectrum_flower.c */
-int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
+int mlxsw_sp_flower_replace(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block,
 			    struct tc_cls_flower_offload *f);
-void mlxsw_sp_flower_destroy(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
+void mlxsw_sp_flower_destroy(struct mlxsw_sp *mlxsw_sp,
+			     struct mlxsw_sp_acl_block *block,
 			     struct tc_cls_flower_offload *f);
-int mlxsw_sp_flower_stats(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
+int mlxsw_sp_flower_stats(struct mlxsw_sp *mlxsw_sp,
+			  struct mlxsw_sp_acl_block *block,
 			  struct tc_cls_flower_offload *f);
 
 /* spectrum_qdisc.c */
+int mlxsw_sp_tc_qdisc_init(struct mlxsw_sp_port *mlxsw_sp_port);
+void mlxsw_sp_tc_qdisc_fini(struct mlxsw_sp_port *mlxsw_sp_port);
 int mlxsw_sp_setup_tc_red(struct mlxsw_sp_port *mlxsw_sp_port,
 			  struct tc_red_qopt_offload *p);
+int mlxsw_sp_setup_tc_prio(struct mlxsw_sp_port *mlxsw_sp_port,
+			   struct tc_prio_qopt_offload *p);
 
 /* spectrum_fid.c */
 int mlxsw_sp_fid_flood_set(struct mlxsw_sp_fid *fid,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c
index 93dcd315f7d6..92d90ed7207e 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c
@@ -39,6 +39,7 @@
 #include <linux/string.h>
 #include <linux/rhashtable.h>
 #include <linux/netdevice.h>
+#include <net/net_namespace.h>
 #include <net/tc_act/tc_vlan.h>
 
 #include "reg.h"
@@ -70,9 +71,23 @@ struct mlxsw_afk *mlxsw_sp_acl_afk(struct mlxsw_sp_acl *acl)
 	return acl->afk;
 }
 
-struct mlxsw_sp_acl_ruleset_ht_key {
-	struct net_device *dev; /* dev this ruleset is bound to */
+struct mlxsw_sp_acl_block_binding {
+	struct list_head list;
+	struct net_device *dev;
+	struct mlxsw_sp_port *mlxsw_sp_port;
 	bool ingress;
+};
+
+struct mlxsw_sp_acl_block {
+	struct list_head binding_list;
+	struct mlxsw_sp_acl_ruleset *ruleset_zero;
+	struct mlxsw_sp *mlxsw_sp;
+	unsigned int rule_count;
+	unsigned int disable_count;
+};
+
+struct mlxsw_sp_acl_ruleset_ht_key {
+	struct mlxsw_sp_acl_block *block;
 	u32 chain_index;
 	const struct mlxsw_sp_acl_profile_ops *ops;
 };
@@ -118,8 +133,185 @@ struct mlxsw_sp_fid *mlxsw_sp_acl_dummy_fid(struct mlxsw_sp *mlxsw_sp)
 	return mlxsw_sp->acl->dummy_fid;
 }
 
+struct mlxsw_sp *mlxsw_sp_acl_block_mlxsw_sp(struct mlxsw_sp_acl_block *block)
+{
+	return block->mlxsw_sp;
+}
+
+unsigned int mlxsw_sp_acl_block_rule_count(struct mlxsw_sp_acl_block *block)
+{
+	return block ? block->rule_count : 0;
+}
+
+void mlxsw_sp_acl_block_disable_inc(struct mlxsw_sp_acl_block *block)
+{
+	if (block)
+		block->disable_count++;
+}
+
+void mlxsw_sp_acl_block_disable_dec(struct mlxsw_sp_acl_block *block)
+{
+	if (block)
+		block->disable_count--;
+}
+
+bool mlxsw_sp_acl_block_disabled(struct mlxsw_sp_acl_block *block)
+{
+	return block->disable_count;
+}
+
+static int
+mlxsw_sp_acl_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
+			  struct mlxsw_sp_acl_block *block,
+			  struct mlxsw_sp_acl_block_binding *binding)
+{
+	struct mlxsw_sp_acl_ruleset *ruleset = block->ruleset_zero;
+	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
+
+	return ops->ruleset_bind(mlxsw_sp, ruleset->priv,
+				 binding->mlxsw_sp_port, binding->ingress);
+}
+
+static void
+mlxsw_sp_acl_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block,
+			    struct mlxsw_sp_acl_block_binding *binding)
+{
+	struct mlxsw_sp_acl_ruleset *ruleset = block->ruleset_zero;
+	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
+
+	ops->ruleset_unbind(mlxsw_sp, ruleset->priv,
+			    binding->mlxsw_sp_port, binding->ingress);
+}
+
+static bool mlxsw_sp_acl_ruleset_block_bound(struct mlxsw_sp_acl_block *block)
+{
+	return block->ruleset_zero;
+}
+
+static int
+mlxsw_sp_acl_ruleset_block_bind(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_acl_ruleset *ruleset,
+				struct mlxsw_sp_acl_block *block)
+{
+	struct mlxsw_sp_acl_block_binding *binding;
+	int err;
+
+	block->ruleset_zero = ruleset;
+	list_for_each_entry(binding, &block->binding_list, list) {
+		err = mlxsw_sp_acl_ruleset_bind(mlxsw_sp, block, binding);
+		if (err)
+			goto rollback;
+	}
+	return 0;
+
+rollback:
+	list_for_each_entry_continue_reverse(binding, &block->binding_list,
+					     list)
+		mlxsw_sp_acl_ruleset_unbind(mlxsw_sp, block, binding);
+	block->ruleset_zero = NULL;
+
+	return err;
+}
+
+static void
+mlxsw_sp_acl_ruleset_block_unbind(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_acl_ruleset *ruleset,
+				  struct mlxsw_sp_acl_block *block)
+{
+	struct mlxsw_sp_acl_block_binding *binding;
+
+	list_for_each_entry(binding, &block->binding_list, list)
+		mlxsw_sp_acl_ruleset_unbind(mlxsw_sp, block, binding);
+	block->ruleset_zero = NULL;
+}
+
+struct mlxsw_sp_acl_block *mlxsw_sp_acl_block_create(struct mlxsw_sp *mlxsw_sp,
+						     struct net *net)
+{
+	struct mlxsw_sp_acl_block *block;
+
+	block = kzalloc(sizeof(*block), GFP_KERNEL);
+	if (!block)
+		return NULL;
+	INIT_LIST_HEAD(&block->binding_list);
+	block->mlxsw_sp = mlxsw_sp;
+	return block;
+}
+
+void mlxsw_sp_acl_block_destroy(struct mlxsw_sp_acl_block *block)
+{
+	WARN_ON(!list_empty(&block->binding_list));
+	kfree(block);
+}
+
+static struct mlxsw_sp_acl_block_binding *
+mlxsw_sp_acl_block_lookup(struct mlxsw_sp_acl_block *block,
+			  struct mlxsw_sp_port *mlxsw_sp_port, bool ingress)
+{
+	struct mlxsw_sp_acl_block_binding *binding;
+
+	list_for_each_entry(binding, &block->binding_list, list)
+		if (binding->mlxsw_sp_port == mlxsw_sp_port &&
+		    binding->ingress == ingress)
+			return binding;
+	return NULL;
+}
+
+int mlxsw_sp_acl_block_bind(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block,
+			    struct mlxsw_sp_port *mlxsw_sp_port,
+			    bool ingress)
+{
+	struct mlxsw_sp_acl_block_binding *binding;
+	int err;
+
+	if (WARN_ON(mlxsw_sp_acl_block_lookup(block, mlxsw_sp_port, ingress)))
+		return -EEXIST;
+
+	binding = kzalloc(sizeof(*binding), GFP_KERNEL);
+	if (!binding)
+		return -ENOMEM;
+	binding->mlxsw_sp_port = mlxsw_sp_port;
+	binding->ingress = ingress;
+
+	if (mlxsw_sp_acl_ruleset_block_bound(block)) {
+		err = mlxsw_sp_acl_ruleset_bind(mlxsw_sp, block, binding);
+		if (err)
+			goto err_ruleset_bind;
+	}
+
+	list_add(&binding->list, &block->binding_list);
+	return 0;
+
+err_ruleset_bind:
+	kfree(binding);
+	return err;
+}
+
+int mlxsw_sp_acl_block_unbind(struct mlxsw_sp *mlxsw_sp,
+			      struct mlxsw_sp_acl_block *block,
+			      struct mlxsw_sp_port *mlxsw_sp_port,
+			      bool ingress)
+{
+	struct mlxsw_sp_acl_block_binding *binding;
+
+	binding = mlxsw_sp_acl_block_lookup(block, mlxsw_sp_port, ingress);
+	if (!binding)
+		return -ENOENT;
+
+	list_del(&binding->list);
+
+	if (mlxsw_sp_acl_ruleset_block_bound(block))
+		mlxsw_sp_acl_ruleset_unbind(mlxsw_sp, block, binding);
+
+	kfree(binding);
+	return 0;
+}
+
 static struct mlxsw_sp_acl_ruleset *
 mlxsw_sp_acl_ruleset_create(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block, u32 chain_index,
 			    const struct mlxsw_sp_acl_profile_ops *ops)
 {
 	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
@@ -132,6 +324,8 @@ mlxsw_sp_acl_ruleset_create(struct mlxsw_sp *mlxsw_sp,
 	if (!ruleset)
 		return ERR_PTR(-ENOMEM);
 	ruleset->ref_count = 1;
+	ruleset->ht_key.block = block;
+	ruleset->ht_key.chain_index = chain_index;
 	ruleset->ht_key.ops = ops;
 
 	err = rhashtable_init(&ruleset->rule_ht, &mlxsw_sp_acl_rule_ht_params);
@@ -142,68 +336,50 @@ mlxsw_sp_acl_ruleset_create(struct mlxsw_sp *mlxsw_sp,
 	if (err)
 		goto err_ops_ruleset_add;
 
-	return ruleset;
-
-err_ops_ruleset_add:
-	rhashtable_destroy(&ruleset->rule_ht);
-err_rhashtable_init:
-	kfree(ruleset);
-	return ERR_PTR(err);
-}
-
-static void mlxsw_sp_acl_ruleset_destroy(struct mlxsw_sp *mlxsw_sp,
-					 struct mlxsw_sp_acl_ruleset *ruleset)
-{
-	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
-
-	ops->ruleset_del(mlxsw_sp, ruleset->priv);
-	rhashtable_destroy(&ruleset->rule_ht);
-	kfree(ruleset);
-}
-
-static int mlxsw_sp_acl_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
-				     struct mlxsw_sp_acl_ruleset *ruleset,
-				     struct net_device *dev, bool ingress,
-				     u32 chain_index)
-{
-	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
-	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
-	int err;
-
-	ruleset->ht_key.dev = dev;
-	ruleset->ht_key.ingress = ingress;
-	ruleset->ht_key.chain_index = chain_index;
 	err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node,
 				     mlxsw_sp_acl_ruleset_ht_params);
 	if (err)
-		return err;
-	if (!ruleset->ht_key.chain_index) {
+		goto err_ht_insert;
+
+	if (!chain_index) {
 		/* We only need ruleset with chain index 0, the implicit one,
 		 * to be directly bound to device. The rest of the rulesets
 		 * are bound by "Goto action set".
 		 */
-		err = ops->ruleset_bind(mlxsw_sp, ruleset->priv, dev, ingress);
+		err = mlxsw_sp_acl_ruleset_block_bind(mlxsw_sp, ruleset, block);
 		if (err)
-			goto err_ops_ruleset_bind;
+			goto err_ruleset_bind;
 	}
-	return 0;
 
-err_ops_ruleset_bind:
+	return ruleset;
+
+err_ruleset_bind:
 	rhashtable_remove_fast(&acl->ruleset_ht, &ruleset->ht_node,
 			       mlxsw_sp_acl_ruleset_ht_params);
-	return err;
+err_ht_insert:
+	ops->ruleset_del(mlxsw_sp, ruleset->priv);
+err_ops_ruleset_add:
+	rhashtable_destroy(&ruleset->rule_ht);
+err_rhashtable_init:
+	kfree(ruleset);
+	return ERR_PTR(err);
 }
 
-static void mlxsw_sp_acl_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
-					struct mlxsw_sp_acl_ruleset *ruleset)
+static void mlxsw_sp_acl_ruleset_destroy(struct mlxsw_sp *mlxsw_sp,
+					 struct mlxsw_sp_acl_ruleset *ruleset)
 {
 	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
+	struct mlxsw_sp_acl_block *block = ruleset->ht_key.block;
+	u32 chain_index = ruleset->ht_key.chain_index;
 	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
 
-	if (!ruleset->ht_key.chain_index)
-		ops->ruleset_unbind(mlxsw_sp, ruleset->priv);
+	if (!chain_index)
+		mlxsw_sp_acl_ruleset_block_unbind(mlxsw_sp, ruleset, block);
 	rhashtable_remove_fast(&acl->ruleset_ht, &ruleset->ht_node,
 			       mlxsw_sp_acl_ruleset_ht_params);
+	ops->ruleset_del(mlxsw_sp, ruleset->priv);
+	rhashtable_destroy(&ruleset->rule_ht);
+	kfree(ruleset);
 }
 
 static void mlxsw_sp_acl_ruleset_ref_inc(struct mlxsw_sp_acl_ruleset *ruleset)
@@ -216,20 +392,18 @@ static void mlxsw_sp_acl_ruleset_ref_dec(struct mlxsw_sp *mlxsw_sp,
 {
 	if (--ruleset->ref_count)
 		return;
-	mlxsw_sp_acl_ruleset_unbind(mlxsw_sp, ruleset);
 	mlxsw_sp_acl_ruleset_destroy(mlxsw_sp, ruleset);
 }
 
 static struct mlxsw_sp_acl_ruleset *
-__mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp_acl *acl, struct net_device *dev,
-			      bool ingress, u32 chain_index,
+__mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp_acl *acl,
+			      struct mlxsw_sp_acl_block *block, u32 chain_index,
 			      const struct mlxsw_sp_acl_profile_ops *ops)
 {
 	struct mlxsw_sp_acl_ruleset_ht_key ht_key;
 
 	memset(&ht_key, 0, sizeof(ht_key));
-	ht_key.dev = dev;
-	ht_key.ingress = ingress;
+	ht_key.block = block;
 	ht_key.chain_index = chain_index;
 	ht_key.ops = ops;
 	return rhashtable_lookup_fast(&acl->ruleset_ht, &ht_key,
@@ -237,8 +411,8 @@ __mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp_acl *acl, struct net_device *dev,
 }
 
 struct mlxsw_sp_acl_ruleset *
-mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
-			    bool ingress, u32 chain_index,
+mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block, u32 chain_index,
 			    enum mlxsw_sp_acl_profile profile)
 {
 	const struct mlxsw_sp_acl_profile_ops *ops;
@@ -248,45 +422,31 @@ mlxsw_sp_acl_ruleset_lookup(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
 	ops = acl->ops->profile_ops(mlxsw_sp, profile);
 	if (!ops)
 		return ERR_PTR(-EINVAL);
-	ruleset = __mlxsw_sp_acl_ruleset_lookup(acl, dev, ingress,
-						chain_index, ops);
+	ruleset = __mlxsw_sp_acl_ruleset_lookup(acl, block, chain_index, ops);
 	if (!ruleset)
 		return ERR_PTR(-ENOENT);
 	return ruleset;
 }
 
 struct mlxsw_sp_acl_ruleset *
-mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp, struct net_device *dev,
-			 bool ingress, u32 chain_index,
+mlxsw_sp_acl_ruleset_get(struct mlxsw_sp *mlxsw_sp,
+			 struct mlxsw_sp_acl_block *block, u32 chain_index,
 			 enum mlxsw_sp_acl_profile profile)
 {
 	const struct mlxsw_sp_acl_profile_ops *ops;
 	struct mlxsw_sp_acl *acl = mlxsw_sp->acl;
 	struct mlxsw_sp_acl_ruleset *ruleset;
-	int err;
 
 	ops = acl->ops->profile_ops(mlxsw_sp, profile);
 	if (!ops)
 		return ERR_PTR(-EINVAL);
 
-	ruleset = __mlxsw_sp_acl_ruleset_lookup(acl, dev, ingress,
-						chain_index, ops);
+	ruleset = __mlxsw_sp_acl_ruleset_lookup(acl, block, chain_index, ops);
 	if (ruleset) {
 		mlxsw_sp_acl_ruleset_ref_inc(ruleset);
 		return ruleset;
 	}
-	ruleset = mlxsw_sp_acl_ruleset_create(mlxsw_sp, ops);
-	if (IS_ERR(ruleset))
-		return ruleset;
-	err = mlxsw_sp_acl_ruleset_bind(mlxsw_sp, ruleset, dev,
-					ingress, chain_index);
-	if (err)
-		goto err_ruleset_bind;
-	return ruleset;
-
-err_ruleset_bind:
-	mlxsw_sp_acl_ruleset_destroy(mlxsw_sp, ruleset);
-	return ERR_PTR(err);
+	return mlxsw_sp_acl_ruleset_create(mlxsw_sp, block, chain_index, ops);
 }
 
 void mlxsw_sp_acl_ruleset_put(struct mlxsw_sp *mlxsw_sp,
@@ -302,27 +462,6 @@ u16 mlxsw_sp_acl_ruleset_group_id(struct mlxsw_sp_acl_ruleset *ruleset)
 	return ops->ruleset_group_id(ruleset->priv);
 }
 
-static int
-mlxsw_sp_acl_rulei_counter_alloc(struct mlxsw_sp *mlxsw_sp,
-				 struct mlxsw_sp_acl_rule_info *rulei)
-{
-	int err;
-
-	err = mlxsw_sp_flow_counter_alloc(mlxsw_sp, &rulei->counter_index);
-	if (err)
-		return err;
-	rulei->counter_valid = true;
-	return 0;
-}
-
-static void
-mlxsw_sp_acl_rulei_counter_free(struct mlxsw_sp *mlxsw_sp,
-				struct mlxsw_sp_acl_rule_info *rulei)
-{
-	rulei->counter_valid = false;
-	mlxsw_sp_flow_counter_free(mlxsw_sp, rulei->counter_index);
-}
-
 struct mlxsw_sp_acl_rule_info *
 mlxsw_sp_acl_rulei_create(struct mlxsw_sp_acl *acl)
 {
@@ -389,6 +528,11 @@ int mlxsw_sp_acl_rulei_act_jump(struct mlxsw_sp_acl_rule_info *rulei,
 	return mlxsw_afa_block_jump(rulei->act_block, group_id);
 }
 
+int mlxsw_sp_acl_rulei_act_terminate(struct mlxsw_sp_acl_rule_info *rulei)
+{
+	return mlxsw_afa_block_terminate(rulei->act_block);
+}
+
 int mlxsw_sp_acl_rulei_act_drop(struct mlxsw_sp_acl_rule_info *rulei)
 {
 	return mlxsw_afa_block_append_drop(rulei->act_block);
@@ -427,6 +571,34 @@ int mlxsw_sp_acl_rulei_act_fwd(struct mlxsw_sp *mlxsw_sp,
 					  local_port, in_port);
 }
 
+int mlxsw_sp_acl_rulei_act_mirror(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_acl_rule_info *rulei,
+				  struct mlxsw_sp_acl_block *block,
+				  struct net_device *out_dev)
+{
+	struct mlxsw_sp_acl_block_binding *binding;
+	struct mlxsw_sp_port *out_port;
+	struct mlxsw_sp_port *in_port;
+
+	if (!list_is_singular(&block->binding_list))
+		return -EOPNOTSUPP;
+
+	binding = list_first_entry(&block->binding_list,
+				   struct mlxsw_sp_acl_block_binding, list);
+	in_port = binding->mlxsw_sp_port;
+	if (!mlxsw_sp_port_dev_check(out_dev))
+		return -EINVAL;
+
+	out_port = netdev_priv(out_dev);
+	if (out_port->mlxsw_sp != mlxsw_sp)
+		return -EINVAL;
+
+	return mlxsw_afa_block_append_mirror(rulei->act_block,
+					     in_port->local_port,
+					     out_port->local_port,
+					     binding->ingress);
+}
+
 int mlxsw_sp_acl_rulei_act_vlan(struct mlxsw_sp *mlxsw_sp,
 				struct mlxsw_sp_acl_rule_info *rulei,
 				u32 action, u16 vid, u16 proto, u8 prio)
@@ -459,7 +631,7 @@ int mlxsw_sp_acl_rulei_act_count(struct mlxsw_sp *mlxsw_sp,
 				 struct mlxsw_sp_acl_rule_info *rulei)
 {
 	return mlxsw_afa_block_append_counter(rulei->act_block,
-					      rulei->counter_index);
+					      &rulei->counter_index);
 }
 
 int mlxsw_sp_acl_rulei_act_fid_set(struct mlxsw_sp *mlxsw_sp,
@@ -493,13 +665,8 @@ mlxsw_sp_acl_rule_create(struct mlxsw_sp *mlxsw_sp,
 		goto err_rulei_create;
 	}
 
-	err = mlxsw_sp_acl_rulei_counter_alloc(mlxsw_sp, rule->rulei);
-	if (err)
-		goto err_counter_alloc;
 	return rule;
 
-err_counter_alloc:
-	mlxsw_sp_acl_rulei_destroy(rule->rulei);
 err_rulei_create:
 	kfree(rule);
 err_alloc:
@@ -512,7 +679,6 @@ void mlxsw_sp_acl_rule_destroy(struct mlxsw_sp *mlxsw_sp,
 {
 	struct mlxsw_sp_acl_ruleset *ruleset = rule->ruleset;
 
-	mlxsw_sp_acl_rulei_counter_free(mlxsw_sp, rule->rulei);
 	mlxsw_sp_acl_rulei_destroy(rule->rulei);
 	kfree(rule);
 	mlxsw_sp_acl_ruleset_ref_dec(mlxsw_sp, ruleset);
@@ -535,6 +701,7 @@ int mlxsw_sp_acl_rule_add(struct mlxsw_sp *mlxsw_sp,
 		goto err_rhashtable_insert;
 
 	list_add_tail(&rule->list, &mlxsw_sp->acl->rules);
+	ruleset->ht_key.block->rule_count++;
 	return 0;
 
 err_rhashtable_insert:
@@ -548,6 +715,7 @@ void mlxsw_sp_acl_rule_del(struct mlxsw_sp *mlxsw_sp,
 	struct mlxsw_sp_acl_ruleset *ruleset = rule->ruleset;
 	const struct mlxsw_sp_acl_profile_ops *ops = ruleset->ht_key.ops;
 
+	ruleset->ht_key.block->rule_count--;
 	list_del(&rule->list);
 	rhashtable_remove_fast(&ruleset->rule_ht, &rule->ht_node,
 			       mlxsw_sp_acl_rule_ht_params);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c
index 4d3340ed0291..6ca6894125f0 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c
@@ -108,11 +108,77 @@ static void mlxsw_sp_act_kvdl_fwd_entry_del(void *priv, u32 kvdl_index)
 	mlxsw_sp_kvdl_free(mlxsw_sp, kvdl_index);
 }
 
+static int
+mlxsw_sp_act_counter_index_get(void *priv, unsigned int *p_counter_index)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	return mlxsw_sp_flow_counter_alloc(mlxsw_sp, p_counter_index);
+}
+
+static void
+mlxsw_sp_act_counter_index_put(void *priv, unsigned int counter_index)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+
+	mlxsw_sp_flow_counter_free(mlxsw_sp, counter_index);
+}
+
+static int
+mlxsw_sp_act_mirror_add(void *priv, u8 local_in_port, u8 local_out_port,
+			bool ingress, int *p_span_id)
+{
+	struct mlxsw_sp_port *in_port, *out_port;
+	struct mlxsw_sp_span_entry *span_entry;
+	struct mlxsw_sp *mlxsw_sp = priv;
+	enum mlxsw_sp_span_type type;
+	int err;
+
+	type = ingress ? MLXSW_SP_SPAN_INGRESS : MLXSW_SP_SPAN_EGRESS;
+	out_port = mlxsw_sp->ports[local_out_port];
+	in_port = mlxsw_sp->ports[local_in_port];
+
+	err = mlxsw_sp_span_mirror_add(in_port, out_port, type, false);
+	if (err)
+		return err;
+
+	span_entry = mlxsw_sp_span_entry_find(mlxsw_sp, local_out_port);
+	if (!span_entry) {
+		err = -ENOENT;
+		goto err_span_entry_find;
+	}
+
+	*p_span_id = span_entry->id;
+	return 0;
+
+err_span_entry_find:
+	mlxsw_sp_span_mirror_del(in_port, local_out_port, type, false);
+	return err;
+}
+
+static void
+mlxsw_sp_act_mirror_del(void *priv, u8 local_in_port, u8 local_out_port,
+			bool ingress)
+{
+	struct mlxsw_sp *mlxsw_sp = priv;
+	struct mlxsw_sp_port *in_port;
+	enum mlxsw_sp_span_type type;
+
+	type = ingress ? MLXSW_SP_SPAN_INGRESS : MLXSW_SP_SPAN_EGRESS;
+	in_port = mlxsw_sp->ports[local_in_port];
+
+	mlxsw_sp_span_mirror_del(in_port, local_out_port, type, false);
+}
+
 static const struct mlxsw_afa_ops mlxsw_sp_act_afa_ops = {
 	.kvdl_set_add		= mlxsw_sp_act_kvdl_set_add,
 	.kvdl_set_del		= mlxsw_sp_act_kvdl_set_del,
 	.kvdl_fwd_entry_add	= mlxsw_sp_act_kvdl_fwd_entry_add,
 	.kvdl_fwd_entry_del	= mlxsw_sp_act_kvdl_fwd_entry_del,
+	.counter_index_get	= mlxsw_sp_act_counter_index_get,
+	.counter_index_put	= mlxsw_sp_act_counter_index_put,
+	.mirror_add		= mlxsw_sp_act_mirror_add,
+	.mirror_del		= mlxsw_sp_act_mirror_del,
 };
 
 int mlxsw_sp_afa_init(struct mlxsw_sp *mlxsw_sp)
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c
index 7e8284b46968..c6e180c2be1e 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c
@@ -154,10 +154,6 @@ struct mlxsw_sp_acl_tcam_group {
 	struct list_head region_list;
 	unsigned int region_count;
 	struct rhashtable chunk_ht;
-	struct {
-		u16 local_port;
-		bool ingress;
-	} bound;
 	struct mlxsw_sp_acl_tcam_group_ops *ops;
 	const struct mlxsw_sp_acl_tcam_pattern *patterns;
 	unsigned int patterns_count;
@@ -262,35 +258,29 @@ static void mlxsw_sp_acl_tcam_group_del(struct mlxsw_sp *mlxsw_sp,
 static int
 mlxsw_sp_acl_tcam_group_bind(struct mlxsw_sp *mlxsw_sp,
 			     struct mlxsw_sp_acl_tcam_group *group,
-			     struct net_device *dev, bool ingress)
+			     struct mlxsw_sp_port *mlxsw_sp_port,
+			     bool ingress)
 {
-	struct mlxsw_sp_port *mlxsw_sp_port;
 	char ppbt_pl[MLXSW_REG_PPBT_LEN];
 
-	if (!mlxsw_sp_port_dev_check(dev))
-		return -EINVAL;
-
-	mlxsw_sp_port = netdev_priv(dev);
-	group->bound.local_port = mlxsw_sp_port->local_port;
-	group->bound.ingress = ingress;
-	mlxsw_reg_ppbt_pack(ppbt_pl,
-			    group->bound.ingress ? MLXSW_REG_PXBT_E_IACL :
-						   MLXSW_REG_PXBT_E_EACL,
-			    MLXSW_REG_PXBT_OP_BIND, group->bound.local_port,
+	mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
+					       MLXSW_REG_PXBT_E_EACL,
+			    MLXSW_REG_PXBT_OP_BIND, mlxsw_sp_port->local_port,
 			    group->id);
 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
 }
 
 static void
 mlxsw_sp_acl_tcam_group_unbind(struct mlxsw_sp *mlxsw_sp,
-			       struct mlxsw_sp_acl_tcam_group *group)
+			       struct mlxsw_sp_acl_tcam_group *group,
+			       struct mlxsw_sp_port *mlxsw_sp_port,
+			       bool ingress)
 {
 	char ppbt_pl[MLXSW_REG_PPBT_LEN];
 
-	mlxsw_reg_ppbt_pack(ppbt_pl,
-			    group->bound.ingress ? MLXSW_REG_PXBT_E_IACL :
-						   MLXSW_REG_PXBT_E_EACL,
-			    MLXSW_REG_PXBT_OP_UNBIND, group->bound.local_port,
+	mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
+					       MLXSW_REG_PXBT_E_EACL,
+			    MLXSW_REG_PXBT_OP_UNBIND, mlxsw_sp_port->local_port,
 			    group->id);
 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
 }
@@ -1056,21 +1046,25 @@ mlxsw_sp_acl_tcam_flower_ruleset_del(struct mlxsw_sp *mlxsw_sp,
 static int
 mlxsw_sp_acl_tcam_flower_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
 				      void *ruleset_priv,
-				      struct net_device *dev, bool ingress)
+				      struct mlxsw_sp_port *mlxsw_sp_port,
+				      bool ingress)
 {
 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
 
 	return mlxsw_sp_acl_tcam_group_bind(mlxsw_sp, &ruleset->group,
-					    dev, ingress);
+					    mlxsw_sp_port, ingress);
 }
 
 static void
 mlxsw_sp_acl_tcam_flower_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
-					void *ruleset_priv)
+					void *ruleset_priv,
+					struct mlxsw_sp_port *mlxsw_sp_port,
+					bool ingress)
 {
 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
 
-	mlxsw_sp_acl_tcam_group_unbind(mlxsw_sp, &ruleset->group);
+	mlxsw_sp_acl_tcam_group_unbind(mlxsw_sp, &ruleset->group,
+				       mlxsw_sp_port, ingress);
 }
 
 static u16
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
index 93728c694e6d..0a9adc5962fb 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c
@@ -385,13 +385,13 @@ static const struct mlxsw_sp_sb_cm mlxsw_sp_sb_cms_egress[] = {
 
 static const struct mlxsw_sp_sb_cm mlxsw_sp_cpu_port_sb_cms[] = {
 	MLXSW_SP_CPU_PORT_SB_CM,
+	MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 0),
+	MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 0),
+	MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 0),
+	MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 0),
+	MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 0),
 	MLXSW_SP_CPU_PORT_SB_CM,
-	MLXSW_SP_CPU_PORT_SB_CM,
-	MLXSW_SP_CPU_PORT_SB_CM,
-	MLXSW_SP_CPU_PORT_SB_CM,
-	MLXSW_SP_CPU_PORT_SB_CM,
-	MLXSW_SP_CPU_PORT_SB_CM,
-	MLXSW_SP_SB_CM(10000, 0, 0),
+	MLXSW_SP_SB_CM(MLXSW_PORT_MAX_MTU, 0, 0),
 	MLXSW_SP_CPU_PORT_SB_CM,
 	MLXSW_SP_CPU_PORT_SB_CM,
 	MLXSW_SP_CPU_PORT_SB_CM,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c
index 96fdba78acab..f56fa18d6b26 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_dpipe.c
@@ -771,14 +771,33 @@ static struct devlink_dpipe_table_ops mlxsw_sp_host4_ops = {
 	.size_get = mlxsw_sp_dpipe_table_host4_size_get,
 };
 
+#define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4 1
+
 static int mlxsw_sp_dpipe_host4_table_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
+	int err;
 
-	return devlink_dpipe_table_register(devlink,
-					    MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
-					    &mlxsw_sp_host4_ops,
-					    mlxsw_sp, false);
+	err = devlink_dpipe_table_register(devlink,
+					   MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
+					   &mlxsw_sp_host4_ops,
+					   mlxsw_sp, false);
+	if (err)
+		return err;
+
+	err = devlink_dpipe_table_resource_set(devlink,
+					       MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
+					       MLXSW_SP_RESOURCE_KVD_HASH_SINGLE,
+					       MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4);
+	if (err)
+		goto err_resource_set;
+
+	return 0;
+
+err_resource_set:
+	devlink_dpipe_table_unregister(devlink,
+				       MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
+	return err;
 }
 
 static void mlxsw_sp_dpipe_host4_table_fini(struct mlxsw_sp *mlxsw_sp)
@@ -829,14 +848,33 @@ static struct devlink_dpipe_table_ops mlxsw_sp_host6_ops = {
 	.size_get = mlxsw_sp_dpipe_table_host6_size_get,
 };
 
+#define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6 2
+
 static int mlxsw_sp_dpipe_host6_table_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
+	int err;
 
-	return devlink_dpipe_table_register(devlink,
-					    MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
-					    &mlxsw_sp_host6_ops,
-					    mlxsw_sp, false);
+	err = devlink_dpipe_table_register(devlink,
+					   MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
+					   &mlxsw_sp_host6_ops,
+					   mlxsw_sp, false);
+	if (err)
+		return err;
+
+	err = devlink_dpipe_table_resource_set(devlink,
+					       MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
+					       MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE,
+					       MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6);
+	if (err)
+		goto err_resource_set;
+
+	return 0;
+
+err_resource_set:
+	devlink_dpipe_table_unregister(devlink,
+				       MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
+	return err;
 }
 
 static void mlxsw_sp_dpipe_host6_table_fini(struct mlxsw_sp *mlxsw_sp)
@@ -1213,14 +1251,33 @@ static struct devlink_dpipe_table_ops mlxsw_sp_dpipe_table_adj_ops = {
 	.size_get = mlxsw_sp_dpipe_table_adj_size_get,
 };
 
+#define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ 1
+
 static int mlxsw_sp_dpipe_adj_table_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
+	int err;
 
-	return devlink_dpipe_table_register(devlink,
-					    MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
-					    &mlxsw_sp_dpipe_table_adj_ops,
-					    mlxsw_sp, false);
+	err = devlink_dpipe_table_register(devlink,
+					   MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
+					   &mlxsw_sp_dpipe_table_adj_ops,
+					   mlxsw_sp, false);
+	if (err)
+		return err;
+
+	err = devlink_dpipe_table_resource_set(devlink,
+					       MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
+					       MLXSW_SP_RESOURCE_KVD_LINEAR,
+					       MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ);
+	if (err)
+		goto err_resource_set;
+
+	return 0;
+
+err_resource_set:
+	devlink_dpipe_table_unregister(devlink,
+				       MLXSW_SP_DPIPE_TABLE_NAME_ADJ);
+	return err;
 }
 
 static void mlxsw_sp_dpipe_adj_table_fini(struct mlxsw_sp *mlxsw_sp)
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
index bbd238e50f05..54262af4e98f 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c
@@ -112,11 +112,11 @@ static const int mlxsw_sp_sfgc_bc_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = {
 	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_NON_IP]	= 1,
 	[MLXSW_REG_SFGC_TYPE_IPV4_LINK_LOCAL]			= 1,
 	[MLXSW_REG_SFGC_TYPE_IPV6_ALL_HOST]			= 1,
+	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV6]	= 1,
 };
 
 static const int mlxsw_sp_sfgc_mc_packet_types[MLXSW_REG_SFGC_TYPE_MAX] = {
 	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV4]	= 1,
-	[MLXSW_REG_SFGC_TYPE_UNREGISTERED_MULTICAST_IPV6]	= 1,
 };
 
 static const int *mlxsw_sp_packet_type_sfgc_types[] = {
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c
index 2f0e57857ea4..89dbf569dff5 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c
@@ -35,6 +35,7 @@
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/netdevice.h>
+#include <net/net_namespace.h>
 #include <net/flow_dissector.h>
 #include <net/pkt_cls.h>
 #include <net/tc_act/tc_gact.h>
@@ -45,7 +46,7 @@
 #include "core_acl_flex_keys.h"
 
 static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
-					 struct net_device *dev, bool ingress,
+					 struct mlxsw_sp_acl_block *block,
 					 struct mlxsw_sp_acl_rule_info *rulei,
 					 struct tcf_exts *exts)
 {
@@ -64,7 +65,7 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
 	tcf_exts_to_list(exts, &actions);
 	list_for_each_entry(a, &actions, list) {
 		if (is_tcf_gact_ok(a)) {
-			err = mlxsw_sp_acl_rulei_act_continue(rulei);
+			err = mlxsw_sp_acl_rulei_act_terminate(rulei);
 			if (err)
 				return err;
 		} else if (is_tcf_gact_shot(a)) {
@@ -80,8 +81,7 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
 			struct mlxsw_sp_acl_ruleset *ruleset;
 			u16 group_id;
 
-			ruleset = mlxsw_sp_acl_ruleset_lookup(mlxsw_sp, dev,
-							      ingress,
+			ruleset = mlxsw_sp_acl_ruleset_lookup(mlxsw_sp, block,
 							      chain_index,
 							      MLXSW_SP_ACL_PROFILE_FLOWER);
 			if (IS_ERR(ruleset))
@@ -92,7 +92,6 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
 			if (err)
 				return err;
 		} else if (is_tcf_mirred_egress_redirect(a)) {
-			int ifindex = tcf_mirred_ifindex(a);
 			struct net_device *out_dev;
 			struct mlxsw_sp_fid *fid;
 			u16 fid_index;
@@ -104,14 +103,18 @@ static int mlxsw_sp_flower_parse_actions(struct mlxsw_sp *mlxsw_sp,
 			if (err)
 				return err;
 
-			out_dev = __dev_get_by_index(dev_net(dev), ifindex);
-			if (out_dev == dev)
-				out_dev = NULL;
-
+			out_dev = tcf_mirred_dev(a);
 			err = mlxsw_sp_acl_rulei_act_fwd(mlxsw_sp, rulei,
 							 out_dev);
 			if (err)
 				return err;
+		} else if (is_tcf_mirred_egress_mirror(a)) {
+			struct net_device *out_dev = tcf_mirred_dev(a);
+
+			err = mlxsw_sp_acl_rulei_act_mirror(mlxsw_sp, rulei,
+							    block, out_dev);
+			if (err)
+				return err;
 		} else if (is_tcf_vlan(a)) {
 			u16 proto = be16_to_cpu(tcf_vlan_push_proto(a));
 			u32 action = tcf_vlan_action(a);
@@ -266,7 +269,7 @@ static int mlxsw_sp_flower_parse_ip(struct mlxsw_sp *mlxsw_sp,
 }
 
 static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
-				 struct net_device *dev, bool ingress,
+				 struct mlxsw_sp_acl_block *block,
 				 struct mlxsw_sp_acl_rule_info *rulei,
 				 struct tc_cls_flower_offload *f)
 {
@@ -384,21 +387,19 @@ static int mlxsw_sp_flower_parse(struct mlxsw_sp *mlxsw_sp,
 	if (err)
 		return err;
 
-	return mlxsw_sp_flower_parse_actions(mlxsw_sp, dev, ingress,
-					     rulei, f->exts);
+	return mlxsw_sp_flower_parse_actions(mlxsw_sp, block, rulei, f->exts);
 }
 
-int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
+int mlxsw_sp_flower_replace(struct mlxsw_sp *mlxsw_sp,
+			    struct mlxsw_sp_acl_block *block,
 			    struct tc_cls_flower_offload *f)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-	struct net_device *dev = mlxsw_sp_port->dev;
 	struct mlxsw_sp_acl_rule_info *rulei;
 	struct mlxsw_sp_acl_ruleset *ruleset;
 	struct mlxsw_sp_acl_rule *rule;
 	int err;
 
-	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, dev, ingress,
+	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, block,
 					   f->common.chain_index,
 					   MLXSW_SP_ACL_PROFILE_FLOWER);
 	if (IS_ERR(ruleset))
@@ -411,7 +412,7 @@ int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	}
 
 	rulei = mlxsw_sp_acl_rule_rulei(rule);
-	err = mlxsw_sp_flower_parse(mlxsw_sp, dev, ingress, rulei, f);
+	err = mlxsw_sp_flower_parse(mlxsw_sp, block, rulei, f);
 	if (err)
 		goto err_flower_parse;
 
@@ -435,15 +436,15 @@ int mlxsw_sp_flower_replace(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	return err;
 }
 
-void mlxsw_sp_flower_destroy(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
+void mlxsw_sp_flower_destroy(struct mlxsw_sp *mlxsw_sp,
+			     struct mlxsw_sp_acl_block *block,
 			     struct tc_cls_flower_offload *f)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	struct mlxsw_sp_acl_ruleset *ruleset;
 	struct mlxsw_sp_acl_rule *rule;
 
-	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, mlxsw_sp_port->dev,
-					   ingress, f->common.chain_index,
+	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, block,
+					   f->common.chain_index,
 					   MLXSW_SP_ACL_PROFILE_FLOWER);
 	if (IS_ERR(ruleset))
 		return;
@@ -457,10 +458,10 @@ void mlxsw_sp_flower_destroy(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	mlxsw_sp_acl_ruleset_put(mlxsw_sp, ruleset);
 }
 
-int mlxsw_sp_flower_stats(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
+int mlxsw_sp_flower_stats(struct mlxsw_sp *mlxsw_sp,
+			  struct mlxsw_sp_acl_block *block,
 			  struct tc_cls_flower_offload *f)
 {
-	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 	struct mlxsw_sp_acl_ruleset *ruleset;
 	struct mlxsw_sp_acl_rule *rule;
 	u64 packets;
@@ -468,8 +469,8 @@ int mlxsw_sp_flower_stats(struct mlxsw_sp_port *mlxsw_sp_port, bool ingress,
 	u64 bytes;
 	int err;
 
-	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, mlxsw_sp_port->dev,
-					   ingress, f->common.chain_index,
+	ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, block,
+					   f->common.chain_index,
 					   MLXSW_SP_ACL_PROFILE_FLOWER);
 	if (WARN_ON(IS_ERR(ruleset)))
 		return -EINVAL;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_kvdl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_kvdl.c
index 310c38247b5c..55f9d2d70f9e 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_kvdl.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_kvdl.c
@@ -286,6 +286,32 @@ static void mlxsw_sp_kvdl_parts_fini(struct mlxsw_sp *mlxsw_sp)
 		mlxsw_sp_kvdl_part_fini(mlxsw_sp, i);
 }
 
+static u64 mlxsw_sp_kvdl_part_occ(struct mlxsw_sp_kvdl_part *part)
+{
+	unsigned int nr_entries;
+	int bit = -1;
+	u64 occ = 0;
+
+	nr_entries = (part->info->end_index -
+		      part->info->start_index + 1) /
+		      part->info->alloc_size;
+	while ((bit = find_next_bit(part->usage, nr_entries, bit + 1))
+		< nr_entries)
+		occ += part->info->alloc_size;
+	return occ;
+}
+
+u64 mlxsw_sp_kvdl_occ_get(const struct mlxsw_sp *mlxsw_sp)
+{
+	struct mlxsw_sp_kvdl_part *part;
+	u64 occ = 0;
+
+	list_for_each_entry(part, &mlxsw_sp->kvdl->parts_list, list)
+		occ += mlxsw_sp_kvdl_part_occ(part);
+
+	return occ;
+}
+
 int mlxsw_sp_kvdl_init(struct mlxsw_sp *mlxsw_sp)
 {
 	struct mlxsw_sp_kvdl *kvdl;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c
index 34a0b632e5dd..4c7f32d4288d 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_mr_tcam.c
@@ -243,7 +243,8 @@ mlxsw_sp_mr_tcam_afa_block_create(struct mlxsw_sp *mlxsw_sp,
 	if (!afa_block)
 		return ERR_PTR(-ENOMEM);
 
-	err = mlxsw_afa_block_append_counter(afa_block, counter_index);
+	err = mlxsw_afa_block_append_allocated_counter(afa_block,
+						       counter_index);
 	if (err)
 		goto err;
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c
index b5397da94d7f..0b7670459051 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c
@@ -41,6 +41,150 @@
 #include "spectrum.h"
 #include "reg.h"
 
+#define MLXSW_SP_PRIO_BAND_TO_TCLASS(band) (IEEE_8021QAZ_MAX_TCS - band - 1)
+
+enum mlxsw_sp_qdisc_type {
+	MLXSW_SP_QDISC_NO_QDISC,
+	MLXSW_SP_QDISC_RED,
+	MLXSW_SP_QDISC_PRIO,
+};
+
+struct mlxsw_sp_qdisc_ops {
+	enum mlxsw_sp_qdisc_type type;
+	int (*check_params)(struct mlxsw_sp_port *mlxsw_sp_port,
+			    struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			    void *params);
+	int (*replace)(struct mlxsw_sp_port *mlxsw_sp_port,
+		       struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, void *params);
+	int (*destroy)(struct mlxsw_sp_port *mlxsw_sp_port,
+		       struct mlxsw_sp_qdisc *mlxsw_sp_qdisc);
+	int (*get_stats)(struct mlxsw_sp_port *mlxsw_sp_port,
+			 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			 struct tc_qopt_offload_stats *stats_ptr);
+	int (*get_xstats)(struct mlxsw_sp_port *mlxsw_sp_port,
+			  struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			  void *xstats_ptr);
+	void (*clean_stats)(struct mlxsw_sp_port *mlxsw_sp_port,
+			    struct mlxsw_sp_qdisc *mlxsw_sp_qdisc);
+	/* unoffload - to be used for a qdisc that stops being offloaded without
+	 * being destroyed.
+	 */
+	void (*unoffload)(struct mlxsw_sp_port *mlxsw_sp_port,
+			  struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, void *params);
+};
+
+struct mlxsw_sp_qdisc {
+	u32 handle;
+	u8 tclass_num;
+	union {
+		struct red_stats red;
+	} xstats_base;
+	struct mlxsw_sp_qdisc_stats {
+		u64 tx_bytes;
+		u64 tx_packets;
+		u64 drops;
+		u64 overlimits;
+		u64 backlog;
+	} stats_base;
+
+	struct mlxsw_sp_qdisc_ops *ops;
+};
+
+static bool
+mlxsw_sp_qdisc_compare(struct mlxsw_sp_qdisc *mlxsw_sp_qdisc, u32 handle,
+		       enum mlxsw_sp_qdisc_type type)
+{
+	return mlxsw_sp_qdisc && mlxsw_sp_qdisc->ops &&
+	       mlxsw_sp_qdisc->ops->type == type &&
+	       mlxsw_sp_qdisc->handle == handle;
+}
+
+static int
+mlxsw_sp_qdisc_destroy(struct mlxsw_sp_port *mlxsw_sp_port,
+		       struct mlxsw_sp_qdisc *mlxsw_sp_qdisc)
+{
+	int err = 0;
+
+	if (!mlxsw_sp_qdisc)
+		return 0;
+
+	if (mlxsw_sp_qdisc->ops && mlxsw_sp_qdisc->ops->destroy)
+		err = mlxsw_sp_qdisc->ops->destroy(mlxsw_sp_port,
+						   mlxsw_sp_qdisc);
+
+	mlxsw_sp_qdisc->handle = TC_H_UNSPEC;
+	mlxsw_sp_qdisc->ops = NULL;
+	return err;
+}
+
+static int
+mlxsw_sp_qdisc_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle,
+		       struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+		       struct mlxsw_sp_qdisc_ops *ops, void *params)
+{
+	int err;
+
+	if (mlxsw_sp_qdisc->ops && mlxsw_sp_qdisc->ops->type != ops->type)
+		/* In case this location contained a different qdisc of the
+		 * same type we can override the old qdisc configuration.
+		 * Otherwise, we need to remove the old qdisc before setting the
+		 * new one.
+		 */
+		mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc);
+	err = ops->check_params(mlxsw_sp_port, mlxsw_sp_qdisc, params);
+	if (err)
+		goto err_bad_param;
+
+	err = ops->replace(mlxsw_sp_port, mlxsw_sp_qdisc, params);
+	if (err)
+		goto err_config;
+
+	if (mlxsw_sp_qdisc->handle != handle) {
+		mlxsw_sp_qdisc->ops = ops;
+		if (ops->clean_stats)
+			ops->clean_stats(mlxsw_sp_port, mlxsw_sp_qdisc);
+	}
+
+	mlxsw_sp_qdisc->handle = handle;
+	return 0;
+
+err_bad_param:
+err_config:
+	if (mlxsw_sp_qdisc->handle == handle && ops->unoffload)
+		ops->unoffload(mlxsw_sp_port, mlxsw_sp_qdisc, params);
+
+	mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc);
+	return err;
+}
+
+static int
+mlxsw_sp_qdisc_get_stats(struct mlxsw_sp_port *mlxsw_sp_port,
+			 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			 struct tc_qopt_offload_stats *stats_ptr)
+{
+	if (mlxsw_sp_qdisc && mlxsw_sp_qdisc->ops &&
+	    mlxsw_sp_qdisc->ops->get_stats)
+		return mlxsw_sp_qdisc->ops->get_stats(mlxsw_sp_port,
+						      mlxsw_sp_qdisc,
+						      stats_ptr);
+
+	return -EOPNOTSUPP;
+}
+
+static int
+mlxsw_sp_qdisc_get_xstats(struct mlxsw_sp_port *mlxsw_sp_port,
+			  struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			  void *xstats_ptr)
+{
+	if (mlxsw_sp_qdisc && mlxsw_sp_qdisc->ops &&
+	    mlxsw_sp_qdisc->ops->get_xstats)
+		return mlxsw_sp_qdisc->ops->get_xstats(mlxsw_sp_port,
+						      mlxsw_sp_qdisc,
+						      xstats_ptr);
+
+	return -EOPNOTSUPP;
+}
+
 static int
 mlxsw_sp_tclass_congestion_enable(struct mlxsw_sp_port *mlxsw_sp_port,
 				  int tclass_num, u32 min, u32 max,
@@ -80,80 +224,78 @@ mlxsw_sp_tclass_congestion_disable(struct mlxsw_sp_port *mlxsw_sp_port,
 }
 
 static void
-mlxsw_sp_setup_tc_qdisc_clean_stats(struct mlxsw_sp_port *mlxsw_sp_port,
-				    struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
-				    int tclass_num)
+mlxsw_sp_setup_tc_qdisc_red_clean_stats(struct mlxsw_sp_port *mlxsw_sp_port,
+					struct mlxsw_sp_qdisc *mlxsw_sp_qdisc)
 {
-	struct red_stats *xstats_base = &mlxsw_sp_qdisc->xstats_base;
+	u8 tclass_num = mlxsw_sp_qdisc->tclass_num;
+	struct mlxsw_sp_qdisc_stats *stats_base;
 	struct mlxsw_sp_port_xstats *xstats;
 	struct rtnl_link_stats64 *stats;
+	struct red_stats *red_base;
 
 	xstats = &mlxsw_sp_port->periodic_hw_stats.xstats;
 	stats = &mlxsw_sp_port->periodic_hw_stats.stats;
+	stats_base = &mlxsw_sp_qdisc->stats_base;
+	red_base = &mlxsw_sp_qdisc->xstats_base.red;
 
-	mlxsw_sp_qdisc->tx_packets = stats->tx_packets;
-	mlxsw_sp_qdisc->tx_bytes = stats->tx_bytes;
+	stats_base->tx_packets = stats->tx_packets;
+	stats_base->tx_bytes = stats->tx_bytes;
 
-	switch (mlxsw_sp_qdisc->type) {
-	case MLXSW_SP_QDISC_RED:
-		xstats_base->prob_mark = xstats->ecn;
-		xstats_base->prob_drop = xstats->wred_drop[tclass_num];
-		xstats_base->pdrop = xstats->tail_drop[tclass_num];
+	red_base->prob_mark = xstats->ecn;
+	red_base->prob_drop = xstats->wred_drop[tclass_num];
+	red_base->pdrop = xstats->tail_drop[tclass_num];
 
-		mlxsw_sp_qdisc->overlimits = xstats_base->prob_drop +
-					     xstats_base->prob_mark;
-		mlxsw_sp_qdisc->drops = xstats_base->prob_drop +
-					xstats_base->pdrop;
-		break;
-	default:
-		break;
-	}
+	stats_base->overlimits = red_base->prob_drop + red_base->prob_mark;
+	stats_base->drops = red_base->prob_drop + red_base->pdrop;
+
+	stats_base->backlog = 0;
 }
 
 static int
-mlxsw_sp_qdisc_red_destroy(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle,
-			   struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
-			   int tclass_num)
+mlxsw_sp_qdisc_red_destroy(struct mlxsw_sp_port *mlxsw_sp_port,
+			   struct mlxsw_sp_qdisc *mlxsw_sp_qdisc)
 {
-	int err;
-
-	if (mlxsw_sp_qdisc->handle != handle)
-		return 0;
-
-	err = mlxsw_sp_tclass_congestion_disable(mlxsw_sp_port, tclass_num);
-	mlxsw_sp_qdisc->handle = TC_H_UNSPEC;
-	mlxsw_sp_qdisc->type = MLXSW_SP_QDISC_NO_QDISC;
-
-	return err;
+	return mlxsw_sp_tclass_congestion_disable(mlxsw_sp_port,
+						  mlxsw_sp_qdisc->tclass_num);
 }
 
 static int
-mlxsw_sp_qdisc_red_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle,
-			   struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
-			   int tclass_num,
-			   struct tc_red_qopt_offload_params *p)
+mlxsw_sp_qdisc_red_check_params(struct mlxsw_sp_port *mlxsw_sp_port,
+				struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+				void *params)
 {
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
-	u32 min, max;
-	u64 prob;
-	int err = 0;
+	struct tc_red_qopt_offload_params *p = params;
 
 	if (p->min > p->max) {
 		dev_err(mlxsw_sp->bus_info->dev,
 			"spectrum: RED: min %u is bigger then max %u\n", p->min,
 			p->max);
-		goto err_bad_param;
+		return -EINVAL;
 	}
 	if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) {
 		dev_err(mlxsw_sp->bus_info->dev,
 			"spectrum: RED: max value %u is too big\n", p->max);
-		goto err_bad_param;
+		return -EINVAL;
 	}
 	if (p->min == 0 || p->max == 0) {
 		dev_err(mlxsw_sp->bus_info->dev,
 			"spectrum: RED: 0 value is illegal for min and max\n");
-		goto err_bad_param;
+		return -EINVAL;
 	}
+	return 0;
+}
+
+static int
+mlxsw_sp_qdisc_red_replace(struct mlxsw_sp_port *mlxsw_sp_port,
+			   struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			   void *params)
+{
+	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
+	struct tc_red_qopt_offload_params *p = params;
+	u8 tclass_num = mlxsw_sp_qdisc->tclass_num;
+	u32 min, max;
+	u64 prob;
 
 	/* calculate probability in percentage */
 	prob = p->probability;
@@ -162,116 +304,309 @@ mlxsw_sp_qdisc_red_replace(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle,
 	prob = DIV_ROUND_UP(prob, 1 << 16);
 	min = mlxsw_sp_bytes_cells(mlxsw_sp, p->min);
 	max = mlxsw_sp_bytes_cells(mlxsw_sp, p->max);
-	err = mlxsw_sp_tclass_congestion_enable(mlxsw_sp_port, tclass_num, min,
-						max, prob, p->is_ecn);
-	if (err)
-		goto err_config;
-
-	mlxsw_sp_qdisc->type = MLXSW_SP_QDISC_RED;
-	if (mlxsw_sp_qdisc->handle != handle)
-		mlxsw_sp_setup_tc_qdisc_clean_stats(mlxsw_sp_port,
-						    mlxsw_sp_qdisc,
-						    tclass_num);
+	return mlxsw_sp_tclass_congestion_enable(mlxsw_sp_port, tclass_num, min,
+						 max, prob, p->is_ecn);
+}
 
-	mlxsw_sp_qdisc->handle = handle;
-	return 0;
+static void
+mlxsw_sp_qdisc_red_unoffload(struct mlxsw_sp_port *mlxsw_sp_port,
+			     struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			     void *params)
+{
+	struct tc_red_qopt_offload_params *p = params;
+	u64 backlog;
 
-err_bad_param:
-	err = -EINVAL;
-err_config:
-	mlxsw_sp_qdisc_red_destroy(mlxsw_sp_port, mlxsw_sp_qdisc->handle,
-				   mlxsw_sp_qdisc, tclass_num);
-	return err;
+	backlog = mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp,
+				       mlxsw_sp_qdisc->stats_base.backlog);
+	p->qstats->backlog -= backlog;
 }
 
 static int
-mlxsw_sp_qdisc_get_red_xstats(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle,
+mlxsw_sp_qdisc_get_red_xstats(struct mlxsw_sp_port *mlxsw_sp_port,
 			      struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
-			      int tclass_num, struct red_stats *res)
+			      void *xstats_ptr)
 {
-	struct red_stats *xstats_base = &mlxsw_sp_qdisc->xstats_base;
+	struct red_stats *xstats_base = &mlxsw_sp_qdisc->xstats_base.red;
+	u8 tclass_num = mlxsw_sp_qdisc->tclass_num;
 	struct mlxsw_sp_port_xstats *xstats;
-
-	if (mlxsw_sp_qdisc->handle != handle ||
-	    mlxsw_sp_qdisc->type != MLXSW_SP_QDISC_RED)
-		return -EOPNOTSUPP;
+	struct red_stats *res = xstats_ptr;
+	int early_drops, marks, pdrops;
 
 	xstats = &mlxsw_sp_port->periodic_hw_stats.xstats;
 
-	res->prob_drop = xstats->wred_drop[tclass_num] - xstats_base->prob_drop;
-	res->prob_mark = xstats->ecn - xstats_base->prob_mark;
-	res->pdrop = xstats->tail_drop[tclass_num] - xstats_base->pdrop;
+	early_drops = xstats->wred_drop[tclass_num] - xstats_base->prob_drop;
+	marks = xstats->ecn - xstats_base->prob_mark;
+	pdrops = xstats->tail_drop[tclass_num] - xstats_base->pdrop;
+
+	res->pdrop += pdrops;
+	res->prob_drop += early_drops;
+	res->prob_mark += marks;
+
+	xstats_base->pdrop += pdrops;
+	xstats_base->prob_drop += early_drops;
+	xstats_base->prob_mark += marks;
 	return 0;
 }
 
 static int
-mlxsw_sp_qdisc_get_red_stats(struct mlxsw_sp_port *mlxsw_sp_port, u32 handle,
+mlxsw_sp_qdisc_get_red_stats(struct mlxsw_sp_port *mlxsw_sp_port,
 			     struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
-			     int tclass_num,
-			     struct tc_red_qopt_offload_stats *res)
+			     struct tc_qopt_offload_stats *stats_ptr)
 {
-	u64 tx_bytes, tx_packets, overlimits, drops;
+	u64 tx_bytes, tx_packets, overlimits, drops, backlog;
+	u8 tclass_num = mlxsw_sp_qdisc->tclass_num;
+	struct mlxsw_sp_qdisc_stats *stats_base;
 	struct mlxsw_sp_port_xstats *xstats;
 	struct rtnl_link_stats64 *stats;
 
-	if (mlxsw_sp_qdisc->handle != handle ||
-	    mlxsw_sp_qdisc->type != MLXSW_SP_QDISC_RED)
-		return -EOPNOTSUPP;
-
 	xstats = &mlxsw_sp_port->periodic_hw_stats.xstats;
 	stats = &mlxsw_sp_port->periodic_hw_stats.stats;
+	stats_base = &mlxsw_sp_qdisc->stats_base;
 
-	tx_bytes = stats->tx_bytes - mlxsw_sp_qdisc->tx_bytes;
-	tx_packets = stats->tx_packets - mlxsw_sp_qdisc->tx_packets;
+	tx_bytes = stats->tx_bytes - stats_base->tx_bytes;
+	tx_packets = stats->tx_packets - stats_base->tx_packets;
 	overlimits = xstats->wred_drop[tclass_num] + xstats->ecn -
-		     mlxsw_sp_qdisc->overlimits;
+		     stats_base->overlimits;
 	drops = xstats->wred_drop[tclass_num] + xstats->tail_drop[tclass_num] -
-		mlxsw_sp_qdisc->drops;
-
-	_bstats_update(res->bstats, tx_bytes, tx_packets);
-	res->qstats->overlimits += overlimits;
-	res->qstats->drops += drops;
-	res->qstats->backlog += mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp,
-						xstats->backlog[tclass_num]);
-
-	mlxsw_sp_qdisc->drops +=  drops;
-	mlxsw_sp_qdisc->overlimits += overlimits;
-	mlxsw_sp_qdisc->tx_bytes += tx_bytes;
-	mlxsw_sp_qdisc->tx_packets += tx_packets;
+		stats_base->drops;
+	backlog = xstats->backlog[tclass_num];
+
+	_bstats_update(stats_ptr->bstats, tx_bytes, tx_packets);
+	stats_ptr->qstats->overlimits += overlimits;
+	stats_ptr->qstats->drops += drops;
+	stats_ptr->qstats->backlog +=
+				mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp,
+						     backlog) -
+				mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp,
+						     stats_base->backlog);
+
+	stats_base->backlog = backlog;
+	stats_base->drops +=  drops;
+	stats_base->overlimits += overlimits;
+	stats_base->tx_bytes += tx_bytes;
+	stats_base->tx_packets += tx_packets;
 	return 0;
 }
 
 #define MLXSW_SP_PORT_DEFAULT_TCLASS 0
 
+static struct mlxsw_sp_qdisc_ops mlxsw_sp_qdisc_ops_red = {
+	.type = MLXSW_SP_QDISC_RED,
+	.check_params = mlxsw_sp_qdisc_red_check_params,
+	.replace = mlxsw_sp_qdisc_red_replace,
+	.unoffload = mlxsw_sp_qdisc_red_unoffload,
+	.destroy = mlxsw_sp_qdisc_red_destroy,
+	.get_stats = mlxsw_sp_qdisc_get_red_stats,
+	.get_xstats = mlxsw_sp_qdisc_get_red_xstats,
+	.clean_stats = mlxsw_sp_setup_tc_qdisc_red_clean_stats,
+};
+
 int mlxsw_sp_setup_tc_red(struct mlxsw_sp_port *mlxsw_sp_port,
 			  struct tc_red_qopt_offload *p)
 {
 	struct mlxsw_sp_qdisc *mlxsw_sp_qdisc;
-	int tclass_num;
 
 	if (p->parent != TC_H_ROOT)
 		return -EOPNOTSUPP;
 
-	mlxsw_sp_qdisc = &mlxsw_sp_port->root_qdisc;
-	tclass_num = MLXSW_SP_PORT_DEFAULT_TCLASS;
+	mlxsw_sp_qdisc = mlxsw_sp_port->root_qdisc;
+
+	if (p->command == TC_RED_REPLACE)
+		return mlxsw_sp_qdisc_replace(mlxsw_sp_port, p->handle,
+					      mlxsw_sp_qdisc,
+					      &mlxsw_sp_qdisc_ops_red,
+					      &p->set);
+
+	if (!mlxsw_sp_qdisc_compare(mlxsw_sp_qdisc, p->handle,
+				    MLXSW_SP_QDISC_RED))
+		return -EOPNOTSUPP;
 
 	switch (p->command) {
-	case TC_RED_REPLACE:
-		return mlxsw_sp_qdisc_red_replace(mlxsw_sp_port, p->handle,
-						  mlxsw_sp_qdisc, tclass_num,
-						  &p->set);
 	case TC_RED_DESTROY:
-		return mlxsw_sp_qdisc_red_destroy(mlxsw_sp_port, p->handle,
-						  mlxsw_sp_qdisc, tclass_num);
+		return mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc);
 	case TC_RED_XSTATS:
-		return mlxsw_sp_qdisc_get_red_xstats(mlxsw_sp_port, p->handle,
-						     mlxsw_sp_qdisc, tclass_num,
-						     p->xstats);
+		return mlxsw_sp_qdisc_get_xstats(mlxsw_sp_port, mlxsw_sp_qdisc,
+						 p->xstats);
 	case TC_RED_STATS:
-		return mlxsw_sp_qdisc_get_red_stats(mlxsw_sp_port, p->handle,
-						    mlxsw_sp_qdisc, tclass_num,
-						    &p->stats);
+		return mlxsw_sp_qdisc_get_stats(mlxsw_sp_port, mlxsw_sp_qdisc,
+						&p->stats);
 	default:
 		return -EOPNOTSUPP;
 	}
 }
+
+static int
+mlxsw_sp_qdisc_prio_destroy(struct mlxsw_sp_port *mlxsw_sp_port,
+			    struct mlxsw_sp_qdisc *mlxsw_sp_qdisc)
+{
+	int i;
+
+	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++)
+		mlxsw_sp_port_prio_tc_set(mlxsw_sp_port, i,
+					  MLXSW_SP_PORT_DEFAULT_TCLASS);
+
+	return 0;
+}
+
+static int
+mlxsw_sp_qdisc_prio_check_params(struct mlxsw_sp_port *mlxsw_sp_port,
+				 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+				 void *params)
+{
+	struct tc_prio_qopt_offload_params *p = params;
+
+	if (p->bands > IEEE_8021QAZ_MAX_TCS)
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+static int
+mlxsw_sp_qdisc_prio_replace(struct mlxsw_sp_port *mlxsw_sp_port,
+			    struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			    void *params)
+{
+	struct tc_prio_qopt_offload_params *p = params;
+	int tclass, i;
+	int err;
+
+	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
+		tclass = MLXSW_SP_PRIO_BAND_TO_TCLASS(p->priomap[i]);
+		err = mlxsw_sp_port_prio_tc_set(mlxsw_sp_port, i, tclass);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static void
+mlxsw_sp_qdisc_prio_unoffload(struct mlxsw_sp_port *mlxsw_sp_port,
+			      struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			      void *params)
+{
+	struct tc_prio_qopt_offload_params *p = params;
+	u64 backlog;
+
+	backlog = mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp,
+				       mlxsw_sp_qdisc->stats_base.backlog);
+	p->qstats->backlog -= backlog;
+}
+
+static int
+mlxsw_sp_qdisc_get_prio_stats(struct mlxsw_sp_port *mlxsw_sp_port,
+			      struct mlxsw_sp_qdisc *mlxsw_sp_qdisc,
+			      struct tc_qopt_offload_stats *stats_ptr)
+{
+	u64 tx_bytes, tx_packets, drops = 0, backlog = 0;
+	struct mlxsw_sp_qdisc_stats *stats_base;
+	struct mlxsw_sp_port_xstats *xstats;
+	struct rtnl_link_stats64 *stats;
+	int i;
+
+	xstats = &mlxsw_sp_port->periodic_hw_stats.xstats;
+	stats = &mlxsw_sp_port->periodic_hw_stats.stats;
+	stats_base = &mlxsw_sp_qdisc->stats_base;
+
+	tx_bytes = stats->tx_bytes - stats_base->tx_bytes;
+	tx_packets = stats->tx_packets - stats_base->tx_packets;
+
+	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) {
+		drops += xstats->tail_drop[i];
+		backlog += xstats->backlog[i];
+	}
+	drops = drops - stats_base->drops;
+
+	_bstats_update(stats_ptr->bstats, tx_bytes, tx_packets);
+	stats_ptr->qstats->drops += drops;
+	stats_ptr->qstats->backlog +=
+				mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp,
+						     backlog) -
+				mlxsw_sp_cells_bytes(mlxsw_sp_port->mlxsw_sp,
+						     stats_base->backlog);
+	stats_base->backlog = backlog;
+	stats_base->drops += drops;
+	stats_base->tx_bytes += tx_bytes;
+	stats_base->tx_packets += tx_packets;
+	return 0;
+}
+
+static void
+mlxsw_sp_setup_tc_qdisc_prio_clean_stats(struct mlxsw_sp_port *mlxsw_sp_port,
+					 struct mlxsw_sp_qdisc *mlxsw_sp_qdisc)
+{
+	struct mlxsw_sp_qdisc_stats *stats_base;
+	struct mlxsw_sp_port_xstats *xstats;
+	struct rtnl_link_stats64 *stats;
+	int i;
+
+	xstats = &mlxsw_sp_port->periodic_hw_stats.xstats;
+	stats = &mlxsw_sp_port->periodic_hw_stats.stats;
+	stats_base = &mlxsw_sp_qdisc->stats_base;
+
+	stats_base->tx_packets = stats->tx_packets;
+	stats_base->tx_bytes = stats->tx_bytes;
+
+	stats_base->drops = 0;
+	for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++)
+		stats_base->drops += xstats->tail_drop[i];
+
+	mlxsw_sp_qdisc->stats_base.backlog = 0;
+}
+
+static struct mlxsw_sp_qdisc_ops mlxsw_sp_qdisc_ops_prio = {
+	.type = MLXSW_SP_QDISC_PRIO,
+	.check_params = mlxsw_sp_qdisc_prio_check_params,
+	.replace = mlxsw_sp_qdisc_prio_replace,
+	.unoffload = mlxsw_sp_qdisc_prio_unoffload,
+	.destroy = mlxsw_sp_qdisc_prio_destroy,
+	.get_stats = mlxsw_sp_qdisc_get_prio_stats,
+	.clean_stats = mlxsw_sp_setup_tc_qdisc_prio_clean_stats,
+};
+
+int mlxsw_sp_setup_tc_prio(struct mlxsw_sp_port *mlxsw_sp_port,
+			   struct tc_prio_qopt_offload *p)
+{
+	struct mlxsw_sp_qdisc *mlxsw_sp_qdisc;
+
+	if (p->parent != TC_H_ROOT)
+		return -EOPNOTSUPP;
+
+	mlxsw_sp_qdisc = mlxsw_sp_port->root_qdisc;
+	if (p->command == TC_PRIO_REPLACE)
+		return mlxsw_sp_qdisc_replace(mlxsw_sp_port, p->handle,
+					      mlxsw_sp_qdisc,
+					      &mlxsw_sp_qdisc_ops_prio,
+					      &p->replace_params);
+
+	if (!mlxsw_sp_qdisc_compare(mlxsw_sp_qdisc, p->handle,
+				    MLXSW_SP_QDISC_PRIO))
+		return -EOPNOTSUPP;
+
+	switch (p->command) {
+	case TC_PRIO_DESTROY:
+		return mlxsw_sp_qdisc_destroy(mlxsw_sp_port, mlxsw_sp_qdisc);
+	case TC_PRIO_STATS:
+		return mlxsw_sp_qdisc_get_stats(mlxsw_sp_port, mlxsw_sp_qdisc,
+						&p->stats);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+int mlxsw_sp_tc_qdisc_init(struct mlxsw_sp_port *mlxsw_sp_port)
+{
+	mlxsw_sp_port->root_qdisc = kzalloc(sizeof(*mlxsw_sp_port->root_qdisc),
+					    GFP_KERNEL);
+	if (!mlxsw_sp_port->root_qdisc)
+		return -ENOMEM;
+
+	mlxsw_sp_port->root_qdisc->tclass_num = MLXSW_SP_PORT_DEFAULT_TCLASS;
+
+	return 0;
+}
+
+void mlxsw_sp_tc_qdisc_fini(struct mlxsw_sp_port *mlxsw_sp_port)
+{
+	kfree(mlxsw_sp_port->root_qdisc);
+}
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 7042c855a5d6..997e24dcb053 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -71,6 +71,7 @@
 #include "spectrum_mr_tcam.h"
 #include "spectrum_router.h"
 
+struct mlxsw_sp_fib;
 struct mlxsw_sp_vr;
 struct mlxsw_sp_lpm_tree;
 struct mlxsw_sp_rif_ops;
@@ -84,6 +85,8 @@ struct mlxsw_sp_router {
 	struct rhashtable nexthop_ht;
 	struct list_head nexthop_list;
 	struct {
+		/* One tree for each protocol: IPv4 and IPv6 */
+		struct mlxsw_sp_lpm_tree *proto_trees[2];
 		struct mlxsw_sp_lpm_tree *trees;
 		unsigned int tree_count;
 	} lpm;
@@ -162,6 +165,15 @@ struct mlxsw_sp_rif_ops {
 	struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 };
 
+static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
+static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
+				  struct mlxsw_sp_lpm_tree *lpm_tree);
+static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
+				     const struct mlxsw_sp_fib *fib,
+				     u8 tree_id);
+static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
+				       const struct mlxsw_sp_fib *fib);
+
 static unsigned int *
 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 			   enum mlxsw_sp_rif_counter_dir dir)
@@ -349,14 +361,6 @@ mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 	return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 }
 
-static bool
-mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
-{
-	struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
-
-	return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
-}
-
 static void
 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 			  struct mlxsw_sp_prefix_usage *prefix_usage2)
@@ -398,7 +402,6 @@ enum mlxsw_sp_fib_entry_type {
 };
 
 struct mlxsw_sp_nexthop_group;
-struct mlxsw_sp_fib;
 
 struct mlxsw_sp_fib_node {
 	struct list_head entry_list;
@@ -445,6 +448,7 @@ struct mlxsw_sp_lpm_tree {
 	u8 id; /* tree ID */
 	unsigned int ref_count;
 	enum mlxsw_sp_l3proto proto;
+	unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 	struct mlxsw_sp_prefix_usage prefix_usage;
 };
 
@@ -453,8 +457,6 @@ struct mlxsw_sp_fib {
 	struct list_head node_list;
 	struct mlxsw_sp_vr *vr;
 	struct mlxsw_sp_lpm_tree *lpm_tree;
-	unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
-	struct mlxsw_sp_prefix_usage prefix_usage;
 	enum mlxsw_sp_l3proto proto;
 };
 
@@ -469,12 +471,15 @@ struct mlxsw_sp_vr {
 
 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 
-static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
+static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
+						struct mlxsw_sp_vr *vr,
 						enum mlxsw_sp_l3proto proto)
 {
+	struct mlxsw_sp_lpm_tree *lpm_tree;
 	struct mlxsw_sp_fib *fib;
 	int err;
 
+	lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 	fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 	if (!fib)
 		return ERR_PTR(-ENOMEM);
@@ -484,17 +489,26 @@ static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 	INIT_LIST_HEAD(&fib->node_list);
 	fib->proto = proto;
 	fib->vr = vr;
+	fib->lpm_tree = lpm_tree;
+	mlxsw_sp_lpm_tree_hold(lpm_tree);
+	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
+	if (err)
+		goto err_lpm_tree_bind;
 	return fib;
 
+err_lpm_tree_bind:
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 err_rhashtable_init:
 	kfree(fib);
 	return ERR_PTR(err);
 }
 
-static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
+static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
+				 struct mlxsw_sp_fib *fib)
 {
+	mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 	WARN_ON(!list_empty(&fib->node_list));
-	WARN_ON(fib->lpm_tree);
 	rhashtable_destroy(&fib->ht);
 	kfree(fib);
 }
@@ -581,6 +595,9 @@ mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 		goto err_left_struct_set;
 	memcpy(&lpm_tree->prefix_usage, prefix_usage,
 	       sizeof(lpm_tree->prefix_usage));
+	memset(&lpm_tree->prefix_ref_count, 0,
+	       sizeof(lpm_tree->prefix_ref_count));
+	lpm_tree->ref_count = 1;
 	return lpm_tree;
 
 err_left_struct_set:
@@ -607,8 +624,10 @@ mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 		if (lpm_tree->ref_count != 0 &&
 		    lpm_tree->proto == proto &&
 		    mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
-					     prefix_usage))
+					     prefix_usage)) {
+			mlxsw_sp_lpm_tree_hold(lpm_tree);
 			return lpm_tree;
+		}
 	}
 	return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 }
@@ -629,9 +648,10 @@ static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 
 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 {
+	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 	struct mlxsw_sp_lpm_tree *lpm_tree;
 	u64 max_trees;
-	int i;
+	int err, i;
 
 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 		return -EIO;
@@ -649,11 +669,42 @@ static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 		lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 	}
 
+	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
+					 MLXSW_SP_L3_PROTO_IPV4);
+	if (IS_ERR(lpm_tree)) {
+		err = PTR_ERR(lpm_tree);
+		goto err_ipv4_tree_get;
+	}
+	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
+
+	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
+					 MLXSW_SP_L3_PROTO_IPV6);
+	if (IS_ERR(lpm_tree)) {
+		err = PTR_ERR(lpm_tree);
+		goto err_ipv6_tree_get;
+	}
+	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
+
 	return 0;
+
+err_ipv6_tree_get:
+	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
+err_ipv4_tree_get:
+	kfree(mlxsw_sp->router->lpm.trees);
+	return err;
 }
 
 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 {
+	struct mlxsw_sp_lpm_tree *lpm_tree;
+
+	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
+
+	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
+
 	kfree(mlxsw_sp->router->lpm.trees);
 }
 
@@ -737,6 +788,9 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 					      u32 tb_id,
 					      struct netlink_ext_ack *extack)
 {
+	struct mlxsw_sp_mr_table *mr4_table;
+	struct mlxsw_sp_fib *fib4;
+	struct mlxsw_sp_fib *fib6;
 	struct mlxsw_sp_vr *vr;
 	int err;
 
@@ -745,39 +799,41 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 		NL_SET_ERR_MSG(extack, "spectrum: Exceeded number of supported virtual routers");
 		return ERR_PTR(-EBUSY);
 	}
-	vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
-	if (IS_ERR(vr->fib4))
-		return ERR_CAST(vr->fib4);
-	vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
-	if (IS_ERR(vr->fib6)) {
-		err = PTR_ERR(vr->fib6);
+	fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
+	if (IS_ERR(fib4))
+		return ERR_CAST(fib4);
+	fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
+	if (IS_ERR(fib6)) {
+		err = PTR_ERR(fib6);
 		goto err_fib6_create;
 	}
-	vr->mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
-						 MLXSW_SP_L3_PROTO_IPV4);
-	if (IS_ERR(vr->mr4_table)) {
-		err = PTR_ERR(vr->mr4_table);
+	mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
+					     MLXSW_SP_L3_PROTO_IPV4);
+	if (IS_ERR(mr4_table)) {
+		err = PTR_ERR(mr4_table);
 		goto err_mr_table_create;
 	}
+	vr->fib4 = fib4;
+	vr->fib6 = fib6;
+	vr->mr4_table = mr4_table;
 	vr->tb_id = tb_id;
 	return vr;
 
 err_mr_table_create:
-	mlxsw_sp_fib_destroy(vr->fib6);
-	vr->fib6 = NULL;
+	mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 err_fib6_create:
-	mlxsw_sp_fib_destroy(vr->fib4);
-	vr->fib4 = NULL;
+	mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 	return ERR_PTR(err);
 }
 
-static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
+static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
+				struct mlxsw_sp_vr *vr)
 {
 	mlxsw_sp_mr_table_destroy(vr->mr4_table);
 	vr->mr4_table = NULL;
-	mlxsw_sp_fib_destroy(vr->fib6);
+	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 	vr->fib6 = NULL;
-	mlxsw_sp_fib_destroy(vr->fib4);
+	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 	vr->fib4 = NULL;
 }
 
@@ -793,12 +849,12 @@ static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 	return vr;
 }
 
-static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
+static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 {
 	if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 	    list_empty(&vr->fib6->node_list) &&
 	    mlxsw_sp_mr_table_empty(vr->mr4_table))
-		mlxsw_sp_vr_destroy(vr);
+		mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 }
 
 static bool
@@ -809,7 +865,7 @@ mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 
 	if (!mlxsw_sp_vr_is_used(vr))
 		return false;
-	if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
+	if (fib->lpm_tree->id == tree_id)
 		return true;
 	return false;
 }
@@ -839,14 +895,13 @@ static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 					 struct mlxsw_sp_fib *fib,
 					 struct mlxsw_sp_lpm_tree *new_tree)
 {
-	struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 	enum mlxsw_sp_l3proto proto = fib->proto;
+	struct mlxsw_sp_lpm_tree *old_tree;
 	u8 old_id, new_id = new_tree->id;
 	struct mlxsw_sp_vr *vr;
 	int i, err;
 
-	if (!old_tree)
-		goto no_replace;
+	old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 	old_id = old_tree->id;
 
 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
@@ -860,6 +915,11 @@ static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 			goto err_tree_replace;
 	}
 
+	memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
+	       sizeof(new_tree->prefix_ref_count));
+	mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
+
 	return 0;
 
 err_tree_replace:
@@ -871,36 +931,6 @@ static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 					     old_tree);
 	}
 	return err;
-
-no_replace:
-	fib->lpm_tree = new_tree;
-	mlxsw_sp_lpm_tree_hold(new_tree);
-	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
-	if (err) {
-		mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
-		fib->lpm_tree = NULL;
-		return err;
-	}
-	return 0;
-}
-
-static void
-mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
-		      enum mlxsw_sp_l3proto proto,
-		      struct mlxsw_sp_prefix_usage *req_prefix_usage)
-{
-	int i;
-
-	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
-		struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
-		struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
-		unsigned char prefix;
-
-		if (!mlxsw_sp_vr_is_used(vr))
-			continue;
-		mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
-			mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
-	}
 }
 
 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
@@ -1350,6 +1380,55 @@ mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
 						  decap_fib_entry);
 }
 
+static int
+mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
+			struct mlxsw_sp_vr *ul_vr, bool enable)
+{
+	struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
+	struct mlxsw_sp_rif *rif = &lb_rif->common;
+	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
+	char ritr_pl[MLXSW_REG_RITR_LEN];
+	u32 saddr4;
+
+	switch (lb_cf.ul_protocol) {
+	case MLXSW_SP_L3_PROTO_IPV4:
+		saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
+		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
+				    rif->rif_index, rif->vr_id, rif->dev->mtu);
+		mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
+			    MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
+			    ul_vr->id, saddr4, lb_cf.okey);
+		break;
+
+	case MLXSW_SP_L3_PROTO_IPV6:
+		return -EAFNOSUPPORT;
+	}
+
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
+}
+
+static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
+						 struct net_device *ol_dev)
+{
+	struct mlxsw_sp_ipip_entry *ipip_entry;
+	struct mlxsw_sp_rif_ipip_lb *lb_rif;
+	struct mlxsw_sp_vr *ul_vr;
+	int err = 0;
+
+	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
+	if (ipip_entry) {
+		lb_rif = ipip_entry->ol_lb;
+		ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
+		err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
+		if (err)
+			goto out;
+		lb_rif->common.mtu = ol_dev->mtu;
+	}
+
+out:
+	return err;
+}
+
 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
 						struct net_device *ol_dev)
 {
@@ -1630,6 +1709,8 @@ int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
 		extack = info->extack;
 		return mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
 							       ol_dev, extack);
+	case NETDEV_CHANGEMTU:
+		return mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
 	}
 	return 0;
 }
@@ -2630,7 +2711,8 @@ struct mlxsw_sp_nexthop_group_cmp_arg {
 
 static bool
 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
-				    const struct in6_addr *gw, int ifindex)
+				    const struct in6_addr *gw, int ifindex,
+				    int weight)
 {
 	int i;
 
@@ -2638,7 +2720,7 @@ mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
 		const struct mlxsw_sp_nexthop *nh;
 
 		nh = &nh_grp->nexthops[i];
-		if (nh->ifindex == ifindex &&
+		if (nh->ifindex == ifindex && nh->nh_weight == weight &&
 		    ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
 			return true;
 	}
@@ -2657,11 +2739,13 @@ mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
 
 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
 		struct in6_addr *gw;
-		int ifindex;
+		int ifindex, weight;
 
 		ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
+		weight = mlxsw_sp_rt6->rt->rt6i_nh_weight;
 		gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
-		if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
+		if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
+							 weight))
 			return false;
 	}
 
@@ -3761,6 +3845,9 @@ mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
 	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
 	int i;
 
+	if (!list_is_singular(&nh_grp->fib_list))
+		return;
+
 	for (i = 0; i < nh_grp->count; i++) {
 		struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
 
@@ -4192,68 +4279,66 @@ mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
 }
 
 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
-				      struct mlxsw_sp_fib *fib,
 				      struct mlxsw_sp_fib_node *fib_node)
 {
-	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
+	struct mlxsw_sp_prefix_usage req_prefix_usage;
+	struct mlxsw_sp_fib *fib = fib_node->fib;
 	struct mlxsw_sp_lpm_tree *lpm_tree;
 	int err;
 
-	/* Since the tree is shared between all virtual routers we must
-	 * make sure it contains all the required prefix lengths. This
-	 * can be computed by either adding the new prefix length to the
-	 * existing prefix usage of a bound tree, or by aggregating the
-	 * prefix lengths across all virtual routers and adding the new
-	 * one as well.
-	 */
-	if (fib->lpm_tree)
-		mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
-					  &fib->lpm_tree->prefix_usage);
-	else
-		mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
-	mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
+	lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
+	if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
+		goto out;
 
+	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
+	mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 					 fib->proto);
 	if (IS_ERR(lpm_tree))
 		return PTR_ERR(lpm_tree);
 
-	if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
-		return 0;
-
 	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
 	if (err)
-		return err;
+		goto err_lpm_tree_replace;
 
+out:
+	lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
 	return 0;
-}
 
-static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
-					 struct mlxsw_sp_fib *fib)
-{
-	if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
-		return;
-	mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
-	mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
-	fib->lpm_tree = NULL;
+err_lpm_tree_replace:
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
+	return err;
 }
 
-static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
+static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
+					 struct mlxsw_sp_fib_node *fib_node)
 {
-	unsigned char prefix_len = fib_node->key.prefix_len;
+	struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
+	struct mlxsw_sp_prefix_usage req_prefix_usage;
 	struct mlxsw_sp_fib *fib = fib_node->fib;
+	int err;
 
-	if (fib->prefix_ref_count[prefix_len]++ == 0)
-		mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
-}
+	if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
+		return;
+	/* Try to construct a new LPM tree from the current prefix usage
+	 * minus the unused one. If we fail, continue using the old one.
+	 */
+	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
+	mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
+				    fib_node->key.prefix_len);
+	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
+					 fib->proto);
+	if (IS_ERR(lpm_tree))
+		return;
 
-static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
-{
-	unsigned char prefix_len = fib_node->key.prefix_len;
-	struct mlxsw_sp_fib *fib = fib_node->fib;
+	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
+	if (err)
+		goto err_lpm_tree_replace;
+
+	return;
 
-	if (--fib->prefix_ref_count[prefix_len] == 0)
-		mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
+err_lpm_tree_replace:
+	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 }
 
 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
@@ -4267,12 +4352,10 @@ static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
 		return err;
 	fib_node->fib = fib;
 
-	err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
+	err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
 	if (err)
 		goto err_fib_lpm_tree_link;
 
-	mlxsw_sp_fib_node_prefix_inc(fib_node);
-
 	return 0;
 
 err_fib_lpm_tree_link:
@@ -4286,8 +4369,7 @@ static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
 {
 	struct mlxsw_sp_fib *fib = fib_node->fib;
 
-	mlxsw_sp_fib_node_prefix_dec(fib_node);
-	mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
+	mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
 	fib_node->fib = NULL;
 	mlxsw_sp_fib_node_remove(fib, fib_node);
 }
@@ -4326,7 +4408,7 @@ mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
 err_fib_node_init:
 	mlxsw_sp_fib_node_destroy(fib_node);
 err_fib_node_create:
-	mlxsw_sp_vr_put(vr);
+	mlxsw_sp_vr_put(mlxsw_sp, vr);
 	return ERR_PTR(err);
 }
 
@@ -4339,7 +4421,7 @@ static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
 		return;
 	mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
 	mlxsw_sp_fib_node_destroy(fib_node);
-	mlxsw_sp_vr_put(vr);
+	mlxsw_sp_vr_put(mlxsw_sp, vr);
 }
 
 static struct mlxsw_sp_fib4_entry *
@@ -4764,7 +4846,7 @@ static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
 	struct net_device *dev = rt->dst.dev;
 
 	nh->nh_grp = nh_grp;
-	nh->nh_weight = 1;
+	nh->nh_weight = rt->rt6i_nh_weight;
 	memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
 
@@ -5362,7 +5444,7 @@ static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
 		return;
 
 	mlxsw_sp_mr_route4_del(vr->mr4_table, men_info->mfc);
-	mlxsw_sp_vr_put(vr);
+	mlxsw_sp_vr_put(mlxsw_sp, vr);
 }
 
 static int
@@ -5399,7 +5481,7 @@ mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
 		return;
 
 	mlxsw_sp_mr_vif_del(vr->mr4_table, ven_info->vif_index);
-	mlxsw_sp_vr_put(vr);
+	mlxsw_sp_vr_put(mlxsw_sp, vr);
 }
 
 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
@@ -6048,7 +6130,7 @@ mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 err_rif_alloc:
 err_rif_index_alloc:
 	vr->rif_count--;
-	mlxsw_sp_vr_put(vr);
+	mlxsw_sp_vr_put(mlxsw_sp, vr);
 	return ERR_PTR(err);
 }
 
@@ -6071,7 +6153,7 @@ void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
 		mlxsw_sp_fid_put(fid);
 	kfree(rif);
 	vr->rif_count--;
-	mlxsw_sp_vr_put(vr);
+	mlxsw_sp_vr_put(mlxsw_sp, vr);
 }
 
 static void
@@ -6812,33 +6894,6 @@ mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
 	rif_lb->lb_config = params_lb->lb_config;
 }
 
-static int
-mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
-			struct mlxsw_sp_vr *ul_vr, bool enable)
-{
-	struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
-	struct mlxsw_sp_rif *rif = &lb_rif->common;
-	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
-	char ritr_pl[MLXSW_REG_RITR_LEN];
-	u32 saddr4;
-
-	switch (lb_cf.ul_protocol) {
-	case MLXSW_SP_L3_PROTO_IPV4:
-		saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
-		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
-				    rif->rif_index, rif->vr_id, rif->dev->mtu);
-		mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
-			    MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
-			    ul_vr->id, saddr4, lb_cf.okey);
-		break;
-
-	case MLXSW_SP_L3_PROTO_IPV6:
-		return -EAFNOSUPPORT;
-	}
-
-	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
-}
-
 static int
 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
 {
@@ -6861,7 +6916,7 @@ mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
 	return 0;
 
 err_loopback_op:
-	mlxsw_sp_vr_put(ul_vr);
+	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
 	return err;
 }
 
@@ -6875,7 +6930,7 @@ static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
 	mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
 
 	--ul_vr->rif_count;
-	mlxsw_sp_vr_put(ul_vr);
+	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
 }
 
 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
@@ -7010,6 +7065,24 @@ static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
 }
 #endif
 
+static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
+{
+	char rdpm_pl[MLXSW_REG_RDPM_LEN];
+	unsigned int i;
+
+	MLXSW_REG_ZERO(rdpm, rdpm_pl);
+
+	/* HW is determining switch priority based on DSCP-bits, but the
+	 * kernel is still doing that based on the ToS. Since there's a
+	 * mismatch in bits we need to make sure to translate the right
+	 * value ToS would observe, skipping the 2 least-significant ECN bits.
+	 */
+	for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
+		mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
+
+	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
+}
+
 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
 {
 	char rgcr_pl[MLXSW_REG_RGCR_LEN];
@@ -7022,6 +7095,7 @@ static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
 
 	mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
 	mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
+	mlxsw_reg_rgcr_usp_set(rgcr_pl, true);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
 	if (err)
 		return err;
@@ -7097,6 +7171,10 @@ int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
 	if (err)
 		goto err_mp_hash_init;
 
+	err = mlxsw_sp_dscp_init(mlxsw_sp);
+	if (err)
+		goto err_dscp_init;
+
 	mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
 	err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
 				    mlxsw_sp_router_fib_dump_flush);
@@ -7106,6 +7184,7 @@ int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
 	return 0;
 
 err_register_fib_notifier:
+err_dscp_init:
 err_mp_hash_init:
 	unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
 err_register_netevent_notifier:
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
index 593ad31be749..161bcdc012f0 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c
@@ -1203,6 +1203,7 @@ static int __mlxsw_sp_port_fdb_uc_op(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 				     bool dynamic)
 {
 	char *sfd_pl;
+	u8 num_rec;
 	int err;
 
 	sfd_pl = kmalloc(MLXSW_REG_SFD_LEN, GFP_KERNEL);
@@ -1212,9 +1213,16 @@ static int __mlxsw_sp_port_fdb_uc_op(struct mlxsw_sp *mlxsw_sp, u8 local_port,
 	mlxsw_reg_sfd_pack(sfd_pl, mlxsw_sp_sfd_op(adding), 0);
 	mlxsw_reg_sfd_uc_pack(sfd_pl, 0, mlxsw_sp_sfd_rec_policy(dynamic),
 			      mac, fid, action, local_port);
+	num_rec = mlxsw_reg_sfd_num_rec_get(sfd_pl);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfd), sfd_pl);
-	kfree(sfd_pl);
+	if (err)
+		goto out;
+
+	if (num_rec != mlxsw_reg_sfd_num_rec_get(sfd_pl))
+		err = -EBUSY;
 
+out:
+	kfree(sfd_pl);
 	return err;
 }
 
@@ -1239,6 +1247,7 @@ static int mlxsw_sp_port_fdb_uc_lag_op(struct mlxsw_sp *mlxsw_sp, u16 lag_id,
 				       bool adding, bool dynamic)
 {
 	char *sfd_pl;
+	u8 num_rec;
 	int err;
 
 	sfd_pl = kmalloc(MLXSW_REG_SFD_LEN, GFP_KERNEL);
@@ -1249,9 +1258,16 @@ static int mlxsw_sp_port_fdb_uc_lag_op(struct mlxsw_sp *mlxsw_sp, u16 lag_id,
 	mlxsw_reg_sfd_uc_lag_pack(sfd_pl, 0, mlxsw_sp_sfd_rec_policy(dynamic),
 				  mac, fid, MLXSW_REG_SFD_REC_ACTION_NOP,
 				  lag_vid, lag_id);
+	num_rec = mlxsw_reg_sfd_num_rec_get(sfd_pl);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfd), sfd_pl);
-	kfree(sfd_pl);
+	if (err)
+		goto out;
+
+	if (num_rec != mlxsw_reg_sfd_num_rec_get(sfd_pl))
+		err = -EBUSY;
 
+out:
+	kfree(sfd_pl);
 	return err;
 }
 
@@ -1296,6 +1312,7 @@ static int mlxsw_sp_port_mdb_op(struct mlxsw_sp *mlxsw_sp, const char *addr,
 				u16 fid, u16 mid_idx, bool adding)
 {
 	char *sfd_pl;
+	u8 num_rec;
 	int err;
 
 	sfd_pl = kmalloc(MLXSW_REG_SFD_LEN, GFP_KERNEL);
@@ -1305,7 +1322,15 @@ static int mlxsw_sp_port_mdb_op(struct mlxsw_sp *mlxsw_sp, const char *addr,
 	mlxsw_reg_sfd_pack(sfd_pl, mlxsw_sp_sfd_op(adding), 0);
 	mlxsw_reg_sfd_mc_pack(sfd_pl, 0, addr, fid,
 			      MLXSW_REG_SFD_REC_ACTION_NOP, mid_idx);
+	num_rec = mlxsw_reg_sfd_num_rec_get(sfd_pl);
 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfd), sfd_pl);
+	if (err)
+		goto out;
+
+	if (num_rec != mlxsw_reg_sfd_num_rec_get(sfd_pl))
+		err = -EBUSY;
+
+out:
 	kfree(sfd_pl);
 	return err;
 }
diff --git a/drivers/net/ethernet/micrel/ks8851.c b/drivers/net/ethernet/micrel/ks8851.c
index 2fe96f1f3fe5..bd6e9014bc74 100644
--- a/drivers/net/ethernet/micrel/ks8851.c
+++ b/drivers/net/ethernet/micrel/ks8851.c
@@ -28,6 +28,7 @@
 #include <linux/spi/spi.h>
 #include <linux/gpio.h>
 #include <linux/of_gpio.h>
+#include <linux/of_net.h>
 
 #include "ks8851.h"
 
@@ -407,15 +408,23 @@ static void ks8851_read_mac_addr(struct net_device *dev)
  * @ks: The device structure
  *
  * Get or create the initial mac address for the device and then set that
- * into the station address register. If there is an EEPROM present, then
+ * into the station address register. A mac address supplied in the device
+ * tree takes precedence. Otherwise, if there is an EEPROM present, then
  * we try that. If no valid mac address is found we use eth_random_addr()
  * to create a new one.
  */
 static void ks8851_init_mac(struct ks8851_net *ks)
 {
 	struct net_device *dev = ks->netdev;
+	const u8 *mac_addr;
+
+	mac_addr = of_get_mac_address(ks->spidev->dev.of_node);
+	if (mac_addr) {
+		memcpy(dev->dev_addr, mac_addr, ETH_ALEN);
+		ks8851_write_mac_addr(dev);
+		return;
+	}
 
-	/* first, try reading what we've got already */
 	if (ks->rc_ccr & CCR_EEPROM) {
 		ks8851_read_mac_addr(dev);
 		if (is_valid_ether_addr(dev->dev_addr))
diff --git a/drivers/net/ethernet/natsemi/Kconfig b/drivers/net/ethernet/natsemi/Kconfig
index a10ef50e4f12..017fb2322589 100644
--- a/drivers/net/ethernet/natsemi/Kconfig
+++ b/drivers/net/ethernet/natsemi/Kconfig
@@ -1,16 +1,16 @@
 #
-# National Semi-conductor device configuration
+# National Semiconductor device configuration
 #
 
 config NET_VENDOR_NATSEMI
-	bool "National Semi-conductor devices"
+	bool "National Semiconductor devices"
 	default y
 	---help---
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
 	  Note that the answer to this question doesn't directly affect the
 	  kernel: saying N will just cause the configurator to skip all
-	  the questions about National Semi-conductor devices. If you say Y,
+	  the questions about National Semiconductor devices. If you say Y,
 	  you will be asked for your specific card in the following questions.
 
 if NET_VENDOR_NATSEMI
diff --git a/drivers/net/ethernet/natsemi/Makefile b/drivers/net/ethernet/natsemi/Makefile
index cc664977596e..a759aa09ef59 100644
--- a/drivers/net/ethernet/natsemi/Makefile
+++ b/drivers/net/ethernet/natsemi/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 #
-# Makefile for the National Semi-conductor Sonic devices.
+# Makefile for the National Semiconductor Sonic devices.
 #
 
 obj-$(CONFIG_MACSONIC) += macsonic.o
diff --git a/drivers/net/ethernet/natsemi/macsonic.c b/drivers/net/ethernet/natsemi/macsonic.c
index a42433fb6949..b922ab5cedea 100644
--- a/drivers/net/ethernet/natsemi/macsonic.c
+++ b/drivers/net/ethernet/natsemi/macsonic.c
@@ -311,7 +311,7 @@ static int mac_onboard_sonic_probe(struct net_device *dev)
 {
 	struct sonic_local* lp = netdev_priv(dev);
 	int sr;
-	int commslot = 0;
+	bool commslot = macintosh_config->expansion_type == MAC_EXP_PDS_COMM;
 
 	if (!MACH_IS_MAC)
 		return -ENODEV;
@@ -322,10 +322,7 @@ static int mac_onboard_sonic_probe(struct net_device *dev)
 	   Ethernet (BTW, the Ethernet *is* always at the same
 	   address, and nothing else lives there, at least if Apple's
 	   documentation is to be believed) */
-	if (macintosh_config->ident == MAC_MODEL_Q630 ||
-	    macintosh_config->ident == MAC_MODEL_P588 ||
-	    macintosh_config->ident == MAC_MODEL_P575 ||
-	    macintosh_config->ident == MAC_MODEL_C610) {
+	if (commslot || macintosh_config->ident == MAC_MODEL_C610) {
 		int card_present;
 
 		card_present = hwreg_present((void*)ONBOARD_SONIC_REGISTERS);
@@ -333,7 +330,6 @@ static int mac_onboard_sonic_probe(struct net_device *dev)
 			printk("none.\n");
 			return -ENODEV;
 		}
-		commslot = 1;
 	}
 
 	printk("yes\n");
@@ -428,26 +424,26 @@ static int mac_nubus_sonic_ethernet_addr(struct net_device *dev,
 	return 0;
 }
 
-static int macsonic_ident(struct nubus_dev *ndev)
+static int macsonic_ident(struct nubus_rsrc *fres)
 {
-	if (ndev->dr_hw == NUBUS_DRHW_ASANTE_LC &&
-	    ndev->dr_sw == NUBUS_DRSW_SONIC_LC)
+	if (fres->dr_hw == NUBUS_DRHW_ASANTE_LC &&
+	    fres->dr_sw == NUBUS_DRSW_SONIC_LC)
 		return MACSONIC_DAYNALINK;
-	if (ndev->dr_hw == NUBUS_DRHW_SONIC &&
-	    ndev->dr_sw == NUBUS_DRSW_APPLE) {
+	if (fres->dr_hw == NUBUS_DRHW_SONIC &&
+	    fres->dr_sw == NUBUS_DRSW_APPLE) {
 		/* There has to be a better way to do this... */
-		if (strstr(ndev->board->name, "DuoDock"))
+		if (strstr(fres->board->name, "DuoDock"))
 			return MACSONIC_DUODOCK;
 		else
 			return MACSONIC_APPLE;
 	}
 
-	if (ndev->dr_hw == NUBUS_DRHW_SMC9194 &&
-	    ndev->dr_sw == NUBUS_DRSW_DAYNA)
+	if (fres->dr_hw == NUBUS_DRHW_SMC9194 &&
+	    fres->dr_sw == NUBUS_DRSW_DAYNA)
 		return MACSONIC_DAYNA;
 
-	if (ndev->dr_hw == NUBUS_DRHW_APPLE_SONIC_LC &&
-	    ndev->dr_sw == 0) { /* huh? */
+	if (fres->dr_hw == NUBUS_DRHW_APPLE_SONIC_LC &&
+	    fres->dr_sw == 0) { /* huh? */
 		return MACSONIC_APPLE16;
 	}
 	return -1;
@@ -456,7 +452,7 @@ static int macsonic_ident(struct nubus_dev *ndev)
 static int mac_nubus_sonic_probe(struct net_device *dev)
 {
 	static int slots;
-	struct nubus_dev* ndev = NULL;
+	struct nubus_rsrc *ndev = NULL;
 	struct sonic_local* lp = netdev_priv(dev);
 	unsigned long base_addr, prom_addr;
 	u16 sonic_dcr;
@@ -464,9 +460,11 @@ static int mac_nubus_sonic_probe(struct net_device *dev)
 	int reg_offset, dma_bitmode;
 
 	/* Find the first SONIC that hasn't been initialized already */
-	while ((ndev = nubus_find_type(NUBUS_CAT_NETWORK,
-				       NUBUS_TYPE_ETHERNET, ndev)) != NULL)
-	{
+	for_each_func_rsrc(ndev) {
+		if (ndev->category != NUBUS_CAT_NETWORK ||
+		    ndev->type != NUBUS_TYPE_ETHERNET)
+			continue;
+
 		/* Have we seen it already? */
 		if (slots & (1<<ndev->board->slot))
 			continue;
diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile
index 24c4408b5734..d5866d708dfa 100644
--- a/drivers/net/ethernet/netronome/nfp/Makefile
+++ b/drivers/net/ethernet/netronome/nfp/Makefile
@@ -22,6 +22,8 @@ nfp-objs := \
 	    nfp_hwmon.o \
 	    nfp_main.o \
 	    nfp_net_common.o \
+	    nfp_net_ctrl.o \
+	    nfp_net_debugdump.o \
 	    nfp_net_ethtool.o \
 	    nfp_net_main.o \
 	    nfp_net_repr.o \
@@ -43,6 +45,7 @@ endif
 
 ifeq ($(CONFIG_BPF_SYSCALL),y)
 nfp-objs += \
+	    bpf/cmsg.o \
 	    bpf/main.o \
 	    bpf/offload.o \
 	    bpf/verifier.o \
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c b/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c
new file mode 100644
index 000000000000..80d3aa0fc9d3
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c
@@ -0,0 +1,453 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/bpf.h>
+#include <linux/bitops.h>
+#include <linux/bug.h>
+#include <linux/jiffies.h>
+#include <linux/skbuff.h>
+#include <linux/wait.h>
+
+#include "../nfp_app.h"
+#include "../nfp_net.h"
+#include "fw.h"
+#include "main.h"
+
+#define cmsg_warn(bpf, msg...)	nn_dp_warn(&(bpf)->app->ctrl->dp, msg)
+
+#define NFP_BPF_TAG_ALLOC_SPAN	(U16_MAX / 4)
+
+static bool nfp_bpf_all_tags_busy(struct nfp_app_bpf *bpf)
+{
+	u16 used_tags;
+
+	used_tags = bpf->tag_alloc_next - bpf->tag_alloc_last;
+
+	return used_tags > NFP_BPF_TAG_ALLOC_SPAN;
+}
+
+static int nfp_bpf_alloc_tag(struct nfp_app_bpf *bpf)
+{
+	/* All FW communication for BPF is request-reply.  To make sure we
+	 * don't reuse the message ID too early after timeout - limit the
+	 * number of requests in flight.
+	 */
+	if (nfp_bpf_all_tags_busy(bpf)) {
+		cmsg_warn(bpf, "all FW request contexts busy!\n");
+		return -EAGAIN;
+	}
+
+	WARN_ON(__test_and_set_bit(bpf->tag_alloc_next, bpf->tag_allocator));
+	return bpf->tag_alloc_next++;
+}
+
+static void nfp_bpf_free_tag(struct nfp_app_bpf *bpf, u16 tag)
+{
+	WARN_ON(!__test_and_clear_bit(tag, bpf->tag_allocator));
+
+	while (!test_bit(bpf->tag_alloc_last, bpf->tag_allocator) &&
+	       bpf->tag_alloc_last != bpf->tag_alloc_next)
+		bpf->tag_alloc_last++;
+}
+
+static struct sk_buff *
+nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size)
+{
+	struct sk_buff *skb;
+
+	skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL);
+	skb_put(skb, size);
+
+	return skb;
+}
+
+static struct sk_buff *
+nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n)
+{
+	unsigned int size;
+
+	size = sizeof(struct cmsg_req_map_op);
+	size += sizeof(struct cmsg_key_value_pair) * n;
+
+	return nfp_bpf_cmsg_alloc(bpf, size);
+}
+
+static unsigned int nfp_bpf_cmsg_get_tag(struct sk_buff *skb)
+{
+	struct cmsg_hdr *hdr;
+
+	hdr = (struct cmsg_hdr *)skb->data;
+
+	return be16_to_cpu(hdr->tag);
+}
+
+static struct sk_buff *__nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
+{
+	unsigned int msg_tag;
+	struct sk_buff *skb;
+
+	skb_queue_walk(&bpf->cmsg_replies, skb) {
+		msg_tag = nfp_bpf_cmsg_get_tag(skb);
+		if (msg_tag == tag) {
+			nfp_bpf_free_tag(bpf, tag);
+			__skb_unlink(skb, &bpf->cmsg_replies);
+			return skb;
+		}
+	}
+
+	return NULL;
+}
+
+static struct sk_buff *nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
+{
+	struct sk_buff *skb;
+
+	nfp_ctrl_lock(bpf->app->ctrl);
+	skb = __nfp_bpf_reply(bpf, tag);
+	nfp_ctrl_unlock(bpf->app->ctrl);
+
+	return skb;
+}
+
+static struct sk_buff *nfp_bpf_reply_drop_tag(struct nfp_app_bpf *bpf, u16 tag)
+{
+	struct sk_buff *skb;
+
+	nfp_ctrl_lock(bpf->app->ctrl);
+	skb = __nfp_bpf_reply(bpf, tag);
+	if (!skb)
+		nfp_bpf_free_tag(bpf, tag);
+	nfp_ctrl_unlock(bpf->app->ctrl);
+
+	return skb;
+}
+
+static struct sk_buff *
+nfp_bpf_cmsg_wait_reply(struct nfp_app_bpf *bpf, enum nfp_bpf_cmsg_type type,
+			int tag)
+{
+	struct sk_buff *skb;
+	int i, err;
+
+	for (i = 0; i < 50; i++) {
+		udelay(4);
+		skb = nfp_bpf_reply(bpf, tag);
+		if (skb)
+			return skb;
+	}
+
+	err = wait_event_interruptible_timeout(bpf->cmsg_wq,
+					       skb = nfp_bpf_reply(bpf, tag),
+					       msecs_to_jiffies(5000));
+	/* We didn't get a response - try last time and atomically drop
+	 * the tag even if no response is matched.
+	 */
+	if (!skb)
+		skb = nfp_bpf_reply_drop_tag(bpf, tag);
+	if (err < 0) {
+		cmsg_warn(bpf, "%s waiting for response to 0x%02x: %d\n",
+			  err == ERESTARTSYS ? "interrupted" : "error",
+			  type, err);
+		return ERR_PTR(err);
+	}
+	if (!skb) {
+		cmsg_warn(bpf, "timeout waiting for response to 0x%02x\n",
+			  type);
+		return ERR_PTR(-ETIMEDOUT);
+	}
+
+	return skb;
+}
+
+static struct sk_buff *
+nfp_bpf_cmsg_communicate(struct nfp_app_bpf *bpf, struct sk_buff *skb,
+			 enum nfp_bpf_cmsg_type type, unsigned int reply_size)
+{
+	struct cmsg_hdr *hdr;
+	int tag;
+
+	nfp_ctrl_lock(bpf->app->ctrl);
+	tag = nfp_bpf_alloc_tag(bpf);
+	if (tag < 0) {
+		nfp_ctrl_unlock(bpf->app->ctrl);
+		dev_kfree_skb_any(skb);
+		return ERR_PTR(tag);
+	}
+
+	hdr = (void *)skb->data;
+	hdr->ver = CMSG_MAP_ABI_VERSION;
+	hdr->type = type;
+	hdr->tag = cpu_to_be16(tag);
+
+	__nfp_app_ctrl_tx(bpf->app, skb);
+
+	nfp_ctrl_unlock(bpf->app->ctrl);
+
+	skb = nfp_bpf_cmsg_wait_reply(bpf, type, tag);
+	if (IS_ERR(skb))
+		return skb;
+
+	hdr = (struct cmsg_hdr *)skb->data;
+	/* 0 reply_size means caller will do the validation */
+	if (reply_size && skb->len != reply_size) {
+		cmsg_warn(bpf, "cmsg drop - wrong size %d != %d!\n",
+			  skb->len, reply_size);
+		goto err_free;
+	}
+	if (hdr->type != __CMSG_REPLY(type)) {
+		cmsg_warn(bpf, "cmsg drop - wrong type 0x%02x != 0x%02lx!\n",
+			  hdr->type, __CMSG_REPLY(type));
+		goto err_free;
+	}
+
+	return skb;
+err_free:
+	dev_kfree_skb_any(skb);
+	return ERR_PTR(-EIO);
+}
+
+static int
+nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf,
+			 struct cmsg_reply_map_simple *reply)
+{
+	static const int res_table[] = {
+		[CMSG_RC_SUCCESS]	= 0,
+		[CMSG_RC_ERR_MAP_FD]	= -EBADFD,
+		[CMSG_RC_ERR_MAP_NOENT]	= -ENOENT,
+		[CMSG_RC_ERR_MAP_ERR]	= -EINVAL,
+		[CMSG_RC_ERR_MAP_PARSE]	= -EIO,
+		[CMSG_RC_ERR_MAP_EXIST]	= -EEXIST,
+		[CMSG_RC_ERR_MAP_NOMEM]	= -ENOMEM,
+		[CMSG_RC_ERR_MAP_E2BIG]	= -E2BIG,
+	};
+	u32 rc;
+
+	rc = be32_to_cpu(reply->rc);
+	if (rc >= ARRAY_SIZE(res_table)) {
+		cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc);
+		return -EIO;
+	}
+
+	return res_table[rc];
+}
+
+long long int
+nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map)
+{
+	struct cmsg_reply_map_alloc_tbl *reply;
+	struct cmsg_req_map_alloc_tbl *req;
+	struct sk_buff *skb;
+	u32 tid;
+	int err;
+
+	skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
+	if (!skb)
+		return -ENOMEM;
+
+	req = (void *)skb->data;
+	req->key_size = cpu_to_be32(map->key_size);
+	req->value_size = cpu_to_be32(map->value_size);
+	req->max_entries = cpu_to_be32(map->max_entries);
+	req->map_type = cpu_to_be32(map->map_type);
+	req->map_flags = 0;
+
+	skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_ALLOC,
+				       sizeof(*reply));
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	reply = (void *)skb->data;
+	err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
+	if (err)
+		goto err_free;
+
+	tid = be32_to_cpu(reply->tid);
+	dev_consume_skb_any(skb);
+
+	return tid;
+err_free:
+	dev_kfree_skb_any(skb);
+	return err;
+}
+
+void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map)
+{
+	struct cmsg_reply_map_free_tbl *reply;
+	struct cmsg_req_map_free_tbl *req;
+	struct sk_buff *skb;
+	int err;
+
+	skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
+	if (!skb) {
+		cmsg_warn(bpf, "leaking map - failed to allocate msg\n");
+		return;
+	}
+
+	req = (void *)skb->data;
+	req->tid = cpu_to_be32(nfp_map->tid);
+
+	skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_FREE,
+				       sizeof(*reply));
+	if (IS_ERR(skb)) {
+		cmsg_warn(bpf, "leaking map - I/O error\n");
+		return;
+	}
+
+	reply = (void *)skb->data;
+	err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
+	if (err)
+		cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err);
+
+	dev_consume_skb_any(skb);
+}
+
+static int
+nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap,
+		      enum nfp_bpf_cmsg_type op,
+		      u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value)
+{
+	struct nfp_bpf_map *nfp_map = offmap->dev_priv;
+	struct nfp_app_bpf *bpf = nfp_map->bpf;
+	struct bpf_map *map = &offmap->map;
+	struct cmsg_reply_map_op *reply;
+	struct cmsg_req_map_op *req;
+	struct sk_buff *skb;
+	int err;
+
+	/* FW messages have no space for more than 32 bits of flags */
+	if (flags >> 32)
+		return -EOPNOTSUPP;
+
+	skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1);
+	if (!skb)
+		return -ENOMEM;
+
+	req = (void *)skb->data;
+	req->tid = cpu_to_be32(nfp_map->tid);
+	req->count = cpu_to_be32(1);
+	req->flags = cpu_to_be32(flags);
+
+	/* Copy inputs */
+	if (key)
+		memcpy(&req->elem[0].key, key, map->key_size);
+	if (value)
+		memcpy(&req->elem[0].value, value, map->value_size);
+
+	skb = nfp_bpf_cmsg_communicate(bpf, skb, op,
+				       sizeof(*reply) + sizeof(*reply->elem));
+	if (IS_ERR(skb))
+		return PTR_ERR(skb);
+
+	reply = (void *)skb->data;
+	err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
+	if (err)
+		goto err_free;
+
+	/* Copy outputs */
+	if (out_key)
+		memcpy(out_key, &reply->elem[0].key, map->key_size);
+	if (out_value)
+		memcpy(out_value, &reply->elem[0].value, map->value_size);
+
+	dev_consume_skb_any(skb);
+
+	return 0;
+err_free:
+	dev_kfree_skb_any(skb);
+	return err;
+}
+
+int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
+			      void *key, void *value, u64 flags)
+{
+	return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_UPDATE,
+				     key, value, flags, NULL, NULL);
+}
+
+int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key)
+{
+	return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_DELETE,
+				     key, NULL, 0, NULL, NULL);
+}
+
+int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
+			      void *key, void *value)
+{
+	return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_LOOKUP,
+				     key, NULL, 0, NULL, value);
+}
+
+int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
+				void *next_key)
+{
+	return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETFIRST,
+				     NULL, NULL, 0, next_key, NULL);
+}
+
+int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
+			       void *key, void *next_key)
+{
+	return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETNEXT,
+				     key, NULL, 0, next_key, NULL);
+}
+
+void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb)
+{
+	struct nfp_app_bpf *bpf = app->priv;
+	unsigned int tag;
+
+	if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) {
+		cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len);
+		goto err_free;
+	}
+
+	nfp_ctrl_lock(bpf->app->ctrl);
+
+	tag = nfp_bpf_cmsg_get_tag(skb);
+	if (unlikely(!test_bit(tag, bpf->tag_allocator))) {
+		cmsg_warn(bpf, "cmsg drop - no one is waiting for tag %u!\n",
+			  tag);
+		goto err_unlock;
+	}
+
+	__skb_queue_tail(&bpf->cmsg_replies, skb);
+	wake_up_interruptible_all(&bpf->cmsg_wq);
+
+	nfp_ctrl_unlock(bpf->app->ctrl);
+
+	return;
+err_unlock:
+	nfp_ctrl_unlock(bpf->app->ctrl);
+err_free:
+	dev_kfree_skb_any(skb);
+}
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/fw.h b/drivers/net/ethernet/netronome/nfp/bpf/fw.h
new file mode 100644
index 000000000000..cfcc7bcb2c67
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/bpf/fw.h
@@ -0,0 +1,157 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef NFP_BPF_FW_H
+#define NFP_BPF_FW_H 1
+
+#include <linux/bitops.h>
+#include <linux/types.h>
+
+enum bpf_cap_tlv_type {
+	NFP_BPF_CAP_TYPE_FUNC		= 1,
+	NFP_BPF_CAP_TYPE_ADJUST_HEAD	= 2,
+	NFP_BPF_CAP_TYPE_MAPS		= 3,
+};
+
+struct nfp_bpf_cap_tlv_func {
+	__le32 func_id;
+	__le32 func_addr;
+};
+
+struct nfp_bpf_cap_tlv_adjust_head {
+	__le32 flags;
+	__le32 off_min;
+	__le32 off_max;
+	__le32 guaranteed_sub;
+	__le32 guaranteed_add;
+};
+
+#define NFP_BPF_ADJUST_HEAD_NO_META	BIT(0)
+
+struct nfp_bpf_cap_tlv_maps {
+	__le32 types;
+	__le32 max_maps;
+	__le32 max_elems;
+	__le32 max_key_sz;
+	__le32 max_val_sz;
+	__le32 max_elem_sz;
+};
+
+/*
+ * Types defined for map related control messages
+ */
+#define CMSG_MAP_ABI_VERSION		1
+
+enum nfp_bpf_cmsg_type {
+	CMSG_TYPE_MAP_ALLOC	= 1,
+	CMSG_TYPE_MAP_FREE	= 2,
+	CMSG_TYPE_MAP_LOOKUP	= 3,
+	CMSG_TYPE_MAP_UPDATE	= 4,
+	CMSG_TYPE_MAP_DELETE	= 5,
+	CMSG_TYPE_MAP_GETNEXT	= 6,
+	CMSG_TYPE_MAP_GETFIRST	= 7,
+	__CMSG_TYPE_MAP_MAX,
+};
+
+#define CMSG_TYPE_MAP_REPLY_BIT		7
+#define __CMSG_REPLY(req)		(BIT(CMSG_TYPE_MAP_REPLY_BIT) | (req))
+
+#define CMSG_MAP_KEY_LW			16
+#define CMSG_MAP_VALUE_LW		16
+
+enum nfp_bpf_cmsg_status {
+	CMSG_RC_SUCCESS			= 0,
+	CMSG_RC_ERR_MAP_FD		= 1,
+	CMSG_RC_ERR_MAP_NOENT		= 2,
+	CMSG_RC_ERR_MAP_ERR		= 3,
+	CMSG_RC_ERR_MAP_PARSE		= 4,
+	CMSG_RC_ERR_MAP_EXIST		= 5,
+	CMSG_RC_ERR_MAP_NOMEM		= 6,
+	CMSG_RC_ERR_MAP_E2BIG		= 7,
+};
+
+struct cmsg_hdr {
+	u8 type;
+	u8 ver;
+	__be16 tag;
+};
+
+struct cmsg_reply_map_simple {
+	struct cmsg_hdr hdr;
+	__be32 rc;
+};
+
+struct cmsg_req_map_alloc_tbl {
+	struct cmsg_hdr hdr;
+	__be32 key_size;		/* in bytes */
+	__be32 value_size;		/* in bytes */
+	__be32 max_entries;
+	__be32 map_type;
+	__be32 map_flags;		/* reserved */
+};
+
+struct cmsg_reply_map_alloc_tbl {
+	struct cmsg_reply_map_simple reply_hdr;
+	__be32 tid;
+};
+
+struct cmsg_req_map_free_tbl {
+	struct cmsg_hdr hdr;
+	__be32 tid;
+};
+
+struct cmsg_reply_map_free_tbl {
+	struct cmsg_reply_map_simple reply_hdr;
+	__be32 count;
+};
+
+struct cmsg_key_value_pair {
+	__be32 key[CMSG_MAP_KEY_LW];
+	__be32 value[CMSG_MAP_VALUE_LW];
+};
+
+struct cmsg_req_map_op {
+	struct cmsg_hdr hdr;
+	__be32 tid;
+	__be32 count;
+	__be32 flags;
+	struct cmsg_key_value_pair elem[0];
+};
+
+struct cmsg_reply_map_op {
+	struct cmsg_reply_map_simple reply_hdr;
+	__be32 count;
+	__be32 resv;
+	struct cmsg_key_value_pair elem[0];
+};
+#endif
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index 995e95410b11..ecd7c33baf3c 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Netronome Systems, Inc.
+ * Copyright (C) 2016-2017 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -33,6 +33,7 @@
 
 #define pr_fmt(fmt)	"NFP net bpf: " fmt
 
+#include <linux/bug.h>
 #include <linux/kernel.h>
 #include <linux/bpf.h>
 #include <linux/filter.h>
@@ -65,12 +66,6 @@
 	     next = nfp_meta_next(pos),				\
 	     next2 = nfp_meta_next(next))
 
-static bool
-nfp_meta_has_next(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
-{
-	return meta->l.next != &nfp_prog->insns;
-}
-
 static bool
 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
@@ -79,7 +74,9 @@ nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 
 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
 {
-	if (nfp_prog->__prog_alloc_len == nfp_prog->prog_len) {
+	if (nfp_prog->__prog_alloc_len / sizeof(u64) == nfp_prog->prog_len) {
+		pr_warn("instruction limit reached (%u NFP instructions)\n",
+			nfp_prog->prog_len);
 		nfp_prog->error = -ENOSPC;
 		return;
 	}
@@ -90,19 +87,25 @@ static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
 
 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
 {
-	return nfp_prog->start_off + nfp_prog->prog_len;
+	return nfp_prog->prog_len;
 }
 
-static unsigned int
-nfp_prog_offset_to_index(struct nfp_prog *nfp_prog, unsigned int offset)
+static bool
+nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off)
 {
-	return offset - nfp_prog->start_off;
+	/* If there is a recorded error we may have dropped instructions;
+	 * that doesn't have to be due to translator bug, and the translation
+	 * will fail anyway, so just return OK.
+	 */
+	if (nfp_prog->error)
+		return true;
+	return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);
 }
 
 /* --- Emitters --- */
 static void
 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
-	   u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync)
+	   u8 mode, u8 xfer, u8 areg, u8 breg, u8 size, bool sync, bool indir)
 {
 	enum cmd_ctx_swap ctx;
 	u64 insn;
@@ -120,14 +123,15 @@ __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
 		FIELD_PREP(OP_CMD_CNT, size) |
 		FIELD_PREP(OP_CMD_SIG, sync) |
 		FIELD_PREP(OP_CMD_TGT_CMD, cmd_tgt_act[op].tgt_cmd) |
+		FIELD_PREP(OP_CMD_INDIR, indir) |
 		FIELD_PREP(OP_CMD_MODE, mode);
 
 	nfp_prog_push(nfp_prog, insn);
 }
 
 static void
-emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
-	 u8 mode, u8 xfer, swreg lreg, swreg rreg, u8 size, bool sync)
+emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
+	     swreg lreg, swreg rreg, u8 size, bool sync, bool indir)
 {
 	struct nfp_insn_re_regs reg;
 	int err;
@@ -148,7 +152,22 @@ emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
 		return;
 	}
 
-	__emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync);
+	__emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, sync,
+		   indir);
+}
+
+static void
+emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
+	 swreg lreg, swreg rreg, u8 size, bool sync)
+{
+	emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, sync, false);
+}
+
+static void
+emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
+	       swreg lreg, swreg rreg, u8 size, bool sync)
+{
+	emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, sync, true);
 }
 
 static void
@@ -172,22 +191,28 @@ __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
 	nfp_prog_push(nfp_prog, insn);
 }
 
-static void emit_br_def(struct nfp_prog *nfp_prog, u16 addr, u8 defer)
+static void
+emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer,
+	     enum nfp_relo_type relo)
 {
-	if (defer > 2) {
+	if (mask == BR_UNC && defer > 2) {
 		pr_err("BUG: branch defer out of bounds %d\n", defer);
 		nfp_prog->error = -EFAULT;
 		return;
 	}
-	__emit_br(nfp_prog, BR_UNC, BR_EV_PIP_UNCOND, BR_CSS_NONE, addr, defer);
+
+	__emit_br(nfp_prog, mask,
+		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
+		  BR_CSS_NONE, addr, defer);
+
+	nfp_prog->prog[nfp_prog->prog_len - 1] |=
+		FIELD_PREP(OP_RELO_TYPE, relo);
 }
 
 static void
 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
 {
-	__emit_br(nfp_prog, mask,
-		  mask != BR_UNC ? BR_EV_PIP_COND : BR_EV_PIP_UNCOND,
-		  BR_CSS_NONE, addr, defer);
+	emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);
 }
 
 static void
@@ -230,9 +255,11 @@ emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm,
 		return;
 	}
 
-	__emit_immed(nfp_prog, reg.areg, reg.breg, imm >> 8, width,
-		     invert, shift, reg.wr_both,
-		     reg.dst_lmextn, reg.src_lmextn);
+	/* Use reg.dst when destination is No-Dest. */
+	__emit_immed(nfp_prog,
+		     swreg_type(dst) == NN_REG_NONE ? reg.dst : reg.areg,
+		     reg.breg, imm >> 8, width, invert, shift,
+		     reg.wr_both, reg.dst_lmextn, reg.src_lmextn);
 }
 
 static void
@@ -458,6 +485,21 @@ static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
 	}
 }
 
+static void
+wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm,
+	       enum nfp_relo_type relo)
+{
+	if (imm > 0xffff) {
+		pr_err("relocation of a large immediate!\n");
+		nfp_prog->error = -EFAULT;
+		return;
+	}
+	emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B);
+
+	nfp_prog->prog[nfp_prog->prog_len - 1] |=
+		FIELD_PREP(OP_RELO_TYPE, relo);
+}
+
 /* ur_load_imm_any() - encode immediate or use tmp register (unrestricted)
  * If the @imm is small enough encode it directly in operand and return
  * otherwise load @imm to a spare register and return its encoding.
@@ -490,24 +532,179 @@ static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count)
 		emit_nop(nfp_prog);
 }
 
+static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
+{
+	emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
+}
+
+static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
+{
+	wrp_mov(nfp_prog, reg_both(dst), reg_b(src));
+}
+
+/* wrp_reg_subpart() - load @field_len bytes from @offset of @src, write the
+ * result to @dst from low end.
+ */
 static void
-wrp_br_special(struct nfp_prog *nfp_prog, enum br_mask mask,
-	       enum br_special special)
+wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len,
+		u8 offset)
 {
-	emit_br(nfp_prog, mask, 0, 0);
+	enum shf_sc sc = offset ? SHF_SC_R_SHF : SHF_SC_NONE;
+	u8 mask = (1 << field_len) - 1;
 
-	nfp_prog->prog[nfp_prog->prog_len - 1] |=
-		FIELD_PREP(OP_BR_SPECIAL, special);
+	emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true);
 }
 
-static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
+static void
+addr40_offset(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
+	      swreg *rega, swreg *regb)
 {
-	emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
+	if (offset == reg_imm(0)) {
+		*rega = reg_a(src_gpr);
+		*regb = reg_b(src_gpr + 1);
+		return;
+	}
+
+	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(src_gpr), ALU_OP_ADD, offset);
+	emit_alu(nfp_prog, imm_b(nfp_prog), reg_b(src_gpr + 1), ALU_OP_ADD_C,
+		 reg_imm(0));
+	*rega = imm_a(nfp_prog);
+	*regb = imm_b(nfp_prog);
 }
 
-static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
+/* NFP has Command Push Pull bus which supports bluk memory operations. */
+static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-	wrp_mov(nfp_prog, reg_both(dst), reg_b(src));
+	bool descending_seq = meta->ldst_gather_len < 0;
+	s16 len = abs(meta->ldst_gather_len);
+	swreg src_base, off;
+	bool src_40bit_addr;
+	unsigned int i;
+	u8 xfer_num;
+
+	off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
+	src_40bit_addr = meta->ptr.type == PTR_TO_MAP_VALUE;
+	src_base = reg_a(meta->insn.src_reg * 2);
+	xfer_num = round_up(len, 4) / 4;
+
+	if (src_40bit_addr)
+		addr40_offset(nfp_prog, meta->insn.src_reg, off, &src_base,
+			      &off);
+
+	/* Setup PREV_ALU fields to override memory read length. */
+	if (len > 32)
+		wrp_immed(nfp_prog, reg_none(),
+			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1));
+
+	/* Memory read from source addr into transfer-in registers. */
+	emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP,
+		     src_40bit_addr ? CMD_MODE_40b_BA : CMD_MODE_32b, 0,
+		     src_base, off, xfer_num - 1, true, len > 32);
+
+	/* Move from transfer-in to transfer-out. */
+	for (i = 0; i < xfer_num; i++)
+		wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i));
+
+	off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog));
+
+	if (len <= 8) {
+		/* Use single direct_ref write8. */
+		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
+			 reg_a(meta->paired_st->dst_reg * 2), off, len - 1,
+			 true);
+	} else if (len <= 32 && IS_ALIGNED(len, 4)) {
+		/* Use single direct_ref write32. */
+		emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
+			 reg_a(meta->paired_st->dst_reg * 2), off, xfer_num - 1,
+			 true);
+	} else if (len <= 32) {
+		/* Use single indirect_ref write8. */
+		wrp_immed(nfp_prog, reg_none(),
+			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, len - 1));
+		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
+			       reg_a(meta->paired_st->dst_reg * 2), off,
+			       len - 1, true);
+	} else if (IS_ALIGNED(len, 4)) {
+		/* Use single indirect_ref write32. */
+		wrp_immed(nfp_prog, reg_none(),
+			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 1));
+		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
+			       reg_a(meta->paired_st->dst_reg * 2), off,
+			       xfer_num - 1, true);
+	} else if (len <= 40) {
+		/* Use one direct_ref write32 to write the first 32-bytes, then
+		 * another direct_ref write8 to write the remaining bytes.
+		 */
+		emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
+			 reg_a(meta->paired_st->dst_reg * 2), off, 7,
+			 true);
+
+		off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32,
+				      imm_b(nfp_prog));
+		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8,
+			 reg_a(meta->paired_st->dst_reg * 2), off, len - 33,
+			 true);
+	} else {
+		/* Use one indirect_ref write32 to write 4-bytes aligned length,
+		 * then another direct_ref write8 to write the remaining bytes.
+		 */
+		u8 new_off;
+
+		wrp_immed(nfp_prog, reg_none(),
+			  CMD_OVE_LEN | FIELD_PREP(CMD_OV_LEN, xfer_num - 2));
+		emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
+			       reg_a(meta->paired_st->dst_reg * 2), off,
+			       xfer_num - 2, true);
+		new_off = meta->paired_st->off + (xfer_num - 1) * 4;
+		off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog));
+		emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b,
+			 xfer_num - 1, reg_a(meta->paired_st->dst_reg * 2), off,
+			 (len & 0x3) - 1, true);
+	}
+
+	/* TODO: The following extra load is to make sure data flow be identical
+	 *  before and after we do memory copy optimization.
+	 *
+	 *  The load destination register is not guaranteed to be dead, so we
+	 *  need to make sure it is loaded with the value the same as before
+	 *  this transformation.
+	 *
+	 *  These extra loads could be removed once we have accurate register
+	 *  usage information.
+	 */
+	if (descending_seq)
+		xfer_num = 0;
+	else if (BPF_SIZE(meta->insn.code) != BPF_DW)
+		xfer_num = xfer_num - 1;
+	else
+		xfer_num = xfer_num - 2;
+
+	switch (BPF_SIZE(meta->insn.code)) {
+	case BPF_B:
+		wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
+				reg_xfer(xfer_num), 1,
+				IS_ALIGNED(len, 4) ? 3 : (len & 3) - 1);
+		break;
+	case BPF_H:
+		wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
+				reg_xfer(xfer_num), 2, (len & 3) ^ 2);
+		break;
+	case BPF_W:
+		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
+			reg_xfer(0));
+		break;
+	case BPF_DW:
+		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
+			reg_xfer(xfer_num));
+		wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1),
+			reg_xfer(xfer_num + 1));
+		break;
+	}
+
+	if (BPF_SIZE(meta->insn.code) != BPF_DW)
+		wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
+
+	return 0;
 }
 
 static int
@@ -540,20 +737,20 @@ data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
 }
 
 static int
-data_ld_host_order(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
-		   u8 dst_gpr, int size)
+data_ld_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr,
+		   swreg lreg, swreg rreg, int size, enum cmd_mode mode)
 {
 	unsigned int i;
 	u8 mask, sz;
 
-	/* We load the value from the address indicated in @offset and then
+	/* We load the value from the address indicated in rreg + lreg and then
 	 * mask out the data we don't need.  Note: this is little endian!
 	 */
 	sz = max(size, 4);
 	mask = size < 4 ? GENMASK(size - 1, 0) : 0;
 
-	emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0,
-		 reg_a(src_gpr), offset, sz / 4 - 1, true);
+	emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, mode, 0,
+		 lreg, rreg, sz / 4 - 1, true);
 
 	i = 0;
 	if (mask)
@@ -569,6 +766,26 @@ data_ld_host_order(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
 	return 0;
 }
 
+static int
+data_ld_host_order_addr32(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
+			  u8 dst_gpr, u8 size)
+{
+	return data_ld_host_order(nfp_prog, dst_gpr, reg_a(src_gpr), offset,
+				  size, CMD_MODE_32b);
+}
+
+static int
+data_ld_host_order_addr40(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
+			  u8 dst_gpr, u8 size)
+{
+	swreg rega, regb;
+
+	addr40_offset(nfp_prog, src_gpr, offset, &rega, &regb);
+
+	return data_ld_host_order(nfp_prog, dst_gpr, rega, regb,
+				  size, CMD_MODE_40b_BA);
+}
+
 static int
 construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
 {
@@ -583,7 +800,7 @@ construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
 		 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
 	emit_alu(nfp_prog, reg_none(),
 		 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
-	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
+	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
 
 	/* Load data */
 	return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
@@ -596,7 +813,7 @@ static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
 	/* Check packet length */
 	tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
 	emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
-	wrp_br_special(nfp_prog, BR_BLO, OP_BR_GO_ABORT);
+	emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
 
 	/* Load data */
 	tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
@@ -975,9 +1192,6 @@ wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 {
 	const struct bpf_insn *insn = &meta->insn;
 
-	if (insn->off < 0) /* TODO */
-		return -EOPNOTSUPP;
-
 	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
 			 insn->src_reg * 2, br_mask, insn->off);
 	wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
@@ -995,9 +1209,6 @@ wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	u8 reg = insn->dst_reg * 2;
 	swreg tmp_reg;
 
-	if (insn->off < 0) /* TODO */
-		return -EOPNOTSUPP;
-
 	tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
 	if (!swap)
 		emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg);
@@ -1027,9 +1238,6 @@ wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	areg = insn->dst_reg * 2;
 	breg = insn->src_reg * 2;
 
-	if (insn->off < 0) /* TODO */
-		return -EOPNOTSUPP;
-
 	if (swap) {
 		areg ^= breg;
 		breg ^= areg;
@@ -1052,6 +1260,136 @@ static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out)
 		      SHF_SC_R_ROT, 16);
 }
 
+static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog);
+	struct nfp_bpf_cap_adjust_head *adjust_head;
+	u32 ret_einval, end;
+
+	adjust_head = &nfp_prog->bpf->adjust_head;
+
+	/* Optimized version - 5 vs 14 cycles */
+	if (nfp_prog->adjust_head_location != UINT_MAX) {
+		if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n))
+			return -EINVAL;
+
+		emit_alu(nfp_prog, pptr_reg(nfp_prog),
+			 reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog));
+		emit_alu(nfp_prog, plen_reg(nfp_prog),
+			 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
+		emit_alu(nfp_prog, pv_len(nfp_prog),
+			 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
+
+		wrp_immed(nfp_prog, reg_both(0), 0);
+		wrp_immed(nfp_prog, reg_both(1), 0);
+
+		/* TODO: when adjust head is guaranteed to succeed we can
+		 * also eliminate the following if (r0 == 0) branch.
+		 */
+
+		return 0;
+	}
+
+	ret_einval = nfp_prog_current_offset(nfp_prog) + 14;
+	end = ret_einval + 2;
+
+	/* We need to use a temp because offset is just a part of the pkt ptr */
+	emit_alu(nfp_prog, tmp,
+		 reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog));
+
+	/* Validate result will fit within FW datapath constraints */
+	emit_alu(nfp_prog, reg_none(),
+		 tmp, ALU_OP_SUB, reg_imm(adjust_head->off_min));
+	emit_br(nfp_prog, BR_BLO, ret_einval, 0);
+	emit_alu(nfp_prog, reg_none(),
+		 reg_imm(adjust_head->off_max), ALU_OP_SUB, tmp);
+	emit_br(nfp_prog, BR_BLO, ret_einval, 0);
+
+	/* Validate the length is at least ETH_HLEN */
+	emit_alu(nfp_prog, tmp_len,
+		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
+	emit_alu(nfp_prog, reg_none(),
+		 tmp_len, ALU_OP_SUB, reg_imm(ETH_HLEN));
+	emit_br(nfp_prog, BR_BMI, ret_einval, 0);
+
+	/* Load the ret code */
+	wrp_immed(nfp_prog, reg_both(0), 0);
+	wrp_immed(nfp_prog, reg_both(1), 0);
+
+	/* Modify the packet metadata */
+	emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);
+
+	/* Skip over the -EINVAL ret code (defer 2) */
+	emit_br(nfp_prog, BR_UNC, end, 2);
+
+	emit_alu(nfp_prog, plen_reg(nfp_prog),
+		 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
+	emit_alu(nfp_prog, pv_len(nfp_prog),
+		 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
+
+	/* return -EINVAL target */
+	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
+		return -EINVAL;
+
+	wrp_immed(nfp_prog, reg_both(0), -22);
+	wrp_immed(nfp_prog, reg_both(1), ~0);
+
+	if (!nfp_prog_confirm_current_offset(nfp_prog, end))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+map_lookup_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	struct bpf_offloaded_map *offmap;
+	struct nfp_bpf_map *nfp_map;
+	bool load_lm_ptr;
+	u32 ret_tgt;
+	s64 lm_off;
+	swreg tid;
+
+	offmap = (struct bpf_offloaded_map *)meta->arg1.map_ptr;
+	nfp_map = offmap->dev_priv;
+
+	/* We only have to reload LM0 if the key is not at start of stack */
+	lm_off = nfp_prog->stack_depth;
+	lm_off += meta->arg2.var_off.value + meta->arg2.off;
+	load_lm_ptr = meta->arg2_var_off || lm_off;
+
+	/* Set LM0 to start of key */
+	if (load_lm_ptr)
+		emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0);
+
+	/* Load map ID into a register, it should actually fit as an immediate
+	 * but in case it doesn't deal with it here, not in the delay slots.
+	 */
+	tid = ur_load_imm_any(nfp_prog, nfp_map->tid, imm_a(nfp_prog));
+
+	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + BPF_FUNC_map_lookup_elem,
+		     2, RELO_BR_HELPER);
+	ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
+
+	/* Load map ID into A0 */
+	wrp_mov(nfp_prog, reg_a(0), tid);
+
+	/* Load the return address into B0 */
+	wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
+
+	if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
+		return -EINVAL;
+
+	/* Reset the LM0 pointer */
+	if (!load_lm_ptr)
+		return 0;
+
+	emit_csr_wr(nfp_prog, stack_reg(nfp_prog),  NFP_CSR_ACT_LM_ADDR0);
+	wrp_nops(nfp_prog, 3);
+
+	return 0;
+}
+
 /* --- Callbacks --- */
 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
@@ -1486,14 +1824,29 @@ mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 
 	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
 
-	return data_ld_host_order(nfp_prog, meta->insn.src_reg * 2, tmp_reg,
-				  meta->insn.dst_reg * 2, size);
+	return data_ld_host_order_addr32(nfp_prog, meta->insn.src_reg * 2,
+					 tmp_reg, meta->insn.dst_reg * 2, size);
+}
+
+static int
+mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+	     unsigned int size)
+{
+	swreg tmp_reg;
+
+	tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
+
+	return data_ld_host_order_addr40(nfp_prog, meta->insn.src_reg * 2,
+					 tmp_reg, meta->insn.dst_reg * 2, size);
 }
 
 static int
 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	unsigned int size)
 {
+	if (meta->ldst_gather_len)
+		return nfp_cpp_memcpy(nfp_prog, meta);
+
 	if (meta->ptr.type == PTR_TO_CTX) {
 		if (nfp_prog->type == BPF_PROG_TYPE_XDP)
 			return mem_ldx_xdp(nfp_prog, meta, size);
@@ -1508,6 +1861,9 @@ mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 		return mem_ldx_stack(nfp_prog, meta, size,
 				     meta->ptr.off + meta->ptr.var_off.value);
 
+	if (meta->ptr.type == PTR_TO_MAP_VALUE)
+		return mem_ldx_emem(nfp_prog, meta, size);
+
 	return -EOPNOTSUPP;
 }
 
@@ -1630,8 +1986,6 @@ static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 
 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-	if (meta->insn.off < 0) /* TODO */
-		return -EOPNOTSUPP;
 	emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
 
 	return 0;
@@ -1646,9 +2000,6 @@ static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	or1 = reg_a(insn->dst_reg * 2);
 	or2 = reg_b(insn->dst_reg * 2 + 1);
 
-	if (insn->off < 0) /* TODO */
-		return -EOPNOTSUPP;
-
 	if (imm & ~0U) {
 		tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
 		emit_alu(nfp_prog, imm_a(nfp_prog),
@@ -1689,15 +2040,32 @@ static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true);
 }
 
+static int jsgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BLT, true);
+}
+
+static int jsge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BGE, false);
+}
+
+static int jslt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BLT, false);
+}
+
+static int jsle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_imm(nfp_prog, meta, BR_BGE, true);
+}
+
 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	const struct bpf_insn *insn = &meta->insn;
 	u64 imm = insn->imm; /* sign extend */
 	swreg tmp_reg;
 
-	if (insn->off < 0) /* TODO */
-		return -EOPNOTSUPP;
-
 	if (!imm) {
 		meta->skip = true;
 		return 0;
@@ -1726,9 +2094,6 @@ static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	u64 imm = insn->imm; /* sign extend */
 	swreg tmp_reg;
 
-	if (insn->off < 0) /* TODO */
-		return -EOPNOTSUPP;
-
 	if (!imm) {
 		emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
 			 ALU_OP_OR, reg_b(insn->dst_reg * 2 + 1));
@@ -1753,9 +2118,6 @@ static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	const struct bpf_insn *insn = &meta->insn;
 
-	if (insn->off < 0) /* TODO */
-		return -EOPNOTSUPP;
-
 	emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
 		 ALU_OP_XOR, reg_b(insn->src_reg * 2));
 	emit_alu(nfp_prog, imm_b(nfp_prog), reg_a(insn->dst_reg * 2 + 1),
@@ -1787,6 +2149,26 @@ static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true);
 }
 
+static int jsgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BLT, true);
+}
+
+static int jsge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BGE, false);
+}
+
+static int jslt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BLT, false);
+}
+
+static int jsle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	return wrp_cmp_reg(nfp_prog, meta, BR_BGE, true);
+}
+
 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
 	return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
@@ -1797,9 +2179,22 @@ static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 	return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
 }
 
+static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
+{
+	switch (meta->insn.imm) {
+	case BPF_FUNC_xdp_adjust_head:
+		return adjust_head(nfp_prog, meta);
+	case BPF_FUNC_map_lookup_elem:
+		return map_lookup_stack(nfp_prog, meta);
+	default:
+		WARN_ONCE(1, "verifier allowed unsupported function\n");
+		return -EOPNOTSUPP;
+	}
+}
+
 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-	wrp_br_special(nfp_prog, BR_UNC, OP_BR_GO_OUT);
+	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT);
 
 	return 0;
 }
@@ -1860,6 +2255,10 @@ static const instr_cb_t instr_cb[256] = {
 	[BPF_JMP | BPF_JGE | BPF_K] =	jge_imm,
 	[BPF_JMP | BPF_JLT | BPF_K] =	jlt_imm,
 	[BPF_JMP | BPF_JLE | BPF_K] =	jle_imm,
+	[BPF_JMP | BPF_JSGT | BPF_K] =  jsgt_imm,
+	[BPF_JMP | BPF_JSGE | BPF_K] =  jsge_imm,
+	[BPF_JMP | BPF_JSLT | BPF_K] =  jslt_imm,
+	[BPF_JMP | BPF_JSLE | BPF_K] =  jsle_imm,
 	[BPF_JMP | BPF_JSET | BPF_K] =	jset_imm,
 	[BPF_JMP | BPF_JNE | BPF_K] =	jne_imm,
 	[BPF_JMP | BPF_JEQ | BPF_X] =	jeq_reg,
@@ -1867,99 +2266,64 @@ static const instr_cb_t instr_cb[256] = {
 	[BPF_JMP | BPF_JGE | BPF_X] =	jge_reg,
 	[BPF_JMP | BPF_JLT | BPF_X] =	jlt_reg,
 	[BPF_JMP | BPF_JLE | BPF_X] =	jle_reg,
+	[BPF_JMP | BPF_JSGT | BPF_X] =  jsgt_reg,
+	[BPF_JMP | BPF_JSGE | BPF_X] =  jsge_reg,
+	[BPF_JMP | BPF_JSLT | BPF_X] =  jslt_reg,
+	[BPF_JMP | BPF_JSLE | BPF_X] =  jsle_reg,
 	[BPF_JMP | BPF_JSET | BPF_X] =	jset_reg,
 	[BPF_JMP | BPF_JNE | BPF_X] =	jne_reg,
+	[BPF_JMP | BPF_CALL] =		call,
 	[BPF_JMP | BPF_EXIT] =		goto_out,
 };
 
-/* --- Misc code --- */
-static void br_set_offset(u64 *instr, u16 offset)
-{
-	u16 addr_lo, addr_hi;
-
-	addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
-	addr_hi = offset != addr_lo;
-	*instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO);
-	*instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
-	*instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo);
-}
-
 /* --- Assembler logic --- */
 static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
 {
-	struct nfp_insn_meta *meta, *next;
-	u32 off, br_idx;
-	u32 idx;
+	struct nfp_insn_meta *meta, *jmp_dst;
+	u32 idx, br_idx;
 
-	nfp_for_each_insn_walk2(nfp_prog, meta, next) {
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
 		if (meta->skip)
 			continue;
+		if (meta->insn.code == (BPF_JMP | BPF_CALL))
+			continue;
 		if (BPF_CLASS(meta->insn.code) != BPF_JMP)
 			continue;
 
-		br_idx = nfp_prog_offset_to_index(nfp_prog, next->off) - 1;
+		if (list_is_last(&meta->l, &nfp_prog->insns))
+			br_idx = nfp_prog->last_bpf_off;
+		else
+			br_idx = list_next_entry(meta, l)->off - 1;
+
 		if (!nfp_is_br(nfp_prog->prog[br_idx])) {
 			pr_err("Fixup found block not ending in branch %d %02x %016llx!!\n",
 			       br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
 			return -ELOOP;
 		}
 		/* Leave special branches for later */
-		if (FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]))
+		if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
+		    RELO_BR_REL)
 			continue;
 
-		/* Find the target offset in assembler realm */
-		off = meta->insn.off;
-		if (!off) {
-			pr_err("Fixup found zero offset!!\n");
+		if (!meta->jmp_dst) {
+			pr_err("Non-exit jump doesn't have destination info recorded!!\n");
 			return -ELOOP;
 		}
 
-		while (off && nfp_meta_has_next(nfp_prog, next)) {
-			next = nfp_meta_next(next);
-			off--;
-		}
-		if (off) {
-			pr_err("Fixup found too large jump!! %d\n", off);
-			return -ELOOP;
-		}
+		jmp_dst = meta->jmp_dst;
 
-		if (next->skip) {
+		if (jmp_dst->skip) {
 			pr_err("Branch landing on removed instruction!!\n");
 			return -ELOOP;
 		}
 
-		for (idx = nfp_prog_offset_to_index(nfp_prog, meta->off);
-		     idx <= br_idx; idx++) {
+		for (idx = meta->off; idx <= br_idx; idx++) {
 			if (!nfp_is_br(nfp_prog->prog[idx]))
 				continue;
-			br_set_offset(&nfp_prog->prog[idx], next->off);
+			br_set_offset(&nfp_prog->prog[idx], jmp_dst->off);
 		}
 	}
 
-	/* Fixup 'goto out's separately, they can be scattered around */
-	for (br_idx = 0; br_idx < nfp_prog->prog_len; br_idx++) {
-		enum br_special special;
-
-		if ((nfp_prog->prog[br_idx] & OP_BR_BASE_MASK) != OP_BR_BASE)
-			continue;
-
-		special = FIELD_GET(OP_BR_SPECIAL, nfp_prog->prog[br_idx]);
-		switch (special) {
-		case OP_BR_NORMAL:
-			break;
-		case OP_BR_GO_OUT:
-			br_set_offset(&nfp_prog->prog[br_idx],
-				      nfp_prog->tgt_out);
-			break;
-		case OP_BR_GO_ABORT:
-			br_set_offset(&nfp_prog->prog[br_idx],
-				      nfp_prog->tgt_abort);
-			break;
-		}
-
-		nfp_prog->prog[br_idx] &= ~OP_BR_SPECIAL;
-	}
-
 	return 0;
 }
 
@@ -1987,7 +2351,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
 	/* Target for aborts */
 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
 
 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
@@ -2014,7 +2378,7 @@ static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
 	emit_shf(nfp_prog, reg_b(2),
 		 reg_imm(0xf), SHF_OP_AND, reg_b(3), SHF_SC_R_SHF, 0);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
 
 	emit_shf(nfp_prog, reg_b(2),
 		 reg_a(2), SHF_OP_OR, reg_b(2), SHF_SC_L_SHF, 4);
@@ -2033,7 +2397,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
 	/* Target for aborts */
 	nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
 
 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
@@ -2054,7 +2418,7 @@ static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
 	emit_shf(nfp_prog, reg_b(2),
 		 reg_imm(0xff), SHF_OP_AND, reg_b(2), SHF_SC_R_SHF, 0);
 
-	emit_br_def(nfp_prog, nfp_prog->tgt_done, 2);
+	emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
 
 	wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
 	emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
@@ -2101,10 +2465,14 @@ static int nfp_translate(struct nfp_prog *nfp_prog)
 		err = cb(nfp_prog, meta);
 		if (err)
 			return err;
+		if (nfp_prog->error)
+			return nfp_prog->error;
 
 		nfp_prog->n_translated++;
 	}
 
+	nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1;
+
 	nfp_outro(nfp_prog);
 	if (nfp_prog->error)
 		return nfp_prog->error;
@@ -2173,6 +2541,9 @@ static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
 		if (next.src_reg || next.dst_reg)
 			continue;
 
+		if (meta2->flags & FLAG_INSN_IS_JUMP_DST)
+			continue;
+
 		meta2->skip = true;
 	}
 }
@@ -2209,40 +2580,294 @@ static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog)
 		if (next1.imm != 0x20 || next2.imm != 0x20)
 			continue;
 
+		if (meta2->flags & FLAG_INSN_IS_JUMP_DST ||
+		    meta3->flags & FLAG_INSN_IS_JUMP_DST)
+			continue;
+
 		meta2->skip = true;
 		meta3->skip = true;
 	}
 }
 
+/* load/store pair that forms memory copy sould look like the following:
+ *
+ *   ld_width R, [addr_src + offset_src]
+ *   st_width [addr_dest + offset_dest], R
+ *
+ * The destination register of load and source register of store should
+ * be the same, load and store should also perform at the same width.
+ * If either of addr_src or addr_dest is stack pointer, we don't do the
+ * CPP optimization as stack is modelled by registers on NFP.
+ */
+static bool
+curr_pair_is_memcpy(struct nfp_insn_meta *ld_meta,
+		    struct nfp_insn_meta *st_meta)
+{
+	struct bpf_insn *ld = &ld_meta->insn;
+	struct bpf_insn *st = &st_meta->insn;
+
+	if (!is_mbpf_load(ld_meta) || !is_mbpf_store(st_meta))
+		return false;
+
+	if (ld_meta->ptr.type != PTR_TO_PACKET)
+		return false;
+
+	if (st_meta->ptr.type != PTR_TO_PACKET)
+		return false;
+
+	if (BPF_SIZE(ld->code) != BPF_SIZE(st->code))
+		return false;
+
+	if (ld->dst_reg != st->src_reg)
+		return false;
+
+	/* There is jump to the store insn in this pair. */
+	if (st_meta->flags & FLAG_INSN_IS_JUMP_DST)
+		return false;
+
+	return true;
+}
+
+/* Currently, we only support chaining load/store pairs if:
+ *
+ *  - Their address base registers are the same.
+ *  - Their address offsets are in the same order.
+ *  - They operate at the same memory width.
+ *  - There is no jump into the middle of them.
+ */
+static bool
+curr_pair_chain_with_previous(struct nfp_insn_meta *ld_meta,
+			      struct nfp_insn_meta *st_meta,
+			      struct bpf_insn *prev_ld,
+			      struct bpf_insn *prev_st)
+{
+	u8 prev_size, curr_size, prev_ld_base, prev_st_base, prev_ld_dst;
+	struct bpf_insn *ld = &ld_meta->insn;
+	struct bpf_insn *st = &st_meta->insn;
+	s16 prev_ld_off, prev_st_off;
+
+	/* This pair is the start pair. */
+	if (!prev_ld)
+		return true;
+
+	prev_size = BPF_LDST_BYTES(prev_ld);
+	curr_size = BPF_LDST_BYTES(ld);
+	prev_ld_base = prev_ld->src_reg;
+	prev_st_base = prev_st->dst_reg;
+	prev_ld_dst = prev_ld->dst_reg;
+	prev_ld_off = prev_ld->off;
+	prev_st_off = prev_st->off;
+
+	if (ld->dst_reg != prev_ld_dst)
+		return false;
+
+	if (ld->src_reg != prev_ld_base || st->dst_reg != prev_st_base)
+		return false;
+
+	if (curr_size != prev_size)
+		return false;
+
+	/* There is jump to the head of this pair. */
+	if (ld_meta->flags & FLAG_INSN_IS_JUMP_DST)
+		return false;
+
+	/* Both in ascending order. */
+	if (prev_ld_off + prev_size == ld->off &&
+	    prev_st_off + prev_size == st->off)
+		return true;
+
+	/* Both in descending order. */
+	if (ld->off + curr_size == prev_ld_off &&
+	    st->off + curr_size == prev_st_off)
+		return true;
+
+	return false;
+}
+
+/* Return TRUE if cross memory access happens. Cross memory access means
+ * store area is overlapping with load area that a later load might load
+ * the value from previous store, for this case we can't treat the sequence
+ * as an memory copy.
+ */
+static bool
+cross_mem_access(struct bpf_insn *ld, struct nfp_insn_meta *head_ld_meta,
+		 struct nfp_insn_meta *head_st_meta)
+{
+	s16 head_ld_off, head_st_off, ld_off;
+
+	/* Different pointer types does not overlap. */
+	if (head_ld_meta->ptr.type != head_st_meta->ptr.type)
+		return false;
+
+	/* load and store are both PTR_TO_PACKET, check ID info.  */
+	if (head_ld_meta->ptr.id != head_st_meta->ptr.id)
+		return true;
+
+	/* Canonicalize the offsets. Turn all of them against the original
+	 * base register.
+	 */
+	head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.off;
+	head_st_off = head_st_meta->insn.off + head_st_meta->ptr.off;
+	ld_off = ld->off + head_ld_meta->ptr.off;
+
+	/* Ascending order cross. */
+	if (ld_off > head_ld_off &&
+	    head_ld_off < head_st_off && ld_off >= head_st_off)
+		return true;
+
+	/* Descending order cross. */
+	if (ld_off < head_ld_off &&
+	    head_ld_off > head_st_off && ld_off <= head_st_off)
+		return true;
+
+	return false;
+}
+
+/* This pass try to identify the following instructoin sequences.
+ *
+ *   load R, [regA + offA]
+ *   store [regB + offB], R
+ *   load R, [regA + offA + const_imm_A]
+ *   store [regB + offB + const_imm_A], R
+ *   load R, [regA + offA + 2 * const_imm_A]
+ *   store [regB + offB + 2 * const_imm_A], R
+ *   ...
+ *
+ * Above sequence is typically generated by compiler when lowering
+ * memcpy. NFP prefer using CPP instructions to accelerate it.
+ */
+static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog)
+{
+	struct nfp_insn_meta *head_ld_meta = NULL;
+	struct nfp_insn_meta *head_st_meta = NULL;
+	struct nfp_insn_meta *meta1, *meta2;
+	struct bpf_insn *prev_ld = NULL;
+	struct bpf_insn *prev_st = NULL;
+	u8 count = 0;
+
+	nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
+		struct bpf_insn *ld = &meta1->insn;
+		struct bpf_insn *st = &meta2->insn;
+
+		/* Reset record status if any of the following if true:
+		 *   - The current insn pair is not load/store.
+		 *   - The load/store pair doesn't chain with previous one.
+		 *   - The chained load/store pair crossed with previous pair.
+		 *   - The chained load/store pair has a total size of memory
+		 *     copy beyond 128 bytes which is the maximum length a
+		 *     single NFP CPP command can transfer.
+		 */
+		if (!curr_pair_is_memcpy(meta1, meta2) ||
+		    !curr_pair_chain_with_previous(meta1, meta2, prev_ld,
+						   prev_st) ||
+		    (head_ld_meta && (cross_mem_access(ld, head_ld_meta,
+						       head_st_meta) ||
+				      head_ld_meta->ldst_gather_len >= 128))) {
+			if (!count)
+				continue;
+
+			if (count > 1) {
+				s16 prev_ld_off = prev_ld->off;
+				s16 prev_st_off = prev_st->off;
+				s16 head_ld_off = head_ld_meta->insn.off;
+
+				if (prev_ld_off < head_ld_off) {
+					head_ld_meta->insn.off = prev_ld_off;
+					head_st_meta->insn.off = prev_st_off;
+					head_ld_meta->ldst_gather_len =
+						-head_ld_meta->ldst_gather_len;
+				}
+
+				head_ld_meta->paired_st = &head_st_meta->insn;
+				head_st_meta->skip = true;
+			} else {
+				head_ld_meta->ldst_gather_len = 0;
+			}
+
+			/* If the chain is ended by an load/store pair then this
+			 * could serve as the new head of the the next chain.
+			 */
+			if (curr_pair_is_memcpy(meta1, meta2)) {
+				head_ld_meta = meta1;
+				head_st_meta = meta2;
+				head_ld_meta->ldst_gather_len =
+					BPF_LDST_BYTES(ld);
+				meta1 = nfp_meta_next(meta1);
+				meta2 = nfp_meta_next(meta2);
+				prev_ld = ld;
+				prev_st = st;
+				count = 1;
+			} else {
+				head_ld_meta = NULL;
+				head_st_meta = NULL;
+				prev_ld = NULL;
+				prev_st = NULL;
+				count = 0;
+			}
+
+			continue;
+		}
+
+		if (!head_ld_meta) {
+			head_ld_meta = meta1;
+			head_st_meta = meta2;
+		} else {
+			meta1->skip = true;
+			meta2->skip = true;
+		}
+
+		head_ld_meta->ldst_gather_len += BPF_LDST_BYTES(ld);
+		meta1 = nfp_meta_next(meta1);
+		meta2 = nfp_meta_next(meta2);
+		prev_ld = ld;
+		prev_st = st;
+		count++;
+	}
+}
+
 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
 {
 	nfp_bpf_opt_reg_init(nfp_prog);
 
 	nfp_bpf_opt_ld_mask(nfp_prog);
 	nfp_bpf_opt_ld_shift(nfp_prog);
+	nfp_bpf_opt_ldst_gather(nfp_prog);
 
 	return 0;
 }
 
-static int nfp_bpf_ustore_calc(struct nfp_prog *nfp_prog, __le64 *ustore)
+static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len)
 {
+	__le64 *ustore = (__force __le64 *)prog;
 	int i;
 
-	for (i = 0; i < nfp_prog->prog_len; i++) {
+	for (i = 0; i < len; i++) {
 		int err;
 
-		err = nfp_ustore_check_valid_no_ecc(nfp_prog->prog[i]);
+		err = nfp_ustore_check_valid_no_ecc(prog[i]);
 		if (err)
 			return err;
 
-		nfp_prog->prog[i] = nfp_ustore_calc_ecc_insn(nfp_prog->prog[i]);
-
-		ustore[i] = cpu_to_le64(nfp_prog->prog[i]);
+		ustore[i] = cpu_to_le64(nfp_ustore_calc_ecc_insn(prog[i]));
 	}
 
 	return 0;
 }
 
+static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog)
+{
+	void *prog;
+
+	prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL);
+	if (!prog)
+		return;
+
+	nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64);
+	memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len);
+	kvfree(nfp_prog->prog);
+	nfp_prog->prog = prog;
+}
+
 int nfp_bpf_jit(struct nfp_prog *nfp_prog)
 {
 	int ret;
@@ -2258,5 +2883,102 @@ int nfp_bpf_jit(struct nfp_prog *nfp_prog)
 		return -EINVAL;
 	}
 
-	return nfp_bpf_ustore_calc(nfp_prog, (__force __le64 *)nfp_prog->prog);
+	nfp_bpf_prog_trim(nfp_prog);
+
+	return ret;
+}
+
+void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt)
+{
+	struct nfp_insn_meta *meta;
+
+	/* Another pass to record jump information. */
+	list_for_each_entry(meta, &nfp_prog->insns, l) {
+		u64 code = meta->insn.code;
+
+		if (BPF_CLASS(code) == BPF_JMP && BPF_OP(code) != BPF_EXIT &&
+		    BPF_OP(code) != BPF_CALL) {
+			struct nfp_insn_meta *dst_meta;
+			unsigned short dst_indx;
+
+			dst_indx = meta->n + 1 + meta->insn.off;
+			dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_indx,
+						     cnt);
+
+			meta->jmp_dst = dst_meta;
+			dst_meta->flags |= FLAG_INSN_IS_JUMP_DST;
+		}
+	}
+}
+
+bool nfp_bpf_supported_opcode(u8 code)
+{
+	return !!instr_cb[code];
+}
+
+void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)
+{
+	unsigned int i;
+	u64 *prog;
+	int err;
+
+	prog = kmemdup(nfp_prog->prog, nfp_prog->prog_len * sizeof(u64),
+		       GFP_KERNEL);
+	if (!prog)
+		return ERR_PTR(-ENOMEM);
+
+	for (i = 0; i < nfp_prog->prog_len; i++) {
+		enum nfp_relo_type special;
+		u32 val;
+
+		special = FIELD_GET(OP_RELO_TYPE, prog[i]);
+		switch (special) {
+		case RELO_NONE:
+			continue;
+		case RELO_BR_REL:
+			br_add_offset(&prog[i], bv->start_off);
+			break;
+		case RELO_BR_GO_OUT:
+			br_set_offset(&prog[i],
+				      nfp_prog->tgt_out + bv->start_off);
+			break;
+		case RELO_BR_GO_ABORT:
+			br_set_offset(&prog[i],
+				      nfp_prog->tgt_abort + bv->start_off);
+			break;
+		case RELO_BR_NEXT_PKT:
+			br_set_offset(&prog[i], bv->tgt_done);
+			break;
+		case RELO_BR_HELPER:
+			val = br_get_offset(prog[i]);
+			val -= BR_OFF_RELO;
+			switch (val) {
+			case BPF_FUNC_map_lookup_elem:
+				val = nfp_prog->bpf->helpers.map_lookup;
+				break;
+			default:
+				pr_err("relocation of unknown helper %d\n",
+				       val);
+				err = -EINVAL;
+				goto err_free_prog;
+			}
+			br_set_offset(&prog[i], val);
+			break;
+		case RELO_IMMED_REL:
+			immed_add_value(&prog[i], bv->start_off);
+			break;
+		}
+
+		prog[i] &= ~OP_RELO_TYPE;
+	}
+
+	err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len);
+	if (err)
+		goto err_free_prog;
+
+	return prog;
+
+err_free_prog:
+	kfree(prog);
+	return ERR_PTR(err);
 }
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c
index 13190aa09faf..34e98aa6b956 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c
@@ -34,10 +34,13 @@
 #include <net/pkt_cls.h>
 
 #include "../nfpcore/nfp_cpp.h"
+#include "../nfpcore/nfp_nffw.h"
+#include "../nfpcore/nfp_nsp.h"
 #include "../nfp_app.h"
 #include "../nfp_main.h"
 #include "../nfp_net.h"
 #include "../nfp_port.h"
+#include "fw.h"
 #include "main.h"
 
 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
@@ -52,7 +55,7 @@ static bool nfp_net_ebpf_capable(struct nfp_net *nn)
 
 static int
 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
-		    struct bpf_prog *prog)
+		    struct bpf_prog *prog, struct netlink_ext_ack *extack)
 {
 	bool running, xdp_running;
 	int ret;
@@ -68,10 +71,10 @@ nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
 	if (prog && running && !xdp_running)
 		return -EBUSY;
 
-	ret = nfp_net_bpf_offload(nn, prog, running);
+	ret = nfp_net_bpf_offload(nn, prog, running, extack);
 	/* Stop offload if replace not possible */
 	if (ret && prog)
-		nfp_bpf_xdp_offload(app, nn, NULL);
+		nfp_bpf_xdp_offload(app, nn, NULL, extack);
 
 	nn->dp.bpf_offload_xdp = prog && !ret;
 	return ret;
@@ -85,16 +88,32 @@ static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
 static int
 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
 {
+	struct nfp_pf *pf = app->pf;
+	struct nfp_bpf_vnic *bv;
 	int err;
 
-	nn->app_priv = kzalloc(sizeof(struct nfp_bpf_vnic), GFP_KERNEL);
-	if (!nn->app_priv)
+	if (!pf->eth_tbl) {
+		nfp_err(pf->cpp, "No ETH table\n");
+		return -EINVAL;
+	}
+	if (pf->max_data_vnics != pf->eth_tbl->count) {
+		nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
+			pf->max_data_vnics, pf->eth_tbl->count);
+		return -EINVAL;
+	}
+
+	bv = kzalloc(sizeof(*bv), GFP_KERNEL);
+	if (!bv)
 		return -ENOMEM;
+	nn->app_priv = bv;
 
 	err = nfp_app_nic_vnic_alloc(app, nn, id);
 	if (err)
 		goto err_free_priv;
 
+	bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
+	bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
+
 	return 0;
 err_free_priv:
 	kfree(nn->app_priv);
@@ -105,8 +124,6 @@ static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
 {
 	struct nfp_bpf_vnic *bv = nn->app_priv;
 
-	if (nn->dp.bpf_offload_xdp)
-		nfp_bpf_xdp_offload(app, nn, NULL);
 	WARN_ON(bv->tc_prog);
 	kfree(bv);
 }
@@ -120,17 +137,29 @@ static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
 	struct nfp_bpf_vnic *bv;
 	int err;
 
-	if (type != TC_SETUP_CLSBPF ||
-	    !tc_can_offload(nn->dp.netdev) ||
-	    !nfp_net_ebpf_capable(nn) ||
-	    cls_bpf->common.protocol != htons(ETH_P_ALL) ||
-	    cls_bpf->common.chain_index)
+	if (type != TC_SETUP_CLSBPF) {
+		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
+				   "only offload of BPF classifiers supported");
+		return -EOPNOTSUPP;
+	}
+	if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
+		return -EOPNOTSUPP;
+	if (!nfp_net_ebpf_capable(nn)) {
+		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
+				   "NFP firmware does not support eBPF offload");
+		return -EOPNOTSUPP;
+	}
+	if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
+		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
+				   "only ETH_P_ALL supported as filter protocol");
 		return -EOPNOTSUPP;
+	}
 
 	/* Only support TC direct action */
 	if (!cls_bpf->exts_integrated ||
 	    tcf_exts_has_actions(cls_bpf->exts)) {
-		nn_err(nn, "only direct action with no legacy actions supported\n");
+		NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
+				   "only direct action with no legacy actions supported");
 		return -EOPNOTSUPP;
 	}
 
@@ -147,11 +176,13 @@ static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
 			return 0;
 	}
 
-	err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog);
+	err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
+				  cls_bpf->common.extack);
 	if (err)
 		return err;
 
 	bv->tc_prog = cls_bpf->prog;
+	nn->port->tc_offload_cnt = !!bv->tc_prog;
 	return 0;
 }
 
@@ -189,25 +220,209 @@ static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
 	}
 }
 
-static bool nfp_bpf_tc_busy(struct nfp_app *app, struct nfp_net *nn)
+static int
+nfp_bpf_change_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+	unsigned int max_mtu;
+
+	if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
+		return 0;
+
+	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
+	if (new_mtu > max_mtu) {
+		nn_info(nn, "BPF offload active, MTU over %u not supported\n",
+			max_mtu);
+		return -EBUSY;
+	}
+	return 0;
+}
+
+static int
+nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
+			      u32 length)
+{
+	struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
+	struct nfp_cpp *cpp = bpf->app->pf->cpp;
+
+	if (length < sizeof(*cap)) {
+		nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
+		return -EINVAL;
+	}
+
+	bpf->adjust_head.flags = readl(&cap->flags);
+	bpf->adjust_head.off_min = readl(&cap->off_min);
+	bpf->adjust_head.off_max = readl(&cap->off_max);
+	bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
+	bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
+
+	if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
+		nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
+		return -EINVAL;
+	}
+	if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
+	    !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
+		nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
+		memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
+		return 0;
+	}
+
+	return 0;
+}
+
+static int
+nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
+{
+	struct nfp_bpf_cap_tlv_func __iomem *cap = value;
+
+	if (length < sizeof(*cap)) {
+		nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
+		return -EINVAL;
+	}
+
+	switch (readl(&cap->func_id)) {
+	case BPF_FUNC_map_lookup_elem:
+		bpf->helpers.map_lookup = readl(&cap->func_addr);
+		break;
+	}
+
+	return 0;
+}
+
+static int
+nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
+{
+	struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
+
+	if (length < sizeof(*cap)) {
+		nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
+		return -EINVAL;
+	}
+
+	bpf->maps.types = readl(&cap->types);
+	bpf->maps.max_maps = readl(&cap->max_maps);
+	bpf->maps.max_elems = readl(&cap->max_elems);
+	bpf->maps.max_key_sz = readl(&cap->max_key_sz);
+	bpf->maps.max_val_sz = readl(&cap->max_val_sz);
+	bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
+
+	return 0;
+}
+
+static int nfp_bpf_parse_capabilities(struct nfp_app *app)
+{
+	struct nfp_cpp *cpp = app->pf->cpp;
+	struct nfp_cpp_area *area;
+	u8 __iomem *mem, *start;
+
+	mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
+			    8, &area);
+	if (IS_ERR(mem))
+		return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
+
+	start = mem;
+	while (mem - start + 8 < nfp_cpp_area_size(area)) {
+		u8 __iomem *value;
+		u32 type, length;
+
+		type = readl(mem);
+		length = readl(mem + 4);
+		value = mem + 8;
+
+		mem += 8 + length;
+		if (mem - start > nfp_cpp_area_size(area))
+			goto err_release_free;
+
+		switch (type) {
+		case NFP_BPF_CAP_TYPE_FUNC:
+			if (nfp_bpf_parse_cap_func(app->priv, value, length))
+				goto err_release_free;
+			break;
+		case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
+			if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
+							  length))
+				goto err_release_free;
+			break;
+		case NFP_BPF_CAP_TYPE_MAPS:
+			if (nfp_bpf_parse_cap_maps(app->priv, value, length))
+				goto err_release_free;
+			break;
+		default:
+			nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
+			break;
+		}
+	}
+	if (mem - start != nfp_cpp_area_size(area)) {
+		nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
+			mem - start, nfp_cpp_area_size(area));
+		goto err_release_free;
+	}
+
+	nfp_cpp_area_release_free(area);
+
+	return 0;
+
+err_release_free:
+	nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
+	nfp_cpp_area_release_free(area);
+	return -EINVAL;
+}
+
+static int nfp_bpf_init(struct nfp_app *app)
+{
+	struct nfp_app_bpf *bpf;
+	int err;
+
+	bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
+	if (!bpf)
+		return -ENOMEM;
+	bpf->app = app;
+	app->priv = bpf;
+
+	skb_queue_head_init(&bpf->cmsg_replies);
+	init_waitqueue_head(&bpf->cmsg_wq);
+	INIT_LIST_HEAD(&bpf->map_list);
+
+	err = nfp_bpf_parse_capabilities(app);
+	if (err)
+		goto err_free_bpf;
+
+	return 0;
+
+err_free_bpf:
+	kfree(bpf);
+	return err;
+}
+
+static void nfp_bpf_clean(struct nfp_app *app)
 {
-	return nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
+	struct nfp_app_bpf *bpf = app->priv;
+
+	WARN_ON(!skb_queue_empty(&bpf->cmsg_replies));
+	WARN_ON(!list_empty(&bpf->map_list));
+	WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
+	kfree(bpf);
 }
 
 const struct nfp_app_type app_bpf = {
 	.id		= NFP_APP_BPF_NIC,
 	.name		= "ebpf",
 
+	.ctrl_cap_mask	= 0,
+
+	.init		= nfp_bpf_init,
+	.clean		= nfp_bpf_clean,
+
+	.change_mtu	= nfp_bpf_change_mtu,
+
 	.extra_cap	= nfp_bpf_extra_cap,
 
 	.vnic_alloc	= nfp_bpf_vnic_alloc,
 	.vnic_free	= nfp_bpf_vnic_free,
 
+	.ctrl_msg_rx	= nfp_bpf_ctrl_msg_rx,
+
 	.setup_tc	= nfp_bpf_setup_tc,
-	.tc_busy	= nfp_bpf_tc_busy,
+	.bpf		= nfp_ndo_bpf,
 	.xdp_offload	= nfp_bpf_xdp_offload,
-
-	.bpf_verifier_prep	= nfp_bpf_verifier_prep,
-	.bpf_translate		= nfp_bpf_translate,
-	.bpf_destroy		= nfp_bpf_destroy,
 };
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h
index 57b6043177a3..054df3dc0698 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.h
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Netronome Systems, Inc.
+ * Copyright (C) 2016-2017 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -37,22 +37,40 @@
 #include <linux/bitfield.h>
 #include <linux/bpf.h>
 #include <linux/bpf_verifier.h>
+#include <linux/kernel.h>
 #include <linux/list.h>
+#include <linux/skbuff.h>
 #include <linux/types.h>
+#include <linux/wait.h>
 
 #include "../nfp_asm.h"
+#include "fw.h"
 
-/* For branch fixup logic use up-most byte of branch instruction as scratch
+/* For relocation logic use up-most byte of branch instruction as scratch
  * area.  Remember to clear this before sending instructions to HW!
  */
-#define OP_BR_SPECIAL	0xff00000000000000ULL
-
-enum br_special {
-	OP_BR_NORMAL = 0,
-	OP_BR_GO_OUT,
-	OP_BR_GO_ABORT,
+#define OP_RELO_TYPE	0xff00000000000000ULL
+
+enum nfp_relo_type {
+	RELO_NONE = 0,
+	/* standard internal jumps */
+	RELO_BR_REL,
+	/* internal jumps to parts of the outro */
+	RELO_BR_GO_OUT,
+	RELO_BR_GO_ABORT,
+	/* external jumps to fixed addresses */
+	RELO_BR_NEXT_PKT,
+	RELO_BR_HELPER,
+	/* immediate relocation against load address */
+	RELO_IMMED_REL,
 };
 
+/* To make absolute relocated branches (branches other than RELO_BR_REL)
+ * distinguishable in user space dumps from normal jumps, add a large offset
+ * to them.
+ */
+#define BR_OFF_RELO		15000
+
 enum static_regs {
 	STATIC_REG_IMM		= 21, /* Bank AB */
 	STATIC_REG_STACK	= 22, /* Bank A */
@@ -78,6 +96,89 @@ enum pkt_vec {
 #define NFP_BPF_ABI_FLAGS	reg_imm(0)
 #define   NFP_BPF_ABI_FLAG_MARK	1
 
+/**
+ * struct nfp_app_bpf - bpf app priv structure
+ * @app:		backpointer to the app
+ *
+ * @tag_allocator:	bitmap of control message tags in use
+ * @tag_alloc_next:	next tag bit to allocate
+ * @tag_alloc_last:	next tag bit to be freed
+ *
+ * @cmsg_replies:	received cmsg replies waiting to be consumed
+ * @cmsg_wq:		work queue for waiting for cmsg replies
+ *
+ * @map_list:		list of offloaded maps
+ * @maps_in_use:	number of currently offloaded maps
+ * @map_elems_in_use:	number of elements allocated to offloaded maps
+ *
+ * @adjust_head:	adjust head capability
+ * @adjust_head.flags:		extra flags for adjust head
+ * @adjust_head.off_min:	minimal packet offset within buffer required
+ * @adjust_head.off_max:	maximum packet offset within buffer required
+ * @adjust_head.guaranteed_sub:	negative adjustment guaranteed possible
+ * @adjust_head.guaranteed_add:	positive adjustment guaranteed possible
+ *
+ * @maps:		map capability
+ * @maps.types:			supported map types
+ * @maps.max_maps:		max number of maps supported
+ * @maps.max_elems:		max number of entries in each map
+ * @maps.max_key_sz:		max size of map key
+ * @maps.max_val_sz:		max size of map value
+ * @maps.max_elem_sz:		max size of map entry (key + value)
+ *
+ * @helpers:		helper addressess for various calls
+ * @helpers.map_lookup:		map lookup helper address
+ */
+struct nfp_app_bpf {
+	struct nfp_app *app;
+
+	DECLARE_BITMAP(tag_allocator, U16_MAX + 1);
+	u16 tag_alloc_next;
+	u16 tag_alloc_last;
+
+	struct sk_buff_head cmsg_replies;
+	struct wait_queue_head cmsg_wq;
+
+	struct list_head map_list;
+	unsigned int maps_in_use;
+	unsigned int map_elems_in_use;
+
+	struct nfp_bpf_cap_adjust_head {
+		u32 flags;
+		int off_min;
+		int off_max;
+		int guaranteed_sub;
+		int guaranteed_add;
+	} adjust_head;
+
+	struct {
+		u32 types;
+		u32 max_maps;
+		u32 max_elems;
+		u32 max_key_sz;
+		u32 max_val_sz;
+		u32 max_elem_sz;
+	} maps;
+
+	struct {
+		u32 map_lookup;
+	} helpers;
+};
+
+/**
+ * struct nfp_bpf_map - private per-map data attached to BPF maps for offload
+ * @offmap:	pointer to the offloaded BPF map
+ * @bpf:	back pointer to bpf app private structure
+ * @tid:	table id identifying map on datapath
+ * @l:		link on the nfp_app_bpf->map_list list
+ */
+struct nfp_bpf_map {
+	struct bpf_offloaded_map *offmap;
+	struct nfp_app_bpf *bpf;
+	u32 tid;
+	struct list_head l;
+};
+
 struct nfp_prog;
 struct nfp_insn_meta;
 typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *);
@@ -89,23 +190,47 @@ typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *);
 #define nfp_meta_next(meta)	list_next_entry(meta, l)
 #define nfp_meta_prev(meta)	list_prev_entry(meta, l)
 
+#define FLAG_INSN_IS_JUMP_DST	BIT(0)
+
 /**
  * struct nfp_insn_meta - BPF instruction wrapper
  * @insn: BPF instruction
  * @ptr: pointer type for memory operations
+ * @ldst_gather_len: memcpy length gathered from load/store sequence
+ * @paired_st: the paired store insn at the head of the sequence
  * @ptr_not_const: pointer is not always constant
+ * @jmp_dst: destination info for jump instructions
+ * @func_id: function id for call instructions
+ * @arg1: arg1 for call instructions
+ * @arg2: arg2 for call instructions
+ * @arg2_var_off: arg2 changes stack offset on different paths
  * @off: index of first generated machine instruction (in nfp_prog.prog)
  * @n: eBPF instruction number
+ * @flags: eBPF instruction extra optimization flags
  * @skip: skip this instruction (optimized out)
  * @double_cb: callback for second part of the instruction
  * @l: link on nfp_prog->insns list
  */
 struct nfp_insn_meta {
 	struct bpf_insn insn;
-	struct bpf_reg_state ptr;
-	bool ptr_not_const;
+	union {
+		struct {
+			struct bpf_reg_state ptr;
+			struct bpf_insn *paired_st;
+			s16 ldst_gather_len;
+			bool ptr_not_const;
+		};
+		struct nfp_insn_meta *jmp_dst;
+		struct {
+			u32 func_id;
+			struct bpf_reg_state arg1;
+			struct bpf_reg_state arg2;
+			bool arg2_var_off;
+		};
+	};
 	unsigned int off;
 	unsigned short n;
+	unsigned short flags;
 	bool skip;
 	instr_cb_t double_cb;
 
@@ -134,23 +259,36 @@ static inline u8 mbpf_mode(const struct nfp_insn_meta *meta)
 	return BPF_MODE(meta->insn.code);
 }
 
+static inline bool is_mbpf_load(const struct nfp_insn_meta *meta)
+{
+	return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM);
+}
+
+static inline bool is_mbpf_store(const struct nfp_insn_meta *meta)
+{
+	return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM);
+}
+
 /**
  * struct nfp_prog - nfp BPF program
+ * @bpf: backpointer to the bpf app priv structure
  * @prog: machine code
  * @prog_len: number of valid instructions in @prog array
  * @__prog_alloc_len: alloc size of @prog array
  * @verifier_meta: temporary storage for verifier's insn meta
  * @type: BPF program type
- * @start_off: address of the first instruction in the memory
+ * @last_bpf_off: address of the last instruction translated from BPF
  * @tgt_out: jump target for normal exit
  * @tgt_abort: jump target for abort (e.g. access outside of packet buffer)
- * @tgt_done: jump target to get the next packet
  * @n_translated: number of successfully translated instructions (for errors)
  * @error: error code if something went wrong
  * @stack_depth: max stack depth from the verifier
+ * @adjust_head_location: if program has single adjust head call - the insn no.
  * @insns: list of BPF instruction wrappers (struct nfp_insn_meta)
  */
 struct nfp_prog {
+	struct nfp_app_bpf *bpf;
+
 	u64 *prog;
 	unsigned int prog_len;
 	unsigned int __prog_alloc_len;
@@ -159,15 +297,15 @@ struct nfp_prog {
 
 	enum bpf_prog_type type;
 
-	unsigned int start_off;
+	unsigned int last_bpf_off;
 	unsigned int tgt_out;
 	unsigned int tgt_abort;
-	unsigned int tgt_done;
 
 	unsigned int n_translated;
 	int error;
 
 	unsigned int stack_depth;
+	unsigned int adjust_head_location;
 
 	struct list_head insns;
 };
@@ -175,26 +313,49 @@ struct nfp_prog {
 /**
  * struct nfp_bpf_vnic - per-vNIC BPF priv structure
  * @tc_prog:	currently loaded cls_bpf program
+ * @start_off:	address of the first instruction in the memory
+ * @tgt_done:	jump target to get the next packet
  */
 struct nfp_bpf_vnic {
 	struct bpf_prog *tc_prog;
+	unsigned int start_off;
+	unsigned int tgt_done;
 };
 
+void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog, unsigned int cnt);
 int nfp_bpf_jit(struct nfp_prog *prog);
+bool nfp_bpf_supported_opcode(u8 code);
 
-extern const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops;
+extern const struct bpf_prog_offload_ops nfp_bpf_analyzer_ops;
 
 struct netdev_bpf;
 struct nfp_app;
 struct nfp_net;
 
+int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn,
+		struct netdev_bpf *bpf);
 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
-			bool old_prog);
-
-int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
-			  struct netdev_bpf *bpf);
-int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
-		      struct bpf_prog *prog);
-int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
-		    struct bpf_prog *prog);
+			bool old_prog, struct netlink_ext_ack *extack);
+
+struct nfp_insn_meta *
+nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+		  unsigned int insn_idx, unsigned int n_insns);
+
+void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv);
+
+long long int
+nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map);
+void
+nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map);
+int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
+				void *next_key);
+int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
+			      void *key, void *value, u64 flags);
+int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key);
+int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
+			      void *key, void *value);
+int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
+			       void *key, void *next_key);
+
+void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb);
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/offload.c b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
index bc879aeb62d4..0a7732385469 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/offload.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Netronome Systems, Inc.
+ * Copyright (C) 2016-2017 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -36,18 +36,23 @@
  * Netronome network device driver: TC offload functions for PF and VF
  */
 
+#define pr_fmt(fmt)	"NFP net bpf: " fmt
+
+#include <linux/bpf.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/pci.h>
 #include <linux/jiffies.h>
 #include <linux/timer.h>
 #include <linux/list.h>
+#include <linux/mm.h>
 
 #include <net/pkt_cls.h>
 #include <net/tc_act/tc_gact.h>
 #include <net/tc_act/tc_mirred.h>
 
 #include "main.h"
+#include "../nfp_app.h"
 #include "../nfp_net_ctrl.h"
 #include "../nfp_net.h"
 
@@ -55,11 +60,10 @@ static int
 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
 		 unsigned int cnt)
 {
+	struct nfp_insn_meta *meta;
 	unsigned int i;
 
 	for (i = 0; i < cnt; i++) {
-		struct nfp_insn_meta *meta;
-
 		meta = kzalloc(sizeof(*meta), GFP_KERNEL);
 		if (!meta)
 			return -ENOMEM;
@@ -70,6 +74,8 @@ nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
 		list_add_tail(&meta->l, &nfp_prog->insns);
 	}
 
+	nfp_bpf_jit_prepare(nfp_prog, cnt);
+
 	return 0;
 }
 
@@ -84,8 +90,9 @@ static void nfp_prog_free(struct nfp_prog *nfp_prog)
 	kfree(nfp_prog);
 }
 
-int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
-			  struct netdev_bpf *bpf)
+static int
+nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
+		      struct netdev_bpf *bpf)
 {
 	struct bpf_prog *prog = bpf->verifier.prog;
 	struct nfp_prog *nfp_prog;
@@ -98,6 +105,7 @@ int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
 
 	INIT_LIST_HEAD(&nfp_prog->insns);
 	nfp_prog->type = prog->type;
+	nfp_prog->bpf = app->priv;
 
 	ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
 	if (ret)
@@ -114,12 +122,12 @@ int nfp_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
 	return ret;
 }
 
-int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
-		      struct bpf_prog *prog)
+static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog)
 {
 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 	unsigned int stack_size;
 	unsigned int max_instr;
+	int err;
 
 	stack_size = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
 	if (prog->aux->stack_depth > stack_size) {
@@ -127,50 +135,179 @@ int nfp_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
 			prog->aux->stack_depth, stack_size);
 		return -EOPNOTSUPP;
 	}
-
-	nfp_prog->stack_depth = prog->aux->stack_depth;
-	nfp_prog->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
-	nfp_prog->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
+	nfp_prog->stack_depth = round_up(prog->aux->stack_depth, 4);
 
 	max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
 	nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
 
-	nfp_prog->prog = kmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
+	nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
 	if (!nfp_prog->prog)
 		return -ENOMEM;
 
-	return nfp_bpf_jit(nfp_prog);
+	err = nfp_bpf_jit(nfp_prog);
+	if (err)
+		return err;
+
+	prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
+	prog->aux->offload->jited_image = nfp_prog->prog;
+
+	return 0;
 }
 
-int nfp_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
-		    struct bpf_prog *prog)
+static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
 {
 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 
-	kfree(nfp_prog->prog);
+	kvfree(nfp_prog->prog);
 	nfp_prog_free(nfp_prog);
 
 	return 0;
 }
 
-static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
+static int
+nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
+			 void *key, void *next_key)
+{
+	if (!key)
+		return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
+	return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
+}
+
+static int
+nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
+{
+	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
+		return -EINVAL;
+	return nfp_bpf_ctrl_del_entry(offmap, key);
+}
+
+static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
+	.map_get_next_key	= nfp_bpf_map_get_next_key,
+	.map_lookup_elem	= nfp_bpf_ctrl_lookup_entry,
+	.map_update_elem	= nfp_bpf_ctrl_update_entry,
+	.map_delete_elem	= nfp_bpf_map_delete_elem,
+};
+
+static int
+nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
+{
+	struct nfp_bpf_map *nfp_map;
+	long long int res;
+
+	if (!bpf->maps.types)
+		return -EOPNOTSUPP;
+
+	if (offmap->map.map_flags ||
+	    offmap->map.numa_node != NUMA_NO_NODE) {
+		pr_info("map flags are not supported\n");
+		return -EINVAL;
+	}
+
+	if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
+		pr_info("map type not supported\n");
+		return -EOPNOTSUPP;
+	}
+	if (bpf->maps.max_maps == bpf->maps_in_use) {
+		pr_info("too many maps for a device\n");
+		return -ENOMEM;
+	}
+	if (bpf->maps.max_elems - bpf->map_elems_in_use <
+	    offmap->map.max_entries) {
+		pr_info("map with too many elements: %u, left: %u\n",
+			offmap->map.max_entries,
+			bpf->maps.max_elems - bpf->map_elems_in_use);
+		return -ENOMEM;
+	}
+	if (offmap->map.key_size > bpf->maps.max_key_sz ||
+	    offmap->map.value_size > bpf->maps.max_val_sz ||
+	    round_up(offmap->map.key_size, 8) +
+	    round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
+		pr_info("elements don't fit in device constraints\n");
+		return -ENOMEM;
+	}
+
+	nfp_map = kzalloc(sizeof(*nfp_map), GFP_USER);
+	if (!nfp_map)
+		return -ENOMEM;
+
+	offmap->dev_priv = nfp_map;
+	nfp_map->offmap = offmap;
+	nfp_map->bpf = bpf;
+
+	res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
+	if (res < 0) {
+		kfree(nfp_map);
+		return res;
+	}
+
+	nfp_map->tid = res;
+	offmap->dev_ops = &nfp_bpf_map_ops;
+	bpf->maps_in_use++;
+	bpf->map_elems_in_use += offmap->map.max_entries;
+	list_add_tail(&nfp_map->l, &bpf->map_list);
+
+	return 0;
+}
+
+static int
+nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
+{
+	struct nfp_bpf_map *nfp_map = offmap->dev_priv;
+
+	nfp_bpf_ctrl_free_map(bpf, nfp_map);
+	list_del_init(&nfp_map->l);
+	bpf->map_elems_in_use -= offmap->map.max_entries;
+	bpf->maps_in_use--;
+	kfree(nfp_map);
+
+	return 0;
+}
+
+int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
+{
+	switch (bpf->command) {
+	case BPF_OFFLOAD_VERIFIER_PREP:
+		return nfp_bpf_verifier_prep(app, nn, bpf);
+	case BPF_OFFLOAD_TRANSLATE:
+		return nfp_bpf_translate(nn, bpf->offload.prog);
+	case BPF_OFFLOAD_DESTROY:
+		return nfp_bpf_destroy(nn, bpf->offload.prog);
+	case BPF_OFFLOAD_MAP_ALLOC:
+		return nfp_bpf_map_alloc(app->priv, bpf->offmap);
+	case BPF_OFFLOAD_MAP_FREE:
+		return nfp_bpf_map_free(app->priv, bpf->offmap);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int
+nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
+		 struct netlink_ext_ack *extack)
 {
 	struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 	unsigned int max_mtu;
 	dma_addr_t dma_addr;
+	void *img;
 	int err;
 
 	max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
 	if (max_mtu < nn->dp.netdev->mtu) {
-		nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
+		NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with MTU larger than HW packet split boundary");
 		return -EOPNOTSUPP;
 	}
 
-	dma_addr = dma_map_single(nn->dp.dev, nfp_prog->prog,
+	img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
+	if (IS_ERR(img))
+		return PTR_ERR(img);
+
+	dma_addr = dma_map_single(nn->dp.dev, img,
 				  nfp_prog->prog_len * sizeof(u64),
 				  DMA_TO_DEVICE);
-	if (dma_mapping_error(nn->dp.dev, dma_addr))
+	if (dma_mapping_error(nn->dp.dev, dma_addr)) {
+		kfree(img);
 		return -ENOMEM;
+	}
 
 	nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
 	nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
@@ -178,15 +315,18 @@ static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog)
 	/* Load up the JITed code */
 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
 	if (err)
-		nn_err(nn, "FW command error while loading BPF: %d\n", err);
+		NL_SET_ERR_MSG_MOD(extack,
+				   "FW command error while loading BPF");
 
 	dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
 			 DMA_TO_DEVICE);
+	kfree(img);
 
 	return err;
 }
 
-static void nfp_net_bpf_start(struct nfp_net *nn)
+static void
+nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
 {
 	int err;
 
@@ -195,7 +335,8 @@ static void nfp_net_bpf_start(struct nfp_net *nn)
 	nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
 	if (err)
-		nn_err(nn, "FW command error while enabling BPF: %d\n", err);
+		NL_SET_ERR_MSG_MOD(extack,
+				   "FW command error while enabling BPF");
 }
 
 static int nfp_net_bpf_stop(struct nfp_net *nn)
@@ -210,12 +351,12 @@ static int nfp_net_bpf_stop(struct nfp_net *nn)
 }
 
 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
-			bool old_prog)
+			bool old_prog, struct netlink_ext_ack *extack)
 {
 	int err;
 
 	if (prog) {
-		struct bpf_dev_offload *offload = prog->aux->offload;
+		struct bpf_prog_offload *offload = prog->aux->offload;
 
 		if (!offload)
 			return -EINVAL;
@@ -228,7 +369,8 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
 
 		cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
 		if (!(cap & NFP_NET_BPF_CAP_RELO)) {
-			nn_err(nn, "FW does not support live reload\n");
+			NL_SET_ERR_MSG_MOD(extack,
+					   "FW does not support live reload");
 			return -EBUSY;
 		}
 	}
@@ -240,12 +382,12 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
 	if (old_prog && !prog)
 		return nfp_net_bpf_stop(nn);
 
-	err = nfp_net_bpf_load(nn, prog);
+	err = nfp_net_bpf_load(nn, prog, extack);
 	if (err)
 		return err;
 
 	if (!old_prog)
-		nfp_net_bpf_start(nn);
+		nfp_net_bpf_start(nn, extack);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 8d43491ddd6b..479f602887e9 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Netronome Systems, Inc.
+ * Copyright (C) 2016-2017 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -31,16 +31,18 @@
  * SOFTWARE.
  */
 
-#define pr_fmt(fmt)	"NFP net bpf: " fmt
-
 #include <linux/bpf.h>
 #include <linux/bpf_verifier.h>
 #include <linux/kernel.h>
 #include <linux/pkt_cls.h>
 
+#include "fw.h"
 #include "main.h"
 
-static struct nfp_insn_meta *
+#define pr_vlog(env, fmt, ...)	\
+	bpf_verifier_log_write(env, "[nfp] " fmt, ##__VA_ARGS__)
+
+struct nfp_insn_meta *
 nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 		  unsigned int insn_idx, unsigned int n_insns)
 {
@@ -68,6 +70,114 @@ nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	return meta;
 }
 
+static void
+nfp_record_adjust_head(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
+		       struct nfp_insn_meta *meta,
+		       const struct bpf_reg_state *reg2)
+{
+	unsigned int location =	UINT_MAX;
+	int imm;
+
+	/* Datapath usually can give us guarantees on how much adjust head
+	 * can be done without the need for any checks.  Optimize the simple
+	 * case where there is only one adjust head by a constant.
+	 */
+	if (reg2->type != SCALAR_VALUE || !tnum_is_const(reg2->var_off))
+		goto exit_set_location;
+	imm = reg2->var_off.value;
+	/* Translator will skip all checks, we need to guarantee min pkt len */
+	if (imm > ETH_ZLEN - ETH_HLEN)
+		goto exit_set_location;
+	if (imm > (int)bpf->adjust_head.guaranteed_add ||
+	    imm < -bpf->adjust_head.guaranteed_sub)
+		goto exit_set_location;
+
+	if (nfp_prog->adjust_head_location) {
+		/* Only one call per program allowed */
+		if (nfp_prog->adjust_head_location != meta->n)
+			goto exit_set_location;
+
+		if (meta->arg2.var_off.value != imm)
+			goto exit_set_location;
+	}
+
+	location = meta->n;
+exit_set_location:
+	nfp_prog->adjust_head_location = location;
+}
+
+static int
+nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct bpf_verifier_env *env,
+		   struct nfp_insn_meta *meta)
+{
+	const struct bpf_reg_state *reg1 = cur_regs(env) + BPF_REG_1;
+	const struct bpf_reg_state *reg2 = cur_regs(env) + BPF_REG_2;
+	struct nfp_app_bpf *bpf = nfp_prog->bpf;
+	u32 func_id = meta->insn.imm;
+	s64 off, old_off;
+
+	switch (func_id) {
+	case BPF_FUNC_xdp_adjust_head:
+		if (!bpf->adjust_head.off_max) {
+			pr_vlog(env, "adjust_head not supported by FW\n");
+			return -EOPNOTSUPP;
+		}
+		if (!(bpf->adjust_head.flags & NFP_BPF_ADJUST_HEAD_NO_META)) {
+			pr_vlog(env, "adjust_head: FW requires shifting metadata, not supported by the driver\n");
+			return -EOPNOTSUPP;
+		}
+
+		nfp_record_adjust_head(bpf, nfp_prog, meta, reg2);
+		break;
+
+	case BPF_FUNC_map_lookup_elem:
+		if (!bpf->helpers.map_lookup) {
+			pr_vlog(env, "map_lookup: not supported by FW\n");
+			return -EOPNOTSUPP;
+		}
+		if (reg2->type != PTR_TO_STACK) {
+			pr_vlog(env,
+				"map_lookup: unsupported key ptr type %d\n",
+				reg2->type);
+			return -EOPNOTSUPP;
+		}
+		if (!tnum_is_const(reg2->var_off)) {
+			pr_vlog(env, "map_lookup: variable key pointer\n");
+			return -EOPNOTSUPP;
+		}
+
+		off = reg2->var_off.value + reg2->off;
+		if (-off % 4) {
+			pr_vlog(env,
+				"map_lookup: unaligned stack pointer %lld\n",
+				-off);
+			return -EOPNOTSUPP;
+		}
+
+		/* Rest of the checks is only if we re-parse the same insn */
+		if (!meta->func_id)
+			break;
+
+		old_off = meta->arg2.var_off.value + meta->arg2.off;
+		meta->arg2_var_off |= off != old_off;
+
+		if (meta->arg1.map_ptr != reg1->map_ptr) {
+			pr_vlog(env, "map_lookup: called for different map\n");
+			return -EOPNOTSUPP;
+		}
+		break;
+	default:
+		pr_vlog(env, "unsupported function id: %d\n", func_id);
+		return -EOPNOTSUPP;
+	}
+
+	meta->func_id = func_id;
+	meta->arg1 = *reg1;
+	meta->arg2 = *reg2;
+
+	return 0;
+}
+
 static int
 nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
 		   struct bpf_verifier_env *env)
@@ -82,7 +192,7 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
 		char tn_buf[48];
 
 		tnum_strn(tn_buf, sizeof(tn_buf), reg0->var_off);
-		pr_info("unsupported exit state: %d, var_off: %s\n",
+		pr_vlog(env, "unsupported exit state: %d, var_off: %s\n",
 			reg0->type, tn_buf);
 		return -EINVAL;
 	}
@@ -92,7 +202,7 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
 	    imm <= TC_ACT_REDIRECT &&
 	    imm != TC_ACT_SHOT && imm != TC_ACT_STOLEN &&
 	    imm != TC_ACT_QUEUED) {
-		pr_info("unsupported exit state: %d, imm: %llx\n",
+		pr_vlog(env, "unsupported exit state: %d, imm: %llx\n",
 			reg0->type, imm);
 		return -EINVAL;
 	}
@@ -103,12 +213,13 @@ nfp_bpf_check_exit(struct nfp_prog *nfp_prog,
 static int
 nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,
 			   struct nfp_insn_meta *meta,
-			   const struct bpf_reg_state *reg)
+			   const struct bpf_reg_state *reg,
+			   struct bpf_verifier_env *env)
 {
 	s32 old_off, new_off;
 
 	if (!tnum_is_const(reg->var_off)) {
-		pr_info("variable ptr stack access\n");
+		pr_vlog(env, "variable ptr stack access\n");
 		return -EINVAL;
 	}
 
@@ -126,7 +237,7 @@ nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,
 	if (old_off % 4 == new_off % 4)
 		return 0;
 
-	pr_info("stack access changed location was:%d is:%d\n",
+	pr_vlog(env, "stack access changed location was:%d is:%d\n",
 		old_off, new_off);
 	return -EINVAL;
 }
@@ -140,19 +251,27 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 
 	if (reg->type != PTR_TO_CTX &&
 	    reg->type != PTR_TO_STACK &&
+	    reg->type != PTR_TO_MAP_VALUE &&
 	    reg->type != PTR_TO_PACKET) {
-		pr_info("unsupported ptr type: %d\n", reg->type);
+		pr_vlog(env, "unsupported ptr type: %d\n", reg->type);
 		return -EINVAL;
 	}
 
 	if (reg->type == PTR_TO_STACK) {
-		err = nfp_bpf_check_stack_access(nfp_prog, meta, reg);
+		err = nfp_bpf_check_stack_access(nfp_prog, meta, reg, env);
 		if (err)
 			return err;
 	}
 
+	if (reg->type == PTR_TO_MAP_VALUE) {
+		if (is_mbpf_store(meta)) {
+			pr_vlog(env, "map writes not supported\n");
+			return -EOPNOTSUPP;
+		}
+	}
+
 	if (meta->ptr.type != NOT_INIT && meta->ptr.type != reg->type) {
-		pr_info("ptr type changed for instruction %d -> %d\n",
+		pr_vlog(env, "ptr type changed for instruction %d -> %d\n",
 			meta->ptr.type, reg->type);
 		return -EINVAL;
 	}
@@ -171,25 +290,33 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
 	meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx, env->prog->len);
 	nfp_prog->verifier_meta = meta;
 
+	if (!nfp_bpf_supported_opcode(meta->insn.code)) {
+		pr_vlog(env, "instruction %#02x not supported\n",
+			meta->insn.code);
+		return -EINVAL;
+	}
+
 	if (meta->insn.src_reg >= MAX_BPF_REG ||
 	    meta->insn.dst_reg >= MAX_BPF_REG) {
-		pr_err("program uses extended registers - jit hardening?\n");
+		pr_vlog(env, "program uses extended registers - jit hardening?\n");
 		return -EINVAL;
 	}
 
+	if (meta->insn.code == (BPF_JMP | BPF_CALL))
+		return nfp_bpf_check_call(nfp_prog, env, meta);
 	if (meta->insn.code == (BPF_JMP | BPF_EXIT))
 		return nfp_bpf_check_exit(nfp_prog, env);
 
-	if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_LDX | BPF_MEM))
+	if (is_mbpf_load(meta))
 		return nfp_bpf_check_ptr(nfp_prog, meta, env,
 					 meta->insn.src_reg);
-	if ((meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_MEM))
+	if (is_mbpf_store(meta))
 		return nfp_bpf_check_ptr(nfp_prog, meta, env,
 					 meta->insn.dst_reg);
 
 	return 0;
 }
 
-const struct bpf_ext_analyzer_ops nfp_bpf_analyzer_ops = {
+const struct bpf_prog_offload_ops nfp_bpf_analyzer_ops = {
 	.insn_hook = nfp_verify_insn,
 };
diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c
index c1c595f8bb87..b3567a596fc1 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/action.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/action.c
@@ -81,6 +81,9 @@ static bool nfp_fl_netdev_is_tunnel_type(struct net_device *out_dev,
 	if (!strcmp(out_dev->rtnl_link_ops->kind, "vxlan"))
 		return tun_type == NFP_FL_TUNNEL_VXLAN;
 
+	if (!strcmp(out_dev->rtnl_link_ops->kind, "geneve"))
+		return tun_type == NFP_FL_TUNNEL_GENEVE;
+
 	return false;
 }
 
@@ -93,13 +96,11 @@ nfp_fl_output(struct nfp_fl_output *output, const struct tc_action *action,
 	size_t act_size = sizeof(struct nfp_fl_output);
 	struct net_device *out_dev;
 	u16 tmp_flags;
-	int ifindex;
 
 	output->head.jump_id = NFP_FL_ACTION_OPCODE_OUTPUT;
 	output->head.len_lw = act_size >> NFP_FL_LW_SIZ;
 
-	ifindex = tcf_mirred_ifindex(action);
-	out_dev = __dev_get_by_index(dev_net(in_dev), ifindex);
+	out_dev = tcf_mirred_dev(action);
 	if (!out_dev)
 		return -EOPNOTSUPP;
 
@@ -138,11 +139,23 @@ nfp_fl_output(struct nfp_fl_output *output, const struct tc_action *action,
 	return 0;
 }
 
-static bool nfp_fl_supported_tun_port(const struct tc_action *action)
+static enum nfp_flower_tun_type
+nfp_fl_get_tun_from_act_l4_port(struct nfp_app *app,
+				const struct tc_action *action)
 {
 	struct ip_tunnel_info *tun = tcf_tunnel_info(action);
-
-	return tun->key.tp_dst == htons(NFP_FL_VXLAN_PORT);
+	struct nfp_flower_priv *priv = app->priv;
+
+	switch (tun->key.tp_dst) {
+	case htons(NFP_FL_VXLAN_PORT):
+		return NFP_FL_TUNNEL_VXLAN;
+	case htons(NFP_FL_GENEVE_PORT):
+		if (priv->flower_ext_feats & NFP_FL_FEATS_GENEVE)
+			return NFP_FL_TUNNEL_GENEVE;
+		/* FALLTHROUGH */
+	default:
+		return NFP_FL_TUNNEL_NONE;
+	}
 }
 
 static struct nfp_fl_pre_tunnel *nfp_fl_pre_tunnel(char *act_data, int act_len)
@@ -167,38 +180,33 @@ static struct nfp_fl_pre_tunnel *nfp_fl_pre_tunnel(char *act_data, int act_len)
 }
 
 static int
-nfp_fl_set_vxlan(struct nfp_fl_set_vxlan *set_vxlan,
-		 const struct tc_action *action,
-		 struct nfp_fl_pre_tunnel *pre_tun)
+nfp_fl_set_ipv4_udp_tun(struct nfp_fl_set_ipv4_udp_tun *set_tun,
+			const struct tc_action *action,
+			struct nfp_fl_pre_tunnel *pre_tun,
+			enum nfp_flower_tun_type tun_type)
 {
-	struct ip_tunnel_info *vxlan = tcf_tunnel_info(action);
-	size_t act_size = sizeof(struct nfp_fl_set_vxlan);
-	u32 tmp_set_vxlan_type_index = 0;
+	size_t act_size = sizeof(struct nfp_fl_set_ipv4_udp_tun);
+	struct ip_tunnel_info *ip_tun = tcf_tunnel_info(action);
+	u32 tmp_set_ip_tun_type_index = 0;
 	/* Currently support one pre-tunnel so index is always 0. */
 	int pretun_idx = 0;
 
-	if (vxlan->options_len) {
-		/* Do not support options e.g. vxlan gpe. */
+	if (ip_tun->options_len)
 		return -EOPNOTSUPP;
-	}
 
-	set_vxlan->head.jump_id = NFP_FL_ACTION_OPCODE_SET_IPV4_TUNNEL;
-	set_vxlan->head.len_lw = act_size >> NFP_FL_LW_SIZ;
+	set_tun->head.jump_id = NFP_FL_ACTION_OPCODE_SET_IPV4_TUNNEL;
+	set_tun->head.len_lw = act_size >> NFP_FL_LW_SIZ;
 
 	/* Set tunnel type and pre-tunnel index. */
-	tmp_set_vxlan_type_index |=
-		FIELD_PREP(NFP_FL_IPV4_TUNNEL_TYPE, NFP_FL_TUNNEL_VXLAN) |
+	tmp_set_ip_tun_type_index |=
+		FIELD_PREP(NFP_FL_IPV4_TUNNEL_TYPE, tun_type) |
 		FIELD_PREP(NFP_FL_IPV4_PRE_TUN_INDEX, pretun_idx);
 
-	set_vxlan->tun_type_index = cpu_to_be32(tmp_set_vxlan_type_index);
-
-	set_vxlan->tun_id = vxlan->key.tun_id;
-	set_vxlan->tun_flags = vxlan->key.tun_flags;
-	set_vxlan->ipv4_ttl = vxlan->key.ttl;
-	set_vxlan->ipv4_tos = vxlan->key.tos;
+	set_tun->tun_type_index = cpu_to_be32(tmp_set_ip_tun_type_index);
+	set_tun->tun_id = ip_tun->key.tun_id;
 
 	/* Complete pre_tunnel action. */
-	pre_tun->ipv4_dst = vxlan->key.u.ipv4.dst;
+	pre_tun->ipv4_dst = ip_tun->key.u.ipv4.dst;
 
 	return 0;
 }
@@ -435,8 +443,8 @@ nfp_flower_loop_action(const struct tc_action *a,
 		       struct net_device *netdev,
 		       enum nfp_flower_tun_type *tun_type, int *tun_out_cnt)
 {
+	struct nfp_fl_set_ipv4_udp_tun *set_tun;
 	struct nfp_fl_pre_tunnel *pre_tun;
-	struct nfp_fl_set_vxlan *s_vxl;
 	struct nfp_fl_push_vlan *psh_v;
 	struct nfp_fl_pop_vlan *pop_v;
 	struct nfp_fl_output *output;
@@ -484,26 +492,29 @@ nfp_flower_loop_action(const struct tc_action *a,
 
 		nfp_fl_push_vlan(psh_v, a);
 		*a_len += sizeof(struct nfp_fl_push_vlan);
-	} else if (is_tcf_tunnel_set(a) && nfp_fl_supported_tun_port(a)) {
+	} else if (is_tcf_tunnel_set(a)) {
+		struct nfp_repr *repr = netdev_priv(netdev);
+		*tun_type = nfp_fl_get_tun_from_act_l4_port(repr->app, a);
+		if (*tun_type == NFP_FL_TUNNEL_NONE)
+			return -EOPNOTSUPP;
+
 		/* Pre-tunnel action is required for tunnel encap.
 		 * This checks for next hop entries on NFP.
 		 * If none, the packet falls back before applying other actions.
 		 */
 		if (*a_len + sizeof(struct nfp_fl_pre_tunnel) +
-		    sizeof(struct nfp_fl_set_vxlan) > NFP_FL_MAX_A_SIZ)
+		    sizeof(struct nfp_fl_set_ipv4_udp_tun) > NFP_FL_MAX_A_SIZ)
 			return -EOPNOTSUPP;
 
-		*tun_type = NFP_FL_TUNNEL_VXLAN;
 		pre_tun = nfp_fl_pre_tunnel(nfp_fl->action_data, *a_len);
 		nfp_fl->meta.shortcut = cpu_to_be32(NFP_FL_SC_ACT_NULL);
 		*a_len += sizeof(struct nfp_fl_pre_tunnel);
 
-		s_vxl = (struct nfp_fl_set_vxlan *)&nfp_fl->action_data[*a_len];
-		err = nfp_fl_set_vxlan(s_vxl, a, pre_tun);
+		set_tun = (void *)&nfp_fl->action_data[*a_len];
+		err = nfp_fl_set_ipv4_udp_tun(set_tun, a, pre_tun, *tun_type);
 		if (err)
 			return err;
-
-		*a_len += sizeof(struct nfp_fl_set_vxlan);
+		*a_len += sizeof(struct nfp_fl_set_ipv4_udp_tun);
 	} else if (is_tcf_tunnel_release(a)) {
 		/* Tunnel decap is handled by default so accept action. */
 		return 0;
diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
index e98bb9cdb6a3..baaea6f1a9d8 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c
@@ -125,6 +125,27 @@ int nfp_flower_cmsg_portmod(struct nfp_repr *repr, bool carrier_ok)
 	return 0;
 }
 
+int nfp_flower_cmsg_portreify(struct nfp_repr *repr, bool exists)
+{
+	struct nfp_flower_cmsg_portreify *msg;
+	struct sk_buff *skb;
+
+	skb = nfp_flower_cmsg_alloc(repr->app, sizeof(*msg),
+				    NFP_FLOWER_CMSG_TYPE_PORT_REIFY,
+				    GFP_KERNEL);
+	if (!skb)
+		return -ENOMEM;
+
+	msg = nfp_flower_cmsg_get_data(skb);
+	msg->portnum = cpu_to_be32(repr->dst->u.port_info.port_id);
+	msg->reserved = 0;
+	msg->info = cpu_to_be16(exists);
+
+	nfp_ctrl_tx(repr->app->ctrl, skb);
+
+	return 0;
+}
+
 static void
 nfp_flower_cmsg_portmod_rx(struct nfp_app *app, struct sk_buff *skb)
 {
@@ -160,6 +181,28 @@ nfp_flower_cmsg_portmod_rx(struct nfp_app *app, struct sk_buff *skb)
 	rtnl_unlock();
 }
 
+static void
+nfp_flower_cmsg_portreify_rx(struct nfp_app *app, struct sk_buff *skb)
+{
+	struct nfp_flower_priv *priv = app->priv;
+	struct nfp_flower_cmsg_portreify *msg;
+	bool exists;
+
+	msg = nfp_flower_cmsg_get_data(skb);
+
+	rcu_read_lock();
+	exists = !!nfp_app_repr_get(app, be32_to_cpu(msg->portnum));
+	rcu_read_unlock();
+	if (!exists) {
+		nfp_flower_cmsg_warn(app, "ctrl msg for unknown port 0x%08x\n",
+				     be32_to_cpu(msg->portnum));
+		return;
+	}
+
+	atomic_inc(&priv->reify_replies);
+	wake_up_interruptible(&priv->reify_wait_queue);
+}
+
 static void
 nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb)
 {
@@ -168,20 +211,14 @@ nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb)
 
 	cmsg_hdr = nfp_flower_cmsg_get_hdr(skb);
 
-	if (unlikely(cmsg_hdr->version != NFP_FLOWER_CMSG_VER1)) {
-		nfp_flower_cmsg_warn(app, "Cannot handle repr control version %u\n",
-				     cmsg_hdr->version);
-		goto out;
-	}
-
 	type = cmsg_hdr->type;
 	switch (type) {
+	case NFP_FLOWER_CMSG_TYPE_PORT_REIFY:
+		nfp_flower_cmsg_portreify_rx(app, skb);
+		break;
 	case NFP_FLOWER_CMSG_TYPE_PORT_MOD:
 		nfp_flower_cmsg_portmod_rx(app, skb);
 		break;
-	case NFP_FLOWER_CMSG_TYPE_FLOW_STATS:
-		nfp_flower_rx_flow_stats(app, skb);
-		break;
 	case NFP_FLOWER_CMSG_TYPE_NO_NEIGH:
 		nfp_tunnel_request_route(app, skb);
 		break;
@@ -217,7 +254,23 @@ void nfp_flower_cmsg_process_rx(struct work_struct *work)
 void nfp_flower_cmsg_rx(struct nfp_app *app, struct sk_buff *skb)
 {
 	struct nfp_flower_priv *priv = app->priv;
+	struct nfp_flower_cmsg_hdr *cmsg_hdr;
+
+	cmsg_hdr = nfp_flower_cmsg_get_hdr(skb);
 
-	skb_queue_tail(&priv->cmsg_skbs, skb);
-	schedule_work(&priv->cmsg_work);
+	if (unlikely(cmsg_hdr->version != NFP_FLOWER_CMSG_VER1)) {
+		nfp_flower_cmsg_warn(app, "Cannot handle repr control version %u\n",
+				     cmsg_hdr->version);
+		dev_kfree_skb_any(skb);
+		return;
+	}
+
+	if (cmsg_hdr->type == NFP_FLOWER_CMSG_TYPE_FLOW_STATS) {
+		/* We need to deal with stats updates from HW asap */
+		nfp_flower_rx_flow_stats(app, skb);
+		dev_consume_skb_any(skb);
+	} else {
+		skb_queue_tail(&priv->cmsg_skbs, skb);
+		schedule_work(&priv->cmsg_work);
+	}
 }
diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
index 66070741d55f..adfe474c2cf0 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h
@@ -41,7 +41,7 @@
 #include "../nfp_app.h"
 #include "../nfpcore/nfp_cpp.h"
 
-#define NFP_FLOWER_LAYER_META		BIT(0)
+#define NFP_FLOWER_LAYER_EXT_META	BIT(0)
 #define NFP_FLOWER_LAYER_PORT		BIT(1)
 #define NFP_FLOWER_LAYER_MAC		BIT(2)
 #define NFP_FLOWER_LAYER_TP		BIT(3)
@@ -50,8 +50,7 @@
 #define NFP_FLOWER_LAYER_CT		BIT(6)
 #define NFP_FLOWER_LAYER_VXLAN		BIT(7)
 
-#define NFP_FLOWER_LAYER_ETHER		BIT(3)
-#define NFP_FLOWER_LAYER_ARP		BIT(4)
+#define NFP_FLOWER_LAYER2_GENEVE	BIT(5)
 
 #define NFP_FLOWER_MASK_VLAN_PRIO	GENMASK(15, 13)
 #define NFP_FLOWER_MASK_VLAN_CFI	BIT(12)
@@ -108,6 +107,7 @@
 enum nfp_flower_tun_type {
 	NFP_FL_TUNNEL_NONE =	0,
 	NFP_FL_TUNNEL_VXLAN =	2,
+	NFP_FL_TUNNEL_GENEVE =	4,
 };
 
 struct nfp_fl_act_head {
@@ -165,20 +165,6 @@ struct nfp_fl_pop_vlan {
 	__be16 reserved;
 };
 
-/* Metadata without L2 (1W/4B)
- * ----------------------------------------------------------------
- *    3                   2                   1
- *  1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
- * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- * |  key_layers   |    mask_id    |           reserved            |
- * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- */
-struct nfp_flower_meta_one {
-	u8 nfp_flow_key_layer;
-	u8 mask_id;
-	u16 reserved;
-};
-
 struct nfp_fl_pre_tunnel {
 	struct nfp_fl_act_head head;
 	__be16 reserved;
@@ -187,16 +173,13 @@ struct nfp_fl_pre_tunnel {
 	__be32 extra[3];
 };
 
-struct nfp_fl_set_vxlan {
+struct nfp_fl_set_ipv4_udp_tun {
 	struct nfp_fl_act_head head;
 	__be16 reserved;
-	__be64 tun_id;
+	__be64 tun_id __packed;
 	__be32 tun_type_index;
-	__be16 tun_flags;
-	u8 ipv4_ttl;
-	u8 ipv4_tos;
-	__be32 extra[2];
-} __packed;
+	__be32 extra[3];
+};
 
 /* Metadata with L2 (1W/4B)
  * ----------------------------------------------------------------
@@ -209,12 +192,24 @@ struct nfp_fl_set_vxlan {
  *                           NOTE: |             TCI               |
  *                                 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  */
-struct nfp_flower_meta_two {
+struct nfp_flower_meta_tci {
 	u8 nfp_flow_key_layer;
 	u8 mask_id;
 	__be16 tci;
 };
 
+/* Extended metadata for additional key_layers (1W/4B)
+ * ----------------------------------------------------------------
+ *    3                   2                   1
+ *  1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |                      nfp_flow_key_layer2                      |
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ */
+struct nfp_flower_ext_meta {
+	__be32 nfp_flow_key_layer2;
+};
+
 /* Port details (1W/4B)
  * ----------------------------------------------------------------
  *    3                   2                   1
@@ -313,7 +308,7 @@ struct nfp_flower_ipv6 {
 	struct in6_addr ipv6_dst;
 };
 
-/* Flow Frame VXLAN --> Tunnel details (4W/16B)
+/* Flow Frame IPv4 UDP TUNNEL --> Tunnel details (4W/16B)
  * -----------------------------------------------------------------
  *    3                   2                   1
  *  1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
@@ -322,22 +317,17 @@ struct nfp_flower_ipv6 {
  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  * |                         ipv4_addr_dst                         |
  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- * |           tun_flags           |       tos     |       ttl     |
+ * |                            Reserved                           |
  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- * |   gpe_flags   |            Reserved           | Next Protocol |
+ * |                            Reserved                           |
  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  * |                     VNI                       |   Reserved    |
  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  */
-struct nfp_flower_vxlan {
+struct nfp_flower_ipv4_udp_tun {
 	__be32 ip_src;
 	__be32 ip_dst;
-	__be16 tun_flags;
-	u8 tos;
-	u8 ttl;
-	u8 gpe_flags;
-	u8 reserved[2];
-	u8 nxt_proto;
+	__be32 reserved[2];
 	__be32 tun_id;
 };
 
@@ -360,6 +350,7 @@ struct nfp_flower_cmsg_hdr {
 enum nfp_flower_cmsg_type_port {
 	NFP_FLOWER_CMSG_TYPE_FLOW_ADD =		0,
 	NFP_FLOWER_CMSG_TYPE_FLOW_DEL =		2,
+	NFP_FLOWER_CMSG_TYPE_PORT_REIFY =	6,
 	NFP_FLOWER_CMSG_TYPE_MAC_REPR =		7,
 	NFP_FLOWER_CMSG_TYPE_PORT_MOD =		8,
 	NFP_FLOWER_CMSG_TYPE_NO_NEIGH =		10,
@@ -396,6 +387,15 @@ struct nfp_flower_cmsg_portmod {
 
 #define NFP_FLOWER_CMSG_PORTMOD_INFO_LINK	BIT(0)
 
+/* NFP_FLOWER_CMSG_TYPE_PORT_REIFY */
+struct nfp_flower_cmsg_portreify {
+	__be32 portnum;
+	u16 reserved;
+	__be16 info;
+};
+
+#define NFP_FLOWER_CMSG_PORTREIFY_INFO_EXIST	BIT(0)
+
 enum nfp_flower_cmsg_port_type {
 	NFP_FLOWER_CMSG_PORT_TYPE_UNSPEC =	0x0,
 	NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT =	0x1,
@@ -454,6 +454,7 @@ nfp_flower_cmsg_mac_repr_add(struct sk_buff *skb, unsigned int idx,
 			     unsigned int nbi, unsigned int nbi_port,
 			     unsigned int phys_port);
 int nfp_flower_cmsg_portmod(struct nfp_repr *repr, bool carrier_ok);
+int nfp_flower_cmsg_portreify(struct nfp_repr *repr, bool exists);
 void nfp_flower_cmsg_process_rx(struct work_struct *work);
 void nfp_flower_cmsg_rx(struct nfp_app *app, struct sk_buff *skb);
 struct sk_buff *
diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.c b/drivers/net/ethernet/netronome/nfp/flower/main.c
index 8fcc90c0d2d3..742d6f1575b5 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/main.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/main.c
@@ -32,6 +32,7 @@
  */
 
 #include <linux/etherdevice.h>
+#include <linux/lockdep.h>
 #include <linux/pci.h>
 #include <linux/skbuff.h>
 #include <linux/vmalloc.h>
@@ -98,7 +99,57 @@ nfp_flower_repr_get(struct nfp_app *app, u32 port_id)
 	if (port >= reprs->num_reprs)
 		return NULL;
 
-	return reprs->reprs[port];
+	return rcu_dereference(reprs->reprs[port]);
+}
+
+static int
+nfp_flower_reprs_reify(struct nfp_app *app, enum nfp_repr_type type,
+		       bool exists)
+{
+	struct nfp_reprs *reprs;
+	int i, err, count = 0;
+
+	reprs = rcu_dereference_protected(app->reprs[type],
+					  lockdep_is_held(&app->pf->lock));
+	if (!reprs)
+		return 0;
+
+	for (i = 0; i < reprs->num_reprs; i++) {
+		struct net_device *netdev;
+
+		netdev = nfp_repr_get_locked(app, reprs, i);
+		if (netdev) {
+			struct nfp_repr *repr = netdev_priv(netdev);
+
+			err = nfp_flower_cmsg_portreify(repr, exists);
+			if (err)
+				return err;
+			count++;
+		}
+	}
+
+	return count;
+}
+
+static int
+nfp_flower_wait_repr_reify(struct nfp_app *app, atomic_t *replies, int tot_repl)
+{
+	struct nfp_flower_priv *priv = app->priv;
+	int err;
+
+	if (!tot_repl)
+		return 0;
+
+	lockdep_assert_held(&app->pf->lock);
+	err = wait_event_interruptible_timeout(priv->reify_wait_queue,
+					       atomic_read(replies) >= tot_repl,
+					       msecs_to_jiffies(10));
+	if (err <= 0) {
+		nfp_warn(app->cpp, "Not all reprs responded to reify\n");
+		return -EIO;
+	}
+
+	return 0;
 }
 
 static int
@@ -110,7 +161,6 @@ nfp_flower_repr_netdev_open(struct nfp_app *app, struct nfp_repr *repr)
 	if (err)
 		return err;
 
-	netif_carrier_on(repr->netdev);
 	netif_tx_wake_all_queues(repr->netdev);
 
 	return 0;
@@ -119,7 +169,6 @@ nfp_flower_repr_netdev_open(struct nfp_app *app, struct nfp_repr *repr)
 static int
 nfp_flower_repr_netdev_stop(struct nfp_app *app, struct nfp_repr *repr)
 {
-	netif_carrier_off(repr->netdev);
 	netif_tx_disable(repr->netdev);
 
 	return nfp_flower_cmsg_portmod(repr, false);
@@ -140,6 +189,24 @@ nfp_flower_repr_netdev_clean(struct nfp_app *app, struct net_device *netdev)
 				     netdev_priv(netdev));
 }
 
+static void
+nfp_flower_repr_netdev_preclean(struct nfp_app *app, struct net_device *netdev)
+{
+	struct nfp_repr *repr = netdev_priv(netdev);
+	struct nfp_flower_priv *priv = app->priv;
+	atomic_t *replies = &priv->reify_replies;
+	int err;
+
+	atomic_set(replies, 0);
+	err = nfp_flower_cmsg_portreify(repr, false);
+	if (err) {
+		nfp_warn(app->cpp, "Failed to notify firmware about repr destruction\n");
+		return;
+	}
+
+	nfp_flower_wait_repr_reify(app, replies, 1);
+}
+
 static void nfp_flower_sriov_disable(struct nfp_app *app)
 {
 	struct nfp_flower_priv *priv = app->priv;
@@ -157,10 +224,11 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 {
 	u8 nfp_pcie = nfp_cppcore_pcie_unit(app->pf->cpp);
 	struct nfp_flower_priv *priv = app->priv;
+	atomic_t *replies = &priv->reify_replies;
 	enum nfp_port_type port_type;
 	struct nfp_reprs *reprs;
+	int i, err, reify_cnt;
 	const u8 queue = 0;
-	int i, err;
 
 	port_type = repr_type == NFP_REPR_TYPE_PF ? NFP_PORT_PF_PORT :
 						    NFP_PORT_VF_PORT;
@@ -170,19 +238,21 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 		return -ENOMEM;
 
 	for (i = 0; i < cnt; i++) {
+		struct net_device *repr;
 		struct nfp_port *port;
 		u32 port_id;
 
-		reprs->reprs[i] = nfp_repr_alloc(app);
-		if (!reprs->reprs[i]) {
+		repr = nfp_repr_alloc(app);
+		if (!repr) {
 			err = -ENOMEM;
 			goto err_reprs_clean;
 		}
+		RCU_INIT_POINTER(reprs->reprs[i], repr);
 
 		/* For now we only support 1 PF */
 		WARN_ON(repr_type == NFP_REPR_TYPE_PF && i);
 
-		port = nfp_port_alloc(app, port_type, reprs->reprs[i]);
+		port = nfp_port_alloc(app, port_type, repr);
 		if (repr_type == NFP_REPR_TYPE_PF) {
 			port->pf_id = i;
 			port->vnic = priv->nn->dp.ctrl_bar;
@@ -193,11 +263,11 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 				app->pf->vf_cfg_mem + i * NFP_NET_CFG_BAR_SZ;
 		}
 
-		eth_hw_addr_random(reprs->reprs[i]);
+		eth_hw_addr_random(repr);
 
 		port_id = nfp_flower_cmsg_pcie_port(nfp_pcie, vnic_type,
 						    i, queue);
-		err = nfp_repr_init(app, reprs->reprs[i],
+		err = nfp_repr_init(app, repr,
 				    port_id, port, priv->nn->dp.netdev);
 		if (err) {
 			nfp_port_free(port);
@@ -206,14 +276,28 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app,
 
 		nfp_info(app->cpp, "%s%d Representor(%s) created\n",
 			 repr_type == NFP_REPR_TYPE_PF ? "PF" : "VF", i,
-			 reprs->reprs[i]->name);
+			 repr->name);
 	}
 
 	nfp_app_reprs_set(app, repr_type, reprs);
 
+	atomic_set(replies, 0);
+	reify_cnt = nfp_flower_reprs_reify(app, repr_type, true);
+	if (reify_cnt < 0) {
+		err = reify_cnt;
+		nfp_warn(app->cpp, "Failed to notify firmware about repr creation\n");
+		goto err_reprs_remove;
+	}
+
+	err = nfp_flower_wait_repr_reify(app, replies, reify_cnt);
+	if (err)
+		goto err_reprs_remove;
+
 	return 0;
+err_reprs_remove:
+	reprs = nfp_app_reprs_set(app, repr_type, NULL);
 err_reprs_clean:
-	nfp_reprs_clean_and_free(reprs);
+	nfp_reprs_clean_and_free(app, reprs);
 	return err;
 }
 
@@ -233,10 +317,11 @@ static int
 nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 {
 	struct nfp_eth_table *eth_tbl = app->pf->eth_tbl;
+	atomic_t *replies = &priv->reify_replies;
 	struct sk_buff *ctrl_skb;
 	struct nfp_reprs *reprs;
+	int err, reify_cnt;
 	unsigned int i;
-	int err;
 
 	ctrl_skb = nfp_flower_cmsg_mac_repr_start(app, eth_tbl->count);
 	if (!ctrl_skb)
@@ -250,17 +335,18 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 
 	for (i = 0; i < eth_tbl->count; i++) {
 		unsigned int phys_port = eth_tbl->ports[i].index;
+		struct net_device *repr;
 		struct nfp_port *port;
 		u32 cmsg_port_id;
 
-		reprs->reprs[phys_port] = nfp_repr_alloc(app);
-		if (!reprs->reprs[phys_port]) {
+		repr = nfp_repr_alloc(app);
+		if (!repr) {
 			err = -ENOMEM;
 			goto err_reprs_clean;
 		}
+		RCU_INIT_POINTER(reprs->reprs[phys_port], repr);
 
-		port = nfp_port_alloc(app, NFP_PORT_PHYS_PORT,
-				      reprs->reprs[phys_port]);
+		port = nfp_port_alloc(app, NFP_PORT_PHYS_PORT, repr);
 		if (IS_ERR(port)) {
 			err = PTR_ERR(port);
 			goto err_reprs_clean;
@@ -271,11 +357,11 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 			goto err_reprs_clean;
 		}
 
-		SET_NETDEV_DEV(reprs->reprs[phys_port], &priv->nn->pdev->dev);
+		SET_NETDEV_DEV(repr, &priv->nn->pdev->dev);
 		nfp_net_get_mac_addr(app->pf, port);
 
 		cmsg_port_id = nfp_flower_cmsg_phys_port(phys_port);
-		err = nfp_repr_init(app, reprs->reprs[phys_port],
+		err = nfp_repr_init(app, repr,
 				    cmsg_port_id, port, priv->nn->dp.netdev);
 		if (err) {
 			nfp_port_free(port);
@@ -288,23 +374,37 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv)
 					     phys_port);
 
 		nfp_info(app->cpp, "Phys Port %d Representor(%s) created\n",
-			 phys_port, reprs->reprs[phys_port]->name);
+			 phys_port, repr->name);
 	}
 
 	nfp_app_reprs_set(app, NFP_REPR_TYPE_PHYS_PORT, reprs);
 
-	/* The MAC_REPR control message should be sent after the MAC
+	/* The REIFY/MAC_REPR control messages should be sent after the MAC
 	 * representors are registered using nfp_app_reprs_set().  This is
 	 * because the firmware may respond with control messages for the
 	 * MAC representors, f.e. to provide the driver with information
 	 * about their state, and without registration the driver will drop
 	 * any such messages.
 	 */
+	atomic_set(replies, 0);
+	reify_cnt = nfp_flower_reprs_reify(app, NFP_REPR_TYPE_PHYS_PORT, true);
+	if (reify_cnt < 0) {
+		err = reify_cnt;
+		nfp_warn(app->cpp, "Failed to notify firmware about repr creation\n");
+		goto err_reprs_remove;
+	}
+
+	err = nfp_flower_wait_repr_reify(app, replies, reify_cnt);
+	if (err)
+		goto err_reprs_remove;
+
 	nfp_ctrl_tx(app->ctrl, ctrl_skb);
 
 	return 0;
+err_reprs_remove:
+	reprs = nfp_app_reprs_set(app, NFP_REPR_TYPE_PHYS_PORT, NULL);
 err_reprs_clean:
-	nfp_reprs_clean_and_free(reprs);
+	nfp_reprs_clean_and_free(app, reprs);
 err_free_ctrl_skb:
 	kfree_skb(ctrl_skb);
 	return err;
@@ -381,7 +481,7 @@ static int nfp_flower_init(struct nfp_app *app)
 {
 	const struct nfp_pf *pf = app->pf;
 	struct nfp_flower_priv *app_priv;
-	u64 version;
+	u64 version, features;
 	int err;
 
 	if (!pf->eth_tbl) {
@@ -419,11 +519,20 @@ static int nfp_flower_init(struct nfp_app *app)
 	app_priv->app = app;
 	skb_queue_head_init(&app_priv->cmsg_skbs);
 	INIT_WORK(&app_priv->cmsg_work, nfp_flower_cmsg_process_rx);
+	init_waitqueue_head(&app_priv->reify_wait_queue);
 
 	err = nfp_flower_metadata_init(app);
 	if (err)
 		goto err_free_app_priv;
 
+	/* Extract the extra features supported by the firmware. */
+	features = nfp_rtsym_read_le(app->pf->rtbl,
+				     "_abi_flower_extra_features", &err);
+	if (err)
+		app_priv->flower_ext_feats = 0;
+	else
+		app_priv->flower_ext_feats = features;
+
 	return 0;
 
 err_free_app_priv:
@@ -456,6 +565,8 @@ static void nfp_flower_stop(struct nfp_app *app)
 const struct nfp_app_type app_flower = {
 	.id		= NFP_APP_FLOWER_NIC,
 	.name		= "flower",
+
+	.ctrl_cap_mask	= ~0U,
 	.ctrl_has_meta	= true,
 
 	.extra_cap	= nfp_flower_extra_cap,
@@ -468,6 +579,7 @@ const struct nfp_app_type app_flower = {
 	.vnic_clean	= nfp_flower_vnic_clean,
 
 	.repr_init	= nfp_flower_repr_netdev_init,
+	.repr_preclean	= nfp_flower_repr_netdev_preclean,
 	.repr_clean	= nfp_flower_repr_netdev_clean,
 
 	.repr_open	= nfp_flower_repr_netdev_open,
diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h
index e6b26c5ae6e0..332ff0fdc038 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/main.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/main.h
@@ -34,6 +34,8 @@
 #ifndef __NFP_FLOWER_H__
 #define __NFP_FLOWER_H__ 1
 
+#include "cmsg.h"
+
 #include <linux/circ_buf.h>
 #include <linux/hashtable.h>
 #include <linux/time64.h>
@@ -58,6 +60,10 @@ struct nfp_app;
 #define NFP_FL_MASK_ID_LOCATION		1
 
 #define NFP_FL_VXLAN_PORT		4789
+#define NFP_FL_GENEVE_PORT		6081
+
+/* Extra features bitmap. */
+#define NFP_FL_FEATS_GENEVE		BIT(0)
 
 struct nfp_fl_mask_id {
 	struct circ_buf mask_id_free_list;
@@ -77,6 +83,7 @@ struct nfp_fl_stats_id {
  * @nn:			Pointer to vNIC
  * @mask_id_seed:	Seed used for mask hash table
  * @flower_version:	HW version of flower
+ * @flower_ext_feats:	Bitmap of extra features the HW supports
  * @stats_ids:		List of free stats ids
  * @mask_ids:		List of free mask ids
  * @mask_table:		Hash table used to store masks
@@ -95,12 +102,16 @@ struct nfp_fl_stats_id {
  * @nfp_mac_off_count:	Number of MACs in address list
  * @nfp_tun_mac_nb:	Notifier to monitor link state
  * @nfp_tun_neigh_nb:	Notifier to monitor neighbour state
+ * @reify_replies:	atomically stores the number of replies received
+ *			from firmware for repr reify
+ * @reify_wait_queue:	wait queue for repr reify response counting
  */
 struct nfp_flower_priv {
 	struct nfp_app *app;
 	struct nfp_net *nn;
 	u32 mask_id_seed;
 	u64 flower_version;
+	u64 flower_ext_feats;
 	struct nfp_fl_stats_id stats_ids;
 	struct nfp_fl_mask_id mask_ids;
 	DECLARE_HASHTABLE(mask_table, NFP_FLOWER_MASK_HASH_BITS);
@@ -119,6 +130,8 @@ struct nfp_flower_priv {
 	int nfp_mac_off_count;
 	struct notifier_block nfp_tun_mac_nb;
 	struct notifier_block nfp_tun_neigh_nb;
+	atomic_t reify_replies;
+	wait_queue_head_t reify_wait_queue;
 };
 
 struct nfp_fl_key_ls {
@@ -172,7 +185,8 @@ int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev,
 int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow,
 				  struct nfp_fl_key_ls *key_ls,
 				  struct net_device *netdev,
-				  struct nfp_fl_payload *nfp_flow);
+				  struct nfp_fl_payload *nfp_flow,
+				  enum nfp_flower_tun_type tun_type);
 int nfp_flower_compile_action(struct tc_cls_flower_offload *flow,
 			      struct net_device *netdev,
 			      struct nfp_fl_payload *nfp_flow);
diff --git a/drivers/net/ethernet/netronome/nfp/flower/match.c b/drivers/net/ethernet/netronome/nfp/flower/match.c
index 60614d4f0e22..37c2ecae2a7a 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/match.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/match.c
@@ -38,7 +38,7 @@
 #include "main.h"
 
 static void
-nfp_flower_compile_meta_tci(struct nfp_flower_meta_two *frame,
+nfp_flower_compile_meta_tci(struct nfp_flower_meta_tci *frame,
 			    struct tc_cls_flower_offload *flow, u8 key_type,
 			    bool mask_version)
 {
@@ -46,7 +46,7 @@ nfp_flower_compile_meta_tci(struct nfp_flower_meta_two *frame,
 	struct flow_dissector_key_vlan *flow_vlan;
 	u16 tmp_tci;
 
-	memset(frame, 0, sizeof(struct nfp_flower_meta_two));
+	memset(frame, 0, sizeof(struct nfp_flower_meta_tci));
 	/* Populate the metadata frame. */
 	frame->nfp_flow_key_layer = key_type;
 	frame->mask_id = ~0;
@@ -68,11 +68,9 @@ nfp_flower_compile_meta_tci(struct nfp_flower_meta_two *frame,
 }
 
 static void
-nfp_flower_compile_meta(struct nfp_flower_meta_one *frame, u8 key_type)
+nfp_flower_compile_ext_meta(struct nfp_flower_ext_meta *frame, u32 key_ext)
 {
-	frame->nfp_flow_key_layer = key_type;
-	frame->mask_id = 0;
-	frame->reserved = 0;
+	frame->nfp_flow_key_layer2 = cpu_to_be32(key_ext);
 }
 
 static int
@@ -224,16 +222,15 @@ nfp_flower_compile_ipv6(struct nfp_flower_ipv6 *frame,
 }
 
 static void
-nfp_flower_compile_vxlan(struct nfp_flower_vxlan *frame,
-			 struct tc_cls_flower_offload *flow,
-			 bool mask_version, __be32 *tun_dst)
+nfp_flower_compile_ipv4_udp_tun(struct nfp_flower_ipv4_udp_tun *frame,
+				struct tc_cls_flower_offload *flow,
+				bool mask_version)
 {
 	struct fl_flow_key *target = mask_version ? flow->mask : flow->key;
-	struct flow_dissector_key_ipv4_addrs *vxlan_ips;
+	struct flow_dissector_key_ipv4_addrs *tun_ips;
 	struct flow_dissector_key_keyid *vni;
 
-	/* Wildcard TOS/TTL/GPE_FLAGS/NXT_PROTO for now. */
-	memset(frame, 0, sizeof(struct nfp_flower_vxlan));
+	memset(frame, 0, sizeof(struct nfp_flower_ipv4_udp_tun));
 
 	if (dissector_uses_key(flow->dissector,
 			       FLOW_DISSECTOR_KEY_ENC_KEYID)) {
@@ -248,80 +245,68 @@ nfp_flower_compile_vxlan(struct nfp_flower_vxlan *frame,
 
 	if (dissector_uses_key(flow->dissector,
 			       FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
-		vxlan_ips =
+		tun_ips =
 		   skb_flow_dissector_target(flow->dissector,
 					     FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
 					     target);
-		frame->ip_src = vxlan_ips->src;
-		frame->ip_dst = vxlan_ips->dst;
-		*tun_dst = vxlan_ips->dst;
+		frame->ip_src = tun_ips->src;
+		frame->ip_dst = tun_ips->dst;
 	}
 }
 
 int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow,
 				  struct nfp_fl_key_ls *key_ls,
 				  struct net_device *netdev,
-				  struct nfp_fl_payload *nfp_flow)
+				  struct nfp_fl_payload *nfp_flow,
+				  enum nfp_flower_tun_type tun_type)
 {
-	enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
-	__be32 tun_dst, tun_dst_mask = 0;
 	struct nfp_repr *netdev_repr;
 	int err;
 	u8 *ext;
 	u8 *msk;
 
-	if (key_ls->key_layer & NFP_FLOWER_LAYER_VXLAN)
-		tun_type = NFP_FL_TUNNEL_VXLAN;
-
 	memset(nfp_flow->unmasked_data, 0, key_ls->key_size);
 	memset(nfp_flow->mask_data, 0, key_ls->key_size);
 
 	ext = nfp_flow->unmasked_data;
 	msk = nfp_flow->mask_data;
-	if (NFP_FLOWER_LAYER_PORT & key_ls->key_layer) {
-		/* Populate Exact Metadata. */
-		nfp_flower_compile_meta_tci((struct nfp_flower_meta_two *)ext,
-					    flow, key_ls->key_layer, false);
-		/* Populate Mask Metadata. */
-		nfp_flower_compile_meta_tci((struct nfp_flower_meta_two *)msk,
-					    flow, key_ls->key_layer, true);
-		ext += sizeof(struct nfp_flower_meta_two);
-		msk += sizeof(struct nfp_flower_meta_two);
-
-		/* Populate Exact Port data. */
-		err = nfp_flower_compile_port((struct nfp_flower_in_port *)ext,
-					      nfp_repr_get_port_id(netdev),
-					      false, tun_type);
-		if (err)
-			return err;
-
-		/* Populate Mask Port Data. */
-		err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk,
-					      nfp_repr_get_port_id(netdev),
-					      true, tun_type);
-		if (err)
-			return err;
-
-		ext += sizeof(struct nfp_flower_in_port);
-		msk += sizeof(struct nfp_flower_in_port);
-	} else {
-		/* Populate Exact Metadata. */
-		nfp_flower_compile_meta((struct nfp_flower_meta_one *)ext,
-					key_ls->key_layer);
-		/* Populate Mask Metadata. */
-		nfp_flower_compile_meta((struct nfp_flower_meta_one *)msk,
-					key_ls->key_layer);
-		ext += sizeof(struct nfp_flower_meta_one);
-		msk += sizeof(struct nfp_flower_meta_one);
-	}
 
-	if (NFP_FLOWER_LAYER_META & key_ls->key_layer) {
-		/* Additional Metadata Fields.
-		 * Currently unsupported.
-		 */
-		return -EOPNOTSUPP;
+	/* Populate Exact Metadata. */
+	nfp_flower_compile_meta_tci((struct nfp_flower_meta_tci *)ext,
+				    flow, key_ls->key_layer, false);
+	/* Populate Mask Metadata. */
+	nfp_flower_compile_meta_tci((struct nfp_flower_meta_tci *)msk,
+				    flow, key_ls->key_layer, true);
+	ext += sizeof(struct nfp_flower_meta_tci);
+	msk += sizeof(struct nfp_flower_meta_tci);
+
+	/* Populate Extended Metadata if Required. */
+	if (NFP_FLOWER_LAYER_EXT_META & key_ls->key_layer) {
+		nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)ext,
+					    key_ls->key_layer_two);
+		nfp_flower_compile_ext_meta((struct nfp_flower_ext_meta *)msk,
+					    key_ls->key_layer_two);
+		ext += sizeof(struct nfp_flower_ext_meta);
+		msk += sizeof(struct nfp_flower_ext_meta);
 	}
 
+	/* Populate Exact Port data. */
+	err = nfp_flower_compile_port((struct nfp_flower_in_port *)ext,
+				      nfp_repr_get_port_id(netdev),
+				      false, tun_type);
+	if (err)
+		return err;
+
+	/* Populate Mask Port Data. */
+	err = nfp_flower_compile_port((struct nfp_flower_in_port *)msk,
+				      nfp_repr_get_port_id(netdev),
+				      true, tun_type);
+	if (err)
+		return err;
+
+	ext += sizeof(struct nfp_flower_in_port);
+	msk += sizeof(struct nfp_flower_in_port);
+
 	if (NFP_FLOWER_LAYER_MAC & key_ls->key_layer) {
 		/* Populate Exact MAC Data. */
 		nfp_flower_compile_mac((struct nfp_flower_mac_mpls *)ext,
@@ -366,15 +351,17 @@ int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow,
 		msk += sizeof(struct nfp_flower_ipv6);
 	}
 
-	if (key_ls->key_layer & NFP_FLOWER_LAYER_VXLAN) {
+	if (key_ls->key_layer & NFP_FLOWER_LAYER_VXLAN ||
+	    key_ls->key_layer_two & NFP_FLOWER_LAYER2_GENEVE) {
+		__be32 tun_dst;
+
 		/* Populate Exact VXLAN Data. */
-		nfp_flower_compile_vxlan((struct nfp_flower_vxlan *)ext,
-					 flow, false, &tun_dst);
+		nfp_flower_compile_ipv4_udp_tun((void *)ext, flow, false);
 		/* Populate Mask VXLAN Data. */
-		nfp_flower_compile_vxlan((struct nfp_flower_vxlan *)msk,
-					 flow, true, &tun_dst_mask);
-		ext += sizeof(struct nfp_flower_vxlan);
-		msk += sizeof(struct nfp_flower_vxlan);
+		nfp_flower_compile_ipv4_udp_tun((void *)msk, flow, true);
+		tun_dst = ((struct nfp_flower_ipv4_udp_tun *)ext)->ip_dst;
+		ext += sizeof(struct nfp_flower_ipv4_udp_tun);
+		msk += sizeof(struct nfp_flower_ipv4_udp_tun);
 
 		/* Configure tunnel end point MAC. */
 		if (nfp_netdev_is_nfp_repr(netdev)) {
diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c
index 553f94f55dce..eb5c13dea8f5 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c
@@ -130,12 +130,15 @@ static bool nfp_flower_check_higher_than_mac(struct tc_cls_flower_offload *f)
 }
 
 static int
-nfp_flower_calculate_key_layers(struct nfp_fl_key_ls *ret_key_ls,
+nfp_flower_calculate_key_layers(struct nfp_app *app,
+				struct nfp_fl_key_ls *ret_key_ls,
 				struct tc_cls_flower_offload *flow,
-				bool egress)
+				bool egress,
+				enum nfp_flower_tun_type *tun_type)
 {
 	struct flow_dissector_key_basic *mask_basic = NULL;
 	struct flow_dissector_key_basic *key_basic = NULL;
+	struct nfp_flower_priv *priv = app->priv;
 	u32 key_layer_two;
 	u8 key_layer;
 	int key_size;
@@ -150,10 +153,15 @@ nfp_flower_calculate_key_layers(struct nfp_fl_key_ls *ret_key_ls,
 		return -EOPNOTSUPP;
 
 	key_layer_two = 0;
-	key_layer = NFP_FLOWER_LAYER_PORT | NFP_FLOWER_LAYER_MAC;
-	key_size = sizeof(struct nfp_flower_meta_one) +
-		   sizeof(struct nfp_flower_in_port) +
-		   sizeof(struct nfp_flower_mac_mpls);
+	key_layer = NFP_FLOWER_LAYER_PORT;
+	key_size = sizeof(struct nfp_flower_meta_tci) +
+		   sizeof(struct nfp_flower_in_port);
+
+	if (dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS) ||
+	    dissector_uses_key(flow->dissector, FLOW_DISSECTOR_KEY_MPLS)) {
+		key_layer |= NFP_FLOWER_LAYER_MAC;
+		key_size += sizeof(struct nfp_flower_mac_mpls);
+	}
 
 	if (dissector_uses_key(flow->dissector,
 			       FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
@@ -192,12 +200,27 @@ nfp_flower_calculate_key_layers(struct nfp_fl_key_ls *ret_key_ls,
 						  FLOW_DISSECTOR_KEY_ENC_PORTS,
 						  flow->key);
 
-		if (mask_enc_ports->dst != cpu_to_be16(~0) ||
-		    enc_ports->dst != htons(NFP_FL_VXLAN_PORT))
+		if (mask_enc_ports->dst != cpu_to_be16(~0))
 			return -EOPNOTSUPP;
 
-		key_layer |= NFP_FLOWER_LAYER_VXLAN;
-		key_size += sizeof(struct nfp_flower_vxlan);
+		switch (enc_ports->dst) {
+		case htons(NFP_FL_VXLAN_PORT):
+			*tun_type = NFP_FL_TUNNEL_VXLAN;
+			key_layer |= NFP_FLOWER_LAYER_VXLAN;
+			key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
+			break;
+		case htons(NFP_FL_GENEVE_PORT):
+			if (!(priv->flower_ext_feats & NFP_FL_FEATS_GENEVE))
+				return -EOPNOTSUPP;
+			*tun_type = NFP_FL_TUNNEL_GENEVE;
+			key_layer |= NFP_FLOWER_LAYER_EXT_META;
+			key_size += sizeof(struct nfp_flower_ext_meta);
+			key_layer_two |= NFP_FLOWER_LAYER2_GENEVE;
+			key_size += sizeof(struct nfp_flower_ipv4_udp_tun);
+			break;
+		default:
+			return -EOPNOTSUPP;
+		}
 	} else if (egress) {
 		/* Reject non tunnel matches offloaded to egress repr. */
 		return -EOPNOTSUPP;
@@ -325,6 +348,8 @@ static int
 nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
 		       struct tc_cls_flower_offload *flow, bool egress)
 {
+	enum nfp_flower_tun_type tun_type = NFP_FL_TUNNEL_NONE;
+	struct nfp_port *port = nfp_port_from_netdev(netdev);
 	struct nfp_flower_priv *priv = app->priv;
 	struct nfp_fl_payload *flow_pay;
 	struct nfp_fl_key_ls *key_layer;
@@ -334,7 +359,8 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
 	if (!key_layer)
 		return -ENOMEM;
 
-	err = nfp_flower_calculate_key_layers(key_layer, flow, egress);
+	err = nfp_flower_calculate_key_layers(app, key_layer, flow, egress,
+					      &tun_type);
 	if (err)
 		goto err_free_key_ls;
 
@@ -344,7 +370,8 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
 		goto err_free_key_ls;
 	}
 
-	err = nfp_flower_compile_flow_match(flow, key_layer, netdev, flow_pay);
+	err = nfp_flower_compile_flow_match(flow, key_layer, netdev, flow_pay,
+					    tun_type);
 	if (err)
 		goto err_destroy_flow;
 
@@ -364,6 +391,7 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev,
 	INIT_HLIST_NODE(&flow_pay->link);
 	flow_pay->tc_flower_cookie = flow->cookie;
 	hash_add_rcu(priv->flow_table, &flow_pay->link, flow->cookie);
+	port->tc_offload_cnt++;
 
 	/* Deallocate flow payload when flower rule has been destroyed. */
 	kfree(key_layer);
@@ -395,6 +423,7 @@ static int
 nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev,
 		       struct tc_cls_flower_offload *flow)
 {
+	struct nfp_port *port = nfp_port_from_netdev(netdev);
 	struct nfp_fl_payload *nfp_flow;
 	int err;
 
@@ -416,6 +445,7 @@ nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev,
 
 err_free_flow:
 	hash_del_rcu(&nfp_flow->link);
+	port->tc_offload_cnt--;
 	kfree(nfp_flow->action_data);
 	kfree(nfp_flow->mask_data);
 	kfree(nfp_flow->unmasked_data);
@@ -457,8 +487,7 @@ static int
 nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev,
 			struct tc_cls_flower_offload *flower, bool egress)
 {
-	if (!eth_proto_is_802_3(flower->common.protocol) ||
-	    flower->common.chain_index)
+	if (!eth_proto_is_802_3(flower->common.protocol))
 		return -EOPNOTSUPP;
 
 	switch (flower->command) {
@@ -478,7 +507,7 @@ int nfp_flower_setup_tc_egress_cb(enum tc_setup_type type, void *type_data,
 {
 	struct nfp_repr *repr = cb_priv;
 
-	if (!tc_can_offload(repr->netdev))
+	if (!tc_cls_can_offload_and_chain0(repr->netdev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
@@ -495,7 +524,7 @@ static int nfp_flower_setup_tc_block_cb(enum tc_setup_type type,
 {
 	struct nfp_repr *repr = cb_priv;
 
-	if (!tc_can_offload(repr->netdev))
+	if (!tc_cls_can_offload_and_chain0(repr->netdev, type_data))
 		return -EOPNOTSUPP;
 
 	switch (type) {
diff --git a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
index b03f22f29612..ec524d97869d 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c
@@ -50,9 +50,9 @@
  * @seq:		sequence number of the message
  * @count:		number of tunnels report in message
  * @flags:		options part of the request
- * @ipv4:		dest IPv4 address of active route
- * @egress_port:	port the encapsulated packet egressed
- * @extra:		reserved for future use
+ * @tun_info.ipv4:		dest IPv4 address of active route
+ * @tun_info.egress_port:	port the encapsulated packet egressed
+ * @tun_info.extra:		reserved for future use
  * @tun_info:		tunnels that have sent traffic in reported period
  */
 struct nfp_tun_active_tuns {
@@ -132,8 +132,8 @@ struct nfp_ipv4_addr_entry {
  * struct nfp_tun_mac_addr - configure MAC address of tunnel EP on NFP
  * @reserved:	reserved for future use
  * @count:	number of MAC addresses in the message
- * @index:	index of MAC address in the lookup table
- * @addr:	interface MAC address
+ * @addresses.index:	index of MAC address in the lookup table
+ * @addresses.addr:	interface MAC address
  * @addresses:	series of MACs to offload
  */
 struct nfp_tun_mac_addr {
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.c b/drivers/net/ethernet/netronome/nfp/nfp_app.c
index 955a9f44d244..6aedef0ad433 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app.c
@@ -32,6 +32,8 @@
  */
 
 #include <linux/bug.h>
+#include <linux/lockdep.h>
+#include <linux/rcupdate.h>
 #include <linux/skbuff.h>
 #include <linux/slab.h>
 
@@ -98,14 +100,20 @@ nfp_app_ctrl_msg_alloc(struct nfp_app *app, unsigned int size, gfp_t priority)
 	return skb;
 }
 
+struct nfp_reprs *
+nfp_reprs_get_locked(struct nfp_app *app, enum nfp_repr_type type)
+{
+	return rcu_dereference_protected(app->reprs[type],
+					 lockdep_is_held(&app->pf->lock));
+}
+
 struct nfp_reprs *
 nfp_app_reprs_set(struct nfp_app *app, enum nfp_repr_type type,
 		  struct nfp_reprs *reprs)
 {
 	struct nfp_reprs *old;
 
-	old = rcu_dereference_protected(app->reprs[type],
-					lockdep_is_held(&app->pf->lock));
+	old = nfp_reprs_get_locked(app, type);
 	rcu_assign_pointer(app->reprs[type], reprs);
 
 	return old;
@@ -116,7 +124,7 @@ struct nfp_app *nfp_app_alloc(struct nfp_pf *pf, enum nfp_app_id id)
 	struct nfp_app *app;
 
 	if (id >= ARRAY_SIZE(apps) || !apps[id]) {
-		nfp_err(pf->cpp, "failed to find app with ID 0x%02hhx\n", id);
+		nfp_err(pf->cpp, "unknown FW app ID 0x%02hhx, driver too old or support for FW not built in\n", id);
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h b/drivers/net/ethernet/netronome/nfp/nfp_app.h
index 0e5e0305ad1c..20546ae67909 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_app.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h
@@ -43,6 +43,7 @@
 struct bpf_prog;
 struct net_device;
 struct netdev_bpf;
+struct netlink_ext_ack;
 struct pci_dev;
 struct sk_buff;
 struct sk_buff;
@@ -66,6 +67,9 @@ extern const struct nfp_app_type app_flower;
  * struct nfp_app_type - application definition
  * @id:		application ID
  * @name:	application name
+ * @ctrl_cap_mask:  ctrl vNIC capability mask, allows disabling features like
+ *		    IRQMOD which are on by default but counter-productive for
+ *		    control messages which are often latency-sensitive
  * @ctrl_has_meta:  control messages have prepend of type:5/port:CTRL
  *
  * Callbacks
@@ -77,18 +81,19 @@ extern const struct nfp_app_type app_flower;
  * @vnic_init:	vNIC netdev was registered
  * @vnic_clean:	vNIC netdev about to be unregistered
  * @repr_init:	representor about to be registered
+ * @repr_preclean:	representor about to unregistered, executed before app
+ *			reference to the it is removed
  * @repr_clean:	representor about to be unregistered
  * @repr_open:	representor netdev open callback
  * @repr_stop:	representor netdev stop callback
+ * @change_mtu:	MTU change on a netdev has been requested (veto-only, change
+ *		is not guaranteed to be committed)
  * @start:	start application logic
  * @stop:	stop application logic
  * @ctrl_msg_rx:    control message handler
  * @setup_tc:	setup TC ndo
- * @tc_busy:	TC HW offload busy (rules loaded)
+ * @bpf:	BPF ndo offload-related calls
  * @xdp_offload:    offload an XDP program
- * @bpf_verifier_prep:	verifier prep for dev-specific BPF programs
- * @bpf_translate:	translate call for dev-specific BPF programs
- * @bpf_destroy:	destroy for dev-specific BPF programs
  * @eswitch_mode_get:    get SR-IOV eswitch mode
  * @sriov_enable: app-specific sriov initialisation
  * @sriov_disable: app-specific sriov clean-up
@@ -98,6 +103,7 @@ struct nfp_app_type {
 	enum nfp_app_id id;
 	const char *name;
 
+	u32 ctrl_cap_mask;
 	bool ctrl_has_meta;
 
 	int (*init)(struct nfp_app *app);
@@ -112,11 +118,15 @@ struct nfp_app_type {
 	void (*vnic_clean)(struct nfp_app *app, struct nfp_net *nn);
 
 	int (*repr_init)(struct nfp_app *app, struct net_device *netdev);
+	void (*repr_preclean)(struct nfp_app *app, struct net_device *netdev);
 	void (*repr_clean)(struct nfp_app *app, struct net_device *netdev);
 
 	int (*repr_open)(struct nfp_app *app, struct nfp_repr *repr);
 	int (*repr_stop)(struct nfp_app *app, struct nfp_repr *repr);
 
+	int (*change_mtu)(struct nfp_app *app, struct net_device *netdev,
+			  int new_mtu);
+
 	int (*start)(struct nfp_app *app);
 	void (*stop)(struct nfp_app *app);
 
@@ -124,15 +134,11 @@ struct nfp_app_type {
 
 	int (*setup_tc)(struct nfp_app *app, struct net_device *netdev,
 			enum tc_setup_type type, void *type_data);
-	bool (*tc_busy)(struct nfp_app *app, struct nfp_net *nn);
+	int (*bpf)(struct nfp_app *app, struct nfp_net *nn,
+		   struct netdev_bpf *xdp);
 	int (*xdp_offload)(struct nfp_app *app, struct nfp_net *nn,
-			   struct bpf_prog *prog);
-	int (*bpf_verifier_prep)(struct nfp_app *app, struct nfp_net *nn,
-				 struct netdev_bpf *bpf);
-	int (*bpf_translate)(struct nfp_app *app, struct nfp_net *nn,
-			     struct bpf_prog *prog);
-	int (*bpf_destroy)(struct nfp_app *app, struct nfp_net *nn,
-			   struct bpf_prog *prog);
+			   struct bpf_prog *prog,
+			   struct netlink_ext_ack *extack);
 
 	int (*sriov_enable)(struct nfp_app *app, int num_vfs);
 	void (*sriov_disable)(struct nfp_app *app);
@@ -163,6 +169,7 @@ struct nfp_app {
 	void *priv;
 };
 
+bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb);
 bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb);
 
 static inline int nfp_app_init(struct nfp_app *app)
@@ -225,6 +232,13 @@ nfp_app_repr_init(struct nfp_app *app, struct net_device *netdev)
 	return app->type->repr_init(app, netdev);
 }
 
+static inline void
+nfp_app_repr_preclean(struct nfp_app *app, struct net_device *netdev)
+{
+	if (app->type->repr_preclean)
+		app->type->repr_preclean(app, netdev);
+}
+
 static inline void
 nfp_app_repr_clean(struct nfp_app *app, struct net_device *netdev)
 {
@@ -232,6 +246,14 @@ nfp_app_repr_clean(struct nfp_app *app, struct net_device *netdev)
 		app->type->repr_clean(app, netdev);
 }
 
+static inline int
+nfp_app_change_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
+{
+	if (!app || !app->type->change_mtu)
+		return 0;
+	return app->type->change_mtu(app, netdev, new_mtu);
+}
+
 static inline int nfp_app_start(struct nfp_app *app, struct nfp_net *ctrl)
 {
 	app->ctrl = ctrl;
@@ -277,13 +299,6 @@ static inline bool nfp_app_has_tc(struct nfp_app *app)
 	return app && app->type->setup_tc;
 }
 
-static inline bool nfp_app_tc_busy(struct nfp_app *app, struct nfp_net *nn)
-{
-	if (!app || !app->type->tc_busy)
-		return false;
-	return app->type->tc_busy(app, nn);
-}
-
 static inline int nfp_app_setup_tc(struct nfp_app *app,
 				   struct net_device *netdev,
 				   enum tc_setup_type type, void *type_data)
@@ -293,39 +308,29 @@ static inline int nfp_app_setup_tc(struct nfp_app *app,
 	return app->type->setup_tc(app, netdev, type, type_data);
 }
 
-static inline int nfp_app_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
-				      struct bpf_prog *prog)
+static inline int nfp_app_bpf(struct nfp_app *app, struct nfp_net *nn,
+			      struct netdev_bpf *bpf)
 {
-	if (!app || !app->type->xdp_offload)
-		return -EOPNOTSUPP;
-	return app->type->xdp_offload(app, nn, prog);
+	if (!app || !app->type->bpf)
+		return -EINVAL;
+	return app->type->bpf(app, nn, bpf);
 }
 
-static inline int
-nfp_app_bpf_verifier_prep(struct nfp_app *app, struct nfp_net *nn,
-			  struct netdev_bpf *bpf)
+static inline int nfp_app_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
+				      struct bpf_prog *prog,
+				      struct netlink_ext_ack *extack)
 {
-	if (!app || !app->type->bpf_verifier_prep)
+	if (!app || !app->type->xdp_offload)
 		return -EOPNOTSUPP;
-	return app->type->bpf_verifier_prep(app, nn, bpf);
+	return app->type->xdp_offload(app, nn, prog, extack);
 }
 
-static inline int
-nfp_app_bpf_translate(struct nfp_app *app, struct nfp_net *nn,
-		      struct bpf_prog *prog)
+static inline bool __nfp_app_ctrl_tx(struct nfp_app *app, struct sk_buff *skb)
 {
-	if (!app || !app->type->bpf_translate)
-		return -EOPNOTSUPP;
-	return app->type->bpf_translate(app, nn, prog);
-}
+	trace_devlink_hwmsg(priv_to_devlink(app->pf), false, 0,
+			    skb->data, skb->len);
 
-static inline int
-nfp_app_bpf_destroy(struct nfp_app *app, struct nfp_net *nn,
-		    struct bpf_prog *prog)
-{
-	if (!app || !app->type->bpf_destroy)
-		return -EOPNOTSUPP;
-	return app->type->bpf_destroy(app, nn, prog);
+	return __nfp_ctrl_tx(app->ctrl, skb);
 }
 
 static inline bool nfp_app_ctrl_tx(struct nfp_app *app, struct sk_buff *skb)
@@ -377,6 +382,8 @@ static inline struct net_device *nfp_app_repr_get(struct nfp_app *app, u32 id)
 
 struct nfp_app *nfp_app_from_netdev(struct net_device *netdev);
 
+struct nfp_reprs *
+nfp_reprs_get_locked(struct nfp_app *app, enum nfp_repr_type type);
 struct nfp_reprs *
 nfp_app_reprs_set(struct nfp_app *app, enum nfp_repr_type type,
 		  struct nfp_reprs *reprs);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_asm.c b/drivers/net/ethernet/netronome/nfp/nfp_asm.c
index 830f6de25f47..1e597600c693 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_asm.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_asm.c
@@ -41,6 +41,7 @@
 
 const struct cmd_tgt_act cmd_tgt_act[__CMD_TGT_MAP_SIZE] = {
 	[CMD_TGT_WRITE8_SWAP] =		{ 0x02, 0x42 },
+	[CMD_TGT_WRITE32_SWAP] =	{ 0x02, 0x5f },
 	[CMD_TGT_READ8] =		{ 0x01, 0x43 },
 	[CMD_TGT_READ32] =		{ 0x00, 0x5c },
 	[CMD_TGT_READ32_LE] =		{ 0x01, 0x5c },
@@ -49,6 +50,94 @@ const struct cmd_tgt_act cmd_tgt_act[__CMD_TGT_MAP_SIZE] = {
 	[CMD_TGT_READ_SWAP_LE] =	{ 0x03, 0x40 },
 };
 
+static bool unreg_is_imm(u16 reg)
+{
+	return (reg & UR_REG_IMM) == UR_REG_IMM;
+}
+
+u16 br_get_offset(u64 instr)
+{
+	u16 addr_lo, addr_hi;
+
+	addr_lo = FIELD_GET(OP_BR_ADDR_LO, instr);
+	addr_hi = FIELD_GET(OP_BR_ADDR_HI, instr);
+
+	return (addr_hi * ((OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO)) + 1)) |
+		addr_lo;
+}
+
+void br_set_offset(u64 *instr, u16 offset)
+{
+	u16 addr_lo, addr_hi;
+
+	addr_lo = offset & (OP_BR_ADDR_LO >> __bf_shf(OP_BR_ADDR_LO));
+	addr_hi = offset != addr_lo;
+	*instr &= ~(OP_BR_ADDR_HI | OP_BR_ADDR_LO);
+	*instr |= FIELD_PREP(OP_BR_ADDR_HI, addr_hi);
+	*instr |= FIELD_PREP(OP_BR_ADDR_LO, addr_lo);
+}
+
+void br_add_offset(u64 *instr, u16 offset)
+{
+	u16 addr;
+
+	addr = br_get_offset(*instr);
+	br_set_offset(instr, addr + offset);
+}
+
+static bool immed_can_modify(u64 instr)
+{
+	if (FIELD_GET(OP_IMMED_INV, instr) ||
+	    FIELD_GET(OP_IMMED_SHIFT, instr) ||
+	    FIELD_GET(OP_IMMED_WIDTH, instr) != IMMED_WIDTH_ALL) {
+		pr_err("Can't decode/encode immed!\n");
+		return false;
+	}
+	return true;
+}
+
+u16 immed_get_value(u64 instr)
+{
+	u16 reg;
+
+	if (!immed_can_modify(instr))
+		return 0;
+
+	reg = FIELD_GET(OP_IMMED_A_SRC, instr);
+	if (!unreg_is_imm(reg))
+		reg = FIELD_GET(OP_IMMED_B_SRC, instr);
+
+	return (reg & 0xff) | FIELD_GET(OP_IMMED_IMM, instr) << 8;
+}
+
+void immed_set_value(u64 *instr, u16 immed)
+{
+	if (!immed_can_modify(*instr))
+		return;
+
+	if (unreg_is_imm(FIELD_GET(OP_IMMED_A_SRC, *instr))) {
+		*instr &= ~FIELD_PREP(OP_IMMED_A_SRC, 0xff);
+		*instr |= FIELD_PREP(OP_IMMED_A_SRC, immed & 0xff);
+	} else {
+		*instr &= ~FIELD_PREP(OP_IMMED_B_SRC, 0xff);
+		*instr |= FIELD_PREP(OP_IMMED_B_SRC, immed & 0xff);
+	}
+
+	*instr &= ~OP_IMMED_IMM;
+	*instr |= FIELD_PREP(OP_IMMED_IMM, immed >> 8);
+}
+
+void immed_add_value(u64 *instr, u16 offset)
+{
+	u16 val;
+
+	if (!immed_can_modify(*instr))
+		return;
+
+	val = immed_get_value(*instr);
+	immed_set_value(instr, val + offset);
+}
+
 static u16 nfp_swreg_to_unreg(swreg reg, bool is_dst)
 {
 	bool lm_id, lm_dec = false;
@@ -120,7 +209,8 @@ int swreg_to_unrestricted(swreg dst, swreg lreg, swreg rreg,
 	reg->dst = nfp_swreg_to_unreg(dst, true);
 
 	/* Decode source operands */
-	if (swreg_type(lreg) == swreg_type(rreg))
+	if (swreg_type(lreg) == swreg_type(rreg) &&
+	    swreg_type(lreg) != NN_REG_NONE)
 		return -EFAULT;
 
 	if (swreg_type(lreg) == NN_REG_GPR_B ||
@@ -200,7 +290,8 @@ int swreg_to_restricted(swreg dst, swreg lreg, swreg rreg,
 	reg->dst = nfp_swreg_to_rereg(dst, true, false, NULL);
 
 	/* Decode source operands */
-	if (swreg_type(lreg) == swreg_type(rreg))
+	if (swreg_type(lreg) == swreg_type(rreg) &&
+	    swreg_type(lreg) != NN_REG_NONE)
 		return -EFAULT;
 
 	if (swreg_type(lreg) == NN_REG_GPR_B ||
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_asm.h b/drivers/net/ethernet/netronome/nfp/nfp_asm.h
index 74d0c11ab2f9..5f9291db98e0 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_asm.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_asm.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2016 Netronome Systems, Inc.
+ * Copyright (C) 2016-2017 Netronome Systems, Inc.
  *
  * This software is dual licensed under the GNU General License Version 2,
  * June 1991 as shown in the file COPYING in the top-level directory of this
@@ -77,9 +77,11 @@
 enum br_mask {
 	BR_BEQ = 0x00,
 	BR_BNE = 0x01,
+	BR_BMI = 0x02,
 	BR_BHS = 0x04,
 	BR_BLO = 0x05,
 	BR_BGE = 0x08,
+	BR_BLT = 0x09,
 	BR_UNC = 0x18,
 };
 
@@ -92,6 +94,10 @@ enum br_ctx_signal_state {
 	BR_CSS_NONE = 2,
 };
 
+u16 br_get_offset(u64 instr);
+void br_set_offset(u64 *instr, u16 offset);
+void br_add_offset(u64 *instr, u16 offset);
+
 #define OP_BBYTE_BASE		0x0c800000000ULL
 #define OP_BB_A_SRC		0x000000000ffULL
 #define OP_BB_BYTE		0x00000000300ULL
@@ -132,6 +138,10 @@ enum immed_shift {
 	IMMED_SHIFT_2B = 2,
 };
 
+u16 immed_get_value(u64 instr);
+void immed_set_value(u64 *instr, u16 immed);
+void immed_add_value(u64 *instr, u16 offset);
+
 #define OP_SHF_BASE		0x08000000000ULL
 #define OP_SHF_A_SRC		0x000000000ffULL
 #define OP_SHF_SC		0x00000000300ULL
@@ -175,6 +185,7 @@ enum alu_op {
 	ALU_OP_NONE	= 0x00,
 	ALU_OP_ADD	= 0x01,
 	ALU_OP_NOT	= 0x04,
+	ALU_OP_ADD_2B	= 0x05,
 	ALU_OP_AND	= 0x08,
 	ALU_OP_SUB_C	= 0x0d,
 	ALU_OP_ADD_C	= 0x11,
@@ -209,6 +220,7 @@ enum alu_dst_ab {
 #define OP_CMD_CNT		0x0000e000000ULL
 #define OP_CMD_SIG		0x000f0000000ULL
 #define OP_CMD_TGT_CMD		0x07f00000000ULL
+#define OP_CMD_INDIR		0x20000000000ULL
 #define OP_CMD_MODE	       0x1c0000000000ULL
 
 struct cmd_tgt_act {
@@ -219,6 +231,7 @@ struct cmd_tgt_act {
 enum cmd_tgt_map {
 	CMD_TGT_READ8,
 	CMD_TGT_WRITE8_SWAP,
+	CMD_TGT_WRITE32_SWAP,
 	CMD_TGT_READ32,
 	CMD_TGT_READ32_LE,
 	CMD_TGT_READ32_SWAP,
@@ -240,6 +253,9 @@ enum cmd_ctx_swap {
 	CMD_CTX_NO_SWAP = 3,
 };
 
+#define CMD_OVE_LEN	BIT(7)
+#define CMD_OV_LEN	GENMASK(12, 8)
+
 #define OP_LCSR_BASE		0x0fc00000000ULL
 #define OP_LCSR_A_SRC		0x000000003ffULL
 #define OP_LCSR_B_SRC		0x000000ffc00ULL
@@ -257,6 +273,7 @@ enum lcsr_wr_src {
 #define OP_CARB_BASE		0x0e000000000ULL
 #define OP_CARB_OR		0x00000010000ULL
 
+#define NFP_CSR_CTX_PTR		0x20
 #define NFP_CSR_ACT_LM_ADDR0	0x64
 #define NFP_CSR_ACT_LM_ADDR1	0x6c
 #define NFP_CSR_ACT_LM_ADDR2	0x94
@@ -377,4 +394,13 @@ int swreg_to_restricted(swreg dst, swreg lreg, swreg rreg,
 int nfp_ustore_check_valid_no_ecc(u64 insn);
 u64 nfp_ustore_calc_ecc_insn(u64 insn);
 
+#define NFP_IND_ME_REFL_WR_SIG_INIT	3
+#define NFP_IND_ME_CTX_PTR_BASE_MASK	GENMASK(9, 0)
+#define NFP_IND_NUM_CONTEXTS		8
+
+static inline u32 nfp_get_ind_csr_ctx_ptr_offs(u32 read_offset)
+{
+	return (read_offset & ~NFP_IND_ME_CTX_PTR_BASE_MASK) | NFP_CSR_CTX_PTR;
+}
+
 #endif
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c
index 6c9f29c2e975..eb0fc614673d 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c
@@ -152,18 +152,8 @@ nfp_devlink_port_unsplit(struct devlink *devlink, unsigned int port_index)
 static int nfp_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
 {
 	struct nfp_pf *pf = devlink_priv(devlink);
-	int ret;
-
-	mutex_lock(&pf->lock);
-	if (!pf->app) {
-		ret = -EBUSY;
-		goto out;
-	}
-	ret = nfp_app_eswitch_mode_get(pf->app, mode);
-out:
-	mutex_unlock(&pf->lock);
 
-	return ret;
+	return nfp_app_eswitch_mode_get(pf->app, mode);
 }
 
 const struct devlink_ops nfp_devlink_ops = {
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c
index 35eaccbece36..ab301d56430b 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c
@@ -45,6 +45,7 @@
 #include <linux/pci.h>
 #include <linux/firmware.h>
 #include <linux/vermagic.h>
+#include <linux/vmalloc.h>
 #include <net/devlink.h>
 
 #include "nfpcore/nfp.h"
@@ -498,17 +499,16 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 	if (err)
 		goto err_hwinfo_free;
 
-	err = devlink_register(devlink, &pdev->dev);
-	if (err)
-		goto err_hwinfo_free;
-
 	err = nfp_nsp_init(pdev, pf);
 	if (err)
-		goto err_devlink_unreg;
+		goto err_hwinfo_free;
 
 	pf->mip = nfp_mip_open(pf->cpp);
 	pf->rtbl = __nfp_rtsym_table_read(pf->cpp, pf->mip);
 
+	pf->dump_flag = NFP_DUMP_NSP_DIAG;
+	pf->dumpspec = nfp_net_dump_load_dumpspec(pf->cpp, pf->rtbl);
+
 	err = nfp_pcie_sriov_read_nfd_limit(pf);
 	if (err)
 		goto err_fw_unload;
@@ -518,6 +518,7 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 		dev_err(&pdev->dev,
 			"Error: %d VFs already enabled, but loaded FW can only support %d\n",
 			pf->num_vfs, pf->limit_vfs);
+		err = -EINVAL;
 		goto err_fw_unload;
 	}
 
@@ -544,8 +545,7 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 		nfp_fw_unload(pf);
 	kfree(pf->eth_tbl);
 	kfree(pf->nspi);
-err_devlink_unreg:
-	devlink_unregister(devlink);
+	vfree(pf->dumpspec);
 err_hwinfo_free:
 	kfree(pf->hwinfo);
 	nfp_cpp_free(pf->cpp);
@@ -566,19 +566,15 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 static void nfp_pci_remove(struct pci_dev *pdev)
 {
 	struct nfp_pf *pf = pci_get_drvdata(pdev);
-	struct devlink *devlink;
 
 	nfp_hwmon_unregister(pf);
 
-	devlink = priv_to_devlink(pf);
-
-	nfp_net_pci_remove(pf);
-
 	nfp_pcie_sriov_disable(pdev);
 	pci_sriov_set_totalvfs(pf->pdev, 0);
 
-	devlink_unregister(devlink);
+	nfp_net_pci_remove(pf);
 
+	vfree(pf->dumpspec);
 	kfree(pf->rtbl);
 	nfp_mip_close(pf->mip);
 	if (pf->fw_loaded)
@@ -592,7 +588,7 @@ static void nfp_pci_remove(struct pci_dev *pdev)
 	kfree(pf->eth_tbl);
 	kfree(pf->nspi);
 	mutex_destroy(&pf->lock);
-	devlink_free(devlink);
+	devlink_free(priv_to_devlink(pf));
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
 }
@@ -653,3 +649,4 @@ MODULE_FIRMWARE("netronome/nic_AMDA0099-0001_2x25.nffw");
 MODULE_AUTHOR("Netronome Systems <oss-drivers@netronome.com>");
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("The Netronome Flow Processor (NFP) driver.");
+MODULE_VERSION(UTS_RELEASE);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.h b/drivers/net/ethernet/netronome/nfp/nfp_main.h
index be0ee59f2eb9..add46e28212b 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.h
@@ -39,6 +39,7 @@
 #ifndef NFP_MAIN_H
 #define NFP_MAIN_H
 
+#include <linux/ethtool.h>
 #include <linux/list.h>
 #include <linux/types.h>
 #include <linux/msi.h>
@@ -61,6 +62,17 @@ struct nfp_nsp_identify;
 struct nfp_port;
 struct nfp_rtsym_table;
 
+/**
+ * struct nfp_dumpspec - NFP FW dump specification structure
+ * @size:	Size of the data
+ * @data:	Sequence of TLVs, each being an instruction to dump some data
+ *		from FW
+ */
+struct nfp_dumpspec {
+	u32 size;
+	u8 data[0];
+};
+
 /**
  * struct nfp_pf - NFP PF-specific device structure
  * @pdev:		Backpointer to PCI device
@@ -83,6 +95,9 @@ struct nfp_rtsym_table;
  * @mip:		MIP handle
  * @rtbl:		RTsym table
  * @hwinfo:		HWInfo table
+ * @dumpspec:		Debug dump specification
+ * @dump_flag:		Store dump flag between set_dump and get_dump_flag
+ * @dump_len:		Store dump length between set_dump and get_dump_flag
  * @eth_tbl:		NSP ETH table
  * @nspi:		NSP identification info
  * @hwmon_dev:		pointer to hwmon device
@@ -124,6 +139,9 @@ struct nfp_pf {
 	const struct nfp_mip *mip;
 	struct nfp_rtsym_table *rtbl;
 	struct nfp_hwinfo *hwinfo;
+	struct nfp_dumpspec *dumpspec;
+	u32 dump_flag;
+	u32 dump_len;
 	struct nfp_eth_table *eth_tbl;
 	struct nfp_nsp_identify *nspi;
 
@@ -157,4 +175,15 @@ void nfp_net_get_mac_addr(struct nfp_pf *pf, struct nfp_port *port);
 
 bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb);
 
+enum nfp_dump_diag {
+	NFP_DUMP_NSP_DIAG = 0,
+};
+
+struct nfp_dumpspec *
+nfp_net_dump_load_dumpspec(struct nfp_cpp *cpp, struct nfp_rtsym_table *rtbl);
+s64 nfp_net_dump_calculate_size(struct nfp_pf *pf, struct nfp_dumpspec *spec,
+				u32 flag);
+int nfp_net_dump_populate_buffer(struct nfp_pf *pf, struct nfp_dumpspec *spec,
+				 struct ethtool_dump *dump_param, void *dest);
+
 #endif /* NFP_MAIN_H */
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h
index 7f9857c276b1..787df47ec430 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h
@@ -47,6 +47,7 @@
 #include <linux/netdevice.h>
 #include <linux/pci.h>
 #include <linux/io-64-nonatomic-hi-lo.h>
+#include <net/xdp.h>
 
 #include "nfp_net_ctrl.h"
 
@@ -192,7 +193,8 @@ struct nfp_net_tx_desc {
 
 /**
  * struct nfp_net_tx_buf - software TX buffer descriptor
- * @skb:	sk_buff associated with this buffer
+ * @skb:	normal ring, sk_buff associated with this buffer
+ * @frag:	XDP ring, page frag associated with this buffer
  * @dma_addr:	DMA mapping address of the buffer
  * @fidx:	Fragment index (-1 for the head and [0..nr_frags-1] for frags)
  * @pkt_cnt:	Number of packets to be produced out of the skb associated
@@ -350,6 +352,7 @@ struct nfp_net_rx_buf {
  * @rxds:       Virtual address of FL/RX ring in host memory
  * @dma:        DMA address of the FL/RX ring
  * @size:       Size, in bytes, of the FL/RX ring (needed to free)
+ * @xdp_rxq:    RX-ring info avail for XDP
  */
 struct nfp_net_rx_ring {
 	struct nfp_net_r_vector *r_vec;
@@ -361,19 +364,23 @@ struct nfp_net_rx_ring {
 	u32 idx;
 
 	int fl_qcidx;
+	unsigned int size;
 	u8 __iomem *qcp_fl;
 
 	struct nfp_net_rx_buf *rxbufs;
 	struct nfp_net_rx_desc *rxds;
 
 	dma_addr_t dma;
-	unsigned int size;
+	struct xdp_rxq_info xdp_rxq;
 } ____cacheline_aligned;
 
 /**
  * struct nfp_net_r_vector - Per ring interrupt vector configuration
  * @nfp_net:        Backpointer to nfp_net structure
  * @napi:           NAPI structure for this ring vec
+ * @tasklet:        ctrl vNIC, tasklet for servicing the r_vec
+ * @queue:          ctrl vNIC, send queue
+ * @lock:           ctrl vNIC, r_vec lock protects @queue
  * @tx_ring:        Pointer to TX ring
  * @rx_ring:        Pointer to RX ring
  * @xdp_ring:	    Pointer to an extra TX ring for XDP
@@ -548,6 +555,8 @@ struct nfp_net_dp {
  * @max_r_vecs:		Number of allocated interrupt vectors for RX/TX
  * @max_tx_rings:       Maximum number of TX rings supported by the Firmware
  * @max_rx_rings:       Maximum number of RX rings supported by the Firmware
+ * @stride_rx:		Queue controller RX queue spacing
+ * @stride_tx:		Queue controller TX queue spacing
  * @r_vecs:             Pre-allocated array of ring vectors
  * @irq_entries:        Pre-allocated array of MSI-X entries
  * @lsc_handler:        Handler for Link State Change interrupt
@@ -573,6 +582,7 @@ struct nfp_net_dp {
  * @qcp_cfg:            Pointer to QCP queue used for configuration notification
  * @tx_bar:             Pointer to mapped TX queues
  * @rx_bar:             Pointer to mapped FL/RX queues
+ * @tlv_caps:		Parsed TLV capabilities
  * @debugfs_dir:	Device directory in debugfs
  * @vnic_list:		Entry on device vNIC list
  * @pdev:		Backpointer to PCI device
@@ -639,6 +649,8 @@ struct nfp_net {
 	u8 __iomem *tx_bar;
 	u8 __iomem *rx_bar;
 
+	struct nfp_net_tlv_caps tlv_caps;
+
 	struct dentry *debugfs_dir;
 
 	struct list_head vnic_list;
@@ -834,6 +846,18 @@ static inline const char *nfp_net_name(struct nfp_net *nn)
 	return nn->dp.netdev ? nn->dp.netdev->name : "ctrl";
 }
 
+static inline void nfp_ctrl_lock(struct nfp_net *nn)
+	__acquires(&nn->r_vecs[0].lock)
+{
+	spin_lock_bh(&nn->r_vecs[0].lock);
+}
+
+static inline void nfp_ctrl_unlock(struct nfp_net *nn)
+	__releases(&nn->r_vecs[0].lock)
+{
+	spin_unlock_bh(&nn->r_vecs[0].lock);
+}
+
 /* Globals */
 extern const char nfp_driver_version[];
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 99b0487b6d82..a05be0ab2713 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -293,9 +293,15 @@ int nfp_net_reconfig(struct nfp_net *nn, u32 update)
  */
 static int nfp_net_reconfig_mbox(struct nfp_net *nn, u32 mbox_cmd)
 {
+	u32 mbox = nn->tlv_caps.mbox_off;
 	int ret;
 
-	nn_writeq(nn, NFP_NET_CFG_MBOX_CMD, mbox_cmd);
+	if (!nfp_net_has_mbox(&nn->tlv_caps)) {
+		nn_err(nn, "no mailbox present, command: %u\n", mbox_cmd);
+		return -EIO;
+	}
+
+	nn_writeq(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_CMD, mbox_cmd);
 
 	ret = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_MBOX);
 	if (ret) {
@@ -303,7 +309,7 @@ static int nfp_net_reconfig_mbox(struct nfp_net *nn, u32 mbox_cmd)
 		return ret;
 	}
 
-	return -nn_readl(nn, NFP_NET_CFG_MBOX_RET);
+	return -nn_readl(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_RET);
 }
 
 /* Interrupt configuration and handling
@@ -1610,11 +1616,13 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 	unsigned int true_bufsz;
 	struct sk_buff *skb;
 	int pkts_polled = 0;
+	struct xdp_buff xdp;
 	int idx;
 
 	rcu_read_lock();
 	xdp_prog = READ_ONCE(dp->xdp_prog);
 	true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz;
+	xdp.rxq = &rx_ring->xdp_rxq;
 	tx_ring = r_vec->xdp_ring;
 
 	while (pkts_polled < budget) {
@@ -1705,7 +1713,6 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget)
 				  dp->bpf_offload_xdp) && !meta.portid) {
 			void *orig_data = rxbuf->frag + pkt_off;
 			unsigned int dma_off;
-			struct xdp_buff xdp;
 			int act;
 
 			xdp.data_hard_start = rxbuf->frag + NFP_NET_RX_BUF_HEADROOM;
@@ -1919,6 +1926,13 @@ nfp_ctrl_tx_one(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,
 	return false;
 }
 
+bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)
+{
+	struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];
+
+	return nfp_ctrl_tx_one(nn, r_vec, skb, false);
+}
+
 bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)
 {
 	struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];
@@ -2254,6 +2268,8 @@ static void nfp_net_rx_ring_free(struct nfp_net_rx_ring *rx_ring)
 	struct nfp_net_r_vector *r_vec = rx_ring->r_vec;
 	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;
 
+	if (dp->netdev)
+		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 	kfree(rx_ring->rxbufs);
 
 	if (rx_ring->rxds)
@@ -2277,7 +2293,14 @@ static void nfp_net_rx_ring_free(struct nfp_net_rx_ring *rx_ring)
 static int
 nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring)
 {
-	int sz;
+	int sz, err;
+
+	if (dp->netdev) {
+		err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev,
+				       rx_ring->idx);
+		if (err < 0)
+			return err;
+	}
 
 	rx_ring->cnt = dp->rxd_cnt;
 	rx_ring->size = sizeof(*rx_ring->rxds) * rx_ring->cnt;
@@ -2441,7 +2464,7 @@ void nfp_net_coalesce_write_cfg(struct nfp_net *nn)
 	 * ME timestamp ticks.  There are 16 ME clock cycles for each timestamp
 	 * count.
 	 */
-	factor = nn->me_freq_mhz / 16;
+	factor = nn->tlv_caps.me_freq_mhz / 16;
 
 	/* copy RX interrupt coalesce parameters */
 	value = (nn->rx_coalesce_max_frames << 16) |
@@ -2852,6 +2875,11 @@ static void nfp_net_set_rx_mode(struct net_device *netdev)
 
 	new_ctrl = nn->dp.ctrl;
 
+	if (!netdev_mc_empty(netdev) || netdev->flags & IFF_ALLMULTI)
+		new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_L2MC;
+	else
+		new_ctrl &= ~NFP_NET_CFG_CTRL_L2MC;
+
 	if (netdev->flags & IFF_PROMISC) {
 		if (nn->cap & NFP_NET_CFG_CTRL_PROMISC)
 			new_ctrl |= NFP_NET_CFG_CTRL_PROMISC;
@@ -3036,6 +3064,11 @@ static int nfp_net_change_mtu(struct net_device *netdev, int new_mtu)
 {
 	struct nfp_net *nn = netdev_priv(netdev);
 	struct nfp_net_dp *dp;
+	int err;
+
+	err = nfp_app_change_mtu(nn->app, netdev, new_mtu);
+	if (err)
+		return err;
 
 	dp = nfp_net_clone_dp(nn);
 	if (!dp)
@@ -3057,8 +3090,9 @@ nfp_net_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
 	if (!vid)
 		return 0;
 
-	nn_writew(nn, NFP_NET_CFG_VLAN_FILTER_VID, vid);
-	nn_writew(nn, NFP_NET_CFG_VLAN_FILTER_PROTO, ETH_P_8021Q);
+	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_VID, vid);
+	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_PROTO,
+		  ETH_P_8021Q);
 
 	return nfp_net_reconfig_mbox(nn, NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_ADD);
 }
@@ -3074,8 +3108,9 @@ nfp_net_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
 	if (!vid)
 		return 0;
 
-	nn_writew(nn, NFP_NET_CFG_VLAN_FILTER_VID, vid);
-	nn_writew(nn, NFP_NET_CFG_VLAN_FILTER_PROTO, ETH_P_8021Q);
+	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_VID, vid);
+	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_PROTO,
+		  ETH_P_8021Q);
 
 	return nfp_net_reconfig_mbox(nn, NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_KILL);
 }
@@ -3175,10 +3210,9 @@ static int nfp_net_set_features(struct net_device *netdev,
 			new_ctrl &= ~NFP_NET_CFG_CTRL_GATHER;
 	}
 
-	if (changed & NETIF_F_HW_TC && nfp_app_tc_busy(nn->app, nn)) {
-		nn_err(nn, "Cannot disable HW TC offload while in use\n");
-		return -EBUSY;
-	}
+	err = nfp_port_set_features(netdev, features);
+	if (err)
+		return err;
 
 	nn_dbg(nn, "Feature change 0x%llx -> 0x%llx (changed=0x%llx)\n",
 	       netdev->features, features, changed);
@@ -3368,7 +3402,7 @@ nfp_net_xdp_setup(struct nfp_net *nn, struct bpf_prog *prog, u32 flags,
 	if (err)
 		return err;
 
-	err = nfp_app_xdp_offload(nn->app, nn, offload_prog);
+	err = nfp_app_xdp_offload(nn->app, nn, offload_prog, extack);
 	if (err && flags & XDP_FLAGS_HW_MODE)
 		return err;
 
@@ -3394,17 +3428,10 @@ static int nfp_net_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
 		if (nn->dp.bpf_offload_xdp)
 			xdp->prog_attached = XDP_ATTACHED_HW;
 		xdp->prog_id = nn->xdp_prog ? nn->xdp_prog->aux->id : 0;
+		xdp->prog_flags = nn->xdp_prog ? nn->xdp_flags : 0;
 		return 0;
-	case BPF_OFFLOAD_VERIFIER_PREP:
-		return nfp_app_bpf_verifier_prep(nn->app, nn, xdp);
-	case BPF_OFFLOAD_TRANSLATE:
-		return nfp_app_bpf_translate(nn->app, nn,
-					     xdp->offload.prog);
-	case BPF_OFFLOAD_DESTROY:
-		return nfp_app_bpf_destroy(nn->app, nn,
-					   xdp->offload.prog);
 	default:
-		return -EINVAL;
+		return nfp_app_bpf(nn->app, nn, xdp);
 	}
 }
 
@@ -3563,9 +3590,6 @@ struct nfp_net *nfp_net_alloc(struct pci_dev *pdev, bool needs_netdev,
  */
 void nfp_net_free(struct nfp_net *nn)
 {
-	if (nn->xdp_prog)
-		bpf_prog_put(nn->xdp_prog);
-
 	if (nn->dp.netdev)
 		free_netdev(nn->dp.netdev);
 	else
@@ -3709,7 +3733,7 @@ static void nfp_net_netdev_init(struct nfp_net *nn)
 
 	netdev->features = netdev->hw_features;
 
-	if (nfp_app_has_tc(nn->app))
+	if (nfp_app_has_tc(nn->app) && nn->port)
 		netdev->hw_features |= NETIF_F_HW_TC;
 
 	/* Advertise but disable TSO by default. */
@@ -3726,23 +3750,15 @@ static void nfp_net_netdev_init(struct nfp_net *nn)
 	netdev->min_mtu = ETH_MIN_MTU;
 	netdev->max_mtu = nn->max_mtu;
 
+	netdev->gso_max_segs = NFP_NET_LSO_MAX_SEGS;
+
 	netif_carrier_off(netdev);
 
 	nfp_net_set_ethtool_ops(netdev);
 }
 
-/**
- * nfp_net_init() - Initialise/finalise the nfp_net structure
- * @nn:		NFP Net device structure
- *
- * Return: 0 on success or negative errno on error.
- */
-int nfp_net_init(struct nfp_net *nn)
+static int nfp_net_read_caps(struct nfp_net *nn)
 {
-	int err;
-
-	nn->dp.rx_dma_dir = DMA_FROM_DEVICE;
-
 	/* Get some of the read-only fields from the BAR */
 	nn->cap = nn_readl(nn, NFP_NET_CFG_CAP);
 	nn->max_mtu = nn_readl(nn, NFP_NET_CFG_MAX_MTU);
@@ -3775,6 +3791,29 @@ int nfp_net_init(struct nfp_net *nn)
 		nn->dp.rx_offset = NFP_NET_RX_OFFSET;
 	}
 
+	/* For control vNICs mask out the capabilities app doesn't want. */
+	if (!nn->dp.netdev)
+		nn->cap &= nn->app->type->ctrl_cap_mask;
+
+	return 0;
+}
+
+/**
+ * nfp_net_init() - Initialise/finalise the nfp_net structure
+ * @nn:		NFP Net device structure
+ *
+ * Return: 0 on success or negative errno on error.
+ */
+int nfp_net_init(struct nfp_net *nn)
+{
+	int err;
+
+	nn->dp.rx_dma_dir = DMA_FROM_DEVICE;
+
+	err = nfp_net_read_caps(nn);
+	if (err)
+		return err;
+
 	/* Set default MTU and Freelist buffer size */
 	if (nn->max_mtu < NFP_NET_DEFAULT_MTU)
 		nn->dp.mtu = nn->max_mtu;
@@ -3791,8 +3830,6 @@ int nfp_net_init(struct nfp_net *nn)
 	/* Allow L2 Broadcast and Multicast through by default, if supported */
 	if (nn->cap & NFP_NET_CFG_CTRL_L2BC)
 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_L2BC;
-	if (nn->cap & NFP_NET_CFG_CTRL_L2MC)
-		nn->dp.ctrl |= NFP_NET_CFG_CTRL_L2MC;
 
 	/* Allow IRQ moderation, if supported */
 	if (nn->cap & NFP_NET_CFG_CTRL_IRQMOD) {
@@ -3800,6 +3837,11 @@ int nfp_net_init(struct nfp_net *nn)
 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_IRQMOD;
 	}
 
+	err = nfp_net_tlv_caps_parse(&nn->pdev->dev, nn->dp.ctrl_bar,
+				     &nn->tlv_caps);
+	if (err)
+		return err;
+
 	if (nn->dp.netdev)
 		nfp_net_netdev_init(nn);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.c
new file mode 100644
index 000000000000..1f9149bb2ae6
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright (C) 2018 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+
+#include "nfp_net_ctrl.h"
+#include "nfp_net.h"
+
+static void nfp_net_tlv_caps_reset(struct nfp_net_tlv_caps *caps)
+{
+	memset(caps, 0, sizeof(*caps));
+	caps->me_freq_mhz = 1200;
+	caps->mbox_off = NFP_NET_CFG_MBOX_BASE;
+	caps->mbox_len = NFP_NET_CFG_MBOX_VAL_MAX_SZ;
+}
+
+int nfp_net_tlv_caps_parse(struct device *dev, u8 __iomem *ctrl_mem,
+			   struct nfp_net_tlv_caps *caps)
+{
+	u8 __iomem *data = ctrl_mem + NFP_NET_CFG_TLV_BASE;
+	u8 __iomem *end = ctrl_mem + NFP_NET_CFG_BAR_SZ;
+	u32 hdr;
+
+	nfp_net_tlv_caps_reset(caps);
+
+	hdr = readl(data);
+	if (!hdr)
+		return 0;
+
+	while (true) {
+		unsigned int length, offset;
+		u32 hdr = readl(data);
+
+		length = FIELD_GET(NFP_NET_CFG_TLV_HEADER_LENGTH, hdr);
+		offset = data - ctrl_mem;
+
+		/* Advance past the header */
+		data += 4;
+
+		if (length % NFP_NET_CFG_TLV_LENGTH_INC) {
+			dev_err(dev, "TLV size not multiple of %u len:%u\n",
+				NFP_NET_CFG_TLV_LENGTH_INC, length);
+			return -EINVAL;
+		}
+		if (data + length > end) {
+			dev_err(dev, "oversized TLV offset:%u len:%u\n",
+				offset, length);
+			return -EINVAL;
+		}
+
+		switch (FIELD_GET(NFP_NET_CFG_TLV_HEADER_TYPE, hdr)) {
+		case NFP_NET_CFG_TLV_TYPE_UNKNOWN:
+			dev_err(dev, "NULL TLV at offset:%u\n", offset);
+			return -EINVAL;
+		case NFP_NET_CFG_TLV_TYPE_RESERVED:
+			break;
+		case NFP_NET_CFG_TLV_TYPE_END:
+			if (!length)
+				return 0;
+
+			dev_err(dev, "END TLV should be empty, has len:%d\n",
+				length);
+			return -EINVAL;
+		case NFP_NET_CFG_TLV_TYPE_ME_FREQ:
+			if (length != 4) {
+				dev_err(dev,
+					"ME FREQ TLV should be 4B, is %dB\n",
+					length);
+				return -EINVAL;
+			}
+
+			caps->me_freq_mhz = readl(data);
+			break;
+		case NFP_NET_CFG_TLV_TYPE_MBOX:
+			if (!length) {
+				caps->mbox_off = 0;
+				caps->mbox_len = 0;
+			} else {
+				caps->mbox_off = data - ctrl_mem;
+				caps->mbox_len = length;
+			}
+			break;
+		default:
+			if (!FIELD_GET(NFP_NET_CFG_TLV_HEADER_REQUIRED, hdr))
+				break;
+
+			dev_err(dev, "unknown TLV type:%u offset:%u len:%u\n",
+				FIELD_GET(NFP_NET_CFG_TLV_HEADER_TYPE, hdr),
+				offset, length);
+			return -EINVAL;
+		}
+
+		data += length;
+		if (data + 4 > end) {
+			dev_err(dev, "reached end of BAR without END TLV\n");
+			return -EINVAL;
+		}
+	}
+
+	/* Not reached */
+	return -EINVAL;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
index 782d452e0fc2..4499a7333078 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
@@ -43,9 +43,7 @@
 #ifndef _NFP_NET_CTRL_H_
 #define _NFP_NET_CTRL_H_
 
-/* IMPORTANT: This header file is shared with the FW,
- *	      no OS specific constructs, please!
- */
+#include <linux/types.h>
 
 /**
  * Configuration BAR size.
@@ -61,9 +59,12 @@
 #define NFP_NET_RX_OFFSET               32
 
 /**
- * Maximum header size supported for LSO frames
+ * LSO parameters
+ * %NFP_NET_LSO_MAX_HDR_SZ:	Maximum header size supported for LSO frames
+ * %NFP_NET_LSO_MAX_SEGS:	Maximum number of segments LSO frame can produce
  */
 #define NFP_NET_LSO_MAX_HDR_SZ		255
+#define NFP_NET_LSO_MAX_SEGS		64
 
 /**
  * Prepend field types
@@ -91,23 +92,24 @@
 #define NFP_NET_RSS_IPV6_EX_UDP         9
 
 /**
- * @NFP_NET_TXR_MAX:         Maximum number of TX rings
- * @NFP_NET_RXR_MAX:         Maximum number of RX rings
+ * Ring counts
+ * %NFP_NET_TXR_MAX:         Maximum number of TX rings
+ * %NFP_NET_RXR_MAX:         Maximum number of RX rings
  */
 #define NFP_NET_TXR_MAX                 64
 #define NFP_NET_RXR_MAX                 64
 
 /**
  * Read/Write config words (0x0000 - 0x002c)
- * @NFP_NET_CFG_CTRL:        Global control
- * @NFP_NET_CFG_UPDATE:      Indicate which fields are updated
- * @NFP_NET_CFG_TXRS_ENABLE: Bitmask of enabled TX rings
- * @NFP_NET_CFG_RXRS_ENABLE: Bitmask of enabled RX rings
- * @NFP_NET_CFG_MTU:         Set MTU size
- * @NFP_NET_CFG_FLBUFSZ:     Set freelist buffer size (must be larger than MTU)
- * @NFP_NET_CFG_EXN:         MSI-X table entry for exceptions
- * @NFP_NET_CFG_LSC:         MSI-X table entry for link state changes
- * @NFP_NET_CFG_MACADDR:     MAC address
+ * %NFP_NET_CFG_CTRL:        Global control
+ * %NFP_NET_CFG_UPDATE:      Indicate which fields are updated
+ * %NFP_NET_CFG_TXRS_ENABLE: Bitmask of enabled TX rings
+ * %NFP_NET_CFG_RXRS_ENABLE: Bitmask of enabled RX rings
+ * %NFP_NET_CFG_MTU:         Set MTU size
+ * %NFP_NET_CFG_FLBUFSZ:     Set freelist buffer size (must be larger than MTU)
+ * %NFP_NET_CFG_EXN:         MSI-X table entry for exceptions
+ * %NFP_NET_CFG_LSC:         MSI-X table entry for link state changes
+ * %NFP_NET_CFG_MACADDR:     MAC address
  *
  * TODO:
  * - define Error details in UPDATE
@@ -176,14 +178,14 @@
 
 /**
  * Read-only words (0x0030 - 0x0050):
- * @NFP_NET_CFG_VERSION:     Firmware version number
- * @NFP_NET_CFG_STS:         Status
- * @NFP_NET_CFG_CAP:         Capabilities (same bits as @NFP_NET_CFG_CTRL)
- * @NFP_NET_CFG_MAX_TXRINGS: Maximum number of TX rings
- * @NFP_NET_CFG_MAX_RXRINGS: Maximum number of RX rings
- * @NFP_NET_CFG_MAX_MTU:     Maximum support MTU
- * @NFP_NET_CFG_START_TXQ:   Start Queue Control Queue to use for TX (PF only)
- * @NFP_NET_CFG_START_RXQ:   Start Queue Control Queue to use for RX (PF only)
+ * %NFP_NET_CFG_VERSION:     Firmware version number
+ * %NFP_NET_CFG_STS:         Status
+ * %NFP_NET_CFG_CAP:         Capabilities (same bits as %NFP_NET_CFG_CTRL)
+ * %NFP_NET_CFG_MAX_TXRINGS: Maximum number of TX rings
+ * %NFP_NET_CFG_MAX_RXRINGS: Maximum number of RX rings
+ * %NFP_NET_CFG_MAX_MTU:     Maximum support MTU
+ * %NFP_NET_CFG_START_TXQ:   Start Queue Control Queue to use for TX (PF only)
+ * %NFP_NET_CFG_START_RXQ:   Start Queue Control Queue to use for RX (PF only)
  *
  * TODO:
  * - define more STS bits
@@ -228,31 +230,37 @@
 
 /**
  * RSS capabilities
- * @NFP_NET_CFG_RSS_CAP_HFUNC:	supported hash functions (same bits as
- *				@NFP_NET_CFG_RSS_HFUNC)
+ * %NFP_NET_CFG_RSS_CAP_HFUNC:	supported hash functions (same bits as
+ *				%NFP_NET_CFG_RSS_HFUNC)
  */
 #define NFP_NET_CFG_RSS_CAP		0x0054
 #define   NFP_NET_CFG_RSS_CAP_HFUNC	  0xff000000
 
+/**
+ * TLV area start
+ * %NFP_NET_CFG_TLV_BASE:	start anchor of the TLV area
+ */
+#define NFP_NET_CFG_TLV_BASE		0x0058
+
 /**
  * VXLAN/UDP encap configuration
- * @NFP_NET_CFG_VXLAN_PORT:	Base address of table of tunnels' UDP dst ports
- * @NFP_NET_CFG_VXLAN_SZ:	Size of the UDP port table in bytes
+ * %NFP_NET_CFG_VXLAN_PORT:	Base address of table of tunnels' UDP dst ports
+ * %NFP_NET_CFG_VXLAN_SZ:	Size of the UDP port table in bytes
  */
 #define NFP_NET_CFG_VXLAN_PORT		0x0060
 #define NFP_NET_CFG_VXLAN_SZ		  0x0008
 
 /**
  * BPF section
- * @NFP_NET_CFG_BPF_ABI:	BPF ABI version
- * @NFP_NET_CFG_BPF_CAP:	BPF capabilities
- * @NFP_NET_CFG_BPF_MAX_LEN:	Maximum size of JITed BPF code in bytes
- * @NFP_NET_CFG_BPF_START:	Offset at which BPF will be loaded
- * @NFP_NET_CFG_BPF_DONE:	Offset to jump to on exit
- * @NFP_NET_CFG_BPF_STACK_SZ:	Total size of stack area in 64B chunks
- * @NFP_NET_CFG_BPF_INL_MTU:	Packet data split offset in 64B chunks
- * @NFP_NET_CFG_BPF_SIZE:	Size of the JITed BPF code in instructions
- * @NFP_NET_CFG_BPF_ADDR:	DMA address of the buffer with JITed BPF code
+ * %NFP_NET_CFG_BPF_ABI:	BPF ABI version
+ * %NFP_NET_CFG_BPF_CAP:	BPF capabilities
+ * %NFP_NET_CFG_BPF_MAX_LEN:	Maximum size of JITed BPF code in bytes
+ * %NFP_NET_CFG_BPF_START:	Offset at which BPF will be loaded
+ * %NFP_NET_CFG_BPF_DONE:	Offset to jump to on exit
+ * %NFP_NET_CFG_BPF_STACK_SZ:	Total size of stack area in 64B chunks
+ * %NFP_NET_CFG_BPF_INL_MTU:	Packet data split offset in 64B chunks
+ * %NFP_NET_CFG_BPF_SIZE:	Size of the JITed BPF code in instructions
+ * %NFP_NET_CFG_BPF_ADDR:	DMA address of the buffer with JITed BPF code
  */
 #define NFP_NET_CFG_BPF_ABI		0x0080
 #define   NFP_NET_BPF_ABI		2
@@ -278,9 +286,9 @@
 /**
  * RSS configuration (0x0100 - 0x01ac):
  * Used only when NFP_NET_CFG_CTRL_RSS is enabled
- * @NFP_NET_CFG_RSS_CFG:     RSS configuration word
- * @NFP_NET_CFG_RSS_KEY:     RSS "secret" key
- * @NFP_NET_CFG_RSS_ITBL:    RSS indirection table
+ * %NFP_NET_CFG_RSS_CFG:     RSS configuration word
+ * %NFP_NET_CFG_RSS_KEY:     RSS "secret" key
+ * %NFP_NET_CFG_RSS_ITBL:    RSS indirection table
  */
 #define NFP_NET_CFG_RSS_BASE            0x0100
 #define NFP_NET_CFG_RSS_CTRL            NFP_NET_CFG_RSS_BASE
@@ -305,13 +313,13 @@
 
 /**
  * TX ring configuration (0x200 - 0x800)
- * @NFP_NET_CFG_TXR_BASE:    Base offset for TX ring configuration
- * @NFP_NET_CFG_TXR_ADDR:    Per TX ring DMA address (8B entries)
- * @NFP_NET_CFG_TXR_WB_ADDR: Per TX ring write back DMA address (8B entries)
- * @NFP_NET_CFG_TXR_SZ:      Per TX ring ring size (1B entries)
- * @NFP_NET_CFG_TXR_VEC:     Per TX ring MSI-X table entry (1B entries)
- * @NFP_NET_CFG_TXR_PRIO:    Per TX ring priority (1B entries)
- * @NFP_NET_CFG_TXR_IRQ_MOD: Per TX ring interrupt moderation packet
+ * %NFP_NET_CFG_TXR_BASE:    Base offset for TX ring configuration
+ * %NFP_NET_CFG_TXR_ADDR:    Per TX ring DMA address (8B entries)
+ * %NFP_NET_CFG_TXR_WB_ADDR: Per TX ring write back DMA address (8B entries)
+ * %NFP_NET_CFG_TXR_SZ:      Per TX ring ring size (1B entries)
+ * %NFP_NET_CFG_TXR_VEC:     Per TX ring MSI-X table entry (1B entries)
+ * %NFP_NET_CFG_TXR_PRIO:    Per TX ring priority (1B entries)
+ * %NFP_NET_CFG_TXR_IRQ_MOD: Per TX ring interrupt moderation packet
  */
 #define NFP_NET_CFG_TXR_BASE            0x0200
 #define NFP_NET_CFG_TXR_ADDR(_x)        (NFP_NET_CFG_TXR_BASE + ((_x) * 0x8))
@@ -325,12 +333,12 @@
 
 /**
  * RX ring configuration (0x0800 - 0x0c00)
- * @NFP_NET_CFG_RXR_BASE:    Base offset for RX ring configuration
- * @NFP_NET_CFG_RXR_ADDR:    Per RX ring DMA address (8B entries)
- * @NFP_NET_CFG_RXR_SZ:      Per RX ring ring size (1B entries)
- * @NFP_NET_CFG_RXR_VEC:     Per RX ring MSI-X table entry (1B entries)
- * @NFP_NET_CFG_RXR_PRIO:    Per RX ring priority (1B entries)
- * @NFP_NET_CFG_RXR_IRQ_MOD: Per RX ring interrupt moderation (4B entries)
+ * %NFP_NET_CFG_RXR_BASE:    Base offset for RX ring configuration
+ * %NFP_NET_CFG_RXR_ADDR:    Per RX ring DMA address (8B entries)
+ * %NFP_NET_CFG_RXR_SZ:      Per RX ring ring size (1B entries)
+ * %NFP_NET_CFG_RXR_VEC:     Per RX ring MSI-X table entry (1B entries)
+ * %NFP_NET_CFG_RXR_PRIO:    Per RX ring priority (1B entries)
+ * %NFP_NET_CFG_RXR_IRQ_MOD: Per RX ring interrupt moderation (4B entries)
  */
 #define NFP_NET_CFG_RXR_BASE            0x0800
 #define NFP_NET_CFG_RXR_ADDR(_x)        (NFP_NET_CFG_RXR_BASE + ((_x) * 0x8))
@@ -343,7 +351,7 @@
 /**
  * Interrupt Control/Cause registers (0x0c00 - 0x0d00)
  * These registers are only used when MSI-X auto-masking is not
- * enabled (@NFP_NET_CFG_CTRL_MSIXAUTO not set).  The array is index
+ * enabled (%NFP_NET_CFG_CTRL_MSIXAUTO not set).  The array is index
  * by MSI-X entry and are 1B in size.  If an entry is zero, the
  * corresponding entry is enabled.  If the FW generates an interrupt,
  * it writes a cause into the corresponding field.  This also masks
@@ -393,8 +401,8 @@
 /**
  * Per ring stats (0x1000 - 0x1800)
  * options, 64bit per entry
- * @NFP_NET_CFG_TXR_STATS:   TX ring statistics (Packet and Byte count)
- * @NFP_NET_CFG_RXR_STATS:   RX ring statistics (Packet and Byte count)
+ * %NFP_NET_CFG_TXR_STATS:   TX ring statistics (Packet and Byte count)
+ * %NFP_NET_CFG_RXR_STATS:   RX ring statistics (Packet and Byte count)
  */
 #define NFP_NET_CFG_TXR_STATS_BASE      0x1000
 #define NFP_NET_CFG_TXR_STATS(_x)       (NFP_NET_CFG_TXR_STATS_BASE + \
@@ -408,24 +416,105 @@
  * 4B used for update command and 4B return code
  * followed by a max of 504B of variable length value
  */
-#define NFP_NET_CFG_MBOX_CMD		0x1800
-#define NFP_NET_CFG_MBOX_RET		0x1804
-#define NFP_NET_CFG_MBOX_VAL		0x1808
+#define NFP_NET_CFG_MBOX_BASE		0x1800
 #define NFP_NET_CFG_MBOX_VAL_MAX_SZ	0x1F8
 
+#define NFP_NET_CFG_MBOX_SIMPLE_CMD	0x0
+#define NFP_NET_CFG_MBOX_SIMPLE_RET	0x4
+#define NFP_NET_CFG_MBOX_SIMPLE_VAL	0x8
+#define NFP_NET_CFG_MBOX_SIMPLE_LEN	0x12
+
 #define NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_ADD 1
 #define NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_KILL 2
 
 /**
  * VLAN filtering using general use mailbox
- * @NFP_NET_CFG_VLAN_FILTER:		Base address of VLAN filter mailbox
- * @NFP_NET_CFG_VLAN_FILTER_VID:	VLAN ID to filter
- * @NFP_NET_CFG_VLAN_FILTER_PROTO:	VLAN proto to filter
- * @NFP_NET_CFG_VXLAN_SZ:		Size of the VLAN filter mailbox in bytes
+ * %NFP_NET_CFG_VLAN_FILTER:		Base address of VLAN filter mailbox
+ * %NFP_NET_CFG_VLAN_FILTER_VID:	VLAN ID to filter
+ * %NFP_NET_CFG_VLAN_FILTER_PROTO:	VLAN proto to filter
+ * %NFP_NET_CFG_VXLAN_SZ:		Size of the VLAN filter mailbox in bytes
  */
-#define NFP_NET_CFG_VLAN_FILTER		NFP_NET_CFG_MBOX_VAL
+#define NFP_NET_CFG_VLAN_FILTER		NFP_NET_CFG_MBOX_SIMPLE_VAL
 #define  NFP_NET_CFG_VLAN_FILTER_VID	NFP_NET_CFG_VLAN_FILTER
 #define  NFP_NET_CFG_VLAN_FILTER_PROTO	 (NFP_NET_CFG_VLAN_FILTER + 2)
 #define NFP_NET_CFG_VLAN_FILTER_SZ	 0x0004
 
+/**
+ * TLV capabilities
+ * %NFP_NET_CFG_TLV_TYPE:	Offset of type within the TLV
+ * %NFP_NET_CFG_TLV_TYPE_REQUIRED: Driver must be able to parse the TLV
+ * %NFP_NET_CFG_TLV_LENGTH:	Offset of length within the TLV
+ * %NFP_NET_CFG_TLV_LENGTH_INC:	TLV length increments
+ * %NFP_NET_CFG_TLV_VALUE:	Offset of value with the TLV
+ *
+ * List of simple TLV structures, first one starts at %NFP_NET_CFG_TLV_BASE.
+ * Last structure must be of type %NFP_NET_CFG_TLV_TYPE_END.  Presence of TLVs
+ * is indicated by %NFP_NET_CFG_TLV_BASE being non-zero.  TLV structures may
+ * fill the entire remainder of the BAR or be shorter.  FW must make sure TLVs
+ * don't conflict with other features which allocate space beyond
+ * %NFP_NET_CFG_TLV_BASE.  %NFP_NET_CFG_TLV_TYPE_RESERVED should be used to wrap
+ * space used by such features.
+ * Note that the 4 byte TLV header is not counted in %NFP_NET_CFG_TLV_LENGTH.
+ */
+#define NFP_NET_CFG_TLV_TYPE		0x00
+#define   NFP_NET_CFG_TLV_TYPE_REQUIRED	  0x8000
+#define NFP_NET_CFG_TLV_LENGTH		0x02
+#define   NFP_NET_CFG_TLV_LENGTH_INC	  4
+#define NFP_NET_CFG_TLV_VALUE		0x04
+
+#define NFP_NET_CFG_TLV_HEADER_REQUIRED	0x80000000
+#define NFP_NET_CFG_TLV_HEADER_TYPE	0x7fff0000
+#define NFP_NET_CFG_TLV_HEADER_LENGTH	0x0000ffff
+
+/**
+ * Capability TLV types
+ *
+ * %NFP_NET_CFG_TLV_TYPE_UNKNOWN:
+ * Special TLV type to catch bugs, should never be encountered.  Drivers should
+ * treat encountering this type as error and refuse to probe.
+ *
+ * %NFP_NET_CFG_TLV_TYPE_RESERVED:
+ * Reserved space, may contain legacy fixed-offset fields, or be used for
+ * padding.  The use of this type should be otherwise avoided.
+ *
+ * %NFP_NET_CFG_TLV_TYPE_END:
+ * Empty, end of TLV list.  Must be the last TLV.  Drivers will stop processing
+ * further TLVs when encountered.
+ *
+ * %NFP_NET_CFG_TLV_TYPE_ME_FREQ:
+ * Single word, ME frequency in MHz as used in calculation for
+ * %NFP_NET_CFG_RXR_IRQ_MOD and %NFP_NET_CFG_TXR_IRQ_MOD.
+ *
+ * %NFP_NET_CFG_TLV_TYPE_MBOX:
+ * Variable, mailbox area.  Overwrites the default location which is
+ * %NFP_NET_CFG_MBOX_BASE and length %NFP_NET_CFG_MBOX_VAL_MAX_SZ.
+ */
+#define NFP_NET_CFG_TLV_TYPE_UNKNOWN		0
+#define NFP_NET_CFG_TLV_TYPE_RESERVED		1
+#define NFP_NET_CFG_TLV_TYPE_END		2
+#define NFP_NET_CFG_TLV_TYPE_ME_FREQ		3
+#define NFP_NET_CFG_TLV_TYPE_MBOX		4
+
+struct device;
+
+/**
+ * struct nfp_net_tlv_caps - parsed control BAR TLV capabilities
+ * @me_freq_mhz:	ME clock_freq (MHz)
+ * @mbox_off:		vNIC mailbox area offset
+ * @mbox_len:		vNIC mailbox area length
+ */
+struct nfp_net_tlv_caps {
+	u32 me_freq_mhz;
+	unsigned int mbox_off;
+	unsigned int mbox_len;
+};
+
+int nfp_net_tlv_caps_parse(struct device *dev, u8 __iomem *ctrl_mem,
+			   struct nfp_net_tlv_caps *caps);
+
+static inline bool nfp_net_has_mbox(struct nfp_net_tlv_caps *caps)
+{
+	return caps->mbox_len >= NFP_NET_CFG_MBOX_SIMPLE_LEN;
+}
+
 #endif /* _NFP_NET_CTRL_H_ */
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_debugdump.c b/drivers/net/ethernet/netronome/nfp/nfp_net_debugdump.c
new file mode 100644
index 000000000000..bb8ed460086e
--- /dev/null
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_debugdump.c
@@ -0,0 +1,811 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is dual licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree or the BSD 2-Clause License provided below.  You have the
+ * option to license this software under the complete terms of either license.
+ *
+ * The BSD 2-Clause License:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      1. Redistributions of source code must retain the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer.
+ *
+ *      2. Redistributions in binary form must reproduce the above
+ *         copyright notice, this list of conditions and the following
+ *         disclaimer in the documentation and/or other materials
+ *         provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/ethtool.h>
+#include <linux/vmalloc.h>
+
+#include "nfp_asm.h"
+#include "nfp_main.h"
+#include "nfpcore/nfp.h"
+#include "nfpcore/nfp_nffw.h"
+#include "nfpcore/nfp6000/nfp6000.h"
+
+#define NFP_DUMP_SPEC_RTSYM	"_abi_dump_spec"
+
+#define ALIGN8(x)	ALIGN(x, 8)
+
+enum nfp_dumpspec_type {
+	NFP_DUMPSPEC_TYPE_CPP_CSR = 0,
+	NFP_DUMPSPEC_TYPE_XPB_CSR = 1,
+	NFP_DUMPSPEC_TYPE_ME_CSR = 2,
+	NFP_DUMPSPEC_TYPE_INDIRECT_ME_CSR = 3,
+	NFP_DUMPSPEC_TYPE_RTSYM = 4,
+	NFP_DUMPSPEC_TYPE_HWINFO = 5,
+	NFP_DUMPSPEC_TYPE_FWNAME = 6,
+	NFP_DUMPSPEC_TYPE_HWINFO_FIELD = 7,
+	NFP_DUMPSPEC_TYPE_PROLOG = 10000,
+	NFP_DUMPSPEC_TYPE_ERROR = 10001,
+};
+
+/* The following structs must be carefully aligned so that they can be used to
+ * interpret the binary dumpspec and populate the dump data in a deterministic
+ * way.
+ */
+
+/* generic type plus length */
+struct nfp_dump_tl {
+	__be32 type;
+	__be32 length;	/* chunk length to follow, aligned to 8 bytes */
+	char data[0];
+};
+
+/* NFP CPP parameters */
+struct nfp_dumpspec_cpp_isl_id {
+	u8 target;
+	u8 action;
+	u8 token;
+	u8 island;
+};
+
+struct nfp_dump_common_cpp {
+	struct nfp_dumpspec_cpp_isl_id cpp_id;
+	__be32 offset;		/* address to start dump */
+	__be32 dump_length;	/* total bytes to dump, aligned to reg size */
+};
+
+/* CSR dumpables */
+struct nfp_dumpspec_csr {
+	struct nfp_dump_tl tl;
+	struct nfp_dump_common_cpp cpp;
+	__be32 register_width;	/* in bits */
+};
+
+struct nfp_dumpspec_rtsym {
+	struct nfp_dump_tl tl;
+	char rtsym[0];
+};
+
+/* header for register dumpable */
+struct nfp_dump_csr {
+	struct nfp_dump_tl tl;
+	struct nfp_dump_common_cpp cpp;
+	__be32 register_width;	/* in bits */
+	__be32 error;		/* error code encountered while reading */
+	__be32 error_offset;	/* offset being read when error occurred */
+};
+
+struct nfp_dump_rtsym {
+	struct nfp_dump_tl tl;
+	struct nfp_dump_common_cpp cpp;
+	__be32 error;		/* error code encountered while reading */
+	u8 padded_name_length;	/* pad so data starts at 8 byte boundary */
+	char rtsym[0];
+	/* after padded_name_length, there is dump_length data */
+};
+
+struct nfp_dump_prolog {
+	struct nfp_dump_tl tl;
+	__be32 dump_level;
+};
+
+struct nfp_dump_error {
+	struct nfp_dump_tl tl;
+	__be32 error;
+	char padding[4];
+	char spec[0];
+};
+
+/* to track state through debug size calculation TLV traversal */
+struct nfp_level_size {
+	__be32 requested_level;	/* input */
+	u32 total_size;		/* output */
+};
+
+/* to track state during debug dump creation TLV traversal */
+struct nfp_dump_state {
+	__be32 requested_level;	/* input param */
+	u32 dumped_size;	/* adds up to size of dumped data */
+	u32 buf_size;		/* size of buffer pointer to by p */
+	void *p;		/* current point in dump buffer */
+};
+
+typedef int (*nfp_tlv_visit)(struct nfp_pf *pf, struct nfp_dump_tl *tl,
+			     void *param);
+
+static int
+nfp_traverse_tlvs(struct nfp_pf *pf, void *data, u32 data_length, void *param,
+		  nfp_tlv_visit tlv_visit)
+{
+	long long remaining = data_length;
+	struct nfp_dump_tl *tl;
+	u32 total_tlv_size;
+	void *p = data;
+	int err;
+
+	while (remaining >= sizeof(*tl)) {
+		tl = p;
+		if (!tl->type && !tl->length)
+			break;
+
+		if (be32_to_cpu(tl->length) > remaining - sizeof(*tl))
+			return -EINVAL;
+
+		total_tlv_size = sizeof(*tl) + be32_to_cpu(tl->length);
+
+		/* Spec TLVs should be aligned to 4 bytes. */
+		if (total_tlv_size % 4 != 0)
+			return -EINVAL;
+
+		p += total_tlv_size;
+		remaining -= total_tlv_size;
+		err = tlv_visit(pf, tl, param);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static u32 nfp_get_numeric_cpp_id(struct nfp_dumpspec_cpp_isl_id *cpp_id)
+{
+	return NFP_CPP_ISLAND_ID(cpp_id->target, cpp_id->action, cpp_id->token,
+				 cpp_id->island);
+}
+
+struct nfp_dumpspec *
+nfp_net_dump_load_dumpspec(struct nfp_cpp *cpp, struct nfp_rtsym_table *rtbl)
+{
+	const struct nfp_rtsym *specsym;
+	struct nfp_dumpspec *dumpspec;
+	int bytes_read;
+	u32 cpp_id;
+
+	specsym = nfp_rtsym_lookup(rtbl, NFP_DUMP_SPEC_RTSYM);
+	if (!specsym)
+		return NULL;
+
+	/* expected size of this buffer is in the order of tens of kilobytes */
+	dumpspec = vmalloc(sizeof(*dumpspec) + specsym->size);
+	if (!dumpspec)
+		return NULL;
+
+	dumpspec->size = specsym->size;
+
+	cpp_id = NFP_CPP_ISLAND_ID(specsym->target, NFP_CPP_ACTION_RW, 0,
+				   specsym->domain);
+
+	bytes_read = nfp_cpp_read(cpp, cpp_id, specsym->addr, dumpspec->data,
+				  specsym->size);
+	if (bytes_read != specsym->size) {
+		vfree(dumpspec);
+		nfp_warn(cpp, "Debug dump specification read failed.\n");
+		return NULL;
+	}
+
+	return dumpspec;
+}
+
+static int nfp_dump_error_tlv_size(struct nfp_dump_tl *spec)
+{
+	return ALIGN8(sizeof(struct nfp_dump_error) + sizeof(*spec) +
+		      be32_to_cpu(spec->length));
+}
+
+static int nfp_calc_fwname_tlv_size(struct nfp_pf *pf)
+{
+	u32 fwname_len = strlen(nfp_mip_name(pf->mip));
+
+	return sizeof(struct nfp_dump_tl) + ALIGN8(fwname_len + 1);
+}
+
+static int nfp_calc_hwinfo_field_sz(struct nfp_pf *pf, struct nfp_dump_tl *spec)
+{
+	u32 tl_len, key_len;
+	const char *value;
+
+	tl_len = be32_to_cpu(spec->length);
+	key_len = strnlen(spec->data, tl_len);
+	if (key_len == tl_len)
+		return nfp_dump_error_tlv_size(spec);
+
+	value = nfp_hwinfo_lookup(pf->hwinfo, spec->data);
+	if (!value)
+		return nfp_dump_error_tlv_size(spec);
+
+	return sizeof(struct nfp_dump_tl) + ALIGN8(key_len + strlen(value) + 2);
+}
+
+static bool nfp_csr_spec_valid(struct nfp_dumpspec_csr *spec_csr)
+{
+	u32 required_read_sz = sizeof(*spec_csr) - sizeof(spec_csr->tl);
+	u32 available_sz = be32_to_cpu(spec_csr->tl.length);
+	u32 reg_width;
+
+	if (available_sz < required_read_sz)
+		return false;
+
+	reg_width = be32_to_cpu(spec_csr->register_width);
+
+	return reg_width == 32 || reg_width == 64;
+}
+
+static int
+nfp_calc_rtsym_dump_sz(struct nfp_pf *pf, struct nfp_dump_tl *spec)
+{
+	struct nfp_rtsym_table *rtbl = pf->rtbl;
+	struct nfp_dumpspec_rtsym *spec_rtsym;
+	const struct nfp_rtsym *sym;
+	u32 tl_len, key_len;
+	u32 size;
+
+	spec_rtsym = (struct nfp_dumpspec_rtsym *)spec;
+	tl_len = be32_to_cpu(spec->length);
+	key_len = strnlen(spec_rtsym->rtsym, tl_len);
+	if (key_len == tl_len)
+		return nfp_dump_error_tlv_size(spec);
+
+	sym = nfp_rtsym_lookup(rtbl, spec_rtsym->rtsym);
+	if (!sym)
+		return nfp_dump_error_tlv_size(spec);
+
+	if (sym->type == NFP_RTSYM_TYPE_ABS)
+		size = sizeof(sym->addr);
+	else
+		size = sym->size;
+
+	return ALIGN8(offsetof(struct nfp_dump_rtsym, rtsym) + key_len + 1) +
+	       ALIGN8(size);
+}
+
+static int
+nfp_add_tlv_size(struct nfp_pf *pf, struct nfp_dump_tl *tl, void *param)
+{
+	struct nfp_dumpspec_csr *spec_csr;
+	u32 *size = param;
+	u32 hwinfo_size;
+
+	switch (be32_to_cpu(tl->type)) {
+	case NFP_DUMPSPEC_TYPE_FWNAME:
+		*size += nfp_calc_fwname_tlv_size(pf);
+		break;
+	case NFP_DUMPSPEC_TYPE_CPP_CSR:
+	case NFP_DUMPSPEC_TYPE_XPB_CSR:
+	case NFP_DUMPSPEC_TYPE_ME_CSR:
+		spec_csr = (struct nfp_dumpspec_csr *)tl;
+		if (!nfp_csr_spec_valid(spec_csr))
+			*size += nfp_dump_error_tlv_size(tl);
+		else
+			*size += ALIGN8(sizeof(struct nfp_dump_csr)) +
+				 ALIGN8(be32_to_cpu(spec_csr->cpp.dump_length));
+		break;
+	case NFP_DUMPSPEC_TYPE_INDIRECT_ME_CSR:
+		spec_csr = (struct nfp_dumpspec_csr *)tl;
+		if (!nfp_csr_spec_valid(spec_csr))
+			*size += nfp_dump_error_tlv_size(tl);
+		else
+			*size += ALIGN8(sizeof(struct nfp_dump_csr)) +
+				 ALIGN8(be32_to_cpu(spec_csr->cpp.dump_length) *
+					NFP_IND_NUM_CONTEXTS);
+		break;
+	case NFP_DUMPSPEC_TYPE_RTSYM:
+		*size += nfp_calc_rtsym_dump_sz(pf, tl);
+		break;
+	case NFP_DUMPSPEC_TYPE_HWINFO:
+		hwinfo_size = nfp_hwinfo_get_packed_str_size(pf->hwinfo);
+		*size += sizeof(struct nfp_dump_tl) + ALIGN8(hwinfo_size);
+		break;
+	case NFP_DUMPSPEC_TYPE_HWINFO_FIELD:
+		*size += nfp_calc_hwinfo_field_sz(pf, tl);
+		break;
+	default:
+		*size += nfp_dump_error_tlv_size(tl);
+		break;
+	}
+
+	return 0;
+}
+
+static int
+nfp_calc_specific_level_size(struct nfp_pf *pf, struct nfp_dump_tl *dump_level,
+			     void *param)
+{
+	struct nfp_level_size *lev_sz = param;
+
+	if (dump_level->type != lev_sz->requested_level)
+		return 0;
+
+	return nfp_traverse_tlvs(pf, dump_level->data,
+				 be32_to_cpu(dump_level->length),
+				 &lev_sz->total_size, nfp_add_tlv_size);
+}
+
+s64 nfp_net_dump_calculate_size(struct nfp_pf *pf, struct nfp_dumpspec *spec,
+				u32 flag)
+{
+	struct nfp_level_size lev_sz;
+	int err;
+
+	lev_sz.requested_level = cpu_to_be32(flag);
+	lev_sz.total_size = ALIGN8(sizeof(struct nfp_dump_prolog));
+
+	err = nfp_traverse_tlvs(pf, spec->data, spec->size, &lev_sz,
+				nfp_calc_specific_level_size);
+	if (err)
+		return err;
+
+	return lev_sz.total_size;
+}
+
+static int nfp_add_tlv(u32 type, u32 total_tlv_sz, struct nfp_dump_state *dump)
+{
+	struct nfp_dump_tl *tl = dump->p;
+
+	if (total_tlv_sz > dump->buf_size)
+		return -ENOSPC;
+
+	if (dump->buf_size - total_tlv_sz < dump->dumped_size)
+		return -ENOSPC;
+
+	tl->type = cpu_to_be32(type);
+	tl->length = cpu_to_be32(total_tlv_sz - sizeof(*tl));
+
+	dump->dumped_size += total_tlv_sz;
+	dump->p += total_tlv_sz;
+
+	return 0;
+}
+
+static int
+nfp_dump_error_tlv(struct nfp_dump_tl *spec, int error,
+		   struct nfp_dump_state *dump)
+{
+	struct nfp_dump_error *dump_header = dump->p;
+	u32 total_spec_size, total_size;
+	int err;
+
+	total_spec_size = sizeof(*spec) + be32_to_cpu(spec->length);
+	total_size = ALIGN8(sizeof(*dump_header) + total_spec_size);
+
+	err = nfp_add_tlv(NFP_DUMPSPEC_TYPE_ERROR, total_size, dump);
+	if (err)
+		return err;
+
+	dump_header->error = cpu_to_be32(error);
+	memcpy(dump_header->spec, spec, total_spec_size);
+
+	return 0;
+}
+
+static int nfp_dump_fwname(struct nfp_pf *pf, struct nfp_dump_state *dump)
+{
+	struct nfp_dump_tl *dump_header = dump->p;
+	u32 fwname_len, total_size;
+	const char *fwname;
+	int err;
+
+	fwname = nfp_mip_name(pf->mip);
+	fwname_len = strlen(fwname);
+	total_size = sizeof(*dump_header) + ALIGN8(fwname_len + 1);
+
+	err = nfp_add_tlv(NFP_DUMPSPEC_TYPE_FWNAME, total_size, dump);
+	if (err)
+		return err;
+
+	memcpy(dump_header->data, fwname, fwname_len);
+
+	return 0;
+}
+
+static int
+nfp_dump_hwinfo(struct nfp_pf *pf, struct nfp_dump_tl *spec,
+		struct nfp_dump_state *dump)
+{
+	struct nfp_dump_tl *dump_header = dump->p;
+	u32 hwinfo_size, total_size;
+	char *hwinfo;
+	int err;
+
+	hwinfo = nfp_hwinfo_get_packed_strings(pf->hwinfo);
+	hwinfo_size = nfp_hwinfo_get_packed_str_size(pf->hwinfo);
+	total_size = sizeof(*dump_header) + ALIGN8(hwinfo_size);
+
+	err = nfp_add_tlv(NFP_DUMPSPEC_TYPE_HWINFO, total_size, dump);
+	if (err)
+		return err;
+
+	memcpy(dump_header->data, hwinfo, hwinfo_size);
+
+	return 0;
+}
+
+static int nfp_dump_hwinfo_field(struct nfp_pf *pf, struct nfp_dump_tl *spec,
+				 struct nfp_dump_state *dump)
+{
+	struct nfp_dump_tl *dump_header = dump->p;
+	u32 tl_len, key_len, val_len;
+	const char *key, *value;
+	u32 total_size;
+	int err;
+
+	tl_len = be32_to_cpu(spec->length);
+	key_len = strnlen(spec->data, tl_len);
+	if (key_len == tl_len)
+		return nfp_dump_error_tlv(spec, -EINVAL, dump);
+
+	key = spec->data;
+	value = nfp_hwinfo_lookup(pf->hwinfo, key);
+	if (!value)
+		return nfp_dump_error_tlv(spec, -ENOENT, dump);
+
+	val_len = strlen(value);
+	total_size = sizeof(*dump_header) + ALIGN8(key_len + val_len + 2);
+	err = nfp_add_tlv(NFP_DUMPSPEC_TYPE_HWINFO_FIELD, total_size, dump);
+	if (err)
+		return err;
+
+	memcpy(dump_header->data, key, key_len + 1);
+	memcpy(dump_header->data + key_len + 1, value, val_len + 1);
+
+	return 0;
+}
+
+static bool is_xpb_read(struct nfp_dumpspec_cpp_isl_id *cpp_id)
+{
+	return cpp_id->target == NFP_CPP_TARGET_ISLAND_XPB &&
+	       cpp_id->action == 0 && cpp_id->token == 0;
+}
+
+static int
+nfp_dump_csr_range(struct nfp_pf *pf, struct nfp_dumpspec_csr *spec_csr,
+		   struct nfp_dump_state *dump)
+{
+	struct nfp_dump_csr *dump_header = dump->p;
+	u32 reg_sz, header_size, total_size;
+	u32 cpp_rd_addr, max_rd_addr;
+	int bytes_read;
+	void *dest;
+	u32 cpp_id;
+	int err;
+
+	if (!nfp_csr_spec_valid(spec_csr))
+		return nfp_dump_error_tlv(&spec_csr->tl, -EINVAL, dump);
+
+	reg_sz = be32_to_cpu(spec_csr->register_width) / BITS_PER_BYTE;
+	header_size = ALIGN8(sizeof(*dump_header));
+	total_size = header_size +
+		     ALIGN8(be32_to_cpu(spec_csr->cpp.dump_length));
+	dest = dump->p + header_size;
+
+	err = nfp_add_tlv(be32_to_cpu(spec_csr->tl.type), total_size, dump);
+	if (err)
+		return err;
+
+	dump_header->cpp = spec_csr->cpp;
+	dump_header->register_width = spec_csr->register_width;
+
+	cpp_id = nfp_get_numeric_cpp_id(&spec_csr->cpp.cpp_id);
+	cpp_rd_addr = be32_to_cpu(spec_csr->cpp.offset);
+	max_rd_addr = cpp_rd_addr + be32_to_cpu(spec_csr->cpp.dump_length);
+
+	while (cpp_rd_addr < max_rd_addr) {
+		if (is_xpb_read(&spec_csr->cpp.cpp_id)) {
+			err = nfp_xpb_readl(pf->cpp, cpp_rd_addr, (u32 *)dest);
+		} else {
+			bytes_read = nfp_cpp_read(pf->cpp, cpp_id, cpp_rd_addr,
+						  dest, reg_sz);
+			err = bytes_read == reg_sz ? 0 : -EIO;
+		}
+		if (err) {
+			dump_header->error = cpu_to_be32(err);
+			dump_header->error_offset = cpu_to_be32(cpp_rd_addr);
+			break;
+		}
+		cpp_rd_addr += reg_sz;
+		dest += reg_sz;
+	}
+
+	return 0;
+}
+
+/* Write context to CSRCtxPtr, then read from it. Then the value can be read
+ * from IndCtxStatus.
+ */
+static int
+nfp_read_indirect_csr(struct nfp_cpp *cpp,
+		      struct nfp_dumpspec_cpp_isl_id cpp_params, u32 offset,
+		      u32 reg_sz, u32 context, void *dest)
+{
+	u32 csr_ctx_ptr_offs;
+	u32 cpp_id;
+	int result;
+
+	csr_ctx_ptr_offs = nfp_get_ind_csr_ctx_ptr_offs(offset);
+	cpp_id = NFP_CPP_ISLAND_ID(cpp_params.target,
+				   NFP_IND_ME_REFL_WR_SIG_INIT,
+				   cpp_params.token, cpp_params.island);
+	result = nfp_cpp_writel(cpp, cpp_id, csr_ctx_ptr_offs, context);
+	if (result)
+		return result;
+
+	cpp_id = nfp_get_numeric_cpp_id(&cpp_params);
+	result = nfp_cpp_read(cpp, cpp_id, csr_ctx_ptr_offs, dest, reg_sz);
+	if (result != reg_sz)
+		return result < 0 ? result : -EIO;
+
+	result = nfp_cpp_read(cpp, cpp_id, offset, dest, reg_sz);
+	if (result != reg_sz)
+		return result < 0 ? result : -EIO;
+
+	return 0;
+}
+
+static int
+nfp_read_all_indirect_csr_ctx(struct nfp_cpp *cpp,
+			      struct nfp_dumpspec_csr *spec_csr, u32 address,
+			      u32 reg_sz, void *dest)
+{
+	u32 ctx;
+	int err;
+
+	for (ctx = 0; ctx < NFP_IND_NUM_CONTEXTS; ctx++) {
+		err = nfp_read_indirect_csr(cpp, spec_csr->cpp.cpp_id, address,
+					    reg_sz, ctx, dest + ctx * reg_sz);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int
+nfp_dump_indirect_csr_range(struct nfp_pf *pf,
+			    struct nfp_dumpspec_csr *spec_csr,
+			    struct nfp_dump_state *dump)
+{
+	struct nfp_dump_csr *dump_header = dump->p;
+	u32 reg_sz, header_size, total_size;
+	u32 cpp_rd_addr, max_rd_addr;
+	u32 reg_data_length;
+	void *dest;
+	int err;
+
+	if (!nfp_csr_spec_valid(spec_csr))
+		return nfp_dump_error_tlv(&spec_csr->tl, -EINVAL, dump);
+
+	reg_sz = be32_to_cpu(spec_csr->register_width) / BITS_PER_BYTE;
+	header_size = ALIGN8(sizeof(*dump_header));
+	reg_data_length = be32_to_cpu(spec_csr->cpp.dump_length) *
+			  NFP_IND_NUM_CONTEXTS;
+	total_size = header_size + ALIGN8(reg_data_length);
+	dest = dump->p + header_size;
+
+	err = nfp_add_tlv(be32_to_cpu(spec_csr->tl.type), total_size, dump);
+	if (err)
+		return err;
+
+	dump_header->cpp = spec_csr->cpp;
+	dump_header->register_width = spec_csr->register_width;
+
+	cpp_rd_addr = be32_to_cpu(spec_csr->cpp.offset);
+	max_rd_addr = cpp_rd_addr + be32_to_cpu(spec_csr->cpp.dump_length);
+	while (cpp_rd_addr < max_rd_addr) {
+		err = nfp_read_all_indirect_csr_ctx(pf->cpp, spec_csr,
+						    cpp_rd_addr, reg_sz, dest);
+		if (err) {
+			dump_header->error = cpu_to_be32(err);
+			dump_header->error_offset = cpu_to_be32(cpp_rd_addr);
+			break;
+		}
+		cpp_rd_addr += reg_sz;
+		dest += reg_sz * NFP_IND_NUM_CONTEXTS;
+	}
+
+	return 0;
+}
+
+static int
+nfp_dump_single_rtsym(struct nfp_pf *pf, struct nfp_dumpspec_rtsym *spec,
+		      struct nfp_dump_state *dump)
+{
+	struct nfp_dump_rtsym *dump_header = dump->p;
+	struct nfp_dumpspec_cpp_isl_id cpp_params;
+	struct nfp_rtsym_table *rtbl = pf->rtbl;
+	u32 header_size, total_size, sym_size;
+	const struct nfp_rtsym *sym;
+	u32 tl_len, key_len;
+	int bytes_read;
+	u32 cpp_id;
+	void *dest;
+	int err;
+
+	tl_len = be32_to_cpu(spec->tl.length);
+	key_len = strnlen(spec->rtsym, tl_len);
+	if (key_len == tl_len)
+		return nfp_dump_error_tlv(&spec->tl, -EINVAL, dump);
+
+	sym = nfp_rtsym_lookup(rtbl, spec->rtsym);
+	if (!sym)
+		return nfp_dump_error_tlv(&spec->tl, -ENOENT, dump);
+
+	if (sym->type == NFP_RTSYM_TYPE_ABS)
+		sym_size = sizeof(sym->addr);
+	else
+		sym_size = sym->size;
+
+	header_size =
+		ALIGN8(offsetof(struct nfp_dump_rtsym, rtsym) + key_len + 1);
+	total_size = header_size + ALIGN8(sym_size);
+	dest = dump->p + header_size;
+
+	err = nfp_add_tlv(be32_to_cpu(spec->tl.type), total_size, dump);
+	if (err)
+		return err;
+
+	dump_header->padded_name_length =
+		header_size - offsetof(struct nfp_dump_rtsym, rtsym);
+	memcpy(dump_header->rtsym, spec->rtsym, key_len + 1);
+	dump_header->cpp.dump_length = cpu_to_be32(sym_size);
+
+	if (sym->type == NFP_RTSYM_TYPE_ABS) {
+		*(u64 *)dest = sym->addr;
+	} else {
+		cpp_params.target = sym->target;
+		cpp_params.action = NFP_CPP_ACTION_RW;
+		cpp_params.token  = 0;
+		cpp_params.island = sym->domain;
+		cpp_id = nfp_get_numeric_cpp_id(&cpp_params);
+		dump_header->cpp.cpp_id = cpp_params;
+		dump_header->cpp.offset = cpu_to_be32(sym->addr);
+		bytes_read = nfp_cpp_read(pf->cpp, cpp_id, sym->addr, dest,
+					  sym_size);
+		if (bytes_read != sym_size) {
+			if (bytes_read >= 0)
+				bytes_read = -EIO;
+			dump_header->error = cpu_to_be32(bytes_read);
+		}
+	}
+
+	return 0;
+}
+
+static int
+nfp_dump_for_tlv(struct nfp_pf *pf, struct nfp_dump_tl *tl, void *param)
+{
+	struct nfp_dumpspec_rtsym *spec_rtsym;
+	struct nfp_dump_state *dump = param;
+	struct nfp_dumpspec_csr *spec_csr;
+	int err;
+
+	switch (be32_to_cpu(tl->type)) {
+	case NFP_DUMPSPEC_TYPE_FWNAME:
+		err = nfp_dump_fwname(pf, dump);
+		if (err)
+			return err;
+		break;
+	case NFP_DUMPSPEC_TYPE_CPP_CSR:
+	case NFP_DUMPSPEC_TYPE_XPB_CSR:
+	case NFP_DUMPSPEC_TYPE_ME_CSR:
+		spec_csr = (struct nfp_dumpspec_csr *)tl;
+		err = nfp_dump_csr_range(pf, spec_csr, dump);
+		if (err)
+			return err;
+		break;
+	case NFP_DUMPSPEC_TYPE_INDIRECT_ME_CSR:
+		spec_csr = (struct nfp_dumpspec_csr *)tl;
+		err = nfp_dump_indirect_csr_range(pf, spec_csr, dump);
+		if (err)
+			return err;
+		break;
+	case NFP_DUMPSPEC_TYPE_RTSYM:
+		spec_rtsym = (struct nfp_dumpspec_rtsym *)tl;
+		err = nfp_dump_single_rtsym(pf, spec_rtsym, dump);
+		if (err)
+			return err;
+		break;
+	case NFP_DUMPSPEC_TYPE_HWINFO:
+		err = nfp_dump_hwinfo(pf, tl, dump);
+		if (err)
+			return err;
+		break;
+	case NFP_DUMPSPEC_TYPE_HWINFO_FIELD:
+		err = nfp_dump_hwinfo_field(pf, tl, dump);
+		if (err)
+			return err;
+		break;
+	default:
+		err = nfp_dump_error_tlv(tl, -EOPNOTSUPP, dump);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int
+nfp_dump_specific_level(struct nfp_pf *pf, struct nfp_dump_tl *dump_level,
+			void *param)
+{
+	struct nfp_dump_state *dump = param;
+
+	if (dump_level->type != dump->requested_level)
+		return 0;
+
+	return nfp_traverse_tlvs(pf, dump_level->data,
+				 be32_to_cpu(dump_level->length), dump,
+				 nfp_dump_for_tlv);
+}
+
+static int nfp_dump_populate_prolog(struct nfp_dump_state *dump)
+{
+	struct nfp_dump_prolog *prolog = dump->p;
+	u32 total_size;
+	int err;
+
+	total_size = ALIGN8(sizeof(*prolog));
+
+	err = nfp_add_tlv(NFP_DUMPSPEC_TYPE_PROLOG, total_size, dump);
+	if (err)
+		return err;
+
+	prolog->dump_level = dump->requested_level;
+
+	return 0;
+}
+
+int nfp_net_dump_populate_buffer(struct nfp_pf *pf, struct nfp_dumpspec *spec,
+				 struct ethtool_dump *dump_param, void *dest)
+{
+	struct nfp_dump_state dump;
+	int err;
+
+	dump.requested_level = cpu_to_be32(dump_param->flag);
+	dump.dumped_size = 0;
+	dump.p = dest;
+	dump.buf_size = dump_param->len;
+
+	err = nfp_dump_populate_prolog(&dump);
+	if (err)
+		return err;
+
+	err = nfp_traverse_tlvs(pf, spec->data, spec->size, &dump,
+				nfp_dump_specific_level);
+	if (err)
+		return err;
+
+	/* Set size of actual dump, to trigger warning if different from
+	 * calculated size.
+	 */
+	dump_param->len = dump.dumped_size;
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
index 6c02b2d6ba06..e1dae0616f52 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
@@ -47,18 +47,16 @@
 #include <linux/interrupt.h>
 #include <linux/pci.h>
 #include <linux/ethtool.h>
+#include <linux/firmware.h>
 
 #include "nfpcore/nfp.h"
 #include "nfpcore/nfp_nsp.h"
 #include "nfp_app.h"
+#include "nfp_main.h"
 #include "nfp_net_ctrl.h"
 #include "nfp_net.h"
 #include "nfp_port.h"
 
-enum nfp_dump_diag {
-	NFP_DUMP_NSP_DIAG = 0,
-};
-
 struct nfp_et_stat {
 	char name[ETH_GSTRING_LEN];
 	int off;
@@ -1066,15 +1064,34 @@ nfp_dump_nsp_diag(struct nfp_app *app, struct ethtool_dump *dump, void *buffer)
 	return ret;
 }
 
+/* Set the dump flag/level. Calculate the dump length for flag > 0 only (new TLV
+ * based dumps), since flag 0 (default) calculates the length in
+ * nfp_app_get_dump_flag(), and we need to support triggering a level 0 dump
+ * without setting the flag first, for backward compatibility.
+ */
 static int nfp_app_set_dump(struct net_device *netdev, struct ethtool_dump *val)
 {
 	struct nfp_app *app = nfp_app_from_netdev(netdev);
+	s64 len;
 
 	if (!app)
 		return -EOPNOTSUPP;
 
-	if (val->flag != NFP_DUMP_NSP_DIAG)
-		return -EINVAL;
+	if (val->flag == NFP_DUMP_NSP_DIAG) {
+		app->pf->dump_flag = val->flag;
+		return 0;
+	}
+
+	if (!app->pf->dumpspec)
+		return -EOPNOTSUPP;
+
+	len = nfp_net_dump_calculate_size(app->pf, app->pf->dumpspec,
+					  val->flag);
+	if (len < 0)
+		return len;
+
+	app->pf->dump_flag = val->flag;
+	app->pf->dump_len = len;
 
 	return 0;
 }
@@ -1082,14 +1099,37 @@ static int nfp_app_set_dump(struct net_device *netdev, struct ethtool_dump *val)
 static int
 nfp_app_get_dump_flag(struct net_device *netdev, struct ethtool_dump *dump)
 {
-	return nfp_dump_nsp_diag(nfp_app_from_netdev(netdev), dump, NULL);
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
+
+	if (!app)
+		return -EOPNOTSUPP;
+
+	if (app->pf->dump_flag == NFP_DUMP_NSP_DIAG)
+		return nfp_dump_nsp_diag(app, dump, NULL);
+
+	dump->flag = app->pf->dump_flag;
+	dump->len = app->pf->dump_len;
+
+	return 0;
 }
 
 static int
 nfp_app_get_dump_data(struct net_device *netdev, struct ethtool_dump *dump,
 		      void *buffer)
 {
-	return nfp_dump_nsp_diag(nfp_app_from_netdev(netdev), dump, buffer);
+	struct nfp_app *app = nfp_app_from_netdev(netdev);
+
+	if (!app)
+		return -EOPNOTSUPP;
+
+	if (app->pf->dump_flag == NFP_DUMP_NSP_DIAG)
+		return nfp_dump_nsp_diag(app, dump, buffer);
+
+	dump->flag = app->pf->dump_flag;
+	dump->len = app->pf->dump_len;
+
+	return nfp_net_dump_populate_buffer(app->pf, app->pf->dumpspec, dump,
+					    buffer);
 }
 
 static int nfp_net_set_coalesce(struct net_device *netdev,
@@ -1230,6 +1270,57 @@ static int nfp_net_set_channels(struct net_device *netdev,
 	return nfp_net_set_num_rings(nn, total_rx, total_tx);
 }
 
+static int
+nfp_net_flash_device(struct net_device *netdev, struct ethtool_flash *flash)
+{
+	const struct firmware *fw;
+	struct nfp_app *app;
+	struct nfp_nsp *nsp;
+	struct device *dev;
+	int err;
+
+	if (flash->region != ETHTOOL_FLASH_ALL_REGIONS)
+		return -EOPNOTSUPP;
+
+	app = nfp_app_from_netdev(netdev);
+	if (!app)
+		return -EOPNOTSUPP;
+
+	dev = &app->pdev->dev;
+
+	nsp = nfp_nsp_open(app->cpp);
+	if (IS_ERR(nsp)) {
+		err = PTR_ERR(nsp);
+		dev_err(dev, "Failed to access the NSP: %d\n", err);
+		return err;
+	}
+
+	err = request_firmware_direct(&fw, flash->data, dev);
+	if (err)
+		goto exit_close_nsp;
+
+	dev_info(dev, "Please be patient while writing flash image: %s\n",
+		 flash->data);
+	dev_hold(netdev);
+	rtnl_unlock();
+
+	err = nfp_nsp_write_flash(nsp, fw);
+	if (err < 0) {
+		dev_err(dev, "Flash write failed: %d\n", err);
+		goto exit_rtnl_lock;
+	}
+	dev_info(dev, "Finished writing flash image\n");
+
+exit_rtnl_lock:
+	rtnl_lock();
+	dev_put(netdev);
+	release_firmware(fw);
+
+exit_close_nsp:
+	nfp_nsp_close(nsp);
+	return err;
+}
+
 static const struct ethtool_ops nfp_net_ethtool_ops = {
 	.get_drvinfo		= nfp_net_get_drvinfo,
 	.get_link		= ethtool_op_get_link,
@@ -1240,6 +1331,7 @@ static const struct ethtool_ops nfp_net_ethtool_ops = {
 	.get_sset_count		= nfp_net_get_sset_count,
 	.get_rxnfc		= nfp_net_get_rxnfc,
 	.set_rxnfc		= nfp_net_set_rxnfc,
+	.flash_device		= nfp_net_flash_device,
 	.get_rxfh_indir_size	= nfp_net_get_rxfh_indir_size,
 	.get_rxfh_key_size	= nfp_net_get_rxfh_key_size,
 	.get_rxfh		= nfp_net_get_rxfh,
@@ -1265,6 +1357,7 @@ const struct ethtool_ops nfp_port_ethtool_ops = {
 	.get_strings		= nfp_port_get_strings,
 	.get_ethtool_stats	= nfp_port_get_stats,
 	.get_sset_count		= nfp_port_get_sset_count,
+	.flash_device		= nfp_net_flash_device,
 	.set_dump		= nfp_app_set_dump,
 	.get_dump_flag		= nfp_app_get_dump_flag,
 	.get_dump_data		= nfp_app_get_dump_data,
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
index c505014121c4..15fa47f622aa 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
@@ -208,12 +208,6 @@ nfp_net_pf_init_vnic(struct nfp_pf *pf, struct nfp_net *nn, unsigned int id)
 {
 	int err;
 
-	/* Get ME clock frequency from ctrl BAR
-	 * XXX for now frequency is hardcoded until we figure out how
-	 * to get the value from nfp-hwinfo into ctrl bar
-	 */
-	nn->me_freq_mhz = 1200;
-
 	err = nfp_net_init(nn);
 	if (err)
 		return err;
@@ -373,7 +367,9 @@ nfp_net_pf_app_init(struct nfp_pf *pf, u8 __iomem *qc_bar, unsigned int stride)
 	if (IS_ERR(pf->app))
 		return PTR_ERR(pf->app);
 
+	mutex_lock(&pf->lock);
 	err = nfp_app_init(pf->app);
+	mutex_unlock(&pf->lock);
 	if (err)
 		goto err_free;
 
@@ -401,7 +397,9 @@ nfp_net_pf_app_init(struct nfp_pf *pf, u8 __iomem *qc_bar, unsigned int stride)
 err_unmap:
 	nfp_cpp_area_release_free(pf->ctrl_vnic_bar);
 err_app_clean:
+	mutex_lock(&pf->lock);
 	nfp_app_clean(pf->app);
+	mutex_unlock(&pf->lock);
 err_free:
 	nfp_app_free(pf->app);
 	pf->app = NULL;
@@ -414,7 +412,11 @@ static void nfp_net_pf_app_clean(struct nfp_pf *pf)
 		nfp_net_pf_free_vnic(pf, pf->ctrl_vnic);
 		nfp_cpp_area_release_free(pf->ctrl_vnic_bar);
 	}
+
+	mutex_lock(&pf->lock);
 	nfp_app_clean(pf->app);
+	mutex_unlock(&pf->lock);
+
 	nfp_app_free(pf->app);
 	pf->app = NULL;
 }
@@ -570,17 +572,6 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf)
 	return err;
 }
 
-static void nfp_net_pci_remove_finish(struct nfp_pf *pf)
-{
-	nfp_net_pf_app_stop(pf);
-	/* stop app first, to avoid double free of ctrl vNIC's ddir */
-	nfp_net_debugfs_dir_clean(&pf->ddir);
-
-	nfp_net_pf_free_irqs(pf);
-	nfp_net_pf_app_clean(pf);
-	nfp_net_pci_unmap_mem(pf);
-}
-
 static int
 nfp_net_eth_port_update(struct nfp_cpp *cpp, struct nfp_port *port,
 			struct nfp_eth_table *eth_table)
@@ -655,9 +646,6 @@ int nfp_net_refresh_port_table_sync(struct nfp_pf *pf)
 		nfp_net_pf_free_vnic(pf, nn);
 	}
 
-	if (list_empty(&pf->vnics))
-		nfp_net_pci_remove_finish(pf);
-
 	return 0;
 }
 
@@ -707,6 +695,7 @@ int nfp_net_refresh_eth_port(struct nfp_port *port)
  */
 int nfp_net_pci_probe(struct nfp_pf *pf)
 {
+	struct devlink *devlink = priv_to_devlink(pf);
 	struct nfp_net_fw_version fw_ver;
 	u8 __iomem *ctrl_bar, *qc_bar;
 	int stride;
@@ -720,16 +709,13 @@ int nfp_net_pci_probe(struct nfp_pf *pf)
 		return -EINVAL;
 	}
 
-	mutex_lock(&pf->lock);
 	pf->max_data_vnics = nfp_net_pf_get_num_ports(pf);
-	if ((int)pf->max_data_vnics < 0) {
-		err = pf->max_data_vnics;
-		goto err_unlock;
-	}
+	if ((int)pf->max_data_vnics < 0)
+		return pf->max_data_vnics;
 
 	err = nfp_net_pci_map_mem(pf);
 	if (err)
-		goto err_unlock;
+		return err;
 
 	ctrl_bar = nfp_cpp_area_iomem(pf->data_vnic_bar);
 	qc_bar = nfp_cpp_area_iomem(pf->qc_area);
@@ -768,6 +754,11 @@ int nfp_net_pci_probe(struct nfp_pf *pf)
 	if (err)
 		goto err_unmap;
 
+	err = devlink_register(devlink, &pf->pdev->dev);
+	if (err)
+		goto err_app_clean;
+
+	mutex_lock(&pf->lock);
 	pf->ddir = nfp_net_debugfs_device_add(pf->pdev);
 
 	/* Allocate the vnics and do basic init */
@@ -799,32 +790,39 @@ int nfp_net_pci_probe(struct nfp_pf *pf)
 	nfp_net_pf_free_vnics(pf);
 err_clean_ddir:
 	nfp_net_debugfs_dir_clean(&pf->ddir);
+	mutex_unlock(&pf->lock);
+	cancel_work_sync(&pf->port_refresh_work);
+	devlink_unregister(devlink);
+err_app_clean:
 	nfp_net_pf_app_clean(pf);
 err_unmap:
 	nfp_net_pci_unmap_mem(pf);
-err_unlock:
-	mutex_unlock(&pf->lock);
-	cancel_work_sync(&pf->port_refresh_work);
 	return err;
 }
 
 void nfp_net_pci_remove(struct nfp_pf *pf)
 {
-	struct nfp_net *nn;
+	struct nfp_net *nn, *next;
 
 	mutex_lock(&pf->lock);
-	if (list_empty(&pf->vnics))
-		goto out;
-
-	list_for_each_entry(nn, &pf->vnics, vnic_list)
-		if (nfp_net_is_data_vnic(nn))
-			nfp_net_pf_clean_vnic(pf, nn);
+	list_for_each_entry_safe(nn, next, &pf->vnics, vnic_list) {
+		if (!nfp_net_is_data_vnic(nn))
+			continue;
+		nfp_net_pf_clean_vnic(pf, nn);
+		nfp_net_pf_free_vnic(pf, nn);
+	}
 
-	nfp_net_pf_free_vnics(pf);
+	nfp_net_pf_app_stop(pf);
+	/* stop app first, to avoid double free of ctrl vNIC's ddir */
+	nfp_net_debugfs_dir_clean(&pf->ddir);
 
-	nfp_net_pci_remove_finish(pf);
-out:
 	mutex_unlock(&pf->lock);
 
+	devlink_unregister(priv_to_devlink(pf));
+
+	nfp_net_pf_free_irqs(pf);
+	nfp_net_pf_app_clean(pf);
+	nfp_net_pci_unmap_mem(pf);
+
 	cancel_work_sync(&pf->port_refresh_work);
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
index 78b36c67c232..619570524d2a 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
@@ -46,6 +46,13 @@
 #include "nfp_net_sriov.h"
 #include "nfp_port.h"
 
+struct net_device *
+nfp_repr_get_locked(struct nfp_app *app, struct nfp_reprs *set, unsigned int id)
+{
+	return rcu_dereference_protected(set->reprs[id],
+					 lockdep_is_held(&app->pf->lock));
+}
+
 static void
 nfp_repr_inc_tx_stats(struct net_device *netdev, unsigned int len,
 		      int tx_status)
@@ -186,6 +193,13 @@ nfp_repr_get_offload_stats(int attr_id, const struct net_device *dev,
 	return -EINVAL;
 }
 
+static int nfp_repr_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	struct nfp_repr *repr = netdev_priv(netdev);
+
+	return nfp_app_change_mtu(repr->app, netdev, new_mtu);
+}
+
 static netdev_tx_t nfp_repr_xmit(struct sk_buff *skb, struct net_device *netdev)
 {
 	struct nfp_repr *repr = netdev_priv(netdev);
@@ -240,6 +254,7 @@ const struct net_device_ops nfp_repr_netdev_ops = {
 	.ndo_open		= nfp_repr_open,
 	.ndo_stop		= nfp_repr_stop,
 	.ndo_start_xmit		= nfp_repr_xmit,
+	.ndo_change_mtu		= nfp_repr_change_mtu,
 	.ndo_get_stats64	= nfp_repr_get_stats64,
 	.ndo_has_offload_stats	= nfp_repr_has_offload_stats,
 	.ndo_get_offload_stats	= nfp_repr_get_offload_stats,
@@ -250,6 +265,7 @@ const struct net_device_ops nfp_repr_netdev_ops = {
 	.ndo_set_vf_spoofchk	= nfp_app_set_vf_spoofchk,
 	.ndo_get_vf_config	= nfp_app_get_vf_config,
 	.ndo_set_vf_link_state	= nfp_app_set_vf_link_state,
+	.ndo_set_features	= nfp_port_set_features,
 };
 
 static void nfp_repr_clean(struct nfp_repr *repr)
@@ -336,6 +352,8 @@ struct net_device *nfp_repr_alloc(struct nfp_app *app)
 	if (!netdev)
 		return NULL;
 
+	netif_carrier_off(netdev);
+
 	repr = netdev_priv(netdev);
 	repr->netdev = netdev;
 	repr->app = app;
@@ -359,29 +377,45 @@ static void nfp_repr_clean_and_free(struct nfp_repr *repr)
 	nfp_repr_free(repr);
 }
 
-void nfp_reprs_clean_and_free(struct nfp_reprs *reprs)
+void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs)
 {
+	struct net_device *netdev;
 	unsigned int i;
 
-	for (i = 0; i < reprs->num_reprs; i++)
-		if (reprs->reprs[i])
-			nfp_repr_clean_and_free(netdev_priv(reprs->reprs[i]));
+	for (i = 0; i < reprs->num_reprs; i++) {
+		netdev = nfp_repr_get_locked(app, reprs, i);
+		if (netdev)
+			nfp_repr_clean_and_free(netdev_priv(netdev));
+	}
 
 	kfree(reprs);
 }
 
 void
-nfp_reprs_clean_and_free_by_type(struct nfp_app *app,
-				 enum nfp_repr_type type)
+nfp_reprs_clean_and_free_by_type(struct nfp_app *app, enum nfp_repr_type type)
 {
+	struct net_device *netdev;
 	struct nfp_reprs *reprs;
+	int i;
 
-	reprs = nfp_app_reprs_set(app, type, NULL);
+	reprs = rcu_dereference_protected(app->reprs[type],
+					  lockdep_is_held(&app->pf->lock));
 	if (!reprs)
 		return;
 
+	/* Preclean must happen before we remove the reprs reference from the
+	 * app below.
+	 */
+	for (i = 0; i < reprs->num_reprs; i++) {
+		netdev = nfp_repr_get_locked(app, reprs, i);
+		if (netdev)
+			nfp_app_repr_preclean(app, netdev);
+	}
+
+	reprs = nfp_app_reprs_set(app, type, NULL);
+
 	synchronize_rcu();
-	nfp_reprs_clean_and_free(reprs);
+	nfp_reprs_clean_and_free(app, reprs);
 }
 
 struct nfp_reprs *nfp_reprs_alloc(unsigned int num_reprs)
@@ -399,47 +433,29 @@ struct nfp_reprs *nfp_reprs_alloc(unsigned int num_reprs)
 
 int nfp_reprs_resync_phys_ports(struct nfp_app *app)
 {
-	struct nfp_reprs *reprs, *old_reprs;
+	struct net_device *netdev;
+	struct nfp_reprs *reprs;
 	struct nfp_repr *repr;
 	int i;
 
-	old_reprs =
-		rcu_dereference_protected(app->reprs[NFP_REPR_TYPE_PHYS_PORT],
-					  lockdep_is_held(&app->pf->lock));
-	if (!old_reprs)
-		return 0;
-
-	reprs = nfp_reprs_alloc(old_reprs->num_reprs);
+	reprs = nfp_reprs_get_locked(app, NFP_REPR_TYPE_PHYS_PORT);
 	if (!reprs)
-		return -ENOMEM;
-
-	for (i = 0; i < old_reprs->num_reprs; i++) {
-		if (!old_reprs->reprs[i])
-			continue;
-
-		repr = netdev_priv(old_reprs->reprs[i]);
-		if (repr->port->type == NFP_PORT_INVALID)
-			continue;
-
-		reprs->reprs[i] = old_reprs->reprs[i];
-	}
-
-	old_reprs = nfp_app_reprs_set(app, NFP_REPR_TYPE_PHYS_PORT, reprs);
-	synchronize_rcu();
+		return 0;
 
-	/* Now we free up removed representors */
-	for (i = 0; i < old_reprs->num_reprs; i++) {
-		if (!old_reprs->reprs[i])
+	for (i = 0; i < reprs->num_reprs; i++) {
+		netdev = nfp_repr_get_locked(app, reprs, i);
+		if (!netdev)
 			continue;
 
-		repr = netdev_priv(old_reprs->reprs[i]);
+		repr = netdev_priv(netdev);
 		if (repr->port->type != NFP_PORT_INVALID)
 			continue;
 
-		nfp_app_repr_stop(app, repr);
+		nfp_app_repr_preclean(app, netdev);
+		rcu_assign_pointer(reprs->reprs[i], NULL);
+		synchronize_rcu();
 		nfp_repr_clean(repr);
 	}
 
-	kfree(old_reprs);
 	return 0;
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h
index 5d4d897bc9c6..a621e8ff528e 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h
@@ -35,6 +35,7 @@
 #define NFP_NET_REPR_H
 
 struct metadata_dst;
+struct nfp_app;
 struct nfp_net;
 struct nfp_port;
 
@@ -47,7 +48,7 @@ struct nfp_port;
  */
 struct nfp_reprs {
 	unsigned int num_reprs;
-	struct net_device *reprs[0];
+	struct net_device __rcu *reprs[0];
 };
 
 /**
@@ -89,6 +90,7 @@ struct nfp_repr {
  * @NFP_REPR_TYPE_PHYS_PORT:	external NIC port
  * @NFP_REPR_TYPE_PF:		physical function
  * @NFP_REPR_TYPE_VF:		virtual function
+ * @__NFP_REPR_TYPE_MAX:	number of representor types
  */
 enum nfp_repr_type {
 	NFP_REPR_TYPE_PHYS_PORT,
@@ -113,16 +115,18 @@ static inline int nfp_repr_get_port_id(struct net_device *netdev)
 	return priv->dst->u.port_info.port_id;
 }
 
+struct net_device *
+nfp_repr_get_locked(struct nfp_app *app, struct nfp_reprs *set,
+		    unsigned int id);
+
 void nfp_repr_inc_rx_stats(struct net_device *netdev, unsigned int len);
 int nfp_repr_init(struct nfp_app *app, struct net_device *netdev,
 		  u32 cmsg_port_id, struct nfp_port *port,
 		  struct net_device *pf_netdev);
 struct net_device *nfp_repr_alloc(struct nfp_app *app);
-void
-nfp_reprs_clean_and_free(struct nfp_reprs *reprs);
-void
-nfp_reprs_clean_and_free_by_type(struct nfp_app *app,
-				 enum nfp_repr_type type);
+void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs);
+void nfp_reprs_clean_and_free_by_type(struct nfp_app *app,
+				      enum nfp_repr_type type);
 struct nfp_reprs *nfp_reprs_alloc(unsigned int num_reprs);
 int nfp_reprs_resync_phys_ports(struct nfp_app *app);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
index c879626e035b..b802a1d55449 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
@@ -277,12 +277,6 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev,
 	}
 	nfp_net_irqs_assign(nn, vf->irq_entries, num_irqs);
 
-	/* Get ME clock frequency from ctrl BAR
-	 * XXX for now frequency is hardcoded until we figure out how
-	 * to get the value from nfp-hwinfo into ctrl bar
-	 */
-	nn->me_freq_mhz = 1200;
-
 	err = nfp_net_init(nn);
 	if (err)
 		goto err_irqs_disable;
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.c b/drivers/net/ethernet/netronome/nfp/nfp_port.c
index 34a6e035fe9a..7bd8be5c833b 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.c
@@ -32,6 +32,7 @@
  */
 
 #include <linux/lockdep.h>
+#include <linux/netdevice.h>
 #include <net/switchdev.h>
 
 #include "nfpcore/nfp_cpp.h"
@@ -100,6 +101,23 @@ int nfp_port_setup_tc(struct net_device *netdev, enum tc_setup_type type,
 	return nfp_app_setup_tc(port->app, netdev, type, type_data);
 }
 
+int nfp_port_set_features(struct net_device *netdev, netdev_features_t features)
+{
+	struct nfp_port *port;
+
+	port = nfp_port_from_netdev(netdev);
+	if (!port)
+		return 0;
+
+	if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) &&
+	    port->tc_offload_cnt) {
+		netdev_err(netdev, "Cannot disable HW TC offload while offloads active\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
 struct nfp_port *
 nfp_port_from_id(struct nfp_pf *pf, enum nfp_port_type type, unsigned int id)
 {
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.h b/drivers/net/ethernet/netronome/nfp/nfp_port.h
index 21bd4aa32646..fa7e669a969c 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.h
@@ -72,6 +72,8 @@ enum nfp_port_flags {
  * @netdev:	backpointer to associated netdev
  * @type:	what port type does the entity represent
  * @flags:	port flags
+ * @tc_offload_cnt:	number of active TC offloads, how offloads are counted
+ *			is not defined, use as a boolean
  * @app:	backpointer to the app structure
  * @dl_port:	devlink port structure
  * @eth_id:	for %NFP_PORT_PHYS_PORT port ID in NFP enumeration scheme
@@ -87,6 +89,7 @@ struct nfp_port {
 	enum nfp_port_type type;
 
 	unsigned long flags;
+	unsigned long tc_offload_cnt;
 
 	struct nfp_app *app;
 
@@ -121,6 +124,9 @@ static inline bool nfp_port_is_vnic(const struct nfp_port *port)
 	return port->type == NFP_PORT_PF_PORT || port->type == NFP_PORT_VF_PORT;
 }
 
+int
+nfp_port_set_features(struct net_device *netdev, netdev_features_t features);
+
 struct nfp_port *nfp_port_from_netdev(struct net_device *netdev);
 struct nfp_port *
 nfp_port_from_id(struct nfp_pf *pf, enum nfp_port_type type, unsigned int id);
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
index 3ce51f03126f..ced62d112aa2 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h
@@ -49,6 +49,8 @@
 struct nfp_hwinfo;
 struct nfp_hwinfo *nfp_hwinfo_read(struct nfp_cpp *cpp);
 const char *nfp_hwinfo_lookup(struct nfp_hwinfo *hwinfo, const char *lookup);
+char *nfp_hwinfo_get_packed_strings(struct nfp_hwinfo *hwinfo);
+u32 nfp_hwinfo_get_packed_str_size(struct nfp_hwinfo *hwinfo);
 
 /* Implemented in nfp_nsp.c, low level functions */
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h
index 5798adc57cbc..c8f2c064cce3 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h
@@ -242,6 +242,7 @@ int nfp_cpp_area_read(struct nfp_cpp_area *area, unsigned long offset,
 		      void *buffer, size_t length);
 int nfp_cpp_area_write(struct nfp_cpp_area *area, unsigned long offset,
 		       const void *buffer, size_t length);
+size_t nfp_cpp_area_size(struct nfp_cpp_area *area);
 const char *nfp_cpp_area_name(struct nfp_cpp_area *cpp_area);
 void *nfp_cpp_area_priv(struct nfp_cpp_area *cpp_area);
 struct nfp_cpp *nfp_cpp_area_cpp(struct nfp_cpp_area *cpp_area);
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c
index 04dd5758ecf5..ef30597aa319 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cppcore.c
@@ -372,8 +372,7 @@ nfp_cpp_area_alloc(struct nfp_cpp *cpp, u32 dest,
  * that it can be accessed directly.
  *
  * NOTE: @address and @size must be 32-bit aligned values.
- *
- * NOTE: The area must also be 'released' when the structure is freed.
+ * The area must also be 'released' when the structure is freed.
  *
  * Return: NFP CPP Area handle, or NULL
  */
@@ -536,8 +535,7 @@ void nfp_cpp_area_release_free(struct nfp_cpp_area *area)
  * Read data from indicated CPP region.
  *
  * NOTE: @offset and @length must be 32-bit aligned values.
- *
- * NOTE: Area must have been locked down with an 'acquire'.
+ * Area must have been locked down with an 'acquire'.
  *
  * Return: length of io, or -ERRNO
  */
@@ -558,8 +556,7 @@ int nfp_cpp_area_read(struct nfp_cpp_area *area,
  * Write data to indicated CPP region.
  *
  * NOTE: @offset and @length must be 32-bit aligned values.
- *
- * NOTE: Area must have been locked down with an 'acquire'.
+ * Area must have been locked down with an 'acquire'.
  *
  * Return: length of io, or -ERRNO
  */
@@ -570,6 +567,17 @@ int nfp_cpp_area_write(struct nfp_cpp_area *area,
 	return area->cpp->op->area_write(area, kernel_vaddr, offset, length);
 }
 
+/**
+ * nfp_cpp_area_size() - return size of a CPP area
+ * @cpp_area:	CPP area handle
+ *
+ * Return: Size of the area
+ */
+size_t nfp_cpp_area_size(struct nfp_cpp_area *cpp_area)
+{
+	return cpp_area->size;
+}
+
 /**
  * nfp_cpp_area_name() - return name of a CPP area
  * @cpp_area:	CPP area handle
@@ -666,18 +674,20 @@ void __iomem *nfp_cpp_area_iomem(struct nfp_cpp_area *area)
  * @offset:	Offset into area
  * @value:	Pointer to read buffer
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_area_readl(struct nfp_cpp_area *area,
 		       unsigned long offset, u32 *value)
 {
 	u8 tmp[4];
-	int err;
+	int n;
 
-	err = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
-	*value = get_unaligned_le32(tmp);
+	n = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
+	if (n != sizeof(tmp))
+		return n < 0 ? n : -EIO;
 
-	return err;
+	*value = get_unaligned_le32(tmp);
+	return 0;
 }
 
 /**
@@ -686,16 +696,18 @@ int nfp_cpp_area_readl(struct nfp_cpp_area *area,
  * @offset:	Offset into area
  * @value:	Value to write
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_area_writel(struct nfp_cpp_area *area,
 			unsigned long offset, u32 value)
 {
 	u8 tmp[4];
+	int n;
 
 	put_unaligned_le32(value, tmp);
+	n = nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));
 
-	return nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));
+	return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO;
 }
 
 /**
@@ -704,18 +716,20 @@ int nfp_cpp_area_writel(struct nfp_cpp_area *area,
  * @offset:	Offset into area
  * @value:	Pointer to read buffer
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_area_readq(struct nfp_cpp_area *area,
 		       unsigned long offset, u64 *value)
 {
 	u8 tmp[8];
-	int err;
+	int n;
 
-	err = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
-	*value = get_unaligned_le64(tmp);
+	n = nfp_cpp_area_read(area, offset, &tmp, sizeof(tmp));
+	if (n != sizeof(tmp))
+		return n < 0 ? n : -EIO;
 
-	return err;
+	*value = get_unaligned_le64(tmp);
+	return 0;
 }
 
 /**
@@ -724,16 +738,18 @@ int nfp_cpp_area_readq(struct nfp_cpp_area *area,
  * @offset:	Offset into area
  * @value:	Value to write
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_area_writeq(struct nfp_cpp_area *area,
 			unsigned long offset, u64 value)
 {
 	u8 tmp[8];
+	int n;
 
 	put_unaligned_le64(value, tmp);
+	n = nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));
 
-	return nfp_cpp_area_write(area, offset, &tmp, sizeof(tmp));
+	return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO;
 }
 
 /**
@@ -1072,7 +1088,7 @@ static u32 nfp_xpb_to_cpp(struct nfp_cpp *cpp, u32 *xpb_addr)
  * @xpb_addr:	Address for operation
  * @value:	Pointer to read buffer
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_xpb_readl(struct nfp_cpp *cpp, u32 xpb_addr, u32 *value)
 {
@@ -1087,7 +1103,7 @@ int nfp_xpb_readl(struct nfp_cpp *cpp, u32 xpb_addr, u32 *value)
  * @xpb_addr:	Address for operation
  * @value:	Value to write
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_xpb_writel(struct nfp_cpp *cpp, u32 xpb_addr, u32 value)
 {
@@ -1105,7 +1121,7 @@ int nfp_xpb_writel(struct nfp_cpp *cpp, u32 xpb_addr, u32 value)
  *
  * KERNEL: This operation is safe to call in interrupt or softirq context.
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_xpb_writelm(struct nfp_cpp *cpp, u32 xpb_tgt,
 		    u32 mask, u32 value)
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpplib.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpplib.c
index ab86bceb93f2..20bad05e2e92 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpplib.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpplib.c
@@ -64,18 +64,20 @@
  * @address:	Address for operation
  * @value:	Pointer to read buffer
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_readl(struct nfp_cpp *cpp, u32 cpp_id,
 		  unsigned long long address, u32 *value)
 {
 	u8 tmp[4];
-	int err;
+	int n;
 
-	err = nfp_cpp_read(cpp, cpp_id, address, tmp, sizeof(tmp));
-	*value = get_unaligned_le32(tmp);
+	n = nfp_cpp_read(cpp, cpp_id, address, tmp, sizeof(tmp));
+	if (n != sizeof(tmp))
+		return n < 0 ? n : -EIO;
 
-	return err;
+	*value = get_unaligned_le32(tmp);
+	return 0;
 }
 
 /**
@@ -85,15 +87,18 @@ int nfp_cpp_readl(struct nfp_cpp *cpp, u32 cpp_id,
  * @address:	Address for operation
  * @value:	Value to write
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_writel(struct nfp_cpp *cpp, u32 cpp_id,
 		   unsigned long long address, u32 value)
 {
 	u8 tmp[4];
+	int n;
 
 	put_unaligned_le32(value, tmp);
-	return nfp_cpp_write(cpp, cpp_id, address, tmp, sizeof(tmp));
+	n = nfp_cpp_write(cpp, cpp_id, address, tmp, sizeof(tmp));
+
+	return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO;
 }
 
 /**
@@ -103,18 +108,20 @@ int nfp_cpp_writel(struct nfp_cpp *cpp, u32 cpp_id,
  * @address:	Address for operation
  * @value:	Pointer to read buffer
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_readq(struct nfp_cpp *cpp, u32 cpp_id,
 		  unsigned long long address, u64 *value)
 {
 	u8 tmp[8];
-	int err;
+	int n;
 
-	err = nfp_cpp_read(cpp, cpp_id, address, tmp, sizeof(tmp));
-	*value = get_unaligned_le64(tmp);
+	n = nfp_cpp_read(cpp, cpp_id, address, tmp, sizeof(tmp));
+	if (n != sizeof(tmp))
+		return n < 0 ? n : -EIO;
 
-	return err;
+	*value = get_unaligned_le64(tmp);
+	return 0;
 }
 
 /**
@@ -124,15 +131,18 @@ int nfp_cpp_readq(struct nfp_cpp *cpp, u32 cpp_id,
  * @address:	Address for operation
  * @value:	Value to write
  *
- * Return: length of the io, or -ERRNO
+ * Return: 0 on success, or -ERRNO
  */
 int nfp_cpp_writeq(struct nfp_cpp *cpp, u32 cpp_id,
 		   unsigned long long address, u64 value)
 {
 	u8 tmp[8];
+	int n;
 
 	put_unaligned_le64(value, tmp);
-	return nfp_cpp_write(cpp, cpp_id, address, tmp, sizeof(tmp));
+	n = nfp_cpp_write(cpp, cpp_id, address, tmp, sizeof(tmp));
+
+	return n == sizeof(tmp) ? 0 : n < 0 ? n : -EIO;
 }
 
 /* NOTE: This code should not use nfp_xpb_* functions,
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_hwinfo.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_hwinfo.c
index 4f24aff1e772..063a9a6243d6 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_hwinfo.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_hwinfo.c
@@ -302,3 +302,13 @@ const char *nfp_hwinfo_lookup(struct nfp_hwinfo *hwinfo, const char *lookup)
 
 	return NULL;
 }
+
+char *nfp_hwinfo_get_packed_strings(struct nfp_hwinfo *hwinfo)
+{
+	return hwinfo->data;
+}
+
+u32 nfp_hwinfo_get_packed_str_size(struct nfp_hwinfo *hwinfo)
+{
+	return le32_to_cpu(hwinfo->size) - sizeof(u32);
+}
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.c
index 14a6d1ba51a9..39abac678b71 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.c
@@ -51,6 +51,9 @@
 #include "nfp_cpp.h"
 #include "nfp_nsp.h"
 
+#define NFP_NSP_TIMEOUT_DEFAULT	30
+#define NFP_NSP_TIMEOUT_BOOT	30
+
 /* Offsets relative to the CSR base */
 #define NSP_STATUS		0x00
 #define   NSP_STATUS_MAGIC	GENMASK_ULL(63, 48)
@@ -93,6 +96,7 @@ enum nfp_nsp_cmd {
 	SPCODE_FW_LOAD		= 6, /* Load fw from buffer, len in option */
 	SPCODE_ETH_RESCAN	= 7, /* Rescan ETHs, write ETH_TABLE to buf */
 	SPCODE_ETH_CONTROL	= 8, /* Update media config from buffer */
+	SPCODE_NSP_WRITE_FLASH	= 11, /* Load and flash image from buffer */
 	SPCODE_NSP_SENSORS	= 12, /* Read NSP sensor(s) */
 	SPCODE_NSP_IDENTIFY	= 13, /* Read NSP version */
 };
@@ -260,10 +264,10 @@ u16 nfp_nsp_get_abi_ver_minor(struct nfp_nsp *state)
 }
 
 static int
-nfp_nsp_wait_reg(struct nfp_cpp *cpp, u64 *reg,
-		 u32 nsp_cpp, u64 addr, u64 mask, u64 val)
+nfp_nsp_wait_reg(struct nfp_cpp *cpp, u64 *reg, u32 nsp_cpp, u64 addr,
+		 u64 mask, u64 val, u32 timeout_sec)
 {
-	const unsigned long wait_until = jiffies + 30 * HZ;
+	const unsigned long wait_until = jiffies + timeout_sec * HZ;
 	int err;
 
 	for (;;) {
@@ -285,12 +289,13 @@ nfp_nsp_wait_reg(struct nfp_cpp *cpp, u64 *reg,
 }
 
 /**
- * nfp_nsp_command() - Execute a command on the NFP Service Processor
+ * __nfp_nsp_command() - Execute a command on the NFP Service Processor
  * @state:	NFP SP state
  * @code:	NFP SP Command Code
  * @option:	NFP SP Command Argument
  * @buff_cpp:	NFP SP Buffer CPP Address info
  * @buff_addr:	NFP SP Buffer Host address
+ * @timeout_sec:Timeout value to wait for completion in seconds
  *
  * Return: 0 for success with no result
  *
@@ -300,10 +305,11 @@ nfp_nsp_wait_reg(struct nfp_cpp *cpp, u64 *reg,
  *	-ENODEV if the NSP is not a supported model
  *	-EBUSY if the NSP is stuck
  *	-EINTR if interrupted while waiting for completion
- *	-ETIMEDOUT if the NSP took longer than 30 seconds to complete
+ *	-ETIMEDOUT if the NSP took longer than @timeout_sec seconds to complete
  */
-static int nfp_nsp_command(struct nfp_nsp *state, u16 code, u32 option,
-			   u32 buff_cpp, u64 buff_addr)
+static int
+__nfp_nsp_command(struct nfp_nsp *state, u16 code, u32 option, u32 buff_cpp,
+		  u64 buff_addr, u32 timeout_sec)
 {
 	u64 reg, ret_val, nsp_base, nsp_buffer, nsp_status, nsp_command;
 	struct nfp_cpp *cpp = state->cpp;
@@ -341,8 +347,8 @@ static int nfp_nsp_command(struct nfp_nsp *state, u16 code, u32 option,
 		return err;
 
 	/* Wait for NSP_COMMAND_START to go to 0 */
-	err = nfp_nsp_wait_reg(cpp, &reg,
-			       nsp_cpp, nsp_command, NSP_COMMAND_START, 0);
+	err = nfp_nsp_wait_reg(cpp, &reg, nsp_cpp, nsp_command,
+			       NSP_COMMAND_START, 0, NFP_NSP_TIMEOUT_DEFAULT);
 	if (err) {
 		nfp_err(cpp, "Error %d waiting for code 0x%04x to start\n",
 			err, code);
@@ -350,8 +356,8 @@ static int nfp_nsp_command(struct nfp_nsp *state, u16 code, u32 option,
 	}
 
 	/* Wait for NSP_STATUS_BUSY to go to 0 */
-	err = nfp_nsp_wait_reg(cpp, &reg,
-			       nsp_cpp, nsp_status, NSP_STATUS_BUSY, 0);
+	err = nfp_nsp_wait_reg(cpp, &reg, nsp_cpp, nsp_status, NSP_STATUS_BUSY,
+			       0, timeout_sec);
 	if (err) {
 		nfp_err(cpp, "Error %d waiting for code 0x%04x to complete\n",
 			err, code);
@@ -374,9 +380,18 @@ static int nfp_nsp_command(struct nfp_nsp *state, u16 code, u32 option,
 	return ret_val;
 }
 
-static int nfp_nsp_command_buf(struct nfp_nsp *nsp, u16 code, u32 option,
-			       const void *in_buf, unsigned int in_size,
-			       void *out_buf, unsigned int out_size)
+static int
+nfp_nsp_command(struct nfp_nsp *state, u16 code, u32 option, u32 buff_cpp,
+		u64 buff_addr)
+{
+	return __nfp_nsp_command(state, code, option, buff_cpp, buff_addr,
+				 NFP_NSP_TIMEOUT_DEFAULT);
+}
+
+static int
+__nfp_nsp_command_buf(struct nfp_nsp *nsp, u16 code, u32 option,
+		      const void *in_buf, unsigned int in_size, void *out_buf,
+		      unsigned int out_size, u32 timeout_sec)
 {
 	struct nfp_cpp *cpp = nsp->cpp;
 	unsigned int max_size;
@@ -429,7 +444,8 @@ static int nfp_nsp_command_buf(struct nfp_nsp *nsp, u16 code, u32 option,
 			return err;
 	}
 
-	ret = nfp_nsp_command(nsp, code, option, cpp_id, cpp_buf);
+	ret = __nfp_nsp_command(nsp, code, option, cpp_id, cpp_buf,
+				timeout_sec);
 	if (ret < 0)
 		return ret;
 
@@ -442,12 +458,23 @@ static int nfp_nsp_command_buf(struct nfp_nsp *nsp, u16 code, u32 option,
 	return ret;
 }
 
+static int
+nfp_nsp_command_buf(struct nfp_nsp *nsp, u16 code, u32 option,
+		    const void *in_buf, unsigned int in_size, void *out_buf,
+		    unsigned int out_size)
+{
+	return __nfp_nsp_command_buf(nsp, code, option, in_buf, in_size,
+				     out_buf, out_size,
+				     NFP_NSP_TIMEOUT_DEFAULT);
+}
+
 int nfp_nsp_wait(struct nfp_nsp *state)
 {
-	const unsigned long wait_until = jiffies + 30 * HZ;
+	const unsigned long wait_until = jiffies + NFP_NSP_TIMEOUT_BOOT * HZ;
 	int err;
 
-	nfp_dbg(state->cpp, "Waiting for NSP to respond (30 sec max).\n");
+	nfp_dbg(state->cpp, "Waiting for NSP to respond (%u sec max).\n",
+		NFP_NSP_TIMEOUT_BOOT);
 
 	for (;;) {
 		const unsigned long start_time = jiffies;
@@ -488,6 +515,17 @@ int nfp_nsp_load_fw(struct nfp_nsp *state, const struct firmware *fw)
 				   fw->size, NULL, 0);
 }
 
+int nfp_nsp_write_flash(struct nfp_nsp *state, const struct firmware *fw)
+{
+	/* The flash time is specified to take a maximum of 70s so we add an
+	 * additional factor to this spec time.
+	 */
+	u32 timeout_sec = 2.5 * 70;
+
+	return __nfp_nsp_command_buf(state, SPCODE_NSP_WRITE_FLASH, fw->size,
+				     fw->data, fw->size, NULL, 0, timeout_sec);
+}
+
 int nfp_nsp_read_eth_table(struct nfp_nsp *state, void *buf, unsigned int size)
 {
 	return nfp_nsp_command_buf(state, SPCODE_ETH_RESCAN, size, NULL, 0,
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h
index 650ca1a5bd21..f23d9e06f097 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h
@@ -48,6 +48,7 @@ u16 nfp_nsp_get_abi_ver_minor(struct nfp_nsp *state);
 int nfp_nsp_wait(struct nfp_nsp *state);
 int nfp_nsp_device_soft_reset(struct nfp_nsp *state);
 int nfp_nsp_load_fw(struct nfp_nsp *state, const struct firmware *fw);
+int nfp_nsp_write_flash(struct nfp_nsp *state, const struct firmware *fw);
 int nfp_nsp_mac_reinit(struct nfp_nsp *state);
 
 static inline bool nfp_nsp_has_mac_reinit(struct nfp_nsp *state)
@@ -97,28 +98,29 @@ enum nfp_eth_fec {
  * @max_index:	max of @index fields of all @ports
  * @ports:	table of ports
  *
- * @eth_index:	port index according to legacy ethX numbering
- * @index:	chip-wide first channel index
- * @nbi:	NBI index
- * @base:	first channel index (within NBI)
- * @lanes:	number of channels
- * @speed:	interface speed (in Mbps)
- * @interface:	interface (module) plugged in
- * @media:	media type of the @interface
- * @fec:	forward error correction mode
- * @aneg:	auto negotiation mode
- * @mac_addr:	interface MAC address
- * @label_port:	port id
- * @label_subport:  id of interface within port (for split ports)
- * @enabled:	is enabled?
- * @tx_enabled:	is TX enabled?
- * @rx_enabled:	is RX enabled?
- * @override_changed: is media reconfig pending?
+ * @ports.eth_index:	port index according to legacy ethX numbering
+ * @ports.index:	chip-wide first channel index
+ * @ports.nbi:		NBI index
+ * @ports.base:		first channel index (within NBI)
+ * @ports.lanes:	number of channels
+ * @ports.speed:	interface speed (in Mbps)
+ * @ports.interface:	interface (module) plugged in
+ * @ports.media:	media type of the @interface
+ * @ports.fec:		forward error correction mode
+ * @ports.aneg:		auto negotiation mode
+ * @ports.mac_addr:	interface MAC address
+ * @ports.label_port:	port id
+ * @ports.label_subport:  id of interface within port (for split ports)
+ * @ports.enabled:	is enabled?
+ * @ports.tx_enabled:	is TX enabled?
+ * @ports.rx_enabled:	is RX enabled?
+ * @ports.override_changed: is media reconfig pending?
  *
- * @port_type:	one of %PORT_* defines for ethtool
- * @port_lanes:	total number of lanes on the port (sum of lanes of all subports)
- * @is_split:	is interface part of a split port
- * @fec_modes_supported:	bitmap of FEC modes supported
+ * @ports.port_type:	one of %PORT_* defines for ethtool
+ * @ports.port_lanes:	total number of lanes on the port (sum of lanes of all
+ *			subports)
+ * @ports.is_split:	is interface part of a split port
+ * @ports.fec_modes_supported:	bitmap of FEC modes supported
  */
 struct nfp_eth_table {
 	unsigned int count;
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
index b1dd13ff282b..7e14725055c7 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c
@@ -56,16 +56,17 @@
 
 /**
  * struct nfp_resource_entry - Resource table entry
- * @owner:		NFP CPP Lock, interface owner
- * @key:		NFP CPP Lock, posix_crc32(name, 8)
- * @region:		Memory region descriptor
- * @name:		ASCII, zero padded name
- * @reserved
- * @cpp_action:		CPP Action
- * @cpp_token:		CPP Token
- * @cpp_target:		CPP Target ID
- * @page_offset:	256-byte page offset into target's CPP address
- * @page_size:		size, in 256-byte pages
+ * @mutex:	NFP CPP Lock
+ * @mutex.owner:	NFP CPP Lock, interface owner
+ * @mutex.key:		NFP CPP Lock, posix_crc32(name, 8)
+ * @region:	Memory region descriptor
+ * @region.name:	ASCII, zero padded name
+ * @region.reserved:	padding
+ * @region.cpp_action:	CPP Action
+ * @region.cpp_token:	CPP Token
+ * @region.cpp_target:	CPP Target ID
+ * @region.page_offset:	256-byte page offset into target's CPP address
+ * @region.page_size:	size, in 256-byte pages
  */
 struct nfp_resource_entry {
 	struct nfp_resource_entry_mutex {
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c
index ecda474ac7c3..46107aefad1c 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c
@@ -277,10 +277,6 @@ u64 nfp_rtsym_read_le(struct nfp_rtsym_table *rtbl, const char *name,
 		break;
 	}
 
-	if (err == sym->size)
-		err = 0;
-	else if (err >= 0)
-		err = -EIO;
 exit:
 	if (error)
 		*error = err;
diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c
index 481876b5424c..66c665d0b926 100644
--- a/drivers/net/ethernet/nvidia/forcedeth.c
+++ b/drivers/net/ethernet/nvidia/forcedeth.c
@@ -793,9 +793,9 @@ struct fe_priv {
 	/* rx specific fields.
 	 * Locking: Within irq hander or disable_irq+spin_lock(&np->lock);
 	 */
-	union ring_type get_rx, put_rx, first_rx, last_rx;
+	union ring_type get_rx, put_rx, last_rx;
 	struct nv_skb_map *get_rx_ctx, *put_rx_ctx;
-	struct nv_skb_map *first_rx_ctx, *last_rx_ctx;
+	struct nv_skb_map *last_rx_ctx;
 	struct nv_skb_map *rx_skb;
 
 	union ring_type rx_ring;
@@ -822,9 +822,9 @@ struct fe_priv {
 	/*
 	 * tx specific fields.
 	 */
-	union ring_type get_tx, put_tx, first_tx, last_tx;
+	union ring_type get_tx, put_tx, last_tx;
 	struct nv_skb_map *get_tx_ctx, *put_tx_ctx;
-	struct nv_skb_map *first_tx_ctx, *last_tx_ctx;
+	struct nv_skb_map *last_tx_ctx;
 	struct nv_skb_map *tx_skb;
 
 	union ring_type tx_ring;
@@ -1812,12 +1812,12 @@ static int nv_alloc_rx(struct net_device *dev)
 	struct ring_desc *less_rx;
 
 	less_rx = np->get_rx.orig;
-	if (less_rx-- == np->first_rx.orig)
+	if (less_rx-- == np->rx_ring.orig)
 		less_rx = np->last_rx.orig;
 
 	while (np->put_rx.orig != less_rx) {
 		struct sk_buff *skb = netdev_alloc_skb(dev, np->rx_buf_sz + NV_RX_ALLOC_PAD);
-		if (skb) {
+		if (likely(skb)) {
 			np->put_rx_ctx->skb = skb;
 			np->put_rx_ctx->dma = dma_map_single(&np->pci_dev->dev,
 							     skb->data,
@@ -1833,9 +1833,9 @@ static int nv_alloc_rx(struct net_device *dev)
 			wmb();
 			np->put_rx.orig->flaglen = cpu_to_le32(np->rx_buf_sz | NV_RX_AVAIL);
 			if (unlikely(np->put_rx.orig++ == np->last_rx.orig))
-				np->put_rx.orig = np->first_rx.orig;
+				np->put_rx.orig = np->rx_ring.orig;
 			if (unlikely(np->put_rx_ctx++ == np->last_rx_ctx))
-				np->put_rx_ctx = np->first_rx_ctx;
+				np->put_rx_ctx = np->rx_skb;
 		} else {
 packet_dropped:
 			u64_stats_update_begin(&np->swstats_rx_syncp);
@@ -1853,12 +1853,12 @@ static int nv_alloc_rx_optimized(struct net_device *dev)
 	struct ring_desc_ex *less_rx;
 
 	less_rx = np->get_rx.ex;
-	if (less_rx-- == np->first_rx.ex)
+	if (less_rx-- == np->rx_ring.ex)
 		less_rx = np->last_rx.ex;
 
 	while (np->put_rx.ex != less_rx) {
 		struct sk_buff *skb = netdev_alloc_skb(dev, np->rx_buf_sz + NV_RX_ALLOC_PAD);
-		if (skb) {
+		if (likely(skb)) {
 			np->put_rx_ctx->skb = skb;
 			np->put_rx_ctx->dma = dma_map_single(&np->pci_dev->dev,
 							     skb->data,
@@ -1875,9 +1875,9 @@ static int nv_alloc_rx_optimized(struct net_device *dev)
 			wmb();
 			np->put_rx.ex->flaglen = cpu_to_le32(np->rx_buf_sz | NV_RX2_AVAIL);
 			if (unlikely(np->put_rx.ex++ == np->last_rx.ex))
-				np->put_rx.ex = np->first_rx.ex;
+				np->put_rx.ex = np->rx_ring.ex;
 			if (unlikely(np->put_rx_ctx++ == np->last_rx_ctx))
-				np->put_rx_ctx = np->first_rx_ctx;
+				np->put_rx_ctx = np->rx_skb;
 		} else {
 packet_dropped:
 			u64_stats_update_begin(&np->swstats_rx_syncp);
@@ -1903,13 +1903,15 @@ static void nv_init_rx(struct net_device *dev)
 	struct fe_priv *np = netdev_priv(dev);
 	int i;
 
-	np->get_rx = np->put_rx = np->first_rx = np->rx_ring;
+	np->get_rx = np->rx_ring;
+	np->put_rx = np->rx_ring;
 
 	if (!nv_optimized(np))
 		np->last_rx.orig = &np->rx_ring.orig[np->rx_ring_size-1];
 	else
 		np->last_rx.ex = &np->rx_ring.ex[np->rx_ring_size-1];
-	np->get_rx_ctx = np->put_rx_ctx = np->first_rx_ctx = np->rx_skb;
+	np->get_rx_ctx = np->rx_skb;
+	np->put_rx_ctx = np->rx_skb;
 	np->last_rx_ctx = &np->rx_skb[np->rx_ring_size-1];
 
 	for (i = 0; i < np->rx_ring_size; i++) {
@@ -1932,13 +1934,15 @@ static void nv_init_tx(struct net_device *dev)
 	struct fe_priv *np = netdev_priv(dev);
 	int i;
 
-	np->get_tx = np->put_tx = np->first_tx = np->tx_ring;
+	np->get_tx = np->tx_ring;
+	np->put_tx = np->tx_ring;
 
 	if (!nv_optimized(np))
 		np->last_tx.orig = &np->tx_ring.orig[np->tx_ring_size-1];
 	else
 		np->last_tx.ex = &np->tx_ring.ex[np->tx_ring_size-1];
-	np->get_tx_ctx = np->put_tx_ctx = np->first_tx_ctx = np->tx_skb;
+	np->get_tx_ctx = np->tx_skb;
+	np->put_tx_ctx = np->tx_skb;
 	np->last_tx_ctx = &np->tx_skb[np->tx_ring_size-1];
 	netdev_reset_queue(np->dev);
 	np->tx_pkts_in_progress = 0;
@@ -2248,9 +2252,9 @@ static netdev_tx_t nv_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		offset += bcnt;
 		size -= bcnt;
 		if (unlikely(put_tx++ == np->last_tx.orig))
-			put_tx = np->first_tx.orig;
+			put_tx = np->tx_ring.orig;
 		if (unlikely(np->put_tx_ctx++ == np->last_tx_ctx))
-			np->put_tx_ctx = np->first_tx_ctx;
+			np->put_tx_ctx = np->tx_skb;
 	} while (size);
 
 	/* setup the fragments */
@@ -2276,7 +2280,7 @@ static netdev_tx_t nv_start_xmit(struct sk_buff *skb, struct net_device *dev)
 				do {
 					nv_unmap_txskb(np, start_tx_ctx);
 					if (unlikely(tmp_tx_ctx++ == np->last_tx_ctx))
-						tmp_tx_ctx = np->first_tx_ctx;
+						tmp_tx_ctx = np->tx_skb;
 				} while (tmp_tx_ctx != np->put_tx_ctx);
 				dev_kfree_skb_any(skb);
 				np->put_tx_ctx = start_tx_ctx;
@@ -2294,18 +2298,18 @@ static netdev_tx_t nv_start_xmit(struct sk_buff *skb, struct net_device *dev)
 			offset += bcnt;
 			frag_size -= bcnt;
 			if (unlikely(put_tx++ == np->last_tx.orig))
-				put_tx = np->first_tx.orig;
+				put_tx = np->tx_ring.orig;
 			if (unlikely(np->put_tx_ctx++ == np->last_tx_ctx))
-				np->put_tx_ctx = np->first_tx_ctx;
+				np->put_tx_ctx = np->tx_skb;
 		} while (frag_size);
 	}
 
-	if (unlikely(put_tx == np->first_tx.orig))
+	if (unlikely(put_tx == np->tx_ring.orig))
 		prev_tx = np->last_tx.orig;
 	else
 		prev_tx = put_tx - 1;
 
-	if (unlikely(np->put_tx_ctx == np->first_tx_ctx))
+	if (unlikely(np->put_tx_ctx == np->tx_skb))
 		prev_tx_ctx = np->last_tx_ctx;
 	else
 		prev_tx_ctx = np->put_tx_ctx - 1;
@@ -2406,9 +2410,9 @@ static netdev_tx_t nv_start_xmit_optimized(struct sk_buff *skb,
 		offset += bcnt;
 		size -= bcnt;
 		if (unlikely(put_tx++ == np->last_tx.ex))
-			put_tx = np->first_tx.ex;
+			put_tx = np->tx_ring.ex;
 		if (unlikely(np->put_tx_ctx++ == np->last_tx_ctx))
-			np->put_tx_ctx = np->first_tx_ctx;
+			np->put_tx_ctx = np->tx_skb;
 	} while (size);
 
 	/* setup the fragments */
@@ -2434,7 +2438,7 @@ static netdev_tx_t nv_start_xmit_optimized(struct sk_buff *skb,
 				do {
 					nv_unmap_txskb(np, start_tx_ctx);
 					if (unlikely(tmp_tx_ctx++ == np->last_tx_ctx))
-						tmp_tx_ctx = np->first_tx_ctx;
+						tmp_tx_ctx = np->tx_skb;
 				} while (tmp_tx_ctx != np->put_tx_ctx);
 				dev_kfree_skb_any(skb);
 				np->put_tx_ctx = start_tx_ctx;
@@ -2452,18 +2456,18 @@ static netdev_tx_t nv_start_xmit_optimized(struct sk_buff *skb,
 			offset += bcnt;
 			frag_size -= bcnt;
 			if (unlikely(put_tx++ == np->last_tx.ex))
-				put_tx = np->first_tx.ex;
+				put_tx = np->tx_ring.ex;
 			if (unlikely(np->put_tx_ctx++ == np->last_tx_ctx))
-				np->put_tx_ctx = np->first_tx_ctx;
+				np->put_tx_ctx = np->tx_skb;
 		} while (frag_size);
 	}
 
-	if (unlikely(put_tx == np->first_tx.ex))
+	if (unlikely(put_tx == np->tx_ring.ex))
 		prev_tx = np->last_tx.ex;
 	else
 		prev_tx = put_tx - 1;
 
-	if (unlikely(np->put_tx_ctx == np->first_tx_ctx))
+	if (unlikely(np->put_tx_ctx == np->tx_skb))
 		prev_tx_ctx = np->last_tx_ctx;
 	else
 		prev_tx_ctx = np->put_tx_ctx - 1;
@@ -2563,7 +2567,7 @@ static int nv_tx_done(struct net_device *dev, int limit)
 
 		if (np->desc_ver == DESC_VER_1) {
 			if (flags & NV_TX_LASTPACKET) {
-				if (flags & NV_TX_ERROR) {
+				if (unlikely(flags & NV_TX_ERROR)) {
 					if ((flags & NV_TX_RETRYERROR)
 					    && !(flags & NV_TX_RETRYCOUNT_MASK))
 						nv_legacybackoff_reseed(dev);
@@ -2580,7 +2584,7 @@ static int nv_tx_done(struct net_device *dev, int limit)
 			}
 		} else {
 			if (flags & NV_TX2_LASTPACKET) {
-				if (flags & NV_TX2_ERROR) {
+				if (unlikely(flags & NV_TX2_ERROR)) {
 					if ((flags & NV_TX2_RETRYERROR)
 					    && !(flags & NV_TX2_RETRYCOUNT_MASK))
 						nv_legacybackoff_reseed(dev);
@@ -2597,9 +2601,9 @@ static int nv_tx_done(struct net_device *dev, int limit)
 			}
 		}
 		if (unlikely(np->get_tx.orig++ == np->last_tx.orig))
-			np->get_tx.orig = np->first_tx.orig;
+			np->get_tx.orig = np->tx_ring.orig;
 		if (unlikely(np->get_tx_ctx++ == np->last_tx_ctx))
-			np->get_tx_ctx = np->first_tx_ctx;
+			np->get_tx_ctx = np->tx_skb;
 	}
 
 	netdev_completed_queue(np->dev, tx_work, bytes_compl);
@@ -2626,7 +2630,7 @@ static int nv_tx_done_optimized(struct net_device *dev, int limit)
 		nv_unmap_txskb(np, np->get_tx_ctx);
 
 		if (flags & NV_TX2_LASTPACKET) {
-			if (flags & NV_TX2_ERROR) {
+			if (unlikely(flags & NV_TX2_ERROR)) {
 				if ((flags & NV_TX2_RETRYERROR)
 				    && !(flags & NV_TX2_RETRYCOUNT_MASK)) {
 					if (np->driver_data & DEV_HAS_GEAR_MODE)
@@ -2651,9 +2655,9 @@ static int nv_tx_done_optimized(struct net_device *dev, int limit)
 		}
 
 		if (unlikely(np->get_tx.ex++ == np->last_tx.ex))
-			np->get_tx.ex = np->first_tx.ex;
+			np->get_tx.ex = np->tx_ring.ex;
 		if (unlikely(np->get_tx_ctx++ == np->last_tx_ctx))
-			np->get_tx_ctx = np->first_tx_ctx;
+			np->get_tx_ctx = np->tx_skb;
 	}
 
 	netdev_completed_queue(np->dev, tx_work, bytes_cleaned);
@@ -2909,9 +2913,9 @@ static int nv_rx_process(struct net_device *dev, int limit)
 		u64_stats_update_end(&np->swstats_rx_syncp);
 next_pkt:
 		if (unlikely(np->get_rx.orig++ == np->last_rx.orig))
-			np->get_rx.orig = np->first_rx.orig;
+			np->get_rx.orig = np->rx_ring.orig;
 		if (unlikely(np->get_rx_ctx++ == np->last_rx_ctx))
-			np->get_rx_ctx = np->first_rx_ctx;
+			np->get_rx_ctx = np->rx_skb;
 
 		rx_work++;
 	}
@@ -2998,9 +3002,9 @@ static int nv_rx_process_optimized(struct net_device *dev, int limit)
 		}
 next_pkt:
 		if (unlikely(np->get_rx.ex++ == np->last_rx.ex))
-			np->get_rx.ex = np->first_rx.ex;
+			np->get_rx.ex = np->rx_ring.ex;
 		if (unlikely(np->get_rx_ctx++ == np->last_rx_ctx))
-			np->get_rx_ctx = np->first_rx_ctx;
+			np->get_rx_ctx = np->rx_skb;
 
 		rx_work++;
 	}
@@ -5507,11 +5511,9 @@ static int nv_open(struct net_device *dev)
 	/* One manual link speed update: Interrupts are enabled, future link
 	 * speed changes cause interrupts and are handled by nv_link_irq().
 	 */
-	{
-		u32 miistat;
-		miistat = readl(base + NvRegMIIStatus);
-		writel(NVREG_MIISTAT_MASK_ALL, base + NvRegMIIStatus);
-	}
+	readl(base + NvRegMIIStatus);
+	writel(NVREG_MIISTAT_MASK_ALL, base + NvRegMIIStatus);
+
 	/* set linkspeed to invalid value, thus force nv_update_linkspeed
 	 * to init hw */
 	np->linkspeed = 0;
diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
index 40e52ffb732f..7cd494611a74 100644
--- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
+++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
@@ -2594,8 +2594,10 @@ static int pch_gbe_probe(struct pci_dev *pdev,
 	if (adapter->pdata && adapter->pdata->platform_init)
 		adapter->pdata->platform_init(pdev);
 
-	adapter->ptp_pdev = pci_get_bus_and_slot(adapter->pdev->bus->number,
-					       PCI_DEVFN(12, 4));
+	adapter->ptp_pdev =
+		pci_get_domain_bus_and_slot(pci_domain_nr(adapter->pdev->bus),
+					    adapter->pdev->bus->number,
+					    PCI_DEVFN(12, 4));
 
 	netdev->netdev_ops = &pch_gbe_netdev_ops;
 	netdev->watchdog_timeo = PCH_GBE_WATCHDOG_PERIOD;
diff --git a/drivers/net/ethernet/pasemi/pasemi_mac.c b/drivers/net/ethernet/pasemi/pasemi_mac.c
index c9a55b774935..07a2eb3781b1 100644
--- a/drivers/net/ethernet/pasemi/pasemi_mac.c
+++ b/drivers/net/ethernet/pasemi/pasemi_mac.c
@@ -212,9 +212,7 @@ static int pasemi_get_mac_addr(struct pasemi_mac *mac)
 		return -ENOENT;
 	}
 
-	if (sscanf(maddr, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
-		   &addr[0], &addr[1], &addr[2], &addr[3], &addr[4], &addr[5])
-	    != ETH_ALEN) {
+	if (!mac_pton(maddr, addr)) {
 		dev_warn(&pdev->dev,
 			 "can't parse mac address, not configuring\n");
 		return -EINVAL;
diff --git a/drivers/net/ethernet/qlogic/Kconfig b/drivers/net/ethernet/qlogic/Kconfig
index 26ddf092e3ec..0ee2490db729 100644
--- a/drivers/net/ethernet/qlogic/Kconfig
+++ b/drivers/net/ethernet/qlogic/Kconfig
@@ -85,6 +85,7 @@ config QED
 	tristate "QLogic QED 25/40/100Gb core driver"
 	depends on PCI
 	select ZLIB_INFLATE
+	select CRC8
 	---help---
 	  This enables the support for ...
 
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c
index 0a66389c06c2..1cd39c9a0345 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_hw.c
@@ -2502,12 +2502,10 @@ netxen_collect_minidump(struct netxen_adapter *adapter)
 {
 	int ret = 0;
 	struct netxen_minidump_template_hdr *hdr;
-	struct timespec val;
 	hdr = (struct netxen_minidump_template_hdr *)
 				adapter->mdump.md_template;
 	hdr->driver_capture_mask = adapter->mdump.md_capture_mask;
-	jiffies_to_timespec(jiffies, &val);
-	hdr->driver_timestamp = (u32) val.tv_sec;
+	hdr->driver_timestamp = ktime_get_seconds();
 	hdr->driver_info_word2 = adapter->fw_version;
 	hdr->driver_info_word3 = NXRD32(adapter, CRB_DRIVER_VERSION);
 	ret = netxen_parse_md_template(adapter);
diff --git a/drivers/net/ethernet/qlogic/qed/qed.h b/drivers/net/ethernet/qlogic/qed/qed.h
index 91003bc6f00b..69488554f4b9 100644
--- a/drivers/net/ethernet/qlogic/qed/qed.h
+++ b/drivers/net/ethernet/qlogic/qed/qed.h
@@ -52,10 +52,10 @@
 
 extern const struct qed_common_ops qed_common_ops_pass;
 
-#define QED_MAJOR_VERSION               8
-#define QED_MINOR_VERSION               10
-#define QED_REVISION_VERSION            11
-#define QED_ENGINEERING_VERSION 21
+#define QED_MAJOR_VERSION		8
+#define QED_MINOR_VERSION		33
+#define QED_REVISION_VERSION		0
+#define QED_ENGINEERING_VERSION		20
 
 #define QED_VERSION						 \
 	((QED_MAJOR_VERSION << 24) | (QED_MINOR_VERSION << 16) | \
@@ -778,8 +778,8 @@ static inline u8 qed_concrete_to_sw_fid(struct qed_dev *cdev,
 	return sw_fid;
 }
 
-#define PURE_LB_TC 8
-#define PKT_LB_TC 9
+#define PKT_LB_TC	9
+#define MAX_NUM_VOQS_E4	20
 
 int qed_configure_vport_wfq(struct qed_dev *cdev, u16 vp_id, u32 rate);
 void qed_configure_vp_wfq_on_link_change(struct qed_dev *cdev,
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
index afd07ad91631..00f41c145d4d 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c
@@ -86,22 +86,22 @@
 
 /* connection context union */
 union conn_context {
-	struct core_conn_context core_ctx;
-	struct eth_conn_context eth_ctx;
-	struct iscsi_conn_context iscsi_ctx;
-	struct fcoe_conn_context fcoe_ctx;
-	struct roce_conn_context roce_ctx;
+	struct e4_core_conn_context core_ctx;
+	struct e4_eth_conn_context eth_ctx;
+	struct e4_iscsi_conn_context iscsi_ctx;
+	struct e4_fcoe_conn_context fcoe_ctx;
+	struct e4_roce_conn_context roce_ctx;
 };
 
 /* TYPE-0 task context - iSCSI, FCOE */
 union type0_task_context {
-	struct iscsi_task_context iscsi_ctx;
-	struct fcoe_task_context fcoe_ctx;
+	struct e4_iscsi_task_context iscsi_ctx;
+	struct e4_fcoe_task_context fcoe_ctx;
 };
 
 /* TYPE-1 task context - ROCE */
 union type1_task_context {
-	struct rdma_task_context roce_ctx;
+	struct e4_rdma_task_context roce_ctx;
 };
 
 struct src_ent {
@@ -109,8 +109,8 @@ struct src_ent {
 	u64 next;
 };
 
-#define CDUT_SEG_ALIGNMET 3	/* in 4k chunks */
-#define CDUT_SEG_ALIGNMET_IN_BYTES (1 << (CDUT_SEG_ALIGNMET + 12))
+#define CDUT_SEG_ALIGNMET		3 /* in 4k chunks */
+#define CDUT_SEG_ALIGNMET_IN_BYTES	BIT(CDUT_SEG_ALIGNMET + 12)
 
 #define CONN_CXT_SIZE(p_hwfn) \
 	ALIGNED_TYPE_SIZE(union conn_context, p_hwfn)
@@ -742,7 +742,7 @@ int qed_cxt_cfg_ilt_compute(struct qed_hwfn *p_hwfn, u32 *line_count)
 	p_blk = qed_cxt_set_blk(&p_cli->pf_blks[0]);
 
 	qed_cxt_qm_iids(p_hwfn, &qm_iids);
-	total = qed_qm_pf_mem_size(p_hwfn->rel_pf_id, qm_iids.cids,
+	total = qed_qm_pf_mem_size(qm_iids.cids,
 				   qm_iids.vf_cids, qm_iids.tids,
 				   p_hwfn->qm_info.num_pqs,
 				   p_hwfn->qm_info.num_vf_pqs);
@@ -1055,11 +1055,10 @@ static int qed_ilt_blk_alloc(struct qed_hwfn *p_hwfn,
 		u32 size;
 
 		size = min_t(u32, sz_left, p_blk->real_size_in_page);
-		p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
-					    size, &p_phys, GFP_KERNEL);
+		p_virt = dma_zalloc_coherent(&p_hwfn->cdev->pdev->dev, size,
+					     &p_phys, GFP_KERNEL);
 		if (!p_virt)
 			return -ENOMEM;
-		memset(p_virt, 0, size);
 
 		ilt_shadow[line].p_phys = p_phys;
 		ilt_shadow[line].p_virt = p_virt;
@@ -1496,20 +1495,24 @@ static void qed_cdu_init_pf(struct qed_hwfn *p_hwfn)
 	}
 }
 
-void qed_qm_init_pf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
+void qed_qm_init_pf(struct qed_hwfn *p_hwfn,
+		    struct qed_ptt *p_ptt, bool is_pf_loading)
 {
-	struct qed_qm_pf_rt_init_params params;
 	struct qed_qm_info *qm_info = &p_hwfn->qm_info;
+	struct qed_qm_pf_rt_init_params params;
+	struct qed_mcp_link_state *p_link;
 	struct qed_qm_iids iids;
 
 	memset(&iids, 0, sizeof(iids));
 	qed_cxt_qm_iids(p_hwfn, &iids);
 
+	p_link = &QED_LEADING_HWFN(p_hwfn->cdev)->mcp_info->link_output;
+
 	memset(&params, 0, sizeof(params));
 	params.port_id = p_hwfn->port_id;
 	params.pf_id = p_hwfn->rel_pf_id;
 	params.max_phys_tcs_per_port = qm_info->max_phys_tcs_per_port;
-	params.is_first_pf = p_hwfn->first_on_engine;
+	params.is_pf_loading = is_pf_loading;
 	params.num_pf_cids = iids.cids;
 	params.num_vf_cids = iids.vf_cids;
 	params.num_tids = iids.tids;
@@ -1520,6 +1523,7 @@ void qed_qm_init_pf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 	params.num_vports = qm_info->num_vports;
 	params.pf_wfq = qm_info->pf_wfq;
 	params.pf_rl = qm_info->pf_rl;
+	params.link_speed = p_link->speed;
 	params.pq_params = qm_info->qm_pq_params;
 	params.vport_params = qm_info->qm_vport_params;
 
@@ -1883,7 +1887,7 @@ void qed_cxt_hw_init_common(struct qed_hwfn *p_hwfn)
 
 void qed_cxt_hw_init_pf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
-	qed_qm_init_pf(p_hwfn, p_ptt);
+	qed_qm_init_pf(p_hwfn, p_ptt, true);
 	qed_cm_init_pf(p_hwfn);
 	qed_dq_init_pf(p_hwfn);
 	qed_cdu_init_pf(p_hwfn);
@@ -2303,14 +2307,13 @@ qed_cxt_dynamic_ilt_alloc(struct qed_hwfn *p_hwfn,
 		goto out0;
 	}
 
-	p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
-				    p_blk->real_size_in_page,
-				    &p_phys, GFP_KERNEL);
+	p_virt = dma_zalloc_coherent(&p_hwfn->cdev->pdev->dev,
+				     p_blk->real_size_in_page, &p_phys,
+				     GFP_KERNEL);
 	if (!p_virt) {
 		rc = -ENOMEM;
 		goto out1;
 	}
-	memset(p_virt, 0, p_blk->real_size_in_page);
 
 	/* configuration of refTagMask to 0xF is required for RoCE DIF MR only,
 	 * to compensate for a HW bug, but it is configured even if DIF is not
@@ -2326,7 +2329,7 @@ qed_cxt_dynamic_ilt_alloc(struct qed_hwfn *p_hwfn,
 		for (elem_i = 0; elem_i < elems_per_p; elem_i++) {
 			elem = (union type1_task_context *)elem_start;
 			SET_FIELD(elem->roce_ctx.tdif_context.flags1,
-				  TDIF_TASK_CONTEXT_REFTAGMASK, 0xf);
+				  TDIF_TASK_CONTEXT_REF_TAG_MASK, 0xf);
 			elem_start += TYPE1_TASK_CXT_SIZE(p_hwfn);
 		}
 	}
@@ -2477,7 +2480,10 @@ int qed_cxt_free_proto_ilt(struct qed_hwfn *p_hwfn, enum protocol_type proto)
 	if (rc)
 		return rc;
 
-	/* Free Task CXT */
+	/* Free Task CXT ( Intentionally RoCE as task-id is shared between
+	 * RoCE and iWARP )
+	 */
+	proto = PROTOCOLID_ROCE;
 	rc = qed_cxt_free_ilt_range(p_hwfn, QED_ELEM_TASK, 0,
 				    qed_cxt_get_proto_tid_count(p_hwfn, proto));
 	if (rc)
diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.h b/drivers/net/ethernet/qlogic/qed/qed_cxt.h
index 17836349a274..a4e95869889f 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_cxt.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.h
@@ -169,8 +169,10 @@ void qed_cxt_hw_init_pf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
  *
  * @param p_hwfn
  * @param p_ptt
+ * @param is_pf_loading
  */
-void qed_qm_init_pf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
+void qed_qm_init_pf(struct qed_hwfn *p_hwfn,
+		    struct qed_ptt *p_ptt, bool is_pf_loading);
 
 /**
  * @brief Reconfigures QM pf on the fly
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
index fe7c1f230028..449777f21237 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c
@@ -954,9 +954,7 @@ void qed_dcbx_set_pf_update_params(struct qed_dcbx_results *p_src,
 				   struct pf_update_ramrod_data *p_dest)
 {
 	struct protocol_dcb_data *p_dcb_data;
-	bool update_flag = false;
-
-	p_dest->pf_id = p_src->pf_id;
+	u8 update_flag;
 
 	update_flag = p_src->arr[DCBX_PROTOCOL_FCOE].update;
 	p_dest->update_fcoe_dcb_data_mode = update_flag;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_debug.c b/drivers/net/ethernet/qlogic/qed/qed_debug.c
index 03c3cf77aaff..fdf37abee3d3 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_debug.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_debug.c
@@ -21,25 +21,26 @@ enum mem_groups {
 	MEM_GROUP_DMAE_MEM,
 	MEM_GROUP_CM_MEM,
 	MEM_GROUP_QM_MEM,
-	MEM_GROUP_TM_MEM,
+	MEM_GROUP_DORQ_MEM,
 	MEM_GROUP_BRB_RAM,
 	MEM_GROUP_BRB_MEM,
 	MEM_GROUP_PRS_MEM,
-	MEM_GROUP_SDM_MEM,
 	MEM_GROUP_IOR,
-	MEM_GROUP_RAM,
 	MEM_GROUP_BTB_RAM,
-	MEM_GROUP_RDIF_CTX,
-	MEM_GROUP_TDIF_CTX,
-	MEM_GROUP_CFC_MEM,
 	MEM_GROUP_CONN_CFC_MEM,
 	MEM_GROUP_TASK_CFC_MEM,
 	MEM_GROUP_CAU_PI,
 	MEM_GROUP_CAU_MEM,
 	MEM_GROUP_PXP_ILT,
+	MEM_GROUP_TM_MEM,
+	MEM_GROUP_SDM_MEM,
 	MEM_GROUP_PBUF,
+	MEM_GROUP_RAM,
 	MEM_GROUP_MULD_MEM,
 	MEM_GROUP_BTB_MEM,
+	MEM_GROUP_RDIF_CTX,
+	MEM_GROUP_TDIF_CTX,
+	MEM_GROUP_CFC_MEM,
 	MEM_GROUP_IGU_MEM,
 	MEM_GROUP_IGU_MSIX,
 	MEM_GROUP_CAU_SB,
@@ -54,25 +55,26 @@ static const char * const s_mem_group_names[] = {
 	"DMAE_MEM",
 	"CM_MEM",
 	"QM_MEM",
-	"TM_MEM",
+	"DORQ_MEM",
 	"BRB_RAM",
 	"BRB_MEM",
 	"PRS_MEM",
-	"SDM_MEM",
 	"IOR",
-	"RAM",
 	"BTB_RAM",
-	"RDIF_CTX",
-	"TDIF_CTX",
-	"CFC_MEM",
 	"CONN_CFC_MEM",
 	"TASK_CFC_MEM",
 	"CAU_PI",
 	"CAU_MEM",
 	"PXP_ILT",
+	"TM_MEM",
+	"SDM_MEM",
 	"PBUF",
+	"RAM",
 	"MULD_MEM",
 	"BTB_MEM",
+	"RDIF_CTX",
+	"TDIF_CTX",
+	"CFC_MEM",
 	"IGU_MEM",
 	"IGU_MSIX",
 	"CAU_SB",
@@ -92,11 +94,6 @@ static u32 cond7(const u32 *r, const u32 *imm)
 	return ((r[0] >> imm[0]) & imm[1]) != imm[2];
 }
 
-static u32 cond14(const u32 *r, const u32 *imm)
-{
-	return (r[0] != imm[0]) && (((r[1] >> imm[1]) & imm[2]) == imm[3]);
-}
-
 static u32 cond6(const u32 *r, const u32 *imm)
 {
 	return (r[0] & imm[0]) != imm[1];
@@ -174,7 +171,6 @@ static u32(*cond_arr[]) (const u32 *r, const u32 *imm) = {
 	cond11,
 	cond12,
 	cond13,
-	cond14,
 };
 
 /******************************* Data Types **********************************/
@@ -203,6 +199,8 @@ struct chip_defs {
 struct platform_defs {
 	const char *name;
 	u32 delay_factor;
+	u32 dmae_thresh;
+	u32 log_thresh;
 };
 
 /* Storm constant definitions.
@@ -234,7 +232,7 @@ struct storm_defs {
 /* Block constant definitions */
 struct block_defs {
 	const char *name;
-	bool has_dbg_bus[MAX_CHIP_IDS];
+	bool exists[MAX_CHIP_IDS];
 	bool associated_to_storm;
 
 	/* Valid only if associated_to_storm is true */
@@ -258,8 +256,8 @@ struct block_defs {
 /* Reset register definitions */
 struct reset_reg_defs {
 	u32 addr;
-	u32 unreset_val;
 	bool exists[MAX_CHIP_IDS];
+	u32 unreset_val[MAX_CHIP_IDS];
 };
 
 struct grc_param_defs {
@@ -276,8 +274,8 @@ struct rss_mem_defs {
 	const char *mem_name;
 	const char *type_name;
 	u32 addr;
+	u32 entry_width;
 	u32 num_entries[MAX_CHIP_IDS];
-	u32 entry_width[MAX_CHIP_IDS];
 };
 
 struct vfc_ram_defs {
@@ -294,7 +292,9 @@ struct big_ram_defs {
 	enum dbg_grc_params grc_param;
 	u32 addr_reg_addr;
 	u32 data_reg_addr;
-	u32 num_of_blocks[MAX_CHIP_IDS];
+	u32 is_256b_reg_addr;
+	u32 is_256b_bit_offset[MAX_CHIP_IDS];
+	u32 ram_size[MAX_CHIP_IDS]; /* In dwords */
 };
 
 struct phy_defs {
@@ -358,20 +358,14 @@ struct phy_defs {
 			(arr)[i] = qed_rd(dev, ptt, addr); \
 	} while (0)
 
-#ifndef DWORDS_TO_BYTES
 #define DWORDS_TO_BYTES(dwords)		((dwords) * BYTES_IN_DWORD)
-#endif
-#ifndef BYTES_TO_DWORDS
 #define BYTES_TO_DWORDS(bytes)		((bytes) / BYTES_IN_DWORD)
-#endif
 
-/* extra lines include a signature line + optional latency events line */
-#ifndef NUM_DBG_LINES
+/* Extra lines include a signature line + optional latency events line */
 #define NUM_EXTRA_DBG_LINES(block_desc) \
 	(1 + ((block_desc)->has_latency_events ? 1 : 0))
 #define NUM_DBG_LINES(block_desc) \
 	((block_desc)->num_of_lines + NUM_EXTRA_DBG_LINES(block_desc))
-#endif
 
 #define RAM_LINES_TO_DWORDS(lines)	((lines) * 2)
 #define RAM_LINES_TO_BYTES(lines) \
@@ -424,9 +418,6 @@ struct phy_defs {
 #define NUM_RSS_MEM_TYPES		5
 
 #define NUM_BIG_RAM_TYPES		3
-#define BIG_RAM_BLOCK_SIZE_BYTES	128
-#define BIG_RAM_BLOCK_SIZE_DWORDS \
-	BYTES_TO_DWORDS(BIG_RAM_BLOCK_SIZE_BYTES)
 
 #define NUM_PHY_TBUS_ADDRESSES		2048
 #define PHY_DUMP_SIZE_DWORDS		(NUM_PHY_TBUS_ADDRESSES / 2)
@@ -441,23 +432,17 @@ struct phy_defs {
 
 #define FW_IMG_MAIN			1
 
-#ifndef REG_FIFO_ELEMENT_DWORDS
 #define REG_FIFO_ELEMENT_DWORDS		2
-#endif
 #define REG_FIFO_DEPTH_ELEMENTS		32
 #define REG_FIFO_DEPTH_DWORDS \
 	(REG_FIFO_ELEMENT_DWORDS * REG_FIFO_DEPTH_ELEMENTS)
 
-#ifndef IGU_FIFO_ELEMENT_DWORDS
 #define IGU_FIFO_ELEMENT_DWORDS		4
-#endif
 #define IGU_FIFO_DEPTH_ELEMENTS		64
 #define IGU_FIFO_DEPTH_DWORDS \
 	(IGU_FIFO_ELEMENT_DWORDS * IGU_FIFO_DEPTH_ELEMENTS)
 
-#ifndef PROTECTION_OVERRIDE_ELEMENT_DWORDS
 #define PROTECTION_OVERRIDE_ELEMENT_DWORDS	2
-#endif
 #define PROTECTION_OVERRIDE_DEPTH_ELEMENTS	20
 #define PROTECTION_OVERRIDE_DEPTH_DWORDS \
 	(PROTECTION_OVERRIDE_DEPTH_ELEMENTS * \
@@ -489,6 +474,11 @@ static struct chip_defs s_chip_defs[MAX_CHIP_IDS] = {
 	   {0, 0, 0} } },
 	{ "ah",
 	  {{MAX_NUM_PORTS_K2, MAX_NUM_PFS_K2, MAX_NUM_VFS_K2},
+	   {0, 0, 0},
+	   {0, 0, 0},
+	   {0, 0, 0} } },
+	{ "reserved",
+	   {{0, 0, 0},
 	   {0, 0, 0},
 	   {0, 0, 0},
 	   {0, 0, 0} } }
@@ -498,7 +488,8 @@ static struct chip_defs s_chip_defs[MAX_CHIP_IDS] = {
 static struct storm_defs s_storm_defs[] = {
 	/* Tstorm */
 	{'T', BLOCK_TSEM,
-	 {DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT}, true,
+	 {DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT,
+	  DBG_BUS_CLIENT_RBCT}, true,
 	 TSEM_REG_FAST_MEMORY,
 	 TSEM_REG_DBG_FRAME_MODE_BB_K2, TSEM_REG_SLOW_DBG_ACTIVE_BB_K2,
 	 TSEM_REG_SLOW_DBG_MODE_BB_K2, TSEM_REG_DBG_MODE1_CFG_BB_K2,
@@ -511,7 +502,8 @@ static struct storm_defs s_storm_defs[] = {
 
 	/* Mstorm */
 	{'M', BLOCK_MSEM,
-	 {DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM}, false,
+	 {DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM,
+	  DBG_BUS_CLIENT_RBCM}, false,
 	 MSEM_REG_FAST_MEMORY,
 	 MSEM_REG_DBG_FRAME_MODE_BB_K2, MSEM_REG_SLOW_DBG_ACTIVE_BB_K2,
 	 MSEM_REG_SLOW_DBG_MODE_BB_K2, MSEM_REG_DBG_MODE1_CFG_BB_K2,
@@ -524,7 +516,8 @@ static struct storm_defs s_storm_defs[] = {
 
 	/* Ustorm */
 	{'U', BLOCK_USEM,
-	 {DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU}, false,
+	 {DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU,
+	  DBG_BUS_CLIENT_RBCU}, false,
 	 USEM_REG_FAST_MEMORY,
 	 USEM_REG_DBG_FRAME_MODE_BB_K2, USEM_REG_SLOW_DBG_ACTIVE_BB_K2,
 	 USEM_REG_SLOW_DBG_MODE_BB_K2, USEM_REG_DBG_MODE1_CFG_BB_K2,
@@ -537,7 +530,8 @@ static struct storm_defs s_storm_defs[] = {
 
 	/* Xstorm */
 	{'X', BLOCK_XSEM,
-	 {DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX}, false,
+	 {DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX,
+	  DBG_BUS_CLIENT_RBCX}, false,
 	 XSEM_REG_FAST_MEMORY,
 	 XSEM_REG_DBG_FRAME_MODE_BB_K2, XSEM_REG_SLOW_DBG_ACTIVE_BB_K2,
 	 XSEM_REG_SLOW_DBG_MODE_BB_K2, XSEM_REG_DBG_MODE1_CFG_BB_K2,
@@ -550,7 +544,8 @@ static struct storm_defs s_storm_defs[] = {
 
 	/* Ystorm */
 	{'Y', BLOCK_YSEM,
-	 {DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY}, false,
+	 {DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY,
+	  DBG_BUS_CLIENT_RBCY}, false,
 	 YSEM_REG_FAST_MEMORY,
 	 YSEM_REG_DBG_FRAME_MODE_BB_K2, YSEM_REG_SLOW_DBG_ACTIVE_BB_K2,
 	 YSEM_REG_SLOW_DBG_MODE_BB_K2, YSEM_REG_DBG_MODE1_CFG_BB_K2,
@@ -563,7 +558,8 @@ static struct storm_defs s_storm_defs[] = {
 
 	/* Pstorm */
 	{'P', BLOCK_PSEM,
-	 {DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS}, true,
+	 {DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS,
+	  DBG_BUS_CLIENT_RBCS}, true,
 	 PSEM_REG_FAST_MEMORY,
 	 PSEM_REG_DBG_FRAME_MODE_BB_K2, PSEM_REG_SLOW_DBG_ACTIVE_BB_K2,
 	 PSEM_REG_SLOW_DBG_MODE_BB_K2, PSEM_REG_DBG_MODE1_CFG_BB_K2,
@@ -579,8 +575,8 @@ static struct storm_defs s_storm_defs[] = {
 
 static struct block_defs block_grc_defs = {
 	"grc",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN},
 	GRC_REG_DBG_SELECT, GRC_REG_DBG_DWORD_ENABLE,
 	GRC_REG_DBG_SHIFT, GRC_REG_DBG_FORCE_VALID,
 	GRC_REG_DBG_FORCE_FRAME,
@@ -588,30 +584,30 @@ static struct block_defs block_grc_defs = {
 };
 
 static struct block_defs block_miscs_defs = {
-	"miscs", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"miscs", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_misc_defs = {
-	"misc", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"misc", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_dbu_defs = {
-	"dbu", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"dbu", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_pglue_b_defs = {
 	"pglue_b",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCH, DBG_BUS_CLIENT_RBCH},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCH, DBG_BUS_CLIENT_RBCH, DBG_BUS_CLIENT_RBCH},
 	PGLUE_B_REG_DBG_SELECT, PGLUE_B_REG_DBG_DWORD_ENABLE,
 	PGLUE_B_REG_DBG_SHIFT, PGLUE_B_REG_DBG_FORCE_VALID,
 	PGLUE_B_REG_DBG_FORCE_FRAME,
@@ -620,25 +616,26 @@ static struct block_defs block_pglue_b_defs = {
 
 static struct block_defs block_cnig_defs = {
 	"cnig",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW},
-	CNIG_REG_DBG_SELECT_K2, CNIG_REG_DBG_DWORD_ENABLE_K2,
-	CNIG_REG_DBG_SHIFT_K2, CNIG_REG_DBG_FORCE_VALID_K2,
-	CNIG_REG_DBG_FORCE_FRAME_K2,
+	{true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW,
+	 DBG_BUS_CLIENT_RBCW},
+	CNIG_REG_DBG_SELECT_K2_E5, CNIG_REG_DBG_DWORD_ENABLE_K2_E5,
+	CNIG_REG_DBG_SHIFT_K2_E5, CNIG_REG_DBG_FORCE_VALID_K2_E5,
+	CNIG_REG_DBG_FORCE_FRAME_K2_E5,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 0
 };
 
 static struct block_defs block_cpmu_defs = {
-	"cpmu", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"cpmu", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 8
 };
 
 static struct block_defs block_ncsi_defs = {
 	"ncsi",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
 	NCSI_REG_DBG_SELECT, NCSI_REG_DBG_DWORD_ENABLE,
 	NCSI_REG_DBG_SHIFT, NCSI_REG_DBG_FORCE_VALID,
 	NCSI_REG_DBG_FORCE_FRAME,
@@ -646,16 +643,16 @@ static struct block_defs block_ncsi_defs = {
 };
 
 static struct block_defs block_opte_defs = {
-	"opte", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"opte", {true, true, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 4
 };
 
 static struct block_defs block_bmb_defs = {
 	"bmb",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCB},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCB, DBG_BUS_CLIENT_RBCB},
 	BMB_REG_DBG_SELECT, BMB_REG_DBG_DWORD_ENABLE,
 	BMB_REG_DBG_SHIFT, BMB_REG_DBG_FORCE_VALID,
 	BMB_REG_DBG_FORCE_FRAME,
@@ -664,27 +661,28 @@ static struct block_defs block_bmb_defs = {
 
 static struct block_defs block_pcie_defs = {
 	"pcie",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH},
-	PCIE_REG_DBG_COMMON_SELECT_K2,
-	PCIE_REG_DBG_COMMON_DWORD_ENABLE_K2,
-	PCIE_REG_DBG_COMMON_SHIFT_K2,
-	PCIE_REG_DBG_COMMON_FORCE_VALID_K2,
-	PCIE_REG_DBG_COMMON_FORCE_FRAME_K2,
+	{true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH,
+	 DBG_BUS_CLIENT_RBCH},
+	PCIE_REG_DBG_COMMON_SELECT_K2_E5,
+	PCIE_REG_DBG_COMMON_DWORD_ENABLE_K2_E5,
+	PCIE_REG_DBG_COMMON_SHIFT_K2_E5,
+	PCIE_REG_DBG_COMMON_FORCE_VALID_K2_E5,
+	PCIE_REG_DBG_COMMON_FORCE_FRAME_K2_E5,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_mcp_defs = {
-	"mcp", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"mcp", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_mcp2_defs = {
 	"mcp2",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
 	MCP2_REG_DBG_SELECT, MCP2_REG_DBG_DWORD_ENABLE,
 	MCP2_REG_DBG_SHIFT, MCP2_REG_DBG_FORCE_VALID,
 	MCP2_REG_DBG_FORCE_FRAME,
@@ -693,8 +691,8 @@ static struct block_defs block_mcp2_defs = {
 
 static struct block_defs block_pswhst_defs = {
 	"pswhst",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PSWHST_REG_DBG_SELECT, PSWHST_REG_DBG_DWORD_ENABLE,
 	PSWHST_REG_DBG_SHIFT, PSWHST_REG_DBG_FORCE_VALID,
 	PSWHST_REG_DBG_FORCE_FRAME,
@@ -703,8 +701,8 @@ static struct block_defs block_pswhst_defs = {
 
 static struct block_defs block_pswhst2_defs = {
 	"pswhst2",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PSWHST2_REG_DBG_SELECT, PSWHST2_REG_DBG_DWORD_ENABLE,
 	PSWHST2_REG_DBG_SHIFT, PSWHST2_REG_DBG_FORCE_VALID,
 	PSWHST2_REG_DBG_FORCE_FRAME,
@@ -713,8 +711,8 @@ static struct block_defs block_pswhst2_defs = {
 
 static struct block_defs block_pswrd_defs = {
 	"pswrd",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PSWRD_REG_DBG_SELECT, PSWRD_REG_DBG_DWORD_ENABLE,
 	PSWRD_REG_DBG_SHIFT, PSWRD_REG_DBG_FORCE_VALID,
 	PSWRD_REG_DBG_FORCE_FRAME,
@@ -723,8 +721,8 @@ static struct block_defs block_pswrd_defs = {
 
 static struct block_defs block_pswrd2_defs = {
 	"pswrd2",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PSWRD2_REG_DBG_SELECT, PSWRD2_REG_DBG_DWORD_ENABLE,
 	PSWRD2_REG_DBG_SHIFT, PSWRD2_REG_DBG_FORCE_VALID,
 	PSWRD2_REG_DBG_FORCE_FRAME,
@@ -733,8 +731,8 @@ static struct block_defs block_pswrd2_defs = {
 
 static struct block_defs block_pswwr_defs = {
 	"pswwr",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PSWWR_REG_DBG_SELECT, PSWWR_REG_DBG_DWORD_ENABLE,
 	PSWWR_REG_DBG_SHIFT, PSWWR_REG_DBG_FORCE_VALID,
 	PSWWR_REG_DBG_FORCE_FRAME,
@@ -742,16 +740,16 @@ static struct block_defs block_pswwr_defs = {
 };
 
 static struct block_defs block_pswwr2_defs = {
-	"pswwr2", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"pswwr2", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	true, false, DBG_RESET_REG_MISC_PL_HV, 3
 };
 
 static struct block_defs block_pswrq_defs = {
 	"pswrq",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PSWRQ_REG_DBG_SELECT, PSWRQ_REG_DBG_DWORD_ENABLE,
 	PSWRQ_REG_DBG_SHIFT, PSWRQ_REG_DBG_FORCE_VALID,
 	PSWRQ_REG_DBG_FORCE_FRAME,
@@ -760,8 +758,8 @@ static struct block_defs block_pswrq_defs = {
 
 static struct block_defs block_pswrq2_defs = {
 	"pswrq2",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PSWRQ2_REG_DBG_SELECT, PSWRQ2_REG_DBG_DWORD_ENABLE,
 	PSWRQ2_REG_DBG_SHIFT, PSWRQ2_REG_DBG_FORCE_VALID,
 	PSWRQ2_REG_DBG_FORCE_FRAME,
@@ -770,18 +768,19 @@ static struct block_defs block_pswrq2_defs = {
 
 static struct block_defs block_pglcs_defs = {
 	"pglcs",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH},
-	PGLCS_REG_DBG_SELECT_K2, PGLCS_REG_DBG_DWORD_ENABLE_K2,
-	PGLCS_REG_DBG_SHIFT_K2, PGLCS_REG_DBG_FORCE_VALID_K2,
-	PGLCS_REG_DBG_FORCE_FRAME_K2,
+	{true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH,
+	 DBG_BUS_CLIENT_RBCH},
+	PGLCS_REG_DBG_SELECT_K2_E5, PGLCS_REG_DBG_DWORD_ENABLE_K2_E5,
+	PGLCS_REG_DBG_SHIFT_K2_E5, PGLCS_REG_DBG_FORCE_VALID_K2_E5,
+	PGLCS_REG_DBG_FORCE_FRAME_K2_E5,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 2
 };
 
 static struct block_defs block_ptu_defs = {
 	"ptu",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	PTU_REG_DBG_SELECT, PTU_REG_DBG_DWORD_ENABLE,
 	PTU_REG_DBG_SHIFT, PTU_REG_DBG_FORCE_VALID,
 	PTU_REG_DBG_FORCE_FRAME,
@@ -790,8 +789,8 @@ static struct block_defs block_ptu_defs = {
 
 static struct block_defs block_dmae_defs = {
 	"dmae",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	DMAE_REG_DBG_SELECT, DMAE_REG_DBG_DWORD_ENABLE,
 	DMAE_REG_DBG_SHIFT, DMAE_REG_DBG_FORCE_VALID,
 	DMAE_REG_DBG_FORCE_FRAME,
@@ -800,8 +799,8 @@ static struct block_defs block_dmae_defs = {
 
 static struct block_defs block_tcm_defs = {
 	"tcm",
-	{true, true}, true, DBG_TSTORM_ID,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	{true, true, true}, true, DBG_TSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
 	TCM_REG_DBG_SELECT, TCM_REG_DBG_DWORD_ENABLE,
 	TCM_REG_DBG_SHIFT, TCM_REG_DBG_FORCE_VALID,
 	TCM_REG_DBG_FORCE_FRAME,
@@ -810,8 +809,8 @@ static struct block_defs block_tcm_defs = {
 
 static struct block_defs block_mcm_defs = {
 	"mcm",
-	{true, true}, true, DBG_MSTORM_ID,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	{true, true, true}, true, DBG_MSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM, DBG_BUS_CLIENT_RBCM},
 	MCM_REG_DBG_SELECT, MCM_REG_DBG_DWORD_ENABLE,
 	MCM_REG_DBG_SHIFT, MCM_REG_DBG_FORCE_VALID,
 	MCM_REG_DBG_FORCE_FRAME,
@@ -820,8 +819,8 @@ static struct block_defs block_mcm_defs = {
 
 static struct block_defs block_ucm_defs = {
 	"ucm",
-	{true, true}, true, DBG_USTORM_ID,
-	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	{true, true, true}, true, DBG_USTORM_ID,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
 	UCM_REG_DBG_SELECT, UCM_REG_DBG_DWORD_ENABLE,
 	UCM_REG_DBG_SHIFT, UCM_REG_DBG_FORCE_VALID,
 	UCM_REG_DBG_FORCE_FRAME,
@@ -830,8 +829,8 @@ static struct block_defs block_ucm_defs = {
 
 static struct block_defs block_xcm_defs = {
 	"xcm",
-	{true, true}, true, DBG_XSTORM_ID,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	{true, true, true}, true, DBG_XSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
 	XCM_REG_DBG_SELECT, XCM_REG_DBG_DWORD_ENABLE,
 	XCM_REG_DBG_SHIFT, XCM_REG_DBG_FORCE_VALID,
 	XCM_REG_DBG_FORCE_FRAME,
@@ -840,8 +839,8 @@ static struct block_defs block_xcm_defs = {
 
 static struct block_defs block_ycm_defs = {
 	"ycm",
-	{true, true}, true, DBG_YSTORM_ID,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	{true, true, true}, true, DBG_YSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY, DBG_BUS_CLIENT_RBCY},
 	YCM_REG_DBG_SELECT, YCM_REG_DBG_DWORD_ENABLE,
 	YCM_REG_DBG_SHIFT, YCM_REG_DBG_FORCE_VALID,
 	YCM_REG_DBG_FORCE_FRAME,
@@ -850,8 +849,8 @@ static struct block_defs block_ycm_defs = {
 
 static struct block_defs block_pcm_defs = {
 	"pcm",
-	{true, true}, true, DBG_PSTORM_ID,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	{true, true, true}, true, DBG_PSTORM_ID,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
 	PCM_REG_DBG_SELECT, PCM_REG_DBG_DWORD_ENABLE,
 	PCM_REG_DBG_SHIFT, PCM_REG_DBG_FORCE_VALID,
 	PCM_REG_DBG_FORCE_FRAME,
@@ -860,8 +859,8 @@ static struct block_defs block_pcm_defs = {
 
 static struct block_defs block_qm_defs = {
 	"qm",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCQ},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCQ, DBG_BUS_CLIENT_RBCQ},
 	QM_REG_DBG_SELECT, QM_REG_DBG_DWORD_ENABLE,
 	QM_REG_DBG_SHIFT, QM_REG_DBG_FORCE_VALID,
 	QM_REG_DBG_FORCE_FRAME,
@@ -870,8 +869,8 @@ static struct block_defs block_qm_defs = {
 
 static struct block_defs block_tm_defs = {
 	"tm",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
 	TM_REG_DBG_SELECT, TM_REG_DBG_DWORD_ENABLE,
 	TM_REG_DBG_SHIFT, TM_REG_DBG_FORCE_VALID,
 	TM_REG_DBG_FORCE_FRAME,
@@ -880,8 +879,8 @@ static struct block_defs block_tm_defs = {
 
 static struct block_defs block_dorq_defs = {
 	"dorq",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY, DBG_BUS_CLIENT_RBCY},
 	DORQ_REG_DBG_SELECT, DORQ_REG_DBG_DWORD_ENABLE,
 	DORQ_REG_DBG_SHIFT, DORQ_REG_DBG_FORCE_VALID,
 	DORQ_REG_DBG_FORCE_FRAME,
@@ -890,8 +889,8 @@ static struct block_defs block_dorq_defs = {
 
 static struct block_defs block_brb_defs = {
 	"brb",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR},
 	BRB_REG_DBG_SELECT, BRB_REG_DBG_DWORD_ENABLE,
 	BRB_REG_DBG_SHIFT, BRB_REG_DBG_FORCE_VALID,
 	BRB_REG_DBG_FORCE_FRAME,
@@ -900,8 +899,8 @@ static struct block_defs block_brb_defs = {
 
 static struct block_defs block_src_defs = {
 	"src",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
 	SRC_REG_DBG_SELECT, SRC_REG_DBG_DWORD_ENABLE,
 	SRC_REG_DBG_SHIFT, SRC_REG_DBG_FORCE_VALID,
 	SRC_REG_DBG_FORCE_FRAME,
@@ -910,8 +909,8 @@ static struct block_defs block_src_defs = {
 
 static struct block_defs block_prs_defs = {
 	"prs",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCR},
 	PRS_REG_DBG_SELECT, PRS_REG_DBG_DWORD_ENABLE,
 	PRS_REG_DBG_SHIFT, PRS_REG_DBG_FORCE_VALID,
 	PRS_REG_DBG_FORCE_FRAME,
@@ -920,8 +919,8 @@ static struct block_defs block_prs_defs = {
 
 static struct block_defs block_tsdm_defs = {
 	"tsdm",
-	{true, true}, true, DBG_TSTORM_ID,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	{true, true, true}, true, DBG_TSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
 	TSDM_REG_DBG_SELECT, TSDM_REG_DBG_DWORD_ENABLE,
 	TSDM_REG_DBG_SHIFT, TSDM_REG_DBG_FORCE_VALID,
 	TSDM_REG_DBG_FORCE_FRAME,
@@ -930,8 +929,8 @@ static struct block_defs block_tsdm_defs = {
 
 static struct block_defs block_msdm_defs = {
 	"msdm",
-	{true, true}, true, DBG_MSTORM_ID,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	{true, true, true}, true, DBG_MSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM, DBG_BUS_CLIENT_RBCM},
 	MSDM_REG_DBG_SELECT, MSDM_REG_DBG_DWORD_ENABLE,
 	MSDM_REG_DBG_SHIFT, MSDM_REG_DBG_FORCE_VALID,
 	MSDM_REG_DBG_FORCE_FRAME,
@@ -940,8 +939,8 @@ static struct block_defs block_msdm_defs = {
 
 static struct block_defs block_usdm_defs = {
 	"usdm",
-	{true, true}, true, DBG_USTORM_ID,
-	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	{true, true, true}, true, DBG_USTORM_ID,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
 	USDM_REG_DBG_SELECT, USDM_REG_DBG_DWORD_ENABLE,
 	USDM_REG_DBG_SHIFT, USDM_REG_DBG_FORCE_VALID,
 	USDM_REG_DBG_FORCE_FRAME,
@@ -950,8 +949,8 @@ static struct block_defs block_usdm_defs = {
 
 static struct block_defs block_xsdm_defs = {
 	"xsdm",
-	{true, true}, true, DBG_XSTORM_ID,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	{true, true, true}, true, DBG_XSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
 	XSDM_REG_DBG_SELECT, XSDM_REG_DBG_DWORD_ENABLE,
 	XSDM_REG_DBG_SHIFT, XSDM_REG_DBG_FORCE_VALID,
 	XSDM_REG_DBG_FORCE_FRAME,
@@ -960,8 +959,8 @@ static struct block_defs block_xsdm_defs = {
 
 static struct block_defs block_ysdm_defs = {
 	"ysdm",
-	{true, true}, true, DBG_YSTORM_ID,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	{true, true, true}, true, DBG_YSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY, DBG_BUS_CLIENT_RBCY},
 	YSDM_REG_DBG_SELECT, YSDM_REG_DBG_DWORD_ENABLE,
 	YSDM_REG_DBG_SHIFT, YSDM_REG_DBG_FORCE_VALID,
 	YSDM_REG_DBG_FORCE_FRAME,
@@ -970,8 +969,8 @@ static struct block_defs block_ysdm_defs = {
 
 static struct block_defs block_psdm_defs = {
 	"psdm",
-	{true, true}, true, DBG_PSTORM_ID,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	{true, true, true}, true, DBG_PSTORM_ID,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
 	PSDM_REG_DBG_SELECT, PSDM_REG_DBG_DWORD_ENABLE,
 	PSDM_REG_DBG_SHIFT, PSDM_REG_DBG_FORCE_VALID,
 	PSDM_REG_DBG_FORCE_FRAME,
@@ -980,8 +979,8 @@ static struct block_defs block_psdm_defs = {
 
 static struct block_defs block_tsem_defs = {
 	"tsem",
-	{true, true}, true, DBG_TSTORM_ID,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	{true, true, true}, true, DBG_TSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
 	TSEM_REG_DBG_SELECT, TSEM_REG_DBG_DWORD_ENABLE,
 	TSEM_REG_DBG_SHIFT, TSEM_REG_DBG_FORCE_VALID,
 	TSEM_REG_DBG_FORCE_FRAME,
@@ -990,8 +989,8 @@ static struct block_defs block_tsem_defs = {
 
 static struct block_defs block_msem_defs = {
 	"msem",
-	{true, true}, true, DBG_MSTORM_ID,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	{true, true, true}, true, DBG_MSTORM_ID,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM, DBG_BUS_CLIENT_RBCM},
 	MSEM_REG_DBG_SELECT, MSEM_REG_DBG_DWORD_ENABLE,
 	MSEM_REG_DBG_SHIFT, MSEM_REG_DBG_FORCE_VALID,
 	MSEM_REG_DBG_FORCE_FRAME,
@@ -1000,8 +999,8 @@ static struct block_defs block_msem_defs = {
 
 static struct block_defs block_usem_defs = {
 	"usem",
-	{true, true}, true, DBG_USTORM_ID,
-	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	{true, true, true}, true, DBG_USTORM_ID,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
 	USEM_REG_DBG_SELECT, USEM_REG_DBG_DWORD_ENABLE,
 	USEM_REG_DBG_SHIFT, USEM_REG_DBG_FORCE_VALID,
 	USEM_REG_DBG_FORCE_FRAME,
@@ -1010,8 +1009,8 @@ static struct block_defs block_usem_defs = {
 
 static struct block_defs block_xsem_defs = {
 	"xsem",
-	{true, true}, true, DBG_XSTORM_ID,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	{true, true, true}, true, DBG_XSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
 	XSEM_REG_DBG_SELECT, XSEM_REG_DBG_DWORD_ENABLE,
 	XSEM_REG_DBG_SHIFT, XSEM_REG_DBG_FORCE_VALID,
 	XSEM_REG_DBG_FORCE_FRAME,
@@ -1020,8 +1019,8 @@ static struct block_defs block_xsem_defs = {
 
 static struct block_defs block_ysem_defs = {
 	"ysem",
-	{true, true}, true, DBG_YSTORM_ID,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY},
+	{true, true, true}, true, DBG_YSTORM_ID,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCY, DBG_BUS_CLIENT_RBCY},
 	YSEM_REG_DBG_SELECT, YSEM_REG_DBG_DWORD_ENABLE,
 	YSEM_REG_DBG_SHIFT, YSEM_REG_DBG_FORCE_VALID,
 	YSEM_REG_DBG_FORCE_FRAME,
@@ -1030,8 +1029,8 @@ static struct block_defs block_ysem_defs = {
 
 static struct block_defs block_psem_defs = {
 	"psem",
-	{true, true}, true, DBG_PSTORM_ID,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	{true, true, true}, true, DBG_PSTORM_ID,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
 	PSEM_REG_DBG_SELECT, PSEM_REG_DBG_DWORD_ENABLE,
 	PSEM_REG_DBG_SHIFT, PSEM_REG_DBG_FORCE_VALID,
 	PSEM_REG_DBG_FORCE_FRAME,
@@ -1040,8 +1039,8 @@ static struct block_defs block_psem_defs = {
 
 static struct block_defs block_rss_defs = {
 	"rss",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCT},
 	RSS_REG_DBG_SELECT, RSS_REG_DBG_DWORD_ENABLE,
 	RSS_REG_DBG_SHIFT, RSS_REG_DBG_FORCE_VALID,
 	RSS_REG_DBG_FORCE_FRAME,
@@ -1050,8 +1049,8 @@ static struct block_defs block_rss_defs = {
 
 static struct block_defs block_tmld_defs = {
 	"tmld",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM, DBG_BUS_CLIENT_RBCM},
 	TMLD_REG_DBG_SELECT, TMLD_REG_DBG_DWORD_ENABLE,
 	TMLD_REG_DBG_SHIFT, TMLD_REG_DBG_FORCE_VALID,
 	TMLD_REG_DBG_FORCE_FRAME,
@@ -1060,8 +1059,8 @@ static struct block_defs block_tmld_defs = {
 
 static struct block_defs block_muld_defs = {
 	"muld",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
 	MULD_REG_DBG_SELECT, MULD_REG_DBG_DWORD_ENABLE,
 	MULD_REG_DBG_SHIFT, MULD_REG_DBG_FORCE_VALID,
 	MULD_REG_DBG_FORCE_FRAME,
@@ -1070,8 +1069,9 @@ static struct block_defs block_muld_defs = {
 
 static struct block_defs block_yuld_defs = {
 	"yuld",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU},
+	{true, true, false}, false, 0,
+	{DBG_BUS_CLIENT_RBCU, DBG_BUS_CLIENT_RBCU,
+	 MAX_DBG_BUS_CLIENTS},
 	YULD_REG_DBG_SELECT_BB_K2, YULD_REG_DBG_DWORD_ENABLE_BB_K2,
 	YULD_REG_DBG_SHIFT_BB_K2, YULD_REG_DBG_FORCE_VALID_BB_K2,
 	YULD_REG_DBG_FORCE_FRAME_BB_K2,
@@ -1081,18 +1081,40 @@ static struct block_defs block_yuld_defs = {
 
 static struct block_defs block_xyld_defs = {
 	"xyld",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX, DBG_BUS_CLIENT_RBCX},
 	XYLD_REG_DBG_SELECT, XYLD_REG_DBG_DWORD_ENABLE,
 	XYLD_REG_DBG_SHIFT, XYLD_REG_DBG_FORCE_VALID,
 	XYLD_REG_DBG_FORCE_FRAME,
 	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 12
 };
 
+static struct block_defs block_ptld_defs = {
+	"ptld",
+	{false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCT},
+	PTLD_REG_DBG_SELECT_E5, PTLD_REG_DBG_DWORD_ENABLE_E5,
+	PTLD_REG_DBG_SHIFT_E5, PTLD_REG_DBG_FORCE_VALID_E5,
+	PTLD_REG_DBG_FORCE_FRAME_E5,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2,
+	28
+};
+
+static struct block_defs block_ypld_defs = {
+	"ypld",
+	{false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCS},
+	YPLD_REG_DBG_SELECT_E5, YPLD_REG_DBG_DWORD_ENABLE_E5,
+	YPLD_REG_DBG_SHIFT_E5, YPLD_REG_DBG_FORCE_VALID_E5,
+	YPLD_REG_DBG_FORCE_FRAME_E5,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2,
+	27
+};
+
 static struct block_defs block_prm_defs = {
 	"prm",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM, DBG_BUS_CLIENT_RBCM},
 	PRM_REG_DBG_SELECT, PRM_REG_DBG_DWORD_ENABLE,
 	PRM_REG_DBG_SHIFT, PRM_REG_DBG_FORCE_VALID,
 	PRM_REG_DBG_FORCE_FRAME,
@@ -1101,8 +1123,8 @@ static struct block_defs block_prm_defs = {
 
 static struct block_defs block_pbf_pb1_defs = {
 	"pbf_pb1",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV, DBG_BUS_CLIENT_RBCV},
 	PBF_PB1_REG_DBG_SELECT, PBF_PB1_REG_DBG_DWORD_ENABLE,
 	PBF_PB1_REG_DBG_SHIFT, PBF_PB1_REG_DBG_FORCE_VALID,
 	PBF_PB1_REG_DBG_FORCE_FRAME,
@@ -1112,8 +1134,8 @@ static struct block_defs block_pbf_pb1_defs = {
 
 static struct block_defs block_pbf_pb2_defs = {
 	"pbf_pb2",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV, DBG_BUS_CLIENT_RBCV},
 	PBF_PB2_REG_DBG_SELECT, PBF_PB2_REG_DBG_DWORD_ENABLE,
 	PBF_PB2_REG_DBG_SHIFT, PBF_PB2_REG_DBG_FORCE_VALID,
 	PBF_PB2_REG_DBG_FORCE_FRAME,
@@ -1123,8 +1145,8 @@ static struct block_defs block_pbf_pb2_defs = {
 
 static struct block_defs block_rpb_defs = {
 	"rpb",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM, DBG_BUS_CLIENT_RBCM},
 	RPB_REG_DBG_SELECT, RPB_REG_DBG_DWORD_ENABLE,
 	RPB_REG_DBG_SHIFT, RPB_REG_DBG_FORCE_VALID,
 	RPB_REG_DBG_FORCE_FRAME,
@@ -1133,8 +1155,8 @@ static struct block_defs block_rpb_defs = {
 
 static struct block_defs block_btb_defs = {
 	"btb",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCV},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCR, DBG_BUS_CLIENT_RBCV, DBG_BUS_CLIENT_RBCV},
 	BTB_REG_DBG_SELECT, BTB_REG_DBG_DWORD_ENABLE,
 	BTB_REG_DBG_SHIFT, BTB_REG_DBG_FORCE_VALID,
 	BTB_REG_DBG_FORCE_FRAME,
@@ -1143,8 +1165,8 @@ static struct block_defs block_btb_defs = {
 
 static struct block_defs block_pbf_defs = {
 	"pbf",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCV, DBG_BUS_CLIENT_RBCV},
 	PBF_REG_DBG_SELECT, PBF_REG_DBG_DWORD_ENABLE,
 	PBF_REG_DBG_SHIFT, PBF_REG_DBG_FORCE_VALID,
 	PBF_REG_DBG_FORCE_FRAME,
@@ -1153,8 +1175,8 @@ static struct block_defs block_pbf_defs = {
 
 static struct block_defs block_rdif_defs = {
 	"rdif",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCT, DBG_BUS_CLIENT_RBCM, DBG_BUS_CLIENT_RBCM},
 	RDIF_REG_DBG_SELECT, RDIF_REG_DBG_DWORD_ENABLE,
 	RDIF_REG_DBG_SHIFT, RDIF_REG_DBG_FORCE_VALID,
 	RDIF_REG_DBG_FORCE_FRAME,
@@ -1163,8 +1185,8 @@ static struct block_defs block_rdif_defs = {
 
 static struct block_defs block_tdif_defs = {
 	"tdif",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS, DBG_BUS_CLIENT_RBCS},
 	TDIF_REG_DBG_SELECT, TDIF_REG_DBG_DWORD_ENABLE,
 	TDIF_REG_DBG_SHIFT, TDIF_REG_DBG_FORCE_VALID,
 	TDIF_REG_DBG_FORCE_FRAME,
@@ -1173,8 +1195,8 @@ static struct block_defs block_tdif_defs = {
 
 static struct block_defs block_cdu_defs = {
 	"cdu",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
 	CDU_REG_DBG_SELECT, CDU_REG_DBG_DWORD_ENABLE,
 	CDU_REG_DBG_SHIFT, CDU_REG_DBG_FORCE_VALID,
 	CDU_REG_DBG_FORCE_FRAME,
@@ -1183,8 +1205,8 @@ static struct block_defs block_cdu_defs = {
 
 static struct block_defs block_ccfc_defs = {
 	"ccfc",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
 	CCFC_REG_DBG_SELECT, CCFC_REG_DBG_DWORD_ENABLE,
 	CCFC_REG_DBG_SHIFT, CCFC_REG_DBG_FORCE_VALID,
 	CCFC_REG_DBG_FORCE_FRAME,
@@ -1193,8 +1215,8 @@ static struct block_defs block_ccfc_defs = {
 
 static struct block_defs block_tcfc_defs = {
 	"tcfc",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF, DBG_BUS_CLIENT_RBCF},
 	TCFC_REG_DBG_SELECT, TCFC_REG_DBG_DWORD_ENABLE,
 	TCFC_REG_DBG_SHIFT, TCFC_REG_DBG_FORCE_VALID,
 	TCFC_REG_DBG_FORCE_FRAME,
@@ -1203,8 +1225,8 @@ static struct block_defs block_tcfc_defs = {
 
 static struct block_defs block_igu_defs = {
 	"igu",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	IGU_REG_DBG_SELECT, IGU_REG_DBG_DWORD_ENABLE,
 	IGU_REG_DBG_SHIFT, IGU_REG_DBG_FORCE_VALID,
 	IGU_REG_DBG_FORCE_FRAME,
@@ -1213,42 +1235,79 @@ static struct block_defs block_igu_defs = {
 
 static struct block_defs block_cau_defs = {
 	"cau",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP, DBG_BUS_CLIENT_RBCP},
 	CAU_REG_DBG_SELECT, CAU_REG_DBG_DWORD_ENABLE,
 	CAU_REG_DBG_SHIFT, CAU_REG_DBG_FORCE_VALID,
 	CAU_REG_DBG_FORCE_FRAME,
 	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 19
 };
 
+static struct block_defs block_rgfs_defs = {
+	"rgfs", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 29
+};
+
+static struct block_defs block_rgsrc_defs = {
+	"rgsrc",
+	{false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH},
+	RGSRC_REG_DBG_SELECT_E5, RGSRC_REG_DBG_DWORD_ENABLE_E5,
+	RGSRC_REG_DBG_SHIFT_E5, RGSRC_REG_DBG_FORCE_VALID_E5,
+	RGSRC_REG_DBG_FORCE_FRAME_E5,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1,
+	30
+};
+
+static struct block_defs block_tgfs_defs = {
+	"tgfs", {false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	0, 0, 0, 0, 0,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_2, 30
+};
+
+static struct block_defs block_tgsrc_defs = {
+	"tgsrc",
+	{false, false, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCV},
+	TGSRC_REG_DBG_SELECT_E5, TGSRC_REG_DBG_DWORD_ENABLE_E5,
+	TGSRC_REG_DBG_SHIFT_E5, TGSRC_REG_DBG_FORCE_VALID_E5,
+	TGSRC_REG_DBG_FORCE_FRAME_E5,
+	true, true, DBG_RESET_REG_MISC_PL_PDA_VMAIN_1,
+	31
+};
+
 static struct block_defs block_umac_defs = {
 	"umac",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ},
-	UMAC_REG_DBG_SELECT_K2, UMAC_REG_DBG_DWORD_ENABLE_K2,
-	UMAC_REG_DBG_SHIFT_K2, UMAC_REG_DBG_FORCE_VALID_K2,
-	UMAC_REG_DBG_FORCE_FRAME_K2,
+	{true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ,
+	 DBG_BUS_CLIENT_RBCZ},
+	UMAC_REG_DBG_SELECT_K2_E5, UMAC_REG_DBG_DWORD_ENABLE_K2_E5,
+	UMAC_REG_DBG_SHIFT_K2_E5, UMAC_REG_DBG_FORCE_VALID_K2_E5,
+	UMAC_REG_DBG_FORCE_FRAME_K2_E5,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 6
 };
 
 static struct block_defs block_xmac_defs = {
-	"xmac", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"xmac", {true, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_dbg_defs = {
-	"dbg", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"dbg", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	true, true, DBG_RESET_REG_MISC_PL_PDA_VAUX, 3
 };
 
 static struct block_defs block_nig_defs = {
 	"nig",
-	{true, true}, false, 0,
-	{DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN},
+	{true, true, true}, false, 0,
+	{DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN, DBG_BUS_CLIENT_RBCN},
 	NIG_REG_DBG_SELECT, NIG_REG_DBG_DWORD_ENABLE,
 	NIG_REG_DBG_SHIFT, NIG_REG_DBG_FORCE_VALID,
 	NIG_REG_DBG_FORCE_FRAME,
@@ -1257,139 +1316,106 @@ static struct block_defs block_nig_defs = {
 
 static struct block_defs block_wol_defs = {
 	"wol",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ},
-	WOL_REG_DBG_SELECT_K2, WOL_REG_DBG_DWORD_ENABLE_K2,
-	WOL_REG_DBG_SHIFT_K2, WOL_REG_DBG_FORCE_VALID_K2,
-	WOL_REG_DBG_FORCE_FRAME_K2,
+	{false, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
+	WOL_REG_DBG_SELECT_K2_E5, WOL_REG_DBG_DWORD_ENABLE_K2_E5,
+	WOL_REG_DBG_SHIFT_K2_E5, WOL_REG_DBG_FORCE_VALID_K2_E5,
+	WOL_REG_DBG_FORCE_FRAME_K2_E5,
 	true, true, DBG_RESET_REG_MISC_PL_PDA_VAUX, 7
 };
 
 static struct block_defs block_bmbn_defs = {
 	"bmbn",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCB},
-	BMBN_REG_DBG_SELECT_K2, BMBN_REG_DBG_DWORD_ENABLE_K2,
-	BMBN_REG_DBG_SHIFT_K2, BMBN_REG_DBG_FORCE_VALID_K2,
-	BMBN_REG_DBG_FORCE_FRAME_K2,
+	{false, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCB,
+	 DBG_BUS_CLIENT_RBCB},
+	BMBN_REG_DBG_SELECT_K2_E5, BMBN_REG_DBG_DWORD_ENABLE_K2_E5,
+	BMBN_REG_DBG_SHIFT_K2_E5, BMBN_REG_DBG_FORCE_VALID_K2_E5,
+	BMBN_REG_DBG_FORCE_FRAME_K2_E5,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_ipc_defs = {
-	"ipc", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"ipc", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	true, false, DBG_RESET_REG_MISCS_PL_UA, 8
 };
 
 static struct block_defs block_nwm_defs = {
 	"nwm",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW},
-	NWM_REG_DBG_SELECT_K2, NWM_REG_DBG_DWORD_ENABLE_K2,
-	NWM_REG_DBG_SHIFT_K2, NWM_REG_DBG_FORCE_VALID_K2,
-	NWM_REG_DBG_FORCE_FRAME_K2,
+	{false, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW, DBG_BUS_CLIENT_RBCW},
+	NWM_REG_DBG_SELECT_K2_E5, NWM_REG_DBG_DWORD_ENABLE_K2_E5,
+	NWM_REG_DBG_SHIFT_K2_E5, NWM_REG_DBG_FORCE_VALID_K2_E5,
+	NWM_REG_DBG_FORCE_FRAME_K2_E5,
 	true, false, DBG_RESET_REG_MISCS_PL_HV_2, 0
 };
 
 static struct block_defs block_nws_defs = {
 	"nws",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW},
-	NWS_REG_DBG_SELECT_K2, NWS_REG_DBG_DWORD_ENABLE_K2,
-	NWS_REG_DBG_SHIFT_K2, NWS_REG_DBG_FORCE_VALID_K2,
-	NWS_REG_DBG_FORCE_FRAME_K2,
+	{false, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCW, DBG_BUS_CLIENT_RBCW},
+	NWS_REG_DBG_SELECT_K2_E5, NWS_REG_DBG_DWORD_ENABLE_K2_E5,
+	NWS_REG_DBG_SHIFT_K2_E5, NWS_REG_DBG_FORCE_VALID_K2_E5,
+	NWS_REG_DBG_FORCE_FRAME_K2_E5,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 12
 };
 
 static struct block_defs block_ms_defs = {
 	"ms",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ},
-	MS_REG_DBG_SELECT_K2, MS_REG_DBG_DWORD_ENABLE_K2,
-	MS_REG_DBG_SHIFT_K2, MS_REG_DBG_FORCE_VALID_K2,
-	MS_REG_DBG_FORCE_FRAME_K2,
+	{false, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCZ, DBG_BUS_CLIENT_RBCZ},
+	MS_REG_DBG_SELECT_K2_E5, MS_REG_DBG_DWORD_ENABLE_K2_E5,
+	MS_REG_DBG_SHIFT_K2_E5, MS_REG_DBG_FORCE_VALID_K2_E5,
+	MS_REG_DBG_FORCE_FRAME_K2_E5,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 13
 };
 
 static struct block_defs block_phy_pcie_defs = {
 	"phy_pcie",
-	{false, true}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH},
-	PCIE_REG_DBG_COMMON_SELECT_K2,
-	PCIE_REG_DBG_COMMON_DWORD_ENABLE_K2,
-	PCIE_REG_DBG_COMMON_SHIFT_K2,
-	PCIE_REG_DBG_COMMON_FORCE_VALID_K2,
-	PCIE_REG_DBG_COMMON_FORCE_FRAME_K2,
+	{false, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, DBG_BUS_CLIENT_RBCH,
+	 DBG_BUS_CLIENT_RBCH},
+	PCIE_REG_DBG_COMMON_SELECT_K2_E5,
+	PCIE_REG_DBG_COMMON_DWORD_ENABLE_K2_E5,
+	PCIE_REG_DBG_COMMON_SHIFT_K2_E5,
+	PCIE_REG_DBG_COMMON_FORCE_VALID_K2_E5,
+	PCIE_REG_DBG_COMMON_FORCE_FRAME_K2_E5,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_led_defs = {
-	"led", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"led", {false, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	true, false, DBG_RESET_REG_MISCS_PL_HV, 14
 };
 
 static struct block_defs block_avs_wrap_defs = {
-	"avs_wrap", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"avs_wrap", {false, true, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	true, false, DBG_RESET_REG_MISCS_PL_UA, 11
 };
 
-static struct block_defs block_rgfs_defs = {
-	"rgfs", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
-	0, 0, 0, 0, 0,
-	false, false, MAX_DBG_RESET_REGS, 0
-};
-
-static struct block_defs block_rgsrc_defs = {
-	"rgsrc", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
-	0, 0, 0, 0, 0,
-	false, false, MAX_DBG_RESET_REGS, 0
-};
-
-static struct block_defs block_tgfs_defs = {
-	"tgfs", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
-	0, 0, 0, 0, 0,
-	false, false, MAX_DBG_RESET_REGS, 0
-};
-
-static struct block_defs block_tgsrc_defs = {
-	"tgsrc", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
-	0, 0, 0, 0, 0,
-	false, false, MAX_DBG_RESET_REGS, 0
-};
-
-static struct block_defs block_ptld_defs = {
-	"ptld", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
-	0, 0, 0, 0, 0,
-	false, false, MAX_DBG_RESET_REGS, 0
-};
-
-static struct block_defs block_ypld_defs = {
-	"ypld", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+static struct block_defs block_pxpreqbus_defs = {
+	"pxpreqbus", {false, false, false}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_misc_aeu_defs = {
-	"misc_aeu", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"misc_aeu", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
 
 static struct block_defs block_bar0_map_defs = {
-	"bar0_map", {false, false}, false, 0,
-	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
+	"bar0_map", {true, true, true}, false, 0,
+	{MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS, MAX_DBG_BUS_CLIENTS},
 	0, 0, 0, 0, 0,
 	false, false, MAX_DBG_RESET_REGS, 0
 };
@@ -1480,164 +1506,160 @@ static struct block_defs *s_block_defs[MAX_BLOCK_ID] = {
 	&block_phy_pcie_defs,
 	&block_led_defs,
 	&block_avs_wrap_defs,
+	&block_pxpreqbus_defs,
 	&block_misc_aeu_defs,
 	&block_bar0_map_defs,
 };
 
 static struct platform_defs s_platform_defs[] = {
-	{"asic", 1},
-	{"reserved", 0},
-	{"reserved2", 0},
-	{"reserved3", 0}
+	{"asic", 1, 256, 32768},
+	{"reserved", 0, 0, 0},
+	{"reserved2", 0, 0, 0},
+	{"reserved3", 0, 0, 0}
 };
 
 static struct grc_param_defs s_grc_param_defs[] = {
 	/* DBG_GRC_PARAM_DUMP_TSTORM */
-	{{1, 1}, 0, 1, false, 1, 1},
+	{{1, 1, 1}, 0, 1, false, 1, 1},
 
 	/* DBG_GRC_PARAM_DUMP_MSTORM */
-	{{1, 1}, 0, 1, false, 1, 1},
+	{{1, 1, 1}, 0, 1, false, 1, 1},
 
 	/* DBG_GRC_PARAM_DUMP_USTORM */
-	{{1, 1}, 0, 1, false, 1, 1},
+	{{1, 1, 1}, 0, 1, false, 1, 1},
 
 	/* DBG_GRC_PARAM_DUMP_XSTORM */
-	{{1, 1}, 0, 1, false, 1, 1},
+	{{1, 1, 1}, 0, 1, false, 1, 1},
 
 	/* DBG_GRC_PARAM_DUMP_YSTORM */
-	{{1, 1}, 0, 1, false, 1, 1},
+	{{1, 1, 1}, 0, 1, false, 1, 1},
 
 	/* DBG_GRC_PARAM_DUMP_PSTORM */
-	{{1, 1}, 0, 1, false, 1, 1},
+	{{1, 1, 1}, 0, 1, false, 1, 1},
 
 	/* DBG_GRC_PARAM_DUMP_REGS */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_RAM */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_PBUF */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_IOR */
-	{{0, 0}, 0, 1, false, 0, 1},
+	{{0, 0, 0}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_VFC */
-	{{0, 0}, 0, 1, false, 0, 1},
+	{{0, 0, 0}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_CM_CTX */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_ILT */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_RSS */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_CAU */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_QM */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_MCP */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_RESERVED */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_CFC */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_IGU */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_BRB */
-	{{0, 0}, 0, 1, false, 0, 1},
+	{{0, 0, 0}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_BTB */
-	{{0, 0}, 0, 1, false, 0, 1},
+	{{0, 0, 0}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_BMB */
-	{{0, 0}, 0, 1, false, 0, 1},
+	{{0, 0, 0}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_NIG */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_MULD */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_PRS */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_DMAE */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_TM */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_SDM */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_DIF */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_STATIC */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_UNSTALL */
-	{{0, 0}, 0, 1, false, 0, 0},
+	{{0, 0, 0}, 0, 1, false, 0, 0},
 
 	/* DBG_GRC_PARAM_NUM_LCIDS */
-	{{MAX_LCIDS, MAX_LCIDS}, 1, MAX_LCIDS, false, MAX_LCIDS,
+	{{MAX_LCIDS, MAX_LCIDS, MAX_LCIDS}, 1, MAX_LCIDS, false, MAX_LCIDS,
 	 MAX_LCIDS},
 
 	/* DBG_GRC_PARAM_NUM_LTIDS */
-	{{MAX_LTIDS, MAX_LTIDS}, 1, MAX_LTIDS, false, MAX_LTIDS,
+	{{MAX_LTIDS, MAX_LTIDS, MAX_LTIDS}, 1, MAX_LTIDS, false, MAX_LTIDS,
 	 MAX_LTIDS},
 
 	/* DBG_GRC_PARAM_EXCLUDE_ALL */
-	{{0, 0}, 0, 1, true, 0, 0},
+	{{0, 0, 0}, 0, 1, true, 0, 0},
 
 	/* DBG_GRC_PARAM_CRASH */
-	{{0, 0}, 0, 1, true, 0, 0},
+	{{0, 0, 0}, 0, 1, true, 0, 0},
 
 	/* DBG_GRC_PARAM_PARITY_SAFE */
-	{{0, 0}, 0, 1, false, 1, 0},
+	{{0, 0, 0}, 0, 1, false, 1, 0},
 
 	/* DBG_GRC_PARAM_DUMP_CM */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_DUMP_PHY */
-	{{1, 1}, 0, 1, false, 0, 1},
+	{{1, 1, 1}, 0, 1, false, 0, 1},
 
 	/* DBG_GRC_PARAM_NO_MCP */
-	{{0, 0}, 0, 1, false, 0, 0},
+	{{0, 0, 0}, 0, 1, false, 0, 0},
 
 	/* DBG_GRC_PARAM_NO_FW_VER */
-	{{0, 0}, 0, 1, false, 0, 0}
+	{{0, 0, 0}, 0, 1, false, 0, 0}
 };
 
 static struct rss_mem_defs s_rss_mem_defs[] = {
-	{ "rss_mem_cid", "rss_cid", 0,
-	  {256, 320},
-	  {32, 32} },
+	{ "rss_mem_cid", "rss_cid", 0, 32,
+	  {256, 320, 512} },
 
-	{ "rss_mem_key_msb", "rss_key", 1024,
-	  {128, 208},
-	  {256, 256} },
+	{ "rss_mem_key_msb", "rss_key", 1024, 256,
+	  {128, 208, 257} },
 
-	{ "rss_mem_key_lsb", "rss_key", 2048,
-	  {128, 208},
-	  {64, 64} },
+	{ "rss_mem_key_lsb", "rss_key", 2048, 64,
+	  {128, 208, 257} },
 
-	{ "rss_mem_info", "rss_info", 3072,
-	  {128, 208},
-	  {16, 16} },
+	{ "rss_mem_info", "rss_info", 3072, 16,
+	  {128, 208, 256} },
 
-	{ "rss_mem_ind", "rss_ind", 4096,
-	  {16384, 26624},
-	  {16, 16} }
+	{ "rss_mem_ind", "rss_ind", 4096, 16,
+	  {16384, 26624, 32768} }
 };
 
 static struct vfc_ram_defs s_vfc_ram_defs[] = {
@@ -1650,72 +1672,75 @@ static struct vfc_ram_defs s_vfc_ram_defs[] = {
 static struct big_ram_defs s_big_ram_defs[] = {
 	{ "BRB", MEM_GROUP_BRB_MEM, MEM_GROUP_BRB_RAM, DBG_GRC_PARAM_DUMP_BRB,
 	  BRB_REG_BIG_RAM_ADDRESS, BRB_REG_BIG_RAM_DATA,
-	  {4800, 5632} },
+	  MISC_REG_BLOCK_256B_EN, {0, 0, 0},
+	  {153600, 180224, 282624} },
 
 	{ "BTB", MEM_GROUP_BTB_MEM, MEM_GROUP_BTB_RAM, DBG_GRC_PARAM_DUMP_BTB,
 	  BTB_REG_BIG_RAM_ADDRESS, BTB_REG_BIG_RAM_DATA,
-	  {2880, 3680} },
+	  MISC_REG_BLOCK_256B_EN, {0, 1, 1},
+	  {92160, 117760, 168960} },
 
 	{ "BMB", MEM_GROUP_BMB_MEM, MEM_GROUP_BMB_RAM, DBG_GRC_PARAM_DUMP_BMB,
 	  BMB_REG_BIG_RAM_ADDRESS, BMB_REG_BIG_RAM_DATA,
-	  {1152, 1152} }
+	  MISCS_REG_BLOCK_256B_EN, {0, 0, 0},
+	  {36864, 36864, 36864} }
 };
 
 static struct reset_reg_defs s_reset_regs_defs[] = {
 	/* DBG_RESET_REG_MISCS_PL_UA */
-	{ MISCS_REG_RESET_PL_UA, 0x0,
-	  {true, true} },
+	{ MISCS_REG_RESET_PL_UA,
+	  {true, true, true}, {0x0, 0x0, 0x0} },
 
 	/* DBG_RESET_REG_MISCS_PL_HV */
-	{ MISCS_REG_RESET_PL_HV, 0x0,
-	  {true, true} },
+	{ MISCS_REG_RESET_PL_HV,
+	  {true, true, true}, {0x0, 0x400, 0x600} },
 
 	/* DBG_RESET_REG_MISCS_PL_HV_2 */
-	{ MISCS_REG_RESET_PL_HV_2_K2, 0x0,
-	  {false, true} },
+	{ MISCS_REG_RESET_PL_HV_2_K2_E5,
+	  {false, true, true}, {0x0, 0x0, 0x0} },
 
 	/* DBG_RESET_REG_MISC_PL_UA */
-	{ MISC_REG_RESET_PL_UA, 0x0,
-	  {true, true} },
+	{ MISC_REG_RESET_PL_UA,
+	  {true, true, true}, {0x0, 0x0, 0x0} },
 
 	/* DBG_RESET_REG_MISC_PL_HV */
-	{ MISC_REG_RESET_PL_HV, 0x0,
-	  {true, true} },
+	{ MISC_REG_RESET_PL_HV,
+	  {true, true, true}, {0x0, 0x0, 0x0} },
 
 	/* DBG_RESET_REG_MISC_PL_PDA_VMAIN_1 */
-	{ MISC_REG_RESET_PL_PDA_VMAIN_1, 0x4404040,
-	  {true, true} },
+	{ MISC_REG_RESET_PL_PDA_VMAIN_1,
+	  {true, true, true}, {0x4404040, 0x4404040, 0x404040} },
 
 	/* DBG_RESET_REG_MISC_PL_PDA_VMAIN_2 */
-	{ MISC_REG_RESET_PL_PDA_VMAIN_2, 0x7c00007,
-	  {true, true} },
+	{ MISC_REG_RESET_PL_PDA_VMAIN_2,
+	  {true, true, true}, {0x7, 0x7c00007, 0x5c08007} },
 
 	/* DBG_RESET_REG_MISC_PL_PDA_VAUX */
-	{ MISC_REG_RESET_PL_PDA_VAUX, 0x2,
-	  {true, true} },
+	{ MISC_REG_RESET_PL_PDA_VAUX,
+	  {true, true, true}, {0x2, 0x2, 0x2} },
 };
 
 static struct phy_defs s_phy_defs[] = {
 	{"nw_phy", NWS_REG_NWS_CMU_K2,
-	 PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_7_0_K2,
-	 PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_15_8_K2,
-	 PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_7_0_K2,
-	 PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_11_8_K2},
-	{"sgmii_phy", MS_REG_MS_CMU_K2,
-	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X132_K2,
-	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X133_K2,
-	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X130_K2,
-	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X131_K2},
-	{"pcie_phy0", PHY_PCIE_REG_PHY0_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131_K2},
-	{"pcie_phy1", PHY_PCIE_REG_PHY1_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130_K2,
-	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131_K2},
+	 PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_7_0_K2_E5,
+	 PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_15_8_K2_E5,
+	 PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_7_0_K2_E5,
+	 PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_11_8_K2_E5},
+	{"sgmii_phy", MS_REG_MS_CMU_K2_E5,
+	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X132_K2_E5,
+	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X133_K2_E5,
+	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X130_K2_E5,
+	 PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X131_K2_E5},
+	{"pcie_phy0", PHY_PCIE_REG_PHY0_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131_K2_E5},
+	{"pcie_phy1", PHY_PCIE_REG_PHY1_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130_K2_E5,
+	 PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131_K2_E5},
 };
 
 /**************************** Private Functions ******************************/
@@ -1774,7 +1799,9 @@ static enum dbg_status qed_dbg_dev_init(struct qed_hwfn *p_hwfn,
 	/* Initializes the GRC parameters */
 	qed_dbg_grc_init_params(p_hwfn);
 
-	dev_data->initialized = true;
+	dev_data->use_dmae = true;
+	dev_data->num_regs_read = 0;
+	dev_data->initialized = 1;
 
 	return DBG_STATUS_OK;
 }
@@ -1807,7 +1834,7 @@ static void qed_read_fw_info(struct qed_hwfn *p_hwfn,
 	 * The address is located in the last line of the Storm RAM.
 	 */
 	addr = storm->sem_fast_mem_addr + SEM_FAST_REG_INT_RAM +
-	       DWORDS_TO_BYTES(SEM_FAST_REG_INT_RAM_SIZE) -
+	       DWORDS_TO_BYTES(SEM_FAST_REG_INT_RAM_SIZE_BB_K2) -
 	       sizeof(fw_info_location);
 	dest = (u32 *)&fw_info_location;
 
@@ -2071,8 +2098,7 @@ static u32 qed_dump_common_global_params(struct qed_hwfn *p_hwfn,
 /* Writes the "last" section (including CRC) to the specified buffer at the
  * given offset. Returns the dumped size in dwords.
  */
-static u32 qed_dump_last_section(struct qed_hwfn *p_hwfn,
-				 u32 *dump_buf, u32 offset, bool dump)
+static u32 qed_dump_last_section(u32 *dump_buf, u32 offset, bool dump)
 {
 	u32 start_offset = offset;
 
@@ -2235,7 +2261,8 @@ static bool qed_grc_is_mem_included(struct qed_hwfn *p_hwfn,
 	case MEM_GROUP_CFC_MEM:
 	case MEM_GROUP_CONN_CFC_MEM:
 	case MEM_GROUP_TASK_CFC_MEM:
-		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_CFC);
+		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_CFC) ||
+		       qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_CM_CTX);
 	case MEM_GROUP_IGU_MEM:
 	case MEM_GROUP_IGU_MSIX:
 		return qed_grc_is_included(p_hwfn, DBG_GRC_PARAM_DUMP_IGU);
@@ -2293,7 +2320,8 @@ static void qed_grc_unreset_blocks(struct qed_hwfn *p_hwfn,
 	for (block_id = 0; block_id < MAX_BLOCK_ID; block_id++) {
 		struct block_defs *block = s_block_defs[block_id];
 
-		if (block->has_reset_bit && block->unreset)
+		if (block->exists[dev_data->chip_id] && block->has_reset_bit &&
+		    block->unreset)
 			reg_val[block->reset_reg] |=
 			    BIT(block->reset_bit_offset);
 	}
@@ -2303,7 +2331,8 @@ static void qed_grc_unreset_blocks(struct qed_hwfn *p_hwfn,
 		if (!s_reset_regs_defs[i].exists[dev_data->chip_id])
 			continue;
 
-		reg_val[i] |= s_reset_regs_defs[i].unreset_val;
+		reg_val[i] |=
+			s_reset_regs_defs[i].unreset_val[dev_data->chip_id];
 
 		if (reg_val[i])
 			qed_wr(p_hwfn,
@@ -2413,6 +2442,18 @@ static u32 qed_grc_dump_regs_hdr(u32 *dump_buf,
 	return offset;
 }
 
+/* Reads the specified registers into the specified buffer.
+ * The addr and len arguments are specified in dwords.
+ */
+void qed_read_regs(struct qed_hwfn *p_hwfn,
+		   struct qed_ptt *p_ptt, u32 *buf, u32 addr, u32 len)
+{
+	u32 i;
+
+	for (i = 0; i < len; i++)
+		buf[i] = qed_rd(p_hwfn, p_ptt, DWORDS_TO_BYTES(addr + i));
+}
+
 /* Dumps the GRC registers in the specified address range.
  * Returns the dumped size in dwords.
  * The addr and len arguments are specified in dwords.
@@ -2422,15 +2463,39 @@ static u32 qed_grc_dump_addr_range(struct qed_hwfn *p_hwfn,
 				   u32 *dump_buf,
 				   bool dump, u32 addr, u32 len, bool wide_bus)
 {
-	u32 byte_addr = DWORDS_TO_BYTES(addr), offset = 0, i;
+	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
 
 	if (!dump)
 		return len;
 
-	for (i = 0; i < len; i++, byte_addr += BYTES_IN_DWORD, offset++)
-		*(dump_buf + offset) = qed_rd(p_hwfn, p_ptt, byte_addr);
+	/* Print log if needed */
+	dev_data->num_regs_read += len;
+	if (dev_data->num_regs_read >=
+	    s_platform_defs[dev_data->platform_id].log_thresh) {
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_DEBUG,
+			   "Dumping %d registers...\n",
+			   dev_data->num_regs_read);
+		dev_data->num_regs_read = 0;
+	}
 
-	return offset;
+	/* Try reading using DMAE */
+	if (dev_data->use_dmae &&
+	    (len >= s_platform_defs[dev_data->platform_id].dmae_thresh ||
+	     wide_bus)) {
+		if (!qed_dmae_grc2host(p_hwfn, p_ptt, DWORDS_TO_BYTES(addr),
+				       (u64)(uintptr_t)(dump_buf), len, 0))
+			return len;
+		dev_data->use_dmae = 0;
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_DEBUG,
+			   "Failed reading from chip using DMAE, using GRC instead\n");
+	}
+
+	/* Read registers */
+	qed_read_regs(p_hwfn, p_ptt, dump_buf, addr, len);
+
+	return len;
 }
 
 /* Dumps GRC registers sequence header. Returns the dumped size in dwords.
@@ -2630,9 +2695,6 @@ static u32 qed_grc_dump_registers(struct qed_hwfn *p_hwfn,
 	chip = &s_chip_defs[dev_data->chip_id];
 	chip_platform = &chip->per_platform[dev_data->platform_id];
 
-	if (dump)
-		DP_VERBOSE(p_hwfn, QED_MSG_DEBUG, "Dumping registers...\n");
-
 	while (input_offset <
 	       s_dbg_arrays[BIN_BUF_DBG_DUMP_REG].size_in_dwords) {
 		const struct dbg_dump_split_hdr *split_hdr;
@@ -2966,22 +3028,12 @@ static u32 qed_grc_dump_mem_hdr(struct qed_hwfn *p_hwfn,
 
 		offset += qed_dump_str_param(dump_buf + offset,
 					     dump, "name", buf);
-		if (dump)
-			DP_VERBOSE(p_hwfn,
-				   QED_MSG_DEBUG,
-				   "Dumping %d registers from %s...\n",
-				   len, buf);
 	} else {
 		/* Dump address */
 		u32 addr_in_bytes = DWORDS_TO_BYTES(addr);
 
 		offset += qed_dump_num_param(dump_buf + offset,
 					     dump, "addr", addr_in_bytes);
-		if (dump && len > 64)
-			DP_VERBOSE(p_hwfn,
-				   QED_MSG_DEBUG,
-				   "Dumping %d registers from address 0x%x...\n",
-				   len, addr_in_bytes);
 	}
 
 	/* Dump len */
@@ -3530,17 +3582,16 @@ static u32 qed_grc_dump_rss(struct qed_hwfn *p_hwfn,
 	u8 rss_mem_id;
 
 	for (rss_mem_id = 0; rss_mem_id < NUM_RSS_MEM_TYPES; rss_mem_id++) {
-		u32 rss_addr, num_entries, entry_width, total_dwords, i;
+		u32 rss_addr, num_entries, total_dwords;
 		struct rss_mem_defs *rss_defs;
-		u32 addr, size;
+		u32 addr, num_dwords_to_read;
 		bool packed;
 
 		rss_defs = &s_rss_mem_defs[rss_mem_id];
 		rss_addr = rss_defs->addr;
 		num_entries = rss_defs->num_entries[dev_data->chip_id];
-		entry_width = rss_defs->entry_width[dev_data->chip_id];
-		total_dwords = (num_entries * entry_width) / 32;
-		packed = (entry_width == 16);
+		total_dwords = (num_entries * rss_defs->entry_width) / 32;
+		packed = (rss_defs->entry_width == 16);
 
 		offset += qed_grc_dump_mem_hdr(p_hwfn,
 					       dump_buf + offset,
@@ -3548,7 +3599,7 @@ static u32 qed_grc_dump_rss(struct qed_hwfn *p_hwfn,
 					       rss_defs->mem_name,
 					       0,
 					       total_dwords,
-					       entry_width,
+					       rss_defs->entry_width,
 					       packed,
 					       rss_defs->type_name, false, 0);
 
@@ -3559,16 +3610,20 @@ static u32 qed_grc_dump_rss(struct qed_hwfn *p_hwfn,
 		}
 
 		addr = BYTES_TO_DWORDS(RSS_REG_RSS_RAM_DATA);
-		size = RSS_REG_RSS_RAM_DATA_SIZE;
-		for (i = 0; i < total_dwords; i += size, rss_addr++) {
+		while (total_dwords) {
+			num_dwords_to_read = min_t(u32,
+						   RSS_REG_RSS_RAM_DATA_SIZE,
+						   total_dwords);
 			qed_wr(p_hwfn, p_ptt, RSS_REG_RSS_RAM_ADDR, rss_addr);
 			offset += qed_grc_dump_addr_range(p_hwfn,
 							  p_ptt,
 							  dump_buf + offset,
 							  dump,
 							  addr,
-							  size,
+							  num_dwords_to_read,
 							  false);
+			total_dwords -= num_dwords_to_read;
+			rss_addr++;
 		}
 	}
 
@@ -3581,19 +3636,21 @@ static u32 qed_grc_dump_big_ram(struct qed_hwfn *p_hwfn,
 				u32 *dump_buf, bool dump, u8 big_ram_id)
 {
 	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
-	u32 total_blocks, ram_size, offset = 0, i;
+	u32 block_size, ram_size, offset = 0, reg_val, i;
 	char mem_name[12] = "???_BIG_RAM";
 	char type_name[8] = "???_RAM";
 	struct big_ram_defs *big_ram;
 
 	big_ram = &s_big_ram_defs[big_ram_id];
-	total_blocks = big_ram->num_of_blocks[dev_data->chip_id];
-	ram_size = total_blocks * BIG_RAM_BLOCK_SIZE_DWORDS;
+	ram_size = big_ram->ram_size[dev_data->chip_id];
 
-	strncpy(type_name, big_ram->instance_name,
-		strlen(big_ram->instance_name));
-	strncpy(mem_name, big_ram->instance_name,
-		strlen(big_ram->instance_name));
+	reg_val = qed_rd(p_hwfn, p_ptt, big_ram->is_256b_reg_addr);
+	block_size = reg_val &
+		     BIT(big_ram->is_256b_bit_offset[dev_data->chip_id]) ? 256
+									 : 128;
+
+	strscpy(type_name, big_ram->instance_name, sizeof(type_name));
+	strscpy(mem_name, big_ram->instance_name, sizeof(mem_name));
 
 	/* Dump memory header */
 	offset += qed_grc_dump_mem_hdr(p_hwfn,
@@ -3602,7 +3659,7 @@ static u32 qed_grc_dump_big_ram(struct qed_hwfn *p_hwfn,
 				       mem_name,
 				       0,
 				       ram_size,
-				       BIG_RAM_BLOCK_SIZE_BYTES * 8,
+				       block_size * 8,
 				       false, type_name, false, 0);
 
 	/* Read and dump Big RAM data */
@@ -3610,12 +3667,13 @@ static u32 qed_grc_dump_big_ram(struct qed_hwfn *p_hwfn,
 		return offset + ram_size;
 
 	/* Dump Big RAM */
-	for (i = 0; i < total_blocks / 2; i++) {
+	for (i = 0; i < DIV_ROUND_UP(ram_size, BRB_REG_BIG_RAM_DATA_SIZE);
+	     i++) {
 		u32 addr, len;
 
 		qed_wr(p_hwfn, p_ptt, big_ram->addr_reg_addr, i);
 		addr = BYTES_TO_DWORDS(big_ram->data_reg_addr);
-		len = 2 * BIG_RAM_BLOCK_SIZE_DWORDS;
+		len = BRB_REG_BIG_RAM_DATA_SIZE;
 		offset += qed_grc_dump_addr_range(p_hwfn,
 						  p_ptt,
 						  dump_buf + offset,
@@ -3649,7 +3707,7 @@ static u32 qed_grc_dump_mcp(struct qed_hwfn *p_hwfn,
 				   dump,
 				   NULL,
 				   BYTES_TO_DWORDS(MCP_REG_SCRATCH),
-				   MCP_REG_SCRATCH_SIZE,
+				   MCP_REG_SCRATCH_SIZE_BB_K2,
 				   false, 0, false, "MCP", false, 0);
 
 	/* Dump MCP cpu_reg_file */
@@ -3710,7 +3768,6 @@ static u32 qed_grc_dump_phy(struct qed_hwfn *p_hwfn,
 			       phy_defs->tbus_data_lo_addr;
 		data_hi_addr = phy_defs->base_addr +
 			       phy_defs->tbus_data_hi_addr;
-		bytes_buf = (u8 *)(dump_buf + offset);
 
 		if (snprintf(mem_name, sizeof(mem_name), "tbus_%s",
 			     phy_defs->phy_name) < 0)
@@ -3730,6 +3787,7 @@ static u32 qed_grc_dump_phy(struct qed_hwfn *p_hwfn,
 			continue;
 		}
 
+		bytes_buf = (u8 *)(dump_buf + offset);
 		for (tbus_hi_offset = 0;
 		     tbus_hi_offset < (NUM_PHY_TBUS_ADDRESSES >> 8);
 		     tbus_hi_offset++) {
@@ -3778,19 +3836,17 @@ static u32 qed_grc_dump_static_debug(struct qed_hwfn *p_hwfn,
 	struct dbg_tools_data *dev_data = &p_hwfn->dbg_info;
 	u32 block_id, line_id, offset = 0;
 
-	/* Skip static debug if a debug bus recording is in progress */
-	if (qed_rd(p_hwfn, p_ptt, DBG_REG_DBG_BLOCK_ON))
+	/* Don't dump static debug if a debug bus recording is in progress */
+	if (dump && qed_rd(p_hwfn, p_ptt, DBG_REG_DBG_BLOCK_ON))
 		return 0;
 
 	if (dump) {
-		DP_VERBOSE(p_hwfn,
-			   QED_MSG_DEBUG, "Dumping static debug data...\n");
-
 		/* Disable all blocks debug output */
 		for (block_id = 0; block_id < MAX_BLOCK_ID; block_id++) {
 			struct block_defs *block = s_block_defs[block_id];
 
-			if (block->has_dbg_bus[dev_data->chip_id])
+			if (block->dbg_client_id[dev_data->chip_id] !=
+			    MAX_DBG_BUS_CLIENTS)
 				qed_wr(p_hwfn, p_ptt, block->dbg_enable_addr,
 				       0);
 		}
@@ -3811,12 +3867,12 @@ static u32 qed_grc_dump_static_debug(struct qed_hwfn *p_hwfn,
 		u32 block_dwords, addr, len;
 		u8 dbg_client_id;
 
-		if (!block->has_dbg_bus[dev_data->chip_id])
+		if (block->dbg_client_id[dev_data->chip_id] ==
+		    MAX_DBG_BUS_CLIENTS)
 			continue;
 
-		block_desc =
-			get_dbg_bus_block_desc(p_hwfn,
-					       (enum block_id)block_id);
+		block_desc = get_dbg_bus_block_desc(p_hwfn,
+						    (enum block_id)block_id);
 		block_dwords = NUM_DBG_LINES(block_desc) *
 			       STATIC_DEBUG_LINE_DWORDS;
 
@@ -4044,7 +4100,7 @@ static enum dbg_status qed_grc_dump(struct qed_hwfn *p_hwfn,
 						    dump_buf + offset, dump);
 
 	/* Dump last section */
-	offset += qed_dump_last_section(p_hwfn, dump_buf, offset, dump);
+	offset += qed_dump_last_section(dump_buf, offset, dump);
 
 	if (dump) {
 		/* Unstall storms */
@@ -4253,30 +4309,33 @@ qed_idle_chk_dump_rule_entries(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 		if (!check_rule && dump)
 			continue;
 
+		if (!dump) {
+			u32 entry_dump_size =
+				qed_idle_chk_dump_failure(p_hwfn,
+							  p_ptt,
+							  dump_buf + offset,
+							  false,
+							  rule->rule_id,
+							  rule,
+							  0,
+							  NULL);
+
+			offset += num_reg_entries * entry_dump_size;
+			(*num_failing_rules) += num_reg_entries;
+			continue;
+		}
+
 		/* Go over all register entries (number of entries is the same
 		 * for all condition registers).
 		 */
 		for (entry_id = 0; entry_id < num_reg_entries; entry_id++) {
 			u32 next_reg_offset = 0;
 
-			if (!dump) {
-				offset += qed_idle_chk_dump_failure(p_hwfn,
-							p_ptt,
-							dump_buf + offset,
-							false,
-							rule->rule_id,
-							rule,
-							entry_id,
-							NULL);
-				(*num_failing_rules)++;
-				break;
-			}
-
 			/* Read current entry of all condition registers */
 			for (reg_id = 0; reg_id < rule->num_cond_regs;
 			     reg_id++) {
 				const struct dbg_idle_chk_cond_reg *reg =
-				    &cond_regs[reg_id];
+					&cond_regs[reg_id];
 				u32 padded_entry_size, addr;
 				bool wide_bus;
 
@@ -4291,9 +4350,9 @@ qed_idle_chk_dump_rule_entries(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 				if (reg->num_entries > 1 ||
 				    reg->start_entry > 0) {
 					padded_entry_size =
-					    reg->entry_size > 1 ?
-					    roundup_pow_of_two(reg->entry_size)
-					    : 1;
+					   reg->entry_size > 1 ?
+					   roundup_pow_of_two(reg->entry_size) :
+					   1;
 					addr += (reg->start_entry + entry_id) *
 						padded_entry_size;
 				}
@@ -4329,7 +4388,6 @@ qed_idle_chk_dump_rule_entries(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 							entry_id,
 							cond_reg_values);
 				(*num_failing_rules)++;
-				break;
 			}
 		}
 	}
@@ -4402,7 +4460,7 @@ static u32 qed_idle_chk_dump(struct qed_hwfn *p_hwfn,
 				   dump, "num_rules", num_failing_rules);
 
 	/* Dump last section */
-	offset += qed_dump_last_section(p_hwfn, dump_buf, offset, dump);
+	offset += qed_dump_last_section(dump_buf, offset, dump);
 
 	return offset;
 }
@@ -4474,7 +4532,7 @@ static enum dbg_status qed_nvram_read(struct qed_hwfn *p_hwfn,
 				       (nvram_offset_bytes +
 					read_offset) |
 				       (bytes_to_copy <<
-					DRV_MB_PARAM_NVM_LEN_SHIFT),
+					DRV_MB_PARAM_NVM_LEN_OFFSET),
 				       &ret_mcp_resp, &ret_mcp_param,
 				       &ret_read_size,
 				       (u32 *)((u8 *)ret_buf + read_offset)))
@@ -4701,7 +4759,7 @@ static enum dbg_status qed_mcp_trace_dump(struct qed_hwfn *p_hwfn,
 		offset += trace_meta_size_dwords;
 
 	/* Dump last section */
-	offset += qed_dump_last_section(p_hwfn, dump_buf, offset, dump);
+	offset += qed_dump_last_section(dump_buf, offset, dump);
 
 	*num_dumped_dwords = offset;
 
@@ -4717,7 +4775,7 @@ static enum dbg_status qed_reg_fifo_dump(struct qed_hwfn *p_hwfn,
 					 u32 *dump_buf,
 					 bool dump, u32 *num_dumped_dwords)
 {
-	u32 dwords_read, size_param_offset, offset = 0;
+	u32 dwords_read, size_param_offset, offset = 0, addr, len;
 	bool fifo_has_data;
 
 	*num_dumped_dwords = 0;
@@ -4753,14 +4811,18 @@ static enum dbg_status qed_reg_fifo_dump(struct qed_hwfn *p_hwfn,
 	 * buffer size since more entries could be added to the buffer as we are
 	 * emptying it.
 	 */
+	addr = BYTES_TO_DWORDS(GRC_REG_TRACE_FIFO);
+	len = REG_FIFO_ELEMENT_DWORDS;
 	for (dwords_read = 0;
 	     fifo_has_data && dwords_read < REG_FIFO_DEPTH_DWORDS;
-	     dwords_read += REG_FIFO_ELEMENT_DWORDS, offset +=
-	     REG_FIFO_ELEMENT_DWORDS) {
-		if (qed_dmae_grc2host(p_hwfn, p_ptt, GRC_REG_TRACE_FIFO,
-				      (u64)(uintptr_t)(&dump_buf[offset]),
-				      REG_FIFO_ELEMENT_DWORDS, 0))
-			return DBG_STATUS_DMAE_FAILED;
+	     dwords_read += REG_FIFO_ELEMENT_DWORDS) {
+		offset += qed_grc_dump_addr_range(p_hwfn,
+						  p_ptt,
+						  dump_buf + offset,
+						  true,
+						  addr,
+						  len,
+						  true);
 		fifo_has_data = qed_rd(p_hwfn, p_ptt,
 				       GRC_REG_TRACE_FIFO_VALID_DATA) > 0;
 	}
@@ -4769,7 +4831,7 @@ static enum dbg_status qed_reg_fifo_dump(struct qed_hwfn *p_hwfn,
 			   dwords_read);
 out:
 	/* Dump last section */
-	offset += qed_dump_last_section(p_hwfn, dump_buf, offset, dump);
+	offset += qed_dump_last_section(dump_buf, offset, dump);
 
 	*num_dumped_dwords = offset;
 
@@ -4782,7 +4844,7 @@ static enum dbg_status qed_igu_fifo_dump(struct qed_hwfn *p_hwfn,
 					 u32 *dump_buf,
 					 bool dump, u32 *num_dumped_dwords)
 {
-	u32 dwords_read, size_param_offset, offset = 0;
+	u32 dwords_read, size_param_offset, offset = 0, addr, len;
 	bool fifo_has_data;
 
 	*num_dumped_dwords = 0;
@@ -4818,16 +4880,19 @@ static enum dbg_status qed_igu_fifo_dump(struct qed_hwfn *p_hwfn,
 	 * buffer size since more entries could be added to the buffer as we are
 	 * emptying it.
 	 */
+	addr = BYTES_TO_DWORDS(IGU_REG_ERROR_HANDLING_MEMORY);
+	len = IGU_FIFO_ELEMENT_DWORDS;
 	for (dwords_read = 0;
 	     fifo_has_data && dwords_read < IGU_FIFO_DEPTH_DWORDS;
-	     dwords_read += IGU_FIFO_ELEMENT_DWORDS, offset +=
-	     IGU_FIFO_ELEMENT_DWORDS) {
-		if (qed_dmae_grc2host(p_hwfn, p_ptt,
-				      IGU_REG_ERROR_HANDLING_MEMORY,
-				      (u64)(uintptr_t)(&dump_buf[offset]),
-				      IGU_FIFO_ELEMENT_DWORDS, 0))
-			return DBG_STATUS_DMAE_FAILED;
-		fifo_has_data =	qed_rd(p_hwfn, p_ptt,
+	     dwords_read += IGU_FIFO_ELEMENT_DWORDS) {
+		offset += qed_grc_dump_addr_range(p_hwfn,
+						  p_ptt,
+						  dump_buf + offset,
+						  true,
+						  addr,
+						  len,
+						  true);
+		fifo_has_data = qed_rd(p_hwfn, p_ptt,
 				       IGU_REG_ERROR_HANDLING_DATA_VALID) > 0;
 	}
 
@@ -4835,7 +4900,7 @@ static enum dbg_status qed_igu_fifo_dump(struct qed_hwfn *p_hwfn,
 			   dwords_read);
 out:
 	/* Dump last section */
-	offset += qed_dump_last_section(p_hwfn, dump_buf, offset, dump);
+	offset += qed_dump_last_section(dump_buf, offset, dump);
 
 	*num_dumped_dwords = offset;
 
@@ -4849,7 +4914,7 @@ static enum dbg_status qed_protection_override_dump(struct qed_hwfn *p_hwfn,
 						    bool dump,
 						    u32 *num_dumped_dwords)
 {
-	u32 size_param_offset, override_window_dwords, offset = 0;
+	u32 size_param_offset, override_window_dwords, offset = 0, addr;
 
 	*num_dumped_dwords = 0;
 
@@ -4875,20 +4940,21 @@ static enum dbg_status qed_protection_override_dump(struct qed_hwfn *p_hwfn,
 
 	/* Add override window info to buffer */
 	override_window_dwords =
-		qed_rd(p_hwfn, p_ptt,
-		       GRC_REG_NUMBER_VALID_OVERRIDE_WINDOW) *
-		       PROTECTION_OVERRIDE_ELEMENT_DWORDS;
-	if (qed_dmae_grc2host(p_hwfn, p_ptt,
-			      GRC_REG_PROTECTION_OVERRIDE_WINDOW,
-			      (u64)(uintptr_t)(dump_buf + offset),
-			      override_window_dwords, 0))
-		return DBG_STATUS_DMAE_FAILED;
-	offset += override_window_dwords;
+		qed_rd(p_hwfn, p_ptt, GRC_REG_NUMBER_VALID_OVERRIDE_WINDOW) *
+		PROTECTION_OVERRIDE_ELEMENT_DWORDS;
+	addr = BYTES_TO_DWORDS(GRC_REG_PROTECTION_OVERRIDE_WINDOW);
+	offset += qed_grc_dump_addr_range(p_hwfn,
+					  p_ptt,
+					  dump_buf + offset,
+					  true,
+					  addr,
+					  override_window_dwords,
+					  true);
 	qed_dump_num_param(dump_buf + size_param_offset, dump, "size",
 			   override_window_dwords);
 out:
 	/* Dump last section */
-	offset += qed_dump_last_section(p_hwfn, dump_buf, offset, dump);
+	offset += qed_dump_last_section(dump_buf, offset, dump);
 
 	*num_dumped_dwords = offset;
 
@@ -4952,9 +5018,9 @@ static u32 qed_fw_asserts_dump(struct qed_hwfn *p_hwfn,
 		next_list_idx_addr = fw_asserts_section_addr +
 			DWORDS_TO_BYTES(asserts->list_next_index_dword_offset);
 		next_list_idx = qed_rd(p_hwfn, p_ptt, next_list_idx_addr);
-		last_list_idx = (next_list_idx > 0
-				 ? next_list_idx
-				 : asserts->list_num_elements) - 1;
+		last_list_idx = (next_list_idx > 0 ?
+				 next_list_idx :
+				 asserts->list_num_elements) - 1;
 		addr = BYTES_TO_DWORDS(fw_asserts_section_addr) +
 		       asserts->list_dword_offset +
 		       last_list_idx * asserts->list_element_dword_size;
@@ -4967,7 +5033,7 @@ static u32 qed_fw_asserts_dump(struct qed_hwfn *p_hwfn,
 	}
 
 	/* Dump last section */
-	offset += qed_dump_last_section(p_hwfn, dump_buf, offset, dump);
+	offset += qed_dump_last_section(dump_buf, offset, dump);
 
 	return offset;
 }
@@ -5596,10 +5662,6 @@ struct igu_fifo_addr_data {
 
 #define PROTECTION_OVERRIDE_ELEMENT_ADDR_FACTOR	4
 
-/********************************* Macros ************************************/
-
-#define BYTES_TO_DWORDS(bytes)			((bytes) / BYTES_IN_DWORD)
-
 /***************************** Constant Arrays *******************************/
 
 struct user_dbg_array {
@@ -5698,6 +5760,7 @@ static struct block_info s_block_info_arr[] = {
 	{"phy_pcie", BLOCK_PHY_PCIE},
 	{"led", BLOCK_LED},
 	{"avs_wrap", BLOCK_AVS_WRAP},
+	{"pxpreqbus", BLOCK_PXPREQBUS},
 	{"misc_aeu", BLOCK_MISC_AEU},
 	{"bar0_map", BLOCK_BAR0_MAP}
 };
@@ -5830,8 +5893,8 @@ static const char * const s_status_str[] = {
 	/* DBG_STATUS_MCP_COULD_NOT_RESUME */
 	"Failed to resume MCP after halt",
 
-	/* DBG_STATUS_DMAE_FAILED */
-	"DMAE transaction failed",
+	/* DBG_STATUS_RESERVED2 */
+	"Reserved debug status - shouldn't be returned",
 
 	/* DBG_STATUS_SEMI_FIFO_NOT_EMPTY */
 	"Failed to empty SEMI sync FIFO",
@@ -6109,6 +6172,7 @@ static u32 qed_read_param(u32 *dump_buf,
 	if (*(char_buf + offset++)) {
 		/* String param */
 		*param_str_val = char_buf + offset;
+		*param_num_val = 0;
 		offset += strlen(*param_str_val) + 1;
 		if (offset & 0x3)
 			offset += (4 - (offset & 0x3));
@@ -6177,8 +6241,7 @@ static u32 qed_print_section_params(u32 *dump_buf,
 /* Parses the idle check rules and returns the number of characters printed.
  * In case of parsing error, returns 0.
  */
-static u32 qed_parse_idle_chk_dump_rules(struct qed_hwfn *p_hwfn,
-					 u32 *dump_buf,
+static u32 qed_parse_idle_chk_dump_rules(u32 *dump_buf,
 					 u32 *dump_buf_end,
 					 u32 num_rules,
 					 bool print_fw_idle_chk,
@@ -6322,8 +6385,7 @@ static u32 qed_parse_idle_chk_dump_rules(struct qed_hwfn *p_hwfn,
  * parsed_results_bytes.
  * The parsing status is returned.
  */
-static enum dbg_status qed_parse_idle_chk_dump(struct qed_hwfn *p_hwfn,
-					       u32 *dump_buf,
+static enum dbg_status qed_parse_idle_chk_dump(u32 *dump_buf,
 					       u32 num_dumped_dwords,
 					       char *results_buf,
 					       u32 *parsed_results_bytes,
@@ -6375,13 +6437,16 @@ static enum dbg_status qed_parse_idle_chk_dump(struct qed_hwfn *p_hwfn,
 					    results_offset),
 			    "FW_IDLE_CHECK:\n");
 		rules_print_size =
-			qed_parse_idle_chk_dump_rules(p_hwfn, dump_buf,
-						      dump_buf_end, num_rules,
+			qed_parse_idle_chk_dump_rules(dump_buf,
+						      dump_buf_end,
+						      num_rules,
 						      true,
 						      results_buf ?
 						      results_buf +
-						      results_offset : NULL,
-						      num_errors, num_warnings);
+						      results_offset :
+						      NULL,
+						      num_errors,
+						      num_warnings);
 		results_offset += rules_print_size;
 		if (!rules_print_size)
 			return DBG_STATUS_IDLE_CHK_PARSE_FAILED;
@@ -6392,13 +6457,16 @@ static enum dbg_status qed_parse_idle_chk_dump(struct qed_hwfn *p_hwfn,
 					    results_offset),
 			    "\nLSI_IDLE_CHECK:\n");
 		rules_print_size =
-			qed_parse_idle_chk_dump_rules(p_hwfn, dump_buf,
-						      dump_buf_end, num_rules,
+			qed_parse_idle_chk_dump_rules(dump_buf,
+						      dump_buf_end,
+						      num_rules,
 						      false,
 						      results_buf ?
 						      results_buf +
-						      results_offset : NULL,
-						      num_errors, num_warnings);
+						      results_offset :
+						      NULL,
+						      num_errors,
+						      num_warnings);
 		results_offset += rules_print_size;
 		if (!rules_print_size)
 			return DBG_STATUS_IDLE_CHK_PARSE_FAILED;
@@ -6537,7 +6605,6 @@ static enum dbg_status qed_mcp_trace_alloc_meta(struct qed_hwfn *p_hwfn,
  */
 static enum dbg_status qed_parse_mcp_trace_dump(struct qed_hwfn *p_hwfn,
 						u32 *dump_buf,
-						u32 num_dumped_dwords,
 						char *results_buf,
 						u32 *parsed_results_bytes)
 {
@@ -6725,9 +6792,7 @@ static enum dbg_status qed_parse_mcp_trace_dump(struct qed_hwfn *p_hwfn,
  * parsed_results_bytes.
  * The parsing status is returned.
  */
-static enum dbg_status qed_parse_reg_fifo_dump(struct qed_hwfn *p_hwfn,
-					       u32 *dump_buf,
-					       u32 num_dumped_dwords,
+static enum dbg_status qed_parse_reg_fifo_dump(u32 *dump_buf,
 					       char *results_buf,
 					       u32 *parsed_results_bytes)
 {
@@ -6834,8 +6899,7 @@ static enum dbg_status qed_parse_reg_fifo_dump(struct qed_hwfn *p_hwfn,
 static enum dbg_status qed_parse_igu_fifo_element(struct igu_fifo_element
 						  *element, char
 						  *results_buf,
-						  u32 *results_offset,
-						  u32 *parsed_results_bytes)
+						  u32 *results_offset)
 {
 	const struct igu_fifo_addr_data *found_addr = NULL;
 	u8 source, err_type, i, is_cleanup;
@@ -6933,9 +6997,9 @@ static enum dbg_status qed_parse_igu_fifo_element(struct igu_fifo_element
 				"cmd_type: prod/cons update, prod/cons: 0x%x, update_flag: %s, en_dis_int_for_sb : %s, segment : %s, timer_mask = %d, ",
 				prod_cons,
 				update_flag ? "update" : "nop",
-				en_dis_int_for_sb
-				? (en_dis_int_for_sb == 1 ? "disable" : "nop")
-				: "enable",
+				en_dis_int_for_sb ?
+				(en_dis_int_for_sb == 1 ? "disable" : "nop") :
+				"enable",
 				segment ? "attn" : "regular",
 				timer_mask);
 		}
@@ -6969,9 +7033,7 @@ static enum dbg_status qed_parse_igu_fifo_element(struct igu_fifo_element
  * parsed_results_bytes.
  * The parsing status is returned.
  */
-static enum dbg_status qed_parse_igu_fifo_dump(struct qed_hwfn *p_hwfn,
-					       u32 *dump_buf,
-					       u32 num_dumped_dwords,
+static enum dbg_status qed_parse_igu_fifo_dump(u32 *dump_buf,
 					       char *results_buf,
 					       u32 *parsed_results_bytes)
 {
@@ -7011,8 +7073,7 @@ static enum dbg_status qed_parse_igu_fifo_dump(struct qed_hwfn *p_hwfn,
 	for (i = 0; i < num_elements; i++) {
 		status = qed_parse_igu_fifo_element(&elements[i],
 						    results_buf,
-						    &results_offset,
-						    parsed_results_bytes);
+						    &results_offset);
 		if (status != DBG_STATUS_OK)
 			return status;
 	}
@@ -7028,9 +7089,7 @@ static enum dbg_status qed_parse_igu_fifo_dump(struct qed_hwfn *p_hwfn,
 }
 
 static enum dbg_status
-qed_parse_protection_override_dump(struct qed_hwfn *p_hwfn,
-				   u32 *dump_buf,
-				   u32 num_dumped_dwords,
+qed_parse_protection_override_dump(u32 *dump_buf,
 				   char *results_buf,
 				   u32 *parsed_results_bytes)
 {
@@ -7105,9 +7164,7 @@ qed_parse_protection_override_dump(struct qed_hwfn *p_hwfn,
  * parsed_results_bytes.
  * The parsing status is returned.
  */
-static enum dbg_status qed_parse_fw_asserts_dump(struct qed_hwfn *p_hwfn,
-						 u32 *dump_buf,
-						 u32 num_dumped_dwords,
+static enum dbg_status qed_parse_fw_asserts_dump(u32 *dump_buf,
 						 char *results_buf,
 						 u32 *parsed_results_bytes)
 {
@@ -7209,8 +7266,7 @@ enum dbg_status qed_get_idle_chk_results_buf_size(struct qed_hwfn *p_hwfn,
 {
 	u32 num_errors, num_warnings;
 
-	return qed_parse_idle_chk_dump(p_hwfn,
-				       dump_buf,
+	return qed_parse_idle_chk_dump(dump_buf,
 				       num_dumped_dwords,
 				       NULL,
 				       results_buf_size,
@@ -7221,12 +7277,12 @@ enum dbg_status qed_print_idle_chk_results(struct qed_hwfn *p_hwfn,
 					   u32 *dump_buf,
 					   u32 num_dumped_dwords,
 					   char *results_buf,
-					   u32 *num_errors, u32 *num_warnings)
+					   u32 *num_errors,
+					   u32 *num_warnings)
 {
 	u32 parsed_buf_size;
 
-	return qed_parse_idle_chk_dump(p_hwfn,
-				       dump_buf,
+	return qed_parse_idle_chk_dump(dump_buf,
 				       num_dumped_dwords,
 				       results_buf,
 				       &parsed_buf_size,
@@ -7245,9 +7301,7 @@ enum dbg_status qed_get_mcp_trace_results_buf_size(struct qed_hwfn *p_hwfn,
 						   u32 *results_buf_size)
 {
 	return qed_parse_mcp_trace_dump(p_hwfn,
-					dump_buf,
-					num_dumped_dwords,
-					NULL, results_buf_size);
+					dump_buf, NULL, results_buf_size);
 }
 
 enum dbg_status qed_print_mcp_trace_results(struct qed_hwfn *p_hwfn,
@@ -7259,7 +7313,6 @@ enum dbg_status qed_print_mcp_trace_results(struct qed_hwfn *p_hwfn,
 
 	return qed_parse_mcp_trace_dump(p_hwfn,
 					dump_buf,
-					num_dumped_dwords,
 					results_buf, &parsed_buf_size);
 }
 
@@ -7268,10 +7321,7 @@ enum dbg_status qed_get_reg_fifo_results_buf_size(struct qed_hwfn *p_hwfn,
 						  u32 num_dumped_dwords,
 						  u32 *results_buf_size)
 {
-	return qed_parse_reg_fifo_dump(p_hwfn,
-				       dump_buf,
-				       num_dumped_dwords,
-				       NULL, results_buf_size);
+	return qed_parse_reg_fifo_dump(dump_buf, NULL, results_buf_size);
 }
 
 enum dbg_status qed_print_reg_fifo_results(struct qed_hwfn *p_hwfn,
@@ -7281,10 +7331,7 @@ enum dbg_status qed_print_reg_fifo_results(struct qed_hwfn *p_hwfn,
 {
 	u32 parsed_buf_size;
 
-	return qed_parse_reg_fifo_dump(p_hwfn,
-				       dump_buf,
-				       num_dumped_dwords,
-				       results_buf, &parsed_buf_size);
+	return qed_parse_reg_fifo_dump(dump_buf, results_buf, &parsed_buf_size);
 }
 
 enum dbg_status qed_get_igu_fifo_results_buf_size(struct qed_hwfn *p_hwfn,
@@ -7292,10 +7339,7 @@ enum dbg_status qed_get_igu_fifo_results_buf_size(struct qed_hwfn *p_hwfn,
 						  u32 num_dumped_dwords,
 						  u32 *results_buf_size)
 {
-	return qed_parse_igu_fifo_dump(p_hwfn,
-				       dump_buf,
-				       num_dumped_dwords,
-				       NULL, results_buf_size);
+	return qed_parse_igu_fifo_dump(dump_buf, NULL, results_buf_size);
 }
 
 enum dbg_status qed_print_igu_fifo_results(struct qed_hwfn *p_hwfn,
@@ -7305,10 +7349,7 @@ enum dbg_status qed_print_igu_fifo_results(struct qed_hwfn *p_hwfn,
 {
 	u32 parsed_buf_size;
 
-	return qed_parse_igu_fifo_dump(p_hwfn,
-				       dump_buf,
-				       num_dumped_dwords,
-				       results_buf, &parsed_buf_size);
+	return qed_parse_igu_fifo_dump(dump_buf, results_buf, &parsed_buf_size);
 }
 
 enum dbg_status
@@ -7317,9 +7358,7 @@ qed_get_protection_override_results_buf_size(struct qed_hwfn *p_hwfn,
 					     u32 num_dumped_dwords,
 					     u32 *results_buf_size)
 {
-	return qed_parse_protection_override_dump(p_hwfn,
-						  dump_buf,
-						  num_dumped_dwords,
+	return qed_parse_protection_override_dump(dump_buf,
 						  NULL, results_buf_size);
 }
 
@@ -7330,9 +7369,7 @@ enum dbg_status qed_print_protection_override_results(struct qed_hwfn *p_hwfn,
 {
 	u32 parsed_buf_size;
 
-	return qed_parse_protection_override_dump(p_hwfn,
-						  dump_buf,
-						  num_dumped_dwords,
+	return qed_parse_protection_override_dump(dump_buf,
 						  results_buf,
 						  &parsed_buf_size);
 }
@@ -7342,10 +7379,7 @@ enum dbg_status qed_get_fw_asserts_results_buf_size(struct qed_hwfn *p_hwfn,
 						    u32 num_dumped_dwords,
 						    u32 *results_buf_size)
 {
-	return qed_parse_fw_asserts_dump(p_hwfn,
-					 dump_buf,
-					 num_dumped_dwords,
-					 NULL, results_buf_size);
+	return qed_parse_fw_asserts_dump(dump_buf, NULL, results_buf_size);
 }
 
 enum dbg_status qed_print_fw_asserts_results(struct qed_hwfn *p_hwfn,
@@ -7355,9 +7389,7 @@ enum dbg_status qed_print_fw_asserts_results(struct qed_hwfn *p_hwfn,
 {
 	u32 parsed_buf_size;
 
-	return qed_parse_fw_asserts_dump(p_hwfn,
-					 dump_buf,
-					 num_dumped_dwords,
+	return qed_parse_fw_asserts_dump(dump_buf,
 					 results_buf, &parsed_buf_size);
 }
 
@@ -7386,30 +7418,30 @@ enum dbg_status qed_dbg_parse_attn(struct qed_hwfn *p_hwfn,
 
 	/* Go over registers with a non-zero attention status */
 	for (i = 0; i < num_regs; i++) {
+		struct dbg_attn_bit_mapping *bit_mapping;
 		struct dbg_attn_reg_result *reg_result;
-		struct dbg_attn_bit_mapping *mapping;
 		u8 num_reg_attn, bit_idx = 0;
 
 		reg_result = &results->reg_results[i];
 		num_reg_attn = GET_FIELD(reg_result->data,
 					 DBG_ATTN_REG_RESULT_NUM_REG_ATTN);
 		block_attn = &s_user_dbg_arrays[BIN_BUF_DBG_ATTN_INDEXES];
-		mapping = &((struct dbg_attn_bit_mapping *)
-			    block_attn->ptr)[reg_result->block_attn_offset];
+		bit_mapping = &((struct dbg_attn_bit_mapping *)
+				block_attn->ptr)[reg_result->block_attn_offset];
 
 		pstrings = &s_user_dbg_arrays[BIN_BUF_DBG_PARSING_STRINGS];
 
 		/* Go over attention status bits */
 		for (j = 0; j < num_reg_attn; j++) {
-			u16 attn_idx_val = GET_FIELD(mapping[j].data,
+			u16 attn_idx_val = GET_FIELD(bit_mapping[j].data,
 						     DBG_ATTN_BIT_MAPPING_VAL);
 			const char *attn_name, *attn_type_str, *masked_str;
-			u32 name_offset, sts_addr;
+			u32 attn_name_offset, sts_addr;
 
 			/* Check if bit mask should be advanced (due to unused
 			 * bits).
 			 */
-			if (GET_FIELD(mapping[j].data,
+			if (GET_FIELD(bit_mapping[j].data,
 				      DBG_ATTN_BIT_MAPPING_IS_UNUSED_BIT_CNT)) {
 				bit_idx += (u8)attn_idx_val;
 				continue;
@@ -7422,9 +7454,10 @@ enum dbg_status qed_dbg_parse_attn(struct qed_hwfn *p_hwfn,
 			}
 
 			/* Find attention name */
-			name_offset = block_attn_name_offsets[attn_idx_val];
+			attn_name_offset =
+				block_attn_name_offsets[attn_idx_val];
 			attn_name = &((const char *)
-				      pstrings->ptr)[name_offset];
+				      pstrings->ptr)[attn_name_offset];
 			attn_type_str = attn_type == ATTN_TYPE_INTERRUPT ?
 					"Interrupt" : "Parity";
 			masked_str = reg_result->mask_val & BIT(bit_idx) ?
diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c
index 58a689fb04db..553a6d17260e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_dev.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c
@@ -758,7 +758,7 @@ static void qed_init_qm_info(struct qed_hwfn *p_hwfn)
 /* This function reconfigures the QM pf on the fly.
  * For this purpose we:
  * 1. reconfigure the QM database
- * 2. set new values to runtime arrat
+ * 2. set new values to runtime array
  * 3. send an sdm_qm_cmd through the rbc interface to stop the QM
  * 4. activate init tool in QM_PF stage
  * 5. send an sdm_qm_cmd through rbc interface to release the QM
@@ -784,7 +784,7 @@ int qed_qm_reconf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 	qed_init_clear_rt_data(p_hwfn);
 
 	/* prepare QM portion of runtime array */
-	qed_qm_init_pf(p_hwfn, p_ptt);
+	qed_qm_init_pf(p_hwfn, p_ptt, false);
 
 	/* activate init tool on runtime array */
 	rc = qed_init_run(p_hwfn, p_ptt, PHASE_QM_PF, p_hwfn->rel_pf_id,
@@ -1515,7 +1515,7 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 			     NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET, 1);
 	}
 
-	/* Protocl Configuration  */
+	/* Protocol Configuration */
 	STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_TCP_RT_OFFSET,
 		     (p_hwfn->hw_info.personality == QED_PCI_ISCSI) ? 1 : 0);
 	STORE_RT_REG(p_hwfn, PRS_REG_SEARCH_FCOE_RT_OFFSET,
@@ -1527,6 +1527,11 @@ static int qed_hw_init_pf(struct qed_hwfn *p_hwfn,
 	if (rc)
 		return rc;
 
+	/* Sanity check before the PF init sequence that uses DMAE */
+	rc = qed_dmae_sanity(p_hwfn, p_ptt, "pf_phase");
+	if (rc)
+		return rc;
+
 	/* PF Init sequence */
 	rc = qed_init_run(p_hwfn, p_ptt, PHASE_PF, rel_pf_id, hw_mode);
 	if (rc)
@@ -2192,7 +2197,7 @@ qed_hw_set_soft_resc_size(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 			/* No need for a case for QED_CMDQS_CQS since
 			 * CNQ/CMDQS are the same resource.
 			 */
-			resc_max_val = NUM_OF_CMDQS_CQS;
+			resc_max_val = NUM_OF_GLOBAL_QUEUES;
 			break;
 		case QED_RDMA_STATS_QUEUE:
 			resc_max_val = b_ah ? RDMA_NUM_STATISTIC_COUNTERS_K2
@@ -2267,7 +2272,7 @@ int qed_hw_get_dflt_resc(struct qed_hwfn *p_hwfn,
 	case QED_RDMA_CNQ_RAM:
 	case QED_CMDQS_CQS:
 		/* CNQ/CMDQS are the same resource */
-		*p_resc_num = NUM_OF_CMDQS_CQS / num_funcs;
+		*p_resc_num = NUM_OF_GLOBAL_QUEUES / num_funcs;
 		break;
 	case QED_RDMA_STATS_QUEUE:
 		*p_resc_num = (b_ah ? RDMA_NUM_STATISTIC_COUNTERS_K2 :
diff --git a/drivers/net/ethernet/qlogic/qed/qed_fcoe.c b/drivers/net/ethernet/qlogic/qed/qed_fcoe.c
index df195c02b711..2dc9b312a795 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_fcoe.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_fcoe.c
@@ -115,7 +115,7 @@ qed_sp_fcoe_func_start(struct qed_hwfn *p_hwfn,
 	struct qed_fcoe_pf_params *fcoe_pf_params = NULL;
 	struct fcoe_init_ramrod_params *p_ramrod = NULL;
 	struct fcoe_init_func_ramrod_data *p_data;
-	struct fcoe_conn_context *p_cxt = NULL;
+	struct e4_fcoe_conn_context *p_cxt = NULL;
 	struct qed_spq_entry *p_ent = NULL;
 	struct qed_sp_init_data init_data;
 	struct qed_cxt_info cxt_info;
@@ -167,7 +167,7 @@ qed_sp_fcoe_func_start(struct qed_hwfn *p_hwfn,
 	}
 	p_cxt = cxt_info.p_cxt;
 	SET_FIELD(p_cxt->tstorm_ag_context.flags3,
-		  TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_EN, 1);
+		  E4_TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_EN, 1);
 
 	fcoe_pf_params->dummy_icid = (u16)dummy_cid;
 
@@ -568,7 +568,7 @@ int qed_fcoe_alloc(struct qed_hwfn *p_hwfn)
 
 void qed_fcoe_setup(struct qed_hwfn *p_hwfn)
 {
-	struct fcoe_task_context *p_task_ctx = NULL;
+	struct e4_fcoe_task_context *p_task_ctx = NULL;
 	int rc;
 	u32 i;
 
@@ -580,13 +580,13 @@ void qed_fcoe_setup(struct qed_hwfn *p_hwfn)
 		if (rc)
 			continue;
 
-		memset(p_task_ctx, 0, sizeof(struct fcoe_task_context));
+		memset(p_task_ctx, 0, sizeof(struct e4_fcoe_task_context));
 		SET_FIELD(p_task_ctx->timer_context.logical_client_0,
 			  TIMERS_CONTEXT_VALIDLC0, 1);
 		SET_FIELD(p_task_ctx->timer_context.logical_client_1,
 			  TIMERS_CONTEXT_VALIDLC1, 1);
 		SET_FIELD(p_task_ctx->tstorm_ag_context.flags0,
-			  TSTORM_FCOE_TASK_AG_CTX_CONNECTION_TYPE, 1);
+			  E4_TSTORM_FCOE_TASK_AG_CTX_CONNECTION_TYPE, 1);
 	}
 }
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
index 3427fe7049b5..de873d770575 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hsi.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_hsi.h
@@ -54,7 +54,7 @@
 struct qed_hwfn;
 struct qed_ptt;
 
-/* opcodes for the event ring */
+/* Opcodes for the event ring */
 enum common_event_opcode {
 	COMMON_EVENT_PF_START,
 	COMMON_EVENT_PF_STOP,
@@ -82,487 +82,7 @@ enum common_ramrod_cmd_id {
 	MAX_COMMON_RAMROD_CMD_ID
 };
 
-/* The core storm context for the Ystorm */
-struct ystorm_core_conn_st_ctx {
-	__le32 reserved[4];
-};
-
-/* The core storm context for the Pstorm */
-struct pstorm_core_conn_st_ctx {
-	__le32 reserved[4];
-};
-
-/* Core Slowpath Connection storm context of Xstorm */
-struct xstorm_core_conn_st_ctx {
-	__le32 spq_base_lo;
-	__le32 spq_base_hi;
-	struct regpair consolid_base_addr;
-	__le16 spq_cons;
-	__le16 consolid_cons;
-	__le32 reserved0[55];
-};
-
-struct xstorm_core_conn_ag_ctx {
-	u8 reserved0;
-	u8 core_state;
-	u8 flags0;
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED1_SHIFT		1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED2_SHIFT		2
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED3_SHIFT		4
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED4_SHIFT		5
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED5_SHIFT		6
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED6_SHIFT		7
-	u8 flags1;
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED7_SHIFT		0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED8_SHIFT		1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED9_SHIFT		2
-#define XSTORM_CORE_CONN_AG_CTX_BIT11_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT11_SHIFT		3
-#define XSTORM_CORE_CONN_AG_CTX_BIT12_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT12_SHIFT		4
-#define XSTORM_CORE_CONN_AG_CTX_BIT13_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT13_SHIFT		5
-#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT	6
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT	7
-	u8 flags2;
-#define XSTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF0_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF1_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF2_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_CF3_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF3_SHIFT	6
-	u8 flags3;
-#define XSTORM_CORE_CONN_AG_CTX_CF4_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF4_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_CF5_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF5_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_CF6_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF6_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_CF7_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF7_SHIFT	6
-	u8 flags4;
-#define XSTORM_CORE_CONN_AG_CTX_CF8_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF8_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_CF9_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF9_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_CF10_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF10_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_CF11_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF11_SHIFT	6
-	u8 flags5;
-#define XSTORM_CORE_CONN_AG_CTX_CF12_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF12_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_CF13_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF13_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_CF14_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF14_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_CF15_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF15_SHIFT	6
-	u8 flags6;
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_CF17_MASK		0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF17_SHIFT		2
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_MASK		0x3
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_SHIFT		4
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_SHIFT	6
-	u8 flags7;
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_MASK		0x3
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_SHIFT		0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_MASK		0x3
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED10_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_MASK		0x3
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_SHIFT		4
-#define XSTORM_CORE_CONN_AG_CTX_CF0EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT		6
-#define XSTORM_CORE_CONN_AG_CTX_CF1EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT		7
-	u8 flags8;
-#define XSTORM_CORE_CONN_AG_CTX_CF2EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_CF3EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT	1
-#define XSTORM_CORE_CONN_AG_CTX_CF4EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_CF5EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT	3
-#define XSTORM_CORE_CONN_AG_CTX_CF6EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_CF7EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT	5
-#define XSTORM_CORE_CONN_AG_CTX_CF8EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT	6
-#define XSTORM_CORE_CONN_AG_CTX_CF9EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT	7
-	u8 flags9;
-#define XSTORM_CORE_CONN_AG_CTX_CF10EN_MASK			0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT			0
-#define XSTORM_CORE_CONN_AG_CTX_CF11EN_MASK			0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF11EN_SHIFT			1
-#define XSTORM_CORE_CONN_AG_CTX_CF12EN_MASK			0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF12EN_SHIFT			2
-#define XSTORM_CORE_CONN_AG_CTX_CF13EN_MASK			0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF13EN_SHIFT			3
-#define XSTORM_CORE_CONN_AG_CTX_CF14EN_MASK			0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF14EN_SHIFT			4
-#define XSTORM_CORE_CONN_AG_CTX_CF15EN_MASK			0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF15EN_SHIFT			5
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_SHIFT	6
-#define XSTORM_CORE_CONN_AG_CTX_CF17EN_MASK			0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF17EN_SHIFT			7
-	u8 flags10;
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_SHIFT		0
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT	1
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED11_SHIFT	3
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_CF23EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_CF23EN_SHIFT		5
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED12_SHIFT	6
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED13_SHIFT	7
-	u8 flags11;
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED14_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RESERVED15_SHIFT	1
-#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT		3
-#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT		4
-#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT		5
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
-#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE9EN_SHIFT		7
-	u8 flags12;
-#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE10EN_SHIFT		0
-#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE11EN_SHIFT		1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
-#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE14EN_SHIFT		4
-#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE15EN_SHIFT		5
-#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE16EN_SHIFT		6
-#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE17EN_SHIFT		7
-	u8 flags13;
-#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE18EN_SHIFT		0
-#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_MASK		0x1
-#define XSTORM_CORE_CONN_AG_CTX_RULE19EN_SHIFT		1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
-	u8 flags14;
-#define XSTORM_CORE_CONN_AG_CTX_BIT16_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT16_SHIFT	0
-#define XSTORM_CORE_CONN_AG_CTX_BIT17_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT17_SHIFT	1
-#define XSTORM_CORE_CONN_AG_CTX_BIT18_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT18_SHIFT	2
-#define XSTORM_CORE_CONN_AG_CTX_BIT19_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT19_SHIFT	3
-#define XSTORM_CORE_CONN_AG_CTX_BIT20_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT20_SHIFT	4
-#define XSTORM_CORE_CONN_AG_CTX_BIT21_MASK	0x1
-#define XSTORM_CORE_CONN_AG_CTX_BIT21_SHIFT	5
-#define XSTORM_CORE_CONN_AG_CTX_CF23_MASK	0x3
-#define XSTORM_CORE_CONN_AG_CTX_CF23_SHIFT	6
-	u8 byte2;
-	__le16 physical_q0;
-	__le16 consolid_prod;
-	__le16 reserved16;
-	__le16 tx_bd_cons;
-	__le16 tx_bd_or_spq_prod;
-	__le16 word5;
-	__le16 conn_dpi;
-	u8 byte3;
-	u8 byte4;
-	u8 byte5;
-	u8 byte6;
-	__le32 reg0;
-	__le32 reg1;
-	__le32 reg2;
-	__le32 reg3;
-	__le32 reg4;
-	__le32 reg5;
-	__le32 reg6;
-	__le16 word7;
-	__le16 word8;
-	__le16 word9;
-	__le16 word10;
-	__le32 reg7;
-	__le32 reg8;
-	__le32 reg9;
-	u8 byte7;
-	u8 byte8;
-	u8 byte9;
-	u8 byte10;
-	u8 byte11;
-	u8 byte12;
-	u8 byte13;
-	u8 byte14;
-	u8 byte15;
-	u8 e5_reserved;
-	__le16 word11;
-	__le32 reg10;
-	__le32 reg11;
-	__le32 reg12;
-	__le32 reg13;
-	__le32 reg14;
-	__le32 reg15;
-	__le32 reg16;
-	__le32 reg17;
-	__le32 reg18;
-	__le32 reg19;
-	__le16 word12;
-	__le16 word13;
-	__le16 word14;
-	__le16 word15;
-};
-
-struct tstorm_core_conn_ag_ctx {
-	u8 byte0;
-	u8 byte1;
-	u8 flags0;
-#define TSTORM_CORE_CONN_AG_CTX_BIT0_MASK     0x1	/* exist_in_qm0 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT    0
-#define TSTORM_CORE_CONN_AG_CTX_BIT1_MASK     0x1	/* exist_in_qm1 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT    1
-#define TSTORM_CORE_CONN_AG_CTX_BIT2_MASK     0x1	/* bit2 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT2_SHIFT    2
-#define TSTORM_CORE_CONN_AG_CTX_BIT3_MASK     0x1	/* bit3 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT3_SHIFT    3
-#define TSTORM_CORE_CONN_AG_CTX_BIT4_MASK     0x1	/* bit4 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT4_SHIFT    4
-#define TSTORM_CORE_CONN_AG_CTX_BIT5_MASK     0x1	/* bit5 */
-#define TSTORM_CORE_CONN_AG_CTX_BIT5_SHIFT    5
-#define TSTORM_CORE_CONN_AG_CTX_CF0_MASK      0x3	/* timer0cf */
-#define TSTORM_CORE_CONN_AG_CTX_CF0_SHIFT     6
-	u8 flags1;
-#define TSTORM_CORE_CONN_AG_CTX_CF1_MASK      0x3	/* timer1cf */
-#define TSTORM_CORE_CONN_AG_CTX_CF1_SHIFT     0
-#define TSTORM_CORE_CONN_AG_CTX_CF2_MASK      0x3	/* timer2cf */
-#define TSTORM_CORE_CONN_AG_CTX_CF2_SHIFT     2
-#define TSTORM_CORE_CONN_AG_CTX_CF3_MASK      0x3	/* timer_stop_all */
-#define TSTORM_CORE_CONN_AG_CTX_CF3_SHIFT     4
-#define TSTORM_CORE_CONN_AG_CTX_CF4_MASK      0x3	/* cf4 */
-#define TSTORM_CORE_CONN_AG_CTX_CF4_SHIFT     6
-	u8 flags2;
-#define TSTORM_CORE_CONN_AG_CTX_CF5_MASK      0x3	/* cf5 */
-#define TSTORM_CORE_CONN_AG_CTX_CF5_SHIFT     0
-#define TSTORM_CORE_CONN_AG_CTX_CF6_MASK      0x3	/* cf6 */
-#define TSTORM_CORE_CONN_AG_CTX_CF6_SHIFT     2
-#define TSTORM_CORE_CONN_AG_CTX_CF7_MASK      0x3	/* cf7 */
-#define TSTORM_CORE_CONN_AG_CTX_CF7_SHIFT     4
-#define TSTORM_CORE_CONN_AG_CTX_CF8_MASK      0x3	/* cf8 */
-#define TSTORM_CORE_CONN_AG_CTX_CF8_SHIFT     6
-	u8 flags3;
-#define TSTORM_CORE_CONN_AG_CTX_CF9_MASK      0x3	/* cf9 */
-#define TSTORM_CORE_CONN_AG_CTX_CF9_SHIFT     0
-#define TSTORM_CORE_CONN_AG_CTX_CF10_MASK     0x3	/* cf10 */
-#define TSTORM_CORE_CONN_AG_CTX_CF10_SHIFT    2
-#define TSTORM_CORE_CONN_AG_CTX_CF0EN_MASK    0x1	/* cf0en */
-#define TSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT   4
-#define TSTORM_CORE_CONN_AG_CTX_CF1EN_MASK    0x1	/* cf1en */
-#define TSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT   5
-#define TSTORM_CORE_CONN_AG_CTX_CF2EN_MASK    0x1	/* cf2en */
-#define TSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT   6
-#define TSTORM_CORE_CONN_AG_CTX_CF3EN_MASK    0x1	/* cf3en */
-#define TSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT   7
-	u8 flags4;
-#define TSTORM_CORE_CONN_AG_CTX_CF4EN_MASK    0x1	/* cf4en */
-#define TSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT   0
-#define TSTORM_CORE_CONN_AG_CTX_CF5EN_MASK    0x1	/* cf5en */
-#define TSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT   1
-#define TSTORM_CORE_CONN_AG_CTX_CF6EN_MASK    0x1	/* cf6en */
-#define TSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT   2
-#define TSTORM_CORE_CONN_AG_CTX_CF7EN_MASK    0x1	/* cf7en */
-#define TSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT   3
-#define TSTORM_CORE_CONN_AG_CTX_CF8EN_MASK    0x1	/* cf8en */
-#define TSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT   4
-#define TSTORM_CORE_CONN_AG_CTX_CF9EN_MASK    0x1	/* cf9en */
-#define TSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT   5
-#define TSTORM_CORE_CONN_AG_CTX_CF10EN_MASK   0x1	/* cf10en */
-#define TSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT  6
-#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK  0x1	/* rule0en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT 7
-	u8 flags5;
-#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK  0x1	/* rule1en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT 0
-#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK  0x1	/* rule2en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT 1
-#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK  0x1	/* rule3en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT 2
-#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK  0x1	/* rule4en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT 3
-#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK  0x1	/* rule5en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT 4
-#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK  0x1	/* rule6en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT 5
-#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK  0x1	/* rule7en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT 6
-#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_MASK  0x1	/* rule8en */
-#define TSTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT 7
-	__le32 reg0;
-	__le32 reg1;
-	__le32 reg2;
-	__le32 reg3;
-	__le32 reg4;
-	__le32 reg5;
-	__le32 reg6;
-	__le32 reg7;
-	__le32 reg8;
-	u8 byte2;
-	u8 byte3;
-	__le16 word0;
-	u8 byte4;
-	u8 byte5;
-	__le16 word1;
-	__le16 word2;
-	__le16 word3;
-	__le32 reg9;
-	__le32 reg10;
-};
-
-struct ustorm_core_conn_ag_ctx {
-	u8 reserved;
-	u8 byte1;
-	u8 flags0;
-#define USTORM_CORE_CONN_AG_CTX_BIT0_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_BIT0_SHIFT	0
-#define USTORM_CORE_CONN_AG_CTX_BIT1_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_BIT1_SHIFT	1
-#define USTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
-#define USTORM_CORE_CONN_AG_CTX_CF0_SHIFT	2
-#define USTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
-#define USTORM_CORE_CONN_AG_CTX_CF1_SHIFT	4
-#define USTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
-#define USTORM_CORE_CONN_AG_CTX_CF2_SHIFT	6
-	u8 flags1;
-#define USTORM_CORE_CONN_AG_CTX_CF3_MASK	0x3
-#define USTORM_CORE_CONN_AG_CTX_CF3_SHIFT	0
-#define USTORM_CORE_CONN_AG_CTX_CF4_MASK	0x3
-#define USTORM_CORE_CONN_AG_CTX_CF4_SHIFT	2
-#define USTORM_CORE_CONN_AG_CTX_CF5_MASK	0x3
-#define USTORM_CORE_CONN_AG_CTX_CF5_SHIFT	4
-#define USTORM_CORE_CONN_AG_CTX_CF6_MASK	0x3
-#define USTORM_CORE_CONN_AG_CTX_CF6_SHIFT	6
-	u8 flags2;
-#define USTORM_CORE_CONN_AG_CTX_CF0EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT	0
-#define USTORM_CORE_CONN_AG_CTX_CF1EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT	1
-#define USTORM_CORE_CONN_AG_CTX_CF2EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT	2
-#define USTORM_CORE_CONN_AG_CTX_CF3EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT	3
-#define USTORM_CORE_CONN_AG_CTX_CF4EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT	4
-#define USTORM_CORE_CONN_AG_CTX_CF5EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT	5
-#define USTORM_CORE_CONN_AG_CTX_CF6EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT	6
-#define USTORM_CORE_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT	7
-	u8 flags3;
-#define USTORM_CORE_CONN_AG_CTX_RULE1EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT	0
-#define USTORM_CORE_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT	1
-#define USTORM_CORE_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT	2
-#define USTORM_CORE_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT	3
-#define USTORM_CORE_CONN_AG_CTX_RULE5EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT	4
-#define USTORM_CORE_CONN_AG_CTX_RULE6EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT	5
-#define USTORM_CORE_CONN_AG_CTX_RULE7EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT	6
-#define USTORM_CORE_CONN_AG_CTX_RULE8EN_MASK	0x1
-#define USTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT	7
-	u8 byte2;
-	u8 byte3;
-	__le16 word0;
-	__le16 word1;
-	__le32 rx_producers;
-	__le32 reg1;
-	__le32 reg2;
-	__le32 reg3;
-	__le16 word2;
-	__le16 word3;
-};
-
-/* The core storm context for the Mstorm */
-struct mstorm_core_conn_st_ctx {
-	__le32 reserved[24];
-};
-
-/* The core storm context for the Ustorm */
-struct ustorm_core_conn_st_ctx {
-	__le32 reserved[4];
-};
-
-/* core connection context */
-struct core_conn_context {
-	struct ystorm_core_conn_st_ctx ystorm_st_context;
-	struct regpair ystorm_st_padding[2];
-	struct pstorm_core_conn_st_ctx pstorm_st_context;
-	struct regpair pstorm_st_padding[2];
-	struct xstorm_core_conn_st_ctx xstorm_st_context;
-	struct xstorm_core_conn_ag_ctx xstorm_ag_context;
-	struct tstorm_core_conn_ag_ctx tstorm_ag_context;
-	struct ustorm_core_conn_ag_ctx ustorm_ag_context;
-	struct mstorm_core_conn_st_ctx mstorm_st_context;
-	struct ustorm_core_conn_st_ctx ustorm_st_context;
-	struct regpair ustorm_st_padding[2];
-};
-
+/* How ll2 should deal with packet upon errors */
 enum core_error_handle {
 	LL2_DROP_PACKET,
 	LL2_DO_NOTHING,
@@ -570,21 +90,25 @@ enum core_error_handle {
 	MAX_CORE_ERROR_HANDLE
 };
 
+/* Opcodes for the event ring */
 enum core_event_opcode {
 	CORE_EVENT_TX_QUEUE_START,
 	CORE_EVENT_TX_QUEUE_STOP,
 	CORE_EVENT_RX_QUEUE_START,
 	CORE_EVENT_RX_QUEUE_STOP,
 	CORE_EVENT_RX_QUEUE_FLUSH,
+	CORE_EVENT_TX_QUEUE_UPDATE,
 	MAX_CORE_EVENT_OPCODE
 };
 
+/* The L4 pseudo checksum mode for Core */
 enum core_l4_pseudo_checksum_mode {
 	CORE_L4_PSEUDO_CSUM_CORRECT_LENGTH,
 	CORE_L4_PSEUDO_CSUM_ZERO_LENGTH,
 	MAX_CORE_L4_PSEUDO_CHECKSUM_MODE
 };
 
+/* Light-L2 RX Producers in Tstorm RAM */
 struct core_ll2_port_stats {
 	struct regpair gsi_invalid_hdr;
 	struct regpair gsi_invalid_pkt_length;
@@ -592,6 +116,7 @@ struct core_ll2_port_stats {
 	struct regpair gsi_crcchksm_error;
 };
 
+/* Ethernet TX Per Queue Stats */
 struct core_ll2_pstorm_per_queue_stat {
 	struct regpair sent_ucast_bytes;
 	struct regpair sent_mcast_bytes;
@@ -601,6 +126,7 @@ struct core_ll2_pstorm_per_queue_stat {
 	struct regpair sent_bcast_pkts;
 };
 
+/* Light-L2 RX Producers in Tstorm RAM */
 struct core_ll2_rx_prod {
 	__le16 bd_prod;
 	__le16 cqe_prod;
@@ -621,6 +147,7 @@ struct core_ll2_ustorm_per_queue_stat {
 	struct regpair rcv_bcast_pkts;
 };
 
+/* Core Ramrod Command IDs (light L2) */
 enum core_ramrod_cmd_id {
 	CORE_RAMROD_UNUSED,
 	CORE_RAMROD_RX_QUEUE_START,
@@ -628,53 +155,64 @@ enum core_ramrod_cmd_id {
 	CORE_RAMROD_RX_QUEUE_STOP,
 	CORE_RAMROD_TX_QUEUE_STOP,
 	CORE_RAMROD_RX_QUEUE_FLUSH,
+	CORE_RAMROD_TX_QUEUE_UPDATE,
 	MAX_CORE_RAMROD_CMD_ID
 };
 
+/* Core RX CQE Type for Light L2 */
 enum core_roce_flavor_type {
 	CORE_ROCE,
 	CORE_RROCE,
 	MAX_CORE_ROCE_FLAVOR_TYPE
 };
 
+/* Specifies how ll2 should deal with packets errors: packet_too_big and
+ * no_buff.
+ */
 struct core_rx_action_on_error {
 	u8 error_type;
 #define CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG_MASK	0x3
-#define CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG_SHIFT 0
-#define CORE_RX_ACTION_ON_ERROR_NO_BUFF_MASK	0x3
-#define CORE_RX_ACTION_ON_ERROR_NO_BUFF_SHIFT	2
-#define CORE_RX_ACTION_ON_ERROR_RESERVED_MASK	0xF
-#define CORE_RX_ACTION_ON_ERROR_RESERVED_SHIFT	4
+#define CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG_SHIFT	0
+#define CORE_RX_ACTION_ON_ERROR_NO_BUFF_MASK		0x3
+#define CORE_RX_ACTION_ON_ERROR_NO_BUFF_SHIFT		2
+#define CORE_RX_ACTION_ON_ERROR_RESERVED_MASK		0xF
+#define CORE_RX_ACTION_ON_ERROR_RESERVED_SHIFT		4
 };
 
+/* Core RX BD for Light L2 */
 struct core_rx_bd {
 	struct regpair addr;
 	__le16 reserved[4];
 };
 
+/* Core RX CM offload BD for Light L2 */
 struct core_rx_bd_with_buff_len {
 	struct regpair addr;
 	__le16 buff_length;
 	__le16 reserved[3];
 };
 
+/* Core RX CM offload BD for Light L2 */
 union core_rx_bd_union {
 	struct core_rx_bd rx_bd;
 	struct core_rx_bd_with_buff_len rx_bd_with_len;
 };
 
+/* Opaque Data for Light L2 RX CQE */
 struct core_rx_cqe_opaque_data {
 	__le32 data[2];
 };
 
+/* Core RX CQE Type for Light L2 */
 enum core_rx_cqe_type {
-	CORE_RX_CQE_ILLIGAL_TYPE,
+	CORE_RX_CQE_ILLEGAL_TYPE,
 	CORE_RX_CQE_TYPE_REGULAR,
 	CORE_RX_CQE_TYPE_GSI_OFFLOAD,
 	CORE_RX_CQE_TYPE_SLOW_PATH,
 	MAX_CORE_RX_CQE_TYPE
 };
 
+/* Core RX CQE for Light L2 */
 struct core_rx_fast_path_cqe {
 	u8 type;
 	u8 placement_offset;
@@ -687,6 +225,7 @@ struct core_rx_fast_path_cqe {
 	__le32 reserved1[3];
 };
 
+/* Core Rx CM offload CQE */
 struct core_rx_gsi_offload_cqe {
 	u8 type;
 	u8 data_length_error;
@@ -696,9 +235,11 @@ struct core_rx_gsi_offload_cqe {
 	__le32 src_mac_addrhi;
 	__le16 src_mac_addrlo;
 	__le16 qp_id;
-	__le32 gid_dst[4];
+	__le32 src_qp;
+	__le32 reserved[3];
 };
 
+/* Core RX CQE for Light L2 */
 struct core_rx_slow_path_cqe {
 	u8 type;
 	u8 ramrod_cmd_id;
@@ -707,12 +248,14 @@ struct core_rx_slow_path_cqe {
 	__le32 reserved1[5];
 };
 
+/* Core RX CM offload BD for Light L2 */
 union core_rx_cqe_union {
 	struct core_rx_fast_path_cqe rx_cqe_fp;
 	struct core_rx_gsi_offload_cqe rx_cqe_gsi;
 	struct core_rx_slow_path_cqe rx_cqe_sp;
 };
 
+/* Ramrod data for rx queue start ramrod */
 struct core_rx_start_ramrod_data {
 	struct regpair bd_base;
 	struct regpair cqe_pbl_addr;
@@ -723,16 +266,18 @@ struct core_rx_start_ramrod_data {
 	u8 complete_event_flg;
 	u8 drop_ttl0_flg;
 	__le16 num_of_pbl_pages;
-	u8 inner_vlan_removal_en;
+	u8 inner_vlan_stripping_en;
+	u8 report_outer_vlan;
 	u8 queue_id;
 	u8 main_func_queue;
 	u8 mf_si_bcast_accept_all;
 	u8 mf_si_mcast_accept_all;
 	struct core_rx_action_on_error action_on_error;
 	u8 gsi_offload_flag;
-	u8 reserved[7];
+	u8 reserved[6];
 };
 
+/* Ramrod data for rx queue stop ramrod */
 struct core_rx_stop_ramrod_data {
 	u8 complete_cqe_flg;
 	u8 complete_event_flg;
@@ -741,46 +286,51 @@ struct core_rx_stop_ramrod_data {
 	__le16 reserved2[2];
 };
 
+/* Flags for Core TX BD */
 struct core_tx_bd_data {
 	__le16 as_bitfield;
-#define CORE_TX_BD_DATA_FORCE_VLAN_MODE_MASK	0x1
-#define CORE_TX_BD_DATA_FORCE_VLAN_MODE_SHIFT     0
-#define CORE_TX_BD_DATA_VLAN_INSERTION_MASK	0x1
-#define CORE_TX_BD_DATA_VLAN_INSERTION_SHIFT      1
-#define CORE_TX_BD_DATA_START_BD_MASK	0x1
-#define CORE_TX_BD_DATA_START_BD_SHIFT            2
-#define CORE_TX_BD_DATA_IP_CSUM_MASK	0x1
-#define CORE_TX_BD_DATA_IP_CSUM_SHIFT             3
-#define CORE_TX_BD_DATA_L4_CSUM_MASK	0x1
-#define CORE_TX_BD_DATA_L4_CSUM_SHIFT             4
-#define CORE_TX_BD_DATA_IPV6_EXT_MASK	0x1
-#define CORE_TX_BD_DATA_IPV6_EXT_SHIFT            5
-#define CORE_TX_BD_DATA_L4_PROTOCOL_MASK	0x1
-#define CORE_TX_BD_DATA_L4_PROTOCOL_SHIFT         6
+#define CORE_TX_BD_DATA_FORCE_VLAN_MODE_MASK		0x1
+#define CORE_TX_BD_DATA_FORCE_VLAN_MODE_SHIFT		0
+#define CORE_TX_BD_DATA_VLAN_INSERTION_MASK		0x1
+#define CORE_TX_BD_DATA_VLAN_INSERTION_SHIFT		1
+#define CORE_TX_BD_DATA_START_BD_MASK			0x1
+#define CORE_TX_BD_DATA_START_BD_SHIFT			2
+#define CORE_TX_BD_DATA_IP_CSUM_MASK			0x1
+#define CORE_TX_BD_DATA_IP_CSUM_SHIFT			3
+#define CORE_TX_BD_DATA_L4_CSUM_MASK			0x1
+#define CORE_TX_BD_DATA_L4_CSUM_SHIFT			4
+#define CORE_TX_BD_DATA_IPV6_EXT_MASK			0x1
+#define CORE_TX_BD_DATA_IPV6_EXT_SHIFT			5
+#define CORE_TX_BD_DATA_L4_PROTOCOL_MASK		0x1
+#define CORE_TX_BD_DATA_L4_PROTOCOL_SHIFT		6
 #define CORE_TX_BD_DATA_L4_PSEUDO_CSUM_MODE_MASK	0x1
-#define CORE_TX_BD_DATA_L4_PSEUDO_CSUM_MODE_SHIFT 7
-#define CORE_TX_BD_DATA_NBDS_MASK	0xF
-#define CORE_TX_BD_DATA_NBDS_SHIFT                8
-#define CORE_TX_BD_DATA_ROCE_FLAV_MASK	0x1
-#define CORE_TX_BD_DATA_ROCE_FLAV_SHIFT           12
-#define CORE_TX_BD_DATA_IP_LEN_MASK	0x1
-#define CORE_TX_BD_DATA_IP_LEN_SHIFT              13
-#define CORE_TX_BD_DATA_RESERVED0_MASK            0x3
-#define CORE_TX_BD_DATA_RESERVED0_SHIFT           14
-};
-
+#define CORE_TX_BD_DATA_L4_PSEUDO_CSUM_MODE_SHIFT	7
+#define CORE_TX_BD_DATA_NBDS_MASK			0xF
+#define CORE_TX_BD_DATA_NBDS_SHIFT			8
+#define CORE_TX_BD_DATA_ROCE_FLAV_MASK			0x1
+#define CORE_TX_BD_DATA_ROCE_FLAV_SHIFT			12
+#define CORE_TX_BD_DATA_IP_LEN_MASK			0x1
+#define CORE_TX_BD_DATA_IP_LEN_SHIFT			13
+#define CORE_TX_BD_DATA_DISABLE_STAG_INSERTION_MASK	0x1
+#define CORE_TX_BD_DATA_DISABLE_STAG_INSERTION_SHIFT	14
+#define CORE_TX_BD_DATA_RESERVED0_MASK			0x1
+#define CORE_TX_BD_DATA_RESERVED0_SHIFT			15
+};
+
+/* Core TX BD for Light L2 */
 struct core_tx_bd {
 	struct regpair addr;
 	__le16 nbytes;
 	__le16 nw_vlan_or_lb_echo;
 	struct core_tx_bd_data bd_data;
 	__le16 bitfield1;
-#define CORE_TX_BD_L4_HDR_OFFSET_W_MASK	0x3FFF
-#define CORE_TX_BD_L4_HDR_OFFSET_W_SHIFT 0
-#define CORE_TX_BD_TX_DST_MASK		0x3
-#define CORE_TX_BD_TX_DST_SHIFT		14
+#define CORE_TX_BD_L4_HDR_OFFSET_W_MASK		0x3FFF
+#define CORE_TX_BD_L4_HDR_OFFSET_W_SHIFT	0
+#define CORE_TX_BD_TX_DST_MASK			0x3
+#define CORE_TX_BD_TX_DST_SHIFT			14
 };
 
+/* Light L2 TX Destination */
 enum core_tx_dest {
 	CORE_TX_DEST_NW,
 	CORE_TX_DEST_LB,
@@ -789,6 +339,7 @@ enum core_tx_dest {
 	MAX_CORE_TX_DEST
 };
 
+/* Ramrod data for tx queue start ramrod */
 struct core_tx_start_ramrod_data {
 	struct regpair pbl_base_addr;
 	__le16 mtu;
@@ -803,10 +354,20 @@ struct core_tx_start_ramrod_data {
 	u8 resrved[3];
 };
 
+/* Ramrod data for tx queue stop ramrod */
 struct core_tx_stop_ramrod_data {
 	__le32 reserved0[2];
 };
 
+/* Ramrod data for tx queue update ramrod */
+struct core_tx_update_ramrod_data {
+	u8 update_qm_pq_id_flg;
+	u8 reserved0;
+	__le16 qm_pq_id;
+	__le32 reserved1[1];
+};
+
+/* Enum flag for what type of dcb data to update */
 enum dcb_dscp_update_mode {
 	DONT_UPDATE_DCB_DSCP,
 	UPDATE_DCB,
@@ -815,6 +376,487 @@ enum dcb_dscp_update_mode {
 	MAX_DCB_DSCP_UPDATE_MODE
 };
 
+/* The core storm context for the Ystorm */
+struct ystorm_core_conn_st_ctx {
+	__le32 reserved[4];
+};
+
+/* The core storm context for the Pstorm */
+struct pstorm_core_conn_st_ctx {
+	__le32 reserved[4];
+};
+
+/* Core Slowpath Connection storm context of Xstorm */
+struct xstorm_core_conn_st_ctx {
+	__le32 spq_base_lo;
+	__le32 spq_base_hi;
+	struct regpair consolid_base_addr;
+	__le16 spq_cons;
+	__le16 consolid_cons;
+	__le32 reserved0[55];
+};
+
+struct e4_xstorm_core_conn_ag_ctx {
+	u8 reserved0;
+	u8 state;
+	u8 flags0;
+#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED1_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED1_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED2_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED2_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED3_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED3_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED4_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED4_SHIFT	5
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED5_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED5_SHIFT	6
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED6_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED6_SHIFT	7
+	u8 flags1;
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED7_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED7_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED8_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED8_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED9_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED9_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT11_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT12_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT12_SHIFT		4
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT13_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT13_SHIFT		5
+#define E4_XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT	6
+#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT	7
+	u8 flags2;
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF0_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF1_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF2_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF3_SHIFT	6
+	u8 flags3;
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF4_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF5_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF6_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF7_SHIFT	6
+	u8 flags4;
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF11_SHIFT	6
+	u8 flags5;
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF12_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF12_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF13_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF13_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF14_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF14_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF15_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF15_SHIFT	6
+	u8 flags6;
+#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF17_MASK			0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF17_SHIFT			2
+#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_MASK			0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_SHIFT			4
+#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_MASK		0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_SHIFT		6
+	u8 flags7;
+#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED10_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED10_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT		6
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT		7
+	u8 flags8;
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF3EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF4EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF5EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT	3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF6EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF7EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT	5
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF8EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT	6
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF9EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT	7
+	u8 flags9;
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF10EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT			0
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF11EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF11EN_SHIFT			1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF12EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF12EN_SHIFT			2
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF13EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF13EN_SHIFT			3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF14EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF14EN_SHIFT			4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF15EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF15EN_SHIFT			5
+#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN_SHIFT	6
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF17EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF17EN_SHIFT			7
+	u8 flags10;
+#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN_SHIFT		0
+#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT		2
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED11_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED11_SHIFT		3
+#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT		4
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF23EN_MASK			0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF23EN_SHIFT			5
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED12_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED12_SHIFT		6
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED13_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED13_SHIFT		7
+	u8 flags11;
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED14_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED14_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED15_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RESERVED15_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT	3
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT	5
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE9EN_MASK		0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE9EN_SHIFT	7
+	u8 flags12;
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE10EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE10EN_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE11EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE11EN_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE14EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE14EN_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE15EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE15EN_SHIFT	5
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE16EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE16EN_SHIFT	6
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE17EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE17EN_SHIFT	7
+	u8 flags13;
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE18EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE18EN_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE19EN_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_RULE19EN_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
+	u8 flags14;
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT16_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT16_SHIFT	0
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT17_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT17_SHIFT	1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT18_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT18_SHIFT	2
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT19_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT19_SHIFT	3
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT20_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT20_SHIFT	4
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT21_MASK	0x1
+#define E4_XSTORM_CORE_CONN_AG_CTX_BIT21_SHIFT	5
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF23_MASK	0x3
+#define E4_XSTORM_CORE_CONN_AG_CTX_CF23_SHIFT	6
+	u8 byte2;
+	__le16 physical_q0;
+	__le16 consolid_prod;
+	__le16 reserved16;
+	__le16 tx_bd_cons;
+	__le16 tx_bd_or_spq_prod;
+	__le16 word5;
+	__le16 conn_dpi;
+	u8 byte3;
+	u8 byte4;
+	u8 byte5;
+	u8 byte6;
+	__le32 reg0;
+	__le32 reg1;
+	__le32 reg2;
+	__le32 reg3;
+	__le32 reg4;
+	__le32 reg5;
+	__le32 reg6;
+	__le16 word7;
+	__le16 word8;
+	__le16 word9;
+	__le16 word10;
+	__le32 reg7;
+	__le32 reg8;
+	__le32 reg9;
+	u8 byte7;
+	u8 byte8;
+	u8 byte9;
+	u8 byte10;
+	u8 byte11;
+	u8 byte12;
+	u8 byte13;
+	u8 byte14;
+	u8 byte15;
+	u8 e5_reserved;
+	__le16 word11;
+	__le32 reg10;
+	__le32 reg11;
+	__le32 reg12;
+	__le32 reg13;
+	__le32 reg14;
+	__le32 reg15;
+	__le32 reg16;
+	__le32 reg17;
+	__le32 reg18;
+	__le32 reg19;
+	__le16 word12;
+	__le16 word13;
+	__le16 word14;
+	__le16 word15;
+};
+
+struct e4_tstorm_core_conn_ag_ctx {
+	u8 byte0;
+	u8 byte1;
+	u8 flags0;
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT2_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT2_SHIFT	2
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT3_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT3_SHIFT	3
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT4_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT4_SHIFT	4
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT5_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_BIT5_SHIFT	5
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF0_SHIFT	6
+	u8 flags1;
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF1_SHIFT	0
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF2_SHIFT	2
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF3_SHIFT	4
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF4_SHIFT	6
+	u8 flags2;
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF5_SHIFT	0
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF6_SHIFT	2
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF7_SHIFT	4
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF8_SHIFT	6
+	u8 flags3;
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF9_SHIFT	0
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF10_SHIFT	2
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT	4
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT	5
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT	6
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF3EN_MASK	0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT	7
+	u8 flags4;
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT		0
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT		1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT		2
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF7EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF7EN_SHIFT		3
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF8EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF8EN_SHIFT		4
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF9EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF9EN_SHIFT		5
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF10EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_CF10EN_SHIFT		6
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT	7
+	u8 flags5;
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_TSTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT	7
+	__le32 reg0;
+	__le32 reg1;
+	__le32 reg2;
+	__le32 reg3;
+	__le32 reg4;
+	__le32 reg5;
+	__le32 reg6;
+	__le32 reg7;
+	__le32 reg8;
+	u8 byte2;
+	u8 byte3;
+	__le16 word0;
+	u8 byte4;
+	u8 byte5;
+	__le16 word1;
+	__le16 word2;
+	__le16 word3;
+	__le32 reg9;
+	__le32 reg10;
+};
+
+struct e4_ustorm_core_conn_ag_ctx {
+	u8 reserved;
+	u8 byte1;
+	u8 flags0;
+#define E4_USTORM_CORE_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_USTORM_CORE_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_USTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_USTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF2_SHIFT	6
+	u8 flags1;
+#define E4_USTORM_CORE_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF3_SHIFT	0
+#define E4_USTORM_CORE_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF4_SHIFT	2
+#define E4_USTORM_CORE_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF5_SHIFT	4
+#define E4_USTORM_CORE_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF6_SHIFT	6
+	u8 flags2;
+#define E4_USTORM_CORE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_USTORM_CORE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_USTORM_CORE_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF3EN_SHIFT		3
+#define E4_USTORM_CORE_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF4EN_SHIFT		4
+#define E4_USTORM_CORE_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF5EN_SHIFT		5
+#define E4_USTORM_CORE_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_CF6EN_SHIFT		6
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT	7
+	u8 flags3;
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_USTORM_CORE_CONN_AG_CTX_RULE8EN_SHIFT	7
+	u8 byte2;
+	u8 byte3;
+	__le16 word0;
+	__le16 word1;
+	__le32 rx_producers;
+	__le32 reg1;
+	__le32 reg2;
+	__le32 reg3;
+	__le16 word2;
+	__le16 word3;
+};
+
+/* The core storm context for the Mstorm */
+struct mstorm_core_conn_st_ctx {
+	__le32 reserved[24];
+};
+
+/* The core storm context for the Ustorm */
+struct ustorm_core_conn_st_ctx {
+	__le32 reserved[4];
+};
+
+/* core connection context */
+struct e4_core_conn_context {
+	struct ystorm_core_conn_st_ctx ystorm_st_context;
+	struct regpair ystorm_st_padding[2];
+	struct pstorm_core_conn_st_ctx pstorm_st_context;
+	struct regpair pstorm_st_padding[2];
+	struct xstorm_core_conn_st_ctx xstorm_st_context;
+	struct e4_xstorm_core_conn_ag_ctx xstorm_ag_context;
+	struct e4_tstorm_core_conn_ag_ctx tstorm_ag_context;
+	struct e4_ustorm_core_conn_ag_ctx ustorm_ag_context;
+	struct mstorm_core_conn_st_ctx mstorm_st_context;
+	struct ustorm_core_conn_st_ctx ustorm_st_context;
+	struct regpair ustorm_st_padding[2];
+};
+
 struct eth_mstorm_per_pf_stat {
 	struct regpair gre_discard_pkts;
 	struct regpair vxlan_discard_pkts;
@@ -896,6 +938,50 @@ struct eth_ustorm_per_queue_stat {
 	struct regpair rcv_bcast_pkts;
 };
 
+/* Event Ring VF-PF Channel data */
+struct vf_pf_channel_eqe_data {
+	struct regpair msg_addr;
+};
+
+/* Event Ring malicious VF data */
+struct malicious_vf_eqe_data {
+	u8 vf_id;
+	u8 err_id;
+	__le16 reserved[3];
+};
+
+/* Event Ring initial cleanup data */
+struct initial_cleanup_eqe_data {
+	u8 vf_id;
+	u8 reserved[7];
+};
+
+/* Event Data Union */
+union event_ring_data {
+	u8 bytes[8];
+	struct vf_pf_channel_eqe_data vf_pf_channel;
+	struct iscsi_eqe_data iscsi_info;
+	struct iscsi_connect_done_results iscsi_conn_done_info;
+	union rdma_eqe_data rdma_data;
+	struct malicious_vf_eqe_data malicious_vf;
+	struct initial_cleanup_eqe_data vf_init_cleanup;
+};
+
+/* Event Ring Entry */
+struct event_ring_entry {
+	u8 protocol_id;
+	u8 opcode;
+	__le16 reserved0;
+	__le16 echo;
+	u8 fw_return_code;
+	u8 flags;
+#define EVENT_RING_ENTRY_ASYNC_MASK		0x1
+#define EVENT_RING_ENTRY_ASYNC_SHIFT		0
+#define EVENT_RING_ENTRY_RESERVED1_MASK		0x7F
+#define EVENT_RING_ENTRY_RESERVED1_SHIFT	1
+	union event_ring_data data;
+};
+
 /* Event Ring Next Page Address */
 struct event_ring_next_addr {
 	struct regpair addr;
@@ -908,12 +994,21 @@ union event_ring_element {
 	struct event_ring_next_addr next_addr;
 };
 
+/* Ports mode */
 enum fw_flow_ctrl_mode {
 	flow_ctrl_pause,
 	flow_ctrl_pfc,
 	MAX_FW_FLOW_CTRL_MODE
 };
 
+/* GFT profile type */
+enum gft_profile_type {
+	GFT_PROFILE_TYPE_4_TUPLE,
+	GFT_PROFILE_TYPE_L4_DST_PORT,
+	GFT_PROFILE_TYPE_IP_DST_PORT,
+	MAX_GFT_PROFILE_TYPE
+};
+
 /* Major and Minor hsi Versions */
 struct hsi_fp_ver_struct {
 	u8 minor_ver_arr[2];
@@ -921,14 +1016,14 @@ struct hsi_fp_ver_struct {
 };
 
 enum iwarp_ll2_tx_queues {
-	IWARP_LL2_IN_ORDER_TX_QUEUE =			1,
+	IWARP_LL2_IN_ORDER_TX_QUEUE = 1,
 	IWARP_LL2_ALIGNED_TX_QUEUE,
 	IWARP_LL2_ALIGNED_RIGHT_TRIMMED_TX_QUEUE,
 	IWARP_LL2_ERROR,
 	MAX_IWARP_LL2_TX_QUEUES
 };
 
-/* Mstorm non-triggering VF zone */
+/* Malicious VF error ID */
 enum malicious_vf_error_id {
 	MALICIOUS_VF_NO_ERROR,
 	VF_PF_CHANNEL_NOT_READY,
@@ -951,9 +1046,11 @@ enum malicious_vf_error_id {
 	ETH_TUNN_IPV6_EXT_NBD_ERR,
 	ETH_CONTROL_PACKET_VIOLATION,
 	ETH_ANTI_SPOOFING_ERR,
+	ETH_PACKET_SIZE_TOO_LARGE,
 	MAX_MALICIOUS_VF_ERROR_ID
 };
 
+/* Mstorm non-triggering VF zone */
 struct mstorm_non_trigger_vf_zone {
 	struct eth_mstorm_per_queue_stat eth_queue_stat;
 	struct eth_rx_prod_data eth_rx_queue_producers[ETH_MAX_NUM_RX_QUEUES_PER_VF_QUAD];
@@ -962,7 +1059,21 @@ struct mstorm_non_trigger_vf_zone {
 /* Mstorm VF zone */
 struct mstorm_vf_zone {
 	struct mstorm_non_trigger_vf_zone non_trigger;
+};
+
+/* vlan header including TPID and TCI fields */
+struct vlan_header {
+	__le16 tpid;
+	__le16 tci;
+};
 
+/* outer tag configurations */
+struct outer_tag_config_struct {
+	u8 enable_stag_pri_change;
+	u8 pri_map_valid;
+	u8 reserved[2];
+	struct vlan_header outer_tag;
+	u8 inner_to_outer_pri_map[8];
 };
 
 /* personality per PF */
@@ -974,7 +1085,7 @@ enum personality_type {
 	PERSONALITY_RDMA,
 	PERSONALITY_CORE,
 	PERSONALITY_ETH,
-	PERSONALITY_RESERVED4,
+	PERSONALITY_RESERVED,
 	MAX_PERSONALITY_TYPE
 };
 
@@ -997,7 +1108,6 @@ struct pf_start_ramrod_data {
 	struct regpair event_ring_pbl_addr;
 	struct regpair consolid_q_pbl_addr;
 	struct pf_start_tunnel_config tunnel_config;
-	__le32 reserved;
 	__le16 event_ring_sb_id;
 	u8 base_vf_id;
 	u8 num_vfs;
@@ -1011,21 +1121,22 @@ struct pf_start_ramrod_data {
 	u8 mf_mode;
 	u8 integ_phase;
 	u8 allow_npar_tx_switching;
-	u8 inner_to_outer_pri_map[8];
-	u8 pri_map_valid;
-	__le32 outer_tag;
+	u8 reserved0;
 	struct hsi_fp_ver_struct hsi_fp_ver;
+	struct outer_tag_config_struct outer_tag_config;
 };
 
+/* Data for port update ramrod */
 struct protocol_dcb_data {
 	u8 dcb_enable_flag;
-	u8 reserved_a;
+	u8 dscp_enable_flag;
 	u8 dcb_priority;
 	u8 dcb_tc;
-	u8 reserved_b;
+	u8 dscp_val;
 	u8 reserved0;
 };
 
+/* Update tunnel configuration */
 struct pf_update_tunnel_config {
 	u8 update_rx_pf_clss;
 	u8 update_rx_def_ucast_clss;
@@ -1042,8 +1153,8 @@ struct pf_update_tunnel_config {
 	__le16 reserved;
 };
 
+/* Data for port update ramrod */
 struct pf_update_ramrod_data {
-	u8 pf_id;
 	u8 update_eth_dcb_data_mode;
 	u8 update_fcoe_dcb_data_mode;
 	u8 update_iscsi_dcb_data_mode;
@@ -1051,6 +1162,7 @@ struct pf_update_ramrod_data {
 	u8 update_rroce_dcb_data_mode;
 	u8 update_iwarp_dcb_data_mode;
 	u8 update_mf_vlan_flag;
+	u8 update_enable_stag_pri_change;
 	struct protocol_dcb_data eth_dcb_data;
 	struct protocol_dcb_data fcoe_dcb_data;
 	struct protocol_dcb_data iscsi_dcb_data;
@@ -1058,7 +1170,8 @@ struct pf_update_ramrod_data {
 	struct protocol_dcb_data rroce_dcb_data;
 	struct protocol_dcb_data iwarp_dcb_data;
 	__le16 mf_vlan;
-	__le16 reserved;
+	u8 enable_stag_pri_change;
+	u8 reserved;
 	struct pf_update_tunnel_config tunnel_config;
 };
 
@@ -1079,11 +1192,13 @@ enum protocol_version_array_key {
 	MAX_PROTOCOL_VERSION_ARRAY_KEY
 };
 
+/* RDMA TX Stats */
 struct rdma_sent_stats {
 	struct regpair sent_bytes;
 	struct regpair sent_pkts;
 };
 
+/* Pstorm non-triggering VF zone */
 struct pstorm_non_trigger_vf_zone {
 	struct eth_pstorm_per_queue_stat eth_queue_stat;
 	struct rdma_sent_stats rdma_stats;
@@ -1103,11 +1218,34 @@ struct ramrod_header {
 	__le16 echo;
 };
 
+/* RDMA RX Stats */
 struct rdma_rcv_stats {
 	struct regpair rcv_bytes;
 	struct regpair rcv_pkts;
 };
 
+/* Data for update QCN/DCQCN RL ramrod */
+struct rl_update_ramrod_data {
+	u8 qcn_update_param_flg;
+	u8 dcqcn_update_param_flg;
+	u8 rl_init_flg;
+	u8 rl_start_flg;
+	u8 rl_stop_flg;
+	u8 rl_id_first;
+	u8 rl_id_last;
+	u8 rl_dc_qcn_flg;
+	__le32 rl_bc_rate;
+	__le16 rl_max_rate;
+	__le16 rl_r_ai;
+	__le16 rl_r_hai;
+	__le16 dcqcn_g;
+	__le32 dcqcn_k_us;
+	__le32 dcqcn_timeuot_us;
+	__le32 qcn_timeuot_us;
+	__le32 reserved[2];
+};
+
+/* Slowpath Element (SPQE) */
 struct slow_path_element {
 	struct ramrod_header hdr;
 	struct regpair data_ptr;
@@ -1130,11 +1268,12 @@ struct tstorm_per_port_stat {
 	struct regpair roce_irregular_pkt;
 	struct regpair iwarp_irregular_pkt;
 	struct regpair eth_irregular_pkt;
-	struct regpair reserved1;
+	struct regpair toe_irregular_pkt;
 	struct regpair preroce_irregular_pkt;
 	struct regpair eth_gre_tunn_filter_discard;
 	struct regpair eth_vxlan_tunn_filter_discard;
 	struct regpair eth_geneve_tunn_filter_discard;
+	struct regpair eth_gft_drop_pkt;
 };
 
 /* Tstorm VF zone */
@@ -1197,6 +1336,7 @@ struct vf_stop_ramrod_data {
 	__le32 reserved2;
 };
 
+/* VF zone size mode */
 enum vf_zone_size_mode {
 	VF_ZONE_SIZE_MODE_DEFAULT,
 	VF_ZONE_SIZE_MODE_DOUBLE,
@@ -1204,6 +1344,7 @@ enum vf_zone_size_mode {
 	MAX_VF_ZONE_SIZE_MODE
 };
 
+/* Attentions status block */
 struct atten_status_block {
 	__le32 atten_bits;
 	__le32 atten_ack;
@@ -1212,12 +1353,6 @@ struct atten_status_block {
 	__le32 reserved1;
 };
 
-enum command_type_bit {
-	IGU_COMMAND_TYPE_NOP = 0,
-	IGU_COMMAND_TYPE_SET = 1,
-	MAX_COMMAND_TYPE_BIT
-};
-
 /* DMAE command */
 struct dmae_cmd {
 	__le32 opcode;
@@ -1327,74 +1462,74 @@ enum dmae_cmd_src_enum {
 	MAX_DMAE_CMD_SRC_ENUM
 };
 
-struct mstorm_core_conn_ag_ctx {
+struct e4_mstorm_core_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define MSTORM_CORE_CONN_AG_CTX_BIT0_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT	0
-#define MSTORM_CORE_CONN_AG_CTX_BIT1_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT	1
-#define MSTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
-#define MSTORM_CORE_CONN_AG_CTX_CF0_SHIFT	2
-#define MSTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
-#define MSTORM_CORE_CONN_AG_CTX_CF1_SHIFT	4
-#define MSTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
-#define MSTORM_CORE_CONN_AG_CTX_CF2_SHIFT	6
+#define E4_MSTORM_CORE_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_MSTORM_CORE_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define MSTORM_CORE_CONN_AG_CTX_CF0EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT	0
-#define MSTORM_CORE_CONN_AG_CTX_CF1EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT	1
-#define MSTORM_CORE_CONN_AG_CTX_CF2EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT	2
-#define MSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT	3
-#define MSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT	4
-#define MSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT	5
-#define MSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT	6
-#define MSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define MSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT	7
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_MSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT	7
 	__le16 word0;
 	__le16 word1;
 	__le32 reg0;
 	__le32 reg1;
 };
 
-struct ystorm_core_conn_ag_ctx {
+struct e4_ystorm_core_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define YSTORM_CORE_CONN_AG_CTX_BIT0_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT	0
-#define YSTORM_CORE_CONN_AG_CTX_BIT1_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT	1
-#define YSTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
-#define YSTORM_CORE_CONN_AG_CTX_CF0_SHIFT	2
-#define YSTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
-#define YSTORM_CORE_CONN_AG_CTX_CF1_SHIFT	4
-#define YSTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
-#define YSTORM_CORE_CONN_AG_CTX_CF2_SHIFT	6
+#define E4_YSTORM_CORE_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_YSTORM_CORE_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define YSTORM_CORE_CONN_AG_CTX_CF0EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT	0
-#define YSTORM_CORE_CONN_AG_CTX_CF1EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT	1
-#define YSTORM_CORE_CONN_AG_CTX_CF2EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT	2
-#define YSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT	3
-#define YSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT	4
-#define YSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT	5
-#define YSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT	6
-#define YSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define YSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT	7
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_YSTORM_CORE_CONN_AG_CTX_RULE4EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -1545,22 +1680,22 @@ struct qm_rf_opportunistic_mask {
 };
 
 /* QM hardware structure of QM map memory */
-struct qm_rf_pq_map {
+struct qm_rf_pq_map_e4 {
 	__le32 reg;
-#define QM_RF_PQ_MAP_PQ_VALID_MASK		0x1
-#define QM_RF_PQ_MAP_PQ_VALID_SHIFT		0
-#define QM_RF_PQ_MAP_RL_ID_MASK			0xFF
-#define QM_RF_PQ_MAP_RL_ID_SHIFT		1
-#define QM_RF_PQ_MAP_VP_PQ_ID_MASK		0x1FF
-#define QM_RF_PQ_MAP_VP_PQ_ID_SHIFT		9
-#define QM_RF_PQ_MAP_VOQ_MASK			0x1F
-#define QM_RF_PQ_MAP_VOQ_SHIFT			18
-#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_MASK	0x3
-#define QM_RF_PQ_MAP_WRR_WEIGHT_GROUP_SHIFT	23
-#define QM_RF_PQ_MAP_RL_VALID_MASK		0x1
-#define QM_RF_PQ_MAP_RL_VALID_SHIFT		25
-#define QM_RF_PQ_MAP_RESERVED_MASK		0x3F
-#define QM_RF_PQ_MAP_RESERVED_SHIFT		26
+#define QM_RF_PQ_MAP_E4_PQ_VALID_MASK		0x1
+#define QM_RF_PQ_MAP_E4_PQ_VALID_SHIFT		0
+#define QM_RF_PQ_MAP_E4_RL_ID_MASK		0xFF
+#define QM_RF_PQ_MAP_E4_RL_ID_SHIFT		1
+#define QM_RF_PQ_MAP_E4_VP_PQ_ID_MASK		0x1FF
+#define QM_RF_PQ_MAP_E4_VP_PQ_ID_SHIFT		9
+#define QM_RF_PQ_MAP_E4_VOQ_MASK		0x1F
+#define QM_RF_PQ_MAP_E4_VOQ_SHIFT		18
+#define QM_RF_PQ_MAP_E4_WRR_WEIGHT_GROUP_MASK	0x3
+#define QM_RF_PQ_MAP_E4_WRR_WEIGHT_GROUP_SHIFT	23
+#define QM_RF_PQ_MAP_E4_RL_VALID_MASK		0x1
+#define QM_RF_PQ_MAP_E4_RL_VALID_SHIFT		25
+#define QM_RF_PQ_MAP_E4_RESERVED_MASK		0x3F
+#define QM_RF_PQ_MAP_E4_RESERVED_SHIFT		26
 };
 
 /* Completion params for aggregated interrupt completion */
@@ -1643,8 +1778,8 @@ enum block_addr {
 	GRCBASE_MULD = 0x4e0000,
 	GRCBASE_YULD = 0x4c8000,
 	GRCBASE_XYLD = 0x4c0000,
-	GRCBASE_PTLD = 0x590000,
-	GRCBASE_YPLD = 0x5b0000,
+	GRCBASE_PTLD = 0x5a0000,
+	GRCBASE_YPLD = 0x5c0000,
 	GRCBASE_PRM = 0x230000,
 	GRCBASE_PBF_PB1 = 0xda0000,
 	GRCBASE_PBF_PB2 = 0xda4000,
@@ -1675,6 +1810,7 @@ enum block_addr {
 	GRCBASE_PHY_PCIE = 0x620000,
 	GRCBASE_LED = 0x6b8000,
 	GRCBASE_AVS_WRAP = 0x6b0000,
+	GRCBASE_PXPREQBUS = 0x56000,
 	GRCBASE_MISC_AEU = 0x8000,
 	GRCBASE_BAR0_MAP = 0x1c00000,
 	MAX_BLOCK_ADDR
@@ -1766,6 +1902,7 @@ enum block_id {
 	BLOCK_PHY_PCIE,
 	BLOCK_LED,
 	BLOCK_AVS_WRAP,
+	BLOCK_PXPREQBUS,
 	BLOCK_MISC_AEU,
 	BLOCK_BAR0_MAP,
 	MAX_BLOCK_ID
@@ -1841,7 +1978,7 @@ struct dbg_attn_block_result {
 	struct dbg_attn_reg_result reg_results[15];
 };
 
-/* mode header */
+/* Mode header */
 struct dbg_mode_hdr {
 	__le16 data;
 #define DBG_MODE_HDR_EVAL_MODE_MASK		0x1
@@ -1863,80 +2000,83 @@ struct dbg_attn_reg {
 	__le32 mask_address;
 };
 
-/* attention types */
+/* Attention types */
 enum dbg_attn_type {
 	ATTN_TYPE_INTERRUPT,
 	ATTN_TYPE_PARITY,
 	MAX_DBG_ATTN_TYPE
 };
 
+/* Debug Bus block data */
 struct dbg_bus_block {
 	u8 num_of_lines;
 	u8 has_latency_events;
 	__le16 lines_offset;
 };
 
+/* Debug Bus block user data */
 struct dbg_bus_block_user_data {
 	u8 num_of_lines;
 	u8 has_latency_events;
 	__le16 names_offset;
 };
 
+/* Block Debug line data */
 struct dbg_bus_line {
 	u8 data;
-#define DBG_BUS_LINE_NUM_OF_GROUPS_MASK  0xF
-#define DBG_BUS_LINE_NUM_OF_GROUPS_SHIFT 0
-#define DBG_BUS_LINE_IS_256B_MASK        0x1
-#define DBG_BUS_LINE_IS_256B_SHIFT       4
-#define DBG_BUS_LINE_RESERVED_MASK       0x7
-#define DBG_BUS_LINE_RESERVED_SHIFT      5
+#define DBG_BUS_LINE_NUM_OF_GROUPS_MASK		0xF
+#define DBG_BUS_LINE_NUM_OF_GROUPS_SHIFT	0
+#define DBG_BUS_LINE_IS_256B_MASK		0x1
+#define DBG_BUS_LINE_IS_256B_SHIFT		4
+#define DBG_BUS_LINE_RESERVED_MASK		0x7
+#define DBG_BUS_LINE_RESERVED_SHIFT		5
 	u8 group_sizes;
 };
 
-/* condition header for registers dump */
+/* Condition header for registers dump */
 struct dbg_dump_cond_hdr {
 	struct dbg_mode_hdr mode; /* Mode header */
 	u8 block_id; /* block ID */
 	u8 data_size; /* size in dwords of the data following this header */
 };
 
-/* memory data for registers dump */
+/* Memory data for registers dump */
 struct dbg_dump_mem {
 	__le32 dword0;
-#define DBG_DUMP_MEM_ADDRESS_MASK       0xFFFFFF
-#define DBG_DUMP_MEM_ADDRESS_SHIFT      0
-#define DBG_DUMP_MEM_MEM_GROUP_ID_MASK  0xFF
-#define DBG_DUMP_MEM_MEM_GROUP_ID_SHIFT 24
+#define DBG_DUMP_MEM_ADDRESS_MASK	0xFFFFFF
+#define DBG_DUMP_MEM_ADDRESS_SHIFT	0
+#define DBG_DUMP_MEM_MEM_GROUP_ID_MASK	0xFF
+#define DBG_DUMP_MEM_MEM_GROUP_ID_SHIFT	24
 	__le32 dword1;
-#define DBG_DUMP_MEM_LENGTH_MASK        0xFFFFFF
-#define DBG_DUMP_MEM_LENGTH_SHIFT       0
-#define DBG_DUMP_MEM_WIDE_BUS_MASK      0x1
-#define DBG_DUMP_MEM_WIDE_BUS_SHIFT     24
-#define DBG_DUMP_MEM_RESERVED_MASK      0x7F
-#define DBG_DUMP_MEM_RESERVED_SHIFT     25
+#define DBG_DUMP_MEM_LENGTH_MASK	0xFFFFFF
+#define DBG_DUMP_MEM_LENGTH_SHIFT	0
+#define DBG_DUMP_MEM_WIDE_BUS_MASK	0x1
+#define DBG_DUMP_MEM_WIDE_BUS_SHIFT	24
+#define DBG_DUMP_MEM_RESERVED_MASK	0x7F
+#define DBG_DUMP_MEM_RESERVED_SHIFT	25
 };
 
-/* register data for registers dump */
+/* Register data for registers dump */
 struct dbg_dump_reg {
 	__le32 data;
-#define DBG_DUMP_REG_ADDRESS_MASK 0x7FFFFF /* register address (in dwords) */
-#define DBG_DUMP_REG_ADDRESS_SHIFT 0
-#define DBG_DUMP_REG_WIDE_BUS_MASK 0x1 /* indicates register is wide-bus */
-#define DBG_DUMP_REG_WIDE_BUS_SHIFT 23
-#define DBG_DUMP_REG_LENGTH_MASK  0xFF /* register size (in dwords) */
-#define DBG_DUMP_REG_LENGTH_SHIFT 24
+#define DBG_DUMP_REG_ADDRESS_MASK	0x7FFFFF
+#define DBG_DUMP_REG_ADDRESS_SHIFT	0
+#define DBG_DUMP_REG_WIDE_BUS_MASK	0x1
+#define DBG_DUMP_REG_WIDE_BUS_SHIFT	23
+#define DBG_DUMP_REG_LENGTH_MASK	0xFF
+#define DBG_DUMP_REG_LENGTH_SHIFT	24
 };
 
-/* split header for registers dump */
+/* Split header for registers dump */
 struct dbg_dump_split_hdr {
 	__le32 hdr;
-#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_MASK      0xFFFFFF
-#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_SHIFT     0
-#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_MASK  0xFF
-#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_SHIFT 24
+#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_MASK	0xFFFFFF
+#define DBG_DUMP_SPLIT_HDR_DATA_SIZE_SHIFT	0
+#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_MASK	0xFF
+#define DBG_DUMP_SPLIT_HDR_SPLIT_TYPE_ID_SHIFT	24
 };
 
-/* condition header for idle check */
+/* Condition header for idle check */
 struct dbg_idle_chk_cond_hdr {
 	struct dbg_mode_hdr mode; /* Mode header */
 	__le16 data_size; /* size in dwords of the data following this header */
@@ -1945,12 +2085,12 @@ struct dbg_idle_chk_cond_hdr {
 /* Idle Check condition register */
 struct dbg_idle_chk_cond_reg {
 	__le32 data;
-#define DBG_IDLE_CHK_COND_REG_ADDRESS_MASK   0x7FFFFF
-#define DBG_IDLE_CHK_COND_REG_ADDRESS_SHIFT  0
-#define DBG_IDLE_CHK_COND_REG_WIDE_BUS_MASK  0x1
-#define DBG_IDLE_CHK_COND_REG_WIDE_BUS_SHIFT 23
-#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_MASK  0xFF
-#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_SHIFT 24
+#define DBG_IDLE_CHK_COND_REG_ADDRESS_MASK	0x7FFFFF
+#define DBG_IDLE_CHK_COND_REG_ADDRESS_SHIFT	0
+#define DBG_IDLE_CHK_COND_REG_WIDE_BUS_MASK	0x1
+#define DBG_IDLE_CHK_COND_REG_WIDE_BUS_SHIFT	23
+#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_MASK	0xFF
+#define DBG_IDLE_CHK_COND_REG_BLOCK_ID_SHIFT	24
 	__le16 num_entries;
 	u8 entry_size;
 	u8 start_entry;
@@ -1959,12 +2099,12 @@ struct dbg_idle_chk_cond_reg {
 /* Idle Check info register */
 struct dbg_idle_chk_info_reg {
 	__le32 data;
-#define DBG_IDLE_CHK_INFO_REG_ADDRESS_MASK   0x7FFFFF
-#define DBG_IDLE_CHK_INFO_REG_ADDRESS_SHIFT  0
-#define DBG_IDLE_CHK_INFO_REG_WIDE_BUS_MASK  0x1
-#define DBG_IDLE_CHK_INFO_REG_WIDE_BUS_SHIFT 23
-#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_MASK  0xFF
-#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_SHIFT 24
+#define DBG_IDLE_CHK_INFO_REG_ADDRESS_MASK	0x7FFFFF
+#define DBG_IDLE_CHK_INFO_REG_ADDRESS_SHIFT	0
+#define DBG_IDLE_CHK_INFO_REG_WIDE_BUS_MASK	0x1
+#define DBG_IDLE_CHK_INFO_REG_WIDE_BUS_SHIFT	23
+#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_MASK	0xFF
+#define DBG_IDLE_CHK_INFO_REG_BLOCK_ID_SHIFT	24
 	__le16 size; /* register size in dwords */
 	struct dbg_mode_hdr mode; /* Mode header */
 };
@@ -2016,13 +2156,13 @@ struct dbg_idle_chk_rule {
 /* Idle Check rule parsing data */
 struct dbg_idle_chk_rule_parsing_data {
 	__le32 data;
-#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_MASK  0x1
-#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_SHIFT 0
-#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_MASK  0x7FFFFFFF
-#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_SHIFT 1
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_MASK	0x1
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_HAS_FW_MSG_SHIFT	0
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_MASK	0x7FFFFFFF
+#define DBG_IDLE_CHK_RULE_PARSING_DATA_STR_OFFSET_SHIFT	1
 };
 
-/* idle check severity types */
+/* Idle check severity types */
 enum dbg_idle_chk_severity_types {
 	/* idle check failure should cause an error */
 	IDLE_CHK_SEVERITY_ERROR,
@@ -2036,14 +2176,14 @@ enum dbg_idle_chk_severity_types {
 /* Debug Bus block data */
 struct dbg_bus_block_data {
 	__le16 data;
-#define DBG_BUS_BLOCK_DATA_ENABLE_MASK_MASK       0xF
-#define DBG_BUS_BLOCK_DATA_ENABLE_MASK_SHIFT      0
-#define DBG_BUS_BLOCK_DATA_RIGHT_SHIFT_MASK       0xF
-#define DBG_BUS_BLOCK_DATA_RIGHT_SHIFT_SHIFT      4
-#define DBG_BUS_BLOCK_DATA_FORCE_VALID_MASK_MASK  0xF
-#define DBG_BUS_BLOCK_DATA_FORCE_VALID_MASK_SHIFT 8
-#define DBG_BUS_BLOCK_DATA_FORCE_FRAME_MASK_MASK  0xF
-#define DBG_BUS_BLOCK_DATA_FORCE_FRAME_MASK_SHIFT 12
+#define DBG_BUS_BLOCK_DATA_ENABLE_MASK_MASK		0xF
+#define DBG_BUS_BLOCK_DATA_ENABLE_MASK_SHIFT		0
+#define DBG_BUS_BLOCK_DATA_RIGHT_SHIFT_MASK		0xF
+#define DBG_BUS_BLOCK_DATA_RIGHT_SHIFT_SHIFT		4
+#define DBG_BUS_BLOCK_DATA_FORCE_VALID_MASK_MASK	0xF
+#define DBG_BUS_BLOCK_DATA_FORCE_VALID_MASK_SHIFT	8
+#define DBG_BUS_BLOCK_DATA_FORCE_FRAME_MASK_MASK	0xF
+#define DBG_BUS_BLOCK_DATA_FORCE_FRAME_MASK_SHIFT	12
 	u8 line_num;
 	u8 hw_id;
 };
@@ -2072,6 +2212,7 @@ enum dbg_bus_clients {
 	MAX_DBG_BUS_CLIENTS
 };
 
+/* Debug Bus constraint operation types */
 enum dbg_bus_constraint_ops {
 	DBG_BUS_CONSTRAINT_OP_EQ,
 	DBG_BUS_CONSTRAINT_OP_NE,
@@ -2086,12 +2227,13 @@ enum dbg_bus_constraint_ops {
 	MAX_DBG_BUS_CONSTRAINT_OPS
 };
 
+/* Debug Bus trigger state data */
 struct dbg_bus_trigger_state_data {
 	u8 data;
-#define DBG_BUS_TRIGGER_STATE_DATA_BLOCK_SHIFTED_ENABLE_MASK_MASK  0xF
-#define DBG_BUS_TRIGGER_STATE_DATA_BLOCK_SHIFTED_ENABLE_MASK_SHIFT 0
-#define DBG_BUS_TRIGGER_STATE_DATA_CONSTRAINT_DWORD_MASK_MASK      0xF
-#define DBG_BUS_TRIGGER_STATE_DATA_CONSTRAINT_DWORD_MASK_SHIFT     4
+#define DBG_BUS_TRIGGER_STATE_DATA_BLOCK_SHIFTED_ENABLE_MASK_MASK	0xF
+#define DBG_BUS_TRIGGER_STATE_DATA_BLOCK_SHIFTED_ENABLE_MASK_SHIFT	0
+#define DBG_BUS_TRIGGER_STATE_DATA_CONSTRAINT_DWORD_MASK_MASK		0xF
+#define DBG_BUS_TRIGGER_STATE_DATA_CONSTRAINT_DWORD_MASK_SHIFT		4
 };
 
 /* Debug Bus memory address */
@@ -2165,6 +2307,7 @@ struct dbg_bus_data {
 	struct dbg_bus_storm_data storms[6];
 };
 
+/* Debug bus filter types */
 enum dbg_bus_filter_types {
 	DBG_BUS_FILTER_TYPE_OFF,
 	DBG_BUS_FILTER_TYPE_PRE,
@@ -2181,6 +2324,7 @@ enum dbg_bus_frame_modes {
 	MAX_DBG_BUS_FRAME_MODES
 };
 
+/* Debug bus other engine mode */
 enum dbg_bus_other_engine_modes {
 	DBG_BUS_OTHER_ENGINE_MODE_NONE,
 	DBG_BUS_OTHER_ENGINE_MODE_DOUBLE_BW_TX,
@@ -2190,12 +2334,14 @@ enum dbg_bus_other_engine_modes {
 	MAX_DBG_BUS_OTHER_ENGINE_MODES
 };
 
+/* Debug bus post-trigger recording types */
 enum dbg_bus_post_trigger_types {
 	DBG_BUS_POST_TRIGGER_RECORD,
 	DBG_BUS_POST_TRIGGER_DROP,
 	MAX_DBG_BUS_POST_TRIGGER_TYPES
 };
 
+/* Debug bus pre-trigger recording types */
 enum dbg_bus_pre_trigger_types {
 	DBG_BUS_PRE_TRIGGER_START_FROM_ZERO,
 	DBG_BUS_PRE_TRIGGER_NUM_CHUNKS,
@@ -2203,11 +2349,10 @@ enum dbg_bus_pre_trigger_types {
 	MAX_DBG_BUS_PRE_TRIGGER_TYPES
 };
 
+/* Debug bus SEMI frame modes */
 enum dbg_bus_semi_frame_modes {
-	DBG_BUS_SEMI_FRAME_MODE_0SLOW_4FAST =
-	    0,
-	DBG_BUS_SEMI_FRAME_MODE_4SLOW_0FAST =
-	    3,
+	DBG_BUS_SEMI_FRAME_MODE_0SLOW_4FAST = 0,
+	DBG_BUS_SEMI_FRAME_MODE_4SLOW_0FAST = 3,
 	MAX_DBG_BUS_SEMI_FRAME_MODES
 };
 
@@ -2220,6 +2365,7 @@ enum dbg_bus_states {
 	MAX_DBG_BUS_STATES
 };
 
+/* Debug Bus Storm modes */
 enum dbg_bus_storm_modes {
 	DBG_BUS_STORM_MODE_PRINTF,
 	DBG_BUS_STORM_MODE_PRAM_ADDR,
@@ -2352,7 +2498,7 @@ enum dbg_status {
 	DBG_STATUS_MCP_TRACE_NO_META,
 	DBG_STATUS_MCP_COULD_NOT_HALT,
 	DBG_STATUS_MCP_COULD_NOT_RESUME,
-	DBG_STATUS_DMAE_FAILED,
+	DBG_STATUS_RESERVED2,
 	DBG_STATUS_SEMI_FIFO_NOT_EMPTY,
 	DBG_STATUS_IGU_FIFO_BAD_DATA,
 	DBG_STATUS_MCP_COULD_NOT_MASK_PRTY,
@@ -2396,7 +2542,8 @@ struct dbg_tools_data {
 	u8 chip_id;
 	u8 platform_id;
 	u8 initialized;
-	u8 reserved;
+	u8 use_dmae;
+	__le32 num_regs_read;
 };
 
 /********************************/
@@ -2406,6 +2553,7 @@ struct dbg_tools_data {
 /* Number of VLAN priorities */
 #define NUM_OF_VLAN_PRIORITIES	8
 
+/* BRB RAM init requirements */
 struct init_brb_ram_req {
 	__le32 guranteed_per_tc;
 	__le32 headroom_per_tc;
@@ -2414,17 +2562,20 @@ struct init_brb_ram_req {
 	u8 num_active_tcs[MAX_NUM_PORTS];
 };
 
+/* ETS per-TC init requirements */
 struct init_ets_tc_req {
 	u8 use_sp;
 	u8 use_wfq;
 	__le16 weight;
 };
 
+/* ETS init requirements */
 struct init_ets_req {
 	__le32 mtu;
 	struct init_ets_tc_req tc_req[NUM_OF_TCS];
 };
 
+/* NIG LB RL init requirements */
 struct init_nig_lb_rl_req {
 	__le16 lb_mac_rate;
 	__le16 lb_rate;
@@ -2432,15 +2583,18 @@ struct init_nig_lb_rl_req {
 	__le16 tc_rate[NUM_OF_PHYS_TCS];
 };
 
+/* NIG TC mapping for each priority */
 struct init_nig_pri_tc_map_entry {
 	u8 tc_id;
 	u8 valid;
 };
 
+/* NIG priority to TC map init requirements */
 struct init_nig_pri_tc_map_req {
 	struct init_nig_pri_tc_map_entry pri[NUM_OF_VLAN_PRIORITIES];
 };
 
+/* QM per-port init parameters */
 struct init_qm_port_params {
 	u8 active;
 	u8 active_phys_tcs;
@@ -2563,7 +2717,7 @@ struct bin_buffer_hdr {
 	__le32 length;
 };
 
-/* binary init buffer types */
+/* Binary init buffer types */
 enum bin_init_buffer_type {
 	BIN_BUF_INIT_FW_VER_INFO,
 	BIN_BUF_INIT_CMD,
@@ -2793,6 +2947,7 @@ struct iro {
 };
 
 /***************************** Public Functions *******************************/
+
 /**
  * @brief qed_dbg_set_bin_ptr - Sets a pointer to the binary data with debug
  *	arrays.
@@ -2801,6 +2956,18 @@ struct iro {
  */
 enum dbg_status qed_dbg_set_bin_ptr(const u8 * const bin_ptr);
 
+/**
+ * @brief qed_read_regs - Reads registers into a buffer (using GRC).
+ *
+ * @param p_hwfn - HW device data
+ * @param p_ptt - Ptt window used for writing the registers.
+ * @param buf - Destination buffer.
+ * @param addr - Source GRC address in dwords.
+ * @param len - Number of registers to read.
+ */
+void qed_read_regs(struct qed_hwfn *p_hwfn,
+		   struct qed_ptt *p_ptt, u32 *buf, u32 addr, u32 len);
+
 /**
  * @brief qed_dbg_grc_set_params_default - Reverts all GRC parameters to their
  *	default value.
@@ -3119,6 +3286,7 @@ enum dbg_status qed_dbg_print_attn(struct qed_hwfn *p_hwfn,
 #define MAX_NAME_LEN	16
 
 /***************************** Public Functions *******************************/
+
 /**
  * @brief qed_dbg_user_set_bin_ptr - Sets a pointer to the binary data with
  *	debug arrays.
@@ -3171,6 +3339,18 @@ enum dbg_status qed_print_idle_chk_results(struct qed_hwfn *p_hwfn,
 					   u32 *num_errors,
 					   u32 *num_warnings);
 
+/**
+ * @brief qed_dbg_mcp_trace_set_meta_data - Sets a pointer to the MCP Trace
+ *	meta data.
+ *
+ * Needed in case the MCP Trace dump doesn't contain the meta data (e.g. due to
+ * no NVRAM access).
+ *
+ * @param data - pointer to MCP Trace meta data
+ * @param size - size of MCP Trace meta data in dwords
+ */
+void qed_dbg_mcp_trace_set_meta_data(u32 *data, u32 size);
+
 /**
  * @brief qed_get_mcp_trace_results_buf_size - Returns the required buffer size
  *	for MCP Trace results (in bytes).
@@ -3607,6 +3787,9 @@ static const u32 dbg_bus_blocks[] = {
 	0x00000000,		/* bar0_map, bb, 0 lines */
 	0x00000000,		/* bar0_map, k2, 0 lines */
 	0x00000000,
+	0x00000000,		/* bar0_map, bb, 0 lines */
+	0x00000000,		/* bar0_map, k2, 0 lines */
+	0x00000000,
 };
 
 /* Win 2 */
@@ -3645,7 +3828,6 @@ static const u32 dbg_bus_blocks[] = {
  * Returns the required host memory size in 4KB units.
  * Must be called before all QM init HSI functions.
  *
- * @param pf_id - physical function ID
  * @param num_pf_cids - number of connections used by this PF
  * @param num_vf_cids - number of connections used by VFs of this PF
  * @param num_tids - number of tasks used by this PF
@@ -3654,8 +3836,7 @@ static const u32 dbg_bus_blocks[] = {
  *
  * @return The required host memory size in 4KB units.
  */
-u32 qed_qm_pf_mem_size(u8 pf_id,
-		       u32 num_pf_cids,
+u32 qed_qm_pf_mem_size(u32 num_pf_cids,
 		       u32 num_vf_cids,
 		       u32 num_tids, u16 num_pf_pqs, u16 num_vf_pqs);
 
@@ -3676,7 +3857,7 @@ struct qed_qm_pf_rt_init_params {
 	u8 port_id;
 	u8 pf_id;
 	u8 max_phys_tcs_per_port;
-	bool is_first_pf;
+	bool is_pf_loading;
 	u32 num_pf_cids;
 	u32 num_vf_cids;
 	u32 num_tids;
@@ -3687,6 +3868,7 @@ struct qed_qm_pf_rt_init_params {
 	u8 num_vports;
 	u16 pf_wfq;
 	u32 pf_rl;
+	u32 link_speed;
 	struct init_qm_pq_params *pq_params;
 	struct init_qm_vport_params *vport_params;
 };
@@ -3744,11 +3926,14 @@ int qed_init_vport_wfq(struct qed_hwfn *p_hwfn,
  * @param p_ptt - ptt window used for writing the registers
  * @param vport_id - VPORT ID
  * @param vport_rl - rate limit in Mb/sec units
+ * @param link_speed - link speed in Mbps.
  *
  * @return 0 on success, -1 on error.
  */
 int qed_init_vport_rl(struct qed_hwfn *p_hwfn,
-		      struct qed_ptt *p_ptt, u8 vport_id, u32 vport_rl);
+		      struct qed_ptt *p_ptt,
+		      u8 vport_id, u32 vport_rl, u32 link_speed);
+
 /**
  * @brief qed_send_qm_stop_cmd  Sends a stop command to the QM
  *
@@ -3759,7 +3944,8 @@ int qed_init_vport_rl(struct qed_hwfn *p_hwfn,
  * @param start_pq - first PQ ID to stop
  * @param num_pqs - Number of PQs to stop, starting from start_pq.
  *
- * @return bool, true if successful, false if timeout occured while waiting for QM command done.
+ * @return bool, true if successful, false if timeout occurred while waiting for
+ *	QM command done.
  */
 bool qed_send_qm_stop_cmd(struct qed_hwfn *p_hwfn,
 			  struct qed_ptt *p_ptt,
@@ -3769,6 +3955,7 @@ bool qed_send_qm_stop_cmd(struct qed_hwfn *p_hwfn,
 /**
  * @brief qed_set_vxlan_dest_port - initializes vxlan tunnel destination udp port
  *
+ * @param p_hwfn
  * @param p_ptt - ptt window used for writing the registers.
  * @param dest_port - vxlan destination udp port.
  */
@@ -3778,6 +3965,7 @@ void qed_set_vxlan_dest_port(struct qed_hwfn *p_hwfn,
 /**
  * @brief qed_set_vxlan_enable - enable or disable VXLAN tunnel in HW
  *
+ * @param p_hwfn
  * @param p_ptt - ptt window used for writing the registers.
  * @param vxlan_enable - vxlan enable flag.
  */
@@ -3787,6 +3975,7 @@ void qed_set_vxlan_enable(struct qed_hwfn *p_hwfn,
 /**
  * @brief qed_set_gre_enable - enable or disable GRE tunnel in HW
  *
+ * @param p_hwfn
  * @param p_ptt - ptt window used for writing the registers.
  * @param eth_gre_enable - eth GRE enable enable flag.
  * @param ip_gre_enable - IP GRE enable enable flag.
@@ -3798,6 +3987,7 @@ void qed_set_gre_enable(struct qed_hwfn *p_hwfn,
 /**
  * @brief qed_set_geneve_dest_port - initializes geneve tunnel destination udp port
  *
+ * @param p_hwfn
  * @param p_ptt - ptt window used for writing the registers.
  * @param dest_port - geneve destination udp port.
  */
@@ -3814,612 +4004,921 @@ void qed_set_geneve_dest_port(struct qed_hwfn *p_hwfn,
 void qed_set_geneve_enable(struct qed_hwfn *p_hwfn,
 			   struct qed_ptt *p_ptt,
 			   bool eth_geneve_enable, bool ip_geneve_enable);
-void qed_set_rfs_mode_disable(struct qed_hwfn *p_hwfn,
-			      struct qed_ptt *p_ptt, u16 pf_id);
-void qed_set_rfs_mode_enable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-			     u16 pf_id, bool tcp, bool udp,
-			     bool ipv4, bool ipv6);
-
-#define	YSTORM_FLOW_CONTROL_MODE_OFFSET			(IRO[0].base)
-#define	YSTORM_FLOW_CONTROL_MODE_SIZE			(IRO[0].size)
-#define	TSTORM_PORT_STAT_OFFSET(port_id) \
+
+/**
+ * @brief qed_gft_disable - Disable GFT
+ *
+ * @param p_hwfn
+ * @param p_ptt - ptt window used for writing the registers.
+ * @param pf_id - pf on which to disable GFT.
+ */
+void qed_gft_disable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 pf_id);
+
+/**
+ * @brief qed_gft_config - Enable and configure HW for GFT
+ *
+ * @param p_hwfn
+ * @param p_ptt - ptt window used for writing the registers.
+ * @param pf_id - pf on which to enable GFT.
+ * @param tcp - set profile tcp packets.
+ * @param udp - set profile udp  packet.
+ * @param ipv4 - set profile ipv4 packet.
+ * @param ipv6 - set profile ipv6 packet.
+ * @param profile_type - define packet same fields. Use enum gft_profile_type.
+ */
+void qed_gft_config(struct qed_hwfn *p_hwfn,
+		    struct qed_ptt *p_ptt,
+		    u16 pf_id,
+		    bool tcp,
+		    bool udp,
+		    bool ipv4, bool ipv6, enum gft_profile_type profile_type);
+
+/**
+ * @brief qed_enable_context_validation - Enable and configure context
+ *	validation.
+ *
+ * @param p_hwfn
+ * @param p_ptt - ptt window used for writing the registers.
+ */
+void qed_enable_context_validation(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt);
+
+/**
+ * @brief qed_calc_session_ctx_validation - Calcualte validation byte for
+ *	session context.
+ *
+ * @param p_ctx_mem - pointer to context memory.
+ * @param ctx_size - context size.
+ * @param ctx_type - context type.
+ * @param cid - context cid.
+ */
+void qed_calc_session_ctx_validation(void *p_ctx_mem,
+				     u16 ctx_size, u8 ctx_type, u32 cid);
+
+/**
+ * @brief qed_calc_task_ctx_validation - Calcualte validation byte for task
+ *	context.
+ *
+ * @param p_ctx_mem - pointer to context memory.
+ * @param ctx_size - context size.
+ * @param ctx_type - context type.
+ * @param tid - context tid.
+ */
+void qed_calc_task_ctx_validation(void *p_ctx_mem,
+				  u16 ctx_size, u8 ctx_type, u32 tid);
+
+/**
+ * @brief qed_memset_session_ctx - Memset session context to 0 while
+ *	preserving validation bytes.
+ *
+ * @param p_hwfn -
+ * @param p_ctx_mem - pointer to context memory.
+ * @param ctx_size - size to initialzie.
+ * @param ctx_type - context type.
+ */
+void qed_memset_session_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type);
+
+/**
+ * @brief qed_memset_task_ctx - Memset task context to 0 while preserving
+ *	validation bytes.
+ *
+ * @param p_ctx_mem - pointer to context memory.
+ * @param ctx_size - size to initialzie.
+ * @param ctx_type - context type.
+ */
+void qed_memset_task_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type);
+
+/* Ystorm flow control mode. Use enum fw_flow_ctrl_mode */
+#define YSTORM_FLOW_CONTROL_MODE_OFFSET			(IRO[0].base)
+#define YSTORM_FLOW_CONTROL_MODE_SIZE			(IRO[0].size)
+
+/* Tstorm port statistics */
+#define TSTORM_PORT_STAT_OFFSET(port_id) \
 	(IRO[1].base + ((port_id) * IRO[1].m1))
-#define	TSTORM_PORT_STAT_SIZE				(IRO[1].size)
+#define TSTORM_PORT_STAT_SIZE				(IRO[1].size)
+
+/* Tstorm ll2 port statistics */
 #define TSTORM_LL2_PORT_STAT_OFFSET(port_id) \
 	(IRO[2].base + ((port_id) * IRO[2].m1))
 #define TSTORM_LL2_PORT_STAT_SIZE			(IRO[2].size)
-#define	USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) \
+
+/* Ustorm VF-PF Channel ready flag */
+#define USTORM_VF_PF_CHANNEL_READY_OFFSET(vf_id) \
 	(IRO[3].base + ((vf_id) * IRO[3].m1))
-#define	USTORM_VF_PF_CHANNEL_READY_SIZE			(IRO[3].size)
-#define	USTORM_FLR_FINAL_ACK_OFFSET(pf_id) \
-	(IRO[4].base + (pf_id) * IRO[4].m1)
-#define	USTORM_FLR_FINAL_ACK_SIZE			(IRO[4].size)
-#define	USTORM_EQE_CONS_OFFSET(pf_id) \
+#define USTORM_VF_PF_CHANNEL_READY_SIZE			(IRO[3].size)
+
+/* Ustorm Final flr cleanup ack */
+#define USTORM_FLR_FINAL_ACK_OFFSET(pf_id) \
+	(IRO[4].base + ((pf_id) * IRO[4].m1))
+#define USTORM_FLR_FINAL_ACK_SIZE			(IRO[4].size)
+
+/* Ustorm Event ring consumer */
+#define USTORM_EQE_CONS_OFFSET(pf_id) \
 	(IRO[5].base + ((pf_id) * IRO[5].m1))
-#define	USTORM_EQE_CONS_SIZE				(IRO[5].size)
-#define	USTORM_ETH_QUEUE_ZONE_OFFSET(queue_zone_id) \
+#define USTORM_EQE_CONS_SIZE				(IRO[5].size)
+
+/* Ustorm eth queue zone */
+#define USTORM_ETH_QUEUE_ZONE_OFFSET(queue_zone_id) \
 	(IRO[6].base + ((queue_zone_id) * IRO[6].m1))
-#define	USTORM_ETH_QUEUE_ZONE_SIZE			(IRO[6].size)
-#define	USTORM_COMMON_QUEUE_CONS_OFFSET(queue_zone_id) \
+#define USTORM_ETH_QUEUE_ZONE_SIZE			(IRO[6].size)
+
+/* Ustorm Common Queue ring consumer */
+#define USTORM_COMMON_QUEUE_CONS_OFFSET(queue_zone_id) \
 	(IRO[7].base + ((queue_zone_id) * IRO[7].m1))
-#define	USTORM_COMMON_QUEUE_CONS_SIZE			(IRO[7].size)
+#define USTORM_COMMON_QUEUE_CONS_SIZE			(IRO[7].size)
+
+/* Xstorm Integration Test Data */
+#define XSTORM_INTEG_TEST_DATA_OFFSET			(IRO[8].base)
+#define XSTORM_INTEG_TEST_DATA_SIZE			(IRO[8].size)
+
+/* Ystorm Integration Test Data */
+#define YSTORM_INTEG_TEST_DATA_OFFSET			(IRO[9].base)
+#define YSTORM_INTEG_TEST_DATA_SIZE			(IRO[9].size)
+
+/* Pstorm Integration Test Data */
+#define PSTORM_INTEG_TEST_DATA_OFFSET			(IRO[10].base)
+#define PSTORM_INTEG_TEST_DATA_SIZE			(IRO[10].size)
+
+/* Tstorm Integration Test Data */
+#define TSTORM_INTEG_TEST_DATA_OFFSET			(IRO[11].base)
+#define TSTORM_INTEG_TEST_DATA_SIZE			(IRO[11].size)
+
+/* Mstorm Integration Test Data */
+#define MSTORM_INTEG_TEST_DATA_OFFSET			(IRO[12].base)
+#define MSTORM_INTEG_TEST_DATA_SIZE			(IRO[12].size)
+
+/* Ustorm Integration Test Data */
+#define USTORM_INTEG_TEST_DATA_OFFSET			(IRO[13].base)
+#define USTORM_INTEG_TEST_DATA_SIZE			(IRO[13].size)
+
+/* Tstorm producers */
 #define TSTORM_LL2_RX_PRODS_OFFSET(core_rx_queue_id) \
-	(IRO[14].base +	((core_rx_queue_id) * IRO[14].m1))
+	(IRO[14].base + ((core_rx_queue_id) * IRO[14].m1))
 #define TSTORM_LL2_RX_PRODS_SIZE			(IRO[14].size)
+
+/* Tstorm LightL2 queue statistics */
 #define CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \
 	(IRO[15].base + ((core_rx_queue_id) * IRO[15].m1))
 #define CORE_LL2_TSTORM_PER_QUEUE_STAT_SIZE		(IRO[15].size)
+
+/* Ustorm LiteL2 queue statistics */
 #define CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(core_rx_queue_id) \
-	(IRO[16].base +	((core_rx_queue_id) * IRO[16].m1))
+	(IRO[16].base + ((core_rx_queue_id) * IRO[16].m1))
 #define CORE_LL2_USTORM_PER_QUEUE_STAT_SIZE		(IRO[16].size)
+
+/* Pstorm LiteL2 queue statistics */
 #define CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(core_tx_stats_id) \
-	(IRO[17].base +	((core_tx_stats_id) * IRO[17].m1))
-#define CORE_LL2_PSTORM_PER_QUEUE_STAT_SIZE	(IRO[17].	size)
-#define	MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
+	(IRO[17].base + ((core_tx_stats_id) * IRO[17].m1))
+#define CORE_LL2_PSTORM_PER_QUEUE_STAT_SIZE		(IRO[17].size)
+
+/* Mstorm queue statistics */
+#define MSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
 	(IRO[18].base + ((stat_counter_id) * IRO[18].m1))
-#define	MSTORM_QUEUE_STAT_SIZE				(IRO[18].size)
-#define	MSTORM_ETH_PF_PRODS_OFFSET(queue_id) \
+#define MSTORM_QUEUE_STAT_SIZE				(IRO[18].size)
+
+/* Mstorm ETH PF queues producers */
+#define MSTORM_ETH_PF_PRODS_OFFSET(queue_id) \
 	(IRO[19].base + ((queue_id) * IRO[19].m1))
-#define	MSTORM_ETH_PF_PRODS_SIZE			(IRO[19].size)
+#define MSTORM_ETH_PF_PRODS_SIZE			(IRO[19].size)
+
+/* Mstorm ETH VF queues producers offset in RAM. Used in default VF zone size
+ * mode.
+ */
 #define MSTORM_ETH_VF_PRODS_OFFSET(vf_id, vf_queue_id) \
-	(IRO[20].base +	((vf_id) * IRO[20].m1) + ((vf_queue_id) * IRO[20].m2))
+	(IRO[20].base + ((vf_id) * IRO[20].m1) + ((vf_queue_id) * IRO[20].m2))
 #define MSTORM_ETH_VF_PRODS_SIZE			(IRO[20].size)
-#define	MSTORM_TPA_TIMEOUT_US_OFFSET			(IRO[21].base)
-#define	MSTORM_TPA_TIMEOUT_US_SIZE			(IRO[21].size)
-#define	MSTORM_ETH_PF_STAT_OFFSET(pf_id) \
+
+/* TPA agregation timeout in us resolution (on ASIC) */
+#define MSTORM_TPA_TIMEOUT_US_OFFSET			(IRO[21].base)
+#define MSTORM_TPA_TIMEOUT_US_SIZE			(IRO[21].size)
+
+/* Mstorm pf statistics */
+#define MSTORM_ETH_PF_STAT_OFFSET(pf_id) \
 	(IRO[22].base + ((pf_id) * IRO[22].m1))
-#define	MSTORM_ETH_PF_STAT_SIZE				(IRO[22].size)
-#define	USTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
+#define MSTORM_ETH_PF_STAT_SIZE				(IRO[22].size)
+
+/* Ustorm queue statistics */
+#define USTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
 	(IRO[23].base + ((stat_counter_id) * IRO[23].m1))
-#define	USTORM_QUEUE_STAT_SIZE				(IRO[23].size)
-#define	USTORM_ETH_PF_STAT_OFFSET(pf_id) \
+#define USTORM_QUEUE_STAT_SIZE				(IRO[23].size)
+
+/* Ustorm pf statistics */
+#define USTORM_ETH_PF_STAT_OFFSET(pf_id)\
 	(IRO[24].base + ((pf_id) * IRO[24].m1))
-#define	USTORM_ETH_PF_STAT_SIZE				(IRO[24].size)
-#define	PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
+#define USTORM_ETH_PF_STAT_SIZE				(IRO[24].size)
+
+/* Pstorm queue statistics */
+#define PSTORM_QUEUE_STAT_OFFSET(stat_counter_id) \
 	(IRO[25].base + ((stat_counter_id) * IRO[25].m1))
-#define	PSTORM_QUEUE_STAT_SIZE				(IRO[25].size)
-#define	PSTORM_ETH_PF_STAT_OFFSET(pf_id) \
+#define PSTORM_QUEUE_STAT_SIZE				(IRO[25].size)
+
+/* Pstorm pf statistics */
+#define PSTORM_ETH_PF_STAT_OFFSET(pf_id) \
 	(IRO[26].base + ((pf_id) * IRO[26].m1))
-#define	PSTORM_ETH_PF_STAT_SIZE				(IRO[26].size)
-#define	PSTORM_CTL_FRAME_ETHTYPE_OFFSET(ethtype) \
-	(IRO[27].base + ((ethtype) * IRO[27].m1))
-#define	PSTORM_CTL_FRAME_ETHTYPE_SIZE			(IRO[27].size)
-#define	TSTORM_ETH_PRS_INPUT_OFFSET			(IRO[28].base)
-#define	TSTORM_ETH_PRS_INPUT_SIZE			(IRO[28].size)
-#define	ETH_RX_RATE_LIMIT_OFFSET(pf_id) \
+#define PSTORM_ETH_PF_STAT_SIZE				(IRO[26].size)
+
+/* Control frame's EthType configuration for TX control frame security */
+#define PSTORM_CTL_FRAME_ETHTYPE_OFFSET(eth_type_id) \
+	(IRO[27].base + ((eth_type_id) * IRO[27].m1))
+#define PSTORM_CTL_FRAME_ETHTYPE_SIZE			(IRO[27].size)
+
+/* Tstorm last parser message */
+#define TSTORM_ETH_PRS_INPUT_OFFSET			(IRO[28].base)
+#define TSTORM_ETH_PRS_INPUT_SIZE			(IRO[28].size)
+
+/* Tstorm Eth limit Rx rate */
+#define ETH_RX_RATE_LIMIT_OFFSET(pf_id) \
 	(IRO[29].base + ((pf_id) * IRO[29].m1))
-#define	ETH_RX_RATE_LIMIT_SIZE				(IRO[29].size)
-#define	XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \
+#define ETH_RX_RATE_LIMIT_SIZE				(IRO[29].size)
+
+/* Xstorm queue zone */
+#define XSTORM_ETH_QUEUE_ZONE_OFFSET(queue_id) \
 	(IRO[30].base + ((queue_id) * IRO[30].m1))
-#define	XSTORM_ETH_QUEUE_ZONE_SIZE			(IRO[30].size)
+#define XSTORM_ETH_QUEUE_ZONE_SIZE			(IRO[30].size)
+
+/* Ystorm cqe producer */
+#define YSTORM_TOE_CQ_PROD_OFFSET(rss_id) \
+	(IRO[31].base + ((rss_id) * IRO[31].m1))
+#define YSTORM_TOE_CQ_PROD_SIZE				(IRO[31].size)
+
+/* Ustorm cqe producer */
+#define USTORM_TOE_CQ_PROD_OFFSET(rss_id) \
+	(IRO[32].base + ((rss_id) * IRO[32].m1))
+#define USTORM_TOE_CQ_PROD_SIZE				(IRO[32].size)
+
+/* Ustorm grq producer */
+#define USTORM_TOE_GRQ_PROD_OFFSET(pf_id) \
+	(IRO[33].base + ((pf_id) * IRO[33].m1))
+#define USTORM_TOE_GRQ_PROD_SIZE			(IRO[33].size)
+
+/* Tstorm cmdq-cons of given command queue-id */
 #define TSTORM_SCSI_CMDQ_CONS_OFFSET(cmdq_queue_id) \
-	(IRO[34].base +	((cmdq_queue_id) * IRO[34].m1))
-#define TSTORM_SCSI_CMDQ_CONS_SIZE				(IRO[34].size)
+	(IRO[34].base + ((cmdq_queue_id) * IRO[34].m1))
+#define TSTORM_SCSI_CMDQ_CONS_SIZE			(IRO[34].size)
+
+/* Tstorm (reflects M-Storm) bdq-external-producer of given function ID,
+ * BDqueue-id.
+ */
 #define TSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \
-	(IRO[35].base +	((func_id) * IRO[35].m1) + ((bdq_id) * IRO[35].m2))
-#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE				(IRO[35].size)
+	(IRO[35].base + ((func_id) * IRO[35].m1) + ((bdq_id) * IRO[35].m2))
+#define TSTORM_SCSI_BDQ_EXT_PROD_SIZE			(IRO[35].size)
+
+/* Mstorm bdq-external-producer of given BDQ resource ID, BDqueue-id */
 #define MSTORM_SCSI_BDQ_EXT_PROD_OFFSET(func_id, bdq_id) \
-	(IRO[36].base +	((func_id) * IRO[36].m1) + ((bdq_id) * IRO[36].m2))
-#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE				(IRO[36].size)
+	(IRO[36].base + ((func_id) * IRO[36].m1) + ((bdq_id) * IRO[36].m2))
+#define MSTORM_SCSI_BDQ_EXT_PROD_SIZE			(IRO[36].size)
+
+/* Tstorm iSCSI RX stats */
 #define TSTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
-	(IRO[37].base +	((pf_id) * IRO[37].m1))
-#define TSTORM_ISCSI_RX_STATS_SIZE				(IRO[37].size)
+	(IRO[37].base + ((pf_id) * IRO[37].m1))
+#define TSTORM_ISCSI_RX_STATS_SIZE			(IRO[37].size)
+
+/* Mstorm iSCSI RX stats */
 #define MSTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
-	(IRO[38].base +	((pf_id) * IRO[38].m1))
-#define MSTORM_ISCSI_RX_STATS_SIZE				(IRO[38].size)
+	(IRO[38].base + ((pf_id) * IRO[38].m1))
+#define MSTORM_ISCSI_RX_STATS_SIZE			(IRO[38].size)
+
+/* Ustorm iSCSI RX stats */
 #define USTORM_ISCSI_RX_STATS_OFFSET(pf_id) \
-	(IRO[39].base +	((pf_id) * IRO[39].m1))
-#define USTORM_ISCSI_RX_STATS_SIZE				(IRO[39].size)
+	(IRO[39].base + ((pf_id) * IRO[39].m1))
+#define USTORM_ISCSI_RX_STATS_SIZE			(IRO[39].size)
+
+/* Xstorm iSCSI TX stats */
 #define XSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
-	(IRO[40].base +	((pf_id) * IRO[40].m1))
-#define XSTORM_ISCSI_TX_STATS_SIZE				(IRO[40].size)
+	(IRO[40].base + ((pf_id) * IRO[40].m1))
+#define XSTORM_ISCSI_TX_STATS_SIZE			(IRO[40].size)
+
+/* Ystorm iSCSI TX stats */
 #define YSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
-	(IRO[41].base +	((pf_id) * IRO[41].m1))
-#define YSTORM_ISCSI_TX_STATS_SIZE				(IRO[41].size)
+	(IRO[41].base + ((pf_id) * IRO[41].m1))
+#define YSTORM_ISCSI_TX_STATS_SIZE			(IRO[41].size)
+
+/* Pstorm iSCSI TX stats */
 #define PSTORM_ISCSI_TX_STATS_OFFSET(pf_id) \
-	(IRO[42].base +	((pf_id) * IRO[42].m1))
-#define PSTORM_ISCSI_TX_STATS_SIZE				(IRO[42].size)
-#define PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \
-	(IRO[45].base +	((rdma_stat_counter_id) * IRO[45].m1))
-#define PSTORM_RDMA_QUEUE_STAT_SIZE				(IRO[45].size)
-#define TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \
-	(IRO[46].base +	((rdma_stat_counter_id) * IRO[46].m1))
-#define TSTORM_RDMA_QUEUE_STAT_SIZE				(IRO[46].size)
+	(IRO[42].base + ((pf_id) * IRO[42].m1))
+#define PSTORM_ISCSI_TX_STATS_SIZE			(IRO[42].size)
+
+/* Tstorm FCoE RX stats */
 #define TSTORM_FCOE_RX_STATS_OFFSET(pf_id) \
-	(IRO[43].base +	((pf_id) * IRO[43].m1))
+	(IRO[43].base + ((pf_id) * IRO[43].m1))
+#define TSTORM_FCOE_RX_STATS_SIZE			(IRO[43].size)
+
+/* Pstorm FCoE TX stats */
 #define PSTORM_FCOE_TX_STATS_OFFSET(pf_id) \
 	(IRO[44].base + ((pf_id) * IRO[44].m1))
+#define PSTORM_FCOE_TX_STATS_SIZE			(IRO[44].size)
+
+/* Pstorm RDMA queue statistics */
+#define PSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \
+	(IRO[45].base + ((rdma_stat_counter_id) * IRO[45].m1))
+#define PSTORM_RDMA_QUEUE_STAT_SIZE			(IRO[45].size)
 
-static const struct iro iro_arr[49] = {
+/* Tstorm RDMA queue statistics */
+#define TSTORM_RDMA_QUEUE_STAT_OFFSET(rdma_stat_counter_id) \
+	(IRO[46].base + ((rdma_stat_counter_id) * IRO[46].m1))
+#define TSTORM_RDMA_QUEUE_STAT_SIZE			(IRO[46].size)
+
+/* Xstorm iWARP rxmit stats */
+#define XSTORM_IWARP_RXMIT_STATS_OFFSET(pf_id) \
+	(IRO[47].base + ((pf_id) * IRO[47].m1))
+#define XSTORM_IWARP_RXMIT_STATS_SIZE			(IRO[47].size)
+
+/* Tstorm RoCE Event Statistics */
+#define TSTORM_ROCE_EVENTS_STAT_OFFSET(roce_pf_id) \
+	(IRO[48].base + ((roce_pf_id) * IRO[48].m1))
+#define TSTORM_ROCE_EVENTS_STAT_SIZE			(IRO[48].size)
+
+/* DCQCN Received Statistics */
+#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(roce_pf_id) \
+	(IRO[49].base + ((roce_pf_id) * IRO[49].m1))
+#define YSTORM_ROCE_DCQCN_RECEIVED_STATS_SIZE		(IRO[49].size)
+
+/* DCQCN Sent Statistics */
+#define PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(roce_pf_id) \
+	(IRO[50].base + ((roce_pf_id) * IRO[50].m1))
+#define PSTORM_ROCE_DCQCN_SENT_STATS_SIZE		(IRO[50].size)
+
+static const struct iro iro_arr[51] = {
 	{0x0, 0x0, 0x0, 0x0, 0x8},
-	{0x4cb0, 0x80, 0x0, 0x0, 0x80},
-	{0x6518, 0x20, 0x0, 0x0, 0x20},
+	{0x4cb8, 0x88, 0x0, 0x0, 0x88},
+	{0x6530, 0x20, 0x0, 0x0, 0x20},
 	{0xb00, 0x8, 0x0, 0x0, 0x4},
 	{0xa80, 0x8, 0x0, 0x0, 0x4},
 	{0x0, 0x8, 0x0, 0x0, 0x2},
 	{0x80, 0x8, 0x0, 0x0, 0x4},
 	{0x84, 0x8, 0x0, 0x0, 0x2},
+	{0x4c48, 0x0, 0x0, 0x0, 0x78},
+	{0x3e18, 0x0, 0x0, 0x0, 0x78},
+	{0x2b58, 0x0, 0x0, 0x0, 0x78},
 	{0x4c40, 0x0, 0x0, 0x0, 0x78},
-	{0x3df0, 0x0, 0x0, 0x0, 0x78},
-	{0x29b0, 0x0, 0x0, 0x0, 0x78},
-	{0x4c38, 0x0, 0x0, 0x0, 0x78},
-	{0x4990, 0x0, 0x0, 0x0, 0x78},
-	{0x7f48, 0x0, 0x0, 0x0, 0x78},
+	{0x4998, 0x0, 0x0, 0x0, 0x78},
+	{0x7f50, 0x0, 0x0, 0x0, 0x78},
 	{0xa28, 0x8, 0x0, 0x0, 0x8},
-	{0x61f8, 0x10, 0x0, 0x0, 0x10},
-	{0xbd20, 0x30, 0x0, 0x0, 0x30},
-	{0x95b8, 0x30, 0x0, 0x0, 0x30},
-	{0x4b60, 0x80, 0x0, 0x0, 0x40},
+	{0x6210, 0x10, 0x0, 0x0, 0x10},
+	{0xb820, 0x30, 0x0, 0x0, 0x30},
+	{0x96c0, 0x30, 0x0, 0x0, 0x30},
+	{0x4b68, 0x80, 0x0, 0x0, 0x40},
 	{0x1f8, 0x4, 0x0, 0x0, 0x4},
-	{0x53a0, 0x80, 0x4, 0x0, 0x4},
-	{0xc7c8, 0x0, 0x0, 0x0, 0x4},
-	{0x4ba0, 0x80, 0x0, 0x0, 0x20},
-	{0x8150, 0x40, 0x0, 0x0, 0x30},
-	{0xec70, 0x60, 0x0, 0x0, 0x60},
-	{0x2b48, 0x80, 0x0, 0x0, 0x38},
-	{0xf1b0, 0x78, 0x0, 0x0, 0x78},
+	{0x53a8, 0x80, 0x4, 0x0, 0x4},
+	{0xc7d0, 0x0, 0x0, 0x0, 0x4},
+	{0x4ba8, 0x80, 0x0, 0x0, 0x20},
+	{0x8158, 0x40, 0x0, 0x0, 0x30},
+	{0xe770, 0x60, 0x0, 0x0, 0x60},
+	{0x2cf0, 0x80, 0x0, 0x0, 0x38},
+	{0xf2b8, 0x78, 0x0, 0x0, 0x78},
 	{0x1f8, 0x4, 0x0, 0x0, 0x4},
-	{0xaef8, 0x0, 0x0, 0x0, 0xf0},
-	{0xafe8, 0x8, 0x0, 0x0, 0x8},
+	{0xaf20, 0x0, 0x0, 0x0, 0xf0},
+	{0xb010, 0x8, 0x0, 0x0, 0x8},
 	{0x1f8, 0x8, 0x0, 0x0, 0x8},
 	{0xac0, 0x8, 0x0, 0x0, 0x8},
 	{0x2578, 0x8, 0x0, 0x0, 0x8},
 	{0x24f8, 0x8, 0x0, 0x0, 0x8},
 	{0x0, 0x8, 0x0, 0x0, 0x8},
-	{0x200, 0x10, 0x8, 0x0, 0x8},
-	{0xb78, 0x10, 0x8, 0x0, 0x2},
-	{0xd9a8, 0x38, 0x0, 0x0, 0x24},
-	{0x12988, 0x10, 0x0, 0x0, 0x8},
-	{0x11fa0, 0x38, 0x0, 0x0, 0x18},
-	{0xa580, 0x38, 0x0, 0x0, 0x10},
-	{0x86f8, 0x30, 0x0, 0x0, 0x18},
-	{0x101f8, 0x10, 0x0, 0x0, 0x10},
-	{0xde28, 0x48, 0x0, 0x0, 0x38},
-	{0x10660, 0x20, 0x0, 0x0, 0x20},
-	{0x2b80, 0x80, 0x0, 0x0, 0x10},
-	{0x5020, 0x10, 0x0, 0x0, 0x10},
-	{0xc9b0, 0x30, 0x0, 0x0, 0x10},
-	{0xeec0, 0x10, 0x0, 0x0, 0x10},
+	{0x400, 0x18, 0x8, 0x0, 0x8},
+	{0xb78, 0x18, 0x8, 0x0, 0x2},
+	{0xd898, 0x50, 0x0, 0x0, 0x3c},
+	{0x12908, 0x18, 0x0, 0x0, 0x10},
+	{0x11aa8, 0x40, 0x0, 0x0, 0x18},
+	{0xa588, 0x50, 0x0, 0x0, 0x20},
+	{0x8700, 0x40, 0x0, 0x0, 0x28},
+	{0x10300, 0x18, 0x0, 0x0, 0x10},
+	{0xde48, 0x48, 0x0, 0x0, 0x38},
+	{0x10768, 0x20, 0x0, 0x0, 0x20},
+	{0x2d28, 0x80, 0x0, 0x0, 0x10},
+	{0x5048, 0x10, 0x0, 0x0, 0x10},
+	{0xc9b8, 0x30, 0x0, 0x0, 0x10},
+	{0xeee0, 0x10, 0x0, 0x0, 0x10},
+	{0xa3a0, 0x10, 0x0, 0x0, 0x10},
+	{0x13108, 0x8, 0x0, 0x0, 0x8},
 };
 
 /* Runtime array offsets */
-#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET	0
-#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET	1
-#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET	2
-#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET	3
-#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET	4
-#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET	5
-#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET	6
-#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET	7
-#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET	8
-#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET	9
-#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET	10
-#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET	11
-#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET	12
-#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET	13
-#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET	14
-#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET	15
-#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET	16
-#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET	17
-#define IGU_REG_PF_CONFIGURATION_RT_OFFSET	18
-#define IGU_REG_VF_CONFIGURATION_RT_OFFSET	19
-#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET	20
-#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET	21
-#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET	22
-#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET	23
-#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET	24
-#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET	761
-#define CAU_REG_SB_VAR_MEMORY_RT_SIZE	736
-#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET	761
-#define CAU_REG_SB_VAR_MEMORY_RT_SIZE	736
-#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET	1497
-#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE	736
-#define CAU_REG_PI_MEMORY_RT_OFFSET	2233
-#define CAU_REG_PI_MEMORY_RT_SIZE	4416
-#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET	6649
-#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET	6650
-#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET	6651
-#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET	6652
-#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET	6653
-#define PRS_REG_SEARCH_TCP_RT_OFFSET	6654
-#define PRS_REG_SEARCH_FCOE_RT_OFFSET	6655
-#define PRS_REG_SEARCH_ROCE_RT_OFFSET	6656
-#define PRS_REG_ROCE_DEST_QP_MAX_VF_RT_OFFSET	6657
-#define PRS_REG_ROCE_DEST_QP_MAX_PF_RT_OFFSET	6658
-#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET	6659
-#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET	6660
-#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET	6661
-#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET	6662
-#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET	6663
-#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET	6664
-#define SRC_REG_FIRSTFREE_RT_OFFSET	6665
-#define SRC_REG_FIRSTFREE_RT_SIZE	2
-#define SRC_REG_LASTFREE_RT_OFFSET	6667
-#define SRC_REG_LASTFREE_RT_SIZE	2
-#define SRC_REG_COUNTFREE_RT_OFFSET	6669
-#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET	6670
-#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET	6671
-#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET	6672
-#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET	6673
-#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET	6674
-#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET	6675
-#define PSWRQ2_REG_TSDM_P_SIZE_RT_OFFSET	6676
-#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET	6677
-#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET	6678
-#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET	6679
-#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET	6680
-#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET	6681
-#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET	6682
-#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET	6683
-#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET	6684
-#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET	6685
-#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET	6686
-#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET	6687
-#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET	6688
-#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET	6689
-#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET	6690
-#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET	6691
-#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET	6692
-#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET	6693
-#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET	6694
-#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET	6695
-#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET	6696
-#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET	6697
-#define PSWRQ2_REG_VF_BASE_RT_OFFSET	6698
-#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET	6699
-#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET	6700
-#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET	6701
-#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET	6702
-#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE	22000
-#define PGLUE_REG_B_VF_BASE_RT_OFFSET	28702
-#define PGLUE_REG_B_MSDM_OFFSET_MASK_B_RT_OFFSET	28703
-#define PGLUE_REG_B_MSDM_VF_SHIFT_B_RT_OFFSET	28704
-#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET	28705
-#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET	28706
-#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET	28707
-#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET	28708
-#define TM_REG_VF_ENABLE_CONN_RT_OFFSET	28709
-#define TM_REG_PF_ENABLE_CONN_RT_OFFSET	28710
-#define TM_REG_PF_ENABLE_TASK_RT_OFFSET	28711
-#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET	28712
-#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET	28713
-#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET	28714
-#define TM_REG_CONFIG_CONN_MEM_RT_SIZE	416
-#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET	29130
-#define TM_REG_CONFIG_TASK_MEM_RT_SIZE	608
-#define QM_REG_MAXPQSIZE_0_RT_OFFSET	29738
-#define QM_REG_MAXPQSIZE_1_RT_OFFSET	29739
-#define QM_REG_MAXPQSIZE_2_RT_OFFSET	29740
-#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET	29741
-#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET	29742
-#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET	29743
-#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET	29744
-#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET	29745
-#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET	29746
-#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET	29747
-#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET	29748
-#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET	29749
-#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET	29750
-#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET	29751
-#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET	29752
-#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET	29753
-#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET	29754
-#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET	29755
-#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET	29756
-#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET	29757
-#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET	29758
-#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET	29759
-#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET	29760
-#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET	29761
-#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET	29762
-#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET	29763
-#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET	29764
-#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET	29765
-#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET	29766
-#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET	29767
-#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET	29768
-#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET	29769
-#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET	29770
-#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET	29771
-#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET	29772
-#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET	29773
-#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET	29774
-#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET	29775
-#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET	29776
-#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET	29777
-#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET	29778
-#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET	29779
-#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET	29780
-#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET	29781
-#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET	29782
-#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET	29783
-#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET	29784
-#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET	29785
-#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET	29786
-#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET	29787
-#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET	29788
-#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET	29789
-#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET	29790
-#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET	29791
-#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET	29792
-#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET	29793
-#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET	29794
-#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET	29795
-#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET	29796
-#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET	29797
-#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET	29798
-#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET	29799
-#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET	29800
-#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET	29801
-#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET	29802
-#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET	29803
-#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET	29804
-#define QM_REG_BASEADDROTHERPQ_RT_OFFSET	29805
-#define QM_REG_BASEADDROTHERPQ_RT_SIZE	128
-#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET	29933
-#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET	29934
-#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET	29935
-#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET	29936
-#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET	29937
-#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET	29938
-#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET	29939
-#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET	29940
-#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET	29941
-#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET	29942
-#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET	29943
-#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET	29944
-#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET	29945
-#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET	29946
-#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET	29947
-#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET	29948
-#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET	29949
-#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET	29950
-#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET	29951
-#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET	29952
-#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET	29953
-#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET	29954
-#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET	29955
-#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET	29956
-#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET	29957
-#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET	29958
-#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET	29959
-#define QM_REG_PQTX2PF_0_RT_OFFSET	29960
-#define QM_REG_PQTX2PF_1_RT_OFFSET	29961
-#define QM_REG_PQTX2PF_2_RT_OFFSET	29962
-#define QM_REG_PQTX2PF_3_RT_OFFSET	29963
-#define QM_REG_PQTX2PF_4_RT_OFFSET	29964
-#define QM_REG_PQTX2PF_5_RT_OFFSET	29965
-#define QM_REG_PQTX2PF_6_RT_OFFSET	29966
-#define QM_REG_PQTX2PF_7_RT_OFFSET	29967
-#define QM_REG_PQTX2PF_8_RT_OFFSET	29968
-#define QM_REG_PQTX2PF_9_RT_OFFSET	29969
-#define QM_REG_PQTX2PF_10_RT_OFFSET	29970
-#define QM_REG_PQTX2PF_11_RT_OFFSET	29971
-#define QM_REG_PQTX2PF_12_RT_OFFSET	29972
-#define QM_REG_PQTX2PF_13_RT_OFFSET	29973
-#define QM_REG_PQTX2PF_14_RT_OFFSET	29974
-#define QM_REG_PQTX2PF_15_RT_OFFSET	29975
-#define QM_REG_PQTX2PF_16_RT_OFFSET	29976
-#define QM_REG_PQTX2PF_17_RT_OFFSET	29977
-#define QM_REG_PQTX2PF_18_RT_OFFSET	29978
-#define QM_REG_PQTX2PF_19_RT_OFFSET	29979
-#define QM_REG_PQTX2PF_20_RT_OFFSET	29980
-#define QM_REG_PQTX2PF_21_RT_OFFSET	29981
-#define QM_REG_PQTX2PF_22_RT_OFFSET	29982
-#define QM_REG_PQTX2PF_23_RT_OFFSET	29983
-#define QM_REG_PQTX2PF_24_RT_OFFSET	29984
-#define QM_REG_PQTX2PF_25_RT_OFFSET	29985
-#define QM_REG_PQTX2PF_26_RT_OFFSET	29986
-#define QM_REG_PQTX2PF_27_RT_OFFSET	29987
-#define QM_REG_PQTX2PF_28_RT_OFFSET	29988
-#define QM_REG_PQTX2PF_29_RT_OFFSET	29989
-#define QM_REG_PQTX2PF_30_RT_OFFSET	29990
-#define QM_REG_PQTX2PF_31_RT_OFFSET	29991
-#define QM_REG_PQTX2PF_32_RT_OFFSET	29992
-#define QM_REG_PQTX2PF_33_RT_OFFSET	29993
-#define QM_REG_PQTX2PF_34_RT_OFFSET	29994
-#define QM_REG_PQTX2PF_35_RT_OFFSET	29995
-#define QM_REG_PQTX2PF_36_RT_OFFSET	29996
-#define QM_REG_PQTX2PF_37_RT_OFFSET	29997
-#define QM_REG_PQTX2PF_38_RT_OFFSET	29998
-#define QM_REG_PQTX2PF_39_RT_OFFSET	29999
-#define QM_REG_PQTX2PF_40_RT_OFFSET	30000
-#define QM_REG_PQTX2PF_41_RT_OFFSET	30001
-#define QM_REG_PQTX2PF_42_RT_OFFSET	30002
-#define QM_REG_PQTX2PF_43_RT_OFFSET	30003
-#define QM_REG_PQTX2PF_44_RT_OFFSET	30004
-#define QM_REG_PQTX2PF_45_RT_OFFSET	30005
-#define QM_REG_PQTX2PF_46_RT_OFFSET	30006
-#define QM_REG_PQTX2PF_47_RT_OFFSET	30007
-#define QM_REG_PQTX2PF_48_RT_OFFSET	30008
-#define QM_REG_PQTX2PF_49_RT_OFFSET	30009
-#define QM_REG_PQTX2PF_50_RT_OFFSET	30010
-#define QM_REG_PQTX2PF_51_RT_OFFSET	30011
-#define QM_REG_PQTX2PF_52_RT_OFFSET	30012
-#define QM_REG_PQTX2PF_53_RT_OFFSET	30013
-#define QM_REG_PQTX2PF_54_RT_OFFSET	30014
-#define QM_REG_PQTX2PF_55_RT_OFFSET	30015
-#define QM_REG_PQTX2PF_56_RT_OFFSET	30016
-#define QM_REG_PQTX2PF_57_RT_OFFSET	30017
-#define QM_REG_PQTX2PF_58_RT_OFFSET	30018
-#define QM_REG_PQTX2PF_59_RT_OFFSET	30019
-#define QM_REG_PQTX2PF_60_RT_OFFSET	30020
-#define QM_REG_PQTX2PF_61_RT_OFFSET	30021
-#define QM_REG_PQTX2PF_62_RT_OFFSET	30022
-#define QM_REG_PQTX2PF_63_RT_OFFSET	30023
-#define QM_REG_PQOTHER2PF_0_RT_OFFSET	30024
-#define QM_REG_PQOTHER2PF_1_RT_OFFSET	30025
-#define QM_REG_PQOTHER2PF_2_RT_OFFSET	30026
-#define QM_REG_PQOTHER2PF_3_RT_OFFSET	30027
-#define QM_REG_PQOTHER2PF_4_RT_OFFSET	30028
-#define QM_REG_PQOTHER2PF_5_RT_OFFSET	30029
-#define QM_REG_PQOTHER2PF_6_RT_OFFSET	30030
-#define QM_REG_PQOTHER2PF_7_RT_OFFSET	30031
-#define QM_REG_PQOTHER2PF_8_RT_OFFSET	30032
-#define QM_REG_PQOTHER2PF_9_RT_OFFSET	30033
-#define QM_REG_PQOTHER2PF_10_RT_OFFSET	30034
-#define QM_REG_PQOTHER2PF_11_RT_OFFSET	30035
-#define QM_REG_PQOTHER2PF_12_RT_OFFSET	30036
-#define QM_REG_PQOTHER2PF_13_RT_OFFSET	30037
-#define QM_REG_PQOTHER2PF_14_RT_OFFSET	30038
-#define QM_REG_PQOTHER2PF_15_RT_OFFSET	30039
-#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET	30040
-#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET	30041
-#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET	30042
-#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET	30043
-#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET	30044
-#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET	30045
-#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET	30046
-#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET	30047
-#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET	30048
-#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET	30049
-#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET	30050
-#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET	30051
-#define QM_REG_RLGLBLINCVAL_RT_OFFSET	30052
-#define QM_REG_RLGLBLINCVAL_RT_SIZE	256
-#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET	30308
-#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE	256
-#define QM_REG_RLGLBLCRD_RT_OFFSET	30564
-#define QM_REG_RLGLBLCRD_RT_SIZE	256
-#define QM_REG_RLGLBLENABLE_RT_OFFSET	30820
-#define QM_REG_RLPFPERIOD_RT_OFFSET	30821
-#define QM_REG_RLPFPERIODTIMER_RT_OFFSET	30822
-#define QM_REG_RLPFINCVAL_RT_OFFSET	30823
-#define QM_REG_RLPFINCVAL_RT_SIZE	16
-#define QM_REG_RLPFUPPERBOUND_RT_OFFSET	30839
-#define QM_REG_RLPFUPPERBOUND_RT_SIZE	16
-#define QM_REG_RLPFCRD_RT_OFFSET	30855
-#define QM_REG_RLPFCRD_RT_SIZE	16
-#define QM_REG_RLPFENABLE_RT_OFFSET	30871
-#define QM_REG_RLPFVOQENABLE_RT_OFFSET	30872
-#define QM_REG_WFQPFWEIGHT_RT_OFFSET	30873
-#define QM_REG_WFQPFWEIGHT_RT_SIZE	16
-#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET	30889
-#define QM_REG_WFQPFUPPERBOUND_RT_SIZE	16
-#define QM_REG_WFQPFCRD_RT_OFFSET	30905
-#define QM_REG_WFQPFCRD_RT_SIZE	256
-#define QM_REG_WFQPFENABLE_RT_OFFSET	31161
-#define QM_REG_WFQVPENABLE_RT_OFFSET	31162
-#define QM_REG_BASEADDRTXPQ_RT_OFFSET	31163
-#define QM_REG_BASEADDRTXPQ_RT_SIZE	512
-#define QM_REG_TXPQMAP_RT_OFFSET	31675
-#define QM_REG_TXPQMAP_RT_SIZE	512
-#define QM_REG_WFQVPWEIGHT_RT_OFFSET	32187
-#define QM_REG_WFQVPWEIGHT_RT_SIZE	512
-#define QM_REG_WFQVPCRD_RT_OFFSET	32699
-#define QM_REG_WFQVPCRD_RT_SIZE	512
-#define QM_REG_WFQVPMAP_RT_OFFSET	33211
-#define QM_REG_WFQVPMAP_RT_SIZE	512
-#define QM_REG_WFQPFCRD_MSB_RT_OFFSET	33723
-#define QM_REG_WFQPFCRD_MSB_RT_SIZE	320
-#define QM_REG_VOQCRDLINE_RT_OFFSET	34043
-#define QM_REG_VOQCRDLINE_RT_SIZE	36
-#define QM_REG_VOQINITCRDLINE_RT_OFFSET	34079
-#define QM_REG_VOQINITCRDLINE_RT_SIZE	36
-#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET	34115
-#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET	34116
-#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET	34117
-#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET	34118
-#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET	34119
-#define NIG_REG_OUTER_TAG_VALUE_MASK_RT_OFFSET	34120
-#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET	34121
-#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET	34122
-#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE	4
-#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_OFFSET	34126
-#define NIG_REG_LLH_FUNC_TAG_HDR_SEL_RT_SIZE	4
-#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET	34130
-#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE	4
-#define NIG_REG_LLH_FUNC_NO_TAG_RT_OFFSET	34134
-#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET	34135
-#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE	32
-#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET	34167
-#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE	16
-#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET	34183
-#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE	16
-#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET	34199
-#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE	16
-#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET	34215
-#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE	16
-#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET	34231
-#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET	34232
-#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET	34233
-#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET	34234
-#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET	34235
-#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET	34236
-#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET	34237
-#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET	34238
-#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET	34239
-#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET	34240
-#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET	34241
-#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET	34242
-#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET	34243
-#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET	34244
-#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET	34245
-#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET	34246
-#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET	34247
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET	34248
-#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET	34249
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET	34250
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET	34251
-#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET	34252
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET	34253
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET	34254
-#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET	34255
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET	34256
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET	34257
-#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET	34258
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET	34259
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET	34260
-#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET	34261
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET	34262
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET	34263
-#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET	34264
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET	34265
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET	34266
-#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET	34267
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET	34268
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET	34269
-#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET	34270
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET	34271
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET	34272
-#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET	34273
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET	34274
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET	34275
-#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET	34276
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET	34277
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET	34278
-#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET	34279
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET	34280
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET	34281
-#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET	34282
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET	34283
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET	34284
-#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET	34285
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET	34286
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET	34287
-#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET	34288
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET	34289
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET	34290
-#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET	34291
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET	34292
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET	34293
-#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET	34294
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET	34295
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET	34296
-#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET	34297
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET	34298
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET	34299
-#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET	34300
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET	34301
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET	34302
-#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET	34303
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET	34304
-#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET	34305
-#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET	34306
-#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET	34307
-#define XCM_REG_CON_PHY_Q3_RT_OFFSET	34308
-
-#define RUNTIME_ARRAY_SIZE 34309
+#define DORQ_REG_PF_MAX_ICID_0_RT_OFFSET			0
+#define DORQ_REG_PF_MAX_ICID_1_RT_OFFSET			1
+#define DORQ_REG_PF_MAX_ICID_2_RT_OFFSET			2
+#define DORQ_REG_PF_MAX_ICID_3_RT_OFFSET			3
+#define DORQ_REG_PF_MAX_ICID_4_RT_OFFSET			4
+#define DORQ_REG_PF_MAX_ICID_5_RT_OFFSET			5
+#define DORQ_REG_PF_MAX_ICID_6_RT_OFFSET			6
+#define DORQ_REG_PF_MAX_ICID_7_RT_OFFSET			7
+#define DORQ_REG_VF_MAX_ICID_0_RT_OFFSET			8
+#define DORQ_REG_VF_MAX_ICID_1_RT_OFFSET			9
+#define DORQ_REG_VF_MAX_ICID_2_RT_OFFSET			10
+#define DORQ_REG_VF_MAX_ICID_3_RT_OFFSET			11
+#define DORQ_REG_VF_MAX_ICID_4_RT_OFFSET			12
+#define DORQ_REG_VF_MAX_ICID_5_RT_OFFSET			13
+#define DORQ_REG_VF_MAX_ICID_6_RT_OFFSET			14
+#define DORQ_REG_VF_MAX_ICID_7_RT_OFFSET			15
+#define DORQ_REG_PF_WAKE_ALL_RT_OFFSET				16
+#define DORQ_REG_TAG1_ETHERTYPE_RT_OFFSET			17
+#define DORQ_REG_GLB_MAX_ICID_0_RT_OFFSET			18
+#define DORQ_REG_GLB_MAX_ICID_1_RT_OFFSET			19
+#define DORQ_REG_GLB_RANGE2CONN_TYPE_0_RT_OFFSET		20
+#define DORQ_REG_GLB_RANGE2CONN_TYPE_1_RT_OFFSET		21
+#define DORQ_REG_PRV_PF_MAX_ICID_2_RT_OFFSET			22
+#define DORQ_REG_PRV_PF_MAX_ICID_3_RT_OFFSET			23
+#define DORQ_REG_PRV_PF_MAX_ICID_4_RT_OFFSET			24
+#define DORQ_REG_PRV_PF_MAX_ICID_5_RT_OFFSET			25
+#define DORQ_REG_PRV_VF_MAX_ICID_2_RT_OFFSET			26
+#define DORQ_REG_PRV_VF_MAX_ICID_3_RT_OFFSET			27
+#define DORQ_REG_PRV_VF_MAX_ICID_4_RT_OFFSET			28
+#define DORQ_REG_PRV_VF_MAX_ICID_5_RT_OFFSET			29
+#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_2_RT_OFFSET		30
+#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_3_RT_OFFSET		31
+#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_4_RT_OFFSET		32
+#define DORQ_REG_PRV_PF_RANGE2CONN_TYPE_5_RT_OFFSET		33
+#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_2_RT_OFFSET		34
+#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_3_RT_OFFSET		35
+#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_4_RT_OFFSET		36
+#define DORQ_REG_PRV_VF_RANGE2CONN_TYPE_5_RT_OFFSET		37
+#define IGU_REG_PF_CONFIGURATION_RT_OFFSET			38
+#define IGU_REG_VF_CONFIGURATION_RT_OFFSET			39
+#define IGU_REG_ATTN_MSG_ADDR_L_RT_OFFSET			40
+#define IGU_REG_ATTN_MSG_ADDR_H_RT_OFFSET			41
+#define IGU_REG_LEADING_EDGE_LATCH_RT_OFFSET			42
+#define IGU_REG_TRAILING_EDGE_LATCH_RT_OFFSET			43
+#define CAU_REG_CQE_AGG_UNIT_SIZE_RT_OFFSET			44
+#define CAU_REG_SB_VAR_MEMORY_RT_OFFSET				45
+#define CAU_REG_SB_VAR_MEMORY_RT_SIZE				1024
+#define CAU_REG_SB_ADDR_MEMORY_RT_OFFSET			1069
+#define CAU_REG_SB_ADDR_MEMORY_RT_SIZE				1024
+#define CAU_REG_PI_MEMORY_RT_OFFSET				2093
+#define CAU_REG_PI_MEMORY_RT_SIZE				4416
+#define PRS_REG_SEARCH_RESP_INITIATOR_TYPE_RT_OFFSET		6509
+#define PRS_REG_TASK_ID_MAX_INITIATOR_PF_RT_OFFSET		6510
+#define PRS_REG_TASK_ID_MAX_INITIATOR_VF_RT_OFFSET		6511
+#define PRS_REG_TASK_ID_MAX_TARGET_PF_RT_OFFSET			6512
+#define PRS_REG_TASK_ID_MAX_TARGET_VF_RT_OFFSET			6513
+#define PRS_REG_SEARCH_TCP_RT_OFFSET				6514
+#define PRS_REG_SEARCH_FCOE_RT_OFFSET				6515
+#define PRS_REG_SEARCH_ROCE_RT_OFFSET				6516
+#define PRS_REG_ROCE_DEST_QP_MAX_VF_RT_OFFSET			6517
+#define PRS_REG_ROCE_DEST_QP_MAX_PF_RT_OFFSET			6518
+#define PRS_REG_SEARCH_OPENFLOW_RT_OFFSET			6519
+#define PRS_REG_SEARCH_NON_IP_AS_OPENFLOW_RT_OFFSET		6520
+#define PRS_REG_OPENFLOW_SUPPORT_ONLY_KNOWN_OVER_IP_RT_OFFSET	6521
+#define PRS_REG_OPENFLOW_SEARCH_KEY_MASK_RT_OFFSET		6522
+#define PRS_REG_TAG_ETHERTYPE_0_RT_OFFSET			6523
+#define PRS_REG_LIGHT_L2_ETHERTYPE_EN_RT_OFFSET			6524
+#define SRC_REG_FIRSTFREE_RT_OFFSET				6525
+#define SRC_REG_FIRSTFREE_RT_SIZE				2
+#define SRC_REG_LASTFREE_RT_OFFSET				6527
+#define SRC_REG_LASTFREE_RT_SIZE				2
+#define SRC_REG_COUNTFREE_RT_OFFSET				6529
+#define SRC_REG_NUMBER_HASH_BITS_RT_OFFSET			6530
+#define PSWRQ2_REG_CDUT_P_SIZE_RT_OFFSET			6531
+#define PSWRQ2_REG_CDUC_P_SIZE_RT_OFFSET			6532
+#define PSWRQ2_REG_TM_P_SIZE_RT_OFFSET				6533
+#define PSWRQ2_REG_QM_P_SIZE_RT_OFFSET				6534
+#define PSWRQ2_REG_SRC_P_SIZE_RT_OFFSET				6535
+#define PSWRQ2_REG_TSDM_P_SIZE_RT_OFFSET			6536
+#define PSWRQ2_REG_TM_FIRST_ILT_RT_OFFSET			6537
+#define PSWRQ2_REG_TM_LAST_ILT_RT_OFFSET			6538
+#define PSWRQ2_REG_QM_FIRST_ILT_RT_OFFSET			6539
+#define PSWRQ2_REG_QM_LAST_ILT_RT_OFFSET			6540
+#define PSWRQ2_REG_SRC_FIRST_ILT_RT_OFFSET			6541
+#define PSWRQ2_REG_SRC_LAST_ILT_RT_OFFSET			6542
+#define PSWRQ2_REG_CDUC_FIRST_ILT_RT_OFFSET			6543
+#define PSWRQ2_REG_CDUC_LAST_ILT_RT_OFFSET			6544
+#define PSWRQ2_REG_CDUT_FIRST_ILT_RT_OFFSET			6545
+#define PSWRQ2_REG_CDUT_LAST_ILT_RT_OFFSET			6546
+#define PSWRQ2_REG_TSDM_FIRST_ILT_RT_OFFSET			6547
+#define PSWRQ2_REG_TSDM_LAST_ILT_RT_OFFSET			6548
+#define PSWRQ2_REG_TM_NUMBER_OF_PF_BLOCKS_RT_OFFSET		6549
+#define PSWRQ2_REG_CDUT_NUMBER_OF_PF_BLOCKS_RT_OFFSET		6550
+#define PSWRQ2_REG_CDUC_NUMBER_OF_PF_BLOCKS_RT_OFFSET		6551
+#define PSWRQ2_REG_TM_VF_BLOCKS_RT_OFFSET			6552
+#define PSWRQ2_REG_CDUT_VF_BLOCKS_RT_OFFSET			6553
+#define PSWRQ2_REG_CDUC_VF_BLOCKS_RT_OFFSET			6554
+#define PSWRQ2_REG_TM_BLOCKS_FACTOR_RT_OFFSET			6555
+#define PSWRQ2_REG_CDUT_BLOCKS_FACTOR_RT_OFFSET			6556
+#define PSWRQ2_REG_CDUC_BLOCKS_FACTOR_RT_OFFSET			6557
+#define PSWRQ2_REG_VF_BASE_RT_OFFSET				6558
+#define PSWRQ2_REG_VF_LAST_ILT_RT_OFFSET			6559
+#define PSWRQ2_REG_DRAM_ALIGN_WR_RT_OFFSET			6560
+#define PSWRQ2_REG_DRAM_ALIGN_RD_RT_OFFSET			6561
+#define PSWRQ2_REG_TGSRC_FIRST_ILT_RT_OFFSET			6562
+#define PSWRQ2_REG_RGSRC_FIRST_ILT_RT_OFFSET			6563
+#define PSWRQ2_REG_TGSRC_LAST_ILT_RT_OFFSET			6564
+#define PSWRQ2_REG_RGSRC_LAST_ILT_RT_OFFSET			6565
+#define PSWRQ2_REG_ILT_MEMORY_RT_OFFSET				6566
+#define PSWRQ2_REG_ILT_MEMORY_RT_SIZE				26414
+#define PGLUE_REG_B_VF_BASE_RT_OFFSET				32980
+#define PGLUE_REG_B_MSDM_OFFSET_MASK_B_RT_OFFSET		32981
+#define PGLUE_REG_B_MSDM_VF_SHIFT_B_RT_OFFSET			32982
+#define PGLUE_REG_B_CACHE_LINE_SIZE_RT_OFFSET			32983
+#define PGLUE_REG_B_PF_BAR0_SIZE_RT_OFFSET			32984
+#define PGLUE_REG_B_PF_BAR1_SIZE_RT_OFFSET			32985
+#define PGLUE_REG_B_VF_BAR1_SIZE_RT_OFFSET			32986
+#define TM_REG_VF_ENABLE_CONN_RT_OFFSET				32987
+#define TM_REG_PF_ENABLE_CONN_RT_OFFSET				32988
+#define TM_REG_PF_ENABLE_TASK_RT_OFFSET				32989
+#define TM_REG_GROUP_SIZE_RESOLUTION_CONN_RT_OFFSET		32990
+#define TM_REG_GROUP_SIZE_RESOLUTION_TASK_RT_OFFSET		32991
+#define TM_REG_CONFIG_CONN_MEM_RT_OFFSET			32992
+#define TM_REG_CONFIG_CONN_MEM_RT_SIZE				416
+#define TM_REG_CONFIG_TASK_MEM_RT_OFFSET			33408
+#define TM_REG_CONFIG_TASK_MEM_RT_SIZE				608
+#define QM_REG_MAXPQSIZE_0_RT_OFFSET				34016
+#define QM_REG_MAXPQSIZE_1_RT_OFFSET				34017
+#define QM_REG_MAXPQSIZE_2_RT_OFFSET				34018
+#define QM_REG_MAXPQSIZETXSEL_0_RT_OFFSET			34019
+#define QM_REG_MAXPQSIZETXSEL_1_RT_OFFSET			34020
+#define QM_REG_MAXPQSIZETXSEL_2_RT_OFFSET			34021
+#define QM_REG_MAXPQSIZETXSEL_3_RT_OFFSET			34022
+#define QM_REG_MAXPQSIZETXSEL_4_RT_OFFSET			34023
+#define QM_REG_MAXPQSIZETXSEL_5_RT_OFFSET			34024
+#define QM_REG_MAXPQSIZETXSEL_6_RT_OFFSET			34025
+#define QM_REG_MAXPQSIZETXSEL_7_RT_OFFSET			34026
+#define QM_REG_MAXPQSIZETXSEL_8_RT_OFFSET			34027
+#define QM_REG_MAXPQSIZETXSEL_9_RT_OFFSET			34028
+#define QM_REG_MAXPQSIZETXSEL_10_RT_OFFSET			34029
+#define QM_REG_MAXPQSIZETXSEL_11_RT_OFFSET			34030
+#define QM_REG_MAXPQSIZETXSEL_12_RT_OFFSET			34031
+#define QM_REG_MAXPQSIZETXSEL_13_RT_OFFSET			34032
+#define QM_REG_MAXPQSIZETXSEL_14_RT_OFFSET			34033
+#define QM_REG_MAXPQSIZETXSEL_15_RT_OFFSET			34034
+#define QM_REG_MAXPQSIZETXSEL_16_RT_OFFSET			34035
+#define QM_REG_MAXPQSIZETXSEL_17_RT_OFFSET			34036
+#define QM_REG_MAXPQSIZETXSEL_18_RT_OFFSET			34037
+#define QM_REG_MAXPQSIZETXSEL_19_RT_OFFSET			34038
+#define QM_REG_MAXPQSIZETXSEL_20_RT_OFFSET			34039
+#define QM_REG_MAXPQSIZETXSEL_21_RT_OFFSET			34040
+#define QM_REG_MAXPQSIZETXSEL_22_RT_OFFSET			34041
+#define QM_REG_MAXPQSIZETXSEL_23_RT_OFFSET			34042
+#define QM_REG_MAXPQSIZETXSEL_24_RT_OFFSET			34043
+#define QM_REG_MAXPQSIZETXSEL_25_RT_OFFSET			34044
+#define QM_REG_MAXPQSIZETXSEL_26_RT_OFFSET			34045
+#define QM_REG_MAXPQSIZETXSEL_27_RT_OFFSET			34046
+#define QM_REG_MAXPQSIZETXSEL_28_RT_OFFSET			34047
+#define QM_REG_MAXPQSIZETXSEL_29_RT_OFFSET			34048
+#define QM_REG_MAXPQSIZETXSEL_30_RT_OFFSET			34049
+#define QM_REG_MAXPQSIZETXSEL_31_RT_OFFSET			34050
+#define QM_REG_MAXPQSIZETXSEL_32_RT_OFFSET			34051
+#define QM_REG_MAXPQSIZETXSEL_33_RT_OFFSET			34052
+#define QM_REG_MAXPQSIZETXSEL_34_RT_OFFSET			34053
+#define QM_REG_MAXPQSIZETXSEL_35_RT_OFFSET			34054
+#define QM_REG_MAXPQSIZETXSEL_36_RT_OFFSET			34055
+#define QM_REG_MAXPQSIZETXSEL_37_RT_OFFSET			34056
+#define QM_REG_MAXPQSIZETXSEL_38_RT_OFFSET			34057
+#define QM_REG_MAXPQSIZETXSEL_39_RT_OFFSET			34058
+#define QM_REG_MAXPQSIZETXSEL_40_RT_OFFSET			34059
+#define QM_REG_MAXPQSIZETXSEL_41_RT_OFFSET			34060
+#define QM_REG_MAXPQSIZETXSEL_42_RT_OFFSET			34061
+#define QM_REG_MAXPQSIZETXSEL_43_RT_OFFSET			34062
+#define QM_REG_MAXPQSIZETXSEL_44_RT_OFFSET			34063
+#define QM_REG_MAXPQSIZETXSEL_45_RT_OFFSET			34064
+#define QM_REG_MAXPQSIZETXSEL_46_RT_OFFSET			34065
+#define QM_REG_MAXPQSIZETXSEL_47_RT_OFFSET			34066
+#define QM_REG_MAXPQSIZETXSEL_48_RT_OFFSET			34067
+#define QM_REG_MAXPQSIZETXSEL_49_RT_OFFSET			34068
+#define QM_REG_MAXPQSIZETXSEL_50_RT_OFFSET			34069
+#define QM_REG_MAXPQSIZETXSEL_51_RT_OFFSET			34070
+#define QM_REG_MAXPQSIZETXSEL_52_RT_OFFSET			34071
+#define QM_REG_MAXPQSIZETXSEL_53_RT_OFFSET			34072
+#define QM_REG_MAXPQSIZETXSEL_54_RT_OFFSET			34073
+#define QM_REG_MAXPQSIZETXSEL_55_RT_OFFSET			34074
+#define QM_REG_MAXPQSIZETXSEL_56_RT_OFFSET			34075
+#define QM_REG_MAXPQSIZETXSEL_57_RT_OFFSET			34076
+#define QM_REG_MAXPQSIZETXSEL_58_RT_OFFSET			34077
+#define QM_REG_MAXPQSIZETXSEL_59_RT_OFFSET			34078
+#define QM_REG_MAXPQSIZETXSEL_60_RT_OFFSET			34079
+#define QM_REG_MAXPQSIZETXSEL_61_RT_OFFSET			34080
+#define QM_REG_MAXPQSIZETXSEL_62_RT_OFFSET			34081
+#define QM_REG_MAXPQSIZETXSEL_63_RT_OFFSET			34082
+#define QM_REG_BASEADDROTHERPQ_RT_OFFSET			34083
+#define QM_REG_BASEADDROTHERPQ_RT_SIZE				128
+#define QM_REG_PTRTBLOTHER_RT_OFFSET				34211
+#define QM_REG_PTRTBLOTHER_RT_SIZE				256
+#define QM_REG_AFULLQMBYPTHRPFWFQ_RT_OFFSET			34467
+#define QM_REG_AFULLQMBYPTHRVPWFQ_RT_OFFSET			34468
+#define QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET			34469
+#define QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET			34470
+#define QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET			34471
+#define QM_REG_WRROTHERPQGRP_0_RT_OFFSET			34472
+#define QM_REG_WRROTHERPQGRP_1_RT_OFFSET			34473
+#define QM_REG_WRROTHERPQGRP_2_RT_OFFSET			34474
+#define QM_REG_WRROTHERPQGRP_3_RT_OFFSET			34475
+#define QM_REG_WRROTHERPQGRP_4_RT_OFFSET			34476
+#define QM_REG_WRROTHERPQGRP_5_RT_OFFSET			34477
+#define QM_REG_WRROTHERPQGRP_6_RT_OFFSET			34478
+#define QM_REG_WRROTHERPQGRP_7_RT_OFFSET			34479
+#define QM_REG_WRROTHERPQGRP_8_RT_OFFSET			34480
+#define QM_REG_WRROTHERPQGRP_9_RT_OFFSET			34481
+#define QM_REG_WRROTHERPQGRP_10_RT_OFFSET			34482
+#define QM_REG_WRROTHERPQGRP_11_RT_OFFSET			34483
+#define QM_REG_WRROTHERPQGRP_12_RT_OFFSET			34484
+#define QM_REG_WRROTHERPQGRP_13_RT_OFFSET			34485
+#define QM_REG_WRROTHERPQGRP_14_RT_OFFSET			34486
+#define QM_REG_WRROTHERPQGRP_15_RT_OFFSET			34487
+#define QM_REG_WRROTHERGRPWEIGHT_0_RT_OFFSET			34488
+#define QM_REG_WRROTHERGRPWEIGHT_1_RT_OFFSET			34489
+#define QM_REG_WRROTHERGRPWEIGHT_2_RT_OFFSET			34490
+#define QM_REG_WRROTHERGRPWEIGHT_3_RT_OFFSET			34491
+#define QM_REG_WRRTXGRPWEIGHT_0_RT_OFFSET			34492
+#define QM_REG_WRRTXGRPWEIGHT_1_RT_OFFSET			34493
+#define QM_REG_PQTX2PF_0_RT_OFFSET				34494
+#define QM_REG_PQTX2PF_1_RT_OFFSET				34495
+#define QM_REG_PQTX2PF_2_RT_OFFSET				34496
+#define QM_REG_PQTX2PF_3_RT_OFFSET				34497
+#define QM_REG_PQTX2PF_4_RT_OFFSET				34498
+#define QM_REG_PQTX2PF_5_RT_OFFSET				34499
+#define QM_REG_PQTX2PF_6_RT_OFFSET				34500
+#define QM_REG_PQTX2PF_7_RT_OFFSET				34501
+#define QM_REG_PQTX2PF_8_RT_OFFSET				34502
+#define QM_REG_PQTX2PF_9_RT_OFFSET				34503
+#define QM_REG_PQTX2PF_10_RT_OFFSET				34504
+#define QM_REG_PQTX2PF_11_RT_OFFSET				34505
+#define QM_REG_PQTX2PF_12_RT_OFFSET				34506
+#define QM_REG_PQTX2PF_13_RT_OFFSET				34507
+#define QM_REG_PQTX2PF_14_RT_OFFSET				34508
+#define QM_REG_PQTX2PF_15_RT_OFFSET				34509
+#define QM_REG_PQTX2PF_16_RT_OFFSET				34510
+#define QM_REG_PQTX2PF_17_RT_OFFSET				34511
+#define QM_REG_PQTX2PF_18_RT_OFFSET				34512
+#define QM_REG_PQTX2PF_19_RT_OFFSET				34513
+#define QM_REG_PQTX2PF_20_RT_OFFSET				34514
+#define QM_REG_PQTX2PF_21_RT_OFFSET				34515
+#define QM_REG_PQTX2PF_22_RT_OFFSET				34516
+#define QM_REG_PQTX2PF_23_RT_OFFSET				34517
+#define QM_REG_PQTX2PF_24_RT_OFFSET				34518
+#define QM_REG_PQTX2PF_25_RT_OFFSET				34519
+#define QM_REG_PQTX2PF_26_RT_OFFSET				34520
+#define QM_REG_PQTX2PF_27_RT_OFFSET				34521
+#define QM_REG_PQTX2PF_28_RT_OFFSET				34522
+#define QM_REG_PQTX2PF_29_RT_OFFSET				34523
+#define QM_REG_PQTX2PF_30_RT_OFFSET				34524
+#define QM_REG_PQTX2PF_31_RT_OFFSET				34525
+#define QM_REG_PQTX2PF_32_RT_OFFSET				34526
+#define QM_REG_PQTX2PF_33_RT_OFFSET				34527
+#define QM_REG_PQTX2PF_34_RT_OFFSET				34528
+#define QM_REG_PQTX2PF_35_RT_OFFSET				34529
+#define QM_REG_PQTX2PF_36_RT_OFFSET				34530
+#define QM_REG_PQTX2PF_37_RT_OFFSET				34531
+#define QM_REG_PQTX2PF_38_RT_OFFSET				34532
+#define QM_REG_PQTX2PF_39_RT_OFFSET				34533
+#define QM_REG_PQTX2PF_40_RT_OFFSET				34534
+#define QM_REG_PQTX2PF_41_RT_OFFSET				34535
+#define QM_REG_PQTX2PF_42_RT_OFFSET				34536
+#define QM_REG_PQTX2PF_43_RT_OFFSET				34537
+#define QM_REG_PQTX2PF_44_RT_OFFSET				34538
+#define QM_REG_PQTX2PF_45_RT_OFFSET				34539
+#define QM_REG_PQTX2PF_46_RT_OFFSET				34540
+#define QM_REG_PQTX2PF_47_RT_OFFSET				34541
+#define QM_REG_PQTX2PF_48_RT_OFFSET				34542
+#define QM_REG_PQTX2PF_49_RT_OFFSET				34543
+#define QM_REG_PQTX2PF_50_RT_OFFSET				34544
+#define QM_REG_PQTX2PF_51_RT_OFFSET				34545
+#define QM_REG_PQTX2PF_52_RT_OFFSET				34546
+#define QM_REG_PQTX2PF_53_RT_OFFSET				34547
+#define QM_REG_PQTX2PF_54_RT_OFFSET				34548
+#define QM_REG_PQTX2PF_55_RT_OFFSET				34549
+#define QM_REG_PQTX2PF_56_RT_OFFSET				34550
+#define QM_REG_PQTX2PF_57_RT_OFFSET				34551
+#define QM_REG_PQTX2PF_58_RT_OFFSET				34552
+#define QM_REG_PQTX2PF_59_RT_OFFSET				34553
+#define QM_REG_PQTX2PF_60_RT_OFFSET				34554
+#define QM_REG_PQTX2PF_61_RT_OFFSET				34555
+#define QM_REG_PQTX2PF_62_RT_OFFSET				34556
+#define QM_REG_PQTX2PF_63_RT_OFFSET				34557
+#define QM_REG_PQOTHER2PF_0_RT_OFFSET				34558
+#define QM_REG_PQOTHER2PF_1_RT_OFFSET				34559
+#define QM_REG_PQOTHER2PF_2_RT_OFFSET				34560
+#define QM_REG_PQOTHER2PF_3_RT_OFFSET				34561
+#define QM_REG_PQOTHER2PF_4_RT_OFFSET				34562
+#define QM_REG_PQOTHER2PF_5_RT_OFFSET				34563
+#define QM_REG_PQOTHER2PF_6_RT_OFFSET				34564
+#define QM_REG_PQOTHER2PF_7_RT_OFFSET				34565
+#define QM_REG_PQOTHER2PF_8_RT_OFFSET				34566
+#define QM_REG_PQOTHER2PF_9_RT_OFFSET				34567
+#define QM_REG_PQOTHER2PF_10_RT_OFFSET				34568
+#define QM_REG_PQOTHER2PF_11_RT_OFFSET				34569
+#define QM_REG_PQOTHER2PF_12_RT_OFFSET				34570
+#define QM_REG_PQOTHER2PF_13_RT_OFFSET				34571
+#define QM_REG_PQOTHER2PF_14_RT_OFFSET				34572
+#define QM_REG_PQOTHER2PF_15_RT_OFFSET				34573
+#define QM_REG_RLGLBLPERIOD_0_RT_OFFSET				34574
+#define QM_REG_RLGLBLPERIOD_1_RT_OFFSET				34575
+#define QM_REG_RLGLBLPERIODTIMER_0_RT_OFFSET			34576
+#define QM_REG_RLGLBLPERIODTIMER_1_RT_OFFSET			34577
+#define QM_REG_RLGLBLPERIODSEL_0_RT_OFFSET			34578
+#define QM_REG_RLGLBLPERIODSEL_1_RT_OFFSET			34579
+#define QM_REG_RLGLBLPERIODSEL_2_RT_OFFSET			34580
+#define QM_REG_RLGLBLPERIODSEL_3_RT_OFFSET			34581
+#define QM_REG_RLGLBLPERIODSEL_4_RT_OFFSET			34582
+#define QM_REG_RLGLBLPERIODSEL_5_RT_OFFSET			34583
+#define QM_REG_RLGLBLPERIODSEL_6_RT_OFFSET			34584
+#define QM_REG_RLGLBLPERIODSEL_7_RT_OFFSET			34585
+#define QM_REG_RLGLBLINCVAL_RT_OFFSET				34586
+#define QM_REG_RLGLBLINCVAL_RT_SIZE				256
+#define QM_REG_RLGLBLUPPERBOUND_RT_OFFSET			34842
+#define QM_REG_RLGLBLUPPERBOUND_RT_SIZE				256
+#define QM_REG_RLGLBLCRD_RT_OFFSET				35098
+#define QM_REG_RLGLBLCRD_RT_SIZE				256
+#define QM_REG_RLGLBLENABLE_RT_OFFSET				35354
+#define QM_REG_RLPFPERIOD_RT_OFFSET				35355
+#define QM_REG_RLPFPERIODTIMER_RT_OFFSET			35356
+#define QM_REG_RLPFINCVAL_RT_OFFSET				35357
+#define QM_REG_RLPFINCVAL_RT_SIZE				16
+#define QM_REG_RLPFUPPERBOUND_RT_OFFSET				35373
+#define QM_REG_RLPFUPPERBOUND_RT_SIZE				16
+#define QM_REG_RLPFCRD_RT_OFFSET				35389
+#define QM_REG_RLPFCRD_RT_SIZE					16
+#define QM_REG_RLPFENABLE_RT_OFFSET				35405
+#define QM_REG_RLPFVOQENABLE_RT_OFFSET				35406
+#define QM_REG_WFQPFWEIGHT_RT_OFFSET				35407
+#define QM_REG_WFQPFWEIGHT_RT_SIZE				16
+#define QM_REG_WFQPFUPPERBOUND_RT_OFFSET			35423
+#define QM_REG_WFQPFUPPERBOUND_RT_SIZE				16
+#define QM_REG_WFQPFCRD_RT_OFFSET				35439
+#define QM_REG_WFQPFCRD_RT_SIZE					256
+#define QM_REG_WFQPFENABLE_RT_OFFSET				35695
+#define QM_REG_WFQVPENABLE_RT_OFFSET				35696
+#define QM_REG_BASEADDRTXPQ_RT_OFFSET				35697
+#define QM_REG_BASEADDRTXPQ_RT_SIZE				512
+#define QM_REG_TXPQMAP_RT_OFFSET				36209
+#define QM_REG_TXPQMAP_RT_SIZE					512
+#define QM_REG_WFQVPWEIGHT_RT_OFFSET				36721
+#define QM_REG_WFQVPWEIGHT_RT_SIZE				512
+#define QM_REG_WFQVPCRD_RT_OFFSET				37233
+#define QM_REG_WFQVPCRD_RT_SIZE					512
+#define QM_REG_WFQVPMAP_RT_OFFSET				37745
+#define QM_REG_WFQVPMAP_RT_SIZE					512
+#define QM_REG_PTRTBLTX_RT_OFFSET				38257
+#define QM_REG_PTRTBLTX_RT_SIZE					1024
+#define QM_REG_WFQPFCRD_MSB_RT_OFFSET				39281
+#define QM_REG_WFQPFCRD_MSB_RT_SIZE				320
+#define QM_REG_VOQCRDLINE_RT_OFFSET				39601
+#define QM_REG_VOQCRDLINE_RT_SIZE				36
+#define QM_REG_VOQINITCRDLINE_RT_OFFSET				39637
+#define QM_REG_VOQINITCRDLINE_RT_SIZE				36
+#define QM_REG_RLPFVOQENABLE_MSB_RT_OFFSET			39673
+#define NIG_REG_TAG_ETHERTYPE_0_RT_OFFSET			39674
+#define NIG_REG_BRB_GATE_DNTFWD_PORT_RT_OFFSET			39675
+#define NIG_REG_OUTER_TAG_VALUE_LIST0_RT_OFFSET			39676
+#define NIG_REG_OUTER_TAG_VALUE_LIST1_RT_OFFSET			39677
+#define NIG_REG_OUTER_TAG_VALUE_LIST2_RT_OFFSET			39678
+#define NIG_REG_OUTER_TAG_VALUE_LIST3_RT_OFFSET			39679
+#define NIG_REG_LLH_FUNC_TAGMAC_CLS_TYPE_RT_OFFSET		39680
+#define NIG_REG_LLH_FUNC_TAG_EN_RT_OFFSET			39681
+#define NIG_REG_LLH_FUNC_TAG_EN_RT_SIZE				4
+#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_OFFSET			39685
+#define NIG_REG_LLH_FUNC_TAG_VALUE_RT_SIZE			4
+#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_OFFSET			39689
+#define NIG_REG_LLH_FUNC_FILTER_VALUE_RT_SIZE			32
+#define NIG_REG_LLH_FUNC_FILTER_EN_RT_OFFSET			39721
+#define NIG_REG_LLH_FUNC_FILTER_EN_RT_SIZE			16
+#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_OFFSET			39737
+#define NIG_REG_LLH_FUNC_FILTER_MODE_RT_SIZE			16
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET		39753
+#define NIG_REG_LLH_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE		16
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_OFFSET		39769
+#define NIG_REG_LLH_FUNC_FILTER_HDR_SEL_RT_SIZE			16
+#define NIG_REG_TX_EDPM_CTRL_RT_OFFSET				39785
+#define NIG_REG_ROCE_DUPLICATE_TO_HOST_RT_OFFSET		39786
+#define NIG_REG_PPF_TO_ENGINE_SEL_RT_OFFSET			39787
+#define NIG_REG_PPF_TO_ENGINE_SEL_RT_SIZE			8
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_VALUE_RT_OFFSET		39795
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_VALUE_RT_SIZE		1024
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_EN_RT_OFFSET		40819
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_EN_RT_SIZE		512
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_MODE_RT_OFFSET		41331
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_MODE_RT_SIZE		512
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_PROTOCOL_TYPE_RT_OFFSET	41843
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_PROTOCOL_TYPE_RT_SIZE	512
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_HDR_SEL_RT_OFFSET	42355
+#define NIG_REG_LLH_PF_CLS_FUNC_FILTER_HDR_SEL_RT_SIZE		512
+#define NIG_REG_LLH_PF_CLS_FILTERS_MAP_RT_OFFSET		42867
+#define NIG_REG_LLH_PF_CLS_FILTERS_MAP_RT_SIZE			32
+#define CDU_REG_CID_ADDR_PARAMS_RT_OFFSET			42899
+#define CDU_REG_SEGMENT0_PARAMS_RT_OFFSET			42900
+#define CDU_REG_SEGMENT1_PARAMS_RT_OFFSET			42901
+#define CDU_REG_PF_SEG0_TYPE_OFFSET_RT_OFFSET			42902
+#define CDU_REG_PF_SEG1_TYPE_OFFSET_RT_OFFSET			42903
+#define CDU_REG_PF_SEG2_TYPE_OFFSET_RT_OFFSET			42904
+#define CDU_REG_PF_SEG3_TYPE_OFFSET_RT_OFFSET			42905
+#define CDU_REG_PF_FL_SEG0_TYPE_OFFSET_RT_OFFSET		42906
+#define CDU_REG_PF_FL_SEG1_TYPE_OFFSET_RT_OFFSET		42907
+#define CDU_REG_PF_FL_SEG2_TYPE_OFFSET_RT_OFFSET		42908
+#define CDU_REG_PF_FL_SEG3_TYPE_OFFSET_RT_OFFSET		42909
+#define CDU_REG_VF_SEG_TYPE_OFFSET_RT_OFFSET			42910
+#define CDU_REG_VF_FL_SEG_TYPE_OFFSET_RT_OFFSET			42911
+#define PBF_REG_TAG_ETHERTYPE_0_RT_OFFSET			42912
+#define PBF_REG_BTB_SHARED_AREA_SIZE_RT_OFFSET			42913
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET		42914
+#define PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET			42915
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ0_RT_OFFSET		42916
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET		42917
+#define PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET			42918
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ1_RT_OFFSET		42919
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ2_RT_OFFSET		42920
+#define PBF_REG_BTB_GUARANTEED_VOQ2_RT_OFFSET			42921
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ2_RT_OFFSET		42922
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ3_RT_OFFSET		42923
+#define PBF_REG_BTB_GUARANTEED_VOQ3_RT_OFFSET			42924
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ3_RT_OFFSET		42925
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ4_RT_OFFSET		42926
+#define PBF_REG_BTB_GUARANTEED_VOQ4_RT_OFFSET			42927
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ4_RT_OFFSET		42928
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ5_RT_OFFSET		42929
+#define PBF_REG_BTB_GUARANTEED_VOQ5_RT_OFFSET			42930
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ5_RT_OFFSET		42931
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ6_RT_OFFSET		42932
+#define PBF_REG_BTB_GUARANTEED_VOQ6_RT_OFFSET			42933
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ6_RT_OFFSET		42934
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ7_RT_OFFSET		42935
+#define PBF_REG_BTB_GUARANTEED_VOQ7_RT_OFFSET			42936
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ7_RT_OFFSET		42937
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ8_RT_OFFSET		42938
+#define PBF_REG_BTB_GUARANTEED_VOQ8_RT_OFFSET			42939
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ8_RT_OFFSET		42940
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ9_RT_OFFSET		42941
+#define PBF_REG_BTB_GUARANTEED_VOQ9_RT_OFFSET			42942
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ9_RT_OFFSET		42943
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ10_RT_OFFSET		42944
+#define PBF_REG_BTB_GUARANTEED_VOQ10_RT_OFFSET			42945
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ10_RT_OFFSET		42946
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ11_RT_OFFSET		42947
+#define PBF_REG_BTB_GUARANTEED_VOQ11_RT_OFFSET			42948
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ11_RT_OFFSET		42949
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ12_RT_OFFSET		42950
+#define PBF_REG_BTB_GUARANTEED_VOQ12_RT_OFFSET			42951
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ12_RT_OFFSET		42952
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ13_RT_OFFSET		42953
+#define PBF_REG_BTB_GUARANTEED_VOQ13_RT_OFFSET			42954
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ13_RT_OFFSET		42955
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ14_RT_OFFSET		42956
+#define PBF_REG_BTB_GUARANTEED_VOQ14_RT_OFFSET			42957
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ14_RT_OFFSET		42958
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ15_RT_OFFSET		42959
+#define PBF_REG_BTB_GUARANTEED_VOQ15_RT_OFFSET			42960
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ15_RT_OFFSET		42961
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ16_RT_OFFSET		42962
+#define PBF_REG_BTB_GUARANTEED_VOQ16_RT_OFFSET			42963
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ16_RT_OFFSET		42964
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ17_RT_OFFSET		42965
+#define PBF_REG_BTB_GUARANTEED_VOQ17_RT_OFFSET			42966
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ17_RT_OFFSET		42967
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ18_RT_OFFSET		42968
+#define PBF_REG_BTB_GUARANTEED_VOQ18_RT_OFFSET			42969
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ18_RT_OFFSET		42970
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ19_RT_OFFSET		42971
+#define PBF_REG_BTB_GUARANTEED_VOQ19_RT_OFFSET			42972
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ19_RT_OFFSET		42973
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ20_RT_OFFSET		42974
+#define PBF_REG_BTB_GUARANTEED_VOQ20_RT_OFFSET			42975
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ20_RT_OFFSET		42976
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ21_RT_OFFSET		42977
+#define PBF_REG_BTB_GUARANTEED_VOQ21_RT_OFFSET			42978
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ21_RT_OFFSET		42979
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ22_RT_OFFSET		42980
+#define PBF_REG_BTB_GUARANTEED_VOQ22_RT_OFFSET			42981
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ22_RT_OFFSET		42982
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ23_RT_OFFSET		42983
+#define PBF_REG_BTB_GUARANTEED_VOQ23_RT_OFFSET			42984
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ23_RT_OFFSET		42985
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ24_RT_OFFSET		42986
+#define PBF_REG_BTB_GUARANTEED_VOQ24_RT_OFFSET			42987
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ24_RT_OFFSET		42988
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ25_RT_OFFSET		42989
+#define PBF_REG_BTB_GUARANTEED_VOQ25_RT_OFFSET			42990
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ25_RT_OFFSET		42991
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ26_RT_OFFSET		42992
+#define PBF_REG_BTB_GUARANTEED_VOQ26_RT_OFFSET			42993
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ26_RT_OFFSET		42994
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ27_RT_OFFSET		42995
+#define PBF_REG_BTB_GUARANTEED_VOQ27_RT_OFFSET			42996
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ27_RT_OFFSET		42997
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ28_RT_OFFSET		42998
+#define PBF_REG_BTB_GUARANTEED_VOQ28_RT_OFFSET			42999
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ28_RT_OFFSET		43000
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ29_RT_OFFSET		43001
+#define PBF_REG_BTB_GUARANTEED_VOQ29_RT_OFFSET			43002
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ29_RT_OFFSET		43003
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ30_RT_OFFSET		43004
+#define PBF_REG_BTB_GUARANTEED_VOQ30_RT_OFFSET			43005
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ30_RT_OFFSET		43006
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ31_RT_OFFSET		43007
+#define PBF_REG_BTB_GUARANTEED_VOQ31_RT_OFFSET			43008
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ31_RT_OFFSET		43009
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ32_RT_OFFSET		43010
+#define PBF_REG_BTB_GUARANTEED_VOQ32_RT_OFFSET			43011
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ32_RT_OFFSET		43012
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ33_RT_OFFSET		43013
+#define PBF_REG_BTB_GUARANTEED_VOQ33_RT_OFFSET			43014
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ33_RT_OFFSET		43015
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ34_RT_OFFSET		43016
+#define PBF_REG_BTB_GUARANTEED_VOQ34_RT_OFFSET			43017
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ34_RT_OFFSET		43018
+#define PBF_REG_YCMD_QS_NUM_LINES_VOQ35_RT_OFFSET		43019
+#define PBF_REG_BTB_GUARANTEED_VOQ35_RT_OFFSET			43020
+#define PBF_REG_BTB_SHARED_AREA_SETUP_VOQ35_RT_OFFSET		43021
+#define XCM_REG_CON_PHY_Q3_RT_OFFSET				43022
+
+#define RUNTIME_ARRAY_SIZE	43023
+
+/* Init Callbacks */
+#define DMAE_READY_CB	0
 
 /* The eth storm context for the Tstorm */
 struct tstorm_eth_conn_st_ctx {
@@ -4436,219 +4935,219 @@ struct xstorm_eth_conn_st_ctx {
 	__le32 reserved[60];
 };
 
-struct xstorm_eth_conn_ag_ctx {
+struct e4_xstorm_eth_conn_ag_ctx {
 	u8 reserved0;
-	u8 eth_state;
+	u8 state;
 	u8 flags0;
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED1_SHIFT		1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED2_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED3_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED4_SHIFT		5
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED5_SHIFT		6
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED6_SHIFT		7
+#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED1_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED1_SHIFT	1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED2_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED2_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED3_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED3_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED4_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED4_SHIFT	5
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED5_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED5_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED6_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED6_SHIFT	7
 		u8 flags1;
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED7_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED8_SHIFT		1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED9_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_BIT11_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_BIT11_SHIFT		3
-#define XSTORM_ETH_CONN_AG_CTX_BIT12_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_BIT12_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_BIT13_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_BIT13_SHIFT		5
-#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT	6
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT	7
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED7_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED7_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED8_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED8_SHIFT	1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED9_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED9_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_BIT11_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_ETH_CONN_AG_CTX_E5_RESERVED2_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_E5_RESERVED2_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_E5_RESERVED3_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_E5_RESERVED3_SHIFT	5
+#define E4_XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT	7
 	u8 flags2;
-#define XSTORM_ETH_CONN_AG_CTX_CF0_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF0_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_CF1_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF1_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_CF2_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF2_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_CF3_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF3_SHIFT		6
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF0_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF1_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF2_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORM_ETH_CONN_AG_CTX_CF4_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF4_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_CF5_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF5_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_CF6_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF6_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_CF7_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF7_SHIFT		6
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF4_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF5_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF6_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF7_SHIFT	6
 		u8 flags4;
-#define XSTORM_ETH_CONN_AG_CTX_CF8_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF8_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_CF9_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF9_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_CF10_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF10_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_CF11_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF11_SHIFT		6
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_ETH_CONN_AG_CTX_CF12_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF12_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_CF13_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF13_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_CF14_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF14_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_CF15_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_CF15_SHIFT		6
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF12_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF12_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF13_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF13_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF14_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF14_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF15_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF15_SHIFT	6
 	u8 flags6;
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK	0x3
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT	0
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_MASK	0x3
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT	2
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_MASK	0x3
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK		0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_MASK		0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_MASK			0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_SHIFT			4
+#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_MASK		0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_SHIFT		6
 	u8 flags7;
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED10_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_CF0EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT		6
-#define XSTORM_ETH_CONN_AG_CTX_CF1EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT		7
+#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_MASK		0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED10_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED10_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT		6
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT		7
 	u8 flags8;
-#define XSTORM_ETH_CONN_AG_CTX_CF2EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_CF3EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT		1
-#define XSTORM_ETH_CONN_AG_CTX_CF4EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_CF5EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT		3
-#define XSTORM_ETH_CONN_AG_CTX_CF6EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_CF7EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT		5
-#define XSTORM_ETH_CONN_AG_CTX_CF8EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT		6
-#define XSTORM_ETH_CONN_AG_CTX_CF9EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT		7
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF3EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT	1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF4EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF5EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT	3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF6EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF7EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT	5
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF8EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF9EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT	7
 	u8 flags9;
-#define XSTORM_ETH_CONN_AG_CTX_CF10EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_CF11EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF11EN_SHIFT		1
-#define XSTORM_ETH_CONN_AG_CTX_CF12EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF12EN_SHIFT		2
-#define XSTORM_ETH_CONN_AG_CTX_CF13EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF13EN_SHIFT		3
-#define XSTORM_ETH_CONN_AG_CTX_CF14EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF14EN_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_CF15EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_CF15EN_SHIFT		5
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT 6
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT 7
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF10EN_MASK			0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT			0
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF11EN_MASK			0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF11EN_SHIFT			1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF12EN_MASK			0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF12EN_SHIFT			2
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF13EN_MASK			0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF13EN_SHIFT			3
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF14EN_MASK			0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF14EN_SHIFT			4
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF15EN_MASK			0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_CF15EN_SHIFT			5
+#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT	7
 	u8 flags10;
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT	1
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT	2
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED11_SHIFT		3
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_SHIFT	4
-#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK 0x1
-#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT 5
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED12_SHIFT		6
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED13_SHIFT		7
+#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_MASK			0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_DQ_CF_EN_SHIFT		0
+#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT		1
+#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT		2
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED11_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED11_SHIFT		3
+#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_SLOW_PATH_EN_SHIFT		4
+#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT	5
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED12_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED12_SHIFT		6
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED13_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED13_SHIFT		7
 	u8 flags11;
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED14_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RESERVED15_SHIFT		1
-#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT	2
-#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT		3
-#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT		5
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
-#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE9EN_SHIFT		7
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED14_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED14_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED15_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RESERVED15_SHIFT	1
+#define E4_XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT		3
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT		4
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT		5
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE9EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE9EN_SHIFT		7
 	u8 flags12;
-#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE10EN_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE11EN_SHIFT		1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
-#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE14EN_SHIFT		4
-#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE15EN_SHIFT		5
-#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE16EN_SHIFT		6
-#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE17EN_SHIFT		7
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE10EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE10EN_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE11EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE11EN_SHIFT	1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE14EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE14EN_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE15EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE15EN_SHIFT	5
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE16EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE16EN_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE17EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE17EN_SHIFT	7
 	u8 flags13;
-#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE18EN_SHIFT		0
-#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_MASK		0x1
-#define XSTORM_ETH_CONN_AG_CTX_RULE19EN_SHIFT		1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE18EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE18EN_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE19EN_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_RULE19EN_SHIFT	1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
 	u8 flags14;
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT	0
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT	1
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK 0x1
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT 2
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK 0x1
-#define XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT 3
-#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT	4
-#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK	0x1
-#define XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT	5
-#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_MASK		0x3
-#define XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_SHIFT		6
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT	0
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT	1
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT	2
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK	0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT	3
+#define E4_XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT		4
+#define E4_XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK		0x1
+#define E4_XSTORM_ETH_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT	5
+#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_MASK		0x3
+#define E4_XSTORM_ETH_CONN_AG_CTX_TPH_ENABLE_SHIFT		6
 	u8 edpm_event_id;
 	__le16 physical_q0;
-	__le16 ereserved1;
+	__le16 e5_reserved1;
 	__le16 edpm_num_bds;
 	__le16 tx_bd_cons;
 	__le16 tx_bd_prod;
@@ -4681,7 +5180,7 @@ struct xstorm_eth_conn_ag_ctx {
 	u8 byte13;
 	u8 byte14;
 	u8 byte15;
-	u8 ereserved;
+	u8 e5_reserved;
 	__le16 word11;
 	__le32 reg10;
 	__le32 reg11;
@@ -4704,37 +5203,37 @@ struct ystorm_eth_conn_st_ctx {
 	__le32 reserved[8];
 };
 
-struct ystorm_eth_conn_ag_ctx {
+struct e4_ystorm_eth_conn_ag_ctx {
 	u8 byte0;
 	u8 state;
 	u8 flags0;
-#define YSTORM_ETH_CONN_AG_CTX_BIT0_MASK		0x1
-#define YSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT		0
-#define YSTORM_ETH_CONN_AG_CTX_BIT1_MASK		0x1
-#define YSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT		1
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK	0x3
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT	2
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_MASK	0x3
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_SHIFT	4
-#define YSTORM_ETH_CONN_AG_CTX_CF2_MASK			0x3
-#define YSTORM_ETH_CONN_AG_CTX_CF2_SHIFT		6
+#define E4_YSTORM_ETH_CONN_AG_CTX_BIT0_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT			0
+#define E4_YSTORM_ETH_CONN_AG_CTX_BIT1_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT			1
+#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK	0x3
+#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT	2
+#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_MASK		0x3
+#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_SHIFT	4
+#define E4_YSTORM_ETH_CONN_AG_CTX_CF2_MASK			0x3
+#define E4_YSTORM_ETH_CONN_AG_CTX_CF2_SHIFT			6
 	u8 flags1;
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK	0x1
-#define YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT	0
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_MASK		0x1
-#define YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_SHIFT	1
-#define YSTORM_ETH_CONN_AG_CTX_CF2EN_MASK			0x1
-#define YSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT			2
-#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK			0x1
-#define YSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT			3
-#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK			0x1
-#define YSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT			4
-#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK			0x1
-#define YSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT			5
-#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK			0x1
-#define YSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT			6
-#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK			0x1
-#define YSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT			7
+#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK	0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT	0
+#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_MASK	0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_PMD_TERMINATE_CF_EN_SHIFT	1
+#define E4_YSTORM_ETH_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT			2
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT			3
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT			4
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT			5
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT			6
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK			0x1
+#define E4_YSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT			7
 	u8 tx_q0_int_coallecing_timeset;
 	u8 byte3;
 	__le16 word0;
@@ -4748,89 +5247,89 @@ struct ystorm_eth_conn_ag_ctx {
 	__le32 reg3;
 };
 
-struct tstorm_eth_conn_ag_ctx {
+struct e4_tstorm_eth_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define TSTORM_ETH_CONN_AG_CTX_BIT0_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT		0
-#define TSTORM_ETH_CONN_AG_CTX_BIT1_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT		1
-#define TSTORM_ETH_CONN_AG_CTX_BIT2_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_BIT2_SHIFT		2
-#define TSTORM_ETH_CONN_AG_CTX_BIT3_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_BIT3_SHIFT		3
-#define TSTORM_ETH_CONN_AG_CTX_BIT4_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_BIT4_SHIFT		4
-#define TSTORM_ETH_CONN_AG_CTX_BIT5_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_BIT5_SHIFT		5
-#define TSTORM_ETH_CONN_AG_CTX_CF0_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF0_SHIFT		6
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT2_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT2_SHIFT	2
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT3_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT3_SHIFT	3
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT4_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT4_SHIFT	4
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT5_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_BIT5_SHIFT	5
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF0_SHIFT	6
 	u8 flags1;
-#define TSTORM_ETH_CONN_AG_CTX_CF1_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF1_SHIFT		0
-#define TSTORM_ETH_CONN_AG_CTX_CF2_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF2_SHIFT		2
-#define TSTORM_ETH_CONN_AG_CTX_CF3_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF3_SHIFT		4
-#define TSTORM_ETH_CONN_AG_CTX_CF4_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF4_SHIFT		6
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF1_SHIFT	0
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF2_SHIFT	2
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF3_SHIFT	4
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF4_SHIFT	6
 	u8 flags2;
-#define TSTORM_ETH_CONN_AG_CTX_CF5_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF5_SHIFT		0
-#define TSTORM_ETH_CONN_AG_CTX_CF6_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF6_SHIFT		2
-#define TSTORM_ETH_CONN_AG_CTX_CF7_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF7_SHIFT		4
-#define TSTORM_ETH_CONN_AG_CTX_CF8_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF8_SHIFT		6
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF5_SHIFT	0
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF6_SHIFT	2
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF7_SHIFT	4
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF8_SHIFT	6
 	u8 flags3;
-#define TSTORM_ETH_CONN_AG_CTX_CF9_MASK			0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF9_SHIFT		0
-#define TSTORM_ETH_CONN_AG_CTX_CF10_MASK		0x3
-#define TSTORM_ETH_CONN_AG_CTX_CF10_SHIFT		2
-#define TSTORM_ETH_CONN_AG_CTX_CF0EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT		4
-#define TSTORM_ETH_CONN_AG_CTX_CF1EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT		5
-#define TSTORM_ETH_CONN_AG_CTX_CF2EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT		6
-#define TSTORM_ETH_CONN_AG_CTX_CF3EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT		7
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF9_SHIFT	0
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF10_SHIFT	2
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT	4
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT	5
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT	6
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF3EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT	7
 	u8 flags4;
-#define TSTORM_ETH_CONN_AG_CTX_CF4EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT		0
-#define TSTORM_ETH_CONN_AG_CTX_CF5EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT		1
-#define TSTORM_ETH_CONN_AG_CTX_CF6EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT		2
-#define TSTORM_ETH_CONN_AG_CTX_CF7EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT		3
-#define TSTORM_ETH_CONN_AG_CTX_CF8EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT		4
-#define TSTORM_ETH_CONN_AG_CTX_CF9EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT		5
-#define TSTORM_ETH_CONN_AG_CTX_CF10EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT		6
-#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT		7
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF4EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF4EN_SHIFT	0
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF5EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF5EN_SHIFT	1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF6EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF6EN_SHIFT	2
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF7EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF7EN_SHIFT	3
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF8EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF8EN_SHIFT	4
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF9EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF9EN_SHIFT	5
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF10EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_CF10EN_SHIFT	6
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT	7
 	u8 flags5;
-#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT		0
-#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT		1
-#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT		2
-#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT		3
-#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT		4
-#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_SHIFT		5
-#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT		6
-#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_MASK		0x1
-#define TSTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT		7
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT		0
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT		1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT		2
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT		3
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT		4
+#define E4_TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RX_BD_EN_SHIFT	5
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT		6
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_TSTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT		7
 	__le32 reg0;
 	__le32 reg1;
 	__le32 reg2;
@@ -4852,63 +5351,63 @@ struct tstorm_eth_conn_ag_ctx {
 	__le32 reg10;
 };
 
-struct ustorm_eth_conn_ag_ctx {
+struct e4_ustorm_eth_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define USTORM_ETH_CONN_AG_CTX_BIT0_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_BIT0_SHIFT			0
-#define USTORM_ETH_CONN_AG_CTX_BIT1_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_BIT1_SHIFT			1
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_MASK		0x3
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_SHIFT	2
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_MASK		0x3
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_SHIFT	4
-#define USTORM_ETH_CONN_AG_CTX_CF2_MASK				0x3
-#define USTORM_ETH_CONN_AG_CTX_CF2_SHIFT			6
+#define E4_USTORM_ETH_CONN_AG_CTX_BIT0_MASK			0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_BIT0_SHIFT			0
+#define E4_USTORM_ETH_CONN_AG_CTX_BIT1_MASK			0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_BIT1_SHIFT			1
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_MASK	0x3
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_SHIFT	2
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_MASK	0x3
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_SHIFT	4
+#define E4_USTORM_ETH_CONN_AG_CTX_CF2_MASK			0x3
+#define E4_USTORM_ETH_CONN_AG_CTX_CF2_SHIFT			6
 	u8 flags1;
-#define USTORM_ETH_CONN_AG_CTX_CF3_MASK				0x3
-#define USTORM_ETH_CONN_AG_CTX_CF3_SHIFT			0
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_MASK			0x3
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_SHIFT			2
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_MASK			0x3
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_SHIFT			4
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK		0x3
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT		6
+#define E4_USTORM_ETH_CONN_AG_CTX_CF3_MASK			0x3
+#define E4_USTORM_ETH_CONN_AG_CTX_CF3_SHIFT			0
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_MASK		0x3
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_SHIFT		2
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_MASK		0x3
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_SHIFT		4
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_MASK	0x3
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_SHIFT	6
 	u8 flags2;
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_MASK	0x1
-#define USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_SHIFT	0
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_MASK	0x1
-#define USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_SHIFT	1
-#define USTORM_ETH_CONN_AG_CTX_CF2EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT			2
-#define USTORM_ETH_CONN_AG_CTX_CF3EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT			3
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_MASK		0x1
-#define USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_SHIFT		4
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_MASK		0x1
-#define USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_SHIFT		5
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK	0x1
-#define USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT	6
-#define USTORM_ETH_CONN_AG_CTX_RULE0EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT			7
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_PMD_TERMINATE_CF_EN_SHIFT	0
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_PMD_TERMINATE_CF_EN_SHIFT	1
+#define E4_USTORM_ETH_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT			2
+#define E4_USTORM_ETH_CONN_AG_CTX_CF3EN_MASK			0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_CF3EN_SHIFT			3
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_MASK		0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_ARM_CF_EN_SHIFT		4
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_MASK		0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RX_ARM_CF_EN_SHIFT		5
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_TX_BD_CONS_UPD_CF_EN_SHIFT	6
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE0EN_MASK			0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT			7
 	u8 flags3;
-#define USTORM_ETH_CONN_AG_CTX_RULE1EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT			0
-#define USTORM_ETH_CONN_AG_CTX_RULE2EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT			1
-#define USTORM_ETH_CONN_AG_CTX_RULE3EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT			2
-#define USTORM_ETH_CONN_AG_CTX_RULE4EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT			3
-#define USTORM_ETH_CONN_AG_CTX_RULE5EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT			4
-#define USTORM_ETH_CONN_AG_CTX_RULE6EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT			5
-#define USTORM_ETH_CONN_AG_CTX_RULE7EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT			6
-#define USTORM_ETH_CONN_AG_CTX_RULE8EN_MASK			0x1
-#define USTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT			7
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE8EN_MASK	0x1
+#define E4_USTORM_ETH_CONN_AG_CTX_RULE8EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -4932,20 +5431,21 @@ struct mstorm_eth_conn_st_ctx {
 };
 
 /* eth connection context */
-struct eth_conn_context {
+struct e4_eth_conn_context {
 	struct tstorm_eth_conn_st_ctx tstorm_st_context;
 	struct regpair tstorm_st_padding[2];
 	struct pstorm_eth_conn_st_ctx pstorm_st_context;
 	struct xstorm_eth_conn_st_ctx xstorm_st_context;
-	struct xstorm_eth_conn_ag_ctx xstorm_ag_context;
+	struct e4_xstorm_eth_conn_ag_ctx xstorm_ag_context;
 	struct ystorm_eth_conn_st_ctx ystorm_st_context;
-	struct ystorm_eth_conn_ag_ctx ystorm_ag_context;
-	struct tstorm_eth_conn_ag_ctx tstorm_ag_context;
-	struct ustorm_eth_conn_ag_ctx ustorm_ag_context;
+	struct e4_ystorm_eth_conn_ag_ctx ystorm_ag_context;
+	struct e4_tstorm_eth_conn_ag_ctx tstorm_ag_context;
+	struct e4_ustorm_eth_conn_ag_ctx ustorm_ag_context;
 	struct ustorm_eth_conn_st_ctx ustorm_st_context;
 	struct mstorm_eth_conn_st_ctx mstorm_st_context;
 };
 
+/* Ethernet filter types: mac/vlan/pair */
 enum eth_error_code {
 	ETH_OK = 0x00,
 	ETH_FILTERS_MAC_ADD_FAIL_FULL,
@@ -4972,6 +5472,7 @@ enum eth_error_code {
 	MAX_ETH_ERROR_CODE
 };
 
+/* Opcodes for the event ring */
 enum eth_event_opcode {
 	ETH_EVENT_UNUSED,
 	ETH_EVENT_VPORT_START,
@@ -4983,13 +5484,14 @@ enum eth_event_opcode {
 	ETH_EVENT_RX_QUEUE_UPDATE,
 	ETH_EVENT_RX_QUEUE_STOP,
 	ETH_EVENT_FILTERS_UPDATE,
-	ETH_EVENT_RESERVED,
-	ETH_EVENT_RESERVED2,
-	ETH_EVENT_RESERVED3,
+	ETH_EVENT_RX_ADD_OPENFLOW_FILTER,
+	ETH_EVENT_RX_DELETE_OPENFLOW_FILTER,
+	ETH_EVENT_RX_CREATE_OPENFLOW_ACTION,
 	ETH_EVENT_RX_ADD_UDP_FILTER,
 	ETH_EVENT_RX_DELETE_UDP_FILTER,
-	ETH_EVENT_RESERVED4,
-	ETH_EVENT_RESERVED5,
+	ETH_EVENT_RX_CREATE_GFT_ACTION,
+	ETH_EVENT_RX_GFT_UPDATE_FILTER,
+	ETH_EVENT_TX_QUEUE_UPDATE,
 	MAX_ETH_EVENT_OPCODE
 };
 
@@ -5039,6 +5541,7 @@ enum eth_filter_type {
 	MAX_ETH_FILTER_TYPE
 };
 
+/* Eth IPv4 Fragment Type */
 enum eth_ipv4_frag_type {
 	ETH_IPV4_NOT_FRAG,
 	ETH_IPV4_FIRST_FRAG,
@@ -5046,12 +5549,14 @@ enum eth_ipv4_frag_type {
 	MAX_ETH_IPV4_FRAG_TYPE
 };
 
+/* eth IPv4 Fragment Type */
 enum eth_ip_type {
 	ETH_IPV4,
 	ETH_IPV6,
 	MAX_ETH_IP_TYPE
 };
 
+/* Ethernet Ramrod Command IDs */
 enum eth_ramrod_cmd_id {
 	ETH_RAMROD_UNUSED,
 	ETH_RAMROD_VPORT_START,
@@ -5070,10 +5575,11 @@ enum eth_ramrod_cmd_id {
 	ETH_RAMROD_RX_DELETE_UDP_FILTER,
 	ETH_RAMROD_RX_CREATE_GFT_ACTION,
 	ETH_RAMROD_GFT_UPDATE_FILTER,
+	ETH_RAMROD_TX_QUEUE_UPDATE,
 	MAX_ETH_RAMROD_CMD_ID
 };
 
-/* return code from eth sp ramrods */
+/* Return code from eth sp ramrods */
 struct eth_return_code {
 	u8 value;
 #define ETH_RETURN_CODE_ERR_CODE_MASK	0x1F
@@ -5209,18 +5715,14 @@ struct eth_vport_tx_mode {
 	__le16 reserved2[3];
 };
 
+/* GFT filter update action type */
 enum gft_filter_update_action {
 	GFT_ADD_FILTER,
 	GFT_DELETE_FILTER,
 	MAX_GFT_FILTER_UPDATE_ACTION
 };
 
-enum gft_logic_filter_type {
-	GFT_FILTER_TYPE,
-	RFS_FILTER_TYPE,
-	MAX_GFT_LOGIC_FILTER_TYPE
-};
-
+/* Ramrod data for rx add openflow filter */
 struct rx_add_openflow_filter_data {
 	__le16 action_icid;
 	u8 priority;
@@ -5244,11 +5746,13 @@ struct rx_add_openflow_filter_data {
 	__le16 l4_src_port;
 };
 
+/* Ramrod data for rx create gft action */
 struct rx_create_gft_action_data {
 	u8 vport_id;
 	u8 reserved[7];
 };
 
+/* Ramrod data for rx create openflow action */
 struct rx_create_openflow_action_data {
 	u8 vport_id;
 	u8 reserved[7];
@@ -5286,7 +5790,7 @@ struct rx_queue_start_ramrod_data {
 	struct regpair reserved2;
 };
 
-/* Ramrod data for rx queue start ramrod */
+/* Ramrod data for rx queue stop ramrod */
 struct rx_queue_stop_ramrod_data {
 	__le16 rx_queue_id;
 	u8 complete_cqe_flg;
@@ -5324,14 +5828,22 @@ struct rx_udp_filter_data {
 	__le32 tenant_id;
 };
 
+/* Add or delete GFT filter - filter is packet header of type of packet wished
+ * to pass certain FW flow.
+ */
 struct rx_update_gft_filter_data {
 	struct regpair pkt_hdr_addr;
 	__le16 pkt_hdr_length;
-	__le16 rx_qid_or_action_icid;
-	u8 vport_id;
-	u8 filter_type;
+	__le16 action_icid;
+	__le16 rx_qid;
+	__le16 flow_id;
+	__le16 vport_id;
+	u8 action_icid_valid;
+	u8 rx_qid_valid;
+	u8 flow_id_valid;
 	u8 filter_action;
 	u8 assert_on_error;
+	u8 reserved;
 };
 
 /* Ramrod data for rx queue start ramrod */
@@ -5377,6 +5889,14 @@ struct tx_queue_stop_ramrod_data {
 	__le16 reserved[4];
 };
 
+/* Ramrod data for tx queue update ramrod */
+struct tx_queue_update_ramrod_data {
+	__le16 update_qm_pq_id_flg;
+	__le16 qm_pq_id;
+	__le32 reserved0;
+	struct regpair reserved1[5];
+};
+
 /* Ramrod data for vport update ramrod */
 struct vport_filter_update_ramrod_data {
 	struct eth_filter_cmd_header filter_cmd_hdr;
@@ -5477,219 +5997,219 @@ struct vport_update_ramrod_data {
 	struct eth_vport_rss_config rss_config;
 };
 
-struct xstorm_eth_conn_agctxdq_ext_ldpart {
+struct e4_xstorm_eth_conn_ag_ctx_dq_ext_ldpart {
 	u8 reserved0;
-	u8 eth_state;
+	u8 state;
 	u8 flags0;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_SHIFT           0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_SHIFT              1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_SHIFT              2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_SHIFT           3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_SHIFT              4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_SHIFT              5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_SHIFT              6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_SHIFT              7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM0_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED1_SHIFT		1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED2_SHIFT		2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EXIST_IN_QM3_SHIFT	3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED3_SHIFT		4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED4_SHIFT		5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED5_SHIFT		6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED6_SHIFT		7
 	u8 flags1;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_SHIFT              0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_SHIFT              1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_SHIFT              2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_SHIFT                  3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT12_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT12_SHIFT                  4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT13_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_BIT13_SHIFT                  5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_SHIFT         6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_SHIFT           7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED7_SHIFT		0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED8_SHIFT		1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED9_SHIFT		2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_BIT11_SHIFT		3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_E5_RESERVED2_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_E5_RESERVED2_SHIFT	4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_E5_RESERVED3_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_E5_RESERVED3_SHIFT	5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_RULE_ACTIVE_SHIFT	6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_ACTIVE_SHIFT	7
 	u8 flags2;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0_SHIFT                    0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1_SHIFT                    2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2_SHIFT                    4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3_SHIFT                    6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2_SHIFT	4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4_SHIFT                    0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5_SHIFT                    2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6_SHIFT                    4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7_SHIFT                    6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6_SHIFT	4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7_SHIFT	6
 	u8 flags4;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8_SHIFT                    0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9_SHIFT                    2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10_SHIFT                   4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11_SHIFT                   6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10_SHIFT	4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12_SHIFT                   0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13_SHIFT                   2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14_SHIFT                   4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15_SHIFT                   6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14_SHIFT	4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15_SHIFT	6
 	u8 flags6;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_SHIFT       0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_SHIFT       2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_SHIFT                  4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_SHIFT           6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_MASK		0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_SHIFT		4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_MASK	0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_SHIFT	6
 	u8 flags7;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_SHIFT               0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_SHIFT             2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_SHIFT              4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_SHIFT                  6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_SHIFT                  7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_MASK		0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_SHIFT		0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_MASK		0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED10_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_MASK		0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_SHIFT		4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF0EN_SHIFT		6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF1EN_SHIFT		7
 	u8 flags8;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_SHIFT                  0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_SHIFT                  1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_SHIFT                  2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_SHIFT                  3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_SHIFT                  4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_SHIFT                  5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_SHIFT                  6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_SHIFT                  7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF2EN_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF3EN_SHIFT	1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF4EN_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF5EN_SHIFT	3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF6EN_SHIFT	4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF7EN_SHIFT	5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF8EN_SHIFT	6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF9EN_SHIFT	7
 	u8 flags9;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_SHIFT                 0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_SHIFT                 1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_SHIFT                 2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_SHIFT                 3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_SHIFT                 4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_SHIFT                 5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_SHIFT    6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_SHIFT    7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF10EN_SHIFT			0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF11EN_SHIFT			1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF12EN_SHIFT			2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF13EN_SHIFT			3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF14EN_SHIFT			4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_CF15EN_SHIFT			5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_GO_TO_BD_CONS_CF_EN_SHIFT	6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_MULTI_UNICAST_CF_EN_SHIFT	7
 	u8 flags10;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_SHIFT               0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_SHIFT        1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_SHIFT            2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_SHIFT             3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_SHIFT           4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_SHIFT 5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_SHIFT             6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_SHIFT             7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_DQ_CF_EN_SHIFT			0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TERMINATE_CF_EN_SHIFT		1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_FLUSH_Q0_EN_SHIFT		2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED11_SHIFT		3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_SLOW_PATH_EN_SHIFT		4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_EN_RESERVED_SHIFT	5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED12_SHIFT		6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_MASK			0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED13_SHIFT		7
 	u8 flags11;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_SHIFT             0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_SHIFT             1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_SHIFT         2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_SHIFT                3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_SHIFT                4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_SHIFT                5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_SHIFT           6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_SHIFT                7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED14_SHIFT	0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RESERVED15_SHIFT	1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TX_DEC_RULE_EN_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE5EN_SHIFT		3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE6EN_SHIFT		4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE7EN_SHIFT		5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED1_SHIFT	6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE9EN_SHIFT		7
 	u8 flags12;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_SHIFT               0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_SHIFT               1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_SHIFT           2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_SHIFT           3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_SHIFT               4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_SHIFT               5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_SHIFT               6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_SHIFT               7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE10EN_SHIFT		0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE11EN_SHIFT		1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED2_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED3_SHIFT	3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE14EN_SHIFT		4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE15EN_SHIFT		5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE16EN_SHIFT		6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE17EN_SHIFT		7
 	u8 flags13;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_SHIFT               0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_SHIFT               1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_SHIFT           2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_SHIFT           3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_SHIFT           4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_SHIFT           5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_SHIFT           6
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_SHIFT           7
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE18EN_SHIFT		0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_RULE19EN_SHIFT		1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED4_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED5_SHIFT	3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED6_SHIFT	4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED7_SHIFT	5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED8_SHIFT	6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_A0_RESERVED9_SHIFT	7
 	u8 flags14;
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_SHIFT       0
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_SHIFT     1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_SHIFT   2
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_SHIFT   3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_SHIFT         4
-#define XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_MASK	0x1
-#define XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_SHIFT       5
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_MASK	0x3
-#define XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_SHIFT             6
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_USE_EXT_HDR_SHIFT		0
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_RAW_L3L4_SHIFT	1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_INBAND_PROP_HDR_SHIFT	2
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_MASK	0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_EDPM_SEND_EXT_TUNNEL_SHIFT	3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_L2_EDPM_ENABLE_SHIFT		4
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_MASK		0x1
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_SHIFT		5
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_MASK			0x3
+#define E4XSTORMETHCONNAGCTXDQEXTLDPART_TPH_ENABLE_SHIFT		6
 	u8 edpm_event_id;
 	__le16 physical_q0;
-	__le16 ereserved1;
+	__le16 e5_reserved1;
 	__le16 edpm_num_bds;
 	__le16 tx_bd_cons;
 	__le16 tx_bd_prod;
@@ -5706,256 +6226,256 @@ struct xstorm_eth_conn_agctxdq_ext_ldpart {
 	__le32 reg4;
 };
 
-struct mstorm_eth_conn_ag_ctx {
+struct e4_mstorm_eth_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT 0
-#define MSTORM_ETH_CONN_AG_CTX_BIT1_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT         1
-#define MSTORM_ETH_CONN_AG_CTX_CF0_MASK	0x3
-#define MSTORM_ETH_CONN_AG_CTX_CF0_SHIFT          2
-#define MSTORM_ETH_CONN_AG_CTX_CF1_MASK	0x3
-#define MSTORM_ETH_CONN_AG_CTX_CF1_SHIFT          4
-#define MSTORM_ETH_CONN_AG_CTX_CF2_MASK	0x3
-#define MSTORM_ETH_CONN_AG_CTX_CF2_SHIFT          6
+#define E4_MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	 0
+#define E4_MSTORM_ETH_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF0_SHIFT		2
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF1_MASK		0x3
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF1_SHIFT		4
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF2_MASK		0x3
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF2_SHIFT		6
 	u8 flags1;
-#define MSTORM_ETH_CONN_AG_CTX_CF0EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT        0
-#define MSTORM_ETH_CONN_AG_CTX_CF1EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT        1
-#define MSTORM_ETH_CONN_AG_CTX_CF2EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT        2
-#define MSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT      3
-#define MSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT      4
-#define MSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT      5
-#define MSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT      6
-#define MSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define MSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT      7
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF0EN_SHIFT	0
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF1EN_SHIFT	1
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_CF2EN_SHIFT	2
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_MSTORM_ETH_CONN_AG_CTX_RULE4EN_SHIFT	7
 	__le16 word0;
 	__le16 word1;
 	__le32 reg0;
 	__le32 reg1;
 };
 
-struct xstorm_eth_hw_conn_ag_ctx {
+struct e4_xstorm_eth_hw_conn_ag_ctx {
 	u8 reserved0;
-	u8 eth_state;
+	u8 state;
 	u8 flags0;
-#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_SHIFT           0
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_SHIFT              1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_SHIFT              2
-#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_SHIFT           3
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_SHIFT              4
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_SHIFT              5
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_SHIFT              6
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_SHIFT              7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED1_SHIFT	1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED2_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED3_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED4_SHIFT	5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED5_SHIFT	6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED6_SHIFT	7
 	u8 flags1;
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_SHIFT              0
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_SHIFT              1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_SHIFT              2
-#define XSTORM_ETH_HW_CONN_AG_CTX_BIT11_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_BIT11_SHIFT                  3
-#define XSTORM_ETH_HW_CONN_AG_CTX_BIT12_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_BIT12_SHIFT                  4
-#define XSTORM_ETH_HW_CONN_AG_CTX_BIT13_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_BIT13_SHIFT                  5
-#define XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT         6
-#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT           7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED7_SHIFT		0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED8_SHIFT		1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED9_SHIFT		2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT11_MASK			0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_E5_RESERVED2_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_E5_RESERVED2_SHIFT		4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_E5_RESERVED3_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_E5_RESERVED3_SHIFT		5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_RULE_ACTIVE_SHIFT	6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_ACTIVE_SHIFT		7
 	u8 flags2;
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF0_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF0_SHIFT                    0
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF1_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF1_SHIFT                    2
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF2_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF2_SHIFT                    4
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF3_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF3_SHIFT                    6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF4_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF4_SHIFT                    0
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF5_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF5_SHIFT                    2
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF6_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF6_SHIFT                    4
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF7_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF7_SHIFT                    6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7_SHIFT	6
 	u8 flags4;
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF8_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF8_SHIFT                    0
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF9_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF9_SHIFT                    2
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF10_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF10_SHIFT                   4
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF11_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF11_SHIFT                   6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF12_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF12_SHIFT                   0
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF13_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF13_SHIFT                   2
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF14_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF14_SHIFT                   4
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF15_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF15_SHIFT                   6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15_SHIFT	6
 	u8 flags6;
-#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT       0
-#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT       2
-#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_SHIFT                  4
-#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_SHIFT           6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_MASK			0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_SHIFT		4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_MASK		0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_SHIFT		6
 	u8 flags7;
-#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_SHIFT               0
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_SHIFT             2
-#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_SHIFT              4
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_SHIFT                  6
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_SHIFT                  7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED10_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF0EN_SHIFT	6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF1EN_SHIFT	7
 	u8 flags8;
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_SHIFT                  0
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_SHIFT                  1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_SHIFT                  2
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_SHIFT                  3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_SHIFT                  4
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_SHIFT                  5
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_SHIFT                  6
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_SHIFT                  7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF2EN_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF3EN_SHIFT	1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF4EN_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF5EN_SHIFT	3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF6EN_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF7EN_SHIFT	5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF8EN_SHIFT	6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF9EN_SHIFT	7
 	u8 flags9;
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_SHIFT                 0
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_SHIFT                 1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_SHIFT                 2
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_SHIFT                 3
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_SHIFT                 4
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_SHIFT                 5
-#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT    6
-#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT    7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF10EN_SHIFT		0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF11EN_SHIFT		1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF12EN_SHIFT		2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF13EN_SHIFT		3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF14EN_SHIFT		4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_CF15EN_SHIFT		5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_GO_TO_BD_CONS_CF_EN_SHIFT	6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_MULTI_UNICAST_CF_EN_SHIFT	7
 	u8 flags10;
-#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_SHIFT               0
-#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT        1
-#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT            2
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_SHIFT             3
-#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_SHIFT           4
-#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT 5
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_SHIFT             6
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_SHIFT             7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_MASK			0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_DQ_CF_EN_SHIFT			0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TERMINATE_CF_EN_SHIFT		1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_MASK			0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT			2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_MASK			0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED11_SHIFT			3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_MASK			0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_SLOW_PATH_EN_SHIFT			4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_EN_RESERVED_SHIFT	5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_MASK			0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED12_SHIFT			6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_MASK			0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED13_SHIFT			7
 	u8 flags11;
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_SHIFT             0
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_SHIFT             1
-#define XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT         2
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_SHIFT                3
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_SHIFT                4
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_SHIFT                5
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_SHIFT           6
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_SHIFT                7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED14_SHIFT		0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RESERVED15_SHIFT		1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TX_DEC_RULE_EN_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE5EN_SHIFT		3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE6EN_SHIFT		4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE7EN_SHIFT		5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED1_SHIFT		6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_MASK		0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE9EN_SHIFT		7
 	u8 flags12;
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_SHIFT               0
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_SHIFT               1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_SHIFT           2
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_SHIFT           3
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_SHIFT               4
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_SHIFT               5
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_SHIFT               6
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_SHIFT               7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE10EN_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE11EN_SHIFT	1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE14EN_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE15EN_SHIFT	5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE16EN_SHIFT	6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE17EN_SHIFT	7
 	u8 flags13;
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_SHIFT               0
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_SHIFT               1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_SHIFT           2
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_SHIFT           3
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_SHIFT           4
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_SHIFT           5
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_SHIFT           6
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_SHIFT           7
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE18EN_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_RULE19EN_SHIFT	1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
 	u8 flags14;
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT       0
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT     1
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT   2
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT   3
-#define XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT         4
-#define XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK	0x1
-#define XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT       5
-#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_MASK	0x3
-#define XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_SHIFT             6
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_USE_EXT_HDR_SHIFT	0
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_RAW_L3L4_SHIFT	1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_INBAND_PROP_HDR_SHIFT	2
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_EDPM_SEND_EXT_TUNNEL_SHIFT	3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_L2_EDPM_ENABLE_SHIFT	4
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK	0x1
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT	5
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_MASK		0x3
+#define E4_XSTORM_ETH_HW_CONN_AG_CTX_TPH_ENABLE_SHIFT		6
 	u8 edpm_event_id;
 	__le16 physical_q0;
-	__le16 ereserved1;
+	__le16 e5_reserved1;
 	__le16 edpm_num_bds;
 	__le16 tx_bd_cons;
 	__le16 tx_bd_prod;
@@ -5963,6 +6483,7 @@ struct xstorm_eth_hw_conn_ag_ctx {
 	__le16 conn_dpi;
 };
 
+/* GFT CAM line struct */
 struct gft_cam_line {
 	__le32 camline;
 #define GFT_CAM_LINE_VALID_MASK		0x1
@@ -5975,6 +6496,7 @@ struct gft_cam_line {
 #define GFT_CAM_LINE_RESERVED1_SHIFT	29
 };
 
+/* GFT CAM line struct with fields breakout */
 struct gft_cam_line_mapped {
 	__le32 camline;
 #define GFT_CAM_LINE_MAPPED_VALID_MASK				0x1
@@ -6008,28 +6530,31 @@ union gft_cam_line_union {
 	struct gft_cam_line_mapped cam_line_mapped;
 };
 
+/* Used in gft_profile_key: Indication for ip version */
 enum gft_profile_ip_version {
 	GFT_PROFILE_IPV4 = 0,
 	GFT_PROFILE_IPV6 = 1,
 	MAX_GFT_PROFILE_IP_VERSION
 };
 
+/* Profile key stucr fot GFT logic in Prs */
 struct gft_profile_key {
 	__le16 profile_key;
-#define GFT_PROFILE_KEY_IP_VERSION_MASK           0x1
-#define GFT_PROFILE_KEY_IP_VERSION_SHIFT          0
-#define GFT_PROFILE_KEY_TUNNEL_IP_VERSION_MASK    0x1
-#define GFT_PROFILE_KEY_TUNNEL_IP_VERSION_SHIFT   1
-#define GFT_PROFILE_KEY_UPPER_PROTOCOL_TYPE_MASK  0xF
-#define GFT_PROFILE_KEY_UPPER_PROTOCOL_TYPE_SHIFT 2
-#define GFT_PROFILE_KEY_TUNNEL_TYPE_MASK          0xF
-#define GFT_PROFILE_KEY_TUNNEL_TYPE_SHIFT         6
-#define GFT_PROFILE_KEY_PF_ID_MASK                0xF
-#define GFT_PROFILE_KEY_PF_ID_SHIFT               10
-#define GFT_PROFILE_KEY_RESERVED0_MASK            0x3
-#define GFT_PROFILE_KEY_RESERVED0_SHIFT           14
-};
-
+#define GFT_PROFILE_KEY_IP_VERSION_MASK			0x1
+#define GFT_PROFILE_KEY_IP_VERSION_SHIFT		0
+#define GFT_PROFILE_KEY_TUNNEL_IP_VERSION_MASK		0x1
+#define GFT_PROFILE_KEY_TUNNEL_IP_VERSION_SHIFT		1
+#define GFT_PROFILE_KEY_UPPER_PROTOCOL_TYPE_MASK	0xF
+#define GFT_PROFILE_KEY_UPPER_PROTOCOL_TYPE_SHIFT	2
+#define GFT_PROFILE_KEY_TUNNEL_TYPE_MASK		0xF
+#define GFT_PROFILE_KEY_TUNNEL_TYPE_SHIFT		6
+#define GFT_PROFILE_KEY_PF_ID_MASK			0xF
+#define GFT_PROFILE_KEY_PF_ID_SHIFT			10
+#define GFT_PROFILE_KEY_RESERVED0_MASK			0x3
+#define GFT_PROFILE_KEY_RESERVED0_SHIFT			14
+};
+
+/* Used in gft_profile_key: Indication for tunnel type */
 enum gft_profile_tunnel_type {
 	GFT_PROFILE_NO_TUNNEL = 0,
 	GFT_PROFILE_VXLAN_TUNNEL = 1,
@@ -6040,6 +6565,7 @@ enum gft_profile_tunnel_type {
 	MAX_GFT_PROFILE_TUNNEL_TYPE
 };
 
+/* Used in gft_profile_key: Indication for protocol type */
 enum gft_profile_upper_protocol_type {
 	GFT_PROFILE_ROCE_PROTOCOL = 0,
 	GFT_PROFILE_RROCE_PROTOCOL = 1,
@@ -6060,6 +6586,7 @@ enum gft_profile_upper_protocol_type {
 	MAX_GFT_PROFILE_UPPER_PROTOCOL_TYPE
 };
 
+/* GFT RAM line struct */
 struct gft_ram_line {
 	__le32 lo;
 #define GFT_RAM_LINE_VLAN_SELECT_MASK			0x3
@@ -6149,6 +6676,7 @@ struct gft_ram_line {
 #define GFT_RAM_LINE_RESERVED1_SHIFT			10
 };
 
+/* Used in the first 2 bits for gft_ram_line: Indication for vlan mask */
 enum gft_vlan_select {
 	INNER_PROVIDER_VLAN = 0,
 	INNER_VLAN = 1,
@@ -6157,10 +6685,205 @@ enum gft_vlan_select {
 	MAX_GFT_VLAN_SELECT
 };
 
+/* The rdma task context of Mstorm */
+struct ystorm_rdma_task_st_ctx {
+	struct regpair temp[4];
+};
+
+struct e4_ystorm_rdma_task_ag_ctx {
+	u8 reserved;
+	u8 byte1;
+	__le16 msem_ctx_upd_seq;
+	u8 flags0;
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK		0xF
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT	0
+#define E4_YSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT		4
+#define E4_YSTORM_RDMA_TASK_AG_CTX_BIT1_MASK			0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT			5
+#define E4_YSTORM_RDMA_TASK_AG_CTX_VALID_MASK			0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_VALID_SHIFT			6
+#define E4_YSTORM_RDMA_TASK_AG_CTX_BIT3_MASK			0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT			7
+	u8 flags1;
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF0_MASK		0x3
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT		0
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF1_MASK		0x3
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT		2
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF2SPECIAL_MASK	0x3
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF2SPECIAL_SHIFT	4
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT		6
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT		7
+	u8 flags2;
+#define E4_YSTORM_RDMA_TASK_AG_CTX_BIT4_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_BIT4_SHIFT		0
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT	1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT	2
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT	3
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT	4
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT	5
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT	6
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK		0x1
+#define E4_YSTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT	7
+	u8 key;
+	__le32 mw_cnt;
+	u8 ref_cnt_seq;
+	u8 ctx_upd_seq;
+	__le16 dif_flags;
+	__le16 tx_ref_count;
+	__le16 last_used_ltid;
+	__le16 parent_mr_lo;
+	__le16 parent_mr_hi;
+	__le32 fbo_lo;
+	__le32 fbo_hi;
+};
+
+struct e4_mstorm_rdma_task_ag_ctx {
+	u8 reserved;
+	u8 byte1;
+	__le16 icid;
+	u8 flags0;
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK		0xF
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT	0
+#define E4_MSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT		4
+#define E4_MSTORM_RDMA_TASK_AG_CTX_BIT1_MASK			0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT			5
+#define E4_MSTORM_RDMA_TASK_AG_CTX_BIT2_MASK			0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_BIT2_SHIFT			6
+#define E4_MSTORM_RDMA_TASK_AG_CTX_BIT3_MASK			0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT			7
+	u8 flags1;
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF0_MASK	0x3
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT	0
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF1_MASK	0x3
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT	2
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF2_MASK	0x3
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF2_SHIFT	4
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK	0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT	6
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK	0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT	7
+	u8 flags2;
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF2EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_CF2EN_SHIFT		0
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT	1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT	2
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT	3
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT	4
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT	5
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT	6
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK		0x1
+#define E4_MSTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT	7
+	u8 key;
+	__le32 mw_cnt;
+	u8 ref_cnt_seq;
+	u8 ctx_upd_seq;
+	__le16 dif_flags;
+	__le16 tx_ref_count;
+	__le16 last_used_ltid;
+	__le16 parent_mr_lo;
+	__le16 parent_mr_hi;
+	__le32 fbo_lo;
+	__le32 fbo_hi;
+};
+
+/* The roce task context of Mstorm */
 struct mstorm_rdma_task_st_ctx {
 	struct regpair temp[4];
 };
 
+/* The roce task context of Ustorm */
+struct ustorm_rdma_task_st_ctx {
+	struct regpair temp[2];
+};
+
+struct e4_ustorm_rdma_task_ag_ctx {
+	u8 reserved;
+	u8 byte1;
+	__le16 icid;
+	u8 flags0;
+#define E4_USTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK		0xF
+#define E4_USTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT	0
+#define E4_USTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT		4
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_RUNT_VALID_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_RUNT_VALID_SHIFT		5
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_MASK	0x3
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_SHIFT	6
+	u8 flags1;
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_RESULT_TOGGLE_BIT_MASK	0x3
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_RESULT_TOGGLE_BIT_SHIFT	0
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_TX_IO_FLG_MASK		0x3
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_TX_IO_FLG_SHIFT		2
+#define E4_USTORM_RDMA_TASK_AG_CTX_CF3_MASK			0x3
+#define E4_USTORM_RDMA_TASK_AG_CTX_CF3_SHIFT			4
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_MASK		0x3
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_SHIFT		6
+	u8 flags2;
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_EN_MASK	0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_EN_SHIFT	0
+#define E4_USTORM_RDMA_TASK_AG_CTX_RESERVED2_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RESERVED2_SHIFT		1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RESERVED3_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RESERVED3_SHIFT		2
+#define E4_USTORM_RDMA_TASK_AG_CTX_CF3EN_MASK			0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_CF3EN_SHIFT			3
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_EN_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_EN_SHIFT	4
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK			0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT		5
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK			0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT		6
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK			0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT		7
+	u8 flags3;
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT	0
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT	1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT	2
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK		0x1
+#define E4_USTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT	3
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_TYPE_MASK	0xF
+#define E4_USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_TYPE_SHIFT	4
+	__le32 dif_err_intervals;
+	__le32 dif_error_1st_interval;
+	__le32 reg2;
+	__le32 dif_runt_value;
+	__le32 reg4;
+	__le32 reg5;
+};
+
+/* RDMA task context */
+struct e4_rdma_task_context {
+	struct ystorm_rdma_task_st_ctx ystorm_st_context;
+	struct e4_ystorm_rdma_task_ag_ctx ystorm_ag_context;
+	struct tdif_task_context tdif_context;
+	struct e4_mstorm_rdma_task_ag_ctx mstorm_ag_context;
+	struct mstorm_rdma_task_st_ctx mstorm_st_context;
+	struct rdif_task_context rdif_context;
+	struct ustorm_rdma_task_st_ctx ustorm_st_context;
+	struct regpair ustorm_st_padding[2];
+	struct e4_ustorm_rdma_task_ag_ctx ustorm_ag_context;
+};
+
+/* rdma function init ramrod data */
 struct rdma_close_func_ramrod_data {
 	u8 cnq_start_offset;
 	u8 num_cnqs;
@@ -6169,6 +6892,7 @@ struct rdma_close_func_ramrod_data {
 	u8 reserved[4];
 };
 
+/* rdma function init CNQ parameters */
 struct rdma_cnq_params {
 	__le16 sb_num;
 	u8 sb_index;
@@ -6179,6 +6903,7 @@ struct rdma_cnq_params {
 	u8 reserved1[6];
 };
 
+/* rdma create cq ramrod data */
 struct rdma_create_cq_ramrod_data {
 	struct regpair cq_handle;
 	struct regpair pbl_addr;
@@ -6193,21 +6918,25 @@ struct rdma_create_cq_ramrod_data {
 	__le16 reserved1;
 };
 
+/* rdma deregister tid ramrod data */
 struct rdma_deregister_tid_ramrod_data {
 	__le32 itid;
 	__le32 reserved;
 };
 
+/* rdma destroy cq output params */
 struct rdma_destroy_cq_output_params {
 	__le16 cnq_num;
 	__le16 reserved0;
 	__le32 reserved1;
 };
 
+/* rdma destroy cq ramrod data */
 struct rdma_destroy_cq_ramrod_data {
 	struct regpair output_params_addr;
 };
 
+/* RDMA slow path EQ cmd IDs */
 enum rdma_event_opcode {
 	RDMA_EVENT_UNUSED,
 	RDMA_EVENT_FUNC_INIT,
@@ -6223,6 +6952,7 @@ enum rdma_event_opcode {
 	MAX_RDMA_EVENT_OPCODE
 };
 
+/* RDMA FW return code for slow path ramrods */
 enum rdma_fw_return_code {
 	RDMA_RETURN_OK = 0,
 	RDMA_RETURN_REGISTER_MR_BAD_STATE_ERR,
@@ -6232,20 +6962,24 @@ enum rdma_fw_return_code {
 	MAX_RDMA_FW_RETURN_CODE
 };
 
+/* rdma function init header */
 struct rdma_init_func_hdr {
 	u8 cnq_start_offset;
 	u8 num_cnqs;
 	u8 cq_ring_mode;
 	u8 vf_id;
 	u8 vf_valid;
-	u8 reserved[3];
+	u8 relaxed_ordering;
+	u8 reserved[2];
 };
 
+/* rdma function init ramrod data */
 struct rdma_init_func_ramrod_data {
 	struct rdma_init_func_hdr params_header;
 	struct rdma_cnq_params cnq_params[NUM_OF_GLOBAL_QUEUES];
 };
 
+/* RDMA ramrod command IDs */
 enum rdma_ramrod_cmd_id {
 	RDMA_RAMROD_UNUSED,
 	RDMA_RAMROD_FUNC_INIT,
@@ -6261,42 +6995,43 @@ enum rdma_ramrod_cmd_id {
 	MAX_RDMA_RAMROD_CMD_ID
 };
 
+/* rdma register tid ramrod data */
 struct rdma_register_tid_ramrod_data {
 	__le16 flags;
 #define RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG_MASK	0x1F
 #define RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG_SHIFT	0
 #define RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL_MASK	0x1
 #define RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL_SHIFT	5
-#define RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED_MASK	0x1
-#define RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED_SHIFT	6
-#define RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR_MASK	0x1
-#define RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR_SHIFT	7
-#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ_MASK	0x1
-#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ_SHIFT	8
-#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE_MASK	0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED_MASK		0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED_SHIFT		6
+#define RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR_MASK		0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR_SHIFT		7
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ_MASK		0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ_SHIFT		8
+#define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE_MASK		0x1
 #define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE_SHIFT	9
 #define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC_MASK	0x1
 #define RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC_SHIFT	10
-#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE_MASK	0x1
-#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE_SHIFT	11
-#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ_MASK	0x1
-#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ_SHIFT	12
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE_MASK		0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE_SHIFT		11
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ_MASK		0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ_SHIFT		12
 #define RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND_MASK	0x1
 #define RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND_SHIFT	13
-#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED_MASK	0x3
-#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED_SHIFT	14
+#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED_MASK		0x3
+#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED_SHIFT		14
 	u8 flags1;
 #define RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG_MASK	0x1F
-#define RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG_SHIFT 0
-#define RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE_MASK	0x7
-#define RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE_SHIFT	5
+#define RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG_SHIFT	0
+#define RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE_MASK		0x7
+#define RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE_SHIFT		5
 	u8 flags2;
-#define RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR_MASK	0x1
-#define RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR_SHIFT	0
+#define RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR_MASK		0x1
+#define RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR_SHIFT		0
 #define RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG_MASK	0x1
 #define RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG_SHIFT	1
-#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED1_MASK	0x3F
-#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED1_SHIFT	2
+#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED1_MASK		0x3F
+#define RDMA_REGISTER_TID_RAMROD_DATA_RESERVED1_SHIFT		2
 	u8 key;
 	u8 length_hi;
 	u8 vf_id;
@@ -6313,19 +7048,21 @@ struct rdma_register_tid_ramrod_data {
 	__le32 reserved4[2];
 };
 
+/* rdma resize cq output params */
 struct rdma_resize_cq_output_params {
 	__le32 old_cq_cons;
 	__le32 old_cq_prod;
 };
 
+/* rdma resize cq ramrod data */
 struct rdma_resize_cq_ramrod_data {
 	u8 flags;
-#define RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK        0x1
-#define RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_SHIFT       0
-#define RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL_MASK  0x1
-#define RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL_SHIFT 1
-#define RDMA_RESIZE_CQ_RAMROD_DATA_RESERVED_MASK          0x3F
-#define RDMA_RESIZE_CQ_RAMROD_DATA_RESERVED_SHIFT         2
+#define RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK		0x1
+#define RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_SHIFT		0
+#define RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL_MASK	0x1
+#define RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL_SHIFT	1
+#define RDMA_RESIZE_CQ_RAMROD_DATA_RESERVED_MASK		0x3F
+#define RDMA_RESIZE_CQ_RAMROD_DATA_RESERVED_SHIFT		2
 	u8 pbl_log_page_size;
 	__le16 pbl_num_pages;
 	__le32 max_cqes;
@@ -6333,10 +7070,12 @@ struct rdma_resize_cq_ramrod_data {
 	struct regpair output_params_addr;
 };
 
+/* The rdma storm context of Mstorm */
 struct rdma_srq_context {
 	struct regpair temp[8];
 };
 
+/* rdma create qp requester ramrod data */
 struct rdma_srq_create_ramrod_data {
 	struct regpair pbl_base_addr;
 	__le16 pages_in_srq_pbl;
@@ -6348,206 +7087,19 @@ struct rdma_srq_create_ramrod_data {
 	struct regpair producers_addr;
 };
 
+/* rdma create qp requester ramrod data */
 struct rdma_srq_destroy_ramrod_data {
 	struct rdma_srq_id srq_id;
 	__le32 reserved;
 };
 
+/* rdma create qp requester ramrod data */
 struct rdma_srq_modify_ramrod_data {
 	struct rdma_srq_id srq_id;
 	__le32 wqe_limit;
 };
 
-struct ystorm_rdma_task_st_ctx {
-	struct regpair temp[4];
-};
-
-struct ystorm_rdma_task_ag_ctx {
-	u8 reserved;
-	u8 byte1;
-	__le16 msem_ctx_upd_seq;
-	u8 flags0;
-#define YSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK  0xF
-#define YSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT 0
-#define YSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK     0x1
-#define YSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT    4
-#define YSTORM_RDMA_TASK_AG_CTX_BIT1_MASK             0x1
-#define YSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT            5
-#define YSTORM_RDMA_TASK_AG_CTX_VALID_MASK            0x1
-#define YSTORM_RDMA_TASK_AG_CTX_VALID_SHIFT           6
-#define YSTORM_RDMA_TASK_AG_CTX_BIT3_MASK             0x1
-#define YSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT            7
-	u8 flags1;
-#define YSTORM_RDMA_TASK_AG_CTX_CF0_MASK              0x3
-#define YSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT             0
-#define YSTORM_RDMA_TASK_AG_CTX_CF1_MASK              0x3
-#define YSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT             2
-#define YSTORM_RDMA_TASK_AG_CTX_CF2SPECIAL_MASK       0x3
-#define YSTORM_RDMA_TASK_AG_CTX_CF2SPECIAL_SHIFT      4
-#define YSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK            0x1
-#define YSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT           6
-#define YSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK            0x1
-#define YSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT           7
-	u8 flags2;
-#define YSTORM_RDMA_TASK_AG_CTX_BIT4_MASK             0x1
-#define YSTORM_RDMA_TASK_AG_CTX_BIT4_SHIFT            0
-#define YSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK          0x1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT         1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK          0x1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT         2
-#define YSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK          0x1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT         3
-#define YSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK          0x1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT         4
-#define YSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK          0x1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT         5
-#define YSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK          0x1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT         6
-#define YSTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK          0x1
-#define YSTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT         7
-	u8 key;
-	__le32 mw_cnt;
-	u8 ref_cnt_seq;
-	u8 ctx_upd_seq;
-	__le16 dif_flags;
-	__le16 tx_ref_count;
-	__le16 last_used_ltid;
-	__le16 parent_mr_lo;
-	__le16 parent_mr_hi;
-	__le32 fbo_lo;
-	__le32 fbo_hi;
-};
-
-struct mstorm_rdma_task_ag_ctx {
-	u8 reserved;
-	u8 byte1;
-	__le16 icid;
-	u8 flags0;
-#define MSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK  0xF
-#define MSTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT 0
-#define MSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK     0x1
-#define MSTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT    4
-#define MSTORM_RDMA_TASK_AG_CTX_BIT1_MASK             0x1
-#define MSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT            5
-#define MSTORM_RDMA_TASK_AG_CTX_BIT2_MASK             0x1
-#define MSTORM_RDMA_TASK_AG_CTX_BIT2_SHIFT            6
-#define MSTORM_RDMA_TASK_AG_CTX_BIT3_MASK             0x1
-#define MSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT            7
-	u8 flags1;
-#define MSTORM_RDMA_TASK_AG_CTX_CF0_MASK              0x3
-#define MSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT             0
-#define MSTORM_RDMA_TASK_AG_CTX_CF1_MASK              0x3
-#define MSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT             2
-#define MSTORM_RDMA_TASK_AG_CTX_CF2_MASK              0x3
-#define MSTORM_RDMA_TASK_AG_CTX_CF2_SHIFT             4
-#define MSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK            0x1
-#define MSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT           6
-#define MSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK            0x1
-#define MSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT           7
-	u8 flags2;
-#define MSTORM_RDMA_TASK_AG_CTX_CF2EN_MASK            0x1
-#define MSTORM_RDMA_TASK_AG_CTX_CF2EN_SHIFT           0
-#define MSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK          0x1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT         1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK          0x1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT         2
-#define MSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK          0x1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT         3
-#define MSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK          0x1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT         4
-#define MSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK          0x1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT         5
-#define MSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK          0x1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT         6
-#define MSTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK          0x1
-#define MSTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT         7
-	u8 key;
-	__le32 mw_cnt;
-	u8 ref_cnt_seq;
-	u8 ctx_upd_seq;
-	__le16 dif_flags;
-	__le16 tx_ref_count;
-	__le16 last_used_ltid;
-	__le16 parent_mr_lo;
-	__le16 parent_mr_hi;
-	__le32 fbo_lo;
-	__le32 fbo_hi;
-};
-
-struct ustorm_rdma_task_st_ctx {
-	struct regpair temp[2];
-};
-
-struct ustorm_rdma_task_ag_ctx {
-	u8 reserved;
-	u8 byte1;
-	__le16 icid;
-	u8 flags0;
-#define USTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_MASK         0xF
-#define USTORM_RDMA_TASK_AG_CTX_CONNECTION_TYPE_SHIFT        0
-#define USTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_MASK            0x1
-#define USTORM_RDMA_TASK_AG_CTX_EXIST_IN_QM0_SHIFT           4
-#define USTORM_RDMA_TASK_AG_CTX_DIF_RUNT_VALID_MASK          0x1
-#define USTORM_RDMA_TASK_AG_CTX_DIF_RUNT_VALID_SHIFT         5
-#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_MASK     0x3
-#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_SHIFT    6
-	u8 flags1;
-#define USTORM_RDMA_TASK_AG_CTX_DIF_RESULT_TOGGLE_BIT_MASK   0x3
-#define USTORM_RDMA_TASK_AG_CTX_DIF_RESULT_TOGGLE_BIT_SHIFT  0
-#define USTORM_RDMA_TASK_AG_CTX_DIF_TX_IO_FLG_MASK           0x3
-#define USTORM_RDMA_TASK_AG_CTX_DIF_TX_IO_FLG_SHIFT          2
-#define USTORM_RDMA_TASK_AG_CTX_CF3_MASK                     0x3
-#define USTORM_RDMA_TASK_AG_CTX_CF3_SHIFT                    4
-#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_MASK            0x3
-#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_SHIFT           6
-	u8 flags2;
-#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_EN_MASK  0x1
-#define USTORM_RDMA_TASK_AG_CTX_DIF_WRITE_RESULT_CF_EN_SHIFT 0
-#define USTORM_RDMA_TASK_AG_CTX_RESERVED2_MASK               0x1
-#define USTORM_RDMA_TASK_AG_CTX_RESERVED2_SHIFT              1
-#define USTORM_RDMA_TASK_AG_CTX_RESERVED3_MASK               0x1
-#define USTORM_RDMA_TASK_AG_CTX_RESERVED3_SHIFT              2
-#define USTORM_RDMA_TASK_AG_CTX_CF3EN_MASK                   0x1
-#define USTORM_RDMA_TASK_AG_CTX_CF3EN_SHIFT                  3
-#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_EN_MASK         0x1
-#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_CF_EN_SHIFT        4
-#define USTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK                 0x1
-#define USTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT                5
-#define USTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK                 0x1
-#define USTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT                6
-#define USTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK                 0x1
-#define USTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT                7
-	u8 flags3;
-#define USTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK                 0x1
-#define USTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT                0
-#define USTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK                 0x1
-#define USTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT                1
-#define USTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK                 0x1
-#define USTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT                2
-#define USTORM_RDMA_TASK_AG_CTX_RULE6EN_MASK                 0x1
-#define USTORM_RDMA_TASK_AG_CTX_RULE6EN_SHIFT                3
-#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_TYPE_MASK          0xF
-#define USTORM_RDMA_TASK_AG_CTX_DIF_ERROR_TYPE_SHIFT         4
-	__le32 dif_err_intervals;
-	__le32 dif_error_1st_interval;
-	__le32 reg2;
-	__le32 dif_runt_value;
-	__le32 reg4;
-	__le32 reg5;
-};
-
-struct rdma_task_context {
-	struct ystorm_rdma_task_st_ctx ystorm_st_context;
-	struct ystorm_rdma_task_ag_ctx ystorm_ag_context;
-	struct tdif_task_context tdif_context;
-	struct mstorm_rdma_task_ag_ctx mstorm_ag_context;
-	struct mstorm_rdma_task_st_ctx mstorm_st_context;
-	struct rdif_task_context rdif_context;
-	struct ustorm_rdma_task_st_ctx ustorm_st_context;
-	struct regpair ustorm_st_padding[2];
-	struct ustorm_rdma_task_ag_ctx ustorm_ag_context;
-};
-
+/* RDMA Tid type enumeration (for register_tid ramrod) */
 enum rdma_tid_type {
 	RDMA_TID_REGISTERED_MR,
 	RDMA_TID_FMR,
@@ -6556,214 +7108,214 @@ enum rdma_tid_type {
 	MAX_RDMA_TID_TYPE
 };
 
-struct xstorm_roce_conn_ag_ctx_dq_ext_ld_part {
+struct e4_xstorm_roce_conn_ag_ctx_dq_ext_ld_part {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM0_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM0_SHIFT     0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT1_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT1_SHIFT             1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT2_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT2_SHIFT             2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM3_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM3_SHIFT     3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT4_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT4_SHIFT             4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT5_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT5_SHIFT             5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT6_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT6_SHIFT             6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT7_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT7_SHIFT             7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM0_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM0_SHIFT	0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT1_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT1_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT2_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT2_SHIFT		2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM3_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_EXIST_IN_QM3_SHIFT	3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT4_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT4_SHIFT		4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT5_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT5_SHIFT		5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT6_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT6_SHIFT		6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT7_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT7_SHIFT		7
 	u8 flags1;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT8_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT8_SHIFT             0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT9_MASK              0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT9_SHIFT             1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT10_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT10_SHIFT            2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT11_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT11_SHIFT            3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT12_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT12_SHIFT            4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_MSTORM_FLUSH_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_MSTORM_FLUSH_SHIFT     5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT14_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT14_SHIFT            6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_YSTORM_FLUSH_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_YSTORM_FLUSH_SHIFT     7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT8_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT8_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT9_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT9_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT10_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT10_SHIFT		2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT11_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT11_SHIFT		3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT12_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT12_SHIFT		4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_MSTORM_FLUSH_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_MSTORM_FLUSH_SHIFT	5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT14_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT14_SHIFT		6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_YSTORM_FLUSH_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_YSTORM_FLUSH_SHIFT	7
 	u8 flags2;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0_SHIFT              0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1_SHIFT              2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2_SHIFT              4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3_SHIFT              6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF0_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF0_SHIFT	0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF1_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF1_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF2_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF2_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF3_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4_SHIFT              0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5_SHIFT              2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6_SHIFT              4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_MASK       0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_SHIFT      6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF4_MASK		0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF4_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF5_MASK		0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF5_SHIFT		2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF6_MASK		0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF6_SHIFT		4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_SHIFT	6
 	u8 flags4;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8_SHIFT              0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9_MASK               0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9_SHIFT              2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10_SHIFT             4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11_SHIFT             6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF8_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF8_SHIFT	0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF9_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF9_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF10_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF10_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF11_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12_SHIFT             0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13_SHIFT             2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14_SHIFT             4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15_SHIFT             6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF12_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF12_SHIFT	0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF13_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF13_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF14_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF14_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF15_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF15_SHIFT	6
 	u8 flags6;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16_SHIFT             0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17_SHIFT             2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18_SHIFT             4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19_SHIFT             6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF16_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF16_SHIFT	0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF17_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF17_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF18_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF18_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF19_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF19_SHIFT	6
 	u8 flags7;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20_SHIFT             0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21_SHIFT             2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_MASK         0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_SHIFT        4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF0EN_SHIFT            6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF1EN_SHIFT            7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF20_MASK		0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF20_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF21_MASK		0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF21_SHIFT		2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_MASK		0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF0EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF0EN_SHIFT		6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF1EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF1EN_SHIFT		7
 	u8 flags8;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF2EN_SHIFT            0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF3EN_SHIFT            1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF4EN_SHIFT            2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF5EN_SHIFT            3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF6EN_SHIFT            4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_EN_MASK    0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_EN_SHIFT   5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF8EN_SHIFT            6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9EN_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF9EN_SHIFT            7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF2EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF2EN_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF3EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF3EN_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF4EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF4EN_SHIFT		2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF5EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF5EN_SHIFT		3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF6EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF6EN_SHIFT		4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_FLUSH_Q0_CF_EN_SHIFT	5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF8EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF8EN_SHIFT		6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF9EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF9EN_SHIFT		7
 	u8 flags9;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF10EN_SHIFT           0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF11EN_SHIFT           1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF12EN_SHIFT           2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF13EN_SHIFT           3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF14EN_SHIFT           4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF15EN_SHIFT           5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF16EN_SHIFT           6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF17EN_SHIFT           7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF10EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF10EN_SHIFT	0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF11EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF11EN_SHIFT	1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF12EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF12EN_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF13EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF13EN_SHIFT	3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF14EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF14EN_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF15EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF15EN_SHIFT	5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF16EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF16EN_SHIFT	6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF17EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF17EN_SHIFT	7
 	u8 flags10;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF18EN_SHIFT           0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF19EN_SHIFT           1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF20EN_SHIFT           2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF21EN_SHIFT           3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_EN_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_EN_SHIFT     4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23EN_MASK            0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23EN_SHIFT           5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE0EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE0EN_SHIFT          6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE1EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE1EN_SHIFT          7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF18EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF18EN_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF19EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF19EN_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF20EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF20EN_SHIFT		2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF21EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF21EN_SHIFT		3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_EN_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_SLOW_PATH_EN_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF23EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF23EN_SHIFT		5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE0EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE0EN_SHIFT		6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE1EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE1EN_SHIFT		7
 	u8 flags11;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE2EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE2EN_SHIFT          0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE3EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE3EN_SHIFT          1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE4EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE4EN_SHIFT          2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE5EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE5EN_SHIFT          3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE6EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE6EN_SHIFT          4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE7EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE7EN_SHIFT          5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED1_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED1_SHIFT     6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE9EN_MASK           0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE9EN_SHIFT          7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE2EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE2EN_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE3EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE3EN_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE4EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE4EN_SHIFT		2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE5EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE5EN_SHIFT		3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE6EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE6EN_SHIFT		4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE7EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE7EN_SHIFT		5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED1_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED1_SHIFT	6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE9EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE9EN_SHIFT		7
 	u8 flags12;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE10EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE10EN_SHIFT         0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE11EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE11EN_SHIFT         1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED2_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED2_SHIFT     2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED3_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED3_SHIFT     3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE14EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE14EN_SHIFT         4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE15EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE15EN_SHIFT         5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE16EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE16EN_SHIFT         6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE17EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE17EN_SHIFT         7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE10EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE10EN_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE11EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE11EN_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED2_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED2_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED3_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED3_SHIFT	3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE14EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE14EN_SHIFT		4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE15EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE15EN_SHIFT		5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE16EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE16EN_SHIFT		6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE17EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE17EN_SHIFT		7
 	u8 flags13;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE18EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE18EN_SHIFT         0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE19EN_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RULE19EN_SHIFT         1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED4_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED4_SHIFT     2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED5_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED5_SHIFT     3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED6_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED6_SHIFT     4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED7_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED7_SHIFT     5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED8_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED8_SHIFT     6
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED9_MASK      0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED9_SHIFT     7
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE18EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE18EN_SHIFT		0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE19EN_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RULE19EN_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED4_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED4_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED5_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED5_SHIFT	3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED6_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED6_SHIFT	4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED7_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED7_SHIFT	5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED8_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED8_SHIFT	6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED9_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_A0_RESERVED9_SHIFT	7
 	u8 flags14;
-#define XSTORMROCECONNAGCTXDQEXTLDPART_MIGRATION_MASK         0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_MIGRATION_SHIFT        0
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT17_MASK             0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_BIT17_SHIFT            1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_DPM_PORT_NUM_MASK      0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_DPM_PORT_NUM_SHIFT     2
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RESERVED_MASK          0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_RESERVED_SHIFT         4
-#define XSTORMROCECONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_MASK  0x1
-#define XSTORMROCECONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_SHIFT 5
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23_MASK              0x3
-#define XSTORMROCECONNAGCTXDQEXTLDPART_CF23_SHIFT             6
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_MIGRATION_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_MIGRATION_SHIFT	0
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT17_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_BIT17_SHIFT		1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_DPM_PORT_NUM_MASK	0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_DPM_PORT_NUM_SHIFT	2
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RESERVED_MASK		0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_RESERVED_SHIFT		4
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_MASK	0x1
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_ROCE_EDPM_ENABLE_SHIFT	5
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF23_MASK		0x3
+#define E4XSTORMROCECONNAGCTXDQEXTLDPART_CF23_SHIFT		6
 	u8 byte2;
 	__le16 physical_q0;
 	__le16 word1;
@@ -6783,126 +7335,126 @@ struct xstorm_roce_conn_ag_ctx_dq_ext_ld_part {
 	__le32 reg4;
 };
 
-struct mstorm_rdma_conn_ag_ctx {
+struct e4_mstorm_rdma_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define MSTORM_RDMA_CONN_AG_CTX_BIT0_MASK     0x1
-#define MSTORM_RDMA_CONN_AG_CTX_BIT0_SHIFT    0
-#define MSTORM_RDMA_CONN_AG_CTX_BIT1_MASK     0x1
-#define MSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT    1
-#define MSTORM_RDMA_CONN_AG_CTX_CF0_MASK      0x3
-#define MSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT     2
-#define MSTORM_RDMA_CONN_AG_CTX_CF1_MASK      0x3
-#define MSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT     4
-#define MSTORM_RDMA_CONN_AG_CTX_CF2_MASK      0x3
-#define MSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_MSTORM_RDMA_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_MSTORM_RDMA_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define MSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK    0x1
-#define MSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT   0
-#define MSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK    0x1
-#define MSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT   1
-#define MSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK    0x1
-#define MSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT   2
-#define MSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define MSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define MSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define MSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define MSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define MSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define MSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define MSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define MSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define MSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_MSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT	7
 	__le16 word0;
 	__le16 word1;
 	__le32 reg0;
 	__le32 reg1;
 };
 
-struct tstorm_rdma_conn_ag_ctx {
+struct e4_tstorm_rdma_conn_ag_ctx {
 	u8 reserved0;
 	u8 byte1;
 	u8 flags0;
-#define TSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK          0x1
-#define TSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT         0
-#define TSTORM_RDMA_CONN_AG_CTX_BIT1_MASK                  0x1
-#define TSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT                 1
-#define TSTORM_RDMA_CONN_AG_CTX_BIT2_MASK                  0x1
-#define TSTORM_RDMA_CONN_AG_CTX_BIT2_SHIFT                 2
-#define TSTORM_RDMA_CONN_AG_CTX_BIT3_MASK                  0x1
-#define TSTORM_RDMA_CONN_AG_CTX_BIT3_SHIFT                 3
-#define TSTORM_RDMA_CONN_AG_CTX_BIT4_MASK                  0x1
-#define TSTORM_RDMA_CONN_AG_CTX_BIT4_SHIFT                 4
-#define TSTORM_RDMA_CONN_AG_CTX_BIT5_MASK                  0x1
-#define TSTORM_RDMA_CONN_AG_CTX_BIT5_SHIFT                 5
-#define TSTORM_RDMA_CONN_AG_CTX_CF0_MASK                   0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT                  6
+#define E4_TSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT2_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT2_SHIFT		2
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT3_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT3_SHIFT		3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT4_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT4_SHIFT		4
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT5_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_BIT5_SHIFT		5
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT		6
 	u8 flags1;
-#define TSTORM_RDMA_CONN_AG_CTX_CF1_MASK                   0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT                  0
-#define TSTORM_RDMA_CONN_AG_CTX_CF2_MASK                   0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT                  2
-#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK     0x3
-#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT    4
-#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK           0x3
-#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT          6
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF1_MASK			0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT			0
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF2_MASK			0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT			2
+#define E4_TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK	0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT	4
+#define E4_TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK		0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT		6
 	u8 flags2;
-#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK       0x3
-#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT      0
-#define TSTORM_RDMA_CONN_AG_CTX_CF6_MASK                   0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF6_SHIFT                  2
-#define TSTORM_RDMA_CONN_AG_CTX_CF7_MASK                   0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF7_SHIFT                  4
-#define TSTORM_RDMA_CONN_AG_CTX_CF8_MASK                   0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF8_SHIFT                  6
+#define E4_TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK		0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT	0
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF6_MASK			0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF6_SHIFT			2
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF7_MASK			0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF7_SHIFT			4
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF8_MASK			0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF8_SHIFT			6
 	u8 flags3;
-#define TSTORM_RDMA_CONN_AG_CTX_CF9_MASK                   0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF9_SHIFT                  0
-#define TSTORM_RDMA_CONN_AG_CTX_CF10_MASK                  0x3
-#define TSTORM_RDMA_CONN_AG_CTX_CF10_SHIFT                 2
-#define TSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK                 0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT                4
-#define TSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK                 0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT                5
-#define TSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK                 0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT                6
-#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK  0x1
-#define TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT 7
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF9_MASK			0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF9_SHIFT			0
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF10_MASK			0x3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF10_SHIFT			2
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT			4
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT			5
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT			6
+#define E4_TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK	0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT	7
 	u8 flags4;
-#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK        0x1
-#define TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT       0
-#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK    0x1
-#define TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT   1
-#define TSTORM_RDMA_CONN_AG_CTX_CF6EN_MASK                 0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT                2
-#define TSTORM_RDMA_CONN_AG_CTX_CF7EN_MASK                 0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF7EN_SHIFT                3
-#define TSTORM_RDMA_CONN_AG_CTX_CF8EN_MASK                 0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF8EN_SHIFT                4
-#define TSTORM_RDMA_CONN_AG_CTX_CF9EN_MASK                 0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF9EN_SHIFT                5
-#define TSTORM_RDMA_CONN_AG_CTX_CF10EN_MASK                0x1
-#define TSTORM_RDMA_CONN_AG_CTX_CF10EN_SHIFT               6
-#define TSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT              7
+#define E4_TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT		0
+#define E4_TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK	0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT	1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF6EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT			2
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF7EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF7EN_SHIFT			3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF8EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF8EN_SHIFT			4
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF9EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF9EN_SHIFT			5
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF10EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_CF10EN_SHIFT			6
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK			0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT		7
 	u8 flags5;
-#define TSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT              0
-#define TSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT              1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT              2
-#define TSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT              3
-#define TSTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT              4
-#define TSTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT              5
-#define TSTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT              6
-#define TSTORM_RDMA_CONN_AG_CTX_RULE8EN_MASK               0x1
-#define TSTORM_RDMA_CONN_AG_CTX_RULE8EN_SHIFT              7
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_TSTORM_RDMA_CONN_AG_CTX_RULE8EN_SHIFT	7
 	__le32 reg0;
 	__le32 reg1;
 	__le32 reg2;
@@ -6924,73 +7476,73 @@ struct tstorm_rdma_conn_ag_ctx {
 	__le32 reg10;
 };
 
-struct tstorm_rdma_task_ag_ctx {
+struct e4_tstorm_rdma_task_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	__le16 word0;
 	u8 flags0;
-#define TSTORM_RDMA_TASK_AG_CTX_NIBBLE0_MASK  0xF
-#define TSTORM_RDMA_TASK_AG_CTX_NIBBLE0_SHIFT 0
-#define TSTORM_RDMA_TASK_AG_CTX_BIT0_MASK     0x1
-#define TSTORM_RDMA_TASK_AG_CTX_BIT0_SHIFT    4
-#define TSTORM_RDMA_TASK_AG_CTX_BIT1_MASK     0x1
-#define TSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT    5
-#define TSTORM_RDMA_TASK_AG_CTX_BIT2_MASK     0x1
-#define TSTORM_RDMA_TASK_AG_CTX_BIT2_SHIFT    6
-#define TSTORM_RDMA_TASK_AG_CTX_BIT3_MASK     0x1
-#define TSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT    7
+#define E4_TSTORM_RDMA_TASK_AG_CTX_NIBBLE0_MASK		0xF
+#define E4_TSTORM_RDMA_TASK_AG_CTX_NIBBLE0_SHIFT	0
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT0_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT0_SHIFT		4
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT1_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT1_SHIFT		5
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT2_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT2_SHIFT		6
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT3_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT3_SHIFT		7
 	u8 flags1;
-#define TSTORM_RDMA_TASK_AG_CTX_BIT4_MASK     0x1
-#define TSTORM_RDMA_TASK_AG_CTX_BIT4_SHIFT    0
-#define TSTORM_RDMA_TASK_AG_CTX_BIT5_MASK     0x1
-#define TSTORM_RDMA_TASK_AG_CTX_BIT5_SHIFT    1
-#define TSTORM_RDMA_TASK_AG_CTX_CF0_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT     2
-#define TSTORM_RDMA_TASK_AG_CTX_CF1_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT     4
-#define TSTORM_RDMA_TASK_AG_CTX_CF2_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF2_SHIFT     6
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT4_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT4_SHIFT	0
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT5_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_BIT5_SHIFT	1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF0_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF0_SHIFT	2
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF1_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF1_SHIFT	4
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF2_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF2_SHIFT	6
 	u8 flags2;
-#define TSTORM_RDMA_TASK_AG_CTX_CF3_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF3_SHIFT     0
-#define TSTORM_RDMA_TASK_AG_CTX_CF4_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF4_SHIFT     2
-#define TSTORM_RDMA_TASK_AG_CTX_CF5_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF5_SHIFT     4
-#define TSTORM_RDMA_TASK_AG_CTX_CF6_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF6_SHIFT     6
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF3_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF3_SHIFT	0
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF4_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF4_SHIFT	2
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF5_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF5_SHIFT	4
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF6_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF6_SHIFT	6
 	u8 flags3;
-#define TSTORM_RDMA_TASK_AG_CTX_CF7_MASK      0x3
-#define TSTORM_RDMA_TASK_AG_CTX_CF7_SHIFT     0
-#define TSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT   2
-#define TSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT   3
-#define TSTORM_RDMA_TASK_AG_CTX_CF2EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF2EN_SHIFT   4
-#define TSTORM_RDMA_TASK_AG_CTX_CF3EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF3EN_SHIFT   5
-#define TSTORM_RDMA_TASK_AG_CTX_CF4EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF4EN_SHIFT   6
-#define TSTORM_RDMA_TASK_AG_CTX_CF5EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF5EN_SHIFT   7
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF7_MASK	0x3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF7_SHIFT	0
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF0EN_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF0EN_SHIFT	2
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF1EN_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF1EN_SHIFT	3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF2EN_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF2EN_SHIFT	4
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF3EN_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF3EN_SHIFT	5
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF4EN_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF4EN_SHIFT	6
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF5EN_MASK	0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF5EN_SHIFT	7
 	u8 flags4;
-#define TSTORM_RDMA_TASK_AG_CTX_CF6EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF6EN_SHIFT   0
-#define TSTORM_RDMA_TASK_AG_CTX_CF7EN_MASK    0x1
-#define TSTORM_RDMA_TASK_AG_CTX_CF7EN_SHIFT   1
-#define TSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK  0x1
-#define TSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT 2
-#define TSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK  0x1
-#define TSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT 3
-#define TSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK  0x1
-#define TSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT 4
-#define TSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK  0x1
-#define TSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT 5
-#define TSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK  0x1
-#define TSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT 6
-#define TSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK  0x1
-#define TSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT 7
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF6EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF6EN_SHIFT		0
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF7EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_CF7EN_SHIFT		1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE0EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE0EN_SHIFT	2
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE1EN_SHIFT	3
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE2EN_SHIFT	4
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE3EN_SHIFT	5
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE4EN_SHIFT	6
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_RDMA_TASK_AG_CTX_RULE5EN_SHIFT	7
 	u8 byte2;
 	__le16 word1;
 	__le32 reg0;
@@ -7003,63 +7555,63 @@ struct tstorm_rdma_task_ag_ctx {
 	__le32 reg2;
 };
 
-struct ustorm_rdma_conn_ag_ctx {
+struct e4_ustorm_rdma_conn_ag_ctx {
 	u8 reserved;
 	u8 byte1;
 	u8 flags0;
-#define USTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK     0x1
-#define USTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT    0
-#define USTORM_RDMA_CONN_AG_CTX_BIT1_MASK             0x1
-#define USTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT            1
-#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK      0x3
-#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT     2
-#define USTORM_RDMA_CONN_AG_CTX_CF1_MASK              0x3
-#define USTORM_RDMA_CONN_AG_CTX_CF1_SHIFT             4
-#define USTORM_RDMA_CONN_AG_CTX_CF2_MASK              0x3
-#define USTORM_RDMA_CONN_AG_CTX_CF2_SHIFT             6
+#define E4_USTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_USTORM_RDMA_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK	0x3
+#define E4_USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT	2
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF1_MASK		0x3
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF1_SHIFT		4
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF2_MASK		0x3
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF2_SHIFT		6
 	u8 flags1;
-#define USTORM_RDMA_CONN_AG_CTX_CF3_MASK              0x3
-#define USTORM_RDMA_CONN_AG_CTX_CF3_SHIFT             0
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_MASK     0x3
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_SHIFT    2
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_MASK        0x3
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_SHIFT       4
-#define USTORM_RDMA_CONN_AG_CTX_CF6_MASK              0x3
-#define USTORM_RDMA_CONN_AG_CTX_CF6_SHIFT             6
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF3_MASK		0x3
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF3_SHIFT		0
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_MASK	0x3
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_SHIFT	2
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_MASK	0x3
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_SHIFT	4
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF6_MASK		0x3
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF6_SHIFT		6
 	u8 flags2;
-#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK   0x1
-#define USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT  0
-#define USTORM_RDMA_CONN_AG_CTX_CF1EN_MASK            0x1
-#define USTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT           1
-#define USTORM_RDMA_CONN_AG_CTX_CF2EN_MASK            0x1
-#define USTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT           2
-#define USTORM_RDMA_CONN_AG_CTX_CF3EN_MASK            0x1
-#define USTORM_RDMA_CONN_AG_CTX_CF3EN_SHIFT           3
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_EN_MASK  0x1
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_EN_SHIFT 4
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_EN_MASK     0x1
-#define USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_EN_SHIFT    5
-#define USTORM_RDMA_CONN_AG_CTX_CF6EN_MASK            0x1
-#define USTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT           6
-#define USTORM_RDMA_CONN_AG_CTX_CQ_SE_EN_MASK         0x1
-#define USTORM_RDMA_CONN_AG_CTX_CQ_SE_EN_SHIFT        7
+#define E4_USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT		0
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF1EN_MASK			0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT			1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT			2
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF3EN_MASK			0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF3EN_SHIFT			3
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_SE_CF_EN_SHIFT	4
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_ARM_CF_EN_SHIFT		5
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF6EN_MASK			0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT			6
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_SE_EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_SE_EN_SHIFT		7
 	u8 flags3;
-#define USTORM_RDMA_CONN_AG_CTX_CQ_EN_MASK            0x1
-#define USTORM_RDMA_CONN_AG_CTX_CQ_EN_SHIFT           0
-#define USTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK          0x1
-#define USTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT         1
-#define USTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK          0x1
-#define USTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT         2
-#define USTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK          0x1
-#define USTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT         3
-#define USTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK          0x1
-#define USTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT         4
-#define USTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK          0x1
-#define USTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT         5
-#define USTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK          0x1
-#define USTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT         6
-#define USTORM_RDMA_CONN_AG_CTX_RULE8EN_MASK          0x1
-#define USTORM_RDMA_CONN_AG_CTX_RULE8EN_SHIFT         7
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_CQ_EN_SHIFT		0
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_USTORM_RDMA_CONN_AG_CTX_RULE8EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 conn_dpi;
@@ -7072,214 +7624,214 @@ struct ustorm_rdma_conn_ag_ctx {
 	__le16 word3;
 };
 
-struct xstorm_rdma_conn_ag_ctx {
+struct e4_xstorm_rdma_conn_ag_ctx {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT     0
-#define XSTORM_RDMA_CONN_AG_CTX_BIT1_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT             1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT2_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT2_SHIFT             2
-#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM3_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM3_SHIFT     3
-#define XSTORM_RDMA_CONN_AG_CTX_BIT4_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT4_SHIFT             4
-#define XSTORM_RDMA_CONN_AG_CTX_BIT5_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT5_SHIFT             5
-#define XSTORM_RDMA_CONN_AG_CTX_BIT6_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT6_SHIFT             6
-#define XSTORM_RDMA_CONN_AG_CTX_BIT7_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT7_SHIFT             7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT2_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT2_SHIFT		2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT4_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT4_SHIFT		4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT5_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT5_SHIFT		5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT6_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT6_SHIFT		6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT7_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT7_SHIFT		7
 	u8 flags1;
-#define XSTORM_RDMA_CONN_AG_CTX_BIT8_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT8_SHIFT             0
-#define XSTORM_RDMA_CONN_AG_CTX_BIT9_MASK              0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT9_SHIFT             1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT10_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT10_SHIFT            2
-#define XSTORM_RDMA_CONN_AG_CTX_BIT11_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT11_SHIFT            3
-#define XSTORM_RDMA_CONN_AG_CTX_BIT12_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT12_SHIFT            4
-#define XSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_SHIFT     5
-#define XSTORM_RDMA_CONN_AG_CTX_BIT14_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT14_SHIFT            6
-#define XSTORM_RDMA_CONN_AG_CTX_YSTORM_FLUSH_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_YSTORM_FLUSH_SHIFT     7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT8_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT8_SHIFT		0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT9_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT9_SHIFT		1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT10_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT10_SHIFT		2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT11_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT12_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT12_SHIFT		4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_MSTORM_FLUSH_SHIFT	5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT14_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT14_SHIFT		6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_YSTORM_FLUSH_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_YSTORM_FLUSH_SHIFT	7
 	u8 flags2;
-#define XSTORM_RDMA_CONN_AG_CTX_CF0_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT              0
-#define XSTORM_RDMA_CONN_AG_CTX_CF1_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT              2
-#define XSTORM_RDMA_CONN_AG_CTX_CF2_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT              4
-#define XSTORM_RDMA_CONN_AG_CTX_CF3_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF3_SHIFT              6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORM_RDMA_CONN_AG_CTX_CF4_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF4_SHIFT              0
-#define XSTORM_RDMA_CONN_AG_CTX_CF5_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF5_SHIFT              2
-#define XSTORM_RDMA_CONN_AG_CTX_CF6_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF6_SHIFT              4
-#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK       0x3
-#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT      6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF4_MASK		0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF4_SHIFT		0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF5_MASK		0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF5_SHIFT		2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF6_MASK		0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF6_SHIFT		4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT	6
 	u8 flags4;
-#define XSTORM_RDMA_CONN_AG_CTX_CF8_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF8_SHIFT              0
-#define XSTORM_RDMA_CONN_AG_CTX_CF9_MASK               0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF9_SHIFT              2
-#define XSTORM_RDMA_CONN_AG_CTX_CF10_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF10_SHIFT             4
-#define XSTORM_RDMA_CONN_AG_CTX_CF11_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF11_SHIFT             6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_RDMA_CONN_AG_CTX_CF12_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF12_SHIFT             0
-#define XSTORM_RDMA_CONN_AG_CTX_CF13_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF13_SHIFT             2
-#define XSTORM_RDMA_CONN_AG_CTX_CF14_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF14_SHIFT             4
-#define XSTORM_RDMA_CONN_AG_CTX_CF15_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF15_SHIFT             6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF12_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF12_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF13_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF13_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF14_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF14_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF15_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF15_SHIFT	6
 	u8 flags6;
-#define XSTORM_RDMA_CONN_AG_CTX_CF16_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF16_SHIFT             0
-#define XSTORM_RDMA_CONN_AG_CTX_CF17_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF17_SHIFT             2
-#define XSTORM_RDMA_CONN_AG_CTX_CF18_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF18_SHIFT             4
-#define XSTORM_RDMA_CONN_AG_CTX_CF19_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF19_SHIFT             6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF16_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF16_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF17_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF17_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF18_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF18_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF19_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF19_SHIFT	6
 	u8 flags7;
-#define XSTORM_RDMA_CONN_AG_CTX_CF20_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF20_SHIFT             0
-#define XSTORM_RDMA_CONN_AG_CTX_CF21_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF21_SHIFT             2
-#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_MASK         0x3
-#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_SHIFT        4
-#define XSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT            6
-#define XSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT            7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF20_MASK		0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF20_SHIFT		0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF21_MASK		0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF21_SHIFT		2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT		6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT		7
 	u8 flags8;
-#define XSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT            0
-#define XSTORM_RDMA_CONN_AG_CTX_CF3EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF3EN_SHIFT            1
-#define XSTORM_RDMA_CONN_AG_CTX_CF4EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF4EN_SHIFT            2
-#define XSTORM_RDMA_CONN_AG_CTX_CF5EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF5EN_SHIFT            3
-#define XSTORM_RDMA_CONN_AG_CTX_CF6EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT            4
-#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK    0x1
-#define XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT   5
-#define XSTORM_RDMA_CONN_AG_CTX_CF8EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF8EN_SHIFT            6
-#define XSTORM_RDMA_CONN_AG_CTX_CF9EN_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF9EN_SHIFT            7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT		0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF3EN_SHIFT		1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF4EN_SHIFT		2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF5EN_SHIFT		3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF6EN_SHIFT		4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT	5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF8EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF8EN_SHIFT		6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF9EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF9EN_SHIFT		7
 	u8 flags9;
-#define XSTORM_RDMA_CONN_AG_CTX_CF10EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF10EN_SHIFT           0
-#define XSTORM_RDMA_CONN_AG_CTX_CF11EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF11EN_SHIFT           1
-#define XSTORM_RDMA_CONN_AG_CTX_CF12EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF12EN_SHIFT           2
-#define XSTORM_RDMA_CONN_AG_CTX_CF13EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF13EN_SHIFT           3
-#define XSTORM_RDMA_CONN_AG_CTX_CF14EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF14EN_SHIFT           4
-#define XSTORM_RDMA_CONN_AG_CTX_CF15EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF15EN_SHIFT           5
-#define XSTORM_RDMA_CONN_AG_CTX_CF16EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF16EN_SHIFT           6
-#define XSTORM_RDMA_CONN_AG_CTX_CF17EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF17EN_SHIFT           7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF10EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF10EN_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF11EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF11EN_SHIFT	1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF12EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF12EN_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF13EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF13EN_SHIFT	3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF14EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF14EN_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF15EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF15EN_SHIFT	5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF16EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF16EN_SHIFT	6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF17EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF17EN_SHIFT	7
 	u8 flags10;
-#define XSTORM_RDMA_CONN_AG_CTX_CF18EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF18EN_SHIFT           0
-#define XSTORM_RDMA_CONN_AG_CTX_CF19EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF19EN_SHIFT           1
-#define XSTORM_RDMA_CONN_AG_CTX_CF20EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF20EN_SHIFT           2
-#define XSTORM_RDMA_CONN_AG_CTX_CF21EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF21EN_SHIFT           3
-#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_EN_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_EN_SHIFT     4
-#define XSTORM_RDMA_CONN_AG_CTX_CF23EN_MASK            0x1
-#define XSTORM_RDMA_CONN_AG_CTX_CF23EN_SHIFT           5
-#define XSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT          6
-#define XSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT          7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF18EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF18EN_SHIFT		0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF19EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF19EN_SHIFT		1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF20EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF20EN_SHIFT		2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF21EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF21EN_SHIFT		3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_SLOW_PATH_EN_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF23EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF23EN_SHIFT		5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT	6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT	7
 	u8 flags11;
-#define XSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT          0
-#define XSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT          1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT          2
-#define XSTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT          3
-#define XSTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT          4
-#define XSTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT          5
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED1_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED1_SHIFT     6
-#define XSTORM_RDMA_CONN_AG_CTX_RULE9EN_MASK           0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE9EN_SHIFT          7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT	1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE5EN_SHIFT	3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE6EN_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE7EN_SHIFT	5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE9EN_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE9EN_SHIFT	7
 	u8 flags12;
-#define XSTORM_RDMA_CONN_AG_CTX_RULE10EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE10EN_SHIFT         0
-#define XSTORM_RDMA_CONN_AG_CTX_RULE11EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE11EN_SHIFT         1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED2_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED2_SHIFT     2
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED3_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED3_SHIFT     3
-#define XSTORM_RDMA_CONN_AG_CTX_RULE14EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE14EN_SHIFT         4
-#define XSTORM_RDMA_CONN_AG_CTX_RULE15EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE15EN_SHIFT         5
-#define XSTORM_RDMA_CONN_AG_CTX_RULE16EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE16EN_SHIFT         6
-#define XSTORM_RDMA_CONN_AG_CTX_RULE17EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE17EN_SHIFT         7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE10EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE10EN_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE11EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE11EN_SHIFT	1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE14EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE14EN_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE15EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE15EN_SHIFT	5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE16EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE16EN_SHIFT	6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE17EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE17EN_SHIFT	7
 	u8 flags13;
-#define XSTORM_RDMA_CONN_AG_CTX_RULE18EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE18EN_SHIFT         0
-#define XSTORM_RDMA_CONN_AG_CTX_RULE19EN_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RULE19EN_SHIFT         1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED4_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED4_SHIFT     2
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED5_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED5_SHIFT     3
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED6_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED6_SHIFT     4
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED7_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED7_SHIFT     5
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED8_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED8_SHIFT     6
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED9_MASK      0x1
-#define XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED9_SHIFT     7
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE18EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE18EN_SHIFT	0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE19EN_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RULE19EN_SHIFT	1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
 	u8 flags14;
-#define XSTORM_RDMA_CONN_AG_CTX_MIGRATION_MASK         0x1
-#define XSTORM_RDMA_CONN_AG_CTX_MIGRATION_SHIFT        0
-#define XSTORM_RDMA_CONN_AG_CTX_BIT17_MASK             0x1
-#define XSTORM_RDMA_CONN_AG_CTX_BIT17_SHIFT            1
-#define XSTORM_RDMA_CONN_AG_CTX_DPM_PORT_NUM_MASK      0x3
-#define XSTORM_RDMA_CONN_AG_CTX_DPM_PORT_NUM_SHIFT     2
-#define XSTORM_RDMA_CONN_AG_CTX_RESERVED_MASK          0x1
-#define XSTORM_RDMA_CONN_AG_CTX_RESERVED_SHIFT         4
-#define XSTORM_RDMA_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK  0x1
-#define XSTORM_RDMA_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT 5
-#define XSTORM_RDMA_CONN_AG_CTX_CF23_MASK              0x3
-#define XSTORM_RDMA_CONN_AG_CTX_CF23_SHIFT             6
+#define E4_XSTORM_RDMA_CONN_AG_CTX_MIGRATION_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_MIGRATION_SHIFT		0
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT17_MASK			0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_BIT17_SHIFT			1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_DPM_PORT_NUM_MASK		0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_DPM_PORT_NUM_SHIFT		2
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RESERVED_MASK		0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_RESERVED_SHIFT		4
+#define E4_XSTORM_RDMA_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK	0x1
+#define E4_XSTORM_RDMA_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT	5
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF23_MASK			0x3
+#define E4_XSTORM_RDMA_CONN_AG_CTX_CF23_SHIFT			6
 	u8 byte2;
 	__le16 physical_q0;
 	__le16 word1;
@@ -7301,37 +7853,37 @@ struct xstorm_rdma_conn_ag_ctx {
 	__le32 reg6;
 };
 
-struct ystorm_rdma_conn_ag_ctx {
+struct e4_ystorm_rdma_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define YSTORM_RDMA_CONN_AG_CTX_BIT0_MASK     0x1
-#define YSTORM_RDMA_CONN_AG_CTX_BIT0_SHIFT    0
-#define YSTORM_RDMA_CONN_AG_CTX_BIT1_MASK     0x1
-#define YSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT    1
-#define YSTORM_RDMA_CONN_AG_CTX_CF0_MASK      0x3
-#define YSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT     2
-#define YSTORM_RDMA_CONN_AG_CTX_CF1_MASK      0x3
-#define YSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT     4
-#define YSTORM_RDMA_CONN_AG_CTX_CF2_MASK      0x3
-#define YSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_YSTORM_RDMA_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_YSTORM_RDMA_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define YSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK    0x1
-#define YSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT   0
-#define YSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK    0x1
-#define YSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT   1
-#define YSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK    0x1
-#define YSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT   2
-#define YSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define YSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define YSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define YSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define YSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define YSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define YSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define YSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define YSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define YSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_YSTORM_RDMA_CONN_AG_CTX_RULE4EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -7345,62 +7897,70 @@ struct ystorm_rdma_conn_ag_ctx {
 	__le32 reg3;
 };
 
-struct mstorm_roce_conn_st_ctx {
-	struct regpair temp[6];
+/* The roce storm context of Ystorm */
+struct ystorm_roce_conn_st_ctx {
+	struct regpair temp[2];
 };
 
+/* The roce storm context of Mstorm */
 struct pstorm_roce_conn_st_ctx {
 	struct regpair temp[16];
 };
 
-struct ystorm_roce_conn_st_ctx {
-	struct regpair temp[2];
-};
-
+/* The roce storm context of Xstorm */
 struct xstorm_roce_conn_st_ctx {
 	struct regpair temp[24];
 };
 
+/* The roce storm context of Tstorm */
 struct tstorm_roce_conn_st_ctx {
 	struct regpair temp[30];
 };
 
+/* The roce storm context of Mstorm */
+struct mstorm_roce_conn_st_ctx {
+	struct regpair temp[6];
+};
+
+/* The roce storm context of Ystorm */
 struct ustorm_roce_conn_st_ctx {
 	struct regpair temp[12];
 };
 
-struct roce_conn_context {
+/* roce connection context */
+struct e4_roce_conn_context {
 	struct ystorm_roce_conn_st_ctx ystorm_st_context;
 	struct regpair ystorm_st_padding[2];
 	struct pstorm_roce_conn_st_ctx pstorm_st_context;
 	struct xstorm_roce_conn_st_ctx xstorm_st_context;
 	struct regpair xstorm_st_padding[2];
-	struct xstorm_rdma_conn_ag_ctx xstorm_ag_context;
-	struct tstorm_rdma_conn_ag_ctx tstorm_ag_context;
+	struct e4_xstorm_rdma_conn_ag_ctx xstorm_ag_context;
+	struct e4_tstorm_rdma_conn_ag_ctx tstorm_ag_context;
 	struct timers_context timer_context;
-	struct ustorm_rdma_conn_ag_ctx ustorm_ag_context;
+	struct e4_ustorm_rdma_conn_ag_ctx ustorm_ag_context;
 	struct tstorm_roce_conn_st_ctx tstorm_st_context;
 	struct mstorm_roce_conn_st_ctx mstorm_st_context;
 	struct ustorm_roce_conn_st_ctx ustorm_st_context;
 	struct regpair ustorm_st_padding[2];
 };
 
+/* roce create qp requester ramrod data */
 struct roce_create_qp_req_ramrod_data {
 	__le16 flags;
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR_MASK          0x3
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR_SHIFT         0
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN_MASK  0x1
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN_SHIFT 2
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP_MASK        0x1
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP_SHIFT       3
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_MASK                  0x7
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_SHIFT                 4
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RESERVED_MASK             0x1
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RESERVED_SHIFT            7
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_MASK        0xF
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_SHIFT       8
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_MASK          0xF
-#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_SHIFT         12
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR_MASK			0x3
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR_SHIFT		0
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN_MASK		0x1
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN_SHIFT	2
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP_MASK		0x1
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP_SHIFT		3
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_MASK				0x7
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_SHIFT			4
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RESERVED_MASK			0x1
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RESERVED_SHIFT			7
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_MASK		0xF
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_SHIFT		8
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_MASK			0xF
+#define ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_SHIFT		12
 	u8 max_ord;
 	u8 traffic_class;
 	u8 hop_limit;
@@ -7431,26 +7991,27 @@ struct roce_create_qp_req_ramrod_data {
 	__le16 dpi;
 };
 
+/* roce create qp responder ramrod data */
 struct roce_create_qp_resp_ramrod_data {
 	__le16 flags;
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR_MASK          0x3
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR_SHIFT         0
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN_MASK           0x1
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN_SHIFT          2
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN_MASK           0x1
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN_SHIFT          3
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN_MASK            0x1
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN_SHIFT           4
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG_MASK              0x1
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG_SHIFT             5
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_MASK  0x1
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_SHIFT 6
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN_MASK	0x1
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN_SHIFT	7
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_MASK                  0x7
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_SHIFT                 8
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_MASK    0x1F
-#define ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_SHIFT   11
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR_MASK		0x3
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR_SHIFT		0
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN_MASK			0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN_SHIFT		2
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN_MASK			0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN_SHIFT		3
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN_MASK			0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN_SHIFT			4
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG_MASK			0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG_SHIFT			5
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_MASK	0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN_SHIFT	6
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN_MASK		0x1
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN_SHIFT		7
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_MASK			0x7
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_SHIFT			8
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_MASK		0x1F
+#define ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_SHIFT		11
 	u8 max_ird;
 	u8 traffic_class;
 	u8 hop_limit;
@@ -7482,24 +8043,40 @@ struct roce_create_qp_resp_ramrod_data {
 	__le16 dpi;
 };
 
+/* roce DCQCN received statistics */
+struct roce_dcqcn_received_stats {
+	struct regpair ecn_pkt_rcv;
+	struct regpair cnp_pkt_rcv;
+};
+
+/* roce DCQCN sent statistics */
+struct roce_dcqcn_sent_stats {
+	struct regpair cnp_pkt_sent;
+};
+
+/* RoCE destroy qp requester output params */
 struct roce_destroy_qp_req_output_params {
 	__le32 num_bound_mw;
 	__le32 cq_prod;
 };
 
+/* RoCE destroy qp requester ramrod data */
 struct roce_destroy_qp_req_ramrod_data {
 	struct regpair output_params_addr;
 };
 
+/* RoCE destroy qp responder output params */
 struct roce_destroy_qp_resp_output_params {
 	__le32 num_invalidated_mw;
 	__le32 cq_prod;
 };
 
+/* RoCE destroy qp responder ramrod data */
 struct roce_destroy_qp_resp_ramrod_data {
 	struct regpair output_params_addr;
 };
 
+/* roce special events statistics */
 struct roce_events_stats {
 	__le16 silent_drops;
 	__le16 rnr_naks_sent;
@@ -7508,6 +8085,7 @@ struct roce_events_stats {
 	__le32 reserved;
 };
 
+/* ROCE slow path EQ cmd IDs */
 enum roce_event_opcode {
 	ROCE_EVENT_CREATE_QP = 11,
 	ROCE_EVENT_MODIFY_QP,
@@ -7518,6 +8096,7 @@ enum roce_event_opcode {
 	MAX_ROCE_EVENT_OPCODE
 };
 
+/* roce func init ramrod data */
 struct roce_init_func_params {
 	u8 ll2_queue_id;
 	u8 cnp_vlan_priority;
@@ -7526,42 +8105,46 @@ struct roce_init_func_params {
 	__le32 cnp_send_timeout;
 };
 
+/* roce func init ramrod data */
 struct roce_init_func_ramrod_data {
 	struct rdma_init_func_ramrod_data rdma;
 	struct roce_init_func_params roce;
 };
 
+/* roce modify qp requester ramrod data */
 struct roce_modify_qp_req_ramrod_data {
 	__le16 flags;
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG_MASK      0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG_SHIFT     0
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG_MASK      0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG_SHIFT     1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY_MASK  0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY_SHIFT 2
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG_MASK            0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG_SHIFT           3
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG_MASK   0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG_SHIFT  4
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG_MASK          0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG_SHIFT         5
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG_MASK      0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG_SHIFT     6
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG_MASK    0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG_SHIFT   7
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG_MASK      0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG_SHIFT     8
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_MASK              0x1
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_SHIFT             9
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_MASK                  0x7
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_SHIFT                 10
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RESERVED1_MASK            0x7
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RESERVED1_SHIFT           13
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG_SHIFT		0
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG_SHIFT		1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY_SHIFT	2
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG_MASK			0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG_SHIFT			3
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG_SHIFT		4
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG_MASK			0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG_SHIFT		5
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG_SHIFT		6
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG_SHIFT		7
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG_SHIFT		8
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_MASK			0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_SHIFT			9
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_MASK				0x7
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_SHIFT			10
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PHYSICAL_QUEUES_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PHYSICAL_QUEUES_FLG_SHIFT	13
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RESERVED1_MASK			0x3
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RESERVED1_SHIFT			14
 	u8 fields;
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_MASK        0xF
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_SHIFT       0
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_MASK          0xF
-#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_SHIFT         4
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_MASK	0xF
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_SHIFT	0
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_MASK		0xF
+#define ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_SHIFT	4
 	u8 max_ord;
 	u8 traffic_class;
 	u8 hop_limit;
@@ -7570,66 +8153,76 @@ struct roce_modify_qp_req_ramrod_data {
 	__le32 ack_timeout_val;
 	__le16 mtu;
 	__le16 reserved2;
-	__le32 reserved3[3];
+	__le32 reserved3[2];
+	__le16 low_latency_phy_queue;
+	__le16 regular_latency_phy_queue;
 	__le32 src_gid[4];
 	__le32 dst_gid[4];
 };
 
+/* roce modify qp responder ramrod data */
 struct roce_modify_qp_resp_ramrod_data {
 	__le16 flags;
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG_MASK        0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG_SHIFT       0
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN_MASK             0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN_SHIFT            1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN_MASK             0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN_SHIFT            2
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN_MASK              0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN_SHIFT             3
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG_MASK              0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG_SHIFT             4
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG_MASK     0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG_SHIFT    5
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG_MASK            0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG_SHIFT           6
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_MASK                0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_SHIFT               7
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG_MASK  0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG_SHIFT 8
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG_MASK        0x1
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG_SHIFT       9
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RESERVED1_MASK              0x3F
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RESERVED1_SHIFT             10
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG_SHIFT		0
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN_MASK			0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN_SHIFT		1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN_MASK			0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN_SHIFT		2
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN_MASK			0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN_SHIFT			3
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG_MASK			0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG_SHIFT			4
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG_SHIFT	5
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG_SHIFT		6
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_MASK			0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_SHIFT			7
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG_MASK	0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG_SHIFT	8
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG_MASK		0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG_SHIFT		9
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PHYSICAL_QUEUES_FLG_MASK	0x1
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PHYSICAL_QUEUES_FLG_SHIFT	10
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RESERVED1_MASK			0x1F
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_RESERVED1_SHIFT			11
 	u8 fields;
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_MASK                    0x7
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_SHIFT                   0
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_MASK      0x1F
-#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_SHIFT     3
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_MASK		0x7
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_SHIFT		0
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_MASK	0x1F
+#define ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_SHIFT	3
 	u8 max_ird;
 	u8 traffic_class;
 	u8 hop_limit;
 	__le16 p_key;
 	__le32 flow_label;
 	__le16 mtu;
-	__le16 reserved2;
+	__le16 low_latency_phy_queue;
+	__le16 regular_latency_phy_queue;
+	u8 reserved2[6];
 	__le32 src_gid[4];
 	__le32 dst_gid[4];
 };
 
+/* RoCE query qp requester output params */
 struct roce_query_qp_req_output_params {
 	__le32 psn;
 	__le32 flags;
-#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG_MASK          0x1
-#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG_SHIFT         0
-#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG_MASK  0x1
-#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG_SHIFT 1
-#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_RESERVED0_MASK        0x3FFFFFFF
-#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_RESERVED0_SHIFT       2
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG_MASK		0x1
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG_SHIFT		0
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG_MASK	0x1
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG_SHIFT	1
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_RESERVED0_MASK		0x3FFFFFFF
+#define ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_RESERVED0_SHIFT		2
 };
 
+/* RoCE query qp requester ramrod data */
 struct roce_query_qp_req_ramrod_data {
 	struct regpair output_params_addr;
 };
 
+/* RoCE query qp responder output params */
 struct roce_query_qp_resp_output_params {
 	__le32 psn;
 	__le32 err_flag;
@@ -7639,10 +8232,12 @@ struct roce_query_qp_resp_output_params {
 #define ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_RESERVED0_SHIFT 1
 };
 
+/* RoCE query qp responder ramrod data */
 struct roce_query_qp_resp_ramrod_data {
 	struct regpair output_params_addr;
 };
 
+/* ROCE ramrod command IDs */
 enum roce_ramrod_cmd_id {
 	ROCE_RAMROD_CREATE_QP = 11,
 	ROCE_RAMROD_MODIFY_QP,
@@ -7653,163 +8248,163 @@ enum roce_ramrod_cmd_id {
 	MAX_ROCE_RAMROD_CMD_ID
 };
 
-struct mstorm_roce_req_conn_ag_ctx {
+struct e4_mstorm_roce_req_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK     0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT    0
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK     0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT    1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK      0x3
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT     2
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK      0x3
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT     4
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK      0x3
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK		0x3
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK		0x3
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK    0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT   0
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK    0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT   1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK    0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT   2
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define MSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT	0
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT	1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT	2
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_MSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT	7
 	__le16 word0;
 	__le16 word1;
 	__le32 reg0;
 	__le32 reg1;
 };
 
-struct mstorm_roce_resp_conn_ag_ctx {
+struct e4_mstorm_roce_resp_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK     0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT    0
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK     0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT    1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK      0x3
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT     2
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK      0x3
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT     4
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK      0x3
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK    0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT   0
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK    0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT   1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK    0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT   2
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define MSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT	0
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT	1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT	2
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_MSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT	7
 	__le16 word0;
 	__le16 word1;
 	__le32 reg0;
 	__le32 reg1;
 };
 
-struct tstorm_roce_req_conn_ag_ctx {
+struct e4_tstorm_roce_req_conn_ag_ctx {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_MASK                0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_SHIFT               0
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_OCCURED_MASK            0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_OCCURED_SHIFT           1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_CQE_ERROR_OCCURED_MASK        0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_CQE_ERROR_OCCURED_SHIFT       2
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_BIT3_MASK                        0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_BIT3_SHIFT                       3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_MASK                0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_SHIFT               4
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_CACHED_ORQ_MASK                  0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_CACHED_ORQ_SHIFT                 5
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_MASK                    0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_SHIFT                   6
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_SHIFT		0
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_OCCURRED_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_OCCURRED_SHIFT		1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TX_CQE_ERROR_OCCURRED_MASK	0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TX_CQE_ERROR_OCCURRED_SHIFT	2
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_BIT3_MASK			0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_BIT3_SHIFT			3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_SHIFT		4
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_CACHED_ORQ_MASK			0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_CACHED_ORQ_SHIFT			5
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_MASK			0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_SHIFT			6
 	u8 flags1;
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK                         0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT                        0
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_MASK                 0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_SHIFT                2
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK           0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT          4
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_MASK                 0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT                6
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK				0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT			0
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_MASK			0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_SHIFT		2
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK		0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT		4
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_MASK			0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT		6
 	u8 flags2;
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK             0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT            0
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_MASK                0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_SHIFT               2
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_MASK           0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_SHIFT          4
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_MASK               0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_SHIFT              6
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK	0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT	0
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_MASK	0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_SHIFT	2
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_MASK	0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_SHIFT	4
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_MASK	0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_SHIFT	6
 	u8 flags3;
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_MASK     0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_SHIFT    0
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_MASK       0x3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_SHIFT      2
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_EN_MASK                 0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_EN_SHIFT                4
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK                       0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT                      5
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_EN_MASK              0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_EN_SHIFT             6
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK        0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT       7
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_MASK	0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_SHIFT	0
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_MASK	0x3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_SHIFT	2
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_EN_MASK			0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_CF_EN_SHIFT		4
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK			0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT			5
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_SQ_CF_EN_SHIFT		6
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK	0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT	7
 	u8 flags4;
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK              0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT             0
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK          0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT         1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_EN_MASK             0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_EN_SHIFT            2
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_EN_MASK        0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_EN_SHIFT       3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_EN_MASK            0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_EN_SHIFT           4
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_EN_MASK  0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_EN_SHIFT 5
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_EN_MASK    0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_EN_SHIFT   6
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT                    7
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT		0
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT		1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SET_TIMER_CF_EN_SHIFT		2
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_EN_MASK	0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_TX_ASYNC_ERROR_CF_EN_SHIFT	3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RXMIT_DONE_CF_EN_SHIFT		4
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_EN_MASK	0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_SCAN_COMPLETED_CF_EN_SHIFT	5
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_EN_MASK	0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SQ_DRAIN_COMPLETED_CF_EN_SHIFT	6
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK			0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT			7
 	u8 flags5;
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT                    0
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT                    1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT                    2
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT                    3
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT                    4
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SND_SQ_CONS_EN_MASK              0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_SND_SQ_CONS_EN_SHIFT             5
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_SHIFT                    6
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_MASK                     0x1
-#define TSTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_SHIFT                    7
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT		0
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT		1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT		2
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT		3
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT		4
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SND_SQ_CONS_EN_MASK	0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_SND_SQ_CONS_EN_SHIFT	5
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_SHIFT		6
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_TSTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_SHIFT		7
 	__le32 reg0;
 	__le32 snd_nxt_psn;
 	__le32 snd_max_psn;
@@ -7825,95 +8420,95 @@ struct tstorm_roce_req_conn_ag_ctx {
 	u8 byte4;
 	u8 byte5;
 	__le16 snd_sq_cons;
-	__le16 word2;
+	__le16 conn_dpi;
 	__le16 word3;
 	__le32 reg9;
 	__le32 reg10;
 };
 
-struct tstorm_roce_resp_conn_ag_ctx {
+struct e4_tstorm_roce_resp_conn_ag_ctx {
 	u8 byte0;
 	u8 state;
 	u8 flags0;
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_MASK        0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT       0
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_NOTIFY_REQUESTER_MASK  0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_NOTIFY_REQUESTER_SHIFT 1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT2_MASK                0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT2_SHIFT               2
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT3_MASK                0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT3_SHIFT               3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_MASK        0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_SHIFT       4
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT5_MASK                0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_BIT5_SHIFT               5
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK                 0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT                6
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT		0
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_NOTIFY_REQUESTER_MASK	0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_NOTIFY_REQUESTER_SHIFT	1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_BIT2_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_BIT2_SHIFT			2
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_BIT3_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_BIT3_SHIFT			3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_SHIFT		4
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_BIT5_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_BIT5_SHIFT			5
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK			0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT			6
 	u8 flags1;
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_MASK         0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_SHIFT        0
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_MASK         0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_SHIFT        2
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK                 0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT                4
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_MASK         0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT        6
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_MASK	0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_SHIFT	0
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_MASK	0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_SHIFT	2
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK		0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT		4
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_MASK	0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT	6
 	u8 flags2;
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK     0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT    0
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6_MASK                 0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6_SHIFT                2
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7_MASK                 0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7_SHIFT                4
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8_MASK                 0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8_SHIFT                6
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_MASK	0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_SHIFT	0
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF6_MASK		0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF6_SHIFT		2
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF7_MASK		0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF7_SHIFT		4
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF8_MASK		0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF8_SHIFT		6
 	u8 flags3;
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9_MASK                 0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9_SHIFT                0
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10_MASK                0x3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10_SHIFT               2
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK               0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT              4
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_MASK      0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT     5
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_EN_MASK      0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_EN_SHIFT     6
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK               0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT              7
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF9_MASK		0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF9_SHIFT		0
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF10_MASK		0x3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF10_SHIFT		2
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT		4
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_MASK	0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT	5
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_EN_MASK	0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_TX_ERROR_CF_EN_SHIFT	6
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT		7
 	u8 flags4;
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK      0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT     0
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK  0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT 1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_MASK               0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_SHIFT              2
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7EN_MASK               0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF7EN_SHIFT              3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_MASK               0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_SHIFT              4
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_MASK               0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_SHIFT              5
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_MASK              0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_SHIFT             6
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT            7
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT		0
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_MSTORM_FLUSH_CF_EN_SHIFT	1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_SHIFT			2
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF7EN_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF7EN_SHIFT			3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_SHIFT			4
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_SHIFT			5
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_SHIFT			6
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK			0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT			7
 	u8 flags5;
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT            0
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT            1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT            2
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT            3
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT            4
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RQ_RULE_EN_MASK          0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RQ_RULE_EN_SHIFT         5
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT            6
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_MASK             0x1
-#define TSTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_SHIFT            7
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT		0
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT		1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT		2
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT		3
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT		4
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RQ_RULE_EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RQ_RULE_EN_SHIFT	5
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT		6
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_TSTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_SHIFT		7
 	__le32 psn_and_rxmit_id_echo;
 	__le32 reg1;
 	__le32 reg2;
@@ -7935,63 +8530,63 @@ struct tstorm_roce_resp_conn_ag_ctx {
 	__le32 reg10;
 };
 
-struct ustorm_roce_req_conn_ag_ctx {
+struct e4_ustorm_roce_req_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK     0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT    0
-#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK     0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT    1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK      0x3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT     2
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK      0x3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT     4
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK      0x3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK		0x3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK		0x3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3_MASK      0x3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3_SHIFT     0
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4_MASK      0x3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4_SHIFT     2
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5_MASK      0x3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5_SHIFT     4
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6_MASK      0x3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6_SHIFT     6
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF3_MASK		0x3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF3_SHIFT	0
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF4_MASK		0x3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF4_SHIFT	2
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF5_MASK		0x3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF5_SHIFT	4
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF6_MASK		0x3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF6_SHIFT	6
 	u8 flags2;
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK    0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT   0
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK    0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT   1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK    0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT   2
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_MASK    0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_SHIFT   3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4EN_MASK    0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF4EN_SHIFT   4
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5EN_MASK    0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF5EN_SHIFT   5
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6EN_MASK    0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_CF6EN_SHIFT   6
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT 7
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT	0
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT	1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT	2
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_SHIFT	3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF4EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF4EN_SHIFT	4
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF5EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF5EN_SHIFT	5
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF6EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_CF6EN_SHIFT	6
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT	7
 	u8 flags3;
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT 0
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT 1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT 2
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT 3
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT 4
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_SHIFT 5
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_SHIFT 6
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_MASK  0x1
-#define USTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_SHIFT 7
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_MASK	0x1
+#define E4_USTORM_ROCE_REQ_CONN_AG_CTX_RULE8EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -8004,63 +8599,63 @@ struct ustorm_roce_req_conn_ag_ctx {
 	__le16 word3;
 };
 
-struct ustorm_roce_resp_conn_ag_ctx {
+struct e4_ustorm_roce_resp_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK     0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT    0
-#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK     0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT    1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK      0x3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT     2
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK      0x3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT     4
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK      0x3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK      0x3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT     0
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4_MASK      0x3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4_SHIFT     2
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5_MASK      0x3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5_SHIFT     4
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6_MASK      0x3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6_SHIFT     6
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT	0
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF4_SHIFT	2
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF5_SHIFT	4
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF6_SHIFT	6
 	u8 flags2;
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK    0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT   0
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK    0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT   1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK    0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT   2
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK    0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT   3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4EN_MASK    0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF4EN_SHIFT   4
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5EN_MASK    0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF5EN_SHIFT   5
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_MASK    0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_SHIFT   6
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT 7
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT	0
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT	1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT	2
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT	3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF4EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF4EN_SHIFT	4
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF5EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF5EN_SHIFT	5
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_CF6EN_SHIFT	6
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT	7
 	u8 flags3;
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT 0
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT 1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT 2
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT 3
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT 4
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_SHIFT 5
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT 6
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_MASK  0x1
-#define USTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_SHIFT 7
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_MASK	0x1
+#define E4_USTORM_ROCE_RESP_CONN_AG_CTX_RULE8EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -8073,214 +8668,214 @@ struct ustorm_roce_resp_conn_ag_ctx {
 	__le16 word3;
 };
 
-struct xstorm_roce_req_conn_ag_ctx {
+struct e4_xstorm_roce_req_conn_ag_ctx {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_SHIFT       0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED1_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED1_SHIFT          1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED2_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED2_SHIFT          2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM3_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM3_SHIFT       3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED3_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED3_SHIFT          4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED4_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED4_SHIFT          5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED5_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED5_SHIFT          6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED6_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED6_SHIFT          7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED1_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED1_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED2_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED2_SHIFT		2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED3_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED3_SHIFT		4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED4_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED4_SHIFT		5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED5_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED5_SHIFT		6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED6_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED6_SHIFT		7
 	u8 flags1;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED7_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED7_SHIFT          0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED8_MASK           0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED8_SHIFT          1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT10_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT10_SHIFT              2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT11_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT11_SHIFT              3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT12_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT12_SHIFT              4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT13_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT13_SHIFT              5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_STATE_MASK         0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_STATE_SHIFT        6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_YSTORM_FLUSH_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_YSTORM_FLUSH_SHIFT       7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED7_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED7_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED8_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED8_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT10_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT10_SHIFT		2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT11_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT12_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT12_SHIFT		4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT13_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT13_SHIFT		5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_STATE_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_ERROR_STATE_SHIFT	6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_YSTORM_FLUSH_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_YSTORM_FLUSH_SHIFT	7
 	u8 flags2;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK                 0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT                0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK                 0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT                2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK                 0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT                4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3_MASK                 0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3_SHIFT                6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT	0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF3_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_MASK         0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_SHIFT        0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_MASK         0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_SHIFT        2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_MASK        0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_SHIFT       4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_MASK         0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT        6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_SHIFT	0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT	6
 	u8 flags4;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8_MASK                 0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8_SHIFT                0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9_MASK                 0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9_SHIFT                2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10_SHIFT               4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11_SHIFT               6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF8_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF9_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12_SHIFT               0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13_SHIFT               2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FMR_ENDED_CF_MASK        0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FMR_ENDED_CF_SHIFT       4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15_SHIFT               6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF12_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF12_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF13_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF13_SHIFT		2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FMR_ENDED_CF_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FMR_ENDED_CF_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF15_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF15_SHIFT		6
 	u8 flags6;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16_SHIFT               0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17_SHIFT               2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18_SHIFT               4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19_SHIFT               6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF16_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF16_SHIFT	0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF17_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF17_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF18_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF18_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF19_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF19_SHIFT	6
 	u8 flags7;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20_SHIFT               0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21_SHIFT               2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_MASK           0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_SHIFT          4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT              6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT              7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF20_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF20_SHIFT	0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF21_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF21_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT	6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT	7
 	u8 flags8;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT              0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_SHIFT              1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_EN_MASK      0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_EN_SHIFT     2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_EN_MASK      0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT     3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_EN_MASK     0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_EN_SHIFT    4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK      0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT     5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8EN_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF8EN_SHIFT              6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9EN_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF9EN_SHIFT              7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF3EN_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_FLUSH_CF_EN_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT	3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SND_RXMIT_CF_EN_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT	5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF8EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF8EN_SHIFT		6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF9EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF9EN_SHIFT		7
 	u8 flags9;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF10EN_SHIFT             0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF11EN_SHIFT             1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF12EN_SHIFT             2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF13EN_SHIFT             3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FME_ENDED_CF_EN_MASK     0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_FME_ENDED_CF_EN_SHIFT    4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF15EN_SHIFT             5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF16EN_SHIFT             6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF17EN_SHIFT             7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF10EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF10EN_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF11EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF11EN_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF12EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF12EN_SHIFT		2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF13EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF13EN_SHIFT		3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FME_ENDED_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_FME_ENDED_CF_EN_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF15EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF15EN_SHIFT		5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF16EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF16EN_SHIFT		6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF17EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF17EN_SHIFT		7
 	u8 flags10;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF18EN_SHIFT             0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF19EN_SHIFT             1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF20EN_SHIFT             2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF21EN_SHIFT             3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_EN_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_EN_SHIFT       4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23EN_MASK              0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23EN_SHIFT             5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT            6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT            7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF18EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF18EN_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF19EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF19EN_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF20EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF20EN_SHIFT		2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF21EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF21EN_SHIFT		3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SLOW_PATH_EN_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF23EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF23EN_SHIFT		5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT		6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT		7
 	u8 flags11;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT            0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT            1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT            2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT            3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_SHIFT            4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_E2E_CREDIT_RULE_EN_MASK  0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_E2E_CREDIT_RULE_EN_SHIFT 5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED1_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED1_SHIFT       6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE9EN_MASK             0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE9EN_SHIFT            7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT		2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE5EN_SHIFT		3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE6EN_SHIFT		4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_E2E_CREDIT_RULE_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_E2E_CREDIT_RULE_EN_SHIFT	5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE9EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE9EN_SHIFT		7
 	u8 flags12;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_PROD_EN_MASK          0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_PROD_EN_SHIFT         0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE11EN_MASK            0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE11EN_SHIFT           1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED2_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED2_SHIFT       2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED3_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED3_SHIFT       3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_INV_FENCE_RULE_EN_MASK   0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_INV_FENCE_RULE_EN_SHIFT  4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE15EN_MASK            0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE15EN_SHIFT           5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_ORQ_FENCE_RULE_EN_MASK   0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_ORQ_FENCE_RULE_EN_SHIFT  6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_MAX_ORD_RULE_EN_MASK     0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_MAX_ORD_RULE_EN_SHIFT    7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_PROD_EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_SQ_PROD_EN_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE11EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE11EN_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_INV_FENCE_RULE_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_INV_FENCE_RULE_EN_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE15EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE15EN_SHIFT		5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_ORQ_FENCE_RULE_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_ORQ_FENCE_RULE_EN_SHIFT	6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_MAX_ORD_RULE_EN_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_MAX_ORD_RULE_EN_SHIFT	7
 	u8 flags13;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE18EN_MASK            0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE18EN_SHIFT           0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE19EN_MASK            0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RULE19EN_SHIFT           1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED4_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED4_SHIFT       2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED5_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED5_SHIFT       3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED6_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED6_SHIFT       4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED7_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED7_SHIFT       5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED8_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED8_SHIFT       6
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED9_MASK        0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED9_SHIFT       7
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE18EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE18EN_SHIFT		0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE19EN_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RULE19EN_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
 	u8 flags14;
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_MIGRATION_FLAG_MASK      0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_MIGRATION_FLAG_SHIFT     0
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT17_MASK               0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_BIT17_SHIFT              1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_DPM_PORT_NUM_MASK        0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_DPM_PORT_NUM_SHIFT       2
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED_MASK            0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED_SHIFT           4
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK    0x1
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT   5
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23_MASK                0x3
-#define XSTORM_ROCE_REQ_CONN_AG_CTX_CF23_SHIFT               6
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_MIGRATION_FLAG_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_MIGRATION_FLAG_SHIFT	0
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT17_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_BIT17_SHIFT		1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_DPM_PORT_NUM_MASK	0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_DPM_PORT_NUM_SHIFT	2
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED_MASK		0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_RESERVED_SHIFT		4
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_ROCE_EDPM_ENABLE_MASK	0x1
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_ROCE_EDPM_ENABLE_SHIFT	5
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF23_MASK		0x3
+#define E4_XSTORM_ROCE_REQ_CONN_AG_CTX_CF23_SHIFT		6
 	u8 byte2;
 	__le16 physical_q0;
 	__le16 word1;
@@ -8302,224 +8897,224 @@ struct xstorm_roce_req_conn_ag_ctx {
 	__le32 orq_cons;
 };
 
-struct xstorm_roce_resp_conn_ag_ctx {
+struct e4_xstorm_roce_resp_conn_ag_ctx {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT     0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED1_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED1_SHIFT        1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED2_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED2_SHIFT        2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM3_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM3_SHIFT     3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED3_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED3_SHIFT        4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED4_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED4_SHIFT        5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED5_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED5_SHIFT        6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED6_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED6_SHIFT        7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED1_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED1_SHIFT		1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED2_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED2_SHIFT		2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED3_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED3_SHIFT		4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED4_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED4_SHIFT		5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED5_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED5_SHIFT		6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED6_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED6_SHIFT		7
 	u8 flags1;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED7_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED7_SHIFT        0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED8_MASK         0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED8_SHIFT        1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT10_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT10_SHIFT            2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT11_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT11_SHIFT            3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT12_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT12_SHIFT            4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT13_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT13_SHIFT            5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_ERROR_STATE_MASK       0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_ERROR_STATE_SHIFT      6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_YSTORM_FLUSH_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_YSTORM_FLUSH_SHIFT     7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED7_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED7_SHIFT		0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED8_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RESERVED8_SHIFT		1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT10_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT10_SHIFT		2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT11_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT12_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT12_SHIFT		4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT13_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT13_SHIFT		5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_ERROR_STATE_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_ERROR_STATE_SHIFT	6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_YSTORM_FLUSH_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_YSTORM_FLUSH_SHIFT	7
 	u8 flags2;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK               0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT              0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK               0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT              2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK               0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT              4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK               0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT              6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_MASK          0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_SHIFT         0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_MASK       0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_SHIFT      2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_MASK      0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_SHIFT     4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_MASK       0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT      6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_MASK		0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_SHIFT		0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT	6
 	u8 flags4;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8_MASK               0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8_SHIFT              0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9_MASK               0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9_SHIFT              2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10_SHIFT             4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11_SHIFT             6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12_SHIFT             0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13_SHIFT             2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14_SHIFT             4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15_SHIFT             6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF12_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF12_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF13_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF13_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF14_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF14_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF15_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF15_SHIFT	6
 	u8 flags6;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16_SHIFT             0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17_SHIFT             2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18_SHIFT             4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19_SHIFT             6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF16_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF16_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF17_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF17_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF18_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF18_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF19_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF19_SHIFT	6
 	u8 flags7;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20_SHIFT             0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21_SHIFT             2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_MASK         0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_SHIFT        4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT            6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT            7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF20_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF20_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF21_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF21_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT	6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT	7
 	u8 flags8;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT            0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT            1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_EN_MASK       0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_EN_SHIFT      2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_MASK    0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT   3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_EN_MASK   0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_EN_SHIFT  4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK    0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT   5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_SHIFT            6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_SHIFT            7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT		0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF3EN_SHIFT		1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RXMIT_CF_EN_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RX_ERROR_CF_EN_SHIFT	3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FORCE_ACK_CF_EN_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT	5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF8EN_SHIFT		6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF9EN_SHIFT		7
 	u8 flags9;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_SHIFT           0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF11EN_SHIFT           1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF12EN_SHIFT           2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF13EN_SHIFT           3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF14EN_SHIFT           4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF15EN_SHIFT           5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF16EN_SHIFT           6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF17EN_SHIFT           7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF10EN_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF11EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF11EN_SHIFT	1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF12EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF12EN_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF13EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF13EN_SHIFT	3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF14EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF14EN_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF15EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF15EN_SHIFT	5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF16EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF16EN_SHIFT	6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF17EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF17EN_SHIFT	7
 	u8 flags10;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF18EN_SHIFT           0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF19EN_SHIFT           1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF20EN_SHIFT           2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF21EN_SHIFT           3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_EN_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_EN_SHIFT     4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23EN_MASK            0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23EN_SHIFT           5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT          6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT          7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF18EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF18EN_SHIFT		0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF19EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF19EN_SHIFT		1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF20EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF20EN_SHIFT		2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF21EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF21EN_SHIFT		3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_SLOW_PATH_EN_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF23EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF23EN_SHIFT		5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT		6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT		7
 	u8 flags11;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT          0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT          1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT          2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT          3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_SHIFT          4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT          5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED1_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED1_SHIFT     6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE9EN_MASK           0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE9EN_SHIFT          7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT		0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT		1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT		2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE5EN_SHIFT		3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE6EN_SHIFT		4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE7EN_SHIFT		5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE9EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE9EN_SHIFT		7
 	u8 flags12;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE10EN_MASK          0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE10EN_SHIFT         0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_IRQ_PROD_RULE_EN_MASK  0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_IRQ_PROD_RULE_EN_SHIFT 1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED2_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED2_SHIFT     2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED3_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED3_SHIFT     3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE14EN_MASK          0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE14EN_SHIFT         4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE15EN_MASK          0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE15EN_SHIFT         5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE16EN_MASK          0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE16EN_SHIFT         6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE17EN_MASK          0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE17EN_SHIFT         7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_IRQ_PROD_RULE_EN_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_IRQ_PROD_RULE_EN_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE11EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE11EN_SHIFT		1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE14EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE14EN_SHIFT		4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE15EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE15EN_SHIFT		5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE16EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE16EN_SHIFT		6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE17EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE17EN_SHIFT		7
 	u8 flags13;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE18EN_MASK          0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE18EN_SHIFT         0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE19EN_MASK          0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_RULE19EN_SHIFT         1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED4_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED4_SHIFT     2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED5_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED5_SHIFT     3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED6_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED6_SHIFT     4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED7_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED7_SHIFT     5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED8_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED8_SHIFT     6
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED9_MASK      0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED9_SHIFT     7
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE18EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE18EN_SHIFT		0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE19EN_MASK		0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_RULE19EN_SHIFT		1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED4_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED4_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED5_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED5_SHIFT	3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED7_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED7_SHIFT	5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
 	u8 flags14;
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT16_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT16_SHIFT            0
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT17_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT17_SHIFT            1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT18_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT18_SHIFT            2
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT19_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT19_SHIFT            3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT20_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT20_SHIFT            4
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT21_MASK             0x1
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_BIT21_SHIFT            5
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23_MASK              0x3
-#define XSTORM_ROCE_RESP_CONN_AG_CTX_CF23_SHIFT             6
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT16_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT16_SHIFT	0
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT17_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT17_SHIFT	1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT18_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT18_SHIFT	2
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT19_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT19_SHIFT	3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT20_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT20_SHIFT	4
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT21_MASK	0x1
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_BIT21_SHIFT	5
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF23_MASK	0x3
+#define E4_XSTORM_ROCE_RESP_CONN_AG_CTX_CF23_SHIFT	6
 	u8 byte2;
 	__le16 physical_q0;
-	__le16 word1;
-	__le16 irq_prod;
-	__le16 word3;
-	__le16 word4;
-	__le16 ereserved1;
+	__le16 irq_prod_shadow;
+	__le16 word2;
 	__le16 irq_cons;
+	__le16 irq_prod;
+	__le16 e5_reserved1;
+	__le16 conn_dpi;
 	u8 rxmit_opcode;
 	u8 byte4;
 	u8 byte5;
@@ -8533,37 +9128,37 @@ struct xstorm_roce_resp_conn_ag_ctx {
 	__le32 msn_and_syndrome;
 };
 
-struct ystorm_roce_req_conn_ag_ctx {
+struct e4_ystorm_roce_req_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK     0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT    0
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK     0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT    1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK      0x3
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT     2
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK      0x3
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT     4
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK      0x3
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF1_MASK		0x3
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF2_MASK		0x3
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK    0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT   0
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK    0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT   1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK    0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT   2
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define YSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF0EN_SHIFT	0
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF1EN_SHIFT	1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_CF2EN_SHIFT	2
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_YSTORM_ROCE_REQ_CONN_AG_CTX_RULE4EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -8577,37 +9172,37 @@ struct ystorm_roce_req_conn_ag_ctx {
 	__le32 reg3;
 };
 
-struct ystorm_roce_resp_conn_ag_ctx {
+struct e4_ystorm_roce_resp_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK     0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT    0
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK     0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT    1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK      0x3
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT     2
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK      0x3
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT     4
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK      0x3
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK    0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT   0
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK    0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT   1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK    0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT   2
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define YSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF0EN_SHIFT	0
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF1EN_SHIFT	1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_CF2EN_SHIFT	2
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_YSTORM_ROCE_RESP_CONN_AG_CTX_RULE4EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -8621,6 +9216,7 @@ struct ystorm_roce_resp_conn_ag_ctx {
 	__le32 reg3;
 };
 
+/* Roce doorbell data */
 enum roce_flavor {
 	PLAIN_ROCE,
 	RROCE_IPV4,
@@ -8628,228 +9224,231 @@ enum roce_flavor {
 	MAX_ROCE_FLAVOR
 };
 
+/* The iwarp storm context of Ystorm */
 struct ystorm_iwarp_conn_st_ctx {
 	__le32 reserved[4];
 };
 
+/* The iwarp storm context of Pstorm */
 struct pstorm_iwarp_conn_st_ctx {
 	__le32 reserved[36];
 };
 
+/* The iwarp storm context of Xstorm */
 struct xstorm_iwarp_conn_st_ctx {
 	__le32 reserved[44];
 };
 
-struct xstorm_iwarp_conn_ag_ctx {
+struct e4_xstorm_iwarp_conn_ag_ctx {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM1_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM1_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM2_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM2_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_BIT4_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT4_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_RESERVED2_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RESERVED2_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_BIT6_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT6_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_BIT7_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT7_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM1_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM1_SHIFT	1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM2_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM2_SHIFT	2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT4_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT4_SHIFT		4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RESERVED2_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RESERVED2_SHIFT	5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT6_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT6_SHIFT		6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT7_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT7_SHIFT		7
 	u8 flags1;
-#define XSTORM_IWARP_CONN_AG_CTX_BIT8_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT8_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_BIT9_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT9_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT10_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT10_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_BIT11_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT11_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_BIT12_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT12_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_BIT13_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT13_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_BIT14_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT14_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_YSTORM_FLUSH_OR_REWIND_SND_MAX_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_YSTORM_FLUSH_OR_REWIND_SND_MAX_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT8_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT8_SHIFT				0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT9_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT9_SHIFT				1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT10_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT10_SHIFT				2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT11_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT11_SHIFT				3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT12_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT12_SHIFT				4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT13_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT13_SHIFT				5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT14_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT14_SHIFT				6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_YSTORM_FLUSH_OR_REWIND_SND_MAX_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_YSTORM_FLUSH_OR_REWIND_SND_MAX_SHIFT 7
 	u8 flags2;
-#define XSTORM_IWARP_CONN_AG_CTX_CF0_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF0_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_CF1_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF1_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_CF2_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF2_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT	6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF0_MASK			0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF0_SHIFT			0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF1_MASK			0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF1_SHIFT			2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF2_MASK			0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF2_SHIFT			4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_MASK		0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT	6
 	u8 flags3;
-#define XSTORM_IWARP_CONN_AG_CTX_CF4_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF4_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_CF5_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF5_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_CF6_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF6_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_CF7_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF7_SHIFT	6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF4_SHIFT	0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF5_SHIFT	2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF6_SHIFT	4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF7_SHIFT	6
 	u8 flags4;
-#define XSTORM_IWARP_CONN_AG_CTX_CF8_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF8_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_CF9_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF9_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_CF10_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF10_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_CF11_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF11_SHIFT	6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_IWARP_CONN_AG_CTX_CF12_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF12_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_CF13_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF13_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_CF15_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF15_SHIFT	6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF12_MASK		0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF12_SHIFT		0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF13_MASK		0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF13_SHIFT		2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_SHIFT	4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF15_MASK		0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF15_SHIFT		6
 	u8 flags6;
-#define XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_CF17_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF17_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_CF18_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF18_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_SHIFT	6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_SHIFT 0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF17_MASK				0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF17_SHIFT				2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF18_MASK				0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF18_SHIFT				4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_MASK			0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_SHIFT			6
 	u8 flags7;
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_CF0EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_CF1EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_SHIFT	0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_SHIFT	2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT		6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT		7
 	u8 flags8;
-#define XSTORM_IWARP_CONN_AG_CTX_CF2EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_CF4EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF4EN_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_CF5EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF5EN_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_CF6EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF6EN_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_CF7EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF7EN_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_CF8EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF8EN_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_CF9EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF9EN_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT			0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT	1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF4EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF4EN_SHIFT			2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF5EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF5EN_SHIFT			3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF6EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF6EN_SHIFT			4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF7EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF7EN_SHIFT			5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF8EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF8EN_SHIFT			6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF9EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF9EN_SHIFT			7
 	u8 flags9;
-#define XSTORM_IWARP_CONN_AG_CTX_CF10EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF10EN_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_CF11EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF11EN_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_CF12EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF12EN_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_CF13EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF13EN_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_EN_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_CF15EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF15EN_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_EN_SHIFT 6
-#define XSTORM_IWARP_CONN_AG_CTX_CF17EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF17EN_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF10EN_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF10EN_SHIFT			0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF11EN_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF11EN_SHIFT			1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF12EN_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF12EN_SHIFT			2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF13EN_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF13EN_SHIFT			3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_FLUSH_CF_EN_SHIFT		4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF15EN_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF15EN_SHIFT			5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_EN_MASK 0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_MPA_OR_ERROR_WAKEUP_TRIGGER_CF_EN_SHIFT 6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF17EN_MASK				0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF17EN_SHIFT			7
 	u8 flags10;
-#define XSTORM_IWARP_CONN_AG_CTX_CF18EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF18EN_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_EN_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_EN_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_EN_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_CF23EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_CF23EN_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_MORE_TO_SEND_RULE_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_MORE_TO_SEND_RULE_EN_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF18EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF18EN_SHIFT		0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_DQ_FLUSH_EN_SHIFT		1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT		2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_FLUSH_Q1_EN_SHIFT		3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SLOW_PATH_EN_SHIFT		4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF23EN_MASK			0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF23EN_SHIFT		5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT		6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_MORE_TO_SEND_RULE_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_MORE_TO_SEND_RULE_EN_SHIFT	7
 	u8 flags11;
-#define XSTORM_IWARP_CONN_AG_CTX_TX_BLOCKED_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_TX_BLOCKED_EN_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_RESERVED3_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RESERVED3_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_RULE5EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE5EN_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_RULE6EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE6EN_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_RULE7EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE7EN_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_RULE9EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE9EN_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_TX_BLOCKED_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_TX_BLOCKED_EN_SHIFT	0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT	1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RESERVED3_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RESERVED3_SHIFT	2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE5EN_SHIFT	3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE6EN_SHIFT	4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE7EN_SHIFT	5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE9EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE9EN_SHIFT	7
 	u8 flags12;
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_NOT_EMPTY_RULE_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_NOT_EMPTY_RULE_EN_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_RULE11EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE11EN_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_FENCE_RULE_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_SQ_FENCE_RULE_EN_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_RULE15EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE15EN_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_RULE16EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE16EN_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_RULE17EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE17EN_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_NOT_EMPTY_RULE_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_NOT_EMPTY_RULE_EN_SHIFT	0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE11EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE11EN_SHIFT		1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED2_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED2_SHIFT		2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED3_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED3_SHIFT		3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_FENCE_RULE_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_SQ_FENCE_RULE_EN_SHIFT	4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE15EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE15EN_SHIFT		5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE16EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE16EN_SHIFT		6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE17EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE17EN_SHIFT		7
 	u8 flags13;
-#define XSTORM_IWARP_CONN_AG_CTX_IRQ_NOT_EMPTY_RULE_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_IRQ_NOT_EMPTY_RULE_EN_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_HQ_NOT_FULL_RULE_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_HQ_NOT_FULL_RULE_EN_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_ORQ_RD_FENCE_RULE_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_ORQ_RD_FENCE_RULE_EN_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_RULE21EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_RULE21EN_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED6_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED6_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_ORQ_NOT_FULL_RULE_EN_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_ORQ_NOT_FULL_RULE_EN_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED8_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED8_SHIFT	6
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED9_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED9_SHIFT	7
+#define E4_XSTORM_IWARP_CONN_AG_CTX_IRQ_NOT_EMPTY_RULE_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_IRQ_NOT_EMPTY_RULE_EN_SHIFT	0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_HQ_NOT_FULL_RULE_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_HQ_NOT_FULL_RULE_EN_SHIFT	1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_ORQ_RD_FENCE_RULE_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_ORQ_RD_FENCE_RULE_EN_SHIFT	2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE21EN_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_RULE21EN_SHIFT		3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED6_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED6_SHIFT		4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_ORQ_NOT_FULL_RULE_EN_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_ORQ_NOT_FULL_RULE_EN_SHIFT	5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED8_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED8_SHIFT		6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED9_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_A0_RESERVED9_SHIFT		7
 	u8 flags14;
-#define XSTORM_IWARP_CONN_AG_CTX_BIT16_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT16_SHIFT	0
-#define XSTORM_IWARP_CONN_AG_CTX_BIT17_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT17_SHIFT	1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT18_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_BIT18_SHIFT	2
-#define XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED1_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED1_SHIFT	3
-#define XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED2_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED2_SHIFT	4
-#define XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED3_MASK	0x1
-#define XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED3_SHIFT	5
-#define XSTORM_IWARP_CONN_AG_CTX_CF23_MASK	0x3
-#define XSTORM_IWARP_CONN_AG_CTX_CF23_SHIFT	6
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT16_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT16_SHIFT		0
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT17_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT17_SHIFT		1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT18_MASK		0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_BIT18_SHIFT		2
+#define E4_XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED1_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED1_SHIFT	3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED2_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED2_SHIFT	4
+#define E4_XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED3_MASK	0x1
+#define E4_XSTORM_IWARP_CONN_AG_CTX_E5_RESERVED3_SHIFT	5
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF23_MASK		0x3
+#define E4_XSTORM_IWARP_CONN_AG_CTX_CF23_SHIFT		6
 	u8 byte2;
 	__le16 physical_q0;
 	__le16 physical_q1;
@@ -8897,89 +9496,89 @@ struct xstorm_iwarp_conn_ag_ctx {
 	__le32 reg17;
 };
 
-struct tstorm_iwarp_conn_ag_ctx {
+struct e4_tstorm_iwarp_conn_ag_ctx {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define TSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
-#define TSTORM_IWARP_CONN_AG_CTX_BIT1_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT	1
-#define TSTORM_IWARP_CONN_AG_CTX_BIT2_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_BIT2_SHIFT	2
-#define TSTORM_IWARP_CONN_AG_CTX_MSTORM_FLUSH_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_MSTORM_FLUSH_SHIFT	3
-#define TSTORM_IWARP_CONN_AG_CTX_BIT4_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_BIT4_SHIFT	4
-#define TSTORM_IWARP_CONN_AG_CTX_CACHED_ORQ_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_CACHED_ORQ_SHIFT	5
-#define TSTORM_IWARP_CONN_AG_CTX_CF0_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_CF0_SHIFT	6
+#define E4_TSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_TSTORM_IWARP_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_BIT2_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_BIT2_SHIFT		2
+#define E4_TSTORM_IWARP_CONN_AG_CTX_MSTORM_FLUSH_MASK	0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_MSTORM_FLUSH_SHIFT	3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_BIT4_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_BIT4_SHIFT		4
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CACHED_ORQ_MASK	0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CACHED_ORQ_SHIFT	5
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF0_SHIFT		6
 	u8 flags1;
-#define TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_SHIFT	0
-#define TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_SHIFT	2
-#define TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT	4
-#define TSTORM_IWARP_CONN_AG_CTX_CF4_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_CF4_SHIFT	6
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_MASK		0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_SHIFT		0
+#define E4_TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_MASK		0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_SHIFT	2
+#define E4_TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_MASK		0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT	4
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF4_MASK			0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF4_SHIFT			6
 	u8 flags2;
-#define TSTORM_IWARP_CONN_AG_CTX_CF5_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_CF5_SHIFT	0
-#define TSTORM_IWARP_CONN_AG_CTX_CF6_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_CF6_SHIFT	2
-#define TSTORM_IWARP_CONN_AG_CTX_CF7_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_CF7_SHIFT	4
-#define TSTORM_IWARP_CONN_AG_CTX_CF8_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_CF8_SHIFT	6
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF5_SHIFT	0
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF6_SHIFT	2
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF7_SHIFT	4
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF8_SHIFT	6
 	u8 flags3;
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_SHIFT	0
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_MASK	0x3
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_SHIFT	2
-#define TSTORM_IWARP_CONN_AG_CTX_CF0EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT	4
-#define TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_EN_SHIFT	5
-#define TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_EN_SHIFT	6
-#define TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT	7
+#define E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_AND_TCP_HANDSHAKE_COMPLETE_MASK 0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_AND_TCP_HANDSHAKE_COMPLETE_SHIFT 0
+#define E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_MASK	0x3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_SHIFT	2
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF0EN_MASK				0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT				4
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_EN_MASK			0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RQ_POST_CF_EN_SHIFT			5
+#define E4_TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_MPA_TIMEOUT_CF_EN_SHIFT		6
+#define E4_TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT		7
 	u8 flags4;
-#define TSTORM_IWARP_CONN_AG_CTX_CF4EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_CF4EN_SHIFT	0
-#define TSTORM_IWARP_CONN_AG_CTX_CF5EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_CF5EN_SHIFT	1
-#define TSTORM_IWARP_CONN_AG_CTX_CF6EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_CF6EN_SHIFT	2
-#define TSTORM_IWARP_CONN_AG_CTX_CF7EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_CF7EN_SHIFT	3
-#define TSTORM_IWARP_CONN_AG_CTX_CF8EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_CF8EN_SHIFT	4
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT	5
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_EN_SHIFT 6
-#define TSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT	7
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF4EN_MASK				0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF4EN_SHIFT				0
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF5EN_MASK				0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF5EN_SHIFT				1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF6EN_MASK				0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF6EN_SHIFT				2
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF7EN_MASK				0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF7EN_SHIFT				3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF8EN_MASK				0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_CF8EN_SHIFT				4
+#define E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_AND_TCP_HANDSHAKE_COMPL_EN_MASK 0x1
+#define	E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_Q0_AND_TCP_HANDSHAKE_COMPL_EN_SHIFT 5
+#define E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_EN_MASK	0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_FLUSH_OR_ERROR_DETECTED_EN_SHIFT	6
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK			0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT			7
 	u8 flags5;
-#define TSTORM_IWARP_CONN_AG_CTX_RULE1EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE1EN_SHIFT	0
-#define TSTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT	1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT	2
-#define TSTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT	3
-#define TSTORM_IWARP_CONN_AG_CTX_RULE5EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE5EN_SHIFT	4
-#define TSTORM_IWARP_CONN_AG_CTX_SND_SQ_CONS_RULE_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_SND_SQ_CONS_RULE_SHIFT	5
-#define TSTORM_IWARP_CONN_AG_CTX_RULE7EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE7EN_SHIFT	6
-#define TSTORM_IWARP_CONN_AG_CTX_RULE8EN_MASK	0x1
-#define TSTORM_IWARP_CONN_AG_CTX_RULE8EN_SHIFT	7
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE1EN_SHIFT		0
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT		1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT		2
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT		3
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE5EN_SHIFT		4
+#define E4_TSTORM_IWARP_CONN_AG_CTX_SND_SQ_CONS_RULE_MASK	0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_SND_SQ_CONS_RULE_SHIFT	5
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE7EN_SHIFT		6
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_TSTORM_IWARP_CONN_AG_CTX_RULE8EN_SHIFT		7
 	__le32 reg0;
 	__le32 reg1;
 	__le32 unaligned_nxt_seq;
@@ -9001,51 +9600,56 @@ struct tstorm_iwarp_conn_ag_ctx {
 	__le32 last_hq_sequence;
 };
 
+/* The iwarp storm context of Tstorm */
 struct tstorm_iwarp_conn_st_ctx {
 	__le32 reserved[60];
 };
 
+/* The iwarp storm context of Mstorm */
 struct mstorm_iwarp_conn_st_ctx {
 	__le32 reserved[32];
 };
 
+/* The iwarp storm context of Ustorm */
 struct ustorm_iwarp_conn_st_ctx {
 	__le32 reserved[24];
 };
 
-struct iwarp_conn_context {
+/* iwarp connection context */
+struct e4_iwarp_conn_context {
 	struct ystorm_iwarp_conn_st_ctx ystorm_st_context;
 	struct regpair ystorm_st_padding[2];
 	struct pstorm_iwarp_conn_st_ctx pstorm_st_context;
 	struct regpair pstorm_st_padding[2];
 	struct xstorm_iwarp_conn_st_ctx xstorm_st_context;
 	struct regpair xstorm_st_padding[2];
-	struct xstorm_iwarp_conn_ag_ctx xstorm_ag_context;
-	struct tstorm_iwarp_conn_ag_ctx tstorm_ag_context;
+	struct e4_xstorm_iwarp_conn_ag_ctx xstorm_ag_context;
+	struct e4_tstorm_iwarp_conn_ag_ctx tstorm_ag_context;
 	struct timers_context timer_context;
-	struct ustorm_rdma_conn_ag_ctx ustorm_ag_context;
+	struct e4_ustorm_rdma_conn_ag_ctx ustorm_ag_context;
 	struct tstorm_iwarp_conn_st_ctx tstorm_st_context;
 	struct regpair tstorm_st_padding[2];
 	struct mstorm_iwarp_conn_st_ctx mstorm_st_context;
 	struct ustorm_iwarp_conn_st_ctx ustorm_st_context;
 };
 
+/* iWARP create QP params passed by driver to FW in CreateQP Request Ramrod */
 struct iwarp_create_qp_ramrod_data {
 	u8 flags;
 #define IWARP_CREATE_QP_RAMROD_DATA_FMR_AND_RESERVED_EN_MASK	0x1
-#define IWARP_CREATE_QP_RAMROD_DATA_FMR_AND_RESERVED_EN_SHIFT 0
-#define IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP_MASK	0x1
-#define IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP_SHIFT	1
-#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN_MASK	0x1
-#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN_SHIFT	2
-#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN_MASK	0x1
-#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN_SHIFT	3
-#define IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN_MASK	0x1
-#define IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN_SHIFT	4
-#define IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG_MASK	0x1
-#define IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG_SHIFT	5
-#define IWARP_CREATE_QP_RAMROD_DATA_RESERVED0_MASK	0x3
-#define IWARP_CREATE_QP_RAMROD_DATA_RESERVED0_SHIFT	6
+#define IWARP_CREATE_QP_RAMROD_DATA_FMR_AND_RESERVED_EN_SHIFT	0
+#define IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP_MASK		0x1
+#define IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP_SHIFT		1
+#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN_MASK		0x1
+#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN_SHIFT		2
+#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN_MASK		0x1
+#define IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN_SHIFT		3
+#define IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN_MASK		0x1
+#define IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN_SHIFT		4
+#define IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG_MASK		0x1
+#define IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG_SHIFT		5
+#define IWARP_CREATE_QP_RAMROD_DATA_RESERVED0_MASK		0x3
+#define IWARP_CREATE_QP_RAMROD_DATA_RESERVED0_SHIFT		6
 	u8 reserved1;
 	__le16 pd;
 	__le16 sq_num_pages;
@@ -9061,6 +9665,7 @@ struct iwarp_create_qp_ramrod_data {
 	u8 reserved2[6];
 };
 
+/* iWARP completion queue types */
 enum iwarp_eqe_async_opcode {
 	IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE,
 	IWARP_EVENT_TYPE_ASYNC_ENHANCED_MPA_REPLY_ARRIVED,
@@ -9083,6 +9688,7 @@ struct iwarp_eqe_data_tcp_async_completion {
 	u8 reserved[5];
 };
 
+/* iWARP completion queue types */
 enum iwarp_eqe_sync_opcode {
 	IWARP_EVENT_TYPE_TCP_OFFLOAD =
 	11,
@@ -9095,6 +9701,7 @@ enum iwarp_eqe_sync_opcode {
 	MAX_IWARP_EQE_SYNC_OPCODE
 };
 
+/* iWARP EQE completion status */
 enum iwarp_fw_return_code {
 	IWARP_CONN_ERROR_TCP_CONNECT_INVALID_PACKET = 5,
 	IWARP_CONN_ERROR_TCP_CONNECTION_RST,
@@ -9125,54 +9732,60 @@ enum iwarp_fw_return_code {
 	MAX_IWARP_FW_RETURN_CODE
 };
 
+/* unaligned opaque data received from LL2 */
 struct iwarp_init_func_params {
 	u8 ll2_ooo_q_index;
 	u8 reserved1[7];
 };
 
+/* iwarp func init ramrod data */
 struct iwarp_init_func_ramrod_data {
 	struct rdma_init_func_ramrod_data rdma;
 	struct tcp_init_params tcp;
 	struct iwarp_init_func_params iwarp;
 };
 
+/* iWARP QP - possible states to transition to */
 enum iwarp_modify_qp_new_state_type {
 	IWARP_MODIFY_QP_STATE_CLOSING = 1,
-	IWARP_MODIFY_QP_STATE_ERROR =
-	2,
+	IWARP_MODIFY_QP_STATE_ERROR = 2,
 	MAX_IWARP_MODIFY_QP_NEW_STATE_TYPE
 };
 
+/* iwarp modify qp responder ramrod data */
 struct iwarp_modify_qp_ramrod_data {
 	__le16 transition_to_state;
 	__le16 flags;
-#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_RD_EN_MASK	0x1
-#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_RD_EN_SHIFT	0
-#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_WR_EN_MASK	0x1
-#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_WR_EN_SHIFT	1
-#define IWARP_MODIFY_QP_RAMROD_DATA_ATOMIC_EN_MASK	0x1
-#define IWARP_MODIFY_QP_RAMROD_DATA_ATOMIC_EN_SHIFT	2
-#define IWARP_MODIFY_QP_RAMROD_DATA_STATE_TRANS_EN_MASK	0x1
+#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_RD_EN_MASK		0x1
+#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_RD_EN_SHIFT		0
+#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_WR_EN_MASK		0x1
+#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_WR_EN_SHIFT		1
+#define IWARP_MODIFY_QP_RAMROD_DATA_ATOMIC_EN_MASK		0x1
+#define IWARP_MODIFY_QP_RAMROD_DATA_ATOMIC_EN_SHIFT		2
+#define IWARP_MODIFY_QP_RAMROD_DATA_STATE_TRANS_EN_MASK		0x1
 #define IWARP_MODIFY_QP_RAMROD_DATA_STATE_TRANS_EN_SHIFT	3
 #define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_OPS_EN_FLG_MASK	0x1
-#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_OPS_EN_FLG_SHIFT 4
-#define IWARP_MODIFY_QP_RAMROD_DATA_RESERVED_MASK	0x7FF
-#define IWARP_MODIFY_QP_RAMROD_DATA_RESERVED_SHIFT	5
+#define IWARP_MODIFY_QP_RAMROD_DATA_RDMA_OPS_EN_FLG_SHIFT	4
+#define IWARP_MODIFY_QP_RAMROD_DATA_RESERVED_MASK		0x7FF
+#define IWARP_MODIFY_QP_RAMROD_DATA_RESERVED_SHIFT		5
 	__le32 reserved3[3];
 	__le32 reserved4[8];
 };
 
+/* MPA params for Enhanced mode */
 struct mpa_rq_params {
 	__le32 ird;
 	__le32 ord;
 };
 
+/* MPA host Address-Len for private data */
 struct mpa_ulp_buffer {
 	struct regpair addr;
 	__le16 len;
 	__le16 reserved[3];
 };
 
+/* iWARP MPA offload params common to Basic and Enhanced modes */
 struct mpa_outgoing_params {
 	u8 crc_needed;
 	u8 reject;
@@ -9181,6 +9794,9 @@ struct mpa_outgoing_params {
 	struct mpa_ulp_buffer outgoing_ulp_buffer;
 };
 
+/* iWARP MPA offload params passed by driver to FW in MPA Offload Request
+ * Ramrod.
+ */
 struct iwarp_mpa_offload_ramrod_data {
 	struct mpa_outgoing_params common;
 	__le32 tcp_cid;
@@ -9188,18 +9804,20 @@ struct iwarp_mpa_offload_ramrod_data {
 	u8 tcp_connect_side;
 	u8 rtr_pref;
 #define IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED_MASK	0x7
-#define IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED_SHIFT 0
-#define IWARP_MPA_OFFLOAD_RAMROD_DATA_RESERVED1_MASK	0x1F
-#define IWARP_MPA_OFFLOAD_RAMROD_DATA_RESERVED1_SHIFT	3
+#define IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED_SHIFT	0
+#define IWARP_MPA_OFFLOAD_RAMROD_DATA_RESERVED1_MASK		0x1F
+#define IWARP_MPA_OFFLOAD_RAMROD_DATA_RESERVED1_SHIFT		3
 	u8 reserved2;
 	struct mpa_ulp_buffer incoming_ulp_buffer;
 	struct regpair async_eqe_output_buf;
 	struct regpair handle_for_async;
 	struct regpair shared_queue_addr;
+	__le16 rcv_wnd;
 	u8 stats_counter_id;
-	u8 reserved3[15];
+	u8 reserved3[13];
 };
 
+/* iWARP TCP connection offload params passed by driver to FW */
 struct iwarp_offload_params {
 	struct mpa_ulp_buffer incoming_ulp_buffer;
 	struct regpair async_eqe_output_buf;
@@ -9211,22 +9829,24 @@ struct iwarp_offload_params {
 	u8 reserved[10];
 };
 
+/* iWARP query QP output params */
 struct iwarp_query_qp_output_params {
 	__le32 flags;
 #define IWARP_QUERY_QP_OUTPUT_PARAMS_ERROR_FLG_MASK	0x1
-#define IWARP_QUERY_QP_OUTPUT_PARAMS_ERROR_FLG_SHIFT 0
+#define IWARP_QUERY_QP_OUTPUT_PARAMS_ERROR_FLG_SHIFT	0
 #define IWARP_QUERY_QP_OUTPUT_PARAMS_RESERVED0_MASK	0x7FFFFFFF
-#define IWARP_QUERY_QP_OUTPUT_PARAMS_RESERVED0_SHIFT 1
+#define IWARP_QUERY_QP_OUTPUT_PARAMS_RESERVED0_SHIFT	1
 	u8 reserved1[4];
 };
 
+/* iWARP query QP ramrod data */
 struct iwarp_query_qp_ramrod_data {
 	struct regpair output_params_addr;
 };
 
+/* iWARP Ramrod Command IDs */
 enum iwarp_ramrod_cmd_id {
-	IWARP_RAMROD_CMD_ID_TCP_OFFLOAD =
-	11,
+	IWARP_RAMROD_CMD_ID_TCP_OFFLOAD = 11,
 	IWARP_RAMROD_CMD_ID_MPA_OFFLOAD,
 	IWARP_RAMROD_CMD_ID_MPA_OFFLOAD_SEND_RTR,
 	IWARP_RAMROD_CMD_ID_CREATE_QP,
@@ -9236,22 +9856,28 @@ enum iwarp_ramrod_cmd_id {
 	MAX_IWARP_RAMROD_CMD_ID
 };
 
+/* Per PF iWARP retransmit path statistics */
 struct iwarp_rxmit_stats_drv {
 	struct regpair tx_go_to_slow_start_event_cnt;
 	struct regpair tx_fast_retransmit_event_cnt;
 };
 
+/* iWARP and TCP connection offload params passed by driver to FW in iWARP
+ * offload ramrod.
+ */
 struct iwarp_tcp_offload_ramrod_data {
 	struct iwarp_offload_params iwarp;
 	struct tcp_offload_params_opt2 tcp;
 };
 
+/* iWARP MPA negotiation types */
 enum mpa_negotiation_mode {
 	MPA_NEGOTIATION_TYPE_BASIC = 1,
 	MPA_NEGOTIATION_TYPE_ENHANCED = 2,
 	MAX_MPA_NEGOTIATION_MODE
 };
 
+/* iWARP MPA Enhanced mode RTR types */
 enum mpa_rtr_type {
 	MPA_RTR_TYPE_NONE = 0,
 	MPA_RTR_TYPE_ZERO_SEND = 1,
@@ -9264,113 +9890,114 @@ enum mpa_rtr_type {
 	MAX_MPA_RTR_TYPE
 };
 
+/* unaligned opaque data received from LL2 */
 struct unaligned_opaque_data {
 	__le16 first_mpa_offset;
 	u8 tcp_payload_offset;
 	u8 flags;
 #define UNALIGNED_OPAQUE_DATA_PKT_REACHED_WIN_RIGHT_EDGE_MASK	0x1
-#define UNALIGNED_OPAQUE_DATA_PKT_REACHED_WIN_RIGHT_EDGE_SHIFT 0
-#define UNALIGNED_OPAQUE_DATA_CONNECTION_CLOSED_MASK	0x1
-#define UNALIGNED_OPAQUE_DATA_CONNECTION_CLOSED_SHIFT	1
-#define UNALIGNED_OPAQUE_DATA_RESERVED_MASK	0x3F
-#define UNALIGNED_OPAQUE_DATA_RESERVED_SHIFT	2
+#define UNALIGNED_OPAQUE_DATA_PKT_REACHED_WIN_RIGHT_EDGE_SHIFT	0
+#define UNALIGNED_OPAQUE_DATA_CONNECTION_CLOSED_MASK		0x1
+#define UNALIGNED_OPAQUE_DATA_CONNECTION_CLOSED_SHIFT		1
+#define UNALIGNED_OPAQUE_DATA_RESERVED_MASK			0x3F
+#define UNALIGNED_OPAQUE_DATA_RESERVED_SHIFT			2
 	__le32 cid;
 };
 
-struct mstorm_iwarp_conn_ag_ctx {
+struct e4_mstorm_iwarp_conn_ag_ctx {
 	u8 reserved;
 	u8 state;
 	u8 flags0;
-#define MSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
-#define MSTORM_IWARP_CONN_AG_CTX_BIT1_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT	1
-#define MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_MASK	0x3
-#define MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_SHIFT	2
-#define MSTORM_IWARP_CONN_AG_CTX_CF1_MASK	0x3
-#define MSTORM_IWARP_CONN_AG_CTX_CF1_SHIFT	4
-#define MSTORM_IWARP_CONN_AG_CTX_CF2_MASK	0x3
-#define MSTORM_IWARP_CONN_AG_CTX_CF2_SHIFT	6
+#define E4_MSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK		0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT		0
+#define E4_MSTORM_IWARP_CONN_AG_CTX_BIT1_MASK			0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT			1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_MASK	0x3
+#define E4_MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_SHIFT	2
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF1_MASK			0x3
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF1_SHIFT			4
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF2_MASK			0x3
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF2_SHIFT			6
 	u8 flags1;
-#define MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_EN_SHIFT 0
-#define MSTORM_IWARP_CONN_AG_CTX_CF1EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT	1
-#define MSTORM_IWARP_CONN_AG_CTX_CF2EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT	2
-#define MSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT	3
-#define MSTORM_IWARP_CONN_AG_CTX_RULE1EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_RULE1EN_SHIFT	4
-#define MSTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT	5
-#define MSTORM_IWARP_CONN_AG_CTX_RCQ_CONS_EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_RCQ_CONS_EN_SHIFT	6
-#define MSTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define MSTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT	7
+#define E4_MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_EN_MASK	0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_INV_STAG_DONE_CF_EN_SHIFT	0
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF1EN_MASK			0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT			1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT			2
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT		3
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE1EN_SHIFT		4
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT		5
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RCQ_CONS_EN_MASK		0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RCQ_CONS_EN_SHIFT		6
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_MSTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT		7
 	__le16 rcq_cons;
 	__le16 rcq_cons_th;
 	__le32 reg0;
 	__le32 reg1;
 };
 
-struct ustorm_iwarp_conn_ag_ctx {
+struct e4_ustorm_iwarp_conn_ag_ctx {
 	u8 reserved;
 	u8 byte1;
 	u8 flags0;
-#define USTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
-#define USTORM_IWARP_CONN_AG_CTX_BIT1_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT	1
-#define USTORM_IWARP_CONN_AG_CTX_CF0_MASK	0x3
-#define USTORM_IWARP_CONN_AG_CTX_CF0_SHIFT	2
-#define USTORM_IWARP_CONN_AG_CTX_CF1_MASK	0x3
-#define USTORM_IWARP_CONN_AG_CTX_CF1_SHIFT	4
-#define USTORM_IWARP_CONN_AG_CTX_CF2_MASK	0x3
-#define USTORM_IWARP_CONN_AG_CTX_CF2_SHIFT	6
+#define E4_USTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_USTORM_IWARP_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF0_SHIFT		2
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF1_MASK		0x3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF1_SHIFT		4
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF2_MASK		0x3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF2_SHIFT		6
 	u8 flags1;
-#define USTORM_IWARP_CONN_AG_CTX_CF3_MASK	0x3
-#define USTORM_IWARP_CONN_AG_CTX_CF3_SHIFT	0
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_MASK	0x3
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_SHIFT	2
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_MASK	0x3
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_SHIFT	4
-#define USTORM_IWARP_CONN_AG_CTX_CF6_MASK	0x3
-#define USTORM_IWARP_CONN_AG_CTX_CF6_SHIFT	6
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF3_MASK		0x3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF3_SHIFT		0
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_MASK	0x3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_SHIFT	2
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_MASK	0x3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_SHIFT	4
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF6_MASK		0x3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF6_SHIFT		6
 	u8 flags2;
-#define USTORM_IWARP_CONN_AG_CTX_CF0EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT	0
-#define USTORM_IWARP_CONN_AG_CTX_CF1EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT	1
-#define USTORM_IWARP_CONN_AG_CTX_CF2EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT	2
-#define USTORM_IWARP_CONN_AG_CTX_CF3EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CF3EN_SHIFT	3
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_EN_SHIFT 4
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_EN_SHIFT	5
-#define USTORM_IWARP_CONN_AG_CTX_CF6EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CF6EN_SHIFT	6
-#define USTORM_IWARP_CONN_AG_CTX_CQ_SE_EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CQ_SE_EN_SHIFT	7
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF0EN_MASK			0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT			0
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF1EN_MASK			0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT			1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT			2
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF3EN_MASK			0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF3EN_SHIFT			3
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_SE_CF_EN_SHIFT	4
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_EN_MASK		0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_ARM_CF_EN_SHIFT		5
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF6EN_MASK			0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CF6EN_SHIFT			6
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_SE_EN_MASK		0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_SE_EN_SHIFT		7
 	u8 flags3;
-#define USTORM_IWARP_CONN_AG_CTX_CQ_EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_CQ_EN_SHIFT	0
-#define USTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT	1
-#define USTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT	2
-#define USTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT	3
-#define USTORM_IWARP_CONN_AG_CTX_RULE5EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_RULE5EN_SHIFT	4
-#define USTORM_IWARP_CONN_AG_CTX_RULE6EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_RULE6EN_SHIFT	5
-#define USTORM_IWARP_CONN_AG_CTX_RULE7EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_RULE7EN_SHIFT	6
-#define USTORM_IWARP_CONN_AG_CTX_RULE8EN_MASK	0x1
-#define USTORM_IWARP_CONN_AG_CTX_RULE8EN_SHIFT	7
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_EN_MASK		0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_CQ_EN_SHIFT		0
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE8EN_MASK	0x1
+#define E4_USTORM_IWARP_CONN_AG_CTX_RULE8EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -9383,37 +10010,37 @@ struct ustorm_iwarp_conn_ag_ctx {
 	__le16 word3;
 };
 
-struct ystorm_iwarp_conn_ag_ctx {
+struct e4_ystorm_iwarp_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define YSTORM_IWARP_CONN_AG_CTX_BIT0_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_BIT0_SHIFT	0
-#define YSTORM_IWARP_CONN_AG_CTX_BIT1_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT	1
-#define YSTORM_IWARP_CONN_AG_CTX_CF0_MASK	0x3
-#define YSTORM_IWARP_CONN_AG_CTX_CF0_SHIFT	2
-#define YSTORM_IWARP_CONN_AG_CTX_CF1_MASK	0x3
-#define YSTORM_IWARP_CONN_AG_CTX_CF1_SHIFT	4
-#define YSTORM_IWARP_CONN_AG_CTX_CF2_MASK	0x3
-#define YSTORM_IWARP_CONN_AG_CTX_CF2_SHIFT	6
+#define E4_YSTORM_IWARP_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_YSTORM_IWARP_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define YSTORM_IWARP_CONN_AG_CTX_CF0EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT	0
-#define YSTORM_IWARP_CONN_AG_CTX_CF1EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT	1
-#define YSTORM_IWARP_CONN_AG_CTX_CF2EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT	2
-#define YSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define YSTORM_IWARP_CONN_AG_CTX_RULE1EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define YSTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define YSTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define YSTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK	0x1
-#define YSTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_YSTORM_IWARP_CONN_AG_CTX_RULE4EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -9427,6 +10054,7 @@ struct ystorm_iwarp_conn_ag_ctx {
 	__le32 reg3;
 };
 
+/* The fcoe storm context of Ystorm */
 struct ystorm_fcoe_conn_st_ctx {
 	u8 func_mode;
 	u8 cos;
@@ -9442,45 +10070,49 @@ struct ystorm_fcoe_conn_st_ctx {
 	struct regpair reserved;
 	__le16 min_frame_size;
 	u8 protection_info_flags;
-#define YSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_MASK  0x1
-#define YSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_SHIFT 0
-#define YSTORM_FCOE_CONN_ST_CTX_VALID_MASK               0x1
-#define YSTORM_FCOE_CONN_ST_CTX_VALID_SHIFT              1
-#define YSTORM_FCOE_CONN_ST_CTX_RESERVED1_MASK           0x3F
-#define YSTORM_FCOE_CONN_ST_CTX_RESERVED1_SHIFT          2
+#define YSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_MASK		0x1
+#define YSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_SHIFT	0
+#define YSTORM_FCOE_CONN_ST_CTX_VALID_MASK			0x1
+#define YSTORM_FCOE_CONN_ST_CTX_VALID_SHIFT			1
+#define YSTORM_FCOE_CONN_ST_CTX_RESERVED1_MASK			0x3F
+#define YSTORM_FCOE_CONN_ST_CTX_RESERVED1_SHIFT			2
 	u8 dst_protection_per_mss;
 	u8 src_protection_per_mss;
 	u8 ptu_log_page_size;
 	u8 flags;
-#define YSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_MASK     0x1
-#define YSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_SHIFT    0
-#define YSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_MASK     0x1
-#define YSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_SHIFT    1
-#define YSTORM_FCOE_CONN_ST_CTX_RSRV_MASK                0x3F
-#define YSTORM_FCOE_CONN_ST_CTX_RSRV_SHIFT               2
+#define YSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_MASK	0x1
+#define YSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_SHIFT	0
+#define YSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_MASK	0x1
+#define YSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_SHIFT	1
+#define YSTORM_FCOE_CONN_ST_CTX_RSRV_MASK		0x3F
+#define YSTORM_FCOE_CONN_ST_CTX_RSRV_SHIFT		2
 	u8 fcp_xfer_size;
 };
 
+/* FCoE 16-bits vlan structure */
 struct fcoe_vlan_fields {
 	__le16 fields;
-#define FCOE_VLAN_FIELDS_VID_MASK  0xFFF
-#define FCOE_VLAN_FIELDS_VID_SHIFT 0
-#define FCOE_VLAN_FIELDS_CLI_MASK  0x1
-#define FCOE_VLAN_FIELDS_CLI_SHIFT 12
-#define FCOE_VLAN_FIELDS_PRI_MASK  0x7
-#define FCOE_VLAN_FIELDS_PRI_SHIFT 13
+#define FCOE_VLAN_FIELDS_VID_MASK	0xFFF
+#define FCOE_VLAN_FIELDS_VID_SHIFT	0
+#define FCOE_VLAN_FIELDS_CLI_MASK	0x1
+#define FCOE_VLAN_FIELDS_CLI_SHIFT	12
+#define FCOE_VLAN_FIELDS_PRI_MASK	0x7
+#define FCOE_VLAN_FIELDS_PRI_SHIFT	13
 };
 
+/* FCoE 16-bits vlan union */
 union fcoe_vlan_field_union {
 	struct fcoe_vlan_fields fields;
 	__le16 val;
 };
 
+/* FCoE 16-bits vlan, vif union */
 union fcoe_vlan_vif_field_union {
 	union fcoe_vlan_field_union vlan;
 	__le16 vif;
 };
 
+/* Ethernet context section */
 struct pstorm_fcoe_eth_context_section {
 	u8 remote_addr_3;
 	u8 remote_addr_2;
@@ -9500,6 +10132,7 @@ struct pstorm_fcoe_eth_context_section {
 	__le16 inner_eth_type;
 };
 
+/* The fcoe storm context of Pstorm */
 struct pstorm_fcoe_conn_st_ctx {
 	u8 func_mode;
 	u8 cos;
@@ -9513,16 +10146,18 @@ struct pstorm_fcoe_conn_st_ctx {
 	u8 sid_1;
 	u8 sid_0;
 	u8 flags;
-#define PSTORM_FCOE_CONN_ST_CTX_VNTAG_VLAN_MASK          0x1
-#define PSTORM_FCOE_CONN_ST_CTX_VNTAG_VLAN_SHIFT         0
-#define PSTORM_FCOE_CONN_ST_CTX_SUPPORT_REC_RR_TOV_MASK  0x1
-#define PSTORM_FCOE_CONN_ST_CTX_SUPPORT_REC_RR_TOV_SHIFT 1
-#define PSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_MASK     0x1
-#define PSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_SHIFT    2
-#define PSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_MASK     0x1
-#define PSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_SHIFT    3
-#define PSTORM_FCOE_CONN_ST_CTX_RESERVED_MASK            0xF
-#define PSTORM_FCOE_CONN_ST_CTX_RESERVED_SHIFT           4
+#define PSTORM_FCOE_CONN_ST_CTX_VNTAG_VLAN_MASK			0x1
+#define PSTORM_FCOE_CONN_ST_CTX_VNTAG_VLAN_SHIFT		0
+#define PSTORM_FCOE_CONN_ST_CTX_SUPPORT_REC_RR_TOV_MASK		0x1
+#define PSTORM_FCOE_CONN_ST_CTX_SUPPORT_REC_RR_TOV_SHIFT	1
+#define PSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_MASK		0x1
+#define PSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_SHIFT		2
+#define PSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_MASK		0x1
+#define PSTORM_FCOE_CONN_ST_CTX_OUTER_VLAN_FLAG_SHIFT		3
+#define PSTORM_FCOE_CONN_ST_CTX_SINGLE_VLAN_FLAG_MASK		0x1
+#define PSTORM_FCOE_CONN_ST_CTX_SINGLE_VLAN_FLAG_SHIFT		4
+#define PSTORM_FCOE_CONN_ST_CTX_RESERVED_MASK			0x7
+#define PSTORM_FCOE_CONN_ST_CTX_RESERVED_SHIFT			5
 	u8 did_2;
 	u8 did_1;
 	u8 did_0;
@@ -9532,6 +10167,7 @@ struct pstorm_fcoe_conn_st_ctx {
 	u8 reserved1;
 };
 
+/* The fcoe storm context of Xstorm */
 struct xstorm_fcoe_conn_st_ctx {
 	u8 func_mode;
 	u8 src_mac_index;
@@ -9539,16 +10175,16 @@ struct xstorm_fcoe_conn_st_ctx {
 	u8 cached_wqes_avail;
 	__le16 stat_ram_addr;
 	u8 flags;
-#define XSTORM_FCOE_CONN_ST_CTX_SQ_DEFERRED_MASK             0x1
-#define XSTORM_FCOE_CONN_ST_CTX_SQ_DEFERRED_SHIFT            0
-#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_MASK         0x1
-#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_SHIFT        1
-#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_ORIG_MASK    0x1
-#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_ORIG_SHIFT   2
-#define XSTORM_FCOE_CONN_ST_CTX_LAST_QUEUE_HANDLED_MASK      0x3
-#define XSTORM_FCOE_CONN_ST_CTX_LAST_QUEUE_HANDLED_SHIFT     3
-#define XSTORM_FCOE_CONN_ST_CTX_RSRV_MASK                    0x7
-#define XSTORM_FCOE_CONN_ST_CTX_RSRV_SHIFT                   5
+#define XSTORM_FCOE_CONN_ST_CTX_SQ_DEFERRED_MASK		0x1
+#define XSTORM_FCOE_CONN_ST_CTX_SQ_DEFERRED_SHIFT		0
+#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_MASK		0x1
+#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_SHIFT		1
+#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_ORIG_MASK	0x1
+#define XSTORM_FCOE_CONN_ST_CTX_INNER_VLAN_FLAG_ORIG_SHIFT	2
+#define XSTORM_FCOE_CONN_ST_CTX_LAST_QUEUE_HANDLED_MASK		0x3
+#define XSTORM_FCOE_CONN_ST_CTX_LAST_QUEUE_HANDLED_SHIFT	3
+#define XSTORM_FCOE_CONN_ST_CTX_RSRV_MASK			0x7
+#define XSTORM_FCOE_CONN_ST_CTX_RSRV_SHIFT			5
 	u8 cached_wqes_offset;
 	u8 reserved2;
 	u8 eth_hdr_size;
@@ -9574,18 +10210,18 @@ struct xstorm_fcoe_conn_st_ctx {
 	u8 fcp_cmd_byte_credit;
 	u8 fcp_rsp_byte_credit;
 	__le16 protection_info;
-#define XSTORM_FCOE_CONN_ST_CTX_PROTECTION_PERF_MASK         0x1
-#define XSTORM_FCOE_CONN_ST_CTX_PROTECTION_PERF_SHIFT        0
-#define XSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_MASK      0x1
-#define XSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_SHIFT     1
-#define XSTORM_FCOE_CONN_ST_CTX_VALID_MASK                   0x1
-#define XSTORM_FCOE_CONN_ST_CTX_VALID_SHIFT                  2
-#define XSTORM_FCOE_CONN_ST_CTX_FRAME_PROT_ALIGNED_MASK      0x1
-#define XSTORM_FCOE_CONN_ST_CTX_FRAME_PROT_ALIGNED_SHIFT     3
-#define XSTORM_FCOE_CONN_ST_CTX_RESERVED3_MASK               0xF
-#define XSTORM_FCOE_CONN_ST_CTX_RESERVED3_SHIFT              4
-#define XSTORM_FCOE_CONN_ST_CTX_DST_PROTECTION_PER_MSS_MASK  0xFF
-#define XSTORM_FCOE_CONN_ST_CTX_DST_PROTECTION_PER_MSS_SHIFT 8
+#define XSTORM_FCOE_CONN_ST_CTX_PROTECTION_PERF_MASK		0x1
+#define XSTORM_FCOE_CONN_ST_CTX_PROTECTION_PERF_SHIFT		0
+#define XSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_MASK		0x1
+#define XSTORM_FCOE_CONN_ST_CTX_SUPPORT_PROTECTION_SHIFT	1
+#define XSTORM_FCOE_CONN_ST_CTX_VALID_MASK			0x1
+#define XSTORM_FCOE_CONN_ST_CTX_VALID_SHIFT			2
+#define XSTORM_FCOE_CONN_ST_CTX_FRAME_PROT_ALIGNED_MASK		0x1
+#define XSTORM_FCOE_CONN_ST_CTX_FRAME_PROT_ALIGNED_SHIFT	3
+#define XSTORM_FCOE_CONN_ST_CTX_RESERVED3_MASK			0xF
+#define XSTORM_FCOE_CONN_ST_CTX_RESERVED3_SHIFT			4
+#define XSTORM_FCOE_CONN_ST_CTX_DST_PROTECTION_PER_MSS_MASK	0xFF
+#define XSTORM_FCOE_CONN_ST_CTX_DST_PROTECTION_PER_MSS_SHIFT	8
 	__le16 xferq_pbl_next_index;
 	__le16 page_size;
 	u8 mid_seq;
@@ -9594,216 +10230,216 @@ struct xstorm_fcoe_conn_st_ctx {
 	struct fcoe_wqe cached_wqes[16];
 };
 
-struct xstorm_fcoe_conn_ag_ctx {
+struct e4_xstorm_fcoe_conn_ag_ctx {
 	u8 reserved0;
-	u8 fcoe_state;
+	u8 state;
 	u8 flags0;
-#define XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT      0
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED1_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED1_SHIFT         1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED2_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED2_SHIFT         2
-#define XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM3_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT      3
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED3_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED3_SHIFT         4
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED4_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED4_SHIFT         5
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED5_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED5_SHIFT         6
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED6_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED6_SHIFT         7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED1_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED1_SHIFT	1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED2_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED2_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED3_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED3_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED4_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED4_SHIFT	5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED5_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED5_SHIFT	6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED6_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED6_SHIFT	7
 	u8 flags1;
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED7_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED7_SHIFT         0
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED8_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED8_SHIFT         1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED9_MASK          0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED9_SHIFT         2
-#define XSTORM_FCOE_CONN_AG_CTX_BIT11_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT11_SHIFT             3
-#define XSTORM_FCOE_CONN_AG_CTX_BIT12_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT12_SHIFT             4
-#define XSTORM_FCOE_CONN_AG_CTX_BIT13_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT13_SHIFT             5
-#define XSTORM_FCOE_CONN_AG_CTX_BIT14_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT14_SHIFT             6
-#define XSTORM_FCOE_CONN_AG_CTX_BIT15_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT15_SHIFT             7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED7_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED7_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED8_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED8_SHIFT	1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED9_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED9_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT11_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT12_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT12_SHIFT		4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT13_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT13_SHIFT		5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT14_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT14_SHIFT		6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT15_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT15_SHIFT		7
 	u8 flags2;
-#define XSTORM_FCOE_CONN_AG_CTX_CF0_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF0_SHIFT               0
-#define XSTORM_FCOE_CONN_AG_CTX_CF1_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF1_SHIFT               2
-#define XSTORM_FCOE_CONN_AG_CTX_CF2_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT               4
-#define XSTORM_FCOE_CONN_AG_CTX_CF3_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF3_SHIFT               6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF0_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF1_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF3_SHIFT	6
 	u8 flags3;
-#define XSTORM_FCOE_CONN_AG_CTX_CF4_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF4_SHIFT               0
-#define XSTORM_FCOE_CONN_AG_CTX_CF5_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF5_SHIFT               2
-#define XSTORM_FCOE_CONN_AG_CTX_CF6_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF6_SHIFT               4
-#define XSTORM_FCOE_CONN_AG_CTX_CF7_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF7_SHIFT               6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF4_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF5_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF6_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF7_SHIFT	6
 	u8 flags4;
-#define XSTORM_FCOE_CONN_AG_CTX_CF8_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF8_SHIFT               0
-#define XSTORM_FCOE_CONN_AG_CTX_CF9_MASK                0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF9_SHIFT               2
-#define XSTORM_FCOE_CONN_AG_CTX_CF10_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF10_SHIFT              4
-#define XSTORM_FCOE_CONN_AG_CTX_CF11_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF11_SHIFT              6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_FCOE_CONN_AG_CTX_CF12_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF12_SHIFT              0
-#define XSTORM_FCOE_CONN_AG_CTX_CF13_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF13_SHIFT              2
-#define XSTORM_FCOE_CONN_AG_CTX_CF14_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF14_SHIFT              4
-#define XSTORM_FCOE_CONN_AG_CTX_CF15_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF15_SHIFT              6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF12_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF12_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF13_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF13_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF14_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF14_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF15_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF15_SHIFT	6
 	u8 flags6;
-#define XSTORM_FCOE_CONN_AG_CTX_CF16_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF16_SHIFT              0
-#define XSTORM_FCOE_CONN_AG_CTX_CF17_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF17_SHIFT              2
-#define XSTORM_FCOE_CONN_AG_CTX_CF18_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF18_SHIFT              4
-#define XSTORM_FCOE_CONN_AG_CTX_DQ_CF_MASK              0x3
-#define XSTORM_FCOE_CONN_AG_CTX_DQ_CF_SHIFT             6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF16_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF16_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF17_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF17_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF18_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF18_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_DQ_CF_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_DQ_CF_SHIFT	6
 	u8 flags7;
-#define XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_MASK           0x3
-#define XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_SHIFT          0
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED10_MASK         0x3
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED10_SHIFT        2
-#define XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_MASK          0x3
-#define XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_SHIFT         4
-#define XSTORM_FCOE_CONN_AG_CTX_CF0EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT             6
-#define XSTORM_FCOE_CONN_AG_CTX_CF1EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT             7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED10_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED10_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT		6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT		7
 	u8 flags8;
-#define XSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT             0
-#define XSTORM_FCOE_CONN_AG_CTX_CF3EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF3EN_SHIFT             1
-#define XSTORM_FCOE_CONN_AG_CTX_CF4EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF4EN_SHIFT             2
-#define XSTORM_FCOE_CONN_AG_CTX_CF5EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF5EN_SHIFT             3
-#define XSTORM_FCOE_CONN_AG_CTX_CF6EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF6EN_SHIFT             4
-#define XSTORM_FCOE_CONN_AG_CTX_CF7EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF7EN_SHIFT             5
-#define XSTORM_FCOE_CONN_AG_CTX_CF8EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF8EN_SHIFT             6
-#define XSTORM_FCOE_CONN_AG_CTX_CF9EN_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF9EN_SHIFT             7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF3EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF3EN_SHIFT	1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF4EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF4EN_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF5EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF5EN_SHIFT	3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF6EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF6EN_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF7EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF7EN_SHIFT	5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF8EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF8EN_SHIFT	6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF9EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF9EN_SHIFT	7
 	u8 flags9;
-#define XSTORM_FCOE_CONN_AG_CTX_CF10EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF10EN_SHIFT            0
-#define XSTORM_FCOE_CONN_AG_CTX_CF11EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF11EN_SHIFT            1
-#define XSTORM_FCOE_CONN_AG_CTX_CF12EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF12EN_SHIFT            2
-#define XSTORM_FCOE_CONN_AG_CTX_CF13EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF13EN_SHIFT            3
-#define XSTORM_FCOE_CONN_AG_CTX_CF14EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF14EN_SHIFT            4
-#define XSTORM_FCOE_CONN_AG_CTX_CF15EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF15EN_SHIFT            5
-#define XSTORM_FCOE_CONN_AG_CTX_CF16EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF16EN_SHIFT            6
-#define XSTORM_FCOE_CONN_AG_CTX_CF17EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF17EN_SHIFT            7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF10EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF10EN_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF11EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF11EN_SHIFT	1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF12EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF12EN_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF13EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF13EN_SHIFT	3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF14EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF14EN_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF15EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF15EN_SHIFT	5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF16EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF16EN_SHIFT	6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF17EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF17EN_SHIFT	7
 	u8 flags10;
-#define XSTORM_FCOE_CONN_AG_CTX_CF18EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF18EN_SHIFT            0
-#define XSTORM_FCOE_CONN_AG_CTX_DQ_CF_EN_MASK           0x1
-#define XSTORM_FCOE_CONN_AG_CTX_DQ_CF_EN_SHIFT          1
-#define XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_EN_MASK        0x1
-#define XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT       2
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED11_MASK         0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED11_SHIFT        3
-#define XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_EN_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT      4
-#define XSTORM_FCOE_CONN_AG_CTX_CF23EN_MASK             0x1
-#define XSTORM_FCOE_CONN_AG_CTX_CF23EN_SHIFT            5
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED12_MASK         0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED12_SHIFT        6
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED13_MASK         0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED13_SHIFT        7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF18EN_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF18EN_SHIFT		0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_DQ_CF_EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_DQ_CF_EN_SHIFT	1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED11_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED11_SHIFT	3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_SLOW_PATH_EN_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF23EN_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF23EN_SHIFT		5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED12_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED12_SHIFT	6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED13_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED13_SHIFT	7
 	u8 flags11;
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED14_MASK         0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED14_SHIFT        0
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED15_MASK         0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED15_SHIFT        1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED16_MASK         0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESERVED16_SHIFT        2
-#define XSTORM_FCOE_CONN_AG_CTX_RULE5EN_MASK            0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE5EN_SHIFT           3
-#define XSTORM_FCOE_CONN_AG_CTX_RULE6EN_MASK            0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE6EN_SHIFT           4
-#define XSTORM_FCOE_CONN_AG_CTX_RULE7EN_MASK            0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE7EN_SHIFT           5
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED1_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED1_SHIFT      6
-#define XSTORM_FCOE_CONN_AG_CTX_XFERQ_DECISION_EN_MASK  0x1
-#define XSTORM_FCOE_CONN_AG_CTX_XFERQ_DECISION_EN_SHIFT 7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED14_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED14_SHIFT		0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED15_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED15_SHIFT		1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED16_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESERVED16_SHIFT		2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE5EN_MASK			0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE5EN_SHIFT		3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE6EN_MASK			0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE6EN_SHIFT		4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE7EN_MASK			0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE7EN_SHIFT		5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED1_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED1_SHIFT		6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_XFERQ_DECISION_EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_XFERQ_DECISION_EN_SHIFT	7
 	u8 flags12;
-#define XSTORM_FCOE_CONN_AG_CTX_SQ_DECISION_EN_MASK     0x1
-#define XSTORM_FCOE_CONN_AG_CTX_SQ_DECISION_EN_SHIFT    0
-#define XSTORM_FCOE_CONN_AG_CTX_RULE11EN_MASK           0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE11EN_SHIFT          1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED2_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED2_SHIFT      2
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED3_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED3_SHIFT      3
-#define XSTORM_FCOE_CONN_AG_CTX_RULE14EN_MASK           0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE14EN_SHIFT          4
-#define XSTORM_FCOE_CONN_AG_CTX_RULE15EN_MASK           0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE15EN_SHIFT          5
-#define XSTORM_FCOE_CONN_AG_CTX_RULE16EN_MASK           0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE16EN_SHIFT          6
-#define XSTORM_FCOE_CONN_AG_CTX_RULE17EN_MASK           0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE17EN_SHIFT          7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_SQ_DECISION_EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_SQ_DECISION_EN_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE11EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE11EN_SHIFT	1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED2_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED2_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED3_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED3_SHIFT	3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE14EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE14EN_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE15EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE15EN_SHIFT	5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE16EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE16EN_SHIFT	6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE17EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE17EN_SHIFT	7
 	u8 flags13;
-#define XSTORM_FCOE_CONN_AG_CTX_RESPQ_DECISION_EN_MASK  0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RESPQ_DECISION_EN_SHIFT 0
-#define XSTORM_FCOE_CONN_AG_CTX_RULE19EN_MASK           0x1
-#define XSTORM_FCOE_CONN_AG_CTX_RULE19EN_SHIFT          1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED4_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED4_SHIFT      2
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED5_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED5_SHIFT      3
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED6_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED6_SHIFT      4
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED7_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED7_SHIFT      5
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED8_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED8_SHIFT      6
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED9_MASK       0x1
-#define XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED9_SHIFT      7
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESPQ_DECISION_EN_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RESPQ_DECISION_EN_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE19EN_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_RULE19EN_SHIFT		1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED4_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED4_SHIFT		2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED5_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED5_SHIFT		3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED6_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED6_SHIFT		4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED7_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED7_SHIFT		5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED8_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED8_SHIFT		6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED9_MASK		0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_A0_RESERVED9_SHIFT		7
 	u8 flags14;
-#define XSTORM_FCOE_CONN_AG_CTX_BIT16_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT16_SHIFT             0
-#define XSTORM_FCOE_CONN_AG_CTX_BIT17_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT17_SHIFT             1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT18_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT18_SHIFT             2
-#define XSTORM_FCOE_CONN_AG_CTX_BIT19_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT19_SHIFT             3
-#define XSTORM_FCOE_CONN_AG_CTX_BIT20_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT20_SHIFT             4
-#define XSTORM_FCOE_CONN_AG_CTX_BIT21_MASK              0x1
-#define XSTORM_FCOE_CONN_AG_CTX_BIT21_SHIFT             5
-#define XSTORM_FCOE_CONN_AG_CTX_CF23_MASK               0x3
-#define XSTORM_FCOE_CONN_AG_CTX_CF23_SHIFT              6
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT16_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT16_SHIFT	0
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT17_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT17_SHIFT	1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT18_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT18_SHIFT	2
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT19_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT19_SHIFT	3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT20_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT20_SHIFT	4
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT21_MASK	0x1
+#define E4_XSTORM_FCOE_CONN_AG_CTX_BIT21_SHIFT	5
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF23_MASK	0x3
+#define E4_XSTORM_FCOE_CONN_AG_CTX_CF23_SHIFT	6
 	u8 byte2;
 	__le16 physical_q0;
 	__le16 word1;
@@ -9831,6 +10467,7 @@ struct xstorm_fcoe_conn_ag_ctx {
 	__le32 reg8;
 };
 
+/* The fcoe storm context of Ustorm */
 struct ustorm_fcoe_conn_st_ctx {
 	struct regpair respq_pbl_addr;
 	__le16 num_pages_in_pbl;
@@ -9840,150 +10477,150 @@ struct ustorm_fcoe_conn_st_ctx {
 	u8 reserved[2];
 };
 
-struct tstorm_fcoe_conn_ag_ctx {
+struct e4_tstorm_fcoe_conn_ag_ctx {
 	u8 reserved0;
-	u8 fcoe_state;
+	u8 state;
 	u8 flags0;
-#define TSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_MASK          0x1
-#define TSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT         0
-#define TSTORM_FCOE_CONN_AG_CTX_BIT1_MASK                  0x1
-#define TSTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT                 1
-#define TSTORM_FCOE_CONN_AG_CTX_BIT2_MASK                  0x1
-#define TSTORM_FCOE_CONN_AG_CTX_BIT2_SHIFT                 2
-#define TSTORM_FCOE_CONN_AG_CTX_BIT3_MASK                  0x1
-#define TSTORM_FCOE_CONN_AG_CTX_BIT3_SHIFT                 3
-#define TSTORM_FCOE_CONN_AG_CTX_BIT4_MASK                  0x1
-#define TSTORM_FCOE_CONN_AG_CTX_BIT4_SHIFT                 4
-#define TSTORM_FCOE_CONN_AG_CTX_BIT5_MASK                  0x1
-#define TSTORM_FCOE_CONN_AG_CTX_BIT5_SHIFT                 5
-#define TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_MASK        0x3
-#define TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_SHIFT       6
+#define E4_TSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT2_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT2_SHIFT		2
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT3_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT3_SHIFT		3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT4_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT4_SHIFT		4
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT5_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_BIT5_SHIFT		5
+#define E4_TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_MASK	0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_SHIFT	6
 	u8 flags1;
-#define TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_MASK           0x3
-#define TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT          0
-#define TSTORM_FCOE_CONN_AG_CTX_CF2_MASK                   0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT                  2
-#define TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK     0x3
-#define TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT    4
-#define TSTORM_FCOE_CONN_AG_CTX_CF4_MASK                   0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF4_SHIFT                  6
+#define E4_TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_MASK		0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_SHIFT		0
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF2_MASK			0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT			2
+#define E4_TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_MASK	0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_SHIFT	4
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF4_MASK			0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF4_SHIFT			6
 	u8 flags2;
-#define TSTORM_FCOE_CONN_AG_CTX_CF5_MASK                   0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF5_SHIFT                  0
-#define TSTORM_FCOE_CONN_AG_CTX_CF6_MASK                   0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF6_SHIFT                  2
-#define TSTORM_FCOE_CONN_AG_CTX_CF7_MASK                   0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF7_SHIFT                  4
-#define TSTORM_FCOE_CONN_AG_CTX_CF8_MASK                   0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF8_SHIFT                  6
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF5_SHIFT	0
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF6_SHIFT	2
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF7_SHIFT	4
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF8_SHIFT	6
 	u8 flags3;
-#define TSTORM_FCOE_CONN_AG_CTX_CF9_MASK                   0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF9_SHIFT                  0
-#define TSTORM_FCOE_CONN_AG_CTX_CF10_MASK                  0x3
-#define TSTORM_FCOE_CONN_AG_CTX_CF10_SHIFT                 2
-#define TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_EN_MASK     0x1
-#define TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_EN_SHIFT    4
-#define TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK        0x1
-#define TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT       5
-#define TSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK                 0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT                6
-#define TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK  0x1
-#define TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT 7
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF9_MASK			0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF9_SHIFT			0
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF10_MASK			0x3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF10_SHIFT			2
+#define E4_TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_EN_MASK	0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_DUMMY_TIMER_CF_EN_SHIFT	4
+#define E4_TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_FLUSH_Q0_CF_EN_SHIFT		5
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT			6
+#define E4_TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_MASK	0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_TIMER_STOP_ALL_CF_EN_SHIFT	7
 	u8 flags4;
-#define TSTORM_FCOE_CONN_AG_CTX_CF4EN_MASK                 0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF4EN_SHIFT                0
-#define TSTORM_FCOE_CONN_AG_CTX_CF5EN_MASK                 0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF5EN_SHIFT                1
-#define TSTORM_FCOE_CONN_AG_CTX_CF6EN_MASK                 0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF6EN_SHIFT                2
-#define TSTORM_FCOE_CONN_AG_CTX_CF7EN_MASK                 0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF7EN_SHIFT                3
-#define TSTORM_FCOE_CONN_AG_CTX_CF8EN_MASK                 0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF8EN_SHIFT                4
-#define TSTORM_FCOE_CONN_AG_CTX_CF9EN_MASK                 0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF9EN_SHIFT                5
-#define TSTORM_FCOE_CONN_AG_CTX_CF10EN_MASK                0x1
-#define TSTORM_FCOE_CONN_AG_CTX_CF10EN_SHIFT               6
-#define TSTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT              7
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF4EN_SHIFT		0
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF5EN_SHIFT		1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF6EN_SHIFT		2
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF7EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF7EN_SHIFT		3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF8EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF8EN_SHIFT		4
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF9EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF9EN_SHIFT		5
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF10EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_CF10EN_SHIFT		6
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT	7
 	u8 flags5;
-#define TSTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT              0
-#define TSTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT              1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT              2
-#define TSTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT              3
-#define TSTORM_FCOE_CONN_AG_CTX_RULE5EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE5EN_SHIFT              4
-#define TSTORM_FCOE_CONN_AG_CTX_RULE6EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE6EN_SHIFT              5
-#define TSTORM_FCOE_CONN_AG_CTX_RULE7EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE7EN_SHIFT              6
-#define TSTORM_FCOE_CONN_AG_CTX_RULE8EN_MASK               0x1
-#define TSTORM_FCOE_CONN_AG_CTX_RULE8EN_SHIFT              7
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_TSTORM_FCOE_CONN_AG_CTX_RULE8EN_SHIFT	7
 	__le32 reg0;
 	__le32 reg1;
 };
 
-struct ustorm_fcoe_conn_ag_ctx {
+struct e4_ustorm_fcoe_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define USTORM_FCOE_CONN_AG_CTX_BIT0_MASK     0x1
-#define USTORM_FCOE_CONN_AG_CTX_BIT0_SHIFT    0
-#define USTORM_FCOE_CONN_AG_CTX_BIT1_MASK     0x1
-#define USTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT    1
-#define USTORM_FCOE_CONN_AG_CTX_CF0_MASK      0x3
-#define USTORM_FCOE_CONN_AG_CTX_CF0_SHIFT     2
-#define USTORM_FCOE_CONN_AG_CTX_CF1_MASK      0x3
-#define USTORM_FCOE_CONN_AG_CTX_CF1_SHIFT     4
-#define USTORM_FCOE_CONN_AG_CTX_CF2_MASK      0x3
-#define USTORM_FCOE_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_USTORM_FCOE_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_USTORM_FCOE_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define USTORM_FCOE_CONN_AG_CTX_CF3_MASK      0x3
-#define USTORM_FCOE_CONN_AG_CTX_CF3_SHIFT     0
-#define USTORM_FCOE_CONN_AG_CTX_CF4_MASK      0x3
-#define USTORM_FCOE_CONN_AG_CTX_CF4_SHIFT     2
-#define USTORM_FCOE_CONN_AG_CTX_CF5_MASK      0x3
-#define USTORM_FCOE_CONN_AG_CTX_CF5_SHIFT     4
-#define USTORM_FCOE_CONN_AG_CTX_CF6_MASK      0x3
-#define USTORM_FCOE_CONN_AG_CTX_CF6_SHIFT     6
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF3_SHIFT	0
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF4_SHIFT	2
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF5_SHIFT	4
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF6_SHIFT	6
 	u8 flags2;
-#define USTORM_FCOE_CONN_AG_CTX_CF0EN_MASK    0x1
-#define USTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT   0
-#define USTORM_FCOE_CONN_AG_CTX_CF1EN_MASK    0x1
-#define USTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT   1
-#define USTORM_FCOE_CONN_AG_CTX_CF2EN_MASK    0x1
-#define USTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT   2
-#define USTORM_FCOE_CONN_AG_CTX_CF3EN_MASK    0x1
-#define USTORM_FCOE_CONN_AG_CTX_CF3EN_SHIFT   3
-#define USTORM_FCOE_CONN_AG_CTX_CF4EN_MASK    0x1
-#define USTORM_FCOE_CONN_AG_CTX_CF4EN_SHIFT   4
-#define USTORM_FCOE_CONN_AG_CTX_CF5EN_MASK    0x1
-#define USTORM_FCOE_CONN_AG_CTX_CF5EN_SHIFT   5
-#define USTORM_FCOE_CONN_AG_CTX_CF6EN_MASK    0x1
-#define USTORM_FCOE_CONN_AG_CTX_CF6EN_SHIFT   6
-#define USTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT 7
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF3EN_SHIFT		3
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF4EN_SHIFT		4
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF5EN_SHIFT		5
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_CF6EN_SHIFT		6
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT	7
 	u8 flags3;
-#define USTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT 0
-#define USTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT 1
-#define USTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT 2
-#define USTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT 3
-#define USTORM_FCOE_CONN_AG_CTX_RULE5EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE5EN_SHIFT 4
-#define USTORM_FCOE_CONN_AG_CTX_RULE6EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE6EN_SHIFT 5
-#define USTORM_FCOE_CONN_AG_CTX_RULE7EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE7EN_SHIFT 6
-#define USTORM_FCOE_CONN_AG_CTX_RULE8EN_MASK  0x1
-#define USTORM_FCOE_CONN_AG_CTX_RULE8EN_SHIFT 7
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE5EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE6EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE7EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE8EN_MASK		0x1
+#define E4_USTORM_FCOE_CONN_AG_CTX_RULE8EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -9996,72 +10633,76 @@ struct ustorm_fcoe_conn_ag_ctx {
 	__le16 word3;
 };
 
+/* The fcoe storm context of Tstorm */
 struct tstorm_fcoe_conn_st_ctx {
 	__le16 stat_ram_addr;
 	__le16 rx_max_fc_payload_len;
 	__le16 e_d_tov_val;
 	u8 flags;
-#define TSTORM_FCOE_CONN_ST_CTX_INC_SEQ_CNT_MASK   0x1
-#define TSTORM_FCOE_CONN_ST_CTX_INC_SEQ_CNT_SHIFT  0
-#define TSTORM_FCOE_CONN_ST_CTX_SUPPORT_CONF_MASK  0x1
-#define TSTORM_FCOE_CONN_ST_CTX_SUPPORT_CONF_SHIFT 1
-#define TSTORM_FCOE_CONN_ST_CTX_DEF_Q_IDX_MASK     0x3F
-#define TSTORM_FCOE_CONN_ST_CTX_DEF_Q_IDX_SHIFT    2
+#define TSTORM_FCOE_CONN_ST_CTX_INC_SEQ_CNT_MASK	0x1
+#define TSTORM_FCOE_CONN_ST_CTX_INC_SEQ_CNT_SHIFT	0
+#define TSTORM_FCOE_CONN_ST_CTX_SUPPORT_CONF_MASK	0x1
+#define TSTORM_FCOE_CONN_ST_CTX_SUPPORT_CONF_SHIFT	1
+#define TSTORM_FCOE_CONN_ST_CTX_DEF_Q_IDX_MASK		0x3F
+#define TSTORM_FCOE_CONN_ST_CTX_DEF_Q_IDX_SHIFT		2
 	u8 timers_cleanup_invocation_cnt;
 	__le32 reserved1[2];
-	__le32 dst_mac_address_bytes0to3;
-	__le16 dst_mac_address_bytes4to5;
+	__le32 dst_mac_address_bytes_0_to_3;
+	__le16 dst_mac_address_bytes_4_to_5;
 	__le16 ramrod_echo;
 	u8 flags1;
-#define TSTORM_FCOE_CONN_ST_CTX_MODE_MASK          0x3
-#define TSTORM_FCOE_CONN_ST_CTX_MODE_SHIFT         0
-#define TSTORM_FCOE_CONN_ST_CTX_RESERVED_MASK      0x3F
-#define TSTORM_FCOE_CONN_ST_CTX_RESERVED_SHIFT     2
-	u8 q_relative_offset;
+#define TSTORM_FCOE_CONN_ST_CTX_MODE_MASK	0x3
+#define TSTORM_FCOE_CONN_ST_CTX_MODE_SHIFT	0
+#define TSTORM_FCOE_CONN_ST_CTX_RESERVED_MASK	0x3F
+#define TSTORM_FCOE_CONN_ST_CTX_RESERVED_SHIFT	2
+	u8 cq_relative_offset;
+	u8 cmdq_relative_offset;
 	u8 bdq_resource_id;
-	u8 reserved0[5];
+	u8 reserved0[4];
 };
 
-struct mstorm_fcoe_conn_ag_ctx {
+struct e4_mstorm_fcoe_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define MSTORM_FCOE_CONN_AG_CTX_BIT0_MASK     0x1
-#define MSTORM_FCOE_CONN_AG_CTX_BIT0_SHIFT    0
-#define MSTORM_FCOE_CONN_AG_CTX_BIT1_MASK     0x1
-#define MSTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT    1
-#define MSTORM_FCOE_CONN_AG_CTX_CF0_MASK      0x3
-#define MSTORM_FCOE_CONN_AG_CTX_CF0_SHIFT     2
-#define MSTORM_FCOE_CONN_AG_CTX_CF1_MASK      0x3
-#define MSTORM_FCOE_CONN_AG_CTX_CF1_SHIFT     4
-#define MSTORM_FCOE_CONN_AG_CTX_CF2_MASK      0x3
-#define MSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_MSTORM_FCOE_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_MSTORM_FCOE_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define MSTORM_FCOE_CONN_AG_CTX_CF0EN_MASK    0x1
-#define MSTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT   0
-#define MSTORM_FCOE_CONN_AG_CTX_CF1EN_MASK    0x1
-#define MSTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT   1
-#define MSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK    0x1
-#define MSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT   2
-#define MSTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define MSTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define MSTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define MSTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define MSTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define MSTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define MSTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define MSTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define MSTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define MSTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_MSTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT	7
 	__le16 word0;
 	__le16 word1;
 	__le32 reg0;
 	__le32 reg1;
 };
 
+/* Fast path part of the fcoe storm context of Mstorm */
 struct fcoe_mstorm_fcoe_conn_st_ctx_fp {
 	__le16 xfer_prod;
-	__le16 reserved1;
+	u8 num_cqs;
+	u8 reserved1;
 	u8 protection_info;
 #define FCOE_MSTORM_FCOE_CONN_ST_CTX_FP_SUPPORT_PROTECTION_MASK  0x1
 #define FCOE_MSTORM_FCOE_CONN_ST_CTX_FP_SUPPORT_PROTECTION_SHIFT 0
@@ -10073,6 +10714,7 @@ struct fcoe_mstorm_fcoe_conn_st_ctx_fp {
 	u8 reserved2[2];
 };
 
+/* Non fast path part of the fcoe storm context of Mstorm */
 struct fcoe_mstorm_fcoe_conn_st_ctx_non_fp {
 	__le16 conn_id;
 	__le16 stat_ram_addr;
@@ -10088,37 +10730,46 @@ struct fcoe_mstorm_fcoe_conn_st_ctx_non_fp {
 	struct regpair reserved2[3];
 };
 
+/* The fcoe storm context of Mstorm */
 struct mstorm_fcoe_conn_st_ctx {
 	struct fcoe_mstorm_fcoe_conn_st_ctx_fp fp;
 	struct fcoe_mstorm_fcoe_conn_st_ctx_non_fp non_fp;
 };
 
-struct fcoe_conn_context {
+/* fcoe connection context */
+struct e4_fcoe_conn_context {
 	struct ystorm_fcoe_conn_st_ctx ystorm_st_context;
 	struct pstorm_fcoe_conn_st_ctx pstorm_st_context;
 	struct regpair pstorm_st_padding[2];
 	struct xstorm_fcoe_conn_st_ctx xstorm_st_context;
-	struct xstorm_fcoe_conn_ag_ctx xstorm_ag_context;
+	struct e4_xstorm_fcoe_conn_ag_ctx xstorm_ag_context;
 	struct regpair xstorm_ag_padding[6];
 	struct ustorm_fcoe_conn_st_ctx ustorm_st_context;
 	struct regpair ustorm_st_padding[2];
-	struct tstorm_fcoe_conn_ag_ctx tstorm_ag_context;
+	struct e4_tstorm_fcoe_conn_ag_ctx tstorm_ag_context;
 	struct regpair tstorm_ag_padding[2];
 	struct timers_context timer_context;
-	struct ustorm_fcoe_conn_ag_ctx ustorm_ag_context;
+	struct e4_ustorm_fcoe_conn_ag_ctx ustorm_ag_context;
 	struct tstorm_fcoe_conn_st_ctx tstorm_st_context;
-	struct mstorm_fcoe_conn_ag_ctx mstorm_ag_context;
+	struct e4_mstorm_fcoe_conn_ag_ctx mstorm_ag_context;
 	struct mstorm_fcoe_conn_st_ctx mstorm_st_context;
 };
 
+/* FCoE connection offload params passed by driver to FW in FCoE offload
+ * ramrod.
+ */
 struct fcoe_conn_offload_ramrod_params {
 	struct fcoe_conn_offload_ramrod_data offload_ramrod_data;
 };
 
+/* FCoE connection terminate params passed by driver to FW in FCoE terminate
+ * conn ramrod.
+ */
 struct fcoe_conn_terminate_ramrod_params {
 	struct fcoe_conn_terminate_ramrod_data terminate_ramrod_data;
 };
 
+/* FCoE event type */
 enum fcoe_event_type {
 	FCOE_EVENT_INIT_FUNC,
 	FCOE_EVENT_DESTROY_FUNC,
@@ -10129,10 +10780,12 @@ enum fcoe_event_type {
 	MAX_FCOE_EVENT_TYPE
 };
 
+/* FCoE init params passed by driver to FW in FCoE init ramrod */
 struct fcoe_init_ramrod_params {
 	struct fcoe_init_func_ramrod_data init_ramrod_data;
 };
 
+/* FCoE ramrod Command IDs */
 enum fcoe_ramrod_cmd_id {
 	FCOE_RAMROD_CMD_ID_INIT_FUNC,
 	FCOE_RAMROD_CMD_ID_DESTROY_FUNC,
@@ -10142,41 +10795,44 @@ enum fcoe_ramrod_cmd_id {
 	MAX_FCOE_RAMROD_CMD_ID
 };
 
+/* FCoE statistics params buffer passed by driver to FW in FCoE statistics
+ * ramrod.
+ */
 struct fcoe_stat_ramrod_params {
 	struct fcoe_stat_ramrod_data stat_ramrod_data;
 };
 
-struct ystorm_fcoe_conn_ag_ctx {
+struct e4_ystorm_fcoe_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define YSTORM_FCOE_CONN_AG_CTX_BIT0_MASK     0x1
-#define YSTORM_FCOE_CONN_AG_CTX_BIT0_SHIFT    0
-#define YSTORM_FCOE_CONN_AG_CTX_BIT1_MASK     0x1
-#define YSTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT    1
-#define YSTORM_FCOE_CONN_AG_CTX_CF0_MASK      0x3
-#define YSTORM_FCOE_CONN_AG_CTX_CF0_SHIFT     2
-#define YSTORM_FCOE_CONN_AG_CTX_CF1_MASK      0x3
-#define YSTORM_FCOE_CONN_AG_CTX_CF1_SHIFT     4
-#define YSTORM_FCOE_CONN_AG_CTX_CF2_MASK      0x3
-#define YSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_YSTORM_FCOE_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_YSTORM_FCOE_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define YSTORM_FCOE_CONN_AG_CTX_CF0EN_MASK    0x1
-#define YSTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT   0
-#define YSTORM_FCOE_CONN_AG_CTX_CF1EN_MASK    0x1
-#define YSTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT   1
-#define YSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK    0x1
-#define YSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT   2
-#define YSTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define YSTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define YSTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define YSTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define YSTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define YSTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define YSTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define YSTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define YSTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define YSTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE0EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE1EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE2EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE3EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE4EN_MASK		0x1
+#define E4_YSTORM_FCOE_CONN_AG_CTX_RULE4EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -10190,230 +10846,233 @@ struct ystorm_fcoe_conn_ag_ctx {
 	__le32 reg3;
 };
 
+/* The iscsi storm connection context of Ystorm */
 struct ystorm_iscsi_conn_st_ctx {
-	__le32 reserved[4];
+	__le32 reserved[8];
 };
 
+/* Combined iSCSI and TCP storm connection of Pstorm */
 struct pstorm_iscsi_tcp_conn_st_ctx {
 	__le32 tcp[32];
 	__le32 iscsi[4];
 };
 
+/* The combined tcp and iscsi storm context of Xstorm */
 struct xstorm_iscsi_tcp_conn_st_ctx {
-	__le32 reserved_iscsi[40];
 	__le32 reserved_tcp[4];
+	__le32 reserved_iscsi[44];
 };
 
-struct xstorm_iscsi_conn_ag_ctx {
+struct e4_xstorm_iscsi_conn_ag_ctx {
 	u8 cdu_validation;
 	u8 state;
 	u8 flags0;
-#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_SHIFT               0
-#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM1_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM1_SHIFT               1
-#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED1_MASK                   0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED1_SHIFT                  2
-#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM3_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM3_SHIFT               3
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT4_MASK                        0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT4_SHIFT                       4
-#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED2_MASK                   0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED2_SHIFT                  5
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT6_MASK                        0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT6_SHIFT                       6
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT7_MASK                        0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT7_SHIFT                       7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM1_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM1_SHIFT	1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RESERVED1_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RESERVED1_SHIFT	2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM3_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM3_SHIFT	3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT4_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT4_SHIFT		4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RESERVED2_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RESERVED2_SHIFT	5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT6_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT6_SHIFT		6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT7_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT7_SHIFT		7
 	u8 flags1;
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT8_MASK                        0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT8_SHIFT                       0
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT9_MASK                        0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT9_SHIFT                       1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT10_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT10_SHIFT                      2
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT11_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT11_SHIFT                      3
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT12_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT12_SHIFT                      4
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT13_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT13_SHIFT                      5
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT14_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT14_SHIFT                      6
-#define XSTORM_ISCSI_CONN_AG_CTX_TX_TRUNCATE_MASK                 0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_TX_TRUNCATE_SHIFT                7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT8_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT8_SHIFT		0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT9_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT9_SHIFT		1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT10_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT10_SHIFT		2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT11_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT11_SHIFT		3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT12_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT12_SHIFT		4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT13_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT13_SHIFT		5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT14_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT14_SHIFT		6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TX_TRUNCATE_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TX_TRUNCATE_SHIFT	7
 	u8 flags2;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF0_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT                        0
-#define XSTORM_ISCSI_CONN_AG_CTX_CF1_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT                        2
-#define XSTORM_ISCSI_CONN_AG_CTX_CF2_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT                        4
-#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_MASK              0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT             6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF0_MASK			0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT			0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF1_MASK			0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT			2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF2_MASK			0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT			4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_MASK		0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT	6
 	u8 flags3;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF4_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT                        0
-#define XSTORM_ISCSI_CONN_AG_CTX_CF5_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT                        2
-#define XSTORM_ISCSI_CONN_AG_CTX_CF6_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT                        4
-#define XSTORM_ISCSI_CONN_AG_CTX_CF7_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF7_SHIFT                        6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT	0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT	2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT	4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF7_SHIFT	6
 	u8 flags4;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF8_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF8_SHIFT                        0
-#define XSTORM_ISCSI_CONN_AG_CTX_CF9_MASK                         0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF9_SHIFT                        2
-#define XSTORM_ISCSI_CONN_AG_CTX_CF10_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF10_SHIFT                       4
-#define XSTORM_ISCSI_CONN_AG_CTX_CF11_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF11_SHIFT                       6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF8_SHIFT	0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF9_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF9_SHIFT	2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF10_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF10_SHIFT	4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF11_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF11_SHIFT	6
 	u8 flags5;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF12_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF12_SHIFT                       0
-#define XSTORM_ISCSI_CONN_AG_CTX_CF13_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF13_SHIFT                       2
-#define XSTORM_ISCSI_CONN_AG_CTX_CF14_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF14_SHIFT                       4
-#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_MASK     0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_SHIFT    6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF12_MASK				0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF12_SHIFT				0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF13_MASK				0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF13_SHIFT				2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF14_MASK				0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF14_SHIFT				4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_SHIFT	6
 	u8 flags6;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF16_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF16_SHIFT                       0
-#define XSTORM_ISCSI_CONN_AG_CTX_CF17_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF17_SHIFT                       2
-#define XSTORM_ISCSI_CONN_AG_CTX_CF18_MASK                        0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF18_SHIFT                       4
-#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_MASK                    0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_SHIFT                   6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF16_MASK		0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF16_SHIFT		0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF17_MASK		0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF17_SHIFT		2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF18_MASK		0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF18_SHIFT		4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_SHIFT	6
 	u8 flags7;
-#define XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_MASK	0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_SHIFT        0
-#define XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_MASK	0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_SHIFT        2
-#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_MASK                   0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_SHIFT                  4
-#define XSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT                      6
-#define XSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT                      7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_SHIFT	0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_SHIFT	2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_MASK		0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_SHIFT		4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT			6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT			7
 	u8 flags8;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT                      0
-#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK           0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT          1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT                      2
-#define XSTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT                      3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT                      4
-#define XSTORM_ISCSI_CONN_AG_CTX_CF7EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF7EN_SHIFT                      5
-#define XSTORM_ISCSI_CONN_AG_CTX_CF8EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF8EN_SHIFT                      6
-#define XSTORM_ISCSI_CONN_AG_CTX_CF9EN_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF9EN_SHIFT                      7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT			0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT	1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT			2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT			3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT			4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF7EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF7EN_SHIFT			5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF8EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF8EN_SHIFT			6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF9EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF9EN_SHIFT			7
 	u8 flags9;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF10EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF10EN_SHIFT                     0
-#define XSTORM_ISCSI_CONN_AG_CTX_CF11EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF11EN_SHIFT                     1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF12EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF12EN_SHIFT                     2
-#define XSTORM_ISCSI_CONN_AG_CTX_CF13EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF13EN_SHIFT                     3
-#define XSTORM_ISCSI_CONN_AG_CTX_CF14EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF14EN_SHIFT                     4
-#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_EN_MASK  0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_EN_SHIFT 5
-#define XSTORM_ISCSI_CONN_AG_CTX_CF16EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF16EN_SHIFT                     6
-#define XSTORM_ISCSI_CONN_AG_CTX_CF17EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF17EN_SHIFT                     7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF10EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF10EN_SHIFT			0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF11EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF11EN_SHIFT			1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF12EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF12EN_SHIFT			2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF13EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF13EN_SHIFT			3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF14EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF14EN_SHIFT			4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UPDATE_STATE_TO_BASE_CF_EN_SHIFT	5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF16EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF16EN_SHIFT			6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF17EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF17EN_SHIFT			7
 	u8 flags10;
-#define XSTORM_ISCSI_CONN_AG_CTX_CF18EN_MASK                      0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_CF18EN_SHIFT                     0
-#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_EN_MASK                 0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_EN_SHIFT                1
-#define XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_EN_MASK	0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_EN_SHIFT     2
-#define XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_EN_MASK	0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_EN_SHIFT     3
-#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_EN_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_EN_SHIFT               4
-#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_EN_MASK        0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_EN_SHIFT       5
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK                     0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT                    6
-#define XSTORM_ISCSI_CONN_AG_CTX_MORE_TO_SEND_DEC_RULE_EN_MASK    0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_MORE_TO_SEND_DEC_RULE_EN_SHIFT   7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF18EN_MASK				0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_CF18EN_SHIFT			0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_DQ_FLUSH_EN_SHIFT			1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_MST_XCM_Q0_FLUSH_CF_EN_SHIFT	2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_UST_XCM_Q1_FLUSH_CF_EN_SHIFT	3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_SLOW_PATH_EN_SHIFT			4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_EN_SHIFT		5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT			6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_MORE_TO_SEND_DEC_RULE_EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_MORE_TO_SEND_DEC_RULE_EN_SHIFT	7
 	u8 flags11;
-#define XSTORM_ISCSI_CONN_AG_CTX_TX_BLOCKED_EN_MASK	0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_TX_BLOCKED_EN_SHIFT              0
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK                     0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT                    1
-#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED3_MASK                   0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RESERVED3_SHIFT                  2
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK                     0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT                    3
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK                     0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT                    4
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK                     0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT                    5
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED1_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED1_SHIFT               6
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE9EN_MASK                     0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE9EN_SHIFT                    7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TX_BLOCKED_EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_TX_BLOCKED_EN_SHIFT	0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT	1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RESERVED3_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RESERVED3_SHIFT	2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT	3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT	4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT	5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED1_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED1_SHIFT	6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE9EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE9EN_SHIFT	7
 	u8 flags12;
-#define XSTORM_ISCSI_CONN_AG_CTX_SQ_DEC_RULE_EN_MASK              0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_SQ_DEC_RULE_EN_SHIFT             0
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE11EN_MASK                    0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE11EN_SHIFT                   1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED2_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED2_SHIFT               2
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED3_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED3_SHIFT               3
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE14EN_MASK                    0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE14EN_SHIFT                   4
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE15EN_MASK                    0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE15EN_SHIFT                   5
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE16EN_MASK                    0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE16EN_SHIFT                   6
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE17EN_MASK                    0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_RULE17EN_SHIFT                   7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_SQ_DEC_RULE_EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_SQ_DEC_RULE_EN_SHIFT	0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE11EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE11EN_SHIFT		1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED2_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED2_SHIFT		2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED3_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED3_SHIFT		3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE14EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE14EN_SHIFT		4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE15EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE15EN_SHIFT		5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE16EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE16EN_SHIFT		6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE17EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_RULE17EN_SHIFT		7
 	u8 flags13;
-#define XSTORM_ISCSI_CONN_AG_CTX_R2TQ_DEC_RULE_EN_MASK            0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_R2TQ_DEC_RULE_EN_SHIFT           0
-#define XSTORM_ISCSI_CONN_AG_CTX_HQ_DEC_RULE_EN_MASK              0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_HQ_DEC_RULE_EN_SHIFT             1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED4_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED4_SHIFT               2
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED5_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED5_SHIFT               3
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED6_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED6_SHIFT               4
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED7_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED7_SHIFT               5
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED8_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED8_SHIFT               6
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED9_MASK                0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED9_SHIFT               7
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_R2TQ_DEC_RULE_EN_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_R2TQ_DEC_RULE_EN_SHIFT	0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_HQ_DEC_RULE_EN_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_HQ_DEC_RULE_EN_SHIFT	1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED4_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED4_SHIFT		2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED5_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED5_SHIFT		3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED6_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED6_SHIFT		4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED7_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED7_SHIFT		5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED8_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED8_SHIFT		6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED9_MASK		0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_A0_RESERVED9_SHIFT		7
 	u8 flags14;
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT16_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT16_SHIFT                      0
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT17_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT17_SHIFT                      1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT18_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT18_SHIFT                      2
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT19_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT19_SHIFT                      3
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT20_MASK                       0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_BIT20_SHIFT                      4
-#define XSTORM_ISCSI_CONN_AG_CTX_DUMMY_READ_DONE_MASK             0x1
-#define XSTORM_ISCSI_CONN_AG_CTX_DUMMY_READ_DONE_SHIFT            5
-#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_MASK           0x3
-#define XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_SHIFT          6
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT16_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT16_SHIFT			0
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT17_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT17_SHIFT			1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT18_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT18_SHIFT			2
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT19_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT19_SHIFT			3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT20_MASK			0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_BIT20_SHIFT			4
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_DUMMY_READ_DONE_MASK	0x1
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_DUMMY_READ_DONE_SHIFT	5
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_MASK	0x3
+#define E4_XSTORM_ISCSI_CONN_AG_CTX_PROC_ONLY_CLEANUP_SHIFT	6
 	u8 byte2;
 	__le16 physical_q0;
 	__le16 physical_q1;
@@ -10449,7 +11108,7 @@ struct xstorm_iscsi_conn_ag_ctx {
 	u8 byte13;
 	u8 byte14;
 	u8 byte15;
-	u8 ereserved;
+	u8 e5_reserved;
 	__le16 word11;
 	__le32 reg10;
 	__le32 reg11;
@@ -10461,89 +11120,89 @@ struct xstorm_iscsi_conn_ag_ctx {
 	__le32 reg17;
 };
 
-struct tstorm_iscsi_conn_ag_ctx {
+struct e4_tstorm_iscsi_conn_ag_ctx {
 	u8 reserved0;
 	u8 state;
 	u8 flags0;
-#define TSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_MASK       0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_SHIFT      0
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK               0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT              1
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT2_MASK               0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT2_SHIFT              2
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT3_MASK               0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT3_SHIFT              3
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT4_MASK               0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT4_SHIFT              4
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT5_MASK               0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_BIT5_SHIFT              5
-#define TSTORM_ISCSI_CONN_AG_CTX_CF0_MASK                0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT               6
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_EXIST_IN_QM0_SHIFT	0
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT		1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT2_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT2_SHIFT		2
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT3_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT3_SHIFT		3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT4_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT4_SHIFT		4
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT5_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_BIT5_SHIFT		5
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF0_MASK		0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT		6
 	u8 flags1;
-#define TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_MASK	0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_SHIFT      0
-#define TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_MASK	0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_SHIFT      2
-#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_MASK     0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT    4
-#define TSTORM_ISCSI_CONN_AG_CTX_CF4_MASK                0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT               6
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_MASK		0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_SHIFT		0
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_MASK		0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_SHIFT		2
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_MASK		0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_SHIFT	4
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF4_MASK			0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT			6
 	u8 flags2;
-#define TSTORM_ISCSI_CONN_AG_CTX_CF5_MASK                0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT               0
-#define TSTORM_ISCSI_CONN_AG_CTX_CF6_MASK                0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT               2
-#define TSTORM_ISCSI_CONN_AG_CTX_CF7_MASK                0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF7_SHIFT               4
-#define TSTORM_ISCSI_CONN_AG_CTX_CF8_MASK                0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF8_SHIFT               6
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT	0
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT	2
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF7_MASK	0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF7_SHIFT	4
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF8_MASK	0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF8_SHIFT	6
 	u8 flags3;
-#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_MASK           0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_SHIFT          0
-#define TSTORM_ISCSI_CONN_AG_CTX_CF10_MASK               0x3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF10_SHIFT              2
-#define TSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK              0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT             4
-#define TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_EN_MASK	0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_EN_SHIFT   5
-#define TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_EN_MASK	0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_EN_SHIFT   6
-#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK  0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT 7
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_MASK		0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_SHIFT		0
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF10_MASK			0x3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF10_SHIFT			2
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK			0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT			4
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_P2T_FLUSH_CF_EN_SHIFT	5
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_M2T_FLUSH_CF_EN_SHIFT	6
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_TIMER_STOP_ALL_EN_SHIFT	7
 	u8 flags4;
-#define TSTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK              0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT             0
-#define TSTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK              0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT             1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK              0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT             2
-#define TSTORM_ISCSI_CONN_AG_CTX_CF7EN_MASK              0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF7EN_SHIFT             3
-#define TSTORM_ISCSI_CONN_AG_CTX_CF8EN_MASK              0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF8EN_SHIFT             4
-#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_MASK        0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT       5
-#define TSTORM_ISCSI_CONN_AG_CTX_CF10EN_MASK             0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_CF10EN_SHIFT            6
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT           7
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT		0
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT		1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT		2
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF7EN_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF7EN_SHIFT		3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF8EN_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF8EN_SHIFT		4
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_FLUSH_Q0_EN_SHIFT	5
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF10EN_MASK		0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_CF10EN_SHIFT	6
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT	7
 	u8 flags5;
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT           0
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT           1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT           2
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT           3
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT           4
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT           5
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT           6
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE8EN_MASK            0x1
-#define TSTORM_ISCSI_CONN_AG_CTX_RULE8EN_SHIFT           7
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE8EN_MASK	0x1
+#define E4_TSTORM_ISCSI_CONN_AG_CTX_RULE8EN_SHIFT	7
 	__le32 reg0;
 	__le32 reg1;
 	__le32 reg2;
@@ -10558,63 +11217,63 @@ struct tstorm_iscsi_conn_ag_ctx {
 	__le16 word0;
 };
 
-struct ustorm_iscsi_conn_ag_ctx {
+struct e4_ustorm_iscsi_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define USTORM_ISCSI_CONN_AG_CTX_BIT0_MASK     0x1
-#define USTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT    0
-#define USTORM_ISCSI_CONN_AG_CTX_BIT1_MASK     0x1
-#define USTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT    1
-#define USTORM_ISCSI_CONN_AG_CTX_CF0_MASK      0x3
-#define USTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT     2
-#define USTORM_ISCSI_CONN_AG_CTX_CF1_MASK      0x3
-#define USTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT     4
-#define USTORM_ISCSI_CONN_AG_CTX_CF2_MASK      0x3
-#define USTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_USTORM_ISCSI_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_USTORM_ISCSI_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define USTORM_ISCSI_CONN_AG_CTX_CF3_MASK      0x3
-#define USTORM_ISCSI_CONN_AG_CTX_CF3_SHIFT     0
-#define USTORM_ISCSI_CONN_AG_CTX_CF4_MASK      0x3
-#define USTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT     2
-#define USTORM_ISCSI_CONN_AG_CTX_CF5_MASK      0x3
-#define USTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT     4
-#define USTORM_ISCSI_CONN_AG_CTX_CF6_MASK      0x3
-#define USTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT     6
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF3_MASK	0x3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF3_SHIFT	0
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF4_MASK	0x3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF4_SHIFT	2
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF5_MASK	0x3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF5_SHIFT	4
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF6_MASK	0x3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF6_SHIFT	6
 	u8 flags2;
-#define USTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK    0x1
-#define USTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT   0
-#define USTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK    0x1
-#define USTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT   1
-#define USTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK    0x1
-#define USTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT   2
-#define USTORM_ISCSI_CONN_AG_CTX_CF3EN_MASK    0x1
-#define USTORM_ISCSI_CONN_AG_CTX_CF3EN_SHIFT   3
-#define USTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK    0x1
-#define USTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT   4
-#define USTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK    0x1
-#define USTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT   5
-#define USTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK    0x1
-#define USTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT   6
-#define USTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT 7
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF3EN_MASK		0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF3EN_SHIFT		3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF4EN_MASK		0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF4EN_SHIFT		4
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF5EN_MASK		0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF5EN_SHIFT		5
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF6EN_MASK		0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_CF6EN_SHIFT		6
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT	7
 	u8 flags3;
-#define USTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT 0
-#define USTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT 1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT 2
-#define USTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT 3
-#define USTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT 4
-#define USTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT 5
-#define USTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT 6
-#define USTORM_ISCSI_CONN_AG_CTX_RULE8EN_MASK  0x1
-#define USTORM_ISCSI_CONN_AG_CTX_RULE8EN_SHIFT 7
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT	0
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT	1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT	2
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT	3
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE5EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE5EN_SHIFT	4
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE6EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE6EN_SHIFT	5
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE7EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE7EN_SHIFT	6
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE8EN_MASK	0x1
+#define E4_USTORM_ISCSI_CONN_AG_CTX_RULE8EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -10627,113 +11286,117 @@ struct ustorm_iscsi_conn_ag_ctx {
 	__le16 word3;
 };
 
+/* The iscsi storm connection context of Tstorm */
 struct tstorm_iscsi_conn_st_ctx {
-	__le32 reserved[40];
+	__le32 reserved[44];
 };
 
-struct mstorm_iscsi_conn_ag_ctx {
+struct e4_mstorm_iscsi_conn_ag_ctx {
 	u8 reserved;
 	u8 state;
 	u8 flags0;
-#define MSTORM_ISCSI_CONN_AG_CTX_BIT0_MASK     0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT    0
-#define MSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK     0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT    1
-#define MSTORM_ISCSI_CONN_AG_CTX_CF0_MASK      0x3
-#define MSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT     2
-#define MSTORM_ISCSI_CONN_AG_CTX_CF1_MASK      0x3
-#define MSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT     4
-#define MSTORM_ISCSI_CONN_AG_CTX_CF2_MASK      0x3
-#define MSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define MSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK    0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT   0
-#define MSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK    0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT   1
-#define MSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK    0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT   2
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define MSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_MSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT	7
 	__le16 word0;
 	__le16 word1;
 	__le32 reg0;
 	__le32 reg1;
 };
 
+/* Combined iSCSI and TCP storm connection of Mstorm */
 struct mstorm_iscsi_tcp_conn_st_ctx {
 	__le32 reserved_tcp[20];
-	__le32 reserved_iscsi[8];
+	__le32 reserved_iscsi[12];
 };
 
+/* The iscsi storm context of Ustorm */
 struct ustorm_iscsi_conn_st_ctx {
 	__le32 reserved[52];
 };
 
-struct iscsi_conn_context {
+/* iscsi connection context */
+struct e4_iscsi_conn_context {
 	struct ystorm_iscsi_conn_st_ctx ystorm_st_context;
-	struct regpair ystorm_st_padding[2];
 	struct pstorm_iscsi_tcp_conn_st_ctx pstorm_st_context;
 	struct regpair pstorm_st_padding[2];
 	struct pb_context xpb2_context;
 	struct xstorm_iscsi_tcp_conn_st_ctx xstorm_st_context;
 	struct regpair xstorm_st_padding[2];
-	struct xstorm_iscsi_conn_ag_ctx xstorm_ag_context;
-	struct tstorm_iscsi_conn_ag_ctx tstorm_ag_context;
+	struct e4_xstorm_iscsi_conn_ag_ctx xstorm_ag_context;
+	struct e4_tstorm_iscsi_conn_ag_ctx tstorm_ag_context;
 	struct regpair tstorm_ag_padding[2];
 	struct timers_context timer_context;
-	struct ustorm_iscsi_conn_ag_ctx ustorm_ag_context;
+	struct e4_ustorm_iscsi_conn_ag_ctx ustorm_ag_context;
 	struct pb_context upb_context;
 	struct tstorm_iscsi_conn_st_ctx tstorm_st_context;
 	struct regpair tstorm_st_padding[2];
-	struct mstorm_iscsi_conn_ag_ctx mstorm_ag_context;
+	struct e4_mstorm_iscsi_conn_ag_ctx mstorm_ag_context;
 	struct mstorm_iscsi_tcp_conn_st_ctx mstorm_st_context;
 	struct ustorm_iscsi_conn_st_ctx ustorm_st_context;
 };
 
+/* iSCSI init params passed by driver to FW in iSCSI init ramrod */
 struct iscsi_init_ramrod_params {
 	struct iscsi_spe_func_init iscsi_init_spe;
 	struct tcp_init_params tcp_init;
 };
 
-struct ystorm_iscsi_conn_ag_ctx {
+struct e4_ystorm_iscsi_conn_ag_ctx {
 	u8 byte0;
 	u8 byte1;
 	u8 flags0;
-#define YSTORM_ISCSI_CONN_AG_CTX_BIT0_MASK     0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT    0
-#define YSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK     0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT    1
-#define YSTORM_ISCSI_CONN_AG_CTX_CF0_MASK      0x3
-#define YSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT     2
-#define YSTORM_ISCSI_CONN_AG_CTX_CF1_MASK      0x3
-#define YSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT     4
-#define YSTORM_ISCSI_CONN_AG_CTX_CF2_MASK      0x3
-#define YSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT     6
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_BIT0_MASK	0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_BIT0_SHIFT	0
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_BIT1_MASK	0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_BIT1_SHIFT	1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF0_MASK	0x3
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF0_SHIFT	2
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF1_MASK	0x3
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF1_SHIFT	4
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF2_MASK	0x3
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF2_SHIFT	6
 	u8 flags1;
-#define YSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK    0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT   0
-#define YSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK    0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT   1
-#define YSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK    0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT   2
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK  0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT 3
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK  0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT 4
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK  0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT 5
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK  0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT 6
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK  0x1
-#define YSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT 7
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF0EN_MASK		0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF0EN_SHIFT		0
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF1EN_MASK		0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF1EN_SHIFT		1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF2EN_MASK		0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_CF2EN_SHIFT		2
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE0EN_MASK	0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE0EN_SHIFT	3
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE1EN_MASK	0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE1EN_SHIFT	4
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE2EN_MASK	0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE2EN_SHIFT	5
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE3EN_MASK	0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE3EN_SHIFT	6
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE4EN_MASK	0x1
+#define E4_YSTORM_ISCSI_CONN_AG_CTX_RULE4EN_SHIFT	7
 	u8 byte2;
 	u8 byte3;
 	__le16 word0;
@@ -11613,7 +12276,7 @@ struct public_drv_mb {
 #define DRV_MB_PARAM_DCBX_NOTIFY_MASK		0x000000FF
 #define DRV_MB_PARAM_DCBX_NOTIFY_SHIFT		3
 
-#define DRV_MB_PARAM_NVM_LEN_SHIFT		24
+#define DRV_MB_PARAM_NVM_LEN_OFFSET		24
 
 #define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_SHIFT	0
 #define DRV_MB_PARAM_CFG_VF_MSIX_VF_ID_MASK	0x000000FF
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hw.c b/drivers/net/ethernet/qlogic/qed/qed_hw.c
index a05feb38c6ee..fca2dbd93ad9 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hw.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_hw.c
@@ -807,3 +807,71 @@ int qed_dmae_host2host(struct qed_hwfn *p_hwfn,
 	return rc;
 }
 
+int qed_dmae_sanity(struct qed_hwfn *p_hwfn,
+		    struct qed_ptt *p_ptt, const char *phase)
+{
+	u32 size = PAGE_SIZE / 2, val;
+	struct qed_dmae_params params;
+	int rc = 0;
+	dma_addr_t p_phys;
+	void *p_virt;
+	u32 *p_tmp;
+
+	p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
+				    2 * size, &p_phys, GFP_KERNEL);
+	if (!p_virt) {
+		DP_NOTICE(p_hwfn,
+			  "DMAE sanity [%s]: failed to allocate memory\n",
+			  phase);
+		return -ENOMEM;
+	}
+
+	/* Fill the bottom half of the allocated memory with a known pattern */
+	for (p_tmp = (u32 *)p_virt;
+	     p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) {
+		/* Save the address itself as the value */
+		val = (u32)(uintptr_t)p_tmp;
+		*p_tmp = val;
+	}
+
+	/* Zero the top half of the allocated memory */
+	memset((u8 *)p_virt + size, 0, size);
+
+	DP_VERBOSE(p_hwfn,
+		   QED_MSG_SP,
+		   "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n",
+		   phase,
+		   (u64)p_phys,
+		   p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size);
+
+	memset(&params, 0, sizeof(params));
+	rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size,
+				size / 4 /* size_in_dwords */, &params);
+	if (rc) {
+		DP_NOTICE(p_hwfn,
+			  "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n",
+			  phase, rc);
+		goto out;
+	}
+
+	/* Verify that the top half of the allocated memory has the pattern */
+	for (p_tmp = (u32 *)((u8 *)p_virt + size);
+	     p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) {
+		/* The corresponding address in the bottom half */
+		val = (u32)(uintptr_t)p_tmp - size;
+
+		if (*p_tmp != val) {
+			DP_NOTICE(p_hwfn,
+				  "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n",
+				  phase,
+				  (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt),
+				  p_tmp, *p_tmp, val);
+			rc = -EINVAL;
+			goto out;
+		}
+	}
+
+out:
+	dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys);
+	return rc;
+}
diff --git a/drivers/net/ethernet/qlogic/qed/qed_hw.h b/drivers/net/ethernet/qlogic/qed/qed_hw.h
index f2505c691c26..8db2839a8ec8 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_hw.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_hw.h
@@ -299,4 +299,8 @@ union qed_qm_pq_params {
 
 int qed_init_fw_data(struct qed_dev *cdev,
 		     const u8 *fw_data);
+
+int qed_dmae_sanity(struct qed_hwfn *p_hwfn,
+		    struct qed_ptt *p_ptt, const char *phase);
+
 #endif
diff --git a/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c b/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c
index b069ad088269..18fb5062a83d 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c
@@ -31,6 +31,7 @@
  */
 
 #include <linux/types.h>
+#include <linux/crc8.h>
 #include <linux/delay.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
@@ -40,102 +41,197 @@
 #include "qed_init_ops.h"
 #include "qed_reg_addr.h"
 
+#define CDU_VALIDATION_DEFAULT_CFG	61
+
+static u16 con_region_offsets[3][NUM_OF_CONNECTION_TYPES_E4] = {
+	{400, 336, 352, 304, 304, 384, 416, 352},	/* region 3 offsets */
+	{528, 496, 416, 448, 448, 512, 544, 480},	/* region 4 offsets */
+	{608, 544, 496, 512, 576, 592, 624, 560}	/* region 5 offsets */
+};
+
+static u16 task_region_offsets[1][NUM_OF_CONNECTION_TYPES_E4] = {
+	{240, 240, 112, 0, 0, 0, 0, 96}	/* region 1 offsets */
+};
+
 /* General constants */
 #define QM_PQ_MEM_4KB(pq_size)	(pq_size ? DIV_ROUND_UP((pq_size + 1) *	\
 							QM_PQ_ELEMENT_SIZE, \
 							0x1000) : 0)
 #define QM_PQ_SIZE_256B(pq_size)	(pq_size ? DIV_ROUND_UP(pq_size, \
 								0x100) - 1 : 0)
-#define QM_INVALID_PQ_ID                        0xffff
+#define QM_INVALID_PQ_ID		0xffff
+
 /* Feature enable */
-#define QM_BYPASS_EN                            1
-#define QM_BYTE_CRD_EN                          1
+#define QM_BYPASS_EN	1
+#define QM_BYTE_CRD_EN	1
+
 /* Other PQ constants */
-#define QM_OTHER_PQS_PER_PF                     4
+#define QM_OTHER_PQS_PER_PF	4
+
 /* WFQ constants */
-#define QM_WFQ_UPPER_BOUND		62500000
-#define QM_WFQ_VP_PQ_VOQ_SHIFT          0
-#define QM_WFQ_VP_PQ_PF_SHIFT           5
-#define QM_WFQ_INC_VAL(weight)          ((weight) * 0x9000)
-#define QM_WFQ_MAX_INC_VAL                      43750000
+
+/* Upper bound in MB, 10 * burst size of 1ms in 50Gbps */
+#define QM_WFQ_UPPER_BOUND	62500000
+
+/* Bit  of VOQ in WFQ VP PQ map */
+#define QM_WFQ_VP_PQ_VOQ_SHIFT	0
+
+/* Bit  of PF in WFQ VP PQ map */
+#define QM_WFQ_VP_PQ_PF_E4_SHIFT	5
+
+/* 0x9000 = 4*9*1024 */
+#define QM_WFQ_INC_VAL(weight)	((weight) * 0x9000)
+
+/* Max WFQ increment value is 0.7 * upper bound */
+#define QM_WFQ_MAX_INC_VAL	((QM_WFQ_UPPER_BOUND * 7) / 10)
 
 /* RL constants */
-#define QM_RL_UPPER_BOUND                       62500000
-#define QM_RL_PERIOD                            5               /* in us */
-#define QM_RL_PERIOD_CLK_25M            (25 * QM_RL_PERIOD)
-#define QM_RL_MAX_INC_VAL                       43750000
-#define QM_RL_INC_VAL(rate)		max_t(u32,	\
-					      (u32)(((rate ? rate : \
-						      1000000) *    \
-						     QM_RL_PERIOD * \
-						     101) / (8 * 100)), 1)
+
+/* Period in us */
+#define QM_RL_PERIOD	5
+
+/* Period in 25MHz cycles */
+#define QM_RL_PERIOD_CLK_25M	(25 * QM_RL_PERIOD)
+
+/* RL increment value - rate is specified in mbps */
+#define QM_RL_INC_VAL(rate) ({ \
+	typeof(rate) __rate = (rate); \
+	max_t(u32, \
+	      (u32)(((__rate ? __rate : 1000000) * QM_RL_PERIOD * 101) / \
+		    (8 * 100)), \
+	      1); })
+
+/* PF RL Upper bound is set to 10 * burst size of 1ms in 50Gbps */
+#define QM_PF_RL_UPPER_BOUND	62500000
+
+/* Max PF RL increment value is 0.7 * upper bound */
+#define QM_PF_RL_MAX_INC_VAL	((QM_PF_RL_UPPER_BOUND * 7) / 10)
+
+/* Vport RL Upper bound, link speed is in Mpbs */
+#define QM_VP_RL_UPPER_BOUND(speed)	((u32)max_t(u32, \
+						    QM_RL_INC_VAL(speed), \
+						    9700 + 1000))
+
+/* Max Vport RL increment value is the Vport RL upper bound */
+#define QM_VP_RL_MAX_INC_VAL(speed)	QM_VP_RL_UPPER_BOUND(speed)
+
+/* Vport RL credit threshold in case of QM bypass */
+#define QM_VP_RL_BYPASS_THRESH_SPEED	(QM_VP_RL_UPPER_BOUND(10000) - 1)
+
 /* AFullOprtnstcCrdMask constants */
-#define QM_OPPOR_LINE_VOQ_DEF           1
-#define QM_OPPOR_FW_STOP_DEF            0
-#define QM_OPPOR_PQ_EMPTY_DEF           1
+#define QM_OPPOR_LINE_VOQ_DEF	1
+#define QM_OPPOR_FW_STOP_DEF	0
+#define QM_OPPOR_PQ_EMPTY_DEF	1
+
 /* Command Queue constants */
-#define PBF_CMDQ_PURE_LB_LINES                          150
-#define PBF_CMDQ_LINES_RT_OFFSET(voq)           (		 \
-		PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET + voq * \
-		(PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET -	 \
-		 PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET))
-#define PBF_BTB_GUARANTEED_RT_OFFSET(voq)       (	      \
-		PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET + voq * \
-		(PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET -      \
-		 PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET))
-#define QM_VOQ_LINE_CRD(pbf_cmd_lines)          ((((pbf_cmd_lines) - \
-						   4) *		     \
-						  2) | QM_LINE_CRD_REG_SIGN_BIT)
+
+/* Pure LB CmdQ lines (+spare) */
+#define PBF_CMDQ_PURE_LB_LINES	150
+
+#define PBF_CMDQ_LINES_E5_RSVD_RATIO	8
+
+#define PBF_CMDQ_LINES_RT_OFFSET(ext_voq) \
+	(PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET + \
+	 (ext_voq) * (PBF_REG_YCMD_QS_NUM_LINES_VOQ1_RT_OFFSET - \
+		PBF_REG_YCMD_QS_NUM_LINES_VOQ0_RT_OFFSET))
+
+#define PBF_BTB_GUARANTEED_RT_OFFSET(ext_voq) \
+	(PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET + \
+	 (ext_voq) * (PBF_REG_BTB_GUARANTEED_VOQ1_RT_OFFSET - \
+		PBF_REG_BTB_GUARANTEED_VOQ0_RT_OFFSET))
+
+#define QM_VOQ_LINE_CRD(pbf_cmd_lines) \
+	((((pbf_cmd_lines) - 4) * 2) | QM_LINE_CRD_REG_SIGN_BIT)
+
 /* BTB: blocks constants (block size = 256B) */
-#define BTB_JUMBO_PKT_BLOCKS            38
-#define BTB_HEADROOM_BLOCKS                     BTB_JUMBO_PKT_BLOCKS
-#define BTB_PURE_LB_FACTOR                      10
-#define BTB_PURE_LB_RATIO                       7
+
+/* 256B blocks in 9700B packet */
+#define BTB_JUMBO_PKT_BLOCKS	38
+
+/* Headroom per-port */
+#define BTB_HEADROOM_BLOCKS	BTB_JUMBO_PKT_BLOCKS
+#define BTB_PURE_LB_FACTOR	10
+
+/* Factored (hence really 0.7) */
+#define BTB_PURE_LB_RATIO	7
+
 /* QM stop command constants */
-#define QM_STOP_PQ_MASK_WIDTH           32
-#define QM_STOP_CMD_ADDR                2
-#define QM_STOP_CMD_STRUCT_SIZE         2
-#define QM_STOP_CMD_PAUSE_MASK_OFFSET   0
-#define QM_STOP_CMD_PAUSE_MASK_SHIFT    0
-#define QM_STOP_CMD_PAUSE_MASK_MASK     -1
-#define QM_STOP_CMD_GROUP_ID_OFFSET     1
-#define QM_STOP_CMD_GROUP_ID_SHIFT      16
-#define QM_STOP_CMD_GROUP_ID_MASK       15
-#define QM_STOP_CMD_PQ_TYPE_OFFSET      1
-#define QM_STOP_CMD_PQ_TYPE_SHIFT       24
-#define QM_STOP_CMD_PQ_TYPE_MASK        1
-#define QM_STOP_CMD_MAX_POLL_COUNT      100
-#define QM_STOP_CMD_POLL_PERIOD_US      500
+#define QM_STOP_PQ_MASK_WIDTH		32
+#define QM_STOP_CMD_ADDR		2
+#define QM_STOP_CMD_STRUCT_SIZE		2
+#define QM_STOP_CMD_PAUSE_MASK_OFFSET	0
+#define QM_STOP_CMD_PAUSE_MASK_SHIFT	0
+#define QM_STOP_CMD_PAUSE_MASK_MASK	-1
+#define QM_STOP_CMD_GROUP_ID_OFFSET	1
+#define QM_STOP_CMD_GROUP_ID_SHIFT	16
+#define QM_STOP_CMD_GROUP_ID_MASK	15
+#define QM_STOP_CMD_PQ_TYPE_OFFSET	1
+#define QM_STOP_CMD_PQ_TYPE_SHIFT	24
+#define QM_STOP_CMD_PQ_TYPE_MASK	1
+#define QM_STOP_CMD_MAX_POLL_COUNT	100
+#define QM_STOP_CMD_POLL_PERIOD_US	500
 
 /* QM command macros */
-#define QM_CMD_STRUCT_SIZE(cmd)			cmd ## \
-	_STRUCT_SIZE
-#define QM_CMD_SET_FIELD(var, cmd, field,				  \
-			 value)        SET_FIELD(var[cmd ## _ ## field ## \
-						     _OFFSET],		  \
-						 cmd ## _ ## field,	  \
-						 value)
-/* QM: VOQ macros */
-#define PHYS_VOQ(port, tc, max_phys_tcs_per_port) ((port) *	\
-						   (max_phys_tcs_per_port) + \
-						   (tc))
-#define LB_VOQ(port)				( \
-		MAX_PHYS_VOQS + (port))
-#define VOQ(port, tc, max_phy_tcs_pr_port)	\
-	((tc) <		\
-	 LB_TC ? PHYS_VOQ(port,		\
-			  tc,			 \
-			  max_phy_tcs_pr_port) \
-		: LB_VOQ(port))
+#define QM_CMD_STRUCT_SIZE(cmd)	cmd ## _STRUCT_SIZE
+#define QM_CMD_SET_FIELD(var, cmd, field, value) \
+	SET_FIELD(var[cmd ## _ ## field ## _OFFSET], \
+		  cmd ## _ ## field, \
+		  value)
+
+#define QM_INIT_TX_PQ_MAP(p_hwfn, map, chip, pq_id, rl_valid, vp_pq_id, rl_id, \
+			  ext_voq, wrr) \
+	do { \
+		typeof(map) __map; \
+		memset(&__map, 0, sizeof(__map)); \
+		SET_FIELD(__map.reg, QM_RF_PQ_MAP_ ## chip ## _PQ_VALID, 1); \
+		SET_FIELD(__map.reg, QM_RF_PQ_MAP_ ## chip ## _RL_VALID, \
+			  rl_valid); \
+		SET_FIELD(__map.reg, QM_RF_PQ_MAP_ ## chip ## _VP_PQ_ID, \
+			  vp_pq_id); \
+		SET_FIELD(__map.reg, QM_RF_PQ_MAP_ ## chip ## _RL_ID, rl_id); \
+		SET_FIELD(__map.reg, QM_RF_PQ_MAP_ ## chip ## _VOQ, ext_voq); \
+		SET_FIELD(__map.reg, \
+			  QM_RF_PQ_MAP_ ## chip ## _WRR_WEIGHT_GROUP, wrr); \
+		STORE_RT_REG(p_hwfn, QM_REG_TXPQMAP_RT_OFFSET + (pq_id), \
+			     *((u32 *)&__map)); \
+		(map) = __map; \
+	} while (0)
+
+#define WRITE_PQ_INFO_TO_RAM	1
+#define PQ_INFO_ELEMENT(vp, pf, tc, port, rl_valid, rl) \
+	(((vp) << 0) | ((pf) << 12) | ((tc) << 16) | ((port) << 20) | \
+	((rl_valid) << 22) | ((rl) << 24))
+#define PQ_INFO_RAM_GRC_ADDRESS(pq_id) \
+	(XSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM + 21776 + (pq_id) * 4)
+
 /******************** INTERNAL IMPLEMENTATION *********************/
+
+/* Returns the external VOQ number */
+static u8 qed_get_ext_voq(struct qed_hwfn *p_hwfn,
+			  u8 port_id, u8 tc, u8 max_phys_tcs_per_port)
+{
+	if (tc == PURE_LB_TC)
+		return NUM_OF_PHYS_TCS * MAX_NUM_PORTS_BB + port_id;
+	else
+		return port_id * max_phys_tcs_per_port + tc;
+}
+
 /* Prepare PF RL enable/disable runtime init values */
 static void qed_enable_pf_rl(struct qed_hwfn *p_hwfn, bool pf_rl_en)
 {
 	STORE_RT_REG(p_hwfn, QM_REG_RLPFENABLE_RT_OFFSET, pf_rl_en ? 1 : 0);
 	if (pf_rl_en) {
+		u8 num_ext_voqs = MAX_NUM_VOQS_E4;
+		u64 voq_bit_mask = ((u64)1 << num_ext_voqs) - 1;
+
 		/* Enable RLs for all VOQs */
-		STORE_RT_REG(p_hwfn, QM_REG_RLPFVOQENABLE_RT_OFFSET,
-			     (1 << MAX_NUM_VOQS) - 1);
+		STORE_RT_REG(p_hwfn,
+			     QM_REG_RLPFVOQENABLE_RT_OFFSET,
+			     (u32)voq_bit_mask);
+		if (num_ext_voqs >= 32)
+			STORE_RT_REG(p_hwfn, QM_REG_RLPFVOQENABLE_MSB_RT_OFFSET,
+				     (u32)(voq_bit_mask >> 32));
+
 		/* Write RL period */
 		STORE_RT_REG(p_hwfn,
 			     QM_REG_RLPFPERIOD_RT_OFFSET, QM_RL_PERIOD_CLK_25M);
@@ -147,7 +243,7 @@ static void qed_enable_pf_rl(struct qed_hwfn *p_hwfn, bool pf_rl_en)
 		if (QM_BYPASS_EN)
 			STORE_RT_REG(p_hwfn,
 				     QM_REG_AFULLQMBYPTHRPFRL_RT_OFFSET,
-				     QM_RL_UPPER_BOUND);
+				     QM_PF_RL_UPPER_BOUND);
 	}
 }
 
@@ -181,7 +277,7 @@ static void qed_enable_vport_rl(struct qed_hwfn *p_hwfn, bool vport_rl_en)
 		if (QM_BYPASS_EN)
 			STORE_RT_REG(p_hwfn,
 				     QM_REG_AFULLQMBYPTHRGLBLRL_RT_OFFSET,
-				     QM_RL_UPPER_BOUND);
+				     QM_VP_RL_BYPASS_THRESH_SPEED);
 	}
 }
 
@@ -202,15 +298,15 @@ static void qed_enable_vport_wfq(struct qed_hwfn *p_hwfn, bool vport_wfq_en)
  * the specified VOQ.
  */
 static void qed_cmdq_lines_voq_rt_init(struct qed_hwfn *p_hwfn,
-				       u8 voq, u16 cmdq_lines)
+				       u8 ext_voq, u16 cmdq_lines)
 {
-	u32 qm_line_crd;
+	u32 qm_line_crd = QM_VOQ_LINE_CRD(cmdq_lines);
 
-	qm_line_crd = QM_VOQ_LINE_CRD(cmdq_lines);
-	OVERWRITE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(voq),
+	OVERWRITE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(ext_voq),
 			 (u32)cmdq_lines);
-	STORE_RT_REG(p_hwfn, QM_REG_VOQCRDLINE_RT_OFFSET + voq, qm_line_crd);
-	STORE_RT_REG(p_hwfn, QM_REG_VOQINITCRDLINE_RT_OFFSET + voq,
+	STORE_RT_REG(p_hwfn, QM_REG_VOQCRDLINE_RT_OFFSET + ext_voq,
+		     qm_line_crd);
+	STORE_RT_REG(p_hwfn, QM_REG_VOQINITCRDLINE_RT_OFFSET + ext_voq,
 		     qm_line_crd);
 }
 
@@ -221,43 +317,52 @@ static void qed_cmdq_lines_rt_init(
 	u8 max_phys_tcs_per_port,
 	struct init_qm_port_params port_params[MAX_NUM_PORTS])
 {
-	u8 tc, voq, port_id, num_tcs_in_port;
+	u8 tc, ext_voq, port_id, num_tcs_in_port;
+	u8 num_ext_voqs = MAX_NUM_VOQS_E4;
+
+	/* Clear PBF lines of all VOQs */
+	for (ext_voq = 0; ext_voq < num_ext_voqs; ext_voq++)
+		STORE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(ext_voq), 0);
 
-	/* Clear PBF lines for all VOQs */
-	for (voq = 0; voq < MAX_NUM_VOQS; voq++)
-		STORE_RT_REG(p_hwfn, PBF_CMDQ_LINES_RT_OFFSET(voq), 0);
 	for (port_id = 0; port_id < max_ports_per_engine; port_id++) {
-		if (port_params[port_id].active) {
-			u16 phys_lines, phys_lines_per_tc;
-
-			/* find #lines to divide between active phys TCs */
-			phys_lines = port_params[port_id].num_pbf_cmd_lines -
-				     PBF_CMDQ_PURE_LB_LINES;
-			/* find #lines per active physical TC */
-			num_tcs_in_port = 0;
-			for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
-				if (((port_params[port_id].active_phys_tcs >>
-				      tc) & 0x1) == 1)
-					num_tcs_in_port++;
-			}
+		u16 phys_lines, phys_lines_per_tc;
 
-			phys_lines_per_tc = phys_lines / num_tcs_in_port;
-			/* init registers per active TC */
-			for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
-				if (((port_params[port_id].active_phys_tcs >>
-				      tc) & 0x1) != 1)
-					continue;
+		if (!port_params[port_id].active)
+			continue;
 
-				voq = PHYS_VOQ(port_id, tc,
-					       max_phys_tcs_per_port);
-				qed_cmdq_lines_voq_rt_init(p_hwfn, voq,
-							   phys_lines_per_tc);
-			}
+		/* Find number of command queue lines to divide between the
+		 * active physical TCs. In E5, 1/8 of the lines are reserved.
+		 * the lines for pure LB TC are subtracted.
+		 */
+		phys_lines = port_params[port_id].num_pbf_cmd_lines;
+		phys_lines -= PBF_CMDQ_PURE_LB_LINES;
+
+		/* Find #lines per active physical TC */
+		num_tcs_in_port = 0;
+		for (tc = 0; tc < max_phys_tcs_per_port; tc++)
+			if (((port_params[port_id].active_phys_tcs >>
+			      tc) & 0x1) == 1)
+				num_tcs_in_port++;
+		phys_lines_per_tc = phys_lines / num_tcs_in_port;
 
-			/* init registers for pure LB TC */
-			qed_cmdq_lines_voq_rt_init(p_hwfn, LB_VOQ(port_id),
-						   PBF_CMDQ_PURE_LB_LINES);
+		/* Init registers per active TC */
+		for (tc = 0; tc < max_phys_tcs_per_port; tc++) {
+			ext_voq = qed_get_ext_voq(p_hwfn,
+						  port_id,
+						  tc, max_phys_tcs_per_port);
+			if (((port_params[port_id].active_phys_tcs >>
+			      tc) & 0x1) == 1)
+				qed_cmdq_lines_voq_rt_init(p_hwfn,
+							   ext_voq,
+							   phys_lines_per_tc);
 		}
+
+		/* Init registers for pure LB TC */
+		ext_voq = qed_get_ext_voq(p_hwfn,
+					  port_id,
+					  PURE_LB_TC, max_phys_tcs_per_port);
+		qed_cmdq_lines_voq_rt_init(p_hwfn,
+					   ext_voq, PBF_CMDQ_PURE_LB_LINES);
 	}
 }
 
@@ -268,11 +373,9 @@ static void qed_btb_blocks_rt_init(
 	struct init_qm_port_params port_params[MAX_NUM_PORTS])
 {
 	u32 usable_blocks, pure_lb_blocks, phys_blocks;
-	u8 tc, voq, port_id, num_tcs_in_port;
+	u8 tc, ext_voq, port_id, num_tcs_in_port;
 
 	for (port_id = 0; port_id < max_ports_per_engine; port_id++) {
-		u32 temp;
-
 		if (!port_params[port_id].active)
 			continue;
 
@@ -280,13 +383,14 @@ static void qed_btb_blocks_rt_init(
 		usable_blocks = port_params[port_id].num_btb_blocks -
 				BTB_HEADROOM_BLOCKS;
 
-		/* find blocks per physical TC */
+		/* Find blocks per physical TC. Use factor to avoid floating
+		 * arithmethic.
+		 */
 		num_tcs_in_port = 0;
-		for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
+		for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++)
 			if (((port_params[port_id].active_phys_tcs >>
 			      tc) & 0x1) == 1)
 				num_tcs_in_port++;
-		}
 
 		pure_lb_blocks = (usable_blocks * BTB_PURE_LB_FACTOR) /
 				 (num_tcs_in_port * BTB_PURE_LB_FACTOR +
@@ -299,47 +403,55 @@ static void qed_btb_blocks_rt_init(
 		/* Init physical TCs */
 		for (tc = 0; tc < NUM_OF_PHYS_TCS; tc++) {
 			if (((port_params[port_id].active_phys_tcs >>
-			      tc) & 0x1) != 1)
-				continue;
-
-			voq = PHYS_VOQ(port_id, tc,
-				       max_phys_tcs_per_port);
-			STORE_RT_REG(p_hwfn, PBF_BTB_GUARANTEED_RT_OFFSET(voq),
-				     phys_blocks);
+			      tc) & 0x1) == 1) {
+				ext_voq =
+					qed_get_ext_voq(p_hwfn,
+							port_id,
+							tc,
+							max_phys_tcs_per_port);
+				STORE_RT_REG(p_hwfn,
+					     PBF_BTB_GUARANTEED_RT_OFFSET
+					     (ext_voq), phys_blocks);
+			}
 		}
 
 		/* Init pure LB TC */
-		temp = LB_VOQ(port_id);
-		STORE_RT_REG(p_hwfn, PBF_BTB_GUARANTEED_RT_OFFSET(temp),
+		ext_voq = qed_get_ext_voq(p_hwfn,
+					  port_id,
+					  PURE_LB_TC, max_phys_tcs_per_port);
+		STORE_RT_REG(p_hwfn, PBF_BTB_GUARANTEED_RT_OFFSET(ext_voq),
 			     pure_lb_blocks);
 	}
 }
 
 /* Prepare Tx PQ mapping runtime init values for the specified PF */
-static void qed_tx_pq_map_rt_init(
-	struct qed_hwfn *p_hwfn,
-	struct qed_ptt *p_ptt,
-	struct qed_qm_pf_rt_init_params *p_params,
-	u32 base_mem_addr_4kb)
+static void qed_tx_pq_map_rt_init(struct qed_hwfn *p_hwfn,
+				  struct qed_ptt *p_ptt,
+				  struct qed_qm_pf_rt_init_params *p_params,
+				  u32 base_mem_addr_4kb)
 {
-	struct init_qm_vport_params *vport_params = p_params->vport_params;
-	u16 num_pqs = p_params->num_pf_pqs + p_params->num_vf_pqs;
-	u16 first_pq_group = p_params->start_pq / QM_PF_QUEUE_GROUP_SIZE;
-	u16 last_pq_group = (p_params->start_pq + num_pqs - 1) /
-			    QM_PF_QUEUE_GROUP_SIZE;
-	u16 i, pq_id, pq_group;
-
-	/* A bit per Tx PQ indicating if the PQ is associated with a VF */
 	u32 tx_pq_vf_mask[MAX_QM_TX_QUEUES / QM_PF_QUEUE_GROUP_SIZE] = { 0 };
+	struct init_qm_vport_params *vport_params = p_params->vport_params;
 	u32 num_tx_pq_vf_masks = MAX_QM_TX_QUEUES / QM_PF_QUEUE_GROUP_SIZE;
-	u32 pq_mem_4kb = QM_PQ_MEM_4KB(p_params->num_pf_cids);
-	u32 vport_pq_mem_4kb = QM_PQ_MEM_4KB(p_params->num_vf_cids);
-	u32 mem_addr_4kb = base_mem_addr_4kb;
+	u16 num_pqs, first_pq_group, last_pq_group, i, j, pq_id, pq_group;
+	struct init_qm_pq_params *pq_params = p_params->pq_params;
+	u32 pq_mem_4kb, vport_pq_mem_4kb, mem_addr_4kb;
+
+	num_pqs = p_params->num_pf_pqs + p_params->num_vf_pqs;
+
+	first_pq_group = p_params->start_pq / QM_PF_QUEUE_GROUP_SIZE;
+	last_pq_group = (p_params->start_pq + num_pqs - 1) /
+			QM_PF_QUEUE_GROUP_SIZE;
+
+	pq_mem_4kb = QM_PQ_MEM_4KB(p_params->num_pf_cids);
+	vport_pq_mem_4kb = QM_PQ_MEM_4KB(p_params->num_vf_cids);
+	mem_addr_4kb = base_mem_addr_4kb;
 
 	/* Set mapping from PQ group to PF */
 	for (pq_group = first_pq_group; pq_group <= last_pq_group; pq_group++)
 		STORE_RT_REG(p_hwfn, QM_REG_PQTX2PF_0_RT_OFFSET + pq_group,
 			     (u32)(p_params->pf_id));
+
 	/* Set PQ sizes */
 	STORE_RT_REG(p_hwfn, QM_REG_MAXPQSIZE_0_RT_OFFSET,
 		     QM_PQ_SIZE_256B(p_params->num_pf_cids));
@@ -348,58 +460,82 @@ static void qed_tx_pq_map_rt_init(
 
 	/* Go over all Tx PQs */
 	for (i = 0, pq_id = p_params->start_pq; i < num_pqs; i++, pq_id++) {
-		u8 voq = VOQ(p_params->port_id, p_params->pq_params[i].tc_id,
-			     p_params->max_phys_tcs_per_port);
-		bool is_vf_pq = (i >= p_params->num_pf_pqs);
-		struct qm_rf_pq_map tx_pq_map;
-
-		bool rl_valid = p_params->pq_params[i].rl_valid &&
-				(p_params->pq_params[i].vport_id <
-				 MAX_QM_GLOBAL_RLS);
+		u8 ext_voq, vport_id_in_pf, tc_id = pq_params[i].tc_id;
+		u32 max_qm_global_rls = MAX_QM_GLOBAL_RLS;
+		struct qm_rf_pq_map_e4 tx_pq_map;
+		bool is_vf_pq, rl_valid;
+		u16 *p_first_tx_pq_id;
+
+		ext_voq = qed_get_ext_voq(p_hwfn,
+					  p_params->port_id,
+					  tc_id,
+					  p_params->max_phys_tcs_per_port);
+		is_vf_pq = (i >= p_params->num_pf_pqs);
+		rl_valid = pq_params[i].rl_valid &&
+			   pq_params[i].vport_id < max_qm_global_rls;
 
 		/* Update first Tx PQ of VPORT/TC */
-		u8 vport_id_in_pf = p_params->pq_params[i].vport_id -
-				    p_params->start_vport;
-		u16 *pq_ids = &vport_params[vport_id_in_pf].first_tx_pq_id[0];
-		u16 first_tx_pq_id = pq_ids[p_params->pq_params[i].tc_id];
+		vport_id_in_pf = pq_params[i].vport_id - p_params->start_vport;
+		p_first_tx_pq_id =
+		    &vport_params[vport_id_in_pf].first_tx_pq_id[tc_id];
+		if (*p_first_tx_pq_id == QM_INVALID_PQ_ID) {
+			u32 map_val =
+				(ext_voq << QM_WFQ_VP_PQ_VOQ_SHIFT) |
+				(p_params->pf_id << QM_WFQ_VP_PQ_PF_E4_SHIFT);
 
-		if (first_tx_pq_id == QM_INVALID_PQ_ID) {
 			/* Create new VP PQ */
-			pq_ids[p_params->pq_params[i].tc_id] = pq_id;
-			first_tx_pq_id = pq_id;
+			*p_first_tx_pq_id = pq_id;
 
 			/* Map VP PQ to VOQ and PF */
 			STORE_RT_REG(p_hwfn,
 				     QM_REG_WFQVPMAP_RT_OFFSET +
-				     first_tx_pq_id,
-				     (voq << QM_WFQ_VP_PQ_VOQ_SHIFT) |
-				     (p_params->pf_id <<
-				      QM_WFQ_VP_PQ_PF_SHIFT));
+				     *p_first_tx_pq_id,
+				     map_val);
 		}
 
-		if (p_params->pq_params[i].rl_valid && !rl_valid)
+		/* Check RL ID */
+		if (pq_params[i].rl_valid && pq_params[i].vport_id >=
+		    max_qm_global_rls)
 			DP_NOTICE(p_hwfn,
-				  "Invalid VPORT ID for rate limiter configuration");
-		/* Fill PQ map entry */
-		memset(&tx_pq_map, 0, sizeof(tx_pq_map));
-		SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_PQ_VALID, 1);
-		SET_FIELD(tx_pq_map.reg,
-			  QM_RF_PQ_MAP_RL_VALID, rl_valid ? 1 : 0);
-		SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_VP_PQ_ID, first_tx_pq_id);
-		SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_RL_ID,
-			  rl_valid ?
-			  p_params->pq_params[i].vport_id : 0);
-		SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_VOQ, voq);
-		SET_FIELD(tx_pq_map.reg, QM_RF_PQ_MAP_WRR_WEIGHT_GROUP,
-			  p_params->pq_params[i].wrr_group);
-		/* Write PQ map entry to CAM */
-		STORE_RT_REG(p_hwfn, QM_REG_TXPQMAP_RT_OFFSET + pq_id,
-			     *((u32 *)&tx_pq_map));
-		/* Set base address */
+				  "Invalid VPORT ID for rate limiter configuration\n");
+
+		/* Prepare PQ map entry */
+		QM_INIT_TX_PQ_MAP(p_hwfn,
+				  tx_pq_map,
+				  E4,
+				  pq_id,
+				  rl_valid ? 1 : 0,
+				  *p_first_tx_pq_id,
+				  rl_valid ? pq_params[i].vport_id : 0,
+				  ext_voq, pq_params[i].wrr_group);
+
+		/* Set PQ base address */
 		STORE_RT_REG(p_hwfn,
 			     QM_REG_BASEADDRTXPQ_RT_OFFSET + pq_id,
 			     mem_addr_4kb);
 
+		/* Clear PQ pointer table entry (64 bit) */
+		if (p_params->is_pf_loading)
+			for (j = 0; j < 2; j++)
+				STORE_RT_REG(p_hwfn,
+					     QM_REG_PTRTBLTX_RT_OFFSET +
+					     (pq_id * 2) + j, 0);
+
+		/* Write PQ info to RAM */
+		if (WRITE_PQ_INFO_TO_RAM != 0) {
+			u32 pq_info = 0;
+
+			pq_info = PQ_INFO_ELEMENT(*p_first_tx_pq_id,
+						  p_params->pf_id,
+						  tc_id,
+						  p_params->port_id,
+						  rl_valid ? 1 : 0,
+						  rl_valid ?
+						  pq_params[i].vport_id : 0);
+			qed_wr(p_hwfn, p_ptt, PQ_INFO_RAM_GRC_ADDRESS(pq_id),
+			       pq_info);
+		}
+
 		/* If VF PQ, add indication to PQ VF mask */
 		if (is_vf_pq) {
 			tx_pq_vf_mask[pq_id /
@@ -421,16 +557,16 @@ static void qed_tx_pq_map_rt_init(
 
 /* Prepare Other PQ mapping runtime init values for the specified PF */
 static void qed_other_pq_map_rt_init(struct qed_hwfn *p_hwfn,
-				     u8 port_id,
 				     u8 pf_id,
+				     bool is_pf_loading,
 				     u32 num_pf_cids,
 				     u32 num_tids, u32 base_mem_addr_4kb)
 {
 	u32 pq_size, pq_mem_4kb, mem_addr_4kb;
-	u16 i, pq_id, pq_group;
+	u16 i, j, pq_id, pq_group;
 
-	/* a single other PQ group is used in each PF,
-	 * where PQ group i is used in PF i.
+	/* A single other PQ group is used in each PF, where PQ group i is used
+	 * in PF i.
 	 */
 	pq_group = pf_id;
 	pq_size = num_pf_cids + num_tids;
@@ -440,16 +576,25 @@ static void qed_other_pq_map_rt_init(struct qed_hwfn *p_hwfn,
 	/* Map PQ group to PF */
 	STORE_RT_REG(p_hwfn, QM_REG_PQOTHER2PF_0_RT_OFFSET + pq_group,
 		     (u32)(pf_id));
+
 	/* Set PQ sizes */
 	STORE_RT_REG(p_hwfn, QM_REG_MAXPQSIZE_2_RT_OFFSET,
 		     QM_PQ_SIZE_256B(pq_size));
 
-	/* Set base address */
 	for (i = 0, pq_id = pf_id * QM_PF_QUEUE_GROUP_SIZE;
 	     i < QM_OTHER_PQS_PER_PF; i++, pq_id++) {
+		/* Set PQ base address */
 		STORE_RT_REG(p_hwfn,
 			     QM_REG_BASEADDROTHERPQ_RT_OFFSET + pq_id,
 			     mem_addr_4kb);
+
+		/* Clear PQ pointer table entry */
+		if (is_pf_loading)
+			for (j = 0; j < 2; j++)
+				STORE_RT_REG(p_hwfn,
+					     QM_REG_PTRTBLOTHER_RT_OFFSET +
+					     (pq_id * 2) + j, 0);
+
 		mem_addr_4kb += pq_mem_4kb;
 	}
 }
@@ -461,16 +606,11 @@ static int qed_pf_wfq_rt_init(struct qed_hwfn *p_hwfn,
 			      struct qed_qm_pf_rt_init_params *p_params)
 {
 	u16 num_tx_pqs = p_params->num_pf_pqs + p_params->num_vf_pqs;
-	u32 crd_reg_offset;
-	u32 inc_val;
+	struct init_qm_pq_params *pq_params = p_params->pq_params;
+	u32 inc_val, crd_reg_offset;
+	u8 ext_voq;
 	u16 i;
 
-	if (p_params->pf_id < MAX_NUM_PFS_BB)
-		crd_reg_offset = QM_REG_WFQPFCRD_RT_OFFSET;
-	else
-		crd_reg_offset = QM_REG_WFQPFCRD_MSB_RT_OFFSET;
-	crd_reg_offset += p_params->pf_id % MAX_NUM_PFS_BB;
-
 	inc_val = QM_WFQ_INC_VAL(p_params->pf_wfq);
 	if (!inc_val || inc_val > QM_WFQ_MAX_INC_VAL) {
 		DP_NOTICE(p_hwfn, "Invalid PF WFQ weight configuration\n");
@@ -478,19 +618,26 @@ static int qed_pf_wfq_rt_init(struct qed_hwfn *p_hwfn,
 	}
 
 	for (i = 0; i < num_tx_pqs; i++) {
-		u8 voq = VOQ(p_params->port_id, p_params->pq_params[i].tc_id,
-			     p_params->max_phys_tcs_per_port);
-
+		ext_voq = qed_get_ext_voq(p_hwfn,
+					  p_params->port_id,
+					  pq_params[i].tc_id,
+					  p_params->max_phys_tcs_per_port);
+		crd_reg_offset =
+			(p_params->pf_id < MAX_NUM_PFS_BB ?
+			 QM_REG_WFQPFCRD_RT_OFFSET :
+			 QM_REG_WFQPFCRD_MSB_RT_OFFSET) +
+			ext_voq * MAX_NUM_PFS_BB +
+			(p_params->pf_id % MAX_NUM_PFS_BB);
 		OVERWRITE_RT_REG(p_hwfn,
-				 crd_reg_offset + voq * MAX_NUM_PFS_BB,
-				 QM_WFQ_CRD_REG_SIGN_BIT);
+				 crd_reg_offset, (u32)QM_WFQ_CRD_REG_SIGN_BIT);
 	}
 
 	STORE_RT_REG(p_hwfn,
 		     QM_REG_WFQPFUPPERBOUND_RT_OFFSET + p_params->pf_id,
-		     QM_WFQ_UPPER_BOUND | QM_WFQ_CRD_REG_SIGN_BIT);
+		     QM_WFQ_UPPER_BOUND | (u32)QM_WFQ_CRD_REG_SIGN_BIT);
 	STORE_RT_REG(p_hwfn, QM_REG_WFQPFWEIGHT_RT_OFFSET + p_params->pf_id,
 		     inc_val);
+
 	return 0;
 }
 
@@ -501,15 +648,19 @@ static int qed_pf_rl_rt_init(struct qed_hwfn *p_hwfn, u8 pf_id, u32 pf_rl)
 {
 	u32 inc_val = QM_RL_INC_VAL(pf_rl);
 
-	if (inc_val > QM_RL_MAX_INC_VAL) {
+	if (inc_val > QM_PF_RL_MAX_INC_VAL) {
 		DP_NOTICE(p_hwfn, "Invalid PF rate limit configuration\n");
 		return -1;
 	}
-	STORE_RT_REG(p_hwfn, QM_REG_RLPFCRD_RT_OFFSET + pf_id,
-		     QM_RL_CRD_REG_SIGN_BIT);
-	STORE_RT_REG(p_hwfn, QM_REG_RLPFUPPERBOUND_RT_OFFSET + pf_id,
-		     QM_RL_UPPER_BOUND | QM_RL_CRD_REG_SIGN_BIT);
+
+	STORE_RT_REG(p_hwfn,
+		     QM_REG_RLPFCRD_RT_OFFSET + pf_id,
+		     (u32)QM_RL_CRD_REG_SIGN_BIT);
+	STORE_RT_REG(p_hwfn,
+		     QM_REG_RLPFUPPERBOUND_RT_OFFSET + pf_id,
+		     QM_PF_RL_UPPER_BOUND | (u32)QM_RL_CRD_REG_SIGN_BIT);
 	STORE_RT_REG(p_hwfn, QM_REG_RLPFINCVAL_RT_OFFSET + pf_id, inc_val);
+
 	return 0;
 }
 
@@ -520,12 +671,12 @@ static int qed_vp_wfq_rt_init(struct qed_hwfn *p_hwfn,
 			      u8 num_vports,
 			      struct init_qm_vport_params *vport_params)
 {
+	u16 vport_pq_id;
 	u32 inc_val;
 	u8 tc, i;
 
 	/* Go over all PF VPORTs */
 	for (i = 0; i < num_vports; i++) {
-
 		if (!vport_params[i].vport_wfq)
 			continue;
 
@@ -536,17 +687,14 @@ static int qed_vp_wfq_rt_init(struct qed_hwfn *p_hwfn,
 			return -1;
 		}
 
-		/* each VPORT can have several VPORT PQ IDs for
-		 * different TCs
-		 */
+		/* Each VPORT can have several VPORT PQ IDs for various TCs */
 		for (tc = 0; tc < NUM_OF_TCS; tc++) {
-			u16 vport_pq_id = vport_params[i].first_tx_pq_id[tc];
-
+			vport_pq_id = vport_params[i].first_tx_pq_id[tc];
 			if (vport_pq_id != QM_INVALID_PQ_ID) {
 				STORE_RT_REG(p_hwfn,
 					     QM_REG_WFQVPCRD_RT_OFFSET +
 					     vport_pq_id,
-					     QM_WFQ_CRD_REG_SIGN_BIT);
+					     (u32)QM_WFQ_CRD_REG_SIGN_BIT);
 				STORE_RT_REG(p_hwfn,
 					     QM_REG_WFQVPWEIGHT_RT_OFFSET +
 					     vport_pq_id, inc_val);
@@ -557,12 +705,17 @@ static int qed_vp_wfq_rt_init(struct qed_hwfn *p_hwfn,
 	return 0;
 }
 
+/* Prepare VPORT RL runtime init values for the specified VPORTs.
+ * Return -1 on error.
+ */
 static int qed_vport_rl_rt_init(struct qed_hwfn *p_hwfn,
 				u8 start_vport,
 				u8 num_vports,
+				u32 link_speed,
 				struct init_qm_vport_params *vport_params)
 {
 	u8 i, vport_id;
+	u32 inc_val;
 
 	if (start_vport + num_vports >= MAX_QM_GLOBAL_RLS) {
 		DP_NOTICE(p_hwfn,
@@ -572,22 +725,22 @@ static int qed_vport_rl_rt_init(struct qed_hwfn *p_hwfn,
 
 	/* Go over all PF VPORTs */
 	for (i = 0, vport_id = start_vport; i < num_vports; i++, vport_id++) {
-		u32 inc_val = QM_RL_INC_VAL(vport_params[i].vport_rl);
-
-		if (inc_val > QM_RL_MAX_INC_VAL) {
+		inc_val = QM_RL_INC_VAL(vport_params[i].vport_rl ?
+			  vport_params[i].vport_rl :
+			  link_speed);
+		if (inc_val > QM_VP_RL_MAX_INC_VAL(link_speed)) {
 			DP_NOTICE(p_hwfn,
 				  "Invalid VPORT rate-limit configuration\n");
 			return -1;
 		}
 
-		STORE_RT_REG(p_hwfn,
-			     QM_REG_RLGLBLCRD_RT_OFFSET + vport_id,
-			     QM_RL_CRD_REG_SIGN_BIT);
+		STORE_RT_REG(p_hwfn, QM_REG_RLGLBLCRD_RT_OFFSET + vport_id,
+			     (u32)QM_RL_CRD_REG_SIGN_BIT);
 		STORE_RT_REG(p_hwfn,
 			     QM_REG_RLGLBLUPPERBOUND_RT_OFFSET + vport_id,
-			     QM_RL_UPPER_BOUND | QM_RL_CRD_REG_SIGN_BIT);
-		STORE_RT_REG(p_hwfn,
-			     QM_REG_RLGLBLINCVAL_RT_OFFSET + vport_id,
+			     QM_VP_RL_UPPER_BOUND(link_speed) |
+			     (u32)QM_RL_CRD_REG_SIGN_BIT);
+		STORE_RT_REG(p_hwfn, QM_REG_RLGLBLINCVAL_RT_OFFSET + vport_id,
 			     inc_val);
 	}
 
@@ -599,7 +752,7 @@ static bool qed_poll_on_qm_cmd_ready(struct qed_hwfn *p_hwfn,
 {
 	u32 reg_val, i;
 
-	for (i = 0, reg_val = 0; i < QM_STOP_CMD_MAX_POLL_COUNT && reg_val == 0;
+	for (i = 0, reg_val = 0; i < QM_STOP_CMD_MAX_POLL_COUNT && !reg_val;
 	     i++) {
 		udelay(QM_STOP_CMD_POLL_PERIOD_US);
 		reg_val = qed_rd(p_hwfn, p_ptt, QM_REG_SDMCMDREADY);
@@ -632,8 +785,8 @@ static bool qed_send_qm_cmd(struct qed_hwfn *p_hwfn,
 }
 
 /******************** INTERFACE IMPLEMENTATION *********************/
-u32 qed_qm_pf_mem_size(u8 pf_id,
-		       u32 num_pf_cids,
+
+u32 qed_qm_pf_mem_size(u32 num_pf_cids,
 		       u32 num_vf_cids,
 		       u32 num_tids, u16 num_pf_pqs, u16 num_vf_pqs)
 {
@@ -642,11 +795,10 @@ u32 qed_qm_pf_mem_size(u8 pf_id,
 	       QM_PQ_MEM_4KB(num_pf_cids + num_tids) * QM_OTHER_PQS_PER_PF;
 }
 
-int qed_qm_common_rt_init(
-	struct qed_hwfn *p_hwfn,
-	struct qed_qm_common_rt_init_params *p_params)
+int qed_qm_common_rt_init(struct qed_hwfn *p_hwfn,
+			  struct qed_qm_common_rt_init_params *p_params)
 {
-	/* init AFullOprtnstcCrdMask */
+	/* Init AFullOprtnstcCrdMask */
 	u32 mask = (QM_OPPOR_LINE_VOQ_DEF <<
 		    QM_RF_OPPORTUNISTIC_MASK_LINEVOQ_SHIFT) |
 		   (QM_BYTE_CRD_EN << QM_RF_OPPORTUNISTIC_MASK_BYTEVOQ_SHIFT) |
@@ -664,18 +816,31 @@ int qed_qm_common_rt_init(
 		    QM_RF_OPPORTUNISTIC_MASK_QUEUEEMPTY_SHIFT);
 
 	STORE_RT_REG(p_hwfn, QM_REG_AFULLOPRTNSTCCRDMASK_RT_OFFSET, mask);
+
+	/* Enable/disable PF RL */
 	qed_enable_pf_rl(p_hwfn, p_params->pf_rl_en);
+
+	/* Enable/disable PF WFQ */
 	qed_enable_pf_wfq(p_hwfn, p_params->pf_wfq_en);
+
+	/* Enable/disable VPORT RL */
 	qed_enable_vport_rl(p_hwfn, p_params->vport_rl_en);
+
+	/* Enable/disable VPORT WFQ */
 	qed_enable_vport_wfq(p_hwfn, p_params->vport_wfq_en);
+
+	/* Init PBF CMDQ line credit */
 	qed_cmdq_lines_rt_init(p_hwfn,
 			       p_params->max_ports_per_engine,
 			       p_params->max_phys_tcs_per_port,
 			       p_params->port_params);
+
+	/* Init BTB blocks in PBF */
 	qed_btb_blocks_rt_init(p_hwfn,
 			       p_params->max_ports_per_engine,
 			       p_params->max_phys_tcs_per_port,
 			       p_params->port_params);
+
 	return 0;
 }
 
@@ -695,24 +860,31 @@ int qed_qm_pf_rt_init(struct qed_hwfn *p_hwfn,
 			vport_params[i].first_tx_pq_id[tc] = QM_INVALID_PQ_ID;
 
 	/* Map Other PQs (if any) */
-	qed_other_pq_map_rt_init(p_hwfn, p_params->port_id, p_params->pf_id,
-				 p_params->num_pf_cids, p_params->num_tids, 0);
+	qed_other_pq_map_rt_init(p_hwfn,
+				 p_params->pf_id,
+				 p_params->is_pf_loading, p_params->num_pf_cids,
+				 p_params->num_tids, 0);
 
 	/* Map Tx PQs */
 	qed_tx_pq_map_rt_init(p_hwfn, p_ptt, p_params, other_mem_size_4kb);
 
+	/* Init PF WFQ */
 	if (p_params->pf_wfq)
 		if (qed_pf_wfq_rt_init(p_hwfn, p_params))
 			return -1;
 
+	/* Init PF RL */
 	if (qed_pf_rl_rt_init(p_hwfn, p_params->pf_id, p_params->pf_rl))
 		return -1;
 
+	/* Set VPORT WFQ */
 	if (qed_vp_wfq_rt_init(p_hwfn, p_params->num_vports, vport_params))
 		return -1;
 
+	/* Set VPORT RL */
 	if (qed_vport_rl_rt_init(p_hwfn, p_params->start_vport,
-				 p_params->num_vports, vport_params))
+				 p_params->num_vports, p_params->link_speed,
+				 vport_params))
 		return -1;
 
 	return 0;
@@ -729,6 +901,7 @@ int qed_init_pf_wfq(struct qed_hwfn *p_hwfn,
 	}
 
 	qed_wr(p_hwfn, p_ptt, QM_REG_WFQPFWEIGHT + pf_id * 4, inc_val);
+
 	return 0;
 }
 
@@ -737,14 +910,13 @@ int qed_init_pf_rl(struct qed_hwfn *p_hwfn,
 {
 	u32 inc_val = QM_RL_INC_VAL(pf_rl);
 
-	if (inc_val > QM_RL_MAX_INC_VAL) {
+	if (inc_val > QM_PF_RL_MAX_INC_VAL) {
 		DP_NOTICE(p_hwfn, "Invalid PF rate limit configuration\n");
 		return -1;
 	}
 
-	qed_wr(p_hwfn, p_ptt,
-	       QM_REG_RLPFCRD + pf_id * 4,
-	       QM_RL_CRD_REG_SIGN_BIT);
+	qed_wr(p_hwfn,
+	       p_ptt, QM_REG_RLPFCRD + pf_id * 4, (u32)QM_RL_CRD_REG_SIGN_BIT);
 	qed_wr(p_hwfn, p_ptt, QM_REG_RLPFINCVAL + pf_id * 4, inc_val);
 
 	return 0;
@@ -767,33 +939,35 @@ int qed_init_vport_wfq(struct qed_hwfn *p_hwfn,
 	for (tc = 0; tc < NUM_OF_TCS; tc++) {
 		vport_pq_id = first_tx_pq_id[tc];
 		if (vport_pq_id != QM_INVALID_PQ_ID)
-			qed_wr(p_hwfn, p_ptt,
-			       QM_REG_WFQVPWEIGHT + vport_pq_id * 4,
-			       inc_val);
+			qed_wr(p_hwfn,
+			       p_ptt,
+			       QM_REG_WFQVPWEIGHT + vport_pq_id * 4, inc_val);
 	}
 
 	return 0;
 }
 
 int qed_init_vport_rl(struct qed_hwfn *p_hwfn,
-		      struct qed_ptt *p_ptt, u8 vport_id, u32 vport_rl)
+		      struct qed_ptt *p_ptt,
+		      u8 vport_id, u32 vport_rl, u32 link_speed)
 {
-	u32 inc_val = QM_RL_INC_VAL(vport_rl);
+	u32 inc_val, max_qm_global_rls = MAX_QM_GLOBAL_RLS;
 
-	if (vport_id >= MAX_QM_GLOBAL_RLS) {
+	if (vport_id >= max_qm_global_rls) {
 		DP_NOTICE(p_hwfn,
 			  "Invalid VPORT ID for rate limiter configuration\n");
 		return -1;
 	}
 
-	if (inc_val > QM_RL_MAX_INC_VAL) {
+	inc_val = QM_RL_INC_VAL(vport_rl ? vport_rl : link_speed);
+	if (inc_val > QM_VP_RL_MAX_INC_VAL(link_speed)) {
 		DP_NOTICE(p_hwfn, "Invalid VPORT rate-limit configuration\n");
 		return -1;
 	}
 
-	qed_wr(p_hwfn, p_ptt,
-	       QM_REG_RLGLBLCRD + vport_id * 4,
-	       QM_RL_CRD_REG_SIGN_BIT);
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       QM_REG_RLGLBLCRD + vport_id * 4, (u32)QM_RL_CRD_REG_SIGN_BIT);
 	qed_wr(p_hwfn, p_ptt, QM_REG_RLGLBLINCVAL + vport_id * 4, inc_val);
 
 	return 0;
@@ -805,23 +979,27 @@ bool qed_send_qm_stop_cmd(struct qed_hwfn *p_hwfn,
 			  bool is_tx_pq, u16 start_pq, u16 num_pqs)
 {
 	u32 cmd_arr[QM_CMD_STRUCT_SIZE(QM_STOP_CMD)] = { 0 };
-	u32 pq_mask = 0, last_pq = start_pq + num_pqs - 1, pq_id;
+	u32 pq_mask = 0, last_pq, pq_id;
+
+	last_pq = start_pq + num_pqs - 1;
 
 	/* Set command's PQ type */
 	QM_CMD_SET_FIELD(cmd_arr, QM_STOP_CMD, PQ_TYPE, is_tx_pq ? 0 : 1);
 
+	/* Go over requested PQs */
 	for (pq_id = start_pq; pq_id <= last_pq; pq_id++) {
 		/* Set PQ bit in mask (stop command only) */
 		if (!is_release_cmd)
-			pq_mask |= (1 << (pq_id % QM_STOP_PQ_MASK_WIDTH));
+			pq_mask |= BIT((pq_id % QM_STOP_PQ_MASK_WIDTH));
 
 		/* If last PQ or end of PQ mask, write command */
 		if ((pq_id == last_pq) ||
 		    (pq_id % QM_STOP_PQ_MASK_WIDTH ==
 		     (QM_STOP_PQ_MASK_WIDTH - 1))) {
-			QM_CMD_SET_FIELD(cmd_arr, QM_STOP_CMD,
-					 PAUSE_MASK, pq_mask);
-			QM_CMD_SET_FIELD(cmd_arr, QM_STOP_CMD,
+			QM_CMD_SET_FIELD(cmd_arr,
+					 QM_STOP_CMD, PAUSE_MASK, pq_mask);
+			QM_CMD_SET_FIELD(cmd_arr,
+					 QM_STOP_CMD,
 					 GROUP_ID,
 					 pq_id / QM_STOP_PQ_MASK_WIDTH);
 			if (!qed_send_qm_cmd(p_hwfn, p_ptt, QM_STOP_CMD_ADDR,
@@ -834,87 +1012,103 @@ bool qed_send_qm_stop_cmd(struct qed_hwfn *p_hwfn,
 	return true;
 }
 
-static void
-qed_set_tunnel_type_enable_bit(unsigned long *var, int bit, bool enable)
-{
-	if (enable)
-		set_bit(bit, var);
-	else
-		clear_bit(bit, var);
-}
 
+#define SET_TUNNEL_TYPE_ENABLE_BIT(var, offset, enable) \
+	do { \
+		typeof(var) *__p_var = &(var); \
+		typeof(offset) __offset = offset; \
+		*__p_var = (*__p_var & ~BIT(__offset)) | \
+			   ((enable) ? BIT(__offset) : 0); \
+	} while (0)
 #define PRS_ETH_TUNN_FIC_FORMAT	-188897008
 
 void qed_set_vxlan_dest_port(struct qed_hwfn *p_hwfn,
 			     struct qed_ptt *p_ptt, u16 dest_port)
 {
+	/* Update PRS register */
 	qed_wr(p_hwfn, p_ptt, PRS_REG_VXLAN_PORT, dest_port);
+
+	/* Update NIG register */
 	qed_wr(p_hwfn, p_ptt, NIG_REG_VXLAN_CTRL, dest_port);
+
+	/* Update PBF register */
 	qed_wr(p_hwfn, p_ptt, PBF_REG_VXLAN_PORT, dest_port);
 }
 
 void qed_set_vxlan_enable(struct qed_hwfn *p_hwfn,
 			  struct qed_ptt *p_ptt, bool vxlan_enable)
 {
-	unsigned long reg_val = 0;
+	u32 reg_val;
 	u8 shift;
 
+	/* Update PRS register */
 	reg_val = qed_rd(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN);
 	shift = PRS_REG_ENCAPSULATION_TYPE_EN_VXLAN_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, vxlan_enable);
-
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, vxlan_enable);
 	qed_wr(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN, reg_val);
-
 	if (reg_val)
-		qed_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0,
-		       PRS_ETH_TUNN_FIC_FORMAT);
+		qed_wr(p_hwfn,
+		       p_ptt,
+		       PRS_REG_OUTPUT_FORMAT_4_0_BB_K2,
+		       (u32)PRS_ETH_TUNN_FIC_FORMAT);
 
+	/* Update NIG register */
 	reg_val = qed_rd(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE);
 	shift = NIG_REG_ENC_TYPE_ENABLE_VXLAN_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, vxlan_enable);
-
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, vxlan_enable);
 	qed_wr(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE, reg_val);
 
-	qed_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_VXLAN_EN,
-	       vxlan_enable ? 1 : 0);
+	/* Update DORQ register */
+	qed_wr(p_hwfn,
+	       p_ptt, DORQ_REG_L2_EDPM_TUNNEL_VXLAN_EN, vxlan_enable ? 1 : 0);
 }
 
-void qed_set_gre_enable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+void qed_set_gre_enable(struct qed_hwfn *p_hwfn,
+			struct qed_ptt *p_ptt,
 			bool eth_gre_enable, bool ip_gre_enable)
 {
-	unsigned long reg_val = 0;
+	u32 reg_val;
 	u8 shift;
 
+	/* Update PRS register */
 	reg_val = qed_rd(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN);
 	shift = PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GRE_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, eth_gre_enable);
-
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, eth_gre_enable);
 	shift = PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GRE_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, ip_gre_enable);
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, ip_gre_enable);
 	qed_wr(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN, reg_val);
 	if (reg_val)
-		qed_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0,
-		       PRS_ETH_TUNN_FIC_FORMAT);
+		qed_wr(p_hwfn,
+		       p_ptt,
+		       PRS_REG_OUTPUT_FORMAT_4_0_BB_K2,
+		       (u32)PRS_ETH_TUNN_FIC_FORMAT);
 
+	/* Update NIG register */
 	reg_val = qed_rd(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE);
 	shift = NIG_REG_ENC_TYPE_ENABLE_ETH_OVER_GRE_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, eth_gre_enable);
-
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, eth_gre_enable);
 	shift = NIG_REG_ENC_TYPE_ENABLE_IP_OVER_GRE_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, ip_gre_enable);
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, ip_gre_enable);
 	qed_wr(p_hwfn, p_ptt, NIG_REG_ENC_TYPE_ENABLE, reg_val);
 
-	qed_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_GRE_ETH_EN,
-	       eth_gre_enable ? 1 : 0);
-	qed_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_GRE_IP_EN,
-	       ip_gre_enable ? 1 : 0);
+	/* Update DORQ registers */
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       DORQ_REG_L2_EDPM_TUNNEL_GRE_ETH_EN, eth_gre_enable ? 1 : 0);
+	qed_wr(p_hwfn,
+	       p_ptt, DORQ_REG_L2_EDPM_TUNNEL_GRE_IP_EN, ip_gre_enable ? 1 : 0);
 }
 
 void qed_set_geneve_dest_port(struct qed_hwfn *p_hwfn,
 			      struct qed_ptt *p_ptt, u16 dest_port)
 {
+	/* Update PRS register */
 	qed_wr(p_hwfn, p_ptt, PRS_REG_NGE_PORT, dest_port);
+
+	/* Update NIG register */
 	qed_wr(p_hwfn, p_ptt, NIG_REG_NGE_PORT, dest_port);
+
+	/* Update PBF register */
 	qed_wr(p_hwfn, p_ptt, PBF_REG_NGE_PORT, dest_port);
 }
 
@@ -922,32 +1116,39 @@ void qed_set_geneve_enable(struct qed_hwfn *p_hwfn,
 			   struct qed_ptt *p_ptt,
 			   bool eth_geneve_enable, bool ip_geneve_enable)
 {
-	unsigned long reg_val = 0;
+	u32 reg_val;
 	u8 shift;
 
+	/* Update PRS register */
 	reg_val = qed_rd(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN);
 	shift = PRS_REG_ENCAPSULATION_TYPE_EN_ETH_OVER_GENEVE_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, eth_geneve_enable);
-
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, eth_geneve_enable);
 	shift = PRS_REG_ENCAPSULATION_TYPE_EN_IP_OVER_GENEVE_ENABLE_SHIFT;
-	qed_set_tunnel_type_enable_bit(&reg_val, shift, ip_geneve_enable);
-
+	SET_TUNNEL_TYPE_ENABLE_BIT(reg_val, shift, ip_geneve_enable);
 	qed_wr(p_hwfn, p_ptt, PRS_REG_ENCAPSULATION_TYPE_EN, reg_val);
 	if (reg_val)
-		qed_wr(p_hwfn, p_ptt, PRS_REG_OUTPUT_FORMAT_4_0,
-		       PRS_ETH_TUNN_FIC_FORMAT);
+		qed_wr(p_hwfn,
+		       p_ptt,
+		       PRS_REG_OUTPUT_FORMAT_4_0_BB_K2,
+		       (u32)PRS_ETH_TUNN_FIC_FORMAT);
 
+	/* Update NIG register */
 	qed_wr(p_hwfn, p_ptt, NIG_REG_NGE_ETH_ENABLE,
 	       eth_geneve_enable ? 1 : 0);
 	qed_wr(p_hwfn, p_ptt, NIG_REG_NGE_IP_ENABLE, ip_geneve_enable ? 1 : 0);
 
-	/* EDPM with geneve tunnel not supported in BB_B0 */
+	/* EDPM with geneve tunnel not supported in BB */
 	if (QED_IS_BB_B0(p_hwfn->cdev))
 		return;
 
-	qed_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_NGE_ETH_EN,
+	/* Update DORQ registers */
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       DORQ_REG_L2_EDPM_TUNNEL_NGE_ETH_EN_K2_E5,
 	       eth_geneve_enable ? 1 : 0);
-	qed_wr(p_hwfn, p_ptt, DORQ_REG_L2_EDPM_TUNNEL_NGE_IP_EN,
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       DORQ_REG_L2_EDPM_TUNNEL_NGE_IP_EN_K2_E5,
 	       ip_geneve_enable ? 1 : 0);
 }
 
@@ -959,117 +1160,297 @@ void qed_set_geneve_enable(struct qed_hwfn *p_hwfn,
 #define RAM_LINE_SIZE sizeof(u64)
 #define REG_SIZE sizeof(u32)
 
-void qed_set_rfs_mode_disable(struct qed_hwfn *p_hwfn,
-			      struct qed_ptt *p_ptt, u16 pf_id)
+void qed_gft_disable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 pf_id)
 {
-	u32 hw_addr = PRS_REG_GFT_PROFILE_MASK_RAM +
-		      pf_id * RAM_LINE_SIZE;
-
-	/*stop using gft logic */
+	/* Disable gft search for PF */
 	qed_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_GFT, 0);
-	qed_wr(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT, 0x0);
+
+	/* Clean ram & cam for next gft session */
+
+	/* Zero camline */
 	qed_wr(p_hwfn, p_ptt, PRS_REG_GFT_CAM + CAM_LINE_SIZE * pf_id, 0);
-	qed_wr(p_hwfn, p_ptt, hw_addr, 0);
-	qed_wr(p_hwfn, p_ptt, hw_addr + 4, 0);
+
+	/* Zero ramline */
+	qed_wr(p_hwfn,
+	       p_ptt, PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id, 0);
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id + REG_SIZE,
+	       0);
 }
 
-void qed_set_rfs_mode_enable(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-			     u16 pf_id, bool tcp, bool udp,
-			     bool ipv4, bool ipv6)
+void qed_set_gft_event_id_cm_hdr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
 {
-	union gft_cam_line_union camline;
-	struct gft_ram_line ramline;
 	u32 rfs_cm_hdr_event_id;
 
+	/* Set RFS event ID to be awakened i Tstorm By Prs */
 	rfs_cm_hdr_event_id = qed_rd(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT);
+	rfs_cm_hdr_event_id |= T_ETH_PACKET_ACTION_GFT_EVENTID <<
+			       PRS_REG_CM_HDR_GFT_EVENT_ID_SHIFT;
+	rfs_cm_hdr_event_id |= PARSER_ETH_CONN_GFT_ACTION_CM_HDR <<
+			       PRS_REG_CM_HDR_GFT_CM_HDR_SHIFT;
+	qed_wr(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT, rfs_cm_hdr_event_id);
+}
+
+void qed_gft_config(struct qed_hwfn *p_hwfn,
+		    struct qed_ptt *p_ptt,
+		    u16 pf_id,
+		    bool tcp,
+		    bool udp,
+		    bool ipv4, bool ipv6, enum gft_profile_type profile_type)
+{
+	u32 reg_val, cam_line, ram_line_lo, ram_line_hi;
 
 	if (!ipv6 && !ipv4)
 		DP_NOTICE(p_hwfn,
-			  "set_rfs_mode_enable: must accept at least on of - ipv4 or ipv6");
+			  "gft_config: must accept at least on of - ipv4 or ipv6'\n");
 	if (!tcp && !udp)
 		DP_NOTICE(p_hwfn,
-			  "set_rfs_mode_enable: must accept at least on of - udp or tcp");
+			  "gft_config: must accept at least on of - udp or tcp\n");
+	if (profile_type >= MAX_GFT_PROFILE_TYPE)
+		DP_NOTICE(p_hwfn, "gft_config: unsupported gft_profile_type\n");
 
-	rfs_cm_hdr_event_id |= T_ETH_PACKET_MATCH_RFS_EVENTID <<
-					PRS_REG_CM_HDR_GFT_EVENT_ID_SHIFT;
-	rfs_cm_hdr_event_id |= PARSER_ETH_CONN_CM_HDR <<
-					PRS_REG_CM_HDR_GFT_CM_HDR_SHIFT;
-	qed_wr(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT, rfs_cm_hdr_event_id);
+	/* Set RFS event ID to be awakened i Tstorm By Prs */
+	reg_val = T_ETH_PACKET_MATCH_RFS_EVENTID <<
+		  PRS_REG_CM_HDR_GFT_EVENT_ID_SHIFT;
+	reg_val |= PARSER_ETH_CONN_CM_HDR << PRS_REG_CM_HDR_GFT_CM_HDR_SHIFT;
+	qed_wr(p_hwfn, p_ptt, PRS_REG_CM_HDR_GFT, reg_val);
 
-	/* Configure Registers for RFS mode */
-	qed_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_GFT, 1);
+	/* Do not load context only cid in PRS on match. */
 	qed_wr(p_hwfn, p_ptt, PRS_REG_LOAD_L2_FILTER, 0);
-	camline.cam_line_mapped.camline = 0;
 
-	/* Cam line is now valid!! */
-	SET_FIELD(camline.cam_line_mapped.camline,
-		  GFT_CAM_LINE_MAPPED_VALID, 1);
+	/* Do not use tenant ID exist bit for gft search */
+	qed_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_TENANT_ID, 0);
 
-	/* filters are per PF!! */
-	SET_FIELD(camline.cam_line_mapped.camline,
+	/* Set Cam */
+	cam_line = 0;
+	SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_VALID, 1);
+
+	/* Filters are per PF!! */
+	SET_FIELD(cam_line,
 		  GFT_CAM_LINE_MAPPED_PF_ID_MASK,
 		  GFT_CAM_LINE_MAPPED_PF_ID_MASK_MASK);
-	SET_FIELD(camline.cam_line_mapped.camline,
-		  GFT_CAM_LINE_MAPPED_PF_ID, pf_id);
+	SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_PF_ID, pf_id);
+
 	if (!(tcp && udp)) {
-		SET_FIELD(camline.cam_line_mapped.camline,
+		SET_FIELD(cam_line,
 			  GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_MASK,
 			  GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE_MASK_MASK);
 		if (tcp)
-			SET_FIELD(camline.cam_line_mapped.camline,
+			SET_FIELD(cam_line,
 				  GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE,
 				  GFT_PROFILE_TCP_PROTOCOL);
 		else
-			SET_FIELD(camline.cam_line_mapped.camline,
+			SET_FIELD(cam_line,
 				  GFT_CAM_LINE_MAPPED_UPPER_PROTOCOL_TYPE,
 				  GFT_PROFILE_UDP_PROTOCOL);
 	}
 
 	if (!(ipv4 && ipv6)) {
-		SET_FIELD(camline.cam_line_mapped.camline,
-			  GFT_CAM_LINE_MAPPED_IP_VERSION_MASK, 1);
+		SET_FIELD(cam_line, GFT_CAM_LINE_MAPPED_IP_VERSION_MASK, 1);
 		if (ipv4)
-			SET_FIELD(camline.cam_line_mapped.camline,
+			SET_FIELD(cam_line,
 				  GFT_CAM_LINE_MAPPED_IP_VERSION,
 				  GFT_PROFILE_IPV4);
 		else
-			SET_FIELD(camline.cam_line_mapped.camline,
+			SET_FIELD(cam_line,
 				  GFT_CAM_LINE_MAPPED_IP_VERSION,
 				  GFT_PROFILE_IPV6);
 	}
 
 	/* Write characteristics to cam */
 	qed_wr(p_hwfn, p_ptt, PRS_REG_GFT_CAM + CAM_LINE_SIZE * pf_id,
-	       camline.cam_line_mapped.camline);
-	camline.cam_line_mapped.camline = qed_rd(p_hwfn, p_ptt,
-						 PRS_REG_GFT_CAM +
-						 CAM_LINE_SIZE * pf_id);
+	       cam_line);
+	cam_line =
+	    qed_rd(p_hwfn, p_ptt, PRS_REG_GFT_CAM + CAM_LINE_SIZE * pf_id);
 
 	/* Write line to RAM - compare to filter 4 tuple */
-	ramline.lo = 0;
-	ramline.hi = 0;
-	SET_FIELD(ramline.hi, GFT_RAM_LINE_DST_IP, 1);
-	SET_FIELD(ramline.hi, GFT_RAM_LINE_SRC_IP, 1);
-	SET_FIELD(ramline.hi, GFT_RAM_LINE_OVER_IP_PROTOCOL, 1);
-	SET_FIELD(ramline.lo, GFT_RAM_LINE_ETHERTYPE, 1);
-	SET_FIELD(ramline.lo, GFT_RAM_LINE_SRC_PORT, 1);
-	SET_FIELD(ramline.lo, GFT_RAM_LINE_DST_PORT, 1);
-
-	/* Each iteration write to reg */
-	qed_wr(p_hwfn, p_ptt,
+	ram_line_lo = 0;
+	ram_line_hi = 0;
+
+	if (profile_type == GFT_PROFILE_TYPE_4_TUPLE) {
+		SET_FIELD(ram_line_hi, GFT_RAM_LINE_DST_IP, 1);
+		SET_FIELD(ram_line_hi, GFT_RAM_LINE_SRC_IP, 1);
+		SET_FIELD(ram_line_hi, GFT_RAM_LINE_OVER_IP_PROTOCOL, 1);
+		SET_FIELD(ram_line_lo, GFT_RAM_LINE_ETHERTYPE, 1);
+		SET_FIELD(ram_line_lo, GFT_RAM_LINE_SRC_PORT, 1);
+		SET_FIELD(ram_line_lo, GFT_RAM_LINE_DST_PORT, 1);
+	} else if (profile_type == GFT_PROFILE_TYPE_L4_DST_PORT) {
+		SET_FIELD(ram_line_hi, GFT_RAM_LINE_OVER_IP_PROTOCOL, 1);
+		SET_FIELD(ram_line_lo, GFT_RAM_LINE_ETHERTYPE, 1);
+		SET_FIELD(ram_line_lo, GFT_RAM_LINE_DST_PORT, 1);
+	} else if (profile_type == GFT_PROFILE_TYPE_IP_DST_PORT) {
+		SET_FIELD(ram_line_hi, GFT_RAM_LINE_DST_IP, 1);
+		SET_FIELD(ram_line_lo, GFT_RAM_LINE_ETHERTYPE, 1);
+	}
+
+	qed_wr(p_hwfn,
+	       p_ptt,
 	       PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id,
-	       ramline.lo);
-	qed_wr(p_hwfn, p_ptt,
-	       PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id + 4,
-	       ramline.hi);
+	       ram_line_lo);
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE * pf_id + REG_SIZE,
+	       ram_line_hi);
 
 	/* Set default profile so that no filter match will happen */
-	qed_wr(p_hwfn, p_ptt,
-	       PRS_REG_GFT_PROFILE_MASK_RAM +
-	       RAM_LINE_SIZE * PRS_GFT_CAM_LINES_NO_MATCH,
-	       ramline.lo);
-	qed_wr(p_hwfn, p_ptt,
-	       PRS_REG_GFT_PROFILE_MASK_RAM +
-	       RAM_LINE_SIZE * PRS_GFT_CAM_LINES_NO_MATCH + 4,
-	       ramline.hi);
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE *
+	       PRS_GFT_CAM_LINES_NO_MATCH, 0xffffffff);
+	qed_wr(p_hwfn,
+	       p_ptt,
+	       PRS_REG_GFT_PROFILE_MASK_RAM + RAM_LINE_SIZE *
+	       PRS_GFT_CAM_LINES_NO_MATCH + REG_SIZE, 0x3ff);
+
+	/* Enable gft search */
+	qed_wr(p_hwfn, p_ptt, PRS_REG_SEARCH_GFT, 1);
+}
+
+DECLARE_CRC8_TABLE(cdu_crc8_table);
+
+/* Calculate and return CDU validation byte per connection type/region/cid */
+static u8 qed_calc_cdu_validation_byte(u8 conn_type, u8 region, u32 cid)
+{
+	const u8 validation_cfg = CDU_VALIDATION_DEFAULT_CFG;
+	u8 crc, validation_byte = 0;
+	static u8 crc8_table_valid; /* automatically initialized to 0 */
+	u32 validation_string = 0;
+	u32 data_to_crc;
+
+	if (!crc8_table_valid) {
+		crc8_populate_msb(cdu_crc8_table, 0x07);
+		crc8_table_valid = 1;
+	}
+
+	/* The CRC is calculated on the String-to-compress:
+	 * [31:8]  = {CID[31:20],CID[11:0]}
+	 * [7:4]   = Region
+	 * [3:0]   = Type
+	 */
+	if ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_CID) & 1)
+		validation_string |= (cid & 0xFFF00000) | ((cid & 0xFFF) << 8);
+
+	if ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_REGION) & 1)
+		validation_string |= ((region & 0xF) << 4);
+
+	if ((validation_cfg >> CDU_CONTEXT_VALIDATION_CFG_USE_TYPE) & 1)
+		validation_string |= (conn_type & 0xF);
+
+	/* Convert to big-endian and calculate CRC8 */
+	data_to_crc = be32_to_cpu(validation_string);
+
+	crc = crc8(cdu_crc8_table,
+		   (u8 *)&data_to_crc, sizeof(data_to_crc), CRC8_INIT_VALUE);
+
+	/* The validation byte [7:0] is composed:
+	 * for type A validation
+	 * [7]          = active configuration bit
+	 * [6:0]        = crc[6:0]
+	 *
+	 * for type B validation
+	 * [7]          = active configuration bit
+	 * [6:3]        = connection_type[3:0]
+	 * [2:0]        = crc[2:0]
+	 */
+	validation_byte |=
+	    ((validation_cfg >>
+	      CDU_CONTEXT_VALIDATION_CFG_USE_ACTIVE) & 1) << 7;
+
+	if ((validation_cfg >>
+	     CDU_CONTEXT_VALIDATION_CFG_VALIDATION_TYPE_SHIFT) & 1)
+		validation_byte |= ((conn_type & 0xF) << 3) | (crc & 0x7);
+	else
+		validation_byte |= crc & 0x7F;
+
+	return validation_byte;
+}
+
+/* Calcualte and set validation bytes for session context */
+void qed_calc_session_ctx_validation(void *p_ctx_mem,
+				     u16 ctx_size, u8 ctx_type, u32 cid)
+{
+	u8 *x_val_ptr, *t_val_ptr, *u_val_ptr, *p_ctx;
+
+	p_ctx = (u8 * const)p_ctx_mem;
+	x_val_ptr = &p_ctx[con_region_offsets[0][ctx_type]];
+	t_val_ptr = &p_ctx[con_region_offsets[1][ctx_type]];
+	u_val_ptr = &p_ctx[con_region_offsets[2][ctx_type]];
+
+	memset(p_ctx, 0, ctx_size);
+
+	*x_val_ptr = qed_calc_cdu_validation_byte(ctx_type, 3, cid);
+	*t_val_ptr = qed_calc_cdu_validation_byte(ctx_type, 4, cid);
+	*u_val_ptr = qed_calc_cdu_validation_byte(ctx_type, 5, cid);
+}
+
+/* Calcualte and set validation bytes for task context */
+void qed_calc_task_ctx_validation(void *p_ctx_mem,
+				  u16 ctx_size, u8 ctx_type, u32 tid)
+{
+	u8 *p_ctx, *region1_val_ptr;
+
+	p_ctx = (u8 * const)p_ctx_mem;
+	region1_val_ptr = &p_ctx[task_region_offsets[0][ctx_type]];
+
+	memset(p_ctx, 0, ctx_size);
+
+	*region1_val_ptr = qed_calc_cdu_validation_byte(ctx_type, 1, tid);
+}
+
+/* Memset session context to 0 while preserving validation bytes */
+void qed_memset_session_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type)
+{
+	u8 *x_val_ptr, *t_val_ptr, *u_val_ptr, *p_ctx;
+	u8 x_val, t_val, u_val;
+
+	p_ctx = (u8 * const)p_ctx_mem;
+	x_val_ptr = &p_ctx[con_region_offsets[0][ctx_type]];
+	t_val_ptr = &p_ctx[con_region_offsets[1][ctx_type]];
+	u_val_ptr = &p_ctx[con_region_offsets[2][ctx_type]];
+
+	x_val = *x_val_ptr;
+	t_val = *t_val_ptr;
+	u_val = *u_val_ptr;
+
+	memset(p_ctx, 0, ctx_size);
+
+	*x_val_ptr = x_val;
+	*t_val_ptr = t_val;
+	*u_val_ptr = u_val;
+}
+
+/* Memset task context to 0 while preserving validation bytes */
+void qed_memset_task_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type)
+{
+	u8 *p_ctx, *region1_val_ptr;
+	u8 region1_val;
+
+	p_ctx = (u8 * const)p_ctx_mem;
+	region1_val_ptr = &p_ctx[task_region_offsets[0][ctx_type]];
+
+	region1_val = *region1_val_ptr;
+
+	memset(p_ctx, 0, ctx_size);
+
+	*region1_val_ptr = region1_val;
+}
+
+/* Enable and configure context validation */
+void qed_enable_context_validation(struct qed_hwfn *p_hwfn,
+				   struct qed_ptt *p_ptt)
+{
+	u32 ctx_validation;
+
+	/* Enable validation for connection region 3: CCFC_CTX_VALID0[31:24] */
+	ctx_validation = CDU_VALIDATION_DEFAULT_CFG << 24;
+	qed_wr(p_hwfn, p_ptt, CDU_REG_CCFC_CTX_VALID0, ctx_validation);
+
+	/* Enable validation for connection region 5: CCFC_CTX_VALID1[15:8] */
+	ctx_validation = CDU_VALIDATION_DEFAULT_CFG << 8;
+	qed_wr(p_hwfn, p_ptt, CDU_REG_CCFC_CTX_VALID1, ctx_validation);
+
+	/* Enable validation for connection region 1: TCFC_CTX_VALID0[15:8] */
+	ctx_validation = CDU_VALIDATION_DEFAULT_CFG << 8;
+	qed_wr(p_hwfn, p_ptt, CDU_REG_TCFC_CTX_VALID0, ctx_validation);
 }
diff --git a/drivers/net/ethernet/qlogic/qed/qed_init_ops.c b/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
index e3f368882f46..3bb76da6baa2 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_init_ops.c
@@ -414,11 +414,23 @@ static void qed_init_cmd_rd(struct qed_hwfn *p_hwfn,
 }
 
 /* init_ops callbacks entry point */
-static void qed_init_cmd_cb(struct qed_hwfn *p_hwfn,
-			    struct qed_ptt *p_ptt,
-			    struct init_callback_op *p_cmd)
+static int qed_init_cmd_cb(struct qed_hwfn *p_hwfn,
+			   struct qed_ptt *p_ptt,
+			   struct init_callback_op *p_cmd)
 {
-	DP_NOTICE(p_hwfn, "Currently init values have no need of callbacks\n");
+	int rc;
+
+	switch (p_cmd->callback_id) {
+	case DMAE_READY_CB:
+		rc = qed_dmae_sanity(p_hwfn, p_ptt, "engine_phase");
+		break;
+	default:
+		DP_NOTICE(p_hwfn, "Unexpected init op callback ID %d\n",
+			  p_cmd->callback_id);
+		return -EINVAL;
+	}
+
+	return rc;
 }
 
 static u8 qed_init_cmd_mode_match(struct qed_hwfn *p_hwfn,
@@ -519,7 +531,7 @@ int qed_init_run(struct qed_hwfn *p_hwfn,
 			break;
 
 		case INIT_OP_CALLBACK:
-			qed_init_cmd_cb(p_hwfn, p_ptt, &cmd->callback);
+			rc = qed_init_cmd_cb(p_hwfn, p_ptt, &cmd->callback);
 			break;
 		}
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c
index 719cdbfe1695..d3eabcf9c86c 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_int.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_int.c
@@ -59,10 +59,10 @@ struct qed_pi_info {
 };
 
 struct qed_sb_sp_info {
-	struct qed_sb_info	sb_info;
+	struct qed_sb_info sb_info;
 
 	/* per protocol index data */
-	struct qed_pi_info	pi_info_arr[PIS_PER_SB];
+	struct qed_pi_info pi_info_arr[PIS_PER_SB_E4];
 };
 
 enum qed_attention_type {
@@ -82,7 +82,7 @@ struct aeu_invert_reg_bit {
 #define ATTENTION_LENGTH_SHIFT          (4)
 #define ATTENTION_LENGTH(flags)         (((flags) & ATTENTION_LENGTH_MASK) >> \
 					 ATTENTION_LENGTH_SHIFT)
-#define ATTENTION_SINGLE                (1 << ATTENTION_LENGTH_SHIFT)
+#define ATTENTION_SINGLE                BIT(ATTENTION_LENGTH_SHIFT)
 #define ATTENTION_PAR                   (ATTENTION_SINGLE | ATTENTION_PARITY)
 #define ATTENTION_PAR_INT               ((2 << ATTENTION_LENGTH_SHIFT) | \
 					 ATTENTION_PARITY)
@@ -1313,7 +1313,7 @@ static void qed_int_cau_conf_pi(struct qed_hwfn *p_hwfn,
 	if (IS_VF(p_hwfn->cdev))
 		return;
 
-	sb_offset = igu_sb_id * PIS_PER_SB;
+	sb_offset = igu_sb_id * PIS_PER_SB_E4;
 	memset(&pi_entry, 0, sizeof(struct cau_pi_entry));
 
 	SET_FIELD(pi_entry.prod, CAU_PI_ENTRY_PI_TIMESET, timeset);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.h b/drivers/net/ethernet/qlogic/qed/qed_int.h
index 5199634ed630..54b4ee0acfd7 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_int.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_int.h
@@ -197,7 +197,7 @@ void qed_int_disable_post_isr_release(struct qed_dev *cdev);
 #define QED_SB_EVENT_MASK       0x0003
 
 #define SB_ALIGNED_SIZE(p_hwfn)	\
-	ALIGNED_TYPE_SIZE(struct status_block, p_hwfn)
+	ALIGNED_TYPE_SIZE(struct status_block_e4, p_hwfn)
 
 #define QED_SB_INVALID_IDX      0xffff
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_iscsi.c b/drivers/net/ethernet/qlogic/qed/qed_iscsi.c
index 813c77cc857f..c0d4a54a5edb 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_iscsi.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_iscsi.c
@@ -62,22 +62,6 @@
 #include "qed_sriov.h"
 #include "qed_reg_addr.h"
 
-static int
-qed_iscsi_async_event(struct qed_hwfn *p_hwfn,
-		      u8 fw_event_code,
-		      u16 echo, union event_ring_data *data, u8 fw_return_code)
-{
-	if (p_hwfn->p_iscsi_info->event_cb) {
-		struct qed_iscsi_info *p_iscsi = p_hwfn->p_iscsi_info;
-
-		return p_iscsi->event_cb(p_iscsi->event_context,
-					 fw_event_code, data);
-	} else {
-		DP_NOTICE(p_hwfn, "iSCSI async completion is not set\n");
-		return -EINVAL;
-	}
-}
-
 struct qed_iscsi_conn {
 	struct list_head list_entry;
 	bool free_on_delete;
@@ -105,7 +89,7 @@ struct qed_iscsi_conn {
 	u8 local_mac[6];
 	u8 remote_mac[6];
 	u16 vlan_id;
-	u8 tcp_flags;
+	u16 tcp_flags;
 	u8 ip_version;
 	u32 remote_ip[4];
 	u32 local_ip[4];
@@ -122,7 +106,6 @@ struct qed_iscsi_conn {
 	u32 ss_thresh;
 	u16 srtt;
 	u16 rtt_var;
-	u32 ts_time;
 	u32 ts_recent;
 	u32 ts_recent_age;
 	u32 total_rt;
@@ -144,7 +127,6 @@ struct qed_iscsi_conn {
 	u16 mss;
 	u8 snd_wnd_scale;
 	u8 rcv_wnd_scale;
-	u32 ts_ticks_per_second;
 	u16 da_timeout_value;
 	u8 ack_frequency;
 
@@ -161,6 +143,22 @@ struct qed_iscsi_conn {
 	u8 abortive_dsconnect;
 };
 
+static int
+qed_iscsi_async_event(struct qed_hwfn *p_hwfn,
+		      u8 fw_event_code,
+		      u16 echo, union event_ring_data *data, u8 fw_return_code)
+{
+	if (p_hwfn->p_iscsi_info->event_cb) {
+		struct qed_iscsi_info *p_iscsi = p_hwfn->p_iscsi_info;
+
+		return p_iscsi->event_cb(p_iscsi->event_context,
+					 fw_event_code, data);
+	} else {
+		DP_NOTICE(p_hwfn, "iSCSI async completion is not set\n");
+		return -EINVAL;
+	}
+}
+
 static int
 qed_sp_iscsi_func_start(struct qed_hwfn *p_hwfn,
 			enum spq_mode comp_mode,
@@ -214,9 +212,9 @@ qed_sp_iscsi_func_start(struct qed_hwfn *p_hwfn,
 	p_init->num_sq_pages_in_ring = p_params->num_sq_pages_in_ring;
 	p_init->num_r2tq_pages_in_ring = p_params->num_r2tq_pages_in_ring;
 	p_init->num_uhq_pages_in_ring = p_params->num_uhq_pages_in_ring;
-	p_init->ooo_enable = p_params->ooo_enable;
 	p_init->ll2_rx_queue_id = p_hwfn->hw_info.resc_start[QED_LL2_QUEUE] +
 				  p_params->ll2_ooo_queue_id;
+
 	p_init->func_params.log_page_size = p_params->log_page_size;
 	val = p_params->num_tasks;
 	p_init->func_params.num_tasks = cpu_to_le16(val);
@@ -276,7 +274,7 @@ qed_sp_iscsi_func_start(struct qed_hwfn *p_hwfn,
 	p_ramrod->tcp_init.two_msl_timer = cpu_to_le32(p_params->two_msl_timer);
 	val = p_params->tx_sws_timer;
 	p_ramrod->tcp_init.tx_sws_timer = cpu_to_le16(val);
-	p_ramrod->tcp_init.maxfinrt = p_params->max_fin_rt;
+	p_ramrod->tcp_init.max_fin_rt = p_params->max_fin_rt;
 
 	p_hwfn->p_iscsi_info->event_context = event_context;
 	p_hwfn->p_iscsi_info->event_cb = async_event_cb;
@@ -304,8 +302,8 @@ static int qed_sp_iscsi_conn_offload(struct qed_hwfn *p_hwfn,
 	int rc = 0;
 	u32 dval;
 	u16 wval;
-	u8 i;
 	u16 *p;
+	u8 i;
 
 	/* Get SPQ entry */
 	memset(&init_data, 0, sizeof(init_data));
@@ -371,7 +369,7 @@ static int qed_sp_iscsi_conn_offload(struct qed_hwfn *p_hwfn,
 
 		p_tcp->vlan_id = cpu_to_le16(p_conn->vlan_id);
 
-		p_tcp->flags = p_conn->tcp_flags;
+		p_tcp->flags = cpu_to_le16(p_conn->tcp_flags);
 		p_tcp->ip_version = p_conn->ip_version;
 		for (i = 0; i < 4; i++) {
 			dval = p_conn->remote_ip[i];
@@ -436,7 +434,7 @@ static int qed_sp_iscsi_conn_offload(struct qed_hwfn *p_hwfn,
 		p_tcp2->remote_mac_addr_lo = swab16(get_unaligned(p + 2));
 
 		p_tcp2->vlan_id = cpu_to_le16(p_conn->vlan_id);
-		p_tcp2->flags = p_conn->tcp_flags;
+		p_tcp2->flags = cpu_to_le16(p_conn->tcp_flags);
 
 		p_tcp2->ip_version = p_conn->ip_version;
 		for (i = 0; i < 4; i++) {
@@ -458,6 +456,11 @@ static int qed_sp_iscsi_conn_offload(struct qed_hwfn *p_hwfn,
 		p_tcp2->syn_ip_payload_length = cpu_to_le16(wval);
 		p_tcp2->syn_phy_addr_lo = DMA_LO_LE(p_conn->syn_phy_addr);
 		p_tcp2->syn_phy_addr_hi = DMA_HI_LE(p_conn->syn_phy_addr);
+		p_tcp2->cwnd = cpu_to_le32(p_conn->cwnd);
+		p_tcp2->ka_max_probe_cnt = p_conn->ka_probe_cnt;
+		p_tcp2->ka_timeout = cpu_to_le32(p_conn->ka_timeout);
+		p_tcp2->max_rt_time = cpu_to_le32(p_conn->max_rt_time);
+		p_tcp2->ka_interval = cpu_to_le32(p_conn->ka_interval);
 	}
 
 	return qed_spq_post(p_hwfn, p_ent, NULL);
@@ -692,8 +695,7 @@ static void __iomem *qed_iscsi_get_secondary_bdq_prod(struct qed_hwfn *p_hwfn,
 	}
 }
 
-static int qed_iscsi_setup_connection(struct qed_hwfn *p_hwfn,
-				      struct qed_iscsi_conn *p_conn)
+static int qed_iscsi_setup_connection(struct qed_iscsi_conn *p_conn)
 {
 	if (!p_conn->queue_cnts_virt_addr)
 		goto nomem;
@@ -844,7 +846,7 @@ static int qed_iscsi_acquire_connection(struct qed_hwfn *p_hwfn,
 		rc = qed_iscsi_allocate_connection(p_hwfn, &p_conn);
 
 	if (!rc)
-		rc = qed_iscsi_setup_connection(p_hwfn, p_conn);
+		rc = qed_iscsi_setup_connection(p_conn);
 
 	if (rc) {
 		spin_lock_bh(&p_hwfn->p_iscsi_info->lock);
@@ -1294,7 +1296,6 @@ static int qed_iscsi_offload_conn(struct qed_dev *cdev,
 	con->ss_thresh = conn_info->ss_thresh;
 	con->srtt = conn_info->srtt;
 	con->rtt_var = conn_info->rtt_var;
-	con->ts_time = conn_info->ts_time;
 	con->ts_recent = conn_info->ts_recent;
 	con->ts_recent_age = conn_info->ts_recent_age;
 	con->total_rt = conn_info->total_rt;
@@ -1316,7 +1317,6 @@ static int qed_iscsi_offload_conn(struct qed_dev *cdev,
 	con->mss = conn_info->mss;
 	con->snd_wnd_scale = conn_info->snd_wnd_scale;
 	con->rcv_wnd_scale = conn_info->rcv_wnd_scale;
-	con->ts_ticks_per_second = conn_info->ts_ticks_per_second;
 	con->da_timeout_value = conn_info->da_timeout_value;
 	con->ack_frequency = conn_info->ack_frequency;
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_iwarp.c b/drivers/net/ethernet/qlogic/qed/qed_iwarp.c
index 409041eab189..d5d02be72947 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_iwarp.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_iwarp.c
@@ -64,14 +64,21 @@ struct mpa_v2_hdr {
 
 #define QED_IWARP_INVALID_TCP_CID	0xffffffff
 #define QED_IWARP_RCV_WND_SIZE_DEF	(256 * 1024)
-#define QED_IWARP_RCV_WND_SIZE_MIN	(64 * 1024)
+#define QED_IWARP_RCV_WND_SIZE_MIN	(0xffff)
 #define TIMESTAMP_HEADER_SIZE		(12)
+#define QED_IWARP_MAX_FIN_RT_DEFAULT	(2)
 
 #define QED_IWARP_TS_EN			BIT(0)
 #define QED_IWARP_DA_EN			BIT(1)
 #define QED_IWARP_PARAM_CRC_NEEDED	(1)
 #define QED_IWARP_PARAM_P2P		(1)
 
+#define QED_IWARP_DEF_MAX_RT_TIME	(0)
+#define QED_IWARP_DEF_CWND_FACTOR	(4)
+#define QED_IWARP_DEF_KA_MAX_PROBE_CNT	(5)
+#define QED_IWARP_DEF_KA_TIMEOUT	(1200000)	/* 20 min */
+#define QED_IWARP_DEF_KA_INTERVAL	(1000)		/* 1 sec */
+
 static int qed_iwarp_async_event(struct qed_hwfn *p_hwfn,
 				 u8 fw_event_code, u16 echo,
 				 union event_ring_data *data,
@@ -120,11 +127,17 @@ static void qed_iwarp_cid_cleaned(struct qed_hwfn *p_hwfn, u32 cid)
 	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 }
 
-void qed_iwarp_init_fw_ramrod(struct qed_hwfn *p_hwfn,
-			      struct iwarp_init_func_params *p_ramrod)
+void
+qed_iwarp_init_fw_ramrod(struct qed_hwfn *p_hwfn,
+			 struct iwarp_init_func_ramrod_data *p_ramrod)
 {
-	p_ramrod->ll2_ooo_q_index = RESC_START(p_hwfn, QED_LL2_QUEUE) +
-				    p_hwfn->p_rdma_info->iwarp.ll2_ooo_handle;
+	p_ramrod->iwarp.ll2_ooo_q_index =
+		RESC_START(p_hwfn, QED_LL2_QUEUE) +
+		p_hwfn->p_rdma_info->iwarp.ll2_ooo_handle;
+
+	p_ramrod->tcp.max_fin_rt = QED_IWARP_MAX_FIN_RT_DEFAULT;
+
+	return;
 }
 
 static int qed_iwarp_alloc_cid(struct qed_hwfn *p_hwfn, u32 *cid)
@@ -699,6 +712,12 @@ qed_iwarp_tcp_offload(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 	tcp->ttl = 0x40;
 	tcp->tos_or_tc = 0;
 
+	tcp->max_rt_time = QED_IWARP_DEF_MAX_RT_TIME;
+	tcp->cwnd = QED_IWARP_DEF_CWND_FACTOR *  tcp->mss;
+	tcp->ka_max_probe_cnt = QED_IWARP_DEF_KA_MAX_PROBE_CNT;
+	tcp->ka_timeout = QED_IWARP_DEF_KA_TIMEOUT;
+	tcp->ka_interval = QED_IWARP_DEF_KA_INTERVAL;
+
 	tcp->rcv_wnd_scale = (u8)p_hwfn->p_rdma_info->iwarp.rcv_wnd_scale;
 	tcp->connect_mode = ep->connect_mode;
 
@@ -807,6 +826,7 @@ static int
 qed_iwarp_mpa_offload(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 {
 	struct iwarp_mpa_offload_ramrod_data *p_mpa_ramrod;
+	struct qed_iwarp_info *iwarp_info;
 	struct qed_sp_init_data init_data;
 	dma_addr_t async_output_phys;
 	struct qed_spq_entry *p_ent;
@@ -874,6 +894,8 @@ qed_iwarp_mpa_offload(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep)
 		p_mpa_ramrod->common.reject = 1;
 	}
 
+	iwarp_info = &p_hwfn->p_rdma_info->iwarp;
+	p_mpa_ramrod->rcv_wnd = iwarp_info->rcv_wnd_size;
 	p_mpa_ramrod->mode = ep->mpa_rev;
 	SET_FIELD(p_mpa_ramrod->rtr_pref,
 		  IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED, ep->rtr_type);
@@ -1681,6 +1703,13 @@ qed_iwarp_parse_rx_pkt(struct qed_hwfn *p_hwfn,
 	iph = (struct iphdr *)((u8 *)(ethh) + eth_hlen);
 
 	if (eth_type == ETH_P_IP) {
+		if (iph->protocol != IPPROTO_TCP) {
+			DP_NOTICE(p_hwfn,
+				  "Unexpected ip protocol on ll2 %x\n",
+				  iph->protocol);
+			return -EINVAL;
+		}
+
 		cm_info->local_ip[0] = ntohl(iph->daddr);
 		cm_info->remote_ip[0] = ntohl(iph->saddr);
 		cm_info->ip_version = TCP_IPV4;
@@ -1689,6 +1718,14 @@ qed_iwarp_parse_rx_pkt(struct qed_hwfn *p_hwfn,
 		*payload_len = ntohs(iph->tot_len) - ip_hlen;
 	} else if (eth_type == ETH_P_IPV6) {
 		ip6h = (struct ipv6hdr *)iph;
+
+		if (ip6h->nexthdr != IPPROTO_TCP) {
+			DP_NOTICE(p_hwfn,
+				  "Unexpected ip protocol on ll2 %x\n",
+				  iph->protocol);
+			return -EINVAL;
+		}
+
 		for (i = 0; i < 4; i++) {
 			cm_info->local_ip[i] =
 			    ntohl(ip6h->daddr.in6_u.u6_addr32[i]);
@@ -1906,8 +1943,8 @@ qed_iwarp_update_fpdu_length(struct qed_hwfn *p_hwfn,
 		/* Missing lower byte is now available */
 		mpa_len = fpdu->fpdu_length | *mpa_data;
 		fpdu->fpdu_length = QED_IWARP_FPDU_LEN_WITH_PAD(mpa_len);
-		fpdu->mpa_frag_len = fpdu->fpdu_length;
 		/* one byte of hdr */
+		fpdu->mpa_frag_len = 1;
 		fpdu->incomplete_bytes = fpdu->fpdu_length - 1;
 		DP_VERBOSE(p_hwfn,
 			   QED_MSG_RDMA,
@@ -2745,6 +2782,7 @@ int qed_iwarp_setup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	/* value 0 is used for ilog2(QED_IWARP_RCV_WND_SIZE_MIN) */
 	iwarp_info->rcv_wnd_scale = ilog2(rcv_wnd_size) -
 	    ilog2(QED_IWARP_RCV_WND_SIZE_MIN);
+	iwarp_info->rcv_wnd_size = rcv_wnd_size >> iwarp_info->rcv_wnd_scale;
 	iwarp_info->crc_needed = QED_IWARP_PARAM_CRC_NEEDED;
 	iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED;
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_iwarp.h b/drivers/net/ethernet/qlogic/qed/qed_iwarp.h
index c1ecd743305f..b8f612d00241 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_iwarp.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_iwarp.h
@@ -95,6 +95,7 @@ struct qed_iwarp_info {
 	spinlock_t iw_lock;	/* for iwarp resources */
 	spinlock_t qp_lock;	/* for teardown races */
 	u32 rcv_wnd_scale;
+	u16 rcv_wnd_size;
 	u16 max_mtu;
 	u8 mac_addr[ETH_ALEN];
 	u8 crc_needed;
@@ -187,7 +188,7 @@ int qed_iwarp_setup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 		    struct qed_rdma_start_in_params *params);
 
 void qed_iwarp_init_fw_ramrod(struct qed_hwfn *p_hwfn,
-			      struct iwarp_init_func_params *p_ramrod);
+			      struct iwarp_init_func_ramrod_data *p_ramrod);
 
 int qed_iwarp_stop(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt);
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.c b/drivers/net/ethernet/qlogic/qed/qed_l2.c
index 085338990f49..893ef08a4b39 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.c
@@ -223,10 +223,9 @@ _qed_eth_queue_to_cid(struct qed_hwfn *p_hwfn,
 	struct qed_queue_cid *p_cid;
 	int rc;
 
-	p_cid = vmalloc(sizeof(*p_cid));
+	p_cid = vzalloc(sizeof(*p_cid));
 	if (!p_cid)
 		return NULL;
-	memset(p_cid, 0, sizeof(*p_cid));
 
 	p_cid->opaque_fid = opaque_fid;
 	p_cid->cid = cid;
@@ -1969,33 +1968,45 @@ void qed_reset_vport_stats(struct qed_dev *cdev)
 		_qed_get_vport_stats(cdev, cdev->reset_stats);
 }
 
-static void
-qed_arfs_mode_configure(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
-			struct qed_arfs_config_params *p_cfg_params)
+static enum gft_profile_type
+qed_arfs_mode_to_hsi(enum qed_filter_config_mode mode)
 {
-	if (p_cfg_params->arfs_enable) {
-		qed_set_rfs_mode_enable(p_hwfn, p_ptt, p_hwfn->rel_pf_id,
-					p_cfg_params->tcp, p_cfg_params->udp,
-					p_cfg_params->ipv4, p_cfg_params->ipv6);
-		DP_VERBOSE(p_hwfn, QED_MSG_SP,
-			   "tcp = %s, udp = %s, ipv4 = %s, ipv6 =%s\n",
+	if (mode == QED_FILTER_CONFIG_MODE_5_TUPLE)
+		return GFT_PROFILE_TYPE_4_TUPLE;
+	if (mode == QED_FILTER_CONFIG_MODE_IP_DEST)
+		return GFT_PROFILE_TYPE_IP_DST_PORT;
+	return GFT_PROFILE_TYPE_L4_DST_PORT;
+}
+
+void qed_arfs_mode_configure(struct qed_hwfn *p_hwfn,
+			     struct qed_ptt *p_ptt,
+			     struct qed_arfs_config_params *p_cfg_params)
+{
+	if (p_cfg_params->mode != QED_FILTER_CONFIG_MODE_DISABLE) {
+		qed_gft_config(p_hwfn, p_ptt, p_hwfn->rel_pf_id,
+			       p_cfg_params->tcp,
+			       p_cfg_params->udp,
+			       p_cfg_params->ipv4,
+			       p_cfg_params->ipv6,
+			       qed_arfs_mode_to_hsi(p_cfg_params->mode));
+		DP_VERBOSE(p_hwfn,
+			   QED_MSG_SP,
+			   "Configured Filtering: tcp = %s, udp = %s, ipv4 = %s, ipv6 =%s mode=%08x\n",
 			   p_cfg_params->tcp ? "Enable" : "Disable",
 			   p_cfg_params->udp ? "Enable" : "Disable",
 			   p_cfg_params->ipv4 ? "Enable" : "Disable",
-			   p_cfg_params->ipv6 ? "Enable" : "Disable");
+			   p_cfg_params->ipv6 ? "Enable" : "Disable",
+			   (u32)p_cfg_params->mode);
 	} else {
-		qed_set_rfs_mode_disable(p_hwfn, p_ptt, p_hwfn->rel_pf_id);
+		DP_VERBOSE(p_hwfn, QED_MSG_SP, "Disabled Filtering\n");
+		qed_gft_disable(p_hwfn, p_ptt, p_hwfn->rel_pf_id);
 	}
-
-	DP_VERBOSE(p_hwfn, QED_MSG_SP, "Configured ARFS mode : %s\n",
-		   p_cfg_params->arfs_enable ? "Enable" : "Disable");
 }
 
-static int
-qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
+int
+qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn,
 				struct qed_spq_comp_cb *p_cb,
-				dma_addr_t p_addr, u16 length, u16 qid,
-				u8 vport_id, bool b_is_add)
+				struct qed_ntuple_filter_params *p_params)
 {
 	struct rx_update_gft_filter_data *p_ramrod = NULL;
 	struct qed_spq_entry *p_ent = NULL;
@@ -2004,13 +2015,15 @@ qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 	u8 abs_vport_id = 0;
 	int rc = -EINVAL;
 
-	rc = qed_fw_vport(p_hwfn, vport_id, &abs_vport_id);
+	rc = qed_fw_vport(p_hwfn, p_params->vport_id, &abs_vport_id);
 	if (rc)
 		return rc;
 
-	rc = qed_fw_l2_queue(p_hwfn, qid, &abs_rx_q_id);
-	if (rc)
-		return rc;
+	if (p_params->qid != QED_RFS_NTUPLE_QID_RSS) {
+		rc = qed_fw_l2_queue(p_hwfn, p_params->qid, &abs_rx_q_id);
+		if (rc)
+			return rc;
+	}
 
 	/* Get SPQ entry */
 	memset(&init_data, 0, sizeof(init_data));
@@ -2032,17 +2045,27 @@ qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
 		return rc;
 
 	p_ramrod = &p_ent->ramrod.rx_update_gft;
-	DMA_REGPAIR_LE(p_ramrod->pkt_hdr_addr, p_addr);
-	p_ramrod->pkt_hdr_length = cpu_to_le16(length);
-	p_ramrod->rx_qid_or_action_icid = cpu_to_le16(abs_rx_q_id);
-	p_ramrod->vport_id = abs_vport_id;
-	p_ramrod->filter_type = RFS_FILTER_TYPE;
-	p_ramrod->filter_action = b_is_add ? GFT_ADD_FILTER : GFT_DELETE_FILTER;
+
+	DMA_REGPAIR_LE(p_ramrod->pkt_hdr_addr, p_params->addr);
+	p_ramrod->pkt_hdr_length = cpu_to_le16(p_params->length);
+
+	if (p_params->qid != QED_RFS_NTUPLE_QID_RSS) {
+		p_ramrod->rx_qid_valid = 1;
+		p_ramrod->rx_qid = cpu_to_le16(abs_rx_q_id);
+	}
+
+	p_ramrod->flow_id_valid = 0;
+	p_ramrod->flow_id = 0;
+
+	p_ramrod->vport_id = cpu_to_le16((u16)abs_vport_id);
+	p_ramrod->filter_action = p_params->b_is_add ? GFT_ADD_FILTER
+	    : GFT_DELETE_FILTER;
 
 	DP_VERBOSE(p_hwfn, QED_MSG_SP,
 		   "V[%0x], Q[%04x] - %s filter from 0x%llx [length %04xb]\n",
 		   abs_vport_id, abs_rx_q_id,
-		   b_is_add ? "Adding" : "Removing", (u64)p_addr, length);
+		   p_params->b_is_add ? "Adding" : "Removing",
+		   (u64)p_params->addr, p_params->length);
 
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
@@ -2743,7 +2766,8 @@ static int qed_configure_filter(struct qed_dev *cdev,
 	}
 }
 
-static int qed_configure_arfs_searcher(struct qed_dev *cdev, bool en_searcher)
+static int qed_configure_arfs_searcher(struct qed_dev *cdev,
+				       enum qed_filter_config_mode mode)
 {
 	struct qed_hwfn *p_hwfn = QED_LEADING_HWFN(cdev);
 	struct qed_arfs_config_params arfs_config_params;
@@ -2753,8 +2777,7 @@ static int qed_configure_arfs_searcher(struct qed_dev *cdev, bool en_searcher)
 	arfs_config_params.udp = true;
 	arfs_config_params.ipv4 = true;
 	arfs_config_params.ipv6 = true;
-	arfs_config_params.arfs_enable = en_searcher;
-
+	arfs_config_params.mode = mode;
 	qed_arfs_mode_configure(p_hwfn, p_hwfn->p_arfs_ptt,
 				&arfs_config_params);
 	return 0;
@@ -2762,8 +2785,8 @@ static int qed_configure_arfs_searcher(struct qed_dev *cdev, bool en_searcher)
 
 static void
 qed_arfs_sp_response_handler(struct qed_hwfn *p_hwfn,
-			     void *cookie, union event_ring_data *data,
-			     u8 fw_return_code)
+			     void *cookie,
+			     union event_ring_data *data, u8 fw_return_code)
 {
 	struct qed_common_cb_ops *op = p_hwfn->cdev->protocol_ops.common;
 	void *dev = p_hwfn->cdev->ops_cookie;
@@ -2771,10 +2794,10 @@ qed_arfs_sp_response_handler(struct qed_hwfn *p_hwfn,
 	op->arfs_filter_op(dev, cookie, fw_return_code);
 }
 
-static int qed_ntuple_arfs_filter_config(struct qed_dev *cdev, void *cookie,
-					 dma_addr_t mapping, u16 length,
-					 u16 vport_id, u16 rx_queue_id,
-					 bool add_filter)
+static int
+qed_ntuple_arfs_filter_config(struct qed_dev *cdev,
+			      void *cookie,
+			      struct qed_ntuple_filter_params *params)
 {
 	struct qed_hwfn *p_hwfn = QED_LEADING_HWFN(cdev);
 	struct qed_spq_comp_cb cb;
@@ -2783,9 +2806,19 @@ static int qed_ntuple_arfs_filter_config(struct qed_dev *cdev, void *cookie,
 	cb.function = qed_arfs_sp_response_handler;
 	cb.cookie = cookie;
 
-	rc = qed_configure_rfs_ntuple_filter(p_hwfn, p_hwfn->p_arfs_ptt,
-					     &cb, mapping, length, rx_queue_id,
-					     vport_id, add_filter);
+	if (params->b_is_vf) {
+		if (!qed_iov_is_valid_vfid(p_hwfn, params->vf_id, false,
+					   false)) {
+			DP_INFO(p_hwfn, "vfid 0x%02x is out of bounds\n",
+				params->vf_id);
+			return rc;
+		}
+
+		params->vport_id = params->vf_id + 1;
+		params->qid = QED_RFS_NTUPLE_QID_RSS;
+	}
+
+	rc = qed_configure_rfs_ntuple_filter(p_hwfn, &cb, params);
 	if (rc)
 		DP_NOTICE(p_hwfn,
 			  "Failed to issue a-RFS filter configuration\n");
diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.h b/drivers/net/ethernet/qlogic/qed/qed_l2.h
index cc1f248551c9..c4030e949cce 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_l2.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_l2.h
@@ -190,7 +190,7 @@ struct qed_arfs_config_params {
 	bool udp;
 	bool ipv4;
 	bool ipv6;
-	bool arfs_enable;
+	enum qed_filter_config_mode mode;
 };
 
 struct qed_sp_vport_update_params {
@@ -277,6 +277,37 @@ void qed_get_vport_stats(struct qed_dev *cdev, struct qed_eth_stats *stats);
 
 void qed_reset_vport_stats(struct qed_dev *cdev);
 
+/**
+ * *@brief qed_arfs_mode_configure -
+ *
+ **Enable or disable rfs mode. It must accept atleast one of tcp or udp true
+ **and atleast one of ipv4 or ipv6 true to enable rfs mode.
+ *
+ **@param p_hwfn
+ **@param p_ptt
+ **@param p_cfg_params - arfs mode configuration parameters.
+ *
+ */
+void qed_arfs_mode_configure(struct qed_hwfn *p_hwfn,
+			     struct qed_ptt *p_ptt,
+			     struct qed_arfs_config_params *p_cfg_params);
+
+/**
+ * @brief - qed_configure_rfs_ntuple_filter
+ *
+ * This ramrod should be used to add or remove arfs hw filter
+ *
+ * @params p_hwfn
+ * @params p_cb - Used for QED_SPQ_MODE_CB,where client would initialize
+ *		  it with cookie and callback function address, if not
+ *		  using this mode then client must pass NULL.
+ * @params p_params
+ */
+int
+qed_configure_rfs_ntuple_filter(struct qed_hwfn *p_hwfn,
+				struct qed_spq_comp_cb *p_cb,
+				struct qed_ntuple_filter_params *p_params);
+
 #define MAX_QUEUES_PER_QZONE    (sizeof(unsigned long) * 8)
 #define QED_QUEUE_CID_SELF	(0xff)
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
index 047f556ca62e..c4f14fdc4e77 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_ll2.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c
@@ -406,6 +406,9 @@ static void qed_ll2_rxq_parse_gsi(struct qed_hwfn *p_hwfn,
 	data->opaque_data_0 = le32_to_cpu(p_cqe->rx_cqe_gsi.src_mac_addrhi);
 	data->opaque_data_1 = le16_to_cpu(p_cqe->rx_cqe_gsi.src_mac_addrlo);
 	data->u.data_length_error = p_cqe->rx_cqe_gsi.data_length_error;
+	data->qp_id = le16_to_cpu(p_cqe->rx_cqe_gsi.qp_id);
+
+	data->src_qp = le32_to_cpu(p_cqe->rx_cqe_gsi.src_qp);
 }
 
 static void qed_ll2_rxq_parse_reg(struct qed_hwfn *p_hwfn,
@@ -927,7 +930,8 @@ static int qed_sp_ll2_rx_queue_start(struct qed_hwfn *p_hwfn,
 		       qed_chain_get_pbl_phys(&p_rx->rcq_chain));
 
 	p_ramrod->drop_ttl0_flg = p_ll2_conn->input.rx_drop_ttl0_flg;
-	p_ramrod->inner_vlan_removal_en = p_ll2_conn->input.rx_vlan_removal_en;
+	p_ramrod->inner_vlan_stripping_en =
+		p_ll2_conn->input.rx_vlan_removal_en;
 	p_ramrod->queue_id = p_ll2_conn->queue_id;
 	p_ramrod->main_func_queue = p_ll2_conn->main_func_queue ? 1 : 0;
 
@@ -1299,8 +1303,20 @@ int qed_ll2_acquire_connection(void *cxt, struct qed_ll2_acquire_data *data)
 
 	memcpy(&p_ll2_info->input, &data->input, sizeof(p_ll2_info->input));
 
-	p_ll2_info->tx_dest = (data->input.tx_dest == QED_LL2_TX_DEST_NW) ?
-			      CORE_TX_DEST_NW : CORE_TX_DEST_LB;
+	switch (data->input.tx_dest) {
+	case QED_LL2_TX_DEST_NW:
+		p_ll2_info->tx_dest = CORE_TX_DEST_NW;
+		break;
+	case QED_LL2_TX_DEST_LB:
+		p_ll2_info->tx_dest = CORE_TX_DEST_LB;
+		break;
+	case QED_LL2_TX_DEST_DROP:
+		p_ll2_info->tx_dest = CORE_TX_DEST_DROP;
+		break;
+	default:
+		return -EINVAL;
+	}
+
 	if (data->input.conn_type == QED_LL2_TYPE_OOO ||
 	    data->input.secondary_queue)
 		p_ll2_info->main_func_queue = false;
@@ -2281,8 +2297,7 @@ static int qed_ll2_start(struct qed_dev *cdev, struct qed_ll2_params *params)
 		goto release_terminate;
 	}
 
-	if (cdev->hwfns[0].hw_info.personality == QED_PCI_ISCSI &&
-	    cdev->hwfns[0].pf_params.iscsi_pf_params.ooo_enable) {
+	if (QED_LEADING_HWFN(cdev)->hw_info.personality == QED_PCI_ISCSI) {
 		DP_VERBOSE(cdev, QED_MSG_STORAGE, "Starting OOO LL2 queue\n");
 		rc = qed_ll2_start_ooo(cdev, params);
 		if (rc) {
@@ -2340,8 +2355,7 @@ static int qed_ll2_stop(struct qed_dev *cdev)
 	qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt);
 	eth_zero_addr(cdev->ll2_mac_address);
 
-	if (cdev->hwfns[0].hw_info.personality == QED_PCI_ISCSI &&
-	    cdev->hwfns[0].pf_params.iscsi_pf_params.ooo_enable)
+	if (QED_LEADING_HWFN(cdev)->hw_info.personality == QED_PCI_ISCSI)
 		qed_ll2_stop_ooo(cdev);
 
 	rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev),
diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
index 8b99c7d26f34..6f46cb11f349 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c
@@ -2234,7 +2234,7 @@ int qed_mcp_nvm_read(struct qed_dev *cdev, u32 addr, u8 *p_buf, u32 len)
 					DRV_MSG_CODE_NVM_READ_NVRAM,
 					addr + offset +
 					(bytes_to_copy <<
-					 DRV_MB_PARAM_NVM_LEN_SHIFT),
+					 DRV_MB_PARAM_NVM_LEN_OFFSET),
 					&resp, &resp_param,
 					&read_len,
 					(u32 *)(p_buf + offset));
diff --git a/drivers/net/ethernet/qlogic/qed/qed_rdma.c b/drivers/net/ethernet/qlogic/qed/qed_rdma.c
index b7abb8205d3a..a411f9c702a1 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_rdma.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_rdma.c
@@ -360,13 +360,13 @@ static void qed_rdma_resc_free(struct qed_hwfn *p_hwfn)
 
 static void qed_rdma_free_tid(void *rdma_cxt, u32 itid)
 {
-        struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
+	struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt;
 
-        DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid);
+	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid);
 
-        spin_lock_bh(&p_hwfn->p_rdma_info->lock);
-        qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tid_map, itid);
-        spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
+	spin_lock_bh(&p_hwfn->p_rdma_info->lock);
+	qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tid_map, itid);
+	spin_unlock_bh(&p_hwfn->p_rdma_info->lock);
 }
 
 static void qed_rdma_free_reserved_lkey(struct qed_hwfn *p_hwfn)
@@ -379,6 +379,7 @@ static void qed_rdma_free(struct qed_hwfn *p_hwfn)
 	DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Freeing RDMA\n");
 
 	qed_rdma_free_reserved_lkey(p_hwfn);
+	qed_cxt_free_proto_ilt(p_hwfn, p_hwfn->p_rdma_info->proto);
 	qed_rdma_resc_free(p_hwfn);
 }
 
@@ -570,7 +571,7 @@ static int qed_rdma_start_fw(struct qed_hwfn *p_hwfn,
 
 	if (QED_IS_IWARP_PERSONALITY(p_hwfn)) {
 		qed_iwarp_init_fw_ramrod(p_hwfn,
-					 &p_ent->ramrod.iwarp_init_func.iwarp);
+					 &p_ent->ramrod.iwarp_init_func);
 		p_ramrod = &p_ent->ramrod.iwarp_init_func.rdma;
 	} else {
 		p_ramrod = &p_ent->ramrod.roce_init_func.rdma;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
index 0cdb4337b3a0..f7122059b6b5 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
@@ -124,6 +124,8 @@
 	0x1f0434UL
 #define PRS_REG_SEARCH_TAG1 \
 	0x1f0444UL
+#define PRS_REG_SEARCH_TENANT_ID \
+	0x1f044cUL
 #define PRS_REG_PKT_LEN_STAT_TAGS_NOT_COUNTED_FIRST \
 	0x1f0a0cUL
 #define PRS_REG_SEARCH_TCP_FIRST_FRAG \
@@ -200,7 +202,13 @@
 	0x2e8800UL
 #define CCFC_REG_STRONG_ENABLE_VF \
 	0x2e070cUL
-#define  CDU_REG_CID_ADDR_PARAMS	\
+#define CDU_REG_CCFC_CTX_VALID0 \
+	0x580400UL
+#define CDU_REG_CCFC_CTX_VALID1 \
+	0x580404UL
+#define CDU_REG_TCFC_CTX_VALID0 \
+	0x580408UL
+#define  CDU_REG_CID_ADDR_PARAMS \
 	0x580900UL
 #define  DBG_REG_CLIENT_ENABLE \
 	0x010004UL
@@ -564,7 +572,7 @@
 #define PRS_REG_ENCAPSULATION_TYPE_EN	0x1f0730UL
 #define PRS_REG_GRE_PROTOCOL		0x1f0734UL
 #define PRS_REG_VXLAN_PORT		0x1f0738UL
-#define PRS_REG_OUTPUT_FORMAT_4_0	0x1f099cUL
+#define PRS_REG_OUTPUT_FORMAT_4_0_BB_K2	0x1f099cUL
 #define NIG_REG_ENC_TYPE_ENABLE		0x501058UL
 
 #define NIG_REG_ENC_TYPE_ENABLE_ETH_OVER_GRE_ENABLE		(0x1 << 0)
@@ -580,11 +588,11 @@
 #define PRS_REG_NGE_PORT		0x1f086cUL
 #define NIG_REG_NGE_PORT		0x508b38UL
 
-#define DORQ_REG_L2_EDPM_TUNNEL_GRE_ETH_EN	0x10090cUL
-#define DORQ_REG_L2_EDPM_TUNNEL_GRE_IP_EN	0x100910UL
-#define DORQ_REG_L2_EDPM_TUNNEL_VXLAN_EN	0x100914UL
-#define DORQ_REG_L2_EDPM_TUNNEL_NGE_IP_EN	0x10092cUL
-#define DORQ_REG_L2_EDPM_TUNNEL_NGE_ETH_EN	0x100930UL
+#define DORQ_REG_L2_EDPM_TUNNEL_GRE_ETH_EN		0x10090cUL
+#define DORQ_REG_L2_EDPM_TUNNEL_GRE_IP_EN		0x100910UL
+#define DORQ_REG_L2_EDPM_TUNNEL_VXLAN_EN		0x100914UL
+#define DORQ_REG_L2_EDPM_TUNNEL_NGE_IP_EN_K2_E5		0x10092cUL
+#define DORQ_REG_L2_EDPM_TUNNEL_NGE_ETH_EN_K2_E5	0x100930UL
 
 #define NIG_REG_NGE_IP_ENABLE			0x508b28UL
 #define NIG_REG_NGE_ETH_ENABLE			0x508b2cUL
@@ -595,15 +603,15 @@
 #define QM_REG_WFQPFWEIGHT	0x2f4e80UL
 #define QM_REG_WFQVPWEIGHT	0x2fa000UL
 
-#define PGLCS_REG_DBG_SELECT_K2 \
+#define PGLCS_REG_DBG_SELECT_K2_E5 \
 	0x001d14UL
-#define PGLCS_REG_DBG_DWORD_ENABLE_K2 \
+#define PGLCS_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x001d18UL
-#define PGLCS_REG_DBG_SHIFT_K2 \
+#define PGLCS_REG_DBG_SHIFT_K2_E5 \
 	0x001d1cUL
-#define PGLCS_REG_DBG_FORCE_VALID_K2 \
+#define PGLCS_REG_DBG_FORCE_VALID_K2_E5 \
 	0x001d20UL
-#define PGLCS_REG_DBG_FORCE_FRAME_K2 \
+#define PGLCS_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x001d24UL
 #define MISC_REG_RESET_PL_PDA_VMAIN_1 \
 	0x008070UL
@@ -615,7 +623,7 @@
 	0x009050UL
 #define MISCS_REG_RESET_PL_HV \
 	0x009060UL
-#define MISCS_REG_RESET_PL_HV_2_K2	\
+#define MISCS_REG_RESET_PL_HV_2_K2_E5 \
 	0x009150UL
 #define DMAE_REG_DBG_SELECT \
 	0x00c510UL
@@ -647,15 +655,15 @@
 	0x0500b0UL
 #define GRC_REG_DBG_FORCE_FRAME	\
 	0x0500b4UL
-#define UMAC_REG_DBG_SELECT_K2 \
+#define UMAC_REG_DBG_SELECT_K2_E5 \
 	0x051094UL
-#define UMAC_REG_DBG_DWORD_ENABLE_K2 \
+#define UMAC_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x051098UL
-#define UMAC_REG_DBG_SHIFT_K2 \
+#define UMAC_REG_DBG_SHIFT_K2_E5 \
 	0x05109cUL
-#define UMAC_REG_DBG_FORCE_VALID_K2 \
+#define UMAC_REG_DBG_FORCE_VALID_K2_E5 \
 	0x0510a0UL
-#define UMAC_REG_DBG_FORCE_FRAME_K2 \
+#define UMAC_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x0510a4UL
 #define MCP2_REG_DBG_SELECT \
 	0x052400UL
@@ -717,15 +725,15 @@
 	0x1f0ba0UL
 #define PRS_REG_DBG_FORCE_FRAME	\
 	0x1f0ba4UL
-#define CNIG_REG_DBG_SELECT_K2 \
+#define CNIG_REG_DBG_SELECT_K2_E5 \
 	0x218254UL
-#define CNIG_REG_DBG_DWORD_ENABLE_K2 \
+#define CNIG_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x218258UL
-#define CNIG_REG_DBG_SHIFT_K2 \
+#define CNIG_REG_DBG_SHIFT_K2_E5 \
 	0x21825cUL
-#define CNIG_REG_DBG_FORCE_VALID_K2 \
+#define CNIG_REG_DBG_FORCE_VALID_K2_E5 \
 	0x218260UL
-#define CNIG_REG_DBG_FORCE_FRAME_K2 \
+#define CNIG_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x218264UL
 #define PRM_REG_DBG_SELECT \
 	0x2306a8UL
@@ -997,35 +1005,35 @@
 	0x580710UL
 #define CDU_REG_DBG_FORCE_FRAME	\
 	0x580714UL
-#define WOL_REG_DBG_SELECT_K2 \
+#define WOL_REG_DBG_SELECT_K2_E5 \
 	0x600140UL
-#define WOL_REG_DBG_DWORD_ENABLE_K2 \
+#define WOL_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x600144UL
-#define WOL_REG_DBG_SHIFT_K2 \
+#define WOL_REG_DBG_SHIFT_K2_E5 \
 	0x600148UL
-#define WOL_REG_DBG_FORCE_VALID_K2 \
+#define WOL_REG_DBG_FORCE_VALID_K2_E5 \
 	0x60014cUL
-#define WOL_REG_DBG_FORCE_FRAME_K2 \
+#define WOL_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x600150UL
-#define BMBN_REG_DBG_SELECT_K2 \
+#define BMBN_REG_DBG_SELECT_K2_E5 \
 	0x610140UL
-#define BMBN_REG_DBG_DWORD_ENABLE_K2 \
+#define BMBN_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x610144UL
-#define BMBN_REG_DBG_SHIFT_K2 \
+#define BMBN_REG_DBG_SHIFT_K2_E5 \
 	0x610148UL
-#define BMBN_REG_DBG_FORCE_VALID_K2 \
+#define BMBN_REG_DBG_FORCE_VALID_K2_E5 \
 	0x61014cUL
-#define BMBN_REG_DBG_FORCE_FRAME_K2 \
+#define BMBN_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x610150UL
-#define NWM_REG_DBG_SELECT_K2 \
+#define NWM_REG_DBG_SELECT_K2_E5 \
 	0x8000ecUL
-#define NWM_REG_DBG_DWORD_ENABLE_K2 \
+#define NWM_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x8000f0UL
-#define NWM_REG_DBG_SHIFT_K2 \
+#define NWM_REG_DBG_SHIFT_K2_E5 \
 	0x8000f4UL
-#define NWM_REG_DBG_FORCE_VALID_K2 \
+#define NWM_REG_DBG_FORCE_VALID_K2_E5 \
 	0x8000f8UL
-#define NWM_REG_DBG_FORCE_FRAME_K2\
+#define NWM_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x8000fcUL
 #define PBF_REG_DBG_SELECT \
 	0xd80060UL
@@ -1247,36 +1255,76 @@
 	0x1901534UL
 #define USEM_REG_DBG_FORCE_FRAME \
 	0x1901538UL
-#define NWS_REG_DBG_SELECT_K2 \
+#define NWS_REG_DBG_SELECT_K2_E5 \
 	0x700128UL
-#define NWS_REG_DBG_DWORD_ENABLE_K2 \
+#define NWS_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x70012cUL
-#define NWS_REG_DBG_SHIFT_K2 \
+#define NWS_REG_DBG_SHIFT_K2_E5 \
 	0x700130UL
-#define NWS_REG_DBG_FORCE_VALID_K2 \
+#define NWS_REG_DBG_FORCE_VALID_K2_E5 \
 	0x700134UL
-#define NWS_REG_DBG_FORCE_FRAME_K2 \
+#define NWS_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x700138UL
-#define MS_REG_DBG_SELECT_K2 \
+#define MS_REG_DBG_SELECT_K2_E5 \
 	0x6a0228UL
-#define MS_REG_DBG_DWORD_ENABLE_K2 \
+#define MS_REG_DBG_DWORD_ENABLE_K2_E5 \
 	0x6a022cUL
-#define MS_REG_DBG_SHIFT_K2 \
+#define MS_REG_DBG_SHIFT_K2_E5 \
 	0x6a0230UL
-#define MS_REG_DBG_FORCE_VALID_K2 \
+#define MS_REG_DBG_FORCE_VALID_K2_E5 \
 	0x6a0234UL
-#define MS_REG_DBG_FORCE_FRAME_K2 \
+#define MS_REG_DBG_FORCE_FRAME_K2_E5 \
 	0x6a0238UL
-#define PCIE_REG_DBG_COMMON_SELECT_K2 \
+#define PCIE_REG_DBG_COMMON_SELECT_K2_E5 \
 	0x054398UL
-#define PCIE_REG_DBG_COMMON_DWORD_ENABLE_K2 \
+#define PCIE_REG_DBG_COMMON_DWORD_ENABLE_K2_E5 \
 	0x05439cUL
-#define PCIE_REG_DBG_COMMON_SHIFT_K2 \
+#define PCIE_REG_DBG_COMMON_SHIFT_K2_E5 \
 	0x0543a0UL
-#define PCIE_REG_DBG_COMMON_FORCE_VALID_K2 \
+#define PCIE_REG_DBG_COMMON_FORCE_VALID_K2_E5 \
 	0x0543a4UL
-#define PCIE_REG_DBG_COMMON_FORCE_FRAME_K2 \
+#define PCIE_REG_DBG_COMMON_FORCE_FRAME_K2_E5 \
 	0x0543a8UL
+#define PTLD_REG_DBG_SELECT_E5 \
+	0x5a1600UL
+#define PTLD_REG_DBG_DWORD_ENABLE_E5 \
+	0x5a1604UL
+#define PTLD_REG_DBG_SHIFT_E5 \
+	0x5a1608UL
+#define PTLD_REG_DBG_FORCE_VALID_E5 \
+	0x5a160cUL
+#define PTLD_REG_DBG_FORCE_FRAME_E5 \
+	0x5a1610UL
+#define YPLD_REG_DBG_SELECT_E5 \
+	0x5c1600UL
+#define YPLD_REG_DBG_DWORD_ENABLE_E5 \
+	0x5c1604UL
+#define YPLD_REG_DBG_SHIFT_E5 \
+	0x5c1608UL
+#define YPLD_REG_DBG_FORCE_VALID_E5 \
+	0x5c160cUL
+#define YPLD_REG_DBG_FORCE_FRAME_E5 \
+	0x5c1610UL
+#define RGSRC_REG_DBG_SELECT_E5	\
+	0x320040UL
+#define RGSRC_REG_DBG_DWORD_ENABLE_E5 \
+	0x320044UL
+#define RGSRC_REG_DBG_SHIFT_E5 \
+	0x320048UL
+#define RGSRC_REG_DBG_FORCE_VALID_E5 \
+	0x32004cUL
+#define RGSRC_REG_DBG_FORCE_FRAME_E5 \
+	0x320050UL
+#define TGSRC_REG_DBG_SELECT_E5	\
+	0x322040UL
+#define TGSRC_REG_DBG_DWORD_ENABLE_E5 \
+	0x322044UL
+#define TGSRC_REG_DBG_SHIFT_E5 \
+	0x322048UL
+#define TGSRC_REG_DBG_FORCE_VALID_E5 \
+	0x32204cUL
+#define TGSRC_REG_DBG_FORCE_FRAME_E5 \
+	0x322050UL
 #define MISC_REG_RESET_PL_UA \
 	0x008050UL
 #define MISC_REG_RESET_PL_HV \
@@ -1415,7 +1463,7 @@
 	0x1940000UL
 #define SEM_FAST_REG_INT_RAM \
 	0x020000UL
-#define SEM_FAST_REG_INT_RAM_SIZE \
+#define SEM_FAST_REG_INT_RAM_SIZE_BB_K2 \
 	20480
 #define GRC_REG_TRACE_FIFO_VALID_DATA \
 	0x050064UL
@@ -1433,6 +1481,8 @@
 	0x340800UL
 #define BRB_REG_BIG_RAM_DATA \
 	0x341500UL
+#define BRB_REG_BIG_RAM_DATA_SIZE \
+	64
 #define SEM_FAST_REG_STALL_0_BB_K2 \
 	0x000488UL
 #define SEM_FAST_REG_STALLED \
@@ -1451,7 +1501,7 @@
 	0x238c30UL
 #define MISCS_REG_BLOCK_256B_EN \
 	0x009074UL
-#define MCP_REG_SCRATCH_SIZE \
+#define MCP_REG_SCRATCH_SIZE_BB_K2 \
 	57344
 #define MCP_REG_CPU_REG_FILE \
 	0xe05200UL
@@ -1485,35 +1535,35 @@
 	0x008c14UL
 #define NWS_REG_NWS_CMU_K2	\
 	0x720000UL
-#define PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_7_0_K2 \
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_7_0_K2_E5 \
 	0x000680UL
-#define PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_15_8_K2 \
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_ADDR_15_8_K2_E5 \
 	0x000684UL
-#define PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_7_0_K2 \
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_7_0_K2_E5 \
 	0x0006c0UL
-#define PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_11_8_K2 \
+#define PHY_NW_IP_REG_PHY0_TOP_TBUS_DATA_11_8_K2_E5 \
 	0x0006c4UL
-#define MS_REG_MS_CMU_K2 \
+#define MS_REG_MS_CMU_K2_E5 \
 	0x6a4000UL
-#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X130_K2 \
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X130_K2_E5 \
 	0x000208UL
-#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X131_K2 \
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X131_K2_E5 \
 	0x00020cUL
-#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X132_K2 \
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X132_K2_E5 \
 	0x000210UL
-#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X133_K2 \
+#define PHY_SGMII_IP_REG_AHB_CMU_CSR_0_X133_K2_E5 \
 	0x000214UL
-#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130_K2 \
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X130_K2_E5 \
 	0x000208UL
-#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131_K2 \
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X131_K2_E5 \
 	0x00020cUL
-#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132_K2 \
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X132_K2_E5 \
 	0x000210UL
-#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133_K2 \
+#define PHY_PCIE_IP_REG_AHB_CMU_CSR_0_X133_K2_E5 \
 	0x000214UL
-#define PHY_PCIE_REG_PHY0_K2 \
+#define PHY_PCIE_REG_PHY0_K2_E5 \
 	0x620000UL
-#define PHY_PCIE_REG_PHY1_K2 \
+#define PHY_PCIE_REG_PHY1_K2_E5 \
 	0x624000UL
 #define NIG_REG_ROCE_DUPLICATE_TO_HOST 0x5088f0UL
 #define PRS_REG_LIGHT_L2_ETHERTYPE_EN 0x1f0968UL
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
index a1d33f35aad3..5e927b6cac22 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c
@@ -351,7 +351,9 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 		DP_NOTICE(p_hwfn, "Unsupported MF mode, init as DEFAULT\n");
 		p_ramrod->mf_mode = MF_NPAR;
 	}
-	p_ramrod->outer_tag = p_hwfn->hw_info.ovlan;
+
+	p_ramrod->outer_tag_config.outer_tag.tci =
+		cpu_to_le16(p_hwfn->hw_info.ovlan);
 
 	/* Place EQ address in RAMROD */
 	DMA_REGPAIR_LE(p_ramrod->event_ring_pbl_addr,
@@ -396,8 +398,8 @@ int qed_sp_pf_start(struct qed_hwfn *p_hwfn,
 	p_ramrod->hsi_fp_ver.minor_ver_arr[ETH_VER_KEY] = ETH_HSI_VER_MINOR;
 
 	DP_VERBOSE(p_hwfn, QED_MSG_SPQ,
-		   "Setting event_ring_sb [id %04x index %02x], outer_tag [%d]\n",
-		   sb, sb_index, p_ramrod->outer_tag);
+		   "Setting event_ring_sb [id %04x index %02x], outer_tag.tci [%d]\n",
+		   sb, sb_index, p_ramrod->outer_tag_config.outer_tag.tci);
 
 	rc = qed_spq_post(p_hwfn, p_ent, NULL);
 
diff --git a/drivers/net/ethernet/qlogic/qed/qed_spq.c b/drivers/net/ethernet/qlogic/qed/qed_spq.c
index 3588081b2e27..1673fc90027f 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_spq.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_spq.c
@@ -97,9 +97,7 @@ static int __qed_spq_block(struct qed_hwfn *p_hwfn,
 
 	while (iter_cnt--) {
 		/* Validate we receive completion update */
-		if (READ_ONCE(comp_done->done) == 1) {
-			/* Read updated FW return value */
-			smp_read_barrier_depends();
+		if (smp_load_acquire(&comp_done->done) == 1) { /* ^^^ */
 			if (p_fw_ret)
 				*p_fw_ret = comp_done->fw_return_code;
 			return 0;
@@ -215,7 +213,7 @@ static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn,
 static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn,
 				  struct qed_spq *p_spq)
 {
-	struct core_conn_context *p_cxt;
+	struct e4_core_conn_context *p_cxt;
 	struct qed_cxt_info cxt_info;
 	u16 physical_q;
 	int rc;
@@ -233,11 +231,11 @@ static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn,
 	p_cxt = cxt_info.p_cxt;
 
 	SET_FIELD(p_cxt->xstorm_ag_context.flags10,
-		  XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1);
+		  E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1);
 	SET_FIELD(p_cxt->xstorm_ag_context.flags1,
-		  XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1);
+		  E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1);
 	SET_FIELD(p_cxt->xstorm_ag_context.flags9,
-		  XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1);
+		  E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1);
 
 	/* QM physical queue */
 	physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB);
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.c b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
index 3f40b1de7957..5acb91b3564c 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.c
@@ -153,9 +153,9 @@ static int qed_sp_vf_stop(struct qed_hwfn *p_hwfn,
 	return qed_spq_post(p_hwfn, p_ent, NULL);
 }
 
-static bool qed_iov_is_valid_vfid(struct qed_hwfn *p_hwfn,
-				  int rel_vf_id,
-				  bool b_enabled_only, bool b_non_malicious)
+bool qed_iov_is_valid_vfid(struct qed_hwfn *p_hwfn,
+			   int rel_vf_id,
+			   bool b_enabled_only, bool b_non_malicious)
 {
 	if (!p_hwfn->pf_iov_info) {
 		DP_NOTICE(p_hwfn->cdev, "No iov info\n");
@@ -1621,7 +1621,7 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
 	/* fill in pfdev info */
 	pfdev_info->chip_num = p_hwfn->cdev->chip_num;
 	pfdev_info->db_size = 0;
-	pfdev_info->indices_per_sb = PIS_PER_SB;
+	pfdev_info->indices_per_sb = PIS_PER_SB_E4;
 
 	pfdev_info->capabilities = PFVF_ACQUIRE_CAP_DEFAULT_UNTAGGED |
 				   PFVF_ACQUIRE_CAP_POST_FW_OVERRIDE;
@@ -3582,11 +3582,11 @@ static int
 qed_iov_vf_flr_poll_pbf(struct qed_hwfn *p_hwfn,
 			struct qed_vf_info *p_vf, struct qed_ptt *p_ptt)
 {
-	u32 cons[MAX_NUM_VOQS], distance[MAX_NUM_VOQS];
+	u32 cons[MAX_NUM_VOQS_E4], distance[MAX_NUM_VOQS_E4];
 	int i, cnt;
 
 	/* Read initial consumers & producers */
-	for (i = 0; i < MAX_NUM_VOQS; i++) {
+	for (i = 0; i < MAX_NUM_VOQS_E4; i++) {
 		u32 prod;
 
 		cons[i] = qed_rd(p_hwfn, p_ptt,
@@ -3601,7 +3601,7 @@ qed_iov_vf_flr_poll_pbf(struct qed_hwfn *p_hwfn,
 	/* Wait for consumers to pass the producers */
 	i = 0;
 	for (cnt = 0; cnt < 50; cnt++) {
-		for (; i < MAX_NUM_VOQS; i++) {
+		for (; i < MAX_NUM_VOQS_E4; i++) {
 			u32 tmp;
 
 			tmp = qed_rd(p_hwfn, p_ptt,
@@ -3611,7 +3611,7 @@ qed_iov_vf_flr_poll_pbf(struct qed_hwfn *p_hwfn,
 				break;
 		}
 
-		if (i == MAX_NUM_VOQS)
+		if (i == MAX_NUM_VOQS_E4)
 			break;
 
 		msleep(20);
@@ -4237,6 +4237,7 @@ qed_iov_bulletin_get_forced_vlan(struct qed_hwfn *p_hwfn, u16 rel_vf_id)
 static int qed_iov_configure_tx_rate(struct qed_hwfn *p_hwfn,
 				     struct qed_ptt *p_ptt, int vfid, int val)
 {
+	struct qed_mcp_link_state *p_link;
 	struct qed_vf_info *vf;
 	u8 abs_vp_id = 0;
 	int rc;
@@ -4249,7 +4250,10 @@ static int qed_iov_configure_tx_rate(struct qed_hwfn *p_hwfn,
 	if (rc)
 		return rc;
 
-	return qed_init_vport_rl(p_hwfn, p_ptt, abs_vp_id, (u32)val);
+	p_link = &QED_LEADING_HWFN(p_hwfn->cdev)->mcp_info->link_output;
+
+	return qed_init_vport_rl(p_hwfn, p_ptt, abs_vp_id, (u32)val,
+				 p_link->speed);
 }
 
 static int
diff --git a/drivers/net/ethernet/qlogic/qed/qed_sriov.h b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
index 3955929ba892..9a8fd79611f2 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_sriov.h
+++ b/drivers/net/ethernet/qlogic/qed/qed_sriov.h
@@ -273,6 +273,23 @@ enum qed_iov_wq_flag {
 };
 
 #ifdef CONFIG_QED_SRIOV
+/**
+ * @brief Check if given VF ID @vfid is valid
+ *        w.r.t. @b_enabled_only value
+ *        if b_enabled_only = true - only enabled VF id is valid
+ *        else any VF id less than max_vfs is valid
+ *
+ * @param p_hwfn
+ * @param rel_vf_id - Relative VF ID
+ * @param b_enabled_only - consider only enabled VF
+ * @param b_non_malicious - true iff we want to validate vf isn't malicious.
+ *
+ * @return bool - true for valid VF ID
+ */
+bool qed_iov_is_valid_vfid(struct qed_hwfn *p_hwfn,
+			   int rel_vf_id,
+			   bool b_enabled_only, bool b_non_malicious);
+
 /**
  * @brief - Given a VF index, return index of next [including that] active VF.
  *
@@ -376,6 +393,13 @@ void qed_vf_start_iov_wq(struct qed_dev *cdev);
 int qed_sriov_disable(struct qed_dev *cdev, bool pci_enabled);
 void qed_inform_vf_link_state(struct qed_hwfn *hwfn);
 #else
+static inline bool
+qed_iov_is_valid_vfid(struct qed_hwfn *p_hwfn,
+		      int rel_vf_id, bool b_enabled_only, bool b_non_malicious)
+{
+	return false;
+}
+
 static inline u16 qed_iov_get_next_active_vf(struct qed_hwfn *p_hwfn,
 					     u16 rel_vf_id)
 {
diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h
index a3a70ade411f..9935978c5542 100644
--- a/drivers/net/ethernet/qlogic/qede/qede.h
+++ b/drivers/net/ethernet/qlogic/qede/qede.h
@@ -40,6 +40,7 @@
 #include <linux/kernel.h>
 #include <linux/mutex.h>
 #include <linux/bpf.h>
+#include <net/xdp.h>
 #include <linux/qed/qede_rdma.h>
 #include <linux/io.h>
 #ifdef CONFIG_RFS_ACCEL
@@ -52,9 +53,9 @@
 #include <linux/qed/qed_eth_if.h>
 
 #define QEDE_MAJOR_VERSION		8
-#define QEDE_MINOR_VERSION		10
-#define QEDE_REVISION_VERSION		10
-#define QEDE_ENGINEERING_VERSION	21
+#define QEDE_MINOR_VERSION		33
+#define QEDE_REVISION_VERSION		0
+#define QEDE_ENGINEERING_VERSION	20
 #define DRV_MODULE_VERSION __stringify(QEDE_MAJOR_VERSION) "."	\
 		__stringify(QEDE_MINOR_VERSION) "."		\
 		__stringify(QEDE_REVISION_VERSION) "."		\
@@ -345,6 +346,7 @@ struct qede_rx_queue {
 	u64 xdp_no_pass;
 
 	void *handle;
+	struct xdp_rxq_info xdp_rxq;
 };
 
 union db_prod {
@@ -494,6 +496,8 @@ int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid);
 void qede_vlan_mark_nonconfigured(struct qede_dev *edev);
 int qede_configure_vlan_filters(struct qede_dev *edev);
 
+netdev_features_t qede_fix_features(struct net_device *dev,
+				    netdev_features_t features);
 int qede_set_features(struct net_device *dev, netdev_features_t features);
 void qede_set_rx_mode(struct net_device *ndev);
 void qede_config_rx_mode(struct net_device *ndev);
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index dae741270022..4ca3847fffd4 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -940,6 +940,9 @@ int qede_change_mtu(struct net_device *ndev, int new_mtu)
 	DP_VERBOSE(edev, (NETIF_MSG_IFUP | NETIF_MSG_IFDOWN),
 		   "Configuring MTU size of %d\n", new_mtu);
 
+	if (new_mtu > PAGE_SIZE)
+		ndev->features &= ~NETIF_F_GRO_HW;
+
 	/* Set the mtu field and re-start the interface if needed */
 	args.u.mtu = new_mtu;
 	args.func = &qede_update_mtu;
diff --git a/drivers/net/ethernet/qlogic/qede/qede_filter.c b/drivers/net/ethernet/qlogic/qede/qede_filter.c
index c1a0708a7d7c..6687e04d1558 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_filter.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_filter.c
@@ -98,10 +98,18 @@ static void qede_configure_arfs_fltr(struct qede_dev *edev,
 				     u16 rxq_id, bool add_fltr)
 {
 	const struct qed_eth_ops *op = edev->ops;
+	struct qed_ntuple_filter_params params;
 
 	if (n->used)
 		return;
 
+	memset(&params, 0, sizeof(params));
+
+	params.addr = n->mapping;
+	params.length = n->buf_len;
+	params.qid = rxq_id;
+	params.b_is_add = add_fltr;
+
 	DP_VERBOSE(edev, NETIF_MSG_RX_STATUS,
 		   "%s arfs filter flow_id=%d, sw_id=%d, src_port=%d, dst_port=%d, rxq=%d\n",
 		   add_fltr ? "Adding" : "Deleting",
@@ -110,8 +118,7 @@ static void qede_configure_arfs_fltr(struct qede_dev *edev,
 
 	n->used = true;
 	n->filter_op = add_fltr;
-	op->ntuple_filter_config(edev->cdev, n, n->mapping, n->buf_len, 0,
-				 rxq_id, add_fltr);
+	op->ntuple_filter_config(edev->cdev, n, &params);
 }
 
 static void
@@ -141,7 +148,10 @@ qede_enqueue_fltr_and_config_searcher(struct qede_dev *edev,
 	edev->arfs->filter_count++;
 
 	if (edev->arfs->filter_count == 1 && !edev->arfs->enable) {
-		edev->ops->configure_arfs_searcher(edev->cdev, true);
+		enum qed_filter_config_mode mode;
+
+		mode = QED_FILTER_CONFIG_MODE_5_TUPLE;
+		edev->ops->configure_arfs_searcher(edev->cdev, mode);
 		edev->arfs->enable = true;
 	}
 
@@ -160,8 +170,11 @@ qede_dequeue_fltr_and_config_searcher(struct qede_dev *edev,
 	edev->arfs->filter_count--;
 
 	if (!edev->arfs->filter_count && edev->arfs->enable) {
+		enum qed_filter_config_mode mode;
+
+		mode = QED_FILTER_CONFIG_MODE_DISABLE;
 		edev->arfs->enable = false;
-		edev->ops->configure_arfs_searcher(edev->cdev, false);
+		edev->ops->configure_arfs_searcher(edev->cdev, mode);
 	}
 }
 
@@ -255,8 +268,11 @@ void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr)
 
 	if (!edev->arfs->filter_count) {
 		if (edev->arfs->enable) {
+			enum qed_filter_config_mode mode;
+
+			mode = QED_FILTER_CONFIG_MODE_DISABLE;
 			edev->arfs->enable = false;
-			edev->ops->configure_arfs_searcher(edev->cdev, false);
+			edev->ops->configure_arfs_searcher(edev->cdev, mode);
 		}
 #ifdef CONFIG_RFS_ACCEL
 	} else {
@@ -895,19 +911,26 @@ static void qede_set_features_reload(struct qede_dev *edev,
 	edev->ndev->features = args->u.features;
 }
 
+netdev_features_t qede_fix_features(struct net_device *dev,
+				    netdev_features_t features)
+{
+	struct qede_dev *edev = netdev_priv(dev);
+
+	if (edev->xdp_prog || edev->ndev->mtu > PAGE_SIZE ||
+	    !(features & NETIF_F_GRO))
+		features &= ~NETIF_F_GRO_HW;
+
+	return features;
+}
+
 int qede_set_features(struct net_device *dev, netdev_features_t features)
 {
 	struct qede_dev *edev = netdev_priv(dev);
 	netdev_features_t changes = features ^ dev->features;
 	bool need_reload = false;
 
-	/* No action needed if hardware GRO is disabled during driver load */
-	if (changes & NETIF_F_GRO) {
-		if (dev->features & NETIF_F_GRO)
-			need_reload = !edev->gro_disable;
-		else
-			need_reload = edev->gro_disable;
-	}
+	if (changes & NETIF_F_GRO_HW)
+		need_reload = true;
 
 	if (need_reload) {
 		struct qede_reload_args args;
diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c
index 48ec4c56cddf..14941303189d 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_fp.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c
@@ -320,13 +320,11 @@ static inline void qede_update_tx_producer(struct qede_tx_queue *txq)
 	barrier();
 	writel(txq->tx_db.raw, txq->doorbell_addr);
 
-	/* mmiowb is needed to synchronize doorbell writes from more than one
-	 * processor. It guarantees that the write arrives to the device before
-	 * the queue lock is released and another start_xmit is called (possibly
-	 * on another CPU). Without this barrier, the next doorbell can bypass
-	 * this doorbell. This is applicable to IA64/Altix systems.
+	/* Fence required to flush the write combined buffer, since another
+	 * CPU may write to the same doorbell address and data may be lost
+	 * due to relaxed order nature of write combined bar.
 	 */
-	mmiowb();
+	wmb();
 }
 
 static int qede_xdp_xmit(struct qede_dev *edev, struct qede_fastpath *fp,
@@ -1006,6 +1004,7 @@ static bool qede_rx_xdp(struct qede_dev *edev,
 	xdp.data = xdp.data_hard_start + *data_offset;
 	xdp_set_data_meta_invalid(&xdp);
 	xdp.data_end = xdp.data + *len;
+	xdp.rxq = &rxq->xdp_rxq;
 
 	/* Queues always have a full reset currently, so for the time
 	 * being until there's atomic program replace just mark read
@@ -1248,16 +1247,10 @@ static int qede_rx_process_cqe(struct qede_dev *edev,
 
 	csum_flag = qede_check_csum(parse_flag);
 	if (unlikely(csum_flag == QEDE_CSUM_ERROR)) {
-		if (qede_pkt_is_ip_fragmented(fp_cqe, parse_flag)) {
+		if (qede_pkt_is_ip_fragmented(fp_cqe, parse_flag))
 			rxq->rx_ip_frags++;
-		} else {
-			DP_NOTICE(edev,
-				  "CQE has error, flags = %x, dropping incoming packet\n",
-				  parse_flag);
+		else
 			rxq->rx_hw_errors++;
-			qede_recycle_rx_bd_ring(rxq, fp_cqe->bd_num);
-			return 0;
-		}
 	}
 
 	/* Basic validation passed; Need to prepare an SKB. This would also
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index 8f9b3eb82137..a01e7d6e5442 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -288,7 +288,7 @@ int __init qede_init(void)
 	}
 
 	/* Must register notifier before pci ops, since we might miss
-	 * interface rename after pci probe and netdev registeration.
+	 * interface rename after pci probe and netdev registration.
 	 */
 	ret = register_netdevice_notifier(&qede_netdev_notifier);
 	if (ret) {
@@ -545,6 +545,7 @@ static const struct net_device_ops qede_netdev_ops = {
 #endif
 	.ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
 	.ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
+	.ndo_fix_features = qede_fix_features,
 	.ndo_set_features = qede_set_features,
 	.ndo_get_stats64 = qede_get_stats64,
 #ifdef CONFIG_QED_SRIOV
@@ -572,6 +573,7 @@ static const struct net_device_ops qede_netdev_vf_ops = {
 	.ndo_change_mtu = qede_change_mtu,
 	.ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
 	.ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
+	.ndo_fix_features = qede_fix_features,
 	.ndo_set_features = qede_set_features,
 	.ndo_get_stats64 = qede_get_stats64,
 	.ndo_udp_tunnel_add = qede_udp_tunnel_add,
@@ -589,6 +591,7 @@ static const struct net_device_ops qede_netdev_vf_xdp_ops = {
 	.ndo_change_mtu = qede_change_mtu,
 	.ndo_vlan_rx_add_vid = qede_vlan_rx_add_vid,
 	.ndo_vlan_rx_kill_vid = qede_vlan_rx_kill_vid,
+	.ndo_fix_features = qede_fix_features,
 	.ndo_set_features = qede_set_features,
 	.ndo_get_stats64 = qede_get_stats64,
 	.ndo_udp_tunnel_add = qede_udp_tunnel_add,
@@ -676,7 +679,7 @@ static void qede_init_ndev(struct qede_dev *edev)
 	ndev->priv_flags |= IFF_UNICAST_FLT;
 
 	/* user-changeble features */
-	hw_features = NETIF_F_GRO | NETIF_F_SG |
+	hw_features = NETIF_F_GRO | NETIF_F_GRO_HW | NETIF_F_SG |
 		      NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 		      NETIF_F_TSO | NETIF_F_TSO6;
 
@@ -762,6 +765,12 @@ static void qede_free_fp_array(struct qede_dev *edev)
 			fp = &edev->fp_array[i];
 
 			kfree(fp->sb_info);
+			/* Handle mem alloc failure case where qede_init_fp
+			 * didn't register xdp_rxq_info yet.
+			 * Implicit only (fp->type & QEDE_FASTPATH_RX)
+			 */
+			if (fp->rxq && xdp_rxq_info_is_reg(&fp->rxq->xdp_rxq))
+				xdp_rxq_info_unreg(&fp->rxq->xdp_rxq);
 			kfree(fp->rxq);
 			kfree(fp->xdp_tx);
 			kfree(fp->txq);
@@ -979,7 +988,7 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 	if (rc)
 		goto err3;
 
-	/* Prepare the lock prior to the registeration of the netdev,
+	/* Prepare the lock prior to the registration of the netdev,
 	 * as once it's registered we might reach flows requiring it
 	 * [it's even possible to reach a flow needing it directly
 	 * from there, although it's unlikely].
@@ -1068,10 +1077,6 @@ static void __qede_remove(struct pci_dev *pdev, enum qede_remove_mode mode)
 
 	pci_set_drvdata(pdev, NULL);
 
-	/* Release edev's reference to XDP's bpf if such exist */
-	if (edev->xdp_prog)
-		bpf_prog_put(edev->xdp_prog);
-
 	/* Use global ops since we've freed edev */
 	qed_ops->common->slowpath_stop(cdev);
 	if (system_state == SYSTEM_POWER_OFF)
@@ -1148,7 +1153,7 @@ static void qede_free_mem_sb(struct qede_dev *edev, struct qed_sb_info *sb_info,
 static int qede_alloc_mem_sb(struct qede_dev *edev,
 			     struct qed_sb_info *sb_info, u16 sb_id)
 {
-	struct status_block *sb_virt;
+	struct status_block_e4 *sb_virt;
 	dma_addr_t sb_phys;
 	int rc;
 
@@ -1232,18 +1237,9 @@ static int qede_alloc_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
 	dma_addr_t mapping;
 	int i;
 
-	/* Don't perform FW aggregations in case of XDP */
-	if (edev->xdp_prog)
-		edev->gro_disable = 1;
-
 	if (edev->gro_disable)
 		return 0;
 
-	if (edev->ndev->mtu > PAGE_SIZE) {
-		edev->gro_disable = 1;
-		return 0;
-	}
-
 	for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) {
 		struct qede_agg_info *tpa_info = &rxq->tpa_info[i];
 		struct sw_rx_data *replace_buf = &tpa_info->buffer;
@@ -1273,6 +1269,7 @@ static int qede_alloc_sge_mem(struct qede_dev *edev, struct qede_rx_queue *rxq)
 err:
 	qede_free_sge_mem(edev, rxq);
 	edev->gro_disable = 1;
+	edev->ndev->features &= ~NETIF_F_GRO_HW;
 	return -ENOMEM;
 }
 
@@ -1502,6 +1499,10 @@ static void qede_init_fp(struct qede_dev *edev)
 			else
 				fp->rxq->data_direction = DMA_FROM_DEVICE;
 			fp->rxq->dev = &edev->pdev->dev;
+
+			/* Driver have no error path from here */
+			WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev,
+						 fp->rxq->rxq_id) < 0);
 		}
 
 		if (fp->type & QEDE_FASTPATH_TX) {
@@ -1515,7 +1516,7 @@ static void qede_init_fp(struct qede_dev *edev)
 			 edev->ndev->name, queue_id);
 	}
 
-	edev->gro_disable = !(edev->ndev->features & NETIF_F_GRO);
+	edev->gro_disable = !(edev->ndev->features & NETIF_F_GRO_HW);
 }
 
 static int qede_set_real_num_queues(struct qede_dev *edev)
@@ -2066,8 +2067,6 @@ static int qede_load(struct qede_dev *edev, enum qede_load_mode mode,
 	link_params.link_up = true;
 	edev->ops->common->set_link(edev->cdev, &link_params);
 
-	qede_rdma_dev_event_open(edev);
-
 	edev->state = QEDE_STATE_OPEN;
 
 	DP_INFO(edev, "Ending successfully qede load\n");
@@ -2168,12 +2167,14 @@ static void qede_link_update(void *dev, struct qed_link_output *link)
 			DP_NOTICE(edev, "Link is up\n");
 			netif_tx_start_all_queues(edev->ndev);
 			netif_carrier_on(edev->ndev);
+			qede_rdma_dev_event_open(edev);
 		}
 	} else {
 		if (netif_carrier_ok(edev->ndev)) {
 			DP_NOTICE(edev, "Link is down\n");
 			netif_tx_disable(edev->ndev);
 			netif_carrier_off(edev->ndev);
+			qede_rdma_dev_event_close(edev);
 		}
 	}
 }
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ptp.c b/drivers/net/ethernet/qlogic/qede/qede_ptp.c
index 9b2280badaf7..02adb513f475 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ptp.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ptp.c
@@ -485,7 +485,7 @@ int qede_ptp_enable(struct qede_dev *edev, bool init_tc)
 	ptp->clock = ptp_clock_register(&ptp->clock_info, &edev->pdev->dev);
 	if (IS_ERR(ptp->clock)) {
 		rc = -EINVAL;
-		DP_ERR(edev, "PTP clock registeration failed\n");
+		DP_ERR(edev, "PTP clock registration failed\n");
 		goto err2;
 	}
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
index f7080d0ab874..46b0372dd032 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_83xx_hw.c
@@ -3891,7 +3891,7 @@ static void qlcnic_83xx_flush_mbx_queue(struct qlcnic_adapter *adapter)
 	struct list_head *head = &mbx->cmd_q;
 	struct qlcnic_cmd_args *cmd = NULL;
 
-	spin_lock(&mbx->queue_lock);
+	spin_lock_bh(&mbx->queue_lock);
 
 	while (!list_empty(head)) {
 		cmd = list_entry(head->next, struct qlcnic_cmd_args, list);
@@ -3902,7 +3902,7 @@ static void qlcnic_83xx_flush_mbx_queue(struct qlcnic_adapter *adapter)
 		qlcnic_83xx_notify_cmd_completion(adapter, cmd);
 	}
 
-	spin_unlock(&mbx->queue_lock);
+	spin_unlock_bh(&mbx->queue_lock);
 }
 
 static int qlcnic_83xx_check_mbx_status(struct qlcnic_adapter *adapter)
@@ -3938,12 +3938,12 @@ static void qlcnic_83xx_dequeue_mbx_cmd(struct qlcnic_adapter *adapter,
 {
 	struct qlcnic_mailbox *mbx = adapter->ahw->mailbox;
 
-	spin_lock(&mbx->queue_lock);
+	spin_lock_bh(&mbx->queue_lock);
 
 	list_del(&cmd->list);
 	mbx->num_cmds--;
 
-	spin_unlock(&mbx->queue_lock);
+	spin_unlock_bh(&mbx->queue_lock);
 
 	qlcnic_83xx_notify_cmd_completion(adapter, cmd);
 }
@@ -4008,7 +4008,7 @@ static int qlcnic_83xx_enqueue_mbx_cmd(struct qlcnic_adapter *adapter,
 		init_completion(&cmd->completion);
 		cmd->rsp_opcode = QLC_83XX_MBX_RESPONSE_UNKNOWN;
 
-		spin_lock(&mbx->queue_lock);
+		spin_lock_bh(&mbx->queue_lock);
 
 		list_add_tail(&cmd->list, &mbx->cmd_q);
 		mbx->num_cmds++;
@@ -4016,7 +4016,7 @@ static int qlcnic_83xx_enqueue_mbx_cmd(struct qlcnic_adapter *adapter,
 		*timeout = cmd->total_cmds * QLC_83XX_MBX_TIMEOUT;
 		queue_work(mbx->work_q, &mbx->work);
 
-		spin_unlock(&mbx->queue_lock);
+		spin_unlock_bh(&mbx->queue_lock);
 
 		return 0;
 	}
@@ -4112,15 +4112,15 @@ static void qlcnic_83xx_mailbox_worker(struct work_struct *work)
 		mbx->rsp_status = QLC_83XX_MBX_RESPONSE_WAIT;
 		spin_unlock_irqrestore(&mbx->aen_lock, flags);
 
-		spin_lock(&mbx->queue_lock);
+		spin_lock_bh(&mbx->queue_lock);
 
 		if (list_empty(head)) {
-			spin_unlock(&mbx->queue_lock);
+			spin_unlock_bh(&mbx->queue_lock);
 			return;
 		}
 		cmd = list_entry(head->next, struct qlcnic_cmd_args, list);
 
-		spin_unlock(&mbx->queue_lock);
+		spin_unlock_bh(&mbx->queue_lock);
 
 		mbx_ops->encode_cmd(adapter, cmd);
 		mbx_ops->nofity_fw(adapter, QLC_83XX_MBX_REQUEST);
diff --git a/drivers/net/ethernet/qlogic/qlge/qlge_main.c b/drivers/net/ethernet/qlogic/qlge/qlge_main.c
index 7b97a9969046..50038d946ced 100644
--- a/drivers/net/ethernet/qlogic/qlge/qlge_main.c
+++ b/drivers/net/ethernet/qlogic/qlge/qlge_main.c
@@ -1747,8 +1747,7 @@ static void ql_realign_skb(struct sk_buff *skb, int len)
 	 */
 	skb->data -= QLGE_SB_PAD - NET_IP_ALIGN;
 	skb->tail -= QLGE_SB_PAD - NET_IP_ALIGN;
-	skb_copy_to_linear_data(skb, temp_addr,
-		(unsigned int)len);
+	memmove(skb->data, temp_addr, len);
 }
 
 /*
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.c b/drivers/net/ethernet/qualcomm/emac/emac-mac.c
index 9cbb27263742..d5a32b7c7dc5 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac-mac.c
+++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.c
@@ -1194,9 +1194,9 @@ void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
 	while (tx_q->tpd.consume_idx != hw_consume_idx) {
 		tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
 		if (tpbuf->dma_addr) {
-			dma_unmap_single(adpt->netdev->dev.parent,
-					 tpbuf->dma_addr, tpbuf->length,
-					 DMA_TO_DEVICE);
+			dma_unmap_page(adpt->netdev->dev.parent,
+				       tpbuf->dma_addr, tpbuf->length,
+				       DMA_TO_DEVICE);
 			tpbuf->dma_addr = 0;
 		}
 
@@ -1353,9 +1353,11 @@ static void emac_tx_fill_tpd(struct emac_adapter *adpt,
 
 		tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
 		tpbuf->length = mapped_len;
-		tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
-						 skb->data, tpbuf->length,
-						 DMA_TO_DEVICE);
+		tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
+					       virt_to_page(skb->data),
+					       offset_in_page(skb->data),
+					       tpbuf->length,
+					       DMA_TO_DEVICE);
 		ret = dma_mapping_error(adpt->netdev->dev.parent,
 					tpbuf->dma_addr);
 		if (ret)
@@ -1371,9 +1373,12 @@ static void emac_tx_fill_tpd(struct emac_adapter *adpt,
 	if (mapped_len < len) {
 		tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
 		tpbuf->length = len - mapped_len;
-		tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
-						 skb->data + mapped_len,
-						 tpbuf->length, DMA_TO_DEVICE);
+		tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
+					       virt_to_page(skb->data +
+							    mapped_len),
+					       offset_in_page(skb->data +
+							      mapped_len),
+					       tpbuf->length, DMA_TO_DEVICE);
 		ret = dma_mapping_error(adpt->netdev->dev.parent,
 					tpbuf->dma_addr);
 		if (ret)
diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.h b/drivers/net/ethernet/qualcomm/emac/emac-mac.h
index 5028fb4bec2b..4beedb8faa1e 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac-mac.h
+++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.h
@@ -114,8 +114,9 @@ struct emac_tpd {
 #define TPD_INSTC_SET(tpd, val)		BITS_SET((tpd)->word[3], 17, 17, val)
 /* High-14bit Buffer Address, So, the 64b-bit address is
  * {DESC_CTRL_11_TX_DATA_HIADDR[17:0],(register) BUFFER_ADDR_H, BUFFER_ADDR_L}
+ * Extend TPD_BUFFER_ADDR_H to [31, 18], because we never enable timestamping.
  */
-#define TPD_BUFFER_ADDR_H_SET(tpd, val)	BITS_SET((tpd)->word[3], 18, 30, val)
+#define TPD_BUFFER_ADDR_H_SET(tpd, val)	BITS_SET((tpd)->word[3], 18, 31, val)
 /* Format D. Word offset from the 1st byte of this packet to start to calculate
  * the custom checksum.
  */
diff --git a/drivers/net/ethernet/qualcomm/emac/emac.c b/drivers/net/ethernet/qualcomm/emac/emac.c
index 38c924bdd32e..13235baf4766 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac.c
+++ b/drivers/net/ethernet/qualcomm/emac/emac.c
@@ -615,8 +615,11 @@ static int emac_probe(struct platform_device *pdev)
 	u32 reg;
 	int ret;
 
-	/* The TPD buffer address is limited to 45 bits. */
-	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(45));
+	/* The TPD buffer address is limited to:
+	 * 1. PTP:	45bits. (Driver doesn't support yet.)
+	 * 2. NON-PTP:	46bits.
+	 */
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
 	if (ret) {
 		dev_err(&pdev->dev, "could not set DMA mask\n");
 		return ret;
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
index df21e900f874..c4949183eef3 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
@@ -43,12 +43,6 @@
 
 /* Local Definitions and Declarations */
 
-struct rmnet_walk_data {
-	struct net_device *real_dev;
-	struct list_head *head;
-	struct rmnet_port *port;
-};
-
 static int rmnet_is_real_dev_registered(const struct net_device *real_dev)
 {
 	return rcu_access_pointer(real_dev->rx_handler) == rmnet_rx_handler;
@@ -112,17 +106,14 @@ static int rmnet_register_real_device(struct net_device *real_dev)
 static void rmnet_unregister_bridge(struct net_device *dev,
 				    struct rmnet_port *port)
 {
-	struct net_device *rmnet_dev, *bridge_dev;
 	struct rmnet_port *bridge_port;
+	struct net_device *bridge_dev;
 
 	if (port->rmnet_mode != RMNET_EPMODE_BRIDGE)
 		return;
 
 	/* bridge slave handling */
 	if (!port->nr_rmnet_devs) {
-		rmnet_dev = netdev_master_upper_dev_get_rcu(dev);
-		netdev_upper_dev_unlink(dev, rmnet_dev);
-
 		bridge_dev = port->bridge_ep;
 
 		bridge_port = rmnet_get_port_rtnl(bridge_dev);
@@ -132,9 +123,6 @@ static void rmnet_unregister_bridge(struct net_device *dev,
 		bridge_dev = port->bridge_ep;
 
 		bridge_port = rmnet_get_port_rtnl(bridge_dev);
-		rmnet_dev = netdev_master_upper_dev_get_rcu(bridge_dev);
-		netdev_upper_dev_unlink(bridge_dev, rmnet_dev);
-
 		rmnet_unregister_real_device(bridge_dev, bridge_port);
 	}
 }
@@ -143,11 +131,7 @@ static int rmnet_newlink(struct net *src_net, struct net_device *dev,
 			 struct nlattr *tb[], struct nlattr *data[],
 			 struct netlink_ext_ack *extack)
 {
-	int ingress_format = RMNET_INGRESS_FORMAT_DEMUXING |
-			     RMNET_INGRESS_FORMAT_DEAGGREGATION |
-			     RMNET_INGRESS_FORMAT_MAP;
-	int egress_format = RMNET_EGRESS_FORMAT_MUXING |
-			    RMNET_EGRESS_FORMAT_MAP;
+	u32 data_format = RMNET_INGRESS_FORMAT_DEAGGREGATION;
 	struct net_device *real_dev;
 	int mode = RMNET_EPMODE_VND;
 	struct rmnet_endpoint *ep;
@@ -177,21 +161,22 @@ static int rmnet_newlink(struct net *src_net, struct net_device *dev,
 	if (err)
 		goto err1;
 
-	err = netdev_master_upper_dev_link(dev, real_dev, NULL, NULL, extack);
-	if (err)
-		goto err2;
-
-	netdev_dbg(dev, "data format [ingress 0x%08X] [egress 0x%08X]\n",
-		   ingress_format, egress_format);
-	port->egress_data_format = egress_format;
-	port->ingress_data_format = ingress_format;
 	port->rmnet_mode = mode;
 
 	hlist_add_head_rcu(&ep->hlnode, &port->muxed_ep[mux_id]);
+
+	if (data[IFLA_VLAN_FLAGS]) {
+		struct ifla_vlan_flags *flags;
+
+		flags = nla_data(data[IFLA_VLAN_FLAGS]);
+		data_format = flags->flags & flags->mask;
+	}
+
+	netdev_dbg(dev, "data format [0x%08X]\n", data_format);
+	port->data_format = data_format;
+
 	return 0;
 
-err2:
-	rmnet_vnd_dellink(mux_id, port, ep);
 err1:
 	rmnet_unregister_real_device(real_dev, port);
 err0:
@@ -201,14 +186,13 @@ static int rmnet_newlink(struct net *src_net, struct net_device *dev,
 
 static void rmnet_dellink(struct net_device *dev, struct list_head *head)
 {
+	struct rmnet_priv *priv = netdev_priv(dev);
 	struct net_device *real_dev;
 	struct rmnet_endpoint *ep;
 	struct rmnet_port *port;
 	u8 mux_id;
 
-	rcu_read_lock();
-	real_dev = netdev_master_upper_dev_get_rcu(dev);
-	rcu_read_unlock();
+	real_dev = priv->real_dev;
 
 	if (!real_dev || !rmnet_is_real_dev_registered(real_dev))
 		return;
@@ -216,7 +200,6 @@ static void rmnet_dellink(struct net_device *dev, struct list_head *head)
 	port = rmnet_get_port_rtnl(real_dev);
 
 	mux_id = rmnet_vnd_get_mux(dev);
-	netdev_upper_dev_unlink(dev, real_dev);
 
 	ep = rmnet_get_endpoint(port, mux_id);
 	if (ep) {
@@ -230,30 +213,13 @@ static void rmnet_dellink(struct net_device *dev, struct list_head *head)
 	unregister_netdevice_queue(dev, head);
 }
 
-static int rmnet_dev_walk_unreg(struct net_device *rmnet_dev, void *data)
-{
-	struct rmnet_walk_data *d = data;
-	struct rmnet_endpoint *ep;
-	u8 mux_id;
-
-	mux_id = rmnet_vnd_get_mux(rmnet_dev);
-	ep = rmnet_get_endpoint(d->port, mux_id);
-	if (ep) {
-		hlist_del_init_rcu(&ep->hlnode);
-		rmnet_vnd_dellink(mux_id, d->port, ep);
-		kfree(ep);
-	}
-	netdev_upper_dev_unlink(rmnet_dev, d->real_dev);
-	unregister_netdevice_queue(rmnet_dev, d->head);
-
-	return 0;
-}
-
 static void rmnet_force_unassociate_device(struct net_device *dev)
 {
 	struct net_device *real_dev = dev;
-	struct rmnet_walk_data d;
+	struct hlist_node *tmp_ep;
+	struct rmnet_endpoint *ep;
 	struct rmnet_port *port;
+	unsigned long bkt_ep;
 	LIST_HEAD(list);
 
 	if (!rmnet_is_real_dev_registered(real_dev))
@@ -261,16 +227,19 @@ static void rmnet_force_unassociate_device(struct net_device *dev)
 
 	ASSERT_RTNL();
 
-	d.real_dev = real_dev;
-	d.head = &list;
-
 	port = rmnet_get_port_rtnl(dev);
-	d.port = port;
 
 	rcu_read_lock();
 	rmnet_unregister_bridge(dev, port);
 
-	netdev_walk_all_lower_dev_rcu(real_dev, rmnet_dev_walk_unreg, &d);
+	hash_for_each_safe(port->muxed_ep, bkt_ep, tmp_ep, ep, hlnode) {
+		unregister_netdevice_queue(ep->egress_dev, &list);
+		rmnet_vnd_dellink(ep->mux_id, port, ep);
+
+		hlist_del_init_rcu(&ep->hlnode);
+		kfree(ep);
+	}
+
 	rcu_read_unlock();
 	unregister_netdevice_many(&list);
 
@@ -317,9 +286,49 @@ static int rmnet_rtnl_validate(struct nlattr *tb[], struct nlattr *data[],
 	return 0;
 }
 
+static int rmnet_changelink(struct net_device *dev, struct nlattr *tb[],
+			    struct nlattr *data[],
+			    struct netlink_ext_ack *extack)
+{
+	struct rmnet_priv *priv = netdev_priv(dev);
+	struct net_device *real_dev;
+	struct rmnet_endpoint *ep;
+	struct rmnet_port *port;
+	u16 mux_id;
+
+	real_dev = __dev_get_by_index(dev_net(dev),
+				      nla_get_u32(tb[IFLA_LINK]));
+
+	if (!real_dev || !dev || !rmnet_is_real_dev_registered(real_dev))
+		return -ENODEV;
+
+	port = rmnet_get_port_rtnl(real_dev);
+
+	if (data[IFLA_VLAN_ID]) {
+		mux_id = nla_get_u16(data[IFLA_VLAN_ID]);
+		ep = rmnet_get_endpoint(port, priv->mux_id);
+
+		hlist_del_init_rcu(&ep->hlnode);
+		hlist_add_head_rcu(&ep->hlnode, &port->muxed_ep[mux_id]);
+
+		ep->mux_id = mux_id;
+		priv->mux_id = mux_id;
+	}
+
+	if (data[IFLA_VLAN_FLAGS]) {
+		struct ifla_vlan_flags *flags;
+
+		flags = nla_data(data[IFLA_VLAN_FLAGS]);
+		port->data_format = flags->flags & flags->mask;
+	}
+
+	return 0;
+}
+
 static size_t rmnet_get_size(const struct net_device *dev)
 {
-	return nla_total_size(2); /* IFLA_VLAN_ID */
+	return nla_total_size(2) /* IFLA_VLAN_ID */ +
+	       nla_total_size(sizeof(struct ifla_vlan_flags)); /* IFLA_VLAN_FLAGS */
 }
 
 struct rtnl_link_ops rmnet_link_ops __read_mostly = {
@@ -331,6 +340,7 @@ struct rtnl_link_ops rmnet_link_ops __read_mostly = {
 	.newlink	= rmnet_newlink,
 	.dellink	= rmnet_dellink,
 	.get_size	= rmnet_get_size,
+	.changelink     = rmnet_changelink,
 };
 
 /* Needs either rcu_read_lock() or rtnl lock */
@@ -378,11 +388,6 @@ int rmnet_add_bridge(struct net_device *rmnet_dev,
 	if (err)
 		return -EBUSY;
 
-	err = netdev_master_upper_dev_link(slave_dev, rmnet_dev, NULL, NULL,
-					   extack);
-	if (err)
-		return -EINVAL;
-
 	slave_port = rmnet_get_port(slave_dev);
 	slave_port->rmnet_mode = RMNET_EPMODE_BRIDGE;
 	slave_port->bridge_ep = real_dev;
@@ -405,7 +410,6 @@ int rmnet_del_bridge(struct net_device *rmnet_dev,
 	port->rmnet_mode = RMNET_EPMODE_VND;
 	port->bridge_ep = NULL;
 
-	netdev_upper_dev_unlink(slave_dev, rmnet_dev);
 	slave_port = rmnet_get_port(slave_dev);
 	rmnet_unregister_real_device(slave_dev, slave_port);
 
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
index c19259eea99e..00e4634100d3 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
@@ -32,8 +32,7 @@ struct rmnet_endpoint {
  */
 struct rmnet_port {
 	struct net_device *dev;
-	u32 ingress_data_format;
-	u32 egress_data_format;
+	u32 data_format;
 	u8 nr_rmnet_devs;
 	u8 rmnet_mode;
 	struct hlist_head muxed_ep[RMNET_MAX_LOGICAL_EP];
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
index 08e4afc0ab39..601edec28c5f 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.c
@@ -15,6 +15,8 @@
 
 #include <linux/netdevice.h>
 #include <linux/netdev_features.h>
+#include <linux/if_arp.h>
+#include <net/sock.h>
 #include "rmnet_private.h"
 #include "rmnet_config.h"
 #include "rmnet_vnd.h"
@@ -64,19 +66,19 @@ __rmnet_map_ingress_handler(struct sk_buff *skb,
 			    struct rmnet_port *port)
 {
 	struct rmnet_endpoint *ep;
+	u16 len, pad;
 	u8 mux_id;
-	u16 len;
 
 	if (RMNET_MAP_GET_CD_BIT(skb)) {
-		if (port->ingress_data_format
-		    & RMNET_INGRESS_FORMAT_MAP_COMMANDS)
+		if (port->data_format & RMNET_INGRESS_FORMAT_MAP_COMMANDS)
 			return rmnet_map_command(skb, port);
 
 		goto free_skb;
 	}
 
 	mux_id = RMNET_MAP_GET_MUX_ID(skb);
-	len = RMNET_MAP_GET_LENGTH(skb) - RMNET_MAP_GET_PAD(skb);
+	pad = RMNET_MAP_GET_PAD(skb);
+	len = RMNET_MAP_GET_LENGTH(skb) - pad;
 
 	if (mux_id >= RMNET_MAX_LOGICAL_EP)
 		goto free_skb;
@@ -89,8 +91,14 @@ __rmnet_map_ingress_handler(struct sk_buff *skb,
 
 	/* Subtract MAP header */
 	skb_pull(skb, sizeof(struct rmnet_map_header));
-	skb_trim(skb, len);
 	rmnet_set_skb_proto(skb);
+
+	if (port->data_format & RMNET_INGRESS_FORMAT_MAP_CKSUMV4) {
+		if (!rmnet_map_checksum_downlink_packet(skb, len + pad))
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+	}
+
+	skb_trim(skb, len);
 	rmnet_deliver_skb(skb);
 	return;
 
@@ -104,8 +112,17 @@ rmnet_map_ingress_handler(struct sk_buff *skb,
 {
 	struct sk_buff *skbn;
 
-	if (port->ingress_data_format & RMNET_INGRESS_FORMAT_DEAGGREGATION) {
-		while ((skbn = rmnet_map_deaggregate(skb)) != NULL)
+	if (skb->dev->type == ARPHRD_ETHER) {
+		if (pskb_expand_head(skb, ETH_HLEN, 0, GFP_KERNEL)) {
+			kfree_skb(skb);
+			return;
+		}
+
+		skb_push(skb, ETH_HLEN);
+	}
+
+	if (port->data_format & RMNET_INGRESS_FORMAT_DEAGGREGATION) {
+		while ((skbn = rmnet_map_deaggregate(skb, port)) != NULL)
 			__rmnet_map_ingress_handler(skbn, port);
 
 		consume_skb(skb);
@@ -124,29 +141,32 @@ static int rmnet_map_egress_handler(struct sk_buff *skb,
 	additional_header_len = 0;
 	required_headroom = sizeof(struct rmnet_map_header);
 
+	if (port->data_format & RMNET_EGRESS_FORMAT_MAP_CKSUMV4) {
+		additional_header_len = sizeof(struct rmnet_map_ul_csum_header);
+		required_headroom += additional_header_len;
+	}
+
 	if (skb_headroom(skb) < required_headroom) {
 		if (pskb_expand_head(skb, required_headroom, 0, GFP_KERNEL))
 			goto fail;
 	}
 
+	if (port->data_format & RMNET_EGRESS_FORMAT_MAP_CKSUMV4)
+		rmnet_map_checksum_uplink_packet(skb, orig_dev);
+
 	map_header = rmnet_map_add_map_header(skb, additional_header_len, 0);
 	if (!map_header)
 		goto fail;
 
-	if (port->egress_data_format & RMNET_EGRESS_FORMAT_MUXING) {
-		if (mux_id == 0xff)
-			map_header->mux_id = 0;
-		else
-			map_header->mux_id = mux_id;
-	}
+	map_header->mux_id = mux_id;
 
 	skb->protocol = htons(ETH_P_MAP);
 
-	return RMNET_MAP_SUCCESS;
+	return 0;
 
 fail:
 	kfree_skb(skb);
-	return RMNET_MAP_CONSUMED;
+	return -ENOMEM;
 }
 
 static void
@@ -178,8 +198,7 @@ rx_handler_result_t rmnet_rx_handler(struct sk_buff **pskb)
 
 	switch (port->rmnet_mode) {
 	case RMNET_EPMODE_VND:
-		if (port->ingress_data_format & RMNET_INGRESS_FORMAT_MAP)
-			rmnet_map_ingress_handler(skb, port);
+		rmnet_map_ingress_handler(skb, port);
 		break;
 	case RMNET_EPMODE_BRIDGE:
 		rmnet_bridge_handler(skb, port->bridge_ep);
@@ -201,6 +220,8 @@ void rmnet_egress_handler(struct sk_buff *skb)
 	struct rmnet_priv *priv;
 	u8 mux_id;
 
+	sk_pacing_shift_update(skb->sk, 8);
+
 	orig_dev = skb->dev;
 	priv = netdev_priv(orig_dev);
 	skb->dev = priv->real_dev;
@@ -212,19 +233,8 @@ void rmnet_egress_handler(struct sk_buff *skb)
 		return;
 	}
 
-	if (port->egress_data_format & RMNET_EGRESS_FORMAT_MAP) {
-		switch (rmnet_map_egress_handler(skb, port, mux_id, orig_dev)) {
-		case RMNET_MAP_CONSUMED:
-			return;
-
-		case RMNET_MAP_SUCCESS:
-			break;
-
-		default:
-			kfree_skb(skb);
-			return;
-		}
-	}
+	if (rmnet_map_egress_handler(skb, port, mux_id, orig_dev))
+		return;
 
 	rmnet_vnd_tx_fixup(skb, orig_dev);
 
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
index 3af3fe7b5457..6ce31e29136d 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
@@ -30,15 +30,6 @@ struct rmnet_map_control_command {
 	};
 }  __aligned(1);
 
-enum rmnet_map_results {
-	RMNET_MAP_SUCCESS,
-	RMNET_MAP_CONSUMED,
-	RMNET_MAP_GENERAL_FAILURE,
-	RMNET_MAP_NOT_ENABLED,
-	RMNET_MAP_FAILED_AGGREGATION,
-	RMNET_MAP_FAILED_MUX
-};
-
 enum rmnet_map_commands {
 	RMNET_MAP_COMMAND_NONE,
 	RMNET_MAP_COMMAND_FLOW_DISABLE,
@@ -56,6 +47,22 @@ struct rmnet_map_header {
 	u16 pkt_len;
 }  __aligned(1);
 
+struct rmnet_map_dl_csum_trailer {
+	u8  reserved1;
+	u8  valid:1;
+	u8  reserved2:7;
+	u16 csum_start_offset;
+	u16 csum_length;
+	__be16 csum_value;
+} __aligned(1);
+
+struct rmnet_map_ul_csum_header {
+	__be16 csum_start_offset;
+	u16 csum_insert_offset:14;
+	u16 udp_ip4_ind:1;
+	u16 csum_enabled:1;
+} __aligned(1);
+
 #define RMNET_MAP_GET_MUX_ID(Y) (((struct rmnet_map_header *) \
 				 (Y)->data)->mux_id)
 #define RMNET_MAP_GET_CD_BIT(Y) (((struct rmnet_map_header *) \
@@ -76,10 +83,13 @@ struct rmnet_map_header {
 #define RMNET_MAP_NO_PAD_BYTES        0
 #define RMNET_MAP_ADD_PAD_BYTES       1
 
-u8 rmnet_map_demultiplex(struct sk_buff *skb);
-struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb);
+struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
+				      struct rmnet_port *port);
 struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
 						  int hdrlen, int pad);
 void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port);
+int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len);
+void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
+				      struct net_device *orig_dev);
 
 #endif /* _RMNET_MAP_H_ */
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
index 51e604923ac1..b0dbca070c00 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_command.c
@@ -38,6 +38,11 @@ static u8 rmnet_map_do_flow_control(struct sk_buff *skb,
 	}
 
 	ep = rmnet_get_endpoint(port, mux_id);
+	if (!ep) {
+		kfree_skb(skb);
+		return RX_HANDLER_CONSUMED;
+	}
+
 	vnd = ep->egress_dev;
 
 	ip_family = cmd->flow_control.ip_family;
@@ -58,11 +63,24 @@ static u8 rmnet_map_do_flow_control(struct sk_buff *skb,
 }
 
 static void rmnet_map_send_ack(struct sk_buff *skb,
-			       unsigned char type)
+			       unsigned char type,
+			       struct rmnet_port *port)
 {
 	struct rmnet_map_control_command *cmd;
 	int xmit_status;
 
+	if (port->data_format & RMNET_INGRESS_FORMAT_MAP_CKSUMV4) {
+		if (skb->len < sizeof(struct rmnet_map_header) +
+		    RMNET_MAP_GET_LENGTH(skb) +
+		    sizeof(struct rmnet_map_dl_csum_trailer)) {
+			kfree_skb(skb);
+			return;
+		}
+
+		skb_trim(skb, skb->len -
+			 sizeof(struct rmnet_map_dl_csum_trailer));
+	}
+
 	skb->protocol = htons(ETH_P_MAP);
 
 	cmd = RMNET_MAP_GET_CMD_START(skb);
@@ -100,5 +118,5 @@ void rmnet_map_command(struct sk_buff *skb, struct rmnet_port *port)
 		break;
 	}
 	if (rc == RMNET_MAP_COMMAND_ACK)
-		rmnet_map_send_ack(skb, rc);
+		rmnet_map_send_ack(skb, rc, port);
 }
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
index 86b8c758f94e..c74a6c56d315 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c
@@ -14,6 +14,9 @@
  */
 
 #include <linux/netdevice.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <net/ip6_checksum.h>
 #include "rmnet_config.h"
 #include "rmnet_map.h"
 #include "rmnet_private.h"
@@ -21,6 +24,233 @@
 #define RMNET_MAP_DEAGGR_SPACING  64
 #define RMNET_MAP_DEAGGR_HEADROOM (RMNET_MAP_DEAGGR_SPACING / 2)
 
+static __sum16 *rmnet_map_get_csum_field(unsigned char protocol,
+					 const void *txporthdr)
+{
+	__sum16 *check = NULL;
+
+	switch (protocol) {
+	case IPPROTO_TCP:
+		check = &(((struct tcphdr *)txporthdr)->check);
+		break;
+
+	case IPPROTO_UDP:
+		check = &(((struct udphdr *)txporthdr)->check);
+		break;
+
+	default:
+		check = NULL;
+		break;
+	}
+
+	return check;
+}
+
+static int
+rmnet_map_ipv4_dl_csum_trailer(struct sk_buff *skb,
+			       struct rmnet_map_dl_csum_trailer *csum_trailer)
+{
+	__sum16 *csum_field, csum_temp, pseudo_csum, hdr_csum, ip_payload_csum;
+	u16 csum_value, csum_value_final;
+	struct iphdr *ip4h;
+	void *txporthdr;
+	__be16 addend;
+
+	ip4h = (struct iphdr *)(skb->data);
+	if ((ntohs(ip4h->frag_off) & IP_MF) ||
+	    ((ntohs(ip4h->frag_off) & IP_OFFSET) > 0))
+		return -EOPNOTSUPP;
+
+	txporthdr = skb->data + ip4h->ihl * 4;
+
+	csum_field = rmnet_map_get_csum_field(ip4h->protocol, txporthdr);
+
+	if (!csum_field)
+		return -EPROTONOSUPPORT;
+
+	/* RFC 768 - Skip IPv4 UDP packets where sender checksum field is 0 */
+	if (*csum_field == 0 && ip4h->protocol == IPPROTO_UDP)
+		return 0;
+
+	csum_value = ~ntohs(csum_trailer->csum_value);
+	hdr_csum = ~ip_fast_csum(ip4h, (int)ip4h->ihl);
+	ip_payload_csum = csum16_sub((__force __sum16)csum_value,
+				     (__force __be16)hdr_csum);
+
+	pseudo_csum = ~csum_tcpudp_magic(ip4h->saddr, ip4h->daddr,
+					 ntohs(ip4h->tot_len) - ip4h->ihl * 4,
+					 ip4h->protocol, 0);
+	addend = (__force __be16)ntohs((__force __be16)pseudo_csum);
+	pseudo_csum = csum16_add(ip_payload_csum, addend);
+
+	addend = (__force __be16)ntohs((__force __be16)*csum_field);
+	csum_temp = ~csum16_sub(pseudo_csum, addend);
+	csum_value_final = (__force u16)csum_temp;
+
+	if (unlikely(csum_value_final == 0)) {
+		switch (ip4h->protocol) {
+		case IPPROTO_UDP:
+			/* RFC 768 - DL4 1's complement rule for UDP csum 0 */
+			csum_value_final = ~csum_value_final;
+			break;
+
+		case IPPROTO_TCP:
+			/* DL4 Non-RFC compliant TCP checksum found */
+			if (*csum_field == (__force __sum16)0xFFFF)
+				csum_value_final = ~csum_value_final;
+			break;
+		}
+	}
+
+	if (csum_value_final == ntohs((__force __be16)*csum_field))
+		return 0;
+	else
+		return -EINVAL;
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static int
+rmnet_map_ipv6_dl_csum_trailer(struct sk_buff *skb,
+			       struct rmnet_map_dl_csum_trailer *csum_trailer)
+{
+	__sum16 *csum_field, ip6_payload_csum, pseudo_csum, csum_temp;
+	u16 csum_value, csum_value_final;
+	__be16 ip6_hdr_csum, addend;
+	struct ipv6hdr *ip6h;
+	void *txporthdr;
+	u32 length;
+
+	ip6h = (struct ipv6hdr *)(skb->data);
+
+	txporthdr = skb->data + sizeof(struct ipv6hdr);
+	csum_field = rmnet_map_get_csum_field(ip6h->nexthdr, txporthdr);
+
+	if (!csum_field)
+		return -EPROTONOSUPPORT;
+
+	csum_value = ~ntohs(csum_trailer->csum_value);
+	ip6_hdr_csum = (__force __be16)
+			~ntohs((__force __be16)ip_compute_csum(ip6h,
+			       (int)(txporthdr - (void *)(skb->data))));
+	ip6_payload_csum = csum16_sub((__force __sum16)csum_value,
+				      ip6_hdr_csum);
+
+	length = (ip6h->nexthdr == IPPROTO_UDP) ?
+		 ntohs(((struct udphdr *)txporthdr)->len) :
+		 ntohs(ip6h->payload_len);
+	pseudo_csum = ~(csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
+			     length, ip6h->nexthdr, 0));
+	addend = (__force __be16)ntohs((__force __be16)pseudo_csum);
+	pseudo_csum = csum16_add(ip6_payload_csum, addend);
+
+	addend = (__force __be16)ntohs((__force __be16)*csum_field);
+	csum_temp = ~csum16_sub(pseudo_csum, addend);
+	csum_value_final = (__force u16)csum_temp;
+
+	if (unlikely(csum_value_final == 0)) {
+		switch (ip6h->nexthdr) {
+		case IPPROTO_UDP:
+			/* RFC 2460 section 8.1
+			 * DL6 One's complement rule for UDP checksum 0
+			 */
+			csum_value_final = ~csum_value_final;
+			break;
+
+		case IPPROTO_TCP:
+			/* DL6 Non-RFC compliant TCP checksum found */
+			if (*csum_field == (__force __sum16)0xFFFF)
+				csum_value_final = ~csum_value_final;
+			break;
+		}
+	}
+
+	if (csum_value_final == ntohs((__force __be16)*csum_field))
+		return 0;
+	else
+		return -EINVAL;
+}
+#endif
+
+static void rmnet_map_complement_ipv4_txporthdr_csum_field(void *iphdr)
+{
+	struct iphdr *ip4h = (struct iphdr *)iphdr;
+	void *txphdr;
+	u16 *csum;
+
+	txphdr = iphdr + ip4h->ihl * 4;
+
+	if (ip4h->protocol == IPPROTO_TCP || ip4h->protocol == IPPROTO_UDP) {
+		csum = (u16 *)rmnet_map_get_csum_field(ip4h->protocol, txphdr);
+		*csum = ~(*csum);
+	}
+}
+
+static void
+rmnet_map_ipv4_ul_csum_header(void *iphdr,
+			      struct rmnet_map_ul_csum_header *ul_header,
+			      struct sk_buff *skb)
+{
+	struct iphdr *ip4h = (struct iphdr *)iphdr;
+	__be16 *hdr = (__be16 *)ul_header, offset;
+
+	offset = htons((__force u16)(skb_transport_header(skb) -
+				     (unsigned char *)iphdr));
+	ul_header->csum_start_offset = offset;
+	ul_header->csum_insert_offset = skb->csum_offset;
+	ul_header->csum_enabled = 1;
+	if (ip4h->protocol == IPPROTO_UDP)
+		ul_header->udp_ip4_ind = 1;
+	else
+		ul_header->udp_ip4_ind = 0;
+
+	/* Changing remaining fields to network order */
+	hdr++;
+	*hdr = htons((__force u16)*hdr);
+
+	skb->ip_summed = CHECKSUM_NONE;
+
+	rmnet_map_complement_ipv4_txporthdr_csum_field(iphdr);
+}
+
+#if IS_ENABLED(CONFIG_IPV6)
+static void rmnet_map_complement_ipv6_txporthdr_csum_field(void *ip6hdr)
+{
+	struct ipv6hdr *ip6h = (struct ipv6hdr *)ip6hdr;
+	void *txphdr;
+	u16 *csum;
+
+	txphdr = ip6hdr + sizeof(struct ipv6hdr);
+
+	if (ip6h->nexthdr == IPPROTO_TCP || ip6h->nexthdr == IPPROTO_UDP) {
+		csum = (u16 *)rmnet_map_get_csum_field(ip6h->nexthdr, txphdr);
+		*csum = ~(*csum);
+	}
+}
+
+static void
+rmnet_map_ipv6_ul_csum_header(void *ip6hdr,
+			      struct rmnet_map_ul_csum_header *ul_header,
+			      struct sk_buff *skb)
+{
+	__be16 *hdr = (__be16 *)ul_header, offset;
+
+	offset = htons((__force u16)(skb_transport_header(skb) -
+				     (unsigned char *)ip6hdr));
+	ul_header->csum_start_offset = offset;
+	ul_header->csum_insert_offset = skb->csum_offset;
+	ul_header->csum_enabled = 1;
+	ul_header->udp_ip4_ind = 0;
+
+	/* Changing remaining fields to network order */
+	hdr++;
+	*hdr = htons((__force u16)*hdr);
+
+	skb->ip_summed = CHECKSUM_NONE;
+
+	rmnet_map_complement_ipv6_txporthdr_csum_field(ip6hdr);
+}
+#endif
+
 /* Adds MAP header to front of skb->data
  * Padding is calculated and set appropriately in MAP header. Mux ID is
  * initialized to 0.
@@ -32,9 +262,6 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
 	u32 padding, map_datalen;
 	u8 *padbytes;
 
-	if (skb_headroom(skb) < sizeof(struct rmnet_map_header))
-		return NULL;
-
 	map_datalen = skb->len - hdrlen;
 	map_header = (struct rmnet_map_header *)
 			skb_push(skb, sizeof(struct rmnet_map_header));
@@ -69,7 +296,8 @@ struct rmnet_map_header *rmnet_map_add_map_header(struct sk_buff *skb,
  * returned, indicating that there are no more packets to deaggregate. Caller
  * is responsible for freeing the original skb.
  */
-struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb)
+struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb,
+				      struct rmnet_port *port)
 {
 	struct rmnet_map_header *maph;
 	struct sk_buff *skbn;
@@ -81,6 +309,9 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb)
 	maph = (struct rmnet_map_header *)skb->data;
 	packet_len = ntohs(maph->pkt_len) + sizeof(struct rmnet_map_header);
 
+	if (port->data_format & RMNET_INGRESS_FORMAT_MAP_CKSUMV4)
+		packet_len += sizeof(struct rmnet_map_dl_csum_trailer);
+
 	if (((int)skb->len - (int)packet_len) < 0)
 		return NULL;
 
@@ -100,3 +331,73 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb)
 
 	return skbn;
 }
+
+/* Validates packet checksums. Function takes a pointer to
+ * the beginning of a buffer which contains the IP payload +
+ * padding + checksum trailer.
+ * Only IPv4 and IPv6 are supported along with TCP & UDP.
+ * Fragmented or tunneled packets are not supported.
+ */
+int rmnet_map_checksum_downlink_packet(struct sk_buff *skb, u16 len)
+{
+	struct rmnet_map_dl_csum_trailer *csum_trailer;
+
+	if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
+		return -EOPNOTSUPP;
+
+	csum_trailer = (struct rmnet_map_dl_csum_trailer *)(skb->data + len);
+
+	if (!csum_trailer->valid)
+		return -EINVAL;
+
+	if (skb->protocol == htons(ETH_P_IP))
+		return rmnet_map_ipv4_dl_csum_trailer(skb, csum_trailer);
+	else if (skb->protocol == htons(ETH_P_IPV6))
+#if IS_ENABLED(CONFIG_IPV6)
+		return rmnet_map_ipv6_dl_csum_trailer(skb, csum_trailer);
+#else
+		return -EPROTONOSUPPORT;
+#endif
+
+	return 0;
+}
+
+/* Generates UL checksum meta info header for IPv4 and IPv6 over TCP and UDP
+ * packets that are supported for UL checksum offload.
+ */
+void rmnet_map_checksum_uplink_packet(struct sk_buff *skb,
+				      struct net_device *orig_dev)
+{
+	struct rmnet_map_ul_csum_header *ul_header;
+	void *iphdr;
+
+	ul_header = (struct rmnet_map_ul_csum_header *)
+		    skb_push(skb, sizeof(struct rmnet_map_ul_csum_header));
+
+	if (unlikely(!(orig_dev->features &
+		     (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))))
+		goto sw_csum;
+
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
+		iphdr = (char *)ul_header +
+			sizeof(struct rmnet_map_ul_csum_header);
+
+		if (skb->protocol == htons(ETH_P_IP)) {
+			rmnet_map_ipv4_ul_csum_header(iphdr, ul_header, skb);
+			return;
+		} else if (skb->protocol == htons(ETH_P_IPV6)) {
+#if IS_ENABLED(CONFIG_IPV6)
+			rmnet_map_ipv6_ul_csum_header(iphdr, ul_header, skb);
+			return;
+#else
+			goto sw_csum;
+#endif
+		}
+	}
+
+sw_csum:
+	ul_header->csum_start_offset = 0;
+	ul_header->csum_insert_offset = 0;
+	ul_header->csum_enabled = 0;
+	ul_header->udp_ip4_ind = 0;
+}
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
index 49102f922b31..de0143eaa05a 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h
@@ -19,14 +19,10 @@
 #define RMNET_TX_QUEUE_LEN         1000
 
 /* Constants */
-#define RMNET_EGRESS_FORMAT_MAP                 BIT(1)
-#define RMNET_EGRESS_FORMAT_AGGREGATION         BIT(2)
-#define RMNET_EGRESS_FORMAT_MUXING              BIT(3)
-
-#define RMNET_INGRESS_FORMAT_MAP                BIT(1)
-#define RMNET_INGRESS_FORMAT_DEAGGREGATION      BIT(2)
-#define RMNET_INGRESS_FORMAT_DEMUXING           BIT(3)
-#define RMNET_INGRESS_FORMAT_MAP_COMMANDS       BIT(4)
+#define RMNET_INGRESS_FORMAT_DEAGGREGATION      BIT(0)
+#define RMNET_INGRESS_FORMAT_MAP_COMMANDS       BIT(1)
+#define RMNET_INGRESS_FORMAT_MAP_CKSUMV4        BIT(2)
+#define RMNET_EGRESS_FORMAT_MAP_CKSUMV4         BIT(3)
 
 /* Replace skb->dev to a virtual rmnet device and pass up the stack */
 #define RMNET_EPMODE_VND (1)
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
index 9caa5e387450..346d310914df 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_vnd.c
@@ -121,7 +121,7 @@ static void rmnet_get_stats64(struct net_device *dev,
 	memset(&total_stats, 0, sizeof(struct rmnet_vnd_stats));
 
 	for_each_possible_cpu(cpu) {
-		pcpu_ptr = this_cpu_ptr(priv->pcpu_stats);
+		pcpu_ptr = per_cpu_ptr(priv->pcpu_stats, cpu);
 
 		do {
 			start = u64_stats_fetch_begin_irq(&pcpu_ptr->syncp);
@@ -185,6 +185,13 @@ int rmnet_vnd_newlink(u8 id, struct net_device *rmnet_dev,
 	if (ep->egress_dev)
 		return -EINVAL;
 
+	if (rmnet_get_endpoint(port, id))
+		return -EBUSY;
+
+	rmnet_dev->hw_features = NETIF_F_RXCSUM;
+	rmnet_dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
+	rmnet_dev->hw_features |= NETIF_F_SG;
+
 	rc = register_netdevice(rmnet_dev);
 	if (!rc) {
 		ep->egress_dev = rmnet_dev;
diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c
index e7ab23e87de2..81045dfa1cd8 100644
--- a/drivers/net/ethernet/realtek/8139cp.c
+++ b/drivers/net/ethernet/realtek/8139cp.c
@@ -748,8 +748,8 @@ static netdev_tx_t cp_start_xmit (struct sk_buff *skb,
 	mss = skb_shinfo(skb)->gso_size;
 
 	if (mss > MSSMask) {
-		WARN_ONCE(1, "Net bug: GSO size %d too large for 8139CP\n",
-			  mss);
+		netdev_WARN_ONCE(dev, "Net bug: GSO size %d too large for 8139CP\n",
+				 mss);
 		goto out_dma_error;
 	}
 
diff --git a/drivers/net/ethernet/realtek/r8169.c b/drivers/net/ethernet/realtek/r8169.c
index 734286ebe5ef..b4779acb6b5c 100644
--- a/drivers/net/ethernet/realtek/r8169.c
+++ b/drivers/net/ethernet/realtek/r8169.c
@@ -1395,7 +1395,7 @@ DECLARE_RTL_COND(rtl_ocp_tx_cond)
 {
 	void __iomem *ioaddr = tp->mmio_addr;
 
-	return RTL_R8(IBISR0) & 0x02;
+	return RTL_R8(IBISR0) & 0x20;
 }
 
 static void rtl8168ep_stop_cmac(struct rtl8169_private *tp)
@@ -1403,7 +1403,7 @@ static void rtl8168ep_stop_cmac(struct rtl8169_private *tp)
 	void __iomem *ioaddr = tp->mmio_addr;
 
 	RTL_W8(IBCR2, RTL_R8(IBCR2) & ~0x01);
-	rtl_msleep_loop_wait_low(tp, &rtl_ocp_tx_cond, 50, 2000);
+	rtl_msleep_loop_wait_high(tp, &rtl_ocp_tx_cond, 50, 2000);
 	RTL_W8(IBISR0, RTL_R8(IBISR0) | 0x20);
 	RTL_W8(IBCR0, RTL_R8(IBCR0) & ~0x01);
 }
@@ -1675,33 +1675,24 @@ static void rtl_link_chg_patch(struct rtl8169_private *tp)
 	}
 }
 
-static void __rtl8169_check_link_status(struct net_device *dev,
-					struct rtl8169_private *tp,
-					void __iomem *ioaddr, bool pm)
+static void rtl8169_check_link_status(struct net_device *dev,
+				      struct rtl8169_private *tp,
+				      void __iomem *ioaddr)
 {
 	if (tp->link_ok(ioaddr)) {
 		rtl_link_chg_patch(tp);
 		/* This is to cancel a scheduled suspend if there's one. */
-		if (pm)
-			pm_request_resume(&tp->pci_dev->dev);
+		pm_request_resume(&tp->pci_dev->dev);
 		netif_carrier_on(dev);
 		if (net_ratelimit())
 			netif_info(tp, ifup, dev, "link up\n");
 	} else {
 		netif_carrier_off(dev);
 		netif_info(tp, ifdown, dev, "link down\n");
-		if (pm)
-			pm_schedule_suspend(&tp->pci_dev->dev, 5000);
+		pm_runtime_idle(&tp->pci_dev->dev);
 	}
 }
 
-static void rtl8169_check_link_status(struct net_device *dev,
-				      struct rtl8169_private *tp,
-				      void __iomem *ioaddr)
-{
-	__rtl8169_check_link_status(dev, tp, ioaddr, false);
-}
-
 #define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST)
 
 static u32 __rtl8169_get_wol(struct rtl8169_private *tp)
@@ -4638,16 +4629,6 @@ static void rtl8169_phy_timer(struct timer_list *t)
 	rtl_schedule_task(tp, RTL_FLAG_TASK_PHY_PENDING);
 }
 
-static void rtl8169_release_board(struct pci_dev *pdev, struct net_device *dev,
-				  void __iomem *ioaddr)
-{
-	iounmap(ioaddr);
-	pci_release_regions(pdev);
-	pci_clear_mwi(pdev);
-	pci_disable_device(pdev);
-	free_netdev(dev);
-}
-
 DECLARE_RTL_COND(rtl_phy_reset_cond)
 {
 	return tp->phy_reset_pending(tp);
@@ -4779,14 +4760,6 @@ static int rtl_tbi_ioctl(struct rtl8169_private *tp, struct mii_ioctl_data *data
 	return -EOPNOTSUPP;
 }
 
-static void rtl_disable_msi(struct pci_dev *pdev, struct rtl8169_private *tp)
-{
-	if (tp->features & RTL_FEATURE_MSI) {
-		pci_disable_msi(pdev);
-		tp->features &= ~RTL_FEATURE_MSI;
-	}
-}
-
 static void rtl_init_mdio_ops(struct rtl8169_private *tp)
 {
 	struct mdio_ops *ops = &tp->mdio_ops;
@@ -7759,7 +7732,7 @@ static void rtl_slow_event_work(struct rtl8169_private *tp)
 		rtl8169_pcierr_interrupt(dev);
 
 	if (status & LinkChg)
-		__rtl8169_check_link_status(dev, tp, tp->mmio_addr, true);
+		rtl8169_check_link_status(dev, tp, tp->mmio_addr);
 
 	rtl_irq_enable_all(tp);
 }
@@ -7972,7 +7945,7 @@ static int rtl_open(struct net_device *dev)
 	rtl_unlock_work(tp);
 
 	tp->saved_wolopts = 0;
-	pm_runtime_put_noidle(&pdev->dev);
+	pm_runtime_put_sync(&pdev->dev);
 
 	rtl8169_check_link_status(dev, tp, ioaddr);
 out:
@@ -8116,8 +8089,10 @@ static int rtl8169_runtime_suspend(struct device *device)
 	struct net_device *dev = pci_get_drvdata(pdev);
 	struct rtl8169_private *tp = netdev_priv(dev);
 
-	if (!tp->TxDescArray)
+	if (!tp->TxDescArray) {
+		rtl_pll_power_down(tp);
 		return 0;
+	}
 
 	rtl_lock_work(tp);
 	tp->saved_wolopts = __rtl8169_get_wol(tp);
@@ -8159,9 +8134,11 @@ static int rtl8169_runtime_idle(struct device *device)
 {
 	struct pci_dev *pdev = to_pci_dev(device);
 	struct net_device *dev = pci_get_drvdata(pdev);
-	struct rtl8169_private *tp = netdev_priv(dev);
 
-	return tp->TxDescArray ? -EBUSY : 0;
+	if (!netif_running(dev) || !netif_carrier_ok(dev))
+		pm_schedule_suspend(device, 10000);
+
+	return -EBUSY;
 }
 
 static const struct dev_pm_ops rtl8169_pm_ops = {
@@ -8208,9 +8185,6 @@ static void rtl_shutdown(struct pci_dev *pdev)
 {
 	struct net_device *dev = pci_get_drvdata(pdev);
 	struct rtl8169_private *tp = netdev_priv(dev);
-	struct device *d = &pdev->dev;
-
-	pm_runtime_get_sync(d);
 
 	rtl8169_net_suspend(dev);
 
@@ -8228,8 +8202,6 @@ static void rtl_shutdown(struct pci_dev *pdev)
 		pci_wake_from_d3(pdev, true);
 		pci_set_power_state(pdev, PCI_D3hot);
 	}
-
-	pm_runtime_put_noidle(d);
 }
 
 static void rtl_remove_one(struct pci_dev *pdev)
@@ -8251,9 +8223,6 @@ static void rtl_remove_one(struct pci_dev *pdev)
 
 	unregister_netdev(dev);
 
-	dma_free_coherent(&tp->pci_dev->dev, sizeof(*tp->counters),
-			  tp->counters, tp->counters_phys_addr);
-
 	rtl_release_firmware(tp);
 
 	if (pci_dev_run_wake(pdev))
@@ -8261,9 +8230,6 @@ static void rtl_remove_one(struct pci_dev *pdev)
 
 	/* restore original MAC address */
 	rtl_rar_set(tp, dev->perm_addr);
-
-	rtl_disable_msi(pdev, tp);
-	rtl8169_release_board(pdev, dev, tp->mmio_addr);
 }
 
 static const struct net_device_ops rtl_netdev_ops = {
@@ -8440,11 +8406,9 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		       MODULENAME, RTL8169_VERSION);
 	}
 
-	dev = alloc_etherdev(sizeof (*tp));
-	if (!dev) {
-		rc = -ENOMEM;
-		goto out;
-	}
+	dev = devm_alloc_etherdev(&pdev->dev, sizeof (*tp));
+	if (!dev)
+		return -ENOMEM;
 
 	SET_NETDEV_DEV(dev, &pdev->dev);
 	dev->netdev_ops = &rtl_netdev_ops;
@@ -8467,13 +8431,13 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 				     PCIE_LINK_STATE_CLKPM);
 
 	/* enable device (incl. PCI PM wakeup and hotplug setup) */
-	rc = pci_enable_device(pdev);
+	rc = pcim_enable_device(pdev);
 	if (rc < 0) {
 		netif_err(tp, probe, dev, "enable failure\n");
-		goto err_out_free_dev_1;
+		return rc;
 	}
 
-	if (pci_set_mwi(pdev) < 0)
+	if (pcim_set_mwi(pdev) < 0)
 		netif_info(tp, probe, dev, "Mem-Wr-Inval unavailable\n");
 
 	/* make sure PCI base addr 1 is MMIO */
@@ -8481,30 +8445,28 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		netif_err(tp, probe, dev,
 			  "region #%d not an MMIO resource, aborting\n",
 			  region);
-		rc = -ENODEV;
-		goto err_out_mwi_2;
+		return -ENODEV;
 	}
 
 	/* check for weird/broken PCI region reporting */
 	if (pci_resource_len(pdev, region) < R8169_REGS_SIZE) {
 		netif_err(tp, probe, dev,
 			  "Invalid PCI region size(s), aborting\n");
-		rc = -ENODEV;
-		goto err_out_mwi_2;
+		return -ENODEV;
 	}
 
 	rc = pci_request_regions(pdev, MODULENAME);
 	if (rc < 0) {
 		netif_err(tp, probe, dev, "could not request regions\n");
-		goto err_out_mwi_2;
+		return rc;
 	}
 
 	/* ioremap MMIO region */
-	ioaddr = ioremap(pci_resource_start(pdev, region), R8169_REGS_SIZE);
+	ioaddr = devm_ioremap(&pdev->dev, pci_resource_start(pdev, region),
+			      R8169_REGS_SIZE);
 	if (!ioaddr) {
 		netif_err(tp, probe, dev, "cannot remap MMIO, aborting\n");
-		rc = -EIO;
-		goto err_out_free_res_3;
+		return -EIO;
 	}
 	tp->mmio_addr = ioaddr;
 
@@ -8530,7 +8492,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
 		if (rc < 0) {
 			netif_err(tp, probe, dev, "DMA configuration failed\n");
-			goto err_out_unmap_4;
+			return rc;
 		}
 	}
 
@@ -8692,18 +8654,17 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	tp->rtl_fw = RTL_FIRMWARE_UNKNOWN;
 
-	tp->counters = dma_alloc_coherent (&pdev->dev, sizeof(*tp->counters),
-					   &tp->counters_phys_addr, GFP_KERNEL);
-	if (!tp->counters) {
-		rc = -ENOMEM;
-		goto err_out_msi_5;
-	}
+	tp->counters = dmam_alloc_coherent (&pdev->dev, sizeof(*tp->counters),
+					    &tp->counters_phys_addr,
+					    GFP_KERNEL);
+	if (!tp->counters)
+		return -ENOMEM;
+
+	pci_set_drvdata(pdev, dev);
 
 	rc = register_netdev(dev);
 	if (rc < 0)
-		goto err_out_cnt_6;
-
-	pci_set_drvdata(pdev, dev);
+		return rc;
 
 	netif_info(tp, probe, dev, "%s at 0x%p, %pM, XID %08x IRQ %d\n",
 		   rtl_chip_infos[chipset].name, ioaddr, dev->dev_addr,
@@ -8725,30 +8686,12 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		rtl8168_driver_start(tp);
 	}
 
-	if (pci_dev_run_wake(pdev))
-		pm_runtime_put_noidle(&pdev->dev);
-
 	netif_carrier_off(dev);
 
-out:
-	return rc;
+	if (pci_dev_run_wake(pdev))
+		pm_runtime_put_sync(&pdev->dev);
 
-err_out_cnt_6:
-	dma_free_coherent(&pdev->dev, sizeof(*tp->counters), tp->counters,
-			  tp->counters_phys_addr);
-err_out_msi_5:
-	netif_napi_del(&tp->napi);
-	rtl_disable_msi(pdev, tp);
-err_out_unmap_4:
-	iounmap(ioaddr);
-err_out_free_res_3:
-	pci_release_regions(pdev);
-err_out_mwi_2:
-	pci_clear_mwi(pdev);
-	pci_disable_device(pdev);
-err_out_free_dev_1:
-	free_netdev(dev);
-	goto out;
+	return 0;
 }
 
 static struct pci_driver rtl8169_pci_driver = {
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 009780df664b..a95fbd5510d9 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -2205,8 +2205,7 @@ static int ravb_probe(struct platform_device *pdev)
 	if (chip_id != RCAR_GEN2)
 		ravb_ptp_stop(ndev);
 out_release:
-	if (ndev)
-		free_netdev(ndev);
+	free_netdev(ndev);
 
 	pm_runtime_put(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
@@ -2256,9 +2255,6 @@ static int ravb_wol_setup(struct net_device *ndev)
 	/* Enable MagicPacket */
 	ravb_modify(ndev, ECMR, ECMR_MPDE, ECMR_MPDE);
 
-	/* Increased clock usage so device won't be suspended */
-	clk_enable(priv->clk);
-
 	return enable_irq_wake(priv->emac_irq);
 }
 
@@ -2277,9 +2273,6 @@ static int ravb_wol_restore(struct net_device *ndev)
 	if (ret < 0)
 		return ret;
 
-	/* Restore clock usage count */
-	clk_disable(priv->clk);
-
 	return disable_irq_wake(priv->emac_irq);
 }
 
diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c
index 53924a4fc31c..14c839bb09e7 100644
--- a/drivers/net/ethernet/renesas/sh_eth.c
+++ b/drivers/net/ethernet/renesas/sh_eth.c
@@ -40,7 +40,6 @@
 #include <linux/slab.h>
 #include <linux/ethtool.h>
 #include <linux/if_vlan.h>
-#include <linux/clk.h>
 #include <linux/sh_eth.h>
 #include <linux/of_mdio.h>
 
@@ -440,6 +439,17 @@ static void sh_eth_modify(struct net_device *ndev, int enum_index, u32 clear,
 		     enum_index);
 }
 
+static void sh_eth_tsu_write(struct sh_eth_private *mdp, u32 data,
+			     int enum_index)
+{
+	iowrite32(data, mdp->tsu_addr + mdp->reg_offset[enum_index]);
+}
+
+static u32 sh_eth_tsu_read(struct sh_eth_private *mdp, int enum_index)
+{
+	return ioread32(mdp->tsu_addr + mdp->reg_offset[enum_index]);
+}
+
 static bool sh_eth_is_gether(struct sh_eth_private *mdp)
 {
 	return mdp->reg_offset == sh_eth_offset_gigabit;
@@ -2304,7 +2314,7 @@ static void sh_eth_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
 	wol->supported = 0;
 	wol->wolopts = 0;
 
-	if (mdp->cd->magic && mdp->clk) {
+	if (mdp->cd->magic) {
 		wol->supported = WAKE_MAGIC;
 		wol->wolopts = mdp->wol_enabled ? WAKE_MAGIC : 0;
 	}
@@ -2314,7 +2324,7 @@ static int sh_eth_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
 {
 	struct sh_eth_private *mdp = netdev_priv(ndev);
 
-	if (!mdp->cd->magic || !mdp->clk || wol->wolopts & ~WAKE_MAGIC)
+	if (!mdp->cd->magic || wol->wolopts & ~WAKE_MAGIC)
 		return -EOPNOTSUPP;
 
 	mdp->wol_enabled = !!(wol->wolopts & WAKE_MAGIC);
@@ -3125,7 +3135,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 	const struct platform_device_id *id = platform_get_device_id(pdev);
 	struct sh_eth_private *mdp;
 	struct net_device *ndev;
-	int ret, devno;
+	int ret;
 
 	/* get base addr */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -3137,10 +3147,6 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_get_sync(&pdev->dev);
 
-	devno = pdev->id;
-	if (devno < 0)
-		devno = 0;
-
 	ret = platform_get_irq(pdev, 0);
 	if (ret < 0)
 		goto out_release;
@@ -3157,11 +3163,6 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 		goto out_release;
 	}
 
-	/* Get clock, if not found that's OK but Wake-On-Lan is unavailable */
-	mdp->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(mdp->clk))
-		mdp->clk = NULL;
-
 	ndev->base_addr = res->start;
 
 	spin_lock_init(&mdp->lock);
@@ -3222,8 +3223,8 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 		eth_hw_addr_random(ndev);
 	}
 
-	/* ioremap the TSU registers */
 	if (mdp->cd->tsu) {
+		int port = pdev->id < 0 ? 0 : pdev->id % 2;
 		struct resource *rtsu;
 
 		rtsu = platform_get_resource(pdev, IORESOURCE_MEM, 1);
@@ -3235,7 +3236,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 		/* We can only request the  TSU region  for the first port
 		 * of the two  sharing this TSU for the probe to succeed...
 		 */
-		if (devno % 2 == 0 &&
+		if (port == 0 &&
 		    !devm_request_mem_region(&pdev->dev, rtsu->start,
 					     resource_size(rtsu),
 					     dev_name(&pdev->dev))) {
@@ -3243,6 +3244,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 			ret = -EBUSY;
 			goto out_release;
 		}
+		/* ioremap the TSU registers */
 		mdp->tsu_addr = devm_ioremap(&pdev->dev, rtsu->start,
 					     resource_size(rtsu));
 		if (!mdp->tsu_addr) {
@@ -3250,16 +3252,14 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 			ret = -ENOMEM;
 			goto out_release;
 		}
-		mdp->port = devno % 2;
+		mdp->port = port;
 		ndev->features = NETIF_F_HW_VLAN_CTAG_FILTER;
-	}
 
-	/* Need to init only the first port of the two sharing a TSU */
-	if (devno % 2 == 0) {
-		if (mdp->cd->chip_reset)
-			mdp->cd->chip_reset(ndev);
+		/* Need to init only the first port of the two sharing a TSU */
+		if (port == 0) {
+			if (mdp->cd->chip_reset)
+				mdp->cd->chip_reset(ndev);
 
-		if (mdp->cd->tsu) {
 			/* TSU init (Init only)*/
 			sh_eth_tsu_init(mdp);
 		}
@@ -3283,7 +3283,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 	if (ret)
 		goto out_napi_del;
 
-	if (mdp->cd->magic && mdp->clk)
+	if (mdp->cd->magic)
 		device_set_wakeup_capable(&pdev->dev, 1);
 
 	/* print device information */
@@ -3301,8 +3301,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 
 out_release:
 	/* net_dev free */
-	if (ndev)
-		free_netdev(ndev);
+	free_netdev(ndev);
 
 	pm_runtime_put(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
@@ -3337,9 +3336,6 @@ static int sh_eth_wol_setup(struct net_device *ndev)
 	/* Enable MagicPacket */
 	sh_eth_modify(ndev, ECMR, ECMR_MPDE, ECMR_MPDE);
 
-	/* Increased clock usage so device won't be suspended */
-	clk_enable(mdp->clk);
-
 	return enable_irq_wake(ndev->irq);
 }
 
@@ -3365,9 +3361,6 @@ static int sh_eth_wol_restore(struct net_device *ndev)
 	if (ret < 0)
 		return ret;
 
-	/* Restore clock usage count */
-	clk_disable(mdp->clk);
-
 	return disable_irq_wake(ndev->irq);
 }
 
diff --git a/drivers/net/ethernet/renesas/sh_eth.h b/drivers/net/ethernet/renesas/sh_eth.h
index a6753ccba711..e5fe70134690 100644
--- a/drivers/net/ethernet/renesas/sh_eth.h
+++ b/drivers/net/ethernet/renesas/sh_eth.h
@@ -567,15 +567,4 @@ static inline void *sh_eth_tsu_get_offset(struct sh_eth_private *mdp,
 	return mdp->tsu_addr + mdp->reg_offset[enum_index];
 }
 
-static inline void sh_eth_tsu_write(struct sh_eth_private *mdp, u32 data,
-				    int enum_index)
-{
-	iowrite32(data, mdp->tsu_addr + mdp->reg_offset[enum_index]);
-}
-
-static inline u32 sh_eth_tsu_read(struct sh_eth_private *mdp, int enum_index)
-{
-	return ioread32(mdp->tsu_addr + mdp->reg_offset[enum_index]);
-}
-
 #endif	/* #ifndef __SH_ETH_H__ */
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index fc8f8bdf6579..056cb6093630 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -2902,6 +2902,12 @@ static int rocker_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_alloc_ordered_workqueue;
 	}
 
+	err = rocker_probe_ports(rocker);
+	if (err) {
+		dev_err(&pdev->dev, "failed to probe ports\n");
+		goto err_probe_ports;
+	}
+
 	/* Only FIBs pointing to our own netdevs are programmed into
 	 * the device, so no need to pass a callback.
 	 */
@@ -2918,22 +2924,16 @@ static int rocker_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	rocker->hw.id = rocker_read64(rocker, SWITCH_ID);
 
-	err = rocker_probe_ports(rocker);
-	if (err) {
-		dev_err(&pdev->dev, "failed to probe ports\n");
-		goto err_probe_ports;
-	}
-
 	dev_info(&pdev->dev, "Rocker switch with id %*phN\n",
 		 (int)sizeof(rocker->hw.id), &rocker->hw.id);
 
 	return 0;
 
-err_probe_ports:
-	unregister_switchdev_notifier(&rocker_switchdev_notifier);
 err_register_switchdev_notifier:
 	unregister_fib_notifier(&rocker->fib_nb);
 err_register_fib_notifier:
+	rocker_remove_ports(rocker);
+err_probe_ports:
 	destroy_workqueue(rocker->rocker_owq);
 err_alloc_ordered_workqueue:
 	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
@@ -2961,9 +2961,9 @@ static void rocker_remove(struct pci_dev *pdev)
 {
 	struct rocker *rocker = pci_get_drvdata(pdev);
 
-	rocker_remove_ports(rocker);
 	unregister_switchdev_notifier(&rocker_switchdev_notifier);
 	unregister_fib_notifier(&rocker->fib_nb);
+	rocker_remove_ports(rocker);
 	rocker_write32(rocker, CONTROL, ROCKER_CONTROL_RESET);
 	destroy_workqueue(rocker->rocker_owq);
 	free_irq(rocker_msix_vector(rocker, ROCKER_MSIX_VEC_EVENT), rocker);
diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c
index 6d6fb8cf3e7c..6473cc68c2d5 100644
--- a/drivers/net/ethernet/rocker/rocker_ofdpa.c
+++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c
@@ -789,7 +789,6 @@ static int ofdpa_flow_tbl_add(struct ofdpa_port *ofdpa_port,
 			       ofdpa_flags_nowait(flags),
 			       ofdpa_cmd_flow_tbl_add,
 			       found, NULL, NULL);
-	return 0;
 }
 
 static int ofdpa_flow_tbl_del(struct ofdpa_port *ofdpa_port,
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index e566dbb3343d..75fbf58e421c 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -160,11 +160,31 @@ static int efx_ef10_get_warm_boot_count(struct efx_nic *efx)
 		EFX_DWORD_FIELD(reg, EFX_WORD_0) : -EIO;
 }
 
+/* On all EF10s up to and including SFC9220 (Medford1), all PFs use BAR 0 for
+ * I/O space and BAR 2(&3) for memory.  On SFC9250 (Medford2), there is no I/O
+ * bar; PFs use BAR 0/1 for memory.
+ */
+static unsigned int efx_ef10_pf_mem_bar(struct efx_nic *efx)
+{
+	switch (efx->pci_dev->device) {
+	case 0x0b03: /* SFC9250 PF */
+		return 0;
+	default:
+		return 2;
+	}
+}
+
+/* All VFs use BAR 0/1 for memory */
+static unsigned int efx_ef10_vf_mem_bar(struct efx_nic *efx)
+{
+	return 0;
+}
+
 static unsigned int efx_ef10_mem_map_size(struct efx_nic *efx)
 {
 	int bar;
 
-	bar = efx->type->mem_bar;
+	bar = efx->type->mem_bar(efx);
 	return resource_size(&efx->pci_dev->resource[bar]);
 }
 
@@ -213,7 +233,7 @@ static int efx_ef10_get_vf_index(struct efx_nic *efx)
 
 static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 {
-	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V2_OUT_LEN);
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V4_OUT_LEN);
 	struct efx_ef10_nic_data *nic_data = efx->nic_data;
 	size_t outlen;
 	int rc;
@@ -257,9 +277,70 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
 		return -ENODEV;
 	}
 
+	if (outlen >= MC_CMD_GET_CAPABILITIES_V3_OUT_LEN) {
+		u8 vi_window_mode = MCDI_BYTE(outbuf,
+				GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE);
+
+		switch (vi_window_mode) {
+		case MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_8K:
+			efx->vi_stride = 8192;
+			break;
+		case MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_16K:
+			efx->vi_stride = 16384;
+			break;
+		case MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_64K:
+			efx->vi_stride = 65536;
+			break;
+		default:
+			netif_err(efx, probe, efx->net_dev,
+				  "Unrecognised VI window mode %d\n",
+				  vi_window_mode);
+			return -EIO;
+		}
+		netif_dbg(efx, probe, efx->net_dev, "vi_stride = %u\n",
+			  efx->vi_stride);
+	} else {
+		/* keep default VI stride */
+		netif_dbg(efx, probe, efx->net_dev,
+			  "firmware did not report VI window mode, assuming vi_stride = %u\n",
+			  efx->vi_stride);
+	}
+
+	if (outlen >= MC_CMD_GET_CAPABILITIES_V4_OUT_LEN) {
+		efx->num_mac_stats = MCDI_WORD(outbuf,
+				GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS);
+		netif_dbg(efx, probe, efx->net_dev,
+			  "firmware reports num_mac_stats = %u\n",
+			  efx->num_mac_stats);
+	} else {
+		/* leave num_mac_stats as the default value, MC_CMD_MAC_NSTATS */
+		netif_dbg(efx, probe, efx->net_dev,
+			  "firmware did not report num_mac_stats, assuming %u\n",
+			  efx->num_mac_stats);
+	}
+
 	return 0;
 }
 
+static void efx_ef10_read_licensed_features(struct efx_nic *efx)
+{
+	MCDI_DECLARE_BUF(inbuf, MC_CMD_LICENSING_V3_IN_LEN);
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_LICENSING_V3_OUT_LEN);
+	struct efx_ef10_nic_data *nic_data = efx->nic_data;
+	size_t outlen;
+	int rc;
+
+	MCDI_SET_DWORD(inbuf, LICENSING_V3_IN_OP,
+		       MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE);
+	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_LICENSING_V3, inbuf, sizeof(inbuf),
+				outbuf, sizeof(outbuf), &outlen);
+	if (rc || (outlen < MC_CMD_LICENSING_V3_OUT_LEN))
+		return;
+
+	nic_data->licensed_features = MCDI_QWORD(outbuf,
+					 LICENSING_V3_OUT_LICENSED_FEATURES);
+}
+
 static int efx_ef10_get_sysclk_freq(struct efx_nic *efx)
 {
 	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CLOCK_OUT_LEN);
@@ -589,17 +670,6 @@ static int efx_ef10_probe(struct efx_nic *efx)
 	struct efx_ef10_nic_data *nic_data;
 	int i, rc;
 
-	/* We can have one VI for each 8K region.  However, until we
-	 * use TX option descriptors we need two TX queues per channel.
-	 */
-	efx->max_channels = min_t(unsigned int,
-				  EFX_MAX_CHANNELS,
-				  efx_ef10_mem_map_size(efx) /
-				  (EFX_VI_PAGE_SIZE * EFX_TXQ_TYPES));
-	efx->max_tx_channels = efx->max_channels;
-	if (WARN_ON(efx->max_channels == 0))
-		return -EIO;
-
 	nic_data = kzalloc(sizeof(*nic_data), GFP_KERNEL);
 	if (!nic_data)
 		return -ENOMEM;
@@ -671,6 +741,22 @@ static int efx_ef10_probe(struct efx_nic *efx)
 	if (rc < 0)
 		goto fail5;
 
+	efx_ef10_read_licensed_features(efx);
+
+	/* We can have one VI for each vi_stride-byte region.
+	 * However, until we use TX option descriptors we need two TX queues
+	 * per channel.
+	 */
+	efx->max_channels = min_t(unsigned int,
+				  EFX_MAX_CHANNELS,
+				  efx_ef10_mem_map_size(efx) /
+				  (efx->vi_stride * EFX_TXQ_TYPES));
+	efx->max_tx_channels = efx->max_channels;
+	if (WARN_ON(efx->max_channels == 0)) {
+		rc = -EIO;
+		goto fail5;
+	}
+
 	efx->rx_packet_len_offset =
 		ES_DZ_RX_PREFIX_PKTLEN_OFST - ES_DZ_RX_PREFIX_SIZE;
 
@@ -695,7 +781,7 @@ static int efx_ef10_probe(struct efx_nic *efx)
 	if (rc && rc != -EPERM)
 		goto fail5;
 
-	efx_ptp_probe(efx, NULL);
+	efx_ptp_defer_probe_with_channel(efx);
 
 #ifdef CONFIG_SFC_SRIOV
 	if ((efx->pci_dev->physfn) && (!efx->pci_dev->is_physfn)) {
@@ -865,6 +951,11 @@ static int efx_ef10_link_piobufs(struct efx_nic *efx)
 
 	/* Link a buffer to each TX queue */
 	efx_for_each_channel(channel, efx) {
+		/* Extra channels, even those with TXQs (PTP), do not require
+		 * PIO resources.
+		 */
+		if (!channel->type->want_pio)
+			continue;
 		efx_for_each_channel_tx_queue(tx_queue, channel) {
 			/* We assign the PIO buffers to queues in
 			 * reverse order to allow for the following
@@ -907,7 +998,7 @@ static int efx_ef10_link_piobufs(struct efx_nic *efx)
 			} else {
 				tx_queue->piobuf =
 					nic_data->pio_write_base +
-					index * EFX_VI_PAGE_SIZE + offset;
+					index * efx->vi_stride + offset;
 				tx_queue->piobuf_offset = offset;
 				netif_dbg(efx, probe, efx->net_dev,
 					  "linked VI %u to PIO buffer %u offset %x addr %p\n",
@@ -1212,7 +1303,9 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
 	void __iomem *membase;
 	int rc;
 
-	channel_vis = max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES);
+	channel_vis = max(efx->n_channels,
+			  (efx->n_tx_channels + efx->n_extra_tx_channels) *
+			  EFX_TXQ_TYPES);
 
 #ifdef EFX_USE_PIO
 	/* Try to allocate PIO buffers if wanted and if the full
@@ -1253,19 +1346,19 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
 	 * for writing PIO buffers through.
 	 *
 	 * The UC mapping contains (channel_vis - 1) complete VIs and the
-	 * first half of the next VI.  Then the WC mapping begins with
-	 * the second half of this last VI.
+	 * first 4K of the next VI.  Then the WC mapping begins with
+	 * the remainder of this last VI.
 	 */
-	uc_mem_map_size = PAGE_ALIGN((channel_vis - 1) * EFX_VI_PAGE_SIZE +
+	uc_mem_map_size = PAGE_ALIGN((channel_vis - 1) * efx->vi_stride +
 				     ER_DZ_TX_PIOBUF);
 	if (nic_data->n_piobufs) {
 		/* pio_write_vi_base rounds down to give the number of complete
 		 * VIs inside the UC mapping.
 		 */
-		pio_write_vi_base = uc_mem_map_size / EFX_VI_PAGE_SIZE;
+		pio_write_vi_base = uc_mem_map_size / efx->vi_stride;
 		wc_mem_map_size = (PAGE_ALIGN((pio_write_vi_base +
 					       nic_data->n_piobufs) *
-					      EFX_VI_PAGE_SIZE) -
+					      efx->vi_stride) -
 				   uc_mem_map_size);
 		max_vis = pio_write_vi_base + nic_data->n_piobufs;
 	} else {
@@ -1337,7 +1430,7 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
 		nic_data->pio_write_vi_base = pio_write_vi_base;
 		nic_data->pio_write_base =
 			nic_data->wc_membase +
-			(pio_write_vi_base * EFX_VI_PAGE_SIZE + ER_DZ_TX_PIOBUF -
+			(pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
 			 uc_mem_map_size);
 
 		rc = efx_ef10_link_piobufs(efx);
@@ -1571,6 +1664,29 @@ static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {
 	EF10_DMA_STAT(tx_bad, VADAPTER_TX_BAD_PACKETS),
 	EF10_DMA_STAT(tx_bad_bytes, VADAPTER_TX_BAD_BYTES),
 	EF10_DMA_STAT(tx_overflow, VADAPTER_TX_OVERFLOW),
+	EF10_DMA_STAT(fec_uncorrected_errors, FEC_UNCORRECTED_ERRORS),
+	EF10_DMA_STAT(fec_corrected_errors, FEC_CORRECTED_ERRORS),
+	EF10_DMA_STAT(fec_corrected_symbols_lane0, FEC_CORRECTED_SYMBOLS_LANE0),
+	EF10_DMA_STAT(fec_corrected_symbols_lane1, FEC_CORRECTED_SYMBOLS_LANE1),
+	EF10_DMA_STAT(fec_corrected_symbols_lane2, FEC_CORRECTED_SYMBOLS_LANE2),
+	EF10_DMA_STAT(fec_corrected_symbols_lane3, FEC_CORRECTED_SYMBOLS_LANE3),
+	EF10_DMA_STAT(ctpio_dmabuf_start, CTPIO_DMABUF_START),
+	EF10_DMA_STAT(ctpio_vi_busy_fallback, CTPIO_VI_BUSY_FALLBACK),
+	EF10_DMA_STAT(ctpio_long_write_success, CTPIO_LONG_WRITE_SUCCESS),
+	EF10_DMA_STAT(ctpio_missing_dbell_fail, CTPIO_MISSING_DBELL_FAIL),
+	EF10_DMA_STAT(ctpio_overflow_fail, CTPIO_OVERFLOW_FAIL),
+	EF10_DMA_STAT(ctpio_underflow_fail, CTPIO_UNDERFLOW_FAIL),
+	EF10_DMA_STAT(ctpio_timeout_fail, CTPIO_TIMEOUT_FAIL),
+	EF10_DMA_STAT(ctpio_noncontig_wr_fail, CTPIO_NONCONTIG_WR_FAIL),
+	EF10_DMA_STAT(ctpio_frm_clobber_fail, CTPIO_FRM_CLOBBER_FAIL),
+	EF10_DMA_STAT(ctpio_invalid_wr_fail, CTPIO_INVALID_WR_FAIL),
+	EF10_DMA_STAT(ctpio_vi_clobber_fallback, CTPIO_VI_CLOBBER_FALLBACK),
+	EF10_DMA_STAT(ctpio_unqualified_fallback, CTPIO_UNQUALIFIED_FALLBACK),
+	EF10_DMA_STAT(ctpio_runt_fallback, CTPIO_RUNT_FALLBACK),
+	EF10_DMA_STAT(ctpio_success, CTPIO_SUCCESS),
+	EF10_DMA_STAT(ctpio_fallback, CTPIO_FALLBACK),
+	EF10_DMA_STAT(ctpio_poison, CTPIO_POISON),
+	EF10_DMA_STAT(ctpio_erase, CTPIO_ERASE),
 };
 
 #define HUNT_COMMON_STAT_MASK ((1ULL << EF10_STAT_port_tx_bytes) |	\
@@ -1646,6 +1762,43 @@ static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {
 	(1ULL << EF10_STAT_port_rx_dp_hlb_fetch) |			\
 	(1ULL << EF10_STAT_port_rx_dp_hlb_wait))
 
+/* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V2,
+ * indicated by returning a value >= MC_CMD_MAC_NSTATS_V2 in
+ * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS.
+ * These bits are in the second u64 of the raw mask.
+ */
+#define EF10_FEC_STAT_MASK (						\
+	(1ULL << (EF10_STAT_fec_uncorrected_errors - 64)) |		\
+	(1ULL << (EF10_STAT_fec_corrected_errors - 64)) |		\
+	(1ULL << (EF10_STAT_fec_corrected_symbols_lane0 - 64)) |	\
+	(1ULL << (EF10_STAT_fec_corrected_symbols_lane1 - 64)) |	\
+	(1ULL << (EF10_STAT_fec_corrected_symbols_lane2 - 64)) |	\
+	(1ULL << (EF10_STAT_fec_corrected_symbols_lane3 - 64)))
+
+/* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V3,
+ * indicated by returning a value >= MC_CMD_MAC_NSTATS_V3 in
+ * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS.
+ * These bits are in the second u64 of the raw mask.
+ */
+#define EF10_CTPIO_STAT_MASK (						\
+	(1ULL << (EF10_STAT_ctpio_dmabuf_start - 64)) |			\
+	(1ULL << (EF10_STAT_ctpio_vi_busy_fallback - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_long_write_success - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_missing_dbell_fail - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_overflow_fail - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_underflow_fail - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_timeout_fail - 64)) |			\
+	(1ULL << (EF10_STAT_ctpio_noncontig_wr_fail - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_frm_clobber_fail - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_invalid_wr_fail - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_vi_clobber_fallback - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_unqualified_fallback - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_runt_fallback - 64)) |		\
+	(1ULL << (EF10_STAT_ctpio_success - 64)) |			\
+	(1ULL << (EF10_STAT_ctpio_fallback - 64)) |			\
+	(1ULL << (EF10_STAT_ctpio_poison - 64)) |			\
+	(1ULL << (EF10_STAT_ctpio_erase - 64)))
+
 static u64 efx_ef10_raw_stat_mask(struct efx_nic *efx)
 {
 	u64 raw_mask = HUNT_COMMON_STAT_MASK;
@@ -1684,10 +1837,22 @@ static void efx_ef10_get_stat_mask(struct efx_nic *efx, unsigned long *mask)
 	if (nic_data->datapath_caps &
 	    (1 << MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN)) {
 		raw_mask[0] |= ~((1ULL << EF10_STAT_rx_unicast) - 1);
-		raw_mask[1] = (1ULL << (EF10_STAT_COUNT - 63)) - 1;
+		raw_mask[1] = (1ULL << (EF10_STAT_V1_COUNT - 64)) - 1;
 	} else {
 		raw_mask[1] = 0;
 	}
+	/* Only show FEC stats when NIC supports MC_CMD_MAC_STATS_V2 */
+	if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V2)
+		raw_mask[1] |= EF10_FEC_STAT_MASK;
+
+	/* CTPIO stats appear in V3. Only show them on devices that actually
+	 * support CTPIO. Although this driver doesn't use CTPIO others might,
+	 * and we may be reporting the stats for the underlying port.
+	 */
+	if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V3 &&
+	    (nic_data->datapath_caps2 &
+	     (1 << MC_CMD_GET_CAPABILITIES_V4_OUT_CTPIO_LBN)))
+		raw_mask[1] |= EF10_CTPIO_STAT_MASK;
 
 #if BITS_PER_LONG == 64
 	BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 2);
@@ -1791,7 +1956,7 @@ static int efx_ef10_try_update_nic_stats_pf(struct efx_nic *efx)
 
 	dma_stats = efx->stats_buffer.addr;
 
-	generation_end = dma_stats[MC_CMD_MAC_GENERATION_END];
+	generation_end = dma_stats[efx->num_mac_stats - 1];
 	if (generation_end == EFX_MC_STATS_GENERATION_INVALID)
 		return 0;
 	rmb();
@@ -1839,7 +2004,7 @@ static int efx_ef10_try_update_nic_stats_vf(struct efx_nic *efx)
 	DECLARE_BITMAP(mask, EF10_STAT_COUNT);
 	__le64 generation_start, generation_end;
 	u64 *stats = nic_data->stats;
-	u32 dma_len = MC_CMD_MAC_NSTATS * sizeof(u64);
+	u32 dma_len = efx->num_mac_stats * sizeof(u64);
 	struct efx_buffer stats_buf;
 	__le64 *dma_stats;
 	int rc;
@@ -1864,7 +2029,7 @@ static int efx_ef10_try_update_nic_stats_vf(struct efx_nic *efx)
 	}
 
 	dma_stats = stats_buf.addr;
-	dma_stats[MC_CMD_MAC_GENERATION_END] = EFX_MC_STATS_GENERATION_INVALID;
+	dma_stats[efx->num_mac_stats - 1] = EFX_MC_STATS_GENERATION_INVALID;
 
 	MCDI_SET_QWORD(inbuf, MAC_STATS_IN_DMA_ADDR, stats_buf.dma_addr);
 	MCDI_POPULATE_DWORD_1(inbuf, MAC_STATS_IN_CMD,
@@ -1883,7 +2048,7 @@ static int efx_ef10_try_update_nic_stats_vf(struct efx_nic *efx)
 		goto out;
 	}
 
-	generation_end = dma_stats[MC_CMD_MAC_GENERATION_END];
+	generation_end = dma_stats[efx->num_mac_stats - 1];
 	if (generation_end == EFX_MC_STATS_GENERATION_INVALID) {
 		WARN_ON_ONCE(1);
 		goto out;
@@ -1951,8 +2116,9 @@ static void efx_ef10_push_irq_moderation(struct efx_channel *channel)
 	} else {
 		unsigned int ticks = efx_usecs_to_ticks(efx, usecs);
 
-		EFX_POPULATE_DWORD_2(timer_cmd, ERF_DZ_TC_TIMER_MODE, mode,
-				     ERF_DZ_TC_TIMER_VAL, ticks);
+		EFX_POPULATE_DWORD_3(timer_cmd, ERF_DZ_TC_TIMER_MODE, mode,
+				     ERF_DZ_TC_TIMER_VAL, ticks,
+				     ERF_FZ_TC_TMR_REL_VAL, ticks);
 		efx_writed_page(efx, &timer_cmd, ER_DZ_EVQ_TMR,
 				channel->channel);
 	}
@@ -2263,12 +2429,25 @@ static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue)
 	int i;
 	BUILD_BUG_ON(MC_CMD_INIT_TXQ_OUT_LEN != 0);
 
+	/* Only attempt to enable TX timestamping if we have the license for it,
+	 * otherwise TXQ init will fail
+	 */
+	if (!(nic_data->licensed_features &
+	      (1 << LICENSED_V3_FEATURES_TX_TIMESTAMPS_LBN))) {
+		tx_queue->timestamping = false;
+		/* Disable sync events on this channel. */
+		if (efx->type->ptp_set_ts_sync_events)
+			efx->type->ptp_set_ts_sync_events(efx, false, false);
+	}
+
 	/* TSOv2 is a limited resource that can only be configured on a limited
 	 * number of queues. TSO without checksum offload is not really a thing,
 	 * so we only enable it for those queues.
+	 * TSOv2 cannot be used with Hardware timestamping.
 	 */
 	if (csum_offload && (nic_data->datapath_caps2 &
-			(1 << MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN))) {
+			(1 << MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN)) &&
+	    !tx_queue->timestamping) {
 		tso_v2 = true;
 		netif_dbg(efx, hw, efx->net_dev, "Using TSOv2 for channel %u\n",
 				channel->channel);
@@ -2294,14 +2473,16 @@ static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue)
 	inlen = MC_CMD_INIT_TXQ_IN_LEN(entries);
 
 	do {
-		MCDI_POPULATE_DWORD_3(inbuf, INIT_TXQ_IN_FLAGS,
+		MCDI_POPULATE_DWORD_4(inbuf, INIT_TXQ_IN_FLAGS,
 				/* This flag was removed from mcdi_pcol.h for
 				 * the non-_EXT version of INIT_TXQ.  However,
 				 * firmware still honours it.
 				 */
 				INIT_TXQ_EXT_IN_FLAG_TSOV2_EN, tso_v2,
 				INIT_TXQ_IN_FLAG_IP_CSUM_DIS, !csum_offload,
-				INIT_TXQ_IN_FLAG_TCP_CSUM_DIS, !csum_offload);
+				INIT_TXQ_IN_FLAG_TCP_CSUM_DIS, !csum_offload,
+				INIT_TXQ_EXT_IN_FLAG_TIMESTAMP,
+						tx_queue->timestamping);
 
 		rc = efx_mcdi_rpc_quiet(efx, MC_CMD_INIT_TXQ, inbuf, inlen,
 					NULL, 0, NULL);
@@ -2327,12 +2508,13 @@ static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue)
 	tx_queue->buffer[0].flags = EFX_TX_BUF_OPTION;
 	tx_queue->insert_count = 1;
 	txd = efx_tx_desc(tx_queue, 0);
-	EFX_POPULATE_QWORD_4(*txd,
+	EFX_POPULATE_QWORD_5(*txd,
 			     ESF_DZ_TX_DESC_IS_OPT, true,
 			     ESF_DZ_TX_OPTION_TYPE,
 			     ESE_DZ_TX_OPTION_DESC_CRC_CSUM,
 			     ESF_DZ_TX_OPTION_UDP_TCP_CSUM, csum_offload,
-			     ESF_DZ_TX_OPTION_IP_CSUM, csum_offload);
+			     ESF_DZ_TX_OPTION_IP_CSUM, csum_offload,
+			     ESF_DZ_TX_TIMESTAMP, tx_queue->timestamping);
 	tx_queue->write_count = 1;
 
 	if (tso_v2) {
@@ -3233,8 +3415,8 @@ static u16 efx_ef10_handle_rx_event_errors(struct efx_channel *channel,
 		if (unlikely(rx_encap_hdr != ESE_EZ_ENCAP_HDR_VXLAN &&
 			     ((rx_l3_class != ESE_DZ_L3_CLASS_IP4 &&
 			       rx_l3_class != ESE_DZ_L3_CLASS_IP6) ||
-			      (rx_l4_class != ESE_DZ_L4_CLASS_TCP &&
-			       rx_l4_class != ESE_DZ_L4_CLASS_UDP))))
+			      (rx_l4_class != ESE_FZ_L4_CLASS_TCP &&
+			       rx_l4_class != ESE_FZ_L4_CLASS_UDP))))
 			netdev_WARN(efx->net_dev,
 				    "invalid class for RX_TCPUDP_CKSUM_ERR: event="
 				    EFX_QWORD_FMT "\n",
@@ -3271,8 +3453,8 @@ static u16 efx_ef10_handle_rx_event_errors(struct efx_channel *channel,
 				    EFX_QWORD_VAL(*event));
 		else if (unlikely((rx_l3_class != ESE_DZ_L3_CLASS_IP4 &&
 				   rx_l3_class != ESE_DZ_L3_CLASS_IP6) ||
-				  (rx_l4_class != ESE_DZ_L4_CLASS_TCP &&
-				   rx_l4_class != ESE_DZ_L4_CLASS_UDP)))
+				  (rx_l4_class != ESE_FZ_L4_CLASS_TCP &&
+				   rx_l4_class != ESE_FZ_L4_CLASS_UDP)))
 			netdev_WARN(efx->net_dev,
 				    "invalid class for RX_TCP_UDP_INNER_CHKSUM_ERR: event="
 				    EFX_QWORD_FMT "\n",
@@ -3307,7 +3489,7 @@ static int efx_ef10_handle_rx_event(struct efx_channel *channel,
 	next_ptr_lbits = EFX_QWORD_FIELD(*event, ESF_DZ_RX_DSC_PTR_LBITS);
 	rx_queue_label = EFX_QWORD_FIELD(*event, ESF_DZ_RX_QLABEL);
 	rx_l3_class = EFX_QWORD_FIELD(*event, ESF_DZ_RX_L3_CLASS);
-	rx_l4_class = EFX_QWORD_FIELD(*event, ESF_DZ_RX_L4_CLASS);
+	rx_l4_class = EFX_QWORD_FIELD(*event, ESF_FZ_RX_L4_CLASS);
 	rx_cont = EFX_QWORD_FIELD(*event, ESF_DZ_RX_CONT);
 	rx_encap_hdr =
 		nic_data->datapath_caps &
@@ -3385,8 +3567,8 @@ static int efx_ef10_handle_rx_event(struct efx_channel *channel,
 							 rx_l3_class, rx_l4_class,
 							 event);
 	} else {
-		bool tcpudp = rx_l4_class == ESE_DZ_L4_CLASS_TCP ||
-			      rx_l4_class == ESE_DZ_L4_CLASS_UDP;
+		bool tcpudp = rx_l4_class == ESE_FZ_L4_CLASS_TCP ||
+			      rx_l4_class == ESE_FZ_L4_CLASS_UDP;
 
 		switch (rx_encap_hdr) {
 		case ESE_EZ_ENCAP_HDR_VXLAN: /* VxLAN or GENEVE */
@@ -3407,7 +3589,7 @@ static int efx_ef10_handle_rx_event(struct efx_channel *channel,
 		}
 	}
 
-	if (rx_l4_class == ESE_DZ_L4_CLASS_TCP)
+	if (rx_l4_class == ESE_FZ_L4_CLASS_TCP)
 		flags |= EFX_RX_PKT_TCP;
 
 	channel->irq_mod_score += 2 * n_packets;
@@ -3427,31 +3609,92 @@ static int efx_ef10_handle_rx_event(struct efx_channel *channel,
 	return n_packets;
 }
 
-static int
+static u32 efx_ef10_extract_event_ts(efx_qword_t *event)
+{
+	u32 tstamp;
+
+	tstamp = EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI);
+	tstamp <<= 16;
+	tstamp |= EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO);
+
+	return tstamp;
+}
+
+static void
 efx_ef10_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 {
 	struct efx_nic *efx = channel->efx;
 	struct efx_tx_queue *tx_queue;
 	unsigned int tx_ev_desc_ptr;
 	unsigned int tx_ev_q_label;
-	int tx_descs = 0;
+	unsigned int tx_ev_type;
+	u64 ts_part;
 
 	if (unlikely(READ_ONCE(efx->reset_pending)))
-		return 0;
+		return;
 
 	if (unlikely(EFX_QWORD_FIELD(*event, ESF_DZ_TX_DROP_EVENT)))
-		return 0;
+		return;
 
-	/* Transmit completion */
-	tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, ESF_DZ_TX_DESCR_INDX);
+	/* Get the transmit queue */
 	tx_ev_q_label = EFX_QWORD_FIELD(*event, ESF_DZ_TX_QLABEL);
 	tx_queue = efx_channel_get_tx_queue(channel,
 					    tx_ev_q_label % EFX_TXQ_TYPES);
-	tx_descs = ((tx_ev_desc_ptr + 1 - tx_queue->read_count) &
-		    tx_queue->ptr_mask);
-	efx_xmit_done(tx_queue, tx_ev_desc_ptr & tx_queue->ptr_mask);
 
-	return tx_descs;
+	if (!tx_queue->timestamping) {
+		/* Transmit completion */
+		tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, ESF_DZ_TX_DESCR_INDX);
+		efx_xmit_done(tx_queue, tx_ev_desc_ptr & tx_queue->ptr_mask);
+		return;
+	}
+
+	/* Transmit timestamps are only available for 8XXX series. They result
+	 * in three events per packet. These occur in order, and are:
+	 *  - the normal completion event
+	 *  - the low part of the timestamp
+	 *  - the high part of the timestamp
+	 *
+	 * Each part of the timestamp is itself split across two 16 bit
+	 * fields in the event.
+	 */
+	tx_ev_type = EFX_QWORD_FIELD(*event, ESF_EZ_TX_SOFT1);
+
+	switch (tx_ev_type) {
+	case TX_TIMESTAMP_EVENT_TX_EV_COMPLETION:
+		/* In case of Queue flush or FLR, we might have received
+		 * the previous TX completion event but not the Timestamp
+		 * events.
+		 */
+		if (tx_queue->completed_desc_ptr != tx_queue->ptr_mask)
+			efx_xmit_done(tx_queue, tx_queue->completed_desc_ptr);
+
+		tx_ev_desc_ptr = EFX_QWORD_FIELD(*event,
+						 ESF_DZ_TX_DESCR_INDX);
+		tx_queue->completed_desc_ptr =
+					tx_ev_desc_ptr & tx_queue->ptr_mask;
+		break;
+
+	case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_LO:
+		ts_part = efx_ef10_extract_event_ts(event);
+		tx_queue->completed_timestamp_minor = ts_part;
+		break;
+
+	case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_HI:
+		ts_part = efx_ef10_extract_event_ts(event);
+		tx_queue->completed_timestamp_major = ts_part;
+
+		efx_xmit_done(tx_queue, tx_queue->completed_desc_ptr);
+		tx_queue->completed_desc_ptr = tx_queue->ptr_mask;
+		break;
+
+	default:
+		netif_err(efx, hw, efx->net_dev,
+			  "channel %d unknown tx event type %d (data "
+			  EFX_QWORD_FMT ")\n",
+			  channel->channel, tx_ev_type,
+			  EFX_QWORD_VAL(*event));
+		break;
+	}
 }
 
 static void
@@ -3513,7 +3756,6 @@ static int efx_ef10_ev_process(struct efx_channel *channel, int quota)
 	efx_qword_t event, *p_event;
 	unsigned int read_ptr;
 	int ev_code;
-	int tx_descs = 0;
 	int spent = 0;
 
 	if (quota <= 0)
@@ -3553,13 +3795,7 @@ static int efx_ef10_ev_process(struct efx_channel *channel, int quota)
 			}
 			break;
 		case ESE_DZ_EV_CODE_TX_EV:
-			tx_descs += efx_ef10_handle_tx_event(channel, &event);
-			if (tx_descs > efx->txq_entries) {
-				spent = quota;
-				goto out;
-			} else if (++spent == quota) {
-				goto out;
-			}
+			efx_ef10_handle_tx_event(channel, &event);
 			break;
 		case ESE_DZ_EV_CODE_DRIVER_EV:
 			efx_ef10_handle_driver_event(channel, &event);
@@ -6034,7 +6270,8 @@ static int efx_ef10_ptp_set_ts_sync_events(struct efx_nic *efx, bool en,
 	      efx_ef10_rx_enable_timestamping :
 	      efx_ef10_rx_disable_timestamping;
 
-	efx_for_each_channel(channel, efx) {
+	channel = efx_ptp_channel(efx);
+	if (channel) {
 		int rc = set(channel, temp);
 		if (en && rc != 0) {
 			efx_ef10_ptp_set_ts_sync_events(efx, false, temp);
@@ -6392,7 +6629,7 @@ static int efx_ef10_udp_tnl_del_port(struct efx_nic *efx,
 
 const struct efx_nic_type efx_hunt_a0_vf_nic_type = {
 	.is_vf = true,
-	.mem_bar = EFX_MEM_VF_BAR,
+	.mem_bar = efx_ef10_vf_mem_bar,
 	.mem_map_size = efx_ef10_mem_map_size,
 	.probe = efx_ef10_probe_vf,
 	.remove = efx_ef10_remove,
@@ -6500,7 +6737,7 @@ const struct efx_nic_type efx_hunt_a0_vf_nic_type = {
 
 const struct efx_nic_type efx_hunt_a0_nic_type = {
 	.is_vf = false,
-	.mem_bar = EFX_MEM_BAR,
+	.mem_bar = efx_ef10_pf_mem_bar,
 	.mem_map_size = efx_ef10_mem_map_size,
 	.probe = efx_ef10_probe_pf,
 	.remove = efx_ef10_remove,
diff --git a/drivers/net/ethernet/sfc/ef10_regs.h b/drivers/net/ethernet/sfc/ef10_regs.h
index 2c4bf9476c37..6a56778cf06c 100644
--- a/drivers/net/ethernet/sfc/ef10_regs.h
+++ b/drivers/net/ethernet/sfc/ef10_regs.h
@@ -1,6 +1,6 @@
 /****************************************************************************
  * Driver for Solarflare network controllers and boards
- * Copyright 2012-2015 Solarflare Communications Inc.
+ * Copyright 2012-2017 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
@@ -79,6 +79,8 @@
 #define	ER_DZ_EVQ_TMR 0x00000420
 #define	ER_DZ_EVQ_TMR_STEP 8192
 #define	ER_DZ_EVQ_TMR_ROWS 2048
+#define	ERF_FZ_TC_TMR_REL_VAL_LBN 16
+#define	ERF_FZ_TC_TMR_REL_VAL_WIDTH 14
 #define	ERF_DZ_TC_TIMER_MODE_LBN 14
 #define	ERF_DZ_TC_TIMER_MODE_WIDTH 2
 #define	ERF_DZ_TC_TIMER_VAL_LBN 0
@@ -159,16 +161,24 @@
 #define	ESF_DZ_RX_EV_SOFT2_WIDTH 2
 #define	ESF_DZ_RX_DSC_PTR_LBITS_LBN 48
 #define	ESF_DZ_RX_DSC_PTR_LBITS_WIDTH 4
-#define	ESF_DZ_RX_L4_CLASS_LBN 45
-#define	ESF_DZ_RX_L4_CLASS_WIDTH 3
-#define	ESE_DZ_L4_CLASS_RSVD7 7
-#define	ESE_DZ_L4_CLASS_RSVD6 6
-#define	ESE_DZ_L4_CLASS_RSVD5 5
-#define	ESE_DZ_L4_CLASS_RSVD4 4
-#define	ESE_DZ_L4_CLASS_RSVD3 3
-#define	ESE_DZ_L4_CLASS_UDP 2
-#define	ESE_DZ_L4_CLASS_TCP 1
-#define	ESE_DZ_L4_CLASS_UNKNOWN 0
+#define	ESF_DE_RX_L4_CLASS_LBN 45
+#define	ESF_DE_RX_L4_CLASS_WIDTH 3
+#define	ESE_DE_L4_CLASS_RSVD7 7
+#define	ESE_DE_L4_CLASS_RSVD6 6
+#define	ESE_DE_L4_CLASS_RSVD5 5
+#define	ESE_DE_L4_CLASS_RSVD4 4
+#define	ESE_DE_L4_CLASS_RSVD3 3
+#define	ESE_DE_L4_CLASS_UDP 2
+#define	ESE_DE_L4_CLASS_TCP 1
+#define	ESE_DE_L4_CLASS_UNKNOWN 0
+#define	ESF_FZ_RX_FASTPD_INDCTR_LBN 47
+#define	ESF_FZ_RX_FASTPD_INDCTR_WIDTH 1
+#define	ESF_FZ_RX_L4_CLASS_LBN 45
+#define	ESF_FZ_RX_L4_CLASS_WIDTH 2
+#define	ESE_FZ_L4_CLASS_RSVD3 3
+#define	ESE_FZ_L4_CLASS_UDP 2
+#define	ESE_FZ_L4_CLASS_TCP 1
+#define	ESE_FZ_L4_CLASS_UNKNOWN 0
 #define	ESF_DZ_RX_L3_CLASS_LBN 42
 #define	ESF_DZ_RX_L3_CLASS_WIDTH 3
 #define	ESE_DZ_L3_CLASS_RSVD7 7
@@ -215,6 +225,8 @@
 #define	ESF_EZ_RX_ABORT_WIDTH 1
 #define	ESF_DZ_RX_ECC_ERR_LBN 29
 #define	ESF_DZ_RX_ECC_ERR_WIDTH 1
+#define	ESF_DZ_RX_TRUNC_ERR_LBN 29
+#define	ESF_DZ_RX_TRUNC_ERR_WIDTH 1
 #define	ESF_DZ_RX_CRC1_ERR_LBN 28
 #define	ESF_DZ_RX_CRC1_ERR_WIDTH 1
 #define	ESF_DZ_RX_CRC0_ERR_LBN 27
@@ -332,6 +344,8 @@
 #define	ESE_DZ_TX_OPTION_DESC_CRC_CSUM 0
 #define	ESF_DZ_TX_TSO_OPTION_TYPE_LBN 56
 #define	ESF_DZ_TX_TSO_OPTION_TYPE_WIDTH 4
+#define	ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B 3
+#define	ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A 2
 #define	ESE_DZ_TX_TSO_OPTION_DESC_ENCAP 1
 #define	ESE_DZ_TX_TSO_OPTION_DESC_NORMAL 0
 #define	ESF_DZ_TX_TSO_TCP_FLAGS_LBN 48
@@ -341,7 +355,7 @@
 #define	ESF_DZ_TX_TSO_TCP_SEQNO_LBN 0
 #define	ESF_DZ_TX_TSO_TCP_SEQNO_WIDTH 32
 
-/* TX_TSO_FATSO2A_DESC */
+/* TX_TSO_V2_DESC_A */
 #define	ESF_DZ_TX_DESC_IS_OPT_LBN 63
 #define	ESF_DZ_TX_DESC_IS_OPT_WIDTH 1
 #define	ESF_DZ_TX_OPTION_TYPE_LBN 60
@@ -360,8 +374,7 @@
 #define	ESF_DZ_TX_TSO_TCP_SEQNO_LBN 0
 #define	ESF_DZ_TX_TSO_TCP_SEQNO_WIDTH 32
 
-
-/* TX_TSO_FATSO2B_DESC */
+/* TX_TSO_V2_DESC_B */
 #define	ESF_DZ_TX_DESC_IS_OPT_LBN 63
 #define	ESF_DZ_TX_DESC_IS_OPT_WIDTH 1
 #define	ESF_DZ_TX_OPTION_TYPE_LBN 60
@@ -375,11 +388,10 @@
 #define	ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A 2
 #define	ESE_DZ_TX_TSO_OPTION_DESC_ENCAP 1
 #define	ESE_DZ_TX_TSO_OPTION_DESC_NORMAL 0
-#define	ESF_DZ_TX_TSO_OUTER_IP_ID_LBN 0
-#define	ESF_DZ_TX_TSO_OUTER_IP_ID_WIDTH 16
 #define	ESF_DZ_TX_TSO_TCP_MSS_LBN 32
 #define	ESF_DZ_TX_TSO_TCP_MSS_WIDTH 16
-
+#define	ESF_DZ_TX_TSO_OUTER_IPID_LBN 0
+#define	ESF_DZ_TX_TSO_OUTER_IPID_WIDTH 16
 
 /*************************************************************************/
 
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index e3c492fcaff0..16757cfc5b29 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -27,6 +27,7 @@
 #include <net/udp_tunnel.h>
 #include "efx.h"
 #include "nic.h"
+#include "io.h"
 #include "selftest.h"
 #include "sriov.h"
 
@@ -895,12 +896,20 @@ void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
 	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
 }
 
+static bool efx_default_channel_want_txqs(struct efx_channel *channel)
+{
+	return channel->channel - channel->efx->tx_channel_offset <
+		channel->efx->n_tx_channels;
+}
+
 static const struct efx_channel_type efx_default_channel_type = {
 	.pre_probe		= efx_channel_dummy_op_int,
 	.post_remove		= efx_channel_dummy_op_void,
 	.get_name		= efx_get_channel_name,
 	.copy			= efx_copy_channel,
+	.want_txqs		= efx_default_channel_want_txqs,
 	.keep_eventq		= false,
+	.want_pio		= true,
 };
 
 int efx_channel_dummy_op_int(struct efx_channel *channel)
@@ -952,31 +961,42 @@ void efx_link_status_changed(struct efx_nic *efx)
 		netif_info(efx, link, efx->net_dev, "link down\n");
 }
 
-void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)
+void efx_link_set_advertising(struct efx_nic *efx,
+			      const unsigned long *advertising)
 {
-	efx->link_advertising = advertising;
-	if (advertising) {
-		if (advertising & ADVERTISED_Pause)
-			efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
-		else
-			efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
-		if (advertising & ADVERTISED_Asym_Pause)
-			efx->wanted_fc ^= EFX_FC_TX;
-	}
+	memcpy(efx->link_advertising, advertising,
+	       sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK()));
+
+	efx->link_advertising[0] |= ADVERTISED_Autoneg;
+	if (advertising[0] & ADVERTISED_Pause)
+		efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
+	else
+		efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
+	if (advertising[0] & ADVERTISED_Asym_Pause)
+		efx->wanted_fc ^= EFX_FC_TX;
+}
+
+/* Equivalent to efx_link_set_advertising with all-zeroes, except does not
+ * force the Autoneg bit on.
+ */
+void efx_link_clear_advertising(struct efx_nic *efx)
+{
+	bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS);
+	efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
 }
 
 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
 {
 	efx->wanted_fc = wanted_fc;
-	if (efx->link_advertising) {
+	if (efx->link_advertising[0]) {
 		if (wanted_fc & EFX_FC_RX)
-			efx->link_advertising |= (ADVERTISED_Pause |
-						  ADVERTISED_Asym_Pause);
+			efx->link_advertising[0] |= (ADVERTISED_Pause |
+						     ADVERTISED_Asym_Pause);
 		else
-			efx->link_advertising &= ~(ADVERTISED_Pause |
-						   ADVERTISED_Asym_Pause);
+			efx->link_advertising[0] &= ~(ADVERTISED_Pause |
+						      ADVERTISED_Asym_Pause);
 		if (wanted_fc & EFX_FC_TX)
-			efx->link_advertising ^= ADVERTISED_Asym_Pause;
+			efx->link_advertising[0] ^= ADVERTISED_Asym_Pause;
 	}
 }
 
@@ -1248,7 +1268,7 @@ static int efx_init_io(struct efx_nic *efx)
 
 	netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
 
-	bar = efx->type->mem_bar;
+	bar = efx->type->mem_bar(efx);
 
 	rc = pci_enable_device(pci_dev);
 	if (rc) {
@@ -1323,7 +1343,7 @@ static void efx_fini_io(struct efx_nic *efx)
 	}
 
 	if (efx->membase_phys) {
-		bar = efx->type->mem_bar;
+		bar = efx->type->mem_bar(efx);
 		pci_release_region(efx->pci_dev, bar);
 		efx->membase_phys = 0;
 	}
@@ -1489,6 +1509,7 @@ static int efx_probe_interrupts(struct efx_nic *efx)
 	}
 
 	/* Assign extra channels if possible */
+	efx->n_extra_tx_channels = 0;
 	j = efx->n_channels;
 	for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
 		if (!efx->extra_channel_type[i])
@@ -1500,6 +1521,8 @@ static int efx_probe_interrupts(struct efx_nic *efx)
 			--j;
 			efx_get_channel(efx, j)->type =
 				efx->extra_channel_type[i];
+			if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
+				efx->n_extra_tx_channels++;
 		}
 	}
 
@@ -2909,6 +2932,10 @@ static const struct pci_device_id efx_pci_table[] = {
 	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
 	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03),  /* SFC9220 VF */
 	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
+	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03),  /* SFC9250 PF */
+	 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
+	{PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03),  /* SFC9250 VF */
+	 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
 	{0}			/* end of list */
 };
 
@@ -2977,6 +3004,9 @@ static int efx_init_struct(struct efx_nic *efx,
 	efx->rx_packet_ts_offset =
 		efx->type->rx_ts_offset - efx->type->rx_prefix_size;
 	spin_lock_init(&efx->stats_lock);
+	efx->vi_stride = EFX_DEFAULT_VI_STRIDE;
+	efx->num_mac_stats = MC_CMD_MAC_NSTATS;
+	BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END);
 	mutex_init(&efx->mac_lock);
 	efx->phy_op = &efx_dummy_phy_operations;
 	efx->mdio.dev = net_dev;
diff --git a/drivers/net/ethernet/sfc/efx.h b/drivers/net/ethernet/sfc/efx.h
index 52c84b782901..0cddc5ad77b1 100644
--- a/drivers/net/ethernet/sfc/efx.h
+++ b/drivers/net/ethernet/sfc/efx.h
@@ -14,11 +14,6 @@
 #include "net_driver.h"
 #include "filter.h"
 
-/* All controllers use BAR 0 for I/O space and BAR 2(&3) for memory */
-/* All VFs use BAR 0/1 for memory */
-#define EFX_MEM_BAR 2
-#define EFX_MEM_VF_BAR 0
-
 int efx_net_open(struct net_device *net_dev);
 int efx_net_stop(struct net_device *net_dev);
 
@@ -263,7 +258,9 @@ static inline void efx_schedule_channel_irq(struct efx_channel *channel)
 }
 
 void efx_link_status_changed(struct efx_nic *efx);
-void efx_link_set_advertising(struct efx_nic *efx, u32);
+void efx_link_set_advertising(struct efx_nic *efx,
+			      const unsigned long *advertising);
+void efx_link_clear_advertising(struct efx_nic *efx);
 void efx_link_set_wanted_fc(struct efx_nic *efx, u8);
 
 static inline void efx_device_detach_sync(struct efx_nic *efx)
diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c
index 3747b5644110..4db2dc2bf52f 100644
--- a/drivers/net/ethernet/sfc/ethtool.c
+++ b/drivers/net/ethernet/sfc/ethtool.c
@@ -720,7 +720,7 @@ static int efx_ethtool_set_pauseparam(struct net_device *net_dev,
 		goto out;
 	}
 
-	if ((wanted_fc & EFX_FC_AUTO) && !efx->link_advertising) {
+	if ((wanted_fc & EFX_FC_AUTO) && !efx->link_advertising[0]) {
 		netif_dbg(efx, drv, efx->net_dev,
 			  "Autonegotiation is disabled\n");
 		rc = -EINVAL;
@@ -732,10 +732,10 @@ static int efx_ethtool_set_pauseparam(struct net_device *net_dev,
 	    (wanted_fc & EFX_FC_TX) && !(efx->wanted_fc & EFX_FC_TX))
 		efx->type->prepare_enable_fc_tx(efx);
 
-	old_adv = efx->link_advertising;
+	old_adv = efx->link_advertising[0];
 	old_fc = efx->wanted_fc;
 	efx_link_set_wanted_fc(efx, wanted_fc);
-	if (efx->link_advertising != old_adv ||
+	if (efx->link_advertising[0] != old_adv ||
 	    (efx->wanted_fc ^ old_fc) & EFX_FC_AUTO) {
 		rc = efx->phy_op->reconfigure(efx);
 		if (rc) {
diff --git a/drivers/net/ethernet/sfc/farch.c b/drivers/net/ethernet/sfc/farch.c
index 5334dc83d926..266b9bee1f3a 100644
--- a/drivers/net/ethernet/sfc/farch.c
+++ b/drivers/net/ethernet/sfc/farch.c
@@ -818,17 +818,16 @@ static void efx_farch_magic_event(struct efx_channel *channel, u32 magic)
  * The NIC batches TX completion events; the message we receive is of
  * the form "complete all TX events up to this index".
  */
-static int
+static void
 efx_farch_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 {
 	unsigned int tx_ev_desc_ptr;
 	unsigned int tx_ev_q_label;
 	struct efx_tx_queue *tx_queue;
 	struct efx_nic *efx = channel->efx;
-	int tx_packets = 0;
 
 	if (unlikely(READ_ONCE(efx->reset_pending)))
-		return 0;
+		return;
 
 	if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
 		/* Transmit completion */
@@ -836,8 +835,6 @@ efx_farch_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 		tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 		tx_queue = efx_channel_get_tx_queue(
 			channel, tx_ev_q_label % EFX_TXQ_TYPES);
-		tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) &
-			      tx_queue->ptr_mask);
 		efx_xmit_done(tx_queue, tx_ev_desc_ptr);
 	} else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
 		/* Rewrite the FIFO write pointer */
@@ -856,8 +853,6 @@ efx_farch_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 			  EFX_QWORD_FMT"\n", channel->channel,
 			  EFX_QWORD_VAL(*event));
 	}
-
-	return tx_packets;
 }
 
 /* Detect errors included in the rx_evt_pkt_ok bit. */
@@ -1090,7 +1085,7 @@ efx_farch_handle_tx_flush_done(struct efx_nic *efx, efx_qword_t *event)
 	int qid;
 
 	qid = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
-	if (qid < EFX_TXQ_TYPES * efx->n_tx_channels) {
+	if (qid < EFX_TXQ_TYPES * (efx->n_tx_channels + efx->n_extra_tx_channels)) {
 		tx_queue = efx_get_tx_queue(efx, qid / EFX_TXQ_TYPES,
 					    qid % EFX_TXQ_TYPES);
 		if (atomic_cmpxchg(&tx_queue->flush_outstanding, 1, 0)) {
@@ -1270,7 +1265,6 @@ int efx_farch_ev_process(struct efx_channel *channel, int budget)
 	unsigned int read_ptr;
 	efx_qword_t event, *p_event;
 	int ev_code;
-	int tx_packets = 0;
 	int spent = 0;
 
 	if (budget <= 0)
@@ -1304,12 +1298,7 @@ int efx_farch_ev_process(struct efx_channel *channel, int budget)
 				goto out;
 			break;
 		case FSE_AZ_EV_CODE_TX_EV:
-			tx_packets += efx_farch_handle_tx_event(channel,
-								&event);
-			if (tx_packets > efx->txq_entries) {
-				spent = budget;
-				goto out;
-			}
+			efx_farch_handle_tx_event(channel, &event);
 			break;
 		case FSE_AZ_EV_CODE_DRV_GEN_EV:
 			efx_farch_handle_generated_event(channel, &event);
@@ -1680,20 +1669,21 @@ void efx_farch_rx_pull_indir_table(struct efx_nic *efx)
  */
 void efx_farch_dimension_resources(struct efx_nic *efx, unsigned sram_lim_qw)
 {
-	unsigned vi_count, buftbl_min;
+	unsigned vi_count, buftbl_min, total_tx_channels;
 
 #ifdef CONFIG_SFC_SRIOV
 	struct siena_nic_data *nic_data = efx->nic_data;
 #endif
 
+	total_tx_channels = efx->n_tx_channels + efx->n_extra_tx_channels;
 	/* Account for the buffer table entries backing the datapath channels
 	 * and the descriptor caches for those channels.
 	 */
 	buftbl_min = ((efx->n_rx_channels * EFX_MAX_DMAQ_SIZE +
-		       efx->n_tx_channels * EFX_TXQ_TYPES * EFX_MAX_DMAQ_SIZE +
+		       total_tx_channels * EFX_TXQ_TYPES * EFX_MAX_DMAQ_SIZE +
 		       efx->n_channels * EFX_MAX_EVQ_SIZE)
 		      * sizeof(efx_qword_t) / EFX_BUF_SIZE);
-	vi_count = max(efx->n_channels, efx->n_tx_channels * EFX_TXQ_TYPES);
+	vi_count = max(efx->n_channels, total_tx_channels * EFX_TXQ_TYPES);
 
 #ifdef CONFIG_SFC_SRIOV
 	if (efx->type->sriov_wanted) {
diff --git a/drivers/net/ethernet/sfc/io.h b/drivers/net/ethernet/sfc/io.h
index afb94aa2c15e..89563170af52 100644
--- a/drivers/net/ethernet/sfc/io.h
+++ b/drivers/net/ethernet/sfc/io.h
@@ -222,18 +222,21 @@ static inline void efx_reado_table(struct efx_nic *efx, efx_oword_t *value,
 	efx_reado(efx, value, reg + index * sizeof(efx_oword_t));
 }
 
-/* Page size used as step between per-VI registers */
-#define EFX_VI_PAGE_SIZE 0x2000
+/* default VI stride (step between per-VI registers) is 8K */
+#define EFX_DEFAULT_VI_STRIDE 0x2000
 
 /* Calculate offset to page-mapped register */
-#define EFX_PAGED_REG(page, reg) \
-	((page) * EFX_VI_PAGE_SIZE + (reg))
+static inline unsigned int efx_paged_reg(struct efx_nic *efx, unsigned int page,
+					 unsigned int reg)
+{
+	return page * efx->vi_stride + reg;
+}
 
 /* Write the whole of RX_DESC_UPD or TX_DESC_UPD */
 static inline void _efx_writeo_page(struct efx_nic *efx, efx_oword_t *value,
 				    unsigned int reg, unsigned int page)
 {
-	reg = EFX_PAGED_REG(page, reg);
+	reg = efx_paged_reg(efx, page, reg);
 
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "writing register %x with " EFX_OWORD_FMT "\n", reg,
@@ -262,7 +265,7 @@ static inline void
 _efx_writed_page(struct efx_nic *efx, const efx_dword_t *value,
 		 unsigned int reg, unsigned int page)
 {
-	efx_writed(efx, value, EFX_PAGED_REG(page, reg));
+	efx_writed(efx, value, efx_paged_reg(efx, page, reg));
 }
 #define efx_writed_page(efx, value, reg, page)				\
 	_efx_writed_page(efx, value,					\
@@ -288,10 +291,10 @@ static inline void _efx_writed_page_locked(struct efx_nic *efx,
 
 	if (page == 0) {
 		spin_lock_irqsave(&efx->biu_lock, flags);
-		efx_writed(efx, value, EFX_PAGED_REG(page, reg));
+		efx_writed(efx, value, efx_paged_reg(efx, page, reg));
 		spin_unlock_irqrestore(&efx->biu_lock, flags);
 	} else {
-		efx_writed(efx, value, EFX_PAGED_REG(page, reg));
+		efx_writed(efx, value, efx_paged_reg(efx, page, reg));
 	}
 }
 #define efx_writed_page_locked(efx, value, reg, page)			\
diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h
index 154ef41d1927..ebd95972ae7b 100644
--- a/drivers/net/ethernet/sfc/mcdi.h
+++ b/drivers/net/ethernet/sfc/mcdi.h
@@ -208,6 +208,9 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev);
 #define _MCDI_DWORD(_buf, _field)					\
 	((_buf) + (_MCDI_CHECK_ALIGN(MC_CMD_ ## _field ## _OFST, 4) >> 2))
 
+#define MCDI_BYTE(_buf, _field)						\
+	((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1),	\
+	 *MCDI_PTR(_buf, _field))
 #define MCDI_WORD(_buf, _field)						\
 	((u16)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 2) +	\
 	 le16_to_cpu(*(__force const __le16 *)MCDI_PTR(_buf, _field)))
diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
index 91fb54fd03d9..869d76f8f589 100644
--- a/drivers/net/ethernet/sfc/mcdi_pcol.h
+++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
@@ -114,6 +114,8 @@
 #define MCDI_HEADER_XFLAGS_WIDTH 8
 /* Request response using event */
 #define MCDI_HEADER_XFLAGS_EVREQ 0x01
+/* Request (and signal) early doorbell return */
+#define MCDI_HEADER_XFLAGS_DBRET 0x02
 
 /* Maximum number of payload bytes */
 #define MCDI_CTL_SDU_LEN_MAX_V1 0xfc
@@ -123,7 +125,7 @@
 
 
 /* The MC can generate events for two reasons:
- *   - To complete a shared memory request if XFLAGS_EVREQ was set
+ *   - To advance a shared memory request if XFLAGS_EVREQ was set
  *   - As a notification (link state, i2c event), controlled
  *     via MC_CMD_LOG_CTRL
  *
@@ -279,6 +281,17 @@
 /* Returned by MC_CMD_TESTASSERT if the action that should
  * have caused an assertion failed to do so.  */
 #define MC_CMD_ERR_UNREACHABLE 0x1016
+/* This command needs to be processed in the background but there were no
+ * resources to do so. Send it again after a command has completed. */
+#define MC_CMD_ERR_QUEUE_FULL 0x1017
+/* The operation could not be completed because the PCIe link has gone
+ * away.  This error code is never expected to be returned over the TLP
+ * transport. */
+#define MC_CMD_ERR_NO_PCIE 0x1018
+/* The operation could not be completed because the datapath has gone
+ * away.  This is distinct from MC_CMD_ERR_DATAPATH_DISABLED in that the
+ * datapath absence may be temporary*/
+#define MC_CMD_ERR_NO_DATAPATH 0x1019
 
 #define MC_CMD_ERR_CODE_OFST 0
 
@@ -360,6 +373,7 @@
 /* enum: Fatal. */
 #define          MCDI_EVENT_LEVEL_FATAL 0x3
 #define       MCDI_EVENT_DATA_OFST 0
+#define       MCDI_EVENT_DATA_LEN 4
 #define        MCDI_EVENT_CMDDONE_SEQ_LBN 0
 #define        MCDI_EVENT_CMDDONE_SEQ_WIDTH 8
 #define        MCDI_EVENT_CMDDONE_DATALEN_LBN 8
@@ -370,6 +384,8 @@
 #define        MCDI_EVENT_LINKCHANGE_LP_CAP_WIDTH 16
 #define        MCDI_EVENT_LINKCHANGE_SPEED_LBN 16
 #define        MCDI_EVENT_LINKCHANGE_SPEED_WIDTH 4
+/* enum: Link is down or link speed could not be determined */
+#define          MCDI_EVENT_LINKCHANGE_SPEED_UNKNOWN  0x0
 /* enum: 100Mbs */
 #define          MCDI_EVENT_LINKCHANGE_SPEED_100M  0x1
 /* enum: 1Gbs */
@@ -378,6 +394,12 @@
 #define          MCDI_EVENT_LINKCHANGE_SPEED_10G  0x3
 /* enum: 40Gbs */
 #define          MCDI_EVENT_LINKCHANGE_SPEED_40G  0x4
+/* enum: 25Gbs */
+#define          MCDI_EVENT_LINKCHANGE_SPEED_25G  0x5
+/* enum: 50Gbs */
+#define          MCDI_EVENT_LINKCHANGE_SPEED_50G  0x6
+/* enum: 100Gbs */
+#define          MCDI_EVENT_LINKCHANGE_SPEED_100G  0x7
 #define        MCDI_EVENT_LINKCHANGE_FCNTL_LBN 20
 #define        MCDI_EVENT_LINKCHANGE_FCNTL_WIDTH 4
 #define        MCDI_EVENT_LINKCHANGE_LINK_FLAGS_LBN 24
@@ -456,8 +478,63 @@
 #define          MCDI_EVENT_AOE_DDR_ECC_STATUS 0xa
 /* enum: PTP status update */
 #define          MCDI_EVENT_AOE_PTP_STATUS 0xb
+/* enum: FPGA header incorrect */
+#define          MCDI_EVENT_AOE_FPGA_LOAD_HEADER_ERR 0xc
+/* enum: FPGA Powered Off due to error in powering up FPGA */
+#define          MCDI_EVENT_AOE_FPGA_POWER_OFF 0xd
+/* enum: AOE FPGA load failed due to MC to MUM communication failure */
+#define          MCDI_EVENT_AOE_FPGA_LOAD_FAILED 0xe
+/* enum: Notify that invalid flash type detected */
+#define          MCDI_EVENT_AOE_INVALID_FPGA_FLASH_TYPE 0xf
+/* enum: Notify that the attempt to run FPGA Controller firmware timedout */
+#define          MCDI_EVENT_AOE_FC_RUN_TIMEDOUT 0x10
+/* enum: Failure to probe one or more FPGA boot flash chips */
+#define          MCDI_EVENT_AOE_FPGA_BOOT_FLASH_INVALID 0x11
+/* enum: FPGA boot-flash contains an invalid image header */
+#define          MCDI_EVENT_AOE_FPGA_BOOT_FLASH_HDR_INVALID 0x12
+/* enum: Failed to program clocks required by the FPGA */
+#define          MCDI_EVENT_AOE_FPGA_CLOCKS_PROGRAM_FAILED 0x13
+/* enum: Notify that FPGA Controller is alive to serve MCDI requests */
+#define          MCDI_EVENT_AOE_FC_RUNNING 0x14
 #define        MCDI_EVENT_AOE_ERR_DATA_LBN 8
 #define        MCDI_EVENT_AOE_ERR_DATA_WIDTH 8
+#define        MCDI_EVENT_AOE_ERR_FC_ASSERT_INFO_LBN 8
+#define        MCDI_EVENT_AOE_ERR_FC_ASSERT_INFO_WIDTH 8
+/* enum: FC Assert happened, but the register information is not available */
+#define          MCDI_EVENT_AOE_ERR_FC_ASSERT_SEEN 0x0
+/* enum: The register information for FC Assert is ready for readinng by driver
+ */
+#define          MCDI_EVENT_AOE_ERR_FC_ASSERT_DATA_READY 0x1
+#define        MCDI_EVENT_AOE_ERR_CODE_FPGA_HEADER_VERIFY_FAILED_LBN 8
+#define        MCDI_EVENT_AOE_ERR_CODE_FPGA_HEADER_VERIFY_FAILED_WIDTH 8
+/* enum: Reading from NV failed */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_NV_READ_FAIL 0x0
+/* enum: Invalid Magic Number if FPGA header */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_MAGIC_FAIL 0x1
+/* enum: Invalid Silicon type detected in header */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_SILICON_TYPE 0x2
+/* enum: Unsupported VRatio */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_VRATIO 0x3
+/* enum: Unsupported DDR Type */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_TYPE 0x4
+/* enum: DDR Voltage out of supported range */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_VOLTAGE 0x5
+/* enum: Unsupported DDR speed */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_SPEED 0x6
+/* enum: Unsupported DDR size */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_SIZE 0x7
+/* enum: Unsupported DDR rank */
+#define          MCDI_EVENT_AOE_ERR_FPGA_HEADER_DDR_RANK 0x8
+#define        MCDI_EVENT_AOE_ERR_CODE_INVALID_FPGA_FLASH_TYPE_INFO_LBN 8
+#define        MCDI_EVENT_AOE_ERR_CODE_INVALID_FPGA_FLASH_TYPE_INFO_WIDTH 8
+/* enum: Primary boot flash */
+#define          MCDI_EVENT_AOE_FLASH_TYPE_BOOT_PRIMARY 0x0
+/* enum: Secondary boot flash */
+#define          MCDI_EVENT_AOE_FLASH_TYPE_BOOT_SECONDARY 0x1
+#define        MCDI_EVENT_AOE_ERR_CODE_FPGA_POWER_OFF_LBN 8
+#define        MCDI_EVENT_AOE_ERR_CODE_FPGA_POWER_OFF_WIDTH 8
+#define        MCDI_EVENT_AOE_ERR_CODE_FPGA_LOAD_FAILED_LBN 8
+#define        MCDI_EVENT_AOE_ERR_CODE_FPGA_LOAD_FAILED_WIDTH 8
 #define        MCDI_EVENT_RX_ERR_RXQ_LBN 0
 #define        MCDI_EVENT_RX_ERR_RXQ_WIDTH 12
 #define        MCDI_EVENT_RX_ERR_TYPE_LBN 12
@@ -480,6 +557,22 @@
 #define          MCDI_EVENT_MUM_WATCHDOG 0x3
 #define        MCDI_EVENT_MUM_ERR_DATA_LBN 8
 #define        MCDI_EVENT_MUM_ERR_DATA_WIDTH 8
+#define        MCDI_EVENT_DBRET_SEQ_LBN 0
+#define        MCDI_EVENT_DBRET_SEQ_WIDTH 8
+#define        MCDI_EVENT_SUC_ERR_TYPE_LBN 0
+#define        MCDI_EVENT_SUC_ERR_TYPE_WIDTH 8
+/* enum: Corrupted or bad SUC application. */
+#define          MCDI_EVENT_SUC_BAD_APP 0x1
+/* enum: SUC application reported an assert. */
+#define          MCDI_EVENT_SUC_ASSERT 0x2
+/* enum: SUC application reported an exception. */
+#define          MCDI_EVENT_SUC_EXCEPTION 0x3
+/* enum: SUC watchdog timer expired. */
+#define          MCDI_EVENT_SUC_WATCHDOG 0x4
+#define        MCDI_EVENT_SUC_ERR_ADDRESS_LBN 8
+#define        MCDI_EVENT_SUC_ERR_ADDRESS_WIDTH 24
+#define        MCDI_EVENT_SUC_ERR_DATA_LBN 8
+#define        MCDI_EVENT_SUC_ERR_DATA_WIDTH 24
 #define       MCDI_EVENT_DATA_LBN 0
 #define       MCDI_EVENT_DATA_WIDTH 32
 #define       MCDI_EVENT_SRC_LBN 36
@@ -552,73 +645,99 @@
  * been processed and it may now resend the command
  */
 #define          MCDI_EVENT_CODE_PROXY_RESPONSE 0x1d
+/* enum: MCDI command accepted. New commands can be issued but this command is
+ * not done yet.
+ */
+#define          MCDI_EVENT_CODE_DBRET 0x1e
+/* enum: The MC has detected a fault on the SUC */
+#define          MCDI_EVENT_CODE_SUC 0x1f
 /* enum: Artificial event generated by host and posted via MC for test
  * purposes.
  */
 #define          MCDI_EVENT_CODE_TESTGEN  0xfa
 #define       MCDI_EVENT_CMDDONE_DATA_OFST 0
+#define       MCDI_EVENT_CMDDONE_DATA_LEN 4
 #define       MCDI_EVENT_CMDDONE_DATA_LBN 0
 #define       MCDI_EVENT_CMDDONE_DATA_WIDTH 32
 #define       MCDI_EVENT_LINKCHANGE_DATA_OFST 0
+#define       MCDI_EVENT_LINKCHANGE_DATA_LEN 4
 #define       MCDI_EVENT_LINKCHANGE_DATA_LBN 0
 #define       MCDI_EVENT_LINKCHANGE_DATA_WIDTH 32
 #define       MCDI_EVENT_SENSOREVT_DATA_OFST 0
+#define       MCDI_EVENT_SENSOREVT_DATA_LEN 4
 #define       MCDI_EVENT_SENSOREVT_DATA_LBN 0
 #define       MCDI_EVENT_SENSOREVT_DATA_WIDTH 32
 #define       MCDI_EVENT_MAC_STATS_DMA_GENERATION_OFST 0
+#define       MCDI_EVENT_MAC_STATS_DMA_GENERATION_LEN 4
 #define       MCDI_EVENT_MAC_STATS_DMA_GENERATION_LBN 0
 #define       MCDI_EVENT_MAC_STATS_DMA_GENERATION_WIDTH 32
 #define       MCDI_EVENT_TX_ERR_DATA_OFST 0
+#define       MCDI_EVENT_TX_ERR_DATA_LEN 4
 #define       MCDI_EVENT_TX_ERR_DATA_LBN 0
 #define       MCDI_EVENT_TX_ERR_DATA_WIDTH 32
 /* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the seconds field of
  * timestamp
  */
 #define       MCDI_EVENT_PTP_SECONDS_OFST 0
+#define       MCDI_EVENT_PTP_SECONDS_LEN 4
 #define       MCDI_EVENT_PTP_SECONDS_LBN 0
 #define       MCDI_EVENT_PTP_SECONDS_WIDTH 32
 /* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the major field of
  * timestamp
  */
 #define       MCDI_EVENT_PTP_MAJOR_OFST 0
+#define       MCDI_EVENT_PTP_MAJOR_LEN 4
 #define       MCDI_EVENT_PTP_MAJOR_LBN 0
 #define       MCDI_EVENT_PTP_MAJOR_WIDTH 32
 /* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the nanoseconds field
  * of timestamp
  */
 #define       MCDI_EVENT_PTP_NANOSECONDS_OFST 0
+#define       MCDI_EVENT_PTP_NANOSECONDS_LEN 4
 #define       MCDI_EVENT_PTP_NANOSECONDS_LBN 0
 #define       MCDI_EVENT_PTP_NANOSECONDS_WIDTH 32
 /* For CODE_PTP_RX, CODE_PTP_PPS and CODE_HW_PPS events the minor field of
  * timestamp
  */
 #define       MCDI_EVENT_PTP_MINOR_OFST 0
+#define       MCDI_EVENT_PTP_MINOR_LEN 4
 #define       MCDI_EVENT_PTP_MINOR_LBN 0
 #define       MCDI_EVENT_PTP_MINOR_WIDTH 32
 /* For CODE_PTP_RX events, the lowest four bytes of sourceUUID from PTP packet
  */
 #define       MCDI_EVENT_PTP_UUID_OFST 0
+#define       MCDI_EVENT_PTP_UUID_LEN 4
 #define       MCDI_EVENT_PTP_UUID_LBN 0
 #define       MCDI_EVENT_PTP_UUID_WIDTH 32
 #define       MCDI_EVENT_RX_ERR_DATA_OFST 0
+#define       MCDI_EVENT_RX_ERR_DATA_LEN 4
 #define       MCDI_EVENT_RX_ERR_DATA_LBN 0
 #define       MCDI_EVENT_RX_ERR_DATA_WIDTH 32
 #define       MCDI_EVENT_PAR_ERR_DATA_OFST 0
+#define       MCDI_EVENT_PAR_ERR_DATA_LEN 4
 #define       MCDI_EVENT_PAR_ERR_DATA_LBN 0
 #define       MCDI_EVENT_PAR_ERR_DATA_WIDTH 32
 #define       MCDI_EVENT_ECC_CORR_ERR_DATA_OFST 0
+#define       MCDI_EVENT_ECC_CORR_ERR_DATA_LEN 4
 #define       MCDI_EVENT_ECC_CORR_ERR_DATA_LBN 0
 #define       MCDI_EVENT_ECC_CORR_ERR_DATA_WIDTH 32
 #define       MCDI_EVENT_ECC_FATAL_ERR_DATA_OFST 0
+#define       MCDI_EVENT_ECC_FATAL_ERR_DATA_LEN 4
 #define       MCDI_EVENT_ECC_FATAL_ERR_DATA_LBN 0
 #define       MCDI_EVENT_ECC_FATAL_ERR_DATA_WIDTH 32
 /* For CODE_PTP_TIME events, the major value of the PTP clock */
 #define       MCDI_EVENT_PTP_TIME_MAJOR_OFST 0
+#define       MCDI_EVENT_PTP_TIME_MAJOR_LEN 4
 #define       MCDI_EVENT_PTP_TIME_MAJOR_LBN 0
 #define       MCDI_EVENT_PTP_TIME_MAJOR_WIDTH 32
 /* For CODE_PTP_TIME events, bits 19-26 of the minor value of the PTP clock */
 #define       MCDI_EVENT_PTP_TIME_MINOR_26_19_LBN 36
 #define       MCDI_EVENT_PTP_TIME_MINOR_26_19_WIDTH 8
+/* For CODE_PTP_TIME events, most significant bits of the minor value of the
+ * PTP clock. This is a more generic equivalent of PTP_TIME_MINOR_26_19.
+ */
+#define       MCDI_EVENT_PTP_TIME_MINOR_MS_8BITS_LBN 36
+#define       MCDI_EVENT_PTP_TIME_MINOR_MS_8BITS_WIDTH 8
 /* For CODE_PTP_TIME events where report sync status is enabled, indicates
  * whether the NIC clock has ever been set
  */
@@ -634,10 +753,17 @@
  */
 #define       MCDI_EVENT_PTP_TIME_MINOR_26_21_LBN 38
 #define       MCDI_EVENT_PTP_TIME_MINOR_26_21_WIDTH 6
+/* For CODE_PTP_TIME events, most significant bits of the minor value of the
+ * PTP clock. This is a more generic equivalent of PTP_TIME_MINOR_26_21.
+ */
+#define       MCDI_EVENT_PTP_TIME_MINOR_MS_6BITS_LBN 38
+#define       MCDI_EVENT_PTP_TIME_MINOR_MS_6BITS_WIDTH 6
 #define       MCDI_EVENT_PROXY_REQUEST_BUFF_INDEX_OFST 0
+#define       MCDI_EVENT_PROXY_REQUEST_BUFF_INDEX_LEN 4
 #define       MCDI_EVENT_PROXY_REQUEST_BUFF_INDEX_LBN 0
 #define       MCDI_EVENT_PROXY_REQUEST_BUFF_INDEX_WIDTH 32
 #define       MCDI_EVENT_PROXY_RESPONSE_HANDLE_OFST 0
+#define       MCDI_EVENT_PROXY_RESPONSE_HANDLE_LEN 4
 #define       MCDI_EVENT_PROXY_RESPONSE_HANDLE_LBN 0
 #define       MCDI_EVENT_PROXY_RESPONSE_HANDLE_WIDTH 32
 /* Zero means that the request has been completed or authorized, and the driver
@@ -646,6 +772,10 @@
  */
 #define       MCDI_EVENT_PROXY_RESPONSE_RC_LBN 36
 #define       MCDI_EVENT_PROXY_RESPONSE_RC_WIDTH 8
+#define       MCDI_EVENT_DBRET_DATA_OFST 0
+#define       MCDI_EVENT_DBRET_DATA_LEN 4
+#define       MCDI_EVENT_DBRET_DATA_LBN 0
+#define       MCDI_EVENT_DBRET_DATA_WIDTH 32
 
 /* FCDI_EVENT structuredef */
 #define    FCDI_EVENT_LEN 8
@@ -662,6 +792,7 @@
 /* enum: Fatal. */
 #define          FCDI_EVENT_LEVEL_FATAL 0x3
 #define       FCDI_EVENT_DATA_OFST 0
+#define       FCDI_EVENT_DATA_LEN 4
 #define        FCDI_EVENT_LINK_STATE_STATUS_LBN 0
 #define        FCDI_EVENT_LINK_STATE_STATUS_WIDTH 1
 #define          FCDI_EVENT_LINK_DOWN 0x0 /* enum */
@@ -701,6 +832,7 @@
 #define          FCDI_EVENT_REBOOT_FC_FW 0x0 /* enum */
 #define          FCDI_EVENT_REBOOT_FC_BOOTLOADER 0x1 /* enum */
 #define       FCDI_EVENT_ASSERT_INSTR_ADDRESS_OFST 0
+#define       FCDI_EVENT_ASSERT_INSTR_ADDRESS_LEN 4
 #define       FCDI_EVENT_ASSERT_INSTR_ADDRESS_LBN 0
 #define       FCDI_EVENT_ASSERT_INSTR_ADDRESS_WIDTH 32
 #define       FCDI_EVENT_ASSERT_TYPE_LBN 36
@@ -708,12 +840,15 @@
 #define       FCDI_EVENT_DDR_TEST_RESULT_STATUS_CODE_LBN 36
 #define       FCDI_EVENT_DDR_TEST_RESULT_STATUS_CODE_WIDTH 8
 #define       FCDI_EVENT_DDR_TEST_RESULT_RESULT_OFST 0
+#define       FCDI_EVENT_DDR_TEST_RESULT_RESULT_LEN 4
 #define       FCDI_EVENT_DDR_TEST_RESULT_RESULT_LBN 0
 #define       FCDI_EVENT_DDR_TEST_RESULT_RESULT_WIDTH 32
 #define       FCDI_EVENT_LINK_STATE_DATA_OFST 0
+#define       FCDI_EVENT_LINK_STATE_DATA_LEN 4
 #define       FCDI_EVENT_LINK_STATE_DATA_LBN 0
 #define       FCDI_EVENT_LINK_STATE_DATA_WIDTH 32
 #define       FCDI_EVENT_PTP_STATE_OFST 0
+#define       FCDI_EVENT_PTP_STATE_LEN 4
 #define          FCDI_EVENT_PTP_UNDEFINED 0x0 /* enum */
 #define          FCDI_EVENT_PTP_SETUP_FAILED 0x1 /* enum */
 #define          FCDI_EVENT_PTP_OPERATIONAL 0x2 /* enum */
@@ -722,6 +857,7 @@
 #define       FCDI_EVENT_DDR_ECC_STATUS_BANK_ID_LBN 36
 #define       FCDI_EVENT_DDR_ECC_STATUS_BANK_ID_WIDTH 8
 #define       FCDI_EVENT_DDR_ECC_STATUS_STATUS_OFST 0
+#define       FCDI_EVENT_DDR_ECC_STATUS_STATUS_LEN 4
 #define       FCDI_EVENT_DDR_ECC_STATUS_STATUS_LBN 0
 #define       FCDI_EVENT_DDR_ECC_STATUS_STATUS_WIDTH 32
 /* Index of MC port being referred to */
@@ -729,9 +865,11 @@
 #define       FCDI_EVENT_PORT_CONFIG_SRC_WIDTH 8
 /* FC Port index that matches the MC port index in SRC */
 #define       FCDI_EVENT_PORT_CONFIG_DATA_OFST 0
+#define       FCDI_EVENT_PORT_CONFIG_DATA_LEN 4
 #define       FCDI_EVENT_PORT_CONFIG_DATA_LBN 0
 #define       FCDI_EVENT_PORT_CONFIG_DATA_WIDTH 32
 #define       FCDI_EVENT_BOOT_RESULT_OFST 0
+#define       FCDI_EVENT_BOOT_RESULT_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_AOE/MC_CMD_AOE_OUT_INFO/FC_BOOT_RESULT */
 #define       FCDI_EVENT_BOOT_RESULT_LBN 0
@@ -748,14 +886,17 @@
 #define    FCDI_EXTENDED_EVENT_PPS_LEN(num) (8+8*(num))
 /* Number of timestamps following */
 #define       FCDI_EXTENDED_EVENT_PPS_COUNT_OFST 0
+#define       FCDI_EXTENDED_EVENT_PPS_COUNT_LEN 4
 #define       FCDI_EXTENDED_EVENT_PPS_COUNT_LBN 0
 #define       FCDI_EXTENDED_EVENT_PPS_COUNT_WIDTH 32
 /* Seconds field of a timestamp record */
 #define       FCDI_EXTENDED_EVENT_PPS_SECONDS_OFST 8
+#define       FCDI_EXTENDED_EVENT_PPS_SECONDS_LEN 4
 #define       FCDI_EXTENDED_EVENT_PPS_SECONDS_LBN 64
 #define       FCDI_EXTENDED_EVENT_PPS_SECONDS_WIDTH 32
 /* Nanoseconds field of a timestamp record */
 #define       FCDI_EXTENDED_EVENT_PPS_NANOSECONDS_OFST 12
+#define       FCDI_EXTENDED_EVENT_PPS_NANOSECONDS_LEN 4
 #define       FCDI_EXTENDED_EVENT_PPS_NANOSECONDS_LBN 96
 #define       FCDI_EXTENDED_EVENT_PPS_NANOSECONDS_WIDTH 32
 /* Timestamp records comprising the event */
@@ -783,6 +924,7 @@
 /* enum: Fatal. */
 #define          MUM_EVENT_LEVEL_FATAL 0x3
 #define       MUM_EVENT_DATA_OFST 0
+#define       MUM_EVENT_DATA_LEN 4
 #define        MUM_EVENT_SENSOR_ID_LBN 0
 #define        MUM_EVENT_SENSOR_ID_WIDTH 8
 /*             Enum values, see field(s): */
@@ -820,18 +962,23 @@
 /* enum: Link fault has been asserted, or has cleared. */
 #define          MUM_EVENT_CODE_QSFP_LASI_INTERRUPT 0x4
 #define       MUM_EVENT_SENSOR_DATA_OFST 0
+#define       MUM_EVENT_SENSOR_DATA_LEN 4
 #define       MUM_EVENT_SENSOR_DATA_LBN 0
 #define       MUM_EVENT_SENSOR_DATA_WIDTH 32
 #define       MUM_EVENT_PORT_PHY_FLAGS_OFST 0
+#define       MUM_EVENT_PORT_PHY_FLAGS_LEN 4
 #define       MUM_EVENT_PORT_PHY_FLAGS_LBN 0
 #define       MUM_EVENT_PORT_PHY_FLAGS_WIDTH 32
 #define       MUM_EVENT_PORT_PHY_COPPER_LEN_OFST 0
+#define       MUM_EVENT_PORT_PHY_COPPER_LEN_LEN 4
 #define       MUM_EVENT_PORT_PHY_COPPER_LEN_LBN 0
 #define       MUM_EVENT_PORT_PHY_COPPER_LEN_WIDTH 32
 #define       MUM_EVENT_PORT_PHY_CAPS_OFST 0
+#define       MUM_EVENT_PORT_PHY_CAPS_LEN 4
 #define       MUM_EVENT_PORT_PHY_CAPS_LBN 0
 #define       MUM_EVENT_PORT_PHY_CAPS_WIDTH 32
 #define       MUM_EVENT_PORT_PHY_TECH_OFST 0
+#define       MUM_EVENT_PORT_PHY_TECH_LEN 4
 #define          MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_UNKNOWN 0x0 /* enum */
 #define          MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_OPTICAL 0x1 /* enum */
 #define          MUM_EVENT_PORT_PHY_STATE_QSFP_MODULE_TECH_COPPER_PASSIVE 0x2 /* enum */
@@ -864,7 +1011,9 @@
 /* MC_CMD_READ32_IN msgrequest */
 #define    MC_CMD_READ32_IN_LEN 8
 #define       MC_CMD_READ32_IN_ADDR_OFST 0
+#define       MC_CMD_READ32_IN_ADDR_LEN 4
 #define       MC_CMD_READ32_IN_NUMWORDS_OFST 4
+#define       MC_CMD_READ32_IN_NUMWORDS_LEN 4
 
 /* MC_CMD_READ32_OUT msgresponse */
 #define    MC_CMD_READ32_OUT_LENMIN 4
@@ -882,13 +1031,14 @@
  */
 #define MC_CMD_WRITE32 0x2
 
-#define MC_CMD_0x2_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x2_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_WRITE32_IN msgrequest */
 #define    MC_CMD_WRITE32_IN_LENMIN 8
 #define    MC_CMD_WRITE32_IN_LENMAX 252
 #define    MC_CMD_WRITE32_IN_LEN(num) (4+4*(num))
 #define       MC_CMD_WRITE32_IN_ADDR_OFST 0
+#define       MC_CMD_WRITE32_IN_ADDR_LEN 4
 #define       MC_CMD_WRITE32_IN_BUFFER_OFST 4
 #define       MC_CMD_WRITE32_IN_BUFFER_LEN 4
 #define       MC_CMD_WRITE32_IN_BUFFER_MINNUM 1
@@ -915,6 +1065,7 @@
  * is a bitfield, with each bit as documented below.
  */
 #define       MC_CMD_COPYCODE_IN_SRC_ADDR_OFST 0
+#define       MC_CMD_COPYCODE_IN_SRC_ADDR_LEN 4
 /* enum: Deprecated; equivalent to setting BOOT_MAGIC_PRESENT (see below) */
 #define          MC_CMD_COPYCODE_HUNT_NO_MAGIC_ADDR 0x10000
 /* enum: Deprecated; equivalent to setting BOOT_MAGIC_PRESENT and
@@ -940,9 +1091,12 @@
 #define        MC_CMD_COPYCODE_IN_BOOT_MAGIC_DISABLE_XIP_WIDTH 1
 /* Destination address */
 #define       MC_CMD_COPYCODE_IN_DEST_ADDR_OFST 4
+#define       MC_CMD_COPYCODE_IN_DEST_ADDR_LEN 4
 #define       MC_CMD_COPYCODE_IN_NUMWORDS_OFST 8
+#define       MC_CMD_COPYCODE_IN_NUMWORDS_LEN 4
 /* Address of where to jump after copy. */
 #define       MC_CMD_COPYCODE_IN_JUMP_OFST 12
+#define       MC_CMD_COPYCODE_IN_JUMP_LEN 4
 /* enum: Control should return to the caller rather than jumping */
 #define          MC_CMD_COPYCODE_JUMP_NONE 0x1
 
@@ -956,12 +1110,13 @@
  */
 #define MC_CMD_SET_FUNC 0x4
 
-#define MC_CMD_0x4_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x4_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_SET_FUNC_IN msgrequest */
 #define    MC_CMD_SET_FUNC_IN_LEN 4
 /* Set function */
 #define       MC_CMD_SET_FUNC_IN_FUNC_OFST 0
+#define       MC_CMD_SET_FUNC_IN_FUNC_LEN 4
 
 /* MC_CMD_SET_FUNC_OUT msgresponse */
 #define    MC_CMD_SET_FUNC_OUT_LEN 0
@@ -973,7 +1128,7 @@
  */
 #define MC_CMD_GET_BOOT_STATUS 0x5
 
-#define MC_CMD_0x5_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x5_PRIVILEGE_CTG SRIOV_CTG_GENERAL
 
 /* MC_CMD_GET_BOOT_STATUS_IN msgrequest */
 #define    MC_CMD_GET_BOOT_STATUS_IN_LEN 0
@@ -982,9 +1137,11 @@
 #define    MC_CMD_GET_BOOT_STATUS_OUT_LEN 8
 /* ?? */
 #define       MC_CMD_GET_BOOT_STATUS_OUT_BOOT_OFFSET_OFST 0
+#define       MC_CMD_GET_BOOT_STATUS_OUT_BOOT_OFFSET_LEN 4
 /* enum: indicates that the MC wasn't flash booted */
 #define          MC_CMD_GET_BOOT_STATUS_OUT_BOOT_OFFSET_NULL  0xdeadbeef
 #define       MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_OFST 4
+#define       MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_LEN 4
 #define        MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_WATCHDOG_LBN 0
 #define        MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_WATCHDOG_WIDTH 1
 #define        MC_CMD_GET_BOOT_STATUS_OUT_FLAGS_PRIMARY_LBN 1
@@ -1007,11 +1164,13 @@
 #define    MC_CMD_GET_ASSERTS_IN_LEN 4
 /* Set to clear assertion */
 #define       MC_CMD_GET_ASSERTS_IN_CLEAR_OFST 0
+#define       MC_CMD_GET_ASSERTS_IN_CLEAR_LEN 4
 
 /* MC_CMD_GET_ASSERTS_OUT msgresponse */
 #define    MC_CMD_GET_ASSERTS_OUT_LEN 140
 /* Assertion status flag. */
 #define       MC_CMD_GET_ASSERTS_OUT_GLOBAL_FLAGS_OFST 0
+#define       MC_CMD_GET_ASSERTS_OUT_GLOBAL_FLAGS_LEN 4
 /* enum: No assertions have failed. */
 #define          MC_CMD_GET_ASSERTS_FLAGS_NO_FAILS 0x1
 /* enum: A system-level assertion has failed. */
@@ -1024,6 +1183,7 @@
 #define          MC_CMD_GET_ASSERTS_FLAGS_ADDR_TRAP 0x5
 /* Failing PC value */
 #define       MC_CMD_GET_ASSERTS_OUT_SAVED_PC_OFFS_OFST 4
+#define       MC_CMD_GET_ASSERTS_OUT_SAVED_PC_OFFS_LEN 4
 /* Saved GP regs */
 #define       MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_OFST 8
 #define       MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_LEN 4
@@ -1034,7 +1194,9 @@
 #define          MC_CMD_GET_ASSERTS_REG_NO_DATA 0xda7a1057
 /* Failing thread address */
 #define       MC_CMD_GET_ASSERTS_OUT_THREAD_OFFS_OFST 132
+#define       MC_CMD_GET_ASSERTS_OUT_THREAD_OFFS_LEN 4
 #define       MC_CMD_GET_ASSERTS_OUT_RESERVED_OFST 136
+#define       MC_CMD_GET_ASSERTS_OUT_RESERVED_LEN 4
 
 
 /***********************************/
@@ -1050,12 +1212,14 @@
 #define    MC_CMD_LOG_CTRL_IN_LEN 8
 /* Log destination */
 #define       MC_CMD_LOG_CTRL_IN_LOG_DEST_OFST 0
+#define       MC_CMD_LOG_CTRL_IN_LOG_DEST_LEN 4
 /* enum: UART. */
 #define          MC_CMD_LOG_CTRL_IN_LOG_DEST_UART 0x1
 /* enum: Event queue. */
 #define          MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ 0x2
 /* Legacy argument. Must be zero. */
 #define       MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ_OFST 4
+#define       MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ_LEN 4
 
 /* MC_CMD_LOG_CTRL_OUT msgresponse */
 #define    MC_CMD_LOG_CTRL_OUT_LEN 0
@@ -1076,23 +1240,29 @@
 #define    MC_CMD_GET_VERSION_EXT_IN_LEN 4
 /* placeholder, set to 0 */
 #define       MC_CMD_GET_VERSION_EXT_IN_EXT_FLAGS_OFST 0
+#define       MC_CMD_GET_VERSION_EXT_IN_EXT_FLAGS_LEN 4
 
 /* MC_CMD_GET_VERSION_V0_OUT msgresponse: deprecated version format */
 #define    MC_CMD_GET_VERSION_V0_OUT_LEN 4
 #define       MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0
+#define       MC_CMD_GET_VERSION_OUT_FIRMWARE_LEN 4
 /* enum: Reserved version number to indicate "any" version. */
 #define          MC_CMD_GET_VERSION_OUT_FIRMWARE_ANY 0xffffffff
 /* enum: Bootrom version value for Siena. */
 #define          MC_CMD_GET_VERSION_OUT_FIRMWARE_SIENA_BOOTROM 0xb0070000
 /* enum: Bootrom version value for Huntington. */
 #define          MC_CMD_GET_VERSION_OUT_FIRMWARE_HUNT_BOOTROM 0xb0070001
+/* enum: Bootrom version value for Medford2. */
+#define          MC_CMD_GET_VERSION_OUT_FIRMWARE_MEDFORD2_BOOTROM 0xb0070002
 
 /* MC_CMD_GET_VERSION_OUT msgresponse */
 #define    MC_CMD_GET_VERSION_OUT_LEN 32
 /*            MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */
+/*            MC_CMD_GET_VERSION_OUT_FIRMWARE_LEN 4 */
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */
 #define       MC_CMD_GET_VERSION_OUT_PCOL_OFST 4
+#define       MC_CMD_GET_VERSION_OUT_PCOL_LEN 4
 /* 128bit mask of functions supported by the current firmware */
 #define       MC_CMD_GET_VERSION_OUT_SUPPORTED_FUNCS_OFST 8
 #define       MC_CMD_GET_VERSION_OUT_SUPPORTED_FUNCS_LEN 16
@@ -1104,9 +1274,11 @@
 /* MC_CMD_GET_VERSION_EXT_OUT msgresponse */
 #define    MC_CMD_GET_VERSION_EXT_OUT_LEN 48
 /*            MC_CMD_GET_VERSION_OUT_FIRMWARE_OFST 0 */
+/*            MC_CMD_GET_VERSION_OUT_FIRMWARE_LEN 4 */
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_VERSION_V0_OUT/MC_CMD_GET_VERSION_OUT_FIRMWARE */
 #define       MC_CMD_GET_VERSION_EXT_OUT_PCOL_OFST 4
+#define       MC_CMD_GET_VERSION_EXT_OUT_PCOL_LEN 4
 /* 128bit mask of functions supported by the current firmware */
 #define       MC_CMD_GET_VERSION_EXT_OUT_SUPPORTED_FUNCS_OFST 8
 #define       MC_CMD_GET_VERSION_EXT_OUT_SUPPORTED_FUNCS_LEN 16
@@ -1136,41 +1308,54 @@
 #define          MC_CMD_PTP_OP_ENABLE 0x1
 /* enum: Disable PTP packet timestamping operation. */
 #define          MC_CMD_PTP_OP_DISABLE 0x2
-/* enum: Send a PTP packet. */
+/* enum: Send a PTP packet. This operation is used on Siena and Huntington.
+ * From Medford onwards it is not supported: on those platforms PTP transmit
+ * timestamping is done using the fast path.
+ */
 #define          MC_CMD_PTP_OP_TRANSMIT 0x3
 /* enum: Read the current NIC time. */
 #define          MC_CMD_PTP_OP_READ_NIC_TIME 0x4
-/* enum: Get the current PTP status. */
+/* enum: Get the current PTP status. Note that the clock frequency returned (in
+ * Hz) is rounded to the nearest MHz (e.g. 666000000 for 666666666).
+ */
 #define          MC_CMD_PTP_OP_STATUS 0x5
 /* enum: Adjust the PTP NIC's time. */
 #define          MC_CMD_PTP_OP_ADJUST 0x6
 /* enum: Synchronize host and NIC time. */
 #define          MC_CMD_PTP_OP_SYNCHRONIZE 0x7
-/* enum: Basic manufacturing tests. */
+/* enum: Basic manufacturing tests. Siena PTP adapters only. */
 #define          MC_CMD_PTP_OP_MANFTEST_BASIC 0x8
-/* enum: Packet based manufacturing tests. */
+/* enum: Packet based manufacturing tests. Siena PTP adapters only. */
 #define          MC_CMD_PTP_OP_MANFTEST_PACKET 0x9
 /* enum: Reset some of the PTP related statistics */
 #define          MC_CMD_PTP_OP_RESET_STATS 0xa
 /* enum: Debug operations to MC. */
 #define          MC_CMD_PTP_OP_DEBUG 0xb
-/* enum: Read an FPGA register */
+/* enum: Read an FPGA register. Siena PTP adapters only. */
 #define          MC_CMD_PTP_OP_FPGAREAD 0xc
-/* enum: Write an FPGA register */
+/* enum: Write an FPGA register. Siena PTP adapters only. */
 #define          MC_CMD_PTP_OP_FPGAWRITE 0xd
 /* enum: Apply an offset to the NIC clock */
 #define          MC_CMD_PTP_OP_CLOCK_OFFSET_ADJUST 0xe
-/* enum: Change Apply an offset to the NIC clock */
+/* enum: Change the frequency correction applied to the NIC clock */
 #define          MC_CMD_PTP_OP_CLOCK_FREQ_ADJUST 0xf
-/* enum: Set the MC packet filter VLAN tags for received PTP packets */
+/* enum: Set the MC packet filter VLAN tags for received PTP packets.
+ * Deprecated for Huntington onwards.
+ */
 #define          MC_CMD_PTP_OP_RX_SET_VLAN_FILTER 0x10
-/* enum: Set the MC packet filter UUID for received PTP packets */
+/* enum: Set the MC packet filter UUID for received PTP packets. Deprecated for
+ * Huntington onwards.
+ */
 #define          MC_CMD_PTP_OP_RX_SET_UUID_FILTER 0x11
-/* enum: Set the MC packet filter Domain for received PTP packets */
+/* enum: Set the MC packet filter Domain for received PTP packets. Deprecated
+ * for Huntington onwards.
+ */
 #define          MC_CMD_PTP_OP_RX_SET_DOMAIN_FILTER 0x12
-/* enum: Set the clock source */
+/* enum: Set the clock source. Required for snapper tests on Huntington and
+ * Medford. Not implemented for Siena or Medford2.
+ */
 #define          MC_CMD_PTP_OP_SET_CLK_SRC 0x13
-/* enum: Reset value of Timer Reg. */
+/* enum: Reset value of Timer Reg. Not implemented. */
 #define          MC_CMD_PTP_OP_RST_CLK 0x14
 /* enum: Enable the forwarding of PPS events to the host */
 #define          MC_CMD_PTP_OP_PPS_ENABLE 0x15
@@ -1191,7 +1376,7 @@
 /* enum: Unsubscribe to stop receiving time events */
 #define          MC_CMD_PTP_OP_TIME_EVENT_UNSUBSCRIBE 0x19
 /* enum: PPS based manfacturing tests. Requires PPS output to be looped to PPS
- * input on the same NIC.
+ * input on the same NIC. Siena PTP adapters only.
  */
 #define          MC_CMD_PTP_OP_MANFTEST_PPS 0x1a
 /* enum: Set the PTP sync status. Status is used by firmware to report to event
@@ -1204,11 +1389,15 @@
 /* MC_CMD_PTP_IN_ENABLE msgrequest */
 #define    MC_CMD_PTP_IN_ENABLE_LEN 16
 #define       MC_CMD_PTP_IN_CMD_OFST 0
+#define       MC_CMD_PTP_IN_CMD_LEN 4
 #define       MC_CMD_PTP_IN_PERIPH_ID_OFST 4
-/* Event queue for PTP events */
+#define       MC_CMD_PTP_IN_PERIPH_ID_LEN 4
+/* Not used. Events are always sent to function relative queue 0. */
 #define       MC_CMD_PTP_IN_ENABLE_QUEUE_OFST 8
-/* PTP timestamping mode */
+#define       MC_CMD_PTP_IN_ENABLE_QUEUE_LEN 4
+/* PTP timestamping mode. Not used from Huntington onwards. */
 #define       MC_CMD_PTP_IN_ENABLE_MODE_OFST 12
+#define       MC_CMD_PTP_IN_ENABLE_MODE_LEN 4
 /* enum: PTP, version 1 */
 #define          MC_CMD_PTP_MODE_V1 0x0
 /* enum: PTP, version 1, with VLAN headers - deprecated */
@@ -1225,16 +1414,21 @@
 /* MC_CMD_PTP_IN_DISABLE msgrequest */
 #define    MC_CMD_PTP_IN_DISABLE_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_TRANSMIT msgrequest */
 #define    MC_CMD_PTP_IN_TRANSMIT_LENMIN 13
 #define    MC_CMD_PTP_IN_TRANSMIT_LENMAX 252
 #define    MC_CMD_PTP_IN_TRANSMIT_LEN(num) (12+1*(num))
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Transmit packet length */
 #define       MC_CMD_PTP_IN_TRANSMIT_LENGTH_OFST 8
+#define       MC_CMD_PTP_IN_TRANSMIT_LENGTH_LEN 4
 /* Transmit packet data */
 #define       MC_CMD_PTP_IN_TRANSMIT_PACKET_OFST 12
 #define       MC_CMD_PTP_IN_TRANSMIT_PACKET_LEN 1
@@ -1244,17 +1438,30 @@
 /* MC_CMD_PTP_IN_READ_NIC_TIME msgrequest */
 #define    MC_CMD_PTP_IN_READ_NIC_TIME_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
+
+/* MC_CMD_PTP_IN_READ_NIC_TIME_V2 msgrequest */
+#define    MC_CMD_PTP_IN_READ_NIC_TIME_V2_LEN 8
+/*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_STATUS msgrequest */
 #define    MC_CMD_PTP_IN_STATUS_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_ADJUST msgrequest */
 #define    MC_CMD_PTP_IN_ADJUST_LEN 24
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Frequency adjustment 40 bit fixed point ns */
 #define       MC_CMD_PTP_IN_ADJUST_FREQ_OFST 8
 #define       MC_CMD_PTP_IN_ADJUST_FREQ_LEN 8
@@ -1262,21 +1469,67 @@
 #define       MC_CMD_PTP_IN_ADJUST_FREQ_HI_OFST 12
 /* enum: Number of fractional bits in frequency adjustment */
 #define          MC_CMD_PTP_IN_ADJUST_BITS 0x28
+/* enum: Number of fractional bits in frequency adjustment when FP44_FREQ_ADJ
+ * is indicated in the MC_CMD_PTP_OUT_GET_ATTRIBUTES command CAPABILITIES
+ * field.
+ */
+#define          MC_CMD_PTP_IN_ADJUST_BITS_FP44 0x2c
 /* Time adjustment in seconds */
 #define       MC_CMD_PTP_IN_ADJUST_SECONDS_OFST 16
+#define       MC_CMD_PTP_IN_ADJUST_SECONDS_LEN 4
 /* Time adjustment major value */
 #define       MC_CMD_PTP_IN_ADJUST_MAJOR_OFST 16
+#define       MC_CMD_PTP_IN_ADJUST_MAJOR_LEN 4
 /* Time adjustment in nanoseconds */
 #define       MC_CMD_PTP_IN_ADJUST_NANOSECONDS_OFST 20
+#define       MC_CMD_PTP_IN_ADJUST_NANOSECONDS_LEN 4
 /* Time adjustment minor value */
 #define       MC_CMD_PTP_IN_ADJUST_MINOR_OFST 20
+#define       MC_CMD_PTP_IN_ADJUST_MINOR_LEN 4
+
+/* MC_CMD_PTP_IN_ADJUST_V2 msgrequest */
+#define    MC_CMD_PTP_IN_ADJUST_V2_LEN 28
+/*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
+/* Frequency adjustment 40 bit fixed point ns */
+#define       MC_CMD_PTP_IN_ADJUST_V2_FREQ_OFST 8
+#define       MC_CMD_PTP_IN_ADJUST_V2_FREQ_LEN 8
+#define       MC_CMD_PTP_IN_ADJUST_V2_FREQ_LO_OFST 8
+#define       MC_CMD_PTP_IN_ADJUST_V2_FREQ_HI_OFST 12
+/* enum: Number of fractional bits in frequency adjustment */
+/*               MC_CMD_PTP_IN_ADJUST_BITS 0x28 */
+/* enum: Number of fractional bits in frequency adjustment when FP44_FREQ_ADJ
+ * is indicated in the MC_CMD_PTP_OUT_GET_ATTRIBUTES command CAPABILITIES
+ * field.
+ */
+/*               MC_CMD_PTP_IN_ADJUST_BITS_FP44 0x2c */
+/* Time adjustment in seconds */
+#define       MC_CMD_PTP_IN_ADJUST_V2_SECONDS_OFST 16
+#define       MC_CMD_PTP_IN_ADJUST_V2_SECONDS_LEN 4
+/* Time adjustment major value */
+#define       MC_CMD_PTP_IN_ADJUST_V2_MAJOR_OFST 16
+#define       MC_CMD_PTP_IN_ADJUST_V2_MAJOR_LEN 4
+/* Time adjustment in nanoseconds */
+#define       MC_CMD_PTP_IN_ADJUST_V2_NANOSECONDS_OFST 20
+#define       MC_CMD_PTP_IN_ADJUST_V2_NANOSECONDS_LEN 4
+/* Time adjustment minor value */
+#define       MC_CMD_PTP_IN_ADJUST_V2_MINOR_OFST 20
+#define       MC_CMD_PTP_IN_ADJUST_V2_MINOR_LEN 4
+/* Upper 32bits of major time offset adjustment */
+#define       MC_CMD_PTP_IN_ADJUST_V2_MAJOR_HI_OFST 24
+#define       MC_CMD_PTP_IN_ADJUST_V2_MAJOR_HI_LEN 4
 
 /* MC_CMD_PTP_IN_SYNCHRONIZE msgrequest */
 #define    MC_CMD_PTP_IN_SYNCHRONIZE_LEN 20
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Number of time readings to capture */
 #define       MC_CMD_PTP_IN_SYNCHRONIZE_NUMTIMESETS_OFST 8
+#define       MC_CMD_PTP_IN_SYNCHRONIZE_NUMTIMESETS_LEN 4
 /* Host address in which to write "synchronization started" indication (64
  * bits)
  */
@@ -1288,42 +1541,59 @@
 /* MC_CMD_PTP_IN_MANFTEST_BASIC msgrequest */
 #define    MC_CMD_PTP_IN_MANFTEST_BASIC_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_MANFTEST_PACKET msgrequest */
 #define    MC_CMD_PTP_IN_MANFTEST_PACKET_LEN 12
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Enable or disable packet testing */
 #define       MC_CMD_PTP_IN_MANFTEST_PACKET_TEST_ENABLE_OFST 8
+#define       MC_CMD_PTP_IN_MANFTEST_PACKET_TEST_ENABLE_LEN 4
 
 /* MC_CMD_PTP_IN_RESET_STATS msgrequest */
 #define    MC_CMD_PTP_IN_RESET_STATS_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /* Reset PTP statistics */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_DEBUG msgrequest */
 #define    MC_CMD_PTP_IN_DEBUG_LEN 12
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Debug operations */
 #define       MC_CMD_PTP_IN_DEBUG_DEBUG_PARAM_OFST 8
+#define       MC_CMD_PTP_IN_DEBUG_DEBUG_PARAM_LEN 4
 
 /* MC_CMD_PTP_IN_FPGAREAD msgrequest */
 #define    MC_CMD_PTP_IN_FPGAREAD_LEN 16
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 #define       MC_CMD_PTP_IN_FPGAREAD_ADDR_OFST 8
+#define       MC_CMD_PTP_IN_FPGAREAD_ADDR_LEN 4
 #define       MC_CMD_PTP_IN_FPGAREAD_NUMBYTES_OFST 12
+#define       MC_CMD_PTP_IN_FPGAREAD_NUMBYTES_LEN 4
 
 /* MC_CMD_PTP_IN_FPGAWRITE msgrequest */
 #define    MC_CMD_PTP_IN_FPGAWRITE_LENMIN 13
 #define    MC_CMD_PTP_IN_FPGAWRITE_LENMAX 252
 #define    MC_CMD_PTP_IN_FPGAWRITE_LEN(num) (12+1*(num))
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 #define       MC_CMD_PTP_IN_FPGAWRITE_ADDR_OFST 8
+#define       MC_CMD_PTP_IN_FPGAWRITE_ADDR_LEN 4
 #define       MC_CMD_PTP_IN_FPGAWRITE_BUFFER_OFST 12
 #define       MC_CMD_PTP_IN_FPGAWRITE_BUFFER_LEN 1
 #define       MC_CMD_PTP_IN_FPGAWRITE_BUFFER_MINNUM 1
@@ -1332,34 +1602,67 @@
 /* MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST msgrequest */
 #define    MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_LEN 16
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Time adjustment in seconds */
 #define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_SECONDS_OFST 8
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_SECONDS_LEN 4
 /* Time adjustment major value */
 #define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_MAJOR_OFST 8
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_MAJOR_LEN 4
 /* Time adjustment in nanoseconds */
 #define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_NANOSECONDS_OFST 12
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_NANOSECONDS_LEN 4
 /* Time adjustment minor value */
 #define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_MINOR_OFST 12
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_MINOR_LEN 4
+
+/* MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2 msgrequest */
+#define    MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_LEN 20
+/*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
+/* Time adjustment in seconds */
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_SECONDS_OFST 8
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_SECONDS_LEN 4
+/* Time adjustment major value */
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_MAJOR_OFST 8
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_MAJOR_LEN 4
+/* Time adjustment in nanoseconds */
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_NANOSECONDS_OFST 12
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_NANOSECONDS_LEN 4
+/* Time adjustment minor value */
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_MINOR_OFST 12
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_MINOR_LEN 4
+/* Upper 32bits of major time offset adjustment */
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_MAJOR_HI_OFST 16
+#define       MC_CMD_PTP_IN_CLOCK_OFFSET_ADJUST_V2_MAJOR_HI_LEN 4
 
 /* MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST msgrequest */
 #define    MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_LEN 16
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Frequency adjustment 40 bit fixed point ns */
 #define       MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_OFST 8
 #define       MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LEN 8
 #define       MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_LO_OFST 8
 #define       MC_CMD_PTP_IN_CLOCK_FREQ_ADJUST_FREQ_HI_OFST 12
-/* enum: Number of fractional bits in frequency adjustment */
-/*               MC_CMD_PTP_IN_ADJUST_BITS 0x28 */
+/*            Enum values, see field(s): */
+/*               MC_CMD_PTP/MC_CMD_PTP_IN_ADJUST/FREQ */
 
 /* MC_CMD_PTP_IN_RX_SET_VLAN_FILTER msgrequest */
 #define    MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_LEN 24
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Number of VLAN tags, 0 if not VLAN */
 #define       MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_NUM_VLAN_TAGS_OFST 8
+#define       MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_NUM_VLAN_TAGS_LEN 4
 /* Set of VLAN tags to filter against */
 #define       MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_VLAN_TAG_OFST 12
 #define       MC_CMD_PTP_IN_RX_SET_VLAN_FILTER_VLAN_TAG_LEN 4
@@ -1368,9 +1671,12 @@
 /* MC_CMD_PTP_IN_RX_SET_UUID_FILTER msgrequest */
 #define    MC_CMD_PTP_IN_RX_SET_UUID_FILTER_LEN 20
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* 1 to enable UUID filtering, 0 to disable */
 #define       MC_CMD_PTP_IN_RX_SET_UUID_FILTER_ENABLE_OFST 8
+#define       MC_CMD_PTP_IN_RX_SET_UUID_FILTER_ENABLE_LEN 4
 /* UUID to filter against */
 #define       MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_OFST 12
 #define       MC_CMD_PTP_IN_RX_SET_UUID_FILTER_UUID_LEN 8
@@ -1380,18 +1686,25 @@
 /* MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER msgrequest */
 #define    MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_LEN 16
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* 1 to enable Domain filtering, 0 to disable */
 #define       MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_ENABLE_OFST 8
+#define       MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_ENABLE_LEN 4
 /* Domain number to filter against */
 #define       MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_DOMAIN_OFST 12
+#define       MC_CMD_PTP_IN_RX_SET_DOMAIN_FILTER_DOMAIN_LEN 4
 
 /* MC_CMD_PTP_IN_SET_CLK_SRC msgrequest */
 #define    MC_CMD_PTP_IN_SET_CLK_SRC_LEN 12
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Set the clock source. */
 #define       MC_CMD_PTP_IN_SET_CLK_SRC_CLK_OFST 8
+#define       MC_CMD_PTP_IN_SET_CLK_SRC_CLK_LEN 4
 /* enum: Internal. */
 #define          MC_CMD_PTP_CLK_SRC_INTERNAL 0x0
 /* enum: External. */
@@ -1400,42 +1713,56 @@
 /* MC_CMD_PTP_IN_RST_CLK msgrequest */
 #define    MC_CMD_PTP_IN_RST_CLK_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /* Reset value of Timer Reg. */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_PPS_ENABLE msgrequest */
 #define    MC_CMD_PTP_IN_PPS_ENABLE_LEN 12
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /* Enable or disable */
 #define       MC_CMD_PTP_IN_PPS_ENABLE_OP_OFST 4
+#define       MC_CMD_PTP_IN_PPS_ENABLE_OP_LEN 4
 /* enum: Enable */
 #define          MC_CMD_PTP_ENABLE_PPS 0x0
 /* enum: Disable */
 #define          MC_CMD_PTP_DISABLE_PPS 0x1
-/* Queue id to send events back */
+/* Not used. Events are always sent to function relative queue 0. */
 #define       MC_CMD_PTP_IN_PPS_ENABLE_QUEUE_ID_OFST 8
+#define       MC_CMD_PTP_IN_PPS_ENABLE_QUEUE_ID_LEN 4
 
 /* MC_CMD_PTP_IN_GET_TIME_FORMAT msgrequest */
 #define    MC_CMD_PTP_IN_GET_TIME_FORMAT_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_GET_ATTRIBUTES msgrequest */
 #define    MC_CMD_PTP_IN_GET_ATTRIBUTES_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_GET_TIMESTAMP_CORRECTIONS msgrequest */
 #define    MC_CMD_PTP_IN_GET_TIMESTAMP_CORRECTIONS_LEN 8
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 
 /* MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE msgrequest */
 #define    MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_LEN 12
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Original field containing queue ID. Now extended to include flags. */
 #define       MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE_OFST 8
+#define       MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE_LEN 4
 #define        MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE_ID_LBN 0
 #define        MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE_ID_WIDTH 16
 #define        MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_REPORT_SYNC_STATUS_LBN 31
@@ -1444,29 +1771,39 @@
 /* MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE msgrequest */
 #define    MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_LEN 16
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* Unsubscribe options */
 #define       MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_CONTROL_OFST 8
+#define       MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_CONTROL_LEN 4
 /* enum: Unsubscribe a single queue */
 #define          MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_SINGLE 0x0
 /* enum: Unsubscribe all queues */
 #define          MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_ALL 0x1
 /* Event queue ID */
 #define       MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_QUEUE_OFST 12
+#define       MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_QUEUE_LEN 4
 
 /* MC_CMD_PTP_IN_MANFTEST_PPS msgrequest */
 #define    MC_CMD_PTP_IN_MANFTEST_PPS_LEN 12
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* 1 to enable PPS test mode, 0 to disable and return result. */
 #define       MC_CMD_PTP_IN_MANFTEST_PPS_TEST_ENABLE_OFST 8
+#define       MC_CMD_PTP_IN_MANFTEST_PPS_TEST_ENABLE_LEN 4
 
 /* MC_CMD_PTP_IN_SET_SYNC_STATUS msgrequest */
 #define    MC_CMD_PTP_IN_SET_SYNC_STATUS_LEN 24
 /*            MC_CMD_PTP_IN_CMD_OFST 0 */
+/*            MC_CMD_PTP_IN_CMD_LEN 4 */
 /*            MC_CMD_PTP_IN_PERIPH_ID_OFST 4 */
+/*            MC_CMD_PTP_IN_PERIPH_ID_LEN 4 */
 /* NIC - Host System Clock Synchronization status */
 #define       MC_CMD_PTP_IN_SET_SYNC_STATUS_STATUS_OFST 8
+#define       MC_CMD_PTP_IN_SET_SYNC_STATUS_STATUS_LEN 4
 /* enum: Host System clock and NIC clock are not in sync */
 #define          MC_CMD_PTP_IN_SET_SYNC_STATUS_NOT_IN_SYNC 0x0
 /* enum: Host System clock and NIC clock are synchronized */
@@ -1475,8 +1812,11 @@
  * no longer in sync.
  */
 #define       MC_CMD_PTP_IN_SET_SYNC_STATUS_TIMEOUT_OFST 12
+#define       MC_CMD_PTP_IN_SET_SYNC_STATUS_TIMEOUT_LEN 4
 #define       MC_CMD_PTP_IN_SET_SYNC_STATUS_RESERVED0_OFST 16
+#define       MC_CMD_PTP_IN_SET_SYNC_STATUS_RESERVED0_LEN 4
 #define       MC_CMD_PTP_IN_SET_SYNC_STATUS_RESERVED1_OFST 20
+#define       MC_CMD_PTP_IN_SET_SYNC_STATUS_RESERVED1_LEN 4
 
 /* MC_CMD_PTP_OUT msgresponse */
 #define    MC_CMD_PTP_OUT_LEN 0
@@ -1485,12 +1825,16 @@
 #define    MC_CMD_PTP_OUT_TRANSMIT_LEN 8
 /* Value of seconds timestamp */
 #define       MC_CMD_PTP_OUT_TRANSMIT_SECONDS_OFST 0
+#define       MC_CMD_PTP_OUT_TRANSMIT_SECONDS_LEN 4
 /* Timestamp major value */
 #define       MC_CMD_PTP_OUT_TRANSMIT_MAJOR_OFST 0
+#define       MC_CMD_PTP_OUT_TRANSMIT_MAJOR_LEN 4
 /* Value of nanoseconds timestamp */
 #define       MC_CMD_PTP_OUT_TRANSMIT_NANOSECONDS_OFST 4
+#define       MC_CMD_PTP_OUT_TRANSMIT_NANOSECONDS_LEN 4
 /* Timestamp minor value */
 #define       MC_CMD_PTP_OUT_TRANSMIT_MINOR_OFST 4
+#define       MC_CMD_PTP_OUT_TRANSMIT_MINOR_LEN 4
 
 /* MC_CMD_PTP_OUT_TIME_EVENT_SUBSCRIBE msgresponse */
 #define    MC_CMD_PTP_OUT_TIME_EVENT_SUBSCRIBE_LEN 0
@@ -1502,47 +1846,85 @@
 #define    MC_CMD_PTP_OUT_READ_NIC_TIME_LEN 8
 /* Value of seconds timestamp */
 #define       MC_CMD_PTP_OUT_READ_NIC_TIME_SECONDS_OFST 0
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_SECONDS_LEN 4
 /* Timestamp major value */
 #define       MC_CMD_PTP_OUT_READ_NIC_TIME_MAJOR_OFST 0
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_MAJOR_LEN 4
 /* Value of nanoseconds timestamp */
 #define       MC_CMD_PTP_OUT_READ_NIC_TIME_NANOSECONDS_OFST 4
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_NANOSECONDS_LEN 4
 /* Timestamp minor value */
 #define       MC_CMD_PTP_OUT_READ_NIC_TIME_MINOR_OFST 4
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_MINOR_LEN 4
+
+/* MC_CMD_PTP_OUT_READ_NIC_TIME_V2 msgresponse */
+#define    MC_CMD_PTP_OUT_READ_NIC_TIME_V2_LEN 12
+/* Value of seconds timestamp */
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_SECONDS_OFST 0
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_SECONDS_LEN 4
+/* Timestamp major value */
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_MAJOR_OFST 0
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_MAJOR_LEN 4
+/* Value of nanoseconds timestamp */
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_NANOSECONDS_OFST 4
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_NANOSECONDS_LEN 4
+/* Timestamp minor value */
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_MINOR_OFST 4
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_MINOR_LEN 4
+/* Upper 32bits of major timestamp value */
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_MAJOR_HI_OFST 8
+#define       MC_CMD_PTP_OUT_READ_NIC_TIME_V2_MAJOR_HI_LEN 4
 
 /* MC_CMD_PTP_OUT_STATUS msgresponse */
 #define    MC_CMD_PTP_OUT_STATUS_LEN 64
 /* Frequency of NIC's hardware clock */
 #define       MC_CMD_PTP_OUT_STATUS_CLOCK_FREQ_OFST 0
+#define       MC_CMD_PTP_OUT_STATUS_CLOCK_FREQ_LEN 4
 /* Number of packets transmitted and timestamped */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_TX_OFST 4
+#define       MC_CMD_PTP_OUT_STATUS_STATS_TX_LEN 4
 /* Number of packets received and timestamped */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_RX_OFST 8
+#define       MC_CMD_PTP_OUT_STATUS_STATS_RX_LEN 4
 /* Number of packets timestamped by the FPGA */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_TS_OFST 12
+#define       MC_CMD_PTP_OUT_STATUS_STATS_TS_LEN 4
 /* Number of packets filter matched */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_FM_OFST 16
+#define       MC_CMD_PTP_OUT_STATUS_STATS_FM_LEN 4
 /* Number of packets not filter matched */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_NFM_OFST 20
+#define       MC_CMD_PTP_OUT_STATUS_STATS_NFM_LEN 4
 /* Number of PPS overflows (noise on input?) */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFLOW_OFST 24
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFLOW_LEN 4
 /* Number of PPS bad periods */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_BAD_OFST 28
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_BAD_LEN 4
 /* Minimum period of PPS pulse in nanoseconds */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MIN_OFST 32
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MIN_LEN 4
 /* Maximum period of PPS pulse in nanoseconds */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MAX_OFST 36
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MAX_LEN 4
 /* Last period of PPS pulse in nanoseconds */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_LAST_OFST 40
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_LAST_LEN 4
 /* Mean period of PPS pulse in nanoseconds */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MEAN_OFST 44
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_PER_MEAN_LEN 4
 /* Minimum offset of PPS pulse in nanoseconds (signed) */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MIN_OFST 48
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MIN_LEN 4
 /* Maximum offset of PPS pulse in nanoseconds (signed) */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MAX_OFST 52
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MAX_LEN 4
 /* Last offset of PPS pulse in nanoseconds (signed) */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_LAST_OFST 56
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_LAST_LEN 4
 /* Mean offset of PPS pulse in nanoseconds (signed) */
 #define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MEAN_OFST 60
+#define       MC_CMD_PTP_OUT_STATUS_STATS_PPS_OFF_MEAN_LEN 4
 
 /* MC_CMD_PTP_OUT_SYNCHRONIZE msgresponse */
 #define    MC_CMD_PTP_OUT_SYNCHRONIZE_LENMIN 20
@@ -1555,23 +1937,31 @@
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_TIMESET_MAXNUM 12
 /* Host time immediately before NIC's hardware clock read */
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_HOSTSTART_OFST 0
+#define       MC_CMD_PTP_OUT_SYNCHRONIZE_HOSTSTART_LEN 4
 /* Value of seconds timestamp */
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_SECONDS_OFST 4
+#define       MC_CMD_PTP_OUT_SYNCHRONIZE_SECONDS_LEN 4
 /* Timestamp major value */
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_MAJOR_OFST 4
+#define       MC_CMD_PTP_OUT_SYNCHRONIZE_MAJOR_LEN 4
 /* Value of nanoseconds timestamp */
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_NANOSECONDS_OFST 8
+#define       MC_CMD_PTP_OUT_SYNCHRONIZE_NANOSECONDS_LEN 4
 /* Timestamp minor value */
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_MINOR_OFST 8
+#define       MC_CMD_PTP_OUT_SYNCHRONIZE_MINOR_LEN 4
 /* Host time immediately after NIC's hardware clock read */
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_HOSTEND_OFST 12
+#define       MC_CMD_PTP_OUT_SYNCHRONIZE_HOSTEND_LEN 4
 /* Number of nanoseconds waited after reading NIC's hardware clock */
 #define       MC_CMD_PTP_OUT_SYNCHRONIZE_WAITNS_OFST 16
+#define       MC_CMD_PTP_OUT_SYNCHRONIZE_WAITNS_LEN 4
 
 /* MC_CMD_PTP_OUT_MANFTEST_BASIC msgresponse */
 #define    MC_CMD_PTP_OUT_MANFTEST_BASIC_LEN 8
 /* Results of testing */
 #define       MC_CMD_PTP_OUT_MANFTEST_BASIC_TEST_RESULT_OFST 0
+#define       MC_CMD_PTP_OUT_MANFTEST_BASIC_TEST_RESULT_LEN 4
 /* enum: Successful test */
 #define          MC_CMD_PTP_MANF_SUCCESS 0x0
 /* enum: FPGA load failed */
@@ -1604,15 +1994,19 @@
 #define          MC_CMD_PTP_MANF_CLOCK_READ 0xe
 /* Presence of external oscillator */
 #define       MC_CMD_PTP_OUT_MANFTEST_BASIC_TEST_EXTOSC_OFST 4
+#define       MC_CMD_PTP_OUT_MANFTEST_BASIC_TEST_EXTOSC_LEN 4
 
 /* MC_CMD_PTP_OUT_MANFTEST_PACKET msgresponse */
 #define    MC_CMD_PTP_OUT_MANFTEST_PACKET_LEN 12
 /* Results of testing */
 #define       MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_RESULT_OFST 0
+#define       MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_RESULT_LEN 4
 /* Number of packets received by FPGA */
 #define       MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_FPGACOUNT_OFST 4
+#define       MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_FPGACOUNT_LEN 4
 /* Number of packets received by Siena filters */
 #define       MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_FILTERCOUNT_OFST 8
+#define       MC_CMD_PTP_OUT_MANFTEST_PACKET_TEST_FILTERCOUNT_LEN 4
 
 /* MC_CMD_PTP_OUT_FPGAREAD msgresponse */
 #define    MC_CMD_PTP_OUT_FPGAREAD_LENMIN 1
@@ -1628,9 +2022,11 @@
 /* Time format required/used by for this NIC. Applies to all PTP MCDI
  * operations that pass times between the host and firmware. If this operation
  * is not supported (older firmware) a format of seconds and nanoseconds should
- * be assumed.
+ * be assumed. Note this enum is deprecated. Do not add to it- use the
+ * TIME_FORMAT field in MC_CMD_PTP_OUT_GET_ATTRIBUTES instead.
  */
 #define       MC_CMD_PTP_OUT_GET_TIME_FORMAT_FORMAT_OFST 0
+#define       MC_CMD_PTP_OUT_GET_TIME_FORMAT_FORMAT_LEN 4
 /* enum: Times are in seconds and nanoseconds */
 #define          MC_CMD_PTP_OUT_GET_TIME_FORMAT_SECONDS_NANOSECONDS 0x0
 /* enum: Major register has units of 16 second per tick, minor 8 ns per tick */
@@ -1646,12 +2042,16 @@
  * be assumed.
  */
 #define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_TIME_FORMAT_OFST 0
+#define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_TIME_FORMAT_LEN 4
 /* enum: Times are in seconds and nanoseconds */
 #define          MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_NANOSECONDS 0x0
 /* enum: Major register has units of 16 second per tick, minor 8 ns per tick */
 #define          MC_CMD_PTP_OUT_GET_ATTRIBUTES_16SECONDS_8NANOSECONDS 0x1
 /* enum: Major register has units of seconds, minor 2^-27s per tick */
 #define          MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_27FRACTION 0x2
+/* enum: Major register units are seconds, minor units are quarter nanoseconds
+ */
+#define          MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_QTR_NANOSECONDS 0x3
 /* Minimum acceptable value for a corrected synchronization timeset. When
  * comparing host and NIC clock times, the MC returns a set of samples that
  * contain the host start and end time, the MC time when the host start was
@@ -1660,46 +2060,66 @@
  * end and start times minus the time that the MC waited for host end.
  */
 #define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_SYNC_WINDOW_MIN_OFST 4
+#define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_SYNC_WINDOW_MIN_LEN 4
 /* Various PTP capabilities */
 #define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_CAPABILITIES_OFST 8
+#define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_CAPABILITIES_LEN 4
 #define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_REPORT_SYNC_STATUS_LBN 0
 #define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_REPORT_SYNC_STATUS_WIDTH 1
 #define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_RX_TSTAMP_OOB_LBN 1
 #define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_RX_TSTAMP_OOB_WIDTH 1
+#define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_64BIT_SECONDS_LBN 2
+#define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_64BIT_SECONDS_WIDTH 1
+#define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_FP44_FREQ_ADJ_LBN 3
+#define        MC_CMD_PTP_OUT_GET_ATTRIBUTES_FP44_FREQ_ADJ_WIDTH 1
 #define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED0_OFST 12
+#define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED0_LEN 4
 #define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED1_OFST 16
+#define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED1_LEN 4
 #define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED2_OFST 20
+#define       MC_CMD_PTP_OUT_GET_ATTRIBUTES_RESERVED2_LEN 4
 
 /* MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS msgresponse */
 #define    MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_LEN 16
 /* Uncorrected error on PTP transmit timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_TRANSMIT_OFST 0
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_TRANSMIT_LEN 4
 /* Uncorrected error on PTP receive timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_RECEIVE_OFST 4
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_RECEIVE_LEN 4
 /* Uncorrected error on PPS output in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_OUT_OFST 8
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_OUT_LEN 4
 /* Uncorrected error on PPS input in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_IN_OFST 12
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_IN_LEN 4
 
 /* MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2 msgresponse */
 #define    MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_LEN 24
 /* Uncorrected error on PTP transmit timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_TX_OFST 0
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_TX_LEN 4
 /* Uncorrected error on PTP receive timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_RX_OFST 4
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PTP_RX_LEN 4
 /* Uncorrected error on PPS output in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_OUT_OFST 8
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_OUT_LEN 4
 /* Uncorrected error on PPS input in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_IN_OFST 12
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_PPS_IN_LEN 4
 /* Uncorrected error on non-PTP transmit timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_TX_OFST 16
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_TX_LEN 4
 /* Uncorrected error on non-PTP receive timestamps in NIC clock format */
 #define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_RX_OFST 20
+#define       MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_RX_LEN 4
 
 /* MC_CMD_PTP_OUT_MANFTEST_PPS msgresponse */
 #define    MC_CMD_PTP_OUT_MANFTEST_PPS_LEN 4
 /* Results of testing */
 #define       MC_CMD_PTP_OUT_MANFTEST_PPS_TEST_RESULT_OFST 0
+#define       MC_CMD_PTP_OUT_MANFTEST_PPS_TEST_RESULT_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_PTP_OUT_MANFTEST_BASIC/TEST_RESULT */
 
@@ -1713,14 +2133,17 @@
  */
 #define MC_CMD_CSR_READ32 0xc
 
-#define MC_CMD_0xc_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xc_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_CSR_READ32_IN msgrequest */
 #define    MC_CMD_CSR_READ32_IN_LEN 12
 /* Address */
 #define       MC_CMD_CSR_READ32_IN_ADDR_OFST 0
+#define       MC_CMD_CSR_READ32_IN_ADDR_LEN 4
 #define       MC_CMD_CSR_READ32_IN_STEP_OFST 4
+#define       MC_CMD_CSR_READ32_IN_STEP_LEN 4
 #define       MC_CMD_CSR_READ32_IN_NUMWORDS_OFST 8
+#define       MC_CMD_CSR_READ32_IN_NUMWORDS_LEN 4
 
 /* MC_CMD_CSR_READ32_OUT msgresponse */
 #define    MC_CMD_CSR_READ32_OUT_LENMIN 4
@@ -1739,7 +2162,7 @@
  */
 #define MC_CMD_CSR_WRITE32 0xd
 
-#define MC_CMD_0xd_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xd_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_CSR_WRITE32_IN msgrequest */
 #define    MC_CMD_CSR_WRITE32_IN_LENMIN 12
@@ -1747,7 +2170,9 @@
 #define    MC_CMD_CSR_WRITE32_IN_LEN(num) (8+4*(num))
 /* Address */
 #define       MC_CMD_CSR_WRITE32_IN_ADDR_OFST 0
+#define       MC_CMD_CSR_WRITE32_IN_ADDR_LEN 4
 #define       MC_CMD_CSR_WRITE32_IN_STEP_OFST 4
+#define       MC_CMD_CSR_WRITE32_IN_STEP_LEN 4
 #define       MC_CMD_CSR_WRITE32_IN_BUFFER_OFST 8
 #define       MC_CMD_CSR_WRITE32_IN_BUFFER_LEN 4
 #define       MC_CMD_CSR_WRITE32_IN_BUFFER_MINNUM 1
@@ -1756,6 +2181,7 @@
 /* MC_CMD_CSR_WRITE32_OUT msgresponse */
 #define    MC_CMD_CSR_WRITE32_OUT_LEN 4
 #define       MC_CMD_CSR_WRITE32_OUT_STATUS_OFST 0
+#define       MC_CMD_CSR_WRITE32_OUT_STATUS_LEN 4
 
 
 /***********************************/
@@ -1776,6 +2202,7 @@
  * sensors.
  */
 #define       MC_CMD_HP_IN_SUBCMD_OFST 0
+#define       MC_CMD_HP_IN_SUBCMD_LEN 4
 /* enum: OCSD (Option Card Sensor Data) sub-command. */
 #define          MC_CMD_HP_IN_OCSD_SUBCMD 0x0
 /* enum: Last known valid HP sub-command. */
@@ -1790,10 +2217,12 @@
  * NULL.)
  */
 #define       MC_CMD_HP_IN_OCSD_INTERVAL_OFST 12
+#define       MC_CMD_HP_IN_OCSD_INTERVAL_LEN 4
 
 /* MC_CMD_HP_OUT msgresponse */
 #define    MC_CMD_HP_OUT_LEN 4
 #define       MC_CMD_HP_OUT_OCSD_STATUS_OFST 0
+#define       MC_CMD_HP_OUT_OCSD_STATUS_LEN 4
 /* enum: OCSD stopped for this card. */
 #define          MC_CMD_HP_OUT_OCSD_STOPPED 0x1
 /* enum: OCSD was successfully started with the address provided. */
@@ -1838,29 +2267,35 @@
  * external devices.
  */
 #define       MC_CMD_MDIO_READ_IN_BUS_OFST 0
+#define       MC_CMD_MDIO_READ_IN_BUS_LEN 4
 /* enum: Internal. */
 #define          MC_CMD_MDIO_BUS_INTERNAL 0x0
 /* enum: External. */
 #define          MC_CMD_MDIO_BUS_EXTERNAL 0x1
 /* Port address */
 #define       MC_CMD_MDIO_READ_IN_PRTAD_OFST 4
+#define       MC_CMD_MDIO_READ_IN_PRTAD_LEN 4
 /* Device Address or clause 22. */
 #define       MC_CMD_MDIO_READ_IN_DEVAD_OFST 8
+#define       MC_CMD_MDIO_READ_IN_DEVAD_LEN 4
 /* enum: By default all the MCDI MDIO operations perform clause45 mode. If you
  * want to use clause22 then set DEVAD = MC_CMD_MDIO_CLAUSE22.
  */
 #define          MC_CMD_MDIO_CLAUSE22 0x20
 /* Address */
 #define       MC_CMD_MDIO_READ_IN_ADDR_OFST 12
+#define       MC_CMD_MDIO_READ_IN_ADDR_LEN 4
 
 /* MC_CMD_MDIO_READ_OUT msgresponse */
 #define    MC_CMD_MDIO_READ_OUT_LEN 8
 /* Value */
 #define       MC_CMD_MDIO_READ_OUT_VALUE_OFST 0
+#define       MC_CMD_MDIO_READ_OUT_VALUE_LEN 4
 /* Status the MDIO commands return the raw status bits from the MDIO block. A
  * "good" transaction should have the DONE bit set and all other bits clear.
  */
 #define       MC_CMD_MDIO_READ_OUT_STATUS_OFST 4
+#define       MC_CMD_MDIO_READ_OUT_STATUS_LEN 4
 /* enum: Good. */
 #define          MC_CMD_MDIO_STATUS_GOOD 0x8
 
@@ -1879,22 +2314,27 @@
  * external devices.
  */
 #define       MC_CMD_MDIO_WRITE_IN_BUS_OFST 0
+#define       MC_CMD_MDIO_WRITE_IN_BUS_LEN 4
 /* enum: Internal. */
 /*               MC_CMD_MDIO_BUS_INTERNAL 0x0 */
 /* enum: External. */
 /*               MC_CMD_MDIO_BUS_EXTERNAL 0x1 */
 /* Port address */
 #define       MC_CMD_MDIO_WRITE_IN_PRTAD_OFST 4
+#define       MC_CMD_MDIO_WRITE_IN_PRTAD_LEN 4
 /* Device Address or clause 22. */
 #define       MC_CMD_MDIO_WRITE_IN_DEVAD_OFST 8
+#define       MC_CMD_MDIO_WRITE_IN_DEVAD_LEN 4
 /* enum: By default all the MCDI MDIO operations perform clause45 mode. If you
  * want to use clause22 then set DEVAD = MC_CMD_MDIO_CLAUSE22.
  */
 /*               MC_CMD_MDIO_CLAUSE22 0x20 */
 /* Address */
 #define       MC_CMD_MDIO_WRITE_IN_ADDR_OFST 12
+#define       MC_CMD_MDIO_WRITE_IN_ADDR_LEN 4
 /* Value */
 #define       MC_CMD_MDIO_WRITE_IN_VALUE_OFST 16
+#define       MC_CMD_MDIO_WRITE_IN_VALUE_LEN 4
 
 /* MC_CMD_MDIO_WRITE_OUT msgresponse */
 #define    MC_CMD_MDIO_WRITE_OUT_LEN 4
@@ -1902,6 +2342,7 @@
  * "good" transaction should have the DONE bit set and all other bits clear.
  */
 #define       MC_CMD_MDIO_WRITE_OUT_STATUS_OFST 0
+#define       MC_CMD_MDIO_WRITE_OUT_STATUS_LEN 4
 /* enum: Good. */
 /*               MC_CMD_MDIO_STATUS_GOOD 0x8 */
 
@@ -1912,7 +2353,7 @@
  */
 #define MC_CMD_DBI_WRITE 0x12
 
-#define MC_CMD_0x12_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x12_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_DBI_WRITE_IN msgrequest */
 #define    MC_CMD_DBI_WRITE_IN_LENMIN 12
@@ -1932,9 +2373,11 @@
 /* MC_CMD_DBIWROP_TYPEDEF structuredef */
 #define    MC_CMD_DBIWROP_TYPEDEF_LEN 12
 #define       MC_CMD_DBIWROP_TYPEDEF_ADDRESS_OFST 0
+#define       MC_CMD_DBIWROP_TYPEDEF_ADDRESS_LEN 4
 #define       MC_CMD_DBIWROP_TYPEDEF_ADDRESS_LBN 0
 #define       MC_CMD_DBIWROP_TYPEDEF_ADDRESS_WIDTH 32
 #define       MC_CMD_DBIWROP_TYPEDEF_PARMS_OFST 4
+#define       MC_CMD_DBIWROP_TYPEDEF_PARMS_LEN 4
 #define        MC_CMD_DBIWROP_TYPEDEF_VF_NUM_LBN 16
 #define        MC_CMD_DBIWROP_TYPEDEF_VF_NUM_WIDTH 16
 #define        MC_CMD_DBIWROP_TYPEDEF_VF_ACTIVE_LBN 15
@@ -1944,6 +2387,7 @@
 #define       MC_CMD_DBIWROP_TYPEDEF_PARMS_LBN 32
 #define       MC_CMD_DBIWROP_TYPEDEF_PARMS_WIDTH 32
 #define       MC_CMD_DBIWROP_TYPEDEF_VALUE_OFST 8
+#define       MC_CMD_DBIWROP_TYPEDEF_VALUE_LEN 4
 #define       MC_CMD_DBIWROP_TYPEDEF_VALUE_LBN 64
 #define       MC_CMD_DBIWROP_TYPEDEF_VALUE_WIDTH 32
 
@@ -1959,13 +2403,16 @@
 #define    MC_CMD_PORT_READ32_IN_LEN 4
 /* Address */
 #define       MC_CMD_PORT_READ32_IN_ADDR_OFST 0
+#define       MC_CMD_PORT_READ32_IN_ADDR_LEN 4
 
 /* MC_CMD_PORT_READ32_OUT msgresponse */
 #define    MC_CMD_PORT_READ32_OUT_LEN 8
 /* Value */
 #define       MC_CMD_PORT_READ32_OUT_VALUE_OFST 0
+#define       MC_CMD_PORT_READ32_OUT_VALUE_LEN 4
 /* Status */
 #define       MC_CMD_PORT_READ32_OUT_STATUS_OFST 4
+#define       MC_CMD_PORT_READ32_OUT_STATUS_LEN 4
 
 
 /***********************************/
@@ -1979,13 +2426,16 @@
 #define    MC_CMD_PORT_WRITE32_IN_LEN 8
 /* Address */
 #define       MC_CMD_PORT_WRITE32_IN_ADDR_OFST 0
+#define       MC_CMD_PORT_WRITE32_IN_ADDR_LEN 4
 /* Value */
 #define       MC_CMD_PORT_WRITE32_IN_VALUE_OFST 4
+#define       MC_CMD_PORT_WRITE32_IN_VALUE_LEN 4
 
 /* MC_CMD_PORT_WRITE32_OUT msgresponse */
 #define    MC_CMD_PORT_WRITE32_OUT_LEN 4
 /* Status */
 #define       MC_CMD_PORT_WRITE32_OUT_STATUS_OFST 0
+#define       MC_CMD_PORT_WRITE32_OUT_STATUS_LEN 4
 
 
 /***********************************/
@@ -1999,6 +2449,7 @@
 #define    MC_CMD_PORT_READ128_IN_LEN 4
 /* Address */
 #define       MC_CMD_PORT_READ128_IN_ADDR_OFST 0
+#define       MC_CMD_PORT_READ128_IN_ADDR_LEN 4
 
 /* MC_CMD_PORT_READ128_OUT msgresponse */
 #define    MC_CMD_PORT_READ128_OUT_LEN 20
@@ -2007,6 +2458,7 @@
 #define       MC_CMD_PORT_READ128_OUT_VALUE_LEN 16
 /* Status */
 #define       MC_CMD_PORT_READ128_OUT_STATUS_OFST 16
+#define       MC_CMD_PORT_READ128_OUT_STATUS_LEN 4
 
 
 /***********************************/
@@ -2020,6 +2472,7 @@
 #define    MC_CMD_PORT_WRITE128_IN_LEN 20
 /* Address */
 #define       MC_CMD_PORT_WRITE128_IN_ADDR_OFST 0
+#define       MC_CMD_PORT_WRITE128_IN_ADDR_LEN 4
 /* Value */
 #define       MC_CMD_PORT_WRITE128_IN_VALUE_OFST 4
 #define       MC_CMD_PORT_WRITE128_IN_VALUE_LEN 16
@@ -2028,6 +2481,7 @@
 #define    MC_CMD_PORT_WRITE128_OUT_LEN 4
 /* Status */
 #define       MC_CMD_PORT_WRITE128_OUT_STATUS_OFST 0
+#define       MC_CMD_PORT_WRITE128_OUT_STATUS_LEN 4
 
 /* MC_CMD_CAPABILITIES structuredef */
 #define    MC_CMD_CAPABILITIES_LEN 4
@@ -2072,24 +2526,54 @@
 #define    MC_CMD_GET_BOARD_CFG_OUT_LENMAX 136
 #define    MC_CMD_GET_BOARD_CFG_OUT_LEN(num) (72+2*(num))
 #define       MC_CMD_GET_BOARD_CFG_OUT_BOARD_TYPE_OFST 0
+#define       MC_CMD_GET_BOARD_CFG_OUT_BOARD_TYPE_LEN 4
 #define       MC_CMD_GET_BOARD_CFG_OUT_BOARD_NAME_OFST 4
 #define       MC_CMD_GET_BOARD_CFG_OUT_BOARD_NAME_LEN 32
-/* See MC_CMD_CAPABILITIES */
+/* Capabilities for Siena Port0 (see struct MC_CMD_CAPABILITIES). Unused on
+ * EF10 and later (use MC_CMD_GET_CAPABILITIES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT0_OFST 36
-/* See MC_CMD_CAPABILITIES */
+#define       MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT0_LEN 4
+/* Capabilities for Siena Port1 (see struct MC_CMD_CAPABILITIES). Unused on
+ * EF10 and later (use MC_CMD_GET_CAPABILITIES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT1_OFST 40
+#define       MC_CMD_GET_BOARD_CFG_OUT_CAPABILITIES_PORT1_LEN 4
+/* Base MAC address for Siena Port0. Unused on EF10 and later (use
+ * MC_CMD_GET_MAC_ADDRESSES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0_OFST 44
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0_LEN 6
+/* Base MAC address for Siena Port1. Unused on EF10 and later (use
+ * MC_CMD_GET_MAC_ADDRESSES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1_OFST 50
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1_LEN 6
+/* Size of MAC address pool for Siena Port0. Unused on EF10 and later (use
+ * MC_CMD_GET_MAC_ADDRESSES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT0_OFST 56
+#define       MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT0_LEN 4
+/* Size of MAC address pool for Siena Port1. Unused on EF10 and later (use
+ * MC_CMD_GET_MAC_ADDRESSES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT1_OFST 60
+#define       MC_CMD_GET_BOARD_CFG_OUT_MAC_COUNT_PORT1_LEN 4
+/* Increment between addresses in MAC address pool for Siena Port0. Unused on
+ * EF10 and later (use MC_CMD_GET_MAC_ADDRESSES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT0_OFST 64
+#define       MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT0_LEN 4
+/* Increment between addresses in MAC address pool for Siena Port1. Unused on
+ * EF10 and later (use MC_CMD_GET_MAC_ADDRESSES).
+ */
 #define       MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT1_OFST 68
-/* This field contains a 16-bit value for each of the types of NVRAM area. The
- * values are defined in the firmware/mc/platform/.c file for a specific board
- * type, but otherwise have no meaning to the MC; they are used by the driver
- * to manage selection of appropriate firmware updates.
+#define       MC_CMD_GET_BOARD_CFG_OUT_MAC_STRIDE_PORT1_LEN 4
+/* Siena only. This field contains a 16-bit value for each of the types of
+ * NVRAM area. The values are defined in the firmware/mc/platform/.c file for a
+ * specific board type, but otherwise have no meaning to the MC; they are used
+ * by the driver to manage selection of appropriate firmware updates. Unused on
+ * EF10 and later (use MC_CMD_NVRAM_METADATA).
  */
 #define       MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_OFST 72
 #define       MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_LEN 2
@@ -2103,7 +2587,7 @@
  */
 #define MC_CMD_DBI_READX 0x19
 
-#define MC_CMD_0x19_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x19_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_DBI_READX_IN msgrequest */
 #define    MC_CMD_DBI_READX_IN_LENMIN 8
@@ -2130,9 +2614,11 @@
 /* MC_CMD_DBIRDOP_TYPEDEF structuredef */
 #define    MC_CMD_DBIRDOP_TYPEDEF_LEN 8
 #define       MC_CMD_DBIRDOP_TYPEDEF_ADDRESS_OFST 0
+#define       MC_CMD_DBIRDOP_TYPEDEF_ADDRESS_LEN 4
 #define       MC_CMD_DBIRDOP_TYPEDEF_ADDRESS_LBN 0
 #define       MC_CMD_DBIRDOP_TYPEDEF_ADDRESS_WIDTH 32
 #define       MC_CMD_DBIRDOP_TYPEDEF_PARMS_OFST 4
+#define       MC_CMD_DBIRDOP_TYPEDEF_PARMS_LEN 4
 #define        MC_CMD_DBIRDOP_TYPEDEF_VF_NUM_LBN 16
 #define        MC_CMD_DBIRDOP_TYPEDEF_VF_NUM_WIDTH 16
 #define        MC_CMD_DBIRDOP_TYPEDEF_VF_ACTIVE_LBN 15
@@ -2149,7 +2635,7 @@
  */
 #define MC_CMD_SET_RAND_SEED 0x1a
 
-#define MC_CMD_0x1a_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x1a_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_SET_RAND_SEED_IN msgrequest */
 #define    MC_CMD_SET_RAND_SEED_IN_LEN 16
@@ -2198,14 +2684,17 @@
 #define    MC_CMD_DRV_ATTACH_IN_LEN 12
 /* new state to set if UPDATE=1 */
 #define       MC_CMD_DRV_ATTACH_IN_NEW_STATE_OFST 0
+#define       MC_CMD_DRV_ATTACH_IN_NEW_STATE_LEN 4
 #define        MC_CMD_DRV_ATTACH_LBN 0
 #define        MC_CMD_DRV_ATTACH_WIDTH 1
 #define        MC_CMD_DRV_PREBOOT_LBN 1
 #define        MC_CMD_DRV_PREBOOT_WIDTH 1
 /* 1 to set new state, or 0 to just report the existing state */
 #define       MC_CMD_DRV_ATTACH_IN_UPDATE_OFST 4
+#define       MC_CMD_DRV_ATTACH_IN_UPDATE_LEN 4
 /* preferred datapath firmware (for Huntington; ignored for Siena) */
 #define       MC_CMD_DRV_ATTACH_IN_FIRMWARE_ID_OFST 8
+#define       MC_CMD_DRV_ATTACH_IN_FIRMWARE_ID_LEN 4
 /* enum: Prefer to use full featured firmware */
 #define          MC_CMD_FW_FULL_FEATURED 0x0
 /* enum: Prefer to use firmware with fewer features but lower latency */
@@ -2229,13 +2718,16 @@
 #define    MC_CMD_DRV_ATTACH_OUT_LEN 4
 /* previous or existing state, see the bitmask at NEW_STATE */
 #define       MC_CMD_DRV_ATTACH_OUT_OLD_STATE_OFST 0
+#define       MC_CMD_DRV_ATTACH_OUT_OLD_STATE_LEN 4
 
 /* MC_CMD_DRV_ATTACH_EXT_OUT msgresponse */
 #define    MC_CMD_DRV_ATTACH_EXT_OUT_LEN 8
 /* previous or existing state, see the bitmask at NEW_STATE */
 #define       MC_CMD_DRV_ATTACH_EXT_OUT_OLD_STATE_OFST 0
+#define       MC_CMD_DRV_ATTACH_EXT_OUT_OLD_STATE_LEN 4
 /* Flags associated with this function */
 #define       MC_CMD_DRV_ATTACH_EXT_OUT_FUNC_FLAGS_OFST 4
+#define       MC_CMD_DRV_ATTACH_EXT_OUT_FUNC_FLAGS_LEN 4
 /* enum: Labels the lowest-numbered function visible to the OS */
 #define          MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_PRIMARY 0x0
 /* enum: The function can control the link state of the physical port it is
@@ -2260,6 +2752,7 @@
 #define    MC_CMD_SHMUART_IN_LEN 4
 /* ??? */
 #define       MC_CMD_SHMUART_IN_FLAG_OFST 0
+#define       MC_CMD_SHMUART_IN_FLAG_LEN 4
 
 /* MC_CMD_SHMUART_OUT msgresponse */
 #define    MC_CMD_SHMUART_OUT_LEN 0
@@ -2297,6 +2790,7 @@
  * (TBD).
  */
 #define       MC_CMD_ENTITY_RESET_IN_FLAG_OFST 0
+#define       MC_CMD_ENTITY_RESET_IN_FLAG_LEN 4
 #define        MC_CMD_ENTITY_RESET_IN_FUNCTION_RESOURCE_RESET_LBN 0
 #define        MC_CMD_ENTITY_RESET_IN_FUNCTION_RESOURCE_RESET_WIDTH 1
 
@@ -2314,8 +2808,10 @@
 #define    MC_CMD_PCIE_CREDITS_IN_LEN 8
 /* poll period. 0 is disabled */
 #define       MC_CMD_PCIE_CREDITS_IN_POLL_PERIOD_OFST 0
+#define       MC_CMD_PCIE_CREDITS_IN_POLL_PERIOD_LEN 4
 /* wipe statistics */
 #define       MC_CMD_PCIE_CREDITS_IN_WIPE_OFST 4
+#define       MC_CMD_PCIE_CREDITS_IN_WIPE_LEN 4
 
 /* MC_CMD_PCIE_CREDITS_OUT msgresponse */
 #define    MC_CMD_PCIE_CREDITS_OUT_LEN 16
@@ -2346,31 +2842,54 @@
 /* MC_CMD_RXD_MONITOR_IN msgrequest */
 #define    MC_CMD_RXD_MONITOR_IN_LEN 12
 #define       MC_CMD_RXD_MONITOR_IN_QID_OFST 0
+#define       MC_CMD_RXD_MONITOR_IN_QID_LEN 4
 #define       MC_CMD_RXD_MONITOR_IN_POLL_PERIOD_OFST 4
+#define       MC_CMD_RXD_MONITOR_IN_POLL_PERIOD_LEN 4
 #define       MC_CMD_RXD_MONITOR_IN_WIPE_OFST 8
+#define       MC_CMD_RXD_MONITOR_IN_WIPE_LEN 4
 
 /* MC_CMD_RXD_MONITOR_OUT msgresponse */
 #define    MC_CMD_RXD_MONITOR_OUT_LEN 80
 #define       MC_CMD_RXD_MONITOR_OUT_QID_OFST 0
+#define       MC_CMD_RXD_MONITOR_OUT_QID_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_FILL_OFST 4
+#define       MC_CMD_RXD_MONITOR_OUT_RING_FILL_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_FILL_OFST 8
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_FILL_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_1_OFST 12
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_1_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_2_OFST 16
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_2_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_4_OFST 20
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_4_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_8_OFST 24
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_8_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_16_OFST 28
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_16_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_32_OFST 32
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_32_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_64_OFST 36
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_64_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_128_OFST 40
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_128_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_LT_256_OFST 44
+#define       MC_CMD_RXD_MONITOR_OUT_RING_LT_256_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_RING_GE_256_OFST 48
+#define       MC_CMD_RXD_MONITOR_OUT_RING_GE_256_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_1_OFST 52
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_1_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_2_OFST 56
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_2_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_4_OFST 60
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_4_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_8_OFST 64
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_8_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_16_OFST 68
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_16_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_32_OFST 72
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_LT_32_LEN 4
 #define       MC_CMD_RXD_MONITOR_OUT_CACHE_GE_32_OFST 76
+#define       MC_CMD_RXD_MONITOR_OUT_CACHE_GE_32_LEN 4
 
 
 /***********************************/
@@ -2379,13 +2898,14 @@
  */
 #define MC_CMD_PUTS 0x23
 
-#define MC_CMD_0x23_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x23_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_PUTS_IN msgrequest */
 #define    MC_CMD_PUTS_IN_LENMIN 13
 #define    MC_CMD_PUTS_IN_LENMAX 252
 #define    MC_CMD_PUTS_IN_LEN(num) (12+1*(num))
 #define       MC_CMD_PUTS_IN_DEST_OFST 0
+#define       MC_CMD_PUTS_IN_DEST_LEN 4
 #define        MC_CMD_PUTS_IN_UART_LBN 0
 #define        MC_CMD_PUTS_IN_UART_WIDTH 1
 #define        MC_CMD_PUTS_IN_PORT_LBN 1
@@ -2417,6 +2937,7 @@
 #define    MC_CMD_GET_PHY_CFG_OUT_LEN 72
 /* flags */
 #define       MC_CMD_GET_PHY_CFG_OUT_FLAGS_OFST 0
+#define       MC_CMD_GET_PHY_CFG_OUT_FLAGS_LEN 4
 #define        MC_CMD_GET_PHY_CFG_OUT_PRESENT_LBN 0
 #define        MC_CMD_GET_PHY_CFG_OUT_PRESENT_WIDTH 1
 #define        MC_CMD_GET_PHY_CFG_OUT_BIST_CABLE_SHORT_LBN 1
@@ -2433,8 +2954,10 @@
 #define        MC_CMD_GET_PHY_CFG_OUT_BIST_WIDTH 1
 /* ?? */
 #define       MC_CMD_GET_PHY_CFG_OUT_TYPE_OFST 4
+#define       MC_CMD_GET_PHY_CFG_OUT_TYPE_LEN 4
 /* Bitmask of supported capabilities */
 #define       MC_CMD_GET_PHY_CFG_OUT_SUPPORTED_CAP_OFST 8
+#define       MC_CMD_GET_PHY_CFG_OUT_SUPPORTED_CAP_LEN 4
 #define        MC_CMD_PHY_CAP_10HDX_LBN 1
 #define        MC_CMD_PHY_CAP_10HDX_WIDTH 1
 #define        MC_CMD_PHY_CAP_10FDX_LBN 2
@@ -2459,17 +2982,39 @@
 #define        MC_CMD_PHY_CAP_40000FDX_WIDTH 1
 #define        MC_CMD_PHY_CAP_DDM_LBN 12
 #define        MC_CMD_PHY_CAP_DDM_WIDTH 1
+#define        MC_CMD_PHY_CAP_100000FDX_LBN 13
+#define        MC_CMD_PHY_CAP_100000FDX_WIDTH 1
+#define        MC_CMD_PHY_CAP_25000FDX_LBN 14
+#define        MC_CMD_PHY_CAP_25000FDX_WIDTH 1
+#define        MC_CMD_PHY_CAP_50000FDX_LBN 15
+#define        MC_CMD_PHY_CAP_50000FDX_WIDTH 1
+#define        MC_CMD_PHY_CAP_BASER_FEC_LBN 16
+#define        MC_CMD_PHY_CAP_BASER_FEC_WIDTH 1
+#define        MC_CMD_PHY_CAP_BASER_FEC_REQUESTED_LBN 17
+#define        MC_CMD_PHY_CAP_BASER_FEC_REQUESTED_WIDTH 1
+#define        MC_CMD_PHY_CAP_RS_FEC_LBN 18
+#define        MC_CMD_PHY_CAP_RS_FEC_WIDTH 1
+#define        MC_CMD_PHY_CAP_RS_FEC_REQUESTED_LBN 19
+#define        MC_CMD_PHY_CAP_RS_FEC_REQUESTED_WIDTH 1
+#define        MC_CMD_PHY_CAP_25G_BASER_FEC_LBN 20
+#define        MC_CMD_PHY_CAP_25G_BASER_FEC_WIDTH 1
+#define        MC_CMD_PHY_CAP_25G_BASER_FEC_REQUESTED_LBN 21
+#define        MC_CMD_PHY_CAP_25G_BASER_FEC_REQUESTED_WIDTH 1
 /* ?? */
 #define       MC_CMD_GET_PHY_CFG_OUT_CHANNEL_OFST 12
+#define       MC_CMD_GET_PHY_CFG_OUT_CHANNEL_LEN 4
 /* ?? */
 #define       MC_CMD_GET_PHY_CFG_OUT_PRT_OFST 16
+#define       MC_CMD_GET_PHY_CFG_OUT_PRT_LEN 4
 /* ?? */
 #define       MC_CMD_GET_PHY_CFG_OUT_STATS_MASK_OFST 20
+#define       MC_CMD_GET_PHY_CFG_OUT_STATS_MASK_LEN 4
 /* ?? */
 #define       MC_CMD_GET_PHY_CFG_OUT_NAME_OFST 24
 #define       MC_CMD_GET_PHY_CFG_OUT_NAME_LEN 20
 /* ?? */
 #define       MC_CMD_GET_PHY_CFG_OUT_MEDIA_TYPE_OFST 44
+#define       MC_CMD_GET_PHY_CFG_OUT_MEDIA_TYPE_LEN 4
 /* enum: Xaui. */
 #define          MC_CMD_MEDIA_XAUI 0x1
 /* enum: CX4. */
@@ -2485,6 +3030,7 @@
 /* enum: QSFP+. */
 #define          MC_CMD_MEDIA_QSFP_PLUS 0x7
 #define       MC_CMD_GET_PHY_CFG_OUT_MMD_MASK_OFST 48
+#define       MC_CMD_GET_PHY_CFG_OUT_MMD_MASK_LEN 4
 /* enum: Native clause 22 */
 #define          MC_CMD_MMD_CLAUSE22 0x0
 #define          MC_CMD_MMD_CLAUSE45_PMAPMD 0x1 /* enum */
@@ -2515,6 +3061,7 @@
 #define    MC_CMD_START_BIST_IN_LEN 4
 /* Type of test. */
 #define       MC_CMD_START_BIST_IN_TYPE_OFST 0
+#define       MC_CMD_START_BIST_IN_TYPE_LEN 4
 /* enum: Run the PHY's short cable BIST. */
 #define          MC_CMD_PHY_BIST_CABLE_SHORT 0x1
 /* enum: Run the PHY's long cable BIST. */
@@ -2556,6 +3103,7 @@
 #define    MC_CMD_POLL_BIST_OUT_LEN 8
 /* result */
 #define       MC_CMD_POLL_BIST_OUT_RESULT_OFST 0
+#define       MC_CMD_POLL_BIST_OUT_RESULT_LEN 4
 /* enum: Running. */
 #define          MC_CMD_POLL_BIST_RUNNING 0x1
 /* enum: Passed. */
@@ -2565,19 +3113,26 @@
 /* enum: Timed-out. */
 #define          MC_CMD_POLL_BIST_TIMEOUT 0x4
 #define       MC_CMD_POLL_BIST_OUT_PRIVATE_OFST 4
+#define       MC_CMD_POLL_BIST_OUT_PRIVATE_LEN 4
 
 /* MC_CMD_POLL_BIST_OUT_SFT9001 msgresponse */
 #define    MC_CMD_POLL_BIST_OUT_SFT9001_LEN 36
 /* result */
 /*            MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */
+/*            MC_CMD_POLL_BIST_OUT_RESULT_LEN 4 */
 /*            Enum values, see field(s): */
 /*               MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_A_OFST 4
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_A_LEN 4
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_B_OFST 8
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_B_LEN 4
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_C_OFST 12
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_C_LEN 4
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_D_OFST 16
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_LENGTH_D_LEN 4
 /* Status of each channel A */
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_A_OFST 20
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_A_LEN 4
 /* enum: Ok. */
 #define          MC_CMD_POLL_BIST_SFT9001_PAIR_OK 0x1
 /* enum: Open. */
@@ -2590,14 +3145,17 @@
 #define          MC_CMD_POLL_BIST_SFT9001_PAIR_BUSY 0x9
 /* Status of each channel B */
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_B_OFST 24
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_B_LEN 4
 /*            Enum values, see field(s): */
 /*               CABLE_STATUS_A */
 /* Status of each channel C */
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_C_OFST 28
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_C_LEN 4
 /*            Enum values, see field(s): */
 /*               CABLE_STATUS_A */
 /* Status of each channel D */
 #define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_D_OFST 32
+#define       MC_CMD_POLL_BIST_OUT_SFT9001_CABLE_STATUS_D_LEN 4
 /*            Enum values, see field(s): */
 /*               CABLE_STATUS_A */
 
@@ -2605,9 +3163,11 @@
 #define    MC_CMD_POLL_BIST_OUT_MRSFP_LEN 8
 /* result */
 /*            MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */
+/*            MC_CMD_POLL_BIST_OUT_RESULT_LEN 4 */
 /*            Enum values, see field(s): */
 /*               MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */
 #define       MC_CMD_POLL_BIST_OUT_MRSFP_TEST_OFST 4
+#define       MC_CMD_POLL_BIST_OUT_MRSFP_TEST_LEN 4
 /* enum: Complete. */
 #define          MC_CMD_POLL_BIST_MRSFP_TEST_COMPLETE 0x0
 /* enum: Bus switch off I2C write. */
@@ -2631,9 +3191,11 @@
 #define    MC_CMD_POLL_BIST_OUT_MEM_LEN 36
 /* result */
 /*            MC_CMD_POLL_BIST_OUT_RESULT_OFST 0 */
+/*            MC_CMD_POLL_BIST_OUT_RESULT_LEN 4 */
 /*            Enum values, see field(s): */
 /*               MC_CMD_POLL_BIST_OUT/MC_CMD_POLL_BIST_OUT_RESULT */
 #define       MC_CMD_POLL_BIST_OUT_MEM_TEST_OFST 4
+#define       MC_CMD_POLL_BIST_OUT_MEM_TEST_LEN 4
 /* enum: Test has completed. */
 #define          MC_CMD_POLL_BIST_MEM_COMPLETE 0x0
 /* enum: RAM test - walk ones. */
@@ -2650,8 +3212,10 @@
 #define          MC_CMD_POLL_BIST_MEM_ECC 0x6
 /* Failure address, only valid if result is POLL_BIST_FAILED */
 #define       MC_CMD_POLL_BIST_OUT_MEM_ADDR_OFST 8
+#define       MC_CMD_POLL_BIST_OUT_MEM_ADDR_LEN 4
 /* Bus or address space to which the failure address corresponds */
 #define       MC_CMD_POLL_BIST_OUT_MEM_BUS_OFST 12
+#define       MC_CMD_POLL_BIST_OUT_MEM_BUS_LEN 4
 /* enum: MC MIPS bus. */
 #define          MC_CMD_POLL_BIST_MEM_BUS_MC 0x0
 /* enum: CSR IREG bus. */
@@ -2672,14 +3236,19 @@
 #define          MC_CMD_POLL_BIST_MEM_BUS_DICPU_RX1 0x8
 /* Pattern written to RAM / register */
 #define       MC_CMD_POLL_BIST_OUT_MEM_EXPECT_OFST 16
+#define       MC_CMD_POLL_BIST_OUT_MEM_EXPECT_LEN 4
 /* Actual value read from RAM / register */
 #define       MC_CMD_POLL_BIST_OUT_MEM_ACTUAL_OFST 20
+#define       MC_CMD_POLL_BIST_OUT_MEM_ACTUAL_LEN 4
 /* ECC error mask */
 #define       MC_CMD_POLL_BIST_OUT_MEM_ECC_OFST 24
+#define       MC_CMD_POLL_BIST_OUT_MEM_ECC_LEN 4
 /* ECC parity error mask */
 #define       MC_CMD_POLL_BIST_OUT_MEM_ECC_PARITY_OFST 28
+#define       MC_CMD_POLL_BIST_OUT_MEM_ECC_PARITY_LEN 4
 /* ECC fatal error mask */
 #define       MC_CMD_POLL_BIST_OUT_MEM_ECC_FATAL_OFST 32
+#define       MC_CMD_POLL_BIST_OUT_MEM_ECC_FATAL_LEN 4
 
 
 /***********************************/
@@ -2831,6 +3400,143 @@
 /*            Enum values, see field(s): */
 /*               100M */
 
+/* MC_CMD_GET_LOOPBACK_MODES_OUT_V2 msgresponse: Supported loopback modes for
+ * newer NICs with 25G/50G/100G support
+ */
+#define    MC_CMD_GET_LOOPBACK_MODES_OUT_V2_LEN 64
+/* Supported loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_OFST 0
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_LO_OFST 0
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100M_HI_OFST 4
+/* enum: None. */
+/*               MC_CMD_LOOPBACK_NONE  0x0 */
+/* enum: Data. */
+/*               MC_CMD_LOOPBACK_DATA  0x1 */
+/* enum: GMAC. */
+/*               MC_CMD_LOOPBACK_GMAC  0x2 */
+/* enum: XGMII. */
+/*               MC_CMD_LOOPBACK_XGMII 0x3 */
+/* enum: XGXS. */
+/*               MC_CMD_LOOPBACK_XGXS  0x4 */
+/* enum: XAUI. */
+/*               MC_CMD_LOOPBACK_XAUI  0x5 */
+/* enum: GMII. */
+/*               MC_CMD_LOOPBACK_GMII  0x6 */
+/* enum: SGMII. */
+/*               MC_CMD_LOOPBACK_SGMII  0x7 */
+/* enum: XGBR. */
+/*               MC_CMD_LOOPBACK_XGBR  0x8 */
+/* enum: XFI. */
+/*               MC_CMD_LOOPBACK_XFI  0x9 */
+/* enum: XAUI Far. */
+/*               MC_CMD_LOOPBACK_XAUI_FAR  0xa */
+/* enum: GMII Far. */
+/*               MC_CMD_LOOPBACK_GMII_FAR  0xb */
+/* enum: SGMII Far. */
+/*               MC_CMD_LOOPBACK_SGMII_FAR  0xc */
+/* enum: XFI Far. */
+/*               MC_CMD_LOOPBACK_XFI_FAR  0xd */
+/* enum: GPhy. */
+/*               MC_CMD_LOOPBACK_GPHY  0xe */
+/* enum: PhyXS. */
+/*               MC_CMD_LOOPBACK_PHYXS  0xf */
+/* enum: PCS. */
+/*               MC_CMD_LOOPBACK_PCS  0x10 */
+/* enum: PMA-PMD. */
+/*               MC_CMD_LOOPBACK_PMAPMD  0x11 */
+/* enum: Cross-Port. */
+/*               MC_CMD_LOOPBACK_XPORT  0x12 */
+/* enum: XGMII-Wireside. */
+/*               MC_CMD_LOOPBACK_XGMII_WS  0x13 */
+/* enum: XAUI Wireside. */
+/*               MC_CMD_LOOPBACK_XAUI_WS  0x14 */
+/* enum: XAUI Wireside Far. */
+/*               MC_CMD_LOOPBACK_XAUI_WS_FAR  0x15 */
+/* enum: XAUI Wireside near. */
+/*               MC_CMD_LOOPBACK_XAUI_WS_NEAR  0x16 */
+/* enum: GMII Wireside. */
+/*               MC_CMD_LOOPBACK_GMII_WS  0x17 */
+/* enum: XFI Wireside. */
+/*               MC_CMD_LOOPBACK_XFI_WS  0x18 */
+/* enum: XFI Wireside Far. */
+/*               MC_CMD_LOOPBACK_XFI_WS_FAR  0x19 */
+/* enum: PhyXS Wireside. */
+/*               MC_CMD_LOOPBACK_PHYXS_WS  0x1a */
+/* enum: PMA lanes MAC-Serdes. */
+/*               MC_CMD_LOOPBACK_PMA_INT  0x1b */
+/* enum: KR Serdes Parallel (Encoder). */
+/*               MC_CMD_LOOPBACK_SD_NEAR  0x1c */
+/* enum: KR Serdes Serial. */
+/*               MC_CMD_LOOPBACK_SD_FAR  0x1d */
+/* enum: PMA lanes MAC-Serdes Wireside. */
+/*               MC_CMD_LOOPBACK_PMA_INT_WS  0x1e */
+/* enum: KR Serdes Parallel Wireside (Full PCS). */
+/*               MC_CMD_LOOPBACK_SD_FEP2_WS  0x1f */
+/* enum: KR Serdes Parallel Wireside (Sym Aligner to TX). */
+/*               MC_CMD_LOOPBACK_SD_FEP1_5_WS  0x20 */
+/* enum: KR Serdes Parallel Wireside (Deserializer to Serializer). */
+/*               MC_CMD_LOOPBACK_SD_FEP_WS  0x21 */
+/* enum: KR Serdes Serial Wireside. */
+/*               MC_CMD_LOOPBACK_SD_FES_WS  0x22 */
+/* enum: Near side of AOE Siena side port */
+/*               MC_CMD_LOOPBACK_AOE_INT_NEAR  0x23 */
+/* enum: Medford Wireside datapath loopback */
+/*               MC_CMD_LOOPBACK_DATA_WS  0x24 */
+/* enum: Force link up without setting up any physical loopback (snapper use
+ * only)
+ */
+/*               MC_CMD_LOOPBACK_FORCE_EXT_LINK  0x25 */
+/* Supported loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_OFST 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_LO_OFST 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_1G_HI_OFST 12
+/*            Enum values, see field(s): */
+/*               100M */
+/* Supported loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_OFST 16
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_LO_OFST 16
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_10G_HI_OFST 20
+/*            Enum values, see field(s): */
+/*               100M */
+/* Supported loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_OFST 24
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_LO_OFST 24
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_SUGGESTED_HI_OFST 28
+/*            Enum values, see field(s): */
+/*               100M */
+/* Supported loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_OFST 32
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_LO_OFST 32
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_40G_HI_OFST 36
+/*            Enum values, see field(s): */
+/*               100M */
+/* Supported 25G loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_OFST 40
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_LO_OFST 40
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_25G_HI_OFST 44
+/*            Enum values, see field(s): */
+/*               100M */
+/* Supported 50 loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_OFST 48
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_LO_OFST 48
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_50G_HI_OFST 52
+/*            Enum values, see field(s): */
+/*               100M */
+/* Supported 100G loopbacks. */
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_OFST 56
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_LEN 8
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_LO_OFST 56
+#define       MC_CMD_GET_LOOPBACK_MODES_OUT_V2_100G_HI_OFST 60
+/*            Enum values, see field(s): */
+/*               100M */
+
 
 /***********************************/
 /* MC_CMD_GET_LINK
@@ -2848,17 +3554,22 @@
 #define    MC_CMD_GET_LINK_OUT_LEN 28
 /* near-side advertised capabilities */
 #define       MC_CMD_GET_LINK_OUT_CAP_OFST 0
+#define       MC_CMD_GET_LINK_OUT_CAP_LEN 4
 /* link-partner advertised capabilities */
 #define       MC_CMD_GET_LINK_OUT_LP_CAP_OFST 4
+#define       MC_CMD_GET_LINK_OUT_LP_CAP_LEN 4
 /* Autonegotiated speed in mbit/s. The link may still be down even if this
  * reads non-zero.
  */
 #define       MC_CMD_GET_LINK_OUT_LINK_SPEED_OFST 8
+#define       MC_CMD_GET_LINK_OUT_LINK_SPEED_LEN 4
 /* Current loopback setting. */
 #define       MC_CMD_GET_LINK_OUT_LOOPBACK_MODE_OFST 12
+#define       MC_CMD_GET_LINK_OUT_LOOPBACK_MODE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_LOOPBACK_MODES/MC_CMD_GET_LOOPBACK_MODES_OUT/100M */
 #define       MC_CMD_GET_LINK_OUT_FLAGS_OFST 16
+#define       MC_CMD_GET_LINK_OUT_FLAGS_LEN 4
 #define        MC_CMD_GET_LINK_OUT_LINK_UP_LBN 0
 #define        MC_CMD_GET_LINK_OUT_LINK_UP_WIDTH 1
 #define        MC_CMD_GET_LINK_OUT_FULL_DUPLEX_LBN 1
@@ -2873,9 +3584,11 @@
 #define        MC_CMD_GET_LINK_OUT_LINK_FAULT_TX_WIDTH 1
 /* This returns the negotiated flow control value. */
 #define       MC_CMD_GET_LINK_OUT_FCNTL_OFST 20
+#define       MC_CMD_GET_LINK_OUT_FCNTL_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_SET_MAC/MC_CMD_SET_MAC_IN/FCNTL */
 #define       MC_CMD_GET_LINK_OUT_MAC_FAULT_OFST 24
+#define       MC_CMD_GET_LINK_OUT_MAC_FAULT_LEN 4
 #define        MC_CMD_MAC_FAULT_XGMII_LOCAL_LBN 0
 #define        MC_CMD_MAC_FAULT_XGMII_LOCAL_WIDTH 1
 #define        MC_CMD_MAC_FAULT_XGMII_REMOTE_LBN 1
@@ -2899,8 +3612,10 @@
 #define    MC_CMD_SET_LINK_IN_LEN 16
 /* ??? */
 #define       MC_CMD_SET_LINK_IN_CAP_OFST 0
+#define       MC_CMD_SET_LINK_IN_CAP_LEN 4
 /* Flags */
 #define       MC_CMD_SET_LINK_IN_FLAGS_OFST 4
+#define       MC_CMD_SET_LINK_IN_FLAGS_LEN 4
 #define        MC_CMD_SET_LINK_IN_LOWPOWER_LBN 0
 #define        MC_CMD_SET_LINK_IN_LOWPOWER_WIDTH 1
 #define        MC_CMD_SET_LINK_IN_POWEROFF_LBN 1
@@ -2909,12 +3624,14 @@
 #define        MC_CMD_SET_LINK_IN_TXDIS_WIDTH 1
 /* Loopback mode. */
 #define       MC_CMD_SET_LINK_IN_LOOPBACK_MODE_OFST 8
+#define       MC_CMD_SET_LINK_IN_LOOPBACK_MODE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_LOOPBACK_MODES/MC_CMD_GET_LOOPBACK_MODES_OUT/100M */
 /* A loopback speed of "0" is supported, and means (choose any available
  * speed).
  */
 #define       MC_CMD_SET_LINK_IN_LOOPBACK_SPEED_OFST 12
+#define       MC_CMD_SET_LINK_IN_LOOPBACK_SPEED_LEN 4
 
 /* MC_CMD_SET_LINK_OUT msgresponse */
 #define    MC_CMD_SET_LINK_OUT_LEN 0
@@ -2932,6 +3649,7 @@
 #define    MC_CMD_SET_ID_LED_IN_LEN 4
 /* Set LED state. */
 #define       MC_CMD_SET_ID_LED_IN_STATE_OFST 0
+#define       MC_CMD_SET_ID_LED_IN_STATE_LEN 4
 #define          MC_CMD_LED_OFF  0x0 /* enum */
 #define          MC_CMD_LED_ON  0x1 /* enum */
 #define          MC_CMD_LED_DEFAULT  0x2 /* enum */
@@ -2954,17 +3672,21 @@
  * EtherII, VLAN, bug16011 padding).
  */
 #define       MC_CMD_SET_MAC_IN_MTU_OFST 0
+#define       MC_CMD_SET_MAC_IN_MTU_LEN 4
 #define       MC_CMD_SET_MAC_IN_DRAIN_OFST 4
+#define       MC_CMD_SET_MAC_IN_DRAIN_LEN 4
 #define       MC_CMD_SET_MAC_IN_ADDR_OFST 8
 #define       MC_CMD_SET_MAC_IN_ADDR_LEN 8
 #define       MC_CMD_SET_MAC_IN_ADDR_LO_OFST 8
 #define       MC_CMD_SET_MAC_IN_ADDR_HI_OFST 12
 #define       MC_CMD_SET_MAC_IN_REJECT_OFST 16
+#define       MC_CMD_SET_MAC_IN_REJECT_LEN 4
 #define        MC_CMD_SET_MAC_IN_REJECT_UNCST_LBN 0
 #define        MC_CMD_SET_MAC_IN_REJECT_UNCST_WIDTH 1
 #define        MC_CMD_SET_MAC_IN_REJECT_BRDCST_LBN 1
 #define        MC_CMD_SET_MAC_IN_REJECT_BRDCST_WIDTH 1
 #define       MC_CMD_SET_MAC_IN_FCNTL_OFST 20
+#define       MC_CMD_SET_MAC_IN_FCNTL_LEN 4
 /* enum: Flow control is off. */
 #define          MC_CMD_FCNTL_OFF 0x0
 /* enum: Respond to flow control. */
@@ -2978,6 +3700,7 @@
 /* enum: Issue flow control. */
 #define          MC_CMD_FCNTL_GENERATE 0x5
 #define       MC_CMD_SET_MAC_IN_FLAGS_OFST 24
+#define       MC_CMD_SET_MAC_IN_FLAGS_LEN 4
 #define        MC_CMD_SET_MAC_IN_FLAG_INCLUDE_FCS_LBN 0
 #define        MC_CMD_SET_MAC_IN_FLAG_INCLUDE_FCS_WIDTH 1
 
@@ -2987,17 +3710,21 @@
  * EtherII, VLAN, bug16011 padding).
  */
 #define       MC_CMD_SET_MAC_EXT_IN_MTU_OFST 0
+#define       MC_CMD_SET_MAC_EXT_IN_MTU_LEN 4
 #define       MC_CMD_SET_MAC_EXT_IN_DRAIN_OFST 4
+#define       MC_CMD_SET_MAC_EXT_IN_DRAIN_LEN 4
 #define       MC_CMD_SET_MAC_EXT_IN_ADDR_OFST 8
 #define       MC_CMD_SET_MAC_EXT_IN_ADDR_LEN 8
 #define       MC_CMD_SET_MAC_EXT_IN_ADDR_LO_OFST 8
 #define       MC_CMD_SET_MAC_EXT_IN_ADDR_HI_OFST 12
 #define       MC_CMD_SET_MAC_EXT_IN_REJECT_OFST 16
+#define       MC_CMD_SET_MAC_EXT_IN_REJECT_LEN 4
 #define        MC_CMD_SET_MAC_EXT_IN_REJECT_UNCST_LBN 0
 #define        MC_CMD_SET_MAC_EXT_IN_REJECT_UNCST_WIDTH 1
 #define        MC_CMD_SET_MAC_EXT_IN_REJECT_BRDCST_LBN 1
 #define        MC_CMD_SET_MAC_EXT_IN_REJECT_BRDCST_WIDTH 1
 #define       MC_CMD_SET_MAC_EXT_IN_FCNTL_OFST 20
+#define       MC_CMD_SET_MAC_EXT_IN_FCNTL_LEN 4
 /* enum: Flow control is off. */
 /*               MC_CMD_FCNTL_OFF 0x0 */
 /* enum: Respond to flow control. */
@@ -3011,6 +3738,7 @@
 /* enum: Issue flow control. */
 /*               MC_CMD_FCNTL_GENERATE 0x5 */
 #define       MC_CMD_SET_MAC_EXT_IN_FLAGS_OFST 24
+#define       MC_CMD_SET_MAC_EXT_IN_FLAGS_LEN 4
 #define        MC_CMD_SET_MAC_EXT_IN_FLAG_INCLUDE_FCS_LBN 0
 #define        MC_CMD_SET_MAC_EXT_IN_FLAG_INCLUDE_FCS_WIDTH 1
 /* Select which parameters to configure. A parameter will only be modified if
@@ -3019,6 +3747,7 @@
  * set).
  */
 #define       MC_CMD_SET_MAC_EXT_IN_CONTROL_OFST 28
+#define       MC_CMD_SET_MAC_EXT_IN_CONTROL_LEN 4
 #define        MC_CMD_SET_MAC_EXT_IN_CFG_MTU_LBN 0
 #define        MC_CMD_SET_MAC_EXT_IN_CFG_MTU_WIDTH 1
 #define        MC_CMD_SET_MAC_EXT_IN_CFG_DRAIN_LBN 1
@@ -3040,6 +3769,7 @@
  * to 0.
  */
 #define       MC_CMD_SET_MAC_V2_OUT_MTU_OFST 0
+#define       MC_CMD_SET_MAC_V2_OUT_MTU_LEN 4
 
 
 /***********************************/
@@ -3144,6 +3874,7 @@
 #define       MC_CMD_MAC_STATS_IN_DMA_ADDR_LO_OFST 0
 #define       MC_CMD_MAC_STATS_IN_DMA_ADDR_HI_OFST 4
 #define       MC_CMD_MAC_STATS_IN_CMD_OFST 8
+#define       MC_CMD_MAC_STATS_IN_CMD_LEN 4
 #define        MC_CMD_MAC_STATS_IN_DMA_LBN 0
 #define        MC_CMD_MAC_STATS_IN_DMA_WIDTH 1
 #define        MC_CMD_MAC_STATS_IN_CLEAR_LBN 1
@@ -3158,9 +3889,16 @@
 #define        MC_CMD_MAC_STATS_IN_PERIODIC_NOEVENT_WIDTH 1
 #define        MC_CMD_MAC_STATS_IN_PERIOD_MS_LBN 16
 #define        MC_CMD_MAC_STATS_IN_PERIOD_MS_WIDTH 16
+/* DMA length. Should be set to MAC_STATS_NUM_STATS * sizeof(uint64_t), as
+ * returned by MC_CMD_GET_CAPABILITIES_V4_OUT. For legacy firmware not
+ * supporting MC_CMD_GET_CAPABILITIES_V4_OUT, DMA_LEN should be set to
+ * MC_CMD_MAC_NSTATS * sizeof(uint64_t)
+ */
 #define       MC_CMD_MAC_STATS_IN_DMA_LEN_OFST 12
+#define       MC_CMD_MAC_STATS_IN_DMA_LEN_LEN 4
 /* port id so vadapter stats can be provided */
 #define       MC_CMD_MAC_STATS_IN_PORT_ID_OFST 16
+#define       MC_CMD_MAC_STATS_IN_PORT_ID_LEN 4
 
 /* MC_CMD_MAC_STATS_OUT_DMA msgresponse */
 #define    MC_CMD_MAC_STATS_OUT_DMA_LEN 0
@@ -3305,9 +4043,126 @@
 #define          MC_CMD_GMAC_DMABUF_START  0x40
 /* enum: End of GMAC stats buffer space, for Siena only. */
 #define          MC_CMD_GMAC_DMABUF_END    0x5f
-#define          MC_CMD_MAC_GENERATION_END 0x60 /* enum */
+/* enum: GENERATION_END value, used together with GENERATION_START to verify
+ * consistency of DMAd data. For legacy firmware / drivers without extended
+ * stats (more precisely, when DMA_LEN == MC_CMD_MAC_NSTATS *
+ * sizeof(uint64_t)), this entry holds the GENERATION_END value. Otherwise,
+ * this value is invalid/ reserved and GENERATION_END is written as the last
+ * 64-bit word of the DMA buffer (at DMA_LEN - sizeof(uint64_t)). Note that
+ * this is consistent with the legacy behaviour, in the sense that entry 96 is
+ * the last 64-bit word in the buffer when DMA_LEN == MC_CMD_MAC_NSTATS *
+ * sizeof(uint64_t). See SF-109306-TC, Section 9.2 for details.
+ */
+#define          MC_CMD_MAC_GENERATION_END 0x60
 #define          MC_CMD_MAC_NSTATS  0x61 /* enum */
 
+/* MC_CMD_MAC_STATS_V2_OUT_DMA msgresponse */
+#define    MC_CMD_MAC_STATS_V2_OUT_DMA_LEN 0
+
+/* MC_CMD_MAC_STATS_V2_OUT_NO_DMA msgresponse */
+#define    MC_CMD_MAC_STATS_V2_OUT_NO_DMA_LEN (((MC_CMD_MAC_NSTATS_V2*64))>>3)
+#define       MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_OFST 0
+#define       MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_LEN 8
+#define       MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_LO_OFST 0
+#define       MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_HI_OFST 4
+#define       MC_CMD_MAC_STATS_V2_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS_V2
+/* enum: Start of FEC stats buffer space, Medford2 and up */
+#define          MC_CMD_MAC_FEC_DMABUF_START  0x61
+/* enum: Number of uncorrected FEC codewords on link (RS-FEC only for Medford2)
+ */
+#define          MC_CMD_MAC_FEC_UNCORRECTED_ERRORS  0x61
+/* enum: Number of corrected FEC codewords on link (RS-FEC only for Medford2)
+ */
+#define          MC_CMD_MAC_FEC_CORRECTED_ERRORS  0x62
+/* enum: Number of corrected 10-bit symbol errors, lane 0 (RS-FEC only) */
+#define          MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE0  0x63
+/* enum: Number of corrected 10-bit symbol errors, lane 1 (RS-FEC only) */
+#define          MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE1  0x64
+/* enum: Number of corrected 10-bit symbol errors, lane 2 (RS-FEC only) */
+#define          MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE2  0x65
+/* enum: Number of corrected 10-bit symbol errors, lane 3 (RS-FEC only) */
+#define          MC_CMD_MAC_FEC_CORRECTED_SYMBOLS_LANE3  0x66
+/* enum: This includes the space at offset 103 which is the final
+ * GENERATION_END in a MAC_STATS_V2 response and otherwise unused.
+ */
+#define          MC_CMD_MAC_NSTATS_V2  0x68
+/*            Other enum values, see field(s): */
+/*               MC_CMD_MAC_STATS_OUT_NO_DMA/STATISTICS */
+
+/* MC_CMD_MAC_STATS_V3_OUT_DMA msgresponse */
+#define    MC_CMD_MAC_STATS_V3_OUT_DMA_LEN 0
+
+/* MC_CMD_MAC_STATS_V3_OUT_NO_DMA msgresponse */
+#define    MC_CMD_MAC_STATS_V3_OUT_NO_DMA_LEN (((MC_CMD_MAC_NSTATS_V3*64))>>3)
+#define       MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_OFST 0
+#define       MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_LEN 8
+#define       MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_LO_OFST 0
+#define       MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_HI_OFST 4
+#define       MC_CMD_MAC_STATS_V3_OUT_NO_DMA_STATISTICS_NUM MC_CMD_MAC_NSTATS_V3
+/* enum: Start of CTPIO stats buffer space, Medford2 and up */
+#define          MC_CMD_MAC_CTPIO_DMABUF_START  0x68
+/* enum: Number of CTPIO fallbacks because a DMA packet was in progress on the
+ * target VI
+ */
+#define          MC_CMD_MAC_CTPIO_VI_BUSY_FALLBACK  0x68
+/* enum: Number of times a CTPIO send wrote beyond frame end (informational
+ * only)
+ */
+#define          MC_CMD_MAC_CTPIO_LONG_WRITE_SUCCESS  0x69
+/* enum: Number of CTPIO failures because the TX doorbell was written before
+ * the end of the frame data
+ */
+#define          MC_CMD_MAC_CTPIO_MISSING_DBELL_FAIL  0x6a
+/* enum: Number of CTPIO failures because the internal FIFO overflowed */
+#define          MC_CMD_MAC_CTPIO_OVERFLOW_FAIL  0x6b
+/* enum: Number of CTPIO failures because the host did not deliver data fast
+ * enough to avoid MAC underflow
+ */
+#define          MC_CMD_MAC_CTPIO_UNDERFLOW_FAIL  0x6c
+/* enum: Number of CTPIO failures because the host did not deliver all the
+ * frame data within the timeout
+ */
+#define          MC_CMD_MAC_CTPIO_TIMEOUT_FAIL  0x6d
+/* enum: Number of CTPIO failures because the frame data arrived out of order
+ * or with gaps
+ */
+#define          MC_CMD_MAC_CTPIO_NONCONTIG_WR_FAIL  0x6e
+/* enum: Number of CTPIO failures because the host started a new frame before
+ * completing the previous one
+ */
+#define          MC_CMD_MAC_CTPIO_FRM_CLOBBER_FAIL  0x6f
+/* enum: Number of CTPIO failures because a write was not a multiple of 32 bits
+ * or not 32-bit aligned
+ */
+#define          MC_CMD_MAC_CTPIO_INVALID_WR_FAIL  0x70
+/* enum: Number of CTPIO fallbacks because another VI on the same port was
+ * sending a CTPIO frame
+ */
+#define          MC_CMD_MAC_CTPIO_VI_CLOBBER_FALLBACK  0x71
+/* enum: Number of CTPIO fallbacks because target VI did not have CTPIO enabled
+ */
+#define          MC_CMD_MAC_CTPIO_UNQUALIFIED_FALLBACK  0x72
+/* enum: Number of CTPIO fallbacks because length in header was less than 29
+ * bytes
+ */
+#define          MC_CMD_MAC_CTPIO_RUNT_FALLBACK  0x73
+/* enum: Total number of successful CTPIO sends on this port */
+#define          MC_CMD_MAC_CTPIO_SUCCESS  0x74
+/* enum: Total number of CTPIO fallbacks on this port */
+#define          MC_CMD_MAC_CTPIO_FALLBACK  0x75
+/* enum: Total number of CTPIO poisoned frames on this port, whether erased or
+ * not
+ */
+#define          MC_CMD_MAC_CTPIO_POISON  0x76
+/* enum: Total number of CTPIO erased frames on this port */
+#define          MC_CMD_MAC_CTPIO_ERASE  0x77
+/* enum: This includes the space at offset 120 which is the final
+ * GENERATION_END in a MAC_STATS_V3 response and otherwise unused.
+ */
+#define          MC_CMD_MAC_NSTATS_V3  0x79
+/*            Other enum values, see field(s): */
+/*               MC_CMD_MAC_STATS_V2_OUT_NO_DMA/STATISTICS */
+
 
 /***********************************/
 /* MC_CMD_SRIOV
@@ -3318,21 +4173,28 @@
 /* MC_CMD_SRIOV_IN msgrequest */
 #define    MC_CMD_SRIOV_IN_LEN 12
 #define       MC_CMD_SRIOV_IN_ENABLE_OFST 0
+#define       MC_CMD_SRIOV_IN_ENABLE_LEN 4
 #define       MC_CMD_SRIOV_IN_VI_BASE_OFST 4
+#define       MC_CMD_SRIOV_IN_VI_BASE_LEN 4
 #define       MC_CMD_SRIOV_IN_VF_COUNT_OFST 8
+#define       MC_CMD_SRIOV_IN_VF_COUNT_LEN 4
 
 /* MC_CMD_SRIOV_OUT msgresponse */
 #define    MC_CMD_SRIOV_OUT_LEN 8
 #define       MC_CMD_SRIOV_OUT_VI_SCALE_OFST 0
+#define       MC_CMD_SRIOV_OUT_VI_SCALE_LEN 4
 #define       MC_CMD_SRIOV_OUT_VF_TOTAL_OFST 4
+#define       MC_CMD_SRIOV_OUT_VF_TOTAL_LEN 4
 
 /* MC_CMD_MEMCPY_RECORD_TYPEDEF structuredef */
 #define    MC_CMD_MEMCPY_RECORD_TYPEDEF_LEN 32
 /* this is only used for the first record */
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_OFST 0
+#define       MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_LEN 4
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_LBN 0
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_NUM_RECORDS_WIDTH 32
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_OFST 4
+#define       MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_LEN 4
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_LBN 32
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_RID_WIDTH 32
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_OFST 8
@@ -3342,6 +4204,7 @@
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_LBN 64
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_TO_ADDR_WIDTH 64
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_OFST 16
+#define       MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_LEN 4
 #define          MC_CMD_MEMCPY_RECORD_TYPEDEF_RID_INLINE 0x100 /* enum */
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_LBN 128
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_RID_WIDTH 32
@@ -3352,6 +4215,7 @@
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_LBN 160
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_FROM_ADDR_WIDTH 64
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_OFST 28
+#define       MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_LEN 4
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_LBN 224
 #define       MC_CMD_MEMCPY_RECORD_TYPEDEF_LENGTH_WIDTH 32
 
@@ -3403,10 +4267,12 @@
 /* MC_CMD_WOL_FILTER_SET_IN msgrequest */
 #define    MC_CMD_WOL_FILTER_SET_IN_LEN 192
 #define       MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0
+#define       MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_LEN 4
 #define          MC_CMD_FILTER_MODE_SIMPLE    0x0 /* enum */
 #define          MC_CMD_FILTER_MODE_STRUCTURED 0xffffffff /* enum */
 /* A type value of 1 is unused. */
 #define       MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4
+#define       MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_LEN 4
 /* enum: Magic */
 #define          MC_CMD_WOL_TYPE_MAGIC      0x0
 /* enum: MS Windows Magic */
@@ -3428,7 +4294,9 @@
 /* MC_CMD_WOL_FILTER_SET_IN_MAGIC msgrequest */
 #define    MC_CMD_WOL_FILTER_SET_IN_MAGIC_LEN 16
 /*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_LEN 4 */
 /*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_LEN 4 */
 #define       MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_OFST 8
 #define       MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LEN 8
 #define       MC_CMD_WOL_FILTER_SET_IN_MAGIC_MAC_LO_OFST 8
@@ -3437,9 +4305,13 @@
 /* MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN msgrequest */
 #define    MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_LEN 20
 /*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_LEN 4 */
 /*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_LEN 4 */
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_IP_OFST 8
+#define       MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_IP_LEN 4
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_IP_OFST 12
+#define       MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_IP_LEN 4
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_PORT_OFST 16
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_SRC_PORT_LEN 2
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV4_SYN_DST_PORT_OFST 18
@@ -3448,7 +4320,9 @@
 /* MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN msgrequest */
 #define    MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_LEN 44
 /*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_LEN 4 */
 /*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_LEN 4 */
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_IP_OFST 8
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_SRC_IP_LEN 16
 #define       MC_CMD_WOL_FILTER_SET_IN_IPV6_SYN_DST_IP_OFST 24
@@ -3461,7 +4335,9 @@
 /* MC_CMD_WOL_FILTER_SET_IN_BITMAP msgrequest */
 #define    MC_CMD_WOL_FILTER_SET_IN_BITMAP_LEN 187
 /*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_LEN 4 */
 /*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_LEN 4 */
 #define       MC_CMD_WOL_FILTER_SET_IN_BITMAP_MASK_OFST 8
 #define       MC_CMD_WOL_FILTER_SET_IN_BITMAP_MASK_LEN 48
 #define       MC_CMD_WOL_FILTER_SET_IN_BITMAP_BITMAP_OFST 56
@@ -3476,8 +4352,11 @@
 /* MC_CMD_WOL_FILTER_SET_IN_LINK msgrequest */
 #define    MC_CMD_WOL_FILTER_SET_IN_LINK_LEN 12
 /*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_OFST 0 */
+/*            MC_CMD_WOL_FILTER_SET_IN_FILTER_MODE_LEN 4 */
 /*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_OFST 4 */
+/*            MC_CMD_WOL_FILTER_SET_IN_WOL_TYPE_LEN 4 */
 #define       MC_CMD_WOL_FILTER_SET_IN_LINK_MASK_OFST 8
+#define       MC_CMD_WOL_FILTER_SET_IN_LINK_MASK_LEN 4
 #define        MC_CMD_WOL_FILTER_SET_IN_LINK_UP_LBN 0
 #define        MC_CMD_WOL_FILTER_SET_IN_LINK_UP_WIDTH 1
 #define        MC_CMD_WOL_FILTER_SET_IN_LINK_DOWN_LBN 1
@@ -3486,6 +4365,7 @@
 /* MC_CMD_WOL_FILTER_SET_OUT msgresponse */
 #define    MC_CMD_WOL_FILTER_SET_OUT_LEN 4
 #define       MC_CMD_WOL_FILTER_SET_OUT_FILTER_ID_OFST 0
+#define       MC_CMD_WOL_FILTER_SET_OUT_FILTER_ID_LEN 4
 
 
 /***********************************/
@@ -3499,6 +4379,7 @@
 /* MC_CMD_WOL_FILTER_REMOVE_IN msgrequest */
 #define    MC_CMD_WOL_FILTER_REMOVE_IN_LEN 4
 #define       MC_CMD_WOL_FILTER_REMOVE_IN_FILTER_ID_OFST 0
+#define       MC_CMD_WOL_FILTER_REMOVE_IN_FILTER_ID_LEN 4
 
 /* MC_CMD_WOL_FILTER_REMOVE_OUT msgresponse */
 #define    MC_CMD_WOL_FILTER_REMOVE_OUT_LEN 0
@@ -3516,6 +4397,7 @@
 /* MC_CMD_WOL_FILTER_RESET_IN msgrequest */
 #define    MC_CMD_WOL_FILTER_RESET_IN_LEN 4
 #define       MC_CMD_WOL_FILTER_RESET_IN_MASK_OFST 0
+#define       MC_CMD_WOL_FILTER_RESET_IN_MASK_LEN 4
 #define          MC_CMD_WOL_FILTER_RESET_IN_WAKE_FILTERS 0x1 /* enum */
 #define          MC_CMD_WOL_FILTER_RESET_IN_LIGHTSOUT_OFFLOADS 0x2 /* enum */
 
@@ -3556,6 +4438,7 @@
 #define    MC_CMD_NVRAM_TYPES_OUT_LEN 4
 /* Bit mask of supported types. */
 #define       MC_CMD_NVRAM_TYPES_OUT_TYPES_OFST 0
+#define       MC_CMD_NVRAM_TYPES_OUT_TYPES_LEN 4
 /* enum: Disabled callisto. */
 #define          MC_CMD_NVRAM_TYPE_DISABLED_CALLISTO 0x0
 /* enum: MC firmware. */
@@ -3612,47 +4495,65 @@
 /* MC_CMD_NVRAM_INFO_IN msgrequest */
 #define    MC_CMD_NVRAM_INFO_IN_LEN 4
 #define       MC_CMD_NVRAM_INFO_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_INFO_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 
 /* MC_CMD_NVRAM_INFO_OUT msgresponse */
 #define    MC_CMD_NVRAM_INFO_OUT_LEN 24
 #define       MC_CMD_NVRAM_INFO_OUT_TYPE_OFST 0
+#define       MC_CMD_NVRAM_INFO_OUT_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_INFO_OUT_SIZE_OFST 4
+#define       MC_CMD_NVRAM_INFO_OUT_SIZE_LEN 4
 #define       MC_CMD_NVRAM_INFO_OUT_ERASESIZE_OFST 8
+#define       MC_CMD_NVRAM_INFO_OUT_ERASESIZE_LEN 4
 #define       MC_CMD_NVRAM_INFO_OUT_FLAGS_OFST 12
+#define       MC_CMD_NVRAM_INFO_OUT_FLAGS_LEN 4
 #define        MC_CMD_NVRAM_INFO_OUT_PROTECTED_LBN 0
 #define        MC_CMD_NVRAM_INFO_OUT_PROTECTED_WIDTH 1
 #define        MC_CMD_NVRAM_INFO_OUT_TLV_LBN 1
 #define        MC_CMD_NVRAM_INFO_OUT_TLV_WIDTH 1
+#define        MC_CMD_NVRAM_INFO_OUT_READ_ONLY_LBN 5
+#define        MC_CMD_NVRAM_INFO_OUT_READ_ONLY_WIDTH 1
 #define        MC_CMD_NVRAM_INFO_OUT_CMAC_LBN 6
 #define        MC_CMD_NVRAM_INFO_OUT_CMAC_WIDTH 1
 #define        MC_CMD_NVRAM_INFO_OUT_A_B_LBN 7
 #define        MC_CMD_NVRAM_INFO_OUT_A_B_WIDTH 1
 #define       MC_CMD_NVRAM_INFO_OUT_PHYSDEV_OFST 16
+#define       MC_CMD_NVRAM_INFO_OUT_PHYSDEV_LEN 4
 #define       MC_CMD_NVRAM_INFO_OUT_PHYSADDR_OFST 20
+#define       MC_CMD_NVRAM_INFO_OUT_PHYSADDR_LEN 4
 
 /* MC_CMD_NVRAM_INFO_V2_OUT msgresponse */
 #define    MC_CMD_NVRAM_INFO_V2_OUT_LEN 28
 #define       MC_CMD_NVRAM_INFO_V2_OUT_TYPE_OFST 0
+#define       MC_CMD_NVRAM_INFO_V2_OUT_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_INFO_V2_OUT_SIZE_OFST 4
+#define       MC_CMD_NVRAM_INFO_V2_OUT_SIZE_LEN 4
 #define       MC_CMD_NVRAM_INFO_V2_OUT_ERASESIZE_OFST 8
+#define       MC_CMD_NVRAM_INFO_V2_OUT_ERASESIZE_LEN 4
 #define       MC_CMD_NVRAM_INFO_V2_OUT_FLAGS_OFST 12
+#define       MC_CMD_NVRAM_INFO_V2_OUT_FLAGS_LEN 4
 #define        MC_CMD_NVRAM_INFO_V2_OUT_PROTECTED_LBN 0
 #define        MC_CMD_NVRAM_INFO_V2_OUT_PROTECTED_WIDTH 1
 #define        MC_CMD_NVRAM_INFO_V2_OUT_TLV_LBN 1
 #define        MC_CMD_NVRAM_INFO_V2_OUT_TLV_WIDTH 1
+#define        MC_CMD_NVRAM_INFO_V2_OUT_READ_ONLY_LBN 5
+#define        MC_CMD_NVRAM_INFO_V2_OUT_READ_ONLY_WIDTH 1
 #define        MC_CMD_NVRAM_INFO_V2_OUT_A_B_LBN 7
 #define        MC_CMD_NVRAM_INFO_V2_OUT_A_B_WIDTH 1
 #define       MC_CMD_NVRAM_INFO_V2_OUT_PHYSDEV_OFST 16
+#define       MC_CMD_NVRAM_INFO_V2_OUT_PHYSDEV_LEN 4
 #define       MC_CMD_NVRAM_INFO_V2_OUT_PHYSADDR_OFST 20
+#define       MC_CMD_NVRAM_INFO_V2_OUT_PHYSADDR_LEN 4
 /* Writes must be multiples of this size. Added to support the MUM on Sorrento.
  */
 #define       MC_CMD_NVRAM_INFO_V2_OUT_WRITESIZE_OFST 24
+#define       MC_CMD_NVRAM_INFO_V2_OUT_WRITESIZE_LEN 4
 
 
 /***********************************/
@@ -3670,6 +4571,7 @@
  */
 #define    MC_CMD_NVRAM_UPDATE_START_IN_LEN 4
 #define       MC_CMD_NVRAM_UPDATE_START_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_UPDATE_START_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 
@@ -3680,9 +4582,11 @@
  */
 #define    MC_CMD_NVRAM_UPDATE_START_V2_IN_LEN 8
 #define       MC_CMD_NVRAM_UPDATE_START_V2_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_UPDATE_START_V2_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_UPDATE_START_V2_IN_FLAGS_OFST 4
+#define       MC_CMD_NVRAM_UPDATE_START_V2_IN_FLAGS_LEN 4
 #define        MC_CMD_NVRAM_UPDATE_START_V2_IN_FLAG_REPORT_VERIFY_RESULT_LBN 0
 #define        MC_CMD_NVRAM_UPDATE_START_V2_IN_FLAG_REPORT_VERIFY_RESULT_WIDTH 1
 
@@ -3703,20 +4607,26 @@
 /* MC_CMD_NVRAM_READ_IN msgrequest */
 #define    MC_CMD_NVRAM_READ_IN_LEN 12
 #define       MC_CMD_NVRAM_READ_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_READ_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_READ_IN_OFFSET_OFST 4
+#define       MC_CMD_NVRAM_READ_IN_OFFSET_LEN 4
 /* amount to read in bytes */
 #define       MC_CMD_NVRAM_READ_IN_LENGTH_OFST 8
+#define       MC_CMD_NVRAM_READ_IN_LENGTH_LEN 4
 
 /* MC_CMD_NVRAM_READ_IN_V2 msgrequest */
 #define    MC_CMD_NVRAM_READ_IN_V2_LEN 16
 #define       MC_CMD_NVRAM_READ_IN_V2_TYPE_OFST 0
+#define       MC_CMD_NVRAM_READ_IN_V2_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_READ_IN_V2_OFFSET_OFST 4
+#define       MC_CMD_NVRAM_READ_IN_V2_OFFSET_LEN 4
 /* amount to read in bytes */
 #define       MC_CMD_NVRAM_READ_IN_V2_LENGTH_OFST 8
+#define       MC_CMD_NVRAM_READ_IN_V2_LENGTH_LEN 4
 /* Optional control info. If a partition is stored with an A/B versioning
  * scheme (i.e. in more than one physical partition in NVRAM) the host can set
  * this to control which underlying physical partition is used to read data
@@ -3726,6 +4636,7 @@
  * verifying by reading with MODE=TARGET_BACKUP.
  */
 #define       MC_CMD_NVRAM_READ_IN_V2_MODE_OFST 12
+#define       MC_CMD_NVRAM_READ_IN_V2_MODE_LEN 4
 /* enum: Same as omitting MODE: caller sees data in current partition unless it
  * holds the write lock in which case it sees data in the partition it is
  * updating.
@@ -3765,10 +4676,13 @@
 #define    MC_CMD_NVRAM_WRITE_IN_LENMAX 252
 #define    MC_CMD_NVRAM_WRITE_IN_LEN(num) (12+1*(num))
 #define       MC_CMD_NVRAM_WRITE_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_WRITE_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_WRITE_IN_OFFSET_OFST 4
+#define       MC_CMD_NVRAM_WRITE_IN_OFFSET_LEN 4
 #define       MC_CMD_NVRAM_WRITE_IN_LENGTH_OFST 8
+#define       MC_CMD_NVRAM_WRITE_IN_LENGTH_LEN 4
 #define       MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_OFST 12
 #define       MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_LEN 1
 #define       MC_CMD_NVRAM_WRITE_IN_WRITE_BUFFER_MINNUM 1
@@ -3791,10 +4705,13 @@
 /* MC_CMD_NVRAM_ERASE_IN msgrequest */
 #define    MC_CMD_NVRAM_ERASE_IN_LEN 12
 #define       MC_CMD_NVRAM_ERASE_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_ERASE_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_ERASE_IN_OFFSET_OFST 4
+#define       MC_CMD_NVRAM_ERASE_IN_OFFSET_LEN 4
 #define       MC_CMD_NVRAM_ERASE_IN_LENGTH_OFST 8
+#define       MC_CMD_NVRAM_ERASE_IN_LENGTH_LEN 4
 
 /* MC_CMD_NVRAM_ERASE_OUT msgresponse */
 #define    MC_CMD_NVRAM_ERASE_OUT_LEN 0
@@ -3815,9 +4732,11 @@
  */
 #define    MC_CMD_NVRAM_UPDATE_FINISH_IN_LEN 8
 #define       MC_CMD_NVRAM_UPDATE_FINISH_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_UPDATE_FINISH_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_UPDATE_FINISH_IN_REBOOT_OFST 4
+#define       MC_CMD_NVRAM_UPDATE_FINISH_IN_REBOOT_LEN 4
 
 /* MC_CMD_NVRAM_UPDATE_FINISH_V2_IN msgrequest: Extended NVRAM_UPDATE_FINISH
  * request with additional flags indicating version of NVRAM_UPDATE commands in
@@ -3826,10 +4745,13 @@
  */
 #define    MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_LEN 12
 #define       MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 #define       MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_REBOOT_OFST 4
+#define       MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_REBOOT_LEN 4
 #define       MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAGS_OFST 8
+#define       MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAGS_LEN 4
 #define        MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_REPORT_VERIFY_RESULT_LBN 0
 #define        MC_CMD_NVRAM_UPDATE_FINISH_V2_IN_FLAG_REPORT_VERIFY_RESULT_WIDTH 1
 
@@ -3848,16 +4770,19 @@
  * This process takes a few seconds to complete. So is likely to take more than
  * the MCDI timeout. Hence signature verification is initiated when
  * MC_CMD_NVRAM_UPDATE_FINISH_V2_IN is received by the firmware, however, the
- * MCDI command returns immediately with error code EAGAIN. Subsequent
- * NVRAM_UPDATE_FINISH_V2_IN requests also return EAGAIN if the verification is
- * in progress. Once the verification has completed, this response payload
- * includes the results of the signature verification. Note that the nvram lock
- * in firmware is only released after the verification has completed and the
- * host has read back the result code from firmware.
+ * MCDI command is run in a background MCDI processing thread. This response
+ * payload includes the results of the signature verification. Note that the
+ * per-partition nvram lock in firmware is only released after the verification
+ * has completed.
  */
 #define    MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_LEN 4
 /* Result of nvram update completion processing */
 #define       MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_RESULT_CODE_OFST 0
+#define       MC_CMD_NVRAM_UPDATE_FINISH_V2_OUT_RESULT_CODE_LEN 4
+/* enum: Invalid return code; only non-zero values are defined. Defined as
+ * unknown for backwards compatibility with NVRAM_UPDATE_FINISH_OUT.
+ */
+#define          MC_CMD_NVRAM_VERIFY_RC_UNKNOWN 0x0
 /* enum: Verify succeeded without any errors. */
 #define          MC_CMD_NVRAM_VERIFY_RC_SUCCESS 0x1
 /* enum: CMS format verification failed due to an internal error. */
@@ -3884,6 +4809,12 @@
  * Trusted approver's list.
  */
 #define          MC_CMD_NVRAM_VERIFY_RC_NO_SIGNATURE_MATCH 0xb
+/* enum: The image contains a test-signed certificate, but the adapter accepts
+ * only production signed images.
+ */
+#define          MC_CMD_NVRAM_VERIFY_RC_REJECT_TEST_SIGNED 0xc
+/* enum: The image has a lower security level than the current firmware. */
+#define          MC_CMD_NVRAM_VERIFY_RC_SECURITY_LEVEL_DOWNGRADE 0xd
 
 
 /***********************************/
@@ -3911,6 +4842,7 @@
 /* MC_CMD_REBOOT_IN msgrequest */
 #define    MC_CMD_REBOOT_IN_LEN 4
 #define       MC_CMD_REBOOT_IN_FLAGS_OFST 0
+#define       MC_CMD_REBOOT_IN_FLAGS_LEN 4
 #define          MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION 0x1 /* enum */
 
 /* MC_CMD_REBOOT_OUT msgresponse */
@@ -3947,11 +4879,12 @@
  */
 #define MC_CMD_REBOOT_MODE 0x3f
 
-#define MC_CMD_0x3f_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x3f_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_REBOOT_MODE_IN msgrequest */
 #define    MC_CMD_REBOOT_MODE_IN_LEN 4
 #define       MC_CMD_REBOOT_MODE_IN_VALUE_OFST 0
+#define       MC_CMD_REBOOT_MODE_IN_VALUE_LEN 4
 /* enum: Normal. */
 #define          MC_CMD_REBOOT_MODE_NORMAL 0x0
 /* enum: Power-on Reset. */
@@ -3966,6 +4899,7 @@
 /* MC_CMD_REBOOT_MODE_OUT msgresponse */
 #define    MC_CMD_REBOOT_MODE_OUT_LEN 4
 #define       MC_CMD_REBOOT_MODE_OUT_VALUE_OFST 0
+#define       MC_CMD_REBOOT_MODE_OUT_VALUE_LEN 4
 
 
 /***********************************/
@@ -4001,7 +4935,7 @@
  */
 #define MC_CMD_SENSOR_INFO 0x41
 
-#define MC_CMD_0x41_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x41_PRIVILEGE_CTG SRIOV_CTG_GENERAL
 
 /* MC_CMD_SENSOR_INFO_IN msgrequest */
 #define    MC_CMD_SENSOR_INFO_IN_LEN 0
@@ -4015,12 +4949,14 @@
  * Page 1 contains sensors 32 to 62 (sensor 63 is the next page bit). etc.
  */
 #define       MC_CMD_SENSOR_INFO_EXT_IN_PAGE_OFST 0
+#define       MC_CMD_SENSOR_INFO_EXT_IN_PAGE_LEN 4
 
 /* MC_CMD_SENSOR_INFO_OUT msgresponse */
 #define    MC_CMD_SENSOR_INFO_OUT_LENMIN 4
 #define    MC_CMD_SENSOR_INFO_OUT_LENMAX 252
 #define    MC_CMD_SENSOR_INFO_OUT_LEN(num) (4+8*(num))
 #define       MC_CMD_SENSOR_INFO_OUT_MASK_OFST 0
+#define       MC_CMD_SENSOR_INFO_OUT_MASK_LEN 4
 /* enum: Controller temperature: degC */
 #define          MC_CMD_SENSOR_CONTROLLER_TEMP  0x0
 /* enum: Phy common temperature: degC */
@@ -4183,6 +5119,20 @@
 #define          MC_CMD_SENSOR_BOARD_FRONT_TEMP  0x4f
 /* enum: Board temperature (back): degC */
 #define          MC_CMD_SENSOR_BOARD_BACK_TEMP  0x50
+/* enum: 1.8v power current: mA */
+#define          MC_CMD_SENSOR_IN_I1V8  0x51
+/* enum: 2.5v power current: mA */
+#define          MC_CMD_SENSOR_IN_I2V5  0x52
+/* enum: 3.3v power current: mA */
+#define          MC_CMD_SENSOR_IN_I3V3  0x53
+/* enum: 12v power current: mA */
+#define          MC_CMD_SENSOR_IN_I12V0  0x54
+/* enum: 1.3v power: mV */
+#define          MC_CMD_SENSOR_IN_1V3  0x55
+/* enum: 1.3v power current: mA */
+#define          MC_CMD_SENSOR_IN_I1V3  0x56
+/* enum: Not a sensor: reserved for the next page flag */
+#define          MC_CMD_SENSOR_PAGE2_NEXT  0x5f
 /* MC_CMD_SENSOR_INFO_ENTRY_TYPEDEF */
 #define       MC_CMD_SENSOR_ENTRY_OFST 4
 #define       MC_CMD_SENSOR_ENTRY_LEN 8
@@ -4196,6 +5146,7 @@
 #define    MC_CMD_SENSOR_INFO_EXT_OUT_LENMAX 252
 #define    MC_CMD_SENSOR_INFO_EXT_OUT_LEN(num) (4+8*(num))
 #define       MC_CMD_SENSOR_INFO_EXT_OUT_MASK_OFST 0
+#define       MC_CMD_SENSOR_INFO_EXT_OUT_MASK_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_SENSOR_INFO_OUT */
 #define        MC_CMD_SENSOR_INFO_EXT_OUT_NEXT_PAGE_LBN 31
@@ -4247,7 +5198,7 @@
  */
 #define MC_CMD_READ_SENSORS 0x42
 
-#define MC_CMD_0x42_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x42_PRIVILEGE_CTG SRIOV_CTG_GENERAL
 
 /* MC_CMD_READ_SENSORS_IN msgrequest */
 #define    MC_CMD_READ_SENSORS_IN_LEN 8
@@ -4266,6 +5217,7 @@
 #define       MC_CMD_READ_SENSORS_EXT_IN_DMA_ADDR_HI_OFST 4
 /* Size in bytes of host buffer. */
 #define       MC_CMD_READ_SENSORS_EXT_IN_LENGTH_OFST 8
+#define       MC_CMD_READ_SENSORS_EXT_IN_LENGTH_LEN 4
 
 /* MC_CMD_READ_SENSORS_OUT msgresponse */
 #define    MC_CMD_READ_SENSORS_OUT_LEN 0
@@ -4319,6 +5271,7 @@
 /* MC_CMD_GET_PHY_STATE_OUT msgresponse */
 #define    MC_CMD_GET_PHY_STATE_OUT_LEN 4
 #define       MC_CMD_GET_PHY_STATE_OUT_STATE_OFST 0
+#define       MC_CMD_GET_PHY_STATE_OUT_STATE_LEN 4
 /* enum: Ok. */
 #define          MC_CMD_PHY_STATE_OK 0x1
 /* enum: Faulty. */
@@ -4355,6 +5308,7 @@
 /* MC_CMD_WOL_FILTER_GET_OUT msgresponse */
 #define    MC_CMD_WOL_FILTER_GET_OUT_LEN 4
 #define       MC_CMD_WOL_FILTER_GET_OUT_FILTER_ID_OFST 0
+#define       MC_CMD_WOL_FILTER_GET_OUT_FILTER_ID_LEN 4
 
 
 /***********************************/
@@ -4371,6 +5325,7 @@
 #define    MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LENMAX 252
 #define    MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_LEN(num) (4+4*(num))
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0
+#define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_LEN 4
 #define          MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_ARP 0x1 /* enum */
 #define          MC_CMD_LIGHTSOUT_OFFLOAD_PROTOCOL_NS  0x2 /* enum */
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_DATA_OFST 4
@@ -4381,13 +5336,16 @@
 /* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP msgrequest */
 #define    MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_LEN 14
 /*            MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 */
+/*            MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_LEN 4 */
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_MAC_OFST 4
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_MAC_LEN 6
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_IP_OFST 10
+#define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_ARP_IP_LEN 4
 
 /* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS msgrequest */
 #define    MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_LEN 42
 /*            MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0 */
+/*            MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_LEN 4 */
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_MAC_OFST 4
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_MAC_LEN 6
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_IN_NS_SNIPV6_OFST 10
@@ -4398,6 +5356,7 @@
 /* MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT msgresponse */
 #define    MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_LEN 4
 #define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_FILTER_ID_OFST 0
+#define       MC_CMD_ADD_LIGHTSOUT_OFFLOAD_OUT_FILTER_ID_LEN 4
 
 
 /***********************************/
@@ -4412,7 +5371,9 @@
 /* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN msgrequest */
 #define    MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_LEN 8
 #define       MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_OFST 0
+#define       MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_PROTOCOL_LEN 4
 #define       MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_FILTER_ID_OFST 4
+#define       MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_IN_FILTER_ID_LEN 4
 
 /* MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_OUT msgresponse */
 #define    MC_CMD_REMOVE_LIGHTSOUT_OFFLOAD_OUT_LEN 0
@@ -4451,6 +5412,7 @@
 #define    MC_CMD_TESTASSERT_V2_IN_LEN 4
 /* How to provoke the assertion */
 #define       MC_CMD_TESTASSERT_V2_IN_TYPE_OFST 0
+#define       MC_CMD_TESTASSERT_V2_IN_TYPE_LEN 4
 /* enum: Assert using the FAIL_ASSERTION_WITH_USEFUL_VALUES macro. Unless
  * you're testing firmware, this is what you want.
  */
@@ -4486,6 +5448,7 @@
 #define    MC_CMD_WORKAROUND_IN_LEN 8
 /* The enums here must correspond with those in MC_CMD_GET_WORKAROUND. */
 #define       MC_CMD_WORKAROUND_IN_TYPE_OFST 0
+#define       MC_CMD_WORKAROUND_IN_TYPE_LEN 4
 /* enum: Bug 17230 work around. */
 #define          MC_CMD_WORKAROUND_BUG17230 0x1
 /* enum: Bug 35388 work around (unsafe EVQ writes). */
@@ -4514,6 +5477,7 @@
  * the workaround
  */
 #define       MC_CMD_WORKAROUND_IN_ENABLED_OFST 4
+#define       MC_CMD_WORKAROUND_IN_ENABLED_LEN 4
 
 /* MC_CMD_WORKAROUND_OUT msgresponse */
 #define    MC_CMD_WORKAROUND_OUT_LEN 0
@@ -4523,6 +5487,7 @@
  */
 #define    MC_CMD_WORKAROUND_EXT_OUT_LEN 4
 #define       MC_CMD_WORKAROUND_EXT_OUT_FLAGS_OFST 0
+#define       MC_CMD_WORKAROUND_EXT_OUT_FLAGS_LEN 4
 #define        MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_LBN 0
 #define        MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_WIDTH 1
 
@@ -4543,6 +5508,7 @@
 /* MC_CMD_GET_PHY_MEDIA_INFO_IN msgrequest */
 #define    MC_CMD_GET_PHY_MEDIA_INFO_IN_LEN 4
 #define       MC_CMD_GET_PHY_MEDIA_INFO_IN_PAGE_OFST 0
+#define       MC_CMD_GET_PHY_MEDIA_INFO_IN_PAGE_LEN 4
 
 /* MC_CMD_GET_PHY_MEDIA_INFO_OUT msgresponse */
 #define    MC_CMD_GET_PHY_MEDIA_INFO_OUT_LENMIN 5
@@ -4550,6 +5516,7 @@
 #define    MC_CMD_GET_PHY_MEDIA_INFO_OUT_LEN(num) (4+1*(num))
 /* in bytes */
 #define       MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATALEN_OFST 0
+#define       MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATALEN_LEN 4
 #define       MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_OFST 4
 #define       MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_LEN 1
 #define       MC_CMD_GET_PHY_MEDIA_INFO_OUT_DATA_MINNUM 1
@@ -4568,12 +5535,14 @@
 /* MC_CMD_NVRAM_TEST_IN msgrequest */
 #define    MC_CMD_NVRAM_TEST_IN_LEN 4
 #define       MC_CMD_NVRAM_TEST_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_TEST_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_NVRAM_TYPES/MC_CMD_NVRAM_TYPES_OUT/TYPES */
 
 /* MC_CMD_NVRAM_TEST_OUT msgresponse */
 #define    MC_CMD_NVRAM_TEST_OUT_LEN 4
 #define       MC_CMD_NVRAM_TEST_OUT_RESULT_OFST 0
+#define       MC_CMD_NVRAM_TEST_OUT_RESULT_LEN 4
 /* enum: Passed. */
 #define          MC_CMD_NVRAM_TEST_PASS 0x0
 /* enum: Failed. */
@@ -4594,12 +5563,16 @@
 #define    MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_LEN 16
 /* 0-6 low->high de-emph. */
 #define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_LEVEL_OFST 0
+#define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_LEVEL_LEN 4
 /* 0-8 low->high ref.V */
 #define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_DT_CFG_OFST 4
+#define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_TXEQ_DT_CFG_LEN 4
 /* 0-8 0-8 low->high boost */
 #define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_BOOST_OFST 8
+#define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_BOOST_LEN 4
 /* 0-8 low->high ref.V */
 #define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_DT_CFG_OFST 12
+#define       MC_CMD_MRSFP_TWEAK_IN_EQ_CONFIG_RXEQ_DT_CFG_LEN 4
 
 /* MC_CMD_MRSFP_TWEAK_IN_READ_ONLY msgrequest */
 #define    MC_CMD_MRSFP_TWEAK_IN_READ_ONLY_LEN 0
@@ -4608,10 +5581,13 @@
 #define    MC_CMD_MRSFP_TWEAK_OUT_LEN 12
 /* input bits */
 #define       MC_CMD_MRSFP_TWEAK_OUT_IOEXP_INPUTS_OFST 0
+#define       MC_CMD_MRSFP_TWEAK_OUT_IOEXP_INPUTS_LEN 4
 /* output bits */
 #define       MC_CMD_MRSFP_TWEAK_OUT_IOEXP_OUTPUTS_OFST 4
+#define       MC_CMD_MRSFP_TWEAK_OUT_IOEXP_OUTPUTS_LEN 4
 /* direction */
 #define       MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_OFST 8
+#define       MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_LEN 4
 /* enum: Out. */
 #define          MC_CMD_MRSFP_TWEAK_OUT_IOEXP_DIRECTION_OUT 0x0
 /* enum: In. */
@@ -4626,21 +5602,26 @@
  */
 #define MC_CMD_SENSOR_SET_LIMS 0x4e
 
-#define MC_CMD_0x4e_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x4e_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_SENSOR_SET_LIMS_IN msgrequest */
 #define    MC_CMD_SENSOR_SET_LIMS_IN_LEN 20
 #define       MC_CMD_SENSOR_SET_LIMS_IN_SENSOR_OFST 0
+#define       MC_CMD_SENSOR_SET_LIMS_IN_SENSOR_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_SENSOR_INFO/MC_CMD_SENSOR_INFO_OUT/MASK */
 /* interpretation is is sensor-specific. */
 #define       MC_CMD_SENSOR_SET_LIMS_IN_LOW0_OFST 4
+#define       MC_CMD_SENSOR_SET_LIMS_IN_LOW0_LEN 4
 /* interpretation is is sensor-specific. */
 #define       MC_CMD_SENSOR_SET_LIMS_IN_HI0_OFST 8
+#define       MC_CMD_SENSOR_SET_LIMS_IN_HI0_LEN 4
 /* interpretation is is sensor-specific. */
 #define       MC_CMD_SENSOR_SET_LIMS_IN_LOW1_OFST 12
+#define       MC_CMD_SENSOR_SET_LIMS_IN_LOW1_LEN 4
 /* interpretation is is sensor-specific. */
 #define       MC_CMD_SENSOR_SET_LIMS_IN_HI1_OFST 16
+#define       MC_CMD_SENSOR_SET_LIMS_IN_HI1_LEN 4
 
 /* MC_CMD_SENSOR_SET_LIMS_OUT msgresponse */
 #define    MC_CMD_SENSOR_SET_LIMS_OUT_LEN 0
@@ -4657,9 +5638,13 @@
 /* MC_CMD_GET_RESOURCE_LIMITS_OUT msgresponse */
 #define    MC_CMD_GET_RESOURCE_LIMITS_OUT_LEN 16
 #define       MC_CMD_GET_RESOURCE_LIMITS_OUT_BUFTBL_OFST 0
+#define       MC_CMD_GET_RESOURCE_LIMITS_OUT_BUFTBL_LEN 4
 #define       MC_CMD_GET_RESOURCE_LIMITS_OUT_EVQ_OFST 4
+#define       MC_CMD_GET_RESOURCE_LIMITS_OUT_EVQ_LEN 4
 #define       MC_CMD_GET_RESOURCE_LIMITS_OUT_RXQ_OFST 8
+#define       MC_CMD_GET_RESOURCE_LIMITS_OUT_RXQ_LEN 4
 #define       MC_CMD_GET_RESOURCE_LIMITS_OUT_TXQ_OFST 12
+#define       MC_CMD_GET_RESOURCE_LIMITS_OUT_TXQ_LEN 4
 
 
 /***********************************/
@@ -4680,6 +5665,7 @@
 #define    MC_CMD_NVRAM_PARTITIONS_OUT_LEN(num) (4+4*(num))
 /* total number of partitions */
 #define       MC_CMD_NVRAM_PARTITIONS_OUT_NUM_PARTITIONS_OFST 0
+#define       MC_CMD_NVRAM_PARTITIONS_OUT_NUM_PARTITIONS_LEN 4
 /* type ID code for each of NUM_PARTITIONS partitions */
 #define       MC_CMD_NVRAM_PARTITIONS_OUT_TYPE_ID_OFST 4
 #define       MC_CMD_NVRAM_PARTITIONS_OUT_TYPE_ID_LEN 4
@@ -4700,6 +5686,7 @@
 #define    MC_CMD_NVRAM_METADATA_IN_LEN 4
 /* Partition type ID code */
 #define       MC_CMD_NVRAM_METADATA_IN_TYPE_OFST 0
+#define       MC_CMD_NVRAM_METADATA_IN_TYPE_LEN 4
 
 /* MC_CMD_NVRAM_METADATA_OUT msgresponse */
 #define    MC_CMD_NVRAM_METADATA_OUT_LENMIN 20
@@ -4707,7 +5694,9 @@
 #define    MC_CMD_NVRAM_METADATA_OUT_LEN(num) (20+1*(num))
 /* Partition type ID code */
 #define       MC_CMD_NVRAM_METADATA_OUT_TYPE_OFST 0
+#define       MC_CMD_NVRAM_METADATA_OUT_TYPE_LEN 4
 #define       MC_CMD_NVRAM_METADATA_OUT_FLAGS_OFST 4
+#define       MC_CMD_NVRAM_METADATA_OUT_FLAGS_LEN 4
 #define        MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_LBN 0
 #define        MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_WIDTH 1
 #define        MC_CMD_NVRAM_METADATA_OUT_VERSION_VALID_LBN 1
@@ -4716,6 +5705,7 @@
 #define        MC_CMD_NVRAM_METADATA_OUT_DESCRIPTION_VALID_WIDTH 1
 /* Subtype ID code for content of this partition */
 #define       MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_OFST 8
+#define       MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_LEN 4
 /* 1st component of W.X.Y.Z version number for content of this partition */
 #define       MC_CMD_NVRAM_METADATA_OUT_VERSION_W_OFST 12
 #define       MC_CMD_NVRAM_METADATA_OUT_VERSION_W_LEN 2
@@ -4756,8 +5746,10 @@
 #define       MC_CMD_GET_MAC_ADDRESSES_OUT_RESERVED_LEN 2
 /* Number of allocated MAC addresses */
 #define       MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_COUNT_OFST 8
+#define       MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_COUNT_LEN 4
 /* Spacing of allocated MAC addresses */
 #define       MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_STRIDE_OFST 12
+#define       MC_CMD_GET_MAC_ADDRESSES_OUT_MAC_STRIDE_LEN 4
 
 
 /***********************************/
@@ -4772,6 +5764,7 @@
 #define    MC_CMD_CLP_IN_LEN 4
 /* Sub operation */
 #define       MC_CMD_CLP_IN_OP_OFST 0
+#define       MC_CMD_CLP_IN_OP_LEN 4
 /* enum: Return to factory default settings */
 #define          MC_CMD_CLP_OP_DEFAULT 0x1
 /* enum: Set MAC address */
@@ -4789,6 +5782,7 @@
 /* MC_CMD_CLP_IN_DEFAULT msgrequest */
 #define    MC_CMD_CLP_IN_DEFAULT_LEN 4
 /*            MC_CMD_CLP_IN_OP_OFST 0 */
+/*            MC_CMD_CLP_IN_OP_LEN 4 */
 
 /* MC_CMD_CLP_OUT_DEFAULT msgresponse */
 #define    MC_CMD_CLP_OUT_DEFAULT_LEN 0
@@ -4796,6 +5790,7 @@
 /* MC_CMD_CLP_IN_SET_MAC msgrequest */
 #define    MC_CMD_CLP_IN_SET_MAC_LEN 12
 /*            MC_CMD_CLP_IN_OP_OFST 0 */
+/*            MC_CMD_CLP_IN_OP_LEN 4 */
 /* MAC address assigned to port */
 #define       MC_CMD_CLP_IN_SET_MAC_ADDR_OFST 4
 #define       MC_CMD_CLP_IN_SET_MAC_ADDR_LEN 6
@@ -4809,6 +5804,7 @@
 /* MC_CMD_CLP_IN_GET_MAC msgrequest */
 #define    MC_CMD_CLP_IN_GET_MAC_LEN 4
 /*            MC_CMD_CLP_IN_OP_OFST 0 */
+/*            MC_CMD_CLP_IN_OP_LEN 4 */
 
 /* MC_CMD_CLP_OUT_GET_MAC msgresponse */
 #define    MC_CMD_CLP_OUT_GET_MAC_LEN 8
@@ -4822,6 +5818,7 @@
 /* MC_CMD_CLP_IN_SET_BOOT msgrequest */
 #define    MC_CMD_CLP_IN_SET_BOOT_LEN 5
 /*            MC_CMD_CLP_IN_OP_OFST 0 */
+/*            MC_CMD_CLP_IN_OP_LEN 4 */
 /* Boot flag */
 #define       MC_CMD_CLP_IN_SET_BOOT_FLAG_OFST 4
 #define       MC_CMD_CLP_IN_SET_BOOT_FLAG_LEN 1
@@ -4832,6 +5829,7 @@
 /* MC_CMD_CLP_IN_GET_BOOT msgrequest */
 #define    MC_CMD_CLP_IN_GET_BOOT_LEN 4
 /*            MC_CMD_CLP_IN_OP_OFST 0 */
+/*            MC_CMD_CLP_IN_OP_LEN 4 */
 
 /* MC_CMD_CLP_OUT_GET_BOOT msgresponse */
 #define    MC_CMD_CLP_OUT_GET_BOOT_LEN 4
@@ -4849,11 +5847,12 @@
  */
 #define MC_CMD_MUM 0x57
 
-#define MC_CMD_0x57_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x57_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_MUM_IN msgrequest */
 #define    MC_CMD_MUM_IN_LEN 4
 #define       MC_CMD_MUM_IN_OP_HDR_OFST 0
+#define       MC_CMD_MUM_IN_OP_HDR_LEN 4
 #define        MC_CMD_MUM_IN_OP_LBN 0
 #define        MC_CMD_MUM_IN_OP_WIDTH 8
 /* enum: NULL MCDI command to MUM */
@@ -4893,26 +5892,32 @@
 #define    MC_CMD_MUM_IN_NULL_LEN 4
 /* MUM cmd header */
 #define       MC_CMD_MUM_IN_CMD_OFST 0
+#define       MC_CMD_MUM_IN_CMD_LEN 4
 
 /* MC_CMD_MUM_IN_GET_VERSION msgrequest */
 #define    MC_CMD_MUM_IN_GET_VERSION_LEN 4
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 
 /* MC_CMD_MUM_IN_READ msgrequest */
 #define    MC_CMD_MUM_IN_READ_LEN 16
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 /* ID of (device connected to MUM) to read from registers of */
 #define       MC_CMD_MUM_IN_READ_DEVICE_OFST 4
+#define       MC_CMD_MUM_IN_READ_DEVICE_LEN 4
 /* enum: Hittite HMC1035 clock generator on Sorrento board */
 #define          MC_CMD_MUM_DEV_HITTITE 0x1
 /* enum: Hittite HMC1035 clock generator for NIC-side on Sorrento board */
 #define          MC_CMD_MUM_DEV_HITTITE_NIC 0x2
 /* 32-bit address to read from */
 #define       MC_CMD_MUM_IN_READ_ADDR_OFST 8
+#define       MC_CMD_MUM_IN_READ_ADDR_LEN 4
 /* Number of words to read. */
 #define       MC_CMD_MUM_IN_READ_NUMWORDS_OFST 12
+#define       MC_CMD_MUM_IN_READ_NUMWORDS_LEN 4
 
 /* MC_CMD_MUM_IN_WRITE msgrequest */
 #define    MC_CMD_MUM_IN_WRITE_LENMIN 16
@@ -4920,12 +5925,15 @@
 #define    MC_CMD_MUM_IN_WRITE_LEN(num) (12+4*(num))
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 /* ID of (device connected to MUM) to write to registers of */
 #define       MC_CMD_MUM_IN_WRITE_DEVICE_OFST 4
+#define       MC_CMD_MUM_IN_WRITE_DEVICE_LEN 4
 /* enum: Hittite HMC1035 clock generator on Sorrento board */
 /*               MC_CMD_MUM_DEV_HITTITE 0x1 */
 /* 32-bit address to write to */
 #define       MC_CMD_MUM_IN_WRITE_ADDR_OFST 8
+#define       MC_CMD_MUM_IN_WRITE_ADDR_LEN 4
 /* Words to write */
 #define       MC_CMD_MUM_IN_WRITE_BUFFER_OFST 12
 #define       MC_CMD_MUM_IN_WRITE_BUFFER_LEN 4
@@ -4938,12 +5946,16 @@
 #define    MC_CMD_MUM_IN_RAW_CMD_LEN(num) (16+1*(num))
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 /* MUM I2C cmd code */
 #define       MC_CMD_MUM_IN_RAW_CMD_CMD_CODE_OFST 4
+#define       MC_CMD_MUM_IN_RAW_CMD_CMD_CODE_LEN 4
 /* Number of bytes to write */
 #define       MC_CMD_MUM_IN_RAW_CMD_NUM_WRITE_OFST 8
+#define       MC_CMD_MUM_IN_RAW_CMD_NUM_WRITE_LEN 4
 /* Number of bytes to read */
 #define       MC_CMD_MUM_IN_RAW_CMD_NUM_READ_OFST 12
+#define       MC_CMD_MUM_IN_RAW_CMD_NUM_READ_LEN 4
 /* Bytes to write */
 #define       MC_CMD_MUM_IN_RAW_CMD_WRITE_DATA_OFST 16
 #define       MC_CMD_MUM_IN_RAW_CMD_WRITE_DATA_LEN 1
@@ -4954,21 +5966,28 @@
 #define    MC_CMD_MUM_IN_LOG_LEN 8
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_LOG_OP_OFST 4
+#define       MC_CMD_MUM_IN_LOG_OP_LEN 4
 #define          MC_CMD_MUM_IN_LOG_OP_UART  0x1 /* enum */
 
 /* MC_CMD_MUM_IN_LOG_OP_UART msgrequest */
 #define    MC_CMD_MUM_IN_LOG_OP_UART_LEN 12
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 /*            MC_CMD_MUM_IN_LOG_OP_OFST 4 */
+/*            MC_CMD_MUM_IN_LOG_OP_LEN 4 */
 /* Enable/disable debug output to UART */
 #define       MC_CMD_MUM_IN_LOG_OP_UART_ENABLE_OFST 8
+#define       MC_CMD_MUM_IN_LOG_OP_UART_ENABLE_LEN 4
 
 /* MC_CMD_MUM_IN_GPIO msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_LEN 8
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_HDR_LEN 4
 #define        MC_CMD_MUM_IN_GPIO_OPCODE_LBN 0
 #define        MC_CMD_MUM_IN_GPIO_OPCODE_WIDTH 8
 #define          MC_CMD_MUM_IN_GPIO_IN_READ 0x0 /* enum */
@@ -4981,40 +6000,56 @@
 /* MC_CMD_MUM_IN_GPIO_IN_READ msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_IN_READ_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_IN_READ_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_IN_READ_HDR_LEN 4
 
 /* MC_CMD_MUM_IN_GPIO_OUT_WRITE msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OUT_WRITE_LEN 16
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OUT_WRITE_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OUT_WRITE_HDR_LEN 4
 /* The first 32-bit word to be written to the GPIO OUT register. */
 #define       MC_CMD_MUM_IN_GPIO_OUT_WRITE_GPIOMASK1_OFST 8
+#define       MC_CMD_MUM_IN_GPIO_OUT_WRITE_GPIOMASK1_LEN 4
 /* The second 32-bit word to be written to the GPIO OUT register. */
 #define       MC_CMD_MUM_IN_GPIO_OUT_WRITE_GPIOMASK2_OFST 12
+#define       MC_CMD_MUM_IN_GPIO_OUT_WRITE_GPIOMASK2_LEN 4
 
 /* MC_CMD_MUM_IN_GPIO_OUT_READ msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OUT_READ_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OUT_READ_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OUT_READ_HDR_LEN 4
 
 /* MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_LEN 16
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_HDR_LEN 4
 /* The first 32-bit word to be written to the GPIO OUT ENABLE register. */
 #define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_GPIOMASK1_OFST 8
+#define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_GPIOMASK1_LEN 4
 /* The second 32-bit word to be written to the GPIO OUT ENABLE register. */
 #define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_GPIOMASK2_OFST 12
+#define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_WRITE_GPIOMASK2_LEN 4
 
 /* MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OUT_ENABLE_READ_HDR_LEN 4
 
 /* MC_CMD_MUM_IN_GPIO_OP msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OP_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OP_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OP_HDR_LEN 4
 #define        MC_CMD_MUM_IN_GPIO_OP_BITWISE_OP_LBN 8
 #define        MC_CMD_MUM_IN_GPIO_OP_BITWISE_OP_WIDTH 8
 #define          MC_CMD_MUM_IN_GPIO_OP_OUT_READ 0x0 /* enum */
@@ -5027,26 +6062,34 @@
 /* MC_CMD_MUM_IN_GPIO_OP_OUT_READ msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OP_OUT_READ_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OP_OUT_READ_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OP_OUT_READ_HDR_LEN 4
 
 /* MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_HDR_LEN 4
 #define        MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_WRITEBIT_LBN 24
 #define        MC_CMD_MUM_IN_GPIO_OP_OUT_WRITE_WRITEBIT_WIDTH 8
 
 /* MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_HDR_LEN 4
 #define        MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_CFG_LBN 24
 #define        MC_CMD_MUM_IN_GPIO_OP_OUT_CONFIG_CFG_WIDTH 8
 
 /* MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE msgrequest */
 #define    MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_LEN 8
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_HDR_OFST 4
+#define       MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_HDR_LEN 4
 #define        MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_ENABLEBIT_LBN 24
 #define        MC_CMD_MUM_IN_GPIO_OP_OUT_ENABLE_ENABLEBIT_WIDTH 8
 
@@ -5054,7 +6097,9 @@
 #define    MC_CMD_MUM_IN_READ_SENSORS_LEN 8
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_READ_SENSORS_PARAMS_OFST 4
+#define       MC_CMD_MUM_IN_READ_SENSORS_PARAMS_LEN 4
 #define        MC_CMD_MUM_IN_READ_SENSORS_SENSOR_ID_LBN 0
 #define        MC_CMD_MUM_IN_READ_SENSORS_SENSOR_ID_WIDTH 8
 #define        MC_CMD_MUM_IN_READ_SENSORS_NUM_SENSORS_LBN 8
@@ -5064,13 +6109,16 @@
 #define    MC_CMD_MUM_IN_PROGRAM_CLOCKS_LEN 12
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 /* Bit-mask of clocks to be programmed */
 #define       MC_CMD_MUM_IN_PROGRAM_CLOCKS_MASK_OFST 4
+#define       MC_CMD_MUM_IN_PROGRAM_CLOCKS_MASK_LEN 4
 #define          MC_CMD_MUM_CLOCK_ID_FPGA 0x0 /* enum */
 #define          MC_CMD_MUM_CLOCK_ID_DDR 0x1 /* enum */
 #define          MC_CMD_MUM_CLOCK_ID_NIC 0x2 /* enum */
 /* Control flags for clock programming */
 #define       MC_CMD_MUM_IN_PROGRAM_CLOCKS_FLAGS_OFST 8
+#define       MC_CMD_MUM_IN_PROGRAM_CLOCKS_FLAGS_LEN 4
 #define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_OVERCLOCK_110_LBN 0
 #define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_OVERCLOCK_110_WIDTH 1
 #define        MC_CMD_MUM_IN_PROGRAM_CLOCKS_CLOCK_NIC_FROM_FPGA_LBN 1
@@ -5082,19 +6130,24 @@
 #define    MC_CMD_MUM_IN_FPGA_LOAD_LEN 8
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 /* Enable/Disable FPGA config from flash */
 #define       MC_CMD_MUM_IN_FPGA_LOAD_ENABLE_OFST 4
+#define       MC_CMD_MUM_IN_FPGA_LOAD_ENABLE_LEN 4
 
 /* MC_CMD_MUM_IN_READ_ATB_SENSOR msgrequest */
 #define    MC_CMD_MUM_IN_READ_ATB_SENSOR_LEN 4
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 
 /* MC_CMD_MUM_IN_QSFP msgrequest */
 #define    MC_CMD_MUM_IN_QSFP_LEN 12
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_QSFP_HDR_OFST 4
+#define       MC_CMD_MUM_IN_QSFP_HDR_LEN 4
 #define        MC_CMD_MUM_IN_QSFP_OPCODE_LBN 0
 #define        MC_CMD_MUM_IN_QSFP_OPCODE_WIDTH 4
 #define          MC_CMD_MUM_IN_QSFP_INIT 0x0 /* enum */
@@ -5104,52 +6157,77 @@
 #define          MC_CMD_MUM_IN_QSFP_FILL_STATS 0x4 /* enum */
 #define          MC_CMD_MUM_IN_QSFP_POLL_BIST 0x5 /* enum */
 #define       MC_CMD_MUM_IN_QSFP_IDX_OFST 8
+#define       MC_CMD_MUM_IN_QSFP_IDX_LEN 4
 
 /* MC_CMD_MUM_IN_QSFP_INIT msgrequest */
 #define    MC_CMD_MUM_IN_QSFP_INIT_LEN 16
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_QSFP_INIT_HDR_OFST 4
+#define       MC_CMD_MUM_IN_QSFP_INIT_HDR_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_INIT_IDX_OFST 8
+#define       MC_CMD_MUM_IN_QSFP_INIT_IDX_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_INIT_CAGE_OFST 12
+#define       MC_CMD_MUM_IN_QSFP_INIT_CAGE_LEN 4
 
 /* MC_CMD_MUM_IN_QSFP_RECONFIGURE msgrequest */
 #define    MC_CMD_MUM_IN_QSFP_RECONFIGURE_LEN 24
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_HDR_OFST 4
+#define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_HDR_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_IDX_OFST 8
+#define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_IDX_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_TX_DISABLE_OFST 12
+#define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_TX_DISABLE_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_PORT_LANES_OFST 16
+#define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_PORT_LANES_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_PORT_LINK_SPEED_OFST 20
+#define       MC_CMD_MUM_IN_QSFP_RECONFIGURE_PORT_LINK_SPEED_LEN 4
 
 /* MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP msgrequest */
 #define    MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_LEN 12
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_HDR_OFST 4
+#define       MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_HDR_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_IDX_OFST 8
+#define       MC_CMD_MUM_IN_QSFP_GET_SUPPORTED_CAP_IDX_LEN 4
 
 /* MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO msgrequest */
 #define    MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_LEN 16
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_HDR_OFST 4
+#define       MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_HDR_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_IDX_OFST 8
+#define       MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_IDX_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_PAGE_OFST 12
+#define       MC_CMD_MUM_IN_QSFP_GET_MEDIA_INFO_PAGE_LEN 4
 
 /* MC_CMD_MUM_IN_QSFP_FILL_STATS msgrequest */
 #define    MC_CMD_MUM_IN_QSFP_FILL_STATS_LEN 12
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_QSFP_FILL_STATS_HDR_OFST 4
+#define       MC_CMD_MUM_IN_QSFP_FILL_STATS_HDR_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_FILL_STATS_IDX_OFST 8
+#define       MC_CMD_MUM_IN_QSFP_FILL_STATS_IDX_LEN 4
 
 /* MC_CMD_MUM_IN_QSFP_POLL_BIST msgrequest */
 #define    MC_CMD_MUM_IN_QSFP_POLL_BIST_LEN 12
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 #define       MC_CMD_MUM_IN_QSFP_POLL_BIST_HDR_OFST 4
+#define       MC_CMD_MUM_IN_QSFP_POLL_BIST_HDR_LEN 4
 #define       MC_CMD_MUM_IN_QSFP_POLL_BIST_IDX_OFST 8
+#define       MC_CMD_MUM_IN_QSFP_POLL_BIST_IDX_LEN 4
 
 /* MC_CMD_MUM_IN_READ_DDR_INFO msgrequest */
 #define    MC_CMD_MUM_IN_READ_DDR_INFO_LEN 4
 /* MUM cmd header */
 /*            MC_CMD_MUM_IN_CMD_OFST 0 */
+/*            MC_CMD_MUM_IN_CMD_LEN 4 */
 
 /* MC_CMD_MUM_OUT msgresponse */
 #define    MC_CMD_MUM_OUT_LEN 0
@@ -5160,6 +6238,7 @@
 /* MC_CMD_MUM_OUT_GET_VERSION msgresponse */
 #define    MC_CMD_MUM_OUT_GET_VERSION_LEN 12
 #define       MC_CMD_MUM_OUT_GET_VERSION_FIRMWARE_OFST 0
+#define       MC_CMD_MUM_OUT_GET_VERSION_FIRMWARE_LEN 4
 #define       MC_CMD_MUM_OUT_GET_VERSION_VERSION_OFST 4
 #define       MC_CMD_MUM_OUT_GET_VERSION_VERSION_LEN 8
 #define       MC_CMD_MUM_OUT_GET_VERSION_VERSION_LO_OFST 4
@@ -5197,8 +6276,10 @@
 #define    MC_CMD_MUM_OUT_GPIO_IN_READ_LEN 8
 /* The first 32-bit word read from the GPIO IN register. */
 #define       MC_CMD_MUM_OUT_GPIO_IN_READ_GPIOMASK1_OFST 0
+#define       MC_CMD_MUM_OUT_GPIO_IN_READ_GPIOMASK1_LEN 4
 /* The second 32-bit word read from the GPIO IN register. */
 #define       MC_CMD_MUM_OUT_GPIO_IN_READ_GPIOMASK2_OFST 4
+#define       MC_CMD_MUM_OUT_GPIO_IN_READ_GPIOMASK2_LEN 4
 
 /* MC_CMD_MUM_OUT_GPIO_OUT_WRITE msgresponse */
 #define    MC_CMD_MUM_OUT_GPIO_OUT_WRITE_LEN 0
@@ -5207,8 +6288,10 @@
 #define    MC_CMD_MUM_OUT_GPIO_OUT_READ_LEN 8
 /* The first 32-bit word read from the GPIO OUT register. */
 #define       MC_CMD_MUM_OUT_GPIO_OUT_READ_GPIOMASK1_OFST 0
+#define       MC_CMD_MUM_OUT_GPIO_OUT_READ_GPIOMASK1_LEN 4
 /* The second 32-bit word read from the GPIO OUT register. */
 #define       MC_CMD_MUM_OUT_GPIO_OUT_READ_GPIOMASK2_OFST 4
+#define       MC_CMD_MUM_OUT_GPIO_OUT_READ_GPIOMASK2_LEN 4
 
 /* MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_WRITE msgresponse */
 #define    MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_WRITE_LEN 0
@@ -5216,11 +6299,14 @@
 /* MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ msgresponse */
 #define    MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_LEN 8
 #define       MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_GPIOMASK1_OFST 0
+#define       MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_GPIOMASK1_LEN 4
 #define       MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_GPIOMASK2_OFST 4
+#define       MC_CMD_MUM_OUT_GPIO_OUT_ENABLE_READ_GPIOMASK2_LEN 4
 
 /* MC_CMD_MUM_OUT_GPIO_OP_OUT_READ msgresponse */
 #define    MC_CMD_MUM_OUT_GPIO_OP_OUT_READ_LEN 4
 #define       MC_CMD_MUM_OUT_GPIO_OP_OUT_READ_BIT_READ_OFST 0
+#define       MC_CMD_MUM_OUT_GPIO_OP_OUT_READ_BIT_READ_LEN 4
 
 /* MC_CMD_MUM_OUT_GPIO_OP_OUT_WRITE msgresponse */
 #define    MC_CMD_MUM_OUT_GPIO_OP_OUT_WRITE_LEN 0
@@ -5249,6 +6335,7 @@
 /* MC_CMD_MUM_OUT_PROGRAM_CLOCKS msgresponse */
 #define    MC_CMD_MUM_OUT_PROGRAM_CLOCKS_LEN 4
 #define       MC_CMD_MUM_OUT_PROGRAM_CLOCKS_OK_MASK_OFST 0
+#define       MC_CMD_MUM_OUT_PROGRAM_CLOCKS_OK_MASK_LEN 4
 
 /* MC_CMD_MUM_OUT_FPGA_LOAD msgresponse */
 #define    MC_CMD_MUM_OUT_FPGA_LOAD_LEN 0
@@ -5256,6 +6343,7 @@
 /* MC_CMD_MUM_OUT_READ_ATB_SENSOR msgresponse */
 #define    MC_CMD_MUM_OUT_READ_ATB_SENSOR_LEN 4
 #define       MC_CMD_MUM_OUT_READ_ATB_SENSOR_RESULT_OFST 0
+#define       MC_CMD_MUM_OUT_READ_ATB_SENSOR_RESULT_LEN 4
 
 /* MC_CMD_MUM_OUT_QSFP_INIT msgresponse */
 #define    MC_CMD_MUM_OUT_QSFP_INIT_LEN 0
@@ -5263,7 +6351,9 @@
 /* MC_CMD_MUM_OUT_QSFP_RECONFIGURE msgresponse */
 #define    MC_CMD_MUM_OUT_QSFP_RECONFIGURE_LEN 8
 #define       MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_LP_CAP_OFST 0
+#define       MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_LP_CAP_LEN 4
 #define       MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_FLAGS_OFST 4
+#define       MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_FLAGS_LEN 4
 #define        MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_READY_LBN 0
 #define        MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_READY_WIDTH 1
 #define        MC_CMD_MUM_OUT_QSFP_RECONFIGURE_PORT_PHY_LINK_UP_LBN 1
@@ -5272,6 +6362,7 @@
 /* MC_CMD_MUM_OUT_QSFP_GET_SUPPORTED_CAP msgresponse */
 #define    MC_CMD_MUM_OUT_QSFP_GET_SUPPORTED_CAP_LEN 4
 #define       MC_CMD_MUM_OUT_QSFP_GET_SUPPORTED_CAP_PORT_PHY_LP_CAP_OFST 0
+#define       MC_CMD_MUM_OUT_QSFP_GET_SUPPORTED_CAP_PORT_PHY_LP_CAP_LEN 4
 
 /* MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO msgresponse */
 #define    MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_LENMIN 5
@@ -5279,6 +6370,7 @@
 #define    MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_LEN(num) (4+1*(num))
 /* in bytes */
 #define       MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATALEN_OFST 0
+#define       MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATALEN_LEN 4
 #define       MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATA_OFST 4
 #define       MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATA_LEN 1
 #define       MC_CMD_MUM_OUT_QSFP_GET_MEDIA_INFO_DATA_MINNUM 1
@@ -5287,11 +6379,14 @@
 /* MC_CMD_MUM_OUT_QSFP_FILL_STATS msgresponse */
 #define    MC_CMD_MUM_OUT_QSFP_FILL_STATS_LEN 8
 #define       MC_CMD_MUM_OUT_QSFP_FILL_STATS_PORT_PHY_STATS_PMA_PMD_LINK_UP_OFST 0
+#define       MC_CMD_MUM_OUT_QSFP_FILL_STATS_PORT_PHY_STATS_PMA_PMD_LINK_UP_LEN 4
 #define       MC_CMD_MUM_OUT_QSFP_FILL_STATS_PORT_PHY_STATS_PCS_LINK_UP_OFST 4
+#define       MC_CMD_MUM_OUT_QSFP_FILL_STATS_PORT_PHY_STATS_PCS_LINK_UP_LEN 4
 
 /* MC_CMD_MUM_OUT_QSFP_POLL_BIST msgresponse */
 #define    MC_CMD_MUM_OUT_QSFP_POLL_BIST_LEN 4
 #define       MC_CMD_MUM_OUT_QSFP_POLL_BIST_TEST_OFST 0
+#define       MC_CMD_MUM_OUT_QSFP_POLL_BIST_TEST_LEN 4
 
 /* MC_CMD_MUM_OUT_READ_DDR_INFO msgresponse */
 #define    MC_CMD_MUM_OUT_READ_DDR_INFO_LENMIN 24
@@ -5299,12 +6394,14 @@
 #define    MC_CMD_MUM_OUT_READ_DDR_INFO_LEN(num) (8+8*(num))
 /* Discrete (soldered) DDR resistor strap info */
 #define       MC_CMD_MUM_OUT_READ_DDR_INFO_DISCRETE_DDR_INFO_OFST 0
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_DISCRETE_DDR_INFO_LEN 4
 #define        MC_CMD_MUM_OUT_READ_DDR_INFO_VRATIO_LBN 0
 #define        MC_CMD_MUM_OUT_READ_DDR_INFO_VRATIO_WIDTH 16
 #define        MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED1_LBN 16
 #define        MC_CMD_MUM_OUT_READ_DDR_INFO_RESERVED1_WIDTH 16
 /* Number of SODIMM info records */
 #define       MC_CMD_MUM_OUT_READ_DDR_INFO_NUM_RECORDS_OFST 4
+#define       MC_CMD_MUM_OUT_READ_DDR_INFO_NUM_RECORDS_LEN 4
 /* Array of SODIMM info records */
 #define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_OFST 8
 #define       MC_CMD_MUM_OUT_READ_DDR_INFO_SODIMM_INFO_RECORD_LEN 8
@@ -5365,6 +6462,7 @@
 /* EVB_PORT_ID structuredef */
 #define    EVB_PORT_ID_LEN 4
 #define       EVB_PORT_ID_PORT_ID_OFST 0
+#define       EVB_PORT_ID_PORT_ID_LEN 4
 /* enum: An invalid port handle. */
 #define          EVB_PORT_ID_NULL  0x0
 /* enum: The port assigned to this function.. */
@@ -5460,6 +6558,10 @@
 #define          NVRAM_PARTITION_TYPE_FC_LOG               0xb04
 /* enum: MUM firmware partition */
 #define          NVRAM_PARTITION_TYPE_MUM_FIRMWARE         0xc00
+/* enum: SUC firmware partition (this is intentionally an alias of
+ * MUM_FIRMWARE)
+ */
+#define          NVRAM_PARTITION_TYPE_SUC_FIRMWARE         0xc00
 /* enum: MUM Non-volatile log output partition. */
 #define          NVRAM_PARTITION_TYPE_MUM_LOG              0xc01
 /* enum: MUM Application table partition. */
@@ -5474,8 +6576,8 @@
 #define          NVRAM_PARTITION_TYPE_MUM_FUSELOCK         0xc06
 /* enum: UEFI expansion ROM if separate from PXE */
 #define          NVRAM_PARTITION_TYPE_EXPANSION_UEFI       0xd00
-/* enum: Spare partition 0 */
-#define          NVRAM_PARTITION_TYPE_SPARE_0              0x1000
+/* enum: Used by the expansion ROM for logging */
+#define          NVRAM_PARTITION_TYPE_PXE_LOG              0x1000
 /* enum: Used for XIP code of shmbooted images */
 #define          NVRAM_PARTITION_TYPE_XIP_SCRATCH          0x1100
 /* enum: Spare partition 2 */
@@ -5488,6 +6590,27 @@
 #define          NVRAM_PARTITION_TYPE_SPARE_4              0x1400
 /* enum: Spare partition 5 */
 #define          NVRAM_PARTITION_TYPE_SPARE_5              0x1500
+/* enum: Partition for reporting MC status. See mc_flash_layout.h
+ * medford_mc_status_hdr_t for layout on Medford.
+ */
+#define          NVRAM_PARTITION_TYPE_STATUS               0x1600
+/* enum: Spare partition 13 */
+#define          NVRAM_PARTITION_TYPE_SPARE_13              0x1700
+/* enum: Spare partition 14 */
+#define          NVRAM_PARTITION_TYPE_SPARE_14              0x1800
+/* enum: Spare partition 15 */
+#define          NVRAM_PARTITION_TYPE_SPARE_15              0x1900
+/* enum: Spare partition 16 */
+#define          NVRAM_PARTITION_TYPE_SPARE_16              0x1a00
+/* enum: Factory defaults for dynamic configuration */
+#define          NVRAM_PARTITION_TYPE_DYNCONFIG_DEFAULTS    0x1b00
+/* enum: Factory defaults for expansion ROM configuration */
+#define          NVRAM_PARTITION_TYPE_ROMCONFIG_DEFAULTS    0x1c00
+/* enum: Field Replaceable Unit inventory information for use on IPMI
+ * platforms. See SF-119124-PS. The STATIC_CONFIG partition may contain a
+ * subset of the information stored in this partition.
+ */
+#define          NVRAM_PARTITION_TYPE_FRU_INFORMATION       0x1d00
 /* enum: Start of reserved value range (firmware may use for any purpose) */
 #define          NVRAM_PARTITION_TYPE_RESERVED_VALUES_MIN  0xff00
 /* enum: End of reserved value range (firmware may use for any purpose) */
@@ -5502,6 +6625,7 @@
 /* LICENSED_APP_ID structuredef */
 #define    LICENSED_APP_ID_LEN 4
 #define       LICENSED_APP_ID_ID_OFST 0
+#define       LICENSED_APP_ID_ID_LEN 4
 /* enum: OpenOnload */
 #define          LICENSED_APP_ID_ONLOAD                  0x1
 /* enum: PTP timestamping */
@@ -5526,6 +6650,14 @@
 #define          LICENSED_APP_ID_SOLARCAPTURE_TAP        0x400
 /* enum: Capture SolarSystem 40G */
 #define          LICENSED_APP_ID_CAPTURE_SOLARSYSTEM_40G 0x800
+/* enum: Capture SolarSystem 1G */
+#define          LICENSED_APP_ID_CAPTURE_SOLARSYSTEM_1G  0x1000
+/* enum: ScaleOut Onload */
+#define          LICENSED_APP_ID_SCALEOUT_ONLOAD         0x2000
+/* enum: SCS Network Analytics Dashboard */
+#define          LICENSED_APP_ID_DSHBRD                  0x4000
+/* enum: SolarCapture Trading Analytics */
+#define          LICENSED_APP_ID_SCATRD                  0x8000
 #define       LICENSED_APP_ID_ID_LBN 0
 #define       LICENSED_APP_ID_ID_WIDTH 32
 
@@ -5590,6 +6722,14 @@
 #define        LICENSED_V3_APPS_SOLARCAPTURE_TAP_WIDTH 1
 #define        LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_40G_LBN 11
 #define        LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_40G_WIDTH 1
+#define        LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_1G_LBN 12
+#define        LICENSED_V3_APPS_CAPTURE_SOLARSYSTEM_1G_WIDTH 1
+#define        LICENSED_V3_APPS_SCALEOUT_ONLOAD_LBN 13
+#define        LICENSED_V3_APPS_SCALEOUT_ONLOAD_WIDTH 1
+#define        LICENSED_V3_APPS_DSHBRD_LBN 14
+#define        LICENSED_V3_APPS_DSHBRD_WIDTH 1
+#define        LICENSED_V3_APPS_SCATRD_LBN 15
+#define        LICENSED_V3_APPS_SCATRD_WIDTH 1
 #define       LICENSED_V3_APPS_MASK_LBN 0
 #define       LICENSED_V3_APPS_MASK_WIDTH 64
 
@@ -5636,6 +6776,18 @@
 #define       TX_TIMESTAMP_EVENT_TX_EV_TYPE_LEN 1
 /* enum: This is a TX completion event, not a timestamp */
 #define          TX_TIMESTAMP_EVENT_TX_EV_COMPLETION  0x0
+/* enum: This is a TX completion event for a CTPIO transmit. The event format
+ * is the same as for TX_EV_COMPLETION.
+ */
+#define          TX_TIMESTAMP_EVENT_TX_EV_CTPIO_COMPLETION  0x11
+/* enum: This is the low part of a TX timestamp for a CTPIO transmission. The
+ * event format is the same as for TX_EV_TSTAMP_LO
+ */
+#define          TX_TIMESTAMP_EVENT_TX_EV_CTPIO_TS_LO  0x12
+/* enum: This is the high part of a TX timestamp for a CTPIO transmission. The
+ * event format is the same as for TX_EV_TSTAMP_HI
+ */
+#define          TX_TIMESTAMP_EVENT_TX_EV_CTPIO_TS_HI  0x13
 /* enum: This is the low part of a TX timestamp event */
 #define          TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_LO  0x51
 /* enum: This is the high part of a TX timestamp event */
@@ -5669,6 +6821,19 @@
 #define       RSS_MODE_HASH_SELECTOR_LBN 0
 #define       RSS_MODE_HASH_SELECTOR_WIDTH 8
 
+/* CTPIO_STATS_MAP structuredef */
+#define    CTPIO_STATS_MAP_LEN 4
+/* The (function relative) VI number */
+#define       CTPIO_STATS_MAP_VI_OFST 0
+#define       CTPIO_STATS_MAP_VI_LEN 2
+#define       CTPIO_STATS_MAP_VI_LBN 0
+#define       CTPIO_STATS_MAP_VI_WIDTH 16
+/* The target bucket for the VI */
+#define       CTPIO_STATS_MAP_BUCKET_OFST 2
+#define       CTPIO_STATS_MAP_BUCKET_LEN 2
+#define       CTPIO_STATS_MAP_BUCKET_LBN 16
+#define       CTPIO_STATS_MAP_BUCKET_WIDTH 16
+
 
 /***********************************/
 /* MC_CMD_READ_REGS
@@ -5676,7 +6841,7 @@
  */
 #define MC_CMD_READ_REGS 0x50
 
-#define MC_CMD_0x50_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x50_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_READ_REGS_IN msgrequest */
 #define    MC_CMD_READ_REGS_IN_LEN 0
@@ -5709,17 +6874,22 @@
 #define    MC_CMD_INIT_EVQ_IN_LEN(num) (36+8*(num))
 /* Size, in entries */
 #define       MC_CMD_INIT_EVQ_IN_SIZE_OFST 0
+#define       MC_CMD_INIT_EVQ_IN_SIZE_LEN 4
 /* Desired instance. Must be set to a specific instance, which is a function
  * local queue index.
  */
 #define       MC_CMD_INIT_EVQ_IN_INSTANCE_OFST 4
+#define       MC_CMD_INIT_EVQ_IN_INSTANCE_LEN 4
 /* The initial timer value. The load value is ignored if the timer mode is DIS.
  */
 #define       MC_CMD_INIT_EVQ_IN_TMR_LOAD_OFST 8
+#define       MC_CMD_INIT_EVQ_IN_TMR_LOAD_LEN 4
 /* The reload value is ignored in one-shot modes */
 #define       MC_CMD_INIT_EVQ_IN_TMR_RELOAD_OFST 12
+#define       MC_CMD_INIT_EVQ_IN_TMR_RELOAD_LEN 4
 /* tbd */
 #define       MC_CMD_INIT_EVQ_IN_FLAGS_OFST 16
+#define       MC_CMD_INIT_EVQ_IN_FLAGS_LEN 4
 #define        MC_CMD_INIT_EVQ_IN_FLAG_INTERRUPTING_LBN 0
 #define        MC_CMD_INIT_EVQ_IN_FLAG_INTERRUPTING_WIDTH 1
 #define        MC_CMD_INIT_EVQ_IN_FLAG_RPTR_DOS_LBN 1
@@ -5735,6 +6905,7 @@
 #define        MC_CMD_INIT_EVQ_IN_FLAG_USE_TIMER_LBN 6
 #define        MC_CMD_INIT_EVQ_IN_FLAG_USE_TIMER_WIDTH 1
 #define       MC_CMD_INIT_EVQ_IN_TMR_MODE_OFST 20
+#define       MC_CMD_INIT_EVQ_IN_TMR_MODE_LEN 4
 /* enum: Disabled */
 #define          MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS 0x0
 /* enum: Immediate */
@@ -5745,13 +6916,16 @@
 #define          MC_CMD_INIT_EVQ_IN_TMR_INT_HLDOFF 0x3
 /* Target EVQ for wakeups if in wakeup mode. */
 #define       MC_CMD_INIT_EVQ_IN_TARGET_EVQ_OFST 24
+#define       MC_CMD_INIT_EVQ_IN_TARGET_EVQ_LEN 4
 /* Target interrupt if in interrupting mode (note union with target EVQ). Use
  * MC_CMD_RESOURCE_INSTANCE_ANY unless a specific one required for test
  * purposes.
  */
 #define       MC_CMD_INIT_EVQ_IN_IRQ_NUM_OFST 24
+#define       MC_CMD_INIT_EVQ_IN_IRQ_NUM_LEN 4
 /* Event Counter Mode. */
 #define       MC_CMD_INIT_EVQ_IN_COUNT_MODE_OFST 28
+#define       MC_CMD_INIT_EVQ_IN_COUNT_MODE_LEN 4
 /* enum: Disabled */
 #define          MC_CMD_INIT_EVQ_IN_COUNT_MODE_DIS 0x0
 /* enum: Disabled */
@@ -5762,6 +6936,7 @@
 #define          MC_CMD_INIT_EVQ_IN_COUNT_MODE_RXTX 0x3
 /* Event queue packet count threshold. */
 #define       MC_CMD_INIT_EVQ_IN_COUNT_THRSHLD_OFST 32
+#define       MC_CMD_INIT_EVQ_IN_COUNT_THRSHLD_LEN 4
 /* 64-bit address of 4k of 4k-aligned host memory buffer */
 #define       MC_CMD_INIT_EVQ_IN_DMA_ADDR_OFST 36
 #define       MC_CMD_INIT_EVQ_IN_DMA_ADDR_LEN 8
@@ -5774,6 +6949,7 @@
 #define    MC_CMD_INIT_EVQ_OUT_LEN 4
 /* Only valid if INTRFLAG was true */
 #define       MC_CMD_INIT_EVQ_OUT_IRQ_OFST 0
+#define       MC_CMD_INIT_EVQ_OUT_IRQ_LEN 4
 
 /* MC_CMD_INIT_EVQ_V2_IN msgrequest */
 #define    MC_CMD_INIT_EVQ_V2_IN_LENMIN 44
@@ -5781,17 +6957,22 @@
 #define    MC_CMD_INIT_EVQ_V2_IN_LEN(num) (36+8*(num))
 /* Size, in entries */
 #define       MC_CMD_INIT_EVQ_V2_IN_SIZE_OFST 0
+#define       MC_CMD_INIT_EVQ_V2_IN_SIZE_LEN 4
 /* Desired instance. Must be set to a specific instance, which is a function
  * local queue index.
  */
 #define       MC_CMD_INIT_EVQ_V2_IN_INSTANCE_OFST 4
+#define       MC_CMD_INIT_EVQ_V2_IN_INSTANCE_LEN 4
 /* The initial timer value. The load value is ignored if the timer mode is DIS.
  */
 #define       MC_CMD_INIT_EVQ_V2_IN_TMR_LOAD_OFST 8
+#define       MC_CMD_INIT_EVQ_V2_IN_TMR_LOAD_LEN 4
 /* The reload value is ignored in one-shot modes */
 #define       MC_CMD_INIT_EVQ_V2_IN_TMR_RELOAD_OFST 12
+#define       MC_CMD_INIT_EVQ_V2_IN_TMR_RELOAD_LEN 4
 /* tbd */
 #define       MC_CMD_INIT_EVQ_V2_IN_FLAGS_OFST 16
+#define       MC_CMD_INIT_EVQ_V2_IN_FLAGS_LEN 4
 #define        MC_CMD_INIT_EVQ_V2_IN_FLAG_INTERRUPTING_LBN 0
 #define        MC_CMD_INIT_EVQ_V2_IN_FLAG_INTERRUPTING_WIDTH 1
 #define        MC_CMD_INIT_EVQ_V2_IN_FLAG_RPTR_DOS_LBN 1
@@ -5828,6 +7009,7 @@
  */
 #define          MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_AUTO 0x3
 #define       MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_OFST 20
+#define       MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_LEN 4
 /* enum: Disabled */
 #define          MC_CMD_INIT_EVQ_V2_IN_TMR_MODE_DIS 0x0
 /* enum: Immediate */
@@ -5838,13 +7020,16 @@
 #define          MC_CMD_INIT_EVQ_V2_IN_TMR_INT_HLDOFF 0x3
 /* Target EVQ for wakeups if in wakeup mode. */
 #define       MC_CMD_INIT_EVQ_V2_IN_TARGET_EVQ_OFST 24
+#define       MC_CMD_INIT_EVQ_V2_IN_TARGET_EVQ_LEN 4
 /* Target interrupt if in interrupting mode (note union with target EVQ). Use
  * MC_CMD_RESOURCE_INSTANCE_ANY unless a specific one required for test
  * purposes.
  */
 #define       MC_CMD_INIT_EVQ_V2_IN_IRQ_NUM_OFST 24
+#define       MC_CMD_INIT_EVQ_V2_IN_IRQ_NUM_LEN 4
 /* Event Counter Mode. */
 #define       MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_OFST 28
+#define       MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_LEN 4
 /* enum: Disabled */
 #define          MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_DIS 0x0
 /* enum: Disabled */
@@ -5855,6 +7040,7 @@
 #define          MC_CMD_INIT_EVQ_V2_IN_COUNT_MODE_RXTX 0x3
 /* Event queue packet count threshold. */
 #define       MC_CMD_INIT_EVQ_V2_IN_COUNT_THRSHLD_OFST 32
+#define       MC_CMD_INIT_EVQ_V2_IN_COUNT_THRSHLD_LEN 4
 /* 64-bit address of 4k of 4k-aligned host memory buffer */
 #define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_OFST 36
 #define       MC_CMD_INIT_EVQ_V2_IN_DMA_ADDR_LEN 8
@@ -5867,8 +7053,10 @@
 #define    MC_CMD_INIT_EVQ_V2_OUT_LEN 8
 /* Only valid if INTRFLAG was true */
 #define       MC_CMD_INIT_EVQ_V2_OUT_IRQ_OFST 0
+#define       MC_CMD_INIT_EVQ_V2_OUT_IRQ_LEN 4
 /* Actual configuration applied on the card */
 #define       MC_CMD_INIT_EVQ_V2_OUT_FLAGS_OFST 4
+#define       MC_CMD_INIT_EVQ_V2_OUT_FLAGS_LEN 4
 #define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_CUT_THRU_LBN 0
 #define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_CUT_THRU_WIDTH 1
 #define        MC_CMD_INIT_EVQ_V2_OUT_FLAG_RX_MERGE_LBN 1
@@ -5916,17 +7104,22 @@
 #define    MC_CMD_INIT_RXQ_IN_LEN(num) (28+8*(num))
 /* Size, in entries */
 #define       MC_CMD_INIT_RXQ_IN_SIZE_OFST 0
+#define       MC_CMD_INIT_RXQ_IN_SIZE_LEN 4
 /* The EVQ to send events to. This is an index originally specified to INIT_EVQ
  */
 #define       MC_CMD_INIT_RXQ_IN_TARGET_EVQ_OFST 4
+#define       MC_CMD_INIT_RXQ_IN_TARGET_EVQ_LEN 4
 /* The value to put in the event data. Check hardware spec. for valid range. */
 #define       MC_CMD_INIT_RXQ_IN_LABEL_OFST 8
+#define       MC_CMD_INIT_RXQ_IN_LABEL_LEN 4
 /* Desired instance. Must be set to a specific instance, which is a function
  * local queue index.
  */
 #define       MC_CMD_INIT_RXQ_IN_INSTANCE_OFST 12
+#define       MC_CMD_INIT_RXQ_IN_INSTANCE_LEN 4
 /* There will be more flags here. */
 #define       MC_CMD_INIT_RXQ_IN_FLAGS_OFST 16
+#define       MC_CMD_INIT_RXQ_IN_FLAGS_LEN 4
 #define        MC_CMD_INIT_RXQ_IN_FLAG_BUFF_MODE_LBN 0
 #define        MC_CMD_INIT_RXQ_IN_FLAG_BUFF_MODE_WIDTH 1
 #define        MC_CMD_INIT_RXQ_IN_FLAG_HDR_SPLIT_LBN 1
@@ -5945,8 +7138,10 @@
 #define        MC_CMD_INIT_RXQ_IN_UNUSED_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_RXQ_IN_OWNER_ID_OFST 20
+#define       MC_CMD_INIT_RXQ_IN_OWNER_ID_LEN 4
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
 #define       MC_CMD_INIT_RXQ_IN_PORT_ID_OFST 24
+#define       MC_CMD_INIT_RXQ_IN_PORT_ID_LEN 4
 /* 64-bit address of 4k of 4k-aligned host memory buffer */
 #define       MC_CMD_INIT_RXQ_IN_DMA_ADDR_OFST 28
 #define       MC_CMD_INIT_RXQ_IN_DMA_ADDR_LEN 8
@@ -5961,17 +7156,22 @@
 #define    MC_CMD_INIT_RXQ_EXT_IN_LEN 544
 /* Size, in entries */
 #define       MC_CMD_INIT_RXQ_EXT_IN_SIZE_OFST 0
+#define       MC_CMD_INIT_RXQ_EXT_IN_SIZE_LEN 4
 /* The EVQ to send events to. This is an index originally specified to INIT_EVQ
  */
 #define       MC_CMD_INIT_RXQ_EXT_IN_TARGET_EVQ_OFST 4
+#define       MC_CMD_INIT_RXQ_EXT_IN_TARGET_EVQ_LEN 4
 /* The value to put in the event data. Check hardware spec. for valid range. */
 #define       MC_CMD_INIT_RXQ_EXT_IN_LABEL_OFST 8
+#define       MC_CMD_INIT_RXQ_EXT_IN_LABEL_LEN 4
 /* Desired instance. Must be set to a specific instance, which is a function
  * local queue index.
  */
 #define       MC_CMD_INIT_RXQ_EXT_IN_INSTANCE_OFST 12
+#define       MC_CMD_INIT_RXQ_EXT_IN_INSTANCE_LEN 4
 /* There will be more flags here. */
 #define       MC_CMD_INIT_RXQ_EXT_IN_FLAGS_OFST 16
+#define       MC_CMD_INIT_RXQ_EXT_IN_FLAGS_LEN 4
 #define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_BUFF_MODE_LBN 0
 #define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_BUFF_MODE_WIDTH 1
 #define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT_LBN 1
@@ -6007,8 +7207,10 @@
 #define        MC_CMD_INIT_RXQ_EXT_IN_FLAG_FORCE_EV_MERGING_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_RXQ_EXT_IN_OWNER_ID_OFST 20
+#define       MC_CMD_INIT_RXQ_EXT_IN_OWNER_ID_LEN 4
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
 #define       MC_CMD_INIT_RXQ_EXT_IN_PORT_ID_OFST 24
+#define       MC_CMD_INIT_RXQ_EXT_IN_PORT_ID_LEN 4
 /* 64-bit address of 4k of 4k-aligned host memory buffer */
 #define       MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_OFST 28
 #define       MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_LEN 8
@@ -6017,6 +7219,7 @@
 #define       MC_CMD_INIT_RXQ_EXT_IN_DMA_ADDR_NUM 64
 /* Maximum length of packet to receive, if SNAPSHOT_MODE flag is set */
 #define       MC_CMD_INIT_RXQ_EXT_IN_SNAPSHOT_LENGTH_OFST 540
+#define       MC_CMD_INIT_RXQ_EXT_IN_SNAPSHOT_LENGTH_LEN 4
 
 /* MC_CMD_INIT_RXQ_OUT msgresponse */
 #define    MC_CMD_INIT_RXQ_OUT_LEN 0
@@ -6040,18 +7243,23 @@
 #define    MC_CMD_INIT_TXQ_IN_LEN(num) (28+8*(num))
 /* Size, in entries */
 #define       MC_CMD_INIT_TXQ_IN_SIZE_OFST 0
+#define       MC_CMD_INIT_TXQ_IN_SIZE_LEN 4
 /* The EVQ to send events to. This is an index originally specified to
  * INIT_EVQ.
  */
 #define       MC_CMD_INIT_TXQ_IN_TARGET_EVQ_OFST 4
+#define       MC_CMD_INIT_TXQ_IN_TARGET_EVQ_LEN 4
 /* The value to put in the event data. Check hardware spec. for valid range. */
 #define       MC_CMD_INIT_TXQ_IN_LABEL_OFST 8
+#define       MC_CMD_INIT_TXQ_IN_LABEL_LEN 4
 /* Desired instance. Must be set to a specific instance, which is a function
  * local queue index.
  */
 #define       MC_CMD_INIT_TXQ_IN_INSTANCE_OFST 12
+#define       MC_CMD_INIT_TXQ_IN_INSTANCE_LEN 4
 /* There will be more flags here. */
 #define       MC_CMD_INIT_TXQ_IN_FLAGS_OFST 16
+#define       MC_CMD_INIT_TXQ_IN_FLAGS_LEN 4
 #define        MC_CMD_INIT_TXQ_IN_FLAG_BUFF_MODE_LBN 0
 #define        MC_CMD_INIT_TXQ_IN_FLAG_BUFF_MODE_WIDTH 1
 #define        MC_CMD_INIT_TXQ_IN_FLAG_IP_CSUM_DIS_LBN 1
@@ -6072,8 +7280,10 @@
 #define        MC_CMD_INIT_TXQ_IN_FLAG_INNER_TCP_CSUM_EN_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_TXQ_IN_OWNER_ID_OFST 20
+#define       MC_CMD_INIT_TXQ_IN_OWNER_ID_LEN 4
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
 #define       MC_CMD_INIT_TXQ_IN_PORT_ID_OFST 24
+#define       MC_CMD_INIT_TXQ_IN_PORT_ID_LEN 4
 /* 64-bit address of 4k of 4k-aligned host memory buffer */
 #define       MC_CMD_INIT_TXQ_IN_DMA_ADDR_OFST 28
 #define       MC_CMD_INIT_TXQ_IN_DMA_ADDR_LEN 8
@@ -6088,18 +7298,23 @@
 #define    MC_CMD_INIT_TXQ_EXT_IN_LEN 544
 /* Size, in entries */
 #define       MC_CMD_INIT_TXQ_EXT_IN_SIZE_OFST 0
+#define       MC_CMD_INIT_TXQ_EXT_IN_SIZE_LEN 4
 /* The EVQ to send events to. This is an index originally specified to
  * INIT_EVQ.
  */
 #define       MC_CMD_INIT_TXQ_EXT_IN_TARGET_EVQ_OFST 4
+#define       MC_CMD_INIT_TXQ_EXT_IN_TARGET_EVQ_LEN 4
 /* The value to put in the event data. Check hardware spec. for valid range. */
 #define       MC_CMD_INIT_TXQ_EXT_IN_LABEL_OFST 8
+#define       MC_CMD_INIT_TXQ_EXT_IN_LABEL_LEN 4
 /* Desired instance. Must be set to a specific instance, which is a function
  * local queue index.
  */
 #define       MC_CMD_INIT_TXQ_EXT_IN_INSTANCE_OFST 12
+#define       MC_CMD_INIT_TXQ_EXT_IN_INSTANCE_LEN 4
 /* There will be more flags here. */
 #define       MC_CMD_INIT_TXQ_EXT_IN_FLAGS_OFST 16
+#define       MC_CMD_INIT_TXQ_EXT_IN_FLAGS_LEN 4
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_BUFF_MODE_LBN 0
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_BUFF_MODE_WIDTH 1
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_IP_CSUM_DIS_LBN 1
@@ -6122,10 +7337,14 @@
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_TSOV2_EN_WIDTH 1
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_CTPIO_LBN 13
 #define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_CTPIO_WIDTH 1
+#define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_CTPIO_UTHRESH_LBN 14
+#define        MC_CMD_INIT_TXQ_EXT_IN_FLAG_CTPIO_UTHRESH_WIDTH 1
 /* Owner ID to use if in buffer mode (zero if physical) */
 #define       MC_CMD_INIT_TXQ_EXT_IN_OWNER_ID_OFST 20
+#define       MC_CMD_INIT_TXQ_EXT_IN_OWNER_ID_LEN 4
 /* The port ID associated with the v-adaptor which should contain this DMAQ. */
 #define       MC_CMD_INIT_TXQ_EXT_IN_PORT_ID_OFST 24
+#define       MC_CMD_INIT_TXQ_EXT_IN_PORT_ID_LEN 4
 /* 64-bit address of 4k of 4k-aligned host memory buffer */
 #define       MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_OFST 28
 #define       MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_LEN 8
@@ -6135,6 +7354,7 @@
 #define       MC_CMD_INIT_TXQ_EXT_IN_DMA_ADDR_MAXNUM 64
 /* Flags related to Qbb flow control mode. */
 #define       MC_CMD_INIT_TXQ_EXT_IN_QBB_FLAGS_OFST 540
+#define       MC_CMD_INIT_TXQ_EXT_IN_QBB_FLAGS_LEN 4
 #define        MC_CMD_INIT_TXQ_EXT_IN_QBB_ENABLE_LBN 0
 #define        MC_CMD_INIT_TXQ_EXT_IN_QBB_ENABLE_WIDTH 1
 #define        MC_CMD_INIT_TXQ_EXT_IN_QBB_PRIORITY_LBN 1
@@ -6161,6 +7381,7 @@
  * passed to INIT_EVQ
  */
 #define       MC_CMD_FINI_EVQ_IN_INSTANCE_OFST 0
+#define       MC_CMD_FINI_EVQ_IN_INSTANCE_LEN 4
 
 /* MC_CMD_FINI_EVQ_OUT msgresponse */
 #define    MC_CMD_FINI_EVQ_OUT_LEN 0
@@ -6178,6 +7399,7 @@
 #define    MC_CMD_FINI_RXQ_IN_LEN 4
 /* Instance of RXQ to destroy */
 #define       MC_CMD_FINI_RXQ_IN_INSTANCE_OFST 0
+#define       MC_CMD_FINI_RXQ_IN_INSTANCE_LEN 4
 
 /* MC_CMD_FINI_RXQ_OUT msgresponse */
 #define    MC_CMD_FINI_RXQ_OUT_LEN 0
@@ -6195,6 +7417,7 @@
 #define    MC_CMD_FINI_TXQ_IN_LEN 4
 /* Instance of TXQ to destroy */
 #define       MC_CMD_FINI_TXQ_IN_INSTANCE_OFST 0
+#define       MC_CMD_FINI_TXQ_IN_INSTANCE_LEN 4
 
 /* MC_CMD_FINI_TXQ_OUT msgresponse */
 #define    MC_CMD_FINI_TXQ_OUT_LEN 0
@@ -6212,6 +7435,7 @@
 #define    MC_CMD_DRIVER_EVENT_IN_LEN 12
 /* Handle of target EVQ */
 #define       MC_CMD_DRIVER_EVENT_IN_EVQ_OFST 0
+#define       MC_CMD_DRIVER_EVENT_IN_EVQ_LEN 4
 /* Bits 0 - 63 of event */
 #define       MC_CMD_DRIVER_EVENT_IN_DATA_OFST 4
 #define       MC_CMD_DRIVER_EVENT_IN_DATA_LEN 8
@@ -6237,6 +7461,7 @@
 #define    MC_CMD_PROXY_CMD_IN_LEN 4
 /* The handle of the target function. */
 #define       MC_CMD_PROXY_CMD_IN_TARGET_OFST 0
+#define       MC_CMD_PROXY_CMD_IN_TARGET_LEN 4
 #define        MC_CMD_PROXY_CMD_IN_TARGET_PF_LBN 0
 #define        MC_CMD_PROXY_CMD_IN_TARGET_PF_WIDTH 16
 #define        MC_CMD_PROXY_CMD_IN_TARGET_VF_LBN 16
@@ -6252,6 +7477,7 @@
 #define    MC_PROXY_STATUS_BUFFER_LEN 16
 /* Handle allocated by the firmware for this proxy transaction */
 #define       MC_PROXY_STATUS_BUFFER_HANDLE_OFST 0
+#define       MC_PROXY_STATUS_BUFFER_HANDLE_LEN 4
 /* enum: An invalid handle. */
 #define          MC_PROXY_STATUS_BUFFER_HANDLE_INVALID  0x0
 #define       MC_PROXY_STATUS_BUFFER_HANDLE_LBN 0
@@ -6282,6 +7508,7 @@
  * elevated privilege mask granted to the requesting function.
  */
 #define       MC_PROXY_STATUS_BUFFER_GRANTED_PRIVILEGES_OFST 12
+#define       MC_PROXY_STATUS_BUFFER_GRANTED_PRIVILEGES_LEN 4
 #define       MC_PROXY_STATUS_BUFFER_GRANTED_PRIVILEGES_LBN 96
 #define       MC_PROXY_STATUS_BUFFER_GRANTED_PRIVILEGES_WIDTH 32
 
@@ -6298,6 +7525,7 @@
 /* MC_CMD_PROXY_CONFIGURE_IN msgrequest */
 #define    MC_CMD_PROXY_CONFIGURE_IN_LEN 108
 #define       MC_CMD_PROXY_CONFIGURE_IN_FLAGS_OFST 0
+#define       MC_CMD_PROXY_CONFIGURE_IN_FLAGS_LEN 4
 #define        MC_CMD_PROXY_CONFIGURE_IN_ENABLE_LBN 0
 #define        MC_CMD_PROXY_CONFIGURE_IN_ENABLE_WIDTH 1
 /* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
@@ -6309,6 +7537,7 @@
 #define       MC_CMD_PROXY_CONFIGURE_IN_STATUS_BUFF_ADDR_HI_OFST 8
 /* Must be a power of 2 */
 #define       MC_CMD_PROXY_CONFIGURE_IN_STATUS_BLOCK_SIZE_OFST 12
+#define       MC_CMD_PROXY_CONFIGURE_IN_STATUS_BLOCK_SIZE_LEN 4
 /* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
  * of blocks, each of the size REPLY_BLOCK_SIZE.
  */
@@ -6318,6 +7547,7 @@
 #define       MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BUFF_ADDR_HI_OFST 20
 /* Must be a power of 2 */
 #define       MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BLOCK_SIZE_OFST 24
+#define       MC_CMD_PROXY_CONFIGURE_IN_REQUEST_BLOCK_SIZE_LEN 4
 /* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
  * of blocks, each of the size STATUS_BLOCK_SIZE. This buffer is only needed if
  * host intends to complete proxied operations by using MC_CMD_PROXY_CMD.
@@ -6328,8 +7558,10 @@
 #define       MC_CMD_PROXY_CONFIGURE_IN_REPLY_BUFF_ADDR_HI_OFST 32
 /* Must be a power of 2, or zero if this buffer is not provided */
 #define       MC_CMD_PROXY_CONFIGURE_IN_REPLY_BLOCK_SIZE_OFST 36
+#define       MC_CMD_PROXY_CONFIGURE_IN_REPLY_BLOCK_SIZE_LEN 4
 /* Applies to all three buffers */
 #define       MC_CMD_PROXY_CONFIGURE_IN_NUM_BLOCKS_OFST 40
+#define       MC_CMD_PROXY_CONFIGURE_IN_NUM_BLOCKS_LEN 4
 /* A bit mask defining which MCDI operations may be proxied */
 #define       MC_CMD_PROXY_CONFIGURE_IN_ALLOWED_MCDI_MASK_OFST 44
 #define       MC_CMD_PROXY_CONFIGURE_IN_ALLOWED_MCDI_MASK_LEN 64
@@ -6337,6 +7569,7 @@
 /* MC_CMD_PROXY_CONFIGURE_EXT_IN msgrequest */
 #define    MC_CMD_PROXY_CONFIGURE_EXT_IN_LEN 112
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_FLAGS_OFST 0
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_FLAGS_LEN 4
 #define        MC_CMD_PROXY_CONFIGURE_EXT_IN_ENABLE_LBN 0
 #define        MC_CMD_PROXY_CONFIGURE_EXT_IN_ENABLE_WIDTH 1
 /* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
@@ -6348,6 +7581,7 @@
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BUFF_ADDR_HI_OFST 8
 /* Must be a power of 2 */
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BLOCK_SIZE_OFST 12
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_STATUS_BLOCK_SIZE_LEN 4
 /* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
  * of blocks, each of the size REPLY_BLOCK_SIZE.
  */
@@ -6357,6 +7591,7 @@
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BUFF_ADDR_HI_OFST 20
 /* Must be a power of 2 */
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BLOCK_SIZE_OFST 24
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REQUEST_BLOCK_SIZE_LEN 4
 /* Host provides a contiguous memory buffer that contains at least NUM_BLOCKS
  * of blocks, each of the size STATUS_BLOCK_SIZE. This buffer is only needed if
  * host intends to complete proxied operations by using MC_CMD_PROXY_CMD.
@@ -6367,12 +7602,15 @@
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BUFF_ADDR_HI_OFST 32
 /* Must be a power of 2, or zero if this buffer is not provided */
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BLOCK_SIZE_OFST 36
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_REPLY_BLOCK_SIZE_LEN 4
 /* Applies to all three buffers */
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_NUM_BLOCKS_OFST 40
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_NUM_BLOCKS_LEN 4
 /* A bit mask defining which MCDI operations may be proxied */
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_ALLOWED_MCDI_MASK_OFST 44
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_ALLOWED_MCDI_MASK_LEN 64
 #define       MC_CMD_PROXY_CONFIGURE_EXT_IN_RESERVED_OFST 108
+#define       MC_CMD_PROXY_CONFIGURE_EXT_IN_RESERVED_LEN 4
 
 /* MC_CMD_PROXY_CONFIGURE_OUT msgresponse */
 #define    MC_CMD_PROXY_CONFIGURE_OUT_LEN 0
@@ -6392,7 +7630,9 @@
 /* MC_CMD_PROXY_COMPLETE_IN msgrequest */
 #define    MC_CMD_PROXY_COMPLETE_IN_LEN 12
 #define       MC_CMD_PROXY_COMPLETE_IN_BLOCK_INDEX_OFST 0
+#define       MC_CMD_PROXY_COMPLETE_IN_BLOCK_INDEX_LEN 4
 #define       MC_CMD_PROXY_COMPLETE_IN_STATUS_OFST 4
+#define       MC_CMD_PROXY_COMPLETE_IN_STATUS_LEN 4
 /* enum: The operation has been completed by using MC_CMD_PROXY_CMD, the reply
  * is stored in the REPLY_BUFF.
  */
@@ -6408,6 +7648,7 @@
  */
 #define          MC_CMD_PROXY_COMPLETE_IN_TIMEDOUT 0x3
 #define       MC_CMD_PROXY_COMPLETE_IN_HANDLE_OFST 8
+#define       MC_CMD_PROXY_COMPLETE_IN_HANDLE_LEN 4
 
 /* MC_CMD_PROXY_COMPLETE_OUT msgresponse */
 #define    MC_CMD_PROXY_COMPLETE_OUT_LEN 0
@@ -6427,17 +7668,22 @@
 #define    MC_CMD_ALLOC_BUFTBL_CHUNK_IN_LEN 8
 /* Owner ID to use */
 #define       MC_CMD_ALLOC_BUFTBL_CHUNK_IN_OWNER_OFST 0
+#define       MC_CMD_ALLOC_BUFTBL_CHUNK_IN_OWNER_LEN 4
 /* Size of buffer table pages to use, in bytes (note that only a few values are
  * legal on any specific hardware).
  */
 #define       MC_CMD_ALLOC_BUFTBL_CHUNK_IN_PAGE_SIZE_OFST 4
+#define       MC_CMD_ALLOC_BUFTBL_CHUNK_IN_PAGE_SIZE_LEN 4
 
 /* MC_CMD_ALLOC_BUFTBL_CHUNK_OUT msgresponse */
 #define    MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_LEN 12
 #define       MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_HANDLE_OFST 0
+#define       MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_HANDLE_LEN 4
 #define       MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_NUMENTRIES_OFST 4
+#define       MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_NUMENTRIES_LEN 4
 /* Buffer table IDs for use in DMA descriptors. */
 #define       MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_ID_OFST 8
+#define       MC_CMD_ALLOC_BUFTBL_CHUNK_OUT_ID_LEN 4
 
 
 /***********************************/
@@ -6453,10 +7699,13 @@
 #define    MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LENMAX 268
 #define    MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_LEN(num) (12+8*(num))
 #define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_HANDLE_OFST 0
+#define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_HANDLE_LEN 4
 /* ID */
 #define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_FIRSTID_OFST 4
+#define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_FIRSTID_LEN 4
 /* Num entries */
 #define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_NUMENTRIES_OFST 8
+#define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_NUMENTRIES_LEN 4
 /* Buffer table entry address */
 #define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_OFST 12
 #define       MC_CMD_PROGRAM_BUFTBL_ENTRIES_IN_ENTRY_LEN 8
@@ -6479,48 +7728,11 @@
 /* MC_CMD_FREE_BUFTBL_CHUNK_IN msgrequest */
 #define    MC_CMD_FREE_BUFTBL_CHUNK_IN_LEN 4
 #define       MC_CMD_FREE_BUFTBL_CHUNK_IN_HANDLE_OFST 0
+#define       MC_CMD_FREE_BUFTBL_CHUNK_IN_HANDLE_LEN 4
 
 /* MC_CMD_FREE_BUFTBL_CHUNK_OUT msgresponse */
 #define    MC_CMD_FREE_BUFTBL_CHUNK_OUT_LEN 0
 
-/* PORT_CONFIG_ENTRY structuredef */
-#define    PORT_CONFIG_ENTRY_LEN 16
-/* External port number (label) */
-#define       PORT_CONFIG_ENTRY_EXT_NUMBER_OFST 0
-#define       PORT_CONFIG_ENTRY_EXT_NUMBER_LEN 1
-#define       PORT_CONFIG_ENTRY_EXT_NUMBER_LBN 0
-#define       PORT_CONFIG_ENTRY_EXT_NUMBER_WIDTH 8
-/* Port core location */
-#define       PORT_CONFIG_ENTRY_CORE_OFST 1
-#define       PORT_CONFIG_ENTRY_CORE_LEN 1
-#define          PORT_CONFIG_ENTRY_STANDALONE  0x0 /* enum */
-#define          PORT_CONFIG_ENTRY_MASTER  0x1 /* enum */
-#define          PORT_CONFIG_ENTRY_SLAVE  0x2 /* enum */
-#define       PORT_CONFIG_ENTRY_CORE_LBN 8
-#define       PORT_CONFIG_ENTRY_CORE_WIDTH 8
-/* Internal number (HW resource) relative to the core */
-#define       PORT_CONFIG_ENTRY_INT_NUMBER_OFST 2
-#define       PORT_CONFIG_ENTRY_INT_NUMBER_LEN 1
-#define       PORT_CONFIG_ENTRY_INT_NUMBER_LBN 16
-#define       PORT_CONFIG_ENTRY_INT_NUMBER_WIDTH 8
-/* Reserved */
-#define       PORT_CONFIG_ENTRY_RSVD_OFST 3
-#define       PORT_CONFIG_ENTRY_RSVD_LEN 1
-#define       PORT_CONFIG_ENTRY_RSVD_LBN 24
-#define       PORT_CONFIG_ENTRY_RSVD_WIDTH 8
-/* Bitmask of KR lanes used by the port */
-#define       PORT_CONFIG_ENTRY_LANES_OFST 4
-#define       PORT_CONFIG_ENTRY_LANES_LBN 32
-#define       PORT_CONFIG_ENTRY_LANES_WIDTH 32
-/* Port capabilities (MC_CMD_PHY_CAP_*) */
-#define       PORT_CONFIG_ENTRY_SUPPORTED_CAPS_OFST 8
-#define       PORT_CONFIG_ENTRY_SUPPORTED_CAPS_LBN 64
-#define       PORT_CONFIG_ENTRY_SUPPORTED_CAPS_WIDTH 32
-/* Reserved (align to 16 bytes) */
-#define       PORT_CONFIG_ENTRY_RSVD2_OFST 12
-#define       PORT_CONFIG_ENTRY_RSVD2_LBN 96
-#define       PORT_CONFIG_ENTRY_RSVD2_WIDTH 32
-
 
 /***********************************/
 /* MC_CMD_FILTER_OP
@@ -6534,6 +7746,7 @@
 #define    MC_CMD_FILTER_OP_IN_LEN 108
 /* identifies the type of operation requested */
 #define       MC_CMD_FILTER_OP_IN_OP_OFST 0
+#define       MC_CMD_FILTER_OP_IN_OP_LEN 4
 /* enum: single-recipient filter insert */
 #define          MC_CMD_FILTER_OP_IN_OP_INSERT  0x0
 /* enum: single-recipient filter remove */
@@ -6554,8 +7767,10 @@
 /* The port ID associated with the v-adaptor which should contain this filter.
  */
 #define       MC_CMD_FILTER_OP_IN_PORT_ID_OFST 12
+#define       MC_CMD_FILTER_OP_IN_PORT_ID_LEN 4
 /* fields to include in match criteria */
 #define       MC_CMD_FILTER_OP_IN_MATCH_FIELDS_OFST 16
+#define       MC_CMD_FILTER_OP_IN_MATCH_FIELDS_LEN 4
 #define        MC_CMD_FILTER_OP_IN_MATCH_SRC_IP_LBN 0
 #define        MC_CMD_FILTER_OP_IN_MATCH_SRC_IP_WIDTH 1
 #define        MC_CMD_FILTER_OP_IN_MATCH_DST_IP_LBN 1
@@ -6586,6 +7801,7 @@
 #define        MC_CMD_FILTER_OP_IN_MATCH_UNKNOWN_UCAST_DST_WIDTH 1
 /* receive destination */
 #define       MC_CMD_FILTER_OP_IN_RX_DEST_OFST 20
+#define       MC_CMD_FILTER_OP_IN_RX_DEST_LEN 4
 /* enum: drop packets */
 #define          MC_CMD_FILTER_OP_IN_RX_DEST_DROP  0x0
 /* enum: receive to host */
@@ -6598,8 +7814,10 @@
 #define          MC_CMD_FILTER_OP_IN_RX_DEST_TX1  0x4
 /* receive queue handle (for multiple queue modes, this is the base queue) */
 #define       MC_CMD_FILTER_OP_IN_RX_QUEUE_OFST 24
+#define       MC_CMD_FILTER_OP_IN_RX_QUEUE_LEN 4
 /* receive mode */
 #define       MC_CMD_FILTER_OP_IN_RX_MODE_OFST 28
+#define       MC_CMD_FILTER_OP_IN_RX_MODE_LEN 4
 /* enum: receive to just the specified queue */
 #define          MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE  0x0
 /* enum: receive to multiple queues using RSS context */
@@ -6614,13 +7832,16 @@
  * MC_CMD_DOT1P_MAPPING_ALLOC.
  */
 #define       MC_CMD_FILTER_OP_IN_RX_CONTEXT_OFST 32
+#define       MC_CMD_FILTER_OP_IN_RX_CONTEXT_LEN 4
 /* transmit domain (reserved; set to 0) */
 #define       MC_CMD_FILTER_OP_IN_TX_DOMAIN_OFST 36
+#define       MC_CMD_FILTER_OP_IN_TX_DOMAIN_LEN 4
 /* transmit destination (either set the MAC and/or PM bits for explicit
  * control, or set this field to TX_DEST_DEFAULT for sensible default
  * behaviour)
  */
 #define       MC_CMD_FILTER_OP_IN_TX_DEST_OFST 40
+#define       MC_CMD_FILTER_OP_IN_TX_DEST_LEN 4
 /* enum: request default behaviour (based on filter type) */
 #define          MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT  0xffffffff
 #define        MC_CMD_FILTER_OP_IN_TX_DEST_MAC_LBN 0
@@ -6653,8 +7874,10 @@
 #define       MC_CMD_FILTER_OP_IN_IP_PROTO_LEN 2
 /* Firmware defined register 0 to match (reserved; set to 0) */
 #define       MC_CMD_FILTER_OP_IN_FWDEF0_OFST 68
+#define       MC_CMD_FILTER_OP_IN_FWDEF0_LEN 4
 /* Firmware defined register 1 to match (reserved; set to 0) */
 #define       MC_CMD_FILTER_OP_IN_FWDEF1_OFST 72
+#define       MC_CMD_FILTER_OP_IN_FWDEF1_LEN 4
 /* source IP address to match (as bytes in network order; set last 12 bytes to
  * 0 for IPv4 address)
  */
@@ -6673,6 +7896,7 @@
 #define    MC_CMD_FILTER_OP_EXT_IN_LEN 172
 /* identifies the type of operation requested */
 #define       MC_CMD_FILTER_OP_EXT_IN_OP_OFST 0
+#define       MC_CMD_FILTER_OP_EXT_IN_OP_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_FILTER_OP_IN/OP */
 /* filter handle (for remove / unsubscribe operations) */
@@ -6683,8 +7907,10 @@
 /* The port ID associated with the v-adaptor which should contain this filter.
  */
 #define       MC_CMD_FILTER_OP_EXT_IN_PORT_ID_OFST 12
+#define       MC_CMD_FILTER_OP_EXT_IN_PORT_ID_LEN 4
 /* fields to include in match criteria */
 #define       MC_CMD_FILTER_OP_EXT_IN_MATCH_FIELDS_OFST 16
+#define       MC_CMD_FILTER_OP_EXT_IN_MATCH_FIELDS_LEN 4
 #define        MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP_LBN 0
 #define        MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP_WIDTH 1
 #define        MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_IP_LBN 1
@@ -6743,6 +7969,7 @@
 #define        MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_WIDTH 1
 /* receive destination */
 #define       MC_CMD_FILTER_OP_EXT_IN_RX_DEST_OFST 20
+#define       MC_CMD_FILTER_OP_EXT_IN_RX_DEST_LEN 4
 /* enum: drop packets */
 #define          MC_CMD_FILTER_OP_EXT_IN_RX_DEST_DROP  0x0
 /* enum: receive to host */
@@ -6755,8 +7982,10 @@
 #define          MC_CMD_FILTER_OP_EXT_IN_RX_DEST_TX1  0x4
 /* receive queue handle (for multiple queue modes, this is the base queue) */
 #define       MC_CMD_FILTER_OP_EXT_IN_RX_QUEUE_OFST 24
+#define       MC_CMD_FILTER_OP_EXT_IN_RX_QUEUE_LEN 4
 /* receive mode */
 #define       MC_CMD_FILTER_OP_EXT_IN_RX_MODE_OFST 28
+#define       MC_CMD_FILTER_OP_EXT_IN_RX_MODE_LEN 4
 /* enum: receive to just the specified queue */
 #define          MC_CMD_FILTER_OP_EXT_IN_RX_MODE_SIMPLE  0x0
 /* enum: receive to multiple queues using RSS context */
@@ -6771,13 +8000,16 @@
  * MC_CMD_DOT1P_MAPPING_ALLOC.
  */
 #define       MC_CMD_FILTER_OP_EXT_IN_RX_CONTEXT_OFST 32
+#define       MC_CMD_FILTER_OP_EXT_IN_RX_CONTEXT_LEN 4
 /* transmit domain (reserved; set to 0) */
 #define       MC_CMD_FILTER_OP_EXT_IN_TX_DOMAIN_OFST 36
+#define       MC_CMD_FILTER_OP_EXT_IN_TX_DOMAIN_LEN 4
 /* transmit destination (either set the MAC and/or PM bits for explicit
  * control, or set this field to TX_DEST_DEFAULT for sensible default
  * behaviour)
  */
 #define       MC_CMD_FILTER_OP_EXT_IN_TX_DEST_OFST 40
+#define       MC_CMD_FILTER_OP_EXT_IN_TX_DEST_LEN 4
 /* enum: request default behaviour (based on filter type) */
 #define          MC_CMD_FILTER_OP_EXT_IN_TX_DEST_DEFAULT  0xffffffff
 #define        MC_CMD_FILTER_OP_EXT_IN_TX_DEST_MAC_LBN 0
@@ -6810,11 +8042,13 @@
 #define       MC_CMD_FILTER_OP_EXT_IN_IP_PROTO_LEN 2
 /* Firmware defined register 0 to match (reserved; set to 0) */
 #define       MC_CMD_FILTER_OP_EXT_IN_FWDEF0_OFST 68
+#define       MC_CMD_FILTER_OP_EXT_IN_FWDEF0_LEN 4
 /* VNI (for VXLAN/Geneve, when IP protocol is UDP) or VSID (for NVGRE, when IP
  * protocol is GRE) to match (as bytes in network order; set last byte to 0 for
  * VXLAN/NVGRE, or 1 for Geneve)
  */
 #define       MC_CMD_FILTER_OP_EXT_IN_VNI_OR_VSID_OFST 72
+#define       MC_CMD_FILTER_OP_EXT_IN_VNI_OR_VSID_LEN 4
 #define        MC_CMD_FILTER_OP_EXT_IN_VNI_VALUE_LBN 0
 #define        MC_CMD_FILTER_OP_EXT_IN_VNI_VALUE_WIDTH 24
 #define        MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_LBN 24
@@ -6880,10 +8114,12 @@
  * to 0)
  */
 #define       MC_CMD_FILTER_OP_EXT_IN_IFRM_FWDEF0_OFST 132
+#define       MC_CMD_FILTER_OP_EXT_IN_IFRM_FWDEF0_LEN 4
 /* VXLAN/NVGRE inner frame Firmware defined register 1 to match (reserved; set
  * to 0)
  */
 #define       MC_CMD_FILTER_OP_EXT_IN_IFRM_FWDEF1_OFST 136
+#define       MC_CMD_FILTER_OP_EXT_IN_IFRM_FWDEF1_LEN 4
 /* VXLAN/NVGRE inner frame source IP address to match (as bytes in network
  * order; set last 12 bytes to 0 for IPv4 address)
  */
@@ -6899,6 +8135,7 @@
 #define    MC_CMD_FILTER_OP_OUT_LEN 12
 /* identifies the type of operation requested */
 #define       MC_CMD_FILTER_OP_OUT_OP_OFST 0
+#define       MC_CMD_FILTER_OP_OUT_OP_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_FILTER_OP_IN/OP */
 /* Returned filter handle (for insert / subscribe operations). Note that these
@@ -6918,6 +8155,7 @@
 #define    MC_CMD_FILTER_OP_EXT_OUT_LEN 12
 /* identifies the type of operation requested */
 #define       MC_CMD_FILTER_OP_EXT_OUT_OP_OFST 0
+#define       MC_CMD_FILTER_OP_EXT_OUT_OP_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_FILTER_OP_EXT_IN/OP */
 /* Returned filter handle (for insert / subscribe operations). Note that these
@@ -6944,6 +8182,7 @@
 #define    MC_CMD_GET_PARSER_DISP_INFO_IN_LEN 4
 /* identifies the type of operation requested */
 #define       MC_CMD_GET_PARSER_DISP_INFO_IN_OP_OFST 0
+#define       MC_CMD_GET_PARSER_DISP_INFO_IN_OP_LEN 4
 /* enum: read the list of supported RX filter matches */
 #define          MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES  0x1
 /* enum: read flags indicating restrictions on filter insertion for the calling
@@ -6966,10 +8205,12 @@
 #define    MC_CMD_GET_PARSER_DISP_INFO_OUT_LEN(num) (8+4*(num))
 /* identifies the type of operation requested */
 #define       MC_CMD_GET_PARSER_DISP_INFO_OUT_OP_OFST 0
+#define       MC_CMD_GET_PARSER_DISP_INFO_OUT_OP_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_PARSER_DISP_INFO_IN/OP */
 /* number of supported match types */
 #define       MC_CMD_GET_PARSER_DISP_INFO_OUT_NUM_SUPPORTED_MATCHES_OFST 4
+#define       MC_CMD_GET_PARSER_DISP_INFO_OUT_NUM_SUPPORTED_MATCHES_LEN 4
 /* array of supported match types (valid MATCH_FIELDS values for
  * MC_CMD_FILTER_OP) sorted in decreasing priority order
  */
@@ -6982,10 +8223,12 @@
 #define    MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_LEN 8
 /* identifies the type of operation requested */
 #define       MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_OP_OFST 0
+#define       MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_OP_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_PARSER_DISP_INFO_IN/OP */
 /* bitfield of filter insertion restrictions */
 #define       MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_RESTRICTION_FLAGS_OFST 4
+#define       MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_RESTRICTION_FLAGS_LEN 4
 #define        MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_DST_IP_MCAST_ONLY_LBN 0
 #define        MC_CMD_GET_PARSER_DISP_RESTRICTIONS_OUT_DST_IP_MCAST_ONLY_WIDTH 1
 
@@ -7005,11 +8248,16 @@
 #define    MC_CMD_PARSER_DISP_RW_IN_LEN 32
 /* identifies the target of the operation */
 #define       MC_CMD_PARSER_DISP_RW_IN_TARGET_OFST 0
+#define       MC_CMD_PARSER_DISP_RW_IN_TARGET_LEN 4
 /* enum: RX dispatcher CPU */
 #define          MC_CMD_PARSER_DISP_RW_IN_RX_DICPU  0x0
 /* enum: TX dispatcher CPU */
 #define          MC_CMD_PARSER_DISP_RW_IN_TX_DICPU  0x1
-/* enum: Lookup engine (with original metadata format) */
+/* enum: Lookup engine (with original metadata format). Deprecated; used only
+ * by cmdclient as a fallback for very old Huntington firmware, and not
+ * supported in firmware beyond v6.4.0.1005. Use LUE_VERSIONED_METADATA
+ * instead.
+ */
 #define          MC_CMD_PARSER_DISP_RW_IN_LUE  0x2
 /* enum: Lookup engine (with requested metadata format) */
 #define          MC_CMD_PARSER_DISP_RW_IN_LUE_VERSIONED_METADATA  0x3
@@ -7021,26 +8269,33 @@
 #define          MC_CMD_PARSER_DISP_RW_IN_MISC_STATE  0x5
 /* identifies the type of operation requested */
 #define       MC_CMD_PARSER_DISP_RW_IN_OP_OFST 4
-/* enum: read a word of DICPU DMEM or a LUE entry */
+#define       MC_CMD_PARSER_DISP_RW_IN_OP_LEN 4
+/* enum: Read a word of DICPU DMEM or a LUE entry */
 #define          MC_CMD_PARSER_DISP_RW_IN_READ  0x0
-/* enum: write a word of DICPU DMEM or a LUE entry */
+/* enum: Write a word of DICPU DMEM or a LUE entry. */
 #define          MC_CMD_PARSER_DISP_RW_IN_WRITE  0x1
-/* enum: read-modify-write a word of DICPU DMEM (not valid for LUE) */
+/* enum: Read-modify-write a word of DICPU DMEM (not valid for LUE). */
 #define          MC_CMD_PARSER_DISP_RW_IN_RMW  0x2
 /* data memory address (DICPU targets) or LUE index (LUE targets) */
 #define       MC_CMD_PARSER_DISP_RW_IN_ADDRESS_OFST 8
+#define       MC_CMD_PARSER_DISP_RW_IN_ADDRESS_LEN 4
 /* selector (for MISC_STATE target) */
 #define       MC_CMD_PARSER_DISP_RW_IN_SELECTOR_OFST 8
+#define       MC_CMD_PARSER_DISP_RW_IN_SELECTOR_LEN 4
 /* enum: Port to datapath mapping */
 #define          MC_CMD_PARSER_DISP_RW_IN_PORT_DP_MAPPING  0x1
 /* value to write (for DMEM writes) */
 #define       MC_CMD_PARSER_DISP_RW_IN_DMEM_WRITE_VALUE_OFST 12
+#define       MC_CMD_PARSER_DISP_RW_IN_DMEM_WRITE_VALUE_LEN 4
 /* XOR value (for DMEM read-modify-writes: new = (old & mask) ^ value) */
 #define       MC_CMD_PARSER_DISP_RW_IN_DMEM_RMW_XOR_VALUE_OFST 12
+#define       MC_CMD_PARSER_DISP_RW_IN_DMEM_RMW_XOR_VALUE_LEN 4
 /* AND mask (for DMEM read-modify-writes: new = (old & mask) ^ value) */
 #define       MC_CMD_PARSER_DISP_RW_IN_DMEM_RMW_AND_MASK_OFST 16
+#define       MC_CMD_PARSER_DISP_RW_IN_DMEM_RMW_AND_MASK_LEN 4
 /* metadata format (for LUE reads using LUE_VERSIONED_METADATA) */
 #define       MC_CMD_PARSER_DISP_RW_IN_LUE_READ_METADATA_VERSION_OFST 12
+#define       MC_CMD_PARSER_DISP_RW_IN_LUE_READ_METADATA_VERSION_LEN 4
 /* value to write (for LUE writes) */
 #define       MC_CMD_PARSER_DISP_RW_IN_LUE_WRITE_VALUE_OFST 12
 #define       MC_CMD_PARSER_DISP_RW_IN_LUE_WRITE_VALUE_LEN 20
@@ -7049,6 +8304,7 @@
 #define    MC_CMD_PARSER_DISP_RW_OUT_LEN 52
 /* value read (for DMEM reads) */
 #define       MC_CMD_PARSER_DISP_RW_OUT_DMEM_READ_VALUE_OFST 0
+#define       MC_CMD_PARSER_DISP_RW_OUT_DMEM_READ_VALUE_LEN 4
 /* value read (for LUE reads) */
 #define       MC_CMD_PARSER_DISP_RW_OUT_LUE_READ_VALUE_OFST 0
 #define       MC_CMD_PARSER_DISP_RW_OUT_LUE_READ_VALUE_LEN 20
@@ -7093,6 +8349,7 @@
 #define    MC_CMD_SET_PF_COUNT_IN_LEN 4
 /* New number of PFs on the device. */
 #define       MC_CMD_SET_PF_COUNT_IN_PF_COUNT_OFST 0
+#define       MC_CMD_SET_PF_COUNT_IN_PF_COUNT_LEN 4
 
 /* MC_CMD_SET_PF_COUNT_OUT msgresponse */
 #define    MC_CMD_SET_PF_COUNT_OUT_LEN 0
@@ -7113,6 +8370,7 @@
 #define    MC_CMD_GET_PORT_ASSIGNMENT_OUT_LEN 4
 /* Identifies the port assignment for this function. */
 #define       MC_CMD_GET_PORT_ASSIGNMENT_OUT_PORT_OFST 0
+#define       MC_CMD_GET_PORT_ASSIGNMENT_OUT_PORT_LEN 4
 
 
 /***********************************/
@@ -7127,6 +8385,7 @@
 #define    MC_CMD_SET_PORT_ASSIGNMENT_IN_LEN 4
 /* Identifies the port assignment for this function. */
 #define       MC_CMD_SET_PORT_ASSIGNMENT_IN_PORT_OFST 0
+#define       MC_CMD_SET_PORT_ASSIGNMENT_IN_PORT_LEN 4
 
 /* MC_CMD_SET_PORT_ASSIGNMENT_OUT msgresponse */
 #define    MC_CMD_SET_PORT_ASSIGNMENT_OUT_LEN 0
@@ -7144,8 +8403,10 @@
 #define    MC_CMD_ALLOC_VIS_IN_LEN 8
 /* The minimum number of VIs that is acceptable */
 #define       MC_CMD_ALLOC_VIS_IN_MIN_VI_COUNT_OFST 0
+#define       MC_CMD_ALLOC_VIS_IN_MIN_VI_COUNT_LEN 4
 /* The maximum number of VIs that would be useful */
 #define       MC_CMD_ALLOC_VIS_IN_MAX_VI_COUNT_OFST 4
+#define       MC_CMD_ALLOC_VIS_IN_MAX_VI_COUNT_LEN 4
 
 /* MC_CMD_ALLOC_VIS_OUT msgresponse: Huntington-compatible VI_ALLOC request.
  * Use extended version in new code.
@@ -7153,21 +8414,26 @@
 #define    MC_CMD_ALLOC_VIS_OUT_LEN 8
 /* The number of VIs allocated on this function */
 #define       MC_CMD_ALLOC_VIS_OUT_VI_COUNT_OFST 0
+#define       MC_CMD_ALLOC_VIS_OUT_VI_COUNT_LEN 4
 /* The base absolute VI number allocated to this function. Required to
  * correctly interpret wakeup events.
  */
 #define       MC_CMD_ALLOC_VIS_OUT_VI_BASE_OFST 4
+#define       MC_CMD_ALLOC_VIS_OUT_VI_BASE_LEN 4
 
 /* MC_CMD_ALLOC_VIS_EXT_OUT msgresponse */
 #define    MC_CMD_ALLOC_VIS_EXT_OUT_LEN 12
 /* The number of VIs allocated on this function */
 #define       MC_CMD_ALLOC_VIS_EXT_OUT_VI_COUNT_OFST 0
+#define       MC_CMD_ALLOC_VIS_EXT_OUT_VI_COUNT_LEN 4
 /* The base absolute VI number allocated to this function. Required to
  * correctly interpret wakeup events.
  */
 #define       MC_CMD_ALLOC_VIS_EXT_OUT_VI_BASE_OFST 4
+#define       MC_CMD_ALLOC_VIS_EXT_OUT_VI_BASE_LEN 4
 /* Function's port vi_shift value (always 0 on Huntington) */
 #define       MC_CMD_ALLOC_VIS_EXT_OUT_VI_SHIFT_OFST 8
+#define       MC_CMD_ALLOC_VIS_EXT_OUT_VI_SHIFT_LEN 4
 
 
 /***********************************/
@@ -7201,15 +8467,20 @@
 #define    MC_CMD_GET_SRIOV_CFG_OUT_LEN 20
 /* Number of VFs currently enabled. */
 #define       MC_CMD_GET_SRIOV_CFG_OUT_VF_CURRENT_OFST 0
+#define       MC_CMD_GET_SRIOV_CFG_OUT_VF_CURRENT_LEN 4
 /* Max number of VFs before sriov stride and offset may need to be changed. */
 #define       MC_CMD_GET_SRIOV_CFG_OUT_VF_MAX_OFST 4
+#define       MC_CMD_GET_SRIOV_CFG_OUT_VF_MAX_LEN 4
 #define       MC_CMD_GET_SRIOV_CFG_OUT_FLAGS_OFST 8
+#define       MC_CMD_GET_SRIOV_CFG_OUT_FLAGS_LEN 4
 #define        MC_CMD_GET_SRIOV_CFG_OUT_VF_ENABLED_LBN 0
 #define        MC_CMD_GET_SRIOV_CFG_OUT_VF_ENABLED_WIDTH 1
 /* RID offset of first VF from PF. */
 #define       MC_CMD_GET_SRIOV_CFG_OUT_VF_OFFSET_OFST 12
+#define       MC_CMD_GET_SRIOV_CFG_OUT_VF_OFFSET_LEN 4
 /* RID offset of each subsequent VF from the previous. */
 #define       MC_CMD_GET_SRIOV_CFG_OUT_VF_STRIDE_OFST 16
+#define       MC_CMD_GET_SRIOV_CFG_OUT_VF_STRIDE_LEN 4
 
 
 /***********************************/
@@ -7224,19 +8495,24 @@
 #define    MC_CMD_SET_SRIOV_CFG_IN_LEN 20
 /* Number of VFs currently enabled. */
 #define       MC_CMD_SET_SRIOV_CFG_IN_VF_CURRENT_OFST 0
+#define       MC_CMD_SET_SRIOV_CFG_IN_VF_CURRENT_LEN 4
 /* Max number of VFs before sriov stride and offset may need to be changed. */
 #define       MC_CMD_SET_SRIOV_CFG_IN_VF_MAX_OFST 4
+#define       MC_CMD_SET_SRIOV_CFG_IN_VF_MAX_LEN 4
 #define       MC_CMD_SET_SRIOV_CFG_IN_FLAGS_OFST 8
+#define       MC_CMD_SET_SRIOV_CFG_IN_FLAGS_LEN 4
 #define        MC_CMD_SET_SRIOV_CFG_IN_VF_ENABLED_LBN 0
 #define        MC_CMD_SET_SRIOV_CFG_IN_VF_ENABLED_WIDTH 1
 /* RID offset of first VF from PF, or 0 for no change, or
  * MC_CMD_RESOURCE_INSTANCE_ANY to allow the system to allocate an offset.
  */
 #define       MC_CMD_SET_SRIOV_CFG_IN_VF_OFFSET_OFST 12
+#define       MC_CMD_SET_SRIOV_CFG_IN_VF_OFFSET_LEN 4
 /* RID offset of each subsequent VF from the previous, 0 for no change, or
  * MC_CMD_RESOURCE_INSTANCE_ANY to allow the system to allocate a stride.
  */
 #define       MC_CMD_SET_SRIOV_CFG_IN_VF_STRIDE_OFST 16
+#define       MC_CMD_SET_SRIOV_CFG_IN_VF_STRIDE_LEN 4
 
 /* MC_CMD_SET_SRIOV_CFG_OUT msgresponse */
 #define    MC_CMD_SET_SRIOV_CFG_OUT_LEN 0
@@ -7258,12 +8534,15 @@
 #define    MC_CMD_GET_VI_ALLOC_INFO_OUT_LEN 12
 /* The number of VIs allocated on this function */
 #define       MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_COUNT_OFST 0
+#define       MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_COUNT_LEN 4
 /* The base absolute VI number allocated to this function. Required to
  * correctly interpret wakeup events.
  */
 #define       MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_BASE_OFST 4
+#define       MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_BASE_LEN 4
 /* Function's port vi_shift value (always 0 on Huntington) */
 #define       MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_SHIFT_OFST 8
+#define       MC_CMD_GET_VI_ALLOC_INFO_OUT_VI_SHIFT_LEN 4
 
 
 /***********************************/
@@ -7278,6 +8557,7 @@
 #define    MC_CMD_DUMP_VI_STATE_IN_LEN 4
 /* The VI number to query. */
 #define       MC_CMD_DUMP_VI_STATE_IN_VI_NUMBER_OFST 0
+#define       MC_CMD_DUMP_VI_STATE_IN_VI_NUMBER_LEN 4
 
 /* MC_CMD_DUMP_VI_STATE_OUT msgresponse */
 #define    MC_CMD_DUMP_VI_STATE_OUT_LEN 96
@@ -7311,6 +8591,7 @@
 #define       MC_CMD_DUMP_VI_STATE_OUT_VI_EV_TIMER_RAW_HI_OFST 24
 /* Combined metadata field. */
 #define       MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_OFST 28
+#define       MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_LEN 4
 #define        MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_BUFS_BASE_LBN 0
 #define        MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_BUFS_BASE_WIDTH 16
 #define        MC_CMD_DUMP_VI_STATE_OUT_VI_EV_META_BUFS_NPAGES_LBN 16
@@ -7392,6 +8673,7 @@
 #define    MC_CMD_ALLOC_PIOBUF_OUT_LEN 4
 /* Handle for allocated push I/O buffer. */
 #define       MC_CMD_ALLOC_PIOBUF_OUT_PIOBUF_HANDLE_OFST 0
+#define       MC_CMD_ALLOC_PIOBUF_OUT_PIOBUF_HANDLE_LEN 4
 
 
 /***********************************/
@@ -7406,6 +8688,7 @@
 #define    MC_CMD_FREE_PIOBUF_IN_LEN 4
 /* Handle for allocated push I/O buffer. */
 #define       MC_CMD_FREE_PIOBUF_IN_PIOBUF_HANDLE_OFST 0
+#define       MC_CMD_FREE_PIOBUF_IN_PIOBUF_HANDLE_LEN 4
 
 /* MC_CMD_FREE_PIOBUF_OUT msgresponse */
 #define    MC_CMD_FREE_PIOBUF_OUT_LEN 0
@@ -7423,6 +8706,7 @@
 #define    MC_CMD_GET_VI_TLP_PROCESSING_IN_LEN 4
 /* VI number to get information for. */
 #define       MC_CMD_GET_VI_TLP_PROCESSING_IN_INSTANCE_OFST 0
+#define       MC_CMD_GET_VI_TLP_PROCESSING_IN_INSTANCE_LEN 4
 
 /* MC_CMD_GET_VI_TLP_PROCESSING_OUT msgresponse */
 #define    MC_CMD_GET_VI_TLP_PROCESSING_OUT_LEN 4
@@ -7445,6 +8729,7 @@
 #define       MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_ON_LBN 19
 #define       MC_CMD_GET_VI_TLP_PROCESSING_OUT_TPH_ON_WIDTH 1
 #define       MC_CMD_GET_VI_TLP_PROCESSING_OUT_DATA_OFST 0
+#define       MC_CMD_GET_VI_TLP_PROCESSING_OUT_DATA_LEN 4
 
 
 /***********************************/
@@ -7459,6 +8744,7 @@
 #define    MC_CMD_SET_VI_TLP_PROCESSING_IN_LEN 8
 /* VI number to set information for. */
 #define       MC_CMD_SET_VI_TLP_PROCESSING_IN_INSTANCE_OFST 0
+#define       MC_CMD_SET_VI_TLP_PROCESSING_IN_INSTANCE_LEN 4
 /* Transaction processing steering hint 1 for use with the Rx Queue. */
 #define       MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_TAG1_RX_OFST 4
 #define       MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_TAG1_RX_LEN 1
@@ -7478,6 +8764,7 @@
 #define       MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_ON_LBN 51
 #define       MC_CMD_SET_VI_TLP_PROCESSING_IN_TPH_ON_WIDTH 1
 #define       MC_CMD_SET_VI_TLP_PROCESSING_IN_DATA_OFST 4
+#define       MC_CMD_SET_VI_TLP_PROCESSING_IN_DATA_LEN 4
 
 /* MC_CMD_SET_VI_TLP_PROCESSING_OUT msgresponse */
 #define    MC_CMD_SET_VI_TLP_PROCESSING_OUT_LEN 0
@@ -7494,6 +8781,7 @@
 /* MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN msgrequest */
 #define    MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_LEN 4
 #define       MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_OFST 0
+#define       MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_LEN 4
 /* enum: MISC. */
 #define          MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_MISC  0x0
 /* enum: IDO. */
@@ -7506,10 +8794,12 @@
 /* MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT msgresponse */
 #define    MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_LEN 8
 #define       MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_GLOBAL_CATEGORY_OFST 0
+#define       MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_GLOBAL_CATEGORY_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN/TLP_GLOBAL_CATEGORY */
 /* Amalgamated TLP info word. */
 #define       MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_WORD_OFST 4
+#define       MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_WORD_LEN 4
 #define        MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_MISC_WTAG_EN_LBN 0
 #define        MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_MISC_WTAG_EN_WIDTH 1
 #define        MC_CMD_GET_TLP_PROCESSING_GLOBALS_OUT_TLP_INFO_MISC_SPARE_LBN 1
@@ -7557,10 +8847,12 @@
 /* MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN msgrequest */
 #define    MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_LEN 8
 #define       MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_OFST 0
+#define       MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_GLOBAL_CATEGORY_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_GET_TLP_PROCESSING_GLOBALS/MC_CMD_GET_TLP_PROCESSING_GLOBALS_IN/TLP_GLOBAL_CATEGORY */
 /* Amalgamated TLP info word. */
 #define       MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_WORD_OFST 4
+#define       MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_WORD_LEN 4
 #define        MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_MISC_WTAG_EN_LBN 0
 #define        MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_MISC_WTAG_EN_WIDTH 1
 #define        MC_CMD_SET_TLP_PROCESSING_GLOBALS_IN_TLP_INFO_IDO_DL_EN_LBN 0
@@ -7627,6 +8919,7 @@
  * in a command from the host.)
  */
 #define       MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_OFST 0
+#define       MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_LEN 4
 #define          MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_IDLE     0x0 /* enum */
 #define          MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_RESET    0x1 /* enum */
 #define          MC_CMD_SATELLITE_DOWNLOAD_IN_PHASE_IMEMS    0x2 /* enum */
@@ -7636,6 +8929,7 @@
  * mc_flash_layout.h.)
  */
 #define       MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_OFST 4
+#define       MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_LEN 4
 /* enum: Valid in phase 2 (PHASE_IMEMS) only */
 #define          MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_TXDI_TEXT  0x0
 /* enum: Valid in phase 2 (PHASE_IMEMS) only */
@@ -7672,12 +8966,14 @@
 #define          MC_CMD_SATELLITE_DOWNLOAD_IN_TARGET_ALL  0xffffffff
 /* Chunk ID, or CHUNK_ID_LAST or CHUNK_ID_ABORT */
 #define       MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_ID_OFST 8
+#define       MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_ID_LEN 4
 /* enum: Last chunk, containing checksum rather than data */
 #define          MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_ID_LAST  0xffffffff
 /* enum: Abort download of this item */
 #define          MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_ID_ABORT  0xfffffffe
 /* Length of this chunk in bytes */
 #define       MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_LEN_OFST 12
+#define       MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_LEN_LEN 4
 /* Data for this chunk */
 #define       MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_DATA_OFST 16
 #define       MC_CMD_SATELLITE_DOWNLOAD_IN_CHUNK_DATA_LEN 4
@@ -7688,8 +8984,10 @@
 #define    MC_CMD_SATELLITE_DOWNLOAD_OUT_LEN 8
 /* Same as MC_CMD_ERR field, but included as 0 in success cases */
 #define       MC_CMD_SATELLITE_DOWNLOAD_OUT_RESULT_OFST 0
+#define       MC_CMD_SATELLITE_DOWNLOAD_OUT_RESULT_LEN 4
 /* Extra status information */
 #define       MC_CMD_SATELLITE_DOWNLOAD_OUT_INFO_OFST 4
+#define       MC_CMD_SATELLITE_DOWNLOAD_OUT_INFO_LEN 4
 /* enum: Code download OK, completed. */
 #define          MC_CMD_SATELLITE_DOWNLOAD_OUT_OK_COMPLETE  0x0
 /* enum: Code download aborted as requested. */
@@ -7726,6 +9024,7 @@
 #define    MC_CMD_GET_CAPABILITIES_OUT_LEN 20
 /* First word of flags. */
 #define       MC_CMD_GET_CAPABILITIES_OUT_FLAGS1_OFST 0
+#define       MC_CMD_GET_CAPABILITIES_OUT_FLAGS1_LEN 4
 #define        MC_CMD_GET_CAPABILITIES_OUT_VPORT_RECONFIGURE_LBN 3
 #define        MC_CMD_GET_CAPABILITIES_OUT_VPORT_RECONFIGURE_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_OUT_TX_STRIPING_LBN 4
@@ -7793,6 +9092,8 @@
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_LOW_LATENCY  0x1
 /* enum: Packed stream RXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_PACKED_STREAM  0x2
+/* enum: Rules engine RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_RULES_ENGINE  0x5
 /* enum: BIST RXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_BIST  0x10a
 /* enum: RXDP Test firmware image 1 */
@@ -7813,6 +9114,8 @@
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_DISABLE_DL  0x108
 /* enum: RXDP Test firmware image 9 */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_DOORBELL_DELAY  0x10b
+/* enum: RXDP Test firmware image 10 */
+#define          MC_CMD_GET_CAPABILITIES_OUT_RXDP_TEST_FW_SLOW  0x10c
 /* TxDPCPU firmware id. */
 #define       MC_CMD_GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID_OFST 6
 #define       MC_CMD_GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID_LEN 2
@@ -7822,6 +9125,8 @@
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXDP_LOW_LATENCY  0x1
 /* enum: High packet rate TXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXDP_HIGH_PACKET_RATE  0x3
+/* enum: Rules engine TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_OUT_TXDP_RULES_ENGINE  0x5
 /* enum: BIST TXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXDP_BIST  0x12d
 /* enum: TXDP Test firmware image 1 */
@@ -7848,7 +9153,9 @@
  * (Huntington development only)
  */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_SIENA_COMPAT  0x2
-/* enum: Virtual switching (full feature) RX PD production firmware */
+/* enum: Full featured RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_VSWITCH  0x3
 /* enum: siena_compat variant RX PD firmware using PM rather than MAC
  * (Huntington development only)
@@ -7864,6 +9171,8 @@
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_LAYER2_PERF  0x7
 /* enum: Rules engine RX PD production firmware */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_RESERVED_9  0x9
 /* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
 #define          MC_CMD_GET_CAPABILITIES_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
 /* enum: RX PD firmware parsing but not filtering network overlay tunnel
@@ -7888,7 +9197,9 @@
  * (Huntington development only)
  */
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_SIENA_COMPAT  0x2
-/* enum: Virtual switching (full feature) TX PD production firmware */
+/* enum: Full featured TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_VSWITCH  0x3
 /* enum: siena_compat variant TX PD firmware using PM rather than MAC
  * (Huntington development only)
@@ -7901,12 +9212,16 @@
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_LAYER2_PERF  0x7
 /* enum: Rules engine TX PD production firmware */
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_RESERVED_9  0x9
 /* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
 #define          MC_CMD_GET_CAPABILITIES_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
 /* Hardware capabilities of NIC */
 #define       MC_CMD_GET_CAPABILITIES_OUT_HW_CAPABILITIES_OFST 12
+#define       MC_CMD_GET_CAPABILITIES_OUT_HW_CAPABILITIES_LEN 4
 /* Licensed capabilities */
 #define       MC_CMD_GET_CAPABILITIES_OUT_LICENSE_CAPABILITIES_OFST 16
+#define       MC_CMD_GET_CAPABILITIES_OUT_LICENSE_CAPABILITIES_LEN 4
 
 /* MC_CMD_GET_CAPABILITIES_V2_IN msgrequest */
 #define    MC_CMD_GET_CAPABILITIES_V2_IN_LEN 0
@@ -7915,6 +9230,7 @@
 #define    MC_CMD_GET_CAPABILITIES_V2_OUT_LEN 72
 /* First word of flags. */
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS1_OFST 0
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS1_LEN 4
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_VPORT_RECONFIGURE_LBN 3
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_VPORT_RECONFIGURE_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_STRIPING_LBN 4
@@ -7982,6 +9298,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_LOW_LATENCY  0x1
 /* enum: Packed stream RXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_PACKED_STREAM  0x2
+/* enum: Rules engine RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_RULES_ENGINE  0x5
 /* enum: BIST RXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_BIST  0x10a
 /* enum: RXDP Test firmware image 1 */
@@ -8002,6 +9320,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_DISABLE_DL  0x108
 /* enum: RXDP Test firmware image 9 */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_DOORBELL_DELAY  0x10b
+/* enum: RXDP Test firmware image 10 */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXDP_TEST_FW_SLOW  0x10c
 /* TxDPCPU firmware id. */
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DPCPU_FW_ID_OFST 6
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_TX_DPCPU_FW_ID_LEN 2
@@ -8011,6 +9331,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_LOW_LATENCY  0x1
 /* enum: High packet rate TXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_HIGH_PACKET_RATE  0x3
+/* enum: Rules engine TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_RULES_ENGINE  0x5
 /* enum: BIST TXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXDP_BIST  0x12d
 /* enum: TXDP Test firmware image 1 */
@@ -8037,7 +9359,9 @@
  * (Huntington development only)
  */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_SIENA_COMPAT  0x2
-/* enum: Virtual switching (full feature) RX PD production firmware */
+/* enum: Full featured RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_VSWITCH  0x3
 /* enum: siena_compat variant RX PD firmware using PM rather than MAC
  * (Huntington development only)
@@ -8053,6 +9377,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_LAYER2_PERF  0x7
 /* enum: Rules engine RX PD production firmware */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_RESERVED_9  0x9
 /* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
 /* enum: RX PD firmware parsing but not filtering network overlay tunnel
@@ -8077,7 +9403,9 @@
  * (Huntington development only)
  */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_SIENA_COMPAT  0x2
-/* enum: Virtual switching (full feature) TX PD production firmware */
+/* enum: Full featured TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_VSWITCH  0x3
 /* enum: siena_compat variant TX PD firmware using PM rather than MAC
  * (Huntington development only)
@@ -8090,14 +9418,19 @@
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_LAYER2_PERF  0x7
 /* enum: Rules engine TX PD production firmware */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_RESERVED_9  0x9
 /* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
 #define          MC_CMD_GET_CAPABILITIES_V2_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
 /* Hardware capabilities of NIC */
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_HW_CAPABILITIES_OFST 12
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_HW_CAPABILITIES_LEN 4
 /* Licensed capabilities */
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_LICENSE_CAPABILITIES_OFST 16
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_LICENSE_CAPABILITIES_LEN 4
 /* Second word of flags. Not present on older firmware (check the length). */
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS2_OFST 20
+#define       MC_CMD_GET_CAPABILITIES_V2_OUT_FLAGS2_LEN 4
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN 0
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_ENCAP_LBN 1
@@ -8124,6 +9457,18 @@
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_TX_SNIFF_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_LBN 12
 #define        MC_CMD_GET_CAPABILITIES_V2_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_BACKGROUND_LBN 13
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_BACKGROUND_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_DB_RETURN_LBN 14
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_MCDI_DB_RETURN_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_CTPIO_LBN 15
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_CTPIO_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TSA_SUPPORT_LBN 16
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TSA_SUPPORT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TSA_BOUND_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_TSA_BOUND_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_SF_ADAPTER_AUTHENTICATION_LBN 18
+#define        MC_CMD_GET_CAPABILITIES_V2_OUT_SF_ADAPTER_AUTHENTICATION_WIDTH 1
 /* Number of FATSOv2 contexts per datapath supported by this NIC. Not present
  * on older firmware (check the length).
  */
@@ -8181,9 +9526,10 @@
 #define       MC_CMD_GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF_LEN 2
 
 /* MC_CMD_GET_CAPABILITIES_V3_OUT msgresponse */
-#define    MC_CMD_GET_CAPABILITIES_V3_OUT_LEN 73
+#define    MC_CMD_GET_CAPABILITIES_V3_OUT_LEN 76
 /* First word of flags. */
 #define       MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS1_OFST 0
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS1_LEN 4
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_VPORT_RECONFIGURE_LBN 3
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_VPORT_RECONFIGURE_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_STRIPING_LBN 4
@@ -8251,6 +9597,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_LOW_LATENCY  0x1
 /* enum: Packed stream RXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_PACKED_STREAM  0x2
+/* enum: Rules engine RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_RULES_ENGINE  0x5
 /* enum: BIST RXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_BIST  0x10a
 /* enum: RXDP Test firmware image 1 */
@@ -8271,6 +9619,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_DISABLE_DL  0x108
 /* enum: RXDP Test firmware image 9 */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_DOORBELL_DELAY  0x10b
+/* enum: RXDP Test firmware image 10 */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXDP_TEST_FW_SLOW  0x10c
 /* TxDPCPU firmware id. */
 #define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DPCPU_FW_ID_OFST 6
 #define       MC_CMD_GET_CAPABILITIES_V3_OUT_TX_DPCPU_FW_ID_LEN 2
@@ -8280,6 +9630,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_LOW_LATENCY  0x1
 /* enum: High packet rate TXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_HIGH_PACKET_RATE  0x3
+/* enum: Rules engine TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_RULES_ENGINE  0x5
 /* enum: BIST TXDP firmware */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXDP_BIST  0x12d
 /* enum: TXDP Test firmware image 1 */
@@ -8306,7 +9658,9 @@
  * (Huntington development only)
  */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_SIENA_COMPAT  0x2
-/* enum: Virtual switching (full feature) RX PD production firmware */
+/* enum: Full featured RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_VSWITCH  0x3
 /* enum: siena_compat variant RX PD firmware using PM rather than MAC
  * (Huntington development only)
@@ -8322,6 +9676,8 @@
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_LAYER2_PERF  0x7
 /* enum: Rules engine RX PD production firmware */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_RESERVED_9  0x9
 /* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
 /* enum: RX PD firmware parsing but not filtering network overlay tunnel
@@ -8346,7 +9702,9 @@
  * (Huntington development only)
  */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_SIENA_COMPAT  0x2
-/* enum: Virtual switching (full feature) TX PD production firmware */
+/* enum: Full featured TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_VSWITCH  0x3
 /* enum: siena_compat variant TX PD firmware using PM rather than MAC
  * (Huntington development only)
@@ -8359,14 +9717,19 @@
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_LAYER2_PERF  0x7
 /* enum: Rules engine TX PD production firmware */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_RESERVED_9  0x9
 /* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
 /* Hardware capabilities of NIC */
 #define       MC_CMD_GET_CAPABILITIES_V3_OUT_HW_CAPABILITIES_OFST 12
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_HW_CAPABILITIES_LEN 4
 /* Licensed capabilities */
 #define       MC_CMD_GET_CAPABILITIES_V3_OUT_LICENSE_CAPABILITIES_OFST 16
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_LICENSE_CAPABILITIES_LEN 4
 /* Second word of flags. Not present on older firmware (check the length). */
 #define       MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS2_OFST 20
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_FLAGS2_LEN 4
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_LBN 0
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_TSO_V2_ENCAP_LBN 1
@@ -8393,6 +9756,18 @@
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_TX_SNIFF_WIDTH 1
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_LBN 12
 #define        MC_CMD_GET_CAPABILITIES_V3_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_BACKGROUND_LBN 13
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_BACKGROUND_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_DB_RETURN_LBN 14
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_MCDI_DB_RETURN_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_CTPIO_LBN 15
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_CTPIO_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TSA_SUPPORT_LBN 16
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TSA_SUPPORT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TSA_BOUND_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_TSA_BOUND_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_SF_ADAPTER_AUTHENTICATION_LBN 18
+#define        MC_CMD_GET_CAPABILITIES_V3_OUT_SF_ADAPTER_AUTHENTICATION_WIDTH 1
 /* Number of FATSOv2 contexts per datapath supported by this NIC. Not present
  * on older firmware (check the length).
  */
@@ -8463,6 +9838,348 @@
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_16K  0x1
 /* enum: Each VI occupies 64k. PIO is at offset 4k. CTPIO is at offset 12k. */
 #define          MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_64K  0x2
+/* Number of vFIFOs per adapter that can be used for VFIFO Stuffing
+ * (SF-115995-SW) in the present configuration of firmware and port mode.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_VFIFOS_OFST 73
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_VFIFOS_LEN 1
+/* Number of buffers per adapter that can be used for VFIFO Stuffing
+ * (SF-115995-SW) in the present configuration of firmware and port mode.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_OFST 74
+#define       MC_CMD_GET_CAPABILITIES_V3_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_LEN 2
+
+/* MC_CMD_GET_CAPABILITIES_V4_OUT msgresponse */
+#define    MC_CMD_GET_CAPABILITIES_V4_OUT_LEN 78
+/* First word of flags. */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS1_OFST 0
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS1_LEN 4
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VPORT_RECONFIGURE_LBN 3
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VPORT_RECONFIGURE_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_STRIPING_LBN 4
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_STRIPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VADAPTOR_QUERY_LBN 5
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VADAPTOR_QUERY_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVB_PORT_VLAN_RESTRICT_LBN 6
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVB_PORT_VLAN_RESTRICT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_DRV_ATTACH_PREBOOT_LBN 7
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_DRV_ATTACH_PREBOOT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_FORCE_EVENT_MERGING_LBN 8
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_FORCE_EVENT_MERGING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_SET_MAC_ENHANCED_LBN 9
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_SET_MAC_ENHANCED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_LBN 10
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_UNKNOWN_UCAST_DST_FILTER_ALWAYS_MULTI_RECIPIENT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 11
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_MAC_SECURITY_FILTERING_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_MAC_SECURITY_FILTERING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_ADDITIONAL_RSS_MODES_LBN 13
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_ADDITIONAL_RSS_MODES_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_QBB_LBN 14
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_QBB_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PACKED_STREAM_VAR_BUFFERS_LBN 15
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PACKED_STREAM_VAR_BUFFERS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_RSS_LIMITED_LBN 16
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_RSS_LIMITED_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PACKED_STREAM_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PACKED_STREAM_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_INCLUDE_FCS_LBN 18
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_INCLUDE_FCS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_VLAN_INSERTION_LBN 19
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_VLAN_INSERTION_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_VLAN_STRIPPING_LBN 20
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_VLAN_STRIPPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_LBN 21
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PREFIX_LEN_0_LBN 22
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PREFIX_LEN_0_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PREFIX_LEN_14_LBN 23
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_PREFIX_LEN_14_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_TIMESTAMP_LBN 24
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_TIMESTAMP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_BATCHING_LBN 25
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_BATCHING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MCAST_FILTER_CHAINING_LBN 26
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MCAST_FILTER_CHAINING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_PM_AND_RXDP_COUNTERS_LBN 27
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_PM_AND_RXDP_COUNTERS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_DISABLE_SCATTER_LBN 28
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_DISABLE_SCATTER_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_MCAST_UDP_LOOPBACK_LBN 29
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_MCAST_UDP_LOOPBACK_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVB_LBN 30
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVB_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VXLAN_NVGRE_LBN 31
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_VXLAN_NVGRE_WIDTH 1
+/* RxDPCPU firmware id. */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_RX_DPCPU_FW_ID_OFST 4
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_RX_DPCPU_FW_ID_LEN 2
+/* enum: Standard RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP  0x0
+/* enum: Low latency RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_LOW_LATENCY  0x1
+/* enum: Packed stream RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_PACKED_STREAM  0x2
+/* enum: Rules engine RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_RULES_ENGINE  0x5
+/* enum: BIST RXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_BIST  0x10a
+/* enum: RXDP Test firmware image 1 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_TO_MC_CUT_THROUGH  0x101
+/* enum: RXDP Test firmware image 2 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD  0x102
+/* enum: RXDP Test firmware image 3 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_TO_MC_STORE_FORWARD_FIRST  0x103
+/* enum: RXDP Test firmware image 4 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_EVERY_EVENT_BATCHABLE  0x104
+/* enum: RXDP Test firmware image 5 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_BACKPRESSURE  0x105
+/* enum: RXDP Test firmware image 6 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_PACKET_EDITS  0x106
+/* enum: RXDP Test firmware image 7 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_RX_HDR_SPLIT  0x107
+/* enum: RXDP Test firmware image 8 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_DISABLE_DL  0x108
+/* enum: RXDP Test firmware image 9 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_DOORBELL_DELAY  0x10b
+/* enum: RXDP Test firmware image 10 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXDP_TEST_FW_SLOW  0x10c
+/* TxDPCPU firmware id. */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TX_DPCPU_FW_ID_OFST 6
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TX_DPCPU_FW_ID_LEN 2
+/* enum: Standard TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP  0x0
+/* enum: Low latency TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP_LOW_LATENCY  0x1
+/* enum: High packet rate TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP_HIGH_PACKET_RATE  0x3
+/* enum: Rules engine TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP_RULES_ENGINE  0x5
+/* enum: BIST TXDP firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP_BIST  0x12d
+/* enum: TXDP Test firmware image 1 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP_TEST_FW_TSO_EDIT  0x101
+/* enum: TXDP Test firmware image 2 */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP_TEST_FW_PACKET_EDITS  0x102
+/* enum: TXDP CSR bus test firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXDP_TEST_FW_CSR  0x103
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_VERSION_OFST 8
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_VERSION_LEN 2
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_VERSION_REV_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_VERSION_REV_WIDTH 12
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_VERSION_TYPE_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_VERSION_TYPE_WIDTH 4
+/* enum: reserved value - do not use (may indicate alternative interpretation
+ * of REV field in future)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_RESERVED  0x0
+/* enum: Trivial RX PD firmware for early Huntington development (Huntington
+ * development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_FIRST_PKT  0x1
+/* enum: RX PD firmware with approximately Siena-compatible behaviour
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_SIENA_COMPAT  0x2
+/* enum: Full featured RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_VSWITCH  0x3
+/* enum: siena_compat variant RX PD firmware using PM rather than MAC
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_SIENA_COMPAT_PM  0x4
+/* enum: Low latency RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_LOW_LATENCY  0x5
+/* enum: Packed stream RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_PACKED_STREAM  0x6
+/* enum: RX PD firmware handling layer 2 only for high packet rate performance
+ * tests (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_LAYER2_PERF  0x7
+/* enum: Rules engine RX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_RESERVED_9  0x9
+/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
+/* enum: RX PD firmware parsing but not filtering network overlay tunnel
+ * encapsulations (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_RXPD_FW_TYPE_TESTFW_ENCAP_PARSING_ONLY  0xf
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_VERSION_OFST 10
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_VERSION_LEN 2
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_VERSION_REV_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_VERSION_REV_WIDTH 12
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_VERSION_TYPE_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_VERSION_TYPE_WIDTH 4
+/* enum: reserved value - do not use (may indicate alternative interpretation
+ * of REV field in future)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_RESERVED  0x0
+/* enum: Trivial TX PD firmware for early Huntington development (Huntington
+ * development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_FIRST_PKT  0x1
+/* enum: TX PD firmware with approximately Siena-compatible behaviour
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_SIENA_COMPAT  0x2
+/* enum: Full featured TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_FULL_FEATURED  0x3
+/* enum: (deprecated original name for the FULL_FEATURED variant) */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_VSWITCH  0x3
+/* enum: siena_compat variant TX PD firmware using PM rather than MAC
+ * (Huntington development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_SIENA_COMPAT_PM  0x4
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_LOW_LATENCY  0x5 /* enum */
+/* enum: TX PD firmware handling layer 2 only for high packet rate performance
+ * tests (Medford development only)
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_LAYER2_PERF  0x7
+/* enum: Rules engine TX PD production firmware */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_RULES_ENGINE  0x8
+/* enum: reserved value - do not use (bug69716) */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_RESERVED_9  0x9
+/* enum: RX PD firmware for GUE parsing prototype (Medford development only) */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_TXPD_FW_TYPE_TESTFW_GUE_PROTOTYPE  0xe
+/* Hardware capabilities of NIC */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_HW_CAPABILITIES_OFST 12
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_HW_CAPABILITIES_LEN 4
+/* Licensed capabilities */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_LICENSE_CAPABILITIES_OFST 16
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_LICENSE_CAPABILITIES_LEN 4
+/* Second word of flags. Not present on older firmware (check the length). */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS2_OFST 20
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS2_LEN 4
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_V2_LBN 0
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_V2_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_V2_ENCAP_LBN 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_V2_ENCAP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVQ_TIMER_CTRL_LBN 2
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVQ_TIMER_CTRL_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVENT_CUT_THROUGH_LBN 3
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_EVENT_CUT_THROUGH_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_CUT_THROUGH_LBN 4
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_CUT_THROUGH_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_VFIFO_ULL_MODE_LBN 5
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_VFIFO_ULL_MODE_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN 6
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_40G_TX_SIZE_BINS_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_INIT_EVQ_V2_LBN 7
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_INIT_EVQ_V2_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_MAC_TIMESTAMPING_LBN 8
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_MAC_TIMESTAMPING_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TIMESTAMP_LBN 9
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TIMESTAMP_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_SNIFF_LBN 10
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_RX_SNIFF_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_SNIFF_LBN 11
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TX_SNIFF_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_LBN 12
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_NVRAM_UPDATE_REPORT_VERIFY_RESULT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MCDI_BACKGROUND_LBN 13
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MCDI_BACKGROUND_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MCDI_DB_RETURN_LBN 14
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_MCDI_DB_RETURN_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_CTPIO_LBN 15
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_CTPIO_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TSA_SUPPORT_LBN 16
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TSA_SUPPORT_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TSA_BOUND_LBN 17
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_TSA_BOUND_WIDTH 1
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_SF_ADAPTER_AUTHENTICATION_LBN 18
+#define        MC_CMD_GET_CAPABILITIES_V4_OUT_SF_ADAPTER_AUTHENTICATION_WIDTH 1
+/* Number of FATSOv2 contexts per datapath supported by this NIC. Not present
+ * on older firmware (check the length).
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_V2_N_CONTEXTS_OFST 24
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TX_TSO_V2_N_CONTEXTS_LEN 2
+/* One byte per PF containing the number of the external port assigned to this
+ * PF, indexed by PF number. Special values indicate that a PF is either not
+ * present or not assigned.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_PFS_TO_PORTS_ASSIGNMENT_OFST 26
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_PFS_TO_PORTS_ASSIGNMENT_LEN 1
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_PFS_TO_PORTS_ASSIGNMENT_NUM 16
+/* enum: The caller is not permitted to access information on this PF. */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_ACCESS_NOT_PERMITTED  0xff
+/* enum: PF does not exist. */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_PF_NOT_PRESENT  0xfe
+/* enum: PF does exist but is not assigned to any external port. */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_PF_NOT_ASSIGNED  0xfd
+/* enum: This value indicates that PF is assigned, but it cannot be expressed
+ * in this field. It is intended for a possible future situation where a more
+ * complex scheme of PFs to ports mapping is being used. The future driver
+ * should look for a new field supporting the new scheme. The current/old
+ * driver should treat this value as PF_NOT_ASSIGNED.
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_INCOMPATIBLE_ASSIGNMENT  0xfc
+/* One byte per PF containing the number of its VFs, indexed by PF number. A
+ * special value indicates that a PF is not present.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_VFS_PER_PF_OFST 42
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_VFS_PER_PF_LEN 1
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_VFS_PER_PF_NUM 16
+/* enum: The caller is not permitted to access information on this PF. */
+/*               MC_CMD_GET_CAPABILITIES_V4_OUT_ACCESS_NOT_PERMITTED  0xff */
+/* enum: PF does not exist. */
+/*               MC_CMD_GET_CAPABILITIES_V4_OUT_PF_NOT_PRESENT  0xfe */
+/* Number of VIs available for each external port */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_VIS_PER_PORT_OFST 58
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_VIS_PER_PORT_LEN 2
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_VIS_PER_PORT_NUM 4
+/* Size of RX descriptor cache expressed as binary logarithm The actual size
+ * equals (2 ^ RX_DESC_CACHE_SIZE)
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_RX_DESC_CACHE_SIZE_OFST 66
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_RX_DESC_CACHE_SIZE_LEN 1
+/* Size of TX descriptor cache expressed as binary logarithm The actual size
+ * equals (2 ^ TX_DESC_CACHE_SIZE)
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TX_DESC_CACHE_SIZE_OFST 67
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_TX_DESC_CACHE_SIZE_LEN 1
+/* Total number of available PIO buffers */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_PIO_BUFFS_OFST 68
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_NUM_PIO_BUFFS_LEN 2
+/* Size of a single PIO buffer */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_SIZE_PIO_BUFF_OFST 70
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_SIZE_PIO_BUFF_LEN 2
+/* On chips later than Medford the amount of address space assigned to each VI
+ * is configurable. This is a global setting that the driver must query to
+ * discover the VI to address mapping. Cut-through PIO (CTPIO) is not available
+ * with 8k VI windows.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_VI_WINDOW_MODE_OFST 72
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_VI_WINDOW_MODE_LEN 1
+/* enum: Each VI occupies 8k as on Huntington and Medford. PIO is at offset 4k.
+ * CTPIO is not mapped.
+ */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_VI_WINDOW_MODE_8K   0x0
+/* enum: Each VI occupies 16k. PIO is at offset 4k. CTPIO is at offset 12k. */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_VI_WINDOW_MODE_16K  0x1
+/* enum: Each VI occupies 64k. PIO is at offset 4k. CTPIO is at offset 12k. */
+#define          MC_CMD_GET_CAPABILITIES_V4_OUT_VI_WINDOW_MODE_64K  0x2
+/* Number of vFIFOs per adapter that can be used for VFIFO Stuffing
+ * (SF-115995-SW) in the present configuration of firmware and port mode.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_VFIFO_STUFFING_NUM_VFIFOS_OFST 73
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_VFIFO_STUFFING_NUM_VFIFOS_LEN 1
+/* Number of buffers per adapter that can be used for VFIFO Stuffing
+ * (SF-115995-SW) in the present configuration of firmware and port mode.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_OFST 74
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_VFIFO_STUFFING_NUM_CP_BUFFERS_LEN 2
+/* Entry count in the MAC stats array, including the final GENERATION_END
+ * entry. For MAC stats DMA, drivers should allocate a buffer large enough to
+ * hold at least this many 64-bit stats values, if they wish to receive all
+ * available stats. If the buffer is shorter than MAC_STATS_NUM_STATS * 8, the
+ * stats array returned will be truncated.
+ */
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS_OFST 76
+#define       MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS_LEN 2
 
 
 /***********************************/
@@ -8502,6 +10219,7 @@
 #define    MC_CMD_TCM_BUCKET_ALLOC_OUT_LEN 4
 /* the bucket id */
 #define       MC_CMD_TCM_BUCKET_ALLOC_OUT_BUCKET_OFST 0
+#define       MC_CMD_TCM_BUCKET_ALLOC_OUT_BUCKET_LEN 4
 
 
 /***********************************/
@@ -8516,6 +10234,7 @@
 #define    MC_CMD_TCM_BUCKET_FREE_IN_LEN 4
 /* the bucket id */
 #define       MC_CMD_TCM_BUCKET_FREE_IN_BUCKET_OFST 0
+#define       MC_CMD_TCM_BUCKET_FREE_IN_BUCKET_LEN 4
 
 /* MC_CMD_TCM_BUCKET_FREE_OUT msgresponse */
 #define    MC_CMD_TCM_BUCKET_FREE_OUT_LEN 0
@@ -8533,17 +10252,22 @@
 #define    MC_CMD_TCM_BUCKET_INIT_IN_LEN 8
 /* the bucket id */
 #define       MC_CMD_TCM_BUCKET_INIT_IN_BUCKET_OFST 0
+#define       MC_CMD_TCM_BUCKET_INIT_IN_BUCKET_LEN 4
 /* the rate in mbps */
 #define       MC_CMD_TCM_BUCKET_INIT_IN_RATE_OFST 4
+#define       MC_CMD_TCM_BUCKET_INIT_IN_RATE_LEN 4
 
 /* MC_CMD_TCM_BUCKET_INIT_EXT_IN msgrequest */
 #define    MC_CMD_TCM_BUCKET_INIT_EXT_IN_LEN 12
 /* the bucket id */
 #define       MC_CMD_TCM_BUCKET_INIT_EXT_IN_BUCKET_OFST 0
+#define       MC_CMD_TCM_BUCKET_INIT_EXT_IN_BUCKET_LEN 4
 /* the rate in mbps */
 #define       MC_CMD_TCM_BUCKET_INIT_EXT_IN_RATE_OFST 4
+#define       MC_CMD_TCM_BUCKET_INIT_EXT_IN_RATE_LEN 4
 /* the desired maximum fill level */
 #define       MC_CMD_TCM_BUCKET_INIT_EXT_IN_MAX_FILL_OFST 8
+#define       MC_CMD_TCM_BUCKET_INIT_EXT_IN_MAX_FILL_LEN 4
 
 /* MC_CMD_TCM_BUCKET_INIT_OUT msgresponse */
 #define    MC_CMD_TCM_BUCKET_INIT_OUT_LEN 0
@@ -8561,10 +10285,13 @@
 #define    MC_CMD_TCM_TXQ_INIT_IN_LEN 28
 /* the txq id */
 #define       MC_CMD_TCM_TXQ_INIT_IN_QID_OFST 0
+#define       MC_CMD_TCM_TXQ_INIT_IN_QID_LEN 4
 /* the static priority associated with the txq */
 #define       MC_CMD_TCM_TXQ_INIT_IN_LABEL_OFST 4
+#define       MC_CMD_TCM_TXQ_INIT_IN_LABEL_LEN 4
 /* bitmask of the priority queues this txq is inserted into when inserted. */
 #define       MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAGS_OFST 8
+#define       MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAGS_LEN 4
 #define        MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_GUARANTEED_LBN 0
 #define        MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_GUARANTEED_WIDTH 1
 #define        MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_NORMAL_LBN 1
@@ -8573,25 +10300,32 @@
 #define        MC_CMD_TCM_TXQ_INIT_IN_PQ_FLAG_LOW_WIDTH 1
 /* the reaction point (RP) bucket */
 #define       MC_CMD_TCM_TXQ_INIT_IN_RP_BKT_OFST 12
+#define       MC_CMD_TCM_TXQ_INIT_IN_RP_BKT_LEN 4
 /* an already reserved bucket (typically set to bucket associated with outer
  * vswitch)
  */
 #define       MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT1_OFST 16
+#define       MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT1_LEN 4
 /* an already reserved bucket (typically set to bucket associated with inner
  * vswitch)
  */
 #define       MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT2_OFST 20
+#define       MC_CMD_TCM_TXQ_INIT_IN_MAX_BKT2_LEN 4
 /* the min bucket (typically for ETS/minimum bandwidth) */
 #define       MC_CMD_TCM_TXQ_INIT_IN_MIN_BKT_OFST 24
+#define       MC_CMD_TCM_TXQ_INIT_IN_MIN_BKT_LEN 4
 
 /* MC_CMD_TCM_TXQ_INIT_EXT_IN msgrequest */
 #define    MC_CMD_TCM_TXQ_INIT_EXT_IN_LEN 32
 /* the txq id */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_QID_OFST 0
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_QID_LEN 4
 /* the static priority associated with the txq */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_LABEL_NORMAL_OFST 4
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_LABEL_NORMAL_LEN 4
 /* bitmask of the priority queues this txq is inserted into when inserted. */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAGS_OFST 8
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAGS_LEN 4
 #define        MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_GUARANTEED_LBN 0
 #define        MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_GUARANTEED_WIDTH 1
 #define        MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_NORMAL_LBN 1
@@ -8600,18 +10334,23 @@
 #define        MC_CMD_TCM_TXQ_INIT_EXT_IN_PQ_FLAG_LOW_WIDTH 1
 /* the reaction point (RP) bucket */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_RP_BKT_OFST 12
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_RP_BKT_LEN 4
 /* an already reserved bucket (typically set to bucket associated with outer
  * vswitch)
  */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_MAX_BKT1_OFST 16
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_MAX_BKT1_LEN 4
 /* an already reserved bucket (typically set to bucket associated with inner
  * vswitch)
  */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_MAX_BKT2_OFST 20
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_MAX_BKT2_LEN 4
 /* the min bucket (typically for ETS/minimum bandwidth) */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_MIN_BKT_OFST 24
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_MIN_BKT_LEN 4
 /* the static priority associated with the txq */
 #define       MC_CMD_TCM_TXQ_INIT_EXT_IN_LABEL_GUARANTEED_OFST 28
+#define       MC_CMD_TCM_TXQ_INIT_EXT_IN_LABEL_GUARANTEED_LEN 4
 
 /* MC_CMD_TCM_TXQ_INIT_OUT msgresponse */
 #define    MC_CMD_TCM_TXQ_INIT_OUT_LEN 0
@@ -8629,8 +10368,10 @@
 #define    MC_CMD_LINK_PIOBUF_IN_LEN 8
 /* Handle for allocated push I/O buffer. */
 #define       MC_CMD_LINK_PIOBUF_IN_PIOBUF_HANDLE_OFST 0
+#define       MC_CMD_LINK_PIOBUF_IN_PIOBUF_HANDLE_LEN 4
 /* Function Local Instance (VI) number. */
 #define       MC_CMD_LINK_PIOBUF_IN_TXQ_INSTANCE_OFST 4
+#define       MC_CMD_LINK_PIOBUF_IN_TXQ_INSTANCE_LEN 4
 
 /* MC_CMD_LINK_PIOBUF_OUT msgresponse */
 #define    MC_CMD_LINK_PIOBUF_OUT_LEN 0
@@ -8648,6 +10389,7 @@
 #define    MC_CMD_UNLINK_PIOBUF_IN_LEN 4
 /* Function Local Instance (VI) number. */
 #define       MC_CMD_UNLINK_PIOBUF_IN_TXQ_INSTANCE_OFST 0
+#define       MC_CMD_UNLINK_PIOBUF_IN_TXQ_INSTANCE_LEN 4
 
 /* MC_CMD_UNLINK_PIOBUF_OUT msgresponse */
 #define    MC_CMD_UNLINK_PIOBUF_OUT_LEN 0
@@ -8665,8 +10407,10 @@
 #define    MC_CMD_VSWITCH_ALLOC_IN_LEN 16
 /* The port to connect to the v-switch's upstream port. */
 #define       MC_CMD_VSWITCH_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VSWITCH_ALLOC_IN_UPSTREAM_PORT_ID_LEN 4
 /* The type of v-switch to create. */
 #define       MC_CMD_VSWITCH_ALLOC_IN_TYPE_OFST 4
+#define       MC_CMD_VSWITCH_ALLOC_IN_TYPE_LEN 4
 /* enum: VLAN */
 #define          MC_CMD_VSWITCH_ALLOC_IN_VSWITCH_TYPE_VLAN  0x1
 /* enum: VEB */
@@ -8679,6 +10423,7 @@
 #define          MC_CMD_VSWITCH_ALLOC_IN_VSWITCH_TYPE_TEST  0x5
 /* Flags controlling v-port creation */
 #define       MC_CMD_VSWITCH_ALLOC_IN_FLAGS_OFST 8
+#define       MC_CMD_VSWITCH_ALLOC_IN_FLAGS_LEN 4
 #define        MC_CMD_VSWITCH_ALLOC_IN_FLAG_AUTO_PORT_LBN 0
 #define        MC_CMD_VSWITCH_ALLOC_IN_FLAG_AUTO_PORT_WIDTH 1
 /* The number of VLAN tags to allow for attached v-ports. For VLAN aggregators,
@@ -8689,6 +10434,7 @@
  * v-ports with this number of tags.
  */
 #define       MC_CMD_VSWITCH_ALLOC_IN_NUM_VLAN_TAGS_OFST 12
+#define       MC_CMD_VSWITCH_ALLOC_IN_NUM_VLAN_TAGS_LEN 4
 
 /* MC_CMD_VSWITCH_ALLOC_OUT msgresponse */
 #define    MC_CMD_VSWITCH_ALLOC_OUT_LEN 0
@@ -8706,6 +10452,7 @@
 #define    MC_CMD_VSWITCH_FREE_IN_LEN 4
 /* The port to which the v-switch is connected. */
 #define       MC_CMD_VSWITCH_FREE_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VSWITCH_FREE_IN_UPSTREAM_PORT_ID_LEN 4
 
 /* MC_CMD_VSWITCH_FREE_OUT msgresponse */
 #define    MC_CMD_VSWITCH_FREE_OUT_LEN 0
@@ -8725,6 +10472,7 @@
 #define    MC_CMD_VSWITCH_QUERY_IN_LEN 4
 /* The port to which the v-switch is connected. */
 #define       MC_CMD_VSWITCH_QUERY_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VSWITCH_QUERY_IN_UPSTREAM_PORT_ID_LEN 4
 
 /* MC_CMD_VSWITCH_QUERY_OUT msgresponse */
 #define    MC_CMD_VSWITCH_QUERY_OUT_LEN 0
@@ -8742,8 +10490,10 @@
 #define    MC_CMD_VPORT_ALLOC_IN_LEN 20
 /* The port to which the v-switch is connected. */
 #define       MC_CMD_VPORT_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VPORT_ALLOC_IN_UPSTREAM_PORT_ID_LEN 4
 /* The type of the new v-port. */
 #define       MC_CMD_VPORT_ALLOC_IN_TYPE_OFST 4
+#define       MC_CMD_VPORT_ALLOC_IN_TYPE_LEN 4
 /* enum: VLAN (obsolete) */
 #define          MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_VLAN  0x1
 /* enum: VEB (obsolete) */
@@ -8764,6 +10514,7 @@
 #define          MC_CMD_VPORT_ALLOC_IN_VPORT_TYPE_TEST  0x6
 /* Flags controlling v-port creation */
 #define       MC_CMD_VPORT_ALLOC_IN_FLAGS_OFST 8
+#define       MC_CMD_VPORT_ALLOC_IN_FLAGS_LEN 4
 #define        MC_CMD_VPORT_ALLOC_IN_FLAG_AUTO_PORT_LBN 0
 #define        MC_CMD_VPORT_ALLOC_IN_FLAG_AUTO_PORT_WIDTH 1
 #define        MC_CMD_VPORT_ALLOC_IN_FLAG_VLAN_RESTRICT_LBN 1
@@ -8773,8 +10524,10 @@
  * v-switch.
  */
 #define       MC_CMD_VPORT_ALLOC_IN_NUM_VLAN_TAGS_OFST 12
+#define       MC_CMD_VPORT_ALLOC_IN_NUM_VLAN_TAGS_LEN 4
 /* The actual VLAN tags to insert/remove */
 #define       MC_CMD_VPORT_ALLOC_IN_VLAN_TAGS_OFST 16
+#define       MC_CMD_VPORT_ALLOC_IN_VLAN_TAGS_LEN 4
 #define        MC_CMD_VPORT_ALLOC_IN_VLAN_TAG_0_LBN 0
 #define        MC_CMD_VPORT_ALLOC_IN_VLAN_TAG_0_WIDTH 16
 #define        MC_CMD_VPORT_ALLOC_IN_VLAN_TAG_1_LBN 16
@@ -8784,6 +10537,7 @@
 #define    MC_CMD_VPORT_ALLOC_OUT_LEN 4
 /* The handle of the new v-port */
 #define       MC_CMD_VPORT_ALLOC_OUT_VPORT_ID_OFST 0
+#define       MC_CMD_VPORT_ALLOC_OUT_VPORT_ID_LEN 4
 
 
 /***********************************/
@@ -8798,6 +10552,7 @@
 #define    MC_CMD_VPORT_FREE_IN_LEN 4
 /* The handle of the v-port */
 #define       MC_CMD_VPORT_FREE_IN_VPORT_ID_OFST 0
+#define       MC_CMD_VPORT_FREE_IN_VPORT_ID_LEN 4
 
 /* MC_CMD_VPORT_FREE_OUT msgresponse */
 #define    MC_CMD_VPORT_FREE_OUT_LEN 0
@@ -8815,18 +10570,23 @@
 #define    MC_CMD_VADAPTOR_ALLOC_IN_LEN 30
 /* The port to connect to the v-adaptor's port. */
 #define       MC_CMD_VADAPTOR_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VADAPTOR_ALLOC_IN_UPSTREAM_PORT_ID_LEN 4
 /* Flags controlling v-adaptor creation */
 #define       MC_CMD_VADAPTOR_ALLOC_IN_FLAGS_OFST 8
+#define       MC_CMD_VADAPTOR_ALLOC_IN_FLAGS_LEN 4
 #define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_AUTO_VADAPTOR_LBN 0
 #define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_AUTO_VADAPTOR_WIDTH 1
 #define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_LBN 1
 #define        MC_CMD_VADAPTOR_ALLOC_IN_FLAG_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED_WIDTH 1
 /* The number of VLAN tags to strip on receive */
 #define       MC_CMD_VADAPTOR_ALLOC_IN_NUM_VLANS_OFST 12
+#define       MC_CMD_VADAPTOR_ALLOC_IN_NUM_VLANS_LEN 4
 /* The number of VLAN tags to transparently insert/remove. */
 #define       MC_CMD_VADAPTOR_ALLOC_IN_NUM_VLAN_TAGS_OFST 16
+#define       MC_CMD_VADAPTOR_ALLOC_IN_NUM_VLAN_TAGS_LEN 4
 /* The actual VLAN tags to insert/remove */
 #define       MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAGS_OFST 20
+#define       MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAGS_LEN 4
 #define        MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAG_0_LBN 0
 #define        MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAG_0_WIDTH 16
 #define        MC_CMD_VADAPTOR_ALLOC_IN_VLAN_TAG_1_LBN 16
@@ -8853,6 +10613,7 @@
 #define    MC_CMD_VADAPTOR_FREE_IN_LEN 4
 /* The port to which the v-adaptor is connected. */
 #define       MC_CMD_VADAPTOR_FREE_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VADAPTOR_FREE_IN_UPSTREAM_PORT_ID_LEN 4
 
 /* MC_CMD_VADAPTOR_FREE_OUT msgresponse */
 #define    MC_CMD_VADAPTOR_FREE_OUT_LEN 0
@@ -8870,6 +10631,7 @@
 #define    MC_CMD_VADAPTOR_SET_MAC_IN_LEN 10
 /* The port to which the v-adaptor is connected. */
 #define       MC_CMD_VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID_LEN 4
 /* The new MAC address to assign to this v-adaptor */
 #define       MC_CMD_VADAPTOR_SET_MAC_IN_MACADDR_OFST 4
 #define       MC_CMD_VADAPTOR_SET_MAC_IN_MACADDR_LEN 6
@@ -8890,6 +10652,7 @@
 #define    MC_CMD_VADAPTOR_GET_MAC_IN_LEN 4
 /* The port to which the v-adaptor is connected. */
 #define       MC_CMD_VADAPTOR_GET_MAC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VADAPTOR_GET_MAC_IN_UPSTREAM_PORT_ID_LEN 4
 
 /* MC_CMD_VADAPTOR_GET_MAC_OUT msgresponse */
 #define    MC_CMD_VADAPTOR_GET_MAC_OUT_LEN 6
@@ -8910,15 +10673,19 @@
 #define    MC_CMD_VADAPTOR_QUERY_IN_LEN 4
 /* The port to which the v-adaptor is connected. */
 #define       MC_CMD_VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID_LEN 4
 
 /* MC_CMD_VADAPTOR_QUERY_OUT msgresponse */
 #define    MC_CMD_VADAPTOR_QUERY_OUT_LEN 12
 /* The EVB port flags as defined at MC_CMD_VPORT_ALLOC. */
 #define       MC_CMD_VADAPTOR_QUERY_OUT_PORT_FLAGS_OFST 0
+#define       MC_CMD_VADAPTOR_QUERY_OUT_PORT_FLAGS_LEN 4
 /* The v-adaptor flags as defined at MC_CMD_VADAPTOR_ALLOC. */
 #define       MC_CMD_VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS_OFST 4
+#define       MC_CMD_VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS_LEN 4
 /* The number of VLAN tags that may still be added */
 #define       MC_CMD_VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_OFST 8
+#define       MC_CMD_VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_LEN 4
 
 
 /***********************************/
@@ -8933,8 +10700,10 @@
 #define    MC_CMD_EVB_PORT_ASSIGN_IN_LEN 8
 /* The port to assign. */
 #define       MC_CMD_EVB_PORT_ASSIGN_IN_PORT_ID_OFST 0
+#define       MC_CMD_EVB_PORT_ASSIGN_IN_PORT_ID_LEN 4
 /* The target function to modify. */
 #define       MC_CMD_EVB_PORT_ASSIGN_IN_FUNCTION_OFST 4
+#define       MC_CMD_EVB_PORT_ASSIGN_IN_FUNCTION_LEN 4
 #define        MC_CMD_EVB_PORT_ASSIGN_IN_PF_LBN 0
 #define        MC_CMD_EVB_PORT_ASSIGN_IN_PF_WIDTH 16
 #define        MC_CMD_EVB_PORT_ASSIGN_IN_VF_LBN 16
@@ -8955,9 +10724,13 @@
 /* MC_CMD_RDWR_A64_REGIONS_IN msgrequest */
 #define    MC_CMD_RDWR_A64_REGIONS_IN_LEN 17
 #define       MC_CMD_RDWR_A64_REGIONS_IN_REGION0_OFST 0
+#define       MC_CMD_RDWR_A64_REGIONS_IN_REGION0_LEN 4
 #define       MC_CMD_RDWR_A64_REGIONS_IN_REGION1_OFST 4
+#define       MC_CMD_RDWR_A64_REGIONS_IN_REGION1_LEN 4
 #define       MC_CMD_RDWR_A64_REGIONS_IN_REGION2_OFST 8
+#define       MC_CMD_RDWR_A64_REGIONS_IN_REGION2_LEN 4
 #define       MC_CMD_RDWR_A64_REGIONS_IN_REGION3_OFST 12
+#define       MC_CMD_RDWR_A64_REGIONS_IN_REGION3_LEN 4
 /* Write enable bits 0-3, set to write, clear to read. */
 #define       MC_CMD_RDWR_A64_REGIONS_IN_WRITE_MASK_LBN 128
 #define       MC_CMD_RDWR_A64_REGIONS_IN_WRITE_MASK_WIDTH 4
@@ -8969,9 +10742,13 @@
  */
 #define    MC_CMD_RDWR_A64_REGIONS_OUT_LEN 16
 #define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION0_OFST 0
+#define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION0_LEN 4
 #define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION1_OFST 4
+#define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION1_LEN 4
 #define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION2_OFST 8
+#define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION2_LEN 4
 #define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION3_OFST 12
+#define       MC_CMD_RDWR_A64_REGIONS_OUT_REGION3_LEN 4
 
 
 /***********************************/
@@ -8986,11 +10763,13 @@
 #define    MC_CMD_ONLOAD_STACK_ALLOC_IN_LEN 4
 /* The handle of the owning upstream port */
 #define       MC_CMD_ONLOAD_STACK_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_ONLOAD_STACK_ALLOC_IN_UPSTREAM_PORT_ID_LEN 4
 
 /* MC_CMD_ONLOAD_STACK_ALLOC_OUT msgresponse */
 #define    MC_CMD_ONLOAD_STACK_ALLOC_OUT_LEN 4
 /* The handle of the new Onload stack */
 #define       MC_CMD_ONLOAD_STACK_ALLOC_OUT_ONLOAD_STACK_ID_OFST 0
+#define       MC_CMD_ONLOAD_STACK_ALLOC_OUT_ONLOAD_STACK_ID_LEN 4
 
 
 /***********************************/
@@ -9005,6 +10784,7 @@
 #define    MC_CMD_ONLOAD_STACK_FREE_IN_LEN 4
 /* The handle of the Onload stack */
 #define       MC_CMD_ONLOAD_STACK_FREE_IN_ONLOAD_STACK_ID_OFST 0
+#define       MC_CMD_ONLOAD_STACK_FREE_IN_ONLOAD_STACK_ID_LEN 4
 
 /* MC_CMD_ONLOAD_STACK_FREE_OUT msgresponse */
 #define    MC_CMD_ONLOAD_STACK_FREE_OUT_LEN 0
@@ -9022,8 +10802,10 @@
 #define    MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN 12
 /* The handle of the owning upstream port */
 #define       MC_CMD_RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID_LEN 4
 /* The type of context to allocate */
 #define       MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_OFST 4
+#define       MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_LEN 4
 /* enum: Allocate a context for exclusive use. The key and indirection table
  * must be explicitly configured.
  */
@@ -9037,6 +10819,7 @@
  * in the indirection table will be in the range 0 to NUM_QUEUES-1.
  */
 #define       MC_CMD_RSS_CONTEXT_ALLOC_IN_NUM_QUEUES_OFST 8
+#define       MC_CMD_RSS_CONTEXT_ALLOC_IN_NUM_QUEUES_LEN 4
 
 /* MC_CMD_RSS_CONTEXT_ALLOC_OUT msgresponse */
 #define    MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN 4
@@ -9045,6 +10828,7 @@
  * handle.
  */
 #define       MC_CMD_RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID_LEN 4
 /* enum: guaranteed invalid RSS context handle value */
 #define          MC_CMD_RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID_INVALID  0xffffffff
 
@@ -9061,6 +10845,7 @@
 #define    MC_CMD_RSS_CONTEXT_FREE_IN_LEN 4
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID_LEN 4
 
 /* MC_CMD_RSS_CONTEXT_FREE_OUT msgresponse */
 #define    MC_CMD_RSS_CONTEXT_FREE_OUT_LEN 0
@@ -9078,6 +10863,7 @@
 #define    MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN 44
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID_LEN 4
 /* The 40-byte Toeplitz hash key (TBD endianness issues?) */
 #define       MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_OFST 4
 #define       MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN 40
@@ -9098,6 +10884,7 @@
 #define    MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN 4
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID_LEN 4
 
 /* MC_CMD_RSS_CONTEXT_GET_KEY_OUT msgresponse */
 #define    MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN 44
@@ -9118,6 +10905,7 @@
 #define    MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN 132
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID_LEN 4
 /* The 128-byte indirection table (1 byte per entry) */
 #define       MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_OFST 4
 #define       MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN 128
@@ -9138,6 +10926,7 @@
 #define    MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN 4
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID_LEN 4
 
 /* MC_CMD_RSS_CONTEXT_GET_TABLE_OUT msgresponse */
 #define    MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN 132
@@ -9158,6 +10947,7 @@
 #define    MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN 8
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID_LEN 4
 /* Hash control flags. The _EN bits are always supported, but new modes are
  * available when ADDITIONAL_RSS_MODES is reported by MC_CMD_GET_CAPABILITIES:
  * in this case, the MODE fields may be set to non-zero values, and will take
@@ -9171,6 +10961,7 @@
  * particular packet type.)
  */
 #define       MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_FLAGS_OFST 4
+#define       MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_FLAGS_LEN 4
 #define        MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN_LBN 0
 #define        MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN_WIDTH 1
 #define        MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN_LBN 1
@@ -9210,6 +11001,7 @@
 #define    MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN 4
 /* The handle of the RSS context */
 #define       MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID_OFST 0
+#define       MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID_LEN 4
 
 /* MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT msgresponse */
 #define    MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN 8
@@ -9227,6 +11019,7 @@
  * always be used for a SET regardless of old/new driver vs. old/new firmware.
  */
 #define       MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST 4
+#define       MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_LEN 4
 #define        MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN 0
 #define        MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_WIDTH 1
 #define        MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN 1
@@ -9263,11 +11056,13 @@
 #define    MC_CMD_DOT1P_MAPPING_ALLOC_IN_LEN 8
 /* The handle of the owning upstream port */
 #define       MC_CMD_DOT1P_MAPPING_ALLOC_IN_UPSTREAM_PORT_ID_OFST 0
+#define       MC_CMD_DOT1P_MAPPING_ALLOC_IN_UPSTREAM_PORT_ID_LEN 4
 /* Number of queues spanned by this mapping, in the range 1-64; valid fixed
  * offsets in the mapping table will be in the range 0 to NUM_QUEUES-1, and
  * referenced RSS contexts must span no more than this number.
  */
 #define       MC_CMD_DOT1P_MAPPING_ALLOC_IN_NUM_QUEUES_OFST 4
+#define       MC_CMD_DOT1P_MAPPING_ALLOC_IN_NUM_QUEUES_LEN 4
 
 /* MC_CMD_DOT1P_MAPPING_ALLOC_OUT msgresponse */
 #define    MC_CMD_DOT1P_MAPPING_ALLOC_OUT_LEN 4
@@ -9276,6 +11071,7 @@
  * handle.
  */
 #define       MC_CMD_DOT1P_MAPPING_ALLOC_OUT_DOT1P_MAPPING_ID_OFST 0
+#define       MC_CMD_DOT1P_MAPPING_ALLOC_OUT_DOT1P_MAPPING_ID_LEN 4
 /* enum: guaranteed invalid .1p mapping handle value */
 #define          MC_CMD_DOT1P_MAPPING_ALLOC_OUT_DOT1P_MAPPING_ID_INVALID  0xffffffff
 
@@ -9292,6 +11088,7 @@
 #define    MC_CMD_DOT1P_MAPPING_FREE_IN_LEN 4
 /* The handle of the .1p mapping */
 #define       MC_CMD_DOT1P_MAPPING_FREE_IN_DOT1P_MAPPING_ID_OFST 0
+#define       MC_CMD_DOT1P_MAPPING_FREE_IN_DOT1P_MAPPING_ID_LEN 4
 
 /* MC_CMD_DOT1P_MAPPING_FREE_OUT msgresponse */
 #define    MC_CMD_DOT1P_MAPPING_FREE_OUT_LEN 0
@@ -9309,6 +11106,7 @@
 #define    MC_CMD_DOT1P_MAPPING_SET_TABLE_IN_LEN 36
 /* The handle of the .1p mapping */
 #define       MC_CMD_DOT1P_MAPPING_SET_TABLE_IN_DOT1P_MAPPING_ID_OFST 0
+#define       MC_CMD_DOT1P_MAPPING_SET_TABLE_IN_DOT1P_MAPPING_ID_LEN 4
 /* Per-priority mappings (1 32-bit word per entry - an offset or RSS context
  * handle)
  */
@@ -9331,6 +11129,7 @@
 #define    MC_CMD_DOT1P_MAPPING_GET_TABLE_IN_LEN 4
 /* The handle of the .1p mapping */
 #define       MC_CMD_DOT1P_MAPPING_GET_TABLE_IN_DOT1P_MAPPING_ID_OFST 0
+#define       MC_CMD_DOT1P_MAPPING_GET_TABLE_IN_DOT1P_MAPPING_ID_LEN 4
 
 /* MC_CMD_DOT1P_MAPPING_GET_TABLE_OUT msgresponse */
 #define    MC_CMD_DOT1P_MAPPING_GET_TABLE_OUT_LEN 36
@@ -9356,10 +11155,13 @@
 #define    MC_CMD_GET_VECTOR_CFG_OUT_LEN 12
 /* Base absolute interrupt vector number. */
 #define       MC_CMD_GET_VECTOR_CFG_OUT_VEC_BASE_OFST 0
+#define       MC_CMD_GET_VECTOR_CFG_OUT_VEC_BASE_LEN 4
 /* Number of interrupt vectors allocate to this PF. */
 #define       MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_PF_OFST 4
+#define       MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_PF_LEN 4
 /* Number of interrupt vectors to allocate per VF. */
 #define       MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_VF_OFST 8
+#define       MC_CMD_GET_VECTOR_CFG_OUT_VECS_PER_VF_LEN 4
 
 
 /***********************************/
@@ -9376,10 +11178,13 @@
  * let the system find a suitable base.
  */
 #define       MC_CMD_SET_VECTOR_CFG_IN_VEC_BASE_OFST 0
+#define       MC_CMD_SET_VECTOR_CFG_IN_VEC_BASE_LEN 4
 /* Number of interrupt vectors allocate to this PF. */
 #define       MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_PF_OFST 4
+#define       MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_PF_LEN 4
 /* Number of interrupt vectors to allocate per VF. */
 #define       MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_VF_OFST 8
+#define       MC_CMD_SET_VECTOR_CFG_IN_VECS_PER_VF_LEN 4
 
 /* MC_CMD_SET_VECTOR_CFG_OUT msgresponse */
 #define    MC_CMD_SET_VECTOR_CFG_OUT_LEN 0
@@ -9397,6 +11202,7 @@
 #define    MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_LEN 10
 /* The handle of the v-port */
 #define       MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_VPORT_ID_OFST 0
+#define       MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_VPORT_ID_LEN 4
 /* MAC address to add */
 #define       MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_MACADDR_OFST 4
 #define       MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_MACADDR_LEN 6
@@ -9417,6 +11223,7 @@
 #define    MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_LEN 10
 /* The handle of the v-port */
 #define       MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_VPORT_ID_OFST 0
+#define       MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_VPORT_ID_LEN 4
 /* MAC address to add */
 #define       MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_MACADDR_OFST 4
 #define       MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_MACADDR_LEN 6
@@ -9437,6 +11244,7 @@
 #define    MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN 4
 /* The handle of the v-port */
 #define       MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_VPORT_ID_OFST 0
+#define       MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_VPORT_ID_LEN 4
 
 /* MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT msgresponse */
 #define    MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMIN 4
@@ -9444,6 +11252,7 @@
 #define    MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LEN(num) (4+6*(num))
 /* The number of MAC addresses returned */
 #define       MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_COUNT_OFST 0
+#define       MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_COUNT_LEN 4
 /* Array of MAC addresses */
 #define       MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_OFST 4
 #define       MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_LEN 6
@@ -9465,8 +11274,10 @@
 #define    MC_CMD_VPORT_RECONFIGURE_IN_LEN 44
 /* The handle of the v-port */
 #define       MC_CMD_VPORT_RECONFIGURE_IN_VPORT_ID_OFST 0
+#define       MC_CMD_VPORT_RECONFIGURE_IN_VPORT_ID_LEN 4
 /* Flags requesting what should be changed. */
 #define       MC_CMD_VPORT_RECONFIGURE_IN_FLAGS_OFST 4
+#define       MC_CMD_VPORT_RECONFIGURE_IN_FLAGS_LEN 4
 #define        MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_VLAN_TAGS_LBN 0
 #define        MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_VLAN_TAGS_WIDTH 1
 #define        MC_CMD_VPORT_RECONFIGURE_IN_REPLACE_MACADDRS_LBN 1
@@ -9476,14 +11287,17 @@
  * v-switch.
  */
 #define       MC_CMD_VPORT_RECONFIGURE_IN_NUM_VLAN_TAGS_OFST 8
+#define       MC_CMD_VPORT_RECONFIGURE_IN_NUM_VLAN_TAGS_LEN 4
 /* The actual VLAN tags to insert/remove */
 #define       MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAGS_OFST 12
+#define       MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAGS_LEN 4
 #define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_0_LBN 0
 #define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_0_WIDTH 16
 #define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_1_LBN 16
 #define        MC_CMD_VPORT_RECONFIGURE_IN_VLAN_TAG_1_WIDTH 16
 /* The number of MAC addresses to add */
 #define       MC_CMD_VPORT_RECONFIGURE_IN_NUM_MACADDRS_OFST 16
+#define       MC_CMD_VPORT_RECONFIGURE_IN_NUM_MACADDRS_LEN 4
 /* MAC addresses to add */
 #define       MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_OFST 20
 #define       MC_CMD_VPORT_RECONFIGURE_IN_MACADDRS_LEN 6
@@ -9492,6 +11306,7 @@
 /* MC_CMD_VPORT_RECONFIGURE_OUT msgresponse */
 #define    MC_CMD_VPORT_RECONFIGURE_OUT_LEN 4
 #define       MC_CMD_VPORT_RECONFIGURE_OUT_FLAGS_OFST 0
+#define       MC_CMD_VPORT_RECONFIGURE_OUT_FLAGS_LEN 4
 #define        MC_CMD_VPORT_RECONFIGURE_OUT_RESET_DONE_LBN 0
 #define        MC_CMD_VPORT_RECONFIGURE_OUT_RESET_DONE_WIDTH 1
 
@@ -9508,15 +11323,18 @@
 #define    MC_CMD_EVB_PORT_QUERY_IN_LEN 4
 /* The handle of the v-port */
 #define       MC_CMD_EVB_PORT_QUERY_IN_PORT_ID_OFST 0
+#define       MC_CMD_EVB_PORT_QUERY_IN_PORT_ID_LEN 4
 
 /* MC_CMD_EVB_PORT_QUERY_OUT msgresponse */
 #define    MC_CMD_EVB_PORT_QUERY_OUT_LEN 8
 /* The EVB port flags as defined at MC_CMD_VPORT_ALLOC. */
 #define       MC_CMD_EVB_PORT_QUERY_OUT_PORT_FLAGS_OFST 0
+#define       MC_CMD_EVB_PORT_QUERY_OUT_PORT_FLAGS_LEN 4
 /* The number of VLAN tags that may be used on a v-adaptor connected to this
  * EVB port.
  */
 #define       MC_CMD_EVB_PORT_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_OFST 4
+#define       MC_CMD_EVB_PORT_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS_LEN 4
 
 
 /***********************************/
@@ -9528,14 +11346,16 @@
  */
 #define MC_CMD_DUMP_BUFTBL_ENTRIES 0xab
 
-#define MC_CMD_0xab_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xab_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_DUMP_BUFTBL_ENTRIES_IN msgrequest */
 #define    MC_CMD_DUMP_BUFTBL_ENTRIES_IN_LEN 8
 /* Index of the first buffer table entry. */
 #define       MC_CMD_DUMP_BUFTBL_ENTRIES_IN_FIRSTID_OFST 0
+#define       MC_CMD_DUMP_BUFTBL_ENTRIES_IN_FIRSTID_LEN 4
 /* Number of buffer table entries to dump. */
 #define       MC_CMD_DUMP_BUFTBL_ENTRIES_IN_NUMENTRIES_OFST 4
+#define       MC_CMD_DUMP_BUFTBL_ENTRIES_IN_NUMENTRIES_LEN 4
 
 /* MC_CMD_DUMP_BUFTBL_ENTRIES_OUT msgresponse */
 #define    MC_CMD_DUMP_BUFTBL_ENTRIES_OUT_LENMIN 12
@@ -9559,6 +11379,7 @@
 /* MC_CMD_SET_RXDP_CONFIG_IN msgrequest */
 #define    MC_CMD_SET_RXDP_CONFIG_IN_LEN 4
 #define       MC_CMD_SET_RXDP_CONFIG_IN_DATA_OFST 0
+#define       MC_CMD_SET_RXDP_CONFIG_IN_DATA_LEN 4
 #define        MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_DMA_LBN 0
 #define        MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_DMA_WIDTH 1
 #define        MC_CMD_SET_RXDP_CONFIG_IN_PAD_HOST_LEN_LBN 1
@@ -9588,6 +11409,7 @@
 /* MC_CMD_GET_RXDP_CONFIG_OUT msgresponse */
 #define    MC_CMD_GET_RXDP_CONFIG_OUT_LEN 4
 #define       MC_CMD_GET_RXDP_CONFIG_OUT_DATA_OFST 0
+#define       MC_CMD_GET_RXDP_CONFIG_OUT_DATA_LEN 4
 #define        MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_DMA_LBN 0
 #define        MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_DMA_WIDTH 1
 #define        MC_CMD_GET_RXDP_CONFIG_OUT_PAD_HOST_LEN_LBN 1
@@ -9611,8 +11433,10 @@
 #define    MC_CMD_GET_CLOCK_OUT_LEN 8
 /* System frequency, MHz */
 #define       MC_CMD_GET_CLOCK_OUT_SYS_FREQ_OFST 0
+#define       MC_CMD_GET_CLOCK_OUT_SYS_FREQ_LEN 4
 /* DPCPU frequency, MHz */
 #define       MC_CMD_GET_CLOCK_OUT_DPCPU_FREQ_OFST 4
+#define       MC_CMD_GET_CLOCK_OUT_DPCPU_FREQ_LEN 4
 
 
 /***********************************/
@@ -9621,36 +11445,43 @@
  */
 #define MC_CMD_SET_CLOCK 0xad
 
-#define MC_CMD_0xad_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xad_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_SET_CLOCK_IN msgrequest */
 #define    MC_CMD_SET_CLOCK_IN_LEN 28
 /* Requested frequency in MHz for system clock domain */
 #define       MC_CMD_SET_CLOCK_IN_SYS_FREQ_OFST 0
+#define       MC_CMD_SET_CLOCK_IN_SYS_FREQ_LEN 4
 /* enum: Leave the system clock domain frequency unchanged */
 #define          MC_CMD_SET_CLOCK_IN_SYS_DOMAIN_DONT_CHANGE  0x0
 /* Requested frequency in MHz for inter-core clock domain */
 #define       MC_CMD_SET_CLOCK_IN_ICORE_FREQ_OFST 4
+#define       MC_CMD_SET_CLOCK_IN_ICORE_FREQ_LEN 4
 /* enum: Leave the inter-core clock domain frequency unchanged */
 #define          MC_CMD_SET_CLOCK_IN_ICORE_DOMAIN_DONT_CHANGE  0x0
 /* Requested frequency in MHz for DPCPU clock domain */
 #define       MC_CMD_SET_CLOCK_IN_DPCPU_FREQ_OFST 8
+#define       MC_CMD_SET_CLOCK_IN_DPCPU_FREQ_LEN 4
 /* enum: Leave the DPCPU clock domain frequency unchanged */
 #define          MC_CMD_SET_CLOCK_IN_DPCPU_DOMAIN_DONT_CHANGE  0x0
 /* Requested frequency in MHz for PCS clock domain */
 #define       MC_CMD_SET_CLOCK_IN_PCS_FREQ_OFST 12
+#define       MC_CMD_SET_CLOCK_IN_PCS_FREQ_LEN 4
 /* enum: Leave the PCS clock domain frequency unchanged */
 #define          MC_CMD_SET_CLOCK_IN_PCS_DOMAIN_DONT_CHANGE  0x0
 /* Requested frequency in MHz for MC clock domain */
 #define       MC_CMD_SET_CLOCK_IN_MC_FREQ_OFST 16
+#define       MC_CMD_SET_CLOCK_IN_MC_FREQ_LEN 4
 /* enum: Leave the MC clock domain frequency unchanged */
 #define          MC_CMD_SET_CLOCK_IN_MC_DOMAIN_DONT_CHANGE  0x0
 /* Requested frequency in MHz for rmon clock domain */
 #define       MC_CMD_SET_CLOCK_IN_RMON_FREQ_OFST 20
+#define       MC_CMD_SET_CLOCK_IN_RMON_FREQ_LEN 4
 /* enum: Leave the rmon clock domain frequency unchanged */
 #define          MC_CMD_SET_CLOCK_IN_RMON_DOMAIN_DONT_CHANGE  0x0
 /* Requested frequency in MHz for vswitch clock domain */
 #define       MC_CMD_SET_CLOCK_IN_VSWITCH_FREQ_OFST 24
+#define       MC_CMD_SET_CLOCK_IN_VSWITCH_FREQ_LEN 4
 /* enum: Leave the vswitch clock domain frequency unchanged */
 #define          MC_CMD_SET_CLOCK_IN_VSWITCH_DOMAIN_DONT_CHANGE  0x0
 
@@ -9658,30 +11489,37 @@
 #define    MC_CMD_SET_CLOCK_OUT_LEN 28
 /* Resulting system frequency in MHz */
 #define       MC_CMD_SET_CLOCK_OUT_SYS_FREQ_OFST 0
+#define       MC_CMD_SET_CLOCK_OUT_SYS_FREQ_LEN 4
 /* enum: The system clock domain doesn't exist */
 #define          MC_CMD_SET_CLOCK_OUT_SYS_DOMAIN_UNSUPPORTED  0x0
 /* Resulting inter-core frequency in MHz */
 #define       MC_CMD_SET_CLOCK_OUT_ICORE_FREQ_OFST 4
+#define       MC_CMD_SET_CLOCK_OUT_ICORE_FREQ_LEN 4
 /* enum: The inter-core clock domain doesn't exist / isn't used */
 #define          MC_CMD_SET_CLOCK_OUT_ICORE_DOMAIN_UNSUPPORTED  0x0
 /* Resulting DPCPU frequency in MHz */
 #define       MC_CMD_SET_CLOCK_OUT_DPCPU_FREQ_OFST 8
+#define       MC_CMD_SET_CLOCK_OUT_DPCPU_FREQ_LEN 4
 /* enum: The dpcpu clock domain doesn't exist */
 #define          MC_CMD_SET_CLOCK_OUT_DPCPU_DOMAIN_UNSUPPORTED  0x0
 /* Resulting PCS frequency in MHz */
 #define       MC_CMD_SET_CLOCK_OUT_PCS_FREQ_OFST 12
+#define       MC_CMD_SET_CLOCK_OUT_PCS_FREQ_LEN 4
 /* enum: The PCS clock domain doesn't exist / isn't controlled */
 #define          MC_CMD_SET_CLOCK_OUT_PCS_DOMAIN_UNSUPPORTED  0x0
 /* Resulting MC frequency in MHz */
 #define       MC_CMD_SET_CLOCK_OUT_MC_FREQ_OFST 16
+#define       MC_CMD_SET_CLOCK_OUT_MC_FREQ_LEN 4
 /* enum: The MC clock domain doesn't exist / isn't controlled */
 #define          MC_CMD_SET_CLOCK_OUT_MC_DOMAIN_UNSUPPORTED  0x0
 /* Resulting rmon frequency in MHz */
 #define       MC_CMD_SET_CLOCK_OUT_RMON_FREQ_OFST 20
+#define       MC_CMD_SET_CLOCK_OUT_RMON_FREQ_LEN 4
 /* enum: The rmon clock domain doesn't exist / isn't controlled */
 #define          MC_CMD_SET_CLOCK_OUT_RMON_DOMAIN_UNSUPPORTED  0x0
 /* Resulting vswitch frequency in MHz */
 #define       MC_CMD_SET_CLOCK_OUT_VSWITCH_FREQ_OFST 24
+#define       MC_CMD_SET_CLOCK_OUT_VSWITCH_FREQ_LEN 4
 /* enum: The vswitch clock domain doesn't exist / isn't controlled */
 #define          MC_CMD_SET_CLOCK_OUT_VSWITCH_DOMAIN_UNSUPPORTED  0x0
 
@@ -9692,11 +11530,12 @@
  */
 #define MC_CMD_DPCPU_RPC 0xae
 
-#define MC_CMD_0xae_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xae_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_DPCPU_RPC_IN msgrequest */
 #define    MC_CMD_DPCPU_RPC_IN_LEN 36
 #define       MC_CMD_DPCPU_RPC_IN_CPU_OFST 0
+#define       MC_CMD_DPCPU_RPC_IN_CPU_LEN 4
 /* enum: RxDPCPU0 */
 #define          MC_CMD_DPCPU_RPC_IN_DPCPU_RX0  0x0
 /* enum: TxDPCPU0 */
@@ -9761,12 +11600,15 @@
 #define       MC_CMD_DPCPU_RPC_IN_WDATA_LEN 24
 /* Register data to write. Only valid in write/write-read. */
 #define       MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_DATA_OFST 16
+#define       MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_DATA_LEN 4
 /* Register address. */
 #define       MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_ADDRESS_OFST 20
+#define       MC_CMD_DPCPU_RPC_IN_CSR_ACCESS_ADDRESS_LEN 4
 
 /* MC_CMD_DPCPU_RPC_OUT msgresponse */
 #define    MC_CMD_DPCPU_RPC_OUT_LEN 36
 #define       MC_CMD_DPCPU_RPC_OUT_RC_OFST 0
+#define       MC_CMD_DPCPU_RPC_OUT_RC_LEN 4
 /* DATA */
 #define       MC_CMD_DPCPU_RPC_OUT_DATA_OFST 4
 #define       MC_CMD_DPCPU_RPC_OUT_DATA_LEN 32
@@ -9777,9 +11619,13 @@
 #define       MC_CMD_DPCPU_RPC_OUT_RDATA_OFST 12
 #define       MC_CMD_DPCPU_RPC_OUT_RDATA_LEN 24
 #define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_1_OFST 12
+#define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_1_LEN 4
 #define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_2_OFST 16
+#define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_2_LEN 4
 #define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_3_OFST 20
+#define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_3_LEN 4
 #define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_4_OFST 24
+#define       MC_CMD_DPCPU_RPC_OUT_CSR_ACCESS_READ_VAL_4_LEN 4
 
 
 /***********************************/
@@ -9794,6 +11640,7 @@
 #define    MC_CMD_TRIGGER_INTERRUPT_IN_LEN 4
 /* Interrupt level relative to base for function. */
 #define       MC_CMD_TRIGGER_INTERRUPT_IN_INTR_LEVEL_OFST 0
+#define       MC_CMD_TRIGGER_INTERRUPT_IN_INTR_LEVEL_LEN 4
 
 /* MC_CMD_TRIGGER_INTERRUPT_OUT msgresponse */
 #define    MC_CMD_TRIGGER_INTERRUPT_OUT_LEN 0
@@ -9811,6 +11658,7 @@
 #define    MC_CMD_SHMBOOT_OP_IN_LEN 4
 /* Identifies the operation to perform */
 #define       MC_CMD_SHMBOOT_OP_IN_SHMBOOT_OP_OFST 0
+#define       MC_CMD_SHMBOOT_OP_IN_SHMBOOT_OP_LEN 4
 /* enum: Copy slave_data section to the slave core. (Greenport only) */
 #define          MC_CMD_SHMBOOT_OP_IN_PUSH_SLAVE_DATA  0x0
 
@@ -9824,13 +11672,16 @@
  */
 #define MC_CMD_CAP_BLK_READ 0xe7
 
-#define MC_CMD_0xe7_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xe7_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_CAP_BLK_READ_IN msgrequest */
 #define    MC_CMD_CAP_BLK_READ_IN_LEN 12
 #define       MC_CMD_CAP_BLK_READ_IN_CAP_REG_OFST 0
+#define       MC_CMD_CAP_BLK_READ_IN_CAP_REG_LEN 4
 #define       MC_CMD_CAP_BLK_READ_IN_ADDR_OFST 4
+#define       MC_CMD_CAP_BLK_READ_IN_ADDR_LEN 4
 #define       MC_CMD_CAP_BLK_READ_IN_COUNT_OFST 8
+#define       MC_CMD_CAP_BLK_READ_IN_COUNT_LEN 4
 
 /* MC_CMD_CAP_BLK_READ_OUT msgresponse */
 #define    MC_CMD_CAP_BLK_READ_OUT_LENMIN 8
@@ -9850,53 +11701,77 @@
  */
 #define MC_CMD_DUMP_DO 0xe8
 
-#define MC_CMD_0xe8_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xe8_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_DUMP_DO_IN msgrequest */
 #define    MC_CMD_DUMP_DO_IN_LEN 52
 #define       MC_CMD_DUMP_DO_IN_PADDING_OFST 0
+#define       MC_CMD_DUMP_DO_IN_PADDING_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_OFST 4
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_LEN 4
 #define          MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM  0x0 /* enum */
 #define          MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_DEFAULT  0x1 /* enum */
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_TYPE_OFST 8
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_TYPE_LEN 4
 #define          MC_CMD_DUMP_DO_IN_DUMP_LOCATION_NVRAM  0x1 /* enum */
 #define          MC_CMD_DUMP_DO_IN_DUMP_LOCATION_HOST_MEMORY  0x2 /* enum */
 #define          MC_CMD_DUMP_DO_IN_DUMP_LOCATION_HOST_MEMORY_MLI  0x3 /* enum */
 #define          MC_CMD_DUMP_DO_IN_DUMP_LOCATION_UART  0x4 /* enum */
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 12
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_PARTITION_TYPE_ID_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_OFFSET_OFST 16
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_OFFSET_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 12
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_LO_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 16
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 12
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_LEN 4
 #define          MC_CMD_DUMP_DO_IN_HOST_MEMORY_MLI_PAGE_SIZE  0x1000 /* enum */
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 16
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 20
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_DEPTH_LEN 4
 #define          MC_CMD_DUMP_DO_IN_HOST_MEMORY_MLI_MAX_DEPTH  0x2 /* enum */
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_UART_PORT_OFST 12
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_UART_PORT_LEN 4
 /* enum: The uart port this command was received over (if using a uart
  * transport)
  */
 #define          MC_CMD_DUMP_DO_IN_UART_PORT_SRC  0xff
 #define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_SIZE_OFST 24
+#define       MC_CMD_DUMP_DO_IN_DUMPSPEC_SRC_CUSTOM_SIZE_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_OFST 28
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_LEN 4
 #define          MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM  0x0 /* enum */
 #define          MC_CMD_DUMP_DO_IN_DUMPFILE_DST_NVRAM_DUMP_PARTITION  0x1 /* enum */
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_TYPE_OFST 32
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC_CUSTOM_TYPE */
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 36
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_NVRAM_PARTITION_TYPE_ID_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_NVRAM_OFFSET_OFST 40
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_NVRAM_OFFSET_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 36
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_LO_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 40
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 36
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 40
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 44
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_DEPTH_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_UART_PORT_OFST 36
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_UART_PORT_LEN 4
 #define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_SIZE_OFST 48
+#define       MC_CMD_DUMP_DO_IN_DUMPFILE_DST_CUSTOM_SIZE_LEN 4
 
 /* MC_CMD_DUMP_DO_OUT msgresponse */
 #define    MC_CMD_DUMP_DO_OUT_LEN 4
 #define       MC_CMD_DUMP_DO_OUT_DUMPFILE_SIZE_OFST 0
+#define       MC_CMD_DUMP_DO_OUT_DUMPFILE_SIZE_LEN 4
 
 
 /***********************************/
@@ -9905,41 +11780,64 @@
  */
 #define MC_CMD_DUMP_CONFIGURE_UNSOLICITED 0xe9
 
-#define MC_CMD_0xe9_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xe9_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN msgrequest */
 #define    MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_LEN 52
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_ENABLE_OFST 0
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_ENABLE_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_OFST 4
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC */
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_TYPE_OFST 8
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC_CUSTOM_TYPE */
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 12
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_PARTITION_TYPE_ID_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_OFFSET_OFST 16
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_NVRAM_OFFSET_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 12
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_LO_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 16
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 12
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 16
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 20
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_HOST_MEMORY_MLI_DEPTH_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_UART_PORT_OFST 12
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_UART_PORT_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_SIZE_OFST 24
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPSPEC_SRC_CUSTOM_SIZE_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_OFST 28
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPFILE_DST */
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_TYPE_OFST 32
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_DUMP_DO/MC_CMD_DUMP_DO_IN/DUMPSPEC_SRC_CUSTOM_TYPE */
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_NVRAM_PARTITION_TYPE_ID_OFST 36
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_NVRAM_PARTITION_TYPE_ID_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_NVRAM_OFFSET_OFST 40
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_NVRAM_OFFSET_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_LO_OFST 36
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_LO_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_HI_OFST 40
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_OFST 36
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_LO_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_OFST 40
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_ROOT_ADDR_HI_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_DEPTH_OFST 44
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_HOST_MEMORY_MLI_DEPTH_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_UART_PORT_OFST 36
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_UART_PORT_LEN 4
 #define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_SIZE_OFST 48
+#define       MC_CMD_DUMP_CONFIGURE_UNSOLICITED_IN_DUMPFILE_DST_CUSTOM_SIZE_LEN 4
 
 
 /***********************************/
@@ -9950,17 +11848,20 @@
  */
 #define MC_CMD_SET_PSU 0xea
 
-#define MC_CMD_0xea_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xea_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_SET_PSU_IN msgrequest */
 #define    MC_CMD_SET_PSU_IN_LEN 12
 #define       MC_CMD_SET_PSU_IN_PARAM_OFST 0
+#define       MC_CMD_SET_PSU_IN_PARAM_LEN 4
 #define          MC_CMD_SET_PSU_IN_PARAM_SUPPLY_VOLTAGE  0x0 /* enum */
 #define       MC_CMD_SET_PSU_IN_RAIL_OFST 4
+#define       MC_CMD_SET_PSU_IN_RAIL_LEN 4
 #define          MC_CMD_SET_PSU_IN_RAIL_0V9  0x0 /* enum */
 #define          MC_CMD_SET_PSU_IN_RAIL_1V2  0x1 /* enum */
 /* desired value, eg voltage in mV */
 #define       MC_CMD_SET_PSU_IN_VALUE_OFST 8
+#define       MC_CMD_SET_PSU_IN_VALUE_LEN 4
 
 /* MC_CMD_SET_PSU_OUT msgresponse */
 #define    MC_CMD_SET_PSU_OUT_LEN 0
@@ -9980,7 +11881,9 @@
 /* MC_CMD_GET_FUNCTION_INFO_OUT msgresponse */
 #define    MC_CMD_GET_FUNCTION_INFO_OUT_LEN 8
 #define       MC_CMD_GET_FUNCTION_INFO_OUT_PF_OFST 0
+#define       MC_CMD_GET_FUNCTION_INFO_OUT_PF_LEN 4
 #define       MC_CMD_GET_FUNCTION_INFO_OUT_VF_OFST 4
+#define       MC_CMD_GET_FUNCTION_INFO_OUT_VF_LEN 4
 
 
 /***********************************/
@@ -10016,12 +11919,16 @@
 #define    MC_CMD_UART_SEND_DATA_OUT_LEN(num) (16+1*(num))
 /* CRC32 over OFFSET, LENGTH, RESERVED, DATA */
 #define       MC_CMD_UART_SEND_DATA_OUT_CHECKSUM_OFST 0
+#define       MC_CMD_UART_SEND_DATA_OUT_CHECKSUM_LEN 4
 /* Offset at which to write the data */
 #define       MC_CMD_UART_SEND_DATA_OUT_OFFSET_OFST 4
+#define       MC_CMD_UART_SEND_DATA_OUT_OFFSET_LEN 4
 /* Length of data */
 #define       MC_CMD_UART_SEND_DATA_OUT_LENGTH_OFST 8
+#define       MC_CMD_UART_SEND_DATA_OUT_LENGTH_LEN 4
 /* Reserved for future use */
 #define       MC_CMD_UART_SEND_DATA_OUT_RESERVED_OFST 12
+#define       MC_CMD_UART_SEND_DATA_OUT_RESERVED_LEN 4
 #define       MC_CMD_UART_SEND_DATA_OUT_DATA_OFST 16
 #define       MC_CMD_UART_SEND_DATA_OUT_DATA_LEN 1
 #define       MC_CMD_UART_SEND_DATA_OUT_DATA_MINNUM 0
@@ -10044,12 +11951,16 @@
 #define    MC_CMD_UART_RECV_DATA_OUT_LEN 16
 /* CRC32 over OFFSET, LENGTH, RESERVED */
 #define       MC_CMD_UART_RECV_DATA_OUT_CHECKSUM_OFST 0
+#define       MC_CMD_UART_RECV_DATA_OUT_CHECKSUM_LEN 4
 /* Offset from which to read the data */
 #define       MC_CMD_UART_RECV_DATA_OUT_OFFSET_OFST 4
+#define       MC_CMD_UART_RECV_DATA_OUT_OFFSET_LEN 4
 /* Length of data */
 #define       MC_CMD_UART_RECV_DATA_OUT_LENGTH_OFST 8
+#define       MC_CMD_UART_RECV_DATA_OUT_LENGTH_LEN 4
 /* Reserved for future use */
 #define       MC_CMD_UART_RECV_DATA_OUT_RESERVED_OFST 12
+#define       MC_CMD_UART_RECV_DATA_OUT_RESERVED_LEN 4
 
 /* MC_CMD_UART_RECV_DATA_IN msgresponse */
 #define    MC_CMD_UART_RECV_DATA_IN_LENMIN 16
@@ -10057,12 +11968,16 @@
 #define    MC_CMD_UART_RECV_DATA_IN_LEN(num) (16+1*(num))
 /* CRC32 over RESERVED1, RESERVED2, RESERVED3, DATA */
 #define       MC_CMD_UART_RECV_DATA_IN_CHECKSUM_OFST 0
+#define       MC_CMD_UART_RECV_DATA_IN_CHECKSUM_LEN 4
 /* Offset at which to write the data */
 #define       MC_CMD_UART_RECV_DATA_IN_RESERVED1_OFST 4
+#define       MC_CMD_UART_RECV_DATA_IN_RESERVED1_LEN 4
 /* Length of data */
 #define       MC_CMD_UART_RECV_DATA_IN_RESERVED2_OFST 8
+#define       MC_CMD_UART_RECV_DATA_IN_RESERVED2_LEN 4
 /* Reserved for future use */
 #define       MC_CMD_UART_RECV_DATA_IN_RESERVED3_OFST 12
+#define       MC_CMD_UART_RECV_DATA_IN_RESERVED3_LEN 4
 #define       MC_CMD_UART_RECV_DATA_IN_DATA_OFST 16
 #define       MC_CMD_UART_RECV_DATA_IN_DATA_LEN 1
 #define       MC_CMD_UART_RECV_DATA_IN_DATA_MINNUM 0
@@ -10075,14 +11990,16 @@
  */
 #define MC_CMD_READ_FUSES 0xf0
 
-#define MC_CMD_0xf0_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0xf0_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_READ_FUSES_IN msgrequest */
 #define    MC_CMD_READ_FUSES_IN_LEN 8
 /* Offset in OTP to read */
 #define       MC_CMD_READ_FUSES_IN_OFFSET_OFST 0
+#define       MC_CMD_READ_FUSES_IN_OFFSET_LEN 4
 /* Length of data to read in bytes */
 #define       MC_CMD_READ_FUSES_IN_LENGTH_OFST 4
+#define       MC_CMD_READ_FUSES_IN_LENGTH_LEN 4
 
 /* MC_CMD_READ_FUSES_OUT msgresponse */
 #define    MC_CMD_READ_FUSES_OUT_LENMIN 4
@@ -10090,6 +12007,7 @@
 #define    MC_CMD_READ_FUSES_OUT_LEN(num) (4+1*(num))
 /* Length of returned OTP data in bytes */
 #define       MC_CMD_READ_FUSES_OUT_LENGTH_OFST 0
+#define       MC_CMD_READ_FUSES_OUT_LENGTH_LEN 4
 /* Returned data */
 #define       MC_CMD_READ_FUSES_OUT_DATA_OFST 4
 #define       MC_CMD_READ_FUSES_OUT_DATA_LEN 1
@@ -10197,6 +12115,60 @@
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_EQC  0x9
 /* enum: CTLE EQ Resistor (0-7, Medford) */
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_EQRES  0xa
+/* enum: CTLE gain (0-31, Medford2) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_GAIN  0xb
+/* enum: CTLE pole (0-31, Medford2) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_POLE  0xc
+/* enum: CTLE peaking (0-31, Medford2) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CTLE_PEAK  0xd
+/* enum: DFE Tap1 - even path (Medford2 - 6 bit signed (-29 - +29)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP1_EVEN  0xe
+/* enum: DFE Tap1 - odd path (Medford2 - 6 bit signed (-29 - +29)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP1_ODD  0xf
+/* enum: DFE Tap2 (Medford2 - 6 bit signed (-20 - +20)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP2  0x10
+/* enum: DFE Tap3 (Medford2 - 6 bit signed (-20 - +20)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP3  0x11
+/* enum: DFE Tap4 (Medford2 - 6 bit signed (-20 - +20)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP4  0x12
+/* enum: DFE Tap5 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP5  0x13
+/* enum: DFE Tap6 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP6  0x14
+/* enum: DFE Tap7 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP7  0x15
+/* enum: DFE Tap8 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP8  0x16
+/* enum: DFE Tap9 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP9  0x17
+/* enum: DFE Tap10 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP10  0x18
+/* enum: DFE Tap11 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP11  0x19
+/* enum: DFE Tap12 (Medford2 - 6 bit signed (-24 - +24)) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_DFE_TAP12  0x1a
+/* enum: I/Q clk offset (Medford2 - 4 bit signed (-5 - +5))) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_IQ_OFF  0x1b
+/* enum: Negative h1 polarity data sampler offset calibration code, even path
+ * (Medford2 - 6 bit signed (-29 - +29)))
+ */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_H1N_OFF_EVEN  0x1c
+/* enum: Negative h1 polarity data sampler offset calibration code, odd path
+ * (Medford2 - 6 bit signed (-29 - +29)))
+ */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_H1N_OFF_ODD  0x1d
+/* enum: Positive h1 polarity data sampler offset calibration code, even path
+ * (Medford2 - 6 bit signed (-29 - +29)))
+ */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_H1P_OFF_EVEN  0x1e
+/* enum: Positive h1 polarity data sampler offset calibration code, odd path
+ * (Medford2 - 6 bit signed (-29 - +29)))
+ */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_H1P_OFF_ODD  0x1f
+/* enum: CDR calibration loop code (Medford2) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CDR_PVT  0x20
+/* enum: CDR integral loop code (Medford2) */
+#define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_CDR_INTEG  0x21
 #define        MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_LANE_LBN 8
 #define        MC_CMD_KR_TUNE_RXEQ_GET_OUT_PARAM_LANE_WIDTH 3
 #define          MC_CMD_KR_TUNE_RXEQ_GET_OUT_LANE_0  0x0 /* enum */
@@ -10268,7 +12240,7 @@
 #define       MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_MAXNUM 63
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_ID_LBN 0
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_ID_WIDTH 8
-/* enum: TX Amplitude (Huntington, Medford) */
+/* enum: TX Amplitude (Huntington, Medford, Medford2) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_LEV  0x0
 /* enum: De-Emphasis Tap1 Magnitude (0-7) (Huntington) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_MODE  0x1
@@ -10290,9 +12262,9 @@
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_RT_SET  0x9
 /* enum: TX Amplitude Fine control (Medford) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TX_LEV_FINE  0xa
-/* enum: Pre-shoot Tap (Medford) */
+/* enum: Pre-shoot Tap (Medford, Medford2) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TAP_ADV  0xb
-/* enum: De-emphasis Tap (Medford) */
+/* enum: De-emphasis Tap (Medford, Medford2) */
 #define          MC_CMD_KR_TUNE_TXEQ_GET_OUT_TAP_DLY  0xc
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_LANE_LBN 8
 #define        MC_CMD_KR_TUNE_TXEQ_GET_OUT_PARAM_LANE_WIDTH 3
@@ -10361,7 +12333,24 @@
 /* Align the arguments to 32 bits */
 #define       MC_CMD_KR_TUNE_START_EYE_PLOT_IN_KR_TUNE_RSVD_OFST 1
 #define       MC_CMD_KR_TUNE_START_EYE_PLOT_IN_KR_TUNE_RSVD_LEN 3
+/* Port-relative lane to scan eye on */
 #define       MC_CMD_KR_TUNE_START_EYE_PLOT_IN_LANE_OFST 4
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_IN_LANE_LEN 4
+
+/* MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN msgrequest */
+#define    MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_LEN 12
+/* Requested operation */
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_KR_TUNE_OP_OFST 0
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_KR_TUNE_OP_LEN 1
+/* Align the arguments to 32 bits */
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_KR_TUNE_RSVD_OFST 1
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_KR_TUNE_RSVD_LEN 3
+/* Port-relative lane to scan eye on */
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_LANE_OFST 4
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_LANE_LEN 4
+/* Scan duration / cycle count */
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_BER_OFST 8
+#define       MC_CMD_KR_TUNE_START_EYE_PLOT_V2_IN_BER_LEN 4
 
 /* MC_CMD_KR_TUNE_START_EYE_PLOT_OUT msgresponse */
 #define    MC_CMD_KR_TUNE_START_EYE_PLOT_OUT_LEN 0
@@ -10393,10 +12382,12 @@
 #define       MC_CMD_KR_TUNE_READ_FOM_IN_KR_TUNE_RSVD_OFST 1
 #define       MC_CMD_KR_TUNE_READ_FOM_IN_KR_TUNE_RSVD_LEN 3
 #define       MC_CMD_KR_TUNE_READ_FOM_IN_LANE_OFST 4
+#define       MC_CMD_KR_TUNE_READ_FOM_IN_LANE_LEN 4
 
 /* MC_CMD_KR_TUNE_READ_FOM_OUT msgresponse */
 #define    MC_CMD_KR_TUNE_READ_FOM_OUT_LEN 4
 #define       MC_CMD_KR_TUNE_READ_FOM_OUT_FOM_OFST 0
+#define       MC_CMD_KR_TUNE_READ_FOM_OUT_FOM_LEN 4
 
 
 /***********************************/
@@ -10594,6 +12585,7 @@
 #define       MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_PCIE_TUNE_RSVD_OFST 1
 #define       MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_PCIE_TUNE_RSVD_LEN 3
 #define       MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_LANE_OFST 4
+#define       MC_CMD_PCIE_TUNE_START_EYE_PLOT_IN_LANE_LEN 4
 
 /* MC_CMD_PCIE_TUNE_START_EYE_PLOT_OUT msgresponse */
 #define    MC_CMD_PCIE_TUNE_START_EYE_PLOT_OUT_LEN 0
@@ -10636,6 +12628,7 @@
 #define    MC_CMD_LICENSING_IN_LEN 4
 /* identifies the type of operation requested */
 #define       MC_CMD_LICENSING_IN_OP_OFST 0
+#define       MC_CMD_LICENSING_IN_OP_LEN 4
 /* enum: re-read and apply licenses after a license key partition update; note
  * that this operation returns a zero-length response
  */
@@ -10647,23 +12640,30 @@
 #define    MC_CMD_LICENSING_OUT_LEN 28
 /* count of application keys which are valid */
 #define       MC_CMD_LICENSING_OUT_VALID_APP_KEYS_OFST 0
+#define       MC_CMD_LICENSING_OUT_VALID_APP_KEYS_LEN 4
 /* sum of UNVERIFIABLE_APP_KEYS + WRONG_NODE_APP_KEYS (for compatibility with
  * MC_CMD_FC_OP_LICENSE)
  */
 #define       MC_CMD_LICENSING_OUT_INVALID_APP_KEYS_OFST 4
+#define       MC_CMD_LICENSING_OUT_INVALID_APP_KEYS_LEN 4
 /* count of application keys which are invalid due to being blacklisted */
 #define       MC_CMD_LICENSING_OUT_BLACKLISTED_APP_KEYS_OFST 8
+#define       MC_CMD_LICENSING_OUT_BLACKLISTED_APP_KEYS_LEN 4
 /* count of application keys which are invalid due to being unverifiable */
 #define       MC_CMD_LICENSING_OUT_UNVERIFIABLE_APP_KEYS_OFST 12
+#define       MC_CMD_LICENSING_OUT_UNVERIFIABLE_APP_KEYS_LEN 4
 /* count of application keys which are invalid due to being for the wrong node
  */
 #define       MC_CMD_LICENSING_OUT_WRONG_NODE_APP_KEYS_OFST 16
+#define       MC_CMD_LICENSING_OUT_WRONG_NODE_APP_KEYS_LEN 4
 /* licensing state (for diagnostics; the exact meaning of the bits in this
  * field are private to the firmware)
  */
 #define       MC_CMD_LICENSING_OUT_LICENSING_STATE_OFST 20
+#define       MC_CMD_LICENSING_OUT_LICENSING_STATE_LEN 4
 /* licensing subsystem self-test report (for manftest) */
 #define       MC_CMD_LICENSING_OUT_LICENSING_SELF_TEST_OFST 24
+#define       MC_CMD_LICENSING_OUT_LICENSING_SELF_TEST_LEN 4
 /* enum: licensing subsystem self-test failed */
 #define          MC_CMD_LICENSING_OUT_SELF_TEST_FAIL  0x0
 /* enum: licensing subsystem self-test passed */
@@ -10683,6 +12683,7 @@
 #define    MC_CMD_LICENSING_V3_IN_LEN 4
 /* identifies the type of operation requested */
 #define       MC_CMD_LICENSING_V3_IN_OP_OFST 0
+#define       MC_CMD_LICENSING_V3_IN_OP_LEN 4
 /* enum: re-read and apply licenses after a license key partition update; note
  * that this operation returns a zero-length response
  */
@@ -10696,20 +12697,26 @@
 #define    MC_CMD_LICENSING_V3_OUT_LEN 88
 /* count of keys which are valid */
 #define       MC_CMD_LICENSING_V3_OUT_VALID_KEYS_OFST 0
+#define       MC_CMD_LICENSING_V3_OUT_VALID_KEYS_LEN 4
 /* sum of UNVERIFIABLE_KEYS + WRONG_NODE_KEYS (for compatibility with
  * MC_CMD_FC_OP_LICENSE)
  */
 #define       MC_CMD_LICENSING_V3_OUT_INVALID_KEYS_OFST 4
+#define       MC_CMD_LICENSING_V3_OUT_INVALID_KEYS_LEN 4
 /* count of keys which are invalid due to being unverifiable */
 #define       MC_CMD_LICENSING_V3_OUT_UNVERIFIABLE_KEYS_OFST 8
+#define       MC_CMD_LICENSING_V3_OUT_UNVERIFIABLE_KEYS_LEN 4
 /* count of keys which are invalid due to being for the wrong node */
 #define       MC_CMD_LICENSING_V3_OUT_WRONG_NODE_KEYS_OFST 12
+#define       MC_CMD_LICENSING_V3_OUT_WRONG_NODE_KEYS_LEN 4
 /* licensing state (for diagnostics; the exact meaning of the bits in this
  * field are private to the firmware)
  */
 #define       MC_CMD_LICENSING_V3_OUT_LICENSING_STATE_OFST 16
+#define       MC_CMD_LICENSING_V3_OUT_LICENSING_STATE_LEN 4
 /* licensing subsystem self-test report (for manftest) */
 #define       MC_CMD_LICENSING_V3_OUT_LICENSING_SELF_TEST_OFST 20
+#define       MC_CMD_LICENSING_V3_OUT_LICENSING_SELF_TEST_LEN 4
 /* enum: licensing subsystem self-test failed */
 #define          MC_CMD_LICENSING_V3_OUT_SELF_TEST_FAIL  0x0
 /* enum: licensing subsystem self-test passed */
@@ -10750,8 +12757,10 @@
 #define    MC_CMD_LICENSING_GET_ID_V3_OUT_LEN(num) (8+1*(num))
 /* type of license (eg 3) */
 #define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_TYPE_OFST 0
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_TYPE_LEN 4
 /* length of the license ID (in bytes) */
 #define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_LENGTH_OFST 4
+#define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_LENGTH_LEN 4
 /* the unique license ID of the adapter */
 #define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_OFST 8
 #define       MC_CMD_LICENSING_GET_ID_V3_OUT_LICENSE_ID_LEN 1
@@ -10789,11 +12798,13 @@
 #define    MC_CMD_GET_LICENSED_APP_STATE_IN_LEN 4
 /* application ID to query (LICENSED_APP_ID_xxx) */
 #define       MC_CMD_GET_LICENSED_APP_STATE_IN_APP_ID_OFST 0
+#define       MC_CMD_GET_LICENSED_APP_STATE_IN_APP_ID_LEN 4
 
 /* MC_CMD_GET_LICENSED_APP_STATE_OUT msgresponse */
 #define    MC_CMD_GET_LICENSED_APP_STATE_OUT_LEN 4
 /* state of this application */
 #define       MC_CMD_GET_LICENSED_APP_STATE_OUT_STATE_OFST 0
+#define       MC_CMD_GET_LICENSED_APP_STATE_OUT_STATE_LEN 4
 /* enum: no (or invalid) license is present for the application */
 #define          MC_CMD_GET_LICENSED_APP_STATE_OUT_NOT_LICENSED  0x0
 /* enum: a valid license is present for the application */
@@ -10824,6 +12835,7 @@
 #define    MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_LEN 4
 /* state of this application */
 #define       MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_STATE_OFST 0
+#define       MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_STATE_LEN 4
 /* enum: no (or invalid) license is present for the application */
 #define          MC_CMD_GET_LICENSED_V3_APP_STATE_OUT_NOT_LICENSED  0x0
 /* enum: a valid license is present for the application */
@@ -10874,8 +12886,10 @@
 #define    MC_CMD_LICENSED_APP_OP_IN_LEN(num) (8+4*(num))
 /* application ID */
 #define       MC_CMD_LICENSED_APP_OP_IN_APP_ID_OFST 0
+#define       MC_CMD_LICENSED_APP_OP_IN_APP_ID_LEN 4
 /* the type of operation requested */
 #define       MC_CMD_LICENSED_APP_OP_IN_OP_OFST 4
+#define       MC_CMD_LICENSED_APP_OP_IN_OP_LEN 4
 /* enum: validate application */
 #define          MC_CMD_LICENSED_APP_OP_IN_OP_VALIDATE  0x0
 /* enum: mask application */
@@ -10900,8 +12914,10 @@
 #define    MC_CMD_LICENSED_APP_OP_VALIDATE_IN_LEN 72
 /* application ID */
 #define       MC_CMD_LICENSED_APP_OP_VALIDATE_IN_APP_ID_OFST 0
+#define       MC_CMD_LICENSED_APP_OP_VALIDATE_IN_APP_ID_LEN 4
 /* the type of operation requested */
 #define       MC_CMD_LICENSED_APP_OP_VALIDATE_IN_OP_OFST 4
+#define       MC_CMD_LICENSED_APP_OP_VALIDATE_IN_OP_LEN 4
 /* validation challenge */
 #define       MC_CMD_LICENSED_APP_OP_VALIDATE_IN_CHALLENGE_OFST 8
 #define       MC_CMD_LICENSED_APP_OP_VALIDATE_IN_CHALLENGE_LEN 64
@@ -10910,6 +12926,7 @@
 #define    MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_LEN 68
 /* feature expiry (time_t) */
 #define       MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_EXPIRY_OFST 0
+#define       MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_EXPIRY_LEN 4
 /* validation response */
 #define       MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_RESPONSE_OFST 4
 #define       MC_CMD_LICENSED_APP_OP_VALIDATE_OUT_RESPONSE_LEN 64
@@ -10918,10 +12935,13 @@
 #define    MC_CMD_LICENSED_APP_OP_MASK_IN_LEN 12
 /* application ID */
 #define       MC_CMD_LICENSED_APP_OP_MASK_IN_APP_ID_OFST 0
+#define       MC_CMD_LICENSED_APP_OP_MASK_IN_APP_ID_LEN 4
 /* the type of operation requested */
 #define       MC_CMD_LICENSED_APP_OP_MASK_IN_OP_OFST 4
+#define       MC_CMD_LICENSED_APP_OP_MASK_IN_OP_LEN 4
 /* flag */
 #define       MC_CMD_LICENSED_APP_OP_MASK_IN_FLAG_OFST 8
+#define       MC_CMD_LICENSED_APP_OP_MASK_IN_FLAG_LEN 4
 
 /* MC_CMD_LICENSED_APP_OP_MASK_OUT msgresponse */
 #define    MC_CMD_LICENSED_APP_OP_MASK_OUT_LEN 0
@@ -10959,8 +12979,10 @@
 #define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_RESPONSE_LEN 96
 /* application expiry time */
 #define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_TIME_OFST 96
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_TIME_LEN 4
 /* application expiry units */
 #define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNITS_OFST 100
+#define       MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNITS_LEN 4
 /* enum: expiry units are accounting units */
 #define          MC_CMD_LICENSED_V3_VALIDATE_APP_OUT_EXPIRY_UNIT_ACC  0x0
 /* enum: expiry units are calendar days */
@@ -10984,7 +13006,7 @@
  */
 #define MC_CMD_LICENSED_V3_MASK_FEATURES 0xd5
 
-#define MC_CMD_0xd5_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+#define MC_CMD_0xd5_PRIVILEGE_CTG SRIOV_CTG_ADMIN
 
 /* MC_CMD_LICENSED_V3_MASK_FEATURES_IN msgrequest */
 #define    MC_CMD_LICENSED_V3_MASK_FEATURES_IN_LEN 12
@@ -10995,6 +13017,7 @@
 #define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_MASK_HI_OFST 4
 /* whether to turn on or turn off the masked features */
 #define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_FLAG_OFST 8
+#define       MC_CMD_LICENSED_V3_MASK_FEATURES_IN_FLAG_LEN 4
 /* enum: turn the features off */
 #define          MC_CMD_LICENSED_V3_MASK_FEATURES_IN_OFF  0x0
 /* enum: turn the features back on */
@@ -11014,12 +13037,13 @@
  */
 #define MC_CMD_LICENSING_V3_TEMPORARY 0xd6
 
-#define MC_CMD_0xd6_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+#define MC_CMD_0xd6_PRIVILEGE_CTG SRIOV_CTG_ADMIN
 
 /* MC_CMD_LICENSING_V3_TEMPORARY_IN msgrequest */
 #define    MC_CMD_LICENSING_V3_TEMPORARY_IN_LEN 4
 /* operation code */
 #define       MC_CMD_LICENSING_V3_TEMPORARY_IN_OP_OFST 0
+#define       MC_CMD_LICENSING_V3_TEMPORARY_IN_OP_LEN 4
 /* enum: install a new license, overwriting any existing temporary license.
  * This is an asynchronous operation owing to the time taken to validate an
  * ECDSA license
@@ -11037,6 +13061,7 @@
 /* MC_CMD_LICENSING_V3_TEMPORARY_IN_SET msgrequest */
 #define    MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_LEN 164
 #define       MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_OP_OFST 0
+#define       MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_OP_LEN 4
 /* ECDSA license and signature */
 #define       MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_LICENSE_OFST 4
 #define       MC_CMD_LICENSING_V3_TEMPORARY_IN_SET_LICENSE_LEN 160
@@ -11044,15 +13069,18 @@
 /* MC_CMD_LICENSING_V3_TEMPORARY_IN_CLEAR msgrequest */
 #define    MC_CMD_LICENSING_V3_TEMPORARY_IN_CLEAR_LEN 4
 #define       MC_CMD_LICENSING_V3_TEMPORARY_IN_CLEAR_OP_OFST 0
+#define       MC_CMD_LICENSING_V3_TEMPORARY_IN_CLEAR_OP_LEN 4
 
 /* MC_CMD_LICENSING_V3_TEMPORARY_IN_STATUS msgrequest */
 #define    MC_CMD_LICENSING_V3_TEMPORARY_IN_STATUS_LEN 4
 #define       MC_CMD_LICENSING_V3_TEMPORARY_IN_STATUS_OP_OFST 0
+#define       MC_CMD_LICENSING_V3_TEMPORARY_IN_STATUS_OP_LEN 4
 
 /* MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS msgresponse */
 #define    MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_LEN 12
 /* status code */
 #define       MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_STATUS_OFST 0
+#define       MC_CMD_LICENSING_V3_TEMPORARY_OUT_STATUS_STATUS_LEN 4
 /* enum: finished validating and installing license */
 #define          MC_CMD_LICENSING_V3_TEMPORARY_STATUS_OK  0x0
 /* enum: license validation and installation in progress */
@@ -11084,14 +13112,17 @@
 #define    MC_CMD_SET_PORT_SNIFF_CONFIG_IN_LEN 16
 /* configuration flags */
 #define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_FLAGS_OFST 0
+#define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_FLAGS_LEN 4
 #define        MC_CMD_SET_PORT_SNIFF_CONFIG_IN_ENABLE_LBN 0
 #define        MC_CMD_SET_PORT_SNIFF_CONFIG_IN_ENABLE_WIDTH 1
 #define        MC_CMD_SET_PORT_SNIFF_CONFIG_IN_PROMISCUOUS_LBN 1
 #define        MC_CMD_SET_PORT_SNIFF_CONFIG_IN_PROMISCUOUS_WIDTH 1
 /* receive queue handle (for RSS mode, this is the base queue) */
 #define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_QUEUE_OFST 4
+#define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_QUEUE_LEN 4
 /* receive mode */
 #define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_MODE_OFST 8
+#define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_MODE_LEN 4
 /* enum: receive to just the specified queue */
 #define          MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_MODE_SIMPLE  0x0
 /* enum: receive to multiple queues using RSS context */
@@ -11101,6 +13132,7 @@
  * of 0xFFFFFFFF is guaranteed never to be a valid handle.
  */
 #define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_CONTEXT_OFST 12
+#define       MC_CMD_SET_PORT_SNIFF_CONFIG_IN_RX_CONTEXT_LEN 4
 
 /* MC_CMD_SET_PORT_SNIFF_CONFIG_OUT msgresponse */
 #define    MC_CMD_SET_PORT_SNIFF_CONFIG_OUT_LEN 0
@@ -11123,20 +13155,24 @@
 #define    MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_LEN 16
 /* configuration flags */
 #define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_FLAGS_OFST 0
+#define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_FLAGS_LEN 4
 #define        MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_ENABLE_LBN 0
 #define        MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_ENABLE_WIDTH 1
 #define        MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_PROMISCUOUS_LBN 1
 #define        MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_PROMISCUOUS_WIDTH 1
 /* receiving queue handle (for RSS mode, this is the base queue) */
 #define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_QUEUE_OFST 4
+#define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_QUEUE_LEN 4
 /* receive mode */
 #define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_MODE_OFST 8
+#define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_MODE_LEN 4
 /* enum: receiving to just the specified queue */
 #define          MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_MODE_SIMPLE  0x0
 /* enum: receiving to multiple queues using RSS context */
 #define          MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_MODE_RSS  0x1
 /* RSS context (for RX_MODE_RSS) */
 #define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_CONTEXT_OFST 12
+#define       MC_CMD_GET_PORT_SNIFF_CONFIG_OUT_RX_CONTEXT_LEN 4
 
 
 /***********************************/
@@ -11153,6 +13189,7 @@
 #define    MC_CMD_SET_PARSER_DISP_CONFIG_IN_LEN(num) (8+4*(num))
 /* the type of configuration setting to change */
 #define       MC_CMD_SET_PARSER_DISP_CONFIG_IN_TYPE_OFST 0
+#define       MC_CMD_SET_PARSER_DISP_CONFIG_IN_TYPE_LEN 4
 /* enum: Per-TXQ enable for multicast UDP destination lookup for possible
  * internal loopback. (ENTITY is a queue handle, VALUE is a single boolean.)
  */
@@ -11166,6 +13203,7 @@
  * on the type of configuration setting being changed
  */
 #define       MC_CMD_SET_PARSER_DISP_CONFIG_IN_ENTITY_OFST 4
+#define       MC_CMD_SET_PARSER_DISP_CONFIG_IN_ENTITY_LEN 4
 /* new value: the details depend on the type of configuration setting being
  * changed
  */
@@ -11190,12 +13228,14 @@
 #define    MC_CMD_GET_PARSER_DISP_CONFIG_IN_LEN 8
 /* the type of configuration setting to read */
 #define       MC_CMD_GET_PARSER_DISP_CONFIG_IN_TYPE_OFST 0
+#define       MC_CMD_GET_PARSER_DISP_CONFIG_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_SET_PARSER_DISP_CONFIG/MC_CMD_SET_PARSER_DISP_CONFIG_IN/TYPE */
 /* handle for the entity to query: queue handle, EVB port ID, etc. depending on
  * the type of configuration setting being read
  */
 #define       MC_CMD_GET_PARSER_DISP_CONFIG_IN_ENTITY_OFST 4
+#define       MC_CMD_GET_PARSER_DISP_CONFIG_IN_ENTITY_LEN 4
 
 /* MC_CMD_GET_PARSER_DISP_CONFIG_OUT msgresponse */
 #define    MC_CMD_GET_PARSER_DISP_CONFIG_OUT_LENMIN 4
@@ -11228,12 +13268,15 @@
 #define    MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_LEN 16
 /* configuration flags */
 #define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_FLAGS_OFST 0
+#define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_FLAGS_LEN 4
 #define        MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_ENABLE_LBN 0
 #define        MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_ENABLE_WIDTH 1
 /* receive queue handle (for RSS mode, this is the base queue) */
 #define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_QUEUE_OFST 4
+#define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_QUEUE_LEN 4
 /* receive mode */
 #define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_MODE_OFST 8
+#define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_MODE_LEN 4
 /* enum: receive to just the specified queue */
 #define          MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_MODE_SIMPLE  0x0
 /* enum: receive to multiple queues using RSS context */
@@ -11243,6 +13286,7 @@
  * of 0xFFFFFFFF is guaranteed never to be a valid handle.
  */
 #define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_CONTEXT_OFST 12
+#define       MC_CMD_SET_TX_PORT_SNIFF_CONFIG_IN_RX_CONTEXT_LEN 4
 
 /* MC_CMD_SET_TX_PORT_SNIFF_CONFIG_OUT msgresponse */
 #define    MC_CMD_SET_TX_PORT_SNIFF_CONFIG_OUT_LEN 0
@@ -11265,18 +13309,22 @@
 #define    MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_LEN 16
 /* configuration flags */
 #define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_FLAGS_OFST 0
+#define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_FLAGS_LEN 4
 #define        MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_ENABLE_LBN 0
 #define        MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_ENABLE_WIDTH 1
 /* receiving queue handle (for RSS mode, this is the base queue) */
 #define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_QUEUE_OFST 4
+#define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_QUEUE_LEN 4
 /* receive mode */
 #define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_MODE_OFST 8
+#define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_MODE_LEN 4
 /* enum: receiving to just the specified queue */
 #define          MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_MODE_SIMPLE  0x0
 /* enum: receiving to multiple queues using RSS context */
 #define          MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_MODE_RSS  0x1
 /* RSS context (for RX_MODE_RSS) */
 #define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_CONTEXT_OFST 12
+#define       MC_CMD_GET_TX_PORT_SNIFF_CONFIG_OUT_RX_CONTEXT_LEN 4
 
 
 /***********************************/
@@ -11291,16 +13339,22 @@
 #define    MC_CMD_RMON_STATS_RX_ERRORS_IN_LEN 8
 /* The rx queue to get stats for. */
 #define       MC_CMD_RMON_STATS_RX_ERRORS_IN_RX_QUEUE_OFST 0
+#define       MC_CMD_RMON_STATS_RX_ERRORS_IN_RX_QUEUE_LEN 4
 #define       MC_CMD_RMON_STATS_RX_ERRORS_IN_FLAGS_OFST 4
+#define       MC_CMD_RMON_STATS_RX_ERRORS_IN_FLAGS_LEN 4
 #define        MC_CMD_RMON_STATS_RX_ERRORS_IN_RST_LBN 0
 #define        MC_CMD_RMON_STATS_RX_ERRORS_IN_RST_WIDTH 1
 
 /* MC_CMD_RMON_STATS_RX_ERRORS_OUT msgresponse */
 #define    MC_CMD_RMON_STATS_RX_ERRORS_OUT_LEN 16
 #define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_CRC_ERRORS_OFST 0
+#define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_CRC_ERRORS_LEN 4
 #define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_TRUNC_ERRORS_OFST 4
+#define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_TRUNC_ERRORS_LEN 4
 #define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_RX_NO_DESC_DROPS_OFST 8
+#define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_RX_NO_DESC_DROPS_LEN 4
 #define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_RX_ABORT_OFST 12
+#define       MC_CMD_RMON_STATS_RX_ERRORS_OUT_RX_ABORT_LEN 4
 
 
 /***********************************/
@@ -11309,6 +13363,8 @@
  */
 #define MC_CMD_GET_PCIE_RESOURCE_INFO 0xfd
 
+#define MC_CMD_0xfd_PRIVILEGE_CTG SRIOV_CTG_GENERAL
+
 /* MC_CMD_GET_PCIE_RESOURCE_INFO_IN msgrequest */
 #define    MC_CMD_GET_PCIE_RESOURCE_INFO_IN_LEN 0
 
@@ -11316,20 +13372,27 @@
 #define    MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_LEN 28
 /* The maximum number of PFs the device can expose */
 #define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_PFS_OFST 0
+#define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_PFS_LEN 4
 /* The maximum number of VFs the device can expose in total */
 #define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VFS_OFST 4
+#define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VFS_LEN 4
 /* The maximum number of MSI-X vectors the device can provide in total */
 #define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VECTORS_OFST 8
+#define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VECTORS_LEN 4
 /* the number of MSI-X vectors the device will allocate by default to each PF
  */
 #define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_DEFAULT_PF_VECTORS_OFST 12
+#define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_DEFAULT_PF_VECTORS_LEN 4
 /* the number of MSI-X vectors the device will allocate by default to each VF
  */
 #define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_DEFAULT_VF_VECTORS_OFST 16
+#define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_DEFAULT_VF_VECTORS_LEN 4
 /* the maximum number of MSI-X vectors the device can allocate to any one PF */
 #define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_PF_VECTORS_OFST 20
+#define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_PF_VECTORS_LEN 4
 /* the maximum number of MSI-X vectors the device can allocate to any one VF */
 #define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VF_VECTORS_OFST 24
+#define       MC_CMD_GET_PCIE_RESOURCE_INFO_OUT_MAX_VF_VECTORS_LEN 4
 
 
 /***********************************/
@@ -11347,10 +13410,13 @@
 #define    MC_CMD_GET_PORT_MODES_OUT_LEN 12
 /* Bitmask of port modes available on the board (indexed by TLV_PORT_MODE_*) */
 #define       MC_CMD_GET_PORT_MODES_OUT_MODES_OFST 0
+#define       MC_CMD_GET_PORT_MODES_OUT_MODES_LEN 4
 /* Default (canonical) board mode */
 #define       MC_CMD_GET_PORT_MODES_OUT_DEFAULT_MODE_OFST 4
+#define       MC_CMD_GET_PORT_MODES_OUT_DEFAULT_MODE_LEN 4
 /* Current board mode */
 #define       MC_CMD_GET_PORT_MODES_OUT_CURRENT_MODE_OFST 8
+#define       MC_CMD_GET_PORT_MODES_OUT_CURRENT_MODE_LEN 4
 
 
 /***********************************/
@@ -11359,21 +13425,26 @@
  */
 #define MC_CMD_READ_ATB 0x100
 
-#define MC_CMD_0x100_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x100_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_READ_ATB_IN msgrequest */
 #define    MC_CMD_READ_ATB_IN_LEN 16
 #define       MC_CMD_READ_ATB_IN_SIGNAL_BUS_OFST 0
+#define       MC_CMD_READ_ATB_IN_SIGNAL_BUS_LEN 4
 #define          MC_CMD_READ_ATB_IN_BUS_CCOM  0x0 /* enum */
 #define          MC_CMD_READ_ATB_IN_BUS_CKR  0x1 /* enum */
 #define          MC_CMD_READ_ATB_IN_BUS_CPCIE  0x8 /* enum */
 #define       MC_CMD_READ_ATB_IN_SIGNAL_EN_BITNO_OFST 4
+#define       MC_CMD_READ_ATB_IN_SIGNAL_EN_BITNO_LEN 4
 #define       MC_CMD_READ_ATB_IN_SIGNAL_SEL_OFST 8
+#define       MC_CMD_READ_ATB_IN_SIGNAL_SEL_LEN 4
 #define       MC_CMD_READ_ATB_IN_SETTLING_TIME_US_OFST 12
+#define       MC_CMD_READ_ATB_IN_SETTLING_TIME_US_LEN 4
 
 /* MC_CMD_READ_ATB_OUT msgresponse */
 #define    MC_CMD_READ_ATB_OUT_LEN 4
 #define       MC_CMD_READ_ATB_OUT_SAMPLE_MV_OFST 0
+#define       MC_CMD_READ_ATB_OUT_SAMPLE_MV_LEN 4
 
 
 /***********************************/
@@ -11390,7 +13461,9 @@
 /* Each workaround is represented by a single bit according to the enums below.
  */
 #define       MC_CMD_GET_WORKAROUNDS_OUT_IMPLEMENTED_OFST 0
+#define       MC_CMD_GET_WORKAROUNDS_OUT_IMPLEMENTED_LEN 4
 #define       MC_CMD_GET_WORKAROUNDS_OUT_ENABLED_OFST 4
+#define       MC_CMD_GET_WORKAROUNDS_OUT_ENABLED_LEN 4
 /* enum: Bug 17230 work around. */
 #define          MC_CMD_GET_WORKAROUNDS_OUT_BUG17230 0x2
 /* enum: Bug 35388 work around (unsafe EVQ writes). */
@@ -11425,6 +13498,7 @@
  * 1,3 = 0x00030001
  */
 #define       MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_OFST 0
+#define       MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_LEN 4
 #define        MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_PF_LBN 0
 #define        MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_PF_WIDTH 16
 #define        MC_CMD_PRIVILEGE_MASK_IN_FUNCTION_VF_LBN 16
@@ -11434,6 +13508,7 @@
  * set to 1.
  */
 #define       MC_CMD_PRIVILEGE_MASK_IN_NEW_MASK_OFST 4
+#define       MC_CMD_PRIVILEGE_MASK_IN_NEW_MASK_LEN 4
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_ADMIN             0x1 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_LINK              0x2 /* enum */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_ONLOAD            0x4 /* enum */
@@ -11460,6 +13535,10 @@
  * only a set of permitted VLANs. See the vPort flag FLAG_VLAN_RESTRICT.
  */
 #define          MC_CMD_PRIVILEGE_MASK_IN_GRP_UNRESTRICTED_VLAN  0x2000
+/* enum: Privilege for insecure commands. Commands that belong to this group
+ * are not permitted on secure adapters regardless of the privilege mask.
+ */
+#define          MC_CMD_PRIVILEGE_MASK_IN_GRP_INSECURE          0x4000
 /* enum: Set this bit to indicate that a new privilege mask is to be set,
  * otherwise the command will only read the existing mask.
  */
@@ -11469,6 +13548,7 @@
 #define    MC_CMD_PRIVILEGE_MASK_OUT_LEN 4
 /* For an admin function, always all the privileges are reported. */
 #define       MC_CMD_PRIVILEGE_MASK_OUT_OLD_MASK_OFST 0
+#define       MC_CMD_PRIVILEGE_MASK_OUT_OLD_MASK_LEN 4
 
 
 /***********************************/
@@ -11485,12 +13565,14 @@
  * e.g. VF 1,3 = 0x00030001
  */
 #define       MC_CMD_LINK_STATE_MODE_IN_FUNCTION_OFST 0
+#define       MC_CMD_LINK_STATE_MODE_IN_FUNCTION_LEN 4
 #define        MC_CMD_LINK_STATE_MODE_IN_FUNCTION_PF_LBN 0
 #define        MC_CMD_LINK_STATE_MODE_IN_FUNCTION_PF_WIDTH 16
 #define        MC_CMD_LINK_STATE_MODE_IN_FUNCTION_VF_LBN 16
 #define        MC_CMD_LINK_STATE_MODE_IN_FUNCTION_VF_WIDTH 16
 /* New link state mode to be set */
 #define       MC_CMD_LINK_STATE_MODE_IN_NEW_MODE_OFST 4
+#define       MC_CMD_LINK_STATE_MODE_IN_NEW_MODE_LEN 4
 #define          MC_CMD_LINK_STATE_MODE_IN_LINK_STATE_AUTO       0x0 /* enum */
 #define          MC_CMD_LINK_STATE_MODE_IN_LINK_STATE_UP         0x1 /* enum */
 #define          MC_CMD_LINK_STATE_MODE_IN_LINK_STATE_DOWN       0x2 /* enum */
@@ -11501,11 +13583,12 @@
 /* MC_CMD_LINK_STATE_MODE_OUT msgresponse */
 #define    MC_CMD_LINK_STATE_MODE_OUT_LEN 4
 #define       MC_CMD_LINK_STATE_MODE_OUT_OLD_MODE_OFST 0
+#define       MC_CMD_LINK_STATE_MODE_OUT_OLD_MODE_LEN 4
 
 
 /***********************************/
 /* MC_CMD_GET_SNAPSHOT_LENGTH
- * Obtain the curent range of allowable values for the SNAPSHOT_LENGTH
+ * Obtain the current range of allowable values for the SNAPSHOT_LENGTH
  * parameter to MC_CMD_INIT_RXQ.
  */
 #define MC_CMD_GET_SNAPSHOT_LENGTH 0x101
@@ -11519,8 +13602,10 @@
 #define    MC_CMD_GET_SNAPSHOT_LENGTH_OUT_LEN 8
 /* Minimum acceptable snapshot length. */
 #define       MC_CMD_GET_SNAPSHOT_LENGTH_OUT_RX_SNAPLEN_MIN_OFST 0
+#define       MC_CMD_GET_SNAPSHOT_LENGTH_OUT_RX_SNAPLEN_MIN_LEN 4
 /* Maximum acceptable snapshot length. */
 #define       MC_CMD_GET_SNAPSHOT_LENGTH_OUT_RX_SNAPLEN_MAX_OFST 4
+#define       MC_CMD_GET_SNAPSHOT_LENGTH_OUT_RX_SNAPLEN_MAX_LEN 4
 
 
 /***********************************/
@@ -11529,7 +13614,7 @@
  */
 #define MC_CMD_FUSE_DIAGS 0x102
 
-#define MC_CMD_0x102_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x102_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_FUSE_DIAGS_IN msgrequest */
 #define    MC_CMD_FUSE_DIAGS_IN_LEN 0
@@ -11538,28 +13623,40 @@
 #define    MC_CMD_FUSE_DIAGS_OUT_LEN 48
 /* Total number of mismatched bits between pairs in area 0 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA0_MISMATCH_BITS_OFST 0
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA0_MISMATCH_BITS_LEN 4
 /* Total number of unexpectedly clear (set in B but not A) bits in area 0 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA0_PAIR_A_BAD_BITS_OFST 4
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA0_PAIR_A_BAD_BITS_LEN 4
 /* Total number of unexpectedly clear (set in A but not B) bits in area 0 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA0_PAIR_B_BAD_BITS_OFST 8
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA0_PAIR_B_BAD_BITS_LEN 4
 /* Checksum of data after logical OR of pairs in area 0 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA0_CHECKSUM_OFST 12
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA0_CHECKSUM_LEN 4
 /* Total number of mismatched bits between pairs in area 1 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA1_MISMATCH_BITS_OFST 16
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA1_MISMATCH_BITS_LEN 4
 /* Total number of unexpectedly clear (set in B but not A) bits in area 1 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA1_PAIR_A_BAD_BITS_OFST 20
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA1_PAIR_A_BAD_BITS_LEN 4
 /* Total number of unexpectedly clear (set in A but not B) bits in area 1 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA1_PAIR_B_BAD_BITS_OFST 24
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA1_PAIR_B_BAD_BITS_LEN 4
 /* Checksum of data after logical OR of pairs in area 1 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA1_CHECKSUM_OFST 28
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA1_CHECKSUM_LEN 4
 /* Total number of mismatched bits between pairs in area 2 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA2_MISMATCH_BITS_OFST 32
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA2_MISMATCH_BITS_LEN 4
 /* Total number of unexpectedly clear (set in B but not A) bits in area 2 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA2_PAIR_A_BAD_BITS_OFST 36
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA2_PAIR_A_BAD_BITS_LEN 4
 /* Total number of unexpectedly clear (set in A but not B) bits in area 2 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA2_PAIR_B_BAD_BITS_OFST 40
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA2_PAIR_B_BAD_BITS_LEN 4
 /* Checksum of data after logical OR of pairs in area 2 */
 #define       MC_CMD_FUSE_DIAGS_OUT_AREA2_CHECKSUM_OFST 44
+#define       MC_CMD_FUSE_DIAGS_OUT_AREA2_CHECKSUM_LEN 4
 
 
 /***********************************/
@@ -11576,6 +13673,7 @@
 #define    MC_CMD_PRIVILEGE_MODIFY_IN_LEN 16
 /* The groups of functions to have their privilege masks modified. */
 #define       MC_CMD_PRIVILEGE_MODIFY_IN_FN_GROUP_OFST 0
+#define       MC_CMD_PRIVILEGE_MODIFY_IN_FN_GROUP_LEN 4
 #define          MC_CMD_PRIVILEGE_MODIFY_IN_NONE       0x0 /* enum */
 #define          MC_CMD_PRIVILEGE_MODIFY_IN_ALL        0x1 /* enum */
 #define          MC_CMD_PRIVILEGE_MODIFY_IN_PFS_ONLY   0x2 /* enum */
@@ -11584,6 +13682,7 @@
 #define          MC_CMD_PRIVILEGE_MODIFY_IN_ONE        0x5 /* enum */
 /* For VFS_OF_PF specify the PF, for ONE specify the target function */
 #define       MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_OFST 4
+#define       MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_LEN 4
 #define        MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_PF_LBN 0
 #define        MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_PF_WIDTH 16
 #define        MC_CMD_PRIVILEGE_MODIFY_IN_FUNCTION_VF_LBN 16
@@ -11592,10 +13691,12 @@
  * refer to the command MC_CMD_PRIVILEGE_MASK
  */
 #define       MC_CMD_PRIVILEGE_MODIFY_IN_ADD_MASK_OFST 8
+#define       MC_CMD_PRIVILEGE_MODIFY_IN_ADD_MASK_LEN 4
 /* Privileges to be removed from the target functions. For privilege
  * definitions refer to the command MC_CMD_PRIVILEGE_MASK
  */
 #define       MC_CMD_PRIVILEGE_MODIFY_IN_REMOVE_MASK_OFST 12
+#define       MC_CMD_PRIVILEGE_MODIFY_IN_REMOVE_MASK_LEN 4
 
 /* MC_CMD_PRIVILEGE_MODIFY_OUT msgresponse */
 #define    MC_CMD_PRIVILEGE_MODIFY_OUT_LEN 0
@@ -11613,8 +13714,10 @@
 #define    MC_CMD_XPM_READ_BYTES_IN_LEN 8
 /* Start address (byte) */
 #define       MC_CMD_XPM_READ_BYTES_IN_ADDR_OFST 0
+#define       MC_CMD_XPM_READ_BYTES_IN_ADDR_LEN 4
 /* Count (bytes) */
 #define       MC_CMD_XPM_READ_BYTES_IN_COUNT_OFST 4
+#define       MC_CMD_XPM_READ_BYTES_IN_COUNT_LEN 4
 
 /* MC_CMD_XPM_READ_BYTES_OUT msgresponse */
 #define    MC_CMD_XPM_READ_BYTES_OUT_LENMIN 0
@@ -11633,7 +13736,7 @@
  */
 #define MC_CMD_XPM_WRITE_BYTES 0x104
 
-#define MC_CMD_0x104_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x104_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_WRITE_BYTES_IN msgrequest */
 #define    MC_CMD_XPM_WRITE_BYTES_IN_LENMIN 8
@@ -11641,8 +13744,10 @@
 #define    MC_CMD_XPM_WRITE_BYTES_IN_LEN(num) (8+1*(num))
 /* Start address (byte) */
 #define       MC_CMD_XPM_WRITE_BYTES_IN_ADDR_OFST 0
+#define       MC_CMD_XPM_WRITE_BYTES_IN_ADDR_LEN 4
 /* Count (bytes) */
 #define       MC_CMD_XPM_WRITE_BYTES_IN_COUNT_OFST 4
+#define       MC_CMD_XPM_WRITE_BYTES_IN_COUNT_LEN 4
 /* Data */
 #define       MC_CMD_XPM_WRITE_BYTES_IN_DATA_OFST 8
 #define       MC_CMD_XPM_WRITE_BYTES_IN_DATA_LEN 1
@@ -11659,14 +13764,16 @@
  */
 #define MC_CMD_XPM_READ_SECTOR 0x105
 
-#define MC_CMD_0x105_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x105_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_READ_SECTOR_IN msgrequest */
 #define    MC_CMD_XPM_READ_SECTOR_IN_LEN 8
 /* Sector index */
 #define       MC_CMD_XPM_READ_SECTOR_IN_INDEX_OFST 0
+#define       MC_CMD_XPM_READ_SECTOR_IN_INDEX_LEN 4
 /* Sector size */
 #define       MC_CMD_XPM_READ_SECTOR_IN_SIZE_OFST 4
+#define       MC_CMD_XPM_READ_SECTOR_IN_SIZE_LEN 4
 
 /* MC_CMD_XPM_READ_SECTOR_OUT msgresponse */
 #define    MC_CMD_XPM_READ_SECTOR_OUT_LENMIN 4
@@ -11674,9 +13781,11 @@
 #define    MC_CMD_XPM_READ_SECTOR_OUT_LEN(num) (4+1*(num))
 /* Sector type */
 #define       MC_CMD_XPM_READ_SECTOR_OUT_TYPE_OFST 0
+#define       MC_CMD_XPM_READ_SECTOR_OUT_TYPE_LEN 4
 #define          MC_CMD_XPM_READ_SECTOR_OUT_BLANK            0x0 /* enum */
 #define          MC_CMD_XPM_READ_SECTOR_OUT_CRYPTO_KEY_128   0x1 /* enum */
 #define          MC_CMD_XPM_READ_SECTOR_OUT_CRYPTO_KEY_256   0x2 /* enum */
+#define          MC_CMD_XPM_READ_SECTOR_OUT_CRYPTO_DATA      0x3 /* enum */
 #define          MC_CMD_XPM_READ_SECTOR_OUT_INVALID          0xff /* enum */
 /* Sector data */
 #define       MC_CMD_XPM_READ_SECTOR_OUT_DATA_OFST 4
@@ -11691,7 +13800,7 @@
  */
 #define MC_CMD_XPM_WRITE_SECTOR 0x106
 
-#define MC_CMD_0x106_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x106_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_WRITE_SECTOR_IN msgrequest */
 #define    MC_CMD_XPM_WRITE_SECTOR_IN_LENMIN 12
@@ -11708,10 +13817,12 @@
 #define       MC_CMD_XPM_WRITE_SECTOR_IN_RESERVED_LEN 3
 /* Sector type */
 #define       MC_CMD_XPM_WRITE_SECTOR_IN_TYPE_OFST 4
+#define       MC_CMD_XPM_WRITE_SECTOR_IN_TYPE_LEN 4
 /*            Enum values, see field(s): */
 /*               MC_CMD_XPM_READ_SECTOR/MC_CMD_XPM_READ_SECTOR_OUT/TYPE */
 /* Sector size */
 #define       MC_CMD_XPM_WRITE_SECTOR_IN_SIZE_OFST 8
+#define       MC_CMD_XPM_WRITE_SECTOR_IN_SIZE_LEN 4
 /* Sector data */
 #define       MC_CMD_XPM_WRITE_SECTOR_IN_DATA_OFST 12
 #define       MC_CMD_XPM_WRITE_SECTOR_IN_DATA_LEN 1
@@ -11722,6 +13833,7 @@
 #define    MC_CMD_XPM_WRITE_SECTOR_OUT_LEN 4
 /* New sector index */
 #define       MC_CMD_XPM_WRITE_SECTOR_OUT_INDEX_OFST 0
+#define       MC_CMD_XPM_WRITE_SECTOR_OUT_INDEX_LEN 4
 
 
 /***********************************/
@@ -11730,12 +13842,13 @@
  */
 #define MC_CMD_XPM_INVALIDATE_SECTOR 0x107
 
-#define MC_CMD_0x107_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x107_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_INVALIDATE_SECTOR_IN msgrequest */
 #define    MC_CMD_XPM_INVALIDATE_SECTOR_IN_LEN 4
 /* Sector index */
 #define       MC_CMD_XPM_INVALIDATE_SECTOR_IN_INDEX_OFST 0
+#define       MC_CMD_XPM_INVALIDATE_SECTOR_IN_INDEX_LEN 4
 
 /* MC_CMD_XPM_INVALIDATE_SECTOR_OUT msgresponse */
 #define    MC_CMD_XPM_INVALIDATE_SECTOR_OUT_LEN 0
@@ -11747,14 +13860,16 @@
  */
 #define MC_CMD_XPM_BLANK_CHECK 0x108
 
-#define MC_CMD_0x108_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x108_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_BLANK_CHECK_IN msgrequest */
 #define    MC_CMD_XPM_BLANK_CHECK_IN_LEN 8
 /* Start address (byte) */
 #define       MC_CMD_XPM_BLANK_CHECK_IN_ADDR_OFST 0
+#define       MC_CMD_XPM_BLANK_CHECK_IN_ADDR_LEN 4
 /* Count (bytes) */
 #define       MC_CMD_XPM_BLANK_CHECK_IN_COUNT_OFST 4
+#define       MC_CMD_XPM_BLANK_CHECK_IN_COUNT_LEN 4
 
 /* MC_CMD_XPM_BLANK_CHECK_OUT msgresponse */
 #define    MC_CMD_XPM_BLANK_CHECK_OUT_LENMIN 4
@@ -11762,6 +13877,7 @@
 #define    MC_CMD_XPM_BLANK_CHECK_OUT_LEN(num) (4+2*(num))
 /* Total number of bad (non-blank) locations */
 #define       MC_CMD_XPM_BLANK_CHECK_OUT_BAD_COUNT_OFST 0
+#define       MC_CMD_XPM_BLANK_CHECK_OUT_BAD_COUNT_LEN 4
 /* Addresses of bad locations (may be less than BAD_COUNT, if all cannot fit
  * into MCDI response)
  */
@@ -11777,14 +13893,16 @@
  */
 #define MC_CMD_XPM_REPAIR 0x109
 
-#define MC_CMD_0x109_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x109_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_REPAIR_IN msgrequest */
 #define    MC_CMD_XPM_REPAIR_IN_LEN 8
 /* Start address (byte) */
 #define       MC_CMD_XPM_REPAIR_IN_ADDR_OFST 0
+#define       MC_CMD_XPM_REPAIR_IN_ADDR_LEN 4
 /* Count (bytes) */
 #define       MC_CMD_XPM_REPAIR_IN_COUNT_OFST 4
+#define       MC_CMD_XPM_REPAIR_IN_COUNT_LEN 4
 
 /* MC_CMD_XPM_REPAIR_OUT msgresponse */
 #define    MC_CMD_XPM_REPAIR_OUT_LEN 0
@@ -11797,7 +13915,7 @@
  */
 #define MC_CMD_XPM_DECODER_TEST 0x10a
 
-#define MC_CMD_0x10a_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x10a_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_DECODER_TEST_IN msgrequest */
 #define    MC_CMD_XPM_DECODER_TEST_IN_LEN 0
@@ -11816,7 +13934,7 @@
  */
 #define MC_CMD_XPM_WRITE_TEST 0x10b
 
-#define MC_CMD_0x10b_PRIVILEGE_CTG SRIOV_CTG_ADMIN
+#define MC_CMD_0x10b_PRIVILEGE_CTG SRIOV_CTG_INSECURE
 
 /* MC_CMD_XPM_WRITE_TEST_IN msgrequest */
 #define    MC_CMD_XPM_WRITE_TEST_IN_LEN 0
@@ -11842,10 +13960,13 @@
 #define    MC_CMD_EXEC_SIGNED_IN_LEN 28
 /* the length of code to include in the CMAC */
 #define       MC_CMD_EXEC_SIGNED_IN_CODELEN_OFST 0
+#define       MC_CMD_EXEC_SIGNED_IN_CODELEN_LEN 4
 /* the length of date to include in the CMAC */
 #define       MC_CMD_EXEC_SIGNED_IN_DATALEN_OFST 4
+#define       MC_CMD_EXEC_SIGNED_IN_DATALEN_LEN 4
 /* the XPM sector containing the key to use */
 #define       MC_CMD_EXEC_SIGNED_IN_KEYSECTOR_OFST 8
+#define       MC_CMD_EXEC_SIGNED_IN_KEYSECTOR_LEN 4
 /* the expected CMAC value */
 #define       MC_CMD_EXEC_SIGNED_IN_CMAC_OFST 12
 #define       MC_CMD_EXEC_SIGNED_IN_CMAC_LEN 16
@@ -11868,11 +13989,34 @@
 #define    MC_CMD_PREPARE_SIGNED_IN_LEN 4
 /* the length of data area to clear */
 #define       MC_CMD_PREPARE_SIGNED_IN_DATALEN_OFST 0
+#define       MC_CMD_PREPARE_SIGNED_IN_DATALEN_LEN 4
 
 /* MC_CMD_PREPARE_SIGNED_OUT msgresponse */
 #define    MC_CMD_PREPARE_SIGNED_OUT_LEN 0
 
 
+/* TUNNEL_ENCAP_UDP_PORT_ENTRY structuredef */
+#define    TUNNEL_ENCAP_UDP_PORT_ENTRY_LEN 4
+/* UDP port (the standard ports are named below but any port may be used) */
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_OFST 0
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_LEN 2
+/* enum: the IANA allocated UDP port for VXLAN */
+#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_IANA_VXLAN_UDP_PORT  0x12b5
+/* enum: the IANA allocated UDP port for Geneve */
+#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_IANA_GENEVE_UDP_PORT  0x17c1
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_LBN 0
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_WIDTH 16
+/* tunnel encapsulation protocol (only those named below are supported) */
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_OFST 2
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_LEN 2
+/* enum: This port will be used for VXLAN on both IPv4 and IPv6 */
+#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN  0x0
+/* enum: This port will be used for Geneve on both IPv4 and IPv6 */
+#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE  0x1
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_LBN 16
+#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_WIDTH 16
+
+
 /***********************************/
 /* MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS
  * Configure UDP ports for tunnel encapsulation hardware acceleration. The
@@ -11913,27 +14057,6 @@
 #define        MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_LBN 0
 #define        MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_WIDTH 1
 
-/* TUNNEL_ENCAP_UDP_PORT_ENTRY structuredef */
-#define    TUNNEL_ENCAP_UDP_PORT_ENTRY_LEN 4
-/* UDP port (the standard ports are named below but any port may be used) */
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_OFST 0
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_LEN 2
-/* enum: the IANA allocated UDP port for VXLAN */
-#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_IANA_VXLAN_UDP_PORT  0x12b5
-/* enum: the IANA allocated UDP port for Geneve */
-#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_IANA_GENEVE_UDP_PORT  0x17c1
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_LBN 0
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT_WIDTH 16
-/* tunnel encapsulation protocol (only those named below are supported) */
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_OFST 2
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_LEN 2
-/* enum: VXLAN */
-#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN  0x0
-/* enum: Geneve */
-#define          TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE  0x1
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_LBN 16
-#define       TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL_WIDTH 16
-
 
 /***********************************/
 /* MC_CMD_RX_BALANCING
@@ -11950,12 +14073,16 @@
 #define    MC_CMD_RX_BALANCING_IN_LEN 16
 /* The RX port whose upconverter table will be modified */
 #define       MC_CMD_RX_BALANCING_IN_PORT_OFST 0
+#define       MC_CMD_RX_BALANCING_IN_PORT_LEN 4
 /* The VLAN priority associated to the table index and vFIFO */
 #define       MC_CMD_RX_BALANCING_IN_PRIORITY_OFST 4
+#define       MC_CMD_RX_BALANCING_IN_PRIORITY_LEN 4
 /* The resulting bit of SRC^DST for indexing the table */
 #define       MC_CMD_RX_BALANCING_IN_SRC_DST_OFST 8
+#define       MC_CMD_RX_BALANCING_IN_SRC_DST_LEN 4
 /* The RX engine to which the vFIFO in the table entry will point to */
 #define       MC_CMD_RX_BALANCING_IN_ENG_OFST 12
+#define       MC_CMD_RX_BALANCING_IN_ENG_LEN 4
 
 /* MC_CMD_RX_BALANCING_OUT msgresponse */
 #define    MC_CMD_RX_BALANCING_OUT_LEN 0
@@ -11976,8 +14103,10 @@
 #define    MC_CMD_NVRAM_PRIVATE_APPEND_IN_LEN(num) (8+1*(num))
 /* The tag to be appended */
 #define       MC_CMD_NVRAM_PRIVATE_APPEND_IN_TAG_OFST 0
+#define       MC_CMD_NVRAM_PRIVATE_APPEND_IN_TAG_LEN 4
 /* The length of the data */
 #define       MC_CMD_NVRAM_PRIVATE_APPEND_IN_LENGTH_OFST 4
+#define       MC_CMD_NVRAM_PRIVATE_APPEND_IN_LENGTH_LEN 4
 /* The data to be contained in the TLV structure */
 #define       MC_CMD_NVRAM_PRIVATE_APPEND_IN_DATA_BUFFER_OFST 8
 #define       MC_CMD_NVRAM_PRIVATE_APPEND_IN_DATA_BUFFER_LEN 1
@@ -12002,6 +14131,7 @@
 #define    MC_CMD_XPM_VERIFY_CONTENTS_IN_LEN 4
 /* Data type to be checked */
 #define       MC_CMD_XPM_VERIFY_CONTENTS_IN_DATA_TYPE_OFST 0
+#define       MC_CMD_XPM_VERIFY_CONTENTS_IN_DATA_TYPE_LEN 4
 
 /* MC_CMD_XPM_VERIFY_CONTENTS_OUT msgresponse */
 #define    MC_CMD_XPM_VERIFY_CONTENTS_OUT_LENMIN 12
@@ -12009,10 +14139,13 @@
 #define    MC_CMD_XPM_VERIFY_CONTENTS_OUT_LEN(num) (12+1*(num))
 /* Number of sectors found (test builds only) */
 #define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_NUM_SECTORS_OFST 0
+#define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_NUM_SECTORS_LEN 4
 /* Number of bytes found (test builds only) */
 #define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_NUM_BYTES_OFST 4
+#define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_NUM_BYTES_LEN 4
 /* Length of signature */
 #define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIG_LENGTH_OFST 8
+#define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIG_LENGTH_LEN 4
 /* Signature */
 #define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIGNATURE_OFST 12
 #define       MC_CMD_XPM_VERIFY_CONTENTS_OUT_SIGNATURE_LEN 1
@@ -12037,12 +14170,16 @@
 #define    MC_CMD_SET_EVQ_TMR_IN_LEN 16
 /* Function-relative queue instance */
 #define       MC_CMD_SET_EVQ_TMR_IN_INSTANCE_OFST 0
+#define       MC_CMD_SET_EVQ_TMR_IN_INSTANCE_LEN 4
 /* Requested value for timer load (in nanoseconds) */
 #define       MC_CMD_SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS_OFST 4
+#define       MC_CMD_SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS_LEN 4
 /* Requested value for timer reload (in nanoseconds) */
 #define       MC_CMD_SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS_OFST 8
+#define       MC_CMD_SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS_LEN 4
 /* Timer mode. Meanings as per EVQ_TMR_REG.TC_TIMER_VAL */
 #define       MC_CMD_SET_EVQ_TMR_IN_TMR_MODE_OFST 12
+#define       MC_CMD_SET_EVQ_TMR_IN_TMR_MODE_LEN 4
 #define          MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_DIS  0x0 /* enum */
 #define          MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_IMMED_START  0x1 /* enum */
 #define          MC_CMD_SET_EVQ_TMR_IN_TIMER_MODE_TRIG_START  0x2 /* enum */
@@ -12052,8 +14189,10 @@
 #define    MC_CMD_SET_EVQ_TMR_OUT_LEN 8
 /* Actual value for timer load (in nanoseconds) */
 #define       MC_CMD_SET_EVQ_TMR_OUT_TMR_LOAD_ACT_NS_OFST 0
+#define       MC_CMD_SET_EVQ_TMR_OUT_TMR_LOAD_ACT_NS_LEN 4
 /* Actual value for timer reload (in nanoseconds) */
 #define       MC_CMD_SET_EVQ_TMR_OUT_TMR_RELOAD_ACT_NS_OFST 4
+#define       MC_CMD_SET_EVQ_TMR_OUT_TMR_RELOAD_ACT_NS_LEN 4
 
 
 /***********************************/
@@ -12071,29 +14210,35 @@
 #define    MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN 36
 /* Reserved for future use. */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_FLAGS_OFST 0
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_FLAGS_LEN 4
 /* For timers updated via writes to EVQ_TMR_REG, this is the time interval (in
  * nanoseconds) for each increment of the timer load/reload count. The
  * requested duration of a timer is this value multiplied by the timer
  * load/reload count.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT_OFST 4
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT_LEN 4
 /* For timers updated via writes to EVQ_TMR_REG, this is the maximum value
  * allowed for timer load/reload counts.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT_OFST 8
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT_LEN 4
 /* For timers updated via writes to EVQ_TMR_REG, timer load/reload counts not a
  * multiple of this step size will be rounded in an implementation defined
  * manner.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_STEP_OFST 12
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_STEP_LEN 4
 /* Maximum timer duration (in nanoseconds) for timers updated via MCDI. Only
  * meaningful if MC_CMD_SET_EVQ_TMR is implemented.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS_OFST 16
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS_LEN 4
 /* Timer durations requested via MCDI that are not a multiple of this step size
  * will be rounded up. Only meaningful if MC_CMD_SET_EVQ_TMR is implemented.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS_OFST 20
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS_LEN 4
 /* For timers updated using the bug35388 workaround, this is the time interval
  * (in nanoseconds) for each increment of the timer load/reload count. The
  * requested duration of a timer is this value multiplied by the timer
@@ -12101,17 +14246,20 @@
  * is enabled.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT_OFST 24
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT_LEN 4
 /* For timers updated using the bug35388 workaround, this is the maximum value
  * allowed for timer load/reload counts. This field is only meaningful if the
  * bug35388 workaround is enabled.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT_OFST 28
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT_LEN 4
 /* For timers updated using the bug35388 workaround, timer load/reload counts
  * not a multiple of this step size will be rounded in an implementation
  * defined manner. This field is only meaningful if the bug35388 workaround is
  * enabled.
  */
 #define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_STEP_OFST 32
+#define       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_STEP_LEN 4
 
 
 /***********************************/
@@ -12129,19 +14277,24 @@
  * local queue index.
  */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INSTANCE_OFST 0
+#define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INSTANCE_LEN 4
 /* Will the common pool be used as TX_vFIFO_ULL (1) */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_MODE_OFST 4
+#define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_MODE_LEN 4
 #define          MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_ENABLED       0x1 /* enum */
 /* enum: Using this interface without TX_vFIFO_ULL is not supported for now */
 #define          MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_DISABLED      0x0
 /* Number of buffers to reserve for the common pool */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_SIZE_OFST 8
+#define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_SIZE_LEN 4
 /* TX datapath to which the Common Pool is connected to. */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INGRESS_OFST 12
+#define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_INGRESS_LEN 4
 /* enum: Extracts information from function */
 #define          MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_USE_FUNCTION_VALUE          -0x1
 /* Network port or RX Engine to which the common pool connects. */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_EGRESS_OFST 16
+#define       MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_EGRESS_LEN 4
 /* enum: Extracts information from function */
 /*               MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_USE_FUNCTION_VALUE          -0x1 */
 #define          MC_CMD_ALLOCATE_TX_VFIFO_CP_IN_PORT0          0x0 /* enum */
@@ -12157,6 +14310,7 @@
 #define    MC_CMD_ALLOCATE_TX_VFIFO_CP_OUT_LEN 4
 /* ID of the common pool allocated */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_CP_OUT_CP_ID_OFST 0
+#define       MC_CMD_ALLOCATE_TX_VFIFO_CP_OUT_CP_ID_LEN 4
 
 
 /***********************************/
@@ -12173,8 +14327,10 @@
 /* Common pool previously allocated to which the new vFIFO will be associated
  */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_CP_OFST 0
+#define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_CP_LEN 4
 /* Port or RX engine to associate the vFIFO egress */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_EGRESS_OFST 4
+#define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_EGRESS_LEN 4
 /* enum: Extracts information from common pool */
 #define          MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_USE_CP_VALUE   -0x1
 #define          MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PORT0          0x0 /* enum */
@@ -12187,12 +14343,15 @@
 #define          MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_RX_ENGINE1     0x5
 /* Minimum number of buffers that the pool must have */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_SIZE_OFST 8
+#define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_SIZE_LEN 4
 /* enum: Do not check the space available */
 #define          MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_NO_MINIMUM     0x0
 /* Will the vFIFO be used as TX_vFIFO_ULL */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_MODE_OFST 12
+#define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_MODE_LEN 4
 /* Network priority of the vFIFO,if applicable */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PRIORITY_OFST 16
+#define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_PRIORITY_LEN 4
 /* enum: Search for the lowest unused priority */
 #define          MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_IN_LOWEST_AVAILABLE  -0x1
 
@@ -12200,8 +14359,10 @@
 #define    MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_LEN 8
 /* Short vFIFO ID */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_VID_OFST 0
+#define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_VID_LEN 4
 /* Network priority of the vFIFO */
 #define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_PRIORITY_OFST 4
+#define       MC_CMD_ALLOCATE_TX_VFIFO_VFIFO_OUT_PRIORITY_LEN 4
 
 
 /***********************************/
@@ -12217,6 +14378,7 @@
 #define    MC_CMD_TEARDOWN_TX_VFIFO_VF_IN_LEN 4
 /* Short vFIFO ID */
 #define       MC_CMD_TEARDOWN_TX_VFIFO_VF_IN_VFIFO_OFST 0
+#define       MC_CMD_TEARDOWN_TX_VFIFO_VF_IN_VFIFO_LEN 4
 
 /* MC_CMD_TEARDOWN_TX_VFIFO_VF_OUT msgresponse */
 #define    MC_CMD_TEARDOWN_TX_VFIFO_VF_OUT_LEN 0
@@ -12235,6 +14397,7 @@
 #define    MC_CMD_DEALLOCATE_TX_VFIFO_CP_IN_LEN 4
 /* Common pool ID given when pool allocated */
 #define       MC_CMD_DEALLOCATE_TX_VFIFO_CP_IN_POOL_ID_OFST 0
+#define       MC_CMD_DEALLOCATE_TX_VFIFO_CP_IN_POOL_ID_LEN 4
 
 /* MC_CMD_DEALLOCATE_TX_VFIFO_CP_OUT msgresponse */
 #define    MC_CMD_DEALLOCATE_TX_VFIFO_CP_OUT_LEN 0
@@ -12256,8 +14419,10 @@
 #define    MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_LEN 8
 /* Available buffers for the ENG to NET vFIFOs. */
 #define       MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_NET_OFST 0
+#define       MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_NET_LEN 4
 /* Available buffers for the ENG to ENG and NET to ENG vFIFOs. */
 #define       MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_ENG_OFST 4
+#define       MC_CMD_SWITCH_GET_UNASSIGNED_BUFFERS_OUT_ENG_LEN 4
 
 
 #endif /* MCDI_PCOL_H */
diff --git a/drivers/net/ethernet/sfc/mcdi_port.c b/drivers/net/ethernet/sfc/mcdi_port.c
index 6e1f282b2976..ce8aabf9091e 100644
--- a/drivers/net/ethernet/sfc/mcdi_port.c
+++ b/drivers/net/ethernet/sfc/mcdi_port.c
@@ -171,89 +171,108 @@ static int efx_mcdi_mdio_write(struct net_device *net_dev,
 	return 0;
 }
 
-static u32 mcdi_to_ethtool_cap(u32 media, u32 cap)
+static void mcdi_to_ethtool_linkset(u32 media, u32 cap, unsigned long *linkset)
 {
-	u32 result = 0;
+	#define SET_BIT(name)	__set_bit(ETHTOOL_LINK_MODE_ ## name ## _BIT, \
+					  linkset)
 
+	bitmap_zero(linkset, __ETHTOOL_LINK_MODE_MASK_NBITS);
 	switch (media) {
 	case MC_CMD_MEDIA_KX4:
-		result |= SUPPORTED_Backplane;
+		SET_BIT(Backplane);
 		if (cap & (1 << MC_CMD_PHY_CAP_1000FDX_LBN))
-			result |= SUPPORTED_1000baseKX_Full;
+			SET_BIT(1000baseKX_Full);
 		if (cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN))
-			result |= SUPPORTED_10000baseKX4_Full;
+			SET_BIT(10000baseKX4_Full);
 		if (cap & (1 << MC_CMD_PHY_CAP_40000FDX_LBN))
-			result |= SUPPORTED_40000baseKR4_Full;
+			SET_BIT(40000baseKR4_Full);
 		break;
 
 	case MC_CMD_MEDIA_XFP:
 	case MC_CMD_MEDIA_SFP_PLUS:
 	case MC_CMD_MEDIA_QSFP_PLUS:
-		result |= SUPPORTED_FIBRE;
+		SET_BIT(FIBRE);
 		if (cap & (1 << MC_CMD_PHY_CAP_1000FDX_LBN))
-			result |= SUPPORTED_1000baseT_Full;
+			SET_BIT(1000baseT_Full);
 		if (cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN))
-			result |= SUPPORTED_10000baseT_Full;
+			SET_BIT(10000baseT_Full);
 		if (cap & (1 << MC_CMD_PHY_CAP_40000FDX_LBN))
-			result |= SUPPORTED_40000baseCR4_Full;
+			SET_BIT(40000baseCR4_Full);
+		if (cap & (1 << MC_CMD_PHY_CAP_100000FDX_LBN))
+			SET_BIT(100000baseCR4_Full);
+		if (cap & (1 << MC_CMD_PHY_CAP_25000FDX_LBN))
+			SET_BIT(25000baseCR_Full);
+		if (cap & (1 << MC_CMD_PHY_CAP_50000FDX_LBN))
+			SET_BIT(50000baseCR2_Full);
 		break;
 
 	case MC_CMD_MEDIA_BASE_T:
-		result |= SUPPORTED_TP;
+		SET_BIT(TP);
 		if (cap & (1 << MC_CMD_PHY_CAP_10HDX_LBN))
-			result |= SUPPORTED_10baseT_Half;
+			SET_BIT(10baseT_Half);
 		if (cap & (1 << MC_CMD_PHY_CAP_10FDX_LBN))
-			result |= SUPPORTED_10baseT_Full;
+			SET_BIT(10baseT_Full);
 		if (cap & (1 << MC_CMD_PHY_CAP_100HDX_LBN))
-			result |= SUPPORTED_100baseT_Half;
+			SET_BIT(100baseT_Half);
 		if (cap & (1 << MC_CMD_PHY_CAP_100FDX_LBN))
-			result |= SUPPORTED_100baseT_Full;
+			SET_BIT(100baseT_Full);
 		if (cap & (1 << MC_CMD_PHY_CAP_1000HDX_LBN))
-			result |= SUPPORTED_1000baseT_Half;
+			SET_BIT(1000baseT_Half);
 		if (cap & (1 << MC_CMD_PHY_CAP_1000FDX_LBN))
-			result |= SUPPORTED_1000baseT_Full;
+			SET_BIT(1000baseT_Full);
 		if (cap & (1 << MC_CMD_PHY_CAP_10000FDX_LBN))
-			result |= SUPPORTED_10000baseT_Full;
+			SET_BIT(10000baseT_Full);
 		break;
 	}
 
 	if (cap & (1 << MC_CMD_PHY_CAP_PAUSE_LBN))
-		result |= SUPPORTED_Pause;
+		SET_BIT(Pause);
 	if (cap & (1 << MC_CMD_PHY_CAP_ASYM_LBN))
-		result |= SUPPORTED_Asym_Pause;
+		SET_BIT(Asym_Pause);
 	if (cap & (1 << MC_CMD_PHY_CAP_AN_LBN))
-		result |= SUPPORTED_Autoneg;
+		SET_BIT(Autoneg);
 
-	return result;
+	#undef SET_BIT
 }
 
-static u32 ethtool_to_mcdi_cap(u32 cap)
+static u32 ethtool_linkset_to_mcdi_cap(const unsigned long *linkset)
 {
 	u32 result = 0;
 
-	if (cap & SUPPORTED_10baseT_Half)
+	#define TEST_BIT(name)	test_bit(ETHTOOL_LINK_MODE_ ## name ## _BIT, \
+					 linkset)
+
+	if (TEST_BIT(10baseT_Half))
 		result |= (1 << MC_CMD_PHY_CAP_10HDX_LBN);
-	if (cap & SUPPORTED_10baseT_Full)
+	if (TEST_BIT(10baseT_Full))
 		result |= (1 << MC_CMD_PHY_CAP_10FDX_LBN);
-	if (cap & SUPPORTED_100baseT_Half)
+	if (TEST_BIT(100baseT_Half))
 		result |= (1 << MC_CMD_PHY_CAP_100HDX_LBN);
-	if (cap & SUPPORTED_100baseT_Full)
+	if (TEST_BIT(100baseT_Full))
 		result |= (1 << MC_CMD_PHY_CAP_100FDX_LBN);
-	if (cap & SUPPORTED_1000baseT_Half)
+	if (TEST_BIT(1000baseT_Half))
 		result |= (1 << MC_CMD_PHY_CAP_1000HDX_LBN);
-	if (cap & (SUPPORTED_1000baseT_Full | SUPPORTED_1000baseKX_Full))
+	if (TEST_BIT(1000baseT_Full) || TEST_BIT(1000baseKX_Full))
 		result |= (1 << MC_CMD_PHY_CAP_1000FDX_LBN);
-	if (cap & (SUPPORTED_10000baseT_Full | SUPPORTED_10000baseKX4_Full))
+	if (TEST_BIT(10000baseT_Full) || TEST_BIT(10000baseKX4_Full))
 		result |= (1 << MC_CMD_PHY_CAP_10000FDX_LBN);
-	if (cap & (SUPPORTED_40000baseCR4_Full | SUPPORTED_40000baseKR4_Full))
+	if (TEST_BIT(40000baseCR4_Full) || TEST_BIT(40000baseKR4_Full))
 		result |= (1 << MC_CMD_PHY_CAP_40000FDX_LBN);
-	if (cap & SUPPORTED_Pause)
+	if (TEST_BIT(100000baseCR4_Full))
+		result |= (1 << MC_CMD_PHY_CAP_100000FDX_LBN);
+	if (TEST_BIT(25000baseCR_Full))
+		result |= (1 << MC_CMD_PHY_CAP_25000FDX_LBN);
+	if (TEST_BIT(50000baseCR2_Full))
+		result |= (1 << MC_CMD_PHY_CAP_50000FDX_LBN);
+	if (TEST_BIT(Pause))
 		result |= (1 << MC_CMD_PHY_CAP_PAUSE_LBN);
-	if (cap & SUPPORTED_Asym_Pause)
+	if (TEST_BIT(Asym_Pause))
 		result |= (1 << MC_CMD_PHY_CAP_ASYM_LBN);
-	if (cap & SUPPORTED_Autoneg)
+	if (TEST_BIT(Autoneg))
 		result |= (1 << MC_CMD_PHY_CAP_AN_LBN);
 
+	#undef TEST_BIT
+
 	return result;
 }
 
@@ -285,7 +304,7 @@ static u32 efx_get_mcdi_phy_flags(struct efx_nic *efx)
 	return flags;
 }
 
-static u32 mcdi_to_ethtool_media(u32 media)
+static u8 mcdi_to_ethtool_media(u32 media)
 {
 	switch (media) {
 	case MC_CMD_MEDIA_XAUI:
@@ -371,8 +390,8 @@ static int efx_mcdi_phy_probe(struct efx_nic *efx)
 
 	caps = MCDI_DWORD(outbuf, GET_LINK_OUT_CAP);
 	if (caps & (1 << MC_CMD_PHY_CAP_AN_LBN))
-		efx->link_advertising =
-			mcdi_to_ethtool_cap(phy_data->media, caps);
+		mcdi_to_ethtool_linkset(phy_data->media, caps,
+					efx->link_advertising);
 	else
 		phy_data->forced_cap = caps;
 
@@ -435,8 +454,8 @@ static int efx_mcdi_phy_probe(struct efx_nic *efx)
 int efx_mcdi_port_reconfigure(struct efx_nic *efx)
 {
 	struct efx_mcdi_phy_data *phy_cfg = efx->phy_data;
-	u32 caps = (efx->link_advertising ?
-		    ethtool_to_mcdi_cap(efx->link_advertising) :
+	u32 caps = (efx->link_advertising[0] ?
+		    ethtool_linkset_to_mcdi_cap(efx->link_advertising) :
 		    phy_cfg->forced_cap);
 
 	return efx_mcdi_set_link(efx, caps, efx_get_mcdi_phy_flags(efx),
@@ -509,34 +528,28 @@ static void efx_mcdi_phy_get_link_ksettings(struct efx_nic *efx,
 	struct efx_mcdi_phy_data *phy_cfg = efx->phy_data;
 	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_LINK_OUT_LEN);
 	int rc;
-	u32 supported, advertising, lp_advertising;
 
-	supported = mcdi_to_ethtool_cap(phy_cfg->media, phy_cfg->supported_cap);
-	advertising = efx->link_advertising;
 	cmd->base.speed = efx->link_state.speed;
 	cmd->base.duplex = efx->link_state.fd;
 	cmd->base.port = mcdi_to_ethtool_media(phy_cfg->media);
 	cmd->base.phy_address = phy_cfg->port;
-	cmd->base.autoneg = !!(efx->link_advertising & ADVERTISED_Autoneg);
+	cmd->base.autoneg = !!(efx->link_advertising[0] & ADVERTISED_Autoneg);
 	cmd->base.mdio_support = (efx->mdio.mode_support &
 			      (MDIO_SUPPORTS_C45 | MDIO_SUPPORTS_C22));
 
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
-						supported);
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
-						advertising);
+	mcdi_to_ethtool_linkset(phy_cfg->media, phy_cfg->supported_cap,
+				cmd->link_modes.supported);
+	memcpy(cmd->link_modes.advertising, efx->link_advertising,
+	       sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK()));
 
 	BUILD_BUG_ON(MC_CMD_GET_LINK_IN_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_GET_LINK, NULL, 0,
 			  outbuf, sizeof(outbuf), NULL);
 	if (rc)
 		return;
-	lp_advertising =
-		mcdi_to_ethtool_cap(phy_cfg->media,
-				    MCDI_DWORD(outbuf, GET_LINK_OUT_LP_CAP));
-
-	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.lp_advertising,
-						lp_advertising);
+	mcdi_to_ethtool_linkset(phy_cfg->media,
+				MCDI_DWORD(outbuf, GET_LINK_OUT_LP_CAP),
+				cmd->link_modes.lp_advertising);
 }
 
 static int
@@ -546,29 +559,28 @@ efx_mcdi_phy_set_link_ksettings(struct efx_nic *efx,
 	struct efx_mcdi_phy_data *phy_cfg = efx->phy_data;
 	u32 caps;
 	int rc;
-	u32 advertising;
-
-	ethtool_convert_link_mode_to_legacy_u32(&advertising,
-						cmd->link_modes.advertising);
 
 	if (cmd->base.autoneg) {
-		caps = (ethtool_to_mcdi_cap(advertising) |
-			 1 << MC_CMD_PHY_CAP_AN_LBN);
+		caps = (ethtool_linkset_to_mcdi_cap(cmd->link_modes.advertising) |
+			1 << MC_CMD_PHY_CAP_AN_LBN);
 	} else if (cmd->base.duplex) {
 		switch (cmd->base.speed) {
-		case 10:    caps = 1 << MC_CMD_PHY_CAP_10FDX_LBN;    break;
-		case 100:   caps = 1 << MC_CMD_PHY_CAP_100FDX_LBN;   break;
-		case 1000:  caps = 1 << MC_CMD_PHY_CAP_1000FDX_LBN;  break;
-		case 10000: caps = 1 << MC_CMD_PHY_CAP_10000FDX_LBN; break;
-		case 40000: caps = 1 << MC_CMD_PHY_CAP_40000FDX_LBN; break;
-		default:    return -EINVAL;
+		case 10:     caps = 1 << MC_CMD_PHY_CAP_10FDX_LBN;     break;
+		case 100:    caps = 1 << MC_CMD_PHY_CAP_100FDX_LBN;    break;
+		case 1000:   caps = 1 << MC_CMD_PHY_CAP_1000FDX_LBN;   break;
+		case 10000:  caps = 1 << MC_CMD_PHY_CAP_10000FDX_LBN;  break;
+		case 40000:  caps = 1 << MC_CMD_PHY_CAP_40000FDX_LBN;  break;
+		case 100000: caps = 1 << MC_CMD_PHY_CAP_100000FDX_LBN; break;
+		case 25000:  caps = 1 << MC_CMD_PHY_CAP_25000FDX_LBN;  break;
+		case 50000:  caps = 1 << MC_CMD_PHY_CAP_50000FDX_LBN;  break;
+		default:     return -EINVAL;
 		}
 	} else {
 		switch (cmd->base.speed) {
-		case 10:    caps = 1 << MC_CMD_PHY_CAP_10HDX_LBN;    break;
-		case 100:   caps = 1 << MC_CMD_PHY_CAP_100HDX_LBN;   break;
-		case 1000:  caps = 1 << MC_CMD_PHY_CAP_1000HDX_LBN;  break;
-		default:    return -EINVAL;
+		case 10:     caps = 1 << MC_CMD_PHY_CAP_10HDX_LBN;     break;
+		case 100:    caps = 1 << MC_CMD_PHY_CAP_100HDX_LBN;    break;
+		case 1000:   caps = 1 << MC_CMD_PHY_CAP_1000HDX_LBN;   break;
+		default:     return -EINVAL;
 		}
 	}
 
@@ -578,11 +590,10 @@ efx_mcdi_phy_set_link_ksettings(struct efx_nic *efx,
 		return rc;
 
 	if (cmd->base.autoneg) {
-		efx_link_set_advertising(
-			efx, advertising | ADVERTISED_Autoneg);
+		efx_link_set_advertising(efx, cmd->link_modes.advertising);
 		phy_cfg->forced_cap = 0;
 	} else {
-		efx_link_set_advertising(efx, 0);
+		efx_link_clear_advertising(efx);
 		phy_cfg->forced_cap = caps;
 	}
 	return 0;
@@ -985,6 +996,9 @@ static unsigned int efx_mcdi_event_link_speed[] = {
 	[MCDI_EVENT_LINKCHANGE_SPEED_1G] = 1000,
 	[MCDI_EVENT_LINKCHANGE_SPEED_10G] = 10000,
 	[MCDI_EVENT_LINKCHANGE_SPEED_40G] = 40000,
+	[MCDI_EVENT_LINKCHANGE_SPEED_25G] = 25000,
+	[MCDI_EVENT_LINKCHANGE_SPEED_50G] = 50000,
+	[MCDI_EVENT_LINKCHANGE_SPEED_100G] = 100000,
 };
 
 void efx_mcdi_process_link_change(struct efx_nic *efx, efx_qword_t *ev)
@@ -1087,7 +1101,7 @@ static int efx_mcdi_mac_stats(struct efx_nic *efx,
 	int period = action == EFX_STATS_ENABLE ? 1000 : 0;
 	dma_addr_t dma_addr = efx->stats_buffer.dma_addr;
 	u32 dma_len = action != EFX_STATS_DISABLE ?
-		MC_CMD_MAC_NSTATS * sizeof(u64) : 0;
+		efx->num_mac_stats * sizeof(u64) : 0;
 
 	BUILD_BUG_ON(MC_CMD_MAC_STATS_OUT_DMA_LEN != 0);
 
@@ -1121,7 +1135,7 @@ void efx_mcdi_mac_start_stats(struct efx_nic *efx)
 {
 	__le64 *dma_stats = efx->stats_buffer.addr;
 
-	dma_stats[MC_CMD_MAC_GENERATION_END] = EFX_MC_STATS_GENERATION_INVALID;
+	dma_stats[efx->num_mac_stats - 1] = EFX_MC_STATS_GENERATION_INVALID;
 
 	efx_mcdi_mac_stats(efx, EFX_STATS_ENABLE, 0);
 }
@@ -1139,10 +1153,10 @@ void efx_mcdi_mac_pull_stats(struct efx_nic *efx)
 	__le64 *dma_stats = efx->stats_buffer.addr;
 	int attempts = EFX_MAC_STATS_WAIT_ATTEMPTS;
 
-	dma_stats[MC_CMD_MAC_GENERATION_END] = EFX_MC_STATS_GENERATION_INVALID;
+	dma_stats[efx->num_mac_stats - 1] = EFX_MC_STATS_GENERATION_INVALID;
 	efx_mcdi_mac_stats(efx, EFX_STATS_PULL, 0);
 
-	while (dma_stats[MC_CMD_MAC_GENERATION_END] ==
+	while (dma_stats[efx->num_mac_stats - 1] ==
 				EFX_MC_STATS_GENERATION_INVALID &&
 			attempts-- != 0)
 		udelay(EFX_MAC_STATS_WAIT_US);
@@ -1167,7 +1181,7 @@ int efx_mcdi_port_probe(struct efx_nic *efx)
 
 	/* Allocate buffer for stats */
 	rc = efx_nic_alloc_buffer(efx, &efx->stats_buffer,
-				  MC_CMD_MAC_NSTATS * sizeof(u64), GFP_KERNEL);
+				  efx->num_mac_stats * sizeof(u64), GFP_KERNEL);
 	if (rc)
 		return rc;
 	netif_dbg(efx, probe, efx->net_dev,
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index c0537ea06c9a..d20a8660ee48 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -191,6 +191,7 @@ struct efx_tx_buffer {
  *	Size of the region is efx_piobuf_size.
  * @piobuf_offset: Buffer offset to be specified in PIO descriptors
  * @initialised: Has hardware queue been initialised?
+ * @timestamping: Is timestamping enabled for this channel?
  * @handle_tso: TSO xmit preparation handler.  Sets up the TSO metadata and
  *	may also map tx data, depending on the nature of the TSO implementation.
  * @read_count: Current read pointer.
@@ -202,6 +203,10 @@ struct efx_tx_buffer {
  *	avoid cache-line ping-pong between the xmit path and the
  *	completion path.
  * @merge_events: Number of TX merged completion events
+ * @completed_desc_ptr: Most recent completed pointer - only used with
+ *      timestamping.
+ * @completed_timestamp_major: Top part of the most recent tx timestamp.
+ * @completed_timestamp_minor: Low part of the most recent tx timestamp.
  * @insert_count: Current insert pointer
  *	This is the number of buffers that have been added to the
  *	software ring.
@@ -247,6 +252,7 @@ struct efx_tx_queue {
 	void __iomem *piobuf;
 	unsigned int piobuf_offset;
 	bool initialised;
+	bool timestamping;
 
 	/* Function pointers used in the fast path. */
 	int (*handle_tso)(struct efx_tx_queue*, struct sk_buff*, bool *);
@@ -257,6 +263,9 @@ struct efx_tx_queue {
 	unsigned int merge_events;
 	unsigned int bytes_compl;
 	unsigned int pkts_compl;
+	unsigned int completed_desc_ptr;
+	u32 completed_timestamp_major;
+	u32 completed_timestamp_minor;
 
 	/* Members used only on the xmit path */
 	unsigned int insert_count ____cacheline_aligned_in_smp;
@@ -522,8 +531,12 @@ struct efx_msi_context {
  * @copy: Copy the channel state prior to reallocation.  May be %NULL if
  *	reallocation is not supported.
  * @receive_skb: Handle an skb ready to be passed to netif_receive_skb()
+ * @want_txqs: Determine whether this channel should have TX queues
+ *	created.  If %NULL, TX queues are not created.
  * @keep_eventq: Flag for whether event queue should be kept initialised
  *	while the device is stopped
+ * @want_pio: Flag for whether PIO buffers should be linked to this
+ *	channel's TX queues.
  */
 struct efx_channel_type {
 	void (*handle_no_channel)(struct efx_nic *);
@@ -532,7 +545,9 @@ struct efx_channel_type {
 	void (*get_name)(struct efx_channel *, char *buf, size_t len);
 	struct efx_channel *(*copy)(const struct efx_channel *);
 	bool (*receive_skb)(struct efx_channel *, struct sk_buff *);
+	bool (*want_txqs)(struct efx_channel *);
 	bool keep_eventq;
+	bool want_pio;
 };
 
 enum efx_led_mode {
@@ -708,6 +723,7 @@ struct vfdi_status;
  * @reset_work: Scheduled reset workitem
  * @membase_phys: Memory BAR value as physical address
  * @membase: Memory BAR value
+ * @vi_stride: step between per-VI registers / memory regions
  * @interrupt_mode: Interrupt mode
  * @timer_quantum_ns: Interrupt timer quantum, in nanoseconds
  * @timer_max_ns: Interrupt timer maximum value, in nanoseconds
@@ -734,6 +750,7 @@ struct vfdi_status;
  * @n_channels: Number of channels in use
  * @n_rx_channels: Number of channels used for RX (= number of RX queues)
  * @n_tx_channels: Number of channels used for TX
+ * @n_extra_tx_channels: Number of extra channels with TX queues
  * @rx_ip_align: RX DMA address offset to have IP header aligned in
  *	in accordance with NET_IP_ALIGN
  * @rx_dma_len: Current maximum RX DMA length
@@ -773,6 +790,8 @@ struct vfdi_status;
  * @port_initialized: Port initialized?
  * @net_dev: Operating system network device. Consider holding the rtnl lock
  * @fixed_features: Features which cannot be turned off
+ * @num_mac_stats: Number of MAC stats reported by firmware (MAC_STATS_NUM_STATS
+ *	field of %MC_CMD_GET_CAPABILITIES_V4 response, or %MC_CMD_MAC_NSTATS)
  * @stats_buffer: DMA buffer for statistics
  * @phy_type: PHY type
  * @phy_op: PHY interface
@@ -812,6 +831,7 @@ struct vfdi_status;
  * @vf_init_count: Number of VFs that have been fully initialised.
  * @vi_scale: log2 number of vnics per VF.
  * @ptp_data: PTP state data
+ * @ptp_warned: has this NIC seen and warned about unexpected PTP events?
  * @vpd_sn: Serial number read from VPD
  * @monitor_work: Hardware monitor workitem
  * @biu_lock: BIU (bus interface unit) lock
@@ -842,6 +862,8 @@ struct efx_nic {
 	resource_size_t membase_phys;
 	void __iomem *membase;
 
+	unsigned int vi_stride;
+
 	enum efx_int_mode interrupt_mode;
 	unsigned int timer_quantum_ns;
 	unsigned int timer_max_ns;
@@ -875,6 +897,7 @@ struct efx_nic {
 	unsigned rss_spread;
 	unsigned tx_channel_offset;
 	unsigned n_tx_channels;
+	unsigned n_extra_tx_channels;
 	unsigned int rx_ip_align;
 	unsigned int rx_dma_len;
 	unsigned int rx_buffer_order;
@@ -918,6 +941,7 @@ struct efx_nic {
 
 	netdev_features_t fixed_features;
 
+	u16 num_mac_stats;
 	struct efx_buffer stats_buffer;
 	u64 rx_nodesc_drops_total;
 	u64 rx_nodesc_drops_while_down;
@@ -930,7 +954,7 @@ struct efx_nic {
 	unsigned int mdio_bus;
 	enum efx_phy_mode phy_mode;
 
-	u32 link_advertising;
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(link_advertising);
 	struct efx_link_state link_state;
 	unsigned int n_link_state_changes;
 
@@ -965,6 +989,7 @@ struct efx_nic {
 #endif
 
 	struct efx_ptp_data *ptp_data;
+	bool ptp_warned;
 
 	char *vpd_sn;
 
@@ -1154,7 +1179,7 @@ struct efx_udp_tunnel {
  */
 struct efx_nic_type {
 	bool is_vf;
-	unsigned int mem_bar;
+	unsigned int (*mem_bar)(struct efx_nic *efx);
 	unsigned int (*mem_map_size)(struct efx_nic *efx);
 	int (*probe)(struct efx_nic *efx);
 	void (*remove)(struct efx_nic *efx);
@@ -1355,8 +1380,8 @@ efx_get_tx_queue(struct efx_nic *efx, unsigned index, unsigned type)
 
 static inline bool efx_channel_has_tx_queues(struct efx_channel *channel)
 {
-	return channel->channel - channel->efx->tx_channel_offset <
-		channel->efx->n_tx_channels;
+	return channel->type && channel->type->want_txqs &&
+				channel->type->want_txqs(channel);
 }
 
 static inline struct efx_tx_queue *
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index 7b51b6371724..6549fc685a48 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -325,6 +325,30 @@ enum {
 	EF10_STAT_tx_bad,
 	EF10_STAT_tx_bad_bytes,
 	EF10_STAT_tx_overflow,
+	EF10_STAT_V1_COUNT,
+	EF10_STAT_fec_uncorrected_errors = EF10_STAT_V1_COUNT,
+	EF10_STAT_fec_corrected_errors,
+	EF10_STAT_fec_corrected_symbols_lane0,
+	EF10_STAT_fec_corrected_symbols_lane1,
+	EF10_STAT_fec_corrected_symbols_lane2,
+	EF10_STAT_fec_corrected_symbols_lane3,
+	EF10_STAT_ctpio_dmabuf_start,
+	EF10_STAT_ctpio_vi_busy_fallback,
+	EF10_STAT_ctpio_long_write_success,
+	EF10_STAT_ctpio_missing_dbell_fail,
+	EF10_STAT_ctpio_overflow_fail,
+	EF10_STAT_ctpio_underflow_fail,
+	EF10_STAT_ctpio_timeout_fail,
+	EF10_STAT_ctpio_noncontig_wr_fail,
+	EF10_STAT_ctpio_frm_clobber_fail,
+	EF10_STAT_ctpio_invalid_wr_fail,
+	EF10_STAT_ctpio_vi_clobber_fallback,
+	EF10_STAT_ctpio_unqualified_fallback,
+	EF10_STAT_ctpio_runt_fallback,
+	EF10_STAT_ctpio_success,
+	EF10_STAT_ctpio_fallback,
+	EF10_STAT_ctpio_poison,
+	EF10_STAT_ctpio_erase,
 	EF10_STAT_COUNT
 };
 
@@ -416,6 +440,7 @@ struct efx_ef10_nic_data {
 	struct efx_udp_tunnel udp_tunnels[16];
 	bool udp_tunnels_dirty;
 	struct mutex udp_tunnels_lock;
+	u64 licensed_features;
 };
 
 int efx_init_sriov(void);
@@ -424,6 +449,7 @@ void efx_fini_sriov(void);
 struct ethtool_ts_info;
 int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel);
 void efx_ptp_defer_probe_with_channel(struct efx_nic *efx);
+struct efx_channel *efx_ptp_channel(struct efx_nic *efx);
 void efx_ptp_remove(struct efx_nic *efx);
 int efx_ptp_set_ts_config(struct efx_nic *efx, struct ifreq *ifr);
 int efx_ptp_get_ts_config(struct efx_nic *efx, struct ifreq *ifr);
@@ -447,6 +473,8 @@ static inline void efx_rx_skb_attach_timestamp(struct efx_channel *channel,
 }
 void efx_ptp_start_datapath(struct efx_nic *efx);
 void efx_ptp_stop_datapath(struct efx_nic *efx);
+bool efx_ptp_use_mac_tx_timestamps(struct efx_nic *efx);
+ktime_t efx_ptp_nic_to_kernel_time(struct efx_tx_queue *tx_queue);
 
 extern const struct efx_nic_type falcon_a1_nic_type;
 extern const struct efx_nic_type falcon_b0_nic_type;
diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c
index caa89bf7603e..f21661532ed3 100644
--- a/drivers/net/ethernet/sfc/ptp.c
+++ b/drivers/net/ethernet/sfc/ptp.c
@@ -149,18 +149,14 @@ enum ptp_packet_state {
 /* Maximum parts-per-billion adjustment that is acceptable */
 #define MAX_PPB			1000000
 
-/* Number of bits required to hold the above */
-#define	MAX_PPB_BITS		20
-
-/* Number of extra bits allowed when calculating fractional ns.
- * EXTRA_BITS + MC_CMD_PTP_IN_ADJUST_BITS + MAX_PPB_BITS should
- * be less than 63.
- */
-#define	PPB_EXTRA_BITS		2
-
 /* Precalculate scale word to avoid long long division at runtime */
-#define	PPB_SCALE_WORD	((1LL << (PPB_EXTRA_BITS + MC_CMD_PTP_IN_ADJUST_BITS +\
-			MAX_PPB_BITS)) / 1000000000LL)
+/* This is equivalent to 2^66 / 10^9. */
+#define PPB_SCALE_WORD  ((1LL << (57)) / 1953125LL)
+
+/* How much to shift down after scaling to convert to FP40 */
+#define PPB_SHIFT_FP40		26
+/* ... and FP44. */
+#define PPB_SHIFT_FP44		22
 
 #define PTP_SYNC_ATTEMPTS	4
 
@@ -218,8 +214,8 @@ struct efx_ptp_timeset {
  * @channel: The PTP channel (Siena only)
  * @rx_ts_inline: Flag for whether RX timestamps are inline (else they are
  *	separate events)
- * @rxq: Receive queue (awaiting timestamps)
- * @txq: Transmit queue
+ * @rxq: Receive SKB queue (awaiting timestamps)
+ * @txq: Transmit SKB queue
  * @evt_list: List of MC receive events awaiting packets
  * @evt_free_list: List of free events
  * @evt_lock: Lock for manipulating evt_list and evt_free_list
@@ -233,19 +229,36 @@ struct efx_ptp_timeset {
  * @config: Current timestamp configuration
  * @enabled: PTP operation enabled
  * @mode: Mode in which PTP operating (PTP version)
- * @time_format: Time format supported by this NIC
  * @ns_to_nic_time: Function to convert from scalar nanoseconds to NIC time
  * @nic_to_kernel_time: Function to convert from NIC to kernel time
+ * @nic_time.minor_max: Wrap point for NIC minor times
+ * @nic_time.sync_event_diff_min: Minimum acceptable difference between time
+ * in packet prefix and last MCDI time sync event i.e. how much earlier than
+ * the last sync event time a packet timestamp can be.
+ * @nic_time.sync_event_diff_max: Maximum acceptable difference between time
+ * in packet prefix and last MCDI time sync event i.e. how much later than
+ * the last sync event time a packet timestamp can be.
+ * @nic_time.sync_event_minor_shift: Shift required to make minor time from
+ * field in MCDI time sync event.
  * @min_synchronisation_ns: Minimum acceptable corrected sync window
- * @ts_corrections.tx: Required driver correction of transmit timestamps
- * @ts_corrections.rx: Required driver correction of receive timestamps
+ * @capabilities: Capabilities flags from the NIC
+ * @ts_corrections.ptp_tx: Required driver correction of PTP packet transmit
+ *                         timestamps
+ * @ts_corrections.ptp_rx: Required driver correction of PTP packet receive
+ *                         timestamps
  * @ts_corrections.pps_out: PPS output error (information only)
  * @ts_corrections.pps_in: Required driver correction of PPS input timestamps
+ * @ts_corrections.general_tx: Required driver correction of general packet
+ *                             transmit timestamps
+ * @ts_corrections.general_rx: Required driver correction of general packet
+ *                             receive timestamps
  * @evt_frags: Partly assembled PTP events
  * @evt_frag_idx: Current fragment number
  * @evt_code: Last event code
  * @start: Address at which MC indicates ready for synchronisation
  * @host_time_pps: Host time at last PPS
+ * @adjfreq_ppb_shift: Shift required to convert scaled parts-per-billion
+ * frequency adjustment into a fixed point fractional nanosecond format.
  * @current_adjfreq: Current ppb adjustment.
  * @phc_clock: Pointer to registered phc device (if primary function)
  * @phc_clock_info: Registration structure for phc device
@@ -264,6 +277,7 @@ struct efx_ptp_timeset {
  * @oversize_sync_windows: Number of corrected sync windows that are too large
  * @rx_no_timestamp: Number of packets received without a timestamp.
  * @timeset: Last set of synchronisation statistics.
+ * @xmit_skb: Transmit SKB function.
  */
 struct efx_ptp_data {
 	struct efx_nic *efx;
@@ -284,22 +298,31 @@ struct efx_ptp_data {
 	struct hwtstamp_config config;
 	bool enabled;
 	unsigned int mode;
-	unsigned int time_format;
 	void (*ns_to_nic_time)(s64 ns, u32 *nic_major, u32 *nic_minor);
 	ktime_t (*nic_to_kernel_time)(u32 nic_major, u32 nic_minor,
 				      s32 correction);
+	struct {
+		u32 minor_max;
+		u32 sync_event_diff_min;
+		u32 sync_event_diff_max;
+		unsigned int sync_event_minor_shift;
+	} nic_time;
 	unsigned int min_synchronisation_ns;
+	unsigned int capabilities;
 	struct {
-		s32 tx;
-		s32 rx;
+		s32 ptp_tx;
+		s32 ptp_rx;
 		s32 pps_out;
 		s32 pps_in;
+		s32 general_tx;
+		s32 general_rx;
 	} ts_corrections;
 	efx_qword_t evt_frags[MAX_EVENT_FRAGS];
 	int evt_frag_idx;
 	int evt_code;
 	struct efx_buffer start;
 	struct pps_event_time host_time_pps;
+	unsigned int adjfreq_ppb_shift;
 	s64 current_adjfreq;
 	struct ptp_clock *phc_clock;
 	struct ptp_clock_info phc_clock_info;
@@ -319,6 +342,7 @@ struct efx_ptp_data {
 	unsigned int rx_no_timestamp;
 	struct efx_ptp_timeset
 	timeset[MC_CMD_PTP_OUT_SYNCHRONIZE_TIMESET_MAXNUM];
+	void (*xmit_skb)(struct efx_nic *efx, struct sk_buff *skb);
 };
 
 static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta);
@@ -329,6 +353,24 @@ static int efx_phc_settime(struct ptp_clock_info *ptp,
 static int efx_phc_enable(struct ptp_clock_info *ptp,
 			  struct ptp_clock_request *request, int on);
 
+bool efx_ptp_use_mac_tx_timestamps(struct efx_nic *efx)
+{
+	struct efx_ef10_nic_data *nic_data = efx->nic_data;
+
+	return ((efx_nic_rev(efx) >= EFX_REV_HUNT_A0) &&
+		(nic_data->datapath_caps2 &
+		 (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_TX_MAC_TIMESTAMPING_LBN)
+		));
+}
+
+/* PTP 'extra' channel is still a traffic channel, but we only create TX queues
+ * if PTP uses MAC TX timestamps, not if PTP uses the MC directly to transmit.
+ */
+static bool efx_ptp_want_txqs(struct efx_channel *channel)
+{
+	return efx_ptp_use_mac_tx_timestamps(channel->efx);
+}
+
 #define PTP_SW_STAT(ext_name, field_name)				\
 	{ #ext_name, 0, offsetof(struct efx_ptp_data, field_name) }
 #define PTP_MC_STAT(ext_name, mcdi_name)				\
@@ -471,6 +513,89 @@ static ktime_t efx_ptp_s27_to_ktime_correction(u32 nic_major, u32 nic_minor,
 	return efx_ptp_s27_to_ktime(nic_major, nic_minor);
 }
 
+/* For Medford2 platforms the time is in seconds and quarter nanoseconds. */
+static void efx_ptp_ns_to_s_qns(s64 ns, u32 *nic_major, u32 *nic_minor)
+{
+	struct timespec64 ts = ns_to_timespec64(ns);
+
+	*nic_major = (u32)ts.tv_sec;
+	*nic_minor = ts.tv_nsec * 4;
+}
+
+static ktime_t efx_ptp_s_qns_to_ktime_correction(u32 nic_major, u32 nic_minor,
+						 s32 correction)
+{
+	ktime_t kt;
+
+	nic_minor = DIV_ROUND_CLOSEST(nic_minor, 4);
+	correction = DIV_ROUND_CLOSEST(correction, 4);
+
+	kt = ktime_set(nic_major, nic_minor);
+
+	if (correction >= 0)
+		kt = ktime_add_ns(kt, (u64)correction);
+	else
+		kt = ktime_sub_ns(kt, (u64)-correction);
+	return kt;
+}
+
+struct efx_channel *efx_ptp_channel(struct efx_nic *efx)
+{
+	return efx->ptp_data ? efx->ptp_data->channel : NULL;
+}
+
+static u32 last_sync_timestamp_major(struct efx_nic *efx)
+{
+	struct efx_channel *channel = efx_ptp_channel(efx);
+	u32 major = 0;
+
+	if (channel)
+		major = channel->sync_timestamp_major;
+	return major;
+}
+
+/* The 8000 series and later can provide the time from the MAC, which is only
+ * 48 bits long and provides meta-information in the top 2 bits.
+ */
+static ktime_t
+efx_ptp_mac_nic_to_ktime_correction(struct efx_nic *efx,
+				    struct efx_ptp_data *ptp,
+				    u32 nic_major, u32 nic_minor,
+				    s32 correction)
+{
+	ktime_t kt = { 0 };
+
+	if (!(nic_major & 0x80000000)) {
+		WARN_ON_ONCE(nic_major >> 16);
+		/* Use the top bits from the latest sync event. */
+		nic_major &= 0xffff;
+		nic_major |= (last_sync_timestamp_major(efx) & 0xffff0000);
+
+		kt = ptp->nic_to_kernel_time(nic_major, nic_minor,
+					     correction);
+	}
+	return kt;
+}
+
+ktime_t efx_ptp_nic_to_kernel_time(struct efx_tx_queue *tx_queue)
+{
+	struct efx_nic *efx = tx_queue->efx;
+	struct efx_ptp_data *ptp = efx->ptp_data;
+	ktime_t kt;
+
+	if (efx_ptp_use_mac_tx_timestamps(efx))
+		kt = efx_ptp_mac_nic_to_ktime_correction(efx, ptp,
+				tx_queue->completed_timestamp_major,
+				tx_queue->completed_timestamp_minor,
+				ptp->ts_corrections.general_tx);
+	else
+		kt = ptp->nic_to_kernel_time(
+				tx_queue->completed_timestamp_major,
+				tx_queue->completed_timestamp_minor,
+				ptp->ts_corrections.general_tx);
+	return kt;
+}
+
 /* Get PTP attributes and set up time conversions */
 static int efx_ptp_get_attributes(struct efx_nic *efx)
 {
@@ -502,31 +627,71 @@ static int efx_ptp_get_attributes(struct efx_nic *efx)
 		return rc;
 	}
 
-	if (fmt == MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_27FRACTION) {
+	switch (fmt) {
+	case MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_27FRACTION:
 		ptp->ns_to_nic_time = efx_ptp_ns_to_s27;
 		ptp->nic_to_kernel_time = efx_ptp_s27_to_ktime_correction;
-	} else if (fmt == MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_NANOSECONDS) {
+		ptp->nic_time.minor_max = 1 << 27;
+		ptp->nic_time.sync_event_minor_shift = 19;
+		break;
+	case MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_NANOSECONDS:
 		ptp->ns_to_nic_time = efx_ptp_ns_to_s_ns;
 		ptp->nic_to_kernel_time = efx_ptp_s_ns_to_ktime_correction;
-	} else {
+		ptp->nic_time.minor_max = 1000000000;
+		ptp->nic_time.sync_event_minor_shift = 22;
+		break;
+	case MC_CMD_PTP_OUT_GET_ATTRIBUTES_SECONDS_QTR_NANOSECONDS:
+		ptp->ns_to_nic_time = efx_ptp_ns_to_s_qns;
+		ptp->nic_to_kernel_time = efx_ptp_s_qns_to_ktime_correction;
+		ptp->nic_time.minor_max = 4000000000UL;
+		ptp->nic_time.sync_event_minor_shift = 24;
+		break;
+	default:
 		return -ERANGE;
 	}
 
-	ptp->time_format = fmt;
-
-	/* MC_CMD_PTP_OP_GET_ATTRIBUTES is an extended version of an older
-	 * operation MC_CMD_PTP_OP_GET_TIME_FORMAT that also returns a value
-	 * to use for the minimum acceptable corrected synchronization window.
+	/* Precalculate acceptable difference between the minor time in the
+	 * packet prefix and the last MCDI time sync event. We expect the
+	 * packet prefix timestamp to be after of sync event by up to one
+	 * sync event interval (0.25s) but we allow it to exceed this by a
+	 * fuzz factor of (0.1s)
+	 */
+	ptp->nic_time.sync_event_diff_min = ptp->nic_time.minor_max
+		- (ptp->nic_time.minor_max / 10);
+	ptp->nic_time.sync_event_diff_max = (ptp->nic_time.minor_max / 4)
+		+ (ptp->nic_time.minor_max / 10);
+
+	/* MC_CMD_PTP_OP_GET_ATTRIBUTES has been extended twice from an older
+	 * operation MC_CMD_PTP_OP_GET_TIME_FORMAT. The function now may return
+	 * a value to use for the minimum acceptable corrected synchronization
+	 * window and may return further capabilities.
 	 * If we have the extra information store it. For older firmware that
 	 * does not implement the extended command use the default value.
 	 */
-	if (rc == 0 && out_len >= MC_CMD_PTP_OUT_GET_ATTRIBUTES_LEN)
+	if (rc == 0 &&
+	    out_len >= MC_CMD_PTP_OUT_GET_ATTRIBUTES_CAPABILITIES_OFST)
 		ptp->min_synchronisation_ns =
 			MCDI_DWORD(outbuf,
 				   PTP_OUT_GET_ATTRIBUTES_SYNC_WINDOW_MIN);
 	else
 		ptp->min_synchronisation_ns = DEFAULT_MIN_SYNCHRONISATION_NS;
 
+	if (rc == 0 &&
+	    out_len >= MC_CMD_PTP_OUT_GET_ATTRIBUTES_LEN)
+		ptp->capabilities = MCDI_DWORD(outbuf,
+					PTP_OUT_GET_ATTRIBUTES_CAPABILITIES);
+	else
+		ptp->capabilities = 0;
+
+	/* Set up the shift for conversion between frequency
+	 * adjustments in parts-per-billion and the fixed-point
+	 * fractional ns format that the adapter uses.
+	 */
+	if (ptp->capabilities & (1 << MC_CMD_PTP_OUT_GET_ATTRIBUTES_FP44_FREQ_ADJ_LBN))
+		ptp->adjfreq_ppb_shift = PPB_SHIFT_FP44;
+	else
+		ptp->adjfreq_ppb_shift = PPB_SHIFT_FP40;
+
 	return 0;
 }
 
@@ -534,8 +699,9 @@ static int efx_ptp_get_attributes(struct efx_nic *efx)
 static int efx_ptp_get_timestamp_corrections(struct efx_nic *efx)
 {
 	MCDI_DECLARE_BUF(inbuf, MC_CMD_PTP_IN_GET_TIMESTAMP_CORRECTIONS_LEN);
-	MCDI_DECLARE_BUF(outbuf, MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_LEN);
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_LEN);
 	int rc;
+	size_t out_len;
 
 	/* Get the timestamp corrections from the NIC. If this operation is
 	 * not supported (older NICs) then no correction is required.
@@ -545,21 +711,37 @@ static int efx_ptp_get_timestamp_corrections(struct efx_nic *efx)
 	MCDI_SET_DWORD(inbuf, PTP_IN_PERIPH_ID, 0);
 
 	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_PTP, inbuf, sizeof(inbuf),
-				outbuf, sizeof(outbuf), NULL);
+				outbuf, sizeof(outbuf), &out_len);
 	if (rc == 0) {
-		efx->ptp_data->ts_corrections.tx = MCDI_DWORD(outbuf,
+		efx->ptp_data->ts_corrections.ptp_tx = MCDI_DWORD(outbuf,
 			PTP_OUT_GET_TIMESTAMP_CORRECTIONS_TRANSMIT);
-		efx->ptp_data->ts_corrections.rx = MCDI_DWORD(outbuf,
+		efx->ptp_data->ts_corrections.ptp_rx = MCDI_DWORD(outbuf,
 			PTP_OUT_GET_TIMESTAMP_CORRECTIONS_RECEIVE);
 		efx->ptp_data->ts_corrections.pps_out = MCDI_DWORD(outbuf,
 			PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_OUT);
 		efx->ptp_data->ts_corrections.pps_in = MCDI_DWORD(outbuf,
 			PTP_OUT_GET_TIMESTAMP_CORRECTIONS_PPS_IN);
+
+		if (out_len >= MC_CMD_PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_LEN) {
+			efx->ptp_data->ts_corrections.general_tx = MCDI_DWORD(
+				outbuf,
+				PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_TX);
+			efx->ptp_data->ts_corrections.general_rx = MCDI_DWORD(
+				outbuf,
+				PTP_OUT_GET_TIMESTAMP_CORRECTIONS_V2_GENERAL_RX);
+		} else {
+			efx->ptp_data->ts_corrections.general_tx =
+				efx->ptp_data->ts_corrections.ptp_tx;
+			efx->ptp_data->ts_corrections.general_rx =
+				efx->ptp_data->ts_corrections.ptp_rx;
+		}
 	} else if (rc == -EINVAL) {
-		efx->ptp_data->ts_corrections.tx = 0;
-		efx->ptp_data->ts_corrections.rx = 0;
+		efx->ptp_data->ts_corrections.ptp_tx = 0;
+		efx->ptp_data->ts_corrections.ptp_rx = 0;
 		efx->ptp_data->ts_corrections.pps_out = 0;
 		efx->ptp_data->ts_corrections.pps_in = 0;
+		efx->ptp_data->ts_corrections.general_tx = 0;
+		efx->ptp_data->ts_corrections.general_rx = 0;
 	} else {
 		efx_mcdi_display_error(efx, MC_CMD_PTP, sizeof(inbuf), outbuf,
 				       sizeof(outbuf), rc);
@@ -873,8 +1055,24 @@ static int efx_ptp_synchronize(struct efx_nic *efx, unsigned int num_readings)
 	return rc;
 }
 
+/* Transmit a PTP packet via the dedicated hardware timestamped queue. */
+static void efx_ptp_xmit_skb_queue(struct efx_nic *efx, struct sk_buff *skb)
+{
+	struct efx_ptp_data *ptp_data = efx->ptp_data;
+	struct efx_tx_queue *tx_queue;
+	u8 type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0;
+
+	tx_queue = &ptp_data->channel->tx_queue[type];
+	if (tx_queue && tx_queue->timestamping) {
+		efx_enqueue_skb(tx_queue, skb);
+	} else {
+		WARN_ONCE(1, "PTP channel has no timestamped tx queue\n");
+		dev_kfree_skb_any(skb);
+	}
+}
+
 /* Transmit a PTP packet, via the MCDI interface, to the wire. */
-static int efx_ptp_xmit_skb(struct efx_nic *efx, struct sk_buff *skb)
+static void efx_ptp_xmit_skb_mc(struct efx_nic *efx, struct sk_buff *skb)
 {
 	struct efx_ptp_data *ptp_data = efx->ptp_data;
 	struct skb_shared_hwtstamps timestamps;
@@ -910,16 +1108,16 @@ static int efx_ptp_xmit_skb(struct efx_nic *efx, struct sk_buff *skb)
 	timestamps.hwtstamp = ptp_data->nic_to_kernel_time(
 		MCDI_DWORD(txtime, PTP_OUT_TRANSMIT_MAJOR),
 		MCDI_DWORD(txtime, PTP_OUT_TRANSMIT_MINOR),
-		ptp_data->ts_corrections.tx);
+		ptp_data->ts_corrections.ptp_tx);
 
 	skb_tstamp_tx(skb, &timestamps);
 
 	rc = 0;
 
 fail:
-	dev_kfree_skb(skb);
+	dev_kfree_skb_any(skb);
 
-	return rc;
+	return;
 }
 
 static void efx_ptp_drop_time_expired_events(struct efx_nic *efx)
@@ -1189,7 +1387,7 @@ static void efx_ptp_worker(struct work_struct *work)
 	efx_ptp_process_events(efx, &tempq);
 
 	while ((skb = skb_dequeue(&ptp_data->txq)))
-		efx_ptp_xmit_skb(efx, skb);
+		ptp_data->xmit_skb(efx, skb);
 
 	while ((skb = __skb_dequeue(&tempq)))
 		efx_ptp_process_rx(efx, skb);
@@ -1239,6 +1437,14 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel)
 		goto fail2;
 	}
 
+	if (efx_ptp_use_mac_tx_timestamps(efx)) {
+		ptp->xmit_skb = efx_ptp_xmit_skb_queue;
+		/* Request sync events on this channel. */
+		channel->sync_events_state = SYNC_EVENTS_QUIESCENT;
+	} else {
+		ptp->xmit_skb = efx_ptp_xmit_skb_mc;
+	}
+
 	INIT_WORK(&ptp->work, efx_ptp_worker);
 	ptp->config.flags = 0;
 	ptp->config.tx_type = HWTSTAMP_TX_OFF;
@@ -1303,11 +1509,21 @@ int efx_ptp_probe(struct efx_nic *efx, struct efx_channel *channel)
 static int efx_ptp_probe_channel(struct efx_channel *channel)
 {
 	struct efx_nic *efx = channel->efx;
+	int rc;
 
 	channel->irq_moderation_us = 0;
 	channel->rx_queue.core_index = 0;
 
-	return efx_ptp_probe(efx, channel);
+	rc = efx_ptp_probe(efx, channel);
+	/* Failure to probe PTP is not fatal; this channel will just not be
+	 * used for anything.
+	 * In the case of EPERM, efx_ptp_probe will print its own message (in
+	 * efx_ptp_get_attributes()), so we don't need to.
+	 */
+	if (rc && rc != -EPERM)
+		netif_warn(efx, drv, efx->net_dev,
+			   "Failed to probe PTP, rc=%d\n", rc);
+	return 0;
 }
 
 void efx_ptp_remove(struct efx_nic *efx)
@@ -1332,6 +1548,7 @@ void efx_ptp_remove(struct efx_nic *efx)
 
 	efx_nic_free_buffer(efx, &efx->ptp_data->start);
 	kfree(efx->ptp_data);
+	efx->ptp_data = NULL;
 }
 
 static void efx_ptp_remove_channel(struct efx_channel *channel)
@@ -1548,6 +1765,17 @@ void efx_ptp_get_ts_info(struct efx_nic *efx, struct ethtool_ts_info *ts_info)
 	ts_info->so_timestamping |= (SOF_TIMESTAMPING_TX_HARDWARE |
 				     SOF_TIMESTAMPING_RX_HARDWARE |
 				     SOF_TIMESTAMPING_RAW_HARDWARE);
+	/* Check licensed features.  If we don't have the license for TX
+	 * timestamps, the NIC will not support them.
+	 */
+	if (efx_ptp_use_mac_tx_timestamps(efx)) {
+		struct efx_ef10_nic_data *nic_data = efx->nic_data;
+
+		if (!(nic_data->licensed_features &
+		      (1 << LICENSED_V3_FEATURES_TX_TIMESTAMPS_LBN)))
+			ts_info->so_timestamping &=
+				~SOF_TIMESTAMPING_TX_HARDWARE;
+	}
 	if (primary && primary->ptp_data && primary->ptp_data->phc_clock)
 		ts_info->phc_index =
 			ptp_clock_index(primary->ptp_data->phc_clock);
@@ -1627,7 +1855,7 @@ static void ptp_event_rx(struct efx_nic *efx, struct efx_ptp_data *ptp)
 		evt->hwtimestamp = efx->ptp_data->nic_to_kernel_time(
 			EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA),
 			EFX_QWORD_FIELD(ptp->evt_frags[1], MCDI_EVENT_DATA),
-			ptp->ts_corrections.rx);
+			ptp->ts_corrections.ptp_rx);
 		evt->expiry = jiffies + msecs_to_jiffies(PKT_EVENT_LIFETIME_MS);
 		list_add_tail(&evt->link, &ptp->evt_list);
 
@@ -1662,9 +1890,11 @@ void efx_ptp_event(struct efx_nic *efx, efx_qword_t *ev)
 	int code = EFX_QWORD_FIELD(*ev, MCDI_EVENT_CODE);
 
 	if (!ptp) {
-		if (net_ratelimit())
+		if (!efx->ptp_warned) {
 			netif_warn(efx, drv, efx->net_dev,
 				   "Received PTP event but PTP not set up\n");
+			efx->ptp_warned = true;
+		}
 		return;
 	}
 
@@ -1707,9 +1937,20 @@ void efx_ptp_event(struct efx_nic *efx, efx_qword_t *ev)
 
 void efx_time_sync_event(struct efx_channel *channel, efx_qword_t *ev)
 {
+	struct efx_nic *efx = channel->efx;
+	struct efx_ptp_data *ptp = efx->ptp_data;
+
+	/* When extracting the sync timestamp minor value, we should discard
+	 * the least significant two bits. These are not required in order
+	 * to reconstruct full-range timestamps and they are optionally used
+	 * to report status depending on the options supplied when subscribing
+	 * for sync events.
+	 */
 	channel->sync_timestamp_major = MCDI_EVENT_FIELD(*ev, PTP_TIME_MAJOR);
 	channel->sync_timestamp_minor =
-		MCDI_EVENT_FIELD(*ev, PTP_TIME_MINOR_26_19) << 19;
+		(MCDI_EVENT_FIELD(*ev, PTP_TIME_MINOR_MS_8BITS) & 0xFC)
+			<< ptp->nic_time.sync_event_minor_shift;
+
 	/* if sync events have been disabled then we want to silently ignore
 	 * this event, so throw away result.
 	 */
@@ -1717,15 +1958,6 @@ void efx_time_sync_event(struct efx_channel *channel, efx_qword_t *ev)
 		       SYNC_EVENTS_VALID);
 }
 
-/* make some assumptions about the time representation rather than abstract it,
- * since we currently only support one type of inline timestamping and only on
- * EF10.
- */
-#define MINOR_TICKS_PER_SECOND 0x8000000
-/* Fuzz factor for sync events to be out of order with RX events */
-#define FUZZ (MINOR_TICKS_PER_SECOND / 10)
-#define EXPECTED_SYNC_EVENTS_PER_SECOND 4
-
 static inline u32 efx_rx_buf_timestamp_minor(struct efx_nic *efx, const u8 *eh)
 {
 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
@@ -1743,31 +1975,33 @@ void __efx_rx_skb_attach_timestamp(struct efx_channel *channel,
 				   struct sk_buff *skb)
 {
 	struct efx_nic *efx = channel->efx;
+	struct efx_ptp_data *ptp = efx->ptp_data;
 	u32 pkt_timestamp_major, pkt_timestamp_minor;
 	u32 diff, carry;
 	struct skb_shared_hwtstamps *timestamps;
 
-	pkt_timestamp_minor = (efx_rx_buf_timestamp_minor(efx,
-							  skb_mac_header(skb)) +
-			       (u32) efx->ptp_data->ts_corrections.rx) &
-			      (MINOR_TICKS_PER_SECOND - 1);
+	if (channel->sync_events_state != SYNC_EVENTS_VALID)
+		return;
+
+	pkt_timestamp_minor = efx_rx_buf_timestamp_minor(efx, skb_mac_header(skb));
 
 	/* get the difference between the packet and sync timestamps,
 	 * modulo one second
 	 */
-	diff = (pkt_timestamp_minor - channel->sync_timestamp_minor) &
-		(MINOR_TICKS_PER_SECOND - 1);
+	diff = pkt_timestamp_minor - channel->sync_timestamp_minor;
+	if (pkt_timestamp_minor < channel->sync_timestamp_minor)
+		diff += ptp->nic_time.minor_max;
+
 	/* do we roll over a second boundary and need to carry the one? */
-	carry = channel->sync_timestamp_minor + diff > MINOR_TICKS_PER_SECOND ?
+	carry = (channel->sync_timestamp_minor >= ptp->nic_time.minor_max - diff) ?
 		1 : 0;
 
-	if (diff <= MINOR_TICKS_PER_SECOND / EXPECTED_SYNC_EVENTS_PER_SECOND +
-		    FUZZ) {
+	if (diff <= ptp->nic_time.sync_event_diff_max) {
 		/* packet is ahead of the sync event by a quarter of a second or
 		 * less (allowing for fuzz)
 		 */
 		pkt_timestamp_major = channel->sync_timestamp_major + carry;
-	} else if (diff >= MINOR_TICKS_PER_SECOND - FUZZ) {
+	} else if (diff >= ptp->nic_time.sync_event_diff_min) {
 		/* packet is behind the sync event but within the fuzz factor.
 		 * This means the RX packet and sync event crossed as they were
 		 * placed on the event queue, which can sometimes happen.
@@ -1789,7 +2023,9 @@ void __efx_rx_skb_attach_timestamp(struct efx_channel *channel,
 	/* attach the timestamps to the skb */
 	timestamps = skb_hwtstamps(skb);
 	timestamps->hwtstamp =
-		efx_ptp_s27_to_ktime(pkt_timestamp_major, pkt_timestamp_minor);
+		ptp->nic_to_kernel_time(pkt_timestamp_major,
+					pkt_timestamp_minor,
+					ptp->ts_corrections.general_rx);
 }
 
 static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta)
@@ -1807,9 +2043,10 @@ static int efx_phc_adjfreq(struct ptp_clock_info *ptp, s32 delta)
 	else if (delta < -MAX_PPB)
 		delta = -MAX_PPB;
 
-	/* Convert ppb to fixed point ns. */
-	adjustment_ns = (((s64)delta * PPB_SCALE_WORD) >>
-			 (PPB_EXTRA_BITS + MAX_PPB_BITS));
+	/* Convert ppb to fixed point ns taking care to round correctly. */
+	adjustment_ns = ((s64)delta * PPB_SCALE_WORD +
+			 (1 << (ptp_data->adjfreq_ppb_shift - 1))) >>
+			ptp_data->adjfreq_ppb_shift;
 
 	MCDI_SET_DWORD(inadj, PTP_IN_OP, MC_CMD_PTP_OP_ADJUST);
 	MCDI_SET_DWORD(inadj, PTP_IN_PERIPH_ID, 0);
@@ -1916,6 +2153,7 @@ static const struct efx_channel_type efx_ptp_channel_type = {
 	.get_name		= efx_ptp_get_channel_name,
 	/* no copy operation; there is no need to reallocate this channel */
 	.receive_skb		= efx_ptp_rx,
+	.want_txqs		= efx_ptp_want_txqs,
 	.keep_eventq		= false,
 };
 
diff --git a/drivers/net/ethernet/sfc/siena.c b/drivers/net/ethernet/sfc/siena.c
index a617f657eae3..ae8645ae4492 100644
--- a/drivers/net/ethernet/sfc/siena.c
+++ b/drivers/net/ethernet/sfc/siena.c
@@ -242,6 +242,14 @@ static int siena_dimension_resources(struct efx_nic *efx)
 	return 0;
 }
 
+/* On all Falcon-architecture NICs, PFs use BAR 0 for I/O space and BAR 2(&3)
+ * for memory.
+ */
+static unsigned int siena_mem_bar(struct efx_nic *efx)
+{
+	return 2;
+}
+
 static unsigned int siena_mem_map_size(struct efx_nic *efx)
 {
 	return FR_CZ_MC_TREG_SMEM +
@@ -547,7 +555,7 @@ static int siena_try_update_nic_stats(struct efx_nic *efx)
 
 	dma_stats = efx->stats_buffer.addr;
 
-	generation_end = dma_stats[MC_CMD_MAC_GENERATION_END];
+	generation_end = dma_stats[efx->num_mac_stats - 1];
 	if (generation_end == EFX_MC_STATS_GENERATION_INVALID)
 		return 0;
 	rmb();
@@ -950,7 +958,7 @@ static int siena_mtd_probe(struct efx_nic *efx)
 
 const struct efx_nic_type siena_a0_nic_type = {
 	.is_vf = false,
-	.mem_bar = EFX_MEM_BAR,
+	.mem_bar = siena_mem_bar,
 	.mem_map_size = siena_mem_map_size,
 	.probe = siena_probe_nic,
 	.remove = siena_remove_nic,
diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c
index 9937a2450e57..cece961f2e82 100644
--- a/drivers/net/ethernet/sfc/tx.c
+++ b/drivers/net/ethernet/sfc/tx.c
@@ -77,9 +77,23 @@ static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
 	}
 
 	if (buffer->flags & EFX_TX_BUF_SKB) {
+		struct sk_buff *skb = (struct sk_buff *)buffer->skb;
+
 		EFX_WARN_ON_PARANOID(!pkts_compl || !bytes_compl);
 		(*pkts_compl)++;
-		(*bytes_compl) += buffer->skb->len;
+		(*bytes_compl) += skb->len;
+		if (tx_queue->timestamping &&
+		    (tx_queue->completed_timestamp_major ||
+		     tx_queue->completed_timestamp_minor)) {
+			struct skb_shared_hwtstamps hwtstamp;
+
+			hwtstamp.hwtstamp =
+				efx_ptp_nic_to_kernel_time(tx_queue);
+			skb_tstamp_tx(skb, &hwtstamp);
+
+			tx_queue->completed_timestamp_major = 0;
+			tx_queue->completed_timestamp_minor = 0;
+		}
 		dev_consume_skb_any((struct sk_buff *)buffer->skb);
 		netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,
 			   "TX queue %d transmission id %x complete\n",
@@ -828,6 +842,11 @@ void efx_init_tx_queue(struct efx_tx_queue *tx_queue)
 	tx_queue->old_read_count = 0;
 	tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID;
 	tx_queue->xmit_more_available = false;
+	tx_queue->timestamping = (efx_ptp_use_mac_tx_timestamps(efx) &&
+				  tx_queue->channel == efx_ptp_channel(efx));
+	tx_queue->completed_desc_ptr = tx_queue->ptr_mask;
+	tx_queue->completed_timestamp_major = 0;
+	tx_queue->completed_timestamp_minor = 0;
 
 	/* Set up default function pointers. These may get replaced by
 	 * efx_nic_init_tx() based off NIC/queue capabilities.
diff --git a/drivers/net/ethernet/smsc/Kconfig b/drivers/net/ethernet/smsc/Kconfig
index 63aca9f847e1..4c2f612e4414 100644
--- a/drivers/net/ethernet/smsc/Kconfig
+++ b/drivers/net/ethernet/smsc/Kconfig
@@ -20,7 +20,7 @@ if NET_VENDOR_SMSC
 
 config SMC9194
 	tristate "SMC 9194 support"
-	depends on (ISA || MAC && BROKEN)
+	depends on ISA
 	select CRC32
 	---help---
 	  This is support for the SMC9xxx based Ethernet cards. Choose this
diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c
index 012fb66eed8d..f0afb88d7bc2 100644
--- a/drivers/net/ethernet/smsc/smsc911x.c
+++ b/drivers/net/ethernet/smsc/smsc911x.c
@@ -2335,14 +2335,14 @@ static int smsc911x_drv_remove(struct platform_device *pdev)
 	pdata = netdev_priv(dev);
 	BUG_ON(!pdata);
 	BUG_ON(!pdata->ioaddr);
-	WARN_ON(dev->phydev);
 
 	SMSC_TRACE(pdata, ifdown, "Stopping driver");
 
+	unregister_netdev(dev);
+
 	mdiobus_unregister(pdata->mii_bus);
 	mdiobus_free(pdata->mii_bus);
 
-	unregister_netdev(dev);
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "smsc911x-memory");
 	if (!res)
diff --git a/drivers/net/ethernet/socionext/Kconfig b/drivers/net/ethernet/socionext/Kconfig
new file mode 100644
index 000000000000..6bcfe27fc560
--- /dev/null
+++ b/drivers/net/ethernet/socionext/Kconfig
@@ -0,0 +1,34 @@
+config NET_VENDOR_SOCIONEXT
+	bool "Socionext ethernet drivers"
+	default y
+	---help---
+	  Option to select ethernet drivers for Socionext platforms.
+
+	  Note that the answer to this question doesn't directly affect the
+	  kernel: saying N will just cause the configurator to skip all
+	  the questions about Socionext devices. If you say Y, you will be asked
+	  for your specific card in the following questions.
+
+if NET_VENDOR_SOCIONEXT
+
+config SNI_AVE
+	tristate "Socionext AVE ethernet support"
+	depends on (ARCH_UNIPHIER || COMPILE_TEST) && OF
+	select PHYLIB
+	---help---
+	  Driver for gigabit ethernet MACs, called AVE, in the
+	  Socionext UniPhier family.
+
+config SNI_NETSEC
+	tristate "Socionext NETSEC ethernet support"
+	depends on (ARCH_SYNQUACER || COMPILE_TEST) && OF
+	select PHYLIB
+	select MII
+	---help---
+	  Enable to add support for the SocioNext NetSec Gigabit Ethernet
+	  controller + PHY, as found on the Synquacer SC2A11 SoC
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called netsec.  If unsure, say N.
+
+endif #NET_VENDOR_SOCIONEXT
diff --git a/drivers/net/ethernet/socionext/Makefile b/drivers/net/ethernet/socionext/Makefile
new file mode 100644
index 000000000000..7fd837a999fd
--- /dev/null
+++ b/drivers/net/ethernet/socionext/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for all ethernet ip drivers on Socionext platforms
+#
+obj-$(CONFIG_SNI_AVE) += sni_ave.o
+obj-$(CONFIG_SNI_NETSEC) += netsec.o
diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c
new file mode 100644
index 000000000000..f4c0b02ddad8
--- /dev/null
+++ b/drivers/net/ethernet/socionext/netsec.c
@@ -0,0 +1,1777 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/types.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/acpi.h>
+#include <linux/of_mdio.h>
+#include <linux/etherdevice.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+
+#include <net/tcp.h>
+#include <net/ip6_checksum.h>
+
+#define NETSEC_REG_SOFT_RST			0x104
+#define NETSEC_REG_COM_INIT			0x120
+
+#define NETSEC_REG_TOP_STATUS			0x200
+#define NETSEC_IRQ_RX				BIT(1)
+#define NETSEC_IRQ_TX				BIT(0)
+
+#define NETSEC_REG_TOP_INTEN			0x204
+#define NETSEC_REG_INTEN_SET			0x234
+#define NETSEC_REG_INTEN_CLR			0x238
+
+#define NETSEC_REG_NRM_TX_STATUS		0x400
+#define NETSEC_REG_NRM_TX_INTEN			0x404
+#define NETSEC_REG_NRM_TX_INTEN_SET		0x428
+#define NETSEC_REG_NRM_TX_INTEN_CLR		0x42c
+#define NRM_TX_ST_NTOWNR	BIT(17)
+#define NRM_TX_ST_TR_ERR	BIT(16)
+#define NRM_TX_ST_TXDONE	BIT(15)
+#define NRM_TX_ST_TMREXP	BIT(14)
+
+#define NETSEC_REG_NRM_RX_STATUS		0x440
+#define NETSEC_REG_NRM_RX_INTEN			0x444
+#define NETSEC_REG_NRM_RX_INTEN_SET		0x468
+#define NETSEC_REG_NRM_RX_INTEN_CLR		0x46c
+#define NRM_RX_ST_RC_ERR	BIT(16)
+#define NRM_RX_ST_PKTCNT	BIT(15)
+#define NRM_RX_ST_TMREXP	BIT(14)
+
+#define NETSEC_REG_PKT_CMD_BUF			0xd0
+
+#define NETSEC_REG_CLK_EN			0x100
+
+#define NETSEC_REG_PKT_CTRL			0x140
+
+#define NETSEC_REG_DMA_TMR_CTRL			0x20c
+#define NETSEC_REG_F_TAIKI_MC_VER		0x22c
+#define NETSEC_REG_F_TAIKI_VER			0x230
+#define NETSEC_REG_DMA_HM_CTRL			0x214
+#define NETSEC_REG_DMA_MH_CTRL			0x220
+#define NETSEC_REG_ADDR_DIS_CORE		0x218
+#define NETSEC_REG_DMAC_HM_CMD_BUF		0x210
+#define NETSEC_REG_DMAC_MH_CMD_BUF		0x21c
+
+#define NETSEC_REG_NRM_TX_PKTCNT		0x410
+
+#define NETSEC_REG_NRM_TX_DONE_PKTCNT		0x414
+#define NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT	0x418
+
+#define NETSEC_REG_NRM_TX_TMR			0x41c
+
+#define NETSEC_REG_NRM_RX_PKTCNT		0x454
+#define NETSEC_REG_NRM_RX_RXINT_PKTCNT		0x458
+#define NETSEC_REG_NRM_TX_TXINT_TMR		0x420
+#define NETSEC_REG_NRM_RX_RXINT_TMR		0x460
+
+#define NETSEC_REG_NRM_RX_TMR			0x45c
+
+#define NETSEC_REG_NRM_TX_DESC_START_UP		0x434
+#define NETSEC_REG_NRM_TX_DESC_START_LW		0x408
+#define NETSEC_REG_NRM_RX_DESC_START_UP		0x474
+#define NETSEC_REG_NRM_RX_DESC_START_LW		0x448
+
+#define NETSEC_REG_NRM_TX_CONFIG		0x430
+#define NETSEC_REG_NRM_RX_CONFIG		0x470
+
+#define MAC_REG_STATUS				0x1024
+#define MAC_REG_DATA				0x11c0
+#define MAC_REG_CMD				0x11c4
+#define MAC_REG_FLOW_TH				0x11cc
+#define MAC_REG_INTF_SEL			0x11d4
+#define MAC_REG_DESC_INIT			0x11fc
+#define MAC_REG_DESC_SOFT_RST			0x1204
+#define NETSEC_REG_MODE_TRANS_COMP_STATUS	0x500
+
+#define GMAC_REG_MCR				0x0000
+#define GMAC_REG_MFFR				0x0004
+#define GMAC_REG_GAR				0x0010
+#define GMAC_REG_GDR				0x0014
+#define GMAC_REG_FCR				0x0018
+#define GMAC_REG_BMR				0x1000
+#define GMAC_REG_RDLAR				0x100c
+#define GMAC_REG_TDLAR				0x1010
+#define GMAC_REG_OMR				0x1018
+
+#define MHZ(n)		((n) * 1000 * 1000)
+
+#define NETSEC_TX_SHIFT_OWN_FIELD		31
+#define NETSEC_TX_SHIFT_LD_FIELD		30
+#define NETSEC_TX_SHIFT_DRID_FIELD		24
+#define NETSEC_TX_SHIFT_PT_FIELD		21
+#define NETSEC_TX_SHIFT_TDRID_FIELD		16
+#define NETSEC_TX_SHIFT_CC_FIELD		15
+#define NETSEC_TX_SHIFT_FS_FIELD		9
+#define NETSEC_TX_LAST				8
+#define NETSEC_TX_SHIFT_CO			7
+#define NETSEC_TX_SHIFT_SO			6
+#define NETSEC_TX_SHIFT_TRS_FIELD		4
+
+#define NETSEC_RX_PKT_OWN_FIELD			31
+#define NETSEC_RX_PKT_LD_FIELD			30
+#define NETSEC_RX_PKT_SDRID_FIELD		24
+#define NETSEC_RX_PKT_FR_FIELD			23
+#define NETSEC_RX_PKT_ER_FIELD			21
+#define NETSEC_RX_PKT_ERR_FIELD			16
+#define NETSEC_RX_PKT_TDRID_FIELD		12
+#define NETSEC_RX_PKT_FS_FIELD			9
+#define NETSEC_RX_PKT_LS_FIELD			8
+#define NETSEC_RX_PKT_CO_FIELD			6
+
+#define NETSEC_RX_PKT_ERR_MASK			3
+
+#define NETSEC_MAX_TX_PKT_LEN			1518
+#define NETSEC_MAX_TX_JUMBO_PKT_LEN		9018
+
+#define NETSEC_RING_GMAC			15
+#define NETSEC_RING_MAX				2
+
+#define NETSEC_TCP_SEG_LEN_MAX			1460
+#define NETSEC_TCP_JUMBO_SEG_LEN_MAX		8960
+
+#define NETSEC_RX_CKSUM_NOTAVAIL		0
+#define NETSEC_RX_CKSUM_OK			1
+#define NETSEC_RX_CKSUM_NG			2
+
+#define NETSEC_TOP_IRQ_REG_CODE_LOAD_END	BIT(20)
+#define NETSEC_IRQ_TRANSITION_COMPLETE		BIT(4)
+
+#define NETSEC_MODE_TRANS_COMP_IRQ_N2T		BIT(20)
+#define NETSEC_MODE_TRANS_COMP_IRQ_T2N		BIT(19)
+
+#define NETSEC_INT_PKTCNT_MAX			2047
+
+#define NETSEC_FLOW_START_TH_MAX		95
+#define NETSEC_FLOW_STOP_TH_MAX			95
+#define NETSEC_FLOW_PAUSE_TIME_MIN		5
+
+#define NETSEC_CLK_EN_REG_DOM_ALL		0x3f
+
+#define NETSEC_PKT_CTRL_REG_MODE_NRM		BIT(28)
+#define NETSEC_PKT_CTRL_REG_EN_JUMBO		BIT(27)
+#define NETSEC_PKT_CTRL_REG_LOG_CHKSUM_ER	BIT(3)
+#define NETSEC_PKT_CTRL_REG_LOG_HD_INCOMPLETE	BIT(2)
+#define NETSEC_PKT_CTRL_REG_LOG_HD_ER		BIT(1)
+#define NETSEC_PKT_CTRL_REG_DRP_NO_MATCH	BIT(0)
+
+#define NETSEC_CLK_EN_REG_DOM_G			BIT(5)
+#define NETSEC_CLK_EN_REG_DOM_C			BIT(1)
+#define NETSEC_CLK_EN_REG_DOM_D			BIT(0)
+
+#define NETSEC_COM_INIT_REG_DB			BIT(2)
+#define NETSEC_COM_INIT_REG_CLS			BIT(1)
+#define NETSEC_COM_INIT_REG_ALL			(NETSEC_COM_INIT_REG_CLS | \
+						 NETSEC_COM_INIT_REG_DB)
+
+#define NETSEC_SOFT_RST_REG_RESET		0
+#define NETSEC_SOFT_RST_REG_RUN			BIT(31)
+
+#define NETSEC_DMA_CTRL_REG_STOP		1
+#define MH_CTRL__MODE_TRANS			BIT(20)
+
+#define NETSEC_GMAC_CMD_ST_READ			0
+#define NETSEC_GMAC_CMD_ST_WRITE		BIT(28)
+#define NETSEC_GMAC_CMD_ST_BUSY			BIT(31)
+
+#define NETSEC_GMAC_BMR_REG_COMMON		0x00412080
+#define NETSEC_GMAC_BMR_REG_RESET		0x00020181
+#define NETSEC_GMAC_BMR_REG_SWR			0x00000001
+
+#define NETSEC_GMAC_OMR_REG_ST			BIT(13)
+#define NETSEC_GMAC_OMR_REG_SR			BIT(1)
+
+#define NETSEC_GMAC_MCR_REG_IBN			BIT(30)
+#define NETSEC_GMAC_MCR_REG_CST			BIT(25)
+#define NETSEC_GMAC_MCR_REG_JE			BIT(20)
+#define NETSEC_MCR_PS				BIT(15)
+#define NETSEC_GMAC_MCR_REG_FES			BIT(14)
+#define NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON	0x0000280c
+#define NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON	0x0001a00c
+
+#define NETSEC_FCR_RFE				BIT(2)
+#define NETSEC_FCR_TFE				BIT(1)
+
+#define NETSEC_GMAC_GAR_REG_GW			BIT(1)
+#define NETSEC_GMAC_GAR_REG_GB			BIT(0)
+
+#define NETSEC_GMAC_GAR_REG_SHIFT_PA		11
+#define NETSEC_GMAC_GAR_REG_SHIFT_GR		6
+#define GMAC_REG_SHIFT_CR_GAR			2
+
+#define NETSEC_GMAC_GAR_REG_CR_25_35_MHZ	2
+#define NETSEC_GMAC_GAR_REG_CR_35_60_MHZ	3
+#define NETSEC_GMAC_GAR_REG_CR_60_100_MHZ	0
+#define NETSEC_GMAC_GAR_REG_CR_100_150_MHZ	1
+#define NETSEC_GMAC_GAR_REG_CR_150_250_MHZ	4
+#define NETSEC_GMAC_GAR_REG_CR_250_300_MHZ	5
+
+#define NETSEC_GMAC_RDLAR_REG_COMMON		0x18000
+#define NETSEC_GMAC_TDLAR_REG_COMMON		0x1c000
+
+#define NETSEC_REG_NETSEC_VER_F_TAIKI		0x50000
+
+#define NETSEC_REG_DESC_RING_CONFIG_CFG_UP	BIT(31)
+#define NETSEC_REG_DESC_RING_CONFIG_CH_RST	BIT(30)
+#define NETSEC_REG_DESC_TMR_MODE		4
+#define NETSEC_REG_DESC_ENDIAN			0
+
+#define NETSEC_MAC_DESC_SOFT_RST_SOFT_RST	1
+#define NETSEC_MAC_DESC_INIT_REG_INIT		1
+
+#define NETSEC_EEPROM_MAC_ADDRESS		0x00
+#define NETSEC_EEPROM_HM_ME_ADDRESS_H		0x08
+#define NETSEC_EEPROM_HM_ME_ADDRESS_L		0x0C
+#define NETSEC_EEPROM_HM_ME_SIZE		0x10
+#define NETSEC_EEPROM_MH_ME_ADDRESS_H		0x14
+#define NETSEC_EEPROM_MH_ME_ADDRESS_L		0x18
+#define NETSEC_EEPROM_MH_ME_SIZE		0x1C
+#define NETSEC_EEPROM_PKT_ME_ADDRESS		0x20
+#define NETSEC_EEPROM_PKT_ME_SIZE		0x24
+
+#define DESC_NUM	128
+#define NAPI_BUDGET	(DESC_NUM / 2)
+
+#define DESC_SZ	sizeof(struct netsec_de)
+
+#define NETSEC_F_NETSEC_VER_MAJOR_NUM(x)	((x) & 0xffff0000)
+
+enum ring_id {
+	NETSEC_RING_TX = 0,
+	NETSEC_RING_RX
+};
+
+struct netsec_desc {
+	struct sk_buff *skb;
+	dma_addr_t dma_addr;
+	void *addr;
+	u16 len;
+};
+
+struct netsec_desc_ring {
+	dma_addr_t desc_dma;
+	struct netsec_desc *desc;
+	void *vaddr;
+	u16 pkt_cnt;
+	u16 head, tail;
+};
+
+struct netsec_priv {
+	struct netsec_desc_ring desc_ring[NETSEC_RING_MAX];
+	struct ethtool_coalesce et_coalesce;
+	spinlock_t reglock; /* protect reg access */
+	struct napi_struct napi;
+	phy_interface_t phy_interface;
+	struct net_device *ndev;
+	struct device_node *phy_np;
+	struct phy_device *phydev;
+	struct mii_bus *mii_bus;
+	void __iomem *ioaddr;
+	void __iomem *eeprom_base;
+	struct device *dev;
+	struct clk *clk;
+	u32 msg_enable;
+	u32 freq;
+	bool rx_cksum_offload_flag;
+};
+
+struct netsec_de { /* Netsec Descriptor layout */
+	u32 attr;
+	u32 data_buf_addr_up;
+	u32 data_buf_addr_lw;
+	u32 buf_len_info;
+};
+
+struct netsec_tx_pkt_ctrl {
+	u16 tcp_seg_len;
+	bool tcp_seg_offload_flag;
+	bool cksum_offload_flag;
+};
+
+struct netsec_rx_pkt_info {
+	int rx_cksum_result;
+	int err_code;
+	bool err_flag;
+};
+
+static void netsec_write(struct netsec_priv *priv, u32 reg_addr, u32 val)
+{
+	writel(val, priv->ioaddr + reg_addr);
+}
+
+static u32 netsec_read(struct netsec_priv *priv, u32 reg_addr)
+{
+	return readl(priv->ioaddr + reg_addr);
+}
+
+/************* MDIO BUS OPS FOLLOW *************/
+
+#define TIMEOUT_SPINS_MAC		1000
+#define TIMEOUT_SECONDARY_MS_MAC	100
+
+static u32 netsec_clk_type(u32 freq)
+{
+	if (freq < MHZ(35))
+		return NETSEC_GMAC_GAR_REG_CR_25_35_MHZ;
+	if (freq < MHZ(60))
+		return NETSEC_GMAC_GAR_REG_CR_35_60_MHZ;
+	if (freq < MHZ(100))
+		return NETSEC_GMAC_GAR_REG_CR_60_100_MHZ;
+	if (freq < MHZ(150))
+		return NETSEC_GMAC_GAR_REG_CR_100_150_MHZ;
+	if (freq < MHZ(250))
+		return NETSEC_GMAC_GAR_REG_CR_150_250_MHZ;
+
+	return NETSEC_GMAC_GAR_REG_CR_250_300_MHZ;
+}
+
+static int netsec_wait_while_busy(struct netsec_priv *priv, u32 addr, u32 mask)
+{
+	u32 timeout = TIMEOUT_SPINS_MAC;
+
+	while (--timeout && netsec_read(priv, addr) & mask)
+		cpu_relax();
+	if (timeout)
+		return 0;
+
+	timeout = TIMEOUT_SECONDARY_MS_MAC;
+	while (--timeout && netsec_read(priv, addr) & mask)
+		usleep_range(1000, 2000);
+
+	if (timeout)
+		return 0;
+
+	netdev_WARN(priv->ndev, "%s: timeout\n", __func__);
+
+	return -ETIMEDOUT;
+}
+
+static int netsec_mac_write(struct netsec_priv *priv, u32 addr, u32 value)
+{
+	netsec_write(priv, MAC_REG_DATA, value);
+	netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_WRITE);
+	return netsec_wait_while_busy(priv,
+				      MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY);
+}
+
+static int netsec_mac_read(struct netsec_priv *priv, u32 addr, u32 *read)
+{
+	int ret;
+
+	netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_READ);
+	ret = netsec_wait_while_busy(priv,
+				     MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY);
+	if (ret)
+		return ret;
+
+	*read = netsec_read(priv, MAC_REG_DATA);
+
+	return 0;
+}
+
+static int netsec_mac_wait_while_busy(struct netsec_priv *priv,
+				      u32 addr, u32 mask)
+{
+	u32 timeout = TIMEOUT_SPINS_MAC;
+	int ret, data;
+
+	do {
+		ret = netsec_mac_read(priv, addr, &data);
+		if (ret)
+			break;
+		cpu_relax();
+	} while (--timeout && (data & mask));
+
+	if (timeout)
+		return 0;
+
+	timeout = TIMEOUT_SECONDARY_MS_MAC;
+	do {
+		usleep_range(1000, 2000);
+
+		ret = netsec_mac_read(priv, addr, &data);
+		if (ret)
+			break;
+		cpu_relax();
+	} while (--timeout && (data & mask));
+
+	if (timeout && !ret)
+		return 0;
+
+	netdev_WARN(priv->ndev, "%s: timeout\n", __func__);
+
+	return -ETIMEDOUT;
+}
+
+static int netsec_mac_update_to_phy_state(struct netsec_priv *priv)
+{
+	struct phy_device *phydev = priv->ndev->phydev;
+	u32 value = 0;
+
+	value = phydev->duplex ? NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON :
+				 NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON;
+
+	if (phydev->speed != SPEED_1000)
+		value |= NETSEC_MCR_PS;
+
+	if (priv->phy_interface != PHY_INTERFACE_MODE_GMII &&
+	    phydev->speed == SPEED_100)
+		value |= NETSEC_GMAC_MCR_REG_FES;
+
+	value |= NETSEC_GMAC_MCR_REG_CST | NETSEC_GMAC_MCR_REG_JE;
+
+	if (phy_interface_mode_is_rgmii(priv->phy_interface))
+		value |= NETSEC_GMAC_MCR_REG_IBN;
+
+	if (netsec_mac_write(priv, GMAC_REG_MCR, value))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int netsec_phy_write(struct mii_bus *bus,
+			    int phy_addr, int reg, u16 val)
+{
+	struct netsec_priv *priv = bus->priv;
+
+	if (netsec_mac_write(priv, GMAC_REG_GDR, val))
+		return -ETIMEDOUT;
+	if (netsec_mac_write(priv, GMAC_REG_GAR,
+			     phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA |
+			     reg << NETSEC_GMAC_GAR_REG_SHIFT_GR |
+			     NETSEC_GMAC_GAR_REG_GW | NETSEC_GMAC_GAR_REG_GB |
+			     (netsec_clk_type(priv->freq) <<
+			      GMAC_REG_SHIFT_CR_GAR)))
+		return -ETIMEDOUT;
+
+	return netsec_mac_wait_while_busy(priv, GMAC_REG_GAR,
+					  NETSEC_GMAC_GAR_REG_GB);
+}
+
+static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr)
+{
+	struct netsec_priv *priv = bus->priv;
+	u32 data;
+	int ret;
+
+	if (netsec_mac_write(priv, GMAC_REG_GAR, NETSEC_GMAC_GAR_REG_GB |
+			     phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA |
+			     reg_addr << NETSEC_GMAC_GAR_REG_SHIFT_GR |
+			     (netsec_clk_type(priv->freq) <<
+			      GMAC_REG_SHIFT_CR_GAR)))
+		return -ETIMEDOUT;
+
+	ret = netsec_mac_wait_while_busy(priv, GMAC_REG_GAR,
+					 NETSEC_GMAC_GAR_REG_GB);
+	if (ret)
+		return ret;
+
+	ret = netsec_mac_read(priv, GMAC_REG_GDR, &data);
+	if (ret)
+		return ret;
+
+	return data;
+}
+
+/************* ETHTOOL_OPS FOLLOW *************/
+
+static void netsec_et_get_drvinfo(struct net_device *net_device,
+				  struct ethtool_drvinfo *info)
+{
+	strlcpy(info->driver, "netsec", sizeof(info->driver));
+	strlcpy(info->bus_info, dev_name(net_device->dev.parent),
+		sizeof(info->bus_info));
+}
+
+static int netsec_et_get_coalesce(struct net_device *net_device,
+				  struct ethtool_coalesce *et_coalesce)
+{
+	struct netsec_priv *priv = netdev_priv(net_device);
+
+	*et_coalesce = priv->et_coalesce;
+
+	return 0;
+}
+
+static int netsec_et_set_coalesce(struct net_device *net_device,
+				  struct ethtool_coalesce *et_coalesce)
+{
+	struct netsec_priv *priv = netdev_priv(net_device);
+
+	priv->et_coalesce = *et_coalesce;
+
+	if (priv->et_coalesce.tx_coalesce_usecs < 50)
+		priv->et_coalesce.tx_coalesce_usecs = 50;
+	if (priv->et_coalesce.tx_max_coalesced_frames < 1)
+		priv->et_coalesce.tx_max_coalesced_frames = 1;
+
+	netsec_write(priv, NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT,
+		     priv->et_coalesce.tx_max_coalesced_frames);
+	netsec_write(priv, NETSEC_REG_NRM_TX_TXINT_TMR,
+		     priv->et_coalesce.tx_coalesce_usecs);
+	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TXDONE);
+	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TMREXP);
+
+	if (priv->et_coalesce.rx_coalesce_usecs < 50)
+		priv->et_coalesce.rx_coalesce_usecs = 50;
+	if (priv->et_coalesce.rx_max_coalesced_frames < 1)
+		priv->et_coalesce.rx_max_coalesced_frames = 1;
+
+	netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_PKTCNT,
+		     priv->et_coalesce.rx_max_coalesced_frames);
+	netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_TMR,
+		     priv->et_coalesce.rx_coalesce_usecs);
+	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_PKTCNT);
+	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_TMREXP);
+
+	return 0;
+}
+
+static u32 netsec_et_get_msglevel(struct net_device *dev)
+{
+	struct netsec_priv *priv = netdev_priv(dev);
+
+	return priv->msg_enable;
+}
+
+static void netsec_et_set_msglevel(struct net_device *dev, u32 datum)
+{
+	struct netsec_priv *priv = netdev_priv(dev);
+
+	priv->msg_enable = datum;
+}
+
+static const struct ethtool_ops netsec_ethtool_ops = {
+	.get_drvinfo		= netsec_et_get_drvinfo,
+	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
+	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
+	.get_link		= ethtool_op_get_link,
+	.get_coalesce		= netsec_et_get_coalesce,
+	.set_coalesce		= netsec_et_set_coalesce,
+	.get_msglevel		= netsec_et_get_msglevel,
+	.set_msglevel		= netsec_et_set_msglevel,
+};
+
+/************* NETDEV_OPS FOLLOW *************/
+
+static struct sk_buff *netsec_alloc_skb(struct netsec_priv *priv,
+					struct netsec_desc *desc)
+{
+	struct sk_buff *skb;
+
+	if (device_get_dma_attr(priv->dev) == DEV_DMA_COHERENT) {
+		skb = netdev_alloc_skb_ip_align(priv->ndev, desc->len);
+	} else {
+		desc->len = L1_CACHE_ALIGN(desc->len);
+		skb = netdev_alloc_skb(priv->ndev, desc->len);
+	}
+	if (!skb)
+		return NULL;
+
+	desc->addr = skb->data;
+	desc->dma_addr = dma_map_single(priv->dev, desc->addr, desc->len,
+					DMA_FROM_DEVICE);
+	if (dma_mapping_error(priv->dev, desc->dma_addr)) {
+		dev_kfree_skb_any(skb);
+		return NULL;
+	}
+	return skb;
+}
+
+static void netsec_set_rx_de(struct netsec_priv *priv,
+			     struct netsec_desc_ring *dring, u16 idx,
+			     const struct netsec_desc *desc,
+			     struct sk_buff *skb)
+{
+	struct netsec_de *de = dring->vaddr + DESC_SZ * idx;
+	u32 attr = (1 << NETSEC_RX_PKT_OWN_FIELD) |
+		   (1 << NETSEC_RX_PKT_FS_FIELD) |
+		   (1 << NETSEC_RX_PKT_LS_FIELD);
+
+	if (idx == DESC_NUM - 1)
+		attr |= (1 << NETSEC_RX_PKT_LD_FIELD);
+
+	de->data_buf_addr_up = upper_32_bits(desc->dma_addr);
+	de->data_buf_addr_lw = lower_32_bits(desc->dma_addr);
+	de->buf_len_info = desc->len;
+	de->attr = attr;
+	dma_wmb();
+
+	dring->desc[idx].dma_addr = desc->dma_addr;
+	dring->desc[idx].addr = desc->addr;
+	dring->desc[idx].len = desc->len;
+	dring->desc[idx].skb = skb;
+}
+
+static struct sk_buff *netsec_get_rx_de(struct netsec_priv *priv,
+					struct netsec_desc_ring *dring,
+					u16 idx,
+					struct netsec_rx_pkt_info *rxpi,
+					struct netsec_desc *desc, u16 *len)
+{
+	struct netsec_de de = {};
+
+	memcpy(&de, dring->vaddr + DESC_SZ * idx, DESC_SZ);
+
+	*len = de.buf_len_info >> 16;
+
+	rxpi->err_flag = (de.attr >> NETSEC_RX_PKT_ER_FIELD) & 1;
+	rxpi->rx_cksum_result = (de.attr >> NETSEC_RX_PKT_CO_FIELD) & 3;
+	rxpi->err_code = (de.attr >> NETSEC_RX_PKT_ERR_FIELD) &
+							NETSEC_RX_PKT_ERR_MASK;
+	*desc = dring->desc[idx];
+	return desc->skb;
+}
+
+static struct sk_buff *netsec_get_rx_pkt_data(struct netsec_priv *priv,
+					      struct netsec_rx_pkt_info *rxpi,
+					      struct netsec_desc *desc,
+					      u16 *len)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
+	struct sk_buff *tmp_skb, *skb = NULL;
+	struct netsec_desc td;
+	int tail;
+
+	*rxpi = (struct netsec_rx_pkt_info){};
+
+	td.len = priv->ndev->mtu + 22;
+
+	tmp_skb = netsec_alloc_skb(priv, &td);
+
+	dma_rmb();
+
+	tail = dring->tail;
+
+	if (!tmp_skb) {
+		netsec_set_rx_de(priv, dring, tail, &dring->desc[tail],
+				 dring->desc[tail].skb);
+	} else {
+		skb = netsec_get_rx_de(priv, dring, tail, rxpi, desc, len);
+		netsec_set_rx_de(priv, dring, tail, &td, tmp_skb);
+	}
+
+	/* move tail ahead */
+	dring->tail = (dring->tail + 1) % DESC_NUM;
+
+	dring->pkt_cnt--;
+
+	return skb;
+}
+
+static int netsec_clean_tx_dring(struct netsec_priv *priv, int budget)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
+	unsigned int pkts, bytes;
+
+	dring->pkt_cnt += netsec_read(priv, NETSEC_REG_NRM_TX_DONE_PKTCNT);
+
+	if (dring->pkt_cnt < budget)
+		budget = dring->pkt_cnt;
+
+	pkts = 0;
+	bytes = 0;
+
+	while (pkts < budget) {
+		struct netsec_desc *desc;
+		struct netsec_de *entry;
+		int tail, eop;
+
+		tail = dring->tail;
+
+		/* move tail ahead */
+		dring->tail = (tail + 1) % DESC_NUM;
+
+		desc = &dring->desc[tail];
+		entry = dring->vaddr + DESC_SZ * tail;
+
+		eop = (entry->attr >> NETSEC_TX_LAST) & 1;
+
+		dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
+				 DMA_TO_DEVICE);
+		if (eop) {
+			pkts++;
+			bytes += desc->skb->len;
+			dev_kfree_skb(desc->skb);
+		}
+		*desc = (struct netsec_desc){};
+	}
+	dring->pkt_cnt -= budget;
+
+	priv->ndev->stats.tx_packets += budget;
+	priv->ndev->stats.tx_bytes += bytes;
+
+	netdev_completed_queue(priv->ndev, budget, bytes);
+
+	return budget;
+}
+
+static int netsec_process_tx(struct netsec_priv *priv, int budget)
+{
+	struct net_device *ndev = priv->ndev;
+	int new, done = 0;
+
+	do {
+		new = netsec_clean_tx_dring(priv, budget);
+		done += new;
+		budget -= new;
+	} while (new);
+
+	if (done && netif_queue_stopped(ndev))
+		netif_wake_queue(ndev);
+
+	return done;
+}
+
+static int netsec_process_rx(struct netsec_priv *priv, int budget)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
+	struct net_device *ndev = priv->ndev;
+	struct netsec_rx_pkt_info rx_info;
+	int done = 0, rx_num = 0;
+	struct netsec_desc desc;
+	struct sk_buff *skb;
+	u16 len;
+
+	while (done < budget) {
+		if (!rx_num) {
+			rx_num = netsec_read(priv, NETSEC_REG_NRM_RX_PKTCNT);
+			dring->pkt_cnt += rx_num;
+
+			/* move head 'rx_num' */
+			dring->head = (dring->head + rx_num) % DESC_NUM;
+
+			rx_num = dring->pkt_cnt;
+			if (!rx_num)
+				break;
+		}
+		done++;
+		rx_num--;
+		skb = netsec_get_rx_pkt_data(priv, &rx_info, &desc, &len);
+		if (unlikely(!skb) || rx_info.err_flag) {
+			netif_err(priv, drv, priv->ndev,
+				  "%s: rx fail err(%d)\n",
+				  __func__, rx_info.err_code);
+			ndev->stats.rx_dropped++;
+			continue;
+		}
+
+		dma_unmap_single(priv->dev, desc.dma_addr, desc.len,
+				 DMA_FROM_DEVICE);
+		skb_put(skb, len);
+		skb->protocol = eth_type_trans(skb, priv->ndev);
+
+		if (priv->rx_cksum_offload_flag &&
+		    rx_info.rx_cksum_result == NETSEC_RX_CKSUM_OK)
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+		if (napi_gro_receive(&priv->napi, skb) != GRO_DROP) {
+			ndev->stats.rx_packets++;
+			ndev->stats.rx_bytes += len;
+		}
+	}
+
+	return done;
+}
+
+static int netsec_napi_poll(struct napi_struct *napi, int budget)
+{
+	struct netsec_priv *priv;
+	struct net_device *ndev;
+	int tx, rx, done, todo;
+
+	priv = container_of(napi, struct netsec_priv, napi);
+	ndev = priv->ndev;
+
+	todo = budget;
+	do {
+		if (!todo)
+			break;
+
+		tx = netsec_process_tx(priv, todo);
+		todo -= tx;
+
+		if (!todo)
+			break;
+
+		rx = netsec_process_rx(priv, todo);
+		todo -= rx;
+	} while (rx || tx);
+
+	done = budget - todo;
+
+	if (done < budget && napi_complete_done(napi, done)) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&priv->reglock, flags);
+		netsec_write(priv, NETSEC_REG_INTEN_SET,
+			     NETSEC_IRQ_RX | NETSEC_IRQ_TX);
+		spin_unlock_irqrestore(&priv->reglock, flags);
+	}
+
+	return done;
+}
+
+static void netsec_set_tx_de(struct netsec_priv *priv,
+			     struct netsec_desc_ring *dring,
+			     const struct netsec_tx_pkt_ctrl *tx_ctrl,
+			     const struct netsec_desc *desc,
+			     struct sk_buff *skb)
+{
+	int idx = dring->head;
+	struct netsec_de *de;
+	u32 attr;
+
+	de = dring->vaddr + (DESC_SZ * idx);
+
+	attr = (1 << NETSEC_TX_SHIFT_OWN_FIELD) |
+	       (1 << NETSEC_TX_SHIFT_PT_FIELD) |
+	       (NETSEC_RING_GMAC << NETSEC_TX_SHIFT_TDRID_FIELD) |
+	       (1 << NETSEC_TX_SHIFT_FS_FIELD) |
+	       (1 << NETSEC_TX_LAST) |
+	       (tx_ctrl->cksum_offload_flag << NETSEC_TX_SHIFT_CO) |
+	       (tx_ctrl->tcp_seg_offload_flag << NETSEC_TX_SHIFT_SO) |
+	       (1 << NETSEC_TX_SHIFT_TRS_FIELD);
+	if (idx == DESC_NUM - 1)
+		attr |= (1 << NETSEC_TX_SHIFT_LD_FIELD);
+
+	de->data_buf_addr_up = upper_32_bits(desc->dma_addr);
+	de->data_buf_addr_lw = lower_32_bits(desc->dma_addr);
+	de->buf_len_info = (tx_ctrl->tcp_seg_len << 16) | desc->len;
+	de->attr = attr;
+	dma_wmb();
+
+	dring->desc[idx] = *desc;
+	dring->desc[idx].skb = skb;
+
+	/* move head ahead */
+	dring->head = (dring->head + 1) % DESC_NUM;
+}
+
+static netdev_tx_t netsec_netdev_start_xmit(struct sk_buff *skb,
+					    struct net_device *ndev)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX];
+	struct netsec_tx_pkt_ctrl tx_ctrl = {};
+	struct netsec_desc tx_desc;
+	u16 tso_seg_len = 0;
+	int filled;
+
+	/* differentiate between full/emtpy ring */
+	if (dring->head >= dring->tail)
+		filled = dring->head - dring->tail;
+	else
+		filled = dring->head + DESC_NUM - dring->tail;
+
+	if (DESC_NUM - filled < 2) { /* if less than 2 available */
+		netif_err(priv, drv, priv->ndev, "%s: TxQFull!\n", __func__);
+		netif_stop_queue(priv->ndev);
+		dma_wmb();
+		return NETDEV_TX_BUSY;
+	}
+
+	if (skb->ip_summed == CHECKSUM_PARTIAL)
+		tx_ctrl.cksum_offload_flag = true;
+
+	if (skb_is_gso(skb))
+		tso_seg_len = skb_shinfo(skb)->gso_size;
+
+	if (tso_seg_len > 0) {
+		if (skb->protocol == htons(ETH_P_IP)) {
+			ip_hdr(skb)->tot_len = 0;
+			tcp_hdr(skb)->check =
+				~tcp_v4_check(0, ip_hdr(skb)->saddr,
+					      ip_hdr(skb)->daddr, 0);
+		} else {
+			ipv6_hdr(skb)->payload_len = 0;
+			tcp_hdr(skb)->check =
+				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
+						 &ipv6_hdr(skb)->daddr,
+						 0, IPPROTO_TCP, 0);
+		}
+
+		tx_ctrl.tcp_seg_offload_flag = true;
+		tx_ctrl.tcp_seg_len = tso_seg_len;
+	}
+
+	tx_desc.dma_addr = dma_map_single(priv->dev, skb->data,
+					  skb_headlen(skb), DMA_TO_DEVICE);
+	if (dma_mapping_error(priv->dev, tx_desc.dma_addr)) {
+		netif_err(priv, drv, priv->ndev,
+			  "%s: DMA mapping failed\n", __func__);
+		ndev->stats.tx_dropped++;
+		dev_kfree_skb_any(skb);
+		return NETDEV_TX_OK;
+	}
+	tx_desc.addr = skb->data;
+	tx_desc.len = skb_headlen(skb);
+
+	skb_tx_timestamp(skb);
+	netdev_sent_queue(priv->ndev, skb->len);
+
+	netsec_set_tx_de(priv, dring, &tx_ctrl, &tx_desc, skb);
+	netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, 1); /* submit another tx */
+
+	return NETDEV_TX_OK;
+}
+
+static void netsec_uninit_pkt_dring(struct netsec_priv *priv, int id)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[id];
+	struct netsec_desc *desc;
+	u16 idx;
+
+	if (!dring->vaddr || !dring->desc)
+		return;
+
+	for (idx = 0; idx < DESC_NUM; idx++) {
+		desc = &dring->desc[idx];
+		if (!desc->addr)
+			continue;
+
+		dma_unmap_single(priv->dev, desc->dma_addr, desc->len,
+				 id == NETSEC_RING_RX ? DMA_FROM_DEVICE :
+							      DMA_TO_DEVICE);
+		dev_kfree_skb(desc->skb);
+	}
+
+	memset(dring->desc, 0, sizeof(struct netsec_desc) * DESC_NUM);
+	memset(dring->vaddr, 0, DESC_SZ * DESC_NUM);
+
+	dring->head = 0;
+	dring->tail = 0;
+	dring->pkt_cnt = 0;
+}
+
+static void netsec_free_dring(struct netsec_priv *priv, int id)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[id];
+
+	if (dring->vaddr) {
+		dma_free_coherent(priv->dev, DESC_SZ * DESC_NUM,
+				  dring->vaddr, dring->desc_dma);
+		dring->vaddr = NULL;
+	}
+
+	kfree(dring->desc);
+	dring->desc = NULL;
+}
+
+static int netsec_alloc_dring(struct netsec_priv *priv, enum ring_id id)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[id];
+	int ret = 0;
+
+	dring->vaddr = dma_zalloc_coherent(priv->dev, DESC_SZ * DESC_NUM,
+					   &dring->desc_dma, GFP_KERNEL);
+	if (!dring->vaddr) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	dring->desc = kzalloc(DESC_NUM * sizeof(*dring->desc), GFP_KERNEL);
+	if (!dring->desc) {
+		ret = -ENOMEM;
+		goto err;
+	}
+
+	return 0;
+err:
+	netsec_free_dring(priv, id);
+
+	return ret;
+}
+
+static int netsec_setup_rx_dring(struct netsec_priv *priv)
+{
+	struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX];
+	struct netsec_desc desc;
+	struct sk_buff *skb;
+	int n;
+
+	desc.len = priv->ndev->mtu + 22;
+
+	for (n = 0; n < DESC_NUM; n++) {
+		skb = netsec_alloc_skb(priv, &desc);
+		if (!skb) {
+			netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
+			return -ENOMEM;
+		}
+		netsec_set_rx_de(priv, dring, n, &desc, skb);
+	}
+
+	return 0;
+}
+
+static int netsec_netdev_load_ucode_region(struct netsec_priv *priv, u32 reg,
+					   u32 addr_h, u32 addr_l, u32 size)
+{
+	u64 base = (u64)addr_h << 32 | addr_l;
+	void __iomem *ucode;
+	u32 i;
+
+	ucode = ioremap(base, size * sizeof(u32));
+	if (!ucode)
+		return -ENOMEM;
+
+	for (i = 0; i < size; i++)
+		netsec_write(priv, reg, readl(ucode + i * 4));
+
+	iounmap(ucode);
+	return 0;
+}
+
+static int netsec_netdev_load_microcode(struct netsec_priv *priv)
+{
+	u32 addr_h, addr_l, size;
+	int err;
+
+	addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_H);
+	addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_L);
+	size = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_SIZE);
+	err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_HM_CMD_BUF,
+					      addr_h, addr_l, size);
+	if (err)
+		return err;
+
+	addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_H);
+	addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_L);
+	size = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_SIZE);
+	err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_MH_CMD_BUF,
+					      addr_h, addr_l, size);
+	if (err)
+		return err;
+
+	addr_h = 0;
+	addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_ADDRESS);
+	size = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_SIZE);
+	err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_PKT_CMD_BUF,
+					      addr_h, addr_l, size);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int netsec_reset_hardware(struct netsec_priv *priv)
+{
+	u32 value;
+	int err;
+
+	/* stop DMA engines */
+	if (!netsec_read(priv, NETSEC_REG_ADDR_DIS_CORE)) {
+		netsec_write(priv, NETSEC_REG_DMA_HM_CTRL,
+			     NETSEC_DMA_CTRL_REG_STOP);
+		netsec_write(priv, NETSEC_REG_DMA_MH_CTRL,
+			     NETSEC_DMA_CTRL_REG_STOP);
+
+		while (netsec_read(priv, NETSEC_REG_DMA_HM_CTRL) &
+		       NETSEC_DMA_CTRL_REG_STOP)
+			cpu_relax();
+
+		while (netsec_read(priv, NETSEC_REG_DMA_MH_CTRL) &
+		       NETSEC_DMA_CTRL_REG_STOP)
+			cpu_relax();
+	}
+
+	netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RESET);
+	netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RUN);
+	netsec_write(priv, NETSEC_REG_COM_INIT, NETSEC_COM_INIT_REG_ALL);
+
+	while (netsec_read(priv, NETSEC_REG_COM_INIT) != 0)
+		cpu_relax();
+
+	/* set desc_start addr */
+	netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_UP,
+		     upper_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma));
+	netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_LW,
+		     lower_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma));
+
+	netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_UP,
+		     upper_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma));
+	netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_LW,
+		     lower_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma));
+
+	/* set normal tx dring ring config */
+	netsec_write(priv, NETSEC_REG_NRM_TX_CONFIG,
+		     1 << NETSEC_REG_DESC_ENDIAN);
+	netsec_write(priv, NETSEC_REG_NRM_RX_CONFIG,
+		     1 << NETSEC_REG_DESC_ENDIAN);
+
+	err = netsec_netdev_load_microcode(priv);
+	if (err) {
+		netif_err(priv, probe, priv->ndev,
+			  "%s: failed to load microcode (%d)\n", __func__, err);
+		return err;
+	}
+
+	/* start DMA engines */
+	netsec_write(priv, NETSEC_REG_DMA_TMR_CTRL, priv->freq / 1000000 - 1);
+	netsec_write(priv, NETSEC_REG_ADDR_DIS_CORE, 0);
+
+	usleep_range(1000, 2000);
+
+	if (!(netsec_read(priv, NETSEC_REG_TOP_STATUS) &
+	      NETSEC_TOP_IRQ_REG_CODE_LOAD_END)) {
+		netif_err(priv, probe, priv->ndev,
+			  "microengine start failed\n");
+		return -ENXIO;
+	}
+	netsec_write(priv, NETSEC_REG_TOP_STATUS,
+		     NETSEC_TOP_IRQ_REG_CODE_LOAD_END);
+
+	value = NETSEC_PKT_CTRL_REG_MODE_NRM;
+	if (priv->ndev->mtu > ETH_DATA_LEN)
+		value |= NETSEC_PKT_CTRL_REG_EN_JUMBO;
+
+	/* change to normal mode */
+	netsec_write(priv, NETSEC_REG_DMA_MH_CTRL, MH_CTRL__MODE_TRANS);
+	netsec_write(priv, NETSEC_REG_PKT_CTRL, value);
+
+	while ((netsec_read(priv, NETSEC_REG_MODE_TRANS_COMP_STATUS) &
+		NETSEC_MODE_TRANS_COMP_IRQ_T2N) == 0)
+		cpu_relax();
+
+	/* clear any pending EMPTY/ERR irq status */
+	netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, ~0);
+
+	/* Disable TX & RX intr */
+	netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0);
+
+	return 0;
+}
+
+static int netsec_start_gmac(struct netsec_priv *priv)
+{
+	struct phy_device *phydev = priv->ndev->phydev;
+	u32 value = 0;
+	int ret;
+
+	if (phydev->speed != SPEED_1000)
+		value = (NETSEC_GMAC_MCR_REG_CST |
+			 NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON);
+
+	if (netsec_mac_write(priv, GMAC_REG_MCR, value))
+		return -ETIMEDOUT;
+	if (netsec_mac_write(priv, GMAC_REG_BMR,
+			     NETSEC_GMAC_BMR_REG_RESET))
+		return -ETIMEDOUT;
+
+	/* Wait soft reset */
+	usleep_range(1000, 5000);
+
+	ret = netsec_mac_read(priv, GMAC_REG_BMR, &value);
+	if (ret)
+		return ret;
+	if (value & NETSEC_GMAC_BMR_REG_SWR)
+		return -EAGAIN;
+
+	netsec_write(priv, MAC_REG_DESC_SOFT_RST, 1);
+	if (netsec_wait_while_busy(priv, MAC_REG_DESC_SOFT_RST, 1))
+		return -ETIMEDOUT;
+
+	netsec_write(priv, MAC_REG_DESC_INIT, 1);
+	if (netsec_wait_while_busy(priv, MAC_REG_DESC_INIT, 1))
+		return -ETIMEDOUT;
+
+	if (netsec_mac_write(priv, GMAC_REG_BMR,
+			     NETSEC_GMAC_BMR_REG_COMMON))
+		return -ETIMEDOUT;
+	if (netsec_mac_write(priv, GMAC_REG_RDLAR,
+			     NETSEC_GMAC_RDLAR_REG_COMMON))
+		return -ETIMEDOUT;
+	if (netsec_mac_write(priv, GMAC_REG_TDLAR,
+			     NETSEC_GMAC_TDLAR_REG_COMMON))
+		return -ETIMEDOUT;
+	if (netsec_mac_write(priv, GMAC_REG_MFFR, 0x80000001))
+		return -ETIMEDOUT;
+
+	ret = netsec_mac_update_to_phy_state(priv);
+	if (ret)
+		return ret;
+
+	ret = netsec_mac_read(priv, GMAC_REG_OMR, &value);
+	if (ret)
+		return ret;
+
+	value |= NETSEC_GMAC_OMR_REG_SR;
+	value |= NETSEC_GMAC_OMR_REG_ST;
+
+	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0);
+	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0);
+
+	netsec_et_set_coalesce(priv->ndev, &priv->et_coalesce);
+
+	if (netsec_mac_write(priv, GMAC_REG_OMR, value))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int netsec_stop_gmac(struct netsec_priv *priv)
+{
+	u32 value;
+	int ret;
+
+	ret = netsec_mac_read(priv, GMAC_REG_OMR, &value);
+	if (ret)
+		return ret;
+	value &= ~NETSEC_GMAC_OMR_REG_SR;
+	value &= ~NETSEC_GMAC_OMR_REG_ST;
+
+	/* disable all interrupts */
+	netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0);
+	netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0);
+
+	return netsec_mac_write(priv, GMAC_REG_OMR, value);
+}
+
+static void netsec_phy_adjust_link(struct net_device *ndev)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+
+	if (ndev->phydev->link)
+		netsec_start_gmac(priv);
+	else
+		netsec_stop_gmac(priv);
+
+	phy_print_status(ndev->phydev);
+}
+
+static irqreturn_t netsec_irq_handler(int irq, void *dev_id)
+{
+	struct netsec_priv *priv = dev_id;
+	u32 val, status = netsec_read(priv, NETSEC_REG_TOP_STATUS);
+	unsigned long flags;
+
+	/* Disable interrupts */
+	if (status & NETSEC_IRQ_TX) {
+		val = netsec_read(priv, NETSEC_REG_NRM_TX_STATUS);
+		netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, val);
+	}
+	if (status & NETSEC_IRQ_RX) {
+		val = netsec_read(priv, NETSEC_REG_NRM_RX_STATUS);
+		netsec_write(priv, NETSEC_REG_NRM_RX_STATUS, val);
+	}
+
+	spin_lock_irqsave(&priv->reglock, flags);
+	netsec_write(priv, NETSEC_REG_INTEN_CLR, NETSEC_IRQ_RX | NETSEC_IRQ_TX);
+	spin_unlock_irqrestore(&priv->reglock, flags);
+
+	napi_schedule(&priv->napi);
+
+	return IRQ_HANDLED;
+}
+
+static int netsec_netdev_open(struct net_device *ndev)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	pm_runtime_get_sync(priv->dev);
+
+	ret = netsec_setup_rx_dring(priv);
+	if (ret) {
+		netif_err(priv, probe, priv->ndev,
+			  "%s: fail setup ring\n", __func__);
+		goto err1;
+	}
+
+	ret = request_irq(priv->ndev->irq, netsec_irq_handler,
+			  IRQF_SHARED, "netsec", priv);
+	if (ret) {
+		netif_err(priv, drv, priv->ndev, "request_irq failed\n");
+		goto err2;
+	}
+
+	if (dev_of_node(priv->dev)) {
+		if (!of_phy_connect(priv->ndev, priv->phy_np,
+				    netsec_phy_adjust_link, 0,
+				    priv->phy_interface)) {
+			netif_err(priv, link, priv->ndev, "missing PHY\n");
+			ret = -ENODEV;
+			goto err3;
+		}
+	} else {
+		ret = phy_connect_direct(priv->ndev, priv->phydev,
+					 netsec_phy_adjust_link,
+					 priv->phy_interface);
+		if (ret) {
+			netif_err(priv, link, priv->ndev,
+				  "phy_connect_direct() failed (%d)\n", ret);
+			goto err3;
+		}
+	}
+
+	phy_start(ndev->phydev);
+
+	netsec_start_gmac(priv);
+	napi_enable(&priv->napi);
+	netif_start_queue(ndev);
+
+	/* Enable RX intr. */
+	netsec_write(priv, NETSEC_REG_INTEN_SET, NETSEC_IRQ_RX);
+
+	return 0;
+err3:
+	free_irq(priv->ndev->irq, priv);
+err2:
+	netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
+err1:
+	pm_runtime_put_sync(priv->dev);
+	return ret;
+}
+
+static int netsec_netdev_stop(struct net_device *ndev)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+
+	netif_stop_queue(priv->ndev);
+	dma_wmb();
+
+	napi_disable(&priv->napi);
+
+	netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0);
+	netsec_stop_gmac(priv);
+
+	free_irq(priv->ndev->irq, priv);
+
+	netsec_uninit_pkt_dring(priv, NETSEC_RING_TX);
+	netsec_uninit_pkt_dring(priv, NETSEC_RING_RX);
+
+	phy_stop(ndev->phydev);
+	phy_disconnect(ndev->phydev);
+
+	pm_runtime_put_sync(priv->dev);
+
+	return 0;
+}
+
+static int netsec_netdev_init(struct net_device *ndev)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+	int ret;
+
+	ret = netsec_alloc_dring(priv, NETSEC_RING_TX);
+	if (ret)
+		return ret;
+
+	ret = netsec_alloc_dring(priv, NETSEC_RING_RX);
+	if (ret)
+		goto err1;
+
+	ret = netsec_reset_hardware(priv);
+	if (ret)
+		goto err2;
+
+	return 0;
+err2:
+	netsec_free_dring(priv, NETSEC_RING_RX);
+err1:
+	netsec_free_dring(priv, NETSEC_RING_TX);
+	return ret;
+}
+
+static void netsec_netdev_uninit(struct net_device *ndev)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+
+	netsec_free_dring(priv, NETSEC_RING_RX);
+	netsec_free_dring(priv, NETSEC_RING_TX);
+}
+
+static int netsec_netdev_set_features(struct net_device *ndev,
+				      netdev_features_t features)
+{
+	struct netsec_priv *priv = netdev_priv(ndev);
+
+	priv->rx_cksum_offload_flag = !!(features & NETIF_F_RXCSUM);
+
+	return 0;
+}
+
+static int netsec_netdev_ioctl(struct net_device *ndev, struct ifreq *ifr,
+			       int cmd)
+{
+	return phy_mii_ioctl(ndev->phydev, ifr, cmd);
+}
+
+static const struct net_device_ops netsec_netdev_ops = {
+	.ndo_init		= netsec_netdev_init,
+	.ndo_uninit		= netsec_netdev_uninit,
+	.ndo_open		= netsec_netdev_open,
+	.ndo_stop		= netsec_netdev_stop,
+	.ndo_start_xmit		= netsec_netdev_start_xmit,
+	.ndo_set_features	= netsec_netdev_set_features,
+	.ndo_set_mac_address    = eth_mac_addr,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_do_ioctl		= netsec_netdev_ioctl,
+};
+
+static int netsec_of_probe(struct platform_device *pdev,
+			   struct netsec_priv *priv)
+{
+	priv->phy_np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0);
+	if (!priv->phy_np) {
+		dev_err(&pdev->dev, "missing required property 'phy-handle'\n");
+		return -EINVAL;
+	}
+
+	priv->clk = devm_clk_get(&pdev->dev, NULL); /* get by 'phy_ref_clk' */
+	if (IS_ERR(priv->clk)) {
+		dev_err(&pdev->dev, "phy_ref_clk not found\n");
+		return PTR_ERR(priv->clk);
+	}
+	priv->freq = clk_get_rate(priv->clk);
+
+	return 0;
+}
+
+static int netsec_acpi_probe(struct platform_device *pdev,
+			     struct netsec_priv *priv, u32 *phy_addr)
+{
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_ACPI))
+		return -ENODEV;
+
+	ret = device_property_read_u32(&pdev->dev, "phy-channel", phy_addr);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"missing required property 'phy-channel'\n");
+		return ret;
+	}
+
+	ret = device_property_read_u32(&pdev->dev,
+				       "socionext,phy-clock-frequency",
+				       &priv->freq);
+	if (ret)
+		dev_err(&pdev->dev,
+			"missing required property 'socionext,phy-clock-frequency'\n");
+	return ret;
+}
+
+static void netsec_unregister_mdio(struct netsec_priv *priv)
+{
+	struct phy_device *phydev = priv->phydev;
+
+	if (!dev_of_node(priv->dev) && phydev) {
+		phy_device_remove(phydev);
+		phy_device_free(phydev);
+	}
+
+	mdiobus_unregister(priv->mii_bus);
+}
+
+static int netsec_register_mdio(struct netsec_priv *priv, u32 phy_addr)
+{
+	struct mii_bus *bus;
+	int ret;
+
+	bus = devm_mdiobus_alloc(priv->dev);
+	if (!bus)
+		return -ENOMEM;
+
+	snprintf(bus->id, MII_BUS_ID_SIZE, "%s", dev_name(priv->dev));
+	bus->priv = priv;
+	bus->name = "SNI NETSEC MDIO";
+	bus->read = netsec_phy_read;
+	bus->write = netsec_phy_write;
+	bus->parent = priv->dev;
+	priv->mii_bus = bus;
+
+	if (dev_of_node(priv->dev)) {
+		struct device_node *mdio_node, *parent = dev_of_node(priv->dev);
+
+		mdio_node = of_get_child_by_name(parent, "mdio");
+		if (mdio_node) {
+			parent = mdio_node;
+		} else {
+			/* older f/w doesn't populate the mdio subnode,
+			 * allow relaxed upgrade of f/w in due time.
+			 */
+			dev_info(priv->dev, "Upgrade f/w for mdio subnode!\n");
+		}
+
+		ret = of_mdiobus_register(bus, parent);
+		of_node_put(mdio_node);
+
+		if (ret) {
+			dev_err(priv->dev, "mdiobus register err(%d)\n", ret);
+			return ret;
+		}
+	} else {
+		/* Mask out all PHYs from auto probing. */
+		bus->phy_mask = ~0;
+		ret = mdiobus_register(bus);
+		if (ret) {
+			dev_err(priv->dev, "mdiobus register err(%d)\n", ret);
+			return ret;
+		}
+
+		priv->phydev = get_phy_device(bus, phy_addr, false);
+		if (IS_ERR(priv->phydev)) {
+			ret = PTR_ERR(priv->phydev);
+			dev_err(priv->dev, "get_phy_device err(%d)\n", ret);
+			priv->phydev = NULL;
+			return -ENODEV;
+		}
+
+		ret = phy_device_register(priv->phydev);
+		if (ret) {
+			mdiobus_unregister(bus);
+			dev_err(priv->dev,
+				"phy_device_register err(%d)\n", ret);
+		}
+	}
+
+	return ret;
+}
+
+static int netsec_probe(struct platform_device *pdev)
+{
+	struct resource *mmio_res, *eeprom_res, *irq_res;
+	u8 *mac, macbuf[ETH_ALEN];
+	struct netsec_priv *priv;
+	u32 hw_ver, phy_addr = 0;
+	struct net_device *ndev;
+	int ret;
+
+	mmio_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!mmio_res) {
+		dev_err(&pdev->dev, "No MMIO resource found.\n");
+		return -ENODEV;
+	}
+
+	eeprom_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+	if (!eeprom_res) {
+		dev_info(&pdev->dev, "No EEPROM resource found.\n");
+		return -ENODEV;
+	}
+
+	irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+	if (!irq_res) {
+		dev_err(&pdev->dev, "No IRQ resource found.\n");
+		return -ENODEV;
+	}
+
+	ndev = alloc_etherdev(sizeof(*priv));
+	if (!ndev)
+		return -ENOMEM;
+
+	priv = netdev_priv(ndev);
+
+	spin_lock_init(&priv->reglock);
+	SET_NETDEV_DEV(ndev, &pdev->dev);
+	platform_set_drvdata(pdev, priv);
+	ndev->irq = irq_res->start;
+	priv->dev = &pdev->dev;
+	priv->ndev = ndev;
+
+	priv->msg_enable = NETIF_MSG_TX_ERR | NETIF_MSG_HW | NETIF_MSG_DRV |
+			   NETIF_MSG_LINK | NETIF_MSG_PROBE;
+
+	priv->phy_interface = device_get_phy_mode(&pdev->dev);
+	if (priv->phy_interface < 0) {
+		dev_err(&pdev->dev, "missing required property 'phy-mode'\n");
+		ret = -ENODEV;
+		goto free_ndev;
+	}
+
+	priv->ioaddr = devm_ioremap(&pdev->dev, mmio_res->start,
+				    resource_size(mmio_res));
+	if (!priv->ioaddr) {
+		dev_err(&pdev->dev, "devm_ioremap() failed\n");
+		ret = -ENXIO;
+		goto free_ndev;
+	}
+
+	priv->eeprom_base = devm_ioremap(&pdev->dev, eeprom_res->start,
+					 resource_size(eeprom_res));
+	if (!priv->eeprom_base) {
+		dev_err(&pdev->dev, "devm_ioremap() failed for EEPROM\n");
+		ret = -ENXIO;
+		goto free_ndev;
+	}
+
+	mac = device_get_mac_address(&pdev->dev, macbuf, sizeof(macbuf));
+	if (mac)
+		ether_addr_copy(ndev->dev_addr, mac);
+
+	if (priv->eeprom_base &&
+	    (!mac || !is_valid_ether_addr(ndev->dev_addr))) {
+		void __iomem *macp = priv->eeprom_base +
+					NETSEC_EEPROM_MAC_ADDRESS;
+
+		ndev->dev_addr[0] = readb(macp + 3);
+		ndev->dev_addr[1] = readb(macp + 2);
+		ndev->dev_addr[2] = readb(macp + 1);
+		ndev->dev_addr[3] = readb(macp + 0);
+		ndev->dev_addr[4] = readb(macp + 7);
+		ndev->dev_addr[5] = readb(macp + 6);
+	}
+
+	if (!is_valid_ether_addr(ndev->dev_addr)) {
+		dev_warn(&pdev->dev, "No MAC address found, using random\n");
+		eth_hw_addr_random(ndev);
+	}
+
+	if (dev_of_node(&pdev->dev))
+		ret = netsec_of_probe(pdev, priv);
+	else
+		ret = netsec_acpi_probe(pdev, priv, &phy_addr);
+	if (ret)
+		goto free_ndev;
+
+	if (!priv->freq) {
+		dev_err(&pdev->dev, "missing PHY reference clock frequency\n");
+		ret = -ENODEV;
+		goto free_ndev;
+	}
+
+	/* default for throughput */
+	priv->et_coalesce.rx_coalesce_usecs = 500;
+	priv->et_coalesce.rx_max_coalesced_frames = 8;
+	priv->et_coalesce.tx_coalesce_usecs = 500;
+	priv->et_coalesce.tx_max_coalesced_frames = 8;
+
+	ret = device_property_read_u32(&pdev->dev, "max-frame-size",
+				       &ndev->max_mtu);
+	if (ret < 0)
+		ndev->max_mtu = ETH_DATA_LEN;
+
+	/* runtime_pm coverage just for probe, open/close also cover it */
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_get_sync(&pdev->dev);
+
+	hw_ver = netsec_read(priv, NETSEC_REG_F_TAIKI_VER);
+	/* this driver only supports F_TAIKI style NETSEC */
+	if (NETSEC_F_NETSEC_VER_MAJOR_NUM(hw_ver) !=
+	    NETSEC_F_NETSEC_VER_MAJOR_NUM(NETSEC_REG_NETSEC_VER_F_TAIKI)) {
+		ret = -ENODEV;
+		goto pm_disable;
+	}
+
+	dev_info(&pdev->dev, "hardware revision %d.%d\n",
+		 hw_ver >> 16, hw_ver & 0xffff);
+
+	netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_BUDGET);
+
+	ndev->netdev_ops = &netsec_netdev_ops;
+	ndev->ethtool_ops = &netsec_ethtool_ops;
+
+	ndev->features |= NETIF_F_HIGHDMA | NETIF_F_RXCSUM | NETIF_F_GSO |
+				NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
+	ndev->hw_features = ndev->features;
+
+	priv->rx_cksum_offload_flag = true;
+
+	ret = netsec_register_mdio(priv, phy_addr);
+	if (ret)
+		goto unreg_napi;
+
+	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)))
+		dev_warn(&pdev->dev, "Failed to enable 64-bit DMA\n");
+
+	ret = register_netdev(ndev);
+	if (ret) {
+		netif_err(priv, probe, ndev, "register_netdev() failed\n");
+		goto unreg_mii;
+	}
+
+	pm_runtime_put_sync(&pdev->dev);
+	return 0;
+
+unreg_mii:
+	netsec_unregister_mdio(priv);
+unreg_napi:
+	netif_napi_del(&priv->napi);
+pm_disable:
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+free_ndev:
+	free_netdev(ndev);
+	dev_err(&pdev->dev, "init failed\n");
+
+	return ret;
+}
+
+static int netsec_remove(struct platform_device *pdev)
+{
+	struct netsec_priv *priv = platform_get_drvdata(pdev);
+
+	unregister_netdev(priv->ndev);
+
+	netsec_unregister_mdio(priv);
+
+	netif_napi_del(&priv->napi);
+
+	pm_runtime_disable(&pdev->dev);
+	free_netdev(priv->ndev);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int netsec_runtime_suspend(struct device *dev)
+{
+	struct netsec_priv *priv = dev_get_drvdata(dev);
+
+	netsec_write(priv, NETSEC_REG_CLK_EN, 0);
+
+	clk_disable_unprepare(priv->clk);
+
+	return 0;
+}
+
+static int netsec_runtime_resume(struct device *dev)
+{
+	struct netsec_priv *priv = dev_get_drvdata(dev);
+
+	clk_prepare_enable(priv->clk);
+
+	netsec_write(priv, NETSEC_REG_CLK_EN, NETSEC_CLK_EN_REG_DOM_D |
+					       NETSEC_CLK_EN_REG_DOM_C |
+					       NETSEC_CLK_EN_REG_DOM_G);
+	return 0;
+}
+#endif
+
+static const struct dev_pm_ops netsec_pm_ops = {
+	SET_RUNTIME_PM_OPS(netsec_runtime_suspend, netsec_runtime_resume, NULL)
+};
+
+static const struct of_device_id netsec_dt_ids[] = {
+	{ .compatible = "socionext,synquacer-netsec" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, netsec_dt_ids);
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id netsec_acpi_ids[] = {
+	{ "SCX0001" },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, netsec_acpi_ids);
+#endif
+
+static struct platform_driver netsec_driver = {
+	.probe	= netsec_probe,
+	.remove	= netsec_remove,
+	.driver = {
+		.name = "netsec",
+		.pm = &netsec_pm_ops,
+		.of_match_table = netsec_dt_ids,
+		.acpi_match_table = ACPI_PTR(netsec_acpi_ids),
+	},
+};
+module_platform_driver(netsec_driver);
+
+MODULE_AUTHOR("Jassi Brar <jaswinder.singh@linaro.org>");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_DESCRIPTION("NETSEC Ethernet driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c
new file mode 100644
index 000000000000..f5c5984afefb
--- /dev/null
+++ b/drivers/net/ethernet/socionext/sni_ave.c
@@ -0,0 +1,1736 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * sni_ave.c - Socionext UniPhier AVE ethernet driver
+ * Copyright 2014 Panasonic Corporation
+ * Copyright 2015-2017 Socionext Inc.
+ */
+
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/etherdevice.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/mii.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/of_net.h>
+#include <linux/of_mdio.h>
+#include <linux/of_platform.h>
+#include <linux/phy.h>
+#include <linux/reset.h>
+#include <linux/types.h>
+#include <linux/u64_stats_sync.h>
+
+/* General Register Group */
+#define AVE_IDR			0x000	/* ID */
+#define AVE_VR			0x004	/* Version */
+#define AVE_GRR			0x008	/* Global Reset */
+#define AVE_CFGR		0x00c	/* Configuration */
+
+/* Interrupt Register Group */
+#define AVE_GIMR		0x100	/* Global Interrupt Mask */
+#define AVE_GISR		0x104	/* Global Interrupt Status */
+
+/* MAC Register Group */
+#define AVE_TXCR		0x200	/* TX Setup */
+#define AVE_RXCR		0x204	/* RX Setup */
+#define AVE_RXMAC1R		0x208	/* MAC address (lower) */
+#define AVE_RXMAC2R		0x20c	/* MAC address (upper) */
+#define AVE_MDIOCTR		0x214	/* MDIO Control */
+#define AVE_MDIOAR		0x218	/* MDIO Address */
+#define AVE_MDIOWDR		0x21c	/* MDIO Data */
+#define AVE_MDIOSR		0x220	/* MDIO Status */
+#define AVE_MDIORDR		0x224	/* MDIO Rd Data */
+
+/* Descriptor Control Register Group */
+#define AVE_DESCC		0x300	/* Descriptor Control */
+#define AVE_TXDC		0x304	/* TX Descriptor Configuration */
+#define AVE_RXDC0		0x308	/* RX Descriptor Ring0 Configuration */
+#define AVE_IIRQC		0x34c	/* Interval IRQ Control */
+
+/* Packet Filter Register Group */
+#define AVE_PKTF_BASE		0x800	/* PF Base Address */
+#define AVE_PFMBYTE_BASE	0xd00	/* PF Mask Byte Base Address */
+#define AVE_PFMBIT_BASE		0xe00	/* PF Mask Bit Base Address */
+#define AVE_PFSEL_BASE		0xf00	/* PF Selector Base Address */
+#define AVE_PFEN		0xffc	/* Packet Filter Enable */
+#define AVE_PKTF(ent)		(AVE_PKTF_BASE + (ent) * 0x40)
+#define AVE_PFMBYTE(ent)	(AVE_PFMBYTE_BASE + (ent) * 8)
+#define AVE_PFMBIT(ent)		(AVE_PFMBIT_BASE + (ent) * 4)
+#define AVE_PFSEL(ent)		(AVE_PFSEL_BASE + (ent) * 4)
+
+/* 64bit descriptor memory */
+#define AVE_DESC_SIZE_64	12	/* Descriptor Size */
+
+#define AVE_TXDM_64		0x1000	/* Tx Descriptor Memory */
+#define AVE_RXDM_64		0x1c00	/* Rx Descriptor Memory */
+
+#define AVE_TXDM_SIZE_64	0x0ba0	/* Tx Descriptor Memory Size 3KB */
+#define AVE_RXDM_SIZE_64	0x6000	/* Rx Descriptor Memory Size 24KB */
+
+/* 32bit descriptor memory */
+#define AVE_DESC_SIZE_32	8	/* Descriptor Size */
+
+#define AVE_TXDM_32		0x1000	/* Tx Descriptor Memory */
+#define AVE_RXDM_32		0x1800	/* Rx Descriptor Memory */
+
+#define AVE_TXDM_SIZE_32	0x07c0	/* Tx Descriptor Memory Size 2KB */
+#define AVE_RXDM_SIZE_32	0x4000	/* Rx Descriptor Memory Size 16KB */
+
+/* RMII Bridge Register Group */
+#define AVE_RSTCTRL		0x8028	/* Reset control */
+#define AVE_RSTCTRL_RMIIRST	BIT(16)
+#define AVE_LINKSEL		0x8034	/* Link speed setting */
+#define AVE_LINKSEL_100M	BIT(0)
+
+/* AVE_GRR */
+#define AVE_GRR_RXFFR		BIT(5)	/* Reset RxFIFO */
+#define AVE_GRR_PHYRST		BIT(4)	/* Reset external PHY */
+#define AVE_GRR_GRST		BIT(0)	/* Reset all MAC */
+
+/* AVE_CFGR */
+#define AVE_CFGR_FLE		BIT(31)	/* Filter Function */
+#define AVE_CFGR_CHE		BIT(30)	/* Checksum Function */
+#define AVE_CFGR_MII		BIT(27)	/* Func mode (1:MII/RMII, 0:RGMII) */
+#define AVE_CFGR_IPFCEN		BIT(24)	/* IP fragment sum Enable */
+
+/* AVE_GISR (common with GIMR) */
+#define AVE_GI_PHY		BIT(24)	/* PHY interrupt */
+#define AVE_GI_TX		BIT(16)	/* Tx complete */
+#define AVE_GI_RXERR		BIT(8)	/* Receive frame more than max size */
+#define AVE_GI_RXOVF		BIT(7)	/* Overflow at the RxFIFO */
+#define AVE_GI_RXDROP		BIT(6)	/* Drop packet */
+#define AVE_GI_RXIINT		BIT(5)	/* Interval interrupt */
+
+/* AVE_TXCR */
+#define AVE_TXCR_FLOCTR		BIT(18)	/* Flow control */
+#define AVE_TXCR_TXSPD_1G	BIT(17)
+#define AVE_TXCR_TXSPD_100	BIT(16)
+
+/* AVE_RXCR */
+#define AVE_RXCR_RXEN		BIT(30)	/* Rx enable */
+#define AVE_RXCR_FDUPEN		BIT(22)	/* Interface mode */
+#define AVE_RXCR_FLOCTR		BIT(21)	/* Flow control */
+#define AVE_RXCR_AFEN		BIT(19)	/* MAC address filter */
+#define AVE_RXCR_DRPEN		BIT(18)	/* Drop pause frame */
+#define AVE_RXCR_MPSIZ_MASK	GENMASK(10, 0)
+
+/* AVE_MDIOCTR */
+#define AVE_MDIOCTR_RREQ	BIT(3)	/* Read request */
+#define AVE_MDIOCTR_WREQ	BIT(2)	/* Write request */
+
+/* AVE_MDIOSR */
+#define AVE_MDIOSR_STS		BIT(0)	/* access status */
+
+/* AVE_DESCC */
+#define AVE_DESCC_STATUS_MASK	GENMASK(31, 16)
+#define AVE_DESCC_RD0		BIT(8)	/* Enable Rx descriptor Ring0 */
+#define AVE_DESCC_RDSTP		BIT(4)	/* Pause Rx descriptor */
+#define AVE_DESCC_TD		BIT(0)	/* Enable Tx descriptor */
+
+/* AVE_TXDC */
+#define AVE_TXDC_SIZE		GENMASK(27, 16)	/* Size of Tx descriptor */
+#define AVE_TXDC_ADDR		GENMASK(11, 0)	/* Start address */
+#define AVE_TXDC_ADDR_START	0
+
+/* AVE_RXDC0 */
+#define AVE_RXDC0_SIZE		GENMASK(30, 16)	/* Size of Rx descriptor */
+#define AVE_RXDC0_ADDR		GENMASK(14, 0)	/* Start address */
+#define AVE_RXDC0_ADDR_START	0
+
+/* AVE_IIRQC */
+#define AVE_IIRQC_EN0		BIT(27)	/* Enable interval interrupt Ring0 */
+#define AVE_IIRQC_BSCK		GENMASK(15, 0)	/* Interval count unit */
+
+/* Command status for descriptor */
+#define AVE_STS_OWN		BIT(31)	/* Descriptor ownership */
+#define AVE_STS_INTR		BIT(29)	/* Request for interrupt */
+#define AVE_STS_OK		BIT(27)	/* Normal transmit */
+/* TX */
+#define AVE_STS_NOCSUM		BIT(28)	/* No use HW checksum */
+#define AVE_STS_1ST		BIT(26)	/* Head of buffer chain */
+#define AVE_STS_LAST		BIT(25)	/* Tail of buffer chain */
+#define AVE_STS_OWC		BIT(21)	/* Out of window,Late Collision */
+#define AVE_STS_EC		BIT(20)	/* Excess collision occurred */
+#define AVE_STS_PKTLEN_TX_MASK	GENMASK(15, 0)
+/* RX */
+#define AVE_STS_CSSV		BIT(21)	/* Checksum check performed */
+#define AVE_STS_CSER		BIT(20)	/* Checksum error detected */
+#define AVE_STS_PKTLEN_RX_MASK	GENMASK(10, 0)
+
+/* Packet filter */
+#define AVE_PFMBYTE_MASK0	(GENMASK(31, 8) | GENMASK(5, 0))
+#define AVE_PFMBYTE_MASK1	GENMASK(25, 0)
+#define AVE_PFMBIT_MASK		GENMASK(15, 0)
+
+#define AVE_PF_SIZE		17	/* Number of all packet filter */
+#define AVE_PF_MULTICAST_SIZE	7	/* Number of multicast filter */
+
+#define AVE_PFNUM_FILTER	0	/* No.0 */
+#define AVE_PFNUM_UNICAST	1	/* No.1 */
+#define AVE_PFNUM_BROADCAST	2	/* No.2 */
+#define AVE_PFNUM_MULTICAST	11	/* No.11-17 */
+
+/* NETIF Message control */
+#define AVE_DEFAULT_MSG_ENABLE	(NETIF_MSG_DRV    |	\
+				 NETIF_MSG_PROBE  |	\
+				 NETIF_MSG_LINK   |	\
+				 NETIF_MSG_TIMER  |	\
+				 NETIF_MSG_IFDOWN |	\
+				 NETIF_MSG_IFUP   |	\
+				 NETIF_MSG_RX_ERR |	\
+				 NETIF_MSG_TX_ERR)
+
+/* Parameter for descriptor */
+#define AVE_NR_TXDESC		32	/* Tx descriptor */
+#define AVE_NR_RXDESC		64	/* Rx descriptor */
+
+#define AVE_DESC_OFS_CMDSTS	0
+#define AVE_DESC_OFS_ADDRL	4
+#define AVE_DESC_OFS_ADDRU	8
+
+/* Parameter for ethernet frame */
+#define AVE_MAX_ETHFRAME	1518
+
+/* Parameter for interrupt */
+#define AVE_INTM_COUNT		20
+#define AVE_FORCE_TXINTCNT	1
+
+#define IS_DESC_64BIT(p)	((p)->data->is_desc_64bit)
+
+enum desc_id {
+	AVE_DESCID_RX,
+	AVE_DESCID_TX,
+};
+
+enum desc_state {
+	AVE_DESC_RX_PERMIT,
+	AVE_DESC_RX_SUSPEND,
+	AVE_DESC_START,
+	AVE_DESC_STOP,
+};
+
+struct ave_desc {
+	struct sk_buff	*skbs;
+	dma_addr_t	skbs_dma;
+	size_t		skbs_dmalen;
+};
+
+struct ave_desc_info {
+	u32	ndesc;		/* number of descriptor */
+	u32	daddr;		/* start address of descriptor */
+	u32	proc_idx;	/* index of processing packet */
+	u32	done_idx;	/* index of processed packet */
+	struct ave_desc *desc;	/* skb info related descriptor */
+};
+
+struct ave_soc_data {
+	bool	is_desc_64bit;
+};
+
+struct ave_stats {
+	struct	u64_stats_sync	syncp;
+	u64	packets;
+	u64	bytes;
+	u64	errors;
+	u64	dropped;
+	u64	collisions;
+	u64	fifo_errors;
+};
+
+struct ave_private {
+	void __iomem            *base;
+	int                     irq;
+	int			phy_id;
+	unsigned int		desc_size;
+	u32			msg_enable;
+	struct clk		*clk;
+	struct reset_control	*rst;
+	phy_interface_t		phy_mode;
+	struct phy_device	*phydev;
+	struct mii_bus		*mdio;
+
+	/* stats */
+	struct ave_stats	stats_rx;
+	struct ave_stats	stats_tx;
+
+	/* NAPI support */
+	struct net_device	*ndev;
+	struct napi_struct	napi_rx;
+	struct napi_struct	napi_tx;
+
+	/* descriptor */
+	struct ave_desc_info	rx;
+	struct ave_desc_info	tx;
+
+	/* flow control */
+	int pause_auto;
+	int pause_rx;
+	int pause_tx;
+
+	const struct ave_soc_data *data;
+};
+
+static u32 ave_desc_read(struct net_device *ndev, enum desc_id id, int entry,
+			 int offset)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 addr;
+
+	addr = ((id == AVE_DESCID_TX) ? priv->tx.daddr : priv->rx.daddr)
+		+ entry * priv->desc_size + offset;
+
+	return readl(priv->base + addr);
+}
+
+static u32 ave_desc_read_cmdsts(struct net_device *ndev, enum desc_id id,
+				int entry)
+{
+	return ave_desc_read(ndev, id, entry, AVE_DESC_OFS_CMDSTS);
+}
+
+static void ave_desc_write(struct net_device *ndev, enum desc_id id,
+			   int entry, int offset, u32 val)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 addr;
+
+	addr = ((id == AVE_DESCID_TX) ? priv->tx.daddr : priv->rx.daddr)
+		+ entry * priv->desc_size + offset;
+
+	writel(val, priv->base + addr);
+}
+
+static void ave_desc_write_cmdsts(struct net_device *ndev, enum desc_id id,
+				  int entry, u32 val)
+{
+	ave_desc_write(ndev, id, entry, AVE_DESC_OFS_CMDSTS, val);
+}
+
+static void ave_desc_write_addr(struct net_device *ndev, enum desc_id id,
+				int entry, dma_addr_t paddr)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	ave_desc_write(ndev, id, entry, AVE_DESC_OFS_ADDRL,
+		       lower_32_bits(paddr));
+	if (IS_DESC_64BIT(priv))
+		ave_desc_write(ndev, id,
+			       entry, AVE_DESC_OFS_ADDRU,
+			       upper_32_bits(paddr));
+}
+
+static u32 ave_irq_disable_all(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 ret;
+
+	ret = readl(priv->base + AVE_GIMR);
+	writel(0, priv->base + AVE_GIMR);
+
+	return ret;
+}
+
+static void ave_irq_restore(struct net_device *ndev, u32 val)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	writel(val, priv->base + AVE_GIMR);
+}
+
+static void ave_irq_enable(struct net_device *ndev, u32 bitflag)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	writel(readl(priv->base + AVE_GIMR) | bitflag, priv->base + AVE_GIMR);
+	writel(bitflag, priv->base + AVE_GISR);
+}
+
+static void ave_hw_write_macaddr(struct net_device *ndev,
+				 const unsigned char *mac_addr,
+				 int reg1, int reg2)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	writel(mac_addr[0] | mac_addr[1] << 8 |
+	       mac_addr[2] << 16 | mac_addr[3] << 24, priv->base + reg1);
+	writel(mac_addr[4] | mac_addr[5] << 8, priv->base + reg2);
+}
+
+static void ave_hw_read_version(struct net_device *ndev, char *buf, int len)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 major, minor, vr;
+
+	vr = readl(priv->base + AVE_VR);
+	major = (vr & GENMASK(15, 8)) >> 8;
+	minor = (vr & GENMASK(7, 0));
+	snprintf(buf, len, "v%u.%u", major, minor);
+}
+
+static void ave_ethtool_get_drvinfo(struct net_device *ndev,
+				    struct ethtool_drvinfo *info)
+{
+	struct device *dev = ndev->dev.parent;
+
+	strlcpy(info->driver, dev->driver->name, sizeof(info->driver));
+	strlcpy(info->bus_info, dev_name(dev), sizeof(info->bus_info));
+	ave_hw_read_version(ndev, info->fw_version, sizeof(info->fw_version));
+}
+
+static u32 ave_ethtool_get_msglevel(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	return priv->msg_enable;
+}
+
+static void ave_ethtool_set_msglevel(struct net_device *ndev, u32 val)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	priv->msg_enable = val;
+}
+
+static void ave_ethtool_get_wol(struct net_device *ndev,
+				struct ethtool_wolinfo *wol)
+{
+	wol->supported = 0;
+	wol->wolopts   = 0;
+
+	if (ndev->phydev)
+		phy_ethtool_get_wol(ndev->phydev, wol);
+}
+
+static int ave_ethtool_set_wol(struct net_device *ndev,
+			       struct ethtool_wolinfo *wol)
+{
+	int ret;
+
+	if (!ndev->phydev ||
+	    (wol->wolopts & (WAKE_ARP | WAKE_MAGICSECURE)))
+		return -EOPNOTSUPP;
+
+	ret = phy_ethtool_set_wol(ndev->phydev, wol);
+	if (!ret)
+		device_set_wakeup_enable(&ndev->dev, !!wol->wolopts);
+
+	return ret;
+}
+
+static void ave_ethtool_get_pauseparam(struct net_device *ndev,
+				       struct ethtool_pauseparam *pause)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	pause->autoneg  = priv->pause_auto;
+	pause->rx_pause = priv->pause_rx;
+	pause->tx_pause = priv->pause_tx;
+}
+
+static int ave_ethtool_set_pauseparam(struct net_device *ndev,
+				      struct ethtool_pauseparam *pause)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	struct phy_device *phydev = ndev->phydev;
+
+	if (!phydev)
+		return -EINVAL;
+
+	priv->pause_auto = pause->autoneg;
+	priv->pause_rx   = pause->rx_pause;
+	priv->pause_tx   = pause->tx_pause;
+
+	phydev->advertising &= ~(ADVERTISED_Pause | ADVERTISED_Asym_Pause);
+	if (pause->rx_pause)
+		phydev->advertising |= ADVERTISED_Pause | ADVERTISED_Asym_Pause;
+	if (pause->tx_pause)
+		phydev->advertising ^= ADVERTISED_Asym_Pause;
+
+	if (pause->autoneg) {
+		if (netif_running(ndev))
+			phy_start_aneg(phydev);
+	}
+
+	return 0;
+}
+
+static const struct ethtool_ops ave_ethtool_ops = {
+	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
+	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
+	.get_drvinfo		= ave_ethtool_get_drvinfo,
+	.nway_reset		= phy_ethtool_nway_reset,
+	.get_link		= ethtool_op_get_link,
+	.get_msglevel		= ave_ethtool_get_msglevel,
+	.set_msglevel		= ave_ethtool_set_msglevel,
+	.get_wol		= ave_ethtool_get_wol,
+	.set_wol		= ave_ethtool_set_wol,
+	.get_pauseparam         = ave_ethtool_get_pauseparam,
+	.set_pauseparam         = ave_ethtool_set_pauseparam,
+};
+
+static int ave_mdiobus_read(struct mii_bus *bus, int phyid, int regnum)
+{
+	struct net_device *ndev = bus->priv;
+	struct ave_private *priv;
+	u32 mdioctl, mdiosr;
+	int ret;
+
+	priv = netdev_priv(ndev);
+
+	/* write address */
+	writel((phyid << 8) | regnum, priv->base + AVE_MDIOAR);
+
+	/* read request */
+	mdioctl = readl(priv->base + AVE_MDIOCTR);
+	writel((mdioctl | AVE_MDIOCTR_RREQ) & ~AVE_MDIOCTR_WREQ,
+	       priv->base + AVE_MDIOCTR);
+
+	ret = readl_poll_timeout(priv->base + AVE_MDIOSR, mdiosr,
+				 !(mdiosr & AVE_MDIOSR_STS), 20, 2000);
+	if (ret) {
+		netdev_err(ndev, "failed to read (phy:%d reg:%x)\n",
+			   phyid, regnum);
+		return ret;
+	}
+
+	return readl(priv->base + AVE_MDIORDR) & GENMASK(15, 0);
+}
+
+static int ave_mdiobus_write(struct mii_bus *bus, int phyid, int regnum,
+			     u16 val)
+{
+	struct net_device *ndev = bus->priv;
+	struct ave_private *priv;
+	u32 mdioctl, mdiosr;
+	int ret;
+
+	priv = netdev_priv(ndev);
+
+	/* write address */
+	writel((phyid << 8) | regnum, priv->base + AVE_MDIOAR);
+
+	/* write data */
+	writel(val, priv->base + AVE_MDIOWDR);
+
+	/* write request */
+	mdioctl = readl(priv->base + AVE_MDIOCTR);
+	writel((mdioctl | AVE_MDIOCTR_WREQ) & ~AVE_MDIOCTR_RREQ,
+	       priv->base + AVE_MDIOCTR);
+
+	ret = readl_poll_timeout(priv->base + AVE_MDIOSR, mdiosr,
+				 !(mdiosr & AVE_MDIOSR_STS), 20, 2000);
+	if (ret)
+		netdev_err(ndev, "failed to write (phy:%d reg:%x)\n",
+			   phyid, regnum);
+
+	return ret;
+}
+
+static int ave_dma_map(struct net_device *ndev, struct ave_desc *desc,
+		       void *ptr, size_t len, enum dma_data_direction dir,
+		       dma_addr_t *paddr)
+{
+	dma_addr_t map_addr;
+
+	map_addr = dma_map_single(ndev->dev.parent, ptr, len, dir);
+	if (unlikely(dma_mapping_error(ndev->dev.parent, map_addr)))
+		return -ENOMEM;
+
+	desc->skbs_dma = map_addr;
+	desc->skbs_dmalen = len;
+	*paddr = map_addr;
+
+	return 0;
+}
+
+static void ave_dma_unmap(struct net_device *ndev, struct ave_desc *desc,
+			  enum dma_data_direction dir)
+{
+	if (!desc->skbs_dma)
+		return;
+
+	dma_unmap_single(ndev->dev.parent,
+			 desc->skbs_dma, desc->skbs_dmalen, dir);
+	desc->skbs_dma = 0;
+}
+
+/* Prepare Rx descriptor and memory */
+static int ave_rxdesc_prepare(struct net_device *ndev, int entry)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	struct sk_buff *skb;
+	dma_addr_t paddr;
+	int ret;
+
+	skb = priv->rx.desc[entry].skbs;
+	if (!skb) {
+		skb = netdev_alloc_skb_ip_align(ndev,
+						AVE_MAX_ETHFRAME);
+		if (!skb) {
+			netdev_err(ndev, "can't allocate skb for Rx\n");
+			return -ENOMEM;
+		}
+	}
+
+	/* set disable to cmdsts */
+	ave_desc_write_cmdsts(ndev, AVE_DESCID_RX, entry,
+			      AVE_STS_INTR | AVE_STS_OWN);
+
+	/* map Rx buffer
+	 * Rx buffer set to the Rx descriptor has two restrictions:
+	 * - Rx buffer address is 4 byte aligned.
+	 * - Rx buffer begins with 2 byte headroom, and data will be put from
+	 *   (buffer + 2).
+	 * To satisfy this, specify the address to put back the buffer
+	 * pointer advanced by NET_IP_ALIGN by netdev_alloc_skb_ip_align(),
+	 * and expand the map size by NET_IP_ALIGN.
+	 */
+	ret = ave_dma_map(ndev, &priv->rx.desc[entry],
+			  skb->data - NET_IP_ALIGN,
+			  AVE_MAX_ETHFRAME + NET_IP_ALIGN,
+			  DMA_FROM_DEVICE, &paddr);
+	if (ret) {
+		netdev_err(ndev, "can't map skb for Rx\n");
+		dev_kfree_skb_any(skb);
+		return ret;
+	}
+	priv->rx.desc[entry].skbs = skb;
+
+	/* set buffer pointer */
+	ave_desc_write_addr(ndev, AVE_DESCID_RX, entry, paddr);
+
+	/* set enable to cmdsts */
+	ave_desc_write_cmdsts(ndev, AVE_DESCID_RX, entry,
+			      AVE_STS_INTR | AVE_MAX_ETHFRAME);
+
+	return ret;
+}
+
+/* Switch state of descriptor */
+static int ave_desc_switch(struct net_device *ndev, enum desc_state state)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	int ret = 0;
+	u32 val;
+
+	switch (state) {
+	case AVE_DESC_START:
+		writel(AVE_DESCC_TD | AVE_DESCC_RD0, priv->base + AVE_DESCC);
+		break;
+
+	case AVE_DESC_STOP:
+		writel(0, priv->base + AVE_DESCC);
+		if (readl_poll_timeout(priv->base + AVE_DESCC, val, !val,
+				       150, 15000)) {
+			netdev_err(ndev, "can't stop descriptor\n");
+			ret = -EBUSY;
+		}
+		break;
+
+	case AVE_DESC_RX_SUSPEND:
+		val = readl(priv->base + AVE_DESCC);
+		val |= AVE_DESCC_RDSTP;
+		val &= ~AVE_DESCC_STATUS_MASK;
+		writel(val, priv->base + AVE_DESCC);
+		if (readl_poll_timeout(priv->base + AVE_DESCC, val,
+				       val & (AVE_DESCC_RDSTP << 16),
+				       150, 150000)) {
+			netdev_err(ndev, "can't suspend descriptor\n");
+			ret = -EBUSY;
+		}
+		break;
+
+	case AVE_DESC_RX_PERMIT:
+		val = readl(priv->base + AVE_DESCC);
+		val &= ~AVE_DESCC_RDSTP;
+		val &= ~AVE_DESCC_STATUS_MASK;
+		writel(val, priv->base + AVE_DESCC);
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int ave_tx_complete(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 proc_idx, done_idx, ndesc, cmdsts;
+	unsigned int nr_freebuf = 0;
+	unsigned int tx_packets = 0;
+	unsigned int tx_bytes = 0;
+
+	proc_idx = priv->tx.proc_idx;
+	done_idx = priv->tx.done_idx;
+	ndesc    = priv->tx.ndesc;
+
+	/* free pre-stored skb from done_idx to proc_idx */
+	while (proc_idx != done_idx) {
+		cmdsts = ave_desc_read_cmdsts(ndev, AVE_DESCID_TX, done_idx);
+
+		/* do nothing if owner is HW (==1 for Tx) */
+		if (cmdsts & AVE_STS_OWN)
+			break;
+
+		/* check Tx status and updates statistics */
+		if (cmdsts & AVE_STS_OK) {
+			tx_bytes += cmdsts & AVE_STS_PKTLEN_TX_MASK;
+			/* success */
+			if (cmdsts & AVE_STS_LAST)
+				tx_packets++;
+		} else {
+			/* error */
+			if (cmdsts & AVE_STS_LAST) {
+				priv->stats_tx.errors++;
+				if (cmdsts & (AVE_STS_OWC | AVE_STS_EC))
+					priv->stats_tx.collisions++;
+			}
+		}
+
+		/* release skb */
+		if (priv->tx.desc[done_idx].skbs) {
+			ave_dma_unmap(ndev, &priv->tx.desc[done_idx],
+				      DMA_TO_DEVICE);
+			dev_consume_skb_any(priv->tx.desc[done_idx].skbs);
+			priv->tx.desc[done_idx].skbs = NULL;
+			nr_freebuf++;
+		}
+		done_idx = (done_idx + 1) % ndesc;
+	}
+
+	priv->tx.done_idx = done_idx;
+
+	/* update stats */
+	u64_stats_update_begin(&priv->stats_tx.syncp);
+	priv->stats_tx.packets += tx_packets;
+	priv->stats_tx.bytes   += tx_bytes;
+	u64_stats_update_end(&priv->stats_tx.syncp);
+
+	/* wake queue for freeing buffer */
+	if (unlikely(netif_queue_stopped(ndev)) && nr_freebuf)
+		netif_wake_queue(ndev);
+
+	return nr_freebuf;
+}
+
+static int ave_rx_receive(struct net_device *ndev, int num)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	unsigned int rx_packets = 0;
+	unsigned int rx_bytes = 0;
+	u32 proc_idx, done_idx;
+	struct sk_buff *skb;
+	unsigned int pktlen;
+	int restpkt, npkts;
+	u32 ndesc, cmdsts;
+
+	proc_idx = priv->rx.proc_idx;
+	done_idx = priv->rx.done_idx;
+	ndesc    = priv->rx.ndesc;
+	restpkt  = ((proc_idx + ndesc - 1) - done_idx) % ndesc;
+
+	for (npkts = 0; npkts < num; npkts++) {
+		/* we can't receive more packet, so fill desc quickly */
+		if (--restpkt < 0)
+			break;
+
+		cmdsts = ave_desc_read_cmdsts(ndev, AVE_DESCID_RX, proc_idx);
+
+		/* do nothing if owner is HW (==0 for Rx) */
+		if (!(cmdsts & AVE_STS_OWN))
+			break;
+
+		if (!(cmdsts & AVE_STS_OK)) {
+			priv->stats_rx.errors++;
+			proc_idx = (proc_idx + 1) % ndesc;
+			continue;
+		}
+
+		pktlen = cmdsts & AVE_STS_PKTLEN_RX_MASK;
+
+		/* get skbuff for rx */
+		skb = priv->rx.desc[proc_idx].skbs;
+		priv->rx.desc[proc_idx].skbs = NULL;
+
+		ave_dma_unmap(ndev, &priv->rx.desc[proc_idx], DMA_FROM_DEVICE);
+
+		skb->dev = ndev;
+		skb_put(skb, pktlen);
+		skb->protocol = eth_type_trans(skb, ndev);
+
+		if ((cmdsts & AVE_STS_CSSV) && (!(cmdsts & AVE_STS_CSER)))
+			skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+		rx_packets++;
+		rx_bytes += pktlen;
+
+		netif_receive_skb(skb);
+
+		proc_idx = (proc_idx + 1) % ndesc;
+	}
+
+	priv->rx.proc_idx = proc_idx;
+
+	/* update stats */
+	u64_stats_update_begin(&priv->stats_rx.syncp);
+	priv->stats_rx.packets += rx_packets;
+	priv->stats_rx.bytes   += rx_bytes;
+	u64_stats_update_end(&priv->stats_rx.syncp);
+
+	/* refill the Rx buffers */
+	while (proc_idx != done_idx) {
+		if (ave_rxdesc_prepare(ndev, done_idx))
+			break;
+		done_idx = (done_idx + 1) % ndesc;
+	}
+
+	priv->rx.done_idx = done_idx;
+
+	return npkts;
+}
+
+static int ave_napi_poll_rx(struct napi_struct *napi, int budget)
+{
+	struct ave_private *priv;
+	struct net_device *ndev;
+	int num;
+
+	priv = container_of(napi, struct ave_private, napi_rx);
+	ndev = priv->ndev;
+
+	num = ave_rx_receive(ndev, budget);
+	if (num < budget) {
+		napi_complete_done(napi, num);
+
+		/* enable Rx interrupt when NAPI finishes */
+		ave_irq_enable(ndev, AVE_GI_RXIINT);
+	}
+
+	return num;
+}
+
+static int ave_napi_poll_tx(struct napi_struct *napi, int budget)
+{
+	struct ave_private *priv;
+	struct net_device *ndev;
+	int num;
+
+	priv = container_of(napi, struct ave_private, napi_tx);
+	ndev = priv->ndev;
+
+	num = ave_tx_complete(ndev);
+	napi_complete(napi);
+
+	/* enable Tx interrupt when NAPI finishes */
+	ave_irq_enable(ndev, AVE_GI_TX);
+
+	return num;
+}
+
+static void ave_global_reset(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 val;
+
+	/* set config register */
+	val = AVE_CFGR_FLE | AVE_CFGR_IPFCEN | AVE_CFGR_CHE;
+	if (!phy_interface_mode_is_rgmii(priv->phy_mode))
+		val |= AVE_CFGR_MII;
+	writel(val, priv->base + AVE_CFGR);
+
+	/* reset RMII register */
+	val = readl(priv->base + AVE_RSTCTRL);
+	val &= ~AVE_RSTCTRL_RMIIRST;
+	writel(val, priv->base + AVE_RSTCTRL);
+
+	/* assert reset */
+	writel(AVE_GRR_GRST | AVE_GRR_PHYRST, priv->base + AVE_GRR);
+	msleep(20);
+
+	/* 1st, negate PHY reset only */
+	writel(AVE_GRR_GRST, priv->base + AVE_GRR);
+	msleep(40);
+
+	/* negate reset */
+	writel(0, priv->base + AVE_GRR);
+	msleep(40);
+
+	/* negate RMII register */
+	val = readl(priv->base + AVE_RSTCTRL);
+	val |= AVE_RSTCTRL_RMIIRST;
+	writel(val, priv->base + AVE_RSTCTRL);
+
+	ave_irq_disable_all(ndev);
+}
+
+static void ave_rxfifo_reset(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 rxcr_org;
+
+	/* save and disable MAC receive op */
+	rxcr_org = readl(priv->base + AVE_RXCR);
+	writel(rxcr_org & (~AVE_RXCR_RXEN), priv->base + AVE_RXCR);
+
+	/* suspend Rx descriptor */
+	ave_desc_switch(ndev, AVE_DESC_RX_SUSPEND);
+
+	/* receive all packets before descriptor starts */
+	ave_rx_receive(ndev, priv->rx.ndesc);
+
+	/* assert reset */
+	writel(AVE_GRR_RXFFR, priv->base + AVE_GRR);
+	usleep_range(40, 50);
+
+	/* negate reset */
+	writel(0, priv->base + AVE_GRR);
+	usleep_range(10, 20);
+
+	/* negate interrupt status */
+	writel(AVE_GI_RXOVF, priv->base + AVE_GISR);
+
+	/* permit descriptor */
+	ave_desc_switch(ndev, AVE_DESC_RX_PERMIT);
+
+	/* restore MAC reccieve op */
+	writel(rxcr_org, priv->base + AVE_RXCR);
+}
+
+static irqreturn_t ave_irq_handler(int irq, void *netdev)
+{
+	struct net_device *ndev = (struct net_device *)netdev;
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 gimr_val, gisr_val;
+
+	gimr_val = ave_irq_disable_all(ndev);
+
+	/* get interrupt status */
+	gisr_val = readl(priv->base + AVE_GISR);
+
+	/* PHY */
+	if (gisr_val & AVE_GI_PHY)
+		writel(AVE_GI_PHY, priv->base + AVE_GISR);
+
+	/* check exceeding packet */
+	if (gisr_val & AVE_GI_RXERR) {
+		writel(AVE_GI_RXERR, priv->base + AVE_GISR);
+		netdev_err(ndev, "receive a packet exceeding frame buffer\n");
+	}
+
+	gisr_val &= gimr_val;
+	if (!gisr_val)
+		goto exit_isr;
+
+	/* RxFIFO overflow */
+	if (gisr_val & AVE_GI_RXOVF) {
+		priv->stats_rx.fifo_errors++;
+		ave_rxfifo_reset(ndev);
+		goto exit_isr;
+	}
+
+	/* Rx drop */
+	if (gisr_val & AVE_GI_RXDROP) {
+		priv->stats_rx.dropped++;
+		writel(AVE_GI_RXDROP, priv->base + AVE_GISR);
+	}
+
+	/* Rx interval */
+	if (gisr_val & AVE_GI_RXIINT) {
+		napi_schedule(&priv->napi_rx);
+		/* still force to disable Rx interrupt until NAPI finishes */
+		gimr_val &= ~AVE_GI_RXIINT;
+	}
+
+	/* Tx completed */
+	if (gisr_val & AVE_GI_TX) {
+		napi_schedule(&priv->napi_tx);
+		/* still force to disable Tx interrupt until NAPI finishes */
+		gimr_val &= ~AVE_GI_TX;
+	}
+
+exit_isr:
+	ave_irq_restore(ndev, gimr_val);
+
+	return IRQ_HANDLED;
+}
+
+static int ave_pfsel_start(struct net_device *ndev, unsigned int entry)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 val;
+
+	if (WARN_ON(entry > AVE_PF_SIZE))
+		return -EINVAL;
+
+	val = readl(priv->base + AVE_PFEN);
+	writel(val | BIT(entry), priv->base + AVE_PFEN);
+
+	return 0;
+}
+
+static int ave_pfsel_stop(struct net_device *ndev, unsigned int entry)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 val;
+
+	if (WARN_ON(entry > AVE_PF_SIZE))
+		return -EINVAL;
+
+	val = readl(priv->base + AVE_PFEN);
+	writel(val & ~BIT(entry), priv->base + AVE_PFEN);
+
+	return 0;
+}
+
+static int ave_pfsel_set_macaddr(struct net_device *ndev,
+				 unsigned int entry,
+				 const unsigned char *mac_addr,
+				 unsigned int set_size)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	if (WARN_ON(entry > AVE_PF_SIZE))
+		return -EINVAL;
+	if (WARN_ON(set_size > 6))
+		return -EINVAL;
+
+	ave_pfsel_stop(ndev, entry);
+
+	/* set MAC address for the filter */
+	ave_hw_write_macaddr(ndev, mac_addr,
+			     AVE_PKTF(entry), AVE_PKTF(entry) + 4);
+
+	/* set byte mask */
+	writel(GENMASK(31, set_size) & AVE_PFMBYTE_MASK0,
+	       priv->base + AVE_PFMBYTE(entry));
+	writel(AVE_PFMBYTE_MASK1, priv->base + AVE_PFMBYTE(entry) + 4);
+
+	/* set bit mask filter */
+	writel(AVE_PFMBIT_MASK, priv->base + AVE_PFMBIT(entry));
+
+	/* set selector to ring 0 */
+	writel(0, priv->base + AVE_PFSEL(entry));
+
+	/* restart filter */
+	ave_pfsel_start(ndev, entry);
+
+	return 0;
+}
+
+static void ave_pfsel_set_promisc(struct net_device *ndev,
+				  unsigned int entry, u32 rxring)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	if (WARN_ON(entry > AVE_PF_SIZE))
+		return;
+
+	ave_pfsel_stop(ndev, entry);
+
+	/* set byte mask */
+	writel(AVE_PFMBYTE_MASK0, priv->base + AVE_PFMBYTE(entry));
+	writel(AVE_PFMBYTE_MASK1, priv->base + AVE_PFMBYTE(entry) + 4);
+
+	/* set bit mask filter */
+	writel(AVE_PFMBIT_MASK, priv->base + AVE_PFMBIT(entry));
+
+	/* set selector to rxring */
+	writel(rxring, priv->base + AVE_PFSEL(entry));
+
+	ave_pfsel_start(ndev, entry);
+}
+
+static void ave_pfsel_init(struct net_device *ndev)
+{
+	unsigned char bcast_mac[ETH_ALEN];
+	int i;
+
+	eth_broadcast_addr(bcast_mac);
+
+	for (i = 0; i < AVE_PF_SIZE; i++)
+		ave_pfsel_stop(ndev, i);
+
+	/* promiscious entry, select ring 0 */
+	ave_pfsel_set_promisc(ndev, AVE_PFNUM_FILTER, 0);
+
+	/* unicast entry */
+	ave_pfsel_set_macaddr(ndev, AVE_PFNUM_UNICAST, ndev->dev_addr, 6);
+
+	/* broadcast entry */
+	ave_pfsel_set_macaddr(ndev, AVE_PFNUM_BROADCAST, bcast_mac, 6);
+}
+
+static void ave_phy_adjust_link(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	struct phy_device *phydev = ndev->phydev;
+	u32 val, txcr, rxcr, rxcr_org;
+	u16 rmt_adv = 0, lcl_adv = 0;
+	u8 cap;
+
+	/* set RGMII speed */
+	val = readl(priv->base + AVE_TXCR);
+	val &= ~(AVE_TXCR_TXSPD_100 | AVE_TXCR_TXSPD_1G);
+
+	if (phy_interface_is_rgmii(phydev) && phydev->speed == SPEED_1000)
+		val |= AVE_TXCR_TXSPD_1G;
+	else if (phydev->speed == SPEED_100)
+		val |= AVE_TXCR_TXSPD_100;
+
+	writel(val, priv->base + AVE_TXCR);
+
+	/* set RMII speed (100M/10M only) */
+	if (!phy_interface_is_rgmii(phydev)) {
+		val = readl(priv->base + AVE_LINKSEL);
+		if (phydev->speed == SPEED_10)
+			val &= ~AVE_LINKSEL_100M;
+		else
+			val |= AVE_LINKSEL_100M;
+		writel(val, priv->base + AVE_LINKSEL);
+	}
+
+	/* check current RXCR/TXCR */
+	rxcr = readl(priv->base + AVE_RXCR);
+	txcr = readl(priv->base + AVE_TXCR);
+	rxcr_org = rxcr;
+
+	if (phydev->duplex) {
+		rxcr |= AVE_RXCR_FDUPEN;
+
+		if (phydev->pause)
+			rmt_adv |= LPA_PAUSE_CAP;
+		if (phydev->asym_pause)
+			rmt_adv |= LPA_PAUSE_ASYM;
+		if (phydev->advertising & ADVERTISED_Pause)
+			lcl_adv |= ADVERTISE_PAUSE_CAP;
+		if (phydev->advertising & ADVERTISED_Asym_Pause)
+			lcl_adv |= ADVERTISE_PAUSE_ASYM;
+
+		cap = mii_resolve_flowctrl_fdx(lcl_adv, rmt_adv);
+		if (cap & FLOW_CTRL_TX)
+			txcr |= AVE_TXCR_FLOCTR;
+		else
+			txcr &= ~AVE_TXCR_FLOCTR;
+		if (cap & FLOW_CTRL_RX)
+			rxcr |= AVE_RXCR_FLOCTR;
+		else
+			rxcr &= ~AVE_RXCR_FLOCTR;
+	} else {
+		rxcr &= ~AVE_RXCR_FDUPEN;
+		rxcr &= ~AVE_RXCR_FLOCTR;
+		txcr &= ~AVE_TXCR_FLOCTR;
+	}
+
+	if (rxcr_org != rxcr) {
+		/* disable Rx mac */
+		writel(rxcr & ~AVE_RXCR_RXEN, priv->base + AVE_RXCR);
+		/* change and enable TX/Rx mac */
+		writel(txcr, priv->base + AVE_TXCR);
+		writel(rxcr, priv->base + AVE_RXCR);
+	}
+
+	phy_print_status(phydev);
+}
+
+static void ave_macaddr_init(struct net_device *ndev)
+{
+	ave_hw_write_macaddr(ndev, ndev->dev_addr, AVE_RXMAC1R, AVE_RXMAC2R);
+
+	/* pfsel unicast entry */
+	ave_pfsel_set_macaddr(ndev, AVE_PFNUM_UNICAST, ndev->dev_addr, 6);
+}
+
+static int ave_init(struct net_device *ndev)
+{
+	struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
+	struct ave_private *priv = netdev_priv(ndev);
+	struct device *dev = ndev->dev.parent;
+	struct device_node *np = dev->of_node;
+	struct device_node *mdio_np;
+	struct phy_device *phydev;
+	int ret;
+
+	/* enable clk because of hw access until ndo_open */
+	ret = clk_prepare_enable(priv->clk);
+	if (ret) {
+		dev_err(dev, "can't enable clock\n");
+		return ret;
+	}
+	ret = reset_control_deassert(priv->rst);
+	if (ret) {
+		dev_err(dev, "can't deassert reset\n");
+		goto out_clk_disable;
+	}
+
+	ave_global_reset(ndev);
+
+	mdio_np = of_get_child_by_name(np, "mdio");
+	if (!mdio_np) {
+		dev_err(dev, "mdio node not found\n");
+		ret = -EINVAL;
+		goto out_reset_assert;
+	}
+	ret = of_mdiobus_register(priv->mdio, mdio_np);
+	of_node_put(mdio_np);
+	if (ret) {
+		dev_err(dev, "failed to register mdiobus\n");
+		goto out_reset_assert;
+	}
+
+	phydev = of_phy_get_and_connect(ndev, np, ave_phy_adjust_link);
+	if (!phydev) {
+		dev_err(dev, "could not attach to PHY\n");
+		ret = -ENODEV;
+		goto out_mdio_unregister;
+	}
+
+	priv->phydev = phydev;
+
+	phy_ethtool_get_wol(phydev, &wol);
+	device_set_wakeup_capable(&ndev->dev, !!wol.supported);
+
+	if (!phy_interface_is_rgmii(phydev)) {
+		phydev->supported &= ~PHY_GBIT_FEATURES;
+		phydev->supported |= PHY_BASIC_FEATURES;
+	}
+	phydev->supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause;
+
+	phy_attached_info(phydev);
+
+	return 0;
+
+out_mdio_unregister:
+	mdiobus_unregister(priv->mdio);
+out_reset_assert:
+	reset_control_assert(priv->rst);
+out_clk_disable:
+	clk_disable_unprepare(priv->clk);
+
+	return ret;
+}
+
+static void ave_uninit(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+
+	phy_disconnect(priv->phydev);
+	mdiobus_unregister(priv->mdio);
+
+	/* disable clk because of hw access after ndo_stop */
+	reset_control_assert(priv->rst);
+	clk_disable_unprepare(priv->clk);
+}
+
+static int ave_open(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	int entry;
+	int ret;
+	u32 val;
+
+	ret = request_irq(priv->irq, ave_irq_handler, IRQF_SHARED, ndev->name,
+			  ndev);
+	if (ret)
+		return ret;
+
+	priv->tx.desc = kcalloc(priv->tx.ndesc, sizeof(*priv->tx.desc),
+				GFP_KERNEL);
+	if (!priv->tx.desc) {
+		ret = -ENOMEM;
+		goto out_free_irq;
+	}
+
+	priv->rx.desc = kcalloc(priv->rx.ndesc, sizeof(*priv->rx.desc),
+				GFP_KERNEL);
+	if (!priv->rx.desc) {
+		kfree(priv->tx.desc);
+		ret = -ENOMEM;
+		goto out_free_irq;
+	}
+
+	/* initialize Tx work and descriptor */
+	priv->tx.proc_idx = 0;
+	priv->tx.done_idx = 0;
+	for (entry = 0; entry < priv->tx.ndesc; entry++) {
+		ave_desc_write_cmdsts(ndev, AVE_DESCID_TX, entry, 0);
+		ave_desc_write_addr(ndev, AVE_DESCID_TX, entry, 0);
+	}
+	writel(AVE_TXDC_ADDR_START |
+	       (((priv->tx.ndesc * priv->desc_size) << 16) & AVE_TXDC_SIZE),
+	       priv->base + AVE_TXDC);
+
+	/* initialize Rx work and descriptor */
+	priv->rx.proc_idx = 0;
+	priv->rx.done_idx = 0;
+	for (entry = 0; entry < priv->rx.ndesc; entry++) {
+		if (ave_rxdesc_prepare(ndev, entry))
+			break;
+	}
+	writel(AVE_RXDC0_ADDR_START |
+	       (((priv->rx.ndesc * priv->desc_size) << 16) & AVE_RXDC0_SIZE),
+	       priv->base + AVE_RXDC0);
+
+	ave_desc_switch(ndev, AVE_DESC_START);
+
+	ave_pfsel_init(ndev);
+	ave_macaddr_init(ndev);
+
+	/* set Rx configuration */
+	/* full duplex, enable pause drop, enalbe flow control */
+	val = AVE_RXCR_RXEN | AVE_RXCR_FDUPEN | AVE_RXCR_DRPEN |
+		AVE_RXCR_FLOCTR | (AVE_MAX_ETHFRAME & AVE_RXCR_MPSIZ_MASK);
+	writel(val, priv->base + AVE_RXCR);
+
+	/* set Tx configuration */
+	/* enable flow control, disable loopback */
+	writel(AVE_TXCR_FLOCTR, priv->base + AVE_TXCR);
+
+	/* enable timer, clear EN,INTM, and mask interval unit(BSCK) */
+	val = readl(priv->base + AVE_IIRQC) & AVE_IIRQC_BSCK;
+	val |= AVE_IIRQC_EN0 | (AVE_INTM_COUNT << 16);
+	writel(val, priv->base + AVE_IIRQC);
+
+	val = AVE_GI_RXIINT | AVE_GI_RXOVF | AVE_GI_TX | AVE_GI_RXDROP;
+	ave_irq_restore(ndev, val);
+
+	napi_enable(&priv->napi_rx);
+	napi_enable(&priv->napi_tx);
+
+	phy_start(ndev->phydev);
+	phy_start_aneg(ndev->phydev);
+	netif_start_queue(ndev);
+
+	return 0;
+
+out_free_irq:
+	disable_irq(priv->irq);
+	free_irq(priv->irq, ndev);
+
+	return ret;
+}
+
+static int ave_stop(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	int entry;
+
+	ave_irq_disable_all(ndev);
+	disable_irq(priv->irq);
+	free_irq(priv->irq, ndev);
+
+	netif_tx_disable(ndev);
+	phy_stop(ndev->phydev);
+	napi_disable(&priv->napi_tx);
+	napi_disable(&priv->napi_rx);
+
+	ave_desc_switch(ndev, AVE_DESC_STOP);
+
+	/* free Tx buffer */
+	for (entry = 0; entry < priv->tx.ndesc; entry++) {
+		if (!priv->tx.desc[entry].skbs)
+			continue;
+
+		ave_dma_unmap(ndev, &priv->tx.desc[entry], DMA_TO_DEVICE);
+		dev_kfree_skb_any(priv->tx.desc[entry].skbs);
+		priv->tx.desc[entry].skbs = NULL;
+	}
+	priv->tx.proc_idx = 0;
+	priv->tx.done_idx = 0;
+
+	/* free Rx buffer */
+	for (entry = 0; entry < priv->rx.ndesc; entry++) {
+		if (!priv->rx.desc[entry].skbs)
+			continue;
+
+		ave_dma_unmap(ndev, &priv->rx.desc[entry], DMA_FROM_DEVICE);
+		dev_kfree_skb_any(priv->rx.desc[entry].skbs);
+		priv->rx.desc[entry].skbs = NULL;
+	}
+	priv->rx.proc_idx = 0;
+	priv->rx.done_idx = 0;
+
+	kfree(priv->tx.desc);
+	kfree(priv->rx.desc);
+
+	return 0;
+}
+
+static int ave_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	u32 proc_idx, done_idx, ndesc, cmdsts;
+	int ret, freepkt;
+	dma_addr_t paddr;
+
+	proc_idx = priv->tx.proc_idx;
+	done_idx = priv->tx.done_idx;
+	ndesc = priv->tx.ndesc;
+	freepkt = ((done_idx + ndesc - 1) - proc_idx) % ndesc;
+
+	/* stop queue when not enough entry */
+	if (unlikely(freepkt < 1)) {
+		netif_stop_queue(ndev);
+		return NETDEV_TX_BUSY;
+	}
+
+	/* add padding for short packet */
+	if (skb_put_padto(skb, ETH_ZLEN)) {
+		priv->stats_tx.dropped++;
+		return NETDEV_TX_OK;
+	}
+
+	/* map Tx buffer
+	 * Tx buffer set to the Tx descriptor doesn't have any restriction.
+	 */
+	ret = ave_dma_map(ndev, &priv->tx.desc[proc_idx],
+			  skb->data, skb->len, DMA_TO_DEVICE, &paddr);
+	if (ret) {
+		dev_kfree_skb_any(skb);
+		priv->stats_tx.dropped++;
+		return NETDEV_TX_OK;
+	}
+
+	priv->tx.desc[proc_idx].skbs = skb;
+
+	ave_desc_write_addr(ndev, AVE_DESCID_TX, proc_idx, paddr);
+
+	cmdsts = AVE_STS_OWN | AVE_STS_1ST | AVE_STS_LAST |
+		(skb->len & AVE_STS_PKTLEN_TX_MASK);
+
+	/* set interrupt per AVE_FORCE_TXINTCNT or when queue is stopped */
+	if (!(proc_idx % AVE_FORCE_TXINTCNT) || netif_queue_stopped(ndev))
+		cmdsts |= AVE_STS_INTR;
+
+	/* disable checksum calculation when skb doesn't calurate checksum */
+	if (skb->ip_summed == CHECKSUM_NONE ||
+	    skb->ip_summed == CHECKSUM_UNNECESSARY)
+		cmdsts |= AVE_STS_NOCSUM;
+
+	ave_desc_write_cmdsts(ndev, AVE_DESCID_TX, proc_idx, cmdsts);
+
+	priv->tx.proc_idx = (proc_idx + 1) % ndesc;
+
+	return NETDEV_TX_OK;
+}
+
+static int ave_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd)
+{
+	return phy_mii_ioctl(ndev->phydev, ifr, cmd);
+}
+
+static const u8 v4multi_macadr[] = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00 };
+static const u8 v6multi_macadr[] = { 0x33, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+static void ave_set_rx_mode(struct net_device *ndev)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	struct netdev_hw_addr *hw_adr;
+	int count, mc_cnt;
+	u32 val;
+
+	/* MAC addr filter enable for promiscious mode */
+	mc_cnt = netdev_mc_count(ndev);
+	val = readl(priv->base + AVE_RXCR);
+	if (ndev->flags & IFF_PROMISC || !mc_cnt)
+		val &= ~AVE_RXCR_AFEN;
+	else
+		val |= AVE_RXCR_AFEN;
+	writel(val, priv->base + AVE_RXCR);
+
+	/* set all multicast address */
+	if ((ndev->flags & IFF_ALLMULTI) || mc_cnt > AVE_PF_MULTICAST_SIZE) {
+		ave_pfsel_set_macaddr(ndev, AVE_PFNUM_MULTICAST,
+				      v4multi_macadr, 1);
+		ave_pfsel_set_macaddr(ndev, AVE_PFNUM_MULTICAST + 1,
+				      v6multi_macadr, 1);
+	} else {
+		/* stop all multicast filter */
+		for (count = 0; count < AVE_PF_MULTICAST_SIZE; count++)
+			ave_pfsel_stop(ndev, AVE_PFNUM_MULTICAST + count);
+
+		/* set multicast addresses */
+		count = 0;
+		netdev_for_each_mc_addr(hw_adr, ndev) {
+			if (count == mc_cnt)
+				break;
+			ave_pfsel_set_macaddr(ndev, AVE_PFNUM_MULTICAST + count,
+					      hw_adr->addr, 6);
+			count++;
+		}
+	}
+}
+
+static void ave_get_stats64(struct net_device *ndev,
+			    struct rtnl_link_stats64 *stats)
+{
+	struct ave_private *priv = netdev_priv(ndev);
+	unsigned int start;
+
+	do {
+		start = u64_stats_fetch_begin_irq(&priv->stats_rx.syncp);
+		stats->rx_packets = priv->stats_rx.packets;
+		stats->rx_bytes	  = priv->stats_rx.bytes;
+	} while (u64_stats_fetch_retry_irq(&priv->stats_rx.syncp, start));
+
+	do {
+		start = u64_stats_fetch_begin_irq(&priv->stats_tx.syncp);
+		stats->tx_packets = priv->stats_tx.packets;
+		stats->tx_bytes	  = priv->stats_tx.bytes;
+	} while (u64_stats_fetch_retry_irq(&priv->stats_tx.syncp, start));
+
+	stats->rx_errors      = priv->stats_rx.errors;
+	stats->tx_errors      = priv->stats_tx.errors;
+	stats->rx_dropped     = priv->stats_rx.dropped;
+	stats->tx_dropped     = priv->stats_tx.dropped;
+	stats->rx_fifo_errors = priv->stats_rx.fifo_errors;
+	stats->collisions     = priv->stats_tx.collisions;
+}
+
+static int ave_set_mac_address(struct net_device *ndev, void *p)
+{
+	int ret = eth_mac_addr(ndev, p);
+
+	if (ret)
+		return ret;
+
+	ave_macaddr_init(ndev);
+
+	return 0;
+}
+
+static const struct net_device_ops ave_netdev_ops = {
+	.ndo_init		= ave_init,
+	.ndo_uninit		= ave_uninit,
+	.ndo_open		= ave_open,
+	.ndo_stop		= ave_stop,
+	.ndo_start_xmit		= ave_start_xmit,
+	.ndo_do_ioctl		= ave_ioctl,
+	.ndo_set_rx_mode	= ave_set_rx_mode,
+	.ndo_get_stats64	= ave_get_stats64,
+	.ndo_set_mac_address	= ave_set_mac_address,
+};
+
+static int ave_probe(struct platform_device *pdev)
+{
+	const struct ave_soc_data *data;
+	struct device *dev = &pdev->dev;
+	char buf[ETHTOOL_FWVERS_LEN];
+	phy_interface_t phy_mode;
+	struct ave_private *priv;
+	struct net_device *ndev;
+	struct device_node *np;
+	struct resource	*res;
+	const void *mac_addr;
+	void __iomem *base;
+	u64 dma_mask;
+	int irq, ret;
+	u32 ave_id;
+
+	data = of_device_get_match_data(dev);
+	if (WARN_ON(!data))
+		return -EINVAL;
+
+	np = dev->of_node;
+	phy_mode = of_get_phy_mode(np);
+	if (phy_mode < 0) {
+		dev_err(dev, "phy-mode not found\n");
+		return -EINVAL;
+	}
+	if ((!phy_interface_mode_is_rgmii(phy_mode)) &&
+	    phy_mode != PHY_INTERFACE_MODE_RMII &&
+	    phy_mode != PHY_INTERFACE_MODE_MII) {
+		dev_err(dev, "phy-mode is invalid\n");
+		return -EINVAL;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(dev, "IRQ not found\n");
+		return irq;
+	}
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	ndev = alloc_etherdev(sizeof(struct ave_private));
+	if (!ndev) {
+		dev_err(dev, "can't allocate ethernet device\n");
+		return -ENOMEM;
+	}
+
+	ndev->netdev_ops = &ave_netdev_ops;
+	ndev->ethtool_ops = &ave_ethtool_ops;
+	SET_NETDEV_DEV(ndev, dev);
+
+	ndev->features    |= (NETIF_F_IP_CSUM | NETIF_F_RXCSUM);
+	ndev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_RXCSUM);
+
+	ndev->max_mtu = AVE_MAX_ETHFRAME - (ETH_HLEN + ETH_FCS_LEN);
+
+	mac_addr = of_get_mac_address(np);
+	if (mac_addr)
+		ether_addr_copy(ndev->dev_addr, mac_addr);
+
+	/* if the mac address is invalid, use random mac address */
+	if (!is_valid_ether_addr(ndev->dev_addr)) {
+		eth_hw_addr_random(ndev);
+		dev_warn(dev, "Using random MAC address: %pM\n",
+			 ndev->dev_addr);
+	}
+
+	priv = netdev_priv(ndev);
+	priv->base = base;
+	priv->irq = irq;
+	priv->ndev = ndev;
+	priv->msg_enable = netif_msg_init(-1, AVE_DEFAULT_MSG_ENABLE);
+	priv->phy_mode = phy_mode;
+	priv->data = data;
+
+	if (IS_DESC_64BIT(priv)) {
+		priv->desc_size = AVE_DESC_SIZE_64;
+		priv->tx.daddr  = AVE_TXDM_64;
+		priv->rx.daddr  = AVE_RXDM_64;
+		dma_mask = DMA_BIT_MASK(64);
+	} else {
+		priv->desc_size = AVE_DESC_SIZE_32;
+		priv->tx.daddr  = AVE_TXDM_32;
+		priv->rx.daddr  = AVE_RXDM_32;
+		dma_mask = DMA_BIT_MASK(32);
+	}
+	ret = dma_set_mask(dev, dma_mask);
+	if (ret)
+		goto out_free_netdev;
+
+	priv->tx.ndesc = AVE_NR_TXDESC;
+	priv->rx.ndesc = AVE_NR_RXDESC;
+
+	u64_stats_init(&priv->stats_tx.syncp);
+	u64_stats_init(&priv->stats_rx.syncp);
+
+	priv->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		ret = PTR_ERR(priv->clk);
+		goto out_free_netdev;
+	}
+
+	priv->rst = devm_reset_control_get_optional_shared(dev, NULL);
+	if (IS_ERR(priv->rst)) {
+		ret = PTR_ERR(priv->rst);
+		goto out_free_netdev;
+	}
+
+	priv->mdio = devm_mdiobus_alloc(dev);
+	if (!priv->mdio) {
+		ret = -ENOMEM;
+		goto out_free_netdev;
+	}
+	priv->mdio->priv = ndev;
+	priv->mdio->parent = dev;
+	priv->mdio->read = ave_mdiobus_read;
+	priv->mdio->write = ave_mdiobus_write;
+	priv->mdio->name = "uniphier-mdio";
+	snprintf(priv->mdio->id, MII_BUS_ID_SIZE, "%s-%x",
+		 pdev->name, pdev->id);
+
+	/* Register as a NAPI supported driver */
+	netif_napi_add(ndev, &priv->napi_rx, ave_napi_poll_rx, priv->rx.ndesc);
+	netif_tx_napi_add(ndev, &priv->napi_tx, ave_napi_poll_tx,
+			  priv->tx.ndesc);
+
+	platform_set_drvdata(pdev, ndev);
+
+	ret = register_netdev(ndev);
+	if (ret) {
+		dev_err(dev, "failed to register netdevice\n");
+		goto out_del_napi;
+	}
+
+	/* get ID and version */
+	ave_id = readl(priv->base + AVE_IDR);
+	ave_hw_read_version(ndev, buf, sizeof(buf));
+
+	dev_info(dev, "Socionext %c%c%c%c Ethernet IP %s (irq=%d, phy=%s)\n",
+		 (ave_id >> 24) & 0xff, (ave_id >> 16) & 0xff,
+		 (ave_id >> 8) & 0xff, (ave_id >> 0) & 0xff,
+		 buf, priv->irq, phy_modes(phy_mode));
+
+	return 0;
+
+out_del_napi:
+	netif_napi_del(&priv->napi_rx);
+	netif_napi_del(&priv->napi_tx);
+out_free_netdev:
+	free_netdev(ndev);
+
+	return ret;
+}
+
+static int ave_remove(struct platform_device *pdev)
+{
+	struct net_device *ndev = platform_get_drvdata(pdev);
+	struct ave_private *priv = netdev_priv(ndev);
+
+	unregister_netdev(ndev);
+	netif_napi_del(&priv->napi_rx);
+	netif_napi_del(&priv->napi_tx);
+	free_netdev(ndev);
+
+	return 0;
+}
+
+static const struct ave_soc_data ave_pro4_data = {
+	.is_desc_64bit = false,
+};
+
+static const struct ave_soc_data ave_pxs2_data = {
+	.is_desc_64bit = false,
+};
+
+static const struct ave_soc_data ave_ld11_data = {
+	.is_desc_64bit = false,
+};
+
+static const struct ave_soc_data ave_ld20_data = {
+	.is_desc_64bit = true,
+};
+
+static const struct of_device_id of_ave_match[] = {
+	{
+		.compatible = "socionext,uniphier-pro4-ave4",
+		.data = &ave_pro4_data,
+	},
+	{
+		.compatible = "socionext,uniphier-pxs2-ave4",
+		.data = &ave_pxs2_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld11-ave4",
+		.data = &ave_ld11_data,
+	},
+	{
+		.compatible = "socionext,uniphier-ld20-ave4",
+		.data = &ave_ld20_data,
+	},
+	{ /* Sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, of_ave_match);
+
+static struct platform_driver ave_driver = {
+	.probe  = ave_probe,
+	.remove = ave_remove,
+	.driver	= {
+		.name = "ave",
+		.of_match_table	= of_ave_match,
+	},
+};
+module_platform_driver(ave_driver);
+
+MODULE_DESCRIPTION("Socionext UniPhier AVE ethernet driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h
index ce2ea2d491ac..2ffe76c0ff74 100644
--- a/drivers/net/ethernet/stmicro/stmmac/common.h
+++ b/drivers/net/ethernet/stmicro/stmmac/common.h
@@ -474,7 +474,7 @@ struct mac_device_info;
 /* Helpers to program the MAC core */
 struct stmmac_ops {
 	/* MAC core initialization */
-	void (*core_init)(struct mac_device_info *hw, int mtu);
+	void (*core_init)(struct mac_device_info *hw, struct net_device *dev);
 	/* Enable the MAC RX/TX */
 	void (*set_mac)(void __iomem *ioaddr, bool enable);
 	/* Enable and verify that the IPC module is supported */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
index 4404650b32c5..5270d26f0bc6 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c
@@ -40,9 +40,7 @@
 #define PRG_ETH0_CLK_M250_DIV_SHIFT	7
 #define PRG_ETH0_CLK_M250_DIV_WIDTH	3
 
-/* divides the result of m25_sel by either 5 (bit unset) or 10 (bit set) */
-#define PRG_ETH0_CLK_M25_DIV_SHIFT	10
-#define PRG_ETH0_CLK_M25_DIV_WIDTH	1
+#define PRG_ETH0_RGMII_TX_CLK_EN	10
 
 #define PRG_ETH0_INVERTED_RMII_CLK	BIT(11)
 #define PRG_ETH0_TX_AND_PHY_REF_CLK	BIT(12)
@@ -63,8 +61,11 @@ struct meson8b_dwmac {
 	struct clk_divider	m250_div;
 	struct clk		*m250_div_clk;
 
-	struct clk_divider	m25_div;
-	struct clk		*m25_div_clk;
+	struct clk_fixed_factor	fixed_div2;
+	struct clk		*fixed_div2_clk;
+
+	struct clk_gate		rgmii_tx_en;
+	struct clk		*rgmii_tx_en_clk;
 
 	u32			tx_delay_ns;
 };
@@ -81,7 +82,7 @@ static void meson8b_dwmac_mask_bits(struct meson8b_dwmac *dwmac, u32 reg,
 	writel(data, dwmac->regs + reg);
 }
 
-static int meson8b_init_clk(struct meson8b_dwmac *dwmac)
+static int meson8b_init_rgmii_tx_clk(struct meson8b_dwmac *dwmac)
 {
 	struct clk_init_data init;
 	int i, ret;
@@ -89,11 +90,6 @@ static int meson8b_init_clk(struct meson8b_dwmac *dwmac)
 	char clk_name[32];
 	const char *clk_div_parents[1];
 	const char *mux_parent_names[MUX_CLK_NUM_PARENTS];
-	static const struct clk_div_table clk_25m_div_table[] = {
-		{ .val = 0, .div = 5 },
-		{ .val = 1, .div = 10 },
-		{ /* sentinel */ },
-	};
 
 	/* get the mux parents from DT */
 	for (i = 0; i < MUX_CLK_NUM_PARENTS; i++) {
@@ -116,7 +112,7 @@ static int meson8b_init_clk(struct meson8b_dwmac *dwmac)
 	snprintf(clk_name, sizeof(clk_name), "%s#m250_sel", dev_name(dev));
 	init.name = clk_name;
 	init.ops = &clk_mux_ops;
-	init.flags = 0;
+	init.flags = CLK_SET_RATE_PARENT;
 	init.parent_names = mux_parent_names;
 	init.num_parents = MUX_CLK_NUM_PARENTS;
 
@@ -144,31 +140,48 @@ static int meson8b_init_clk(struct meson8b_dwmac *dwmac)
 	dwmac->m250_div.shift = PRG_ETH0_CLK_M250_DIV_SHIFT;
 	dwmac->m250_div.width = PRG_ETH0_CLK_M250_DIV_WIDTH;
 	dwmac->m250_div.hw.init = &init;
-	dwmac->m250_div.flags = CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO;
+	dwmac->m250_div.flags = CLK_DIVIDER_ONE_BASED |
+				CLK_DIVIDER_ALLOW_ZERO |
+				CLK_DIVIDER_ROUND_CLOSEST;
 
 	dwmac->m250_div_clk = devm_clk_register(dev, &dwmac->m250_div.hw);
 	if (WARN_ON(IS_ERR(dwmac->m250_div_clk)))
 		return PTR_ERR(dwmac->m250_div_clk);
 
-	/* create the m25_div */
-	snprintf(clk_name, sizeof(clk_name), "%s#m25_div", dev_name(dev));
+	/* create the fixed_div2 */
+	snprintf(clk_name, sizeof(clk_name), "%s#fixed_div2", dev_name(dev));
 	init.name = devm_kstrdup(dev, clk_name, GFP_KERNEL);
-	init.ops = &clk_divider_ops;
-	init.flags = CLK_IS_BASIC | CLK_SET_RATE_PARENT;
+	init.ops = &clk_fixed_factor_ops;
+	init.flags = CLK_SET_RATE_PARENT;
 	clk_div_parents[0] = __clk_get_name(dwmac->m250_div_clk);
 	init.parent_names = clk_div_parents;
 	init.num_parents = ARRAY_SIZE(clk_div_parents);
 
-	dwmac->m25_div.reg = dwmac->regs + PRG_ETH0;
-	dwmac->m25_div.shift = PRG_ETH0_CLK_M25_DIV_SHIFT;
-	dwmac->m25_div.width = PRG_ETH0_CLK_M25_DIV_WIDTH;
-	dwmac->m25_div.table = clk_25m_div_table;
-	dwmac->m25_div.hw.init = &init;
-	dwmac->m25_div.flags = CLK_DIVIDER_ALLOW_ZERO;
+	dwmac->fixed_div2.mult = 1;
+	dwmac->fixed_div2.div = 2;
+	dwmac->fixed_div2.hw.init = &init;
 
-	dwmac->m25_div_clk = devm_clk_register(dev, &dwmac->m25_div.hw);
-	if (WARN_ON(IS_ERR(dwmac->m25_div_clk)))
-		return PTR_ERR(dwmac->m25_div_clk);
+	dwmac->fixed_div2_clk = devm_clk_register(dev, &dwmac->fixed_div2.hw);
+	if (WARN_ON(IS_ERR(dwmac->fixed_div2_clk)))
+		return PTR_ERR(dwmac->fixed_div2_clk);
+
+	/* create the rgmii_tx_en */
+	init.name = devm_kasprintf(dev, GFP_KERNEL, "%s#rgmii_tx_en",
+				   dev_name(dev));
+	init.ops = &clk_gate_ops;
+	init.flags = CLK_SET_RATE_PARENT;
+	clk_div_parents[0] = __clk_get_name(dwmac->fixed_div2_clk);
+	init.parent_names = clk_div_parents;
+	init.num_parents = ARRAY_SIZE(clk_div_parents);
+
+	dwmac->rgmii_tx_en.reg = dwmac->regs + PRG_ETH0;
+	dwmac->rgmii_tx_en.bit_idx = PRG_ETH0_RGMII_TX_CLK_EN;
+	dwmac->rgmii_tx_en.hw.init = &init;
+
+	dwmac->rgmii_tx_en_clk = devm_clk_register(dev,
+						   &dwmac->rgmii_tx_en.hw);
+	if (WARN_ON(IS_ERR(dwmac->rgmii_tx_en_clk)))
+		return PTR_ERR(dwmac->rgmii_tx_en_clk);
 
 	return 0;
 }
@@ -176,7 +189,6 @@ static int meson8b_init_clk(struct meson8b_dwmac *dwmac)
 static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
 {
 	int ret;
-	unsigned long clk_rate;
 	u8 tx_dly_val = 0;
 
 	switch (dwmac->phy_mode) {
@@ -191,9 +203,6 @@ static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
 
 	case PHY_INTERFACE_MODE_RGMII_ID:
 	case PHY_INTERFACE_MODE_RGMII_TXID:
-		/* Generate a 25MHz clock for the PHY */
-		clk_rate = 25 * 1000 * 1000;
-
 		/* enable RGMII mode */
 		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_RGMII_MODE,
 					PRG_ETH0_RGMII_MODE);
@@ -204,12 +213,28 @@ static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
 
 		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TXDLY_MASK,
 					tx_dly_val << PRG_ETH0_TXDLY_SHIFT);
+
+		/* Configure the 125MHz RGMII TX clock, the IP block changes
+		 * the output automatically (= without us having to configure
+		 * a register) based on the line-speed (125MHz for Gbit speeds,
+		 * 25MHz for 100Mbit/s and 2.5MHz for 10Mbit/s).
+		 */
+		ret = clk_set_rate(dwmac->rgmii_tx_en_clk, 125 * 1000 * 1000);
+		if (ret) {
+			dev_err(&dwmac->pdev->dev,
+				"failed to set RGMII TX clock\n");
+			return ret;
+		}
+
+		ret = clk_prepare_enable(dwmac->rgmii_tx_en_clk);
+		if (ret) {
+			dev_err(&dwmac->pdev->dev,
+				"failed to enable the RGMII TX clock\n");
+			return ret;
+		}
 		break;
 
 	case PHY_INTERFACE_MODE_RMII:
-		/* Use the rate of the mux clock for the internal RMII PHY */
-		clk_rate = clk_get_rate(dwmac->m250_mux_clk);
-
 		/* disable RGMII mode -> enables RMII mode */
 		meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_RGMII_MODE,
 					0);
@@ -231,20 +256,6 @@ static int meson8b_init_prg_eth(struct meson8b_dwmac *dwmac)
 		return -EINVAL;
 	}
 
-	ret = clk_prepare_enable(dwmac->m25_div_clk);
-	if (ret) {
-		dev_err(&dwmac->pdev->dev, "failed to enable the PHY clock\n");
-		return ret;
-	}
-
-	ret = clk_set_rate(dwmac->m25_div_clk, clk_rate);
-	if (ret) {
-		clk_disable_unprepare(dwmac->m25_div_clk);
-
-		dev_err(&dwmac->pdev->dev, "failed to set PHY clock\n");
-		return ret;
-	}
-
 	/* enable TX_CLK and PHY_REF_CLK generator */
 	meson8b_dwmac_mask_bits(dwmac, PRG_ETH0, PRG_ETH0_TX_AND_PHY_REF_CLK,
 				PRG_ETH0_TX_AND_PHY_REF_CLK);
@@ -294,7 +305,7 @@ static int meson8b_dwmac_probe(struct platform_device *pdev)
 				 &dwmac->tx_delay_ns))
 		dwmac->tx_delay_ns = 2;
 
-	ret = meson8b_init_clk(dwmac);
+	ret = meson8b_init_rgmii_tx_clk(dwmac);
 	if (ret)
 		goto err_remove_config_dt;
 
@@ -311,7 +322,8 @@ static int meson8b_dwmac_probe(struct platform_device *pdev)
 	return 0;
 
 err_clk_disable:
-	clk_disable_unprepare(dwmac->m25_div_clk);
+	if (phy_interface_mode_is_rgmii(dwmac->phy_mode))
+		clk_disable_unprepare(dwmac->rgmii_tx_en_clk);
 err_remove_config_dt:
 	stmmac_remove_config_dt(pdev, plat_dat);
 
@@ -322,7 +334,8 @@ static int meson8b_dwmac_remove(struct platform_device *pdev)
 {
 	struct meson8b_dwmac *dwmac = get_stmmac_bsp_priv(&pdev->dev);
 
-	clk_disable_unprepare(dwmac->m25_div_clk);
+	if (phy_interface_mode_is_rgmii(dwmac->phy_mode))
+		clk_disable_unprepare(dwmac->rgmii_tx_en_clk);
 
 	return stmmac_pltfr_remove(pdev);
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
index 9eb7f65d8000..a3fa65b1ca8e 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c
@@ -483,7 +483,8 @@ static int sun8i_dwmac_init(struct platform_device *pdev, void *priv)
 	return 0;
 }
 
-static void sun8i_dwmac_core_init(struct mac_device_info *hw, int mtu)
+static void sun8i_dwmac_core_init(struct mac_device_info *hw,
+				  struct net_device *dev)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	u32 v;
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
index 8a86340ff2d3..ef10baf14186 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
@@ -25,18 +25,28 @@
 #include <linux/crc32.h>
 #include <linux/slab.h>
 #include <linux/ethtool.h>
+#include <net/dsa.h>
 #include <asm/io.h>
 #include "stmmac_pcs.h"
 #include "dwmac1000.h"
 
-static void dwmac1000_core_init(struct mac_device_info *hw, int mtu)
+static void dwmac1000_core_init(struct mac_device_info *hw,
+				struct net_device *dev)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	u32 value = readl(ioaddr + GMAC_CONTROL);
+	int mtu = dev->mtu;
 
 	/* Configure GMAC core */
 	value |= GMAC_CORE_INIT;
 
+	/* Clear ACS bit because Ethernet switch tagging formats such as
+	 * Broadcom tags can look like invalid LLC/SNAP packets and cause the
+	 * hardware to truncate packets on reception.
+	 */
+	if (netdev_uses_dsa(dev))
+		value &= ~GMAC_CONTROL_ACS;
+
 	if (mtu > 1500)
 		value |= GMAC_CONTROL_2K;
 	if (mtu > 2000)
@@ -64,8 +74,6 @@ static void dwmac1000_core_init(struct mac_device_info *hw, int mtu)
 	/* Mask GMAC interrupts */
 	value = GMAC_INT_DEFAULT_MASK;
 
-	if (hw->pmt)
-		value &= ~GMAC_INT_DISABLE_PMT;
 	if (hw->pcs)
 		value &= ~GMAC_INT_DISABLE_PCS;
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
index 8ef517356313..91b23f9db31a 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
@@ -25,15 +25,26 @@
 *******************************************************************************/
 
 #include <linux/crc32.h>
+#include <net/dsa.h>
 #include <asm/io.h>
 #include "dwmac100.h"
 
-static void dwmac100_core_init(struct mac_device_info *hw, int mtu)
+static void dwmac100_core_init(struct mac_device_info *hw,
+			       struct net_device *dev)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	u32 value = readl(ioaddr + MAC_CONTROL);
 
-	writel((value | MAC_CORE_INIT), ioaddr + MAC_CONTROL);
+	value |= MAC_CORE_INIT;
+
+	/* Clear ASTP bit because Ethernet switch tagging formats such as
+	 * Broadcom tags can look like invalid LLC/SNAP packets and cause the
+	 * hardware to truncate packets on reception.
+	 */
+	if (netdev_uses_dsa(dev))
+		value &= ~MAC_CONTROL_ASTP;
+
+	writel(value, ioaddr + MAC_CONTROL);
 
 #ifdef STMMAC_VLAN_TAG_USED
 	writel(ETH_P_8021Q, ioaddr + MAC_VLAN1);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
index 789dad8a07b5..7761a26ec9c5 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h
@@ -98,7 +98,7 @@
 #define	GMAC_PCS_IRQ_DEFAULT	(GMAC_INT_RGSMIIS | GMAC_INT_PCS_LINK |	\
 				 GMAC_INT_PCS_ANE)
 
-#define	GMAC_INT_DEFAULT_MASK	(GMAC_INT_PMT_EN | GMAC_INT_LPI_EN)
+#define	GMAC_INT_DEFAULT_ENABLE	(GMAC_INT_PMT_EN | GMAC_INT_LPI_EN)
 
 enum dwmac4_irq_status {
 	time_stamp_irq = 0x00001000,
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index f3ed8f7853eb..63795ecafc8d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -17,16 +17,26 @@
 #include <linux/slab.h>
 #include <linux/ethtool.h>
 #include <linux/io.h>
+#include <net/dsa.h>
 #include "stmmac_pcs.h"
 #include "dwmac4.h"
 
-static void dwmac4_core_init(struct mac_device_info *hw, int mtu)
+static void dwmac4_core_init(struct mac_device_info *hw,
+			     struct net_device *dev)
 {
 	void __iomem *ioaddr = hw->pcsr;
 	u32 value = readl(ioaddr + GMAC_CONFIG);
+	int mtu = dev->mtu;
 
 	value |= GMAC_CORE_INIT;
 
+	/* Clear ACS bit because Ethernet switch tagging formats such as
+	 * Broadcom tags can look like invalid LLC/SNAP packets and cause the
+	 * hardware to truncate packets on reception.
+	 */
+	if (netdev_uses_dsa(dev))
+		value &= ~GMAC_CONFIG_ACS;
+
 	if (mtu > 1500)
 		value |= GMAC_CONFIG_2K;
 	if (mtu > 2000)
@@ -51,10 +61,9 @@ static void dwmac4_core_init(struct mac_device_info *hw, int mtu)
 
 	writel(value, ioaddr + GMAC_CONFIG);
 
-	/* Mask GMAC interrupts */
-	value = GMAC_INT_DEFAULT_MASK;
-	if (hw->pmt)
-		value |= GMAC_INT_PMT_EN;
+	/* Enable GMAC interrupts */
+	value = GMAC_INT_DEFAULT_ENABLE;
+
 	if (hw->pcs)
 		value |= GMAC_PCS_IRQ_DEFAULT;
 
@@ -562,10 +571,12 @@ static int dwmac4_irq_status(struct mac_device_info *hw,
 			     struct stmmac_extra_stats *x)
 {
 	void __iomem *ioaddr = hw->pcsr;
-	u32 intr_status;
+	u32 intr_status = readl(ioaddr + GMAC_INT_STATUS);
+	u32 intr_enable = readl(ioaddr + GMAC_INT_EN);
 	int ret = 0;
 
-	intr_status = readl(ioaddr + GMAC_INT_STATUS);
+	/* Discard disabled bits */
+	intr_status &= intr_enable;
 
 	/* Not used events (e.g. MMC interrupts) are not handled. */
 	if ((intr_status & mmc_tx_irq))
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
index 7e089bf906b4..c728ffa095de 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c
@@ -334,7 +334,7 @@ static void dwmac4_rd_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
 	if (tx_own)
 		tdes3 |= TDES3_OWN;
 
-	if (is_fs & tx_own)
+	if (is_fs && tx_own)
 		/* When the own bit, for the first frame, has to be set, all
 		 * descriptors for the same frame has to be set before, to
 		 * avoid race condition.
@@ -377,7 +377,7 @@ static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs,
 	if (tx_own)
 		tdes3 |= TDES3_OWN;
 
-	if (is_fs & tx_own)
+	if (is_fs && tx_own)
 		/* When the own bit, for the first frame, has to be set, all
 		 * descriptors for the same frame has to be set before, to
 		 * avoid race condition.
@@ -406,7 +406,7 @@ static void dwmac4_display_ring(void *head, unsigned int size, bool rx)
 	pr_info("%s descriptor ring:\n", rx ? "RX" : "TX");
 
 	for (i = 0; i < size; i++) {
-		pr_info("%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
+		pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
 			i, (unsigned int)virt_to_phys(p),
 			le32_to_cpu(p->des0), le32_to_cpu(p->des1),
 			le32_to_cpu(p->des2), le32_to_cpu(p->des3));
diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
index 2a828a312814..6768a25b6aa0 100644
--- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c
@@ -341,7 +341,7 @@ static void enh_desc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len,
 	if (tx_own)
 		tdes0 |= ETDES0_OWN;
 
-	if (is_fs & tx_own)
+	if (is_fs && tx_own)
 		/* When the own bit, for the first frame, has to be set, all
 		 * descriptors for the same frame has to be set before, to
 		 * avoid race condition.
@@ -428,7 +428,7 @@ static void enh_desc_display_ring(void *head, unsigned int size, bool rx)
 		u64 x;
 
 		x = *(u64 *)ep;
-		pr_info("%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
+		pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
 			i, (unsigned int)virt_to_phys(ep),
 			(unsigned int)x, (unsigned int)(x >> 32),
 			ep->basic.des2, ep->basic.des3);
diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
index db4cee57bb24..ebd9e5e00f16 100644
--- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
+++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c
@@ -288,7 +288,7 @@ static void ndesc_display_ring(void *head, unsigned int size, bool rx)
 		u64 x;
 
 		x = *(u64 *)p;
-		pr_info("%d [0x%x]: 0x%x 0x%x 0x%x 0x%x",
+		pr_info("%03d [0x%x]: 0x%x 0x%x 0x%x 0x%x",
 			i, (unsigned int)virt_to_phys(p),
 			(unsigned int)x, (unsigned int)(x >> 32),
 			p->des2, p->des3);
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index c0af0bc4e714..7ad841434ec8 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -2003,22 +2003,60 @@ static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
 {
 	u32 tx_channel_count = priv->plat->tx_queues_to_use;
-	int status;
+	u32 rx_channel_count = priv->plat->rx_queues_to_use;
+	u32 channels_to_check = tx_channel_count > rx_channel_count ?
+				tx_channel_count : rx_channel_count;
 	u32 chan;
+	bool poll_scheduled = false;
+	int status[channels_to_check];
+
+	/* Each DMA channel can be used for rx and tx simultaneously, yet
+	 * napi_struct is embedded in struct stmmac_rx_queue rather than in a
+	 * stmmac_channel struct.
+	 * Because of this, stmmac_poll currently checks (and possibly wakes)
+	 * all tx queues rather than just a single tx queue.
+	 */
+	for (chan = 0; chan < channels_to_check; chan++)
+		status[chan] = priv->hw->dma->dma_interrupt(priv->ioaddr,
+							    &priv->xstats,
+							    chan);
 
-	for (chan = 0; chan < tx_channel_count; chan++) {
-		struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
+	for (chan = 0; chan < rx_channel_count; chan++) {
+		if (likely(status[chan] & handle_rx)) {
+			struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
 
-		status = priv->hw->dma->dma_interrupt(priv->ioaddr,
-						      &priv->xstats, chan);
-		if (likely((status & handle_rx)) || (status & handle_tx)) {
 			if (likely(napi_schedule_prep(&rx_q->napi))) {
 				stmmac_disable_dma_irq(priv, chan);
 				__napi_schedule(&rx_q->napi);
+				poll_scheduled = true;
+			}
+		}
+	}
+
+	/* If we scheduled poll, we already know that tx queues will be checked.
+	 * If we didn't schedule poll, see if any DMA channel (used by tx) has a
+	 * completed transmission, if so, call stmmac_poll (once).
+	 */
+	if (!poll_scheduled) {
+		for (chan = 0; chan < tx_channel_count; chan++) {
+			if (status[chan] & handle_tx) {
+				/* It doesn't matter what rx queue we choose
+				 * here. We use 0 since it always exists.
+				 */
+				struct stmmac_rx_queue *rx_q =
+					&priv->rx_queue[0];
+
+				if (likely(napi_schedule_prep(&rx_q->napi))) {
+					stmmac_disable_dma_irq(priv, chan);
+					__napi_schedule(&rx_q->napi);
+				}
+				break;
 			}
 		}
+	}
 
-		if (unlikely(status & tx_hard_error_bump_tc)) {
+	for (chan = 0; chan < tx_channel_count; chan++) {
+		if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
 			/* Try to bump up the dma threshold on this failure */
 			if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
 			    (tc <= 256)) {
@@ -2035,7 +2073,7 @@ static void stmmac_dma_interrupt(struct stmmac_priv *priv)
 								    chan);
 				priv->xstats.threshold = tc;
 			}
-		} else if (unlikely(status == tx_hard_error)) {
+		} else if (unlikely(status[chan] == tx_hard_error)) {
 			stmmac_tx_err(priv, chan);
 		}
 	}
@@ -2489,7 +2527,7 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
 	}
 
 	/* Initialize the MAC Core */
-	priv->hw->mac->core_init(priv->hw, dev->mtu);
+	priv->hw->mac->core_init(priv->hw, dev);
 
 	/* Initialize MTL*/
 	if (priv->synopsys_id >= DWMAC_CORE_4_00)
@@ -3404,9 +3442,8 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
 			if (netif_msg_rx_status(priv)) {
 				netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
 					   p, entry, des);
-				if (frame_len > ETH_FRAME_LEN)
-					netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
-						   frame_len, status);
+				netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
+					   frame_len, status);
 			}
 
 			/* The zero-copy is always used for all the sizes
diff --git a/drivers/net/ethernet/sun/Kconfig b/drivers/net/ethernet/sun/Kconfig
index b2caf5132bd2..7b982e02ea3a 100644
--- a/drivers/net/ethernet/sun/Kconfig
+++ b/drivers/net/ethernet/sun/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Sun network device configuration
 #
diff --git a/drivers/net/ethernet/sun/cassini.c b/drivers/net/ethernet/sun/cassini.c
index 113bd57e2ea0..9020b084b953 100644
--- a/drivers/net/ethernet/sun/cassini.c
+++ b/drivers/net/ethernet/sun/cassini.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* cassini.c: Sun Microsystems Cassini(+) ethernet driver.
  *
  * Copyright (C) 2004 Sun Microsystems Inc.
diff --git a/drivers/net/ethernet/sun/cassini.h b/drivers/net/ethernet/sun/cassini.h
index 882ce168a799..13f3860496a8 100644
--- a/drivers/net/ethernet/sun/cassini.h
+++ b/drivers/net/ethernet/sun/cassini.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /* $Id: cassini.h,v 1.16 2004/08/17 21:15:16 zaumen Exp $
  * cassini.h: Definitions for Sun Microsystems Cassini(+) ethernet driver.
  *
diff --git a/drivers/net/ethernet/sun/ldmvsw.c b/drivers/net/ethernet/sun/ldmvsw.c
index 5ea037672e6f..a5dd627fe2f9 100644
--- a/drivers/net/ethernet/sun/ldmvsw.c
+++ b/drivers/net/ethernet/sun/ldmvsw.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* ldmvsw.c: Sun4v LDOM Virtual Switch Driver.
  *
  * Copyright (C) 2016-2017 Oracle. All rights reserved.
diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c
index 06001bacbe0f..8dd545fed30d 100644
--- a/drivers/net/ethernet/sun/niu.c
+++ b/drivers/net/ethernet/sun/niu.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* niu.c: Neptune ethernet driver.
  *
  * Copyright (C) 2007, 2008 David S. Miller (davem@davemloft.net)
diff --git a/drivers/net/ethernet/sun/sunbmac.c b/drivers/net/ethernet/sun/sunbmac.c
index 0b1f41f6bceb..f047b2797156 100644
--- a/drivers/net/ethernet/sun/sunbmac.c
+++ b/drivers/net/ethernet/sun/sunbmac.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* sunbmac.c: Driver for Sparc BigMAC 100baseT ethernet adapters.
  *
  * Copyright (C) 1997, 1998, 1999, 2003, 2008 David S. Miller (davem@davemloft.net)
diff --git a/drivers/net/ethernet/sun/sungem.c b/drivers/net/ethernet/sun/sungem.c
index a7afcee3c5ae..7a16d40a72d1 100644
--- a/drivers/net/ethernet/sun/sungem.c
+++ b/drivers/net/ethernet/sun/sungem.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* $Id: sungem.c,v 1.44.2.22 2002/03/13 01:18:12 davem Exp $
  * sungem.c: Sun GEM ethernet driver.
  *
diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c
index 0431f1e5f511..06da2f59fcbf 100644
--- a/drivers/net/ethernet/sun/sunhme.c
+++ b/drivers/net/ethernet/sun/sunhme.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* sunhme.c: Sparc HME/BigMac 10/100baseT half/full duplex auto switching,
  *           auto carrier detecting ethernet driver.  Also known as the
  *           "Happy Meal Ethernet" found on SunSwift SBUS cards.
diff --git a/drivers/net/ethernet/sun/sunqe.c b/drivers/net/ethernet/sun/sunqe.c
index a6bcdcdd947e..7fe0d5e33922 100644
--- a/drivers/net/ethernet/sun/sunqe.c
+++ b/drivers/net/ethernet/sun/sunqe.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* sunqe.c: Sparc QuadEthernet 10baseT SBUS card driver.
  *          Once again I am out to prove that every ethernet
  *          controller out there can be most efficiently programmed
diff --git a/drivers/net/ethernet/sun/sunvnet.c b/drivers/net/ethernet/sun/sunvnet.c
index 27fb22638885..a94f50442613 100644
--- a/drivers/net/ethernet/sun/sunvnet.c
+++ b/drivers/net/ethernet/sun/sunvnet.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* sunvnet.c: Sun LDOM Virtual Network Driver.
  *
  * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
@@ -311,7 +312,7 @@ static struct vnet *vnet_new(const u64 *local_mac,
 	dev->ethtool_ops = &vnet_ethtool_ops;
 	dev->watchdog_timeo = VNET_TX_TIMEOUT;
 
-	dev->hw_features = NETIF_F_TSO | NETIF_F_GSO | NETIF_F_GSO_SOFTWARE |
+	dev->hw_features = NETIF_F_TSO | NETIF_F_GSO | NETIF_F_ALL_TSO |
 			   NETIF_F_HW_CSUM | NETIF_F_SG;
 	dev->features = dev->hw_features;
 
diff --git a/drivers/net/ethernet/sun/sunvnet_common.c b/drivers/net/ethernet/sun/sunvnet_common.c
index 8aa3ce46bb81..d8f4c3f28150 100644
--- a/drivers/net/ethernet/sun/sunvnet_common.c
+++ b/drivers/net/ethernet/sun/sunvnet_common.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* sunvnet.c: Sun LDOM Virtual Network Driver.
  *
  * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c
index d655a4261e98..eb1c6b03c329 100644
--- a/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c
+++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c
@@ -333,9 +333,8 @@ void xlgmac_print_pkt(struct net_device *netdev,
 		      struct sk_buff *skb, bool tx_rx)
 {
 	struct ethhdr *eth = (struct ethhdr *)skb->data;
-	unsigned char *buf = skb->data;
 	unsigned char buffer[128];
-	unsigned int i, j;
+	unsigned int i;
 
 	netdev_dbg(netdev, "\n************** SKB dump ****************\n");
 
@@ -346,22 +345,13 @@ void xlgmac_print_pkt(struct net_device *netdev,
 	netdev_dbg(netdev, "Src MAC addr: %pM\n", eth->h_source);
 	netdev_dbg(netdev, "Protocol: %#06hx\n", ntohs(eth->h_proto));
 
-	for (i = 0, j = 0; i < skb->len;) {
-		j += snprintf(buffer + j, sizeof(buffer) - j, "%02hhx",
-			      buf[i++]);
-
-		if ((i % 32) == 0) {
-			netdev_dbg(netdev, "  %#06x: %s\n", i - 32, buffer);
-			j = 0;
-		} else if ((i % 16) == 0) {
-			buffer[j++] = ' ';
-			buffer[j++] = ' ';
-		} else if ((i % 4) == 0) {
-			buffer[j++] = ' ';
-		}
+	for (i = 0; i < skb->len; i += 32) {
+		unsigned int len = min(skb->len - i, 32U);
+
+		hex_dump_to_buffer(&skb->data[i], len, 32, 1,
+				   buffer, sizeof(buffer), false);
+		netdev_dbg(netdev, "  %#06x: %s\n", i, buffer);
 	}
-	if (i % 32)
-		netdev_dbg(netdev, "  %#06x: %s\n", i - (i % 32), buffer);
 
 	netdev_dbg(netdev, "\n************** SKB dump ****************\n");
 }
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index a73600dceb8b..b2b30c9df037 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -88,6 +88,7 @@ do {								\
 #define CPSW_VERSION_4		0x190112
 
 #define HOST_PORT_NUM		0
+#define CPSW_ALE_PORTS_NUM	3
 #define SLIVER_SIZE		0x40
 
 #define CPSW1_HOST_PORT_OFFSET	0x028
@@ -352,6 +353,27 @@ struct cpsw_hw_stats {
 	u32	rxdmaoverruns;
 };
 
+struct cpsw_slave_data {
+	struct device_node *phy_node;
+	char		phy_id[MII_BUS_ID_SIZE];
+	int		phy_if;
+	u8		mac_addr[ETH_ALEN];
+	u16		dual_emac_res_vlan;	/* Reserved VLAN for DualEMAC */
+};
+
+struct cpsw_platform_data {
+	struct cpsw_slave_data	*slave_data;
+	u32	ss_reg_ofs;	/* Subsystem control register offset */
+	u32	channels;	/* number of cpdma channels (symmetric) */
+	u32	slaves;		/* number of slave cpgmac ports */
+	u32	active_slave; /* time stamping, ethtool and SIOCGMIIPHY slave */
+	u32	ale_entries;	/* ale table size */
+	u32	bd_ram_size;  /*buffer descriptor ram size */
+	u32	mac_control;	/* Mac control register */
+	u16	default_vlan;	/* Def VLAN for ALE lookup in VLAN aware mode*/
+	bool	dual_emac;	/* Enable Dual EMAC mode */
+};
+
 struct cpsw_slave {
 	void __iomem			*regs;
 	struct cpsw_sliver_regs __iomem	*sliver;
@@ -365,12 +387,12 @@ struct cpsw_slave {
 
 static inline u32 slave_read(struct cpsw_slave *slave, u32 offset)
 {
-	return __raw_readl(slave->regs + offset);
+	return readl_relaxed(slave->regs + offset);
 }
 
 static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
 {
-	__raw_writel(val, slave->regs + offset);
+	writel_relaxed(val, slave->regs + offset);
 }
 
 struct cpsw_vector {
@@ -660,8 +682,8 @@ static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
 
 static void cpsw_intr_enable(struct cpsw_common *cpsw)
 {
-	__raw_writel(0xFF, &cpsw->wr_regs->tx_en);
-	__raw_writel(0xFF, &cpsw->wr_regs->rx_en);
+	writel_relaxed(0xFF, &cpsw->wr_regs->tx_en);
+	writel_relaxed(0xFF, &cpsw->wr_regs->rx_en);
 
 	cpdma_ctlr_int_ctrl(cpsw->dma, true);
 	return;
@@ -669,8 +691,8 @@ static void cpsw_intr_enable(struct cpsw_common *cpsw)
 
 static void cpsw_intr_disable(struct cpsw_common *cpsw)
 {
-	__raw_writel(0, &cpsw->wr_regs->tx_en);
-	__raw_writel(0, &cpsw->wr_regs->rx_en);
+	writel_relaxed(0, &cpsw->wr_regs->tx_en);
+	writel_relaxed(0, &cpsw->wr_regs->rx_en);
 
 	cpdma_ctlr_int_ctrl(cpsw->dma, false);
 	return;
@@ -949,18 +971,14 @@ static inline void soft_reset(const char *module, void __iomem *reg)
 {
 	unsigned long timeout = jiffies + HZ;
 
-	__raw_writel(1, reg);
+	writel_relaxed(1, reg);
 	do {
 		cpu_relax();
-	} while ((__raw_readl(reg) & 1) && time_after(timeout, jiffies));
+	} while ((readl_relaxed(reg) & 1) && time_after(timeout, jiffies));
 
-	WARN(__raw_readl(reg) & 1, "failed to soft-reset %s\n", module);
+	WARN(readl_relaxed(reg) & 1, "failed to soft-reset %s\n", module);
 }
 
-#define mac_hi(mac)	(((mac)[0] << 0) | ((mac)[1] << 8) |	\
-			 ((mac)[2] << 16) | ((mac)[3] << 24))
-#define mac_lo(mac)	(((mac)[4] << 0) | ((mac)[5] << 8))
-
 static void cpsw_set_slave_mac(struct cpsw_slave *slave,
 			       struct cpsw_priv *priv)
 {
@@ -996,7 +1014,8 @@ static void _cpsw_adjust_link(struct cpsw_slave *slave,
 		/* set speed_in input in case RMII mode is used in 100Mbps */
 		if (phy->speed == 100)
 			mac_control |= BIT(15);
-		else if (phy->speed == 10)
+		/* in band mode only works in 10Mbps RGMII mode */
+		else if ((phy->speed == 10) && phy_interface_is_rgmii(phy))
 			mac_control |= BIT(18); /* In Band mode */
 
 		if (priv->rx_pause)
@@ -1015,7 +1034,7 @@ static void _cpsw_adjust_link(struct cpsw_slave *slave,
 
 	if (mac_control != slave->mac_control) {
 		phy_print_status(phy);
-		__raw_writel(mac_control, &slave->sliver->mac_control);
+		writel_relaxed(mac_control, &slave->sliver->mac_control);
 	}
 
 	slave->mac_control = mac_control;
@@ -1278,7 +1297,7 @@ static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
 	soft_reset_slave(slave);
 
 	/* setup priority mapping */
-	__raw_writel(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
+	writel_relaxed(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
 
 	switch (cpsw->version) {
 	case CPSW_VERSION_1:
@@ -1304,7 +1323,7 @@ static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
 	}
 
 	/* setup max packet size, and mac address */
-	__raw_writel(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
+	writel_relaxed(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
 	cpsw_set_slave_mac(slave, priv);
 
 	slave->mac_control = 0;	/* no link yet */
@@ -1395,9 +1414,9 @@ static void cpsw_init_host_port(struct cpsw_priv *priv)
 	writel(fifo_mode, &cpsw->host_port_regs->tx_in_ctl);
 
 	/* setup host port priority mapping */
-	__raw_writel(CPDMA_TX_PRIORITY_MAP,
-		     &cpsw->host_port_regs->cpdma_tx_pri_map);
-	__raw_writel(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
+	writel_relaxed(CPDMA_TX_PRIORITY_MAP,
+		       &cpsw->host_port_regs->cpdma_tx_pri_map);
+	writel_relaxed(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
 
 	cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM,
 			     ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
@@ -1514,10 +1533,10 @@ static int cpsw_ndo_open(struct net_device *ndev)
 	/* initialize shared resources for every ndev */
 	if (!cpsw->usage_count) {
 		/* disable priority elevation */
-		__raw_writel(0, &cpsw->regs->ptype);
+		writel_relaxed(0, &cpsw->regs->ptype);
 
 		/* enable statistics collection only on all ports */
-		__raw_writel(0x7, &cpsw->regs->stat_port_en);
+		writel_relaxed(0x7, &cpsw->regs->stat_port_en);
 
 		/* Enable internal fifo flow control */
 		writel(0x7, &cpsw->regs->flow_control);
@@ -1618,6 +1637,7 @@ static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
 		q_idx = q_idx % cpsw->tx_ch_num;
 
 	txch = cpsw->txv[q_idx].ch;
+	txq = netdev_get_tx_queue(ndev, q_idx);
 	ret = cpsw_tx_packet_submit(priv, skb, txch);
 	if (unlikely(ret != 0)) {
 		cpsw_err(priv, tx_err, "desc submit failed\n");
@@ -1628,15 +1648,26 @@ static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
 	 * tell the kernel to stop sending us tx frames.
 	 */
 	if (unlikely(!cpdma_check_free_tx_desc(txch))) {
-		txq = netdev_get_tx_queue(ndev, q_idx);
 		netif_tx_stop_queue(txq);
+
+		/* Barrier, so that stop_queue visible to other cpus */
+		smp_mb__after_atomic();
+
+		if (cpdma_check_free_tx_desc(txch))
+			netif_tx_wake_queue(txq);
 	}
 
 	return NETDEV_TX_OK;
 fail:
 	ndev->stats.tx_dropped++;
-	txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
 	netif_tx_stop_queue(txq);
+
+	/* Barrier, so that stop_queue visible to other cpus */
+	smp_mb__after_atomic();
+
+	if (cpdma_check_free_tx_desc(txch))
+		netif_tx_wake_queue(txq);
+
 	return NETDEV_TX_BUSY;
 }
 
@@ -1701,7 +1732,7 @@ static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
 
 	slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
 	slave_write(slave, ctrl, CPSW2_CONTROL);
-	__raw_writel(ETH_P_1588, &cpsw->regs->ts_ltype);
+	writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);
 }
 
 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
@@ -2298,7 +2329,6 @@ static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
 
 static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
 {
-	int (*poll)(struct napi_struct *, int);
 	struct cpsw_common *cpsw = priv->cpsw;
 	void (*handler)(void *, int, int);
 	struct netdev_queue *queue;
@@ -2309,12 +2339,10 @@ static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
 		ch = &cpsw->rx_ch_num;
 		vec = cpsw->rxv;
 		handler = cpsw_rx_handler;
-		poll = cpsw_rx_poll;
 	} else {
 		ch = &cpsw->tx_ch_num;
 		vec = cpsw->txv;
 		handler = cpsw_tx_handler;
-		poll = cpsw_tx_poll;
 	}
 
 	while (*ch < ch_num) {
@@ -3050,17 +3078,23 @@ static int cpsw_probe(struct platform_device *pdev)
 	}
 
 	cpsw->txv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_tx_handler, 0);
+	if (IS_ERR(cpsw->txv[0].ch)) {
+		dev_err(priv->dev, "error initializing tx dma channel\n");
+		ret = PTR_ERR(cpsw->txv[0].ch);
+		goto clean_dma_ret;
+	}
+
 	cpsw->rxv[0].ch = cpdma_chan_create(cpsw->dma, 0, cpsw_rx_handler, 1);
-	if (WARN_ON(!cpsw->rxv[0].ch || !cpsw->txv[0].ch)) {
-		dev_err(priv->dev, "error initializing dma channels\n");
-		ret = -ENOMEM;
+	if (IS_ERR(cpsw->rxv[0].ch)) {
+		dev_err(priv->dev, "error initializing rx dma channel\n");
+		ret = PTR_ERR(cpsw->rxv[0].ch);
 		goto clean_dma_ret;
 	}
 
 	ale_params.dev			= &pdev->dev;
 	ale_params.ale_ageout		= ale_ageout;
 	ale_params.ale_entries		= data->ale_entries;
-	ale_params.ale_ports		= data->slaves;
+	ale_params.ale_ports		= CPSW_ALE_PORTS_NUM;
 
 	cpsw->ale = cpsw_ale_create(&ale_params);
 	if (!cpsw->ale) {
@@ -3072,14 +3106,14 @@ static int cpsw_probe(struct platform_device *pdev)
 	cpsw->cpts = cpts_create(cpsw->dev, cpts_regs, cpsw->dev->of_node);
 	if (IS_ERR(cpsw->cpts)) {
 		ret = PTR_ERR(cpsw->cpts);
-		goto clean_ale_ret;
+		goto clean_dma_ret;
 	}
 
 	ndev->irq = platform_get_irq(pdev, 1);
 	if (ndev->irq < 0) {
 		dev_err(priv->dev, "error getting irq resource\n");
 		ret = ndev->irq;
-		goto clean_ale_ret;
+		goto clean_dma_ret;
 	}
 
 	of_id = of_match_device(cpsw_of_mtable, &pdev->dev);
@@ -3103,7 +3137,7 @@ static int cpsw_probe(struct platform_device *pdev)
 	if (ret) {
 		dev_err(priv->dev, "error registering net device\n");
 		ret = -ENODEV;
-		goto clean_ale_ret;
+		goto clean_dma_ret;
 	}
 
 	if (cpsw->data.dual_emac) {
@@ -3126,7 +3160,7 @@ static int cpsw_probe(struct platform_device *pdev)
 	irq = platform_get_irq(pdev, 1);
 	if (irq < 0) {
 		ret = irq;
-		goto clean_ale_ret;
+		goto clean_dma_ret;
 	}
 
 	cpsw->irqs_table[0] = irq;
@@ -3134,14 +3168,14 @@ static int cpsw_probe(struct platform_device *pdev)
 			       0, dev_name(&pdev->dev), cpsw);
 	if (ret < 0) {
 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
-		goto clean_ale_ret;
+		goto clean_dma_ret;
 	}
 
 	/* TX IRQ */
 	irq = platform_get_irq(pdev, 2);
 	if (irq < 0) {
 		ret = irq;
-		goto clean_ale_ret;
+		goto clean_dma_ret;
 	}
 
 	cpsw->irqs_table[1] = irq;
@@ -3149,7 +3183,7 @@ static int cpsw_probe(struct platform_device *pdev)
 			       0, dev_name(&pdev->dev), cpsw);
 	if (ret < 0) {
 		dev_err(priv->dev, "error attaching irq (%d)\n", ret);
-		goto clean_ale_ret;
+		goto clean_dma_ret;
 	}
 
 	cpsw_notice(priv, probe,
@@ -3162,8 +3196,6 @@ static int cpsw_probe(struct platform_device *pdev)
 
 clean_unregister_netdev_ret:
 	unregister_netdev(ndev);
-clean_ale_ret:
-	cpsw_ale_destroy(cpsw->ale);
 clean_dma_ret:
 	cpdma_ctlr_destroy(cpsw->dma);
 clean_dt_ret:
@@ -3193,7 +3225,6 @@ static int cpsw_remove(struct platform_device *pdev)
 	unregister_netdev(ndev);
 
 	cpts_release(cpsw->cpts);
-	cpsw_ale_destroy(cpsw->ale);
 	cpdma_ctlr_destroy(cpsw->dma);
 	cpsw_remove_dt(pdev);
 	pm_runtime_put_sync(&pdev->dev);
diff --git a/drivers/net/ethernet/ti/cpsw.h b/drivers/net/ethernet/ti/cpsw.h
index 6c3037aa2cd3..cf111db3dc27 100644
--- a/drivers/net/ethernet/ti/cpsw.h
+++ b/drivers/net/ethernet/ti/cpsw.h
@@ -17,26 +17,9 @@
 #include <linux/if_ether.h>
 #include <linux/phy.h>
 
-struct cpsw_slave_data {
-	struct device_node *phy_node;
-	char		phy_id[MII_BUS_ID_SIZE];
-	int		phy_if;
-	u8		mac_addr[ETH_ALEN];
-	u16		dual_emac_res_vlan;	/* Reserved VLAN for DualEMAC */
-};
-
-struct cpsw_platform_data {
-	struct cpsw_slave_data	*slave_data;
-	u32	ss_reg_ofs;	/* Subsystem control register offset */
-	u32	channels;	/* number of cpdma channels (symmetric) */
-	u32	slaves;		/* number of slave cpgmac ports */
-	u32	active_slave; /* time stamping, ethtool and SIOCGMIIPHY slave */
-	u32	ale_entries;	/* ale table size */
-	u32	bd_ram_size;  /*buffer descriptor ram size */
-	u32	mac_control;	/* Mac control register */
-	u16	default_vlan;	/* Def VLAN for ALE lookup in VLAN aware mode*/
-	bool	dual_emac;	/* Enable Dual EMAC mode */
-};
+#define mac_hi(mac)	(((mac)[0] << 0) | ((mac)[1] << 8) |	\
+			 ((mac)[2] << 16) | ((mac)[3] << 24))
+#define mac_lo(mac)	(((mac)[4] << 0) | ((mac)[5] << 8))
 
 void cpsw_phy_sel(struct device *dev, phy_interface_t phy_mode, int slave);
 int ti_cm_get_macid(struct device *dev, int slave, u8 *mac_addr);
diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c
index b432a75fb874..93dc05c194d3 100644
--- a/drivers/net/ethernet/ti/cpsw_ale.c
+++ b/drivers/net/ethernet/ti/cpsw_ale.c
@@ -150,11 +150,11 @@ static int cpsw_ale_read(struct cpsw_ale *ale, int idx, u32 *ale_entry)
 
 	WARN_ON(idx > ale->params.ale_entries);
 
-	__raw_writel(idx, ale->params.ale_regs + ALE_TABLE_CONTROL);
+	writel_relaxed(idx, ale->params.ale_regs + ALE_TABLE_CONTROL);
 
 	for (i = 0; i < ALE_ENTRY_WORDS; i++)
-		ale_entry[i] = __raw_readl(ale->params.ale_regs +
-					   ALE_TABLE + 4 * i);
+		ale_entry[i] = readl_relaxed(ale->params.ale_regs +
+					     ALE_TABLE + 4 * i);
 
 	return idx;
 }
@@ -166,11 +166,11 @@ static int cpsw_ale_write(struct cpsw_ale *ale, int idx, u32 *ale_entry)
 	WARN_ON(idx > ale->params.ale_entries);
 
 	for (i = 0; i < ALE_ENTRY_WORDS; i++)
-		__raw_writel(ale_entry[i], ale->params.ale_regs +
-			     ALE_TABLE + 4 * i);
+		writel_relaxed(ale_entry[i], ale->params.ale_regs +
+			       ALE_TABLE + 4 * i);
 
-	__raw_writel(idx | ALE_TABLE_WRITE, ale->params.ale_regs +
-		     ALE_TABLE_CONTROL);
+	writel_relaxed(idx | ALE_TABLE_WRITE, ale->params.ale_regs +
+		       ALE_TABLE_CONTROL);
 
 	return idx;
 }
@@ -723,7 +723,7 @@ int cpsw_ale_control_set(struct cpsw_ale *ale, int port, int control,
 	if (info->port_offset == 0 && info->port_shift == 0)
 		port = 0; /* global, port is a dont care */
 
-	if (port < 0 || port > ale->params.ale_ports)
+	if (port < 0 || port >= ale->params.ale_ports)
 		return -EINVAL;
 
 	mask = BITMASK(info->bits);
@@ -733,9 +733,9 @@ int cpsw_ale_control_set(struct cpsw_ale *ale, int port, int control,
 	offset = info->offset + (port * info->port_offset);
 	shift  = info->shift  + (port * info->port_shift);
 
-	tmp = __raw_readl(ale->params.ale_regs + offset);
+	tmp = readl_relaxed(ale->params.ale_regs + offset);
 	tmp = (tmp & ~(mask << shift)) | (value << shift);
-	__raw_writel(tmp, ale->params.ale_regs + offset);
+	writel_relaxed(tmp, ale->params.ale_regs + offset);
 
 	return 0;
 }
@@ -754,13 +754,13 @@ int cpsw_ale_control_get(struct cpsw_ale *ale, int port, int control)
 	if (info->port_offset == 0 && info->port_shift == 0)
 		port = 0; /* global, port is a dont care */
 
-	if (port < 0 || port > ale->params.ale_ports)
+	if (port < 0 || port >= ale->params.ale_ports)
 		return -EINVAL;
 
 	offset = info->offset + (port * info->port_offset);
 	shift  = info->shift  + (port * info->port_shift);
 
-	tmp = __raw_readl(ale->params.ale_regs + offset) >> shift;
+	tmp = readl_relaxed(ale->params.ale_regs + offset) >> shift;
 	return tmp & BITMASK(info->bits);
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_control_get);
@@ -779,9 +779,37 @@ static void cpsw_ale_timer(struct timer_list *t)
 
 void cpsw_ale_start(struct cpsw_ale *ale)
 {
+	cpsw_ale_control_set(ale, 0, ALE_ENABLE, 1);
+	cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1);
+
+	timer_setup(&ale->timer, cpsw_ale_timer, 0);
+	if (ale->ageout) {
+		ale->timer.expires = jiffies + ale->ageout;
+		add_timer(&ale->timer);
+	}
+}
+EXPORT_SYMBOL_GPL(cpsw_ale_start);
+
+void cpsw_ale_stop(struct cpsw_ale *ale)
+{
+	del_timer_sync(&ale->timer);
+	cpsw_ale_control_set(ale, 0, ALE_ENABLE, 0);
+}
+EXPORT_SYMBOL_GPL(cpsw_ale_stop);
+
+struct cpsw_ale *cpsw_ale_create(struct cpsw_ale_params *params)
+{
+	struct cpsw_ale *ale;
 	u32 rev, ale_entries;
 
-	rev = __raw_readl(ale->params.ale_regs + ALE_IDVER);
+	ale = devm_kzalloc(params->dev, sizeof(*ale), GFP_KERNEL);
+	if (!ale)
+		return NULL;
+
+	ale->params = *params;
+	ale->ageout = ale->params.ale_ageout * HZ;
+
+	rev = readl_relaxed(ale->params.ale_regs + ALE_IDVER);
 	if (!ale->params.major_ver_mask)
 		ale->params.major_ver_mask = 0xff;
 	ale->version =
@@ -793,8 +821,8 @@ void cpsw_ale_start(struct cpsw_ale *ale)
 
 	if (!ale->params.ale_entries) {
 		ale_entries =
-			__raw_readl(ale->params.ale_regs + ALE_STATUS) &
-				    ALE_STATUS_SIZE_MASK;
+			readl_relaxed(ale->params.ale_regs + ALE_STATUS) &
+			ALE_STATUS_SIZE_MASK;
 		/* ALE available on newer NetCP switches has introduced
 		 * a register, ALE_STATUS, to indicate the size of ALE
 		 * table which shows the size as a multiple of 1024 entries.
@@ -816,9 +844,9 @@ void cpsw_ale_start(struct cpsw_ale *ale)
 		 "ALE Table size %ld\n", ale->params.ale_entries);
 
 	/* set default bits for existing h/w */
-	ale->port_mask_bits = 3;
-	ale->port_num_bits = 2;
-	ale->vlan_field_bits = 3;
+	ale->port_mask_bits = ale->params.ale_ports;
+	ale->port_num_bits = order_base_2(ale->params.ale_ports);
+	ale->vlan_field_bits = ale->params.ale_ports;
 
 	/* Set defaults override for ALE on NetCP NU switch and for version
 	 * 1R3
@@ -847,57 +875,12 @@ void cpsw_ale_start(struct cpsw_ale *ale)
 		ale_controls[ALE_PORT_UNTAGGED_EGRESS].shift = 0;
 		ale_controls[ALE_PORT_UNTAGGED_EGRESS].offset =
 					ALE_UNKNOWNVLAN_FORCE_UNTAG_EGRESS;
-		ale->port_mask_bits = ale->params.ale_ports;
-		ale->port_num_bits = ale->params.ale_ports - 1;
-		ale->vlan_field_bits = ale->params.ale_ports;
-	} else if (ale->version == ALE_VERSION_1R3) {
-		ale->port_mask_bits = ale->params.ale_ports;
-		ale->port_num_bits = 3;
-		ale->vlan_field_bits = ale->params.ale_ports;
 	}
 
-	cpsw_ale_control_set(ale, 0, ALE_ENABLE, 1);
-	cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1);
-
-	timer_setup(&ale->timer, cpsw_ale_timer, 0);
-	if (ale->ageout) {
-		ale->timer.expires = jiffies + ale->ageout;
-		add_timer(&ale->timer);
-	}
-}
-EXPORT_SYMBOL_GPL(cpsw_ale_start);
-
-void cpsw_ale_stop(struct cpsw_ale *ale)
-{
-	del_timer_sync(&ale->timer);
-}
-EXPORT_SYMBOL_GPL(cpsw_ale_stop);
-
-struct cpsw_ale *cpsw_ale_create(struct cpsw_ale_params *params)
-{
-	struct cpsw_ale *ale;
-
-	ale = kzalloc(sizeof(*ale), GFP_KERNEL);
-	if (!ale)
-		return NULL;
-
-	ale->params = *params;
-	ale->ageout = ale->params.ale_ageout * HZ;
-
 	return ale;
 }
 EXPORT_SYMBOL_GPL(cpsw_ale_create);
 
-int cpsw_ale_destroy(struct cpsw_ale *ale)
-{
-	if (!ale)
-		return -EINVAL;
-	cpsw_ale_control_set(ale, 0, ALE_ENABLE, 0);
-	kfree(ale);
-	return 0;
-}
-EXPORT_SYMBOL_GPL(cpsw_ale_destroy);
-
 void cpsw_ale_dump(struct cpsw_ale *ale, u32 *data)
 {
 	int i;
diff --git a/drivers/net/ethernet/ti/cpsw_ale.h b/drivers/net/ethernet/ti/cpsw_ale.h
index 25d24e8d0904..d4fe9016429b 100644
--- a/drivers/net/ethernet/ti/cpsw_ale.h
+++ b/drivers/net/ethernet/ti/cpsw_ale.h
@@ -100,7 +100,6 @@ enum cpsw_ale_port_state {
 #define ALE_ENTRY_WORDS	DIV_ROUND_UP(ALE_ENTRY_BITS, 32)
 
 struct cpsw_ale *cpsw_ale_create(struct cpsw_ale_params *params);
-int cpsw_ale_destroy(struct cpsw_ale *ale);
 
 void cpsw_ale_start(struct cpsw_ale *ale);
 void cpsw_ale_stop(struct cpsw_ale *ale);
diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c
index e4d6edf387b3..6f9173ff9414 100644
--- a/drivers/net/ethernet/ti/davinci_cpdma.c
+++ b/drivers/net/ethernet/ti/davinci_cpdma.c
@@ -893,7 +893,7 @@ struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num,
 	chan_num = rx_type ? rx_chan_num(chan_num) : tx_chan_num(chan_num);
 
 	if (__chan_linear(chan_num) >= ctlr->num_chan)
-		return NULL;
+		return ERR_PTR(-EINVAL);
 
 	chan = devm_kzalloc(ctlr->dev, sizeof(*chan), GFP_KERNEL);
 	if (!chan)
diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
index 4bb561856af5..abceea802ea1 100644
--- a/drivers/net/ethernet/ti/davinci_emac.c
+++ b/drivers/net/ethernet/ti/davinci_emac.c
@@ -1385,11 +1385,6 @@ static int emac_devioctl(struct net_device *ndev, struct ifreq *ifrq, int cmd)
 		return -EOPNOTSUPP;
 }
 
-static int match_first_device(struct device *dev, void *data)
-{
-	return !strncmp(dev_name(dev), "davinci_mdio", 12);
-}
-
 /**
  * emac_dev_open - EMAC device open
  * @ndev: The DaVinci EMAC network adapter
@@ -1489,8 +1484,8 @@ static int emac_dev_open(struct net_device *ndev)
 
 	/* use the first phy on the bus if pdata did not give us a phy id */
 	if (!phydev && !priv->phy_id) {
-		phy = bus_find_device(&mdio_bus_type, NULL, NULL,
-				      match_first_device);
+		phy = bus_find_device_by_name(&mdio_bus_type, NULL,
+					      "davinci_mdio");
 		if (phy) {
 			priv->phy_id = dev_name(phy);
 			if (!priv->phy_id || !*priv->phy_id)
@@ -1875,10 +1870,17 @@ static int davinci_emac_probe(struct platform_device *pdev)
 
 	priv->txchan = cpdma_chan_create(priv->dma, EMAC_DEF_TX_CH,
 					 emac_tx_handler, 0);
+	if (IS_ERR(priv->txchan)) {
+		dev_err(&pdev->dev, "error initializing tx dma channel\n");
+		rc = PTR_ERR(priv->txchan);
+		goto no_cpdma_chan;
+	}
+
 	priv->rxchan = cpdma_chan_create(priv->dma, EMAC_DEF_RX_CH,
 					 emac_rx_handler, 1);
-	if (WARN_ON(!priv->txchan || !priv->rxchan)) {
-		rc = -ENOMEM;
+	if (IS_ERR(priv->rxchan)) {
+		dev_err(&pdev->dev, "error initializing rx dma channel\n");
+		rc = PTR_ERR(priv->rxchan);
 		goto no_cpdma_chan;
 	}
 
diff --git a/drivers/net/ethernet/ti/netcp_ethss.c b/drivers/net/ethernet/ti/netcp_ethss.c
index e831c49713ee..56dbc0b9fedc 100644
--- a/drivers/net/ethernet/ti/netcp_ethss.c
+++ b/drivers/net/ethernet/ti/netcp_ethss.c
@@ -27,6 +27,7 @@
 #include <linux/net_tstamp.h>
 #include <linux/ethtool.h>
 
+#include "cpsw.h"
 #include "cpsw_ale.h"
 #include "netcp.h"
 #include "cpts.h"
@@ -2047,10 +2048,6 @@ static const struct ethtool_ops keystone_ethtool_ops = {
 	.get_ts_info		= keystone_get_ts_info,
 };
 
-#define mac_hi(mac)	(((mac)[0] << 0) | ((mac)[1] << 8) |	\
-			 ((mac)[2] << 16) | ((mac)[3] << 24))
-#define mac_lo(mac)	(((mac)[4] << 0) | ((mac)[5] << 8))
-
 static void gbe_set_slave_mac(struct gbe_slave *slave,
 			      struct gbe_intf *gbe_intf)
 {
@@ -3692,7 +3689,6 @@ static int gbe_remove(struct netcp_device *netcp_device, void *inst_priv)
 	del_timer_sync(&gbe_dev->timer);
 	cpts_release(gbe_dev->cpts);
 	cpsw_ale_stop(gbe_dev->ale);
-	cpsw_ale_destroy(gbe_dev->ale);
 	netcp_txpipe_close(&gbe_dev->tx_pipe);
 	free_secondary_ports(gbe_dev);
 
diff --git a/drivers/net/ethernet/tundra/tsi108_eth.c b/drivers/net/ethernet/tundra/tsi108_eth.c
index 0624b71ab5d4..edcd1e60b30d 100644
--- a/drivers/net/ethernet/tundra/tsi108_eth.c
+++ b/drivers/net/ethernet/tundra/tsi108_eth.c
@@ -152,6 +152,8 @@ struct tsi108_prv_data {
 	u32 msg_enable;			/* debug message level */
 	struct mii_if_info mii_if;
 	unsigned int init_media;
+
+	struct platform_device *pdev;
 };
 
 /* Structure for a device driver */
@@ -703,17 +705,18 @@ static int tsi108_send_packet(struct sk_buff * skb, struct net_device *dev)
 		data->txskbs[tx] = skb;
 
 		if (i == 0) {
-			data->txring[tx].buf0 = dma_map_single(NULL, skb->data,
-					skb_headlen(skb), DMA_TO_DEVICE);
+			data->txring[tx].buf0 = dma_map_single(&data->pdev->dev,
+					skb->data, skb_headlen(skb),
+					DMA_TO_DEVICE);
 			data->txring[tx].len = skb_headlen(skb);
 			misc |= TSI108_TX_SOF;
 		} else {
 			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
 
-			data->txring[tx].buf0 = skb_frag_dma_map(NULL, frag,
-								 0,
-								 skb_frag_size(frag),
-								 DMA_TO_DEVICE);
+			data->txring[tx].buf0 =
+				skb_frag_dma_map(&data->pdev->dev, frag,
+						0, skb_frag_size(frag),
+						DMA_TO_DEVICE);
 			data->txring[tx].len = skb_frag_size(frag);
 		}
 
@@ -808,9 +811,9 @@ static int tsi108_refill_rx(struct net_device *dev, int budget)
 		if (!skb)
 			break;
 
-		data->rxring[rx].buf0 = dma_map_single(NULL, skb->data,
-							TSI108_RX_SKB_SIZE,
-							DMA_FROM_DEVICE);
+		data->rxring[rx].buf0 = dma_map_single(&data->pdev->dev,
+				skb->data, TSI108_RX_SKB_SIZE,
+				DMA_FROM_DEVICE);
 
 		/* Sometimes the hardware sets blen to zero after packet
 		 * reception, even though the manual says that it's only ever
@@ -1308,15 +1311,15 @@ static int tsi108_open(struct net_device *dev)
 		       data->id, dev->irq, dev->name);
 	}
 
-	data->rxring = dma_zalloc_coherent(NULL, rxring_size, &data->rxdma,
-					   GFP_KERNEL);
+	data->rxring = dma_zalloc_coherent(&data->pdev->dev, rxring_size,
+			&data->rxdma, GFP_KERNEL);
 	if (!data->rxring)
 		return -ENOMEM;
 
-	data->txring = dma_zalloc_coherent(NULL, txring_size, &data->txdma,
-					   GFP_KERNEL);
+	data->txring = dma_zalloc_coherent(&data->pdev->dev, txring_size,
+			&data->txdma, GFP_KERNEL);
 	if (!data->txring) {
-		pci_free_consistent(NULL, rxring_size, data->rxring,
+		dma_free_coherent(&data->pdev->dev, rxring_size, data->rxring,
 				    data->rxdma);
 		return -ENOMEM;
 	}
@@ -1428,10 +1431,10 @@ static int tsi108_close(struct net_device *dev)
 		dev_kfree_skb(skb);
 	}
 
-	dma_free_coherent(0,
+	dma_free_coherent(&data->pdev->dev,
 			    TSI108_RXRING_LEN * sizeof(rx_desc),
 			    data->rxring, data->rxdma);
-	dma_free_coherent(0,
+	dma_free_coherent(&data->pdev->dev,
 			    TSI108_TXRING_LEN * sizeof(tx_desc),
 			    data->txring, data->txdma);
 
@@ -1576,6 +1579,7 @@ tsi108_init_one(struct platform_device *pdev)
 	printk("tsi108_eth%d: probe...\n", pdev->id);
 	data = netdev_priv(dev);
 	data->dev = dev;
+	data->pdev = pdev;
 
 	pr_debug("tsi108_eth%d:regs:phyresgs:phy:irq_num=0x%x:0x%x:0x%x:0x%x\n",
 			pdev->id, einfo->regs, einfo->phyregs,
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index 64fda2e1040e..b919e89a9b93 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -1652,19 +1652,16 @@ static __net_init int geneve_init_net(struct net *net)
 	return 0;
 }
 
-static void __net_exit geneve_exit_net(struct net *net)
+static void geneve_destroy_tunnels(struct net *net, struct list_head *head)
 {
 	struct geneve_net *gn = net_generic(net, geneve_net_id);
 	struct geneve_dev *geneve, *next;
 	struct net_device *dev, *aux;
-	LIST_HEAD(list);
-
-	rtnl_lock();
 
 	/* gather any geneve devices that were moved into this ns */
 	for_each_netdev_safe(net, dev, aux)
 		if (dev->rtnl_link_ops == &geneve_link_ops)
-			unregister_netdevice_queue(dev, &list);
+			unregister_netdevice_queue(dev, head);
 
 	/* now gather any other geneve devices that were created in this ns */
 	list_for_each_entry_safe(geneve, next, &gn->geneve_list, next) {
@@ -1672,18 +1669,29 @@ static void __net_exit geneve_exit_net(struct net *net)
 		 * to the list by the previous loop.
 		 */
 		if (!net_eq(dev_net(geneve->dev), net))
-			unregister_netdevice_queue(geneve->dev, &list);
+			unregister_netdevice_queue(geneve->dev, head);
 	}
 
+	WARN_ON_ONCE(!list_empty(&gn->sock_list));
+}
+
+static void __net_exit geneve_exit_batch_net(struct list_head *net_list)
+{
+	struct net *net;
+	LIST_HEAD(list);
+
+	rtnl_lock();
+	list_for_each_entry(net, net_list, exit_list)
+		geneve_destroy_tunnels(net, &list);
+
 	/* unregister the devices gathered above */
 	unregister_netdevice_many(&list);
 	rtnl_unlock();
-	WARN_ON_ONCE(!list_empty(&gn->sock_list));
 }
 
 static struct pernet_operations geneve_net_ops = {
 	.init = geneve_init_net,
-	.exit = geneve_exit_net,
+	.exit_batch = geneve_exit_batch_net,
 	.id   = &geneve_net_id,
 	.size = sizeof(struct geneve_net),
 };
diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h
index 88ddfb92122b..32861036c3fc 100644
--- a/drivers/net/hyperv/hyperv_net.h
+++ b/drivers/net/hyperv/hyperv_net.h
@@ -146,7 +146,6 @@ struct hv_netvsc_packet {
 
 struct netvsc_device_info {
 	unsigned char mac_adr[ETH_ALEN];
-	int  ring_size;
 	u32  num_chn;
 	u32  send_sections;
 	u32  recv_sections;
@@ -174,6 +173,7 @@ struct rndis_device {
 	struct list_head req_list;
 
 	struct work_struct mcast_work;
+	u32 filter;
 
 	bool link_state;        /* 0 - link up, 1 - link down */
 
@@ -188,18 +188,22 @@ struct rndis_message;
 struct netvsc_device;
 struct net_device_context;
 
+extern u32 netvsc_ring_bytes;
+extern struct reciprocal_value netvsc_ring_reciprocal;
+
 struct netvsc_device *netvsc_device_add(struct hv_device *device,
 					const struct netvsc_device_info *info);
 int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx);
 void netvsc_device_remove(struct hv_device *device);
-int netvsc_send(struct net_device_context *ndc,
+int netvsc_send(struct net_device *net,
 		struct hv_netvsc_packet *packet,
 		struct rndis_message *rndis_msg,
 		struct hv_page_buffer *page_buffer,
 		struct sk_buff *skb);
-void netvsc_linkstatus_callback(struct hv_device *device_obj,
+void netvsc_linkstatus_callback(struct net_device *net,
 				struct rndis_message *resp);
 int netvsc_recv_callback(struct net_device *net,
+			 struct netvsc_device *nvdev,
 			 struct vmbus_channel *channel,
 			 void  *data, u32 len,
 			 const struct ndis_tcp_ip_checksum_info *csum_info,
@@ -208,7 +212,6 @@ void netvsc_channel_cb(void *context);
 int netvsc_poll(struct napi_struct *napi, int budget);
 
 void rndis_set_subchannel(struct work_struct *w);
-bool rndis_filter_opened(const struct netvsc_device *nvdev);
 int rndis_filter_open(struct netvsc_device *nvdev);
 int rndis_filter_close(struct netvsc_device *nvdev);
 struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
@@ -220,7 +223,6 @@ int rndis_filter_set_rss_param(struct rndis_device *rdev,
 			       const u8 *key);
 int rndis_filter_receive(struct net_device *ndev,
 			 struct netvsc_device *net_dev,
-			 struct hv_device *dev,
 			 struct vmbus_channel *channel,
 			 void *data, u32 buflen);
 
@@ -635,14 +637,27 @@ struct nvsp_message {
 #define NETVSC_MTU 65535
 #define NETVSC_MTU_MIN ETH_MIN_MTU
 
-#define NETVSC_RECEIVE_BUFFER_SIZE		(1024*1024*16)	/* 16MB */
-#define NETVSC_RECEIVE_BUFFER_SIZE_LEGACY	(1024*1024*15)  /* 15MB */
-#define NETVSC_SEND_BUFFER_SIZE			(1024 * 1024 * 15)   /* 15MB */
+/* Max buffer sizes allowed by a host */
+#define NETVSC_RECEIVE_BUFFER_SIZE		(1024 * 1024 * 31) /* 31MB */
+#define NETVSC_RECEIVE_BUFFER_SIZE_LEGACY	(1024 * 1024 * 15) /* 15MB */
+#define NETVSC_RECEIVE_BUFFER_DEFAULT		(1024 * 1024 * 16)
+
+#define NETVSC_SEND_BUFFER_SIZE			(1024 * 1024 * 15)  /* 15MB */
+#define NETVSC_SEND_BUFFER_DEFAULT		(1024 * 1024)
+
 #define NETVSC_INVALID_INDEX			-1
 
 #define NETVSC_SEND_SECTION_SIZE		6144
 #define NETVSC_RECV_SECTION_SIZE		1728
 
+/* Default size of TX buf: 1MB, RX buf: 16MB */
+#define NETVSC_MIN_TX_SECTIONS	10
+#define NETVSC_DEFAULT_TX	(NETVSC_SEND_BUFFER_DEFAULT \
+				 / NETVSC_SEND_SECTION_SIZE)
+#define NETVSC_MIN_RX_SECTIONS	10
+#define NETVSC_DEFAULT_RX	(NETVSC_RECEIVE_BUFFER_DEFAULT \
+				 / NETVSC_RECV_SECTION_SIZE)
+
 #define NETVSC_RECEIVE_BUFFER_ID		0xcafe
 #define NETVSC_SEND_BUFFER_ID			0
 
@@ -690,6 +705,7 @@ struct netvsc_ethtool_stats {
 	unsigned long tx_busy;
 	unsigned long tx_send_full;
 	unsigned long rx_comp_busy;
+	unsigned long rx_no_memory;
 	unsigned long stop_queue;
 	unsigned long wake_queue;
 };
@@ -804,13 +820,9 @@ struct netvsc_device {
 
 	struct rndis_device *extension;
 
-	int ring_size;
-
 	u32 max_pkt; /* max number of pkt in one send, e.g. 8 */
 	u32 pkt_align; /* alignment bytes, e.g. 8 */
 
-	atomic_t open_cnt;
-
 	struct netvsc_channel chan_table[VRSS_CHANNEL_MAX];
 
 	struct rcu_head rcu;
@@ -1425,32 +1437,6 @@ struct rndis_message {
 	(sizeof(msg) + (sizeof(struct rndis_message) -	\
 	 sizeof(union rndis_message_container)))
 
-/* get pointer to info buffer with message pointer */
-#define MESSAGE_TO_INFO_BUFFER(msg)				\
-	(((unsigned char *)(msg)) + msg->info_buf_offset)
-
-/* get pointer to status buffer with message pointer */
-#define MESSAGE_TO_STATUS_BUFFER(msg)			\
-	(((unsigned char *)(msg)) + msg->status_buf_offset)
-
-/* get pointer to OOBD buffer with message pointer */
-#define MESSAGE_TO_OOBD_BUFFER(msg)				\
-	(((unsigned char *)(msg)) + msg->oob_data_offset)
-
-/* get pointer to data buffer with message pointer */
-#define MESSAGE_TO_DATA_BUFFER(msg)				\
-	(((unsigned char *)(msg)) + msg->per_pkt_info_offset)
-
-/* get pointer to contained message from NDIS_MESSAGE pointer */
-#define RNDIS_MESSAGE_PTR_TO_MESSAGE_PTR(rndis_msg)		\
-	((void *) &rndis_msg->msg)
-
-/* get pointer to contained message from NDIS_MESSAGE pointer */
-#define RNDIS_MESSAGE_RAW_PTR_TO_MESSAGE_PTR(rndis_msg)	\
-	((void *) rndis_msg)
-
-
-
 #define RNDIS_HEADER_SIZE	(sizeof(struct rndis_message) - \
 				 sizeof(union rndis_message_container))
 
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index bfc79698b8f4..7472172823f3 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -31,6 +31,7 @@
 #include <linux/vmalloc.h>
 #include <linux/rtnetlink.h>
 #include <linux/prefetch.h>
+#include <linux/reciprocal_div.h>
 
 #include <asm/sync_bitops.h>
 
@@ -72,7 +73,7 @@ static struct netvsc_device *alloc_net_device(void)
 
 	init_waitqueue_head(&net_device->wait_drain);
 	net_device->destroy = false;
-	atomic_set(&net_device->open_cnt, 0);
+
 	net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT;
 	net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT;
 
@@ -89,6 +90,11 @@ static void free_netvsc_device(struct rcu_head *head)
 		= container_of(head, struct netvsc_device, rcu);
 	int i;
 
+	kfree(nvdev->extension);
+	vfree(nvdev->recv_buf);
+	vfree(nvdev->send_buf);
+	kfree(nvdev->send_section_map);
+
 	for (i = 0; i < VRSS_CHANNEL_MAX; i++)
 		vfree(nvdev->chan_table[i].mrc.slots);
 
@@ -210,12 +216,6 @@ static void netvsc_teardown_gpadl(struct hv_device *device,
 		net_device->recv_buf_gpadl_handle = 0;
 	}
 
-	if (net_device->recv_buf) {
-		/* Free up the receive buffer */
-		vfree(net_device->recv_buf);
-		net_device->recv_buf = NULL;
-	}
-
 	if (net_device->send_buf_gpadl_handle) {
 		ret = vmbus_teardown_gpadl(device->channel,
 					   net_device->send_buf_gpadl_handle);
@@ -230,12 +230,6 @@ static void netvsc_teardown_gpadl(struct hv_device *device,
 		}
 		net_device->send_buf_gpadl_handle = 0;
 	}
-	if (net_device->send_buf) {
-		/* Free up the send buffer */
-		vfree(net_device->send_buf);
-		net_device->send_buf = NULL;
-	}
-	kfree(net_device->send_section_map);
 }
 
 int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx)
@@ -267,6 +261,11 @@ static int netvsc_init_buf(struct hv_device *device,
 	buf_size = device_info->recv_sections * device_info->recv_section_size;
 	buf_size = roundup(buf_size, PAGE_SIZE);
 
+	/* Legacy hosts only allow smaller receive buffer */
+	if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
+		buf_size = min_t(unsigned int, buf_size,
+				 NETVSC_RECEIVE_BUFFER_SIZE_LEGACY);
+
 	net_device->recv_buf = vzalloc(buf_size);
 	if (!net_device->recv_buf) {
 		netdev_err(ndev,
@@ -556,26 +555,29 @@ void netvsc_device_remove(struct hv_device *device)
 		= rtnl_dereference(net_device_ctx->nvdev);
 	int i;
 
-	cancel_work_sync(&net_device->subchan_work);
-
 	netvsc_revoke_buf(device, net_device);
 
 	RCU_INIT_POINTER(net_device_ctx->nvdev, NULL);
 
+	/* And disassociate NAPI context from device */
+	for (i = 0; i < net_device->num_chn; i++)
+		netif_napi_del(&net_device->chan_table[i].napi);
+
 	/*
 	 * At this point, no one should be accessing net_device
 	 * except in here
 	 */
 	netdev_dbg(ndev, "net device safe to remove\n");
 
+	/* older versions require that buffer be revoked before close */
+	if (net_device->nvsp_version < NVSP_PROTOCOL_VERSION_4)
+		netvsc_teardown_gpadl(device, net_device);
+
 	/* Now, we can close the channel safely */
 	vmbus_close(device->channel);
 
-	netvsc_teardown_gpadl(device, net_device);
-
-	/* And dissassociate NAPI context from device */
-	for (i = 0; i < net_device->num_chn; i++)
-		netif_napi_del(&net_device->chan_table[i].napi);
+	if (net_device->nvsp_version >= NVSP_PROTOCOL_VERSION_4)
+		netvsc_teardown_gpadl(device, net_device);
 
 	/* Release all resources */
 	free_netvsc_device_rcu(net_device);
@@ -588,14 +590,11 @@ void netvsc_device_remove(struct hv_device *device)
  * Get the percentage of available bytes to write in the ring.
  * The return value is in range from 0 to 100.
  */
-static inline u32 hv_ringbuf_avail_percent(
-		struct hv_ring_buffer_info *ring_info)
+static u32 hv_ringbuf_avail_percent(const struct hv_ring_buffer_info *ring_info)
 {
-	u32 avail_read, avail_write;
+	u32 avail_write = hv_get_bytes_to_write(ring_info);
 
-	hv_get_ringbuffer_availbytes(ring_info, &avail_read, &avail_write);
-
-	return avail_write * 100 / ring_info->ring_datasize;
+	return reciprocal_divide(avail_write  * 100, netvsc_ring_reciprocal);
 }
 
 static inline void netvsc_free_send_slot(struct netvsc_device *net_device,
@@ -642,14 +641,18 @@ static void netvsc_send_tx_complete(struct netvsc_device *net_device,
 	queue_sends =
 		atomic_dec_return(&net_device->chan_table[q_idx].queue_sends);
 
-	if (net_device->destroy && queue_sends == 0)
-		wake_up(&net_device->wait_drain);
+	if (unlikely(net_device->destroy)) {
+		if (queue_sends == 0)
+			wake_up(&net_device->wait_drain);
+	} else {
+		struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx);
 
-	if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) &&
-	    (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER ||
-	     queue_sends < 1)) {
-		netif_tx_wake_queue(netdev_get_tx_queue(ndev, q_idx));
-		ndev_ctx->eth_stats.wake_queue++;
+		if (netif_tx_queue_stopped(txq) &&
+		    (hv_ringbuf_avail_percent(&channel->outbound) > RING_AVAIL_PERCENT_HIWATER ||
+		     queue_sends < 1)) {
+			netif_tx_wake_queue(txq);
+			ndev_ctx->eth_stats.wake_queue++;
+		}
 	}
 }
 
@@ -698,26 +701,26 @@ static u32 netvsc_get_next_send_section(struct netvsc_device *net_device)
 	return NETVSC_INVALID_INDEX;
 }
 
-static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device,
-				   unsigned int section_index,
-				   u32 pend_size,
-				   struct hv_netvsc_packet *packet,
-				   struct rndis_message *rndis_msg,
-				   struct hv_page_buffer *pb,
-				   struct sk_buff *skb)
+static void netvsc_copy_to_send_buf(struct netvsc_device *net_device,
+				    unsigned int section_index,
+				    u32 pend_size,
+				    struct hv_netvsc_packet *packet,
+				    struct rndis_message *rndis_msg,
+				    struct hv_page_buffer *pb,
+				    bool xmit_more)
 {
 	char *start = net_device->send_buf;
 	char *dest = start + (section_index * net_device->send_section_size)
 		     + pend_size;
 	int i;
-	u32 msg_size = 0;
 	u32 padding = 0;
-	u32 remain = packet->total_data_buflen % net_device->pkt_align;
 	u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt :
 		packet->page_buf_cnt;
+	u32 remain;
 
 	/* Add padding */
-	if (skb->xmit_more && remain && !packet->cp_partial) {
+	remain = packet->total_data_buflen & (net_device->pkt_align - 1);
+	if (xmit_more && remain) {
 		padding = net_device->pkt_align - remain;
 		rndis_msg->msg_len += padding;
 		packet->total_data_buflen += padding;
@@ -729,16 +732,11 @@ static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device,
 		u32 len = pb[i].len;
 
 		memcpy(dest, (src + offset), len);
-		msg_size += len;
 		dest += len;
 	}
 
-	if (padding) {
+	if (padding)
 		memset(dest, 0, padding);
-		msg_size += padding;
-	}
-
-	return msg_size;
 }
 
 static inline int netvsc_send_pkt(
@@ -831,12 +829,13 @@ static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send,
 }
 
 /* RCU already held by caller */
-int netvsc_send(struct net_device_context *ndev_ctx,
+int netvsc_send(struct net_device *ndev,
 		struct hv_netvsc_packet *packet,
 		struct rndis_message *rndis_msg,
 		struct hv_page_buffer *pb,
 		struct sk_buff *skb)
 {
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
 	struct netvsc_device *net_device
 		= rcu_dereference_bh(ndev_ctx->nvdev);
 	struct hv_device *device = ndev_ctx->device_ctx;
@@ -847,20 +846,12 @@ int netvsc_send(struct net_device_context *ndev_ctx,
 	struct multi_send_data *msdp;
 	struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL;
 	struct sk_buff *msd_skb = NULL;
-	bool try_batch;
-	bool xmit_more = (skb != NULL) ? skb->xmit_more : false;
+	bool try_batch, xmit_more;
 
 	/* If device is rescinded, return error and packet will get dropped. */
 	if (unlikely(!net_device || net_device->destroy))
 		return -ENODEV;
 
-	/* We may race with netvsc_connect_vsp()/netvsc_init_buf() and get
-	 * here before the negotiation with the host is finished and
-	 * send_section_map may not be allocated yet.
-	 */
-	if (unlikely(!net_device->send_section_map))
-		return -EAGAIN;
-
 	nvchan = &net_device->chan_table[packet->q_idx];
 	packet->send_buf_index = NETVSC_INVALID_INDEX;
 	packet->cp_partial = false;
@@ -868,10 +859,8 @@ int netvsc_send(struct net_device_context *ndev_ctx,
 	/* Send control message directly without accessing msd (Multi-Send
 	 * Data) field which may be changed during data packet processing.
 	 */
-	if (!skb) {
-		cur_send = packet;
-		goto send_now;
-	}
+	if (!skb)
+		return netvsc_send_pkt(device, packet, net_device, pb, skb);
 
 	/* batch packets in send buffer if possible */
 	msdp = &nvchan->msd;
@@ -899,10 +888,17 @@ int netvsc_send(struct net_device_context *ndev_ctx,
 		}
 	}
 
+	/* Keep aggregating only if stack says more data is coming
+	 * and not doing mixed modes send and not flow blocked
+	 */
+	xmit_more = skb->xmit_more &&
+		!packet->cp_partial &&
+		!netif_xmit_stopped(netdev_get_tx_queue(ndev, packet->q_idx));
+
 	if (section_index != NETVSC_INVALID_INDEX) {
 		netvsc_copy_to_send_buf(net_device,
 					section_index, msd_len,
-					packet, rndis_msg, pb, skb);
+					packet, rndis_msg, pb, xmit_more);
 
 		packet->send_buf_index = section_index;
 
@@ -922,7 +918,7 @@ int netvsc_send(struct net_device_context *ndev_ctx,
 		if (msdp->skb)
 			dev_consume_skb_any(msdp->skb);
 
-		if (xmit_more && !packet->cp_partial) {
+		if (xmit_more) {
 			msdp->skb = skb;
 			msdp->pkt = packet;
 			msdp->count++;
@@ -948,7 +944,6 @@ int netvsc_send(struct net_device_context *ndev_ctx,
 		}
 	}
 
-send_now:
 	if (cur_send)
 		ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb);
 
@@ -1085,7 +1080,7 @@ static int netvsc_receive(struct net_device *ndev,
 		u32 buflen = vmxferpage_packet->ranges[i].byte_count;
 
 		/* Pass it to the upper layer */
-		status = rndis_filter_receive(ndev, net_device, device,
+		status = rndis_filter_receive(ndev, net_device,
 					      channel, data, buflen);
 	}
 
@@ -1212,9 +1207,10 @@ int netvsc_poll(struct napi_struct *napi, int budget)
 	if (send_recv_completions(ndev, net_device, nvchan) == 0 &&
 	    work_done < budget &&
 	    napi_complete_done(napi, work_done) &&
-	    hv_end_read(&channel->inbound)) {
+	    hv_end_read(&channel->inbound) &&
+	    napi_schedule_prep(napi)) {
 		hv_begin_read(&channel->inbound);
-		napi_reschedule(napi);
+		__napi_schedule(napi);
 	}
 
 	/* Driver may overshoot since multiple packets per descriptor */
@@ -1237,7 +1233,7 @@ void netvsc_channel_cb(void *context)
 		/* disable interupts from host */
 		hv_begin_read(rbi);
 
-		__napi_schedule(&nvchan->napi);
+		__napi_schedule_irqoff(&nvchan->napi);
 	}
 }
 
@@ -1249,7 +1245,6 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
 				const struct netvsc_device_info *device_info)
 {
 	int i, ret = 0;
-	int ring_size = device_info->ring_size;
 	struct netvsc_device *net_device;
 	struct net_device *ndev = hv_get_drvdata(device);
 	struct net_device_context *net_device_ctx = netdev_priv(ndev);
@@ -1261,8 +1256,6 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
 	for (i = 0; i < VRSS_SEND_TAB_SIZE; i++)
 		net_device_ctx->tx_table[i] = 0;
 
-	net_device->ring_size = ring_size;
-
 	/* Because the device uses NAPI, all the interrupt batching and
 	 * control is done via Net softirq, not the channel handling
 	 */
@@ -1289,13 +1282,11 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
 		       netvsc_poll, NAPI_POLL_WEIGHT);
 
 	/* Open the channel */
-	ret = vmbus_open(device->channel, ring_size * PAGE_SIZE,
-			 ring_size * PAGE_SIZE, NULL, 0,
-			 netvsc_channel_cb,
-			 net_device->chan_table);
+	ret = vmbus_open(device->channel, netvsc_ring_bytes,
+			 netvsc_ring_bytes,  NULL, 0,
+			 netvsc_channel_cb, net_device->chan_table);
 
 	if (ret != 0) {
-		netif_napi_del(&net_device->chan_table[0].napi);
 		netdev_err(ndev, "unable to open channel: %d\n", ret);
 		goto cleanup;
 	}
@@ -1305,11 +1296,6 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
 
 	napi_enable(&net_device->chan_table[0].napi);
 
-	/* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is
-	 * populated.
-	 */
-	rcu_assign_pointer(net_device_ctx->nvdev, net_device);
-
 	/* Connect with the NetVsp */
 	ret = netvsc_connect_vsp(device, net_device, device_info);
 	if (ret != 0) {
@@ -1318,6 +1304,11 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
 		goto close;
 	}
 
+	/* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is
+	 * populated.
+	 */
+	rcu_assign_pointer(net_device_ctx->nvdev, net_device);
+
 	return net_device;
 
 close:
@@ -1328,6 +1319,7 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
 	vmbus_close(device->channel);
 
 cleanup:
+	netif_napi_del(&net_device->chan_table[0].napi);
 	free_netvsc_device(&net_device->rcu);
 
 	return ERR_PTR(ret);
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 5129647d420c..f28c85d212ce 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -35,6 +35,7 @@
 #include <linux/slab.h>
 #include <linux/rtnetlink.h>
 #include <linux/netpoll.h>
+#include <linux/reciprocal_div.h>
 
 #include <net/arp.h>
 #include <net/route.h>
@@ -45,18 +46,19 @@
 
 #include "hyperv_net.h"
 
-#define RING_SIZE_MIN		64
-#define NETVSC_MIN_TX_SECTIONS	10
-#define NETVSC_DEFAULT_TX	192	/* ~1M */
-#define NETVSC_MIN_RX_SECTIONS	10	/* ~64K */
-#define NETVSC_DEFAULT_RX	10485   /* Max ~16M */
+#define RING_SIZE_MIN	64
+#define RETRY_US_LO	5000
+#define RETRY_US_HI	10000
+#define RETRY_MAX	2000	/* >10 sec */
 
 #define LINKCHANGE_INT (2 * HZ)
 #define VF_TAKEOVER_INT (HZ / 10)
 
-static int ring_size = 128;
-module_param(ring_size, int, S_IRUGO);
+static unsigned int ring_size __ro_after_init = 128;
+module_param(ring_size, uint, S_IRUGO);
 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
+unsigned int netvsc_ring_bytes __ro_after_init;
+struct reciprocal_value netvsc_ring_reciprocal __ro_after_init;
 
 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
 				NETIF_MSG_LINK | NETIF_MSG_IFUP |
@@ -67,12 +69,43 @@ static int debug = -1;
 module_param(debug, int, S_IRUGO);
 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 
-static void netvsc_set_multicast_list(struct net_device *net)
+static void netvsc_change_rx_flags(struct net_device *net, int change)
 {
-	struct net_device_context *net_device_ctx = netdev_priv(net);
-	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
+	struct net_device_context *ndev_ctx = netdev_priv(net);
+	struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
+	int inc;
+
+	if (!vf_netdev)
+		return;
+
+	if (change & IFF_PROMISC) {
+		inc = (net->flags & IFF_PROMISC) ? 1 : -1;
+		dev_set_promiscuity(vf_netdev, inc);
+	}
+
+	if (change & IFF_ALLMULTI) {
+		inc = (net->flags & IFF_ALLMULTI) ? 1 : -1;
+		dev_set_allmulti(vf_netdev, inc);
+	}
+}
+
+static void netvsc_set_rx_mode(struct net_device *net)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(net);
+	struct net_device *vf_netdev;
+	struct netvsc_device *nvdev;
+
+	rcu_read_lock();
+	vf_netdev = rcu_dereference(ndev_ctx->vf_netdev);
+	if (vf_netdev) {
+		dev_uc_sync(vf_netdev, net);
+		dev_mc_sync(vf_netdev, net);
+	}
 
-	rndis_filter_update(nvdev);
+	nvdev = rcu_dereference(ndev_ctx->nvdev);
+	if (nvdev)
+		rndis_filter_update(nvdev);
+	rcu_read_unlock();
 }
 
 static int netvsc_open(struct net_device *net)
@@ -92,10 +125,7 @@ static int netvsc_open(struct net_device *net)
 		return ret;
 	}
 
-	netif_tx_wake_all_queues(net);
-
 	rdev = nvdev->extension;
-
 	if (!rdev->link_state)
 		netif_carrier_on(net);
 
@@ -113,36 +143,25 @@ static int netvsc_open(struct net_device *net)
 	return 0;
 }
 
-static int netvsc_close(struct net_device *net)
+static int netvsc_wait_until_empty(struct netvsc_device *nvdev)
 {
-	struct net_device_context *net_device_ctx = netdev_priv(net);
-	struct net_device *vf_netdev
-		= rtnl_dereference(net_device_ctx->vf_netdev);
-	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
-	int ret = 0;
-	u32 aread, i, msec = 10, retry = 0, retry_max = 20;
-	struct vmbus_channel *chn;
-
-	netif_tx_disable(net);
-
-	/* No need to close rndis filter if it is removed already */
-	if (!nvdev)
-		goto out;
-
-	ret = rndis_filter_close(nvdev);
-	if (ret != 0) {
-		netdev_err(net, "unable to close device (ret %d).\n", ret);
-		return ret;
-	}
+	unsigned int retry = 0;
+	int i;
 
 	/* Ensure pending bytes in ring are read */
-	while (true) {
-		aread = 0;
+	for (;;) {
+		u32 aread = 0;
+
 		for (i = 0; i < nvdev->num_chn; i++) {
-			chn = nvdev->chan_table[i].channel;
+			struct vmbus_channel *chn
+				= nvdev->chan_table[i].channel;
+
 			if (!chn)
 				continue;
 
+			/* make sure receive not running now */
+			napi_synchronize(&nvdev->chan_table[i].napi);
+
 			aread = hv_get_bytes_to_read(&chn->inbound);
 			if (aread)
 				break;
@@ -152,39 +171,55 @@ static int netvsc_close(struct net_device *net)
 				break;
 		}
 
-		retry++;
-		if (retry > retry_max || aread == 0)
-			break;
+		if (aread == 0)
+			return 0;
+
+		if (++retry > RETRY_MAX)
+			return -ETIMEDOUT;
 
-		msleep(msec);
+		usleep_range(RETRY_US_LO, RETRY_US_HI);
+	}
+}
+
+static int netvsc_close(struct net_device *net)
+{
+	struct net_device_context *net_device_ctx = netdev_priv(net);
+	struct net_device *vf_netdev
+		= rtnl_dereference(net_device_ctx->vf_netdev);
+	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
+	int ret;
+
+	netif_tx_disable(net);
 
-		if (msec < 1000)
-			msec *= 2;
+	/* No need to close rndis filter if it is removed already */
+	if (!nvdev)
+		return 0;
+
+	ret = rndis_filter_close(nvdev);
+	if (ret != 0) {
+		netdev_err(net, "unable to close device (ret %d).\n", ret);
+		return ret;
 	}
 
-	if (aread) {
+	ret = netvsc_wait_until_empty(nvdev);
+	if (ret)
 		netdev_err(net, "Ring buffer not empty after closing rndis\n");
-		ret = -ETIMEDOUT;
-	}
 
-out:
 	if (vf_netdev)
 		dev_close(vf_netdev);
 
 	return ret;
 }
 
-static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size,
-			   int pkt_type)
+static inline void *init_ppi_data(struct rndis_message *msg,
+				  u32 ppi_size, u32 pkt_type)
 {
-	struct rndis_packet *rndis_pkt;
+	struct rndis_packet *rndis_pkt = &msg->msg.pkt;
 	struct rndis_per_packet_info *ppi;
 
-	rndis_pkt = &msg->msg.pkt;
 	rndis_pkt->data_offset += ppi_size;
-
-	ppi = (struct rndis_per_packet_info *)((void *)rndis_pkt +
-		rndis_pkt->per_pkt_info_offset + rndis_pkt->per_pkt_info_len);
+	ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset
+		+ rndis_pkt->per_pkt_info_len;
 
 	ppi->size = ppi_size;
 	ppi->type = pkt_type;
@@ -192,7 +227,7 @@ static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size,
 
 	rndis_pkt->per_pkt_info_len += ppi_size;
 
-	return ppi;
+	return ppi + 1;
 }
 
 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented
@@ -302,8 +337,19 @@ static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
 	rcu_read_lock();
 	vf_netdev = rcu_dereference(ndc->vf_netdev);
 	if (vf_netdev) {
-		txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 0;
-		qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;
+		const struct net_device_ops *vf_ops = vf_netdev->netdev_ops;
+
+		if (vf_ops->ndo_select_queue)
+			txq = vf_ops->ndo_select_queue(vf_netdev, skb,
+						       accel_priv, fallback);
+		else
+			txq = fallback(vf_netdev, skb);
+
+		/* Record the queue selected by VF so that it can be
+		 * used for common case where VF has more queues than
+		 * the synthetic device.
+		 */
+		qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq;
 	} else {
 		txq = netvsc_pick_tx(ndev, skb);
 	}
@@ -469,10 +515,8 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 	int ret;
 	unsigned int num_data_pgs;
 	struct rndis_message *rndis_msg;
-	struct rndis_packet *rndis_pkt;
 	struct net_device *vf_netdev;
 	u32 rndis_msg_size;
-	struct rndis_per_packet_info *ppi;
 	u32 hash;
 	struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
 
@@ -527,34 +571,36 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 
 	rndis_msg = (struct rndis_message *)skb->head;
 
-	memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE);
-
 	/* Add the rndis header */
 	rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
 	rndis_msg->msg_len = packet->total_data_buflen;
-	rndis_pkt = &rndis_msg->msg.pkt;
-	rndis_pkt->data_offset = sizeof(struct rndis_packet);
-	rndis_pkt->data_len = packet->total_data_buflen;
-	rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet);
+
+	rndis_msg->msg.pkt = (struct rndis_packet) {
+		.data_offset = sizeof(struct rndis_packet),
+		.data_len = packet->total_data_buflen,
+		.per_pkt_info_offset = sizeof(struct rndis_packet),
+	};
 
 	rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
 
 	hash = skb_get_hash_raw(skb);
 	if (hash != 0 && net->real_num_tx_queues > 1) {
+		u32 *hash_info;
+
 		rndis_msg_size += NDIS_HASH_PPI_SIZE;
-		ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
-				    NBL_HASH_VALUE);
-		*(u32 *)((void *)ppi + ppi->ppi_offset) = hash;
+		hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
+					  NBL_HASH_VALUE);
+		*hash_info = hash;
 	}
 
 	if (skb_vlan_tag_present(skb)) {
 		struct ndis_pkt_8021q_info *vlan;
 
 		rndis_msg_size += NDIS_VLAN_PPI_SIZE;
-		ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
-				    IEEE_8021Q_INFO);
+		vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
+				     IEEE_8021Q_INFO);
 
-		vlan = (void *)ppi + ppi->ppi_offset;
+		vlan->value = 0;
 		vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
 		vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
 				VLAN_PRIO_SHIFT;
@@ -564,11 +610,10 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 		struct ndis_tcp_lso_info *lso_info;
 
 		rndis_msg_size += NDIS_LSO_PPI_SIZE;
-		ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
-				    TCP_LARGESEND_PKTINFO);
-
-		lso_info = (void *)ppi + ppi->ppi_offset;
+		lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
+					 TCP_LARGESEND_PKTINFO);
 
+		lso_info->value = 0;
 		lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
 		if (skb->protocol == htons(ETH_P_IP)) {
 			lso_info->lso_v2_transmit.ip_version =
@@ -593,12 +638,10 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 			struct ndis_tcp_ip_checksum_info *csum_info;
 
 			rndis_msg_size += NDIS_CSUM_PPI_SIZE;
-			ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
-					    TCPIP_CHKSUM_PKTINFO);
-
-			csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi +
-									 ppi->ppi_offset);
+			csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
+						  TCPIP_CHKSUM_PKTINFO);
 
+			csum_info->value = 0;
 			csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
 
 			if (skb->protocol == htons(ETH_P_IP)) {
@@ -632,7 +675,7 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 	/* timestamp packet in software */
 	skb_tx_timestamp(skb);
 
-	ret = netvsc_send(net_device_ctx, packet, rndis_msg, pb, skb);
+	ret = netvsc_send(net, packet, rndis_msg, pb, skb);
 	if (likely(ret == 0))
 		return NETDEV_TX_OK;
 
@@ -658,22 +701,14 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
 /*
  * netvsc_linkstatus_callback - Link up/down notification
  */
-void netvsc_linkstatus_callback(struct hv_device *device_obj,
+void netvsc_linkstatus_callback(struct net_device *net,
 				struct rndis_message *resp)
 {
 	struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
-	struct net_device *net;
-	struct net_device_context *ndev_ctx;
+	struct net_device_context *ndev_ctx = netdev_priv(net);
 	struct netvsc_reconfig *event;
 	unsigned long flags;
 
-	net = hv_get_drvdata(device_obj);
-
-	if (!net)
-		return;
-
-	ndev_ctx = netdev_priv(net);
-
 	/* Update the physical link speed when changing to another vSwitch */
 	if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
 		u32 speed;
@@ -753,34 +788,26 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
  * "wire" on the specified device.
  */
 int netvsc_recv_callback(struct net_device *net,
+			 struct netvsc_device *net_device,
 			 struct vmbus_channel *channel,
 			 void  *data, u32 len,
 			 const struct ndis_tcp_ip_checksum_info *csum_info,
 			 const struct ndis_pkt_8021q_info *vlan)
 {
 	struct net_device_context *net_device_ctx = netdev_priv(net);
-	struct netvsc_device *net_device;
 	u16 q_idx = channel->offermsg.offer.sub_channel_index;
-	struct netvsc_channel *nvchan;
+	struct netvsc_channel *nvchan = &net_device->chan_table[q_idx];
 	struct sk_buff *skb;
 	struct netvsc_stats *rx_stats;
 
 	if (net->reg_state != NETREG_REGISTERED)
 		return NVSP_STAT_FAIL;
 
-	rcu_read_lock();
-	net_device = rcu_dereference(net_device_ctx->nvdev);
-	if (unlikely(!net_device))
-		goto drop;
-
-	nvchan = &net_device->chan_table[q_idx];
-
 	/* Allocate a skb - TODO direct I/O to pages? */
 	skb = netvsc_alloc_recv_skb(net, &nvchan->napi,
 				    csum_info, vlan, data, len);
 	if (unlikely(!skb)) {
-drop:
-		++net->stats.rx_dropped;
+		++net_device_ctx->eth_stats.rx_no_memory;
 		rcu_read_unlock();
 		return NVSP_STAT_FAIL;
 	}
@@ -804,8 +831,6 @@ int netvsc_recv_callback(struct net_device *net,
 	u64_stats_update_end(&rx_stats->syncp);
 
 	napi_gro_receive(&nvchan->napi, skb);
-	rcu_read_unlock();
-
 	return 0;
 }
 
@@ -828,16 +853,81 @@ static void netvsc_get_channels(struct net_device *net,
 	}
 }
 
+static int netvsc_detach(struct net_device *ndev,
+			 struct netvsc_device *nvdev)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+	struct hv_device *hdev = ndev_ctx->device_ctx;
+	int ret;
+
+	/* Don't try continuing to try and setup sub channels */
+	if (cancel_work_sync(&nvdev->subchan_work))
+		nvdev->num_chn = 1;
+
+	/* If device was up (receiving) then shutdown */
+	if (netif_running(ndev)) {
+		netif_tx_disable(ndev);
+
+		ret = rndis_filter_close(nvdev);
+		if (ret) {
+			netdev_err(ndev,
+				   "unable to close device (ret %d).\n", ret);
+			return ret;
+		}
+
+		ret = netvsc_wait_until_empty(nvdev);
+		if (ret) {
+			netdev_err(ndev,
+				   "Ring buffer not empty after closing rndis\n");
+			return ret;
+		}
+	}
+
+	netif_device_detach(ndev);
+
+	rndis_filter_device_remove(hdev, nvdev);
+
+	return 0;
+}
+
+static int netvsc_attach(struct net_device *ndev,
+			 struct netvsc_device_info *dev_info)
+{
+	struct net_device_context *ndev_ctx = netdev_priv(ndev);
+	struct hv_device *hdev = ndev_ctx->device_ctx;
+	struct netvsc_device *nvdev;
+	struct rndis_device *rdev;
+	int ret;
+
+	nvdev = rndis_filter_device_add(hdev, dev_info);
+	if (IS_ERR(nvdev))
+		return PTR_ERR(nvdev);
+
+	/* Note: enable and attach happen when sub-channels setup */
+
+	netif_carrier_off(ndev);
+
+	if (netif_running(ndev)) {
+		ret = rndis_filter_open(nvdev);
+		if (ret)
+			return ret;
+
+		rdev = nvdev->extension;
+		if (!rdev->link_state)
+			netif_carrier_on(ndev);
+	}
+
+	return 0;
+}
+
 static int netvsc_set_channels(struct net_device *net,
 			       struct ethtool_channels *channels)
 {
 	struct net_device_context *net_device_ctx = netdev_priv(net);
-	struct hv_device *dev = net_device_ctx->device_ctx;
 	struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
 	unsigned int orig, count = channels->combined_count;
 	struct netvsc_device_info device_info;
-	bool was_opened;
-	int ret = 0;
+	int ret;
 
 	/* We do not support separate count for rx, tx, or other */
 	if (count == 0 ||
@@ -854,40 +944,25 @@ static int netvsc_set_channels(struct net_device *net,
 		return -EINVAL;
 
 	orig = nvdev->num_chn;
-	was_opened = rndis_filter_opened(nvdev);
-	if (was_opened)
-		rndis_filter_close(nvdev);
 
 	memset(&device_info, 0, sizeof(device_info));
 	device_info.num_chn = count;
-	device_info.ring_size = ring_size;
 	device_info.send_sections = nvdev->send_section_cnt;
 	device_info.send_section_size = nvdev->send_section_size;
 	device_info.recv_sections = nvdev->recv_section_cnt;
 	device_info.recv_section_size = nvdev->recv_section_size;
 
-	rndis_filter_device_remove(dev, nvdev);
+	ret = netvsc_detach(net, nvdev);
+	if (ret)
+		return ret;
 
-	nvdev = rndis_filter_device_add(dev, &device_info);
-	if (IS_ERR(nvdev)) {
-		ret = PTR_ERR(nvdev);
+	ret = netvsc_attach(net, &device_info);
+	if (ret) {
 		device_info.num_chn = orig;
-		nvdev = rndis_filter_device_add(dev, &device_info);
-
-		if (IS_ERR(nvdev)) {
-			netdev_err(net, "restoring channel setting failed: %ld\n",
-				   PTR_ERR(nvdev));
-			return ret;
-		}
+		if (netvsc_attach(net, &device_info))
+			netdev_err(net, "restoring channel setting failed\n");
 	}
 
-	if (was_opened)
-		rndis_filter_open(nvdev);
-
-	/* We may have missed link change notifications */
-	net_device_ctx->last_reconfig = 0;
-	schedule_delayed_work(&net_device_ctx->dwork, 0);
-
 	return ret;
 }
 
@@ -953,10 +1028,8 @@ static int netvsc_change_mtu(struct net_device *ndev, int mtu)
 	struct net_device_context *ndevctx = netdev_priv(ndev);
 	struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
 	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
-	struct hv_device *hdev = ndevctx->device_ctx;
 	int orig_mtu = ndev->mtu;
 	struct netvsc_device_info device_info;
-	bool was_opened;
 	int ret = 0;
 
 	if (!nvdev || nvdev->destroy)
@@ -969,48 +1042,34 @@ static int netvsc_change_mtu(struct net_device *ndev, int mtu)
 			return ret;
 	}
 
-	netif_device_detach(ndev);
-	was_opened = rndis_filter_opened(nvdev);
-	if (was_opened)
-		rndis_filter_close(nvdev);
-
 	memset(&device_info, 0, sizeof(device_info));
-	device_info.ring_size = ring_size;
 	device_info.num_chn = nvdev->num_chn;
 	device_info.send_sections = nvdev->send_section_cnt;
 	device_info.send_section_size = nvdev->send_section_size;
 	device_info.recv_sections = nvdev->recv_section_cnt;
 	device_info.recv_section_size = nvdev->recv_section_size;
 
-	rndis_filter_device_remove(hdev, nvdev);
+	ret = netvsc_detach(ndev, nvdev);
+	if (ret)
+		goto rollback_vf;
 
 	ndev->mtu = mtu;
 
-	nvdev = rndis_filter_device_add(hdev, &device_info);
-	if (IS_ERR(nvdev)) {
-		ret = PTR_ERR(nvdev);
-
-		/* Attempt rollback to original MTU */
-		ndev->mtu = orig_mtu;
-		nvdev = rndis_filter_device_add(hdev, &device_info);
-
-		if (vf_netdev)
-			dev_set_mtu(vf_netdev, orig_mtu);
-
-		if (IS_ERR(nvdev)) {
-			netdev_err(ndev, "restoring mtu failed: %ld\n",
-				   PTR_ERR(nvdev));
-			return ret;
-		}
-	}
+	ret = netvsc_attach(ndev, &device_info);
+	if (ret)
+		goto rollback;
 
-	if (was_opened)
-		rndis_filter_open(nvdev);
+	return 0;
 
-	netif_device_attach(ndev);
+rollback:
+	/* Attempt rollback to original MTU */
+	ndev->mtu = orig_mtu;
 
-	/* We may have missed link change notifications */
-	schedule_delayed_work(&ndevctx->dwork, 0);
+	if (netvsc_attach(ndev, &device_info))
+		netdev_err(ndev, "restoring mtu failed\n");
+rollback_vf:
+	if (vf_netdev)
+		dev_set_mtu(vf_netdev, orig_mtu);
 
 	return ret;
 }
@@ -1133,12 +1192,13 @@ static const struct {
 	u16 offset;
 } netvsc_stats[] = {
 	{ "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
-	{ "tx_no_memory",  offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
+	{ "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
 	{ "tx_no_space",  offsetof(struct netvsc_ethtool_stats, tx_no_space) },
 	{ "tx_too_big",	  offsetof(struct netvsc_ethtool_stats, tx_too_big) },
 	{ "tx_busy",	  offsetof(struct netvsc_ethtool_stats, tx_busy) },
 	{ "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
 	{ "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
+	{ "rx_no_memory", offsetof(struct netvsc_ethtool_stats, rx_no_memory) },
 	{ "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) },
 	{ "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) },
 }, vf_stats[] = {
@@ -1515,11 +1575,9 @@ static int netvsc_set_ringparam(struct net_device *ndev,
 {
 	struct net_device_context *ndevctx = netdev_priv(ndev);
 	struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
-	struct hv_device *hdev = ndevctx->device_ctx;
 	struct netvsc_device_info device_info;
 	struct ethtool_ringparam orig;
 	u32 new_tx, new_rx;
-	bool was_opened;
 	int ret = 0;
 
 	if (!nvdev || nvdev->destroy)
@@ -1539,40 +1597,23 @@ static int netvsc_set_ringparam(struct net_device *ndev,
 
 	memset(&device_info, 0, sizeof(device_info));
 	device_info.num_chn = nvdev->num_chn;
-	device_info.ring_size = ring_size;
 	device_info.send_sections = new_tx;
 	device_info.send_section_size = nvdev->send_section_size;
 	device_info.recv_sections = new_rx;
 	device_info.recv_section_size = nvdev->recv_section_size;
 
-	netif_device_detach(ndev);
-	was_opened = rndis_filter_opened(nvdev);
-	if (was_opened)
-		rndis_filter_close(nvdev);
-
-	rndis_filter_device_remove(hdev, nvdev);
-
-	nvdev = rndis_filter_device_add(hdev, &device_info);
-	if (IS_ERR(nvdev)) {
-		ret = PTR_ERR(nvdev);
+	ret = netvsc_detach(ndev, nvdev);
+	if (ret)
+		return ret;
 
+	ret = netvsc_attach(ndev, &device_info);
+	if (ret) {
 		device_info.send_sections = orig.tx_pending;
 		device_info.recv_sections = orig.rx_pending;
-		nvdev = rndis_filter_device_add(hdev, &device_info);
-		if (IS_ERR(nvdev)) {
-			netdev_err(ndev, "restoring ringparam failed: %ld\n",
-				   PTR_ERR(nvdev));
-			return ret;
-		}
-	}
 
-	if (was_opened)
-		rndis_filter_open(nvdev);
-	netif_device_attach(ndev);
-
-	/* We may have missed link change notifications */
-	ndevctx->last_reconfig = 0;
-	schedule_delayed_work(&ndevctx->dwork, 0);
+		if (netvsc_attach(ndev, &device_info))
+			netdev_err(ndev, "restoring ringparam failed");
+	}
 
 	return ret;
 }
@@ -1602,7 +1643,8 @@ static const struct net_device_ops device_ops = {
 	.ndo_open =			netvsc_open,
 	.ndo_stop =			netvsc_close,
 	.ndo_start_xmit =		netvsc_start_xmit,
-	.ndo_set_rx_mode =		netvsc_set_multicast_list,
+	.ndo_change_rx_flags =		netvsc_change_rx_flags,
+	.ndo_set_rx_mode =		netvsc_set_rx_mode,
 	.ndo_change_mtu =		netvsc_change_mtu,
 	.ndo_validate_addr =		eth_validate_addr,
 	.ndo_set_mac_address =		netvsc_set_mac_addr,
@@ -1833,6 +1875,15 @@ static void __netvsc_vf_setup(struct net_device *ndev,
 		netdev_warn(vf_netdev,
 			    "unable to change mtu to %u\n", ndev->mtu);
 
+	/* set multicast etc flags on VF */
+	dev_change_flags(vf_netdev, ndev->flags | IFF_SLAVE);
+
+	/* sync address list from ndev to VF */
+	netif_addr_lock_bh(ndev);
+	dev_uc_sync(vf_netdev, ndev);
+	dev_mc_sync(vf_netdev, ndev);
+	netif_addr_unlock_bh(ndev);
+
 	if (netif_running(ndev)) {
 		ret = dev_open(vf_netdev);
 		if (ret)
@@ -1995,7 +2046,6 @@ static int netvsc_probe(struct hv_device *dev,
 
 	/* Notify the netvsc driver of the new device */
 	memset(&device_info, 0, sizeof(device_info));
-	device_info.ring_size = ring_size;
 	device_info.num_chn = VRSS_CHANNEL_DEFAULT;
 	device_info.send_sections = NETVSC_DEFAULT_TX;
 	device_info.send_section_size = NETVSC_SEND_SECTION_SIZE;
@@ -2048,8 +2098,8 @@ static int netvsc_probe(struct hv_device *dev,
 static int netvsc_remove(struct hv_device *dev)
 {
 	struct net_device_context *ndev_ctx;
-	struct net_device *vf_netdev;
-	struct net_device *net;
+	struct net_device *vf_netdev, *net;
+	struct netvsc_device *nvdev;
 
 	net = hv_get_drvdata(dev);
 	if (net == NULL) {
@@ -2059,10 +2109,14 @@ static int netvsc_remove(struct hv_device *dev)
 
 	ndev_ctx = netdev_priv(net);
 
-	netif_device_detach(net);
-
 	cancel_delayed_work_sync(&ndev_ctx->dwork);
 
+	rcu_read_lock();
+	nvdev = rcu_dereference(ndev_ctx->nvdev);
+
+	if  (nvdev)
+		cancel_work_sync(&nvdev->subchan_work);
+
 	/*
 	 * Call to the vsc driver to let it know that the device is being
 	 * removed. Also blocks mtu and channel changes.
@@ -2072,11 +2126,13 @@ static int netvsc_remove(struct hv_device *dev)
 	if (vf_netdev)
 		netvsc_unregister_vf(vf_netdev);
 
+	if (nvdev)
+		rndis_filter_device_remove(dev, nvdev);
+
 	unregister_netdevice(net);
 
-	rndis_filter_device_remove(dev,
-				   rtnl_dereference(ndev_ctx->nvdev));
 	rtnl_unlock();
+	rcu_read_unlock();
 
 	hv_set_drvdata(dev, NULL);
 
@@ -2158,11 +2214,13 @@ static int __init netvsc_drv_init(void)
 
 	if (ring_size < RING_SIZE_MIN) {
 		ring_size = RING_SIZE_MIN;
-		pr_info("Increased ring_size to %d (min allowed)\n",
+		pr_info("Increased ring_size to %u (min allowed)\n",
 			ring_size);
 	}
-	ret = vmbus_driver_register(&netvsc_drv);
+	netvsc_ring_bytes = ring_size * PAGE_SIZE;
+	netvsc_ring_reciprocal = reciprocal_value(netvsc_ring_bytes);
 
+	ret = vmbus_driver_register(&netvsc_drv);
 	if (ret)
 		return ret;
 
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index 7b637c7dd1e5..465c42e30508 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -134,11 +134,9 @@ static void put_rndis_request(struct rndis_device *dev,
 	kfree(req);
 }
 
-static void dump_rndis_message(struct hv_device *hv_dev,
+static void dump_rndis_message(struct net_device *netdev,
 			       const struct rndis_message *rndis_msg)
 {
-	struct net_device *netdev = hv_get_drvdata(hv_dev);
-
 	switch (rndis_msg->ndis_msg_type) {
 	case RNDIS_MSG_PACKET:
 		netdev_dbg(netdev, "RNDIS_MSG_PACKET (len %u, "
@@ -217,7 +215,6 @@ static int rndis_filter_send_request(struct rndis_device *dev,
 	struct hv_netvsc_packet *packet;
 	struct hv_page_buffer page_buf[2];
 	struct hv_page_buffer *pb = page_buf;
-	struct net_device_context *net_device_ctx = netdev_priv(dev->ndev);
 	int ret;
 
 	/* Setup the packet to send it */
@@ -245,7 +242,7 @@ static int rndis_filter_send_request(struct rndis_device *dev,
 	}
 
 	rcu_read_lock_bh();
-	ret = netvsc_send(net_device_ctx, packet, NULL, pb, NULL);
+	ret = netvsc_send(dev->ndev, packet, NULL, pb, NULL);
 	rcu_read_unlock_bh();
 
 	return ret;
@@ -267,13 +264,23 @@ static void rndis_set_link_state(struct rndis_device *rdev,
 	}
 }
 
-static void rndis_filter_receive_response(struct rndis_device *dev,
-				       struct rndis_message *resp)
+static void rndis_filter_receive_response(struct net_device *ndev,
+					  struct netvsc_device *nvdev,
+					  const struct rndis_message *resp)
 {
+	struct rndis_device *dev = nvdev->extension;
 	struct rndis_request *request = NULL;
 	bool found = false;
 	unsigned long flags;
-	struct net_device *ndev = dev->ndev;
+
+	/* This should never happen, it means control message
+	 * response received after device removed.
+	 */
+	if (dev->state == RNDIS_DEV_UNINITIALIZED) {
+		netdev_err(ndev,
+			   "got rndis message uninitialized\n");
+		return;
+	}
 
 	spin_lock_irqsave(&dev->request_lock, flags);
 	list_for_each_entry(request, &dev->req_list, list_ent) {
@@ -354,7 +361,7 @@ static inline void *rndis_get_ppi(struct rndis_packet *rpkt, u32 type)
 }
 
 static int rndis_filter_receive_data(struct net_device *ndev,
-				     struct rndis_device *dev,
+				     struct netvsc_device *nvdev,
 				     struct rndis_message *msg,
 				     struct vmbus_channel *channel,
 				     void *data, u32 data_buflen)
@@ -374,7 +381,7 @@ static int rndis_filter_receive_data(struct net_device *ndev,
 	 * should be the data packet size plus the trailer padding size
 	 */
 	if (unlikely(data_buflen < rndis_pkt->data_len)) {
-		netdev_err(dev->ndev, "rndis message buffer "
+		netdev_err(ndev, "rndis message buffer "
 			   "overflow detected (got %u, min %u)"
 			   "...dropping this message!\n",
 			   data_buflen, rndis_pkt->data_len);
@@ -390,51 +397,37 @@ static int rndis_filter_receive_data(struct net_device *ndev,
 	 */
 	data = (void *)((unsigned long)data + data_offset);
 	csum_info = rndis_get_ppi(rndis_pkt, TCPIP_CHKSUM_PKTINFO);
-	return netvsc_recv_callback(ndev, channel,
+
+	return netvsc_recv_callback(ndev, nvdev, channel,
 				    data, rndis_pkt->data_len,
 				    csum_info, vlan);
 }
 
 int rndis_filter_receive(struct net_device *ndev,
 			 struct netvsc_device *net_dev,
-			 struct hv_device *dev,
 			 struct vmbus_channel *channel,
 			 void *data, u32 buflen)
 {
 	struct net_device_context *net_device_ctx = netdev_priv(ndev);
-	struct rndis_device *rndis_dev = net_dev->extension;
 	struct rndis_message *rndis_msg = data;
 
-	/* Make sure the rndis device state is initialized */
-	if (unlikely(!rndis_dev)) {
-		netif_dbg(net_device_ctx, rx_err, ndev,
-			  "got rndis message but no rndis device!\n");
-		return NVSP_STAT_FAIL;
-	}
-
-	if (unlikely(rndis_dev->state == RNDIS_DEV_UNINITIALIZED)) {
-		netif_dbg(net_device_ctx, rx_err, ndev,
-			  "got rndis message uninitialized\n");
-		return NVSP_STAT_FAIL;
-	}
-
 	if (netif_msg_rx_status(net_device_ctx))
-		dump_rndis_message(dev, rndis_msg);
+		dump_rndis_message(ndev, rndis_msg);
 
 	switch (rndis_msg->ndis_msg_type) {
 	case RNDIS_MSG_PACKET:
-		return rndis_filter_receive_data(ndev, rndis_dev, rndis_msg,
+		return rndis_filter_receive_data(ndev, net_dev, rndis_msg,
 						 channel, data, buflen);
 	case RNDIS_MSG_INIT_C:
 	case RNDIS_MSG_QUERY_C:
 	case RNDIS_MSG_SET_C:
 		/* completion msgs */
-		rndis_filter_receive_response(rndis_dev, rndis_msg);
+		rndis_filter_receive_response(ndev, net_dev, rndis_msg);
 		break;
 
 	case RNDIS_MSG_INDICATE:
 		/* notification msgs */
-		netvsc_linkstatus_callback(dev, rndis_msg);
+		netvsc_linkstatus_callback(ndev, rndis_msg);
 		break;
 	default:
 		netdev_err(ndev,
@@ -826,13 +819,15 @@ static int rndis_filter_set_packet_filter(struct rndis_device *dev,
 	struct rndis_set_request *set;
 	int ret;
 
+	if (dev->filter == new_filter)
+		return 0;
+
 	request = get_rndis_request(dev, RNDIS_MSG_SET,
 			RNDIS_MESSAGE_SIZE(struct rndis_set_request) +
 			sizeof(u32));
 	if (!request)
 		return -ENOMEM;
 
-
 	/* Setup the rndis set */
 	set = &request->request_msg.msg.set_req;
 	set->oid = RNDIS_OID_GEN_CURRENT_PACKET_FILTER;
@@ -843,8 +838,10 @@ static int rndis_filter_set_packet_filter(struct rndis_device *dev,
 	       &new_filter, sizeof(u32));
 
 	ret = rndis_filter_send_request(dev, request);
-	if (ret == 0)
+	if (ret == 0) {
 		wait_for_completion(&request->wait_event);
+		dev->filter = new_filter;
+	}
 
 	put_rndis_request(dev, request);
 
@@ -855,15 +852,19 @@ static void rndis_set_multicast(struct work_struct *w)
 {
 	struct rndis_device *rdev
 		= container_of(w, struct rndis_device, mcast_work);
+	u32 filter = NDIS_PACKET_TYPE_DIRECTED;
+	unsigned int flags = rdev->ndev->flags;
 
-	if (rdev->ndev->flags & IFF_PROMISC)
-		rndis_filter_set_packet_filter(rdev,
-					       NDIS_PACKET_TYPE_PROMISCUOUS);
-	else
-		rndis_filter_set_packet_filter(rdev,
-					       NDIS_PACKET_TYPE_BROADCAST |
-					       NDIS_PACKET_TYPE_ALL_MULTICAST |
-					       NDIS_PACKET_TYPE_DIRECTED);
+	if (flags & IFF_PROMISC) {
+		filter = NDIS_PACKET_TYPE_PROMISCUOUS;
+	} else {
+		if (!netdev_mc_empty(rdev->ndev) || (flags & IFF_ALLMULTI))
+			filter |= NDIS_PACKET_TYPE_ALL_MULTICAST;
+		if (flags & IFF_BROADCAST)
+			filter |= NDIS_PACKET_TYPE_BROADCAST;
+	}
+
+	rndis_filter_set_packet_filter(rdev, filter);
 }
 
 void rndis_filter_update(struct netvsc_device *nvdev)
@@ -1040,8 +1041,8 @@ static void netvsc_sc_open(struct vmbus_channel *new_sc)
 	/* Set the channel before opening.*/
 	nvchan->channel = new_sc;
 
-	ret = vmbus_open(new_sc, nvscdev->ring_size * PAGE_SIZE,
-			 nvscdev->ring_size * PAGE_SIZE, NULL, 0,
+	ret = vmbus_open(new_sc, netvsc_ring_bytes,
+			 netvsc_ring_bytes, NULL, 0,
 			 netvsc_channel_cb, nvchan);
 	if (ret == 0)
 		napi_enable(&nvchan->napi);
@@ -1117,6 +1118,7 @@ void rndis_set_subchannel(struct work_struct *w)
 	for (i = 0; i < VRSS_SEND_TAB_SIZE; i++)
 		ndev_ctx->tx_table[i] = i % nvdev->num_chn;
 
+	netif_device_attach(ndev);
 	rtnl_unlock();
 	return;
 
@@ -1127,6 +1129,8 @@ void rndis_set_subchannel(struct work_struct *w)
 
 	nvdev->max_chn = 1;
 	nvdev->num_chn = 1;
+
+	netif_device_attach(ndev);
 unlock:
 	rtnl_unlock();
 }
@@ -1222,7 +1226,6 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
 	struct ndis_recv_scale_cap rsscap;
 	u32 rsscap_size = sizeof(struct ndis_recv_scale_cap);
 	u32 mtu, size;
-	const struct cpumask *node_cpu_mask;
 	u32 num_possible_rss_qs;
 	int i, ret;
 
@@ -1291,14 +1294,8 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
 	if (ret || rsscap.num_recv_que < 2)
 		goto out;
 
-	/*
-	 * We will limit the VRSS channels to the number CPUs in the NUMA node
-	 * the primary channel is currently bound to.
-	 *
-	 * This also guarantees that num_possible_rss_qs <= num_online_cpus
-	 */
-	node_cpu_mask = cpumask_of_node(cpu_to_node(dev->channel->target_cpu));
-	num_possible_rss_qs = min_t(u32, cpumask_weight(node_cpu_mask),
+	/* This guarantees that num_possible_rss_qs <= num_online_cpus */
+	num_possible_rss_qs = min_t(u32, num_online_cpus(),
 				    rsscap.num_recv_que);
 
 	net_device->max_chn = min_t(u32, VRSS_CHANNEL_MAX, num_possible_rss_qs);
@@ -1336,6 +1333,10 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
 		net_device->num_chn = 1;
 	}
 
+	/* No sub channels, device is ready */
+	if (net_device->num_chn == 1)
+		netif_device_attach(net);
+
 	return net_device;
 
 err_dev_remv:
@@ -1354,7 +1355,6 @@ void rndis_filter_device_remove(struct hv_device *dev,
 	net_dev->extension = NULL;
 
 	netvsc_device_remove(dev);
-	kfree(rndis_dev);
 }
 
 int rndis_filter_open(struct netvsc_device *nvdev)
@@ -1362,9 +1362,6 @@ int rndis_filter_open(struct netvsc_device *nvdev)
 	if (!nvdev)
 		return -EINVAL;
 
-	if (atomic_inc_return(&nvdev->open_cnt) != 1)
-		return 0;
-
 	return rndis_filter_open_device(nvdev->extension);
 }
 
@@ -1373,13 +1370,5 @@ int rndis_filter_close(struct netvsc_device *nvdev)
 	if (!nvdev)
 		return -EINVAL;
 
-	if (atomic_dec_return(&nvdev->open_cnt) != 0)
-		return 0;
-
 	return rndis_filter_close_device(nvdev->extension);
 }
-
-bool rndis_filter_opened(const struct netvsc_device *nvdev)
-{
-	return atomic_read(&nvdev->open_cnt) > 0;
-}
diff --git a/drivers/net/ieee802154/adf7242.c b/drivers/net/ieee802154/adf7242.c
index 400fdbd3a120..64f1b1e77bc0 100644
--- a/drivers/net/ieee802154/adf7242.c
+++ b/drivers/net/ieee802154/adf7242.c
@@ -1,7 +1,7 @@
 /*
  * Analog Devices ADF7242 Low-Power IEEE 802.15.4 Transceiver
  *
- * Copyright 2009-2015 Analog Devices Inc.
+ * Copyright 2009-2017 Analog Devices Inc.
  *
  * Licensed under the GPL-2 or later.
  *
@@ -344,12 +344,18 @@ static int adf7242_wait_status(struct adf7242_local *lp, unsigned int status,
 	return ret;
 }
 
-static int adf7242_wait_ready(struct adf7242_local *lp, int line)
+static int adf7242_wait_rc_ready(struct adf7242_local *lp, int line)
 {
 	return adf7242_wait_status(lp, STAT_RC_READY | STAT_SPI_READY,
 				   STAT_RC_READY | STAT_SPI_READY, line);
 }
 
+static int adf7242_wait_spi_ready(struct adf7242_local *lp, int line)
+{
+	return adf7242_wait_status(lp, STAT_SPI_READY,
+				   STAT_SPI_READY, line);
+}
+
 static int adf7242_write_fbuf(struct adf7242_local *lp, u8 *data, u8 len)
 {
 	u8 *buf = lp->buf;
@@ -369,7 +375,7 @@ static int adf7242_write_fbuf(struct adf7242_local *lp, u8 *data, u8 len)
 	spi_message_add_tail(&xfer_head, &msg);
 	spi_message_add_tail(&xfer_buf, &msg);
 
-	adf7242_wait_ready(lp, __LINE__);
+	adf7242_wait_spi_ready(lp, __LINE__);
 
 	mutex_lock(&lp->bmux);
 	buf[0] = CMD_SPI_PKT_WR;
@@ -401,7 +407,7 @@ static int adf7242_read_fbuf(struct adf7242_local *lp,
 	spi_message_add_tail(&xfer_head, &msg);
 	spi_message_add_tail(&xfer_buf, &msg);
 
-	adf7242_wait_ready(lp, __LINE__);
+	adf7242_wait_spi_ready(lp, __LINE__);
 
 	mutex_lock(&lp->bmux);
 	if (packet_read) {
@@ -432,7 +438,7 @@ static int adf7242_read_reg(struct adf7242_local *lp, u16 addr, u8 *data)
 		.rx_buf = lp->buf_read_rx,
 	};
 
-	adf7242_wait_ready(lp, __LINE__);
+	adf7242_wait_spi_ready(lp, __LINE__);
 
 	mutex_lock(&lp->bmux);
 	lp->buf_read_tx[0] = CMD_SPI_MEM_RD(addr);
@@ -462,7 +468,7 @@ static int adf7242_write_reg(struct adf7242_local *lp, u16 addr, u8 data)
 {
 	int status;
 
-	adf7242_wait_ready(lp, __LINE__);
+	adf7242_wait_spi_ready(lp, __LINE__);
 
 	mutex_lock(&lp->bmux);
 	lp->buf_reg_tx[0] = CMD_SPI_MEM_WR(addr);
@@ -484,7 +490,7 @@ static int adf7242_cmd(struct adf7242_local *lp, unsigned int cmd)
 	dev_vdbg(&lp->spi->dev, "%s : CMD=0x%X\n", __func__, cmd);
 
 	if (cmd != CMD_RC_PC_RESET_NO_WAIT)
-		adf7242_wait_ready(lp, __LINE__);
+		adf7242_wait_rc_ready(lp, __LINE__);
 
 	mutex_lock(&lp->bmux);
 	lp->buf_cmd = cmd;
@@ -557,6 +563,22 @@ static int adf7242_verify_firmware(struct adf7242_local *lp,
 	return 0;
 }
 
+static void adf7242_clear_irqstat(struct adf7242_local *lp)
+{
+	adf7242_write_reg(lp, REG_IRQ1_SRC1, IRQ_CCA_COMPLETE | IRQ_SFD_RX |
+			  IRQ_SFD_TX | IRQ_RX_PKT_RCVD | IRQ_TX_PKT_SENT |
+			  IRQ_FRAME_VALID | IRQ_ADDRESS_VALID | IRQ_CSMA_CA);
+}
+
+static int adf7242_cmd_rx(struct adf7242_local *lp)
+{
+	/* Wait until the ACK is sent */
+	adf7242_wait_status(lp, RC_STATUS_PHY_RDY, RC_STATUS_MASK, __LINE__);
+	adf7242_clear_irqstat(lp);
+
+	return adf7242_cmd(lp, CMD_RC_RX);
+}
+
 static int adf7242_set_txpower(struct ieee802154_hw *hw, int mbm)
 {
 	struct adf7242_local *lp = hw->priv;
@@ -660,7 +682,7 @@ static int adf7242_start(struct ieee802154_hw *hw)
 	struct adf7242_local *lp = hw->priv;
 
 	adf7242_cmd(lp, CMD_RC_PHY_RDY);
-	adf7242_write_reg(lp, REG_IRQ1_SRC1, 0xFF);
+	adf7242_clear_irqstat(lp);
 	enable_irq(lp->spi->irq);
 	set_bit(FLAG_START, &lp->flags);
 
@@ -671,10 +693,10 @@ static void adf7242_stop(struct ieee802154_hw *hw)
 {
 	struct adf7242_local *lp = hw->priv;
 
+	disable_irq(lp->spi->irq);
 	adf7242_cmd(lp, CMD_RC_IDLE);
 	clear_bit(FLAG_START, &lp->flags);
-	disable_irq(lp->spi->irq);
-	adf7242_write_reg(lp, REG_IRQ1_SRC1, 0xFF);
+	adf7242_clear_irqstat(lp);
 }
 
 static int adf7242_channel(struct ieee802154_hw *hw, u8 page, u8 channel)
@@ -789,9 +811,12 @@ static int adf7242_xmit(struct ieee802154_hw *hw, struct sk_buff *skb)
 	struct adf7242_local *lp = hw->priv;
 	int ret;
 
+	/* ensure existing instances of the IRQ handler have completed */
+	disable_irq(lp->spi->irq);
 	set_bit(FLAG_XMIT, &lp->flags);
 	reinit_completion(&lp->tx_complete);
 	adf7242_cmd(lp, CMD_RC_PHY_RDY);
+	adf7242_clear_irqstat(lp);
 
 	ret = adf7242_write_fbuf(lp, skb->data, skb->len);
 	if (ret)
@@ -800,6 +825,7 @@ static int adf7242_xmit(struct ieee802154_hw *hw, struct sk_buff *skb)
 	ret = adf7242_cmd(lp, CMD_RC_CSMACA);
 	if (ret)
 		goto err;
+	enable_irq(lp->spi->irq);
 
 	ret = wait_for_completion_interruptible_timeout(&lp->tx_complete,
 							HZ / 10);
@@ -822,7 +848,7 @@ static int adf7242_xmit(struct ieee802154_hw *hw, struct sk_buff *skb)
 
 err:
 	clear_bit(FLAG_XMIT, &lp->flags);
-	adf7242_cmd(lp, CMD_RC_RX);
+	adf7242_cmd_rx(lp);
 
 	return ret;
 }
@@ -846,7 +872,7 @@ static int adf7242_rx(struct adf7242_local *lp)
 
 	skb = dev_alloc_skb(len);
 	if (!skb) {
-		adf7242_cmd(lp, CMD_RC_RX);
+		adf7242_cmd_rx(lp);
 		return -ENOMEM;
 	}
 
@@ -854,14 +880,14 @@ static int adf7242_rx(struct adf7242_local *lp)
 	ret = adf7242_read_fbuf(lp, data, len, true);
 	if (ret < 0) {
 		kfree_skb(skb);
-		adf7242_cmd(lp, CMD_RC_RX);
+		adf7242_cmd_rx(lp);
 		return ret;
 	}
 
 	lqi = data[len - 2];
 	lp->rssi = data[len - 1];
 
-	adf7242_cmd(lp, CMD_RC_RX);
+	ret = adf7242_cmd_rx(lp);
 
 	skb_trim(skb, len - 2);	/* Don't put RSSI/LQI or CRC into the frame */
 
@@ -870,7 +896,7 @@ static int adf7242_rx(struct adf7242_local *lp)
 	dev_dbg(&lp->spi->dev, "%s: ret=%d len=%d lqi=%d rssi=%d\n",
 		__func__, ret, (int)len, (int)lqi, lp->rssi);
 
-	return 0;
+	return ret;
 }
 
 static const struct ieee802154_ops adf7242_ops = {
@@ -888,7 +914,7 @@ static const struct ieee802154_ops adf7242_ops = {
 	.set_cca_ed_level = adf7242_set_cca_ed_level,
 };
 
-static void adf7242_debug(u8 irq1)
+static void adf7242_debug(struct adf7242_local *lp, u8 irq1)
 {
 #ifdef DEBUG
 	u8 stat;
@@ -906,9 +932,12 @@ static void adf7242_debug(u8 irq1)
 		irq1 & IRQ_FRAME_VALID ? "IRQ_FRAME_VALID\n" : "",
 		irq1 & IRQ_ADDRESS_VALID ? "IRQ_ADDRESS_VALID\n" : "");
 
-	dev_dbg(&lp->spi->dev, "%s STATUS = %X:\n%s\n%s%s%s%s%s\n",
+	dev_dbg(&lp->spi->dev, "%s STATUS = %X:\n%s\n%s\n%s\n%s\n%s%s%s%s%s\n",
 		__func__, stat,
+		stat & STAT_SPI_READY ? "SPI_READY" : "SPI_BUSY",
+		stat & STAT_IRQ_STATUS ? "IRQ_PENDING" : "IRQ_CLEAR",
 		stat & STAT_RC_READY ? "RC_READY" : "RC_BUSY",
+		stat & STAT_CCA_RESULT ? "CHAN_IDLE" : "CHAN_BUSY",
 		(stat & 0xf) == RC_STATUS_IDLE ? "RC_STATUS_IDLE" : "",
 		(stat & 0xf) == RC_STATUS_MEAS ? "RC_STATUS_MEAS" : "",
 		(stat & 0xf) == RC_STATUS_PHY_RDY ? "RC_STATUS_PHY_RDY" : "",
@@ -923,20 +952,20 @@ static irqreturn_t adf7242_isr(int irq, void *data)
 	unsigned int xmit;
 	u8 irq1;
 
-	adf7242_wait_status(lp, RC_STATUS_PHY_RDY, RC_STATUS_MASK, __LINE__);
-
 	adf7242_read_reg(lp, REG_IRQ1_SRC1, &irq1);
-	adf7242_write_reg(lp, REG_IRQ1_SRC1, irq1);
 
 	if (!(irq1 & (IRQ_RX_PKT_RCVD | IRQ_CSMA_CA)))
 		dev_err(&lp->spi->dev, "%s :ERROR IRQ1 = 0x%X\n",
 			__func__, irq1);
 
-	adf7242_debug(irq1);
+	adf7242_debug(lp, irq1);
 
 	xmit = test_bit(FLAG_XMIT, &lp->flags);
 
 	if (xmit && (irq1 & IRQ_CSMA_CA)) {
+		adf7242_wait_status(lp, RC_STATUS_PHY_RDY,
+				    RC_STATUS_MASK, __LINE__);
+
 		if (ADF7242_REPORT_CSMA_CA_STAT) {
 			u8 astat;
 
@@ -957,6 +986,7 @@ static irqreturn_t adf7242_isr(int irq, void *data)
 			lp->tx_stat = SUCCESS;
 		}
 		complete(&lp->tx_complete);
+		adf7242_clear_irqstat(lp);
 	} else if (!xmit && (irq1 & IRQ_RX_PKT_RCVD) &&
 		   (irq1 & IRQ_FRAME_VALID)) {
 		adf7242_rx(lp);
@@ -965,16 +995,19 @@ static irqreturn_t adf7242_isr(int irq, void *data)
 		dev_dbg(&lp->spi->dev, "%s:%d : ERROR IRQ1 = 0x%X\n",
 			__func__, __LINE__, irq1);
 		adf7242_cmd(lp, CMD_RC_PHY_RDY);
-		adf7242_write_reg(lp, REG_IRQ1_SRC1, 0xFF);
-		adf7242_cmd(lp, CMD_RC_RX);
+		adf7242_cmd_rx(lp);
 	} else {
 		/* This can only be xmit without IRQ, likely a RX packet.
 		 * we get an TX IRQ shortly - do nothing or let the xmit
 		 * timeout handle this
 		 */
+
 		dev_dbg(&lp->spi->dev, "%s:%d : ERROR IRQ1 = 0x%X, xmit %d\n",
 			__func__, __LINE__, irq1, xmit);
+		adf7242_wait_status(lp, RC_STATUS_PHY_RDY,
+				    RC_STATUS_MASK, __LINE__);
 		complete(&lp->tx_complete);
+		adf7242_clear_irqstat(lp);
 	}
 
 	return IRQ_HANDLED;
@@ -994,7 +1027,7 @@ static int adf7242_soft_reset(struct adf7242_local *lp, int line)
 	adf7242_set_promiscuous_mode(lp->hw, lp->promiscuous);
 	adf7242_set_csma_params(lp->hw, lp->min_be, lp->max_be,
 				lp->max_cca_retries);
-	adf7242_write_reg(lp, REG_IRQ1_SRC1, 0xFF);
+	adf7242_clear_irqstat(lp);
 
 	if (test_bit(FLAG_START, &lp->flags)) {
 		enable_irq(lp->spi->irq);
@@ -1060,7 +1093,7 @@ static int adf7242_hw_init(struct adf7242_local *lp)
 	adf7242_write_reg(lp, REG_IRQ1_EN0, 0);
 	adf7242_write_reg(lp, REG_IRQ1_EN1, IRQ_RX_PKT_RCVD | IRQ_CSMA_CA);
 
-	adf7242_write_reg(lp, REG_IRQ1_SRC1, 0xFF);
+	adf7242_clear_irqstat(lp);
 	adf7242_write_reg(lp, REG_IRQ1_SRC0, 0xFF);
 
 	adf7242_cmd(lp, CMD_RC_IDLE);
@@ -1086,8 +1119,11 @@ static int adf7242_stats_show(struct seq_file *file, void *offset)
 		   irq1 & IRQ_FRAME_VALID ? "IRQ_FRAME_VALID\n" : "",
 		   irq1 & IRQ_ADDRESS_VALID ? "IRQ_ADDRESS_VALID\n" : "");
 
-	seq_printf(file, "STATUS = %X:\n%s\n%s%s%s%s%s\n", stat,
+	seq_printf(file, "STATUS = %X:\n%s\n%s\n%s\n%s\n%s%s%s%s%s\n", stat,
+		   stat & STAT_SPI_READY ? "SPI_READY" : "SPI_BUSY",
+		   stat & STAT_IRQ_STATUS ? "IRQ_PENDING" : "IRQ_CLEAR",
 		   stat & STAT_RC_READY ? "RC_READY" : "RC_BUSY",
+		   stat & STAT_CCA_RESULT ? "CHAN_IDLE" : "CHAN_BUSY",
 		   (stat & 0xf) == RC_STATUS_IDLE ? "RC_STATUS_IDLE" : "",
 		   (stat & 0xf) == RC_STATUS_MEAS ? "RC_STATUS_MEAS" : "",
 		   (stat & 0xf) == RC_STATUS_PHY_RDY ? "RC_STATUS_PHY_RDY" : "",
@@ -1257,12 +1293,14 @@ static int adf7242_remove(struct spi_device *spi)
 
 static const struct of_device_id adf7242_of_match[] = {
 	{ .compatible = "adi,adf7242", },
+	{ .compatible = "adi,adf7241", },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, adf7242_of_match);
 
 static const struct spi_device_id adf7242_device_id[] = {
 	{ .name = "adf7242", },
+	{ .name = "adf7241", },
 	{ },
 };
 MODULE_DEVICE_TABLE(spi, adf7242_device_id);
diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c
index 7900ed066d8a..377af43b81b3 100644
--- a/drivers/net/ieee802154/ca8210.c
+++ b/drivers/net/ieee802154/ca8210.c
@@ -2638,21 +2638,21 @@ static long ca8210_test_int_ioctl(
  *
  * Return: set of poll return flags
  */
-static unsigned int ca8210_test_int_poll(
+static __poll_t ca8210_test_int_poll(
 	struct file *filp,
 	struct poll_table_struct *ptable
 )
 {
-	unsigned int return_flags = 0;
+	__poll_t return_flags = 0;
 	struct ca8210_priv *priv = filp->private_data;
 
 	poll_wait(filp, &priv->test.readq, ptable);
 	if (!kfifo_is_empty(&priv->test.up_fifo))
-		return_flags |= (POLLIN | POLLRDNORM);
+		return_flags |= (EPOLLIN | EPOLLRDNORM);
 	if (wait_event_interruptible(
 		priv->test.readq,
 		!kfifo_is_empty(&priv->test.up_fifo))) {
-		return POLLERR;
+		return EPOLLERR;
 	}
 	return return_flags;
 }
diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c
index 77cc4fbaeace..c1f008fe4e1d 100644
--- a/drivers/net/ipvlan/ipvlan_core.c
+++ b/drivers/net/ipvlan/ipvlan_core.c
@@ -315,13 +315,13 @@ static int ipvlan_rcv_frame(struct ipvl_addr *addr, struct sk_buff **pskb,
 
 		*pskb = skb;
 	}
-	ipvlan_skb_crossing_ns(skb, dev);
 
 	if (local) {
 		skb->pkt_type = PACKET_HOST;
 		if (dev_forward_skb(ipvlan->dev, skb) == NET_RX_SUCCESS)
 			success = true;
 	} else {
+		skb->dev = dev;
 		ret = RX_HANDLER_ANOTHER;
 		success = true;
 	}
@@ -586,7 +586,7 @@ static int ipvlan_xmit_mode_l2(struct sk_buff *skb, struct net_device *dev)
 		return NET_XMIT_SUCCESS;
 	}
 
-	ipvlan_skb_crossing_ns(skb, ipvlan->phy_dev);
+	skb->dev = ipvlan->phy_dev;
 	return dev_queue_xmit(skb);
 }
 
@@ -664,8 +664,6 @@ static rx_handler_result_t ipvlan_handle_mode_l2(struct sk_buff **pskb,
 	struct sk_buff *skb = *pskb;
 	struct ethhdr *eth = eth_hdr(skb);
 	rx_handler_result_t ret = RX_HANDLER_PASS;
-	void *lyr3h;
-	int addr_type;
 
 	if (is_multicast_ether_addr(eth->h_dest)) {
 		if (ipvlan_external_frame(skb, port)) {
@@ -683,15 +681,8 @@ static rx_handler_result_t ipvlan_handle_mode_l2(struct sk_buff **pskb,
 			}
 		}
 	} else {
-		struct ipvl_addr *addr;
-
-		lyr3h = ipvlan_get_L3_hdr(port, skb, &addr_type);
-		if (!lyr3h)
-			return ret;
-
-		addr = ipvlan_addr_lookup(port, lyr3h, addr_type, true);
-		if (addr)
-			ret = ipvlan_rcv_frame(addr, pskb, false);
+		/* Perform like l3 mode for non-multicast packet */
+		ret = ipvlan_handle_mode_l3(pskb, port);
 	}
 
 	return ret;
diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c
index 30cb803e2fe5..2469df118fbf 100644
--- a/drivers/net/ipvlan/ipvlan_main.c
+++ b/drivers/net/ipvlan/ipvlan_main.c
@@ -850,6 +850,19 @@ static void ipvlan_del_addr6(struct ipvl_dev *ipvlan, struct in6_addr *ip6_addr)
 	return ipvlan_del_addr(ipvlan, ip6_addr, true);
 }
 
+static bool ipvlan_is_valid_dev(const struct net_device *dev)
+{
+	struct ipvl_dev *ipvlan = netdev_priv(dev);
+
+	if (!netif_is_ipvlan(dev))
+		return false;
+
+	if (!ipvlan || !ipvlan->port)
+		return false;
+
+	return true;
+}
+
 static int ipvlan_addr6_event(struct notifier_block *unused,
 			      unsigned long event, void *ptr)
 {
@@ -857,10 +870,7 @@ static int ipvlan_addr6_event(struct notifier_block *unused,
 	struct net_device *dev = (struct net_device *)if6->idev->dev;
 	struct ipvl_dev *ipvlan = netdev_priv(dev);
 
-	if (!netif_is_ipvlan(dev))
-		return NOTIFY_DONE;
-
-	if (!ipvlan || !ipvlan->port)
+	if (!ipvlan_is_valid_dev(dev))
 		return NOTIFY_DONE;
 
 	switch (event) {
@@ -888,10 +898,7 @@ static int ipvlan_addr6_validator_event(struct notifier_block *unused,
 	if (in_softirq())
 		return NOTIFY_DONE;
 
-	if (!netif_is_ipvlan(dev))
-		return NOTIFY_DONE;
-
-	if (!ipvlan || !ipvlan->port)
+	if (!ipvlan_is_valid_dev(dev))
 		return NOTIFY_DONE;
 
 	switch (event) {
@@ -932,10 +939,7 @@ static int ipvlan_addr4_event(struct notifier_block *unused,
 	struct ipvl_dev *ipvlan = netdev_priv(dev);
 	struct in_addr ip4_addr;
 
-	if (!netif_is_ipvlan(dev))
-		return NOTIFY_DONE;
-
-	if (!ipvlan || !ipvlan->port)
+	if (!ipvlan_is_valid_dev(dev))
 		return NOTIFY_DONE;
 
 	switch (event) {
@@ -961,10 +965,7 @@ static int ipvlan_addr4_validator_event(struct notifier_block *unused,
 	struct net_device *dev = (struct net_device *)ivi->ivi_dev->dev;
 	struct ipvl_dev *ipvlan = netdev_priv(dev);
 
-	if (!netif_is_ipvlan(dev))
-		return NOTIFY_DONE;
-
-	if (!ipvlan || !ipvlan->port)
+	if (!ipvlan_is_valid_dev(dev))
 		return NOTIFY_DONE;
 
 	switch (event) {
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index 1d025ab9568f..9cbb0c8a896a 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -393,7 +393,10 @@ static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
 #define MACSEC_PORT_SCB (0x0000)
 #define MACSEC_UNDEF_SCI ((__force sci_t)0xffffffffffffffffULL)
 
-#define DEFAULT_SAK_LEN 16
+#define MACSEC_GCM_AES_128_SAK_LEN 16
+#define MACSEC_GCM_AES_256_SAK_LEN 32
+
+#define DEFAULT_SAK_LEN MACSEC_GCM_AES_128_SAK_LEN
 #define DEFAULT_SEND_SCI true
 #define DEFAULT_ENCRYPT false
 #define DEFAULT_ENCODING_SA 0
@@ -2362,15 +2365,26 @@ static int nla_put_secy(struct macsec_secy *secy, struct sk_buff *skb)
 {
 	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
 	struct nlattr *secy_nest = nla_nest_start(skb, MACSEC_ATTR_SECY);
+	u64 csid;
 
 	if (!secy_nest)
 		return 1;
 
+	switch (secy->key_len) {
+	case MACSEC_GCM_AES_128_SAK_LEN:
+		csid = MACSEC_DEFAULT_CIPHER_ID;
+		break;
+	case MACSEC_GCM_AES_256_SAK_LEN:
+		csid = MACSEC_CIPHER_ID_GCM_AES_256;
+		break;
+	default:
+		goto cancel;
+	}
+
 	if (nla_put_sci(skb, MACSEC_SECY_ATTR_SCI, secy->sci,
 			MACSEC_SECY_ATTR_PAD) ||
 	    nla_put_u64_64bit(skb, MACSEC_SECY_ATTR_CIPHER_SUITE,
-			      MACSEC_DEFAULT_CIPHER_ID,
-			      MACSEC_SECY_ATTR_PAD) ||
+			      csid, MACSEC_SECY_ATTR_PAD) ||
 	    nla_put_u8(skb, MACSEC_SECY_ATTR_ICV_LEN, secy->icv_len) ||
 	    nla_put_u8(skb, MACSEC_SECY_ATTR_OPER, secy->operational) ||
 	    nla_put_u8(skb, MACSEC_SECY_ATTR_PROTECT, secy->protect_frames) ||
@@ -3015,8 +3029,8 @@ static void macsec_setup(struct net_device *dev)
 	eth_zero_addr(dev->broadcast);
 }
 
-static void macsec_changelink_common(struct net_device *dev,
-				     struct nlattr *data[])
+static int macsec_changelink_common(struct net_device *dev,
+				    struct nlattr *data[])
 {
 	struct macsec_secy *secy;
 	struct macsec_tx_sc *tx_sc;
@@ -3056,6 +3070,22 @@ static void macsec_changelink_common(struct net_device *dev,
 
 	if (data[IFLA_MACSEC_VALIDATION])
 		secy->validate_frames = nla_get_u8(data[IFLA_MACSEC_VALIDATION]);
+
+	if (data[IFLA_MACSEC_CIPHER_SUITE]) {
+		switch (nla_get_u64(data[IFLA_MACSEC_CIPHER_SUITE])) {
+		case MACSEC_CIPHER_ID_GCM_AES_128:
+		case MACSEC_DEFAULT_CIPHER_ID:
+			secy->key_len = MACSEC_GCM_AES_128_SAK_LEN;
+			break;
+		case MACSEC_CIPHER_ID_GCM_AES_256:
+			secy->key_len = MACSEC_GCM_AES_256_SAK_LEN;
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	return 0;
 }
 
 static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
@@ -3071,9 +3101,7 @@ static int macsec_changelink(struct net_device *dev, struct nlattr *tb[],
 	    data[IFLA_MACSEC_PORT])
 		return -EINVAL;
 
-	macsec_changelink_common(dev, data);
-
-	return 0;
+	return macsec_changelink_common(dev, data);
 }
 
 static void macsec_del_dev(struct macsec_dev *macsec)
@@ -3249,7 +3277,7 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 
 	err = netdev_upper_dev_link(real_dev, dev, extack);
 	if (err < 0)
-		goto unregister;
+		goto put_dev;
 
 	/* need to be already registered so that ->init has run and
 	 * the MAC addr is set
@@ -3270,8 +3298,11 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 	if (err)
 		goto unlink;
 
-	if (data)
-		macsec_changelink_common(dev, data);
+	if (data) {
+		err = macsec_changelink_common(dev, data);
+		if (err)
+			goto del_dev;
+	}
 
 	err = register_macsec_dev(real_dev, dev);
 	if (err < 0)
@@ -3285,7 +3316,8 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 	macsec_del_dev(macsec);
 unlink:
 	netdev_upper_dev_unlink(real_dev, dev);
-unregister:
+put_dev:
+	dev_put(real_dev);
 	unregister_netdevice(dev);
 	return err;
 }
@@ -3320,8 +3352,9 @@ static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[],
 	}
 
 	switch (csid) {
+	case MACSEC_CIPHER_ID_GCM_AES_128:
+	case MACSEC_CIPHER_ID_GCM_AES_256:
 	case MACSEC_DEFAULT_CIPHER_ID:
-	case MACSEC_DEFAULT_CIPHER_ALT:
 		if (icv_len < MACSEC_MIN_ICV_LEN ||
 		    icv_len > MACSEC_STD_ICV_LEN)
 			return -EINVAL;
@@ -3390,12 +3423,24 @@ static int macsec_fill_info(struct sk_buff *skb,
 {
 	struct macsec_secy *secy = &macsec_priv(dev)->secy;
 	struct macsec_tx_sc *tx_sc = &secy->tx_sc;
+	u64 csid;
+
+	switch (secy->key_len) {
+	case MACSEC_GCM_AES_128_SAK_LEN:
+		csid = MACSEC_DEFAULT_CIPHER_ID;
+		break;
+	case MACSEC_GCM_AES_256_SAK_LEN:
+		csid = MACSEC_CIPHER_ID_GCM_AES_256;
+		break;
+	default:
+		goto nla_put_failure;
+	}
 
 	if (nla_put_sci(skb, IFLA_MACSEC_SCI, secy->sci,
 			IFLA_MACSEC_PAD) ||
 	    nla_put_u8(skb, IFLA_MACSEC_ICV_LEN, secy->icv_len) ||
 	    nla_put_u64_64bit(skb, IFLA_MACSEC_CIPHER_SUITE,
-			      MACSEC_DEFAULT_CIPHER_ID, IFLA_MACSEC_PAD) ||
+			      csid, IFLA_MACSEC_PAD) ||
 	    nla_put_u8(skb, IFLA_MACSEC_ENCODING_SA, tx_sc->encoding_sa) ||
 	    nla_put_u8(skb, IFLA_MACSEC_ENCRYPT, tx_sc->encrypt) ||
 	    nla_put_u8(skb, IFLA_MACSEC_PROTECT, secy->protect_frames) ||
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index a0f2be81d52e..725f4b4afc6d 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -1036,7 +1036,7 @@ static netdev_features_t macvlan_fix_features(struct net_device *dev,
 	lowerdev_features &= (features | ~NETIF_F_LRO);
 	features = netdev_increment_features(lowerdev_features, features, mask);
 	features |= ALWAYS_ON_FEATURES;
-	features &= ~NETIF_F_NETNS_LOCAL;
+	features &= (ALWAYS_ON_FEATURES | MACVLAN_FEATURES);
 
 	return features;
 }
@@ -1451,7 +1451,7 @@ int macvlan_common_newlink(struct net *src_net, struct net_device *dev,
 	/* the macvlan port may be freed by macvlan_uninit when fail to register.
 	 * so we destroy the macvlan port only when it's valid.
 	 */
-	if (create && macvlan_port_get_rtnl(dev))
+	if (create && macvlan_port_get_rtnl(lowerdev))
 		macvlan_port_destroy(port->dev);
 	return err;
 }
diff --git a/drivers/net/netdevsim/Makefile b/drivers/net/netdevsim/Makefile
new file mode 100644
index 000000000000..09388c06171d
--- /dev/null
+++ b/drivers/net/netdevsim/Makefile
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_NETDEVSIM) += netdevsim.o
+
+netdevsim-objs := \
+	netdev.o \
+
+ifeq ($(CONFIG_BPF_SYSCALL),y)
+netdevsim-objs += \
+	bpf.o
+endif
diff --git a/drivers/net/netdevsim/bpf.c b/drivers/net/netdevsim/bpf.c
new file mode 100644
index 000000000000..75c25306d234
--- /dev/null
+++ b/drivers/net/netdevsim/bpf.c
@@ -0,0 +1,642 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree.
+ *
+ * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
+ * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
+ * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
+ * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+ */
+
+#include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
+#include <linux/debugfs.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/rtnetlink.h>
+#include <net/pkt_cls.h>
+
+#include "netdevsim.h"
+
+#define pr_vlog(env, fmt, ...)	\
+	bpf_verifier_log_write(env, "[netdevsim] " fmt, ##__VA_ARGS__)
+
+struct nsim_bpf_bound_prog {
+	struct netdevsim *ns;
+	struct bpf_prog *prog;
+	struct dentry *ddir;
+	const char *state;
+	bool is_loaded;
+	struct list_head l;
+};
+
+#define NSIM_BPF_MAX_KEYS		2
+
+struct nsim_bpf_bound_map {
+	struct netdevsim *ns;
+	struct bpf_offloaded_map *map;
+	struct mutex mutex;
+	struct nsim_map_entry {
+		void *key;
+		void *value;
+	} entry[NSIM_BPF_MAX_KEYS];
+	struct list_head l;
+};
+
+static int nsim_debugfs_bpf_string_read(struct seq_file *file, void *data)
+{
+	const char **str = file->private;
+
+	if (*str)
+		seq_printf(file, "%s\n", *str);
+
+	return 0;
+}
+
+static int nsim_debugfs_bpf_string_open(struct inode *inode, struct file *f)
+{
+	return single_open(f, nsim_debugfs_bpf_string_read, inode->i_private);
+}
+
+static const struct file_operations nsim_bpf_string_fops = {
+	.owner = THIS_MODULE,
+	.open = nsim_debugfs_bpf_string_open,
+	.release = single_release,
+	.read = seq_read,
+	.llseek = seq_lseek
+};
+
+static int
+nsim_bpf_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn)
+{
+	struct nsim_bpf_bound_prog *state;
+
+	state = env->prog->aux->offload->dev_priv;
+	if (state->ns->bpf_bind_verifier_delay && !insn_idx)
+		msleep(state->ns->bpf_bind_verifier_delay);
+
+	if (insn_idx == env->prog->len - 1)
+		pr_vlog(env, "Hello from netdevsim!\n");
+
+	return 0;
+}
+
+static const struct bpf_prog_offload_ops nsim_bpf_analyzer_ops = {
+	.insn_hook = nsim_bpf_verify_insn,
+};
+
+static bool nsim_xdp_offload_active(struct netdevsim *ns)
+{
+	return ns->xdp_prog_mode == XDP_ATTACHED_HW;
+}
+
+static void nsim_prog_set_loaded(struct bpf_prog *prog, bool loaded)
+{
+	struct nsim_bpf_bound_prog *state;
+
+	if (!prog || !prog->aux->offload)
+		return;
+
+	state = prog->aux->offload->dev_priv;
+	state->is_loaded = loaded;
+}
+
+static int
+nsim_bpf_offload(struct netdevsim *ns, struct bpf_prog *prog, bool oldprog)
+{
+	nsim_prog_set_loaded(ns->bpf_offloaded, false);
+
+	WARN(!!ns->bpf_offloaded != oldprog,
+	     "bad offload state, expected offload %sto be active",
+	     oldprog ? "" : "not ");
+	ns->bpf_offloaded = prog;
+	ns->bpf_offloaded_id = prog ? prog->aux->id : 0;
+	nsim_prog_set_loaded(prog, true);
+
+	return 0;
+}
+
+int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
+			       void *type_data, void *cb_priv)
+{
+	struct tc_cls_bpf_offload *cls_bpf = type_data;
+	struct bpf_prog *prog = cls_bpf->prog;
+	struct netdevsim *ns = cb_priv;
+	struct bpf_prog *oldprog;
+
+	if (type != TC_SETUP_CLSBPF) {
+		NSIM_EA(cls_bpf->common.extack,
+			"only offload of BPF classifiers supported");
+		return -EOPNOTSUPP;
+	}
+
+	if (!tc_cls_can_offload_and_chain0(ns->netdev, &cls_bpf->common))
+		return -EOPNOTSUPP;
+
+	if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
+		NSIM_EA(cls_bpf->common.extack,
+			"only ETH_P_ALL supported as filter protocol");
+		return -EOPNOTSUPP;
+	}
+
+	if (!ns->bpf_tc_accept) {
+		NSIM_EA(cls_bpf->common.extack,
+			"netdevsim configured to reject BPF TC offload");
+		return -EOPNOTSUPP;
+	}
+	/* Note: progs without skip_sw will probably not be dev bound */
+	if (prog && !prog->aux->offload && !ns->bpf_tc_non_bound_accept) {
+		NSIM_EA(cls_bpf->common.extack,
+			"netdevsim configured to reject unbound programs");
+		return -EOPNOTSUPP;
+	}
+
+	if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
+		return -EOPNOTSUPP;
+
+	oldprog = cls_bpf->oldprog;
+
+	/* Don't remove if oldprog doesn't match driver's state */
+	if (ns->bpf_offloaded != oldprog) {
+		oldprog = NULL;
+		if (!cls_bpf->prog)
+			return 0;
+		if (ns->bpf_offloaded) {
+			NSIM_EA(cls_bpf->common.extack,
+				"driver and netdev offload states mismatch");
+			return -EBUSY;
+		}
+	}
+
+	return nsim_bpf_offload(ns, cls_bpf->prog, oldprog);
+}
+
+int nsim_bpf_disable_tc(struct netdevsim *ns)
+{
+	if (ns->bpf_offloaded && !nsim_xdp_offload_active(ns))
+		return -EBUSY;
+	return 0;
+}
+
+static int nsim_xdp_offload_prog(struct netdevsim *ns, struct netdev_bpf *bpf)
+{
+	if (!nsim_xdp_offload_active(ns) && !bpf->prog)
+		return 0;
+	if (!nsim_xdp_offload_active(ns) && bpf->prog && ns->bpf_offloaded) {
+		NSIM_EA(bpf->extack, "TC program is already loaded");
+		return -EBUSY;
+	}
+
+	return nsim_bpf_offload(ns, bpf->prog, nsim_xdp_offload_active(ns));
+}
+
+static int nsim_xdp_set_prog(struct netdevsim *ns, struct netdev_bpf *bpf)
+{
+	int err;
+
+	if (ns->xdp_prog && (bpf->flags ^ ns->xdp_flags) & XDP_FLAGS_MODES) {
+		NSIM_EA(bpf->extack, "program loaded with different flags");
+		return -EBUSY;
+	}
+
+	if (bpf->command == XDP_SETUP_PROG && !ns->bpf_xdpdrv_accept) {
+		NSIM_EA(bpf->extack, "driver XDP disabled in DebugFS");
+		return -EOPNOTSUPP;
+	}
+	if (bpf->command == XDP_SETUP_PROG_HW && !ns->bpf_xdpoffload_accept) {
+		NSIM_EA(bpf->extack, "XDP offload disabled in DebugFS");
+		return -EOPNOTSUPP;
+	}
+
+	if (bpf->command == XDP_SETUP_PROG_HW) {
+		err = nsim_xdp_offload_prog(ns, bpf);
+		if (err)
+			return err;
+	}
+
+	if (ns->xdp_prog)
+		bpf_prog_put(ns->xdp_prog);
+
+	ns->xdp_prog = bpf->prog;
+	ns->xdp_flags = bpf->flags;
+
+	if (!bpf->prog)
+		ns->xdp_prog_mode = XDP_ATTACHED_NONE;
+	else if (bpf->command == XDP_SETUP_PROG)
+		ns->xdp_prog_mode = XDP_ATTACHED_DRV;
+	else
+		ns->xdp_prog_mode = XDP_ATTACHED_HW;
+
+	return 0;
+}
+
+static int nsim_bpf_create_prog(struct netdevsim *ns, struct bpf_prog *prog)
+{
+	struct nsim_bpf_bound_prog *state;
+	char name[16];
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return -ENOMEM;
+
+	state->ns = ns;
+	state->prog = prog;
+	state->state = "verify";
+
+	/* Program id is not populated yet when we create the state. */
+	sprintf(name, "%u", ns->prog_id_gen++);
+	state->ddir = debugfs_create_dir(name, ns->ddir_bpf_bound_progs);
+	if (IS_ERR_OR_NULL(state->ddir)) {
+		kfree(state);
+		return -ENOMEM;
+	}
+
+	debugfs_create_u32("id", 0400, state->ddir, &prog->aux->id);
+	debugfs_create_file("state", 0400, state->ddir,
+			    &state->state, &nsim_bpf_string_fops);
+	debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);
+
+	list_add_tail(&state->l, &ns->bpf_bound_progs);
+
+	prog->aux->offload->dev_priv = state;
+
+	return 0;
+}
+
+static void nsim_bpf_destroy_prog(struct bpf_prog *prog)
+{
+	struct nsim_bpf_bound_prog *state;
+
+	state = prog->aux->offload->dev_priv;
+	WARN(state->is_loaded,
+	     "offload state destroyed while program still bound");
+	debugfs_remove_recursive(state->ddir);
+	list_del(&state->l);
+	kfree(state);
+}
+
+static int nsim_setup_prog_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
+{
+	if (bpf->prog && bpf->prog->aux->offload) {
+		NSIM_EA(bpf->extack, "attempt to load offloaded prog to drv");
+		return -EINVAL;
+	}
+	if (ns->netdev->mtu > NSIM_XDP_MAX_MTU) {
+		NSIM_EA(bpf->extack, "MTU too large w/ XDP enabled");
+		return -EINVAL;
+	}
+	if (nsim_xdp_offload_active(ns)) {
+		NSIM_EA(bpf->extack, "xdp offload active, can't load drv prog");
+		return -EBUSY;
+	}
+	return 0;
+}
+
+static int
+nsim_setup_prog_hw_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
+{
+	struct nsim_bpf_bound_prog *state;
+
+	if (!bpf->prog)
+		return 0;
+
+	if (!bpf->prog->aux->offload) {
+		NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
+		return -EINVAL;
+	}
+	if (bpf->prog->aux->offload->netdev != ns->netdev) {
+		NSIM_EA(bpf->extack, "program bound to different dev");
+		return -EINVAL;
+	}
+
+	state = bpf->prog->aux->offload->dev_priv;
+	if (WARN_ON(strcmp(state->state, "xlated"))) {
+		NSIM_EA(bpf->extack, "offloading program in bad state");
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static bool
+nsim_map_key_match(struct bpf_map *map, struct nsim_map_entry *e, void *key)
+{
+	return e->key && !memcmp(key, e->key, map->key_size);
+}
+
+static int nsim_map_key_find(struct bpf_offloaded_map *offmap, void *key)
+{
+	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(nmap->entry); i++)
+		if (nsim_map_key_match(&offmap->map, &nmap->entry[i], key))
+			return i;
+
+	return -ENOENT;
+}
+
+static int
+nsim_map_alloc_elem(struct bpf_offloaded_map *offmap, unsigned int idx)
+{
+	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
+
+	nmap->entry[idx].key = kmalloc(offmap->map.key_size, GFP_USER);
+	if (!nmap->entry[idx].key)
+		return -ENOMEM;
+	nmap->entry[idx].value = kmalloc(offmap->map.value_size, GFP_USER);
+	if (!nmap->entry[idx].value) {
+		kfree(nmap->entry[idx].key);
+		nmap->entry[idx].key = NULL;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int
+nsim_map_get_next_key(struct bpf_offloaded_map *offmap,
+		      void *key, void *next_key)
+{
+	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
+	int idx = -ENOENT;
+
+	mutex_lock(&nmap->mutex);
+
+	if (key)
+		idx = nsim_map_key_find(offmap, key);
+	if (idx == -ENOENT)
+		idx = 0;
+	else
+		idx++;
+
+	for (; idx < ARRAY_SIZE(nmap->entry); idx++) {
+		if (nmap->entry[idx].key) {
+			memcpy(next_key, nmap->entry[idx].key,
+			       offmap->map.key_size);
+			break;
+		}
+	}
+
+	mutex_unlock(&nmap->mutex);
+
+	if (idx == ARRAY_SIZE(nmap->entry))
+		return -ENOENT;
+	return 0;
+}
+
+static int
+nsim_map_lookup_elem(struct bpf_offloaded_map *offmap, void *key, void *value)
+{
+	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
+	int idx;
+
+	mutex_lock(&nmap->mutex);
+
+	idx = nsim_map_key_find(offmap, key);
+	if (idx >= 0)
+		memcpy(value, nmap->entry[idx].value, offmap->map.value_size);
+
+	mutex_unlock(&nmap->mutex);
+
+	return idx < 0 ? idx : 0;
+}
+
+static int
+nsim_map_update_elem(struct bpf_offloaded_map *offmap,
+		     void *key, void *value, u64 flags)
+{
+	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
+	int idx, err = 0;
+
+	mutex_lock(&nmap->mutex);
+
+	idx = nsim_map_key_find(offmap, key);
+	if (idx < 0 && flags == BPF_EXIST) {
+		err = idx;
+		goto exit_unlock;
+	}
+	if (idx >= 0 && flags == BPF_NOEXIST) {
+		err = -EEXIST;
+		goto exit_unlock;
+	}
+
+	if (idx < 0) {
+		for (idx = 0; idx < ARRAY_SIZE(nmap->entry); idx++)
+			if (!nmap->entry[idx].key)
+				break;
+		if (idx == ARRAY_SIZE(nmap->entry)) {
+			err = -E2BIG;
+			goto exit_unlock;
+		}
+
+		err = nsim_map_alloc_elem(offmap, idx);
+		if (err)
+			goto exit_unlock;
+	}
+
+	memcpy(nmap->entry[idx].key, key, offmap->map.key_size);
+	memcpy(nmap->entry[idx].value, value, offmap->map.value_size);
+exit_unlock:
+	mutex_unlock(&nmap->mutex);
+
+	return err;
+}
+
+static int nsim_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
+{
+	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
+	int idx;
+
+	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
+		return -EINVAL;
+
+	mutex_lock(&nmap->mutex);
+
+	idx = nsim_map_key_find(offmap, key);
+	if (idx >= 0) {
+		kfree(nmap->entry[idx].key);
+		kfree(nmap->entry[idx].value);
+		memset(&nmap->entry[idx], 0, sizeof(nmap->entry[idx]));
+	}
+
+	mutex_unlock(&nmap->mutex);
+
+	return idx < 0 ? idx : 0;
+}
+
+static const struct bpf_map_dev_ops nsim_bpf_map_ops = {
+	.map_get_next_key	= nsim_map_get_next_key,
+	.map_lookup_elem	= nsim_map_lookup_elem,
+	.map_update_elem	= nsim_map_update_elem,
+	.map_delete_elem	= nsim_map_delete_elem,
+};
+
+static int
+nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
+{
+	struct nsim_bpf_bound_map *nmap;
+	int i, err;
+
+	if (WARN_ON(offmap->map.map_type != BPF_MAP_TYPE_ARRAY &&
+		    offmap->map.map_type != BPF_MAP_TYPE_HASH))
+		return -EINVAL;
+	if (offmap->map.max_entries > NSIM_BPF_MAX_KEYS)
+		return -ENOMEM;
+	if (offmap->map.map_flags)
+		return -EINVAL;
+
+	nmap = kzalloc(sizeof(*nmap), GFP_USER);
+	if (!nmap)
+		return -ENOMEM;
+
+	offmap->dev_priv = nmap;
+	nmap->ns = ns;
+	nmap->map = offmap;
+	mutex_init(&nmap->mutex);
+
+	if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) {
+		for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
+			u32 *key;
+
+			err = nsim_map_alloc_elem(offmap, i);
+			if (err)
+				goto err_free;
+			key = nmap->entry[i].key;
+			*key = i;
+		}
+	}
+
+	offmap->dev_ops = &nsim_bpf_map_ops;
+	list_add_tail(&nmap->l, &ns->bpf_bound_maps);
+
+	return 0;
+
+err_free:
+	while (--i >= 0) {
+		kfree(nmap->entry[i].key);
+		kfree(nmap->entry[i].value);
+	}
+	kfree(nmap);
+	return err;
+}
+
+static void nsim_bpf_map_free(struct bpf_offloaded_map *offmap)
+{
+	struct nsim_bpf_bound_map *nmap = offmap->dev_priv;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(nmap->entry); i++) {
+		kfree(nmap->entry[i].key);
+		kfree(nmap->entry[i].value);
+	}
+	list_del_init(&nmap->l);
+	mutex_destroy(&nmap->mutex);
+	kfree(nmap);
+}
+
+int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+	struct nsim_bpf_bound_prog *state;
+	int err;
+
+	ASSERT_RTNL();
+
+	switch (bpf->command) {
+	case BPF_OFFLOAD_VERIFIER_PREP:
+		if (!ns->bpf_bind_accept)
+			return -EOPNOTSUPP;
+
+		err = nsim_bpf_create_prog(ns, bpf->verifier.prog);
+		if (err)
+			return err;
+
+		bpf->verifier.ops = &nsim_bpf_analyzer_ops;
+		return 0;
+	case BPF_OFFLOAD_TRANSLATE:
+		state = bpf->offload.prog->aux->offload->dev_priv;
+
+		state->state = "xlated";
+		return 0;
+	case BPF_OFFLOAD_DESTROY:
+		nsim_bpf_destroy_prog(bpf->offload.prog);
+		return 0;
+	case XDP_QUERY_PROG:
+		bpf->prog_attached = ns->xdp_prog_mode;
+		bpf->prog_id = ns->xdp_prog ? ns->xdp_prog->aux->id : 0;
+		bpf->prog_flags = ns->xdp_prog ? ns->xdp_flags : 0;
+		return 0;
+	case XDP_SETUP_PROG:
+		err = nsim_setup_prog_checks(ns, bpf);
+		if (err)
+			return err;
+
+		return nsim_xdp_set_prog(ns, bpf);
+	case XDP_SETUP_PROG_HW:
+		err = nsim_setup_prog_hw_checks(ns, bpf);
+		if (err)
+			return err;
+
+		return nsim_xdp_set_prog(ns, bpf);
+	case BPF_OFFLOAD_MAP_ALLOC:
+		if (!ns->bpf_map_accept)
+			return -EOPNOTSUPP;
+
+		return nsim_bpf_map_alloc(ns, bpf->offmap);
+	case BPF_OFFLOAD_MAP_FREE:
+		nsim_bpf_map_free(bpf->offmap);
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+int nsim_bpf_init(struct netdevsim *ns)
+{
+	INIT_LIST_HEAD(&ns->bpf_bound_progs);
+	INIT_LIST_HEAD(&ns->bpf_bound_maps);
+
+	debugfs_create_u32("bpf_offloaded_id", 0400, ns->ddir,
+			   &ns->bpf_offloaded_id);
+
+	ns->bpf_bind_accept = true;
+	debugfs_create_bool("bpf_bind_accept", 0600, ns->ddir,
+			    &ns->bpf_bind_accept);
+	debugfs_create_u32("bpf_bind_verifier_delay", 0600, ns->ddir,
+			   &ns->bpf_bind_verifier_delay);
+	ns->ddir_bpf_bound_progs =
+		debugfs_create_dir("bpf_bound_progs", ns->ddir);
+	if (IS_ERR_OR_NULL(ns->ddir_bpf_bound_progs))
+		return -ENOMEM;
+
+	ns->bpf_tc_accept = true;
+	debugfs_create_bool("bpf_tc_accept", 0600, ns->ddir,
+			    &ns->bpf_tc_accept);
+	debugfs_create_bool("bpf_tc_non_bound_accept", 0600, ns->ddir,
+			    &ns->bpf_tc_non_bound_accept);
+	ns->bpf_xdpdrv_accept = true;
+	debugfs_create_bool("bpf_xdpdrv_accept", 0600, ns->ddir,
+			    &ns->bpf_xdpdrv_accept);
+	ns->bpf_xdpoffload_accept = true;
+	debugfs_create_bool("bpf_xdpoffload_accept", 0600, ns->ddir,
+			    &ns->bpf_xdpoffload_accept);
+
+	ns->bpf_map_accept = true;
+	debugfs_create_bool("bpf_map_accept", 0600, ns->ddir,
+			    &ns->bpf_map_accept);
+
+	return 0;
+}
+
+void nsim_bpf_uninit(struct netdevsim *ns)
+{
+	WARN_ON(!list_empty(&ns->bpf_bound_progs));
+	WARN_ON(!list_empty(&ns->bpf_bound_maps));
+	WARN_ON(ns->xdp_prog);
+	WARN_ON(ns->bpf_offloaded);
+}
diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c
new file mode 100644
index 000000000000..3fd567928f3d
--- /dev/null
+++ b/drivers/net/netdevsim/netdev.c
@@ -0,0 +1,504 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree.
+ *
+ * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
+ * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
+ * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
+ * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+ */
+
+#include <linux/debugfs.h>
+#include <linux/etherdevice.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/slab.h>
+#include <net/netlink.h>
+#include <net/pkt_cls.h>
+#include <net/rtnetlink.h>
+
+#include "netdevsim.h"
+
+struct nsim_vf_config {
+	int link_state;
+	u16 min_tx_rate;
+	u16 max_tx_rate;
+	u16 vlan;
+	__be16 vlan_proto;
+	u16 qos;
+	u8 vf_mac[ETH_ALEN];
+	bool spoofchk_enabled;
+	bool trusted;
+	bool rss_query_enabled;
+};
+
+static u32 nsim_dev_id;
+
+static int nsim_num_vf(struct device *dev)
+{
+	struct netdevsim *ns = to_nsim(dev);
+
+	return ns->num_vfs;
+}
+
+static struct bus_type nsim_bus = {
+	.name		= DRV_NAME,
+	.dev_name	= DRV_NAME,
+	.num_vf		= nsim_num_vf,
+};
+
+static int nsim_vfs_enable(struct netdevsim *ns, unsigned int num_vfs)
+{
+	ns->vfconfigs = kcalloc(num_vfs, sizeof(struct nsim_vf_config),
+				GFP_KERNEL);
+	if (!ns->vfconfigs)
+		return -ENOMEM;
+	ns->num_vfs = num_vfs;
+
+	return 0;
+}
+
+static void nsim_vfs_disable(struct netdevsim *ns)
+{
+	kfree(ns->vfconfigs);
+	ns->vfconfigs = NULL;
+	ns->num_vfs = 0;
+}
+
+static ssize_t
+nsim_numvfs_store(struct device *dev, struct device_attribute *attr,
+		  const char *buf, size_t count)
+{
+	struct netdevsim *ns = to_nsim(dev);
+	unsigned int num_vfs;
+	int ret;
+
+	ret = kstrtouint(buf, 0, &num_vfs);
+	if (ret)
+		return ret;
+
+	rtnl_lock();
+	if (ns->num_vfs == num_vfs)
+		goto exit_good;
+	if (ns->num_vfs && num_vfs) {
+		ret = -EBUSY;
+		goto exit_unlock;
+	}
+
+	if (num_vfs) {
+		ret = nsim_vfs_enable(ns, num_vfs);
+		if (ret)
+			goto exit_unlock;
+	} else {
+		nsim_vfs_disable(ns);
+	}
+exit_good:
+	ret = count;
+exit_unlock:
+	rtnl_unlock();
+
+	return ret;
+}
+
+static ssize_t
+nsim_numvfs_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct netdevsim *ns = to_nsim(dev);
+
+	return sprintf(buf, "%u\n", ns->num_vfs);
+}
+
+static struct device_attribute nsim_numvfs_attr =
+	__ATTR(sriov_numvfs, 0664, nsim_numvfs_show, nsim_numvfs_store);
+
+static struct attribute *nsim_dev_attrs[] = {
+	&nsim_numvfs_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group nsim_dev_attr_group = {
+	.attrs = nsim_dev_attrs,
+};
+
+static const struct attribute_group *nsim_dev_attr_groups[] = {
+	&nsim_dev_attr_group,
+	NULL,
+};
+
+static void nsim_dev_release(struct device *dev)
+{
+	struct netdevsim *ns = to_nsim(dev);
+
+	nsim_vfs_disable(ns);
+	free_netdev(ns->netdev);
+}
+
+static struct device_type nsim_dev_type = {
+	.groups = nsim_dev_attr_groups,
+	.release = nsim_dev_release,
+};
+
+static int nsim_init(struct net_device *dev)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+	int err;
+
+	ns->netdev = dev;
+	ns->ddir = debugfs_create_dir(netdev_name(dev), nsim_ddir);
+	if (IS_ERR_OR_NULL(ns->ddir))
+		return -ENOMEM;
+
+	err = nsim_bpf_init(ns);
+	if (err)
+		goto err_debugfs_destroy;
+
+	ns->dev.id = nsim_dev_id++;
+	ns->dev.bus = &nsim_bus;
+	ns->dev.type = &nsim_dev_type;
+	err = device_register(&ns->dev);
+	if (err)
+		goto err_bpf_uninit;
+
+	SET_NETDEV_DEV(dev, &ns->dev);
+
+	return 0;
+
+err_bpf_uninit:
+	nsim_bpf_uninit(ns);
+err_debugfs_destroy:
+	debugfs_remove_recursive(ns->ddir);
+	return err;
+}
+
+static void nsim_uninit(struct net_device *dev)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	debugfs_remove_recursive(ns->ddir);
+	nsim_bpf_uninit(ns);
+}
+
+static void nsim_free(struct net_device *dev)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	device_unregister(&ns->dev);
+	/* netdev and vf state will be freed out of device_release() */
+}
+
+static netdev_tx_t nsim_start_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	u64_stats_update_begin(&ns->syncp);
+	ns->tx_packets++;
+	ns->tx_bytes += skb->len;
+	u64_stats_update_end(&ns->syncp);
+
+	dev_kfree_skb(skb);
+
+	return NETDEV_TX_OK;
+}
+
+static void nsim_set_rx_mode(struct net_device *dev)
+{
+}
+
+static int nsim_change_mtu(struct net_device *dev, int new_mtu)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (ns->xdp_prog_mode == XDP_ATTACHED_DRV &&
+	    new_mtu > NSIM_XDP_MAX_MTU)
+		return -EBUSY;
+
+	dev->mtu = new_mtu;
+
+	return 0;
+}
+
+static void
+nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+	unsigned int start;
+
+	do {
+		start = u64_stats_fetch_begin(&ns->syncp);
+		stats->tx_bytes = ns->tx_bytes;
+		stats->tx_packets = ns->tx_packets;
+	} while (u64_stats_fetch_retry(&ns->syncp, start));
+}
+
+static int
+nsim_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
+{
+	return nsim_bpf_setup_tc_block_cb(type, type_data, cb_priv);
+}
+
+static int
+nsim_setup_tc_block(struct net_device *dev, struct tc_block_offload *f)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+		return -EOPNOTSUPP;
+
+	switch (f->command) {
+	case TC_BLOCK_BIND:
+		return tcf_block_cb_register(f->block, nsim_setup_tc_block_cb,
+					     ns, ns);
+	case TC_BLOCK_UNBIND:
+		tcf_block_cb_unregister(f->block, nsim_setup_tc_block_cb, ns);
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int nsim_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	/* Only refuse multicast addresses, zero address can mean unset/any. */
+	if (vf >= ns->num_vfs || is_multicast_ether_addr(mac))
+		return -EINVAL;
+	memcpy(ns->vfconfigs[vf].vf_mac, mac, ETH_ALEN);
+
+	return 0;
+}
+
+static int nsim_set_vf_vlan(struct net_device *dev, int vf,
+			    u16 vlan, u8 qos, __be16 vlan_proto)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (vf >= ns->num_vfs || vlan > 4095 || qos > 7)
+		return -EINVAL;
+
+	ns->vfconfigs[vf].vlan = vlan;
+	ns->vfconfigs[vf].qos = qos;
+	ns->vfconfigs[vf].vlan_proto = vlan_proto;
+
+	return 0;
+}
+
+static int nsim_set_vf_rate(struct net_device *dev, int vf, int min, int max)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (vf >= ns->num_vfs)
+		return -EINVAL;
+
+	ns->vfconfigs[vf].min_tx_rate = min;
+	ns->vfconfigs[vf].max_tx_rate = max;
+
+	return 0;
+}
+
+static int nsim_set_vf_spoofchk(struct net_device *dev, int vf, bool val)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (vf >= ns->num_vfs)
+		return -EINVAL;
+	ns->vfconfigs[vf].spoofchk_enabled = val;
+
+	return 0;
+}
+
+static int nsim_set_vf_rss_query_en(struct net_device *dev, int vf, bool val)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (vf >= ns->num_vfs)
+		return -EINVAL;
+	ns->vfconfigs[vf].rss_query_enabled = val;
+
+	return 0;
+}
+
+static int nsim_set_vf_trust(struct net_device *dev, int vf, bool val)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (vf >= ns->num_vfs)
+		return -EINVAL;
+	ns->vfconfigs[vf].trusted = val;
+
+	return 0;
+}
+
+static int
+nsim_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (vf >= ns->num_vfs)
+		return -EINVAL;
+
+	ivi->vf = vf;
+	ivi->linkstate = ns->vfconfigs[vf].link_state;
+	ivi->min_tx_rate = ns->vfconfigs[vf].min_tx_rate;
+	ivi->max_tx_rate = ns->vfconfigs[vf].max_tx_rate;
+	ivi->vlan = ns->vfconfigs[vf].vlan;
+	ivi->vlan_proto = ns->vfconfigs[vf].vlan_proto;
+	ivi->qos = ns->vfconfigs[vf].qos;
+	memcpy(&ivi->mac, ns->vfconfigs[vf].vf_mac, ETH_ALEN);
+	ivi->spoofchk = ns->vfconfigs[vf].spoofchk_enabled;
+	ivi->trusted = ns->vfconfigs[vf].trusted;
+	ivi->rss_query_en = ns->vfconfigs[vf].rss_query_enabled;
+
+	return 0;
+}
+
+static int nsim_set_vf_link_state(struct net_device *dev, int vf, int state)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if (vf >= ns->num_vfs)
+		return -EINVAL;
+
+	switch (state) {
+	case IFLA_VF_LINK_STATE_AUTO:
+	case IFLA_VF_LINK_STATE_ENABLE:
+	case IFLA_VF_LINK_STATE_DISABLE:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ns->vfconfigs[vf].link_state = state;
+
+	return 0;
+}
+
+static int
+nsim_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data)
+{
+	switch (type) {
+	case TC_SETUP_BLOCK:
+		return nsim_setup_tc_block(dev, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int
+nsim_set_features(struct net_device *dev, netdev_features_t features)
+{
+	struct netdevsim *ns = netdev_priv(dev);
+
+	if ((dev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC))
+		return nsim_bpf_disable_tc(ns);
+
+	return 0;
+}
+
+static const struct net_device_ops nsim_netdev_ops = {
+	.ndo_init		= nsim_init,
+	.ndo_uninit		= nsim_uninit,
+	.ndo_start_xmit		= nsim_start_xmit,
+	.ndo_set_rx_mode	= nsim_set_rx_mode,
+	.ndo_set_mac_address	= eth_mac_addr,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_change_mtu		= nsim_change_mtu,
+	.ndo_get_stats64	= nsim_get_stats64,
+	.ndo_set_vf_mac		= nsim_set_vf_mac,
+	.ndo_set_vf_vlan	= nsim_set_vf_vlan,
+	.ndo_set_vf_rate	= nsim_set_vf_rate,
+	.ndo_set_vf_spoofchk	= nsim_set_vf_spoofchk,
+	.ndo_set_vf_trust	= nsim_set_vf_trust,
+	.ndo_get_vf_config	= nsim_get_vf_config,
+	.ndo_set_vf_link_state	= nsim_set_vf_link_state,
+	.ndo_set_vf_rss_query_en = nsim_set_vf_rss_query_en,
+	.ndo_setup_tc		= nsim_setup_tc,
+	.ndo_set_features	= nsim_set_features,
+	.ndo_bpf		= nsim_bpf,
+};
+
+static void nsim_setup(struct net_device *dev)
+{
+	ether_setup(dev);
+	eth_hw_addr_random(dev);
+
+	dev->netdev_ops = &nsim_netdev_ops;
+	dev->priv_destructor = nsim_free;
+
+	dev->tx_queue_len = 0;
+	dev->flags |= IFF_NOARP;
+	dev->flags &= ~IFF_MULTICAST;
+	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE |
+			   IFF_NO_QUEUE;
+	dev->features |= NETIF_F_HIGHDMA |
+			 NETIF_F_SG |
+			 NETIF_F_FRAGLIST |
+			 NETIF_F_HW_CSUM |
+			 NETIF_F_TSO;
+	dev->hw_features |= NETIF_F_HW_TC;
+	dev->max_mtu = ETH_MAX_MTU;
+}
+
+static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
+			 struct netlink_ext_ack *extack)
+{
+	if (tb[IFLA_ADDRESS]) {
+		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)
+			return -EINVAL;
+		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))
+			return -EADDRNOTAVAIL;
+	}
+	return 0;
+}
+
+static struct rtnl_link_ops nsim_link_ops __read_mostly = {
+	.kind		= DRV_NAME,
+	.priv_size	= sizeof(struct netdevsim),
+	.setup		= nsim_setup,
+	.validate	= nsim_validate,
+};
+
+struct dentry *nsim_ddir;
+
+static int __init nsim_module_init(void)
+{
+	int err;
+
+	nsim_ddir = debugfs_create_dir(DRV_NAME, NULL);
+	if (IS_ERR_OR_NULL(nsim_ddir))
+		return -ENOMEM;
+
+	err = bus_register(&nsim_bus);
+	if (err)
+		goto err_debugfs_destroy;
+
+	err = rtnl_link_register(&nsim_link_ops);
+	if (err)
+		goto err_unreg_bus;
+
+	return 0;
+
+err_unreg_bus:
+	bus_unregister(&nsim_bus);
+err_debugfs_destroy:
+	debugfs_remove_recursive(nsim_ddir);
+	return err;
+}
+
+static void __exit nsim_module_exit(void)
+{
+	rtnl_link_unregister(&nsim_link_ops);
+	bus_unregister(&nsim_bus);
+	debugfs_remove_recursive(nsim_ddir);
+}
+
+module_init(nsim_module_init);
+module_exit(nsim_module_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS_RTNL_LINK(DRV_NAME);
diff --git a/drivers/net/netdevsim/netdevsim.h b/drivers/net/netdevsim/netdevsim.h
new file mode 100644
index 000000000000..ea081c10efb8
--- /dev/null
+++ b/drivers/net/netdevsim/netdevsim.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2017 Netronome Systems, Inc.
+ *
+ * This software is licensed under the GNU General License Version 2,
+ * June 1991 as shown in the file COPYING in the top-level directory of this
+ * source tree.
+ *
+ * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
+ * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
+ * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
+ * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+ */
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/netdevice.h>
+#include <linux/u64_stats_sync.h>
+
+#define DRV_NAME	"netdevsim"
+
+#define NSIM_XDP_MAX_MTU	4000
+
+#define NSIM_EA(extack, msg)	NL_SET_ERR_MSG_MOD((extack), msg)
+
+struct bpf_prog;
+struct dentry;
+struct nsim_vf_config;
+
+struct netdevsim {
+	struct net_device *netdev;
+
+	u64 tx_packets;
+	u64 tx_bytes;
+	struct u64_stats_sync syncp;
+
+	struct device dev;
+
+	struct dentry *ddir;
+
+	unsigned int num_vfs;
+	struct nsim_vf_config *vfconfigs;
+
+	struct bpf_prog	*bpf_offloaded;
+	u32 bpf_offloaded_id;
+
+	u32 xdp_flags;
+	int xdp_prog_mode;
+	struct bpf_prog	*xdp_prog;
+
+	u32 prog_id_gen;
+
+	bool bpf_bind_accept;
+	u32 bpf_bind_verifier_delay;
+	struct dentry *ddir_bpf_bound_progs;
+	struct list_head bpf_bound_progs;
+
+	bool bpf_tc_accept;
+	bool bpf_tc_non_bound_accept;
+	bool bpf_xdpdrv_accept;
+	bool bpf_xdpoffload_accept;
+
+	bool bpf_map_accept;
+	struct list_head bpf_bound_maps;
+};
+
+extern struct dentry *nsim_ddir;
+
+#ifdef CONFIG_BPF_SYSCALL
+int nsim_bpf_init(struct netdevsim *ns);
+void nsim_bpf_uninit(struct netdevsim *ns);
+int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf);
+int nsim_bpf_disable_tc(struct netdevsim *ns);
+int nsim_bpf_setup_tc_block_cb(enum tc_setup_type type,
+			       void *type_data, void *cb_priv);
+#else
+static inline int nsim_bpf_init(struct netdevsim *ns)
+{
+	return 0;
+}
+
+static inline void nsim_bpf_uninit(struct netdevsim *ns)
+{
+}
+
+static inline int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
+{
+	return bpf->command == XDP_QUERY_PROG ? 0 : -EOPNOTSUPP;
+}
+
+static inline int nsim_bpf_disable_tc(struct netdevsim *ns)
+{
+	return 0;
+}
+
+static inline int
+nsim_bpf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
+			   void *cb_priv)
+{
+	return -EOPNOTSUPP;
+}
+#endif
+
+static inline struct netdevsim *to_nsim(struct device *ptr)
+{
+	return container_of(ptr, struct netdevsim, dev);
+}
diff --git a/drivers/net/phy/amd.c b/drivers/net/phy/amd.c
index 18141c022b13..6fe5dc9201d0 100644
--- a/drivers/net/phy/amd.c
+++ b/drivers/net/phy/amd.c
@@ -68,8 +68,6 @@ static struct phy_driver am79c_driver[] = { {
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= am79c_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= am79c_ack_interrupt,
 	.config_intr	= am79c_config_intr,
 } };
diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c
index e911e4990b20..411cf1072bae 100644
--- a/drivers/net/phy/at803x.c
+++ b/drivers/net/phy/at803x.c
@@ -71,7 +71,6 @@ MODULE_LICENSE("GPL");
 
 struct at803x_priv {
 	bool phy_reset:1;
-	struct gpio_desc *gpiod_reset;
 };
 
 struct at803x_context {
@@ -216,56 +215,33 @@ static int at803x_suspend(struct phy_device *phydev)
 	int value;
 	int wol_enabled;
 
-	mutex_lock(&phydev->lock);
-
 	value = phy_read(phydev, AT803X_INTR_ENABLE);
 	wol_enabled = value & AT803X_INTR_ENABLE_WOL;
 
-	value = phy_read(phydev, MII_BMCR);
-
 	if (wol_enabled)
-		value |= BMCR_ISOLATE;
+		value = BMCR_ISOLATE;
 	else
-		value |= BMCR_PDOWN;
-
-	phy_write(phydev, MII_BMCR, value);
+		value = BMCR_PDOWN;
 
-	mutex_unlock(&phydev->lock);
+	phy_modify(phydev, MII_BMCR, 0, value);
 
 	return 0;
 }
 
 static int at803x_resume(struct phy_device *phydev)
 {
-	int value;
-
-	value = phy_read(phydev, MII_BMCR);
-	value &= ~(BMCR_PDOWN | BMCR_ISOLATE);
-	phy_write(phydev, MII_BMCR, value);
-
-	return 0;
+	return phy_modify(phydev, MII_BMCR, BMCR_PDOWN | BMCR_ISOLATE, 0);
 }
 
 static int at803x_probe(struct phy_device *phydev)
 {
 	struct device *dev = &phydev->mdio.dev;
 	struct at803x_priv *priv;
-	struct gpio_desc *gpiod_reset;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	if (phydev->drv->phy_id != ATH8030_PHY_ID)
-		goto does_not_require_reset_workaround;
-
-	gpiod_reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(gpiod_reset))
-		return PTR_ERR(gpiod_reset);
-
-	priv->gpiod_reset = gpiod_reset;
-
-does_not_require_reset_workaround:
 	phydev->priv = priv;
 
 	return 0;
@@ -339,14 +315,14 @@ static void at803x_link_change_notify(struct phy_device *phydev)
 	 * cannot recover from by software.
 	 */
 	if (phydev->state == PHY_NOLINK) {
-		if (priv->gpiod_reset && !priv->phy_reset) {
+		if (phydev->mdio.reset && !priv->phy_reset) {
 			struct at803x_context context;
 
 			at803x_context_save(phydev, &context);
 
-			gpiod_set_value(priv->gpiod_reset, 1);
+			phy_device_reset(phydev, 1);
 			msleep(1);
-			gpiod_set_value(priv->gpiod_reset, 0);
+			phy_device_reset(phydev, 0);
 			msleep(1);
 
 			at803x_context_restore(phydev, &context);
@@ -404,8 +380,6 @@ static struct phy_driver at803x_driver[] = {
 	.resume			= at803x_resume,
 	.features		= PHY_GBIT_FEATURES,
 	.flags			= PHY_HAS_INTERRUPT,
-	.config_aneg		= genphy_config_aneg,
-	.read_status		= genphy_read_status,
 	.ack_interrupt		= at803x_ack_interrupt,
 	.config_intr		= at803x_config_intr,
 }, {
@@ -422,8 +396,6 @@ static struct phy_driver at803x_driver[] = {
 	.resume			= at803x_resume,
 	.features		= PHY_BASIC_FEATURES,
 	.flags			= PHY_HAS_INTERRUPT,
-	.config_aneg		= genphy_config_aneg,
-	.read_status		= genphy_read_status,
 	.ack_interrupt		= at803x_ack_interrupt,
 	.config_intr		= at803x_config_intr,
 }, {
@@ -439,8 +411,6 @@ static struct phy_driver at803x_driver[] = {
 	.resume			= at803x_resume,
 	.features		= PHY_GBIT_FEATURES,
 	.flags			= PHY_HAS_INTERRUPT,
-	.config_aneg		= genphy_config_aneg,
-	.read_status		= genphy_read_status,
 	.aneg_done		= at803x_aneg_done,
 	.ack_interrupt		= &at803x_ack_interrupt,
 	.config_intr		= &at803x_config_intr,
diff --git a/drivers/net/phy/bcm-cygnus.c b/drivers/net/phy/bcm-cygnus.c
index 3fe8cc5c177e..6838129839ca 100644
--- a/drivers/net/phy/bcm-cygnus.c
+++ b/drivers/net/phy/bcm-cygnus.c
@@ -136,8 +136,6 @@ static struct phy_driver bcm_cygnus_phy_driver[] = {
 	.name          = "Broadcom Cygnus PHY",
 	.features      = PHY_GBIT_FEATURES,
 	.config_init   = bcm_cygnus_config_init,
-	.config_aneg   = genphy_config_aneg,
-	.read_status   = genphy_read_status,
 	.ack_interrupt = bcm_phy_ack_intr,
 	.config_intr   = bcm_phy_config_intr,
 	.suspend       = genphy_suspend,
diff --git a/drivers/net/phy/bcm-phy-lib.c b/drivers/net/phy/bcm-phy-lib.c
index 171010eb4d9c..5ad130c3da43 100644
--- a/drivers/net/phy/bcm-phy-lib.c
+++ b/drivers/net/phy/bcm-phy-lib.c
@@ -341,8 +341,8 @@ void bcm_phy_get_strings(struct phy_device *phydev, u8 *data)
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(bcm_phy_hw_stats); i++)
-		memcpy(data + i * ETH_GSTRING_LEN,
-		       bcm_phy_hw_stats[i].string, ETH_GSTRING_LEN);
+		strlcpy(data + i * ETH_GSTRING_LEN,
+			bcm_phy_hw_stats[i].string, ETH_GSTRING_LEN);
 }
 EXPORT_SYMBOL_GPL(bcm_phy_get_strings);
 
diff --git a/drivers/net/phy/bcm63xx.c b/drivers/net/phy/bcm63xx.c
index b0492ef2cdaa..cf14613745c9 100644
--- a/drivers/net/phy/bcm63xx.c
+++ b/drivers/net/phy/bcm63xx.c
@@ -69,8 +69,6 @@ static struct phy_driver bcm63xx_driver[] = {
 	.features	= (PHY_BASIC_FEATURES | SUPPORTED_Pause),
 	.flags		= PHY_HAS_INTERRUPT | PHY_IS_INTERNAL,
 	.config_init	= bcm63xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm63xx_config_intr,
 }, {
@@ -81,8 +79,6 @@ static struct phy_driver bcm63xx_driver[] = {
 	.features	= (PHY_BASIC_FEATURES | SUPPORTED_Pause),
 	.flags		= PHY_HAS_INTERRUPT | PHY_IS_INTERNAL,
 	.config_init	= bcm63xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm63xx_config_intr,
 } };
diff --git a/drivers/net/phy/bcm7xxx.c b/drivers/net/phy/bcm7xxx.c
index 8b33f688ac8a..421feb8f92fe 100644
--- a/drivers/net/phy/bcm7xxx.c
+++ b/drivers/net/phy/bcm7xxx.c
@@ -611,8 +611,6 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
 	.features	= PHY_GBIT_FEATURES,				\
 	.flags		= PHY_IS_INTERNAL,				\
 	.config_init	= bcm7xxx_28nm_config_init,			\
-	.config_aneg	= genphy_config_aneg,				\
-	.read_status	= genphy_read_status,				\
 	.resume		= bcm7xxx_28nm_resume,				\
 	.get_tunable	= bcm7xxx_28nm_get_tunable,			\
 	.set_tunable	= bcm7xxx_28nm_set_tunable,			\
@@ -630,8 +628,6 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
 	.features	= PHY_BASIC_FEATURES,				\
 	.flags		= PHY_IS_INTERNAL,				\
 	.config_init	= bcm7xxx_28nm_ephy_config_init,		\
-	.config_aneg	= genphy_config_aneg,				\
-	.read_status	= genphy_read_status,				\
 	.resume		= bcm7xxx_28nm_ephy_resume,			\
 	.get_sset_count	= bcm_phy_get_sset_count,			\
 	.get_strings	= bcm_phy_get_strings,				\
@@ -647,8 +643,6 @@ static int bcm7xxx_28nm_probe(struct phy_device *phydev)
 	.features       = PHY_BASIC_FEATURES,				\
 	.flags          = PHY_IS_INTERNAL,				\
 	.config_init    = bcm7xxx_config_init,				\
-	.config_aneg    = genphy_config_aneg,				\
-	.read_status    = genphy_read_status,				\
 	.suspend        = bcm7xxx_suspend,				\
 	.resume         = bcm7xxx_config_init,				\
 }
diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c
index d7ed69deabfb..3bb6b66dc7bf 100644
--- a/drivers/net/phy/broadcom.c
+++ b/drivers/net/phy/broadcom.c
@@ -540,6 +540,37 @@ static int brcm_fet_config_intr(struct phy_device *phydev)
 	return err;
 }
 
+struct bcm53xx_phy_priv {
+	u64	*stats;
+};
+
+static int bcm53xx_phy_probe(struct phy_device *phydev)
+{
+	struct bcm53xx_phy_priv *priv;
+
+	priv = devm_kzalloc(&phydev->mdio.dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	phydev->priv = priv;
+
+	priv->stats = devm_kcalloc(&phydev->mdio.dev,
+				   bcm_phy_get_sset_count(phydev), sizeof(u64),
+				   GFP_KERNEL);
+	if (!priv->stats)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void bcm53xx_phy_get_stats(struct phy_device *phydev,
+				  struct ethtool_stats *stats, u64 *data)
+{
+	struct bcm53xx_phy_priv *priv = phydev->priv;
+
+	bcm_phy_get_stats(phydev, priv->stats, stats, data);
+}
+
 static struct phy_driver broadcom_drivers[] = {
 {
 	.phy_id		= PHY_ID_BCM5411,
@@ -548,8 +579,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -559,8 +588,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -570,8 +597,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -581,8 +606,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -592,8 +615,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -603,8 +624,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -614,8 +633,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -626,7 +643,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
 	.config_aneg	= bcm5481_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -637,7 +653,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.flags          = PHY_HAS_INTERRUPT,
 	.config_init    = bcm54xx_config_init,
 	.config_aneg    = bcm5481_config_aneg,
-	.read_status    = genphy_read_status,
 	.ack_interrupt  = bcm_phy_ack_intr,
 	.config_intr    = bcm_phy_config_intr,
 }, {
@@ -647,7 +662,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm5482_config_init,
-	.config_aneg	= genphy_config_aneg,
 	.read_status	= bcm5482_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
@@ -658,8 +672,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -669,8 +681,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -680,8 +690,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= bcm54xx_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= bcm_phy_ack_intr,
 	.config_intr	= bcm_phy_config_intr,
 }, {
@@ -691,8 +699,6 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= brcm_fet_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= brcm_fet_ack_interrupt,
 	.config_intr	= brcm_fet_config_intr,
 }, {
@@ -702,10 +708,18 @@ static struct phy_driver broadcom_drivers[] = {
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= brcm_fet_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= brcm_fet_ack_interrupt,
 	.config_intr	= brcm_fet_config_intr,
+}, {
+	.phy_id		= PHY_ID_BCM5395,
+	.phy_id_mask	= 0xfffffff0,
+	.name		= "Broadcom BCM5395",
+	.flags		= PHY_IS_INTERNAL,
+	.features	= PHY_GBIT_FEATURES,
+	.get_sset_count	= bcm_phy_get_sset_count,
+	.get_strings	= bcm_phy_get_strings,
+	.get_stats	= bcm53xx_phy_get_stats,
+	.probe		= bcm53xx_phy_probe,
 } };
 
 module_phy_driver(broadcom_drivers);
@@ -726,6 +740,7 @@ static struct mdio_device_id __maybe_unused broadcom_tbl[] = {
 	{ PHY_ID_BCM57780, 0xfffffff0 },
 	{ PHY_ID_BCMAC131, 0xfffffff0 },
 	{ PHY_ID_BCM5241, 0xfffffff0 },
+	{ PHY_ID_BCM5395, 0xfffffff0 },
 	{ }
 };
 
diff --git a/drivers/net/phy/cicada.c b/drivers/net/phy/cicada.c
index d339c1afea77..c05af00bf4b6 100644
--- a/drivers/net/phy/cicada.c
+++ b/drivers/net/phy/cicada.c
@@ -110,8 +110,6 @@ static struct phy_driver cis820x_driver[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= &cis820x_config_init,
-	.config_aneg	= &genphy_config_aneg,
-	.read_status	= &genphy_read_status,
 	.ack_interrupt	= &cis820x_ack_interrupt,
 	.config_intr	= &cis820x_config_intr,
 }, {
@@ -121,8 +119,6 @@ static struct phy_driver cis820x_driver[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= &cis820x_config_init,
-	.config_aneg	= &genphy_config_aneg,
-	.read_status	= &genphy_read_status,
 	.ack_interrupt	= &cis820x_ack_interrupt,
 	.config_intr	= &cis820x_config_intr,
 } };
diff --git a/drivers/net/phy/davicom.c b/drivers/net/phy/davicom.c
index e28913d9ea7e..5ee99b3b428c 100644
--- a/drivers/net/phy/davicom.c
+++ b/drivers/net/phy/davicom.c
@@ -153,7 +153,6 @@ static struct phy_driver dm91xx_driver[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= dm9161_config_init,
 	.config_aneg	= dm9161_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= dm9161_ack_interrupt,
 	.config_intr	= dm9161_config_intr,
 }, {
@@ -164,7 +163,6 @@ static struct phy_driver dm91xx_driver[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= dm9161_config_init,
 	.config_aneg	= dm9161_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= dm9161_ack_interrupt,
 	.config_intr	= dm9161_config_intr,
 }, {
@@ -175,7 +173,6 @@ static struct phy_driver dm91xx_driver[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= dm9161_config_init,
 	.config_aneg	= dm9161_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= dm9161_ack_interrupt,
 	.config_intr	= dm9161_config_intr,
 }, {
@@ -184,8 +181,6 @@ static struct phy_driver dm91xx_driver[] = {
 	.phy_id_mask	= 0x0ffffff0,
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= dm9161_ack_interrupt,
 	.config_intr	= dm9161_config_intr,
 } };
diff --git a/drivers/net/phy/dp83640.c b/drivers/net/phy/dp83640.c
index cbd629822f04..654f42d00092 100644
--- a/drivers/net/phy/dp83640.c
+++ b/drivers/net/phy/dp83640.c
@@ -1502,8 +1502,6 @@ static struct phy_driver dp83640_driver = {
 	.probe		= dp83640_probe,
 	.remove		= dp83640_remove,
 	.config_init	= dp83640_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt  = dp83640_ack_interrupt,
 	.config_intr    = dp83640_config_intr,
 	.ts_info	= dp83640_ts_info,
diff --git a/drivers/net/phy/dp83822.c b/drivers/net/phy/dp83822.c
index 14335d14e9e4..6e8a2a4f3a6e 100644
--- a/drivers/net/phy/dp83822.c
+++ b/drivers/net/phy/dp83822.c
@@ -325,8 +325,6 @@ static struct phy_driver dp83822_driver[] = {
 		.set_wol = dp83822_set_wol,
 		.ack_interrupt = dp83822_ack_interrupt,
 		.config_intr = dp83822_config_intr,
-		.config_aneg = genphy_config_aneg,
-		.read_status = genphy_read_status,
 		.suspend = dp83822_suspend,
 		.resume = dp83822_resume,
 	 },
diff --git a/drivers/net/phy/dp83848.c b/drivers/net/phy/dp83848.c
index 3966d43c5146..cd09c3af2117 100644
--- a/drivers/net/phy/dp83848.c
+++ b/drivers/net/phy/dp83848.c
@@ -95,8 +95,6 @@ MODULE_DEVICE_TABLE(mdio, dp83848_tbl);
 		.config_init	= genphy_config_init,		\
 		.suspend	= genphy_suspend,		\
 		.resume		= genphy_resume,		\
-		.config_aneg	= genphy_config_aneg,		\
-		.read_status	= genphy_read_status,		\
 								\
 		/* IRQ related */				\
 		.ack_interrupt	= dp83848_ack_interrupt,	\
diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c
index c1ab976cc800..ab58224f897f 100644
--- a/drivers/net/phy/dp83867.c
+++ b/drivers/net/phy/dp83867.c
@@ -324,8 +324,6 @@ static struct phy_driver dp83867_driver[] = {
 		.ack_interrupt	= dp83867_ack_interrupt,
 		.config_intr	= dp83867_config_intr,
 
-		.config_aneg	= genphy_config_aneg,
-		.read_status	= genphy_read_status,
 		.suspend	= genphy_suspend,
 		.resume		= genphy_resume,
 	},
diff --git a/drivers/net/phy/fixed_phy.c b/drivers/net/phy/fixed_phy.c
index eb5167210681..001fe1df7557 100644
--- a/drivers/net/phy/fixed_phy.c
+++ b/drivers/net/phy/fixed_phy.c
@@ -115,37 +115,6 @@ int fixed_phy_set_link_update(struct phy_device *phydev,
 }
 EXPORT_SYMBOL_GPL(fixed_phy_set_link_update);
 
-int fixed_phy_update_state(struct phy_device *phydev,
-			   const struct fixed_phy_status *status,
-			   const struct fixed_phy_status *changed)
-{
-	struct fixed_mdio_bus *fmb = &platform_fmb;
-	struct fixed_phy *fp;
-
-	if (!phydev || phydev->mdio.bus != fmb->mii_bus)
-		return -EINVAL;
-
-	list_for_each_entry(fp, &fmb->phys, node) {
-		if (fp->addr == phydev->mdio.addr) {
-			write_seqcount_begin(&fp->seqcount);
-#define _UPD(x) if (changed->x) \
-	fp->status.x = status->x
-			_UPD(link);
-			_UPD(speed);
-			_UPD(duplex);
-			_UPD(pause);
-			_UPD(asym_pause);
-#undef _UPD
-			fixed_phy_update(fp);
-			write_seqcount_end(&fp->seqcount);
-			return 0;
-		}
-	}
-
-	return -ENOENT;
-}
-EXPORT_SYMBOL(fixed_phy_update_state);
-
 int fixed_phy_add(unsigned int irq, int phy_addr,
 		  struct fixed_phy_status *status,
 		  int link_gpio)
diff --git a/drivers/net/phy/icplus.c b/drivers/net/phy/icplus.c
index 567280a72241..791587a49215 100644
--- a/drivers/net/phy/icplus.c
+++ b/drivers/net/phy/icplus.c
@@ -227,8 +227,6 @@ static struct phy_driver icplus_driver[] = {
 	.phy_id_mask	= 0x0ffffff0,
 	.features	= PHY_GBIT_FEATURES,
 	.config_init	= &ip1001_config_init,
-	.config_aneg	= &genphy_config_aneg,
-	.read_status	= &genphy_read_status,
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 }, {
@@ -239,8 +237,6 @@ static struct phy_driver icplus_driver[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.ack_interrupt	= ip101a_g_ack_interrupt,
 	.config_init	= &ip101a_g_config_init,
-	.config_aneg	= &genphy_config_aneg,
-	.read_status	= &genphy_read_status,
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 } };
diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c
index 55f8c52dd2f1..a11f80cb5388 100644
--- a/drivers/net/phy/intel-xway.c
+++ b/drivers/net/phy/intel-xway.c
@@ -243,7 +243,6 @@ static struct phy_driver xway_gphy[] = {
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
 		.config_aneg	= xway_gphy14_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -257,7 +256,6 @@ static struct phy_driver xway_gphy[] = {
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
 		.config_aneg	= xway_gphy14_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -271,7 +269,6 @@ static struct phy_driver xway_gphy[] = {
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
 		.config_aneg	= xway_gphy14_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -285,7 +282,6 @@ static struct phy_driver xway_gphy[] = {
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
 		.config_aneg	= xway_gphy14_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -298,8 +294,6 @@ static struct phy_driver xway_gphy[] = {
 		.features	= PHY_GBIT_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
-		.config_aneg	= genphy_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -312,8 +306,6 @@ static struct phy_driver xway_gphy[] = {
 		.features	= PHY_BASIC_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
-		.config_aneg	= genphy_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -326,8 +318,6 @@ static struct phy_driver xway_gphy[] = {
 		.features	= PHY_GBIT_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
-		.config_aneg	= genphy_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -340,8 +330,6 @@ static struct phy_driver xway_gphy[] = {
 		.features	= PHY_BASIC_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
 		.config_init	= xway_gphy_config_init,
-		.config_aneg	= genphy_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= xway_gphy_ack_interrupt,
 		.did_interrupt	= xway_gphy_did_interrupt,
 		.config_intr	= xway_gphy_config_intr,
diff --git a/drivers/net/phy/lxt.c b/drivers/net/phy/lxt.c
index 09d215177fff..c14b254b2879 100644
--- a/drivers/net/phy/lxt.c
+++ b/drivers/net/phy/lxt.c
@@ -259,8 +259,6 @@ static struct phy_driver lxt97x_driver[] = {
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= lxt970_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= lxt970_ack_interrupt,
 	.config_intr	= lxt970_config_intr,
 }, {
@@ -269,8 +267,6 @@ static struct phy_driver lxt97x_driver[] = {
 	.phy_id_mask	= 0xfffffff0,
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= lxt971_ack_interrupt,
 	.config_intr	= lxt971_config_intr,
 }, {
@@ -290,7 +286,6 @@ static struct phy_driver lxt97x_driver[] = {
 	.flags		= 0,
 	.probe		= lxt973_probe,
 	.config_aneg	= lxt973_config_aneg,
-	.read_status	= genphy_read_status,
 } };
 
 module_phy_driver(lxt97x_driver);
diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
index 82104edca393..0e0978d8a0eb 100644
--- a/drivers/net/phy/marvell.c
+++ b/drivers/net/phy/marvell.c
@@ -83,7 +83,7 @@
 #define MII_88E1121_PHY_MSCR_REG	21
 #define MII_88E1121_PHY_MSCR_RX_DELAY	BIT(5)
 #define MII_88E1121_PHY_MSCR_TX_DELAY	BIT(4)
-#define MII_88E1121_PHY_MSCR_DELAY_MASK	(~(BIT(5) | BIT(4)))
+#define MII_88E1121_PHY_MSCR_DELAY_MASK	(BIT(5) | BIT(4))
 
 #define MII_88E1121_MISC_TEST				0x1a
 #define MII_88E1510_MISC_TEST_TEMP_THRESHOLD_MASK	0x1f00
@@ -96,6 +96,17 @@
 #define MII_88E1510_TEMP_SENSOR		0x1b
 #define MII_88E1510_TEMP_SENSOR_MASK	0xff
 
+#define MII_88E6390_MISC_TEST		0x1b
+#define MII_88E6390_MISC_TEST_SAMPLE_1S		0
+#define MII_88E6390_MISC_TEST_SAMPLE_10MS	BIT(14)
+#define MII_88E6390_MISC_TEST_SAMPLE_DISABLE	BIT(15)
+#define MII_88E6390_MISC_TEST_SAMPLE_ENABLE	0
+#define MII_88E6390_MISC_TEST_SAMPLE_MASK	(0x3 << 14)
+
+#define MII_88E6390_TEMP_SENSOR		0x1c
+#define MII_88E6390_TEMP_SENSOR_MASK	0xff
+#define MII_88E6390_TEMP_SENSOR_SAMPLES 10
+
 #define MII_88E1318S_PHY_MSCR1_REG	16
 #define MII_88E1318S_PHY_MSCR1_PAD_ODD	BIT(6)
 
@@ -177,27 +188,19 @@ struct marvell_priv {
 	struct device *hwmon_dev;
 };
 
-static int marvell_get_page(struct phy_device *phydev)
+static int marvell_read_page(struct phy_device *phydev)
 {
-	return phy_read(phydev, MII_MARVELL_PHY_PAGE);
+	return __phy_read(phydev, MII_MARVELL_PHY_PAGE);
 }
 
-static int marvell_set_page(struct phy_device *phydev, int page)
+static int marvell_write_page(struct phy_device *phydev, int page)
 {
-	return phy_write(phydev, MII_MARVELL_PHY_PAGE, page);
+	return __phy_write(phydev, MII_MARVELL_PHY_PAGE, page);
 }
 
-static int marvell_get_set_page(struct phy_device *phydev, int page)
+static int marvell_set_page(struct phy_device *phydev, int page)
 {
-	int oldpage = marvell_get_page(phydev);
-
-	if (oldpage < 0)
-		return oldpage;
-
-	if (page != oldpage)
-		return marvell_set_page(phydev, page);
-
-	return 0;
+	return phy_write(phydev, MII_MARVELL_PHY_PAGE, page);
 }
 
 static int marvell_ack_interrupt(struct phy_device *phydev)
@@ -399,7 +402,7 @@ static int m88e1111_config_aneg(struct phy_device *phydev)
 static int marvell_of_reg_init(struct phy_device *phydev)
 {
 	const __be32 *paddr;
-	int len, i, saved_page, current_page, ret;
+	int len, i, saved_page, current_page, ret = 0;
 
 	if (!phydev->mdio.dev.of_node)
 		return 0;
@@ -409,12 +412,11 @@ static int marvell_of_reg_init(struct phy_device *phydev)
 	if (!paddr || len < (4 * sizeof(*paddr)))
 		return 0;
 
-	saved_page = marvell_get_page(phydev);
+	saved_page = phy_save_page(phydev);
 	if (saved_page < 0)
-		return saved_page;
+		goto err;
 	current_page = saved_page;
 
-	ret = 0;
 	len /= sizeof(*paddr);
 	for (i = 0; i < len - 3; i += 4) {
 		u16 page = be32_to_cpup(paddr + i);
@@ -425,14 +427,14 @@ static int marvell_of_reg_init(struct phy_device *phydev)
 
 		if (page != current_page) {
 			current_page = page;
-			ret = marvell_set_page(phydev, page);
+			ret = marvell_write_page(phydev, page);
 			if (ret < 0)
 				goto err;
 		}
 
 		val = 0;
 		if (mask) {
-			val = phy_read(phydev, reg);
+			val = __phy_read(phydev, reg);
 			if (val < 0) {
 				ret = val;
 				goto err;
@@ -441,17 +443,12 @@ static int marvell_of_reg_init(struct phy_device *phydev)
 		}
 		val |= val_bits;
 
-		ret = phy_write(phydev, reg, val);
+		ret = __phy_write(phydev, reg, val);
 		if (ret < 0)
 			goto err;
 	}
 err:
-	if (current_page != saved_page) {
-		i = marvell_set_page(phydev, saved_page);
-		if (ret == 0)
-			ret = i;
-	}
-	return ret;
+	return phy_restore_page(phydev, saved_page, ret);
 }
 #else
 static int marvell_of_reg_init(struct phy_device *phydev)
@@ -462,34 +459,21 @@ static int marvell_of_reg_init(struct phy_device *phydev)
 
 static int m88e1121_config_aneg_rgmii_delays(struct phy_device *phydev)
 {
-	int err, oldpage, mscr;
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_MSCR_PAGE);
-	if (oldpage < 0)
-		return oldpage;
-
-	mscr = phy_read(phydev, MII_88E1121_PHY_MSCR_REG);
-	if (mscr < 0) {
-		err = mscr;
-		goto out;
-	}
-
-	mscr &= MII_88E1121_PHY_MSCR_DELAY_MASK;
+	int mscr;
 
 	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID)
-		mscr |= (MII_88E1121_PHY_MSCR_RX_DELAY |
-			 MII_88E1121_PHY_MSCR_TX_DELAY);
+		mscr = MII_88E1121_PHY_MSCR_RX_DELAY |
+		       MII_88E1121_PHY_MSCR_TX_DELAY;
 	else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID)
-		mscr |= MII_88E1121_PHY_MSCR_RX_DELAY;
+		mscr = MII_88E1121_PHY_MSCR_RX_DELAY;
 	else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
-		mscr |= MII_88E1121_PHY_MSCR_TX_DELAY;
-
-	err = phy_write(phydev, MII_88E1121_PHY_MSCR_REG, mscr);
-
-out:
-	marvell_set_page(phydev, oldpage);
+		mscr = MII_88E1121_PHY_MSCR_TX_DELAY;
+	else
+		mscr = 0;
 
-	return err;
+	return phy_modify_paged(phydev, MII_MARVELL_MSCR_PAGE,
+				MII_88E1121_PHY_MSCR_REG,
+				MII_88E1121_PHY_MSCR_DELAY_MASK, mscr);
 }
 
 static int m88e1121_config_aneg(struct phy_device *phydev)
@@ -498,7 +482,7 @@ static int m88e1121_config_aneg(struct phy_device *phydev)
 
 	if (phy_interface_is_rgmii(phydev)) {
 		err = m88e1121_config_aneg_rgmii_delays(phydev);
-		if (err)
+		if (err < 0)
 			return err;
 	}
 
@@ -515,20 +499,11 @@ static int m88e1121_config_aneg(struct phy_device *phydev)
 
 static int m88e1318_config_aneg(struct phy_device *phydev)
 {
-	int err, oldpage, mscr;
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_MSCR_PAGE);
-	if (oldpage < 0)
-		return oldpage;
-
-	mscr = phy_read(phydev, MII_88E1318S_PHY_MSCR1_REG);
-	mscr |= MII_88E1318S_PHY_MSCR1_PAD_ODD;
-
-	err = phy_write(phydev, MII_88E1318S_PHY_MSCR1_REG, mscr);
-	if (err < 0)
-		return err;
+	int err;
 
-	err = marvell_set_page(phydev, oldpage);
+	err = phy_modify_paged(phydev, MII_MARVELL_MSCR_PAGE,
+			       MII_88E1318S_PHY_MSCR1_REG,
+			       0, MII_88E1318S_PHY_MSCR1_PAD_ODD);
 	if (err < 0)
 		return err;
 
@@ -700,19 +675,14 @@ static int m88e1116r_config_init(struct phy_device *phydev)
 
 static int m88e3016_config_init(struct phy_device *phydev)
 {
-	int reg;
+	int ret;
 
 	/* Enable Scrambler and Auto-Crossover */
-	reg = phy_read(phydev, MII_88E3016_PHY_SPEC_CTRL);
-	if (reg < 0)
-		return reg;
-
-	reg &= ~MII_88E3016_DISABLE_SCRAMBLER;
-	reg |= MII_88E3016_AUTO_MDIX_CROSSOVER;
-
-	reg = phy_write(phydev, MII_88E3016_PHY_SPEC_CTRL, reg);
-	if (reg < 0)
-		return reg;
+	ret = phy_modify(phydev, MII_88E3016_PHY_SPEC_CTRL,
+			 MII_88E3016_DISABLE_SCRAMBLER,
+			 MII_88E3016_AUTO_MDIX_CROSSOVER);
+	if (ret < 0)
+		return ret;
 
 	return marvell_config_init(phydev);
 }
@@ -721,42 +691,33 @@ static int m88e1111_config_init_hwcfg_mode(struct phy_device *phydev,
 					   u16 mode,
 					   int fibre_copper_auto)
 {
-	int temp;
-
-	temp = phy_read(phydev, MII_M1111_PHY_EXT_SR);
-	if (temp < 0)
-		return temp;
-
-	temp &= ~(MII_M1111_HWCFG_MODE_MASK |
-		  MII_M1111_HWCFG_FIBER_COPPER_AUTO |
-		  MII_M1111_HWCFG_FIBER_COPPER_RES);
-	temp |= mode;
-
 	if (fibre_copper_auto)
-		temp |= MII_M1111_HWCFG_FIBER_COPPER_AUTO;
+		mode |= MII_M1111_HWCFG_FIBER_COPPER_AUTO;
 
-	return phy_write(phydev, MII_M1111_PHY_EXT_SR, temp);
+	return phy_modify(phydev, MII_M1111_PHY_EXT_SR,
+			  MII_M1111_HWCFG_MODE_MASK |
+			  MII_M1111_HWCFG_FIBER_COPPER_AUTO |
+			  MII_M1111_HWCFG_FIBER_COPPER_RES,
+			  mode);
 }
 
 static int m88e1111_config_init_rgmii_delays(struct phy_device *phydev)
 {
-	int temp;
-
-	temp = phy_read(phydev, MII_M1111_PHY_EXT_CR);
-	if (temp < 0)
-		return temp;
+	int delay;
 
 	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID) {
-		temp |= (MII_M1111_RGMII_RX_DELAY | MII_M1111_RGMII_TX_DELAY);
+		delay = MII_M1111_RGMII_RX_DELAY | MII_M1111_RGMII_TX_DELAY;
 	} else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_RXID) {
-		temp &= ~MII_M1111_RGMII_TX_DELAY;
-		temp |= MII_M1111_RGMII_RX_DELAY;
+		delay = MII_M1111_RGMII_RX_DELAY;
 	} else if (phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID) {
-		temp &= ~MII_M1111_RGMII_RX_DELAY;
-		temp |= MII_M1111_RGMII_TX_DELAY;
+		delay = MII_M1111_RGMII_TX_DELAY;
+	} else {
+		delay = 0;
 	}
 
-	return phy_write(phydev, MII_M1111_PHY_EXT_CR, temp);
+	return phy_modify(phydev, MII_M1111_PHY_EXT_CR,
+			  MII_M1111_RGMII_RX_DELAY | MII_M1111_RGMII_TX_DELAY,
+			  delay);
 }
 
 static int m88e1111_config_init_rgmii(struct phy_device *phydev)
@@ -802,7 +763,7 @@ static int m88e1111_config_init_rtbi(struct phy_device *phydev)
 	int err;
 
 	err = m88e1111_config_init_rgmii_delays(phydev);
-	if (err)
+	if (err < 0)
 		return err;
 
 	err = m88e1111_config_init_hwcfg_mode(
@@ -829,7 +790,7 @@ static int m88e1111_config_init(struct phy_device *phydev)
 
 	if (phy_interface_is_rgmii(phydev)) {
 		err = m88e1111_config_init_rgmii(phydev);
-		if (err)
+		if (err < 0)
 			return err;
 	}
 
@@ -854,20 +815,15 @@ static int m88e1111_config_init(struct phy_device *phydev)
 
 static int m88e1121_config_init(struct phy_device *phydev)
 {
-	int err, oldpage;
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_LED_PAGE);
-	if (oldpage < 0)
-		return oldpage;
+	int err;
 
 	/* Default PHY LED config: LED[0] .. Link, LED[1] .. Activity */
-	err = phy_write(phydev, MII_88E1121_PHY_LED_CTRL,
-			MII_88E1121_PHY_LED_DEF);
+	err = phy_write_paged(phydev, MII_MARVELL_LED_PAGE,
+			      MII_88E1121_PHY_LED_CTRL,
+			      MII_88E1121_PHY_LED_DEF);
 	if (err < 0)
 		return err;
 
-	marvell_set_page(phydev, oldpage);
-
 	/* Set marvell,reg-init configuration from device tree */
 	return marvell_config_init(phydev);
 }
@@ -875,7 +831,6 @@ static int m88e1121_config_init(struct phy_device *phydev)
 static int m88e1510_config_init(struct phy_device *phydev)
 {
 	int err;
-	int temp;
 
 	/* SGMII-to-Copper mode initialization */
 	if (phydev->interface == PHY_INTERFACE_MODE_SGMII) {
@@ -887,16 +842,15 @@ static int m88e1510_config_init(struct phy_device *phydev)
 			return err;
 
 		/* In reg 20, write MODE[2:0] = 0x1 (SGMII to Copper) */
-		temp = phy_read(phydev, MII_88E1510_GEN_CTRL_REG_1);
-		temp &= ~MII_88E1510_GEN_CTRL_REG_1_MODE_MASK;
-		temp |= MII_88E1510_GEN_CTRL_REG_1_MODE_SGMII;
-		err = phy_write(phydev, MII_88E1510_GEN_CTRL_REG_1, temp);
+		err = phy_modify(phydev, MII_88E1510_GEN_CTRL_REG_1,
+				 MII_88E1510_GEN_CTRL_REG_1_MODE_MASK,
+				 MII_88E1510_GEN_CTRL_REG_1_MODE_SGMII);
 		if (err < 0)
 			return err;
 
 		/* PHY reset is necessary after changing MODE[2:0] */
-		temp |= MII_88E1510_GEN_CTRL_REG_1_RESET;
-		err = phy_write(phydev, MII_88E1510_GEN_CTRL_REG_1, temp);
+		err = phy_modify(phydev, MII_88E1510_GEN_CTRL_REG_1, 0,
+				 MII_88E1510_GEN_CTRL_REG_1_RESET);
 		if (err < 0)
 			return err;
 
@@ -1002,7 +956,6 @@ static int m88e1149_config_init(struct phy_device *phydev)
 
 static int m88e1145_config_init_rgmii(struct phy_device *phydev)
 {
-	int temp;
 	int err;
 
 	err = m88e1111_config_init_rgmii_delays(phydev);
@@ -1014,15 +967,9 @@ static int m88e1145_config_init_rgmii(struct phy_device *phydev)
 		if (err < 0)
 			return err;
 
-		temp = phy_read(phydev, 0x1e);
-		if (temp < 0)
-			return temp;
-
-		temp &= 0xf03f;
-		temp |= 2 << 9;	/* 36 ohm */
-		temp |= 2 << 6;	/* 39 ohm */
-
-		err = phy_write(phydev, 0x1e, temp);
+		err = phy_modify(phydev, 0x1e, 0x0fc0,
+				 2 << 9 | /* 36 ohm */
+				 2 << 6); /* 39 ohm */
 		if (err < 0)
 			return err;
 
@@ -1398,100 +1345,98 @@ static int m88e1121_did_interrupt(struct phy_device *phydev)
 static void m88e1318_get_wol(struct phy_device *phydev,
 			     struct ethtool_wolinfo *wol)
 {
+	int oldpage, ret = 0;
+
 	wol->supported = WAKE_MAGIC;
 	wol->wolopts = 0;
 
-	if (marvell_set_page(phydev, MII_MARVELL_WOL_PAGE) < 0)
-		return;
+	oldpage = phy_select_page(phydev, MII_MARVELL_WOL_PAGE);
+	if (oldpage < 0)
+		goto error;
 
-	if (phy_read(phydev, MII_88E1318S_PHY_WOL_CTRL) &
-	    MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE)
+	ret = __phy_read(phydev, MII_88E1318S_PHY_WOL_CTRL);
+	if (ret & MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE)
 		wol->wolopts |= WAKE_MAGIC;
 
-	if (marvell_set_page(phydev, MII_MARVELL_COPPER_PAGE) < 0)
-		return;
+error:
+	phy_restore_page(phydev, oldpage, ret);
 }
 
 static int m88e1318_set_wol(struct phy_device *phydev,
 			    struct ethtool_wolinfo *wol)
 {
-	int err, oldpage, temp;
+	int err = 0, oldpage;
 
-	oldpage = marvell_get_page(phydev);
+	oldpage = phy_save_page(phydev);
+	if (oldpage < 0)
+		goto error;
 
 	if (wol->wolopts & WAKE_MAGIC) {
 		/* Explicitly switch to page 0x00, just to be sure */
-		err = marvell_set_page(phydev, MII_MARVELL_COPPER_PAGE);
+		err = marvell_write_page(phydev, MII_MARVELL_COPPER_PAGE);
 		if (err < 0)
-			return err;
+			goto error;
 
 		/* Enable the WOL interrupt */
-		temp = phy_read(phydev, MII_88E1318S_PHY_CSIER);
-		temp |= MII_88E1318S_PHY_CSIER_WOL_EIE;
-		err = phy_write(phydev, MII_88E1318S_PHY_CSIER, temp);
+		err = __phy_modify(phydev, MII_88E1318S_PHY_CSIER, 0,
+				   MII_88E1318S_PHY_CSIER_WOL_EIE);
 		if (err < 0)
-			return err;
+			goto error;
 
-		err = marvell_set_page(phydev, MII_MARVELL_LED_PAGE);
+		err = marvell_write_page(phydev, MII_MARVELL_LED_PAGE);
 		if (err < 0)
-			return err;
+			goto error;
 
 		/* Setup LED[2] as interrupt pin (active low) */
-		temp = phy_read(phydev, MII_88E1318S_PHY_LED_TCR);
-		temp &= ~MII_88E1318S_PHY_LED_TCR_FORCE_INT;
-		temp |= MII_88E1318S_PHY_LED_TCR_INTn_ENABLE;
-		temp |= MII_88E1318S_PHY_LED_TCR_INT_ACTIVE_LOW;
-		err = phy_write(phydev, MII_88E1318S_PHY_LED_TCR, temp);
+		err = __phy_modify(phydev, MII_88E1318S_PHY_LED_TCR,
+				   MII_88E1318S_PHY_LED_TCR_FORCE_INT,
+				   MII_88E1318S_PHY_LED_TCR_INTn_ENABLE |
+				   MII_88E1318S_PHY_LED_TCR_INT_ACTIVE_LOW);
 		if (err < 0)
-			return err;
+			goto error;
 
-		err = marvell_set_page(phydev, MII_MARVELL_WOL_PAGE);
+		err = marvell_write_page(phydev, MII_MARVELL_WOL_PAGE);
 		if (err < 0)
-			return err;
+			goto error;
 
 		/* Store the device address for the magic packet */
-		err = phy_write(phydev, MII_88E1318S_PHY_MAGIC_PACKET_WORD2,
+		err = __phy_write(phydev, MII_88E1318S_PHY_MAGIC_PACKET_WORD2,
 				((phydev->attached_dev->dev_addr[5] << 8) |
 				 phydev->attached_dev->dev_addr[4]));
 		if (err < 0)
-			return err;
-		err = phy_write(phydev, MII_88E1318S_PHY_MAGIC_PACKET_WORD1,
+			goto error;
+		err = __phy_write(phydev, MII_88E1318S_PHY_MAGIC_PACKET_WORD1,
 				((phydev->attached_dev->dev_addr[3] << 8) |
 				 phydev->attached_dev->dev_addr[2]));
 		if (err < 0)
-			return err;
-		err = phy_write(phydev, MII_88E1318S_PHY_MAGIC_PACKET_WORD0,
+			goto error;
+		err = __phy_write(phydev, MII_88E1318S_PHY_MAGIC_PACKET_WORD0,
 				((phydev->attached_dev->dev_addr[1] << 8) |
 				 phydev->attached_dev->dev_addr[0]));
 		if (err < 0)
-			return err;
+			goto error;
 
 		/* Clear WOL status and enable magic packet matching */
-		temp = phy_read(phydev, MII_88E1318S_PHY_WOL_CTRL);
-		temp |= MII_88E1318S_PHY_WOL_CTRL_CLEAR_WOL_STATUS;
-		temp |= MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE;
-		err = phy_write(phydev, MII_88E1318S_PHY_WOL_CTRL, temp);
+		err = __phy_modify(phydev, MII_88E1318S_PHY_WOL_CTRL, 0,
+				   MII_88E1318S_PHY_WOL_CTRL_CLEAR_WOL_STATUS |
+				   MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE);
 		if (err < 0)
-			return err;
+			goto error;
 	} else {
-		err = marvell_set_page(phydev, MII_MARVELL_WOL_PAGE);
+		err = marvell_write_page(phydev, MII_MARVELL_WOL_PAGE);
 		if (err < 0)
-			return err;
+			goto error;
 
 		/* Clear WOL status and disable magic packet matching */
-		temp = phy_read(phydev, MII_88E1318S_PHY_WOL_CTRL);
-		temp |= MII_88E1318S_PHY_WOL_CTRL_CLEAR_WOL_STATUS;
-		temp &= ~MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE;
-		err = phy_write(phydev, MII_88E1318S_PHY_WOL_CTRL, temp);
+		err = __phy_modify(phydev, MII_88E1318S_PHY_WOL_CTRL,
+				   MII_88E1318S_PHY_WOL_CTRL_MAGIC_PACKET_MATCH_ENABLE,
+				   MII_88E1318S_PHY_WOL_CTRL_CLEAR_WOL_STATUS);
 		if (err < 0)
-			return err;
+			goto error;
 	}
 
-	err = marvell_set_page(phydev, oldpage);
-	if (err < 0)
-		return err;
-
-	return 0;
+error:
+	return phy_restore_page(phydev, oldpage, err);
 }
 
 static int marvell_get_sset_count(struct phy_device *phydev)
@@ -1507,8 +1452,8 @@ static void marvell_get_strings(struct phy_device *phydev, u8 *data)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(marvell_hw_stats); i++) {
-		memcpy(data + i * ETH_GSTRING_LEN,
-		       marvell_hw_stats[i].string, ETH_GSTRING_LEN);
+		strlcpy(data + i * ETH_GSTRING_LEN,
+			marvell_hw_stats[i].string, ETH_GSTRING_LEN);
 	}
 }
 
@@ -1519,14 +1464,10 @@ static u64 marvell_get_stat(struct phy_device *phydev, int i)
 {
 	struct marvell_hw_stat stat = marvell_hw_stats[i];
 	struct marvell_priv *priv = phydev->priv;
-	int oldpage, val;
+	int val;
 	u64 ret;
 
-	oldpage = marvell_get_set_page(phydev, stat.page);
-	if (oldpage < 0)
-		return UINT64_MAX;
-
-	val = phy_read(phydev, stat.reg);
+	val = phy_read_paged(phydev, stat.page, stat.reg);
 	if (val < 0) {
 		ret = UINT64_MAX;
 	} else {
@@ -1535,8 +1476,6 @@ static u64 marvell_get_stat(struct phy_device *phydev, int i)
 		ret = priv->stats[i];
 	}
 
-	marvell_set_page(phydev, oldpage);
-
 	return ret;
 }
 
@@ -1553,51 +1492,44 @@ static void marvell_get_stats(struct phy_device *phydev,
 static int m88e1121_get_temp(struct phy_device *phydev, long *temp)
 {
 	int oldpage;
-	int ret;
+	int ret = 0;
 	int val;
 
 	*temp = 0;
 
-	mutex_lock(&phydev->lock);
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_MISC_TEST_PAGE);
-	if (oldpage < 0) {
-		mutex_unlock(&phydev->lock);
-		return oldpage;
-	}
+	oldpage = phy_select_page(phydev, MII_MARVELL_MISC_TEST_PAGE);
+	if (oldpage < 0)
+		goto error;
 
 	/* Enable temperature sensor */
-	ret = phy_read(phydev, MII_88E1121_MISC_TEST);
+	ret = __phy_read(phydev, MII_88E1121_MISC_TEST);
 	if (ret < 0)
 		goto error;
 
-	ret = phy_write(phydev, MII_88E1121_MISC_TEST,
-			ret | MII_88E1121_MISC_TEST_TEMP_SENSOR_EN);
+	ret = __phy_write(phydev, MII_88E1121_MISC_TEST,
+			  ret | MII_88E1121_MISC_TEST_TEMP_SENSOR_EN);
 	if (ret < 0)
 		goto error;
 
 	/* Wait for temperature to stabilize */
 	usleep_range(10000, 12000);
 
-	val = phy_read(phydev, MII_88E1121_MISC_TEST);
+	val = __phy_read(phydev, MII_88E1121_MISC_TEST);
 	if (val < 0) {
 		ret = val;
 		goto error;
 	}
 
 	/* Disable temperature sensor */
-	ret = phy_write(phydev, MII_88E1121_MISC_TEST,
-			ret & ~MII_88E1121_MISC_TEST_TEMP_SENSOR_EN);
+	ret = __phy_write(phydev, MII_88E1121_MISC_TEST,
+			  ret & ~MII_88E1121_MISC_TEST_TEMP_SENSOR_EN);
 	if (ret < 0)
 		goto error;
 
 	*temp = ((val & MII_88E1121_MISC_TEST_TEMP_MASK) - 5) * 5000;
 
 error:
-	marvell_set_page(phydev, oldpage);
-	mutex_unlock(&phydev->lock);
-
-	return ret;
+	return phy_restore_page(phydev, oldpage, ret);
 }
 
 static int m88e1121_hwmon_read(struct device *dev,
@@ -1671,118 +1603,64 @@ static const struct hwmon_chip_info m88e1121_hwmon_chip_info = {
 
 static int m88e1510_get_temp(struct phy_device *phydev, long *temp)
 {
-	int oldpage;
 	int ret;
 
 	*temp = 0;
 
-	mutex_lock(&phydev->lock);
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_MISC_TEST_PAGE);
-	if (oldpage < 0) {
-		mutex_unlock(&phydev->lock);
-		return oldpage;
-	}
-
-	ret = phy_read(phydev, MII_88E1510_TEMP_SENSOR);
+	ret = phy_read_paged(phydev, MII_MARVELL_MISC_TEST_PAGE,
+			     MII_88E1510_TEMP_SENSOR);
 	if (ret < 0)
-		goto error;
+		return ret;
 
 	*temp = ((ret & MII_88E1510_TEMP_SENSOR_MASK) - 25) * 1000;
 
-error:
-	marvell_set_page(phydev, oldpage);
-	mutex_unlock(&phydev->lock);
-
-	return ret;
+	return 0;
 }
 
 static int m88e1510_get_temp_critical(struct phy_device *phydev, long *temp)
 {
-	int oldpage;
 	int ret;
 
 	*temp = 0;
 
-	mutex_lock(&phydev->lock);
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_MISC_TEST_PAGE);
-	if (oldpage < 0) {
-		mutex_unlock(&phydev->lock);
-		return oldpage;
-	}
-
-	ret = phy_read(phydev, MII_88E1121_MISC_TEST);
+	ret = phy_read_paged(phydev, MII_MARVELL_MISC_TEST_PAGE,
+			     MII_88E1121_MISC_TEST);
 	if (ret < 0)
-		goto error;
+		return ret;
 
 	*temp = (((ret & MII_88E1510_MISC_TEST_TEMP_THRESHOLD_MASK) >>
 		  MII_88E1510_MISC_TEST_TEMP_THRESHOLD_SHIFT) * 5) - 25;
 	/* convert to mC */
 	*temp *= 1000;
 
-error:
-	marvell_set_page(phydev, oldpage);
-	mutex_unlock(&phydev->lock);
-
-	return ret;
+	return 0;
 }
 
 static int m88e1510_set_temp_critical(struct phy_device *phydev, long temp)
 {
-	int oldpage;
-	int ret;
-
-	mutex_lock(&phydev->lock);
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_MISC_TEST_PAGE);
-	if (oldpage < 0) {
-		mutex_unlock(&phydev->lock);
-		return oldpage;
-	}
-
-	ret = phy_read(phydev, MII_88E1121_MISC_TEST);
-	if (ret < 0)
-		goto error;
-
 	temp = temp / 1000;
 	temp = clamp_val(DIV_ROUND_CLOSEST(temp, 5) + 5, 0, 0x1f);
-	ret = phy_write(phydev, MII_88E1121_MISC_TEST,
-			(ret & ~MII_88E1510_MISC_TEST_TEMP_THRESHOLD_MASK) |
-			(temp << MII_88E1510_MISC_TEST_TEMP_THRESHOLD_SHIFT));
-
-error:
-	marvell_set_page(phydev, oldpage);
-	mutex_unlock(&phydev->lock);
 
-	return ret;
+	return phy_modify_paged(phydev, MII_MARVELL_MISC_TEST_PAGE,
+				MII_88E1121_MISC_TEST,
+				MII_88E1510_MISC_TEST_TEMP_THRESHOLD_MASK,
+				temp << MII_88E1510_MISC_TEST_TEMP_THRESHOLD_SHIFT);
 }
 
 static int m88e1510_get_temp_alarm(struct phy_device *phydev, long *alarm)
 {
-	int oldpage;
 	int ret;
 
 	*alarm = false;
 
-	mutex_lock(&phydev->lock);
-
-	oldpage = marvell_get_set_page(phydev, MII_MARVELL_MISC_TEST_PAGE);
-	if (oldpage < 0) {
-		mutex_unlock(&phydev->lock);
-		return oldpage;
-	}
-
-	ret = phy_read(phydev, MII_88E1121_MISC_TEST);
+	ret = phy_read_paged(phydev, MII_MARVELL_MISC_TEST_PAGE,
+			     MII_88E1121_MISC_TEST);
 	if (ret < 0)
-		goto error;
-	*alarm = !!(ret & MII_88E1510_MISC_TEST_TEMP_IRQ);
+		return ret;
 
-error:
-	marvell_set_page(phydev, oldpage);
-	mutex_unlock(&phydev->lock);
+	*alarm = !!(ret & MII_88E1510_MISC_TEST_TEMP_IRQ);
 
-	return ret;
+	return 0;
 }
 
 static int m88e1510_hwmon_read(struct device *dev,
@@ -1871,6 +1749,123 @@ static const struct hwmon_chip_info m88e1510_hwmon_chip_info = {
 	.info = m88e1510_hwmon_info,
 };
 
+static int m88e6390_get_temp(struct phy_device *phydev, long *temp)
+{
+	int sum = 0;
+	int oldpage;
+	int ret = 0;
+	int i;
+
+	*temp = 0;
+
+	oldpage = phy_select_page(phydev, MII_MARVELL_MISC_TEST_PAGE);
+	if (oldpage < 0)
+		goto error;
+
+	/* Enable temperature sensor */
+	ret = __phy_read(phydev, MII_88E6390_MISC_TEST);
+	if (ret < 0)
+		goto error;
+
+	ret = ret & ~MII_88E6390_MISC_TEST_SAMPLE_MASK;
+	ret |= MII_88E6390_MISC_TEST_SAMPLE_ENABLE |
+		MII_88E6390_MISC_TEST_SAMPLE_1S;
+
+	ret = __phy_write(phydev, MII_88E6390_MISC_TEST, ret);
+	if (ret < 0)
+		goto error;
+
+	/* Wait for temperature to stabilize */
+	usleep_range(10000, 12000);
+
+	/* Reading the temperature sense has an errata. You need to read
+	 * a number of times and take an average.
+	 */
+	for (i = 0; i < MII_88E6390_TEMP_SENSOR_SAMPLES; i++) {
+		ret = __phy_read(phydev, MII_88E6390_TEMP_SENSOR);
+		if (ret < 0)
+			goto error;
+		sum += ret & MII_88E6390_TEMP_SENSOR_MASK;
+	}
+
+	sum /= MII_88E6390_TEMP_SENSOR_SAMPLES;
+	*temp = (sum  - 75) * 1000;
+
+	/* Disable temperature sensor */
+	ret = __phy_read(phydev, MII_88E6390_MISC_TEST);
+	if (ret < 0)
+		goto error;
+
+	ret = ret & ~MII_88E6390_MISC_TEST_SAMPLE_MASK;
+	ret |= MII_88E6390_MISC_TEST_SAMPLE_DISABLE;
+
+	ret = __phy_write(phydev, MII_88E6390_MISC_TEST, ret);
+
+error:
+	phy_restore_page(phydev, oldpage, ret);
+
+	return ret;
+}
+
+static int m88e6390_hwmon_read(struct device *dev,
+			       enum hwmon_sensor_types type,
+			       u32 attr, int channel, long *temp)
+{
+	struct phy_device *phydev = dev_get_drvdata(dev);
+	int err;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		err = m88e6390_get_temp(phydev, temp);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return err;
+}
+
+static umode_t m88e6390_hwmon_is_visible(const void *data,
+					 enum hwmon_sensor_types type,
+					 u32 attr, int channel)
+{
+	if (type != hwmon_temp)
+		return 0;
+
+	switch (attr) {
+	case hwmon_temp_input:
+		return 0444;
+	default:
+		return 0;
+	}
+}
+
+static u32 m88e6390_hwmon_temp_config[] = {
+	HWMON_T_INPUT,
+	0
+};
+
+static const struct hwmon_channel_info m88e6390_hwmon_temp = {
+	.type = hwmon_temp,
+	.config = m88e6390_hwmon_temp_config,
+};
+
+static const struct hwmon_channel_info *m88e6390_hwmon_info[] = {
+	&m88e1121_hwmon_chip,
+	&m88e6390_hwmon_temp,
+	NULL
+};
+
+static const struct hwmon_ops m88e6390_hwmon_hwmon_ops = {
+	.is_visible = m88e6390_hwmon_is_visible,
+	.read = m88e6390_hwmon_read,
+};
+
+static const struct hwmon_chip_info m88e6390_hwmon_chip_info = {
+	.ops = &m88e6390_hwmon_hwmon_ops,
+	.info = m88e6390_hwmon_info,
+};
+
 static int marvell_hwmon_name(struct phy_device *phydev)
 {
 	struct marvell_priv *priv = phydev->priv;
@@ -1917,6 +1912,11 @@ static int m88e1510_hwmon_probe(struct phy_device *phydev)
 {
 	return marvell_hwmon_probe(phydev, &m88e1510_hwmon_chip_info);
 }
+
+static int m88e6390_hwmon_probe(struct phy_device *phydev)
+{
+	return marvell_hwmon_probe(phydev, &m88e6390_hwmon_chip_info);
+}
 #else
 static int m88e1121_hwmon_probe(struct phy_device *phydev)
 {
@@ -1927,6 +1927,11 @@ static int m88e1510_hwmon_probe(struct phy_device *phydev)
 {
 	return 0;
 }
+
+static int m88e6390_hwmon_probe(struct phy_device *phydev)
+{
+	return 0;
+}
 #endif
 
 static int marvell_probe(struct phy_device *phydev)
@@ -1964,6 +1969,17 @@ static int m88e1510_probe(struct phy_device *phydev)
 	return m88e1510_hwmon_probe(phydev);
 }
 
+static int m88e6390_probe(struct phy_device *phydev)
+{
+	int err;
+
+	err = marvell_probe(phydev);
+	if (err)
+		return err;
+
+	return m88e6390_hwmon_probe(phydev);
+}
+
 static struct phy_driver marvell_drivers[] = {
 	{
 		.phy_id = MARVELL_PHY_ID_88E1101,
@@ -1974,11 +1990,12 @@ static struct phy_driver marvell_drivers[] = {
 		.probe = marvell_probe,
 		.config_init = &marvell_config_init,
 		.config_aneg = &m88e1101_config_aneg,
-		.read_status = &genphy_read_status,
 		.ack_interrupt = &marvell_ack_interrupt,
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -1992,11 +2009,12 @@ static struct phy_driver marvell_drivers[] = {
 		.probe = marvell_probe,
 		.config_init = &m88e1111_config_init,
 		.config_aneg = &marvell_config_aneg,
-		.read_status = &genphy_read_status,
 		.ack_interrupt = &marvell_ack_interrupt,
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2015,6 +2033,8 @@ static struct phy_driver marvell_drivers[] = {
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2028,11 +2048,12 @@ static struct phy_driver marvell_drivers[] = {
 		.probe = marvell_probe,
 		.config_init = &m88e1118_config_init,
 		.config_aneg = &m88e1118_config_aneg,
-		.read_status = &genphy_read_status,
 		.ack_interrupt = &marvell_ack_interrupt,
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2052,6 +2073,8 @@ static struct phy_driver marvell_drivers[] = {
 		.did_interrupt = &m88e1121_did_interrupt,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2073,6 +2096,8 @@ static struct phy_driver marvell_drivers[] = {
 		.set_wol = &m88e1318_set_wol,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2091,6 +2116,8 @@ static struct phy_driver marvell_drivers[] = {
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2104,11 +2131,12 @@ static struct phy_driver marvell_drivers[] = {
 		.probe = marvell_probe,
 		.config_init = &m88e1149_config_init,
 		.config_aneg = &m88e1118_config_aneg,
-		.read_status = &genphy_read_status,
 		.ack_interrupt = &marvell_ack_interrupt,
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2122,11 +2150,12 @@ static struct phy_driver marvell_drivers[] = {
 		.probe = marvell_probe,
 		.config_init = &m88e1111_config_init,
 		.config_aneg = &marvell_config_aneg,
-		.read_status = &genphy_read_status,
 		.ack_interrupt = &marvell_ack_interrupt,
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2139,12 +2168,12 @@ static struct phy_driver marvell_drivers[] = {
 		.flags = PHY_HAS_INTERRUPT,
 		.probe = marvell_probe,
 		.config_init = &m88e1116r_config_init,
-		.config_aneg = &genphy_config_aneg,
-		.read_status = &genphy_read_status,
 		.ack_interrupt = &marvell_ack_interrupt,
 		.config_intr = &marvell_config_intr,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2166,6 +2195,8 @@ static struct phy_driver marvell_drivers[] = {
 		.set_wol = &m88e1318_set_wol,
 		.resume = &marvell_resume,
 		.suspend = &marvell_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2186,6 +2217,8 @@ static struct phy_driver marvell_drivers[] = {
 		.did_interrupt = &m88e1121_did_interrupt,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2205,6 +2238,8 @@ static struct phy_driver marvell_drivers[] = {
 		.did_interrupt = &m88e1121_did_interrupt,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2216,7 +2251,6 @@ static struct phy_driver marvell_drivers[] = {
 		.features = PHY_BASIC_FEATURES,
 		.flags = PHY_HAS_INTERRUPT,
 		.probe = marvell_probe,
-		.config_aneg = &genphy_config_aneg,
 		.config_init = &m88e3016_config_init,
 		.aneg_done = &marvell_aneg_done,
 		.read_status = &marvell_read_status,
@@ -2225,6 +2259,8 @@ static struct phy_driver marvell_drivers[] = {
 		.did_interrupt = &m88e1121_did_interrupt,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
@@ -2235,7 +2271,7 @@ static struct phy_driver marvell_drivers[] = {
 		.name = "Marvell 88E6390",
 		.features = PHY_GBIT_FEATURES,
 		.flags = PHY_HAS_INTERRUPT,
-		.probe = m88e1510_probe,
+		.probe = m88e6390_probe,
 		.config_init = &marvell_config_init,
 		.config_aneg = &m88e1510_config_aneg,
 		.read_status = &marvell_read_status,
@@ -2244,6 +2280,8 @@ static struct phy_driver marvell_drivers[] = {
 		.did_interrupt = &m88e1121_did_interrupt,
 		.resume = &genphy_resume,
 		.suspend = &genphy_suspend,
+		.read_page = marvell_read_page,
+		.write_page = marvell_write_page,
 		.get_sset_count = marvell_get_sset_count,
 		.get_strings = marvell_get_strings,
 		.get_stats = marvell_get_stats,
diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c
index 21b3f36e023a..8a0bd98fdec7 100644
--- a/drivers/net/phy/marvell10g.c
+++ b/drivers/net/phy/marvell10g.c
@@ -6,12 +6,18 @@
  *
  * There appears to be several different data paths through the PHY which
  * are automatically managed by the PHY.  The following has been determined
- * via observation and experimentation:
+ * via observation and experimentation for a setup using single-lane Serdes:
  *
  *       SGMII PHYXS -- BASE-T PCS -- 10G PMA -- AN -- Copper (for <= 1G)
  *  10GBASE-KR PHYXS -- BASE-T PCS -- 10G PMA -- AN -- Copper (for 10G)
  *  10GBASE-KR PHYXS -- BASE-R PCS -- Fiber
  *
+ * With XAUI, observation shows:
+ *
+ *        XAUI PHYXS -- <appropriate PCS as above>
+ *
+ * and no switching of the host interface mode occurs.
+ *
  * If both the fiber and copper ports are connected, the first to gain
  * link takes priority and the other port is completely locked out.
  */
@@ -23,19 +29,17 @@ enum {
 	MV_PCS_BASE_R		= 0x1000,
 	MV_PCS_1000BASEX	= 0x2000,
 
+	MV_PCS_PAIRSWAP		= 0x8182,
+	MV_PCS_PAIRSWAP_MASK	= 0x0003,
+	MV_PCS_PAIRSWAP_AB	= 0x0002,
+	MV_PCS_PAIRSWAP_NONE	= 0x0003,
+
 	/* These registers appear at 0x800X and 0xa00X - the 0xa00X control
 	 * registers appear to set themselves to the 0x800X when AN is
 	 * restarted, but status registers appear readable from either.
 	 */
 	MV_AN_CTRL1000		= 0x8000, /* 1000base-T control register */
 	MV_AN_STAT1000		= 0x8001, /* 1000base-T status register */
-
-	/* This register appears to reflect the copper status */
-	MV_AN_RESULT		= 0xa016,
-	MV_AN_RESULT_SPD_10	= BIT(12),
-	MV_AN_RESULT_SPD_100	= BIT(13),
-	MV_AN_RESULT_SPD_1000	= BIT(14),
-	MV_AN_RESULT_SPD_10000	= BIT(15),
 };
 
 static int mv3310_modify(struct phy_device *phydev, int devad, u16 reg,
@@ -84,7 +88,6 @@ static int mv3310_config_init(struct phy_device *phydev)
 
 	/* Check that the PHY interface type is compatible */
 	if (phydev->interface != PHY_INTERFACE_MODE_SGMII &&
-	    phydev->interface != PHY_INTERFACE_MODE_XGMII &&
 	    phydev->interface != PHY_INTERFACE_MODE_XAUI &&
 	    phydev->interface != PHY_INTERFACE_MODE_RXAUI &&
 	    phydev->interface != PHY_INTERFACE_MODE_10GKR)
@@ -150,12 +153,18 @@ static int mv3310_config_init(struct phy_device *phydev)
 		if (val & MDIO_PMA_EXTABLE_1000BKX)
 			__set_bit(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT,
 				  supported);
-		if (val & MDIO_PMA_EXTABLE_100BTX)
+		if (val & MDIO_PMA_EXTABLE_100BTX) {
 			__set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT,
 				  supported);
-		if (val & MDIO_PMA_EXTABLE_10BT)
+			__set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT,
+				  supported);
+		}
+		if (val & MDIO_PMA_EXTABLE_10BT) {
 			__set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT,
 				  supported);
+			__set_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT,
+				  supported);
+		}
 	}
 
 	if (!ethtool_convert_link_mode_to_legacy_u32(&mask, supported))
@@ -175,6 +184,9 @@ static int mv3310_config_aneg(struct phy_device *phydev)
 	u32 advertising;
 	int ret;
 
+	/* We don't support manual MDI control */
+	phydev->mdix_ctrl = ETH_TP_MDI_AUTO;
+
 	if (phydev->autoneg == AUTONEG_DISABLE) {
 		ret = genphy_c45_pma_setup_forced(phydev);
 		if (ret < 0)
@@ -233,6 +245,24 @@ static int mv3310_aneg_done(struct phy_device *phydev)
 	return genphy_c45_aneg_done(phydev);
 }
 
+static void mv3310_update_interface(struct phy_device *phydev)
+{
+	if ((phydev->interface == PHY_INTERFACE_MODE_SGMII ||
+	     phydev->interface == PHY_INTERFACE_MODE_10GKR) && phydev->link) {
+		/* The PHY automatically switches its serdes interface (and
+		 * active PHYXS instance) between Cisco SGMII and 10GBase-KR
+		 * modes according to the speed.  Florian suggests setting
+		 * phydev->interface to communicate this to the MAC. Only do
+		 * this if we are already in either SGMII or 10GBase-KR mode.
+		 */
+		if (phydev->speed == SPEED_10000)
+			phydev->interface = PHY_INTERFACE_MODE_10GKR;
+		else if (phydev->speed >= SPEED_10 &&
+			 phydev->speed < SPEED_10000)
+			phydev->interface = PHY_INTERFACE_MODE_SGMII;
+	}
+}
+
 /* 10GBASE-ER,LR,LRM,SR do not support autonegotiation. */
 static int mv3310_read_10gbr_status(struct phy_device *phydev)
 {
@@ -240,8 +270,7 @@ static int mv3310_read_10gbr_status(struct phy_device *phydev)
 	phydev->speed = SPEED_10000;
 	phydev->duplex = DUPLEX_FULL;
 
-	if (phydev->interface == PHY_INTERFACE_MODE_SGMII)
-		phydev->interface = PHY_INTERFACE_MODE_10GKR;
+	mv3310_update_interface(phydev);
 
 	return 0;
 }
@@ -264,6 +293,7 @@ static int mv3310_read_status(struct phy_device *phydev)
 	phydev->link = 0;
 	phydev->pause = 0;
 	phydev->asym_pause = 0;
+	phydev->mdix = 0;
 
 	val = phy_read_mmd(phydev, MDIO_MMD_PCS, MV_PCS_BASE_R + MDIO_STAT1);
 	if (val < 0)
@@ -294,22 +324,8 @@ static int mv3310_read_status(struct phy_device *phydev)
 
 		phydev->lp_advertising |= mii_stat1000_to_ethtool_lpa_t(val);
 
-		if (phydev->autoneg == AUTONEG_ENABLE) {
-			val = phy_read_mmd(phydev, MDIO_MMD_AN, MV_AN_RESULT);
-			if (val < 0)
-				return val;
-
-			if (val & MV_AN_RESULT_SPD_10000)
-				phydev->speed = SPEED_10000;
-			else if (val & MV_AN_RESULT_SPD_1000)
-				phydev->speed = SPEED_1000;
-			else if (val & MV_AN_RESULT_SPD_100)
-				phydev->speed = SPEED_100;
-			else if (val & MV_AN_RESULT_SPD_10)
-				phydev->speed = SPEED_10;
-
-			phydev->duplex = DUPLEX_FULL;
-		}
+		if (phydev->autoneg == AUTONEG_ENABLE)
+			phy_resolve_aneg_linkmode(phydev);
 	}
 
 	if (phydev->autoneg != AUTONEG_ENABLE) {
@@ -318,21 +334,30 @@ static int mv3310_read_status(struct phy_device *phydev)
 			return val;
 	}
 
-	if ((phydev->interface == PHY_INTERFACE_MODE_SGMII ||
-	     phydev->interface == PHY_INTERFACE_MODE_10GKR) && phydev->link) {
-		/* The PHY automatically switches its serdes interface (and
-		 * active PHYXS instance) between Cisco SGMII and 10GBase-KR
-		 * modes according to the speed.  Florian suggests setting
-		 * phydev->interface to communicate this to the MAC. Only do
-		 * this if we are already in either SGMII or 10GBase-KR mode.
-		 */
-		if (phydev->speed == SPEED_10000)
-			phydev->interface = PHY_INTERFACE_MODE_10GKR;
-		else if (phydev->speed >= SPEED_10 &&
-			 phydev->speed < SPEED_10000)
-			phydev->interface = PHY_INTERFACE_MODE_SGMII;
+	if (phydev->speed == SPEED_10000) {
+		val = genphy_c45_read_mdix(phydev);
+		if (val < 0)
+			return val;
+	} else {
+		val = phy_read_mmd(phydev, MDIO_MMD_PCS, MV_PCS_PAIRSWAP);
+		if (val < 0)
+			return val;
+
+		switch (val & MV_PCS_PAIRSWAP_MASK) {
+		case MV_PCS_PAIRSWAP_AB:
+			phydev->mdix = ETH_TP_MDI_X;
+			break;
+		case MV_PCS_PAIRSWAP_NONE:
+			phydev->mdix = ETH_TP_MDI;
+			break;
+		default:
+			phydev->mdix = ETH_TP_MDI_INVALID;
+			break;
+		}
 	}
 
+	mv3310_update_interface(phydev);
+
 	return 0;
 }
 
@@ -342,7 +367,9 @@ static struct phy_driver mv3310_drivers[] = {
 		.phy_id_mask	= MARVELL_PHY_ID_MASK,
 		.name		= "mv88x3310",
 		.features	= SUPPORTED_10baseT_Full |
+				  SUPPORTED_10baseT_Half |
 				  SUPPORTED_100baseT_Full |
+				  SUPPORTED_100baseT_Half |
 				  SUPPORTED_1000baseT_Full |
 				  SUPPORTED_Autoneg |
 				  SUPPORTED_TP |
diff --git a/drivers/net/phy/mdio-bcm-unimac.c b/drivers/net/phy/mdio-bcm-unimac.c
index 08e0647b85e2..8d370667fa1b 100644
--- a/drivers/net/phy/mdio-bcm-unimac.c
+++ b/drivers/net/phy/mdio-bcm-unimac.c
@@ -205,6 +205,8 @@ static int unimac_mdio_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!r)
+		return -EINVAL;
 
 	/* Just ioremap, as this MDIO block is usually integrated into an
 	 * Ethernet MAC controller register range
diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c
index 54d00a1d2bef..24b5511222c8 100644
--- a/drivers/net/phy/mdio_bus.c
+++ b/drivers/net/phy/mdio_bus.c
@@ -38,6 +38,7 @@
 #include <linux/phy.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
+#include <linux/gpio/consumer.h>
 
 #include <asm/irq.h>
 
@@ -46,11 +47,42 @@
 
 #include "mdio-boardinfo.h"
 
+static int mdiobus_register_gpiod(struct mdio_device *mdiodev)
+{
+	struct gpio_desc *gpiod = NULL;
+
+	/* Deassert the optional reset signal */
+	if (mdiodev->dev.of_node)
+		gpiod = fwnode_get_named_gpiod(&mdiodev->dev.of_node->fwnode,
+					       "reset-gpios", 0, GPIOD_OUT_LOW,
+					       "PHY reset");
+	if (PTR_ERR(gpiod) == -ENOENT ||
+	    PTR_ERR(gpiod) == -ENOSYS)
+		gpiod = NULL;
+	else if (IS_ERR(gpiod))
+		return PTR_ERR(gpiod);
+
+	mdiodev->reset = gpiod;
+
+	/* Assert the reset signal again */
+	mdio_device_reset(mdiodev, 1);
+
+	return 0;
+}
+
 int mdiobus_register_device(struct mdio_device *mdiodev)
 {
+	int err;
+
 	if (mdiodev->bus->mdio_map[mdiodev->addr])
 		return -EBUSY;
 
+	if (mdiodev->flags & MDIO_DEVICE_FLAG_PHY) {
+		err = mdiobus_register_gpiod(mdiodev);
+		if (err)
+			return err;
+	}
+
 	mdiodev->bus->mdio_map[mdiodev->addr] = mdiodev;
 
 	return 0;
@@ -421,6 +453,9 @@ void mdiobus_unregister(struct mii_bus *bus)
 		if (!mdiodev)
 			continue;
 
+		if (mdiodev->reset)
+			gpiod_put(mdiodev->reset);
+
 		mdiodev->device_remove(mdiodev);
 		mdiodev->device_free(mdiodev);
 	}
@@ -493,6 +528,55 @@ struct phy_device *mdiobus_scan(struct mii_bus *bus, int addr)
 }
 EXPORT_SYMBOL(mdiobus_scan);
 
+/**
+ * __mdiobus_read - Unlocked version of the mdiobus_read function
+ * @bus: the mii_bus struct
+ * @addr: the phy address
+ * @regnum: register number to read
+ *
+ * Read a MDIO bus register. Caller must hold the mdio bus lock.
+ *
+ * NOTE: MUST NOT be called from interrupt context.
+ */
+int __mdiobus_read(struct mii_bus *bus, int addr, u32 regnum)
+{
+	int retval;
+
+	WARN_ON_ONCE(!mutex_is_locked(&bus->mdio_lock));
+
+	retval = bus->read(bus, addr, regnum);
+
+	trace_mdio_access(bus, 1, addr, regnum, retval, retval);
+
+	return retval;
+}
+EXPORT_SYMBOL(__mdiobus_read);
+
+/**
+ * __mdiobus_write - Unlocked version of the mdiobus_write function
+ * @bus: the mii_bus struct
+ * @addr: the phy address
+ * @regnum: register number to write
+ * @val: value to write to @regnum
+ *
+ * Write a MDIO bus register. Caller must hold the mdio bus lock.
+ *
+ * NOTE: MUST NOT be called from interrupt context.
+ */
+int __mdiobus_write(struct mii_bus *bus, int addr, u32 regnum, u16 val)
+{
+	int err;
+
+	WARN_ON_ONCE(!mutex_is_locked(&bus->mdio_lock));
+
+	err = bus->write(bus, addr, regnum, val);
+
+	trace_mdio_access(bus, 0, addr, regnum, val, err);
+
+	return err;
+}
+EXPORT_SYMBOL(__mdiobus_write);
+
 /**
  * mdiobus_read_nested - Nested version of the mdiobus_read function
  * @bus: the mii_bus struct
@@ -513,11 +597,9 @@ int mdiobus_read_nested(struct mii_bus *bus, int addr, u32 regnum)
 	BUG_ON(in_interrupt());
 
 	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
-	retval = bus->read(bus, addr, regnum);
+	retval = __mdiobus_read(bus, addr, regnum);
 	mutex_unlock(&bus->mdio_lock);
 
-	trace_mdio_access(bus, 1, addr, regnum, retval, retval);
-
 	return retval;
 }
 EXPORT_SYMBOL(mdiobus_read_nested);
@@ -539,11 +621,9 @@ int mdiobus_read(struct mii_bus *bus, int addr, u32 regnum)
 	BUG_ON(in_interrupt());
 
 	mutex_lock(&bus->mdio_lock);
-	retval = bus->read(bus, addr, regnum);
+	retval = __mdiobus_read(bus, addr, regnum);
 	mutex_unlock(&bus->mdio_lock);
 
-	trace_mdio_access(bus, 1, addr, regnum, retval, retval);
-
 	return retval;
 }
 EXPORT_SYMBOL(mdiobus_read);
@@ -569,11 +649,9 @@ int mdiobus_write_nested(struct mii_bus *bus, int addr, u32 regnum, u16 val)
 	BUG_ON(in_interrupt());
 
 	mutex_lock_nested(&bus->mdio_lock, MDIO_MUTEX_NESTED);
-	err = bus->write(bus, addr, regnum, val);
+	err = __mdiobus_write(bus, addr, regnum, val);
 	mutex_unlock(&bus->mdio_lock);
 
-	trace_mdio_access(bus, 0, addr, regnum, val, err);
-
 	return err;
 }
 EXPORT_SYMBOL(mdiobus_write_nested);
@@ -596,11 +674,9 @@ int mdiobus_write(struct mii_bus *bus, int addr, u32 regnum, u16 val)
 	BUG_ON(in_interrupt());
 
 	mutex_lock(&bus->mdio_lock);
-	err = bus->write(bus, addr, regnum, val);
+	err = __mdiobus_write(bus, addr, regnum, val);
 	mutex_unlock(&bus->mdio_lock);
 
-	trace_mdio_access(bus, 0, addr, regnum, val, err);
-
 	return err;
 }
 EXPORT_SYMBOL(mdiobus_write);
diff --git a/drivers/net/phy/mdio_device.c b/drivers/net/phy/mdio_device.c
index e24f28924af8..c924700cf37b 100644
--- a/drivers/net/phy/mdio_device.c
+++ b/drivers/net/phy/mdio_device.c
@@ -12,6 +12,8 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/errno.h>
+#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
@@ -22,6 +24,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/unistd.h>
+#include <linux/delay.h>
 
 void mdio_device_free(struct mdio_device *mdiodev)
 {
@@ -114,6 +117,21 @@ void mdio_device_remove(struct mdio_device *mdiodev)
 }
 EXPORT_SYMBOL(mdio_device_remove);
 
+void mdio_device_reset(struct mdio_device *mdiodev, int value)
+{
+	unsigned int d;
+
+	if (!mdiodev->reset)
+		return;
+
+	gpiod_set_value(mdiodev->reset, value);
+
+	d = value ? mdiodev->reset_assert_delay : mdiodev->reset_deassert_delay;
+	if (d)
+		usleep_range(d, d + max_t(unsigned int, d / 10, 100));
+}
+EXPORT_SYMBOL(mdio_device_reset);
+
 /**
  * mdio_probe - probe an MDIO device
  * @dev: device to probe
@@ -128,8 +146,16 @@ static int mdio_probe(struct device *dev)
 	struct mdio_driver *mdiodrv = to_mdio_driver(drv);
 	int err = 0;
 
-	if (mdiodrv->probe)
+	if (mdiodrv->probe) {
+		/* Deassert the reset signal */
+		mdio_device_reset(mdiodev, 0);
+
 		err = mdiodrv->probe(mdiodev);
+		if (err) {
+			/* Assert the reset signal */
+			mdio_device_reset(mdiodev, 1);
+		}
+	}
 
 	return err;
 }
@@ -140,9 +166,13 @@ static int mdio_remove(struct device *dev)
 	struct device_driver *drv = mdiodev->dev.driver;
 	struct mdio_driver *mdiodrv = to_mdio_driver(drv);
 
-	if (mdiodrv->remove)
+	if (mdiodrv->remove) {
 		mdiodrv->remove(mdiodev);
 
+		/* Assert the reset signal */
+		mdio_device_reset(mdiodev, 1);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/phy/meson-gxl.c b/drivers/net/phy/meson-gxl.c
index 842eb871a6e3..ddc2c5ea3787 100644
--- a/drivers/net/phy/meson-gxl.c
+++ b/drivers/net/phy/meson-gxl.c
@@ -24,31 +24,129 @@
 #include <linux/netdevice.h>
 #include <linux/bitfield.h>
 
-static int meson_gxl_config_init(struct phy_device *phydev)
+#define TSTCNTL		20
+#define  TSTCNTL_READ		BIT(15)
+#define  TSTCNTL_WRITE		BIT(14)
+#define  TSTCNTL_REG_BANK_SEL	GENMASK(12, 11)
+#define  TSTCNTL_TEST_MODE	BIT(10)
+#define  TSTCNTL_READ_ADDRESS	GENMASK(9, 5)
+#define  TSTCNTL_WRITE_ADDRESS	GENMASK(4, 0)
+#define TSTREAD1	21
+#define TSTWRITE	23
+#define INTSRC_FLAG	29
+#define  INTSRC_ANEG_PR		BIT(1)
+#define  INTSRC_PARALLEL_FAULT	BIT(2)
+#define  INTSRC_ANEG_LP_ACK	BIT(3)
+#define  INTSRC_LINK_DOWN	BIT(4)
+#define  INTSRC_REMOTE_FAULT	BIT(5)
+#define  INTSRC_ANEG_COMPLETE	BIT(6)
+#define INTSRC_MASK	30
+
+#define BANK_ANALOG_DSP		0
+#define BANK_WOL		1
+#define BANK_BIST		3
+
+/* WOL Registers */
+#define LPI_STATUS	0xc
+#define  LPI_STATUS_RSV12	BIT(12)
+
+/* BIST Registers */
+#define FR_PLL_CONTROL	0x1b
+#define FR_PLL_DIV0	0x1c
+#define FR_PLL_DIV1	0x1d
+
+static int meson_gxl_open_banks(struct phy_device *phydev)
 {
-	/* Enable Analog and DSP register Bank access by */
-	phy_write(phydev, 0x14, 0x0000);
-	phy_write(phydev, 0x14, 0x0400);
-	phy_write(phydev, 0x14, 0x0000);
-	phy_write(phydev, 0x14, 0x0400);
+	int ret;
+
+	/* Enable Analog and DSP register Bank access by
+	 * toggling TSTCNTL_TEST_MODE bit in the TSTCNTL register
+	 */
+	ret = phy_write(phydev, TSTCNTL, 0);
+	if (ret)
+		return ret;
+	ret = phy_write(phydev, TSTCNTL, TSTCNTL_TEST_MODE);
+	if (ret)
+		return ret;
+	ret = phy_write(phydev, TSTCNTL, 0);
+	if (ret)
+		return ret;
+	return phy_write(phydev, TSTCNTL, TSTCNTL_TEST_MODE);
+}
+
+static void meson_gxl_close_banks(struct phy_device *phydev)
+{
+	phy_write(phydev, TSTCNTL, 0);
+}
+
+static int meson_gxl_read_reg(struct phy_device *phydev,
+			      unsigned int bank, unsigned int reg)
+{
+	int ret;
+
+	ret = meson_gxl_open_banks(phydev);
+	if (ret)
+		goto out;
+
+	ret = phy_write(phydev, TSTCNTL, TSTCNTL_READ |
+			FIELD_PREP(TSTCNTL_REG_BANK_SEL, bank) |
+			TSTCNTL_TEST_MODE |
+			FIELD_PREP(TSTCNTL_READ_ADDRESS, reg));
+	if (ret)
+		goto out;
+
+	ret = phy_read(phydev, TSTREAD1);
+out:
+	/* Close the bank access on our way out */
+	meson_gxl_close_banks(phydev);
+	return ret;
+}
+
+static int meson_gxl_write_reg(struct phy_device *phydev,
+			       unsigned int bank, unsigned int reg,
+			       uint16_t value)
+{
+	int ret;
+
+	ret = meson_gxl_open_banks(phydev);
+	if (ret)
+		goto out;
+
+	ret = phy_write(phydev, TSTWRITE, value);
+	if (ret)
+		goto out;
+
+	ret = phy_write(phydev, TSTCNTL, TSTCNTL_WRITE |
+			FIELD_PREP(TSTCNTL_REG_BANK_SEL, bank) |
+			TSTCNTL_TEST_MODE |
+			FIELD_PREP(TSTCNTL_WRITE_ADDRESS, reg));
 
-	/* Write Analog register 23 */
-	phy_write(phydev, 0x17, 0x8E0D);
-	phy_write(phydev, 0x14, 0x4417);
+out:
+	/* Close the bank access on our way out */
+	meson_gxl_close_banks(phydev);
+	return ret;
+}
+
+static int meson_gxl_config_init(struct phy_device *phydev)
+{
+	int ret;
 
 	/* Enable fractional PLL */
-	phy_write(phydev, 0x17, 0x0005);
-	phy_write(phydev, 0x14, 0x5C1B);
+	ret = meson_gxl_write_reg(phydev, BANK_BIST, FR_PLL_CONTROL, 0x5);
+	if (ret)
+		return ret;
 
 	/* Program fraction FR_PLL_DIV1 */
-	phy_write(phydev, 0x17, 0x029A);
-	phy_write(phydev, 0x14, 0x5C1D);
+	ret = meson_gxl_write_reg(phydev, BANK_BIST, FR_PLL_DIV1, 0x029a);
+	if (ret)
+		return ret;
 
 	/* Program fraction FR_PLL_DIV1 */
-	phy_write(phydev, 0x17, 0xAAAA);
-	phy_write(phydev, 0x14, 0x5C1C);
+	ret = meson_gxl_write_reg(phydev, BANK_BIST, FR_PLL_DIV0, 0xaaaa);
+	if (ret)
+		return ret;
 
-	return 0;
+	return genphy_config_init(phydev);
 }
 
 /* This function is provided to cope with the possible failures of this phy
@@ -78,27 +176,8 @@ static int meson_gxl_read_status(struct phy_device *phydev)
 		else if (!ret)
 			goto read_status_continue;
 
-		/* Need to access WOL bank, make sure the access is open */
-		ret = phy_write(phydev, 0x14, 0x0000);
-		if (ret)
-			return ret;
-		ret = phy_write(phydev, 0x14, 0x0400);
-		if (ret)
-			return ret;
-		ret = phy_write(phydev, 0x14, 0x0000);
-		if (ret)
-			return ret;
-		ret = phy_write(phydev, 0x14, 0x0400);
-		if (ret)
-			return ret;
-
-		/* Request LPI_STATUS WOL register */
-		ret = phy_write(phydev, 0x14, 0x8D80);
-		if (ret)
-			return ret;
-
-		/* Read LPI_STATUS value */
-		wol = phy_read(phydev, 0x15);
+		/* Aneg is done, let's check everything is fine */
+		wol = meson_gxl_read_reg(phydev, BANK_WOL, LPI_STATUS);
 		if (wol < 0)
 			return wol;
 
@@ -110,7 +189,7 @@ static int meson_gxl_read_status(struct phy_device *phydev)
 		if (exp < 0)
 			return exp;
 
-		if (!(wol & BIT(12)) ||
+		if (!(wol & LPI_STATUS_RSV12) ||
 		    ((exp & EXPANSION_NWAY) && !(lpa & LPA_LPACK))) {
 			/* Looks like aneg failed after all */
 			phydev_dbg(phydev, "LPA corruption - aneg restart\n");
@@ -122,17 +201,43 @@ static int meson_gxl_read_status(struct phy_device *phydev)
 	return genphy_read_status(phydev);
 }
 
+static int meson_gxl_ack_interrupt(struct phy_device *phydev)
+{
+	int ret = phy_read(phydev, INTSRC_FLAG);
+
+	return ret < 0 ? ret : 0;
+}
+
+static int meson_gxl_config_intr(struct phy_device *phydev)
+{
+	u16 val;
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		val = INTSRC_ANEG_PR
+			| INTSRC_PARALLEL_FAULT
+			| INTSRC_ANEG_LP_ACK
+			| INTSRC_LINK_DOWN
+			| INTSRC_REMOTE_FAULT
+			| INTSRC_ANEG_COMPLETE;
+	} else {
+		val = 0;
+	}
+
+	return phy_write(phydev, INTSRC_MASK, val);
+}
+
 static struct phy_driver meson_gxl_phy[] = {
 	{
 		.phy_id		= 0x01814400,
 		.phy_id_mask	= 0xfffffff0,
 		.name		= "Meson GXL Internal PHY",
 		.features	= PHY_BASIC_FEATURES,
-		.flags		= PHY_IS_INTERNAL,
+		.flags		= PHY_IS_INTERNAL | PHY_HAS_INTERRUPT,
 		.config_init	= meson_gxl_config_init,
-		.config_aneg	= genphy_config_aneg,
 		.aneg_done      = genphy_aneg_done,
 		.read_status	= meson_gxl_read_status,
+		.ack_interrupt	= meson_gxl_ack_interrupt,
+		.config_intr	= meson_gxl_config_intr,
 		.suspend        = genphy_suspend,
 		.resume         = genphy_resume,
 	},
@@ -150,4 +255,5 @@ MODULE_DEVICE_TABLE(mdio, meson_gxl_tbl);
 MODULE_DESCRIPTION("Amlogic Meson GXL Internal PHY driver");
 MODULE_AUTHOR("Baoqi wang");
 MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index 422ff6333c52..f41b224a9cdb 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -635,25 +635,6 @@ static int ksz8873mll_config_aneg(struct phy_device *phydev)
 	return 0;
 }
 
-/* This routine returns -1 as an indication to the caller that the
- * Micrel ksz9021 10/100/1000 PHY does not support standard IEEE
- * MMD extended PHY registers.
- */
-static int
-ksz9021_rd_mmd_phyreg(struct phy_device *phydev, int devad, u16 regnum)
-{
-	return -1;
-}
-
-/* This routine does nothing since the Micrel ksz9021 does not support
- * standard IEEE MMD extended PHY registers.
- */
-static int
-ksz9021_wr_mmd_phyreg(struct phy_device *phydev, int devad, u16 regnum, u16 val)
-{
-	return -1;
-}
-
 static int kszphy_get_sset_count(struct phy_device *phydev)
 {
 	return ARRAY_SIZE(kszphy_hw_stats);
@@ -664,8 +645,8 @@ static void kszphy_get_strings(struct phy_device *phydev, u8 *data)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(kszphy_hw_stats); i++) {
-		memcpy(data + i * ETH_GSTRING_LEN,
-		       kszphy_hw_stats[i].string, ETH_GSTRING_LEN);
+		strlcpy(data + i * ETH_GSTRING_LEN,
+			kszphy_hw_stats[i].string, ETH_GSTRING_LEN);
 	}
 }
 
@@ -802,8 +783,6 @@ static struct phy_driver ksphy_driver[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.driver_data	= &ks8737_type,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.suspend	= genphy_suspend,
@@ -817,8 +796,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz8021_type,
 	.probe		= kszphy_probe,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -835,8 +812,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz8021_type,
 	.probe		= kszphy_probe,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -854,7 +829,6 @@ static struct phy_driver ksphy_driver[] = {
 	.probe		= kszphy_probe,
 	.config_init	= ksz8041_config_init,
 	.config_aneg	= ksz8041_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -871,8 +845,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz8041_type,
 	.probe		= kszphy_probe,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -889,8 +861,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz8051_type,
 	.probe		= kszphy_probe,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -907,8 +877,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz8041_type,
 	.probe		= kszphy_probe,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -925,8 +893,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz8081_type,
 	.probe		= kszphy_probe,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -941,8 +907,6 @@ static struct phy_driver ksphy_driver[] = {
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.suspend	= genphy_suspend,
@@ -956,8 +920,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz9021_type,
 	.probe		= kszphy_probe,
 	.config_init	= ksz9021_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
 	.get_sset_count = kszphy_get_sset_count,
@@ -965,8 +927,8 @@ static struct phy_driver ksphy_driver[] = {
 	.get_stats	= kszphy_get_stats,
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
-	.read_mmd	= ksz9021_rd_mmd_phyreg,
-	.write_mmd	= ksz9021_wr_mmd_phyreg,
+	.read_mmd	= genphy_read_mmd_unsupported,
+	.write_mmd	= genphy_write_mmd_unsupported,
 }, {
 	.phy_id		= PHY_ID_KSZ9031,
 	.phy_id_mask	= MICREL_PHY_ID_MASK,
@@ -976,7 +938,6 @@ static struct phy_driver ksphy_driver[] = {
 	.driver_data	= &ksz9021_type,
 	.probe		= kszphy_probe,
 	.config_init	= ksz9031_config_init,
-	.config_aneg	= genphy_config_aneg,
 	.read_status	= ksz9031_read_status,
 	.ack_interrupt	= kszphy_ack_interrupt,
 	.config_intr	= kszphy_config_intr,
@@ -1001,8 +962,6 @@ static struct phy_driver ksphy_driver[] = {
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 }, {
@@ -1022,8 +981,6 @@ static struct phy_driver ksphy_driver[] = {
 	.name		= "Microchip KSZ9477",
 	.features	= PHY_GBIT_FEATURES,
 	.config_init	= kszphy_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 } };
diff --git a/drivers/net/phy/microchip.c b/drivers/net/phy/microchip.c
index 37ee856c7680..0f293ef28935 100644
--- a/drivers/net/phy/microchip.c
+++ b/drivers/net/phy/microchip.c
@@ -153,7 +153,6 @@ static struct phy_driver microchip_phy_driver[] = {
 
 	.config_init	= genphy_config_init,
 	.config_aneg	= lan88xx_config_aneg,
-	.read_status	= genphy_read_status,
 
 	.ack_interrupt	= lan88xx_phy_ack_interrupt,
 	.config_intr	= lan88xx_phy_config_intr,
diff --git a/drivers/net/phy/national.c b/drivers/net/phy/national.c
index 2addf1d3f619..2b1e336961f9 100644
--- a/drivers/net/phy/national.c
+++ b/drivers/net/phy/national.c
@@ -136,8 +136,6 @@ static struct phy_driver dp83865_driver[] = { {
 	.features = PHY_GBIT_FEATURES,
 	.flags = PHY_HAS_INTERRUPT,
 	.config_init = ns_config_init,
-	.config_aneg = genphy_config_aneg,
-	.read_status = genphy_read_status,
 	.ack_interrupt = ns_ack_interrupt,
 	.config_intr = ns_config_intr,
 } };
diff --git a/drivers/net/phy/phy-c45.c b/drivers/net/phy/phy-c45.c
index dada819c6b78..a4576859afae 100644
--- a/drivers/net/phy/phy-c45.c
+++ b/drivers/net/phy/phy-c45.c
@@ -233,6 +233,39 @@ int genphy_c45_read_pma(struct phy_device *phydev)
 }
 EXPORT_SYMBOL_GPL(genphy_c45_read_pma);
 
+/**
+ * genphy_c45_read_mdix - read mdix status from PMA
+ * @phydev: target phy_device struct
+ */
+int genphy_c45_read_mdix(struct phy_device *phydev)
+{
+	int val;
+
+	if (phydev->speed == SPEED_10000) {
+		val = phy_read_mmd(phydev, MDIO_MMD_PMAPMD,
+				   MDIO_PMA_10GBT_SWAPPOL);
+		if (val < 0)
+			return val;
+
+		switch (val) {
+		case MDIO_PMA_10GBT_SWAPPOL_ABNX | MDIO_PMA_10GBT_SWAPPOL_CDNX:
+			phydev->mdix = ETH_TP_MDI;
+			break;
+
+		case 0:
+			phydev->mdix = ETH_TP_MDI_X;
+			break;
+
+		default:
+			phydev->mdix = ETH_TP_MDI_INVALID;
+			break;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(genphy_c45_read_mdix);
+
 /* The gen10g_* functions are the old Clause 45 stub */
 
 static int gen10g_config_aneg(struct phy_device *phydev)
diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c
index 21f75ae244b3..4083f00c97a5 100644
--- a/drivers/net/phy/phy-core.c
+++ b/drivers/net/phy/phy-core.c
@@ -189,17 +189,61 @@ size_t phy_speeds(unsigned int *speeds, size_t size,
 	return count;
 }
 
+/**
+ * phy_resolve_aneg_linkmode - resolve the advertisments into phy settings
+ * @phydev: The phy_device struct
+ *
+ * Resolve our and the link partner advertisments into their corresponding
+ * speed and duplex. If full duplex was negotiated, extract the pause mode
+ * from the link partner mask.
+ */
+void phy_resolve_aneg_linkmode(struct phy_device *phydev)
+{
+	u32 common = phydev->lp_advertising & phydev->advertising;
+
+	if (common & ADVERTISED_10000baseT_Full) {
+		phydev->speed = SPEED_10000;
+		phydev->duplex = DUPLEX_FULL;
+	} else if (common & ADVERTISED_1000baseT_Full) {
+		phydev->speed = SPEED_1000;
+		phydev->duplex = DUPLEX_FULL;
+	} else if (common & ADVERTISED_1000baseT_Half) {
+		phydev->speed = SPEED_1000;
+		phydev->duplex = DUPLEX_HALF;
+	} else if (common & ADVERTISED_100baseT_Full) {
+		phydev->speed = SPEED_100;
+		phydev->duplex = DUPLEX_FULL;
+	} else if (common & ADVERTISED_100baseT_Half) {
+		phydev->speed = SPEED_100;
+		phydev->duplex = DUPLEX_HALF;
+	} else if (common & ADVERTISED_10baseT_Full) {
+		phydev->speed = SPEED_10;
+		phydev->duplex = DUPLEX_FULL;
+	} else if (common & ADVERTISED_10baseT_Half) {
+		phydev->speed = SPEED_10;
+		phydev->duplex = DUPLEX_HALF;
+	}
+
+	if (phydev->duplex == DUPLEX_FULL) {
+		phydev->pause = !!(phydev->lp_advertising & ADVERTISED_Pause);
+		phydev->asym_pause = !!(phydev->lp_advertising &
+					ADVERTISED_Asym_Pause);
+	}
+}
+EXPORT_SYMBOL_GPL(phy_resolve_aneg_linkmode);
+
 static void mmd_phy_indirect(struct mii_bus *bus, int phy_addr, int devad,
 			     u16 regnum)
 {
 	/* Write the desired MMD Devad */
-	bus->write(bus, phy_addr, MII_MMD_CTRL, devad);
+	__mdiobus_write(bus, phy_addr, MII_MMD_CTRL, devad);
 
 	/* Write the desired MMD register address */
-	bus->write(bus, phy_addr, MII_MMD_DATA, regnum);
+	__mdiobus_write(bus, phy_addr, MII_MMD_DATA, regnum);
 
 	/* Select the Function : DATA with no post increment */
-	bus->write(bus, phy_addr, MII_MMD_CTRL, devad | MII_MMD_CTRL_NOINCR);
+	__mdiobus_write(bus, phy_addr, MII_MMD_CTRL,
+			devad | MII_MMD_CTRL_NOINCR);
 }
 
 /**
@@ -232,7 +276,7 @@ int phy_read_mmd(struct phy_device *phydev, int devad, u32 regnum)
 		mmd_phy_indirect(bus, phy_addr, devad, regnum);
 
 		/* Read the content of the MMD's selected register */
-		val = bus->read(bus, phy_addr, MII_MMD_DATA);
+		val = __mdiobus_read(bus, phy_addr, MII_MMD_DATA);
 		mutex_unlock(&bus->mdio_lock);
 	}
 	return val;
@@ -271,7 +315,7 @@ int phy_write_mmd(struct phy_device *phydev, int devad, u32 regnum, u16 val)
 		mmd_phy_indirect(bus, phy_addr, devad, regnum);
 
 		/* Write the data into MMD's selected register */
-		bus->write(bus, phy_addr, MII_MMD_DATA, val);
+		__mdiobus_write(bus, phy_addr, MII_MMD_DATA, val);
 		mutex_unlock(&bus->mdio_lock);
 
 		ret = 0;
@@ -279,3 +323,207 @@ int phy_write_mmd(struct phy_device *phydev, int devad, u32 regnum, u16 val)
 	return ret;
 }
 EXPORT_SYMBOL(phy_write_mmd);
+
+/**
+ * __phy_modify() - Convenience function for modifying a PHY register
+ * @phydev: a pointer to a &struct phy_device
+ * @regnum: register number
+ * @mask: bit mask of bits to clear
+ * @set: bit mask of bits to set
+ *
+ * Unlocked helper function which allows a PHY register to be modified as
+ * new register value = (old register value & ~mask) | set
+ */
+int __phy_modify(struct phy_device *phydev, u32 regnum, u16 mask, u16 set)
+{
+	int ret;
+
+	ret = __phy_read(phydev, regnum);
+	if (ret < 0)
+		return ret;
+
+	ret = __phy_write(phydev, regnum, (ret & ~mask) | set);
+
+	return ret < 0 ? ret : 0;
+}
+EXPORT_SYMBOL_GPL(__phy_modify);
+
+/**
+ * phy_modify - Convenience function for modifying a given PHY register
+ * @phydev: the phy_device struct
+ * @regnum: register number to write
+ * @mask: bit mask of bits to clear
+ * @set: new value of bits set in mask to write to @regnum
+ *
+ * NOTE: MUST NOT be called from interrupt context,
+ * because the bus read/write functions may wait for an interrupt
+ * to conclude the operation.
+ */
+int phy_modify(struct phy_device *phydev, u32 regnum, u16 mask, u16 set)
+{
+	int ret;
+
+	mutex_lock(&phydev->mdio.bus->mdio_lock);
+	ret = __phy_modify(phydev, regnum, mask, set);
+	mutex_unlock(&phydev->mdio.bus->mdio_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phy_modify);
+
+static int __phy_read_page(struct phy_device *phydev)
+{
+	return phydev->drv->read_page(phydev);
+}
+
+static int __phy_write_page(struct phy_device *phydev, int page)
+{
+	return phydev->drv->write_page(phydev, page);
+}
+
+/**
+ * phy_save_page() - take the bus lock and save the current page
+ * @phydev: a pointer to a &struct phy_device
+ *
+ * Take the MDIO bus lock, and return the current page number. On error,
+ * returns a negative errno. phy_restore_page() must always be called
+ * after this, irrespective of success or failure of this call.
+ */
+int phy_save_page(struct phy_device *phydev)
+{
+	mutex_lock(&phydev->mdio.bus->mdio_lock);
+	return __phy_read_page(phydev);
+}
+EXPORT_SYMBOL_GPL(phy_save_page);
+
+/**
+ * phy_select_page() - take the bus lock, save the current page, and set a page
+ * @phydev: a pointer to a &struct phy_device
+ * @page: desired page
+ *
+ * Take the MDIO bus lock to protect against concurrent access, save the
+ * current PHY page, and set the current page.  On error, returns a
+ * negative errno, otherwise returns the previous page number.
+ * phy_restore_page() must always be called after this, irrespective
+ * of success or failure of this call.
+ */
+int phy_select_page(struct phy_device *phydev, int page)
+{
+	int ret, oldpage;
+
+	oldpage = ret = phy_save_page(phydev);
+	if (ret < 0)
+		return ret;
+
+	if (oldpage != page) {
+		ret = __phy_write_page(phydev, page);
+		if (ret < 0)
+			return ret;
+	}
+
+	return oldpage;
+}
+EXPORT_SYMBOL_GPL(phy_select_page);
+
+/**
+ * phy_restore_page() - restore the page register and release the bus lock
+ * @phydev: a pointer to a &struct phy_device
+ * @oldpage: the old page, return value from phy_save_page() or phy_select_page()
+ * @ret: operation's return code
+ *
+ * Release the MDIO bus lock, restoring @oldpage if it is a valid page.
+ * This function propagates the earliest error code from the group of
+ * operations.
+ *
+ * Returns:
+ *   @oldpage if it was a negative value, otherwise
+ *   @ret if it was a negative errno value, otherwise
+ *   phy_write_page()'s negative value if it were in error, otherwise
+ *   @ret.
+ */
+int phy_restore_page(struct phy_device *phydev, int oldpage, int ret)
+{
+	int r;
+
+	if (oldpage >= 0) {
+		r = __phy_write_page(phydev, oldpage);
+
+		/* Propagate the operation return code if the page write
+		 * was successful.
+		 */
+		if (ret >= 0 && r < 0)
+			ret = r;
+	} else {
+		/* Propagate the phy page selection error code */
+		ret = oldpage;
+	}
+
+	mutex_unlock(&phydev->mdio.bus->mdio_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phy_restore_page);
+
+/**
+ * phy_read_paged() - Convenience function for reading a paged register
+ * @phydev: a pointer to a &struct phy_device
+ * @page: the page for the phy
+ * @regnum: register number
+ *
+ * Same rules as for phy_read().
+ */
+int phy_read_paged(struct phy_device *phydev, int page, u32 regnum)
+{
+	int ret = 0, oldpage;
+
+	oldpage = phy_select_page(phydev, page);
+	if (oldpage >= 0)
+		ret = __phy_read(phydev, regnum);
+
+	return phy_restore_page(phydev, oldpage, ret);
+}
+EXPORT_SYMBOL(phy_read_paged);
+
+/**
+ * phy_write_paged() - Convenience function for writing a paged register
+ * @phydev: a pointer to a &struct phy_device
+ * @page: the page for the phy
+ * @regnum: register number
+ * @val: value to write
+ *
+ * Same rules as for phy_write().
+ */
+int phy_write_paged(struct phy_device *phydev, int page, u32 regnum, u16 val)
+{
+	int ret = 0, oldpage;
+
+	oldpage = phy_select_page(phydev, page);
+	if (oldpage >= 0)
+		ret = __phy_write(phydev, regnum, val);
+
+	return phy_restore_page(phydev, oldpage, ret);
+}
+EXPORT_SYMBOL(phy_write_paged);
+
+/**
+ * phy_modify_paged() - Convenience function for modifying a paged register
+ * @phydev: a pointer to a &struct phy_device
+ * @page: the page for the phy
+ * @regnum: register number
+ * @mask: bit mask of bits to clear
+ * @set: bit mask of bits to set
+ *
+ * Same rules as for phy_read() and phy_write().
+ */
+int phy_modify_paged(struct phy_device *phydev, int page, u32 regnum,
+		     u16 mask, u16 set)
+{
+	int ret = 0, oldpage;
+
+	oldpage = phy_select_page(phydev, page);
+	if (oldpage >= 0)
+		ret = __phy_modify(phydev, regnum, mask, set);
+
+	return phy_restore_page(phydev, oldpage, ret);
+}
+EXPORT_SYMBOL(phy_modify_paged);
diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
index ed10d1fc8f59..9aabfa1a455a 100644
--- a/drivers/net/phy/phy.c
+++ b/drivers/net/phy/phy.c
@@ -493,7 +493,10 @@ static int phy_start_aneg_priv(struct phy_device *phydev, bool sync)
 	/* Invalidate LP advertising flags */
 	phydev->lp_advertising = 0;
 
-	err = phydev->drv->config_aneg(phydev);
+	if (phydev->drv->config_aneg)
+		err = phydev->drv->config_aneg(phydev);
+	else
+		err = genphy_config_aneg(phydev);
 	if (err < 0)
 		goto out_unlock;
 
@@ -614,6 +617,77 @@ static void phy_error(struct phy_device *phydev)
 	phy_trigger_machine(phydev, false);
 }
 
+/**
+ * phy_disable_interrupts - Disable the PHY interrupts from the PHY side
+ * @phydev: target phy_device struct
+ */
+static int phy_disable_interrupts(struct phy_device *phydev)
+{
+	int err;
+
+	/* Disable PHY interrupts */
+	err = phy_config_interrupt(phydev, PHY_INTERRUPT_DISABLED);
+	if (err)
+		goto phy_err;
+
+	/* Clear the interrupt */
+	err = phy_clear_interrupt(phydev);
+	if (err)
+		goto phy_err;
+
+	return 0;
+
+phy_err:
+	phy_error(phydev);
+
+	return err;
+}
+
+/**
+ * phy_change - Called by the phy_interrupt to handle PHY changes
+ * @phydev: phy_device struct that interrupted
+ */
+static irqreturn_t phy_change(struct phy_device *phydev)
+{
+	if (phy_interrupt_is_valid(phydev)) {
+		if (phydev->drv->did_interrupt &&
+		    !phydev->drv->did_interrupt(phydev))
+			return IRQ_NONE;
+
+		if (phydev->state == PHY_HALTED)
+			if (phy_disable_interrupts(phydev))
+				goto phy_err;
+	}
+
+	mutex_lock(&phydev->lock);
+	if ((PHY_RUNNING == phydev->state) || (PHY_NOLINK == phydev->state))
+		phydev->state = PHY_CHANGELINK;
+	mutex_unlock(&phydev->lock);
+
+	/* reschedule state queue work to run as soon as possible */
+	phy_trigger_machine(phydev, true);
+
+	if (phy_interrupt_is_valid(phydev) && phy_clear_interrupt(phydev))
+		goto phy_err;
+	return IRQ_HANDLED;
+
+phy_err:
+	phy_error(phydev);
+	return IRQ_NONE;
+}
+
+/**
+ * phy_change_work - Scheduled by the phy_mac_interrupt to handle PHY changes
+ * @work: work_struct that describes the work to be done
+ */
+void phy_change_work(struct work_struct *work)
+{
+	struct phy_device *phydev =
+		container_of(work, struct phy_device, phy_queue);
+
+	phy_change(phydev);
+}
+
 /**
  * phy_interrupt - PHY interrupt handler
  * @irq: interrupt line
@@ -629,12 +703,7 @@ static irqreturn_t phy_interrupt(int irq, void *phy_dat)
 	if (PHY_HALTED == phydev->state)
 		return IRQ_NONE;		/* It can't be ours.  */
 
-	disable_irq_nosync(irq);
-	atomic_inc(&phydev->irq_disable);
-
-	phy_change(phydev);
-
-	return IRQ_HANDLED;
+	return phy_change(phydev);
 }
 
 /**
@@ -651,32 +720,6 @@ static int phy_enable_interrupts(struct phy_device *phydev)
 	return phy_config_interrupt(phydev, PHY_INTERRUPT_ENABLED);
 }
 
-/**
- * phy_disable_interrupts - Disable the PHY interrupts from the PHY side
- * @phydev: target phy_device struct
- */
-static int phy_disable_interrupts(struct phy_device *phydev)
-{
-	int err;
-
-	/* Disable PHY interrupts */
-	err = phy_config_interrupt(phydev, PHY_INTERRUPT_DISABLED);
-	if (err)
-		goto phy_err;
-
-	/* Clear the interrupt */
-	err = phy_clear_interrupt(phydev);
-	if (err)
-		goto phy_err;
-
-	return 0;
-
-phy_err:
-	phy_error(phydev);
-
-	return err;
-}
-
 /**
  * phy_start_interrupts - request and enable interrupts for a PHY device
  * @phydev: target phy_device struct
@@ -689,7 +732,6 @@ static int phy_disable_interrupts(struct phy_device *phydev)
  */
 int phy_start_interrupts(struct phy_device *phydev)
 {
-	atomic_set(&phydev->irq_disable, 0);
 	if (request_threaded_irq(phydev->irq, NULL, phy_interrupt,
 				 IRQF_ONESHOT | IRQF_SHARED,
 				 phydev_name(phydev), phydev) < 0) {
@@ -716,75 +758,10 @@ int phy_stop_interrupts(struct phy_device *phydev)
 
 	free_irq(phydev->irq, phydev);
 
-	/* If work indeed has been cancelled, disable_irq() will have
-	 * been left unbalanced from phy_interrupt() and enable_irq()
-	 * has to be called so that other devices on the line work.
-	 */
-	while (atomic_dec_return(&phydev->irq_disable) >= 0)
-		enable_irq(phydev->irq);
-
 	return err;
 }
 EXPORT_SYMBOL(phy_stop_interrupts);
 
-/**
- * phy_change - Called by the phy_interrupt to handle PHY changes
- * @phydev: phy_device struct that interrupted
- */
-void phy_change(struct phy_device *phydev)
-{
-	if (phy_interrupt_is_valid(phydev)) {
-		if (phydev->drv->did_interrupt &&
-		    !phydev->drv->did_interrupt(phydev))
-			goto ignore;
-
-		if (phy_disable_interrupts(phydev))
-			goto phy_err;
-	}
-
-	mutex_lock(&phydev->lock);
-	if ((PHY_RUNNING == phydev->state) || (PHY_NOLINK == phydev->state))
-		phydev->state = PHY_CHANGELINK;
-	mutex_unlock(&phydev->lock);
-
-	if (phy_interrupt_is_valid(phydev)) {
-		atomic_dec(&phydev->irq_disable);
-		enable_irq(phydev->irq);
-
-		/* Reenable interrupts */
-		if (PHY_HALTED != phydev->state &&
-		    phy_config_interrupt(phydev, PHY_INTERRUPT_ENABLED))
-			goto irq_enable_err;
-	}
-
-	/* reschedule state queue work to run as soon as possible */
-	phy_trigger_machine(phydev, true);
-	return;
-
-ignore:
-	atomic_dec(&phydev->irq_disable);
-	enable_irq(phydev->irq);
-	return;
-
-irq_enable_err:
-	disable_irq(phydev->irq);
-	atomic_inc(&phydev->irq_disable);
-phy_err:
-	phy_error(phydev);
-}
-
-/**
- * phy_change_work - Scheduled by the phy_mac_interrupt to handle PHY changes
- * @work: work_struct that describes the work to be done
- */
-void phy_change_work(struct work_struct *work)
-{
-	struct phy_device *phydev =
-		container_of(work, struct phy_device, phy_queue);
-
-	phy_change(phydev);
-}
-
 /**
  * phy_stop - Bring down the PHY link, and stop checking the status
  * @phydev: target phy_device struct
@@ -841,10 +818,10 @@ void phy_start(struct phy_device *phydev)
 		break;
 	case PHY_HALTED:
 		/* if phy was suspended, bring the physical link up again */
-		phy_resume(phydev);
+		__phy_resume(phydev);
 
 		/* make sure interrupts are re-enabled for the PHY */
-		if (phydev->irq != PHY_POLL) {
+		if (phy_interrupt_is_valid(phydev)) {
 			err = phy_enable_interrupts(phydev);
 			if (err < 0)
 				break;
@@ -1003,10 +980,6 @@ void phy_state_machine(struct work_struct *work)
 			phydev->state = PHY_NOLINK;
 			phy_link_down(phydev, true);
 		}
-
-		if (phy_interrupt_is_valid(phydev))
-			err = phy_config_interrupt(phydev,
-						   PHY_INTERRUPT_ENABLED);
 		break;
 	case PHY_HALTED:
 		if (phydev->link) {
@@ -1083,16 +1056,12 @@ void phy_state_machine(struct work_struct *work)
 /**
  * phy_mac_interrupt - MAC says the link has changed
  * @phydev: phy_device struct with changed link
- * @new_link: Link is Up/Down.
  *
- * Description: The MAC layer is able indicate there has been a change
- *   in the PHY link status. Set the new link status, and trigger the
- *   state machine, work a work queue.
+ * The MAC layer is able to indicate there has been a change in the PHY link
+ * status. Trigger the state machine and work a work queue.
  */
-void phy_mac_interrupt(struct phy_device *phydev, int new_link)
+void phy_mac_interrupt(struct phy_device *phydev)
 {
-	phydev->link = new_link;
-
 	/* Trigger a state machine change */
 	queue_work(system_power_efficient_wq, &phydev->phy_queue);
 }
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index b15b31ca2618..74664a6c0cdc 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -135,9 +135,7 @@ static int mdio_bus_phy_resume(struct device *dev)
 	if (!mdio_bus_phy_may_suspend(phydev))
 		goto no_resume;
 
-	mutex_lock(&phydev->lock);
 	ret = phy_resume(phydev);
-	mutex_unlock(&phydev->lock);
 	if (ret < 0)
 		return ret;
 
@@ -634,6 +632,9 @@ int phy_device_register(struct phy_device *phydev)
 	if (err)
 		return err;
 
+	/* Deassert the reset signal */
+	phy_device_reset(phydev, 0);
+
 	/* Run all of the fixups for this PHY */
 	err = phy_scan_fixups(phydev);
 	if (err) {
@@ -652,6 +653,9 @@ int phy_device_register(struct phy_device *phydev)
 	return 0;
 
  out:
+	/* Assert the reset signal */
+	phy_device_reset(phydev, 1);
+
 	mdiobus_unregister_device(&phydev->mdio);
 	return err;
 }
@@ -668,6 +672,10 @@ EXPORT_SYMBOL(phy_device_register);
 void phy_device_remove(struct phy_device *phydev)
 {
 	device_del(&phydev->mdio.dev);
+
+	/* Assert the reset signal */
+	phy_device_reset(phydev, 1);
+
 	mdiobus_unregister_device(&phydev->mdio);
 }
 EXPORT_SYMBOL(phy_device_remove);
@@ -851,6 +859,9 @@ int phy_init_hw(struct phy_device *phydev)
 {
 	int ret = 0;
 
+	/* Deassert the reset signal */
+	phy_device_reset(phydev, 0);
+
 	if (!phydev->drv || !phydev->drv->config_init)
 		return 0;
 
@@ -1001,10 +1012,17 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 	err = sysfs_create_link(&phydev->mdio.dev.kobj, &dev->dev.kobj,
 				"attached_dev");
 	if (!err) {
-		err = sysfs_create_link(&dev->dev.kobj, &phydev->mdio.dev.kobj,
-					"phydev");
-		if (err)
-			goto error;
+		err = sysfs_create_link_nowarn(&dev->dev.kobj,
+					       &phydev->mdio.dev.kobj,
+					       "phydev");
+		if (err) {
+			dev_err(&dev->dev, "could not add device link to %s err %d\n",
+				kobject_name(&phydev->mdio.dev.kobj),
+				err);
+			/* non-fatal - some net drivers can use one netdevice
+			 * with more then one phy
+			 */
+		}
 
 		phydev->sysfs_links = true;
 	}
@@ -1028,9 +1046,7 @@ int phy_attach_direct(struct net_device *dev, struct phy_device *phydev,
 	if (err)
 		goto error;
 
-	mutex_lock(&phydev->lock);
 	phy_resume(phydev);
-	mutex_unlock(&phydev->lock);
 	phy_led_triggers_register(phydev);
 
 	return err;
@@ -1130,6 +1146,9 @@ void phy_detach(struct phy_device *phydev)
 	put_device(&phydev->mdio.dev);
 	if (ndev_owner != bus->owner)
 		module_put(bus->owner);
+
+	/* Assert the reset signal */
+	phy_device_reset(phydev, 1);
 }
 EXPORT_SYMBOL(phy_detach);
 
@@ -1156,7 +1175,7 @@ int phy_suspend(struct phy_device *phydev)
 }
 EXPORT_SYMBOL(phy_suspend);
 
-int phy_resume(struct phy_device *phydev)
+int __phy_resume(struct phy_device *phydev)
 {
 	struct phy_driver *phydrv = to_phy_driver(phydev->mdio.dev.driver);
 	int ret = 0;
@@ -1173,6 +1192,18 @@ int phy_resume(struct phy_device *phydev)
 
 	return ret;
 }
+EXPORT_SYMBOL(__phy_resume);
+
+int phy_resume(struct phy_device *phydev)
+{
+	int ret;
+
+	mutex_lock(&phydev->lock);
+	ret = __phy_resume(phydev);
+	mutex_unlock(&phydev->lock);
+
+	return ret;
+}
 EXPORT_SYMBOL(phy_resume);
 
 int phy_loopback(struct phy_device *phydev, bool enable)
@@ -1208,6 +1239,30 @@ int phy_loopback(struct phy_device *phydev, bool enable)
 }
 EXPORT_SYMBOL(phy_loopback);
 
+/**
+ * phy_reset_after_clk_enable - perform a PHY reset if needed
+ * @phydev: target phy_device struct
+ *
+ * Description: Some PHYs are known to need a reset after their refclk was
+ *   enabled. This function evaluates the flags and perform the reset if it's
+ *   needed. Returns < 0 on error, 0 if the phy wasn't reset and 1 if the phy
+ *   was reset.
+ */
+int phy_reset_after_clk_enable(struct phy_device *phydev)
+{
+	if (!phydev || !phydev->drv)
+		return -ENODEV;
+
+	if (phydev->drv->flags & PHY_RST_AFTER_CLK_EN) {
+		phy_device_reset(phydev, 1);
+		phy_device_reset(phydev, 0);
+		return 1;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(phy_reset_after_clk_enable);
+
 /* Generic PHY support and helper functions */
 
 /**
@@ -1328,9 +1383,8 @@ static int genphy_config_eee_advert(struct phy_device *phydev)
  */
 int genphy_setup_forced(struct phy_device *phydev)
 {
-	int ctl = phy_read(phydev, MII_BMCR);
+	u16 ctl = 0;
 
-	ctl &= BMCR_LOOPBACK | BMCR_ISOLATE | BMCR_PDOWN;
 	phydev->pause = 0;
 	phydev->asym_pause = 0;
 
@@ -1342,7 +1396,8 @@ int genphy_setup_forced(struct phy_device *phydev)
 	if (DUPLEX_FULL == phydev->duplex)
 		ctl |= BMCR_FULLDPLX;
 
-	return phy_write(phydev, MII_BMCR, ctl);
+	return phy_modify(phydev, MII_BMCR,
+			  ~(BMCR_LOOPBACK | BMCR_ISOLATE | BMCR_PDOWN), ctl);
 }
 EXPORT_SYMBOL(genphy_setup_forced);
 
@@ -1352,17 +1407,9 @@ EXPORT_SYMBOL(genphy_setup_forced);
  */
 int genphy_restart_aneg(struct phy_device *phydev)
 {
-	int ctl = phy_read(phydev, MII_BMCR);
-
-	if (ctl < 0)
-		return ctl;
-
-	ctl |= BMCR_ANENABLE | BMCR_ANRESTART;
-
 	/* Don't isolate the PHY if we're negotiating */
-	ctl &= ~BMCR_ISOLATE;
-
-	return phy_write(phydev, MII_BMCR, ctl);
+	return phy_modify(phydev, MII_BMCR, BMCR_ISOLATE,
+			  BMCR_ANENABLE | BMCR_ANRESTART);
 }
 EXPORT_SYMBOL(genphy_restart_aneg);
 
@@ -1626,46 +1673,39 @@ int genphy_config_init(struct phy_device *phydev)
 }
 EXPORT_SYMBOL(genphy_config_init);
 
-int genphy_suspend(struct phy_device *phydev)
+/* This is used for the phy device which doesn't support the MMD extended
+ * register access, but it does have side effect when we are trying to access
+ * the MMD register via indirect method.
+ */
+int genphy_read_mmd_unsupported(struct phy_device *phdev, int devad, u16 regnum)
 {
-	int value;
-
-	mutex_lock(&phydev->lock);
-
-	value = phy_read(phydev, MII_BMCR);
-	phy_write(phydev, MII_BMCR, value | BMCR_PDOWN);
+	return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(genphy_read_mmd_unsupported);
 
-	mutex_unlock(&phydev->lock);
+int genphy_write_mmd_unsupported(struct phy_device *phdev, int devnum,
+				 u16 regnum, u16 val)
+{
+	return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(genphy_write_mmd_unsupported);
 
-	return 0;
+int genphy_suspend(struct phy_device *phydev)
+{
+	return phy_set_bits(phydev, MII_BMCR, BMCR_PDOWN);
 }
 EXPORT_SYMBOL(genphy_suspend);
 
 int genphy_resume(struct phy_device *phydev)
 {
-	int value;
-
-	value = phy_read(phydev, MII_BMCR);
-	phy_write(phydev, MII_BMCR, value & ~BMCR_PDOWN);
-
-	return 0;
+	return phy_clear_bits(phydev, MII_BMCR, BMCR_PDOWN);
 }
 EXPORT_SYMBOL(genphy_resume);
 
 int genphy_loopback(struct phy_device *phydev, bool enable)
 {
-	int value;
-
-	value = phy_read(phydev, MII_BMCR);
-	if (value < 0)
-		return value;
-
-	if (enable)
-		value |= BMCR_LOOPBACK;
-	else
-		value &= ~BMCR_LOOPBACK;
-
-	return phy_write(phydev, MII_BMCR, value);
+	return phy_modify(phydev, MII_BMCR, BMCR_LOOPBACK,
+			  enable ? BMCR_LOOPBACK : 0);
 }
 EXPORT_SYMBOL(genphy_loopback);
 
@@ -1813,8 +1853,16 @@ static int phy_probe(struct device *dev)
 	/* Set the state to READY by default */
 	phydev->state = PHY_READY;
 
-	if (phydev->drv->probe)
+	if (phydev->drv->probe) {
+		/* Deassert the reset signal */
+		phy_device_reset(phydev, 0);
+
 		err = phydev->drv->probe(phydev);
+		if (err) {
+			/* Assert the reset signal */
+			phy_device_reset(phydev, 1);
+		}
+	}
 
 	mutex_unlock(&phydev->lock);
 
@@ -1831,8 +1879,12 @@ static int phy_remove(struct device *dev)
 	phydev->state = PHY_DOWN;
 	mutex_unlock(&phydev->lock);
 
-	if (phydev->drv && phydev->drv->remove)
+	if (phydev->drv && phydev->drv->remove) {
 		phydev->drv->remove(phydev);
+
+		/* Assert the reset signal */
+		phy_device_reset(phydev, 1);
+	}
 	phydev->drv = NULL;
 
 	return 0;
@@ -1909,9 +1961,7 @@ static struct phy_driver genphy_driver = {
 	.features	= PHY_GBIT_FEATURES | SUPPORTED_MII |
 			  SUPPORTED_AUI | SUPPORTED_FIBRE |
 			  SUPPORTED_BNC,
-	.config_aneg	= genphy_config_aneg,
 	.aneg_done	= genphy_aneg_done,
-	.read_status	= genphy_read_status,
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 	.set_loopback   = genphy_loopback,
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
index 249ce5cbea22..6ac8b29b2dc3 100644
--- a/drivers/net/phy/phylink.c
+++ b/drivers/net/phy/phylink.c
@@ -36,7 +36,11 @@ enum {
 	PHYLINK_DISABLE_LINK,
 };
 
+/**
+ * struct phylink - internal data type for phylink
+ */
 struct phylink {
+	/* private: */
 	struct net_device *netdev;
 	const struct phylink_mac_ops *ops;
 
@@ -50,6 +54,8 @@ struct phylink {
 	/* The link configuration settings */
 	struct phylink_link_state link_config;
 	struct gpio_desc *link_gpio;
+	void (*get_fixed_state)(struct net_device *dev,
+				struct phylink_link_state *s);
 
 	struct mutex state_mutex;
 	struct phylink_link_state phy_state;
@@ -87,6 +93,13 @@ static inline bool linkmode_empty(const unsigned long *src)
 	return bitmap_empty(src, __ETHTOOL_LINK_MODE_MASK_NBITS);
 }
 
+/**
+ * phylink_set_port_modes() - set the port type modes in the ethtool mask
+ * @mask: ethtool link mode mask
+ *
+ * Sets all the port type modes in the ethtool mask.  MAC drivers should
+ * use this in their 'validate' callback.
+ */
 void phylink_set_port_modes(unsigned long *mask)
 {
 	phylink_set(mask, TP);
@@ -117,8 +130,7 @@ static const char *phylink_an_mode_str(unsigned int mode)
 	static const char *modestr[] = {
 		[MLO_AN_PHY] = "phy",
 		[MLO_AN_FIXED] = "fixed",
-		[MLO_AN_SGMII] = "SGMII",
-		[MLO_AN_8023Z] = "802.3z",
+		[MLO_AN_INBAND] = "inband",
 	};
 
 	return mode < ARRAY_SIZE(modestr) ? modestr[mode] : "unknown";
@@ -132,59 +144,64 @@ static int phylink_validate(struct phylink *pl, unsigned long *supported,
 	return phylink_is_empty_linkmode(supported) ? -EINVAL : 0;
 }
 
-static int phylink_parse_fixedlink(struct phylink *pl, struct device_node *np)
+static int phylink_parse_fixedlink(struct phylink *pl,
+				   struct fwnode_handle *fwnode)
 {
-	struct device_node *fixed_node;
+	struct fwnode_handle *fixed_node;
 	const struct phy_setting *s;
 	struct gpio_desc *desc;
-	const __be32 *fixed_prop;
 	u32 speed;
-	int ret, len;
+	int ret;
 
-	fixed_node = of_get_child_by_name(np, "fixed-link");
+	fixed_node = fwnode_get_named_child_node(fwnode, "fixed-link");
 	if (fixed_node) {
-		ret = of_property_read_u32(fixed_node, "speed", &speed);
+		ret = fwnode_property_read_u32(fixed_node, "speed", &speed);
 
 		pl->link_config.speed = speed;
 		pl->link_config.duplex = DUPLEX_HALF;
 
-		if (of_property_read_bool(fixed_node, "full-duplex"))
+		if (fwnode_property_read_bool(fixed_node, "full-duplex"))
 			pl->link_config.duplex = DUPLEX_FULL;
 
 		/* We treat the "pause" and "asym-pause" terminology as
 		 * defining the link partner's ability. */
-		if (of_property_read_bool(fixed_node, "pause"))
+		if (fwnode_property_read_bool(fixed_node, "pause"))
 			pl->link_config.pause |= MLO_PAUSE_SYM;
-		if (of_property_read_bool(fixed_node, "asym-pause"))
+		if (fwnode_property_read_bool(fixed_node, "asym-pause"))
 			pl->link_config.pause |= MLO_PAUSE_ASYM;
 
 		if (ret == 0) {
-			desc = fwnode_get_named_gpiod(&fixed_node->fwnode,
-						      "link-gpios", 0,
-						      GPIOD_IN, "?");
+			desc = fwnode_get_named_gpiod(fixed_node, "link-gpios",
+						      0, GPIOD_IN, "?");
 
 			if (!IS_ERR(desc))
 				pl->link_gpio = desc;
 			else if (desc == ERR_PTR(-EPROBE_DEFER))
 				ret = -EPROBE_DEFER;
 		}
-		of_node_put(fixed_node);
+		fwnode_handle_put(fixed_node);
 
 		if (ret)
 			return ret;
 	} else {
-		fixed_prop = of_get_property(np, "fixed-link", &len);
-		if (!fixed_prop) {
+		u32 prop[5];
+
+		ret = fwnode_property_read_u32_array(fwnode, "fixed-link",
+						     NULL, 0);
+		if (ret != ARRAY_SIZE(prop)) {
 			netdev_err(pl->netdev, "broken fixed-link?\n");
 			return -EINVAL;
 		}
-		if (len == 5 * sizeof(*fixed_prop)) {
-			pl->link_config.duplex = be32_to_cpu(fixed_prop[1]) ?
+
+		ret = fwnode_property_read_u32_array(fwnode, "fixed-link",
+						     prop, ARRAY_SIZE(prop));
+		if (!ret) {
+			pl->link_config.duplex = prop[1] ?
 						DUPLEX_FULL : DUPLEX_HALF;
-			pl->link_config.speed = be32_to_cpu(fixed_prop[2]);
-			if (be32_to_cpu(fixed_prop[3]))
+			pl->link_config.speed = prop[2];
+			if (prop[3])
 				pl->link_config.pause |= MLO_PAUSE_SYM;
-			if (be32_to_cpu(fixed_prop[4]))
+			if (prop[4])
 				pl->link_config.pause |= MLO_PAUSE_ASYM;
 		}
 	}
@@ -220,17 +237,17 @@ static int phylink_parse_fixedlink(struct phylink *pl, struct device_node *np)
 	return 0;
 }
 
-static int phylink_parse_mode(struct phylink *pl, struct device_node *np)
+static int phylink_parse_mode(struct phylink *pl, struct fwnode_handle *fwnode)
 {
-	struct device_node *dn;
+	struct fwnode_handle *dn;
 	const char *managed;
 
-	dn = of_get_child_by_name(np, "fixed-link");
-	if (dn || of_find_property(np, "fixed-link", NULL))
+	dn = fwnode_get_named_child_node(fwnode, "fixed-link");
+	if (dn || fwnode_property_present(fwnode, "fixed-link"))
 		pl->link_an_mode = MLO_AN_FIXED;
-	of_node_put(dn);
+	fwnode_handle_put(dn);
 
-	if (of_property_read_string(np, "managed", &managed) == 0 &&
+	if (fwnode_property_read_string(fwnode, "managed", &managed) == 0 &&
 	    strcmp(managed, "in-band-status") == 0) {
 		if (pl->link_an_mode == MLO_AN_FIXED) {
 			netdev_err(pl->netdev,
@@ -244,6 +261,7 @@ static int phylink_parse_mode(struct phylink *pl, struct device_node *np)
 		phylink_set(pl->supported, Asym_Pause);
 		phylink_set(pl->supported, Pause);
 		pl->link_config.an_enabled = true;
+		pl->link_an_mode = MLO_AN_INBAND;
 
 		switch (pl->link_config.interface) {
 		case PHY_INTERFACE_MODE_SGMII:
@@ -253,17 +271,14 @@ static int phylink_parse_mode(struct phylink *pl, struct device_node *np)
 			phylink_set(pl->supported, 100baseT_Full);
 			phylink_set(pl->supported, 1000baseT_Half);
 			phylink_set(pl->supported, 1000baseT_Full);
-			pl->link_an_mode = MLO_AN_SGMII;
 			break;
 
 		case PHY_INTERFACE_MODE_1000BASEX:
 			phylink_set(pl->supported, 1000baseX_Full);
-			pl->link_an_mode = MLO_AN_8023Z;
 			break;
 
 		case PHY_INTERFACE_MODE_2500BASEX:
 			phylink_set(pl->supported, 2500baseX_Full);
-			pl->link_an_mode = MLO_AN_8023Z;
 			break;
 
 		case PHY_INTERFACE_MODE_10GKR:
@@ -280,7 +295,6 @@ static int phylink_parse_mode(struct phylink *pl, struct device_node *np)
 			phylink_set(pl->supported, 10000baseLR_Full);
 			phylink_set(pl->supported, 10000baseLRM_Full);
 			phylink_set(pl->supported, 10000baseER_Full);
-			pl->link_an_mode = MLO_AN_SGMII;
 			break;
 
 		default:
@@ -320,8 +334,7 @@ static void phylink_mac_config(struct phylink *pl,
 static void phylink_mac_an_restart(struct phylink *pl)
 {
 	if (pl->link_config.an_enabled &&
-	    (pl->link_config.interface == PHY_INTERFACE_MODE_1000BASEX ||
-	     pl->link_config.interface == PHY_INTERFACE_MODE_2500BASEX))
+	    phy_interface_mode_is_8023z(pl->link_config.interface))
 		pl->ops->mac_an_restart(pl->netdev);
 }
 
@@ -339,12 +352,14 @@ static int phylink_get_mac_state(struct phylink *pl, struct phylink_link_state *
 }
 
 /* The fixed state is... fixed except for the link state,
- * which may be determined by a GPIO.
+ * which may be determined by a GPIO or a callback.
  */
 static void phylink_get_fixed_state(struct phylink *pl, struct phylink_link_state *state)
 {
 	*state = pl->link_config;
-	if (pl->link_gpio)
+	if (pl->get_fixed_state)
+		pl->get_fixed_state(pl->netdev, state);
+	else if (pl->link_gpio)
 		state->link = !!gpiod_get_value(pl->link_gpio);
 }
 
@@ -423,7 +438,7 @@ static void phylink_resolve(struct work_struct *w)
 			phylink_mac_config(pl, &link_state);
 			break;
 
-		case MLO_AN_SGMII:
+		case MLO_AN_INBAND:
 			phylink_get_mac_state(pl, &link_state);
 			if (pl->phydev) {
 				bool changed = false;
@@ -449,10 +464,6 @@ static void phylink_resolve(struct work_struct *w)
 				}
 			}
 			break;
-
-		case MLO_AN_8023Z:
-			phylink_get_mac_state(pl, &link_state);
-			break;
 		}
 	}
 
@@ -489,15 +500,27 @@ static void phylink_run_resolve(struct phylink *pl)
 
 static const struct sfp_upstream_ops sfp_phylink_ops;
 
-static int phylink_register_sfp(struct phylink *pl, struct device_node *np)
+static int phylink_register_sfp(struct phylink *pl,
+				struct fwnode_handle *fwnode)
 {
-	struct device_node *sfp_np;
+	struct fwnode_reference_args ref;
+	int ret;
 
-	sfp_np = of_parse_phandle(np, "sfp", 0);
-	if (!sfp_np)
+	if (!fwnode)
 		return 0;
 
-	pl->sfp_bus = sfp_register_upstream(sfp_np, pl->netdev, pl,
+	ret = fwnode_property_get_reference_args(fwnode, "sfp", NULL,
+						 0, 0, &ref);
+	if (ret < 0) {
+		if (ret == -ENOENT)
+			return 0;
+
+		netdev_err(pl->netdev, "unable to parse \"sfp\" node: %d\n",
+			   ret);
+		return ret;
+	}
+
+	pl->sfp_bus = sfp_register_upstream(ref.fwnode, pl->netdev, pl,
 					    &sfp_phylink_ops);
 	if (!pl->sfp_bus)
 		return -ENOMEM;
@@ -505,7 +528,22 @@ static int phylink_register_sfp(struct phylink *pl, struct device_node *np)
 	return 0;
 }
 
-struct phylink *phylink_create(struct net_device *ndev, struct device_node *np,
+/**
+ * phylink_create() - create a phylink instance
+ * @ndev: a pointer to the &struct net_device
+ * @fwnode: a pointer to a &struct fwnode_handle describing the network
+ *	interface
+ * @iface: the desired link mode defined by &typedef phy_interface_t
+ * @ops: a pointer to a &struct phylink_mac_ops for the MAC.
+ *
+ * Create a new phylink instance, and parse the link parameters found in @np.
+ * This will parse in-band modes, fixed-link or SFP configuration.
+ *
+ * Returns a pointer to a &struct phylink, or an error-pointer value. Users
+ * must use IS_ERR() to check for errors from this function.
+ */
+struct phylink *phylink_create(struct net_device *ndev,
+			       struct fwnode_handle *fwnode,
 			       phy_interface_t iface,
 			       const struct phylink_mac_ops *ops)
 {
@@ -521,7 +559,10 @@ struct phylink *phylink_create(struct net_device *ndev, struct device_node *np,
 	pl->netdev = ndev;
 	pl->phy_state.interface = iface;
 	pl->link_interface = iface;
-	pl->link_port = PORT_MII;
+	if (iface == PHY_INTERFACE_MODE_MOCA)
+		pl->link_port = PORT_BNC;
+	else
+		pl->link_port = PORT_MII;
 	pl->link_config.interface = iface;
 	pl->link_config.pause = MLO_PAUSE_AN;
 	pl->link_config.speed = SPEED_UNKNOWN;
@@ -534,21 +575,21 @@ struct phylink *phylink_create(struct net_device *ndev, struct device_node *np,
 	linkmode_copy(pl->link_config.advertising, pl->supported);
 	phylink_validate(pl, pl->supported, &pl->link_config);
 
-	ret = phylink_parse_mode(pl, np);
+	ret = phylink_parse_mode(pl, fwnode);
 	if (ret < 0) {
 		kfree(pl);
 		return ERR_PTR(ret);
 	}
 
 	if (pl->link_an_mode == MLO_AN_FIXED) {
-		ret = phylink_parse_fixedlink(pl, np);
+		ret = phylink_parse_fixedlink(pl, fwnode);
 		if (ret < 0) {
 			kfree(pl);
 			return ERR_PTR(ret);
 		}
 	}
 
-	ret = phylink_register_sfp(pl, np);
+	ret = phylink_register_sfp(pl, fwnode);
 	if (ret < 0) {
 		kfree(pl);
 		return ERR_PTR(ret);
@@ -558,6 +599,13 @@ struct phylink *phylink_create(struct net_device *ndev, struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(phylink_create);
 
+/**
+ * phylink_destroy() - cleanup and destroy the phylink instance
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ *
+ * Destroy a phylink instance. Any PHY that has been attached must have been
+ * cleaned up via phylink_disconnect_phy() prior to calling this function.
+ */
 void phylink_destroy(struct phylink *pl)
 {
 	if (pl->sfp_bus)
@@ -654,10 +702,39 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy)
 	return 0;
 }
 
+/**
+ * phylink_connect_phy() - connect a PHY to the phylink instance
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @phy: a pointer to a &struct phy_device.
+ *
+ * Connect @phy to the phylink instance specified by @pl by calling
+ * phy_attach_direct(). Configure the @phy according to the MAC driver's
+ * capabilities, start the PHYLIB state machine and enable any interrupts
+ * that the PHY supports.
+ *
+ * This updates the phylink's ethtool supported and advertising link mode
+ * masks.
+ *
+ * Returns 0 on success or a negative errno.
+ */
 int phylink_connect_phy(struct phylink *pl, struct phy_device *phy)
 {
 	int ret;
 
+	if (WARN_ON(pl->link_an_mode == MLO_AN_FIXED ||
+		    (pl->link_an_mode == MLO_AN_INBAND &&
+		     phy_interface_mode_is_8023z(pl->link_interface))))
+		return -EINVAL;
+
+	if (pl->phydev)
+		return -EBUSY;
+
+	/* Use PHY device/driver interface */
+	if (pl->link_interface == PHY_INTERFACE_MODE_NA) {
+		pl->link_interface = phy->interface;
+		pl->link_config.interface = pl->link_interface;
+	}
+
 	ret = phy_attach_direct(pl->netdev, phy, 0, pl->link_interface);
 	if (ret)
 		return ret;
@@ -670,14 +747,29 @@ int phylink_connect_phy(struct phylink *pl, struct phy_device *phy)
 }
 EXPORT_SYMBOL_GPL(phylink_connect_phy);
 
-int phylink_of_phy_connect(struct phylink *pl, struct device_node *dn)
+/**
+ * phylink_of_phy_connect() - connect the PHY specified in the DT mode.
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @dn: a pointer to a &struct device_node.
+ * @flags: PHY-specific flags to communicate to the PHY device driver
+ *
+ * Connect the phy specified in the device node @dn to the phylink instance
+ * specified by @pl. Actions specified in phylink_connect_phy() will be
+ * performed.
+ *
+ * Returns 0 on success or a negative errno.
+ */
+int phylink_of_phy_connect(struct phylink *pl, struct device_node *dn,
+			   u32 flags)
 {
 	struct device_node *phy_node;
 	struct phy_device *phy_dev;
 	int ret;
 
-	/* Fixed links are handled without needing a PHY */
-	if (pl->link_an_mode == MLO_AN_FIXED)
+	/* Fixed links and 802.3z are handled without needing a PHY */
+	if (pl->link_an_mode == MLO_AN_FIXED ||
+	    (pl->link_an_mode == MLO_AN_INBAND &&
+	     phy_interface_mode_is_8023z(pl->link_interface)))
 		return 0;
 
 	phy_node = of_parse_phandle(dn, "phy-handle", 0);
@@ -687,14 +779,13 @@ int phylink_of_phy_connect(struct phylink *pl, struct device_node *dn)
 		phy_node = of_parse_phandle(dn, "phy-device", 0);
 
 	if (!phy_node) {
-		if (pl->link_an_mode == MLO_AN_PHY) {
-			netdev_err(pl->netdev, "unable to find PHY node\n");
+		if (pl->link_an_mode == MLO_AN_PHY)
 			return -ENODEV;
-		}
 		return 0;
 	}
 
-	phy_dev = of_phy_attach(pl->netdev, phy_node, 0, pl->link_interface);
+	phy_dev = of_phy_attach(pl->netdev, phy_node, flags,
+				pl->link_interface);
 	/* We're done with the phy_node handle */
 	of_node_put(phy_node);
 
@@ -709,11 +800,18 @@ int phylink_of_phy_connect(struct phylink *pl, struct device_node *dn)
 }
 EXPORT_SYMBOL_GPL(phylink_of_phy_connect);
 
+/**
+ * phylink_disconnect_phy() - disconnect any PHY attached to the phylink
+ *   instance.
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ *
+ * Disconnect any current PHY from the phylink instance described by @pl.
+ */
 void phylink_disconnect_phy(struct phylink *pl)
 {
 	struct phy_device *phy;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	phy = pl->phydev;
 	if (phy) {
@@ -730,6 +828,40 @@ void phylink_disconnect_phy(struct phylink *pl)
 }
 EXPORT_SYMBOL_GPL(phylink_disconnect_phy);
 
+/**
+ * phylink_fixed_state_cb() - allow setting a fixed link callback
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @cb: callback to execute to determine the fixed link state.
+ *
+ * The MAC driver should call this driver when the state of its link
+ * can be determined through e.g: an out of band MMIO register.
+ */
+int phylink_fixed_state_cb(struct phylink *pl,
+			   void (*cb)(struct net_device *dev,
+				      struct phylink_link_state *state))
+{
+	/* It does not make sense to let the link be overriden unless we use
+	 * MLO_AN_FIXED
+	 */
+	if (pl->link_an_mode != MLO_AN_FIXED)
+		return -EINVAL;
+
+	mutex_lock(&pl->state_mutex);
+	pl->get_fixed_state = cb;
+	mutex_unlock(&pl->state_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(phylink_fixed_state_cb);
+
+/**
+ * phylink_mac_change() - notify phylink of a change in MAC state
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @up: indicates whether the link is currently up.
+ *
+ * The MAC driver should call this driver when the state of its link
+ * changes (eg, link failure, new negotiation results, etc.)
+ */
 void phylink_mac_change(struct phylink *pl, bool up)
 {
 	if (!up)
@@ -739,9 +871,17 @@ void phylink_mac_change(struct phylink *pl, bool up)
 }
 EXPORT_SYMBOL_GPL(phylink_mac_change);
 
+/**
+ * phylink_start() - start a phylink instance
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ *
+ * Start the phylink instance specified by @pl, configuring the MAC for the
+ * desired link mode(s) and negotiation style. This should be called from the
+ * network device driver's &struct net_device_ops ndo_open() method.
+ */
 void phylink_start(struct phylink *pl)
 {
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	netdev_info(pl->netdev, "configuring for %s/%s link mode\n",
 		    phylink_an_mode_str(pl->link_an_mode),
@@ -754,6 +894,12 @@ void phylink_start(struct phylink *pl)
 	phylink_resolve_flow(pl, &pl->link_config);
 	phylink_mac_config(pl, &pl->link_config);
 
+	/* Restart autonegotiation if using 802.3z to ensure that the link
+	 * parameters are properly negotiated.  This is necessary for DSA
+	 * switches using 802.3z negotiation to ensure they see our modes.
+	 */
+	phylink_mac_an_restart(pl);
+
 	clear_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state);
 	phylink_run_resolve(pl);
 
@@ -764,9 +910,18 @@ void phylink_start(struct phylink *pl)
 }
 EXPORT_SYMBOL_GPL(phylink_start);
 
+/**
+ * phylink_stop() - stop a phylink instance
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ *
+ * Stop the phylink instance specified by @pl. This should be called from the
+ * network device driver's &struct net_device_ops ndo_stop() method.  The
+ * network device's carrier state should not be changed prior to calling this
+ * function.
+ */
 void phylink_stop(struct phylink *pl)
 {
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev)
 		phy_stop(pl->phydev);
@@ -779,9 +934,18 @@ void phylink_stop(struct phylink *pl)
 }
 EXPORT_SYMBOL_GPL(phylink_stop);
 
+/**
+ * phylink_ethtool_get_wol() - get the wake on lan parameters for the PHY
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @wol: a pointer to &struct ethtool_wolinfo to hold the read parameters
+ *
+ * Read the wake on lan parameters from the PHY attached to the phylink
+ * instance specified by @pl. If no PHY is currently attached, report no
+ * support for wake on lan.
+ */
 void phylink_ethtool_get_wol(struct phylink *pl, struct ethtool_wolinfo *wol)
 {
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	wol->supported = 0;
 	wol->wolopts = 0;
@@ -791,11 +955,22 @@ void phylink_ethtool_get_wol(struct phylink *pl, struct ethtool_wolinfo *wol)
 }
 EXPORT_SYMBOL_GPL(phylink_ethtool_get_wol);
 
+/**
+ * phylink_ethtool_set_wol() - set wake on lan parameters
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @wol: a pointer to &struct ethtool_wolinfo for the desired parameters
+ *
+ * Set the wake on lan parameters for the PHY attached to the phylink
+ * instance specified by @pl. If no PHY is attached, returns %EOPNOTSUPP
+ * error.
+ *
+ * Returns zero on success or negative errno code.
+ */
 int phylink_ethtool_set_wol(struct phylink *pl, struct ethtool_wolinfo *wol)
 {
 	int ret = -EOPNOTSUPP;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev)
 		ret = phy_ethtool_set_wol(pl->phydev, wol);
@@ -826,12 +1001,21 @@ static void phylink_get_ksettings(const struct phylink_link_state *state,
 				AUTONEG_DISABLE;
 }
 
+/**
+ * phylink_ethtool_ksettings_get() - get the current link settings
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @kset: a pointer to a &struct ethtool_link_ksettings to hold link settings
+ *
+ * Read the current link settings for the phylink instance specified by @pl.
+ * This will be the link settings read from the MAC, PHY or fixed link
+ * settings depending on the current negotiation mode.
+ */
 int phylink_ethtool_ksettings_get(struct phylink *pl,
 				  struct ethtool_link_ksettings *kset)
 {
 	struct phylink_link_state link_state;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev) {
 		phy_ethtool_ksettings_get(pl->phydev, kset);
@@ -851,14 +1035,13 @@ int phylink_ethtool_ksettings_get(struct phylink *pl,
 		phylink_get_ksettings(&link_state, kset);
 		break;
 
-	case MLO_AN_SGMII:
+	case MLO_AN_INBAND:
 		/* If there is a phy attached, then use the reported
 		 * settings from the phy with no modification.
 		 */
 		if (pl->phydev)
 			break;
 
-	case MLO_AN_8023Z:
 		phylink_get_mac_state(pl, &link_state);
 
 		/* The MAC is reporting the link results from its own PCS
@@ -873,6 +1056,11 @@ int phylink_ethtool_ksettings_get(struct phylink *pl,
 }
 EXPORT_SYMBOL_GPL(phylink_ethtool_ksettings_get);
 
+/**
+ * phylink_ethtool_ksettings_set() - set the link settings
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @kset: a pointer to a &struct ethtool_link_ksettings for the desired modes
+ */
 int phylink_ethtool_ksettings_set(struct phylink *pl,
 				  const struct ethtool_link_ksettings *kset)
 {
@@ -880,7 +1068,7 @@ int phylink_ethtool_ksettings_set(struct phylink *pl,
 	struct phylink_link_state config;
 	int ret;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (kset->base.autoneg != AUTONEG_DISABLE &&
 	    kset->base.autoneg != AUTONEG_ENABLE)
@@ -967,11 +1155,22 @@ int phylink_ethtool_ksettings_set(struct phylink *pl,
 }
 EXPORT_SYMBOL_GPL(phylink_ethtool_ksettings_set);
 
+/**
+ * phylink_ethtool_nway_reset() - restart negotiation
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ *
+ * Restart negotiation for the phylink instance specified by @pl. This will
+ * cause any attached phy to restart negotiation with the link partner, and
+ * if the MAC is in a BaseX mode, the MAC will also be requested to restart
+ * negotiation.
+ *
+ * Returns zero on success, or negative error code.
+ */
 int phylink_ethtool_nway_reset(struct phylink *pl)
 {
 	int ret = 0;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev)
 		ret = phy_restart_aneg(pl->phydev);
@@ -981,10 +1180,15 @@ int phylink_ethtool_nway_reset(struct phylink *pl)
 }
 EXPORT_SYMBOL_GPL(phylink_ethtool_nway_reset);
 
+/**
+ * phylink_ethtool_get_pauseparam() - get the current pause parameters
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @pause: a pointer to a &struct ethtool_pauseparam
+ */
 void phylink_ethtool_get_pauseparam(struct phylink *pl,
 				    struct ethtool_pauseparam *pause)
 {
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	pause->autoneg = !!(pl->link_config.pause & MLO_PAUSE_AN);
 	pause->rx_pause = !!(pl->link_config.pause & MLO_PAUSE_RX);
@@ -992,12 +1196,17 @@ void phylink_ethtool_get_pauseparam(struct phylink *pl,
 }
 EXPORT_SYMBOL_GPL(phylink_ethtool_get_pauseparam);
 
+/**
+ * phylink_ethtool_set_pauseparam() - set the current pause parameters
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @pause: a pointer to a &struct ethtool_pauseparam
+ */
 int phylink_ethtool_set_pauseparam(struct phylink *pl,
 				   struct ethtool_pauseparam *pause)
 {
 	struct phylink_link_state *config = &pl->link_config;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (!phylink_test(pl->supported, Pause) &&
 	    !phylink_test(pl->supported, Asym_Pause))
@@ -1030,8 +1239,7 @@ int phylink_ethtool_set_pauseparam(struct phylink *pl,
 			phylink_mac_config(pl, config);
 			break;
 
-		case MLO_AN_SGMII:
-		case MLO_AN_8023Z:
+		case MLO_AN_INBAND:
 			phylink_mac_config(pl, config);
 			phylink_mac_an_restart(pl);
 			break;
@@ -1070,24 +1278,21 @@ int phylink_ethtool_get_module_eeprom(struct phylink *pl,
 }
 EXPORT_SYMBOL_GPL(phylink_ethtool_get_module_eeprom);
 
-int phylink_init_eee(struct phylink *pl, bool clk_stop_enable)
-{
-	int ret = -EPROTONOSUPPORT;
-
-	WARN_ON(!lockdep_rtnl_is_held());
-
-	if (pl->phydev)
-		ret = phy_init_eee(pl->phydev, clk_stop_enable);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(phylink_init_eee);
-
+/**
+ * phylink_ethtool_get_eee_err() - read the energy efficient ethernet error
+ *   counter
+ * @pl: a pointer to a &struct phylink returned from phylink_create().
+ *
+ * Read the Energy Efficient Ethernet error counter from the PHY associated
+ * with the phylink instance specified by @pl.
+ *
+ * Returns positive error counter value, or negative error code.
+ */
 int phylink_get_eee_err(struct phylink *pl)
 {
 	int ret = 0;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev)
 		ret = phy_get_eee_err(pl->phydev);
@@ -1096,11 +1301,16 @@ int phylink_get_eee_err(struct phylink *pl)
 }
 EXPORT_SYMBOL_GPL(phylink_get_eee_err);
 
+/**
+ * phylink_ethtool_get_eee() - read the energy efficient ethernet parameters
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @eee: a pointer to a &struct ethtool_eee for the read parameters
+ */
 int phylink_ethtool_get_eee(struct phylink *pl, struct ethtool_eee *eee)
 {
 	int ret = -EOPNOTSUPP;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev)
 		ret = phy_ethtool_get_eee(pl->phydev, eee);
@@ -1109,11 +1319,16 @@ int phylink_ethtool_get_eee(struct phylink *pl, struct ethtool_eee *eee)
 }
 EXPORT_SYMBOL_GPL(phylink_ethtool_get_eee);
 
+/**
+ * phylink_ethtool_set_eee() - set the energy efficient ethernet parameters
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @eee: a pointer to a &struct ethtool_eee for the desired parameters
+ */
 int phylink_ethtool_set_eee(struct phylink *pl, struct ethtool_eee *eee)
 {
 	int ret = -EOPNOTSUPP;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev)
 		ret = phy_ethtool_set_eee(pl->phydev, eee);
@@ -1248,9 +1463,7 @@ static int phylink_mii_read(struct phylink *pl, unsigned int phy_id,
 	case MLO_AN_PHY:
 		return -EOPNOTSUPP;
 
-	case MLO_AN_SGMII:
-		/* No phy, fall through to 8023z method */
-	case MLO_AN_8023Z:
+	case MLO_AN_INBAND:
 		if (phy_id == 0) {
 			val = phylink_get_mac_state(pl, &state);
 			if (val < 0)
@@ -1275,24 +1488,40 @@ static int phylink_mii_write(struct phylink *pl, unsigned int phy_id,
 	case MLO_AN_PHY:
 		return -EOPNOTSUPP;
 
-	case MLO_AN_SGMII:
-		/* No phy, fall through to 8023z method */
-	case MLO_AN_8023Z:
+	case MLO_AN_INBAND:
 		break;
 	}
 
 	return 0;
 }
 
+/**
+ * phylink_mii_ioctl() - generic mii ioctl interface
+ * @pl: a pointer to a &struct phylink returned from phylink_create()
+ * @ifr: a pointer to a &struct ifreq for socket ioctls
+ * @cmd: ioctl cmd to execute
+ *
+ * Perform the specified MII ioctl on the PHY attached to the phylink instance
+ * specified by @pl. If no PHY is attached, emulate the presence of the PHY.
+ *
+ * Returns: zero on success or negative error code.
+ *
+ * %SIOCGMIIPHY:
+ *  read register from the current PHY.
+ * %SIOCGMIIREG:
+ *  read register from the specified PHY.
+ * %SIOCSMIIREG:
+ *  set a register on the specified PHY.
+ */
 int phylink_mii_ioctl(struct phylink *pl, struct ifreq *ifr, int cmd)
 {
 	struct mii_ioctl_data *mii = if_mii(ifr);
 	int  ret;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	if (pl->phydev) {
-		/* PHYs only exist for MLO_AN_PHY and MLO_AN_SGMII */
+		/* PHYs only exist for MLO_AN_PHY and SGMII */
 		switch (cmd) {
 		case SIOCGMIIPHY:
 			mii->phy_id = pl->phydev->mdio.addr;
@@ -1351,7 +1580,7 @@ static int phylink_sfp_module_insert(void *upstream,
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(support) = { 0, };
 	struct phylink_link_state config;
 	phy_interface_t iface;
-	int mode, ret = 0;
+	int ret = 0;
 	bool changed;
 	u8 port;
 
@@ -1359,14 +1588,13 @@ static int phylink_sfp_module_insert(void *upstream,
 	port = sfp_parse_port(pl->sfp_bus, id, support);
 	iface = sfp_parse_interface(pl->sfp_bus, id);
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	switch (iface) {
 	case PHY_INTERFACE_MODE_SGMII:
-		mode = MLO_AN_SGMII;
-		break;
 	case PHY_INTERFACE_MODE_1000BASEX:
-		mode = MLO_AN_8023Z;
+	case PHY_INTERFACE_MODE_2500BASEX:
+	case PHY_INTERFACE_MODE_10GKR:
 		break;
 	default:
 		return -EINVAL;
@@ -1384,16 +1612,18 @@ static int phylink_sfp_module_insert(void *upstream,
 	ret = phylink_validate(pl, support, &config);
 	if (ret) {
 		netdev_err(pl->netdev, "validation of %s/%s with support %*pb failed: %d\n",
-			   phylink_an_mode_str(mode), phy_modes(config.interface),
+			   phylink_an_mode_str(MLO_AN_INBAND),
+			   phy_modes(config.interface),
 			   __ETHTOOL_LINK_MODE_MASK_NBITS, support, ret);
 		return ret;
 	}
 
 	netdev_dbg(pl->netdev, "requesting link mode %s/%s with support %*pb\n",
-		   phylink_an_mode_str(mode), phy_modes(config.interface),
+		   phylink_an_mode_str(MLO_AN_INBAND),
+		   phy_modes(config.interface),
 		   __ETHTOOL_LINK_MODE_MASK_NBITS, support);
 
-	if (mode == MLO_AN_8023Z && pl->phydev)
+	if (phy_interface_mode_is_8023z(iface) && pl->phydev)
 		return -EINVAL;
 
 	changed = !bitmap_equal(pl->supported, support,
@@ -1403,15 +1633,15 @@ static int phylink_sfp_module_insert(void *upstream,
 		linkmode_copy(pl->link_config.advertising, config.advertising);
 	}
 
-	if (pl->link_an_mode != mode ||
+	if (pl->link_an_mode != MLO_AN_INBAND ||
 	    pl->link_config.interface != config.interface) {
 		pl->link_config.interface = config.interface;
-		pl->link_an_mode = mode;
+		pl->link_an_mode = MLO_AN_INBAND;
 
 		changed = true;
 
 		netdev_info(pl->netdev, "switched to %s/%s link mode\n",
-			    phylink_an_mode_str(mode),
+			    phylink_an_mode_str(MLO_AN_INBAND),
 			    phy_modes(config.interface));
 	}
 
@@ -1428,7 +1658,7 @@ static void phylink_sfp_link_down(void *upstream)
 {
 	struct phylink *pl = upstream;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	set_bit(PHYLINK_DISABLE_LINK, &pl->phylink_disable_state);
 	queue_work(system_power_efficient_wq, &pl->resolve);
@@ -1439,7 +1669,7 @@ static void phylink_sfp_link_up(void *upstream)
 {
 	struct phylink *pl = upstream;
 
-	WARN_ON(!lockdep_rtnl_is_held());
+	ASSERT_RTNL();
 
 	clear_bit(PHYLINK_DISABLE_LINK, &pl->phylink_disable_state);
 	phylink_run_resolve(pl);
diff --git a/drivers/net/phy/qsemi.c b/drivers/net/phy/qsemi.c
index dbef8002bc28..889a4dce1648 100644
--- a/drivers/net/phy/qsemi.c
+++ b/drivers/net/phy/qsemi.c
@@ -118,8 +118,6 @@ static struct phy_driver qs6612_driver[] = { {
 	.features	= PHY_BASIC_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= qs6612_config_init,
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.ack_interrupt	= qs6612_ack_interrupt,
 	.config_intr	= qs6612_config_intr,
 } };
diff --git a/drivers/net/phy/realtek.c b/drivers/net/phy/realtek.c
index eda0a6e86918..9f48ecf9c627 100644
--- a/drivers/net/phy/realtek.c
+++ b/drivers/net/phy/realtek.c
@@ -13,29 +13,44 @@
  * option) any later version.
  *
  */
+#include <linux/bitops.h>
 #include <linux/phy.h>
 #include <linux/module.h>
 
-#define RTL821x_PHYSR		0x11
-#define RTL821x_PHYSR_DUPLEX	0x2000
-#define RTL821x_PHYSR_SPEED	0xc000
-#define RTL821x_INER		0x12
-#define RTL821x_INER_INIT	0x6400
-#define RTL821x_INSR		0x13
-#define RTL821x_PAGE_SELECT	0x1f
-#define RTL8211E_INER_LINK_STATUS 0x400
+#define RTL821x_PHYSR				0x11
+#define RTL821x_PHYSR_DUPLEX			BIT(13)
+#define RTL821x_PHYSR_SPEED			GENMASK(15, 14)
 
-#define RTL8211F_INER_LINK_STATUS 0x0010
-#define RTL8211F_INSR		0x1d
-#define RTL8211F_TX_DELAY	0x100
+#define RTL821x_INER				0x12
+#define RTL8211B_INER_INIT			0x6400
+#define RTL8211E_INER_LINK_STATUS		BIT(10)
+#define RTL8211F_INER_LINK_STATUS		BIT(4)
 
-#define RTL8201F_ISR		0x1e
-#define RTL8201F_IER		0x13
+#define RTL821x_INSR				0x13
+
+#define RTL821x_PAGE_SELECT			0x1f
+
+#define RTL8211F_INSR				0x1d
+
+#define RTL8211F_TX_DELAY			BIT(8)
+
+#define RTL8201F_ISR				0x1e
+#define RTL8201F_IER				0x13
 
 MODULE_DESCRIPTION("Realtek PHY driver");
 MODULE_AUTHOR("Johnson Leung");
 MODULE_LICENSE("GPL");
 
+static int rtl821x_read_page(struct phy_device *phydev)
+{
+	return __phy_read(phydev, RTL821x_PAGE_SELECT);
+}
+
+static int rtl821x_write_page(struct phy_device *phydev, int page)
+{
+	return __phy_write(phydev, RTL821x_PAGE_SELECT, page);
+}
+
 static int rtl8201_ack_interrupt(struct phy_device *phydev)
 {
 	int err;
@@ -58,31 +73,21 @@ static int rtl8211f_ack_interrupt(struct phy_device *phydev)
 {
 	int err;
 
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0xa43);
-	err = phy_read(phydev, RTL8211F_INSR);
-	/* restore to default page 0 */
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0x0);
+	err = phy_read_paged(phydev, 0xa43, RTL8211F_INSR);
 
 	return (err < 0) ? err : 0;
 }
 
 static int rtl8201_config_intr(struct phy_device *phydev)
 {
-	int err;
-
-	/* switch to page 7 */
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0x7);
+	u16 val;
 
 	if (phydev->interrupts == PHY_INTERRUPT_ENABLED)
-		err = phy_write(phydev, RTL8201F_IER,
-				BIT(13) | BIT(12) | BIT(11));
+		val = BIT(13) | BIT(12) | BIT(11);
 	else
-		err = phy_write(phydev, RTL8201F_IER, 0);
-
-	/* restore to default page 0 */
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0x0);
+		val = 0;
 
-	return err;
+	return phy_write_paged(phydev, 0x7, RTL8201F_IER, val);
 }
 
 static int rtl8211b_config_intr(struct phy_device *phydev)
@@ -91,7 +96,7 @@ static int rtl8211b_config_intr(struct phy_device *phydev)
 
 	if (phydev->interrupts == PHY_INTERRUPT_ENABLED)
 		err = phy_write(phydev, RTL821x_INER,
-				RTL821x_INER_INIT);
+				RTL8211B_INER_INIT);
 	else
 		err = phy_write(phydev, RTL821x_INER, 0);
 
@@ -113,43 +118,31 @@ static int rtl8211e_config_intr(struct phy_device *phydev)
 
 static int rtl8211f_config_intr(struct phy_device *phydev)
 {
-	int err;
+	u16 val;
 
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0xa42);
 	if (phydev->interrupts == PHY_INTERRUPT_ENABLED)
-		err = phy_write(phydev, RTL821x_INER,
-				RTL8211F_INER_LINK_STATUS);
+		val = RTL8211F_INER_LINK_STATUS;
 	else
-		err = phy_write(phydev, RTL821x_INER, 0);
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0);
+		val = 0;
 
-	return err;
+	return phy_write_paged(phydev, 0xa42, RTL821x_INER, val);
 }
 
 static int rtl8211f_config_init(struct phy_device *phydev)
 {
 	int ret;
-	u16 reg;
+	u16 val = 0;
 
 	ret = genphy_config_init(phydev);
 	if (ret < 0)
 		return ret;
 
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0xd08);
-	reg = phy_read(phydev, 0x11);
-
 	/* enable TX-delay for rgmii-id and rgmii-txid, otherwise disable it */
 	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
 	    phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
-		reg |= RTL8211F_TX_DELAY;
-	else
-		reg &= ~RTL8211F_TX_DELAY;
-
-	phy_write(phydev, 0x11, reg);
-	/* restore to default page 0 */
-	phy_write(phydev, RTL821x_PAGE_SELECT, 0x0);
+		val = RTL8211F_TX_DELAY;
 
-	return 0;
+	return phy_modify_paged(phydev, 0xd08, 0x11, RTL8211F_TX_DELAY, val);
 }
 
 static struct phy_driver realtek_drvs[] = {
@@ -159,38 +152,34 @@ static struct phy_driver realtek_drvs[] = {
 		.phy_id_mask    = 0x0000ffff,
 		.features       = PHY_BASIC_FEATURES,
 		.flags          = PHY_HAS_INTERRUPT,
-		.config_aneg    = &genphy_config_aneg,
-		.read_status    = &genphy_read_status,
 	}, {
 		.phy_id		= 0x001cc816,
 		.name		= "RTL8201F 10/100Mbps Ethernet",
 		.phy_id_mask	= 0x001fffff,
 		.features	= PHY_BASIC_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
-		.config_aneg	= &genphy_config_aneg,
-		.read_status	= &genphy_read_status,
 		.ack_interrupt	= &rtl8201_ack_interrupt,
 		.config_intr	= &rtl8201_config_intr,
 		.suspend	= genphy_suspend,
 		.resume		= genphy_resume,
+		.read_page	= rtl821x_read_page,
+		.write_page	= rtl821x_write_page,
 	}, {
 		.phy_id		= 0x001cc912,
 		.name		= "RTL8211B Gigabit Ethernet",
 		.phy_id_mask	= 0x001fffff,
 		.features	= PHY_GBIT_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
-		.config_aneg	= &genphy_config_aneg,
-		.read_status	= &genphy_read_status,
 		.ack_interrupt	= &rtl821x_ack_interrupt,
 		.config_intr	= &rtl8211b_config_intr,
+		.read_mmd	= &genphy_read_mmd_unsupported,
+		.write_mmd	= &genphy_write_mmd_unsupported,
 	}, {
 		.phy_id		= 0x001cc914,
 		.name		= "RTL8211DN Gigabit Ethernet",
 		.phy_id_mask	= 0x001fffff,
 		.features	= PHY_GBIT_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
-		.config_aneg	= genphy_config_aneg,
-		.read_status	= genphy_read_status,
 		.ack_interrupt	= rtl821x_ack_interrupt,
 		.config_intr	= rtl8211e_config_intr,
 		.suspend	= genphy_suspend,
@@ -201,8 +190,6 @@ static struct phy_driver realtek_drvs[] = {
 		.phy_id_mask	= 0x001fffff,
 		.features	= PHY_GBIT_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
-		.config_aneg	= &genphy_config_aneg,
-		.read_status	= &genphy_read_status,
 		.ack_interrupt	= &rtl821x_ack_interrupt,
 		.config_intr	= &rtl8211e_config_intr,
 		.suspend	= genphy_suspend,
@@ -213,13 +200,13 @@ static struct phy_driver realtek_drvs[] = {
 		.phy_id_mask	= 0x001fffff,
 		.features	= PHY_GBIT_FEATURES,
 		.flags		= PHY_HAS_INTERRUPT,
-		.config_aneg	= &genphy_config_aneg,
 		.config_init	= &rtl8211f_config_init,
-		.read_status	= &genphy_read_status,
 		.ack_interrupt	= &rtl8211f_ack_interrupt,
 		.config_intr	= &rtl8211f_config_intr,
 		.suspend	= genphy_suspend,
 		.resume		= genphy_resume,
+		.read_page	= rtl821x_read_page,
+		.write_page	= rtl821x_write_page,
 	},
 };
 
diff --git a/drivers/net/phy/rockchip.c b/drivers/net/phy/rockchip.c
index c092af137056..f1da70b9b55f 100644
--- a/drivers/net/phy/rockchip.c
+++ b/drivers/net/phy/rockchip.c
@@ -213,7 +213,6 @@ static struct phy_driver rockchip_phy_driver[] = {
 	.soft_reset		= genphy_soft_reset,
 	.config_init		= rockchip_integrated_phy_config_init,
 	.config_aneg		= rockchip_config_aneg,
-	.read_status		= genphy_read_status,
 	.suspend		= genphy_suspend,
 	.resume			= rockchip_phy_resume,
 },
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
index ab64a142b832..8961209ee949 100644
--- a/drivers/net/phy/sfp-bus.c
+++ b/drivers/net/phy/sfp-bus.c
@@ -8,10 +8,14 @@
 
 #include "sfp.h"
 
+/**
+ * struct sfp_bus - internal representation of a sfp bus
+ */
 struct sfp_bus {
+	/* private: */
 	struct kref kref;
 	struct list_head node;
-	struct device_node *device_node;
+	struct fwnode_handle *fwnode;
 
 	const struct sfp_socket_ops *socket_ops;
 	struct device *sfp_dev;
@@ -26,6 +30,20 @@ struct sfp_bus {
 	bool started;
 };
 
+/**
+ * sfp_parse_port() - Parse the EEPROM base ID, setting the port type
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ * @id: a pointer to the module's &struct sfp_eeprom_id
+ * @support: optional pointer to an array of unsigned long for the
+ *   ethtool support mask
+ *
+ * Parse the EEPROM identification given in @id, and return one of
+ * %PORT_TP, %PORT_FIBRE or %PORT_OTHER. If @support is non-%NULL,
+ * also set the ethtool %ETHTOOL_LINK_MODE_xxx_BIT corresponding with
+ * the connector type.
+ *
+ * If the port type is not known, returns %PORT_OTHER.
+ */
 int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 		   unsigned long *support)
 {
@@ -39,21 +57,19 @@ int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	case SFP_CONNECTOR_MT_RJ:
 	case SFP_CONNECTOR_MU:
 	case SFP_CONNECTOR_OPTICAL_PIGTAIL:
-		if (support)
-			phylink_set(support, FIBRE);
 		port = PORT_FIBRE;
 		break;
 
 	case SFP_CONNECTOR_RJ45:
-		if (support)
-			phylink_set(support, TP);
 		port = PORT_TP;
 		break;
 
+	case SFP_CONNECTOR_COPPER_PIGTAIL:
+		port = PORT_DA;
+		break;
+
 	case SFP_CONNECTOR_UNSPEC:
 		if (id->base.e1000_base_t) {
-			if (support)
-				phylink_set(support, TP);
 			port = PORT_TP;
 			break;
 		}
@@ -62,7 +78,6 @@ int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	case SFP_CONNECTOR_MPO_1X12:
 	case SFP_CONNECTOR_MPO_2X16:
 	case SFP_CONNECTOR_HSSDC_II:
-	case SFP_CONNECTOR_COPPER_PIGTAIL:
 	case SFP_CONNECTOR_NOSEPARATE:
 	case SFP_CONNECTOR_MXC_2X16:
 		port = PORT_OTHER;
@@ -74,10 +89,40 @@ int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 		break;
 	}
 
+	if (support) {
+		switch (port) {
+		case PORT_FIBRE:
+			phylink_set(support, FIBRE);
+			break;
+
+		case PORT_TP:
+			phylink_set(support, TP);
+			break;
+		}
+	}
+
 	return port;
 }
 EXPORT_SYMBOL_GPL(sfp_parse_port);
 
+/**
+ * sfp_parse_interface() - Parse the phy_interface_t
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ * @id: a pointer to the module's &struct sfp_eeprom_id
+ *
+ * Derive the phy_interface_t mode for the information found in the
+ * module's identifying EEPROM. There is no standard or defined way
+ * to derive this information, so we use some heuristics.
+ *
+ * If the encoding is 64b66b, then the module must be >= 10G, so
+ * return %PHY_INTERFACE_MODE_10GKR.
+ *
+ * If it's 8b10b, then it's 1G or slower. If it's definitely a fibre
+ * module, return %PHY_INTERFACE_MODE_1000BASEX mode, otherwise return
+ * %PHY_INTERFACE_MODE_SGMII mode.
+ *
+ * If the encoding is not known, return %PHY_INTERFACE_MODE_NA.
+ */
 phy_interface_t sfp_parse_interface(struct sfp_bus *bus,
 				    const struct sfp_eeprom_id *id)
 {
@@ -107,6 +152,11 @@ phy_interface_t sfp_parse_interface(struct sfp_bus *bus,
 		break;
 
 	default:
+		if (id->base.e1000_base_cx) {
+			iface = PHY_INTERFACE_MODE_1000BASEX;
+			break;
+		}
+
 		iface = PHY_INTERFACE_MODE_NA;
 		dev_err(bus->sfp_dev,
 			"SFP module encoding does not support 8b10b nor 64b66b\n");
@@ -117,13 +167,38 @@ phy_interface_t sfp_parse_interface(struct sfp_bus *bus,
 }
 EXPORT_SYMBOL_GPL(sfp_parse_interface);
 
+/**
+ * sfp_parse_support() - Parse the eeprom id for supported link modes
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ * @id: a pointer to the module's &struct sfp_eeprom_id
+ * @support: pointer to an array of unsigned long for the ethtool support mask
+ *
+ * Parse the EEPROM identification information and derive the supported
+ * ethtool link modes for the module.
+ */
 void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 		       unsigned long *support)
 {
+	unsigned int br_min, br_nom, br_max;
+
 	phylink_set(support, Autoneg);
 	phylink_set(support, Pause);
 	phylink_set(support, Asym_Pause);
 
+	/* Decode the bitrate information to MBd */
+	br_min = br_nom = br_max = 0;
+	if (id->base.br_nominal) {
+		if (id->base.br_nominal != 255) {
+			br_nom = id->base.br_nominal * 100;
+			br_min = br_nom + id->base.br_nominal * id->ext.br_min;
+			br_max = br_nom + id->base.br_nominal * id->ext.br_max;
+		} else if (id->ext.br_max) {
+			br_nom = 250 * id->ext.br_max;
+			br_max = br_nom + br_nom * id->ext.br_min / 100;
+			br_min = br_nom - br_nom * id->ext.br_min / 100;
+		}
+	}
+
 	/* Set ethtool support from the compliance fields. */
 	if (id->base.e10g_base_sr)
 		phylink_set(support, 10000baseSR_Full);
@@ -142,6 +217,34 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 		phylink_set(support, 1000baseT_Full);
 	}
 
+	/* 1000Base-PX or 1000Base-BX10 */
+	if ((id->base.e_base_px || id->base.e_base_bx10) &&
+	    br_min <= 1300 && br_max >= 1200)
+		phylink_set(support, 1000baseX_Full);
+
+	/* For active or passive cables, select the link modes
+	 * based on the bit rates and the cable compliance bytes.
+	 */
+	if ((id->base.sfp_ct_passive || id->base.sfp_ct_active) && br_nom) {
+		/* This may look odd, but some manufacturers use 12000MBd */
+		if (br_min <= 12000 && br_max >= 10300)
+			phylink_set(support, 10000baseCR_Full);
+		if (br_min <= 3200 && br_max >= 3100)
+			phylink_set(support, 2500baseX_Full);
+		if (br_min <= 1300 && br_max >= 1200)
+			phylink_set(support, 1000baseX_Full);
+	}
+	if (id->base.sfp_ct_passive) {
+		if (id->base.passive.sff8431_app_e)
+			phylink_set(support, 10000baseCR_Full);
+	}
+	if (id->base.sfp_ct_active) {
+		if (id->base.active.sff8431_app_e ||
+		    id->base.active.sff8431_lim) {
+			phylink_set(support, 10000baseCR_Full);
+		}
+	}
+
 	switch (id->base.extended_cc) {
 	case 0x00: /* Unspecified */
 		break;
@@ -175,35 +278,6 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 		if (id->base.br_nominal >= 12)
 			phylink_set(support, 1000baseX_Full);
 	}
-
-	switch (id->base.connector) {
-	case SFP_CONNECTOR_SC:
-	case SFP_CONNECTOR_FIBERJACK:
-	case SFP_CONNECTOR_LC:
-	case SFP_CONNECTOR_MT_RJ:
-	case SFP_CONNECTOR_MU:
-	case SFP_CONNECTOR_OPTICAL_PIGTAIL:
-		break;
-
-	case SFP_CONNECTOR_UNSPEC:
-		if (id->base.e1000_base_t)
-			break;
-
-	case SFP_CONNECTOR_SG: /* guess */
-	case SFP_CONNECTOR_MPO_1X12:
-	case SFP_CONNECTOR_MPO_2X16:
-	case SFP_CONNECTOR_HSSDC_II:
-	case SFP_CONNECTOR_COPPER_PIGTAIL:
-	case SFP_CONNECTOR_NOSEPARATE:
-	case SFP_CONNECTOR_MXC_2X16:
-	default:
-		/* a guess at the supported link modes */
-		dev_warn(bus->sfp_dev,
-			 "Guessing link modes, please report...\n");
-		phylink_set(support, 1000baseT_Half);
-		phylink_set(support, 1000baseT_Full);
-		break;
-	}
 }
 EXPORT_SYMBOL_GPL(sfp_parse_support);
 
@@ -215,7 +289,7 @@ static const struct sfp_upstream_ops *sfp_get_upstream_ops(struct sfp_bus *bus)
 	return bus->registered ? bus->upstream_ops : NULL;
 }
 
-static struct sfp_bus *sfp_bus_get(struct device_node *np)
+static struct sfp_bus *sfp_bus_get(struct fwnode_handle *fwnode)
 {
 	struct sfp_bus *sfp, *new, *found = NULL;
 
@@ -224,7 +298,7 @@ static struct sfp_bus *sfp_bus_get(struct device_node *np)
 	mutex_lock(&sfp_mutex);
 
 	list_for_each_entry(sfp, &sfp_buses, node) {
-		if (sfp->device_node == np) {
+		if (sfp->fwnode == fwnode) {
 			kref_get(&sfp->kref);
 			found = sfp;
 			break;
@@ -233,7 +307,7 @@ static struct sfp_bus *sfp_bus_get(struct device_node *np)
 
 	if (!found && new) {
 		kref_init(&new->kref);
-		new->device_node = np;
+		new->fwnode = fwnode;
 		list_add(&new->node, &sfp_buses);
 		found = new;
 		new = NULL;
@@ -246,7 +320,7 @@ static struct sfp_bus *sfp_bus_get(struct device_node *np)
 	return found;
 }
 
-static void sfp_bus_release(struct kref *kref) __releases(sfp_mutex)
+static void sfp_bus_release(struct kref *kref)
 {
 	struct sfp_bus *bus = container_of(kref, struct sfp_bus, kref);
 
@@ -293,6 +367,16 @@ static void sfp_unregister_bus(struct sfp_bus *bus)
 	bus->registered = false;
 }
 
+/**
+ * sfp_get_module_info() - Get the ethtool_modinfo for a SFP module
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ * @modinfo: a &struct ethtool_modinfo
+ *
+ * Fill in the type and eeprom_len parameters in @modinfo for a module on
+ * the sfp bus specified by @bus.
+ *
+ * Returns 0 on success or a negative errno number.
+ */
 int sfp_get_module_info(struct sfp_bus *bus, struct ethtool_modinfo *modinfo)
 {
 	if (!bus->registered)
@@ -301,6 +385,17 @@ int sfp_get_module_info(struct sfp_bus *bus, struct ethtool_modinfo *modinfo)
 }
 EXPORT_SYMBOL_GPL(sfp_get_module_info);
 
+/**
+ * sfp_get_module_eeprom() - Read the SFP module EEPROM
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ * @ee: a &struct ethtool_eeprom
+ * @data: buffer to contain the EEPROM data (must be at least @ee->len bytes)
+ *
+ * Read the EEPROM as specified by the supplied @ee. See the documentation
+ * for &struct ethtool_eeprom for the region to be read.
+ *
+ * Returns 0 on success or a negative errno number.
+ */
 int sfp_get_module_eeprom(struct sfp_bus *bus, struct ethtool_eeprom *ee,
 			  u8 *data)
 {
@@ -310,6 +405,15 @@ int sfp_get_module_eeprom(struct sfp_bus *bus, struct ethtool_eeprom *ee,
 }
 EXPORT_SYMBOL_GPL(sfp_get_module_eeprom);
 
+/**
+ * sfp_upstream_start() - Inform the SFP that the network device is up
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ *
+ * Inform the SFP socket that the network device is now up, so that the
+ * module can be enabled by allowing TX_DISABLE to be deasserted. This
+ * should be called from the network device driver's &struct net_device_ops
+ * ndo_open() method.
+ */
 void sfp_upstream_start(struct sfp_bus *bus)
 {
 	if (bus->registered)
@@ -318,6 +422,15 @@ void sfp_upstream_start(struct sfp_bus *bus)
 }
 EXPORT_SYMBOL_GPL(sfp_upstream_start);
 
+/**
+ * sfp_upstream_stop() - Inform the SFP that the network device is down
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ *
+ * Inform the SFP socket that the network device is now up, so that the
+ * module can be disabled by asserting TX_DISABLE, disabling the laser
+ * in optical modules. This should be called from the network device
+ * driver's &struct net_device_ops ndo_stop() method.
+ */
 void sfp_upstream_stop(struct sfp_bus *bus)
 {
 	if (bus->registered)
@@ -326,11 +439,24 @@ void sfp_upstream_stop(struct sfp_bus *bus)
 }
 EXPORT_SYMBOL_GPL(sfp_upstream_stop);
 
-struct sfp_bus *sfp_register_upstream(struct device_node *np,
+/**
+ * sfp_register_upstream() - Register the neighbouring device
+ * @fwnode: firmware node for the SFP bus
+ * @ndev: network device associated with the interface
+ * @upstream: the upstream private data
+ * @ops: the upstream's &struct sfp_upstream_ops
+ *
+ * Register the upstream device (eg, PHY) with the SFP bus. MAC drivers
+ * should use phylink, which will call this function for them. Returns
+ * a pointer to the allocated &struct sfp_bus.
+ *
+ * On error, returns %NULL.
+ */
+struct sfp_bus *sfp_register_upstream(struct fwnode_handle *fwnode,
 				      struct net_device *ndev, void *upstream,
 				      const struct sfp_upstream_ops *ops)
 {
-	struct sfp_bus *bus = sfp_bus_get(np);
+	struct sfp_bus *bus = sfp_bus_get(fwnode);
 	int ret = 0;
 
 	if (bus) {
@@ -353,6 +479,13 @@ struct sfp_bus *sfp_register_upstream(struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(sfp_register_upstream);
 
+/**
+ * sfp_unregister_upstream() - Unregister sfp bus
+ * @bus: a pointer to the &struct sfp_bus structure for the sfp module
+ *
+ * Unregister a previously registered upstream connection for the SFP
+ * module. @bus is returned from sfp_register_upstream().
+ */
 void sfp_unregister_upstream(struct sfp_bus *bus)
 {
 	rtnl_lock();
@@ -434,7 +567,7 @@ EXPORT_SYMBOL_GPL(sfp_module_remove);
 struct sfp_bus *sfp_register_socket(struct device *dev, struct sfp *sfp,
 				    const struct sfp_socket_ops *ops)
 {
-	struct sfp_bus *bus = sfp_bus_get(dev->of_node);
+	struct sfp_bus *bus = sfp_bus_get(dev->fwnode);
 	int ret = 0;
 
 	if (bus) {
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
index 9dfc1c4c954f..6c7d9289078d 100644
--- a/drivers/net/phy/sfp.c
+++ b/drivers/net/phy/sfp.c
@@ -98,12 +98,18 @@ static const enum gpiod_flags gpio_flags[] = {
 
 static DEFINE_MUTEX(sfp_mutex);
 
+struct sff_data {
+	unsigned int gpios;
+	bool (*module_supported)(const struct sfp_eeprom_id *id);
+};
+
 struct sfp {
 	struct device *dev;
 	struct i2c_adapter *i2c;
 	struct mii_bus *i2c_mii;
 	struct sfp_bus *sfp_bus;
 	struct phy_device *mod_phy;
+	const struct sff_data *type;
 
 	unsigned int (*get_state)(struct sfp *);
 	void (*set_state)(struct sfp *, unsigned int);
@@ -123,6 +129,36 @@ struct sfp {
 	struct sfp_eeprom_id id;
 };
 
+static bool sff_module_supported(const struct sfp_eeprom_id *id)
+{
+	return id->base.phys_id == SFP_PHYS_ID_SFF &&
+	       id->base.phys_ext_id == SFP_PHYS_EXT_ID_SFP;
+}
+
+static const struct sff_data sff_data = {
+	.gpios = SFP_F_LOS | SFP_F_TX_FAULT | SFP_F_TX_DISABLE,
+	.module_supported = sff_module_supported,
+};
+
+static bool sfp_module_supported(const struct sfp_eeprom_id *id)
+{
+	return id->base.phys_id == SFP_PHYS_ID_SFP &&
+	       id->base.phys_ext_id == SFP_PHYS_EXT_ID_SFP;
+}
+
+static const struct sff_data sfp_data = {
+	.gpios = SFP_F_PRESENT | SFP_F_LOS | SFP_F_TX_FAULT |
+		 SFP_F_TX_DISABLE | SFP_F_RATE_SELECT,
+	.module_supported = sfp_module_supported,
+};
+
+static const struct of_device_id sfp_of_match[] = {
+	{ .compatible = "sff,sff", .data = &sff_data, },
+	{ .compatible = "sff,sfp", .data = &sfp_data, },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, sfp_of_match);
+
 static unsigned long poll_jiffies;
 
 static unsigned int sfp_gpio_get_state(struct sfp *sfp)
@@ -141,6 +177,11 @@ static unsigned int sfp_gpio_get_state(struct sfp *sfp)
 	return state;
 }
 
+static unsigned int sff_gpio_get_state(struct sfp *sfp)
+{
+	return sfp_gpio_get_state(sfp) | SFP_F_PRESENT;
+}
+
 static void sfp_gpio_set_state(struct sfp *sfp, unsigned int state)
 {
 	if (state & SFP_F_PRESENT) {
@@ -315,12 +356,12 @@ static void sfp_sm_probe_phy(struct sfp *sfp)
 	msleep(T_PHY_RESET_MS);
 
 	phy = mdiobus_scan(sfp->i2c_mii, SFP_PHY_ADDR);
-	if (IS_ERR(phy)) {
-		dev_err(sfp->dev, "mdiobus scan returned %ld\n", PTR_ERR(phy));
+	if (phy == ERR_PTR(-ENODEV)) {
+		dev_info(sfp->dev, "no PHY detected\n");
 		return;
 	}
-	if (!phy) {
-		dev_info(sfp->dev, "no PHY detected\n");
+	if (IS_ERR(phy)) {
+		dev_err(sfp->dev, "mdiobus scan returned %ld\n", PTR_ERR(phy));
 		return;
 	}
 
@@ -425,11 +466,6 @@ static int sfp_sm_mod_probe(struct sfp *sfp)
 {
 	/* SFP module inserted - read I2C data */
 	struct sfp_eeprom_id id;
-	char vendor[17];
-	char part[17];
-	char sn[17];
-	char date[9];
-	char rev[5];
 	u8 check;
 	int err;
 
@@ -465,24 +501,17 @@ static int sfp_sm_mod_probe(struct sfp *sfp)
 
 	sfp->id = id;
 
-	memcpy(vendor, sfp->id.base.vendor_name, 16);
-	vendor[16] = '\0';
-	memcpy(part, sfp->id.base.vendor_pn, 16);
-	part[16] = '\0';
-	memcpy(rev, sfp->id.base.vendor_rev, 4);
-	rev[4] = '\0';
-	memcpy(sn, sfp->id.ext.vendor_sn, 16);
-	sn[16] = '\0';
-	memcpy(date, sfp->id.ext.datecode, 8);
-	date[8] = '\0';
-
-	dev_info(sfp->dev, "module %s %s rev %s sn %s dc %s\n",
-		 vendor, part, rev, sn, date);
-
-	/* We only support SFP modules, not the legacy GBIC modules. */
-	if (sfp->id.base.phys_id != SFP_PHYS_ID_SFP ||
-	    sfp->id.base.phys_ext_id != SFP_PHYS_EXT_ID_SFP) {
-		dev_err(sfp->dev, "module is not SFP - phys id 0x%02x 0x%02x\n",
+	dev_info(sfp->dev, "module %.*s %.*s rev %.*s sn %.*s dc %.*s\n",
+		 (int)sizeof(id.base.vendor_name), id.base.vendor_name,
+		 (int)sizeof(id.base.vendor_pn), id.base.vendor_pn,
+		 (int)sizeof(id.base.vendor_rev), id.base.vendor_rev,
+		 (int)sizeof(id.ext.vendor_sn), id.ext.vendor_sn,
+		 (int)sizeof(id.ext.datecode), id.ext.datecode);
+
+	/* Check whether we support this module */
+	if (!sfp->type->module_supported(&sfp->id)) {
+		dev_err(sfp->dev,
+			"module is not supported - phys id 0x%02x 0x%02x\n",
 			sfp->id.base.phys_id, sfp->id.base.phys_ext_id);
 		return -EINVAL;
 	}
@@ -683,20 +712,19 @@ static int sfp_module_eeprom(struct sfp *sfp, struct ethtool_eeprom *ee,
 		len = min_t(unsigned int, last, ETH_MODULE_SFF_8079_LEN);
 		len -= first;
 
-		ret = sfp->read(sfp, false, first, data, len);
+		ret = sfp_read(sfp, false, first, data, len);
 		if (ret < 0)
 			return ret;
 
 		first += len;
 		data += len;
 	}
-	if (first >= ETH_MODULE_SFF_8079_LEN &&
-	    first < ETH_MODULE_SFF_8472_LEN) {
+	if (first < ETH_MODULE_SFF_8472_LEN && last > ETH_MODULE_SFF_8079_LEN) {
 		len = min_t(unsigned int, last, ETH_MODULE_SFF_8472_LEN);
 		len -= first;
 		first -= ETH_MODULE_SFF_8079_LEN;
 
-		ret = sfp->read(sfp, true, first, data, len);
+		ret = sfp_read(sfp, true, first, data, len);
 		if (ret < 0)
 			return ret;
 	}
@@ -801,6 +829,7 @@ static void sfp_cleanup(void *data)
 
 static int sfp_probe(struct platform_device *pdev)
 {
+	const struct sff_data *sff;
 	struct sfp *sfp;
 	bool poll = false;
 	int irq, err, i;
@@ -815,10 +844,19 @@ static int sfp_probe(struct platform_device *pdev)
 	if (err < 0)
 		return err;
 
+	sff = sfp->type = &sfp_data;
+
 	if (pdev->dev.of_node) {
 		struct device_node *node = pdev->dev.of_node;
+		const struct of_device_id *id;
 		struct device_node *np;
 
+		id = of_match_node(sfp_of_match, node);
+		if (WARN_ON(!id))
+			return -EINVAL;
+
+		sff = sfp->type = id->data;
+
 		np = of_parse_phandle(node, "i2c-bus", 0);
 		if (np) {
 			struct i2c_adapter *i2c;
@@ -834,17 +872,22 @@ static int sfp_probe(struct platform_device *pdev)
 				return err;
 			}
 		}
+	}
 
-		for (i = 0; i < GPIO_MAX; i++) {
+	for (i = 0; i < GPIO_MAX; i++)
+		if (sff->gpios & BIT(i)) {
 			sfp->gpio[i] = devm_gpiod_get_optional(sfp->dev,
 					   gpio_of_names[i], gpio_flags[i]);
 			if (IS_ERR(sfp->gpio[i]))
 				return PTR_ERR(sfp->gpio[i]);
 		}
 
-		sfp->get_state = sfp_gpio_get_state;
-		sfp->set_state = sfp_gpio_set_state;
-	}
+	sfp->get_state = sfp_gpio_get_state;
+	sfp->set_state = sfp_gpio_set_state;
+
+	/* Modules that have no detect signal are always present */
+	if (!(sfp->gpio[GPIO_MODDEF0]))
+		sfp->get_state = sff_gpio_get_state;
 
 	sfp->sfp_bus = sfp_register_socket(sfp->dev, sfp, &sfp_module_ops);
 	if (!sfp->sfp_bus)
@@ -899,12 +942,6 @@ static int sfp_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id sfp_of_match[] = {
-	{ .compatible = "sff,sfp", },
-	{ },
-};
-MODULE_DEVICE_TABLE(of, sfp_of_match);
-
 static struct platform_driver sfp_driver = {
 	.probe = sfp_probe,
 	.remove = sfp_remove,
diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c
index 2306bfae057f..be399d645224 100644
--- a/drivers/net/phy/smsc.c
+++ b/drivers/net/phy/smsc.c
@@ -227,8 +227,6 @@ static struct phy_driver smsc_phy_driver[] = {
 	.probe		= smsc_phy_probe,
 
 	/* basic functions */
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.config_init	= smsc_phy_config_init,
 	.soft_reset	= smsc_phy_reset,
 
@@ -249,8 +247,6 @@ static struct phy_driver smsc_phy_driver[] = {
 	.probe		= smsc_phy_probe,
 
 	/* basic functions */
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.config_init	= smsc_phy_config_init,
 	.soft_reset	= smsc_phy_reset,
 
@@ -276,7 +272,6 @@ static struct phy_driver smsc_phy_driver[] = {
 	.probe		= smsc_phy_probe,
 
 	/* basic functions */
-	.config_aneg	= genphy_config_aneg,
 	.read_status	= lan87xx_read_status,
 	.config_init	= smsc_phy_config_init,
 	.soft_reset	= smsc_phy_reset,
@@ -303,8 +298,6 @@ static struct phy_driver smsc_phy_driver[] = {
 	.probe		= smsc_phy_probe,
 
 	/* basic functions */
-	.config_aneg	= genphy_config_aneg,
-	.read_status	= genphy_read_status,
 	.config_init	= lan911x_config_init,
 
 	/* IRQ related */
@@ -319,12 +312,11 @@ static struct phy_driver smsc_phy_driver[] = {
 	.name		= "SMSC LAN8710/LAN8720",
 
 	.features	= PHY_BASIC_FEATURES,
-	.flags		= PHY_HAS_INTERRUPT,
+	.flags		= PHY_HAS_INTERRUPT | PHY_RST_AFTER_CLK_EN,
 
 	.probe		= smsc_phy_probe,
 
 	/* basic functions */
-	.config_aneg	= genphy_config_aneg,
 	.read_status	= lan87xx_read_status,
 	.config_init	= smsc_phy_config_init,
 	.soft_reset	= smsc_phy_reset,
@@ -351,7 +343,6 @@ static struct phy_driver smsc_phy_driver[] = {
 	.probe		= smsc_phy_probe,
 
 	/* basic functions */
-	.config_aneg	= genphy_config_aneg,
 	.read_status	= lan87xx_read_status,
 	.config_init	= smsc_phy_config_init,
 	.soft_reset	= smsc_phy_reset,
diff --git a/drivers/net/phy/ste10Xp.c b/drivers/net/phy/ste10Xp.c
index d00cfb64529e..fbd548a1ad84 100644
--- a/drivers/net/phy/ste10Xp.c
+++ b/drivers/net/phy/ste10Xp.c
@@ -89,8 +89,6 @@ static struct phy_driver ste10xp_pdriver[] = {
 	.features = PHY_BASIC_FEATURES | SUPPORTED_Pause,
 	.flags = PHY_HAS_INTERRUPT,
 	.config_init = ste10Xp_config_init,
-	.config_aneg = genphy_config_aneg,
-	.read_status = genphy_read_status,
 	.ack_interrupt = ste10Xp_ack_interrupt,
 	.config_intr = ste10Xp_config_intr,
 	.suspend = genphy_suspend,
@@ -102,8 +100,6 @@ static struct phy_driver ste10xp_pdriver[] = {
 	.features = PHY_BASIC_FEATURES | SUPPORTED_Pause,
 	.flags = PHY_HAS_INTERRUPT,
 	.config_init = ste10Xp_config_init,
-	.config_aneg = genphy_config_aneg,
-	.read_status = genphy_read_status,
 	.ack_interrupt = ste10Xp_ack_interrupt,
 	.config_intr = ste10Xp_config_intr,
 	.suspend = genphy_suspend,
diff --git a/drivers/net/phy/uPD60620.c b/drivers/net/phy/uPD60620.c
index 96b33475ea5e..55f48ee3595a 100644
--- a/drivers/net/phy/uPD60620.c
+++ b/drivers/net/phy/uPD60620.c
@@ -95,7 +95,6 @@ static struct phy_driver upd60620_driver[1] = { {
 	.features       = PHY_BASIC_FEATURES,
 	.flags          = 0,
 	.config_init    = upd60620_config_init,
-	.config_aneg    = genphy_config_aneg,
 	.read_status    = upd60620_read_status,
 } };
 
diff --git a/drivers/net/phy/vitesse.c b/drivers/net/phy/vitesse.c
index f78ff0279648..d9dd8fbfffc7 100644
--- a/drivers/net/phy/vitesse.c
+++ b/drivers/net/phy/vitesse.c
@@ -267,7 +267,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.flags          = PHY_HAS_INTERRUPT,
 	.config_init    = &vsc824x_config_init,
 	.config_aneg    = &vsc82x4_config_aneg,
-	.read_status    = &genphy_read_status,
 	.ack_interrupt  = &vsc824x_ack_interrupt,
 	.config_intr    = &vsc82xx_config_intr,
 }, {
@@ -278,7 +277,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= &vsc824x_config_init,
 	.config_aneg	= &vsc82x4_config_aneg,
-	.read_status	= &genphy_read_status,
 	.ack_interrupt	= &vsc824x_ack_interrupt,
 	.config_intr	= &vsc82xx_config_intr,
 }, {
@@ -289,7 +287,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= &vsc824x_config_init,
 	.config_aneg	= &vsc82x4_config_aneg,
-	.read_status	= &genphy_read_status,
 	.ack_interrupt	= &vsc824x_ack_interrupt,
 	.config_intr	= &vsc82xx_config_intr,
 }, {
@@ -300,7 +297,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.flags          = PHY_HAS_INTERRUPT,
 	.config_init    = &vsc824x_config_init,
 	.config_aneg    = &vsc82x4_config_aneg,
-	.read_status    = &genphy_read_status,
 	.ack_interrupt  = &vsc824x_ack_interrupt,
 	.config_intr    = &vsc82xx_config_intr,
 }, {
@@ -311,7 +307,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.flags          = PHY_HAS_INTERRUPT,
 	.config_init    = &vsc824x_config_init,
 	.config_aneg    = &vsc82x4_config_aneg,
-	.read_status    = &genphy_read_status,
 	.ack_interrupt  = &vsc824x_ack_interrupt,
 	.config_intr    = &vsc82xx_config_intr,
 }, {
@@ -321,8 +316,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.features       = PHY_GBIT_FEATURES,
 	.flags          = PHY_HAS_INTERRUPT,
 	.config_init    = &vsc8601_config_init,
-	.config_aneg    = &genphy_config_aneg,
-	.read_status    = &genphy_read_status,
 	.ack_interrupt  = &vsc824x_ack_interrupt,
 	.config_intr    = &vsc82xx_config_intr,
 }, {
@@ -333,7 +326,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.flags          = PHY_HAS_INTERRUPT,
 	.config_init    = &vsc824x_config_init,
 	.config_aneg    = &vsc82x4_config_aneg,
-	.read_status    = &genphy_read_status,
 	.ack_interrupt  = &vsc824x_ack_interrupt,
 	.config_intr    = &vsc82xx_config_intr,
 }, {
@@ -344,8 +336,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= &vsc8221_config_init,
-	.config_aneg	= &genphy_config_aneg,
-	.read_status	= &genphy_read_status,
 	.ack_interrupt	= &vsc824x_ack_interrupt,
 	.config_intr	= &vsc82xx_config_intr,
 }, {
@@ -356,8 +346,6 @@ static struct phy_driver vsc82xx_driver[] = {
 	.features	= PHY_GBIT_FEATURES,
 	.flags		= PHY_HAS_INTERRUPT,
 	.config_init	= &vsc8221_config_init,
-	.config_aneg	= &genphy_config_aneg,
-	.read_status	= &genphy_read_status,
 	.ack_interrupt	= &vsc824x_ack_interrupt,
 	.config_intr	= &vsc82xx_config_intr,
 } };
diff --git a/drivers/net/ppp/ppp_async.c b/drivers/net/ppp/ppp_async.c
index 1b28e6e702f5..bdc4d23627c5 100644
--- a/drivers/net/ppp/ppp_async.c
+++ b/drivers/net/ppp/ppp_async.c
@@ -334,7 +334,7 @@ ppp_asynctty_ioctl(struct tty_struct *tty, struct file *file,
 }
 
 /* No kernel lock - fine */
-static unsigned int
+static __poll_t
 ppp_asynctty_poll(struct tty_struct *tty, struct file *file, poll_table *wait)
 {
 	return 0;
diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
index 264d4af0bf69..da1937832c99 100644
--- a/drivers/net/ppp/ppp_generic.c
+++ b/drivers/net/ppp/ppp_generic.c
@@ -257,7 +257,7 @@ struct ppp_net {
 /* Prototypes. */
 static int ppp_unattached_ioctl(struct net *net, struct ppp_file *pf,
 			struct file *file, unsigned int cmd, unsigned long arg);
-static void ppp_xmit_process(struct ppp *ppp);
+static void ppp_xmit_process(struct ppp *ppp, struct sk_buff *skb);
 static void ppp_send_frame(struct ppp *ppp, struct sk_buff *skb);
 static void ppp_push(struct ppp *ppp);
 static void ppp_channel_push(struct channel *pch);
@@ -513,13 +513,12 @@ static ssize_t ppp_write(struct file *file, const char __user *buf,
 		goto out;
 	}
 
-	skb_queue_tail(&pf->xq, skb);
-
 	switch (pf->kind) {
 	case INTERFACE:
-		ppp_xmit_process(PF_TO_PPP(pf));
+		ppp_xmit_process(PF_TO_PPP(pf), skb);
 		break;
 	case CHANNEL:
+		skb_queue_tail(&pf->xq, skb);
 		ppp_channel_push(PF_TO_CHANNEL(pf));
 		break;
 	}
@@ -531,19 +530,19 @@ static ssize_t ppp_write(struct file *file, const char __user *buf,
 }
 
 /* No kernel lock - fine */
-static unsigned int ppp_poll(struct file *file, poll_table *wait)
+static __poll_t ppp_poll(struct file *file, poll_table *wait)
 {
 	struct ppp_file *pf = file->private_data;
-	unsigned int mask;
+	__poll_t mask;
 
 	if (!pf)
 		return 0;
 	poll_wait(file, &pf->rwait, wait);
-	mask = POLLOUT | POLLWRNORM;
+	mask = EPOLLOUT | EPOLLWRNORM;
 	if (skb_peek(&pf->rq))
-		mask |= POLLIN | POLLRDNORM;
+		mask |= EPOLLIN | EPOLLRDNORM;
 	if (pf->dead)
-		mask |= POLLHUP;
+		mask |= EPOLLHUP;
 	else if (pf->kind == INTERFACE) {
 		/* see comment in ppp_read */
 		struct ppp *ppp = PF_TO_PPP(pf);
@@ -551,7 +550,7 @@ static unsigned int ppp_poll(struct file *file, poll_table *wait)
 		ppp_recv_lock(ppp);
 		if (ppp->n_channels == 0 &&
 		    (ppp->flags & SC_LOOP_TRAFFIC) == 0)
-			mask |= POLLIN | POLLRDNORM;
+			mask |= EPOLLIN | EPOLLRDNORM;
 		ppp_recv_unlock(ppp);
 	}
 
@@ -1267,8 +1266,8 @@ ppp_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	put_unaligned_be16(proto, pp);
 
 	skb_scrub_packet(skb, !net_eq(ppp->ppp_net, dev_net(dev)));
-	skb_queue_tail(&ppp->file.xq, skb);
-	ppp_xmit_process(ppp);
+	ppp_xmit_process(ppp, skb);
+
 	return NETDEV_TX_OK;
 
  outf:
@@ -1420,13 +1419,14 @@ static void ppp_setup(struct net_device *dev)
  */
 
 /* Called to do any work queued up on the transmit side that can now be done */
-static void __ppp_xmit_process(struct ppp *ppp)
+static void __ppp_xmit_process(struct ppp *ppp, struct sk_buff *skb)
 {
-	struct sk_buff *skb;
-
 	ppp_xmit_lock(ppp);
 	if (!ppp->closing) {
 		ppp_push(ppp);
+
+		if (skb)
+			skb_queue_tail(&ppp->file.xq, skb);
 		while (!ppp->xmit_pending &&
 		       (skb = skb_dequeue(&ppp->file.xq)))
 			ppp_send_frame(ppp, skb);
@@ -1440,7 +1440,7 @@ static void __ppp_xmit_process(struct ppp *ppp)
 	ppp_xmit_unlock(ppp);
 }
 
-static void ppp_xmit_process(struct ppp *ppp)
+static void ppp_xmit_process(struct ppp *ppp, struct sk_buff *skb)
 {
 	local_bh_disable();
 
@@ -1448,7 +1448,7 @@ static void ppp_xmit_process(struct ppp *ppp)
 		goto err;
 
 	(*this_cpu_ptr(ppp->xmit_recursion))++;
-	__ppp_xmit_process(ppp);
+	__ppp_xmit_process(ppp, skb);
 	(*this_cpu_ptr(ppp->xmit_recursion))--;
 
 	local_bh_enable();
@@ -1458,6 +1458,8 @@ static void ppp_xmit_process(struct ppp *ppp)
 err:
 	local_bh_enable();
 
+	kfree_skb(skb);
+
 	if (net_ratelimit())
 		netdev_err(ppp->dev, "recursion detected\n");
 }
@@ -1942,7 +1944,7 @@ static void __ppp_channel_push(struct channel *pch)
 	if (skb_queue_empty(&pch->file.xq)) {
 		ppp = pch->ppp;
 		if (ppp)
-			__ppp_xmit_process(ppp);
+			__ppp_xmit_process(ppp, NULL);
 	}
 }
 
@@ -3161,6 +3163,15 @@ ppp_connect_channel(struct channel *pch, int unit)
 		goto outl;
 
 	ppp_lock(ppp);
+	spin_lock_bh(&pch->downl);
+	if (!pch->chan) {
+		/* Don't connect unregistered channels */
+		spin_unlock_bh(&pch->downl);
+		ppp_unlock(ppp);
+		ret = -ENOTCONN;
+		goto outl;
+	}
+	spin_unlock_bh(&pch->downl);
 	if (pch->file.hdrlen > ppp->file.hdrlen)
 		ppp->file.hdrlen = pch->file.hdrlen;
 	hdrlen = pch->file.hdrlen + 2;	/* for protocol bytes */
diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c
index 7196f00f0991..047f6c68a441 100644
--- a/drivers/net/ppp/ppp_synctty.c
+++ b/drivers/net/ppp/ppp_synctty.c
@@ -327,7 +327,7 @@ ppp_synctty_ioctl(struct tty_struct *tty, struct file *file,
 }
 
 /* No kernel lock - fine */
-static unsigned int
+static __poll_t
 ppp_sync_poll(struct tty_struct *tty, struct file *file, poll_table *wait)
 {
 	return 0;
diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c
index cc63102ca96e..8940417c30e5 100644
--- a/drivers/net/slip/slip.c
+++ b/drivers/net/slip/slip.c
@@ -731,7 +731,7 @@ static void sl_sync(void)
 
 
 /* Find a free SLIP channel, and link in this `tty' line. */
-static struct slip *sl_alloc(dev_t line)
+static struct slip *sl_alloc(void)
 {
 	int i;
 	char name[IFNAMSIZ];
@@ -809,7 +809,7 @@ static int slip_open(struct tty_struct *tty)
 
 	/* OK.  Find a free SLIP channel to use. */
 	err = -ENFILE;
-	sl = sl_alloc(tty_devnum(tty));
+	sl = sl_alloc();
 	if (sl == NULL)
 		goto err_exit;
 
diff --git a/drivers/net/tap.c b/drivers/net/tap.c
index 0a886fda0129..9b6cb780affe 100644
--- a/drivers/net/tap.c
+++ b/drivers/net/tap.c
@@ -330,9 +330,6 @@ rx_handler_result_t tap_handle_frame(struct sk_buff **pskb)
 	if (!q)
 		return RX_HANDLER_PASS;
 
-	if (__skb_array_full(&q->skb_array))
-		goto drop;
-
 	skb_push(skb, ETH_HLEN);
 
 	/* Apply the forward feature mask so that we perform segmentation
@@ -348,7 +345,7 @@ rx_handler_result_t tap_handle_frame(struct sk_buff **pskb)
 			goto drop;
 
 		if (!segs) {
-			if (skb_array_produce(&q->skb_array, skb))
+			if (ptr_ring_produce(&q->ring, skb))
 				goto drop;
 			goto wake_up;
 		}
@@ -358,7 +355,7 @@ rx_handler_result_t tap_handle_frame(struct sk_buff **pskb)
 			struct sk_buff *nskb = segs->next;
 
 			segs->next = NULL;
-			if (skb_array_produce(&q->skb_array, segs)) {
+			if (ptr_ring_produce(&q->ring, segs)) {
 				kfree_skb(segs);
 				kfree_skb_list(nskb);
 				break;
@@ -375,12 +372,12 @@ rx_handler_result_t tap_handle_frame(struct sk_buff **pskb)
 		    !(features & NETIF_F_CSUM_MASK) &&
 		    skb_checksum_help(skb))
 			goto drop;
-		if (skb_array_produce(&q->skb_array, skb))
+		if (ptr_ring_produce(&q->ring, skb))
 			goto drop;
 	}
 
 wake_up:
-	wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND);
+	wake_up_interruptible_poll(sk_sleep(&q->sk), EPOLLIN | EPOLLRDNORM | EPOLLRDBAND);
 	return RX_HANDLER_CONSUMED;
 
 drop:
@@ -490,14 +487,14 @@ static void tap_sock_write_space(struct sock *sk)
 
 	wqueue = sk_sleep(sk);
 	if (wqueue && waitqueue_active(wqueue))
-		wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND);
+		wake_up_interruptible_poll(wqueue, EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
 }
 
 static void tap_sock_destruct(struct sock *sk)
 {
 	struct tap_queue *q = container_of(sk, struct tap_queue, sk);
 
-	skb_array_cleanup(&q->skb_array);
+	ptr_ring_cleanup(&q->ring, __skb_array_destroy_skb);
 }
 
 static int tap_open(struct inode *inode, struct file *file)
@@ -517,7 +514,7 @@ static int tap_open(struct inode *inode, struct file *file)
 					     &tap_proto, 0);
 	if (!q)
 		goto err;
-	if (skb_array_init(&q->skb_array, tap->dev->tx_queue_len, GFP_KERNEL)) {
+	if (ptr_ring_init(&q->ring, tap->dev->tx_queue_len, GFP_KERNEL)) {
 		sk_free(&q->sk);
 		goto err;
 	}
@@ -546,7 +543,7 @@ static int tap_open(struct inode *inode, struct file *file)
 
 	err = tap_set_queue(tap, file, q);
 	if (err) {
-		/* tap_sock_destruct() will take care of freeing skb_array */
+		/* tap_sock_destruct() will take care of freeing ptr_ring */
 		goto err_put;
 	}
 
@@ -572,10 +569,10 @@ static int tap_release(struct inode *inode, struct file *file)
 	return 0;
 }
 
-static unsigned int tap_poll(struct file *file, poll_table *wait)
+static __poll_t tap_poll(struct file *file, poll_table *wait)
 {
 	struct tap_queue *q = file->private_data;
-	unsigned int mask = POLLERR;
+	__poll_t mask = EPOLLERR;
 
 	if (!q)
 		goto out;
@@ -583,13 +580,13 @@ static unsigned int tap_poll(struct file *file, poll_table *wait)
 	mask = 0;
 	poll_wait(file, &q->wq.wait, wait);
 
-	if (!skb_array_empty(&q->skb_array))
-		mask |= POLLIN | POLLRDNORM;
+	if (!ptr_ring_empty(&q->ring))
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	if (sock_writeable(&q->sk) ||
 	    (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &q->sock.flags) &&
 	     sock_writeable(&q->sk)))
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 
 out:
 	return mask;
@@ -844,7 +841,7 @@ static ssize_t tap_do_read(struct tap_queue *q,
 					TASK_INTERRUPTIBLE);
 
 		/* Read frames from the queue */
-		skb = skb_array_consume(&q->skb_array);
+		skb = ptr_ring_consume(&q->ring);
 		if (skb)
 			break;
 		if (noblock) {
@@ -1176,7 +1173,7 @@ static int tap_peek_len(struct socket *sock)
 {
 	struct tap_queue *q = container_of(sock, struct tap_queue,
 					       sock);
-	return skb_array_peek_len(&q->skb_array);
+	return PTR_RING_PEEK_CALL(&q->ring, __skb_array_len_with_tag);
 }
 
 /* Ops structure to mimic raw sockets with tun */
@@ -1202,7 +1199,7 @@ struct socket *tap_get_socket(struct file *file)
 }
 EXPORT_SYMBOL_GPL(tap_get_socket);
 
-struct skb_array *tap_get_skb_array(struct file *file)
+struct ptr_ring *tap_get_ptr_ring(struct file *file)
 {
 	struct tap_queue *q;
 
@@ -1211,29 +1208,30 @@ struct skb_array *tap_get_skb_array(struct file *file)
 	q = file->private_data;
 	if (!q)
 		return ERR_PTR(-EBADFD);
-	return &q->skb_array;
+	return &q->ring;
 }
-EXPORT_SYMBOL_GPL(tap_get_skb_array);
+EXPORT_SYMBOL_GPL(tap_get_ptr_ring);
 
 int tap_queue_resize(struct tap_dev *tap)
 {
 	struct net_device *dev = tap->dev;
 	struct tap_queue *q;
-	struct skb_array **arrays;
+	struct ptr_ring **rings;
 	int n = tap->numqueues;
 	int ret, i = 0;
 
-	arrays = kmalloc_array(n, sizeof(*arrays), GFP_KERNEL);
-	if (!arrays)
+	rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL);
+	if (!rings)
 		return -ENOMEM;
 
 	list_for_each_entry(q, &tap->queue_list, next)
-		arrays[i++] = &q->skb_array;
+		rings[i++] = &q->ring;
 
-	ret = skb_array_resize_multiple(arrays, n,
-					dev->tx_queue_len, GFP_KERNEL);
+	ret = ptr_ring_resize_multiple(rings, n,
+				       dev->tx_queue_len, GFP_KERNEL,
+				       __skb_array_destroy_skb);
 
-	kfree(arrays);
+	kfree(rings);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(tap_queue_resize);
diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c
index a468439969df..befed2d22bf4 100644
--- a/drivers/net/team/team.c
+++ b/drivers/net/team/team.c
@@ -1197,11 +1197,6 @@ static int team_port_add(struct team *team, struct net_device *port_dev)
 		goto err_dev_open;
 	}
 
-	netif_addr_lock_bh(dev);
-	dev_uc_sync_multiple(port_dev, dev);
-	dev_mc_sync_multiple(port_dev, dev);
-	netif_addr_unlock_bh(dev);
-
 	err = vlan_vids_add_by_dev(port_dev, dev);
 	if (err) {
 		netdev_err(dev, "Failed to add vlan ids to device %s\n",
@@ -1241,6 +1236,11 @@ static int team_port_add(struct team *team, struct net_device *port_dev)
 		goto err_option_port_add;
 	}
 
+	netif_addr_lock_bh(dev);
+	dev_uc_sync_multiple(port_dev, dev);
+	dev_mc_sync_multiple(port_dev, dev);
+	netif_addr_unlock_bh(dev);
+
 	port->index = -1;
 	list_add_tail_rcu(&port->list, &team->port_list);
 	team_port_enable(team, port);
@@ -1265,8 +1265,6 @@ static int team_port_add(struct team *team, struct net_device *port_dev)
 	vlan_vids_del_by_dev(port_dev, dev);
 
 err_vids_add:
-	dev_uc_unsync(port_dev, dev);
-	dev_mc_unsync(port_dev, dev);
 	dev_close(port_dev);
 
 err_dev_open:
@@ -2395,7 +2393,7 @@ static int team_nl_send_options_get(struct team *team, u32 portid, u32 seq,
 	if (!nlh) {
 		err = __send_and_alloc_skb(&skb, team, portid, send_func);
 		if (err)
-			goto errout;
+			return err;
 		goto send_done;
 	}
 
@@ -2681,7 +2679,7 @@ static int team_nl_send_port_list_get(struct team *team, u32 portid, u32 seq,
 	if (!nlh) {
 		err = __send_and_alloc_skb(&skb, team, portid, send_func);
 		if (err)
-			goto errout;
+			return err;
 		goto send_done;
 	}
 
diff --git a/drivers/net/thunderbolt.c b/drivers/net/thunderbolt.c
index ca5e375de27c..e0d6760f3219 100644
--- a/drivers/net/thunderbolt.c
+++ b/drivers/net/thunderbolt.c
@@ -166,6 +166,8 @@ struct tbnet_ring {
  * @connected_work: Worker that finalizes the ThunderboltIP connection
  *		    setup and enables DMA paths for high speed data
  *		    transfers
+ * @disconnect_work: Worker that handles tearing down the ThunderboltIP
+ *		     connection
  * @rx_hdr: Copy of the currently processed Rx frame. Used when a
  *	    network packet consists of multiple Thunderbolt frames.
  *	    In host byte order.
@@ -190,6 +192,7 @@ struct tbnet {
 	int login_retries;
 	struct delayed_work login_work;
 	struct work_struct connected_work;
+	struct work_struct disconnect_work;
 	struct thunderbolt_ip_frame_header rx_hdr;
 	struct tbnet_ring rx_ring;
 	atomic_t frame_id;
@@ -445,7 +448,7 @@ static int tbnet_handle_packet(const void *buf, size_t size, void *data)
 	case TBIP_LOGOUT:
 		ret = tbnet_logout_response(net, route, sequence, command_id);
 		if (!ret)
-			tbnet_tear_down(net, false);
+			queue_work(system_long_wq, &net->disconnect_work);
 		break;
 
 	default:
@@ -659,6 +662,13 @@ static void tbnet_login_work(struct work_struct *work)
 	}
 }
 
+static void tbnet_disconnect_work(struct work_struct *work)
+{
+	struct tbnet *net = container_of(work, typeof(*net), disconnect_work);
+
+	tbnet_tear_down(net, false);
+}
+
 static bool tbnet_check_frame(struct tbnet *net, const struct tbnet_frame *tf,
 			      const struct thunderbolt_ip_frame_header *hdr)
 {
@@ -881,6 +891,7 @@ static int tbnet_stop(struct net_device *dev)
 
 	napi_disable(&net->napi);
 
+	cancel_work_sync(&net->disconnect_work);
 	tbnet_tear_down(net, true);
 
 	tb_ring_free(net->rx_ring.ring);
@@ -1195,6 +1206,7 @@ static int tbnet_probe(struct tb_service *svc, const struct tb_service_id *id)
 	net = netdev_priv(dev);
 	INIT_DELAYED_WORK(&net->login_work, tbnet_login_work);
 	INIT_WORK(&net->connected_work, tbnet_connected_work);
+	INIT_WORK(&net->disconnect_work, tbnet_disconnect_work);
 	mutex_init(&net->connection_lock);
 	atomic_set(&net->command_id, 0);
 	atomic_set(&net->frame_id, 0);
@@ -1270,10 +1282,7 @@ static int __maybe_unused tbnet_suspend(struct device *dev)
 	stop_login(net);
 	if (netif_running(net->dev)) {
 		netif_device_detach(net->dev);
-		tb_ring_stop(net->rx_ring.ring);
-		tb_ring_stop(net->tx_ring.ring);
-		tbnet_free_buffers(&net->rx_ring);
-		tbnet_free_buffers(&net->tx_ring);
+		tbnet_tear_down(net, true);
 	}
 
 	return 0;
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index a8ec589d1359..28cfa642e39a 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -179,7 +179,8 @@ struct tun_file {
 	struct mutex napi_mutex;	/* Protects access to the above napi */
 	struct list_head next;
 	struct tun_struct *detached;
-	struct skb_array tx_array;
+	struct ptr_ring tx_ring;
+	struct xdp_rxq_info xdp_rxq;
 };
 
 struct tun_flow_entry {
@@ -195,6 +196,11 @@ struct tun_flow_entry {
 
 #define TUN_NUM_FLOW_ENTRIES 1024
 
+struct tun_prog {
+	struct rcu_head rcu;
+	struct bpf_prog *prog;
+};
+
 /* Since the socket were moved to tun_file, to preserve the behavior of persist
  * device, socket filter, sndbuf and vnet header size were restore when the
  * file were attached to a persist device.
@@ -232,8 +238,33 @@ struct tun_struct {
 	u32 rx_batched;
 	struct tun_pcpu_stats __percpu *pcpu_stats;
 	struct bpf_prog __rcu *xdp_prog;
+	struct tun_prog __rcu *steering_prog;
+	struct tun_prog __rcu *filter_prog;
 };
 
+struct veth {
+	__be16 h_vlan_proto;
+	__be16 h_vlan_TCI;
+};
+
+bool tun_is_xdp_buff(void *ptr)
+{
+	return (unsigned long)ptr & TUN_XDP_FLAG;
+}
+EXPORT_SYMBOL(tun_is_xdp_buff);
+
+void *tun_xdp_to_ptr(void *ptr)
+{
+	return (void *)((unsigned long)ptr | TUN_XDP_FLAG);
+}
+EXPORT_SYMBOL(tun_xdp_to_ptr);
+
+void *tun_ptr_to_xdp(void *ptr)
+{
+	return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);
+}
+EXPORT_SYMBOL(tun_ptr_to_xdp);
+
 static int tun_napi_receive(struct napi_struct *napi, int budget)
 {
 	struct tun_file *tfile = container_of(napi, struct tun_file, napi);
@@ -537,15 +568,12 @@ static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
  * different rxq no. here. If we could not get rxhash, then we would
  * hope the rxq no. may help here.
  */
-static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
-			    void *accel_priv, select_queue_fallback_t fallback)
+static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
 {
-	struct tun_struct *tun = netdev_priv(dev);
 	struct tun_flow_entry *e;
 	u32 txq = 0;
 	u32 numqueues = 0;
 
-	rcu_read_lock();
 	numqueues = READ_ONCE(tun->numqueues);
 
 	txq = __skb_get_hash_symmetric(skb);
@@ -563,10 +591,37 @@ static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
 			txq -= numqueues;
 	}
 
-	rcu_read_unlock();
 	return txq;
 }
 
+static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
+{
+	struct tun_prog *prog;
+	u16 ret = 0;
+
+	prog = rcu_dereference(tun->steering_prog);
+	if (prog)
+		ret = bpf_prog_run_clear_cb(prog->prog, skb);
+
+	return ret % tun->numqueues;
+}
+
+static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
+			    void *accel_priv, select_queue_fallback_t fallback)
+{
+	struct tun_struct *tun = netdev_priv(dev);
+	u16 ret;
+
+	rcu_read_lock();
+	if (rcu_dereference(tun->steering_prog))
+		ret = tun_ebpf_select_queue(tun, skb);
+	else
+		ret = tun_automq_select_queue(tun, skb);
+	rcu_read_unlock();
+
+	return ret;
+}
+
 static inline bool tun_not_capable(struct tun_struct *tun)
 {
 	const struct cred *cred = current_cred();
@@ -600,22 +655,37 @@ static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
 	return tun;
 }
 
+void tun_ptr_free(void *ptr)
+{
+	if (!ptr)
+		return;
+	if (tun_is_xdp_buff(ptr)) {
+		struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
+
+		put_page(virt_to_head_page(xdp->data));
+	} else {
+		__skb_array_destroy_skb(ptr);
+	}
+}
+EXPORT_SYMBOL_GPL(tun_ptr_free);
+
 static void tun_queue_purge(struct tun_file *tfile)
 {
-	struct sk_buff *skb;
+	void *ptr;
 
-	while ((skb = skb_array_consume(&tfile->tx_array)) != NULL)
-		kfree_skb(skb);
+	while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
+		tun_ptr_free(ptr);
 
 	skb_queue_purge(&tfile->sk.sk_write_queue);
 	skb_queue_purge(&tfile->sk.sk_error_queue);
 }
 
-static void tun_cleanup_tx_array(struct tun_file *tfile)
+static void tun_cleanup_tx_ring(struct tun_file *tfile)
 {
-	if (tfile->tx_array.ring.queue) {
-		skb_array_cleanup(&tfile->tx_array);
-		memset(&tfile->tx_array, 0, sizeof(tfile->tx_array));
+	if (tfile->tx_ring.queue) {
+		ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
+		xdp_rxq_info_unreg(&tfile->xdp_rxq);
+		memset(&tfile->tx_ring, 0, sizeof(tfile->tx_ring));
 	}
 }
 
@@ -665,7 +735,7 @@ static void __tun_detach(struct tun_file *tfile, bool clean)
 			    tun->dev->reg_state == NETREG_REGISTERED)
 				unregister_netdevice(tun->dev);
 		}
-		tun_cleanup_tx_array(tfile);
+		tun_cleanup_tx_ring(tfile);
 		sock_put(&tfile->sk);
 	}
 }
@@ -680,7 +750,6 @@ static void tun_detach(struct tun_file *tfile, bool clean)
 static void tun_detach_all(struct net_device *dev)
 {
 	struct tun_struct *tun = netdev_priv(dev);
-	struct bpf_prog *xdp_prog = rtnl_dereference(tun->xdp_prog);
 	struct tun_file *tfile, *tmp;
 	int i, n = tun->numqueues;
 
@@ -707,19 +776,16 @@ static void tun_detach_all(struct net_device *dev)
 		/* Drop read queue */
 		tun_queue_purge(tfile);
 		sock_put(&tfile->sk);
-		tun_cleanup_tx_array(tfile);
+		tun_cleanup_tx_ring(tfile);
 	}
 	list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
 		tun_enable_queue(tfile);
 		tun_queue_purge(tfile);
 		sock_put(&tfile->sk);
-		tun_cleanup_tx_array(tfile);
+		tun_cleanup_tx_ring(tfile);
 	}
 	BUG_ON(tun->numdisabled != 0);
 
-	if (xdp_prog)
-		bpf_prog_put(xdp_prog);
-
 	if (tun->flags & IFF_PERSIST)
 		module_put(THIS_MODULE);
 }
@@ -760,13 +826,29 @@ static int tun_attach(struct tun_struct *tun, struct file *file,
 	}
 
 	if (!tfile->detached &&
-	    skb_array_init(&tfile->tx_array, dev->tx_queue_len, GFP_KERNEL)) {
+	    ptr_ring_init(&tfile->tx_ring, dev->tx_queue_len, GFP_KERNEL)) {
 		err = -ENOMEM;
 		goto out;
 	}
 
 	tfile->queue_index = tun->numqueues;
 	tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
+
+	if (tfile->detached) {
+		/* Re-attach detached tfile, updating XDP queue_index */
+		WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
+
+		if (tfile->xdp_rxq.queue_index    != tfile->queue_index)
+			tfile->xdp_rxq.queue_index = tfile->queue_index;
+	} else {
+		/* Setup XDP RX-queue info, for new tfile getting attached */
+		err = xdp_rxq_info_reg(&tfile->xdp_rxq,
+				       tun->dev, tfile->queue_index);
+		if (err < 0)
+			goto out;
+		err = 0;
+	}
+
 	rcu_assign_pointer(tfile->tun, tun);
 	rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
 	tun->numqueues++;
@@ -946,23 +1028,10 @@ static int tun_net_close(struct net_device *dev)
 }
 
 /* Net device start xmit */
-static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
+static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
 {
-	struct tun_struct *tun = netdev_priv(dev);
-	int txq = skb->queue_mapping;
-	struct tun_file *tfile;
-	u32 numqueues = 0;
-
-	rcu_read_lock();
-	tfile = rcu_dereference(tun->tfiles[txq]);
-	numqueues = READ_ONCE(tun->numqueues);
-
-	/* Drop packet if interface is not attached */
-	if (txq >= numqueues)
-		goto drop;
-
 #ifdef CONFIG_RPS
-	if (numqueues == 1 && static_key_false(&rps_needed)) {
+	if (tun->numqueues == 1 && static_key_false(&rps_needed)) {
 		/* Select queue was not called for the skbuff, so we extract the
 		 * RPS hash and save it into the flow_table here.
 		 */
@@ -978,6 +1047,37 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 		}
 	}
 #endif
+}
+
+static unsigned int run_ebpf_filter(struct tun_struct *tun,
+				    struct sk_buff *skb,
+				    int len)
+{
+	struct tun_prog *prog = rcu_dereference(tun->filter_prog);
+
+	if (prog)
+		len = bpf_prog_run_clear_cb(prog->prog, skb);
+
+	return len;
+}
+
+/* Net device start xmit */
+static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct tun_struct *tun = netdev_priv(dev);
+	int txq = skb->queue_mapping;
+	struct tun_file *tfile;
+	int len = skb->len;
+
+	rcu_read_lock();
+	tfile = rcu_dereference(tun->tfiles[txq]);
+
+	/* Drop packet if interface is not attached */
+	if (txq >= tun->numqueues)
+		goto drop;
+
+	if (!rcu_dereference(tun->steering_prog))
+		tun_automq_xmit(tun, skb);
 
 	tun_debug(KERN_INFO, tun, "tun_net_xmit %d\n", skb->len);
 
@@ -993,6 +1093,15 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	    sk_filter(tfile->socket.sk, skb))
 		goto drop;
 
+	len = run_ebpf_filter(tun, skb, len);
+
+	/* Trim extra bytes since we may insert vlan proto & TCI
+	 * in tun_put_user().
+	 */
+	len -= skb_vlan_tag_present(skb) ? sizeof(struct veth) : 0;
+	if (len <= 0 || pskb_trim(skb, len))
+		goto drop;
+
 	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
 		goto drop;
 
@@ -1005,7 +1114,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	nf_reset(skb);
 
-	if (skb_array_produce(&tfile->tx_array, skb))
+	if (ptr_ring_produce(&tfile->tx_ring, skb))
 		goto drop;
 
 	/* Notify and wake up reader process */
@@ -1178,6 +1287,67 @@ static const struct net_device_ops tun_netdev_ops = {
 	.ndo_get_stats64	= tun_net_get_stats64,
 };
 
+static int tun_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
+{
+	struct tun_struct *tun = netdev_priv(dev);
+	struct xdp_buff *buff = xdp->data_hard_start;
+	int headroom = xdp->data - xdp->data_hard_start;
+	struct tun_file *tfile;
+	u32 numqueues;
+	int ret = 0;
+
+	/* Assure headroom is available and buff is properly aligned */
+	if (unlikely(headroom < sizeof(*xdp) || tun_is_xdp_buff(xdp)))
+		return -ENOSPC;
+
+	*buff = *xdp;
+
+	rcu_read_lock();
+
+	numqueues = READ_ONCE(tun->numqueues);
+	if (!numqueues) {
+		ret = -ENOSPC;
+		goto out;
+	}
+
+	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
+					    numqueues]);
+	/* Encode the XDP flag into lowest bit for consumer to differ
+	 * XDP buffer from sk_buff.
+	 */
+	if (ptr_ring_produce(&tfile->tx_ring, tun_xdp_to_ptr(buff))) {
+		this_cpu_inc(tun->pcpu_stats->tx_dropped);
+		ret = -ENOSPC;
+	}
+
+out:
+	rcu_read_unlock();
+	return ret;
+}
+
+static void tun_xdp_flush(struct net_device *dev)
+{
+	struct tun_struct *tun = netdev_priv(dev);
+	struct tun_file *tfile;
+	u32 numqueues;
+
+	rcu_read_lock();
+
+	numqueues = READ_ONCE(tun->numqueues);
+	if (!numqueues)
+		goto out;
+
+	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
+					    numqueues]);
+	/* Notify and wake up reader process */
+	if (tfile->flags & TUN_FASYNC)
+		kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
+	tfile->socket.sk->sk_data_ready(tfile->socket.sk);
+
+out:
+	rcu_read_unlock();
+}
+
 static const struct net_device_ops tap_netdev_ops = {
 	.ndo_uninit		= tun_net_uninit,
 	.ndo_open		= tun_net_open,
@@ -1195,6 +1365,8 @@ static const struct net_device_ops tap_netdev_ops = {
 	.ndo_set_rx_headroom	= tun_set_headroom,
 	.ndo_get_stats64	= tun_net_get_stats64,
 	.ndo_bpf		= tun_xdp,
+	.ndo_xdp_xmit		= tun_xdp_xmit,
+	.ndo_xdp_flush		= tun_xdp_flush,
 };
 
 static void tun_flow_init(struct tun_struct *tun)
@@ -1257,15 +1429,15 @@ static void tun_net_init(struct net_device *dev)
 /* Character device part */
 
 /* Poll */
-static unsigned int tun_chr_poll(struct file *file, poll_table *wait)
+static __poll_t tun_chr_poll(struct file *file, poll_table *wait)
 {
 	struct tun_file *tfile = file->private_data;
 	struct tun_struct *tun = tun_get(tfile);
 	struct sock *sk;
-	unsigned int mask = 0;
+	__poll_t mask = 0;
 
 	if (!tun)
-		return POLLERR;
+		return EPOLLERR;
 
 	sk = tfile->socket.sk;
 
@@ -1273,17 +1445,17 @@ static unsigned int tun_chr_poll(struct file *file, poll_table *wait)
 
 	poll_wait(file, sk_sleep(sk), wait);
 
-	if (!skb_array_empty(&tfile->tx_array))
-		mask |= POLLIN | POLLRDNORM;
+	if (!ptr_ring_empty(&tfile->tx_ring))
+		mask |= EPOLLIN | EPOLLRDNORM;
 
 	if (tun->dev->flags & IFF_UP &&
 	    (sock_writeable(sk) ||
 	     (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) &&
 	      sock_writeable(sk))))
-		mask |= POLLOUT | POLLWRNORM;
+		mask |= EPOLLOUT | EPOLLWRNORM;
 
 	if (tun->dev->reg_state != NETREG_REGISTERED)
-		mask = POLLERR;
+		mask = EPOLLERR;
 
 	tun_put(tun);
 	return mask;
@@ -1317,27 +1489,23 @@ static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
 	skb->truesize += skb->data_len;
 
 	for (i = 1; i < it->nr_segs; i++) {
+		struct page_frag *pfrag = &current->task_frag;
 		size_t fragsz = it->iov[i].iov_len;
-		unsigned long offset;
-		struct page *page;
-		void *data;
 
 		if (fragsz == 0 || fragsz > PAGE_SIZE) {
 			err = -EINVAL;
 			goto free;
 		}
 
-		local_bh_disable();
-		data = napi_alloc_frag(fragsz);
-		local_bh_enable();
-		if (!data) {
+		if (!skb_page_frag_refill(fragsz, pfrag, GFP_KERNEL)) {
 			err = -ENOMEM;
 			goto free;
 		}
 
-		page = virt_to_head_page(data);
-		offset = data - page_address(page);
-		skb_fill_page_desc(skb, i - 1, page, offset, fragsz);
+		skb_fill_page_desc(skb, i - 1, pfrag->page,
+				   pfrag->offset, fragsz);
+		page_ref_inc(pfrag->page);
+		pfrag->offset += fragsz;
 	}
 
 	return skb;
@@ -1475,6 +1643,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 	else
 		*skb_xdp = 0;
 
+	preempt_disable();
 	rcu_read_lock();
 	xdp_prog = rcu_dereference(tun->xdp_prog);
 	if (xdp_prog && !*skb_xdp) {
@@ -1486,6 +1655,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 		xdp.data = buf + pad;
 		xdp_set_data_meta_invalid(&xdp);
 		xdp.data_end = xdp.data + len;
+		xdp.rxq = &tfile->xdp_rxq;
 		orig_data = xdp.data;
 		act = bpf_prog_run_xdp(xdp_prog, &xdp);
 
@@ -1494,9 +1664,11 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 			get_page(alloc_frag->page);
 			alloc_frag->offset += buflen;
 			err = xdp_do_redirect(tun->dev, &xdp, xdp_prog);
+			xdp_do_flush_map();
 			if (err)
 				goto err_redirect;
 			rcu_read_unlock();
+			preempt_enable();
 			return NULL;
 		case XDP_TX:
 			xdp_xmit = true;
@@ -1518,6 +1690,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 	skb = build_skb(buf, buflen);
 	if (!skb) {
 		rcu_read_unlock();
+		preempt_enable();
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -1530,10 +1703,12 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 		skb->dev = tun->dev;
 		generic_xdp_tx(skb, xdp_prog);
 		rcu_read_unlock();
+		preempt_enable();
 		return NULL;
 	}
 
 	rcu_read_unlock();
+	preempt_enable();
 
 	return skb;
 
@@ -1541,6 +1716,7 @@ static struct sk_buff *tun_build_skb(struct tun_struct *tun,
 	put_page(alloc_frag->page);
 err_xdp:
 	rcu_read_unlock();
+	preempt_enable();
 	this_cpu_inc(tun->pcpu_stats->rx_dropped);
 	return NULL;
 }
@@ -1560,7 +1736,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 	int copylen;
 	bool zerocopy = false;
 	int err;
-	u32 rxhash;
+	u32 rxhash = 0;
 	int skb_xdp = 1;
 	bool frags = tun_napi_frags_enabled(tun);
 
@@ -1748,7 +1924,10 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 		rcu_read_unlock();
 	}
 
-	rxhash = __skb_get_hash_symmetric(skb);
+	rcu_read_lock();
+	if (!rcu_dereference(tun->steering_prog))
+		rxhash = __skb_get_hash_symmetric(skb);
+	rcu_read_unlock();
 
 	if (frags) {
 		/* Exercise flow dissector code path. */
@@ -1792,7 +1971,9 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
 	u64_stats_update_end(&stats->syncp);
 	put_cpu_ptr(stats);
 
-	tun_flow_update(tun, rxhash, tfile);
+	if (rxhash)
+		tun_flow_update(tun, rxhash, tfile);
+
 	return total_len;
 }
 
@@ -1813,6 +1994,40 @@ static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	return result;
 }
 
+static ssize_t tun_put_user_xdp(struct tun_struct *tun,
+				struct tun_file *tfile,
+				struct xdp_buff *xdp,
+				struct iov_iter *iter)
+{
+	int vnet_hdr_sz = 0;
+	size_t size = xdp->data_end - xdp->data;
+	struct tun_pcpu_stats *stats;
+	size_t ret;
+
+	if (tun->flags & IFF_VNET_HDR) {
+		struct virtio_net_hdr gso = { 0 };
+
+		vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
+		if (unlikely(iov_iter_count(iter) < vnet_hdr_sz))
+			return -EINVAL;
+		if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) !=
+			     sizeof(gso)))
+			return -EFAULT;
+		iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
+	}
+
+	ret = copy_to_iter(xdp->data, size, iter) + vnet_hdr_sz;
+
+	stats = get_cpu_ptr(tun->pcpu_stats);
+	u64_stats_update_begin(&stats->syncp);
+	stats->tx_packets++;
+	stats->tx_bytes += ret;
+	u64_stats_update_end(&stats->syncp);
+	put_cpu_ptr(tun->pcpu_stats);
+
+	return ret;
+}
+
 /* Put packet to the user space buffer */
 static ssize_t tun_put_user(struct tun_struct *tun,
 			    struct tun_file *tfile,
@@ -1877,10 +2092,7 @@ static ssize_t tun_put_user(struct tun_struct *tun,
 
 	if (vlan_hlen) {
 		int ret;
-		struct {
-			__be16 h_vlan_proto;
-			__be16 h_vlan_TCI;
-		} veth;
+		struct veth veth;
 
 		veth.h_vlan_proto = skb->vlan_proto;
 		veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
@@ -1910,15 +2122,14 @@ static ssize_t tun_put_user(struct tun_struct *tun,
 	return total;
 }
 
-static struct sk_buff *tun_ring_recv(struct tun_file *tfile, int noblock,
-				     int *err)
+static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
 {
 	DECLARE_WAITQUEUE(wait, current);
-	struct sk_buff *skb = NULL;
+	void *ptr = NULL;
 	int error = 0;
 
-	skb = skb_array_consume(&tfile->tx_array);
-	if (skb)
+	ptr = ptr_ring_consume(&tfile->tx_ring);
+	if (ptr)
 		goto out;
 	if (noblock) {
 		error = -EAGAIN;
@@ -1929,8 +2140,8 @@ static struct sk_buff *tun_ring_recv(struct tun_file *tfile, int noblock,
 	current->state = TASK_INTERRUPTIBLE;
 
 	while (1) {
-		skb = skb_array_consume(&tfile->tx_array);
-		if (skb)
+		ptr = ptr_ring_consume(&tfile->tx_ring);
+		if (ptr)
 			break;
 		if (signal_pending(current)) {
 			error = -ERESTARTSYS;
@@ -1949,12 +2160,12 @@ static struct sk_buff *tun_ring_recv(struct tun_file *tfile, int noblock,
 
 out:
 	*err = error;
-	return skb;
+	return ptr;
 }
 
 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
 			   struct iov_iter *to,
-			   int noblock, struct sk_buff *skb)
+			   int noblock, void *ptr)
 {
 	ssize_t ret;
 	int err;
@@ -1962,23 +2173,31 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
 	tun_debug(KERN_INFO, tun, "tun_do_read\n");
 
 	if (!iov_iter_count(to)) {
-		if (skb)
-			kfree_skb(skb);
+		tun_ptr_free(ptr);
 		return 0;
 	}
 
-	if (!skb) {
+	if (!ptr) {
 		/* Read frames from ring */
-		skb = tun_ring_recv(tfile, noblock, &err);
-		if (!skb)
+		ptr = tun_ring_recv(tfile, noblock, &err);
+		if (!ptr)
 			return err;
 	}
 
-	ret = tun_put_user(tun, tfile, skb, to);
-	if (unlikely(ret < 0))
-		kfree_skb(skb);
-	else
-		consume_skb(skb);
+	if (tun_is_xdp_buff(ptr)) {
+		struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
+
+		ret = tun_put_user_xdp(tun, tfile, xdp, to);
+		put_page(virt_to_head_page(xdp->data));
+	} else {
+		struct sk_buff *skb = ptr;
+
+		ret = tun_put_user(tun, tfile, skb, to);
+		if (unlikely(ret < 0))
+			kfree_skb(skb);
+		else
+			consume_skb(skb);
+	}
 
 	return ret;
 }
@@ -2000,6 +2219,39 @@ static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	return ret;
 }
 
+static void tun_prog_free(struct rcu_head *rcu)
+{
+	struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu);
+
+	bpf_prog_destroy(prog->prog);
+	kfree(prog);
+}
+
+static int __tun_set_ebpf(struct tun_struct *tun,
+			  struct tun_prog __rcu **prog_p,
+			  struct bpf_prog *prog)
+{
+	struct tun_prog *old, *new = NULL;
+
+	if (prog) {
+		new = kmalloc(sizeof(*new), GFP_KERNEL);
+		if (!new)
+			return -ENOMEM;
+		new->prog = prog;
+	}
+
+	spin_lock_bh(&tun->lock);
+	old = rcu_dereference_protected(*prog_p,
+					lockdep_is_held(&tun->lock));
+	rcu_assign_pointer(*prog_p, new);
+	spin_unlock_bh(&tun->lock);
+
+	if (old)
+		call_rcu(&old->rcu, tun_prog_free);
+
+	return 0;
+}
+
 static void tun_free_netdev(struct net_device *dev)
 {
 	struct tun_struct *tun = netdev_priv(dev);
@@ -2008,6 +2260,8 @@ static void tun_free_netdev(struct net_device *dev)
 	free_percpu(tun->pcpu_stats);
 	tun_flow_uninit(tun);
 	security_tun_dev_free_security(tun->security);
+	__tun_set_ebpf(tun, &tun->steering_prog, NULL);
+	__tun_set_ebpf(tun, &tun->filter_prog, NULL);
 }
 
 static void tun_setup(struct net_device *dev)
@@ -2053,8 +2307,8 @@ static void tun_sock_write_space(struct sock *sk)
 
 	wqueue = sk_sleep(sk);
 	if (wqueue && waitqueue_active(wqueue))
-		wake_up_interruptible_sync_poll(wqueue, POLLOUT |
-						POLLWRNORM | POLLWRBAND);
+		wake_up_interruptible_sync_poll(wqueue, EPOLLOUT |
+						EPOLLWRNORM | EPOLLWRBAND);
 
 	tfile = container_of(sk, struct tun_file, sk);
 	kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
@@ -2081,12 +2335,12 @@ static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
 {
 	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
 	struct tun_struct *tun = tun_get(tfile);
-	struct sk_buff *skb = m->msg_control;
+	void *ptr = m->msg_control;
 	int ret;
 
 	if (!tun) {
 		ret = -EBADFD;
-		goto out_free_skb;
+		goto out_free;
 	}
 
 	if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) {
@@ -2098,7 +2352,7 @@ static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
 					 SOL_PACKET, TUN_TX_TIMESTAMP);
 		goto out;
 	}
-	ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, skb);
+	ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
 	if (ret > (ssize_t)total_len) {
 		m->msg_flags |= MSG_TRUNC;
 		ret = flags & MSG_TRUNC ? ret : total_len;
@@ -2109,12 +2363,25 @@ static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
 
 out_put_tun:
 	tun_put(tun);
-out_free_skb:
-	if (skb)
-		kfree_skb(skb);
+out_free:
+	tun_ptr_free(ptr);
 	return ret;
 }
 
+static int tun_ptr_peek_len(void *ptr)
+{
+	if (likely(ptr)) {
+		if (tun_is_xdp_buff(ptr)) {
+			struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
+
+			return xdp->data_end - xdp->data;
+		}
+		return __skb_array_len_with_tag(ptr);
+	} else {
+		return 0;
+	}
+}
+
 static int tun_peek_len(struct socket *sock)
 {
 	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
@@ -2125,7 +2392,7 @@ static int tun_peek_len(struct socket *sock)
 	if (!tun)
 		return 0;
 
-	ret = skb_array_peek_len(&tfile->tx_array);
+	ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
 	tun_put(tun);
 
 	return ret;
@@ -2296,6 +2563,7 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
 		tun->filter_attached = false;
 		tun->sndbuf = tfile->socket.sk->sk_sndbuf;
 		tun->rx_batched = 0;
+		RCU_INIT_POINTER(tun->steering_prog, NULL);
 
 		tun->pcpu_stats = netdev_alloc_pcpu_stats(struct tun_pcpu_stats);
 		if (!tun->pcpu_stats) {
@@ -2488,6 +2756,26 @@ static int tun_set_queue(struct file *file, struct ifreq *ifr)
 	return ret;
 }
 
+static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog **prog_p,
+			void __user *data)
+{
+	struct bpf_prog *prog;
+	int fd;
+
+	if (copy_from_user(&fd, data, sizeof(fd)))
+		return -EFAULT;
+
+	if (fd == -1) {
+		prog = NULL;
+	} else {
+		prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
+		if (IS_ERR(prog))
+			return PTR_ERR(prog);
+	}
+
+	return __tun_set_ebpf(tun, prog_p, prog);
+}
+
 static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 			    unsigned long arg, int ifreq_len)
 {
@@ -2764,6 +3052,14 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 		ret = 0;
 		break;
 
+	case TUNSETSTEERINGEBPF:
+		ret = tun_set_ebpf(tun, &tun->steering_prog, argp);
+		break;
+
+	case TUNSETFILTEREBPF:
+		ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
+		break;
+
 	default:
 		ret = -EINVAL;
 		break;
@@ -2860,7 +3156,7 @@ static int tun_chr_open(struct inode *inode, struct file * file)
 
 	sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
 
-	memset(&tfile->tx_array, 0, sizeof(tfile->tx_array));
+	memset(&tfile->tx_ring, 0, sizeof(tfile->tx_ring));
 
 	return 0;
 }
@@ -3009,25 +3305,26 @@ static int tun_queue_resize(struct tun_struct *tun)
 {
 	struct net_device *dev = tun->dev;
 	struct tun_file *tfile;
-	struct skb_array **arrays;
+	struct ptr_ring **rings;
 	int n = tun->numqueues + tun->numdisabled;
 	int ret, i;
 
-	arrays = kmalloc_array(n, sizeof(*arrays), GFP_KERNEL);
-	if (!arrays)
+	rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL);
+	if (!rings)
 		return -ENOMEM;
 
 	for (i = 0; i < tun->numqueues; i++) {
 		tfile = rtnl_dereference(tun->tfiles[i]);
-		arrays[i] = &tfile->tx_array;
+		rings[i] = &tfile->tx_ring;
 	}
 	list_for_each_entry(tfile, &tun->disabled, next)
-		arrays[i++] = &tfile->tx_array;
+		rings[i++] = &tfile->tx_ring;
 
-	ret = skb_array_resize_multiple(arrays, n,
-					dev->tx_queue_len, GFP_KERNEL);
+	ret = ptr_ring_resize_multiple(rings, n,
+				       dev->tx_queue_len, GFP_KERNEL,
+				       tun_ptr_free);
 
-	kfree(arrays);
+	kfree(rings);
 	return ret;
 }
 
@@ -3113,7 +3410,7 @@ struct socket *tun_get_socket(struct file *file)
 }
 EXPORT_SYMBOL_GPL(tun_get_socket);
 
-struct skb_array *tun_get_skb_array(struct file *file)
+struct ptr_ring *tun_get_tx_ring(struct file *file)
 {
 	struct tun_file *tfile;
 
@@ -3122,9 +3419,9 @@ struct skb_array *tun_get_skb_array(struct file *file)
 	tfile = file->private_data;
 	if (!tfile)
 		return ERR_PTR(-EBADFD);
-	return &tfile->tx_array;
+	return &tfile->tx_ring;
 }
-EXPORT_SYMBOL_GPL(tun_get_skb_array);
+EXPORT_SYMBOL_GPL(tun_get_tx_ring);
 
 module_init(tun_init);
 module_exit(tun_cleanup);
diff --git a/drivers/net/usb/cdc_ether.c b/drivers/net/usb/cdc_ether.c
index 05dca3e5c93d..fff4b13eece2 100644
--- a/drivers/net/usb/cdc_ether.c
+++ b/drivers/net/usb/cdc_ether.c
@@ -895,6 +895,12 @@ static const struct usb_device_id	products[] = {
 				      USB_CDC_SUBCLASS_ETHERNET,
 				      USB_CDC_PROTO_NONE),
 	.driver_info = (unsigned long)&wwan_info,
+}, {
+	/* Cinterion PLS8 modem by GEMALTO */
+	USB_DEVICE_AND_INTERFACE_INFO(0x1e2d, 0x0061, USB_CLASS_COMM,
+				      USB_CDC_SUBCLASS_ETHERNET,
+				      USB_CDC_PROTO_NONE),
+	.driver_info = (unsigned long)&wwan_info,
 }, {
 	USB_INTERFACE_INFO(USB_CLASS_COMM, USB_CDC_SUBCLASS_ETHERNET,
 			USB_CDC_PROTO_NONE),
diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
index ec56ff29aac4..55a78eb96961 100644
--- a/drivers/net/usb/lan78xx.c
+++ b/drivers/net/usb/lan78xx.c
@@ -2006,7 +2006,7 @@ static int lan78xx_phy_init(struct lan78xx_net *dev)
 {
 	int ret;
 	u32 mii_adv;
-	struct phy_device *phydev = dev->net->phydev;
+	struct phy_device *phydev;
 
 	phydev = phy_find_first(dev->mdiobus);
 	if (!phydev) {
@@ -2351,6 +2351,7 @@ static int lan78xx_reset(struct lan78xx_net *dev)
 	u32 buf;
 	int ret = 0;
 	unsigned long timeout;
+	u8 sig;
 
 	ret = lan78xx_read_reg(dev, HW_CFG, &buf);
 	buf |= HW_CFG_LRST_;
@@ -2450,6 +2451,15 @@ static int lan78xx_reset(struct lan78xx_net *dev)
 	/* LAN7801 only has RGMII mode */
 	if (dev->chipid == ID_REV_CHIP_ID_7801_)
 		buf &= ~MAC_CR_GMII_EN_;
+
+	if (dev->chipid == ID_REV_CHIP_ID_7800_) {
+		ret = lan78xx_read_raw_eeprom(dev, 0, 1, &sig);
+		if (!ret && sig != EEPROM_INDICATOR) {
+			/* Implies there is no external eeprom. Set mac speed */
+			netdev_info(dev->net, "No External EEPROM. Setting MAC Speed\n");
+			buf |= MAC_CR_AUTO_DUPLEX_ | MAC_CR_AUTO_SPEED_;
+		}
+	}
 	ret = lan78xx_write_reg(dev, MAC_CR, buf);
 
 	ret = lan78xx_read_reg(dev, MAC_TX, &buf);
@@ -2863,8 +2873,7 @@ static int lan78xx_bind(struct lan78xx_net *dev, struct usb_interface *intf)
 	if (ret < 0) {
 		netdev_warn(dev->net,
 			    "lan78xx_setup_irq_domain() failed : %d", ret);
-		kfree(pdata);
-		return ret;
+		goto out1;
 	}
 
 	dev->net->hard_header_len += TX_OVERHEAD;
@@ -2872,14 +2881,32 @@ static int lan78xx_bind(struct lan78xx_net *dev, struct usb_interface *intf)
 
 	/* Init all registers */
 	ret = lan78xx_reset(dev);
+	if (ret) {
+		netdev_warn(dev->net, "Registers INIT FAILED....");
+		goto out2;
+	}
 
 	ret = lan78xx_mdio_init(dev);
+	if (ret) {
+		netdev_warn(dev->net, "MDIO INIT FAILED.....");
+		goto out2;
+	}
 
 	dev->net->flags |= IFF_MULTICAST;
 
 	pdata->wol = WAKE_MAGIC;
 
 	return ret;
+
+out2:
+	lan78xx_remove_irq_domain(dev);
+
+out1:
+	netdev_warn(dev->net, "Bind routine FAILED");
+	cancel_work_sync(&pdata->set_multicast);
+	cancel_work_sync(&pdata->set_vlan);
+	kfree(pdata);
+	return ret;
 }
 
 static void lan78xx_unbind(struct lan78xx_net *dev, struct usb_interface *intf)
@@ -2891,6 +2918,8 @@ static void lan78xx_unbind(struct lan78xx_net *dev, struct usb_interface *intf)
 	lan78xx_remove_mdio(dev);
 
 	if (pdata) {
+		cancel_work_sync(&pdata->set_multicast);
+		cancel_work_sync(&pdata->set_vlan);
 		netif_dbg(dev, ifdown, dev->net, "free pdata");
 		kfree(pdata);
 		pdata = NULL;
diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c
index 728819feab44..ca066b785e9f 100644
--- a/drivers/net/usb/qmi_wwan.c
+++ b/drivers/net/usb/qmi_wwan.c
@@ -826,7 +826,7 @@ static int qmi_wwan_resume(struct usb_interface *intf)
 
 static const struct driver_info	qmi_wwan_info = {
 	.description	= "WWAN/QMI device",
-	.flags		= FLAG_WWAN,
+	.flags		= FLAG_WWAN | FLAG_SEND_ZLP,
 	.bind		= qmi_wwan_bind,
 	.unbind		= qmi_wwan_unbind,
 	.manage_power	= qmi_wwan_manage_power,
@@ -835,7 +835,7 @@ static const struct driver_info	qmi_wwan_info = {
 
 static const struct driver_info	qmi_wwan_info_quirk_dtr = {
 	.description	= "WWAN/QMI device",
-	.flags		= FLAG_WWAN,
+	.flags		= FLAG_WWAN | FLAG_SEND_ZLP,
 	.bind		= qmi_wwan_bind,
 	.unbind		= qmi_wwan_unbind,
 	.manage_power	= qmi_wwan_manage_power,
@@ -1104,6 +1104,9 @@ static const struct usb_device_id products[] = {
 	{QMI_FIXED_INTF(0x0846, 0x68a2, 8)},
 	{QMI_FIXED_INTF(0x12d1, 0x140c, 1)},	/* Huawei E173 */
 	{QMI_FIXED_INTF(0x12d1, 0x14ac, 1)},	/* Huawei E1820 */
+	{QMI_FIXED_INTF(0x1435, 0xd181, 3)},	/* Wistron NeWeb D18Q1 */
+	{QMI_FIXED_INTF(0x1435, 0xd181, 4)},	/* Wistron NeWeb D18Q1 */
+	{QMI_FIXED_INTF(0x1435, 0xd181, 5)},	/* Wistron NeWeb D18Q1 */
 	{QMI_FIXED_INTF(0x16d8, 0x6003, 0)},	/* CMOTech 6003 */
 	{QMI_FIXED_INTF(0x16d8, 0x6007, 0)},	/* CMOTech CHE-628S */
 	{QMI_FIXED_INTF(0x16d8, 0x6008, 0)},	/* CMOTech CMU-301 */
@@ -1180,6 +1183,7 @@ static const struct usb_device_id products[] = {
 	{QMI_FIXED_INTF(0x19d2, 0x2002, 4)},	/* ZTE (Vodafone) K3765-Z */
 	{QMI_FIXED_INTF(0x2001, 0x7e19, 4)},	/* D-Link DWM-221 B1 */
 	{QMI_FIXED_INTF(0x2001, 0x7e35, 4)},	/* D-Link DWM-222 */
+	{QMI_FIXED_INTF(0x2020, 0x2033, 4)},	/* BroadMobi BM806U */
 	{QMI_FIXED_INTF(0x0f3d, 0x68a2, 8)},    /* Sierra Wireless MC7700 */
 	{QMI_FIXED_INTF(0x114f, 0x68a2, 8)},    /* Sierra Wireless MC7750 */
 	{QMI_FIXED_INTF(0x1199, 0x68a2, 8)},	/* Sierra Wireless MC7710 in QMI mode */
@@ -1240,11 +1244,13 @@ static const struct usb_device_id products[] = {
 	{QMI_FIXED_INTF(0x413c, 0x81b6, 8)},	/* Dell Wireless 5811e */
 	{QMI_FIXED_INTF(0x413c, 0x81b6, 10)},	/* Dell Wireless 5811e */
 	{QMI_FIXED_INTF(0x03f0, 0x4e1d, 8)},	/* HP lt4111 LTE/EV-DO/HSPA+ Gobi 4G Module */
+	{QMI_FIXED_INTF(0x03f0, 0x9d1d, 1)},	/* HP lt4120 Snapdragon X5 LTE */
 	{QMI_FIXED_INTF(0x22de, 0x9061, 3)},	/* WeTelecom WPD-600N */
 	{QMI_FIXED_INTF(0x1e0e, 0x9001, 5)},	/* SIMCom 7230E */
 	{QMI_QUIRK_SET_DTR(0x2c7c, 0x0125, 4)},	/* Quectel EC25, EC20 R2.0  Mini PCIe */
 	{QMI_QUIRK_SET_DTR(0x2c7c, 0x0121, 4)},	/* Quectel EC21 Mini PCIe */
 	{QMI_FIXED_INTF(0x2c7c, 0x0296, 4)},	/* Quectel BG96 */
+	{QMI_QUIRK_SET_DTR(0x2c7c, 0x0306, 4)},	/* Quectel EP06 Mini PCIe */
 
 	/* 4. Gobi 1000 devices */
 	{QMI_GOBI1K_DEVICE(0x05c6, 0x9212)},	/* Acer Gobi Modem Device */
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 0657203ffb91..86f7196f9d91 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -1794,7 +1794,7 @@ static int r8152_tx_agg_fill(struct r8152 *tp, struct tx_agg *agg)
 
 		tx_data += len;
 		agg->skb_len += len;
-		agg->skb_num++;
+		agg->skb_num += skb_shinfo(skb)->gso_segs ?: 1;
 
 		dev_kfree_skb_any(skb);
 
@@ -1848,11 +1848,9 @@ static u8 r8152_rx_csum(struct r8152 *tp, struct rx_desc *rx_desc)
 	if (opts2 & RD_IPV4_CS) {
 		if (opts3 & IPF)
 			checksum = CHECKSUM_NONE;
-		else if ((opts2 & RD_UDP_CS) && (opts3 & UDPF))
-			checksum = CHECKSUM_NONE;
-		else if ((opts2 & RD_TCP_CS) && (opts3 & TCPF))
-			checksum = CHECKSUM_NONE;
-		else
+		else if ((opts2 & RD_UDP_CS) && !(opts3 & UDPF))
+			checksum = CHECKSUM_UNNECESSARY;
+		else if ((opts2 & RD_TCP_CS) && !(opts3 & TCPF))
 			checksum = CHECKSUM_UNNECESSARY;
 	} else if (opts2 & RD_IPV6_CS) {
 		if ((opts2 & RD_UDP_CS) && !(opts3 & UDPF))
@@ -3797,11 +3795,12 @@ static void set_carrier(struct r8152 *tp)
 	if (speed & LINK_STATUS) {
 		if (!netif_carrier_ok(netdev)) {
 			tp->rtl_ops.enable(tp);
-			set_bit(RTL8152_SET_RX_MODE, &tp->flags);
 			netif_stop_queue(netdev);
 			napi_disable(napi);
 			netif_carrier_on(netdev);
 			rtl_start_rx(tp);
+			clear_bit(RTL8152_SET_RX_MODE, &tp->flags);
+			_rtl8152_set_rx_mode(netdev);
 			napi_enable(&tp->napi);
 			netif_wake_queue(netdev);
 			netif_info(tp, link, netdev, "carrier on\n");
@@ -4261,7 +4260,7 @@ static int rtl8152_post_reset(struct usb_interface *intf)
 		mutex_lock(&tp->control);
 		tp->rtl_ops.enable(tp);
 		rtl_start_rx(tp);
-		rtl8152_set_rx_mode(netdev);
+		_rtl8152_set_rx_mode(netdev);
 		mutex_unlock(&tp->control);
 	}
 
diff --git a/drivers/net/usb/smsc75xx.c b/drivers/net/usb/smsc75xx.c
index d0a113743195..7a6a1fe79309 100644
--- a/drivers/net/usb/smsc75xx.c
+++ b/drivers/net/usb/smsc75xx.c
@@ -954,10 +954,11 @@ static int smsc75xx_set_features(struct net_device *netdev,
 	/* it's racing here! */
 
 	ret = smsc75xx_write_reg(dev, RFE_CTL, pdata->rfe_ctl);
-	if (ret < 0)
+	if (ret < 0) {
 		netdev_warn(dev->net, "Error writing RFE_CTL\n");
-
-	return ret;
+		return ret;
+	}
+	return 0;
 }
 
 static int smsc75xx_wait_ready(struct usbnet *dev, int in_pm)
diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c
index 8a22ff67b026..d9eea8cfe6cb 100644
--- a/drivers/net/usb/usbnet.c
+++ b/drivers/net/usb/usbnet.c
@@ -315,6 +315,7 @@ static void __usbnet_status_stop_force(struct usbnet *dev)
 void usbnet_skb_return (struct usbnet *dev, struct sk_buff *skb)
 {
 	struct pcpu_sw_netstats *stats64 = this_cpu_ptr(dev->stats64);
+	unsigned long flags;
 	int	status;
 
 	if (test_bit(EVENT_RX_PAUSED, &dev->flags)) {
@@ -326,10 +327,10 @@ void usbnet_skb_return (struct usbnet *dev, struct sk_buff *skb)
 	if (skb->protocol == 0)
 		skb->protocol = eth_type_trans (skb, dev->net);
 
-	u64_stats_update_begin(&stats64->syncp);
+	flags = u64_stats_update_begin_irqsave(&stats64->syncp);
 	stats64->rx_packets++;
 	stats64->rx_bytes += skb->len;
-	u64_stats_update_end(&stats64->syncp);
+	u64_stats_update_end_irqrestore(&stats64->syncp, flags);
 
 	netif_dbg(dev, rx_status, dev->net, "< rx, len %zu, type 0x%x\n",
 		  skb->len + sizeof (struct ethhdr), skb->protocol);
@@ -1248,11 +1249,12 @@ static void tx_complete (struct urb *urb)
 
 	if (urb->status == 0) {
 		struct pcpu_sw_netstats *stats64 = this_cpu_ptr(dev->stats64);
+		unsigned long flags;
 
-		u64_stats_update_begin(&stats64->syncp);
+		flags = u64_stats_update_begin_irqsave(&stats64->syncp);
 		stats64->tx_packets += entry->packets;
 		stats64->tx_bytes += entry->length;
-		u64_stats_update_end(&stats64->syncp);
+		u64_stats_update_end_irqrestore(&stats64->syncp, flags);
 	} else {
 		dev->net->stats.tx_errors++;
 
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index f5438d0978ca..a69ad39ee57e 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -410,6 +410,9 @@ static int veth_newlink(struct net *src_net, struct net_device *dev,
 	if (ifmp && (dev->ifindex != 0))
 		peer->ifindex = ifmp->ifi_index;
 
+	peer->gso_max_size = dev->gso_max_size;
+	peer->gso_max_segs = dev->gso_max_segs;
+
 	err = register_netdevice(peer);
 	put_net(net);
 	net = NULL;
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 559b215c0169..23374603e4d9 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -31,6 +31,7 @@
 #include <linux/average.h>
 #include <linux/filter.h>
 #include <net/route.h>
+#include <net/xdp.h>
 
 static int napi_weight = NAPI_POLL_WEIGHT;
 module_param(napi_weight, int, 0444);
@@ -65,16 +66,39 @@ static const unsigned long guest_offloads[] = {
 	VIRTIO_NET_F_GUEST_UFO
 };
 
-struct virtnet_stats {
-	struct u64_stats_sync tx_syncp;
-	struct u64_stats_sync rx_syncp;
-	u64 tx_bytes;
-	u64 tx_packets;
+struct virtnet_stat_desc {
+	char desc[ETH_GSTRING_LEN];
+	size_t offset;
+};
+
+struct virtnet_sq_stats {
+	struct u64_stats_sync syncp;
+	u64 packets;
+	u64 bytes;
+};
+
+struct virtnet_rq_stats {
+	struct u64_stats_sync syncp;
+	u64 packets;
+	u64 bytes;
+};
+
+#define VIRTNET_SQ_STAT(m)	offsetof(struct virtnet_sq_stats, m)
+#define VIRTNET_RQ_STAT(m)	offsetof(struct virtnet_rq_stats, m)
+
+static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = {
+	{ "packets",	VIRTNET_SQ_STAT(packets) },
+	{ "bytes",	VIRTNET_SQ_STAT(bytes) },
+};
 
-	u64 rx_bytes;
-	u64 rx_packets;
+static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
+	{ "packets",	VIRTNET_RQ_STAT(packets) },
+	{ "bytes",	VIRTNET_RQ_STAT(bytes) },
 };
 
+#define VIRTNET_SQ_STATS_LEN	ARRAY_SIZE(virtnet_sq_stats_desc)
+#define VIRTNET_RQ_STATS_LEN	ARRAY_SIZE(virtnet_rq_stats_desc)
+
 /* Internal representation of a send virtqueue */
 struct send_queue {
 	/* Virtqueue associated with this send _queue */
@@ -86,6 +110,8 @@ struct send_queue {
 	/* Name of the send queue: output.$index */
 	char name[40];
 
+	struct virtnet_sq_stats stats;
+
 	struct napi_struct napi;
 };
 
@@ -98,6 +124,8 @@ struct receive_queue {
 
 	struct bpf_prog __rcu *xdp_prog;
 
+	struct virtnet_rq_stats stats;
+
 	/* Chain pages by the private ptr. */
 	struct page *pages;
 
@@ -115,6 +143,8 @@ struct receive_queue {
 
 	/* Name of this receive queue: input.$index */
 	char name[40];
+
+	struct xdp_rxq_info xdp_rxq;
 };
 
 struct virtnet_info {
@@ -149,9 +179,6 @@ struct virtnet_info {
 	/* Packet virtio header size */
 	u8 hdr_len;
 
-	/* Active statistics */
-	struct virtnet_stats __percpu *stats;
-
 	/* Work struct for refilling if we run low on memory. */
 	struct delayed_work refill;
 
@@ -261,9 +288,12 @@ static void virtqueue_napi_complete(struct napi_struct *napi,
 	int opaque;
 
 	opaque = virtqueue_enable_cb_prepare(vq);
-	if (napi_complete_done(napi, processed) &&
-	    unlikely(virtqueue_poll(vq, opaque)))
-		virtqueue_napi_schedule(napi, vq);
+	if (napi_complete_done(napi, processed)) {
+		if (unlikely(virtqueue_poll(vq, opaque)))
+			virtqueue_napi_schedule(napi, vq);
+	} else {
+		virtqueue_disable_cb(vq);
+	}
 }
 
 static void skb_xmit_done(struct virtqueue *vq)
@@ -413,12 +443,8 @@ static bool __virtnet_xdp_xmit(struct virtnet_info *vi,
 	sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
 
 	err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
-	if (unlikely(err)) {
-		struct page *page = virt_to_head_page(xdp->data);
-
-		put_page(page);
-		return false;
-	}
+	if (unlikely(err))
+		return false; /* Caller handle free/refcnt */
 
 	return true;
 }
@@ -426,8 +452,18 @@ static bool __virtnet_xdp_xmit(struct virtnet_info *vi,
 static int virtnet_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
-	bool sent = __virtnet_xdp_xmit(vi, xdp);
+	struct receive_queue *rq = vi->rq;
+	struct bpf_prog *xdp_prog;
+	bool sent;
 
+	/* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this
+	 * indicate XDP resources have been successfully allocated.
+	 */
+	xdp_prog = rcu_dereference(rq->xdp_prog);
+	if (!xdp_prog)
+		return -ENXIO;
+
+	sent = __virtnet_xdp_xmit(vi, xdp);
 	if (!sent)
 		return -ENOSPC;
 	return 0;
@@ -468,6 +504,7 @@ static struct page *xdp_linearize_page(struct receive_queue *rq,
 	page_off += *len;
 
 	while (--*num_buf) {
+		int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 		unsigned int buflen;
 		void *buf;
 		int off;
@@ -482,7 +519,7 @@ static struct page *xdp_linearize_page(struct receive_queue *rq,
 		/* guard against a misconfigured or uncooperative backend that
 		 * is sending packet larger than the MTU.
 		 */
-		if ((page_off + buflen) > PAGE_SIZE) {
+		if ((page_off + buflen + tailroom) > PAGE_SIZE) {
 			put_page(p);
 			goto err_buf;
 		}
@@ -516,8 +553,11 @@ static struct sk_buff *receive_small(struct net_device *dev,
 	unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 			      SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
 	struct page *page = virt_to_head_page(buf);
-	unsigned int delta = 0, err;
+	unsigned int delta = 0;
 	struct page *xdp_page;
+	bool sent;
+	int err;
+
 	len -= vi->hdr_len;
 
 	rcu_read_lock();
@@ -528,7 +568,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
 		void *orig_data;
 		u32 act;
 
-		if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
+		if (unlikely(hdr->hdr.gso_type))
 			goto err_xdp;
 
 		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
@@ -556,6 +596,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
 		xdp.data = xdp.data_hard_start + xdp_headroom;
 		xdp_set_data_meta_invalid(&xdp);
 		xdp.data_end = xdp.data + len;
+		xdp.rxq = &rq->xdp_rxq;
 		orig_data = xdp.data;
 		act = bpf_prog_run_xdp(xdp_prog, &xdp);
 
@@ -565,16 +606,19 @@ static struct sk_buff *receive_small(struct net_device *dev,
 			delta = orig_data - xdp.data;
 			break;
 		case XDP_TX:
-			if (unlikely(!__virtnet_xdp_xmit(vi, &xdp)))
+			sent = __virtnet_xdp_xmit(vi, &xdp);
+			if (unlikely(!sent)) {
 				trace_xdp_exception(vi->dev, xdp_prog, act);
-			else
-				*xdp_xmit = true;
+				goto err_xdp;
+			}
+			*xdp_xmit = true;
 			rcu_read_unlock();
 			goto xdp_xmit;
 		case XDP_REDIRECT:
 			err = xdp_do_redirect(dev, &xdp, xdp_prog);
-			if (!err)
-				*xdp_xmit = true;
+			if (err)
+				goto err_xdp;
+			*xdp_xmit = true;
 			rcu_read_unlock();
 			goto xdp_xmit;
 		default:
@@ -646,6 +690,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	struct bpf_prog *xdp_prog;
 	unsigned int truesize;
 	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
+	bool sent;
 	int err;
 
 	head_skb = NULL;
@@ -658,7 +703,12 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		void *data;
 		u32 act;
 
-		/* This happens when rx buffer size is underestimated */
+		/* This happens when rx buffer size is underestimated
+		 * or headroom is not enough because of the buffer
+		 * was refilled before XDP is set. This should only
+		 * happen for the first several packets, so we don't
+		 * care much about its performance.
+		 */
 		if (unlikely(num_buf > 1 ||
 			     headroom < virtnet_get_headroom(vi))) {
 			/* linearize data for XDP */
@@ -689,10 +739,9 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		xdp.data = data + vi->hdr_len;
 		xdp_set_data_meta_invalid(&xdp);
 		xdp.data_end = xdp.data + (len - vi->hdr_len);
-		act = bpf_prog_run_xdp(xdp_prog, &xdp);
+		xdp.rxq = &rq->xdp_rxq;
 
-		if (act != XDP_PASS)
-			ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
+		act = bpf_prog_run_xdp(xdp_prog, &xdp);
 
 		switch (act) {
 		case XDP_PASS:
@@ -713,18 +762,28 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 			}
 			break;
 		case XDP_TX:
-			if (unlikely(!__virtnet_xdp_xmit(vi, &xdp)))
+			sent = __virtnet_xdp_xmit(vi, &xdp);
+			if (unlikely(!sent)) {
 				trace_xdp_exception(vi->dev, xdp_prog, act);
-			else
-				*xdp_xmit = true;
+				if (unlikely(xdp_page != page))
+					put_page(xdp_page);
+				goto err_xdp;
+			}
+			*xdp_xmit = true;
 			if (unlikely(xdp_page != page))
 				goto err_xdp;
 			rcu_read_unlock();
 			goto xdp_xmit;
 		case XDP_REDIRECT:
 			err = xdp_do_redirect(dev, &xdp, xdp_prog);
-			if (!err)
-				*xdp_xmit = true;
+			if (err) {
+				if (unlikely(xdp_page != page))
+					put_page(xdp_page);
+				goto err_xdp;
+			}
+			*xdp_xmit = true;
+			if (unlikely(xdp_page != page))
+				goto err_xdp;
 			rcu_read_unlock();
 			goto xdp_xmit;
 		default:
@@ -970,13 +1029,18 @@ static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
 }
 
 static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
-					  struct ewma_pkt_len *avg_pkt_len)
+					  struct ewma_pkt_len *avg_pkt_len,
+					  unsigned int room)
 {
 	const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
 	unsigned int len;
 
-	len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
+	if (room)
+		return PAGE_SIZE - room;
+
+	len = hdr_len +	clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len),
 				rq->min_buf_len, PAGE_SIZE - hdr_len);
+
 	return ALIGN(len, L1_CACHE_BYTES);
 }
 
@@ -985,21 +1049,27 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
 {
 	struct page_frag *alloc_frag = &rq->alloc_frag;
 	unsigned int headroom = virtnet_get_headroom(vi);
+	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
+	unsigned int room = SKB_DATA_ALIGN(headroom + tailroom);
 	char *buf;
 	void *ctx;
 	int err;
 	unsigned int len, hole;
 
-	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len);
-	if (unlikely(!skb_page_frag_refill(len + headroom, alloc_frag, gfp)))
+	/* Extra tailroom is needed to satisfy XDP's assumption. This
+	 * means rx frags coalescing won't work, but consider we've
+	 * disabled GSO for XDP, it won't be a big issue.
+	 */
+	len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room);
+	if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp)))
 		return -ENOMEM;
 
 	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
 	buf += headroom; /* advance address leaving hole at front of pkt */
 	get_page(alloc_frag->page);
-	alloc_frag->offset += len + headroom;
+	alloc_frag->offset += len + room;
 	hole = alloc_frag->size - alloc_frag->offset;
-	if (hole < len + headroom) {
+	if (hole < len + room) {
 		/* To avoid internal fragmentation, if there is very likely not
 		 * enough space for another buffer, add the remaining space to
 		 * the current buffer.
@@ -1118,7 +1188,6 @@ static int virtnet_receive(struct receive_queue *rq, int budget, bool *xdp_xmit)
 	struct virtnet_info *vi = rq->vq->vdev->priv;
 	unsigned int len, received = 0, bytes = 0;
 	void *buf;
-	struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
 
 	if (!vi->big_packets || vi->mergeable_rx_bufs) {
 		void *ctx;
@@ -1141,10 +1210,10 @@ static int virtnet_receive(struct receive_queue *rq, int budget, bool *xdp_xmit)
 			schedule_delayed_work(&vi->refill, 0);
 	}
 
-	u64_stats_update_begin(&stats->rx_syncp);
-	stats->rx_bytes += bytes;
-	stats->rx_packets += received;
-	u64_stats_update_end(&stats->rx_syncp);
+	u64_stats_update_begin(&rq->stats.syncp);
+	rq->stats.bytes += bytes;
+	rq->stats.packets += received;
+	u64_stats_update_end(&rq->stats.syncp);
 
 	return received;
 }
@@ -1153,8 +1222,6 @@ static void free_old_xmit_skbs(struct send_queue *sq)
 {
 	struct sk_buff *skb;
 	unsigned int len;
-	struct virtnet_info *vi = sq->vq->vdev->priv;
-	struct virtnet_stats *stats = this_cpu_ptr(vi->stats);
 	unsigned int packets = 0;
 	unsigned int bytes = 0;
 
@@ -1173,10 +1240,10 @@ static void free_old_xmit_skbs(struct send_queue *sq)
 	if (!packets)
 		return;
 
-	u64_stats_update_begin(&stats->tx_syncp);
-	stats->tx_bytes += bytes;
-	stats->tx_packets += packets;
-	u64_stats_update_end(&stats->tx_syncp);
+	u64_stats_update_begin(&sq->stats.syncp);
+	sq->stats.bytes += bytes;
+	sq->stats.packets += packets;
+	u64_stats_update_end(&sq->stats.syncp);
 }
 
 static void virtnet_poll_cleantx(struct receive_queue *rq)
@@ -1222,13 +1289,18 @@ static int virtnet_poll(struct napi_struct *napi, int budget)
 static int virtnet_open(struct net_device *dev)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
-	int i;
+	int i, err;
 
 	for (i = 0; i < vi->max_queue_pairs; i++) {
 		if (i < vi->curr_queue_pairs)
 			/* Make sure we have some buffers: if oom use wq. */
 			if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL))
 				schedule_delayed_work(&vi->refill, 0);
+
+		err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i);
+		if (err < 0)
+			return err;
+
 		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
 		virtnet_napi_tx_enable(vi, vi->sq[i].vq, &vi->sq[i].napi);
 	}
@@ -1460,24 +1532,25 @@ static void virtnet_stats(struct net_device *dev,
 			  struct rtnl_link_stats64 *tot)
 {
 	struct virtnet_info *vi = netdev_priv(dev);
-	int cpu;
 	unsigned int start;
+	int i;
 
-	for_each_possible_cpu(cpu) {
-		struct virtnet_stats *stats = per_cpu_ptr(vi->stats, cpu);
+	for (i = 0; i < vi->max_queue_pairs; i++) {
 		u64 tpackets, tbytes, rpackets, rbytes;
+		struct receive_queue *rq = &vi->rq[i];
+		struct send_queue *sq = &vi->sq[i];
 
 		do {
-			start = u64_stats_fetch_begin_irq(&stats->tx_syncp);
-			tpackets = stats->tx_packets;
-			tbytes   = stats->tx_bytes;
-		} while (u64_stats_fetch_retry_irq(&stats->tx_syncp, start));
+			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
+			tpackets = sq->stats.packets;
+			tbytes   = sq->stats.bytes;
+		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
 
 		do {
-			start = u64_stats_fetch_begin_irq(&stats->rx_syncp);
-			rpackets = stats->rx_packets;
-			rbytes   = stats->rx_bytes;
-		} while (u64_stats_fetch_retry_irq(&stats->rx_syncp, start));
+			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
+			rpackets = rq->stats.packets;
+			rbytes   = rq->stats.bytes;
+		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
 
 		tot->rx_packets += rpackets;
 		tot->tx_packets += tpackets;
@@ -1557,6 +1630,7 @@ static int virtnet_close(struct net_device *dev)
 	cancel_delayed_work_sync(&vi->refill);
 
 	for (i = 0; i < vi->max_queue_pairs; i++) {
+		xdp_rxq_info_unreg(&vi->rq[i].xdp_rxq);
 		napi_disable(&vi->rq[i].napi);
 		virtnet_napi_tx_disable(&vi->sq[i].napi);
 	}
@@ -1814,6 +1888,83 @@ static int virtnet_set_channels(struct net_device *dev,
 	return err;
 }
 
+static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	char *p = (char *)data;
+	unsigned int i, j;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		for (i = 0; i < vi->curr_queue_pairs; i++) {
+			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
+				snprintf(p, ETH_GSTRING_LEN, "rx_queue_%u_%s",
+					 i, virtnet_rq_stats_desc[j].desc);
+				p += ETH_GSTRING_LEN;
+			}
+		}
+
+		for (i = 0; i < vi->curr_queue_pairs; i++) {
+			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
+				snprintf(p, ETH_GSTRING_LEN, "tx_queue_%u_%s",
+					 i, virtnet_sq_stats_desc[j].desc);
+				p += ETH_GSTRING_LEN;
+			}
+		}
+		break;
+	}
+}
+
+static int virtnet_get_sset_count(struct net_device *dev, int sset)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+
+	switch (sset) {
+	case ETH_SS_STATS:
+		return vi->curr_queue_pairs * (VIRTNET_RQ_STATS_LEN +
+					       VIRTNET_SQ_STATS_LEN);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void virtnet_get_ethtool_stats(struct net_device *dev,
+				      struct ethtool_stats *stats, u64 *data)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	unsigned int idx = 0, start, i, j;
+	const u8 *stats_base;
+	size_t offset;
+
+	for (i = 0; i < vi->curr_queue_pairs; i++) {
+		struct receive_queue *rq = &vi->rq[i];
+
+		stats_base = (u8 *)&rq->stats;
+		do {
+			start = u64_stats_fetch_begin_irq(&rq->stats.syncp);
+			for (j = 0; j < VIRTNET_RQ_STATS_LEN; j++) {
+				offset = virtnet_rq_stats_desc[j].offset;
+				data[idx + j] = *(u64 *)(stats_base + offset);
+			}
+		} while (u64_stats_fetch_retry_irq(&rq->stats.syncp, start));
+		idx += VIRTNET_RQ_STATS_LEN;
+	}
+
+	for (i = 0; i < vi->curr_queue_pairs; i++) {
+		struct send_queue *sq = &vi->sq[i];
+
+		stats_base = (u8 *)&sq->stats;
+		do {
+			start = u64_stats_fetch_begin_irq(&sq->stats.syncp);
+			for (j = 0; j < VIRTNET_SQ_STATS_LEN; j++) {
+				offset = virtnet_sq_stats_desc[j].offset;
+				data[idx + j] = *(u64 *)(stats_base + offset);
+			}
+		} while (u64_stats_fetch_retry_irq(&sq->stats.syncp, start));
+		idx += VIRTNET_SQ_STATS_LEN;
+	}
+}
+
 static void virtnet_get_channels(struct net_device *dev,
 				 struct ethtool_channels *channels)
 {
@@ -1891,10 +2042,31 @@ static void virtnet_init_settings(struct net_device *dev)
 	vi->duplex = DUPLEX_UNKNOWN;
 }
 
+static void virtnet_update_settings(struct virtnet_info *vi)
+{
+	u32 speed;
+	u8 duplex;
+
+	if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX))
+		return;
+
+	speed = virtio_cread32(vi->vdev, offsetof(struct virtio_net_config,
+						  speed));
+	if (ethtool_validate_speed(speed))
+		vi->speed = speed;
+	duplex = virtio_cread8(vi->vdev, offsetof(struct virtio_net_config,
+						  duplex));
+	if (ethtool_validate_duplex(duplex))
+		vi->duplex = duplex;
+}
+
 static const struct ethtool_ops virtnet_ethtool_ops = {
 	.get_drvinfo = virtnet_get_drvinfo,
 	.get_link = ethtool_op_get_link,
 	.get_ringparam = virtnet_get_ringparam,
+	.get_strings = virtnet_get_strings,
+	.get_sset_count = virtnet_get_sset_count,
+	.get_ethtool_stats = virtnet_get_ethtool_stats,
 	.set_channels = virtnet_set_channels,
 	.get_channels = virtnet_get_channels,
 	.get_ts_info = ethtool_op_get_ts_info,
@@ -2040,8 +2212,9 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
 	}
 
 	/* Make sure NAPI is not using any XDP TX queues for RX. */
-	for (i = 0; i < vi->max_queue_pairs; i++)
-		napi_disable(&vi->rq[i].napi);
+	if (netif_running(dev))
+		for (i = 0; i < vi->max_queue_pairs; i++)
+			napi_disable(&vi->rq[i].napi);
 
 	netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp);
 	err = _virtnet_set_queues(vi, curr_qp + xdp_qp);
@@ -2060,7 +2233,8 @@ static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog,
 		}
 		if (old_prog)
 			bpf_prog_put(old_prog);
-		virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
+		if (netif_running(dev))
+			virtnet_napi_enable(vi->rq[i].vq, &vi->rq[i].napi);
 	}
 
 	return 0;
@@ -2144,6 +2318,7 @@ static void virtnet_config_changed_work(struct work_struct *work)
 	vi->status = v;
 
 	if (vi->status & VIRTIO_NET_S_LINK_UP) {
+		virtnet_update_settings(vi);
 		netif_carrier_on(vi->dev);
 		netif_tx_wake_all_queues(vi->dev);
 	} else {
@@ -2386,6 +2561,9 @@ static int virtnet_alloc_queues(struct virtnet_info *vi)
 		sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg));
 		ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len);
 		sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg));
+
+		u64_stats_init(&vi->rq[i].stats.syncp);
+		u64_stats_init(&vi->sq[i].stats.syncp);
 	}
 
 	return 0;
@@ -2427,12 +2605,15 @@ static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue,
 {
 	struct virtnet_info *vi = netdev_priv(queue->dev);
 	unsigned int queue_index = get_netdev_rx_queue_index(queue);
+	unsigned int headroom = virtnet_get_headroom(vi);
+	unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0;
 	struct ewma_pkt_len *avg;
 
 	BUG_ON(queue_index >= vi->max_queue_pairs);
 	avg = &vi->rq[queue_index].mrg_avg_pkt_len;
 	return sprintf(buf, "%u\n",
-		       get_mergeable_buf_len(&vi->rq[queue_index], avg));
+		       get_mergeable_buf_len(&vi->rq[queue_index], avg,
+				       SKB_DATA_ALIGN(headroom + tailroom)));
 }
 
 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute =
@@ -2510,7 +2691,7 @@ static int virtnet_validate(struct virtio_device *vdev)
 
 static int virtnet_probe(struct virtio_device *vdev)
 {
-	int i, err;
+	int i, err = -ENOMEM;
 	struct net_device *dev;
 	struct virtnet_info *vi;
 	u16 max_queue_pairs;
@@ -2587,17 +2768,6 @@ static int virtnet_probe(struct virtio_device *vdev)
 	vi->dev = dev;
 	vi->vdev = vdev;
 	vdev->priv = vi;
-	vi->stats = alloc_percpu(struct virtnet_stats);
-	err = -ENOMEM;
-	if (vi->stats == NULL)
-		goto free;
-
-	for_each_possible_cpu(i) {
-		struct virtnet_stats *virtnet_stats;
-		virtnet_stats = per_cpu_ptr(vi->stats, i);
-		u64_stats_init(&virtnet_stats->tx_syncp);
-		u64_stats_init(&virtnet_stats->rx_syncp);
-	}
 
 	INIT_WORK(&vi->config_work, virtnet_config_changed_work);
 
@@ -2634,7 +2804,7 @@ static int virtnet_probe(struct virtio_device *vdev)
 			 */
 			dev_err(&vdev->dev, "device MTU appears to have changed "
 				"it is now %d < %d", mtu, dev->min_mtu);
-			goto free_stats;
+			goto free;
 		}
 
 		dev->mtu = mtu;
@@ -2658,7 +2828,7 @@ static int virtnet_probe(struct virtio_device *vdev)
 	/* Allocate/initialize the rx/tx queues, and invoke find_vqs */
 	err = init_vqs(vi);
 	if (err)
-		goto free_stats;
+		goto free;
 
 #ifdef CONFIG_SYSFS
 	if (vi->mergeable_rx_bufs)
@@ -2692,6 +2862,7 @@ static int virtnet_probe(struct virtio_device *vdev)
 		schedule_work(&vi->config_work);
 	} else {
 		vi->status = VIRTIO_NET_S_LINK_UP;
+		virtnet_update_settings(vi);
 		netif_carrier_on(dev);
 	}
 
@@ -2712,8 +2883,6 @@ static int virtnet_probe(struct virtio_device *vdev)
 	cancel_delayed_work_sync(&vi->refill);
 	free_receive_page_frags(vi);
 	virtnet_del_vqs(vi);
-free_stats:
-	free_percpu(vi->stats);
 free:
 	free_netdev(dev);
 	return err;
@@ -2746,7 +2915,6 @@ static void virtnet_remove(struct virtio_device *vdev)
 
 	remove_vq_common(vi);
 
-	free_percpu(vi->stats);
 	free_netdev(vi->dev);
 }
 
@@ -2793,7 +2961,8 @@ static struct virtio_device_id id_table[] = {
 	VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \
 	VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
 	VIRTIO_NET_F_CTRL_MAC_ADDR, \
-	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
+	VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
+	VIRTIO_NET_F_SPEED_DUPLEX
 
 static unsigned int features[] = {
 	VIRTNET_FEATURES,
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index cf95290b160c..e04937f44f33 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -977,6 +977,8 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 {
 	int ret;
 	u32 count;
+	int num_pkts;
+	int tx_num_deferred;
 	unsigned long flags;
 	struct vmxnet3_tx_ctx ctx;
 	union Vmxnet3_GenericDesc *gdesc;
@@ -1075,12 +1077,12 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 #else
 	gdesc = ctx.sop_txd;
 #endif
+	tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);
 	if (ctx.mss) {
 		gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
 		gdesc->txd.om = VMXNET3_OM_TSO;
 		gdesc->txd.msscof = ctx.mss;
-		le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
-			     gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
+		num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss;
 	} else {
 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
 			gdesc->txd.hlen = ctx.eth_ip_hdr_size;
@@ -1091,8 +1093,10 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 			gdesc->txd.om = 0;
 			gdesc->txd.msscof = 0;
 		}
-		le32_add_cpu(&tq->shared->txNumDeferred, 1);
+		num_pkts = 1;
 	}
+	le32_add_cpu(&tq->shared->txNumDeferred, num_pkts);
+	tx_num_deferred += num_pkts;
 
 	if (skb_vlan_tag_present(skb)) {
 		gdesc->txd.ti = 1;
@@ -1118,8 +1122,7 @@ vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 
 	spin_unlock_irqrestore(&tq->tx_lock, flags);
 
-	if (le32_to_cpu(tq->shared->txNumDeferred) >=
-					le32_to_cpu(tq->shared->txThreshold)) {
+	if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {
 		tq->shared->txNumDeferred = 0;
 		VMXNET3_WRITE_BAR0_REG(adapter,
 				       VMXNET3_REG_TXPROD + tq->qid * 8,
@@ -1470,7 +1473,8 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
 			vmxnet3_rx_csum(adapter, skb,
 					(union Vmxnet3_GenericDesc *)rcd);
 			skb->protocol = eth_type_trans(skb, adapter->netdev);
-			if (!rcd->tcp || !adapter->lro)
+			if (!rcd->tcp ||
+			    !(adapter->netdev->features & NETIF_F_LRO))
 				goto not_lro;
 
 			if (segCnt != 0 && mss != 0) {
@@ -2760,9 +2764,6 @@ static void
 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
 {
 	size_t sz, i, ring0_size, ring1_size, comp_size;
-	struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[0];
-
-
 	if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
 				    VMXNET3_MAX_ETH_HDR_SIZE) {
 		adapter->skb_buf_size = adapter->netdev->mtu +
@@ -2794,7 +2795,8 @@ vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
 	comp_size = ring0_size + ring1_size;
 
 	for (i = 0; i < adapter->num_rx_queues; i++) {
-		rq = &adapter->rx_queue[i];
+		struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[i];
+
 		rq->rx_ring[0].size = ring0_size;
 		rq->rx_ring[1].size = ring1_size;
 		rq->comp_ring.size = comp_size;
diff --git a/drivers/net/vmxnet3/vmxnet3_int.h b/drivers/net/vmxnet3/vmxnet3_int.h
index 9c51b8be0038..59ec34052a65 100644
--- a/drivers/net/vmxnet3/vmxnet3_int.h
+++ b/drivers/net/vmxnet3/vmxnet3_int.h
@@ -69,10 +69,10 @@
 /*
  * Version numbers
  */
-#define VMXNET3_DRIVER_VERSION_STRING   "1.4.a.0-k"
+#define VMXNET3_DRIVER_VERSION_STRING   "1.4.13.0-k"
 
 /* a 32-bit int, each byte encode a verion number in VMXNET3_DRIVER_VERSION */
-#define VMXNET3_DRIVER_VERSION_NUM      0x01040a00
+#define VMXNET3_DRIVER_VERSION_NUM      0x01040d00
 
 #if defined(CONFIG_PCI_MSI)
 	/* RSS only makes sense if MSI-X is supported. */
@@ -342,9 +342,6 @@ struct vmxnet3_adapter {
 	u8			__iomem *hw_addr1; /* for BAR 1 */
 	u8                              version;
 
-	bool				rxcsum;
-	bool				lro;
-
 #ifdef VMXNET3_RSS
 	struct UPT1_RSSConf		*rss_conf;
 	bool				rss;
@@ -416,8 +413,8 @@ struct vmxnet3_adapter {
 
 /* must be a multiple of VMXNET3_RING_SIZE_ALIGN */
 #define VMXNET3_DEF_TX_RING_SIZE    512
-#define VMXNET3_DEF_RX_RING_SIZE    256
-#define VMXNET3_DEF_RX_RING2_SIZE   128
+#define VMXNET3_DEF_RX_RING_SIZE    1024
+#define VMXNET3_DEF_RX_RING2_SIZE   256
 
 #define VMXNET3_DEF_RXDATA_DESC_SIZE 128
 
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index 139c61c8244a..ac40924fe437 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -578,12 +578,13 @@ static int vrf_finish_output(struct net *net, struct sock *sk, struct sk_buff *s
 	if (!IS_ERR(neigh)) {
 		sock_confirm_neigh(skb, neigh);
 		ret = neigh_output(neigh, skb);
+		rcu_read_unlock_bh();
+		return ret;
 	}
 
 	rcu_read_unlock_bh();
 err:
-	if (unlikely(ret < 0))
-		vrf_tx_error(skb->dev, skb);
+	vrf_tx_error(skb->dev, skb);
 	return ret;
 }
 
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index c3e34e3c82a7..fab7a4db249e 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -3709,18 +3709,16 @@ static __net_init int vxlan_init_net(struct net *net)
 	return 0;
 }
 
-static void __net_exit vxlan_exit_net(struct net *net)
+static void vxlan_destroy_tunnels(struct net *net, struct list_head *head)
 {
 	struct vxlan_net *vn = net_generic(net, vxlan_net_id);
 	struct vxlan_dev *vxlan, *next;
 	struct net_device *dev, *aux;
 	unsigned int h;
-	LIST_HEAD(list);
 
-	rtnl_lock();
 	for_each_netdev_safe(net, dev, aux)
 		if (dev->rtnl_link_ops == &vxlan_link_ops)
-			unregister_netdevice_queue(dev, &list);
+			unregister_netdevice_queue(dev, head);
 
 	list_for_each_entry_safe(vxlan, next, &vn->vxlan_list, next) {
 		/* If vxlan->dev is in the same netns, it has already been added
@@ -3728,20 +3726,30 @@ static void __net_exit vxlan_exit_net(struct net *net)
 		 */
 		if (!net_eq(dev_net(vxlan->dev), net)) {
 			gro_cells_destroy(&vxlan->gro_cells);
-			unregister_netdevice_queue(vxlan->dev, &list);
+			unregister_netdevice_queue(vxlan->dev, head);
 		}
 	}
 
-	unregister_netdevice_many(&list);
-	rtnl_unlock();
-
 	for (h = 0; h < PORT_HASH_SIZE; ++h)
 		WARN_ON_ONCE(!hlist_empty(&vn->sock_list[h]));
 }
 
+static void __net_exit vxlan_exit_batch_net(struct list_head *net_list)
+{
+	struct net *net;
+	LIST_HEAD(list);
+
+	rtnl_lock();
+	list_for_each_entry(net, net_list, exit_list)
+		vxlan_destroy_tunnels(net, &list);
+
+	unregister_netdevice_many(&list);
+	rtnl_unlock();
+}
+
 static struct pernet_operations vxlan_net_ops = {
 	.init = vxlan_init_net,
-	.exit = vxlan_exit_net,
+	.exit_batch = vxlan_exit_batch_net,
 	.id   = &vxlan_net_id,
 	.size = sizeof(struct vxlan_net),
 };
diff --git a/drivers/net/wan/cosa.c b/drivers/net/wan/cosa.c
index 6ea16260ec76..f6b000ddcd15 100644
--- a/drivers/net/wan/cosa.c
+++ b/drivers/net/wan/cosa.c
@@ -924,7 +924,7 @@ static int chrdev_tx_done(struct channel_data *chan, int size)
 	return 1;
 }
 
-static unsigned int cosa_poll(struct file *file, poll_table *poll)
+static __poll_t cosa_poll(struct file *file, poll_table *poll)
 {
 	pr_info("cosa_poll is here\n");
 	return 0;
diff --git a/drivers/net/wan/hdlc_ppp.c b/drivers/net/wan/hdlc_ppp.c
index afeca6bcdade..ab8b3cbbb205 100644
--- a/drivers/net/wan/hdlc_ppp.c
+++ b/drivers/net/wan/hdlc_ppp.c
@@ -574,7 +574,10 @@ static void ppp_timer(struct timer_list *t)
 			ppp_cp_event(proto->dev, proto->pid, TO_GOOD, 0, 0,
 				     0, NULL);
 			proto->restart_counter--;
-		} else
+		} else if (netif_carrier_ok(proto->dev))
+			ppp_cp_event(proto->dev, proto->pid, TO_GOOD, 0, 0,
+				     0, NULL);
+		else
 			ppp_cp_event(proto->dev, proto->pid, TO_BAD, 0, 0,
 				     0, NULL);
 		break;
diff --git a/drivers/net/wimax/i2400m/sysfs.c b/drivers/net/wimax/i2400m/sysfs.c
index 1237109f251a..8c67df11105c 100644
--- a/drivers/net/wimax/i2400m/sysfs.c
+++ b/drivers/net/wimax/i2400m/sysfs.c
@@ -65,8 +65,7 @@ ssize_t i2400m_idle_timeout_store(struct device *dev,
 }
 
 static
-DEVICE_ATTR(i2400m_idle_timeout, S_IWUSR,
-	    NULL, i2400m_idle_timeout_store);
+DEVICE_ATTR_WO(i2400m_idle_timeout);
 
 static
 struct attribute *i2400m_dev_attrs[] = {
diff --git a/drivers/net/wireless/ath/ath10k/Kconfig b/drivers/net/wireless/ath/ath10k/Kconfig
index 87f56d0e17a6..deb5ae21a559 100644
--- a/drivers/net/wireless/ath/ath10k/Kconfig
+++ b/drivers/net/wireless/ath/ath10k/Kconfig
@@ -47,12 +47,19 @@ config ATH10K_DEBUG
 config ATH10K_DEBUGFS
 	bool "Atheros ath10k debugfs support"
 	depends on ATH10K && DEBUG_FS
-	select RELAY
 	---help---
 	  Enabled debugfs support
 
 	  If unsure, say Y to make it easier to debug problems.
 
+config ATH10K_SPECTRAL
+	bool "Atheros ath10k spectral scan support"
+	depends on ATH10K_DEBUGFS
+	select RELAY
+	default n
+	---help---
+	  Say Y to enable access to the FFT/spectral data via debugfs.
+
 config ATH10K_TRACING
 	bool "Atheros ath10k tracing support"
 	depends on ATH10K
diff --git a/drivers/net/wireless/ath/ath10k/Makefile b/drivers/net/wireless/ath/ath10k/Makefile
index 9492177e9063..6739ac26fd29 100644
--- a/drivers/net/wireless/ath/ath10k/Makefile
+++ b/drivers/net/wireless/ath/ath10k/Makefile
@@ -15,12 +15,13 @@ ath10k_core-y += mac.o \
 		 p2p.o \
 		 swap.o
 
-ath10k_core-$(CONFIG_ATH10K_DEBUGFS) += spectral.o
+ath10k_core-$(CONFIG_ATH10K_SPECTRAL) += spectral.o
 ath10k_core-$(CONFIG_NL80211_TESTMODE) += testmode.o
 ath10k_core-$(CONFIG_ATH10K_TRACING) += trace.o
 ath10k_core-$(CONFIG_THERMAL) += thermal.o
 ath10k_core-$(CONFIG_MAC80211_DEBUGFS) += debugfs_sta.o
 ath10k_core-$(CONFIG_PM) += wow.o
+ath10k_core-$(CONFIG_DEV_COREDUMP) += coredump.o
 
 obj-$(CONFIG_ATH10K_PCI) += ath10k_pci.o
 ath10k_pci-y += pci.o \
diff --git a/drivers/net/wireless/ath/ath10k/ahb.c b/drivers/net/wireless/ath/ath10k/ahb.c
index ff6815e95684..35d10490f6c3 100644
--- a/drivers/net/wireless/ath/ath10k/ahb.c
+++ b/drivers/net/wireless/ath/ath10k/ahb.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016 Qualcomm Atheros, Inc. All rights reserved.
+ * Copyright (c) 2016-2017 Qualcomm Atheros, Inc. All rights reserved.
  * Copyright (c) 2015 The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
diff --git a/drivers/net/wireless/ath/ath10k/bmi.c b/drivers/net/wireless/ath/ath10k/bmi.c
index 2d3a2f31123d..af4978d6a14b 100644
--- a/drivers/net/wireless/ath/ath10k/bmi.c
+++ b/drivers/net/wireless/ath/ath10k/bmi.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2014,2016-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/bmi.h b/drivers/net/wireless/ath/ath10k/bmi.h
index 9c0839b2ca8f..9a396817aa55 100644
--- a/drivers/net/wireless/ath/ath10k/bmi.h
+++ b/drivers/net/wireless/ath/ath10k/bmi.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2015,2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c
index a8afd690290f..b9def7bace2f 100644
--- a/drivers/net/wireless/ath/ath10k/ce.c
+++ b/drivers/net/wireless/ath/ath10k/ce.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -327,12 +327,12 @@ static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
  * Guts of ath10k_ce_send.
  * The caller takes responsibility for any needed locking.
  */
-int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
-			  void *per_transfer_context,
-			  u32 buffer,
-			  unsigned int nbytes,
-			  unsigned int transfer_id,
-			  unsigned int flags)
+static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
+				  void *per_transfer_context,
+				  dma_addr_t buffer,
+				  unsigned int nbytes,
+				  unsigned int transfer_id,
+				  unsigned int flags)
 {
 	struct ath10k *ar = ce_state->ar;
 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
@@ -384,6 +384,87 @@ int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 	return ret;
 }
 
+static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
+				     void *per_transfer_context,
+				     dma_addr_t buffer,
+				     unsigned int nbytes,
+				     unsigned int transfer_id,
+				     unsigned int flags)
+{
+	struct ath10k *ar = ce_state->ar;
+	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
+	struct ce_desc_64 *desc, sdesc;
+	unsigned int nentries_mask = src_ring->nentries_mask;
+	unsigned int sw_index = src_ring->sw_index;
+	unsigned int write_index = src_ring->write_index;
+	u32 ctrl_addr = ce_state->ctrl_addr;
+	__le32 *addr;
+	u32 desc_flags = 0;
+	int ret = 0;
+
+	if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
+		return -ESHUTDOWN;
+
+	if (nbytes > ce_state->src_sz_max)
+		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
+			    __func__, nbytes, ce_state->src_sz_max);
+
+	if (unlikely(CE_RING_DELTA(nentries_mask,
+				   write_index, sw_index - 1) <= 0)) {
+		ret = -ENOSR;
+		goto exit;
+	}
+
+	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
+				      write_index);
+
+	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
+
+	if (flags & CE_SEND_FLAG_GATHER)
+		desc_flags |= CE_DESC_FLAGS_GATHER;
+
+	if (flags & CE_SEND_FLAG_BYTE_SWAP)
+		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
+
+	addr = (__le32 *)&sdesc.addr;
+
+	flags |= upper_32_bits(buffer) & CE_DESC_FLAGS_GET_MASK;
+	addr[0] = __cpu_to_le32(buffer);
+	addr[1] = __cpu_to_le32(flags);
+	if (flags & CE_SEND_FLAG_GATHER)
+		addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
+	else
+		addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
+
+	sdesc.nbytes = __cpu_to_le16(nbytes);
+	sdesc.flags  = __cpu_to_le16(desc_flags);
+
+	*desc = sdesc;
+
+	src_ring->per_transfer_context[write_index] = per_transfer_context;
+
+	/* Update Source Ring Write Index */
+	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
+
+	if (!(flags & CE_SEND_FLAG_GATHER))
+		ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
+
+	src_ring->write_index = write_index;
+exit:
+	return ret;
+}
+
+int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
+			  void *per_transfer_context,
+			  dma_addr_t buffer,
+			  unsigned int nbytes,
+			  unsigned int transfer_id,
+			  unsigned int flags)
+{
+	return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
+				    buffer, nbytes, transfer_id, flags);
+}
+
 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 {
 	struct ath10k *ar = pipe->ar;
@@ -413,7 +494,7 @@ void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 
 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 		   void *per_transfer_context,
-		   u32 buffer,
+		   dma_addr_t buffer,
 		   unsigned int nbytes,
 		   unsigned int transfer_id,
 		   unsigned int flags)
@@ -459,7 +540,8 @@ int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 	return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 }
 
-int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
+static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
+				   dma_addr_t paddr)
 {
 	struct ath10k *ar = pipe->ar;
 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
@@ -488,6 +570,39 @@ int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
 	return 0;
 }
 
+static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
+				      void *ctx,
+				      dma_addr_t paddr)
+{
+	struct ath10k *ar = pipe->ar;
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
+	unsigned int nentries_mask = dest_ring->nentries_mask;
+	unsigned int write_index = dest_ring->write_index;
+	unsigned int sw_index = dest_ring->sw_index;
+	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
+	struct ce_desc_64 *desc =
+			CE_DEST_RING_TO_DESC_64(base, write_index);
+	u32 ctrl_addr = pipe->ctrl_addr;
+
+	lockdep_assert_held(&ce->ce_lock);
+
+	if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
+		return -ENOSPC;
+
+	desc->addr = __cpu_to_le64(paddr);
+	desc->addr &= __cpu_to_le64(CE_DESC_37BIT_ADDR_MASK);
+
+	desc->nbytes = 0;
+
+	dest_ring->per_transfer_context[write_index] = ctx;
+	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
+	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
+	dest_ring->write_index = write_index;
+
+	return 0;
+}
+
 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 {
 	struct ath10k *ar = pipe->ar;
@@ -508,14 +623,15 @@ void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 	dest_ring->write_index = write_index;
 }
 
-int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
+int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
+			  dma_addr_t paddr)
 {
 	struct ath10k *ar = pipe->ar;
 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	int ret;
 
 	spin_lock_bh(&ce->ce_lock);
-	ret = __ath10k_ce_rx_post_buf(pipe, ctx, paddr);
+	ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
 	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
@@ -525,9 +641,10 @@ int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
  * Guts of ath10k_ce_completed_recv_next.
  * The caller takes responsibility for any necessary locking.
  */
-int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
-					 void **per_transfer_contextp,
-					 unsigned int *nbytesp)
+static int
+	 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
+					       void **per_transfer_contextp,
+					       unsigned int *nbytesp)
 {
 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 	unsigned int nentries_mask = dest_ring->nentries_mask;
@@ -574,6 +691,64 @@ int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 	return 0;
 }
 
+static int
+_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
+					 void **per_transfer_contextp,
+					 unsigned int *nbytesp)
+{
+	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
+	unsigned int nentries_mask = dest_ring->nentries_mask;
+	unsigned int sw_index = dest_ring->sw_index;
+	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
+	struct ce_desc_64 *desc =
+		CE_DEST_RING_TO_DESC_64(base, sw_index);
+	struct ce_desc_64 sdesc;
+	u16 nbytes;
+
+	/* Copy in one go for performance reasons */
+	sdesc = *desc;
+
+	nbytes = __le16_to_cpu(sdesc.nbytes);
+	if (nbytes == 0) {
+		/* This closes a relatively unusual race where the Host
+		 * sees the updated DRRI before the update to the
+		 * corresponding descriptor has completed. We treat this
+		 * as a descriptor that is not yet done.
+		 */
+		return -EIO;
+	}
+
+	desc->nbytes = 0;
+
+	/* Return data from completed destination descriptor */
+	*nbytesp = nbytes;
+
+	if (per_transfer_contextp)
+		*per_transfer_contextp =
+			dest_ring->per_transfer_context[sw_index];
+
+	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
+	 * So update transfer context all CEs except CE5.
+	 */
+	if (ce_state->id != 5)
+		dest_ring->per_transfer_context[sw_index] = NULL;
+
+	/* Update sw_index */
+	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
+	dest_ring->sw_index = sw_index;
+
+	return 0;
+}
+
+int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
+					 void **per_transfer_ctx,
+					 unsigned int *nbytesp)
+{
+	return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
+							    per_transfer_ctx,
+							    nbytesp);
+}
+
 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 				  void **per_transfer_contextp,
 				  unsigned int *nbytesp)
@@ -583,17 +758,18 @@ int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 	int ret;
 
 	spin_lock_bh(&ce->ce_lock);
-	ret = ath10k_ce_completed_recv_next_nolock(ce_state,
+	ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 						   per_transfer_contextp,
 						   nbytesp);
+
 	spin_unlock_bh(&ce->ce_lock);
 
 	return ret;
 }
 
-int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
-			       void **per_transfer_contextp,
-			       u32 *bufferp)
+static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
+				       void **per_transfer_contextp,
+				       dma_addr_t *bufferp)
 {
 	struct ath10k_ce_ring *dest_ring;
 	unsigned int nentries_mask;
@@ -644,6 +820,69 @@ int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 	return ret;
 }
 
+static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
+					  void **per_transfer_contextp,
+					  dma_addr_t *bufferp)
+{
+	struct ath10k_ce_ring *dest_ring;
+	unsigned int nentries_mask;
+	unsigned int sw_index;
+	unsigned int write_index;
+	int ret;
+	struct ath10k *ar;
+	struct ath10k_ce *ce;
+
+	dest_ring = ce_state->dest_ring;
+
+	if (!dest_ring)
+		return -EIO;
+
+	ar = ce_state->ar;
+	ce = ath10k_ce_priv(ar);
+
+	spin_lock_bh(&ce->ce_lock);
+
+	nentries_mask = dest_ring->nentries_mask;
+	sw_index = dest_ring->sw_index;
+	write_index = dest_ring->write_index;
+	if (write_index != sw_index) {
+		struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
+		struct ce_desc_64 *desc =
+			CE_DEST_RING_TO_DESC_64(base, sw_index);
+
+		/* Return data from completed destination descriptor */
+		*bufferp = __le64_to_cpu(desc->addr);
+
+		if (per_transfer_contextp)
+			*per_transfer_contextp =
+				dest_ring->per_transfer_context[sw_index];
+
+		/* sanity */
+		dest_ring->per_transfer_context[sw_index] = NULL;
+		desc->nbytes = 0;
+
+		/* Update sw_index */
+		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
+		dest_ring->sw_index = sw_index;
+		ret = 0;
+	} else {
+		ret = -EIO;
+	}
+
+	spin_unlock_bh(&ce->ce_lock);
+
+	return ret;
+}
+
+int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
+			       void **per_transfer_contextp,
+			       dma_addr_t *bufferp)
+{
+	return ce_state->ops->ce_revoke_recv_next(ce_state,
+						  per_transfer_contextp,
+						  bufferp);
+}
+
 /*
  * Guts of ath10k_ce_completed_send_next.
  * The caller takes responsibility for any necessary locking.
@@ -698,10 +937,45 @@ int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
 	return 0;
 }
 
+static void ath10k_ce_extract_desc_data(struct ath10k *ar,
+					struct ath10k_ce_ring *src_ring,
+					u32 sw_index,
+					dma_addr_t *bufferp,
+					u32 *nbytesp,
+					u32 *transfer_idp)
+{
+		struct ce_desc *base = src_ring->base_addr_owner_space;
+		struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
+
+		/* Return data from completed source descriptor */
+		*bufferp = __le32_to_cpu(desc->addr);
+		*nbytesp = __le16_to_cpu(desc->nbytes);
+		*transfer_idp = MS(__le16_to_cpu(desc->flags),
+				   CE_DESC_FLAGS_META_DATA);
+}
+
+static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
+					   struct ath10k_ce_ring *src_ring,
+					   u32 sw_index,
+					   dma_addr_t *bufferp,
+					   u32 *nbytesp,
+					   u32 *transfer_idp)
+{
+		struct ce_desc_64 *base = src_ring->base_addr_owner_space;
+		struct ce_desc_64 *desc =
+			CE_SRC_RING_TO_DESC_64(base, sw_index);
+
+		/* Return data from completed source descriptor */
+		*bufferp = __le64_to_cpu(desc->addr);
+		*nbytesp = __le16_to_cpu(desc->nbytes);
+		*transfer_idp = MS(__le16_to_cpu(desc->flags),
+				   CE_DESC_FLAGS_META_DATA);
+}
+
 /* NB: Modeled after ath10k_ce_completed_send_next */
 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
 			       void **per_transfer_contextp,
-			       u32 *bufferp,
+			       dma_addr_t *bufferp,
 			       unsigned int *nbytesp,
 			       unsigned int *transfer_idp)
 {
@@ -728,14 +1002,9 @@ int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
 	write_index = src_ring->write_index;
 
 	if (write_index != sw_index) {
-		struct ce_desc *base = src_ring->base_addr_owner_space;
-		struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
-
-		/* Return data from completed source descriptor */
-		*bufferp = __le32_to_cpu(desc->addr);
-		*nbytesp = __le16_to_cpu(desc->nbytes);
-		*transfer_idp = MS(__le16_to_cpu(desc->flags),
-						CE_DESC_FLAGS_META_DATA);
+		ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
+						    bufferp, nbytesp,
+						    transfer_idp);
 
 		if (per_transfer_contextp)
 			*per_transfer_contextp =
@@ -897,8 +1166,12 @@ static int ath10k_ce_init_src_ring(struct ath10k *ar,
 
 	nentries = roundup_pow_of_two(attr->src_nentries);
 
-	memset(src_ring->base_addr_owner_space, 0,
-	       nentries * sizeof(struct ce_desc));
+	if (ar->hw_params.target_64bit)
+		memset(src_ring->base_addr_owner_space, 0,
+		       nentries * sizeof(struct ce_desc_64));
+	else
+		memset(src_ring->base_addr_owner_space, 0,
+		       nentries * sizeof(struct ce_desc));
 
 	src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
 	src_ring->sw_index &= src_ring->nentries_mask;
@@ -934,8 +1207,12 @@ static int ath10k_ce_init_dest_ring(struct ath10k *ar,
 
 	nentries = roundup_pow_of_two(attr->dest_nentries);
 
-	memset(dest_ring->base_addr_owner_space, 0,
-	       nentries * sizeof(struct ce_desc));
+	if (ar->hw_params.target_64bit)
+		memset(dest_ring->base_addr_owner_space, 0,
+		       nentries * sizeof(struct ce_desc_64));
+	else
+		memset(dest_ring->base_addr_owner_space, 0,
+		       nentries * sizeof(struct ce_desc));
 
 	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
 	dest_ring->sw_index &= dest_ring->nentries_mask;
@@ -993,12 +1270,57 @@ ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
 
 	src_ring->base_addr_ce_space_unaligned = base_addr;
 
-	src_ring->base_addr_owner_space = PTR_ALIGN(
-			src_ring->base_addr_owner_space_unaligned,
-			CE_DESC_RING_ALIGN);
-	src_ring->base_addr_ce_space = ALIGN(
-			src_ring->base_addr_ce_space_unaligned,
-			CE_DESC_RING_ALIGN);
+	src_ring->base_addr_owner_space =
+			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
+				  CE_DESC_RING_ALIGN);
+	src_ring->base_addr_ce_space =
+			ALIGN(src_ring->base_addr_ce_space_unaligned,
+			      CE_DESC_RING_ALIGN);
+
+	return src_ring;
+}
+
+static struct ath10k_ce_ring *
+ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
+			    const struct ce_attr *attr)
+{
+	struct ath10k_ce_ring *src_ring;
+	u32 nentries = attr->src_nentries;
+	dma_addr_t base_addr;
+
+	nentries = roundup_pow_of_two(nentries);
+
+	src_ring = kzalloc(sizeof(*src_ring) +
+			   (nentries *
+			    sizeof(*src_ring->per_transfer_context)),
+			   GFP_KERNEL);
+	if (!src_ring)
+		return ERR_PTR(-ENOMEM);
+
+	src_ring->nentries = nentries;
+	src_ring->nentries_mask = nentries - 1;
+
+	/* Legacy platforms that do not support cache
+	 * coherent DMA are unsupported
+	 */
+	src_ring->base_addr_owner_space_unaligned =
+		dma_alloc_coherent(ar->dev,
+				   (nentries * sizeof(struct ce_desc_64) +
+				    CE_DESC_RING_ALIGN),
+				   &base_addr, GFP_KERNEL);
+	if (!src_ring->base_addr_owner_space_unaligned) {
+		kfree(src_ring);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	src_ring->base_addr_ce_space_unaligned = base_addr;
+
+	src_ring->base_addr_owner_space =
+			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
+				  CE_DESC_RING_ALIGN);
+	src_ring->base_addr_ce_space =
+			ALIGN(src_ring->base_addr_ce_space_unaligned,
+			      CE_DESC_RING_ALIGN);
 
 	return src_ring;
 }
@@ -1039,12 +1361,63 @@ ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
 
 	dest_ring->base_addr_ce_space_unaligned = base_addr;
 
-	dest_ring->base_addr_owner_space = PTR_ALIGN(
-			dest_ring->base_addr_owner_space_unaligned,
-			CE_DESC_RING_ALIGN);
-	dest_ring->base_addr_ce_space = ALIGN(
-			dest_ring->base_addr_ce_space_unaligned,
-			CE_DESC_RING_ALIGN);
+	dest_ring->base_addr_owner_space =
+			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
+				  CE_DESC_RING_ALIGN);
+	dest_ring->base_addr_ce_space =
+				ALIGN(dest_ring->base_addr_ce_space_unaligned,
+				      CE_DESC_RING_ALIGN);
+
+	return dest_ring;
+}
+
+static struct ath10k_ce_ring *
+ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
+			     const struct ce_attr *attr)
+{
+	struct ath10k_ce_ring *dest_ring;
+	u32 nentries;
+	dma_addr_t base_addr;
+
+	nentries = roundup_pow_of_two(attr->dest_nentries);
+
+	dest_ring = kzalloc(sizeof(*dest_ring) +
+			    (nentries *
+			     sizeof(*dest_ring->per_transfer_context)),
+			    GFP_KERNEL);
+	if (!dest_ring)
+		return ERR_PTR(-ENOMEM);
+
+	dest_ring->nentries = nentries;
+	dest_ring->nentries_mask = nentries - 1;
+
+	/* Legacy platforms that do not support cache
+	 * coherent DMA are unsupported
+	 */
+	dest_ring->base_addr_owner_space_unaligned =
+		dma_alloc_coherent(ar->dev,
+				   (nentries * sizeof(struct ce_desc_64) +
+				    CE_DESC_RING_ALIGN),
+				   &base_addr, GFP_KERNEL);
+	if (!dest_ring->base_addr_owner_space_unaligned) {
+		kfree(dest_ring);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	dest_ring->base_addr_ce_space_unaligned = base_addr;
+
+	/* Correctly initialize memory to 0 to prevent garbage
+	 * data crashing system when download firmware
+	 */
+	memset(dest_ring->base_addr_owner_space_unaligned, 0,
+	       nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN);
+
+	dest_ring->base_addr_owner_space =
+			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
+				  CE_DESC_RING_ALIGN);
+	dest_ring->base_addr_ce_space =
+			ALIGN(dest_ring->base_addr_ce_space_unaligned,
+			      CE_DESC_RING_ALIGN);
 
 	return dest_ring;
 }
@@ -1107,65 +1480,36 @@ void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
 	ath10k_ce_deinit_dest_ring(ar, ce_id);
 }
 
-int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
-			 const struct ce_attr *attr)
+static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 {
 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
-	int ret;
-
-	/*
-	 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid
-	 * additional TX locking checks.
-	 *
-	 * For the lack of a better place do the check here.
-	 */
-	BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
-		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
-	BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
-		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
-	BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
-		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
-
-	ce_state->ar = ar;
-	ce_state->id = ce_id;
-	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
-	ce_state->attr_flags = attr->flags;
-	ce_state->src_sz_max = attr->src_sz_max;
 
-	if (attr->src_nentries)
-		ce_state->send_cb = attr->send_cb;
-
-	if (attr->dest_nentries)
-		ce_state->recv_cb = attr->recv_cb;
-
-	if (attr->src_nentries) {
-		ce_state->src_ring = ath10k_ce_alloc_src_ring(ar, ce_id, attr);
-		if (IS_ERR(ce_state->src_ring)) {
-			ret = PTR_ERR(ce_state->src_ring);
-			ath10k_err(ar, "failed to allocate copy engine source ring %d: %d\n",
-				   ce_id, ret);
-			ce_state->src_ring = NULL;
-			return ret;
-		}
+	if (ce_state->src_ring) {
+		dma_free_coherent(ar->dev,
+				  (ce_state->src_ring->nentries *
+				   sizeof(struct ce_desc) +
+				   CE_DESC_RING_ALIGN),
+				  ce_state->src_ring->base_addr_owner_space,
+				  ce_state->src_ring->base_addr_ce_space);
+		kfree(ce_state->src_ring);
 	}
 
-	if (attr->dest_nentries) {
-		ce_state->dest_ring = ath10k_ce_alloc_dest_ring(ar, ce_id,
-								attr);
-		if (IS_ERR(ce_state->dest_ring)) {
-			ret = PTR_ERR(ce_state->dest_ring);
-			ath10k_err(ar, "failed to allocate copy engine destination ring %d: %d\n",
-				   ce_id, ret);
-			ce_state->dest_ring = NULL;
-			return ret;
-		}
+	if (ce_state->dest_ring) {
+		dma_free_coherent(ar->dev,
+				  (ce_state->dest_ring->nentries *
+				   sizeof(struct ce_desc) +
+				   CE_DESC_RING_ALIGN),
+				  ce_state->dest_ring->base_addr_owner_space,
+				  ce_state->dest_ring->base_addr_ce_space);
+		kfree(ce_state->dest_ring);
 	}
 
-	return 0;
+	ce_state->src_ring = NULL;
+	ce_state->dest_ring = NULL;
 }
 
-void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
+static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
 {
 	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
@@ -1173,7 +1517,7 @@ void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 	if (ce_state->src_ring) {
 		dma_free_coherent(ar->dev,
 				  (ce_state->src_ring->nentries *
-				   sizeof(struct ce_desc) +
+				   sizeof(struct ce_desc_64) +
 				   CE_DESC_RING_ALIGN),
 				  ce_state->src_ring->base_addr_owner_space,
 				  ce_state->src_ring->base_addr_ce_space);
@@ -1183,7 +1527,7 @@ void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 	if (ce_state->dest_ring) {
 		dma_free_coherent(ar->dev,
 				  (ce_state->dest_ring->nentries *
-				   sizeof(struct ce_desc) +
+				   sizeof(struct ce_desc_64) +
 				   CE_DESC_RING_ALIGN),
 				  ce_state->dest_ring->base_addr_owner_space,
 				  ce_state->dest_ring->base_addr_ce_space);
@@ -1194,6 +1538,14 @@ void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
 	ce_state->dest_ring = NULL;
 }
 
+void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
+
+	ce_state->ops->ce_free_pipe(ar, ce_id);
+}
+
 void ath10k_ce_dump_registers(struct ath10k *ar,
 			      struct ath10k_fw_crash_data *crash_data)
 {
@@ -1232,3 +1584,99 @@ void ath10k_ce_dump_registers(struct ath10k *ar,
 
 	spin_unlock_bh(&ce->ce_lock);
 }
+
+static const struct ath10k_ce_ops ce_ops = {
+	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
+	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
+	.ce_rx_post_buf = __ath10k_ce_rx_post_buf,
+	.ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
+	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
+	.ce_extract_desc_data = ath10k_ce_extract_desc_data,
+	.ce_free_pipe = _ath10k_ce_free_pipe,
+	.ce_send_nolock = _ath10k_ce_send_nolock,
+};
+
+static const struct ath10k_ce_ops ce_64_ops = {
+	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
+	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
+	.ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
+	.ce_completed_recv_next_nolock =
+				_ath10k_ce_completed_recv_next_nolock_64,
+	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
+	.ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
+	.ce_free_pipe = _ath10k_ce_free_pipe_64,
+	.ce_send_nolock = _ath10k_ce_send_nolock_64,
+};
+
+static void ath10k_ce_set_ops(struct ath10k *ar,
+			      struct ath10k_ce_pipe *ce_state)
+{
+	switch (ar->hw_rev) {
+	case ATH10K_HW_WCN3990:
+		ce_state->ops = &ce_64_ops;
+		break;
+	default:
+		ce_state->ops = &ce_ops;
+		break;
+	}
+}
+
+int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
+			 const struct ce_attr *attr)
+{
+	struct ath10k_ce *ce = ath10k_ce_priv(ar);
+	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
+	int ret;
+
+	ath10k_ce_set_ops(ar, ce_state);
+	/* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
+	 * additional TX locking checks.
+	 *
+	 * For the lack of a better place do the check here.
+	 */
+	BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
+		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
+	BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
+		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
+	BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
+		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
+
+	ce_state->ar = ar;
+	ce_state->id = ce_id;
+	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
+	ce_state->attr_flags = attr->flags;
+	ce_state->src_sz_max = attr->src_sz_max;
+
+	if (attr->src_nentries)
+		ce_state->send_cb = attr->send_cb;
+
+	if (attr->dest_nentries)
+		ce_state->recv_cb = attr->recv_cb;
+
+	if (attr->src_nentries) {
+		ce_state->src_ring =
+			ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
+		if (IS_ERR(ce_state->src_ring)) {
+			ret = PTR_ERR(ce_state->src_ring);
+			ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
+				   ce_id, ret);
+			ce_state->src_ring = NULL;
+			return ret;
+		}
+	}
+
+	if (attr->dest_nentries) {
+		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
+									ce_id,
+									attr);
+		if (IS_ERR(ce_state->dest_ring)) {
+			ret = PTR_ERR(ce_state->dest_ring);
+			ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
+				   ce_id, ret);
+			ce_state->dest_ring = NULL;
+			return ret;
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/net/wireless/ath/ath10k/ce.h b/drivers/net/wireless/ath/ath10k/ce.h
index bdec794704d9..2c3c8f5e90ea 100644
--- a/drivers/net/wireless/ath/ath10k/ce.h
+++ b/drivers/net/wireless/ath/ath10k/ce.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -36,6 +36,10 @@ struct ath10k_ce_pipe;
 
 #define CE_DESC_FLAGS_GATHER         (1 << 0)
 #define CE_DESC_FLAGS_BYTE_SWAP      (1 << 1)
+#define CE_WCN3990_DESC_FLAGS_GATHER BIT(31)
+
+#define CE_DESC_FLAGS_GET_MASK		GENMASK(4, 0)
+#define CE_DESC_37BIT_ADDR_MASK		GENMASK_ULL(37, 0)
 
 /* Following desc flags are used in QCA99X0 */
 #define CE_DESC_FLAGS_HOST_INT_DIS	(1 << 2)
@@ -50,6 +54,16 @@ struct ce_desc {
 	__le16 flags; /* %CE_DESC_FLAGS_ */
 };
 
+struct ce_desc_64 {
+	__le64 addr;
+	__le16 nbytes; /* length in register map */
+	__le16 flags; /* fw_metadata_high */
+	__le32 toeplitz_hash_result;
+};
+
+#define CE_DESC_SIZE sizeof(struct ce_desc)
+#define CE_DESC_SIZE_64 sizeof(struct ce_desc_64)
+
 struct ath10k_ce_ring {
 	/* Number of entries in this ring; must be power of 2 */
 	unsigned int nentries;
@@ -117,6 +131,7 @@ struct ath10k_ce_pipe {
 	unsigned int src_sz_max;
 	struct ath10k_ce_ring *src_ring;
 	struct ath10k_ce_ring *dest_ring;
+	const struct ath10k_ce_ops *ops;
 };
 
 /* Copy Engine settable attributes */
@@ -160,7 +175,7 @@ struct ath10k_ce {
  */
 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 		   void *per_transfer_send_context,
-		   u32 buffer,
+		   dma_addr_t buffer,
 		   unsigned int nbytes,
 		   /* 14 bits */
 		   unsigned int transfer_id,
@@ -168,7 +183,7 @@ int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 
 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 			  void *per_transfer_context,
-			  u32 buffer,
+			  dma_addr_t buffer,
 			  unsigned int nbytes,
 			  unsigned int transfer_id,
 			  unsigned int flags);
@@ -180,8 +195,8 @@ int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe);
 /*==================Recv=======================*/
 
 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe);
-int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr);
-int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr);
+int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
+			  dma_addr_t paddr);
 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries);
 
 /* recv flags */
@@ -222,7 +237,7 @@ void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id);
  */
 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 			       void **per_transfer_contextp,
-			       u32 *bufferp);
+			       dma_addr_t *bufferp);
 
 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 					 void **per_transfer_contextp,
@@ -235,7 +250,7 @@ int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
  */
 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
 			       void **per_transfer_contextp,
-			       u32 *bufferp,
+			       dma_addr_t *bufferp,
 			       unsigned int *nbytesp,
 			       unsigned int *transfer_idp);
 
@@ -281,6 +296,32 @@ struct ce_attr {
 	void (*recv_cb)(struct ath10k_ce_pipe *);
 };
 
+struct ath10k_ce_ops {
+	struct ath10k_ce_ring *(*ce_alloc_src_ring)(struct ath10k *ar,
+						    u32 ce_id,
+						    const struct ce_attr *attr);
+	struct ath10k_ce_ring *(*ce_alloc_dst_ring)(struct ath10k *ar,
+						    u32 ce_id,
+						    const struct ce_attr *attr);
+	int (*ce_rx_post_buf)(struct ath10k_ce_pipe *pipe, void *ctx,
+			      dma_addr_t paddr);
+	int (*ce_completed_recv_next_nolock)(struct ath10k_ce_pipe *ce_state,
+					     void **per_transfer_contextp,
+					     u32 *nbytesp);
+	int (*ce_revoke_recv_next)(struct ath10k_ce_pipe *ce_state,
+				   void **per_transfer_contextp,
+				   dma_addr_t *nbytesp);
+	void (*ce_extract_desc_data)(struct ath10k *ar,
+				     struct ath10k_ce_ring *src_ring,
+				     u32 sw_index, dma_addr_t *bufferp,
+				     u32 *nbytesp, u32 *transfer_idp);
+	void (*ce_free_pipe)(struct ath10k *ar, int ce_id);
+	int (*ce_send_nolock)(struct ath10k_ce_pipe *pipe,
+			      void *per_transfer_context,
+			      dma_addr_t buffer, u32 nbytes,
+			      u32 transfer_id, u32 flags);
+};
+
 static inline u32 ath10k_ce_base_address(struct ath10k *ar, unsigned int ce_id)
 {
 	return CE0_BASE_ADDRESS + (CE1_BASE_ADDRESS - CE0_BASE_ADDRESS) * ce_id;
@@ -292,6 +333,12 @@ static inline u32 ath10k_ce_base_address(struct ath10k *ar, unsigned int ce_id)
 #define CE_DEST_RING_TO_DESC(baddr, idx) \
 	(&(((struct ce_desc *)baddr)[idx]))
 
+#define CE_SRC_RING_TO_DESC_64(baddr, idx) \
+	(&(((struct ce_desc_64 *)baddr)[idx]))
+
+#define CE_DEST_RING_TO_DESC_64(baddr, idx) \
+	(&(((struct ce_desc_64 *)baddr)[idx]))
+
 /* Ring arithmetic (modulus number of entries in ring, which is a pwr of 2). */
 #define CE_RING_DELTA(nentries_mask, fromidx, toidx) \
 	(((int)(toidx) - (int)(fromidx)) & (nentries_mask))
diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c
index b29fdbd21ead..f3ec13b80b20 100644
--- a/drivers/net/wireless/ath/ath10k/core.c
+++ b/drivers/net/wireless/ath/ath10k/core.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -32,6 +32,7 @@
 #include "htt.h"
 #include "testmode.h"
 #include "wmi-ops.h"
+#include "coredump.h"
 
 unsigned int ath10k_debug_mask;
 static unsigned int ath10k_cryptmode_param;
@@ -39,17 +40,25 @@ static bool uart_print;
 static bool skip_otp;
 static bool rawmode;
 
+/* Enable ATH10K_FW_CRASH_DUMP_REGISTERS and ATH10K_FW_CRASH_DUMP_CE_DATA
+ * by default.
+ */
+unsigned long ath10k_coredump_mask = 0x3;
+
+/* FIXME: most of these should be readonly */
 module_param_named(debug_mask, ath10k_debug_mask, uint, 0644);
 module_param_named(cryptmode, ath10k_cryptmode_param, uint, 0644);
 module_param(uart_print, bool, 0644);
 module_param(skip_otp, bool, 0644);
 module_param(rawmode, bool, 0644);
+module_param_named(coredump_mask, ath10k_coredump_mask, ulong, 0444);
 
 MODULE_PARM_DESC(debug_mask, "Debugging mask");
 MODULE_PARM_DESC(uart_print, "Uart target debugging");
 MODULE_PARM_DESC(skip_otp, "Skip otp failure for calibration in testmode");
 MODULE_PARM_DESC(cryptmode, "Crypto mode: 0-hardware, 1-software");
 MODULE_PARM_DESC(rawmode, "Use raw 802.11 frame datapath");
+MODULE_PARM_DESC(coredump_mask, "Bitfield of what to include in firmware crash file");
 
 static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 	{
@@ -75,6 +84,40 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+	},
+	{
+		.id = QCA988X_HW_2_0_VERSION,
+		.dev_id = QCA988X_2_0_DEVICE_ID_UBNT,
+		.name = "qca988x hw2.0 ubiquiti",
+		.patch_load_addr = QCA988X_HW_2_0_PATCH_LOAD_ADDR,
+		.uart_pin = 7,
+		.cc_wraparound_type = ATH10K_HW_CC_WRAP_SHIFTED_ALL,
+		.otp_exe_param = 0,
+		.channel_counters_freq_hz = 88000,
+		.max_probe_resp_desc_thres = 0,
+		.cal_data_len = 2116,
+		.fw = {
+			.dir = QCA988X_HW_2_0_FW_DIR,
+			.board = QCA988X_HW_2_0_BOARD_DATA_FILE,
+			.board_size = QCA988X_BOARD_DATA_SZ,
+			.board_ext_size = QCA988X_BOARD_EXT_DATA_SZ,
+		},
+		.hw_ops = &qca988x_ops,
+		.decap_align_bytes = 4,
+		.spectral_bin_discard = 0,
+		.vht160_mcs_rx_highest = 0,
+		.vht160_mcs_tx_highest = 0,
+		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA9887_HW_1_0_VERSION,
@@ -99,6 +142,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -122,6 +170,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -145,6 +198,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA6174_HW_3_0_VERSION,
@@ -168,6 +226,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA6174_HW_3_2_VERSION,
@@ -194,6 +257,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA99X0_HW_2_0_DEV_VERSION,
@@ -223,6 +291,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 11,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA9984_HW_1_0_DEV_VERSION,
@@ -257,6 +330,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 1560,
 		.vht160_mcs_tx_highest = 1560,
 		.n_cipher_suites = 11,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA9888_HW_2_0_DEV_VERSION,
@@ -290,6 +368,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 780,
 		.vht160_mcs_tx_highest = 780,
 		.n_cipher_suites = 11,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA9377_HW_1_0_DEV_VERSION,
@@ -313,6 +396,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA9377_HW_1_1_DEV_VERSION,
@@ -338,6 +426,11 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 8,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
 	},
 	{
 		.id = QCA4019_HW_1_0_DEV_VERSION,
@@ -368,6 +461,31 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.vht160_mcs_rx_highest = 0,
 		.vht160_mcs_tx_highest = 0,
 		.n_cipher_suites = 11,
+		.num_peers = TARGET_TLV_NUM_PEERS,
+		.ast_skid_limit = 0x10,
+		.num_wds_entries = 0x20,
+		.target_64bit = false,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL,
+	},
+	{
+		.id = WCN3990_HW_1_0_DEV_VERSION,
+		.dev_id = 0,
+		.name = "wcn3990 hw1.0",
+		.continuous_frag_desc = true,
+		.tx_chain_mask = 0x7,
+		.rx_chain_mask = 0x7,
+		.max_spatial_stream = 4,
+		.fw = {
+			.dir = WCN3990_HW_1_0_FW_DIR,
+		},
+		.sw_decrypt_mcast_mgmt = true,
+		.hw_ops = &wcn3990_ops,
+		.decap_align_bytes = 1,
+		.num_peers = TARGET_HL_10_TLV_NUM_PEERS,
+		.ast_skid_limit = TARGET_HL_10_TLV_AST_SKID_LIMIT,
+		.num_wds_entries = TARGET_HL_10_TLV_NUM_WDS_ENTRIES,
+		.target_64bit = true,
+		.rx_ring_fill_level = HTT_RX_RING_FILL_LEVEL_DUAL_MAC,
 	},
 };
 
@@ -390,6 +508,8 @@ static const char *const ath10k_core_fw_feature_str[] = {
 	[ATH10K_FW_FEATURE_SKIP_NULL_FUNC_WAR] = "skip-null-func-war",
 	[ATH10K_FW_FEATURE_ALLOWS_MESH_BCAST] = "allows-mesh-bcast",
 	[ATH10K_FW_FEATURE_NO_PS] = "no-ps",
+	[ATH10K_FW_FEATURE_MGMT_TX_BY_REF] = "mgmt-tx-by-reference",
+	[ATH10K_FW_FEATURE_NON_BMI] = "non-bmi",
 };
 
 static unsigned int ath10k_core_get_fw_feature_str(char *buf,
@@ -860,6 +980,28 @@ static int ath10k_core_check_smbios(struct ath10k *ar)
 	return 0;
 }
 
+static int ath10k_core_check_dt(struct ath10k *ar)
+{
+	struct device_node *node;
+	const char *variant = NULL;
+
+	node = ar->dev->of_node;
+	if (!node)
+		return -ENOENT;
+
+	of_property_read_string(node, "qcom,ath10k-calibration-variant",
+				&variant);
+	if (!variant)
+		return -ENODATA;
+
+	if (strscpy(ar->id.bdf_ext, variant, sizeof(ar->id.bdf_ext)) < 0)
+		ath10k_dbg(ar, ATH10K_DBG_BOOT,
+			   "bdf variant string is longer than the buffer can accommodate (variant: %s)\n",
+			    variant);
+
+	return 0;
+}
+
 static int ath10k_download_and_run_otp(struct ath10k *ar)
 {
 	u32 result, address = ar->hw_params.patch_load_addr;
@@ -1231,19 +1373,19 @@ static int ath10k_core_create_board_name(struct ath10k *ar, char *name,
 	/* strlen(',variant=') + strlen(ar->id.bdf_ext) */
 	char variant[9 + ATH10K_SMBIOS_BDF_EXT_STR_LENGTH] = { 0 };
 
+	if (ar->id.bdf_ext[0] != '\0')
+		scnprintf(variant, sizeof(variant), ",variant=%s",
+			  ar->id.bdf_ext);
+
 	if (ar->id.bmi_ids_valid) {
 		scnprintf(name, name_len,
-			  "bus=%s,bmi-chip-id=%d,bmi-board-id=%d",
+			  "bus=%s,bmi-chip-id=%d,bmi-board-id=%d%s",
 			  ath10k_bus_str(ar->hif.bus),
 			  ar->id.bmi_chip_id,
-			  ar->id.bmi_board_id);
+			  ar->id.bmi_board_id, variant);
 		goto out;
 	}
 
-	if (ar->id.bdf_ext[0] != '\0')
-		scnprintf(variant, sizeof(variant), ",variant=%s",
-			  ar->id.bdf_ext);
-
 	scnprintf(name, name_len,
 		  "bus=%s,vendor=%04x,device=%04x,subsystem-vendor=%04x,subsystem-device=%04x%s",
 		  ath10k_bus_str(ar->hif.bus),
@@ -1447,8 +1589,8 @@ int ath10k_core_fetch_firmware_api_n(struct ath10k *ar, const char *name,
 		data += ie_len;
 	}
 
-	if (!fw_file->firmware_data ||
-	    !fw_file->firmware_len) {
+	if (!test_bit(ATH10K_FW_FEATURE_NON_BMI, fw_file->fw_features) &&
+	    (!fw_file->firmware_data || !fw_file->firmware_len)) {
 		ath10k_warn(ar, "No ATH10K_FW_IE_FW_IMAGE found from '%s/%s', skipping\n",
 			    ar->hw_params.fw.dir, name);
 		ret = -ENOMEDIUM;
@@ -1474,6 +1616,7 @@ static void ath10k_core_get_fw_name(struct ath10k *ar, char *fw_name,
 		break;
 	case ATH10K_BUS_PCI:
 	case ATH10K_BUS_AHB:
+	case ATH10K_BUS_SNOC:
 		scnprintf(fw_name, fw_name_len, "%s-%d.bin",
 			  ATH10K_FW_FILE_BASE, fw_api);
 		break;
@@ -1759,7 +1902,7 @@ static void ath10k_core_restart(struct work_struct *work)
 
 	mutex_unlock(&ar->conf_mutex);
 
-	ret = ath10k_debug_fw_devcoredump(ar);
+	ret = ath10k_coredump_submit(ar);
 	if (ret)
 		ath10k_warn(ar, "failed to send firmware crash dump via devcoredump: %d",
 			    ret);
@@ -2001,43 +2144,47 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 
 	ar->running_fw = fw;
 
-	ath10k_bmi_start(ar);
-
-	if (ath10k_init_configure_target(ar)) {
-		status = -EINVAL;
-		goto err;
-	}
+	if (!test_bit(ATH10K_FW_FEATURE_NON_BMI,
+		      ar->running_fw->fw_file.fw_features)) {
+		ath10k_bmi_start(ar);
 
-	status = ath10k_download_cal_data(ar);
-	if (status)
-		goto err;
+		if (ath10k_init_configure_target(ar)) {
+			status = -EINVAL;
+			goto err;
+		}
 
-	/* Some of of qca988x solutions are having global reset issue
-	 * during target initialization. Bypassing PLL setting before
-	 * downloading firmware and letting the SoC run on REF_CLK is
-	 * fixing the problem. Corresponding firmware change is also needed
-	 * to set the clock source once the target is initialized.
-	 */
-	if (test_bit(ATH10K_FW_FEATURE_SUPPORTS_SKIP_CLOCK_INIT,
-		     ar->running_fw->fw_file.fw_features)) {
-		status = ath10k_bmi_write32(ar, hi_skip_clock_init, 1);
-		if (status) {
-			ath10k_err(ar, "could not write to skip_clock_init: %d\n",
-				   status);
+		status = ath10k_download_cal_data(ar);
+		if (status)
 			goto err;
+
+		/* Some of of qca988x solutions are having global reset issue
+		 * during target initialization. Bypassing PLL setting before
+		 * downloading firmware and letting the SoC run on REF_CLK is
+		 * fixing the problem. Corresponding firmware change is also
+		 * needed to set the clock source once the target is
+		 * initialized.
+		 */
+		if (test_bit(ATH10K_FW_FEATURE_SUPPORTS_SKIP_CLOCK_INIT,
+			     ar->running_fw->fw_file.fw_features)) {
+			status = ath10k_bmi_write32(ar, hi_skip_clock_init, 1);
+			if (status) {
+				ath10k_err(ar, "could not write to skip_clock_init: %d\n",
+					   status);
+				goto err;
+			}
 		}
-	}
 
-	status = ath10k_download_fw(ar);
-	if (status)
-		goto err;
+		status = ath10k_download_fw(ar);
+		if (status)
+			goto err;
 
-	status = ath10k_init_uart(ar);
-	if (status)
-		goto err;
+		status = ath10k_init_uart(ar);
+		if (status)
+			goto err;
 
-	if (ar->hif.bus == ATH10K_BUS_SDIO)
-		ath10k_init_sdio(ar);
+		if (ar->hif.bus == ATH10K_BUS_SDIO)
+			ath10k_init_sdio(ar);
+	}
 
 	ar->htc.htc_ops.target_send_suspend_complete =
 		ath10k_send_suspend_complete;
@@ -2048,9 +2195,12 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 		goto err;
 	}
 
-	status = ath10k_bmi_done(ar);
-	if (status)
-		goto err;
+	if (!test_bit(ATH10K_FW_FEATURE_NON_BMI,
+		      ar->running_fw->fw_file.fw_features)) {
+		status = ath10k_bmi_done(ar);
+		if (status)
+			goto err;
+	}
 
 	status = ath10k_wmi_attach(ar);
 	if (status) {
@@ -2293,19 +2443,35 @@ static int ath10k_core_probe_fw(struct ath10k *ar)
 		return ret;
 	}
 
-	memset(&target_info, 0, sizeof(target_info));
-	if (ar->hif.bus == ATH10K_BUS_SDIO)
+	switch (ar->hif.bus) {
+	case ATH10K_BUS_SDIO:
+		memset(&target_info, 0, sizeof(target_info));
 		ret = ath10k_bmi_get_target_info_sdio(ar, &target_info);
-	else
+		if (ret) {
+			ath10k_err(ar, "could not get target info (%d)\n", ret);
+			goto err_power_down;
+		}
+		ar->target_version = target_info.version;
+		ar->hw->wiphy->hw_version = target_info.version;
+		break;
+	case ATH10K_BUS_PCI:
+	case ATH10K_BUS_AHB:
+	case ATH10K_BUS_USB:
+		memset(&target_info, 0, sizeof(target_info));
 		ret = ath10k_bmi_get_target_info(ar, &target_info);
-	if (ret) {
-		ath10k_err(ar, "could not get target info (%d)\n", ret);
-		goto err_power_down;
+		if (ret) {
+			ath10k_err(ar, "could not get target info (%d)\n", ret);
+			goto err_power_down;
+		}
+		ar->target_version = target_info.version;
+		ar->hw->wiphy->hw_version = target_info.version;
+		break;
+	case ATH10K_BUS_SNOC:
+		break;
+	default:
+		ath10k_err(ar, "incorrect hif bus type: %d\n", ar->hif.bus);
 	}
 
-	ar->target_version = target_info.version;
-	ar->hw->wiphy->hw_version = target_info.version;
-
 	ret = ath10k_init_hw_params(ar);
 	if (ret) {
 		ath10k_err(ar, "could not get hw params (%d)\n", ret);
@@ -2325,33 +2491,40 @@ static int ath10k_core_probe_fw(struct ath10k *ar)
 
 	ath10k_debug_print_hwfw_info(ar);
 
-	ret = ath10k_core_pre_cal_download(ar);
-	if (ret) {
-		/* pre calibration data download is not necessary
-		 * for all the chipsets. Ignore failures and continue.
-		 */
-		ath10k_dbg(ar, ATH10K_DBG_BOOT,
-			   "could not load pre cal data: %d\n", ret);
-	}
+	if (!test_bit(ATH10K_FW_FEATURE_NON_BMI,
+		      ar->normal_mode_fw.fw_file.fw_features)) {
+		ret = ath10k_core_pre_cal_download(ar);
+		if (ret) {
+			/* pre calibration data download is not necessary
+			 * for all the chipsets. Ignore failures and continue.
+			 */
+			ath10k_dbg(ar, ATH10K_DBG_BOOT,
+				   "could not load pre cal data: %d\n", ret);
+		}
 
-	ret = ath10k_core_get_board_id_from_otp(ar);
-	if (ret && ret != -EOPNOTSUPP) {
-		ath10k_err(ar, "failed to get board id from otp: %d\n",
-			   ret);
-		goto err_free_firmware_files;
-	}
+		ret = ath10k_core_get_board_id_from_otp(ar);
+		if (ret && ret != -EOPNOTSUPP) {
+			ath10k_err(ar, "failed to get board id from otp: %d\n",
+				   ret);
+			goto err_free_firmware_files;
+		}
 
-	ret = ath10k_core_check_smbios(ar);
-	if (ret)
-		ath10k_dbg(ar, ATH10K_DBG_BOOT, "bdf variant name not set.\n");
+		ret = ath10k_core_check_smbios(ar);
+		if (ret)
+			ath10k_dbg(ar, ATH10K_DBG_BOOT, "SMBIOS bdf variant name not set.\n");
 
-	ret = ath10k_core_fetch_board_file(ar);
-	if (ret) {
-		ath10k_err(ar, "failed to fetch board file: %d\n", ret);
-		goto err_free_firmware_files;
-	}
+		ret = ath10k_core_check_dt(ar);
+		if (ret)
+			ath10k_dbg(ar, ATH10K_DBG_BOOT, "DT bdf variant name not set.\n");
+
+		ret = ath10k_core_fetch_board_file(ar);
+		if (ret) {
+			ath10k_err(ar, "failed to fetch board file: %d\n", ret);
+			goto err_free_firmware_files;
+		}
 
-	ath10k_debug_print_board_info(ar);
+		ath10k_debug_print_board_info(ar);
+	}
 
 	ret = ath10k_core_init_firmware_features(ar);
 	if (ret) {
@@ -2360,11 +2533,15 @@ static int ath10k_core_probe_fw(struct ath10k *ar)
 		goto err_free_firmware_files;
 	}
 
-	ret = ath10k_swap_code_seg_init(ar, &ar->normal_mode_fw.fw_file);
-	if (ret) {
-		ath10k_err(ar, "failed to initialize code swap segment: %d\n",
-			   ret);
-		goto err_free_firmware_files;
+	if (!test_bit(ATH10K_FW_FEATURE_NON_BMI,
+		      ar->normal_mode_fw.fw_file.fw_features)) {
+		ret = ath10k_swap_code_seg_init(ar,
+						&ar->normal_mode_fw.fw_file);
+		if (ret) {
+			ath10k_err(ar, "failed to initialize code swap segment: %d\n",
+				   ret);
+			goto err_free_firmware_files;
+		}
 	}
 
 	mutex_lock(&ar->conf_mutex);
@@ -2416,10 +2593,16 @@ static void ath10k_core_register_work(struct work_struct *work)
 		goto err_release_fw;
 	}
 
+	status = ath10k_coredump_register(ar);
+	if (status) {
+		ath10k_err(ar, "unable to register coredump\n");
+		goto err_unregister_mac;
+	}
+
 	status = ath10k_debug_register(ar);
 	if (status) {
 		ath10k_err(ar, "unable to initialize debugfs\n");
-		goto err_unregister_mac;
+		goto err_unregister_coredump;
 	}
 
 	status = ath10k_spectral_create(ar);
@@ -2442,6 +2625,8 @@ static void ath10k_core_register_work(struct work_struct *work)
 	ath10k_spectral_destroy(ar);
 err_debug_destroy:
 	ath10k_debug_destroy(ar);
+err_unregister_coredump:
+	ath10k_coredump_unregister(ar);
 err_unregister_mac:
 	ath10k_mac_unregister(ar);
 err_release_fw:
@@ -2596,12 +2781,19 @@ struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev,
 
 	init_dummy_netdev(&ar->napi_dev);
 
-	ret = ath10k_debug_create(ar);
+	ret = ath10k_coredump_create(ar);
 	if (ret)
 		goto err_free_aux_wq;
 
+	ret = ath10k_debug_create(ar);
+	if (ret)
+		goto err_free_coredump;
+
 	return ar;
 
+err_free_coredump:
+	ath10k_coredump_destroy(ar);
+
 err_free_aux_wq:
 	destroy_workqueue(ar->workqueue_aux);
 err_free_wq:
@@ -2623,6 +2815,7 @@ void ath10k_core_destroy(struct ath10k *ar)
 	destroy_workqueue(ar->workqueue_aux);
 
 	ath10k_debug_destroy(ar);
+	ath10k_coredump_destroy(ar);
 	ath10k_htt_tx_destroy(&ar->htt);
 	ath10k_wmi_free_host_mem(ar);
 	ath10k_mac_destroy(ar);
diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h
index 643041ef3271..fe6b30356d3b 100644
--- a/drivers/net/wireless/ath/ath10k/core.h
+++ b/drivers/net/wireless/ath/ath10k/core.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -67,7 +67,6 @@
 
 /* NAPI poll budget */
 #define ATH10K_NAPI_BUDGET      64
-#define ATH10K_NAPI_QUOTA_LIMIT 60
 
 /* SMBIOS type containing Board Data File Name Extension */
 #define ATH10K_SMBIOS_BDF_EXT_TYPE 0xF8
@@ -93,6 +92,7 @@ enum ath10k_bus {
 	ATH10K_BUS_AHB,
 	ATH10K_BUS_SDIO,
 	ATH10K_BUS_USB,
+	ATH10K_BUS_SNOC,
 };
 
 static inline const char *ath10k_bus_str(enum ath10k_bus bus)
@@ -106,6 +106,8 @@ static inline const char *ath10k_bus_str(enum ath10k_bus bus)
 		return "sdio";
 	case ATH10K_BUS_USB:
 		return "usb";
+	case ATH10K_BUS_SNOC:
+		return "snoc";
 	}
 
 	return "unknown";
@@ -364,11 +366,11 @@ struct ath10k_sta {
 	struct rate_info txrate;
 
 	struct work_struct update_wk;
+	u64 rx_duration;
 
 #ifdef CONFIG_MAC80211_DEBUGFS
 	/* protected by conf_mutex */
 	bool aggr_mode;
-	u64 rx_duration;
 #endif
 };
 
@@ -458,14 +460,17 @@ struct ath10k_ce_crash_hdr {
 	struct ath10k_ce_crash_data entries[];
 };
 
+#define MAX_MEM_DUMP_TYPE	5
+
 /* used for crash-dump storage, protected by data-lock */
 struct ath10k_fw_crash_data {
-	bool crashed_since_read;
-
 	guid_t guid;
-	struct timespec timestamp;
+	struct timespec64 timestamp;
 	__le32 registers[REG_DUMP_COUNT_QCA988X];
 	struct ath10k_ce_crash_data ce_crash_data[CE_COUNT_MAX];
+
+	u8 *ramdump_buf;
+	size_t ramdump_buf_len;
 };
 
 struct ath10k_debug {
@@ -488,12 +493,9 @@ struct ath10k_debug {
 	/* protected by conf_mutex */
 	u64 fw_dbglog_mask;
 	u32 fw_dbglog_level;
-	u32 pktlog_filter;
 	u32 reg_addr;
 	u32 nf_cal_period;
 	void *cal_data;
-
-	struct ath10k_fw_crash_data *fw_crash_data;
 };
 
 enum ath10k_state {
@@ -615,6 +617,12 @@ enum ath10k_fw_features {
 	/* Firmware does not support power save in station mode. */
 	ATH10K_FW_FEATURE_NO_PS = 17,
 
+	/* Firmware allows management tx by reference instead of by value. */
+	ATH10K_FW_FEATURE_MGMT_TX_BY_REF = 18,
+
+	/* Firmware load is done externally, not by bmi */
+	ATH10K_FW_FEATURE_NON_BMI = 19,
+
 	/* keep last */
 	ATH10K_FW_FEATURE_COUNT,
 };
@@ -963,6 +971,14 @@ struct ath10k {
 	} spectral;
 #endif
 
+	u32 pktlog_filter;
+
+#ifdef CONFIG_DEV_COREDUMP
+	struct {
+		struct ath10k_fw_crash_data *fw_crash_data;
+	} coredump;
+#endif
+
 	struct {
 		/* protected by conf_mutex */
 		struct ath10k_fw_components utf_mode_fw;
@@ -1016,6 +1032,8 @@ static inline bool ath10k_peer_stats_enabled(struct ath10k *ar)
 	return false;
 }
 
+extern unsigned long ath10k_coredump_mask;
+
 struct ath10k *ath10k_core_create(size_t priv_size, struct device *dev,
 				  enum ath10k_bus bus,
 				  enum ath10k_hw_rev hw_rev,
diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c
new file mode 100644
index 000000000000..7173b3743b43
--- /dev/null
+++ b/drivers/net/wireless/ath/ath10k/coredump.c
@@ -0,0 +1,994 @@
+/*
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "coredump.h"
+
+#include <linux/devcoredump.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/utsname.h>
+
+#include "debug.h"
+#include "hw.h"
+
+static const struct ath10k_mem_section qca6174_hw21_register_sections[] = {
+	{0x800, 0x810},
+	{0x820, 0x82C},
+	{0x830, 0x8F4},
+	{0x90C, 0x91C},
+	{0xA14, 0xA18},
+	{0xA84, 0xA94},
+	{0xAA8, 0xAD4},
+	{0xADC, 0xB40},
+	{0x1000, 0x10A4},
+	{0x10BC, 0x111C},
+	{0x1134, 0x1138},
+	{0x1144, 0x114C},
+	{0x1150, 0x115C},
+	{0x1160, 0x1178},
+	{0x1240, 0x1260},
+	{0x2000, 0x207C},
+	{0x3000, 0x3014},
+	{0x4000, 0x4014},
+	{0x5000, 0x5124},
+	{0x6000, 0x6040},
+	{0x6080, 0x60CC},
+	{0x6100, 0x611C},
+	{0x6140, 0x61D8},
+	{0x6200, 0x6238},
+	{0x6240, 0x628C},
+	{0x62C0, 0x62EC},
+	{0x6380, 0x63E8},
+	{0x6400, 0x6440},
+	{0x6480, 0x64CC},
+	{0x6500, 0x651C},
+	{0x6540, 0x6580},
+	{0x6600, 0x6638},
+	{0x6640, 0x668C},
+	{0x66C0, 0x66EC},
+	{0x6780, 0x67E8},
+	{0x7080, 0x708C},
+	{0x70C0, 0x70C8},
+	{0x7400, 0x741C},
+	{0x7440, 0x7454},
+	{0x7800, 0x7818},
+	{0x8000, 0x8004},
+	{0x8010, 0x8064},
+	{0x8080, 0x8084},
+	{0x80A0, 0x80A4},
+	{0x80C0, 0x80C4},
+	{0x80E0, 0x80F4},
+	{0x8100, 0x8104},
+	{0x8110, 0x812C},
+	{0x9000, 0x9004},
+	{0x9800, 0x982C},
+	{0x9830, 0x9838},
+	{0x9840, 0x986C},
+	{0x9870, 0x9898},
+	{0x9A00, 0x9C00},
+	{0xD580, 0xD59C},
+	{0xF000, 0xF0E0},
+	{0xF140, 0xF190},
+	{0xF250, 0xF25C},
+	{0xF260, 0xF268},
+	{0xF26C, 0xF2A8},
+	{0x10008, 0x1000C},
+	{0x10014, 0x10018},
+	{0x1001C, 0x10020},
+	{0x10024, 0x10028},
+	{0x10030, 0x10034},
+	{0x10040, 0x10054},
+	{0x10058, 0x1007C},
+	{0x10080, 0x100C4},
+	{0x100C8, 0x10114},
+	{0x1012C, 0x10130},
+	{0x10138, 0x10144},
+	{0x10200, 0x10220},
+	{0x10230, 0x10250},
+	{0x10260, 0x10280},
+	{0x10290, 0x102B0},
+	{0x102C0, 0x102DC},
+	{0x102E0, 0x102F4},
+	{0x102FC, 0x1037C},
+	{0x10380, 0x10390},
+	{0x10800, 0x10828},
+	{0x10840, 0x10844},
+	{0x10880, 0x10884},
+	{0x108C0, 0x108E8},
+	{0x10900, 0x10928},
+	{0x10940, 0x10944},
+	{0x10980, 0x10984},
+	{0x109C0, 0x109E8},
+	{0x10A00, 0x10A28},
+	{0x10A40, 0x10A50},
+	{0x11000, 0x11028},
+	{0x11030, 0x11034},
+	{0x11038, 0x11068},
+	{0x11070, 0x11074},
+	{0x11078, 0x110A8},
+	{0x110B0, 0x110B4},
+	{0x110B8, 0x110E8},
+	{0x110F0, 0x110F4},
+	{0x110F8, 0x11128},
+	{0x11138, 0x11144},
+	{0x11178, 0x11180},
+	{0x111B8, 0x111C0},
+	{0x111F8, 0x11200},
+	{0x11238, 0x1123C},
+	{0x11270, 0x11274},
+	{0x11278, 0x1127C},
+	{0x112B0, 0x112B4},
+	{0x112B8, 0x112BC},
+	{0x112F0, 0x112F4},
+	{0x112F8, 0x112FC},
+	{0x11338, 0x1133C},
+	{0x11378, 0x1137C},
+	{0x113B8, 0x113BC},
+	{0x113F8, 0x113FC},
+	{0x11438, 0x11440},
+	{0x11478, 0x11480},
+	{0x114B8, 0x114BC},
+	{0x114F8, 0x114FC},
+	{0x11538, 0x1153C},
+	{0x11578, 0x1157C},
+	{0x115B8, 0x115BC},
+	{0x115F8, 0x115FC},
+	{0x11638, 0x1163C},
+	{0x11678, 0x1167C},
+	{0x116B8, 0x116BC},
+	{0x116F8, 0x116FC},
+	{0x11738, 0x1173C},
+	{0x11778, 0x1177C},
+	{0x117B8, 0x117BC},
+	{0x117F8, 0x117FC},
+	{0x17000, 0x1701C},
+	{0x17020, 0x170AC},
+	{0x18000, 0x18050},
+	{0x18054, 0x18074},
+	{0x18080, 0x180D4},
+	{0x180DC, 0x18104},
+	{0x18108, 0x1813C},
+	{0x18144, 0x18148},
+	{0x18168, 0x18174},
+	{0x18178, 0x18180},
+	{0x181C8, 0x181E0},
+	{0x181E4, 0x181E8},
+	{0x181EC, 0x1820C},
+	{0x1825C, 0x18280},
+	{0x18284, 0x18290},
+	{0x18294, 0x182A0},
+	{0x18300, 0x18304},
+	{0x18314, 0x18320},
+	{0x18328, 0x18350},
+	{0x1835C, 0x1836C},
+	{0x18370, 0x18390},
+	{0x18398, 0x183AC},
+	{0x183BC, 0x183D8},
+	{0x183DC, 0x183F4},
+	{0x18400, 0x186F4},
+	{0x186F8, 0x1871C},
+	{0x18720, 0x18790},
+	{0x19800, 0x19830},
+	{0x19834, 0x19840},
+	{0x19880, 0x1989C},
+	{0x198A4, 0x198B0},
+	{0x198BC, 0x19900},
+	{0x19C00, 0x19C88},
+	{0x19D00, 0x19D20},
+	{0x19E00, 0x19E7C},
+	{0x19E80, 0x19E94},
+	{0x19E98, 0x19EAC},
+	{0x19EB0, 0x19EBC},
+	{0x19F70, 0x19F74},
+	{0x19F80, 0x19F8C},
+	{0x19FA0, 0x19FB4},
+	{0x19FC0, 0x19FD8},
+	{0x1A000, 0x1A200},
+	{0x1A204, 0x1A210},
+	{0x1A228, 0x1A22C},
+	{0x1A230, 0x1A248},
+	{0x1A250, 0x1A270},
+	{0x1A280, 0x1A290},
+	{0x1A2A0, 0x1A2A4},
+	{0x1A2C0, 0x1A2EC},
+	{0x1A300, 0x1A3BC},
+	{0x1A3F0, 0x1A3F4},
+	{0x1A3F8, 0x1A434},
+	{0x1A438, 0x1A444},
+	{0x1A448, 0x1A468},
+	{0x1A580, 0x1A58C},
+	{0x1A644, 0x1A654},
+	{0x1A670, 0x1A698},
+	{0x1A6AC, 0x1A6B0},
+	{0x1A6D0, 0x1A6D4},
+	{0x1A6EC, 0x1A70C},
+	{0x1A710, 0x1A738},
+	{0x1A7C0, 0x1A7D0},
+	{0x1A7D4, 0x1A7D8},
+	{0x1A7DC, 0x1A7E4},
+	{0x1A7F0, 0x1A7F8},
+	{0x1A888, 0x1A89C},
+	{0x1A8A8, 0x1A8AC},
+	{0x1A8C0, 0x1A8DC},
+	{0x1A8F0, 0x1A8FC},
+	{0x1AE04, 0x1AE08},
+	{0x1AE18, 0x1AE24},
+	{0x1AF80, 0x1AF8C},
+	{0x1AFA0, 0x1AFB4},
+	{0x1B000, 0x1B200},
+	{0x1B284, 0x1B288},
+	{0x1B2D0, 0x1B2D8},
+	{0x1B2DC, 0x1B2EC},
+	{0x1B300, 0x1B340},
+	{0x1B374, 0x1B378},
+	{0x1B380, 0x1B384},
+	{0x1B388, 0x1B38C},
+	{0x1B404, 0x1B408},
+	{0x1B420, 0x1B428},
+	{0x1B440, 0x1B444},
+	{0x1B448, 0x1B44C},
+	{0x1B450, 0x1B458},
+	{0x1B45C, 0x1B468},
+	{0x1B584, 0x1B58C},
+	{0x1B68C, 0x1B690},
+	{0x1B6AC, 0x1B6B0},
+	{0x1B7F0, 0x1B7F8},
+	{0x1C800, 0x1CC00},
+	{0x1CE00, 0x1CE04},
+	{0x1CF80, 0x1CF84},
+	{0x1D200, 0x1D800},
+	{0x1E000, 0x20014},
+	{0x20100, 0x20124},
+	{0x21400, 0x217A8},
+	{0x21800, 0x21BA8},
+	{0x21C00, 0x21FA8},
+	{0x22000, 0x223A8},
+	{0x22400, 0x227A8},
+	{0x22800, 0x22BA8},
+	{0x22C00, 0x22FA8},
+	{0x23000, 0x233A8},
+	{0x24000, 0x24034},
+	{0x26000, 0x26064},
+	{0x27000, 0x27024},
+	{0x34000, 0x3400C},
+	{0x34400, 0x3445C},
+	{0x34800, 0x3485C},
+	{0x34C00, 0x34C5C},
+	{0x35000, 0x3505C},
+	{0x35400, 0x3545C},
+	{0x35800, 0x3585C},
+	{0x35C00, 0x35C5C},
+	{0x36000, 0x3605C},
+	{0x38000, 0x38064},
+	{0x38070, 0x380E0},
+	{0x3A000, 0x3A064},
+	{0x40000, 0x400A4},
+	{0x80000, 0x8000C},
+	{0x80010, 0x80020},
+};
+
+static const struct ath10k_mem_section qca6174_hw30_register_sections[] = {
+	{0x800, 0x810},
+	{0x820, 0x82C},
+	{0x830, 0x8F4},
+	{0x90C, 0x91C},
+	{0xA14, 0xA18},
+	{0xA84, 0xA94},
+	{0xAA8, 0xAD4},
+	{0xADC, 0xB40},
+	{0x1000, 0x10A4},
+	{0x10BC, 0x111C},
+	{0x1134, 0x1138},
+	{0x1144, 0x114C},
+	{0x1150, 0x115C},
+	{0x1160, 0x1178},
+	{0x1240, 0x1260},
+	{0x2000, 0x207C},
+	{0x3000, 0x3014},
+	{0x4000, 0x4014},
+	{0x5000, 0x5124},
+	{0x6000, 0x6040},
+	{0x6080, 0x60CC},
+	{0x6100, 0x611C},
+	{0x6140, 0x61D8},
+	{0x6200, 0x6238},
+	{0x6240, 0x628C},
+	{0x62C0, 0x62EC},
+	{0x6380, 0x63E8},
+	{0x6400, 0x6440},
+	{0x6480, 0x64CC},
+	{0x6500, 0x651C},
+	{0x6540, 0x6580},
+	{0x6600, 0x6638},
+	{0x6640, 0x668C},
+	{0x66C0, 0x66EC},
+	{0x6780, 0x67E8},
+	{0x7080, 0x708C},
+	{0x70C0, 0x70C8},
+	{0x7400, 0x741C},
+	{0x7440, 0x7454},
+	{0x7800, 0x7818},
+	{0x8000, 0x8004},
+	{0x8010, 0x8064},
+	{0x8080, 0x8084},
+	{0x80A0, 0x80A4},
+	{0x80C0, 0x80C4},
+	{0x80E0, 0x80F4},
+	{0x8100, 0x8104},
+	{0x8110, 0x812C},
+	{0x9000, 0x9004},
+	{0x9800, 0x982C},
+	{0x9830, 0x9838},
+	{0x9840, 0x986C},
+	{0x9870, 0x9898},
+	{0x9A00, 0x9C00},
+	{0xD580, 0xD59C},
+	{0xF000, 0xF0E0},
+	{0xF140, 0xF190},
+	{0xF250, 0xF25C},
+	{0xF260, 0xF268},
+	{0xF26C, 0xF2A8},
+	{0x10008, 0x1000C},
+	{0x10014, 0x10018},
+	{0x1001C, 0x10020},
+	{0x10024, 0x10028},
+	{0x10030, 0x10034},
+	{0x10040, 0x10054},
+	{0x10058, 0x1007C},
+	{0x10080, 0x100C4},
+	{0x100C8, 0x10114},
+	{0x1012C, 0x10130},
+	{0x10138, 0x10144},
+	{0x10200, 0x10220},
+	{0x10230, 0x10250},
+	{0x10260, 0x10280},
+	{0x10290, 0x102B0},
+	{0x102C0, 0x102DC},
+	{0x102E0, 0x102F4},
+	{0x102FC, 0x1037C},
+	{0x10380, 0x10390},
+	{0x10800, 0x10828},
+	{0x10840, 0x10844},
+	{0x10880, 0x10884},
+	{0x108C0, 0x108E8},
+	{0x10900, 0x10928},
+	{0x10940, 0x10944},
+	{0x10980, 0x10984},
+	{0x109C0, 0x109E8},
+	{0x10A00, 0x10A28},
+	{0x10A40, 0x10A50},
+	{0x11000, 0x11028},
+	{0x11030, 0x11034},
+	{0x11038, 0x11068},
+	{0x11070, 0x11074},
+	{0x11078, 0x110A8},
+	{0x110B0, 0x110B4},
+	{0x110B8, 0x110E8},
+	{0x110F0, 0x110F4},
+	{0x110F8, 0x11128},
+	{0x11138, 0x11144},
+	{0x11178, 0x11180},
+	{0x111B8, 0x111C0},
+	{0x111F8, 0x11200},
+	{0x11238, 0x1123C},
+	{0x11270, 0x11274},
+	{0x11278, 0x1127C},
+	{0x112B0, 0x112B4},
+	{0x112B8, 0x112BC},
+	{0x112F0, 0x112F4},
+	{0x112F8, 0x112FC},
+	{0x11338, 0x1133C},
+	{0x11378, 0x1137C},
+	{0x113B8, 0x113BC},
+	{0x113F8, 0x113FC},
+	{0x11438, 0x11440},
+	{0x11478, 0x11480},
+	{0x114B8, 0x114BC},
+	{0x114F8, 0x114FC},
+	{0x11538, 0x1153C},
+	{0x11578, 0x1157C},
+	{0x115B8, 0x115BC},
+	{0x115F8, 0x115FC},
+	{0x11638, 0x1163C},
+	{0x11678, 0x1167C},
+	{0x116B8, 0x116BC},
+	{0x116F8, 0x116FC},
+	{0x11738, 0x1173C},
+	{0x11778, 0x1177C},
+	{0x117B8, 0x117BC},
+	{0x117F8, 0x117FC},
+	{0x17000, 0x1701C},
+	{0x17020, 0x170AC},
+	{0x18000, 0x18050},
+	{0x18054, 0x18074},
+	{0x18080, 0x180D4},
+	{0x180DC, 0x18104},
+	{0x18108, 0x1813C},
+	{0x18144, 0x18148},
+	{0x18168, 0x18174},
+	{0x18178, 0x18180},
+	{0x181C8, 0x181E0},
+	{0x181E4, 0x181E8},
+	{0x181EC, 0x1820C},
+	{0x1825C, 0x18280},
+	{0x18284, 0x18290},
+	{0x18294, 0x182A0},
+	{0x18300, 0x18304},
+	{0x18314, 0x18320},
+	{0x18328, 0x18350},
+	{0x1835C, 0x1836C},
+	{0x18370, 0x18390},
+	{0x18398, 0x183AC},
+	{0x183BC, 0x183D8},
+	{0x183DC, 0x183F4},
+	{0x18400, 0x186F4},
+	{0x186F8, 0x1871C},
+	{0x18720, 0x18790},
+	{0x19800, 0x19830},
+	{0x19834, 0x19840},
+	{0x19880, 0x1989C},
+	{0x198A4, 0x198B0},
+	{0x198BC, 0x19900},
+	{0x19C00, 0x19C88},
+	{0x19D00, 0x19D20},
+	{0x19E00, 0x19E7C},
+	{0x19E80, 0x19E94},
+	{0x19E98, 0x19EAC},
+	{0x19EB0, 0x19EBC},
+	{0x19F70, 0x19F74},
+	{0x19F80, 0x19F8C},
+	{0x19FA0, 0x19FB4},
+	{0x19FC0, 0x19FD8},
+	{0x1A000, 0x1A200},
+	{0x1A204, 0x1A210},
+	{0x1A228, 0x1A22C},
+	{0x1A230, 0x1A248},
+	{0x1A250, 0x1A270},
+	{0x1A280, 0x1A290},
+	{0x1A2A0, 0x1A2A4},
+	{0x1A2C0, 0x1A2EC},
+	{0x1A300, 0x1A3BC},
+	{0x1A3F0, 0x1A3F4},
+	{0x1A3F8, 0x1A434},
+	{0x1A438, 0x1A444},
+	{0x1A448, 0x1A468},
+	{0x1A580, 0x1A58C},
+	{0x1A644, 0x1A654},
+	{0x1A670, 0x1A698},
+	{0x1A6AC, 0x1A6B0},
+	{0x1A6D0, 0x1A6D4},
+	{0x1A6EC, 0x1A70C},
+	{0x1A710, 0x1A738},
+	{0x1A7C0, 0x1A7D0},
+	{0x1A7D4, 0x1A7D8},
+	{0x1A7DC, 0x1A7E4},
+	{0x1A7F0, 0x1A7F8},
+	{0x1A888, 0x1A89C},
+	{0x1A8A8, 0x1A8AC},
+	{0x1A8C0, 0x1A8DC},
+	{0x1A8F0, 0x1A8FC},
+	{0x1AE04, 0x1AE08},
+	{0x1AE18, 0x1AE24},
+	{0x1AF80, 0x1AF8C},
+	{0x1AFA0, 0x1AFB4},
+	{0x1B000, 0x1B200},
+	{0x1B284, 0x1B288},
+	{0x1B2D0, 0x1B2D8},
+	{0x1B2DC, 0x1B2EC},
+	{0x1B300, 0x1B340},
+	{0x1B374, 0x1B378},
+	{0x1B380, 0x1B384},
+	{0x1B388, 0x1B38C},
+	{0x1B404, 0x1B408},
+	{0x1B420, 0x1B428},
+	{0x1B440, 0x1B444},
+	{0x1B448, 0x1B44C},
+	{0x1B450, 0x1B458},
+	{0x1B45C, 0x1B468},
+	{0x1B584, 0x1B58C},
+	{0x1B68C, 0x1B690},
+	{0x1B6AC, 0x1B6B0},
+	{0x1B7F0, 0x1B7F8},
+	{0x1C800, 0x1CC00},
+	{0x1CE00, 0x1CE04},
+	{0x1CF80, 0x1CF84},
+	{0x1D200, 0x1D800},
+	{0x1E000, 0x20014},
+	{0x20100, 0x20124},
+	{0x21400, 0x217A8},
+	{0x21800, 0x21BA8},
+	{0x21C00, 0x21FA8},
+	{0x22000, 0x223A8},
+	{0x22400, 0x227A8},
+	{0x22800, 0x22BA8},
+	{0x22C00, 0x22FA8},
+	{0x23000, 0x233A8},
+	{0x24000, 0x24034},
+	{0x26000, 0x26064},
+	{0x27000, 0x27024},
+	{0x34000, 0x3400C},
+	{0x34400, 0x3445C},
+	{0x34800, 0x3485C},
+	{0x34C00, 0x34C5C},
+	{0x35000, 0x3505C},
+	{0x35400, 0x3545C},
+	{0x35800, 0x3585C},
+	{0x35C00, 0x35C5C},
+	{0x36000, 0x3605C},
+	{0x38000, 0x38064},
+	{0x38070, 0x380E0},
+	{0x3A000, 0x3A074},
+	{0x40000, 0x400A4},
+	{0x80000, 0x8000C},
+	{0x80010, 0x80020},
+};
+
+static const struct ath10k_mem_region qca6174_hw10_mem_regions[] = {
+	{
+		.type = ATH10K_MEM_REGION_TYPE_DRAM,
+		.start = 0x400000,
+		.len = 0x70000,
+		.name = "DRAM",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_REG,
+
+		/* RTC_SOC_BASE_ADDRESS */
+		.start = 0x0,
+
+		/* WLAN_MBOX_BASE_ADDRESS - RTC_SOC_BASE_ADDRESS */
+		.len = 0x800 - 0x0,
+
+		.name = "REG_PART1",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_REG,
+
+		/* STEREO_BASE_ADDRESS */
+		.start = 0x27000,
+
+		/* USB_BASE_ADDRESS - STEREO_BASE_ADDRESS */
+		.len = 0x60000 - 0x27000,
+
+		.name = "REG_PART2",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+};
+
+static const struct ath10k_mem_region qca6174_hw21_mem_regions[] = {
+	{
+		.type = ATH10K_MEM_REGION_TYPE_DRAM,
+		.start = 0x400000,
+		.len = 0x70000,
+		.name = "DRAM",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_AXI,
+		.start = 0xa0000,
+		.len = 0x18000,
+		.name = "AXI",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_REG,
+		.start = 0x800,
+		.len = 0x80020 - 0x800,
+		.name = "REG_TOTAL",
+		.section_table = {
+			.sections = qca6174_hw21_register_sections,
+			.size = ARRAY_SIZE(qca6174_hw21_register_sections),
+		},
+	},
+};
+
+static const struct ath10k_mem_region qca6174_hw30_mem_regions[] = {
+	{
+		.type = ATH10K_MEM_REGION_TYPE_DRAM,
+		.start = 0x400000,
+		.len = 0xa8000,
+		.name = "DRAM",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_AXI,
+		.start = 0xa0000,
+		.len = 0x18000,
+		.name = "AXI",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_REG,
+		.start = 0x800,
+		.len = 0x80020 - 0x800,
+		.name = "REG_TOTAL",
+		.section_table = {
+			.sections = qca6174_hw30_register_sections,
+			.size = ARRAY_SIZE(qca6174_hw30_register_sections),
+		},
+	},
+
+	/* IRAM dump must be put last */
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IRAM1,
+		.start = 0x00980000,
+		.len = 0x00080000,
+		.name = "IRAM1",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_IRAM2,
+		.start = 0x00a00000,
+		.len = 0x00040000,
+		.name = "IRAM2",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+};
+
+static const struct ath10k_mem_region qca988x_hw20_mem_regions[] = {
+	{
+		.type = ATH10K_MEM_REGION_TYPE_DRAM,
+		.start = 0x400000,
+		.len = 0x50000,
+		.name = "DRAM",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_REG,
+		.start = 0x4000,
+		.len = 0x2000,
+		.name = "REG_PART1",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+	{
+		.type = ATH10K_MEM_REGION_TYPE_REG,
+		.start = 0x8000,
+		.len = 0x58000,
+		.name = "REG_PART2",
+		.section_table = {
+			.sections = NULL,
+			.size = 0,
+		},
+	},
+};
+
+static const struct ath10k_hw_mem_layout hw_mem_layouts[] = {
+	{
+		.hw_id = QCA6174_HW_1_0_VERSION,
+		.region_table = {
+			.regions = qca6174_hw10_mem_regions,
+			.size = ARRAY_SIZE(qca6174_hw10_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA6174_HW_1_1_VERSION,
+		.region_table = {
+			.regions = qca6174_hw10_mem_regions,
+			.size = ARRAY_SIZE(qca6174_hw10_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA6174_HW_1_3_VERSION,
+		.region_table = {
+			.regions = qca6174_hw10_mem_regions,
+			.size = ARRAY_SIZE(qca6174_hw10_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA6174_HW_2_1_VERSION,
+		.region_table = {
+			.regions = qca6174_hw21_mem_regions,
+			.size = ARRAY_SIZE(qca6174_hw21_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA6174_HW_3_0_VERSION,
+		.region_table = {
+			.regions = qca6174_hw30_mem_regions,
+			.size = ARRAY_SIZE(qca6174_hw30_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA6174_HW_3_2_VERSION,
+		.region_table = {
+			.regions = qca6174_hw30_mem_regions,
+			.size = ARRAY_SIZE(qca6174_hw30_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA9377_HW_1_1_DEV_VERSION,
+		.region_table = {
+			.regions = qca6174_hw30_mem_regions,
+			.size = ARRAY_SIZE(qca6174_hw30_mem_regions),
+		},
+	},
+	{
+		.hw_id = QCA988X_HW_2_0_VERSION,
+		.region_table = {
+			.regions = qca988x_hw20_mem_regions,
+			.size = ARRAY_SIZE(qca988x_hw20_mem_regions),
+		},
+	},
+};
+
+static u32 ath10k_coredump_get_ramdump_size(struct ath10k *ar)
+{
+	const struct ath10k_hw_mem_layout *hw;
+	const struct ath10k_mem_region *mem_region;
+	size_t size = 0;
+	int i;
+
+	hw = ath10k_coredump_get_mem_layout(ar);
+
+	if (!hw)
+		return 0;
+
+	mem_region = &hw->region_table.regions[0];
+
+	for (i = 0; i < hw->region_table.size; i++) {
+		size += mem_region->len;
+		mem_region++;
+	}
+
+	/* reserve space for the headers */
+	size += hw->region_table.size * sizeof(struct ath10k_dump_ram_data_hdr);
+
+	/* make sure it is aligned 16 bytes for debug message print out */
+	size = ALIGN(size, 16);
+
+	return size;
+}
+
+const struct ath10k_hw_mem_layout *ath10k_coredump_get_mem_layout(struct ath10k *ar)
+{
+	int i;
+
+	if (!test_bit(ATH10K_FW_CRASH_DUMP_RAM_DATA, &ath10k_coredump_mask))
+		return NULL;
+
+	if (WARN_ON(ar->target_version == 0))
+		return NULL;
+
+	for (i = 0; i < ARRAY_SIZE(hw_mem_layouts); i++) {
+		if (ar->target_version == hw_mem_layouts[i].hw_id)
+			return &hw_mem_layouts[i];
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL(ath10k_coredump_get_mem_layout);
+
+struct ath10k_fw_crash_data *ath10k_coredump_new(struct ath10k *ar)
+{
+	struct ath10k_fw_crash_data *crash_data = ar->coredump.fw_crash_data;
+
+	lockdep_assert_held(&ar->data_lock);
+
+	if (ath10k_coredump_mask == 0)
+		/* coredump disabled */
+		return NULL;
+
+	guid_gen(&crash_data->guid);
+	ktime_get_real_ts64(&crash_data->timestamp);
+
+	return crash_data;
+}
+EXPORT_SYMBOL(ath10k_coredump_new);
+
+static struct ath10k_dump_file_data *ath10k_coredump_build(struct ath10k *ar)
+{
+	struct ath10k_fw_crash_data *crash_data = ar->coredump.fw_crash_data;
+	struct ath10k_ce_crash_hdr *ce_hdr;
+	struct ath10k_dump_file_data *dump_data;
+	struct ath10k_tlv_dump_data *dump_tlv;
+	size_t hdr_len = sizeof(*dump_data);
+	size_t len, sofar = 0;
+	unsigned char *buf;
+
+	len = hdr_len;
+
+	if (test_bit(ATH10K_FW_CRASH_DUMP_REGISTERS, &ath10k_coredump_mask))
+		len += sizeof(*dump_tlv) + sizeof(crash_data->registers);
+
+	if (test_bit(ATH10K_FW_CRASH_DUMP_CE_DATA, &ath10k_coredump_mask))
+		len += sizeof(*dump_tlv) + sizeof(*ce_hdr) +
+			CE_COUNT * sizeof(ce_hdr->entries[0]);
+
+	if (test_bit(ATH10K_FW_CRASH_DUMP_RAM_DATA, &ath10k_coredump_mask))
+		len += sizeof(*dump_tlv) + crash_data->ramdump_buf_len;
+
+	sofar += hdr_len;
+
+	/* This is going to get big when we start dumping FW RAM and such,
+	 * so go ahead and use vmalloc.
+	 */
+	buf = vzalloc(len);
+	if (!buf)
+		return NULL;
+
+	spin_lock_bh(&ar->data_lock);
+
+	dump_data = (struct ath10k_dump_file_data *)(buf);
+	strlcpy(dump_data->df_magic, "ATH10K-FW-DUMP",
+		sizeof(dump_data->df_magic));
+	dump_data->len = cpu_to_le32(len);
+
+	dump_data->version = cpu_to_le32(ATH10K_FW_CRASH_DUMP_VERSION);
+
+	guid_copy(&dump_data->guid, &crash_data->guid);
+	dump_data->chip_id = cpu_to_le32(ar->chip_id);
+	dump_data->bus_type = cpu_to_le32(0);
+	dump_data->target_version = cpu_to_le32(ar->target_version);
+	dump_data->fw_version_major = cpu_to_le32(ar->fw_version_major);
+	dump_data->fw_version_minor = cpu_to_le32(ar->fw_version_minor);
+	dump_data->fw_version_release = cpu_to_le32(ar->fw_version_release);
+	dump_data->fw_version_build = cpu_to_le32(ar->fw_version_build);
+	dump_data->phy_capability = cpu_to_le32(ar->phy_capability);
+	dump_data->hw_min_tx_power = cpu_to_le32(ar->hw_min_tx_power);
+	dump_data->hw_max_tx_power = cpu_to_le32(ar->hw_max_tx_power);
+	dump_data->ht_cap_info = cpu_to_le32(ar->ht_cap_info);
+	dump_data->vht_cap_info = cpu_to_le32(ar->vht_cap_info);
+	dump_data->num_rf_chains = cpu_to_le32(ar->num_rf_chains);
+
+	strlcpy(dump_data->fw_ver, ar->hw->wiphy->fw_version,
+		sizeof(dump_data->fw_ver));
+
+	dump_data->kernel_ver_code = 0;
+	strlcpy(dump_data->kernel_ver, init_utsname()->release,
+		sizeof(dump_data->kernel_ver));
+
+	dump_data->tv_sec = cpu_to_le64(crash_data->timestamp.tv_sec);
+	dump_data->tv_nsec = cpu_to_le64(crash_data->timestamp.tv_nsec);
+
+	if (test_bit(ATH10K_FW_CRASH_DUMP_REGISTERS, &ath10k_coredump_mask)) {
+		dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
+		dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_REGISTERS);
+		dump_tlv->tlv_len = cpu_to_le32(sizeof(crash_data->registers));
+		memcpy(dump_tlv->tlv_data, &crash_data->registers,
+		       sizeof(crash_data->registers));
+		sofar += sizeof(*dump_tlv) + sizeof(crash_data->registers);
+	}
+
+	if (test_bit(ATH10K_FW_CRASH_DUMP_CE_DATA, &ath10k_coredump_mask)) {
+		dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
+		dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_CE_DATA);
+		dump_tlv->tlv_len = cpu_to_le32(sizeof(*ce_hdr) +
+						CE_COUNT * sizeof(ce_hdr->entries[0]));
+		ce_hdr = (struct ath10k_ce_crash_hdr *)(dump_tlv->tlv_data);
+		ce_hdr->ce_count = cpu_to_le32(CE_COUNT);
+		memset(ce_hdr->reserved, 0, sizeof(ce_hdr->reserved));
+		memcpy(ce_hdr->entries, crash_data->ce_crash_data,
+		       CE_COUNT * sizeof(ce_hdr->entries[0]));
+		sofar += sizeof(*dump_tlv) + sizeof(*ce_hdr) +
+			CE_COUNT * sizeof(ce_hdr->entries[0]);
+	}
+
+	/* Gather ram dump */
+	if (test_bit(ATH10K_FW_CRASH_DUMP_RAM_DATA, &ath10k_coredump_mask)) {
+		dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
+		dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_RAM_DATA);
+		dump_tlv->tlv_len = cpu_to_le32(crash_data->ramdump_buf_len);
+		memcpy(dump_tlv->tlv_data, crash_data->ramdump_buf,
+		       crash_data->ramdump_buf_len);
+		sofar += sizeof(*dump_tlv) + crash_data->ramdump_buf_len;
+	}
+
+	spin_unlock_bh(&ar->data_lock);
+
+	return dump_data;
+}
+
+int ath10k_coredump_submit(struct ath10k *ar)
+{
+	struct ath10k_dump_file_data *dump;
+
+	if (ath10k_coredump_mask == 0)
+		/* coredump disabled */
+		return 0;
+
+	dump = ath10k_coredump_build(ar);
+	if (!dump) {
+		ath10k_warn(ar, "no crash dump data found for devcoredump");
+		return -ENODATA;
+	}
+
+	dev_coredumpv(ar->dev, dump, le32_to_cpu(dump->len), GFP_KERNEL);
+
+	return 0;
+}
+
+int ath10k_coredump_create(struct ath10k *ar)
+{
+	if (ath10k_coredump_mask == 0)
+		/* coredump disabled */
+		return 0;
+
+	ar->coredump.fw_crash_data = vzalloc(sizeof(*ar->coredump.fw_crash_data));
+	if (!ar->coredump.fw_crash_data)
+		return -ENOMEM;
+
+	return 0;
+}
+
+int ath10k_coredump_register(struct ath10k *ar)
+{
+	struct ath10k_fw_crash_data *crash_data = ar->coredump.fw_crash_data;
+
+	if (test_bit(ATH10K_FW_CRASH_DUMP_RAM_DATA, &ath10k_coredump_mask)) {
+		crash_data->ramdump_buf_len = ath10k_coredump_get_ramdump_size(ar);
+
+		crash_data->ramdump_buf = vzalloc(crash_data->ramdump_buf_len);
+		if (!crash_data->ramdump_buf)
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void ath10k_coredump_unregister(struct ath10k *ar)
+{
+	struct ath10k_fw_crash_data *crash_data = ar->coredump.fw_crash_data;
+
+	vfree(crash_data->ramdump_buf);
+}
+
+void ath10k_coredump_destroy(struct ath10k *ar)
+{
+	if (ar->coredump.fw_crash_data->ramdump_buf) {
+		vfree(ar->coredump.fw_crash_data->ramdump_buf);
+		ar->coredump.fw_crash_data->ramdump_buf = NULL;
+		ar->coredump.fw_crash_data->ramdump_buf_len = 0;
+	}
+
+	vfree(ar->coredump.fw_crash_data);
+	ar->coredump.fw_crash_data = NULL;
+}
diff --git a/drivers/net/wireless/ath/ath10k/coredump.h b/drivers/net/wireless/ath/ath10k/coredump.h
new file mode 100644
index 000000000000..bfee13038e59
--- /dev/null
+++ b/drivers/net/wireless/ath/ath10k/coredump.h
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _COREDUMP_H_
+#define _COREDUMP_H_
+
+#include "core.h"
+
+#define ATH10K_FW_CRASH_DUMP_VERSION 1
+
+/**
+ * enum ath10k_fw_crash_dump_type - types of data in the dump file
+ * @ATH10K_FW_CRASH_DUMP_REGDUMP: Register crash dump in binary format
+ */
+enum ath10k_fw_crash_dump_type {
+	ATH10K_FW_CRASH_DUMP_REGISTERS = 0,
+	ATH10K_FW_CRASH_DUMP_CE_DATA = 1,
+
+	/* contains multiple struct ath10k_dump_ram_data_hdr */
+	ATH10K_FW_CRASH_DUMP_RAM_DATA = 2,
+
+	ATH10K_FW_CRASH_DUMP_MAX,
+};
+
+struct ath10k_tlv_dump_data {
+	/* see ath10k_fw_crash_dump_type above */
+	__le32 type;
+
+	/* in bytes */
+	__le32 tlv_len;
+
+	/* pad to 32-bit boundaries as needed */
+	u8 tlv_data[];
+} __packed;
+
+struct ath10k_dump_file_data {
+	/* dump file information */
+
+	/* "ATH10K-FW-DUMP" */
+	char df_magic[16];
+
+	__le32 len;
+
+	/* file dump version */
+	__le32 version;
+
+	/* some info we can get from ath10k struct that might help */
+
+	guid_t guid;
+
+	__le32 chip_id;
+
+	/* 0 for now, in place for later hardware */
+	__le32 bus_type;
+
+	__le32 target_version;
+	__le32 fw_version_major;
+	__le32 fw_version_minor;
+	__le32 fw_version_release;
+	__le32 fw_version_build;
+	__le32 phy_capability;
+	__le32 hw_min_tx_power;
+	__le32 hw_max_tx_power;
+	__le32 ht_cap_info;
+	__le32 vht_cap_info;
+	__le32 num_rf_chains;
+
+	/* firmware version string */
+	char fw_ver[ETHTOOL_FWVERS_LEN];
+
+	/* Kernel related information */
+
+	/* time-of-day stamp */
+	__le64 tv_sec;
+
+	/* time-of-day stamp, nano-seconds */
+	__le64 tv_nsec;
+
+	/* LINUX_VERSION_CODE */
+	__le32 kernel_ver_code;
+
+	/* VERMAGIC_STRING */
+	char kernel_ver[64];
+
+	/* room for growth w/out changing binary format */
+	u8 unused[128];
+
+	/* struct ath10k_tlv_dump_data + more */
+	u8 data[0];
+} __packed;
+
+struct ath10k_dump_ram_data_hdr {
+	/* enum ath10k_mem_region_type */
+	__le32 region_type;
+
+	__le32 start;
+
+	/* length of payload data, not including this header */
+	__le32 length;
+
+	u8 data[0];
+};
+
+/* magic number to fill the holes not copied due to sections in regions */
+#define ATH10K_MAGIC_NOT_COPIED		0xAA
+
+/* part of user space ABI */
+enum ath10k_mem_region_type {
+	ATH10K_MEM_REGION_TYPE_REG	= 1,
+	ATH10K_MEM_REGION_TYPE_DRAM	= 2,
+	ATH10K_MEM_REGION_TYPE_AXI	= 3,
+	ATH10K_MEM_REGION_TYPE_IRAM1	= 4,
+	ATH10K_MEM_REGION_TYPE_IRAM2	= 5,
+};
+
+/* Define a section of the region which should be copied. As not all parts
+ * of the memory is possible to copy, for example some of the registers can
+ * be like that, sections can be used to define what is safe to copy.
+ *
+ * To minimize the size of the array, the list must obey the format:
+ * '{start0,stop0},{start1,stop1},{start2,stop2}....' The values below must
+ * also obey to 'start0 < stop0 < start1 < stop1 < start2 < ...', otherwise
+ * we may encouter error in the dump processing.
+ */
+struct ath10k_mem_section {
+	u32 start;
+	u32 end;
+};
+
+/* One region of a memory layout. If the sections field is null entire
+ * region is copied. If sections is non-null only the areas specified in
+ * sections are copied and rest of the areas are filled with
+ * ATH10K_MAGIC_NOT_COPIED.
+ */
+struct ath10k_mem_region {
+	enum ath10k_mem_region_type type;
+	u32 start;
+	u32 len;
+
+	const char *name;
+
+	struct {
+		const struct ath10k_mem_section *sections;
+		u32 size;
+	} section_table;
+};
+
+/* Contains the memory layout of a hardware version identified with the
+ * hardware id, split into regions.
+ */
+struct ath10k_hw_mem_layout {
+	u32 hw_id;
+
+	struct {
+		const struct ath10k_mem_region *regions;
+		int size;
+	} region_table;
+};
+
+/* FIXME: where to put this? */
+extern unsigned long ath10k_coredump_mask;
+
+#ifdef CONFIG_DEV_COREDUMP
+
+int ath10k_coredump_submit(struct ath10k *ar);
+struct ath10k_fw_crash_data *ath10k_coredump_new(struct ath10k *ar);
+int ath10k_coredump_create(struct ath10k *ar);
+int ath10k_coredump_register(struct ath10k *ar);
+void ath10k_coredump_unregister(struct ath10k *ar);
+void ath10k_coredump_destroy(struct ath10k *ar);
+
+const struct ath10k_hw_mem_layout *ath10k_coredump_get_mem_layout(struct ath10k *ar);
+
+#else /* CONFIG_DEV_COREDUMP */
+
+static inline int ath10k_coredump_submit(struct ath10k *ar)
+{
+	return 0;
+}
+
+static inline struct ath10k_fw_crash_data *ath10k_coredump_new(struct ath10k *ar)
+{
+	return NULL;
+}
+
+static inline int ath10k_coredump_create(struct ath10k *ar)
+{
+	return 0;
+}
+
+static inline int ath10k_coredump_register(struct ath10k *ar)
+{
+	return 0;
+}
+
+static inline void ath10k_coredump_unregister(struct ath10k *ar)
+{
+}
+
+static inline void ath10k_coredump_destroy(struct ath10k *ar)
+{
+}
+
+static inline const struct ath10k_hw_mem_layout *
+ath10k_coredump_get_mem_layout(struct ath10k *ar)
+{
+	return NULL;
+}
+
+#endif /* CONFIG_DEV_COREDUMP */
+
+#endif /* _COREDUMP_H_ */
diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c
index df514507d3f1..554cd7856cb6 100644
--- a/drivers/net/wireless/ath/ath10k/debug.c
+++ b/drivers/net/wireless/ath/ath10k/debug.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -18,10 +19,8 @@
 #include <linux/module.h>
 #include <linux/debugfs.h>
 #include <linux/vmalloc.h>
-#include <linux/utsname.h>
 #include <linux/crc32.h>
 #include <linux/firmware.h>
-#include <linux/devcoredump.h>
 
 #include "core.h"
 #include "debug.h"
@@ -33,86 +32,6 @@
 
 #define ATH10K_DEBUG_CAL_DATA_LEN 12064
 
-#define ATH10K_FW_CRASH_DUMP_VERSION 1
-
-/**
- * enum ath10k_fw_crash_dump_type - types of data in the dump file
- * @ATH10K_FW_CRASH_DUMP_REGDUMP: Register crash dump in binary format
- */
-enum ath10k_fw_crash_dump_type {
-	ATH10K_FW_CRASH_DUMP_REGISTERS = 0,
-	ATH10K_FW_CRASH_DUMP_CE_DATA = 1,
-
-	ATH10K_FW_CRASH_DUMP_MAX,
-};
-
-struct ath10k_tlv_dump_data {
-	/* see ath10k_fw_crash_dump_type above */
-	__le32 type;
-
-	/* in bytes */
-	__le32 tlv_len;
-
-	/* pad to 32-bit boundaries as needed */
-	u8 tlv_data[];
-} __packed;
-
-struct ath10k_dump_file_data {
-	/* dump file information */
-
-	/* "ATH10K-FW-DUMP" */
-	char df_magic[16];
-
-	__le32 len;
-
-	/* file dump version */
-	__le32 version;
-
-	/* some info we can get from ath10k struct that might help */
-
-	guid_t guid;
-
-	__le32 chip_id;
-
-	/* 0 for now, in place for later hardware */
-	__le32 bus_type;
-
-	__le32 target_version;
-	__le32 fw_version_major;
-	__le32 fw_version_minor;
-	__le32 fw_version_release;
-	__le32 fw_version_build;
-	__le32 phy_capability;
-	__le32 hw_min_tx_power;
-	__le32 hw_max_tx_power;
-	__le32 ht_cap_info;
-	__le32 vht_cap_info;
-	__le32 num_rf_chains;
-
-	/* firmware version string */
-	char fw_ver[ETHTOOL_FWVERS_LEN];
-
-	/* Kernel related information */
-
-	/* time-of-day stamp */
-	__le64 tv_sec;
-
-	/* time-of-day stamp, nano-seconds */
-	__le64 tv_nsec;
-
-	/* LINUX_VERSION_CODE */
-	__le32 kernel_ver_code;
-
-	/* VERMAGIC_STRING */
-	char kernel_ver[64];
-
-	/* room for growth w/out changing binary format */
-	u8 unused[128];
-
-	/* struct ath10k_tlv_dump_data + more */
-	u8 data[0];
-} __packed;
-
 void ath10k_info(struct ath10k *ar, const char *fmt, ...)
 {
 	struct va_format vaf = {
@@ -163,6 +82,8 @@ void ath10k_debug_print_hwfw_info(struct ath10k *ar)
 void ath10k_debug_print_board_info(struct ath10k *ar)
 {
 	char boardinfo[100];
+	const struct firmware *board;
+	u32 crc;
 
 	if (ar->id.bmi_ids_valid)
 		scnprintf(boardinfo, sizeof(boardinfo), "%d:%d",
@@ -170,11 +91,16 @@ void ath10k_debug_print_board_info(struct ath10k *ar)
 	else
 		scnprintf(boardinfo, sizeof(boardinfo), "N/A");
 
+	board = ar->normal_mode_fw.board;
+	if (!IS_ERR_OR_NULL(board))
+		crc = crc32_le(0, board->data, board->size);
+	else
+		crc = 0;
+
 	ath10k_info(ar, "board_file api %d bmi_id %s crc32 %08x",
 		    ar->bd_api,
 		    boardinfo,
-		    crc32_le(0, ar->normal_mode_fw.board->data,
-			     ar->normal_mode_fw.board->size));
+		    crc);
 }
 
 void ath10k_debug_print_boot_info(struct ath10k *ar)
@@ -711,189 +637,6 @@ static const struct file_operations fops_chip_id = {
 	.llseek = default_llseek,
 };
 
-struct ath10k_fw_crash_data *
-ath10k_debug_get_new_fw_crash_data(struct ath10k *ar)
-{
-	struct ath10k_fw_crash_data *crash_data = ar->debug.fw_crash_data;
-
-	lockdep_assert_held(&ar->data_lock);
-
-	crash_data->crashed_since_read = true;
-	guid_gen(&crash_data->guid);
-	getnstimeofday(&crash_data->timestamp);
-
-	return crash_data;
-}
-EXPORT_SYMBOL(ath10k_debug_get_new_fw_crash_data);
-
-static struct ath10k_dump_file_data *ath10k_build_dump_file(struct ath10k *ar,
-							    bool mark_read)
-{
-	struct ath10k_fw_crash_data *crash_data = ar->debug.fw_crash_data;
-	struct ath10k_ce_crash_hdr *ce_hdr;
-	struct ath10k_dump_file_data *dump_data;
-	struct ath10k_tlv_dump_data *dump_tlv;
-	size_t hdr_len = sizeof(*dump_data);
-	size_t len, sofar = 0;
-	unsigned char *buf;
-
-	len = hdr_len;
-	len += sizeof(*dump_tlv) + sizeof(crash_data->registers);
-	len += sizeof(*dump_tlv) + sizeof(*ce_hdr) +
-		CE_COUNT * sizeof(ce_hdr->entries[0]);
-
-	sofar += hdr_len;
-
-	/* This is going to get big when we start dumping FW RAM and such,
-	 * so go ahead and use vmalloc.
-	 */
-	buf = vzalloc(len);
-	if (!buf)
-		return NULL;
-
-	spin_lock_bh(&ar->data_lock);
-
-	if (!crash_data->crashed_since_read) {
-		spin_unlock_bh(&ar->data_lock);
-		vfree(buf);
-		return NULL;
-	}
-
-	dump_data = (struct ath10k_dump_file_data *)(buf);
-	strlcpy(dump_data->df_magic, "ATH10K-FW-DUMP",
-		sizeof(dump_data->df_magic));
-	dump_data->len = cpu_to_le32(len);
-
-	dump_data->version = cpu_to_le32(ATH10K_FW_CRASH_DUMP_VERSION);
-
-	guid_copy(&dump_data->guid, &crash_data->guid);
-	dump_data->chip_id = cpu_to_le32(ar->chip_id);
-	dump_data->bus_type = cpu_to_le32(0);
-	dump_data->target_version = cpu_to_le32(ar->target_version);
-	dump_data->fw_version_major = cpu_to_le32(ar->fw_version_major);
-	dump_data->fw_version_minor = cpu_to_le32(ar->fw_version_minor);
-	dump_data->fw_version_release = cpu_to_le32(ar->fw_version_release);
-	dump_data->fw_version_build = cpu_to_le32(ar->fw_version_build);
-	dump_data->phy_capability = cpu_to_le32(ar->phy_capability);
-	dump_data->hw_min_tx_power = cpu_to_le32(ar->hw_min_tx_power);
-	dump_data->hw_max_tx_power = cpu_to_le32(ar->hw_max_tx_power);
-	dump_data->ht_cap_info = cpu_to_le32(ar->ht_cap_info);
-	dump_data->vht_cap_info = cpu_to_le32(ar->vht_cap_info);
-	dump_data->num_rf_chains = cpu_to_le32(ar->num_rf_chains);
-
-	strlcpy(dump_data->fw_ver, ar->hw->wiphy->fw_version,
-		sizeof(dump_data->fw_ver));
-
-	dump_data->kernel_ver_code = 0;
-	strlcpy(dump_data->kernel_ver, init_utsname()->release,
-		sizeof(dump_data->kernel_ver));
-
-	dump_data->tv_sec = cpu_to_le64(crash_data->timestamp.tv_sec);
-	dump_data->tv_nsec = cpu_to_le64(crash_data->timestamp.tv_nsec);
-
-	/* Gather crash-dump */
-	dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
-	dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_REGISTERS);
-	dump_tlv->tlv_len = cpu_to_le32(sizeof(crash_data->registers));
-	memcpy(dump_tlv->tlv_data, &crash_data->registers,
-	       sizeof(crash_data->registers));
-	sofar += sizeof(*dump_tlv) + sizeof(crash_data->registers);
-
-	dump_tlv = (struct ath10k_tlv_dump_data *)(buf + sofar);
-	dump_tlv->type = cpu_to_le32(ATH10K_FW_CRASH_DUMP_CE_DATA);
-	dump_tlv->tlv_len = cpu_to_le32(sizeof(*ce_hdr) +
-					CE_COUNT * sizeof(ce_hdr->entries[0]));
-	ce_hdr = (struct ath10k_ce_crash_hdr *)(dump_tlv->tlv_data);
-	ce_hdr->ce_count = cpu_to_le32(CE_COUNT);
-	memset(ce_hdr->reserved, 0, sizeof(ce_hdr->reserved));
-	memcpy(ce_hdr->entries, crash_data->ce_crash_data,
-	       CE_COUNT * sizeof(ce_hdr->entries[0]));
-	sofar += sizeof(*dump_tlv) + sizeof(*ce_hdr) +
-		 CE_COUNT * sizeof(ce_hdr->entries[0]);
-
-	ar->debug.fw_crash_data->crashed_since_read = !mark_read;
-
-	spin_unlock_bh(&ar->data_lock);
-
-	return dump_data;
-}
-
-int ath10k_debug_fw_devcoredump(struct ath10k *ar)
-{
-	struct ath10k_dump_file_data *dump;
-	void *dump_ptr;
-	u32 dump_len;
-
-	/* To keep the dump file available also for debugfs don't mark the
-	 * file read, only debugfs should do that.
-	 */
-	dump = ath10k_build_dump_file(ar, false);
-	if (!dump) {
-		ath10k_warn(ar, "no crash dump data found for devcoredump");
-		return -ENODATA;
-	}
-
-	/* Make a copy of the dump file for dev_coredumpv() as during the
-	 * transition period we need to own the original file. Once
-	 * fw_crash_dump debugfs file is removed no need to have a copy
-	 * anymore.
-	 */
-	dump_len = le32_to_cpu(dump->len);
-	dump_ptr = vzalloc(dump_len);
-
-	if (!dump_ptr)
-		return -ENOMEM;
-
-	memcpy(dump_ptr, dump, dump_len);
-
-	dev_coredumpv(ar->dev, dump_ptr, dump_len, GFP_KERNEL);
-
-	return 0;
-}
-
-static int ath10k_fw_crash_dump_open(struct inode *inode, struct file *file)
-{
-	struct ath10k *ar = inode->i_private;
-	struct ath10k_dump_file_data *dump;
-
-	ath10k_warn(ar, "fw_crash_dump debugfs file is deprecated, please use /sys/class/devcoredump instead.");
-
-	dump = ath10k_build_dump_file(ar, true);
-	if (!dump)
-		return -ENODATA;
-
-	file->private_data = dump;
-
-	return 0;
-}
-
-static ssize_t ath10k_fw_crash_dump_read(struct file *file,
-					 char __user *user_buf,
-					 size_t count, loff_t *ppos)
-{
-	struct ath10k_dump_file_data *dump_file = file->private_data;
-
-	return simple_read_from_buffer(user_buf, count, ppos,
-				       dump_file,
-				       le32_to_cpu(dump_file->len));
-}
-
-static int ath10k_fw_crash_dump_release(struct inode *inode,
-					struct file *file)
-{
-	vfree(file->private_data);
-
-	return 0;
-}
-
-static const struct file_operations fops_fw_crash_dump = {
-	.open = ath10k_fw_crash_dump_open,
-	.read = ath10k_fw_crash_dump_read,
-	.release = ath10k_fw_crash_dump_release,
-	.owner = THIS_MODULE,
-	.llseek = default_llseek,
-};
-
 static ssize_t ath10k_reg_addr_read(struct file *file,
 				    char __user *user_buf,
 				    size_t count, loff_t *ppos)
@@ -1950,14 +1693,14 @@ int ath10k_debug_start(struct ath10k *ar)
 				    ret);
 	}
 
-	if (ar->debug.pktlog_filter) {
+	if (ar->pktlog_filter) {
 		ret = ath10k_wmi_pdev_pktlog_enable(ar,
-						    ar->debug.pktlog_filter);
+						    ar->pktlog_filter);
 		if (ret)
 			/* not serious */
 			ath10k_warn(ar,
 				    "failed to enable pktlog filter %x: %d\n",
-				    ar->debug.pktlog_filter, ret);
+				    ar->pktlog_filter, ret);
 	} else {
 		ret = ath10k_wmi_pdev_pktlog_disable(ar);
 		if (ret)
@@ -2097,12 +1840,12 @@ static ssize_t ath10k_write_pktlog_filter(struct file *file,
 	mutex_lock(&ar->conf_mutex);
 
 	if (ar->state != ATH10K_STATE_ON) {
-		ar->debug.pktlog_filter = filter;
+		ar->pktlog_filter = filter;
 		ret = count;
 		goto out;
 	}
 
-	if (filter == ar->debug.pktlog_filter) {
+	if (filter == ar->pktlog_filter) {
 		ret = count;
 		goto out;
 	}
@@ -2111,7 +1854,7 @@ static ssize_t ath10k_write_pktlog_filter(struct file *file,
 		ret = ath10k_wmi_pdev_pktlog_enable(ar, filter);
 		if (ret) {
 			ath10k_warn(ar, "failed to enable pktlog filter %x: %d\n",
-				    ar->debug.pktlog_filter, ret);
+				    ar->pktlog_filter, ret);
 			goto out;
 		}
 	} else {
@@ -2122,7 +1865,7 @@ static ssize_t ath10k_write_pktlog_filter(struct file *file,
 		}
 	}
 
-	ar->debug.pktlog_filter = filter;
+	ar->pktlog_filter = filter;
 	ret = count;
 
 out:
@@ -2139,7 +1882,7 @@ static ssize_t ath10k_read_pktlog_filter(struct file *file, char __user *ubuf,
 
 	mutex_lock(&ar->conf_mutex);
 	len = scnprintf(buf, sizeof(buf) - len, "%08x\n",
-			ar->debug.pktlog_filter);
+			ar->pktlog_filter);
 	mutex_unlock(&ar->conf_mutex);
 
 	return simple_read_from_buffer(ubuf, count, ppos, buf, len);
@@ -2402,10 +2145,6 @@ static const struct file_operations fops_fw_checksums = {
 
 int ath10k_debug_create(struct ath10k *ar)
 {
-	ar->debug.fw_crash_data = vzalloc(sizeof(*ar->debug.fw_crash_data));
-	if (!ar->debug.fw_crash_data)
-		return -ENOMEM;
-
 	ar->debug.cal_data = vzalloc(ATH10K_DEBUG_CAL_DATA_LEN);
 	if (!ar->debug.cal_data)
 		return -ENOMEM;
@@ -2420,9 +2159,6 @@ int ath10k_debug_create(struct ath10k *ar)
 
 void ath10k_debug_destroy(struct ath10k *ar)
 {
-	vfree(ar->debug.fw_crash_data);
-	ar->debug.fw_crash_data = NULL;
-
 	vfree(ar->debug.cal_data);
 	ar->debug.cal_data = NULL;
 
@@ -2460,9 +2196,6 @@ int ath10k_debug_register(struct ath10k *ar)
 	debugfs_create_file("simulate_fw_crash", 0600, ar->debug.debugfs_phy, ar,
 			    &fops_simulate_fw_crash);
 
-	debugfs_create_file("fw_crash_dump", 0400, ar->debug.debugfs_phy, ar,
-			    &fops_fw_crash_dump);
-
 	debugfs_create_file("reg_addr", 0600, ar->debug.debugfs_phy, ar,
 			    &fops_reg_addr);
 
diff --git a/drivers/net/wireless/ath/ath10k/debug.h b/drivers/net/wireless/ath/ath10k/debug.h
index 548ad5483a4a..e54308889e59 100644
--- a/drivers/net/wireless/ath/ath10k/debug.h
+++ b/drivers/net/wireless/ath/ath10k/debug.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -42,6 +42,7 @@ enum ath10k_debug_mask {
 	ATH10K_DBG_SDIO_DUMP	= 0x00020000,
 	ATH10K_DBG_USB		= 0x00040000,
 	ATH10K_DBG_USB_BULK	= 0x00080000,
+	ATH10K_DBG_SNOC		= 0x00100000,
 	ATH10K_DBG_ANY		= 0xffffffff,
 };
 
@@ -51,7 +52,8 @@ enum ath10k_pktlog_filter {
 	ATH10K_PKTLOG_RCFIND     = 0x000000004,
 	ATH10K_PKTLOG_RCUPDATE   = 0x000000008,
 	ATH10K_PKTLOG_DBG_PRINT  = 0x000000010,
-	ATH10K_PKTLOG_ANY        = 0x00000001f,
+	ATH10K_PKTLOG_PEER_STATS = 0x000000040,
+	ATH10K_PKTLOG_ANY        = 0x00000005f,
 };
 
 enum ath10k_dbg_aggr_mode {
@@ -60,6 +62,21 @@ enum ath10k_dbg_aggr_mode {
 	ATH10K_DBG_AGGR_MODE_MAX,
 };
 
+/* Types of packet log events */
+enum ath_pktlog_type {
+	ATH_PKTLOG_TYPE_TX_CTRL = 1,
+	ATH_PKTLOG_TYPE_TX_STAT,
+};
+
+struct ath10k_pktlog_hdr {
+	__le16 flags;
+	__le16 missed_cnt;
+	__le16 log_type; /* Type of log information foll this header */
+	__le16 size; /* Size of variable length log information in bytes */
+	__le32 timestamp;
+	u8 payload[0];
+} __packed;
+
 /* FIXME: How to calculate the buffer size sanely? */
 #define ATH10K_FW_STATS_BUF_SIZE (1024 * 1024)
 
@@ -84,13 +101,8 @@ void ath10k_debug_unregister(struct ath10k *ar);
 void ath10k_debug_fw_stats_process(struct ath10k *ar, struct sk_buff *skb);
 void ath10k_debug_tpc_stats_process(struct ath10k *ar,
 				    struct ath10k_tpc_stats *tpc_stats);
-struct ath10k_fw_crash_data *
-ath10k_debug_get_new_fw_crash_data(struct ath10k *ar);
-
 void ath10k_debug_dbglog_add(struct ath10k *ar, u8 *buffer, int len);
 
-int ath10k_debug_fw_devcoredump(struct ath10k *ar);
-
 #define ATH10K_DFS_STAT_INC(ar, c) (ar->debug.dfs_stats.c++)
 
 void ath10k_debug_get_et_strings(struct ieee80211_hw *hw,
@@ -157,12 +169,6 @@ static inline void ath10k_debug_dbglog_add(struct ath10k *ar, u8 *buffer,
 {
 }
 
-static inline struct ath10k_fw_crash_data *
-ath10k_debug_get_new_fw_crash_data(struct ath10k *ar)
-{
-	return NULL;
-}
-
 static inline u64 ath10k_debug_get_fw_dbglog_mask(struct ath10k *ar)
 {
 	return 0;
@@ -173,11 +179,6 @@ static inline u32 ath10k_debug_get_fw_dbglog_level(struct ath10k *ar)
 	return 0;
 }
 
-static inline int ath10k_debug_fw_devcoredump(struct ath10k *ar)
-{
-	return 0;
-}
-
 #define ATH10K_DFS_STAT_INC(ar, c) do { } while (0)
 
 #define ath10k_debug_get_et_strings NULL
@@ -190,9 +191,6 @@ void ath10k_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			    struct ieee80211_sta *sta, struct dentry *dir);
 void ath10k_sta_update_rx_duration(struct ath10k *ar,
 				   struct ath10k_fw_stats *stats);
-void ath10k_sta_statistics(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-			   struct ieee80211_sta *sta,
-			   struct station_info *sinfo);
 #else
 static inline
 void ath10k_sta_update_rx_duration(struct ath10k *ar,
diff --git a/drivers/net/wireless/ath/ath10k/debugfs_sta.c b/drivers/net/wireless/ath/ath10k/debugfs_sta.c
index d59ac6b83340..b260b09dd4d3 100644
--- a/drivers/net/wireless/ath/ath10k/debugfs_sta.c
+++ b/drivers/net/wireless/ath/ath10k/debugfs_sta.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -65,33 +65,6 @@ void ath10k_sta_update_rx_duration(struct ath10k *ar,
 		ath10k_sta_update_stats_rx_duration(ar, stats);
 }
 
-void ath10k_sta_statistics(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-			   struct ieee80211_sta *sta,
-			   struct station_info *sinfo)
-{
-	struct ath10k_sta *arsta = (struct ath10k_sta *)sta->drv_priv;
-	struct ath10k *ar = arsta->arvif->ar;
-
-	if (!ath10k_peer_stats_enabled(ar))
-		return;
-
-	sinfo->rx_duration = arsta->rx_duration;
-	sinfo->filled |= 1ULL << NL80211_STA_INFO_RX_DURATION;
-
-	if (!arsta->txrate.legacy && !arsta->txrate.nss)
-		return;
-
-	if (arsta->txrate.legacy) {
-		sinfo->txrate.legacy = arsta->txrate.legacy;
-	} else {
-		sinfo->txrate.mcs = arsta->txrate.mcs;
-		sinfo->txrate.nss = arsta->txrate.nss;
-		sinfo->txrate.bw = arsta->txrate.bw;
-	}
-	sinfo->txrate.flags = arsta->txrate.flags;
-	sinfo->filled |= 1ULL << NL80211_STA_INFO_TX_BITRATE;
-}
-
 static ssize_t ath10k_dbg_sta_read_aggr_mode(struct file *file,
 					     char __user *user_buf,
 					     size_t count, loff_t *ppos)
diff --git a/drivers/net/wireless/ath/ath10k/hif.h b/drivers/net/wireless/ath/ath10k/hif.h
index 6679dd9cfd12..6da4e3369c5a 100644
--- a/drivers/net/wireless/ath/ath10k/hif.h
+++ b/drivers/net/wireless/ath/ath10k/hif.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2015,2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/htc.c b/drivers/net/wireless/ath/ath10k/htc.c
index e5c80f582ff5..492dc5b4bbf2 100644
--- a/drivers/net/wireless/ath/ath10k/htc.c
+++ b/drivers/net/wireless/ath/ath10k/htc.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/htc.h b/drivers/net/wireless/ath/ath10k/htc.h
index 24663b07eeac..a2f8814b3e53 100644
--- a/drivers/net/wireless/ath/ath10k/htc.h
+++ b/drivers/net/wireless/ath/ath10k/htc.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/htt.c b/drivers/net/wireless/ath/ath10k/htt.c
index cd160b16db1e..625198dea18b 100644
--- a/drivers/net/wireless/ath/ath10k/htt.c
+++ b/drivers/net/wireless/ath/ath10k/htt.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -207,6 +207,9 @@ int ath10k_htt_init(struct ath10k *ar)
 		WARN_ON(1);
 		return -EINVAL;
 	}
+	ath10k_htt_set_tx_ops(htt);
+	ath10k_htt_set_rx_ops(htt);
+
 	return 0;
 }
 
@@ -254,11 +257,11 @@ int ath10k_htt_setup(struct ath10k_htt *htt)
 		return status;
 	}
 
-	status = ath10k_htt_send_frag_desc_bank_cfg(htt);
+	status = htt->tx_ops->htt_send_frag_desc_bank_cfg(htt);
 	if (status)
 		return status;
 
-	status = ath10k_htt_send_rx_ring_cfg_ll(htt);
+	status = htt->tx_ops->htt_send_rx_ring_cfg(htt);
 	if (status) {
 		ath10k_warn(ar, "failed to setup rx ring: %d\n",
 			    status);
diff --git a/drivers/net/wireless/ath/ath10k/htt.h b/drivers/net/wireless/ath/ath10k/htt.h
index 6305308422c4..8cc2a8b278e4 100644
--- a/drivers/net/wireless/ath/ath10k/htt.h
+++ b/drivers/net/wireless/ath/ath10k/htt.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -107,6 +107,14 @@ struct htt_msdu_ext_desc {
 	struct htt_data_tx_desc_frag frags[6];
 };
 
+struct htt_msdu_ext_desc_64 {
+	__le32 tso_flag[5];
+	__le16 ip_identification;
+	u8 flags;
+	u8 reserved;
+	struct htt_data_tx_desc_frag frags[6];
+};
+
 #define	HTT_MSDU_EXT_DESC_FLAG_IPV4_CSUM_ENABLE		BIT(0)
 #define	HTT_MSDU_EXT_DESC_FLAG_UDP_IPV4_CSUM_ENABLE	BIT(1)
 #define	HTT_MSDU_EXT_DESC_FLAG_UDP_IPV6_CSUM_ENABLE	BIT(2)
@@ -179,6 +187,22 @@ struct htt_data_tx_desc {
 	u8 prefetch[0]; /* start of frame, for FW classification engine */
 } __packed;
 
+struct htt_data_tx_desc_64 {
+	u8 flags0; /* %HTT_DATA_TX_DESC_FLAGS0_ */
+	__le16 flags1; /* %HTT_DATA_TX_DESC_FLAGS1_ */
+	__le16 len;
+	__le16 id;
+	__le64 frags_paddr;
+	union {
+		__le32 peerid;
+		struct {
+			__le16 peerid;
+			__le16 freq;
+		} __packed offchan_tx;
+	} __packed;
+	u8 prefetch[0]; /* start of frame, for FW classification engine */
+} __packed;
+
 enum htt_rx_ring_flags {
 	HTT_RX_RING_FLAGS_MAC80211_HDR = 1 << 0,
 	HTT_RX_RING_FLAGS_MSDU_PAYLOAD = 1 << 1,
@@ -200,8 +224,11 @@ enum htt_rx_ring_flags {
 
 #define HTT_RX_RING_SIZE_MIN 128
 #define HTT_RX_RING_SIZE_MAX 2048
+#define HTT_RX_RING_SIZE HTT_RX_RING_SIZE_MAX
+#define HTT_RX_RING_FILL_LEVEL (((HTT_RX_RING_SIZE) / 2) - 1)
+#define HTT_RX_RING_FILL_LEVEL_DUAL_MAC (HTT_RX_RING_SIZE - 1)
 
-struct htt_rx_ring_setup_ring {
+struct htt_rx_ring_setup_ring32 {
 	__le32 fw_idx_shadow_reg_paddr;
 	__le32 rx_ring_base_paddr;
 	__le16 rx_ring_len; /* in 4-byte words */
@@ -222,14 +249,40 @@ struct htt_rx_ring_setup_ring {
 	__le16 frag_info_offset;
 } __packed;
 
+struct htt_rx_ring_setup_ring64 {
+	__le64 fw_idx_shadow_reg_paddr;
+	__le64 rx_ring_base_paddr;
+	__le16 rx_ring_len; /* in 4-byte words */
+	__le16 rx_ring_bufsize; /* rx skb size - in bytes */
+	__le16 flags; /* %HTT_RX_RING_FLAGS_ */
+	__le16 fw_idx_init_val;
+
+	/* the following offsets are in 4-byte units */
+	__le16 mac80211_hdr_offset;
+	__le16 msdu_payload_offset;
+	__le16 ppdu_start_offset;
+	__le16 ppdu_end_offset;
+	__le16 mpdu_start_offset;
+	__le16 mpdu_end_offset;
+	__le16 msdu_start_offset;
+	__le16 msdu_end_offset;
+	__le16 rx_attention_offset;
+	__le16 frag_info_offset;
+} __packed;
+
 struct htt_rx_ring_setup_hdr {
 	u8 num_rings; /* supported values: 1, 2 */
 	__le16 rsvd0;
 } __packed;
 
-struct htt_rx_ring_setup {
+struct htt_rx_ring_setup_32 {
 	struct htt_rx_ring_setup_hdr hdr;
-	struct htt_rx_ring_setup_ring rings[0];
+	struct htt_rx_ring_setup_ring32 rings[0];
+} __packed;
+
+struct htt_rx_ring_setup_64 {
+	struct htt_rx_ring_setup_hdr hdr;
+	struct htt_rx_ring_setup_ring64 rings[0];
 } __packed;
 
 /*
@@ -855,13 +908,23 @@ struct htt_rx_in_ord_msdu_desc {
 	u8 reserved;
 } __packed;
 
+struct htt_rx_in_ord_msdu_desc_ext {
+	__le64 msdu_paddr;
+	__le16 msdu_len;
+	u8 fw_desc;
+	u8 reserved;
+} __packed;
+
 struct htt_rx_in_ord_ind {
 	u8 info;
 	__le16 peer_id;
 	u8 vdev_id;
 	u8 reserved;
 	__le16 msdu_count;
-	struct htt_rx_in_ord_msdu_desc msdu_descs[0];
+	union {
+		struct htt_rx_in_ord_msdu_desc msdu_descs32[0];
+		struct htt_rx_in_ord_msdu_desc_ext msdu_descs64[0];
+	} __packed;
 } __packed;
 
 #define HTT_RX_IN_ORD_IND_INFO_TID_MASK		0x0000001f
@@ -1351,7 +1414,7 @@ struct htt_q_state_conf {
 	u8 pad[2];
 } __packed;
 
-struct htt_frag_desc_bank_cfg {
+struct htt_frag_desc_bank_cfg32 {
 	u8 info; /* HTT_FRAG_DESC_BANK_CFG_INFO_ */
 	u8 num_banks;
 	u8 desc_size;
@@ -1360,6 +1423,15 @@ struct htt_frag_desc_bank_cfg {
 	struct htt_q_state_conf q_state;
 } __packed;
 
+struct htt_frag_desc_bank_cfg64 {
+	u8 info; /* HTT_FRAG_DESC_BANK_CFG_INFO_ */
+	u8 num_banks;
+	u8 desc_size;
+	__le64 bank_base_addrs[HTT_FRAG_DESC_BANK_MAX];
+	struct htt_frag_desc_bank_id bank_id[HTT_FRAG_DESC_BANK_MAX];
+	struct htt_q_state_conf q_state;
+} __packed;
+
 #define HTT_TX_Q_STATE_ENTRY_COEFFICIENT	128
 #define HTT_TX_Q_STATE_ENTRY_FACTOR_MASK	0x3f
 #define HTT_TX_Q_STATE_ENTRY_FACTOR_LSB		0
@@ -1497,6 +1569,23 @@ struct htt_peer_tx_stats {
 	u8 payload[0];
 } __packed;
 
+#define ATH10K_10_2_TX_STATS_OFFSET	136
+#define PEER_STATS_FOR_NO_OF_PPDUS	4
+
+struct ath10k_10_2_peer_tx_stats {
+	u8 ratecode[PEER_STATS_FOR_NO_OF_PPDUS];
+	u8 success_pkts[PEER_STATS_FOR_NO_OF_PPDUS];
+	__le16 success_bytes[PEER_STATS_FOR_NO_OF_PPDUS];
+	u8 retry_pkts[PEER_STATS_FOR_NO_OF_PPDUS];
+	__le16 retry_bytes[PEER_STATS_FOR_NO_OF_PPDUS];
+	u8 failed_pkts[PEER_STATS_FOR_NO_OF_PPDUS];
+	__le16 failed_bytes[PEER_STATS_FOR_NO_OF_PPDUS];
+	u8 flags[PEER_STATS_FOR_NO_OF_PPDUS];
+	__le32 tx_duration;
+	u8 tx_ppdu_cnt;
+	u8 peer_id;
+} __packed;
+
 union htt_rx_pn_t {
 	/* WEP: 24-bit PN */
 	u32 pn24;
@@ -1514,11 +1603,13 @@ struct htt_cmd {
 		struct htt_ver_req ver_req;
 		struct htt_mgmt_tx_desc mgmt_tx;
 		struct htt_data_tx_desc data_tx;
-		struct htt_rx_ring_setup rx_setup;
+		struct htt_rx_ring_setup_32 rx_setup_32;
+		struct htt_rx_ring_setup_64 rx_setup_64;
 		struct htt_stats_req stats_req;
 		struct htt_oob_sync_req oob_sync_req;
 		struct htt_aggr_conf aggr_conf;
-		struct htt_frag_desc_bank_cfg frag_desc_bank_cfg;
+		struct htt_frag_desc_bank_cfg32 frag_desc_bank_cfg32;
+		struct htt_frag_desc_bank_cfg64 frag_desc_bank_cfg64;
 		struct htt_tx_fetch_resp tx_fetch_resp;
 	};
 } __packed;
@@ -1576,13 +1667,20 @@ struct htt_peer_unmap_event {
 	u16 peer_id;
 };
 
-struct ath10k_htt_txbuf {
+struct ath10k_htt_txbuf_32 {
 	struct htt_data_tx_desc_frag frags[2];
 	struct ath10k_htc_hdr htc_hdr;
 	struct htt_cmd_hdr cmd_hdr;
 	struct htt_data_tx_desc cmd_tx;
 } __packed;
 
+struct ath10k_htt_txbuf_64 {
+	struct htt_data_tx_desc_frag frags[2];
+	struct ath10k_htc_hdr htc_hdr;
+	struct htt_cmd_hdr cmd_hdr;
+	struct htt_data_tx_desc_64 cmd_tx;
+} __packed;
+
 struct ath10k_htt {
 	struct ath10k *ar;
 	enum ath10k_htc_ep_id eid;
@@ -1627,7 +1725,10 @@ struct ath10k_htt {
 		 * rx buffers the host SW provides for the MAC HW to
 		 * fill.
 		 */
-		__le32 *paddrs_ring;
+		union {
+			__le64 *paddrs_ring_64;
+			__le32 *paddrs_ring_32;
+		};
 
 		/*
 		 * Base address of ring, as a "physical" device address
@@ -1695,7 +1796,7 @@ struct ath10k_htt {
 	/* This is used to group tx/rx completions separately and process them
 	 * in batches to reduce cache stalls
 	 */
-	struct sk_buff_head rx_compl_q;
+	struct sk_buff_head rx_msdus_q;
 	struct sk_buff_head rx_in_ord_compl_q;
 	struct sk_buff_head tx_fetch_ind_q;
 
@@ -1704,12 +1805,20 @@ struct ath10k_htt {
 
 	struct {
 		dma_addr_t paddr;
-		struct htt_msdu_ext_desc *vaddr;
+		union {
+			struct htt_msdu_ext_desc *vaddr_desc_32;
+			struct htt_msdu_ext_desc_64 *vaddr_desc_64;
+		};
+		size_t size;
 	} frag_desc;
 
 	struct {
 		dma_addr_t paddr;
-		struct ath10k_htt_txbuf *vaddr;
+		union {
+			struct ath10k_htt_txbuf_32 *vaddr_txbuff_32;
+			struct ath10k_htt_txbuf_64 *vaddr_txbuff_64;
+		};
+		size_t size;
 	} txbuf;
 
 	struct {
@@ -1724,6 +1833,28 @@ struct ath10k_htt {
 	} tx_q_state;
 
 	bool tx_mem_allocated;
+	const struct ath10k_htt_tx_ops *tx_ops;
+	const struct ath10k_htt_rx_ops *rx_ops;
+};
+
+struct ath10k_htt_tx_ops {
+	int (*htt_send_rx_ring_cfg)(struct ath10k_htt *htt);
+	int (*htt_send_frag_desc_bank_cfg)(struct ath10k_htt *htt);
+	int (*htt_alloc_frag_desc)(struct ath10k_htt *htt);
+	void (*htt_free_frag_desc)(struct ath10k_htt *htt);
+	int (*htt_tx)(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
+		      struct sk_buff *msdu);
+	int (*htt_alloc_txbuff)(struct ath10k_htt *htt);
+	void (*htt_free_txbuff)(struct ath10k_htt *htt);
+};
+
+struct ath10k_htt_rx_ops {
+	size_t (*htt_get_rx_ring_size)(struct ath10k_htt *htt);
+	void (*htt_config_paddrs_ring)(struct ath10k_htt *htt, void *vaddr);
+	void (*htt_set_paddrs_ring)(struct ath10k_htt *htt, dma_addr_t paddr,
+				    int idx);
+	void* (*htt_get_vaddr_ring)(struct ath10k_htt *htt);
+	void (*htt_reset_paddrs_ring)(struct ath10k_htt *htt, int idx);
 };
 
 #define RX_HTT_HDR_STATUS_LEN 64
@@ -1803,8 +1934,6 @@ void ath10k_htt_htc_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb);
 bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb);
 int ath10k_htt_h2t_ver_req_msg(struct ath10k_htt *htt);
 int ath10k_htt_h2t_stats_req(struct ath10k_htt *htt, u8 mask, u64 cookie);
-int ath10k_htt_send_frag_desc_bank_cfg(struct ath10k_htt *htt);
-int ath10k_htt_send_rx_ring_cfg_ll(struct ath10k_htt *htt);
 int ath10k_htt_h2t_aggr_cfg_msg(struct ath10k_htt *htt,
 				u8 max_subfrms_ampdu,
 				u8 max_subfrms_amsdu);
@@ -1829,11 +1958,9 @@ int ath10k_htt_tx_mgmt_inc_pending(struct ath10k_htt *htt, bool is_mgmt,
 int ath10k_htt_tx_alloc_msdu_id(struct ath10k_htt *htt, struct sk_buff *skb);
 void ath10k_htt_tx_free_msdu_id(struct ath10k_htt *htt, u16 msdu_id);
 int ath10k_htt_mgmt_tx(struct ath10k_htt *htt, struct sk_buff *msdu);
-int ath10k_htt_tx(struct ath10k_htt *htt,
-		  enum ath10k_hw_txrx_mode txmode,
-		  struct sk_buff *msdu);
 void ath10k_htt_rx_pktlog_completion_handler(struct ath10k *ar,
 					     struct sk_buff *skb);
 int ath10k_htt_txrx_compl_task(struct ath10k *ar, int budget);
-
+void ath10k_htt_set_tx_ops(struct ath10k_htt *htt);
+void ath10k_htt_set_rx_ops(struct ath10k_htt *htt);
 #endif
diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c
index 7d295ee71534..6d96f9560950 100644
--- a/drivers/net/wireless/ath/ath10k/htt_rx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_rx.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -25,9 +25,6 @@
 
 #include <linux/log2.h>
 
-#define HTT_RX_RING_SIZE HTT_RX_RING_SIZE_MAX
-#define HTT_RX_RING_FILL_LEVEL (((HTT_RX_RING_SIZE) / 2) - 1)
-
 /* when under memory pressure rx ring refill may fail and needs a retry */
 #define HTT_RX_RING_REFILL_RETRY_MS 50
 
@@ -36,7 +33,7 @@
 static int ath10k_htt_rx_get_csum_state(struct sk_buff *skb);
 
 static struct sk_buff *
-ath10k_htt_rx_find_skb_paddr(struct ath10k *ar, u32 paddr)
+ath10k_htt_rx_find_skb_paddr(struct ath10k *ar, u64 paddr)
 {
 	struct ath10k_skb_rxcb *rxcb;
 
@@ -84,6 +81,60 @@ static void ath10k_htt_rx_ring_free(struct ath10k_htt *htt)
 	       htt->rx_ring.size * sizeof(htt->rx_ring.netbufs_ring[0]));
 }
 
+static size_t ath10k_htt_get_rx_ring_size_32(struct ath10k_htt *htt)
+{
+	return htt->rx_ring.size * sizeof(htt->rx_ring.paddrs_ring_32);
+}
+
+static size_t ath10k_htt_get_rx_ring_size_64(struct ath10k_htt *htt)
+{
+	return htt->rx_ring.size * sizeof(htt->rx_ring.paddrs_ring_64);
+}
+
+static void ath10k_htt_config_paddrs_ring_32(struct ath10k_htt *htt,
+					     void *vaddr)
+{
+	htt->rx_ring.paddrs_ring_32 = vaddr;
+}
+
+static void ath10k_htt_config_paddrs_ring_64(struct ath10k_htt *htt,
+					     void *vaddr)
+{
+	htt->rx_ring.paddrs_ring_64 = vaddr;
+}
+
+static void ath10k_htt_set_paddrs_ring_32(struct ath10k_htt *htt,
+					  dma_addr_t paddr, int idx)
+{
+	htt->rx_ring.paddrs_ring_32[idx] = __cpu_to_le32(paddr);
+}
+
+static void ath10k_htt_set_paddrs_ring_64(struct ath10k_htt *htt,
+					  dma_addr_t paddr, int idx)
+{
+	htt->rx_ring.paddrs_ring_64[idx] = __cpu_to_le64(paddr);
+}
+
+static void ath10k_htt_reset_paddrs_ring_32(struct ath10k_htt *htt, int idx)
+{
+	htt->rx_ring.paddrs_ring_32[idx] = 0;
+}
+
+static void ath10k_htt_reset_paddrs_ring_64(struct ath10k_htt *htt, int idx)
+{
+	htt->rx_ring.paddrs_ring_64[idx] = 0;
+}
+
+static void *ath10k_htt_get_vaddr_ring_32(struct ath10k_htt *htt)
+{
+	return (void *)htt->rx_ring.paddrs_ring_32;
+}
+
+static void *ath10k_htt_get_vaddr_ring_64(struct ath10k_htt *htt)
+{
+	return (void *)htt->rx_ring.paddrs_ring_64;
+}
+
 static int __ath10k_htt_rx_ring_fill_n(struct ath10k_htt *htt, int num)
 {
 	struct htt_rx_desc *rx_desc;
@@ -129,13 +180,13 @@ static int __ath10k_htt_rx_ring_fill_n(struct ath10k_htt *htt, int num)
 		rxcb = ATH10K_SKB_RXCB(skb);
 		rxcb->paddr = paddr;
 		htt->rx_ring.netbufs_ring[idx] = skb;
-		htt->rx_ring.paddrs_ring[idx] = __cpu_to_le32(paddr);
+		htt->rx_ops->htt_set_paddrs_ring(htt, paddr, idx);
 		htt->rx_ring.fill_cnt++;
 
 		if (htt->rx_ring.in_ord_rx) {
 			hash_add(htt->rx_ring.skb_table,
 				 &ATH10K_SKB_RXCB(skb)->hlist,
-				 (u32)paddr);
+				 paddr);
 		}
 
 		num--;
@@ -227,16 +278,15 @@ void ath10k_htt_rx_free(struct ath10k_htt *htt)
 {
 	del_timer_sync(&htt->rx_ring.refill_retry_timer);
 
-	skb_queue_purge(&htt->rx_compl_q);
+	skb_queue_purge(&htt->rx_msdus_q);
 	skb_queue_purge(&htt->rx_in_ord_compl_q);
 	skb_queue_purge(&htt->tx_fetch_ind_q);
 
 	ath10k_htt_rx_ring_free(htt);
 
 	dma_free_coherent(htt->ar->dev,
-			  (htt->rx_ring.size *
-			   sizeof(htt->rx_ring.paddrs_ring)),
-			  htt->rx_ring.paddrs_ring,
+			  htt->rx_ops->htt_get_rx_ring_size(htt),
+			  htt->rx_ops->htt_get_vaddr_ring(htt),
 			  htt->rx_ring.base_paddr);
 
 	dma_free_coherent(htt->ar->dev,
@@ -263,7 +313,7 @@ static inline struct sk_buff *ath10k_htt_rx_netbuf_pop(struct ath10k_htt *htt)
 	idx = htt->rx_ring.sw_rd_idx.msdu_payld;
 	msdu = htt->rx_ring.netbufs_ring[idx];
 	htt->rx_ring.netbufs_ring[idx] = NULL;
-	htt->rx_ring.paddrs_ring[idx] = 0;
+	htt->rx_ops->htt_reset_paddrs_ring(htt, idx);
 
 	idx++;
 	idx &= htt->rx_ring.size_mask;
@@ -383,7 +433,7 @@ static int ath10k_htt_rx_amsdu_pop(struct ath10k_htt *htt,
 }
 
 static struct sk_buff *ath10k_htt_rx_pop_paddr(struct ath10k_htt *htt,
-					       u32 paddr)
+					       u64 paddr)
 {
 	struct ath10k *ar = htt->ar;
 	struct ath10k_skb_rxcb *rxcb;
@@ -408,12 +458,12 @@ static struct sk_buff *ath10k_htt_rx_pop_paddr(struct ath10k_htt *htt,
 	return msdu;
 }
 
-static int ath10k_htt_rx_pop_paddr_list(struct ath10k_htt *htt,
-					struct htt_rx_in_ord_ind *ev,
-					struct sk_buff_head *list)
+static int ath10k_htt_rx_pop_paddr32_list(struct ath10k_htt *htt,
+					  struct htt_rx_in_ord_ind *ev,
+					  struct sk_buff_head *list)
 {
 	struct ath10k *ar = htt->ar;
-	struct htt_rx_in_ord_msdu_desc *msdu_desc = ev->msdu_descs;
+	struct htt_rx_in_ord_msdu_desc *msdu_desc = ev->msdu_descs32;
 	struct htt_rx_desc *rxd;
 	struct sk_buff *msdu;
 	int msdu_count;
@@ -458,11 +508,60 @@ static int ath10k_htt_rx_pop_paddr_list(struct ath10k_htt *htt,
 	return 0;
 }
 
+static int ath10k_htt_rx_pop_paddr64_list(struct ath10k_htt *htt,
+					  struct htt_rx_in_ord_ind *ev,
+					  struct sk_buff_head *list)
+{
+	struct ath10k *ar = htt->ar;
+	struct htt_rx_in_ord_msdu_desc_ext *msdu_desc = ev->msdu_descs64;
+	struct htt_rx_desc *rxd;
+	struct sk_buff *msdu;
+	int msdu_count;
+	bool is_offload;
+	u64 paddr;
+
+	lockdep_assert_held(&htt->rx_ring.lock);
+
+	msdu_count = __le16_to_cpu(ev->msdu_count);
+	is_offload = !!(ev->info & HTT_RX_IN_ORD_IND_INFO_OFFLOAD_MASK);
+
+	while (msdu_count--) {
+		paddr = __le64_to_cpu(msdu_desc->msdu_paddr);
+		msdu = ath10k_htt_rx_pop_paddr(htt, paddr);
+		if (!msdu) {
+			__skb_queue_purge(list);
+			return -ENOENT;
+		}
+
+		__skb_queue_tail(list, msdu);
+
+		if (!is_offload) {
+			rxd = (void *)msdu->data;
+
+			trace_ath10k_htt_rx_desc(ar, rxd, sizeof(*rxd));
+
+			skb_put(msdu, sizeof(*rxd));
+			skb_pull(msdu, sizeof(*rxd));
+			skb_put(msdu, __le16_to_cpu(msdu_desc->msdu_len));
+
+			if (!(__le32_to_cpu(rxd->attention.flags) &
+			      RX_ATTENTION_FLAGS_MSDU_DONE)) {
+				ath10k_warn(htt->ar, "tried to pop an incomplete frame, oops!\n");
+				return -EIO;
+			}
+		}
+
+		msdu_desc++;
+	}
+
+	return 0;
+}
+
 int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 {
 	struct ath10k *ar = htt->ar;
 	dma_addr_t paddr;
-	void *vaddr;
+	void *vaddr, *vaddr_ring;
 	size_t size;
 	struct timer_list *timer = &htt->rx_ring.refill_retry_timer;
 
@@ -473,7 +572,7 @@ int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 	 */
 	htt->rx_ring.size = HTT_RX_RING_SIZE;
 	htt->rx_ring.size_mask = htt->rx_ring.size - 1;
-	htt->rx_ring.fill_level = HTT_RX_RING_FILL_LEVEL;
+	htt->rx_ring.fill_level = ar->hw_params.rx_ring_fill_level;
 
 	if (!is_power_of_2(htt->rx_ring.size)) {
 		ath10k_warn(ar, "htt rx ring size is not power of 2\n");
@@ -486,13 +585,13 @@ int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 	if (!htt->rx_ring.netbufs_ring)
 		goto err_netbuf;
 
-	size = htt->rx_ring.size * sizeof(htt->rx_ring.paddrs_ring);
+	size = htt->rx_ops->htt_get_rx_ring_size(htt);
 
-	vaddr = dma_alloc_coherent(htt->ar->dev, size, &paddr, GFP_KERNEL);
-	if (!vaddr)
+	vaddr_ring = dma_alloc_coherent(htt->ar->dev, size, &paddr, GFP_KERNEL);
+	if (!vaddr_ring)
 		goto err_dma_ring;
 
-	htt->rx_ring.paddrs_ring = vaddr;
+	htt->rx_ops->htt_config_paddrs_ring(htt, vaddr_ring);
 	htt->rx_ring.base_paddr = paddr;
 
 	vaddr = dma_alloc_coherent(htt->ar->dev,
@@ -515,7 +614,7 @@ int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 	htt->rx_ring.sw_rd_idx.msdu_payld = 0;
 	hash_init(htt->rx_ring.skb_table);
 
-	skb_queue_head_init(&htt->rx_compl_q);
+	skb_queue_head_init(&htt->rx_msdus_q);
 	skb_queue_head_init(&htt->rx_in_ord_compl_q);
 	skb_queue_head_init(&htt->tx_fetch_ind_q);
 	atomic_set(&htt->num_mpdus_ready, 0);
@@ -526,9 +625,8 @@ int ath10k_htt_rx_alloc(struct ath10k_htt *htt)
 
 err_dma_idx:
 	dma_free_coherent(htt->ar->dev,
-			  (htt->rx_ring.size *
-			   sizeof(htt->rx_ring.paddrs_ring)),
-			  htt->rx_ring.paddrs_ring,
+			  htt->rx_ops->htt_get_rx_ring_size(htt),
+			  vaddr_ring,
 			  htt->rx_ring.base_paddr);
 err_dma_ring:
 	kfree(htt->rx_ring.netbufs_ring);
@@ -974,16 +1072,25 @@ static char *ath10k_get_tid(struct ieee80211_hdr *hdr, char *out, size_t size)
 	return out;
 }
 
-static void ath10k_process_rx(struct ath10k *ar,
-			      struct ieee80211_rx_status *rx_status,
-			      struct sk_buff *skb)
+static void ath10k_htt_rx_h_queue_msdu(struct ath10k *ar,
+				       struct ieee80211_rx_status *rx_status,
+				       struct sk_buff *skb)
+{
+	struct ieee80211_rx_status *status;
+
+	status = IEEE80211_SKB_RXCB(skb);
+	*status = *rx_status;
+
+	__skb_queue_tail(&ar->htt.rx_msdus_q, skb);
+}
+
+static void ath10k_process_rx(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct ieee80211_rx_status *status;
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	char tid[32];
 
 	status = IEEE80211_SKB_RXCB(skb);
-	*status = *rx_status;
 
 	ath10k_dbg(ar, ATH10K_DBG_DATA,
 		   "rx skb %pK len %u peer %pM %s %s sn %u %s%s%s%s%s%s %srate_idx %u vht_nss %u freq %u band %u flag 0x%x fcs-err %i mic-err %i amsdu-more %i\n",
@@ -1517,7 +1624,7 @@ static void ath10k_htt_rx_h_mpdu(struct ath10k *ar,
 	}
 }
 
-static void ath10k_htt_rx_h_deliver(struct ath10k *ar,
+static void ath10k_htt_rx_h_enqueue(struct ath10k *ar,
 				    struct sk_buff_head *amsdu,
 				    struct ieee80211_rx_status *status)
 {
@@ -1540,7 +1647,7 @@ static void ath10k_htt_rx_h_deliver(struct ath10k *ar,
 			status->flag |= RX_FLAG_ALLOW_SAME_PN;
 		}
 
-		ath10k_process_rx(ar, status, msdu);
+		ath10k_htt_rx_h_queue_msdu(ar, status, msdu);
 	}
 }
 
@@ -1652,7 +1759,7 @@ static int ath10k_htt_rx_handle_amsdu(struct ath10k_htt *htt)
 	struct ath10k *ar = htt->ar;
 	struct ieee80211_rx_status *rx_status = &htt->rx_status;
 	struct sk_buff_head amsdu;
-	int ret, num_msdus;
+	int ret;
 
 	__skb_queue_head_init(&amsdu);
 
@@ -1674,7 +1781,6 @@ static int ath10k_htt_rx_handle_amsdu(struct ath10k_htt *htt)
 		return ret;
 	}
 
-	num_msdus = skb_queue_len(&amsdu);
 	ath10k_htt_rx_h_ppdu(ar, &amsdu, rx_status, 0xffff);
 
 	/* only for ret = 1 indicates chained msdus */
@@ -1683,9 +1789,9 @@ static int ath10k_htt_rx_handle_amsdu(struct ath10k_htt *htt)
 
 	ath10k_htt_rx_h_filter(ar, &amsdu, rx_status);
 	ath10k_htt_rx_h_mpdu(ar, &amsdu, rx_status, true);
-	ath10k_htt_rx_h_deliver(ar, &amsdu, rx_status);
+	ath10k_htt_rx_h_enqueue(ar, &amsdu, rx_status);
 
-	return num_msdus;
+	return 0;
 }
 
 static void ath10k_htt_rx_proc_rx_ind(struct ath10k_htt *htt,
@@ -1893,15 +1999,14 @@ static void ath10k_htt_rx_h_rx_offload_prot(struct ieee80211_rx_status *status,
 			RX_FLAG_MMIC_STRIPPED;
 }
 
-static int ath10k_htt_rx_h_rx_offload(struct ath10k *ar,
-				      struct sk_buff_head *list)
+static void ath10k_htt_rx_h_rx_offload(struct ath10k *ar,
+				       struct sk_buff_head *list)
 {
 	struct ath10k_htt *htt = &ar->htt;
 	struct ieee80211_rx_status *status = &htt->rx_status;
 	struct htt_rx_offload_msdu *rx;
 	struct sk_buff *msdu;
 	size_t offset;
-	int num_msdu = 0;
 
 	while ((msdu = __skb_dequeue(list))) {
 		/* Offloaded frames don't have Rx descriptor. Instead they have
@@ -1940,10 +2045,8 @@ static int ath10k_htt_rx_h_rx_offload(struct ath10k *ar,
 
 		ath10k_htt_rx_h_rx_offload_prot(status, msdu);
 		ath10k_htt_rx_h_channel(ar, status, NULL, rx->vdev_id);
-		ath10k_process_rx(ar, status, msdu);
-		num_msdu++;
+		ath10k_htt_rx_h_queue_msdu(ar, status, msdu);
 	}
-	return num_msdu;
 }
 
 static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
@@ -1959,7 +2062,7 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 	u8 tid;
 	bool offload;
 	bool frag;
-	int ret, num_msdus = 0;
+	int ret;
 
 	lockdep_assert_held(&htt->rx_ring.lock);
 
@@ -1981,7 +2084,7 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 		   "htt rx in ord vdev %i peer %i tid %i offload %i frag %i msdu count %i\n",
 		   vdev_id, peer_id, tid, offload, frag, msdu_count);
 
-	if (skb->len < msdu_count * sizeof(*resp->rx_in_ord_ind.msdu_descs)) {
+	if (skb->len < msdu_count * sizeof(*resp->rx_in_ord_ind.msdu_descs32)) {
 		ath10k_warn(ar, "dropping invalid in order rx indication\n");
 		return -EINVAL;
 	}
@@ -1990,7 +2093,13 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 	 * extracted and processed.
 	 */
 	__skb_queue_head_init(&list);
-	ret = ath10k_htt_rx_pop_paddr_list(htt, &resp->rx_in_ord_ind, &list);
+	if (ar->hw_params.target_64bit)
+		ret = ath10k_htt_rx_pop_paddr64_list(htt, &resp->rx_in_ord_ind,
+						     &list);
+	else
+		ret = ath10k_htt_rx_pop_paddr32_list(htt, &resp->rx_in_ord_ind,
+						     &list);
+
 	if (ret < 0) {
 		ath10k_warn(ar, "failed to pop paddr list: %d\n", ret);
 		htt->rx_confused = true;
@@ -2001,7 +2110,7 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 	 * separately.
 	 */
 	if (offload)
-		num_msdus = ath10k_htt_rx_h_rx_offload(ar, &list);
+		ath10k_htt_rx_h_rx_offload(ar, &list);
 
 	while (!skb_queue_empty(&list)) {
 		__skb_queue_head_init(&amsdu);
@@ -2014,11 +2123,10 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 			 * better to report something than nothing though. This
 			 * should still give an idea about rx rate to the user.
 			 */
-			num_msdus += skb_queue_len(&amsdu);
 			ath10k_htt_rx_h_ppdu(ar, &amsdu, status, vdev_id);
 			ath10k_htt_rx_h_filter(ar, &amsdu, status);
 			ath10k_htt_rx_h_mpdu(ar, &amsdu, status, false);
-			ath10k_htt_rx_h_deliver(ar, &amsdu, status);
+			ath10k_htt_rx_h_enqueue(ar, &amsdu, status);
 			break;
 		case -EAGAIN:
 			/* fall through */
@@ -2030,7 +2138,7 @@ static int ath10k_htt_rx_in_ord_ind(struct ath10k *ar, struct sk_buff *skb)
 			return -EIO;
 		}
 	}
-	return num_msdus;
+	return ret;
 }
 
 static void ath10k_htt_rx_tx_fetch_resp_id_confirm(struct ath10k *ar,
@@ -2449,6 +2557,62 @@ static void ath10k_htt_fetch_peer_stats(struct ath10k *ar,
 	rcu_read_unlock();
 }
 
+static void ath10k_fetch_10_2_tx_stats(struct ath10k *ar, u8 *data)
+{
+	struct ath10k_pktlog_hdr *hdr = (struct ath10k_pktlog_hdr *)data;
+	struct ath10k_per_peer_tx_stats *p_tx_stats = &ar->peer_tx_stats;
+	struct ath10k_10_2_peer_tx_stats *tx_stats;
+	struct ieee80211_sta *sta;
+	struct ath10k_peer *peer;
+	u16 log_type = __le16_to_cpu(hdr->log_type);
+	u32 peer_id = 0, i;
+
+	if (log_type != ATH_PKTLOG_TYPE_TX_STAT)
+		return;
+
+	tx_stats = (struct ath10k_10_2_peer_tx_stats *)((hdr->payload) +
+		    ATH10K_10_2_TX_STATS_OFFSET);
+
+	if (!tx_stats->tx_ppdu_cnt)
+		return;
+
+	peer_id = tx_stats->peer_id;
+
+	rcu_read_lock();
+	spin_lock_bh(&ar->data_lock);
+	peer = ath10k_peer_find_by_id(ar, peer_id);
+	if (!peer) {
+		ath10k_warn(ar, "Invalid peer id %d in peer stats buffer\n",
+			    peer_id);
+		goto out;
+	}
+
+	sta = peer->sta;
+	for (i = 0; i < tx_stats->tx_ppdu_cnt; i++) {
+		p_tx_stats->succ_bytes =
+			__le16_to_cpu(tx_stats->success_bytes[i]);
+		p_tx_stats->retry_bytes =
+			__le16_to_cpu(tx_stats->retry_bytes[i]);
+		p_tx_stats->failed_bytes =
+			__le16_to_cpu(tx_stats->failed_bytes[i]);
+		p_tx_stats->ratecode = tx_stats->ratecode[i];
+		p_tx_stats->flags = tx_stats->flags[i];
+		p_tx_stats->succ_pkts = tx_stats->success_pkts[i];
+		p_tx_stats->retry_pkts = tx_stats->retry_pkts[i];
+		p_tx_stats->failed_pkts = tx_stats->failed_pkts[i];
+
+		ath10k_update_per_peer_tx_stats(ar, sta, p_tx_stats);
+	}
+	spin_unlock_bh(&ar->data_lock);
+	rcu_read_unlock();
+
+	return;
+
+out:
+	spin_unlock_bh(&ar->data_lock);
+	rcu_read_unlock();
+}
+
 bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 {
 	struct ath10k_htt *htt = &ar->htt;
@@ -2566,6 +2730,10 @@ bool ath10k_htt_t2h_msg_handler(struct ath10k *ar, struct sk_buff *skb)
 					skb->len -
 					offsetof(struct htt_resp,
 						 pktlog_msg.payload));
+
+		if (ath10k_peer_stats_enabled(ar))
+			ath10k_fetch_10_2_tx_stats(ar,
+						   resp->pktlog_msg.payload);
 		break;
 	}
 	case HTT_T2H_MSG_TYPE_RX_FLUSH: {
@@ -2631,6 +2799,24 @@ void ath10k_htt_rx_pktlog_completion_handler(struct ath10k *ar,
 }
 EXPORT_SYMBOL(ath10k_htt_rx_pktlog_completion_handler);
 
+static int ath10k_htt_rx_deliver_msdu(struct ath10k *ar, int quota, int budget)
+{
+	struct sk_buff *skb;
+
+	while (quota < budget) {
+		if (skb_queue_empty(&ar->htt.rx_msdus_q))
+			break;
+
+		skb = __skb_dequeue(&ar->htt.rx_msdus_q);
+		if (!skb)
+			break;
+		ath10k_process_rx(ar, skb);
+		quota++;
+	}
+
+	return quota;
+}
+
 int ath10k_htt_txrx_compl_task(struct ath10k *ar, int budget)
 {
 	struct ath10k_htt *htt = &ar->htt;
@@ -2638,63 +2824,44 @@ int ath10k_htt_txrx_compl_task(struct ath10k *ar, int budget)
 	struct sk_buff_head tx_ind_q;
 	struct sk_buff *skb;
 	unsigned long flags;
-	int quota = 0, done, num_rx_msdus;
+	int quota = 0, done, ret;
 	bool resched_napi = false;
 
 	__skb_queue_head_init(&tx_ind_q);
 
-	/* Since in-ord-ind can deliver more than 1 A-MSDU in single event,
-	 * process it first to utilize full available quota.
+	/* Process pending frames before dequeuing more data
+	 * from hardware.
 	 */
-	while (quota < budget) {
-		if (skb_queue_empty(&htt->rx_in_ord_compl_q))
-			break;
-
-		skb = __skb_dequeue(&htt->rx_in_ord_compl_q);
-		if (!skb) {
-			resched_napi = true;
-			goto exit;
-		}
+	quota = ath10k_htt_rx_deliver_msdu(ar, quota, budget);
+	if (quota == budget) {
+		resched_napi = true;
+		goto exit;
+	}
 
+	while ((skb = __skb_dequeue(&htt->rx_in_ord_compl_q))) {
 		spin_lock_bh(&htt->rx_ring.lock);
-		num_rx_msdus = ath10k_htt_rx_in_ord_ind(ar, skb);
+		ret = ath10k_htt_rx_in_ord_ind(ar, skb);
 		spin_unlock_bh(&htt->rx_ring.lock);
-		if (num_rx_msdus < 0) {
-			resched_napi = true;
-			goto exit;
-		}
 
 		dev_kfree_skb_any(skb);
-		if (num_rx_msdus > 0)
-			quota += num_rx_msdus;
-
-		if ((quota > ATH10K_NAPI_QUOTA_LIMIT) &&
-		    !skb_queue_empty(&htt->rx_in_ord_compl_q)) {
+		if (ret == -EIO) {
 			resched_napi = true;
 			goto exit;
 		}
 	}
 
-	while (quota < budget) {
-		/* no more data to receive */
-		if (!atomic_read(&htt->num_mpdus_ready))
-			break;
-
-		num_rx_msdus = ath10k_htt_rx_handle_amsdu(htt);
-		if (num_rx_msdus < 0) {
+	while (atomic_read(&htt->num_mpdus_ready)) {
+		ret = ath10k_htt_rx_handle_amsdu(htt);
+		if (ret == -EIO) {
 			resched_napi = true;
 			goto exit;
 		}
-
-		quota += num_rx_msdus;
 		atomic_dec(&htt->num_mpdus_ready);
-		if ((quota > ATH10K_NAPI_QUOTA_LIMIT) &&
-		    atomic_read(&htt->num_mpdus_ready)) {
-			resched_napi = true;
-			goto exit;
-		}
 	}
 
+	/* Deliver received data after processing data from hardware */
+	quota = ath10k_htt_rx_deliver_msdu(ar, quota, budget);
+
 	/* From NAPI documentation:
 	 *  The napi poll() function may also process TX completions, in which
 	 *  case if it processes the entire TX ring then it should count that
@@ -2732,3 +2899,29 @@ int ath10k_htt_txrx_compl_task(struct ath10k *ar, int budget)
 	return done;
 }
 EXPORT_SYMBOL(ath10k_htt_txrx_compl_task);
+
+static const struct ath10k_htt_rx_ops htt_rx_ops_32 = {
+	.htt_get_rx_ring_size = ath10k_htt_get_rx_ring_size_32,
+	.htt_config_paddrs_ring = ath10k_htt_config_paddrs_ring_32,
+	.htt_set_paddrs_ring = ath10k_htt_set_paddrs_ring_32,
+	.htt_get_vaddr_ring = ath10k_htt_get_vaddr_ring_32,
+	.htt_reset_paddrs_ring = ath10k_htt_reset_paddrs_ring_32,
+};
+
+static const struct ath10k_htt_rx_ops htt_rx_ops_64 = {
+	.htt_get_rx_ring_size = ath10k_htt_get_rx_ring_size_64,
+	.htt_config_paddrs_ring = ath10k_htt_config_paddrs_ring_64,
+	.htt_set_paddrs_ring = ath10k_htt_set_paddrs_ring_64,
+	.htt_get_vaddr_ring = ath10k_htt_get_vaddr_ring_64,
+	.htt_reset_paddrs_ring = ath10k_htt_reset_paddrs_ring_64,
+};
+
+void ath10k_htt_set_rx_ops(struct ath10k_htt *htt)
+{
+	struct ath10k *ar = htt->ar;
+
+	if (ar->hw_params.target_64bit)
+		htt->rx_ops = &htt_rx_ops_64;
+	else
+		htt->rx_ops = &htt_rx_ops_32;
+}
diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c
index 685faac1368f..d334b7be1fea 100644
--- a/drivers/net/wireless/ath/ath10k/htt_tx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_tx.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -229,50 +229,91 @@ void ath10k_htt_tx_free_msdu_id(struct ath10k_htt *htt, u16 msdu_id)
 	idr_remove(&htt->pending_tx, msdu_id);
 }
 
-static void ath10k_htt_tx_free_cont_txbuf(struct ath10k_htt *htt)
+static void ath10k_htt_tx_free_cont_txbuf_32(struct ath10k_htt *htt)
 {
 	struct ath10k *ar = htt->ar;
 	size_t size;
 
-	if (!htt->txbuf.vaddr)
+	if (!htt->txbuf.vaddr_txbuff_32)
 		return;
 
-	size = htt->max_num_pending_tx * sizeof(struct ath10k_htt_txbuf);
-	dma_free_coherent(ar->dev, size, htt->txbuf.vaddr, htt->txbuf.paddr);
-	htt->txbuf.vaddr = NULL;
+	size = htt->txbuf.size;
+	dma_free_coherent(ar->dev, size, htt->txbuf.vaddr_txbuff_32,
+			  htt->txbuf.paddr);
+	htt->txbuf.vaddr_txbuff_32 = NULL;
 }
 
-static int ath10k_htt_tx_alloc_cont_txbuf(struct ath10k_htt *htt)
+static int ath10k_htt_tx_alloc_cont_txbuf_32(struct ath10k_htt *htt)
 {
 	struct ath10k *ar = htt->ar;
 	size_t size;
 
-	size = htt->max_num_pending_tx * sizeof(struct ath10k_htt_txbuf);
-	htt->txbuf.vaddr = dma_alloc_coherent(ar->dev, size, &htt->txbuf.paddr,
-					      GFP_KERNEL);
-	if (!htt->txbuf.vaddr)
+	size = htt->max_num_pending_tx *
+			sizeof(struct ath10k_htt_txbuf_32);
+
+	htt->txbuf.vaddr_txbuff_32 = dma_alloc_coherent(ar->dev, size,
+							&htt->txbuf.paddr,
+							GFP_KERNEL);
+	if (!htt->txbuf.vaddr_txbuff_32)
 		return -ENOMEM;
 
+	htt->txbuf.size = size;
+
 	return 0;
 }
 
-static void ath10k_htt_tx_free_cont_frag_desc(struct ath10k_htt *htt)
+static void ath10k_htt_tx_free_cont_txbuf_64(struct ath10k_htt *htt)
 {
+	struct ath10k *ar = htt->ar;
 	size_t size;
 
-	if (!htt->frag_desc.vaddr)
+	if (!htt->txbuf.vaddr_txbuff_64)
 		return;
 
-	size = htt->max_num_pending_tx * sizeof(struct htt_msdu_ext_desc);
+	size = htt->txbuf.size;
+	dma_free_coherent(ar->dev, size, htt->txbuf.vaddr_txbuff_64,
+			  htt->txbuf.paddr);
+	htt->txbuf.vaddr_txbuff_64 = NULL;
+}
+
+static int ath10k_htt_tx_alloc_cont_txbuf_64(struct ath10k_htt *htt)
+{
+	struct ath10k *ar = htt->ar;
+	size_t size;
+
+	size = htt->max_num_pending_tx *
+			sizeof(struct ath10k_htt_txbuf_64);
+
+	htt->txbuf.vaddr_txbuff_64 = dma_alloc_coherent(ar->dev, size,
+							&htt->txbuf.paddr,
+							GFP_KERNEL);
+	if (!htt->txbuf.vaddr_txbuff_64)
+		return -ENOMEM;
+
+	htt->txbuf.size = size;
+
+	return 0;
+}
+
+static void ath10k_htt_tx_free_cont_frag_desc_32(struct ath10k_htt *htt)
+{
+	size_t size;
+
+	if (!htt->frag_desc.vaddr_desc_32)
+		return;
+
+	size = htt->max_num_pending_tx *
+			sizeof(struct htt_msdu_ext_desc);
 
 	dma_free_coherent(htt->ar->dev,
 			  size,
-			  htt->frag_desc.vaddr,
+			  htt->frag_desc.vaddr_desc_32,
 			  htt->frag_desc.paddr);
-	htt->frag_desc.vaddr = NULL;
+
+	htt->frag_desc.vaddr_desc_32 = NULL;
 }
 
-static int ath10k_htt_tx_alloc_cont_frag_desc(struct ath10k_htt *htt)
+static int ath10k_htt_tx_alloc_cont_frag_desc_32(struct ath10k_htt *htt)
 {
 	struct ath10k *ar = htt->ar;
 	size_t size;
@@ -280,12 +321,57 @@ static int ath10k_htt_tx_alloc_cont_frag_desc(struct ath10k_htt *htt)
 	if (!ar->hw_params.continuous_frag_desc)
 		return 0;
 
-	size = htt->max_num_pending_tx * sizeof(struct htt_msdu_ext_desc);
-	htt->frag_desc.vaddr = dma_alloc_coherent(ar->dev, size,
-						  &htt->frag_desc.paddr,
-						  GFP_KERNEL);
-	if (!htt->frag_desc.vaddr)
+	size = htt->max_num_pending_tx *
+			sizeof(struct htt_msdu_ext_desc);
+	htt->frag_desc.vaddr_desc_32 = dma_alloc_coherent(ar->dev, size,
+							  &htt->frag_desc.paddr,
+							  GFP_KERNEL);
+	if (!htt->frag_desc.vaddr_desc_32) {
+		ath10k_err(ar, "failed to alloc fragment desc memory\n");
 		return -ENOMEM;
+	}
+	htt->frag_desc.size = size;
+
+	return 0;
+}
+
+static void ath10k_htt_tx_free_cont_frag_desc_64(struct ath10k_htt *htt)
+{
+	size_t size;
+
+	if (!htt->frag_desc.vaddr_desc_64)
+		return;
+
+	size = htt->max_num_pending_tx *
+			sizeof(struct htt_msdu_ext_desc_64);
+
+	dma_free_coherent(htt->ar->dev,
+			  size,
+			  htt->frag_desc.vaddr_desc_64,
+			  htt->frag_desc.paddr);
+
+	htt->frag_desc.vaddr_desc_64 = NULL;
+}
+
+static int ath10k_htt_tx_alloc_cont_frag_desc_64(struct ath10k_htt *htt)
+{
+	struct ath10k *ar = htt->ar;
+	size_t size;
+
+	if (!ar->hw_params.continuous_frag_desc)
+		return 0;
+
+	size = htt->max_num_pending_tx *
+			sizeof(struct htt_msdu_ext_desc_64);
+
+	htt->frag_desc.vaddr_desc_64 = dma_alloc_coherent(ar->dev, size,
+							  &htt->frag_desc.paddr,
+							  GFP_KERNEL);
+	if (!htt->frag_desc.vaddr_desc_64) {
+		ath10k_err(ar, "failed to alloc fragment desc memory\n");
+		return -ENOMEM;
+	}
+	htt->frag_desc.size = size;
 
 	return 0;
 }
@@ -357,13 +443,13 @@ static int ath10k_htt_tx_alloc_buf(struct ath10k_htt *htt)
 	struct ath10k *ar = htt->ar;
 	int ret;
 
-	ret = ath10k_htt_tx_alloc_cont_txbuf(htt);
+	ret = htt->tx_ops->htt_alloc_txbuff(htt);
 	if (ret) {
 		ath10k_err(ar, "failed to alloc cont tx buffer: %d\n", ret);
 		return ret;
 	}
 
-	ret = ath10k_htt_tx_alloc_cont_frag_desc(htt);
+	ret = htt->tx_ops->htt_alloc_frag_desc(htt);
 	if (ret) {
 		ath10k_err(ar, "failed to alloc cont frag desc: %d\n", ret);
 		goto free_txbuf;
@@ -387,10 +473,10 @@ static int ath10k_htt_tx_alloc_buf(struct ath10k_htt *htt)
 	ath10k_htt_tx_free_txq(htt);
 
 free_frag_desc:
-	ath10k_htt_tx_free_cont_frag_desc(htt);
+	htt->tx_ops->htt_free_frag_desc(htt);
 
 free_txbuf:
-	ath10k_htt_tx_free_cont_txbuf(htt);
+	htt->tx_ops->htt_free_txbuff(htt);
 
 	return ret;
 }
@@ -444,9 +530,9 @@ void ath10k_htt_tx_destroy(struct ath10k_htt *htt)
 	if (!htt->tx_mem_allocated)
 		return;
 
-	ath10k_htt_tx_free_cont_txbuf(htt);
+	htt->tx_ops->htt_free_txbuff(htt);
 	ath10k_htt_tx_free_txq(htt);
-	ath10k_htt_tx_free_cont_frag_desc(htt);
+	htt->tx_ops->htt_free_frag_desc(htt);
 	ath10k_htt_tx_free_txdone_fifo(htt);
 	htt->tx_mem_allocated = false;
 }
@@ -545,12 +631,12 @@ int ath10k_htt_h2t_stats_req(struct ath10k_htt *htt, u8 mask, u64 cookie)
 	return 0;
 }
 
-int ath10k_htt_send_frag_desc_bank_cfg(struct ath10k_htt *htt)
+static int ath10k_htt_send_frag_desc_bank_cfg_32(struct ath10k_htt *htt)
 {
 	struct ath10k *ar = htt->ar;
 	struct sk_buff *skb;
 	struct htt_cmd *cmd;
-	struct htt_frag_desc_bank_cfg *cfg;
+	struct htt_frag_desc_bank_cfg32 *cfg;
 	int ret, size;
 	u8 info;
 
@@ -562,7 +648,7 @@ int ath10k_htt_send_frag_desc_bank_cfg(struct ath10k_htt *htt)
 		return -EINVAL;
 	}
 
-	size = sizeof(cmd->hdr) + sizeof(cmd->frag_desc_bank_cfg);
+	size = sizeof(cmd->hdr) + sizeof(cmd->frag_desc_bank_cfg32);
 	skb = ath10k_htc_alloc_skb(ar, size);
 	if (!skb)
 		return -ENOMEM;
@@ -579,7 +665,7 @@ int ath10k_htt_send_frag_desc_bank_cfg(struct ath10k_htt *htt)
 		     ar->running_fw->fw_file.fw_features))
 		info |= HTT_FRAG_DESC_BANK_CFG_INFO_Q_STATE_VALID;
 
-	cfg = &cmd->frag_desc_bank_cfg;
+	cfg = &cmd->frag_desc_bank_cfg32;
 	cfg->info = info;
 	cfg->num_banks = 1;
 	cfg->desc_size = sizeof(struct htt_msdu_ext_desc);
@@ -607,12 +693,112 @@ int ath10k_htt_send_frag_desc_bank_cfg(struct ath10k_htt *htt)
 	return 0;
 }
 
-int ath10k_htt_send_rx_ring_cfg_ll(struct ath10k_htt *htt)
+static int ath10k_htt_send_frag_desc_bank_cfg_64(struct ath10k_htt *htt)
 {
 	struct ath10k *ar = htt->ar;
 	struct sk_buff *skb;
 	struct htt_cmd *cmd;
-	struct htt_rx_ring_setup_ring *ring;
+	struct htt_frag_desc_bank_cfg64 *cfg;
+	int ret, size;
+	u8 info;
+
+	if (!ar->hw_params.continuous_frag_desc)
+		return 0;
+
+	if (!htt->frag_desc.paddr) {
+		ath10k_warn(ar, "invalid frag desc memory\n");
+		return -EINVAL;
+	}
+
+	size = sizeof(cmd->hdr) + sizeof(cmd->frag_desc_bank_cfg64);
+	skb = ath10k_htc_alloc_skb(ar, size);
+	if (!skb)
+		return -ENOMEM;
+
+	skb_put(skb, size);
+	cmd = (struct htt_cmd *)skb->data;
+	cmd->hdr.msg_type = HTT_H2T_MSG_TYPE_FRAG_DESC_BANK_CFG;
+
+	info = 0;
+	info |= SM(htt->tx_q_state.type,
+		   HTT_FRAG_DESC_BANK_CFG_INFO_Q_STATE_DEPTH_TYPE);
+
+	if (test_bit(ATH10K_FW_FEATURE_PEER_FLOW_CONTROL,
+		     ar->running_fw->fw_file.fw_features))
+		info |= HTT_FRAG_DESC_BANK_CFG_INFO_Q_STATE_VALID;
+
+	cfg = &cmd->frag_desc_bank_cfg64;
+	cfg->info = info;
+	cfg->num_banks = 1;
+	cfg->desc_size = sizeof(struct htt_msdu_ext_desc_64);
+	cfg->bank_base_addrs[0] =  __cpu_to_le64(htt->frag_desc.paddr);
+	cfg->bank_id[0].bank_min_id = 0;
+	cfg->bank_id[0].bank_max_id = __cpu_to_le16(htt->max_num_pending_tx -
+						    1);
+
+	cfg->q_state.paddr = cpu_to_le32(htt->tx_q_state.paddr);
+	cfg->q_state.num_peers = cpu_to_le16(htt->tx_q_state.num_peers);
+	cfg->q_state.num_tids = cpu_to_le16(htt->tx_q_state.num_tids);
+	cfg->q_state.record_size = HTT_TX_Q_STATE_ENTRY_SIZE;
+	cfg->q_state.record_multiplier = HTT_TX_Q_STATE_ENTRY_MULTIPLIER;
+
+	ath10k_dbg(ar, ATH10K_DBG_HTT, "htt frag desc bank cmd\n");
+
+	ret = ath10k_htc_send(&htt->ar->htc, htt->eid, skb);
+	if (ret) {
+		ath10k_warn(ar, "failed to send frag desc bank cfg request: %d\n",
+			    ret);
+		dev_kfree_skb_any(skb);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void ath10k_htt_fill_rx_desc_offset_32(void *rx_ring)
+{
+	struct htt_rx_ring_setup_ring32 *ring =
+			(struct htt_rx_ring_setup_ring32 *)rx_ring;
+
+#define desc_offset(x) (offsetof(struct htt_rx_desc, x) / 4)
+	ring->mac80211_hdr_offset = __cpu_to_le16(desc_offset(rx_hdr_status));
+	ring->msdu_payload_offset = __cpu_to_le16(desc_offset(msdu_payload));
+	ring->ppdu_start_offset = __cpu_to_le16(desc_offset(ppdu_start));
+	ring->ppdu_end_offset = __cpu_to_le16(desc_offset(ppdu_end));
+	ring->mpdu_start_offset = __cpu_to_le16(desc_offset(mpdu_start));
+	ring->mpdu_end_offset = __cpu_to_le16(desc_offset(mpdu_end));
+	ring->msdu_start_offset = __cpu_to_le16(desc_offset(msdu_start));
+	ring->msdu_end_offset = __cpu_to_le16(desc_offset(msdu_end));
+	ring->rx_attention_offset = __cpu_to_le16(desc_offset(attention));
+	ring->frag_info_offset = __cpu_to_le16(desc_offset(frag_info));
+#undef desc_offset
+}
+
+static void ath10k_htt_fill_rx_desc_offset_64(void *rx_ring)
+{
+	struct htt_rx_ring_setup_ring64 *ring =
+			(struct htt_rx_ring_setup_ring64 *)rx_ring;
+
+#define desc_offset(x) (offsetof(struct htt_rx_desc, x) / 4)
+	ring->mac80211_hdr_offset = __cpu_to_le16(desc_offset(rx_hdr_status));
+	ring->msdu_payload_offset = __cpu_to_le16(desc_offset(msdu_payload));
+	ring->ppdu_start_offset = __cpu_to_le16(desc_offset(ppdu_start));
+	ring->ppdu_end_offset = __cpu_to_le16(desc_offset(ppdu_end));
+	ring->mpdu_start_offset = __cpu_to_le16(desc_offset(mpdu_start));
+	ring->mpdu_end_offset = __cpu_to_le16(desc_offset(mpdu_end));
+	ring->msdu_start_offset = __cpu_to_le16(desc_offset(msdu_start));
+	ring->msdu_end_offset = __cpu_to_le16(desc_offset(msdu_end));
+	ring->rx_attention_offset = __cpu_to_le16(desc_offset(attention));
+	ring->frag_info_offset = __cpu_to_le16(desc_offset(frag_info));
+#undef desc_offset
+}
+
+static int ath10k_htt_send_rx_ring_cfg_32(struct ath10k_htt *htt)
+{
+	struct ath10k *ar = htt->ar;
+	struct sk_buff *skb;
+	struct htt_cmd *cmd;
+	struct htt_rx_ring_setup_ring32 *ring;
 	const int num_rx_ring = 1;
 	u16 flags;
 	u32 fw_idx;
@@ -626,7 +812,7 @@ int ath10k_htt_send_rx_ring_cfg_ll(struct ath10k_htt *htt)
 	BUILD_BUG_ON(!IS_ALIGNED(HTT_RX_BUF_SIZE, 4));
 	BUILD_BUG_ON((HTT_RX_BUF_SIZE & HTT_MAX_CACHE_LINE_SIZE_MASK) != 0);
 
-	len = sizeof(cmd->hdr) + sizeof(cmd->rx_setup.hdr)
+	len = sizeof(cmd->hdr) + sizeof(cmd->rx_setup_32.hdr)
 	    + (sizeof(*ring) * num_rx_ring);
 	skb = ath10k_htc_alloc_skb(ar, len);
 	if (!skb)
@@ -635,10 +821,10 @@ int ath10k_htt_send_rx_ring_cfg_ll(struct ath10k_htt *htt)
 	skb_put(skb, len);
 
 	cmd = (struct htt_cmd *)skb->data;
-	ring = &cmd->rx_setup.rings[0];
+	ring = &cmd->rx_setup_32.rings[0];
 
 	cmd->hdr.msg_type = HTT_H2T_MSG_TYPE_RX_RING_CFG;
-	cmd->rx_setup.hdr.num_rings = 1;
+	cmd->rx_setup_32.hdr.num_rings = 1;
 
 	/* FIXME: do we need all of this? */
 	flags = 0;
@@ -669,21 +855,76 @@ int ath10k_htt_send_rx_ring_cfg_ll(struct ath10k_htt *htt)
 	ring->flags = __cpu_to_le16(flags);
 	ring->fw_idx_init_val = __cpu_to_le16(fw_idx);
 
-#define desc_offset(x) (offsetof(struct htt_rx_desc, x) / 4)
+	ath10k_htt_fill_rx_desc_offset_32(ring);
+	ret = ath10k_htc_send(&htt->ar->htc, htt->eid, skb);
+	if (ret) {
+		dev_kfree_skb_any(skb);
+		return ret;
+	}
 
-	ring->mac80211_hdr_offset = __cpu_to_le16(desc_offset(rx_hdr_status));
-	ring->msdu_payload_offset = __cpu_to_le16(desc_offset(msdu_payload));
-	ring->ppdu_start_offset = __cpu_to_le16(desc_offset(ppdu_start));
-	ring->ppdu_end_offset = __cpu_to_le16(desc_offset(ppdu_end));
-	ring->mpdu_start_offset = __cpu_to_le16(desc_offset(mpdu_start));
-	ring->mpdu_end_offset = __cpu_to_le16(desc_offset(mpdu_end));
-	ring->msdu_start_offset = __cpu_to_le16(desc_offset(msdu_start));
-	ring->msdu_end_offset = __cpu_to_le16(desc_offset(msdu_end));
-	ring->rx_attention_offset = __cpu_to_le16(desc_offset(attention));
-	ring->frag_info_offset = __cpu_to_le16(desc_offset(frag_info));
+	return 0;
+}
 
-#undef desc_offset
+static int ath10k_htt_send_rx_ring_cfg_64(struct ath10k_htt *htt)
+{
+	struct ath10k *ar = htt->ar;
+	struct sk_buff *skb;
+	struct htt_cmd *cmd;
+	struct htt_rx_ring_setup_ring64 *ring;
+	const int num_rx_ring = 1;
+	u16 flags;
+	u32 fw_idx;
+	int len;
+	int ret;
+
+	/* HW expects the buffer to be an integral number of 4-byte
+	 * "words"
+	 */
+	BUILD_BUG_ON(!IS_ALIGNED(HTT_RX_BUF_SIZE, 4));
+	BUILD_BUG_ON((HTT_RX_BUF_SIZE & HTT_MAX_CACHE_LINE_SIZE_MASK) != 0);
+
+	len = sizeof(cmd->hdr) + sizeof(cmd->rx_setup_64.hdr)
+	    + (sizeof(*ring) * num_rx_ring);
+	skb = ath10k_htc_alloc_skb(ar, len);
+	if (!skb)
+		return -ENOMEM;
+
+	skb_put(skb, len);
+
+	cmd = (struct htt_cmd *)skb->data;
+	ring = &cmd->rx_setup_64.rings[0];
+
+	cmd->hdr.msg_type = HTT_H2T_MSG_TYPE_RX_RING_CFG;
+	cmd->rx_setup_64.hdr.num_rings = 1;
+
+	flags = 0;
+	flags |= HTT_RX_RING_FLAGS_MAC80211_HDR;
+	flags |= HTT_RX_RING_FLAGS_MSDU_PAYLOAD;
+	flags |= HTT_RX_RING_FLAGS_PPDU_START;
+	flags |= HTT_RX_RING_FLAGS_PPDU_END;
+	flags |= HTT_RX_RING_FLAGS_MPDU_START;
+	flags |= HTT_RX_RING_FLAGS_MPDU_END;
+	flags |= HTT_RX_RING_FLAGS_MSDU_START;
+	flags |= HTT_RX_RING_FLAGS_MSDU_END;
+	flags |= HTT_RX_RING_FLAGS_RX_ATTENTION;
+	flags |= HTT_RX_RING_FLAGS_FRAG_INFO;
+	flags |= HTT_RX_RING_FLAGS_UNICAST_RX;
+	flags |= HTT_RX_RING_FLAGS_MULTICAST_RX;
+	flags |= HTT_RX_RING_FLAGS_CTRL_RX;
+	flags |= HTT_RX_RING_FLAGS_MGMT_RX;
+	flags |= HTT_RX_RING_FLAGS_NULL_RX;
+	flags |= HTT_RX_RING_FLAGS_PHY_DATA_RX;
 
+	fw_idx = __le32_to_cpu(*htt->rx_ring.alloc_idx.vaddr);
+
+	ring->fw_idx_shadow_reg_paddr = __cpu_to_le64(htt->rx_ring.alloc_idx.paddr);
+	ring->rx_ring_base_paddr = __cpu_to_le64(htt->rx_ring.base_paddr);
+	ring->rx_ring_len = __cpu_to_le16(htt->rx_ring.size);
+	ring->rx_ring_bufsize = __cpu_to_le16(HTT_RX_BUF_SIZE);
+	ring->flags = __cpu_to_le16(flags);
+	ring->fw_idx_init_val = __cpu_to_le16(fw_idx);
+
+	ath10k_htt_fill_rx_desc_offset_64(ring);
 	ret = ath10k_htc_send(&htt->ar->htc, htt->eid, skb);
 	if (ret) {
 		dev_kfree_skb_any(skb);
@@ -895,8 +1136,9 @@ int ath10k_htt_mgmt_tx(struct ath10k_htt *htt, struct sk_buff *msdu)
 	return res;
 }
 
-int ath10k_htt_tx(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
-		  struct sk_buff *msdu)
+static int ath10k_htt_tx_32(struct ath10k_htt *htt,
+			    enum ath10k_hw_txrx_mode txmode,
+			    struct sk_buff *msdu)
 {
 	struct ath10k *ar = htt->ar;
 	struct device *dev = ar->dev;
@@ -904,7 +1146,7 @@ int ath10k_htt_tx(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(msdu);
 	struct ath10k_skb_cb *skb_cb = ATH10K_SKB_CB(msdu);
 	struct ath10k_hif_sg_item sg_items[2];
-	struct ath10k_htt_txbuf *txbuf;
+	struct ath10k_htt_txbuf_32 *txbuf;
 	struct htt_data_tx_desc_frag *frags;
 	bool is_eth = (txmode == ATH10K_HW_TXRX_ETHERNET);
 	u8 vdev_id = ath10k_htt_tx_get_vdev_id(ar, msdu);
@@ -917,6 +1159,7 @@ int ath10k_htt_tx(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
 	u32 frags_paddr = 0;
 	u32 txbuf_paddr;
 	struct htt_msdu_ext_desc *ext_desc = NULL;
+	struct htt_msdu_ext_desc *ext_desc_t = NULL;
 
 	spin_lock_bh(&htt->tx_lock);
 	res = ath10k_htt_tx_alloc_msdu_id(htt, msdu);
@@ -929,9 +1172,9 @@ int ath10k_htt_tx(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
 	prefetch_len = min(htt->prefetch_len, msdu->len);
 	prefetch_len = roundup(prefetch_len, 4);
 
-	txbuf = &htt->txbuf.vaddr[msdu_id];
+	txbuf = htt->txbuf.vaddr_txbuff_32 + msdu_id;
 	txbuf_paddr = htt->txbuf.paddr +
-		      (sizeof(struct ath10k_htt_txbuf) * msdu_id);
+		      (sizeof(struct ath10k_htt_txbuf_32) * msdu_id);
 
 	if ((ieee80211_is_action(hdr->frame_control) ||
 	     ieee80211_is_deauth(hdr->frame_control) ||
@@ -962,11 +1205,12 @@ int ath10k_htt_tx(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
 		/* pass through */
 	case ATH10K_HW_TXRX_ETHERNET:
 		if (ar->hw_params.continuous_frag_desc) {
-			memset(&htt->frag_desc.vaddr[msdu_id], 0,
+			ext_desc_t = htt->frag_desc.vaddr_desc_32;
+			memset(&ext_desc_t[msdu_id], 0,
 			       sizeof(struct htt_msdu_ext_desc));
 			frags = (struct htt_data_tx_desc_frag *)
-				&htt->frag_desc.vaddr[msdu_id].frags;
-			ext_desc = &htt->frag_desc.vaddr[msdu_id];
+				&ext_desc_t[msdu_id].frags;
+			ext_desc = &ext_desc_t[msdu_id];
 			frags[0].tword_addr.paddr_lo =
 				__cpu_to_le32(skb_cb->paddr);
 			frags[0].tword_addr.paddr_hi = 0;
@@ -1055,9 +1299,9 @@ int ath10k_htt_tx(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
 
 	trace_ath10k_htt_tx(ar, msdu_id, msdu->len, vdev_id, tid);
 	ath10k_dbg(ar, ATH10K_DBG_HTT,
-		   "htt tx flags0 %hhu flags1 %hu len %d id %hu frags_paddr %08x, msdu_paddr %08x vdev %hhu tid %hhu freq %hu\n",
-		   flags0, flags1, msdu->len, msdu_id, frags_paddr,
-		   (u32)skb_cb->paddr, vdev_id, tid, freq);
+		   "htt tx flags0 %hhu flags1 %hu len %d id %hu frags_paddr %pad, msdu_paddr %pad vdev %hhu tid %hhu freq %hu\n",
+		   flags0, flags1, msdu->len, msdu_id, &frags_paddr,
+		   &skb_cb->paddr, vdev_id, tid, freq);
 	ath10k_dbg_dump(ar, ATH10K_DBG_HTT_DUMP, NULL, "htt tx msdu: ",
 			msdu->data, msdu->len);
 	trace_ath10k_tx_hdr(ar, msdu->data, msdu->len);
@@ -1093,3 +1337,239 @@ int ath10k_htt_tx(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txmode,
 err:
 	return res;
 }
+
+static int ath10k_htt_tx_64(struct ath10k_htt *htt,
+			    enum ath10k_hw_txrx_mode txmode,
+			    struct sk_buff *msdu)
+{
+	struct ath10k *ar = htt->ar;
+	struct device *dev = ar->dev;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)msdu->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(msdu);
+	struct ath10k_skb_cb *skb_cb = ATH10K_SKB_CB(msdu);
+	struct ath10k_hif_sg_item sg_items[2];
+	struct ath10k_htt_txbuf_64 *txbuf;
+	struct htt_data_tx_desc_frag *frags;
+	bool is_eth = (txmode == ATH10K_HW_TXRX_ETHERNET);
+	u8 vdev_id = ath10k_htt_tx_get_vdev_id(ar, msdu);
+	u8 tid = ath10k_htt_tx_get_tid(msdu, is_eth);
+	int prefetch_len;
+	int res;
+	u8 flags0 = 0;
+	u16 msdu_id, flags1 = 0;
+	u16 freq = 0;
+	dma_addr_t frags_paddr = 0;
+	u32 txbuf_paddr;
+	struct htt_msdu_ext_desc_64 *ext_desc = NULL;
+	struct htt_msdu_ext_desc_64 *ext_desc_t = NULL;
+
+	spin_lock_bh(&htt->tx_lock);
+	res = ath10k_htt_tx_alloc_msdu_id(htt, msdu);
+	spin_unlock_bh(&htt->tx_lock);
+	if (res < 0)
+		goto err;
+
+	msdu_id = res;
+
+	prefetch_len = min(htt->prefetch_len, msdu->len);
+	prefetch_len = roundup(prefetch_len, 4);
+
+	txbuf = htt->txbuf.vaddr_txbuff_64 + msdu_id;
+	txbuf_paddr = htt->txbuf.paddr +
+		      (sizeof(struct ath10k_htt_txbuf_64) * msdu_id);
+
+	if ((ieee80211_is_action(hdr->frame_control) ||
+	     ieee80211_is_deauth(hdr->frame_control) ||
+	     ieee80211_is_disassoc(hdr->frame_control)) &&
+	     ieee80211_has_protected(hdr->frame_control)) {
+		skb_put(msdu, IEEE80211_CCMP_MIC_LEN);
+	} else if (!(skb_cb->flags & ATH10K_SKB_F_NO_HWCRYPT) &&
+		   txmode == ATH10K_HW_TXRX_RAW &&
+		   ieee80211_has_protected(hdr->frame_control)) {
+		skb_put(msdu, IEEE80211_CCMP_MIC_LEN);
+	}
+
+	skb_cb->paddr = dma_map_single(dev, msdu->data, msdu->len,
+				       DMA_TO_DEVICE);
+	res = dma_mapping_error(dev, skb_cb->paddr);
+	if (res) {
+		res = -EIO;
+		goto err_free_msdu_id;
+	}
+
+	if (unlikely(info->flags & IEEE80211_TX_CTL_TX_OFFCHAN))
+		freq = ar->scan.roc_freq;
+
+	switch (txmode) {
+	case ATH10K_HW_TXRX_RAW:
+	case ATH10K_HW_TXRX_NATIVE_WIFI:
+		flags0 |= HTT_DATA_TX_DESC_FLAGS0_MAC_HDR_PRESENT;
+		/* pass through */
+	case ATH10K_HW_TXRX_ETHERNET:
+		if (ar->hw_params.continuous_frag_desc) {
+			ext_desc_t = htt->frag_desc.vaddr_desc_64;
+			memset(&ext_desc_t[msdu_id], 0,
+			       sizeof(struct htt_msdu_ext_desc_64));
+			frags = (struct htt_data_tx_desc_frag *)
+				&ext_desc_t[msdu_id].frags;
+			ext_desc = &ext_desc_t[msdu_id];
+			frags[0].tword_addr.paddr_lo =
+				__cpu_to_le32(skb_cb->paddr);
+			frags[0].tword_addr.paddr_hi =
+				__cpu_to_le16(upper_32_bits(skb_cb->paddr));
+			frags[0].tword_addr.len_16 = __cpu_to_le16(msdu->len);
+
+			frags_paddr =  htt->frag_desc.paddr +
+			   (sizeof(struct htt_msdu_ext_desc_64) * msdu_id);
+		} else {
+			frags = txbuf->frags;
+			frags[0].tword_addr.paddr_lo =
+						__cpu_to_le32(skb_cb->paddr);
+			frags[0].tword_addr.paddr_hi =
+				__cpu_to_le16(upper_32_bits(skb_cb->paddr));
+			frags[0].tword_addr.len_16 = __cpu_to_le16(msdu->len);
+			frags[1].tword_addr.paddr_lo = 0;
+			frags[1].tword_addr.paddr_hi = 0;
+			frags[1].tword_addr.len_16 = 0;
+		}
+		flags0 |= SM(txmode, HTT_DATA_TX_DESC_FLAGS0_PKT_TYPE);
+		break;
+	case ATH10K_HW_TXRX_MGMT:
+		flags0 |= SM(ATH10K_HW_TXRX_MGMT,
+			     HTT_DATA_TX_DESC_FLAGS0_PKT_TYPE);
+		flags0 |= HTT_DATA_TX_DESC_FLAGS0_MAC_HDR_PRESENT;
+
+		frags_paddr = skb_cb->paddr;
+		break;
+	}
+
+	/* Normally all commands go through HTC which manages tx credits for
+	 * each endpoint and notifies when tx is completed.
+	 *
+	 * HTT endpoint is creditless so there's no need to care about HTC
+	 * flags. In that case it is trivial to fill the HTC header here.
+	 *
+	 * MSDU transmission is considered completed upon HTT event. This
+	 * implies no relevant resources can be freed until after the event is
+	 * received. That's why HTC tx completion handler itself is ignored by
+	 * setting NULL to transfer_context for all sg items.
+	 *
+	 * There is simply no point in pushing HTT TX_FRM through HTC tx path
+	 * as it's a waste of resources. By bypassing HTC it is possible to
+	 * avoid extra memory allocations, compress data structures and thus
+	 * improve performance.
+	 */
+
+	txbuf->htc_hdr.eid = htt->eid;
+	txbuf->htc_hdr.len = __cpu_to_le16(sizeof(txbuf->cmd_hdr) +
+					   sizeof(txbuf->cmd_tx) +
+					   prefetch_len);
+	txbuf->htc_hdr.flags = 0;
+
+	if (skb_cb->flags & ATH10K_SKB_F_NO_HWCRYPT)
+		flags0 |= HTT_DATA_TX_DESC_FLAGS0_NO_ENCRYPT;
+
+	flags1 |= SM((u16)vdev_id, HTT_DATA_TX_DESC_FLAGS1_VDEV_ID);
+	flags1 |= SM((u16)tid, HTT_DATA_TX_DESC_FLAGS1_EXT_TID);
+	if (msdu->ip_summed == CHECKSUM_PARTIAL &&
+	    !test_bit(ATH10K_FLAG_RAW_MODE, &ar->dev_flags)) {
+		flags1 |= HTT_DATA_TX_DESC_FLAGS1_CKSUM_L3_OFFLOAD;
+		flags1 |= HTT_DATA_TX_DESC_FLAGS1_CKSUM_L4_OFFLOAD;
+		if (ar->hw_params.continuous_frag_desc)
+			ext_desc->flags |= HTT_MSDU_CHECKSUM_ENABLE;
+	}
+
+	/* Prevent firmware from sending up tx inspection requests. There's
+	 * nothing ath10k can do with frames requested for inspection so force
+	 * it to simply rely a regular tx completion with discard status.
+	 */
+	flags1 |= HTT_DATA_TX_DESC_FLAGS1_POSTPONED;
+
+	txbuf->cmd_hdr.msg_type = HTT_H2T_MSG_TYPE_TX_FRM;
+	txbuf->cmd_tx.flags0 = flags0;
+	txbuf->cmd_tx.flags1 = __cpu_to_le16(flags1);
+	txbuf->cmd_tx.len = __cpu_to_le16(msdu->len);
+	txbuf->cmd_tx.id = __cpu_to_le16(msdu_id);
+
+	/* fill fragment descriptor */
+	txbuf->cmd_tx.frags_paddr = __cpu_to_le64(frags_paddr);
+	if (ath10k_mac_tx_frm_has_freq(ar)) {
+		txbuf->cmd_tx.offchan_tx.peerid =
+				__cpu_to_le16(HTT_INVALID_PEERID);
+		txbuf->cmd_tx.offchan_tx.freq =
+				__cpu_to_le16(freq);
+	} else {
+		txbuf->cmd_tx.peerid =
+				__cpu_to_le32(HTT_INVALID_PEERID);
+	}
+
+	trace_ath10k_htt_tx(ar, msdu_id, msdu->len, vdev_id, tid);
+	ath10k_dbg(ar, ATH10K_DBG_HTT,
+		   "htt tx flags0 %hhu flags1 %hu len %d id %hu frags_paddr %pad, msdu_paddr %pad vdev %hhu tid %hhu freq %hu\n",
+		   flags0, flags1, msdu->len, msdu_id, &frags_paddr,
+		   &skb_cb->paddr, vdev_id, tid, freq);
+	ath10k_dbg_dump(ar, ATH10K_DBG_HTT_DUMP, NULL, "htt tx msdu: ",
+			msdu->data, msdu->len);
+	trace_ath10k_tx_hdr(ar, msdu->data, msdu->len);
+	trace_ath10k_tx_payload(ar, msdu->data, msdu->len);
+
+	sg_items[0].transfer_id = 0;
+	sg_items[0].transfer_context = NULL;
+	sg_items[0].vaddr = &txbuf->htc_hdr;
+	sg_items[0].paddr = txbuf_paddr +
+			    sizeof(txbuf->frags);
+	sg_items[0].len = sizeof(txbuf->htc_hdr) +
+			  sizeof(txbuf->cmd_hdr) +
+			  sizeof(txbuf->cmd_tx);
+
+	sg_items[1].transfer_id = 0;
+	sg_items[1].transfer_context = NULL;
+	sg_items[1].vaddr = msdu->data;
+	sg_items[1].paddr = skb_cb->paddr;
+	sg_items[1].len = prefetch_len;
+
+	res = ath10k_hif_tx_sg(htt->ar,
+			       htt->ar->htc.endpoint[htt->eid].ul_pipe_id,
+			       sg_items, ARRAY_SIZE(sg_items));
+	if (res)
+		goto err_unmap_msdu;
+
+	return 0;
+
+err_unmap_msdu:
+	dma_unmap_single(dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE);
+err_free_msdu_id:
+	ath10k_htt_tx_free_msdu_id(htt, msdu_id);
+err:
+	return res;
+}
+
+static const struct ath10k_htt_tx_ops htt_tx_ops_32 = {
+	.htt_send_rx_ring_cfg = ath10k_htt_send_rx_ring_cfg_32,
+	.htt_send_frag_desc_bank_cfg = ath10k_htt_send_frag_desc_bank_cfg_32,
+	.htt_alloc_frag_desc = ath10k_htt_tx_alloc_cont_frag_desc_32,
+	.htt_free_frag_desc = ath10k_htt_tx_free_cont_frag_desc_32,
+	.htt_tx = ath10k_htt_tx_32,
+	.htt_alloc_txbuff = ath10k_htt_tx_alloc_cont_txbuf_32,
+	.htt_free_txbuff = ath10k_htt_tx_free_cont_txbuf_32,
+};
+
+static const struct ath10k_htt_tx_ops htt_tx_ops_64 = {
+	.htt_send_rx_ring_cfg = ath10k_htt_send_rx_ring_cfg_64,
+	.htt_send_frag_desc_bank_cfg = ath10k_htt_send_frag_desc_bank_cfg_64,
+	.htt_alloc_frag_desc = ath10k_htt_tx_alloc_cont_frag_desc_64,
+	.htt_free_frag_desc = ath10k_htt_tx_free_cont_frag_desc_64,
+	.htt_tx = ath10k_htt_tx_64,
+	.htt_alloc_txbuff = ath10k_htt_tx_alloc_cont_txbuf_64,
+	.htt_free_txbuff = ath10k_htt_tx_free_cont_txbuf_64,
+};
+
+void ath10k_htt_set_tx_ops(struct ath10k_htt *htt)
+{
+	struct ath10k *ar = htt->ar;
+
+	if (ar->hw_params.target_64bit)
+		htt->tx_ops = &htt_tx_ops_64;
+	else
+		htt->tx_ops = &htt_tx_ops_32;
+}
diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c
index 88955bbe20bd..497ac33e0fbf 100644
--- a/drivers/net/wireless/ath/ath10k/hw.c
+++ b/drivers/net/wireless/ath/ath10k/hw.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -931,3 +931,5 @@ const struct ath10k_hw_ops qca6174_ops = {
 	.set_coverage_class = ath10k_hw_qca988x_set_coverage_class,
 	.enable_pll_clk = ath10k_hw_qca6174_enable_pll_clock,
 };
+
+const struct ath10k_hw_ops wcn3990_ops = {};
diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h
index 05f26e5858ad..413b1b4321f7 100644
--- a/drivers/net/wireless/ath/ath10k/hw.h
+++ b/drivers/net/wireless/ath/ath10k/hw.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -22,6 +22,7 @@
 
 #define ATH10K_FW_DIR			"ath10k"
 
+#define QCA988X_2_0_DEVICE_ID_UBNT   (0x11ac)
 #define QCA988X_2_0_DEVICE_ID   (0x003c)
 #define QCA6164_2_1_DEVICE_ID   (0x0041)
 #define QCA6174_2_1_DEVICE_ID   (0x003e)
@@ -128,6 +129,10 @@ enum qca9377_chip_id_rev {
 #define QCA4019_HW_1_0_BOARD_DATA_FILE "board.bin"
 #define QCA4019_HW_1_0_PATCH_LOAD_ADDR  0x1234
 
+/* WCN3990 1.0 definitions */
+#define WCN3990_HW_1_0_DEV_VERSION	ATH10K_HW_WCN3990
+#define WCN3990_HW_1_0_FW_DIR		ATH10K_FW_DIR "/WCN3990/hw3.0"
+
 #define ATH10K_FW_FILE_BASE		"firmware"
 #define ATH10K_FW_API_MAX		6
 #define ATH10K_FW_API_MIN		2
@@ -553,6 +558,16 @@ struct ath10k_hw_params {
 
 	/* Number of ciphers supported (i.e First N) in cipher_suites array */
 	int n_cipher_suites;
+
+	u32 num_peers;
+	u32 ast_skid_limit;
+	u32 num_wds_entries;
+
+	/* Targets supporting physical addressing capability above 32-bits */
+	bool target_64bit;
+
+	/* Target rx ring fill level */
+	u32 rx_ring_fill_level;
 };
 
 struct htt_rx_desc;
@@ -567,6 +582,7 @@ struct ath10k_hw_ops {
 extern const struct ath10k_hw_ops qca988x_ops;
 extern const struct ath10k_hw_ops qca99x0_ops;
 extern const struct ath10k_hw_ops qca6174_ops;
+extern const struct ath10k_hw_ops wcn3990_ops;
 
 extern const struct ath10k_hw_clk_params qca6174_clk[];
 
@@ -663,6 +679,11 @@ ath10k_rx_desc_get_l3_pad_bytes(struct ath10k_hw_params *hw,
 #define TARGET_TLV_NUM_MSDU_DESC		(1024 + 32)
 #define TARGET_TLV_NUM_WOW_PATTERNS		22
 
+/* Target specific defines for WMI-HL-1.0 firmware */
+#define TARGET_HL_10_TLV_NUM_PEERS		14
+#define TARGET_HL_10_TLV_AST_SKID_LIMIT		6
+#define TARGET_HL_10_TLV_NUM_WDS_ENTRIES	2
+
 /* Diagnostic Window */
 #define CE_DIAG_PIPE	7
 
@@ -868,6 +889,7 @@ ath10k_rx_desc_get_l3_pad_bytes(struct ath10k_hw_params *hw,
 #define PCIE_INTR_CLR_ADDRESS			ar->regs->pcie_intr_clr_address
 #define SCRATCH_3_ADDRESS			ar->regs->scratch_3_address
 #define CPU_INTR_ADDRESS			0x0010
+#define FW_RAM_CONFIG_ADDRESS			0x0018
 
 #define CCNT_TO_MSEC(ar, x) ((x) / ar->hw_params.channel_counters_freq_hz)
 
diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
index 0a947eef348d..ebb3f1b046f3 100644
--- a/drivers/net/wireless/ath/ath10k/mac.c
+++ b/drivers/net/wireless/ath/ath10k/mac.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -2563,7 +2563,7 @@ static void ath10k_peer_assoc_h_qos(struct ath10k *ar,
 		}
 		break;
 	case WMI_VDEV_TYPE_STA:
-		if (vif->bss_conf.qos)
+		if (sta->wme)
 			arg->peer_flags |= arvif->ar->wmi.peer_flags->qos;
 		break;
 	case WMI_VDEV_TYPE_IBSS:
@@ -3574,7 +3574,9 @@ ath10k_mac_tx_h_get_txpath(struct ath10k *ar,
 		return ATH10K_MAC_TX_HTT;
 	case ATH10K_HW_TXRX_MGMT:
 		if (test_bit(ATH10K_FW_FEATURE_HAS_WMI_MGMT_TX,
-			     ar->running_fw->fw_file.fw_features))
+			     ar->running_fw->fw_file.fw_features) ||
+			     test_bit(WMI_SERVICE_MGMT_TX_WMI,
+				      ar->wmi.svc_map))
 			return ATH10K_MAC_TX_WMI_MGMT;
 		else if (ar->htt.target_version_major >= 3)
 			return ATH10K_MAC_TX_HTT;
@@ -3595,7 +3597,7 @@ static int ath10k_mac_tx_submit(struct ath10k *ar,
 
 	switch (txpath) {
 	case ATH10K_MAC_TX_HTT:
-		ret = ath10k_htt_tx(htt, txmode, skb);
+		ret = htt->tx_ops->htt_tx(htt, txmode, skb);
 		break;
 	case ATH10K_MAC_TX_HTT_MGMT:
 		ret = ath10k_htt_mgmt_tx(htt, skb);
@@ -6201,6 +6203,16 @@ static int ath10k_sta_state(struct ieee80211_hw *hw,
 			   "mac vdev %d peer delete %pM sta %pK (sta gone)\n",
 			   arvif->vdev_id, sta->addr, sta);
 
+		if (sta->tdls) {
+			ret = ath10k_mac_tdls_peer_update(ar, arvif->vdev_id,
+							  sta,
+							  WMI_TDLS_PEER_STATE_TEARDOWN);
+			if (ret)
+				ath10k_warn(ar, "failed to update tdls peer state for %pM state %d: %i\n",
+					    sta->addr,
+					    WMI_TDLS_PEER_STATE_TEARDOWN, ret);
+		}
+
 		ret = ath10k_peer_delete(ar, arvif->vdev_id, sta->addr);
 		if (ret)
 			ath10k_warn(ar, "failed to delete peer %pM for vdev %d: %i\n",
@@ -7536,6 +7548,16 @@ ath10k_mac_op_assign_vif_chanctx(struct ieee80211_hw *hw,
 				    arvif->vdev_id, ret);
 	}
 
+	if (ath10k_peer_stats_enabled(ar)) {
+		ar->pktlog_filter |= ATH10K_PKTLOG_PEER_STATS;
+		ret = ath10k_wmi_pdev_pktlog_enable(ar,
+						    ar->pktlog_filter);
+		if (ret) {
+			ath10k_warn(ar, "failed to enable pktlog %d\n", ret);
+			goto err_stop;
+		}
+	}
+
 	mutex_unlock(&ar->conf_mutex);
 	return 0;
 
@@ -7620,6 +7642,34 @@ static void ath10k_mac_op_sta_pre_rcu_remove(struct ieee80211_hw *hw,
 			peer->removed = true;
 }
 
+static void ath10k_sta_statistics(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_sta *sta,
+				  struct station_info *sinfo)
+{
+	struct ath10k_sta *arsta = (struct ath10k_sta *)sta->drv_priv;
+	struct ath10k *ar = arsta->arvif->ar;
+
+	if (!ath10k_peer_stats_enabled(ar))
+		return;
+
+	sinfo->rx_duration = arsta->rx_duration;
+	sinfo->filled |= 1ULL << NL80211_STA_INFO_RX_DURATION;
+
+	if (!arsta->txrate.legacy && !arsta->txrate.nss)
+		return;
+
+	if (arsta->txrate.legacy) {
+		sinfo->txrate.legacy = arsta->txrate.legacy;
+	} else {
+		sinfo->txrate.mcs = arsta->txrate.mcs;
+		sinfo->txrate.nss = arsta->txrate.nss;
+		sinfo->txrate.bw = arsta->txrate.bw;
+	}
+	sinfo->txrate.flags = arsta->txrate.flags;
+	sinfo->filled |= 1ULL << NL80211_STA_INFO_TX_BITRATE;
+}
+
 static const struct ieee80211_ops ath10k_ops = {
 	.tx				= ath10k_mac_op_tx,
 	.wake_tx_queue			= ath10k_mac_op_wake_tx_queue,
@@ -7661,6 +7711,7 @@ static const struct ieee80211_ops ath10k_ops = {
 	.unassign_vif_chanctx		= ath10k_mac_op_unassign_vif_chanctx,
 	.switch_vif_chanctx		= ath10k_mac_op_switch_vif_chanctx,
 	.sta_pre_rcu_remove		= ath10k_mac_op_sta_pre_rcu_remove,
+	.sta_statistics			= ath10k_sta_statistics,
 
 	CFG80211_TESTMODE_CMD(ath10k_tm_cmd)
 
@@ -7671,7 +7722,6 @@ static const struct ieee80211_ops ath10k_ops = {
 #endif
 #ifdef CONFIG_MAC80211_DEBUGFS
 	.sta_add_debugfs		= ath10k_sta_add_debugfs,
-	.sta_statistics			= ath10k_sta_statistics,
 #endif
 };
 
@@ -8244,7 +8294,8 @@ int ath10k_mac_register(struct ath10k *ar)
 	if (test_bit(WMI_SERVICE_TDLS, ar->wmi.svc_map) ||
 	    test_bit(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY, ar->wmi.svc_map)) {
 		ar->hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS;
-		ieee80211_hw_set(ar->hw, TDLS_WIDER_BW);
+		if (test_bit(WMI_SERVICE_TDLS_WIDER_BANDWIDTH, ar->wmi.svc_map))
+			ieee80211_hw_set(ar->hw, TDLS_WIDER_BW);
 	}
 
 	ar->hw->wiphy->flags |= WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL;
@@ -8329,15 +8380,6 @@ int ath10k_mac_register(struct ath10k *ar)
 			ath10k_warn(ar, "failed to initialise DFS pattern detector\n");
 	}
 
-	/* Current wake_tx_queue implementation imposes a significant
-	 * performance penalty in some setups. The tx scheduling code needs
-	 * more work anyway so disable the wake_tx_queue unless firmware
-	 * supports the pull-push mechanism.
-	 */
-	if (!test_bit(ATH10K_FW_FEATURE_PEER_FLOW_CONTROL,
-		      ar->running_fw->fw_file.fw_features))
-		ar->ops->wake_tx_queue = NULL;
-
 	ret = ath10k_mac_init_rd(ar);
 	if (ret) {
 		ath10k_err(ar, "failed to derive regdom: %d\n", ret);
diff --git a/drivers/net/wireless/ath/ath10k/mac.h b/drivers/net/wireless/ath/ath10k/mac.h
index 553747bc19ed..81f8d6c0af35 100644
--- a/drivers/net/wireless/ath/ath10k/mac.h
+++ b/drivers/net/wireless/ath/ath10k/mac.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index ffea348b2190..1b266cd0c2ec 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -23,6 +23,7 @@
 
 #include "core.h"
 #include "debug.h"
+#include "coredump.h"
 
 #include "targaddrs.h"
 #include "bmi.h"
@@ -51,7 +52,15 @@ MODULE_PARM_DESC(reset_mode, "0: auto, 1: warm only (default: 0)");
 #define ATH10K_PCI_TARGET_WAIT 3000
 #define ATH10K_PCI_NUM_WARM_RESET_ATTEMPTS 3
 
+/* Maximum number of bytes that can be handled atomically by
+ * diag read and write.
+ */
+#define ATH10K_DIAG_TRANSFER_LIMIT	0x5000
+
 static const struct pci_device_id ath10k_pci_id_table[] = {
+	/* PCI-E QCA988X V2 (Ubiquiti branded) */
+	{ PCI_VDEVICE(UBIQUITI, QCA988X_2_0_DEVICE_ID_UBNT) },
+
 	{ PCI_VDEVICE(ATHEROS, QCA988X_2_0_DEVICE_ID) }, /* PCI-E QCA988X V2 */
 	{ PCI_VDEVICE(ATHEROS, QCA6164_2_1_DEVICE_ID) }, /* PCI-E QCA6164 V2.1 */
 	{ PCI_VDEVICE(ATHEROS, QCA6174_2_1_DEVICE_ID) }, /* PCI-E QCA6174 V2.1 */
@@ -68,6 +77,7 @@ static const struct ath10k_pci_supp_chip ath10k_pci_supp_chips[] = {
 	 * hacks. ath10k doesn't have them and these devices crash horribly
 	 * because of that.
 	 */
+	{ QCA988X_2_0_DEVICE_ID_UBNT, QCA988X_HW_2_0_CHIP_ID_REV },
 	{ QCA988X_2_0_DEVICE_ID, QCA988X_HW_2_0_CHIP_ID_REV },
 
 	{ QCA6164_2_1_DEVICE_ID, QCA6174_HW_2_1_CHIP_ID_REV },
@@ -785,7 +795,7 @@ static int __ath10k_pci_rx_post_buf(struct ath10k_pci_pipe *pipe)
 	ATH10K_SKB_RXCB(skb)->paddr = paddr;
 
 	spin_lock_bh(&ce->ce_lock);
-	ret = __ath10k_ce_rx_post_buf(ce_pipe, skb, paddr);
+	ret = ce_pipe->ops->ce_rx_post_buf(ce_pipe, skb, paddr);
 	spin_unlock_bh(&ce->ce_lock);
 	if (ret) {
 		dma_unmap_single(ar->dev, paddr, skb->len + skb_tailroom(skb),
@@ -923,7 +933,7 @@ static int ath10k_pci_diag_read_mem(struct ath10k *ar, u32 address, void *data,
 		nbytes = min_t(unsigned int, remaining_bytes,
 			       DIAG_TRANSFER_LIMIT);
 
-		ret = __ath10k_ce_rx_post_buf(ce_diag, &ce_data, ce_data);
+		ret = ce_diag->ops->ce_rx_post_buf(ce_diag, &ce_data, ce_data);
 		if (ret != 0)
 			goto done;
 
@@ -1089,7 +1099,7 @@ int ath10k_pci_diag_write_mem(struct ath10k *ar, u32 address,
 		nbytes = min_t(int, remaining_bytes, DIAG_TRANSFER_LIMIT);
 
 		/* Set up to receive directly into Target(!) address */
-		ret = __ath10k_ce_rx_post_buf(ce_diag, &address, address);
+		ret = ce_diag->ops->ce_rx_post_buf(ce_diag, &address, address);
 		if (ret != 0)
 			goto done;
 
@@ -1461,6 +1471,215 @@ static void ath10k_pci_dump_registers(struct ath10k *ar,
 		crash_data->registers[i] = reg_dump_values[i];
 }
 
+static int ath10k_pci_dump_memory_section(struct ath10k *ar,
+					  const struct ath10k_mem_region *mem_region,
+					  u8 *buf, size_t buf_len)
+{
+	const struct ath10k_mem_section *cur_section, *next_section;
+	unsigned int count, section_size, skip_size;
+	int ret, i, j;
+
+	if (!mem_region || !buf)
+		return 0;
+
+	cur_section = &mem_region->section_table.sections[0];
+
+	if (mem_region->start > cur_section->start) {
+		ath10k_warn(ar, "incorrect memdump region 0x%x with section start address 0x%x.\n",
+			    mem_region->start, cur_section->start);
+		return 0;
+	}
+
+	skip_size = cur_section->start - mem_region->start;
+
+	/* fill the gap between the first register section and register
+	 * start address
+	 */
+	for (i = 0; i < skip_size; i++) {
+		*buf = ATH10K_MAGIC_NOT_COPIED;
+		buf++;
+	}
+
+	count = 0;
+
+	for (i = 0; cur_section != NULL; i++) {
+		section_size = cur_section->end - cur_section->start;
+
+		if (section_size <= 0) {
+			ath10k_warn(ar, "incorrect ramdump format with start address 0x%x and stop address 0x%x\n",
+				    cur_section->start,
+				    cur_section->end);
+			break;
+		}
+
+		if ((i + 1) == mem_region->section_table.size) {
+			/* last section */
+			next_section = NULL;
+			skip_size = 0;
+		} else {
+			next_section = cur_section + 1;
+
+			if (cur_section->end > next_section->start) {
+				ath10k_warn(ar, "next ramdump section 0x%x is smaller than current end address 0x%x\n",
+					    next_section->start,
+					    cur_section->end);
+				break;
+			}
+
+			skip_size = next_section->start - cur_section->end;
+		}
+
+		if (buf_len < (skip_size + section_size)) {
+			ath10k_warn(ar, "ramdump buffer is too small: %zu\n", buf_len);
+			break;
+		}
+
+		buf_len -= skip_size + section_size;
+
+		/* read section to dest memory */
+		ret = ath10k_pci_diag_read_mem(ar, cur_section->start,
+					       buf, section_size);
+		if (ret) {
+			ath10k_warn(ar, "failed to read ramdump from section 0x%x: %d\n",
+				    cur_section->start, ret);
+			break;
+		}
+
+		buf += section_size;
+		count += section_size;
+
+		/* fill in the gap between this section and the next */
+		for (j = 0; j < skip_size; j++) {
+			*buf = ATH10K_MAGIC_NOT_COPIED;
+			buf++;
+		}
+
+		count += skip_size;
+
+		if (!next_section)
+			/* this was the last section */
+			break;
+
+		cur_section = next_section;
+	}
+
+	return count;
+}
+
+static int ath10k_pci_set_ram_config(struct ath10k *ar, u32 config)
+{
+	u32 val;
+
+	ath10k_pci_write32(ar, SOC_CORE_BASE_ADDRESS +
+			   FW_RAM_CONFIG_ADDRESS, config);
+
+	val = ath10k_pci_read32(ar, SOC_CORE_BASE_ADDRESS +
+				FW_RAM_CONFIG_ADDRESS);
+	if (val != config) {
+		ath10k_warn(ar, "failed to set RAM config from 0x%x to 0x%x\n",
+			    val, config);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void ath10k_pci_dump_memory(struct ath10k *ar,
+				   struct ath10k_fw_crash_data *crash_data)
+{
+	const struct ath10k_hw_mem_layout *mem_layout;
+	const struct ath10k_mem_region *current_region;
+	struct ath10k_dump_ram_data_hdr *hdr;
+	u32 count, shift;
+	size_t buf_len;
+	int ret, i;
+	u8 *buf;
+
+	lockdep_assert_held(&ar->data_lock);
+
+	if (!crash_data)
+		return;
+
+	mem_layout = ath10k_coredump_get_mem_layout(ar);
+	if (!mem_layout)
+		return;
+
+	current_region = &mem_layout->region_table.regions[0];
+
+	buf = crash_data->ramdump_buf;
+	buf_len = crash_data->ramdump_buf_len;
+
+	memset(buf, 0, buf_len);
+
+	for (i = 0; i < mem_layout->region_table.size; i++) {
+		count = 0;
+
+		if (current_region->len > buf_len) {
+			ath10k_warn(ar, "memory region %s size %d is larger that remaining ramdump buffer size %zu\n",
+				    current_region->name,
+				    current_region->len,
+				    buf_len);
+			break;
+		}
+
+		/* To get IRAM dump, the host driver needs to switch target
+		 * ram config from DRAM to IRAM.
+		 */
+		if (current_region->type == ATH10K_MEM_REGION_TYPE_IRAM1 ||
+		    current_region->type == ATH10K_MEM_REGION_TYPE_IRAM2) {
+			shift = current_region->start >> 20;
+
+			ret = ath10k_pci_set_ram_config(ar, shift);
+			if (ret) {
+				ath10k_warn(ar, "failed to switch ram config to IRAM for section %s: %d\n",
+					    current_region->name, ret);
+				break;
+			}
+		}
+
+		/* Reserve space for the header. */
+		hdr = (void *)buf;
+		buf += sizeof(*hdr);
+		buf_len -= sizeof(*hdr);
+
+		if (current_region->section_table.size > 0) {
+			/* Copy each section individually. */
+			count = ath10k_pci_dump_memory_section(ar,
+							       current_region,
+							       buf,
+							       current_region->len);
+		} else {
+			/* No individiual memory sections defined so we can
+			 * copy the entire memory region.
+			 */
+			ret = ath10k_pci_diag_read_mem(ar,
+						       current_region->start,
+						       buf,
+						       current_region->len);
+			if (ret) {
+				ath10k_warn(ar, "failed to copy ramdump region %s: %d\n",
+					    current_region->name, ret);
+				break;
+			}
+
+			count = current_region->len;
+		}
+
+		hdr->region_type = cpu_to_le32(current_region->type);
+		hdr->start = cpu_to_le32(current_region->start);
+		hdr->length = cpu_to_le32(count);
+
+		if (count == 0)
+			/* Note: the header remains, just with zero length. */
+			break;
+
+		buf += count;
+		buf_len -= count;
+
+		current_region++;
+	}
+}
+
 static void ath10k_pci_fw_crashed_dump(struct ath10k *ar)
 {
 	struct ath10k_fw_crash_data *crash_data;
@@ -1470,7 +1689,7 @@ static void ath10k_pci_fw_crashed_dump(struct ath10k *ar)
 
 	ar->stats.fw_crash_counter++;
 
-	crash_data = ath10k_debug_get_new_fw_crash_data(ar);
+	crash_data = ath10k_coredump_new(ar);
 
 	if (crash_data)
 		scnprintf(guid, sizeof(guid), "%pUl", &crash_data->guid);
@@ -1481,6 +1700,7 @@ static void ath10k_pci_fw_crashed_dump(struct ath10k *ar)
 	ath10k_print_driver_info(ar);
 	ath10k_pci_dump_registers(ar, crash_data);
 	ath10k_ce_dump_registers(ar, crash_data);
+	ath10k_pci_dump_memory(ar, crash_data);
 
 	spin_unlock_bh(&ar->data_lock);
 
@@ -1858,7 +2078,7 @@ int ath10k_pci_hif_exchange_bmi_msg(struct ath10k *ar,
 
 	ret = ath10k_pci_bmi_wait(ar, ce_tx, ce_rx, &xfer);
 	if (ret) {
-		u32 unused_buffer;
+		dma_addr_t unused_buffer;
 		unsigned int unused_nbytes;
 		unsigned int unused_id;
 
@@ -1871,7 +2091,7 @@ int ath10k_pci_hif_exchange_bmi_msg(struct ath10k *ar,
 
 err_resp:
 	if (resp) {
-		u32 unused_buffer;
+		dma_addr_t unused_buffer;
 
 		ath10k_ce_revoke_recv_next(ce_rx, NULL, &unused_buffer);
 		dma_unmap_single(ar->dev, resp_paddr,
@@ -1977,6 +2197,7 @@ static int ath10k_pci_get_num_banks(struct ath10k *ar)
 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
 
 	switch (ar_pci->pdev->device) {
+	case QCA988X_2_0_DEVICE_ID_UBNT:
 	case QCA988X_2_0_DEVICE_ID:
 	case QCA99X0_2_0_DEVICE_ID:
 	case QCA9888_2_0_DEVICE_ID:
@@ -3208,6 +3429,7 @@ static int ath10k_pci_probe(struct pci_dev *pdev,
 	u32 (*targ_cpu_to_ce_addr)(struct ath10k *ar, u32 addr);
 
 	switch (pci_dev->device) {
+	case QCA988X_2_0_DEVICE_ID_UBNT:
 	case QCA988X_2_0_DEVICE_ID:
 		hw_rev = ATH10K_HW_QCA988X;
 		pci_ps = false;
diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h
index 08704fbc11e3..e52fd83156b6 100644
--- a/drivers/net/wireless/ath/ath10k/pci.h
+++ b/drivers/net/wireless/ath/ath10k/pci.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/rx_desc.h b/drivers/net/wireless/ath/ath10k/rx_desc.h
index 28da14398951..545deb6d7af1 100644
--- a/drivers/net/wireless/ath/ath10k/rx_desc.h
+++ b/drivers/net/wireless/ath/ath10k/rx_desc.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -210,6 +210,10 @@ struct rx_frag_info {
 	u8 ring1_more_count;
 	u8 ring2_more_count;
 	u8 ring3_more_count;
+	u8 ring4_more_count;
+	u8 ring5_more_count;
+	u8 ring6_more_count;
+	u8 ring7_more_count;
 } __packed;
 
 /*
@@ -471,10 +475,16 @@ struct rx_msdu_start_qca99x0 {
 	__le32 info2; /* %RX_MSDU_START_INFO2_ */
 } __packed;
 
+struct rx_msdu_start_wcn3990 {
+	__le32 info2; /* %RX_MSDU_START_INFO2_ */
+	__le32 info3; /* %RX_MSDU_START_INFO3_ */
+} __packed;
+
 struct rx_msdu_start {
 	struct rx_msdu_start_common common;
 	union {
 		struct rx_msdu_start_qca99x0 qca99x0;
+		struct rx_msdu_start_wcn3990 wcn3990;
 	} __packed;
 } __packed;
 
@@ -595,10 +605,23 @@ struct rx_msdu_end_qca99x0 {
 	__le32 info2;
 } __packed;
 
+struct rx_msdu_end_wcn3990 {
+	__le32 ipv6_crc;
+	__le32 tcp_seq_no;
+	__le32 tcp_ack_no;
+	__le32 info1;
+	__le32 info2;
+	__le32 rule_indication_0;
+	__le32 rule_indication_1;
+	__le32 rule_indication_2;
+	__le32 rule_indication_3;
+} __packed;
+
 struct rx_msdu_end {
 	struct rx_msdu_end_common common;
 	union {
 		struct rx_msdu_end_qca99x0 qca99x0;
+		struct rx_msdu_end_wcn3990 wcn3990;
 	} __packed;
 } __packed;
 
@@ -963,6 +986,12 @@ struct rx_pkt_end {
 	__le32 phy_timestamp_2;
 } __packed;
 
+struct rx_pkt_end_wcn3990 {
+	__le32 info0; /* %RX_PKT_END_INFO0_ */
+	__le64 phy_timestamp_1;
+	__le64 phy_timestamp_2;
+} __packed;
+
 #define RX_LOCATION_INFO0_RTT_FAC_LEGACY_MASK		0x00003fff
 #define RX_LOCATION_INFO0_RTT_FAC_LEGACY_LSB		0
 #define RX_LOCATION_INFO0_RTT_FAC_VHT_MASK		0x1fff8000
@@ -998,6 +1027,12 @@ struct rx_location_info {
 	__le32 rx_location_info1; /* %RX_LOCATION_INFO1_ */
 } __packed;
 
+struct rx_location_info_wcn3990 {
+	__le32 rx_location_info0; /* %RX_LOCATION_INFO0_ */
+	__le32 rx_location_info1; /* %RX_LOCATION_INFO1_ */
+	__le32 rx_location_info2; /* %RX_LOCATION_INFO2_ */
+} __packed;
+
 enum rx_phy_ppdu_end_info0 {
 	RX_PHY_PPDU_END_INFO0_ERR_RADAR           = BIT(2),
 	RX_PHY_PPDU_END_INFO0_ERR_RX_ABORT        = BIT(3),
@@ -1086,6 +1121,20 @@ struct rx_ppdu_end_qca9984 {
 	__le16 info1; /* %RX_PPDU_END_INFO1_ */
 } __packed;
 
+struct rx_ppdu_end_wcn3990 {
+	struct rx_pkt_end_wcn3990 rx_pkt_end;
+	struct rx_location_info_wcn3990 rx_location_info;
+	struct rx_phy_ppdu_end rx_phy_ppdu_end;
+	__le32 rx_timing_offset;
+	__le32 reserved_info_0;
+	__le32 reserved_info_1;
+	__le32 rx_antenna_info;
+	__le32 rx_coex_info;
+	__le32 rx_mpdu_cnt_info;
+	__le64 phy_timestamp_tx;
+	__le32 rx_bb_length;
+} __packed;
+
 struct rx_ppdu_end {
 	struct rx_ppdu_end_common common;
 	union {
@@ -1093,6 +1142,7 @@ struct rx_ppdu_end {
 		struct rx_ppdu_end_qca6174 qca6174;
 		struct rx_ppdu_end_qca99x0 qca99x0;
 		struct rx_ppdu_end_qca9984 qca9984;
+		struct rx_ppdu_end_wcn3990 wcn3990;
 	} __packed;
 } __packed;
 
diff --git a/drivers/net/wireless/ath/ath10k/spectral.c b/drivers/net/wireless/ath/ath10k/spectral.c
index 2048b1e5262b..af6995de7e00 100644
--- a/drivers/net/wireless/ath/ath10k/spectral.c
+++ b/drivers/net/wireless/ath/ath10k/spectral.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2013-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/spectral.h b/drivers/net/wireless/ath/ath10k/spectral.h
index 89b0ad769d4f..13276f4dc12c 100644
--- a/drivers/net/wireless/ath/ath10k/spectral.h
+++ b/drivers/net/wireless/ath/ath10k/spectral.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2013-2015 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -44,7 +44,7 @@ enum ath10k_spectral_mode {
 	SPECTRAL_MANUAL,
 };
 
-#ifdef CONFIG_ATH10K_DEBUGFS
+#ifdef CONFIG_ATH10K_SPECTRAL
 
 int ath10k_spectral_process_fft(struct ath10k *ar,
 				struct wmi_phyerr_ev_arg *phyerr,
@@ -85,6 +85,6 @@ static inline void ath10k_spectral_destroy(struct ath10k *ar)
 {
 }
 
-#endif /* CONFIG_ATH10K_DEBUGFS */
+#endif /* CONFIG_ATH10K_SPECTRAL */
 
 #endif /* SPECTRAL_H */
diff --git a/drivers/net/wireless/ath/ath10k/swap.c b/drivers/net/wireless/ath/ath10k/swap.c
index adf4592374b4..e7f57efadae1 100644
--- a/drivers/net/wireless/ath/ath10k/swap.c
+++ b/drivers/net/wireless/ath/ath10k/swap.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2015-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/swap.h b/drivers/net/wireless/ath/ath10k/swap.h
index f5dc0476493e..fa602f15fa93 100644
--- a/drivers/net/wireless/ath/ath10k/swap.h
+++ b/drivers/net/wireless/ath/ath10k/swap.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2015-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/targaddrs.h b/drivers/net/wireless/ath/ath10k/targaddrs.h
index 8bded5da9d0d..c2b5bad0459b 100644
--- a/drivers/net/wireless/ath/ath10k/targaddrs.h
+++ b/drivers/net/wireless/ath/ath10k/targaddrs.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/testmode.c b/drivers/net/wireless/ath/ath10k/testmode.c
index 9d3eb258ac2f..568810b41657 100644
--- a/drivers/net/wireless/ath/ath10k/testmode.c
+++ b/drivers/net/wireless/ath/ath10k/testmode.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/testmode_i.h b/drivers/net/wireless/ath/ath10k/testmode_i.h
index 191a8f34c8ea..6514d1a14242 100644
--- a/drivers/net/wireless/ath/ath10k/testmode_i.h
+++ b/drivers/net/wireless/ath/ath10k/testmode_i.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014,2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/thermal.c b/drivers/net/wireless/ath/ath10k/thermal.c
index ef717b631ff8..aa8978a8d751 100644
--- a/drivers/net/wireless/ath/ath10k/thermal.c
+++ b/drivers/net/wireless/ath/ath10k/thermal.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014-2015 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/thermal.h b/drivers/net/wireless/ath/ath10k/thermal.h
index 3abb97f63b1e..65e2419543f9 100644
--- a/drivers/net/wireless/ath/ath10k/thermal.h
+++ b/drivers/net/wireless/ath/ath10k/thermal.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2014-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/trace.h b/drivers/net/wireless/ath/ath10k/trace.h
index e0d00cef0bd8..e40edced1d82 100644
--- a/drivers/net/wireless/ath/ath10k/trace.h
+++ b/drivers/net/wireless/ath/ath10k/trace.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/txrx.c b/drivers/net/wireless/ath/ath10k/txrx.c
index d4986f626c35..5b3b021526ab 100644
--- a/drivers/net/wireless/ath/ath10k/txrx.c
+++ b/drivers/net/wireless/ath/ath10k/txrx.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/txrx.h b/drivers/net/wireless/ath/ath10k/txrx.h
index e7ea1ae1c438..2bf401e436d3 100644
--- a/drivers/net/wireless/ath/ath10k/txrx.h
+++ b/drivers/net/wireless/ath/ath10k/txrx.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2014,2016 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/wmi-ops.h b/drivers/net/wireless/ath/ath10k/wmi-ops.h
index 2fc3f24ff1ca..14093cfdc505 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-ops.h
+++ b/drivers/net/wireless/ath/ath10k/wmi-ops.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -377,6 +377,7 @@ ath10k_wmi_mgmt_tx(struct ath10k *ar, struct sk_buff *msdu)
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(msdu);
 	struct sk_buff *skb;
 	int ret;
+	u32 mgmt_tx_cmdid;
 
 	if (!ar->wmi.ops->gen_mgmt_tx)
 		return -EOPNOTSUPP;
@@ -385,7 +386,13 @@ ath10k_wmi_mgmt_tx(struct ath10k *ar, struct sk_buff *msdu)
 	if (IS_ERR(skb))
 		return PTR_ERR(skb);
 
-	ret = ath10k_wmi_cmd_send(ar, skb, ar->wmi.cmd->mgmt_tx_cmdid);
+	if (test_bit(ATH10K_FW_FEATURE_MGMT_TX_BY_REF,
+		     ar->running_fw->fw_file.fw_features))
+		mgmt_tx_cmdid = ar->wmi.cmd->mgmt_tx_send_cmdid;
+	else
+		mgmt_tx_cmdid = ar->wmi.cmd->mgmt_tx_cmdid;
+
+	ret = ath10k_wmi_cmd_send(ar, skb, mgmt_tx_cmdid);
 	if (ret)
 		return ret;
 
diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.c b/drivers/net/wireless/ath/ath10k/wmi-tlv.c
index 7616c1c4bbd3..ae77a007ae07 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-tlv.c
+++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -917,33 +917,69 @@ ath10k_wmi_tlv_parse_mem_reqs(struct ath10k *ar, u16 tag, u16 len,
 	return -ENOMEM;
 }
 
+struct wmi_tlv_svc_rdy_parse {
+	const struct hal_reg_capabilities *reg;
+	const struct wmi_tlv_svc_rdy_ev *ev;
+	const __le32 *svc_bmap;
+	const struct wlan_host_mem_req *mem_reqs;
+	bool svc_bmap_done;
+	bool dbs_hw_mode_done;
+};
+
+static int ath10k_wmi_tlv_svc_rdy_parse(struct ath10k *ar, u16 tag, u16 len,
+					const void *ptr, void *data)
+{
+	struct wmi_tlv_svc_rdy_parse *svc_rdy = data;
+
+	switch (tag) {
+	case WMI_TLV_TAG_STRUCT_SERVICE_READY_EVENT:
+		svc_rdy->ev = ptr;
+		break;
+	case WMI_TLV_TAG_STRUCT_HAL_REG_CAPABILITIES:
+		svc_rdy->reg = ptr;
+		break;
+	case WMI_TLV_TAG_ARRAY_STRUCT:
+		svc_rdy->mem_reqs = ptr;
+		break;
+	case WMI_TLV_TAG_ARRAY_UINT32:
+		if (!svc_rdy->svc_bmap_done) {
+			svc_rdy->svc_bmap_done = true;
+			svc_rdy->svc_bmap = ptr;
+		} else if (!svc_rdy->dbs_hw_mode_done) {
+			svc_rdy->dbs_hw_mode_done = true;
+		}
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
 static int ath10k_wmi_tlv_op_pull_svc_rdy_ev(struct ath10k *ar,
 					     struct sk_buff *skb,
 					     struct wmi_svc_rdy_ev_arg *arg)
 {
-	const void **tb;
 	const struct hal_reg_capabilities *reg;
 	const struct wmi_tlv_svc_rdy_ev *ev;
 	const __le32 *svc_bmap;
 	const struct wlan_host_mem_req *mem_reqs;
+	struct wmi_tlv_svc_rdy_parse svc_rdy = { };
 	int ret;
 
-	tb = ath10k_wmi_tlv_parse_alloc(ar, skb->data, skb->len, GFP_ATOMIC);
-	if (IS_ERR(tb)) {
-		ret = PTR_ERR(tb);
+	ret = ath10k_wmi_tlv_iter(ar, skb->data, skb->len,
+				  ath10k_wmi_tlv_svc_rdy_parse, &svc_rdy);
+	if (ret) {
 		ath10k_warn(ar, "failed to parse tlv: %d\n", ret);
 		return ret;
 	}
 
-	ev = tb[WMI_TLV_TAG_STRUCT_SERVICE_READY_EVENT];
-	reg = tb[WMI_TLV_TAG_STRUCT_HAL_REG_CAPABILITIES];
-	svc_bmap = tb[WMI_TLV_TAG_ARRAY_UINT32];
-	mem_reqs = tb[WMI_TLV_TAG_ARRAY_STRUCT];
+	ev = svc_rdy.ev;
+	reg = svc_rdy.reg;
+	svc_bmap = svc_rdy.svc_bmap;
+	mem_reqs = svc_rdy.mem_reqs;
 
-	if (!ev || !reg || !svc_bmap || !mem_reqs) {
-		kfree(tb);
+	if (!ev || !reg || !svc_bmap || !mem_reqs)
 		return -EPROTO;
-	}
 
 	/* This is an internal ABI compatibility check for WMI TLV so check it
 	 * here instead of the generic WMI code.
@@ -961,7 +997,6 @@ static int ath10k_wmi_tlv_op_pull_svc_rdy_ev(struct ath10k *ar,
 	    __le32_to_cpu(ev->abi.abi_ver_ns1) != WMI_TLV_ABI_VER_NS1 ||
 	    __le32_to_cpu(ev->abi.abi_ver_ns2) != WMI_TLV_ABI_VER_NS2 ||
 	    __le32_to_cpu(ev->abi.abi_ver_ns3) != WMI_TLV_ABI_VER_NS3) {
-		kfree(tb);
 		return -ENOTSUPP;
 	}
 
@@ -982,12 +1017,10 @@ static int ath10k_wmi_tlv_op_pull_svc_rdy_ev(struct ath10k *ar,
 	ret = ath10k_wmi_tlv_iter(ar, mem_reqs, ath10k_wmi_tlv_len(mem_reqs),
 				  ath10k_wmi_tlv_parse_mem_reqs, arg);
 	if (ret) {
-		kfree(tb);
 		ath10k_warn(ar, "failed to parse mem_reqs tlv: %d\n", ret);
 		return ret;
 	}
 
-	kfree(tb);
 	return 0;
 }
 
@@ -1406,7 +1439,10 @@ static struct sk_buff *ath10k_wmi_tlv_op_gen_init(struct ath10k *ar)
 	cmd->num_host_mem_chunks = __cpu_to_le32(ar->wmi.num_mem_chunks);
 
 	cfg->num_vdevs = __cpu_to_le32(TARGET_TLV_NUM_VDEVS);
-	cfg->num_peers = __cpu_to_le32(TARGET_TLV_NUM_PEERS);
+
+	cfg->num_peers = __cpu_to_le32(ar->hw_params.num_peers);
+	cfg->ast_skid_limit = __cpu_to_le32(ar->hw_params.ast_skid_limit);
+	cfg->num_wds_entries = __cpu_to_le32(ar->hw_params.num_wds_entries);
 
 	if (test_bit(WMI_SERVICE_RX_FULL_REORDER, ar->wmi.svc_map)) {
 		cfg->num_offload_peers = __cpu_to_le32(TARGET_TLV_NUM_VDEVS);
@@ -1418,7 +1454,6 @@ static struct sk_buff *ath10k_wmi_tlv_op_gen_init(struct ath10k *ar)
 
 	cfg->num_peer_keys = __cpu_to_le32(2);
 	cfg->num_tids = __cpu_to_le32(TARGET_TLV_NUM_TIDS);
-	cfg->ast_skid_limit = __cpu_to_le32(0x10);
 	cfg->tx_chain_mask = __cpu_to_le32(0x7);
 	cfg->rx_chain_mask = __cpu_to_le32(0x7);
 	cfg->rx_timeout_pri[0] = __cpu_to_le32(0x64);
@@ -1434,7 +1469,6 @@ static struct sk_buff *ath10k_wmi_tlv_op_gen_init(struct ath10k *ar)
 	cfg->num_mcast_table_elems = __cpu_to_le32(0);
 	cfg->mcast2ucast_mode = __cpu_to_le32(0);
 	cfg->tx_dbg_log_size = __cpu_to_le32(0x400);
-	cfg->num_wds_entries = __cpu_to_le32(0x20);
 	cfg->dma_burst_size = __cpu_to_le32(0);
 	cfg->mac_aggr_delim = __cpu_to_le32(0);
 	cfg->rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(0);
@@ -2449,6 +2483,80 @@ ath10k_wmi_tlv_op_gen_request_stats(struct ath10k *ar, u32 stats_mask)
 	return skb;
 }
 
+static struct sk_buff *
+ath10k_wmi_tlv_op_gen_mgmt_tx(struct ath10k *ar, struct sk_buff *msdu)
+{
+	struct ath10k_skb_cb *cb = ATH10K_SKB_CB(msdu);
+	struct wmi_tlv_mgmt_tx_cmd *cmd;
+	struct wmi_tlv *tlv;
+	struct ieee80211_hdr *hdr;
+	struct sk_buff *skb;
+	void *ptr;
+	int len;
+	u32 buf_len = msdu->len;
+	struct ath10k_vif *arvif;
+	dma_addr_t mgmt_frame_dma;
+	u32 vdev_id;
+
+	if (!cb->vif)
+		return ERR_PTR(-EINVAL);
+
+	hdr = (struct ieee80211_hdr *)msdu->data;
+	arvif = (void *)cb->vif->drv_priv;
+	vdev_id = arvif->vdev_id;
+
+	if (WARN_ON_ONCE(!ieee80211_is_mgmt(hdr->frame_control)))
+		return ERR_PTR(-EINVAL);
+
+	len = sizeof(*cmd) + 2 * sizeof(*tlv);
+
+	if ((ieee80211_is_action(hdr->frame_control) ||
+	     ieee80211_is_deauth(hdr->frame_control) ||
+	     ieee80211_is_disassoc(hdr->frame_control)) &&
+	     ieee80211_has_protected(hdr->frame_control)) {
+		len += IEEE80211_CCMP_MIC_LEN;
+		buf_len += IEEE80211_CCMP_MIC_LEN;
+	}
+
+	buf_len = min_t(u32, buf_len, WMI_TLV_MGMT_TX_FRAME_MAX_LEN);
+	buf_len = round_up(buf_len, 4);
+
+	len += buf_len;
+	len = round_up(len, 4);
+	skb = ath10k_wmi_alloc_skb(ar, len);
+	if (!skb)
+		return ERR_PTR(-ENOMEM);
+
+	ptr = (void *)skb->data;
+	tlv = ptr;
+	tlv->tag = __cpu_to_le16(WMI_TLV_TAG_STRUCT_MGMT_TX_CMD);
+	tlv->len = __cpu_to_le16(sizeof(*cmd));
+	cmd = (void *)tlv->value;
+	cmd->vdev_id = __cpu_to_le32(vdev_id);
+	cmd->desc_id = 0;
+	cmd->chanfreq = 0;
+	cmd->buf_len = __cpu_to_le32(buf_len);
+	cmd->frame_len = __cpu_to_le32(msdu->len);
+	mgmt_frame_dma = dma_map_single(arvif->ar->dev, msdu->data,
+					msdu->len, DMA_TO_DEVICE);
+	if (!mgmt_frame_dma)
+		return ERR_PTR(-ENOMEM);
+
+	cmd->paddr = __cpu_to_le64(mgmt_frame_dma);
+
+	ptr += sizeof(*tlv);
+	ptr += sizeof(*cmd);
+
+	tlv = ptr;
+	tlv->tag = __cpu_to_le16(WMI_TLV_TAG_ARRAY_BYTE);
+	tlv->len = __cpu_to_le16(buf_len);
+
+	ptr += sizeof(*tlv);
+	memcpy(ptr, msdu->data, buf_len);
+
+	return skb;
+}
+
 static struct sk_buff *
 ath10k_wmi_tlv_op_gen_force_fw_hang(struct ath10k *ar,
 				    enum wmi_force_fw_hang_type type,
@@ -3258,6 +3366,7 @@ static struct wmi_cmd_map wmi_tlv_cmd_map = {
 	.bcn_filter_rx_cmdid = WMI_TLV_BCN_FILTER_RX_CMDID,
 	.prb_req_filter_rx_cmdid = WMI_TLV_PRB_REQ_FILTER_RX_CMDID,
 	.mgmt_tx_cmdid = WMI_TLV_MGMT_TX_CMDID,
+	.mgmt_tx_send_cmdid = WMI_TLV_MGMT_TX_SEND_CMD,
 	.prb_tmpl_cmdid = WMI_TLV_PRB_TMPL_CMDID,
 	.addba_clear_resp_cmdid = WMI_TLV_ADDBA_CLEAR_RESP_CMDID,
 	.addba_send_cmdid = WMI_TLV_ADDBA_SEND_CMDID,
@@ -3592,6 +3701,7 @@ static const struct wmi_ops wmi_tlv_ops = {
 	.gen_request_stats = ath10k_wmi_tlv_op_gen_request_stats,
 	.gen_force_fw_hang = ath10k_wmi_tlv_op_gen_force_fw_hang,
 	/* .gen_mgmt_tx = not implemented; HTT is used */
+	.gen_mgmt_tx =  ath10k_wmi_tlv_op_gen_mgmt_tx,
 	.gen_dbglog_cfg = ath10k_wmi_tlv_op_gen_dbglog_cfg,
 	.gen_pktlog_enable = ath10k_wmi_tlv_op_gen_pktlog_enable,
 	.gen_pktlog_disable = ath10k_wmi_tlv_op_gen_pktlog_disable,
diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.h b/drivers/net/wireless/ath/ath10k/wmi-tlv.h
index 22cf011e839a..da89128e8dd6 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-tlv.h
+++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2014 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -22,6 +22,7 @@
 #define WMI_TLV_CMD_UNSUPPORTED 0
 #define WMI_TLV_PDEV_PARAM_UNSUPPORTED 0
 #define WMI_TLV_VDEV_PARAM_UNSUPPORTED 0
+#define WMI_TLV_MGMT_TX_FRAME_MAX_LEN	64
 
 enum wmi_tlv_grp_id {
 	WMI_TLV_GRP_START = 0x3,
@@ -132,6 +133,7 @@ enum wmi_tlv_cmd_id {
 	WMI_TLV_PRB_REQ_FILTER_RX_CMDID,
 	WMI_TLV_MGMT_TX_CMDID,
 	WMI_TLV_PRB_TMPL_CMDID,
+	WMI_TLV_MGMT_TX_SEND_CMD,
 	WMI_TLV_ADDBA_CLEAR_RESP_CMDID = WMI_TLV_CMD(WMI_TLV_GRP_BA_NEG),
 	WMI_TLV_ADDBA_SEND_CMDID,
 	WMI_TLV_ADDBA_STATUS_CMDID,
@@ -890,6 +892,63 @@ enum wmi_tlv_tag {
 	WMI_TLV_TAG_STRUCT_SAP_OFL_DEL_STA_EVENT,
 	WMI_TLV_TAG_STRUCT_APFIND_CMD_PARAM,
 	WMI_TLV_TAG_STRUCT_APFIND_EVENT_HDR,
+	WMI_TLV_TAG_STRUCT_OCB_SET_SCHED_CMD,
+	WMI_TLV_TAG_STRUCT_OCB_SET_SCHED_EVENT,
+	WMI_TLV_TAG_STRUCT_OCB_SET_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_OCB_SET_CONFIG_RESP_EVENT,
+	WMI_TLV_TAG_STRUCT_OCB_SET_UTC_TIME_CMD,
+	WMI_TLV_TAG_STRUCT_OCB_START_TIMING_ADVERT_CMD,
+	WMI_TLV_TAG_STRUCT_OCB_STOP_TIMING_ADVERT_CMD,
+	WMI_TLV_TAG_STRUCT_OCB_GET_TSF_TIMER_CMD,
+	WMI_TLV_TAG_STRUCT_OCB_GET_TSF_TIMER_RESP_EVENT,
+	WMI_TLV_TAG_STRUCT_DCC_GET_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_DCC_CHANNEL_STATS_REQUEST,
+	WMI_TLV_TAG_STRUCT_DCC_GET_STATS_RESP_EVENT,
+	WMI_TLV_TAG_STRUCT_DCC_CLEAR_STATS_CMD,
+	WMI_TLV_TAG_STRUCT_DCC_UPDATE_NDL_CMD,
+	WMI_TLV_TAG_STRUCT_DCC_UPDATE_NDL_RESP_EVENT,
+	WMI_TLV_TAG_STRUCT_DCC_STATS_EVENT,
+	WMI_TLV_TAG_STRUCT_OCB_CHANNEL,
+	WMI_TLV_TAG_STRUCT_OCB_SCHEDULE_ELEMENT,
+	WMI_TLV_TAG_STRUCT_DCC_NDL_STATS_PER_CHANNEL,
+	WMI_TLV_TAG_STRUCT_DCC_NDL_CHAN,
+	WMI_TLV_TAG_STRUCT_QOS_PARAMETER,
+	WMI_TLV_TAG_STRUCT_DCC_NDL_ACTIVE_STATE_CONFIG,
+	WMI_TLV_TAG_STRUCT_ROAM_SCAN_EXTENDED_THRESHOLD_PARAM,
+	WMI_TLV_TAG_STRUCT_ROAM_FILTER_FIXED_PARAM,
+	WMI_TLV_TAG_STRUCT_PASSPOINT_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_PASSPOINT_EVENT_HDR,
+	WMI_TLV_TAG_STRUCT_EXTSCAN_CONFIGURE_HOTLIST_SSID_MONITOR_CMD,
+	WMI_TLV_TAG_STRUCT_EXTSCAN_HOTLIST_SSID_MATCH_EVENT,
+	WMI_TLV_TAG_STRUCT_VDEV_TSF_TSTAMP_ACTION_CMD,
+	WMI_TLV_TAG_STRUCT_VDEV_TSF_REPORT_EVENT,
+	WMI_TLV_TAG_STRUCT_GET_FW_MEM_DUMP,
+	WMI_TLV_TAG_STRUCT_UPDATE_FW_MEM_DUMP,
+	WMI_TLV_TAG_STRUCT_FW_MEM_DUMP_PARAMS,
+	WMI_TLV_TAG_STRUCT_DEBUG_MESG_FLUSH,
+	WMI_TLV_TAG_STRUCT_DEBUG_MESG_FLUSH_COMPLETE,
+	WMI_TLV_TAG_STRUCT_PEER_SET_RATE_REPORT_CONDITION,
+	WMI_TLV_TAG_STRUCT_ROAM_SUBNET_CHANGE_CONFIG,
+	WMI_TLV_TAG_STRUCT_VDEV_SET_IE_CMD,
+	WMI_TLV_TAG_STRUCT_RSSI_BREACH_MONITOR_CONFIG,
+	WMI_TLV_TAG_STRUCT_RSSI_BREACH_EVENT,
+	WMI_TLV_TAG_STRUCT_EVENT_INITIAL_WAKEUP,
+	WMI_TLV_TAG_STRUCT_SOC_SET_PCL_CMD,
+	WMI_TLV_TAG_STRUCT_SOC_SET_HW_MODE_CMD,
+	WMI_TLV_TAG_STRUCT_SOC_SET_HW_MODE_RESPONSE_EVENT,
+	WMI_TLV_TAG_STRUCT_SOC_HW_MODE_TRANSITION_EVENT,
+	WMI_TLV_TAG_STRUCT_VDEV_TXRX_STREAMS,
+	WMI_TLV_TAG_STRUCT_SOC_SET_HW_MODE_RESPONSE_VDEV_MAC_ENTRY,
+	WMI_TLV_TAG_STRUCT_SOC_SET_DUAL_MAC_CONFIG_CMD,
+	WMI_TLV_TAG_STRUCT_SOC_SET_DUAL_MAC_CONFIG_RESPONSE_EVENT,
+	WMI_TLV_TAG_STRUCT_IOAC_SOCK_PATTERN_T,
+	WMI_TLV_TAG_STRUCT_WOW_ENABLE_ICMPV6_NA_FLT_CMD,
+	WMI_TLV_TAG_STRUCT_DIAG_EVENT_LOG_CONFIG,
+	WMI_TLV_TAG_STRUCT_DIAG_EVENT_LOG_SUPPORTED_EVENT,
+	WMI_TLV_TAG_STRUCT_PACKET_FILTER_CONFIG,
+	WMI_TLV_TAG_STRUCT_PACKET_FILTER_ENABLE,
+	WMI_TLV_TAG_STRUCT_SAP_SET_BLACKLIST_PARAM_CMD,
+	WMI_TLV_TAG_STRUCT_MGMT_TX_CMD,
 
 	WMI_TLV_TAG_MAX
 };
@@ -965,6 +1024,50 @@ enum wmi_tlv_service {
 	WMI_TLV_SERVICE_STA_RX_IPA_OFFLOAD_SUPPORT,
 	WMI_TLV_SERVICE_MDNS_OFFLOAD,
 	WMI_TLV_SERVICE_SAP_AUTH_OFFLOAD,
+	WMI_TLV_SERVICE_DUAL_BAND_SIMULTANEOUS_SUPPORT,
+	WMI_TLV_SERVICE_OCB,
+	WMI_TLV_SERVICE_AP_ARPNS_OFFLOAD,
+	WMI_TLV_SERVICE_PER_BAND_CHAINMASK_SUPPORT,
+	WMI_TLV_SERVICE_PACKET_FILTER_OFFLOAD,
+	WMI_TLV_SERVICE_MGMT_TX_HTT,
+	WMI_TLV_SERVICE_MGMT_TX_WMI,
+	WMI_TLV_SERVICE_EXT_MSG,
+	WMI_TLV_SERVICE_MAWC,
+	WMI_TLV_SERVICE_PEER_ASSOC_CONF,
+	WMI_TLV_SERVICE_EGAP,
+	WMI_TLV_SERVICE_STA_PMF_OFFLOAD,
+	WMI_TLV_SERVICE_UNIFIED_WOW_CAPABILITY,
+	WMI_TLV_SERVICE_ENHANCED_PROXY_STA,
+	WMI_TLV_SERVICE_ATF,
+	WMI_TLV_SERVICE_COEX_GPIO,
+	WMI_TLV_SERVICE_AUX_SPECTRAL_INTF,
+	WMI_TLV_SERVICE_AUX_CHAN_LOAD_INTF,
+	WMI_TLV_SERVICE_BSS_CHANNEL_INFO_64,
+	WMI_TLV_SERVICE_ENTERPRISE_MESH,
+	WMI_TLV_SERVICE_RESTRT_CHNL_SUPPORT,
+	WMI_TLV_SERVICE_BPF_OFFLOAD,
+	WMI_TLV_SERVICE_SYNC_DELETE_CMDS,
+	WMI_TLV_SERVICE_SMART_ANTENNA_SW_SUPPORT,
+	WMI_TLV_SERVICE_SMART_ANTENNA_HW_SUPPORT,
+	WMI_TLV_SERVICE_RATECTRL_LIMIT_MAX_MIN_RATES,
+	WMI_TLV_SERVICE_NAN_DATA,
+	WMI_TLV_SERVICE_NAN_RTT,
+	WMI_TLV_SERVICE_11AX,
+	WMI_TLV_SERVICE_DEPRECATED_REPLACE,
+	WMI_TLV_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE,
+	WMI_TLV_SERVICE_ENHANCED_MCAST_FILTER,
+	WMI_TLV_SERVICE_PERIODIC_CHAN_STAT_SUPPORT,
+	WMI_TLV_SERVICE_MESH_11S,
+	WMI_TLV_SERVICE_HALF_RATE_QUARTER_RATE_SUPPORT,
+	WMI_TLV_SERVICE_VDEV_RX_FILTER,
+	WMI_TLV_SERVICE_P2P_LISTEN_OFFLOAD_SUPPORT,
+	WMI_TLV_SERVICE_MARK_FIRST_WAKEUP_PACKET,
+	WMI_TLV_SERVICE_MULTIPLE_MCAST_FILTER_SET,
+	WMI_TLV_SERVICE_HOST_MANAGED_RX_REORDER,
+	WMI_TLV_SERVICE_FLASH_RDWR_SUPPORT,
+	WMI_TLV_SERVICE_WLAN_STATS_REPORT,
+	WMI_TLV_SERVICE_TX_MSDU_ID_NEW_PARTITION_SUPPORT,
+	WMI_TLV_SERVICE_DFS_PHYERR_OFFLOAD,
 };
 
 #define WMI_SERVICE_IS_ENABLED(wmi_svc_bmap, svc_id, len) \
@@ -1121,6 +1224,8 @@ wmi_tlv_svc_map(const __le32 *in, unsigned long *out, size_t len)
 	       WMI_SERVICE_MDNS_OFFLOAD, len);
 	SVCMAP(WMI_TLV_SERVICE_SAP_AUTH_OFFLOAD,
 	       WMI_SERVICE_SAP_AUTH_OFFLOAD, len);
+	SVCMAP(WMI_TLV_SERVICE_MGMT_TX_WMI,
+	       WMI_SERVICE_MGMT_TX_WMI, len);
 }
 
 #undef SVCMAP
@@ -1643,4 +1748,12 @@ struct wmi_tlv_tx_pause_ev {
 
 void ath10k_wmi_tlv_attach(struct ath10k *ar);
 
+struct wmi_tlv_mgmt_tx_cmd {
+	__le32 vdev_id;
+	__le32 desc_id;
+	__le32 chanfreq;
+	__le64 paddr;
+	__le32 frame_len;
+	__le32 buf_len;
+} __packed;
 #endif
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index cad2e42dcef6..58dc2189ba49 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -29,6 +29,7 @@
 #include "p2p.h"
 #include "hw.h"
 #include "hif.h"
+#include "txrx.h"
 
 #define ATH10K_WMI_BARRIER_ECHO_ID 0xBA991E9
 #define ATH10K_WMI_BARRIER_TIMEOUT_HZ (3 * HZ)
@@ -4456,6 +4457,74 @@ void ath10k_wmi_event_pdev_tpc_config(struct ath10k *ar, struct sk_buff *skb)
 		   __le32_to_cpu(ev->rate_max));
 }
 
+static void
+ath10k_wmi_handle_tdls_peer_event(struct ath10k *ar, struct sk_buff *skb)
+{
+	struct wmi_tdls_peer_event *ev;
+	struct ath10k_peer *peer;
+	struct ath10k_vif *arvif;
+	int vdev_id;
+	int peer_status;
+	int peer_reason;
+	u8 reason;
+
+	if (skb->len < sizeof(*ev)) {
+		ath10k_err(ar, "received tdls peer event with invalid size (%d bytes)\n",
+			   skb->len);
+		return;
+	}
+
+	ev = (struct wmi_tdls_peer_event *)skb->data;
+	vdev_id = __le32_to_cpu(ev->vdev_id);
+	peer_status = __le32_to_cpu(ev->peer_status);
+	peer_reason = __le32_to_cpu(ev->peer_reason);
+
+	spin_lock_bh(&ar->data_lock);
+	peer = ath10k_peer_find(ar, vdev_id, ev->peer_macaddr.addr);
+	spin_unlock_bh(&ar->data_lock);
+
+	if (!peer) {
+		ath10k_warn(ar, "failed to find peer entry for %pM\n",
+			    ev->peer_macaddr.addr);
+		return;
+	}
+
+	switch (peer_status) {
+	case WMI_TDLS_SHOULD_TEARDOWN:
+		switch (peer_reason) {
+		case WMI_TDLS_TEARDOWN_REASON_PTR_TIMEOUT:
+		case WMI_TDLS_TEARDOWN_REASON_NO_RESPONSE:
+		case WMI_TDLS_TEARDOWN_REASON_RSSI:
+			reason = WLAN_REASON_TDLS_TEARDOWN_UNREACHABLE;
+			break;
+		default:
+			reason = WLAN_REASON_TDLS_TEARDOWN_UNSPECIFIED;
+			break;
+		}
+
+		arvif = ath10k_get_arvif(ar, vdev_id);
+		if (!arvif) {
+			ath10k_warn(ar, "received tdls peer event for invalid vdev id %u\n",
+				    vdev_id);
+			return;
+		}
+
+		ieee80211_tdls_oper_request(arvif->vif, ev->peer_macaddr.addr,
+					    NL80211_TDLS_TEARDOWN, reason,
+					    GFP_ATOMIC);
+
+		ath10k_dbg(ar, ATH10K_DBG_WMI,
+			   "received tdls teardown event for peer %pM reason %u\n",
+			   ev->peer_macaddr.addr, peer_reason);
+		break;
+	default:
+		ath10k_dbg(ar, ATH10K_DBG_WMI,
+			   "received unknown tdls peer event %u\n",
+			   peer_status);
+		break;
+	}
+}
+
 void ath10k_wmi_event_pdev_ftm_intg(struct ath10k *ar, struct sk_buff *skb)
 {
 	ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_PDEV_FTM_INTG_EVENTID\n");
@@ -5477,6 +5546,9 @@ static void ath10k_wmi_10_4_op_rx(struct ath10k *ar, struct sk_buff *skb)
 	case WMI_10_4_PDEV_TPC_CONFIG_EVENTID:
 		ath10k_wmi_event_pdev_tpc_config(ar, skb);
 		break;
+	case WMI_10_4_TDLS_PEER_EVENTID:
+		ath10k_wmi_handle_tdls_peer_event(ar, skb);
+		break;
 	default:
 		ath10k_warn(ar, "Unknown eventid: %d\n", id);
 		break;
diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h
index c02b21cff38d..c7b30ed9015d 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.h
+++ b/drivers/net/wireless/ath/ath10k/wmi.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2005-2011 Atheros Communications Inc.
- * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
+ * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -195,6 +195,8 @@ enum wmi_service {
 	WMI_SERVICE_SMART_LOGGING_SUPPORT,
 	WMI_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE,
 	WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY,
+	WMI_SERVICE_MGMT_TX_WMI,
+	WMI_SERVICE_TDLS_WIDER_BANDWIDTH,
 
 	/* keep last */
 	WMI_SERVICE_MAX,
@@ -336,6 +338,7 @@ enum wmi_10_4_service {
 	WMI_10_4_SERVICE_TDLS_UAPSD_SLEEP_STA,
 	WMI_10_4_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE,
 	WMI_10_4_SERVICE_TDLS_EXPLICIT_MODE_ONLY,
+	WMI_10_4_SERVICE_TDLS_WIDER_BANDWIDTH,
 };
 
 static inline char *wmi_service_name(int service_id)
@@ -444,6 +447,7 @@ static inline char *wmi_service_name(int service_id)
 	SVCSTR(WMI_SERVICE_SMART_LOGGING_SUPPORT);
 	SVCSTR(WMI_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE);
 	SVCSTR(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY);
+	SVCSTR(WMI_SERVICE_TDLS_WIDER_BANDWIDTH);
 	default:
 		return NULL;
 	}
@@ -740,6 +744,8 @@ static inline void wmi_10_4_svc_map(const __le32 *in, unsigned long *out,
 	       WMI_SERVICE_TDLS_CONN_TRACKER_IN_HOST_MODE, len);
 	SVCMAP(WMI_10_4_SERVICE_TDLS_EXPLICIT_MODE_ONLY,
 	       WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY, len);
+	SVCMAP(WMI_10_4_SERVICE_TDLS_WIDER_BANDWIDTH,
+	       WMI_SERVICE_TDLS_WIDER_BANDWIDTH, len);
 }
 
 #undef SVCMAP
@@ -797,6 +803,7 @@ struct wmi_cmd_map {
 	u32 bcn_filter_rx_cmdid;
 	u32 prb_req_filter_rx_cmdid;
 	u32 mgmt_tx_cmdid;
+	u32 mgmt_tx_send_cmdid;
 	u32 prb_tmpl_cmdid;
 	u32 addba_clear_resp_cmdid;
 	u32 addba_send_cmdid;
@@ -2922,7 +2929,7 @@ struct wmi_ext_resource_config_10_4_cmd {
 	__le32 max_tdls_concurrent_buffer_sta;
 };
 
-/* strucutre describing host memory chunk. */
+/* structure describing host memory chunk. */
 struct host_memory_chunk {
 	/* id of the request that is passed up in service ready */
 	__le32 req_id;
@@ -5236,7 +5243,8 @@ enum wmi_10_4_vdev_param {
 #define WMI_VDEV_PARAM_TXBF_MU_TX_BFER BIT(3)
 
 #define WMI_TXBF_STS_CAP_OFFSET_LSB	4
-#define WMI_TXBF_STS_CAP_OFFSET_MASK	0xf0
+#define WMI_TXBF_STS_CAP_OFFSET_MASK	0x70
+#define WMI_TXBF_CONF_IMPLICIT_BF       BIT(7)
 #define WMI_BF_SOUND_DIM_OFFSET_LSB	8
 #define WMI_BF_SOUND_DIM_OFFSET_MASK	0xf00
 
diff --git a/drivers/net/wireless/ath/ath10k/wow.c b/drivers/net/wireless/ath/ath10k/wow.c
index 0d46d6dc7578..c4cbccb29b31 100644
--- a/drivers/net/wireless/ath/ath10k/wow.c
+++ b/drivers/net/wireless/ath/ath10k/wow.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2015-2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath10k/wow.h b/drivers/net/wireless/ath/ath10k/wow.h
index 9745b9ddc7f5..6e810105b775 100644
--- a/drivers/net/wireless/ath/ath10k/wow.h
+++ b/drivers/net/wireless/ath/ath10k/wow.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2015,2017 Qualcomm Atheros, Inc.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c
index b53eb2b85f02..2ba8cf3f38af 100644
--- a/drivers/net/wireless/ath/ath6kl/cfg80211.c
+++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c
@@ -2766,7 +2766,6 @@ static int ath6kl_start_ap(struct wiphy *wiphy, struct net_device *dev,
 	struct ieee80211_mgmt *mgmt;
 	bool hidden = false;
 	u8 *ies;
-	int ies_len;
 	struct wmi_connect_cmd p;
 	int res;
 	int i, ret;
@@ -2804,7 +2803,6 @@ static int ath6kl_start_ap(struct wiphy *wiphy, struct net_device *dev,
 	ies = mgmt->u.beacon.variable;
 	if (ies > info->beacon.head + info->beacon.head_len)
 		return -EINVAL;
-	ies_len = info->beacon.head + info->beacon.head_len - ies;
 
 	if (info->ssid == NULL)
 		return -EINVAL;
diff --git a/drivers/net/wireless/ath/ath6kl/txrx.c b/drivers/net/wireless/ath/ath6kl/txrx.c
index 1379906bf849..8da9506f8c2b 100644
--- a/drivers/net/wireless/ath/ath6kl/txrx.c
+++ b/drivers/net/wireless/ath/ath6kl/txrx.c
@@ -1001,7 +1001,7 @@ static void aggr_slice_amsdu(struct aggr_info *p_aggr,
 
 	while (amsdu_len > mac_hdr_len) {
 		hdr = (struct ethhdr *) framep;
-		payload_8023_len = ntohs(hdr->h_proto);
+		payload_8023_len = be16_to_cpu(hdr->h_proto);
 
 		if (payload_8023_len < MIN_MSDU_SUBFRAME_PAYLOAD_LEN ||
 		    payload_8023_len > MAX_MSDU_SUBFRAME_PAYLOAD_LEN) {
diff --git a/drivers/net/wireless/ath/ath9k/Kconfig b/drivers/net/wireless/ath/ath9k/Kconfig
index 783a38f1a626..1f3523019509 100644
--- a/drivers/net/wireless/ath/ath9k/Kconfig
+++ b/drivers/net/wireless/ath/ath9k/Kconfig
@@ -61,13 +61,12 @@ config ATH9K_DEBUGFS
 	depends on ATH9K && DEBUG_FS
 	select MAC80211_DEBUGFS
 	select ATH9K_COMMON_DEBUG
-	select RELAY
 	---help---
 	  Say Y, if you need access to ath9k's statistics for
 	  interrupts, rate control, etc.
 
-	  Also required for changing debug message flags at run time.
-	  As well as access to the FFT/spectral data and TX99.
+	  Also required for changing debug message flags at run time and for
+	  TX99.
 
 config ATH9K_STATION_STATISTICS
 	bool "Detailed station statistics"
@@ -177,7 +176,6 @@ config ATH9K_HTC_DEBUGFS
 	bool "Atheros ath9k_htc debugging"
 	depends on ATH9K_HTC && DEBUG_FS
 	select ATH9K_COMMON_DEBUG
-	select RELAY
 	---help---
 	  Say Y, if you need access to ath9k_htc's statistics.
 	  As well as access to the FFT/spectral data.
@@ -192,3 +190,11 @@ config ATH9K_HWRNG
 
 	  Say Y, feeds the entropy directly from the WiFi driver to the input
 	  pool.
+
+config ATH9K_COMMON_SPECTRAL
+	bool "Atheros ath9k/ath9k_htc spectral scan support"
+	depends on ATH9K_DEBUGFS || ATH9K_HTC_DEBUGFS
+	select RELAY
+	default n
+	---help---
+	  Say Y to enable access to the FFT/spectral data via debugfs.
diff --git a/drivers/net/wireless/ath/ath9k/Makefile b/drivers/net/wireless/ath/ath9k/Makefile
index d804ce7391a0..f71b2ad8275c 100644
--- a/drivers/net/wireless/ath/ath9k/Makefile
+++ b/drivers/net/wireless/ath/ath9k/Makefile
@@ -62,8 +62,8 @@ ath9k_common-y:=	common.o \
 			common-init.o \
 			common-beacon.o \
 
-ath9k_common-$(CONFIG_ATH9K_COMMON_DEBUG) += common-debug.o \
-					     common-spectral.o
+ath9k_common-$(CONFIG_ATH9K_COMMON_DEBUG) += common-debug.o
+ath9k_common-$(CONFIG_ATH9K_COMMON_SPECTRAL) += common-spectral.o
 
 ath9k_htc-y +=	htc_hst.o \
 		hif_usb.o \
diff --git a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
index c2e210c0a770..f019a20e5a1f 100644
--- a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
+++ b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.c
@@ -3310,6 +3310,12 @@ static int ar9300_eeprom_restore_internal(struct ath_hw *ah,
 	if (ar9300_check_eeprom_header(ah, read, cptr))
 		goto found;
 
+	cptr = AR9300_BASE_ADDR_4K;
+	ath_dbg(common, EEPROM, "Trying EEPROM access at Address 0x%04x\n",
+		cptr);
+	if (ar9300_check_eeprom_header(ah, read, cptr))
+		goto found;
+
 	cptr = AR9300_BASE_ADDR_512;
 	ath_dbg(common, EEPROM, "Trying EEPROM access at Address 0x%04x\n",
 		cptr);
@@ -3430,6 +3436,60 @@ static u32 ar9003_dump_modal_eeprom(char *buf, u32 len, u32 size,
 	return len;
 }
 
+static u32 ar9003_dump_cal_data(struct ath_hw *ah, char *buf, u32 len, u32 size,
+				bool is_2g)
+{
+	struct ar9300_eeprom *eep = &ah->eeprom.ar9300_eep;
+	struct ar9300_base_eep_hdr *pBase;
+	struct ar9300_cal_data_per_freq_op_loop *cal_pier;
+	int cal_pier_nr;
+	int freq;
+	int i, j;
+
+	pBase = &eep->baseEepHeader;
+
+	if (is_2g)
+		cal_pier_nr = AR9300_NUM_2G_CAL_PIERS;
+	else
+		cal_pier_nr = AR9300_NUM_5G_CAL_PIERS;
+
+	for (i = 0; i < AR9300_MAX_CHAINS; i++) {
+		if (!((pBase->txrxMask >> i) & 1))
+			continue;
+
+		len += snprintf(buf + len, size - len, "Chain %d\n", i);
+
+		len += snprintf(buf + len, size - len,
+			"Freq\t ref\tvolt\ttemp\tnf_cal\tnf_pow\trx_temp\n");
+
+		for (j = 0; j < cal_pier_nr; j++) {
+			if (is_2g) {
+				cal_pier = &eep->calPierData2G[i][j];
+				freq = 2300 + eep->calFreqPier2G[j];
+			} else {
+				cal_pier = &eep->calPierData5G[i][j];
+				freq = 4800 + eep->calFreqPier5G[j] * 5;
+			}
+
+			len += snprintf(buf + len, size - len,
+				"%d\t", freq);
+
+			len += snprintf(buf + len, size - len,
+				"%d\t%d\t%d\t%d\t%d\t%d\n",
+				cal_pier->refPower,
+				cal_pier->voltMeas,
+				cal_pier->tempMeas,
+				cal_pier->rxTempMeas ?
+				N2DBM(cal_pier->rxNoisefloorCal) : 0,
+				cal_pier->rxTempMeas ?
+				N2DBM(cal_pier->rxNoisefloorPower) : 0,
+				cal_pier->rxTempMeas);
+		}
+	}
+
+	return len;
+}
+
 static u32 ath9k_hw_ar9003_dump_eeprom(struct ath_hw *ah, bool dump_base_hdr,
 				       u8 *buf, u32 len, u32 size)
 {
@@ -3441,10 +3501,18 @@ static u32 ath9k_hw_ar9003_dump_eeprom(struct ath_hw *ah, bool dump_base_hdr,
 				 "%20s :\n", "2GHz modal Header");
 		len = ar9003_dump_modal_eeprom(buf, len, size,
 						&eep->modalHeader2G);
-		len += scnprintf(buf + len, size - len,
+
+		len += scnprintf(buf + len, size - len, "Calibration data\n");
+		len = ar9003_dump_cal_data(ah, buf, len, size, true);
+
+		len +=  snprintf(buf + len, size - len,
 				 "%20s :\n", "5GHz modal Header");
 		len = ar9003_dump_modal_eeprom(buf, len, size,
 						&eep->modalHeader5G);
+
+		len += snprintf(buf + len, size - len, "Calibration data\n");
+		len = ar9003_dump_cal_data(ah, buf, len, size, false);
+
 		goto out;
 	}
 
@@ -4683,7 +4751,8 @@ static int ar9003_hw_cal_pier_get(struct ath_hw *ah,
 				  int ichain,
 				  int *pfrequency,
 				  int *pcorrection,
-				  int *ptemperature, int *pvoltage)
+				  int *ptemperature, int *pvoltage,
+				  int *pnf_cal, int *pnf_power)
 {
 	u8 *pCalPier;
 	struct ar9300_cal_data_per_freq_op_loop *pCalPierStruct;
@@ -4725,6 +4794,10 @@ static int ar9003_hw_cal_pier_get(struct ath_hw *ah,
 	*pcorrection = pCalPierStruct->refPower;
 	*ptemperature = pCalPierStruct->tempMeas;
 	*pvoltage = pCalPierStruct->voltMeas;
+	*pnf_cal = pCalPierStruct->rxTempMeas ?
+			N2DBM(pCalPierStruct->rxNoisefloorCal) : 0;
+	*pnf_power = pCalPierStruct->rxTempMeas ?
+			N2DBM(pCalPierStruct->rxNoisefloorPower) : 0;
 
 	return 0;
 }
@@ -4889,14 +4962,18 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 	int mode;
 	int lfrequency[AR9300_MAX_CHAINS],
 	    lcorrection[AR9300_MAX_CHAINS],
-	    ltemperature[AR9300_MAX_CHAINS], lvoltage[AR9300_MAX_CHAINS];
+	    ltemperature[AR9300_MAX_CHAINS], lvoltage[AR9300_MAX_CHAINS],
+	    lnf_cal[AR9300_MAX_CHAINS], lnf_pwr[AR9300_MAX_CHAINS];
 	int hfrequency[AR9300_MAX_CHAINS],
 	    hcorrection[AR9300_MAX_CHAINS],
-	    htemperature[AR9300_MAX_CHAINS], hvoltage[AR9300_MAX_CHAINS];
+	    htemperature[AR9300_MAX_CHAINS], hvoltage[AR9300_MAX_CHAINS],
+	    hnf_cal[AR9300_MAX_CHAINS], hnf_pwr[AR9300_MAX_CHAINS];
 	int fdiff;
 	int correction[AR9300_MAX_CHAINS],
-	    voltage[AR9300_MAX_CHAINS], temperature[AR9300_MAX_CHAINS];
-	int pfrequency, pcorrection, ptemperature, pvoltage;
+	    voltage[AR9300_MAX_CHAINS], temperature[AR9300_MAX_CHAINS],
+	    nf_cal[AR9300_MAX_CHAINS], nf_pwr[AR9300_MAX_CHAINS];
+	int pfrequency, pcorrection, ptemperature, pvoltage,
+	    pnf_cal, pnf_pwr;
 	struct ath_common *common = ath9k_hw_common(ah);
 
 	mode = (frequency >= 4000);
@@ -4914,7 +4991,8 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 		for (ipier = 0; ipier < npier; ipier++) {
 			if (!ar9003_hw_cal_pier_get(ah, mode, ipier, ichain,
 						    &pfrequency, &pcorrection,
-						    &ptemperature, &pvoltage)) {
+						    &ptemperature, &pvoltage,
+						    &pnf_cal, &pnf_pwr)) {
 				fdiff = frequency - pfrequency;
 
 				/*
@@ -4936,6 +5014,8 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 						htemperature[ichain] =
 						    ptemperature;
 						hvoltage[ichain] = pvoltage;
+						hnf_cal[ichain] = pnf_cal;
+						hnf_pwr[ichain] = pnf_pwr;
 					}
 				}
 				if (fdiff >= 0) {
@@ -4952,6 +5032,8 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 						ltemperature[ichain] =
 						    ptemperature;
 						lvoltage[ichain] = pvoltage;
+						lnf_cal[ichain] = pnf_cal;
+						lnf_pwr[ichain] = pnf_pwr;
 					}
 				}
 			}
@@ -4960,15 +5042,20 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 
 	/* interpolate  */
 	for (ichain = 0; ichain < AR9300_MAX_CHAINS; ichain++) {
-		ath_dbg(common, EEPROM, "ch=%d f=%d low=%d %d h=%d %d\n",
+		ath_dbg(common, EEPROM,
+			"ch=%d f=%d low=%d %d h=%d %d n=%d %d p=%d %d\n",
 			ichain, frequency, lfrequency[ichain],
 			lcorrection[ichain], hfrequency[ichain],
-			hcorrection[ichain]);
+			hcorrection[ichain], lnf_cal[ichain],
+			hnf_cal[ichain], lnf_pwr[ichain],
+			hnf_pwr[ichain]);
 		/* they're the same, so just pick one */
 		if (hfrequency[ichain] == lfrequency[ichain]) {
 			correction[ichain] = lcorrection[ichain];
 			voltage[ichain] = lvoltage[ichain];
 			temperature[ichain] = ltemperature[ichain];
+			nf_cal[ichain] = lnf_cal[ichain];
+			nf_pwr[ichain] = lnf_pwr[ichain];
 		}
 		/* the low frequency is good */
 		else if (frequency - lfrequency[ichain] < 1000) {
@@ -4992,12 +5079,26 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 						hfrequency[ichain],
 						lvoltage[ichain],
 						hvoltage[ichain]);
+
+				nf_cal[ichain] = interpolate(frequency,
+						lfrequency[ichain],
+						hfrequency[ichain],
+						lnf_cal[ichain],
+						hnf_cal[ichain]);
+
+				nf_pwr[ichain] = interpolate(frequency,
+						lfrequency[ichain],
+						hfrequency[ichain],
+						lnf_pwr[ichain],
+						hnf_pwr[ichain]);
 			}
 			/* only low is good, use it */
 			else {
 				correction[ichain] = lcorrection[ichain];
 				temperature[ichain] = ltemperature[ichain];
 				voltage[ichain] = lvoltage[ichain];
+				nf_cal[ichain] = lnf_cal[ichain];
+				nf_pwr[ichain] = lnf_pwr[ichain];
 			}
 		}
 		/* only high is good, use it */
@@ -5005,10 +5106,14 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 			correction[ichain] = hcorrection[ichain];
 			temperature[ichain] = htemperature[ichain];
 			voltage[ichain] = hvoltage[ichain];
+			nf_cal[ichain] = hnf_cal[ichain];
+			nf_pwr[ichain] = hnf_pwr[ichain];
 		} else {	/* nothing is good, presume 0???? */
 			correction[ichain] = 0;
 			temperature[ichain] = 0;
 			voltage[ichain] = 0;
+			nf_cal[ichain] = 0;
+			nf_pwr[ichain] = 0;
 		}
 	}
 
@@ -5019,6 +5124,16 @@ static int ar9003_hw_calibration_apply(struct ath_hw *ah, int frequency)
 		"for frequency=%d, calibration correction = %d %d %d\n",
 		frequency, correction[0], correction[1], correction[2]);
 
+	/* Store calibrated noise floor values */
+	for (ichain = 0; ichain < AR5416_MAX_CHAINS; ichain++)
+		if (mode) {
+			ah->nf_5g.cal[ichain] = nf_cal[ichain];
+			ah->nf_5g.pwr[ichain] = nf_pwr[ichain];
+		} else {
+			ah->nf_2g.cal[ichain] = nf_cal[ichain];
+			ah->nf_2g.pwr[ichain] = nf_pwr[ichain];
+		}
+
 	return 0;
 }
 
diff --git a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.h b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.h
index bd2269c7de6b..e8fda54acfe3 100644
--- a/drivers/net/wireless/ath/ath9k/ar9003_eeprom.h
+++ b/drivers/net/wireless/ath/ath9k/ar9003_eeprom.h
@@ -62,6 +62,16 @@
  */
 #define AR9300_PWR_TABLE_OFFSET  0
 
+/* Noise power data definitions
+ * units are: 4 x dBm - NOISE_PWR_DATA_OFFSET
+ * (e.g. -25 = (-25/4 - 90) = -96.25 dBm)
+ * range (for 6 signed bits) is (-32 to 31) + offset => -122dBm to -59dBm
+ * resolution (2 bits) is 0.25dBm
+ */
+#define NOISE_PWR_DATA_OFFSET	-90
+#define NOISE_PWR_DBM_2_INT(_p)	((((_p) + 3) >> 2) + NOISE_PWR_DATA_OFFSET)
+#define N2DBM(_p)		NOISE_PWR_DBM_2_INT(_p)
+
 /* byte addressable */
 #define AR9300_EEPROM_SIZE (16*1024)
 
diff --git a/drivers/net/wireless/ath/ath9k/calib.c b/drivers/net/wireless/ath/ath9k/calib.c
index 13ab6bc46775..695c779ae8cf 100644
--- a/drivers/net/wireless/ath/ath9k/calib.c
+++ b/drivers/net/wireless/ath/ath9k/calib.c
@@ -58,9 +58,15 @@ static struct ath_nf_limits *ath9k_hw_get_nf_limits(struct ath_hw *ah,
 }
 
 static s16 ath9k_hw_get_default_nf(struct ath_hw *ah,
-				   struct ath9k_channel *chan)
+				   struct ath9k_channel *chan,
+				   int chain)
 {
-	return ath9k_hw_get_nf_limits(ah, chan)->nominal;
+	s16 calib_nf = ath9k_hw_get_nf_limits(ah, chan)->cal[chain];
+
+	if (calib_nf)
+		return calib_nf;
+	else
+		return ath9k_hw_get_nf_limits(ah, chan)->nominal;
 }
 
 s16 ath9k_hw_getchan_noise(struct ath_hw *ah, struct ath9k_channel *chan,
@@ -70,7 +76,7 @@ s16 ath9k_hw_getchan_noise(struct ath_hw *ah, struct ath9k_channel *chan,
 
 	if (nf) {
 		s8 delta = nf - ATH9K_NF_CAL_NOISE_THRESH -
-			   ath9k_hw_get_default_nf(ah, chan);
+			   ath9k_hw_get_default_nf(ah, chan, 0);
 		if (delta > 0)
 			noise += delta;
 	}
@@ -240,7 +246,7 @@ int ath9k_hw_loadnf(struct ath_hw *ah, struct ath9k_channel *chan)
 	unsigned i, j;
 	u8 chainmask = (ah->rxchainmask << 3) | ah->rxchainmask;
 	struct ath_common *common = ath9k_hw_common(ah);
-	s16 default_nf = ath9k_hw_get_default_nf(ah, chan);
+	s16 default_nf = ath9k_hw_get_nf_limits(ah, chan)->nominal;
 	u32 bb_agc_ctl = REG_READ(ah, AR_PHY_AGC_CONTROL);
 
 	if (ah->caldata)
@@ -258,8 +264,13 @@ int ath9k_hw_loadnf(struct ath_hw *ah, struct ath9k_channel *chan)
 				nfval = ah->nf_override;
 			else if (h)
 				nfval = h[i].privNF;
-			else
-				nfval = default_nf;
+			else {
+				/* Try to get calibrated noise floor value */
+				nfval =
+				    ath9k_hw_get_nf_limits(ah, chan)->cal[i];
+				if (nfval > -60 || nfval < -127)
+					nfval = default_nf;
+			}
 
 			REG_RMW(ah, ah->nf_regs[i],
 				(((u32) nfval << 1) & 0x1ff), 0x1ff);
@@ -429,20 +440,19 @@ void ath9k_init_nfcal_hist_buffer(struct ath_hw *ah,
 				  struct ath9k_channel *chan)
 {
 	struct ath9k_nfcal_hist *h;
-	s16 default_nf;
-	int i, j;
+	int i, j, k = 0;
 
 	ah->caldata->channel = chan->channel;
 	ah->caldata->channelFlags = chan->channelFlags;
 	h = ah->caldata->nfCalHist;
-	default_nf = ath9k_hw_get_default_nf(ah, chan);
 	for (i = 0; i < NUM_NF_READINGS; i++) {
 		h[i].currIndex = 0;
-		h[i].privNF = default_nf;
+		h[i].privNF = ath9k_hw_get_default_nf(ah, chan, k);
 		h[i].invalidNFcount = AR_PHY_CCA_FILTERWINDOW_LENGTH;
-		for (j = 0; j < ATH9K_NF_CAL_HIST_MAX; j++) {
-			h[i].nfCalBuffer[j] = default_nf;
-		}
+		for (j = 0; j < ATH9K_NF_CAL_HIST_MAX; j++)
+			h[i].nfCalBuffer[j] = h[i].privNF;
+		if (++k >= AR5416_MAX_CHAINS)
+			k = 0;
 	}
 }
 
diff --git a/drivers/net/wireless/ath/ath9k/common-spectral.h b/drivers/net/wireless/ath/ath9k/common-spectral.h
index 5d1a51d83aa6..303ab470ce34 100644
--- a/drivers/net/wireless/ath/ath9k/common-spectral.h
+++ b/drivers/net/wireless/ath/ath9k/common-spectral.h
@@ -151,7 +151,7 @@ static inline u8 spectral_bitmap_weight(u8 *bins)
 	return bins[0] & 0x3f;
 }
 
-#ifdef CONFIG_ATH9K_COMMON_DEBUG
+#ifdef CONFIG_ATH9K_COMMON_SPECTRAL
 void ath9k_cmn_spectral_init_debug(struct ath_spec_scan_priv *spec_priv, struct dentry *debugfs_phy);
 void ath9k_cmn_spectral_deinit_debug(struct ath_spec_scan_priv *spec_priv);
 
@@ -183,6 +183,6 @@ static inline int ath_cmn_process_fft(struct ath_spec_scan_priv *spec_priv,
 {
 	return 0;
 }
-#endif /* CONFIG_ATH9K_COMMON_DEBUG */
+#endif /* CONFIG_ATH9K_COMMON_SPECTRAL */
 
 #endif /* SPECTRAL_H */
diff --git a/drivers/net/wireless/ath/ath9k/dfs.c b/drivers/net/wireless/ath/ath9k/dfs.c
index 40a397fd0e0e..6fee9a464cce 100644
--- a/drivers/net/wireless/ath/ath9k/dfs.c
+++ b/drivers/net/wireless/ath/ath9k/dfs.c
@@ -123,11 +123,9 @@ static bool ath9k_check_chirping(struct ath_softc *sc, u8 *data,
 			fft = (struct ath9k_dfs_fft_40 *) (data + 2);
 			ath_dbg(common, DFS, "fixing datalen by 2\n");
 		}
-		if (IS_CHAN_HT40MINUS(ah->curchan)) {
-			int temp = is_ctl;
-			is_ctl = is_ext;
-			is_ext = temp;
-		}
+		if (IS_CHAN_HT40MINUS(ah->curchan))
+			swap(is_ctl, is_ext);
+
 		for (i = 0; i < FFT_NUM_SAMPLES; i++)
 			max_bin[i] = ath9k_get_max_index_ht40(fft + i, is_ctl,
 							      is_ext);
diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c
index 56676eaff24c..cb0eef13af1c 100644
--- a/drivers/net/wireless/ath/ath9k/hif_usb.c
+++ b/drivers/net/wireless/ath/ath9k/hif_usb.c
@@ -24,6 +24,7 @@ static const struct usb_device_id ath9k_hif_usb_ids[] = {
 	{ USB_DEVICE(0x0cf3, 0x9271) }, /* Atheros */
 	{ USB_DEVICE(0x0cf3, 0x1006) }, /* Atheros */
 	{ USB_DEVICE(0x0846, 0x9030) }, /* Netgear N150 */
+	{ USB_DEVICE(0x07b8, 0x9271) }, /* Altai WA1011N-GU */
 	{ USB_DEVICE(0x07D1, 0x3A10) }, /* Dlink Wireless 150 */
 	{ USB_DEVICE(0x13D3, 0x3327) }, /* Azurewave */
 	{ USB_DEVICE(0x13D3, 0x3328) }, /* Azurewave */
diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_init.c b/drivers/net/wireless/ath/ath9k/htc_drv_init.c
index e89e5ef2c2a4..f246e9ed4a81 100644
--- a/drivers/net/wireless/ath/ath9k/htc_drv_init.c
+++ b/drivers/net/wireless/ath/ath9k/htc_drv_init.c
@@ -729,6 +729,7 @@ static void ath9k_set_hw_capab(struct ath9k_htc_priv *priv,
 	ieee80211_hw_set(hw, SPECTRUM_MGMT);
 	ieee80211_hw_set(hw, SIGNAL_DBM);
 	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
+	ieee80211_hw_set(hw, DOESNT_SUPPORT_QOS_NDP);
 
 	if (ath9k_ps_enable)
 		ieee80211_hw_set(hw, SUPPORTS_PS);
diff --git a/drivers/net/wireless/ath/ath9k/htc_drv_main.c b/drivers/net/wireless/ath/ath9k/htc_drv_main.c
index f808e5833d7e..a82ad739ab80 100644
--- a/drivers/net/wireless/ath/ath9k/htc_drv_main.c
+++ b/drivers/net/wireless/ath/ath9k/htc_drv_main.c
@@ -1683,6 +1683,10 @@ static int ath9k_htc_ampdu_action(struct ieee80211_hw *hw,
 		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
 		break;
 	case IEEE80211_AMPDU_TX_OPERATIONAL:
+		if (tid >= ATH9K_HTC_MAX_TID) {
+			ret = -EINVAL;
+			break;
+		}
 		ista = (struct ath9k_htc_sta *) sta->drv_priv;
 		spin_lock_bh(&priv->tx.tx_lock);
 		ista->tid_state[tid] = AGGR_OPERATIONAL;
diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c
index 8c5c2dd8fa7f..cd0f023ccf77 100644
--- a/drivers/net/wireless/ath/ath9k/hw.c
+++ b/drivers/net/wireless/ath/ath9k/hw.c
@@ -922,6 +922,7 @@ static void ath9k_hw_init_interrupt_masks(struct ath_hw *ah,
 		AR_IMR_RXERR |
 		AR_IMR_RXORN |
 		AR_IMR_BCNMISC;
+	u32 msi_cfg = 0;
 
 	if (AR_SREV_9340(ah) || AR_SREV_9550(ah) || AR_SREV_9531(ah) ||
 	    AR_SREV_9561(ah))
@@ -929,22 +930,30 @@ static void ath9k_hw_init_interrupt_masks(struct ath_hw *ah,
 
 	if (AR_SREV_9300_20_OR_LATER(ah)) {
 		imr_reg |= AR_IMR_RXOK_HP;
-		if (ah->config.rx_intr_mitigation)
+		if (ah->config.rx_intr_mitigation) {
 			imr_reg |= AR_IMR_RXINTM | AR_IMR_RXMINTR;
-		else
+			msi_cfg |= AR_INTCFG_MSI_RXINTM | AR_INTCFG_MSI_RXMINTR;
+		} else {
 			imr_reg |= AR_IMR_RXOK_LP;
-
+			msi_cfg |= AR_INTCFG_MSI_RXOK;
+		}
 	} else {
-		if (ah->config.rx_intr_mitigation)
+		if (ah->config.rx_intr_mitigation) {
 			imr_reg |= AR_IMR_RXINTM | AR_IMR_RXMINTR;
-		else
+			msi_cfg |= AR_INTCFG_MSI_RXINTM | AR_INTCFG_MSI_RXMINTR;
+		} else {
 			imr_reg |= AR_IMR_RXOK;
+			msi_cfg |= AR_INTCFG_MSI_RXOK;
+		}
 	}
 
-	if (ah->config.tx_intr_mitigation)
+	if (ah->config.tx_intr_mitigation) {
 		imr_reg |= AR_IMR_TXINTM | AR_IMR_TXMINTR;
-	else
+		msi_cfg |= AR_INTCFG_MSI_TXINTM | AR_INTCFG_MSI_TXMINTR;
+	} else {
 		imr_reg |= AR_IMR_TXOK;
+		msi_cfg |= AR_INTCFG_MSI_TXOK;
+	}
 
 	ENABLE_REGWRITE_BUFFER(ah);
 
@@ -952,6 +961,16 @@ static void ath9k_hw_init_interrupt_masks(struct ath_hw *ah,
 	ah->imrs2_reg |= AR_IMR_S2_GTT;
 	REG_WRITE(ah, AR_IMR_S2, ah->imrs2_reg);
 
+	if (ah->msi_enabled) {
+		ah->msi_reg = REG_READ(ah, AR_PCIE_MSI);
+		ah->msi_reg |= AR_PCIE_MSI_HW_DBI_WR_EN;
+		ah->msi_reg &= AR_PCIE_MSI_HW_INT_PENDING_ADDR_MSI_64;
+		REG_WRITE(ah, AR_INTCFG, msi_cfg);
+		ath_dbg(ath9k_hw_common(ah), ANY,
+			"value of AR_INTCFG=0x%X, msi_cfg=0x%X\n",
+			REG_READ(ah, AR_INTCFG), msi_cfg);
+	}
+
 	if (!AR_SREV_9100(ah)) {
 		REG_WRITE(ah, AR_INTR_SYNC_CAUSE, 0xFFFFFFFF);
 		REG_WRITE(ah, AR_INTR_SYNC_ENABLE, sync_default);
diff --git a/drivers/net/wireless/ath/ath9k/hw.h b/drivers/net/wireless/ath/ath9k/hw.h
index 4ac70827d142..9804a24a2dc0 100644
--- a/drivers/net/wireless/ath/ath9k/hw.h
+++ b/drivers/net/wireless/ath/ath9k/hw.h
@@ -754,6 +754,8 @@ struct ath_nf_limits {
 	s16 max;
 	s16 min;
 	s16 nominal;
+	s16 cal[AR5416_MAX_CHAINS];
+	s16 pwr[AR5416_MAX_CHAINS];
 };
 
 enum ath_cal_list {
@@ -977,6 +979,9 @@ struct ath_hw {
 	bool tpc_enabled;
 	u8 tx_power[Ar5416RateSize];
 	u8 tx_power_stbc[Ar5416RateSize];
+	bool msi_enabled;
+	u32 msi_mask;
+	u32 msi_reg;
 };
 
 struct ath_bus_ops {
diff --git a/drivers/net/wireless/ath/ath9k/init.c b/drivers/net/wireless/ath/ath9k/init.c
index fa58a32227f5..e479fae5aab9 100644
--- a/drivers/net/wireless/ath/ath9k/init.c
+++ b/drivers/net/wireless/ath/ath9k/init.c
@@ -23,6 +23,7 @@
 #include <linux/of.h>
 #include <linux/of_net.h>
 #include <linux/relay.h>
+#include <linux/dmi.h>
 #include <net/ieee80211_radiotap.h>
 
 #include "ath9k.h"
@@ -75,6 +76,10 @@ MODULE_PARM_DESC(use_chanctx, "Enable channel context for concurrency");
 
 #endif /* CONFIG_ATH9K_CHANNEL_CONTEXT */
 
+int ath9k_use_msi;
+module_param_named(use_msi, ath9k_use_msi, int, 0444);
+MODULE_PARM_DESC(use_msi, "Use MSI instead of INTx if possible");
+
 bool is_ath9k_unloaded;
 
 #ifdef CONFIG_MAC80211_LEDS
@@ -92,6 +97,56 @@ static const struct ieee80211_tpt_blink ath9k_tpt_blink[] = {
 };
 #endif
 
+static int __init set_use_msi(const struct dmi_system_id *dmi)
+{
+	ath9k_use_msi = 1;
+	return 1;
+}
+
+static const struct dmi_system_id ath9k_quirks[] __initconst = {
+	{
+		.callback = set_use_msi,
+		.ident = "Dell Inspiron 24-3460",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 24-3460"),
+		},
+	},
+	{
+		.callback = set_use_msi,
+		.ident = "Dell Vostro 3262",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Vostro 3262"),
+		},
+	},
+	{
+		.callback = set_use_msi,
+		.ident = "Dell Inspiron 3472",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 3472"),
+		},
+	},
+	{
+		.callback = set_use_msi,
+		.ident = "Dell Vostro 15-3572",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Vostro 15-3572"),
+		},
+	},
+	{
+		.callback = set_use_msi,
+		.ident = "Dell Inspiron 14-3473",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 14-3473"),
+		},
+	},
+	{}
+};
+
 static void ath9k_deinit_softc(struct ath_softc *sc);
 
 static void ath9k_op_ps_wakeup(struct ath_common *common)
@@ -1100,6 +1155,8 @@ static int __init ath9k_init(void)
 		goto err_pci_exit;
 	}
 
+	dmi_check_system(ath9k_quirks);
+
 	return 0;
 
  err_pci_exit:
diff --git a/drivers/net/wireless/ath/ath9k/mac.c b/drivers/net/wireless/ath/ath9k/mac.c
index 77c94f9e7b61..58d02c19b6d0 100644
--- a/drivers/net/wireless/ath/ath9k/mac.c
+++ b/drivers/net/wireless/ath/ath9k/mac.c
@@ -832,6 +832,43 @@ static void __ath9k_hw_enable_interrupts(struct ath_hw *ah)
 	}
 	ath_dbg(common, INTERRUPT, "AR_IMR 0x%x IER 0x%x\n",
 		REG_READ(ah, AR_IMR), REG_READ(ah, AR_IER));
+
+	if (ah->msi_enabled) {
+		u32 _msi_reg = 0;
+		u32 i = 0;
+		u32 msi_pend_addr_mask = AR_PCIE_MSI_HW_INT_PENDING_ADDR_MSI_64;
+
+		ath_dbg(ath9k_hw_common(ah), INTERRUPT,
+			"Enabling MSI, msi_mask=0x%X\n", ah->msi_mask);
+
+		REG_WRITE(ah, AR_INTR_PRIO_ASYNC_ENABLE, ah->msi_mask);
+		REG_WRITE(ah, AR_INTR_PRIO_ASYNC_MASK, ah->msi_mask);
+		ath_dbg(ath9k_hw_common(ah), INTERRUPT,
+			"AR_INTR_PRIO_ASYNC_ENABLE=0x%X, AR_INTR_PRIO_ASYNC_MASK=0x%X\n",
+			REG_READ(ah, AR_INTR_PRIO_ASYNC_ENABLE),
+			REG_READ(ah, AR_INTR_PRIO_ASYNC_MASK));
+
+		if (ah->msi_reg == 0)
+			ah->msi_reg = REG_READ(ah, AR_PCIE_MSI);
+
+		ath_dbg(ath9k_hw_common(ah), INTERRUPT,
+			"AR_PCIE_MSI=0x%X, ah->msi_reg = 0x%X\n",
+			AR_PCIE_MSI, ah->msi_reg);
+
+		i = 0;
+		do {
+			REG_WRITE(ah, AR_PCIE_MSI,
+				  (ah->msi_reg | AR_PCIE_MSI_ENABLE)
+				  & msi_pend_addr_mask);
+			_msi_reg = REG_READ(ah, AR_PCIE_MSI);
+			i++;
+		} while ((_msi_reg & AR_PCIE_MSI_ENABLE) == 0 && i < 200);
+
+		if (i >= 200)
+			ath_err(ath9k_hw_common(ah),
+				"%s: _msi_reg = 0x%X\n",
+				__func__, _msi_reg);
+	}
 }
 
 void ath9k_hw_resume_interrupts(struct ath_hw *ah)
@@ -878,12 +915,21 @@ void ath9k_hw_set_interrupts(struct ath_hw *ah)
 	if (!(ints & ATH9K_INT_GLOBAL))
 		ath9k_hw_disable_interrupts(ah);
 
+	if (ah->msi_enabled) {
+		ath_dbg(common, INTERRUPT, "Clearing AR_INTR_PRIO_ASYNC_ENABLE\n");
+
+		REG_WRITE(ah, AR_INTR_PRIO_ASYNC_ENABLE, 0);
+		REG_READ(ah, AR_INTR_PRIO_ASYNC_ENABLE);
+	}
+
 	ath_dbg(common, INTERRUPT, "New interrupt mask 0x%x\n", ints);
 
 	mask = ints & ATH9K_INT_COMMON;
 	mask2 = 0;
 
+	ah->msi_mask = 0;
 	if (ints & ATH9K_INT_TX) {
+		ah->msi_mask |= AR_INTR_PRIO_TX;
 		if (ah->config.tx_intr_mitigation)
 			mask |= AR_IMR_TXMINTR | AR_IMR_TXINTM;
 		else {
@@ -898,6 +944,7 @@ void ath9k_hw_set_interrupts(struct ath_hw *ah)
 			mask |= AR_IMR_TXEOL;
 	}
 	if (ints & ATH9K_INT_RX) {
+		ah->msi_mask |= AR_INTR_PRIO_RXLP | AR_INTR_PRIO_RXHP;
 		if (AR_SREV_9300_20_OR_LATER(ah)) {
 			mask |= AR_IMR_RXERR | AR_IMR_RXOK_HP;
 			if (ah->config.rx_intr_mitigation) {
diff --git a/drivers/net/wireless/ath/ath9k/pci.c b/drivers/net/wireless/ath/ath9k/pci.c
index 223606311261..645f0fbd9179 100644
--- a/drivers/net/wireless/ath/ath9k/pci.c
+++ b/drivers/net/wireless/ath/ath9k/pci.c
@@ -22,6 +22,8 @@
 #include <linux/module.h>
 #include "ath9k.h"
 
+extern int ath9k_use_msi;
+
 static const struct pci_device_id ath_pci_id_table[] = {
 	{ PCI_VDEVICE(ATHEROS, 0x0023) }, /* PCI   */
 	{ PCI_VDEVICE(ATHEROS, 0x0024) }, /* PCI-E */
@@ -889,6 +891,7 @@ static int ath_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	u32 val;
 	int ret = 0;
 	char hw_name[64];
+	int msi_enabled = 0;
 
 	if (pcim_enable_device(pdev))
 		return -EIO;
@@ -960,7 +963,20 @@ static int ath_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	sc->mem = pcim_iomap_table(pdev)[0];
 	sc->driver_data = id->driver_data;
 
-	ret = request_irq(pdev->irq, ath_isr, IRQF_SHARED, "ath9k", sc);
+	if (ath9k_use_msi) {
+		if (pci_enable_msi(pdev) == 0) {
+			msi_enabled = 1;
+			dev_err(&pdev->dev, "Using MSI\n");
+		} else {
+			dev_err(&pdev->dev, "Using INTx\n");
+		}
+	}
+
+	if (!msi_enabled)
+		ret = request_irq(pdev->irq, ath_isr, IRQF_SHARED, "ath9k", sc);
+	else
+		ret = request_irq(pdev->irq, ath_isr, 0, "ath9k", sc);
+
 	if (ret) {
 		dev_err(&pdev->dev, "request_irq failed\n");
 		goto err_irq;
@@ -974,6 +990,9 @@ static int ath_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_init;
 	}
 
+	sc->sc_ah->msi_enabled = msi_enabled;
+	sc->sc_ah->msi_reg = 0;
+
 	ath9k_hw_name(sc->sc_ah, hw_name, sizeof(hw_name));
 	wiphy_info(hw->wiphy, "%s mem=0x%lx, irq=%d\n",
 		   hw_name, (unsigned long)sc->mem, pdev->irq);
diff --git a/drivers/net/wireless/ath/ath9k/recv.c b/drivers/net/wireless/ath/ath9k/recv.c
index 2197aee2bb72..a8ac42c96d71 100644
--- a/drivers/net/wireless/ath/ath9k/recv.c
+++ b/drivers/net/wireless/ath/ath9k/recv.c
@@ -826,9 +826,9 @@ static int ath9k_rx_skb_preprocess(struct ath_softc *sc,
 	sc->rx.discard_next = false;
 
 	/*
-	 * Discard zero-length packets.
+	 * Discard zero-length packets and packets smaller than an ACK
 	 */
-	if (!rx_stats->rs_datalen) {
+	if (rx_stats->rs_datalen < 10) {
 		RX_STAT_INC(rx_len_err);
 		goto corrupt;
 	}
diff --git a/drivers/net/wireless/ath/ath9k/reg.h b/drivers/net/wireless/ath/ath9k/reg.h
index 80ff69f99229..653e79611830 100644
--- a/drivers/net/wireless/ath/ath9k/reg.h
+++ b/drivers/net/wireless/ath/ath9k/reg.h
@@ -146,6 +146,14 @@
 #define AR_MACMISC_MISC_OBS_BUS_MSB_S   15
 #define AR_MACMISC_MISC_OBS_BUS_1       1
 
+#define AR_INTCFG               0x005C
+#define AR_INTCFG_MSI_RXOK      0x00000000
+#define AR_INTCFG_MSI_RXINTM    0x00000004
+#define AR_INTCFG_MSI_RXMINTR   0x00000006
+#define AR_INTCFG_MSI_TXOK      0x00000000
+#define AR_INTCFG_MSI_TXINTM    0x00000010
+#define AR_INTCFG_MSI_TXMINTR   0x00000018
+
 #define AR_DATABUF_SIZE		0x0060
 #define AR_DATABUF_SIZE_MASK	0x00000FFF
 
@@ -1256,6 +1264,13 @@ enum {
 #define AR_PCIE_MSI                             (AR_SREV_9340(ah) ? 0x40d8 : \
 						 (AR_SREV_9300_20_OR_LATER(ah) ? 0x40a4 : 0x4094))
 #define AR_PCIE_MSI_ENABLE                       0x00000001
+#define AR_PCIE_MSI_HW_DBI_WR_EN                 0x02000000
+#define AR_PCIE_MSI_HW_INT_PENDING_ADDR          0xFFA0C1FF /* bits 8..11: value must be 0x5060 */
+#define AR_PCIE_MSI_HW_INT_PENDING_ADDR_MSI_64   0xFFA0C9FF /* bits 8..11: value must be 0x5064 */
+
+#define AR_INTR_PRIO_TX               0x00000001
+#define AR_INTR_PRIO_RXLP             0x00000002
+#define AR_INTR_PRIO_RXHP             0x00000004
 
 #define AR_INTR_PRIO_SYNC_ENABLE  (AR_SREV_9340(ah) ? 0x4088 : 0x40c4)
 #define AR_INTR_PRIO_ASYNC_MASK   (AR_SREV_9340(ah) ? 0x408c : 0x40c8)
diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.c b/drivers/net/wireless/ath/wcn36xx/dxe.c
index d5c810a8cc52..a3f1f7d042a4 100644
--- a/drivers/net/wireless/ath/wcn36xx/dxe.c
+++ b/drivers/net/wireless/ath/wcn36xx/dxe.c
@@ -236,6 +236,14 @@ static int wcn36xx_dxe_init_descs(struct device *dev, struct wcn36xx_dxe_ch *wcn
 	return 0;
 }
 
+static void wcn36xx_dxe_deinit_descs(struct device *dev, struct wcn36xx_dxe_ch *wcn_ch)
+{
+	size_t size;
+
+	size = wcn_ch->desc_num * sizeof(struct wcn36xx_dxe_desc);
+	dma_free_coherent(dev, size,wcn_ch->cpu_addr, wcn_ch->dma_addr);
+}
+
 static void wcn36xx_dxe_init_tx_bd(struct wcn36xx_dxe_ch *ch,
 				   struct wcn36xx_dxe_mem_pool *pool)
 {
@@ -722,7 +730,11 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/***************************************/
 	/* Init descriptors for TX LOW channel */
 	/***************************************/
-	wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_tx_l_ch);
+	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_tx_l_ch);
+	if (ret) {
+		dev_err(wcn->dev, "Error allocating descriptor\n");
+		return ret;
+	}
 	wcn36xx_dxe_init_tx_bd(&wcn->dxe_tx_l_ch, &wcn->data_mem_pool);
 
 	/* Write channel head to a NEXT register */
@@ -740,7 +752,12 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/***************************************/
 	/* Init descriptors for TX HIGH channel */
 	/***************************************/
-	wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_tx_h_ch);
+	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_tx_h_ch);
+	if (ret) {
+		dev_err(wcn->dev, "Error allocating descriptor\n");
+		goto out_err_txh_ch;
+	}
+
 	wcn36xx_dxe_init_tx_bd(&wcn->dxe_tx_h_ch, &wcn->mgmt_mem_pool);
 
 	/* Write channel head to a NEXT register */
@@ -760,7 +777,12 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/***************************************/
 	/* Init descriptors for RX LOW channel */
 	/***************************************/
-	wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_rx_l_ch);
+	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_rx_l_ch);
+	if (ret) {
+		dev_err(wcn->dev, "Error allocating descriptor\n");
+		goto out_err_rxl_ch;
+	}
+
 
 	/* For RX we need to preallocated buffers */
 	wcn36xx_dxe_ch_alloc_skb(wcn, &wcn->dxe_rx_l_ch);
@@ -790,7 +812,11 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 	/***************************************/
 	/* Init descriptors for RX HIGH channel */
 	/***************************************/
-	wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_rx_h_ch);
+	ret = wcn36xx_dxe_init_descs(wcn->dev, &wcn->dxe_rx_h_ch);
+	if (ret) {
+		dev_err(wcn->dev, "Error allocating descriptor\n");
+		goto out_err_rxh_ch;
+	}
 
 	/* For RX we need to prealocat buffers */
 	wcn36xx_dxe_ch_alloc_skb(wcn, &wcn->dxe_rx_h_ch);
@@ -819,11 +845,19 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn)
 
 	ret = wcn36xx_dxe_request_irqs(wcn);
 	if (ret < 0)
-		goto out_err;
+		goto out_err_irq;
 
 	return 0;
 
-out_err:
+out_err_irq:
+	wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_rx_h_ch);
+out_err_rxh_ch:
+	wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_rx_l_ch);
+out_err_rxl_ch:
+	wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_tx_h_ch);
+out_err_txh_ch:
+	wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_tx_l_ch);
+
 	return ret;
 }
 
diff --git a/drivers/net/wireless/ath/wcn36xx/hal.h b/drivers/net/wireless/ath/wcn36xx/hal.h
index b765c647319d..182963522941 100644
--- a/drivers/net/wireless/ath/wcn36xx/hal.h
+++ b/drivers/net/wireless/ath/wcn36xx/hal.h
@@ -348,6 +348,13 @@ enum wcn36xx_hal_host_msg_type {
 	WCN36XX_HAL_DHCP_START_IND = 189,
 	WCN36XX_HAL_DHCP_STOP_IND = 190,
 
+	/* Scan Offload(hw) APIs */
+	WCN36XX_HAL_START_SCAN_OFFLOAD_REQ = 204,
+	WCN36XX_HAL_START_SCAN_OFFLOAD_RSP = 205,
+	WCN36XX_HAL_STOP_SCAN_OFFLOAD_REQ = 206,
+	WCN36XX_HAL_STOP_SCAN_OFFLOAD_RSP = 207,
+	WCN36XX_HAL_SCAN_OFFLOAD_IND = 210,
+
 	WCN36XX_HAL_AVOID_FREQ_RANGE_IND = 233,
 
 	WCN36XX_HAL_PRINT_REG_INFO_IND = 259,
@@ -1115,6 +1122,101 @@ struct wcn36xx_hal_finish_scan_rsp_msg {
 
 } __packed;
 
+enum wcn36xx_hal_scan_type {
+	WCN36XX_HAL_SCAN_TYPE_PASSIVE = 0x00,
+	WCN36XX_HAL_SCAN_TYPE_ACTIVE = WCN36XX_HAL_MAX_ENUM_SIZE
+};
+
+struct wcn36xx_hal_mac_ssid {
+	u8 length;
+	u8 ssid[32];
+} __packed;
+
+struct wcn36xx_hal_start_scan_offload_req_msg {
+	struct wcn36xx_hal_msg_header header;
+
+	/* BSSIDs hot list */
+	u8 num_bssid;
+	u8 bssids[4][ETH_ALEN];
+
+	/* Directed probe-requests will be sent for listed SSIDs (max 10)*/
+	u8 num_ssid;
+	struct wcn36xx_hal_mac_ssid ssids[10];
+
+	/* Report AP with hidden ssid */
+	u8 scan_hidden;
+
+	/* Self MAC address */
+	u8 mac[ETH_ALEN];
+
+	/* BSS type */
+	enum wcn36xx_hal_bss_type bss_type;
+
+	/* Scan type */
+	enum wcn36xx_hal_scan_type scan_type;
+
+	/* Minimum scanning time on each channel (ms) */
+	u32 min_ch_time;
+
+	/* Maximum scanning time on each channel */
+	u32 max_ch_time;
+
+	/* Is a p2p search */
+	u8 p2p_search;
+
+	/* Channels to scan */
+	u8 num_channel;
+	u8 channels[80];
+
+	/* IE field */
+	u16 ie_len;
+	u8 ie[0];
+} __packed;
+
+struct wcn36xx_hal_start_scan_offload_rsp_msg {
+	struct wcn36xx_hal_msg_header header;
+
+	/* success or failure */
+	u32 status;
+} __packed;
+
+enum wcn36xx_hal_scan_offload_ind_type {
+	/* Scan has been started */
+	WCN36XX_HAL_SCAN_IND_STARTED = 0x01,
+	/* Scan has been completed */
+	WCN36XX_HAL_SCAN_IND_COMPLETED = 0x02,
+	/* Moved to foreign channel */
+	WCN36XX_HAL_SCAN_IND_FOREIGN_CHANNEL = 0x08,
+	/* scan request has been dequeued */
+	WCN36XX_HAL_SCAN_IND_DEQUEUED = 0x10,
+	/* preempted by other high priority scan */
+	WCN36XX_HAL_SCAN_IND_PREEMPTED = 0x20,
+	/* scan start failed */
+	WCN36XX_HAL_SCAN_IND_FAILED = 0x40,
+	 /*scan restarted */
+	WCN36XX_HAL_SCAN_IND_RESTARTED = 0x80,
+	WCN36XX_HAL_SCAN_IND_MAX = WCN36XX_HAL_MAX_ENUM_SIZE
+};
+
+struct wcn36xx_hal_scan_offload_ind {
+	struct wcn36xx_hal_msg_header header;
+
+	u32 type;
+	u32 channel_mhz;
+	u32 scan_id;
+} __packed;
+
+struct wcn36xx_hal_stop_scan_offload_req_msg {
+	struct wcn36xx_hal_msg_header header;
+} __packed;
+
+struct wcn36xx_hal_stop_scan_offload_rsp_msg {
+	struct wcn36xx_hal_msg_header header;
+
+	/* success or failure */
+	u32 status;
+} __packed;
+
 enum wcn36xx_hal_rate_index {
 	HW_RATE_INDEX_1MBPS	= 0x82,
 	HW_RATE_INDEX_2MBPS	= 0x84,
@@ -1507,11 +1609,6 @@ struct wcn36xx_hal_edca_param_record {
 	u16 txop_limit;
 } __packed;
 
-struct wcn36xx_hal_mac_ssid {
-	u8 length;
-	u8 ssid[32];
-} __packed;
-
 /* Concurrency role. These are generic IDs that identify the various roles
  *  in the software system. */
 enum wcn36xx_hal_con_mode {
diff --git a/drivers/net/wireless/ath/wcn36xx/main.c b/drivers/net/wireless/ath/wcn36xx/main.c
index 987f1252a3cf..ab5be6d2c691 100644
--- a/drivers/net/wireless/ath/wcn36xx/main.c
+++ b/drivers/net/wireless/ath/wcn36xx/main.c
@@ -641,7 +641,6 @@ static int wcn36xx_hw_scan(struct ieee80211_hw *hw,
 			   struct ieee80211_scan_request *hw_req)
 {
 	struct wcn36xx *wcn = hw->priv;
-
 	mutex_lock(&wcn->scan_lock);
 	if (wcn->scan_req) {
 		mutex_unlock(&wcn->scan_lock);
@@ -650,11 +649,16 @@ static int wcn36xx_hw_scan(struct ieee80211_hw *hw,
 
 	wcn->scan_aborted = false;
 	wcn->scan_req = &hw_req->req;
+
 	mutex_unlock(&wcn->scan_lock);
 
-	schedule_work(&wcn->scan_work);
+	if (!get_feat_caps(wcn->fw_feat_caps, SCAN_OFFLOAD)) {
+		/* legacy manual/sw scan */
+		schedule_work(&wcn->scan_work);
+		return 0;
+	}
 
-	return 0;
+	return wcn36xx_smd_start_hw_scan(wcn, vif, &hw_req->req);
 }
 
 static void wcn36xx_cancel_hw_scan(struct ieee80211_hw *hw,
@@ -662,6 +666,12 @@ static void wcn36xx_cancel_hw_scan(struct ieee80211_hw *hw,
 {
 	struct wcn36xx *wcn = hw->priv;
 
+	if (!wcn36xx_smd_stop_hw_scan(wcn)) {
+		struct cfg80211_scan_info scan_info = { .aborted = true };
+
+		ieee80211_scan_completed(wcn->hw, &scan_info);
+	}
+
 	mutex_lock(&wcn->scan_lock);
 	wcn->scan_aborted = true;
 	mutex_unlock(&wcn->scan_lock);
diff --git a/drivers/net/wireless/ath/wcn36xx/smd.c b/drivers/net/wireless/ath/wcn36xx/smd.c
index 9c6590d5348a..2a4871ca9c72 100644
--- a/drivers/net/wireless/ath/wcn36xx/smd.c
+++ b/drivers/net/wireless/ath/wcn36xx/smd.c
@@ -73,6 +73,8 @@ static struct wcn36xx_cfg_val wcn36xx_cfg_vals[] = {
 	WCN36XX_CFG_VAL(TX_PWR_CTRL_ENABLE, 1),
 	WCN36XX_CFG_VAL(ENABLE_CLOSE_LOOP, 1),
 	WCN36XX_CFG_VAL(ENABLE_LPWR_IMG_TRANSITION, 0),
+	WCN36XX_CFG_VAL(BTC_STATIC_LEN_LE_BT, 120000),
+	WCN36XX_CFG_VAL(BTC_STATIC_LEN_LE_WLAN, 30000),
 	WCN36XX_CFG_VAL(MAX_ASSOC_LIMIT, 10),
 	WCN36XX_CFG_VAL(ENABLE_MCC_ADAPTIVE_SCHEDULER, 0),
 };
@@ -613,6 +615,85 @@ int wcn36xx_smd_finish_scan(struct wcn36xx *wcn,
 	return ret;
 }
 
+int wcn36xx_smd_start_hw_scan(struct wcn36xx *wcn, struct ieee80211_vif *vif,
+			      struct cfg80211_scan_request *req)
+{
+	struct wcn36xx_hal_start_scan_offload_req_msg msg_body;
+	int ret, i;
+
+	mutex_lock(&wcn->hal_mutex);
+	INIT_HAL_MSG(msg_body, WCN36XX_HAL_START_SCAN_OFFLOAD_REQ);
+
+	msg_body.scan_type = WCN36XX_HAL_SCAN_TYPE_ACTIVE;
+	msg_body.min_ch_time = 30;
+	msg_body.max_ch_time = 100;
+	msg_body.scan_hidden = 1;
+	memcpy(msg_body.mac, vif->addr, ETH_ALEN);
+	msg_body.p2p_search = vif->p2p;
+
+	msg_body.num_ssid = min_t(u8, req->n_ssids, ARRAY_SIZE(msg_body.ssids));
+	for (i = 0; i < msg_body.num_ssid; i++) {
+		msg_body.ssids[i].length = min_t(u8, req->ssids[i].ssid_len,
+						sizeof(msg_body.ssids[i].ssid));
+		memcpy(msg_body.ssids[i].ssid, req->ssids[i].ssid,
+		       msg_body.ssids[i].length);
+	}
+
+	msg_body.num_channel = min_t(u8, req->n_channels,
+				     sizeof(msg_body.channels));
+	for (i = 0; i < msg_body.num_channel; i++)
+		msg_body.channels[i] = req->channels[i]->hw_value;
+
+	PREPARE_HAL_BUF(wcn->hal_buf, msg_body);
+
+	wcn36xx_dbg(WCN36XX_DBG_HAL,
+		    "hal start hw-scan (channels: %u; ssids: %u; p2p: %s)\n",
+		    msg_body.num_channel, msg_body.num_ssid,
+		    msg_body.p2p_search ? "yes" : "no");
+
+	ret = wcn36xx_smd_send_and_wait(wcn, msg_body.header.len);
+	if (ret) {
+		wcn36xx_err("Sending hal_start_scan_offload failed\n");
+		goto out;
+	}
+	ret = wcn36xx_smd_rsp_status_check(wcn->hal_buf, wcn->hal_rsp_len);
+	if (ret) {
+		wcn36xx_err("hal_start_scan_offload response failed err=%d\n",
+			    ret);
+		goto out;
+	}
+out:
+	mutex_unlock(&wcn->hal_mutex);
+	return ret;
+}
+
+int wcn36xx_smd_stop_hw_scan(struct wcn36xx *wcn)
+{
+	struct wcn36xx_hal_stop_scan_offload_req_msg msg_body;
+	int ret;
+
+	mutex_lock(&wcn->hal_mutex);
+	INIT_HAL_MSG(msg_body, WCN36XX_HAL_STOP_SCAN_OFFLOAD_REQ);
+	PREPARE_HAL_BUF(wcn->hal_buf, msg_body);
+
+	wcn36xx_dbg(WCN36XX_DBG_HAL, "hal stop hw-scan\n");
+
+	ret = wcn36xx_smd_send_and_wait(wcn, msg_body.header.len);
+	if (ret) {
+		wcn36xx_err("Sending hal_stop_scan_offload failed\n");
+		goto out;
+	}
+	ret = wcn36xx_smd_rsp_status_check(wcn->hal_buf, wcn->hal_rsp_len);
+	if (ret) {
+		wcn36xx_err("hal_stop_scan_offload response failed err=%d\n",
+			    ret);
+		goto out;
+	}
+out:
+	mutex_unlock(&wcn->hal_mutex);
+	return ret;
+}
+
 static int wcn36xx_smd_switch_channel_rsp(void *buf, size_t len)
 {
 	struct wcn36xx_hal_switch_channel_rsp_msg *rsp;
@@ -2039,6 +2120,40 @@ static int wcn36xx_smd_tx_compl_ind(struct wcn36xx *wcn, void *buf, size_t len)
 	return 0;
 }
 
+static int wcn36xx_smd_hw_scan_ind(struct wcn36xx *wcn, void *buf, size_t len)
+{
+	struct wcn36xx_hal_scan_offload_ind *rsp = buf;
+	struct cfg80211_scan_info scan_info = {};
+
+	if (len != sizeof(*rsp)) {
+		wcn36xx_warn("Corrupted delete scan indication\n");
+		return -EIO;
+	}
+
+	wcn36xx_dbg(WCN36XX_DBG_HAL, "scan indication (type %x)", rsp->type);
+
+	switch (rsp->type) {
+	case WCN36XX_HAL_SCAN_IND_FAILED:
+		scan_info.aborted = true;
+	case WCN36XX_HAL_SCAN_IND_COMPLETED:
+		mutex_lock(&wcn->scan_lock);
+		wcn->scan_req = NULL;
+		mutex_unlock(&wcn->scan_lock);
+		ieee80211_scan_completed(wcn->hw, &scan_info);
+		break;
+	case WCN36XX_HAL_SCAN_IND_STARTED:
+	case WCN36XX_HAL_SCAN_IND_FOREIGN_CHANNEL:
+	case WCN36XX_HAL_SCAN_IND_DEQUEUED:
+	case WCN36XX_HAL_SCAN_IND_PREEMPTED:
+	case WCN36XX_HAL_SCAN_IND_RESTARTED:
+		break;
+	default:
+		wcn36xx_warn("Unknown scan indication type %x\n", rsp->type);
+	}
+
+	return 0;
+}
+
 static int wcn36xx_smd_missed_beacon_ind(struct wcn36xx *wcn,
 					 void *buf,
 					 size_t len)
@@ -2250,6 +2365,8 @@ int wcn36xx_smd_rsp_process(struct rpmsg_device *rpdev,
 	case WCN36XX_HAL_CH_SWITCH_RSP:
 	case WCN36XX_HAL_FEATURE_CAPS_EXCHANGE_RSP:
 	case WCN36XX_HAL_8023_MULTICAST_LIST_RSP:
+	case WCN36XX_HAL_START_SCAN_OFFLOAD_RSP:
+	case WCN36XX_HAL_STOP_SCAN_OFFLOAD_RSP:
 		memcpy(wcn->hal_buf, buf, len);
 		wcn->hal_rsp_len = len;
 		complete(&wcn->hal_rsp_compl);
@@ -2262,6 +2379,7 @@ int wcn36xx_smd_rsp_process(struct rpmsg_device *rpdev,
 	case WCN36XX_HAL_MISSED_BEACON_IND:
 	case WCN36XX_HAL_DELETE_STA_CONTEXT_IND:
 	case WCN36XX_HAL_PRINT_REG_INFO_IND:
+	case WCN36XX_HAL_SCAN_OFFLOAD_IND:
 		msg_ind = kmalloc(sizeof(*msg_ind) + len, GFP_ATOMIC);
 		if (!msg_ind) {
 			wcn36xx_err("Run out of memory while handling SMD_EVENT (%d)\n",
@@ -2298,6 +2416,8 @@ static void wcn36xx_ind_smd_work(struct work_struct *work)
 	hal_ind_msg = list_first_entry(&wcn->hal_ind_queue,
 				       struct wcn36xx_hal_ind_msg,
 				       list);
+	list_del(wcn->hal_ind_queue.next);
+	spin_unlock_irqrestore(&wcn->hal_ind_lock, flags);
 
 	msg_header = (struct wcn36xx_hal_msg_header *)hal_ind_msg->msg;
 
@@ -2326,12 +2446,14 @@ static void wcn36xx_ind_smd_work(struct work_struct *work)
 					       hal_ind_msg->msg,
 					       hal_ind_msg->msg_len);
 		break;
+	case WCN36XX_HAL_SCAN_OFFLOAD_IND:
+		wcn36xx_smd_hw_scan_ind(wcn, hal_ind_msg->msg,
+					hal_ind_msg->msg_len);
+		break;
 	default:
 		wcn36xx_err("SMD_EVENT (%d) not supported\n",
 			      msg_header->msg_type);
 	}
-	list_del(wcn->hal_ind_queue.next);
-	spin_unlock_irqrestore(&wcn->hal_ind_lock, flags);
 	kfree(hal_ind_msg);
 }
 int wcn36xx_smd_open(struct wcn36xx *wcn)
diff --git a/drivers/net/wireless/ath/wcn36xx/smd.h b/drivers/net/wireless/ath/wcn36xx/smd.h
index 013fc9546f56..8076edf40ac8 100644
--- a/drivers/net/wireless/ath/wcn36xx/smd.h
+++ b/drivers/net/wireless/ath/wcn36xx/smd.h
@@ -65,6 +65,9 @@ int wcn36xx_smd_end_scan(struct wcn36xx *wcn, u8 scan_channel);
 int wcn36xx_smd_finish_scan(struct wcn36xx *wcn,
 			    enum wcn36xx_hal_sys_mode mode);
 int wcn36xx_smd_update_scan_params(struct wcn36xx *wcn, u8 *channels, size_t channel_count);
+int wcn36xx_smd_start_hw_scan(struct wcn36xx *wcn, struct ieee80211_vif *vif,
+			      struct cfg80211_scan_request *req);
+int wcn36xx_smd_stop_hw_scan(struct wcn36xx *wcn);
 int wcn36xx_smd_add_sta_self(struct wcn36xx *wcn, struct ieee80211_vif *vif);
 int wcn36xx_smd_delete_sta_self(struct wcn36xx *wcn, u8 *addr);
 int wcn36xx_smd_delete_sta(struct wcn36xx *wcn, u8 sta_index);
diff --git a/drivers/net/wireless/ath/wil6210/boot_loader.h b/drivers/net/wireless/ath/wil6210/boot_loader.h
index c131b5e1292f..d32c1f4e533a 100644
--- a/drivers/net/wireless/ath/wil6210/boot_loader.h
+++ b/drivers/net/wireless/ath/wil6210/boot_loader.h
@@ -1,4 +1,5 @@
 /* Copyright (c) 2015 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -39,7 +40,8 @@ struct bl_dedicated_registers_v1 {
 	/* valid only for version 2 and above */
 	__le32  bl_assert_code;         /* 0x880A58 BL Assert code */
 	__le32  bl_assert_blink;        /* 0x880A5C BL Assert Branch */
-	__le32  bl_reserved[22];        /* 0x880A60 - 0x880AB4 */
+	__le32  bl_shutdown_handshake;  /* 0x880A60 BL cleaner shutdown */
+	__le32  bl_reserved[21];        /* 0x880A64 - 0x880AB4 */
 	__le32  bl_magic_number;        /* 0x880AB8 BL Magic number */
 } __packed;
 
@@ -58,4 +60,9 @@ struct bl_dedicated_registers_v0 {
 	u8	mac_address[6];			/* 0x880A4c BL mac address */
 } __packed;
 
+/* bits for bl_shutdown_handshake */
+#define BL_SHUTDOWN_HS_GRTD		BIT(0)
+#define BL_SHUTDOWN_HS_RTD		BIT(1)
+#define BL_SHUTDOWN_HS_PROT_VER(x) WIL_GET_BITS(x, 28, 31)
+
 #endif /* BOOT_LOADER_EXPORT_H_ */
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index 85d5c04618eb..768f63f38341 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -901,7 +901,7 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 			 u64 *cookie)
 {
 	const u8 *buf = params->buf;
-	size_t len = params->len;
+	size_t len = params->len, total;
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	int rc;
 	bool tx_status = false;
@@ -926,7 +926,11 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	if (len < sizeof(struct ieee80211_hdr_3addr))
 		return -EINVAL;
 
-	cmd = kmalloc(sizeof(*cmd) + len, GFP_KERNEL);
+	total = sizeof(*cmd) + len;
+	if (total < len)
+		return -EINVAL;
+
+	cmd = kmalloc(total, GFP_KERNEL);
 	if (!cmd) {
 		rc = -ENOMEM;
 		goto out;
@@ -936,7 +940,7 @@ int wil_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	cmd->len = cpu_to_le16(len);
 	memcpy(cmd->payload, buf, len);
 
-	rc = wmi_call(wil, WMI_SW_TX_REQ_CMDID, cmd, sizeof(*cmd) + len,
+	rc = wmi_call(wil, WMI_SW_TX_REQ_CMDID, cmd, total,
 		      WMI_SW_TX_COMPLETE_EVENTID, &evt, sizeof(evt), 2000);
 	if (rc == 0)
 		tx_status = !evt.evt.status;
@@ -952,9 +956,8 @@ static int wil_cfg80211_set_channel(struct wiphy *wiphy,
 				    struct cfg80211_chan_def *chandef)
 {
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
-	struct wireless_dev *wdev = wil_to_wdev(wil);
 
-	wdev->preset_chandef = *chandef;
+	wil->monitor_chandef = *chandef;
 
 	return 0;
 }
@@ -1727,9 +1730,12 @@ static int wil_cfg80211_suspend(struct wiphy *wiphy,
 
 	wil_dbg_pm(wil, "suspending\n");
 
-	wil_p2p_stop_discovery(wil);
-
+	mutex_lock(&wil->mutex);
+	mutex_lock(&wil->p2p_wdev_mutex);
+	wil_p2p_stop_radio_operations(wil);
 	wil_abort_scan(wil, true);
+	mutex_unlock(&wil->p2p_wdev_mutex);
+	mutex_unlock(&wil->mutex);
 
 out:
 	return rc;
@@ -1744,6 +1750,69 @@ static int wil_cfg80211_resume(struct wiphy *wiphy)
 	return 0;
 }
 
+static int
+wil_cfg80211_sched_scan_start(struct wiphy *wiphy,
+			      struct net_device *dev,
+			      struct cfg80211_sched_scan_request *request)
+{
+	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	int i, rc;
+
+	wil_dbg_misc(wil,
+		     "sched scan start: n_ssids %d, ie_len %zu, flags 0x%x\n",
+		     request->n_ssids, request->ie_len, request->flags);
+	for (i = 0; i < request->n_ssids; i++) {
+		wil_dbg_misc(wil, "SSID[%d]:", i);
+		wil_hex_dump_misc("SSID ", DUMP_PREFIX_OFFSET, 16, 1,
+				  request->ssids[i].ssid,
+				  request->ssids[i].ssid_len, true);
+	}
+	wil_dbg_misc(wil, "channels:");
+	for (i = 0; i < request->n_channels; i++)
+		wil_dbg_misc(wil, " %d%s", request->channels[i]->hw_value,
+			     i == request->n_channels - 1 ? "\n" : "");
+	wil_dbg_misc(wil, "n_match_sets %d, min_rssi_thold %d, delay %d\n",
+		     request->n_match_sets, request->min_rssi_thold,
+		     request->delay);
+	for (i = 0; i < request->n_match_sets; i++) {
+		struct cfg80211_match_set *ms = &request->match_sets[i];
+
+		wil_dbg_misc(wil, "MATCHSET[%d]: rssi_thold %d\n",
+			     i, ms->rssi_thold);
+		wil_hex_dump_misc("SSID ", DUMP_PREFIX_OFFSET, 16, 1,
+				  ms->ssid.ssid,
+				  ms->ssid.ssid_len, true);
+	}
+	wil_dbg_misc(wil, "n_scan_plans %d\n", request->n_scan_plans);
+	for (i = 0; i < request->n_scan_plans; i++) {
+		struct cfg80211_sched_scan_plan *sp = &request->scan_plans[i];
+
+		wil_dbg_misc(wil, "SCAN PLAN[%d]: interval %d iterations %d\n",
+			     i, sp->interval, sp->iterations);
+	}
+
+	rc = wmi_set_ie(wil, WMI_FRAME_PROBE_REQ, request->ie_len, request->ie);
+	if (rc)
+		return rc;
+	return wmi_start_sched_scan(wil, request);
+}
+
+static int
+wil_cfg80211_sched_scan_stop(struct wiphy *wiphy, struct net_device *dev,
+			     u64 reqid)
+{
+	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
+	int rc;
+
+	rc = wmi_stop_sched_scan(wil);
+	/* device would return error if it thinks PNO is already stopped.
+	 * ignore the return code so user space and driver gets back in-sync
+	 */
+	wil_dbg_misc(wil, "sched scan stopped (%d)\n", rc);
+
+	return 0;
+}
+
 static const struct cfg80211_ops wil_cfg80211_ops = {
 	.add_virtual_intf = wil_cfg80211_add_iface,
 	.del_virtual_intf = wil_cfg80211_del_iface,
@@ -1777,6 +1846,8 @@ static const struct cfg80211_ops wil_cfg80211_ops = {
 	.set_power_mgmt = wil_cfg80211_set_power_mgmt,
 	.suspend = wil_cfg80211_suspend,
 	.resume = wil_cfg80211_resume,
+	.sched_scan_start = wil_cfg80211_sched_scan_start,
+	.sched_scan_stop = wil_cfg80211_sched_scan_stop,
 };
 
 static void wil_wiphy_init(struct wiphy *wiphy)
diff --git a/drivers/net/wireless/ath/wil6210/debugfs.c b/drivers/net/wireless/ath/wil6210/debugfs.c
index e58dc6dc1f9c..4a4888246e8c 100644
--- a/drivers/net/wireless/ath/wil6210/debugfs.c
+++ b/drivers/net/wireless/ath/wil6210/debugfs.c
@@ -242,12 +242,19 @@ static void wil_print_ring(struct seq_file *s, const char *prefix,
 static int wil_mbox_debugfs_show(struct seq_file *s, void *data)
 {
 	struct wil6210_priv *wil = s->private;
+	int ret;
+
+	ret = wil_pm_runtime_get(wil);
+	if (ret < 0)
+		return ret;
 
 	wil_print_ring(s, "tx", wil->csr + HOST_MBOX +
 		       offsetof(struct wil6210_mbox_ctl, tx));
 	wil_print_ring(s, "rx", wil->csr + HOST_MBOX +
 		       offsetof(struct wil6210_mbox_ctl, rx));
 
+	wil_pm_runtime_put(wil);
+
 	return 0;
 }
 
@@ -265,15 +272,37 @@ static const struct file_operations fops_mbox = {
 
 static int wil_debugfs_iomem_x32_set(void *data, u64 val)
 {
-	writel(val, (void __iomem *)data);
+	struct wil_debugfs_iomem_data *d = (struct
+					    wil_debugfs_iomem_data *)data;
+	struct wil6210_priv *wil = d->wil;
+	int ret;
+
+	ret = wil_pm_runtime_get(wil);
+	if (ret < 0)
+		return ret;
+
+	writel(val, (void __iomem *)d->offset);
 	wmb(); /* make sure write propagated to HW */
 
+	wil_pm_runtime_put(wil);
+
 	return 0;
 }
 
 static int wil_debugfs_iomem_x32_get(void *data, u64 *val)
 {
-	*val = readl((void __iomem *)data);
+	struct wil_debugfs_iomem_data *d = (struct
+					    wil_debugfs_iomem_data *)data;
+	struct wil6210_priv *wil = d->wil;
+	int ret;
+
+	ret = wil_pm_runtime_get(wil);
+	if (ret < 0)
+		return ret;
+
+	*val = readl((void __iomem *)d->offset);
+
+	wil_pm_runtime_put(wil);
 
 	return 0;
 }
@@ -284,10 +313,21 @@ DEFINE_SIMPLE_ATTRIBUTE(fops_iomem_x32, wil_debugfs_iomem_x32_get,
 static struct dentry *wil_debugfs_create_iomem_x32(const char *name,
 						   umode_t mode,
 						   struct dentry *parent,
-						   void *value)
+						   void *value,
+						   struct wil6210_priv *wil)
 {
-	return debugfs_create_file(name, mode, parent, value,
-				   &fops_iomem_x32);
+	struct dentry *file;
+	struct wil_debugfs_iomem_data *data = &wil->dbg_data.data_arr[
+					      wil->dbg_data.iomem_data_count];
+
+	data->wil = wil;
+	data->offset = value;
+
+	file = debugfs_create_file(name, mode, parent, data, &fops_iomem_x32);
+	if (!IS_ERR_OR_NULL(file))
+		wil->dbg_data.iomem_data_count++;
+
+	return file;
 }
 
 static int wil_debugfs_ulong_set(void *data, u64 val)
@@ -346,7 +386,8 @@ static void wil6210_debugfs_init_offset(struct wil6210_priv *wil,
 		case doff_io32:
 			f = wil_debugfs_create_iomem_x32(tbl[i].name,
 							 tbl[i].mode, dbg,
-							 base + tbl[i].off);
+							 base + tbl[i].off,
+							 wil);
 			break;
 		case doff_u8:
 			f = debugfs_create_u8(tbl[i].name, tbl[i].mode, dbg,
@@ -475,13 +516,22 @@ static int wil6210_debugfs_create_ITR_CNT(struct wil6210_priv *wil,
 static int wil_memread_debugfs_show(struct seq_file *s, void *data)
 {
 	struct wil6210_priv *wil = s->private;
-	void __iomem *a = wmi_buffer(wil, cpu_to_le32(mem_addr));
+	void __iomem *a;
+	int ret;
+
+	ret = wil_pm_runtime_get(wil);
+	if (ret < 0)
+		return ret;
+
+	a = wmi_buffer(wil, cpu_to_le32(mem_addr));
 
 	if (a)
 		seq_printf(s, "[0x%08x] = 0x%08x\n", mem_addr, readl(a));
 	else
 		seq_printf(s, "[0x%08x] = INVALID\n", mem_addr);
 
+	wil_pm_runtime_put(wil);
+
 	return 0;
 }
 
@@ -502,10 +552,12 @@ static ssize_t wil_read_file_ioblob(struct file *file, char __user *user_buf,
 {
 	enum { max_count = 4096 };
 	struct wil_blob_wrapper *wil_blob = file->private_data;
+	struct wil6210_priv *wil = wil_blob->wil;
 	loff_t pos = *ppos;
 	size_t available = wil_blob->blob.size;
 	void *buf;
 	size_t ret;
+	int rc;
 
 	if (test_bit(wil_status_suspending, wil_blob->wil->status) ||
 	    test_bit(wil_status_suspended, wil_blob->wil->status))
@@ -526,10 +578,19 @@ static ssize_t wil_read_file_ioblob(struct file *file, char __user *user_buf,
 	if (!buf)
 		return -ENOMEM;
 
+	rc = wil_pm_runtime_get(wil);
+	if (rc < 0) {
+		kfree(buf);
+		return rc;
+	}
+
 	wil_memcpy_fromio_32(buf, (const void __iomem *)
 			     wil_blob->blob.data + pos, count);
 
 	ret = copy_to_user(user_buf, buf, count);
+
+	wil_pm_runtime_put(wil);
+
 	kfree(buf);
 	if (ret == count)
 		return -EFAULT;
@@ -808,7 +869,6 @@ static ssize_t wil_write_file_txmgmt(struct file *file, const char __user *buf,
 
 	params.buf = frame;
 	params.len = len;
-	params.chan = wdev->preset_chandef.chan;
 
 	rc = wil_cfg80211_mgmt_tx(wiphy, wdev, &params, NULL);
 
@@ -1571,8 +1631,6 @@ static ssize_t wil_write_suspend_stats(struct file *file,
 	struct wil6210_priv *wil = file->private_data;
 
 	memset(&wil->suspend_stats, 0, sizeof(wil->suspend_stats));
-	wil->suspend_stats.min_suspend_time = ULONG_MAX;
-	wil->suspend_stats.collection_start = ktime_get();
 
 	return len;
 }
@@ -1582,33 +1640,41 @@ static ssize_t wil_read_suspend_stats(struct file *file,
 				      size_t count, loff_t *ppos)
 {
 	struct wil6210_priv *wil = file->private_data;
-	static char text[400];
-	int n;
-	unsigned long long stats_collection_time =
-		ktime_to_us(ktime_sub(ktime_get(),
-				      wil->suspend_stats.collection_start));
+	char *text;
+	int n, ret, text_size = 500;
 
-	n = snprintf(text, sizeof(text),
-		     "Suspend statistics:\n"
+	text = kmalloc(text_size, GFP_KERNEL);
+	if (!text)
+		return -ENOMEM;
+
+	n = snprintf(text, text_size,
+		     "Radio on suspend statistics:\n"
+		     "successful suspends:%ld failed suspends:%ld\n"
+		     "successful resumes:%ld failed resumes:%ld\n"
+		     "rejected by device:%ld\n"
+		     "Radio off suspend statistics:\n"
 		     "successful suspends:%ld failed suspends:%ld\n"
 		     "successful resumes:%ld failed resumes:%ld\n"
-		     "rejected by host:%ld rejected by device:%ld\n"
-		     "total suspend time:%lld min suspend time:%lld\n"
-		     "max suspend time:%lld stats collection time: %lld\n",
-		     wil->suspend_stats.successful_suspends,
-		     wil->suspend_stats.failed_suspends,
-		     wil->suspend_stats.successful_resumes,
-		     wil->suspend_stats.failed_resumes,
-		     wil->suspend_stats.rejected_by_host,
+		     "General statistics:\n"
+		     "rejected by host:%ld\n",
+		     wil->suspend_stats.r_on.successful_suspends,
+		     wil->suspend_stats.r_on.failed_suspends,
+		     wil->suspend_stats.r_on.successful_resumes,
+		     wil->suspend_stats.r_on.failed_resumes,
 		     wil->suspend_stats.rejected_by_device,
-		     wil->suspend_stats.total_suspend_time,
-		     wil->suspend_stats.min_suspend_time,
-		     wil->suspend_stats.max_suspend_time,
-		     stats_collection_time);
+		     wil->suspend_stats.r_off.successful_suspends,
+		     wil->suspend_stats.r_off.failed_suspends,
+		     wil->suspend_stats.r_off.successful_resumes,
+		     wil->suspend_stats.r_off.failed_resumes,
+		     wil->suspend_stats.rejected_by_host);
+
+	n = min_t(int, n, text_size);
 
-	n = min_t(int, n, sizeof(text));
+	ret = simple_read_from_buffer(user_buf, count, ppos, text, n);
 
-	return simple_read_from_buffer(user_buf, count, ppos, text, n);
+	kfree(text);
+
+	return ret;
 }
 
 static const struct file_operations fops_suspend_stats = {
@@ -1736,14 +1802,31 @@ static const struct dbg_off dbg_statics[] = {
 	{},
 };
 
+static const int dbg_off_count = 4 * (ARRAY_SIZE(isr_off) - 1) +
+				ARRAY_SIZE(dbg_wil_regs) - 1 +
+				ARRAY_SIZE(pseudo_isr_off) - 1 +
+				ARRAY_SIZE(lgc_itr_cnt_off) - 1 +
+				ARRAY_SIZE(tx_itr_cnt_off) - 1 +
+				ARRAY_SIZE(rx_itr_cnt_off) - 1;
+
 int wil6210_debugfs_init(struct wil6210_priv *wil)
 {
 	struct dentry *dbg = wil->debug = debugfs_create_dir(WIL_NAME,
 			wil_to_wiphy(wil)->debugfsdir);
-
 	if (IS_ERR_OR_NULL(dbg))
 		return -ENODEV;
 
+	wil->dbg_data.data_arr = kcalloc(dbg_off_count,
+					 sizeof(struct wil_debugfs_iomem_data),
+					 GFP_KERNEL);
+	if (!wil->dbg_data.data_arr) {
+		debugfs_remove_recursive(dbg);
+		wil->debug = NULL;
+		return -ENOMEM;
+	}
+
+	wil->dbg_data.iomem_data_count = 0;
+
 	wil_pmc_init(wil);
 
 	wil6210_debugfs_init_files(wil, dbg);
@@ -1758,8 +1841,6 @@ int wil6210_debugfs_init(struct wil6210_priv *wil)
 
 	wil6210_debugfs_create_ITR_CNT(wil, dbg);
 
-	wil->suspend_stats.collection_start = ktime_get();
-
 	return 0;
 }
 
@@ -1768,6 +1849,8 @@ void wil6210_debugfs_remove(struct wil6210_priv *wil)
 	debugfs_remove_recursive(wil->debug);
 	wil->debug = NULL;
 
+	kfree(wil->dbg_data.data_arr);
+
 	/* free pmc memory without sending command to fw, as it will
 	 * be reset on the way down anyway
 	 */
diff --git a/drivers/net/wireless/ath/wil6210/ethtool.c b/drivers/net/wireless/ath/wil6210/ethtool.c
index adcfef4dabf7..66200f616a37 100644
--- a/drivers/net/wireless/ath/wil6210/ethtool.c
+++ b/drivers/net/wireless/ath/wil6210/ethtool.c
@@ -47,9 +47,14 @@ static int wil_ethtoolops_get_coalesce(struct net_device *ndev,
 	struct wil6210_priv *wil = ndev_to_wil(ndev);
 	u32 tx_itr_en, tx_itr_val = 0;
 	u32 rx_itr_en, rx_itr_val = 0;
+	int ret;
 
 	wil_dbg_misc(wil, "ethtoolops_get_coalesce\n");
 
+	ret = wil_pm_runtime_get(wil);
+	if (ret < 0)
+		return ret;
+
 	tx_itr_en = wil_r(wil, RGF_DMA_ITR_TX_CNT_CTL);
 	if (tx_itr_en & BIT_DMA_ITR_TX_CNT_CTL_EN)
 		tx_itr_val = wil_r(wil, RGF_DMA_ITR_TX_CNT_TRSH);
@@ -58,6 +63,8 @@ static int wil_ethtoolops_get_coalesce(struct net_device *ndev,
 	if (rx_itr_en & BIT_DMA_ITR_RX_CNT_CTL_EN)
 		rx_itr_val = wil_r(wil, RGF_DMA_ITR_RX_CNT_TRSH);
 
+	wil_pm_runtime_put(wil);
+
 	cp->tx_coalesce_usecs = tx_itr_val;
 	cp->rx_coalesce_usecs = rx_itr_val;
 	return 0;
@@ -67,6 +74,7 @@ static int wil_ethtoolops_set_coalesce(struct net_device *ndev,
 				       struct ethtool_coalesce *cp)
 {
 	struct wil6210_priv *wil = ndev_to_wil(ndev);
+	int ret;
 
 	wil_dbg_misc(wil, "ethtoolops_set_coalesce: rx %d usec, tx %d usec\n",
 		     cp->rx_coalesce_usecs, cp->tx_coalesce_usecs);
@@ -86,8 +94,15 @@ static int wil_ethtoolops_set_coalesce(struct net_device *ndev,
 
 	wil->tx_max_burst_duration = cp->tx_coalesce_usecs;
 	wil->rx_max_burst_duration = cp->rx_coalesce_usecs;
+
+	ret = wil_pm_runtime_get(wil);
+	if (ret < 0)
+		return ret;
+
 	wil_configure_interrupt_moderation(wil);
 
+	wil_pm_runtime_put(wil);
+
 	return 0;
 
 out_bad:
diff --git a/drivers/net/wireless/ath/wil6210/fw.h b/drivers/net/wireless/ath/wil6210/fw.h
index 2f2b910501ba..2c7b24f61587 100644
--- a/drivers/net/wireless/ath/wil6210/fw.h
+++ b/drivers/net/wireless/ath/wil6210/fw.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2014,2016 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -58,15 +59,30 @@ struct wil_fw_record_comment { /* type == wil_fw_type_comment */
 	u8 data[0]; /* free-form data [data_size], see above */
 } __packed;
 
+/* Comment header - common for all comment record types */
+struct wil_fw_record_comment_hdr {
+	__le32 magic;
+};
+
 /* FW capabilities encoded inside a comment record */
 #define WIL_FW_CAPABILITIES_MAGIC (0xabcddcba)
 struct wil_fw_record_capabilities { /* type == wil_fw_type_comment */
 	/* identifies capabilities record */
-	__le32 magic;
+	struct wil_fw_record_comment_hdr hdr;
 	/* capabilities (variable size), see enum wmi_fw_capability */
 	u8 capabilities[0];
 };
 
+/* brd file info encoded inside a comment record */
+#define WIL_BRD_FILE_MAGIC (0xabcddcbb)
+struct wil_fw_record_brd_file { /* type == wil_fw_type_comment */
+	/* identifies brd file record */
+	struct wil_fw_record_comment_hdr hdr;
+	__le32 version;
+	__le32 base_addr;
+	__le32 max_size_bytes;
+} __packed;
+
 /* perform action
  * data_size = @head.size - offsetof(struct wil_fw_record_action, data)
  */
diff --git a/drivers/net/wireless/ath/wil6210/fw_inc.c b/drivers/net/wireless/ath/wil6210/fw_inc.c
index e01acac88825..914c0106e94b 100644
--- a/drivers/net/wireless/ath/wil6210/fw_inc.c
+++ b/drivers/net/wireless/ath/wil6210/fw_inc.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2014-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -26,14 +27,17 @@
 					     prefix_type, rowsize,	\
 					     groupsize, buf, len, ascii)
 
-#define FW_ADDR_CHECK(ioaddr, val, msg) do { \
-		ioaddr = wmi_buffer(wil, val); \
-		if (!ioaddr) { \
-			wil_err_fw(wil, "bad " msg ": 0x%08x\n", \
-				   le32_to_cpu(val)); \
-			return -EINVAL; \
-		} \
-	} while (0)
+static bool wil_fw_addr_check(struct wil6210_priv *wil,
+			      void __iomem **ioaddr, __le32 val,
+			      u32 size, const char *msg)
+{
+	*ioaddr = wmi_buffer_block(wil, val, size);
+	if (!(*ioaddr)) {
+		wil_err_fw(wil, "bad %s: 0x%08x\n", msg, le32_to_cpu(val));
+		return false;
+	}
+	return true;
+}
 
 /**
  * wil_fw_verify - verify firmware file validity
@@ -124,14 +128,6 @@ static int fw_ignore_section(struct wil6210_priv *wil, const void *data,
 	return 0;
 }
 
-static int fw_handle_comment(struct wil6210_priv *wil, const void *data,
-			     size_t size)
-{
-	wil_hex_dump_fw("", DUMP_PREFIX_OFFSET, 16, 1, data, size, true);
-
-	return 0;
-}
-
 static int
 fw_handle_capabilities(struct wil6210_priv *wil, const void *data,
 		       size_t size)
@@ -139,9 +135,11 @@ fw_handle_capabilities(struct wil6210_priv *wil, const void *data,
 	const struct wil_fw_record_capabilities *rec = data;
 	size_t capa_size;
 
-	if (size < sizeof(*rec) ||
-	    le32_to_cpu(rec->magic) != WIL_FW_CAPABILITIES_MAGIC)
+	if (size < sizeof(*rec)) {
+		wil_hex_dump_fw("", DUMP_PREFIX_OFFSET, 16, 1,
+				data, size, true);
 		return 0;
+	}
 
 	capa_size = size - offsetof(struct wil_fw_record_capabilities,
 				    capabilities);
@@ -153,8 +151,56 @@ fw_handle_capabilities(struct wil6210_priv *wil, const void *data,
 	return 0;
 }
 
-static int fw_handle_data(struct wil6210_priv *wil, const void *data,
-			  size_t size)
+static int
+fw_handle_brd_file(struct wil6210_priv *wil, const void *data,
+		   size_t size)
+{
+	const struct wil_fw_record_brd_file *rec = data;
+
+	if (size < sizeof(*rec)) {
+		wil_hex_dump_fw("", DUMP_PREFIX_OFFSET, 16, 1,
+				data, size, true);
+		return 0;
+	}
+
+	wil->brd_file_addr = le32_to_cpu(rec->base_addr);
+	wil->brd_file_max_size = le32_to_cpu(rec->max_size_bytes);
+
+	wil_dbg_fw(wil, "brd_file_addr 0x%x, brd_file_max_size %d\n",
+		   wil->brd_file_addr, wil->brd_file_max_size);
+
+	return 0;
+}
+
+static int
+fw_handle_comment(struct wil6210_priv *wil, const void *data,
+		  size_t size)
+{
+	const struct wil_fw_record_comment_hdr *hdr = data;
+	u32 magic;
+	int rc = 0;
+
+	if (size < sizeof(*hdr))
+		return 0;
+
+	magic = le32_to_cpu(hdr->magic);
+
+	switch (magic) {
+	case WIL_FW_CAPABILITIES_MAGIC:
+		wil_dbg_fw(wil, "magic is WIL_FW_CAPABILITIES_MAGIC\n");
+		rc = fw_handle_capabilities(wil, data, size);
+		break;
+	case WIL_BRD_FILE_MAGIC:
+		wil_dbg_fw(wil, "magic is WIL_BRD_FILE_MAGIC\n");
+		rc = fw_handle_brd_file(wil, data, size);
+		break;
+	}
+
+	return rc;
+}
+
+static int __fw_handle_data(struct wil6210_priv *wil, const void *data,
+			    size_t size, __le32 addr)
 {
 	const struct wil_fw_record_data *d = data;
 	void __iomem *dst;
@@ -165,15 +211,23 @@ static int fw_handle_data(struct wil6210_priv *wil, const void *data,
 		return -EINVAL;
 	}
 
-	FW_ADDR_CHECK(dst, d->addr, "address");
-	wil_dbg_fw(wil, "write [0x%08x] <== %zu bytes\n", le32_to_cpu(d->addr),
-		   s);
+	if (!wil_fw_addr_check(wil, &dst, addr, s, "address"))
+		return -EINVAL;
+	wil_dbg_fw(wil, "write [0x%08x] <== %zu bytes\n", le32_to_cpu(addr), s);
 	wil_memcpy_toio_32(dst, d->data, s);
 	wmb(); /* finish before processing next record */
 
 	return 0;
 }
 
+static int fw_handle_data(struct wil6210_priv *wil, const void *data,
+			  size_t size)
+{
+	const struct wil_fw_record_data *d = data;
+
+	return __fw_handle_data(wil, data, size, d->addr);
+}
+
 static int fw_handle_fill(struct wil6210_priv *wil, const void *data,
 			  size_t size)
 {
@@ -197,7 +251,8 @@ static int fw_handle_fill(struct wil6210_priv *wil, const void *data,
 		return -EINVAL;
 	}
 
-	FW_ADDR_CHECK(dst, d->addr, "address");
+	if (!wil_fw_addr_check(wil, &dst, d->addr, s, "address"))
+		return -EINVAL;
 
 	v = le32_to_cpu(d->value);
 	wil_dbg_fw(wil, "fill [0x%08x] <== 0x%08x, %zu bytes\n",
@@ -253,7 +308,8 @@ static int fw_handle_direct_write(struct wil6210_priv *wil, const void *data,
 		u32 v = le32_to_cpu(block[i].value);
 		u32 x, y;
 
-		FW_ADDR_CHECK(dst, block[i].addr, "address");
+		if (!wil_fw_addr_check(wil, &dst, block[i].addr, 0, "address"))
+			return -EINVAL;
 
 		x = readl(dst);
 		y = (x & m) | (v & ~m);
@@ -319,10 +375,15 @@ static int fw_handle_gateway_data(struct wil6210_priv *wil, const void *data,
 	wil_dbg_fw(wil, "gw write record [%3d] blocks, cmd 0x%08x\n",
 		   n, gw_cmd);
 
-	FW_ADDR_CHECK(gwa_addr, d->gateway_addr_addr, "gateway_addr_addr");
-	FW_ADDR_CHECK(gwa_val, d->gateway_value_addr, "gateway_value_addr");
-	FW_ADDR_CHECK(gwa_cmd, d->gateway_cmd_addr, "gateway_cmd_addr");
-	FW_ADDR_CHECK(gwa_ctl, d->gateway_ctrl_address, "gateway_ctrl_address");
+	if (!wil_fw_addr_check(wil, &gwa_addr, d->gateway_addr_addr, 0,
+			       "gateway_addr_addr") ||
+	    !wil_fw_addr_check(wil, &gwa_val, d->gateway_value_addr, 0,
+			       "gateway_value_addr") ||
+	    !wil_fw_addr_check(wil, &gwa_cmd, d->gateway_cmd_addr, 0,
+			       "gateway_cmd_addr") ||
+	    !wil_fw_addr_check(wil, &gwa_ctl, d->gateway_ctrl_address, 0,
+			       "gateway_ctrl_address"))
+		return -EINVAL;
 
 	wil_dbg_fw(wil, "gw addresses: addr 0x%08x val 0x%08x"
 		   " cmd 0x%08x ctl 0x%08x\n",
@@ -378,12 +439,19 @@ static int fw_handle_gateway_data4(struct wil6210_priv *wil, const void *data,
 	wil_dbg_fw(wil, "gw4 write record [%3d] blocks, cmd 0x%08x\n",
 		   n, gw_cmd);
 
-	FW_ADDR_CHECK(gwa_addr, d->gateway_addr_addr, "gateway_addr_addr");
+	if (!wil_fw_addr_check(wil, &gwa_addr, d->gateway_addr_addr, 0,
+			       "gateway_addr_addr"))
+		return -EINVAL;
 	for (k = 0; k < ARRAY_SIZE(block->value); k++)
-		FW_ADDR_CHECK(gwa_val[k], d->gateway_value_addr[k],
-			      "gateway_value_addr");
-	FW_ADDR_CHECK(gwa_cmd, d->gateway_cmd_addr, "gateway_cmd_addr");
-	FW_ADDR_CHECK(gwa_ctl, d->gateway_ctrl_address, "gateway_ctrl_address");
+		if (!wil_fw_addr_check(wil, &gwa_val[k],
+				       d->gateway_value_addr[k],
+				       0, "gateway_value_addr"))
+			return -EINVAL;
+	if (!wil_fw_addr_check(wil, &gwa_cmd, d->gateway_cmd_addr, 0,
+			       "gateway_cmd_addr") ||
+	    !wil_fw_addr_check(wil, &gwa_ctl, d->gateway_ctrl_address, 0,
+			       "gateway_ctrl_address"))
+		return -EINVAL;
 
 	wil_dbg_fw(wil, "gw4 addresses: addr 0x%08x cmd 0x%08x ctl 0x%08x\n",
 		   le32_to_cpu(d->gateway_addr_addr),
@@ -422,7 +490,7 @@ static const struct {
 	int (*parse_handler)(struct wil6210_priv *wil, const void *data,
 			     size_t size);
 } wil_fw_handlers[] = {
-	{wil_fw_type_comment, fw_handle_comment, fw_handle_capabilities},
+	{wil_fw_type_comment, fw_handle_comment, fw_handle_comment},
 	{wil_fw_type_data, fw_handle_data, fw_ignore_section},
 	{wil_fw_type_fill, fw_handle_fill, fw_ignore_section},
 	/* wil_fw_type_action */
@@ -517,7 +585,7 @@ int wil_request_firmware(struct wil6210_priv *wil, const char *name,
 
 	rc = request_firmware(&fw, name, wil_to_dev(wil));
 	if (rc) {
-		wil_err_fw(wil, "Failed to load firmware %s\n", name);
+		wil_err_fw(wil, "Failed to load firmware %s rc %d\n", name, rc);
 		return rc;
 	}
 	wil_dbg_fw(wil, "Loading <%s>, %zu bytes\n", name, fw->size);
@@ -538,6 +606,100 @@ int wil_request_firmware(struct wil6210_priv *wil, const char *name,
 	return rc;
 }
 
+/**
+ * wil_brd_process - process section from BRD file
+ *
+ * Return error code
+ */
+static int wil_brd_process(struct wil6210_priv *wil, const void *data,
+			   size_t size)
+{
+	int rc = 0;
+	const struct wil_fw_record_head *hdr = data;
+	size_t s, hdr_sz;
+	u16 type;
+
+	/* Assuming the board file includes only one header record and one data
+	 * record. Each record starts with wil_fw_record_head.
+	 */
+	if (size < sizeof(*hdr))
+		return -EINVAL;
+	s = sizeof(*hdr) + le32_to_cpu(hdr->size);
+	if (s > size)
+		return -EINVAL;
+
+	/* Skip the header record and handle the data record */
+	hdr = (const void *)hdr + s;
+	size -= s;
+	if (size < sizeof(*hdr))
+		return -EINVAL;
+	hdr_sz = le32_to_cpu(hdr->size);
+
+	if (wil->brd_file_max_size && hdr_sz > wil->brd_file_max_size)
+		return -EINVAL;
+	if (sizeof(*hdr) + hdr_sz > size)
+		return -EINVAL;
+	if (hdr_sz % 4) {
+		wil_err_fw(wil, "unaligned record size: %zu\n",
+			   hdr_sz);
+		return -EINVAL;
+	}
+	type = le16_to_cpu(hdr->type);
+	if (type != wil_fw_type_data) {
+		wil_err_fw(wil, "invalid record type for board file: %d\n",
+			   type);
+		return -EINVAL;
+	}
+	if (hdr_sz < sizeof(struct wil_fw_record_data)) {
+		wil_err_fw(wil, "data record too short: %zu\n", hdr_sz);
+		return -EINVAL;
+	}
+
+	wil_dbg_fw(wil, "using addr from fw file: [0x%08x]\n",
+		   wil->brd_file_addr);
+
+	rc = __fw_handle_data(wil, &hdr[1], hdr_sz,
+			      cpu_to_le32(wil->brd_file_addr));
+
+	return rc;
+}
+
+/**
+ * wil_request_board - Request board file
+ *
+ * Request board image from the file
+ * board file address and max size are read from FW file
+ * during initialization.
+ * brd file shall include one header and one data section.
+ *
+ * Return error code
+ */
+int wil_request_board(struct wil6210_priv *wil, const char *name)
+{
+	int rc, dlen;
+	const struct firmware *brd;
+
+	rc = request_firmware(&brd, name, wil_to_dev(wil));
+	if (rc) {
+		wil_err_fw(wil, "Failed to load brd %s\n", name);
+		return rc;
+	}
+	wil_dbg_fw(wil, "Loading <%s>, %zu bytes\n", name, brd->size);
+
+	/* Verify the header */
+	dlen = wil_fw_verify(wil, brd->data, brd->size);
+	if (dlen < 0) {
+		rc = dlen;
+		goto out;
+	}
+	/* Process the data record */
+	rc = wil_brd_process(wil, brd->data, dlen);
+
+out:
+	release_firmware(brd);
+	return rc;
+}
+
 /**
  * wil_fw_verify_file_exists - checks if firmware file exist
  *
diff --git a/drivers/net/wireless/ath/wil6210/interrupt.c b/drivers/net/wireless/ath/wil6210/interrupt.c
index 59def4f3fcf3..1835187ea075 100644
--- a/drivers/net/wireless/ath/wil6210/interrupt.c
+++ b/drivers/net/wireless/ath/wil6210/interrupt.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -358,6 +359,25 @@ static void wil_cache_mbox_regs(struct wil6210_priv *wil)
 	wil_mbox_ring_le2cpus(&wil->mbox_ctl.tx);
 }
 
+static bool wil_validate_mbox_regs(struct wil6210_priv *wil)
+{
+	size_t min_size = sizeof(struct wil6210_mbox_hdr) +
+		sizeof(struct wmi_cmd_hdr);
+
+	if (wil->mbox_ctl.rx.entry_size < min_size) {
+		wil_err(wil, "rx mbox entry too small (%d)\n",
+			wil->mbox_ctl.rx.entry_size);
+		return false;
+	}
+	if (wil->mbox_ctl.tx.entry_size < min_size) {
+		wil_err(wil, "tx mbox entry too small (%d)\n",
+			wil->mbox_ctl.tx.entry_size);
+		return false;
+	}
+
+	return true;
+}
+
 static irqreturn_t wil6210_irq_misc(int irq, void *cookie)
 {
 	struct wil6210_priv *wil = cookie;
@@ -376,8 +396,9 @@ static irqreturn_t wil6210_irq_misc(int irq, void *cookie)
 	wil6210_mask_irq_misc(wil, false);
 
 	if (isr & ISR_MISC_FW_ERROR) {
-		u32 fw_assert_code = wil_r(wil, RGF_FW_ASSERT_CODE);
-		u32 ucode_assert_code = wil_r(wil, RGF_UCODE_ASSERT_CODE);
+		u32 fw_assert_code = wil_r(wil, wil->rgf_fw_assert_code_addr);
+		u32 ucode_assert_code =
+			wil_r(wil, wil->rgf_ucode_assert_code_addr);
 
 		wil_err(wil,
 			"Firmware error detected, assert codes FW 0x%08x, UCODE 0x%08x\n",
@@ -393,7 +414,8 @@ static irqreturn_t wil6210_irq_misc(int irq, void *cookie)
 	if (isr & ISR_MISC_FW_READY) {
 		wil_dbg_irq(wil, "IRQ: FW ready\n");
 		wil_cache_mbox_regs(wil);
-		set_bit(wil_status_mbox_ready, wil->status);
+		if (wil_validate_mbox_regs(wil))
+			set_bit(wil_status_mbox_ready, wil->status);
 		/**
 		 * Actual FW ready indicated by the
 		 * WMI_FW_READY_EVENTID
@@ -545,7 +567,7 @@ static irqreturn_t wil6210_hardirq(int irq, void *cookie)
 	if (unlikely((pseudo_cause == 0) || ((pseudo_cause & 0xff) == 0xff)))
 		return IRQ_NONE;
 
-	/* FIXME: IRQ mask debug */
+	/* IRQ mask debug */
 	if (unlikely(wil6210_debug_irq_mask(wil, pseudo_cause)))
 		return IRQ_NONE;
 
diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c
index 885924abf61c..0c61a6c13991 100644
--- a/drivers/net/wireless/ath/wil6210/main.c
+++ b/drivers/net/wireless/ath/wil6210/main.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -579,7 +580,6 @@ int wil_priv_init(struct wil6210_priv *wil)
 	wil->wakeup_trigger = WMI_WAKEUP_TRIGGER_UCAST |
 			      WMI_WAKEUP_TRIGGER_BCAST;
 	memset(&wil->suspend_stats, 0, sizeof(wil->suspend_stats));
-	wil->suspend_stats.min_suspend_time = ULONG_MAX;
 	wil->vring_idle_trsh = 16;
 
 	return 0;
@@ -638,6 +638,98 @@ void wil_priv_deinit(struct wil6210_priv *wil)
 	destroy_workqueue(wil->wmi_wq);
 }
 
+static void wil_shutdown_bl(struct wil6210_priv *wil)
+{
+	u32 val;
+
+	wil_s(wil, RGF_USER_BL +
+	      offsetof(struct bl_dedicated_registers_v1,
+		       bl_shutdown_handshake), BL_SHUTDOWN_HS_GRTD);
+
+	usleep_range(100, 150);
+
+	val = wil_r(wil, RGF_USER_BL +
+		    offsetof(struct bl_dedicated_registers_v1,
+			     bl_shutdown_handshake));
+	if (val & BL_SHUTDOWN_HS_RTD) {
+		wil_dbg_misc(wil, "BL is ready for halt\n");
+		return;
+	}
+
+	wil_err(wil, "BL did not report ready for halt\n");
+}
+
+/* this format is used by ARC embedded CPU for instruction memory */
+static inline u32 ARC_me_imm32(u32 d)
+{
+	return ((d & 0xffff0000) >> 16) | ((d & 0x0000ffff) << 16);
+}
+
+/* defines access to interrupt vectors for wil_freeze_bl */
+#define ARC_IRQ_VECTOR_OFFSET(N)	((N) * 8)
+/* ARC long jump instruction */
+#define ARC_JAL_INST			(0x20200f80)
+
+static void wil_freeze_bl(struct wil6210_priv *wil)
+{
+	u32 jal, upc, saved;
+	u32 ivt3 = ARC_IRQ_VECTOR_OFFSET(3);
+
+	jal = wil_r(wil, wil->iccm_base + ivt3);
+	if (jal != ARC_me_imm32(ARC_JAL_INST)) {
+		wil_dbg_misc(wil, "invalid IVT entry found, skipping\n");
+		return;
+	}
+
+	/* prevent the target from entering deep sleep
+	 * and disabling memory access
+	 */
+	saved = wil_r(wil, RGF_USER_USAGE_8);
+	wil_w(wil, RGF_USER_USAGE_8, saved | BIT_USER_PREVENT_DEEP_SLEEP);
+	usleep_range(20, 25); /* let the BL process the bit */
+
+	/* redirect to endless loop in the INT_L1 context and let it trap */
+	wil_w(wil, wil->iccm_base + ivt3 + 4, ARC_me_imm32(ivt3));
+	usleep_range(20, 25); /* let the BL get into the trap */
+
+	/* verify the BL is frozen */
+	upc = wil_r(wil, RGF_USER_CPU_PC);
+	if (upc < ivt3 || (upc > (ivt3 + 8)))
+		wil_dbg_misc(wil, "BL freeze failed, PC=0x%08X\n", upc);
+
+	wil_w(wil, RGF_USER_USAGE_8, saved);
+}
+
+static void wil_bl_prepare_halt(struct wil6210_priv *wil)
+{
+	u32 tmp, ver;
+
+	/* before halting device CPU driver must make sure BL is not accessing
+	 * host memory. This is done differently depending on BL version:
+	 * 1. For very old BL versions the procedure is skipped
+	 * (not supported).
+	 * 2. For old BL version we use a special trick to freeze the BL
+	 * 3. For new BL versions we shutdown the BL using handshake procedure.
+	 */
+	tmp = wil_r(wil, RGF_USER_BL +
+		    offsetof(struct bl_dedicated_registers_v0,
+			     boot_loader_struct_version));
+	if (!tmp) {
+		wil_dbg_misc(wil, "old BL, skipping halt preperation\n");
+		return;
+	}
+
+	tmp = wil_r(wil, RGF_USER_BL +
+		    offsetof(struct bl_dedicated_registers_v1,
+			     bl_shutdown_handshake));
+	ver = BL_SHUTDOWN_HS_PROT_VER(tmp);
+
+	if (ver > 0)
+		wil_shutdown_bl(wil);
+	else
+		wil_freeze_bl(wil);
+}
+
 static inline void wil_halt_cpu(struct wil6210_priv *wil)
 {
 	wil_w(wil, RGF_USER_USER_CPU_0, BIT_USER_USER_CPU_MAN_RST);
@@ -671,7 +763,7 @@ static void wil_set_oob_mode(struct wil6210_priv *wil, u8 mode)
 	}
 }
 
-static int wil_target_reset(struct wil6210_priv *wil)
+static int wil_target_reset(struct wil6210_priv *wil, int no_flash)
 {
 	int delay = 0;
 	u32 x, x1 = 0;
@@ -685,9 +777,16 @@ static int wil_target_reset(struct wil6210_priv *wil)
 
 	wil_halt_cpu(wil);
 
-	/* clear all boot loader "ready" bits */
-	wil_w(wil, RGF_USER_BL +
-	      offsetof(struct bl_dedicated_registers_v0, boot_loader_ready), 0);
+	if (!no_flash) {
+		/* clear all boot loader "ready" bits */
+		wil_w(wil, RGF_USER_BL +
+		      offsetof(struct bl_dedicated_registers_v0,
+			       boot_loader_ready), 0);
+		/* this should be safe to write even with old BLs */
+		wil_w(wil, RGF_USER_BL +
+		      offsetof(struct bl_dedicated_registers_v1,
+			       bl_shutdown_handshake), 0);
+	}
 	/* Clear Fw Download notification */
 	wil_c(wil, RGF_USER_USAGE_6, BIT(0));
 
@@ -728,21 +827,33 @@ static int wil_target_reset(struct wil6210_priv *wil)
 	wil_w(wil, RGF_USER_CLKS_CTL_SW_RST_VEC_0, 0);
 
 	/* wait until device ready. typical time is 20..80 msec */
-	do {
-		msleep(RST_DELAY);
-		x = wil_r(wil, RGF_USER_BL +
-			  offsetof(struct bl_dedicated_registers_v0,
-				   boot_loader_ready));
-		if (x1 != x) {
-			wil_dbg_misc(wil, "BL.ready 0x%08x => 0x%08x\n", x1, x);
-			x1 = x;
-		}
-		if (delay++ > RST_COUNT) {
-			wil_err(wil, "Reset not completed, bl.ready 0x%08x\n",
-				x);
-			return -ETIME;
-		}
-	} while (x != BL_READY);
+	if (no_flash)
+		do {
+			msleep(RST_DELAY);
+			x = wil_r(wil, USER_EXT_USER_PMU_3);
+			if (delay++ > RST_COUNT) {
+				wil_err(wil, "Reset not completed, PMU_3 0x%08x\n",
+					x);
+				return -ETIME;
+			}
+		} while ((x & BIT_PMU_DEVICE_RDY) == 0);
+	else
+		do {
+			msleep(RST_DELAY);
+			x = wil_r(wil, RGF_USER_BL +
+				  offsetof(struct bl_dedicated_registers_v0,
+					   boot_loader_ready));
+			if (x1 != x) {
+				wil_dbg_misc(wil, "BL.ready 0x%08x => 0x%08x\n",
+					     x1, x);
+				x1 = x;
+			}
+			if (delay++ > RST_COUNT) {
+				wil_err(wil, "Reset not completed, bl.ready 0x%08x\n",
+					x);
+				return -ETIME;
+			}
+		} while (x != BL_READY);
 
 	wil_c(wil, RGF_USER_CLKS_CTL_0, BIT_USER_CLKS_RST_PWGD);
 
@@ -750,6 +861,21 @@ static int wil_target_reset(struct wil6210_priv *wil)
 	wil_s(wil, RGF_DMA_OFUL_NID_0, BIT_DMA_OFUL_NID_0_RX_EXT_TR_EN |
 	      BIT_DMA_OFUL_NID_0_RX_EXT_A3_SRC);
 
+	if (no_flash) {
+		/* Reset OTP HW vectors to fit 40MHz */
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME1, 0x60001);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME2, 0x20027);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME3, 0x1);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME4, 0x20027);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME5, 0x30003);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME6, 0x20002);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME7, 0x60001);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME8, 0x60001);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME9, 0x60001);
+		wil_w(wil, RGF_USER_XPM_IFC_RD_TIME10, 0x60001);
+		wil_w(wil, RGF_USER_XPM_RD_DOUT_SAMPLE_TIME, 0x57);
+	}
+
 	wil_dbg_misc(wil, "Reset completed in %d ms\n", delay * RST_DELAY);
 	return 0;
 }
@@ -760,6 +886,8 @@ static void wil_collect_fw_info(struct wil6210_priv *wil)
 	u8 retry_short;
 	int rc;
 
+	wil_refresh_fw_capabilities(wil);
+
 	rc = wmi_get_mgmt_retry(wil, &retry_short);
 	if (!rc) {
 		wiphy->retry_short = retry_short;
@@ -767,6 +895,43 @@ static void wil_collect_fw_info(struct wil6210_priv *wil)
 	}
 }
 
+void wil_refresh_fw_capabilities(struct wil6210_priv *wil)
+{
+	struct wiphy *wiphy = wil_to_wiphy(wil);
+	int features;
+
+	wil->keep_radio_on_during_sleep =
+		test_bit(WIL_PLATFORM_CAPA_RADIO_ON_IN_SUSPEND,
+			 wil->platform_capa) &&
+		test_bit(WMI_FW_CAPABILITY_D3_SUSPEND, wil->fw_capabilities);
+
+	wil_info(wil, "keep_radio_on_during_sleep (%d)\n",
+		 wil->keep_radio_on_during_sleep);
+
+	if (test_bit(WMI_FW_CAPABILITY_RSSI_REPORTING, wil->fw_capabilities))
+		wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM;
+	else
+		wiphy->signal_type = CFG80211_SIGNAL_TYPE_UNSPEC;
+
+	if (test_bit(WMI_FW_CAPABILITY_PNO, wil->fw_capabilities)) {
+		wiphy->max_sched_scan_reqs = 1;
+		wiphy->max_sched_scan_ssids = WMI_MAX_PNO_SSID_NUM;
+		wiphy->max_match_sets = WMI_MAX_PNO_SSID_NUM;
+		wiphy->max_sched_scan_ie_len = WMI_MAX_IE_LEN;
+		wiphy->max_sched_scan_plans = WMI_MAX_PLANS_NUM;
+	}
+
+	if (wil->platform_ops.set_features) {
+		features = (test_bit(WMI_FW_CAPABILITY_REF_CLOCK_CONTROL,
+				     wil->fw_capabilities) &&
+			    test_bit(WIL_PLATFORM_CAPA_EXT_CLK,
+				     wil->platform_capa)) ?
+			BIT(WIL_PLATFORM_FEATURE_FW_EXT_CLK_CONTROL) : 0;
+
+		wil->platform_ops.set_features(wil->platform_handle, features);
+	}
+}
+
 void wil_mbox_ring_le2cpus(struct wil6210_mbox_ring *r)
 {
 	le32_to_cpus(&r->base);
@@ -868,6 +1033,27 @@ static void wil_bl_crash_info(struct wil6210_priv *wil, bool is_err)
 	}
 }
 
+static int wil_get_otp_info(struct wil6210_priv *wil)
+{
+	struct net_device *ndev = wil_to_ndev(wil);
+	struct wiphy *wiphy = wil_to_wiphy(wil);
+	u8 mac[8];
+
+	wil_memcpy_fromio_32(mac, wil->csr + HOSTADDR(RGF_OTP_MAC),
+			     sizeof(mac));
+	if (!is_valid_ether_addr(mac)) {
+		wil_err(wil, "Invalid MAC %pM\n", mac);
+		return -EINVAL;
+	}
+
+	ether_addr_copy(ndev->perm_addr, mac);
+	ether_addr_copy(wiphy->perm_addr, mac);
+	if (!is_valid_ether_addr(ndev->dev_addr))
+		ether_addr_copy(ndev->dev_addr, mac);
+
+	return 0;
+}
+
 static int wil_wait_for_fw_ready(struct wil6210_priv *wil)
 {
 	ulong to = msecs_to_jiffies(1000);
@@ -960,6 +1146,8 @@ static void wil_pre_fw_config(struct wil6210_priv *wil)
 int wil_reset(struct wil6210_priv *wil, bool load_fw)
 {
 	int rc;
+	unsigned long status_flags = BIT(wil_status_resetting);
+	int no_flash;
 
 	wil_dbg_misc(wil, "reset\n");
 
@@ -980,6 +1168,16 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 	if (wil->hw_version == HW_VER_UNKNOWN)
 		return -ENODEV;
 
+	if (test_bit(WIL_PLATFORM_CAPA_T_PWR_ON_0, wil->platform_capa)) {
+		wil_dbg_misc(wil, "Notify FW to set T_POWER_ON=0\n");
+		wil_s(wil, RGF_USER_USAGE_8, BIT_USER_SUPPORT_T_POWER_ON_0);
+	}
+
+	if (test_bit(WIL_PLATFORM_CAPA_EXT_CLK, wil->platform_capa)) {
+		wil_dbg_misc(wil, "Notify FW on ext clock configuration\n");
+		wil_s(wil, RGF_USER_USAGE_8, BIT_USER_EXT_CLK);
+	}
+
 	if (wil->platform_ops.notify) {
 		rc = wil->platform_ops.notify(wil->platform_handle,
 					      WIL_PLATFORM_EVT_PRE_RESET);
@@ -989,6 +1187,14 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 	}
 
 	set_bit(wil_status_resetting, wil->status);
+	if (test_bit(wil_status_collecting_dumps, wil->status)) {
+		/* Device collects crash dump, cancel the reset.
+		 * following crash dump collection, reset would take place.
+		 */
+		wil_dbg_misc(wil, "reject reset while collecting crash dump\n");
+		rc = -EBUSY;
+		goto out;
+	}
 
 	cancel_work_sync(&wil->disconnect_worker);
 	wil6210_disconnect(wil, NULL, WLAN_REASON_DEAUTH_LEAVING, false);
@@ -1003,7 +1209,11 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 
 	/* prevent NAPI from being scheduled and prevent wmi commands */
 	mutex_lock(&wil->wmi_mutex);
-	bitmap_zero(wil->status, wil_status_last);
+	if (test_bit(wil_status_suspending, wil->status))
+		status_flags |= BIT(wil_status_suspending);
+	bitmap_and(wil->status, wil->status, &status_flags,
+		   wil_status_last);
+	wil_dbg_misc(wil, "wil->status (0x%lx)\n", *wil->status);
 	mutex_unlock(&wil->wmi_mutex);
 
 	wil_mask_irq(wil);
@@ -1013,37 +1223,53 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 	flush_workqueue(wil->wq_service);
 	flush_workqueue(wil->wmi_wq);
 
-	wil_bl_crash_info(wil, false);
+	no_flash = test_bit(hw_capa_no_flash, wil->hw_capa);
+	if (!no_flash)
+		wil_bl_crash_info(wil, false);
 	wil_disable_irq(wil);
-	rc = wil_target_reset(wil);
+	rc = wil_target_reset(wil, no_flash);
 	wil6210_clear_irq(wil);
 	wil_enable_irq(wil);
 	wil_rx_fini(wil);
 	if (rc) {
-		wil_bl_crash_info(wil, true);
-		return rc;
+		if (!no_flash)
+			wil_bl_crash_info(wil, true);
+		goto out;
 	}
 
-	rc = wil_get_bl_info(wil);
-	if (rc == -EAGAIN && !load_fw) /* ignore RF error if not going up */
-		rc = 0;
+	if (no_flash) {
+		rc = wil_get_otp_info(wil);
+	} else {
+		rc = wil_get_bl_info(wil);
+		if (rc == -EAGAIN && !load_fw)
+			/* ignore RF error if not going up */
+			rc = 0;
+	}
 	if (rc)
-		return rc;
+		goto out;
 
 	wil_set_oob_mode(wil, oob_mode);
 	if (load_fw) {
 		wil_info(wil, "Use firmware <%s> + board <%s>\n",
 			 wil->wil_fw_name, WIL_BOARD_FILE_NAME);
 
+		if (!no_flash)
+			wil_bl_prepare_halt(wil);
+
 		wil_halt_cpu(wil);
 		memset(wil->fw_version, 0, sizeof(wil->fw_version));
 		/* Loading f/w from the file */
 		rc = wil_request_firmware(wil, wil->wil_fw_name, true);
 		if (rc)
-			return rc;
-		rc = wil_request_firmware(wil, WIL_BOARD_FILE_NAME, true);
+			goto out;
+		if (wil->brd_file_addr)
+			rc = wil_request_board(wil, WIL_BOARD_FILE_NAME);
+		else
+			rc = wil_request_firmware(wil,
+						  WIL_BOARD_FILE_NAME,
+						  true);
 		if (rc)
-			return rc;
+			goto out;
 
 		wil_pre_fw_config(wil);
 		wil_release_cpu(wil);
@@ -1055,6 +1281,8 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 	reinit_completion(&wil->wmi_call);
 	reinit_completion(&wil->halp.comp);
 
+	clear_bit(wil_status_resetting, wil->status);
+
 	if (load_fw) {
 		wil_configure_interrupt_moderation(wil);
 		wil_unmask_irq(wil);
@@ -1071,11 +1299,11 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 			return rc;
 		}
 
+		wil_collect_fw_info(wil);
+
 		if (wil->ps_profile != WMI_PS_PROFILE_TYPE_DEFAULT)
 			wil_ps_update(wil, wil->ps_profile);
 
-		wil_collect_fw_info(wil);
-
 		if (wil->platform_ops.notify) {
 			rc = wil->platform_ops.notify(wil->platform_handle,
 						      WIL_PLATFORM_EVT_FW_RDY);
@@ -1088,6 +1316,10 @@ int wil_reset(struct wil6210_priv *wil, bool load_fw)
 	}
 
 	return rc;
+
+out:
+	clear_bit(wil_status_resetting, wil->status);
+	return rc;
 }
 
 void wil_fw_error_recovery(struct wil6210_priv *wil)
@@ -1193,9 +1425,7 @@ int __wil_down(struct wil6210_priv *wil)
 	wil_abort_scan(wil, false);
 	mutex_unlock(&wil->p2p_wdev_mutex);
 
-	wil_reset(wil, false);
-
-	return 0;
+	return wil_reset(wil, false);
 }
 
 int wil_down(struct wil6210_priv *wil)
diff --git a/drivers/net/wireless/ath/wil6210/netdev.c b/drivers/net/wireless/ath/wil6210/netdev.c
index 4a6ab2d0fdf1..7ba4e0af8f57 100644
--- a/drivers/net/wireless/ath/wil6210/netdev.c
+++ b/drivers/net/wireless/ath/wil6210/netdev.c
@@ -21,6 +21,7 @@
 static int wil_open(struct net_device *ndev)
 {
 	struct wil6210_priv *wil = ndev_to_wil(ndev);
+	int rc;
 
 	wil_dbg_misc(wil, "open\n");
 
@@ -30,16 +31,29 @@ static int wil_open(struct net_device *ndev)
 		return -EINVAL;
 	}
 
-	return wil_up(wil);
+	rc = wil_pm_runtime_get(wil);
+	if (rc < 0)
+		return rc;
+
+	rc = wil_up(wil);
+	if (rc)
+		wil_pm_runtime_put(wil);
+
+	return rc;
 }
 
 static int wil_stop(struct net_device *ndev)
 {
 	struct wil6210_priv *wil = ndev_to_wil(ndev);
+	int rc;
 
 	wil_dbg_misc(wil, "stop\n");
 
-	return wil_down(wil);
+	rc = wil_down(wil);
+	if (!rc)
+		wil_pm_runtime_put(wil);
+
+	return rc;
 }
 
 static const struct net_device_ops wil_netdev_ops = {
@@ -136,7 +150,7 @@ void *wil_if_alloc(struct device *dev)
 	wdev->iftype = NL80211_IFTYPE_STATION; /* TODO */
 	/* default monitor channel */
 	ch = wdev->wiphy->bands[NL80211_BAND_60GHZ]->channels;
-	cfg80211_chandef_create(&wdev->preset_chandef, ch, NL80211_CHAN_NO_HT);
+	cfg80211_chandef_create(&wil->monitor_chandef, ch, NL80211_CHAN_NO_HT);
 
 	ndev = alloc_netdev(0, "wlan%d", NET_NAME_UNKNOWN, wil_dev_setup);
 	if (!ndev) {
diff --git a/drivers/net/wireless/ath/wil6210/pcie_bus.c b/drivers/net/wireless/ath/wil6210/pcie_bus.c
index 6a3ab4bf916d..809092a49192 100644
--- a/drivers/net/wireless/ath/wil6210/pcie_bus.c
+++ b/drivers/net/wireless/ath/wil6210/pcie_bus.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -21,6 +22,7 @@
 #include <linux/suspend.h>
 #include "wil6210.h"
 #include <linux/rtnetlink.h>
+#include <linux/pm_runtime.h>
 
 static bool use_msi = true;
 module_param(use_msi, bool, 0444);
@@ -30,29 +32,30 @@ static bool ftm_mode;
 module_param(ftm_mode, bool, 0444);
 MODULE_PARM_DESC(ftm_mode, " Set factory test mode, default - false");
 
-#ifdef CONFIG_PM
-#ifdef CONFIG_PM_SLEEP
 static int wil6210_pm_notify(struct notifier_block *notify_block,
 			     unsigned long mode, void *unused);
-#endif /* CONFIG_PM_SLEEP */
-#endif /* CONFIG_PM */
 
 static
-void wil_set_capabilities(struct wil6210_priv *wil)
+int wil_set_capabilities(struct wil6210_priv *wil)
 {
 	const char *wil_fw_name;
 	u32 jtag_id = wil_r(wil, RGF_USER_JTAG_DEV_ID);
 	u8 chip_revision = (wil_r(wil, RGF_USER_REVISION_ID) &
 			    RGF_USER_REVISION_ID_MASK);
+	int platform_capa;
+	struct fw_map *iccm_section, *sct;
 
-	bitmap_zero(wil->hw_capabilities, hw_capability_last);
+	bitmap_zero(wil->hw_capa, hw_capa_last);
 	bitmap_zero(wil->fw_capabilities, WMI_FW_CAPABILITY_MAX);
+	bitmap_zero(wil->platform_capa, WIL_PLATFORM_CAPA_MAX);
 	wil->wil_fw_name = ftm_mode ? WIL_FW_NAME_FTM_DEFAULT :
 			   WIL_FW_NAME_DEFAULT;
 	wil->chip_revision = chip_revision;
 
 	switch (jtag_id) {
 	case JTAG_DEV_ID_SPARROW:
+		memcpy(fw_mapping, sparrow_fw_mapping,
+		       sizeof(sparrow_fw_mapping));
 		switch (chip_revision) {
 		case REVISION_ID_SPARROW_D0:
 			wil->hw_name = "Sparrow D0";
@@ -62,6 +65,12 @@ void wil_set_capabilities(struct wil6210_priv *wil)
 
 			if (wil_fw_verify_file_exists(wil, wil_fw_name))
 				wil->wil_fw_name = wil_fw_name;
+			sct = wil_find_fw_mapping("mac_rgf_ext");
+			if (!sct) {
+				wil_err(wil, "mac_rgf_ext section not found in fw_mapping\n");
+				return -EINVAL;
+			}
+			memcpy(sct, &sparrow_d0_mac_rgf_ext, sizeof(*sct));
 			break;
 		case REVISION_ID_SPARROW_B0:
 			wil->hw_name = "Sparrow B0";
@@ -72,21 +81,50 @@ void wil_set_capabilities(struct wil6210_priv *wil)
 			wil->hw_version = HW_VER_UNKNOWN;
 			break;
 		}
+		wil->rgf_fw_assert_code_addr = SPARROW_RGF_FW_ASSERT_CODE;
+		wil->rgf_ucode_assert_code_addr = SPARROW_RGF_UCODE_ASSERT_CODE;
+		break;
+	case JTAG_DEV_ID_TALYN:
+		wil->hw_name = "Talyn";
+		wil->hw_version = HW_VER_TALYN;
+		memcpy(fw_mapping, talyn_fw_mapping, sizeof(talyn_fw_mapping));
+		wil->rgf_fw_assert_code_addr = TALYN_RGF_FW_ASSERT_CODE;
+		wil->rgf_ucode_assert_code_addr = TALYN_RGF_UCODE_ASSERT_CODE;
+		if (wil_r(wil, RGF_USER_OTP_HW_RD_MACHINE_1) &
+		    BIT_NO_FLASH_INDICATION)
+			set_bit(hw_capa_no_flash, wil->hw_capa);
 		break;
 	default:
 		wil_err(wil, "Unknown board hardware, chip_id 0x%08x, chip_revision 0x%08x\n",
 			jtag_id, chip_revision);
 		wil->hw_name = "Unknown";
 		wil->hw_version = HW_VER_UNKNOWN;
+		return -EINVAL;
+	}
+
+	iccm_section = wil_find_fw_mapping("fw_code");
+	if (!iccm_section) {
+		wil_err(wil, "fw_code section not found in fw_mapping\n");
+		return -EINVAL;
 	}
+	wil->iccm_base = iccm_section->host;
 
-	wil_info(wil, "Board hardware is %s\n", wil->hw_name);
+	wil_info(wil, "Board hardware is %s, flash %sexist\n", wil->hw_name,
+		 test_bit(hw_capa_no_flash, wil->hw_capa) ? "doesn't " : "");
+
+	/* Get platform capabilities */
+	if (wil->platform_ops.get_capa) {
+		platform_capa =
+			wil->platform_ops.get_capa(wil->platform_handle);
+		memcpy(wil->platform_capa, &platform_capa,
+		       min(sizeof(wil->platform_capa), sizeof(platform_capa)));
+	}
 
 	/* extract FW capabilities from file without loading the FW */
 	wil_request_firmware(wil, wil->wil_fw_name, false);
+	wil_refresh_fw_capabilities(wil);
 
-	if (test_bit(WMI_FW_CAPABILITY_RSSI_REPORTING, wil->fw_capabilities))
-		wil_to_wiphy(wil)->signal_type = CFG80211_SIGNAL_TYPE_MBM;
+	return 0;
 }
 
 void wil_disable_irq(struct wil6210_priv *wil)
@@ -209,6 +247,8 @@ static int wil_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		.fw_recovery = wil_platform_rop_fw_recovery,
 	};
 	u32 bar_size = pci_resource_len(pdev, 0);
+	int dma_addr_size[] = {48, 40, 32}; /* keep descending order */
+	int i;
 
 	/* check HW */
 	dev_info(&pdev->dev, WIL_NAME
@@ -244,21 +284,23 @@ static int wil_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	}
 	/* rollback to err_plat */
 
-	/* device supports 48bit addresses */
-	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
-	if (rc) {
-		dev_err(dev, "dma_set_mask_and_coherent(48) failed: %d\n", rc);
-		rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	/* device supports >32bit addresses */
+	for (i = 0; i < ARRAY_SIZE(dma_addr_size); i++) {
+		rc = dma_set_mask_and_coherent(dev,
+					       DMA_BIT_MASK(dma_addr_size[i]));
 		if (rc) {
-			dev_err(dev,
-				"dma_set_mask_and_coherent(32) failed: %d\n",
-				rc);
-			goto err_plat;
+			dev_err(dev, "dma_set_mask_and_coherent(%d) failed: %d\n",
+				dma_addr_size[i], rc);
+			continue;
 		}
-	} else {
-		wil->use_extended_dma_addr = 1;
+		dev_info(dev, "using dma mask %d", dma_addr_size[i]);
+		wil->dma_addr_size = dma_addr_size[i];
+		break;
 	}
 
+	if (wil->dma_addr_size == 0)
+		goto err_plat;
+
 	rc = pci_enable_device(pdev);
 	if (rc && pdev->msi_enabled == 0) {
 		wil_err(wil,
@@ -293,18 +335,13 @@ static int wil_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	/* rollback to err_iounmap */
 	wil_info(wil, "CSR at %pR -> 0x%p\n", &pdev->resource[0], wil->csr);
 
-	wil_set_capabilities(wil);
+	rc = wil_set_capabilities(wil);
+	if (rc) {
+		wil_err(wil, "wil_set_capabilities failed, rc %d\n", rc);
+		goto err_iounmap;
+	}
 	wil6210_clear_irq(wil);
 
-	wil->keep_radio_on_during_sleep =
-		wil->platform_ops.keep_radio_on_during_sleep &&
-		wil->platform_ops.keep_radio_on_during_sleep(
-			wil->platform_handle) &&
-		test_bit(WMI_FW_CAPABILITY_D3_SUSPEND, wil->fw_capabilities);
-
-	wil_info(wil, "keep_radio_on_during_sleep (%d)\n",
-		 wil->keep_radio_on_during_sleep);
-
 	/* FW should raise IRQ when ready */
 	rc = wil_if_pcie_enable(wil);
 	if (rc) {
@@ -319,20 +356,19 @@ static int wil_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto bus_disable;
 	}
 
-#ifdef CONFIG_PM
-#ifdef CONFIG_PM_SLEEP
-	wil->pm_notify.notifier_call = wil6210_pm_notify;
+	if (IS_ENABLED(CONFIG_PM))
+		wil->pm_notify.notifier_call = wil6210_pm_notify;
+
 	rc = register_pm_notifier(&wil->pm_notify);
 	if (rc)
 		/* Do not fail the driver initialization, as suspend can
 		 * be prevented in a later phase if needed
 		 */
 		wil_err(wil, "register_pm_notifier failed: %d\n", rc);
-#endif /* CONFIG_PM_SLEEP */
-#endif /* CONFIG_PM */
 
 	wil6210_debugfs_init(wil);
 
+	wil_pm_runtime_allow(wil);
 
 	return 0;
 
@@ -359,11 +395,9 @@ static void wil_pcie_remove(struct pci_dev *pdev)
 
 	wil_dbg_misc(wil, "pcie_remove\n");
 
-#ifdef CONFIG_PM
-#ifdef CONFIG_PM_SLEEP
 	unregister_pm_notifier(&wil->pm_notify);
-#endif /* CONFIG_PM_SLEEP */
-#endif /* CONFIG_PM */
+
+	wil_pm_runtime_forbid(wil);
 
 	wil6210_debugfs_remove(wil);
 	rtnl_lock();
@@ -381,18 +415,19 @@ static void wil_pcie_remove(struct pci_dev *pdev)
 static const struct pci_device_id wil6210_pcie_ids[] = {
 	{ PCI_DEVICE(0x1ae9, 0x0310) },
 	{ PCI_DEVICE(0x1ae9, 0x0302) }, /* same as above, firmware broken */
+	{ PCI_DEVICE(0x17cb, 0x1201) }, /* Talyn */
 	{ /* end: all zeroes */	},
 };
 MODULE_DEVICE_TABLE(pci, wil6210_pcie_ids);
 
-#ifdef CONFIG_PM
-#ifdef CONFIG_PM_SLEEP
-
 static int wil6210_suspend(struct device *dev, bool is_runtime)
 {
 	int rc = 0;
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct wil6210_priv *wil = pci_get_drvdata(pdev);
+	struct net_device *ndev = wil_to_ndev(wil);
+	bool keep_radio_on = ndev->flags & IFF_UP &&
+			     wil->keep_radio_on_during_sleep;
 
 	wil_dbg_pm(wil, "suspend: %s\n", is_runtime ? "runtime" : "system");
 
@@ -400,16 +435,18 @@ static int wil6210_suspend(struct device *dev, bool is_runtime)
 	if (rc)
 		goto out;
 
-	rc = wil_suspend(wil, is_runtime);
+	rc = wil_suspend(wil, is_runtime, keep_radio_on);
 	if (!rc) {
-		wil->suspend_stats.successful_suspends++;
-
-		/* If platform device supports keep_radio_on_during_sleep
-		 * it will control PCIe master
+		/* In case radio stays on, platform device will control
+		 * PCIe master
 		 */
-		if (!wil->keep_radio_on_during_sleep)
+		if (!keep_radio_on) {
 			/* disable bus mastering */
 			pci_clear_master(pdev);
+			wil->suspend_stats.r_off.successful_suspends++;
+		} else {
+			wil->suspend_stats.r_on.successful_suspends++;
+		}
 	}
 out:
 	return rc;
@@ -420,23 +457,32 @@ static int wil6210_resume(struct device *dev, bool is_runtime)
 	int rc = 0;
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct wil6210_priv *wil = pci_get_drvdata(pdev);
+	struct net_device *ndev = wil_to_ndev(wil);
+	bool keep_radio_on = ndev->flags & IFF_UP &&
+			     wil->keep_radio_on_during_sleep;
 
 	wil_dbg_pm(wil, "resume: %s\n", is_runtime ? "runtime" : "system");
 
-	/* If platform device supports keep_radio_on_during_sleep it will
-	 * control PCIe master
+	/* In case radio stays on, platform device will control
+	 * PCIe master
 	 */
-	if (!wil->keep_radio_on_during_sleep)
+	if (!keep_radio_on)
 		/* allow master */
 		pci_set_master(pdev);
-	rc = wil_resume(wil, is_runtime);
+	rc = wil_resume(wil, is_runtime, keep_radio_on);
 	if (rc) {
 		wil_err(wil, "device failed to resume (%d)\n", rc);
-		wil->suspend_stats.failed_resumes++;
-		if (!wil->keep_radio_on_during_sleep)
+		if (!keep_radio_on) {
 			pci_clear_master(pdev);
+			wil->suspend_stats.r_off.failed_resumes++;
+		} else {
+			wil->suspend_stats.r_on.failed_resumes++;
+		}
 	} else {
-		wil->suspend_stats.successful_resumes++;
+		if (keep_radio_on)
+			wil->suspend_stats.r_on.successful_resumes++;
+		else
+			wil->suspend_stats.r_off.successful_resumes++;
 	}
 
 	return rc;
@@ -481,21 +527,49 @@ static int wil6210_pm_notify(struct notifier_block *notify_block,
 	return rc;
 }
 
-static int wil6210_pm_suspend(struct device *dev)
+static int __maybe_unused wil6210_pm_suspend(struct device *dev)
 {
 	return wil6210_suspend(dev, false);
 }
 
-static int wil6210_pm_resume(struct device *dev)
+static int __maybe_unused wil6210_pm_resume(struct device *dev)
 {
 	return wil6210_resume(dev, false);
 }
-#endif /* CONFIG_PM_SLEEP */
 
-#endif /* CONFIG_PM */
+static int __maybe_unused wil6210_pm_runtime_idle(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct wil6210_priv *wil = pci_get_drvdata(pdev);
+
+	wil_dbg_pm(wil, "Runtime idle\n");
+
+	return wil_can_suspend(wil, true);
+}
+
+static int __maybe_unused wil6210_pm_runtime_resume(struct device *dev)
+{
+	return wil6210_resume(dev, true);
+}
+
+static int __maybe_unused wil6210_pm_runtime_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct wil6210_priv *wil = pci_get_drvdata(pdev);
+
+	if (test_bit(wil_status_suspended, wil->status)) {
+		wil_dbg_pm(wil, "trying to suspend while suspended\n");
+		return 1;
+	}
+
+	return wil6210_suspend(dev, true);
+}
 
 static const struct dev_pm_ops wil6210_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(wil6210_pm_suspend, wil6210_pm_resume)
+	SET_RUNTIME_PM_OPS(wil6210_pm_runtime_suspend,
+			   wil6210_pm_runtime_resume,
+			   wil6210_pm_runtime_idle)
 };
 
 static struct pci_driver wil6210_driver = {
diff --git a/drivers/net/wireless/ath/wil6210/pm.c b/drivers/net/wireless/ath/wil6210/pm.c
index 8f5d1b447aaa..0a96518a566f 100644
--- a/drivers/net/wireless/ath/wil6210/pm.c
+++ b/drivers/net/wireless/ath/wil6210/pm.c
@@ -16,15 +16,30 @@
 
 #include "wil6210.h"
 #include <linux/jiffies.h>
+#include <linux/pm_runtime.h>
+
+#define WIL6210_AUTOSUSPEND_DELAY_MS (1000)
 
 int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime)
 {
 	int rc = 0;
 	struct wireless_dev *wdev = wil->wdev;
 	struct net_device *ndev = wil_to_ndev(wil);
+	bool wmi_only = test_bit(WMI_FW_CAPABILITY_WMI_ONLY,
+				 wil->fw_capabilities);
 
 	wil_dbg_pm(wil, "can_suspend: %s\n", is_runtime ? "runtime" : "system");
 
+	if (wmi_only || debug_fw) {
+		wil_dbg_pm(wil, "Deny any suspend - %s mode\n",
+			   wmi_only ? "wmi_only" : "debug_fw");
+		rc = -EBUSY;
+		goto out;
+	}
+	if (is_runtime && !wil->platform_ops.suspend) {
+		rc = -EBUSY;
+		goto out;
+	}
 	if (!(ndev->flags & IFF_UP)) {
 		/* can always sleep when down */
 		wil_dbg_pm(wil, "Interface is down\n");
@@ -44,6 +59,10 @@ int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime)
 	/* interface is running */
 	switch (wdev->iftype) {
 	case NL80211_IFTYPE_MONITOR:
+		wil_dbg_pm(wil, "Sniffer\n");
+		rc = -EBUSY;
+		goto out;
+	/* for STA-like interface, don't runtime suspend */
 	case NL80211_IFTYPE_STATION:
 	case NL80211_IFTYPE_P2P_CLIENT:
 		if (test_bit(wil_status_fwconnecting, wil->status)) {
@@ -51,6 +70,12 @@ int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime)
 			rc = -EBUSY;
 			goto out;
 		}
+		/* Runtime pm not supported in case the interface is up */
+		if (is_runtime) {
+			wil_dbg_pm(wil, "STA-like interface\n");
+			rc = -EBUSY;
+			goto out;
+		}
 		break;
 	/* AP-like interface - can't suspend */
 	default:
@@ -120,6 +145,13 @@ static int wil_suspend_keep_radio_on(struct wil6210_priv *wil)
 
 	/* Prevent handling of new tx and wmi commands */
 	set_bit(wil_status_suspending, wil->status);
+	if (test_bit(wil_status_collecting_dumps, wil->status)) {
+		/* Device collects crash dump, cancel the suspend */
+		wil_dbg_pm(wil, "reject suspend while collecting crash dump\n");
+		clear_bit(wil_status_suspending, wil->status);
+		wil->suspend_stats.rejected_by_host++;
+		return -EBUSY;
+	}
 	wil_update_net_queues_bh(wil, NULL, true);
 
 	if (!wil_is_tx_idle(wil)) {
@@ -158,7 +190,7 @@ static int wil_suspend_keep_radio_on(struct wil6210_priv *wil)
 					break;
 				wil_err(wil,
 					"TO waiting for idle RX, suspend failed\n");
-				wil->suspend_stats.failed_suspends++;
+				wil->suspend_stats.r_on.failed_suspends++;
 				goto resume_after_fail;
 			}
 			wil_dbg_ratelimited(wil, "rx vring is not empty -> NAPI\n");
@@ -174,7 +206,7 @@ static int wil_suspend_keep_radio_on(struct wil6210_priv *wil)
 	 */
 	if (!wil_is_wmi_idle(wil)) {
 		wil_err(wil, "suspend failed due to pending WMI events\n");
-		wil->suspend_stats.failed_suspends++;
+		wil->suspend_stats.r_on.failed_suspends++;
 		goto resume_after_fail;
 	}
 
@@ -188,7 +220,7 @@ static int wil_suspend_keep_radio_on(struct wil6210_priv *wil)
 		if (rc) {
 			wil_err(wil, "platform device failed to suspend (%d)\n",
 				rc);
-			wil->suspend_stats.failed_suspends++;
+			wil->suspend_stats.r_on.failed_suspends++;
 			wil_c(wil, RGF_USER_CLKS_CTL_0, BIT_USER_CLKS_RST_PWGD);
 			wil_unmask_irq(wil);
 			goto resume_after_fail;
@@ -230,11 +262,21 @@ static int wil_suspend_radio_off(struct wil6210_priv *wil)
 
 	wil_dbg_pm(wil, "suspend radio off\n");
 
+	set_bit(wil_status_suspending, wil->status);
+	if (test_bit(wil_status_collecting_dumps, wil->status)) {
+		/* Device collects crash dump, cancel the suspend */
+		wil_dbg_pm(wil, "reject suspend while collecting crash dump\n");
+		clear_bit(wil_status_suspending, wil->status);
+		wil->suspend_stats.rejected_by_host++;
+		return -EBUSY;
+	}
+
 	/* if netif up, hardware is alive, shut it down */
 	if (ndev->flags & IFF_UP) {
 		rc = wil_down(wil);
 		if (rc) {
 			wil_err(wil, "wil_down : %d\n", rc);
+			wil->suspend_stats.r_off.failed_suspends++;
 			goto out;
 		}
 	}
@@ -247,6 +289,7 @@ static int wil_suspend_radio_off(struct wil6210_priv *wil)
 		rc = wil->platform_ops.suspend(wil->platform_handle, false);
 		if (rc) {
 			wil_enable_irq(wil);
+			wil->suspend_stats.r_off.failed_suspends++;
 			goto out;
 		}
 	}
@@ -254,6 +297,7 @@ static int wil_suspend_radio_off(struct wil6210_priv *wil)
 	set_bit(wil_status_suspended, wil->status);
 
 out:
+	clear_bit(wil_status_suspending, wil->status);
 	wil_dbg_pm(wil, "suspend radio off: %d\n", rc);
 
 	return rc;
@@ -279,12 +323,9 @@ static int wil_resume_radio_off(struct wil6210_priv *wil)
 	return rc;
 }
 
-int wil_suspend(struct wil6210_priv *wil, bool is_runtime)
+int wil_suspend(struct wil6210_priv *wil, bool is_runtime, bool keep_radio_on)
 {
 	int rc = 0;
-	struct net_device *ndev = wil_to_ndev(wil);
-	bool keep_radio_on = ndev->flags & IFF_UP &&
-			     wil->keep_radio_on_during_sleep;
 
 	wil_dbg_pm(wil, "suspend: %s\n", is_runtime ? "runtime" : "system");
 
@@ -301,19 +342,12 @@ int wil_suspend(struct wil6210_priv *wil, bool is_runtime)
 	wil_dbg_pm(wil, "suspend: %s => %d\n",
 		   is_runtime ? "runtime" : "system", rc);
 
-	if (!rc)
-		wil->suspend_stats.suspend_start_time = ktime_get();
-
 	return rc;
 }
 
-int wil_resume(struct wil6210_priv *wil, bool is_runtime)
+int wil_resume(struct wil6210_priv *wil, bool is_runtime, bool keep_radio_on)
 {
 	int rc = 0;
-	struct net_device *ndev = wil_to_ndev(wil);
-	bool keep_radio_on = ndev->flags & IFF_UP &&
-			     wil->keep_radio_on_during_sleep;
-	unsigned long long suspend_time_usec = 0;
 
 	wil_dbg_pm(wil, "resume: %s\n", is_runtime ? "runtime" : "system");
 
@@ -331,20 +365,49 @@ int wil_resume(struct wil6210_priv *wil, bool is_runtime)
 	else
 		rc = wil_resume_radio_off(wil);
 
-	if (rc)
-		goto out;
-
-	suspend_time_usec =
-		ktime_to_us(ktime_sub(ktime_get(),
-				      wil->suspend_stats.suspend_start_time));
-	wil->suspend_stats.total_suspend_time += suspend_time_usec;
-	if (suspend_time_usec < wil->suspend_stats.min_suspend_time)
-		wil->suspend_stats.min_suspend_time = suspend_time_usec;
-	if (suspend_time_usec > wil->suspend_stats.max_suspend_time)
-		wil->suspend_stats.max_suspend_time = suspend_time_usec;
-
 out:
-	wil_dbg_pm(wil, "resume: %s => %d, suspend time %lld usec\n",
-		   is_runtime ? "runtime" : "system", rc, suspend_time_usec);
+	wil_dbg_pm(wil, "resume: %s => %d\n", is_runtime ? "runtime" : "system",
+		   rc);
 	return rc;
 }
+
+void wil_pm_runtime_allow(struct wil6210_priv *wil)
+{
+	struct device *dev = wil_to_dev(wil);
+
+	pm_runtime_put_noidle(dev);
+	pm_runtime_set_autosuspend_delay(dev, WIL6210_AUTOSUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_allow(dev);
+}
+
+void wil_pm_runtime_forbid(struct wil6210_priv *wil)
+{
+	struct device *dev = wil_to_dev(wil);
+
+	pm_runtime_forbid(dev);
+	pm_runtime_get_noresume(dev);
+}
+
+int wil_pm_runtime_get(struct wil6210_priv *wil)
+{
+	int rc;
+	struct device *dev = wil_to_dev(wil);
+
+	rc = pm_runtime_get_sync(dev);
+	if (rc < 0) {
+		wil_err(wil, "pm_runtime_get_sync() failed, rc = %d\n", rc);
+		pm_runtime_put_noidle(dev);
+		return rc;
+	}
+
+	return 0;
+}
+
+void wil_pm_runtime_put(struct wil6210_priv *wil)
+{
+	struct device *dev = wil_to_dev(wil);
+
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+}
diff --git a/drivers/net/wireless/ath/wil6210/pmc.c b/drivers/net/wireless/ath/wil6210/pmc.c
index 2e301b6b32a9..4ea27b0bd278 100644
--- a/drivers/net/wireless/ath/wil6210/pmc.c
+++ b/drivers/net/wireless/ath/wil6210/pmc.c
@@ -111,14 +111,14 @@ void wil_pmc_alloc(struct wil6210_priv *wil,
 	 *
 	 * HW has limitation that all vrings addresses must share the same
 	 * upper 16 msb bits part of 48 bits address. To workaround that,
-	 * if we are using 48 bit addresses switch to 32 bit allocation
-	 * before allocating vring memory.
+	 * if we are using more than 32 bit addresses switch to 32 bit
+	 * allocation before allocating vring memory.
 	 *
 	 * There's no check for the return value of dma_set_mask_and_coherent,
 	 * since we assume if we were able to set the mask during
 	 * initialization in this system it will not fail if we set it again
 	 */
-	if (wil->use_extended_dma_addr)
+	if (wil->dma_addr_size > 32)
 		dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 
 	pmc->pring_va = dma_alloc_coherent(dev,
@@ -126,8 +126,9 @@ void wil_pmc_alloc(struct wil6210_priv *wil,
 			&pmc->pring_pa,
 			GFP_KERNEL);
 
-	if (wil->use_extended_dma_addr)
-		dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (wil->dma_addr_size > 32)
+		dma_set_mask_and_coherent(dev,
+					  DMA_BIT_MASK(wil->dma_addr_size));
 
 	wil_dbg_misc(wil,
 		     "pmc_alloc: allocated pring %p => %pad. %zd x %d = total %zd bytes\n",
diff --git a/drivers/net/wireless/ath/wil6210/txrx.c b/drivers/net/wireless/ath/wil6210/txrx.c
index 389c718cd257..16b8a4e5201f 100644
--- a/drivers/net/wireless/ath/wil6210/txrx.c
+++ b/drivers/net/wireless/ath/wil6210/txrx.c
@@ -178,14 +178,14 @@ static int wil_vring_alloc(struct wil6210_priv *wil, struct vring *vring)
 	 *
 	 * HW has limitation that all vrings addresses must share the same
 	 * upper 16 msb bits part of 48 bits address. To workaround that,
-	 * if we are using 48 bit addresses switch to 32 bit allocation
-	 * before allocating vring memory.
+	 * if we are using more than 32 bit addresses switch to 32 bit
+	 * allocation before allocating vring memory.
 	 *
 	 * There's no check for the return value of dma_set_mask_and_coherent,
 	 * since we assume if we were able to set the mask during
 	 * initialization in this system it will not fail if we set it again
 	 */
-	if (wil->use_extended_dma_addr)
+	if (wil->dma_addr_size > 32)
 		dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 
 	vring->va = dma_alloc_coherent(dev, sz, &vring->pa, GFP_KERNEL);
@@ -195,8 +195,9 @@ static int wil_vring_alloc(struct wil6210_priv *wil, struct vring *vring)
 		return -ENOMEM;
 	}
 
-	if (wil->use_extended_dma_addr)
-		dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
+	if (wil->dma_addr_size > 32)
+		dma_set_mask_and_coherent(dev,
+					  DMA_BIT_MASK(wil->dma_addr_size));
 
 	/* initially, all descriptors are SW owned
 	 * For Tx and Rx, ownership bit is at the same location, thus
@@ -347,7 +348,6 @@ static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct vring *vring,
 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,
 				       struct sk_buff *skb)
 {
-	struct wireless_dev *wdev = wil->wdev;
 	struct wil6210_rtap {
 		struct ieee80211_radiotap_header rthdr;
 		/* fields should be in the order of bits in rthdr.it_present */
@@ -374,7 +374,7 @@ static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,
 	int rtap_len = sizeof(struct wil6210_rtap);
 	int phy_length = 0; /* phy info header size, bytes */
 	static char phy_data[128];
-	struct ieee80211_channel *ch = wdev->preset_chandef.chan;
+	struct ieee80211_channel *ch = wil->monitor_chandef.chan;
 
 	if (rtap_include_phy_info) {
 		rtap_len = sizeof(*rtap_vendor) + sizeof(*d);
diff --git a/drivers/net/wireless/ath/wil6210/wil6210.h b/drivers/net/wireless/ath/wil6210/wil6210.h
index 1e340d04bd70..0df2aada6659 100644
--- a/drivers/net/wireless/ath/wil6210/wil6210.h
+++ b/drivers/net/wireless/ath/wil6210/wil6210.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -82,18 +83,18 @@ static inline u32 WIL_GET_BITS(u32 x, int b0, int b1)
  */
 #define WIL_MAX_MPDU_OVERHEAD	(62)
 
-struct wil_suspend_stats {
+struct wil_suspend_count_stats {
 	unsigned long successful_suspends;
-	unsigned long failed_suspends;
 	unsigned long successful_resumes;
+	unsigned long failed_suspends;
 	unsigned long failed_resumes;
-	unsigned long rejected_by_device;
+};
+
+struct wil_suspend_stats {
+	struct wil_suspend_count_stats r_off;
+	struct wil_suspend_count_stats r_on;
+	unsigned long rejected_by_device; /* only radio on */
 	unsigned long rejected_by_host;
-	unsigned long long total_suspend_time;
-	unsigned long long min_suspend_time;
-	unsigned long long max_suspend_time;
-	ktime_t collection_start;
-	ktime_t suspend_start_time;
 };
 
 /* Calculate MAC buffer size for the firmware. It includes all overhead,
@@ -161,10 +162,15 @@ struct RGF_ICR {
 #define RGF_USER_USAGE_6		(0x880018)
 	#define BIT_USER_OOB_MODE		BIT(31)
 	#define BIT_USER_OOB_R2_MODE		BIT(30)
+#define RGF_USER_USAGE_8		(0x880020)
+	#define BIT_USER_PREVENT_DEEP_SLEEP	BIT(0)
+	#define BIT_USER_SUPPORT_T_POWER_ON_0	BIT(1)
+	#define BIT_USER_EXT_CLK		BIT(2)
 #define RGF_USER_HW_MACHINE_STATE	(0x8801dc)
 	#define HW_MACHINE_BOOT_DONE	(0x3fffffd)
 #define RGF_USER_USER_CPU_0		(0x8801e0)
 	#define BIT_USER_USER_CPU_MAN_RST	BIT(1) /* user_cpu_man_rst */
+#define RGF_USER_CPU_PC			(0x8801e8)
 #define RGF_USER_MAC_CPU_0		(0x8801fc)
 	#define BIT_USER_MAC_CPU_MAN_RST	BIT(1) /* mac_cpu_man_rst */
 #define RGF_USER_USER_SCRATCH_PAD	(0x8802bc)
@@ -190,6 +196,19 @@ struct RGF_ICR {
 #define RGF_USER_CLKS_CTL_EXT_SW_RST_VEC_1	(0x880c2c)
 #define RGF_USER_SPARROW_M_4			(0x880c50) /* Sparrow */
 	#define BIT_SPARROW_M_4_SEL_SLEEP_OR_REF	BIT(2)
+#define RGF_USER_OTP_HW_RD_MACHINE_1	(0x880ce0)
+	#define BIT_NO_FLASH_INDICATION		BIT(8)
+#define RGF_USER_XPM_IFC_RD_TIME1	(0x880cec)
+#define RGF_USER_XPM_IFC_RD_TIME2	(0x880cf0)
+#define RGF_USER_XPM_IFC_RD_TIME3	(0x880cf4)
+#define RGF_USER_XPM_IFC_RD_TIME4	(0x880cf8)
+#define RGF_USER_XPM_IFC_RD_TIME5	(0x880cfc)
+#define RGF_USER_XPM_IFC_RD_TIME6	(0x880d00)
+#define RGF_USER_XPM_IFC_RD_TIME7	(0x880d04)
+#define RGF_USER_XPM_IFC_RD_TIME8	(0x880d08)
+#define RGF_USER_XPM_IFC_RD_TIME9	(0x880d0c)
+#define RGF_USER_XPM_IFC_RD_TIME10	(0x880d10)
+#define RGF_USER_XPM_RD_DOUT_SAMPLE_TIME (0x880d64)
 
 #define RGF_DMA_EP_TX_ICR		(0x881bb4) /* struct RGF_ICR */
 	#define BIT_DMA_EP_TX_ICR_TX_DONE	BIT(0)
@@ -280,22 +299,33 @@ struct RGF_ICR {
 #define RGF_CAF_PLL_LOCK_STATUS		(0x88afec)
 	#define BIT_CAF_OSC_DIG_XTAL_STABLE	BIT(0)
 
+#define USER_EXT_USER_PMU_3		(0x88d00c)
+	#define BIT_PMU_DEVICE_RDY		BIT(0)
+
 #define RGF_USER_JTAG_DEV_ID	(0x880b34) /* device ID */
 	#define JTAG_DEV_ID_SPARROW	(0x2632072f)
+	#define JTAG_DEV_ID_TALYN	(0x7e0e1)
 
 #define RGF_USER_REVISION_ID		(0x88afe4)
 #define RGF_USER_REVISION_ID_MASK	(3)
 	#define REVISION_ID_SPARROW_B0	(0x0)
 	#define REVISION_ID_SPARROW_D0	(0x3)
 
+#define RGF_OTP_MAC			(0x8a0620)
+
 /* crash codes for FW/Ucode stored here */
-#define RGF_FW_ASSERT_CODE		(0x91f020)
-#define RGF_UCODE_ASSERT_CODE		(0x91f028)
+
+/* ASSERT RGFs */
+#define SPARROW_RGF_FW_ASSERT_CODE	(0x91f020)
+#define SPARROW_RGF_UCODE_ASSERT_CODE	(0x91f028)
+#define TALYN_RGF_FW_ASSERT_CODE	(0xa37020)
+#define TALYN_RGF_UCODE_ASSERT_CODE	(0xa37028)
 
 enum {
 	HW_VER_UNKNOWN,
 	HW_VER_SPARROW_B0, /* REVISION_ID_SPARROW_B0 */
 	HW_VER_SPARROW_D0, /* REVISION_ID_SPARROW_D0 */
+	HW_VER_TALYN,	/* JTAG_DEV_ID_TALYN */
 };
 
 /* popular locations */
@@ -311,6 +341,10 @@ enum {
 #define WIL_DATA_COMPLETION_TO_MS 200
 
 /* Hardware definitions end */
+#define SPARROW_FW_MAPPING_TABLE_SIZE 10
+#define TALYN_FW_MAPPING_TABLE_SIZE 13
+#define MAX_FW_MAPPING_TABLE_SIZE 13
+
 struct fw_map {
 	u32 from; /* linker address - from, inclusive */
 	u32 to;   /* linker address - to, exclusive */
@@ -320,7 +354,10 @@ struct fw_map {
 };
 
 /* array size should be in sync with actual definition in the wmi.c */
-extern const struct fw_map fw_mapping[10];
+extern const struct fw_map sparrow_fw_mapping[SPARROW_FW_MAPPING_TABLE_SIZE];
+extern const struct fw_map sparrow_d0_mac_rgf_ext;
+extern const struct fw_map talyn_fw_mapping[TALYN_FW_MAPPING_TABLE_SIZE];
+extern struct fw_map fw_mapping[MAX_FW_MAPPING_TABLE_SIZE];
 
 /**
  * mk_cidxtid - construct @cidxtid field
@@ -435,12 +472,13 @@ enum { /* for wil6210_priv.status */
 	wil_status_fwconnected,
 	wil_status_dontscan,
 	wil_status_mbox_ready, /* MBOX structures ready */
-	wil_status_irqen, /* FIXME: interrupts enabled - for debug */
+	wil_status_irqen, /* interrupts enabled - for debug */
 	wil_status_napi_en, /* NAPI enabled protected by wil->mutex */
 	wil_status_resetting, /* reset in progress */
 	wil_status_suspending, /* suspend in progress */
 	wil_status_suspended, /* suspend completed, device is suspended */
 	wil_status_resuming, /* resume in progress */
+	wil_status_collecting_dumps, /* crashdump collection in progress */
 	wil_status_last /* keep last */
 };
 
@@ -565,7 +603,8 @@ enum {
 };
 
 enum {
-	hw_capability_last
+	hw_capa_no_flash,
+	hw_capa_last
 };
 
 struct wil_probe_client_req {
@@ -616,6 +655,16 @@ struct blink_on_off_time {
 	u32 off_ms;
 };
 
+struct wil_debugfs_iomem_data {
+	void *offset;
+	struct wil6210_priv *wil;
+};
+
+struct wil_debugfs_data {
+	struct wil_debugfs_iomem_data *data_arr;
+	int iomem_data_count;
+};
+
 extern struct blink_on_off_time led_blink_time[WIL_LED_TIME_LAST];
 extern u8 led_id;
 extern u8 led_polarity;
@@ -631,14 +680,19 @@ struct wil6210_priv {
 	u8 chip_revision;
 	const char *hw_name;
 	const char *wil_fw_name;
-	DECLARE_BITMAP(hw_capabilities, hw_capability_last);
+	char *board_file;
+	u32 brd_file_addr;
+	u32 brd_file_max_size;
+	DECLARE_BITMAP(hw_capa, hw_capa_last);
 	DECLARE_BITMAP(fw_capabilities, WMI_FW_CAPABILITY_MAX);
+	DECLARE_BITMAP(platform_capa, WIL_PLATFORM_CAPA_MAX);
 	u8 n_mids; /* number of additional MIDs as reported by FW */
 	u32 recovery_count; /* num of FW recovery attempts in a short time */
 	u32 recovery_state; /* FW recovery state machine */
 	unsigned long last_fw_recovery; /* jiffies of last fw recovery */
 	wait_queue_head_t wq; /* for all wait_event() use */
 	/* profile */
+	struct cfg80211_chan_def monitor_chandef;
 	u32 monitor_flags;
 	u32 privacy; /* secure connection? */
 	u8 hidden_ssid; /* relevant in AP mode */
@@ -694,7 +748,7 @@ struct wil6210_priv {
 	struct wil_sta_info sta[WIL6210_MAX_CID];
 	int bcast_vring;
 	u32 vring_idle_trsh; /* HW fetches up to 16 descriptors at once  */
-	bool use_extended_dma_addr; /* indicates whether we are using 48 bits */
+	u32 dma_addr_size; /* indicates dma addr size */
 	/* scan */
 	struct cfg80211_scan_request *scan_request;
 
@@ -703,11 +757,12 @@ struct wil6210_priv {
 	atomic_t isr_count_rx, isr_count_tx;
 	/* debugfs */
 	struct dentry *debug;
-	struct wil_blob_wrapper blobs[ARRAY_SIZE(fw_mapping)];
+	struct wil_blob_wrapper blobs[MAX_FW_MAPPING_TABLE_SIZE];
 	u8 discovery_mode;
 	u8 abft_len;
 	u8 wakeup_trigger;
 	struct wil_suspend_stats suspend_stats;
+	struct wil_debugfs_data dbg_data;
 
 	void *platform_handle;
 	struct wil_platform_ops platform_ops;
@@ -731,16 +786,16 @@ struct wil6210_priv {
 
 	int fw_calib_result;
 
-#ifdef CONFIG_PM
-#ifdef CONFIG_PM_SLEEP
 	struct notifier_block pm_notify;
-#endif /* CONFIG_PM_SLEEP */
-#endif /* CONFIG_PM */
 
 	bool suspend_resp_rcvd;
 	bool suspend_resp_comp;
 	u32 bus_request_kbps;
 	u32 bus_request_kbps_pre_suspend;
+
+	u32 rgf_fw_assert_code_addr;
+	u32 rgf_ucode_assert_code_addr;
+	u32 iccm_base;
 };
 
 #define wil_to_wiphy(i) (i->wdev->wiphy)
@@ -861,10 +916,13 @@ int wil_up(struct wil6210_priv *wil);
 int __wil_up(struct wil6210_priv *wil);
 int wil_down(struct wil6210_priv *wil);
 int __wil_down(struct wil6210_priv *wil);
+void wil_refresh_fw_capabilities(struct wil6210_priv *wil);
 void wil_mbox_ring_le2cpus(struct wil6210_mbox_ring *r);
 int wil_find_cid(struct wil6210_priv *wil, const u8 *mac);
 void wil_set_ethtoolops(struct net_device *ndev);
 
+struct fw_map *wil_find_fw_mapping(const char *section);
+void __iomem *wmi_buffer_block(struct wil6210_priv *wil, __le32 ptr, u32 size);
 void __iomem *wmi_buffer(struct wil6210_priv *wil, __le32 ptr);
 void __iomem *wmi_addr(struct wil6210_priv *wil, u32 ptr);
 int wmi_read_hdr(struct wil6210_priv *wil, __le32 ptr,
@@ -997,11 +1055,17 @@ int wil_iftype_nl2wmi(enum nl80211_iftype type);
 
 int wil_request_firmware(struct wil6210_priv *wil, const char *name,
 			 bool load);
+int wil_request_board(struct wil6210_priv *wil, const char *name);
 bool wil_fw_verify_file_exists(struct wil6210_priv *wil, const char *name);
 
+void wil_pm_runtime_allow(struct wil6210_priv *wil);
+void wil_pm_runtime_forbid(struct wil6210_priv *wil);
+int wil_pm_runtime_get(struct wil6210_priv *wil);
+void wil_pm_runtime_put(struct wil6210_priv *wil);
+
 int wil_can_suspend(struct wil6210_priv *wil, bool is_runtime);
-int wil_suspend(struct wil6210_priv *wil, bool is_runtime);
-int wil_resume(struct wil6210_priv *wil, bool is_runtime);
+int wil_suspend(struct wil6210_priv *wil, bool is_runtime, bool keep_radio_on);
+int wil_resume(struct wil6210_priv *wil, bool is_runtime, bool keep_radio_on);
 bool wil_is_wmi_idle(struct wil6210_priv *wil);
 int wmi_resume(struct wil6210_priv *wil);
 int wmi_suspend(struct wil6210_priv *wil);
@@ -1016,4 +1080,8 @@ void wil_halp_unvote(struct wil6210_priv *wil);
 void wil6210_set_halp(struct wil6210_priv *wil);
 void wil6210_clear_halp(struct wil6210_priv *wil);
 
+int wmi_start_sched_scan(struct wil6210_priv *wil,
+			 struct cfg80211_sched_scan_request *request);
+int wmi_stop_sched_scan(struct wil6210_priv *wil);
+
 #endif /* __WIL6210_H__ */
diff --git a/drivers/net/wireless/ath/wil6210/wil_crash_dump.c b/drivers/net/wireless/ath/wil6210/wil_crash_dump.c
index e53cf0cf7031..1ed330674d9b 100644
--- a/drivers/net/wireless/ath/wil6210/wil_crash_dump.c
+++ b/drivers/net/wireless/ath/wil6210/wil_crash_dump.c
@@ -72,6 +72,15 @@ int wil_fw_copy_crash_dump(struct wil6210_priv *wil, void *dest, u32 size)
 		return -EINVAL;
 	}
 
+	set_bit(wil_status_collecting_dumps, wil->status);
+	if (test_bit(wil_status_suspending, wil->status) ||
+	    test_bit(wil_status_suspended, wil->status) ||
+	    test_bit(wil_status_resetting, wil->status)) {
+		wil_err(wil, "cannot collect fw dump during suspend/reset\n");
+		clear_bit(wil_status_collecting_dumps, wil->status);
+		return -EINVAL;
+	}
+
 	/* copy to crash dump area */
 	for (i = 0; i < ARRAY_SIZE(fw_mapping); i++) {
 		map = &fw_mapping[i];
@@ -91,6 +100,8 @@ int wil_fw_copy_crash_dump(struct wil6210_priv *wil, void *dest, u32 size)
 				     (const void __iomem * __force)data, len);
 	}
 
+	clear_bit(wil_status_collecting_dumps, wil->status);
+
 	return 0;
 }
 
diff --git a/drivers/net/wireless/ath/wil6210/wil_platform.h b/drivers/net/wireless/ath/wil6210/wil_platform.h
index 5d9e4bfcb045..177026e5323b 100644
--- a/drivers/net/wireless/ath/wil6210/wil_platform.h
+++ b/drivers/net/wireless/ath/wil6210/wil_platform.h
@@ -27,6 +27,18 @@ enum wil_platform_event {
 	WIL_PLATFORM_EVT_POST_SUSPEND = 4,
 };
 
+enum wil_platform_features {
+	WIL_PLATFORM_FEATURE_FW_EXT_CLK_CONTROL = 0,
+	WIL_PLATFORM_FEATURE_MAX,
+};
+
+enum wil_platform_capa {
+	WIL_PLATFORM_CAPA_RADIO_ON_IN_SUSPEND = 0,
+	WIL_PLATFORM_CAPA_T_PWR_ON_0 = 1,
+	WIL_PLATFORM_CAPA_EXT_CLK = 2,
+	WIL_PLATFORM_CAPA_MAX,
+};
+
 /**
  * struct wil_platform_ops - wil platform module calls from this
  * driver to platform driver
@@ -37,7 +49,8 @@ struct wil_platform_ops {
 	int (*resume)(void *handle, bool device_powered_on);
 	void (*uninit)(void *handle);
 	int (*notify)(void *handle, enum wil_platform_event evt);
-	bool (*keep_radio_on_during_sleep)(void *handle);
+	int (*get_capa)(void *handle);
+	void (*set_features)(void *handle, int features);
 };
 
 /**
diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c
index ffdd2fa401b1..b31e2514f8c2 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.c
+++ b/drivers/net/wireless/ath/wil6210/wmi.c
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
+ * Copyright (c) 2018, The Linux Foundation. All rights reserved.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -38,6 +39,7 @@ MODULE_PARM_DESC(led_id,
 		 " 60G device led enablement. Set the led ID (0-2) to enable");
 
 #define WIL_WAIT_FOR_SUSPEND_RESUME_COMP 200
+#define WIL_WMI_CALL_GENERAL_TO_MS 100
 
 /**
  * WMI event receiving - theory of operations
@@ -69,23 +71,23 @@ MODULE_PARM_DESC(led_id,
  * On the PCI bus, there is one BAR (BAR0) of 2Mb size, exposing
  * AHB addresses starting from 0x880000
  *
- * Internally, firmware uses addresses that allows faster access but
+ * Internally, firmware uses addresses that allow faster access but
  * are invisible from the host. To read from these addresses, alternative
  * AHB address must be used.
- *
- * Memory mapping
- * Linker address         PCI/Host address
- *                        0x880000 .. 0xa80000  2Mb BAR0
- * 0x800000 .. 0x807000   0x900000 .. 0x907000  28k DCCM
- * 0x840000 .. 0x857000   0x908000 .. 0x91f000  92k PERIPH
  */
 
 /**
- * @fw_mapping provides memory remapping table
+ * @sparrow_fw_mapping provides memory remapping table for sparrow
  *
  * array size should be in sync with the declaration in the wil6210.h
+ *
+ * Sparrow memory mapping:
+ * Linker address         PCI/Host address
+ *                        0x880000 .. 0xa80000  2Mb BAR0
+ * 0x800000 .. 0x808000   0x900000 .. 0x908000  32k DCCM
+ * 0x840000 .. 0x860000   0x908000 .. 0x928000  128k PERIPH
  */
-const struct fw_map fw_mapping[] = {
+const struct fw_map sparrow_fw_mapping[] = {
 	/* FW code RAM 256k */
 	{0x000000, 0x040000, 0x8c0000, "fw_code", true},
 	/* FW data RAM 32k */
@@ -111,6 +113,59 @@ const struct fw_map fw_mapping[] = {
 	{0x800000, 0x804000, 0x940000, "uc_data", false},
 };
 
+/**
+ * @sparrow_d0_mac_rgf_ext - mac_rgf_ext section for Sparrow D0
+ * it is a bit larger to support extra features
+ */
+const struct fw_map sparrow_d0_mac_rgf_ext = {
+	0x88c000, 0x88c500, 0x88c000, "mac_rgf_ext", true
+};
+
+/**
+ * @talyn_fw_mapping provides memory remapping table for Talyn
+ *
+ * array size should be in sync with the declaration in the wil6210.h
+ *
+ * Talyn memory mapping:
+ * Linker address         PCI/Host address
+ *                        0x880000 .. 0xc80000  4Mb BAR0
+ * 0x800000 .. 0x820000   0xa00000 .. 0xa20000  128k DCCM
+ * 0x840000 .. 0x858000   0xa20000 .. 0xa38000  96k PERIPH
+ */
+const struct fw_map talyn_fw_mapping[] = {
+	/* FW code RAM 1M */
+	{0x000000, 0x100000, 0x900000, "fw_code", true},
+	/* FW data RAM 128k */
+	{0x800000, 0x820000, 0xa00000, "fw_data", true},
+	/* periph. data RAM 96k */
+	{0x840000, 0x858000, 0xa20000, "fw_peri", true},
+	/* various RGF 40k */
+	{0x880000, 0x88a000, 0x880000, "rgf", true},
+	/* AGC table 4k */
+	{0x88a000, 0x88b000, 0x88a000, "AGC_tbl", true},
+	/* Pcie_ext_rgf 4k */
+	{0x88b000, 0x88c000, 0x88b000, "rgf_ext", true},
+	/* mac_ext_rgf 1344b */
+	{0x88c000, 0x88c540, 0x88c000, "mac_rgf_ext", true},
+	/* ext USER RGF 4k */
+	{0x88d000, 0x88e000, 0x88d000, "ext_user_rgf", true},
+	/* OTP 4k */
+	{0x8a0000, 0x8a1000, 0x8a0000, "otp", true},
+	/* DMA EXT RGF 64k */
+	{0x8b0000, 0x8c0000, 0x8b0000, "dma_ext_rgf", true},
+	/* upper area 1536k */
+	{0x900000, 0xa80000, 0x900000, "upper", true},
+	/* UCODE areas - accessible by debugfs blobs but not by
+	 * wmi_addr_remap. UCODE areas MUST be added AFTER FW areas!
+	 */
+	/* ucode code RAM 256k */
+	{0x000000, 0x040000, 0xa38000, "uc_code", false},
+	/* ucode data RAM 32k */
+	{0x800000, 0x808000, 0xa78000, "uc_data", false},
+};
+
+struct fw_map fw_mapping[MAX_FW_MAPPING_TABLE_SIZE];
+
 struct blink_on_off_time led_blink_time[] = {
 	{WIL_LED_BLINK_ON_SLOW_MS, WIL_LED_BLINK_OFF_SLOW_MS},
 	{WIL_LED_BLINK_ON_MED_MS, WIL_LED_BLINK_OFF_MED_MS},
@@ -137,16 +192,36 @@ static u32 wmi_addr_remap(u32 x)
 	return 0;
 }
 
+/**
+ * find fw_mapping entry by section name
+ * @section - section name
+ *
+ * Return pointer to section or NULL if not found
+ */
+struct fw_map *wil_find_fw_mapping(const char *section)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(fw_mapping); i++)
+		if (fw_mapping[i].name &&
+		    !strcmp(section, fw_mapping[i].name))
+			return &fw_mapping[i];
+
+	return NULL;
+}
+
 /**
  * Check address validity for WMI buffer; remap if needed
  * @ptr - internal (linker) fw/ucode address
+ * @size - if non zero, validate the block does not
+ *  exceed the device memory (bar)
  *
  * Valid buffer should be DWORD aligned
  *
  * return address for accessing buffer from the host;
  * if buffer is not valid, return NULL.
  */
-void __iomem *wmi_buffer(struct wil6210_priv *wil, __le32 ptr_)
+void __iomem *wmi_buffer_block(struct wil6210_priv *wil, __le32 ptr_, u32 size)
 {
 	u32 off;
 	u32 ptr = le32_to_cpu(ptr_);
@@ -161,10 +236,17 @@ void __iomem *wmi_buffer(struct wil6210_priv *wil, __le32 ptr_)
 	off = HOSTADDR(ptr);
 	if (off > wil->bar_size - 4)
 		return NULL;
+	if (size && ((off + size > wil->bar_size) || (off + size < off)))
+		return NULL;
 
 	return wil->csr + off;
 }
 
+void __iomem *wmi_buffer(struct wil6210_priv *wil, __le32 ptr_)
+{
+	return wmi_buffer_block(wil, ptr_, 0);
+}
+
 /**
  * Check address validity
  */
@@ -198,6 +280,242 @@ int wmi_read_hdr(struct wil6210_priv *wil, __le32 ptr,
 	return 0;
 }
 
+static const char *cmdid2name(u16 cmdid)
+{
+	switch (cmdid) {
+	case WMI_NOTIFY_REQ_CMDID:
+		return "WMI_NOTIFY_REQ_CMD";
+	case WMI_START_SCAN_CMDID:
+		return "WMI_START_SCAN_CMD";
+	case WMI_CONNECT_CMDID:
+		return "WMI_CONNECT_CMD";
+	case WMI_DISCONNECT_CMDID:
+		return "WMI_DISCONNECT_CMD";
+	case WMI_SW_TX_REQ_CMDID:
+		return "WMI_SW_TX_REQ_CMD";
+	case WMI_GET_RF_SECTOR_PARAMS_CMDID:
+		return "WMI_GET_RF_SECTOR_PARAMS_CMD";
+	case WMI_SET_RF_SECTOR_PARAMS_CMDID:
+		return "WMI_SET_RF_SECTOR_PARAMS_CMD";
+	case WMI_GET_SELECTED_RF_SECTOR_INDEX_CMDID:
+		return "WMI_GET_SELECTED_RF_SECTOR_INDEX_CMD";
+	case WMI_SET_SELECTED_RF_SECTOR_INDEX_CMDID:
+		return "WMI_SET_SELECTED_RF_SECTOR_INDEX_CMD";
+	case WMI_BRP_SET_ANT_LIMIT_CMDID:
+		return "WMI_BRP_SET_ANT_LIMIT_CMD";
+	case WMI_TOF_SESSION_START_CMDID:
+		return "WMI_TOF_SESSION_START_CMD";
+	case WMI_AOA_MEAS_CMDID:
+		return "WMI_AOA_MEAS_CMD";
+	case WMI_PMC_CMDID:
+		return "WMI_PMC_CMD";
+	case WMI_TOF_GET_TX_RX_OFFSET_CMDID:
+		return "WMI_TOF_GET_TX_RX_OFFSET_CMD";
+	case WMI_TOF_SET_TX_RX_OFFSET_CMDID:
+		return "WMI_TOF_SET_TX_RX_OFFSET_CMD";
+	case WMI_VRING_CFG_CMDID:
+		return "WMI_VRING_CFG_CMD";
+	case WMI_BCAST_VRING_CFG_CMDID:
+		return "WMI_BCAST_VRING_CFG_CMD";
+	case WMI_TRAFFIC_SUSPEND_CMDID:
+		return "WMI_TRAFFIC_SUSPEND_CMD";
+	case WMI_TRAFFIC_RESUME_CMDID:
+		return "WMI_TRAFFIC_RESUME_CMD";
+	case WMI_ECHO_CMDID:
+		return "WMI_ECHO_CMD";
+	case WMI_SET_MAC_ADDRESS_CMDID:
+		return "WMI_SET_MAC_ADDRESS_CMD";
+	case WMI_LED_CFG_CMDID:
+		return "WMI_LED_CFG_CMD";
+	case WMI_PCP_START_CMDID:
+		return "WMI_PCP_START_CMD";
+	case WMI_PCP_STOP_CMDID:
+		return "WMI_PCP_STOP_CMD";
+	case WMI_SET_SSID_CMDID:
+		return "WMI_SET_SSID_CMD";
+	case WMI_GET_SSID_CMDID:
+		return "WMI_GET_SSID_CMD";
+	case WMI_SET_PCP_CHANNEL_CMDID:
+		return "WMI_SET_PCP_CHANNEL_CMD";
+	case WMI_GET_PCP_CHANNEL_CMDID:
+		return "WMI_GET_PCP_CHANNEL_CMD";
+	case WMI_P2P_CFG_CMDID:
+		return "WMI_P2P_CFG_CMD";
+	case WMI_START_LISTEN_CMDID:
+		return "WMI_START_LISTEN_CMD";
+	case WMI_START_SEARCH_CMDID:
+		return "WMI_START_SEARCH_CMD";
+	case WMI_DISCOVERY_STOP_CMDID:
+		return "WMI_DISCOVERY_STOP_CMD";
+	case WMI_DELETE_CIPHER_KEY_CMDID:
+		return "WMI_DELETE_CIPHER_KEY_CMD";
+	case WMI_ADD_CIPHER_KEY_CMDID:
+		return "WMI_ADD_CIPHER_KEY_CMD";
+	case WMI_SET_APPIE_CMDID:
+		return "WMI_SET_APPIE_CMD";
+	case WMI_CFG_RX_CHAIN_CMDID:
+		return "WMI_CFG_RX_CHAIN_CMD";
+	case WMI_TEMP_SENSE_CMDID:
+		return "WMI_TEMP_SENSE_CMD";
+	case WMI_DEL_STA_CMDID:
+		return "WMI_DEL_STA_CMD";
+	case WMI_DISCONNECT_STA_CMDID:
+		return "WMI_DISCONNECT_STA_CMD";
+	case WMI_VRING_BA_EN_CMDID:
+		return "WMI_VRING_BA_EN_CMD";
+	case WMI_VRING_BA_DIS_CMDID:
+		return "WMI_VRING_BA_DIS_CMD";
+	case WMI_RCP_DELBA_CMDID:
+		return "WMI_RCP_DELBA_CMD";
+	case WMI_RCP_ADDBA_RESP_CMDID:
+		return "WMI_RCP_ADDBA_RESP_CMD";
+	case WMI_PS_DEV_PROFILE_CFG_CMDID:
+		return "WMI_PS_DEV_PROFILE_CFG_CMD";
+	case WMI_SET_MGMT_RETRY_LIMIT_CMDID:
+		return "WMI_SET_MGMT_RETRY_LIMIT_CMD";
+	case WMI_GET_MGMT_RETRY_LIMIT_CMDID:
+		return "WMI_GET_MGMT_RETRY_LIMIT_CMD";
+	case WMI_ABORT_SCAN_CMDID:
+		return "WMI_ABORT_SCAN_CMD";
+	case WMI_NEW_STA_CMDID:
+		return "WMI_NEW_STA_CMD";
+	case WMI_SET_THERMAL_THROTTLING_CFG_CMDID:
+		return "WMI_SET_THERMAL_THROTTLING_CFG_CMD";
+	case WMI_GET_THERMAL_THROTTLING_CFG_CMDID:
+		return "WMI_GET_THERMAL_THROTTLING_CFG_CMD";
+	case WMI_LINK_MAINTAIN_CFG_WRITE_CMDID:
+		return "WMI_LINK_MAINTAIN_CFG_WRITE_CMD";
+	case WMI_LO_POWER_CALIB_FROM_OTP_CMDID:
+		return "WMI_LO_POWER_CALIB_FROM_OTP_CMD";
+	case WMI_START_SCHED_SCAN_CMDID:
+		return "WMI_START_SCHED_SCAN_CMD";
+	case WMI_STOP_SCHED_SCAN_CMDID:
+		return "WMI_STOP_SCHED_SCAN_CMD";
+	default:
+		return "Untracked CMD";
+	}
+}
+
+static const char *eventid2name(u16 eventid)
+{
+	switch (eventid) {
+	case WMI_NOTIFY_REQ_DONE_EVENTID:
+		return "WMI_NOTIFY_REQ_DONE_EVENT";
+	case WMI_DISCONNECT_EVENTID:
+		return "WMI_DISCONNECT_EVENT";
+	case WMI_SW_TX_COMPLETE_EVENTID:
+		return "WMI_SW_TX_COMPLETE_EVENT";
+	case WMI_GET_RF_SECTOR_PARAMS_DONE_EVENTID:
+		return "WMI_GET_RF_SECTOR_PARAMS_DONE_EVENT";
+	case WMI_SET_RF_SECTOR_PARAMS_DONE_EVENTID:
+		return "WMI_SET_RF_SECTOR_PARAMS_DONE_EVENT";
+	case WMI_GET_SELECTED_RF_SECTOR_INDEX_DONE_EVENTID:
+		return "WMI_GET_SELECTED_RF_SECTOR_INDEX_DONE_EVENT";
+	case WMI_SET_SELECTED_RF_SECTOR_INDEX_DONE_EVENTID:
+		return "WMI_SET_SELECTED_RF_SECTOR_INDEX_DONE_EVENT";
+	case WMI_BRP_SET_ANT_LIMIT_EVENTID:
+		return "WMI_BRP_SET_ANT_LIMIT_EVENT";
+	case WMI_FW_READY_EVENTID:
+		return "WMI_FW_READY_EVENT";
+	case WMI_TRAFFIC_RESUME_EVENTID:
+		return "WMI_TRAFFIC_RESUME_EVENT";
+	case WMI_TOF_GET_TX_RX_OFFSET_EVENTID:
+		return "WMI_TOF_GET_TX_RX_OFFSET_EVENT";
+	case WMI_TOF_SET_TX_RX_OFFSET_EVENTID:
+		return "WMI_TOF_SET_TX_RX_OFFSET_EVENT";
+	case WMI_VRING_CFG_DONE_EVENTID:
+		return "WMI_VRING_CFG_DONE_EVENT";
+	case WMI_READY_EVENTID:
+		return "WMI_READY_EVENT";
+	case WMI_RX_MGMT_PACKET_EVENTID:
+		return "WMI_RX_MGMT_PACKET_EVENT";
+	case WMI_TX_MGMT_PACKET_EVENTID:
+		return "WMI_TX_MGMT_PACKET_EVENT";
+	case WMI_SCAN_COMPLETE_EVENTID:
+		return "WMI_SCAN_COMPLETE_EVENT";
+	case WMI_ACS_PASSIVE_SCAN_COMPLETE_EVENTID:
+		return "WMI_ACS_PASSIVE_SCAN_COMPLETE_EVENT";
+	case WMI_CONNECT_EVENTID:
+		return "WMI_CONNECT_EVENT";
+	case WMI_EAPOL_RX_EVENTID:
+		return "WMI_EAPOL_RX_EVENT";
+	case WMI_BA_STATUS_EVENTID:
+		return "WMI_BA_STATUS_EVENT";
+	case WMI_RCP_ADDBA_REQ_EVENTID:
+		return "WMI_RCP_ADDBA_REQ_EVENT";
+	case WMI_DELBA_EVENTID:
+		return "WMI_DELBA_EVENT";
+	case WMI_VRING_EN_EVENTID:
+		return "WMI_VRING_EN_EVENT";
+	case WMI_DATA_PORT_OPEN_EVENTID:
+		return "WMI_DATA_PORT_OPEN_EVENT";
+	case WMI_AOA_MEAS_EVENTID:
+		return "WMI_AOA_MEAS_EVENT";
+	case WMI_TOF_SESSION_END_EVENTID:
+		return "WMI_TOF_SESSION_END_EVENT";
+	case WMI_TOF_GET_CAPABILITIES_EVENTID:
+		return "WMI_TOF_GET_CAPABILITIES_EVENT";
+	case WMI_TOF_SET_LCR_EVENTID:
+		return "WMI_TOF_SET_LCR_EVENT";
+	case WMI_TOF_SET_LCI_EVENTID:
+		return "WMI_TOF_SET_LCI_EVENT";
+	case WMI_TOF_FTM_PER_DEST_RES_EVENTID:
+		return "WMI_TOF_FTM_PER_DEST_RES_EVENT";
+	case WMI_TOF_CHANNEL_INFO_EVENTID:
+		return "WMI_TOF_CHANNEL_INFO_EVENT";
+	case WMI_TRAFFIC_SUSPEND_EVENTID:
+		return "WMI_TRAFFIC_SUSPEND_EVENT";
+	case WMI_ECHO_RSP_EVENTID:
+		return "WMI_ECHO_RSP_EVENT";
+	case WMI_LED_CFG_DONE_EVENTID:
+		return "WMI_LED_CFG_DONE_EVENT";
+	case WMI_PCP_STARTED_EVENTID:
+		return "WMI_PCP_STARTED_EVENT";
+	case WMI_PCP_STOPPED_EVENTID:
+		return "WMI_PCP_STOPPED_EVENT";
+	case WMI_GET_SSID_EVENTID:
+		return "WMI_GET_SSID_EVENT";
+	case WMI_GET_PCP_CHANNEL_EVENTID:
+		return "WMI_GET_PCP_CHANNEL_EVENT";
+	case WMI_P2P_CFG_DONE_EVENTID:
+		return "WMI_P2P_CFG_DONE_EVENT";
+	case WMI_LISTEN_STARTED_EVENTID:
+		return "WMI_LISTEN_STARTED_EVENT";
+	case WMI_SEARCH_STARTED_EVENTID:
+		return "WMI_SEARCH_STARTED_EVENT";
+	case WMI_DISCOVERY_STOPPED_EVENTID:
+		return "WMI_DISCOVERY_STOPPED_EVENT";
+	case WMI_CFG_RX_CHAIN_DONE_EVENTID:
+		return "WMI_CFG_RX_CHAIN_DONE_EVENT";
+	case WMI_TEMP_SENSE_DONE_EVENTID:
+		return "WMI_TEMP_SENSE_DONE_EVENT";
+	case WMI_RCP_ADDBA_RESP_SENT_EVENTID:
+		return "WMI_RCP_ADDBA_RESP_SENT_EVENT";
+	case WMI_PS_DEV_PROFILE_CFG_EVENTID:
+		return "WMI_PS_DEV_PROFILE_CFG_EVENT";
+	case WMI_SET_MGMT_RETRY_LIMIT_EVENTID:
+		return "WMI_SET_MGMT_RETRY_LIMIT_EVENT";
+	case WMI_GET_MGMT_RETRY_LIMIT_EVENTID:
+		return "WMI_GET_MGMT_RETRY_LIMIT_EVENT";
+	case WMI_SET_THERMAL_THROTTLING_CFG_EVENTID:
+		return "WMI_SET_THERMAL_THROTTLING_CFG_EVENT";
+	case WMI_GET_THERMAL_THROTTLING_CFG_EVENTID:
+		return "WMI_GET_THERMAL_THROTTLING_CFG_EVENT";
+	case WMI_LINK_MAINTAIN_CFG_WRITE_DONE_EVENTID:
+		return "WMI_LINK_MAINTAIN_CFG_WRITE_DONE_EVENT";
+	case WMI_LO_POWER_CALIB_FROM_OTP_EVENTID:
+		return "WMI_LO_POWER_CALIB_FROM_OTP_EVENT";
+	case WMI_START_SCHED_SCAN_EVENTID:
+		return "WMI_START_SCHED_SCAN_EVENT";
+	case WMI_STOP_SCHED_SCAN_EVENTID:
+		return "WMI_STOP_SCHED_SCAN_EVENT";
+	case WMI_SCHED_SCAN_RESULT_EVENTID:
+		return "WMI_SCHED_SCAN_RESULT_EVENT";
+	default:
+		return "Untracked EVENT";
+	}
+}
+
 static int __wmi_send(struct wil6210_priv *wil, u16 cmdid, void *buf, u16 len)
 {
 	struct {
@@ -222,7 +540,7 @@ static int __wmi_send(struct wil6210_priv *wil, u16 cmdid, void *buf, u16 len)
 	uint retry;
 	int rc = 0;
 
-	if (sizeof(cmd) + len > r->entry_size) {
+	if (len > r->entry_size - sizeof(cmd)) {
 		wil_err(wil, "WMI size too large: %d bytes, max is %d\n",
 			(int)(sizeof(cmd) + len), r->entry_size);
 		return -ERANGE;
@@ -294,7 +612,8 @@ static int __wmi_send(struct wil6210_priv *wil, u16 cmdid, void *buf, u16 len)
 	}
 	cmd.hdr.seq = cpu_to_le16(++wil->wmi_seq);
 	/* set command */
-	wil_dbg_wmi(wil, "WMI command 0x%04x [%d]\n", cmdid, len);
+	wil_dbg_wmi(wil, "sending %s (0x%04x) [%d]\n",
+		    cmdid2name(cmdid), cmdid, len);
 	wil_hex_dump_wmi("Cmd ", DUMP_PREFIX_OFFSET, 16, 1, &cmd,
 			 sizeof(cmd), true);
 	wil_hex_dump_wmi("cmd ", DUMP_PREFIX_OFFSET, 16, 1, buf,
@@ -566,8 +885,6 @@ static void wmi_evt_connect(struct wil6210_priv *wil, int id, void *d, int len)
 		}
 	}
 
-	/* FIXME FW can transmit only ucast frames to peer */
-	/* FIXME real ring_id instead of hard coded 0 */
 	ether_addr_copy(wil->sta[evt->cid].addr, evt->bssid);
 	wil->sta[evt->cid].status = wil_sta_conn_pending;
 
@@ -830,6 +1147,75 @@ __acquires(&sta->tid_rx_lock) __releases(&sta->tid_rx_lock)
 	spin_unlock_bh(&sta->tid_rx_lock);
 }
 
+static void
+wmi_evt_sched_scan_result(struct wil6210_priv *wil, int id, void *d, int len)
+{
+	struct wmi_sched_scan_result_event *data = d;
+	struct wiphy *wiphy = wil_to_wiphy(wil);
+	struct ieee80211_mgmt *rx_mgmt_frame =
+		(struct ieee80211_mgmt *)data->payload;
+	int flen = len - offsetof(struct wmi_sched_scan_result_event, payload);
+	int ch_no;
+	u32 freq;
+	struct ieee80211_channel *channel;
+	s32 signal;
+	__le16 fc;
+	u32 d_len;
+	struct cfg80211_bss *bss;
+
+	if (flen < 0) {
+		wil_err(wil, "sched scan result event too short, len %d\n",
+			len);
+		return;
+	}
+
+	d_len = le32_to_cpu(data->info.len);
+	if (d_len != flen) {
+		wil_err(wil,
+			"sched scan result length mismatch, d_len %d should be %d\n",
+			d_len, flen);
+		return;
+	}
+
+	fc = rx_mgmt_frame->frame_control;
+	if (!ieee80211_is_probe_resp(fc)) {
+		wil_err(wil, "sched scan result invalid frame, fc 0x%04x\n",
+			fc);
+		return;
+	}
+
+	ch_no = data->info.channel + 1;
+	freq = ieee80211_channel_to_frequency(ch_no, NL80211_BAND_60GHZ);
+	channel = ieee80211_get_channel(wiphy, freq);
+	if (test_bit(WMI_FW_CAPABILITY_RSSI_REPORTING, wil->fw_capabilities))
+		signal = 100 * data->info.rssi;
+	else
+		signal = data->info.sqi;
+
+	wil_dbg_wmi(wil, "sched scan result: channel %d MCS %d RSSI %d\n",
+		    data->info.channel, data->info.mcs, data->info.rssi);
+	wil_dbg_wmi(wil, "len %d qid %d mid %d cid %d\n",
+		    d_len, data->info.qid, data->info.mid, data->info.cid);
+	wil_hex_dump_wmi("PROBE ", DUMP_PREFIX_OFFSET, 16, 1, rx_mgmt_frame,
+			 d_len, true);
+
+	if (!channel) {
+		wil_err(wil, "Frame on unsupported channel\n");
+		return;
+	}
+
+	bss = cfg80211_inform_bss_frame(wiphy, channel, rx_mgmt_frame,
+					d_len, signal, GFP_KERNEL);
+	if (bss) {
+		wil_dbg_wmi(wil, "Added BSS %pM\n", rx_mgmt_frame->bssid);
+		cfg80211_put_bss(wiphy, bss);
+	} else {
+		wil_err(wil, "cfg80211_inform_bss_frame() failed\n");
+	}
+
+	cfg80211_sched_scan_results(wiphy, 0);
+}
+
 /**
  * Some events are ignored for purpose; and need not be interpreted as
  * "unhandled events"
@@ -857,6 +1243,7 @@ static const struct {
 	{WMI_DELBA_EVENTID,		wmi_evt_delba},
 	{WMI_VRING_EN_EVENTID,		wmi_evt_vring_en},
 	{WMI_DATA_PORT_OPEN_EVENTID,		wmi_evt_ignore},
+	{WMI_SCHED_SCAN_RESULT_EVENTID,		wmi_evt_sched_scan_result},
 };
 
 /*
@@ -963,8 +1350,8 @@ void wmi_recv_cmd(struct wil6210_priv *wil)
 			}
 			spin_unlock_irqrestore(&wil->wmi_ev_lock, flags);
 
-			wil_dbg_wmi(wil, "WMI event 0x%04x MID %d @%d msec\n",
-				    id, wmi->mid, tstamp);
+			wil_dbg_wmi(wil, "recv %s (0x%04x) MID %d @%d msec\n",
+				    eventid2name(id), id, wmi->mid, tstamp);
 			trace_wil6210_wmi_event(wmi, &wmi[1],
 						len - sizeof(*wmi));
 		}
@@ -1380,8 +1767,14 @@ int wmi_set_ie(struct wil6210_priv *wil, u8 type, u16 ie_len, const void *ie)
 	};
 	int rc;
 	u16 len = sizeof(struct wmi_set_appie_cmd) + ie_len;
-	struct wmi_set_appie_cmd *cmd = kzalloc(len, GFP_KERNEL);
+	struct wmi_set_appie_cmd *cmd;
+
+	if (len < ie_len) {
+		rc = -EINVAL;
+		goto out;
+	}
 
+	cmd = kzalloc(len, GFP_KERNEL);
 	if (!cmd) {
 		rc = -ENOMEM;
 		goto out;
@@ -1461,7 +1854,7 @@ int wmi_rx_chain_add(struct wil6210_priv *wil, struct vring *vring)
 	int rc;
 
 	if (wdev->iftype == NL80211_IFTYPE_MONITOR) {
-		struct ieee80211_channel *ch = wdev->preset_chandef.chan;
+		struct ieee80211_channel *ch = wil->monitor_chandef.chan;
 
 		cmd.sniffer_cfg.mode = cpu_to_le32(WMI_SNIFFER_ON);
 		if (ch)
@@ -1801,6 +2194,16 @@ void wmi_event_flush(struct wil6210_priv *wil)
 	spin_unlock_irqrestore(&wil->wmi_ev_lock, flags);
 }
 
+static const char *suspend_status2name(u8 status)
+{
+	switch (status) {
+	case WMI_TRAFFIC_SUSPEND_REJECTED_LINK_NOT_IDLE:
+		return "LINK_NOT_IDLE";
+	default:
+		return "Untracked status";
+	}
+}
+
 int wmi_suspend(struct wil6210_priv *wil)
 {
 	int rc;
@@ -1816,7 +2219,7 @@ int wmi_suspend(struct wil6210_priv *wil)
 	wil->suspend_resp_rcvd = false;
 	wil->suspend_resp_comp = false;
 
-	reply.evt.status = WMI_TRAFFIC_SUSPEND_REJECTED;
+	reply.evt.status = WMI_TRAFFIC_SUSPEND_REJECTED_LINK_NOT_IDLE;
 
 	rc = wmi_call(wil, WMI_TRAFFIC_SUSPEND_CMDID, &cmd, sizeof(cmd),
 		      WMI_TRAFFIC_SUSPEND_EVENTID, &reply, sizeof(reply),
@@ -1848,8 +2251,9 @@ int wmi_suspend(struct wil6210_priv *wil)
 	}
 
 	wil_dbg_wmi(wil, "suspend_response_completed rcvd\n");
-	if (reply.evt.status == WMI_TRAFFIC_SUSPEND_REJECTED) {
-		wil_dbg_pm(wil, "device rejected the suspend\n");
+	if (reply.evt.status != WMI_TRAFFIC_SUSPEND_APPROVED) {
+		wil_dbg_pm(wil, "device rejected the suspend, %s\n",
+			   suspend_status2name(reply.evt.status));
 		wil->suspend_stats.rejected_by_device++;
 	}
 	rc = reply.evt.status;
@@ -1861,21 +2265,50 @@ int wmi_suspend(struct wil6210_priv *wil)
 	return rc;
 }
 
+static void resume_triggers2string(u32 triggers, char *string, int str_size)
+{
+	string[0] = '\0';
+
+	if (!triggers) {
+		strlcat(string, " UNKNOWN", str_size);
+		return;
+	}
+
+	if (triggers & WMI_RESUME_TRIGGER_HOST)
+		strlcat(string, " HOST", str_size);
+
+	if (triggers & WMI_RESUME_TRIGGER_UCAST_RX)
+		strlcat(string, " UCAST_RX", str_size);
+
+	if (triggers & WMI_RESUME_TRIGGER_BCAST_RX)
+		strlcat(string, " BCAST_RX", str_size);
+
+	if (triggers & WMI_RESUME_TRIGGER_WMI_EVT)
+		strlcat(string, " WMI_EVT", str_size);
+}
+
 int wmi_resume(struct wil6210_priv *wil)
 {
 	int rc;
+	char string[100];
 	struct {
 		struct wmi_cmd_hdr wmi;
 		struct wmi_traffic_resume_event evt;
 	} __packed reply;
 
 	reply.evt.status = WMI_TRAFFIC_RESUME_FAILED;
+	reply.evt.resume_triggers = WMI_RESUME_TRIGGER_UNKNOWN;
 
 	rc = wmi_call(wil, WMI_TRAFFIC_RESUME_CMDID, NULL, 0,
 		      WMI_TRAFFIC_RESUME_EVENTID, &reply, sizeof(reply),
 		      WIL_WAIT_FOR_SUSPEND_RESUME_COMP);
 	if (rc)
 		return rc;
+	resume_triggers2string(le32_to_cpu(reply.evt.resume_triggers), string,
+			       sizeof(string));
+	wil_dbg_pm(wil, "device resume %s, resume triggers:%s (0x%x)\n",
+		   reply.evt.status ? "failed" : "passed", string,
+		   le32_to_cpu(reply.evt.resume_triggers));
 
 	return reply.evt.status;
 }
@@ -1906,8 +2339,8 @@ static void wmi_event_handle(struct wil6210_priv *wil,
 		void *evt_data = (void *)(&wmi[1]);
 		u16 id = le16_to_cpu(wmi->command_id);
 
-		wil_dbg_wmi(wil, "Handle WMI 0x%04x (reply_id 0x%04x)\n",
-			    id, wil->reply_id);
+		wil_dbg_wmi(wil, "Handle %s (0x%04x) (reply_id 0x%04x)\n",
+			    eventid2name(id), id, wil->reply_id);
 		/* check if someone waits for this event */
 		if (wil->reply_id && wil->reply_id == id) {
 			WARN_ON(wil->reply_buf);
@@ -2002,3 +2435,159 @@ bool wil_is_wmi_idle(struct wil6210_priv *wil)
 	spin_unlock_irqrestore(&wil->wmi_ev_lock, flags);
 	return rc;
 }
+
+static void
+wmi_sched_scan_set_ssids(struct wil6210_priv *wil,
+			 struct wmi_start_sched_scan_cmd *cmd,
+			 struct cfg80211_ssid *ssids, int n_ssids,
+			 struct cfg80211_match_set *match_sets,
+			 int n_match_sets)
+{
+	int i;
+
+	if (n_match_sets > WMI_MAX_PNO_SSID_NUM) {
+		wil_dbg_wmi(wil, "too many match sets (%d), use first %d\n",
+			    n_match_sets, WMI_MAX_PNO_SSID_NUM);
+		n_match_sets = WMI_MAX_PNO_SSID_NUM;
+	}
+	cmd->num_of_ssids = n_match_sets;
+
+	for (i = 0; i < n_match_sets; i++) {
+		struct wmi_sched_scan_ssid_match *wmi_match =
+			&cmd->ssid_for_match[i];
+		struct cfg80211_match_set *cfg_match = &match_sets[i];
+		int j;
+
+		wmi_match->ssid_len = cfg_match->ssid.ssid_len;
+		memcpy(wmi_match->ssid, cfg_match->ssid.ssid,
+		       min_t(u8, wmi_match->ssid_len, WMI_MAX_SSID_LEN));
+		wmi_match->rssi_threshold = S8_MIN;
+		if (cfg_match->rssi_thold >= S8_MIN &&
+		    cfg_match->rssi_thold <= S8_MAX)
+			wmi_match->rssi_threshold = cfg_match->rssi_thold;
+
+		for (j = 0; j < n_ssids; j++)
+			if (wmi_match->ssid_len == ssids[j].ssid_len &&
+			    memcmp(wmi_match->ssid, ssids[j].ssid,
+				   wmi_match->ssid_len) == 0)
+				wmi_match->add_ssid_to_probe = true;
+	}
+}
+
+static void
+wmi_sched_scan_set_channels(struct wil6210_priv *wil,
+			    struct wmi_start_sched_scan_cmd *cmd,
+			    u32 n_channels,
+			    struct ieee80211_channel **channels)
+{
+	int i;
+
+	if (n_channels > WMI_MAX_CHANNEL_NUM) {
+		wil_dbg_wmi(wil, "too many channels (%d), use first %d\n",
+			    n_channels, WMI_MAX_CHANNEL_NUM);
+		n_channels = WMI_MAX_CHANNEL_NUM;
+	}
+	cmd->num_of_channels = n_channels;
+
+	for (i = 0; i < n_channels; i++) {
+		struct ieee80211_channel *cfg_chan = channels[i];
+
+		cmd->channel_list[i] = cfg_chan->hw_value - 1;
+	}
+}
+
+static void
+wmi_sched_scan_set_plans(struct wil6210_priv *wil,
+			 struct wmi_start_sched_scan_cmd *cmd,
+			 struct cfg80211_sched_scan_plan *scan_plans,
+			 int n_scan_plans)
+{
+	int i;
+
+	if (n_scan_plans > WMI_MAX_PLANS_NUM) {
+		wil_dbg_wmi(wil, "too many plans (%d), use first %d\n",
+			    n_scan_plans, WMI_MAX_PLANS_NUM);
+		n_scan_plans = WMI_MAX_PLANS_NUM;
+	}
+
+	for (i = 0; i < n_scan_plans; i++) {
+		struct cfg80211_sched_scan_plan *cfg_plan = &scan_plans[i];
+
+		cmd->scan_plans[i].interval_sec =
+			cpu_to_le16(cfg_plan->interval);
+		cmd->scan_plans[i].num_of_iterations =
+			cpu_to_le16(cfg_plan->iterations);
+	}
+}
+
+int wmi_start_sched_scan(struct wil6210_priv *wil,
+			 struct cfg80211_sched_scan_request *request)
+{
+	int rc;
+	struct wmi_start_sched_scan_cmd cmd = {
+		.min_rssi_threshold = S8_MIN,
+		.initial_delay_sec = cpu_to_le16(request->delay),
+	};
+	struct {
+		struct wmi_cmd_hdr wmi;
+		struct wmi_start_sched_scan_event evt;
+	} __packed reply;
+
+	if (!test_bit(WMI_FW_CAPABILITY_PNO, wil->fw_capabilities))
+		return -ENOTSUPP;
+
+	if (request->min_rssi_thold >= S8_MIN &&
+	    request->min_rssi_thold <= S8_MAX)
+		cmd.min_rssi_threshold = request->min_rssi_thold;
+
+	wmi_sched_scan_set_ssids(wil, &cmd, request->ssids, request->n_ssids,
+				 request->match_sets, request->n_match_sets);
+	wmi_sched_scan_set_channels(wil, &cmd,
+				    request->n_channels, request->channels);
+	wmi_sched_scan_set_plans(wil, &cmd,
+				 request->scan_plans, request->n_scan_plans);
+
+	reply.evt.result = WMI_PNO_REJECT;
+
+	rc = wmi_call(wil, WMI_START_SCHED_SCAN_CMDID, &cmd, sizeof(cmd),
+		      WMI_START_SCHED_SCAN_EVENTID, &reply, sizeof(reply),
+		      WIL_WMI_CALL_GENERAL_TO_MS);
+	if (rc)
+		return rc;
+
+	if (reply.evt.result != WMI_PNO_SUCCESS) {
+		wil_err(wil, "start sched scan failed, result %d\n",
+			reply.evt.result);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int wmi_stop_sched_scan(struct wil6210_priv *wil)
+{
+	int rc;
+	struct {
+		struct wmi_cmd_hdr wmi;
+		struct wmi_stop_sched_scan_event evt;
+	} __packed reply;
+
+	if (!test_bit(WMI_FW_CAPABILITY_PNO, wil->fw_capabilities))
+		return -ENOTSUPP;
+
+	reply.evt.result = WMI_PNO_REJECT;
+
+	rc = wmi_call(wil, WMI_STOP_SCHED_SCAN_CMDID, NULL, 0,
+		      WMI_STOP_SCHED_SCAN_EVENTID, &reply, sizeof(reply),
+		      WIL_WMI_CALL_GENERAL_TO_MS);
+	if (rc)
+		return rc;
+
+	if (reply.evt.result != WMI_PNO_SUCCESS) {
+		wil_err(wil, "stop sched scan failed, result %d\n",
+			reply.evt.result);
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/wireless/ath/wil6210/wmi.h b/drivers/net/wireless/ath/wil6210/wmi.h
index 5263ee717a4f..d3e75f0ff245 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.h
+++ b/drivers/net/wireless/ath/wil6210/wmi.h
@@ -71,6 +71,8 @@ enum wmi_fw_capability {
 	WMI_FW_CAPABILITY_RSSI_REPORTING		= 12,
 	WMI_FW_CAPABILITY_SET_SILENT_RSSI_TABLE		= 13,
 	WMI_FW_CAPABILITY_LO_POWER_CALIB_FROM_OTP	= 14,
+	WMI_FW_CAPABILITY_PNO				= 15,
+	WMI_FW_CAPABILITY_REF_CLOCK_CONTROL		= 18,
 	WMI_FW_CAPABILITY_MAX,
 };
 
@@ -87,6 +89,8 @@ enum wmi_command_id {
 	WMI_CONNECT_CMDID				= 0x01,
 	WMI_DISCONNECT_CMDID				= 0x03,
 	WMI_DISCONNECT_STA_CMDID			= 0x04,
+	WMI_START_SCHED_SCAN_CMDID			= 0x05,
+	WMI_STOP_SCHED_SCAN_CMDID			= 0x06,
 	WMI_START_SCAN_CMDID				= 0x07,
 	WMI_SET_BSS_FILTER_CMDID			= 0x09,
 	WMI_SET_PROBED_SSID_CMDID			= 0x0A,
@@ -385,6 +389,38 @@ struct wmi_start_scan_cmd {
 	} channel_list[0];
 } __packed;
 
+#define WMI_MAX_PNO_SSID_NUM	(16)
+#define WMI_MAX_CHANNEL_NUM	(6)
+#define WMI_MAX_PLANS_NUM	(2)
+
+/* WMI_START_SCHED_SCAN_CMDID */
+struct wmi_sched_scan_ssid_match {
+	u8 ssid_len;
+	u8 ssid[WMI_MAX_SSID_LEN];
+	s8 rssi_threshold;
+	/* boolean */
+	u8 add_ssid_to_probe;
+	u8 reserved;
+} __packed;
+
+/* WMI_START_SCHED_SCAN_CMDID */
+struct wmi_sched_scan_plan {
+	__le16 interval_sec;
+	__le16 num_of_iterations;
+} __packed;
+
+/* WMI_START_SCHED_SCAN_CMDID */
+struct wmi_start_sched_scan_cmd {
+	struct wmi_sched_scan_ssid_match ssid_for_match[WMI_MAX_PNO_SSID_NUM];
+	u8 num_of_ssids;
+	s8 min_rssi_threshold;
+	u8 channel_list[WMI_MAX_CHANNEL_NUM];
+	u8 num_of_channels;
+	u8 reserved;
+	__le16 initial_delay_sec;
+	struct wmi_sched_scan_plan scan_plans[WMI_MAX_PLANS_NUM];
+} __packed;
+
 /* WMI_SET_PROBED_SSID_CMDID */
 #define MAX_PROBED_SSID_INDEX	(3)
 
@@ -1238,6 +1274,9 @@ enum wmi_event_id {
 	WMI_READY_EVENTID				= 0x1001,
 	WMI_CONNECT_EVENTID				= 0x1002,
 	WMI_DISCONNECT_EVENTID				= 0x1003,
+	WMI_START_SCHED_SCAN_EVENTID			= 0x1005,
+	WMI_STOP_SCHED_SCAN_EVENTID			= 0x1006,
+	WMI_SCHED_SCAN_RESULT_EVENTID			= 0x1007,
 	WMI_SCAN_COMPLETE_EVENTID			= 0x100A,
 	WMI_REPORT_STATISTICS_EVENTID			= 0x100B,
 	WMI_RD_MEM_RSP_EVENTID				= 0x1800,
@@ -1600,6 +1639,49 @@ struct wmi_scan_complete_event {
 	__le32 status;
 } __packed;
 
+/* wmi_rx_mgmt_info */
+struct wmi_rx_mgmt_info {
+	u8 mcs;
+	s8 rssi;
+	u8 range;
+	u8 sqi;
+	__le16 stype;
+	__le16 status;
+	__le32 len;
+	/* Not resolved when == 0xFFFFFFFF == > Broadcast to all MIDS */
+	u8 qid;
+	/* Not resolved when == 0xFFFFFFFF == > Broadcast to all MIDS */
+	u8 mid;
+	u8 cid;
+	/* From Radio MNGR */
+	u8 channel;
+} __packed;
+
+/* WMI_START_SCHED_SCAN_EVENTID */
+enum wmi_pno_result {
+	WMI_PNO_SUCCESS			= 0x00,
+	WMI_PNO_REJECT			= 0x01,
+	WMI_PNO_INVALID_PARAMETERS	= 0x02,
+	WMI_PNO_NOT_ENABLED		= 0x03,
+};
+
+struct wmi_start_sched_scan_event {
+	/* pno_result */
+	u8 result;
+	u8 reserved[3];
+} __packed;
+
+struct wmi_stop_sched_scan_event {
+	/* pno_result */
+	u8 result;
+	u8 reserved[3];
+} __packed;
+
+struct wmi_sched_scan_result_event {
+	struct wmi_rx_mgmt_info info;
+	u8 payload[0];
+} __packed;
+
 /* WMI_ACS_PASSIVE_SCAN_COMPLETE_EVENT */
 enum wmi_acs_info_bitmask {
 	WMI_ACS_INFO_BITMASK_BEACON_FOUND	= 0x01,
@@ -1814,24 +1896,6 @@ struct wmi_get_ssid_event {
 	u8 ssid[WMI_MAX_SSID_LEN];
 } __packed;
 
-/* wmi_rx_mgmt_info */
-struct wmi_rx_mgmt_info {
-	u8 mcs;
-	s8 rssi;
-	u8 range;
-	u8 sqi;
-	__le16 stype;
-	__le16 status;
-	__le32 len;
-	/* Not resolved when == 0xFFFFFFFF == > Broadcast to all MIDS */
-	u8 qid;
-	/* Not resolved when == 0xFFFFFFFF == > Broadcast to all MIDS */
-	u8 mid;
-	u8 cid;
-	/* From Radio MNGR */
-	u8 channel;
-} __packed;
-
 /* EVENT: WMI_RF_XPM_READ_RESULT_EVENTID */
 struct wmi_rf_xpm_read_result_event {
 	/* enum wmi_fw_status_e - success=0 or fail=1 */
@@ -2267,8 +2331,8 @@ struct wmi_link_maintain_cfg_read_done_event {
 } __packed;
 
 enum wmi_traffic_suspend_status {
-	WMI_TRAFFIC_SUSPEND_APPROVED	= 0x0,
-	WMI_TRAFFIC_SUSPEND_REJECTED	= 0x1,
+	WMI_TRAFFIC_SUSPEND_APPROVED			= 0x0,
+	WMI_TRAFFIC_SUSPEND_REJECTED_LINK_NOT_IDLE	= 0x1,
 };
 
 /* WMI_TRAFFIC_SUSPEND_EVENTID */
@@ -2282,10 +2346,21 @@ enum wmi_traffic_resume_status {
 	WMI_TRAFFIC_RESUME_FAILED	= 0x1,
 };
 
+enum wmi_resume_trigger {
+	WMI_RESUME_TRIGGER_UNKNOWN	= 0x0,
+	WMI_RESUME_TRIGGER_HOST		= 0x1,
+	WMI_RESUME_TRIGGER_UCAST_RX	= 0x2,
+	WMI_RESUME_TRIGGER_BCAST_RX	= 0x4,
+	WMI_RESUME_TRIGGER_WMI_EVT	= 0x8,
+};
+
 /* WMI_TRAFFIC_RESUME_EVENTID */
 struct wmi_traffic_resume_event {
-	/* enum wmi_traffic_resume_status_e */
+	/* enum wmi_traffic_resume_status */
 	u8 status;
+	u8 reserved[3];
+	/* enum wmi_resume_trigger bitmap */
+	__le32 resume_triggers;
 } __packed;
 
 /* Power Save command completion status codes */
diff --git a/drivers/net/wireless/broadcom/b43/phy_n.c b/drivers/net/wireless/broadcom/b43/phy_n.c
index a5557d70689f..f2a2f41e3c96 100644
--- a/drivers/net/wireless/broadcom/b43/phy_n.c
+++ b/drivers/net/wireless/broadcom/b43/phy_n.c
@@ -1031,7 +1031,7 @@ static void b43_radio_2057_init_post(struct b43_wldev *dev)
 
 	b43_radio_set(dev, R2057_RFPLL_MISC_CAL_RESETN, 0x78);
 	b43_radio_set(dev, R2057_XTAL_CONFIG2, 0x80);
-	mdelay(2);
+	usleep_range(2000, 3000);
 	b43_radio_mask(dev, R2057_RFPLL_MISC_CAL_RESETN, ~0x78);
 	b43_radio_mask(dev, R2057_XTAL_CONFIG2, ~0x80);
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
index 9f2d0b0cf6e5..2d3a5dd07a3f 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
@@ -165,7 +165,7 @@ static int brcmf_proto_bcdc_cmplt(struct brcmf_pub *drvr, u32 id, u32 len)
 
 static int
 brcmf_proto_bcdc_query_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd,
-			    void *buf, uint len)
+			    void *buf, uint len, int *fwerr)
 {
 	struct brcmf_bcdc *bcdc = (struct brcmf_bcdc *)drvr->proto->pd;
 	struct brcmf_proto_bcdc_dcmd *msg = &bcdc->msg;
@@ -175,6 +175,7 @@ brcmf_proto_bcdc_query_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd,
 
 	brcmf_dbg(BCDC, "Enter, cmd %d len %d\n", cmd, len);
 
+	*fwerr = 0;
 	ret = brcmf_proto_bcdc_msg(drvr, ifidx, cmd, buf, len, false);
 	if (ret < 0) {
 		brcmf_err("brcmf_proto_bcdc_msg failed w/status %d\n",
@@ -211,25 +212,27 @@ brcmf_proto_bcdc_query_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd,
 		memcpy(buf, info, len);
 	}
 
+	ret = 0;
+
 	/* Check the ERROR flag */
 	if (flags & BCDC_DCMD_ERROR)
-		ret = le32_to_cpu(msg->status);
-
+		*fwerr = le32_to_cpu(msg->status);
 done:
 	return ret;
 }
 
 static int
 brcmf_proto_bcdc_set_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd,
-			  void *buf, uint len)
+			  void *buf, uint len, int *fwerr)
 {
 	struct brcmf_bcdc *bcdc = (struct brcmf_bcdc *)drvr->proto->pd;
 	struct brcmf_proto_bcdc_dcmd *msg = &bcdc->msg;
-	int ret = 0;
+	int ret;
 	u32 flags, id;
 
 	brcmf_dbg(BCDC, "Enter, cmd %d len %d\n", cmd, len);
 
+	*fwerr = 0;
 	ret = brcmf_proto_bcdc_msg(drvr, ifidx, cmd, buf, len, true);
 	if (ret < 0)
 		goto done;
@@ -249,9 +252,11 @@ brcmf_proto_bcdc_set_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd,
 		goto done;
 	}
 
+	ret = 0;
+
 	/* Check the ERROR flag */
 	if (flags & BCDC_DCMD_ERROR)
-		ret = le32_to_cpu(msg->status);
+		*fwerr = le32_to_cpu(msg->status);
 
 done:
 	return ret;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
index cd587325e286..0b68240ec7b4 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
@@ -118,7 +118,7 @@ int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev)
 
 		ret = request_irq(pdata->oob_irq_nr, brcmf_sdiod_oob_irqhandler,
 				  pdata->oob_irq_flags, "brcmf_oob_intr",
-				  &sdiodev->func[1]->dev);
+				  &sdiodev->func1->dev);
 		if (ret != 0) {
 			brcmf_err("request_irq failed %d\n", ret);
 			return ret;
@@ -132,39 +132,40 @@ int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev)
 		}
 		sdiodev->irq_wake = true;
 
-		sdio_claim_host(sdiodev->func[1]);
+		sdio_claim_host(sdiodev->func1);
 
 		if (sdiodev->bus_if->chip == BRCM_CC_43362_CHIP_ID) {
 			/* assign GPIO to SDIO core */
 			addr = CORE_CC_REG(SI_ENUM_BASE, gpiocontrol);
-			gpiocontrol = brcmf_sdiod_regrl(sdiodev, addr, &ret);
+			gpiocontrol = brcmf_sdiod_readl(sdiodev, addr, &ret);
 			gpiocontrol |= 0x2;
-			brcmf_sdiod_regwl(sdiodev, addr, gpiocontrol, &ret);
+			brcmf_sdiod_writel(sdiodev, addr, gpiocontrol, &ret);
 
-			brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_SELECT, 0xf,
-					  &ret);
-			brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_OUT, 0, &ret);
-			brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_EN, 0x2, &ret);
+			brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_SELECT,
+					   0xf, &ret);
+			brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_OUT, 0, &ret);
+			brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_EN, 0x2, &ret);
 		}
 
 		/* must configure SDIO_CCCR_IENx to enable irq */
-		data = brcmf_sdiod_regrb(sdiodev, SDIO_CCCR_IENx, &ret);
-		data |= 1 << SDIO_FUNC_1 | 1 << SDIO_FUNC_2 | 1;
-		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_IENx, data, &ret);
+		data = brcmf_sdiod_func0_rb(sdiodev, SDIO_CCCR_IENx, &ret);
+		data |= SDIO_CCCR_IEN_FUNC1 | SDIO_CCCR_IEN_FUNC2 |
+			SDIO_CCCR_IEN_FUNC0;
+		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_IENx, data, &ret);
 
 		/* redirect, configure and enable io for interrupt signal */
-		data = SDIO_SEPINT_MASK | SDIO_SEPINT_OE;
+		data = SDIO_CCCR_BRCM_SEPINT_MASK | SDIO_CCCR_BRCM_SEPINT_OE;
 		if (pdata->oob_irq_flags & IRQF_TRIGGER_HIGH)
-			data |= SDIO_SEPINT_ACT_HI;
-		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_SEPINT, data, &ret);
-
-		sdio_release_host(sdiodev->func[1]);
+			data |= SDIO_CCCR_BRCM_SEPINT_ACT_HI;
+		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_SEPINT,
+				     data, &ret);
+		sdio_release_host(sdiodev->func1);
 	} else {
 		brcmf_dbg(SDIO, "Entering\n");
-		sdio_claim_host(sdiodev->func[1]);
-		sdio_claim_irq(sdiodev->func[1], brcmf_sdiod_ib_irqhandler);
-		sdio_claim_irq(sdiodev->func[2], brcmf_sdiod_dummy_irqhandler);
-		sdio_release_host(sdiodev->func[1]);
+		sdio_claim_host(sdiodev->func1);
+		sdio_claim_irq(sdiodev->func1, brcmf_sdiod_ib_irqhandler);
+		sdio_claim_irq(sdiodev->func2, brcmf_sdiod_dummy_irqhandler);
+		sdio_release_host(sdiodev->func1);
 		sdiodev->sd_irq_requested = true;
 	}
 
@@ -182,26 +183,26 @@ void brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev)
 		struct brcmfmac_sdio_pd *pdata;
 
 		pdata = &sdiodev->settings->bus.sdio;
-		sdio_claim_host(sdiodev->func[1]);
-		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_SEPINT, 0, NULL);
-		brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_IENx, 0, NULL);
-		sdio_release_host(sdiodev->func[1]);
+		sdio_claim_host(sdiodev->func1);
+		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_SEPINT, 0, NULL);
+		brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_IENx, 0, NULL);
+		sdio_release_host(sdiodev->func1);
 
 		sdiodev->oob_irq_requested = false;
 		if (sdiodev->irq_wake) {
 			disable_irq_wake(pdata->oob_irq_nr);
 			sdiodev->irq_wake = false;
 		}
-		free_irq(pdata->oob_irq_nr, &sdiodev->func[1]->dev);
+		free_irq(pdata->oob_irq_nr, &sdiodev->func1->dev);
 		sdiodev->irq_en = false;
 		sdiodev->oob_irq_requested = false;
 	}
 
 	if (sdiodev->sd_irq_requested) {
-		sdio_claim_host(sdiodev->func[1]);
-		sdio_release_irq(sdiodev->func[2]);
-		sdio_release_irq(sdiodev->func[1]);
-		sdio_release_host(sdiodev->func[1]);
+		sdio_claim_host(sdiodev->func1);
+		sdio_release_irq(sdiodev->func2);
+		sdio_release_irq(sdiodev->func1);
+		sdio_release_host(sdiodev->func1);
 		sdiodev->sd_irq_requested = false;
 	}
 }
@@ -230,271 +231,121 @@ void brcmf_sdiod_change_state(struct brcmf_sdio_dev *sdiodev,
 	sdiodev->state = state;
 }
 
-static inline int brcmf_sdiod_f0_writeb(struct sdio_func *func,
-					uint regaddr, u8 byte)
-{
-	int err_ret;
-
-	/*
-	 * Can only directly write to some F0 registers.
-	 * Handle CCCR_IENx and CCCR_ABORT command
-	 * as a special case.
-	 */
-	if ((regaddr == SDIO_CCCR_ABORT) ||
-	    (regaddr == SDIO_CCCR_IENx))
-		sdio_writeb(func, byte, regaddr, &err_ret);
-	else
-		sdio_f0_writeb(func, byte, regaddr, &err_ret);
-
-	return err_ret;
-}
-
-static int brcmf_sdiod_request_data(struct brcmf_sdio_dev *sdiodev, u8 fn,
-				    u32 addr, u8 regsz, void *data, bool write)
-{
-	struct sdio_func *func;
-	int ret = -EINVAL;
-
-	brcmf_dbg(SDIO, "rw=%d, func=%d, addr=0x%05x, nbytes=%d\n",
-		  write, fn, addr, regsz);
-
-	/* only allow byte access on F0 */
-	if (WARN_ON(regsz > 1 && !fn))
-		return -EINVAL;
-	func = sdiodev->func[fn];
-
-	switch (regsz) {
-	case sizeof(u8):
-		if (write) {
-			if (fn)
-				sdio_writeb(func, *(u8 *)data, addr, &ret);
-			else
-				ret = brcmf_sdiod_f0_writeb(func, addr,
-							    *(u8 *)data);
-		} else {
-			if (fn)
-				*(u8 *)data = sdio_readb(func, addr, &ret);
-			else
-				*(u8 *)data = sdio_f0_readb(func, addr, &ret);
-		}
-		break;
-	case sizeof(u16):
-		if (write)
-			sdio_writew(func, *(u16 *)data, addr, &ret);
-		else
-			*(u16 *)data = sdio_readw(func, addr, &ret);
-		break;
-	case sizeof(u32):
-		if (write)
-			sdio_writel(func, *(u32 *)data, addr, &ret);
-		else
-			*(u32 *)data = sdio_readl(func, addr, &ret);
-		break;
-	default:
-		brcmf_err("invalid size: %d\n", regsz);
-		break;
-	}
-
-	if (ret)
-		brcmf_dbg(SDIO, "failed to %s data F%d@0x%05x, err: %d\n",
-			  write ? "write" : "read", fn, addr, ret);
-
-	return ret;
-}
-
-static int brcmf_sdiod_regrw_helper(struct brcmf_sdio_dev *sdiodev, u32 addr,
-				   u8 regsz, void *data, bool write)
-{
-	u8 func;
-	s32 retry = 0;
-	int ret;
-
-	if (sdiodev->state == BRCMF_SDIOD_NOMEDIUM)
-		return -ENOMEDIUM;
-
-	/*
-	 * figure out how to read the register based on address range
-	 * 0x00 ~ 0x7FF: function 0 CCCR and FBR
-	 * 0x10000 ~ 0x1FFFF: function 1 miscellaneous registers
-	 * The rest: function 1 silicon backplane core registers
-	 */
-	if ((addr & ~REG_F0_REG_MASK) == 0)
-		func = SDIO_FUNC_0;
-	else
-		func = SDIO_FUNC_1;
-
-	do {
-		if (!write)
-			memset(data, 0, regsz);
-		/* for retry wait for 1 ms till bus get settled down */
-		if (retry)
-			usleep_range(1000, 2000);
-		ret = brcmf_sdiod_request_data(sdiodev, func, addr, regsz,
-					       data, write);
-	} while (ret != 0 && ret != -ENOMEDIUM &&
-		 retry++ < SDIOH_API_ACCESS_RETRY_LIMIT);
-
-	if (ret == -ENOMEDIUM)
-		brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM);
-	else if (ret != 0) {
-		/*
-		 * SleepCSR register access can fail when
-		 * waking up the device so reduce this noise
-		 * in the logs.
-		 */
-		if (addr != SBSDIO_FUNC1_SLEEPCSR)
-			brcmf_err("failed to %s data F%d@0x%05x, err: %d\n",
-				  write ? "write" : "read", func, addr, ret);
-		else
-			brcmf_dbg(SDIO, "failed to %s data F%d@0x%05x, err: %d\n",
-				  write ? "write" : "read", func, addr, ret);
-	}
-	return ret;
-}
-
-static int
-brcmf_sdiod_set_sbaddr_window(struct brcmf_sdio_dev *sdiodev, u32 address)
+static int brcmf_sdiod_set_backplane_window(struct brcmf_sdio_dev *sdiodev,
+					    u32 addr)
 {
+	u32 v, bar0 = addr & SBSDIO_SBWINDOW_MASK;
 	int err = 0, i;
-	u8 addr[3];
-
-	if (sdiodev->state == BRCMF_SDIOD_NOMEDIUM)
-		return -ENOMEDIUM;
-
-	addr[0] = (address >> 8) & SBSDIO_SBADDRLOW_MASK;
-	addr[1] = (address >> 16) & SBSDIO_SBADDRMID_MASK;
-	addr[2] = (address >> 24) & SBSDIO_SBADDRHIGH_MASK;
-
-	for (i = 0; i < 3; i++) {
-		err = brcmf_sdiod_regrw_helper(sdiodev,
-					       SBSDIO_FUNC1_SBADDRLOW + i,
-					       sizeof(u8), &addr[i], true);
-		if (err) {
-			brcmf_err("failed at addr: 0x%0x\n",
-				  SBSDIO_FUNC1_SBADDRLOW + i);
-			break;
-		}
-	}
 
-	return err;
-}
+	if (bar0 == sdiodev->sbwad)
+		return 0;
 
-static int
-brcmf_sdiod_addrprep(struct brcmf_sdio_dev *sdiodev, uint width, u32 *addr)
-{
-	uint bar0 = *addr & ~SBSDIO_SB_OFT_ADDR_MASK;
-	int err = 0;
+	v = bar0 >> 8;
 
-	if (bar0 != sdiodev->sbwad) {
-		err = brcmf_sdiod_set_sbaddr_window(sdiodev, bar0);
-		if (err)
-			return err;
+	for (i = 0 ; i < 3 && !err ; i++, v >>= 8)
+		brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_SBADDRLOW + i,
+				   v & 0xff, &err);
 
+	if (!err)
 		sdiodev->sbwad = bar0;
-	}
-
-	*addr &= SBSDIO_SB_OFT_ADDR_MASK;
 
-	if (width == 4)
-		*addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
-
-	return 0;
+	return err;
 }
 
-u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
+u32 brcmf_sdiod_readl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
 {
-	u8 data;
+	u32 data = 0;
 	int retval;
 
-	brcmf_dbg(SDIO, "addr:0x%08x\n", addr);
-	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
-					  false);
-	brcmf_dbg(SDIO, "data:0x%02x\n", data);
+	retval = brcmf_sdiod_set_backplane_window(sdiodev, addr);
+	if (retval)
+		goto out;
+
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	data = sdio_readl(sdiodev->func1, addr, &retval);
 
+out:
 	if (ret)
 		*ret = retval;
 
 	return data;
 }
 
-u32 brcmf_sdiod_regrl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret)
+void brcmf_sdiod_writel(struct brcmf_sdio_dev *sdiodev, u32 addr,
+			u32 data, int *ret)
 {
-	u32 data = 0;
 	int retval;
 
-	brcmf_dbg(SDIO, "addr:0x%08x\n", addr);
-	retval = brcmf_sdiod_addrprep(sdiodev, sizeof(data), &addr);
+	retval = brcmf_sdiod_set_backplane_window(sdiodev, addr);
 	if (retval)
-		goto done;
-	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
-					  false);
-	brcmf_dbg(SDIO, "data:0x%08x\n", data);
-
-done:
-	if (ret)
-		*ret = retval;
+		goto out;
 
-	return data;
-}
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
 
-void brcmf_sdiod_regwb(struct brcmf_sdio_dev *sdiodev, u32 addr,
-		      u8 data, int *ret)
-{
-	int retval;
+	sdio_writel(sdiodev->func1, data, addr, &retval);
 
-	brcmf_dbg(SDIO, "addr:0x%08x, data:0x%02x\n", addr, data);
-	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
-					  true);
+out:
 	if (ret)
 		*ret = retval;
 }
 
-void brcmf_sdiod_regwl(struct brcmf_sdio_dev *sdiodev, u32 addr,
-		      u32 data, int *ret)
+static int brcmf_sdiod_skbuff_read(struct brcmf_sdio_dev *sdiodev,
+				   struct sdio_func *func, u32 addr,
+				   struct sk_buff *skb)
 {
-	int retval;
+	unsigned int req_sz;
+	int err;
 
-	brcmf_dbg(SDIO, "addr:0x%08x, data:0x%08x\n", addr, data);
-	retval = brcmf_sdiod_addrprep(sdiodev, sizeof(data), &addr);
-	if (retval)
-		goto done;
-	retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data,
-					  true);
+	/* Single skb use the standard mmc interface */
+	req_sz = skb->len + 3;
+	req_sz &= (uint)~3;
 
-done:
-	if (ret)
-		*ret = retval;
+	switch (func->num) {
+	case 1:
+		err = sdio_memcpy_fromio(func, ((u8 *)(skb->data)), addr,
+					 req_sz);
+		break;
+	case 2:
+		err = sdio_readsb(func, ((u8 *)(skb->data)), addr, req_sz);
+		break;
+	default:
+		/* bail out as things are really fishy here */
+		WARN(1, "invalid sdio function number: %d\n", func->num);
+		err = -ENOMEDIUM;
+	};
+
+	if (err == -ENOMEDIUM)
+		brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM);
+
+	return err;
 }
 
-static int brcmf_sdiod_buffrw(struct brcmf_sdio_dev *sdiodev, uint fn,
-			     bool write, u32 addr, struct sk_buff *pkt)
+static int brcmf_sdiod_skbuff_write(struct brcmf_sdio_dev *sdiodev,
+				    struct sdio_func *func, u32 addr,
+				    struct sk_buff *skb)
 {
 	unsigned int req_sz;
 	int err;
 
 	/* Single skb use the standard mmc interface */
-	req_sz = pkt->len + 3;
+	req_sz = skb->len + 3;
 	req_sz &= (uint)~3;
 
-	if (write)
-		err = sdio_memcpy_toio(sdiodev->func[fn], addr,
-				       ((u8 *)(pkt->data)), req_sz);
-	else if (fn == 1)
-		err = sdio_memcpy_fromio(sdiodev->func[fn], ((u8 *)(pkt->data)),
-					 addr, req_sz);
-	else
-		/* function 2 read is FIFO operation */
-		err = sdio_readsb(sdiodev->func[fn], ((u8 *)(pkt->data)), addr,
-				  req_sz);
+	err = sdio_memcpy_toio(func, addr, ((u8 *)(skb->data)), req_sz);
+
 	if (err == -ENOMEDIUM)
 		brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM);
+
 	return err;
 }
 
 /**
  * brcmf_sdiod_sglist_rw - SDIO interface function for block data access
  * @sdiodev: brcmfmac sdio device
- * @fn: SDIO function number
+ * @func: SDIO function
  * @write: direction flag
  * @addr: dongle memory address as source/destination
  * @pkt: skb pointer
@@ -503,7 +354,8 @@ static int brcmf_sdiod_buffrw(struct brcmf_sdio_dev *sdiodev, uint fn,
  * stack for block data access. It assumes that the skb passed down by the
  * caller has already been padded and aligned.
  */
-static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, uint fn,
+static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev,
+				 struct sdio_func *func,
 				 bool write, u32 addr,
 				 struct sk_buff_head *pktlist)
 {
@@ -529,7 +381,7 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, uint fn,
 		req_sz = 0;
 		skb_queue_walk(pktlist, pkt_next)
 			req_sz += pkt_next->len;
-		req_sz = ALIGN(req_sz, sdiodev->func[fn]->cur_blksize);
+		req_sz = ALIGN(req_sz, func->cur_blksize);
 		while (req_sz > PAGE_SIZE) {
 			pkt_next = brcmu_pkt_buf_get_skb(PAGE_SIZE);
 			if (pkt_next == NULL) {
@@ -548,7 +400,7 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, uint fn,
 		target_list = &local_list;
 	}
 
-	func_blk_sz = sdiodev->func[fn]->cur_blksize;
+	func_blk_sz = func->cur_blksize;
 	max_req_sz = sdiodev->max_request_size;
 	max_seg_cnt = min_t(unsigned short, sdiodev->max_segment_count,
 			    target_list->qlen);
@@ -565,10 +417,10 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, uint fn,
 	mmc_dat.flags = write ? MMC_DATA_WRITE : MMC_DATA_READ;
 	mmc_cmd.opcode = SD_IO_RW_EXTENDED;
 	mmc_cmd.arg = write ? 1<<31 : 0;	/* write flag  */
-	mmc_cmd.arg |= (fn & 0x7) << 28;	/* SDIO func num */
-	mmc_cmd.arg |= 1<<27;			/* block mode */
+	mmc_cmd.arg |= (func->num & 0x7) << 28;	/* SDIO func num */
+	mmc_cmd.arg |= 1 << 27;			/* block mode */
 	/* for function 1 the addr will be incremented */
-	mmc_cmd.arg |= (fn == 1) ? 1<<26 : 0;
+	mmc_cmd.arg |= (func->num == 1) ? 1 << 26 : 0;
 	mmc_cmd.flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_ADTC;
 	mmc_req.cmd = &mmc_cmd;
 	mmc_req.data = &mmc_dat;
@@ -614,11 +466,11 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, uint fn,
 		mmc_cmd.arg |= (addr & 0x1FFFF) << 9;	/* address */
 		mmc_cmd.arg |= mmc_dat.blocks & 0x1FF;	/* block count */
 		/* incrementing addr for function 1 */
-		if (fn == 1)
+		if (func->num == 1)
 			addr += req_sz;
 
-		mmc_set_data_timeout(&mmc_dat, sdiodev->func[fn]->card);
-		mmc_wait_for_req(sdiodev->func[fn]->card->host, &mmc_req);
+		mmc_set_data_timeout(&mmc_dat, func->card);
+		mmc_wait_for_req(func->card->host, &mmc_req);
 
 		ret = mmc_cmd.error ? mmc_cmd.error : mmc_dat.error;
 		if (ret == -ENOMEDIUM) {
@@ -686,16 +538,19 @@ int brcmf_sdiod_recv_buf(struct brcmf_sdio_dev *sdiodev, u8 *buf, uint nbytes)
 
 int brcmf_sdiod_recv_pkt(struct brcmf_sdio_dev *sdiodev, struct sk_buff *pkt)
 {
-	u32 addr = sdiodev->sbwad;
+	u32 addr = sdiodev->cc_core->base;
 	int err = 0;
 
 	brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n", addr, pkt->len);
 
-	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
+	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
 	if (err)
 		goto done;
 
-	err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr, pkt);
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr, pkt);
 
 done:
 	return err;
@@ -706,25 +561,28 @@ int brcmf_sdiod_recv_chain(struct brcmf_sdio_dev *sdiodev,
 {
 	struct sk_buff *glom_skb = NULL;
 	struct sk_buff *skb;
-	u32 addr = sdiodev->sbwad;
+	u32 addr = sdiodev->cc_core->base;
 	int err = 0;
 
 	brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n",
 		  addr, pktq->qlen);
 
-	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
+	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
 	if (err)
 		goto done;
 
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
 	if (pktq->qlen == 1)
-		err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr,
-					 pktq->next);
+		err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr,
+					      pktq->next);
 	else if (!sdiodev->sg_support) {
 		glom_skb = brcmu_pkt_buf_get_skb(totlen);
 		if (!glom_skb)
 			return -ENOMEM;
-		err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr,
-					 glom_skb);
+		err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr,
+					      glom_skb);
 		if (err)
 			goto done;
 
@@ -733,8 +591,8 @@ int brcmf_sdiod_recv_chain(struct brcmf_sdio_dev *sdiodev,
 			skb_pull(glom_skb, skb->len);
 		}
 	} else
-		err = brcmf_sdiod_sglist_rw(sdiodev, SDIO_FUNC_2, false, addr,
-					    pktq);
+		err = brcmf_sdiod_sglist_rw(sdiodev, sdiodev->func2, false,
+					    addr, pktq);
 
 done:
 	brcmu_pkt_buf_free_skb(glom_skb);
@@ -744,10 +602,11 @@ int brcmf_sdiod_recv_chain(struct brcmf_sdio_dev *sdiodev,
 int brcmf_sdiod_send_buf(struct brcmf_sdio_dev *sdiodev, u8 *buf, uint nbytes)
 {
 	struct sk_buff *mypkt;
-	u32 addr = sdiodev->sbwad;
+	u32 addr = sdiodev->cc_core->base;
 	int err;
 
 	mypkt = brcmu_pkt_buf_get_skb(nbytes);
+
 	if (!mypkt) {
 		brcmf_err("brcmu_pkt_buf_get_skb failed: len %d\n",
 			  nbytes);
@@ -756,40 +615,49 @@ int brcmf_sdiod_send_buf(struct brcmf_sdio_dev *sdiodev, u8 *buf, uint nbytes)
 
 	memcpy(mypkt->data, buf, nbytes);
 
-	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
+	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
+	if (err)
+		return err;
+
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
 
 	if (!err)
-		err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, true, addr,
-					 mypkt);
+		err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func2, addr,
+					       mypkt);
 
 	brcmu_pkt_buf_free_skb(mypkt);
-	return err;
 
+	return err;
 }
 
 int brcmf_sdiod_send_pkt(struct brcmf_sdio_dev *sdiodev,
 			 struct sk_buff_head *pktq)
 {
 	struct sk_buff *skb;
-	u32 addr = sdiodev->sbwad;
+	u32 addr = sdiodev->cc_core->base;
 	int err;
 
 	brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n", addr, pktq->qlen);
 
-	err = brcmf_sdiod_addrprep(sdiodev, 4, &addr);
+	err = brcmf_sdiod_set_backplane_window(sdiodev, addr);
 	if (err)
 		return err;
 
-	if (pktq->qlen == 1 || !sdiodev->sg_support)
+	addr &= SBSDIO_SB_OFT_ADDR_MASK;
+	addr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
+
+	if (pktq->qlen == 1 || !sdiodev->sg_support) {
 		skb_queue_walk(pktq, skb) {
-			err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, true,
-						 addr, skb);
+			err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func2,
+						       addr, skb);
 			if (err)
 				break;
 		}
-	else
-		err = brcmf_sdiod_sglist_rw(sdiodev, SDIO_FUNC_2, true, addr,
-					    pktq);
+	} else {
+		err = brcmf_sdiod_sglist_rw(sdiodev, sdiodev->func2, true,
+					    addr, pktq);
+	}
 
 	return err;
 }
@@ -798,7 +666,7 @@ int
 brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address,
 		  u8 *data, uint size)
 {
-	int bcmerror = 0;
+	int err = 0;
 	struct sk_buff *pkt;
 	u32 sdaddr;
 	uint dsize;
@@ -818,13 +686,13 @@ brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address,
 	else
 		dsize = size;
 
-	sdio_claim_host(sdiodev->func[1]);
+	sdio_claim_host(sdiodev->func1);
 
 	/* Do the transfer(s) */
 	while (size) {
 		/* Set the backplane window to include the start address */
-		bcmerror = brcmf_sdiod_set_sbaddr_window(sdiodev, address);
-		if (bcmerror)
+		err = brcmf_sdiod_set_backplane_window(sdiodev, address);
+		if (err)
 			break;
 
 		brcmf_dbg(SDIO, "%s %d bytes at offset 0x%08x in window 0x%08x\n",
@@ -835,11 +703,17 @@ brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address,
 		sdaddr |= SBSDIO_SB_ACCESS_2_4B_FLAG;
 
 		skb_put(pkt, dsize);
-		if (write)
+
+		if (write) {
 			memcpy(pkt->data, data, dsize);
-		bcmerror = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_1, write,
-					      sdaddr, pkt);
-		if (bcmerror) {
+			err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func1,
+						       sdaddr, pkt);
+		} else {
+			err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func1,
+						      sdaddr, pkt);
+		}
+
+		if (err) {
 			brcmf_err("membytes transfer failed\n");
 			break;
 		}
@@ -859,24 +733,17 @@ brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address,
 
 	dev_kfree_skb(pkt);
 
-	/* Return the window to backplane enumeration space for core access */
-	if (brcmf_sdiod_set_sbaddr_window(sdiodev, sdiodev->sbwad))
-		brcmf_err("FAILED to set window back to 0x%x\n",
-			  sdiodev->sbwad);
+	sdio_release_host(sdiodev->func1);
 
-	sdio_release_host(sdiodev->func[1]);
-
-	return bcmerror;
+	return err;
 }
 
-int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, uint fn)
+int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, struct sdio_func *func)
 {
-	char t_func = (char)fn;
 	brcmf_dbg(SDIO, "Enter\n");
 
-	/* issue abort cmd52 command through F0 */
-	brcmf_sdiod_request_data(sdiodev, SDIO_FUNC_0, SDIO_CCCR_ABORT,
-				 sizeof(t_func), &t_func, true);
+	/* Issue abort cmd52 command through F0 */
+	brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_ABORT, func->num, NULL);
 
 	brcmf_dbg(SDIO, "Exit\n");
 	return 0;
@@ -890,7 +757,7 @@ void brcmf_sdiod_sgtable_alloc(struct brcmf_sdio_dev *sdiodev)
 	uint nents;
 	int err;
 
-	func = sdiodev->func[2];
+	func = sdiodev->func2;
 	host = func->card->host;
 	sdiodev->sg_support = host->max_segs > 1;
 	max_blocks = min_t(uint, host->max_blk_count, 511u);
@@ -951,17 +818,17 @@ static int brcmf_sdiod_freezer_on(struct brcmf_sdio_dev *sdiodev)
 	brcmf_sdio_trigger_dpc(sdiodev->bus);
 	wait_event(sdiodev->freezer->thread_freeze,
 		   atomic_read(expect) == sdiodev->freezer->frozen_count);
-	sdio_claim_host(sdiodev->func[1]);
+	sdio_claim_host(sdiodev->func1);
 	res = brcmf_sdio_sleep(sdiodev->bus, true);
-	sdio_release_host(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func1);
 	return res;
 }
 
 static void brcmf_sdiod_freezer_off(struct brcmf_sdio_dev *sdiodev)
 {
-	sdio_claim_host(sdiodev->func[1]);
+	sdio_claim_host(sdiodev->func1);
 	brcmf_sdio_sleep(sdiodev->bus, false);
-	sdio_release_host(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func1);
 	atomic_set(&sdiodev->freezer->freezing, 0);
 	complete_all(&sdiodev->freezer->resumed);
 }
@@ -1011,19 +878,19 @@ static int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev)
 	brcmf_sdiod_freezer_detach(sdiodev);
 
 	/* Disable Function 2 */
-	sdio_claim_host(sdiodev->func[2]);
-	sdio_disable_func(sdiodev->func[2]);
-	sdio_release_host(sdiodev->func[2]);
+	sdio_claim_host(sdiodev->func2);
+	sdio_disable_func(sdiodev->func2);
+	sdio_release_host(sdiodev->func2);
 
 	/* Disable Function 1 */
-	sdio_claim_host(sdiodev->func[1]);
-	sdio_disable_func(sdiodev->func[1]);
-	sdio_release_host(sdiodev->func[1]);
+	sdio_claim_host(sdiodev->func1);
+	sdio_disable_func(sdiodev->func1);
+	sdio_release_host(sdiodev->func1);
 
 	sg_free_table(&sdiodev->sgtable);
 	sdiodev->sbwad = 0;
 
-	pm_runtime_allow(sdiodev->func[1]->card->host->parent);
+	pm_runtime_allow(sdiodev->func1->card->host->parent);
 	return 0;
 }
 
@@ -1039,29 +906,27 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev)
 {
 	int ret = 0;
 
-	sdiodev->num_funcs = 2;
+	sdio_claim_host(sdiodev->func1);
 
-	sdio_claim_host(sdiodev->func[1]);
-
-	ret = sdio_set_block_size(sdiodev->func[1], SDIO_FUNC1_BLOCKSIZE);
+	ret = sdio_set_block_size(sdiodev->func1, SDIO_FUNC1_BLOCKSIZE);
 	if (ret) {
 		brcmf_err("Failed to set F1 blocksize\n");
-		sdio_release_host(sdiodev->func[1]);
+		sdio_release_host(sdiodev->func1);
 		goto out;
 	}
-	ret = sdio_set_block_size(sdiodev->func[2], SDIO_FUNC2_BLOCKSIZE);
+	ret = sdio_set_block_size(sdiodev->func2, SDIO_FUNC2_BLOCKSIZE);
 	if (ret) {
 		brcmf_err("Failed to set F2 blocksize\n");
-		sdio_release_host(sdiodev->func[1]);
+		sdio_release_host(sdiodev->func1);
 		goto out;
 	}
 
 	/* increase F2 timeout */
-	sdiodev->func[2]->enable_timeout = SDIO_WAIT_F2RDY;
+	sdiodev->func2->enable_timeout = SDIO_WAIT_F2RDY;
 
 	/* Enable Function 1 */
-	ret = sdio_enable_func(sdiodev->func[1]);
-	sdio_release_host(sdiodev->func[1]);
+	ret = sdio_enable_func(sdiodev->func1);
+	sdio_release_host(sdiodev->func1);
 	if (ret) {
 		brcmf_err("Failed to enable F1: err=%d\n", ret);
 		goto out;
@@ -1077,7 +942,7 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev)
 		ret = -ENODEV;
 		goto out;
 	}
-	brcmf_sdiod_host_fixup(sdiodev->func[2]->card->host);
+	brcmf_sdiod_host_fixup(sdiodev->func2->card->host);
 out:
 	if (ret)
 		brcmf_sdiod_remove(sdiodev);
@@ -1138,6 +1003,10 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func,
 	brcmf_dbg(SDIO, "Function#: %d\n", func->num);
 
 	dev = &func->dev;
+
+	/* Set MMC_QUIRK_LENIENT_FN0 for this card */
+	func->card->quirks |= MMC_QUIRK_LENIENT_FN0;
+
 	/* prohibit ACPI power management for this device */
 	brcmf_sdiod_acpi_set_power_manageable(dev, 0);
 
@@ -1161,17 +1030,15 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func,
 	/* store refs to functions used. mmc_card does
 	 * not hold the F0 function pointer.
 	 */
-	sdiodev->func[0] = kmemdup(func, sizeof(*func), GFP_KERNEL);
-	sdiodev->func[0]->num = 0;
-	sdiodev->func[1] = func->card->sdio_func[0];
-	sdiodev->func[2] = func;
+	sdiodev->func1 = func->card->sdio_func[0];
+	sdiodev->func2 = func;
 
 	sdiodev->bus_if = bus_if;
 	bus_if->bus_priv.sdio = sdiodev;
 	bus_if->proto_type = BRCMF_PROTO_BCDC;
 	dev_set_drvdata(&func->dev, bus_if);
-	dev_set_drvdata(&sdiodev->func[1]->dev, bus_if);
-	sdiodev->dev = &sdiodev->func[1]->dev;
+	dev_set_drvdata(&sdiodev->func1->dev, bus_if);
+	sdiodev->dev = &sdiodev->func1->dev;
 
 	brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_DOWN);
 
@@ -1187,8 +1054,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func,
 
 fail:
 	dev_set_drvdata(&func->dev, NULL);
-	dev_set_drvdata(&sdiodev->func[1]->dev, NULL);
-	kfree(sdiodev->func[0]);
+	dev_set_drvdata(&sdiodev->func1->dev, NULL);
 	kfree(sdiodev);
 	kfree(bus_if);
 	return err;
@@ -1217,11 +1083,10 @@ static void brcmf_ops_sdio_remove(struct sdio_func *func)
 		/* only proceed with rest of cleanup if func 1 */
 		brcmf_sdiod_remove(sdiodev);
 
-		dev_set_drvdata(&sdiodev->func[1]->dev, NULL);
-		dev_set_drvdata(&sdiodev->func[2]->dev, NULL);
+		dev_set_drvdata(&sdiodev->func1->dev, NULL);
+		dev_set_drvdata(&sdiodev->func2->dev, NULL);
 
 		kfree(bus_if);
-		kfree(sdiodev->func[0]);
 		kfree(sdiodev);
 	}
 
@@ -1247,7 +1112,7 @@ static int brcmf_ops_sdio_suspend(struct device *dev)
 
 	func = container_of(dev, struct sdio_func, dev);
 	brcmf_dbg(SDIO, "Enter: F%d\n", func->num);
-	if (func->num != SDIO_FUNC_1)
+	if (func->num != 1)
 		return 0;
 
 
@@ -1264,7 +1129,7 @@ static int brcmf_ops_sdio_suspend(struct device *dev)
 		else
 			sdio_flags |= MMC_PM_WAKE_SDIO_IRQ;
 	}
-	if (sdio_set_host_pm_flags(sdiodev->func[1], sdio_flags))
+	if (sdio_set_host_pm_flags(sdiodev->func1, sdio_flags))
 		brcmf_err("Failed to set pm_flags %x\n", sdio_flags);
 	return 0;
 }
@@ -1276,7 +1141,7 @@ static int brcmf_ops_sdio_resume(struct device *dev)
 	struct sdio_func *func = container_of(dev, struct sdio_func, dev);
 
 	brcmf_dbg(SDIO, "Enter: F%d\n", func->num);
-	if (func->num != SDIO_FUNC_2)
+	if (func->num != 2)
 		return 0;
 
 	brcmf_sdiod_freezer_off(sdiodev);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index 15fa00d79fc6..1ad97a40940d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -6802,7 +6802,7 @@ static void brcmf_cfg80211_reg_notifier(struct wiphy *wiphy,
 		return;
 
 	/* ignore non-ISO3166 country codes */
-	for (i = 0; i < sizeof(req->alpha2); i++)
+	for (i = 0; i < 2; i++)
 		if (req->alpha2[i] < 'A' || req->alpha2[i] > 'Z') {
 			brcmf_err("not an ISO3166 code (0x%02x 0x%02x)\n",
 				  req->alpha2[0], req->alpha2[1]);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
index c5d1a1cbf601..f7b30ce2300d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
@@ -1338,6 +1338,7 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub)
 	switch (pub->chip) {
 	case BRCM_CC_4354_CHIP_ID:
 	case BRCM_CC_4356_CHIP_ID:
+	case BRCM_CC_4345_CHIP_ID:
 		/* explicitly check SR engine enable bit */
 		pmu_cc3_mask = BIT(2);
 		/* fall-through */
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
index 9be0b051066a..ebe35e6a40e7 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
@@ -75,6 +75,10 @@ static int brcmf_roamoff;
 module_param_named(roamoff, brcmf_roamoff, int, S_IRUSR);
 MODULE_PARM_DESC(roamoff, "Do not use internal roaming engine");
 
+static int brcmf_iapp_enable;
+module_param_named(iapp, brcmf_iapp_enable, int, 0);
+MODULE_PARM_DESC(iapp, "Enable partial support for the obsoleted Inter-Access Point Protocol");
+
 #ifdef DEBUG
 /* always succeed brcmf_bus_started() */
 static int brcmf_ignore_probe_fail;
@@ -441,6 +445,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev,
 	settings->feature_disable = brcmf_feature_disable;
 	settings->fcmode = brcmf_fcmode;
 	settings->roamoff = !!brcmf_roamoff;
+	settings->iapp = !!brcmf_iapp_enable;
 #ifdef DEBUG
 	settings->ignore_probe_fail = !!brcmf_ignore_probe_fail;
 #endif
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
index a62f8e70b320..ef914619e8e1 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
@@ -58,6 +58,7 @@ struct brcmf_mp_device {
 	unsigned int	feature_disable;
 	int		fcmode;
 	bool		roamoff;
+	bool		iapp;
 	bool		ignore_probe_fail;
 	struct brcmfmac_pd_cc *country_codes;
 	union {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
index 930e423f83a8..44b7774ebd02 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
@@ -230,6 +230,37 @@ static void brcmf_netdev_set_multicast_list(struct net_device *ndev)
 	schedule_work(&ifp->multicast_work);
 }
 
+/**
+ * brcmf_skb_is_iapp - checks if skb is an IAPP packet
+ *
+ * @skb: skb to check
+ */
+static bool brcmf_skb_is_iapp(struct sk_buff *skb)
+{
+	static const u8 iapp_l2_update_packet[6] __aligned(2) = {
+		0x00, 0x01, 0xaf, 0x81, 0x01, 0x00,
+	};
+	unsigned char *eth_data;
+#if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
+	const u16 *a, *b;
+#endif
+
+	if (skb->len - skb->mac_len != 6 ||
+	    !is_multicast_ether_addr(eth_hdr(skb)->h_dest))
+		return false;
+
+	eth_data = skb_mac_header(skb) + ETH_HLEN;
+#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
+	return !(((*(const u32 *)eth_data) ^ (*(const u32 *)iapp_l2_update_packet)) |
+		 ((*(const u16 *)(eth_data + 4)) ^ (*(const u16 *)(iapp_l2_update_packet + 4))));
+#else
+	a = (const u16 *)eth_data;
+	b = (const u16 *)iapp_l2_update_packet;
+
+	return !((a[0] ^ b[0]) | (a[1] ^ b[1]) | (a[2] ^ b[2]));
+#endif
+}
+
 static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb,
 					   struct net_device *ndev)
 {
@@ -250,6 +281,23 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb,
 		goto done;
 	}
 
+	/* Some recent Broadcom's firmwares disassociate STA when they receive
+	 * an 802.11f ADD frame. This behavior can lead to a local DoS security
+	 * issue. Attacker may trigger disassociation of any STA by sending a
+	 * proper Ethernet frame to the wireless interface.
+	 *
+	 * Moreover this feature may break AP interfaces in some specific
+	 * setups. This applies e.g. to the bridge with hairpin mode enabled and
+	 * IFLA_BRPORT_MCAST_TO_UCAST set. IAPP packet generated by a firmware
+	 * will get passed back to the wireless interface and cause immediate
+	 * disassociation of a just-connected STA.
+	 */
+	if (!drvr->settings->iapp && brcmf_skb_is_iapp(skb)) {
+		dev_kfree_skb(skb);
+		ret = -EINVAL;
+		goto done;
+	}
+
 	/* Make sure there's enough writeable headroom */
 	if (skb_headroom(skb) < drvr->hdrlen || skb_header_cloned(skb)) {
 		head_delta = max_t(int, drvr->hdrlen - skb_headroom(skb), 0);
@@ -325,6 +373,15 @@ void brcmf_txflowblock_if(struct brcmf_if *ifp,
 
 void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb)
 {
+	/* Most of Broadcom's firmwares send 802.11f ADD frame every time a new
+	 * STA connects to the AP interface. This is an obsoleted standard most
+	 * users don't use, so don't pass these frames up unless requested.
+	 */
+	if (!ifp->drvr->settings->iapp && brcmf_skb_is_iapp(skb)) {
+		brcmu_pkt_buf_free_skb(skb);
+		return;
+	}
+
 	if (skb->pkt_type == PACKET_MULTICAST)
 		ifp->ndev->stats.multicast++;
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
index df8a1ecb9924..232dcbb83311 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h
@@ -181,6 +181,7 @@ enum brcmf_netif_stop_reason {
  * @netif_stop_lock: spinlock for update netif_stop from multiple sources.
  * @pend_8021x_cnt: tracks outstanding number of 802.1x frames.
  * @pend_8021x_wait: used for signalling change in count.
+ * @fwil_fwerr: flag indicating fwil layer should return firmware error codes.
  */
 struct brcmf_if {
 	struct brcmf_pub *drvr;
@@ -198,6 +199,7 @@ struct brcmf_if {
 	wait_queue_head_t pend_8021x_wait;
 	struct in6_addr ipv6_addr_tbl[NDOL_MAX_ENTRIES];
 	u8 ipv6addr_idx;
+	bool fwil_fwerr;
 };
 
 int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
index 53ae30259989..bede7b7fd996 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
@@ -104,6 +104,9 @@ static void brcmf_feat_iovar_int_get(struct brcmf_if *ifp,
 	u32 data;
 	int err;
 
+	/* we need to know firmware error */
+	ifp->fwil_fwerr = true;
+
 	err = brcmf_fil_iovar_int_get(ifp, name, &data);
 	if (err == 0) {
 		brcmf_dbg(INFO, "enabling feature: %s\n", brcmf_feat_names[id]);
@@ -112,6 +115,8 @@ static void brcmf_feat_iovar_int_get(struct brcmf_if *ifp,
 		brcmf_dbg(TRACE, "%s feature check failed: %d\n",
 			  brcmf_feat_names[id], err);
 	}
+
+	ifp->fwil_fwerr = false;
 }
 
 static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp,
@@ -120,6 +125,9 @@ static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp,
 {
 	int err;
 
+	/* we need to know firmware error */
+	ifp->fwil_fwerr = true;
+
 	err = brcmf_fil_iovar_data_set(ifp, name, data, len);
 	if (err != -BRCMF_FW_UNSUPPORTED) {
 		brcmf_dbg(INFO, "enabling feature: %s\n", brcmf_feat_names[id]);
@@ -128,15 +136,23 @@ static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp,
 		brcmf_dbg(TRACE, "%s feature check failed: %d\n",
 			  brcmf_feat_names[id], err);
 	}
+
+	ifp->fwil_fwerr = false;
 }
 
+#define MAX_CAPS_BUFFER_SIZE	512
 static void brcmf_feat_firmware_capabilities(struct brcmf_if *ifp)
 {
-	char caps[256];
+	char caps[MAX_CAPS_BUFFER_SIZE];
 	enum brcmf_feat_id id;
-	int i;
+	int i, err;
+
+	err = brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps));
+	if (err) {
+		brcmf_err("could not get firmware cap (%d)\n", err);
+		return;
+	}
 
-	brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps));
 	brcmf_dbg(INFO, "[ %s]\n", caps);
 
 	for (i = 0; i < ARRAY_SIZE(brcmf_fwcap_map); i++) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c
index f6a2df94dba7..fc5751116d99 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c
@@ -107,7 +107,7 @@ static s32
 brcmf_fil_cmd_data(struct brcmf_if *ifp, u32 cmd, void *data, u32 len, bool set)
 {
 	struct brcmf_pub *drvr = ifp->drvr;
-	s32 err;
+	s32 err, fwerr;
 
 	if (drvr->bus_if->state != BRCMF_BUS_UP) {
 		brcmf_err("bus is down. we have nothing to do.\n");
@@ -117,15 +117,22 @@ brcmf_fil_cmd_data(struct brcmf_if *ifp, u32 cmd, void *data, u32 len, bool set)
 	if (data != NULL)
 		len = min_t(uint, len, BRCMF_DCMD_MAXLEN);
 	if (set)
-		err = brcmf_proto_set_dcmd(drvr, ifp->ifidx, cmd, data, len);
+		err = brcmf_proto_set_dcmd(drvr, ifp->ifidx, cmd,
+					   data, len, &fwerr);
 	else
-		err = brcmf_proto_query_dcmd(drvr, ifp->ifidx, cmd, data, len);
-
-	if (err >= 0)
-		return 0;
-
-	brcmf_dbg(FIL, "Failed: %s (%d)\n",
-		  brcmf_fil_get_errstr((u32)(-err)), err);
+		err = brcmf_proto_query_dcmd(drvr, ifp->ifidx, cmd,
+					     data, len, &fwerr);
+
+	if (err) {
+		brcmf_dbg(FIL, "Failed: %s (%d)\n",
+			  brcmf_fil_get_errstr((u32)(-err)), err);
+	} else if (fwerr < 0) {
+		brcmf_dbg(FIL, "Firmware error: %s (%d)\n",
+			  brcmf_fil_get_errstr((u32)(-fwerr)), fwerr);
+		err = -EBADE;
+	}
+	if (ifp->fwil_fwerr)
+		return fwerr;
 
 	return err;
 }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c
index d2c834c3b2fc..e212a791a072 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c
@@ -477,7 +477,7 @@ static void brcmf_msgbuf_ioctl_resp_wake(struct brcmf_msgbuf *msgbuf)
 
 
 static int brcmf_msgbuf_query_dcmd(struct brcmf_pub *drvr, int ifidx,
-				   uint cmd, void *buf, uint len)
+				   uint cmd, void *buf, uint len, int *fwerr)
 {
 	struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd;
 	struct sk_buff *skb = NULL;
@@ -485,6 +485,7 @@ static int brcmf_msgbuf_query_dcmd(struct brcmf_pub *drvr, int ifidx,
 	int err;
 
 	brcmf_dbg(MSGBUF, "ifidx=%d, cmd=%d, len=%d\n", ifidx, cmd, len);
+	*fwerr = 0;
 	msgbuf->ctl_completed = false;
 	err = brcmf_msgbuf_tx_ioctl(drvr, ifidx, cmd, buf, len);
 	if (err)
@@ -508,14 +509,15 @@ static int brcmf_msgbuf_query_dcmd(struct brcmf_pub *drvr, int ifidx,
 	}
 	brcmu_pkt_buf_free_skb(skb);
 
-	return msgbuf->ioctl_resp_status;
+	*fwerr = msgbuf->ioctl_resp_status;
+	return 0;
 }
 
 
 static int brcmf_msgbuf_set_dcmd(struct brcmf_pub *drvr, int ifidx,
-				 uint cmd, void *buf, uint len)
+				 uint cmd, void *buf, uint len, int *fwerr)
 {
-	return brcmf_msgbuf_query_dcmd(drvr, ifidx, cmd, buf, len);
+	return brcmf_msgbuf_query_dcmd(drvr, ifidx, cmd, buf, len, fwerr);
 }
 
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
index 2ee54133efa1..82064e909784 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
@@ -462,25 +462,23 @@ static int brcmf_p2p_set_firmware(struct brcmf_if *ifp, u8 *p2p_mac)
  * @dev_addr: optional device address.
  *
  * P2P needs mac addresses for P2P device and interface. If no device
- * address it specified, these are derived from the primary net device, ie.
- * the permanent ethernet address of the device.
+ * address it specified, these are derived from a random ethernet
+ * address.
  */
 static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr)
 {
-	struct brcmf_if *pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp;
-	bool local_admin = false;
+	bool random_addr = false;
 
-	if (!dev_addr || is_zero_ether_addr(dev_addr)) {
-		dev_addr = pri_ifp->mac_addr;
-		local_admin = true;
-	}
+	if (!dev_addr || is_zero_ether_addr(dev_addr))
+		random_addr = true;
 
-	/* Generate the P2P Device Address.  This consists of the device's
-	 * primary MAC address with the locally administered bit set.
+	/* Generate the P2P Device Address obtaining a random ethernet
+	 * address with the locally administered bit set.
 	 */
-	memcpy(p2p->dev_addr, dev_addr, ETH_ALEN);
-	if (local_admin)
-		p2p->dev_addr[0] |= 0x02;
+	if (random_addr)
+		eth_random_addr(p2p->dev_addr);
+	else
+		memcpy(p2p->dev_addr, dev_addr, ETH_ALEN);
 
 	/* Generate the P2P Interface Address.  If the discovery and connection
 	 * BSSCFGs need to simultaneously co-exist, then this address must be
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
index 3c87157f5b85..8752707557bf 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
@@ -1251,14 +1251,14 @@ static int brcmf_pcie_init_scratchbuffers(struct brcmf_pciedev_info *devinfo)
 	u64 address;
 	u32 addr;
 
-	devinfo->shared.scratch = dma_alloc_coherent(&devinfo->pdev->dev,
-		BRCMF_DMA_D2H_SCRATCH_BUF_LEN,
-		&devinfo->shared.scratch_dmahandle, GFP_KERNEL);
+	devinfo->shared.scratch =
+		dma_zalloc_coherent(&devinfo->pdev->dev,
+					BRCMF_DMA_D2H_SCRATCH_BUF_LEN,
+					&devinfo->shared.scratch_dmahandle,
+					GFP_KERNEL);
 	if (!devinfo->shared.scratch)
 		goto fail;
 
-	memset(devinfo->shared.scratch, 0, BRCMF_DMA_D2H_SCRATCH_BUF_LEN);
-
 	addr = devinfo->shared.tcm_base_address +
 	       BRCMF_SHARED_DMA_SCRATCH_ADDR_OFFSET;
 	address = (u64)devinfo->shared.scratch_dmahandle;
@@ -1268,14 +1268,14 @@ static int brcmf_pcie_init_scratchbuffers(struct brcmf_pciedev_info *devinfo)
 	       BRCMF_SHARED_DMA_SCRATCH_LEN_OFFSET;
 	brcmf_pcie_write_tcm32(devinfo, addr, BRCMF_DMA_D2H_SCRATCH_BUF_LEN);
 
-	devinfo->shared.ringupd = dma_alloc_coherent(&devinfo->pdev->dev,
-		BRCMF_DMA_D2H_RINGUPD_BUF_LEN,
-		&devinfo->shared.ringupd_dmahandle, GFP_KERNEL);
+	devinfo->shared.ringupd =
+		dma_zalloc_coherent(&devinfo->pdev->dev,
+					BRCMF_DMA_D2H_RINGUPD_BUF_LEN,
+					&devinfo->shared.ringupd_dmahandle,
+					GFP_KERNEL);
 	if (!devinfo->shared.ringupd)
 		goto fail;
 
-	memset(devinfo->shared.ringupd, 0, BRCMF_DMA_D2H_RINGUPD_BUF_LEN);
-
 	addr = devinfo->shared.tcm_base_address +
 	       BRCMF_SHARED_DMA_RINGUPD_ADDR_OFFSET;
 	address = (u64)devinfo->shared.ringupd_dmahandle;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h
index 2404f8a7c31c..8a8e08f09ea0 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h
@@ -30,9 +30,9 @@ struct brcmf_proto {
 	int (*hdrpull)(struct brcmf_pub *drvr, bool do_fws,
 		       struct sk_buff *skb, struct brcmf_if **ifp);
 	int (*query_dcmd)(struct brcmf_pub *drvr, int ifidx, uint cmd,
-			  void *buf, uint len);
+			  void *buf, uint len, int *fwerr);
 	int (*set_dcmd)(struct brcmf_pub *drvr, int ifidx, uint cmd, void *buf,
-			uint len);
+			uint len, int *fwerr);
 	int (*tx_queue_data)(struct brcmf_pub *drvr, int ifidx,
 			     struct sk_buff *skb);
 	int (*txdata)(struct brcmf_pub *drvr, int ifidx, u8 offset,
@@ -71,14 +71,16 @@ static inline int brcmf_proto_hdrpull(struct brcmf_pub *drvr, bool do_fws,
 	return drvr->proto->hdrpull(drvr, do_fws, skb, ifp);
 }
 static inline int brcmf_proto_query_dcmd(struct brcmf_pub *drvr, int ifidx,
-					 uint cmd, void *buf, uint len)
+					 uint cmd, void *buf, uint len,
+					 int *fwerr)
 {
-	return drvr->proto->query_dcmd(drvr, ifidx, cmd, buf, len);
+	return drvr->proto->query_dcmd(drvr, ifidx, cmd, buf, len,fwerr);
 }
 static inline int brcmf_proto_set_dcmd(struct brcmf_pub *drvr, int ifidx,
-				       uint cmd, void *buf, uint len)
+				       uint cmd, void *buf, uint len,
+				       int *fwerr)
 {
-	return drvr->proto->set_dcmd(drvr, ifidx, cmd, buf, len);
+	return drvr->proto->set_dcmd(drvr, ifidx, cmd, buf, len, fwerr);
 }
 
 static inline int brcmf_proto_tx_queue_data(struct brcmf_pub *drvr, int ifidx,
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index cdf9e4161592..08686147b59d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -159,8 +159,8 @@ struct rte_console {
 /* manfid tuple length, include tuple, link bytes */
 #define SBSDIO_CIS_MANFID_TUPLE_LEN	6
 
-#define CORE_BUS_REG(base, field) \
-		(base + offsetof(struct sdpcmd_regs, field))
+#define SD_REG(field) \
+		(offsetof(struct sdpcmd_regs, field))
 
 /* SDIO function 1 register CHIPCLKCSR */
 /* Force ALP request to backplane */
@@ -436,6 +436,7 @@ struct brcmf_sdio_count {
 struct brcmf_sdio {
 	struct brcmf_sdio_dev *sdiodev;	/* sdio device handler */
 	struct brcmf_chip *ci;	/* Chip info struct */
+	struct brcmf_core *sdio_core; /* sdio core info struct */
 
 	u32 hostintmask;	/* Copy of Host Interrupt Mask */
 	atomic_t intstatus;	/* Intstatus bits (events) pending */
@@ -659,32 +660,6 @@ static bool data_ok(struct brcmf_sdio *bus)
 	       ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0;
 }
 
-/*
- * Reads a register in the SDIO hardware block. This block occupies a series of
- * adresses on the 32 bit backplane bus.
- */
-static int r_sdreg32(struct brcmf_sdio *bus, u32 *regvar, u32 offset)
-{
-	struct brcmf_core *core;
-	int ret;
-
-	core = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
-	*regvar = brcmf_sdiod_regrl(bus->sdiodev, core->base + offset, &ret);
-
-	return ret;
-}
-
-static int w_sdreg32(struct brcmf_sdio *bus, u32 regval, u32 reg_offset)
-{
-	struct brcmf_core *core;
-	int ret;
-
-	core = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
-	brcmf_sdiod_regwl(bus->sdiodev, core->base + reg_offset, regval, &ret);
-
-	return ret;
-}
-
 static int
 brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on)
 {
@@ -697,8 +672,7 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on)
 
 	wr_val = (on << SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT);
 	/* 1st KSO write goes to AOS wake up core if device is asleep  */
-	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
-			  wr_val, &err);
+	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val, &err);
 
 	if (on) {
 		/* device WAKEUP through KSO:
@@ -724,7 +698,7 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on)
 		 * just one write attempt may fail,
 		 * read it back until it matches written value
 		 */
-		rd_val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
+		rd_val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
 					   &err);
 		if (!err) {
 			if ((rd_val & bmask) == cmp_val)
@@ -734,9 +708,11 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on)
 		/* bail out upon subsequent access errors */
 		if (err && (err_cnt++ > BRCMF_SDIO_MAX_ACCESS_ERRORS))
 			break;
+
 		udelay(KSO_WAIT_US);
-		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
-				  wr_val, &err);
+		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val,
+				   &err);
+
 	} while (try_cnt++ < MAX_KSO_ATTEMPTS);
 
 	if (try_cnt > 2)
@@ -772,15 +748,15 @@ static int brcmf_sdio_htclk(struct brcmf_sdio *bus, bool on, bool pendok)
 		clkreq =
 		    bus->alp_only ? SBSDIO_ALP_AVAIL_REQ : SBSDIO_HT_AVAIL_REQ;
 
-		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-				  clkreq, &err);
+		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+				   clkreq, &err);
 		if (err) {
 			brcmf_err("HT Avail request error: %d\n", err);
 			return -EBADE;
 		}
 
 		/* Check current status */
-		clkctl = brcmf_sdiod_regrb(bus->sdiodev,
+		clkctl = brcmf_sdiod_readb(bus->sdiodev,
 					   SBSDIO_FUNC1_CHIPCLKCSR, &err);
 		if (err) {
 			brcmf_err("HT Avail read error: %d\n", err);
@@ -790,35 +766,34 @@ static int brcmf_sdio_htclk(struct brcmf_sdio *bus, bool on, bool pendok)
 		/* Go to pending and await interrupt if appropriate */
 		if (!SBSDIO_CLKAV(clkctl, bus->alp_only) && pendok) {
 			/* Allow only clock-available interrupt */
-			devctl = brcmf_sdiod_regrb(bus->sdiodev,
+			devctl = brcmf_sdiod_readb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			if (err) {
-				brcmf_err("Devctl error setting CA: %d\n",
-					  err);
+				brcmf_err("Devctl error setting CA: %d\n", err);
 				return -EBADE;
 			}
 
 			devctl |= SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					  devctl, &err);
+			brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					   devctl, &err);
 			brcmf_dbg(SDIO, "CLKCTL: set PENDING\n");
 			bus->clkstate = CLK_PENDING;
 
 			return 0;
 		} else if (bus->clkstate == CLK_PENDING) {
 			/* Cancel CA-only interrupt filter */
-			devctl = brcmf_sdiod_regrb(bus->sdiodev,
+			devctl = brcmf_sdiod_readb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					  devctl, &err);
+			brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					   devctl, &err);
 		}
 
 		/* Otherwise, wait here (polling) for HT Avail */
 		timeout = jiffies +
 			  msecs_to_jiffies(PMU_MAX_TRANSITION_DLY/1000);
 		while (!SBSDIO_CLKAV(clkctl, bus->alp_only)) {
-			clkctl = brcmf_sdiod_regrb(bus->sdiodev,
+			clkctl = brcmf_sdiod_readb(bus->sdiodev,
 						   SBSDIO_FUNC1_CHIPCLKCSR,
 						   &err);
 			if (time_after(jiffies, timeout))
@@ -852,16 +827,16 @@ static int brcmf_sdio_htclk(struct brcmf_sdio *bus, bool on, bool pendok)
 
 		if (bus->clkstate == CLK_PENDING) {
 			/* Cancel CA-only interrupt filter */
-			devctl = brcmf_sdiod_regrb(bus->sdiodev,
+			devctl = brcmf_sdiod_readb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					  devctl, &err);
+			brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					   devctl, &err);
 		}
 
 		bus->clkstate = CLK_SDONLY;
-		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-				  clkreq, &err);
+		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+				   clkreq, &err);
 		brcmf_dbg(SDIO, "CLKCTL: turned OFF\n");
 		if (err) {
 			brcmf_err("Failed access turning clock off: %d\n",
@@ -951,14 +926,14 @@ brcmf_sdio_bus_sleep(struct brcmf_sdio *bus, bool sleep, bool pendok)
 
 		/* Going to sleep */
 		if (sleep) {
-			clkcsr = brcmf_sdiod_regrb(bus->sdiodev,
+			clkcsr = brcmf_sdiod_readb(bus->sdiodev,
 						   SBSDIO_FUNC1_CHIPCLKCSR,
 						   &err);
 			if ((clkcsr & SBSDIO_CSR_MASK) == 0) {
 				brcmf_dbg(SDIO, "no clock, set ALP\n");
-				brcmf_sdiod_regwb(bus->sdiodev,
-						  SBSDIO_FUNC1_CHIPCLKCSR,
-						  SBSDIO_ALP_AVAIL_REQ, &err);
+				brcmf_sdiod_writeb(bus->sdiodev,
+						   SBSDIO_FUNC1_CHIPCLKCSR,
+						   SBSDIO_ALP_AVAIL_REQ, &err);
 			}
 			err = brcmf_sdio_kso_control(bus, false);
 		} else {
@@ -1004,7 +979,7 @@ static int brcmf_sdio_readshared(struct brcmf_sdio *bus,
 	struct sdpcm_shared_le sh_le;
 	__le32 addr_le;
 
-	sdio_claim_host(bus->sdiodev->func[1]);
+	sdio_claim_host(bus->sdiodev->func1);
 	brcmf_sdio_bus_sleep(bus, false, false);
 
 	/*
@@ -1038,7 +1013,7 @@ static int brcmf_sdio_readshared(struct brcmf_sdio *bus,
 	if (rv < 0)
 		goto fail;
 
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 
 	/* Endianness */
 	sh->flags = le32_to_cpu(sh_le.flags);
@@ -1060,7 +1035,7 @@ static int brcmf_sdio_readshared(struct brcmf_sdio *bus,
 fail:
 	brcmf_err("unable to obtain sdpcm_shared info: rv=%d (addr=0x%x)\n",
 		  rv, addr);
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 	return rv;
 }
 
@@ -1079,6 +1054,8 @@ static void brcmf_sdio_get_console_addr(struct brcmf_sdio *bus)
 
 static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus)
 {
+	struct brcmf_sdio_dev *sdiod = bus->sdiodev;
+	struct brcmf_core *core = bus->sdio_core;
 	u32 intstatus = 0;
 	u32 hmb_data;
 	u8 fcbits;
@@ -1087,12 +1064,14 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus)
 	brcmf_dbg(SDIO, "Enter\n");
 
 	/* Read mailbox data and ack that we did so */
-	ret = r_sdreg32(bus, &hmb_data,
-			offsetof(struct sdpcmd_regs, tohostmailboxdata));
+	hmb_data = brcmf_sdiod_readl(sdiod,
+				     core->base + SD_REG(tohostmailboxdata),
+				     &ret);
+
+	if (!ret)
+		brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailbox),
+				   SMB_INT_ACK, &ret);
 
-	if (ret == 0)
-		w_sdreg32(bus, SMB_INT_ACK,
-			  offsetof(struct sdpcmd_regs, tosbmailbox));
 	bus->sdcnt.f1regdata += 2;
 
 	/* dongle indicates the firmware has halted/crashed */
@@ -1166,6 +1145,8 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus)
 
 static void brcmf_sdio_rxfail(struct brcmf_sdio *bus, bool abort, bool rtx)
 {
+	struct brcmf_sdio_dev *sdiod = bus->sdiodev;
+	struct brcmf_core *core = bus->sdio_core;
 	uint retries = 0;
 	u16 lastrbc;
 	u8 hi, lo;
@@ -1176,18 +1157,18 @@ static void brcmf_sdio_rxfail(struct brcmf_sdio *bus, bool abort, bool rtx)
 		  rtx ? ", send NAK" : "");
 
 	if (abort)
-		brcmf_sdiod_abort(bus->sdiodev, SDIO_FUNC_2);
+		brcmf_sdiod_abort(bus->sdiodev, bus->sdiodev->func2);
 
-	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_FRAMECTRL,
-			  SFC_RF_TERM, &err);
+	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_RF_TERM,
+			   &err);
 	bus->sdcnt.f1regdata++;
 
 	/* Wait until the packet has been flushed (device/FIFO stable) */
 	for (lastrbc = retries = 0xffff; retries > 0; retries--) {
-		hi = brcmf_sdiod_regrb(bus->sdiodev,
-				       SBSDIO_FUNC1_RFRAMEBCHI, &err);
-		lo = brcmf_sdiod_regrb(bus->sdiodev,
-				       SBSDIO_FUNC1_RFRAMEBCLO, &err);
+		hi = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_RFRAMEBCHI,
+				       &err);
+		lo = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_RFRAMEBCLO,
+				       &err);
 		bus->sdcnt.f1regdata += 2;
 
 		if ((hi == 0) && (lo == 0))
@@ -1207,8 +1188,8 @@ static void brcmf_sdio_rxfail(struct brcmf_sdio *bus, bool abort, bool rtx)
 
 	if (rtx) {
 		bus->sdcnt.rxrtx++;
-		err = w_sdreg32(bus, SMB_NAK,
-				offsetof(struct sdpcmd_regs, tosbmailbox));
+		brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailbox),
+				   SMB_NAK, &err);
 
 		bus->sdcnt.f1regdata++;
 		if (err == 0)
@@ -1228,13 +1209,13 @@ static void brcmf_sdio_txfail(struct brcmf_sdio *bus)
 	brcmf_err("sdio error, abort command and terminate frame\n");
 	bus->sdcnt.tx_sderrs++;
 
-	brcmf_sdiod_abort(sdiodev, SDIO_FUNC_2);
-	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_WF_TERM, NULL);
+	brcmf_sdiod_abort(sdiodev, sdiodev->func2);
+	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_WF_TERM, NULL);
 	bus->sdcnt.f1regdata++;
 
 	for (i = 0; i < 3; i++) {
-		hi = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_WFRAMEBCHI, NULL);
-		lo = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_WFRAMEBCLO, NULL);
+		hi = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_WFRAMEBCHI, NULL);
+		lo = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_WFRAMEBCLO, NULL);
 		bus->sdcnt.f1regdata += 2;
 		if ((hi == 0) && (lo == 0))
 			break;
@@ -1584,10 +1565,10 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq)
 		 * read directly into the chained packet, or allocate a large
 		 * packet and and copy into the chain.
 		 */
-		sdio_claim_host(bus->sdiodev->func[1]);
+		sdio_claim_host(bus->sdiodev->func1);
 		errcode = brcmf_sdiod_recv_chain(bus->sdiodev,
 						 &bus->glom, dlen);
-		sdio_release_host(bus->sdiodev->func[1]);
+		sdio_release_host(bus->sdiodev->func1);
 		bus->sdcnt.f2rxdata++;
 
 		/* On failure, kill the superframe */
@@ -1595,11 +1576,11 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq)
 			brcmf_err("glom read of %d bytes failed: %d\n",
 				  dlen, errcode);
 
-			sdio_claim_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
 			brcmf_sdio_rxfail(bus, true, false);
 			bus->sdcnt.rxglomfail++;
 			brcmf_sdio_free_glom(bus);
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 			return 0;
 		}
 
@@ -1609,10 +1590,10 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq)
 
 		rd_new.seq_num = rxseq;
 		rd_new.len = dlen;
-		sdio_claim_host(bus->sdiodev->func[1]);
+		sdio_claim_host(bus->sdiodev->func1);
 		errcode = brcmf_sdio_hdparse(bus, pfirst->data, &rd_new,
 					     BRCMF_SDIO_FT_SUPER);
-		sdio_release_host(bus->sdiodev->func[1]);
+		sdio_release_host(bus->sdiodev->func1);
 		bus->cur_read.len = rd_new.len_nxtfrm << 4;
 
 		/* Remove superframe header, remember offset */
@@ -1628,10 +1609,10 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq)
 
 			rd_new.len = pnext->len;
 			rd_new.seq_num = rxseq++;
-			sdio_claim_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
 			errcode = brcmf_sdio_hdparse(bus, pnext->data, &rd_new,
 						     BRCMF_SDIO_FT_SUB);
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 			brcmf_dbg_hex_dump(BRCMF_GLOM_ON(),
 					   pnext->data, 32, "subframe:\n");
 
@@ -1640,11 +1621,11 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq)
 
 		if (errcode) {
 			/* Terminate frame on error */
-			sdio_claim_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
 			brcmf_sdio_rxfail(bus, true, false);
 			bus->sdcnt.rxglomfail++;
 			brcmf_sdio_free_glom(bus);
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 			bus->cur_read.len = 0;
 			return 0;
 		}
@@ -1852,7 +1833,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 
 		rd->len_left = rd->len;
 		/* read header first for unknow frame length */
-		sdio_claim_host(bus->sdiodev->func[1]);
+		sdio_claim_host(bus->sdiodev->func1);
 		if (!rd->len) {
 			ret = brcmf_sdiod_recv_buf(bus->sdiodev,
 						   bus->rxhdr, BRCMF_FIRSTREAD);
@@ -1862,7 +1843,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 					  ret);
 				bus->sdcnt.rx_hdrfail++;
 				brcmf_sdio_rxfail(bus, true, true);
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 				continue;
 			}
 
@@ -1872,7 +1853,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 
 			if (brcmf_sdio_hdparse(bus, bus->rxhdr, rd,
 					       BRCMF_SDIO_FT_NORMAL)) {
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 				if (!bus->rxpending)
 					break;
 				else
@@ -1888,7 +1869,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 				rd->len_nxtfrm = 0;
 				/* treat all packet as event if we don't know */
 				rd->channel = SDPCM_EVENT_CHANNEL;
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 				continue;
 			}
 			rd->len_left = rd->len > BRCMF_FIRSTREAD ?
@@ -1905,7 +1886,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 			brcmf_err("brcmu_pkt_buf_get_skb failed\n");
 			brcmf_sdio_rxfail(bus, false,
 					    RETRYCHAN(rd->channel));
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 			continue;
 		}
 		skb_pull(pkt, head_read);
@@ -1913,16 +1894,16 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 
 		ret = brcmf_sdiod_recv_pkt(bus->sdiodev, pkt);
 		bus->sdcnt.f2rxdata++;
-		sdio_release_host(bus->sdiodev->func[1]);
+		sdio_release_host(bus->sdiodev->func1);
 
 		if (ret < 0) {
 			brcmf_err("read %d bytes from channel %d failed: %d\n",
 				  rd->len, rd->channel, ret);
 			brcmu_pkt_buf_free_skb(pkt);
-			sdio_claim_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
 			brcmf_sdio_rxfail(bus, true,
 					    RETRYCHAN(rd->channel));
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 			continue;
 		}
 
@@ -1933,7 +1914,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 		} else {
 			memcpy(bus->rxhdr, pkt->data, SDPCM_HDRLEN);
 			rd_new.seq_num = rd->seq_num;
-			sdio_claim_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
 			if (brcmf_sdio_hdparse(bus, bus->rxhdr, &rd_new,
 					       BRCMF_SDIO_FT_NORMAL)) {
 				rd->len = 0;
@@ -1946,11 +1927,11 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 					  roundup(rd_new.len, 16) >> 4);
 				rd->len = 0;
 				brcmf_sdio_rxfail(bus, true, true);
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 				brcmu_pkt_buf_free_skb(pkt);
 				continue;
 			}
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 			rd->len_nxtfrm = rd_new.len_nxtfrm;
 			rd->channel = rd_new.channel;
 			rd->dat_offset = rd_new.dat_offset;
@@ -1966,9 +1947,9 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 					  rd_new.seq_num);
 				/* Force retry w/normal header read */
 				rd->len = 0;
-				sdio_claim_host(bus->sdiodev->func[1]);
+				sdio_claim_host(bus->sdiodev->func1);
 				brcmf_sdio_rxfail(bus, false, true);
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 				brcmu_pkt_buf_free_skb(pkt);
 				continue;
 			}
@@ -1991,9 +1972,9 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes)
 			} else {
 				brcmf_err("%s: glom superframe w/o "
 					  "descriptor!\n", __func__);
-				sdio_claim_host(bus->sdiodev->func[1]);
+				sdio_claim_host(bus->sdiodev->func1);
 				brcmf_sdio_rxfail(bus, false, false);
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 			}
 			/* prepare the descriptor for the next read */
 			rd->len = rd->len_nxtfrm << 4;
@@ -2091,7 +2072,7 @@ static int brcmf_sdio_txpkt_prep_sg(struct brcmf_sdio *bus,
 	int ntail, ret;
 
 	sdiodev = bus->sdiodev;
-	blksize = sdiodev->func[SDIO_FUNC_2]->cur_blksize;
+	blksize = sdiodev->func2->cur_blksize;
 	/* sg entry alignment should be a divisor of block size */
 	WARN_ON(blksize % bus->sgentry_align);
 
@@ -2270,14 +2251,14 @@ static int brcmf_sdio_txpkt(struct brcmf_sdio *bus, struct sk_buff_head *pktq,
 	if (ret)
 		goto done;
 
-	sdio_claim_host(bus->sdiodev->func[1]);
+	sdio_claim_host(bus->sdiodev->func1);
 	ret = brcmf_sdiod_send_pkt(bus->sdiodev, pktq);
 	bus->sdcnt.f2txdata++;
 
 	if (ret < 0)
 		brcmf_sdio_txfail(bus);
 
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 
 done:
 	brcmf_sdio_txpkt_postp(bus, pktq);
@@ -2295,6 +2276,7 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes)
 {
 	struct sk_buff *pkt;
 	struct sk_buff_head pktq;
+	u32 intstat_addr = bus->sdio_core->base + SD_REG(intstatus);
 	u32 intstatus = 0;
 	int ret = 0, prec_out, i;
 	uint cnt = 0;
@@ -2332,11 +2314,11 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes)
 		/* In poll mode, need to check for other events */
 		if (!bus->intr) {
 			/* Check device status, signal pending interrupt */
-			sdio_claim_host(bus->sdiodev->func[1]);
-			ret = r_sdreg32(bus, &intstatus,
-					offsetof(struct sdpcmd_regs,
-						 intstatus));
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
+			intstatus = brcmf_sdiod_readl(bus->sdiodev,
+						      intstat_addr, &ret);
+			sdio_release_host(bus->sdiodev->func1);
+
 			bus->sdcnt.f2txdata++;
 			if (ret != 0)
 				break;
@@ -2419,12 +2401,13 @@ static int brcmf_sdio_tx_ctrlframe(struct brcmf_sdio *bus, u8 *frame, u16 len)
 
 static void brcmf_sdio_bus_stop(struct device *dev)
 {
-	u32 local_hostintmask;
-	u8 saveclk;
-	int err;
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
 	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
 	struct brcmf_sdio *bus = sdiodev->bus;
+	struct brcmf_core *core = bus->sdio_core;
+	u32 local_hostintmask;
+	u8 saveclk;
+	int err;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
@@ -2435,35 +2418,37 @@ static void brcmf_sdio_bus_stop(struct device *dev)
 	}
 
 	if (sdiodev->state != BRCMF_SDIOD_NOMEDIUM) {
-		sdio_claim_host(sdiodev->func[1]);
+		sdio_claim_host(sdiodev->func1);
 
 		/* Enable clock for device interrupts */
 		brcmf_sdio_bus_sleep(bus, false, false);
 
 		/* Disable and clear interrupts at the chip level also */
-		w_sdreg32(bus, 0, offsetof(struct sdpcmd_regs, hostintmask));
+		brcmf_sdiod_writel(sdiodev, core->base + SD_REG(hostintmask),
+				   0, NULL);
+
 		local_hostintmask = bus->hostintmask;
 		bus->hostintmask = 0;
 
 		/* Force backplane clocks to assure F2 interrupt propagates */
-		saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+		saveclk = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
 					    &err);
 		if (!err)
-			brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-					  (saveclk | SBSDIO_FORCE_HT), &err);
+			brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+					   (saveclk | SBSDIO_FORCE_HT), &err);
 		if (err)
 			brcmf_err("Failed to force clock for F2: err %d\n",
 				  err);
 
 		/* Turn off the bus (F2), free any pending packets */
 		brcmf_dbg(INTR, "disable SDIO interrupts\n");
-		sdio_disable_func(sdiodev->func[SDIO_FUNC_2]);
+		sdio_disable_func(sdiodev->func2);
 
 		/* Clear any pending interrupts now that F2 is disabled */
-		w_sdreg32(bus, local_hostintmask,
-			  offsetof(struct sdpcmd_regs, intstatus));
+		brcmf_sdiod_writel(sdiodev, core->base + SD_REG(intstatus),
+				   local_hostintmask, NULL);
 
-		sdio_release_host(sdiodev->func[1]);
+		sdio_release_host(sdiodev->func1);
 	}
 	/* Clear the data packet queues */
 	brcmu_pktq_flush(&bus->txq, true, NULL, NULL);
@@ -2501,15 +2486,14 @@ static inline void brcmf_sdio_clrintr(struct brcmf_sdio *bus)
 
 static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus)
 {
-	struct brcmf_core *buscore;
+	struct brcmf_core *core = bus->sdio_core;
 	u32 addr;
 	unsigned long val;
 	int ret;
 
-	buscore = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
-	addr = buscore->base + offsetof(struct sdpcmd_regs, intstatus);
+	addr = core->base + SD_REG(intstatus);
 
-	val = brcmf_sdiod_regrl(bus->sdiodev, addr, &ret);
+	val = brcmf_sdiod_readl(bus->sdiodev, addr, &ret);
 	bus->sdcnt.f1regdata++;
 	if (ret != 0)
 		return ret;
@@ -2519,7 +2503,7 @@ static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus)
 
 	/* Clear interrupts */
 	if (val) {
-		brcmf_sdiod_regwl(bus->sdiodev, addr, val, &ret);
+		brcmf_sdiod_writel(bus->sdiodev, addr, val, &ret);
 		bus->sdcnt.f1regdata++;
 		atomic_or(val, &bus->intstatus);
 	}
@@ -2529,7 +2513,9 @@ static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus)
 
 static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 {
+	struct brcmf_sdio_dev *sdiod = bus->sdiodev;
 	u32 newstatus = 0;
+	u32 intstat_addr = bus->sdio_core->base + SD_REG(intstatus);
 	unsigned long intstatus;
 	uint txlimit = bus->txbound;	/* Tx frames to send before resched */
 	uint framecnt;			/* Temporary counter of tx/rx frames */
@@ -2537,7 +2523,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 
 	brcmf_dbg(TRACE, "Enter\n");
 
-	sdio_claim_host(bus->sdiodev->func[1]);
+	sdio_claim_host(bus->sdiodev->func1);
 
 	/* If waiting for HTAVAIL, check status */
 	if (!bus->sr_enabled && bus->clkstate == CLK_PENDING) {
@@ -2545,23 +2531,23 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 
 #ifdef DEBUG
 		/* Check for inconsistent device control */
-		devctl = brcmf_sdiod_regrb(bus->sdiodev,
-					   SBSDIO_DEVICE_CTL, &err);
+		devctl = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_DEVICE_CTL,
+					   &err);
 #endif				/* DEBUG */
 
 		/* Read CSR, if clock on switch to AVAIL, else ignore */
-		clkctl = brcmf_sdiod_regrb(bus->sdiodev,
+		clkctl = brcmf_sdiod_readb(bus->sdiodev,
 					   SBSDIO_FUNC1_CHIPCLKCSR, &err);
 
 		brcmf_dbg(SDIO, "DPC: PENDING, devctl 0x%02x clkctl 0x%02x\n",
 			  devctl, clkctl);
 
 		if (SBSDIO_HTAV(clkctl)) {
-			devctl = brcmf_sdiod_regrb(bus->sdiodev,
+			devctl = brcmf_sdiod_readb(bus->sdiodev,
 						   SBSDIO_DEVICE_CTL, &err);
 			devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY;
-			brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL,
-					  devctl, &err);
+			brcmf_sdiod_writeb(bus->sdiodev,
+					   SBSDIO_DEVICE_CTL, devctl, &err);
 			bus->clkstate = CLK_AVAIL;
 		}
 	}
@@ -2584,11 +2570,10 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 	 */
 	if (intstatus & I_HMB_FC_CHANGE) {
 		intstatus &= ~I_HMB_FC_CHANGE;
-		err = w_sdreg32(bus, I_HMB_FC_CHANGE,
-				offsetof(struct sdpcmd_regs, intstatus));
+		brcmf_sdiod_writel(sdiod, intstat_addr, I_HMB_FC_CHANGE, &err);
+
+		newstatus = brcmf_sdiod_readl(sdiod, intstat_addr, &err);
 
-		err = r_sdreg32(bus, &newstatus,
-				offsetof(struct sdpcmd_regs, intstatus));
 		bus->sdcnt.f1regdata += 2;
 		atomic_set(&bus->fcstate,
 			   !!(newstatus & (I_HMB_FC_STATE | I_HMB_FC_CHANGE)));
@@ -2601,7 +2586,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 		intstatus |= brcmf_sdio_hostmail(bus);
 	}
 
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 
 	/* Generally don't ask for these, can get CRC errors... */
 	if (intstatus & I_WR_OOSYNC) {
@@ -2644,7 +2629,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 
 	if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) &&
 	    data_ok(bus)) {
-		sdio_claim_host(bus->sdiodev->func[1]);
+		sdio_claim_host(bus->sdiodev->func1);
 		if (bus->ctrl_frame_stat) {
 			err = brcmf_sdio_tx_ctrlframe(bus,  bus->ctrl_frame_buf,
 						      bus->ctrl_frame_len);
@@ -2652,7 +2637,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 			wmb();
 			bus->ctrl_frame_stat = false;
 		}
-		sdio_release_host(bus->sdiodev->func[1]);
+		sdio_release_host(bus->sdiodev->func1);
 		brcmf_sdio_wait_event_wakeup(bus);
 	}
 	/* Send queued frames (limit 1 if rx may still be pending) */
@@ -2668,14 +2653,14 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus)
 		brcmf_err("failed backplane access over SDIO, halting operation\n");
 		atomic_set(&bus->intstatus, 0);
 		if (bus->ctrl_frame_stat) {
-			sdio_claim_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
 			if (bus->ctrl_frame_stat) {
 				bus->ctrl_frame_err = -ENODEV;
 				wmb();
 				bus->ctrl_frame_stat = false;
 				brcmf_sdio_wait_event_wakeup(bus);
 			}
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 		}
 	} else if (atomic_read(&bus->intstatus) ||
 		   atomic_read(&bus->ipend) > 0 ||
@@ -2890,13 +2875,13 @@ brcmf_sdio_bus_txctl(struct device *dev, unsigned char *msg, uint msglen)
 					 CTL_DONE_TIMEOUT);
 	ret = 0;
 	if (bus->ctrl_frame_stat) {
-		sdio_claim_host(bus->sdiodev->func[1]);
+		sdio_claim_host(bus->sdiodev->func1);
 		if (bus->ctrl_frame_stat) {
 			brcmf_dbg(SDIO, "ctrl_frame timeout\n");
 			bus->ctrl_frame_stat = false;
 			ret = -ETIMEDOUT;
 		}
-		sdio_release_host(bus->sdiodev->func[1]);
+		sdio_release_host(bus->sdiodev->func1);
 	}
 	if (!ret) {
 		brcmf_dbg(SDIO, "ctrl_frame complete, err=%d\n",
@@ -3020,7 +3005,7 @@ static int brcmf_sdio_assert_info(struct seq_file *seq, struct brcmf_sdio *bus,
 		return 0;
 	}
 
-	sdio_claim_host(bus->sdiodev->func[1]);
+	sdio_claim_host(bus->sdiodev->func1);
 	if (sh->assert_file_addr != 0) {
 		error = brcmf_sdiod_ramrw(bus->sdiodev, false,
 					  sh->assert_file_addr, (u8 *)file, 80);
@@ -3033,7 +3018,7 @@ static int brcmf_sdio_assert_info(struct seq_file *seq, struct brcmf_sdio *bus,
 		if (error < 0)
 			return error;
 	}
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 
 	seq_printf(seq, "dongle assert: %s:%d: assert(%s)\n",
 		   file, sh->assert_line, expr);
@@ -3307,7 +3292,7 @@ static int brcmf_sdio_download_firmware(struct brcmf_sdio *bus,
 	int bcmerror;
 	u32 rstvec;
 
-	sdio_claim_host(bus->sdiodev->func[1]);
+	sdio_claim_host(bus->sdiodev->func1);
 	brcmf_sdio_clkctl(bus, CLK_AVAIL, false);
 
 	rstvec = get_unaligned_le32(fw->data);
@@ -3336,7 +3321,7 @@ static int brcmf_sdio_download_firmware(struct brcmf_sdio *bus,
 
 err:
 	brcmf_sdio_clkctl(bus, CLK_SDONLY, false);
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 	return bcmerror;
 }
 
@@ -3347,31 +3332,31 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus)
 
 	brcmf_dbg(TRACE, "Enter\n");
 
-	val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err);
+	val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err);
 	if (err) {
 		brcmf_err("error reading SBSDIO_FUNC1_WAKEUPCTRL\n");
 		return;
 	}
 
 	val |= 1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT;
-	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err);
+	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err);
 	if (err) {
 		brcmf_err("error writing SBSDIO_FUNC1_WAKEUPCTRL\n");
 		return;
 	}
 
 	/* Add CMD14 Support */
-	brcmf_sdiod_regwb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP,
-			  (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT |
-			   SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT),
-			  &err);
+	brcmf_sdiod_func0_wb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP,
+			     (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT |
+			      SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT),
+			     &err);
 	if (err) {
 		brcmf_err("error writing SDIO_CCCR_BRCM_CARDCAP\n");
 		return;
 	}
 
-	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-			  SBSDIO_FORCE_HT, &err);
+	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+			   SBSDIO_FORCE_HT, &err);
 	if (err) {
 		brcmf_err("error writing SBSDIO_FUNC1_CHIPCLKCSR\n");
 		return;
@@ -3385,16 +3370,17 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus)
 /* enable KSO bit */
 static int brcmf_sdio_kso_init(struct brcmf_sdio *bus)
 {
+	struct brcmf_core *core = bus->sdio_core;
 	u8 val;
 	int err = 0;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
 	/* KSO bit added in SDIO core rev 12 */
-	if (brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV)->rev < 12)
+	if (core->rev < 12)
 		return 0;
 
-	val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, &err);
+	val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, &err);
 	if (err) {
 		brcmf_err("error reading SBSDIO_FUNC1_SLEEPCSR\n");
 		return err;
@@ -3403,8 +3389,8 @@ static int brcmf_sdio_kso_init(struct brcmf_sdio *bus)
 	if (!(val & SBSDIO_FUNC1_SLEEPCSR_KSO_MASK)) {
 		val |= (SBSDIO_FUNC1_SLEEPCSR_KSO_EN <<
 			SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT);
-		brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
-				  val, &err);
+		brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR,
+				   val, &err);
 		if (err) {
 			brcmf_err("error writing SBSDIO_FUNC1_SLEEPCSR\n");
 			return err;
@@ -3420,6 +3406,7 @@ static int brcmf_sdio_bus_preinit(struct device *dev)
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
 	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
 	struct brcmf_sdio *bus = sdiodev->bus;
+	struct brcmf_core *core = bus->sdio_core;
 	uint pad_size;
 	u32 value;
 	int err;
@@ -3428,7 +3415,7 @@ static int brcmf_sdio_bus_preinit(struct device *dev)
 	 * a device perspective, ie. bus:txglom affects the
 	 * bus transfers from device to host.
 	 */
-	if (brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV)->rev < 12) {
+	if (core->rev < 12) {
 		/* for sdio core rev < 12, disable txgloming */
 		value = 0;
 		err = brcmf_iovar_data_set(dev, "bus:txglom", &value,
@@ -3449,7 +3436,7 @@ static int brcmf_sdio_bus_preinit(struct device *dev)
 	if (sdiodev->sg_support) {
 		bus->txglom = false;
 		value = 1;
-		pad_size = bus->sdiodev->func[2]->cur_blksize << 1;
+		pad_size = bus->sdiodev->func2->cur_blksize << 1;
 		err = brcmf_iovar_data_set(bus->sdiodev->dev, "bus:rxglom",
 					   &value, sizeof(u32));
 		if (err < 0) {
@@ -3491,7 +3478,7 @@ static int brcmf_sdio_bus_get_memdump(struct device *dev, void *data,
 
 	address = bus->ci->rambase;
 	offset = err = 0;
-	sdio_claim_host(sdiodev->func[1]);
+	sdio_claim_host(sdiodev->func1);
 	while (offset < mem_size) {
 		len = ((offset + MEMBLOCK) < mem_size) ? MEMBLOCK :
 		      mem_size - offset;
@@ -3507,7 +3494,7 @@ static int brcmf_sdio_bus_get_memdump(struct device *dev, void *data,
 	}
 
 done:
-	sdio_release_host(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func1);
 	return err;
 }
 
@@ -3564,11 +3551,10 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus)
 			if (!bus->dpc_triggered) {
 				u8 devpend;
 
-				sdio_claim_host(bus->sdiodev->func[1]);
-				devpend = brcmf_sdiod_regrb(bus->sdiodev,
-							    SDIO_CCCR_INTx,
-							    NULL);
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_claim_host(bus->sdiodev->func1);
+				devpend = brcmf_sdiod_func0_rb(bus->sdiodev,
+						  SDIO_CCCR_INTx, NULL);
+				sdio_release_host(bus->sdiodev->func1);
 				intstatus = devpend & (INTR_STATUS_FUNC1 |
 						       INTR_STATUS_FUNC2);
 			}
@@ -3594,13 +3580,13 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus)
 		bus->console.count += jiffies_to_msecs(BRCMF_WD_POLL);
 		if (bus->console.count >= bus->console_interval) {
 			bus->console.count -= bus->console_interval;
-			sdio_claim_host(bus->sdiodev->func[1]);
+			sdio_claim_host(bus->sdiodev->func1);
 			/* Make sure backplane clock is on */
 			brcmf_sdio_bus_sleep(bus, false, false);
 			if (brcmf_sdio_readconsole(bus) < 0)
 				/* stop on error */
 				bus->console_interval = 0;
-			sdio_release_host(bus->sdiodev->func[1]);
+			sdio_release_host(bus->sdiodev->func1);
 		}
 	}
 #endif				/* DEBUG */
@@ -3613,11 +3599,11 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus)
 			bus->idlecount++;
 			if (bus->idlecount > bus->idletime) {
 				brcmf_dbg(SDIO, "idle\n");
-				sdio_claim_host(bus->sdiodev->func[1]);
+				sdio_claim_host(bus->sdiodev->func1);
 				brcmf_sdio_wd_timer(bus, false);
 				bus->idlecount = 0;
 				brcmf_sdio_bus_sleep(bus, true, false);
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 			}
 		} else {
 			bus->idlecount = 0;
@@ -3705,12 +3691,12 @@ brcmf_sdio_drivestrengthinit(struct brcmf_sdio_dev *sdiodev,
 			}
 		}
 		addr = CORE_CC_REG(pmu->base, chipcontrol_addr);
-		brcmf_sdiod_regwl(sdiodev, addr, 1, NULL);
-		cc_data_temp = brcmf_sdiod_regrl(sdiodev, addr, NULL);
+		brcmf_sdiod_writel(sdiodev, addr, 1, NULL);
+		cc_data_temp = brcmf_sdiod_readl(sdiodev, addr, NULL);
 		cc_data_temp &= ~str_mask;
 		drivestrength_sel <<= str_shift;
 		cc_data_temp |= drivestrength_sel;
-		brcmf_sdiod_regwl(sdiodev, addr, cc_data_temp, NULL);
+		brcmf_sdiod_writel(sdiodev, addr, cc_data_temp, NULL);
 
 		brcmf_dbg(INFO, "SDIO: %d mA (req=%d mA) drive strength selected, set to 0x%08x\n",
 			  str_tab[i].strength, drivestrength, cc_data_temp);
@@ -3725,7 +3711,7 @@ static int brcmf_sdio_buscoreprep(void *ctx)
 
 	/* Try forcing SDIO core to do ALPAvail request only */
 	clkset = SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_ALP_AVAIL_REQ;
-	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
+	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
 	if (err) {
 		brcmf_err("error writing for HT off\n");
 		return err;
@@ -3733,8 +3719,7 @@ static int brcmf_sdio_buscoreprep(void *ctx)
 
 	/* If register supported, wait for ALPAvail and then force ALP */
 	/* This may take up to 15 milliseconds */
-	clkval = brcmf_sdiod_regrb(sdiodev,
-				   SBSDIO_FUNC1_CHIPCLKCSR, NULL);
+	clkval = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, NULL);
 
 	if ((clkval & ~SBSDIO_AVBITS) != clkset) {
 		brcmf_err("ChipClkCSR access: wrote 0x%02x read 0x%02x\n",
@@ -3742,10 +3727,11 @@ static int brcmf_sdio_buscoreprep(void *ctx)
 		return -EACCES;
 	}
 
-	SPINWAIT(((clkval = brcmf_sdiod_regrb(sdiodev,
-					      SBSDIO_FUNC1_CHIPCLKCSR, NULL)),
-			!SBSDIO_ALPAV(clkval)),
-			PMU_MAX_TRANSITION_DLY);
+	SPINWAIT(((clkval = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+					      NULL)),
+		 !SBSDIO_ALPAV(clkval)),
+		 PMU_MAX_TRANSITION_DLY);
+
 	if (!SBSDIO_ALPAV(clkval)) {
 		brcmf_err("timeout on ALPAV wait, clkval 0x%02x\n",
 			  clkval);
@@ -3753,11 +3739,11 @@ static int brcmf_sdio_buscoreprep(void *ctx)
 	}
 
 	clkset = SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_FORCE_ALP;
-	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
+	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err);
 	udelay(65);
 
 	/* Also, disable the extra SDIO pull-ups */
-	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_SDIOPULLUP, 0, NULL);
+	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_SDIOPULLUP, 0, NULL);
 
 	return 0;
 }
@@ -3766,13 +3752,12 @@ static void brcmf_sdio_buscore_activate(void *ctx, struct brcmf_chip *chip,
 					u32 rstvec)
 {
 	struct brcmf_sdio_dev *sdiodev = ctx;
-	struct brcmf_core *core;
+	struct brcmf_core *core = sdiodev->bus->sdio_core;
 	u32 reg_addr;
 
 	/* clear all interrupts */
-	core = brcmf_chip_get_core(chip, BCMA_CORE_SDIO_DEV);
-	reg_addr = core->base + offsetof(struct sdpcmd_regs, intstatus);
-	brcmf_sdiod_regwl(sdiodev, reg_addr, 0xFFFFFFFF, NULL);
+	reg_addr = core->base + SD_REG(intstatus);
+	brcmf_sdiod_writel(sdiodev, reg_addr, 0xFFFFFFFF, NULL);
 
 	if (rstvec)
 		/* Write reset vector to address 0 */
@@ -3785,16 +3770,25 @@ static u32 brcmf_sdio_buscore_read32(void *ctx, u32 addr)
 	struct brcmf_sdio_dev *sdiodev = ctx;
 	u32 val, rev;
 
-	val = brcmf_sdiod_regrl(sdiodev, addr, NULL);
-	if ((sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4335_4339 ||
-	     sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4339) &&
-	    addr == CORE_CC_REG(SI_ENUM_BASE, chipid)) {
+	val = brcmf_sdiod_readl(sdiodev, addr, NULL);
+
+	/*
+	 * this is a bit of special handling if reading the chipcommon chipid
+	 * register. The 4339 is a next-gen of the 4335. It uses the same
+	 * SDIO device id as 4335 and the chipid register returns 4335 as well.
+	 * It can be identified as 4339 by looking at the chip revision. It
+	 * is corrected here so the chip.c module has the right info.
+	 */
+	if (addr == CORE_CC_REG(SI_ENUM_BASE, chipid) &&
+	    (sdiodev->func1->device == SDIO_DEVICE_ID_BROADCOM_4339 ||
+	     sdiodev->func1->device == SDIO_DEVICE_ID_BROADCOM_4335_4339)) {
 		rev = (val & CID_REV_MASK) >> CID_REV_SHIFT;
 		if (rev >= 2) {
 			val &= ~CID_ID_MASK;
 			val |= BRCM_CC_4339_CHIP_ID;
 		}
 	}
+
 	return val;
 }
 
@@ -3802,7 +3796,7 @@ static void brcmf_sdio_buscore_write32(void *ctx, u32 addr, u32 val)
 {
 	struct brcmf_sdio_dev *sdiodev = ctx;
 
-	brcmf_sdiod_regwl(sdiodev, addr, val, NULL);
+	brcmf_sdiod_writel(sdiodev, addr, val, NULL);
 }
 
 static const struct brcmf_buscore_ops brcmf_sdio_buscore_ops = {
@@ -3823,21 +3817,21 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus)
 	u32 drivestrength;
 
 	sdiodev = bus->sdiodev;
-	sdio_claim_host(sdiodev->func[1]);
+	sdio_claim_host(sdiodev->func1);
 
 	pr_debug("F1 signature read @0x18000000=0x%4x\n",
-		 brcmf_sdiod_regrl(sdiodev, SI_ENUM_BASE, NULL));
+		 brcmf_sdiod_readl(sdiodev, SI_ENUM_BASE, NULL));
 
 	/*
 	 * Force PLL off until brcmf_chip_attach()
 	 * programs PLL control regs
 	 */
 
-	brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-			  BRCMF_INIT_CLKCTL1, &err);
+	brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, BRCMF_INIT_CLKCTL1,
+			   &err);
 	if (!err)
-		clkctl = brcmf_sdiod_regrb(sdiodev,
-					   SBSDIO_FUNC1_CHIPCLKCSR, &err);
+		clkctl = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+					   &err);
 
 	if (err || ((clkctl & ~SBSDIO_AVBITS) != BRCMF_INIT_CLKCTL1)) {
 		brcmf_err("ChipClkCSR access: err %d wrote 0x%02x read 0x%02x\n",
@@ -3851,6 +3845,17 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus)
 		bus->ci = NULL;
 		goto fail;
 	}
+
+	/* Pick up the SDIO core info struct from chip.c */
+	bus->sdio_core   = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV);
+	if (!bus->sdio_core)
+		goto fail;
+
+	/* Pick up the CHIPCOMMON core info struct, for bulk IO in bcmsdh.c */
+	sdiodev->cc_core = brcmf_chip_get_core(bus->ci, BCMA_CORE_CHIPCOMMON);
+	if (!sdiodev->cc_core)
+		goto fail;
+
 	sdiodev->settings = brcmf_get_module_param(sdiodev->dev,
 						   BRCMF_BUSTYPE_SDIO,
 						   bus->ci->chip,
@@ -3879,8 +3884,8 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus)
 	/* wowl can be supported when KEEP_POWER is true and (WAKE_SDIO_IRQ
 	 * is true or when platform data OOB irq is true).
 	 */
-	if ((sdio_get_host_pm_caps(sdiodev->func[1]) & MMC_PM_KEEP_POWER) &&
-	    ((sdio_get_host_pm_caps(sdiodev->func[1]) & MMC_PM_WAKE_SDIO_IRQ) ||
+	if ((sdio_get_host_pm_caps(sdiodev->func1) & MMC_PM_KEEP_POWER) &&
+	    ((sdio_get_host_pm_caps(sdiodev->func1) & MMC_PM_WAKE_SDIO_IRQ) ||
 	     (sdiodev->settings->bus.sdio.oob_irq_supported)))
 		sdiodev->bus_if->wowl_supported = true;
 #endif
@@ -3897,29 +3902,29 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus)
 	brcmf_sdio_drivestrengthinit(sdiodev, bus->ci, drivestrength);
 
 	/* Set card control so an SDIO card reset does a WLAN backplane reset */
-	reg_val = brcmf_sdiod_regrb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, &err);
+	reg_val = brcmf_sdiod_func0_rb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, &err);
 	if (err)
 		goto fail;
 
 	reg_val |= SDIO_CCCR_BRCM_CARDCTRL_WLANRESET;
 
-	brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, reg_val, &err);
+	brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, reg_val, &err);
 	if (err)
 		goto fail;
 
 	/* set PMUControl so a backplane reset does PMU state reload */
 	reg_addr = CORE_CC_REG(brcmf_chip_get_pmu(bus->ci)->base, pmucontrol);
-	reg_val = brcmf_sdiod_regrl(sdiodev, reg_addr, &err);
+	reg_val = brcmf_sdiod_readl(sdiodev, reg_addr, &err);
 	if (err)
 		goto fail;
 
 	reg_val |= (BCMA_CC_PMU_CTL_RES_RELOAD << BCMA_CC_PMU_CTL_RES_SHIFT);
 
-	brcmf_sdiod_regwl(sdiodev, reg_addr, reg_val, &err);
+	brcmf_sdiod_writel(sdiodev, reg_addr, reg_val, &err);
 	if (err)
 		goto fail;
 
-	sdio_release_host(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func1);
 
 	brcmu_pktq_init(&bus->txq, (PRIOMASK + 1), TXQLEN);
 
@@ -3940,7 +3945,7 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus)
 	return true;
 
 fail:
-	sdio_release_host(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func1);
 	return false;
 }
 
@@ -4020,22 +4025,21 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 					 const struct firmware *code,
 					 void *nvram, u32 nvram_len)
 {
-	struct brcmf_bus *bus_if;
-	struct brcmf_sdio_dev *sdiodev;
-	struct brcmf_sdio *bus;
+	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
+	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
+	struct brcmf_sdio *bus = sdiodev->bus;
+	struct brcmf_sdio_dev *sdiod = bus->sdiodev;
+	struct brcmf_core *core = bus->sdio_core;
 	u8 saveclk;
 
 	brcmf_dbg(TRACE, "Enter: dev=%s, err=%d\n", dev_name(dev), err);
-	bus_if = dev_get_drvdata(dev);
-	sdiodev = bus_if->bus_priv.sdio;
+
 	if (err)
 		goto fail;
 
 	if (!bus_if->drvr)
 		return;
 
-	bus = sdiodev->bus;
-
 	/* try to download image and nvram to the dongle */
 	bus->alp_only = true;
 	err = brcmf_sdio_download_firmware(bus, code, nvram, nvram_len);
@@ -4047,7 +4051,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 	bus->sdcnt.tickcnt = 0;
 	brcmf_sdio_wd_timer(bus, true);
 
-	sdio_claim_host(sdiodev->func[1]);
+	sdio_claim_host(sdiodev->func1);
 
 	/* Make sure backplane clock is on, needed to generate F2 interrupt */
 	brcmf_sdio_clkctl(bus, CLK_AVAIL, false);
@@ -4055,10 +4059,10 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 		goto release;
 
 	/* Force clocks on backplane to be sure F2 interrupt propagates */
-	saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err);
+	saveclk = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err);
 	if (!err) {
-		brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-				  (saveclk | SBSDIO_FORCE_HT), &err);
+		brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+				   (saveclk | SBSDIO_FORCE_HT), &err);
 	}
 	if (err) {
 		brcmf_err("Failed to force clock for F2: err %d\n", err);
@@ -4066,10 +4070,10 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 	}
 
 	/* Enable function 2 (frame transfers) */
-	w_sdreg32(bus, SDPCM_PROT_VERSION << SMB_DATA_VERSION_SHIFT,
-		  offsetof(struct sdpcmd_regs, tosbmailboxdata));
-	err = sdio_enable_func(sdiodev->func[SDIO_FUNC_2]);
+	brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailboxdata),
+			   SDPCM_PROT_VERSION << SMB_DATA_VERSION_SHIFT, NULL);
 
+	err = sdio_enable_func(sdiodev->func2);
 
 	brcmf_dbg(INFO, "enable F2: err=%d\n", err);
 
@@ -4077,13 +4081,14 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 	if (!err) {
 		/* Set up the interrupt mask and enable interrupts */
 		bus->hostintmask = HOSTINTMASK;
-		w_sdreg32(bus, bus->hostintmask,
-			  offsetof(struct sdpcmd_regs, hostintmask));
+		brcmf_sdiod_writel(sdiod, core->base + SD_REG(hostintmask),
+				   bus->hostintmask, NULL);
+
 
-		brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, 8, &err);
+		brcmf_sdiod_writeb(sdiodev, SBSDIO_WATERMARK, 8, &err);
 	} else {
 		/* Disable F2 again */
-		sdio_disable_func(sdiodev->func[SDIO_FUNC_2]);
+		sdio_disable_func(sdiodev->func2);
 		goto release;
 	}
 
@@ -4091,8 +4096,8 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 		brcmf_sdio_sr_init(bus);
 	} else {
 		/* Restore previous clock setting */
-		brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
-				  saveclk, &err);
+		brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR,
+				   saveclk, &err);
 	}
 
 	if (err == 0) {
@@ -4108,7 +4113,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 	if (err != 0)
 		brcmf_sdio_clkctl(bus, CLK_NONE, false);
 
-	sdio_release_host(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func1);
 
 	err = brcmf_bus_started(dev);
 	if (err != 0) {
@@ -4118,10 +4123,10 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 	return;
 
 release:
-	sdio_release_host(sdiodev->func[1]);
+	sdio_release_host(sdiodev->func1);
 fail:
 	brcmf_dbg(TRACE, "failed: dev=%s, err=%d\n", dev_name(dev), err);
-	device_release_driver(&sdiodev->func[2]->dev);
+	device_release_driver(&sdiodev->func2->dev);
 	device_release_driver(dev);
 }
 
@@ -4148,7 +4153,7 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
 
 	/* single-threaded workqueue */
 	wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM,
-				     dev_name(&sdiodev->func[1]->dev));
+				     dev_name(&sdiodev->func1->dev));
 	if (!wq) {
 		brcmf_err("insufficient memory to create txworkqueue\n");
 		goto fail;
@@ -4174,7 +4179,7 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
 	init_completion(&bus->watchdog_wait);
 	bus->watchdog_tsk = kthread_run(brcmf_sdio_watchdog_thread,
 					bus, "brcmf_wdog/%s",
-					dev_name(&sdiodev->func[1]->dev));
+					dev_name(&sdiodev->func1->dev));
 	if (IS_ERR(bus->watchdog_tsk)) {
 		pr_warn("brcmf_watchdog thread failed to start\n");
 		bus->watchdog_tsk = NULL;
@@ -4200,7 +4205,7 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
 	}
 
 	/* Query the F2 block size, set roundup accordingly */
-	bus->blocksize = bus->sdiodev->func[2]->cur_blksize;
+	bus->blocksize = bus->sdiodev->func2->cur_blksize;
 	bus->roundup = min(max_roundup, bus->blocksize);
 
 	/* Allocate buffers */
@@ -4216,17 +4221,17 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev)
 		}
 	}
 
-	sdio_claim_host(bus->sdiodev->func[1]);
+	sdio_claim_host(bus->sdiodev->func1);
 
 	/* Disable F2 to clear any intermediate frame state on the dongle */
-	sdio_disable_func(bus->sdiodev->func[SDIO_FUNC_2]);
+	sdio_disable_func(bus->sdiodev->func2);
 
 	bus->rxflow = false;
 
 	/* Done with backplane-dependent accesses, can drop clock... */
-	brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, 0, NULL);
+	brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, 0, NULL);
 
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 
 	/* ...and initialize clock/power states */
 	bus->clkstate = CLK_SDONLY;
@@ -4278,7 +4283,7 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus)
 
 		if (bus->ci) {
 			if (bus->sdiodev->state != BRCMF_SDIOD_NOMEDIUM) {
-				sdio_claim_host(bus->sdiodev->func[1]);
+				sdio_claim_host(bus->sdiodev->func1);
 				brcmf_sdio_wd_timer(bus, false);
 				brcmf_sdio_clkctl(bus, CLK_AVAIL, false);
 				/* Leave the device in state where it is
@@ -4288,7 +4293,7 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus)
 				msleep(20);
 				brcmf_chip_set_passive(bus->ci);
 				brcmf_sdio_clkctl(bus, CLK_NONE, false);
-				sdio_release_host(bus->sdiodev->func[1]);
+				sdio_release_host(bus->sdiodev->func1);
 			}
 			brcmf_chip_detach(bus->ci);
 		}
@@ -4335,9 +4340,9 @@ int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep)
 {
 	int ret;
 
-	sdio_claim_host(bus->sdiodev->func[1]);
+	sdio_claim_host(bus->sdiodev->func1);
 	ret = brcmf_sdio_bus_sleep(bus, sleep, false);
-	sdio_release_host(bus->sdiodev->func[1]);
+	sdio_release_host(bus->sdiodev->func1);
 
 	return ret;
 }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
index f3da32fc6360..7faed831f07d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
@@ -21,10 +21,6 @@
 #include <linux/firmware.h>
 #include "firmware.h"
 
-#define SDIO_FUNC_0		0
-#define SDIO_FUNC_1		1
-#define SDIO_FUNC_2		2
-
 #define SDIOD_FBR_SIZE		0x100
 
 /* io_en */
@@ -39,28 +35,29 @@
 #define INTR_STATUS_FUNC1	0x2
 #define INTR_STATUS_FUNC2	0x4
 
-/* Maximum number of I/O funcs */
-#define SDIOD_MAX_IOFUNCS	7
-
 /* mask of register map */
 #define REG_F0_REG_MASK		0x7FF
 #define REG_F1_MISC_MASK	0x1FFFF
 
-/* as of sdiod rev 0, supports 3 functions */
-#define SBSDIO_NUM_FUNCTION		3
-
 /* function 0 vendor specific CCCR registers */
+
 #define SDIO_CCCR_BRCM_CARDCAP			0xf0
-#define SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT	0x02
-#define SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT	0x04
-#define SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC	0x08
-#define SDIO_CCCR_BRCM_CARDCTRL		0xf1
-#define SDIO_CCCR_BRCM_CARDCTRL_WLANRESET	0x02
-#define SDIO_CCCR_BRCM_SEPINT			0xf2
+#define SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT	BIT(1)
+#define SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT	BIT(2)
+#define SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC	BIT(3)
+
+/* Interrupt enable bits for each function */
+#define SDIO_CCCR_IEN_FUNC0			BIT(0)
+#define SDIO_CCCR_IEN_FUNC1			BIT(1)
+#define SDIO_CCCR_IEN_FUNC2			BIT(2)
+
+#define SDIO_CCCR_BRCM_CARDCTRL			0xf1
+#define SDIO_CCCR_BRCM_CARDCTRL_WLANRESET	BIT(1)
 
-#define  SDIO_SEPINT_MASK		0x01
-#define  SDIO_SEPINT_OE			0x02
-#define  SDIO_SEPINT_ACT_HI		0x04
+#define SDIO_CCCR_BRCM_SEPINT			0xf2
+#define SDIO_CCCR_BRCM_SEPINT_MASK		BIT(0)
+#define SDIO_CCCR_BRCM_SEPINT_OE		BIT(1)
+#define SDIO_CCCR_BRCM_SEPINT_ACT_HI		BIT(2)
 
 /* function 1 miscellaneous registers */
 
@@ -131,11 +128,6 @@
 /* with b15, maps to 32-bit SB access */
 #define SBSDIO_SB_ACCESS_2_4B_FLAG	0x08000
 
-/* valid bits in SBSDIO_FUNC1_SBADDRxxx regs */
-
-#define SBSDIO_SBADDRLOW_MASK		0x80	/* Valid bits in SBADDRLOW */
-#define SBSDIO_SBADDRMID_MASK		0xff	/* Valid bits in SBADDRMID */
-#define SBSDIO_SBADDRHIGH_MASK		0xffU	/* Valid bits in SBADDRHIGH */
 /* Address bits from SBADDR regs */
 #define SBSDIO_SBWINDOW_MASK		0xffff8000
 
@@ -178,9 +170,10 @@ struct brcmf_sdio;
 struct brcmf_sdiod_freezer;
 
 struct brcmf_sdio_dev {
-	struct sdio_func *func[SDIO_MAX_FUNCS];
-	u8 num_funcs;			/* Supported funcs on client */
+	struct sdio_func *func1;
+	struct sdio_func *func2;
 	u32 sbwad;			/* Save backplane window address */
+	struct brcmf_core *cc_core;	/* chipcommon core info struct */
 	struct brcmf_sdio *bus;
 	struct device *dev;
 	struct brcmf_bus *bus_if;
@@ -296,13 +289,24 @@ struct sdpcmd_regs {
 int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev);
 void brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev);
 
-/* sdio device register access interface */
-u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret);
-u32 brcmf_sdiod_regrl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret);
-void brcmf_sdiod_regwb(struct brcmf_sdio_dev *sdiodev, u32 addr, u8 data,
-		       int *ret);
-void brcmf_sdiod_regwl(struct brcmf_sdio_dev *sdiodev, u32 addr, u32 data,
-		       int *ret);
+/* SDIO device register access interface */
+/* Accessors for SDIO Function 0 */
+#define brcmf_sdiod_func0_rb(sdiodev, addr, r) \
+	sdio_f0_readb((sdiodev)->func1, (addr), (r))
+
+#define brcmf_sdiod_func0_wb(sdiodev, addr, v, ret) \
+	sdio_f0_writeb((sdiodev)->func1, (v), (addr), (ret))
+
+/* Accessors for SDIO Function 1 */
+#define brcmf_sdiod_readb(sdiodev, addr, r) \
+	sdio_readb((sdiodev)->func1, (addr), (r))
+
+#define brcmf_sdiod_writeb(sdiodev, addr, v, ret) \
+	sdio_writeb((sdiodev)->func1, (v), (addr), (ret))
+
+u32 brcmf_sdiod_readl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret);
+void brcmf_sdiod_writel(struct brcmf_sdio_dev *sdiodev, u32 addr, u32 data,
+			int *ret);
 
 /* Buffer transfer to/from device (client) core via cmd53.
  *   fn:       function number
@@ -342,7 +346,8 @@ int brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address,
 		      u8 *data, uint size);
 
 /* Issue an abort to the specified function */
-int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, uint fn);
+int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, struct sdio_func *func);
+
 void brcmf_sdiod_sgtable_alloc(struct brcmf_sdio_dev *sdiodev);
 void brcmf_sdiod_change_state(struct brcmf_sdio_dev *sdiodev,
 			      enum brcmf_sdiod_state state);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c
index 763e8ba6b178..7e01981bc5c8 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c
@@ -16049,8 +16049,7 @@ static void wlc_phy_workarounds_nphy_gainctrl(struct brcms_phy *pi)
 		wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_UPDATEGAINU,
 				       rfseq_updategainu_events,
 				       rfseq_updategainu_dlys,
-				       sizeof(rfseq_updategainu_events) /
-				       sizeof(rfseq_updategainu_events[0]));
+				       ARRAY_SIZE(rfseq_updategainu_events));
 
 		mod_phy_reg(pi, 0x153, (0xff << 8), (90 << 8));
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c
index dbf50ef6cd75..533bd4b0277e 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c
@@ -14,6 +14,7 @@
  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
+#include <linux/kernel.h>
 #include <types.h>
 #include "phytbl_n.h"
 
@@ -4437,109 +4438,39 @@ static const u16 loft_lut_core1_rev0[] = {
 };
 
 const struct phytbl_info mimophytbl_info_rev0_volatile[] = {
-	{&bdi_tbl_rev0, sizeof(bdi_tbl_rev0) / sizeof(bdi_tbl_rev0[0]), 21, 0,
-	 16}
-	,
-	{&pltlut_tbl_rev0, sizeof(pltlut_tbl_rev0) / sizeof(pltlut_tbl_rev0[0]),
-	 20, 0, 32}
-	,
-	{&gainctrl_lut_core0_rev0,
-	 sizeof(gainctrl_lut_core0_rev0) / sizeof(gainctrl_lut_core0_rev0[0]),
-	 26, 192, 32}
-	,
-	{&gainctrl_lut_core1_rev0,
-	 sizeof(gainctrl_lut_core1_rev0) / sizeof(gainctrl_lut_core1_rev0[0]),
-	 27, 192, 32}
-	,
-
-	{&est_pwr_lut_core0_rev0,
-	 sizeof(est_pwr_lut_core0_rev0) / sizeof(est_pwr_lut_core0_rev0[0]), 26,
-	 0, 8}
-	,
-	{&est_pwr_lut_core1_rev0,
-	 sizeof(est_pwr_lut_core1_rev0) / sizeof(est_pwr_lut_core1_rev0[0]), 27,
-	 0, 8}
-	,
-	{&adj_pwr_lut_core0_rev0,
-	 sizeof(adj_pwr_lut_core0_rev0) / sizeof(adj_pwr_lut_core0_rev0[0]), 26,
-	 64, 8}
-	,
-	{&adj_pwr_lut_core1_rev0,
-	 sizeof(adj_pwr_lut_core1_rev0) / sizeof(adj_pwr_lut_core1_rev0[0]), 27,
-	 64, 8}
-	,
-	{&iq_lut_core0_rev0,
-	 sizeof(iq_lut_core0_rev0) / sizeof(iq_lut_core0_rev0[0]), 26, 320, 32}
-	,
-	{&iq_lut_core1_rev0,
-	 sizeof(iq_lut_core1_rev0) / sizeof(iq_lut_core1_rev0[0]), 27, 320, 32}
-	,
-	{&loft_lut_core0_rev0,
-	 sizeof(loft_lut_core0_rev0) / sizeof(loft_lut_core0_rev0[0]), 26, 448,
-	 16}
-	,
-	{&loft_lut_core1_rev0,
-	 sizeof(loft_lut_core1_rev0) / sizeof(loft_lut_core1_rev0[0]), 27, 448,
-	 16}
-	,
+	{&bdi_tbl_rev0, ARRAY_SIZE(bdi_tbl_rev0), 21, 0, 16},
+	{&pltlut_tbl_rev0, ARRAY_SIZE(pltlut_tbl_rev0), 20, 0, 32},
+	{&gainctrl_lut_core0_rev0, ARRAY_SIZE(gainctrl_lut_core0_rev0), 26, 192, 32},
+	{&gainctrl_lut_core1_rev0, ARRAY_SIZE(gainctrl_lut_core1_rev0), 27, 192, 32},
+	{&est_pwr_lut_core0_rev0, ARRAY_SIZE(est_pwr_lut_core0_rev0), 26, 0, 8},
+	{&est_pwr_lut_core1_rev0, ARRAY_SIZE(est_pwr_lut_core1_rev0), 27, 0, 8},
+	{&adj_pwr_lut_core0_rev0, ARRAY_SIZE(adj_pwr_lut_core0_rev0), 26, 64, 8},
+	{&adj_pwr_lut_core1_rev0, ARRAY_SIZE(adj_pwr_lut_core1_rev0), 27, 64, 8},
+	{&iq_lut_core0_rev0, ARRAY_SIZE(iq_lut_core0_rev0), 26, 320, 32},
+	{&iq_lut_core1_rev0, ARRAY_SIZE(iq_lut_core1_rev0), 27, 320, 32},
+	{&loft_lut_core0_rev0, ARRAY_SIZE(loft_lut_core0_rev0), 26, 448, 16},
+	{&loft_lut_core1_rev0, ARRAY_SIZE(loft_lut_core1_rev0), 27, 448, 16},
 };
 
 const struct phytbl_info mimophytbl_info_rev0[] = {
-	{&frame_struct_rev0,
-	 sizeof(frame_struct_rev0) / sizeof(frame_struct_rev0[0]), 10, 0, 32}
-	,
-	{&frame_lut_rev0, sizeof(frame_lut_rev0) / sizeof(frame_lut_rev0[0]),
-	 24, 0, 8}
-	,
-	{&tmap_tbl_rev0, sizeof(tmap_tbl_rev0) / sizeof(tmap_tbl_rev0[0]), 12,
-	 0, 32}
-	,
-	{&tdtrn_tbl_rev0, sizeof(tdtrn_tbl_rev0) / sizeof(tdtrn_tbl_rev0[0]),
-	 14, 0, 32}
-	,
-	{&intlv_tbl_rev0, sizeof(intlv_tbl_rev0) / sizeof(intlv_tbl_rev0[0]),
-	 13, 0, 32}
-	,
-	{&pilot_tbl_rev0, sizeof(pilot_tbl_rev0) / sizeof(pilot_tbl_rev0[0]),
-	 11, 0, 16}
-	,
-	{&tdi_tbl20_ant0_rev0,
-	 sizeof(tdi_tbl20_ant0_rev0) / sizeof(tdi_tbl20_ant0_rev0[0]), 19, 128,
-	 32}
-	,
-	{&tdi_tbl20_ant1_rev0,
-	 sizeof(tdi_tbl20_ant1_rev0) / sizeof(tdi_tbl20_ant1_rev0[0]), 19, 256,
-	 32}
-	,
-	{&tdi_tbl40_ant0_rev0,
-	 sizeof(tdi_tbl40_ant0_rev0) / sizeof(tdi_tbl40_ant0_rev0[0]), 19, 640,
-	 32}
-	,
-	{&tdi_tbl40_ant1_rev0,
-	 sizeof(tdi_tbl40_ant1_rev0) / sizeof(tdi_tbl40_ant1_rev0[0]), 19, 768,
-	 32}
-	,
-	{&chanest_tbl_rev0,
-	 sizeof(chanest_tbl_rev0) / sizeof(chanest_tbl_rev0[0]), 22, 0, 32}
-	,
-	{&mcs_tbl_rev0, sizeof(mcs_tbl_rev0) / sizeof(mcs_tbl_rev0[0]), 18, 0,
-	 8}
-	,
-	{&noise_var_tbl0_rev0,
-	 sizeof(noise_var_tbl0_rev0) / sizeof(noise_var_tbl0_rev0[0]), 16, 0,
-	 32}
-	,
-	{&noise_var_tbl1_rev0,
-	 sizeof(noise_var_tbl1_rev0) / sizeof(noise_var_tbl1_rev0[0]), 16, 128,
-	 32}
-	,
+	{&frame_struct_rev0, ARRAY_SIZE(frame_struct_rev0), 10, 0, 32},
+	{&frame_lut_rev0, ARRAY_SIZE(frame_lut_rev0), 24, 0, 8},
+	{&tmap_tbl_rev0, ARRAY_SIZE(tmap_tbl_rev0), 12, 0, 32},
+	{&tdtrn_tbl_rev0, ARRAY_SIZE(tdtrn_tbl_rev0), 14, 0, 32},
+	{&intlv_tbl_rev0, ARRAY_SIZE(intlv_tbl_rev0), 13, 0, 32},
+	{&pilot_tbl_rev0, ARRAY_SIZE(pilot_tbl_rev0), 11, 0, 16},
+	{&tdi_tbl20_ant0_rev0, ARRAY_SIZE(tdi_tbl20_ant0_rev0), 19, 128, 32},
+	{&tdi_tbl20_ant1_rev0, ARRAY_SIZE(tdi_tbl20_ant1_rev0), 19, 256, 32},
+	{&tdi_tbl40_ant0_rev0, ARRAY_SIZE(tdi_tbl40_ant0_rev0), 19, 640, 32},
+	{&tdi_tbl40_ant1_rev0, ARRAY_SIZE(tdi_tbl40_ant1_rev0), 19, 768, 32},
+	{&chanest_tbl_rev0, ARRAY_SIZE(chanest_tbl_rev0), 22, 0, 32},
+	{&mcs_tbl_rev0, ARRAY_SIZE(mcs_tbl_rev0), 18, 0, 8},
+	{&noise_var_tbl0_rev0, ARRAY_SIZE(noise_var_tbl0_rev0), 16, 0, 32},
+	{&noise_var_tbl1_rev0, ARRAY_SIZE(noise_var_tbl1_rev0), 16, 128, 32},
 };
 
-const u32 mimophytbl_info_sz_rev0 =
-	sizeof(mimophytbl_info_rev0) / sizeof(mimophytbl_info_rev0[0]);
-const u32 mimophytbl_info_sz_rev0_volatile =
-	sizeof(mimophytbl_info_rev0_volatile) /
-	sizeof(mimophytbl_info_rev0_volatile[0]);
+const u32 mimophytbl_info_sz_rev0 = ARRAY_SIZE(mimophytbl_info_rev0);
+const u32 mimophytbl_info_sz_rev0_volatile = ARRAY_SIZE(mimophytbl_info_rev0_volatile);
 
 static const u16 ant_swctrl_tbl_rev3[] = {
 	0x0082,
@@ -9363,132 +9294,53 @@ static const u32 papd_cal_scalars_tbl_core1_rev3[] = {
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile[] = {
-	{&ant_swctrl_tbl_rev3,
-	 sizeof(ant_swctrl_tbl_rev3) / sizeof(ant_swctrl_tbl_rev3[0]), 9, 0, 16}
-	,
+	{&ant_swctrl_tbl_rev3, ARRAY_SIZE(ant_swctrl_tbl_rev3), 9, 0, 16},
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile1[] = {
-	{&ant_swctrl_tbl_rev3_1,
-	 sizeof(ant_swctrl_tbl_rev3_1) / sizeof(ant_swctrl_tbl_rev3_1[0]), 9, 0,
-	 16}
-	,
+	{&ant_swctrl_tbl_rev3_1, ARRAY_SIZE(ant_swctrl_tbl_rev3_1), 9, 0, 16},
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile2[] = {
-	{&ant_swctrl_tbl_rev3_2,
-	 sizeof(ant_swctrl_tbl_rev3_2) / sizeof(ant_swctrl_tbl_rev3_2[0]), 9, 0,
-	 16}
-	,
+	{&ant_swctrl_tbl_rev3_2, ARRAY_SIZE(ant_swctrl_tbl_rev3_2), 9, 0, 16},
 };
 
 const struct phytbl_info mimophytbl_info_rev3_volatile3[] = {
-	{&ant_swctrl_tbl_rev3_3,
-	 sizeof(ant_swctrl_tbl_rev3_3) / sizeof(ant_swctrl_tbl_rev3_3[0]), 9, 0,
-	 16}
-	,
+	{&ant_swctrl_tbl_rev3_3, ARRAY_SIZE(ant_swctrl_tbl_rev3_3), 9, 0, 16},
 };
 
 const struct phytbl_info mimophytbl_info_rev3[] = {
-	{&frame_struct_rev3,
-	 sizeof(frame_struct_rev3) / sizeof(frame_struct_rev3[0]), 10, 0, 32}
-	,
-	{&pilot_tbl_rev3, sizeof(pilot_tbl_rev3) / sizeof(pilot_tbl_rev3[0]),
-	 11, 0, 16}
-	,
-	{&tmap_tbl_rev3, sizeof(tmap_tbl_rev3) / sizeof(tmap_tbl_rev3[0]), 12,
-	 0, 32}
-	,
-	{&intlv_tbl_rev3, sizeof(intlv_tbl_rev3) / sizeof(intlv_tbl_rev3[0]),
-	 13, 0, 32}
-	,
-	{&tdtrn_tbl_rev3, sizeof(tdtrn_tbl_rev3) / sizeof(tdtrn_tbl_rev3[0]),
-	 14, 0, 32}
-	,
-	{&noise_var_tbl_rev3,
-	 sizeof(noise_var_tbl_rev3) / sizeof(noise_var_tbl_rev3[0]), 16, 0, 32}
-	,
-	{&mcs_tbl_rev3, sizeof(mcs_tbl_rev3) / sizeof(mcs_tbl_rev3[0]), 18, 0,
-	 16}
-	,
-	{&tdi_tbl20_ant0_rev3,
-	 sizeof(tdi_tbl20_ant0_rev3) / sizeof(tdi_tbl20_ant0_rev3[0]), 19, 128,
-	 32}
-	,
-	{&tdi_tbl20_ant1_rev3,
-	 sizeof(tdi_tbl20_ant1_rev3) / sizeof(tdi_tbl20_ant1_rev3[0]), 19, 256,
-	 32}
-	,
-	{&tdi_tbl40_ant0_rev3,
-	 sizeof(tdi_tbl40_ant0_rev3) / sizeof(tdi_tbl40_ant0_rev3[0]), 19, 640,
-	 32}
-	,
-	{&tdi_tbl40_ant1_rev3,
-	 sizeof(tdi_tbl40_ant1_rev3) / sizeof(tdi_tbl40_ant1_rev3[0]), 19, 768,
-	 32}
-	,
-	{&pltlut_tbl_rev3, sizeof(pltlut_tbl_rev3) / sizeof(pltlut_tbl_rev3[0]),
-	 20, 0, 32}
-	,
-	{&chanest_tbl_rev3,
-	 sizeof(chanest_tbl_rev3) / sizeof(chanest_tbl_rev3[0]), 22, 0, 32}
-	,
-	{&frame_lut_rev3, sizeof(frame_lut_rev3) / sizeof(frame_lut_rev3[0]),
-	 24, 0, 8}
-	,
-	{&est_pwr_lut_core0_rev3,
-	 sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26,
-	 0, 8}
-	,
-	{&est_pwr_lut_core1_rev3,
-	 sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27,
-	 0, 8}
-	,
-	{&adj_pwr_lut_core0_rev3,
-	 sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26,
-	 64, 8}
-	,
-	{&adj_pwr_lut_core1_rev3,
-	 sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27,
-	 64, 8}
-	,
-	{&gainctrl_lut_core0_rev3,
-	 sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]),
-	 26, 192, 32}
-	,
-	{&gainctrl_lut_core1_rev3,
-	 sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]),
-	 27, 192, 32}
-	,
-	{&iq_lut_core0_rev3,
-	 sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32}
-	,
-	{&iq_lut_core1_rev3,
-	 sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32}
-	,
-	{&loft_lut_core0_rev3,
-	 sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448,
-	 16}
-	,
-	{&loft_lut_core1_rev3,
-	 sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448,
-	 16}
+	{&frame_struct_rev3, ARRAY_SIZE(frame_struct_rev3), 10, 0, 32},
+	{&pilot_tbl_rev3, ARRAY_SIZE(pilot_tbl_rev3), 11, 0, 16},
+	{&tmap_tbl_rev3, ARRAY_SIZE(tmap_tbl_rev3), 12, 0, 32},
+	{&intlv_tbl_rev3, ARRAY_SIZE(intlv_tbl_rev3), 13, 0, 32},
+	{&tdtrn_tbl_rev3, ARRAY_SIZE(tdtrn_tbl_rev3), 14, 0, 32},
+	{&noise_var_tbl_rev3, ARRAY_SIZE(noise_var_tbl_rev3), 16, 0, 32},
+	{&mcs_tbl_rev3, ARRAY_SIZE(mcs_tbl_rev3), 18, 0, 16},
+	{&tdi_tbl20_ant0_rev3, ARRAY_SIZE(tdi_tbl20_ant0_rev3), 19, 128, 32},
+	{&tdi_tbl20_ant1_rev3, ARRAY_SIZE(tdi_tbl20_ant1_rev3), 19, 256, 32},
+	{&tdi_tbl40_ant0_rev3, ARRAY_SIZE(tdi_tbl40_ant0_rev3), 19, 640, 32},
+	{&tdi_tbl40_ant1_rev3, ARRAY_SIZE(tdi_tbl40_ant1_rev3), 19, 768, 32},
+	{&pltlut_tbl_rev3, ARRAY_SIZE(pltlut_tbl_rev3), 20, 0, 32},
+	{&chanest_tbl_rev3, ARRAY_SIZE(chanest_tbl_rev3), 22, 0, 32},
+	{&frame_lut_rev3, ARRAY_SIZE(frame_lut_rev3), 24, 0, 8},
+	{&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8},
+	{&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8},
+	{&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8},
+	{&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8},
+	{&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32},
+	{&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32},
+	{&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32},
+	{&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32},
+	{&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16},
+	{&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16}
 };
 
-const u32 mimophytbl_info_sz_rev3 =
-	sizeof(mimophytbl_info_rev3) / sizeof(mimophytbl_info_rev3[0]);
-const u32 mimophytbl_info_sz_rev3_volatile =
-	sizeof(mimophytbl_info_rev3_volatile) /
-	sizeof(mimophytbl_info_rev3_volatile[0]);
-const u32 mimophytbl_info_sz_rev3_volatile1 =
-	sizeof(mimophytbl_info_rev3_volatile1) /
-	sizeof(mimophytbl_info_rev3_volatile1[0]);
-const u32 mimophytbl_info_sz_rev3_volatile2 =
-	sizeof(mimophytbl_info_rev3_volatile2) /
-	sizeof(mimophytbl_info_rev3_volatile2[0]);
-const u32 mimophytbl_info_sz_rev3_volatile3 =
-	sizeof(mimophytbl_info_rev3_volatile3) /
-	sizeof(mimophytbl_info_rev3_volatile3[0]);
+const u32 mimophytbl_info_sz_rev3 = ARRAY_SIZE(mimophytbl_info_rev3);
+const u32 mimophytbl_info_sz_rev3_volatile = ARRAY_SIZE(mimophytbl_info_rev3_volatile);
+const u32 mimophytbl_info_sz_rev3_volatile1 = ARRAY_SIZE(mimophytbl_info_rev3_volatile1);
+const u32 mimophytbl_info_sz_rev3_volatile2 = ARRAY_SIZE(mimophytbl_info_rev3_volatile2);
+const u32 mimophytbl_info_sz_rev3_volatile3 = ARRAY_SIZE(mimophytbl_info_rev3_volatile3);
 
 static const u32 tmap_tbl_rev7[] = {
 	0x8a88aa80,
@@ -10469,162 +10321,58 @@ static const u32 papd_cal_scalars_tbl_core1_rev7[] = {
 };
 
 const struct phytbl_info mimophytbl_info_rev7[] = {
-	{&frame_struct_rev3,
-	 sizeof(frame_struct_rev3) / sizeof(frame_struct_rev3[0]), 10, 0, 32}
-	,
-	{&pilot_tbl_rev3, sizeof(pilot_tbl_rev3) / sizeof(pilot_tbl_rev3[0]),
-	 11, 0, 16}
-	,
-	{&tmap_tbl_rev7, sizeof(tmap_tbl_rev7) / sizeof(tmap_tbl_rev7[0]), 12,
-	 0, 32}
-	,
-	{&intlv_tbl_rev3, sizeof(intlv_tbl_rev3) / sizeof(intlv_tbl_rev3[0]),
-	 13, 0, 32}
-	,
-	{&tdtrn_tbl_rev3, sizeof(tdtrn_tbl_rev3) / sizeof(tdtrn_tbl_rev3[0]),
-	 14, 0, 32}
-	,
-	{&noise_var_tbl_rev7,
-	 sizeof(noise_var_tbl_rev7) / sizeof(noise_var_tbl_rev7[0]), 16, 0, 32}
-	,
-	{&mcs_tbl_rev3, sizeof(mcs_tbl_rev3) / sizeof(mcs_tbl_rev3[0]), 18, 0,
-	 16}
-	,
-	{&tdi_tbl20_ant0_rev3,
-	 sizeof(tdi_tbl20_ant0_rev3) / sizeof(tdi_tbl20_ant0_rev3[0]), 19, 128,
-	 32}
-	,
-	{&tdi_tbl20_ant1_rev3,
-	 sizeof(tdi_tbl20_ant1_rev3) / sizeof(tdi_tbl20_ant1_rev3[0]), 19, 256,
-	 32}
-	,
-	{&tdi_tbl40_ant0_rev3,
-	 sizeof(tdi_tbl40_ant0_rev3) / sizeof(tdi_tbl40_ant0_rev3[0]), 19, 640,
-	 32}
-	,
-	{&tdi_tbl40_ant1_rev3,
-	 sizeof(tdi_tbl40_ant1_rev3) / sizeof(tdi_tbl40_ant1_rev3[0]), 19, 768,
-	 32}
-	,
-	{&pltlut_tbl_rev3, sizeof(pltlut_tbl_rev3) / sizeof(pltlut_tbl_rev3[0]),
-	 20, 0, 32}
-	,
-	{&chanest_tbl_rev3,
-	 sizeof(chanest_tbl_rev3) / sizeof(chanest_tbl_rev3[0]), 22, 0, 32}
-	,
-	{&frame_lut_rev3, sizeof(frame_lut_rev3) / sizeof(frame_lut_rev3[0]),
-	 24, 0, 8}
-	,
-	{&est_pwr_lut_core0_rev3,
-	 sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26,
-	 0, 8}
-	,
-	{&est_pwr_lut_core1_rev3,
-	 sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27,
-	 0, 8}
-	,
-	{&adj_pwr_lut_core0_rev3,
-	 sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26,
-	 64, 8}
-	,
-	{&adj_pwr_lut_core1_rev3,
-	 sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27,
-	 64, 8}
-	,
-	{&gainctrl_lut_core0_rev3,
-	 sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]),
-	 26, 192, 32}
-	,
-	{&gainctrl_lut_core1_rev3,
-	 sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]),
-	 27, 192, 32}
-	,
-	{&iq_lut_core0_rev3,
-	 sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32}
-	,
-	{&iq_lut_core1_rev3,
-	 sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32}
-	,
-	{&loft_lut_core0_rev3,
-	 sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448,
-	 16}
-	,
-	{&loft_lut_core1_rev3,
-	 sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448,
-	 16}
-	,
+	{&frame_struct_rev3, ARRAY_SIZE(frame_struct_rev3), 10, 0, 32},
+	{&pilot_tbl_rev3, ARRAY_SIZE(pilot_tbl_rev3), 11, 0, 16},
+	{&tmap_tbl_rev7, ARRAY_SIZE(tmap_tbl_rev7), 12, 0, 32},
+	{&intlv_tbl_rev3, ARRAY_SIZE(intlv_tbl_rev3), 13, 0, 32},
+	{&tdtrn_tbl_rev3, ARRAY_SIZE(tdtrn_tbl_rev3), 14, 0, 32},
+	{&noise_var_tbl_rev7, ARRAY_SIZE(noise_var_tbl_rev7), 16, 0, 32},
+	{&mcs_tbl_rev3, ARRAY_SIZE(mcs_tbl_rev3), 18, 0, 16},
+	{&tdi_tbl20_ant0_rev3, ARRAY_SIZE(tdi_tbl20_ant0_rev3), 19, 128, 32},
+	{&tdi_tbl20_ant1_rev3, ARRAY_SIZE(tdi_tbl20_ant1_rev3), 19, 256, 32},
+	{&tdi_tbl40_ant0_rev3, ARRAY_SIZE(tdi_tbl40_ant0_rev3), 19, 640, 32},
+	{&tdi_tbl40_ant1_rev3, ARRAY_SIZE(tdi_tbl40_ant1_rev3), 19, 768, 32},
+	{&pltlut_tbl_rev3, ARRAY_SIZE(pltlut_tbl_rev3), 20, 0, 32},
+	{&chanest_tbl_rev3, ARRAY_SIZE(chanest_tbl_rev3), 22, 0, 32},
+	{&frame_lut_rev3, ARRAY_SIZE(frame_lut_rev3), 24, 0, 8},
+	{&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8},
+	{&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8},
+	{&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8},
+	{&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8},
+	{&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32},
+	{&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32},
+	{&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32},
+	{&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32},
+	{&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16},
+	{&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16},
 	{&papd_comp_rfpwr_tbl_core0_rev3,
-	 sizeof(papd_comp_rfpwr_tbl_core0_rev3) /
-	 sizeof(papd_comp_rfpwr_tbl_core0_rev3[0]), 26, 576, 16}
-	,
+	 ARRAY_SIZE(papd_comp_rfpwr_tbl_core0_rev3), 26, 576, 16},
 	{&papd_comp_rfpwr_tbl_core1_rev3,
-	 sizeof(papd_comp_rfpwr_tbl_core1_rev3) /
-	 sizeof(papd_comp_rfpwr_tbl_core1_rev3[0]), 27, 576, 16}
-	,
+	 ARRAY_SIZE(papd_comp_rfpwr_tbl_core1_rev3), 27, 576, 16},
 	{&papd_comp_epsilon_tbl_core0_rev7,
-	 sizeof(papd_comp_epsilon_tbl_core0_rev7) /
-	 sizeof(papd_comp_epsilon_tbl_core0_rev7[0]), 31, 0, 32}
-	,
+	 ARRAY_SIZE(papd_comp_epsilon_tbl_core0_rev7), 31, 0, 32},
 	{&papd_cal_scalars_tbl_core0_rev7,
-	 sizeof(papd_cal_scalars_tbl_core0_rev7) /
-	 sizeof(papd_cal_scalars_tbl_core0_rev7[0]), 32, 0, 32}
-	,
+	 ARRAY_SIZE(papd_cal_scalars_tbl_core0_rev7), 32, 0, 32},
 	{&papd_comp_epsilon_tbl_core1_rev7,
-	 sizeof(papd_comp_epsilon_tbl_core1_rev7) /
-	 sizeof(papd_comp_epsilon_tbl_core1_rev7[0]), 33, 0, 32}
-	,
+	 ARRAY_SIZE(papd_comp_epsilon_tbl_core1_rev7), 33, 0, 32},
 	{&papd_cal_scalars_tbl_core1_rev7,
-	 sizeof(papd_cal_scalars_tbl_core1_rev7) /
-	 sizeof(papd_cal_scalars_tbl_core1_rev7[0]), 34, 0, 32}
-	,
+	 ARRAY_SIZE(papd_cal_scalars_tbl_core1_rev7), 34, 0, 32},
 };
 
-const u32 mimophytbl_info_sz_rev7 =
-	sizeof(mimophytbl_info_rev7) / sizeof(mimophytbl_info_rev7[0]);
+const u32 mimophytbl_info_sz_rev7 = ARRAY_SIZE(mimophytbl_info_rev7);
 
 const struct phytbl_info mimophytbl_info_rev16[] = {
-	{&noise_var_tbl_rev7,
-	 sizeof(noise_var_tbl_rev7) / sizeof(noise_var_tbl_rev7[0]), 16, 0, 32}
-	,
-	{&est_pwr_lut_core0_rev3,
-	 sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26,
-	 0, 8}
-	,
-	{&est_pwr_lut_core1_rev3,
-	 sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27,
-	 0, 8}
-	,
-	{&adj_pwr_lut_core0_rev3,
-	 sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26,
-	 64, 8}
-	,
-	{&adj_pwr_lut_core1_rev3,
-	 sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27,
-	 64, 8}
-	,
-	{&gainctrl_lut_core0_rev3,
-	 sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]),
-	 26, 192, 32}
-	,
-	{&gainctrl_lut_core1_rev3,
-	 sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]),
-	 27, 192, 32}
-	,
-	{&iq_lut_core0_rev3,
-	 sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32}
-	,
-	{&iq_lut_core1_rev3,
-	 sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32}
-	,
-	{&loft_lut_core0_rev3,
-	 sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448,
-	 16}
-	,
-	{&loft_lut_core1_rev3,
-	 sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448,
-	 16}
-	,
+	{&noise_var_tbl_rev7, ARRAY_SIZE(noise_var_tbl_rev7), 16, 0, 32},
+	{&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8},
+	{&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8},
+	{&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8},
+	{&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8},
+	{&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32},
+	{&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32},
+	{&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32},
+	{&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32},
+	{&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16},
+	{&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16},
 };
 
-const u32 mimophytbl_info_sz_rev16 =
-	sizeof(mimophytbl_info_rev16) / sizeof(mimophytbl_info_rev16[0]);
+const u32 mimophytbl_info_sz_rev16 = ARRAY_SIZE(mimophytbl_info_rev16);
diff --git a/drivers/net/wireless/intel/iwlwifi/Kconfig b/drivers/net/wireless/intel/iwlwifi/Kconfig
index c5f2ddf9b0fe..e5a2fc738ac3 100644
--- a/drivers/net/wireless/intel/iwlwifi/Kconfig
+++ b/drivers/net/wireless/intel/iwlwifi/Kconfig
@@ -91,7 +91,6 @@ config IWLWIFI_BCAST_FILTERING
 config IWLWIFI_PCIE_RTPM
        bool "Enable runtime power management mode for PCIe devices"
        depends on IWLMVM && PM && EXPERT
-       default false
        help
          Say Y here to enable runtime power management for PCIe
          devices.  If enabled, the device will go into low power mode
diff --git a/drivers/net/wireless/intel/iwlwifi/Makefile b/drivers/net/wireless/intel/iwlwifi/Makefile
index ff136f299a0a..e6205eae51fd 100644
--- a/drivers/net/wireless/intel/iwlwifi/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/Makefile
@@ -9,12 +9,13 @@ iwlwifi-objs		+= iwl-phy-db.o iwl-nvm-parse.o
 iwlwifi-objs		+= pcie/drv.o pcie/rx.o pcie/tx.o pcie/trans.o
 iwlwifi-objs		+= pcie/ctxt-info.o pcie/trans-gen2.o pcie/tx-gen2.o
 iwlwifi-$(CONFIG_IWLDVM) += cfg/1000.o cfg/2000.o cfg/5000.o cfg/6000.o
-iwlwifi-$(CONFIG_IWLMVM) += cfg/7000.o cfg/8000.o cfg/9000.o cfg/a000.o
+iwlwifi-$(CONFIG_IWLMVM) += cfg/7000.o cfg/8000.o cfg/9000.o cfg/22000.o
 iwlwifi-objs		+= iwl-trans.o
 iwlwifi-objs		+= fw/notif-wait.o
 iwlwifi-$(CONFIG_IWLMVM) += fw/paging.o fw/smem.o fw/init.o fw/dbg.o
 iwlwifi-$(CONFIG_IWLMVM) += fw/common_rx.o fw/nvm.o
 iwlwifi-$(CONFIG_ACPI) += fw/acpi.o
+iwlwifi-$(CONFIG_IWLWIFI_DEBUGFS) += fw/debugfs.o
 
 iwlwifi-objs += $(iwlwifi-m)
 
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
new file mode 100644
index 000000000000..48f6f80eb24b
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
@@ -0,0 +1,216 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2015-2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015-2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/stringify.h>
+#include "iwl-config.h"
+#include "iwl-agn-hw.h"
+
+/* Highest firmware API version supported */
+#define IWL_22000_UCODE_API_MAX	36
+
+/* Lowest firmware API version supported */
+#define IWL_22000_UCODE_API_MIN	24
+
+/* NVM versions */
+#define IWL_22000_NVM_VERSION		0x0a1d
+#define IWL_22000_TX_POWER_VERSION	0xffff /* meaningless */
+
+/* Memory offsets and lengths */
+#define IWL_22000_DCCM_OFFSET		0x800000 /* LMAC1 */
+#define IWL_22000_DCCM_LEN		0x10000 /* LMAC1 */
+#define IWL_22000_DCCM2_OFFSET		0x880000
+#define IWL_22000_DCCM2_LEN		0x8000
+#define IWL_22000_SMEM_OFFSET		0x400000
+#define IWL_22000_SMEM_LEN		0xD0000
+
+#define IWL_22000_JF_FW_PRE	"iwlwifi-Qu-a0-jf-b0-"
+#define IWL_22000_HR_FW_PRE	"iwlwifi-Qu-a0-hr-a0-"
+#define IWL_22000_HR_CDB_FW_PRE	"iwlwifi-QuIcp-z0-hrcdb-a0-"
+#define IWL_22000_HR_F0_FW_PRE	"iwlwifi-QuQnj-f0-hr-a0-"
+#define IWL_22000_JF_B0_FW_PRE	"iwlwifi-QuQnj-a0-jf-b0-"
+#define IWL_22000_HR_A0_FW_PRE	"iwlwifi-QuQnj-a0-hr-a0-"
+
+#define IWL_22000_HR_MODULE_FIRMWARE(api) \
+	IWL_22000_HR_FW_PRE __stringify(api) ".ucode"
+#define IWL_22000_JF_MODULE_FIRMWARE(api) \
+	IWL_22000_JF_FW_PRE __stringify(api) ".ucode"
+#define IWL_22000_HR_F0_QNJ_MODULE_FIRMWARE(api) \
+	IWL_22000_HR_F0_FW_PRE __stringify(api) ".ucode"
+#define IWL_22000_JF_B0_QNJ_MODULE_FIRMWARE(api) \
+	IWL_22000_JF_B0_FW_PRE __stringify(api) ".ucode"
+#define IWL_22000_HR_A0_QNJ_MODULE_FIRMWARE(api) \
+	IWL_22000_HR_A0_FW_PRE __stringify(api) ".ucode"
+
+#define NVM_HW_SECTION_NUM_FAMILY_22000		10
+
+static const struct iwl_base_params iwl_22000_base_params = {
+	.eeprom_size = OTP_LOW_IMAGE_SIZE_FAMILY_22000,
+	.num_of_queues = 512,
+	.shadow_ram_support = true,
+	.led_compensation = 57,
+	.wd_timeout = IWL_LONG_WD_TIMEOUT,
+	.max_event_log_size = 512,
+	.shadow_reg_enable = true,
+	.pcie_l1_allowed = true,
+};
+
+static const struct iwl_ht_params iwl_22000_ht_params = {
+	.stbc = true,
+	.ldpc = true,
+	.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
+};
+
+#define IWL_DEVICE_22000						\
+	.ucode_api_max = IWL_22000_UCODE_API_MAX,			\
+	.ucode_api_min = IWL_22000_UCODE_API_MIN,			\
+	.device_family = IWL_DEVICE_FAMILY_22000,			\
+	.max_inst_size = IWL60_RTC_INST_SIZE,				\
+	.max_data_size = IWL60_RTC_DATA_SIZE,				\
+	.base_params = &iwl_22000_base_params,				\
+	.led_mode = IWL_LED_RF_STATE,					\
+	.nvm_hw_section_num = NVM_HW_SECTION_NUM_FAMILY_22000,		\
+	.non_shared_ant = ANT_A,					\
+	.dccm_offset = IWL_22000_DCCM_OFFSET,				\
+	.dccm_len = IWL_22000_DCCM_LEN,					\
+	.dccm2_offset = IWL_22000_DCCM2_OFFSET,				\
+	.dccm2_len = IWL_22000_DCCM2_LEN,				\
+	.smem_offset = IWL_22000_SMEM_OFFSET,				\
+	.smem_len = IWL_22000_SMEM_LEN,					\
+	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,		\
+	.apmg_not_supported = true,					\
+	.mq_rx_supported = true,					\
+	.vht_mu_mimo_supported = true,					\
+	.mac_addr_from_csr = true,					\
+	.use_tfh = true,						\
+	.rf_id = true,							\
+	.gen2 = true,							\
+	.nvm_type = IWL_NVM_EXT,					\
+	.dbgc_supported = true,						\
+	.tx_cmd_queue_size = 32,					\
+	.min_umac_error_event_table = 0x400000
+
+const struct iwl_cfg iwl22000_2ac_cfg_hr = {
+	.name = "Intel(R) Dual Band Wireless AC 22000",
+	.fw_name_pre = IWL_22000_HR_FW_PRE,
+	IWL_DEVICE_22000,
+	.ht_params = &iwl_22000_ht_params,
+	.nvm_ver = IWL_22000_NVM_VERSION,
+	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwl22000_2ac_cfg_hr_cdb = {
+	.name = "Intel(R) Dual Band Wireless AC 22000",
+	.fw_name_pre = IWL_22000_HR_CDB_FW_PRE,
+	IWL_DEVICE_22000,
+	.ht_params = &iwl_22000_ht_params,
+	.nvm_ver = IWL_22000_NVM_VERSION,
+	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.cdb = true,
+};
+
+const struct iwl_cfg iwl22000_2ac_cfg_jf = {
+	.name = "Intel(R) Dual Band Wireless AC 22000",
+	.fw_name_pre = IWL_22000_JF_FW_PRE,
+	IWL_DEVICE_22000,
+	.ht_params = &iwl_22000_ht_params,
+	.nvm_ver = IWL_22000_NVM_VERSION,
+	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwl22000_2ax_cfg_hr = {
+	.name = "Intel(R) Dual Band Wireless AX 22000",
+	.fw_name_pre = IWL_22000_HR_FW_PRE,
+	IWL_DEVICE_22000,
+	.ht_params = &iwl_22000_ht_params,
+	.nvm_ver = IWL_22000_NVM_VERSION,
+	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwl22000_2ax_cfg_qnj_hr_f0 = {
+	.name = "Intel(R) Dual Band Wireless AX 22000",
+	.fw_name_pre = IWL_22000_HR_F0_FW_PRE,
+	IWL_DEVICE_22000,
+	.ht_params = &iwl_22000_ht_params,
+	.nvm_ver = IWL_22000_NVM_VERSION,
+	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwl22000_2ax_cfg_qnj_jf_b0 = {
+	.name = "Intel(R) Dual Band Wireless AX 22000",
+	.fw_name_pre = IWL_22000_JF_B0_FW_PRE,
+	IWL_DEVICE_22000,
+	.ht_params = &iwl_22000_ht_params,
+	.nvm_ver = IWL_22000_NVM_VERSION,
+	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+const struct iwl_cfg iwl22000_2ax_cfg_qnj_hr_a0 = {
+	.name = "Intel(R) Dual Band Wireless AX 22000",
+	.fw_name_pre = IWL_22000_HR_A0_FW_PRE,
+	IWL_DEVICE_22000,
+	.ht_params = &iwl_22000_ht_params,
+	.nvm_ver = IWL_22000_NVM_VERSION,
+	.nvm_calib_ver = IWL_22000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+};
+
+MODULE_FIRMWARE(IWL_22000_HR_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_22000_JF_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_22000_HR_F0_QNJ_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_22000_JF_B0_QNJ_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_22000_HR_A0_QNJ_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
index 9bb7c19d48eb..3f4d9bac9f73 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
@@ -70,8 +70,8 @@
 #include "iwl-agn-hw.h"
 
 /* Highest firmware API version supported */
-#define IWL8000_UCODE_API_MAX	34
-#define IWL8265_UCODE_API_MAX	34
+#define IWL8000_UCODE_API_MAX	36
+#define IWL8265_UCODE_API_MAX	36
 
 /* Lowest firmware API version supported */
 #define IWL8000_UCODE_API_MIN	22
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
index e7e75b458005..ab1469473e4f 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
@@ -53,9 +53,10 @@
 #include <linux/stringify.h>
 #include "iwl-config.h"
 #include "iwl-agn-hw.h"
+#include "fw/file.h"
 
 /* Highest firmware API version supported */
-#define IWL9000_UCODE_API_MAX	34
+#define IWL9000_UCODE_API_MAX	36
 
 /* Lowest firmware API version supported */
 #define IWL9000_UCODE_API_MIN	30
@@ -265,6 +266,67 @@ const struct iwl_cfg iwl9560_2ac_cfg_soc = {
 	.integrated = true,
 	.soc_latency = 5000,
 };
+
+const struct iwl_cfg iwl9460_2ac_cfg_shared_clk = {
+	.name = "Intel(R) Dual Band Wireless AC 9460",
+	.fw_name_pre = IWL9000A_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9000B_FW_PRE,
+	.fw_name_pre_rf_next_step = IWL9000RFB_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.integrated = true,
+	.soc_latency = 5000,
+	.extra_phy_cfg_flags = FW_PHY_CFG_SHARED_CLK
+};
+
+const struct iwl_cfg iwl9461_2ac_cfg_shared_clk = {
+	.name = "Intel(R) Dual Band Wireless AC 9461",
+	.fw_name_pre = IWL9000A_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9000B_FW_PRE,
+	.fw_name_pre_rf_next_step = IWL9000RFB_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.integrated = true,
+	.soc_latency = 5000,
+	.extra_phy_cfg_flags = FW_PHY_CFG_SHARED_CLK
+};
+
+const struct iwl_cfg iwl9462_2ac_cfg_shared_clk = {
+	.name = "Intel(R) Dual Band Wireless AC 9462",
+	.fw_name_pre = IWL9000A_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9000B_FW_PRE,
+	.fw_name_pre_rf_next_step = IWL9000RFB_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.integrated = true,
+	.soc_latency = 5000,
+	.extra_phy_cfg_flags = FW_PHY_CFG_SHARED_CLK
+};
+
+const struct iwl_cfg iwl9560_2ac_cfg_shared_clk = {
+	.name = "Intel(R) Dual Band Wireless AC 9560",
+	.fw_name_pre = IWL9000A_FW_PRE,
+	.fw_name_pre_b_or_c_step = IWL9000B_FW_PRE,
+	.fw_name_pre_rf_next_step = IWL9000RFB_FW_PRE,
+	IWL_DEVICE_9000,
+	.ht_params = &iwl9000_ht_params,
+	.nvm_ver = IWL9000_NVM_VERSION,
+	.nvm_calib_ver = IWL9000_TX_POWER_VERSION,
+	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
+	.integrated = true,
+	.soc_latency = 5000,
+	.extra_phy_cfg_flags = FW_PHY_CFG_SHARED_CLK
+};
+
 MODULE_FIRMWARE(IWL9000A_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL9000B_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL9000RFB_MODULE_FIRMWARE(IWL9000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/a000.c b/drivers/net/wireless/intel/iwlwifi/cfg/a000.c
deleted file mode 100644
index 705f83b02e13..000000000000
--- a/drivers/net/wireless/intel/iwlwifi/cfg/a000.c
+++ /dev/null
@@ -1,216 +0,0 @@
-/******************************************************************************
- *
- * This file is provided under a dual BSD/GPLv2 license.  When using or
- * redistributing this file, you may do so under either license.
- *
- * GPL LICENSE SUMMARY
- *
- * Copyright(c) 2015-2017 Intel Deutschland GmbH
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of version 2 of the GNU General Public License as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * BSD LICENSE
- *
- * Copyright(c) 2015-2017 Intel Deutschland GmbH
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- *  * Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- *  * Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *  * Neither the name Intel Corporation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- *****************************************************************************/
-
-#include <linux/module.h>
-#include <linux/stringify.h>
-#include "iwl-config.h"
-#include "iwl-agn-hw.h"
-
-/* Highest firmware API version supported */
-#define IWL_A000_UCODE_API_MAX	34
-
-/* Lowest firmware API version supported */
-#define IWL_A000_UCODE_API_MIN	24
-
-/* NVM versions */
-#define IWL_A000_NVM_VERSION		0x0a1d
-#define IWL_A000_TX_POWER_VERSION	0xffff /* meaningless */
-
-/* Memory offsets and lengths */
-#define IWL_A000_DCCM_OFFSET		0x800000 /* LMAC1 */
-#define IWL_A000_DCCM_LEN		0x10000 /* LMAC1 */
-#define IWL_A000_DCCM2_OFFSET		0x880000
-#define IWL_A000_DCCM2_LEN		0x8000
-#define IWL_A000_SMEM_OFFSET		0x400000
-#define IWL_A000_SMEM_LEN		0xD0000
-
-#define IWL_A000_JF_FW_PRE	"iwlwifi-Qu-a0-jf-b0-"
-#define IWL_A000_HR_FW_PRE	"iwlwifi-Qu-a0-hr-a0-"
-#define IWL_A000_HR_CDB_FW_PRE	"iwlwifi-QuIcp-z0-hrcdb-a0-"
-#define IWL_A000_HR_F0_FW_PRE	"iwlwifi-QuQnj-f0-hr-a0-"
-#define IWL_A000_JF_B0_FW_PRE	"iwlwifi-QuQnj-a0-jf-b0-"
-#define IWL_A000_HR_A0_FW_PRE	"iwlwifi-QuQnj-a0-hr-a0-"
-
-#define IWL_A000_HR_MODULE_FIRMWARE(api) \
-	IWL_A000_HR_FW_PRE __stringify(api) ".ucode"
-#define IWL_A000_JF_MODULE_FIRMWARE(api) \
-	IWL_A000_JF_FW_PRE __stringify(api) ".ucode"
-#define IWL_A000_HR_F0_QNJ_MODULE_FIRMWARE(api) \
-	IWL_A000_HR_F0_FW_PRE __stringify(api) ".ucode"
-#define IWL_A000_JF_B0_QNJ_MODULE_FIRMWARE(api) \
-	IWL_A000_JF_B0_FW_PRE __stringify(api) ".ucode"
-#define IWL_A000_HR_A0_QNJ_MODULE_FIRMWARE(api) \
-	IWL_A000_HR_A0_FW_PRE __stringify(api) ".ucode"
-
-#define NVM_HW_SECTION_NUM_FAMILY_A000		10
-
-static const struct iwl_base_params iwl_a000_base_params = {
-	.eeprom_size = OTP_LOW_IMAGE_SIZE_FAMILY_A000,
-	.num_of_queues = 512,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
-	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
-	.shadow_reg_enable = true,
-	.pcie_l1_allowed = true,
-};
-
-static const struct iwl_ht_params iwl_a000_ht_params = {
-	.stbc = true,
-	.ldpc = true,
-	.ht40_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ),
-};
-
-#define IWL_DEVICE_A000							\
-	.ucode_api_max = IWL_A000_UCODE_API_MAX,			\
-	.ucode_api_min = IWL_A000_UCODE_API_MIN,			\
-	.device_family = IWL_DEVICE_FAMILY_A000,			\
-	.max_inst_size = IWL60_RTC_INST_SIZE,				\
-	.max_data_size = IWL60_RTC_DATA_SIZE,				\
-	.base_params = &iwl_a000_base_params,				\
-	.led_mode = IWL_LED_RF_STATE,					\
-	.nvm_hw_section_num = NVM_HW_SECTION_NUM_FAMILY_A000,		\
-	.non_shared_ant = ANT_A,					\
-	.dccm_offset = IWL_A000_DCCM_OFFSET,				\
-	.dccm_len = IWL_A000_DCCM_LEN,					\
-	.dccm2_offset = IWL_A000_DCCM2_OFFSET,				\
-	.dccm2_len = IWL_A000_DCCM2_LEN,				\
-	.smem_offset = IWL_A000_SMEM_OFFSET,				\
-	.smem_len = IWL_A000_SMEM_LEN,					\
-	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,		\
-	.apmg_not_supported = true,					\
-	.mq_rx_supported = true,					\
-	.vht_mu_mimo_supported = true,					\
-	.mac_addr_from_csr = true,					\
-	.use_tfh = true,						\
-	.rf_id = true,							\
-	.gen2 = true,							\
-	.nvm_type = IWL_NVM_EXT,					\
-	.dbgc_supported = true,						\
-	.tx_cmd_queue_size = 32,					\
-	.min_umac_error_event_table = 0x400000
-
-const struct iwl_cfg iwla000_2ac_cfg_hr = {
-	.name = "Intel(R) Dual Band Wireless AC a000",
-	.fw_name_pre = IWL_A000_HR_FW_PRE,
-	IWL_DEVICE_A000,
-	.ht_params = &iwl_a000_ht_params,
-	.nvm_ver = IWL_A000_NVM_VERSION,
-	.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-};
-
-const struct iwl_cfg iwla000_2ac_cfg_hr_cdb = {
-	.name = "Intel(R) Dual Band Wireless AC a000",
-	.fw_name_pre = IWL_A000_HR_CDB_FW_PRE,
-	IWL_DEVICE_A000,
-	.ht_params = &iwl_a000_ht_params,
-	.nvm_ver = IWL_A000_NVM_VERSION,
-	.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-	.cdb = true,
-};
-
-const struct iwl_cfg iwla000_2ac_cfg_jf = {
-	.name = "Intel(R) Dual Band Wireless AC a000",
-	.fw_name_pre = IWL_A000_JF_FW_PRE,
-	IWL_DEVICE_A000,
-	.ht_params = &iwl_a000_ht_params,
-	.nvm_ver = IWL_A000_NVM_VERSION,
-	.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-};
-
-const struct iwl_cfg iwla000_2ax_cfg_hr = {
-	.name = "Intel(R) Dual Band Wireless AX a000",
-	.fw_name_pre = IWL_A000_HR_FW_PRE,
-	IWL_DEVICE_A000,
-	.ht_params = &iwl_a000_ht_params,
-	.nvm_ver = IWL_A000_NVM_VERSION,
-	.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-};
-
-const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_f0 = {
-	.name = "Intel(R) Dual Band Wireless AX a000",
-	.fw_name_pre = IWL_A000_HR_F0_FW_PRE,
-	IWL_DEVICE_A000,
-	.ht_params = &iwl_a000_ht_params,
-	.nvm_ver = IWL_A000_NVM_VERSION,
-	.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-};
-
-const struct iwl_cfg iwla000_2ax_cfg_qnj_jf_b0 = {
-	.name = "Intel(R) Dual Band Wireless AX a000",
-	.fw_name_pre = IWL_A000_JF_B0_FW_PRE,
-	IWL_DEVICE_A000,
-	.ht_params = &iwl_a000_ht_params,
-	.nvm_ver = IWL_A000_NVM_VERSION,
-	.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-};
-
-const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_a0 = {
-	.name = "Intel(R) Dual Band Wireless AX a000",
-	.fw_name_pre = IWL_A000_HR_A0_FW_PRE,
-	IWL_DEVICE_A000,
-	.ht_params = &iwl_a000_ht_params,
-	.nvm_ver = IWL_A000_NVM_VERSION,
-	.nvm_calib_ver = IWL_A000_TX_POWER_VERSION,
-	.max_ht_ampdu_exponent = IEEE80211_HT_MAX_AMPDU_64K,
-};
-
-MODULE_FIRMWARE(IWL_A000_HR_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_A000_JF_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_A000_HR_F0_QNJ_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_A000_JF_B0_QNJ_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_A000_HR_A0_QNJ_MODULE_FIRMWARE(IWL_A000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/alive.h b/drivers/net/wireless/intel/iwlwifi/fw/api/alive.h
index 3684a3e180e5..007bfe7656a4 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/alive.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/alive.h
@@ -95,8 +95,8 @@ enum {
 #define IWL_ALIVE_FLG_RFKILL	BIT(0)
 
 struct iwl_lmac_alive {
-	__le32 ucode_minor;
 	__le32 ucode_major;
+	__le32 ucode_minor;
 	u8 ver_subtype;
 	u8 ver_type;
 	u8 mac;
@@ -113,8 +113,8 @@ struct iwl_lmac_alive {
 } __packed; /* UCODE_ALIVE_NTFY_API_S_VER_3 */
 
 struct iwl_umac_alive {
-	__le32 umac_minor;		/* UMAC version: minor */
 	__le32 umac_major;		/* UMAC version: major */
+	__le32 umac_minor;		/* UMAC version: minor */
 	__le32 error_info_addr;		/* SRAM address for UMAC error log */
 	__le32 dbg_print_buff_addr;
 } __packed; /* UMAC_ALIVE_DATA_API_S_VER_2 */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h b/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h
index d09555afe2c5..87c1ddea75ae 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/coex.h
@@ -188,11 +188,6 @@ enum iwl_bt_mxbox_dw3 {
 	BT_MBOX(3, UPDATE_REQUEST, 21, 1),
 };
 
-enum iwl_bt_mxbox_dw4 {
-	BT_MBOX(4, ATS_BT_INTERVAL, 0, 7),
-	BT_MBOX(4, ATS_BT_ACTIVE_MAX_TH, 7, 7),
-};
-
 #define BT_MBOX_MSG(_notif, _num, _field)				     \
 	((le32_to_cpu((_notif)->mbox_msg[(_num)]) & BT_MBOX##_num##_##_field)\
 	>> BT_MBOX##_num##_##_field##_POS)
@@ -232,31 +227,6 @@ enum iwl_bt_ci_compliance {
  * @reserved: reserved
  */
 struct iwl_bt_coex_profile_notif {
-	__le32 mbox_msg[8];
-	__le32 msg_idx;
-	__le32 bt_ci_compliance;
-
-	__le32 primary_ch_lut;
-	__le32 secondary_ch_lut;
-	__le32 bt_activity_grading;
-	u8 ttc_status;
-	u8 rrc_status;
-	__le16 reserved;
-} __packed; /* BT_COEX_PROFILE_NTFY_API_S_VER_5 */
-
-/**
- * struct iwl_bt_coex_profile_notif - notification about BT coex
- * @mbox_msg: message from BT to WiFi
- * @msg_idx: the index of the message
- * @bt_ci_compliance: enum %iwl_bt_ci_compliance
- * @primary_ch_lut: LUT used for primary channel &enum iwl_bt_coex_lut_type
- * @secondary_ch_lut: LUT used for secondary channel &enum iwl_bt_coex_lut_type
- * @bt_activity_grading: the activity of BT &enum iwl_bt_activity_grading
- * @ttc_status: is TTC enabled - one bit per PHY
- * @rrc_status: is RRC enabled - one bit per PHY
- * @reserved: reserved
- */
-struct iwl_bt_coex_profile_notif_v4 {
 	__le32 mbox_msg[4];
 	__le32 msg_idx;
 	__le32 bt_ci_compliance;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
index 7ebbf097488b..f285bacc8726 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
@@ -215,7 +215,7 @@ enum iwl_legacy_cmds {
 	/**
 	 * @SCD_QUEUE_CFG: &struct iwl_scd_txq_cfg_cmd for older hardware,
 	 *	&struct iwl_tx_queue_cfg_cmd with &struct iwl_tx_queue_cfg_rsp
-	 *	for newer (A000) hardware.
+	 *	for newer (22000) hardware.
 	 */
 	SCD_QUEUE_CFG = 0x1d,
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
index aa76dcc148bd..a57c7223df0f 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
@@ -82,6 +82,21 @@ enum iwl_data_path_subcmd_ids {
 	 */
 	TRIGGER_RX_QUEUES_NOTIF_CMD = 0x2,
 
+	/**
+	 * @TLC_MNG_CONFIG_CMD: &struct iwl_tlc_config_cmd
+	 */
+	TLC_MNG_CONFIG_CMD = 0xF,
+
+	/**
+	 * @TLC_MNG_NOTIF_REQ_CMD: &struct iwl_tlc_notif_req_config_cmd
+	 */
+	TLC_MNG_NOTIF_REQ_CMD = 0x10,
+
+	/**
+	 * @TLC_MNG_UPDATE_NOTIF: &struct iwl_tlc_update_notif
+	 */
+	TLC_MNG_UPDATE_NOTIF = 0xF7,
+
 	/**
 	 * @STA_PM_NOTIF: &struct iwl_mvm_pm_state_notification
 	 */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
index 0a81fb1b6ed4..106782341544 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/debug.h
@@ -250,10 +250,12 @@ struct iwl_mfu_assert_dump_notif {
  * The ids for different type of markers to insert into the usniffer logs
  *
  * @MARKER_ID_TX_FRAME_LATENCY: TX latency marker
+ * @MARKER_ID_SYNC_CLOCK: sync FW time and systime
  */
 enum iwl_mvm_marker_id {
 	MARKER_ID_TX_FRAME_LATENCY = 1,
-}; /* MARKER_ID_API_E_VER_1 */
+	MARKER_ID_SYNC_CLOCK = 2,
+}; /* MARKER_ID_API_E_VER_2 */
 
 /**
  * struct iwl_mvm_marker - mark info into the usniffer logs
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
index ec42c84e5df2..17c7ef1662a9 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
@@ -67,6 +67,10 @@
  * enum iwl_mac_conf_subcmd_ids - mac configuration command IDs
  */
 enum iwl_mac_conf_subcmd_ids {
+	/**
+	 * @LOW_LATENCY_CMD: &struct iwl_mac_low_latency_cmd
+	 */
+	LOW_LATENCY_CMD = 0x3,
 	/**
 	 * @CHANNEL_SWITCH_NOA_NOTIF: &struct iwl_channel_switch_noa_notif
 	 */
@@ -82,4 +86,19 @@ struct iwl_channel_switch_noa_notif {
 	__le32 id_and_color;
 } __packed; /* CHANNEL_SWITCH_START_NTFY_API_S_VER_1 */
 
+/**
+ * struct iwl_mac_low_latency_cmd - set/clear mac to 'low-latency mode'
+ *
+ * @mac_id: MAC ID to whom to apply the low-latency configurations
+ * @low_latency_rx: 1/0 to set/clear Rx low latency direction
+ * @low_latency_tx: 1/0 to set/clear Tx low latency direction
+ * @reserved: reserved for alignment purposes
+ */
+struct iwl_mac_low_latency_cmd {
+	__le32 mac_id;
+	u8 low_latency_rx;
+	u8 low_latency_tx;
+	__le16 reserved;
+} __packed; /* MAC_LOW_LATENCY_API_S_VER_1 */
+
 #endif /* __iwl_fw_api_mac_cfg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
index a13fd8a1be62..e49a6f7be613 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rs.h
@@ -62,6 +62,267 @@
 
 #include "mac.h"
 
+/**
+ * enum iwl_tlc_mng_cfg_flags_enum - options for TLC config flags
+ * @IWL_TLC_MNG_CFG_FLAGS_CCK_MSK: CCK support
+ * @IWL_TLC_MNG_CFG_FLAGS_DD_MSK: enable DD
+ * @IWL_TLC_MNG_CFG_FLAGS_STBC_MSK: enable STBC
+ * @IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK: enable LDPC
+ * @IWL_TLC_MNG_CFG_FLAGS_BF_MSK: enable BFER
+ * @IWL_TLC_MNG_CFG_FLAGS_DCM_MSK: enable DCM
+ */
+enum iwl_tlc_mng_cfg_flags {
+	IWL_TLC_MNG_CFG_FLAGS_CCK_MSK	= BIT(0),
+	IWL_TLC_MNG_CFG_FLAGS_DD_MSK	= BIT(1),
+	IWL_TLC_MNG_CFG_FLAGS_STBC_MSK	= BIT(2),
+	IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK	= BIT(3),
+	IWL_TLC_MNG_CFG_FLAGS_BF_MSK	= BIT(4),
+	IWL_TLC_MNG_CFG_FLAGS_DCM_MSK	= BIT(5),
+};
+
+/**
+ * enum iwl_tlc_mng_cfg_cw - channel width options
+ * @IWL_TLC_MNG_MAX_CH_WIDTH_20MHZ: 20MHZ channel
+ * @IWL_TLC_MNG_MAX_CH_WIDTH_40MHZ: 40MHZ channel
+ * @IWL_TLC_MNG_MAX_CH_WIDTH_80MHZ: 80MHZ channel
+ * @IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ: 160MHZ channel
+ * @IWL_TLC_MNG_MAX_CH_WIDTH_LAST: maximum value
+ */
+enum iwl_tlc_mng_cfg_cw {
+	IWL_TLC_MNG_MAX_CH_WIDTH_20MHZ,
+	IWL_TLC_MNG_MAX_CH_WIDTH_40MHZ,
+	IWL_TLC_MNG_MAX_CH_WIDTH_80MHZ,
+	IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ,
+	IWL_TLC_MNG_MAX_CH_WIDTH_LAST = IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ,
+};
+
+/**
+ * enum iwl_tlc_mng_cfg_chains - possible chains
+ * @IWL_TLC_MNG_CHAIN_A_MSK: chain A
+ * @IWL_TLC_MNG_CHAIN_B_MSK: chain B
+ * @IWL_TLC_MNG_CHAIN_C_MSK: chain C
+ */
+enum iwl_tlc_mng_cfg_chains {
+	IWL_TLC_MNG_CHAIN_A_MSK = BIT(0),
+	IWL_TLC_MNG_CHAIN_B_MSK = BIT(1),
+	IWL_TLC_MNG_CHAIN_C_MSK = BIT(2),
+};
+
+/**
+ * enum iwl_tlc_mng_cfg_gi - guard interval options
+ * @IWL_TLC_MNG_SGI_20MHZ_MSK: enable short GI for 20MHZ
+ * @IWL_TLC_MNG_SGI_40MHZ_MSK: enable short GI for 40MHZ
+ * @IWL_TLC_MNG_SGI_80MHZ_MSK: enable short GI for 80MHZ
+ * @IWL_TLC_MNG_SGI_160MHZ_MSK: enable short GI for 160MHZ
+ */
+enum iwl_tlc_mng_cfg_gi {
+	IWL_TLC_MNG_SGI_20MHZ_MSK  = BIT(0),
+	IWL_TLC_MNG_SGI_40MHZ_MSK  = BIT(1),
+	IWL_TLC_MNG_SGI_80MHZ_MSK  = BIT(2),
+	IWL_TLC_MNG_SGI_160MHZ_MSK = BIT(3),
+};
+
+/**
+ * enum iwl_tlc_mng_cfg_mode - supported modes
+ * @IWL_TLC_MNG_MODE_CCK: enable CCK
+ * @IWL_TLC_MNG_MODE_OFDM_NON_HT: enable OFDM (non HT)
+ * @IWL_TLC_MNG_MODE_NON_HT: enable non HT
+ * @IWL_TLC_MNG_MODE_HT: enable HT
+ * @IWL_TLC_MNG_MODE_VHT: enable VHT
+ * @IWL_TLC_MNG_MODE_HE: enable HE
+ * @IWL_TLC_MNG_MODE_INVALID: invalid value
+ * @IWL_TLC_MNG_MODE_NUM: a count of possible modes
+ */
+enum iwl_tlc_mng_cfg_mode {
+	IWL_TLC_MNG_MODE_CCK = 0,
+	IWL_TLC_MNG_MODE_OFDM_NON_HT = IWL_TLC_MNG_MODE_CCK,
+	IWL_TLC_MNG_MODE_NON_HT = IWL_TLC_MNG_MODE_CCK,
+	IWL_TLC_MNG_MODE_HT,
+	IWL_TLC_MNG_MODE_VHT,
+	IWL_TLC_MNG_MODE_HE,
+	IWL_TLC_MNG_MODE_INVALID,
+	IWL_TLC_MNG_MODE_NUM = IWL_TLC_MNG_MODE_INVALID,
+};
+
+/**
+ * enum iwl_tlc_mng_vht_he_types - VHT HE types
+ * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU: VHT HT single user
+ * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU_EXT: VHT HT single user extended
+ * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_MU: VHT HT multiple users
+ * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_TRIG_BASED: trigger based
+ * @IWL_TLC_MNG_VALID_VHT_HE_TYPES_NUM: a count of possible types
+ */
+enum iwl_tlc_mng_vht_he_types {
+	IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU = 0,
+	IWL_TLC_MNG_VALID_VHT_HE_TYPES_SU_EXT,
+	IWL_TLC_MNG_VALID_VHT_HE_TYPES_MU,
+	IWL_TLC_MNG_VALID_VHT_HE_TYPES_TRIG_BASED,
+	IWL_TLC_MNG_VALID_VHT_HE_TYPES_NUM =
+		IWL_TLC_MNG_VALID_VHT_HE_TYPES_TRIG_BASED,
+
+};
+
+/**
+ * enum iwl_tlc_mng_ht_rates - HT/VHT rates
+ * @IWL_TLC_MNG_HT_RATE_MCS0: index of MCS0
+ * @IWL_TLC_MNG_HT_RATE_MCS1: index of MCS1
+ * @IWL_TLC_MNG_HT_RATE_MCS2: index of MCS2
+ * @IWL_TLC_MNG_HT_RATE_MCS3: index of MCS3
+ * @IWL_TLC_MNG_HT_RATE_MCS4: index of MCS4
+ * @IWL_TLC_MNG_HT_RATE_MCS5: index of MCS5
+ * @IWL_TLC_MNG_HT_RATE_MCS6: index of MCS6
+ * @IWL_TLC_MNG_HT_RATE_MCS7: index of MCS7
+ * @IWL_TLC_MNG_HT_RATE_MCS8: index of MCS8
+ * @IWL_TLC_MNG_HT_RATE_MCS9: index of MCS9
+ * @IWL_TLC_MNG_HT_RATE_MAX: maximal rate for HT/VHT
+ */
+enum iwl_tlc_mng_ht_rates {
+	IWL_TLC_MNG_HT_RATE_MCS0 = 0,
+	IWL_TLC_MNG_HT_RATE_MCS1,
+	IWL_TLC_MNG_HT_RATE_MCS2,
+	IWL_TLC_MNG_HT_RATE_MCS3,
+	IWL_TLC_MNG_HT_RATE_MCS4,
+	IWL_TLC_MNG_HT_RATE_MCS5,
+	IWL_TLC_MNG_HT_RATE_MCS6,
+	IWL_TLC_MNG_HT_RATE_MCS7,
+	IWL_TLC_MNG_HT_RATE_MCS8,
+	IWL_TLC_MNG_HT_RATE_MCS9,
+	IWL_TLC_MNG_HT_RATE_MAX = IWL_TLC_MNG_HT_RATE_MCS9,
+};
+
+/* Maximum supported tx antennas number */
+#define MAX_RS_ANT_NUM 3
+
+/**
+ * struct tlc_config_cmd - TLC configuration
+ * @sta_id: station id
+ * @reserved1: reserved
+ * @max_supp_ch_width: channel width
+ * @flags: bitmask of &enum iwl_tlc_mng_cfg_flags
+ * @chains: bitmask of &enum iwl_tlc_mng_cfg_chains
+ * @max_supp_ss: valid values are 0-3, 0 - spatial streams are not supported
+ * @valid_vht_he_types: bitmap of &enum iwl_tlc_mng_vht_he_types
+ * @non_ht_supp_rates: bitmap of supported legacy rates
+ * @ht_supp_rates: bitmap of supported HT/VHT rates, valid bits are 0-9
+ * @mode: &enum iwl_tlc_mng_cfg_mode
+ * @reserved2: reserved
+ * @he_supp_rates: bitmap of supported HE rates
+ * @sgi_ch_width_supp: bitmap of SGI support per channel width
+ * @he_gi_support: 11ax HE guard interval
+ * @max_ampdu_cnt: max AMPDU size (frames count)
+ */
+struct iwl_tlc_config_cmd {
+	u8 sta_id;
+	u8 reserved1[3];
+	u8 max_supp_ch_width;
+	u8 chains;
+	u8 max_supp_ss;
+	u8 valid_vht_he_types;
+	__le16 flags;
+	__le16 non_ht_supp_rates;
+	__le16 ht_supp_rates[MAX_RS_ANT_NUM];
+	u8 mode;
+	u8 reserved2;
+	__le16 he_supp_rates;
+	u8 sgi_ch_width_supp;
+	u8 he_gi_support;
+	__le32 max_ampdu_cnt;
+} __packed; /* TLC_MNG_CONFIG_CMD_API_S_VER_1 */
+
+#define IWL_TLC_NOTIF_INIT_RATE_POS 0
+#define IWL_TLC_NOTIF_INIT_RATE_MSK BIT(IWL_TLC_NOTIF_INIT_RATE_POS)
+#define IWL_TLC_NOTIF_REQ_INTERVAL (500)
+
+/**
+ * struct iwl_tlc_notif_req_config_cmd - request notif on specific changes
+ * @sta_id: relevant station
+ * @reserved1: reserved
+ * @flags: bitmap of requested notifications %IWL_TLC_NOTIF_INIT_\*
+ * @interval: minimum time between notifications from TLC to the driver (msec)
+ * @reserved2: reserved
+ */
+struct iwl_tlc_notif_req_config_cmd {
+	u8 sta_id;
+	u8 reserved1;
+	__le16 flags;
+	__le16 interval;
+	__le16 reserved2;
+} __packed; /* TLC_MNG_NOTIF_REQ_CMD_API_S_VER_1 */
+
+/**
+ * struct iwl_tlc_update_notif - TLC notification from FW
+ * @sta_id: station id
+ * @reserved: reserved
+ * @flags: bitmap of notifications reported
+ * @values: field per flag in struct iwl_tlc_notif_req_config_cmd
+ */
+struct iwl_tlc_update_notif {
+	u8 sta_id;
+	u8 reserved;
+	__le16 flags;
+	__le32 values[16];
+} __packed; /* TLC_MNG_UPDATE_NTFY_API_S_VER_1 */
+
+/**
+ * enum iwl_tlc_debug_flags - debug options
+ * @IWL_TLC_DEBUG_FIXED_RATE: set fixed rate for rate scaling
+ * @IWL_TLC_DEBUG_STATS_TH: threshold for sending statistics to the driver, in
+ *	frames
+ * @IWL_TLC_DEBUG_STATS_TIME_TH: threshold for sending statistics to the
+ *	driver, in msec
+ * @IWL_TLC_DEBUG_AGG_TIME_LIM: time limit for a BA session
+ * @IWL_TLC_DEBUG_AGG_DIS_START_TH: frame with try-count greater than this
+ *	threshold should not start an aggregation session
+ * @IWL_TLC_DEBUG_AGG_FRAME_CNT_LIM: set max number of frames in an aggregation
+ * @IWL_TLC_DEBUG_RENEW_ADDBA_DELAY: delay between retries of ADD BA
+ * @IWL_TLC_DEBUG_START_AC_RATE_IDX: frames per second to start a BA session
+ * @IWL_TLC_DEBUG_NO_FAR_RANGE_TWEAK: disable BW scaling
+ */
+enum iwl_tlc_debug_flags {
+	IWL_TLC_DEBUG_FIXED_RATE,
+	IWL_TLC_DEBUG_STATS_TH,
+	IWL_TLC_DEBUG_STATS_TIME_TH,
+	IWL_TLC_DEBUG_AGG_TIME_LIM,
+	IWL_TLC_DEBUG_AGG_DIS_START_TH,
+	IWL_TLC_DEBUG_AGG_FRAME_CNT_LIM,
+	IWL_TLC_DEBUG_RENEW_ADDBA_DELAY,
+	IWL_TLC_DEBUG_START_AC_RATE_IDX,
+	IWL_TLC_DEBUG_NO_FAR_RANGE_TWEAK,
+}; /* TLC_MNG_DEBUG_FLAGS_API_E_VER_1 */
+
+/**
+ * struct iwl_dhc_tlc_dbg - fixed debug config
+ * @sta_id: bit 0 - enable/disable, bits 1 - 7 hold station id
+ * @reserved1: reserved
+ * @flags: bitmap of %IWL_TLC_DEBUG_\*
+ * @fixed_rate: rate value
+ * @stats_threshold: if number of tx-ed frames is greater, send statistics
+ * @time_threshold: statistics threshold in usec
+ * @agg_time_lim: max agg time
+ * @agg_dis_start_threshold: frames with try-cont greater than this count will
+ *			     not be aggregated
+ * @agg_frame_count_lim: agg size
+ * @addba_retry_delay: delay between retries of ADD BA
+ * @start_ac_rate_idx: frames per second to start a BA session
+ * @no_far_range_tweak: disable BW scaling
+ * @reserved2: reserved
+ */
+struct iwl_dhc_tlc_cmd {
+	u8 sta_id;
+	u8 reserved1[3];
+	__le32 flags;
+	__le32 fixed_rate;
+	__le16 stats_threshold;
+	__le16 time_threshold;
+	__le16 agg_time_lim;
+	__le16 agg_dis_start_threshold;
+	__le16 agg_frame_count_lim;
+	__le16 addba_retry_delay;
+	u8 start_ac_rate_idx[IEEE80211_NUM_ACS];
+	u8 no_far_range_tweak;
+	u8 reserved2[3];
+} __packed;
+
 /*
  * These serve as indexes into
  * struct iwl_rate_info fw_rate_idx_to_plcp[IWL_RATE_COUNT];
@@ -253,7 +514,6 @@ enum {
 #define RATE_MCS_ANT_ABC_MSK		(RATE_MCS_ANT_AB_MSK | \
 					 RATE_MCS_ANT_C_MSK)
 #define RATE_MCS_ANT_MSK		RATE_MCS_ANT_ABC_MSK
-#define RATE_MCS_ANT_NUM 3
 
 /* Bit 17: (0) SS, (1) SS*2 */
 #define RATE_MCS_STBC_POS		17
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/time-event.h b/drivers/net/wireless/intel/iwlwifi/fw/api/time-event.h
index 3721a3ed358b..f824bebceb06 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/time-event.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/time-event.h
@@ -211,7 +211,7 @@ enum {
  * @TE_V2_NOTIF_HOST_FRAG_END:request/receive notification on frag end
  * @TE_V2_NOTIF_INTERNAL_FRAG_START: internal FW use.
  * @TE_V2_NOTIF_INTERNAL_FRAG_END: internal FW use.
- * @T2_V2_START_IMMEDIATELY: start time event immediately
+ * @TE_V2_START_IMMEDIATELY: start time event immediately
  * @TE_V2_DEP_OTHER: depends on another time event
  * @TE_V2_DEP_TSF: depends on a specific time
  * @TE_V2_EVENT_SOCIOPATHIC: can't co-exist with other events of tha same MAC
@@ -230,7 +230,7 @@ enum iwl_time_event_policy {
 	TE_V2_NOTIF_HOST_FRAG_END = BIT(5),
 	TE_V2_NOTIF_INTERNAL_FRAG_START = BIT(6),
 	TE_V2_NOTIF_INTERNAL_FRAG_END = BIT(7),
-	T2_V2_START_IMMEDIATELY = BIT(11),
+	TE_V2_START_IMMEDIATELY = BIT(11),
 
 	/* placement characteristics */
 	TE_V2_DEP_OTHER = BIT(TE_V2_PLACEMENT_POS),
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
index f5d5ba7e37ec..a2a40b515a3c 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/tx.h
@@ -121,7 +121,7 @@ enum iwl_tx_flags {
 }; /* TX_FLAGS_BITS_API_S_VER_1 */
 
 /**
- * enum iwl_tx_cmd_flags - bitmasks for tx_flags in TX command for a000
+ * enum iwl_tx_cmd_flags - bitmasks for tx_flags in TX command for 22000
  * @IWL_TX_FLAGS_CMD_RATE: use rate from the TX command
  * @IWL_TX_FLAGS_ENCRYPT_DIS: frame should not be encrypted, even if it belongs
  *	to a secured STA
@@ -301,7 +301,7 @@ struct iwl_dram_sec_info {
 } __packed; /* DRAM_SEC_INFO_API_S_VER_1 */
 
 /**
- * struct iwl_tx_cmd_gen2 - TX command struct to FW for a000 devices
+ * struct iwl_tx_cmd_gen2 - TX command struct to FW for 22000 devices
  * ( TX_CMD = 0x1c )
  * @len: in bytes of the payload, see below for details
  * @offload_assist: TX offload configuration
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
index 8106fd4be996..7bd704a3e640 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -33,6 +34,7 @@
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -942,7 +944,6 @@ void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt)
 
 out:
 	iwl_fw_free_dump_desc(fwrt);
-	fwrt->dump.trig = NULL;
 	clear_bit(IWL_FWRT_STATUS_DUMPING, &fwrt->status);
 	IWL_DEBUG_INFO(fwrt, "WRT dump done\n");
 }
@@ -964,7 +965,20 @@ int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
 	if (trigger)
 		delay = msecs_to_jiffies(le32_to_cpu(trigger->stop_delay));
 
-	if (WARN(fwrt->trans->state == IWL_TRANS_NO_FW,
+	/*
+	 * If the loading of the FW completed successfully, the next step is to
+	 * get the SMEM config data. Thus, if fwrt->smem_cfg.num_lmacs is non
+	 * zero, the FW was already loaded successully. If the state is "NO_FW"
+	 * in such a case - WARN and exit, since FW may be dead. Otherwise, we
+	 * can try to collect the data, since FW might just not be fully
+	 * loaded (no "ALIVE" yet), and the debug data is accessible.
+	 *
+	 * Corner case: got the FW alive but crashed before getting the SMEM
+	 *	config. In such a case, due to HW access problems, we might
+	 *	collect garbage.
+	 */
+	if (WARN((fwrt->trans->state == IWL_TRANS_NO_FW) &&
+		 fwrt->smem_cfg.num_lmacs,
 		 "Can't collect dbg data when FW isn't alive\n"))
 		return -EIO;
 
@@ -1099,6 +1113,14 @@ void iwl_fw_error_dump_wk(struct work_struct *work)
 	    fwrt->ops->dump_start(fwrt->ops_ctx))
 		return;
 
+	if (fwrt->ops && fwrt->ops->fw_running &&
+	    !fwrt->ops->fw_running(fwrt->ops_ctx)) {
+		IWL_ERR(fwrt, "Firmware not running - cannot dump error\n");
+		iwl_fw_free_dump_desc(fwrt);
+		clear_bit(IWL_FWRT_STATUS_DUMPING, &fwrt->status);
+		goto out;
+	}
+
 	if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_7000) {
 		/* stop recording */
 		iwl_fw_dbg_stop_recording(fwrt);
@@ -1132,7 +1154,7 @@ void iwl_fw_error_dump_wk(struct work_struct *work)
 			iwl_write_prph(fwrt->trans, DBGC_OUT_CTRL, out_ctrl);
 		}
 	}
-
+out:
 	if (fwrt->ops && fwrt->ops->dump_end)
 		fwrt->ops->dump_end(fwrt->ops_ctx);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
index 223fb77a3aa9..72259bff9922 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
@@ -8,6 +8,7 @@
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -33,6 +34,7 @@
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2015 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -91,6 +93,7 @@ static inline void iwl_fw_free_dump_desc(struct iwl_fw_runtime *fwrt)
 	if (fwrt->dump.desc != &iwl_dump_desc_assert)
 		kfree(fwrt->dump.desc);
 	fwrt->dump.desc = NULL;
+	fwrt->dump.trig = NULL;
 }
 
 void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
new file mode 100644
index 000000000000..e2ded29a145d
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
@@ -0,0 +1,195 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#include "api/commands.h"
+#include "debugfs.h"
+
+#define FWRT_DEBUGFS_READ_FILE_OPS(name)				\
+static ssize_t iwl_dbgfs_##name##_read(struct iwl_fw_runtime *fwrt,	\
+				       char *buf, size_t count,		\
+				       loff_t *ppos);			\
+static const struct file_operations iwl_dbgfs_##name##_ops = {		\
+	.read = iwl_dbgfs_##name##_read,				\
+	.open = simple_open,						\
+	.llseek = generic_file_llseek,					\
+}
+
+#define FWRT_DEBUGFS_WRITE_WRAPPER(name, buflen)			\
+static ssize_t iwl_dbgfs_##name##_write(struct iwl_fw_runtime *fwrt,	\
+					char *buf, size_t count,	\
+					loff_t *ppos);			\
+static ssize_t _iwl_dbgfs_##name##_write(struct file *file,		\
+					 const char __user *user_buf,	\
+					 size_t count, loff_t *ppos)	\
+{									\
+	struct iwl_fw_runtime *fwrt = file->private_data;		\
+	char buf[buflen] = {};						\
+	size_t buf_size = min(count, sizeof(buf) -  1);			\
+									\
+	if (copy_from_user(buf, user_buf, buf_size))			\
+		return -EFAULT;						\
+									\
+	return iwl_dbgfs_##name##_write(fwrt, buf, buf_size, ppos);	\
+}
+
+#define FWRT_DEBUGFS_READ_WRITE_FILE_OPS(name, buflen)			\
+FWRT_DEBUGFS_WRITE_WRAPPER(name, buflen)				\
+static const struct file_operations iwl_dbgfs_##name##_ops = {		\
+	.write = _iwl_dbgfs_##name##_write,				\
+	.read = iwl_dbgfs_##name##_read,				\
+	.open = simple_open,						\
+	.llseek = generic_file_llseek,					\
+}
+
+#define FWRT_DEBUGFS_WRITE_FILE_OPS(name, buflen)			\
+FWRT_DEBUGFS_WRITE_WRAPPER(name, buflen)				\
+static const struct file_operations iwl_dbgfs_##name##_ops = {		\
+	.write = _iwl_dbgfs_##name##_write,				\
+	.open = simple_open,						\
+	.llseek = generic_file_llseek,					\
+}
+
+#define FWRT_DEBUGFS_ADD_FILE_ALIAS(alias, name, parent, mode) do {	\
+		if (!debugfs_create_file(alias, mode, parent, fwrt,	\
+					 &iwl_dbgfs_##name##_ops))	\
+			goto err;					\
+	} while (0)
+#define FWRT_DEBUGFS_ADD_FILE(name, parent, mode) \
+	FWRT_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
+
+static int iwl_fw_send_timestamp_marker_cmd(struct iwl_fw_runtime *fwrt)
+{
+	struct iwl_mvm_marker marker = {
+		.dw_len = sizeof(struct iwl_mvm_marker) / 4,
+		.marker_id = MARKER_ID_SYNC_CLOCK,
+
+		/* the real timestamp is taken from the ftrace clock
+		 * this is for finding the match between fw and kernel logs
+		 */
+		.timestamp = cpu_to_le64(fwrt->timestamp.seq++),
+	};
+
+	struct iwl_host_cmd hcmd = {
+		.id = MARKER_CMD,
+		.flags = CMD_ASYNC,
+		.data[0] = &marker,
+		.len[0] = sizeof(marker),
+	};
+
+	return iwl_trans_send_cmd(fwrt->trans, &hcmd);
+}
+
+static void iwl_fw_timestamp_marker_wk(struct work_struct *work)
+{
+	int ret;
+	struct iwl_fw_runtime *fwrt =
+		container_of(work, struct iwl_fw_runtime, timestamp.wk.work);
+	unsigned long delay = fwrt->timestamp.delay;
+
+	ret = iwl_fw_send_timestamp_marker_cmd(fwrt);
+	if (!ret && delay)
+		schedule_delayed_work(&fwrt->timestamp.wk,
+				      round_jiffies_relative(delay));
+	else
+		IWL_INFO(fwrt,
+			 "stopping timestamp_marker, ret: %d, delay: %u\n",
+			 ret, jiffies_to_msecs(delay) / 1000);
+}
+
+static ssize_t iwl_dbgfs_timestamp_marker_write(struct iwl_fw_runtime *fwrt,
+						char *buf, size_t count,
+						loff_t *ppos)
+{
+	int ret;
+	u32 delay;
+
+	ret = kstrtou32(buf, 10, &delay);
+	if (ret < 0)
+		return ret;
+
+	IWL_INFO(fwrt,
+		 "starting timestamp_marker trigger with delay: %us\n",
+		 delay);
+
+	iwl_fw_cancel_timestamp(fwrt);
+
+	fwrt->timestamp.delay = msecs_to_jiffies(delay * 1000);
+
+	schedule_delayed_work(&fwrt->timestamp.wk,
+			      round_jiffies_relative(fwrt->timestamp.delay));
+	return count;
+}
+
+FWRT_DEBUGFS_WRITE_FILE_OPS(timestamp_marker, 10);
+
+int iwl_fwrt_dbgfs_register(struct iwl_fw_runtime *fwrt,
+			    struct dentry *dbgfs_dir)
+{
+	INIT_DELAYED_WORK(&fwrt->timestamp.wk, iwl_fw_timestamp_marker_wk);
+	FWRT_DEBUGFS_ADD_FILE(timestamp_marker, dbgfs_dir, S_IWUSR);
+	return 0;
+err:
+	IWL_ERR(fwrt, "Can't create the fwrt debugfs directory\n");
+	return -ENOMEM;
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.h b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.h
new file mode 100644
index 000000000000..3da468d2cc92
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.h
@@ -0,0 +1,105 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+
+#include "runtime.h"
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+int iwl_fwrt_dbgfs_register(struct iwl_fw_runtime *fwrt,
+			    struct dentry *dbgfs_dir);
+
+static inline void iwl_fw_cancel_timestamp(struct iwl_fw_runtime *fwrt)
+{
+	fwrt->timestamp.delay = 0;
+	cancel_delayed_work_sync(&fwrt->timestamp.wk);
+}
+
+static inline void iwl_fw_suspend_timestamp(struct iwl_fw_runtime *fwrt)
+{
+	cancel_delayed_work_sync(&fwrt->timestamp.wk);
+}
+
+static inline void iwl_fw_resume_timestamp(struct iwl_fw_runtime *fwrt)
+{
+	if (!fwrt->timestamp.delay)
+		return;
+
+	schedule_delayed_work(&fwrt->timestamp.wk,
+			      round_jiffies_relative(fwrt->timestamp.delay));
+}
+
+#else
+static inline int iwl_fwrt_dbgfs_register(struct iwl_fw_runtime *fwrt,
+					  struct dentry *dbgfs_dir)
+{
+	return 0;
+}
+
+static inline void iwl_fw_cancel_timestamp(struct iwl_fw_runtime *fwrt) {}
+
+static inline void iwl_fw_suspend_timestamp(struct iwl_fw_runtime *fwrt) {}
+
+static inline void iwl_fw_resume_timestamp(struct iwl_fw_runtime *fwrt) {}
+
+#endif /* CONFIG_IWLWIFI_DEBUGFS */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index 37a5c5b4eda6..2cb303c5c42e 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -7,7 +7,7 @@
  *
  * Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2016        Intel Deutschland GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -34,7 +34,7 @@
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2016        Intel Deutschland GmbH
+ * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -246,10 +246,10 @@ typedef unsigned int __bitwise iwl_ucode_tlv_api_t;
  * @IWL_UCODE_TLV_API_STA_TYPE: This ucode supports station type assignement.
  * @IWL_UCODE_TLV_API_NAN2_VER2: This ucode supports NAN API version 2
  * @IWL_UCODE_TLV_API_NEW_RX_STATS: should new RX STATISTICS API be used
- * @IWL_UCODE_TLV_API_ATS_COEX_EXTERNAL: the coex notification is enlared to
- *	include information about ACL time sharing.
  * @IWL_UCODE_TLV_API_QUOTA_LOW_LATENCY: Quota command includes a field
  *	indicating low latency direction.
+ * @IWL_UCODE_TLV_API_DEPRECATE_TTAK: RX status flag TTAK ok (bit 7) is
+ *	deprecated.
  *
  * @NUM_IWL_UCODE_TLV_API: number of bits used
  */
@@ -267,8 +267,8 @@ enum iwl_ucode_tlv_api {
 	IWL_UCODE_TLV_API_ADAPTIVE_DWELL	= (__force iwl_ucode_tlv_api_t)32,
 	IWL_UCODE_TLV_API_NEW_BEACON_TEMPLATE	= (__force iwl_ucode_tlv_api_t)34,
 	IWL_UCODE_TLV_API_NEW_RX_STATS		= (__force iwl_ucode_tlv_api_t)35,
-	IWL_UCODE_TLV_API_COEX_ATS_EXTERNAL	= (__force iwl_ucode_tlv_api_t)37,
 	IWL_UCODE_TLV_API_QUOTA_LOW_LATENCY	= (__force iwl_ucode_tlv_api_t)38,
+	IWL_UCODE_TLV_API_DEPRECATE_TTAK	= (__force iwl_ucode_tlv_api_t)41,
 
 	NUM_IWL_UCODE_TLV_API
 #ifdef __CHECKER__
@@ -313,6 +313,8 @@ typedef unsigned int __bitwise iwl_ucode_tlv_capa_t;
  * @IWL_UCODE_TLV_CAPA_BT_COEX_RRC: supports BT Coex RRC
  * @IWL_UCODE_TLV_CAPA_GSCAN_SUPPORT: supports gscan
  * @IWL_UCODE_TLV_CAPA_STA_PM_NOTIF: firmware will send STA PM notification
+ * @IWL_UCODE_TLV_CAPA_TLC_OFFLOAD: firmware implements rate scaling algorithm
+ * @IWL_UCODE_TLV_CAPA_DYNAMIC_QUOTA: firmware implements quota related
  * @IWL_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE: extended DTS measurement
  * @IWL_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS: supports short PM timeouts
  * @IWL_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT: supports bt-coex Multi-priority LUT
@@ -367,6 +369,8 @@ enum iwl_ucode_tlv_capa {
 	IWL_UCODE_TLV_CAPA_BINDING_CDB_SUPPORT		= (__force iwl_ucode_tlv_capa_t)39,
 	IWL_UCODE_TLV_CAPA_CDB_SUPPORT			= (__force iwl_ucode_tlv_capa_t)40,
 	IWL_UCODE_TLV_CAPA_D0I3_END_FIRST		= (__force iwl_ucode_tlv_capa_t)41,
+	IWL_UCODE_TLV_CAPA_TLC_OFFLOAD                  = (__force iwl_ucode_tlv_capa_t)43,
+	IWL_UCODE_TLV_CAPA_DYNAMIC_QUOTA                = (__force iwl_ucode_tlv_capa_t)44,
 	IWL_UCODE_TLV_CAPA_EXTENDED_DTS_MEASURE		= (__force iwl_ucode_tlv_capa_t)64,
 	IWL_UCODE_TLV_CAPA_SHORT_PM_TIMEOUTS		= (__force iwl_ucode_tlv_capa_t)65,
 	IWL_UCODE_TLV_CAPA_BT_MPLUT_SUPPORT		= (__force iwl_ucode_tlv_capa_t)67,
@@ -437,6 +441,7 @@ enum iwl_fw_phy_cfg {
 	FW_PHY_CFG_TX_CHAIN = 0xf << FW_PHY_CFG_TX_CHAIN_POS,
 	FW_PHY_CFG_RX_CHAIN_POS = 20,
 	FW_PHY_CFG_RX_CHAIN = 0xf << FW_PHY_CFG_RX_CHAIN_POS,
+	FW_PHY_CFG_SHARED_CLK = BIT(31),
 };
 
 #define IWL_UCODE_MAX_CS		1
@@ -541,7 +546,7 @@ struct iwl_fw_dbg_mem_seg_tlv {
 } __packed;
 
 /**
- * struct iwl_fw_dbg_dest_tlv - configures the destination of the debug data
+ * struct iwl_fw_dbg_dest_tlv_v1 - configures the destination of the debug data
  *
  * @version: version of the TLV - currently 0
  * @monitor_mode: &enum iwl_fw_dbg_monitor_mode
@@ -556,7 +561,7 @@ struct iwl_fw_dbg_mem_seg_tlv {
  *
  * This parses IWL_UCODE_TLV_FW_DBG_DEST
  */
-struct iwl_fw_dbg_dest_tlv {
+struct iwl_fw_dbg_dest_tlv_v1 {
 	u8 version;
 	u8 monitor_mode;
 	u8 size_power;
@@ -570,6 +575,26 @@ struct iwl_fw_dbg_dest_tlv {
 	struct iwl_fw_dbg_reg_op reg_ops[0];
 } __packed;
 
+/* Mask of the register for defining the LDBG MAC2SMEM buffer SMEM size */
+#define IWL_LDBG_M2S_BUF_SIZE_MSK	0x0fff0000
+/* Mask of the register for defining the LDBG MAC2SMEM SMEM base address */
+#define IWL_LDBG_M2S_BUF_BA_MSK		0x00000fff
+/* The smem buffer chunks are in units of 256 bits */
+#define IWL_M2S_UNIT_SIZE			0x100
+
+struct iwl_fw_dbg_dest_tlv {
+	u8 version;
+	u8 monitor_mode;
+	u8 size_power;
+	u8 reserved;
+	__le32 cfg_reg;
+	__le32 write_ptr_reg;
+	__le32 wrap_count;
+	u8 base_shift;
+	u8 size_shift;
+	struct iwl_fw_dbg_reg_op reg_ops[0];
+} __packed;
+
 struct iwl_fw_dbg_conf_hcmd {
 	u8 id;
 	u8 reserved;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/img.h b/drivers/net/wireless/intel/iwlwifi/fw/img.h
index 985496cc01d0..b23ffe12ad84 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/img.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/img.h
@@ -284,7 +284,7 @@ struct iwl_fw {
 	struct iwl_fw_cipher_scheme cs[IWL_UCODE_MAX_CS];
 	u8 human_readable[FW_VER_HUMAN_READABLE_SZ];
 
-	struct iwl_fw_dbg_dest_tlv *dbg_dest_tlv;
+	struct iwl_fw_dbg_dest_tlv_v1 *dbg_dest_tlv;
 	struct iwl_fw_dbg_conf_tlv *dbg_conf_tlv[FW_DBG_CONF_MAX];
 	size_t dbg_conf_tlv_len[FW_DBG_CONF_MAX];
 	struct iwl_fw_dbg_trigger_tlv *dbg_trigger_tlv[FW_DBG_TRIGGER_MAX];
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/init.c b/drivers/net/wireless/intel/iwlwifi/fw/init.c
index bfe5316bbb6a..2efac307909e 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/init.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/init.c
@@ -58,10 +58,12 @@
 #include "iwl-drv.h"
 #include "runtime.h"
 #include "dbg.h"
+#include "debugfs.h"
 
 void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
-			 const struct iwl_fw *fw,
-			 const struct iwl_fw_runtime_ops *ops, void *ops_ctx)
+			const struct iwl_fw *fw,
+			const struct iwl_fw_runtime_ops *ops, void *ops_ctx,
+			struct dentry *dbgfs_dir)
 {
 	memset(fwrt, 0, sizeof(*fwrt));
 	fwrt->trans = trans;
@@ -71,5 +73,18 @@ void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
 	fwrt->ops = ops;
 	fwrt->ops_ctx = ops_ctx;
 	INIT_DELAYED_WORK(&fwrt->dump.wk, iwl_fw_error_dump_wk);
+	iwl_fwrt_dbgfs_register(fwrt, dbgfs_dir);
 }
 IWL_EXPORT_SYMBOL(iwl_fw_runtime_init);
+
+void iwl_fw_runtime_suspend(struct iwl_fw_runtime *fwrt)
+{
+	iwl_fw_suspend_timestamp(fwrt);
+}
+IWL_EXPORT_SYMBOL(iwl_fw_runtime_suspend);
+
+void iwl_fw_runtime_resume(struct iwl_fw_runtime *fwrt)
+{
+	iwl_fw_resume_timestamp(fwrt);
+}
+IWL_EXPORT_SYMBOL(iwl_fw_runtime_resume);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
index 50cfb6d795a5..3fb940ebd74a 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/runtime.h
@@ -6,6 +6,7 @@
  * GPL LICENSE SUMMARY
  *
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -26,6 +27,7 @@
  * BSD LICENSE
  *
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -68,6 +70,7 @@
 struct iwl_fw_runtime_ops {
 	int (*dump_start)(void *ctx);
 	void (*dump_end)(void *ctx);
+	bool (*fw_running)(void *ctx);
 };
 
 #define MAX_NUM_LMAC 2
@@ -134,11 +137,25 @@ struct iwl_fw_runtime {
 		/* ts of the beginning of a non-collect fw dbg data period */
 		unsigned long non_collect_ts_start[FW_DBG_TRIGGER_MAX - 1];
 	} dump;
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	struct {
+		struct delayed_work wk;
+		u32 delay;
+		u64 seq;
+	} timestamp;
+#endif /* CONFIG_IWLWIFI_DEBUGFS */
 };
 
 void iwl_fw_runtime_init(struct iwl_fw_runtime *fwrt, struct iwl_trans *trans,
-			 const struct iwl_fw *fw,
-			 const struct iwl_fw_runtime_ops *ops, void *ops_ctx);
+			const struct iwl_fw *fw,
+			const struct iwl_fw_runtime_ops *ops, void *ops_ctx,
+			struct dentry *dbgfs_dir);
+
+void iwl_fw_runtime_exit(struct iwl_fw_runtime *fwrt);
+
+void iwl_fw_runtime_suspend(struct iwl_fw_runtime *fwrt);
+
+void iwl_fw_runtime_resume(struct iwl_fw_runtime *fwrt);
 
 static inline void iwl_fw_set_current_image(struct iwl_fw_runtime *fwrt,
 					    enum iwl_ucode_type cur_fw_img)
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/smem.c b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
index 76675736ba4f..fb4b6442b4d7 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/smem.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/smem.c
@@ -63,8 +63,8 @@
 #include "runtime.h"
 #include "fw/api/commands.h"
 
-static void iwl_parse_shared_mem_a000(struct iwl_fw_runtime *fwrt,
-				      struct iwl_rx_packet *pkt)
+static void iwl_parse_shared_mem_22000(struct iwl_fw_runtime *fwrt,
+				       struct iwl_rx_packet *pkt)
 {
 	struct iwl_shared_mem_cfg *mem_cfg = (void *)pkt->data;
 	int i, lmac;
@@ -143,8 +143,8 @@ void iwl_get_shared_mem_conf(struct iwl_fw_runtime *fwrt)
 		return;
 
 	pkt = cmd.resp_pkt;
-	if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_A000)
-		iwl_parse_shared_mem_a000(fwrt, pkt);
+	if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_22000)
+		iwl_parse_shared_mem_22000(fwrt, pkt);
 	else
 		iwl_parse_shared_mem(fwrt, pkt);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index e21e46cf6f9a..f0f5636dd3ea 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -89,7 +89,7 @@ enum iwl_device_family {
 	IWL_DEVICE_FAMILY_7000,
 	IWL_DEVICE_FAMILY_8000,
 	IWL_DEVICE_FAMILY_9000,
-	IWL_DEVICE_FAMILY_A000,
+	IWL_DEVICE_FAMILY_22000,
 };
 
 /*
@@ -266,7 +266,7 @@ struct iwl_tt_params {
 #define OTP_LOW_IMAGE_SIZE_FAMILY_7000	(16 * 512 * sizeof(u16)) /* 16 KB */
 #define OTP_LOW_IMAGE_SIZE_FAMILY_8000	(32 * 512 * sizeof(u16)) /* 32 KB */
 #define OTP_LOW_IMAGE_SIZE_FAMILY_9000	OTP_LOW_IMAGE_SIZE_FAMILY_8000
-#define OTP_LOW_IMAGE_SIZE_FAMILY_A000	OTP_LOW_IMAGE_SIZE_FAMILY_9000
+#define OTP_LOW_IMAGE_SIZE_FAMILY_22000	OTP_LOW_IMAGE_SIZE_FAMILY_9000
 
 struct iwl_eeprom_params {
 	const u8 regulatory_bands[7];
@@ -330,7 +330,7 @@ struct iwl_pwr_tx_backoff {
  * @vht_mu_mimo_supported: VHT MU-MIMO support
  * @rf_id: need to read rf_id to determine the firmware image
  * @integrated: discrete or integrated
- * @gen2: a000 and on transport operation
+ * @gen2: 22000 and on transport operation
  * @cdb: CDB support
  * @nvm_type: see &enum iwl_nvm_type
  * @tx_cmd_queue_size: size of the cmd queue. If zero, use the same value as
@@ -398,6 +398,7 @@ struct iwl_cfg {
 	u8 ucode_api_max;
 	u8 ucode_api_min;
 	u32 min_umac_error_event_table;
+	u32 extra_phy_cfg_flags;
 };
 
 /*
@@ -477,13 +478,17 @@ extern const struct iwl_cfg iwl9460_2ac_cfg_soc;
 extern const struct iwl_cfg iwl9461_2ac_cfg_soc;
 extern const struct iwl_cfg iwl9462_2ac_cfg_soc;
 extern const struct iwl_cfg iwl9560_2ac_cfg_soc;
-extern const struct iwl_cfg iwla000_2ac_cfg_hr;
-extern const struct iwl_cfg iwla000_2ac_cfg_hr_cdb;
-extern const struct iwl_cfg iwla000_2ac_cfg_jf;
-extern const struct iwl_cfg iwla000_2ax_cfg_hr;
-extern const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_f0;
-extern const struct iwl_cfg iwla000_2ax_cfg_qnj_jf_b0;
-extern const struct iwl_cfg iwla000_2ax_cfg_qnj_hr_a0;
+extern const struct iwl_cfg iwl9460_2ac_cfg_shared_clk;
+extern const struct iwl_cfg iwl9461_2ac_cfg_shared_clk;
+extern const struct iwl_cfg iwl9462_2ac_cfg_shared_clk;
+extern const struct iwl_cfg iwl9560_2ac_cfg_shared_clk;
+extern const struct iwl_cfg iwl22000_2ac_cfg_hr;
+extern const struct iwl_cfg iwl22000_2ac_cfg_hr_cdb;
+extern const struct iwl_cfg iwl22000_2ac_cfg_jf;
+extern const struct iwl_cfg iwl22000_2ax_cfg_hr;
+extern const struct iwl_cfg iwl22000_2ax_cfg_qnj_hr_f0;
+extern const struct iwl_cfg iwl22000_2ax_cfg_qnj_jf_b0;
+extern const struct iwl_cfg iwl22000_2ax_cfg_qnj_hr_a0;
 #endif /* CONFIG_IWLMVM */
 
 #endif /* __IWL_CONFIG_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h
index 7f16dcce0995..9518a82f44c2 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-devtrace-iwlwifi.h
@@ -95,7 +95,7 @@ TRACE_EVENT(iwlwifi_dev_tx,
 	TP_ARGS(dev, skb, tfd, tfdlen, buf0, buf0_len, hdr_len),
 	TP_STRUCT__entry(
 		DEV_ENTRY
-
+		__field(void *, skbaddr)
 		__field(size_t, framelen)
 		__dynamic_array(u8, tfd, tfdlen)
 
@@ -110,6 +110,7 @@ TRACE_EVENT(iwlwifi_dev_tx,
 	),
 	TP_fast_assign(
 		DEV_ASSIGN;
+		__entry->skbaddr = skb;
 		__entry->framelen = buf0_len;
 		if (hdr_len > 0)
 			__entry->framelen += skb->len - hdr_len;
@@ -120,9 +121,9 @@ TRACE_EVENT(iwlwifi_dev_tx,
 				      __get_dynamic_array(buf1),
 				      skb->len - hdr_len);
 	),
-	TP_printk("[%s] TX %.2x (%zu bytes)",
+	TP_printk("[%s] TX %.2x (%zu bytes) skbaddr=%p",
 		  __get_str(dev), ((u8 *)__get_dynamic_array(buf0))[0],
-		  __entry->framelen)
+		  __entry->framelen, __entry->skbaddr)
 );
 
 TRACE_EVENT(iwlwifi_dev_ucode_error,
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
index 4b224d7d967c..9c4a7f648a44 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
@@ -296,7 +296,12 @@ struct iwl_firmware_pieces {
 	u32 inst_evtlog_ptr, inst_evtlog_size, inst_errlog_ptr;
 
 	/* FW debug data parsed for driver usage */
-	struct iwl_fw_dbg_dest_tlv *dbg_dest_tlv;
+	bool dbg_dest_tlv_init;
+	u8 *dbg_dest_ver;
+	union {
+		struct iwl_fw_dbg_dest_tlv *dbg_dest_tlv;
+		struct iwl_fw_dbg_dest_tlv_v1 *dbg_dest_tlv_v1;
+	};
 	struct iwl_fw_dbg_conf_tlv *dbg_conf_tlv[FW_DBG_CONF_MAX];
 	size_t dbg_conf_tlv_len[FW_DBG_CONF_MAX];
 	struct iwl_fw_dbg_trigger_tlv *dbg_trigger_tlv[FW_DBG_TRIGGER_MAX];
@@ -919,27 +924,60 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			minor = le32_to_cpup(ptr++);
 			local_comp = le32_to_cpup(ptr);
 
-			snprintf(drv->fw.fw_version,
-				 sizeof(drv->fw.fw_version), "%u.%u.%u",
-				 major, minor, local_comp);
+			if (major >= 35)
+				snprintf(drv->fw.fw_version,
+					 sizeof(drv->fw.fw_version),
+					"%u.%08x.%u", major, minor, local_comp);
+			else
+				snprintf(drv->fw.fw_version,
+					 sizeof(drv->fw.fw_version),
+					"%u.%u.%u", major, minor, local_comp);
 			break;
 			}
 		case IWL_UCODE_TLV_FW_DBG_DEST: {
-			struct iwl_fw_dbg_dest_tlv *dest = (void *)tlv_data;
+			struct iwl_fw_dbg_dest_tlv *dest = NULL;
+			struct iwl_fw_dbg_dest_tlv_v1 *dest_v1 = NULL;
+			u8 mon_mode;
+
+			pieces->dbg_dest_ver = (u8 *)tlv_data;
+			if (*pieces->dbg_dest_ver == 1) {
+				dest = (void *)tlv_data;
+			} else if (*pieces->dbg_dest_ver == 0) {
+				dest_v1 = (void *)tlv_data;
+			} else {
+				IWL_ERR(drv,
+					"The version is %d, and it is invalid\n",
+					*pieces->dbg_dest_ver);
+				break;
+			}
 
-			if (pieces->dbg_dest_tlv) {
+			if (pieces->dbg_dest_tlv_init) {
 				IWL_ERR(drv,
 					"dbg destination ignored, already exists\n");
 				break;
 			}
 
-			pieces->dbg_dest_tlv = dest;
+			pieces->dbg_dest_tlv_init = true;
+
+			if (dest_v1) {
+				pieces->dbg_dest_tlv_v1 = dest_v1;
+				mon_mode = dest_v1->monitor_mode;
+			} else {
+				pieces->dbg_dest_tlv = dest;
+				mon_mode = dest->monitor_mode;
+			}
+
 			IWL_INFO(drv, "Found debug destination: %s\n",
-				 get_fw_dbg_mode_string(dest->monitor_mode));
+				 get_fw_dbg_mode_string(mon_mode));
+
+			drv->fw.dbg_dest_reg_num = (dest_v1) ?
+				tlv_len -
+				offsetof(struct iwl_fw_dbg_dest_tlv_v1,
+					 reg_ops) :
+				tlv_len -
+				offsetof(struct iwl_fw_dbg_dest_tlv,
+					 reg_ops);
 
-			drv->fw.dbg_dest_reg_num =
-				tlv_len - offsetof(struct iwl_fw_dbg_dest_tlv,
-						   reg_ops);
 			drv->fw.dbg_dest_reg_num /=
 				sizeof(drv->fw.dbg_dest_tlv->reg_ops[0]);
 
@@ -948,7 +986,7 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 		case IWL_UCODE_TLV_FW_DBG_CONF: {
 			struct iwl_fw_dbg_conf_tlv *conf = (void *)tlv_data;
 
-			if (!pieces->dbg_dest_tlv) {
+			if (!pieces->dbg_dest_tlv_init) {
 				IWL_ERR(drv,
 					"Ignore dbg config %d - no destination configured\n",
 					conf->id);
@@ -1335,15 +1373,51 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 		if (iwl_alloc_ucode(drv, pieces, i))
 			goto out_free_fw;
 
-	if (pieces->dbg_dest_tlv) {
-		drv->fw.dbg_dest_tlv =
-			kmemdup(pieces->dbg_dest_tlv,
-				sizeof(*pieces->dbg_dest_tlv) +
-				sizeof(pieces->dbg_dest_tlv->reg_ops[0]) *
-				drv->fw.dbg_dest_reg_num, GFP_KERNEL);
+	if (pieces->dbg_dest_tlv_init) {
+		size_t dbg_dest_size = sizeof(*drv->fw.dbg_dest_tlv) +
+			sizeof(drv->fw.dbg_dest_tlv->reg_ops[0]) *
+			drv->fw.dbg_dest_reg_num;
+
+		drv->fw.dbg_dest_tlv = kmalloc(dbg_dest_size, GFP_KERNEL);
 
 		if (!drv->fw.dbg_dest_tlv)
 			goto out_free_fw;
+
+		if (*pieces->dbg_dest_ver == 0) {
+			memcpy(drv->fw.dbg_dest_tlv, pieces->dbg_dest_tlv_v1,
+			       dbg_dest_size);
+		} else {
+			struct iwl_fw_dbg_dest_tlv_v1 *dest_tlv =
+				drv->fw.dbg_dest_tlv;
+
+			dest_tlv->version = pieces->dbg_dest_tlv->version;
+			dest_tlv->monitor_mode =
+				pieces->dbg_dest_tlv->monitor_mode;
+			dest_tlv->size_power =
+				pieces->dbg_dest_tlv->size_power;
+			dest_tlv->wrap_count =
+				pieces->dbg_dest_tlv->wrap_count;
+			dest_tlv->write_ptr_reg =
+				pieces->dbg_dest_tlv->write_ptr_reg;
+			dest_tlv->base_shift =
+				pieces->dbg_dest_tlv->base_shift;
+			memcpy(dest_tlv->reg_ops,
+			       pieces->dbg_dest_tlv->reg_ops,
+			       sizeof(drv->fw.dbg_dest_tlv->reg_ops[0]) *
+			       drv->fw.dbg_dest_reg_num);
+
+			/* In version 1 of the destination tlv, which is
+			 * relevant for internal buffer exclusively,
+			 * the base address is part of given with the length
+			 * of the buffer, and the size shift is give instead of
+			 * end shift. We now store these values in base_reg,
+			 * and end shift, and when dumping the data we'll
+			 * manipulate it for extracting both the length and
+			 * base address */
+			dest_tlv->base_reg = pieces->dbg_dest_tlv->cfg_reg;
+			dest_tlv->end_shift =
+				pieces->dbg_dest_tlv->size_shift;
+		}
 	}
 
 	for (i = 0; i < ARRAY_SIZE(drv->fw.dbg_conf_tlv); i++) {
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-fh.h b/drivers/net/wireless/intel/iwlwifi/iwl-fh.h
index 66e5db41e559..11789ffb6512 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-fh.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-fh.h
@@ -121,7 +121,7 @@
 #define FH_MEM_CBBC_16_19_UPPER_BOUND		(FH_MEM_LOWER_BOUND + 0xC00)
 #define FH_MEM_CBBC_20_31_LOWER_BOUND		(FH_MEM_LOWER_BOUND + 0xB20)
 #define FH_MEM_CBBC_20_31_UPPER_BOUND		(FH_MEM_LOWER_BOUND + 0xB80)
-/* a000 TFD table address, 64 bit */
+/* 22000 TFD table address, 64 bit */
 #define TFH_TFDQ_CBB_TABLE			(0x1C00)
 
 /* Find TFD CB base pointer for given queue */
@@ -140,7 +140,7 @@ static inline unsigned int FH_MEM_CBBC_QUEUE(struct iwl_trans *trans,
 	return FH_MEM_CBBC_20_31_LOWER_BOUND + 4 * (chnl - 20);
 }
 
-/* a000 configuration registers */
+/* 22000 configuration registers */
 
 /*
  * TFH Configuration register.
@@ -697,8 +697,8 @@ struct iwl_tfh_tb {
  * Each Tx queue uses a circular buffer of 256 TFDs stored in host DRAM.
  * Both driver and device share these circular buffers, each of which must be
  * contiguous 256 TFDs.
- * For pre a000 HW it is 256 x 128 bytes-per-TFD = 32 KBytes
- * For a000 HW and on it is 256 x 256 bytes-per-TFD = 65 KBytes
+ * For pre 22000 HW it is 256 x 128 bytes-per-TFD = 32 KBytes
+ * For 22000 HW and on it is 256 x 256 bytes-per-TFD = 65 KBytes
  *
  * Driver must indicate the physical address of the base of each
  * circular buffer via the FH_MEM_CBBC_QUEUE registers.
@@ -750,10 +750,10 @@ struct iwl_tfh_tfd {
 /**
  * struct iwlagn_schedq_bc_tbl scheduler byte count table
  *	base physical address provided by SCD_DRAM_BASE_ADDR
- * For devices up to a000:
+ * For devices up to 22000:
  * @tfd_offset  0-12 - tx command byte count
  *		12-16 - station index
- * For a000 and on:
+ * For 22000 and on:
  * @tfd_offset  0-12 - tx command byte count
  *		12-13 - number of 64 byte chunks
  *		14-16 - reserved
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index 921cab9e2d73..c25ed1a0bbb0 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -551,7 +551,7 @@ struct iwl_trans_ops {
 			   unsigned int queue_wdg_timeout);
 	void (*txq_disable)(struct iwl_trans *trans, int queue,
 			    bool configure_scd);
-	/* a000 functions */
+	/* 22000 functions */
 	int (*txq_alloc)(struct iwl_trans *trans,
 			 struct iwl_tx_queue_cfg_cmd *cmd,
 			 int cmd_id,
@@ -579,6 +579,7 @@ struct iwl_trans_ops {
 	void (*configure)(struct iwl_trans *trans,
 			  const struct iwl_trans_config *trans_cfg);
 	void (*set_pmi)(struct iwl_trans *trans, bool state);
+	void (*sw_reset)(struct iwl_trans *trans);
 	bool (*grab_nic_access)(struct iwl_trans *trans, unsigned long *flags);
 	void (*release_nic_access)(struct iwl_trans *trans,
 				   unsigned long *flags);
@@ -744,7 +745,7 @@ struct iwl_trans {
 	struct lockdep_map sync_cmd_lockdep_map;
 #endif
 
-	const struct iwl_fw_dbg_dest_tlv *dbg_dest_tlv;
+	const struct iwl_fw_dbg_dest_tlv_v1 *dbg_dest_tlv;
 	const struct iwl_fw_dbg_conf_tlv *dbg_conf_tlv[FW_DBG_CONF_MAX];
 	struct iwl_fw_dbg_trigger_tlv * const *dbg_trigger_tlv;
 	u8 dbg_dest_reg_num;
@@ -1124,6 +1125,12 @@ static inline void iwl_trans_set_pmi(struct iwl_trans *trans, bool state)
 		trans->ops->set_pmi(trans, state);
 }
 
+static inline void iwl_trans_sw_reset(struct iwl_trans *trans)
+{
+	if (trans->ops->sw_reset)
+		trans->ops->sw_reset(trans);
+}
+
 static inline void
 iwl_trans_set_bits_mask(struct iwl_trans *trans, u32 reg, u32 mask, u32 value)
 {
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
index a47635c32c11..9ffd21918b5a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile
@@ -2,7 +2,7 @@
 obj-$(CONFIG_IWLMVM)   += iwlmvm.o
 iwlmvm-y += fw.o mac80211.o nvm.o ops.o phy-ctxt.o mac-ctxt.o
 iwlmvm-y += utils.o rx.o rxmq.o tx.o binding.o quota.o sta.o sf.o
-iwlmvm-y += scan.o time-event.o rs.o
+iwlmvm-y += scan.o time-event.o rs.o rs-fw.o
 iwlmvm-y += power.o coex.o
 iwlmvm-y += tt.o offloading.o tdls.o
 iwlmvm-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o debugfs-vif.o
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
index 79c80f181f7d..890dbfff3a06 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/coex.c
@@ -7,7 +7,6 @@
  *
  * Copyright(c) 2013 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2017        Intel Deutschland GmbH
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -34,7 +33,6 @@
  *
  * Copyright(c) 2013 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
- * Copyright(c) 2017        Intel Deutschland GmbH
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -514,36 +512,17 @@ void iwl_mvm_rx_bt_coex_notif(struct iwl_mvm *mvm,
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_bt_coex_profile_notif *notif = (void *)pkt->data;
 
-	if (!iwl_mvm_has_new_ats_coex_api(mvm)) {
-		struct iwl_bt_coex_profile_notif_v4 *v4 = (void *)pkt->data;
-
-		mvm->last_bt_notif.mbox_msg[0] = v4->mbox_msg[0];
-		mvm->last_bt_notif.mbox_msg[1] = v4->mbox_msg[1];
-		mvm->last_bt_notif.mbox_msg[2] = v4->mbox_msg[2];
-		mvm->last_bt_notif.mbox_msg[3] = v4->mbox_msg[3];
-		mvm->last_bt_notif.msg_idx = v4->msg_idx;
-		mvm->last_bt_notif.bt_ci_compliance = v4->bt_ci_compliance;
-		mvm->last_bt_notif.primary_ch_lut = v4->primary_ch_lut;
-		mvm->last_bt_notif.secondary_ch_lut = v4->secondary_ch_lut;
-		mvm->last_bt_notif.bt_activity_grading =
-			v4->bt_activity_grading;
-		mvm->last_bt_notif.ttc_status = v4->ttc_status;
-		mvm->last_bt_notif.rrc_status = v4->rrc_status;
-	} else {
-		/* save this notification for future use: rssi fluctuations */
-		memcpy(&mvm->last_bt_notif, notif, sizeof(mvm->last_bt_notif));
-	}
-
 	IWL_DEBUG_COEX(mvm, "BT Coex Notification received\n");
-	IWL_DEBUG_COEX(mvm, "\tBT ci compliance %d\n",
-		       mvm->last_bt_notif.bt_ci_compliance);
+	IWL_DEBUG_COEX(mvm, "\tBT ci compliance %d\n", notif->bt_ci_compliance);
 	IWL_DEBUG_COEX(mvm, "\tBT primary_ch_lut %d\n",
-		       le32_to_cpu(mvm->last_bt_notif.primary_ch_lut));
+		       le32_to_cpu(notif->primary_ch_lut));
 	IWL_DEBUG_COEX(mvm, "\tBT secondary_ch_lut %d\n",
-		       le32_to_cpu(mvm->last_bt_notif.secondary_ch_lut));
+		       le32_to_cpu(notif->secondary_ch_lut));
 	IWL_DEBUG_COEX(mvm, "\tBT activity grading %d\n",
-		       le32_to_cpu(mvm->last_bt_notif.bt_activity_grading));
+		       le32_to_cpu(notif->bt_activity_grading));
 
+	/* remember this notification for future use: rssi fluctuations */
+	memcpy(&mvm->last_bt_notif, notif, sizeof(mvm->last_bt_notif));
 
 	iwl_mvm_bt_coex_notif_handle(mvm);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
index b1f73dcabd31..2efe9b099556 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
@@ -429,231 +429,6 @@ static int iwl_mvm_send_patterns(struct iwl_mvm *mvm,
 	return err;
 }
 
-enum iwl_mvm_tcp_packet_type {
-	MVM_TCP_TX_SYN,
-	MVM_TCP_RX_SYNACK,
-	MVM_TCP_TX_DATA,
-	MVM_TCP_RX_ACK,
-	MVM_TCP_RX_WAKE,
-	MVM_TCP_TX_FIN,
-};
-
-static __le16 pseudo_hdr_check(int len, __be32 saddr, __be32 daddr)
-{
-	__sum16 check = tcp_v4_check(len, saddr, daddr, 0);
-	return cpu_to_le16(be16_to_cpu((__force __be16)check));
-}
-
-static void iwl_mvm_build_tcp_packet(struct ieee80211_vif *vif,
-				     struct cfg80211_wowlan_tcp *tcp,
-				     void *_pkt, u8 *mask,
-				     __le16 *pseudo_hdr_csum,
-				     enum iwl_mvm_tcp_packet_type ptype)
-{
-	struct {
-		struct ethhdr eth;
-		struct iphdr ip;
-		struct tcphdr tcp;
-		u8 data[];
-	} __packed *pkt = _pkt;
-	u16 ip_tot_len = sizeof(struct iphdr) + sizeof(struct tcphdr);
-	int i;
-
-	pkt->eth.h_proto = cpu_to_be16(ETH_P_IP),
-	pkt->ip.version = 4;
-	pkt->ip.ihl = 5;
-	pkt->ip.protocol = IPPROTO_TCP;
-
-	switch (ptype) {
-	case MVM_TCP_TX_SYN:
-	case MVM_TCP_TX_DATA:
-	case MVM_TCP_TX_FIN:
-		memcpy(pkt->eth.h_dest, tcp->dst_mac, ETH_ALEN);
-		memcpy(pkt->eth.h_source, vif->addr, ETH_ALEN);
-		pkt->ip.ttl = 128;
-		pkt->ip.saddr = tcp->src;
-		pkt->ip.daddr = tcp->dst;
-		pkt->tcp.source = cpu_to_be16(tcp->src_port);
-		pkt->tcp.dest = cpu_to_be16(tcp->dst_port);
-		/* overwritten for TX SYN later */
-		pkt->tcp.doff = sizeof(struct tcphdr) / 4;
-		pkt->tcp.window = cpu_to_be16(65000);
-		break;
-	case MVM_TCP_RX_SYNACK:
-	case MVM_TCP_RX_ACK:
-	case MVM_TCP_RX_WAKE:
-		memcpy(pkt->eth.h_dest, vif->addr, ETH_ALEN);
-		memcpy(pkt->eth.h_source, tcp->dst_mac, ETH_ALEN);
-		pkt->ip.saddr = tcp->dst;
-		pkt->ip.daddr = tcp->src;
-		pkt->tcp.source = cpu_to_be16(tcp->dst_port);
-		pkt->tcp.dest = cpu_to_be16(tcp->src_port);
-		break;
-	default:
-		WARN_ON(1);
-		return;
-	}
-
-	switch (ptype) {
-	case MVM_TCP_TX_SYN:
-		/* firmware assumes 8 option bytes - 8 NOPs for now */
-		memset(pkt->data, 0x01, 8);
-		ip_tot_len += 8;
-		pkt->tcp.doff = (sizeof(struct tcphdr) + 8) / 4;
-		pkt->tcp.syn = 1;
-		break;
-	case MVM_TCP_TX_DATA:
-		ip_tot_len += tcp->payload_len;
-		memcpy(pkt->data, tcp->payload, tcp->payload_len);
-		pkt->tcp.psh = 1;
-		pkt->tcp.ack = 1;
-		break;
-	case MVM_TCP_TX_FIN:
-		pkt->tcp.fin = 1;
-		pkt->tcp.ack = 1;
-		break;
-	case MVM_TCP_RX_SYNACK:
-		pkt->tcp.syn = 1;
-		pkt->tcp.ack = 1;
-		break;
-	case MVM_TCP_RX_ACK:
-		pkt->tcp.ack = 1;
-		break;
-	case MVM_TCP_RX_WAKE:
-		ip_tot_len += tcp->wake_len;
-		pkt->tcp.psh = 1;
-		pkt->tcp.ack = 1;
-		memcpy(pkt->data, tcp->wake_data, tcp->wake_len);
-		break;
-	}
-
-	switch (ptype) {
-	case MVM_TCP_TX_SYN:
-	case MVM_TCP_TX_DATA:
-	case MVM_TCP_TX_FIN:
-		pkt->ip.tot_len = cpu_to_be16(ip_tot_len);
-		pkt->ip.check = ip_fast_csum(&pkt->ip, pkt->ip.ihl);
-		break;
-	case MVM_TCP_RX_WAKE:
-		for (i = 0; i < DIV_ROUND_UP(tcp->wake_len, 8); i++) {
-			u8 tmp = tcp->wake_mask[i];
-			mask[i + 6] |= tmp << 6;
-			if (i + 1 < DIV_ROUND_UP(tcp->wake_len, 8))
-				mask[i + 7] = tmp >> 2;
-		}
-		/* fall through for ethernet/IP/TCP headers mask */
-	case MVM_TCP_RX_SYNACK:
-	case MVM_TCP_RX_ACK:
-		mask[0] = 0xff; /* match ethernet */
-		/*
-		 * match ethernet, ip.version, ip.ihl
-		 * the ip.ihl half byte is really masked out by firmware
-		 */
-		mask[1] = 0x7f;
-		mask[2] = 0x80; /* match ip.protocol */
-		mask[3] = 0xfc; /* match ip.saddr, ip.daddr */
-		mask[4] = 0x3f; /* match ip.daddr, tcp.source, tcp.dest */
-		mask[5] = 0x80; /* match tcp flags */
-		/* leave rest (0 or set for MVM_TCP_RX_WAKE) */
-		break;
-	};
-
-	*pseudo_hdr_csum = pseudo_hdr_check(ip_tot_len - sizeof(struct iphdr),
-					    pkt->ip.saddr, pkt->ip.daddr);
-}
-
-static int iwl_mvm_send_remote_wake_cfg(struct iwl_mvm *mvm,
-					struct ieee80211_vif *vif,
-					struct cfg80211_wowlan_tcp *tcp)
-{
-	struct iwl_wowlan_remote_wake_config *cfg;
-	struct iwl_host_cmd cmd = {
-		.id = REMOTE_WAKE_CONFIG_CMD,
-		.len = { sizeof(*cfg), },
-		.dataflags = { IWL_HCMD_DFL_NOCOPY, },
-	};
-	int ret;
-
-	if (!tcp)
-		return 0;
-
-	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
-	if (!cfg)
-		return -ENOMEM;
-	cmd.data[0] = cfg;
-
-	cfg->max_syn_retries = 10;
-	cfg->max_data_retries = 10;
-	cfg->tcp_syn_ack_timeout = 1; /* seconds */
-	cfg->tcp_ack_timeout = 1; /* seconds */
-
-	/* SYN (TX) */
-	iwl_mvm_build_tcp_packet(
-		vif, tcp, cfg->syn_tx.data, NULL,
-		&cfg->syn_tx.info.tcp_pseudo_header_checksum,
-		MVM_TCP_TX_SYN);
-	cfg->syn_tx.info.tcp_payload_length = 0;
-
-	/* SYN/ACK (RX) */
-	iwl_mvm_build_tcp_packet(
-		vif, tcp, cfg->synack_rx.data, cfg->synack_rx.rx_mask,
-		&cfg->synack_rx.info.tcp_pseudo_header_checksum,
-		MVM_TCP_RX_SYNACK);
-	cfg->synack_rx.info.tcp_payload_length = 0;
-
-	/* KEEPALIVE/ACK (TX) */
-	iwl_mvm_build_tcp_packet(
-		vif, tcp, cfg->keepalive_tx.data, NULL,
-		&cfg->keepalive_tx.info.tcp_pseudo_header_checksum,
-		MVM_TCP_TX_DATA);
-	cfg->keepalive_tx.info.tcp_payload_length =
-		cpu_to_le16(tcp->payload_len);
-	cfg->sequence_number_offset = tcp->payload_seq.offset;
-	/* length must be 0..4, the field is little endian */
-	cfg->sequence_number_length = tcp->payload_seq.len;
-	cfg->initial_sequence_number = cpu_to_le32(tcp->payload_seq.start);
-	cfg->keepalive_interval = cpu_to_le16(tcp->data_interval);
-	if (tcp->payload_tok.len) {
-		cfg->token_offset = tcp->payload_tok.offset;
-		cfg->token_length = tcp->payload_tok.len;
-		cfg->num_tokens =
-			cpu_to_le16(tcp->tokens_size % tcp->payload_tok.len);
-		memcpy(cfg->tokens, tcp->payload_tok.token_stream,
-		       tcp->tokens_size);
-	} else {
-		/* set tokens to max value to almost never run out */
-		cfg->num_tokens = cpu_to_le16(65535);
-	}
-
-	/* ACK (RX) */
-	iwl_mvm_build_tcp_packet(
-		vif, tcp, cfg->keepalive_ack_rx.data,
-		cfg->keepalive_ack_rx.rx_mask,
-		&cfg->keepalive_ack_rx.info.tcp_pseudo_header_checksum,
-		MVM_TCP_RX_ACK);
-	cfg->keepalive_ack_rx.info.tcp_payload_length = 0;
-
-	/* WAKEUP (RX) */
-	iwl_mvm_build_tcp_packet(
-		vif, tcp, cfg->wake_rx.data, cfg->wake_rx.rx_mask,
-		&cfg->wake_rx.info.tcp_pseudo_header_checksum,
-		MVM_TCP_RX_WAKE);
-	cfg->wake_rx.info.tcp_payload_length =
-		cpu_to_le16(tcp->wake_len);
-
-	/* FIN */
-	iwl_mvm_build_tcp_packet(
-		vif, tcp, cfg->fin_tx.data, NULL,
-		&cfg->fin_tx.info.tcp_pseudo_header_checksum,
-		MVM_TCP_TX_FIN);
-	cfg->fin_tx.info.tcp_payload_length = 0;
-
-	ret = iwl_mvm_send_cmd(mvm, &cmd);
-	kfree(cfg);
-
-	return ret;
-}
-
 static int iwl_mvm_d3_reprogram(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 				struct ieee80211_sta *ap_sta)
 {
@@ -1082,12 +857,7 @@ iwl_mvm_wowlan_config(struct iwl_mvm *mvm,
 	if (ret)
 		return ret;
 
-	ret = iwl_mvm_send_proto_offload(mvm, vif, false, true, 0);
-	if (ret)
-		return ret;
-
-	ret = iwl_mvm_send_remote_wake_cfg(mvm, vif, wowlan->tcp);
-	return ret;
+	return iwl_mvm_send_proto_offload(mvm, vif, false, true, 0);
 }
 
 static int
@@ -1328,6 +1098,8 @@ int iwl_mvm_suspend(struct ieee80211_hw *hw, struct cfg80211_wowlan *wowlan)
 	/* make sure the d0i3 exit work is not pending */
 	flush_work(&mvm->d0i3_exit_work);
 
+	iwl_fw_runtime_suspend(&mvm->fwrt);
+
 	ret = iwl_trans_suspend(trans);
 	if (ret)
 		return ret;
@@ -2242,6 +2014,8 @@ int iwl_mvm_resume(struct ieee80211_hw *hw)
 
 	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
 
+	iwl_fw_runtime_resume(&mvm->fwrt);
+
 	return ret;
 }
 
@@ -2268,6 +2042,8 @@ static int iwl_mvm_d3_test_open(struct inode *inode, struct file *file)
 
 	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_D3;
 
+	iwl_fw_runtime_suspend(&mvm->fwrt);
+
 	/* start pseudo D3 */
 	rtnl_lock();
 	err = __iwl_mvm_suspend(mvm->hw, mvm->hw->wiphy->wowlan_config, true);
@@ -2328,6 +2104,8 @@ static int iwl_mvm_d3_test_release(struct inode *inode, struct file *file)
 	__iwl_mvm_resume(mvm, true);
 	rtnl_unlock();
 
+	iwl_fw_runtime_resume(&mvm->fwrt);
+
 	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
 
 	iwl_abort_notification_waits(&mvm->notif_wait);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index 2ff594f11259..9c436d8d001d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -425,6 +427,50 @@ static ssize_t iwl_dbgfs_stations_read(struct file *file, char __user *user_buf,
 	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
 }
 
+static ssize_t iwl_dbgfs_rs_data_read(struct file *file, char __user *user_buf,
+				      size_t count, loff_t *ppos)
+{
+	struct ieee80211_sta *sta = file->private_data;
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->lq_sta.rs_fw;
+	struct iwl_mvm *mvm = lq_sta->pers.drv;
+	static const size_t bufsz = 2048;
+	char *buff;
+	int desc = 0;
+	ssize_t ret;
+
+	buff = kmalloc(bufsz, GFP_KERNEL);
+	if (!buff)
+		return -ENOMEM;
+
+	mutex_lock(&mvm->mutex);
+
+	desc += scnprintf(buff + desc, bufsz - desc, "sta_id %d\n",
+			  lq_sta->pers.sta_id);
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "fixed rate 0x%X\n",
+			  lq_sta->pers.dbg_fixed_rate);
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "A-MPDU size limit %d\n",
+			  lq_sta->pers.dbg_agg_frame_count_lim);
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "valid_tx_ant %s%s%s\n",
+		(iwl_mvm_get_valid_tx_ant(mvm) & ANT_A) ? "ANT_A," : "",
+		(iwl_mvm_get_valid_tx_ant(mvm) & ANT_B) ? "ANT_B," : "",
+		(iwl_mvm_get_valid_tx_ant(mvm) & ANT_C) ? "ANT_C" : "");
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "last tx rate=0x%X ",
+			  lq_sta->last_rate_n_flags);
+
+	desc += rs_pretty_print_rate(buff + desc, bufsz - desc,
+				     lq_sta->last_rate_n_flags);
+	mutex_unlock(&mvm->mutex);
+
+	ret = simple_read_from_buffer(user_buf, count, ppos, buff, desc);
+	kfree(buff);
+	return ret;
+}
+
 static ssize_t iwl_dbgfs_disable_power_off_read(struct file *file,
 						char __user *user_buf,
 						size_t count, loff_t *ppos)
@@ -470,8 +516,7 @@ static ssize_t iwl_dbgfs_disable_power_off_write(struct iwl_mvm *mvm, char *buf,
 }
 
 static
-int iwl_mvm_coex_dump_mbox(struct iwl_mvm *mvm,
-			   struct iwl_bt_coex_profile_notif *notif, char *buf,
+int iwl_mvm_coex_dump_mbox(struct iwl_bt_coex_profile_notif *notif, char *buf,
 			   int pos, int bufsz)
 {
 	pos += scnprintf(buf+pos, bufsz-pos, "MBOX dw0:\n");
@@ -525,12 +570,7 @@ int iwl_mvm_coex_dump_mbox(struct iwl_mvm *mvm,
 	BT_MBOX_PRINT(3, INBAND_P, false);
 	BT_MBOX_PRINT(3, MSG_TYPE_2, false);
 	BT_MBOX_PRINT(3, SSN_2, false);
-	BT_MBOX_PRINT(3, UPDATE_REQUEST, !iwl_mvm_has_new_ats_coex_api(mvm));
-
-	if (iwl_mvm_has_new_ats_coex_api(mvm)) {
-		BT_MBOX_PRINT(4, ATS_BT_INTERVAL, false);
-		BT_MBOX_PRINT(4, ATS_BT_ACTIVE_MAX_TH, true);
-	}
+	BT_MBOX_PRINT(3, UPDATE_REQUEST, true);
 
 	return pos;
 }
@@ -549,7 +589,7 @@ static ssize_t iwl_dbgfs_bt_notif_read(struct file *file, char __user *user_buf,
 
 	mutex_lock(&mvm->mutex);
 
-	pos += iwl_mvm_coex_dump_mbox(mvm, notif, buf, pos, bufsz);
+	pos += iwl_mvm_coex_dump_mbox(notif, buf, pos, bufsz);
 
 	pos += scnprintf(buf + pos, bufsz - pos, "bt_ci_compliance = %d\n",
 			 notif->bt_ci_compliance);
@@ -721,6 +761,9 @@ static ssize_t iwl_dbgfs_fw_rx_stats_read(struct file *file,
 
 	mutex_lock(&mvm->mutex);
 
+	if (iwl_mvm_firmware_running(mvm))
+		iwl_mvm_request_statistics(mvm, false);
+
 	pos += scnprintf(buf + pos, bufsz - pos, fmt_header,
 			 "Statistics_Rx - OFDM");
 	if (!iwl_mvm_has_new_rx_stats_api(mvm)) {
@@ -936,7 +979,8 @@ static ssize_t iwl_dbgfs_frame_stats_read(struct iwl_mvm *mvm,
 			continue;
 		pos += scnprintf(pos, endpos - pos, "Rate[%d]: ",
 				 (int)(ARRAY_SIZE(stats->last_rates) - i));
-		pos += rs_pretty_print_rate(pos, stats->last_rates[idx]);
+		pos += rs_pretty_print_rate(pos, endpos - pos,
+					    stats->last_rates[idx]);
 	}
 	spin_unlock_bh(&mvm->drv_stats_lock);
 
@@ -1179,7 +1223,7 @@ static ssize_t iwl_dbgfs_cont_recording_write(struct iwl_mvm *mvm,
 					      loff_t *ppos)
 {
 	struct iwl_trans *trans = mvm->trans;
-	const struct iwl_fw_dbg_dest_tlv *dest = trans->dbg_dest_tlv;
+	const struct iwl_fw_dbg_dest_tlv_v1 *dest = trans->dbg_dest_tlv;
 	struct iwl_continuous_record_cmd cont_rec = {};
 	int ret, rec_mode;
 
@@ -1239,9 +1283,6 @@ static ssize_t iwl_dbgfs_fw_dbg_collect_write(struct iwl_mvm *mvm,
 {
 	int ret;
 
-	if (!iwl_mvm_firmware_running(mvm))
-		return -EIO;
-
 	ret = iwl_mvm_ref_sync(mvm, IWL_MVM_REF_PRPH_WRITE);
 	if (ret)
 		return ret;
@@ -1603,6 +1644,19 @@ static ssize_t iwl_dbgfs_d0i3_refs_write(struct iwl_mvm *mvm, char *buf,
 #define MVM_DEBUGFS_ADD_FILE(name, parent, mode) \
 	MVM_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
 
+#define MVM_DEBUGFS_WRITE_STA_FILE_OPS(name, bufsz) \
+	_MVM_DEBUGFS_WRITE_FILE_OPS(name, bufsz, struct ieee80211_sta)
+#define MVM_DEBUGFS_READ_WRITE_STA_FILE_OPS(name, bufsz) \
+	_MVM_DEBUGFS_READ_WRITE_FILE_OPS(name, bufsz, struct ieee80211_sta)
+
+#define MVM_DEBUGFS_ADD_STA_FILE_ALIAS(alias, name, parent, mode) do {	\
+		if (!debugfs_create_file(alias, mode, parent, sta,	\
+					 &iwl_dbgfs_##name##_ops))	\
+			goto err;					\
+	} while (0)
+#define MVM_DEBUGFS_ADD_STA_FILE(name, parent, mode) \
+	MVM_DEBUGFS_ADD_STA_FILE_ALIAS(#name, name, parent, mode)
+
 static ssize_t
 iwl_dbgfs_prph_reg_read(struct file *file,
 			char __user *user_buf,
@@ -1687,6 +1741,7 @@ MVM_DEBUGFS_READ_WRITE_FILE_OPS(sram, 64);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(set_nic_temperature, 64);
 MVM_DEBUGFS_READ_FILE_OPS(nic_temp);
 MVM_DEBUGFS_READ_FILE_OPS(stations);
+MVM_DEBUGFS_READ_FILE_OPS(rs_data);
 MVM_DEBUGFS_READ_FILE_OPS(bt_notif);
 MVM_DEBUGFS_READ_FILE_OPS(bt_cmd);
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(disable_power_off, 64);
@@ -1851,6 +1906,21 @@ static const struct file_operations iwl_dbgfs_mem_ops = {
 	.llseek = default_llseek,
 };
 
+void iwl_mvm_sta_add_debugfs(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta,
+			     struct dentry *dir)
+{
+	struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw);
+
+	if (iwl_mvm_has_tlc_offload(mvm))
+		MVM_DEBUGFS_ADD_STA_FILE(rs_data, dir, S_IRUSR);
+
+	return;
+err:
+	IWL_ERR(mvm, "Can't create the mvm station debugfs entry\n");
+}
+
 int iwl_mvm_dbgfs_register(struct iwl_mvm *mvm, struct dentry *dbgfs_dir)
 {
 	struct dentry *bcast_dir __maybe_unused;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
index c0de7bb86cf7..3c59109bea20 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c
@@ -297,7 +297,7 @@ static int iwl_mvm_load_ucode_wait_alive(struct iwl_mvm *mvm,
 	if (ret) {
 		struct iwl_trans *trans = mvm->trans;
 
-		if (trans->cfg->device_family == IWL_DEVICE_FAMILY_A000)
+		if (trans->cfg->device_family == IWL_DEVICE_FAMILY_22000)
 			IWL_ERR(mvm,
 				"SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
 				iwl_read_prph(trans, UMAG_SB_CPU_1_STATUS),
@@ -433,6 +433,10 @@ static int iwl_send_phy_cfg_cmd(struct iwl_mvm *mvm)
 
 	/* Set parameters */
 	phy_cfg_cmd.phy_cfg = cpu_to_le32(iwl_mvm_get_phy_config(mvm));
+
+	/* set flags extra PHY configuration flags from the device's cfg */
+	phy_cfg_cmd.phy_cfg |= cpu_to_le32(mvm->cfg->extra_phy_cfg_flags);
+
 	phy_cfg_cmd.calib_control.event_trigger =
 		mvm->fw->default_calib[ucode_type].event_trigger;
 	phy_cfg_cmd.calib_control.flow_trigger =
@@ -923,11 +927,11 @@ static int iwl_mvm_load_rt_fw(struct iwl_mvm *mvm)
 
 	ret = iwl_run_init_mvm_ucode(mvm, false);
 
-	if (iwlmvm_mod_params.init_dbg)
-		return 0;
-
 	if (ret) {
 		IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", ret);
+
+		if (iwlmvm_mod_params.init_dbg)
+			return 0;
 		return ret;
 	}
 
@@ -998,9 +1002,9 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 		goto error;
 
 	/* Init RSS configuration */
-	/* TODO - remove a000 disablement when we have RXQ config API */
+	/* TODO - remove 22000 disablement when we have RXQ config API */
 	if (iwl_mvm_has_new_rx_api(mvm) &&
-	    mvm->trans->cfg->device_family != IWL_DEVICE_FAMILY_A000) {
+	    mvm->trans->cfg->device_family != IWL_DEVICE_FAMILY_22000) {
 		ret = iwl_send_rss_cfg_cmd(mvm);
 		if (ret) {
 			IWL_ERR(mvm, "Failed to configure RSS queues: %d\n",
@@ -1111,7 +1115,7 @@ int iwl_mvm_up(struct iwl_mvm *mvm)
 	IWL_DEBUG_INFO(mvm, "RT uCode started.\n");
 	return 0;
  error:
-	if (!iwlmvm_mod_params.init_dbg)
+	if (!iwlmvm_mod_params.init_dbg || !ret)
 		iwl_mvm_stop_device(mvm);
 	return ret;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index 2f22e14e00fe..8ba16fc24e3a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -438,7 +438,8 @@ int iwl_mvm_mac_ctxt_init(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	}
 
 	/* Allocate the CAB queue for softAP and GO interfaces */
-	if (vif->type == NL80211_IFTYPE_AP) {
+	if (vif->type == NL80211_IFTYPE_AP ||
+	    vif->type == NL80211_IFTYPE_ADHOC) {
 		/*
 		 * For TVQM this will be overwritten later with the FW assigned
 		 * queue value (when queue is enabled).
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index 3e92a117c0b8..7152fdc00fb1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -114,29 +115,6 @@ static const struct ieee80211_iface_combination iwl_mvm_iface_combinations[] = {
 	},
 };
 
-#ifdef CONFIG_PM_SLEEP
-static const struct nl80211_wowlan_tcp_data_token_feature
-iwl_mvm_wowlan_tcp_token_feature = {
-	.min_len = 0,
-	.max_len = 255,
-	.bufsize = IWL_WOWLAN_REMOTE_WAKE_MAX_TOKENS,
-};
-
-static const struct wiphy_wowlan_tcp_support iwl_mvm_wowlan_tcp_support = {
-	.tok = &iwl_mvm_wowlan_tcp_token_feature,
-	.data_payload_max = IWL_WOWLAN_TCP_MAX_PACKET_LEN -
-			    sizeof(struct ethhdr) -
-			    sizeof(struct iphdr) -
-			    sizeof(struct tcphdr),
-	.data_interval_max = 65535, /* __le16 in API */
-	.wake_payload_max = IWL_WOWLAN_REMOTE_WAKE_MAX_PACKET_LEN -
-			    sizeof(struct ethhdr) -
-			    sizeof(struct iphdr) -
-			    sizeof(struct tcphdr),
-	.seq = true,
-};
-#endif
-
 #ifdef CONFIG_IWLWIFI_BCAST_FILTERING
 /*
  * Use the reserved field to indicate magic values.
@@ -443,6 +421,12 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	ieee80211_hw_set(hw, SUPPORTS_CLONED_SKBS);
 	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
 	ieee80211_hw_set(hw, NEEDS_UNIQUE_STA_ADDR);
+
+	if (iwl_mvm_has_tlc_offload(mvm)) {
+		ieee80211_hw_set(hw, TX_AMPDU_SETUP_IN_HW);
+		ieee80211_hw_set(hw, HAS_RATE_CONTROL);
+	}
+
 	if (iwl_mvm_has_new_rx_api(mvm))
 		ieee80211_hw_set(hw, SUPPORTS_REORDERING_BUFFER);
 
@@ -477,7 +461,9 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 	/* this is the case for CCK frames, it's better (only 8) for OFDM */
 	hw->radiotap_timestamp.accuracy = 22;
 
-	hw->rate_control_algorithm = "iwl-mvm-rs";
+	if (!iwl_mvm_has_tlc_offload(mvm))
+		hw->rate_control_algorithm = RS_NAME;
+
 	hw->uapsd_queues = IWL_MVM_UAPSD_QUEUES;
 	hw->uapsd_max_sp_len = IWL_UAPSD_MAX_SP;
 
@@ -702,7 +688,6 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm)
 		mvm->wowlan.pattern_min_len = IWL_WOWLAN_MIN_PATTERN_LEN;
 		mvm->wowlan.pattern_max_len = IWL_WOWLAN_MAX_PATTERN_LEN;
 		mvm->wowlan.max_nd_match_sets = IWL_SCAN_MAX_PROFILES;
-		mvm->wowlan.tcp = &iwl_mvm_wowlan_tcp_support;
 		hw->wiphy->wowlan = &mvm->wowlan;
 	}
 #endif
@@ -2122,15 +2107,40 @@ static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
 	if (ret)
 		goto out_remove;
 
-	ret = iwl_mvm_add_mcast_sta(mvm, vif);
-	if (ret)
-		goto out_unbind;
-
-	/* Send the bcast station. At this stage the TBTT and DTIM time events
-	 * are added and applied to the scheduler */
-	ret = iwl_mvm_send_add_bcast_sta(mvm, vif);
-	if (ret)
-		goto out_rm_mcast;
+	/*
+	 * This is not very nice, but the simplest:
+	 * For older FWs adding the mcast sta before the bcast station may
+	 * cause assert 0x2b00.
+	 * This is fixed in later FW so make the order of removal depend on
+	 * the TLV
+	 */
+	if (fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_STA_TYPE)) {
+		ret = iwl_mvm_add_mcast_sta(mvm, vif);
+		if (ret)
+			goto out_unbind;
+		/*
+		 * Send the bcast station. At this stage the TBTT and DTIM time
+		 * events are added and applied to the scheduler
+		 */
+		ret = iwl_mvm_send_add_bcast_sta(mvm, vif);
+		if (ret) {
+			iwl_mvm_rm_mcast_sta(mvm, vif);
+			goto out_unbind;
+		}
+	} else {
+		/*
+		 * Send the bcast station. At this stage the TBTT and DTIM time
+		 * events are added and applied to the scheduler
+		 */
+		ret = iwl_mvm_send_add_bcast_sta(mvm, vif);
+		if (ret)
+			goto out_unbind;
+		ret = iwl_mvm_add_mcast_sta(mvm, vif);
+		if (ret) {
+			iwl_mvm_send_rm_bcast_sta(mvm, vif);
+			goto out_unbind;
+		}
+	}
 
 	/* must be set before quota calculations */
 	mvmvif->ap_ibss_active = true;
@@ -2160,7 +2170,6 @@ static int iwl_mvm_start_ap_ibss(struct ieee80211_hw *hw,
 	iwl_mvm_power_update_mac(mvm);
 	mvmvif->ap_ibss_active = false;
 	iwl_mvm_send_rm_bcast_sta(mvm, vif);
-out_rm_mcast:
 	iwl_mvm_rm_mcast_sta(mvm, vif);
 out_unbind:
 	iwl_mvm_binding_remove_vif(mvm, vif);
@@ -2698,6 +2707,10 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 
 		/* enable beacon filtering */
 		WARN_ON(iwl_mvm_enable_beacon_filter(mvm, vif, 0));
+
+		iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band,
+				     false);
+
 		ret = 0;
 	} else if (old_state == IEEE80211_STA_AUTHORIZED &&
 		   new_state == IEEE80211_STA_ASSOC) {
@@ -3216,6 +3229,10 @@ static int iwl_mvm_roc(struct ieee80211_hw *hw,
 	IWL_DEBUG_MAC80211(mvm, "enter (%d, %d, %d)\n", channel->hw_value,
 			   duration, type);
 
+	/*
+	 * Flush the done work, just in case it's still pending, so that
+	 * the work it does can complete and we can accept new frames.
+	 */
 	flush_work(&mvm->roc_done_wk);
 
 	mutex_lock(&mvm->mutex);
@@ -3477,6 +3494,7 @@ static int __iwl_mvm_assign_vif_chanctx(struct iwl_mvm *mvm,
 		ret = 0;
 		goto out;
 	case NL80211_IFTYPE_STATION:
+		mvmvif->csa_bcn_pending = false;
 		break;
 	case NL80211_IFTYPE_MONITOR:
 		/* always disable PS when a monitor interface is active */
@@ -3520,7 +3538,7 @@ static int __iwl_mvm_assign_vif_chanctx(struct iwl_mvm *mvm,
 	}
 
 	if (switching_chanctx && vif->type == NL80211_IFTYPE_STATION) {
-		u32 duration = 2 * vif->bss_conf.beacon_int;
+		u32 duration = 3 * vif->bss_conf.beacon_int;
 
 		/* iwl_mvm_protect_session() reads directly from the
 		 * device (the system time), so make sure it is
@@ -3533,6 +3551,7 @@ static int __iwl_mvm_assign_vif_chanctx(struct iwl_mvm *mvm,
 		/* Protect the session to make sure we hear the first
 		 * beacon on the new channel.
 		 */
+		mvmvif->csa_bcn_pending = true;
 		iwl_mvm_protect_session(mvm, vif, duration, duration,
 					vif->bss_conf.beacon_int / 2,
 					true);
@@ -3813,7 +3832,7 @@ static int __iwl_mvm_mac_testmode_cmd(struct iwl_mvm *mvm,
 		mvm->noa_duration = noa_duration;
 		mvm->noa_vif = vif;
 
-		return iwl_mvm_update_quotas(mvm, false, NULL);
+		return iwl_mvm_update_quotas(mvm, true, NULL);
 	case IWL_MVM_TM_CMD_SET_BEACON_FILTER:
 		/* must be associated client vif - ignore authorized */
 		if (!vif || vif->type != NL80211_IFTYPE_STATION ||
@@ -3971,6 +3990,7 @@ static int iwl_mvm_post_channel_switch(struct ieee80211_hw *hw,
 	if (vif->type == NL80211_IFTYPE_STATION) {
 		struct iwl_mvm_sta *mvmsta;
 
+		mvmvif->csa_bcn_pending = false;
 		mvmsta = iwl_mvm_sta_from_staid_protected(mvm,
 							  mvmvif->ap_sta_id);
 
@@ -4301,7 +4321,7 @@ void iwl_mvm_sync_rx_queues_internal(struct iwl_mvm *mvm,
 			   mvm->trans->num_rx_queues);
 
 	/* TODO - remove this when we have RXQ config API */
-	if (mvm->trans->cfg->device_family == IWL_DEVICE_FAMILY_A000) {
+	if (mvm->trans->cfg->device_family == IWL_DEVICE_FAMILY_22000) {
 		qmask = BIT(0);
 		if (notif->sync)
 			atomic_set(&mvm->queue_sync_counter, 1);
@@ -4414,4 +4434,7 @@ const struct ieee80211_ops iwl_mvm_hw_ops = {
 #endif
 	.get_survey = iwl_mvm_mac_get_survey,
 	.sta_statistics = iwl_mvm_mac_sta_statistics,
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	.sta_add_debugfs = iwl_mvm_sta_add_debugfs,
+#endif
 };
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index 55ab5349dd40..625b238a3f0a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -90,6 +90,7 @@
 #include "fw/runtime.h"
 #include "fw/dbg.h"
 #include "fw/acpi.h"
+#include "fw/debugfs.h"
 
 #define IWL_MVM_MAX_ADDRESSES		5
 /* RSSI offset for WkP */
@@ -437,6 +438,9 @@ struct iwl_mvm_vif {
 	bool csa_failed;
 	u16 csa_target_freq;
 
+	/* Indicates that we are waiting for a beacon on a new channel */
+	bool csa_bcn_pending;
+
 	/* TCP Checksum Offload */
 	netdev_features_t features;
 };
@@ -1248,7 +1252,7 @@ static inline bool iwl_mvm_has_new_tx_api(struct iwl_mvm *mvm)
 static inline bool iwl_mvm_has_unified_ucode(struct iwl_mvm *mvm)
 {
 	/* TODO - better define this */
-	return mvm->trans->cfg->device_family >= IWL_DEVICE_FAMILY_A000;
+	return mvm->trans->cfg->device_family >= IWL_DEVICE_FAMILY_22000;
 }
 
 static inline bool iwl_mvm_is_cdb_supported(struct iwl_mvm *mvm)
@@ -1272,16 +1276,16 @@ static inline bool iwl_mvm_has_new_rx_stats_api(struct iwl_mvm *mvm)
 			  IWL_UCODE_TLV_API_NEW_RX_STATS);
 }
 
-static inline bool iwl_mvm_has_new_ats_coex_api(struct iwl_mvm *mvm)
+static inline bool iwl_mvm_has_quota_low_latency(struct iwl_mvm *mvm)
 {
 	return fw_has_api(&mvm->fw->ucode_capa,
-			  IWL_UCODE_TLV_API_COEX_ATS_EXTERNAL);
+			  IWL_UCODE_TLV_API_QUOTA_LOW_LATENCY);
 }
 
-static inline bool iwl_mvm_has_quota_low_latency(struct iwl_mvm *mvm)
+static inline bool iwl_mvm_has_tlc_offload(const struct iwl_mvm *mvm)
 {
-	return fw_has_api(&mvm->fw->ucode_capa,
-			  IWL_UCODE_TLV_API_QUOTA_LOW_LATENCY);
+	return fw_has_capa(&mvm->fw->ucode_capa,
+			   IWL_UCODE_TLV_CAPA_TLC_OFFLOAD);
 }
 
 static inline struct agg_tx_status *
@@ -1600,9 +1604,9 @@ iwl_mvm_vif_dbgfs_clean(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 /* rate scaling */
 int iwl_mvm_send_lq_cmd(struct iwl_mvm *mvm, struct iwl_lq_cmd *lq, bool init);
 void iwl_mvm_update_frame_stats(struct iwl_mvm *mvm, u32 rate, bool agg);
-int rs_pretty_print_rate(char *buf, const u32 rate);
+int rs_pretty_print_rate(char *buf, int bufsz, const u32 rate);
 void rs_update_last_rssi(struct iwl_mvm *mvm,
-			 struct iwl_lq_sta *lq_sta,
+			 struct iwl_mvm_sta *mvmsta,
 			 struct ieee80211_rx_status *rx_status);
 
 /* power management */
@@ -1783,6 +1787,7 @@ static inline u32 iwl_mvm_flushable_queues(struct iwl_mvm *mvm)
 
 static inline void iwl_mvm_stop_device(struct iwl_mvm *mvm)
 {
+	iwl_fw_cancel_timestamp(&mvm->fwrt);
 	iwl_free_fw_paging(&mvm->fwrt);
 	clear_bit(IWL_MVM_STATUS_FIRMWARE_RUNNING, &mvm->status);
 	iwl_fw_dump_conf_clear(&mvm->fwrt);
@@ -1882,5 +1887,11 @@ void iwl_mvm_event_frame_timeout_callback(struct iwl_mvm *mvm,
 
 int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b);
 int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm);
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+void iwl_mvm_sta_add_debugfs(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_sta *sta,
+			     struct dentry *dir);
+#endif
 
 #endif /* __IWL_MVM_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index 45470b6b351a..ab7fb5aad984 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -35,6 +36,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018        Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -127,11 +129,8 @@ static int __init iwl_mvm_init(void)
 	}
 
 	ret = iwl_opmode_register("iwlmvm", &iwl_mvm_ops);
-
-	if (ret) {
+	if (ret)
 		pr_err("Unable to register MVM op_mode: %d\n", ret);
-		iwl_mvm_rate_control_unregister();
-	}
 
 	return ret;
 }
@@ -555,9 +554,15 @@ static void iwl_mvm_fwrt_dump_end(void *ctx)
 	iwl_mvm_unref(mvm, IWL_MVM_REF_FW_DBG_COLLECT);
 }
 
+static bool iwl_mvm_fwrt_fw_running(void *ctx)
+{
+	return iwl_mvm_firmware_running(ctx);
+}
+
 static const struct iwl_fw_runtime_ops iwl_mvm_fwrt_ops = {
 	.dump_start = iwl_mvm_fwrt_dump_start,
 	.dump_end = iwl_mvm_fwrt_dump_end,
+	.fw_running = iwl_mvm_fwrt_fw_running,
 };
 
 static struct iwl_op_mode *
@@ -605,7 +610,8 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	mvm->fw = fw;
 	mvm->hw = hw;
 
-	iwl_fw_runtime_init(&mvm->fwrt, trans, fw, &iwl_mvm_fwrt_ops, mvm);
+	iwl_fw_runtime_init(&mvm->fwrt, trans, fw, &iwl_mvm_fwrt_ops, mvm,
+			    dbgfs_dir);
 
 	mvm->init_status = 0;
 
@@ -750,7 +756,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	mutex_lock(&mvm->mutex);
 	iwl_mvm_ref(mvm, IWL_MVM_REF_INIT_UCODE);
 	err = iwl_run_init_mvm_ucode(mvm, true);
-	if (!iwlmvm_mod_params.init_dbg)
+	if (!iwlmvm_mod_params.init_dbg || !err)
 		iwl_mvm_stop_device(mvm);
 	iwl_mvm_unref(mvm, IWL_MVM_REF_INIT_UCODE);
 	mutex_unlock(&mvm->mutex);
@@ -844,7 +850,6 @@ static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode)
 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_IWLWIFI_DEBUGFS)
 	kfree(mvm->d3_resume_sram);
 #endif
-
 	iwl_trans_op_mode_leave(mvm->trans);
 
 	iwl_phy_db_free(mvm->phy_db);
@@ -1021,6 +1026,8 @@ static void iwl_mvm_rx_mq(struct iwl_op_mode *op_mode,
 		iwl_mvm_rx_queue_notif(mvm, rxb, 0);
 	else if (cmd == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE))
 		iwl_mvm_rx_frame_release(mvm, napi, rxb, 0);
+	else if (cmd == WIDE_ID(DATA_PATH_GROUP, TLC_MNG_UPDATE_NOTIF))
+		iwl_mvm_tlc_update_notif(mvm, pkt);
 	else
 		iwl_mvm_rx_common(mvm, rxb, pkt);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
index 305cd56bf746..7f5434b34d0d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/phy-ctxt.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
  * Copyright(c) 2017           Intel Deutschland GmbH
+ * Copyright(c) 2018           Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -18,11 +19,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -34,6 +30,7 @@
  *
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
+ * Copyright(c) 2018           Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -286,6 +283,20 @@ void iwl_mvm_phy_ctxt_unref(struct iwl_mvm *mvm, struct iwl_mvm_phy_ctxt *ctxt)
 		return;
 
 	ctxt->ref--;
+
+	/*
+	 * Move unused phy's to a default channel. When the phy is moved the,
+	 * fw will cleanup immediate quiet bit if it was previously set,
+	 * otherwise we might not be able to reuse this phy.
+	 */
+	if (ctxt->ref == 0) {
+		struct ieee80211_channel *chan;
+		struct cfg80211_chan_def chandef;
+
+		chan = &mvm->hw->wiphy->bands[NL80211_BAND_2GHZ]->channels[0];
+		cfg80211_chandef_create(&chandef, chan, NL80211_CHAN_NO_HT);
+		iwl_mvm_phy_ctxt_changed(mvm, ctxt, &chandef, 1, 1);
+	}
 }
 
 static void iwl_mvm_binding_iterator(void *_data, u8 *mac,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/quota.c b/drivers/net/wireless/intel/iwlwifi/mvm/quota.c
index b4a0264329b7..690559bdf421 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/quota.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/quota.c
@@ -202,6 +202,10 @@ int iwl_mvm_update_quotas(struct iwl_mvm *mvm,
 
 	lockdep_assert_held(&mvm->mutex);
 
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_DYNAMIC_QUOTA))
+		return 0;
+
 	/* update all upon completion */
 	if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))
 		return 0;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
new file mode 100644
index 000000000000..55d1274c6092
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
@@ -0,0 +1,314 @@
+/******************************************************************************
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2017        Intel Deutschland GmbH
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution
+ * in the file called COPYING.
+ *
+ * Contact Information:
+ *  Intel Linux Wireless <linuxwifi@intel.com>
+ * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2017        Intel Deutschland GmbH
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *  * Neither the name Intel Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ *****************************************************************************/
+#include "rs.h"
+#include "fw-api.h"
+#include "sta.h"
+#include "iwl-op-mode.h"
+#include "mvm.h"
+
+static u8 rs_fw_bw_from_sta_bw(struct ieee80211_sta *sta)
+{
+	switch (sta->bandwidth) {
+	case IEEE80211_STA_RX_BW_160:
+		return IWL_TLC_MNG_MAX_CH_WIDTH_160MHZ;
+	case IEEE80211_STA_RX_BW_80:
+		return IWL_TLC_MNG_MAX_CH_WIDTH_80MHZ;
+	case IEEE80211_STA_RX_BW_40:
+		return IWL_TLC_MNG_MAX_CH_WIDTH_40MHZ;
+	case IEEE80211_STA_RX_BW_20:
+	default:
+		return IWL_TLC_MNG_MAX_CH_WIDTH_20MHZ;
+	}
+}
+
+static u8 rs_fw_set_active_chains(u8 chains)
+{
+	u8 fw_chains = 0;
+
+	if (chains & ANT_A)
+		fw_chains |= IWL_TLC_MNG_CHAIN_A_MSK;
+	if (chains & ANT_B)
+		fw_chains |= IWL_TLC_MNG_CHAIN_B_MSK;
+	if (chains & ANT_C)
+		fw_chains |= IWL_TLC_MNG_CHAIN_C_MSK;
+
+	return fw_chains;
+}
+
+static u8 rs_fw_sgi_cw_support(struct ieee80211_sta *sta)
+{
+	struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
+	u8 supp = 0;
+
+	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_20)
+		supp |= IWL_TLC_MNG_SGI_20MHZ_MSK;
+	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_40)
+		supp |= IWL_TLC_MNG_SGI_40MHZ_MSK;
+	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_80)
+		supp |= IWL_TLC_MNG_SGI_80MHZ_MSK;
+	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_160)
+		supp |= IWL_TLC_MNG_SGI_160MHZ_MSK;
+
+	return supp;
+}
+
+static u16 rs_fw_set_config_flags(struct iwl_mvm *mvm,
+				  struct ieee80211_sta *sta)
+{
+	struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
+	bool vht_ena = vht_cap && vht_cap->vht_supported;
+	u16 flags = IWL_TLC_MNG_CFG_FLAGS_CCK_MSK |
+		    IWL_TLC_MNG_CFG_FLAGS_DCM_MSK |
+		    IWL_TLC_MNG_CFG_FLAGS_DD_MSK;
+
+	if (mvm->cfg->ht_params->stbc &&
+	    (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) > 1) &&
+	    ((ht_cap && (ht_cap->cap & IEEE80211_HT_CAP_RX_STBC)) ||
+	     (vht_ena && (vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK))))
+		flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
+
+	if (mvm->cfg->ht_params->ldpc &&
+	    ((ht_cap && (ht_cap->cap & IEEE80211_HT_CAP_LDPC_CODING)) ||
+	     (vht_ena && (vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC))))
+		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
+
+	if (fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_BEAMFORMER) &&
+	    (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) > 1) &&
+	    (vht_cap->cap & IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE))
+		flags |= IWL_TLC_MNG_CFG_FLAGS_BF_MSK;
+
+	return flags;
+}
+
+static
+int rs_fw_vht_highest_rx_mcs_index(struct ieee80211_sta_vht_cap *vht_cap,
+				   int nss)
+{
+	u16 rx_mcs = le16_to_cpu(vht_cap->vht_mcs.rx_mcs_map) &
+		(0x3 << (2 * (nss - 1)));
+	rx_mcs >>= (2 * (nss - 1));
+
+	switch (rx_mcs) {
+	case IEEE80211_VHT_MCS_SUPPORT_0_7:
+		return IWL_TLC_MNG_HT_RATE_MCS7;
+	case IEEE80211_VHT_MCS_SUPPORT_0_8:
+		return IWL_TLC_MNG_HT_RATE_MCS8;
+	case IEEE80211_VHT_MCS_SUPPORT_0_9:
+		return IWL_TLC_MNG_HT_RATE_MCS9;
+	default:
+		WARN_ON_ONCE(1);
+		break;
+	}
+
+	return 0;
+}
+
+static void rs_fw_vht_set_enabled_rates(struct ieee80211_sta *sta,
+					struct ieee80211_sta_vht_cap *vht_cap,
+					struct iwl_tlc_config_cmd *cmd)
+{
+	u16 supp;
+	int i, highest_mcs;
+
+	for (i = 0; i < sta->rx_nss; i++) {
+		if (i == MAX_RS_ANT_NUM)
+			break;
+
+		highest_mcs = rs_fw_vht_highest_rx_mcs_index(vht_cap, i + 1);
+		if (!highest_mcs)
+			continue;
+
+		supp = BIT(highest_mcs + 1) - 1;
+		if (sta->bandwidth == IEEE80211_STA_RX_BW_20)
+			supp &= ~BIT(IWL_TLC_MNG_HT_RATE_MCS9);
+
+		cmd->ht_supp_rates[i] = cpu_to_le16(supp);
+	}
+}
+
+static void rs_fw_set_supp_rates(struct ieee80211_sta *sta,
+				 struct ieee80211_supported_band *sband,
+				 struct iwl_tlc_config_cmd *cmd)
+{
+	int i;
+	unsigned long tmp;
+	unsigned long supp; /* must be unsigned long for for_each_set_bit */
+	struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
+
+	/* non HT rates */
+	supp = 0;
+	tmp = sta->supp_rates[sband->band];
+	for_each_set_bit(i, &tmp, BITS_PER_LONG)
+		supp |= BIT(sband->bitrates[i].hw_value);
+
+	cmd->non_ht_supp_rates = cpu_to_le16(supp);
+	cmd->mode = IWL_TLC_MNG_MODE_NON_HT;
+
+	/* HT/VHT rates */
+	if (vht_cap && vht_cap->vht_supported) {
+		cmd->mode = IWL_TLC_MNG_MODE_VHT;
+		rs_fw_vht_set_enabled_rates(sta, vht_cap, cmd);
+	} else if (ht_cap && ht_cap->ht_supported) {
+		cmd->mode = IWL_TLC_MNG_MODE_HT;
+		cmd->ht_supp_rates[0] = cpu_to_le16(ht_cap->mcs.rx_mask[0]);
+		cmd->ht_supp_rates[1] = cpu_to_le16(ht_cap->mcs.rx_mask[1]);
+	}
+}
+
+static void rs_fw_tlc_mng_notif_req_config(struct iwl_mvm *mvm, u8 sta_id)
+{
+	u32 cmd_id = iwl_cmd_id(TLC_MNG_NOTIF_REQ_CMD, DATA_PATH_GROUP, 0);
+	struct iwl_tlc_notif_req_config_cmd cfg_cmd = {
+		.sta_id = sta_id,
+		.flags = cpu_to_le16(IWL_TLC_NOTIF_INIT_RATE_MSK),
+		.interval = cpu_to_le16(IWL_TLC_NOTIF_REQ_INTERVAL),
+	};
+	int ret;
+
+	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cfg_cmd), &cfg_cmd);
+	if (ret)
+		IWL_ERR(mvm, "Failed to send TLC notif request (%d)\n", ret);
+}
+
+void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt)
+{
+	struct iwl_tlc_update_notif *notif;
+	struct iwl_mvm_sta *mvmsta;
+	struct iwl_lq_sta_rs_fw *lq_sta;
+
+	notif = (void *)pkt->data;
+	mvmsta = iwl_mvm_sta_from_staid_rcu(mvm, notif->sta_id);
+
+	if (!mvmsta) {
+		IWL_ERR(mvm, "Invalid sta id (%d) in FW TLC notification\n",
+			notif->sta_id);
+		return;
+	}
+
+	lq_sta = &mvmsta->lq_sta.rs_fw;
+
+	if (le16_to_cpu(notif->flags) & IWL_TLC_NOTIF_INIT_RATE_MSK) {
+		lq_sta->last_rate_n_flags =
+			le32_to_cpu(notif->values[IWL_TLC_NOTIF_INIT_RATE_POS]);
+		IWL_DEBUG_RATE(mvm, "new rate_n_flags: 0x%X\n",
+			       lq_sta->last_rate_n_flags);
+	}
+}
+
+void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
+		     enum nl80211_band band)
+{
+	struct ieee80211_hw *hw = mvm->hw;
+	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
+	struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->lq_sta.rs_fw;
+	u32 cmd_id = iwl_cmd_id(TLC_MNG_CONFIG_CMD, DATA_PATH_GROUP, 0);
+	struct ieee80211_supported_band *sband;
+	struct iwl_tlc_config_cmd cfg_cmd = {
+		.sta_id = mvmsta->sta_id,
+		.max_supp_ch_width = rs_fw_bw_from_sta_bw(sta),
+		.flags = cpu_to_le16(rs_fw_set_config_flags(mvm, sta)),
+		.chains = rs_fw_set_active_chains(iwl_mvm_get_valid_tx_ant(mvm)),
+		.max_supp_ss = sta->rx_nss,
+		.max_ampdu_cnt = cpu_to_le32(mvmsta->max_agg_bufsize),
+		.sgi_ch_width_supp = rs_fw_sgi_cw_support(sta),
+	};
+	int ret;
+
+	memset(lq_sta, 0, offsetof(typeof(*lq_sta), pers));
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	iwl_mvm_reset_frame_stats(mvm);
+#endif
+	sband = hw->wiphy->bands[band];
+	rs_fw_set_supp_rates(sta, sband, &cfg_cmd);
+
+	ret = iwl_mvm_send_cmd_pdu(mvm, cmd_id, 0, sizeof(cfg_cmd), &cfg_cmd);
+	if (ret)
+		IWL_ERR(mvm, "Failed to send rate scale config (%d)\n", ret);
+
+	rs_fw_tlc_mng_notif_req_config(mvm, cfg_cmd.sta_id);
+}
+
+int rs_fw_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
+			bool enable)
+{
+	/* TODO: need to introduce a new FW cmd since LQ cmd is not relevant */
+	IWL_DEBUG_RATE(mvm, "tx protection - not implemented yet.\n");
+	return 0;
+}
+
+void iwl_mvm_rs_add_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta)
+{
+	struct iwl_lq_sta_rs_fw *lq_sta = &mvmsta->lq_sta.rs_fw;
+
+	IWL_DEBUG_RATE(mvm, "create station rate scale window\n");
+
+	lq_sta->pers.drv = mvm;
+	lq_sta->pers.sta_id = mvmsta->sta_id;
+	lq_sta->pers.chains = 0;
+	memset(lq_sta->pers.chain_signal, 0, sizeof(lq_sta->pers.chain_signal));
+	lq_sta->pers.last_rssi = S8_MIN;
+	lq_sta->last_rate_n_flags = 0;
+
+#ifdef CONFIG_MAC80211_DEBUGFS
+	lq_sta->pers.dbg_fixed_rate = 0;
+#endif
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
index c69515ed72df..47f4c7a1d80d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
@@ -42,8 +42,6 @@
 #include "mvm.h"
 #include "debugfs.h"
 
-#define RS_NAME "iwl-mvm-rs"
-
 #define IWL_RATE_MAX_WINDOW		62	/* # tx in history window */
 
 /* Calculations of success ratio are done in fixed point where 12800 is 100%.
@@ -809,7 +807,7 @@ static int rs_collect_tlc_data(struct iwl_mvm *mvm,
 		return -EINVAL;
 
 	if (tbl->column != RS_COLUMN_INVALID) {
-		struct lq_sta_pers *pers = &mvmsta->lq_sta.pers;
+		struct lq_sta_pers *pers = &mvmsta->lq_sta.rs_drv.pers;
 
 		pers->tx_stats[tbl->column][scale_index].total += attempts;
 		pers->tx_stats[tbl->column][scale_index].success += successes;
@@ -1206,7 +1204,7 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	u8 lq_color = RS_DRV_DATA_LQ_COLOR_GET(tlc_info);
 	u32 tx_resp_hwrate = (uintptr_t)info->status.status_driver_data[1];
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta;
+	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
 
 	/* Treat uninitialized rate scaling data same as non-existing. */
 	if (!lq_sta) {
@@ -1416,13 +1414,13 @@ void iwl_mvm_rs_tx_status(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 /*
  * mac80211 sends us Tx status
  */
-static void rs_mac80211_tx_status(void *mvm_r,
-				  struct ieee80211_supported_band *sband,
-				  struct ieee80211_sta *sta, void *priv_sta,
-				  struct sk_buff *skb)
+static void rs_drv_mac80211_tx_status(void *mvm_r,
+				      struct ieee80211_supported_band *sband,
+				      struct ieee80211_sta *sta, void *priv_sta,
+				      struct sk_buff *skb)
 {
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
-	struct iwl_op_mode *op_mode = (struct iwl_op_mode *)mvm_r;
+	struct iwl_op_mode *op_mode = mvm_r;
 	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 
@@ -1877,12 +1875,10 @@ static int rs_switch_to_column(struct iwl_mvm *mvm,
 	struct rs_rate *rate = &search_tbl->rate;
 	const struct rs_tx_column *column = &rs_tx_columns[col_id];
 	const struct rs_tx_column *curr_column = &rs_tx_columns[tbl->column];
-	u32 sz = (sizeof(struct iwl_scale_tbl_info) -
-		  (sizeof(struct iwl_rate_scale_data) * IWL_RATE_COUNT));
 	unsigned long rate_mask = 0;
 	u32 rate_idx = 0;
 
-	memcpy(search_tbl, tbl, sz);
+	memcpy(search_tbl, tbl, offsetof(struct iwl_scale_tbl_info, win));
 
 	rate->sgi = column->sgi;
 	rate->ant = column->ant;
@@ -2688,7 +2684,8 @@ static void rs_get_initial_rate(struct iwl_mvm *mvm,
 				struct ieee80211_sta *sta,
 				struct iwl_lq_sta *lq_sta,
 				enum nl80211_band band,
-				struct rs_rate *rate)
+				struct rs_rate *rate,
+				bool init)
 {
 	int i, nentries;
 	unsigned long active_rate;
@@ -2742,14 +2739,25 @@ static void rs_get_initial_rate(struct iwl_mvm *mvm,
 	 */
 	if (sta->vht_cap.vht_supported &&
 	    best_rssi > IWL_RS_LOW_RSSI_THRESHOLD) {
-		switch (sta->bandwidth) {
-		case IEEE80211_STA_RX_BW_160:
-		case IEEE80211_STA_RX_BW_80:
-		case IEEE80211_STA_RX_BW_40:
+		/*
+		 * In AP mode, when a new station associates, rs is initialized
+		 * immediately upon association completion, before the phy
+		 * context is updated with the association parameters, so the
+		 * sta bandwidth might be wider than the phy context allows.
+		 * To avoid this issue, always initialize rs with 20mhz
+		 * bandwidth rate, and after authorization, when the phy context
+		 * is already up-to-date, re-init rs with the correct bw.
+		 */
+		u32 bw = init ? RATE_MCS_CHAN_WIDTH_20 : rs_bw_from_sta_bw(sta);
+
+		switch (bw) {
+		case RATE_MCS_CHAN_WIDTH_40:
+		case RATE_MCS_CHAN_WIDTH_80:
+		case RATE_MCS_CHAN_WIDTH_160:
 			initial_rates = rs_optimal_rates_vht;
 			nentries = ARRAY_SIZE(rs_optimal_rates_vht);
 			break;
-		case IEEE80211_STA_RX_BW_20:
+		case RATE_MCS_CHAN_WIDTH_20:
 			initial_rates = rs_optimal_rates_vht_20mhz;
 			nentries = ARRAY_SIZE(rs_optimal_rates_vht_20mhz);
 			break;
@@ -2760,7 +2768,7 @@ static void rs_get_initial_rate(struct iwl_mvm *mvm,
 
 		active_rate = lq_sta->active_siso_rate;
 		rate->type = LQ_VHT_SISO;
-		rate->bw = rs_bw_from_sta_bw(sta);
+		rate->bw = bw;
 	} else if (sta->ht_cap.ht_supported &&
 		   best_rssi > IWL_RS_LOW_RSSI_THRESHOLD) {
 		initial_rates = rs_optimal_rates_ht;
@@ -2787,9 +2795,10 @@ static void rs_get_initial_rate(struct iwl_mvm *mvm,
 
 /* Save info about RSSI of last Rx */
 void rs_update_last_rssi(struct iwl_mvm *mvm,
-			 struct iwl_lq_sta *lq_sta,
+			 struct iwl_mvm_sta *mvmsta,
 			 struct ieee80211_rx_status *rx_status)
 {
+	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
 	int i;
 
 	lq_sta->pers.chains = rx_status->chains;
@@ -2842,7 +2851,7 @@ static void rs_initialize_lq(struct iwl_mvm *mvm,
 	tbl = &(lq_sta->lq_info[active_tbl]);
 	rate = &tbl->rate;
 
-	rs_get_initial_rate(mvm, sta, lq_sta, band, rate);
+	rs_get_initial_rate(mvm, sta, lq_sta, band, rate, init);
 	rs_init_optimal_rate(mvm, sta, lq_sta);
 
 	WARN_ONCE(rate->ant != ANT_A && rate->ant != ANT_B,
@@ -2858,15 +2867,15 @@ static void rs_initialize_lq(struct iwl_mvm *mvm,
 	iwl_mvm_send_lq_cmd(mvm, &lq_sta->lq, init);
 }
 
-static void rs_get_rate(void *mvm_r, struct ieee80211_sta *sta, void *mvm_sta,
-			struct ieee80211_tx_rate_control *txrc)
+static void rs_drv_get_rate(void *mvm_r, struct ieee80211_sta *sta,
+			    void *mvm_sta,
+			    struct ieee80211_tx_rate_control *txrc)
 {
-	struct sk_buff *skb = txrc->skb;
-	struct iwl_op_mode *op_mode __maybe_unused =
-			(struct iwl_op_mode *)mvm_r;
+	struct iwl_op_mode *op_mode = mvm_r;
 	struct iwl_mvm *mvm __maybe_unused = IWL_OP_MODE_GET_MVM(op_mode);
+	struct sk_buff *skb = txrc->skb;
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-	struct iwl_lq_sta *lq_sta = mvm_sta;
+	struct iwl_lq_sta *lq_sta;
 	struct rs_rate *optimal_rate;
 	u32 last_ucode_rate;
 
@@ -2878,18 +2887,14 @@ static void rs_get_rate(void *mvm_r, struct ieee80211_sta *sta, void *mvm_sta,
 		mvm_sta = NULL;
 	}
 
-	/* TODO: handle rate_idx_mask and rate_idx_mcs_mask */
-
-	/* Treat uninitialized rate scaling data same as non-existing. */
-	if (lq_sta && !lq_sta->pers.drv) {
-		IWL_DEBUG_RATE(mvm, "Rate scaling not initialized yet.\n");
-		mvm_sta = NULL;
-	}
-
 	/* Send management frames and NO_ACK data using lowest rate. */
 	if (rate_control_send_low(sta, mvm_sta, txrc))
 		return;
 
+	if (!mvm_sta)
+		return;
+
+	lq_sta = mvm_sta;
 	iwl_mvm_hwrate_to_tx_rate(lq_sta->last_rate_n_flags,
 				  info->band, &info->control.rates[0]);
 	info->control.rates[0].count = 1;
@@ -2906,13 +2911,13 @@ static void rs_get_rate(void *mvm_r, struct ieee80211_sta *sta, void *mvm_sta,
 	}
 }
 
-static void *rs_alloc_sta(void *mvm_rate, struct ieee80211_sta *sta,
-			  gfp_t gfp)
+static void *rs_drv_alloc_sta(void *mvm_rate, struct ieee80211_sta *sta,
+			      gfp_t gfp)
 {
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	struct iwl_op_mode *op_mode = (struct iwl_op_mode *)mvm_rate;
 	struct iwl_mvm *mvm  = IWL_OP_MODE_GET_MVM(op_mode);
-	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta;
+	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
 
 	IWL_DEBUG_RATE(mvm, "create station rate scale window\n");
 
@@ -2926,7 +2931,7 @@ static void *rs_alloc_sta(void *mvm_rate, struct ieee80211_sta *sta,
 	memset(lq_sta->pers.chain_signal, 0, sizeof(lq_sta->pers.chain_signal));
 	lq_sta->pers.last_rssi = S8_MIN;
 
-	return &mvmsta->lq_sta;
+	return lq_sta;
 }
 
 static int rs_vht_highest_rx_mcs_index(struct ieee80211_sta_vht_cap *vht_cap,
@@ -3043,7 +3048,7 @@ static void rs_vht_init(struct iwl_mvm *mvm,
 }
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
-static void iwl_mvm_reset_frame_stats(struct iwl_mvm *mvm)
+void iwl_mvm_reset_frame_stats(struct iwl_mvm *mvm)
 {
 	spin_lock_bh(&mvm->drv_stats_lock);
 	memset(&mvm->drv_rx_stats, 0, sizeof(mvm->drv_rx_stats));
@@ -3111,15 +3116,15 @@ void iwl_mvm_update_frame_stats(struct iwl_mvm *mvm, u32 rate, bool agg)
 /*
  * Called after adding a new station to initialize rate scaling
  */
-void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
-			  enum nl80211_band band, bool init)
+static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
+			     enum nl80211_band band, bool init)
 {
 	int i, j;
 	struct ieee80211_hw *hw = mvm->hw;
 	struct ieee80211_sta_ht_cap *ht_cap = &sta->ht_cap;
 	struct ieee80211_sta_vht_cap *vht_cap = &sta->vht_cap;
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta;
+	struct iwl_lq_sta *lq_sta = &mvmsta->lq_sta.rs_drv;
 	struct ieee80211_supported_band *sband;
 	unsigned long supp; /* must be unsigned long for for_each_set_bit */
 
@@ -3194,16 +3199,15 @@ void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	rs_initialize_lq(mvm, sta, lq_sta, band, init);
 }
 
-static void rs_rate_update(void *mvm_r,
-			   struct ieee80211_supported_band *sband,
-			   struct cfg80211_chan_def *chandef,
-			   struct ieee80211_sta *sta, void *priv_sta,
-			   u32 changed)
+static void rs_drv_rate_update(void *mvm_r,
+			       struct ieee80211_supported_band *sband,
+			       struct cfg80211_chan_def *chandef,
+			       struct ieee80211_sta *sta,
+			       void *priv_sta, u32 changed)
 {
+	struct iwl_op_mode *op_mode = mvm_r;
+	struct iwl_mvm *mvm __maybe_unused = IWL_OP_MODE_GET_MVM(op_mode);
 	u8 tid;
-	struct iwl_op_mode *op_mode  =
-			(struct iwl_op_mode *)mvm_r;
-	struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode);
 
 	if (!iwl_mvm_sta_from_mac80211(sta)->vif)
 		return;
@@ -3385,7 +3389,7 @@ static void rs_bfer_active_iter(void *_data,
 {
 	struct rs_bfer_active_iter_data *data = _data;
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
-	struct iwl_lq_cmd *lq_cmd = &mvmsta->lq_sta.lq;
+	struct iwl_lq_cmd *lq_cmd = &mvmsta->lq_sta.rs_drv.lq;
 	u32 ss_params = le32_to_cpu(lq_cmd->ss_params);
 
 	if (sta == data->exclude_sta)
@@ -3497,7 +3501,8 @@ static void rs_set_lq_ss_params(struct iwl_mvm *mvm,
 
 	/* Disallow BFER on another STA if active and we're a higher priority */
 	if (rs_bfer_priority_cmp(mvmsta, bfer_mvmsta) > 0) {
-		struct iwl_lq_cmd *bfersta_lq_cmd = &bfer_mvmsta->lq_sta.lq;
+		struct iwl_lq_cmd *bfersta_lq_cmd =
+			&bfer_mvmsta->lq_sta.rs_drv.lq;
 		u32 bfersta_ss_params = le32_to_cpu(bfersta_lq_cmd->ss_params);
 
 		bfersta_ss_params &= ~LQ_SS_BFER_ALLOWED;
@@ -3569,14 +3574,14 @@ static void *rs_alloc(struct ieee80211_hw *hw, struct dentry *debugfsdir)
 {
 	return hw->priv;
 }
+
 /* rate scale requires free function to be implemented */
 static void rs_free(void *mvm_rate)
 {
 	return;
 }
 
-static void rs_free_sta(void *mvm_r, struct ieee80211_sta *sta,
-			void *mvm_sta)
+static void rs_free_sta(void *mvm_r, struct ieee80211_sta *sta, void *mvm_sta)
 {
 	struct iwl_op_mode *op_mode __maybe_unused = mvm_r;
 	struct iwl_mvm *mvm __maybe_unused = IWL_OP_MODE_GET_MVM(op_mode);
@@ -3586,7 +3591,7 @@ static void rs_free_sta(void *mvm_r, struct ieee80211_sta *sta,
 }
 
 #ifdef CONFIG_MAC80211_DEBUGFS
-int rs_pretty_print_rate(char *buf, const u32 rate)
+int rs_pretty_print_rate(char *buf, int bufsz, const u32 rate)
 {
 
 	char *type, *bw;
@@ -3597,10 +3602,10 @@ int rs_pretty_print_rate(char *buf, const u32 rate)
 	    !(rate & RATE_MCS_VHT_MSK)) {
 		int index = iwl_hwrate_to_plcp_idx(rate);
 
-		return sprintf(buf, "Legacy | ANT: %s Rate: %s Mbps\n",
-			       rs_pretty_ant(ant),
-			       index == IWL_RATE_INVALID ? "BAD" :
-			       iwl_rate_mcs[index].mbps);
+		return scnprintf(buf, bufsz, "Legacy | ANT: %s Rate: %s Mbps\n",
+				 rs_pretty_ant(ant),
+				 index == IWL_RATE_INVALID ? "BAD" :
+				 iwl_rate_mcs[index].mbps);
 	}
 
 	if (rate & RATE_MCS_VHT_MSK) {
@@ -3634,12 +3639,13 @@ int rs_pretty_print_rate(char *buf, const u32 rate)
 		bw = "BAD BW";
 	}
 
-	return sprintf(buf, "%s | ANT: %s BW: %s MCS: %d NSS: %d %s%s%s%s\n",
-		       type, rs_pretty_ant(ant), bw, mcs, nss,
-		       (rate & RATE_MCS_SGI_MSK) ? "SGI " : "NGI ",
-		       (rate & RATE_MCS_STBC_MSK) ? "STBC " : "",
-		       (rate & RATE_MCS_LDPC_MSK) ? "LDPC " : "",
-		       (rate & RATE_MCS_BF_MSK) ? "BF " : "");
+	return scnprintf(buf, bufsz,
+			 "%s | ANT: %s BW: %s MCS: %d NSS: %d %s%s%s%s\n",
+			 type, rs_pretty_ant(ant), bw, mcs, nss,
+			 (rate & RATE_MCS_SGI_MSK) ? "SGI " : "NGI ",
+			 (rate & RATE_MCS_STBC_MSK) ? "STBC " : "",
+			 (rate & RATE_MCS_LDPC_MSK) ? "LDPC " : "",
+			 (rate & RATE_MCS_BF_MSK) ? "BF " : "");
 }
 
 /**
@@ -3696,65 +3702,70 @@ static ssize_t rs_sta_dbgfs_scale_table_read(struct file *file,
 	int desc = 0;
 	int i = 0;
 	ssize_t ret;
+	static const size_t bufsz = 2048;
 
 	struct iwl_lq_sta *lq_sta = file->private_data;
 	struct iwl_mvm_sta *mvmsta =
-		container_of(lq_sta, struct iwl_mvm_sta, lq_sta);
+		container_of(lq_sta, struct iwl_mvm_sta, lq_sta.rs_drv);
 	struct iwl_mvm *mvm;
 	struct iwl_scale_tbl_info *tbl = &(lq_sta->lq_info[lq_sta->active_tbl]);
 	struct rs_rate *rate = &tbl->rate;
 	u32 ss_params;
 
 	mvm = lq_sta->pers.drv;
-	buff = kmalloc(2048, GFP_KERNEL);
+	buff = kmalloc(bufsz, GFP_KERNEL);
 	if (!buff)
 		return -ENOMEM;
 
-	desc += sprintf(buff+desc, "sta_id %d\n", lq_sta->lq.sta_id);
-	desc += sprintf(buff+desc, "failed=%d success=%d rate=0%lX\n",
-			lq_sta->total_failed, lq_sta->total_success,
-			lq_sta->active_legacy_rate);
-	desc += sprintf(buff+desc, "fixed rate 0x%X\n",
-			lq_sta->pers.dbg_fixed_rate);
-	desc += sprintf(buff+desc, "valid_tx_ant %s%s%s\n",
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "sta_id %d\n", lq_sta->lq.sta_id);
+	desc += scnprintf(buff + desc, bufsz - desc,
+			  "failed=%d success=%d rate=0%lX\n",
+			  lq_sta->total_failed, lq_sta->total_success,
+			  lq_sta->active_legacy_rate);
+	desc += scnprintf(buff + desc, bufsz - desc, "fixed rate 0x%X\n",
+			  lq_sta->pers.dbg_fixed_rate);
+	desc += scnprintf(buff + desc, bufsz - desc, "valid_tx_ant %s%s%s\n",
 	    (iwl_mvm_get_valid_tx_ant(mvm) & ANT_A) ? "ANT_A," : "",
 	    (iwl_mvm_get_valid_tx_ant(mvm) & ANT_B) ? "ANT_B," : "",
 	    (iwl_mvm_get_valid_tx_ant(mvm) & ANT_C) ? "ANT_C" : "");
-	desc += sprintf(buff+desc, "lq type %s\n",
-			(is_legacy(rate)) ? "legacy" :
-			is_vht(rate) ? "VHT" : "HT");
+	desc += scnprintf(buff + desc, bufsz - desc, "lq type %s\n",
+			  (is_legacy(rate)) ? "legacy" :
+			  is_vht(rate) ? "VHT" : "HT");
 	if (!is_legacy(rate)) {
-		desc += sprintf(buff + desc, " %s",
+		desc += scnprintf(buff + desc, bufsz - desc, " %s",
 		   (is_siso(rate)) ? "SISO" : "MIMO2");
-		desc += sprintf(buff + desc, " %s",
+		desc += scnprintf(buff + desc, bufsz - desc, " %s",
 				(is_ht20(rate)) ? "20MHz" :
 				(is_ht40(rate)) ? "40MHz" :
 				(is_ht80(rate)) ? "80MHz" :
 				(is_ht160(rate)) ? "160MHz" : "BAD BW");
-		desc += sprintf(buff + desc, " %s %s %s %s\n",
+		desc += scnprintf(buff + desc, bufsz - desc, " %s %s %s %s\n",
 				(rate->sgi) ? "SGI" : "NGI",
 				(rate->ldpc) ? "LDPC" : "BCC",
 				(lq_sta->is_agg) ? "AGG on" : "",
 				(mvmsta->tlc_amsdu) ? "AMSDU on" : "");
 	}
-	desc += sprintf(buff+desc, "last tx rate=0x%X\n",
+	desc += scnprintf(buff + desc, bufsz - desc, "last tx rate=0x%X\n",
 			lq_sta->last_rate_n_flags);
-	desc += sprintf(buff+desc,
+	desc += scnprintf(buff + desc, bufsz - desc,
 			"general: flags=0x%X mimo-d=%d s-ant=0x%x d-ant=0x%x\n",
 			lq_sta->lq.flags,
 			lq_sta->lq.mimo_delim,
 			lq_sta->lq.single_stream_ant_msk,
 			lq_sta->lq.dual_stream_ant_msk);
 
-	desc += sprintf(buff+desc,
+	desc += scnprintf(buff + desc, bufsz - desc,
 			"agg: time_limit=%d dist_start_th=%d frame_cnt_limit=%d\n",
 			le16_to_cpu(lq_sta->lq.agg_time_limit),
 			lq_sta->lq.agg_disable_start_th,
 			lq_sta->lq.agg_frame_cnt_limit);
 
-	desc += sprintf(buff+desc, "reduced tpc=%d\n", lq_sta->lq.reduced_tpc);
+	desc += scnprintf(buff + desc, bufsz - desc, "reduced tpc=%d\n",
+			  lq_sta->lq.reduced_tpc);
 	ss_params = le32_to_cpu(lq_sta->lq.ss_params);
-	desc += sprintf(buff+desc, "single stream params: %s%s%s%s\n",
+	desc += scnprintf(buff + desc, bufsz - desc,
+			"single stream params: %s%s%s%s\n",
 			(ss_params & LQ_SS_PARAMS_VALID) ?
 			"VALID" : "INVALID",
 			(ss_params & LQ_SS_BFER_ALLOWED) ?
@@ -3763,7 +3774,7 @@ static ssize_t rs_sta_dbgfs_scale_table_read(struct file *file,
 			", STBC" : "",
 			(ss_params & LQ_SS_FORCE) ?
 			", FORCE" : "");
-	desc += sprintf(buff+desc,
+	desc += scnprintf(buff + desc, bufsz - desc,
 			"Start idx [0]=0x%x [1]=0x%x [2]=0x%x [3]=0x%x\n",
 			lq_sta->lq.initial_rate_index[0],
 			lq_sta->lq.initial_rate_index[1],
@@ -3773,8 +3784,9 @@ static ssize_t rs_sta_dbgfs_scale_table_read(struct file *file,
 	for (i = 0; i < LINK_QUAL_MAX_RETRY_NUM; i++) {
 		u32 r = le32_to_cpu(lq_sta->lq.rs_table[i]);
 
-		desc += sprintf(buff+desc, " rate[%d] 0x%X ", i, r);
-		desc += rs_pretty_print_rate(buff+desc, r);
+		desc += scnprintf(buff + desc, bufsz - desc,
+				  " rate[%d] 0x%X ", i, r);
+		desc += rs_pretty_print_rate(buff + desc, bufsz - desc, r);
 	}
 
 	ret = simple_read_from_buffer(user_buf, count, ppos, buff, desc);
@@ -3987,12 +3999,13 @@ static ssize_t iwl_dbgfs_ss_force_write(struct iwl_lq_sta *lq_sta, char *buf,
 
 MVM_DEBUGFS_READ_WRITE_FILE_OPS(ss_force, 32);
 
-static void rs_add_debugfs(void *mvm, void *priv_sta, struct dentry *dir)
+static void rs_drv_add_sta_debugfs(void *mvm, void *priv_sta,
+				   struct dentry *dir)
 {
 	struct iwl_lq_sta *lq_sta = priv_sta;
 	struct iwl_mvm_sta *mvmsta;
 
-	mvmsta = container_of(lq_sta, struct iwl_mvm_sta, lq_sta);
+	mvmsta = container_of(lq_sta, struct iwl_mvm_sta, lq_sta.rs_drv);
 
 	if (!mvmsta->vif)
 		return;
@@ -4014,7 +4027,7 @@ static void rs_add_debugfs(void *mvm, void *priv_sta, struct dentry *dir)
 	IWL_ERR((struct iwl_mvm *)mvm, "Can't create debugfs entity\n");
 }
 
-static void rs_remove_debugfs(void *mvm, void *mvm_sta)
+void rs_remove_sta_debugfs(void *mvm, void *mvm_sta)
 {
 }
 #endif
@@ -4024,50 +4037,53 @@ static void rs_remove_debugfs(void *mvm, void *mvm_sta)
  * the station is added. Since mac80211 calls this function before a
  * station is added we ignore it.
  */
-static void rs_rate_init_stub(void *mvm_r,
-			      struct ieee80211_supported_band *sband,
-			      struct cfg80211_chan_def *chandef,
-			      struct ieee80211_sta *sta, void *mvm_sta)
+static void rs_rate_init_ops(void *mvm_r,
+			     struct ieee80211_supported_band *sband,
+			     struct cfg80211_chan_def *chandef,
+			     struct ieee80211_sta *sta, void *mvm_sta)
 {
 }
 
-static const struct rate_control_ops rs_mvm_ops = {
+/* ops for rate scaling implemented in the driver */
+static const struct rate_control_ops rs_mvm_ops_drv = {
 	.name = RS_NAME,
-	.tx_status = rs_mac80211_tx_status,
-	.get_rate = rs_get_rate,
-	.rate_init = rs_rate_init_stub,
+	.tx_status = rs_drv_mac80211_tx_status,
+	.get_rate = rs_drv_get_rate,
+	.rate_init = rs_rate_init_ops,
 	.alloc = rs_alloc,
 	.free = rs_free,
-	.alloc_sta = rs_alloc_sta,
+	.alloc_sta = rs_drv_alloc_sta,
 	.free_sta = rs_free_sta,
-	.rate_update = rs_rate_update,
+	.rate_update = rs_drv_rate_update,
 #ifdef CONFIG_MAC80211_DEBUGFS
-	.add_sta_debugfs = rs_add_debugfs,
-	.remove_sta_debugfs = rs_remove_debugfs,
+	.add_sta_debugfs = rs_drv_add_sta_debugfs,
+	.remove_sta_debugfs = rs_remove_sta_debugfs,
 #endif
 };
 
+void iwl_mvm_rs_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
+			  enum nl80211_band band, bool init)
+{
+	if (iwl_mvm_has_tlc_offload(mvm))
+		rs_fw_rate_init(mvm, sta, band);
+	else
+		rs_drv_rate_init(mvm, sta, band, init);
+}
+
 int iwl_mvm_rate_control_register(void)
 {
-	return ieee80211_rate_control_register(&rs_mvm_ops);
+	return ieee80211_rate_control_register(&rs_mvm_ops_drv);
 }
 
 void iwl_mvm_rate_control_unregister(void)
 {
-	ieee80211_rate_control_unregister(&rs_mvm_ops);
+	ieee80211_rate_control_unregister(&rs_mvm_ops_drv);
 }
 
-/**
- * iwl_mvm_tx_protection - Gets LQ command, change it to enable/disable
- * Tx protection, according to this request and previous requests,
- * and send the LQ command.
- * @mvmsta: The station
- * @enable: Enable Tx protection?
- */
-int iwl_mvm_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
-			  bool enable)
+static int rs_drv_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
+				bool enable)
 {
-	struct iwl_lq_cmd *lq = &mvmsta->lq_sta.lq;
+	struct iwl_lq_cmd *lq = &mvmsta->lq_sta.rs_drv.lq;
 
 	lockdep_assert_held(&mvm->mutex);
 
@@ -4083,3 +4099,17 @@ int iwl_mvm_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 
 	return iwl_mvm_send_lq_cmd(mvm, lq, false);
 }
+
+/**
+ * iwl_mvm_tx_protection - ask FW to enable RTS/CTS protection
+ * @mvmsta: The station
+ * @enable: Enable Tx protection?
+ */
+int iwl_mvm_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
+			  bool enable)
+{
+	if (iwl_mvm_has_tlc_offload(mvm))
+		return rs_fw_tx_protection(mvm, mvmsta, enable);
+	else
+		return rs_drv_tx_protection(mvm, mvmsta, enable);
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.h b/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
index 32b4d66debea..fb18cb8c233d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.h
@@ -36,6 +36,8 @@
 #include "fw-api.h"
 #include "iwl-trans.h"
 
+#define RS_NAME "iwl-mvm-rs"
+
 struct iwl_rs_rate_info {
 	u8 plcp;	  /* uCode API:  IWL_RATE_6M_PLCP, etc. */
 	u8 plcp_ht_siso;  /* uCode API:  IWL_RATE_SISO_6M_PLCP, etc. */
@@ -217,6 +219,38 @@ struct iwl_rate_mcs_info {
 	char	mcs[IWL_MAX_MCS_DISPLAY_SIZE];
 };
 
+/**
+ * struct iwl_lq_sta_rs_fw - rate and related statistics for RS in FW
+ * @last_rate_n_flags: last rate reported by FW
+ * @sta_id: the id of the station
+#ifdef CONFIG_MAC80211_DEBUGFS
+ * @dbg_fixed_rate: for debug, use fixed rate if not 0
+ * @dbg_agg_frame_count_lim: for debug, max number of frames in A-MPDU
+#endif
+ * @chains: bitmask of chains reported in %chain_signal
+ * @chain_signal: per chain signal strength
+ * @last_rssi: last rssi reported
+ * @drv: pointer back to the driver data
+ */
+
+struct iwl_lq_sta_rs_fw {
+	/* last tx rate_n_flags */
+	u32 last_rate_n_flags;
+
+	/* persistent fields - initialized only once - keep last! */
+	struct lq_sta_pers_rs_fw {
+		u32 sta_id;
+#ifdef CONFIG_MAC80211_DEBUGFS
+		u32 dbg_fixed_rate;
+		u16 dbg_agg_frame_count_lim;
+#endif
+		u8 chains;
+		s8 chain_signal[IEEE80211_MAX_CHAINS];
+		s8 last_rssi;
+		struct iwl_mvm *drv;
+	} pers;
+};
+
 /**
  * struct iwl_rate_scale_data -- tx success history for one rate
  */
@@ -407,4 +441,18 @@ struct iwl_mvm_sta;
 int iwl_mvm_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
 			  bool enable);
 
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+void iwl_mvm_reset_frame_stats(struct iwl_mvm *mvm);
+#endif
+
+#ifdef CONFIG_MAC80211_DEBUGFS
+void rs_remove_sta_debugfs(void *mvm, void *mvm_sta);
+#endif
+
+void iwl_mvm_rs_add_sta(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta);
+void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
+		     enum nl80211_band band);
+int rs_fw_tx_protection(struct iwl_mvm *mvm, struct iwl_mvm_sta *mvmsta,
+			bool enable);
+void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt);
 #endif /* __rs__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
index d1a40688d5e1..d26833c5ce1f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
@@ -222,7 +222,9 @@ static u32 iwl_mvm_set_mac80211_rx_flag(struct iwl_mvm *mvm,
 
 	case RX_MPDU_RES_STATUS_SEC_TKIP_ENC:
 		/* Don't drop the frame and decrypt it in SW */
-		if (!(rx_pkt_status & RX_MPDU_RES_STATUS_TTAK_OK))
+		if (!fw_has_api(&mvm->fw->ucode_capa,
+				IWL_UCODE_TLV_API_DEPRECATE_TTAK) &&
+		    !(rx_pkt_status & RX_MPDU_RES_STATUS_TTAK_OK))
 			return 0;
 		*crypt_len = IEEE80211_TKIP_IV_LEN;
 		/* fall through if TTAK OK */
@@ -383,7 +385,7 @@ void iwl_mvm_rx_rx_mpdu(struct iwl_mvm *mvm, struct napi_struct *napi,
 								 false);
 		}
 
-		rs_update_last_rssi(mvm, &mvmsta->lq_sta, rx_status);
+		rs_update_last_rssi(mvm, mvmsta, rx_status);
 
 		if (iwl_fw_dbg_trigger_enabled(mvm->fw, FW_DBG_TRIGGER_RSSI) &&
 		    ieee80211_is_beacon(hdr->frame_control)) {
@@ -439,7 +441,8 @@ void iwl_mvm_rx_rx_mpdu(struct iwl_mvm *mvm, struct napi_struct *napi,
 		rx_status->bw = RATE_INFO_BW_160;
 		break;
 	}
-	if (rate_n_flags & RATE_MCS_SGI_MSK)
+	if (!(rate_n_flags & RATE_MCS_CCK_MSK) &&
+	    rate_n_flags & RATE_MCS_SGI_MSK)
 		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
 	if (rate_n_flags & RATE_HT_MCS_GF_MSK)
 		rx_status->enc_flags |= RX_ENC_FLAG_HT_GF;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
index 3b8d44361380..580de5851fc7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
@@ -71,6 +71,7 @@ static inline int iwl_mvm_check_pn(struct iwl_mvm *mvm, struct sk_buff *skb,
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct ieee80211_rx_status *stats = IEEE80211_SKB_RXCB(skb);
 	struct iwl_mvm_key_pn *ptk_pn;
+	int res;
 	u8 tid, keyidx;
 	u8 pn[IEEE80211_CCMP_PN_LEN];
 	u8 *extiv;
@@ -127,12 +128,13 @@ static inline int iwl_mvm_check_pn(struct iwl_mvm *mvm, struct sk_buff *skb,
 	pn[4] = extiv[1];
 	pn[5] = extiv[0];
 
-	if (memcmp(pn, ptk_pn->q[queue].pn[tid],
-		   IEEE80211_CCMP_PN_LEN) <= 0)
+	res = memcmp(pn, ptk_pn->q[queue].pn[tid], IEEE80211_CCMP_PN_LEN);
+	if (res < 0)
+		return -1;
+	if (!res && !(stats->flag & RX_FLAG_ALLOW_SAME_PN))
 		return -1;
 
-	if (!(stats->flag & RX_FLAG_AMSDU_MORE))
-		memcpy(ptk_pn->q[queue].pn[tid], pn, IEEE80211_CCMP_PN_LEN);
+	memcpy(ptk_pn->q[queue].pn[tid], pn, IEEE80211_CCMP_PN_LEN);
 	stats->flag |= RX_FLAG_PN_VALIDATED;
 
 	return 0;
@@ -261,7 +263,9 @@ static int iwl_mvm_rx_crypto(struct iwl_mvm *mvm, struct ieee80211_hdr *hdr,
 		return 0;
 	case IWL_RX_MPDU_STATUS_SEC_TKIP:
 		/* Don't drop the frame and decrypt it in SW */
-		if (!(status & IWL_RX_MPDU_RES_STATUS_TTAK_OK))
+		if (!fw_has_api(&mvm->fw->ucode_capa,
+				IWL_UCODE_TLV_API_DEPRECATE_TTAK) &&
+		    !(status & IWL_RX_MPDU_RES_STATUS_TTAK_OK))
 			return 0;
 
 		*crypt_len = IEEE80211_TKIP_IV_LEN;
@@ -312,28 +316,21 @@ static void iwl_mvm_rx_csum(struct ieee80211_sta *sta,
 }
 
 /*
- * returns true if a packet outside BA session is a duplicate and
- * should be dropped
+ * returns true if a packet is a duplicate and should be dropped.
+ * Updates AMSDU PN tracking info
  */
-static bool iwl_mvm_is_nonagg_dup(struct ieee80211_sta *sta, int queue,
-				  struct ieee80211_rx_status *rx_status,
-				  struct ieee80211_hdr *hdr,
-				  struct iwl_rx_mpdu_desc *desc)
+static bool iwl_mvm_is_dup(struct ieee80211_sta *sta, int queue,
+			   struct ieee80211_rx_status *rx_status,
+			   struct ieee80211_hdr *hdr,
+			   struct iwl_rx_mpdu_desc *desc)
 {
 	struct iwl_mvm_sta *mvm_sta;
 	struct iwl_mvm_rxq_dup_data *dup_data;
-	u8 baid, tid, sub_frame_idx;
+	u8 tid, sub_frame_idx;
 
 	if (WARN_ON(IS_ERR_OR_NULL(sta)))
 		return false;
 
-	baid = (le32_to_cpu(desc->reorder_data) &
-		IWL_RX_MPDU_REORDER_BAID_MASK) >>
-		IWL_RX_MPDU_REORDER_BAID_SHIFT;
-
-	if (baid != IWL_RX_REORDER_DATA_INVALID_BAID)
-		return false;
-
 	mvm_sta = iwl_mvm_sta_from_mac80211(sta);
 	dup_data = &mvm_sta->dup_data[queue];
 
@@ -363,6 +360,12 @@ static bool iwl_mvm_is_nonagg_dup(struct ieee80211_sta *sta, int queue,
 		     dup_data->last_sub_frame[tid] >= sub_frame_idx))
 		return true;
 
+	/* Allow same PN as the first subframe for following sub frames */
+	if (dup_data->last_seq[tid] == hdr->seq_ctrl &&
+	    sub_frame_idx > dup_data->last_sub_frame[tid] &&
+	    desc->mac_flags2 & IWL_RX_MPDU_MFLG2_AMSDU)
+		rx_status->flag |= RX_FLAG_ALLOW_SAME_PN;
+
 	dup_data->last_seq[tid] = hdr->seq_ctrl;
 	dup_data->last_sub_frame[tid] = sub_frame_idx;
 
@@ -943,7 +946,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 								 false);
 		}
 
-		rs_update_last_rssi(mvm, &mvmsta->lq_sta, rx_status);
+		rs_update_last_rssi(mvm, mvmsta, rx_status);
 
 		if (iwl_fw_dbg_trigger_enabled(mvm->fw, FW_DBG_TRIGGER_RSSI) &&
 		    ieee80211_is_beacon(hdr->frame_control)) {
@@ -969,7 +972,7 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		if (ieee80211_is_data(hdr->frame_control))
 			iwl_mvm_rx_csum(sta, skb, desc);
 
-		if (iwl_mvm_is_nonagg_dup(sta, queue, rx_status, hdr, desc)) {
+		if (iwl_mvm_is_dup(sta, queue, rx_status, hdr, desc)) {
 			kfree_skb(skb);
 			goto out;
 		}
@@ -1020,7 +1023,9 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		rx_status->bw = RATE_INFO_BW_160;
 		break;
 	}
-	if (rate_n_flags & RATE_MCS_SGI_MSK)
+
+	if (!(rate_n_flags & RATE_MCS_CCK_MSK) &&
+	    rate_n_flags & RATE_MCS_SGI_MSK)
 		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
 	if (rate_n_flags & RATE_HT_MCS_GF_MSK)
 		rx_status->enc_flags |= RX_ENC_FLAG_HT_GF;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index e4fd476e9ccb..356b16f40e78 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -664,6 +664,22 @@ static u8 *iwl_mvm_copy_and_insert_ds_elem(struct iwl_mvm *mvm, const u8 *ies,
 	return newpos;
 }
 
+#define WFA_TPC_IE_LEN	9
+
+static void iwl_mvm_add_tpc_report_ie(u8 *pos)
+{
+	pos[0] = WLAN_EID_VENDOR_SPECIFIC;
+	pos[1] = WFA_TPC_IE_LEN - 2;
+	pos[2] = (WLAN_OUI_MICROSOFT >> 16) & 0xff;
+	pos[3] = (WLAN_OUI_MICROSOFT >> 8) & 0xff;
+	pos[4] = WLAN_OUI_MICROSOFT & 0xff;
+	pos[5] = WLAN_OUI_TYPE_MICROSOFT_TPC;
+	pos[6] = 0;
+	/* pos[7] - tx power will be inserted by the FW */
+	pos[7] = 0;
+	pos[8] = 0;
+}
+
 static void
 iwl_mvm_build_scan_probe(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			 struct ieee80211_scan_ies *ies,
@@ -716,7 +732,16 @@ iwl_mvm_build_scan_probe(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	memcpy(pos, ies->common_ies, ies->common_ie_len);
 	params->preq.common_data.offset = cpu_to_le16(pos - params->preq.buf);
-	params->preq.common_data.len = cpu_to_le16(ies->common_ie_len);
+
+	if (iwl_mvm_rrm_scan_needed(mvm) &&
+	    !fw_has_capa(&mvm->fw->ucode_capa,
+			 IWL_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT)) {
+		iwl_mvm_add_tpc_report_ie(pos + ies->common_ie_len);
+		params->preq.common_data.len = cpu_to_le16(ies->common_ie_len +
+							   WFA_TPC_IE_LEN);
+	} else {
+		params->preq.common_data.len = cpu_to_le16(ies->common_ie_len);
+	}
 }
 
 static void iwl_mvm_scan_lmac_dwell(struct iwl_mvm *mvm,
@@ -781,7 +806,9 @@ static int iwl_mvm_scan_lmac_flags(struct iwl_mvm *mvm,
 	if (params->type == IWL_SCAN_TYPE_FRAGMENTED)
 		flags |= IWL_MVM_LMAC_SCAN_FLAG_FRAGMENTED;
 
-	if (iwl_mvm_rrm_scan_needed(mvm))
+	if (iwl_mvm_rrm_scan_needed(mvm) &&
+	    fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT))
 		flags |= IWL_MVM_LMAC_SCAN_FLAGS_RRM_ENABLED;
 
 	if (params->pass_all)
@@ -1183,7 +1210,9 @@ static u16 iwl_mvm_scan_umac_flags(struct iwl_mvm *mvm,
 			flags |= IWL_UMAC_SCAN_GEN_FLAGS_LMAC2_FRAGMENTED;
 	}
 
-	if (iwl_mvm_rrm_scan_needed(mvm))
+	if (iwl_mvm_rrm_scan_needed(mvm) &&
+	    fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_WFA_TPC_REP_IE_SUPPORT))
 		flags |= IWL_UMAC_SCAN_GEN_FLAGS_RRM_ENABLED;
 
 	if (params->pass_all)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
index 1add5615fc3a..80067eb9ea05 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
@@ -1439,6 +1439,13 @@ int iwl_mvm_add_sta(struct iwl_mvm *mvm,
 			goto err;
 	}
 
+	/*
+	 * if rs is registered with mac80211, then "add station" will be handled
+	 * via the corresponding ops, otherwise need to notify rate scaling here
+	 */
+	if (iwl_mvm_has_tlc_offload(mvm))
+		iwl_mvm_rs_add_sta(mvm, mvm_sta);
+
 update_fw:
 	ret = iwl_mvm_sta_send_to_fw(mvm, sta, sta_update, sta_flags);
 	if (ret)
@@ -1688,7 +1695,8 @@ int iwl_mvm_allocate_int_sta(struct iwl_mvm *mvm,
 			     u32 qmask, enum nl80211_iftype iftype,
 			     enum iwl_sta_type type)
 {
-	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) {
+	if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) ||
+	    sta->sta_id == IWL_MVM_INVALID_STA) {
 		sta->sta_id = iwl_mvm_find_free_sta_id(mvm, iftype);
 		if (WARN_ON_ONCE(sta->sta_id == IWL_MVM_INVALID_STA))
 			return -ENOSPC;
@@ -1762,7 +1770,7 @@ int iwl_mvm_add_aux_sta(struct iwl_mvm *mvm)
 	}
 
 	/*
-	 * For a000 firmware and on we cannot add queue to a station unknown
+	 * For 22000 firmware and on we cannot add queue to a station unknown
 	 * to firmware so enable queue here - after the station was added
 	 */
 	if (iwl_mvm_has_new_tx_api(mvm))
@@ -1885,7 +1893,7 @@ int iwl_mvm_send_add_bcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		return ret;
 
 	/*
-	 * For a000 firmware and on we cannot add queue to a station unknown
+	 * For 22000 firmware and on we cannot add queue to a station unknown
 	 * to firmware so enable queue here - after the station was added
 	 */
 	if (iwl_mvm_has_new_tx_api(mvm)) {
@@ -2032,7 +2040,7 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	struct iwl_trans_txq_scd_cfg cfg = {
 		.fifo = IWL_MVM_TX_FIFO_MCAST,
 		.sta_id = msta->sta_id,
-		.tid = IWL_MAX_TID_COUNT,
+		.tid = 0,
 		.aggregate = false,
 		.frame_limit = IWL_FRAME_LIMIT,
 	};
@@ -2045,6 +2053,17 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 		    vif->type != NL80211_IFTYPE_ADHOC))
 		return -ENOTSUPP;
 
+	/*
+	 * In IBSS, ieee80211_check_queues() sets the cab_queue to be
+	 * invalid, so make sure we use the queue we want.
+	 * Note that this is done here as we want to avoid making DQA
+	 * changes in mac80211 layer.
+	 */
+	if (vif->type == NL80211_IFTYPE_ADHOC) {
+		vif->cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
+		mvmvif->cab_queue = vif->cab_queue;
+	}
+
 	/*
 	 * While in previous FWs we had to exclude cab queue from TFD queue
 	 * mask, now it is needed as any other queue.
@@ -2064,7 +2083,7 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 
 	/*
 	 * Enable cab queue after the ADD_STA command is sent.
-	 * This is needed for a000 firmware which won't accept SCD_QUEUE_CFG
+	 * This is needed for 22000 firmware which won't accept SCD_QUEUE_CFG
 	 * command with unknown station id, and for FW that doesn't support
 	 * station API since the cab queue is not included in the
 	 * tfd_queue_mask.
@@ -2072,24 +2091,13 @@ int iwl_mvm_add_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	if (iwl_mvm_has_new_tx_api(mvm)) {
 		int queue = iwl_mvm_tvqm_enable_txq(mvm, vif->cab_queue,
 						    msta->sta_id,
-						    IWL_MAX_TID_COUNT,
+						    0,
 						    timeout);
 		mvmvif->cab_queue = queue;
 	} else if (!fw_has_api(&mvm->fw->ucode_capa,
-			       IWL_UCODE_TLV_API_STA_TYPE)) {
-		/*
-		 * In IBSS, ieee80211_check_queues() sets the cab_queue to be
-		 * invalid, so make sure we use the queue we want.
-		 * Note that this is done here as we want to avoid making DQA
-		 * changes in mac80211 layer.
-		 */
-		if (vif->type == NL80211_IFTYPE_ADHOC) {
-			vif->cab_queue = IWL_MVM_DQA_GCAST_QUEUE;
-			mvmvif->cab_queue = vif->cab_queue;
-		}
+			       IWL_UCODE_TLV_API_STA_TYPE))
 		iwl_mvm_enable_txq(mvm, vif->cab_queue, vif->cab_queue, 0,
 				   &cfg, timeout);
-	}
 
 	return 0;
 }
@@ -2108,7 +2116,7 @@ int iwl_mvm_rm_mcast_sta(struct iwl_mvm *mvm, struct ieee80211_vif *vif)
 	iwl_mvm_flush_sta(mvm, &mvmvif->mcast_sta, true, 0);
 
 	iwl_mvm_disable_txq(mvm, mvmvif->cab_queue, vif->cab_queue,
-			    IWL_MAX_TID_COUNT, 0);
+			    0, 0);
 
 	ret = iwl_mvm_rm_sta_common(mvm, mvmvif->mcast_sta.sta_id);
 	if (ret)
@@ -2471,28 +2479,12 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	/*
 	 * Note the possible cases:
-	 *  1. In DQA mode with an enabled TXQ - TXQ needs to become agg'ed
-	 *  2. Non-DQA mode: the TXQ hasn't yet been enabled, so find a free
-	 *	one and mark it as reserved
-	 *  3. In DQA mode, but no traffic yet on this TID: same treatment as in
-	 *	non-DQA mode, since the TXQ hasn't yet been allocated
-	 * Don't support case 3 for new TX path as it is not expected to happen
-	 * and aggregation will be offloaded soon anyway
+	 *  1. An enabled TXQ - TXQ needs to become agg'ed
+	 *  2. The TXQ hasn't yet been enabled, so find a free one and mark
+	 *	it as reserved
 	 */
 	txq_id = mvmsta->tid_data[tid].txq_id;
-	if (iwl_mvm_has_new_tx_api(mvm)) {
-		if (txq_id == IWL_MVM_INVALID_QUEUE) {
-			ret = -ENXIO;
-			goto release_locks;
-		}
-	} else if (unlikely(mvm->queue_info[txq_id].status ==
-			    IWL_MVM_QUEUE_SHARED)) {
-		ret = -ENXIO;
-		IWL_DEBUG_TX_QUEUES(mvm,
-				    "Can't start tid %d agg on shared queue!\n",
-				    tid);
-		goto release_locks;
-	} else if (mvm->queue_info[txq_id].status != IWL_MVM_QUEUE_READY) {
+	if (txq_id == IWL_MVM_INVALID_QUEUE) {
 		txq_id = iwl_mvm_find_free_queue(mvm, mvmsta->sta_id,
 						 IWL_MVM_DQA_MIN_DATA_QUEUE,
 						 IWL_MVM_DQA_MAX_DATA_QUEUE);
@@ -2501,16 +2493,16 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			IWL_ERR(mvm, "Failed to allocate agg queue\n");
 			goto release_locks;
 		}
-		/*
-		 * TXQ shouldn't be in inactive mode for non-DQA, so getting
-		 * an inactive queue from iwl_mvm_find_free_queue() is
-		 * certainly a bug
-		 */
-		WARN_ON(mvm->queue_info[txq_id].status ==
-			IWL_MVM_QUEUE_INACTIVE);
 
 		/* TXQ hasn't yet been enabled, so mark it only as reserved */
 		mvm->queue_info[txq_id].status = IWL_MVM_QUEUE_RESERVED;
+	} else if (unlikely(mvm->queue_info[txq_id].status ==
+			    IWL_MVM_QUEUE_SHARED)) {
+		ret = -ENXIO;
+		IWL_DEBUG_TX_QUEUES(mvm,
+				    "Can't start tid %d agg on shared queue!\n",
+				    tid);
+		goto release_locks;
 	}
 
 	spin_unlock(&mvm->queue_info_lock);
@@ -2530,7 +2522,7 @@ int iwl_mvm_sta_tx_agg_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			    tid_data->next_reclaimed);
 
 	/*
-	 * In A000 HW, the next_reclaimed index is only 8 bit, so we'll need
+	 * In 22000 HW, the next_reclaimed index is only 8 bit, so we'll need
 	 * to align the wrap around of ssn so we compare relevant values.
 	 */
 	normalized_ssn = tid_data->ssn;
@@ -2575,6 +2567,13 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		.aggregate = true,
 	};
 
+	/*
+	 * When FW supports TLC_OFFLOAD, it also implements Tx aggregation
+	 * manager, so this function should never be called in this case.
+	 */
+	if (WARN_ON_ONCE(iwl_mvm_has_tlc_offload(mvm)))
+		return -EINVAL;
+
 	BUILD_BUG_ON((sizeof(mvmsta->agg_tids) * BITS_PER_BYTE)
 		     != IWL_MAX_TID_COUNT);
 
@@ -2672,18 +2671,20 @@ int iwl_mvm_sta_tx_agg_oper(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	 */
 	mvmsta->max_agg_bufsize =
 		min(mvmsta->max_agg_bufsize, buf_size);
-	mvmsta->lq_sta.lq.agg_frame_cnt_limit = mvmsta->max_agg_bufsize;
+	mvmsta->lq_sta.rs_drv.lq.agg_frame_cnt_limit = mvmsta->max_agg_bufsize;
 
 	IWL_DEBUG_HT(mvm, "Tx aggregation enabled on ra = %pM tid = %d\n",
 		     sta->addr, tid);
 
-	return iwl_mvm_send_lq_cmd(mvm, &mvmsta->lq_sta.lq, false);
+	return iwl_mvm_send_lq_cmd(mvm, &mvmsta->lq_sta.rs_drv.lq, false);
 }
 
 static void iwl_mvm_unreserve_agg_queue(struct iwl_mvm *mvm,
 					struct iwl_mvm_sta *mvmsta,
-					u16 txq_id)
+					struct iwl_mvm_tid_data *tid_data)
 {
+	u16 txq_id = tid_data->txq_id;
+
 	if (iwl_mvm_has_new_tx_api(mvm))
 		return;
 
@@ -2695,8 +2696,10 @@ static void iwl_mvm_unreserve_agg_queue(struct iwl_mvm *mvm,
 	 * allocated through iwl_mvm_enable_txq, so we can just mark it back as
 	 * free.
 	 */
-	if (mvm->queue_info[txq_id].status == IWL_MVM_QUEUE_RESERVED)
+	if (mvm->queue_info[txq_id].status == IWL_MVM_QUEUE_RESERVED) {
 		mvm->queue_info[txq_id].status = IWL_MVM_QUEUE_FREE;
+		tid_data->txq_id = IWL_MVM_INVALID_QUEUE;
+	}
 
 	spin_unlock_bh(&mvm->queue_info_lock);
 }
@@ -2727,7 +2730,7 @@ int iwl_mvm_sta_tx_agg_stop(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 
 	mvmsta->agg_tids &= ~BIT(tid);
 
-	iwl_mvm_unreserve_agg_queue(mvm, mvmsta, txq_id);
+	iwl_mvm_unreserve_agg_queue(mvm, mvmsta, tid_data);
 
 	switch (tid_data->state) {
 	case IWL_AGG_ON:
@@ -2794,7 +2797,7 @@ int iwl_mvm_sta_tx_agg_flush(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	mvmsta->agg_tids &= ~BIT(tid);
 	spin_unlock_bh(&mvmsta->lock);
 
-	iwl_mvm_unreserve_agg_queue(mvm, mvmsta, txq_id);
+	iwl_mvm_unreserve_agg_queue(mvm, mvmsta, tid_data);
 
 	if (old_state >= IWL_AGG_ON) {
 		iwl_mvm_drain_sta(mvm, mvmsta, true);
@@ -3156,8 +3159,9 @@ static int __iwl_mvm_remove_sta_key(struct iwl_mvm *mvm, u8 sta_id,
 	int ret, size;
 	u32 status;
 
+	/* This is a valid situation for GTK removal */
 	if (sta_id == IWL_MVM_INVALID_STA)
-		return -EINVAL;
+		return 0;
 
 	key_flags = cpu_to_le16((keyconf->keyidx << STA_KEY_FLG_KEYID_POS) &
 				 STA_KEY_FLG_KEYID_MSK);
@@ -3218,17 +3222,9 @@ int iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 		}
 		sta_id = mvm_sta->sta_id;
 
-		if (keyconf->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
-		    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_128 ||
-		    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256) {
-			ret = iwl_mvm_send_sta_igtk(mvm, keyconf, sta_id,
-						    false);
-			goto end;
-		}
-
 		/*
 		 * It is possible that the 'sta' parameter is NULL, and thus
-		 * there is a need to retrieve  the sta from the local station
+		 * there is a need to retrieve the sta from the local station
 		 * table.
 		 */
 		if (!sta) {
@@ -3243,6 +3239,17 @@ int iwl_mvm_set_sta_key(struct iwl_mvm *mvm,
 
 		if (WARN_ON_ONCE(iwl_mvm_sta_from_mac80211(sta)->vif != vif))
 			return -EINVAL;
+	} else {
+		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
+		sta_id = mvmvif->mcast_sta.sta_id;
+	}
+
+	if (keyconf->cipher == WLAN_CIPHER_SUITE_AES_CMAC ||
+	    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_128 ||
+	    keyconf->cipher == WLAN_CIPHER_SUITE_BIP_GMAC_256) {
+		ret = iwl_mvm_send_sta_igtk(mvm, keyconf, sta_id, false);
+		goto end;
 	}
 
 	/* If the key_offset is not pre-assigned, we need to find a
@@ -3615,7 +3622,7 @@ u16 iwl_mvm_tid_queued(struct iwl_mvm *mvm, struct iwl_mvm_tid_data *tid_data)
 	u16 sn = IEEE80211_SEQ_TO_SN(tid_data->seq_number);
 
 	/*
-	 * In A000 HW, the next_reclaimed index is only 8 bit, so we'll need
+	 * In 22000 HW, the next_reclaimed index is only 8 bit, so we'll need
 	 * to align the wrap around of ssn so we compare relevant values.
 	 */
 	if (mvm->trans->cfg->gen2)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
index aedabe101cf0..5ffd6adbc383 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h
@@ -383,6 +383,8 @@ struct iwl_mvm_rxq_dup_data {
  * and from Tx response flow, it needs a spinlock.
  * @tid_data: per tid data + mgmt. Look at %iwl_mvm_tid_data.
  * @tid_to_baid: a simple map of TID to baid
+ * @lq_sta: holds rate scaling data, either for the case when RS is done in
+ *	the driver - %rs_drv or in the FW - %rs_fw.
  * @reserved_queue: the queue reserved for this STA for DQA purposes
  *	Every STA has is given one reserved queue to allow it to operate. If no
  *	such queue can be guaranteed, the STA addition will fail.
@@ -417,7 +419,10 @@ struct iwl_mvm_sta {
 	spinlock_t lock;
 	struct iwl_mvm_tid_data tid_data[IWL_MAX_TID_COUNT + 1];
 	u8 tid_to_baid[IWL_MAX_TID_COUNT];
-	struct iwl_lq_sta lq_sta;
+	union {
+		struct iwl_lq_sta_rs_fw rs_fw;
+		struct iwl_lq_sta rs_drv;
+	} lq_sta;
 	struct ieee80211_vif *vif;
 	struct iwl_mvm_key_pn __rcu *ptk_pn[4];
 	struct iwl_mvm_rxq_dup_data *dup_data;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
index e25cda9fbf6c..cd91bc44259c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c
@@ -8,6 +8,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of version 2 of the GNU General Public License as
@@ -18,11 +19,6 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
- * USA
- *
  * The full GNU General Public License is included in this distribution
  * in the file called COPYING.
  *
@@ -35,6 +31,7 @@
  * Copyright(c) 2012 - 2014 Intel Corporation. All rights reserved.
  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  * Copyright(c) 2017 Intel Deutschland GmbH
+ * Copyright(c) 2018 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -101,7 +98,6 @@ void iwl_mvm_te_clear_data(struct iwl_mvm *mvm,
 void iwl_mvm_roc_done_wk(struct work_struct *wk)
 {
 	struct iwl_mvm *mvm = container_of(wk, struct iwl_mvm, roc_done_wk);
-	u32 queues = 0;
 
 	/*
 	 * Clear the ROC_RUNNING /ROC_AUX_RUNNING status bit.
@@ -110,14 +106,10 @@ void iwl_mvm_roc_done_wk(struct work_struct *wk)
 	 * in the case that the time event actually completed in the firmware
 	 * (which is handled in iwl_mvm_te_handle_notif).
 	 */
-	if (test_and_clear_bit(IWL_MVM_STATUS_ROC_RUNNING, &mvm->status)) {
-		queues |= BIT(IWL_MVM_OFFCHANNEL_QUEUE);
+	if (test_and_clear_bit(IWL_MVM_STATUS_ROC_RUNNING, &mvm->status))
 		iwl_mvm_unref(mvm, IWL_MVM_REF_ROC);
-	}
-	if (test_and_clear_bit(IWL_MVM_STATUS_ROC_AUX_RUNNING, &mvm->status)) {
-		queues |= BIT(mvm->aux_queue);
+	if (test_and_clear_bit(IWL_MVM_STATUS_ROC_AUX_RUNNING, &mvm->status))
 		iwl_mvm_unref(mvm, IWL_MVM_REF_ROC_AUX);
-	}
 
 	synchronize_net();
 
@@ -203,9 +195,13 @@ static bool iwl_mvm_te_check_disconnect(struct iwl_mvm *mvm,
 					struct ieee80211_vif *vif,
 					const char *errmsg)
 {
+	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+
 	if (vif->type != NL80211_IFTYPE_STATION)
 		return false;
-	if (vif->bss_conf.assoc && vif->bss_conf.dtim_period)
+
+	if (!mvmvif->csa_bcn_pending && vif->bss_conf.assoc &&
+	    vif->bss_conf.dtim_period)
 		return false;
 	if (errmsg)
 		IWL_ERR(mvm, "%s\n", errmsg);
@@ -349,7 +345,7 @@ static void iwl_mvm_te_handle_notif(struct iwl_mvm *mvm,
 			 * and know the dtim period.
 			 */
 			iwl_mvm_te_check_disconnect(mvm, te_data->vif,
-				"No association and the time event is over already...");
+				"No beacon heard and the time event is over already...");
 			break;
 		default:
 			break;
@@ -621,7 +617,7 @@ void iwl_mvm_protect_session(struct iwl_mvm *mvm,
 	time_cmd.repeat = 1;
 	time_cmd.policy = cpu_to_le16(TE_V2_NOTIF_HOST_EVENT_START |
 				      TE_V2_NOTIF_HOST_EVENT_END |
-				      T2_V2_START_IMMEDIATELY);
+				      TE_V2_START_IMMEDIATELY);
 
 	if (!wait_for_notif) {
 		iwl_mvm_time_event_send_add(mvm, vif, te_data, &time_cmd);
@@ -777,12 +773,6 @@ int iwl_mvm_start_p2p_roc(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 		return -EBUSY;
 	}
 
-	/*
-	 * Flush the done work, just in case it's still pending, so that
-	 * the work it does can complete and we can accept new frames.
-	 */
-	flush_work(&mvm->roc_done_wk);
-
 	time_cmd.action = cpu_to_le32(FW_CTXT_ACTION_ADD);
 	time_cmd.id_and_color =
 		cpu_to_le32(FW_CMD_ID_AND_COLOR(mvmvif->id, mvmvif->color));
@@ -814,7 +804,7 @@ int iwl_mvm_start_p2p_roc(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 	time_cmd.repeat = 1;
 	time_cmd.policy = cpu_to_le16(TE_V2_NOTIF_HOST_EVENT_START |
 				      TE_V2_NOTIF_HOST_EVENT_END |
-				      T2_V2_START_IMMEDIATELY);
+				      TE_V2_START_IMMEDIATELY);
 
 	return iwl_mvm_time_event_send_add(mvm, vif, te_data, &time_cmd);
 }
@@ -924,6 +914,8 @@ int iwl_mvm_schedule_csa_period(struct iwl_mvm *mvm,
 	time_cmd.interval = cpu_to_le32(1);
 	time_cmd.policy = cpu_to_le16(TE_V2_NOTIF_HOST_EVENT_START |
 				      TE_V2_ABSENCE);
+	if (!apply_time)
+		time_cmd.policy |= cpu_to_le16(TE_V2_START_IMMEDIATELY);
 
 	return iwl_mvm_time_event_send_add(mvm, vif, te_data, &time_cmd);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
index 333bcb75b8af..7dfe4cde55e3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
@@ -419,11 +419,11 @@ static void iwl_mvm_set_tx_cmd_crypto(struct iwl_mvm *mvm,
 {
 	struct ieee80211_key_conf *keyconf = info->control.hw_key;
 	u8 *crypto_hdr = skb_frag->data + hdrlen;
+	enum iwl_tx_cmd_sec_ctrl type = TX_CMD_SEC_CCM;
 	u64 pn;
 
 	switch (keyconf->cipher) {
 	case WLAN_CIPHER_SUITE_CCMP:
-	case WLAN_CIPHER_SUITE_CCMP_256:
 		iwl_mvm_set_tx_cmd_ccmp(info, tx_cmd);
 		iwl_mvm_set_tx_cmd_pn(info, crypto_hdr);
 		break;
@@ -447,13 +447,16 @@ static void iwl_mvm_set_tx_cmd_crypto(struct iwl_mvm *mvm,
 		break;
 	case WLAN_CIPHER_SUITE_GCMP:
 	case WLAN_CIPHER_SUITE_GCMP_256:
+		type = TX_CMD_SEC_GCMP;
+		/* Fall through */
+	case WLAN_CIPHER_SUITE_CCMP_256:
 		/* TODO: Taking the key from the table might introduce a race
 		 * when PTK rekeying is done, having an old packets with a PN
 		 * based on the old key but the message encrypted with a new
 		 * one.
 		 * Need to handle this.
 		 */
-		tx_cmd->sec_ctl |= TX_CMD_SEC_GCMP | TX_CMD_SEC_KEY_FROM_TABLE;
+		tx_cmd->sec_ctl |= type | TX_CMD_SEC_KEY_FROM_TABLE;
 		tx_cmd->key[0] = keyconf->hw_key_idx;
 		iwl_mvm_set_tx_cmd_pn(info, crypto_hdr);
 		break;
@@ -645,7 +648,11 @@ int iwl_mvm_tx_skb_non_sta(struct iwl_mvm *mvm, struct sk_buff *skb)
 		if (info.control.vif->type == NL80211_IFTYPE_P2P_DEVICE ||
 		    info.control.vif->type == NL80211_IFTYPE_AP ||
 		    info.control.vif->type == NL80211_IFTYPE_ADHOC) {
-			sta_id = mvmvif->bcast_sta.sta_id;
+			if (info.control.vif->type == NL80211_IFTYPE_P2P_DEVICE)
+				sta_id = mvmvif->bcast_sta.sta_id;
+			else
+				sta_id = mvmvif->mcast_sta.sta_id;
+
 			queue = iwl_mvm_get_ctrl_vif_queue(mvm, &info,
 							   hdr->frame_control);
 			if (queue < 0)
@@ -888,10 +895,9 @@ static void iwl_mvm_tx_add_stream(struct iwl_mvm *mvm,
 	/*
 	 * The first deferred frame should've stopped the MAC queues, so we
 	 * should never get a second deferred frame for the RA/TID.
+	 * In case of GSO the first packet may have been split, so don't warn.
 	 */
-	if (!WARN(skb_queue_len(deferred_tx_frames) != 1,
-		  "RATID %d/%d has %d deferred frames\n", mvm_sta->sta_id, tid,
-		  skb_queue_len(deferred_tx_frames))) {
+	if (skb_queue_len(deferred_tx_frames) == 1) {
 		iwl_mvm_stop_mac_queues(mvm, BIT(mac_queue));
 		schedule_work(&mvm->add_stream_wk);
 	}
@@ -1132,7 +1138,7 @@ static void iwl_mvm_check_ratid_empty(struct iwl_mvm *mvm,
 	}
 
 	/*
-	 * In A000 HW, the next_reclaimed index is only 8 bit, so we'll need
+	 * In 22000 HW, the next_reclaimed index is only 8 bit, so we'll need
 	 * to align the wrap around of ssn so we compare relevant values.
 	 */
 	normalized_ssn = tid_data->ssn;
@@ -1624,7 +1630,7 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 	int freed;
 
 	if (WARN_ONCE(sta_id >= IWL_MVM_STATION_COUNT ||
-		      tid >= IWL_MAX_TID_COUNT,
+		      tid > IWL_MAX_TID_COUNT,
 		      "sta_id %d tid %d", sta_id, tid))
 		return;
 
@@ -1679,7 +1685,7 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 		if (ieee80211_is_data_qos(hdr->frame_control))
 			freed++;
 		else
-			WARN_ON_ONCE(1);
+			WARN_ON_ONCE(tid != IWL_MAX_TID_COUNT);
 
 		iwl_trans_free_tx_cmd(mvm->trans, info->driver_data[1]);
 
@@ -1719,8 +1725,11 @@ static void iwl_mvm_tx_reclaim(struct iwl_mvm *mvm, int sta_id, int tid,
 		ba_info->band = chanctx_conf->def.chan->band;
 		iwl_mvm_hwrate_to_tx_status(rate, ba_info);
 
-		IWL_DEBUG_TX_REPLY(mvm, "No reclaim. Update rs directly\n");
-		iwl_mvm_rs_tx_status(mvm, sta, tid, ba_info, false);
+		if (!iwl_mvm_has_tlc_offload(mvm)) {
+			IWL_DEBUG_TX_REPLY(mvm,
+					   "No reclaim. Update rs directly\n");
+			iwl_mvm_rs_tx_status(mvm, sta, tid, ba_info, false);
+		}
 	}
 
 out:
@@ -1771,8 +1780,12 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb)
 			struct iwl_mvm_compressed_ba_tfd *ba_tfd =
 				&ba_res->tfd[i];
 
+			tid = ba_tfd->tid;
+			if (tid == IWL_MGMT_TID)
+				tid = IWL_MAX_TID_COUNT;
+
 			mvmsta->tid_data[i].lq_color = lq_color;
-			iwl_mvm_tx_reclaim(mvm, sta_id, ba_tfd->tid,
+			iwl_mvm_tx_reclaim(mvm, sta_id, tid,
 					   (int)(le16_to_cpu(ba_tfd->q_num)),
 					   le16_to_cpu(ba_tfd->tfd_index),
 					   &ba_info,
@@ -1881,14 +1894,12 @@ int iwl_mvm_flush_sta(struct iwl_mvm *mvm, void *sta, bool internal, u32 flags)
 	struct iwl_mvm_int_sta *int_sta = sta;
 	struct iwl_mvm_sta *mvm_sta = sta;
 
-	if (iwl_mvm_has_new_tx_api(mvm)) {
-		if (internal)
-			return iwl_mvm_flush_sta_tids(mvm, int_sta->sta_id,
-						      BIT(IWL_MGMT_TID), flags);
+	BUILD_BUG_ON(offsetof(struct iwl_mvm_int_sta, sta_id) !=
+		     offsetof(struct iwl_mvm_sta, sta_id));
 
+	if (iwl_mvm_has_new_tx_api(mvm))
 		return iwl_mvm_flush_sta_tids(mvm, mvm_sta->sta_id,
-					      0xFF, flags);
-	}
+					      0xff | BIT(IWL_MGMT_TID), flags);
 
 	if (internal)
 		return iwl_mvm_flush_tx_path(mvm, int_sta->tfd_queue_msk,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index 03ffd84786ca..d65e1db7c097 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -278,8 +278,8 @@ u8 iwl_mvm_next_antenna(struct iwl_mvm *mvm, u8 valid, u8 last_idx)
 	u8 ind = last_idx;
 	int i;
 
-	for (i = 0; i < RATE_MCS_ANT_NUM; i++) {
-		ind = (ind + 1) % RATE_MCS_ANT_NUM;
+	for (i = 0; i < MAX_RS_ANT_NUM; i++) {
+		ind = (ind + 1) % MAX_RS_ANT_NUM;
 		if (valid & BIT(ind))
 			return ind;
 	}
@@ -516,8 +516,7 @@ static void iwl_mvm_dump_lmac_error_log(struct iwl_mvm *mvm, u32 base)
 		IWL_ERR(trans, "HW error, resetting before reading\n");
 
 		/* reset the device */
-		iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
-		usleep_range(5000, 6000);
+		iwl_trans_sw_reset(trans);
 
 		/* set INIT_DONE flag */
 		iwl_set_bit(trans, CSR_GP_CNTRL,
@@ -595,6 +594,12 @@ static void iwl_mvm_dump_lmac_error_log(struct iwl_mvm *mvm, u32 base)
 
 void iwl_mvm_dump_nic_error_log(struct iwl_mvm *mvm)
 {
+	if (!test_bit(STATUS_DEVICE_ENABLED, &mvm->trans->status)) {
+		IWL_ERR(mvm,
+			"DEVICE_ENABLED bit is not set. Aborting dump.\n");
+		return;
+	}
+
 	iwl_mvm_dump_lmac_error_log(mvm, mvm->error_event_table[0]);
 
 	if (mvm->error_event_table[1])
@@ -906,7 +911,8 @@ int iwl_mvm_send_lq_cmd(struct iwl_mvm *mvm, struct iwl_lq_cmd *lq, bool init)
 		.data = { lq, },
 	};
 
-	if (WARN_ON(lq->sta_id == IWL_MVM_INVALID_STA))
+	if (WARN_ON(lq->sta_id == IWL_MVM_INVALID_STA ||
+		    iwl_mvm_has_tlc_offload(mvm)))
 		return -EINVAL;
 
 	return iwl_mvm_send_cmd(mvm, &cmd);
@@ -1025,12 +1031,34 @@ int iwl_mvm_update_low_latency(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	int res;
+	bool low_latency;
 
 	lockdep_assert_held(&mvm->mutex);
 
-	if (iwl_mvm_vif_low_latency(mvmvif) == prev)
+	low_latency = iwl_mvm_vif_low_latency(mvmvif);
+
+	if (low_latency == prev)
 		return 0;
 
+	if (fw_has_capa(&mvm->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_DYNAMIC_QUOTA)) {
+		struct iwl_mac_low_latency_cmd cmd = {
+			.mac_id = cpu_to_le32(mvmvif->id)
+		};
+
+		if (low_latency) {
+			/* currently we don't care about the direction */
+			cmd.low_latency_rx = 1;
+			cmd.low_latency_tx = 1;
+		}
+		res = iwl_mvm_send_cmd_pdu(mvm,
+					   iwl_cmd_id(LOW_LATENCY_CMD,
+						      MAC_CONF_GROUP, 0),
+					   0, sizeof(cmd), &cmd);
+		if (res)
+			IWL_ERR(mvm, "Failed to send low latency command\n");
+	}
+
 	res = iwl_mvm_update_quotas(mvm, false, NULL);
 	if (res)
 		return res;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index ccd7c33c4c28..e323d3abb6ac 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -579,25 +579,25 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0x30DC, 0x0264, iwl9461_2ac_cfg_soc)},
 	{IWL_PCI_DEVICE(0x30DC, 0x02A0, iwl9462_2ac_cfg_soc)},
 	{IWL_PCI_DEVICE(0x30DC, 0x02A4, iwl9462_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0030, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0034, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0038, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x003C, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0060, iwl9460_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0064, iwl9461_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x00A0, iwl9462_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x00A4, iwl9462_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0230, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0234, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0238, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x023C, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0260, iwl9461_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x0264, iwl9461_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x02A0, iwl9462_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x02A4, iwl9462_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x4030, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x4034, iwl9560_2ac_cfg_soc)},
-	{IWL_PCI_DEVICE(0x31DC, 0x40A4, iwl9462_2ac_cfg_soc)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0030, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0034, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0038, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x003C, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0060, iwl9460_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0064, iwl9461_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x00A0, iwl9462_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x00A4, iwl9462_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0230, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0234, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0238, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x023C, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0260, iwl9461_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x0264, iwl9461_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x02A0, iwl9462_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x02A4, iwl9462_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x4030, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x4034, iwl9560_2ac_cfg_shared_clk)},
+	{IWL_PCI_DEVICE(0x31DC, 0x40A4, iwl9462_2ac_cfg_shared_clk)},
 	{IWL_PCI_DEVICE(0x34F0, 0x0030, iwl9560_2ac_cfg_soc)},
 	{IWL_PCI_DEVICE(0x34F0, 0x0034, iwl9560_2ac_cfg_soc)},
 	{IWL_PCI_DEVICE(0x34F0, 0x02A4, iwl9462_2ac_cfg_soc)},
@@ -652,20 +652,20 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0xA370, 0x4034, iwl9560_2ac_cfg_soc)},
 	{IWL_PCI_DEVICE(0xA370, 0x40A4, iwl9462_2ac_cfg_soc)},
 
-/* a000 Series */
-	{IWL_PCI_DEVICE(0x2720, 0x0A10, iwla000_2ac_cfg_hr_cdb)},
-	{IWL_PCI_DEVICE(0x34F0, 0x0310, iwla000_2ac_cfg_jf)},
-	{IWL_PCI_DEVICE(0x2720, 0x0000, iwla000_2ax_cfg_hr)},
-	{IWL_PCI_DEVICE(0x34F0, 0x0070, iwla000_2ax_cfg_hr)},
-	{IWL_PCI_DEVICE(0x2720, 0x0078, iwla000_2ax_cfg_hr)},
-	{IWL_PCI_DEVICE(0x2720, 0x0070, iwla000_2ac_cfg_hr_cdb)},
-	{IWL_PCI_DEVICE(0x2720, 0x0030, iwla000_2ac_cfg_hr_cdb)},
-	{IWL_PCI_DEVICE(0x2720, 0x1080, iwla000_2ax_cfg_hr)},
-	{IWL_PCI_DEVICE(0x2720, 0x0090, iwla000_2ac_cfg_hr_cdb)},
-	{IWL_PCI_DEVICE(0x2720, 0x0310, iwla000_2ac_cfg_hr_cdb)},
-	{IWL_PCI_DEVICE(0x40C0, 0x0000, iwla000_2ax_cfg_hr)},
-	{IWL_PCI_DEVICE(0x40C0, 0x0A10, iwla000_2ax_cfg_hr)},
-	{IWL_PCI_DEVICE(0xA0F0, 0x0000, iwla000_2ax_cfg_hr)},
+/* 22000 Series */
+	{IWL_PCI_DEVICE(0x2720, 0x0A10, iwl22000_2ac_cfg_hr_cdb)},
+	{IWL_PCI_DEVICE(0x34F0, 0x0310, iwl22000_2ac_cfg_jf)},
+	{IWL_PCI_DEVICE(0x2720, 0x0000, iwl22000_2ax_cfg_hr)},
+	{IWL_PCI_DEVICE(0x34F0, 0x0070, iwl22000_2ax_cfg_hr)},
+	{IWL_PCI_DEVICE(0x2720, 0x0078, iwl22000_2ax_cfg_hr)},
+	{IWL_PCI_DEVICE(0x2720, 0x0070, iwl22000_2ac_cfg_hr_cdb)},
+	{IWL_PCI_DEVICE(0x2720, 0x0030, iwl22000_2ac_cfg_hr_cdb)},
+	{IWL_PCI_DEVICE(0x2720, 0x1080, iwl22000_2ax_cfg_hr)},
+	{IWL_PCI_DEVICE(0x2720, 0x0090, iwl22000_2ac_cfg_hr_cdb)},
+	{IWL_PCI_DEVICE(0x2720, 0x0310, iwl22000_2ac_cfg_hr_cdb)},
+	{IWL_PCI_DEVICE(0x40C0, 0x0000, iwl22000_2ax_cfg_hr)},
+	{IWL_PCI_DEVICE(0x40C0, 0x0A10, iwl22000_2ax_cfg_hr)},
+	{IWL_PCI_DEVICE(0xA0F0, 0x0000, iwl22000_2ax_cfg_hr)},
 
 #endif /* CONFIG_IWLMVM */
 
@@ -707,7 +707,7 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		iwl_trans->cfg = cfg_7265d;
 	}
 
-	if (iwl_trans->cfg->rf_id && cfg == &iwla000_2ac_cfg_hr_cdb &&
+	if (iwl_trans->cfg->rf_id && cfg == &iwl22000_2ac_cfg_hr_cdb &&
 	    iwl_trans->hw_rev != CSR_HW_REV_TYPE_HR_CDB) {
 		u32 rf_id_chp = CSR_HW_RF_ID_TYPE_CHIP_ID(iwl_trans->hw_rf_id);
 		u32 jf_chp_id = CSR_HW_RF_ID_TYPE_CHIP_ID(CSR_HW_RF_ID_TYPE_JF);
@@ -715,14 +715,14 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 		if (rf_id_chp == jf_chp_id) {
 			if (iwl_trans->hw_rev == CSR_HW_REV_TYPE_QNJ)
-				cfg = &iwla000_2ax_cfg_qnj_jf_b0;
+				cfg = &iwl22000_2ax_cfg_qnj_jf_b0;
 			else
-				cfg = &iwla000_2ac_cfg_jf;
+				cfg = &iwl22000_2ac_cfg_jf;
 		} else if (rf_id_chp == hr_chp_id) {
 			if (iwl_trans->hw_rev == CSR_HW_REV_TYPE_QNJ)
-				cfg = &iwla000_2ax_cfg_qnj_hr_a0;
+				cfg = &iwl22000_2ax_cfg_qnj_hr_a0;
 			else
-				cfg = &iwla000_2ac_cfg_hr;
+				cfg = &iwl22000_2ac_cfg_hr;
 		}
 		iwl_trans->cfg = cfg;
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
index 403e65c309d0..ca3b64ff4dad 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
@@ -658,13 +658,6 @@ static inline void iwl_enable_fw_load_int(struct iwl_trans *trans)
 	}
 }
 
-static inline void iwl_pcie_sw_reset(struct iwl_trans *trans)
-{
-	/* Reset entire device - do controller reset (results in SHRD_HW_RST) */
-	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
-	usleep_range(5000, 6000);
-}
-
 static inline u8 iwl_pcie_get_cmd_index(struct iwl_txq *q, u32 index)
 {
 	return index & (q->n_window - 1);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
index ac05fd1e74c4..cb4012541f45 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans-gen2.c
@@ -137,7 +137,7 @@ static void iwl_pcie_gen2_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 	/* Stop device's DMA activity */
 	iwl_pcie_apm_stop_master(trans);
 
-	iwl_pcie_sw_reset(trans);
+	iwl_trans_sw_reset(trans);
 
 	/*
 	 * Clear "initialization complete" bit to move adapter from
@@ -192,7 +192,7 @@ void _iwl_trans_pcie_gen2_stop_device(struct iwl_trans *trans, bool low_power)
 	/* Stop the device, and put it in low power state */
 	iwl_pcie_gen2_apm_stop(trans, false);
 
-	iwl_pcie_sw_reset(trans);
+	iwl_trans_sw_reset(trans);
 
 	/*
 	 * Upon stop, the IVAR table gets erased, so msi-x won't
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
index 4541c86881d6..b406b536c850 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
@@ -176,6 +176,13 @@ static void iwl_trans_pcie_dump_regs(struct iwl_trans *trans)
 	kfree(buf);
 }
 
+static void iwl_trans_pcie_sw_reset(struct iwl_trans *trans)
+{
+	/* Reset entire device - do controller reset (results in SHRD_HW_RST) */
+	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
+	usleep_range(5000, 6000);
+}
+
 static void iwl_pcie_free_fw_monitor(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
@@ -446,7 +453,7 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
 				 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
 
-	iwl_pcie_sw_reset(trans);
+	iwl_trans_pcie_sw_reset(trans);
 
 	/*
 	 * Set "initialization complete" bit to move adapter from
@@ -487,7 +494,7 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 				 apmg_xtal_cfg_reg |
 				 SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
 
-	iwl_pcie_sw_reset(trans);
+	iwl_trans_pcie_sw_reset(trans);
 
 	/* Enable LP XTAL by indirect access through CSR */
 	apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG);
@@ -580,7 +587,7 @@ static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 		return;
 	}
 
-	iwl_pcie_sw_reset(trans);
+	iwl_trans_pcie_sw_reset(trans);
 
 	/*
 	 * Clear "initialization complete" bit to move adapter from
@@ -915,14 +922,9 @@ static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans,
 void iwl_pcie_apply_destination(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-	const struct iwl_fw_dbg_dest_tlv *dest = trans->dbg_dest_tlv;
+	const struct iwl_fw_dbg_dest_tlv_v1 *dest = trans->dbg_dest_tlv;
 	int i;
 
-	if (dest->version)
-		IWL_ERR(trans,
-			"DBG DEST version is %d - expect issues\n",
-			dest->version);
-
 	IWL_INFO(trans, "Applying debug destination %s\n",
 		 get_fw_dbg_mode_string(dest->monitor_mode));
 
@@ -1270,7 +1272,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power)
 	/* Stop the device, and put it in low power state */
 	iwl_pcie_apm_stop(trans, false);
 
-	iwl_pcie_sw_reset(trans);
+	iwl_trans_pcie_sw_reset(trans);
 
 	/*
 	 * Upon stop, the IVAR table gets erased, so msi-x won't
@@ -1744,7 +1746,7 @@ static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
 		return err;
 	}
 
-	iwl_pcie_sw_reset(trans);
+	iwl_trans_pcie_sw_reset(trans);
 
 	err = iwl_pcie_apm_init(trans);
 	if (err)
@@ -2816,8 +2818,17 @@ iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
 			 * Update pointers to reflect actual values after
 			 * shifting
 			 */
-			base = iwl_read_prph(trans, base) <<
-			       trans->dbg_dest_tlv->base_shift;
+			if (trans->dbg_dest_tlv->version) {
+				base = (iwl_read_prph(trans, base) &
+					IWL_LDBG_M2S_BUF_BA_MSK) <<
+				       trans->dbg_dest_tlv->base_shift;
+				base *= IWL_M2S_UNIT_SIZE;
+				base += trans->cfg->smem_offset;
+			} else {
+				base = iwl_read_prph(trans, base) <<
+				       trans->dbg_dest_tlv->base_shift;
+			}
+
 			iwl_trans_read_mem(trans, base, fw_mon_data->data,
 					   monitor_len / sizeof(u32));
 		} else if (trans->dbg_dest_tlv->monitor_mode == MARBH_MODE) {
@@ -2865,21 +2876,36 @@ static struct iwl_trans_dump_data
 		       trans_pcie->fw_mon_size;
 		monitor_len = trans_pcie->fw_mon_size;
 	} else if (trans->dbg_dest_tlv) {
-		u32 base, end;
+		u32 base, end, cfg_reg;
 
-		base = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
-		end = le32_to_cpu(trans->dbg_dest_tlv->end_reg);
+		if (trans->dbg_dest_tlv->version == 1) {
+			cfg_reg = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
+			cfg_reg = iwl_read_prph(trans, cfg_reg);
+			base = (cfg_reg & IWL_LDBG_M2S_BUF_BA_MSK) <<
+				trans->dbg_dest_tlv->base_shift;
+			base *= IWL_M2S_UNIT_SIZE;
+			base += trans->cfg->smem_offset;
 
-		base = iwl_read_prph(trans, base) <<
-		       trans->dbg_dest_tlv->base_shift;
-		end = iwl_read_prph(trans, end) <<
-		      trans->dbg_dest_tlv->end_shift;
+			monitor_len =
+				(cfg_reg & IWL_LDBG_M2S_BUF_SIZE_MSK) >>
+				trans->dbg_dest_tlv->end_shift;
+			monitor_len *= IWL_M2S_UNIT_SIZE;
+		} else {
+			base = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
+			end = le32_to_cpu(trans->dbg_dest_tlv->end_reg);
 
-		/* Make "end" point to the actual end */
-		if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000 ||
-		    trans->dbg_dest_tlv->monitor_mode == MARBH_MODE)
-			end += (1 << trans->dbg_dest_tlv->end_shift);
-		monitor_len = end - base;
+			base = iwl_read_prph(trans, base) <<
+			       trans->dbg_dest_tlv->base_shift;
+			end = iwl_read_prph(trans, end) <<
+			      trans->dbg_dest_tlv->end_shift;
+
+			/* Make "end" point to the actual end */
+			if (trans->cfg->device_family >=
+			    IWL_DEVICE_FAMILY_8000 ||
+			    trans->dbg_dest_tlv->monitor_mode == MARBH_MODE)
+				end += (1 << trans->dbg_dest_tlv->end_shift);
+			monitor_len = end - base;
+		}
 		len += sizeof(*data) + sizeof(struct iwl_fw_error_dump_fw_mon) +
 		       monitor_len;
 	} else {
@@ -3025,6 +3051,7 @@ static void iwl_trans_pcie_resume(struct iwl_trans *trans)
 	.write_mem = iwl_trans_pcie_write_mem,				\
 	.configure = iwl_trans_pcie_configure,				\
 	.set_pmi = iwl_trans_pcie_set_pmi,				\
+	.sw_reset = iwl_trans_pcie_sw_reset,				\
 	.grab_nic_access = iwl_trans_pcie_grab_nic_access,		\
 	.release_nic_access = iwl_trans_pcie_release_nic_access,	\
 	.set_bits_mask = iwl_trans_pcie_set_bits_mask,			\
@@ -3250,9 +3277,9 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
 
 		hw_status = iwl_read_prph(trans, UMAG_GEN_HW_STATUS);
 		if (hw_status & UMAG_GEN_HW_IS_FPGA)
-			trans->cfg = &iwla000_2ax_cfg_qnj_hr_f0;
+			trans->cfg = &iwl22000_2ax_cfg_qnj_hr_f0;
 		else
-			trans->cfg = &iwla000_2ac_cfg_hr;
+			trans->cfg = &iwl22000_2ac_cfg_hr;
 	}
 #endif
 
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
index 6d0a907d5ba5..fabae0f60683 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx-gen2.c
@@ -147,7 +147,7 @@ static void iwl_pcie_gen2_tfd_unmap(struct iwl_trans *trans,
 	/* Sanity check on number of chunks */
 	num_tbs = iwl_pcie_gen2_get_num_tbs(trans, tfd);
 
-	if (num_tbs >= trans_pcie->max_tbs) {
+	if (num_tbs > trans_pcie->max_tbs) {
 		IWL_ERR(trans, "Too many chunks: %i\n", num_tbs);
 		return;
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
index 3f85713c41dc..1a566287993d 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
@@ -378,7 +378,7 @@ static void iwl_pcie_tfd_unmap(struct iwl_trans *trans,
 	/* Sanity check on number of chunks */
 	num_tbs = iwl_pcie_tfd_get_num_tbs(trans, tfd);
 
-	if (num_tbs >= trans_pcie->max_tbs) {
+	if (num_tbs > trans_pcie->max_tbs) {
 		IWL_ERR(trans, "Too many chunks: %i\n", num_tbs);
 		/* @todo issue fatal error, it is quite serious situation */
 		return;
diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c
index f6d4a50f1bdb..35b21f8152bb 100644
--- a/drivers/net/wireless/mac80211_hwsim.c
+++ b/drivers/net/wireless/mac80211_hwsim.c
@@ -32,6 +32,7 @@
 #include <net/genetlink.h>
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
+#include <linux/rhashtable.h>
 #include "mac80211_hwsim.h"
 
 #define WARN_QUEUE 100
@@ -490,6 +491,7 @@ static const struct ieee80211_iface_combination hwsim_if_comb_p2p_dev[] = {
 static spinlock_t hwsim_radio_lock;
 static LIST_HEAD(hwsim_radios);
 static struct workqueue_struct *hwsim_wq;
+static struct rhashtable hwsim_radios_rht;
 static int hwsim_radio_idx;
 
 static struct platform_driver mac80211_hwsim_driver = {
@@ -500,6 +502,7 @@ static struct platform_driver mac80211_hwsim_driver = {
 
 struct mac80211_hwsim_data {
 	struct list_head list;
+	struct rhash_head rht;
 	struct ieee80211_hw *hw;
 	struct device *dev;
 	struct ieee80211_supported_band bands[NUM_NL80211_BANDS];
@@ -574,6 +577,13 @@ struct mac80211_hwsim_data {
 	u64 tx_failed;
 };
 
+static const struct rhashtable_params hwsim_rht_params = {
+	.nelem_hint = 2,
+	.automatic_shrinking = true,
+	.key_len = ETH_ALEN,
+	.key_offset = offsetof(struct mac80211_hwsim_data, addresses[1]),
+	.head_offset = offsetof(struct mac80211_hwsim_data, rht),
+};
 
 struct hwsim_radiotap_hdr {
 	struct ieee80211_radiotap_header hdr;
@@ -729,16 +739,21 @@ static int hwsim_fops_ps_write(void *dat, u64 val)
 	    val != PS_MANUAL_POLL)
 		return -EINVAL;
 
-	old_ps = data->ps;
-	data->ps = val;
-
-	local_bh_disable();
 	if (val == PS_MANUAL_POLL) {
+		if (data->ps != PS_ENABLED)
+			return -EINVAL;
+		local_bh_disable();
 		ieee80211_iterate_active_interfaces_atomic(
 			data->hw, IEEE80211_IFACE_ITER_NORMAL,
 			hwsim_send_ps_poll, data);
-		data->ps_poll_pending = true;
-	} else if (old_ps == PS_DISABLED && val != PS_DISABLED) {
+		local_bh_enable();
+		return 0;
+	}
+	old_ps = data->ps;
+	data->ps = val;
+
+	local_bh_disable();
+	if (old_ps == PS_DISABLED && val != PS_DISABLED) {
 		ieee80211_iterate_active_interfaces_atomic(
 			data->hw, IEEE80211_IFACE_ITER_NORMAL,
 			hwsim_send_nullfunc_ps, data);
@@ -1004,6 +1019,36 @@ static int hwsim_unicast_netgroup(struct mac80211_hwsim_data *data,
 	return res;
 }
 
+static inline u16 trans_tx_rate_flags_ieee2hwsim(struct ieee80211_tx_rate *rate)
+{
+	u16 result = 0;
+
+	if (rate->flags & IEEE80211_TX_RC_USE_RTS_CTS)
+		result |= MAC80211_HWSIM_TX_RC_USE_RTS_CTS;
+	if (rate->flags & IEEE80211_TX_RC_USE_CTS_PROTECT)
+		result |= MAC80211_HWSIM_TX_RC_USE_CTS_PROTECT;
+	if (rate->flags & IEEE80211_TX_RC_USE_SHORT_PREAMBLE)
+		result |= MAC80211_HWSIM_TX_RC_USE_SHORT_PREAMBLE;
+	if (rate->flags & IEEE80211_TX_RC_MCS)
+		result |= MAC80211_HWSIM_TX_RC_MCS;
+	if (rate->flags & IEEE80211_TX_RC_GREEN_FIELD)
+		result |= MAC80211_HWSIM_TX_RC_GREEN_FIELD;
+	if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+		result |= MAC80211_HWSIM_TX_RC_40_MHZ_WIDTH;
+	if (rate->flags & IEEE80211_TX_RC_DUP_DATA)
+		result |= MAC80211_HWSIM_TX_RC_DUP_DATA;
+	if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
+		result |= MAC80211_HWSIM_TX_RC_SHORT_GI;
+	if (rate->flags & IEEE80211_TX_RC_VHT_MCS)
+		result |= MAC80211_HWSIM_TX_RC_VHT_MCS;
+	if (rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
+		result |= MAC80211_HWSIM_TX_RC_80_MHZ_WIDTH;
+	if (rate->flags & IEEE80211_TX_RC_160_MHZ_WIDTH)
+		result |= MAC80211_HWSIM_TX_RC_160_MHZ_WIDTH;
+
+	return result;
+}
+
 static void mac80211_hwsim_tx_frame_nl(struct ieee80211_hw *hw,
 				       struct sk_buff *my_skb,
 				       int dst_portid)
@@ -1016,6 +1061,7 @@ static void mac80211_hwsim_tx_frame_nl(struct ieee80211_hw *hw,
 	unsigned int hwsim_flags = 0;
 	int i;
 	struct hwsim_tx_rate tx_attempts[IEEE80211_TX_MAX_RATES];
+	struct hwsim_tx_rate_flag tx_attempts_flags[IEEE80211_TX_MAX_RATES];
 	uintptr_t cookie;
 
 	if (data->ps != PS_DISABLED)
@@ -1067,7 +1113,11 @@ static void mac80211_hwsim_tx_frame_nl(struct ieee80211_hw *hw,
 
 	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
 		tx_attempts[i].idx = info->status.rates[i].idx;
+		tx_attempts_flags[i].idx = info->status.rates[i].idx;
 		tx_attempts[i].count = info->status.rates[i].count;
+		tx_attempts_flags[i].flags =
+				trans_tx_rate_flags_ieee2hwsim(
+						&info->status.rates[i]);
 	}
 
 	if (nla_put(skb, HWSIM_ATTR_TX_INFO,
@@ -1075,6 +1125,11 @@ static void mac80211_hwsim_tx_frame_nl(struct ieee80211_hw *hw,
 		    tx_attempts))
 		goto nla_put_failure;
 
+	if (nla_put(skb, HWSIM_ATTR_TX_INFO_FLAGS,
+		    sizeof(struct hwsim_tx_rate_flag) * IEEE80211_TX_MAX_RATES,
+		    tx_attempts_flags))
+		goto nla_put_failure;
+
 	/* We create a cookie to identify this skb */
 	data->pending_cookie++;
 	cookie = data->pending_cookie;
@@ -2672,6 +2727,7 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 	mutex_init(&data->mutex);
 
 	data->netgroup = hwsim_net_get_netgroup(net);
+	data->wmediumd = hwsim_net_get_wmediumd(net);
 
 	/* Enable frame retransmissions for lossy channels */
 	hw->max_rates = 4;
@@ -2727,6 +2783,15 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 			     CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 
 	spin_lock_bh(&hwsim_radio_lock);
+	err = rhashtable_insert_fast(&hwsim_radios_rht, &data->rht,
+				     hwsim_rht_params);
+	if (err < 0) {
+		pr_debug("mac80211_hwsim: radio index %d already present\n",
+			 idx);
+		spin_unlock_bh(&hwsim_radio_lock);
+		goto failed_final_insert;
+	}
+
 	list_add_tail(&data->list, &hwsim_radios);
 	spin_unlock_bh(&hwsim_radio_lock);
 
@@ -2735,6 +2800,9 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 
 	return idx;
 
+failed_final_insert:
+	debugfs_remove_recursive(data->debugfs);
+	ieee80211_unregister_hw(data->hw);
 failed_hw:
 	device_release_driver(data->dev);
 failed_bind:
@@ -2870,22 +2938,9 @@ static void hwsim_mon_setup(struct net_device *dev)
 
 static struct mac80211_hwsim_data *get_hwsim_data_ref_from_addr(const u8 *addr)
 {
-	struct mac80211_hwsim_data *data;
-	bool _found = false;
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry(data, &hwsim_radios, list) {
-		if (memcmp(data->addresses[1].addr, addr, ETH_ALEN) == 0) {
-			_found = true;
-			break;
-		}
-	}
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	if (!_found)
-		return NULL;
-
-	return data;
+	return rhashtable_lookup_fast(&hwsim_radios_rht,
+				      addr,
+				      hwsim_rht_params);
 }
 
 static void hwsim_register_wmediumd(struct net *net, u32 portid)
@@ -2970,7 +3025,6 @@ static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
 	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
 		txi->status.rates[i].idx = tx_attempts[i].idx;
 		txi->status.rates[i].count = tx_attempts[i].count;
-		/*txi->status.rates[i].flags = 0;*/
 	}
 
 	txi->status.ack_signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]);
@@ -3150,8 +3204,10 @@ static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
 	if (info->attrs[HWSIM_ATTR_REG_CUSTOM_REG]) {
 		u32 idx = nla_get_u32(info->attrs[HWSIM_ATTR_REG_CUSTOM_REG]);
 
-		if (idx >= ARRAY_SIZE(hwsim_world_regdom_custom))
+		if (idx >= ARRAY_SIZE(hwsim_world_regdom_custom)) {
+			kfree(hwname);
 			return -EINVAL;
+		}
 		param.regd = hwsim_world_regdom_custom[idx];
 	}
 
@@ -3192,6 +3248,8 @@ static int hwsim_del_radio_nl(struct sk_buff *msg, struct genl_info *info)
 			continue;
 
 		list_del(&data->list);
+		rhashtable_remove_fast(&hwsim_radios_rht, &data->rht,
+				       hwsim_rht_params);
 		spin_unlock_bh(&hwsim_radio_lock);
 		mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
 					 info);
@@ -3347,6 +3405,8 @@ static void remove_user_radios(u32 portid)
 	list_for_each_entry_safe(entry, tmp, &hwsim_radios, list) {
 		if (entry->destroy_on_close && entry->portid == portid) {
 			list_del(&entry->list);
+			rhashtable_remove_fast(&hwsim_radios_rht, &entry->rht,
+					       hwsim_rht_params);
 			INIT_WORK(&entry->destroy_work, destroy_radio);
 			queue_work(hwsim_wq, &entry->destroy_work);
 		}
@@ -3422,6 +3482,8 @@ static void __net_exit hwsim_exit_net(struct net *net)
 			continue;
 
 		list_del(&data->list);
+		rhashtable_remove_fast(&hwsim_radios_rht, &data->rht,
+				       hwsim_rht_params);
 		INIT_WORK(&data->destroy_work, destroy_radio);
 		queue_work(hwsim_wq, &data->destroy_work);
 	}
@@ -3455,9 +3517,10 @@ static int __init init_mac80211_hwsim(void)
 
 	spin_lock_init(&hwsim_radio_lock);
 
-	hwsim_wq = alloc_workqueue("hwsim_wq",WQ_MEM_RECLAIM,0);
+	hwsim_wq = alloc_workqueue("hwsim_wq", 0, 0);
 	if (!hwsim_wq)
 		return -ENOMEM;
+	rhashtable_init(&hwsim_radios_rht, &hwsim_rht_params);
 
 	err = register_pernet_device(&hwsim_net_ops);
 	if (err)
@@ -3599,6 +3662,7 @@ static void __exit exit_mac80211_hwsim(void)
 	mac80211_hwsim_free();
 	flush_workqueue(hwsim_wq);
 
+	rhashtable_destroy(&hwsim_radios_rht);
 	unregister_netdev(hwsim_mon);
 	platform_driver_unregister(&mac80211_hwsim_driver);
 	unregister_pernet_device(&hwsim_net_ops);
diff --git a/drivers/net/wireless/mac80211_hwsim.h b/drivers/net/wireless/mac80211_hwsim.h
index 3f5eda591dba..a96a79c1eff5 100644
--- a/drivers/net/wireless/mac80211_hwsim.h
+++ b/drivers/net/wireless/mac80211_hwsim.h
@@ -64,7 +64,8 @@ enum hwsim_tx_control_flags {
  * @HWSIM_CMD_TX_INFO_FRAME: Transmission info report from user space to
  *	kernel, uses:
  *	%HWSIM_ATTR_ADDR_TRANSMITTER, %HWSIM_ATTR_FLAGS,
- *	%HWSIM_ATTR_TX_INFO, %HWSIM_ATTR_SIGNAL, %HWSIM_ATTR_COOKIE
+ *	%HWSIM_ATTR_TX_INFO, %WSIM_ATTR_TX_INFO_FLAGS,
+ *	%HWSIM_ATTR_SIGNAL, %HWSIM_ATTR_COOKIE
  * @HWSIM_CMD_NEW_RADIO: create a new radio with the given parameters,
  *	returns the radio ID (>= 0) or negative on errors, if successful
  *	then multicast the result
@@ -123,6 +124,8 @@ enum {
  * @HWSIM_ATTR_RADIO_NAME: Name of radio, e.g. phy666
  * @HWSIM_ATTR_NO_VIF:  Do not create vif (wlanX) when creating radio.
  * @HWSIM_ATTR_FREQ: Frequency at which packet is transmitted or received.
+ * @HWSIM_ATTR_TX_INFO_FLAGS: additional flags for corresponding
+ *	rates of %HWSIM_ATTR_TX_INFO
  * @__HWSIM_ATTR_MAX: enum limit
  */
 
@@ -149,6 +152,7 @@ enum {
 	HWSIM_ATTR_NO_VIF,
 	HWSIM_ATTR_FREQ,
 	HWSIM_ATTR_PAD,
+	HWSIM_ATTR_TX_INFO_FLAGS,
 	__HWSIM_ATTR_MAX,
 };
 #define HWSIM_ATTR_MAX (__HWSIM_ATTR_MAX - 1)
@@ -171,4 +175,66 @@ struct hwsim_tx_rate {
 	u8 count;
 } __packed;
 
+/**
+ * enum hwsim_tx_rate_flags - per-rate flags set by the rate control algorithm.
+ *	Inspired by structure mac80211_rate_control_flags. New flags may be
+ *	appended, but old flags not deleted, to keep compatibility for
+ *	userspace.
+ *
+ * These flags are set by the Rate control algorithm for each rate during tx,
+ * in the @flags member of struct ieee80211_tx_rate.
+ *
+ * @MAC80211_HWSIM_TX_RC_USE_RTS_CTS: Use RTS/CTS exchange for this rate.
+ * @MAC80211_HWSIM_TX_RC_USE_CTS_PROTECT: CTS-to-self protection is required.
+ *	This is set if the current BSS requires ERP protection.
+ * @MAC80211_HWSIM_TX_RC_USE_SHORT_PREAMBLE: Use short preamble.
+ * @MAC80211_HWSIM_TX_RC_MCS: HT rate.
+ * @MAC80211_HWSIM_TX_RC_VHT_MCS: VHT MCS rate, in this case the idx field is
+ *	split into a higher 4 bits (Nss) and lower 4 bits (MCS number)
+ * @MAC80211_HWSIM_TX_RC_GREEN_FIELD: Indicates whether this rate should be used
+ *	in Greenfield mode.
+ * @MAC80211_HWSIM_TX_RC_40_MHZ_WIDTH: Indicates if the Channel Width should be
+ *	40 MHz.
+ * @MAC80211_HWSIM_TX_RC_80_MHZ_WIDTH: Indicates 80 MHz transmission
+ * @MAC80211_HWSIM_TX_RC_160_MHZ_WIDTH: Indicates 160 MHz transmission
+ *	(80+80 isn't supported yet)
+ * @MAC80211_HWSIM_TX_RC_DUP_DATA: The frame should be transmitted on both of
+ *	the adjacent 20 MHz channels, if the current channel type is
+ *	NL80211_CHAN_HT40MINUS or NL80211_CHAN_HT40PLUS.
+ * @MAC80211_HWSIM_TX_RC_SHORT_GI: Short Guard interval should be used for this
+ *	rate.
+ */
+enum hwsim_tx_rate_flags {
+	MAC80211_HWSIM_TX_RC_USE_RTS_CTS		= BIT(0),
+	MAC80211_HWSIM_TX_RC_USE_CTS_PROTECT		= BIT(1),
+	MAC80211_HWSIM_TX_RC_USE_SHORT_PREAMBLE	= BIT(2),
+
+	/* rate index is an HT/VHT MCS instead of an index */
+	MAC80211_HWSIM_TX_RC_MCS			= BIT(3),
+	MAC80211_HWSIM_TX_RC_GREEN_FIELD		= BIT(4),
+	MAC80211_HWSIM_TX_RC_40_MHZ_WIDTH		= BIT(5),
+	MAC80211_HWSIM_TX_RC_DUP_DATA		= BIT(6),
+	MAC80211_HWSIM_TX_RC_SHORT_GI		= BIT(7),
+	MAC80211_HWSIM_TX_RC_VHT_MCS			= BIT(8),
+	MAC80211_HWSIM_TX_RC_80_MHZ_WIDTH		= BIT(9),
+	MAC80211_HWSIM_TX_RC_160_MHZ_WIDTH		= BIT(10),
+};
+
+/**
+ * struct hwsim_tx_rate - rate selection/status
+ *
+ * @idx: rate index to attempt to send with
+ * @count: number of tries in this rate before going to the next rate
+ *
+ * A value of -1 for @idx indicates an invalid rate and, if used
+ * in an array of retry rates, that no more rates should be tried.
+ *
+ * When used for transmit status reporting, the driver should
+ * always report the rate and number of retries used.
+ *
+ */
+struct hwsim_tx_rate_flag {
+	s8 idx;
+	u16 flags;
+} __packed;
 #endif /* __MAC80211_HWSIM_H */
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index 6e0d9a9c5cfb..ce4432c535f0 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -1116,6 +1116,12 @@ mwifiex_cfg80211_change_virtual_intf(struct wiphy *wiphy,
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(dev);
 	enum nl80211_iftype curr_iftype = dev->ieee80211_ptr->iftype;
 
+	if (priv->scan_request) {
+		mwifiex_dbg(priv->adapter, ERROR,
+			    "change virtual interface: scan in process\n");
+		return -EBUSY;
+	}
+
 	switch (curr_iftype) {
 	case NL80211_IFTYPE_ADHOC:
 		switch (type) {
@@ -1180,7 +1186,6 @@ mwifiex_cfg80211_change_virtual_intf(struct wiphy *wiphy,
 	case NL80211_IFTYPE_AP:
 		switch (type) {
 		case NL80211_IFTYPE_ADHOC:
-		case NL80211_IFTYPE_STATION:
 			return mwifiex_change_vif_to_sta_adhoc(dev, curr_iftype,
 							       type, params);
 			break;
diff --git a/drivers/net/wireless/marvell/mwifiex/cmdevt.c b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
index dcc529e9c0ef..874660052055 100644
--- a/drivers/net/wireless/marvell/mwifiex/cmdevt.c
+++ b/drivers/net/wireless/marvell/mwifiex/cmdevt.c
@@ -290,13 +290,16 @@ static int mwifiex_dnld_cmd_to_fw(struct mwifiex_private *priv,
 	adapter->dbg.last_cmd_act[adapter->dbg.last_cmd_index] =
 			get_unaligned_le16((u8 *)host_cmd + S_DS_GEN);
 
+	/* Setup the timer after transmit command, except that specific
+	 * command might not have command response.
+	 */
+	if (cmd_code != HostCmd_CMD_FW_DUMP_EVENT)
+		mod_timer(&adapter->cmd_timer,
+			  jiffies + msecs_to_jiffies(MWIFIEX_TIMER_10S));
+
 	/* Clear BSS_NO_BITS from HostCmd */
 	cmd_code &= HostCmd_CMD_ID_MASK;
 
-	/* Setup the timer after transmit command */
-	mod_timer(&adapter->cmd_timer,
-		  jiffies + msecs_to_jiffies(MWIFIEX_TIMER_10S));
-
 	return 0;
 }
 
diff --git a/drivers/net/wireless/marvell/mwifiex/debugfs.c b/drivers/net/wireless/marvell/mwifiex/debugfs.c
index 6f4239be609d..db2872daae97 100644
--- a/drivers/net/wireless/marvell/mwifiex/debugfs.c
+++ b/drivers/net/wireless/marvell/mwifiex/debugfs.c
@@ -168,10 +168,15 @@ mwifiex_device_dump_read(struct file *file, char __user *ubuf,
 {
 	struct mwifiex_private *priv = file->private_data;
 
-	if (!priv->adapter->if_ops.device_dump)
-		return -EIO;
-
-	priv->adapter->if_ops.device_dump(priv->adapter);
+	/* For command timeouts, USB firmware will automatically emit
+	 * firmware dump events, so we don't implement device_dump().
+	 * For user-initiated dumps, we trigger it ourselves.
+	 */
+	if (priv->adapter->iface_type == MWIFIEX_USB)
+		mwifiex_send_cmd(priv, HostCmd_CMD_FW_DUMP_EVENT,
+				 HostCmd_ACT_GEN_SET, 0, NULL, true);
+	else
+		priv->adapter->if_ops.device_dump(priv->adapter);
 
 	return 0;
 }
diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h
index 13cd58e963b3..9c2cdef54074 100644
--- a/drivers/net/wireless/marvell/mwifiex/fw.h
+++ b/drivers/net/wireless/marvell/mwifiex/fw.h
@@ -56,6 +56,15 @@ struct mwifiex_fw_data {
 	u8 data[1];
 } __packed;
 
+struct mwifiex_fw_dump_header {
+	__le16          seq_num;
+	__le16          reserved;
+	__le16          type;
+	__le16          len;
+} __packed;
+
+#define FW_DUMP_INFO_ENDED 0x0002
+
 #define MWIFIEX_FW_DNLD_CMD_1 0x1
 #define MWIFIEX_FW_DNLD_CMD_5 0x5
 #define MWIFIEX_FW_DNLD_CMD_6 0x6
@@ -400,6 +409,7 @@ enum MWIFIEX_802_11_PRIVACY_FILTER {
 #define HostCmd_CMD_TDLS_CONFIG                       0x0100
 #define HostCmd_CMD_MC_POLICY                         0x0121
 #define HostCmd_CMD_TDLS_OPER                         0x0122
+#define HostCmd_CMD_FW_DUMP_EVENT		      0x0125
 #define HostCmd_CMD_SDIO_SP_RX_AGGR_CFG               0x0223
 #define HostCmd_CMD_CHAN_REGION_CFG		      0x0242
 #define HostCmd_CMD_PACKET_AGGR_CTRL		      0x0251
@@ -570,6 +580,7 @@ enum mwifiex_channel_flags {
 #define EVENT_BG_SCAN_STOPPED           0x00000065
 #define EVENT_REMAIN_ON_CHAN_EXPIRED    0x0000005f
 #define EVENT_MULTI_CHAN_INFO           0x0000006a
+#define EVENT_FW_DUMP_INFO		0x00000073
 #define EVENT_TX_STATUS_REPORT		0x00000074
 #define EVENT_BT_COEX_WLAN_PARA_CHANGE	0X00000076
 
diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c
index e1aa86042469..d239e9248c05 100644
--- a/drivers/net/wireless/marvell/mwifiex/init.c
+++ b/drivers/net/wireless/marvell/mwifiex/init.c
@@ -64,6 +64,13 @@ static void wakeup_timer_fn(struct timer_list *t)
 		adapter->if_ops.card_reset(adapter);
 }
 
+static void fw_dump_timer_fn(struct timer_list *t)
+{
+	struct mwifiex_adapter *adapter = from_timer(adapter, t, devdump_timer);
+
+	mwifiex_upload_device_dump(adapter);
+}
+
 /*
  * This function initializes the private structure and sets default
  * values to the members.
@@ -314,6 +321,8 @@ static void mwifiex_init_adapter(struct mwifiex_adapter *adapter)
 	adapter->iface_limit.p2p_intf = MWIFIEX_MAX_P2P_NUM;
 	adapter->active_scan_triggered = false;
 	timer_setup(&adapter->wakeup_timer, wakeup_timer_fn, 0);
+	adapter->devdump_len = 0;
+	timer_setup(&adapter->devdump_timer, fw_dump_timer_fn, 0);
 }
 
 /*
@@ -396,6 +405,7 @@ static void
 mwifiex_adapter_cleanup(struct mwifiex_adapter *adapter)
 {
 	del_timer(&adapter->wakeup_timer);
+	del_timer_sync(&adapter->devdump_timer);
 	mwifiex_cancel_all_pending_cmd(adapter);
 	wake_up_interruptible(&adapter->cmd_wait_q.wait);
 	wake_up_interruptible(&adapter->hs_activate_wait_q);
diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c
index a96bd7e653bf..12e739950332 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.c
+++ b/drivers/net/wireless/marvell/mwifiex/main.c
@@ -1051,9 +1051,30 @@ void mwifiex_multi_chan_resync(struct mwifiex_adapter *adapter)
 }
 EXPORT_SYMBOL_GPL(mwifiex_multi_chan_resync);
 
-int mwifiex_drv_info_dump(struct mwifiex_adapter *adapter, void **drv_info)
+void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter)
 {
-	void *p;
+	/* Dump all the memory data into single file, a userspace script will
+	 * be used to split all the memory data to multiple files
+	 */
+	mwifiex_dbg(adapter, MSG,
+		    "== mwifiex dump information to /sys/class/devcoredump start\n");
+	dev_coredumpv(adapter->dev, adapter->devdump_data, adapter->devdump_len,
+		      GFP_KERNEL);
+	mwifiex_dbg(adapter, MSG,
+		    "== mwifiex dump information to /sys/class/devcoredump end\n");
+
+	/* Device dump data will be freed in device coredump release function
+	 * after 5 min. Here reset adapter->devdump_data and ->devdump_len
+	 * to avoid it been accidentally reused.
+	 */
+	adapter->devdump_data = NULL;
+	adapter->devdump_len = 0;
+}
+EXPORT_SYMBOL_GPL(mwifiex_upload_device_dump);
+
+void mwifiex_drv_info_dump(struct mwifiex_adapter *adapter)
+{
+	char *p;
 	char drv_version[64];
 	struct usb_card_rec *cardp;
 	struct sdio_mmc_card *sdio_card;
@@ -1061,17 +1082,12 @@ int mwifiex_drv_info_dump(struct mwifiex_adapter *adapter, void **drv_info)
 	int i, idx;
 	struct netdev_queue *txq;
 	struct mwifiex_debug_info *debug_info;
-	void *drv_info_dump;
 
 	mwifiex_dbg(adapter, MSG, "===mwifiex driverinfo dump start===\n");
 
-	/* memory allocate here should be free in mwifiex_upload_device_dump*/
-	drv_info_dump = vzalloc(MWIFIEX_DRV_INFO_SIZE_MAX);
-
-	if (!drv_info_dump)
-		return 0;
-
-	p = (char *)(drv_info_dump);
+	p = adapter->devdump_data;
+	strcpy(p, "========Start dump driverinfo========\n");
+	p += strlen("========Start dump driverinfo========\n");
 	p += sprintf(p, "driver_name = " "\"mwifiex\"\n");
 
 	mwifiex_drv_get_driver_version(adapter, drv_version,
@@ -1155,21 +1171,18 @@ int mwifiex_drv_info_dump(struct mwifiex_adapter *adapter, void **drv_info)
 		kfree(debug_info);
 	}
 
+	strcpy(p, "\n========End dump========\n");
+	p += strlen("\n========End dump========\n");
 	mwifiex_dbg(adapter, MSG, "===mwifiex driverinfo dump end===\n");
-	*drv_info = drv_info_dump;
-	return p - drv_info_dump;
+	adapter->devdump_len = p - (char *)adapter->devdump_data;
 }
 EXPORT_SYMBOL_GPL(mwifiex_drv_info_dump);
 
-void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter, void *drv_info,
-				int drv_info_size)
+void mwifiex_prepare_fw_dump_info(struct mwifiex_adapter *adapter)
 {
-	u8 idx, *dump_data, *fw_dump_ptr;
-	u32 dump_len;
-
-	dump_len = (strlen("========Start dump driverinfo========\n") +
-		       drv_info_size +
-		       strlen("\n========End dump========\n"));
+	u8 idx;
+	char *fw_dump_ptr;
+	u32 dump_len = 0;
 
 	for (idx = 0; idx < adapter->num_mem_types; idx++) {
 		struct memory_type_mapping *entry =
@@ -1184,24 +1197,24 @@ void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter, void *drv_info,
 		}
 	}
 
-	dump_data = vzalloc(dump_len + 1);
-	if (!dump_data)
-		goto done;
-
-	fw_dump_ptr = dump_data;
+	if (dump_len + 1 + adapter->devdump_len > MWIFIEX_FW_DUMP_SIZE) {
+		/* Realloc in case buffer overflow */
+		fw_dump_ptr = vzalloc(dump_len + 1 + adapter->devdump_len);
+		mwifiex_dbg(adapter, MSG, "Realloc device dump data.\n");
+		if (!fw_dump_ptr) {
+			vfree(adapter->devdump_data);
+			mwifiex_dbg(adapter, ERROR,
+				    "vzalloc devdump data failure!\n");
+			return;
+		}
 
-	/* Dump all the memory data into single file, a userspace script will
-	 * be used to split all the memory data to multiple files
-	 */
-	mwifiex_dbg(adapter, MSG,
-		    "== mwifiex dump information to /sys/class/devcoredump start");
+		memmove(fw_dump_ptr, adapter->devdump_data,
+			adapter->devdump_len);
+		vfree(adapter->devdump_data);
+		adapter->devdump_data = fw_dump_ptr;
+	}
 
-	strcpy(fw_dump_ptr, "========Start dump driverinfo========\n");
-	fw_dump_ptr += strlen("========Start dump driverinfo========\n");
-	memcpy(fw_dump_ptr, drv_info, drv_info_size);
-	fw_dump_ptr += drv_info_size;
-	strcpy(fw_dump_ptr, "\n========End dump========\n");
-	fw_dump_ptr += strlen("\n========End dump========\n");
+	fw_dump_ptr = (char *)adapter->devdump_data + adapter->devdump_len;
 
 	for (idx = 0; idx < adapter->num_mem_types; idx++) {
 		struct memory_type_mapping *entry =
@@ -1225,14 +1238,8 @@ void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter, void *drv_info,
 		}
 	}
 
-	/* device dump data will be free in device coredump release function
-	 * after 5 min
-	 */
-	dev_coredumpv(adapter->dev, dump_data, dump_len, GFP_KERNEL);
-	mwifiex_dbg(adapter, MSG,
-		    "== mwifiex dump information to /sys/class/devcoredump end");
+	adapter->devdump_len = fw_dump_ptr - (char *)adapter->devdump_data;
 
-done:
 	for (idx = 0; idx < adapter->num_mem_types; idx++) {
 		struct memory_type_mapping *entry =
 			&adapter->mem_type_mapping_tbl[idx];
@@ -1241,10 +1248,8 @@ void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter, void *drv_info,
 		entry->mem_ptr = NULL;
 		entry->mem_size = 0;
 	}
-
-	vfree(drv_info);
 }
-EXPORT_SYMBOL_GPL(mwifiex_upload_device_dump);
+EXPORT_SYMBOL_GPL(mwifiex_prepare_fw_dump_info);
 
 /*
  * CFG802.11 network device handler for statistics retrieval.
diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h
index 154c0796c0c5..6b5539b1f4d8 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.h
+++ b/drivers/net/wireless/marvell/mwifiex/main.h
@@ -94,6 +94,8 @@ enum {
 
 #define MAX_EVENT_SIZE                  2048
 
+#define MWIFIEX_FW_DUMP_SIZE       (2 * 1024 * 1024)
+
 #define ARP_FILTER_MAX_BUF_SIZE         68
 
 #define MWIFIEX_KEY_BUFFER_SIZE			16
@@ -1032,6 +1034,10 @@ struct mwifiex_adapter {
 	bool wake_by_wifi;
 	/* Aggregation parameters*/
 	struct bus_aggr_params bus_aggr;
+	/* Device dump data/length */
+	void *devdump_data;
+	int devdump_len;
+	struct timer_list devdump_timer;
 };
 
 void mwifiex_process_tx_queue(struct mwifiex_adapter *adapter);
@@ -1656,9 +1662,9 @@ void mwifiex_hist_data_add(struct mwifiex_private *priv,
 u8 mwifiex_adjust_data_rate(struct mwifiex_private *priv,
 			    u8 rx_rate, u8 ht_info);
 
-int mwifiex_drv_info_dump(struct mwifiex_adapter *adapter, void **drv_info);
-void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter, void *drv_info,
-				int drv_info_size);
+void mwifiex_drv_info_dump(struct mwifiex_adapter *adapter);
+void mwifiex_prepare_fw_dump_info(struct mwifiex_adapter *adapter);
+void mwifiex_upload_device_dump(struct mwifiex_adapter *adapter);
 void *mwifiex_alloc_dma_align_buf(int rx_len, gfp_t flags);
 void mwifiex_queue_main_work(struct mwifiex_adapter *adapter);
 int mwifiex_get_wakeup_reason(struct mwifiex_private *priv, u16 action,
@@ -1677,6 +1683,7 @@ void mwifiex_process_multi_chan_event(struct mwifiex_private *priv,
 void mwifiex_multi_chan_resync(struct mwifiex_adapter *adapter);
 int mwifiex_set_mac_address(struct mwifiex_private *priv,
 			    struct net_device *dev);
+void mwifiex_devdump_tmo_func(unsigned long function_context);
 
 #ifdef CONFIG_DEBUG_FS
 void mwifiex_debugfs_init(void);
diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c
index cd314946452c..97a6199692ab 100644
--- a/drivers/net/wireless/marvell/mwifiex/pcie.c
+++ b/drivers/net/wireless/marvell/mwifiex/pcie.c
@@ -2769,19 +2769,27 @@ static void mwifiex_pcie_fw_dump(struct mwifiex_adapter *adapter)
 
 static void mwifiex_pcie_device_dump_work(struct mwifiex_adapter *adapter)
 {
-	int drv_info_size;
-	void *drv_info;
+	adapter->devdump_data = vzalloc(MWIFIEX_FW_DUMP_SIZE);
+	if (!adapter->devdump_data) {
+		mwifiex_dbg(adapter, ERROR,
+			    "vzalloc devdump data failure!\n");
+		return;
+	}
 
-	drv_info_size = mwifiex_drv_info_dump(adapter, &drv_info);
+	mwifiex_drv_info_dump(adapter);
 	mwifiex_pcie_fw_dump(adapter);
-	mwifiex_upload_device_dump(adapter, drv_info, drv_info_size);
+	mwifiex_prepare_fw_dump_info(adapter);
+	mwifiex_upload_device_dump(adapter);
 }
 
 static void mwifiex_pcie_card_reset_work(struct mwifiex_adapter *adapter)
 {
 	struct pcie_service_card *card = adapter->card;
 
-	pci_reset_function(card->dev);
+	/* We can't afford to wait here; remove() might be waiting on us. If we
+	 * can't grab the device lock, maybe we'll get another chance later.
+	 */
+	pci_try_reset_function(card->dev);
 }
 
 static void mwifiex_pcie_work(struct work_struct *work)
diff --git a/drivers/net/wireless/marvell/mwifiex/sdio.c b/drivers/net/wireless/marvell/mwifiex/sdio.c
index fd5183c10c4e..a82880132af4 100644
--- a/drivers/net/wireless/marvell/mwifiex/sdio.c
+++ b/drivers/net/wireless/marvell/mwifiex/sdio.c
@@ -2505,15 +2505,21 @@ static void mwifiex_sdio_generic_fw_dump(struct mwifiex_adapter *adapter)
 static void mwifiex_sdio_device_dump_work(struct mwifiex_adapter *adapter)
 {
 	struct sdio_mmc_card *card = adapter->card;
-	int drv_info_size;
-	void *drv_info;
 
-	drv_info_size = mwifiex_drv_info_dump(adapter, &drv_info);
+	adapter->devdump_data = vzalloc(MWIFIEX_FW_DUMP_SIZE);
+	if (!adapter->devdump_data) {
+		mwifiex_dbg(adapter, ERROR,
+			    "vzalloc devdump data failure!\n");
+		return;
+	}
+
+	mwifiex_drv_info_dump(adapter);
 	if (card->fw_dump_enh)
 		mwifiex_sdio_generic_fw_dump(adapter);
 	else
 		mwifiex_sdio_fw_dump(adapter);
-	mwifiex_upload_device_dump(adapter, drv_info, drv_info_size);
+	mwifiex_prepare_fw_dump_info(adapter);
+	mwifiex_upload_device_dump(adapter);
 }
 
 static void mwifiex_sdio_work(struct work_struct *work)
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
index fb090144a6d8..211e47d8b318 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
@@ -2206,6 +2206,10 @@ int mwifiex_sta_prepare_cmd(struct mwifiex_private *priv, uint16_t cmd_no,
 	case HostCmd_CMD_CHAN_REGION_CFG:
 		ret = mwifiex_cmd_chan_region_cfg(priv, cmd_ptr, cmd_action);
 		break;
+	case HostCmd_CMD_FW_DUMP_EVENT:
+		cmd_ptr->command = cpu_to_le16(cmd_no);
+		cmd_ptr->size = cpu_to_le16(S_DS_GEN);
+		break;
 	default:
 		mwifiex_dbg(priv->adapter, ERROR,
 			    "PREP_CMD: unknown cmd- %#x\n", cmd_no);
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c
index d8db412b76c6..93dfb76cd8a6 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c
@@ -584,6 +584,62 @@ void mwifiex_bt_coex_wlan_param_update_event(struct mwifiex_private *priv,
 		adapter->coex_rx_win_size);
 }
 
+static void
+mwifiex_fw_dump_info_event(struct mwifiex_private *priv,
+			   struct sk_buff *event_skb)
+{
+	struct mwifiex_adapter *adapter = priv->adapter;
+	struct mwifiex_fw_dump_header *fw_dump_hdr =
+				(void *)adapter->event_body;
+
+	if (adapter->iface_type != MWIFIEX_USB) {
+		mwifiex_dbg(adapter, MSG,
+			    "event is not on usb interface, ignore it\n");
+		return;
+	}
+
+	if (!adapter->devdump_data) {
+		/* When receive the first event, allocate device dump
+		 * buffer, dump driver info.
+		 */
+		adapter->devdump_data = vzalloc(MWIFIEX_FW_DUMP_SIZE);
+		if (!adapter->devdump_data) {
+			mwifiex_dbg(adapter, ERROR,
+				    "vzalloc devdump data failure!\n");
+			return;
+		}
+
+		mwifiex_drv_info_dump(adapter);
+
+		/* If no proceeded event arrive in 10s, upload device
+		 * dump data, this will be useful if the end of
+		 * transmission event get lost, in this cornel case,
+		 * user would still get partial of the dump.
+		 */
+		mod_timer(&adapter->devdump_timer,
+			  jiffies + msecs_to_jiffies(MWIFIEX_TIMER_10S));
+	}
+
+	/* Overflow check */
+	if (adapter->devdump_len + event_skb->len >= MWIFIEX_FW_DUMP_SIZE)
+		goto upload_dump;
+
+	memmove(adapter->devdump_data + adapter->devdump_len,
+		adapter->event_skb->data, event_skb->len);
+	adapter->devdump_len += event_skb->len;
+
+	if (le16_to_cpu(fw_dump_hdr->type == FW_DUMP_INFO_ENDED)) {
+		mwifiex_dbg(adapter, MSG,
+			    "receive end of transmission flag event!\n");
+		goto upload_dump;
+	}
+	return;
+
+upload_dump:
+	del_timer_sync(&adapter->devdump_timer);
+	mwifiex_upload_device_dump(adapter);
+}
+
 /*
  * This function handles events generated by firmware.
  *
@@ -636,6 +692,7 @@ void mwifiex_bt_coex_wlan_param_update_event(struct mwifiex_private *priv,
  *      - EVENT_DELBA
  *      - EVENT_BA_STREAM_TIEMOUT
  *      - EVENT_AMSDU_AGGR_CTRL
+ *      - EVENT_FW_DUMP_INFO
  */
 int mwifiex_process_sta_event(struct mwifiex_private *priv)
 {
@@ -1007,6 +1064,10 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv)
 					    adapter->event_skb->len -
 					    sizeof(eventcause));
 		break;
+	case EVENT_FW_DUMP_INFO:
+		mwifiex_dbg(adapter, EVENT, "event: firmware debug info\n");
+		mwifiex_fw_dump_info_event(priv, adapter->event_skb);
+		break;
 	/* Debugging event; not used, but let's not print an ERROR for it. */
 	case EVENT_UNKNOWN_DEBUG:
 		mwifiex_dbg(adapter, EVENT, "event: debug\n");
diff --git a/drivers/net/wireless/marvell/mwl8k.c b/drivers/net/wireless/marvell/mwl8k.c
index e813b2ca740c..8e4e9b6919e0 100644
--- a/drivers/net/wireless/marvell/mwl8k.c
+++ b/drivers/net/wireless/marvell/mwl8k.c
@@ -199,7 +199,7 @@ struct mwl8k_priv {
 	struct ieee80211_channel channels_24[14];
 	struct ieee80211_rate rates_24[13];
 	struct ieee80211_supported_band band_50;
-	struct ieee80211_channel channels_50[4];
+	struct ieee80211_channel channels_50[9];
 	struct ieee80211_rate rates_50[8];
 	u32 ap_macids_supported;
 	u32 sta_macids_supported;
@@ -383,6 +383,11 @@ static const struct ieee80211_channel mwl8k_channels_50[] = {
 	{ .band = NL80211_BAND_5GHZ, .center_freq = 5200, .hw_value = 40, },
 	{ .band = NL80211_BAND_5GHZ, .center_freq = 5220, .hw_value = 44, },
 	{ .band = NL80211_BAND_5GHZ, .center_freq = 5240, .hw_value = 48, },
+	{ .band = NL80211_BAND_5GHZ, .center_freq = 5745, .hw_value = 149, },
+	{ .band = NL80211_BAND_5GHZ, .center_freq = 5765, .hw_value = 153, },
+	{ .band = NL80211_BAND_5GHZ, .center_freq = 5785, .hw_value = 157, },
+	{ .band = NL80211_BAND_5GHZ, .center_freq = 5805, .hw_value = 161, },
+	{ .band = NL80211_BAND_5GHZ, .center_freq = 5825, .hw_value = 165, },
 };
 
 static const struct ieee80211_rate mwl8k_rates_50[] = {
diff --git a/drivers/net/wireless/mediatek/Kconfig b/drivers/net/wireless/mediatek/Kconfig
index 28843fed750a..92ce4062f307 100644
--- a/drivers/net/wireless/mediatek/Kconfig
+++ b/drivers/net/wireless/mediatek/Kconfig
@@ -11,4 +11,5 @@ config WLAN_VENDOR_MEDIATEK
 
 if WLAN_VENDOR_MEDIATEK
 source "drivers/net/wireless/mediatek/mt7601u/Kconfig"
+source "drivers/net/wireless/mediatek/mt76/Kconfig"
 endif # WLAN_VENDOR_MEDIATEK
diff --git a/drivers/net/wireless/mediatek/Makefile b/drivers/net/wireless/mediatek/Makefile
index 9d5f182fd7fd..00f945f59b38 100644
--- a/drivers/net/wireless/mediatek/Makefile
+++ b/drivers/net/wireless/mediatek/Makefile
@@ -1 +1,2 @@
 obj-$(CONFIG_MT7601U)	+= mt7601u/
+obj-$(CONFIG_MT76_CORE)	+= mt76/
diff --git a/drivers/net/wireless/mediatek/mt76/Kconfig b/drivers/net/wireless/mediatek/mt76/Kconfig
new file mode 100644
index 000000000000..fc05d79c80d0
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/Kconfig
@@ -0,0 +1,10 @@
+config MT76_CORE
+	tristate
+
+config MT76x2E
+	tristate "MediaTek MT76x2E (PCIe) support"
+	select MT76_CORE
+	depends on MAC80211
+	depends on PCI
+	---help---
+	  This adds support for MT7612/MT7602/MT7662-based wireless PCIe devices.
diff --git a/drivers/net/wireless/mediatek/mt76/Makefile b/drivers/net/wireless/mediatek/mt76/Makefile
new file mode 100644
index 000000000000..a0156bc01dea
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/Makefile
@@ -0,0 +1,15 @@
+obj-$(CONFIG_MT76_CORE) += mt76.o
+obj-$(CONFIG_MT76x2E) += mt76x2e.o
+
+mt76-y := \
+	mmio.o util.o trace.o dma.o mac80211.o debugfs.o eeprom.o tx.o agg-rx.o
+
+CFLAGS_trace.o := -I$(src)
+
+mt76x2e-y := \
+	mt76x2_pci.o mt76x2_dma.o \
+	mt76x2_main.o mt76x2_init.o mt76x2_debugfs.o mt76x2_tx.o \
+	mt76x2_core.o mt76x2_mac.o mt76x2_eeprom.o mt76x2_mcu.o mt76x2_phy.o \
+	mt76x2_dfs.o mt76x2_trace.o
+
+CFLAGS_mt76x2_trace.o := -I$(src)
diff --git a/drivers/net/wireless/mediatek/mt76/agg-rx.c b/drivers/net/wireless/mediatek/mt76/agg-rx.c
new file mode 100644
index 000000000000..fcb208d1f276
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/agg-rx.c
@@ -0,0 +1,294 @@
+/*
+ * Copyright (C) 2018 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#include "mt76.h"
+
+#define REORDER_TIMEOUT (HZ / 10)
+
+static void
+mt76_aggr_release(struct mt76_rx_tid *tid, struct sk_buff_head *frames, int idx)
+{
+	struct sk_buff *skb;
+
+	tid->head = ieee80211_sn_inc(tid->head);
+
+	skb = tid->reorder_buf[idx];
+	if (!skb)
+		return;
+
+	tid->reorder_buf[idx] = NULL;
+	tid->nframes--;
+	__skb_queue_tail(frames, skb);
+}
+
+static void
+mt76_rx_aggr_release_frames(struct mt76_rx_tid *tid, struct sk_buff_head *frames,
+			 u16 head)
+{
+	int idx;
+
+	while (ieee80211_sn_less(tid->head, head)) {
+		idx = tid->head % tid->size;
+		mt76_aggr_release(tid, frames, idx);
+	}
+}
+
+static void
+mt76_rx_aggr_release_head(struct mt76_rx_tid *tid, struct sk_buff_head *frames)
+{
+	int idx = tid->head % tid->size;
+
+	while (tid->reorder_buf[idx]) {
+		mt76_aggr_release(tid, frames, idx);
+		idx = tid->head % tid->size;
+	}
+}
+
+static void
+mt76_rx_aggr_check_release(struct mt76_rx_tid *tid, struct sk_buff_head *frames)
+{
+	struct mt76_rx_status *status;
+	struct sk_buff *skb;
+	int start, idx, nframes;
+
+	if (!tid->nframes)
+		return;
+
+	mt76_rx_aggr_release_head(tid, frames);
+
+	start = tid->head % tid->size;
+	nframes = tid->nframes;
+
+	for (idx = (tid->head + 1) % tid->size;
+	     idx != start && nframes;
+	     idx = (idx + 1) % tid->size) {
+
+		skb = tid->reorder_buf[idx];
+		if (!skb)
+			continue;
+
+		nframes--;
+		status = (struct mt76_rx_status *) skb->cb;
+		if (!time_after(jiffies, status->reorder_time +
+					 REORDER_TIMEOUT))
+			continue;
+
+		mt76_rx_aggr_release_frames(tid, frames, status->seqno);
+	}
+
+	mt76_rx_aggr_release_head(tid, frames);
+}
+
+static void
+mt76_rx_aggr_reorder_work(struct work_struct *work)
+{
+	struct mt76_rx_tid *tid = container_of(work, struct mt76_rx_tid,
+					       reorder_work.work);
+	struct mt76_dev *dev = tid->dev;
+	struct sk_buff_head frames;
+	int nframes;
+
+	__skb_queue_head_init(&frames);
+
+	local_bh_disable();
+
+	spin_lock(&tid->lock);
+	mt76_rx_aggr_check_release(tid, &frames);
+	nframes = tid->nframes;
+	spin_unlock(&tid->lock);
+
+	if (nframes)
+		ieee80211_queue_delayed_work(tid->dev->hw, &tid->reorder_work,
+					     REORDER_TIMEOUT);
+	mt76_rx_complete(dev, &frames, -1);
+
+	local_bh_enable();
+}
+
+static void
+mt76_rx_aggr_check_ctl(struct sk_buff *skb, struct sk_buff_head *frames)
+{
+	struct mt76_rx_status *status = (struct mt76_rx_status *) skb->cb;
+	struct ieee80211_bar *bar = (struct ieee80211_bar *) skb->data;
+	struct mt76_wcid *wcid = status->wcid;
+	struct mt76_rx_tid *tid;
+	u16 seqno;
+
+	if (!ieee80211_is_ctl(bar->frame_control))
+		return;
+
+	if (!ieee80211_is_back_req(bar->frame_control))
+		return;
+
+	status->tid = le16_to_cpu(bar->control) >> 12;
+	seqno = le16_to_cpu(bar->start_seq_num) >> 4;
+	tid = rcu_dereference(wcid->aggr[status->tid]);
+	if (!tid)
+		return;
+
+	spin_lock_bh(&tid->lock);
+	mt76_rx_aggr_release_frames(tid, frames, seqno);
+	mt76_rx_aggr_release_head(tid, frames);
+	spin_unlock_bh(&tid->lock);
+}
+
+void mt76_rx_aggr_reorder(struct sk_buff *skb, struct sk_buff_head *frames)
+{
+	struct mt76_rx_status *status = (struct mt76_rx_status *) skb->cb;
+	struct mt76_wcid *wcid = status->wcid;
+	struct ieee80211_sta *sta;
+	struct mt76_rx_tid *tid;
+	bool sn_less;
+	u16 seqno, head, size;
+	u8 idx;
+
+	__skb_queue_tail(frames, skb);
+
+	sta = wcid_to_sta(wcid);
+	if (!sta)
+		return;
+
+	if (!status->aggr) {
+		mt76_rx_aggr_check_ctl(skb, frames);
+		return;
+	}
+
+	tid = rcu_dereference(wcid->aggr[status->tid]);
+	if (!tid)
+		return;
+
+	spin_lock_bh(&tid->lock);
+
+	if (tid->stopped)
+		goto out;
+
+	head = tid->head;
+	seqno = status->seqno;
+	size = tid->size;
+	sn_less = ieee80211_sn_less(seqno, head);
+
+	if (!tid->started) {
+		if (sn_less)
+			goto out;
+
+		tid->started = true;
+	}
+
+	if (sn_less) {
+		__skb_unlink(skb, frames);
+		dev_kfree_skb(skb);
+		goto out;
+	}
+
+	if (seqno == head) {
+		tid->head = ieee80211_sn_inc(head);
+		if (tid->nframes)
+			mt76_rx_aggr_release_head(tid, frames);
+		goto out;
+	}
+
+	__skb_unlink(skb, frames);
+
+	/*
+	 * Frame sequence number exceeds buffering window, free up some space
+	 * by releasing previous frames
+	 */
+	if (!ieee80211_sn_less(seqno, head + size)) {
+		head = ieee80211_sn_inc(ieee80211_sn_sub(seqno, size));
+		mt76_rx_aggr_release_frames(tid, frames, head);
+	}
+
+	idx = seqno % size;
+
+	/* Discard if the current slot is already in use */
+	if (tid->reorder_buf[idx]) {
+		dev_kfree_skb(skb);
+		goto out;
+	}
+
+	status->reorder_time = jiffies;
+	tid->reorder_buf[idx] = skb;
+	tid->nframes++;
+	mt76_rx_aggr_release_head(tid, frames);
+
+	ieee80211_queue_delayed_work(tid->dev->hw, &tid->reorder_work, REORDER_TIMEOUT);
+
+out:
+	spin_unlock_bh(&tid->lock);
+}
+
+int mt76_rx_aggr_start(struct mt76_dev *dev, struct mt76_wcid *wcid, u8 tidno,
+		       u16 ssn, u8 size)
+{
+	struct mt76_rx_tid *tid;
+
+	mt76_rx_aggr_stop(dev, wcid, tidno);
+
+	tid = kzalloc(sizeof(*tid) + size * sizeof(tid->reorder_buf[0]),
+		      GFP_KERNEL);
+	if (!tid)
+		return -ENOMEM;
+
+	tid->dev = dev;
+	tid->head = ssn;
+	tid->size = size;
+	INIT_DELAYED_WORK(&tid->reorder_work, mt76_rx_aggr_reorder_work);
+	spin_lock_init(&tid->lock);
+
+	rcu_assign_pointer(wcid->aggr[tidno], tid);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mt76_rx_aggr_start);
+
+static void mt76_rx_aggr_shutdown(struct mt76_dev *dev, struct mt76_rx_tid *tid)
+{
+	u8 size = tid->size;
+	int i;
+
+	spin_lock_bh(&tid->lock);
+
+	tid->stopped = true;
+	for (i = 0; tid->nframes && i < size; i++) {
+		struct sk_buff *skb = tid->reorder_buf[i];
+
+		if (!skb)
+			continue;
+
+		tid->nframes--;
+		dev_kfree_skb(skb);
+	}
+
+	spin_unlock_bh(&tid->lock);
+
+	cancel_delayed_work_sync(&tid->reorder_work);
+}
+
+void mt76_rx_aggr_stop(struct mt76_dev *dev, struct mt76_wcid *wcid, u8 tidno)
+{
+	struct mt76_rx_tid *tid;
+
+	rcu_read_lock();
+
+	tid = rcu_dereference(wcid->aggr[tidno]);
+	if (tid) {
+		rcu_assign_pointer(wcid->aggr[tidno], NULL);
+		mt76_rx_aggr_shutdown(dev, tid);
+		kfree_rcu(tid, rcu_head);
+	}
+
+	rcu_read_unlock();
+}
+EXPORT_SYMBOL_GPL(mt76_rx_aggr_stop);
diff --git a/drivers/net/wireless/mediatek/mt76/debugfs.c b/drivers/net/wireless/mediatek/mt76/debugfs.c
new file mode 100644
index 000000000000..c121b502a462
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/debugfs.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#include "mt76.h"
+
+static int
+mt76_reg_set(void *data, u64 val)
+{
+	struct mt76_dev *dev = data;
+
+	dev->bus->wr(dev, dev->debugfs_reg, val);
+	return 0;
+}
+
+static int
+mt76_reg_get(void *data, u64 *val)
+{
+	struct mt76_dev *dev = data;
+
+	*val = dev->bus->rr(dev, dev->debugfs_reg);
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(fops_regval, mt76_reg_get, mt76_reg_set,
+			 "0x%08llx\n");
+
+static int
+mt76_queues_read(struct seq_file *s, void *data)
+{
+	struct mt76_dev *dev = dev_get_drvdata(s->private);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++) {
+		struct mt76_queue *q = &dev->q_tx[i];
+
+		if (!q->ndesc)
+			continue;
+
+		seq_printf(s,
+			   "%d:	queued=%d head=%d tail=%d swq_queued=%d\n",
+			   i, q->queued, q->head, q->tail, q->swq_queued);
+	}
+
+	return 0;
+}
+
+struct dentry *mt76_register_debugfs(struct mt76_dev *dev)
+{
+	struct dentry *dir;
+
+	dir = debugfs_create_dir("mt76", dev->hw->wiphy->debugfsdir);
+	if (!dir)
+		return NULL;
+
+	debugfs_create_u8("led_pin", S_IRUSR | S_IWUSR, dir, &dev->led_pin);
+	debugfs_create_u32("regidx", S_IRUSR | S_IWUSR, dir, &dev->debugfs_reg);
+	debugfs_create_file_unsafe("regval", S_IRUSR | S_IWUSR, dir, dev,
+				   &fops_regval);
+	debugfs_create_blob("eeprom", S_IRUSR, dir, &dev->eeprom);
+	if (dev->otp.data)
+		debugfs_create_blob("otp", S_IRUSR, dir, &dev->otp);
+	debugfs_create_devm_seqfile(dev->dev, "queues", dir, mt76_queues_read);
+
+	return dir;
+}
+EXPORT_SYMBOL_GPL(mt76_register_debugfs);
diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c
new file mode 100644
index 000000000000..3518703524e7
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/dma.c
@@ -0,0 +1,459 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/dma-mapping.h>
+#include "mt76.h"
+#include "dma.h"
+
+#define DMA_DUMMY_TXWI	((void *) ~0)
+
+static int
+mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q)
+{
+	int size;
+	int i;
+
+	spin_lock_init(&q->lock);
+	INIT_LIST_HEAD(&q->swq);
+
+	size = q->ndesc * sizeof(struct mt76_desc);
+	q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
+	if (!q->desc)
+		return -ENOMEM;
+
+	size = q->ndesc * sizeof(*q->entry);
+	q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
+	if (!q->entry)
+		return -ENOMEM;
+
+	/* clear descriptors */
+	for (i = 0; i < q->ndesc; i++)
+		q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
+
+	iowrite32(q->desc_dma, &q->regs->desc_base);
+	iowrite32(0, &q->regs->cpu_idx);
+	iowrite32(0, &q->regs->dma_idx);
+	iowrite32(q->ndesc, &q->regs->ring_size);
+
+	return 0;
+}
+
+static int
+mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
+		 struct mt76_queue_buf *buf, int nbufs, u32 info,
+		 struct sk_buff *skb, void *txwi)
+{
+	struct mt76_desc *desc;
+	u32 ctrl;
+	int i, idx = -1;
+
+	if (txwi)
+		q->entry[q->head].txwi = DMA_DUMMY_TXWI;
+
+	for (i = 0; i < nbufs; i += 2, buf += 2) {
+		u32 buf0 = buf[0].addr, buf1 = 0;
+
+		ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
+		if (i < nbufs - 1) {
+			buf1 = buf[1].addr;
+			ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
+		}
+
+		if (i == nbufs - 1)
+			ctrl |= MT_DMA_CTL_LAST_SEC0;
+		else if (i == nbufs - 2)
+			ctrl |= MT_DMA_CTL_LAST_SEC1;
+
+		idx = q->head;
+		q->head = (q->head + 1) % q->ndesc;
+
+		desc = &q->desc[idx];
+
+		WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
+		WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
+		WRITE_ONCE(desc->info, cpu_to_le32(info));
+		WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
+
+		q->queued++;
+	}
+
+	q->entry[idx].txwi = txwi;
+	q->entry[idx].skb = skb;
+
+	return idx;
+}
+
+static void
+mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
+			struct mt76_queue_entry *prev_e)
+{
+	struct mt76_queue_entry *e = &q->entry[idx];
+	__le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
+	u32 ctrl = le32_to_cpu(__ctrl);
+
+	if (!e->txwi || !e->skb) {
+		__le32 addr = READ_ONCE(q->desc[idx].buf0);
+		u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
+
+		dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
+				 DMA_TO_DEVICE);
+	}
+
+	if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
+		__le32 addr = READ_ONCE(q->desc[idx].buf1);
+		u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);
+
+		dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
+				 DMA_TO_DEVICE);
+	}
+
+	if (e->txwi == DMA_DUMMY_TXWI)
+		e->txwi = NULL;
+
+	*prev_e = *e;
+	memset(e, 0, sizeof(*e));
+}
+
+static void
+mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
+{
+	q->head = ioread32(&q->regs->dma_idx);
+	q->tail = q->head;
+	iowrite32(q->head, &q->regs->cpu_idx);
+}
+
+static void
+mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
+{
+	struct mt76_queue *q = &dev->q_tx[qid];
+	struct mt76_queue_entry entry;
+	bool wake = false;
+	int last;
+
+	if (!q->ndesc)
+		return;
+
+	spin_lock_bh(&q->lock);
+	if (flush)
+		last = -1;
+	else
+		last = ioread32(&q->regs->dma_idx);
+
+	while (q->queued && q->tail != last) {
+		mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
+		if (entry.schedule)
+			q->swq_queued--;
+
+		if (entry.skb)
+			dev->drv->tx_complete_skb(dev, q, &entry, flush);
+
+		if (entry.txwi) {
+			mt76_put_txwi(dev, entry.txwi);
+			wake = true;
+		}
+
+		q->tail = (q->tail + 1) % q->ndesc;
+		q->queued--;
+
+		if (!flush && q->tail == last)
+			last = ioread32(&q->regs->dma_idx);
+	}
+
+	if (!flush)
+		mt76_txq_schedule(dev, q);
+	else
+		mt76_dma_sync_idx(dev, q);
+
+	wake = wake && qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
+	spin_unlock_bh(&q->lock);
+
+	if (wake)
+		ieee80211_wake_queue(dev->hw, qid);
+}
+
+static void *
+mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
+		 int *len, u32 *info, bool *more)
+{
+	struct mt76_queue_entry *e = &q->entry[idx];
+	struct mt76_desc *desc = &q->desc[idx];
+	dma_addr_t buf_addr;
+	void *buf = e->buf;
+	int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
+
+	buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
+	if (len) {
+		u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
+		*len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
+		*more = !(ctl & MT_DMA_CTL_LAST_SEC0);
+	}
+
+	if (info)
+		*info = le32_to_cpu(desc->info);
+
+	dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
+	e->buf = NULL;
+
+	return buf;
+}
+
+static void *
+mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
+		 int *len, u32 *info, bool *more)
+{
+	int idx = q->tail;
+
+	*more = false;
+	if (!q->queued)
+		return NULL;
+
+	if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
+		return NULL;
+
+	q->tail = (q->tail + 1) % q->ndesc;
+	q->queued--;
+
+	return mt76_dma_get_buf(dev, q, idx, len, info, more);
+}
+
+static void
+mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
+{
+	iowrite32(q->head, &q->regs->cpu_idx);
+}
+
+static int
+mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, bool napi)
+{
+	dma_addr_t addr;
+	void *buf;
+	int frames = 0;
+	int len = SKB_WITH_OVERHEAD(q->buf_size);
+	int offset = q->buf_offset;
+	int idx;
+	void *(*alloc)(unsigned int fragsz);
+
+	if (napi)
+		alloc = napi_alloc_frag;
+	else
+		alloc = netdev_alloc_frag;
+
+	spin_lock_bh(&q->lock);
+
+	while (q->queued < q->ndesc - 1) {
+		struct mt76_queue_buf qbuf;
+
+		buf = alloc(q->buf_size);
+		if (!buf)
+			break;
+
+		addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
+		if (dma_mapping_error(dev->dev, addr)) {
+			skb_free_frag(buf);
+			break;
+		}
+
+		qbuf.addr = addr + offset;
+		qbuf.len = len - offset;
+		idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
+		frames++;
+	}
+
+	if (frames)
+		mt76_dma_kick_queue(dev, q);
+
+	spin_unlock_bh(&q->lock);
+
+	return frames;
+}
+
+static void
+mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
+{
+	void *buf;
+	bool more;
+
+	spin_lock_bh(&q->lock);
+	do {
+		buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
+		if (!buf)
+			break;
+
+		skb_free_frag(buf);
+	} while (1);
+	spin_unlock_bh(&q->lock);
+}
+
+static void
+mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
+{
+	struct mt76_queue *q = &dev->q_rx[qid];
+	int i;
+
+	for (i = 0; i < q->ndesc; i++)
+		q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);
+
+	mt76_dma_rx_cleanup(dev, q);
+	mt76_dma_sync_idx(dev, q);
+	mt76_dma_rx_fill(dev, q, false);
+}
+
+static void
+mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
+		  int len, bool more)
+{
+	struct page *page = virt_to_head_page(data);
+	int offset = data - page_address(page);
+	struct sk_buff *skb = q->rx_head;
+
+	offset += q->buf_offset;
+	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len,
+			q->buf_size);
+
+	if (more)
+		return;
+
+	q->rx_head = NULL;
+	dev->drv->rx_skb(dev, q - dev->q_rx, skb);
+}
+
+static int
+mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
+{
+	struct sk_buff *skb;
+	unsigned char *data;
+	int len;
+	int done = 0;
+	bool more;
+
+	while (done < budget) {
+		u32 info;
+
+		data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
+		if (!data)
+			break;
+
+		if (q->rx_head) {
+			mt76_add_fragment(dev, q, data, len, more);
+			continue;
+		}
+
+		skb = build_skb(data, q->buf_size);
+		if (!skb) {
+			skb_free_frag(data);
+			continue;
+		}
+
+		skb_reserve(skb, q->buf_offset);
+		if (skb->tail + len > skb->end) {
+			dev_kfree_skb(skb);
+			continue;
+		}
+
+		if (q == &dev->q_rx[MT_RXQ_MCU]) {
+			u32 *rxfce = (u32 *) skb->cb;
+			*rxfce = info;
+		}
+
+		__skb_put(skb, len);
+		done++;
+
+		if (more) {
+			q->rx_head = skb;
+			continue;
+		}
+
+		dev->drv->rx_skb(dev, q - dev->q_rx, skb);
+	}
+
+	mt76_dma_rx_fill(dev, q, true);
+	return done;
+}
+
+static int
+mt76_dma_rx_poll(struct napi_struct *napi, int budget)
+{
+	struct mt76_dev *dev;
+	int qid, done = 0, cur;
+
+	dev = container_of(napi->dev, struct mt76_dev, napi_dev);
+	qid = napi - dev->napi;
+
+	rcu_read_lock();
+
+	do {
+		cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
+		mt76_rx_poll_complete(dev, qid);
+		done += cur;
+	} while (cur && done < budget);
+
+	rcu_read_unlock();
+
+	if (done < budget) {
+		napi_complete(napi);
+		dev->drv->rx_poll_complete(dev, qid);
+	}
+
+	return done;
+}
+
+static int
+mt76_dma_init(struct mt76_dev *dev)
+{
+	int i;
+
+	init_dummy_netdev(&dev->napi_dev);
+
+	for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
+		netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
+			       64);
+		mt76_dma_rx_fill(dev, &dev->q_rx[i], false);
+		skb_queue_head_init(&dev->rx_skb[i]);
+		napi_enable(&dev->napi[i]);
+	}
+
+	return 0;
+}
+
+static const struct mt76_queue_ops mt76_dma_ops = {
+	.init = mt76_dma_init,
+	.alloc = mt76_dma_alloc_queue,
+	.add_buf = mt76_dma_add_buf,
+	.tx_cleanup = mt76_dma_tx_cleanup,
+	.rx_reset = mt76_dma_rx_reset,
+	.kick = mt76_dma_kick_queue,
+};
+
+int mt76_dma_attach(struct mt76_dev *dev)
+{
+	dev->queue_ops = &mt76_dma_ops;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mt76_dma_attach);
+
+void mt76_dma_cleanup(struct mt76_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
+		mt76_dma_tx_cleanup(dev, i, true);
+
+	for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
+		netif_napi_del(&dev->napi[i]);
+		mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
+	}
+}
+EXPORT_SYMBOL_GPL(mt76_dma_cleanup);
diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h
new file mode 100644
index 000000000000..1dad39697929
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/dma.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#ifndef __MT76_DMA_H
+#define __MT76_DMA_H
+
+#define MT_RING_SIZE			0x10
+
+#define MT_DMA_CTL_SD_LEN1		GENMASK(13, 0)
+#define MT_DMA_CTL_LAST_SEC1		BIT(14)
+#define MT_DMA_CTL_BURST		BIT(15)
+#define MT_DMA_CTL_SD_LEN0		GENMASK(29, 16)
+#define MT_DMA_CTL_LAST_SEC0		BIT(30)
+#define MT_DMA_CTL_DMA_DONE		BIT(31)
+
+struct mt76_desc {
+	__le32 buf0;
+	__le32 ctrl;
+	__le32 buf1;
+	__le32 info;
+} __packed __aligned(4);
+
+int mt76_dma_attach(struct mt76_dev *dev);
+void mt76_dma_cleanup(struct mt76_dev *dev);
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/eeprom.c b/drivers/net/wireless/mediatek/mt76/eeprom.c
new file mode 100644
index 000000000000..530e5593765c
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/eeprom.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#include <linux/of.h>
+#include <linux/of_net.h>
+#include <linux/mtd/mtd.h>
+#include <linux/mtd/partitions.h>
+#include <linux/etherdevice.h>
+#include "mt76.h"
+
+static int
+mt76_get_of_eeprom(struct mt76_dev *dev, int len)
+{
+#if defined(CONFIG_OF) && defined(CONFIG_MTD)
+	struct device_node *np = dev->dev->of_node;
+	struct mtd_info *mtd;
+	const __be32 *list;
+	const char *part;
+	phandle phandle;
+	int offset = 0;
+	int size;
+	size_t retlen;
+	int ret;
+
+	if (!np)
+		return -ENOENT;
+
+	list = of_get_property(np, "mediatek,mtd-eeprom", &size);
+	if (!list)
+		return -ENOENT;
+
+	phandle = be32_to_cpup(list++);
+	if (!phandle)
+		return -ENOENT;
+
+	np = of_find_node_by_phandle(phandle);
+	if (!np)
+		return -EINVAL;
+
+	part = of_get_property(np, "label", NULL);
+	if (!part)
+		part = np->name;
+
+	mtd = get_mtd_device_nm(part);
+	if (IS_ERR(mtd))
+		return PTR_ERR(mtd);
+
+	if (size <= sizeof(*list))
+		return -EINVAL;
+
+	offset = be32_to_cpup(list);
+	ret = mtd_read(mtd, offset, len, &retlen, dev->eeprom.data);
+	put_mtd_device(mtd);
+	if (ret)
+		return ret;
+
+	if (retlen < len)
+		return -EINVAL;
+
+	return 0;
+#else
+	return -ENOENT;
+#endif
+}
+
+void
+mt76_eeprom_override(struct mt76_dev *dev)
+{
+#ifdef CONFIG_OF
+	struct device_node *np = dev->dev->of_node;
+	const u8 *mac;
+
+	if (!np)
+		return;
+
+	mac = of_get_mac_address(np);
+	if (mac)
+		memcpy(dev->macaddr, mac, ETH_ALEN);
+#endif
+
+	if (!is_valid_ether_addr(dev->macaddr)) {
+		eth_random_addr(dev->macaddr);
+		dev_info(dev->dev,
+			 "Invalid MAC address, using random address %pM\n",
+			 dev->macaddr);
+	}
+}
+EXPORT_SYMBOL_GPL(mt76_eeprom_override);
+
+int
+mt76_eeprom_init(struct mt76_dev *dev, int len)
+{
+	dev->eeprom.size = len;
+	dev->eeprom.data = devm_kzalloc(dev->dev, len, GFP_KERNEL);
+	if (!dev->eeprom.data)
+		return -ENOMEM;
+
+	return !mt76_get_of_eeprom(dev, len);
+}
+EXPORT_SYMBOL_GPL(mt76_eeprom_init);
diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c
new file mode 100644
index 000000000000..85f8d324ebf8
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mac80211.c
@@ -0,0 +1,555 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#include <linux/of.h>
+#include "mt76.h"
+
+#define CHAN2G(_idx, _freq) {			\
+	.band = NL80211_BAND_2GHZ,		\
+	.center_freq = (_freq),			\
+	.hw_value = (_idx),			\
+	.max_power = 30,			\
+}
+
+#define CHAN5G(_idx, _freq) {			\
+	.band = NL80211_BAND_5GHZ,		\
+	.center_freq = (_freq),			\
+	.hw_value = (_idx),			\
+	.max_power = 30,			\
+}
+
+static const struct ieee80211_channel mt76_channels_2ghz[] = {
+	CHAN2G(1, 2412),
+	CHAN2G(2, 2417),
+	CHAN2G(3, 2422),
+	CHAN2G(4, 2427),
+	CHAN2G(5, 2432),
+	CHAN2G(6, 2437),
+	CHAN2G(7, 2442),
+	CHAN2G(8, 2447),
+	CHAN2G(9, 2452),
+	CHAN2G(10, 2457),
+	CHAN2G(11, 2462),
+	CHAN2G(12, 2467),
+	CHAN2G(13, 2472),
+	CHAN2G(14, 2484),
+};
+
+static const struct ieee80211_channel mt76_channels_5ghz[] = {
+	CHAN5G(36, 5180),
+	CHAN5G(40, 5200),
+	CHAN5G(44, 5220),
+	CHAN5G(48, 5240),
+
+	CHAN5G(52, 5260),
+	CHAN5G(56, 5280),
+	CHAN5G(60, 5300),
+	CHAN5G(64, 5320),
+
+	CHAN5G(100, 5500),
+	CHAN5G(104, 5520),
+	CHAN5G(108, 5540),
+	CHAN5G(112, 5560),
+	CHAN5G(116, 5580),
+	CHAN5G(120, 5600),
+	CHAN5G(124, 5620),
+	CHAN5G(128, 5640),
+	CHAN5G(132, 5660),
+	CHAN5G(136, 5680),
+	CHAN5G(140, 5700),
+
+	CHAN5G(149, 5745),
+	CHAN5G(153, 5765),
+	CHAN5G(157, 5785),
+	CHAN5G(161, 5805),
+	CHAN5G(165, 5825),
+};
+
+static const struct ieee80211_tpt_blink mt76_tpt_blink[] = {
+	{ .throughput =   0 * 1024, .blink_time = 334 },
+	{ .throughput =   1 * 1024, .blink_time = 260 },
+	{ .throughput =   5 * 1024, .blink_time = 220 },
+	{ .throughput =  10 * 1024, .blink_time = 190 },
+	{ .throughput =  20 * 1024, .blink_time = 170 },
+	{ .throughput =  50 * 1024, .blink_time = 150 },
+	{ .throughput =  70 * 1024, .blink_time = 130 },
+	{ .throughput = 100 * 1024, .blink_time = 110 },
+	{ .throughput = 200 * 1024, .blink_time =  80 },
+	{ .throughput = 300 * 1024, .blink_time =  50 },
+};
+
+static int mt76_led_init(struct mt76_dev *dev)
+{
+	struct device_node *np = dev->dev->of_node;
+	struct ieee80211_hw *hw = dev->hw;
+	int led_pin;
+
+	if (!dev->led_cdev.brightness_set && !dev->led_cdev.blink_set)
+		return 0;
+
+	snprintf(dev->led_name, sizeof(dev->led_name),
+		 "mt76-%s", wiphy_name(hw->wiphy));
+
+	dev->led_cdev.name = dev->led_name;
+	dev->led_cdev.default_trigger =
+		ieee80211_create_tpt_led_trigger(hw,
+					IEEE80211_TPT_LEDTRIG_FL_RADIO,
+					mt76_tpt_blink,
+					ARRAY_SIZE(mt76_tpt_blink));
+
+	np = of_get_child_by_name(np, "led");
+	if (np) {
+		if (!of_property_read_u32(np, "led-sources", &led_pin))
+			dev->led_pin = led_pin;
+		dev->led_al = of_property_read_bool(np, "led-active-low");
+	}
+
+	return devm_led_classdev_register(dev->dev, &dev->led_cdev);
+}
+
+static int
+mt76_init_sband(struct mt76_dev *dev, struct mt76_sband *msband,
+		const struct ieee80211_channel *chan, int n_chan,
+		struct ieee80211_rate *rates, int n_rates, bool vht)
+{
+	struct ieee80211_supported_band *sband = &msband->sband;
+	struct ieee80211_sta_ht_cap *ht_cap;
+	struct ieee80211_sta_vht_cap *vht_cap;
+	void *chanlist;
+	u16 mcs_map;
+	int size;
+
+	size = n_chan * sizeof(*chan);
+	chanlist = devm_kmemdup(dev->dev, chan, size, GFP_KERNEL);
+	if (!chanlist)
+		return -ENOMEM;
+
+	msband->chan = devm_kzalloc(dev->dev, n_chan * sizeof(*msband->chan),
+				    GFP_KERNEL);
+	if (!msband->chan)
+		return -ENOMEM;
+
+	sband->channels = chanlist;
+	sband->n_channels = n_chan;
+	sband->bitrates = rates;
+	sband->n_bitrates = n_rates;
+	dev->chandef.chan = &sband->channels[0];
+
+	ht_cap = &sband->ht_cap;
+	ht_cap->ht_supported = true;
+	ht_cap->cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
+		       IEEE80211_HT_CAP_GRN_FLD |
+		       IEEE80211_HT_CAP_SGI_20 |
+		       IEEE80211_HT_CAP_SGI_40 |
+		       IEEE80211_HT_CAP_TX_STBC |
+		       (1 << IEEE80211_HT_CAP_RX_STBC_SHIFT);
+
+	ht_cap->mcs.rx_mask[0] = 0xff;
+	ht_cap->mcs.rx_mask[1] = 0xff;
+	ht_cap->mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED;
+	ht_cap->ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K;
+	ht_cap->ampdu_density = IEEE80211_HT_MPDU_DENSITY_4;
+
+	if (!vht)
+		return 0;
+
+	vht_cap = &sband->vht_cap;
+	vht_cap->vht_supported = true;
+
+	mcs_map = (IEEE80211_VHT_MCS_SUPPORT_0_9 << (0 * 2)) |
+		  (IEEE80211_VHT_MCS_SUPPORT_0_9 << (1 * 2)) |
+		  (IEEE80211_VHT_MCS_NOT_SUPPORTED << (2 * 2)) |
+		  (IEEE80211_VHT_MCS_NOT_SUPPORTED << (3 * 2)) |
+		  (IEEE80211_VHT_MCS_NOT_SUPPORTED << (4 * 2)) |
+		  (IEEE80211_VHT_MCS_NOT_SUPPORTED << (5 * 2)) |
+		  (IEEE80211_VHT_MCS_NOT_SUPPORTED << (6 * 2)) |
+		  (IEEE80211_VHT_MCS_NOT_SUPPORTED << (7 * 2));
+
+	vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(mcs_map);
+	vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(mcs_map);
+	vht_cap->cap |= IEEE80211_VHT_CAP_RXLDPC |
+			IEEE80211_VHT_CAP_TXSTBC |
+			IEEE80211_VHT_CAP_RXSTBC_1 |
+			IEEE80211_VHT_CAP_SHORT_GI_80;
+
+	return 0;
+}
+
+static int
+mt76_init_sband_2g(struct mt76_dev *dev, struct ieee80211_rate *rates,
+		   int n_rates)
+{
+	dev->hw->wiphy->bands[NL80211_BAND_2GHZ] = &dev->sband_2g.sband;
+
+	return mt76_init_sband(dev, &dev->sband_2g,
+			       mt76_channels_2ghz,
+			       ARRAY_SIZE(mt76_channels_2ghz),
+			       rates, n_rates, false);
+}
+
+static int
+mt76_init_sband_5g(struct mt76_dev *dev, struct ieee80211_rate *rates,
+		   int n_rates, bool vht)
+{
+	dev->hw->wiphy->bands[NL80211_BAND_5GHZ] = &dev->sband_5g.sband;
+
+	return mt76_init_sband(dev, &dev->sband_5g,
+			       mt76_channels_5ghz,
+			       ARRAY_SIZE(mt76_channels_5ghz),
+			       rates, n_rates, vht);
+}
+
+static void
+mt76_check_sband(struct mt76_dev *dev, int band)
+{
+	struct ieee80211_supported_band *sband = dev->hw->wiphy->bands[band];
+	bool found = false;
+	int i;
+
+	if (!sband)
+		return;
+
+	for (i = 0; i < sband->n_channels; i++) {
+		if (sband->channels[i].flags & IEEE80211_CHAN_DISABLED)
+			continue;
+
+		found = true;
+		break;
+	}
+
+	if (found)
+		return;
+
+	sband->n_channels = 0;
+	dev->hw->wiphy->bands[band] = NULL;
+}
+
+int mt76_register_device(struct mt76_dev *dev, bool vht,
+			 struct ieee80211_rate *rates, int n_rates)
+{
+	struct ieee80211_hw *hw = dev->hw;
+	struct wiphy *wiphy = hw->wiphy;
+	int ret;
+
+	dev_set_drvdata(dev->dev, dev);
+
+	spin_lock_init(&dev->lock);
+	spin_lock_init(&dev->cc_lock);
+	INIT_LIST_HEAD(&dev->txwi_cache);
+
+	SET_IEEE80211_DEV(hw, dev->dev);
+	SET_IEEE80211_PERM_ADDR(hw, dev->macaddr);
+
+	wiphy->interface_modes =
+		BIT(NL80211_IFTYPE_STATION) |
+		BIT(NL80211_IFTYPE_AP) |
+#ifdef CONFIG_MAC80211_MESH
+		BIT(NL80211_IFTYPE_MESH_POINT) |
+#endif
+		BIT(NL80211_IFTYPE_ADHOC);
+
+	wiphy->features |= NL80211_FEATURE_ACTIVE_MONITOR;
+
+	hw->txq_data_size = sizeof(struct mt76_txq);
+	hw->max_tx_fragments = 16;
+
+	ieee80211_hw_set(hw, SIGNAL_DBM);
+	ieee80211_hw_set(hw, PS_NULLFUNC_STACK);
+	ieee80211_hw_set(hw, HOST_BROADCAST_PS_BUFFERING);
+	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
+	ieee80211_hw_set(hw, SUPPORTS_RC_TABLE);
+	ieee80211_hw_set(hw, SUPPORT_FAST_XMIT);
+	ieee80211_hw_set(hw, SUPPORTS_CLONED_SKBS);
+	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
+	ieee80211_hw_set(hw, TX_AMSDU);
+	ieee80211_hw_set(hw, TX_FRAG_LIST);
+	ieee80211_hw_set(hw, MFP_CAPABLE);
+	ieee80211_hw_set(hw, AP_LINK_PS);
+
+	wiphy->flags |= WIPHY_FLAG_IBSS_RSN;
+
+	if (dev->cap.has_2ghz) {
+		ret = mt76_init_sband_2g(dev, rates, n_rates);
+		if (ret)
+			return ret;
+	}
+
+	if (dev->cap.has_5ghz) {
+		ret = mt76_init_sband_5g(dev, rates + 4, n_rates - 4, vht);
+		if (ret)
+			return ret;
+	}
+
+	wiphy_read_of_freq_limits(dev->hw->wiphy);
+	mt76_check_sband(dev, NL80211_BAND_2GHZ);
+	mt76_check_sband(dev, NL80211_BAND_5GHZ);
+
+	ret = mt76_led_init(dev);
+	if (ret)
+		return ret;
+
+	return ieee80211_register_hw(hw);
+}
+EXPORT_SYMBOL_GPL(mt76_register_device);
+
+void mt76_unregister_device(struct mt76_dev *dev)
+{
+	struct ieee80211_hw *hw = dev->hw;
+
+	ieee80211_unregister_hw(hw);
+	mt76_tx_free(dev);
+}
+EXPORT_SYMBOL_GPL(mt76_unregister_device);
+
+void mt76_rx(struct mt76_dev *dev, enum mt76_rxq_id q, struct sk_buff *skb)
+{
+	if (!test_bit(MT76_STATE_RUNNING, &dev->state)) {
+		dev_kfree_skb(skb);
+		return;
+	}
+
+	__skb_queue_tail(&dev->rx_skb[q], skb);
+}
+EXPORT_SYMBOL_GPL(mt76_rx);
+
+void mt76_set_channel(struct mt76_dev *dev)
+{
+	struct ieee80211_hw *hw = dev->hw;
+	struct cfg80211_chan_def *chandef = &hw->conf.chandef;
+	struct mt76_channel_state *state;
+	bool offchannel = hw->conf.flags & IEEE80211_CONF_OFFCHANNEL;
+
+	if (dev->drv->update_survey)
+		dev->drv->update_survey(dev);
+
+	dev->chandef = *chandef;
+
+	if (!offchannel)
+		dev->main_chan = chandef->chan;
+
+	if (chandef->chan != dev->main_chan) {
+		state = mt76_channel_state(dev, chandef->chan);
+		memset(state, 0, sizeof(*state));
+	}
+}
+EXPORT_SYMBOL_GPL(mt76_set_channel);
+
+int mt76_get_survey(struct ieee80211_hw *hw, int idx,
+		    struct survey_info *survey)
+{
+	struct mt76_dev *dev = hw->priv;
+	struct mt76_sband *sband;
+	struct ieee80211_channel *chan;
+	struct mt76_channel_state *state;
+	int ret = 0;
+
+	if (idx == 0 && dev->drv->update_survey)
+		dev->drv->update_survey(dev);
+
+	sband = &dev->sband_2g;
+	if (idx >= sband->sband.n_channels) {
+		idx -= sband->sband.n_channels;
+		sband = &dev->sband_5g;
+	}
+
+	if (idx >= sband->sband.n_channels)
+		return -ENOENT;
+
+	chan = &sband->sband.channels[idx];
+	state = mt76_channel_state(dev, chan);
+
+	memset(survey, 0, sizeof(*survey));
+	survey->channel = chan;
+	survey->filled = SURVEY_INFO_TIME | SURVEY_INFO_TIME_BUSY;
+	if (chan == dev->main_chan)
+		survey->filled |= SURVEY_INFO_IN_USE;
+
+	spin_lock_bh(&dev->cc_lock);
+	survey->time = div_u64(state->cc_active, 1000);
+	survey->time_busy = div_u64(state->cc_busy, 1000);
+	spin_unlock_bh(&dev->cc_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(mt76_get_survey);
+
+void mt76_wcid_key_setup(struct mt76_dev *dev, struct mt76_wcid *wcid,
+			 struct ieee80211_key_conf *key)
+{
+	struct ieee80211_key_seq seq;
+	int i;
+
+	wcid->rx_check_pn = false;
+
+	if (!key)
+		return;
+
+	if (key->cipher == WLAN_CIPHER_SUITE_CCMP)
+		wcid->rx_check_pn = true;
+
+	for (i = 0; i < IEEE80211_NUM_TIDS; i++) {
+		ieee80211_get_key_rx_seq(key, i, &seq);
+		memcpy(wcid->rx_key_pn[i], seq.ccmp.pn, sizeof(seq.ccmp.pn));
+	}
+}
+EXPORT_SYMBOL(mt76_wcid_key_setup);
+
+static struct ieee80211_sta *mt76_rx_convert(struct sk_buff *skb)
+{
+	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
+	struct mt76_rx_status mstat;
+
+	mstat = *((struct mt76_rx_status *) skb->cb);
+	memset(status, 0, sizeof(*status));
+
+	status->flag = mstat.flag;
+	status->freq = mstat.freq;
+	status->enc_flags = mstat.enc_flags;
+	status->encoding = mstat.encoding;
+	status->bw = mstat.bw;
+	status->rate_idx = mstat.rate_idx;
+	status->nss = mstat.nss;
+	status->band = mstat.band;
+	status->signal = mstat.signal;
+	status->chains = mstat.chains;
+
+	BUILD_BUG_ON(sizeof(mstat) > sizeof(skb->cb));
+	BUILD_BUG_ON(sizeof(status->chain_signal) != sizeof(mstat.chain_signal));
+	memcpy(status->chain_signal, mstat.chain_signal, sizeof(mstat.chain_signal));
+
+	return wcid_to_sta(mstat.wcid);
+}
+
+static int
+mt76_check_ccmp_pn(struct sk_buff *skb)
+{
+	struct mt76_rx_status *status = (struct mt76_rx_status *) skb->cb;
+	struct mt76_wcid *wcid = status->wcid;
+	struct ieee80211_hdr *hdr;
+	int ret;
+
+	if (!(status->flag & RX_FLAG_DECRYPTED))
+		return 0;
+
+	if (!wcid || !wcid->rx_check_pn)
+		return 0;
+
+	if (!(status->flag & RX_FLAG_IV_STRIPPED)) {
+		/*
+		 * Validate the first fragment both here and in mac80211
+		 * All further fragments will be validated by mac80211 only.
+		 */
+		hdr = (struct ieee80211_hdr *) skb->data;
+		if (ieee80211_is_frag(hdr) &&
+		    !ieee80211_is_first_frag(hdr->frame_control))
+			return 0;
+	}
+
+	BUILD_BUG_ON(sizeof(status->iv) != sizeof(wcid->rx_key_pn[0]));
+	ret = memcmp(status->iv, wcid->rx_key_pn[status->tid],
+		     sizeof(status->iv));
+	if (ret <= 0)
+		return -EINVAL; /* replay */
+
+	memcpy(wcid->rx_key_pn[status->tid], status->iv, sizeof(status->iv));
+
+	if (status->flag & RX_FLAG_IV_STRIPPED)
+		status->flag |= RX_FLAG_PN_VALIDATED;
+
+	return 0;
+}
+
+static void
+mt76_check_ps(struct mt76_dev *dev, struct sk_buff *skb)
+{
+	struct mt76_rx_status *status = (struct mt76_rx_status *) skb->cb;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
+	struct ieee80211_sta *sta;
+	struct mt76_wcid *wcid = status->wcid;
+	bool ps;
+
+	if (!wcid || !wcid->sta)
+		return;
+
+	sta = container_of((void *) wcid, struct ieee80211_sta, drv_priv);
+
+	if (!test_bit(MT_WCID_FLAG_CHECK_PS, &wcid->flags))
+		return;
+
+	if (ieee80211_is_pspoll(hdr->frame_control)) {
+		ieee80211_sta_pspoll(sta);
+		return;
+	}
+
+	if (ieee80211_has_morefrags(hdr->frame_control) ||
+		!(ieee80211_is_mgmt(hdr->frame_control) ||
+		  ieee80211_is_data(hdr->frame_control)))
+		return;
+
+	ps = ieee80211_has_pm(hdr->frame_control);
+
+	if (ps && (ieee80211_is_data_qos(hdr->frame_control) ||
+		   ieee80211_is_qos_nullfunc(hdr->frame_control)))
+		ieee80211_sta_uapsd_trigger(sta, status->tid);
+
+	if (!!test_bit(MT_WCID_FLAG_PS, &wcid->flags) == ps)
+		return;
+
+	if (ps) {
+		set_bit(MT_WCID_FLAG_PS, &wcid->flags);
+		mt76_stop_tx_queues(dev, sta, true);
+	} else {
+		clear_bit(MT_WCID_FLAG_PS, &wcid->flags);
+	}
+
+	ieee80211_sta_ps_transition(sta, ps);
+	dev->drv->sta_ps(dev, sta, ps);
+}
+
+void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
+		      int queue)
+{
+	struct napi_struct *napi = NULL;
+	struct ieee80211_sta *sta;
+	struct sk_buff *skb;
+
+	if (queue >= 0)
+	    napi = &dev->napi[queue];
+
+	while ((skb = __skb_dequeue(frames)) != NULL) {
+		if (mt76_check_ccmp_pn(skb)) {
+			dev_kfree_skb(skb);
+			continue;
+		}
+
+		sta = mt76_rx_convert(skb);
+		ieee80211_rx_napi(dev->hw, sta, skb, napi);
+	}
+}
+
+void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q)
+{
+	struct sk_buff_head frames;
+	struct sk_buff *skb;
+
+	__skb_queue_head_init(&frames);
+
+	while ((skb = __skb_dequeue(&dev->rx_skb[q])) != NULL) {
+		mt76_check_ps(dev, skb);
+		mt76_rx_aggr_reorder(skb, &frames);
+	}
+
+	mt76_rx_complete(dev, &frames, q);
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mmio.c b/drivers/net/wireless/mediatek/mt76/mmio.c
new file mode 100644
index 000000000000..09a14dead6e3
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mmio.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "mt76.h"
+#include "trace.h"
+
+static u32 mt76_mmio_rr(struct mt76_dev *dev, u32 offset)
+{
+	u32 val;
+
+	val = ioread32(dev->regs + offset);
+	trace_reg_rr(dev, offset, val);
+
+	return val;
+}
+
+static void mt76_mmio_wr(struct mt76_dev *dev, u32 offset, u32 val)
+{
+	trace_reg_wr(dev, offset, val);
+	iowrite32(val, dev->regs + offset);
+}
+
+static u32 mt76_mmio_rmw(struct mt76_dev *dev, u32 offset, u32 mask, u32 val)
+{
+	val |= mt76_mmio_rr(dev, offset) & ~mask;
+	mt76_mmio_wr(dev, offset, val);
+	return val;
+}
+
+static void mt76_mmio_copy(struct mt76_dev *dev, u32 offset, const void *data,
+			   int len)
+{
+	__iowrite32_copy(dev->regs + offset, data, len >> 2);
+}
+
+void mt76_mmio_init(struct mt76_dev *dev, void __iomem *regs)
+{
+	static const struct mt76_bus_ops mt76_mmio_ops = {
+		.rr = mt76_mmio_rr,
+		.rmw = mt76_mmio_rmw,
+		.wr = mt76_mmio_wr,
+		.copy = mt76_mmio_copy,
+	};
+
+	dev->bus = &mt76_mmio_ops;
+	dev->regs = regs;
+}
+EXPORT_SYMBOL_GPL(mt76_mmio_init);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h
new file mode 100644
index 000000000000..d2ce15093edd
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76.h
@@ -0,0 +1,442 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76_H
+#define __MT76_H
+
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/skbuff.h>
+#include <linux/leds.h>
+#include <net/mac80211.h>
+#include "util.h"
+
+#define MT_TX_RING_SIZE     256
+#define MT_MCU_RING_SIZE    32
+#define MT_RX_BUF_SIZE      2048
+
+struct mt76_dev;
+
+struct mt76_bus_ops {
+	u32 (*rr)(struct mt76_dev *dev, u32 offset);
+	void (*wr)(struct mt76_dev *dev, u32 offset, u32 val);
+	u32 (*rmw)(struct mt76_dev *dev, u32 offset, u32 mask, u32 val);
+	void (*copy)(struct mt76_dev *dev, u32 offset, const void *data,
+		     int len);
+};
+
+enum mt76_txq_id {
+	MT_TXQ_VO = IEEE80211_AC_VO,
+	MT_TXQ_VI = IEEE80211_AC_VI,
+	MT_TXQ_BE = IEEE80211_AC_BE,
+	MT_TXQ_BK = IEEE80211_AC_BK,
+	MT_TXQ_PSD,
+	MT_TXQ_MCU,
+	MT_TXQ_BEACON,
+	MT_TXQ_CAB,
+	__MT_TXQ_MAX
+};
+
+enum mt76_rxq_id {
+	MT_RXQ_MAIN,
+	MT_RXQ_MCU,
+	__MT_RXQ_MAX
+};
+
+struct mt76_queue_buf {
+	dma_addr_t addr;
+	int len;
+};
+
+struct mt76_queue_entry {
+	union {
+		void *buf;
+		struct sk_buff *skb;
+	};
+	struct mt76_txwi_cache *txwi;
+	bool schedule;
+};
+
+struct mt76_queue_regs {
+	u32 desc_base;
+	u32 ring_size;
+	u32 cpu_idx;
+	u32 dma_idx;
+} __packed __aligned(4);
+
+struct mt76_queue {
+	struct mt76_queue_regs __iomem *regs;
+
+	spinlock_t lock;
+	struct mt76_queue_entry *entry;
+	struct mt76_desc *desc;
+
+	struct list_head swq;
+	int swq_queued;
+
+	u16 head;
+	u16 tail;
+	int ndesc;
+	int queued;
+	int buf_size;
+
+	u8 buf_offset;
+	u8 hw_idx;
+
+	dma_addr_t desc_dma;
+	struct sk_buff *rx_head;
+};
+
+struct mt76_queue_ops {
+	int (*init)(struct mt76_dev *dev);
+
+	int (*alloc)(struct mt76_dev *dev, struct mt76_queue *q);
+
+	int (*add_buf)(struct mt76_dev *dev, struct mt76_queue *q,
+		       struct mt76_queue_buf *buf, int nbufs, u32 info,
+		       struct sk_buff *skb, void *txwi);
+
+	void *(*dequeue)(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
+			 int *len, u32 *info, bool *more);
+
+	void (*rx_reset)(struct mt76_dev *dev, enum mt76_rxq_id qid);
+
+	void (*tx_cleanup)(struct mt76_dev *dev, enum mt76_txq_id qid,
+			   bool flush);
+
+	void (*kick)(struct mt76_dev *dev, struct mt76_queue *q);
+};
+
+enum mt76_wcid_flags {
+	MT_WCID_FLAG_CHECK_PS,
+	MT_WCID_FLAG_PS,
+};
+
+struct mt76_wcid {
+	struct mt76_rx_tid __rcu *aggr[IEEE80211_NUM_TIDS];
+
+	struct work_struct aggr_work;
+
+	unsigned long flags;
+
+	u8 idx;
+	u8 hw_key_idx;
+
+	u8 sta:1;
+
+	u8 rx_check_pn;
+	u8 rx_key_pn[IEEE80211_NUM_TIDS][6];
+
+	__le16 tx_rate;
+	bool tx_rate_set;
+	u8 tx_rate_nss;
+	s8 max_txpwr_adj;
+	bool sw_iv;
+};
+
+struct mt76_txq {
+	struct list_head list;
+	struct mt76_queue *hwq;
+	struct mt76_wcid *wcid;
+
+	struct sk_buff_head retry_q;
+
+	u16 agg_ssn;
+	bool send_bar;
+	bool aggr;
+};
+
+struct mt76_txwi_cache {
+	u32 txwi[8];
+	dma_addr_t dma_addr;
+	struct list_head list;
+};
+
+
+struct mt76_rx_tid {
+	struct rcu_head rcu_head;
+
+	struct mt76_dev *dev;
+
+	spinlock_t lock;
+	struct delayed_work reorder_work;
+
+	u16 head;
+	u8 size;
+	u8 nframes;
+
+	u8 started:1, stopped:1, timer_pending:1;
+
+	struct sk_buff *reorder_buf[];
+};
+
+enum {
+	MT76_STATE_INITIALIZED,
+	MT76_STATE_RUNNING,
+	MT76_SCANNING,
+	MT76_RESET,
+};
+
+struct mt76_hw_cap {
+	bool has_2ghz;
+	bool has_5ghz;
+};
+
+struct mt76_driver_ops {
+	u16 txwi_size;
+
+	void (*update_survey)(struct mt76_dev *dev);
+
+	int (*tx_prepare_skb)(struct mt76_dev *dev, void *txwi_ptr,
+			      struct sk_buff *skb, struct mt76_queue *q,
+			      struct mt76_wcid *wcid,
+			      struct ieee80211_sta *sta, u32 *tx_info);
+
+	void (*tx_complete_skb)(struct mt76_dev *dev, struct mt76_queue *q,
+				struct mt76_queue_entry *e, bool flush);
+
+	void (*rx_skb)(struct mt76_dev *dev, enum mt76_rxq_id q,
+		       struct sk_buff *skb);
+
+	void (*rx_poll_complete)(struct mt76_dev *dev, enum mt76_rxq_id q);
+
+	void (*sta_ps)(struct mt76_dev *dev, struct ieee80211_sta *sta,
+		       bool ps);
+};
+
+struct mt76_channel_state {
+	u64 cc_active;
+	u64 cc_busy;
+};
+
+struct mt76_sband {
+	struct ieee80211_supported_band sband;
+	struct mt76_channel_state *chan;
+};
+
+struct mt76_dev {
+	struct ieee80211_hw *hw;
+	struct cfg80211_chan_def chandef;
+	struct ieee80211_channel *main_chan;
+
+	spinlock_t lock;
+	spinlock_t cc_lock;
+	const struct mt76_bus_ops *bus;
+	const struct mt76_driver_ops *drv;
+	void __iomem *regs;
+	struct device *dev;
+
+	struct net_device napi_dev;
+	struct napi_struct napi[__MT_RXQ_MAX];
+	struct sk_buff_head rx_skb[__MT_RXQ_MAX];
+
+	struct list_head txwi_cache;
+	struct mt76_queue q_tx[__MT_TXQ_MAX];
+	struct mt76_queue q_rx[__MT_RXQ_MAX];
+	const struct mt76_queue_ops *queue_ops;
+
+	u8 macaddr[ETH_ALEN];
+	u32 rev;
+	unsigned long state;
+
+	struct mt76_sband sband_2g;
+	struct mt76_sband sband_5g;
+	struct debugfs_blob_wrapper eeprom;
+	struct debugfs_blob_wrapper otp;
+	struct mt76_hw_cap cap;
+
+	u32 debugfs_reg;
+
+	struct led_classdev led_cdev;
+	char led_name[32];
+	bool led_al;
+	u8 led_pin;
+};
+
+enum mt76_phy_type {
+	MT_PHY_TYPE_CCK,
+	MT_PHY_TYPE_OFDM,
+	MT_PHY_TYPE_HT,
+	MT_PHY_TYPE_HT_GF,
+	MT_PHY_TYPE_VHT,
+};
+
+struct mt76_rate_power {
+	union {
+		struct {
+			s8 cck[4];
+			s8 ofdm[8];
+			s8 ht[16];
+			s8 vht[10];
+		};
+		s8 all[38];
+	};
+};
+
+struct mt76_rx_status {
+	struct mt76_wcid *wcid;
+
+	unsigned long reorder_time;
+
+	u8 iv[6];
+
+	u8 aggr:1;
+	u8 tid;
+	u16 seqno;
+
+	u16 freq;
+	u32 flag;
+	u8 enc_flags;
+	u8 encoding:2, bw:3;
+	u8 rate_idx;
+	u8 nss;
+	u8 band;
+	u8 signal;
+	u8 chains;
+	s8 chain_signal[IEEE80211_MAX_CHAINS];
+};
+
+#define mt76_rr(dev, ...)	(dev)->mt76.bus->rr(&((dev)->mt76), __VA_ARGS__)
+#define mt76_wr(dev, ...)	(dev)->mt76.bus->wr(&((dev)->mt76), __VA_ARGS__)
+#define mt76_rmw(dev, ...)	(dev)->mt76.bus->rmw(&((dev)->mt76), __VA_ARGS__)
+#define mt76_wr_copy(dev, ...)	(dev)->mt76.bus->copy(&((dev)->mt76), __VA_ARGS__)
+
+#define mt76_set(dev, offset, val)	mt76_rmw(dev, offset, 0, val)
+#define mt76_clear(dev, offset, val)	mt76_rmw(dev, offset, val, 0)
+
+#define mt76_get_field(_dev, _reg, _field)		\
+	FIELD_GET(_field, mt76_rr(dev, _reg))
+
+#define mt76_rmw_field(_dev, _reg, _field, _val)	\
+	mt76_rmw(_dev, _reg, _field, FIELD_PREP(_field, _val))
+
+#define mt76_hw(dev) (dev)->mt76.hw
+
+bool __mt76_poll(struct mt76_dev *dev, u32 offset, u32 mask, u32 val,
+		 int timeout);
+
+#define mt76_poll(dev, ...) __mt76_poll(&((dev)->mt76), __VA_ARGS__)
+
+bool __mt76_poll_msec(struct mt76_dev *dev, u32 offset, u32 mask, u32 val,
+		      int timeout);
+
+#define mt76_poll_msec(dev, ...) __mt76_poll_msec(&((dev)->mt76), __VA_ARGS__)
+
+void mt76_mmio_init(struct mt76_dev *dev, void __iomem *regs);
+
+static inline u16 mt76_chip(struct mt76_dev *dev)
+{
+	return dev->rev >> 16;
+}
+
+static inline u16 mt76_rev(struct mt76_dev *dev)
+{
+	return dev->rev & 0xffff;
+}
+
+#define mt76xx_chip(dev) mt76_chip(&((dev)->mt76))
+#define mt76xx_rev(dev) mt76_rev(&((dev)->mt76))
+
+#define mt76_init_queues(dev)		(dev)->mt76.queue_ops->init(&((dev)->mt76))
+#define mt76_queue_alloc(dev, ...)	(dev)->mt76.queue_ops->alloc(&((dev)->mt76), __VA_ARGS__)
+#define mt76_queue_add_buf(dev, ...)	(dev)->mt76.queue_ops->add_buf(&((dev)->mt76), __VA_ARGS__)
+#define mt76_queue_rx_reset(dev, ...)	(dev)->mt76.queue_ops->rx_reset(&((dev)->mt76), __VA_ARGS__)
+#define mt76_queue_tx_cleanup(dev, ...)	(dev)->mt76.queue_ops->tx_cleanup(&((dev)->mt76), __VA_ARGS__)
+#define mt76_queue_kick(dev, ...)	(dev)->mt76.queue_ops->kick(&((dev)->mt76), __VA_ARGS__)
+
+static inline struct mt76_channel_state *
+mt76_channel_state(struct mt76_dev *dev, struct ieee80211_channel *c)
+{
+	struct mt76_sband *msband;
+	int idx;
+
+	if (c->band == NL80211_BAND_2GHZ)
+		msband = &dev->sband_2g;
+	else
+		msband = &dev->sband_5g;
+
+	idx = c - &msband->sband.channels[0];
+	return &msband->chan[idx];
+}
+
+int mt76_register_device(struct mt76_dev *dev, bool vht,
+			 struct ieee80211_rate *rates, int n_rates);
+void mt76_unregister_device(struct mt76_dev *dev);
+
+struct dentry *mt76_register_debugfs(struct mt76_dev *dev);
+
+int mt76_eeprom_init(struct mt76_dev *dev, int len);
+void mt76_eeprom_override(struct mt76_dev *dev);
+
+static inline struct ieee80211_txq *
+mtxq_to_txq(struct mt76_txq *mtxq)
+{
+	void *ptr = mtxq;
+
+	return container_of(ptr, struct ieee80211_txq, drv_priv);
+}
+
+static inline struct ieee80211_sta *
+wcid_to_sta(struct mt76_wcid *wcid)
+{
+	void *ptr = wcid;
+
+	if (!wcid || !wcid->sta)
+		return NULL;
+
+	return container_of(ptr, struct ieee80211_sta, drv_priv);
+}
+
+int mt76_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
+		      struct sk_buff *skb, struct mt76_wcid *wcid,
+		      struct ieee80211_sta *sta);
+
+void mt76_rx(struct mt76_dev *dev, enum mt76_rxq_id q, struct sk_buff *skb);
+void mt76_tx(struct mt76_dev *dev, struct ieee80211_sta *sta,
+	     struct mt76_wcid *wcid, struct sk_buff *skb);
+void mt76_txq_init(struct mt76_dev *dev, struct ieee80211_txq *txq);
+void mt76_txq_remove(struct mt76_dev *dev, struct ieee80211_txq *txq);
+void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq);
+void mt76_stop_tx_queues(struct mt76_dev *dev, struct ieee80211_sta *sta,
+			 bool send_bar);
+void mt76_txq_schedule(struct mt76_dev *dev, struct mt76_queue *hwq);
+void mt76_txq_schedule_all(struct mt76_dev *dev);
+void mt76_release_buffered_frames(struct ieee80211_hw *hw,
+				  struct ieee80211_sta *sta,
+				  u16 tids, int nframes,
+				  enum ieee80211_frame_release_type reason,
+				  bool more_data);
+void mt76_set_channel(struct mt76_dev *dev);
+int mt76_get_survey(struct ieee80211_hw *hw, int idx,
+		    struct survey_info *survey);
+
+int mt76_rx_aggr_start(struct mt76_dev *dev, struct mt76_wcid *wcid, u8 tid,
+		       u16 ssn, u8 size);
+void mt76_rx_aggr_stop(struct mt76_dev *dev, struct mt76_wcid *wcid, u8 tid);
+
+void mt76_wcid_key_setup(struct mt76_dev *dev, struct mt76_wcid *wcid,
+			 struct ieee80211_key_conf *key);
+
+/* internal */
+void mt76_tx_free(struct mt76_dev *dev);
+void mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t);
+void mt76_rx_complete(struct mt76_dev *dev, struct sk_buff_head *frames,
+		      int queue);
+void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q);
+void mt76_rx_aggr_reorder(struct sk_buff *skb, struct sk_buff_head *frames);
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2.h b/drivers/net/wireless/mediatek/mt76/mt76x2.h
new file mode 100644
index 000000000000..e62131b88102
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2.h
@@ -0,0 +1,230 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76x2_H
+#define __MT76x2_H
+
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/spinlock.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <linux/mutex.h>
+#include <linux/bitops.h>
+#include <linux/kfifo.h>
+
+#define MT7662_FIRMWARE		"mt7662.bin"
+#define MT7662_ROM_PATCH	"mt7662_rom_patch.bin"
+#define MT7662_EEPROM_SIZE	512
+
+#define MT76x2_RX_RING_SIZE	256
+#define MT_RX_HEADROOM		32
+
+#define MT_MAX_CHAINS		2
+
+#define MT_CALIBRATE_INTERVAL	HZ
+
+#include "mt76.h"
+#include "mt76x2_regs.h"
+#include "mt76x2_mac.h"
+#include "mt76x2_dfs.h"
+
+struct mt76x2_mcu {
+	struct mutex mutex;
+
+	wait_queue_head_t wait;
+	struct sk_buff_head res_q;
+
+	u32 msg_seq;
+};
+
+struct mt76x2_rx_freq_cal {
+	s8 high_gain[MT_MAX_CHAINS];
+	s8 rssi_offset[MT_MAX_CHAINS];
+	s8 lna_gain;
+	u32 mcu_gain;
+};
+
+struct mt76x2_calibration {
+	struct mt76x2_rx_freq_cal rx;
+
+	u8 agc_gain_init[MT_MAX_CHAINS];
+	u8 agc_gain_cur[MT_MAX_CHAINS];
+
+	int avg_rssi[MT_MAX_CHAINS];
+	int avg_rssi_all;
+
+	s8 agc_gain_adjust;
+	s8 low_gain;
+
+	u8 temp;
+
+	bool init_cal_done;
+	bool tssi_cal_done;
+	bool tssi_comp_pending;
+	bool dpd_cal_done;
+	bool channel_cal_done;
+};
+
+struct mt76x2_dev {
+	struct mt76_dev mt76; /* must be first */
+
+	struct mac_address macaddr_list[8];
+
+	struct mutex mutex;
+
+	const u16 *beacon_offsets;
+	unsigned long wcid_mask[128 / BITS_PER_LONG];
+
+	int txpower_conf;
+	int txpower_cur;
+
+	u8 txdone_seq;
+	DECLARE_KFIFO_PTR(txstatus_fifo, struct mt76x2_tx_status);
+
+	struct mt76x2_mcu mcu;
+	struct sk_buff *rx_head;
+
+	struct tasklet_struct tx_tasklet;
+	struct tasklet_struct pre_tbtt_tasklet;
+	struct delayed_work cal_work;
+	struct delayed_work mac_work;
+
+	u32 aggr_stats[32];
+
+	struct mt76_wcid global_wcid;
+	struct mt76_wcid __rcu *wcid[128];
+
+	spinlock_t irq_lock;
+	u32 irqmask;
+
+	struct sk_buff *beacons[8];
+	u8 beacon_mask;
+	u8 beacon_data_mask;
+
+	u32 rev;
+	u32 rxfilter;
+
+	u16 chainmask;
+
+	struct mt76x2_calibration cal;
+
+	s8 target_power;
+	s8 target_power_delta[2];
+	struct mt76_rate_power rate_power;
+	bool enable_tpc;
+
+	u8 coverage_class;
+	u8 slottime;
+
+	struct mt76x2_dfs_pattern_detector dfs_pd;
+};
+
+struct mt76x2_vif {
+	u8 idx;
+
+	struct mt76_wcid group_wcid;
+};
+
+struct mt76x2_sta {
+	struct mt76_wcid wcid; /* must be first */
+
+	struct mt76x2_vif *vif;
+	struct mt76x2_tx_status status;
+	int n_frames;
+};
+
+static inline bool is_mt7612(struct mt76x2_dev *dev)
+{
+	return (dev->rev >> 16) == 0x7612;
+}
+
+void mt76x2_set_irq_mask(struct mt76x2_dev *dev, u32 clear, u32 set);
+
+static inline void mt76x2_irq_enable(struct mt76x2_dev *dev, u32 mask)
+{
+	mt76x2_set_irq_mask(dev, 0, mask);
+}
+
+static inline void mt76x2_irq_disable(struct mt76x2_dev *dev, u32 mask)
+{
+	mt76x2_set_irq_mask(dev, mask, 0);
+}
+
+extern const struct ieee80211_ops mt76x2_ops;
+
+struct mt76x2_dev *mt76x2_alloc_device(struct device *pdev);
+int mt76x2_register_device(struct mt76x2_dev *dev);
+void mt76x2_init_debugfs(struct mt76x2_dev *dev);
+
+irqreturn_t mt76x2_irq_handler(int irq, void *dev_instance);
+void mt76x2_phy_power_on(struct mt76x2_dev *dev);
+int mt76x2_init_hardware(struct mt76x2_dev *dev);
+void mt76x2_stop_hardware(struct mt76x2_dev *dev);
+int mt76x2_eeprom_init(struct mt76x2_dev *dev);
+int mt76x2_apply_calibration_data(struct mt76x2_dev *dev, int channel);
+void mt76x2_set_tx_ackto(struct mt76x2_dev *dev);
+
+int mt76x2_phy_start(struct mt76x2_dev *dev);
+int mt76x2_phy_set_channel(struct mt76x2_dev *dev,
+			 struct cfg80211_chan_def *chandef);
+int mt76x2_phy_get_rssi(struct mt76x2_dev *dev, s8 rssi, int chain);
+void mt76x2_phy_calibrate(struct work_struct *work);
+void mt76x2_phy_set_txpower(struct mt76x2_dev *dev);
+
+int mt76x2_mcu_init(struct mt76x2_dev *dev);
+int mt76x2_mcu_set_channel(struct mt76x2_dev *dev, u8 channel, u8 bw,
+			   u8 bw_index, bool scan);
+int mt76x2_mcu_set_radio_state(struct mt76x2_dev *dev, bool on);
+int mt76x2_mcu_load_cr(struct mt76x2_dev *dev, u8 type, u8 temp_level,
+		       u8 channel);
+int mt76x2_mcu_cleanup(struct mt76x2_dev *dev);
+
+int mt76x2_dma_init(struct mt76x2_dev *dev);
+void mt76x2_dma_cleanup(struct mt76x2_dev *dev);
+
+void mt76x2_cleanup(struct mt76x2_dev *dev);
+
+int mt76x2_tx_queue_mcu(struct mt76x2_dev *dev, enum mt76_txq_id qid,
+			struct sk_buff *skb, int cmd, int seq);
+void mt76x2_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control,
+	       struct sk_buff *skb);
+void mt76x2_tx_complete(struct mt76x2_dev *dev, struct sk_buff *skb);
+int mt76x2_tx_prepare_skb(struct mt76_dev *mdev, void *txwi,
+			  struct sk_buff *skb, struct mt76_queue *q,
+			  struct mt76_wcid *wcid, struct ieee80211_sta *sta,
+			  u32 *tx_info);
+void mt76x2_tx_complete_skb(struct mt76_dev *mdev, struct mt76_queue *q,
+			    struct mt76_queue_entry *e, bool flush);
+
+void mt76x2_pre_tbtt_tasklet(unsigned long arg);
+
+void mt76x2_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q);
+void mt76x2_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
+			 struct sk_buff *skb);
+
+void mt76x2_sta_ps(struct mt76_dev *dev, struct ieee80211_sta *sta, bool ps);
+
+void mt76x2_update_channel(struct mt76_dev *mdev);
+
+s8 mt76x2_tx_get_max_txpwr_adj(struct mt76x2_dev *dev,
+			       const struct ieee80211_tx_rate *rate);
+s8 mt76x2_tx_get_txpwr_adj(struct mt76x2_dev *dev, s8 txpwr, s8 max_txpwr_adj);
+void mt76x2_tx_set_txpwr_auto(struct mt76x2_dev *dev, s8 txpwr);
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_core.c b/drivers/net/wireless/mediatek/mt76/mt76x2_core.c
new file mode 100644
index 000000000000..2629779e8d3e
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_core.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include "mt76x2.h"
+#include "mt76x2_trace.h"
+
+void mt76x2_set_irq_mask(struct mt76x2_dev *dev, u32 clear, u32 set)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->irq_lock, flags);
+	dev->irqmask &= ~clear;
+	dev->irqmask |= set;
+	mt76_wr(dev, MT_INT_MASK_CSR, dev->irqmask);
+	spin_unlock_irqrestore(&dev->irq_lock, flags);
+}
+
+void mt76x2_rx_poll_complete(struct mt76_dev *mdev, enum mt76_rxq_id q)
+{
+	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
+
+	mt76x2_irq_enable(dev, MT_INT_RX_DONE(q));
+}
+
+irqreturn_t mt76x2_irq_handler(int irq, void *dev_instance)
+{
+	struct mt76x2_dev *dev = dev_instance;
+	u32 intr;
+
+	intr = mt76_rr(dev, MT_INT_SOURCE_CSR);
+	mt76_wr(dev, MT_INT_SOURCE_CSR, intr);
+
+	if (!test_bit(MT76_STATE_INITIALIZED, &dev->mt76.state))
+		return IRQ_NONE;
+
+	trace_dev_irq(dev, intr, dev->irqmask);
+
+	intr &= dev->irqmask;
+
+	if (intr & MT_INT_TX_DONE_ALL) {
+		mt76x2_irq_disable(dev, MT_INT_TX_DONE_ALL);
+		tasklet_schedule(&dev->tx_tasklet);
+	}
+
+	if (intr & MT_INT_RX_DONE(0)) {
+		mt76x2_irq_disable(dev, MT_INT_RX_DONE(0));
+		napi_schedule(&dev->mt76.napi[0]);
+	}
+
+	if (intr & MT_INT_RX_DONE(1)) {
+		mt76x2_irq_disable(dev, MT_INT_RX_DONE(1));
+		napi_schedule(&dev->mt76.napi[1]);
+	}
+
+	if (intr & MT_INT_PRE_TBTT)
+		tasklet_schedule(&dev->pre_tbtt_tasklet);
+
+	/* send buffered multicast frames now */
+	if (intr & MT_INT_TBTT)
+		mt76_queue_kick(dev, &dev->mt76.q_tx[MT_TXQ_PSD]);
+
+	if (intr & MT_INT_TX_STAT) {
+		mt76x2_mac_poll_tx_status(dev, true);
+		tasklet_schedule(&dev->tx_tasklet);
+	}
+
+	if (intr & MT_INT_GPTIMER) {
+		mt76x2_irq_disable(dev, MT_INT_GPTIMER);
+		tasklet_schedule(&dev->dfs_pd.dfs_tasklet);
+	}
+
+	return IRQ_HANDLED;
+}
+
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_debugfs.c b/drivers/net/wireless/mediatek/mt76/mt76x2_debugfs.c
new file mode 100644
index 000000000000..612feb593d7d
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_debugfs.c
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/debugfs.h>
+#include "mt76x2.h"
+
+static int
+mt76x2_ampdu_stat_read(struct seq_file *file, void *data)
+{
+	struct mt76x2_dev *dev = file->private;
+	int i, j;
+
+	for (i = 0; i < 4; i++) {
+		seq_puts(file, "Length: ");
+		for (j = 0; j < 8; j++)
+			seq_printf(file, "%8d | ", i * 8 + j + 1);
+		seq_puts(file, "\n");
+		seq_puts(file, "Count:  ");
+		for (j = 0; j < 8; j++)
+			seq_printf(file, "%8d | ", dev->aggr_stats[i * 8 + j]);
+		seq_puts(file, "\n");
+		seq_puts(file, "--------");
+		for (j = 0; j < 8; j++)
+			seq_puts(file, "-----------");
+		seq_puts(file, "\n");
+	}
+
+	return 0;
+}
+
+static int
+mt76x2_ampdu_stat_open(struct inode *inode, struct file *f)
+{
+	return single_open(f, mt76x2_ampdu_stat_read, inode->i_private);
+}
+
+static void
+seq_puts_array(struct seq_file *file, const char *str, s8 *val, int len)
+{
+	int i;
+
+	seq_printf(file, "%10s:", str);
+	for (i = 0; i < len; i++)
+		seq_printf(file, " %2d", val[i]);
+	seq_puts(file, "\n");
+}
+
+static int read_txpower(struct seq_file *file, void *data)
+{
+	struct mt76x2_dev *dev = dev_get_drvdata(file->private);
+
+	seq_printf(file, "Target power: %d\n", dev->target_power);
+
+	seq_puts_array(file, "Delta", dev->target_power_delta,
+		       ARRAY_SIZE(dev->target_power_delta));
+	seq_puts_array(file, "CCK", dev->rate_power.cck,
+		       ARRAY_SIZE(dev->rate_power.cck));
+	seq_puts_array(file, "OFDM", dev->rate_power.ofdm,
+		       ARRAY_SIZE(dev->rate_power.ofdm));
+	seq_puts_array(file, "HT", dev->rate_power.ht,
+		       ARRAY_SIZE(dev->rate_power.ht));
+	seq_puts_array(file, "VHT", dev->rate_power.vht,
+		       ARRAY_SIZE(dev->rate_power.vht));
+	return 0;
+}
+
+static const struct file_operations fops_ampdu_stat = {
+	.open = mt76x2_ampdu_stat_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static int
+mt76x2_dfs_stat_read(struct seq_file *file, void *data)
+{
+	int i;
+	struct mt76x2_dev *dev = file->private;
+	struct mt76x2_dfs_pattern_detector *dfs_pd = &dev->dfs_pd;
+
+	for (i = 0; i < MT_DFS_NUM_ENGINES; i++) {
+		seq_printf(file, "engine: %d\n", i);
+		seq_printf(file, "  hw pattern detected:\t%d\n",
+			   dfs_pd->stats[i].hw_pattern);
+		seq_printf(file, "  hw pulse discarded:\t%d\n",
+			   dfs_pd->stats[i].hw_pulse_discarded);
+	}
+
+	return 0;
+}
+
+static int
+mt76x2_dfs_stat_open(struct inode *inode, struct file *f)
+{
+	return single_open(f, mt76x2_dfs_stat_read, inode->i_private);
+}
+
+static const struct file_operations fops_dfs_stat = {
+	.open = mt76x2_dfs_stat_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+void mt76x2_init_debugfs(struct mt76x2_dev *dev)
+{
+	struct dentry *dir;
+
+	dir = mt76_register_debugfs(&dev->mt76);
+	if (!dir)
+		return;
+
+	debugfs_create_u8("temperature", S_IRUSR, dir, &dev->cal.temp);
+	debugfs_create_bool("tpc", S_IRUSR | S_IWUSR, dir, &dev->enable_tpc);
+
+	debugfs_create_file("ampdu_stat", S_IRUSR, dir, dev, &fops_ampdu_stat);
+	debugfs_create_file("dfs_stats", S_IRUSR, dir, dev, &fops_dfs_stat);
+	debugfs_create_devm_seqfile(dev->mt76.dev, "txpower", dir,
+				    read_txpower);
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_dfs.c b/drivers/net/wireless/mediatek/mt76/mt76x2_dfs.c
new file mode 100644
index 000000000000..f936dc9a5476
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_dfs.c
@@ -0,0 +1,506 @@
+/*
+ * Copyright (C) 2016 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "mt76x2.h"
+
+#define RADAR_SPEC(m, len, el, eh, wl, wh,		\
+		   w_tolerance, tl, th, t_tolerance,	\
+		   bl, bh, event_exp, power_jmp)	\
+{							\
+	.mode = m,					\
+	.avg_len = len,					\
+	.e_low = el,					\
+	.e_high = eh,					\
+	.w_low = wl,					\
+	.w_high = wh,					\
+	.w_margin = w_tolerance,			\
+	.t_low = tl,					\
+	.t_high = th,					\
+	.t_margin = t_tolerance,			\
+	.b_low = bl,					\
+	.b_high = bh,					\
+	.event_expiration = event_exp,			\
+	.pwr_jmp = power_jmp				\
+}
+
+static const struct mt76x2_radar_specs etsi_radar_specs[] = {
+	/* 20MHz */
+	RADAR_SPEC(0, 8, 2, 15, 106, 150, 10, 4900, 100096, 10, 0,
+		   0x7fffffff, 0x155cc0, 0x19cc),
+	RADAR_SPEC(0, 40, 4, 59, 96, 380, 150, 4900, 100096, 40, 0,
+		   0x7fffffff, 0x155cc0, 0x19cc),
+	RADAR_SPEC(3, 60, 20, 46, 300, 640, 80, 4900, 10100, 80, 0,
+		   0x7fffffff, 0x155cc0, 0x19dd),
+	RADAR_SPEC(8, 8, 2, 9, 106, 150, 32, 4900, 296704, 32, 0,
+		   0x7fffffff, 0x2191c0, 0x15cc),
+	/* 40MHz */
+	RADAR_SPEC(0, 8, 2, 15, 106, 150, 10, 4900, 100096, 10, 0,
+		   0x7fffffff, 0x155cc0, 0x19cc),
+	RADAR_SPEC(0, 40, 4, 59, 96, 380, 150, 4900, 100096, 40, 0,
+		   0x7fffffff, 0x155cc0, 0x19cc),
+	RADAR_SPEC(3, 60, 20, 46, 300, 640, 80, 4900, 10100, 80, 0,
+		   0x7fffffff, 0x155cc0, 0x19dd),
+	RADAR_SPEC(8, 8, 2, 9, 106, 150, 32, 4900, 296704, 32, 0,
+		   0x7fffffff, 0x2191c0, 0x15cc),
+	/* 80MHz */
+	RADAR_SPEC(0, 8, 2, 15, 106, 150, 10, 4900, 100096, 10, 0,
+		   0x7fffffff, 0x155cc0, 0x19cc),
+	RADAR_SPEC(0, 40, 4, 59, 96, 380, 150, 4900, 100096, 40, 0,
+		   0x7fffffff, 0x155cc0, 0x19cc),
+	RADAR_SPEC(3, 60, 20, 46, 300, 640, 80, 4900, 10100, 80, 0,
+		   0x7fffffff, 0x155cc0, 0x19dd),
+	RADAR_SPEC(8, 8, 2, 9, 106, 150, 32, 4900, 296704, 32, 0,
+		   0x7fffffff, 0x2191c0, 0x15cc)
+};
+
+static const struct mt76x2_radar_specs fcc_radar_specs[] = {
+	/* 20MHz */
+	RADAR_SPEC(0, 8, 2, 12, 106, 150, 5, 2900, 80100, 5, 0,
+		   0x7fffffff, 0xfe808, 0x13dc),
+	RADAR_SPEC(0, 8, 2, 7, 106, 140, 5, 27600, 27900, 5, 0,
+		   0x7fffffff, 0xfe808, 0x19dd),
+	RADAR_SPEC(0, 40, 4, 54, 96, 480, 150, 2900, 80100, 40, 0,
+		   0x7fffffff, 0xfe808, 0x12cc),
+	RADAR_SPEC(2, 60, 15, 63, 640, 2080, 32, 19600, 40200, 32, 0,
+		   0x3938700, 0x57bcf00, 0x1289),
+	/* 40MHz */
+	RADAR_SPEC(0, 8, 2, 12, 106, 150, 5, 2900, 80100, 5, 0,
+		   0x7fffffff, 0xfe808, 0x13dc),
+	RADAR_SPEC(0, 8, 2, 7, 106, 140, 5, 27600, 27900, 5, 0,
+		   0x7fffffff, 0xfe808, 0x19dd),
+	RADAR_SPEC(0, 40, 4, 54, 96, 480, 150, 2900, 80100, 40, 0,
+		   0x7fffffff, 0xfe808, 0x12cc),
+	RADAR_SPEC(2, 60, 15, 63, 640, 2080, 32, 19600, 40200, 32, 0,
+		   0x3938700, 0x57bcf00, 0x1289),
+	/* 80MHz */
+	RADAR_SPEC(0, 8, 2, 14, 106, 150, 15, 2900, 80100, 15, 0,
+		   0x7fffffff, 0xfe808, 0x16cc),
+	RADAR_SPEC(0, 8, 2, 7, 106, 140, 5, 27600, 27900, 5, 0,
+		   0x7fffffff, 0xfe808, 0x19dd),
+	RADAR_SPEC(0, 40, 4, 54, 96, 480, 150, 2900, 80100, 40, 0,
+		   0x7fffffff, 0xfe808, 0x12cc),
+	RADAR_SPEC(2, 60, 15, 63, 640, 2080, 32, 19600, 40200, 32, 0,
+		   0x3938700, 0x57bcf00, 0x1289)
+};
+
+static const struct mt76x2_radar_specs jp_w56_radar_specs[] = {
+	/* 20MHz */
+	RADAR_SPEC(0, 8, 2, 7, 106, 150, 5, 2900, 80100, 5, 0,
+		   0x7fffffff, 0x14c080, 0x13dc),
+	RADAR_SPEC(0, 8, 2, 7, 106, 140, 5, 27600, 27900, 5, 0,
+		   0x7fffffff, 0x14c080, 0x19dd),
+	RADAR_SPEC(0, 40, 4, 44, 96, 480, 150, 2900, 80100, 40, 0,
+		   0x7fffffff, 0x14c080, 0x12cc),
+	RADAR_SPEC(2, 60, 15, 48, 940, 2080, 32, 19600, 40200, 32, 0,
+		   0x3938700, 0X57bcf00, 0x1289),
+	/* 40MHz */
+	RADAR_SPEC(0, 8, 2, 7, 106, 150, 5, 2900, 80100, 5, 0,
+		   0x7fffffff, 0x14c080, 0x13dc),
+	RADAR_SPEC(0, 8, 2, 7, 106, 140, 5, 27600, 27900, 5, 0,
+		   0x7fffffff, 0x14c080, 0x19dd),
+	RADAR_SPEC(0, 40, 4, 44, 96, 480, 150, 2900, 80100, 40, 0,
+		   0x7fffffff, 0x14c080, 0x12cc),
+	RADAR_SPEC(2, 60, 15, 48, 940, 2080, 32, 19600, 40200, 32, 0,
+		   0x3938700, 0X57bcf00, 0x1289),
+	/* 80MHz */
+	RADAR_SPEC(0, 8, 2, 9, 106, 150, 15, 2900, 80100, 15, 0,
+		   0x7fffffff, 0x14c080, 0x16cc),
+	RADAR_SPEC(0, 8, 2, 7, 106, 140, 5, 27600, 27900, 5, 0,
+		   0x7fffffff, 0x14c080, 0x19dd),
+	RADAR_SPEC(0, 40, 4, 44, 96, 480, 150, 2900, 80100, 40, 0,
+		   0x7fffffff, 0x14c080, 0x12cc),
+	RADAR_SPEC(2, 60, 15, 48, 940, 2080, 32, 19600, 40200, 32, 0,
+		   0x3938700, 0X57bcf00, 0x1289)
+};
+
+static const struct mt76x2_radar_specs jp_w53_radar_specs[] = {
+	/* 20MHz */
+	RADAR_SPEC(0, 8, 2, 9, 106, 150, 20, 28400, 77000, 20, 0,
+		   0x7fffffff, 0x14c080, 0x16cc),
+	{ 0 },
+	RADAR_SPEC(0, 40, 4, 44, 96, 200, 150, 28400, 77000, 60, 0,
+		   0x7fffffff, 0x14c080, 0x16cc),
+	{ 0 },
+	/* 40MHz */
+	RADAR_SPEC(0, 8, 2, 9, 106, 150, 20, 28400, 77000, 20, 0,
+		   0x7fffffff, 0x14c080, 0x16cc),
+	{ 0 },
+	RADAR_SPEC(0, 40, 4, 44, 96, 200, 150, 28400, 77000, 60, 0,
+		   0x7fffffff, 0x14c080, 0x16cc),
+	{ 0 },
+	/* 80MHz */
+	RADAR_SPEC(0, 8, 2, 9, 106, 150, 20, 28400, 77000, 20, 0,
+		   0x7fffffff, 0x14c080, 0x16cc),
+	{ 0 },
+	RADAR_SPEC(0, 40, 4, 44, 96, 200, 150, 28400, 77000, 60, 0,
+		   0x7fffffff, 0x14c080, 0x16cc),
+	{ 0 }
+};
+
+static void mt76x2_dfs_set_capture_mode_ctrl(struct mt76x2_dev *dev,
+					     u8 enable)
+{
+	u32 data;
+
+	data = (1 << 1) | enable;
+	mt76_wr(dev, MT_BBP(DFS, 36), data);
+}
+
+static bool mt76x2_dfs_check_chirp(struct mt76x2_dev *dev)
+{
+	bool ret = false;
+	u32 current_ts, delta_ts;
+	struct mt76x2_dfs_pattern_detector *dfs_pd = &dev->dfs_pd;
+
+	current_ts = mt76_rr(dev, MT_PBF_LIFE_TIMER);
+	delta_ts = current_ts - dfs_pd->chirp_pulse_ts;
+	dfs_pd->chirp_pulse_ts = current_ts;
+
+	/* 12 sec */
+	if (delta_ts <= (12 * (1 << 20))) {
+		if (++dfs_pd->chirp_pulse_cnt > 8)
+			ret = true;
+	} else {
+		dfs_pd->chirp_pulse_cnt = 1;
+	}
+
+	return ret;
+}
+
+static void mt76x2_dfs_get_hw_pulse(struct mt76x2_dev *dev,
+				    struct mt76x2_dfs_hw_pulse *pulse)
+{
+	u32 data;
+
+	/* select channel */
+	data = (MT_DFS_CH_EN << 16) | pulse->engine;
+	mt76_wr(dev, MT_BBP(DFS, 0), data);
+
+	/* reported period */
+	pulse->period = mt76_rr(dev, MT_BBP(DFS, 19));
+
+	/* reported width */
+	pulse->w1 = mt76_rr(dev, MT_BBP(DFS, 20));
+	pulse->w2 = mt76_rr(dev, MT_BBP(DFS, 23));
+
+	/* reported burst number */
+	pulse->burst = mt76_rr(dev, MT_BBP(DFS, 22));
+}
+
+static bool mt76x2_dfs_check_hw_pulse(struct mt76x2_dev *dev,
+				      struct mt76x2_dfs_hw_pulse *pulse)
+{
+	bool ret = false;
+
+	if (!pulse->period || !pulse->w1)
+		return false;
+
+	switch (dev->dfs_pd.region) {
+	case NL80211_DFS_FCC:
+		if (pulse->engine > 3)
+			break;
+
+		if (pulse->engine == 3) {
+			ret = mt76x2_dfs_check_chirp(dev);
+			break;
+		}
+
+		/* check short pulse*/
+		if (pulse->w1 < 120)
+			ret = (pulse->period >= 2900 &&
+			       (pulse->period <= 4700 ||
+				pulse->period >= 6400) &&
+			       (pulse->period <= 6800 ||
+				pulse->period >= 10200) &&
+			       pulse->period <= 61600);
+		else if (pulse->w1 < 130) /* 120 - 130 */
+			ret = (pulse->period >= 2900 &&
+			       pulse->period <= 61600);
+		else
+			ret = (pulse->period >= 3500 &&
+			       pulse->period <= 10100);
+		break;
+	case NL80211_DFS_ETSI:
+		if (pulse->engine >= 3)
+			break;
+
+		ret = (pulse->period >= 4900 &&
+		       (pulse->period <= 10200 ||
+			pulse->period >= 12400) &&
+		       pulse->period <= 100100);
+		break;
+	case NL80211_DFS_JP:
+		if (dev->mt76.chandef.chan->center_freq >= 5250 &&
+		    dev->mt76.chandef.chan->center_freq <= 5350) {
+			/* JPW53 */
+			if (pulse->w1 <= 130)
+				ret = (pulse->period >= 28360 &&
+				       (pulse->period <= 28700 ||
+					pulse->period >= 76900) &&
+				       pulse->period <= 76940);
+			break;
+		}
+
+		if (pulse->engine > 3)
+			break;
+
+		if (pulse->engine == 3) {
+			ret = mt76x2_dfs_check_chirp(dev);
+			break;
+		}
+
+		/* check short pulse*/
+		if (pulse->w1 < 120)
+			ret = (pulse->period >= 2900 &&
+			       (pulse->period <= 4700 ||
+				pulse->period >= 6400) &&
+			       (pulse->period <= 6800 ||
+				pulse->period >= 27560) &&
+			       (pulse->period <= 27960 ||
+				pulse->period >= 28360) &&
+			       (pulse->period <= 28700 ||
+				pulse->period >= 79900) &&
+			       pulse->period <= 80100);
+		else if (pulse->w1 < 130) /* 120 - 130 */
+			ret = (pulse->period >= 2900 &&
+			       (pulse->period <= 10100 ||
+				pulse->period >= 27560) &&
+			       (pulse->period <= 27960 ||
+				pulse->period >= 28360) &&
+			       (pulse->period <= 28700 ||
+				pulse->period >= 79900) &&
+			       pulse->period <= 80100);
+		else
+			ret = (pulse->period >= 3900 &&
+			       pulse->period <= 10100);
+		break;
+	case NL80211_DFS_UNSET:
+	default:
+		return false;
+	}
+
+	return ret;
+}
+
+static void mt76x2_dfs_tasklet(unsigned long arg)
+{
+	struct mt76x2_dev *dev = (struct mt76x2_dev *)arg;
+	struct mt76x2_dfs_pattern_detector *dfs_pd = &dev->dfs_pd;
+	u32 engine_mask;
+	int i;
+
+	if (test_bit(MT76_SCANNING, &dev->mt76.state))
+		goto out;
+
+	engine_mask = mt76_rr(dev, MT_BBP(DFS, 1));
+	if (!(engine_mask & 0xf))
+		goto out;
+
+	for (i = 0; i < MT_DFS_NUM_ENGINES; i++) {
+		struct mt76x2_dfs_hw_pulse pulse;
+
+		if (!(engine_mask & (1 << i)))
+			continue;
+
+		pulse.engine = i;
+		mt76x2_dfs_get_hw_pulse(dev, &pulse);
+
+		if (!mt76x2_dfs_check_hw_pulse(dev, &pulse)) {
+			dfs_pd->stats[i].hw_pulse_discarded++;
+			continue;
+		}
+
+		/* hw detector rx radar pattern */
+		dfs_pd->stats[i].hw_pattern++;
+		ieee80211_radar_detected(dev->mt76.hw);
+
+		/* reset hw detector */
+		mt76_wr(dev, MT_BBP(DFS, 1), 0xf);
+
+		return;
+	}
+
+	/* reset hw detector */
+	mt76_wr(dev, MT_BBP(DFS, 1), 0xf);
+
+out:
+	mt76x2_irq_enable(dev, MT_INT_GPTIMER);
+}
+
+static void mt76x2_dfs_set_bbp_params(struct mt76x2_dev *dev)
+{
+	u32 data;
+	u8 i, shift;
+	const struct mt76x2_radar_specs *radar_specs;
+
+	switch (dev->mt76.chandef.width) {
+	case NL80211_CHAN_WIDTH_40:
+		shift = MT_DFS_NUM_ENGINES;
+		break;
+	case NL80211_CHAN_WIDTH_80:
+		shift = 2 * MT_DFS_NUM_ENGINES;
+		break;
+	default:
+		shift = 0;
+		break;
+	}
+
+	switch (dev->dfs_pd.region) {
+	case NL80211_DFS_FCC:
+		radar_specs = &fcc_radar_specs[shift];
+		break;
+	case NL80211_DFS_ETSI:
+		radar_specs = &etsi_radar_specs[shift];
+		break;
+	case NL80211_DFS_JP:
+		if (dev->mt76.chandef.chan->center_freq >= 5250 &&
+		    dev->mt76.chandef.chan->center_freq <= 5350)
+			radar_specs = &jp_w53_radar_specs[shift];
+		else
+			radar_specs = &jp_w56_radar_specs[shift];
+		break;
+	case NL80211_DFS_UNSET:
+	default:
+		return;
+	}
+
+	data = (MT_DFS_VGA_MASK << 16) |
+	       (MT_DFS_PWR_GAIN_OFFSET << 12) |
+	       (MT_DFS_PWR_DOWN_TIME << 8) |
+	       (MT_DFS_SYM_ROUND << 4) |
+	       (MT_DFS_DELTA_DELAY & 0xf);
+	mt76_wr(dev, MT_BBP(DFS, 2), data);
+
+	data = (MT_DFS_RX_PE_MASK << 16) | MT_DFS_PKT_END_MASK;
+	mt76_wr(dev, MT_BBP(DFS, 3), data);
+
+	for (i = 0; i < MT_DFS_NUM_ENGINES; i++) {
+		/* configure engine */
+		mt76_wr(dev, MT_BBP(DFS, 0), i);
+
+		/* detection mode + avg_len */
+		data = ((radar_specs[i].avg_len & 0x1ff) << 16) |
+		       (radar_specs[i].mode & 0xf);
+		mt76_wr(dev, MT_BBP(DFS, 4), data);
+
+		/* dfs energy */
+		data = ((radar_specs[i].e_high & 0x0fff) << 16) |
+		       (radar_specs[i].e_low & 0x0fff);
+		mt76_wr(dev, MT_BBP(DFS, 5), data);
+
+		/* dfs period */
+		mt76_wr(dev, MT_BBP(DFS, 7), radar_specs[i].t_low);
+		mt76_wr(dev, MT_BBP(DFS, 9), radar_specs[i].t_high);
+
+		/* dfs burst */
+		mt76_wr(dev, MT_BBP(DFS, 11), radar_specs[i].b_low);
+		mt76_wr(dev, MT_BBP(DFS, 13), radar_specs[i].b_high);
+
+		/* dfs width */
+		data = ((radar_specs[i].w_high & 0x0fff) << 16) |
+		       (radar_specs[i].w_low & 0x0fff);
+		mt76_wr(dev, MT_BBP(DFS, 14), data);
+
+		/* dfs margins */
+		data = (radar_specs[i].w_margin << 16) |
+		       radar_specs[i].t_margin;
+		mt76_wr(dev, MT_BBP(DFS, 15), data);
+
+		/* dfs event expiration */
+		mt76_wr(dev, MT_BBP(DFS, 17), radar_specs[i].event_expiration);
+
+		/* dfs pwr adj */
+		mt76_wr(dev, MT_BBP(DFS, 30), radar_specs[i].pwr_jmp);
+	}
+
+	/* reset status */
+	mt76_wr(dev, MT_BBP(DFS, 1), 0xf);
+	mt76_wr(dev, MT_BBP(DFS, 36), 0x3);
+
+	/* enable detection*/
+	mt76_wr(dev, MT_BBP(DFS, 0), MT_DFS_CH_EN << 16);
+	mt76_wr(dev, 0x212c, 0x0c350001);
+}
+
+void mt76x2_dfs_adjust_agc(struct mt76x2_dev *dev)
+{
+	u32 agc_r8, agc_r4, val_r8, val_r4, dfs_r31;
+
+	agc_r8 = mt76_rr(dev, MT_BBP(AGC, 8));
+	agc_r4 = mt76_rr(dev, MT_BBP(AGC, 4));
+
+	val_r8 = (agc_r8 & 0x00007e00) >> 9;
+	val_r4 = agc_r4 & ~0x1f000000;
+	val_r4 += (((val_r8 + 1) >> 1) << 24);
+	mt76_wr(dev, MT_BBP(AGC, 4), val_r4);
+
+	dfs_r31 = FIELD_GET(MT_BBP_AGC_LNA_HIGH_GAIN, val_r4);
+	dfs_r31 += val_r8;
+	dfs_r31 -= (agc_r8 & 0x00000038) >> 3;
+	dfs_r31 = (dfs_r31 << 16) | 0x00000307;
+	mt76_wr(dev, MT_BBP(DFS, 31), dfs_r31);
+
+	mt76_wr(dev, MT_BBP(DFS, 32), 0x00040071);
+}
+
+void mt76x2_dfs_init_params(struct mt76x2_dev *dev)
+{
+	struct cfg80211_chan_def *chandef = &dev->mt76.chandef;
+
+	if ((chandef->chan->flags & IEEE80211_CHAN_RADAR) &&
+	    dev->dfs_pd.region != NL80211_DFS_UNSET) {
+		mt76x2_dfs_set_bbp_params(dev);
+		/* enable debug mode */
+		mt76x2_dfs_set_capture_mode_ctrl(dev, true);
+
+		mt76x2_irq_enable(dev, MT_INT_GPTIMER);
+		mt76_rmw_field(dev, MT_INT_TIMER_EN,
+			       MT_INT_TIMER_EN_GP_TIMER_EN, 1);
+	} else {
+		/* disable hw detector */
+		mt76_wr(dev, MT_BBP(DFS, 0), 0);
+		/* clear detector status */
+		mt76_wr(dev, MT_BBP(DFS, 1), 0xf);
+		mt76_wr(dev, 0x212c, 0);
+
+		mt76x2_irq_disable(dev, MT_INT_GPTIMER);
+		mt76_rmw_field(dev, MT_INT_TIMER_EN,
+			       MT_INT_TIMER_EN_GP_TIMER_EN, 0);
+	}
+}
+
+void mt76x2_dfs_init_detector(struct mt76x2_dev *dev)
+{
+	struct mt76x2_dfs_pattern_detector *dfs_pd = &dev->dfs_pd;
+
+	dfs_pd->region = NL80211_DFS_UNSET;
+	tasklet_init(&dfs_pd->dfs_tasklet, mt76x2_dfs_tasklet,
+		     (unsigned long)dev);
+}
+
+void mt76x2_dfs_set_domain(struct mt76x2_dev *dev,
+			   enum nl80211_dfs_regions region)
+{
+	struct mt76x2_dfs_pattern_detector *dfs_pd = &dev->dfs_pd;
+
+	if (dfs_pd->region != region) {
+		tasklet_disable(&dfs_pd->dfs_tasklet);
+		dfs_pd->region = region;
+		mt76x2_dfs_init_params(dev);
+		tasklet_enable(&dfs_pd->dfs_tasklet);
+	}
+}
+
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_dfs.h b/drivers/net/wireless/mediatek/mt76/mt76x2_dfs.h
new file mode 100644
index 000000000000..8dbc783cc6bc
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_dfs.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2016 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76x2_DFS_H
+#define __MT76x2_DFS_H
+
+#include <linux/types.h>
+#include <linux/nl80211.h>
+
+#define MT_DFS_GP_INTERVAL		(10 << 4) /* 64 us unit */
+#define MT_DFS_NUM_ENGINES		4
+
+/* bbp params */
+#define MT_DFS_SYM_ROUND		0
+#define MT_DFS_DELTA_DELAY		2
+#define MT_DFS_VGA_MASK			0
+#define MT_DFS_PWR_GAIN_OFFSET		3
+#define MT_DFS_PWR_DOWN_TIME		0xf
+#define MT_DFS_RX_PE_MASK		0xff
+#define MT_DFS_PKT_END_MASK		0
+#define MT_DFS_CH_EN			0xf
+
+struct mt76x2_radar_specs {
+	u8 mode;
+	u16 avg_len;
+	u16 e_low;
+	u16 e_high;
+	u16 w_low;
+	u16 w_high;
+	u16 w_margin;
+	u32 t_low;
+	u32 t_high;
+	u16 t_margin;
+	u32 b_low;
+	u32 b_high;
+	u32 event_expiration;
+	u16 pwr_jmp;
+};
+
+struct mt76x2_dfs_hw_pulse {
+	u8 engine;
+	u32 period;
+	u32 w1;
+	u32 w2;
+	u32 burst;
+};
+
+struct mt76x2_dfs_engine_stats {
+	u32 hw_pattern;
+	u32 hw_pulse_discarded;
+};
+
+struct mt76x2_dfs_pattern_detector {
+	enum nl80211_dfs_regions region;
+
+	u8 chirp_pulse_cnt;
+	u32 chirp_pulse_ts;
+
+	struct mt76x2_dfs_engine_stats stats[MT_DFS_NUM_ENGINES];
+	struct tasklet_struct dfs_tasklet;
+};
+
+void mt76x2_dfs_init_params(struct mt76x2_dev *dev);
+void mt76x2_dfs_init_detector(struct mt76x2_dev *dev);
+void mt76x2_dfs_adjust_agc(struct mt76x2_dev *dev);
+void mt76x2_dfs_set_domain(struct mt76x2_dev *dev,
+			   enum nl80211_dfs_regions region);
+
+#endif /* __MT76x2_DFS_H */
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_dma.c b/drivers/net/wireless/mediatek/mt76/mt76x2_dma.c
new file mode 100644
index 000000000000..fd1ec4743e0b
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_dma.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "mt76x2.h"
+#include "mt76x2_dma.h"
+
+int
+mt76x2_tx_queue_mcu(struct mt76x2_dev *dev, enum mt76_txq_id qid,
+		    struct sk_buff *skb, int cmd, int seq)
+{
+	struct mt76_queue *q = &dev->mt76.q_tx[qid];
+	struct mt76_queue_buf buf;
+	dma_addr_t addr;
+	u32 tx_info;
+
+	tx_info = MT_MCU_MSG_TYPE_CMD |
+		  FIELD_PREP(MT_MCU_MSG_CMD_TYPE, cmd) |
+		  FIELD_PREP(MT_MCU_MSG_CMD_SEQ, seq) |
+		  FIELD_PREP(MT_MCU_MSG_PORT, CPU_TX_PORT) |
+		  FIELD_PREP(MT_MCU_MSG_LEN, skb->len);
+
+	addr = dma_map_single(dev->mt76.dev, skb->data, skb->len,
+			      DMA_TO_DEVICE);
+	if (dma_mapping_error(dev->mt76.dev, addr))
+		return -ENOMEM;
+
+	buf.addr = addr;
+	buf.len = skb->len;
+	spin_lock_bh(&q->lock);
+	mt76_queue_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
+	mt76_queue_kick(dev, q);
+	spin_unlock_bh(&q->lock);
+
+	return 0;
+}
+
+static int
+mt76x2_init_tx_queue(struct mt76x2_dev *dev, struct mt76_queue *q,
+		     int idx, int n_desc)
+{
+	int ret;
+
+	q->regs = dev->mt76.regs + MT_TX_RING_BASE + idx * MT_RING_SIZE;
+	q->ndesc = n_desc;
+	q->hw_idx = idx;
+
+	ret = mt76_queue_alloc(dev, q);
+	if (ret)
+		return ret;
+
+	mt76x2_irq_enable(dev, MT_INT_TX_DONE(idx));
+
+	return 0;
+}
+
+void mt76x2_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
+			 struct sk_buff *skb)
+{
+	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
+	void *rxwi = skb->data;
+
+	if (q == MT_RXQ_MCU) {
+		skb_queue_tail(&dev->mcu.res_q, skb);
+		wake_up(&dev->mcu.wait);
+		return;
+	}
+
+	skb_pull(skb, sizeof(struct mt76x2_rxwi));
+	if (mt76x2_mac_process_rx(dev, skb, rxwi)) {
+		dev_kfree_skb(skb);
+		return;
+	}
+
+	mt76_rx(&dev->mt76, q, skb);
+}
+
+static int
+mt76x2_init_rx_queue(struct mt76x2_dev *dev, struct mt76_queue *q,
+		     int idx, int n_desc, int bufsize)
+{
+	int ret;
+
+	q->regs = dev->mt76.regs + MT_RX_RING_BASE + idx * MT_RING_SIZE;
+	q->ndesc = n_desc;
+	q->buf_size = bufsize;
+
+	ret = mt76_queue_alloc(dev, q);
+	if (ret)
+		return ret;
+
+	mt76x2_irq_enable(dev, MT_INT_RX_DONE(idx));
+
+	return 0;
+}
+
+static void
+mt76x2_tx_tasklet(unsigned long data)
+{
+	struct mt76x2_dev *dev = (struct mt76x2_dev *) data;
+	int i;
+
+	mt76x2_mac_process_tx_status_fifo(dev);
+
+	for (i = MT_TXQ_MCU; i >= 0; i--)
+		mt76_queue_tx_cleanup(dev, i, false);
+
+	mt76x2_mac_poll_tx_status(dev, false);
+	mt76x2_irq_enable(dev, MT_INT_TX_DONE_ALL);
+}
+
+int mt76x2_dma_init(struct mt76x2_dev *dev)
+{
+	static const u8 wmm_queue_map[] = {
+		[IEEE80211_AC_BE] = 0,
+		[IEEE80211_AC_BK] = 1,
+		[IEEE80211_AC_VI] = 2,
+		[IEEE80211_AC_VO] = 3,
+	};
+	int ret;
+	int i;
+	struct mt76_txwi_cache __maybe_unused *t;
+	struct mt76_queue *q;
+
+	BUILD_BUG_ON(sizeof(t->txwi) < sizeof(struct mt76x2_txwi));
+	BUILD_BUG_ON(sizeof(struct mt76x2_rxwi) > MT_RX_HEADROOM);
+
+	mt76_dma_attach(&dev->mt76);
+
+	init_waitqueue_head(&dev->mcu.wait);
+	skb_queue_head_init(&dev->mcu.res_q);
+
+	tasklet_init(&dev->tx_tasklet, mt76x2_tx_tasklet, (unsigned long) dev);
+
+	mt76_wr(dev, MT_WPDMA_RST_IDX, ~0);
+
+	for (i = 0; i < ARRAY_SIZE(wmm_queue_map); i++) {
+		ret = mt76x2_init_tx_queue(dev, &dev->mt76.q_tx[i],
+					   wmm_queue_map[i], MT_TX_RING_SIZE);
+		if (ret)
+			return ret;
+	}
+
+	ret = mt76x2_init_tx_queue(dev, &dev->mt76.q_tx[MT_TXQ_PSD],
+				   MT_TX_HW_QUEUE_MGMT, MT_TX_RING_SIZE);
+	if (ret)
+		return ret;
+
+	ret = mt76x2_init_tx_queue(dev, &dev->mt76.q_tx[MT_TXQ_MCU],
+				   MT_TX_HW_QUEUE_MCU, MT_MCU_RING_SIZE);
+	if (ret)
+		return ret;
+
+	ret = mt76x2_init_rx_queue(dev, &dev->mt76.q_rx[MT_RXQ_MCU], 1,
+				   MT_MCU_RING_SIZE, MT_RX_BUF_SIZE);
+	if (ret)
+		return ret;
+
+	q = &dev->mt76.q_rx[MT_RXQ_MAIN];
+	q->buf_offset = MT_RX_HEADROOM - sizeof(struct mt76x2_rxwi);
+	ret = mt76x2_init_rx_queue(dev, q, 0, MT76x2_RX_RING_SIZE, MT_RX_BUF_SIZE);
+	if (ret)
+		return ret;
+
+	return mt76_init_queues(dev);
+}
+
+void mt76x2_dma_cleanup(struct mt76x2_dev *dev)
+{
+	tasklet_kill(&dev->tx_tasklet);
+	mt76_dma_cleanup(&dev->mt76);
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_dma.h b/drivers/net/wireless/mediatek/mt76/mt76x2_dma.h
new file mode 100644
index 000000000000..47f79d83fcb4
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_dma.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76x2_DMA_H
+#define __MT76x2_DMA_H
+
+#include "dma.h"
+
+#define MT_TXD_INFO_LEN			GENMASK(13, 0)
+#define MT_TXD_INFO_NEXT_VLD		BIT(16)
+#define MT_TXD_INFO_TX_BURST		BIT(17)
+#define MT_TXD_INFO_80211		BIT(19)
+#define MT_TXD_INFO_TSO			BIT(20)
+#define MT_TXD_INFO_CSO			BIT(21)
+#define MT_TXD_INFO_WIV			BIT(24)
+#define MT_TXD_INFO_QSEL		GENMASK(26, 25)
+#define MT_TXD_INFO_TCO			BIT(29)
+#define MT_TXD_INFO_UCO			BIT(30)
+#define MT_TXD_INFO_ICO			BIT(31)
+
+#define MT_RX_FCE_INFO_LEN		GENMASK(13, 0)
+#define MT_RX_FCE_INFO_SELF_GEN		BIT(15)
+#define MT_RX_FCE_INFO_CMD_SEQ		GENMASK(19, 16)
+#define MT_RX_FCE_INFO_EVT_TYPE		GENMASK(23, 20)
+#define MT_RX_FCE_INFO_PCIE_INTR	BIT(24)
+#define MT_RX_FCE_INFO_QSEL		GENMASK(26, 25)
+#define MT_RX_FCE_INFO_D_PORT		GENMASK(29, 27)
+#define MT_RX_FCE_INFO_TYPE		GENMASK(31, 30)
+
+/* MCU request message header  */
+#define MT_MCU_MSG_LEN			GENMASK(15, 0)
+#define MT_MCU_MSG_CMD_SEQ		GENMASK(19, 16)
+#define MT_MCU_MSG_CMD_TYPE		GENMASK(26, 20)
+#define MT_MCU_MSG_PORT			GENMASK(29, 27)
+#define MT_MCU_MSG_TYPE			GENMASK(31, 30)
+#define MT_MCU_MSG_TYPE_CMD		BIT(30)
+
+enum mt76x2_qsel {
+	MT_QSEL_MGMT,
+	MT_QSEL_HCCA,
+	MT_QSEL_EDCA,
+	MT_QSEL_EDCA_2,
+};
+
+enum dma_msg_port {
+	WLAN_PORT,
+	CPU_RX_PORT,
+	CPU_TX_PORT,
+	HOST_PORT,
+	VIRTUAL_CPU_RX_PORT,
+	VIRTUAL_CPU_TX_PORT,
+	DISCARD,
+};
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c
new file mode 100644
index 000000000000..9c9bf3e785ba
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.c
@@ -0,0 +1,664 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <asm/unaligned.h>
+#include "mt76x2.h"
+#include "mt76x2_eeprom.h"
+
+#define EE_FIELD(_name, _value) [MT_EE_##_name] = (_value) | 1
+
+static int
+mt76x2_eeprom_copy(struct mt76x2_dev *dev, enum mt76x2_eeprom_field field,
+		   void *dest, int len)
+{
+	if (field + len > dev->mt76.eeprom.size)
+		return -1;
+
+	memcpy(dest, dev->mt76.eeprom.data + field, len);
+	return 0;
+}
+
+static int
+mt76x2_eeprom_get_macaddr(struct mt76x2_dev *dev)
+{
+	void *src = dev->mt76.eeprom.data + MT_EE_MAC_ADDR;
+
+	memcpy(dev->mt76.macaddr, src, ETH_ALEN);
+	return 0;
+}
+
+static void
+mt76x2_eeprom_parse_hw_cap(struct mt76x2_dev *dev)
+{
+	u16 val = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_0);
+
+	switch (FIELD_GET(MT_EE_NIC_CONF_0_BOARD_TYPE, val)) {
+	case BOARD_TYPE_5GHZ:
+		dev->mt76.cap.has_5ghz = true;
+		break;
+	case BOARD_TYPE_2GHZ:
+		dev->mt76.cap.has_2ghz = true;
+		break;
+	default:
+		dev->mt76.cap.has_2ghz = true;
+		dev->mt76.cap.has_5ghz = true;
+		break;
+	}
+}
+
+static int
+mt76x2_efuse_read(struct mt76x2_dev *dev, u16 addr, u8 *data)
+{
+	u32 val;
+	int i;
+
+	val = mt76_rr(dev, MT_EFUSE_CTRL);
+	val &= ~(MT_EFUSE_CTRL_AIN |
+		 MT_EFUSE_CTRL_MODE);
+	val |= FIELD_PREP(MT_EFUSE_CTRL_AIN, addr & ~0xf);
+	val |= MT_EFUSE_CTRL_KICK;
+	mt76_wr(dev, MT_EFUSE_CTRL, val);
+
+	if (!mt76_poll(dev, MT_EFUSE_CTRL, MT_EFUSE_CTRL_KICK, 0, 1000))
+		return -ETIMEDOUT;
+
+	udelay(2);
+
+	val = mt76_rr(dev, MT_EFUSE_CTRL);
+	if ((val & MT_EFUSE_CTRL_AOUT) == MT_EFUSE_CTRL_AOUT) {
+		memset(data, 0xff, 16);
+		return 0;
+	}
+
+	for (i = 0; i < 4; i++) {
+		val = mt76_rr(dev, MT_EFUSE_DATA(i));
+		put_unaligned_le32(val, data + 4 * i);
+	}
+
+	return 0;
+}
+
+static int
+mt76x2_get_efuse_data(struct mt76x2_dev *dev, void *buf, int len)
+{
+	int ret, i;
+
+	for (i = 0; i + 16 <= len; i += 16) {
+		ret = mt76x2_efuse_read(dev, i, buf + i);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static bool
+mt76x2_has_cal_free_data(struct mt76x2_dev *dev, u8 *efuse)
+{
+	u16 *efuse_w = (u16 *) efuse;
+
+	if (efuse_w[MT_EE_NIC_CONF_0] != 0)
+		return false;
+
+	if (efuse_w[MT_EE_XTAL_TRIM_1] == 0xffff)
+		return false;
+
+	if (efuse_w[MT_EE_TX_POWER_DELTA_BW40] != 0)
+		return false;
+
+	if (efuse_w[MT_EE_TX_POWER_0_START_2G] == 0xffff)
+		return false;
+
+	if (efuse_w[MT_EE_TX_POWER_0_GRP3_TX_POWER_DELTA] != 0)
+		return false;
+
+	if (efuse_w[MT_EE_TX_POWER_0_GRP4_TSSI_SLOPE] == 0xffff)
+		return false;
+
+	return true;
+}
+
+static void
+mt76x2_apply_cal_free_data(struct mt76x2_dev *dev, u8 *efuse)
+{
+#define GROUP_5G(_id)							   \
+	MT_EE_TX_POWER_0_START_5G + MT_TX_POWER_GROUP_SIZE_5G * (_id),	   \
+	MT_EE_TX_POWER_0_START_5G + MT_TX_POWER_GROUP_SIZE_5G * (_id) + 1, \
+	MT_EE_TX_POWER_1_START_5G + MT_TX_POWER_GROUP_SIZE_5G * (_id),	   \
+	MT_EE_TX_POWER_1_START_5G + MT_TX_POWER_GROUP_SIZE_5G * (_id) + 1
+
+	static const u8 cal_free_bytes[] = {
+		MT_EE_XTAL_TRIM_1,
+		MT_EE_TX_POWER_EXT_PA_5G + 1,
+		MT_EE_TX_POWER_0_START_2G,
+		MT_EE_TX_POWER_0_START_2G + 1,
+		MT_EE_TX_POWER_1_START_2G,
+		MT_EE_TX_POWER_1_START_2G + 1,
+		GROUP_5G(0),
+		GROUP_5G(1),
+		GROUP_5G(2),
+		GROUP_5G(3),
+		GROUP_5G(4),
+		GROUP_5G(5),
+		MT_EE_RF_2G_TSSI_OFF_TXPOWER,
+		MT_EE_RF_2G_RX_HIGH_GAIN + 1,
+		MT_EE_RF_5G_GRP0_1_RX_HIGH_GAIN,
+		MT_EE_RF_5G_GRP0_1_RX_HIGH_GAIN + 1,
+		MT_EE_RF_5G_GRP2_3_RX_HIGH_GAIN,
+		MT_EE_RF_5G_GRP2_3_RX_HIGH_GAIN + 1,
+		MT_EE_RF_5G_GRP4_5_RX_HIGH_GAIN,
+		MT_EE_RF_5G_GRP4_5_RX_HIGH_GAIN + 1,
+	};
+	u8 *eeprom = dev->mt76.eeprom.data;
+	u8 prev_grp0[4] = {
+		eeprom[MT_EE_TX_POWER_0_START_5G],
+		eeprom[MT_EE_TX_POWER_0_START_5G + 1],
+		eeprom[MT_EE_TX_POWER_1_START_5G],
+		eeprom[MT_EE_TX_POWER_1_START_5G + 1]
+	};
+	u16 val;
+	int i;
+
+	if (!mt76x2_has_cal_free_data(dev, efuse))
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(cal_free_bytes); i++) {
+		int offset = cal_free_bytes[i];
+
+		eeprom[offset] = efuse[offset];
+	}
+
+	if (!(efuse[MT_EE_TX_POWER_0_START_5G] |
+	      efuse[MT_EE_TX_POWER_0_START_5G + 1]))
+		memcpy(eeprom + MT_EE_TX_POWER_0_START_5G, prev_grp0, 2);
+	if (!(efuse[MT_EE_TX_POWER_1_START_5G] |
+	      efuse[MT_EE_TX_POWER_1_START_5G + 1]))
+		memcpy(eeprom + MT_EE_TX_POWER_1_START_5G, prev_grp0 + 2, 2);
+
+	val = get_unaligned_le16(efuse + MT_EE_BT_RCAL_RESULT);
+	if (val != 0xffff)
+		eeprom[MT_EE_BT_RCAL_RESULT] = val & 0xff;
+
+	val = get_unaligned_le16(efuse + MT_EE_BT_VCDL_CALIBRATION);
+	if (val != 0xffff)
+		eeprom[MT_EE_BT_VCDL_CALIBRATION + 1] = val >> 8;
+
+	val = get_unaligned_le16(efuse + MT_EE_BT_PMUCFG);
+	if (val != 0xffff)
+		eeprom[MT_EE_BT_PMUCFG] = val & 0xff;
+}
+
+static int mt76x2_check_eeprom(struct mt76x2_dev *dev)
+{
+	u16 val = get_unaligned_le16(dev->mt76.eeprom.data);
+
+	if (!val)
+		val = get_unaligned_le16(dev->mt76.eeprom.data + MT_EE_PCI_ID);
+
+	switch (val) {
+	case 0x7662:
+	case 0x7612:
+		return 0;
+	default:
+		dev_err(dev->mt76.dev, "EEPROM data check failed: %04x\n", val);
+		return -EINVAL;
+	}
+}
+
+static int
+mt76x2_eeprom_load(struct mt76x2_dev *dev)
+{
+	void *efuse;
+	int len = MT7662_EEPROM_SIZE;
+	bool found;
+	int ret;
+
+	ret = mt76_eeprom_init(&dev->mt76, len);
+	if (ret < 0)
+		return ret;
+
+	found = ret;
+	if (found)
+		found = !mt76x2_check_eeprom(dev);
+
+	dev->mt76.otp.data = devm_kzalloc(dev->mt76.dev, len, GFP_KERNEL);
+	dev->mt76.otp.size = len;
+	if (!dev->mt76.otp.data)
+		return -ENOMEM;
+
+	efuse = dev->mt76.otp.data;
+
+	if (mt76x2_get_efuse_data(dev, efuse, len))
+		goto out;
+
+	if (found) {
+		mt76x2_apply_cal_free_data(dev, efuse);
+	} else {
+		/* FIXME: check if efuse data is complete */
+		found = true;
+		memcpy(dev->mt76.eeprom.data, efuse, len);
+	}
+
+out:
+	if (!found)
+		return -ENOENT;
+
+	return 0;
+}
+
+static inline int
+mt76x2_sign_extend(u32 val, unsigned int size)
+{
+	bool sign = val & BIT(size - 1);
+
+	val &= BIT(size - 1) - 1;
+
+	return sign ? val : -val;
+}
+
+static inline int
+mt76x2_sign_extend_optional(u32 val, unsigned int size)
+{
+	bool enable = val & BIT(size);
+
+	return enable ? mt76x2_sign_extend(val, size) : 0;
+}
+
+static bool
+field_valid(u8 val)
+{
+	return val != 0 && val != 0xff;
+}
+
+static void
+mt76x2_set_rx_gain_group(struct mt76x2_dev *dev, u8 val)
+{
+	s8 *dest = dev->cal.rx.high_gain;
+
+	if (!field_valid(val)) {
+		dest[0] = 0;
+		dest[1] = 0;
+		return;
+	}
+
+	dest[0] = mt76x2_sign_extend(val, 4);
+	dest[1] = mt76x2_sign_extend(val >> 4, 4);
+}
+
+static void
+mt76x2_set_rssi_offset(struct mt76x2_dev *dev, int chain, u8 val)
+{
+	s8 *dest = dev->cal.rx.rssi_offset;
+
+	if (!field_valid(val)) {
+		dest[chain] = 0;
+		return;
+	}
+
+	dest[chain] = mt76x2_sign_extend_optional(val, 7);
+}
+
+static enum mt76x2_cal_channel_group
+mt76x2_get_cal_channel_group(int channel)
+{
+	if (channel >= 184 && channel <= 196)
+		return MT_CH_5G_JAPAN;
+	if (channel <= 48)
+		return MT_CH_5G_UNII_1;
+	if (channel <= 64)
+		return MT_CH_5G_UNII_2;
+	if (channel <= 114)
+		return MT_CH_5G_UNII_2E_1;
+	if (channel <= 144)
+		return MT_CH_5G_UNII_2E_2;
+	return MT_CH_5G_UNII_3;
+}
+
+static u8
+mt76x2_get_5g_rx_gain(struct mt76x2_dev *dev, u8 channel)
+{
+	enum mt76x2_cal_channel_group group;
+
+	group = mt76x2_get_cal_channel_group(channel);
+	switch (group) {
+	case MT_CH_5G_JAPAN:
+		return mt76x2_eeprom_get(dev, MT_EE_RF_5G_GRP0_1_RX_HIGH_GAIN);
+	case MT_CH_5G_UNII_1:
+		return mt76x2_eeprom_get(dev, MT_EE_RF_5G_GRP0_1_RX_HIGH_GAIN) >> 8;
+	case MT_CH_5G_UNII_2:
+		return mt76x2_eeprom_get(dev, MT_EE_RF_5G_GRP2_3_RX_HIGH_GAIN);
+	case MT_CH_5G_UNII_2E_1:
+		return mt76x2_eeprom_get(dev, MT_EE_RF_5G_GRP2_3_RX_HIGH_GAIN) >> 8;
+	case MT_CH_5G_UNII_2E_2:
+		return mt76x2_eeprom_get(dev, MT_EE_RF_5G_GRP4_5_RX_HIGH_GAIN);
+	default:
+		return mt76x2_eeprom_get(dev, MT_EE_RF_5G_GRP4_5_RX_HIGH_GAIN) >> 8;
+	}
+}
+
+void mt76x2_read_rx_gain(struct mt76x2_dev *dev)
+{
+	struct ieee80211_channel *chan = dev->mt76.chandef.chan;
+	int channel = chan->hw_value;
+	s8 lna_5g[3], lna_2g;
+	u8 lna;
+	u16 val;
+
+	if (chan->band == NL80211_BAND_2GHZ)
+		val = mt76x2_eeprom_get(dev, MT_EE_RF_2G_RX_HIGH_GAIN) >> 8;
+	else
+		val = mt76x2_get_5g_rx_gain(dev, channel);
+
+	mt76x2_set_rx_gain_group(dev, val);
+
+	if (chan->band == NL80211_BAND_2GHZ) {
+		val = mt76x2_eeprom_get(dev, MT_EE_RSSI_OFFSET_2G_0);
+		mt76x2_set_rssi_offset(dev, 0, val);
+		mt76x2_set_rssi_offset(dev, 1, val >> 8);
+	} else {
+		val = mt76x2_eeprom_get(dev, MT_EE_RSSI_OFFSET_5G_0);
+		mt76x2_set_rssi_offset(dev, 0, val);
+		mt76x2_set_rssi_offset(dev, 1, val >> 8);
+	}
+
+	val = mt76x2_eeprom_get(dev, MT_EE_LNA_GAIN);
+	lna_2g = val & 0xff;
+	lna_5g[0] = val >> 8;
+
+	val = mt76x2_eeprom_get(dev, MT_EE_RSSI_OFFSET_2G_1);
+	lna_5g[1] = val >> 8;
+
+	val = mt76x2_eeprom_get(dev, MT_EE_RSSI_OFFSET_5G_1);
+	lna_5g[2] = val >> 8;
+
+	if (!field_valid(lna_5g[1]))
+		lna_5g[1] = lna_5g[0];
+
+	if (!field_valid(lna_5g[2]))
+		lna_5g[2] = lna_5g[0];
+
+	dev->cal.rx.mcu_gain =  (lna_2g & 0xff);
+	dev->cal.rx.mcu_gain |= (lna_5g[0] & 0xff) << 8;
+	dev->cal.rx.mcu_gain |= (lna_5g[1] & 0xff) << 16;
+	dev->cal.rx.mcu_gain |= (lna_5g[2] & 0xff) << 24;
+
+	val = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1);
+	if (val & MT_EE_NIC_CONF_1_LNA_EXT_2G)
+		lna_2g = 0;
+	if (val & MT_EE_NIC_CONF_1_LNA_EXT_5G)
+		memset(lna_5g, 0, sizeof(lna_5g));
+
+	if (chan->band == NL80211_BAND_2GHZ)
+		lna = lna_2g;
+	else if (channel <= 64)
+		lna = lna_5g[0];
+	else if (channel <= 128)
+		lna = lna_5g[1];
+	else
+		lna = lna_5g[2];
+
+	if (lna == 0xff)
+		lna = 0;
+
+	dev->cal.rx.lna_gain = mt76x2_sign_extend(lna, 8);
+}
+
+static s8
+mt76x2_rate_power_val(u8 val)
+{
+	if (!field_valid(val))
+		return 0;
+
+	return mt76x2_sign_extend_optional(val, 7);
+}
+
+void mt76x2_get_rate_power(struct mt76x2_dev *dev, struct mt76_rate_power *t,
+			   struct ieee80211_channel *chan)
+{
+	bool is_5ghz;
+	u16 val;
+
+	is_5ghz = chan->band == NL80211_BAND_5GHZ;
+
+	memset(t, 0, sizeof(*t));
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_CCK);
+	t->cck[0] = t->cck[1] = mt76x2_rate_power_val(val);
+	t->cck[2] = t->cck[3] = mt76x2_rate_power_val(val >> 8);
+
+	if (is_5ghz)
+		val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_OFDM_5G_6M);
+	else
+		val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_OFDM_2G_6M);
+	t->ofdm[0] = t->ofdm[1] = mt76x2_rate_power_val(val);
+	t->ofdm[2] = t->ofdm[3] = mt76x2_rate_power_val(val >> 8);
+
+	if (is_5ghz)
+		val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_OFDM_5G_24M);
+	else
+		val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_OFDM_2G_24M);
+	t->ofdm[4] = t->ofdm[5] = mt76x2_rate_power_val(val);
+	t->ofdm[6] = t->ofdm[7] = mt76x2_rate_power_val(val >> 8);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_HT_MCS0);
+	t->ht[0] = t->ht[1] = mt76x2_rate_power_val(val);
+	t->ht[2] = t->ht[3] = mt76x2_rate_power_val(val >> 8);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_HT_MCS4);
+	t->ht[4] = t->ht[5] = mt76x2_rate_power_val(val);
+	t->ht[6] = t->ht[7] = mt76x2_rate_power_val(val >> 8);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_HT_MCS8);
+	t->ht[8] = t->ht[9] = mt76x2_rate_power_val(val);
+	t->ht[10] = t->ht[11] = mt76x2_rate_power_val(val >> 8);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_HT_MCS12);
+	t->ht[12] = t->ht[13] = mt76x2_rate_power_val(val);
+	t->ht[14] = t->ht[15] = mt76x2_rate_power_val(val >> 8);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_VHT_MCS0);
+	t->vht[0] = t->vht[1] = mt76x2_rate_power_val(val);
+	t->vht[2] = t->vht[3] = mt76x2_rate_power_val(val >> 8);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_VHT_MCS4);
+	t->vht[4] = t->vht[5] = mt76x2_rate_power_val(val);
+	t->vht[6] = t->vht[7] = mt76x2_rate_power_val(val >> 8);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_VHT_MCS8);
+	if (!is_5ghz)
+		val >>= 8;
+	t->vht[8] = t->vht[9] = mt76x2_rate_power_val(val >> 8);
+}
+
+int mt76x2_get_max_rate_power(struct mt76_rate_power *r)
+{
+	int i;
+	s8 ret = 0;
+
+	for (i = 0; i < sizeof(r->all); i++)
+		ret = max(ret, r->all[i]);
+
+	return ret;
+}
+
+static void
+mt76x2_get_power_info_2g(struct mt76x2_dev *dev, struct mt76x2_tx_power_info *t,
+		         struct ieee80211_channel *chan, int chain, int offset)
+{
+	int channel = chan->hw_value;
+	int delta_idx;
+	u8 data[6];
+	u16 val;
+
+	if (channel < 6)
+		delta_idx = 3;
+	else if (channel < 11)
+		delta_idx = 4;
+	else
+		delta_idx = 5;
+
+	mt76x2_eeprom_copy(dev, offset, data, sizeof(data));
+
+	t->chain[chain].tssi_slope = data[0];
+	t->chain[chain].tssi_offset = data[1];
+	t->chain[chain].target_power = data[2];
+	t->chain[chain].delta = mt76x2_sign_extend_optional(data[delta_idx], 7);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_RF_2G_TSSI_OFF_TXPOWER);
+	t->target_power = val >> 8;
+}
+
+static void
+mt76x2_get_power_info_5g(struct mt76x2_dev *dev, struct mt76x2_tx_power_info *t,
+		         struct ieee80211_channel *chan, int chain, int offset)
+{
+	int channel = chan->hw_value;
+	enum mt76x2_cal_channel_group group;
+	int delta_idx;
+	u16 val;
+	u8 data[5];
+
+	group = mt76x2_get_cal_channel_group(channel);
+	offset += group * MT_TX_POWER_GROUP_SIZE_5G;
+
+	if (channel >= 192)
+		delta_idx = 4;
+	else if (channel >= 184)
+		delta_idx = 3;
+	else if (channel < 44)
+		delta_idx = 3;
+	else if (channel < 52)
+		delta_idx = 4;
+	else if (channel < 58)
+		delta_idx = 3;
+	else if (channel < 98)
+		delta_idx = 4;
+	else if (channel < 106)
+		delta_idx = 3;
+	else if (channel < 116)
+		delta_idx = 4;
+	else if (channel < 130)
+		delta_idx = 3;
+	else if (channel < 149)
+		delta_idx = 4;
+	else if (channel < 157)
+		delta_idx = 3;
+	else
+		delta_idx = 4;
+
+	mt76x2_eeprom_copy(dev, offset, data, sizeof(data));
+
+	t->chain[chain].tssi_slope = data[0];
+	t->chain[chain].tssi_offset = data[1];
+	t->chain[chain].target_power = data[2];
+	t->chain[chain].delta = mt76x2_sign_extend_optional(data[delta_idx], 7);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_RF_2G_RX_HIGH_GAIN);
+	t->target_power = val & 0xff;
+}
+
+void mt76x2_get_power_info(struct mt76x2_dev *dev,
+			   struct mt76x2_tx_power_info *t,
+			   struct ieee80211_channel *chan)
+{
+	u16 bw40, bw80;
+
+	memset(t, 0, sizeof(*t));
+
+	bw40 = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_DELTA_BW40);
+	bw80 = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_DELTA_BW80);
+
+	if (chan->band == NL80211_BAND_5GHZ) {
+		bw40 >>= 8;
+		mt76x2_get_power_info_5g(dev, t, chan, 0,
+					 MT_EE_TX_POWER_0_START_5G);
+		mt76x2_get_power_info_5g(dev, t, chan, 1,
+					 MT_EE_TX_POWER_1_START_5G);
+	} else {
+		mt76x2_get_power_info_2g(dev, t, chan, 0,
+					 MT_EE_TX_POWER_0_START_2G);
+		mt76x2_get_power_info_2g(dev, t, chan, 1,
+					 MT_EE_TX_POWER_1_START_2G);
+	}
+
+	if (mt76x2_tssi_enabled(dev) || !field_valid(t->target_power))
+		t->target_power = t->chain[0].target_power;
+
+	t->delta_bw40 = mt76x2_rate_power_val(bw40);
+	t->delta_bw80 = mt76x2_rate_power_val(bw80);
+}
+
+int mt76x2_get_temp_comp(struct mt76x2_dev *dev, struct mt76x2_temp_comp *t)
+{
+	enum nl80211_band band = dev->mt76.chandef.chan->band;
+	u16 val, slope;
+	u8 bounds;
+
+	memset(t, 0, sizeof(*t));
+
+	val = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1);
+	if (!(val & MT_EE_NIC_CONF_1_TEMP_TX_ALC))
+		return -EINVAL;
+
+	if (!mt76x2_ext_pa_enabled(dev, band))
+		return -EINVAL;
+
+	val = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_EXT_PA_5G) >> 8;
+	if (!(val & BIT(7)))
+		return -EINVAL;
+
+	t->temp_25_ref = val & 0x7f;
+	if (band == NL80211_BAND_5GHZ) {
+		slope = mt76x2_eeprom_get(dev, MT_EE_RF_TEMP_COMP_SLOPE_5G);
+		bounds = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_EXT_PA_5G);
+	} else {
+		slope = mt76x2_eeprom_get(dev, MT_EE_RF_TEMP_COMP_SLOPE_2G);
+		bounds = mt76x2_eeprom_get(dev, MT_EE_TX_POWER_DELTA_BW80) >> 8;
+	}
+
+	t->high_slope = slope & 0xff;
+	t->low_slope = slope >> 8;
+	t->lower_bound = 0 - (bounds & 0xf);
+	t->upper_bound = (bounds >> 4) & 0xf;
+
+	return 0;
+}
+
+bool mt76x2_ext_pa_enabled(struct mt76x2_dev *dev, enum nl80211_band band)
+{
+	u16 conf0 = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_0);
+
+	if (band == NL80211_BAND_5GHZ)
+		return !(conf0 & MT_EE_NIC_CONF_0_PA_INT_5G);
+	else
+		return !(conf0 & MT_EE_NIC_CONF_0_PA_INT_2G);
+}
+
+int mt76x2_eeprom_init(struct mt76x2_dev *dev)
+{
+	int ret;
+
+	ret = mt76x2_eeprom_load(dev);
+	if (ret)
+		return ret;
+
+	mt76x2_eeprom_parse_hw_cap(dev);
+	mt76x2_eeprom_get_macaddr(dev);
+	mt76_eeprom_override(&dev->mt76);
+	dev->mt76.macaddr[0] &= ~BIT(1);
+
+	return 0;
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h
new file mode 100644
index 000000000000..d79122728dca
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_eeprom.h
@@ -0,0 +1,185 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76x2_EEPROM_H
+#define __MT76x2_EEPROM_H
+
+#include "mt76x2.h"
+
+enum mt76x2_eeprom_field {
+	MT_EE_CHIP_ID =				0x000,
+	MT_EE_VERSION =				0x002,
+	MT_EE_MAC_ADDR =			0x004,
+	MT_EE_PCI_ID =				0x00A,
+	MT_EE_NIC_CONF_0 =			0x034,
+	MT_EE_NIC_CONF_1 =			0x036,
+	MT_EE_NIC_CONF_2 =			0x042,
+
+	MT_EE_XTAL_TRIM_1 =			0x03a,
+	MT_EE_XTAL_TRIM_2 =			0x09e,
+
+	MT_EE_LNA_GAIN =			0x044,
+	MT_EE_RSSI_OFFSET_2G_0 =		0x046,
+	MT_EE_RSSI_OFFSET_2G_1 =		0x048,
+	MT_EE_RSSI_OFFSET_5G_0 =		0x04a,
+	MT_EE_RSSI_OFFSET_5G_1 =		0x04c,
+
+	MT_EE_TX_POWER_DELTA_BW40 =		0x050,
+	MT_EE_TX_POWER_DELTA_BW80 =		0x052,
+
+	MT_EE_TX_POWER_EXT_PA_5G =		0x054,
+
+	MT_EE_TX_POWER_0_START_2G =		0x056,
+	MT_EE_TX_POWER_1_START_2G =		0x05c,
+
+	/* used as byte arrays */
+#define MT_TX_POWER_GROUP_SIZE_5G		5
+#define MT_TX_POWER_GROUPS_5G			6
+	MT_EE_TX_POWER_0_START_5G =		0x062,
+
+	MT_EE_TX_POWER_0_GRP3_TX_POWER_DELTA =	0x074,
+	MT_EE_TX_POWER_0_GRP4_TSSI_SLOPE =	0x076,
+
+	MT_EE_TX_POWER_1_START_5G =		0x080,
+
+	MT_EE_TX_POWER_CCK =			0x0a0,
+	MT_EE_TX_POWER_OFDM_2G_6M =		0x0a2,
+	MT_EE_TX_POWER_OFDM_2G_24M =		0x0a4,
+	MT_EE_TX_POWER_OFDM_5G_6M =		0x0b2,
+	MT_EE_TX_POWER_OFDM_5G_24M =		0x0b4,
+	MT_EE_TX_POWER_HT_MCS0 =		0x0a6,
+	MT_EE_TX_POWER_HT_MCS4 =		0x0a8,
+	MT_EE_TX_POWER_HT_MCS8 =		0x0aa,
+	MT_EE_TX_POWER_HT_MCS12 =		0x0ac,
+	MT_EE_TX_POWER_VHT_MCS0 =		0x0ba,
+	MT_EE_TX_POWER_VHT_MCS4 =		0x0bc,
+	MT_EE_TX_POWER_VHT_MCS8 =		0x0be,
+
+	MT_EE_RF_TEMP_COMP_SLOPE_5G =		0x0f2,
+	MT_EE_RF_TEMP_COMP_SLOPE_2G =		0x0f4,
+
+	MT_EE_RF_2G_TSSI_OFF_TXPOWER =		0x0f6,
+	MT_EE_RF_2G_RX_HIGH_GAIN =		0x0f8,
+	MT_EE_RF_5G_GRP0_1_RX_HIGH_GAIN =	0x0fa,
+	MT_EE_RF_5G_GRP2_3_RX_HIGH_GAIN =	0x0fc,
+	MT_EE_RF_5G_GRP4_5_RX_HIGH_GAIN =	0x0fe,
+
+	MT_EE_BT_RCAL_RESULT =			0x138,
+	MT_EE_BT_VCDL_CALIBRATION =		0x13c,
+	MT_EE_BT_PMUCFG =			0x13e,
+
+	__MT_EE_MAX
+};
+
+#define MT_EE_NIC_CONF_0_PA_INT_2G		BIT(8)
+#define MT_EE_NIC_CONF_0_PA_INT_5G		BIT(9)
+#define MT_EE_NIC_CONF_0_BOARD_TYPE		GENMASK(13, 12)
+
+#define MT_EE_NIC_CONF_1_TEMP_TX_ALC		BIT(1)
+#define MT_EE_NIC_CONF_1_LNA_EXT_2G		BIT(2)
+#define MT_EE_NIC_CONF_1_LNA_EXT_5G		BIT(3)
+#define MT_EE_NIC_CONF_1_TX_ALC_EN		BIT(13)
+
+#define MT_EE_NIC_CONF_2_RX_STREAM		GENMASK(3, 0)
+#define MT_EE_NIC_CONF_2_TX_STREAM		GENMASK(7, 4)
+#define MT_EE_NIC_CONF_2_HW_ANTDIV		BIT(8)
+#define MT_EE_NIC_CONF_2_XTAL_OPTION		GENMASK(10, 9)
+#define MT_EE_NIC_CONF_2_TEMP_DISABLE		BIT(11)
+#define MT_EE_NIC_CONF_2_COEX_METHOD		GENMASK(15, 13)
+
+enum mt76x2_board_type {
+	BOARD_TYPE_2GHZ = 1,
+	BOARD_TYPE_5GHZ = 2,
+};
+
+enum mt76x2_cal_channel_group {
+	MT_CH_5G_JAPAN,
+	MT_CH_5G_UNII_1,
+	MT_CH_5G_UNII_2,
+	MT_CH_5G_UNII_2E_1,
+	MT_CH_5G_UNII_2E_2,
+	MT_CH_5G_UNII_3,
+	__MT_CH_MAX
+};
+
+struct mt76x2_tx_power_info {
+	u8 target_power;
+
+	s8 delta_bw40;
+	s8 delta_bw80;
+
+	struct {
+		s8 tssi_slope;
+		s8 tssi_offset;
+		s8 target_power;
+		s8 delta;
+	} chain[MT_MAX_CHAINS];
+};
+
+struct mt76x2_temp_comp {
+	u8 temp_25_ref;
+	int lower_bound; /* J */
+	int upper_bound; /* J */
+	unsigned int high_slope; /* J / dB */
+	unsigned int low_slope; /* J / dB */
+};
+
+static inline int
+mt76x2_eeprom_get(struct mt76x2_dev *dev, enum mt76x2_eeprom_field field)
+{
+	if ((field & 1) || field >= __MT_EE_MAX)
+		return -1;
+
+	return get_unaligned_le16(dev->mt76.eeprom.data + field);
+}
+
+void mt76x2_get_rate_power(struct mt76x2_dev *dev, struct mt76_rate_power *t,
+			   struct ieee80211_channel *chan);
+int mt76x2_get_max_rate_power(struct mt76_rate_power *r);
+void mt76x2_get_power_info(struct mt76x2_dev *dev,
+			   struct mt76x2_tx_power_info *t,
+			   struct ieee80211_channel *chan);
+int mt76x2_get_temp_comp(struct mt76x2_dev *dev, struct mt76x2_temp_comp *t);
+bool mt76x2_ext_pa_enabled(struct mt76x2_dev *dev, enum nl80211_band band);
+void mt76x2_read_rx_gain(struct mt76x2_dev *dev);
+
+static inline bool
+mt76x2_temp_tx_alc_enabled(struct mt76x2_dev *dev)
+{
+	return mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1) &
+	       MT_EE_NIC_CONF_1_TEMP_TX_ALC;
+}
+
+static inline bool
+mt76x2_tssi_enabled(struct mt76x2_dev *dev)
+{
+	return !mt76x2_temp_tx_alc_enabled(dev) &&
+	       (mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1) &
+		MT_EE_NIC_CONF_1_TX_ALC_EN);
+}
+
+static inline bool
+mt76x2_has_ext_lna(struct mt76x2_dev *dev)
+{
+	u32 val = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1);
+
+	if (dev->mt76.chandef.chan->band == NL80211_BAND_2GHZ)
+		return val & MT_EE_NIC_CONF_1_LNA_EXT_2G;
+	else
+		return val & MT_EE_NIC_CONF_1_LNA_EXT_5G;
+}
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_init.c b/drivers/net/wireless/mediatek/mt76/mt76x2_init.c
new file mode 100644
index 000000000000..9dbf94947324
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_init.c
@@ -0,0 +1,876 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include "mt76x2.h"
+#include "mt76x2_eeprom.h"
+#include "mt76x2_mcu.h"
+
+struct mt76x2_reg_pair {
+	u32 reg;
+	u32 value;
+};
+
+static bool
+mt76x2_wait_for_mac(struct mt76x2_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < 500; i++) {
+		switch (mt76_rr(dev, MT_MAC_CSR0)) {
+		case 0:
+		case ~0:
+			break;
+		default:
+			return true;
+		}
+		usleep_range(5000, 10000);
+	}
+
+	return false;
+}
+
+static bool
+wait_for_wpdma(struct mt76x2_dev *dev)
+{
+	return mt76_poll(dev, MT_WPDMA_GLO_CFG,
+			 MT_WPDMA_GLO_CFG_TX_DMA_BUSY |
+			 MT_WPDMA_GLO_CFG_RX_DMA_BUSY,
+			 0, 1000);
+}
+
+static void
+mt76x2_mac_pbf_init(struct mt76x2_dev *dev)
+{
+	u32 val;
+
+	val = MT_PBF_SYS_CTRL_MCU_RESET |
+	      MT_PBF_SYS_CTRL_DMA_RESET |
+	      MT_PBF_SYS_CTRL_MAC_RESET |
+	      MT_PBF_SYS_CTRL_PBF_RESET |
+	      MT_PBF_SYS_CTRL_ASY_RESET;
+
+	mt76_set(dev, MT_PBF_SYS_CTRL, val);
+	mt76_clear(dev, MT_PBF_SYS_CTRL, val);
+
+	mt76_wr(dev, MT_PBF_TX_MAX_PCNT, 0xefef3f1f);
+	mt76_wr(dev, MT_PBF_RX_MAX_PCNT, 0xfebf);
+}
+
+static void
+mt76x2_write_reg_pairs(struct mt76x2_dev *dev,
+		       const struct mt76x2_reg_pair *data, int len)
+{
+	while (len > 0) {
+		mt76_wr(dev, data->reg, data->value);
+		len--;
+		data++;
+	}
+}
+
+static void
+mt76_write_mac_initvals(struct mt76x2_dev *dev)
+{
+#define DEFAULT_PROT_CFG				\
+	(FIELD_PREP(MT_PROT_CFG_RATE, 0x2004) |		\
+	 FIELD_PREP(MT_PROT_CFG_NAV, 1) |			\
+	 FIELD_PREP(MT_PROT_CFG_TXOP_ALLOW, 0x3f) |	\
+	 MT_PROT_CFG_RTS_THRESH)
+
+#define DEFAULT_PROT_CFG_20				\
+	(FIELD_PREP(MT_PROT_CFG_RATE, 0x2004) |		\
+	 FIELD_PREP(MT_PROT_CFG_CTRL, 1) |		\
+	 FIELD_PREP(MT_PROT_CFG_NAV, 1) |			\
+	 FIELD_PREP(MT_PROT_CFG_TXOP_ALLOW, 0x17))
+
+#define DEFAULT_PROT_CFG_40				\
+	(FIELD_PREP(MT_PROT_CFG_RATE, 0x2084) |		\
+	 FIELD_PREP(MT_PROT_CFG_CTRL, 1) |		\
+	 FIELD_PREP(MT_PROT_CFG_NAV, 1) |			\
+	 FIELD_PREP(MT_PROT_CFG_TXOP_ALLOW, 0x3f))
+
+	static const struct mt76x2_reg_pair vals[] = {
+		/* Copied from MediaTek reference source */
+		{ MT_PBF_SYS_CTRL,		0x00080c00 },
+		{ MT_PBF_CFG,			0x1efebcff },
+		{ MT_FCE_PSE_CTRL,		0x00000001 },
+		{ MT_MAC_SYS_CTRL,		0x0000000c },
+		{ MT_MAX_LEN_CFG,		0x003e3f00 },
+		{ MT_AMPDU_MAX_LEN_20M1S,	0xaaa99887 },
+		{ MT_AMPDU_MAX_LEN_20M2S,	0x000000aa },
+		{ MT_XIFS_TIME_CFG,		0x33a40d0a },
+		{ MT_BKOFF_SLOT_CFG,		0x00000209 },
+		{ MT_TBTT_SYNC_CFG,		0x00422010 },
+		{ MT_PWR_PIN_CFG,		0x00000000 },
+		{ 0x1238,			0x001700c8 },
+		{ MT_TX_SW_CFG0,		0x00101001 },
+		{ MT_TX_SW_CFG1,		0x00010000 },
+		{ MT_TX_SW_CFG2,		0x00000000 },
+		{ MT_TXOP_CTRL_CFG,		0x0400583f },
+		{ MT_TX_RTS_CFG,		0x00100020 },
+		{ MT_TX_TIMEOUT_CFG,		0x000a2290 },
+		{ MT_TX_RETRY_CFG,		0x47f01f0f },
+		{ MT_EXP_ACK_TIME,		0x002c00dc },
+		{ MT_TX_PROT_CFG6,		0xe3f42004 },
+		{ MT_TX_PROT_CFG7,		0xe3f42084 },
+		{ MT_TX_PROT_CFG8,		0xe3f42104 },
+		{ MT_PIFS_TX_CFG,		0x00060fff },
+		{ MT_RX_FILTR_CFG,		0x00015f97 },
+		{ MT_LEGACY_BASIC_RATE,		0x0000017f },
+		{ MT_HT_BASIC_RATE,		0x00004003 },
+		{ MT_PN_PAD_MODE,		0x00000003 },
+		{ MT_TXOP_HLDR_ET,		0x00000002 },
+		{ 0xa44,			0x00000000 },
+		{ MT_HEADER_TRANS_CTRL_REG,	0x00000000 },
+		{ MT_TSO_CTRL,			0x00000000 },
+		{ MT_AUX_CLK_CFG,		0x00000000 },
+		{ MT_DACCLK_EN_DLY_CFG,		0x00000000 },
+		{ MT_TX_ALC_CFG_4,		0x00000000 },
+		{ MT_TX_ALC_VGA3,		0x00000000 },
+		{ MT_TX_PWR_CFG_0,		0x3a3a3a3a },
+		{ MT_TX_PWR_CFG_1,		0x3a3a3a3a },
+		{ MT_TX_PWR_CFG_2,		0x3a3a3a3a },
+		{ MT_TX_PWR_CFG_3,		0x3a3a3a3a },
+		{ MT_TX_PWR_CFG_4,		0x3a3a3a3a },
+		{ MT_TX_PWR_CFG_7,		0x3a3a3a3a },
+		{ MT_TX_PWR_CFG_8,		0x0000003a },
+		{ MT_TX_PWR_CFG_9,		0x0000003a },
+		{ MT_EFUSE_CTRL,		0x0000d000 },
+		{ MT_PAUSE_ENABLE_CONTROL1,	0x0000000a },
+		{ MT_FCE_WLAN_FLOW_CONTROL1,	0x60401c18 },
+		{ MT_WPDMA_DELAY_INT_CFG,	0x94ff0000 },
+		{ MT_TX_SW_CFG3,		0x00000004 },
+		{ MT_HT_FBK_TO_LEGACY,		0x00001818 },
+		{ MT_VHT_HT_FBK_CFG1,		0xedcba980 },
+		{ MT_PROT_AUTO_TX_CFG,		0x00830083 },
+		{ MT_HT_CTRL_CFG,		0x000001ff },
+	};
+	struct mt76x2_reg_pair prot_vals[] = {
+		{ MT_CCK_PROT_CFG,		DEFAULT_PROT_CFG },
+		{ MT_OFDM_PROT_CFG,		DEFAULT_PROT_CFG },
+		{ MT_MM20_PROT_CFG,		DEFAULT_PROT_CFG_20 },
+		{ MT_MM40_PROT_CFG,		DEFAULT_PROT_CFG_40 },
+		{ MT_GF20_PROT_CFG,		DEFAULT_PROT_CFG_20 },
+		{ MT_GF40_PROT_CFG,		DEFAULT_PROT_CFG_40 },
+	};
+
+	mt76x2_write_reg_pairs(dev, vals, ARRAY_SIZE(vals));
+	mt76x2_write_reg_pairs(dev, prot_vals, ARRAY_SIZE(prot_vals));
+}
+
+static void
+mt76x2_fixup_xtal(struct mt76x2_dev *dev)
+{
+	u16 eep_val;
+	s8 offset = 0;
+
+	eep_val = mt76x2_eeprom_get(dev, MT_EE_XTAL_TRIM_2);
+
+	offset = eep_val & 0x7f;
+	if ((eep_val & 0xff) == 0xff)
+		offset = 0;
+	else if (eep_val & 0x80)
+		offset = 0 - offset;
+
+	eep_val >>= 8;
+	if (eep_val == 0x00 || eep_val == 0xff) {
+		eep_val = mt76x2_eeprom_get(dev, MT_EE_XTAL_TRIM_1);
+		eep_val &= 0xff;
+
+		if (eep_val == 0x00 || eep_val == 0xff)
+			eep_val = 0x14;
+	}
+
+	eep_val &= 0x7f;
+	mt76_rmw_field(dev, MT_XO_CTRL5, MT_XO_CTRL5_C2_VAL, eep_val + offset);
+	mt76_set(dev, MT_XO_CTRL6, MT_XO_CTRL6_C2_CTRL);
+
+	eep_val = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_2);
+	switch (FIELD_GET(MT_EE_NIC_CONF_2_XTAL_OPTION, eep_val)) {
+	case 0:
+		mt76_wr(dev, MT_XO_CTRL7, 0x5c1fee80);
+		break;
+	case 1:
+		mt76_wr(dev, MT_XO_CTRL7, 0x5c1feed0);
+		break;
+	default:
+		break;
+	}
+}
+
+static void
+mt76x2_init_beacon_offsets(struct mt76x2_dev *dev)
+{
+	u16 base = MT_BEACON_BASE;
+	u32 regs[4] = {};
+	int i;
+
+	for (i = 0; i < 16; i++) {
+		u16 addr = dev->beacon_offsets[i];
+
+		regs[i / 4] |= ((addr - base) / 64) << (8 * (i % 4));
+	}
+
+	for (i = 0; i < 4; i++)
+		mt76_wr(dev, MT_BCN_OFFSET(i), regs[i]);
+}
+
+int mt76x2_mac_reset(struct mt76x2_dev *dev, bool hard)
+{
+	static const u8 null_addr[ETH_ALEN] = {};
+	const u8 *macaddr = dev->mt76.macaddr;
+	u32 val;
+	int i, k;
+
+	if (!mt76x2_wait_for_mac(dev))
+		return -ETIMEDOUT;
+
+	val = mt76_rr(dev, MT_WPDMA_GLO_CFG);
+
+	val &= ~(MT_WPDMA_GLO_CFG_TX_DMA_EN |
+		 MT_WPDMA_GLO_CFG_TX_DMA_BUSY |
+		 MT_WPDMA_GLO_CFG_RX_DMA_EN |
+		 MT_WPDMA_GLO_CFG_RX_DMA_BUSY |
+		 MT_WPDMA_GLO_CFG_DMA_BURST_SIZE);
+	val |= FIELD_PREP(MT_WPDMA_GLO_CFG_DMA_BURST_SIZE, 3);
+
+	mt76_wr(dev, MT_WPDMA_GLO_CFG, val);
+
+	mt76x2_mac_pbf_init(dev);
+	mt76_write_mac_initvals(dev);
+	mt76x2_fixup_xtal(dev);
+
+	mt76_clear(dev, MT_MAC_SYS_CTRL,
+		   MT_MAC_SYS_CTRL_RESET_CSR |
+		   MT_MAC_SYS_CTRL_RESET_BBP);
+
+	if (is_mt7612(dev))
+		mt76_clear(dev, MT_COEXCFG0, MT_COEXCFG0_COEX_EN);
+
+	mt76_set(dev, MT_EXT_CCA_CFG, 0x0000f000);
+	mt76_clear(dev, MT_TX_ALC_CFG_4, BIT(31));
+
+	mt76_wr(dev, MT_RF_BYPASS_0, 0x06000000);
+	mt76_wr(dev, MT_RF_SETTING_0, 0x08800000);
+	usleep_range(5000, 10000);
+	mt76_wr(dev, MT_RF_BYPASS_0, 0x00000000);
+
+	mt76_wr(dev, MT_MCU_CLOCK_CTL, 0x1401);
+	mt76_clear(dev, MT_FCE_L2_STUFF, MT_FCE_L2_STUFF_WR_MPDU_LEN_EN);
+
+	mt76_wr(dev, MT_MAC_ADDR_DW0, get_unaligned_le32(macaddr));
+	mt76_wr(dev, MT_MAC_ADDR_DW1, get_unaligned_le16(macaddr + 4));
+
+	mt76_wr(dev, MT_MAC_BSSID_DW0, get_unaligned_le32(macaddr));
+	mt76_wr(dev, MT_MAC_BSSID_DW1, get_unaligned_le16(macaddr + 4) |
+		FIELD_PREP(MT_MAC_BSSID_DW1_MBSS_MODE, 3) | /* 8 beacons */
+		MT_MAC_BSSID_DW1_MBSS_LOCAL_BIT);
+
+	/* Fire a pre-TBTT interrupt 8 ms before TBTT */
+	mt76_rmw_field(dev, MT_INT_TIMER_CFG, MT_INT_TIMER_CFG_PRE_TBTT,
+		       8 << 4);
+	mt76_rmw_field(dev, MT_INT_TIMER_CFG, MT_INT_TIMER_CFG_GP_TIMER,
+		       MT_DFS_GP_INTERVAL);
+	mt76_wr(dev, MT_INT_TIMER_EN, 0);
+
+	mt76_wr(dev, MT_BCN_BYPASS_MASK, 0xffff);
+	if (!hard)
+		return 0;
+
+	for (i = 0; i < 256 / 32; i++)
+		mt76_wr(dev, MT_WCID_DROP_BASE + i * 4, 0);
+
+	for (i = 0; i < 256; i++)
+		mt76x2_mac_wcid_setup(dev, i, 0, NULL);
+
+	for (i = 0; i < 16; i++)
+		for (k = 0; k < 4; k++)
+			mt76x2_mac_shared_key_setup(dev, i, k, NULL);
+
+	for (i = 0; i < 8; i++) {
+		mt76x2_mac_set_bssid(dev, i, null_addr);
+		mt76x2_mac_set_beacon(dev, i, NULL);
+	}
+
+	for (i = 0; i < 16; i++)
+		mt76_rr(dev, MT_TX_STAT_FIFO);
+
+	mt76_wr(dev, MT_CH_TIME_CFG,
+		MT_CH_TIME_CFG_TIMER_EN |
+		MT_CH_TIME_CFG_TX_AS_BUSY |
+		MT_CH_TIME_CFG_RX_AS_BUSY |
+		MT_CH_TIME_CFG_NAV_AS_BUSY |
+		MT_CH_TIME_CFG_EIFS_AS_BUSY |
+		FIELD_PREP(MT_CH_TIME_CFG_CH_TIMER_CLR, 1));
+
+	mt76x2_init_beacon_offsets(dev);
+
+	mt76x2_set_tx_ackto(dev);
+
+	return 0;
+}
+
+int mt76x2_mac_start(struct mt76x2_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < 16; i++)
+		mt76_rr(dev, MT_TX_AGG_CNT(i));
+
+	for (i = 0; i < 16; i++)
+		mt76_rr(dev, MT_TX_STAT_FIFO);
+
+	memset(dev->aggr_stats, 0, sizeof(dev->aggr_stats));
+
+	mt76_wr(dev, MT_MAC_SYS_CTRL, MT_MAC_SYS_CTRL_ENABLE_TX);
+	wait_for_wpdma(dev);
+	usleep_range(50, 100);
+
+	mt76_set(dev, MT_WPDMA_GLO_CFG,
+		 MT_WPDMA_GLO_CFG_TX_DMA_EN |
+		 MT_WPDMA_GLO_CFG_RX_DMA_EN);
+
+	mt76_clear(dev, MT_WPDMA_GLO_CFG, MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE);
+
+	mt76_wr(dev, MT_RX_FILTR_CFG, dev->rxfilter);
+
+	mt76_wr(dev, MT_MAC_SYS_CTRL,
+		MT_MAC_SYS_CTRL_ENABLE_TX |
+		MT_MAC_SYS_CTRL_ENABLE_RX);
+
+	mt76x2_irq_enable(dev, MT_INT_RX_DONE_ALL | MT_INT_TX_DONE_ALL |
+			       MT_INT_TX_STAT);
+
+	return 0;
+}
+
+void mt76x2_mac_stop(struct mt76x2_dev *dev, bool force)
+{
+	bool stopped = false;
+	u32 rts_cfg;
+	int i;
+
+	mt76_wr(dev, MT_MAC_SYS_CTRL, 0);
+
+	rts_cfg = mt76_rr(dev, MT_TX_RTS_CFG);
+	mt76_wr(dev, MT_TX_RTS_CFG, rts_cfg & ~MT_TX_RTS_CFG_RETRY_LIMIT);
+
+	/* Wait for MAC to become idle */
+	for (i = 0; i < 300; i++) {
+		if (mt76_rr(dev, MT_MAC_STATUS) &
+		    (MT_MAC_STATUS_RX | MT_MAC_STATUS_TX))
+			continue;
+
+		if (mt76_rr(dev, MT_BBP(IBI, 12)))
+			continue;
+
+		stopped = true;
+		break;
+	}
+
+	if (force && !stopped) {
+		mt76_set(dev, MT_BBP(CORE, 4), BIT(1));
+		mt76_clear(dev, MT_BBP(CORE, 4), BIT(1));
+
+		mt76_set(dev, MT_BBP(CORE, 4), BIT(0));
+		mt76_clear(dev, MT_BBP(CORE, 4), BIT(0));
+	}
+
+	mt76_wr(dev, MT_TX_RTS_CFG, rts_cfg);
+}
+
+void mt76x2_mac_resume(struct mt76x2_dev *dev)
+{
+	mt76_wr(dev, MT_MAC_SYS_CTRL,
+		MT_MAC_SYS_CTRL_ENABLE_TX |
+		MT_MAC_SYS_CTRL_ENABLE_RX);
+}
+
+static void
+mt76x2_power_on_rf_patch(struct mt76x2_dev *dev)
+{
+	mt76_set(dev, 0x10130, BIT(0) | BIT(16));
+	udelay(1);
+
+	mt76_clear(dev, 0x1001c, 0xff);
+	mt76_set(dev, 0x1001c, 0x30);
+
+	mt76_wr(dev, 0x10014, 0x484f);
+	udelay(1);
+
+	mt76_set(dev, 0x10130, BIT(17));
+	udelay(125);
+
+	mt76_clear(dev, 0x10130, BIT(16));
+	udelay(50);
+
+	mt76_set(dev, 0x1014c, BIT(19) | BIT(20));
+}
+
+static void
+mt76x2_power_on_rf(struct mt76x2_dev *dev, int unit)
+{
+	int shift = unit ? 8 : 0;
+
+	/* Enable RF BG */
+	mt76_set(dev, 0x10130, BIT(0) << shift);
+	udelay(10);
+
+	/* Enable RFDIG LDO/AFE/ABB/ADDA */
+	mt76_set(dev, 0x10130, (BIT(1) | BIT(3) | BIT(4) | BIT(5)) << shift);
+	udelay(10);
+
+	/* Switch RFDIG power to internal LDO */
+	mt76_clear(dev, 0x10130, BIT(2) << shift);
+	udelay(10);
+
+	mt76x2_power_on_rf_patch(dev);
+
+	mt76_set(dev, 0x530, 0xf);
+}
+
+static void
+mt76x2_power_on(struct mt76x2_dev *dev)
+{
+	u32 val;
+
+	/* Turn on WL MTCMOS */
+	mt76_set(dev, MT_WLAN_MTC_CTRL, MT_WLAN_MTC_CTRL_MTCMOS_PWR_UP);
+
+	val = MT_WLAN_MTC_CTRL_STATE_UP |
+	      MT_WLAN_MTC_CTRL_PWR_ACK |
+	      MT_WLAN_MTC_CTRL_PWR_ACK_S;
+
+	mt76_poll(dev, MT_WLAN_MTC_CTRL, val, val, 1000);
+
+	mt76_clear(dev, MT_WLAN_MTC_CTRL, 0x7f << 16);
+	udelay(10);
+
+	mt76_clear(dev, MT_WLAN_MTC_CTRL, 0xf << 24);
+	udelay(10);
+
+	mt76_set(dev, MT_WLAN_MTC_CTRL, 0xf << 24);
+	mt76_clear(dev, MT_WLAN_MTC_CTRL, 0xfff);
+
+	/* Turn on AD/DA power down */
+	mt76_clear(dev, 0x11204, BIT(3));
+
+	/* WLAN function enable */
+	mt76_set(dev, 0x10080, BIT(0));
+
+	/* Release BBP software reset */
+	mt76_clear(dev, 0x10064, BIT(18));
+
+	mt76x2_power_on_rf(dev, 0);
+	mt76x2_power_on_rf(dev, 1);
+}
+
+void mt76x2_set_tx_ackto(struct mt76x2_dev *dev)
+{
+	u8 ackto, sifs, slottime = dev->slottime;
+
+	slottime += 3 * dev->coverage_class;
+
+	sifs = mt76_get_field(dev, MT_XIFS_TIME_CFG,
+			      MT_XIFS_TIME_CFG_OFDM_SIFS);
+
+	ackto = slottime + sifs;
+	mt76_rmw_field(dev, MT_TX_TIMEOUT_CFG,
+		       MT_TX_TIMEOUT_CFG_ACKTO, ackto);
+}
+
+static void
+mt76x2_set_wlan_state(struct mt76x2_dev *dev, bool enable)
+{
+	u32 val = mt76_rr(dev, MT_WLAN_FUN_CTRL);
+
+	if (enable)
+		val |= (MT_WLAN_FUN_CTRL_WLAN_EN |
+			MT_WLAN_FUN_CTRL_WLAN_CLK_EN);
+	else
+		val &= ~(MT_WLAN_FUN_CTRL_WLAN_EN |
+			 MT_WLAN_FUN_CTRL_WLAN_CLK_EN);
+
+	mt76_wr(dev, MT_WLAN_FUN_CTRL, val);
+	udelay(20);
+}
+
+static void
+mt76x2_reset_wlan(struct mt76x2_dev *dev, bool enable)
+{
+	u32 val;
+
+	val = mt76_rr(dev, MT_WLAN_FUN_CTRL);
+
+	val &= ~MT_WLAN_FUN_CTRL_FRC_WL_ANT_SEL;
+
+	if (val & MT_WLAN_FUN_CTRL_WLAN_EN) {
+		val |= MT_WLAN_FUN_CTRL_WLAN_RESET_RF;
+		mt76_wr(dev, MT_WLAN_FUN_CTRL, val);
+		udelay(20);
+
+		val &= ~MT_WLAN_FUN_CTRL_WLAN_RESET_RF;
+	}
+
+	mt76_wr(dev, MT_WLAN_FUN_CTRL, val);
+	udelay(20);
+
+	mt76x2_set_wlan_state(dev, enable);
+}
+
+int mt76x2_init_hardware(struct mt76x2_dev *dev)
+{
+	static const u16 beacon_offsets[16] = {
+		/* 1024 byte per beacon */
+		0xc000,
+		0xc400,
+		0xc800,
+		0xcc00,
+		0xd000,
+		0xd400,
+		0xd800,
+		0xdc00,
+
+		/* BSS idx 8-15 not used for beacons */
+		0xc000,
+		0xc000,
+		0xc000,
+		0xc000,
+		0xc000,
+		0xc000,
+		0xc000,
+		0xc000,
+	};
+	u32 val;
+	int ret;
+
+	dev->beacon_offsets = beacon_offsets;
+	tasklet_init(&dev->pre_tbtt_tasklet, mt76x2_pre_tbtt_tasklet,
+		     (unsigned long) dev);
+
+	dev->chainmask = 0x202;
+	dev->global_wcid.idx = 255;
+	dev->global_wcid.hw_key_idx = -1;
+	dev->slottime = 9;
+
+	val = mt76_rr(dev, MT_WPDMA_GLO_CFG);
+	val &= MT_WPDMA_GLO_CFG_DMA_BURST_SIZE |
+	       MT_WPDMA_GLO_CFG_BIG_ENDIAN |
+	       MT_WPDMA_GLO_CFG_HDR_SEG_LEN;
+	val |= MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE;
+	mt76_wr(dev, MT_WPDMA_GLO_CFG, val);
+
+	mt76x2_reset_wlan(dev, true);
+	mt76x2_power_on(dev);
+
+	ret = mt76x2_eeprom_init(dev);
+	if (ret)
+		return ret;
+
+	ret = mt76x2_mac_reset(dev, true);
+	if (ret)
+		return ret;
+
+	dev->rxfilter = mt76_rr(dev, MT_RX_FILTR_CFG);
+
+	ret = mt76x2_dma_init(dev);
+	if (ret)
+		return ret;
+
+	set_bit(MT76_STATE_INITIALIZED, &dev->mt76.state);
+	ret = mt76x2_mac_start(dev);
+	if (ret)
+		return ret;
+
+	ret = mt76x2_mcu_init(dev);
+	if (ret)
+		return ret;
+
+	mt76x2_mac_stop(dev, false);
+
+	return 0;
+}
+
+void mt76x2_stop_hardware(struct mt76x2_dev *dev)
+{
+	cancel_delayed_work_sync(&dev->cal_work);
+	cancel_delayed_work_sync(&dev->mac_work);
+	mt76x2_mcu_set_radio_state(dev, false);
+	mt76x2_mac_stop(dev, false);
+}
+
+void mt76x2_cleanup(struct mt76x2_dev *dev)
+{
+	tasklet_disable(&dev->dfs_pd.dfs_tasklet);
+	tasklet_disable(&dev->pre_tbtt_tasklet);
+	mt76x2_stop_hardware(dev);
+	mt76x2_dma_cleanup(dev);
+	mt76x2_mcu_cleanup(dev);
+}
+
+struct mt76x2_dev *mt76x2_alloc_device(struct device *pdev)
+{
+	static const struct mt76_driver_ops drv_ops = {
+		.txwi_size = sizeof(struct mt76x2_txwi),
+		.update_survey = mt76x2_update_channel,
+		.tx_prepare_skb = mt76x2_tx_prepare_skb,
+		.tx_complete_skb = mt76x2_tx_complete_skb,
+		.rx_skb = mt76x2_queue_rx_skb,
+		.rx_poll_complete = mt76x2_rx_poll_complete,
+		.sta_ps = mt76x2_sta_ps,
+	};
+	struct ieee80211_hw *hw;
+	struct mt76x2_dev *dev;
+
+	hw = ieee80211_alloc_hw(sizeof(*dev), &mt76x2_ops);
+	if (!hw)
+		return NULL;
+
+	dev = hw->priv;
+	dev->mt76.dev = pdev;
+	dev->mt76.hw = hw;
+	dev->mt76.drv = &drv_ops;
+	mutex_init(&dev->mutex);
+	spin_lock_init(&dev->irq_lock);
+
+	return dev;
+}
+
+static void mt76x2_regd_notifier(struct wiphy *wiphy,
+				 struct regulatory_request *request)
+{
+	struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy);
+	struct mt76x2_dev *dev = hw->priv;
+
+	mt76x2_dfs_set_domain(dev, request->dfs_region);
+}
+
+#define CCK_RATE(_idx, _rate) {					\
+	.bitrate = _rate,					\
+	.flags = IEEE80211_RATE_SHORT_PREAMBLE,			\
+	.hw_value = (MT_PHY_TYPE_CCK << 8) | _idx,		\
+	.hw_value_short = (MT_PHY_TYPE_CCK << 8) | (8 + _idx),	\
+}
+
+#define OFDM_RATE(_idx, _rate) {				\
+	.bitrate = _rate,					\
+	.hw_value = (MT_PHY_TYPE_OFDM << 8) | _idx,		\
+	.hw_value_short = (MT_PHY_TYPE_OFDM << 8) | _idx,	\
+}
+
+static struct ieee80211_rate mt76x2_rates[] = {
+	CCK_RATE(0, 10),
+	CCK_RATE(1, 20),
+	CCK_RATE(2, 55),
+	CCK_RATE(3, 110),
+	OFDM_RATE(0, 60),
+	OFDM_RATE(1, 90),
+	OFDM_RATE(2, 120),
+	OFDM_RATE(3, 180),
+	OFDM_RATE(4, 240),
+	OFDM_RATE(5, 360),
+	OFDM_RATE(6, 480),
+	OFDM_RATE(7, 540),
+};
+
+static const struct ieee80211_iface_limit if_limits[] = {
+	{
+		.max = 1,
+		.types = BIT(NL80211_IFTYPE_ADHOC)
+	}, {
+		.max = 8,
+		.types = BIT(NL80211_IFTYPE_STATION) |
+#ifdef CONFIG_MAC80211_MESH
+			 BIT(NL80211_IFTYPE_MESH_POINT) |
+#endif
+			 BIT(NL80211_IFTYPE_AP)
+	 },
+};
+
+static const struct ieee80211_iface_combination if_comb[] = {
+	{
+		.limits = if_limits,
+		.n_limits = ARRAY_SIZE(if_limits),
+		.max_interfaces = 8,
+		.num_different_channels = 1,
+		.beacon_int_infra_match = true,
+		.radar_detect_widths = BIT(NL80211_CHAN_WIDTH_20_NOHT) |
+				       BIT(NL80211_CHAN_WIDTH_20) |
+				       BIT(NL80211_CHAN_WIDTH_40) |
+				       BIT(NL80211_CHAN_WIDTH_80),
+	}
+};
+
+static void mt76x2_led_set_config(struct mt76_dev *mt76, u8 delay_on,
+				  u8 delay_off)
+{
+	struct mt76x2_dev *dev = container_of(mt76, struct mt76x2_dev,
+					      mt76);
+	u32 val;
+
+	val = MT_LED_STATUS_DURATION(0xff) |
+	      MT_LED_STATUS_OFF(delay_off) |
+	      MT_LED_STATUS_ON(delay_on);
+
+	mt76_wr(dev, MT_LED_S0(mt76->led_pin), val);
+	mt76_wr(dev, MT_LED_S1(mt76->led_pin), val);
+
+	val = MT_LED_CTRL_REPLAY(mt76->led_pin) |
+	      MT_LED_CTRL_KICK(mt76->led_pin);
+	if (mt76->led_al)
+		val |= MT_LED_CTRL_POLARITY(mt76->led_pin);
+	mt76_wr(dev, MT_LED_CTRL, val);
+}
+
+static int mt76x2_led_set_blink(struct led_classdev *led_cdev,
+				unsigned long *delay_on,
+				unsigned long *delay_off)
+{
+	struct mt76_dev *mt76 = container_of(led_cdev, struct mt76_dev,
+					     led_cdev);
+	u8 delta_on, delta_off;
+
+	delta_off = max_t(u8, *delay_off / 10, 1);
+	delta_on = max_t(u8, *delay_on / 10, 1);
+
+	mt76x2_led_set_config(mt76, delta_on, delta_off);
+	return 0;
+}
+
+static void mt76x2_led_set_brightness(struct led_classdev *led_cdev,
+				      enum led_brightness brightness)
+{
+	struct mt76_dev *mt76 = container_of(led_cdev, struct mt76_dev,
+					     led_cdev);
+
+	if (!brightness)
+		mt76x2_led_set_config(mt76, 0, 0xff);
+	else
+		mt76x2_led_set_config(mt76, 0xff, 0);
+}
+
+static void
+mt76x2_init_txpower(struct mt76x2_dev *dev,
+		    struct ieee80211_supported_band *sband)
+{
+	struct ieee80211_channel *chan;
+	struct mt76x2_tx_power_info txp;
+	struct mt76_rate_power t = {};
+	int target_power;
+	int i;
+
+	for (i = 0; i < sband->n_channels; i++) {
+		chan = &sband->channels[i];
+
+		mt76x2_get_power_info(dev, &txp, chan);
+
+		target_power = max_t(int, (txp.chain[0].target_power +
+					   txp.chain[0].delta),
+					  (txp.chain[1].target_power +
+					   txp.chain[1].delta));
+
+		mt76x2_get_rate_power(dev, &t, chan);
+
+		chan->max_power = mt76x2_get_max_rate_power(&t) +
+				  target_power;
+		chan->max_power /= 2;
+
+		/* convert to combined output power on 2x2 devices */
+		chan->max_power += 3;
+	}
+}
+
+int mt76x2_register_device(struct mt76x2_dev *dev)
+{
+	struct ieee80211_hw *hw = mt76_hw(dev);
+	struct wiphy *wiphy = hw->wiphy;
+	void *status_fifo;
+	int fifo_size;
+	int i, ret;
+
+	fifo_size = roundup_pow_of_two(32 * sizeof(struct mt76x2_tx_status));
+	status_fifo = devm_kzalloc(dev->mt76.dev, fifo_size, GFP_KERNEL);
+	if (!status_fifo)
+		return -ENOMEM;
+
+	kfifo_init(&dev->txstatus_fifo, status_fifo, fifo_size);
+
+	ret = mt76x2_init_hardware(dev);
+	if (ret)
+		return ret;
+
+	hw->queues = 4;
+	hw->max_rates = 1;
+	hw->max_report_rates = 7;
+	hw->max_rate_tries = 1;
+	hw->extra_tx_headroom = 2;
+
+	hw->sta_data_size = sizeof(struct mt76x2_sta);
+	hw->vif_data_size = sizeof(struct mt76x2_vif);
+
+	for (i = 0; i < ARRAY_SIZE(dev->macaddr_list); i++) {
+		u8 *addr = dev->macaddr_list[i].addr;
+
+		memcpy(addr, dev->mt76.macaddr, ETH_ALEN);
+
+		if (!i)
+			continue;
+
+		addr[0] |= BIT(1);
+		addr[0] ^= ((i - 1) << 2);
+	}
+	wiphy->addresses = dev->macaddr_list;
+	wiphy->n_addresses = ARRAY_SIZE(dev->macaddr_list);
+
+	wiphy->iface_combinations = if_comb;
+	wiphy->n_iface_combinations = ARRAY_SIZE(if_comb);
+
+	wiphy->reg_notifier = mt76x2_regd_notifier;
+
+	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
+
+	ieee80211_hw_set(hw, SUPPORTS_HT_CCK_RATES);
+	ieee80211_hw_set(hw, SUPPORTS_REORDERING_BUFFER);
+
+	INIT_DELAYED_WORK(&dev->cal_work, mt76x2_phy_calibrate);
+	INIT_DELAYED_WORK(&dev->mac_work, mt76x2_mac_work);
+
+	dev->mt76.sband_2g.sband.ht_cap.cap |= IEEE80211_HT_CAP_LDPC_CODING;
+	dev->mt76.sband_5g.sband.ht_cap.cap |= IEEE80211_HT_CAP_LDPC_CODING;
+
+	mt76x2_dfs_init_detector(dev);
+
+	/* init led callbacks */
+	dev->mt76.led_cdev.brightness_set = mt76x2_led_set_brightness;
+	dev->mt76.led_cdev.blink_set = mt76x2_led_set_blink;
+
+	ret = mt76_register_device(&dev->mt76, true, mt76x2_rates,
+				   ARRAY_SIZE(mt76x2_rates));
+	if (ret)
+		goto fail;
+
+	mt76x2_init_debugfs(dev);
+	mt76x2_init_txpower(dev, &dev->mt76.sband_2g.sband);
+	mt76x2_init_txpower(dev, &dev->mt76.sband_5g.sband);
+
+	return 0;
+
+fail:
+	mt76x2_stop_hardware(dev);
+	return ret;
+}
+
+
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.c
new file mode 100644
index 000000000000..7ea3d841918e
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.c
@@ -0,0 +1,839 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include "mt76x2.h"
+#include "mt76x2_mcu.h"
+#include "mt76x2_eeprom.h"
+#include "mt76x2_trace.h"
+
+void mt76x2_mac_set_bssid(struct mt76x2_dev *dev, u8 idx, const u8 *addr)
+{
+	idx &= 7;
+	mt76_wr(dev, MT_MAC_APC_BSSID_L(idx), get_unaligned_le32(addr));
+	mt76_rmw_field(dev, MT_MAC_APC_BSSID_H(idx), MT_MAC_APC_BSSID_H_ADDR,
+		       get_unaligned_le16(addr + 4));
+}
+
+static int
+mt76x2_mac_process_rate(struct mt76_rx_status *status, u16 rate)
+{
+	u8 idx = FIELD_GET(MT_RXWI_RATE_INDEX, rate);
+
+	switch (FIELD_GET(MT_RXWI_RATE_PHY, rate)) {
+	case MT_PHY_TYPE_OFDM:
+		if (idx >= 8)
+			idx = 0;
+
+		if (status->band == NL80211_BAND_2GHZ)
+			idx += 4;
+
+		status->rate_idx = idx;
+		return 0;
+	case MT_PHY_TYPE_CCK:
+		if (idx >= 8) {
+			idx -= 8;
+			status->enc_flags |= RX_ENC_FLAG_SHORTPRE;
+		}
+
+		if (idx >= 4)
+			idx = 0;
+
+		status->rate_idx = idx;
+		return 0;
+	case MT_PHY_TYPE_HT_GF:
+		status->enc_flags |= RX_ENC_FLAG_HT_GF;
+		/* fall through */
+	case MT_PHY_TYPE_HT:
+		status->encoding = RX_ENC_HT;
+		status->rate_idx = idx;
+		break;
+	case MT_PHY_TYPE_VHT:
+		status->encoding = RX_ENC_VHT;
+		status->rate_idx = FIELD_GET(MT_RATE_INDEX_VHT_IDX, idx);
+		status->nss = FIELD_GET(MT_RATE_INDEX_VHT_NSS, idx) + 1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (rate & MT_RXWI_RATE_LDPC)
+		status->enc_flags |= RX_ENC_FLAG_LDPC;
+
+	if (rate & MT_RXWI_RATE_SGI)
+		status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
+
+	if (rate & MT_RXWI_RATE_STBC)
+		status->enc_flags |= 1 << RX_ENC_FLAG_STBC_SHIFT;
+
+	switch (FIELD_GET(MT_RXWI_RATE_BW, rate)) {
+	case MT_PHY_BW_20:
+		break;
+	case MT_PHY_BW_40:
+		status->bw = RATE_INFO_BW_40;
+		break;
+	case MT_PHY_BW_80:
+		status->bw = RATE_INFO_BW_80;
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static __le16
+mt76x2_mac_tx_rate_val(struct mt76x2_dev *dev,
+		       const struct ieee80211_tx_rate *rate, u8 *nss_val)
+{
+	u16 rateval;
+	u8 phy, rate_idx;
+	u8 nss = 1;
+	u8 bw = 0;
+
+	if (rate->flags & IEEE80211_TX_RC_VHT_MCS) {
+		rate_idx = rate->idx;
+		nss = 1 + (rate->idx >> 4);
+		phy = MT_PHY_TYPE_VHT;
+		if (rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
+			bw = 2;
+		else if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+			bw = 1;
+	} else if (rate->flags & IEEE80211_TX_RC_MCS) {
+		rate_idx = rate->idx;
+		nss = 1 + (rate->idx >> 3);
+		phy = MT_PHY_TYPE_HT;
+		if (rate->flags & IEEE80211_TX_RC_GREEN_FIELD)
+			phy = MT_PHY_TYPE_HT_GF;
+		if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+			bw = 1;
+	} else {
+		const struct ieee80211_rate *r;
+		int band = dev->mt76.chandef.chan->band;
+		u16 val;
+
+		r = &mt76_hw(dev)->wiphy->bands[band]->bitrates[rate->idx];
+		if (rate->flags & IEEE80211_TX_RC_USE_SHORT_PREAMBLE)
+			val = r->hw_value_short;
+		else
+			val = r->hw_value;
+
+		phy = val >> 8;
+		rate_idx = val & 0xff;
+		bw = 0;
+	}
+
+	rateval = FIELD_PREP(MT_RXWI_RATE_INDEX, rate_idx);
+	rateval |= FIELD_PREP(MT_RXWI_RATE_PHY, phy);
+	rateval |= FIELD_PREP(MT_RXWI_RATE_BW, bw);
+	if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
+		rateval |= MT_RXWI_RATE_SGI;
+
+	*nss_val = nss;
+	return cpu_to_le16(rateval);
+}
+
+void mt76x2_mac_wcid_set_drop(struct mt76x2_dev *dev, u8 idx, bool drop)
+{
+	u32 val = mt76_rr(dev, MT_WCID_DROP(idx));
+	u32 bit = MT_WCID_DROP_MASK(idx);
+
+	/* prevent unnecessary writes */
+	if ((val & bit) != (bit * drop))
+		mt76_wr(dev, MT_WCID_DROP(idx), (val & ~bit) | (bit * drop));
+}
+
+void mt76x2_mac_wcid_set_rate(struct mt76x2_dev *dev, struct mt76_wcid *wcid,
+			      const struct ieee80211_tx_rate *rate)
+{
+	spin_lock_bh(&dev->mt76.lock);
+	wcid->tx_rate = mt76x2_mac_tx_rate_val(dev, rate, &wcid->tx_rate_nss);
+	wcid->tx_rate_set = true;
+	spin_unlock_bh(&dev->mt76.lock);
+}
+
+void mt76x2_mac_write_txwi(struct mt76x2_dev *dev, struct mt76x2_txwi *txwi,
+			   struct sk_buff *skb, struct mt76_wcid *wcid,
+			   struct ieee80211_sta *sta)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_tx_rate *rate = &info->control.rates[0];
+	struct ieee80211_key_conf *key = info->control.hw_key;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
+	u16 rate_ht_mask = FIELD_PREP(MT_RXWI_RATE_PHY, BIT(1) | BIT(2));
+	u16 txwi_flags = 0;
+	u8 nss;
+	s8 txpwr_adj, max_txpwr_adj;
+	u8 ccmp_pn[8];
+
+	memset(txwi, 0, sizeof(*txwi));
+
+	if (wcid)
+		txwi->wcid = wcid->idx;
+	else
+		txwi->wcid = 0xff;
+
+	txwi->pktid = 1;
+
+	if (wcid && wcid->sw_iv && key) {
+		u64 pn = atomic64_inc_return(&key->tx_pn);
+		ccmp_pn[0] = pn;
+		ccmp_pn[1] = pn >> 8;
+		ccmp_pn[2] = 0;
+		ccmp_pn[3] = 0x20 | (key->keyidx << 6);
+		ccmp_pn[4] = pn >> 16;
+		ccmp_pn[5] = pn >> 24;
+		ccmp_pn[6] = pn >> 32;
+		ccmp_pn[7] = pn >> 40;
+		txwi->iv = *((u32 *) &ccmp_pn[0]);
+		txwi->eiv = *((u32 *) &ccmp_pn[1]);
+	}
+
+	spin_lock_bh(&dev->mt76.lock);
+	if (wcid && (rate->idx < 0 || !rate->count)) {
+		txwi->rate = wcid->tx_rate;
+		max_txpwr_adj = wcid->max_txpwr_adj;
+		nss = wcid->tx_rate_nss;
+	} else {
+		txwi->rate = mt76x2_mac_tx_rate_val(dev, rate, &nss);
+		max_txpwr_adj = mt76x2_tx_get_max_txpwr_adj(dev, rate);
+	}
+	spin_unlock_bh(&dev->mt76.lock);
+
+	txpwr_adj = mt76x2_tx_get_txpwr_adj(dev, dev->txpower_conf,
+					    max_txpwr_adj);
+	txwi->ctl2 = FIELD_PREP(MT_TX_PWR_ADJ, txpwr_adj);
+
+	if (mt76xx_rev(dev) >= MT76XX_REV_E4)
+		txwi->txstream = 0x13;
+	else if (mt76xx_rev(dev) >= MT76XX_REV_E3 &&
+		 !(txwi->rate & cpu_to_le16(rate_ht_mask)))
+		txwi->txstream = 0x93;
+
+	if (info->flags & IEEE80211_TX_CTL_LDPC)
+		txwi->rate |= cpu_to_le16(MT_RXWI_RATE_LDPC);
+	if ((info->flags & IEEE80211_TX_CTL_STBC) && nss == 1)
+		txwi->rate |= cpu_to_le16(MT_RXWI_RATE_STBC);
+	if (nss > 1 && sta && sta->smps_mode == IEEE80211_SMPS_DYNAMIC)
+		txwi_flags |= MT_TXWI_FLAGS_MMPS;
+	if (!(info->flags & IEEE80211_TX_CTL_NO_ACK))
+		txwi->ack_ctl |= MT_TXWI_ACK_CTL_REQ;
+	if (info->flags & IEEE80211_TX_CTL_ASSIGN_SEQ)
+		txwi->ack_ctl |= MT_TXWI_ACK_CTL_NSEQ;
+	if (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)
+		txwi->pktid |= MT_TXWI_PKTID_PROBE;
+	if ((info->flags & IEEE80211_TX_CTL_AMPDU) && sta) {
+		u8 ba_size = IEEE80211_MIN_AMPDU_BUF;
+
+		ba_size <<= sta->ht_cap.ampdu_factor;
+		ba_size = min_t(int, 63, ba_size - 1);
+		if (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)
+			ba_size = 0;
+		txwi->ack_ctl |= FIELD_PREP(MT_TXWI_ACK_CTL_BA_WINDOW, ba_size);
+
+		txwi_flags |= MT_TXWI_FLAGS_AMPDU |
+			 FIELD_PREP(MT_TXWI_FLAGS_MPDU_DENSITY,
+				    sta->ht_cap.ampdu_density);
+	}
+
+	if (ieee80211_is_probe_resp(hdr->frame_control) ||
+	    ieee80211_is_beacon(hdr->frame_control))
+		txwi_flags |= MT_TXWI_FLAGS_TS;
+
+	txwi->flags |= cpu_to_le16(txwi_flags);
+	txwi->len_ctl = cpu_to_le16(skb->len);
+}
+
+static void mt76x2_remove_hdr_pad(struct sk_buff *skb, int len)
+{
+	int hdrlen;
+
+	if (!len)
+		return;
+
+	hdrlen = ieee80211_get_hdrlen_from_skb(skb);
+	memmove(skb->data + len, skb->data, hdrlen);
+	skb_pull(skb, len);
+}
+
+static struct mt76_wcid *
+mt76x2_rx_get_sta_wcid(struct mt76x2_dev *dev, u8 idx, bool unicast)
+{
+	struct mt76x2_sta *sta;
+	struct mt76_wcid *wcid;
+
+	if (idx >= ARRAY_SIZE(dev->wcid))
+		return NULL;
+
+	wcid = rcu_dereference(dev->wcid[idx]);
+	if (unicast || !wcid)
+		return wcid;
+
+	sta = container_of(wcid, struct mt76x2_sta, wcid);
+	return &sta->vif->group_wcid;
+}
+
+int mt76x2_mac_process_rx(struct mt76x2_dev *dev, struct sk_buff *skb,
+			  void *rxi)
+{
+	struct mt76_rx_status *status = (struct mt76_rx_status *) skb->cb;
+	struct mt76x2_rxwi *rxwi = rxi;
+	u32 rxinfo = le32_to_cpu(rxwi->rxinfo);
+	u32 ctl = le32_to_cpu(rxwi->ctl);
+	u16 rate = le16_to_cpu(rxwi->rate);
+	u16 tid_sn = le16_to_cpu(rxwi->tid_sn);
+	bool unicast = rxwi->rxinfo & cpu_to_le32(MT_RXINFO_UNICAST);
+	int pad_len = 0;
+	u8 pn_len;
+	u8 wcid;
+	int len;
+
+	if (rxinfo & MT_RXINFO_L2PAD)
+		pad_len += 2;
+
+	if (rxinfo & MT_RXINFO_DECRYPT) {
+		status->flag |= RX_FLAG_DECRYPTED;
+		status->flag |= RX_FLAG_MMIC_STRIPPED;
+		status->flag |= RX_FLAG_MIC_STRIPPED;
+		status->flag |= RX_FLAG_IV_STRIPPED;
+	}
+
+	wcid = FIELD_GET(MT_RXWI_CTL_WCID, ctl);
+	status->wcid = mt76x2_rx_get_sta_wcid(dev, wcid, unicast);
+
+	len = FIELD_GET(MT_RXWI_CTL_MPDU_LEN, ctl);
+	pn_len = FIELD_GET(MT_RXINFO_PN_LEN, rxinfo);
+	if (pn_len) {
+		int offset = ieee80211_get_hdrlen_from_skb(skb) + pad_len;
+		u8 *data = skb->data + offset;
+
+		status->iv[0] = data[7];
+		status->iv[1] = data[6];
+		status->iv[2] = data[5];
+		status->iv[3] = data[4];
+		status->iv[4] = data[1];
+		status->iv[5] = data[0];
+
+		/*
+		 * Driver CCMP validation can't deal with fragments.
+		 * Let mac80211 take care of it.
+		 */
+		if (rxinfo & MT_RXINFO_FRAG) {
+			status->flag &= ~RX_FLAG_IV_STRIPPED;
+		} else {
+			pad_len += pn_len << 2;
+			len -= pn_len << 2;
+		}
+	}
+
+	mt76x2_remove_hdr_pad(skb, pad_len);
+
+	if ((rxinfo & MT_RXINFO_BA) && !(rxinfo & MT_RXINFO_NULL))
+		status->aggr = true;
+
+	if (WARN_ON_ONCE(len > skb->len))
+		return -EINVAL;
+
+	pskb_trim(skb, len);
+	status->chains = BIT(0) | BIT(1);
+	status->chain_signal[0] = mt76x2_phy_get_rssi(dev, rxwi->rssi[0], 0);
+	status->chain_signal[1] = mt76x2_phy_get_rssi(dev, rxwi->rssi[1], 1);
+	status->signal = max(status->chain_signal[0], status->chain_signal[1]);
+	status->freq = dev->mt76.chandef.chan->center_freq;
+	status->band = dev->mt76.chandef.chan->band;
+
+	status->tid = FIELD_GET(MT_RXWI_TID, tid_sn);
+	status->seqno = FIELD_GET(MT_RXWI_SN, tid_sn);
+
+	return mt76x2_mac_process_rate(status, rate);
+}
+
+static int
+mt76x2_mac_process_tx_rate(struct ieee80211_tx_rate *txrate, u16 rate,
+			   enum nl80211_band band)
+{
+	u8 idx = FIELD_GET(MT_RXWI_RATE_INDEX, rate);
+
+	txrate->idx = 0;
+	txrate->flags = 0;
+	txrate->count = 1;
+
+	switch (FIELD_GET(MT_RXWI_RATE_PHY, rate)) {
+	case MT_PHY_TYPE_OFDM:
+		if (band == NL80211_BAND_2GHZ)
+			idx += 4;
+
+		txrate->idx = idx;
+		return 0;
+	case MT_PHY_TYPE_CCK:
+		if (idx >= 8)
+			idx -= 8;
+
+		txrate->idx = idx;
+		return 0;
+	case MT_PHY_TYPE_HT_GF:
+		txrate->flags |= IEEE80211_TX_RC_GREEN_FIELD;
+		/* fall through */
+	case MT_PHY_TYPE_HT:
+		txrate->flags |= IEEE80211_TX_RC_MCS;
+		txrate->idx = idx;
+		break;
+	case MT_PHY_TYPE_VHT:
+		txrate->flags |= IEEE80211_TX_RC_VHT_MCS;
+		txrate->idx = idx;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (FIELD_GET(MT_RXWI_RATE_BW, rate)) {
+	case MT_PHY_BW_20:
+		break;
+	case MT_PHY_BW_40:
+		txrate->flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;
+		break;
+	case MT_PHY_BW_80:
+		txrate->flags |= IEEE80211_TX_RC_80_MHZ_WIDTH;
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+
+	if (rate & MT_RXWI_RATE_SGI)
+		txrate->flags |= IEEE80211_TX_RC_SHORT_GI;
+
+	return 0;
+}
+
+static void
+mt76x2_mac_fill_tx_status(struct mt76x2_dev *dev,
+			  struct ieee80211_tx_info *info,
+			  struct mt76x2_tx_status *st, int n_frames)
+{
+	struct ieee80211_tx_rate *rate = info->status.rates;
+	int cur_idx, last_rate;
+	int i;
+
+	if (!n_frames)
+		return;
+
+	last_rate = min_t(int, st->retry, IEEE80211_TX_MAX_RATES - 1);
+	mt76x2_mac_process_tx_rate(&rate[last_rate], st->rate,
+				 dev->mt76.chandef.chan->band);
+	if (last_rate < IEEE80211_TX_MAX_RATES - 1)
+		rate[last_rate + 1].idx = -1;
+
+	cur_idx = rate[last_rate].idx + st->retry;
+	for (i = 0; i <= last_rate; i++) {
+		rate[i].flags = rate[last_rate].flags;
+		rate[i].idx = max_t(int, 0, cur_idx - i);
+		rate[i].count = 1;
+	}
+
+	if (last_rate > 0)
+		rate[last_rate - 1].count = st->retry + 1 - last_rate;
+
+	info->status.ampdu_len = n_frames;
+	info->status.ampdu_ack_len = st->success ? n_frames : 0;
+
+	if (st->pktid & MT_TXWI_PKTID_PROBE)
+		info->flags |= IEEE80211_TX_CTL_RATE_CTRL_PROBE;
+
+	if (st->aggr)
+		info->flags |= IEEE80211_TX_CTL_AMPDU |
+			       IEEE80211_TX_STAT_AMPDU;
+
+	if (!st->ack_req)
+		info->flags |= IEEE80211_TX_CTL_NO_ACK;
+	else if (st->success)
+		info->flags |= IEEE80211_TX_STAT_ACK;
+}
+
+static void
+mt76x2_send_tx_status(struct mt76x2_dev *dev, struct mt76x2_tx_status *stat,
+		      u8 *update)
+{
+	struct ieee80211_tx_info info = {};
+	struct ieee80211_sta *sta = NULL;
+	struct mt76_wcid *wcid = NULL;
+	struct mt76x2_sta *msta = NULL;
+
+	rcu_read_lock();
+	if (stat->wcid < ARRAY_SIZE(dev->wcid))
+		wcid = rcu_dereference(dev->wcid[stat->wcid]);
+
+	if (wcid) {
+		void *priv;
+
+		priv = msta = container_of(wcid, struct mt76x2_sta, wcid);
+		sta = container_of(priv, struct ieee80211_sta,
+				   drv_priv);
+	}
+
+	if (msta && stat->aggr) {
+		u32 stat_val, stat_cache;
+
+		stat_val = stat->rate;
+		stat_val |= ((u32) stat->retry) << 16;
+		stat_cache = msta->status.rate;
+		stat_cache |= ((u32) msta->status.retry) << 16;
+
+		if (*update == 0 && stat_val == stat_cache &&
+		    stat->wcid == msta->status.wcid && msta->n_frames < 32) {
+			msta->n_frames++;
+			goto out;
+		}
+
+		mt76x2_mac_fill_tx_status(dev, &info, &msta->status,
+					  msta->n_frames);
+
+		msta->status = *stat;
+		msta->n_frames = 1;
+		*update = 0;
+	} else {
+		mt76x2_mac_fill_tx_status(dev, &info, stat, 1);
+		*update = 1;
+	}
+
+	ieee80211_tx_status_noskb(mt76_hw(dev), sta, &info);
+
+out:
+	rcu_read_unlock();
+}
+
+void mt76x2_mac_poll_tx_status(struct mt76x2_dev *dev, bool irq)
+{
+	struct mt76x2_tx_status stat = {};
+	unsigned long flags;
+	u8 update = 1;
+
+	if (!test_bit(MT76_STATE_RUNNING, &dev->mt76.state))
+		return;
+
+	trace_mac_txstat_poll(dev);
+
+	while (!irq || !kfifo_is_full(&dev->txstatus_fifo)) {
+		u32 stat1, stat2;
+
+		spin_lock_irqsave(&dev->irq_lock, flags);
+		stat2 = mt76_rr(dev, MT_TX_STAT_FIFO_EXT);
+		stat1 = mt76_rr(dev, MT_TX_STAT_FIFO);
+		if (!(stat1 & MT_TX_STAT_FIFO_VALID)) {
+			spin_unlock_irqrestore(&dev->irq_lock, flags);
+			break;
+		}
+
+		spin_unlock_irqrestore(&dev->irq_lock, flags);
+
+		stat.valid = 1;
+		stat.success = !!(stat1 & MT_TX_STAT_FIFO_SUCCESS);
+		stat.aggr = !!(stat1 & MT_TX_STAT_FIFO_AGGR);
+		stat.ack_req = !!(stat1 & MT_TX_STAT_FIFO_ACKREQ);
+		stat.wcid = FIELD_GET(MT_TX_STAT_FIFO_WCID, stat1);
+		stat.rate = FIELD_GET(MT_TX_STAT_FIFO_RATE, stat1);
+		stat.retry = FIELD_GET(MT_TX_STAT_FIFO_EXT_RETRY, stat2);
+		stat.pktid = FIELD_GET(MT_TX_STAT_FIFO_EXT_PKTID, stat2);
+		trace_mac_txstat_fetch(dev, &stat);
+
+		if (!irq) {
+			mt76x2_send_tx_status(dev, &stat, &update);
+			continue;
+		}
+
+		kfifo_put(&dev->txstatus_fifo, stat);
+	}
+}
+
+static void
+mt76x2_mac_queue_txdone(struct mt76x2_dev *dev, struct sk_buff *skb,
+			void *txwi_ptr)
+{
+	struct mt76x2_tx_info *txi = mt76x2_skb_tx_info(skb);
+	struct mt76x2_txwi *txwi = txwi_ptr;
+
+	mt76x2_mac_poll_tx_status(dev, false);
+
+	txi->tries = 0;
+	txi->jiffies = jiffies;
+	txi->wcid = txwi->wcid;
+	txi->pktid = txwi->pktid;
+	trace_mac_txdone_add(dev, txwi->wcid, txwi->pktid);
+	mt76x2_tx_complete(dev, skb);
+}
+
+void mt76x2_mac_process_tx_status_fifo(struct mt76x2_dev *dev)
+{
+	struct mt76x2_tx_status stat;
+	u8 update = 1;
+
+	while (kfifo_get(&dev->txstatus_fifo, &stat))
+		mt76x2_send_tx_status(dev, &stat, &update);
+}
+
+void mt76x2_tx_complete_skb(struct mt76_dev *mdev, struct mt76_queue *q,
+			    struct mt76_queue_entry *e, bool flush)
+{
+	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
+
+	if (e->txwi)
+		mt76x2_mac_queue_txdone(dev, e->skb, &e->txwi->txwi);
+	else
+		dev_kfree_skb_any(e->skb);
+}
+
+static enum mt76x2_cipher_type
+mt76x2_mac_get_key_info(struct ieee80211_key_conf *key, u8 *key_data)
+{
+	memset(key_data, 0, 32);
+	if (!key)
+		return MT_CIPHER_NONE;
+
+	if (key->keylen > 32)
+		return MT_CIPHER_NONE;
+
+	memcpy(key_data, key->key, key->keylen);
+
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_WEP40:
+		return MT_CIPHER_WEP40;
+	case WLAN_CIPHER_SUITE_WEP104:
+		return MT_CIPHER_WEP104;
+	case WLAN_CIPHER_SUITE_TKIP:
+		return MT_CIPHER_TKIP;
+	case WLAN_CIPHER_SUITE_CCMP:
+		return MT_CIPHER_AES_CCMP;
+	default:
+		return MT_CIPHER_NONE;
+	}
+}
+
+void mt76x2_mac_wcid_setup(struct mt76x2_dev *dev, u8 idx, u8 vif_idx, u8 *mac)
+{
+	struct mt76_wcid_addr addr = {};
+	u32 attr;
+
+	attr = FIELD_PREP(MT_WCID_ATTR_BSS_IDX, vif_idx & 7) |
+	       FIELD_PREP(MT_WCID_ATTR_BSS_IDX_EXT, !!(vif_idx & 8));
+
+	mt76_wr(dev, MT_WCID_ATTR(idx), attr);
+
+	mt76_wr(dev, MT_WCID_TX_RATE(idx), 0);
+	mt76_wr(dev, MT_WCID_TX_RATE(idx) + 4, 0);
+
+	if (idx >= 128)
+		return;
+
+	if (mac)
+		memcpy(addr.macaddr, mac, ETH_ALEN);
+
+	mt76_wr_copy(dev, MT_WCID_ADDR(idx), &addr, sizeof(addr));
+}
+
+int mt76x2_mac_wcid_set_key(struct mt76x2_dev *dev, u8 idx,
+			    struct ieee80211_key_conf *key)
+{
+	enum mt76x2_cipher_type cipher;
+	u8 key_data[32];
+	u8 iv_data[8];
+
+	cipher = mt76x2_mac_get_key_info(key, key_data);
+	if (cipher == MT_CIPHER_NONE && key)
+		return -EOPNOTSUPP;
+
+	mt76_rmw_field(dev, MT_WCID_ATTR(idx), MT_WCID_ATTR_PKEY_MODE, cipher);
+	mt76_wr_copy(dev, MT_WCID_KEY(idx), key_data, sizeof(key_data));
+
+	memset(iv_data, 0, sizeof(iv_data));
+	if (key) {
+		mt76_rmw_field(dev, MT_WCID_ATTR(idx), MT_WCID_ATTR_PAIRWISE,
+			       !!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE));
+		iv_data[3] = key->keyidx << 6;
+		if (cipher >= MT_CIPHER_TKIP)
+			iv_data[3] |= 0x20;
+	}
+
+	mt76_wr_copy(dev, MT_WCID_IV(idx), iv_data, sizeof(iv_data));
+
+	return 0;
+}
+
+int mt76x2_mac_shared_key_setup(struct mt76x2_dev *dev, u8 vif_idx, u8 key_idx,
+			      struct ieee80211_key_conf *key)
+{
+	enum mt76x2_cipher_type cipher;
+	u8 key_data[32];
+	u32 val;
+
+	cipher = mt76x2_mac_get_key_info(key, key_data);
+	if (cipher == MT_CIPHER_NONE && key)
+		return -EOPNOTSUPP;
+
+	val = mt76_rr(dev, MT_SKEY_MODE(vif_idx));
+	val &= ~(MT_SKEY_MODE_MASK << MT_SKEY_MODE_SHIFT(vif_idx, key_idx));
+	val |= cipher << MT_SKEY_MODE_SHIFT(vif_idx, key_idx);
+	mt76_wr(dev, MT_SKEY_MODE(vif_idx), val);
+
+	mt76_wr_copy(dev, MT_SKEY(vif_idx, key_idx), key_data,
+		     sizeof(key_data));
+
+	return 0;
+}
+
+static int
+mt76_write_beacon(struct mt76x2_dev *dev, int offset, struct sk_buff *skb)
+{
+	int beacon_len = dev->beacon_offsets[1] - dev->beacon_offsets[0];
+	struct mt76x2_txwi txwi;
+
+	if (WARN_ON_ONCE(beacon_len < skb->len + sizeof(struct mt76x2_txwi)))
+		return -ENOSPC;
+
+	mt76x2_mac_write_txwi(dev, &txwi, skb, NULL, NULL);
+
+	mt76_wr_copy(dev, offset, &txwi, sizeof(txwi));
+	offset += sizeof(txwi);
+
+	mt76_wr_copy(dev, offset, skb->data, skb->len);
+	return 0;
+}
+
+static int
+__mt76x2_mac_set_beacon(struct mt76x2_dev *dev, u8 bcn_idx, struct sk_buff *skb)
+{
+	int beacon_len = dev->beacon_offsets[1] - dev->beacon_offsets[0];
+	int beacon_addr = dev->beacon_offsets[bcn_idx];
+	int ret = 0;
+	int i;
+
+	/* Prevent corrupt transmissions during update */
+	mt76_set(dev, MT_BCN_BYPASS_MASK, BIT(bcn_idx));
+
+	if (skb) {
+		ret = mt76_write_beacon(dev, beacon_addr, skb);
+		if (!ret)
+			dev->beacon_data_mask |= BIT(bcn_idx) &
+						 dev->beacon_mask;
+	} else {
+		dev->beacon_data_mask &= ~BIT(bcn_idx);
+		for (i = 0; i < beacon_len; i += 4)
+			mt76_wr(dev, beacon_addr + i, 0);
+	}
+
+	mt76_wr(dev, MT_BCN_BYPASS_MASK, 0xff00 | ~dev->beacon_data_mask);
+
+	return ret;
+}
+
+int mt76x2_mac_set_beacon(struct mt76x2_dev *dev, u8 vif_idx,
+			  struct sk_buff *skb)
+{
+	bool force_update = false;
+	int bcn_idx = 0;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dev->beacons); i++) {
+		if (vif_idx == i) {
+			force_update = !!dev->beacons[i] ^ !!skb;
+
+			if (dev->beacons[i])
+				dev_kfree_skb(dev->beacons[i]);
+
+			dev->beacons[i] = skb;
+			__mt76x2_mac_set_beacon(dev, bcn_idx, skb);
+		} else if (force_update && dev->beacons[i]) {
+			__mt76x2_mac_set_beacon(dev, bcn_idx, dev->beacons[i]);
+		}
+
+		bcn_idx += !!dev->beacons[i];
+	}
+
+	for (i = bcn_idx; i < ARRAY_SIZE(dev->beacons); i++) {
+		if (!(dev->beacon_data_mask & BIT(i)))
+			break;
+
+		__mt76x2_mac_set_beacon(dev, i, NULL);
+	}
+
+	mt76_rmw_field(dev, MT_MAC_BSSID_DW1, MT_MAC_BSSID_DW1_MBEACON_N,
+		       bcn_idx - 1);
+	return 0;
+}
+
+void mt76x2_mac_set_beacon_enable(struct mt76x2_dev *dev, u8 vif_idx, bool val)
+{
+	u8 old_mask = dev->beacon_mask;
+	bool en;
+	u32 reg;
+
+	if (val) {
+		dev->beacon_mask |= BIT(vif_idx);
+	} else {
+		dev->beacon_mask &= ~BIT(vif_idx);
+		mt76x2_mac_set_beacon(dev, vif_idx, NULL);
+	}
+
+	if (!!old_mask == !!dev->beacon_mask)
+		return;
+
+	en = dev->beacon_mask;
+
+	mt76_rmw_field(dev, MT_INT_TIMER_EN, MT_INT_TIMER_EN_PRE_TBTT_EN, en);
+	reg = MT_BEACON_TIME_CFG_BEACON_TX |
+	      MT_BEACON_TIME_CFG_TBTT_EN |
+	      MT_BEACON_TIME_CFG_TIMER_EN;
+	mt76_rmw(dev, MT_BEACON_TIME_CFG, reg, reg * en);
+
+	if (en)
+		mt76x2_irq_enable(dev, MT_INT_PRE_TBTT | MT_INT_TBTT);
+	else
+		mt76x2_irq_disable(dev, MT_INT_PRE_TBTT | MT_INT_TBTT);
+}
+
+void mt76x2_update_channel(struct mt76_dev *mdev)
+{
+	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
+	struct mt76_channel_state *state;
+	u32 active, busy;
+
+	state = mt76_channel_state(&dev->mt76, dev->mt76.chandef.chan);
+
+	busy = mt76_rr(dev, MT_CH_BUSY);
+	active = busy + mt76_rr(dev, MT_CH_IDLE);
+
+	spin_lock_bh(&dev->mt76.cc_lock);
+	state->cc_busy += busy;
+	state->cc_active += active;
+	spin_unlock_bh(&dev->mt76.cc_lock);
+}
+
+void mt76x2_mac_work(struct work_struct *work)
+{
+	struct mt76x2_dev *dev = container_of(work, struct mt76x2_dev,
+					    mac_work.work);
+	int i, idx;
+
+	mt76x2_update_channel(&dev->mt76);
+	for (i = 0, idx = 0; i < 16; i++) {
+		u32 val = mt76_rr(dev, MT_TX_AGG_CNT(i));
+
+		dev->aggr_stats[idx++] += val & 0xffff;
+		dev->aggr_stats[idx++] += val >> 16;
+	}
+
+	ieee80211_queue_delayed_work(mt76_hw(dev), &dev->mac_work,
+				     MT_CALIBRATE_INTERVAL);
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_mac.h b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.h
new file mode 100644
index 000000000000..8a8a25e32d5f
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_mac.h
@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76x2_MAC_H
+#define __MT76x2_MAC_H
+
+#include "mt76.h"
+
+struct mt76x2_dev;
+struct mt76x2_sta;
+struct mt76x2_vif;
+struct mt76x2_txwi;
+
+struct mt76x2_tx_status {
+	u8 valid:1;
+	u8 success:1;
+	u8 aggr:1;
+	u8 ack_req:1;
+	u8 wcid;
+	u8 pktid;
+	u8 retry;
+	u16 rate;
+} __packed __aligned(2);
+
+struct mt76x2_tx_info {
+	unsigned long jiffies;
+	u8 tries;
+
+	u8 wcid;
+	u8 pktid;
+	u8 retry;
+};
+
+struct mt76x2_rxwi {
+	__le32 rxinfo;
+
+	__le32 ctl;
+
+	__le16 tid_sn;
+	__le16 rate;
+
+	u8 rssi[4];
+
+	__le32 bbp_rxinfo[4];
+};
+
+#define MT_RXINFO_BA			BIT(0)
+#define MT_RXINFO_DATA			BIT(1)
+#define MT_RXINFO_NULL			BIT(2)
+#define MT_RXINFO_FRAG			BIT(3)
+#define MT_RXINFO_UNICAST		BIT(4)
+#define MT_RXINFO_MULTICAST		BIT(5)
+#define MT_RXINFO_BROADCAST		BIT(6)
+#define MT_RXINFO_MYBSS			BIT(7)
+#define MT_RXINFO_CRCERR		BIT(8)
+#define MT_RXINFO_ICVERR		BIT(9)
+#define MT_RXINFO_MICERR		BIT(10)
+#define MT_RXINFO_AMSDU			BIT(11)
+#define MT_RXINFO_HTC			BIT(12)
+#define MT_RXINFO_RSSI			BIT(13)
+#define MT_RXINFO_L2PAD			BIT(14)
+#define MT_RXINFO_AMPDU			BIT(15)
+#define MT_RXINFO_DECRYPT		BIT(16)
+#define MT_RXINFO_BSSIDX3		BIT(17)
+#define MT_RXINFO_WAPI_KEY		BIT(18)
+#define MT_RXINFO_PN_LEN		GENMASK(21, 19)
+#define MT_RXINFO_SW_FTYPE0		BIT(22)
+#define MT_RXINFO_SW_FTYPE1		BIT(23)
+#define MT_RXINFO_PROBE_RESP		BIT(24)
+#define MT_RXINFO_BEACON		BIT(25)
+#define MT_RXINFO_DISASSOC		BIT(26)
+#define MT_RXINFO_DEAUTH		BIT(27)
+#define MT_RXINFO_ACTION		BIT(28)
+#define MT_RXINFO_TCP_SUM_ERR		BIT(30)
+#define MT_RXINFO_IP_SUM_ERR		BIT(31)
+
+#define MT_RXWI_CTL_WCID		GENMASK(7, 0)
+#define MT_RXWI_CTL_KEY_IDX		GENMASK(9, 8)
+#define MT_RXWI_CTL_BSS_IDX		GENMASK(12, 10)
+#define MT_RXWI_CTL_UDF			GENMASK(15, 13)
+#define MT_RXWI_CTL_MPDU_LEN		GENMASK(29, 16)
+#define MT_RXWI_CTL_EOF			BIT(31)
+
+#define MT_RXWI_TID			GENMASK(3, 0)
+#define MT_RXWI_SN			GENMASK(15, 4)
+
+#define MT_RXWI_RATE_INDEX		GENMASK(5, 0)
+#define MT_RXWI_RATE_LDPC		BIT(6)
+#define MT_RXWI_RATE_BW			GENMASK(8, 7)
+#define MT_RXWI_RATE_SGI		BIT(9)
+#define MT_RXWI_RATE_STBC		BIT(10)
+#define MT_RXWI_RATE_LDPC_EXSYM		BIT(11)
+#define MT_RXWI_RATE_PHY		GENMASK(15, 13)
+
+#define MT_RATE_INDEX_VHT_IDX		GENMASK(3, 0)
+#define MT_RATE_INDEX_VHT_NSS		GENMASK(5, 4)
+
+#define MT_TX_PWR_ADJ			GENMASK(3, 0)
+
+enum mt76x2_phy_bandwidth {
+	MT_PHY_BW_20,
+	MT_PHY_BW_40,
+	MT_PHY_BW_80,
+};
+
+#define MT_TXWI_FLAGS_FRAG		BIT(0)
+#define MT_TXWI_FLAGS_MMPS		BIT(1)
+#define MT_TXWI_FLAGS_CFACK		BIT(2)
+#define MT_TXWI_FLAGS_TS		BIT(3)
+#define MT_TXWI_FLAGS_AMPDU		BIT(4)
+#define MT_TXWI_FLAGS_MPDU_DENSITY	GENMASK(7, 5)
+#define MT_TXWI_FLAGS_TXOP		GENMASK(9, 8)
+#define MT_TXWI_FLAGS_NDPS		BIT(10)
+#define MT_TXWI_FLAGS_RTSBWSIG		BIT(11)
+#define MT_TXWI_FLAGS_NDP_BW		GENMASK(13, 12)
+#define MT_TXWI_FLAGS_SOUND		BIT(14)
+#define MT_TXWI_FLAGS_TX_RATE_LUT	BIT(15)
+
+#define MT_TXWI_ACK_CTL_REQ		BIT(0)
+#define MT_TXWI_ACK_CTL_NSEQ		BIT(1)
+#define MT_TXWI_ACK_CTL_BA_WINDOW	GENMASK(7, 2)
+
+#define MT_TXWI_PKTID_PROBE		BIT(7)
+
+struct mt76x2_txwi {
+	__le16 flags;
+	__le16 rate;
+	u8 ack_ctl;
+	u8 wcid;
+	__le16 len_ctl;
+	__le32 iv;
+	__le32 eiv;
+	u8 aid;
+	u8 txstream;
+	u8 ctl2;
+	u8 pktid;
+} __packed __aligned(4);
+
+static inline struct mt76x2_tx_info *
+mt76x2_skb_tx_info(struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+	return (void *) info->status.status_driver_data;
+}
+
+int mt76x2_mac_reset(struct mt76x2_dev *dev, bool hard);
+int mt76x2_mac_start(struct mt76x2_dev *dev);
+void mt76x2_mac_stop(struct mt76x2_dev *dev, bool force);
+void mt76x2_mac_resume(struct mt76x2_dev *dev);
+void mt76x2_mac_set_bssid(struct mt76x2_dev *dev, u8 idx, const u8 *addr);
+
+int mt76x2_mac_process_rx(struct mt76x2_dev *dev, struct sk_buff *skb,
+			  void *rxi);
+void mt76x2_mac_write_txwi(struct mt76x2_dev *dev, struct mt76x2_txwi *txwi,
+			   struct sk_buff *skb, struct mt76_wcid *wcid,
+			   struct ieee80211_sta *sta);
+void mt76x2_mac_wcid_setup(struct mt76x2_dev *dev, u8 idx, u8 vif_idx, u8 *mac);
+int mt76x2_mac_wcid_set_key(struct mt76x2_dev *dev, u8 idx,
+			    struct ieee80211_key_conf *key);
+void mt76x2_mac_wcid_set_rate(struct mt76x2_dev *dev, struct mt76_wcid *wcid,
+			      const struct ieee80211_tx_rate *rate);
+void mt76x2_mac_wcid_set_drop(struct mt76x2_dev *dev, u8 idx, bool drop);
+
+int mt76x2_mac_shared_key_setup(struct mt76x2_dev *dev, u8 vif_idx, u8 key_idx,
+				struct ieee80211_key_conf *key);
+
+int mt76x2_mac_set_beacon(struct mt76x2_dev *dev, u8 vif_idx,
+			  struct sk_buff *skb);
+void mt76x2_mac_set_beacon_enable(struct mt76x2_dev *dev, u8 vif_idx, bool val);
+
+void mt76x2_mac_poll_tx_status(struct mt76x2_dev *dev, bool irq);
+void mt76x2_mac_process_tx_status_fifo(struct mt76x2_dev *dev);
+
+void mt76x2_mac_work(struct work_struct *work);
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_main.c b/drivers/net/wireless/mediatek/mt76/mt76x2_main.c
new file mode 100644
index 000000000000..205043b470b2
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_main.c
@@ -0,0 +1,577 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "mt76x2.h"
+
+static int
+mt76x2_start(struct ieee80211_hw *hw)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	int ret;
+
+	mutex_lock(&dev->mutex);
+
+	ret = mt76x2_mac_start(dev);
+	if (ret)
+		goto out;
+
+	ret = mt76x2_phy_start(dev);
+	if (ret)
+		goto out;
+
+	ieee80211_queue_delayed_work(mt76_hw(dev), &dev->mac_work,
+				     MT_CALIBRATE_INTERVAL);
+
+	set_bit(MT76_STATE_RUNNING, &dev->mt76.state);
+
+out:
+	mutex_unlock(&dev->mutex);
+	return ret;
+}
+
+static void
+mt76x2_stop(struct ieee80211_hw *hw)
+{
+	struct mt76x2_dev *dev = hw->priv;
+
+	mutex_lock(&dev->mutex);
+	clear_bit(MT76_STATE_RUNNING, &dev->mt76.state);
+	mt76x2_stop_hardware(dev);
+	mutex_unlock(&dev->mutex);
+}
+
+static void
+mt76x2_txq_init(struct mt76x2_dev *dev, struct ieee80211_txq *txq)
+{
+	struct mt76_txq *mtxq;
+
+	if (!txq)
+		return;
+
+	mtxq = (struct mt76_txq *) txq->drv_priv;
+	if (txq->sta) {
+		struct mt76x2_sta *sta;
+
+		sta = (struct mt76x2_sta *) txq->sta->drv_priv;
+		mtxq->wcid = &sta->wcid;
+	} else {
+		struct mt76x2_vif *mvif;
+
+		mvif = (struct mt76x2_vif *) txq->vif->drv_priv;
+		mtxq->wcid = &mvif->group_wcid;
+	}
+
+	mt76_txq_init(&dev->mt76, txq);
+}
+
+static int
+mt76x2_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	struct mt76x2_vif *mvif = (struct mt76x2_vif *) vif->drv_priv;
+	unsigned int idx = 0;
+
+	if (vif->addr[0] & BIT(1))
+		idx = 1 + (((dev->mt76.macaddr[0] ^ vif->addr[0]) >> 2) & 7);
+
+	/*
+	 * Client mode typically only has one configurable BSSID register,
+	 * which is used for bssidx=0. This is linked to the MAC address.
+	 * Since mac80211 allows changing interface types, and we cannot
+	 * force the use of the primary MAC address for a station mode
+	 * interface, we need some other way of configuring a per-interface
+	 * remote BSSID.
+	 * The hardware provides an AP-Client feature, where bssidx 0-7 are
+	 * used for AP mode and bssidx 8-15 for client mode.
+	 * We shift the station interface bss index by 8 to force the
+	 * hardware to recognize the BSSID.
+	 * The resulting bssidx mismatch for unicast frames is ignored by hw.
+	 */
+	if (vif->type == NL80211_IFTYPE_STATION)
+		idx += 8;
+
+	mvif->idx = idx;
+	mvif->group_wcid.idx = 254 - idx;
+	mvif->group_wcid.hw_key_idx = -1;
+	mt76x2_txq_init(dev, vif->txq);
+
+	return 0;
+}
+
+static void
+mt76x2_remove_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct mt76x2_dev *dev = hw->priv;
+
+	mt76_txq_remove(&dev->mt76, vif->txq);
+}
+
+static int
+mt76x2_set_channel(struct mt76x2_dev *dev, struct cfg80211_chan_def *chandef)
+{
+	int ret;
+
+	mt76_set_channel(&dev->mt76);
+
+	tasklet_disable(&dev->pre_tbtt_tasklet);
+	tasklet_disable(&dev->dfs_pd.dfs_tasklet);
+	cancel_delayed_work_sync(&dev->cal_work);
+
+	mt76x2_mac_stop(dev, true);
+	ret = mt76x2_phy_set_channel(dev, chandef);
+
+	/* channel cycle counters read-and-clear */
+	mt76_rr(dev, MT_CH_IDLE);
+	mt76_rr(dev, MT_CH_BUSY);
+
+	mt76x2_dfs_init_params(dev);
+
+	mt76x2_mac_resume(dev);
+	tasklet_enable(&dev->dfs_pd.dfs_tasklet);
+	tasklet_enable(&dev->pre_tbtt_tasklet);
+
+	return ret;
+}
+
+static int
+mt76x2_config(struct ieee80211_hw *hw, u32 changed)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	int ret = 0;
+
+	mutex_lock(&dev->mutex);
+
+	if (changed & IEEE80211_CONF_CHANGE_MONITOR) {
+		if (!(hw->conf.flags & IEEE80211_CONF_MONITOR))
+			dev->rxfilter |= MT_RX_FILTR_CFG_PROMISC;
+		else
+			dev->rxfilter &= ~MT_RX_FILTR_CFG_PROMISC;
+
+		mt76_wr(dev, MT_RX_FILTR_CFG, dev->rxfilter);
+	}
+
+	if (changed & IEEE80211_CONF_CHANGE_POWER) {
+		dev->txpower_conf = hw->conf.power_level * 2;
+
+		/* convert to per-chain power for 2x2 devices */
+		dev->txpower_conf -= 6;
+
+		if (test_bit(MT76_STATE_RUNNING, &dev->mt76.state)) {
+			mt76x2_phy_set_txpower(dev);
+			mt76x2_tx_set_txpwr_auto(dev, dev->txpower_conf);
+		}
+	}
+
+	if (changed & IEEE80211_CONF_CHANGE_CHANNEL) {
+		ieee80211_stop_queues(hw);
+		ret = mt76x2_set_channel(dev, &hw->conf.chandef);
+		ieee80211_wake_queues(hw);
+	}
+
+	mutex_unlock(&dev->mutex);
+
+	return ret;
+}
+
+static void
+mt76x2_configure_filter(struct ieee80211_hw *hw, unsigned int changed_flags,
+			unsigned int *total_flags, u64 multicast)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	u32 flags = 0;
+
+#define MT76_FILTER(_flag, _hw) do { \
+		flags |= *total_flags & FIF_##_flag;			\
+		dev->rxfilter &= ~(_hw);				\
+		dev->rxfilter |= !(flags & FIF_##_flag) * (_hw);	\
+	} while (0)
+
+	mutex_lock(&dev->mutex);
+
+	dev->rxfilter &= ~MT_RX_FILTR_CFG_OTHER_BSS;
+
+	MT76_FILTER(FCSFAIL, MT_RX_FILTR_CFG_CRC_ERR);
+	MT76_FILTER(PLCPFAIL, MT_RX_FILTR_CFG_PHY_ERR);
+	MT76_FILTER(CONTROL, MT_RX_FILTR_CFG_ACK |
+			     MT_RX_FILTR_CFG_CTS |
+			     MT_RX_FILTR_CFG_CFEND |
+			     MT_RX_FILTR_CFG_CFACK |
+			     MT_RX_FILTR_CFG_BA |
+			     MT_RX_FILTR_CFG_CTRL_RSV);
+	MT76_FILTER(PSPOLL, MT_RX_FILTR_CFG_PSPOLL);
+
+	*total_flags = flags;
+	mt76_wr(dev, MT_RX_FILTR_CFG, dev->rxfilter);
+
+	mutex_unlock(&dev->mutex);
+}
+
+static void
+mt76x2_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			struct ieee80211_bss_conf *info, u32 changed)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	struct mt76x2_vif *mvif = (struct mt76x2_vif *) vif->drv_priv;
+
+	mutex_lock(&dev->mutex);
+
+	if (changed & BSS_CHANGED_BSSID)
+		mt76x2_mac_set_bssid(dev, mvif->idx, info->bssid);
+
+	if (changed & BSS_CHANGED_BEACON_INT)
+		mt76_rmw_field(dev, MT_BEACON_TIME_CFG,
+			       MT_BEACON_TIME_CFG_INTVAL,
+			       info->beacon_int << 4);
+
+	if (changed & BSS_CHANGED_BEACON_ENABLED) {
+		tasklet_disable(&dev->pre_tbtt_tasklet);
+		mt76x2_mac_set_beacon_enable(dev, mvif->idx,
+					     info->enable_beacon);
+		tasklet_enable(&dev->pre_tbtt_tasklet);
+	}
+
+	if (changed & BSS_CHANGED_ERP_SLOT) {
+		int slottime = info->use_short_slot ? 9 : 20;
+
+		dev->slottime = slottime;
+		mt76_rmw_field(dev, MT_BKOFF_SLOT_CFG,
+			       MT_BKOFF_SLOT_CFG_SLOTTIME, slottime);
+	}
+
+	mutex_unlock(&dev->mutex);
+}
+
+static int
+mt76x2_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+	       struct ieee80211_sta *sta)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	struct mt76x2_sta *msta = (struct mt76x2_sta *) sta->drv_priv;
+	struct mt76x2_vif *mvif = (struct mt76x2_vif *) vif->drv_priv;
+	int ret = 0;
+	int idx = 0;
+	int i;
+
+	mutex_lock(&dev->mutex);
+
+	idx = mt76_wcid_alloc(dev->wcid_mask, ARRAY_SIZE(dev->wcid));
+	if (idx < 0) {
+		ret = -ENOSPC;
+		goto out;
+	}
+
+	msta->vif = mvif;
+	msta->wcid.sta = 1;
+	msta->wcid.idx = idx;
+	msta->wcid.hw_key_idx = -1;
+	mt76x2_mac_wcid_setup(dev, idx, mvif->idx, sta->addr);
+	mt76x2_mac_wcid_set_drop(dev, idx, false);
+	for (i = 0; i < ARRAY_SIZE(sta->txq); i++)
+		mt76x2_txq_init(dev, sta->txq[i]);
+
+	if (vif->type == NL80211_IFTYPE_AP)
+		set_bit(MT_WCID_FLAG_CHECK_PS, &msta->wcid.flags);
+
+	rcu_assign_pointer(dev->wcid[idx], &msta->wcid);
+
+out:
+	mutex_unlock(&dev->mutex);
+
+	return ret;
+}
+
+static int
+mt76x2_sta_remove(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		  struct ieee80211_sta *sta)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	struct mt76x2_sta *msta = (struct mt76x2_sta *) sta->drv_priv;
+	int idx = msta->wcid.idx;
+	int i;
+
+	mutex_lock(&dev->mutex);
+	rcu_assign_pointer(dev->wcid[idx], NULL);
+	for (i = 0; i < ARRAY_SIZE(sta->txq); i++)
+		mt76_txq_remove(&dev->mt76, sta->txq[i]);
+	mt76x2_mac_wcid_set_drop(dev, idx, true);
+	mt76_wcid_free(dev->wcid_mask, idx);
+	mt76x2_mac_wcid_setup(dev, idx, 0, NULL);
+	mutex_unlock(&dev->mutex);
+
+	return 0;
+}
+
+void
+mt76x2_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps)
+{
+	struct mt76x2_sta *msta = (struct mt76x2_sta *) sta->drv_priv;
+	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
+	int idx = msta->wcid.idx;
+
+	mt76x2_mac_wcid_set_drop(dev, idx, ps);
+}
+
+static int
+mt76x2_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+	       struct ieee80211_vif *vif, struct ieee80211_sta *sta,
+	       struct ieee80211_key_conf *key)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	struct mt76x2_vif *mvif = (struct mt76x2_vif *) vif->drv_priv;
+	struct mt76x2_sta *msta;
+	struct mt76_wcid *wcid;
+	int idx = key->keyidx;
+	int ret;
+
+	/*
+	 * The hardware does not support per-STA RX GTK, fall back
+	 * to software mode for these.
+	 */
+	if ((vif->type == NL80211_IFTYPE_ADHOC ||
+	     vif->type == NL80211_IFTYPE_MESH_POINT) &&
+	    (key->cipher == WLAN_CIPHER_SUITE_TKIP ||
+	     key->cipher == WLAN_CIPHER_SUITE_CCMP) &&
+	    !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
+		return -EOPNOTSUPP;
+
+	msta = sta ? (struct mt76x2_sta *) sta->drv_priv : NULL;
+	wcid = msta ? &msta->wcid : &mvif->group_wcid;
+
+	if (cmd == SET_KEY) {
+		key->hw_key_idx = wcid->idx;
+		wcid->hw_key_idx = idx;
+		if (key->flags & IEEE80211_KEY_FLAG_RX_MGMT) {
+			key->flags |= IEEE80211_KEY_FLAG_SW_MGMT_TX;
+			wcid->sw_iv = true;
+		}
+	} else {
+		if (idx == wcid->hw_key_idx) {
+			wcid->hw_key_idx = -1;
+			wcid->sw_iv = true;
+		}
+
+		key = NULL;
+	}
+	mt76_wcid_key_setup(&dev->mt76, wcid, key);
+
+	if (!msta) {
+		if (key || wcid->hw_key_idx == idx) {
+			ret = mt76x2_mac_wcid_set_key(dev, wcid->idx, key);
+			if (ret)
+				return ret;
+		}
+
+		return mt76x2_mac_shared_key_setup(dev, mvif->idx, idx, key);
+	}
+
+	return mt76x2_mac_wcid_set_key(dev, msta->wcid.idx, key);
+}
+
+static int
+mt76x2_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif, u16 queue,
+	       const struct ieee80211_tx_queue_params *params)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	u8 cw_min = 5, cw_max = 10, qid;
+	u32 val;
+
+	qid = dev->mt76.q_tx[queue].hw_idx;
+
+	if (params->cw_min)
+		cw_min = fls(params->cw_min);
+	if (params->cw_max)
+		cw_max = fls(params->cw_max);
+
+	val = FIELD_PREP(MT_EDCA_CFG_TXOP, params->txop) |
+	      FIELD_PREP(MT_EDCA_CFG_AIFSN, params->aifs) |
+	      FIELD_PREP(MT_EDCA_CFG_CWMIN, cw_min) |
+	      FIELD_PREP(MT_EDCA_CFG_CWMAX, cw_max);
+	mt76_wr(dev, MT_EDCA_CFG_AC(qid), val);
+
+	val = mt76_rr(dev, MT_WMM_TXOP(qid));
+	val &= ~(MT_WMM_TXOP_MASK << MT_WMM_TXOP_SHIFT(qid));
+	val |= params->txop << MT_WMM_TXOP_SHIFT(qid);
+	mt76_wr(dev, MT_WMM_TXOP(qid), val);
+
+	val = mt76_rr(dev, MT_WMM_AIFSN);
+	val &= ~(MT_WMM_AIFSN_MASK << MT_WMM_AIFSN_SHIFT(qid));
+	val |= params->aifs << MT_WMM_AIFSN_SHIFT(qid);
+	mt76_wr(dev, MT_WMM_AIFSN, val);
+
+	val = mt76_rr(dev, MT_WMM_CWMIN);
+	val &= ~(MT_WMM_CWMIN_MASK << MT_WMM_CWMIN_SHIFT(qid));
+	val |= cw_min << MT_WMM_CWMIN_SHIFT(qid);
+	mt76_wr(dev, MT_WMM_CWMIN, val);
+
+	val = mt76_rr(dev, MT_WMM_CWMAX);
+	val &= ~(MT_WMM_CWMAX_MASK << MT_WMM_CWMAX_SHIFT(qid));
+	val |= cw_max << MT_WMM_CWMAX_SHIFT(qid);
+	mt76_wr(dev, MT_WMM_CWMAX, val);
+
+	return 0;
+}
+
+static void
+mt76x2_sw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+	       const u8 *mac)
+{
+	struct mt76x2_dev *dev = hw->priv;
+
+	tasklet_disable(&dev->pre_tbtt_tasklet);
+	set_bit(MT76_SCANNING, &dev->mt76.state);
+}
+
+static void
+mt76x2_sw_scan_complete(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	struct mt76x2_dev *dev = hw->priv;
+
+	clear_bit(MT76_SCANNING, &dev->mt76.state);
+	tasklet_enable(&dev->pre_tbtt_tasklet);
+	mt76_txq_schedule_all(&dev->mt76);
+}
+
+static void
+mt76x2_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+	     u32 queues, bool drop)
+{
+}
+
+static int
+mt76x2_get_txpower(struct ieee80211_hw *hw, struct ieee80211_vif *vif, int *dbm)
+{
+	struct mt76x2_dev *dev = hw->priv;
+
+	*dbm = dev->txpower_cur / 2;
+
+	/* convert from per-chain power to combined output on 2x2 devices */
+	*dbm += 3;
+
+	return 0;
+}
+
+static int
+mt76x2_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+		    struct ieee80211_ampdu_params *params)
+{
+	enum ieee80211_ampdu_mlme_action action = params->action;
+	struct ieee80211_sta *sta = params->sta;
+	struct mt76x2_dev *dev = hw->priv;
+	struct mt76x2_sta *msta = (struct mt76x2_sta *) sta->drv_priv;
+	struct ieee80211_txq *txq = sta->txq[params->tid];
+	u16 tid = params->tid;
+	u16 *ssn = &params->ssn;
+	struct mt76_txq *mtxq;
+
+	if (!txq)
+		return -EINVAL;
+
+	mtxq = (struct mt76_txq *)txq->drv_priv;
+
+	switch (action) {
+	case IEEE80211_AMPDU_RX_START:
+		mt76_rx_aggr_start(&dev->mt76, &msta->wcid, tid, *ssn, params->buf_size);
+		mt76_set(dev, MT_WCID_ADDR(msta->wcid.idx) + 4, BIT(16 + tid));
+		break;
+	case IEEE80211_AMPDU_RX_STOP:
+		mt76_rx_aggr_stop(&dev->mt76, &msta->wcid, tid);
+		mt76_clear(dev, MT_WCID_ADDR(msta->wcid.idx) + 4,
+			   BIT(16 + tid));
+		break;
+	case IEEE80211_AMPDU_TX_OPERATIONAL:
+		mtxq->aggr = true;
+		mtxq->send_bar = false;
+		ieee80211_send_bar(vif, sta->addr, tid, mtxq->agg_ssn);
+		break;
+	case IEEE80211_AMPDU_TX_STOP_FLUSH:
+	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
+		mtxq->aggr = false;
+		ieee80211_send_bar(vif, sta->addr, tid, mtxq->agg_ssn);
+		break;
+	case IEEE80211_AMPDU_TX_START:
+		mtxq->agg_ssn = *ssn << 4;
+		ieee80211_start_tx_ba_cb_irqsafe(vif, sta->addr, tid);
+		break;
+	case IEEE80211_AMPDU_TX_STOP_CONT:
+		mtxq->aggr = false;
+		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
+		break;
+	}
+
+	return 0;
+}
+
+static void
+mt76x2_sta_rate_tbl_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			   struct ieee80211_sta *sta)
+{
+	struct mt76x2_dev *dev = hw->priv;
+	struct mt76x2_sta *msta = (struct mt76x2_sta *) sta->drv_priv;
+	struct ieee80211_sta_rates *rates = rcu_dereference(sta->rates);
+	struct ieee80211_tx_rate rate = {};
+
+	if (!rates)
+		return;
+
+	rate.idx = rates->rate[0].idx;
+	rate.flags = rates->rate[0].flags;
+	mt76x2_mac_wcid_set_rate(dev, &msta->wcid, &rate);
+	msta->wcid.max_txpwr_adj = mt76x2_tx_get_max_txpwr_adj(dev, &rate);
+}
+
+static void mt76x2_set_coverage_class(struct ieee80211_hw *hw,
+				      s16 coverage_class)
+{
+	struct mt76x2_dev *dev = hw->priv;
+
+	mutex_lock(&dev->mutex);
+	dev->coverage_class = coverage_class;
+	mt76x2_set_tx_ackto(dev);
+	mutex_unlock(&dev->mutex);
+}
+
+static int
+mt76x2_set_tim(struct ieee80211_hw *hw, struct ieee80211_sta *sta, bool set)
+{
+	return 0;
+}
+
+const struct ieee80211_ops mt76x2_ops = {
+	.tx = mt76x2_tx,
+	.start = mt76x2_start,
+	.stop = mt76x2_stop,
+	.add_interface = mt76x2_add_interface,
+	.remove_interface = mt76x2_remove_interface,
+	.config = mt76x2_config,
+	.configure_filter = mt76x2_configure_filter,
+	.bss_info_changed = mt76x2_bss_info_changed,
+	.sta_add = mt76x2_sta_add,
+	.sta_remove = mt76x2_sta_remove,
+	.set_key = mt76x2_set_key,
+	.conf_tx = mt76x2_conf_tx,
+	.sw_scan_start = mt76x2_sw_scan,
+	.sw_scan_complete = mt76x2_sw_scan_complete,
+	.flush = mt76x2_flush,
+	.ampdu_action = mt76x2_ampdu_action,
+	.get_txpower = mt76x2_get_txpower,
+	.wake_tx_queue = mt76_wake_tx_queue,
+	.sta_rate_tbl_update = mt76x2_sta_rate_tbl_update,
+	.release_buffered_frames = mt76_release_buffered_frames,
+	.set_coverage_class = mt76x2_set_coverage_class,
+	.get_survey = mt76_get_survey,
+	.set_tim = mt76x2_set_tim,
+};
+
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76x2_mcu.c
new file mode 100644
index 000000000000..15820b11f9db
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_mcu.c
@@ -0,0 +1,453 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/firmware.h>
+#include <linux/delay.h>
+
+#include "mt76x2.h"
+#include "mt76x2_mcu.h"
+#include "mt76x2_dma.h"
+#include "mt76x2_eeprom.h"
+
+struct mt76x2_fw_header {
+	__le32 ilm_len;
+	__le32 dlm_len;
+	__le16 build_ver;
+	__le16 fw_ver;
+	u8 pad[4];
+	char build_time[16];
+};
+
+struct mt76x2_patch_header {
+	char build_time[16];
+	char platform[4];
+	char hw_version[4];
+	char patch_version[4];
+	u8 pad[2];
+};
+
+static struct sk_buff *mt76x2_mcu_msg_alloc(const void *data, int len)
+{
+	struct sk_buff *skb;
+
+	skb = alloc_skb(len, GFP_KERNEL);
+	if (!skb)
+		return NULL;
+	memcpy(skb_put(skb, len), data, len);
+
+	return skb;
+}
+
+static struct sk_buff *
+mt76x2_mcu_get_response(struct mt76x2_dev *dev, unsigned long expires)
+{
+	unsigned long timeout;
+
+	if (!time_is_after_jiffies(expires))
+		return NULL;
+
+	timeout = expires - jiffies;
+	wait_event_timeout(dev->mcu.wait, !skb_queue_empty(&dev->mcu.res_q),
+			   timeout);
+	return skb_dequeue(&dev->mcu.res_q);
+}
+
+static int
+mt76x2_mcu_msg_send(struct mt76x2_dev *dev, struct sk_buff *skb,
+		    enum mcu_cmd cmd)
+{
+	unsigned long expires = jiffies + HZ;
+	int ret;
+	u8 seq;
+
+	if (!skb)
+		return -EINVAL;
+
+	mutex_lock(&dev->mcu.mutex);
+
+	seq = ++dev->mcu.msg_seq & 0xf;
+	if (!seq)
+		seq = ++dev->mcu.msg_seq & 0xf;
+
+	ret = mt76x2_tx_queue_mcu(dev, MT_TXQ_MCU, skb, cmd, seq);
+	if (ret)
+		goto out;
+
+	while (1) {
+		u32 *rxfce;
+		bool check_seq = false;
+
+		skb = mt76x2_mcu_get_response(dev, expires);
+		if (!skb) {
+			dev_err(dev->mt76.dev,
+				"MCU message %d (seq %d) timed out\n", cmd,
+				seq);
+			ret = -ETIMEDOUT;
+			break;
+		}
+
+		rxfce = (u32 *) skb->cb;
+
+		if (seq == FIELD_GET(MT_RX_FCE_INFO_CMD_SEQ, *rxfce))
+			check_seq = true;
+
+		dev_kfree_skb(skb);
+		if (check_seq)
+			break;
+	}
+
+out:
+	mutex_unlock(&dev->mcu.mutex);
+
+	return ret;
+}
+
+static int
+mt76pci_load_rom_patch(struct mt76x2_dev *dev)
+{
+	const struct firmware *fw = NULL;
+	struct mt76x2_patch_header *hdr;
+	bool rom_protect = !is_mt7612(dev);
+	int len, ret = 0;
+	__le32 *cur;
+	u32 patch_mask, patch_reg;
+
+	if (rom_protect && !mt76_poll(dev, MT_MCU_SEMAPHORE_03, 1, 1, 600)) {
+		dev_err(dev->mt76.dev,
+			"Could not get hardware semaphore for ROM PATCH\n");
+		return -ETIMEDOUT;
+	}
+
+	if (mt76xx_rev(dev) >= MT76XX_REV_E3) {
+		patch_mask = BIT(0);
+		patch_reg = MT_MCU_CLOCK_CTL;
+	} else {
+		patch_mask = BIT(1);
+		patch_reg = MT_MCU_COM_REG0;
+	}
+
+	if (rom_protect && (mt76_rr(dev, patch_reg) & patch_mask)) {
+		dev_info(dev->mt76.dev, "ROM patch already applied\n");
+		goto out;
+	}
+
+	ret = request_firmware(&fw, MT7662_ROM_PATCH, dev->mt76.dev);
+	if (ret)
+		goto out;
+
+	if (!fw || !fw->data || fw->size <= sizeof(*hdr)) {
+		ret = -EIO;
+		dev_err(dev->mt76.dev, "Failed to load firmware\n");
+		goto out;
+	}
+
+	hdr = (struct mt76x2_patch_header *) fw->data;
+	dev_info(dev->mt76.dev, "ROM patch build: %.15s\n", hdr->build_time);
+
+	mt76_wr(dev, MT_MCU_PCIE_REMAP_BASE4, MT_MCU_ROM_PATCH_OFFSET);
+
+	cur = (__le32 *) (fw->data + sizeof(*hdr));
+	len = fw->size - sizeof(*hdr);
+	mt76_wr_copy(dev, MT_MCU_ROM_PATCH_ADDR, cur, len);
+
+	mt76_wr(dev, MT_MCU_PCIE_REMAP_BASE4, 0);
+
+	/* Trigger ROM */
+	mt76_wr(dev, MT_MCU_INT_LEVEL, 4);
+
+	if (!mt76_poll_msec(dev, patch_reg, patch_mask, patch_mask, 2000)) {
+		dev_err(dev->mt76.dev, "Failed to load ROM patch\n");
+		ret = -ETIMEDOUT;
+	}
+
+out:
+	/* release semaphore */
+	if (rom_protect)
+		mt76_wr(dev, MT_MCU_SEMAPHORE_03, 1);
+	release_firmware(fw);
+	return ret;
+}
+
+static int
+mt76pci_load_firmware(struct mt76x2_dev *dev)
+{
+	const struct firmware *fw;
+	const struct mt76x2_fw_header *hdr;
+	int i, len, ret;
+	__le32 *cur;
+	u32 offset, val;
+
+	ret = request_firmware(&fw, MT7662_FIRMWARE, dev->mt76.dev);
+	if (ret)
+		return ret;
+
+	if (!fw || !fw->data || fw->size < sizeof(*hdr))
+		goto error;
+
+	hdr = (const struct mt76x2_fw_header *) fw->data;
+
+	len = sizeof(*hdr);
+	len += le32_to_cpu(hdr->ilm_len);
+	len += le32_to_cpu(hdr->dlm_len);
+
+	if (fw->size != len)
+		goto error;
+
+	val = le16_to_cpu(hdr->fw_ver);
+	dev_info(dev->mt76.dev, "Firmware Version: %d.%d.%02d\n",
+		 (val >> 12) & 0xf, (val >> 8) & 0xf, val & 0xf);
+
+	val = le16_to_cpu(hdr->build_ver);
+	dev_info(dev->mt76.dev, "Build: %x\n", val);
+	dev_info(dev->mt76.dev, "Build Time: %.16s\n", hdr->build_time);
+
+	cur = (__le32 *) (fw->data + sizeof(*hdr));
+	len = le32_to_cpu(hdr->ilm_len);
+
+	mt76_wr(dev, MT_MCU_PCIE_REMAP_BASE4, MT_MCU_ILM_OFFSET);
+	mt76_wr_copy(dev, MT_MCU_ILM_ADDR, cur, len);
+
+	cur += len / sizeof(*cur);
+	len = le32_to_cpu(hdr->dlm_len);
+
+	if (mt76xx_rev(dev) >= MT76XX_REV_E3)
+		offset = MT_MCU_DLM_ADDR_E3;
+	else
+		offset = MT_MCU_DLM_ADDR;
+
+	mt76_wr(dev, MT_MCU_PCIE_REMAP_BASE4, MT_MCU_DLM_OFFSET);
+	mt76_wr_copy(dev, offset, cur, len);
+
+	mt76_wr(dev, MT_MCU_PCIE_REMAP_BASE4, 0);
+
+	val = mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_2);
+	if (FIELD_GET(MT_EE_NIC_CONF_2_XTAL_OPTION, val) == 1)
+		mt76_set(dev, MT_MCU_COM_REG0, BIT(30));
+
+	/* trigger firmware */
+	mt76_wr(dev, MT_MCU_INT_LEVEL, 2);
+	for (i = 200; i > 0; i--) {
+		val = mt76_rr(dev, MT_MCU_COM_REG0);
+
+		if (val & 1)
+			break;
+
+		msleep(10);
+	}
+
+	if (!i) {
+		dev_err(dev->mt76.dev, "Firmware failed to start\n");
+		release_firmware(fw);
+		return -ETIMEDOUT;
+	}
+
+	dev_info(dev->mt76.dev, "Firmware running!\n");
+
+	release_firmware(fw);
+
+	return ret;
+
+error:
+	dev_err(dev->mt76.dev, "Invalid firmware\n");
+	release_firmware(fw);
+	return -ENOENT;
+}
+
+static int
+mt76x2_mcu_function_select(struct mt76x2_dev *dev, enum mcu_function func,
+			   u32 val)
+{
+	struct sk_buff *skb;
+	struct {
+	    __le32 id;
+	    __le32 value;
+	} __packed __aligned(4) msg = {
+	    .id = cpu_to_le32(func),
+	    .value = cpu_to_le32(val),
+	};
+
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	return mt76x2_mcu_msg_send(dev, skb, CMD_FUN_SET_OP);
+}
+
+int mt76x2_mcu_load_cr(struct mt76x2_dev *dev, u8 type, u8 temp_level,
+		       u8 channel)
+{
+	struct sk_buff *skb;
+	struct {
+		u8 cr_mode;
+		u8 temp;
+		u8 ch;
+		u8 _pad0;
+
+		__le32 cfg;
+	} __packed __aligned(4) msg = {
+		.cr_mode = type,
+		.temp = temp_level,
+		.ch = channel,
+	};
+	u32 val;
+
+	val = BIT(31);
+	val |= (mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_0) >> 8) & 0x00ff;
+	val |= (mt76x2_eeprom_get(dev, MT_EE_NIC_CONF_1) << 8) & 0xff00;
+	msg.cfg = cpu_to_le32(val);
+
+	/* first set the channel without the extension channel info */
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	return mt76x2_mcu_msg_send(dev, skb, CMD_LOAD_CR);
+}
+
+int mt76x2_mcu_set_channel(struct mt76x2_dev *dev, u8 channel, u8 bw,
+			   u8 bw_index, bool scan)
+{
+	struct sk_buff *skb;
+	struct {
+		u8 idx;
+		u8 scan;
+		u8 bw;
+		u8 _pad0;
+
+		__le16 chainmask;
+		u8 ext_chan;
+		u8 _pad1;
+
+	} __packed __aligned(4) msg = {
+		.idx = channel,
+		.scan = scan,
+		.bw = bw,
+		.chainmask = cpu_to_le16(dev->chainmask),
+	};
+
+	/* first set the channel without the extension channel info */
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	mt76x2_mcu_msg_send(dev, skb, CMD_SWITCH_CHANNEL_OP);
+
+	usleep_range(5000, 10000);
+
+	msg.ext_chan = 0xe0 + bw_index;
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	return mt76x2_mcu_msg_send(dev, skb, CMD_SWITCH_CHANNEL_OP);
+}
+
+int mt76x2_mcu_set_radio_state(struct mt76x2_dev *dev, bool on)
+{
+	struct sk_buff *skb;
+	struct {
+		__le32 mode;
+		__le32 level;
+	} __packed __aligned(4) msg = {
+		.mode = cpu_to_le32(on ? RADIO_ON : RADIO_OFF),
+		.level = cpu_to_le32(0),
+	};
+
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	return mt76x2_mcu_msg_send(dev, skb, CMD_POWER_SAVING_OP);
+}
+
+int mt76x2_mcu_calibrate(struct mt76x2_dev *dev, enum mcu_calibration type,
+			 u32 param)
+{
+	struct sk_buff *skb;
+	struct {
+		__le32 id;
+		__le32 value;
+	} __packed __aligned(4) msg = {
+		.id = cpu_to_le32(type),
+		.value = cpu_to_le32(param),
+	};
+	int ret;
+
+	mt76_clear(dev, MT_MCU_COM_REG0, BIT(31));
+
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	ret = mt76x2_mcu_msg_send(dev, skb, CMD_CALIBRATION_OP);
+	if (ret)
+		return ret;
+
+	if (WARN_ON(!mt76_poll_msec(dev, MT_MCU_COM_REG0,
+				    BIT(31), BIT(31), 100)))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+int mt76x2_mcu_tssi_comp(struct mt76x2_dev *dev,
+			 struct mt76x2_tssi_comp *tssi_data)
+{
+	struct sk_buff *skb;
+	struct {
+		__le32 id;
+		struct mt76x2_tssi_comp data;
+	} __packed __aligned(4) msg = {
+		.id = cpu_to_le32(MCU_CAL_TSSI_COMP),
+		.data = *tssi_data,
+	};
+
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	return mt76x2_mcu_msg_send(dev, skb, CMD_CALIBRATION_OP);
+}
+
+int mt76x2_mcu_init_gain(struct mt76x2_dev *dev, u8 channel, u32 gain,
+			 bool force)
+{
+	struct sk_buff *skb;
+	struct {
+		__le32 channel;
+		__le32 gain_val;
+	} __packed __aligned(4) msg = {
+		.channel = cpu_to_le32(channel),
+		.gain_val = cpu_to_le32(gain),
+	};
+
+	if (force)
+		msg.channel |= cpu_to_le32(BIT(31));
+
+	skb = mt76x2_mcu_msg_alloc(&msg, sizeof(msg));
+	return mt76x2_mcu_msg_send(dev, skb, CMD_INIT_GAIN_OP);
+}
+
+int mt76x2_mcu_init(struct mt76x2_dev *dev)
+{
+	int ret;
+
+	mutex_init(&dev->mcu.mutex);
+
+	ret = mt76pci_load_rom_patch(dev);
+	if (ret)
+		return ret;
+
+	ret = mt76pci_load_firmware(dev);
+	if (ret)
+		return ret;
+
+	mt76x2_mcu_function_select(dev, Q_SELECT, 1);
+	return 0;
+}
+
+int mt76x2_mcu_cleanup(struct mt76x2_dev *dev)
+{
+	struct sk_buff *skb;
+
+	mt76_wr(dev, MT_MCU_INT_LEVEL, 1);
+	usleep_range(20000, 30000);
+
+	while ((skb = skb_dequeue(&dev->mcu.res_q)) != NULL)
+		dev_kfree_skb(skb);
+
+	return 0;
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76x2_mcu.h
new file mode 100644
index 000000000000..d7a7e83262ce
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_mcu.h
@@ -0,0 +1,155 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76x2_MCU_H
+#define __MT76x2_MCU_H
+
+/* Register definitions */
+#define MT_MCU_CPU_CTL			0x0704
+#define MT_MCU_CLOCK_CTL		0x0708
+#define MT_MCU_RESET_CTL		0x070C
+#define MT_MCU_INT_LEVEL		0x0718
+#define MT_MCU_COM_REG0			0x0730
+#define MT_MCU_COM_REG1			0x0734
+#define MT_MCU_COM_REG2			0x0738
+#define MT_MCU_COM_REG3			0x073C
+#define MT_MCU_PCIE_REMAP_BASE1		0x0740
+#define MT_MCU_PCIE_REMAP_BASE2		0x0744
+#define MT_MCU_PCIE_REMAP_BASE3		0x0748
+#define MT_MCU_PCIE_REMAP_BASE4		0x074C
+
+#define MT_LED_CTRL			0x0770
+#define MT_LED_CTRL_REPLAY(_n)		BIT(0 + (8 * (_n)))
+#define MT_LED_CTRL_POLARITY(_n)	BIT(1 + (8 * (_n)))
+#define MT_LED_CTRL_TX_BLINK_MODE(_n)	BIT(2 + (8 * (_n)))
+#define MT_LED_CTRL_KICK(_n)		BIT(7 + (8 * (_n)))
+
+#define MT_LED_TX_BLINK_0		0x0774
+#define MT_LED_TX_BLINK_1		0x0778
+
+#define MT_LED_S0_BASE			0x077C
+#define MT_LED_S0(_n)			(MT_LED_S0_BASE + 8 * (_n))
+#define MT_LED_S1_BASE			0x0780
+#define MT_LED_S1(_n)			(MT_LED_S1_BASE + 8 * (_n))
+#define MT_LED_STATUS_OFF_MASK		GENMASK(31, 24)
+#define MT_LED_STATUS_OFF(_v)		(((_v) << __ffs(MT_LED_STATUS_OFF_MASK)) & \
+					 MT_LED_STATUS_OFF_MASK)
+#define MT_LED_STATUS_ON_MASK		GENMASK(23, 16)
+#define MT_LED_STATUS_ON(_v)		(((_v) << __ffs(MT_LED_STATUS_ON_MASK)) & \
+					 MT_LED_STATUS_ON_MASK)
+#define MT_LED_STATUS_DURATION_MASK	GENMASK(15, 8)
+#define MT_LED_STATUS_DURATION(_v)	(((_v) << __ffs(MT_LED_STATUS_DURATION_MASK)) & \
+					 MT_LED_STATUS_DURATION_MASK)
+
+#define MT_MCU_SEMAPHORE_00		0x07B0
+#define MT_MCU_SEMAPHORE_01		0x07B4
+#define MT_MCU_SEMAPHORE_02		0x07B8
+#define MT_MCU_SEMAPHORE_03		0x07BC
+
+#define MT_MCU_ROM_PATCH_OFFSET		0x80000
+#define MT_MCU_ROM_PATCH_ADDR		0x90000
+
+#define MT_MCU_ILM_OFFSET		0x80000
+#define MT_MCU_ILM_ADDR			0x80000
+
+#define MT_MCU_DLM_OFFSET		0x100000
+#define MT_MCU_DLM_ADDR			0x90000
+#define MT_MCU_DLM_ADDR_E3		0x90800
+
+enum mcu_cmd {
+	CMD_FUN_SET_OP = 1,
+	CMD_LOAD_CR = 2,
+	CMD_INIT_GAIN_OP = 3,
+	CMD_DYNC_VGA_OP = 6,
+	CMD_TDLS_CH_SW = 7,
+	CMD_BURST_WRITE = 8,
+	CMD_READ_MODIFY_WRITE = 9,
+	CMD_RANDOM_READ = 10,
+	CMD_BURST_READ = 11,
+	CMD_RANDOM_WRITE = 12,
+	CMD_LED_MODE_OP = 16,
+	CMD_POWER_SAVING_OP = 20,
+	CMD_WOW_CONFIG = 21,
+	CMD_WOW_QUERY = 22,
+	CMD_WOW_FEATURE = 24,
+	CMD_CARRIER_DETECT_OP = 28,
+	CMD_RADOR_DETECT_OP = 29,
+	CMD_SWITCH_CHANNEL_OP = 30,
+	CMD_CALIBRATION_OP = 31,
+	CMD_BEACON_OP = 32,
+	CMD_ANTENNA_OP = 33,
+};
+
+enum mcu_function {
+	Q_SELECT = 1,
+	BW_SETTING = 2,
+	USB2_SW_DISCONNECT = 2,
+	USB3_SW_DISCONNECT = 3,
+	LOG_FW_DEBUG_MSG = 4,
+	GET_FW_VERSION = 5,
+};
+
+enum mcu_power_mode {
+	RADIO_OFF = 0x30,
+	RADIO_ON = 0x31,
+	RADIO_OFF_AUTO_WAKEUP = 0x32,
+	RADIO_OFF_ADVANCE = 0x33,
+	RADIO_ON_ADVANCE = 0x34,
+};
+
+enum mcu_calibration {
+	MCU_CAL_R = 1,
+	MCU_CAL_TEMP_SENSOR,
+	MCU_CAL_RXDCOC,
+	MCU_CAL_RC,
+	MCU_CAL_SX_LOGEN,
+	MCU_CAL_LC,
+	MCU_CAL_TX_LOFT,
+	MCU_CAL_TXIQ,
+	MCU_CAL_TSSI,
+	MCU_CAL_TSSI_COMP,
+	MCU_CAL_DPD,
+	MCU_CAL_RXIQC_FI,
+	MCU_CAL_RXIQC_FD,
+	MCU_CAL_PWRON,
+	MCU_CAL_TX_SHAPING,
+};
+
+enum mt76x2_mcu_cr_mode {
+	MT_RF_CR,
+	MT_BBP_CR,
+	MT_RF_BBP_CR,
+	MT_HL_TEMP_CR_UPDATE,
+};
+
+struct mt76x2_tssi_comp {
+	u8 pa_mode;
+	u8 cal_mode;
+	u16 pad;
+
+	u8 slope0;
+	u8 slope1;
+	u8 offset0;
+	u8 offset1;
+} __packed __aligned(4);
+
+int mt76x2_mcu_calibrate(struct mt76x2_dev *dev, enum mcu_calibration type,
+			 u32 param);
+int mt76x2_mcu_tssi_comp(struct mt76x2_dev *dev, struct mt76x2_tssi_comp *tssi_data);
+int mt76x2_mcu_init_gain(struct mt76x2_dev *dev, u8 channel, u32 gain,
+			 bool force);
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_pci.c b/drivers/net/wireless/mediatek/mt76/mt76x2_pci.c
new file mode 100644
index 000000000000..e66f047ea448
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_pci.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "mt76x2.h"
+#include "mt76x2_trace.h"
+
+static const struct pci_device_id mt76pci_device_table[] = {
+	{ PCI_DEVICE(0x14c3, 0x7662) },
+	{ PCI_DEVICE(0x14c3, 0x7612) },
+	{ PCI_DEVICE(0x14c3, 0x7602) },
+	{ },
+};
+
+static int
+mt76pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct mt76x2_dev *dev;
+	int ret;
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
+	if (ret)
+		return ret;
+
+	pci_set_master(pdev);
+
+	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
+	dev = mt76x2_alloc_device(&pdev->dev);
+	if (!dev)
+		return -ENOMEM;
+
+	mt76_mmio_init(&dev->mt76, pcim_iomap_table(pdev)[0]);
+
+	dev->mt76.rev = mt76_rr(dev, MT_ASIC_VERSION);
+	dev_info(dev->mt76.dev, "ASIC revision: %08x\n", dev->mt76.rev);
+
+	ret = devm_request_irq(dev->mt76.dev, pdev->irq, mt76x2_irq_handler,
+			       IRQF_SHARED, KBUILD_MODNAME, dev);
+	if (ret)
+		goto error;
+
+	ret = mt76x2_register_device(dev);
+	if (ret)
+		goto error;
+
+	/* Fix up ASPM configuration */
+
+	/* RG_SSUSB_G1_CDR_BIR_LTR = 0x9 */
+	mt76_rmw_field(dev, 0x15a10, 0x1f << 16, 0x9);
+
+	/* RG_SSUSB_G1_CDR_BIC_LTR = 0xf */
+	mt76_rmw_field(dev, 0x15a0c, 0xf << 28, 0xf);
+
+	/* RG_SSUSB_CDR_BR_PE1D = 0x3 */
+	mt76_rmw_field(dev, 0x15c58, 0x3 << 6, 0x3);
+
+	return 0;
+
+error:
+	ieee80211_free_hw(mt76_hw(dev));
+	return ret;
+}
+
+static void
+mt76pci_remove(struct pci_dev *pdev)
+{
+	struct mt76_dev *mdev = pci_get_drvdata(pdev);
+	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
+
+	mt76_unregister_device(mdev);
+	mt76x2_cleanup(dev);
+	ieee80211_free_hw(mdev->hw);
+}
+
+MODULE_DEVICE_TABLE(pci, mt76pci_device_table);
+MODULE_FIRMWARE(MT7662_FIRMWARE);
+MODULE_FIRMWARE(MT7662_ROM_PATCH);
+MODULE_LICENSE("Dual BSD/GPL");
+
+static struct pci_driver mt76pci_driver = {
+	.name		= KBUILD_MODNAME,
+	.id_table	= mt76pci_device_table,
+	.probe		= mt76pci_probe,
+	.remove		= mt76pci_remove,
+};
+
+module_pci_driver(mt76pci_driver);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_phy.c b/drivers/net/wireless/mediatek/mt76/mt76x2_phy.c
new file mode 100644
index 000000000000..5b742749d5de
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_phy.c
@@ -0,0 +1,740 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include "mt76x2.h"
+#include "mt76x2_mcu.h"
+#include "mt76x2_eeprom.h"
+
+static void
+mt76x2_adjust_high_lna_gain(struct mt76x2_dev *dev, int reg, s8 offset)
+{
+	s8 gain;
+
+	gain = FIELD_GET(MT_BBP_AGC_LNA_HIGH_GAIN, mt76_rr(dev, MT_BBP(AGC, reg)));
+	gain -= offset / 2;
+	mt76_rmw_field(dev, MT_BBP(AGC, reg), MT_BBP_AGC_LNA_HIGH_GAIN, gain);
+}
+
+static void
+mt76x2_adjust_agc_gain(struct mt76x2_dev *dev, int reg, s8 offset)
+{
+	s8 gain;
+
+	gain = FIELD_GET(MT_BBP_AGC_GAIN, mt76_rr(dev, MT_BBP(AGC, reg)));
+	gain += offset;
+	mt76_rmw_field(dev, MT_BBP(AGC, reg), MT_BBP_AGC_GAIN, gain);
+}
+
+static void
+mt76x2_apply_gain_adj(struct mt76x2_dev *dev)
+{
+	s8 *gain_adj = dev->cal.rx.high_gain;
+
+	mt76x2_adjust_high_lna_gain(dev, 4, gain_adj[0]);
+	mt76x2_adjust_high_lna_gain(dev, 5, gain_adj[1]);
+
+	mt76x2_adjust_agc_gain(dev, 8, gain_adj[0]);
+	mt76x2_adjust_agc_gain(dev, 9, gain_adj[1]);
+}
+
+static u32
+mt76x2_tx_power_mask(u8 v1, u8 v2, u8 v3, u8 v4)
+{
+	u32 val = 0;
+
+	val |= (v1 & (BIT(6) - 1)) << 0;
+	val |= (v2 & (BIT(6) - 1)) << 8;
+	val |= (v3 & (BIT(6) - 1)) << 16;
+	val |= (v4 & (BIT(6) - 1)) << 24;
+	return val;
+}
+
+int mt76x2_phy_get_rssi(struct mt76x2_dev *dev, s8 rssi, int chain)
+{
+	struct mt76x2_rx_freq_cal *cal = &dev->cal.rx;
+
+	rssi += cal->rssi_offset[chain];
+	rssi -= cal->lna_gain;
+
+	return rssi;
+}
+
+static u8
+mt76x2_txpower_check(int value)
+{
+	if (value < 0)
+		return 0;
+	if (value > 0x2f)
+		return 0x2f;
+	return value;
+}
+
+static void
+mt76x2_add_rate_power_offset(struct mt76_rate_power *r, int offset)
+{
+	int i;
+
+	for (i = 0; i < sizeof(r->all); i++)
+		r->all[i] += offset;
+}
+
+static void
+mt76x2_limit_rate_power(struct mt76_rate_power *r, int limit)
+{
+	int i;
+
+	for (i = 0; i < sizeof(r->all); i++)
+		if (r->all[i] > limit)
+			r->all[i] = limit;
+}
+
+void mt76x2_phy_set_txpower(struct mt76x2_dev *dev)
+{
+	enum nl80211_chan_width width = dev->mt76.chandef.width;
+	struct ieee80211_channel *chan = dev->mt76.chandef.chan;
+	struct mt76x2_tx_power_info txp;
+	int txp_0, txp_1, delta = 0;
+	struct mt76_rate_power t = {};
+
+	mt76x2_get_power_info(dev, &txp, chan);
+
+	if (width == NL80211_CHAN_WIDTH_40)
+		delta = txp.delta_bw40;
+	else if (width == NL80211_CHAN_WIDTH_80)
+		delta = txp.delta_bw80;
+
+	if (txp.target_power > dev->txpower_conf)
+		delta -= txp.target_power - dev->txpower_conf;
+
+	mt76x2_get_rate_power(dev, &t, chan);
+	mt76x2_add_rate_power_offset(&t, txp.chain[0].target_power +
+				   txp.chain[0].delta);
+	mt76x2_limit_rate_power(&t, dev->txpower_conf);
+	dev->txpower_cur = mt76x2_get_max_rate_power(&t);
+	mt76x2_add_rate_power_offset(&t, -(txp.chain[0].target_power +
+					 txp.chain[0].delta + delta));
+	dev->target_power = txp.chain[0].target_power;
+	dev->target_power_delta[0] = txp.chain[0].delta + delta;
+	dev->target_power_delta[1] = txp.chain[1].delta + delta;
+	dev->rate_power = t;
+
+	txp_0 = mt76x2_txpower_check(txp.chain[0].target_power +
+				   txp.chain[0].delta + delta);
+
+	txp_1 = mt76x2_txpower_check(txp.chain[1].target_power +
+				   txp.chain[1].delta + delta);
+
+	mt76_rmw_field(dev, MT_TX_ALC_CFG_0, MT_TX_ALC_CFG_0_CH_INIT_0, txp_0);
+	mt76_rmw_field(dev, MT_TX_ALC_CFG_0, MT_TX_ALC_CFG_0_CH_INIT_1, txp_1);
+
+	mt76_wr(dev, MT_TX_PWR_CFG_0,
+		mt76x2_tx_power_mask(t.cck[0], t.cck[2], t.ofdm[0], t.ofdm[2]));
+	mt76_wr(dev, MT_TX_PWR_CFG_1,
+		mt76x2_tx_power_mask(t.ofdm[4], t.ofdm[6], t.ht[0], t.ht[2]));
+	mt76_wr(dev, MT_TX_PWR_CFG_2,
+		mt76x2_tx_power_mask(t.ht[4], t.ht[6], t.ht[8], t.ht[10]));
+	mt76_wr(dev, MT_TX_PWR_CFG_3,
+		mt76x2_tx_power_mask(t.ht[12], t.ht[14], t.ht[0], t.ht[2]));
+	mt76_wr(dev, MT_TX_PWR_CFG_4,
+		mt76x2_tx_power_mask(t.ht[4], t.ht[6], 0, 0));
+	mt76_wr(dev, MT_TX_PWR_CFG_7,
+		mt76x2_tx_power_mask(t.ofdm[6], t.vht[8], t.ht[6], t.vht[8]));
+	mt76_wr(dev, MT_TX_PWR_CFG_8,
+		mt76x2_tx_power_mask(t.ht[14], t.vht[8], t.vht[8], 0));
+	mt76_wr(dev, MT_TX_PWR_CFG_9,
+		mt76x2_tx_power_mask(t.ht[6], t.vht[8], t.vht[8], 0));
+}
+
+static bool
+mt76x2_channel_silent(struct mt76x2_dev *dev)
+{
+	struct ieee80211_channel *chan = dev->mt76.chandef.chan;
+
+	return ((chan->flags & IEEE80211_CHAN_RADAR) &&
+		chan->dfs_state != NL80211_DFS_AVAILABLE);
+}
+
+static bool
+mt76x2_phy_tssi_init_cal(struct mt76x2_dev *dev)
+{
+	struct ieee80211_channel *chan = dev->mt76.chandef.chan;
+	u32 flag = 0;
+
+	if (!mt76x2_tssi_enabled(dev))
+		return false;
+
+	if (mt76x2_channel_silent(dev))
+		return false;
+
+	if (chan->band == NL80211_BAND_2GHZ)
+		flag |= BIT(0);
+
+	if (mt76x2_ext_pa_enabled(dev, chan->band))
+		flag |= BIT(8);
+
+	mt76x2_mcu_calibrate(dev, MCU_CAL_TSSI, flag);
+	dev->cal.tssi_cal_done = true;
+	return true;
+}
+
+static void
+mt76x2_phy_channel_calibrate(struct mt76x2_dev *dev, bool mac_stopped)
+{
+	struct ieee80211_channel *chan = dev->mt76.chandef.chan;
+	bool is_5ghz = chan->band == NL80211_BAND_5GHZ;
+
+	if (dev->cal.channel_cal_done)
+		return;
+
+	if (mt76x2_channel_silent(dev))
+		return;
+
+	if (!dev->cal.tssi_cal_done)
+		mt76x2_phy_tssi_init_cal(dev);
+
+	if (!mac_stopped)
+		mt76x2_mac_stop(dev, false);
+
+	if (is_5ghz)
+		mt76x2_mcu_calibrate(dev, MCU_CAL_LC, 0);
+
+	mt76x2_mcu_calibrate(dev, MCU_CAL_TX_LOFT, is_5ghz);
+	mt76x2_mcu_calibrate(dev, MCU_CAL_TXIQ, is_5ghz);
+	mt76x2_mcu_calibrate(dev, MCU_CAL_RXIQC_FI, is_5ghz);
+	mt76x2_mcu_calibrate(dev, MCU_CAL_TEMP_SENSOR, 0);
+	mt76x2_mcu_calibrate(dev, MCU_CAL_TX_SHAPING, 0);
+
+	if (!mac_stopped)
+		mt76x2_mac_resume(dev);
+
+	mt76x2_apply_gain_adj(dev);
+
+	dev->cal.channel_cal_done = true;
+}
+
+static void
+mt76x2_phy_set_txpower_regs(struct mt76x2_dev *dev, enum nl80211_band band)
+{
+	u32 pa_mode[2];
+	u32 pa_mode_adj;
+
+	if (band == NL80211_BAND_2GHZ) {
+		pa_mode[0] = 0x010055ff;
+		pa_mode[1] = 0x00550055;
+
+		mt76_wr(dev, MT_TX_ALC_CFG_2, 0x35160a00);
+		mt76_wr(dev, MT_TX_ALC_CFG_3, 0x35160a06);
+
+		if (mt76x2_ext_pa_enabled(dev, band)) {
+			mt76_wr(dev, MT_RF_PA_MODE_ADJ0, 0x0000ec00);
+			mt76_wr(dev, MT_RF_PA_MODE_ADJ1, 0x0000ec00);
+		} else {
+			mt76_wr(dev, MT_RF_PA_MODE_ADJ0, 0xf4000200);
+			mt76_wr(dev, MT_RF_PA_MODE_ADJ1, 0xfa000200);
+		}
+	} else {
+		pa_mode[0] = 0x0000ffff;
+		pa_mode[1] = 0x00ff00ff;
+
+		if (mt76x2_ext_pa_enabled(dev, band)) {
+			mt76_wr(dev, MT_TX_ALC_CFG_2, 0x2f0f0400);
+			mt76_wr(dev, MT_TX_ALC_CFG_3, 0x2f0f0476);
+		} else {
+			mt76_wr(dev, MT_TX_ALC_CFG_2, 0x1b0f0400);
+			mt76_wr(dev, MT_TX_ALC_CFG_3, 0x1b0f0476);
+		}
+		mt76_wr(dev, MT_TX_ALC_CFG_4, 0);
+
+		if (mt76x2_ext_pa_enabled(dev, band))
+			pa_mode_adj = 0x04000000;
+		else
+			pa_mode_adj = 0;
+
+		mt76_wr(dev, MT_RF_PA_MODE_ADJ0, pa_mode_adj);
+		mt76_wr(dev, MT_RF_PA_MODE_ADJ1, pa_mode_adj);
+	}
+
+	mt76_wr(dev, MT_BB_PA_MODE_CFG0, pa_mode[0]);
+	mt76_wr(dev, MT_BB_PA_MODE_CFG1, pa_mode[1]);
+	mt76_wr(dev, MT_RF_PA_MODE_CFG0, pa_mode[0]);
+	mt76_wr(dev, MT_RF_PA_MODE_CFG1, pa_mode[1]);
+
+	if (mt76x2_ext_pa_enabled(dev, band)) {
+		u32 val;
+
+		if (band == NL80211_BAND_2GHZ)
+			val = 0x3c3c023c;
+		else
+			val = 0x363c023c;
+
+		mt76_wr(dev, MT_TX0_RF_GAIN_CORR, val);
+		mt76_wr(dev, MT_TX1_RF_GAIN_CORR, val);
+		mt76_wr(dev, MT_TX_ALC_CFG_4, 0x00001818);
+	} else {
+		if (band == NL80211_BAND_2GHZ) {
+			u32 val = 0x0f3c3c3c;
+
+			mt76_wr(dev, MT_TX0_RF_GAIN_CORR, val);
+			mt76_wr(dev, MT_TX1_RF_GAIN_CORR, val);
+			mt76_wr(dev, MT_TX_ALC_CFG_4, 0x00000606);
+		} else {
+			mt76_wr(dev, MT_TX0_RF_GAIN_CORR, 0x383c023c);
+			mt76_wr(dev, MT_TX1_RF_GAIN_CORR, 0x24282e28);
+			mt76_wr(dev, MT_TX_ALC_CFG_4, 0);
+		}
+	}
+}
+
+static void
+mt76x2_configure_tx_delay(struct mt76x2_dev *dev, enum nl80211_band band, u8 bw)
+{
+	u32 cfg0, cfg1;
+
+	if (mt76x2_ext_pa_enabled(dev, band)) {
+		cfg0 = bw ? 0x000b0c01 : 0x00101101;
+		cfg1 = 0x00011414;
+	} else {
+		cfg0 = bw ? 0x000b0b01 : 0x00101001;
+		cfg1 = 0x00021414;
+	}
+	mt76_wr(dev, MT_TX_SW_CFG0, cfg0);
+	mt76_wr(dev, MT_TX_SW_CFG1, cfg1);
+
+	mt76_rmw_field(dev, MT_XIFS_TIME_CFG, MT_XIFS_TIME_CFG_OFDM_SIFS, 15);
+}
+
+static void
+mt76x2_phy_set_bw(struct mt76x2_dev *dev, int width, u8 ctrl)
+{
+	int core_val, agc_val;
+
+	switch (width) {
+	case NL80211_CHAN_WIDTH_80:
+		core_val = 3;
+		agc_val = 7;
+		break;
+	case NL80211_CHAN_WIDTH_40:
+		core_val = 2;
+		agc_val = 3;
+		break;
+	default:
+		core_val = 0;
+		agc_val = 1;
+		break;
+	}
+
+	mt76_rmw_field(dev, MT_BBP(CORE, 1), MT_BBP_CORE_R1_BW, core_val);
+	mt76_rmw_field(dev, MT_BBP(AGC, 0), MT_BBP_AGC_R0_BW, agc_val);
+	mt76_rmw_field(dev, MT_BBP(AGC, 0), MT_BBP_AGC_R0_CTRL_CHAN, ctrl);
+	mt76_rmw_field(dev, MT_BBP(TXBE, 0), MT_BBP_TXBE_R0_CTRL_CHAN, ctrl);
+}
+
+static void
+mt76x2_phy_set_band(struct mt76x2_dev *dev, int band, bool primary_upper)
+{
+	switch (band) {
+	case NL80211_BAND_2GHZ:
+		mt76_set(dev, MT_TX_BAND_CFG, MT_TX_BAND_CFG_2G);
+		mt76_clear(dev, MT_TX_BAND_CFG, MT_TX_BAND_CFG_5G);
+		break;
+	case NL80211_BAND_5GHZ:
+		mt76_clear(dev, MT_TX_BAND_CFG, MT_TX_BAND_CFG_2G);
+		mt76_set(dev, MT_TX_BAND_CFG, MT_TX_BAND_CFG_5G);
+		break;
+	}
+
+	mt76_rmw_field(dev, MT_TX_BAND_CFG, MT_TX_BAND_CFG_UPPER_40M,
+		       primary_upper);
+}
+
+static void
+mt76x2_set_rx_chains(struct mt76x2_dev *dev)
+{
+	u32 val;
+
+	val = mt76_rr(dev, MT_BBP(AGC, 0));
+	val &= ~(BIT(3) | BIT(4));
+
+	if (dev->chainmask & BIT(1))
+		val |= BIT(3);
+
+	mt76_wr(dev, MT_BBP(AGC, 0), val);
+}
+
+static void
+mt76x2_set_tx_dac(struct mt76x2_dev *dev)
+{
+	if (dev->chainmask & BIT(1))
+		mt76_set(dev, MT_BBP(TXBE, 5), 3);
+	else
+		mt76_clear(dev, MT_BBP(TXBE, 5), 3);
+}
+
+static void
+mt76x2_get_agc_gain(struct mt76x2_dev *dev, u8 *dest)
+{
+	dest[0] = mt76_get_field(dev, MT_BBP(AGC, 8), MT_BBP_AGC_GAIN);
+	dest[1] = mt76_get_field(dev, MT_BBP(AGC, 9), MT_BBP_AGC_GAIN);
+}
+
+static int
+mt76x2_get_rssi_gain_thresh(struct mt76x2_dev *dev)
+{
+	switch (dev->mt76.chandef.width) {
+	case NL80211_CHAN_WIDTH_80:
+		return -62;
+	case NL80211_CHAN_WIDTH_40:
+		return -65;
+	default:
+		return -68;
+	}
+}
+
+static int
+mt76x2_get_low_rssi_gain_thresh(struct mt76x2_dev *dev)
+{
+	switch (dev->mt76.chandef.width) {
+	case NL80211_CHAN_WIDTH_80:
+		return -76;
+	case NL80211_CHAN_WIDTH_40:
+		return -79;
+	default:
+		return -82;
+	}
+}
+
+static void
+mt76x2_phy_set_gain_val(struct mt76x2_dev *dev)
+{
+	u32 val;
+	u8 gain_val[2];
+
+	gain_val[0] = dev->cal.agc_gain_cur[0] - dev->cal.agc_gain_adjust;
+	gain_val[1] = dev->cal.agc_gain_cur[1] - dev->cal.agc_gain_adjust;
+
+	if (dev->mt76.chandef.width >= NL80211_CHAN_WIDTH_40)
+		val = 0x1e42 << 16;
+	else
+		val = 0x1836 << 16;
+
+	val |= 0xf8;
+
+	mt76_wr(dev, MT_BBP(AGC, 8),
+		val | FIELD_PREP(MT_BBP_AGC_GAIN, gain_val[0]));
+	mt76_wr(dev, MT_BBP(AGC, 9),
+		val | FIELD_PREP(MT_BBP_AGC_GAIN, gain_val[1]));
+
+	if (dev->mt76.chandef.chan->flags & IEEE80211_CHAN_RADAR)
+		mt76x2_dfs_adjust_agc(dev);
+}
+
+static void
+mt76x2_phy_adjust_vga_gain(struct mt76x2_dev *dev)
+{
+	u32 false_cca;
+	u8 limit = dev->cal.low_gain > 1 ? 4 : 16;
+
+	false_cca = FIELD_GET(MT_RX_STAT_1_CCA_ERRORS, mt76_rr(dev, MT_RX_STAT_1));
+	if (false_cca > 800 && dev->cal.agc_gain_adjust < limit)
+		dev->cal.agc_gain_adjust += 2;
+	else if (false_cca < 10 && dev->cal.agc_gain_adjust > 0)
+		dev->cal.agc_gain_adjust -= 2;
+	else
+		return;
+
+	mt76x2_phy_set_gain_val(dev);
+}
+
+static void
+mt76x2_phy_update_channel_gain(struct mt76x2_dev *dev)
+{
+	u32 val = mt76_rr(dev, MT_BBP(AGC, 20));
+	int rssi0 = (s8) FIELD_GET(MT_BBP_AGC20_RSSI0, val);
+	int rssi1 = (s8) FIELD_GET(MT_BBP_AGC20_RSSI1, val);
+	u8 *gain = dev->cal.agc_gain_init;
+	u8 gain_delta;
+	int low_gain;
+
+	dev->cal.avg_rssi[0] = (dev->cal.avg_rssi[0] * 15) / 16 + (rssi0 << 8);
+	dev->cal.avg_rssi[1] = (dev->cal.avg_rssi[1] * 15) / 16 + (rssi1 << 8);
+	dev->cal.avg_rssi_all = (dev->cal.avg_rssi[0] +
+				 dev->cal.avg_rssi[1]) / 512;
+
+	low_gain = (dev->cal.avg_rssi_all > mt76x2_get_rssi_gain_thresh(dev)) +
+		   (dev->cal.avg_rssi_all > mt76x2_get_low_rssi_gain_thresh(dev));
+
+	if (dev->cal.low_gain == low_gain) {
+		mt76x2_phy_adjust_vga_gain(dev);
+		return;
+	}
+
+	dev->cal.low_gain = low_gain;
+
+	if (dev->mt76.chandef.width == NL80211_CHAN_WIDTH_80)
+		mt76_wr(dev, MT_BBP(RXO, 14), 0x00560211);
+	else
+		mt76_wr(dev, MT_BBP(RXO, 14), 0x00560423);
+
+	if (low_gain) {
+		mt76_wr(dev, MT_BBP(RXO, 18), 0xf000a991);
+		mt76_wr(dev, MT_BBP(AGC, 35), 0x08080808);
+		mt76_wr(dev, MT_BBP(AGC, 37), 0x08080808);
+		if (mt76x2_has_ext_lna(dev))
+			gain_delta = 10;
+		else
+			gain_delta = 14;
+	} else {
+		mt76_wr(dev, MT_BBP(RXO, 18), 0xf000a990);
+		if (dev->mt76.chandef.width == NL80211_CHAN_WIDTH_80)
+			mt76_wr(dev, MT_BBP(AGC, 35), 0x10101014);
+		else
+			mt76_wr(dev, MT_BBP(AGC, 35), 0x11111116);
+		mt76_wr(dev, MT_BBP(AGC, 37), 0x2121262C);
+		gain_delta = 0;
+	}
+
+	dev->cal.agc_gain_cur[0] = gain[0] - gain_delta;
+	dev->cal.agc_gain_cur[1] = gain[1] - gain_delta;
+	dev->cal.agc_gain_adjust = 0;
+	mt76x2_phy_set_gain_val(dev);
+}
+
+int mt76x2_phy_set_channel(struct mt76x2_dev *dev,
+			   struct cfg80211_chan_def *chandef)
+{
+	struct ieee80211_channel *chan = chandef->chan;
+	bool scan = test_bit(MT76_SCANNING, &dev->mt76.state);
+	enum nl80211_band band = chan->band;
+	u8 channel;
+
+	u32 ext_cca_chan[4] = {
+		[0] = FIELD_PREP(MT_EXT_CCA_CFG_CCA0, 0) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA1, 1) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA2, 2) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA3, 3) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA_MASK, BIT(0)),
+		[1] = FIELD_PREP(MT_EXT_CCA_CFG_CCA0, 1) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA1, 0) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA2, 2) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA3, 3) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA_MASK, BIT(1)),
+		[2] = FIELD_PREP(MT_EXT_CCA_CFG_CCA0, 2) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA1, 3) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA2, 1) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA3, 0) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA_MASK, BIT(2)),
+		[3] = FIELD_PREP(MT_EXT_CCA_CFG_CCA0, 3) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA1, 2) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA2, 1) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA3, 0) |
+		      FIELD_PREP(MT_EXT_CCA_CFG_CCA_MASK, BIT(3)),
+	};
+	int ch_group_index;
+	u8 bw, bw_index;
+	int freq, freq1;
+	int ret;
+
+	dev->cal.channel_cal_done = false;
+	freq = chandef->chan->center_freq;
+	freq1 = chandef->center_freq1;
+	channel = chan->hw_value;
+
+	switch (chandef->width) {
+	case NL80211_CHAN_WIDTH_40:
+		bw = 1;
+		if (freq1 > freq) {
+			bw_index = 1;
+			ch_group_index = 0;
+		} else {
+			bw_index = 3;
+			ch_group_index = 1;
+		}
+		channel += 2 - ch_group_index * 4;
+		break;
+	case NL80211_CHAN_WIDTH_80:
+		ch_group_index = (freq - freq1 + 30) / 20;
+		if (WARN_ON(ch_group_index < 0 || ch_group_index > 3))
+			ch_group_index = 0;
+		bw = 2;
+		bw_index = ch_group_index;
+		channel += 6 - ch_group_index * 4;
+		break;
+	default:
+		bw = 0;
+		bw_index = 0;
+		ch_group_index = 0;
+		break;
+	}
+
+	mt76x2_read_rx_gain(dev);
+	mt76x2_phy_set_txpower_regs(dev, band);
+	mt76x2_configure_tx_delay(dev, band, bw);
+	mt76x2_phy_set_txpower(dev);
+
+	mt76x2_set_rx_chains(dev);
+	mt76x2_phy_set_band(dev, chan->band, ch_group_index & 1);
+	mt76x2_phy_set_bw(dev, chandef->width, ch_group_index);
+	mt76x2_set_tx_dac(dev);
+
+	mt76_rmw(dev, MT_EXT_CCA_CFG,
+		 (MT_EXT_CCA_CFG_CCA0 |
+		  MT_EXT_CCA_CFG_CCA1 |
+		  MT_EXT_CCA_CFG_CCA2 |
+		  MT_EXT_CCA_CFG_CCA3 |
+		  MT_EXT_CCA_CFG_CCA_MASK),
+		 ext_cca_chan[ch_group_index]);
+
+	ret = mt76x2_mcu_set_channel(dev, channel, bw, bw_index, scan);
+	if (ret)
+		return ret;
+
+	mt76x2_mcu_init_gain(dev, channel, dev->cal.rx.mcu_gain, true);
+
+	/* Enable LDPC Rx */
+	if (mt76xx_rev(dev) >= MT76XX_REV_E3)
+		mt76_set(dev, MT_BBP(RXO, 13), BIT(10));
+
+	if (!dev->cal.init_cal_done) {
+		u8 val = mt76x2_eeprom_get(dev, MT_EE_BT_RCAL_RESULT);
+
+		if (val != 0xff)
+			mt76x2_mcu_calibrate(dev, MCU_CAL_R, 0);
+	}
+
+	mt76x2_mcu_calibrate(dev, MCU_CAL_RXDCOC, channel);
+
+	/* Rx LPF calibration */
+	if (!dev->cal.init_cal_done)
+		mt76x2_mcu_calibrate(dev, MCU_CAL_RC, 0);
+
+	dev->cal.init_cal_done = true;
+
+	mt76_wr(dev, MT_BBP(AGC, 61), 0xFF64A4E2);
+	mt76_wr(dev, MT_BBP(AGC, 7), 0x08081010);
+	mt76_wr(dev, MT_BBP(AGC, 11), 0x00000404);
+	mt76_wr(dev, MT_BBP(AGC, 2), 0x00007070);
+	mt76_wr(dev, MT_TXOP_CTRL_CFG, 0x04101B3F);
+
+	if (scan)
+		return 0;
+
+	dev->cal.low_gain = -1;
+	mt76x2_phy_channel_calibrate(dev, true);
+	mt76x2_get_agc_gain(dev, dev->cal.agc_gain_init);
+	memcpy(dev->cal.agc_gain_cur, dev->cal.agc_gain_init,
+	       sizeof(dev->cal.agc_gain_cur));
+
+	ieee80211_queue_delayed_work(mt76_hw(dev), &dev->cal_work,
+				     MT_CALIBRATE_INTERVAL);
+
+	return 0;
+}
+
+static void
+mt76x2_phy_tssi_compensate(struct mt76x2_dev *dev)
+{
+	struct ieee80211_channel *chan = dev->mt76.chandef.chan;
+	struct mt76x2_tx_power_info txp;
+	struct mt76x2_tssi_comp t = {};
+
+	if (!dev->cal.tssi_cal_done)
+		return;
+
+	if (!dev->cal.tssi_comp_pending) {
+		/* TSSI trigger */
+		t.cal_mode = BIT(0);
+		mt76x2_mcu_tssi_comp(dev, &t);
+		dev->cal.tssi_comp_pending = true;
+	} else {
+		if (mt76_rr(dev, MT_BBP(CORE, 34)) & BIT(4))
+			return;
+
+		dev->cal.tssi_comp_pending = false;
+		mt76x2_get_power_info(dev, &txp, chan);
+
+		if (mt76x2_ext_pa_enabled(dev, chan->band))
+			t.pa_mode = 1;
+
+		t.cal_mode = BIT(1);
+		t.slope0 = txp.chain[0].tssi_slope;
+		t.offset0 = txp.chain[0].tssi_offset;
+		t.slope1 = txp.chain[1].tssi_slope;
+		t.offset1 = txp.chain[1].tssi_offset;
+		mt76x2_mcu_tssi_comp(dev, &t);
+
+		if (t.pa_mode || dev->cal.dpd_cal_done)
+			return;
+
+		usleep_range(10000, 20000);
+		mt76x2_mcu_calibrate(dev, MCU_CAL_DPD, chan->hw_value);
+		dev->cal.dpd_cal_done = true;
+	}
+}
+
+static void
+mt76x2_phy_temp_compensate(struct mt76x2_dev *dev)
+{
+	struct mt76x2_temp_comp t;
+	int temp, db_diff;
+
+	if (mt76x2_get_temp_comp(dev, &t))
+		return;
+
+	temp = mt76_get_field(dev, MT_TEMP_SENSOR, MT_TEMP_SENSOR_VAL);
+	temp -= t.temp_25_ref;
+	temp = (temp * 1789) / 1000 + 25;
+	dev->cal.temp = temp;
+
+	if (temp > 25)
+		db_diff = (temp - 25) / t.high_slope;
+	else
+		db_diff = (25 - temp) / t.low_slope;
+
+	db_diff = min(db_diff, t.upper_bound);
+	db_diff = max(db_diff, t.lower_bound);
+
+	mt76_rmw_field(dev, MT_TX_ALC_CFG_1, MT_TX_ALC_CFG_1_TEMP_COMP,
+		       db_diff * 2);
+	mt76_rmw_field(dev, MT_TX_ALC_CFG_2, MT_TX_ALC_CFG_2_TEMP_COMP,
+		       db_diff * 2);
+}
+
+void mt76x2_phy_calibrate(struct work_struct *work)
+{
+	struct mt76x2_dev *dev;
+
+	dev = container_of(work, struct mt76x2_dev, cal_work.work);
+	mt76x2_phy_channel_calibrate(dev, false);
+	mt76x2_phy_tssi_compensate(dev);
+	mt76x2_phy_temp_compensate(dev);
+	mt76x2_phy_update_channel_gain(dev);
+	ieee80211_queue_delayed_work(mt76_hw(dev), &dev->cal_work,
+				     MT_CALIBRATE_INTERVAL);
+}
+
+int mt76x2_phy_start(struct mt76x2_dev *dev)
+{
+	int ret;
+
+	ret = mt76x2_mcu_set_radio_state(dev, true);
+	if (ret)
+		return ret;
+
+	mt76x2_mcu_load_cr(dev, MT_RF_BBP_CR, 0, 0);
+
+	return ret;
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_regs.h b/drivers/net/wireless/mediatek/mt76/mt76x2_regs.h
new file mode 100644
index 000000000000..ce3ab85c8b0f
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_regs.h
@@ -0,0 +1,587 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __MT76x2_REGS_H
+#define __MT76x2_REGS_H
+
+#define MT_ASIC_VERSION			0x0000
+
+#define MT76XX_REV_E3		0x22
+#define MT76XX_REV_E4		0x33
+
+#define MT_CMB_CTRL			0x0020
+#define MT_CMB_CTRL_XTAL_RDY		BIT(22)
+#define MT_CMB_CTRL_PLL_LD		BIT(23)
+
+#define MT_EFUSE_CTRL			0x0024
+#define MT_EFUSE_CTRL_AOUT		GENMASK(5, 0)
+#define MT_EFUSE_CTRL_MODE		GENMASK(7, 6)
+#define MT_EFUSE_CTRL_LDO_OFF_TIME	GENMASK(13, 8)
+#define MT_EFUSE_CTRL_LDO_ON_TIME	GENMASK(15, 14)
+#define MT_EFUSE_CTRL_AIN		GENMASK(25, 16)
+#define MT_EFUSE_CTRL_KICK		BIT(30)
+#define MT_EFUSE_CTRL_SEL		BIT(31)
+
+#define MT_EFUSE_DATA_BASE		0x0028
+#define MT_EFUSE_DATA(_n)		(MT_EFUSE_DATA_BASE + ((_n) << 2))
+
+#define MT_COEXCFG0			0x0040
+#define MT_COEXCFG0_COEX_EN		BIT(0)
+
+#define MT_WLAN_FUN_CTRL		0x0080
+#define MT_WLAN_FUN_CTRL_WLAN_EN	BIT(0)
+#define MT_WLAN_FUN_CTRL_WLAN_CLK_EN	BIT(1)
+#define MT_WLAN_FUN_CTRL_WLAN_RESET_RF	BIT(2)
+
+#define MT_WLAN_FUN_CTRL_WLAN_RESET	BIT(3) /* MT76x0 */
+#define MT_WLAN_FUN_CTRL_CSR_F20M_CKEN	BIT(3) /* MT76x2 */
+
+#define MT_WLAN_FUN_CTRL_PCIE_CLK_REQ	BIT(4)
+#define MT_WLAN_FUN_CTRL_FRC_WL_ANT_SEL	BIT(5)
+#define MT_WLAN_FUN_CTRL_INV_ANT_SEL	BIT(6)
+#define MT_WLAN_FUN_CTRL_WAKE_HOST	BIT(7)
+
+#define MT_WLAN_FUN_CTRL_THERM_RST	BIT(8) /* MT76x2 */
+#define MT_WLAN_FUN_CTRL_THERM_CKEN	BIT(9) /* MT76x2 */
+
+#define MT_WLAN_FUN_CTRL_GPIO_IN	GENMASK(15, 8) /* MT76x0 */
+#define MT_WLAN_FUN_CTRL_GPIO_OUT	GENMASK(23, 16) /* MT76x0 */
+#define MT_WLAN_FUN_CTRL_GPIO_OUT_EN	GENMASK(31, 24) /* MT76x0 */
+
+#define MT_XO_CTRL0			0x0100
+#define MT_XO_CTRL1			0x0104
+#define MT_XO_CTRL2			0x0108
+#define MT_XO_CTRL3			0x010c
+#define MT_XO_CTRL4			0x0110
+
+#define MT_XO_CTRL5			0x0114
+#define MT_XO_CTRL5_C2_VAL		GENMASK(14, 8)
+
+#define MT_XO_CTRL6			0x0118
+#define MT_XO_CTRL6_C2_CTRL		GENMASK(14, 8)
+
+#define MT_XO_CTRL7			0x011c
+
+#define MT_WLAN_MTC_CTRL		0x10148
+#define MT_WLAN_MTC_CTRL_MTCMOS_PWR_UP	BIT(0)
+#define MT_WLAN_MTC_CTRL_PWR_ACK	BIT(12)
+#define MT_WLAN_MTC_CTRL_PWR_ACK_S	BIT(13)
+#define MT_WLAN_MTC_CTRL_BBP_MEM_PD	GENMASK(19, 16)
+#define MT_WLAN_MTC_CTRL_PBF_MEM_PD	BIT(20)
+#define MT_WLAN_MTC_CTRL_FCE_MEM_PD	BIT(21)
+#define MT_WLAN_MTC_CTRL_TSO_MEM_PD	BIT(22)
+#define MT_WLAN_MTC_CTRL_BBP_MEM_RB	BIT(24)
+#define MT_WLAN_MTC_CTRL_PBF_MEM_RB	BIT(25)
+#define MT_WLAN_MTC_CTRL_FCE_MEM_RB	BIT(26)
+#define MT_WLAN_MTC_CTRL_TSO_MEM_RB	BIT(27)
+#define MT_WLAN_MTC_CTRL_STATE_UP	BIT(28)
+
+#define MT_INT_SOURCE_CSR		0x0200
+#define MT_INT_MASK_CSR			0x0204
+
+#define MT_INT_RX_DONE(_n)		BIT(_n)
+#define MT_INT_RX_DONE_ALL		GENMASK(1, 0)
+#define MT_INT_TX_DONE_ALL		GENMASK(13, 4)
+#define MT_INT_TX_DONE(_n)		BIT(_n + 4)
+#define MT_INT_RX_COHERENT		BIT(16)
+#define MT_INT_TX_COHERENT		BIT(17)
+#define MT_INT_ANY_COHERENT		BIT(18)
+#define MT_INT_MCU_CMD			BIT(19)
+#define MT_INT_TBTT			BIT(20)
+#define MT_INT_PRE_TBTT			BIT(21)
+#define MT_INT_TX_STAT			BIT(22)
+#define MT_INT_AUTO_WAKEUP		BIT(23)
+#define MT_INT_GPTIMER			BIT(24)
+#define MT_INT_RXDELAYINT		BIT(26)
+#define MT_INT_TXDELAYINT		BIT(27)
+
+#define MT_WPDMA_GLO_CFG		0x0208
+#define MT_WPDMA_GLO_CFG_TX_DMA_EN	BIT(0)
+#define MT_WPDMA_GLO_CFG_TX_DMA_BUSY	BIT(1)
+#define MT_WPDMA_GLO_CFG_RX_DMA_EN	BIT(2)
+#define MT_WPDMA_GLO_CFG_RX_DMA_BUSY	BIT(3)
+#define MT_WPDMA_GLO_CFG_DMA_BURST_SIZE	GENMASK(5, 4)
+#define MT_WPDMA_GLO_CFG_TX_WRITEBACK_DONE	BIT(6)
+#define MT_WPDMA_GLO_CFG_BIG_ENDIAN	BIT(7)
+#define MT_WPDMA_GLO_CFG_HDR_SEG_LEN	GENMASK(15, 8)
+#define MT_WPDMA_GLO_CFG_CLK_GATE_DIS	BIT(30)
+#define MT_WPDMA_GLO_CFG_RX_2B_OFFSET	BIT(31)
+
+#define MT_WPDMA_RST_IDX		0x020c
+
+#define MT_WPDMA_DELAY_INT_CFG		0x0210
+
+#define MT_WMM_AIFSN		0x0214
+#define MT_WMM_AIFSN_MASK		GENMASK(3, 0)
+#define MT_WMM_AIFSN_SHIFT(_n)		((_n) * 4)
+
+#define MT_WMM_CWMIN		0x0218
+#define MT_WMM_CWMIN_MASK		GENMASK(3, 0)
+#define MT_WMM_CWMIN_SHIFT(_n)		((_n) * 4)
+
+#define MT_WMM_CWMAX		0x021c
+#define MT_WMM_CWMAX_MASK		GENMASK(3, 0)
+#define MT_WMM_CWMAX_SHIFT(_n)		((_n) * 4)
+
+#define MT_WMM_TXOP_BASE		0x0220
+#define MT_WMM_TXOP(_n)			(MT_WMM_TXOP_BASE + (((_n) / 2) << 2))
+#define MT_WMM_TXOP_SHIFT(_n)		((_n & 1) * 16)
+#define MT_WMM_TXOP_MASK		GENMASK(15, 0)
+
+#define MT_TSO_CTRL			0x0250
+#define MT_HEADER_TRANS_CTRL_REG	0x0260
+
+#define MT_TX_RING_BASE			0x0300
+#define MT_RX_RING_BASE			0x03c0
+
+#define MT_TX_HW_QUEUE_MCU		8
+#define MT_TX_HW_QUEUE_MGMT		9
+
+#define MT_PBF_SYS_CTRL			0x0400
+#define MT_PBF_SYS_CTRL_MCU_RESET	BIT(0)
+#define MT_PBF_SYS_CTRL_DMA_RESET	BIT(1)
+#define MT_PBF_SYS_CTRL_MAC_RESET	BIT(2)
+#define MT_PBF_SYS_CTRL_PBF_RESET	BIT(3)
+#define MT_PBF_SYS_CTRL_ASY_RESET	BIT(4)
+
+#define MT_PBF_CFG			0x0404
+#define MT_PBF_CFG_TX0Q_EN		BIT(0)
+#define MT_PBF_CFG_TX1Q_EN		BIT(1)
+#define MT_PBF_CFG_TX2Q_EN		BIT(2)
+#define MT_PBF_CFG_TX3Q_EN		BIT(3)
+#define MT_PBF_CFG_RX0Q_EN		BIT(4)
+#define MT_PBF_CFG_RX_DROP_EN		BIT(8)
+
+#define MT_PBF_TX_MAX_PCNT		0x0408
+#define MT_PBF_RX_MAX_PCNT		0x040c
+
+#define MT_BCN_OFFSET_BASE		0x041c
+#define MT_BCN_OFFSET(_n)		(MT_BCN_OFFSET_BASE + ((_n) << 2))
+
+#define MT_RF_BYPASS_0			0x0504
+#define MT_RF_BYPASS_1			0x0508
+#define MT_RF_SETTING_0			0x050c
+
+#define MT_RF_DATA_WRITE		0x0524
+
+#define MT_RF_CTRL			0x0528
+#define MT_RF_CTRL_ADDR			GENMASK(11, 0)
+#define MT_RF_CTRL_WRITE		BIT(12)
+#define MT_RF_CTRL_BUSY			BIT(13)
+#define MT_RF_CTRL_IDX			BIT(16)
+
+#define MT_RF_DATA_READ			0x052c
+
+#define MT_FCE_PSE_CTRL			0x0800
+#define MT_FCE_PARAMETERS		0x0804
+#define MT_FCE_CSO			0x0808
+
+#define MT_FCE_L2_STUFF			0x080c
+#define MT_FCE_L2_STUFF_HT_L2_EN	BIT(0)
+#define MT_FCE_L2_STUFF_QOS_L2_EN	BIT(1)
+#define MT_FCE_L2_STUFF_RX_STUFF_EN	BIT(2)
+#define MT_FCE_L2_STUFF_TX_STUFF_EN	BIT(3)
+#define MT_FCE_L2_STUFF_WR_MPDU_LEN_EN	BIT(4)
+#define MT_FCE_L2_STUFF_MVINV_BSWAP	BIT(5)
+#define MT_FCE_L2_STUFF_TS_CMD_QSEL_EN	GENMASK(15, 8)
+#define MT_FCE_L2_STUFF_TS_LEN_EN	GENMASK(23, 16)
+#define MT_FCE_L2_STUFF_OTHER_PORT	GENMASK(25, 24)
+
+#define MT_FCE_WLAN_FLOW_CONTROL1	0x0824
+
+#define MT_PAUSE_ENABLE_CONTROL1	0x0a38
+
+#define MT_MAC_CSR0			0x1000
+
+#define MT_MAC_SYS_CTRL			0x1004
+#define MT_MAC_SYS_CTRL_RESET_CSR	BIT(0)
+#define MT_MAC_SYS_CTRL_RESET_BBP	BIT(1)
+#define MT_MAC_SYS_CTRL_ENABLE_TX	BIT(2)
+#define MT_MAC_SYS_CTRL_ENABLE_RX	BIT(3)
+
+#define MT_MAC_ADDR_DW0			0x1008
+#define MT_MAC_ADDR_DW1			0x100c
+
+#define MT_MAC_BSSID_DW0		0x1010
+#define MT_MAC_BSSID_DW1		0x1014
+#define MT_MAC_BSSID_DW1_ADDR		GENMASK(15, 0)
+#define MT_MAC_BSSID_DW1_MBSS_MODE	GENMASK(17, 16)
+#define MT_MAC_BSSID_DW1_MBEACON_N	GENMASK(20, 18)
+#define MT_MAC_BSSID_DW1_MBSS_LOCAL_BIT	BIT(21)
+#define MT_MAC_BSSID_DW1_MBSS_MODE_B2	BIT(22)
+#define MT_MAC_BSSID_DW1_MBEACON_N_B3	BIT(23)
+#define MT_MAC_BSSID_DW1_MBSS_IDX_BYTE	GENMASK(26, 24)
+
+#define MT_MAX_LEN_CFG			0x1018
+
+#define MT_AMPDU_MAX_LEN_20M1S		0x1030
+#define MT_AMPDU_MAX_LEN_20M2S		0x1034
+#define MT_AMPDU_MAX_LEN_40M1S		0x1038
+#define MT_AMPDU_MAX_LEN_40M2S		0x103c
+#define MT_AMPDU_MAX_LEN		0x1040
+
+#define MT_WCID_DROP_BASE		0x106c
+#define MT_WCID_DROP(_n)		(MT_WCID_DROP_BASE + ((_n) >> 5) * 4)
+#define MT_WCID_DROP_MASK(_n)		BIT((_n) % 32)
+
+#define MT_BCN_BYPASS_MASK		0x108c
+
+#define MT_MAC_APC_BSSID_BASE		0x1090
+#define MT_MAC_APC_BSSID_L(_n)		(MT_MAC_APC_BSSID_BASE + ((_n) * 8))
+#define MT_MAC_APC_BSSID_H(_n)		(MT_MAC_APC_BSSID_BASE + ((_n) * 8 + 4))
+#define MT_MAC_APC_BSSID_H_ADDR		GENMASK(15, 0)
+#define MT_MAC_APC_BSSID0_H_EN		BIT(16)
+
+#define MT_XIFS_TIME_CFG		0x1100
+#define MT_XIFS_TIME_CFG_CCK_SIFS	GENMASK(7, 0)
+#define MT_XIFS_TIME_CFG_OFDM_SIFS	GENMASK(15, 8)
+#define MT_XIFS_TIME_CFG_OFDM_XIFS	GENMASK(19, 16)
+#define MT_XIFS_TIME_CFG_EIFS		GENMASK(28, 20)
+#define MT_XIFS_TIME_CFG_BB_RXEND_EN	BIT(29)
+
+#define MT_BKOFF_SLOT_CFG		0x1104
+#define MT_BKOFF_SLOT_CFG_SLOTTIME	GENMASK(7, 0)
+#define MT_BKOFF_SLOT_CFG_CC_DELAY	GENMASK(11, 8)
+
+#define MT_CH_TIME_CFG			0x110c
+#define MT_CH_TIME_CFG_TIMER_EN		BIT(0)
+#define MT_CH_TIME_CFG_TX_AS_BUSY	BIT(1)
+#define MT_CH_TIME_CFG_RX_AS_BUSY	BIT(2)
+#define MT_CH_TIME_CFG_NAV_AS_BUSY	BIT(3)
+#define MT_CH_TIME_CFG_EIFS_AS_BUSY	BIT(4)
+#define MT_CH_TIME_CFG_MDRDY_CNT_EN	BIT(5)
+#define MT_CH_TIME_CFG_CH_TIMER_CLR	GENMASK(9, 8)
+#define MT_CH_TIME_CFG_MDRDY_CLR	GENMASK(11, 10)
+
+#define MT_PBF_LIFE_TIMER		0x1110
+
+#define MT_BEACON_TIME_CFG		0x1114
+#define MT_BEACON_TIME_CFG_INTVAL	GENMASK(15, 0)
+#define MT_BEACON_TIME_CFG_TIMER_EN	BIT(16)
+#define MT_BEACON_TIME_CFG_SYNC_MODE	GENMASK(18, 17)
+#define MT_BEACON_TIME_CFG_TBTT_EN	BIT(19)
+#define MT_BEACON_TIME_CFG_BEACON_TX	BIT(20)
+#define MT_BEACON_TIME_CFG_TSF_COMP	GENMASK(31, 24)
+
+#define MT_TBTT_SYNC_CFG		0x1118
+#define MT_TBTT_TIMER_CFG		0x1124
+
+#define MT_INT_TIMER_CFG		0x1128
+#define MT_INT_TIMER_CFG_PRE_TBTT	GENMASK(15, 0)
+#define MT_INT_TIMER_CFG_GP_TIMER	GENMASK(31, 16)
+
+#define MT_INT_TIMER_EN			0x112c
+#define MT_INT_TIMER_EN_PRE_TBTT_EN	BIT(0)
+#define MT_INT_TIMER_EN_GP_TIMER_EN	BIT(1)
+
+#define MT_CH_IDLE			0x1130
+#define MT_CH_BUSY			0x1134
+#define MT_EXT_CH_BUSY			0x1138
+#define MT_ED_CCA_TIMER			0x1140
+
+#define MT_MAC_STATUS			0x1200
+#define MT_MAC_STATUS_TX		BIT(0)
+#define MT_MAC_STATUS_RX		BIT(1)
+
+#define MT_PWR_PIN_CFG			0x1204
+#define MT_AUX_CLK_CFG			0x120c
+
+#define MT_BB_PA_MODE_CFG0		0x1214
+#define MT_BB_PA_MODE_CFG1		0x1218
+#define MT_RF_PA_MODE_CFG0		0x121c
+#define MT_RF_PA_MODE_CFG1		0x1220
+
+#define MT_RF_PA_MODE_ADJ0		0x1228
+#define MT_RF_PA_MODE_ADJ1		0x122c
+
+#define MT_DACCLK_EN_DLY_CFG		0x1264
+
+#define MT_EDCA_CFG_BASE		0x1300
+#define MT_EDCA_CFG_AC(_n)		(MT_EDCA_CFG_BASE + ((_n) << 2))
+#define MT_EDCA_CFG_TXOP		GENMASK(7, 0)
+#define MT_EDCA_CFG_AIFSN		GENMASK(11, 8)
+#define MT_EDCA_CFG_CWMIN		GENMASK(15, 12)
+#define MT_EDCA_CFG_CWMAX		GENMASK(19, 16)
+
+#define MT_TX_PWR_CFG_0			0x1314
+#define MT_TX_PWR_CFG_1			0x1318
+#define MT_TX_PWR_CFG_2			0x131c
+#define MT_TX_PWR_CFG_3			0x1320
+#define MT_TX_PWR_CFG_4			0x1324
+
+#define MT_TX_BAND_CFG			0x132c
+#define MT_TX_BAND_CFG_UPPER_40M	BIT(0)
+#define MT_TX_BAND_CFG_5G		BIT(1)
+#define MT_TX_BAND_CFG_2G		BIT(2)
+
+#define MT_HT_FBK_TO_LEGACY		0x1384
+#define MT_TX_MPDU_ADJ_INT		0x1388
+
+#define MT_TX_PWR_CFG_7			0x13d4
+#define MT_TX_PWR_CFG_8			0x13d8
+#define MT_TX_PWR_CFG_9			0x13dc
+
+#define MT_TX_SW_CFG0			0x1330
+#define MT_TX_SW_CFG1			0x1334
+#define MT_TX_SW_CFG2			0x1338
+
+#define MT_TXOP_CTRL_CFG		0x1340
+
+#define MT_TX_RTS_CFG			0x1344
+#define MT_TX_RTS_CFG_RETRY_LIMIT	GENMASK(7, 0)
+#define MT_TX_RTS_CFG_THRESH		GENMASK(23, 8)
+#define MT_TX_RTS_FALLBACK		BIT(24)
+
+#define MT_TX_TIMEOUT_CFG		0x1348
+#define MT_TX_TIMEOUT_CFG_ACKTO		GENMASK(15, 8)
+
+#define MT_TX_RETRY_CFG			0x134c
+#define MT_VHT_HT_FBK_CFG1		0x1358
+
+#define MT_PROT_CFG_RATE		GENMASK(15, 0)
+#define MT_PROT_CFG_CTRL		GENMASK(17, 16)
+#define MT_PROT_CFG_NAV			GENMASK(19, 18)
+#define MT_PROT_CFG_TXOP_ALLOW		GENMASK(25, 20)
+#define MT_PROT_CFG_RTS_THRESH		BIT(26)
+
+#define MT_CCK_PROT_CFG			0x1364
+#define MT_OFDM_PROT_CFG		0x1368
+#define MT_MM20_PROT_CFG		0x136c
+#define MT_MM40_PROT_CFG		0x1370
+#define MT_GF20_PROT_CFG		0x1374
+#define MT_GF40_PROT_CFG		0x1378
+
+#define MT_EXP_ACK_TIME			0x1380
+
+#define MT_TX_PWR_CFG_0_EXT		0x1390
+#define MT_TX_PWR_CFG_1_EXT		0x1394
+
+#define MT_TX_FBK_LIMIT			0x1398
+#define MT_TX_FBK_LIMIT_MPDU_FBK	GENMASK(7, 0)
+#define MT_TX_FBK_LIMIT_AMPDU_FBK	GENMASK(15, 8)
+#define MT_TX_FBK_LIMIT_MPDU_UP_CLEAR	BIT(16)
+#define MT_TX_FBK_LIMIT_AMPDU_UP_CLEAR	BIT(17)
+#define MT_TX_FBK_LIMIT_RATE_LUT	BIT(18)
+
+#define MT_TX0_RF_GAIN_CORR		0x13a0
+#define MT_TX1_RF_GAIN_CORR		0x13a4
+
+#define MT_TX_ALC_CFG_0			0x13b0
+#define MT_TX_ALC_CFG_0_CH_INIT_0	GENMASK(5, 0)
+#define MT_TX_ALC_CFG_0_CH_INIT_1	GENMASK(13, 8)
+#define MT_TX_ALC_CFG_0_LIMIT_0		GENMASK(21, 16)
+#define MT_TX_ALC_CFG_0_LIMIT_1		GENMASK(29, 24)
+
+#define MT_TX_ALC_CFG_1			0x13b4
+#define MT_TX_ALC_CFG_1_TEMP_COMP	GENMASK(5, 0)
+
+#define MT_TX_ALC_CFG_2			0x13a8
+#define MT_TX_ALC_CFG_2_TEMP_COMP	GENMASK(5, 0)
+
+#define MT_TX_ALC_CFG_3			0x13ac
+#define MT_TX_ALC_CFG_4			0x13c0
+#define MT_TX_ALC_CFG_4_LOWGAIN_CH_EN	BIT(31)
+
+#define MT_TX_ALC_VGA3			0x13c8
+
+#define MT_TX_PROT_CFG6			0x13e0
+#define MT_TX_PROT_CFG7			0x13e4
+#define MT_TX_PROT_CFG8			0x13e8
+
+#define MT_PIFS_TX_CFG			0x13ec
+
+#define MT_RX_FILTR_CFG			0x1400
+
+#define MT_RX_FILTR_CFG_CRC_ERR		BIT(0)
+#define MT_RX_FILTR_CFG_PHY_ERR		BIT(1)
+#define MT_RX_FILTR_CFG_PROMISC		BIT(2)
+#define MT_RX_FILTR_CFG_OTHER_BSS	BIT(3)
+#define MT_RX_FILTR_CFG_VER_ERR		BIT(4)
+#define MT_RX_FILTR_CFG_MCAST		BIT(5)
+#define MT_RX_FILTR_CFG_BCAST		BIT(6)
+#define MT_RX_FILTR_CFG_DUP		BIT(7)
+#define MT_RX_FILTR_CFG_CFACK		BIT(8)
+#define MT_RX_FILTR_CFG_CFEND		BIT(9)
+#define MT_RX_FILTR_CFG_ACK		BIT(10)
+#define MT_RX_FILTR_CFG_CTS		BIT(11)
+#define MT_RX_FILTR_CFG_RTS		BIT(12)
+#define MT_RX_FILTR_CFG_PSPOLL		BIT(13)
+#define MT_RX_FILTR_CFG_BA		BIT(14)
+#define MT_RX_FILTR_CFG_BAR		BIT(15)
+#define MT_RX_FILTR_CFG_CTRL_RSV	BIT(16)
+
+#define MT_LEGACY_BASIC_RATE		0x1408
+#define MT_HT_BASIC_RATE		0x140c
+
+#define MT_HT_CTRL_CFG			0x1410
+
+#define MT_EXT_CCA_CFG			0x141c
+#define MT_EXT_CCA_CFG_CCA0		GENMASK(1, 0)
+#define MT_EXT_CCA_CFG_CCA1		GENMASK(3, 2)
+#define MT_EXT_CCA_CFG_CCA2		GENMASK(5, 4)
+#define MT_EXT_CCA_CFG_CCA3		GENMASK(7, 6)
+#define MT_EXT_CCA_CFG_CCA_MASK		GENMASK(11, 8)
+#define MT_EXT_CCA_CFG_ED_CCA_MASK	GENMASK(15, 12)
+
+#define MT_TX_SW_CFG3			0x1478
+
+#define MT_PN_PAD_MODE			0x150c
+
+#define MT_TXOP_HLDR_ET			0x1608
+
+#define MT_PROT_AUTO_TX_CFG		0x1648
+#define MT_PROT_AUTO_TX_CFG_PROT_PADJ	GENMASK(11, 8)
+#define MT_PROT_AUTO_TX_CFG_AUTO_PADJ	GENMASK(27, 24)
+
+#define MT_RX_STAT_0			0x1700
+#define MT_RX_STAT_0_CRC_ERRORS		GENMASK(15, 0)
+#define MT_RX_STAT_0_PHY_ERRORS		GENMASK(31, 16)
+
+#define MT_RX_STAT_1			0x1704
+#define MT_RX_STAT_1_CCA_ERRORS		GENMASK(15, 0)
+#define MT_RX_STAT_1_PLCP_ERRORS	GENMASK(31, 16)
+
+#define MT_RX_STAT_2			0x1708
+#define MT_RX_STAT_2_DUP_ERRORS		GENMASK(15, 0)
+#define MT_RX_STAT_2_OVERFLOW_ERRORS	GENMASK(31, 16)
+
+#define MT_TX_STAT_FIFO			0x1718
+#define MT_TX_STAT_FIFO_VALID		BIT(0)
+#define MT_TX_STAT_FIFO_SUCCESS		BIT(5)
+#define MT_TX_STAT_FIFO_AGGR		BIT(6)
+#define MT_TX_STAT_FIFO_ACKREQ		BIT(7)
+#define MT_TX_STAT_FIFO_WCID		GENMASK(15, 8)
+#define MT_TX_STAT_FIFO_RATE		GENMASK(31, 16)
+
+#define MT_TX_AGG_CNT_BASE0		0x1720
+#define MT_TX_AGG_CNT_BASE1		0x174c
+
+#define MT_TX_AGG_CNT(_id)		((_id) < 8 ?			\
+					 MT_TX_AGG_CNT_BASE0 + ((_id) << 2) : \
+					 MT_TX_AGG_CNT_BASE1 + ((_id - 8) << 2))
+
+#define MT_TX_STAT_FIFO_EXT		0x1798
+#define MT_TX_STAT_FIFO_EXT_RETRY	GENMASK(7, 0)
+#define MT_TX_STAT_FIFO_EXT_PKTID	GENMASK(15, 8)
+
+#define MT_WCID_TX_RATE_BASE		0x1c00
+#define MT_WCID_TX_RATE(_i)		(MT_WCID_TX_RATE_BASE + ((_i) << 3))
+
+#define MT_BBP_CORE_BASE		0x2000
+#define MT_BBP_IBI_BASE			0x2100
+#define MT_BBP_AGC_BASE			0x2300
+#define MT_BBP_TXC_BASE			0x2400
+#define MT_BBP_RXC_BASE			0x2500
+#define MT_BBP_TXO_BASE			0x2600
+#define MT_BBP_TXBE_BASE		0x2700
+#define MT_BBP_RXFE_BASE		0x2800
+#define MT_BBP_RXO_BASE			0x2900
+#define MT_BBP_DFS_BASE			0x2a00
+#define MT_BBP_TR_BASE			0x2b00
+#define MT_BBP_CAL_BASE			0x2c00
+#define MT_BBP_DSC_BASE			0x2e00
+#define MT_BBP_PFMU_BASE		0x2f00
+
+#define MT_BBP(_type, _n)		(MT_BBP_##_type##_BASE + ((_n) << 2))
+
+#define MT_BBP_CORE_R1_BW		GENMASK(4, 3)
+
+#define MT_BBP_AGC_R0_CTRL_CHAN		GENMASK(9, 8)
+#define MT_BBP_AGC_R0_BW		GENMASK(14, 12)
+
+/* AGC, R4/R5 */
+#define MT_BBP_AGC_LNA_HIGH_GAIN	GENMASK(21, 16)
+#define MT_BBP_AGC_LNA_MID_GAIN		GENMASK(13, 8)
+#define MT_BBP_AGC_LNA_LOW_GAIN		GENMASK(5, 0)
+
+/* AGC, R6/R7 */
+#define MT_BBP_AGC_LNA_ULOW_GAIN	GENMASK(5, 0)
+
+/* AGC, R8/R9 */
+#define MT_BBP_AGC_LNA_GAIN_MODE	GENMASK(7, 6)
+#define MT_BBP_AGC_GAIN			GENMASK(14, 8)
+
+#define MT_BBP_AGC20_RSSI0		GENMASK(7, 0)
+#define MT_BBP_AGC20_RSSI1		GENMASK(15, 8)
+
+#define MT_BBP_TXBE_R0_CTRL_CHAN	GENMASK(1, 0)
+
+#define MT_WCID_ADDR_BASE		0x1800
+#define MT_WCID_ADDR(_n)		(MT_WCID_ADDR_BASE + (_n) * 8)
+
+#define MT_SRAM_BASE			0x4000
+
+#define MT_WCID_KEY_BASE		0x8000
+#define MT_WCID_KEY(_n)			(MT_WCID_KEY_BASE + (_n) * 32)
+
+#define MT_WCID_IV_BASE			0xa000
+#define MT_WCID_IV(_n)			(MT_WCID_IV_BASE + (_n) * 8)
+
+#define MT_WCID_ATTR_BASE		0xa800
+#define MT_WCID_ATTR(_n)		(MT_WCID_ATTR_BASE + (_n) * 4)
+
+#define MT_WCID_ATTR_PAIRWISE		BIT(0)
+#define MT_WCID_ATTR_PKEY_MODE		GENMASK(3, 1)
+#define MT_WCID_ATTR_BSS_IDX		GENMASK(6, 4)
+#define MT_WCID_ATTR_RXWI_UDF		GENMASK(9, 7)
+#define MT_WCID_ATTR_PKEY_MODE_EXT	BIT(10)
+#define MT_WCID_ATTR_BSS_IDX_EXT	BIT(11)
+#define MT_WCID_ATTR_WAPI_MCBC		BIT(15)
+#define MT_WCID_ATTR_WAPI_KEYID		GENMASK(31, 24)
+
+#define MT_SKEY_BASE_0			0xac00
+#define MT_SKEY_BASE_1			0xb400
+#define MT_SKEY_0(_bss, _idx)		(MT_SKEY_BASE_0 + (4 * (_bss) + _idx) * 32)
+#define MT_SKEY_1(_bss, _idx)		(MT_SKEY_BASE_1 + (4 * ((_bss) & 7) + _idx) * 32)
+#define MT_SKEY(_bss, _idx)		((_bss & 8) ? MT_SKEY_1(_bss, _idx) : MT_SKEY_0(_bss, _idx))
+
+#define MT_SKEY_MODE_BASE_0		0xb000
+#define MT_SKEY_MODE_BASE_1		0xb3f0
+#define MT_SKEY_MODE_0(_bss)		(MT_SKEY_MODE_BASE_0 + ((_bss / 2) << 2))
+#define MT_SKEY_MODE_1(_bss)		(MT_SKEY_MODE_BASE_1 + ((((_bss) & 7) / 2) << 2))
+#define MT_SKEY_MODE(_bss)		((_bss & 8) ? MT_SKEY_MODE_1(_bss) : MT_SKEY_MODE_0(_bss))
+#define MT_SKEY_MODE_MASK		GENMASK(3, 0)
+#define MT_SKEY_MODE_SHIFT(_bss, _idx)	(4 * ((_idx) + 4 * (_bss & 1)))
+
+#define MT_BEACON_BASE			0xc000
+
+#define MT_TEMP_SENSOR			0x1d000
+#define MT_TEMP_SENSOR_VAL		GENMASK(6, 0)
+
+struct mt76_wcid_addr {
+	u8 macaddr[6];
+	__le16 ba_mask;
+} __packed __aligned(4);
+
+struct mt76_wcid_key {
+	u8 key[16];
+	u8 tx_mic[8];
+	u8 rx_mic[8];
+} __packed __aligned(4);
+
+enum mt76x2_cipher_type {
+	MT_CIPHER_NONE,
+	MT_CIPHER_WEP40,
+	MT_CIPHER_WEP104,
+	MT_CIPHER_TKIP,
+	MT_CIPHER_AES_CCMP,
+	MT_CIPHER_CKIP40,
+	MT_CIPHER_CKIP104,
+	MT_CIPHER_CKIP128,
+	MT_CIPHER_WAPI,
+};
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_trace.c b/drivers/net/wireless/mediatek/mt76/mt76x2_trace.c
new file mode 100644
index 000000000000..a09f117848d6
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_trace.c
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/module.h>
+
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "mt76x2_trace.h"
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_trace.h b/drivers/net/wireless/mediatek/mt76/mt76x2_trace.h
new file mode 100644
index 000000000000..4cd424148d4b
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_trace.h
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#if !defined(__MT76x2_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define __MT76x2_TRACE_H
+
+#include <linux/tracepoint.h>
+#include "mt76x2.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mt76x2
+
+#define MAXNAME		32
+#define DEV_ENTRY	__array(char, wiphy_name, 32)
+#define DEV_ASSIGN	strlcpy(__entry->wiphy_name, wiphy_name(mt76_hw(dev)->wiphy), MAXNAME)
+#define DEV_PR_FMT	"%s"
+#define DEV_PR_ARG	__entry->wiphy_name
+
+#define TXID_ENTRY	__field(u8, wcid) __field(u8, pktid)
+#define TXID_ASSIGN	__entry->wcid = wcid; __entry->pktid = pktid
+#define TXID_PR_FMT	" [%d:%d]"
+#define TXID_PR_ARG	__entry->wcid, __entry->pktid
+
+DECLARE_EVENT_CLASS(dev_evt,
+	TP_PROTO(struct mt76x2_dev *dev),
+	TP_ARGS(dev),
+	TP_STRUCT__entry(
+		DEV_ENTRY
+	),
+	TP_fast_assign(
+		DEV_ASSIGN;
+	),
+	TP_printk(DEV_PR_FMT, DEV_PR_ARG)
+);
+
+DECLARE_EVENT_CLASS(dev_txid_evt,
+	TP_PROTO(struct mt76x2_dev *dev, u8 wcid, u8 pktid),
+	TP_ARGS(dev, wcid, pktid),
+	TP_STRUCT__entry(
+		DEV_ENTRY
+		TXID_ENTRY
+	),
+	TP_fast_assign(
+		DEV_ASSIGN;
+		TXID_ASSIGN;
+	),
+	TP_printk(
+		DEV_PR_FMT TXID_PR_FMT,
+		DEV_PR_ARG, TXID_PR_ARG
+	)
+);
+
+DEFINE_EVENT(dev_evt, mac_txstat_poll,
+	TP_PROTO(struct mt76x2_dev *dev),
+	TP_ARGS(dev)
+);
+
+DEFINE_EVENT(dev_txid_evt, mac_txdone_add,
+	TP_PROTO(struct mt76x2_dev *dev, u8 wcid, u8 pktid),
+	TP_ARGS(dev, wcid, pktid)
+);
+
+TRACE_EVENT(mac_txstat_fetch,
+	TP_PROTO(struct mt76x2_dev *dev,
+		 struct mt76x2_tx_status *stat),
+
+	TP_ARGS(dev, stat),
+
+	TP_STRUCT__entry(
+		DEV_ENTRY
+		TXID_ENTRY
+		__field(bool, success)
+		__field(bool, aggr)
+		__field(bool, ack_req)
+		__field(u16, rate)
+		__field(u8, retry)
+	),
+
+	TP_fast_assign(
+		DEV_ASSIGN;
+		__entry->success = stat->success;
+		__entry->aggr = stat->aggr;
+		__entry->ack_req = stat->ack_req;
+		__entry->wcid = stat->wcid;
+		__entry->pktid = stat->pktid;
+		__entry->rate = stat->rate;
+		__entry->retry = stat->retry;
+	),
+
+	TP_printk(
+		DEV_PR_FMT TXID_PR_FMT
+		" success:%d aggr:%d ack_req:%d"
+		" rate:%04x retry:%d",
+		DEV_PR_ARG, TXID_PR_ARG,
+		__entry->success, __entry->aggr, __entry->ack_req,
+		__entry->rate, __entry->retry
+	)
+);
+
+
+TRACE_EVENT(dev_irq,
+	TP_PROTO(struct mt76x2_dev *dev, u32 val, u32 mask),
+
+	TP_ARGS(dev, val, mask),
+
+	TP_STRUCT__entry(
+		DEV_ENTRY
+		__field(u32, val)
+		__field(u32, mask)
+	),
+
+	TP_fast_assign(
+		DEV_ASSIGN;
+		__entry->val = val;
+		__entry->mask = mask;
+	),
+
+	TP_printk(
+		DEV_PR_FMT " %08x & %08x",
+		DEV_PR_ARG, __entry->val, __entry->mask
+	)
+);
+
+#endif
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE mt76x2_trace
+
+#include <trace/define_trace.h>
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x2_tx.c b/drivers/net/wireless/mediatek/mt76/mt76x2_tx.c
new file mode 100644
index 000000000000..534e4bf9a34c
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/mt76x2_tx.c
@@ -0,0 +1,260 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "mt76x2.h"
+#include "mt76x2_dma.h"
+
+struct beacon_bc_data {
+	struct mt76x2_dev *dev;
+	struct sk_buff_head q;
+	struct sk_buff *tail[8];
+};
+
+void mt76x2_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control,
+	     struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct mt76x2_dev *dev = hw->priv;
+	struct ieee80211_vif *vif = info->control.vif;
+	struct mt76_wcid *wcid = &dev->global_wcid;
+
+	if (control->sta) {
+		struct mt76x2_sta *msta;
+
+		msta = (struct mt76x2_sta *) control->sta->drv_priv;
+		wcid = &msta->wcid;
+	}
+
+	if (vif || (!info->control.hw_key && wcid->hw_key_idx != -1)) {
+		struct mt76x2_vif *mvif;
+
+		mvif = (struct mt76x2_vif *) vif->drv_priv;
+		wcid = &mvif->group_wcid;
+	}
+
+	mt76_tx(&dev->mt76, control->sta, wcid, skb);
+}
+
+void mt76x2_tx_complete(struct mt76x2_dev *dev, struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+	if (info->flags & IEEE80211_TX_CTL_AMPDU) {
+		ieee80211_free_txskb(mt76_hw(dev), skb);
+	} else {
+		ieee80211_tx_info_clear_status(info);
+		info->status.rates[0].idx = -1;
+		info->flags |= IEEE80211_TX_STAT_ACK;
+		ieee80211_tx_status(mt76_hw(dev), skb);
+	}
+}
+
+s8 mt76x2_tx_get_max_txpwr_adj(struct mt76x2_dev *dev,
+			       const struct ieee80211_tx_rate *rate)
+{
+	s8 max_txpwr;
+
+	if (rate->flags & IEEE80211_TX_RC_VHT_MCS) {
+		u8 mcs = ieee80211_rate_get_vht_mcs(rate);
+
+		if (mcs == 8 || mcs == 9) {
+			max_txpwr = dev->rate_power.vht[8];
+		} else {
+			u8 nss, idx;
+
+			nss = ieee80211_rate_get_vht_nss(rate);
+			idx = ((nss - 1) << 3) + mcs;
+			max_txpwr = dev->rate_power.ht[idx & 0xf];
+		}
+	} else if (rate->flags & IEEE80211_TX_RC_MCS) {
+		max_txpwr = dev->rate_power.ht[rate->idx & 0xf];
+	} else {
+		enum nl80211_band band = dev->mt76.chandef.chan->band;
+
+		if (band == NL80211_BAND_2GHZ) {
+			const struct ieee80211_rate *r;
+			struct wiphy *wiphy = mt76_hw(dev)->wiphy;
+			struct mt76_rate_power *rp = &dev->rate_power;
+
+			r = &wiphy->bands[band]->bitrates[rate->idx];
+			if (r->flags & IEEE80211_RATE_SHORT_PREAMBLE)
+				max_txpwr = rp->cck[r->hw_value & 0x3];
+			else
+				max_txpwr = rp->ofdm[r->hw_value & 0x7];
+		} else {
+			max_txpwr = dev->rate_power.ofdm[rate->idx & 0x7];
+		}
+	}
+
+	return max_txpwr;
+}
+
+s8 mt76x2_tx_get_txpwr_adj(struct mt76x2_dev *dev, s8 txpwr, s8 max_txpwr_adj)
+{
+	txpwr = min_t(s8, txpwr, dev->txpower_conf);
+	txpwr -= (dev->target_power + dev->target_power_delta[0]);
+	txpwr = min_t(s8, txpwr, max_txpwr_adj);
+
+	if (!dev->enable_tpc)
+		return 0;
+	else if (txpwr >= 0)
+		return min_t(s8, txpwr, 7);
+	else
+		return (txpwr < -16) ? 8 : (txpwr + 32) / 2;
+}
+
+void mt76x2_tx_set_txpwr_auto(struct mt76x2_dev *dev, s8 txpwr)
+{
+	s8 txpwr_adj;
+
+	txpwr_adj = mt76x2_tx_get_txpwr_adj(dev, txpwr,
+					    dev->rate_power.ofdm[4]);
+	mt76_rmw_field(dev, MT_PROT_AUTO_TX_CFG,
+		       MT_PROT_AUTO_TX_CFG_PROT_PADJ, txpwr_adj);
+	mt76_rmw_field(dev, MT_PROT_AUTO_TX_CFG,
+		       MT_PROT_AUTO_TX_CFG_AUTO_PADJ, txpwr_adj);
+}
+
+static int mt76x2_insert_hdr_pad(struct sk_buff *skb)
+{
+	int len = ieee80211_get_hdrlen_from_skb(skb);
+
+	if (len % 4 == 0)
+		return 0;
+
+	skb_push(skb, 2);
+	memmove(skb->data, skb->data + 2, len);
+
+	skb->data[len] = 0;
+	skb->data[len + 1] = 0;
+	return 2;
+}
+
+int mt76x2_tx_prepare_skb(struct mt76_dev *mdev, void *txwi,
+			  struct sk_buff *skb, struct mt76_queue *q,
+			  struct mt76_wcid *wcid, struct ieee80211_sta *sta,
+			  u32 *tx_info)
+{
+	struct mt76x2_dev *dev = container_of(mdev, struct mt76x2_dev, mt76);
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	int qsel = MT_QSEL_EDCA;
+	int ret;
+
+	if (q == &dev->mt76.q_tx[MT_TXQ_PSD] && wcid && wcid->idx < 128)
+		mt76x2_mac_wcid_set_drop(dev, wcid->idx, false);
+
+	mt76x2_mac_write_txwi(dev, txwi, skb, wcid, sta);
+
+	ret = mt76x2_insert_hdr_pad(skb);
+	if (ret < 0)
+		return ret;
+
+	if (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)
+		qsel = MT_QSEL_MGMT;
+
+	*tx_info = FIELD_PREP(MT_TXD_INFO_QSEL, qsel) |
+		   MT_TXD_INFO_80211;
+
+	if (!wcid || wcid->hw_key_idx == 0xff || wcid->sw_iv)
+		*tx_info |= MT_TXD_INFO_WIV;
+
+	return 0;
+}
+
+static void
+mt76x2_update_beacon_iter(void *priv, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct mt76x2_dev *dev = (struct mt76x2_dev *) priv;
+	struct mt76x2_vif *mvif = (struct mt76x2_vif *) vif->drv_priv;
+	struct sk_buff *skb = NULL;
+
+	if (!(dev->beacon_mask & BIT(mvif->idx)))
+		return;
+
+	skb = ieee80211_beacon_get(mt76_hw(dev), vif);
+	if (!skb)
+		return;
+
+	mt76x2_mac_set_beacon(dev, mvif->idx, skb);
+}
+
+static void
+mt76x2_add_buffered_bc(void *priv, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct beacon_bc_data *data = priv;
+	struct mt76x2_dev *dev = data->dev;
+	struct mt76x2_vif *mvif = (struct mt76x2_vif *) vif->drv_priv;
+	struct ieee80211_tx_info *info;
+	struct sk_buff *skb;
+
+	if (!(dev->beacon_mask & BIT(mvif->idx)))
+		return;
+
+	skb = ieee80211_get_buffered_bc(mt76_hw(dev), vif);
+	if (!skb)
+		return;
+
+	info = IEEE80211_SKB_CB(skb);
+	info->control.vif = vif;
+	info->flags |= IEEE80211_TX_CTL_ASSIGN_SEQ;
+	mt76_skb_set_moredata(skb, true);
+	__skb_queue_tail(&data->q, skb);
+	data->tail[mvif->idx] = skb;
+}
+
+void mt76x2_pre_tbtt_tasklet(unsigned long arg)
+{
+	struct mt76x2_dev *dev = (struct mt76x2_dev *) arg;
+	struct mt76_queue *q = &dev->mt76.q_tx[MT_TXQ_PSD];
+	struct beacon_bc_data data = {};
+	struct sk_buff *skb;
+	int i, nframes;
+
+	data.dev = dev;
+	__skb_queue_head_init(&data.q);
+
+	ieee80211_iterate_active_interfaces_atomic(mt76_hw(dev),
+		IEEE80211_IFACE_ITER_RESUME_ALL,
+		mt76x2_update_beacon_iter, dev);
+
+	do {
+		nframes = skb_queue_len(&data.q);
+		ieee80211_iterate_active_interfaces_atomic(mt76_hw(dev),
+			IEEE80211_IFACE_ITER_RESUME_ALL,
+			mt76x2_add_buffered_bc, &data);
+	} while (nframes != skb_queue_len(&data.q));
+
+	if (!nframes)
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(data.tail); i++) {
+		if (!data.tail[i])
+			continue;
+
+		mt76_skb_set_moredata(data.tail[i], false);
+	}
+
+	spin_lock_bh(&q->lock);
+	while ((skb = __skb_dequeue(&data.q)) != NULL) {
+		struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+		struct ieee80211_vif *vif = info->control.vif;
+		struct mt76x2_vif *mvif = (struct mt76x2_vif *) vif->drv_priv;
+
+		mt76_tx_queue_skb(&dev->mt76, q, skb, &mvif->group_wcid, NULL);
+	}
+	spin_unlock_bh(&q->lock);
+}
+
diff --git a/drivers/net/wireless/mediatek/mt76/trace.c b/drivers/net/wireless/mediatek/mt76/trace.c
new file mode 100644
index 000000000000..ea4ab8729ae4
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/trace.c
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/module.h>
+
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+
+#endif
diff --git a/drivers/net/wireless/mediatek/mt76/trace.h b/drivers/net/wireless/mediatek/mt76/trace.h
new file mode 100644
index 000000000000..ea30895933c5
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/trace.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#if !defined(__MT76_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define __MT76_TRACE_H
+
+#include <linux/tracepoint.h>
+#include "mt76.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mt76
+
+#define MAXNAME		32
+#define DEV_ENTRY   __array(char, wiphy_name, 32)
+#define DEV_ASSIGN  strlcpy(__entry->wiphy_name, wiphy_name(dev->hw->wiphy), MAXNAME)
+#define DEV_PR_FMT  "%s"
+#define DEV_PR_ARG  __entry->wiphy_name
+
+#define REG_ENTRY	__field(u32, reg) __field(u32, val)
+#define REG_ASSIGN	__entry->reg = reg; __entry->val = val
+#define REG_PR_FMT	" %04x=%08x"
+#define REG_PR_ARG	__entry->reg, __entry->val
+
+DECLARE_EVENT_CLASS(dev_reg_evt,
+	TP_PROTO(struct mt76_dev *dev, u32 reg, u32 val),
+	TP_ARGS(dev, reg, val),
+	TP_STRUCT__entry(
+		DEV_ENTRY
+		REG_ENTRY
+	),
+	TP_fast_assign(
+		DEV_ASSIGN;
+		REG_ASSIGN;
+	),
+	TP_printk(
+		DEV_PR_FMT REG_PR_FMT,
+		DEV_PR_ARG, REG_PR_ARG
+	)
+);
+
+DEFINE_EVENT(dev_reg_evt, reg_rr,
+	TP_PROTO(struct mt76_dev *dev, u32 reg, u32 val),
+	TP_ARGS(dev, reg, val)
+);
+
+DEFINE_EVENT(dev_reg_evt, reg_wr,
+	TP_PROTO(struct mt76_dev *dev, u32 reg, u32 val),
+	TP_ARGS(dev, reg, val)
+);
+
+#endif
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+
+#include <trace/define_trace.h>
diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c
new file mode 100644
index 000000000000..4eef69bd8a9e
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/tx.c
@@ -0,0 +1,511 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "mt76.h"
+
+static struct mt76_txwi_cache *
+mt76_alloc_txwi(struct mt76_dev *dev)
+{
+	struct mt76_txwi_cache *t;
+	dma_addr_t addr;
+	int size;
+
+	size = (sizeof(*t) + L1_CACHE_BYTES - 1) & ~(L1_CACHE_BYTES - 1);
+	t = devm_kzalloc(dev->dev, size, GFP_ATOMIC);
+	if (!t)
+		return NULL;
+
+	addr = dma_map_single(dev->dev, &t->txwi, sizeof(t->txwi),
+			      DMA_TO_DEVICE);
+	t->dma_addr = addr;
+
+	return t;
+}
+
+static struct mt76_txwi_cache *
+__mt76_get_txwi(struct mt76_dev *dev)
+{
+	struct mt76_txwi_cache *t = NULL;
+
+	spin_lock_bh(&dev->lock);
+	if (!list_empty(&dev->txwi_cache)) {
+		t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
+				     list);
+		list_del(&t->list);
+	}
+	spin_unlock_bh(&dev->lock);
+
+	return t;
+}
+
+static struct mt76_txwi_cache *
+mt76_get_txwi(struct mt76_dev *dev)
+{
+	struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
+
+	if (t)
+		return t;
+
+	return mt76_alloc_txwi(dev);
+}
+
+void
+mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
+{
+	if (!t)
+		return;
+
+	spin_lock_bh(&dev->lock);
+	list_add(&t->list, &dev->txwi_cache);
+	spin_unlock_bh(&dev->lock);
+}
+
+void mt76_tx_free(struct mt76_dev *dev)
+{
+	struct mt76_txwi_cache *t;
+
+	while ((t = __mt76_get_txwi(dev)) != NULL)
+		dma_unmap_single(dev->dev, t->dma_addr, sizeof(t->txwi),
+				 DMA_TO_DEVICE);
+}
+
+static int
+mt76_txq_get_qid(struct ieee80211_txq *txq)
+{
+	if (!txq->sta)
+		return MT_TXQ_BE;
+
+	return txq->ac;
+}
+
+int mt76_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
+		      struct sk_buff *skb, struct mt76_wcid *wcid,
+		      struct ieee80211_sta *sta)
+{
+	struct mt76_queue_entry e;
+	struct mt76_txwi_cache *t;
+	struct mt76_queue_buf buf[32];
+	struct sk_buff *iter;
+	dma_addr_t addr;
+	int len;
+	u32 tx_info = 0;
+	int n, ret;
+
+	t = mt76_get_txwi(dev);
+	if (!t) {
+		ieee80211_free_txskb(dev->hw, skb);
+		return -ENOMEM;
+	}
+
+	dma_sync_single_for_cpu(dev->dev, t->dma_addr, sizeof(t->txwi),
+				DMA_TO_DEVICE);
+	ret = dev->drv->tx_prepare_skb(dev, &t->txwi, skb, q, wcid, sta,
+				       &tx_info);
+	dma_sync_single_for_device(dev->dev, t->dma_addr, sizeof(t->txwi),
+				   DMA_TO_DEVICE);
+	if (ret < 0)
+		goto free;
+
+	len = skb->len - skb->data_len;
+	addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
+	if (dma_mapping_error(dev->dev, addr)) {
+		ret = -ENOMEM;
+		goto free;
+	}
+
+	n = 0;
+	buf[n].addr = t->dma_addr;
+	buf[n++].len = dev->drv->txwi_size;
+	buf[n].addr = addr;
+	buf[n++].len = len;
+
+	skb_walk_frags(skb, iter) {
+		if (n == ARRAY_SIZE(buf))
+			goto unmap;
+
+		addr = dma_map_single(dev->dev, iter->data, iter->len,
+				      DMA_TO_DEVICE);
+		if (dma_mapping_error(dev->dev, addr))
+			goto unmap;
+
+		buf[n].addr = addr;
+		buf[n++].len = iter->len;
+	}
+
+	if (q->queued + (n + 1) / 2 >= q->ndesc - 1)
+		goto unmap;
+
+	return dev->queue_ops->add_buf(dev, q, buf, n, tx_info, skb, t);
+
+unmap:
+	ret = -ENOMEM;
+	for (n--; n > 0; n--)
+		dma_unmap_single(dev->dev, buf[n].addr, buf[n].len,
+				 DMA_TO_DEVICE);
+
+free:
+	e.skb = skb;
+	e.txwi = t;
+	dev->drv->tx_complete_skb(dev, q, &e, true);
+	mt76_put_txwi(dev, t);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(mt76_tx_queue_skb);
+
+void
+mt76_tx(struct mt76_dev *dev, struct ieee80211_sta *sta,
+	struct mt76_wcid *wcid, struct sk_buff *skb)
+{
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct mt76_queue *q;
+	int qid = skb_get_queue_mapping(skb);
+
+	if (WARN_ON(qid >= MT_TXQ_PSD)) {
+		qid = MT_TXQ_BE;
+		skb_set_queue_mapping(skb, qid);
+	}
+
+	if (!wcid->tx_rate_set)
+		ieee80211_get_tx_rates(info->control.vif, sta, skb,
+				       info->control.rates, 1);
+
+	q = &dev->q_tx[qid];
+
+	spin_lock_bh(&q->lock);
+	mt76_tx_queue_skb(dev, q, skb, wcid, sta);
+	dev->queue_ops->kick(dev, q);
+
+	if (q->queued > q->ndesc - 8)
+		ieee80211_stop_queue(dev->hw, skb_get_queue_mapping(skb));
+	spin_unlock_bh(&q->lock);
+}
+EXPORT_SYMBOL_GPL(mt76_tx);
+
+static struct sk_buff *
+mt76_txq_dequeue(struct mt76_dev *dev, struct mt76_txq *mtxq, bool ps)
+{
+	struct ieee80211_txq *txq = mtxq_to_txq(mtxq);
+	struct sk_buff *skb;
+
+	skb = skb_dequeue(&mtxq->retry_q);
+	if (skb) {
+		u8 tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK;
+
+		if (ps && skb_queue_empty(&mtxq->retry_q))
+			ieee80211_sta_set_buffered(txq->sta, tid, false);
+
+		return skb;
+	}
+
+	skb = ieee80211_tx_dequeue(dev->hw, txq);
+	if (!skb)
+		return NULL;
+
+	return skb;
+}
+
+static void
+mt76_check_agg_ssn(struct mt76_txq *mtxq, struct sk_buff *skb)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
+
+	if (!ieee80211_is_data_qos(hdr->frame_control))
+		return;
+
+	mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10;
+}
+
+static void
+mt76_queue_ps_skb(struct mt76_dev *dev, struct ieee80211_sta *sta,
+		  struct sk_buff *skb, bool last)
+{
+	struct mt76_wcid *wcid = (struct mt76_wcid *) sta->drv_priv;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct mt76_queue *hwq = &dev->q_tx[MT_TXQ_PSD];
+
+	info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE;
+	if (last)
+		info->flags |= IEEE80211_TX_STATUS_EOSP;
+
+	mt76_skb_set_moredata(skb, !last);
+	mt76_tx_queue_skb(dev, hwq, skb, wcid, sta);
+}
+
+void
+mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
+			     u16 tids, int nframes,
+			     enum ieee80211_frame_release_type reason,
+			     bool more_data)
+{
+	struct mt76_dev *dev = hw->priv;
+	struct sk_buff *last_skb = NULL;
+	struct mt76_queue *hwq = &dev->q_tx[MT_TXQ_PSD];
+	int i;
+
+	spin_lock_bh(&hwq->lock);
+	for (i = 0; tids && nframes; i++, tids >>= 1) {
+		struct ieee80211_txq *txq = sta->txq[i];
+		struct mt76_txq *mtxq = (struct mt76_txq *) txq->drv_priv;
+		struct sk_buff *skb;
+
+		if (!(tids & 1))
+			continue;
+
+		do {
+			skb = mt76_txq_dequeue(dev, mtxq, true);
+			if (!skb)
+				break;
+
+			if (mtxq->aggr)
+				mt76_check_agg_ssn(mtxq, skb);
+
+			nframes--;
+			if (last_skb)
+				mt76_queue_ps_skb(dev, sta, last_skb, false);
+
+			last_skb = skb;
+		} while (nframes);
+	}
+
+	if (last_skb) {
+		mt76_queue_ps_skb(dev, sta, last_skb, true);
+		dev->queue_ops->kick(dev, hwq);
+	}
+	spin_unlock_bh(&hwq->lock);
+}
+EXPORT_SYMBOL_GPL(mt76_release_buffered_frames);
+
+static int
+mt76_txq_send_burst(struct mt76_dev *dev, struct mt76_queue *hwq,
+		    struct mt76_txq *mtxq, bool *empty)
+{
+	struct ieee80211_txq *txq = mtxq_to_txq(mtxq);
+	struct ieee80211_tx_info *info;
+	struct mt76_wcid *wcid = mtxq->wcid;
+	struct sk_buff *skb;
+	int n_frames = 1, limit;
+	struct ieee80211_tx_rate tx_rate;
+	bool ampdu;
+	bool probe;
+	int idx;
+
+	skb = mt76_txq_dequeue(dev, mtxq, false);
+	if (!skb) {
+		*empty = true;
+		return 0;
+	}
+
+	info = IEEE80211_SKB_CB(skb);
+	if (!wcid->tx_rate_set)
+		ieee80211_get_tx_rates(txq->vif, txq->sta, skb,
+				       info->control.rates, 1);
+	tx_rate = info->control.rates[0];
+
+	probe = (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE);
+	ampdu = IEEE80211_SKB_CB(skb)->flags & IEEE80211_TX_CTL_AMPDU;
+	limit = ampdu ? 16 : 3;
+
+	if (ampdu)
+		mt76_check_agg_ssn(mtxq, skb);
+
+	idx = mt76_tx_queue_skb(dev, hwq, skb, wcid, txq->sta);
+
+	if (idx < 0)
+		return idx;
+
+	do {
+		bool cur_ampdu;
+
+		if (probe)
+			break;
+
+		if (test_bit(MT76_SCANNING, &dev->state) ||
+		    test_bit(MT76_RESET, &dev->state))
+			return -EBUSY;
+
+		skb = mt76_txq_dequeue(dev, mtxq, false);
+		if (!skb) {
+			*empty = true;
+			break;
+		}
+
+		info = IEEE80211_SKB_CB(skb);
+		cur_ampdu = info->flags & IEEE80211_TX_CTL_AMPDU;
+
+		if (ampdu != cur_ampdu ||
+		    (info->flags & IEEE80211_TX_CTL_RATE_CTRL_PROBE)) {
+			skb_queue_tail(&mtxq->retry_q, skb);
+			break;
+		}
+
+		info->control.rates[0] = tx_rate;
+
+		if (cur_ampdu)
+			mt76_check_agg_ssn(mtxq, skb);
+
+		idx = mt76_tx_queue_skb(dev, hwq, skb, wcid, txq->sta);
+		if (idx < 0)
+			return idx;
+
+		n_frames++;
+	} while (n_frames < limit);
+
+	if (!probe) {
+		hwq->swq_queued++;
+		hwq->entry[idx].schedule = true;
+	}
+
+	dev->queue_ops->kick(dev, hwq);
+
+	return n_frames;
+}
+
+static int
+mt76_txq_schedule_list(struct mt76_dev *dev, struct mt76_queue *hwq)
+{
+	struct mt76_txq *mtxq, *mtxq_last;
+	int len = 0;
+
+restart:
+	mtxq_last = list_last_entry(&hwq->swq, struct mt76_txq, list);
+	while (!list_empty(&hwq->swq)) {
+		bool empty = false;
+		int cur;
+
+		mtxq = list_first_entry(&hwq->swq, struct mt76_txq, list);
+		if (mtxq->send_bar && mtxq->aggr) {
+			struct ieee80211_txq *txq = mtxq_to_txq(mtxq);
+			struct ieee80211_sta *sta = txq->sta;
+			struct ieee80211_vif *vif = txq->vif;
+			u16 agg_ssn = mtxq->agg_ssn;
+			u8 tid = txq->tid;
+
+			mtxq->send_bar = false;
+			spin_unlock_bh(&hwq->lock);
+			ieee80211_send_bar(vif, sta->addr, tid, agg_ssn);
+			spin_lock_bh(&hwq->lock);
+			goto restart;
+		}
+
+		list_del_init(&mtxq->list);
+
+		cur = mt76_txq_send_burst(dev, hwq, mtxq, &empty);
+		if (!empty)
+			list_add_tail(&mtxq->list, &hwq->swq);
+
+		if (cur < 0)
+			return cur;
+
+		len += cur;
+
+		if (mtxq == mtxq_last)
+			break;
+	}
+
+	return len;
+}
+
+void mt76_txq_schedule(struct mt76_dev *dev, struct mt76_queue *hwq)
+{
+	int len;
+
+	do {
+		if (hwq->swq_queued >= 4 || list_empty(&hwq->swq))
+			break;
+
+		len = mt76_txq_schedule_list(dev, hwq);
+	} while (len > 0);
+}
+EXPORT_SYMBOL_GPL(mt76_txq_schedule);
+
+void mt76_txq_schedule_all(struct mt76_dev *dev)
+{
+	int i;
+
+	for (i = 0; i <= MT_TXQ_BK; i++) {
+		struct mt76_queue *q = &dev->q_tx[i];
+
+		spin_lock_bh(&q->lock);
+		mt76_txq_schedule(dev, q);
+		spin_unlock_bh(&q->lock);
+	}
+}
+EXPORT_SYMBOL_GPL(mt76_txq_schedule_all);
+
+void mt76_stop_tx_queues(struct mt76_dev *dev, struct ieee80211_sta *sta,
+			 bool send_bar)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sta->txq); i++) {
+		struct ieee80211_txq *txq = sta->txq[i];
+		struct mt76_txq *mtxq = (struct mt76_txq *) txq->drv_priv;
+
+		spin_lock_bh(&mtxq->hwq->lock);
+		mtxq->send_bar = mtxq->aggr && send_bar;
+		if (!list_empty(&mtxq->list))
+			list_del_init(&mtxq->list);
+		spin_unlock_bh(&mtxq->hwq->lock);
+	}
+}
+EXPORT_SYMBOL_GPL(mt76_stop_tx_queues);
+
+void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq)
+{
+	struct mt76_dev *dev = hw->priv;
+	struct mt76_txq *mtxq = (struct mt76_txq *) txq->drv_priv;
+	struct mt76_queue *hwq = mtxq->hwq;
+
+	spin_lock_bh(&hwq->lock);
+	if (list_empty(&mtxq->list))
+		list_add_tail(&mtxq->list, &hwq->swq);
+	mt76_txq_schedule(dev, hwq);
+	spin_unlock_bh(&hwq->lock);
+}
+EXPORT_SYMBOL_GPL(mt76_wake_tx_queue);
+
+void mt76_txq_remove(struct mt76_dev *dev, struct ieee80211_txq *txq)
+{
+	struct mt76_txq *mtxq;
+	struct mt76_queue *hwq;
+	struct sk_buff *skb;
+
+	if (!txq)
+		return;
+
+	mtxq = (struct mt76_txq *) txq->drv_priv;
+	hwq = mtxq->hwq;
+
+	spin_lock_bh(&hwq->lock);
+	if (!list_empty(&mtxq->list))
+		list_del(&mtxq->list);
+	spin_unlock_bh(&hwq->lock);
+
+	while ((skb = skb_dequeue(&mtxq->retry_q)) != NULL)
+		ieee80211_free_txskb(dev->hw, skb);
+}
+EXPORT_SYMBOL_GPL(mt76_txq_remove);
+
+void mt76_txq_init(struct mt76_dev *dev, struct ieee80211_txq *txq)
+{
+	struct mt76_txq *mtxq = (struct mt76_txq *) txq->drv_priv;
+
+	INIT_LIST_HEAD(&mtxq->list);
+	skb_queue_head_init(&mtxq->retry_q);
+
+	mtxq->hwq = &dev->q_tx[mt76_txq_get_qid(txq)];
+}
+EXPORT_SYMBOL_GPL(mt76_txq_init);
diff --git a/drivers/net/wireless/mediatek/mt76/util.c b/drivers/net/wireless/mediatek/mt76/util.c
new file mode 100644
index 000000000000..0c35b8db58cd
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/util.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include "mt76.h"
+
+bool __mt76_poll(struct mt76_dev *dev, u32 offset, u32 mask, u32 val,
+		 int timeout)
+{
+	u32 cur;
+
+	timeout /= 10;
+	do {
+		cur = dev->bus->rr(dev, offset) & mask;
+		if (cur == val)
+			return true;
+
+		udelay(10);
+	} while (timeout-- > 0);
+
+	return false;
+}
+EXPORT_SYMBOL_GPL(__mt76_poll);
+
+bool __mt76_poll_msec(struct mt76_dev *dev, u32 offset, u32 mask, u32 val,
+		      int timeout)
+{
+	u32 cur;
+
+	timeout /= 10;
+	do {
+		cur = dev->bus->rr(dev, offset) & mask;
+		if (cur == val)
+			return true;
+
+		usleep_range(10000, 20000);
+	} while (timeout-- > 0);
+
+	return false;
+}
+EXPORT_SYMBOL_GPL(__mt76_poll_msec);
+
+int mt76_wcid_alloc(unsigned long *mask, int size)
+{
+	int i, idx = 0, cur;
+
+	for (i = 0; i < size / BITS_PER_LONG; i++) {
+		idx = ffs(~mask[i]);
+		if (!idx)
+			continue;
+
+		idx--;
+		cur = i * BITS_PER_LONG + idx;
+		if (cur >= size)
+			break;
+
+		mask[i] |= BIT(idx);
+		return cur;
+	}
+
+	return -1;
+}
+EXPORT_SYMBOL_GPL(mt76_wcid_alloc);
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/mediatek/mt76/util.h b/drivers/net/wireless/mediatek/mt76/util.h
new file mode 100644
index 000000000000..018d475504a2
--- /dev/null
+++ b/drivers/net/wireless/mediatek/mt76/util.h
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
+ * Copyright (C) 2004 - 2009 Ivo van Doorn <IvDoorn@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef __MT76_UTIL_H
+#define __MT76_UTIL_H
+
+#include <linux/skbuff.h>
+#include <linux/bitops.h>
+#include <linux/bitfield.h>
+
+#define MT76_INCR(_var, _size) \
+	_var = (((_var) + 1) % _size)
+
+int mt76_wcid_alloc(unsigned long *mask, int size);
+
+static inline void
+mt76_wcid_free(unsigned long *mask, int idx)
+{
+	mask[idx / BITS_PER_LONG] &= ~BIT(idx % BITS_PER_LONG);
+}
+
+static inline void
+mt76_skb_set_moredata(struct sk_buff *skb, bool enable)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
+
+	if (enable)
+		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_MOREDATA);
+	else
+		hdr->frame_control &= ~cpu_to_le16(IEEE80211_FCTL_MOREDATA);
+}
+
+#endif
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
index 6711e7fb6926..0398bece5782 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
@@ -80,6 +80,41 @@ qtnf_mgmt_stypes[NUM_NL80211_IFTYPES] = {
 	},
 };
 
+static int
+qtnf_validate_iface_combinations(struct wiphy *wiphy,
+				 struct qtnf_vif *change_vif,
+				 enum nl80211_iftype new_type)
+{
+	struct qtnf_wmac *mac;
+	struct qtnf_vif *vif;
+	int i;
+	int ret = 0;
+	struct iface_combination_params params = {
+		.num_different_channels = 1,
+	};
+
+	mac = wiphy_priv(wiphy);
+	if (!mac)
+		return -EFAULT;
+
+	for (i = 0; i < QTNF_MAX_INTF; i++) {
+		vif = &mac->iflist[i];
+		if (vif->wdev.iftype != NL80211_IFTYPE_UNSPECIFIED)
+			params.iftype_num[vif->wdev.iftype]++;
+	}
+
+	if (change_vif) {
+		params.iftype_num[new_type]++;
+		params.iftype_num[change_vif->wdev.iftype]--;
+	} else {
+		params.iftype_num[new_type]++;
+	}
+
+	ret = cfg80211_check_combinations(wiphy, &params);
+
+	return ret;
+}
+
 static int
 qtnf_change_virtual_intf(struct wiphy *wiphy,
 			 struct net_device *dev,
@@ -90,6 +125,13 @@ qtnf_change_virtual_intf(struct wiphy *wiphy,
 	u8 *mac_addr;
 	int ret;
 
+	ret = qtnf_validate_iface_combinations(wiphy, vif, type);
+	if (ret) {
+		pr_err("VIF%u.%u combination check: failed to set type %d\n",
+		       vif->mac->macid, vif->vifid, type);
+		return ret;
+	}
+
 	if (params)
 		mac_addr = params->macaddr;
 	else
@@ -120,10 +162,6 @@ int qtnf_del_virtual_intf(struct wiphy *wiphy, struct wireless_dev *wdev)
 
 	qtnf_scan_done(vif->mac, true);
 
-	if (qtnf_cmd_send_del_intf(vif))
-		pr_err("VIF%u.%u: failed to delete VIF\n", vif->mac->macid,
-		       vif->vifid);
-
 	/* Stop data */
 	netif_tx_stop_all_queues(netdev);
 	if (netif_carrier_ok(netdev))
@@ -132,6 +170,10 @@ int qtnf_del_virtual_intf(struct wiphy *wiphy, struct wireless_dev *wdev)
 	if (netdev->reg_state == NETREG_REGISTERED)
 		unregister_netdevice(netdev);
 
+	if (qtnf_cmd_send_del_intf(vif))
+		pr_err("VIF%u.%u: failed to delete VIF\n", vif->mac->macid,
+		       vif->vifid);
+
 	vif->netdev->ieee80211_ptr = NULL;
 	vif->netdev = NULL;
 	vif->wdev.iftype = NL80211_IFTYPE_UNSPECIFIED;
@@ -150,12 +192,20 @@ static struct wireless_dev *qtnf_add_virtual_intf(struct wiphy *wiphy,
 	struct qtnf_wmac *mac;
 	struct qtnf_vif *vif;
 	u8 *mac_addr = NULL;
+	int ret;
 
 	mac = wiphy_priv(wiphy);
 
 	if (!mac)
 		return ERR_PTR(-EFAULT);
 
+	ret = qtnf_validate_iface_combinations(wiphy, NULL, type);
+	if (ret) {
+		pr_err("MAC%u invalid combination: failed to add type %d\n",
+		       mac->macid, type);
+		return ERR_PTR(ret);
+	}
+
 	switch (type) {
 	case NL80211_IFTYPE_STATION:
 	case NL80211_IFTYPE_AP:
@@ -190,7 +240,7 @@ static struct wireless_dev *qtnf_add_virtual_intf(struct wiphy *wiphy,
 		goto err_mac;
 	}
 
-	if (qtnf_core_net_attach(mac, vif, name, name_assign_t, type)) {
+	if (qtnf_core_net_attach(mac, vif, name, name_assign_t)) {
 		pr_err("VIF%u.%u: failed to attach netdev\n", mac->macid,
 		       vif->vifid);
 		goto err_net;
@@ -381,6 +431,7 @@ qtnf_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	const struct ieee80211_mgmt *mgmt_frame = (void *)params->buf;
 	u32 short_cookie = prandom_u32();
 	u16 flags = 0;
+	u16 freq;
 
 	*cookie = short_cookie;
 
@@ -393,13 +444,21 @@ qtnf_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	if (params->dont_wait_for_ack)
 		flags |= QLINK_MGMT_FRAME_TX_FLAG_ACK_NOWAIT;
 
+	/* If channel is not specified, pass "freq = 0" to tell device
+	 * firmware to use current channel.
+	 */
+	if (params->chan)
+		freq = params->chan->center_freq;
+	else
+		freq = 0;
+
 	pr_debug("%s freq:%u; FC:%.4X; DA:%pM; len:%zu; C:%.8X; FL:%.4X\n",
-		 wdev->netdev->name, params->chan->center_freq,
+		 wdev->netdev->name, freq,
 		 le16_to_cpu(mgmt_frame->frame_control), mgmt_frame->da,
 		 params->len, short_cookie, flags);
 
 	return qtnf_cmd_send_mgmt_frame(vif, short_cookie, flags,
-					params->chan->center_freq,
+					freq,
 					params->buf, params->len);
 }
 
@@ -409,6 +468,7 @@ qtnf_get_station(struct wiphy *wiphy, struct net_device *dev,
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
 
+	sinfo->generation = vif->generation;
 	return qtnf_cmd_get_sta_info(vif, mac, sinfo);
 }
 
@@ -430,11 +490,13 @@ qtnf_dump_station(struct wiphy *wiphy, struct net_device *dev,
 	ret = qtnf_cmd_get_sta_info(vif, sta_node->mac_addr, sinfo);
 
 	if (unlikely(ret == -ENOENT)) {
-		qtnf_sta_list_del(&vif->sta_list, mac);
+		qtnf_sta_list_del(vif, mac);
 		cfg80211_del_sta(vif->netdev, mac, GFP_KERNEL);
 		sinfo->filled = 0;
 	}
 
+	sinfo->generation = vif->generation;
+
 	return ret;
 }
 
@@ -533,19 +595,13 @@ qtnf_del_station(struct wiphy *wiphy, struct net_device *dev,
 	return ret;
 }
 
-static void qtnf_scan_timeout(struct timer_list *t)
-{
-	struct qtnf_wmac *mac = from_timer(mac, t, scan_timeout);
-
-	pr_warn("mac%d scan timed out\n", mac->macid);
-	qtnf_scan_done(mac, true);
-}
-
 static int
 qtnf_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request)
 {
 	struct qtnf_wmac *mac = wiphy_priv(wiphy);
 
+	cancel_delayed_work_sync(&mac->scan_timeout);
+
 	mac->scan_req = request;
 
 	if (qtnf_cmd_send_scan(mac)) {
@@ -554,9 +610,8 @@ qtnf_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request)
 		return -EFAULT;
 	}
 
-	mac->scan_timeout.function = qtnf_scan_timeout;
-	mod_timer(&mac->scan_timeout,
-		  jiffies + QTNF_SCAN_TIMEOUT_SEC * HZ);
+	queue_delayed_work(mac->bus->workqueue, &mac->scan_timeout,
+			   QTNF_SCAN_TIMEOUT_SEC * HZ);
 
 	return 0;
 }
@@ -617,7 +672,6 @@ qtnf_disconnect(struct wiphy *wiphy, struct net_device *dev,
 		return ret;
 	}
 
-	vif->sta_state = QTNF_STA_DISCONNECTED;
 	return 0;
 }
 
@@ -717,7 +771,8 @@ qtnf_get_channel(struct wiphy *wiphy, struct wireless_dev *wdev,
 	}
 
 	if (!cfg80211_chandef_valid(chandef)) {
-		pr_err("%s: bad chan freq1=%u freq2=%u bw=%u\n", ndev->name,
+		pr_err("%s: bad channel freq=%u cf1=%u cf2=%u bw=%u\n",
+		       ndev->name, chandef->chan->center_freq,
 		       chandef->center_freq1, chandef->center_freq2,
 		       chandef->width);
 		ret = -ENODATA;
@@ -750,6 +805,35 @@ static int qtnf_channel_switch(struct wiphy *wiphy, struct net_device *dev,
 	return ret;
 }
 
+static int qtnf_start_radar_detection(struct wiphy *wiphy,
+				      struct net_device *ndev,
+				      struct cfg80211_chan_def *chandef,
+				      u32 cac_time_ms)
+{
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(ndev);
+	int ret;
+
+	ret = qtnf_cmd_start_cac(vif, chandef, cac_time_ms);
+	if (ret)
+		pr_err("%s: failed to start CAC ret=%d\n", ndev->name, ret);
+
+	return ret;
+}
+
+static int qtnf_set_mac_acl(struct wiphy *wiphy,
+			    struct net_device *dev,
+			    const struct cfg80211_acl_data *params)
+{
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
+	int ret;
+
+	ret = qtnf_cmd_set_mac_acl(vif, params);
+	if (ret)
+		pr_err("%s: failed to set mac ACL ret=%d\n", dev->name, ret);
+
+	return ret;
+}
+
 static struct cfg80211_ops qtn_cfg80211_ops = {
 	.add_virtual_intf	= qtnf_add_virtual_intf,
 	.change_virtual_intf	= qtnf_change_virtual_intf,
@@ -773,7 +857,9 @@ static struct cfg80211_ops qtn_cfg80211_ops = {
 	.disconnect		= qtnf_disconnect,
 	.dump_survey		= qtnf_dump_survey,
 	.get_channel		= qtnf_get_channel,
-	.channel_switch		= qtnf_channel_switch
+	.channel_switch		= qtnf_channel_switch,
+	.start_radar_detection	= qtnf_start_radar_detection,
+	.set_mac_acl		= qtnf_set_mac_acl,
 };
 
 static void qtnf_cfg80211_reg_notifier(struct wiphy *wiphy_in,
@@ -802,6 +888,9 @@ static void qtnf_cfg80211_reg_notifier(struct wiphy *wiphy_in,
 			continue;
 
 		mac = bus->mac[mac_idx];
+		if (!mac)
+			continue;
+
 		wiphy = priv_to_wiphy(mac);
 
 		for (band = 0; band < NUM_NL80211_BANDS; ++band) {
@@ -829,29 +918,29 @@ struct wiphy *qtnf_wiphy_allocate(struct qtnf_bus *bus)
 	return wiphy;
 }
 
-static int qtnf_wiphy_setup_if_comb(struct wiphy *wiphy,
-				    struct ieee80211_iface_combination *if_comb,
-				    const struct qtnf_mac_info *mac_info)
+static int
+qtnf_wiphy_setup_if_comb(struct wiphy *wiphy, struct qtnf_mac_info *mac_info)
 {
-	size_t max_interfaces = 0;
+	struct ieee80211_iface_combination *if_comb;
+	size_t n_if_comb;
 	u16 interface_modes = 0;
-	size_t i;
+	size_t i, j;
+
+	if_comb = mac_info->if_comb;
+	n_if_comb = mac_info->n_if_comb;
 
-	if (unlikely(!mac_info->limits || !mac_info->n_limits))
+	if (!if_comb || !n_if_comb)
 		return -ENOENT;
 
-	if_comb->limits = mac_info->limits;
-	if_comb->n_limits = mac_info->n_limits;
+	for (i = 0; i < n_if_comb; i++) {
+		if_comb[i].radar_detect_widths = mac_info->radar_detect_widths;
 
-	for (i = 0; i < mac_info->n_limits; i++) {
-		max_interfaces += mac_info->limits[i].max;
-		interface_modes |= mac_info->limits[i].types;
+		for (j = 0; j < if_comb[i].n_limits; j++)
+			interface_modes |= if_comb[i].limits[j].types;
 	}
 
-	if_comb->num_different_channels = 1;
-	if_comb->beacon_int_infra_match = true;
-	if_comb->max_interfaces = max_interfaces;
-	if_comb->radar_detect_widths = mac_info->radar_detect_widths;
+	wiphy->iface_combinations = if_comb;
+	wiphy->n_iface_combinations = n_if_comb;
 	wiphy->interface_modes = interface_modes;
 
 	return 0;
@@ -860,7 +949,6 @@ static int qtnf_wiphy_setup_if_comb(struct wiphy *wiphy,
 int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 {
 	struct wiphy *wiphy = priv_to_wiphy(mac);
-	struct ieee80211_iface_combination *iface_comb = NULL;
 	int ret;
 
 	if (!wiphy) {
@@ -868,14 +956,6 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 		return -EFAULT;
 	}
 
-	iface_comb = kzalloc(sizeof(*iface_comb), GFP_KERNEL);
-	if (!iface_comb)
-		return -ENOMEM;
-
-	ret = qtnf_wiphy_setup_if_comb(wiphy, iface_comb, &mac->macinfo);
-	if (ret)
-		goto out;
-
 	wiphy->frag_threshold = mac->macinfo.frag_thr;
 	wiphy->rts_threshold = mac->macinfo.rts_thr;
 	wiphy->retry_short = mac->macinfo.sretry_limit;
@@ -886,11 +966,13 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 	wiphy->max_scan_ie_len = QTNF_MAX_VSIE_LEN;
 	wiphy->mgmt_stypes = qtnf_mgmt_stypes;
 	wiphy->max_remain_on_channel_duration = 5000;
-
-	wiphy->iface_combinations = iface_comb;
-	wiphy->n_iface_combinations = 1;
+	wiphy->max_acl_mac_addrs = mac->macinfo.max_acl_mac_addrs;
 	wiphy->max_num_csa_counters = 2;
 
+	ret = qtnf_wiphy_setup_if_comb(wiphy, &mac->macinfo);
+	if (ret)
+		goto out;
+
 	/* Initialize cipher suits */
 	wiphy->cipher_suites = qtnf_cipher_suites;
 	wiphy->n_cipher_suites = ARRAY_SIZE(qtnf_cipher_suites);
@@ -924,6 +1006,10 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 		wiphy->regulatory_flags |= REGULATORY_WIPHY_SELF_MANAGED;
 	}
 
+	strlcpy(wiphy->fw_version, hw_info->fw_version,
+		sizeof(wiphy->fw_version));
+	wiphy->hw_version = hw_info->hw_version;
+
 	ret = wiphy_register(wiphy);
 	if (ret < 0)
 		goto out;
@@ -935,12 +1021,7 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 		ret = regulatory_hint(wiphy, hw_info->rd->alpha2);
 
 out:
-	if (ret) {
-		kfree(iface_comb);
-		return ret;
-	}
-
-	return 0;
+	return ret;
 }
 
 void qtnf_netdev_updown(struct net_device *ndev, bool up)
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h
index 66db26613b1f..b73425122a10 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.h
@@ -28,23 +28,4 @@ void qtnf_band_init_rates(struct ieee80211_supported_band *band);
 void qtnf_band_setup_htvht_caps(struct qtnf_mac_info *macinfo,
 				struct ieee80211_supported_band *band);
 
-static inline void qtnf_scan_done(struct qtnf_wmac *mac, bool aborted)
-{
-	struct cfg80211_scan_info info = {
-		.aborted = aborted,
-	};
-
-	if (timer_pending(&mac->scan_timeout))
-		del_timer_sync(&mac->scan_timeout);
-
-	mutex_lock(&mac->mac_lock);
-
-	if (mac->scan_req) {
-		cfg80211_scan_done(mac->scan_req, &info);
-		mac->scan_req = NULL;
-	}
-
-	mutex_unlock(&mac->mac_lock);
-}
-
 #endif /* _QTN_FMAC_CFG80211_H_ */
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c
index 8bc8dd637315..deca0060eb27 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c
@@ -162,6 +162,14 @@ static void qtnf_cmd_tlv_ie_set_add(struct sk_buff *cmd_skb, u8 frame_type,
 		memcpy(tlv->ie_data, buf, len);
 }
 
+static inline size_t qtnf_cmd_acl_data_size(const struct cfg80211_acl_data *acl)
+{
+	size_t size = sizeof(struct qlink_acl_data) +
+		      acl->n_acl_entries * sizeof(struct qlink_mac_address);
+
+	return size;
+}
+
 static bool qtnf_cmd_start_ap_can_fit(const struct qtnf_vif *vif,
 				      const struct cfg80211_ap_settings *s)
 {
@@ -178,6 +186,10 @@ static bool qtnf_cmd_start_ap_can_fit(const struct qtnf_vif *vif,
 	if (cfg80211_chandef_valid(&s->chandef))
 		len += sizeof(struct qlink_tlv_chandef);
 
+	if (s->acl)
+		len += sizeof(struct qlink_tlv_hdr) +
+		       qtnf_cmd_acl_data_size(s->acl);
+
 	if (len > (sizeof(struct qlink_cmd) + QTNF_MAX_CMD_BUF_SIZE)) {
 		pr_err("VIF%u.%u: can not fit AP settings: %u\n",
 		       vif->mac->macid, vif->vifid, len);
@@ -203,7 +215,7 @@ int qtnf_cmd_send_start_ap(struct qtnf_vif *vif,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_START_AP,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	cmd = (struct qlink_cmd_start_ap *)cmd_skb->data;
@@ -247,7 +259,7 @@ int qtnf_cmd_send_start_ap(struct qtnf_vif *vif,
 		chtlv->hdr.type = cpu_to_le16(QTN_TLV_ID_CHANDEF);
 		chtlv->hdr.len = cpu_to_le16(sizeof(*chtlv) -
 					     sizeof(chtlv->hdr));
-		qlink_chandef_cfg2q(&s->chandef, &chtlv->chan);
+		qlink_chandef_cfg2q(&s->chandef, &chtlv->chdef);
 	}
 
 	qtnf_cmd_tlv_ie_set_add(cmd_skb, QLINK_IE_SET_BEACON_HEAD,
@@ -283,6 +295,16 @@ int qtnf_cmd_send_start_ap(struct qtnf_vif *vif,
 		memcpy(tlv->val, s->vht_cap, sizeof(*s->vht_cap));
 	}
 
+	if (s->acl) {
+		size_t acl_size = qtnf_cmd_acl_data_size(s->acl);
+		struct qlink_tlv_hdr *tlv =
+			skb_put(cmd_skb, sizeof(*tlv) + acl_size);
+
+		tlv->type = cpu_to_le16(QTN_TLV_ID_ACL_DATA);
+		tlv->len = cpu_to_le16(acl_size);
+		qlink_acl_data_cfg2q(s->acl, (struct qlink_acl_data *)tlv->val);
+	}
+
 	qtnf_bus_lock(vif->mac->bus);
 
 	ret = qtnf_cmd_send(vif->mac->bus, cmd_skb, &res_code);
@@ -313,7 +335,7 @@ int qtnf_cmd_send_stop_ap(struct qtnf_vif *vif)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_STOP_AP,
 					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -347,7 +369,7 @@ int qtnf_cmd_send_register_mgmt(struct qtnf_vif *vif, u16 frame_type, bool reg)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_REGISTER_MGMT,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -390,7 +412,7 @@ int qtnf_cmd_send_mgmt_frame(struct qtnf_vif *vif, u32 cookie, u16 flags,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_SEND_MGMT_FRAME,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -436,7 +458,7 @@ int qtnf_cmd_send_mgmt_set_appie(struct qtnf_vif *vif, u8 frame_type,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_MGMT_SET_APPIE,
 					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_cmd_tlv_ie_set_add(cmd_skb, frame_type, buf, len);
@@ -460,31 +482,9 @@ int qtnf_cmd_send_mgmt_set_appie(struct qtnf_vif *vif, u8 frame_type,
 	return ret;
 }
 
-static void
-qtnf_sta_info_parse_basic_counters(struct station_info *sinfo,
-		const struct qlink_sta_stat_basic_counters *counters)
-{
-	sinfo->filled |= BIT(NL80211_STA_INFO_RX_BYTES) |
-			 BIT(NL80211_STA_INFO_TX_BYTES);
-	sinfo->rx_bytes = get_unaligned_le64(&counters->rx_bytes);
-	sinfo->tx_bytes = get_unaligned_le64(&counters->tx_bytes);
-
-	sinfo->filled |= BIT(NL80211_STA_INFO_RX_PACKETS) |
-			 BIT(NL80211_STA_INFO_TX_PACKETS) |
-			 BIT(NL80211_STA_INFO_BEACON_RX);
-	sinfo->rx_packets = get_unaligned_le32(&counters->rx_packets);
-	sinfo->tx_packets = get_unaligned_le32(&counters->tx_packets);
-	sinfo->rx_beacon = get_unaligned_le64(&counters->rx_beacons);
-
-	sinfo->filled |= BIT(NL80211_STA_INFO_RX_DROP_MISC) |
-			 BIT(NL80211_STA_INFO_TX_FAILED);
-	sinfo->rx_dropped_misc = get_unaligned_le32(&counters->rx_dropped);
-	sinfo->tx_failed = get_unaligned_le32(&counters->tx_failed);
-}
-
 static void
 qtnf_sta_info_parse_rate(struct rate_info *rate_dst,
-			 const struct  qlink_sta_info_rate *rate_src)
+			 const struct qlink_sta_info_rate *rate_src)
 {
 	rate_dst->legacy = get_unaligned_le16(&rate_src->rate) * 10;
 
@@ -493,22 +493,23 @@ qtnf_sta_info_parse_rate(struct rate_info *rate_dst,
 	rate_dst->flags = 0;
 
 	switch (rate_src->bw) {
-	case QLINK_STA_INFO_RATE_BW_5:
+	case QLINK_CHAN_WIDTH_5:
 		rate_dst->bw = RATE_INFO_BW_5;
 		break;
-	case QLINK_STA_INFO_RATE_BW_10:
+	case QLINK_CHAN_WIDTH_10:
 		rate_dst->bw = RATE_INFO_BW_10;
 		break;
-	case QLINK_STA_INFO_RATE_BW_20:
+	case QLINK_CHAN_WIDTH_20:
+	case QLINK_CHAN_WIDTH_20_NOHT:
 		rate_dst->bw = RATE_INFO_BW_20;
 		break;
-	case QLINK_STA_INFO_RATE_BW_40:
+	case QLINK_CHAN_WIDTH_40:
 		rate_dst->bw = RATE_INFO_BW_40;
 		break;
-	case QLINK_STA_INFO_RATE_BW_80:
+	case QLINK_CHAN_WIDTH_80:
 		rate_dst->bw = RATE_INFO_BW_80;
 		break;
-	case QLINK_STA_INFO_RATE_BW_160:
+	case QLINK_CHAN_WIDTH_160:
 		rate_dst->bw = RATE_INFO_BW_160;
 		break;
 	default:
@@ -578,87 +579,125 @@ qtnf_sta_info_parse_flags(struct nl80211_sta_flag_update *dst,
 }
 
 static void
-qtnf_sta_info_parse_generic_info(struct station_info *sinfo,
-				 const struct qlink_sta_info_generic *info)
+qtnf_cmd_sta_info_parse(struct station_info *sinfo,
+			const struct qlink_tlv_hdr *tlv,
+			size_t resp_size)
 {
-	sinfo->filled |= BIT(NL80211_STA_INFO_CONNECTED_TIME) |
-			 BIT(NL80211_STA_INFO_INACTIVE_TIME);
-	sinfo->connected_time = get_unaligned_le32(&info->connected_time);
-	sinfo->inactive_time = get_unaligned_le32(&info->inactive_time);
+	const struct qlink_sta_stats *stats = NULL;
+	const u8 *map = NULL;
+	unsigned int map_len = 0;
+	unsigned int stats_len = 0;
+	u16 tlv_len;
+
+#define qtnf_sta_stat_avail(stat_name, bitn)	\
+	(qtnf_utils_is_bit_set(map, bitn, map_len) && \
+	 (offsetofend(struct qlink_sta_stats, stat_name) <= stats_len))
+
+	while (resp_size >= sizeof(*tlv)) {
+		tlv_len = le16_to_cpu(tlv->len);
+
+		switch (le16_to_cpu(tlv->type)) {
+		case QTN_TLV_ID_STA_STATS_MAP:
+			map_len = tlv_len;
+			map = tlv->val;
+			break;
+		case QTN_TLV_ID_STA_STATS:
+			stats_len = tlv_len;
+			stats = (const struct qlink_sta_stats *)tlv->val;
+			break;
+		default:
+			break;
+		}
 
-	sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL) |
-			 BIT(NL80211_STA_INFO_SIGNAL_AVG);
-	sinfo->signal = info->rssi - 120;
-	sinfo->signal_avg = info->rssi_avg - QLINK_RSSI_OFFSET;
+		resp_size -= tlv_len + sizeof(*tlv);
+		tlv = (const struct qlink_tlv_hdr *)(tlv->val + tlv_len);
+	}
 
-	if (info->rx_rate.rate) {
+	if (!map || !stats)
+		return;
+
+	if (qtnf_sta_stat_avail(inactive_time, QLINK_STA_INFO_INACTIVE_TIME)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_INACTIVE_TIME);
+		sinfo->inactive_time = le32_to_cpu(stats->inactive_time);
+	}
+
+	if (qtnf_sta_stat_avail(connected_time,
+				QLINK_STA_INFO_CONNECTED_TIME)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_CONNECTED_TIME);
+		sinfo->connected_time = le32_to_cpu(stats->connected_time);
+	}
+
+	if (qtnf_sta_stat_avail(signal, QLINK_STA_INFO_SIGNAL)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL);
+		sinfo->signal = stats->signal - QLINK_RSSI_OFFSET;
+	}
+
+	if (qtnf_sta_stat_avail(signal_avg, QLINK_STA_INFO_SIGNAL_AVG)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL_AVG);
+		sinfo->signal_avg = stats->signal_avg - QLINK_RSSI_OFFSET;
+	}
+
+	if (qtnf_sta_stat_avail(rxrate, QLINK_STA_INFO_RX_BITRATE)) {
 		sinfo->filled |= BIT(NL80211_STA_INFO_RX_BITRATE);
-		qtnf_sta_info_parse_rate(&sinfo->rxrate, &info->rx_rate);
+		qtnf_sta_info_parse_rate(&sinfo->rxrate, &stats->rxrate);
 	}
 
-	if (info->tx_rate.rate) {
+	if (qtnf_sta_stat_avail(txrate, QLINK_STA_INFO_TX_BITRATE)) {
 		sinfo->filled |= BIT(NL80211_STA_INFO_TX_BITRATE);
-		qtnf_sta_info_parse_rate(&sinfo->txrate, &info->tx_rate);
+		qtnf_sta_info_parse_rate(&sinfo->txrate, &stats->txrate);
 	}
 
-	sinfo->filled |= BIT(NL80211_STA_INFO_STA_FLAGS);
-	qtnf_sta_info_parse_flags(&sinfo->sta_flags, &info->state);
-}
+	if (qtnf_sta_stat_avail(sta_flags, QLINK_STA_INFO_STA_FLAGS)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_STA_FLAGS);
+		qtnf_sta_info_parse_flags(&sinfo->sta_flags, &stats->sta_flags);
+	}
 
-static int qtnf_cmd_sta_info_parse(struct station_info *sinfo,
-				   const u8 *payload, size_t payload_size)
-{
-	const struct qlink_sta_stat_basic_counters *counters;
-	const struct qlink_sta_info_generic *sta_info;
-	u16 tlv_type;
-	u16 tlv_value_len;
-	size_t tlv_full_len;
-	const struct qlink_tlv_hdr *tlv;
+	if (qtnf_sta_stat_avail(rx_bytes, QLINK_STA_INFO_RX_BYTES)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_RX_BYTES);
+		sinfo->rx_bytes = le64_to_cpu(stats->rx_bytes);
+	}
 
-	sinfo->filled = 0;
+	if (qtnf_sta_stat_avail(tx_bytes, QLINK_STA_INFO_TX_BYTES)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_TX_BYTES);
+		sinfo->tx_bytes = le64_to_cpu(stats->tx_bytes);
+	}
 
-	tlv = (const struct qlink_tlv_hdr *)payload;
-	while (payload_size >= sizeof(struct qlink_tlv_hdr)) {
-		tlv_type = le16_to_cpu(tlv->type);
-		tlv_value_len = le16_to_cpu(tlv->len);
-		tlv_full_len = tlv_value_len + sizeof(struct qlink_tlv_hdr);
-		if (tlv_full_len > payload_size) {
-			pr_warn("malformed TLV 0x%.2X; LEN: %u\n",
-				tlv_type, tlv_value_len);
-			return -EINVAL;
-		}
-		switch (tlv_type) {
-		case QTN_TLV_ID_STA_BASIC_COUNTERS:
-			if (unlikely(tlv_value_len < sizeof(*counters))) {
-				pr_err("invalid TLV size %.4X: %u\n",
-				       tlv_type, tlv_value_len);
-				break;
-			}
+	if (qtnf_sta_stat_avail(rx_bytes, QLINK_STA_INFO_RX_BYTES64)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_RX_BYTES64);
+		sinfo->rx_bytes = le64_to_cpu(stats->rx_bytes);
+	}
 
-			counters = (void *)tlv->val;
-			qtnf_sta_info_parse_basic_counters(sinfo, counters);
-			break;
-		case QTN_TLV_ID_STA_GENERIC_INFO:
-			if (unlikely(tlv_value_len < sizeof(*sta_info)))
-				break;
+	if (qtnf_sta_stat_avail(tx_bytes, QLINK_STA_INFO_TX_BYTES64)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_TX_BYTES64);
+		sinfo->tx_bytes = le64_to_cpu(stats->tx_bytes);
+	}
 
-			sta_info = (void *)tlv->val;
-			qtnf_sta_info_parse_generic_info(sinfo, sta_info);
-			break;
-		default:
-			pr_warn("unexpected TLV type: %.4X\n", tlv_type);
-			break;
-		}
-		payload_size -= tlv_full_len;
-		tlv = (struct qlink_tlv_hdr *)(tlv->val + tlv_value_len);
+	if (qtnf_sta_stat_avail(rx_packets, QLINK_STA_INFO_RX_PACKETS)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_RX_PACKETS);
+		sinfo->rx_packets = le32_to_cpu(stats->rx_packets);
 	}
 
-	if (payload_size) {
-		pr_warn("malformed TLV buf; bytes left: %zu\n", payload_size);
-		return -EINVAL;
+	if (qtnf_sta_stat_avail(tx_packets, QLINK_STA_INFO_TX_PACKETS)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_TX_PACKETS);
+		sinfo->tx_packets = le32_to_cpu(stats->tx_packets);
 	}
 
-	return 0;
+	if (qtnf_sta_stat_avail(rx_beacon, QLINK_STA_INFO_BEACON_RX)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_BEACON_RX);
+		sinfo->rx_beacon = le64_to_cpu(stats->rx_beacon);
+	}
+
+	if (qtnf_sta_stat_avail(rx_dropped_misc, QLINK_STA_INFO_RX_DROP_MISC)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_RX_DROP_MISC);
+		sinfo->rx_dropped_misc = le32_to_cpu(stats->rx_dropped_misc);
+	}
+
+	if (qtnf_sta_stat_avail(tx_failed, QLINK_STA_INFO_TX_FAILED)) {
+		sinfo->filled |= BIT(NL80211_STA_INFO_TX_FAILED);
+		sinfo->tx_failed = le32_to_cpu(stats->tx_failed);
+	}
+
+#undef qtnf_sta_stat_avail
 }
 
 int qtnf_cmd_get_sta_info(struct qtnf_vif *vif, const u8 *sta_mac,
@@ -674,8 +713,7 @@ int qtnf_cmd_get_sta_info(struct qtnf_vif *vif, const u8 *sta_mac,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_GET_STA_INFO,
 					    sizeof(*cmd));
-
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -715,7 +753,9 @@ int qtnf_cmd_get_sta_info(struct qtnf_vif *vif, const u8 *sta_mac,
 		goto out;
 	}
 
-	ret = qtnf_cmd_sta_info_parse(sinfo, resp->info, var_resp_len);
+	qtnf_cmd_sta_info_parse(sinfo,
+				(const struct qlink_tlv_hdr *)resp->info,
+				var_resp_len);
 
 out:
 	qtnf_bus_unlock(vif->mac->bus);
@@ -738,7 +778,7 @@ static int qtnf_cmd_send_add_change_intf(struct qtnf_vif *vif,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    cmd_type,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -811,7 +851,7 @@ int qtnf_cmd_send_del_intf(struct qtnf_vif *vif)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_DEL_INTF,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -908,6 +948,16 @@ qtnf_cmd_resp_proc_hw_info(struct qtnf_bus *bus,
 	struct qtnf_hw_info *hwinfo = &bus->hw_info;
 	const struct qlink_tlv_hdr *tlv;
 	const struct qlink_tlv_reg_rule *tlv_rule;
+	const char *bld_name = NULL;
+	const char *bld_rev = NULL;
+	const char *bld_type = NULL;
+	const char *bld_label = NULL;
+	u32 bld_tmstamp = 0;
+	u32 plat_id = 0;
+	const char *hw_id = NULL;
+	const char *calibration_ver = NULL;
+	const char *uboot_ver = NULL;
+	u32 hw_ver = 0;
 	struct ieee80211_reg_rule *rule;
 	u16 tlv_type;
 	u16 tlv_value_len;
@@ -934,6 +984,10 @@ qtnf_cmd_resp_proc_hw_info(struct qtnf_bus *bus,
 	hwinfo->rd->alpha2[0] = resp->alpha2[0];
 	hwinfo->rd->alpha2[1] = resp->alpha2[1];
 
+	bld_tmstamp = le32_to_cpu(resp->bld_tmstamp);
+	plat_id = le32_to_cpu(resp->plat_id);
+	hw_ver = le32_to_cpu(resp->hw_ver);
+
 	switch (resp->dfs_region) {
 	case QLINK_DFS_FCC:
 		hwinfo->rd->dfs_region = NL80211_DFS_FCC;
@@ -994,6 +1048,27 @@ qtnf_cmd_resp_proc_hw_info(struct qtnf_bus *bus,
 			rule->flags = qtnf_cmd_resp_reg_rule_flags_parse(
 					le32_to_cpu(tlv_rule->flags));
 			break;
+		case QTN_TLV_ID_BUILD_NAME:
+			bld_name = (const void *)tlv->val;
+			break;
+		case QTN_TLV_ID_BUILD_REV:
+			bld_rev = (const void *)tlv->val;
+			break;
+		case QTN_TLV_ID_BUILD_TYPE:
+			bld_type = (const void *)tlv->val;
+			break;
+		case QTN_TLV_ID_BUILD_LABEL:
+			bld_label = (const void *)tlv->val;
+			break;
+		case QTN_TLV_ID_HW_ID:
+			hw_id = (const void *)tlv->val;
+			break;
+		case QTN_TLV_ID_CALIBRATION_VER:
+			calibration_ver = (const void *)tlv->val;
+			break;
+		case QTN_TLV_ID_UBOOT_VER:
+			uboot_ver = (const void *)tlv->val;
+			break;
 		default:
 			break;
 		}
@@ -1016,21 +1091,46 @@ qtnf_cmd_resp_proc_hw_info(struct qtnf_bus *bus,
 		hwinfo->total_tx_chain, hwinfo->total_rx_chain,
 		hwinfo->hw_capab);
 
+	pr_info("\nBuild name:            %s"  \
+		"\nBuild revision:        %s"  \
+		"\nBuild type:            %s"  \
+		"\nBuild label:           %s"  \
+		"\nBuild timestamp:       %lu" \
+		"\nPlatform ID:           %lu" \
+		"\nHardware ID:           %s"  \
+		"\nCalibration version:   %s"  \
+		"\nU-Boot version:        %s"  \
+		"\nHardware version:      0x%08x",
+		bld_name, bld_rev, bld_type, bld_label,
+		(unsigned long)bld_tmstamp,
+		(unsigned long)plat_id,
+		hw_id, calibration_ver, uboot_ver, hw_ver);
+
+	strlcpy(hwinfo->fw_version, bld_label, sizeof(hwinfo->fw_version));
+	hwinfo->hw_version = hw_ver;
+
 	return 0;
 }
 
 static int qtnf_parse_variable_mac_info(struct qtnf_wmac *mac,
 					const u8 *tlv_buf, size_t tlv_buf_size)
 {
-	struct ieee80211_iface_limit *limits = NULL;
-	const struct qlink_iface_limit *limit_record;
-	size_t record_count = 0, rec = 0;
-	u16 tlv_type, tlv_value_len;
-	struct qlink_iface_comb_num *comb;
+	struct ieee80211_iface_combination *comb = NULL;
+	size_t n_comb = 0;
+	struct ieee80211_iface_limit *limits;
+	const struct qlink_iface_comb_num *comb_num;
+	const struct qlink_iface_limit_record *rec;
+	const struct qlink_iface_limit *lim;
+	u16 rec_len;
+	u16 tlv_type;
+	u16 tlv_value_len;
 	size_t tlv_full_len;
 	const struct qlink_tlv_hdr *tlv;
-
-	mac->macinfo.n_limits = 0;
+	u8 *ext_capa = NULL;
+	u8 *ext_capa_mask = NULL;
+	u8 ext_capa_len = 0;
+	u8 ext_capa_mask_len = 0;
+	int i = 0;
 
 	tlv = (const struct qlink_tlv_hdr *)tlv_buf;
 	while (tlv_buf_size >= sizeof(struct qlink_tlv_hdr)) {
@@ -1045,52 +1145,89 @@ static int qtnf_parse_variable_mac_info(struct qtnf_wmac *mac,
 
 		switch (tlv_type) {
 		case QTN_TLV_ID_NUM_IFACE_COMB:
-			if (unlikely(tlv_value_len != sizeof(*comb)))
+			if (tlv_value_len != sizeof(*comb_num))
 				return -EINVAL;
 
-			comb = (void *)tlv->val;
-			record_count = le16_to_cpu(comb->iface_comb_num);
+			comb_num = (void *)tlv->val;
+
+			/* free earlier iface comb memory */
+			qtnf_mac_iface_comb_free(mac);
 
-			mac->macinfo.n_limits = record_count;
-			/* free earlier iface limits memory */
-			kfree(mac->macinfo.limits);
-			mac->macinfo.limits =
-				kzalloc(sizeof(*mac->macinfo.limits) *
-					record_count, GFP_KERNEL);
+			mac->macinfo.n_if_comb =
+				le32_to_cpu(comb_num->iface_comb_num);
 
-			if (unlikely(!mac->macinfo.limits))
+			mac->macinfo.if_comb =
+				kcalloc(mac->macinfo.n_if_comb,
+					sizeof(*mac->macinfo.if_comb),
+					GFP_KERNEL);
+
+			if (!mac->macinfo.if_comb)
 				return -ENOMEM;
 
-			limits = mac->macinfo.limits;
+			comb = mac->macinfo.if_comb;
+
+			pr_debug("MAC%u: %zu iface combinations\n",
+				 mac->macid, mac->macinfo.n_if_comb);
+
 			break;
 		case QTN_TLV_ID_IFACE_LIMIT:
-			if (unlikely(!limits)) {
-				pr_warn("MAC%u: limits are not inited\n",
+			if (unlikely(!comb)) {
+				pr_warn("MAC%u: no combinations advertised\n",
 					mac->macid);
 				return -EINVAL;
 			}
 
-			if (unlikely(tlv_value_len != sizeof(*limit_record))) {
-				pr_warn("MAC%u: record size mismatch\n",
+			if (n_comb >= mac->macinfo.n_if_comb) {
+				pr_warn("MAC%u: combinations count exceeded\n",
 					mac->macid);
-				return -EINVAL;
+				n_comb++;
+				break;
 			}
 
-			limit_record = (void *)tlv->val;
-			limits[rec].max = le16_to_cpu(limit_record->max_num);
-			limits[rec].types = qlink_iface_type_to_nl_mask(
-				le16_to_cpu(limit_record->type));
+			rec = (void *)tlv->val;
+			rec_len = sizeof(*rec) + rec->n_limits * sizeof(*lim);
+
+			if (unlikely(tlv_value_len != rec_len)) {
+				pr_warn("MAC%u: record %zu size mismatch\n",
+					mac->macid, n_comb);
+				return -EINVAL;
+			}
 
-			/* supported modes: STA, AP */
-			limits[rec].types &= BIT(NL80211_IFTYPE_AP) |
-					     BIT(NL80211_IFTYPE_AP_VLAN) |
-					     BIT(NL80211_IFTYPE_STATION);
+			limits = kzalloc(sizeof(*limits) * rec->n_limits,
+					 GFP_KERNEL);
+			if (!limits)
+				return -ENOMEM;
 
-			pr_debug("MAC%u: MAX: %u; TYPES: %.4X\n", mac->macid,
-				 limits[rec].max, limits[rec].types);
+			comb[n_comb].num_different_channels =
+				rec->num_different_channels;
+			comb[n_comb].max_interfaces =
+				le16_to_cpu(rec->max_interfaces);
+			comb[n_comb].n_limits = rec->n_limits;
+			comb[n_comb].limits = limits;
+
+			for (i = 0; i < rec->n_limits; i++) {
+				lim = &rec->limits[i];
+				limits[i].max = le16_to_cpu(lim->max_num);
+				limits[i].types =
+					qlink_iface_type_to_nl_mask(le16_to_cpu(lim->type));
+				pr_debug("MAC%u: comb[%zu]: MAX:%u TYPES:%.4X\n",
+					 mac->macid, n_comb,
+					 limits[i].max, limits[i].types);
+			}
 
-			if (limits[rec].types)
-				rec++;
+			n_comb++;
+			break;
+		case WLAN_EID_EXT_CAPABILITY:
+			if (unlikely(tlv_value_len > U8_MAX))
+				return -EINVAL;
+			ext_capa = (u8 *)tlv->val;
+			ext_capa_len = tlv_value_len;
+			break;
+		case QTN_TLV_ID_EXT_CAPABILITY_MASK:
+			if (unlikely(tlv_value_len > U8_MAX))
+				return -EINVAL;
+			ext_capa_mask = (u8 *)tlv->val;
+			ext_capa_mask_len = tlv_value_len;
 			break;
 		default:
 			break;
@@ -1106,12 +1243,40 @@ static int qtnf_parse_variable_mac_info(struct qtnf_wmac *mac,
 		return -EINVAL;
 	}
 
-	if (mac->macinfo.n_limits != rec) {
+	if (mac->macinfo.n_if_comb != n_comb) {
 		pr_err("MAC%u: combination mismatch: reported=%zu parsed=%zu\n",
-		       mac->macid, mac->macinfo.n_limits, rec);
+		       mac->macid, mac->macinfo.n_if_comb, n_comb);
+		return -EINVAL;
+	}
+
+	if (ext_capa_len != ext_capa_mask_len) {
+		pr_err("MAC%u: ext_capa/_mask lengths mismatch: %u != %u\n",
+		       mac->macid, ext_capa_len, ext_capa_mask_len);
 		return -EINVAL;
 	}
 
+	if (ext_capa_len > 0) {
+		ext_capa = kmemdup(ext_capa, ext_capa_len, GFP_KERNEL);
+		if (!ext_capa)
+			return -ENOMEM;
+
+		ext_capa_mask =
+			kmemdup(ext_capa_mask, ext_capa_mask_len, GFP_KERNEL);
+		if (!ext_capa_mask) {
+			kfree(ext_capa);
+			return -ENOMEM;
+		}
+	} else {
+		ext_capa = NULL;
+		ext_capa_mask = NULL;
+	}
+
+	kfree(mac->macinfo.extended_capabilities);
+	kfree(mac->macinfo.extended_capabilities_mask);
+	mac->macinfo.extended_capabilities = ext_capa;
+	mac->macinfo.extended_capabilities_mask = ext_capa_mask;
+	mac->macinfo.extended_capabilities_len = ext_capa_len;
+
 	return 0;
 }
 
@@ -1143,6 +1308,7 @@ qtnf_cmd_resp_proc_mac_info(struct qtnf_wmac *mac,
 	mac_info->radar_detect_widths =
 			qlink_chan_width_mask_to_nl(le16_to_cpu(
 					resp_info->radar_detect_widths));
+	mac_info->max_acl_mac_addrs = le32_to_cpu(resp_info->max_acl_mac_addrs);
 
 	memcpy(&mac_info->ht_cap_mod_mask, &resp_info->ht_cap_mod_mask,
 	       sizeof(mac_info->ht_cap_mod_mask));
@@ -1186,7 +1352,7 @@ qtnf_cmd_resp_fill_band_info(struct ieee80211_supported_band *band,
 	size_t tlv_len;
 	size_t tlv_dlen;
 	const struct qlink_tlv_hdr *tlv;
-	const struct qlink_tlv_channel *qchan;
+	const struct qlink_channel *qchan;
 	struct ieee80211_channel *chan;
 	unsigned int chidx = 0;
 	u32 qflags;
@@ -1232,7 +1398,7 @@ qtnf_cmd_resp_fill_band_info(struct ieee80211_supported_band *band,
 
 		switch (tlv_type) {
 		case QTN_TLV_ID_CHANNEL:
-			if (unlikely(tlv_len != sizeof(*qchan))) {
+			if (unlikely(tlv_dlen != sizeof(*qchan))) {
 				pr_err("invalid channel TLV len %zu\n",
 				       tlv_len);
 				goto error_ret;
@@ -1243,7 +1409,7 @@ qtnf_cmd_resp_fill_band_info(struct ieee80211_supported_band *band,
 				goto error_ret;
 			}
 
-			qchan = (const struct qlink_tlv_channel *)tlv;
+			qchan = (const struct qlink_channel *)tlv->val;
 			chan = &band->channels[chidx++];
 			qflags = le32_to_cpu(qchan->flags);
 
@@ -1490,7 +1656,7 @@ int qtnf_cmd_get_mac_info(struct qtnf_wmac *mac)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(mac->macid, QLINK_VIFID_RSVD,
 					    QLINK_CMD_MAC_INFO,
 					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(mac->bus);
@@ -1528,7 +1694,7 @@ int qtnf_cmd_get_hw_info(struct qtnf_bus *bus)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(QLINK_MACID_RSVD, QLINK_VIFID_RSVD,
 					    QLINK_CMD_GET_HW_INFO,
 					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(bus);
@@ -1708,7 +1874,7 @@ int qtnf_cmd_send_init_fw(struct qtnf_bus *bus)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(QLINK_MACID_RSVD, QLINK_VIFID_RSVD,
 					    QLINK_CMD_FW_INIT,
 					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(bus);
@@ -1757,7 +1923,7 @@ int qtnf_cmd_send_add_key(struct qtnf_vif *vif, u8 key_index, bool pairwise,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_ADD_KEY,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -1810,7 +1976,7 @@ int qtnf_cmd_send_del_key(struct qtnf_vif *vif, u8 key_index, bool pairwise,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_DEL_KEY,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -1851,7 +2017,7 @@ int qtnf_cmd_send_set_default_key(struct qtnf_vif *vif, u8 key_index,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_SET_DEFAULT_KEY,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -1886,7 +2052,7 @@ int qtnf_cmd_send_set_default_mgmt_key(struct qtnf_vif *vif, u8 key_index)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_SET_DEFAULT_MGMT_KEY,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -1941,28 +2107,24 @@ int qtnf_cmd_send_change_sta(struct qtnf_vif *vif, const u8 *mac,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_CHANGE_STA,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
 
 	cmd = (struct qlink_cmd_change_sta *)cmd_skb->data;
 	ether_addr_copy(cmd->sta_addr, mac);
+	cmd->flag_update.mask =
+		cpu_to_le32(qtnf_encode_sta_flags(params->sta_flags_mask));
+	cmd->flag_update.value =
+		cpu_to_le32(qtnf_encode_sta_flags(params->sta_flags_set));
 
 	switch (vif->wdev.iftype) {
 	case NL80211_IFTYPE_AP:
 		cmd->if_type = cpu_to_le16(QLINK_IFTYPE_AP);
-		cmd->sta_flags_mask = cpu_to_le32(qtnf_encode_sta_flags(
-						  params->sta_flags_mask));
-		cmd->sta_flags_set = cpu_to_le32(qtnf_encode_sta_flags(
-						 params->sta_flags_set));
 		break;
 	case NL80211_IFTYPE_STATION:
 		cmd->if_type = cpu_to_le16(QLINK_IFTYPE_STATION);
-		cmd->sta_flags_mask = cpu_to_le32(qtnf_encode_sta_flags(
-						  params->sta_flags_mask));
-		cmd->sta_flags_set = cpu_to_le32(qtnf_encode_sta_flags(
-						 params->sta_flags_set));
 		break;
 	default:
 		pr_err("unsupported iftype %d\n", vif->wdev.iftype);
@@ -1997,7 +2159,7 @@ int qtnf_cmd_send_del_sta(struct qtnf_vif *vif,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_DEL_STA,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -2037,8 +2199,8 @@ static void qtnf_cmd_channel_tlv_add(struct sk_buff *cmd_skb,
 	qchan = skb_put_zero(cmd_skb, sizeof(*qchan));
 	qchan->hdr.type = cpu_to_le16(QTN_TLV_ID_CHANNEL);
 	qchan->hdr.len = cpu_to_le16(sizeof(*qchan) - sizeof(qchan->hdr));
-	qchan->center_freq = cpu_to_le16(sc->center_freq);
-	qchan->hw_value = cpu_to_le16(sc->hw_value);
+	qchan->chan.center_freq = cpu_to_le16(sc->center_freq);
+	qchan->chan.hw_value = cpu_to_le16(sc->hw_value);
 
 	if (sc->flags & IEEE80211_CHAN_NO_IR)
 		flags |= QLINK_CHAN_NO_IR;
@@ -2046,7 +2208,7 @@ static void qtnf_cmd_channel_tlv_add(struct sk_buff *cmd_skb,
 	if (sc->flags & IEEE80211_CHAN_RADAR)
 		flags |= QLINK_CHAN_RADAR;
 
-	qchan->flags = cpu_to_le32(flags);
+	qchan->chan.flags = cpu_to_le32(flags);
 }
 
 int qtnf_cmd_send_scan(struct qtnf_wmac *mac)
@@ -2067,7 +2229,7 @@ int qtnf_cmd_send_scan(struct qtnf_wmac *mac)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(mac->macid, QLINK_VIFID_RSVD,
 					    QLINK_CMD_SCAN,
 					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(mac->bus);
@@ -2137,7 +2299,7 @@ int qtnf_cmd_send_connect(struct qtnf_vif *vif,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_CONNECT,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	cmd = (struct qlink_cmd_connect *)cmd_skb->data;
@@ -2239,7 +2401,7 @@ int qtnf_cmd_send_disconnect(struct qtnf_vif *vif, u16 reason_code)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_DISCONNECT,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(vif->mac->bus);
@@ -2273,7 +2435,7 @@ int qtnf_cmd_send_updown_intf(struct qtnf_vif *vif, bool up)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_UPDOWN_INTF,
 					    sizeof(*cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	cmd = (struct qlink_cmd_updown *)cmd_skb->data;
@@ -2434,8 +2596,7 @@ int qtnf_cmd_send_chan_switch(struct qtnf_vif *vif,
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(mac->macid, vif->vifid,
 					    QLINK_CMD_CHAN_SWITCH,
 					    sizeof(*cmd));
-
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(mac->bus);
@@ -2487,7 +2648,7 @@ int qtnf_cmd_get_channel(struct qtnf_vif *vif, struct cfg80211_chan_def *chdef)
 	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
 					    QLINK_CMD_CHAN_GET,
 					    sizeof(struct qlink_cmd));
-	if (unlikely(!cmd_skb))
+	if (!cmd_skb)
 		return -ENOMEM;
 
 	qtnf_bus_lock(bus);
@@ -2512,3 +2673,83 @@ int qtnf_cmd_get_channel(struct qtnf_vif *vif, struct cfg80211_chan_def *chdef)
 	consume_skb(resp_skb);
 	return ret;
 }
+
+int qtnf_cmd_start_cac(const struct qtnf_vif *vif,
+		       const struct cfg80211_chan_def *chdef,
+		       u32 cac_time_ms)
+{
+	struct qtnf_bus *bus = vif->mac->bus;
+	struct sk_buff *cmd_skb;
+	struct qlink_cmd_start_cac *cmd;
+	int ret;
+	u16 res_code;
+
+	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
+					    QLINK_CMD_START_CAC,
+					    sizeof(*cmd));
+	if (!cmd_skb)
+		return -ENOMEM;
+
+	cmd = (struct qlink_cmd_start_cac *)cmd_skb->data;
+	cmd->cac_time_ms = cpu_to_le32(cac_time_ms);
+	qlink_chandef_cfg2q(chdef, &cmd->chan);
+
+	qtnf_bus_lock(bus);
+	ret = qtnf_cmd_send(bus, cmd_skb, &res_code);
+	qtnf_bus_unlock(bus);
+
+	if (ret)
+		return ret;
+
+	switch (res_code) {
+	case QLINK_CMD_RESULT_OK:
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+
+	return ret;
+}
+
+int qtnf_cmd_set_mac_acl(const struct qtnf_vif *vif,
+			 const struct cfg80211_acl_data *params)
+{
+	struct qtnf_bus *bus = vif->mac->bus;
+	struct sk_buff *cmd_skb;
+	struct qlink_tlv_hdr *tlv;
+	size_t acl_size = qtnf_cmd_acl_data_size(params);
+	u16 res_code;
+	int ret;
+
+	cmd_skb = qtnf_cmd_alloc_new_cmdskb(vif->mac->macid, vif->vifid,
+					    QLINK_CMD_SET_MAC_ACL,
+					    sizeof(struct qlink_cmd));
+	if (!cmd_skb)
+		return -ENOMEM;
+
+	tlv = skb_put(cmd_skb, sizeof(*tlv) + acl_size);
+	tlv->type = cpu_to_le16(QTN_TLV_ID_ACL_DATA);
+	tlv->len = cpu_to_le16(acl_size);
+	qlink_acl_data_cfg2q(params, (struct qlink_acl_data *)tlv->val);
+
+	qtnf_bus_lock(bus);
+	ret = qtnf_cmd_send(bus, cmd_skb, &res_code);
+	qtnf_bus_unlock(bus);
+
+	if (unlikely(ret))
+		return ret;
+
+	switch (res_code) {
+	case QLINK_CMD_RESULT_OK:
+		break;
+	case QLINK_CMD_RESULT_INVALID:
+		ret = -EINVAL;
+		break;
+	default:
+		ret = -EOPNOTSUPP;
+		break;
+	}
+
+	return ret;
+}
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.h b/drivers/net/wireless/quantenna/qtnfmac/commands.h
index d981a76e5835..69a7d56f7e58 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.h
@@ -76,5 +76,10 @@ int qtnf_cmd_get_chan_stats(struct qtnf_wmac *mac, u16 channel,
 int qtnf_cmd_send_chan_switch(struct qtnf_vif *vif,
 			      struct cfg80211_csa_settings *params);
 int qtnf_cmd_get_channel(struct qtnf_vif *vif, struct cfg80211_chan_def *chdef);
+int qtnf_cmd_start_cac(const struct qtnf_vif *vif,
+		       const struct cfg80211_chan_def *chdef,
+		       u32 cac_time_ms);
+int qtnf_cmd_set_mac_acl(const struct qtnf_vif *vif,
+			 const struct cfg80211_acl_data *params);
 
 #endif /* QLINK_COMMANDS_H_ */
diff --git a/drivers/net/wireless/quantenna/qtnfmac/core.c b/drivers/net/wireless/quantenna/qtnfmac/core.c
index 3423dc51198b..cf26c15a84f8 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/core.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/core.c
@@ -119,9 +119,38 @@ qtnf_netdev_hard_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 
 /* Netdev handler for getting stats.
  */
-static struct net_device_stats *qtnf_netdev_get_stats(struct net_device *dev)
+static void qtnf_netdev_get_stats64(struct net_device *ndev,
+				    struct rtnl_link_stats64 *stats)
 {
-	return &dev->stats;
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(ndev);
+	unsigned int start;
+	int cpu;
+
+	netdev_stats_to_stats64(stats, &ndev->stats);
+
+	if (!vif->stats64)
+		return;
+
+	for_each_possible_cpu(cpu) {
+		struct pcpu_sw_netstats *stats64;
+		u64 rx_packets, rx_bytes;
+		u64 tx_packets, tx_bytes;
+
+		stats64 = per_cpu_ptr(vif->stats64, cpu);
+
+		do {
+			start = u64_stats_fetch_begin_irq(&stats64->syncp);
+			rx_packets = stats64->rx_packets;
+			rx_bytes = stats64->rx_bytes;
+			tx_packets = stats64->tx_packets;
+			tx_bytes = stats64->tx_bytes;
+		} while (u64_stats_fetch_retry_irq(&stats64->syncp, start));
+
+		stats->rx_packets += rx_packets;
+		stats->rx_bytes += rx_bytes;
+		stats->tx_packets += tx_packets;
+		stats->tx_bytes += tx_bytes;
+	}
 }
 
 /* Netdev handler for transmission timeout.
@@ -156,7 +185,7 @@ const struct net_device_ops qtnf_netdev_ops = {
 	.ndo_stop = qtnf_netdev_close,
 	.ndo_start_xmit = qtnf_netdev_hard_start_xmit,
 	.ndo_tx_timeout = qtnf_netdev_tx_timeout,
-	.ndo_get_stats = qtnf_netdev_get_stats,
+	.ndo_get_stats64 = qtnf_netdev_get_stats64,
 };
 
 static int qtnf_mac_init_single_band(struct wiphy *wiphy,
@@ -233,6 +262,23 @@ struct qtnf_vif *qtnf_mac_get_base_vif(struct qtnf_wmac *mac)
 	return vif;
 }
 
+void qtnf_mac_iface_comb_free(struct qtnf_wmac *mac)
+{
+	struct ieee80211_iface_combination *comb;
+	int i;
+
+	if (mac->macinfo.if_comb) {
+		for (i = 0; i < mac->macinfo.n_if_comb; i++) {
+			comb = &mac->macinfo.if_comb[i];
+			kfree(comb->limits);
+			comb->limits = NULL;
+		}
+
+		kfree(mac->macinfo.if_comb);
+		mac->macinfo.if_comb = NULL;
+	}
+}
+
 static void qtnf_vif_reset_handler(struct work_struct *work)
 {
 	struct qtnf_vif *vif = container_of(work, struct qtnf_vif, reset_work);
@@ -258,13 +304,44 @@ static void qtnf_mac_init_primary_intf(struct qtnf_wmac *mac)
 {
 	struct qtnf_vif *vif = &mac->iflist[QTNF_PRIMARY_VIF_IDX];
 
-	vif->wdev.iftype = NL80211_IFTYPE_AP;
+	vif->wdev.iftype = NL80211_IFTYPE_STATION;
 	vif->bss_priority = QTNF_DEF_BSS_PRIORITY;
 	vif->wdev.wiphy = priv_to_wiphy(mac);
 	INIT_WORK(&vif->reset_work, qtnf_vif_reset_handler);
 	vif->cons_tx_timeout_cnt = 0;
 }
 
+static void qtnf_mac_scan_finish(struct qtnf_wmac *mac, bool aborted)
+{
+	struct cfg80211_scan_info info = {
+		.aborted = aborted,
+	};
+
+	mutex_lock(&mac->mac_lock);
+
+	if (mac->scan_req) {
+		cfg80211_scan_done(mac->scan_req, &info);
+		mac->scan_req = NULL;
+	}
+
+	mutex_unlock(&mac->mac_lock);
+}
+
+void qtnf_scan_done(struct qtnf_wmac *mac, bool aborted)
+{
+	cancel_delayed_work_sync(&mac->scan_timeout);
+	qtnf_mac_scan_finish(mac, aborted);
+}
+
+static void qtnf_mac_scan_timeout(struct work_struct *work)
+{
+	struct qtnf_wmac *mac =
+		container_of(work, struct qtnf_wmac, scan_timeout.work);
+
+	pr_warn("MAC%d: scan timed out\n", mac->macid);
+	qtnf_mac_scan_finish(mac, true);
+}
+
 static struct qtnf_wmac *qtnf_core_mac_alloc(struct qtnf_bus *bus,
 					     unsigned int macid)
 {
@@ -288,7 +365,12 @@ static struct qtnf_wmac *qtnf_core_mac_alloc(struct qtnf_bus *bus,
 		mac->iflist[i].vifid = i;
 		qtnf_sta_list_init(&mac->iflist[i].sta_list);
 		mutex_init(&mac->mac_lock);
-		timer_setup(&mac->scan_timeout, NULL, 0);
+		INIT_DELAYED_WORK(&mac->scan_timeout, qtnf_mac_scan_timeout);
+		mac->iflist[i].stats64 =
+			netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
+		if (!mac->iflist[i].stats64)
+			pr_warn("VIF%u.%u: per cpu stats allocation failed\n",
+				macid, i);
 	}
 
 	qtnf_mac_init_primary_intf(mac);
@@ -297,9 +379,12 @@ static struct qtnf_wmac *qtnf_core_mac_alloc(struct qtnf_bus *bus,
 	return mac;
 }
 
+static const struct ethtool_ops qtnf_ethtool_ops = {
+	.get_drvinfo = cfg80211_get_drvinfo,
+};
+
 int qtnf_core_net_attach(struct qtnf_wmac *mac, struct qtnf_vif *vif,
-			 const char *name, unsigned char name_assign_type,
-			 enum nl80211_iftype iftype)
+			 const char *name, unsigned char name_assign_type)
 {
 	struct wiphy *wiphy = priv_to_wiphy(mac);
 	struct net_device *dev;
@@ -320,12 +405,12 @@ int qtnf_core_net_attach(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 	dev->needs_free_netdev = true;
 	dev_net_set(dev, wiphy_net(wiphy));
 	dev->ieee80211_ptr = &vif->wdev;
-	dev->ieee80211_ptr->iftype = iftype;
 	ether_addr_copy(dev->dev_addr, vif->mac_addr);
 	SET_NETDEV_DEV(dev, wiphy_dev(wiphy));
 	dev->flags |= IFF_BROADCAST | IFF_MULTICAST;
 	dev->watchdog_timeo = QTNF_DEF_WDOG_TIMEOUT;
 	dev->tx_queue_len = 100;
+	dev->ethtool_ops = &qtnf_ethtool_ops;
 
 	qdev_vif = netdev_priv(dev);
 	*((void **)qdev_vif) = vif;
@@ -366,6 +451,7 @@ static void qtnf_core_mac_detach(struct qtnf_bus *bus, unsigned int macid)
 		}
 		rtnl_unlock();
 		qtnf_sta_list_free(&vif->sta_list);
+		free_percpu(vif->stats64);
 	}
 
 	if (mac->wiphy_registered)
@@ -382,8 +468,9 @@ static void qtnf_core_mac_detach(struct qtnf_bus *bus, unsigned int macid)
 		wiphy->bands[band] = NULL;
 	}
 
-	kfree(mac->macinfo.limits);
-	kfree(wiphy->iface_combinations);
+	qtnf_mac_iface_comb_free(mac);
+	kfree(mac->macinfo.extended_capabilities);
+	kfree(mac->macinfo.extended_capabilities_mask);
 	wiphy_free(wiphy);
 	bus->mac[macid] = NULL;
 }
@@ -418,7 +505,7 @@ static int qtnf_core_mac_attach(struct qtnf_bus *bus, unsigned int macid)
 		goto error;
 	}
 
-	ret = qtnf_cmd_send_add_intf(vif, NL80211_IFTYPE_AP, vif->mac_addr);
+	ret = qtnf_cmd_send_add_intf(vif, vif->wdev.iftype, vif->mac_addr);
 	if (ret) {
 		pr_err("MAC%u: failed to add VIF\n", macid);
 		goto error;
@@ -446,8 +533,7 @@ static int qtnf_core_mac_attach(struct qtnf_bus *bus, unsigned int macid)
 
 	rtnl_lock();
 
-	ret = qtnf_core_net_attach(mac, vif, "wlan%d", NET_NAME_ENUM,
-				   NL80211_IFTYPE_AP);
+	ret = qtnf_core_net_attach(mac, vif, "wlan%d", NET_NAME_ENUM);
 	rtnl_unlock();
 
 	if (ret) {
@@ -644,6 +730,46 @@ void qtnf_wake_all_queues(struct net_device *ndev)
 }
 EXPORT_SYMBOL_GPL(qtnf_wake_all_queues);
 
+void qtnf_update_rx_stats(struct net_device *ndev, const struct sk_buff *skb)
+{
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(ndev);
+	struct pcpu_sw_netstats *stats64;
+
+	if (unlikely(!vif || !vif->stats64)) {
+		ndev->stats.rx_packets++;
+		ndev->stats.rx_bytes += skb->len;
+		return;
+	}
+
+	stats64 = this_cpu_ptr(vif->stats64);
+
+	u64_stats_update_begin(&stats64->syncp);
+	stats64->rx_packets++;
+	stats64->rx_bytes += skb->len;
+	u64_stats_update_end(&stats64->syncp);
+}
+EXPORT_SYMBOL_GPL(qtnf_update_rx_stats);
+
+void qtnf_update_tx_stats(struct net_device *ndev, const struct sk_buff *skb)
+{
+	struct qtnf_vif *vif = qtnf_netdev_get_priv(ndev);
+	struct pcpu_sw_netstats *stats64;
+
+	if (unlikely(!vif || !vif->stats64)) {
+		ndev->stats.tx_packets++;
+		ndev->stats.tx_bytes += skb->len;
+		return;
+	}
+
+	stats64 = this_cpu_ptr(vif->stats64);
+
+	u64_stats_update_begin(&stats64->syncp);
+	stats64->tx_packets++;
+	stats64->tx_bytes += skb->len;
+	u64_stats_update_end(&stats64->syncp);
+}
+EXPORT_SYMBOL_GPL(qtnf_update_tx_stats);
+
 MODULE_AUTHOR("Quantenna Communications");
 MODULE_DESCRIPTION("Quantenna 802.11 wireless LAN FullMAC driver.");
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/wireless/quantenna/qtnfmac/core.h b/drivers/net/wireless/quantenna/qtnfmac/core.h
index 1b7bc0318f3e..3b884c80b6ab 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/core.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/core.h
@@ -88,6 +88,9 @@ struct qtnf_vif {
 	struct work_struct reset_work;
 	struct qtnf_sta_list sta_list;
 	unsigned long cons_tx_timeout_cnt;
+	int generation;
+
+	struct pcpu_sw_netstats __percpu *stats64;
 };
 
 struct qtnf_mac_info {
@@ -102,10 +105,14 @@ struct qtnf_mac_info {
 	u8 sretry_limit;
 	u8 coverage_class;
 	u8 radar_detect_widths;
+	u32 max_acl_mac_addrs;
 	struct ieee80211_ht_cap ht_cap_mod_mask;
 	struct ieee80211_vht_cap vht_cap_mod_mask;
-	struct ieee80211_iface_limit *limits;
-	size_t n_limits;
+	struct ieee80211_iface_combination *if_comb;
+	size_t n_if_comb;
+	u8 *extended_capabilities;
+	u8 *extended_capabilities_mask;
+	u8 extended_capabilities_len;
 };
 
 struct qtnf_chan_stats {
@@ -126,7 +133,7 @@ struct qtnf_wmac {
 	struct qtnf_vif iflist[QTNF_MAX_INTF];
 	struct cfg80211_scan_request *scan_req;
 	struct mutex mac_lock;	/* lock during wmac speicific ops */
-	struct timer_list scan_timeout;
+	struct delayed_work scan_timeout;
 };
 
 struct qtnf_hw_info {
@@ -138,14 +145,16 @@ struct qtnf_hw_info {
 	struct ieee80211_regdomain *rd;
 	u8 total_tx_chain;
 	u8 total_rx_chain;
+	char fw_version[ETHTOOL_FWVERS_LEN];
+	u32 hw_version;
 };
 
 struct qtnf_vif *qtnf_mac_get_free_vif(struct qtnf_wmac *mac);
 struct qtnf_vif *qtnf_mac_get_base_vif(struct qtnf_wmac *mac);
+void qtnf_mac_iface_comb_free(struct qtnf_wmac *mac);
 struct wiphy *qtnf_wiphy_allocate(struct qtnf_bus *bus);
 int qtnf_core_net_attach(struct qtnf_wmac *mac, struct qtnf_vif *priv,
-			 const char *name, unsigned char name_assign_type,
-			 enum nl80211_iftype iftype);
+			 const char *name, unsigned char name_assign_type);
 void qtnf_main_work_queue(struct work_struct *work);
 int qtnf_cmd_send_update_phy_params(struct qtnf_wmac *mac, u32 changed);
 int qtnf_cmd_send_get_phy_params(struct qtnf_wmac *mac);
@@ -153,9 +162,13 @@ int qtnf_cmd_send_get_phy_params(struct qtnf_wmac *mac);
 struct qtnf_wmac *qtnf_core_get_mac(const struct qtnf_bus *bus, u8 macid);
 struct net_device *qtnf_classify_skb(struct qtnf_bus *bus, struct sk_buff *skb);
 void qtnf_wake_all_queues(struct net_device *ndev);
+void qtnf_update_rx_stats(struct net_device *ndev, const struct sk_buff *skb);
+void qtnf_update_tx_stats(struct net_device *ndev, const struct sk_buff *skb);
+
 void qtnf_virtual_intf_cleanup(struct net_device *ndev);
 
 void qtnf_netdev_updown(struct net_device *ndev, bool up);
+void qtnf_scan_done(struct qtnf_wmac *mac, bool aborted);
 
 static inline struct qtnf_vif *qtnf_netdev_get_priv(struct net_device *dev)
 {
diff --git a/drivers/net/wireless/quantenna/qtnfmac/event.c b/drivers/net/wireless/quantenna/qtnfmac/event.c
index 4abc6d9ed560..bcd415f96412 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/event.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/event.c
@@ -59,10 +59,11 @@ qtnf_event_handle_sta_assoc(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 	pr_debug("VIF%u.%u: MAC:%pM FC:%x\n", mac->macid, vif->vifid, sta_addr,
 		 frame_control);
 
-	qtnf_sta_list_add(&vif->sta_list, sta_addr);
+	qtnf_sta_list_add(vif, sta_addr);
 
 	sinfo.assoc_req_ies = NULL;
 	sinfo.assoc_req_ies_len = 0;
+	sinfo.generation = vif->generation;
 
 	payload_len = len - sizeof(*sta_assoc);
 	tlv = (const struct qlink_tlv_hdr *)sta_assoc->ies;
@@ -132,7 +133,7 @@ qtnf_event_handle_sta_deauth(struct qtnf_wmac *mac, struct qtnf_vif *vif,
 	pr_debug("VIF%u.%u: MAC:%pM reason:%x\n", mac->macid, vif->vifid,
 		 sta_addr, reason);
 
-	if (qtnf_sta_list_del(&vif->sta_list, sta_addr))
+	if (qtnf_sta_list_del(vif, sta_addr))
 		cfg80211_del_sta(vif->netdev, sta_deauth->sta_addr,
 				 GFP_KERNEL);
 
@@ -237,9 +238,8 @@ qtnf_event_handle_mgmt_received(struct qtnf_vif *vif,
 	pr_debug("%s LEN:%u FC:%.4X SA:%pM\n", vif->netdev->name, frame_len,
 		 le16_to_cpu(frame->frame_control), frame->addr2);
 
-	cfg80211_rx_mgmt(&vif->wdev, le32_to_cpu(rxmgmt->freq),
-			 le32_to_cpu(rxmgmt->sig_dbm), rxmgmt->frame_data,
-			 frame_len, flags);
+	cfg80211_rx_mgmt(&vif->wdev, le32_to_cpu(rxmgmt->freq), rxmgmt->sig_dbm,
+			 rxmgmt->frame_data, frame_len, flags);
 
 	return 0;
 }
@@ -324,7 +324,7 @@ qtnf_event_handle_scan_results(struct qtnf_vif *vif,
 				  sr->bssid, get_unaligned_le64(&sr->tsf),
 				  le16_to_cpu(sr->capab),
 				  le16_to_cpu(sr->bintval), ies, ies_len,
-				  sr->signal, GFP_KERNEL);
+				  DBM_TO_MBM(sr->sig_dbm), GFP_KERNEL);
 	if (!bss)
 		return -ENOMEM;
 
@@ -369,7 +369,8 @@ qtnf_event_handle_freq_change(struct qtnf_wmac *mac,
 	qlink_chandef_q2cfg(wiphy, &data->chan, &chandef);
 
 	if (!cfg80211_chandef_valid(&chandef)) {
-		pr_err("MAC%u: bad channel f1=%u f2=%u bw=%u\n", mac->macid,
+		pr_err("MAC%u: bad channel freq=%u cf1=%u cf2=%u bw=%u\n",
+		       mac->macid, chandef.chan->center_freq,
 		       chandef.center_freq1, chandef.center_freq2,
 		       chandef.width);
 		return -EINVAL;
@@ -394,6 +395,63 @@ qtnf_event_handle_freq_change(struct qtnf_wmac *mac,
 	return 0;
 }
 
+static int qtnf_event_handle_radar(struct qtnf_vif *vif,
+				   const struct qlink_event_radar *ev,
+				   u16 len)
+{
+	struct wiphy *wiphy = priv_to_wiphy(vif->mac);
+	struct cfg80211_chan_def chandef;
+
+	if (len < sizeof(*ev)) {
+		pr_err("MAC%u: payload is too short\n", vif->mac->macid);
+		return -EINVAL;
+	}
+
+	if (!wiphy->registered || !vif->netdev)
+		return 0;
+
+	qlink_chandef_q2cfg(wiphy, &ev->chan, &chandef);
+
+	if (!cfg80211_chandef_valid(&chandef)) {
+		pr_err("MAC%u: bad channel f1=%u f2=%u bw=%u\n",
+		       vif->mac->macid,
+		       chandef.center_freq1, chandef.center_freq2,
+		       chandef.width);
+		return -EINVAL;
+	}
+
+	pr_info("%s: radar event=%u f1=%u f2=%u bw=%u\n",
+		vif->netdev->name, ev->event,
+		chandef.center_freq1, chandef.center_freq2,
+		chandef.width);
+
+	switch (ev->event) {
+	case QLINK_RADAR_DETECTED:
+		cfg80211_radar_event(wiphy, &chandef, GFP_KERNEL);
+		break;
+	case QLINK_RADAR_CAC_FINISHED:
+		if (!vif->wdev.cac_started)
+			break;
+
+		cfg80211_cac_event(vif->netdev, &chandef,
+				   NL80211_RADAR_CAC_FINISHED, GFP_KERNEL);
+		break;
+	case QLINK_RADAR_CAC_ABORTED:
+		if (!vif->wdev.cac_started)
+			break;
+
+		cfg80211_cac_event(vif->netdev, &chandef,
+				   NL80211_RADAR_CAC_ABORTED, GFP_KERNEL);
+		break;
+	default:
+		pr_warn("%s: unhandled radar event %u\n",
+			vif->netdev->name, ev->event);
+		break;
+	}
+
+	return 0;
+}
+
 static int qtnf_event_parse(struct qtnf_wmac *mac,
 			    const struct sk_buff *event_skb)
 {
@@ -448,6 +506,10 @@ static int qtnf_event_parse(struct qtnf_wmac *mac,
 		ret = qtnf_event_handle_freq_change(mac, (const void *)event,
 						    event_len);
 		break;
+	case QLINK_EVENT_RADAR:
+		ret = qtnf_event_handle_radar(vif, (const void *)event,
+					      event_len);
+		break;
 	default:
 		pr_warn("unknown event type: %x\n", event_id);
 		break;
@@ -479,9 +541,9 @@ static int qtnf_event_process_skb(struct qtnf_bus *bus,
 	if (unlikely(!mac))
 		return -ENXIO;
 
-	qtnf_bus_lock(bus);
+	rtnl_lock();
 	res = qtnf_event_parse(mac, skb);
-	qtnf_bus_unlock(bus);
+	rtnl_unlock();
 
 	return res;
 }
diff --git a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
index 7e487622d87d..6f6190964320 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/pearl/pcie.c
@@ -615,8 +615,7 @@ static void qtnf_pcie_data_tx_reclaim(struct qtnf_pcie_bus_priv *priv)
 					 PCI_DMA_TODEVICE);
 
 			if (skb->dev) {
-				skb->dev->stats.tx_packets++;
-				skb->dev->stats.tx_bytes += skb->len;
+				qtnf_update_tx_stats(skb->dev, skb);
 				if (unlikely(priv->tx_stopped)) {
 					qtnf_wake_all_queues(skb->dev);
 					priv->tx_stopped = 0;
@@ -855,9 +854,7 @@ static int qtnf_rx_poll(struct napi_struct *napi, int budget)
 			skb_put(skb, psize);
 			ndev = qtnf_classify_skb(bus, skb);
 			if (likely(ndev)) {
-				ndev->stats.rx_packets++;
-				ndev->stats.rx_bytes += skb->len;
-
+				qtnf_update_rx_stats(ndev, skb);
 				skb->protocol = eth_type_trans(skb, ndev);
 				napi_gro_receive(napi, skb);
 			} else {
diff --git a/drivers/net/wireless/quantenna/qtnfmac/qlink.h b/drivers/net/wireless/quantenna/qtnfmac/qlink.h
index a432fb001c41..9bf3ae4d1b3b 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/qlink.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/qlink.h
@@ -19,7 +19,7 @@
 
 #include <linux/ieee80211.h>
 
-#define QLINK_PROTO_VER		6
+#define QLINK_PROTO_VER		11
 
 #define QLINK_MACID_RSVD		0xFF
 #define QLINK_VIFID_RSVD		0xFF
@@ -121,18 +121,50 @@ enum qlink_channel_width {
 	QLINK_CHAN_WIDTH_160,
 };
 
+/**
+ * struct qlink_channel - qlink control channel definition
+ *
+ * @hw_value: hardware-specific value for the channel
+ * @center_freq: center frequency in MHz
+ * @flags: channel flags from &enum qlink_channel_flags
+ * @band: band this channel belongs to
+ * @max_antenna_gain: maximum antenna gain in dBi
+ * @max_power: maximum transmission power (in dBm)
+ * @max_reg_power: maximum regulatory transmission power (in dBm)
+ * @dfs_state: current state of this channel.
+ *      Only relevant if radar is required on this channel.
+ * @beacon_found: helper to regulatory code to indicate when a beacon
+ *	has been found on this channel. Use regulatory_hint_found_beacon()
+ *	to enable this, this is useful only on 5 GHz band.
+ */
+struct qlink_channel {
+	__le16 hw_value;
+	__le16 center_freq;
+	__le32 flags;
+	u8 band;
+	u8 max_antenna_gain;
+	u8 max_power;
+	u8 max_reg_power;
+	__le32 dfs_cac_ms;
+	u8 dfs_state;
+	u8 beacon_found;
+	u8 rsvd[2];
+} __packed;
+
 /**
  * struct qlink_chandef - qlink channel definition
  *
+ * @chan: primary channel definition
  * @center_freq1: center frequency of first segment
  * @center_freq2: center frequency of second segment (80+80 only)
  * @width: channel width, one of @enum qlink_channel_width
  */
 struct qlink_chandef {
+	struct qlink_channel chan;
 	__le16 center_freq1;
 	__le16 center_freq2;
 	u8 width;
-	u8 rsvd[3];
+	u8 rsvd;
 } __packed;
 
 #define QLINK_MAX_NR_CIPHER_SUITES            5
@@ -153,6 +185,17 @@ struct qlink_auth_encr {
 	u8 rsvd[2];
 } __packed;
 
+/**
+ * struct qlink_sta_info_state - station flags mask/value
+ *
+ * @mask: STA flags mask, bitmap of &enum qlink_sta_flags
+ * @value: STA flags values, bitmap of &enum qlink_sta_flags
+ */
+struct qlink_sta_info_state {
+	__le32 mask;
+	__le32 value;
+} __packed;
+
 /* QLINK Command messages related definitions
  */
 
@@ -173,6 +216,7 @@ struct qlink_auth_encr {
  * @QLINK_CMD_REG_NOTIFY: notify device about regulatory domain change. This
  *	command is supported only if device reports QLINK_HW_SUPPORTS_REG_UPDATE
  *	capability.
+ * @QLINK_CMD_START_CAC: start radar detection procedure on a specified channel.
  */
 enum qlink_cmd_type {
 	QLINK_CMD_FW_INIT		= 0x0001,
@@ -192,8 +236,10 @@ enum qlink_cmd_type {
 	QLINK_CMD_BAND_INFO_GET		= 0x001A,
 	QLINK_CMD_CHAN_SWITCH		= 0x001B,
 	QLINK_CMD_CHAN_GET		= 0x001C,
+	QLINK_CMD_START_CAC		= 0x001D,
 	QLINK_CMD_START_AP		= 0x0021,
 	QLINK_CMD_STOP_AP		= 0x0022,
+	QLINK_CMD_SET_MAC_ACL		= 0x0023,
 	QLINK_CMD_GET_STA_INFO		= 0x0030,
 	QLINK_CMD_ADD_KEY		= 0x0040,
 	QLINK_CMD_DEL_KEY		= 0x0041,
@@ -368,16 +414,14 @@ struct qlink_cmd_set_def_mgmt_key {
 /**
  * struct qlink_cmd_change_sta - data for QLINK_CMD_CHANGE_STA command
  *
- * @sta_flags_mask: STA flags mask, bitmap of &enum qlink_sta_flags
- * @sta_flags_set: STA flags values, bitmap of &enum qlink_sta_flags
+ * @flag_update: STA flags to update
  * @if_type: Mode of interface operation, one of &enum qlink_iface_type
  * @vlanid: VLAN ID to assign to specific STA
  * @sta_addr: address of the STA for which parameters are set.
  */
 struct qlink_cmd_change_sta {
 	struct qlink_cmd chdr;
-	__le32 sta_flags_mask;
-	__le32 sta_flags_set;
+	struct qlink_sta_info_state flag_update;
 	__le16 if_type;
 	__le16 vlanid;
 	u8 sta_addr[ETH_ALEN];
@@ -585,6 +629,40 @@ struct qlink_cmd_start_ap {
 	u8 info[0];
 } __packed;
 
+/**
+ * struct qlink_cmd_start_cac - data for QLINK_CMD_START_CAC command
+ *
+ * @chan: a channel to start a radar detection procedure on.
+ * @cac_time_ms: CAC time.
+ */
+struct qlink_cmd_start_cac {
+	struct qlink_cmd chdr;
+	struct qlink_chandef chan;
+	__le32 cac_time_ms;
+} __packed;
+
+enum qlink_acl_policy {
+	QLINK_ACL_POLICY_ACCEPT_UNLESS_LISTED,
+	QLINK_ACL_POLICY_DENY_UNLESS_LISTED,
+};
+
+struct qlink_mac_address {
+	u8 addr[ETH_ALEN];
+} __packed;
+
+/**
+ * struct qlink_acl_data - ACL data
+ *
+ * @policy: filter policy, one of &enum qlink_acl_policy.
+ * @num_entries: number of MAC addresses in array.
+ * @mac_address: MAC addresses array.
+ */
+struct qlink_acl_data {
+	__le32 policy;
+	__le32 num_entries;
+	struct qlink_mac_address mac_addrs[0];
+} __packed;
+
 /* QLINK Command Responses messages related definitions
  */
 
@@ -646,6 +724,7 @@ struct qlink_resp_get_mac_info {
 	struct ieee80211_ht_cap ht_cap_mod_mask;
 	__le16 max_ap_assoc_sta;
 	__le16 radar_detect_widths;
+	__le32 max_acl_mac_addrs;
 	u8 bands_cap;
 	u8 rsvd[1];
 	u8 var_info[0];
@@ -685,6 +764,9 @@ struct qlink_resp_get_hw_info {
 	struct qlink_resp rhdr;
 	__le32 fw_ver;
 	__le32 hw_capab;
+	__le32 bld_tmstamp;
+	__le32 plat_id;
+	__le32 hw_ver;
 	__le16 ql_proto_ver;
 	u8 num_mac;
 	u8 mac_bitmap;
@@ -709,17 +791,27 @@ struct qlink_resp_manage_intf {
 	struct qlink_intf_info intf_info;
 } __packed;
 
+enum qlink_sta_info_rate_flags {
+	QLINK_STA_INFO_RATE_FLAG_HT_MCS		= BIT(0),
+	QLINK_STA_INFO_RATE_FLAG_VHT_MCS	= BIT(1),
+	QLINK_STA_INFO_RATE_FLAG_SHORT_GI	= BIT(2),
+	QLINK_STA_INFO_RATE_FLAG_60G		= BIT(3),
+};
+
 /**
  * struct qlink_resp_get_sta_info - response for QLINK_CMD_GET_STA_INFO command
  *
  * Response data containing statistics for specified STA.
  *
+ * @filled: a bitmask of &enum qlink_sta_info, specifies which info in response
+ *	is valid.
  * @sta_addr: MAC address of STA the response carries statistic for.
- * @info: statistics for specified STA.
+ * @info: variable statistics for specified STA.
  */
 struct qlink_resp_get_sta_info {
 	struct qlink_resp rhdr;
 	u8 sta_addr[ETH_ALEN];
+	u8 rsvd[2];
 	u8 info[0];
 } __packed;
 
@@ -782,6 +874,7 @@ enum qlink_event_type {
 	QLINK_EVENT_BSS_JOIN		= 0x0026,
 	QLINK_EVENT_BSS_LEAVE		= 0x0027,
 	QLINK_EVENT_FREQ_CHANGE		= 0x0028,
+	QLINK_EVENT_RADAR		= 0x0029,
 };
 
 /**
@@ -869,15 +962,16 @@ enum qlink_rxmgmt_flags {
  * struct qlink_event_rxmgmt - data for QLINK_EVENT_MGMT_RECEIVED event
  *
  * @freq: Frequency on which the frame was received in MHz.
- * @sig_dbm: signal strength in dBm.
  * @flags: bitmap of &enum qlink_rxmgmt_flags.
+ * @sig_dbm: signal strength in dBm.
  * @frame_data: data of Rx'd frame itself.
  */
 struct qlink_event_rxmgmt {
 	struct qlink_event ehdr;
 	__le32 freq;
-	__le32 sig_dbm;
 	__le32 flags;
+	s8 sig_dbm;
+	u8 rsvd[3];
 	u8 frame_data[0];
 } __packed;
 
@@ -889,7 +983,7 @@ struct qlink_event_rxmgmt {
  *	event was generated was discovered.
  * @capab: capabilities field.
  * @bintval: beacon interval announced by discovered BSS.
- * @signal: signal strength.
+ * @sig_dbm: signal strength in dBm.
  * @bssid: BSSID announced by discovered BSS.
  * @ssid_len: length of SSID announced by BSS.
  * @ssid: SSID announced by discovered BSS.
@@ -901,7 +995,7 @@ struct qlink_event_scan_result {
 	__le16 freq;
 	__le16 capab;
 	__le16 bintval;
-	s8 signal;
+	s8 sig_dbm;
 	u8 ssid_len;
 	u8 ssid[IEEE80211_MAX_SSID_LEN];
 	u8 bssid[ETH_ALEN];
@@ -931,9 +1025,39 @@ struct qlink_event_scan_complete {
 	__le32 flags;
 } __packed;
 
+enum qlink_radar_event {
+	QLINK_RADAR_DETECTED,
+	QLINK_RADAR_CAC_FINISHED,
+	QLINK_RADAR_CAC_ABORTED,
+	QLINK_RADAR_NOP_FINISHED,
+	QLINK_RADAR_PRE_CAC_EXPIRED,
+};
+
+/**
+ * struct qlink_event_radar - data for QLINK_EVENT_RADAR event
+ *
+ * @chan: channel on which radar event happened.
+ * @event: radar event type, one of &enum qlink_radar_event.
+ */
+struct qlink_event_radar {
+	struct qlink_event ehdr;
+	struct qlink_chandef chan;
+	u8 event;
+	u8 rsvd[3];
+} __packed;
+
 /* QLINK TLVs (Type-Length Values) definitions
  */
 
+/**
+ * enum qlink_tlv_id - list of TLVs that Qlink messages can carry
+ *
+ * @QTN_TLV_ID_STA_STATS_MAP: a bitmap of &enum qlink_sta_info, used to
+ *	indicate which statistic carried in QTN_TLV_ID_STA_STATS is valid.
+ * @QTN_TLV_ID_STA_STATS: per-STA statistics as defined by
+ *	&struct qlink_sta_stats. Valid values are marked as such in a bitmap
+ *	carried by QTN_TLV_ID_STA_STATS_MAP.
+ */
 enum qlink_tlv_id {
 	QTN_TLV_ID_FRAG_THRESH		= 0x0201,
 	QTN_TLV_ID_RTS_THRESH		= 0x0202,
@@ -942,15 +1066,24 @@ enum qlink_tlv_id {
 	QTN_TLV_ID_REG_RULE		= 0x0207,
 	QTN_TLV_ID_CHANNEL		= 0x020F,
 	QTN_TLV_ID_CHANDEF		= 0x0210,
+	QTN_TLV_ID_STA_STATS_MAP	= 0x0211,
+	QTN_TLV_ID_STA_STATS		= 0x0212,
 	QTN_TLV_ID_COVERAGE_CLASS	= 0x0213,
 	QTN_TLV_ID_IFACE_LIMIT		= 0x0214,
 	QTN_TLV_ID_NUM_IFACE_COMB	= 0x0215,
 	QTN_TLV_ID_CHANNEL_STATS	= 0x0216,
-	QTN_TLV_ID_STA_BASIC_COUNTERS	= 0x0300,
-	QTN_TLV_ID_STA_GENERIC_INFO	= 0x0301,
 	QTN_TLV_ID_KEY			= 0x0302,
 	QTN_TLV_ID_SEQ			= 0x0303,
 	QTN_TLV_ID_IE_SET		= 0x0305,
+	QTN_TLV_ID_EXT_CAPABILITY_MASK	= 0x0306,
+	QTN_TLV_ID_ACL_DATA		= 0x0307,
+	QTN_TLV_ID_BUILD_NAME		= 0x0401,
+	QTN_TLV_ID_BUILD_REV		= 0x0402,
+	QTN_TLV_ID_BUILD_TYPE		= 0x0403,
+	QTN_TLV_ID_BUILD_LABEL		= 0x0404,
+	QTN_TLV_ID_HW_ID		= 0x0405,
+	QTN_TLV_ID_CALIBRATION_VER	= 0x0406,
+	QTN_TLV_ID_UBOOT_VER		= 0x0407,
 };
 
 struct qlink_tlv_hdr {
@@ -959,76 +1092,24 @@ struct qlink_tlv_hdr {
 	u8 val[0];
 } __packed;
 
-struct qlink_iface_limit {
-	__le16 max_num;
-	__le16 type;
-} __packed;
-
 struct qlink_iface_comb_num {
-	__le16 iface_comb_num;
+	__le32 iface_comb_num;
 } __packed;
 
-struct qlink_sta_stat_basic_counters {
-	__le64 rx_bytes;
-	__le64 tx_bytes;
-	__le64 rx_beacons;
-	__le32 rx_packets;
-	__le32 tx_packets;
-	__le32 rx_dropped;
-	__le32 tx_failed;
-} __packed;
-
-enum qlink_sta_info_rate_flags {
-	QLINK_STA_INFO_RATE_FLAG_INVALID	= 0,
-	QLINK_STA_INFO_RATE_FLAG_HT_MCS		= BIT(0),
-	QLINK_STA_INFO_RATE_FLAG_VHT_MCS	= BIT(1),
-	QLINK_STA_INFO_RATE_FLAG_SHORT_GI	= BIT(2),
-	QLINK_STA_INFO_RATE_FLAG_60G		= BIT(3),
-};
-
-enum qlink_sta_info_rate_bw {
-	QLINK_STA_INFO_RATE_BW_5		= 0,
-	QLINK_STA_INFO_RATE_BW_10		= 1,
-	QLINK_STA_INFO_RATE_BW_20		= 2,
-	QLINK_STA_INFO_RATE_BW_40		= 3,
-	QLINK_STA_INFO_RATE_BW_80		= 4,
-	QLINK_STA_INFO_RATE_BW_160		= 5,
-};
-
-/**
- * struct qlink_sta_info_rate - STA rate statistics
- *
- * @rate: data rate in Mbps.
- * @flags: bitmap of &enum qlink_sta_flags.
- * @mcs: 802.11-defined MCS index.
- * nss: Number of Spatial Streams.
- * @bw: bandwidth, one of &enum qlink_sta_info_rate_bw.
- */
-struct qlink_sta_info_rate {
-	__le16 rate;
-	u8 flags;
-	u8 mcs;
-	u8 nss;
-	u8 bw;
+struct qlink_iface_limit {
+	__le16 max_num;
+	__le16 type;
 } __packed;
 
-struct qlink_sta_info_state {
-	__le32 mask;
-	__le32 value;
+struct qlink_iface_limit_record {
+	__le16 max_interfaces;
+	u8 num_different_channels;
+	u8 n_limits;
+	struct qlink_iface_limit limits[0];
 } __packed;
 
 #define QLINK_RSSI_OFFSET	120
 
-struct qlink_sta_info_generic {
-	struct qlink_sta_info_state state;
-	__le32 connected_time;
-	__le32 inactive_time;
-	struct qlink_sta_info_rate rx_rate;
-	struct qlink_sta_info_rate tx_rate;
-	u8 rssi;
-	u8 rssi_avg;
-} __packed;
-
 struct qlink_tlv_frag_rts_thr {
 	struct qlink_tlv_hdr hdr;
 	__le16 thr;
@@ -1113,19 +1194,16 @@ enum qlink_dfs_state {
 	QLINK_DFS_AVAILABLE,
 };
 
+/**
+ * struct qlink_tlv_channel - data for QTN_TLV_ID_CHANNEL TLV
+ *
+ * Channel settings.
+ *
+ * @channel: ieee80211 channel settings.
+ */
 struct qlink_tlv_channel {
 	struct qlink_tlv_hdr hdr;
-	__le16 hw_value;
-	__le16 center_freq;
-	__le32 flags;
-	u8 band;
-	u8 max_antenna_gain;
-	u8 max_power;
-	u8 max_reg_power;
-	__le32 dfs_cac_ms;
-	u8 dfs_state;
-	u8 beacon_found;
-	u8 rsvd[2];
+	struct qlink_channel chan;
 } __packed;
 
 /**
@@ -1137,7 +1215,7 @@ struct qlink_tlv_channel {
  */
 struct qlink_tlv_chandef {
 	struct qlink_tlv_hdr hdr;
-	struct qlink_chandef chan;
+	struct qlink_chandef chdef;
 } __packed;
 
 enum qlink_ie_set_type {
@@ -1176,4 +1254,105 @@ struct qlink_chan_stats {
 	s8 chan_noise;
 } __packed;
 
+/**
+ * enum qlink_sta_info - station information bitmap
+ *
+ * Used to indicate which statistics values in &struct qlink_sta_stats
+ * are valid. Individual values are used to fill a bitmap carried in a
+ * payload of QTN_TLV_ID_STA_STATS_MAP.
+ *
+ * @QLINK_STA_INFO_CONNECTED_TIME: connected_time value is valid.
+ * @QLINK_STA_INFO_INACTIVE_TIME: inactive_time value is valid.
+ * @QLINK_STA_INFO_RX_BYTES: lower 32 bits of rx_bytes value are valid.
+ * @QLINK_STA_INFO_TX_BYTES: lower 32 bits of tx_bytes value are valid.
+ * @QLINK_STA_INFO_RX_BYTES64: rx_bytes value is valid.
+ * @QLINK_STA_INFO_TX_BYTES64: tx_bytes value is valid.
+ * @QLINK_STA_INFO_RX_DROP_MISC: rx_dropped_misc value is valid.
+ * @QLINK_STA_INFO_BEACON_RX: rx_beacon value is valid.
+ * @QLINK_STA_INFO_SIGNAL: signal value is valid.
+ * @QLINK_STA_INFO_SIGNAL_AVG: signal_avg value is valid.
+ * @QLINK_STA_INFO_RX_BITRATE: rxrate value is valid.
+ * @QLINK_STA_INFO_TX_BITRATE: txrate value is valid.
+ * @QLINK_STA_INFO_RX_PACKETS: rx_packets value is valid.
+ * @QLINK_STA_INFO_TX_PACKETS: tx_packets value is valid.
+ * @QLINK_STA_INFO_TX_RETRIES: tx_retries value is valid.
+ * @QLINK_STA_INFO_TX_FAILED: tx_failed value is valid.
+ * @QLINK_STA_INFO_STA_FLAGS: sta_flags value is valid.
+ */
+enum qlink_sta_info {
+	QLINK_STA_INFO_CONNECTED_TIME,
+	QLINK_STA_INFO_INACTIVE_TIME,
+	QLINK_STA_INFO_RX_BYTES,
+	QLINK_STA_INFO_TX_BYTES,
+	QLINK_STA_INFO_RX_BYTES64,
+	QLINK_STA_INFO_TX_BYTES64,
+	QLINK_STA_INFO_RX_DROP_MISC,
+	QLINK_STA_INFO_BEACON_RX,
+	QLINK_STA_INFO_SIGNAL,
+	QLINK_STA_INFO_SIGNAL_AVG,
+	QLINK_STA_INFO_RX_BITRATE,
+	QLINK_STA_INFO_TX_BITRATE,
+	QLINK_STA_INFO_RX_PACKETS,
+	QLINK_STA_INFO_TX_PACKETS,
+	QLINK_STA_INFO_TX_RETRIES,
+	QLINK_STA_INFO_TX_FAILED,
+	QLINK_STA_INFO_STA_FLAGS,
+	QLINK_STA_INFO_NUM,
+};
+
+/**
+ * struct qlink_sta_info_rate - STA rate statistics
+ *
+ * @rate: data rate in Mbps.
+ * @flags: bitmap of &enum qlink_sta_info_rate_flags.
+ * @mcs: 802.11-defined MCS index.
+ * nss: Number of Spatial Streams.
+ * @bw: bandwidth, one of &enum qlink_channel_width.
+ */
+struct qlink_sta_info_rate {
+	__le16 rate;
+	u8 flags;
+	u8 mcs;
+	u8 nss;
+	u8 bw;
+} __packed;
+
+/**
+ * struct qlink_sta_stats - data for QTN_TLV_ID_STA_STATS
+ *
+ * Carries statistics of a STA. Not all fields may be filled with
+ * valid values. Valid fields should be indicated as such using a bitmap of
+ * &enum qlink_sta_info. Bitmap is carried separately in a payload of
+ * QTN_TLV_ID_STA_STATS_MAP.
+ */
+struct qlink_sta_stats {
+	__le64 rx_bytes;
+	__le64 tx_bytes;
+	__le64 rx_beacon;
+	__le64 rx_duration;
+	__le64 t_offset;
+	__le32 connected_time;
+	__le32 inactive_time;
+	__le32 rx_packets;
+	__le32 tx_packets;
+	__le32 tx_retries;
+	__le32 tx_failed;
+	__le32 rx_dropped_misc;
+	__le32 beacon_loss_count;
+	__le32 expected_throughput;
+	struct qlink_sta_info_state sta_flags;
+	struct qlink_sta_info_rate txrate;
+	struct qlink_sta_info_rate rxrate;
+	__le16 llid;
+	__le16 plid;
+	u8 local_pm;
+	u8 peer_pm;
+	u8 nonpeer_pm;
+	u8 rx_beacon_signal_avg;
+	u8 plink_state;
+	u8 signal;
+	u8 signal_avg;
+	u8 rsvd[1];
+};
+
 #endif /* _QTN_QLINK_H_ */
diff --git a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c
index 61d999affb09..aeeda81b09ea 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.c
@@ -100,34 +100,6 @@ static enum nl80211_chan_width qlink_chanwidth_to_nl(u8 qlw)
 	}
 }
 
-void qlink_chandef_q2cfg(struct wiphy *wiphy,
-			 const struct qlink_chandef *qch,
-			 struct cfg80211_chan_def *chdef)
-{
-	chdef->center_freq1 = le16_to_cpu(qch->center_freq1);
-	chdef->center_freq2 = le16_to_cpu(qch->center_freq2);
-	chdef->width = qlink_chanwidth_to_nl(qch->width);
-
-	switch (chdef->width) {
-	case NL80211_CHAN_WIDTH_20_NOHT:
-	case NL80211_CHAN_WIDTH_20:
-	case NL80211_CHAN_WIDTH_5:
-	case NL80211_CHAN_WIDTH_10:
-		chdef->chan = ieee80211_get_channel(wiphy, chdef->center_freq1);
-		break;
-	case NL80211_CHAN_WIDTH_40:
-	case NL80211_CHAN_WIDTH_80:
-	case NL80211_CHAN_WIDTH_80P80:
-	case NL80211_CHAN_WIDTH_160:
-		chdef->chan = ieee80211_get_channel(wiphy,
-						    chdef->center_freq1 - 10);
-		break;
-	default:
-		chdef->chan = NULL;
-		break;
-	}
-}
-
 static u8 qlink_chanwidth_nl_to_qlink(enum nl80211_chan_width nlwidth)
 {
 	switch (nlwidth) {
@@ -152,9 +124,29 @@ static u8 qlink_chanwidth_nl_to_qlink(enum nl80211_chan_width nlwidth)
 	}
 }
 
+void qlink_chandef_q2cfg(struct wiphy *wiphy,
+			 const struct qlink_chandef *qch,
+			 struct cfg80211_chan_def *chdef)
+{
+	struct ieee80211_channel *chan;
+
+	chan = ieee80211_get_channel(wiphy, le16_to_cpu(qch->chan.center_freq));
+
+	chdef->chan = chan;
+	chdef->center_freq1 = le16_to_cpu(qch->center_freq1);
+	chdef->center_freq2 = le16_to_cpu(qch->center_freq2);
+	chdef->width = qlink_chanwidth_to_nl(qch->width);
+}
+
 void qlink_chandef_cfg2q(const struct cfg80211_chan_def *chdef,
 			 struct qlink_chandef *qch)
 {
+	struct ieee80211_channel *chan = chdef->chan;
+
+	qch->chan.hw_value = cpu_to_le16(chan->hw_value);
+	qch->chan.center_freq = cpu_to_le16(chan->center_freq);
+	qch->chan.flags = cpu_to_le32(chan->flags);
+
 	qch->center_freq1 = cpu_to_le16(chdef->center_freq1);
 	qch->center_freq2 = cpu_to_le16(chdef->center_freq2);
 	qch->width = qlink_chanwidth_nl_to_qlink(chdef->width);
@@ -172,3 +164,33 @@ enum qlink_hidden_ssid qlink_hidden_ssid_nl2q(enum nl80211_hidden_ssid nl_val)
 		return QLINK_HIDDEN_SSID_NOT_IN_USE;
 	}
 }
+
+bool qtnf_utils_is_bit_set(const u8 *arr, unsigned int bit,
+			   unsigned int arr_max_len)
+{
+	unsigned int idx = bit / BITS_PER_BYTE;
+	u8 mask = 1 << (bit - (idx * BITS_PER_BYTE));
+
+	if (idx >= arr_max_len)
+		return false;
+
+	return arr[idx] & mask;
+}
+
+void qlink_acl_data_cfg2q(const struct cfg80211_acl_data *acl,
+			  struct qlink_acl_data *qacl)
+{
+	switch (acl->acl_policy) {
+	case NL80211_ACL_POLICY_ACCEPT_UNLESS_LISTED:
+		qacl->policy =
+			cpu_to_le32(QLINK_ACL_POLICY_ACCEPT_UNLESS_LISTED);
+		break;
+	case NL80211_ACL_POLICY_DENY_UNLESS_LISTED:
+		qacl->policy = cpu_to_le32(QLINK_ACL_POLICY_DENY_UNLESS_LISTED);
+		break;
+	}
+
+	qacl->num_entries = cpu_to_le32(acl->n_acl_entries);
+	memcpy(qacl->mac_addrs, acl->mac_addrs,
+	       acl->n_acl_entries * sizeof(*qacl->mac_addrs));
+}
diff --git a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h
index 260383d6d109..54caeb38917c 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/qlink_util.h
@@ -69,5 +69,9 @@ void qlink_chandef_q2cfg(struct wiphy *wiphy,
 void qlink_chandef_cfg2q(const struct cfg80211_chan_def *chdef,
 			 struct qlink_chandef *qch);
 enum qlink_hidden_ssid qlink_hidden_ssid_nl2q(enum nl80211_hidden_ssid nl_val);
+bool qtnf_utils_is_bit_set(const u8 *arr, unsigned int bit,
+			   unsigned int arr_max_len);
+void qlink_acl_data_cfg2q(const struct cfg80211_acl_data *acl,
+			  struct qlink_acl_data *qacl);
 
 #endif /* _QTN_FMAC_QLINK_UTIL_H_ */
diff --git a/drivers/net/wireless/quantenna/qtnfmac/util.c b/drivers/net/wireless/quantenna/qtnfmac/util.c
index ed38e87471bf..e745733ba417 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/util.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/util.c
@@ -57,9 +57,10 @@ struct qtnf_sta_node *qtnf_sta_list_lookup_index(struct qtnf_sta_list *list,
 	return NULL;
 }
 
-struct qtnf_sta_node *qtnf_sta_list_add(struct qtnf_sta_list *list,
+struct qtnf_sta_node *qtnf_sta_list_add(struct qtnf_vif *vif,
 					const u8 *mac)
 {
+	struct qtnf_sta_list *list = &vif->sta_list;
 	struct qtnf_sta_node *node;
 
 	if (unlikely(!mac))
@@ -77,13 +78,15 @@ struct qtnf_sta_node *qtnf_sta_list_add(struct qtnf_sta_list *list,
 	ether_addr_copy(node->mac_addr, mac);
 	list_add_tail(&node->list, &list->head);
 	atomic_inc(&list->size);
+	++vif->generation;
 
 done:
 	return node;
 }
 
-bool qtnf_sta_list_del(struct qtnf_sta_list *list, const u8 *mac)
+bool qtnf_sta_list_del(struct qtnf_vif *vif, const u8 *mac)
 {
+	struct qtnf_sta_list *list = &vif->sta_list;
 	struct qtnf_sta_node *node;
 	bool ret = false;
 
@@ -93,6 +96,7 @@ bool qtnf_sta_list_del(struct qtnf_sta_list *list, const u8 *mac)
 		list_del(&node->list);
 		atomic_dec(&list->size);
 		kfree(node);
+		++vif->generation;
 		ret = true;
 	}
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/util.h b/drivers/net/wireless/quantenna/qtnfmac/util.h
index 0359eae8c24b..0d4d92b11540 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/util.h
+++ b/drivers/net/wireless/quantenna/qtnfmac/util.h
@@ -26,9 +26,9 @@ struct qtnf_sta_node *qtnf_sta_list_lookup(struct qtnf_sta_list *list,
 					   const u8 *mac);
 struct qtnf_sta_node *qtnf_sta_list_lookup_index(struct qtnf_sta_list *list,
 						 size_t index);
-struct qtnf_sta_node *qtnf_sta_list_add(struct qtnf_sta_list *list,
+struct qtnf_sta_node *qtnf_sta_list_add(struct qtnf_vif *vif,
 					const u8 *mac);
-bool qtnf_sta_list_del(struct qtnf_sta_list *list, const u8 *mac);
+bool qtnf_sta_list_del(struct qtnf_vif *vif, const u8 *mac);
 
 void qtnf_sta_list_free(struct qtnf_sta_list *list);
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
index d2c289446c00..429d07b651dd 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
@@ -4903,7 +4903,7 @@ void rt2800_vco_calibration(struct rt2x00_dev *rt2x00dev)
 		min_sleep = 2000;
 		break;
 	default:
-		WARN_ONCE(1, "Not supported RF chipet %x for VCO recalibration",
+		WARN_ONCE(1, "Not supported RF chipset %x for VCO recalibration",
 			  rt2x00dev->chip.rf);
 		return;
 	}
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00debug.c b/drivers/net/wireless/ralink/rt2x00/rt2x00debug.c
index f4fdad2ed319..ac2572943ed0 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00debug.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00debug.c
@@ -301,7 +301,7 @@ static ssize_t rt2x00debug_read_queue_dump(struct file *file,
 	return status;
 }
 
-static unsigned int rt2x00debug_poll_queue_dump(struct file *file,
+static __poll_t rt2x00debug_poll_queue_dump(struct file *file,
 						poll_table *wait)
 {
 	struct rt2x00debug_intf *intf = file->private_data;
@@ -309,7 +309,7 @@ static unsigned int rt2x00debug_poll_queue_dump(struct file *file,
 	poll_wait(file, &intf->frame_dump_waitqueue, wait);
 
 	if (!skb_queue_empty(&intf->frame_dump_skbqueue))
-		return POLLOUT | POLLWRNORM;
+		return EPOLLOUT | EPOLLWRNORM;
 
 	return 0;
 }
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
index ecc96312a370..a971bc7a6b63 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00mac.c
@@ -142,32 +142,28 @@ void rt2x00mac_tx(struct ieee80211_hw *hw,
 	if (!rt2x00dev->ops->hw->set_rts_threshold &&
 	    (tx_info->control.rates[0].flags & (IEEE80211_TX_RC_USE_RTS_CTS |
 						IEEE80211_TX_RC_USE_CTS_PROTECT))) {
-		if (rt2x00queue_available(queue) <= 1)
-			goto exit_fail;
+		if (rt2x00queue_available(queue) <= 1) {
+			/*
+			 * Recheck for full queue under lock to avoid race
+			 * conditions with rt2x00lib_txdone().
+			 */
+			spin_lock(&queue->tx_lock);
+			if (rt2x00queue_threshold(queue))
+				rt2x00queue_pause_queue(queue);
+			spin_unlock(&queue->tx_lock);
+
+			goto exit_free_skb;
+		}
 
 		if (rt2x00mac_tx_rts_cts(rt2x00dev, queue, skb))
-			goto exit_fail;
+			goto exit_free_skb;
 	}
 
 	if (unlikely(rt2x00queue_write_tx_frame(queue, skb, control->sta, false)))
-		goto exit_fail;
-
-	/*
-	 * Pausing queue has to be serialized with rt2x00lib_txdone(). Note
-	 * we should not use spin_lock_bh variant as bottom halve was already
-	 * disabled before ieee80211_xmit() call.
-	 */
-	spin_lock(&queue->tx_lock);
-	if (rt2x00queue_threshold(queue))
-		rt2x00queue_pause_queue(queue);
-	spin_unlock(&queue->tx_lock);
+		goto exit_free_skb;
 
 	return;
 
- exit_fail:
-	spin_lock(&queue->tx_lock);
-	rt2x00queue_pause_queue(queue);
-	spin_unlock(&queue->tx_lock);
  exit_free_skb:
 	ieee80211_free_txskb(hw, skb);
 }
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
index a2c1ca5c76d1..a6884e73d2ab 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
@@ -715,6 +715,14 @@ int rt2x00queue_write_tx_frame(struct data_queue *queue, struct sk_buff *skb,
 	rt2x00queue_kick_tx_queue(queue, &txdesc);
 
 out:
+	/*
+	 * Pausing queue has to be serialized with rt2x00lib_txdone(), so we
+	 * do this under queue->tx_lock. Bottom halve was already disabled
+	 * before ieee80211_xmit() call.
+	 */
+	if (rt2x00queue_threshold(queue))
+		rt2x00queue_pause_queue(queue);
+
 	spin_unlock(&queue->tx_lock);
 	return ret;
 }
@@ -1244,10 +1252,8 @@ int rt2x00queue_allocate(struct rt2x00_dev *rt2x00dev)
 	rt2x00dev->data_queues = 2 + rt2x00dev->ops->tx_queues + req_atim;
 
 	queue = kcalloc(rt2x00dev->data_queues, sizeof(*queue), GFP_KERNEL);
-	if (!queue) {
-		rt2x00_err(rt2x00dev, "Queue allocation failed\n");
+	if (!queue)
 		return -ENOMEM;
-	}
 
 	/*
 	 * Initialize pointers
diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c
index e6668ffb77e6..ff0971f1e2c8 100644
--- a/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c
+++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/rtl8225.c
@@ -688,10 +688,7 @@ static void rtl8225z2_b_rf_set_tx_power(struct ieee80211_hw *dev, int channel)
 	cck_power = priv->channels[channel - 1].hw_value & 0xF;
 	ofdm_power = priv->channels[channel - 1].hw_value >> 4;
 
-	if (cck_power > 15)
-		cck_power = (priv->hw_rev == RTL8187BvB) ? 15 : 22;
-	else
-		cck_power += (priv->hw_rev == RTL8187BvB) ? 0 : 7;
+	cck_power += (priv->hw_rev == RTL8187BvB) ? 0 : 7;
 	cck_power += priv->txpwr_base & 0xF;
 	cck_power = min(cck_power, (u8)35);
 
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
index 95e3993d8a33..8828baf26e7b 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
@@ -1172,7 +1172,7 @@ struct rtl8723bu_c2h {
 
 			u8 basic_rate:1;
 			u8 bt_has_reset:1;
-			u8 dummy4_1:1;;
+			u8 dummy4_1:1;
 			u8 ignore_wlan:1;
 			u8 auto_report:1;
 			u8 dummy4_2:3;
diff --git a/drivers/net/wireless/realtek/rtlwifi/base.c b/drivers/net/wireless/realtek/rtlwifi/base.c
index cad2272ae21b..d6c03bd5cc65 100644
--- a/drivers/net/wireless/realtek/rtlwifi/base.c
+++ b/drivers/net/wireless/realtek/rtlwifi/base.c
@@ -244,7 +244,8 @@ static void _rtl_init_hw_vht_capab(struct ieee80211_hw *hw,
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
 
-	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE) {
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE ||
+	    rtlhal->hw_type == HARDWARE_TYPE_RTL8822BE) {
 		u16 mcs_map;
 
 		vht_cap->vht_supported = true;
@@ -395,6 +396,7 @@ static void _rtl_init_mac80211(struct ieee80211_hw *hw)
 	ieee80211_hw_set(hw, CONNECTION_MONITOR);
 	ieee80211_hw_set(hw, MFP_CAPABLE);
 	ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS);
+	ieee80211_hw_set(hw, SUPPORTS_AMSDU_IN_AMPDU);
 
 	/* swlps or hwlps has been set in diff chip in init_sw_vars */
 	if (rtlpriv->psc.swctrl_lps) {
@@ -551,7 +553,8 @@ int rtl_init_core(struct ieee80211_hw *hw)
 
 	/* <4> locks */
 	mutex_init(&rtlpriv->locks.conf_mutex);
-	spin_lock_init(&rtlpriv->locks.ips_lock);
+	mutex_init(&rtlpriv->locks.ips_mutex);
+	mutex_init(&rtlpriv->locks.lps_mutex);
 	spin_lock_init(&rtlpriv->locks.irq_th_lock);
 	spin_lock_init(&rtlpriv->locks.h2c_lock);
 	spin_lock_init(&rtlpriv->locks.rf_ps_lock);
@@ -561,9 +564,7 @@ int rtl_init_core(struct ieee80211_hw *hw)
 	spin_lock_init(&rtlpriv->locks.c2hcmd_lock);
 	spin_lock_init(&rtlpriv->locks.scan_list_lock);
 	spin_lock_init(&rtlpriv->locks.cck_and_rw_pagea_lock);
-	spin_lock_init(&rtlpriv->locks.check_sendpkt_lock);
 	spin_lock_init(&rtlpriv->locks.fw_ps_lock);
-	spin_lock_init(&rtlpriv->locks.lps_lock);
 	spin_lock_init(&rtlpriv->locks.iqk_lock);
 	/* <5> init list */
 	INIT_LIST_HEAD(&rtlpriv->entry_list);
@@ -697,14 +698,94 @@ static void _rtl_query_protection_mode(struct ieee80211_hw *hw,
 	}
 }
 
+u8 rtl_mrate_idx_to_arfr_id(struct ieee80211_hw *hw, u8 rate_index,
+			    enum wireless_mode wirelessmode)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
+	u8 ret = 0;
+
+	switch (rate_index) {
+	case RATR_INX_WIRELESS_NGB:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_BGN_40M_1SS;
+		else
+			ret = RATEID_IDX_BGN_40M_2SS;
+		; break;
+	case RATR_INX_WIRELESS_N:
+	case RATR_INX_WIRELESS_NG:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_GN_N1SS;
+		else
+			ret = RATEID_IDX_GN_N2SS;
+		; break;
+	case RATR_INX_WIRELESS_NB:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_BGN_20M_1SS_BN;
+		else
+			ret = RATEID_IDX_BGN_20M_2SS_BN;
+		; break;
+	case RATR_INX_WIRELESS_GB:
+		ret = RATEID_IDX_BG;
+		break;
+	case RATR_INX_WIRELESS_G:
+		ret = RATEID_IDX_G;
+		break;
+	case RATR_INX_WIRELESS_B:
+		ret = RATEID_IDX_B;
+		break;
+	case RATR_INX_WIRELESS_MC:
+		if (wirelessmode == WIRELESS_MODE_B ||
+		    wirelessmode == WIRELESS_MODE_G ||
+		    wirelessmode == WIRELESS_MODE_N_24G ||
+		    wirelessmode == WIRELESS_MODE_AC_24G)
+			ret = RATEID_IDX_BG;
+		else
+			ret = RATEID_IDX_G;
+		break;
+	case RATR_INX_WIRELESS_AC_5N:
+		if (rtlphy->rf_type == RF_1T1R)
+			ret = RATEID_IDX_VHT_1SS;
+		else
+			ret = RATEID_IDX_VHT_2SS;
+		break;
+	case RATR_INX_WIRELESS_AC_24N:
+		if (rtlphy->current_chan_bw == HT_CHANNEL_WIDTH_80) {
+			if (rtlphy->rf_type == RF_1T1R)
+				ret = RATEID_IDX_VHT_1SS;
+			else
+				ret = RATEID_IDX_VHT_2SS;
+		} else {
+			if (rtlphy->rf_type == RF_1T1R)
+				ret = RATEID_IDX_MIX1;
+			else
+				ret = RATEID_IDX_MIX2;
+		}
+		break;
+	default:
+		ret = RATEID_IDX_BGN_40M_2SS;
+		break;
+	}
+	return ret;
+}
+EXPORT_SYMBOL(rtl_mrate_idx_to_arfr_id);
+
 static void _rtl_txrate_selectmode(struct ieee80211_hw *hw,
 				   struct ieee80211_sta *sta,
 				   struct rtl_tcb_desc *tcb_desc)
 {
+#define SET_RATE_ID(rate_id)					\
+	({typeof(rate_id) _id = rate_id;			\
+	  ((rtlpriv->cfg->spec_ver & RTL_SPEC_NEW_RATEID) ?	\
+		rtl_mrate_idx_to_arfr_id(hw, _id,		\
+			(sta_entry ? sta_entry->wireless_mode :	\
+			 WIRELESS_MODE_G)) :			\
+		_id); })
+
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
 	struct rtl_sta_info *sta_entry = NULL;
-	u8 ratr_index = 7;
+	u8 ratr_index = SET_RATE_ID(RATR_INX_WIRELESS_MC);
 
 	if (sta) {
 		sta_entry = (struct rtl_sta_info *) sta->drv_priv;
@@ -719,7 +800,8 @@ static void _rtl_txrate_selectmode(struct ieee80211_hw *hw,
 				tcb_desc->hw_rate =
 				    rtlpriv->cfg->maps[RTL_RC_CCK_RATE2M];
 				tcb_desc->use_driver_rate = 1;
-				tcb_desc->ratr_index = RATR_INX_WIRELESS_MC;
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_MC);
 			} else {
 				tcb_desc->ratr_index = ratr_index;
 			}
@@ -735,22 +817,30 @@ static void _rtl_txrate_selectmode(struct ieee80211_hw *hw,
 		    mac->opmode == NL80211_IFTYPE_MESH_POINT) {
 			tcb_desc->mac_id = 0;
 
-			if (mac->mode == WIRELESS_MODE_AC_5G)
+			if (sta &&
+			    (rtlpriv->cfg->spec_ver & RTL_SPEC_NEW_RATEID))
+				;	/* use sta_entry->ratr_index */
+			else if (mac->mode == WIRELESS_MODE_AC_5G)
 				tcb_desc->ratr_index =
-					RATR_INX_WIRELESS_AC_5N;
+					SET_RATE_ID(RATR_INX_WIRELESS_AC_5N);
 			else if (mac->mode == WIRELESS_MODE_AC_24G)
 				tcb_desc->ratr_index =
-					RATR_INX_WIRELESS_AC_24N;
+					SET_RATE_ID(RATR_INX_WIRELESS_AC_24N);
 			else if (mac->mode == WIRELESS_MODE_N_24G)
-				tcb_desc->ratr_index = RATR_INX_WIRELESS_NGB;
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_NGB);
 			else if (mac->mode == WIRELESS_MODE_N_5G)
-				tcb_desc->ratr_index = RATR_INX_WIRELESS_NG;
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_NG);
 			else if (mac->mode & WIRELESS_MODE_G)
-				tcb_desc->ratr_index = RATR_INX_WIRELESS_GB;
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_GB);
 			else if (mac->mode & WIRELESS_MODE_B)
-				tcb_desc->ratr_index = RATR_INX_WIRELESS_B;
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_B);
 			else if (mac->mode & WIRELESS_MODE_A)
-				tcb_desc->ratr_index = RATR_INX_WIRELESS_G;
+				tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_G);
 
 		} else if (mac->opmode == NL80211_IFTYPE_AP ||
 			mac->opmode == NL80211_IFTYPE_ADHOC) {
@@ -764,6 +854,7 @@ static void _rtl_txrate_selectmode(struct ieee80211_hw *hw,
 			}
 		}
 	}
+#undef SET_RATE_ID
 }
 
 static void _rtl_query_bandwidth_mode(struct ieee80211_hw *hw,
@@ -859,8 +950,8 @@ static u8 _rtl_get_highest_n_rate(struct ieee80211_hw *hw,
 	struct rtl_phy *rtlphy = &rtlpriv->phy;
 	u8 hw_rate;
 
-	if ((get_rf_type(rtlphy) == RF_2T2R) &&
-	    (sta->ht_cap.mcs.rx_mask[1] != 0))
+	if (get_rf_type(rtlphy) == RF_2T2R &&
+	    sta->ht_cap.mcs.rx_mask[1] != 0)
 		hw_rate = rtlpriv->cfg->maps[RTL_RC_HT_RATEMCS15];
 	else
 		hw_rate = rtlpriv->cfg->maps[RTL_RC_HT_RATEMCS7];
@@ -1140,9 +1231,19 @@ void rtl_get_tcb_desc(struct ieee80211_hw *hw,
 		      struct ieee80211_sta *sta,
 		      struct sk_buff *skb, struct rtl_tcb_desc *tcb_desc)
 {
+#define SET_RATE_ID(rate_id)					\
+	({typeof(rate_id) _id = rate_id;			\
+	  ((rtlpriv->cfg->spec_ver & RTL_SPEC_NEW_RATEID) ?	\
+		rtl_mrate_idx_to_arfr_id(hw, _id,		\
+			(sta_entry ? sta_entry->wireless_mode :	\
+			 WIRELESS_MODE_G)) :			\
+		_id); })
+
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_mac *rtlmac = rtl_mac(rtl_priv(hw));
 	struct ieee80211_hdr *hdr = rtl_get_hdr(skb);
+	struct rtl_sta_info *sta_entry =
+		(sta ? (struct rtl_sta_info *)sta->drv_priv : NULL);
 
 	__le16 fc = rtl_get_fc(skb);
 
@@ -1165,7 +1266,8 @@ void rtl_get_tcb_desc(struct ieee80211_hw *hw,
 		if (info->control.rates[0].idx == 0 ||
 				ieee80211_is_nullfunc(fc)) {
 			tcb_desc->use_driver_rate = true;
-			tcb_desc->ratr_index = RATR_INX_WIRELESS_MC;
+			tcb_desc->ratr_index =
+					SET_RATE_ID(RATR_INX_WIRELESS_MC);
 
 			tcb_desc->disable_ratefallback = 1;
 		} else {
@@ -1180,7 +1282,7 @@ void rtl_get_tcb_desc(struct ieee80211_hw *hw,
 				tcb_desc->hw_rate =
 				_rtl_get_vht_highest_n_rate(hw, sta);
 			} else {
-				if (sta && (sta->ht_cap.ht_supported)) {
+				if (sta && sta->ht_cap.ht_supported) {
 					tcb_desc->hw_rate =
 						_rtl_get_highest_n_rate(hw, sta);
 				} else {
@@ -1207,11 +1309,12 @@ void rtl_get_tcb_desc(struct ieee80211_hw *hw,
 		_rtl_query_protection_mode(hw, tcb_desc, info);
 	} else {
 		tcb_desc->use_driver_rate = true;
-		tcb_desc->ratr_index = RATR_INX_WIRELESS_MC;
+		tcb_desc->ratr_index = SET_RATE_ID(RATR_INX_WIRELESS_MC);
 		tcb_desc->disable_ratefallback = 1;
 		tcb_desc->mac_id = 0;
 		tcb_desc->packet_bw = false;
 	}
+#undef SET_RATE_ID
 }
 EXPORT_SYMBOL(rtl_get_tcb_desc);
 
@@ -1229,7 +1332,6 @@ bool rtl_tx_mgmt_proc(struct ieee80211_hw *hw, struct sk_buff *skb)
 	}
 	if (ieee80211_is_auth(fc)) {
 		RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG, "MAC80211_LINKING\n");
-		rtl_ips_nic_on(hw);
 
 		mac->link_state = MAC80211_LINKING;
 		/* Dul mac */
@@ -1321,6 +1423,10 @@ bool rtl_action_proc(struct ieee80211_hw *hw, struct sk_buff *skb, u8 is_tx)
 				  le16_to_cpu(mgmt->u.action.u.addba_req.capab);
 				tid = (capab &
 				       IEEE80211_ADDBA_PARAM_TID_MASK) >> 2;
+				if (tid >= MAX_TID_COUNT) {
+					rcu_read_unlock();
+					return true;
+				}
 				tid_data = &sta_entry->tids[tid];
 				if (tid_data->agg.rx_agg_state ==
 				    RTL_RX_AGG_START)
@@ -1726,7 +1832,7 @@ int rtl_tx_agg_oper(struct ieee80211_hw *hw,
 void rtl_rx_ampdu_apply(struct rtl_priv *rtlpriv)
 {
 	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
-	u8 reject_agg, ctrl_agg_size = 0, agg_size;
+	u8 reject_agg = 0, ctrl_agg_size = 0, agg_size = 0;
 
 	if (rtlpriv->cfg->ops->get_btc_status())
 		btc_ops->btc_get_ampdu_cfg(rtlpriv, &reject_agg,
@@ -1972,9 +2078,9 @@ void rtl_watchdog_wq_callback(void *data)
 		    rtlpriv->btcoexist.btc_ops->btc_is_bt_ctrl_lps(rtlpriv))
 			goto label_lps_done;
 
-		if (((rtlpriv->link_info.num_rx_inperiod +
-		      rtlpriv->link_info.num_tx_inperiod) > 8) ||
-		    (rtlpriv->link_info.num_rx_inperiod > 2))
+		if (rtlpriv->link_info.num_rx_inperiod +
+		      rtlpriv->link_info.num_tx_inperiod > 8 ||
+		    rtlpriv->link_info.num_rx_inperiod > 2)
 			rtl_lps_leave(hw);
 		else
 			rtl_lps_enter(hw);
@@ -2225,9 +2331,7 @@ static struct sk_buff *rtl_make_smps_action(struct ieee80211_hw *hw,
 	case IEEE80211_SMPS_AUTOMATIC:/* 0 */
 	case IEEE80211_SMPS_NUM_MODES:/* 4 */
 		WARN_ON(1);
-	/* Here will get a 'MISSING_BREAK' in Coverity Test, just ignore it.
-	 * According to Kernel Code, here is right.
-	 */
+	/* fall through */
 	case IEEE80211_SMPS_OFF:/* 1 */ /*MIMO_PS_NOLIMIT*/
 		action_frame->u.action.u.ht_smps.smps_control =
 				WLAN_HT_SMPS_CONTROL_DISABLED;/* 0 */
@@ -2528,6 +2632,9 @@ static int __init rtl_core_module_init(void)
 	if (rtl_rate_control_register())
 		pr_err("rtl: Unable to register rtl_rc, use default RC !!\n");
 
+	/* add debugfs */
+	rtl_debugfs_add_topdir();
+
 	/* init some global vars */
 	INIT_LIST_HEAD(&rtl_global_var.glb_priv_list);
 	spin_lock_init(&rtl_global_var.glb_list_lock);
@@ -2539,6 +2646,9 @@ static void __exit rtl_core_module_exit(void)
 {
 	/*RC*/
 	rtl_rate_control_unregister();
+
+	/* remove debugfs */
+	rtl_debugfs_remove_topdir();
 }
 
 module_init(rtl_core_module_init);
diff --git a/drivers/net/wireless/realtek/rtlwifi/base.h b/drivers/net/wireless/realtek/rtlwifi/base.h
index 26735319b38f..acc924635818 100644
--- a/drivers/net/wireless/realtek/rtlwifi/base.h
+++ b/drivers/net/wireless/realtek/rtlwifi/base.h
@@ -162,6 +162,8 @@ void rtl_c2hcmd_wq_callback(void *data);
 void rtl_c2hcmd_launcher(struct ieee80211_hw *hw, int exec);
 void rtl_c2hcmd_enqueue(struct ieee80211_hw *hw, u8 tag, u8 len, u8 *val);
 
+u8 rtl_mrate_idx_to_arfr_id(struct ieee80211_hw *hw, u8 rate_index,
+			    enum wireless_mode wirelessmode);
 void rtl_get_tcb_desc(struct ieee80211_hw *hw,
 		      struct ieee80211_tx_info *info,
 		      struct ieee80211_sta *sta,
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c
index 44c25724529e..8fce371749d3 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.c
@@ -2704,11 +2704,11 @@ void ex_btc8192e2ant_init_coex_dm(struct btc_coexist *btcoexist)
 	btc8192e2ant_init_coex_dm(btcoexist);
 }
 
-void ex_btc8192e2ant_display_coex_info(struct btc_coexist *btcoexist)
+void ex_btc8192e2ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
 {
 	struct btc_board_info *board_info = &btcoexist->board_info;
 	struct btc_stack_info *stack_info = &btcoexist->stack_info;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	u8 u8tmp[4], i, bt_info_ext, ps_tdma_case = 0;
 	u16 u16tmp[4];
 	u32 u32tmp[4];
@@ -2719,75 +2719,64 @@ void ex_btc8192e2ant_display_coex_info(struct btc_coexist *btcoexist)
 	u8 wifi_dot11_chnl, wifi_hs_chnl;
 	u32 fw_ver = 0, bt_patch_ver = 0;
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n ============[BT Coexist info]============");
+	seq_puts(m, "\n ============[BT Coexist info]============");
 
 	if (btcoexist->manual_control) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ===========[Under Manual Control]===========");
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ==========================================");
-	}
-
-	if (!board_info->bt_exist) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n BT not exists !!!");
-		return;
+		seq_puts(m, "\n ===========[Under Manual Control]===========");
+		seq_puts(m, "\n ==========================================");
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d ", "Ant PG number/ Ant mechanism:",
+	seq_printf(m, "\n %-35s = %d/ %d ", "Ant PG number/ Ant mechanism:",
 		   board_info->pg_ant_num, board_info->btdm_ant_num);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s / %d",
-		 "BT stack/ hci ext ver",
+	seq_printf(m, "\n %-35s = %s / %d", "BT stack/ hci ext ver",
 		   ((stack_info->profile_notified) ? "Yes" : "No"),
 		   stack_info->hci_version);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_BT_PATCH_VER, &bt_patch_ver);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_FW_VER, &fw_ver);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d_%d/ 0x%x/ 0x%x(%d)",
-		 "CoexVer/ FwVer/ PatchVer",
-		 glcoex_ver_date_8192e_2ant, glcoex_ver_8192e_2ant,
-		 fw_ver, bt_patch_ver, bt_patch_ver);
+	seq_printf(m, "\n %-35s = %d_%d/ 0x%x/ 0x%x(%d)",
+		   "CoexVer/ FwVer/ PatchVer",
+		   glcoex_ver_date_8192e_2ant, glcoex_ver_8192e_2ant,
+		   fw_ver, bt_patch_ver, bt_patch_ver);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_DOT11_CHNL,
 			   &wifi_dot11_chnl);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_HS_CHNL, &wifi_hs_chnl);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d / %d(%d)",
-		 "Dot11 channel / HsMode(HsChnl)",
-		 wifi_dot11_chnl, bt_hs_on, wifi_hs_chnl);
+	seq_printf(m, "\n %-35s = %d / %d(%d)",
+		   "Dot11 channel / HsMode(HsChnl)",
+		   wifi_dot11_chnl, bt_hs_on, wifi_hs_chnl);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %3ph ",
-		 "H2C Wifi inform bt chnl Info", coex_dm->wifi_chnl_info);
+	seq_printf(m, "\n %-35s = %3ph ",
+		   "H2C Wifi inform bt chnl Info", coex_dm->wifi_chnl_info);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_HS_RSSI, &bt_hs_rssi);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "Wifi rssi/ HS rssi", wifi_rssi, bt_hs_rssi);
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "Wifi rssi/ HS rssi", wifi_rssi, bt_hs_rssi);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d ",
-		 "Wifi link/ roam/ scan", link, roam, scan);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ",
+		   "Wifi link/ roam/ scan", link, roam, scan);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_TRAFFIC_DIRECTION,
 			   &wifi_traffic_dir);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s / %s/ %s ",
-		 "Wifi status", (wifi_under_5g ? "5G" : "2.4G"),
+	seq_printf(m, "\n %-35s = %s / %s/ %s ",
+		   "Wifi status", (wifi_under_5g ? "5G" : "2.4G"),
 		   ((BTC_WIFI_BW_LEGACY == wifi_bw) ? "Legacy" :
 			(((BTC_WIFI_BW_HT40 == wifi_bw) ? "HT40" : "HT20"))),
 		   ((!wifi_busy) ? "idle" :
 			((BTC_WIFI_TRAFFIC_TX == wifi_traffic_dir) ?
 				"uplink" : "downlink")));
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = [%s/ %d/ %d] ",
-		 "BT [status/ rssi/ retryCnt]",
+	seq_printf(m, "\n %-35s = [%s/ %d/ %d] ",
+		   "BT [status/ rssi/ retryCnt]",
 		   ((btcoexist->bt_info.bt_disabled) ? ("disabled") :
 		    ((coex_sta->c2h_bt_inquiry_page) ?
 		     ("inquiry/page scan") :
@@ -2797,131 +2786,129 @@ void ex_btc8192e2ant_display_coex_info(struct btc_coexist *btcoexist)
 			   coex_dm->bt_status) ? "connected-idle" : "busy")))),
 		   coex_sta->bt_rssi, coex_sta->bt_retry_cnt);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d / %d / %d / %d",
-		 "SCO/HID/PAN/A2DP", stack_info->sco_exist,
+	seq_printf(m, "\n %-35s = %d / %d / %d / %d",
+		   "SCO/HID/PAN/A2DP", stack_info->sco_exist,
 		   stack_info->hid_exist, stack_info->pan_exist,
 		   stack_info->a2dp_exist);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO, m);
 
 	bt_info_ext = coex_sta->bt_info_ext;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s",
-		 "BT Info A2DP rate",
+	seq_printf(m, "\n %-35s = %s",
+		   "BT Info A2DP rate",
 		   (bt_info_ext&BIT0) ? "Basic rate" : "EDR rate");
 
 	for (i = 0; i < BT_INFO_SRC_8192E_2ANT_MAX; i++) {
 		if (coex_sta->bt_info_c2h_cnt[i]) {
-			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-				 "\r\n %-35s = %7ph(%d)",
-				 glbt_info_src_8192e_2ant[i],
-				 coex_sta->bt_info_c2h[i],
-				 coex_sta->bt_info_c2h_cnt[i]);
+			seq_printf(m, "\n %-35s = %7ph(%d)",
+				   glbt_info_src_8192e_2ant[i],
+				   coex_sta->bt_info_c2h[i],
+				   coex_sta->bt_info_c2h_cnt[i]);
 		}
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s/%s",
-		 "PS state, IPS/LPS",
-		 ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
-		 ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")));
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD);
+	seq_printf(m, "\n %-35s = %s/%s",
+		   "PS state, IPS/LPS",
+		   ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
+		   ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")));
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD, m);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x ", "SS Type",
-		 coex_dm->cur_ss_type);
+	seq_printf(m, "\n %-35s = 0x%x ", "SS Type",
+		   coex_dm->cur_ss_type);
 
 	/* Sw mechanism	*/
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Sw mechanism]============");
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d ",
-		 "SM1[ShRf/ LpRA/ LimDig]", coex_dm->cur_rf_rx_lpf_shrink,
-		 coex_dm->cur_low_penalty_ra, coex_dm->limited_dig);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d(0x%x) ",
-		 "SM2[AgcT/ AdcB/ SwDacSwing(lvl)]",
-		 coex_dm->cur_agc_table_en, coex_dm->cur_adc_back_off,
-		 coex_dm->cur_dac_swing_on, coex_dm->cur_dac_swing_lvl);
-
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x ", "Rate Mask",
-		 btcoexist->bt_info.ra_mask);
+	seq_printf(m, "\n %-35s",
+		   "============[Sw mechanism]============");
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ",
+		   "SM1[ShRf/ LpRA/ LimDig]", coex_dm->cur_rf_rx_lpf_shrink,
+		   coex_dm->cur_low_penalty_ra, coex_dm->limited_dig);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d(0x%x) ",
+		   "SM2[AgcT/ AdcB/ SwDacSwing(lvl)]",
+		   coex_dm->cur_agc_table_en, coex_dm->cur_adc_back_off,
+		   coex_dm->cur_dac_swing_on, coex_dm->cur_dac_swing_lvl);
+
+	seq_printf(m, "\n %-35s = 0x%x ", "Rate Mask",
+		   btcoexist->bt_info.ra_mask);
 
 	/* Fw mechanism	*/
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Fw mechanism]============");
+	seq_printf(m, "\n %-35s",
+		   "============[Fw mechanism]============");
 
 	ps_tdma_case = coex_dm->cur_ps_tdma;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %5ph case-%d (auto:%d)",
-		 "PS TDMA", coex_dm->ps_tdma_para,
-		 ps_tdma_case, coex_dm->auto_tdma_adjust);
+	seq_printf(m,
+		   "\n %-35s = %5ph case-%d (auto:%d)",
+		   "PS TDMA", coex_dm->ps_tdma_para,
+		   ps_tdma_case, coex_dm->auto_tdma_adjust);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d ",
-		 "DecBtPwr/ IgnWlanAct",
-		 coex_dm->cur_dec_bt_pwr, coex_dm->cur_ignore_wlan_act);
+	seq_printf(m, "\n %-35s = %d/ %d ",
+		   "DecBtPwr/ IgnWlanAct",
+		   coex_dm->cur_dec_bt_pwr, coex_dm->cur_ignore_wlan_act);
 
 	/* Hw setting */
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Hw setting]============");
+	seq_printf(m, "\n %-35s",
+		   "============[Hw setting]============");
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x",
-		 "RF-A, 0x1e initVal", coex_dm->bt_rf0x1e_backup);
+	seq_printf(m, "\n %-35s = 0x%x",
+		   "RF-A, 0x1e initVal", coex_dm->bt_rf0x1e_backup);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/0x%x/0x%x/0x%x",
-		 "backup ARFR1/ARFR2/RL/AMaxTime", coex_dm->backup_arfr_cnt1,
-		 coex_dm->backup_arfr_cnt2, coex_dm->backup_retry_limit,
-		 coex_dm->backup_ampdu_maxtime);
+	seq_printf(m, "\n %-35s = 0x%x/0x%x/0x%x/0x%x",
+		   "backup ARFR1/ARFR2/RL/AMaxTime", coex_dm->backup_arfr_cnt1,
+		   coex_dm->backup_arfr_cnt2, coex_dm->backup_retry_limit,
+		   coex_dm->backup_ampdu_maxtime);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x430);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x434);
 	u16tmp[0] = btcoexist->btc_read_2byte(btcoexist, 0x42a);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x456);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/0x%x/0x%x/0x%x",
-		 "0x430/0x434/0x42a/0x456",
-		 u32tmp[0], u32tmp[1], u16tmp[0], u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/0x%x/0x%x/0x%x",
+		   "0x430/0x434/0x42a/0x456",
+		   u32tmp[0], u32tmp[1], u16tmp[0], u8tmp[0]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xc04);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0xd04);
 	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x90c);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0xc04/ 0xd04/ 0x90c", u32tmp[0], u32tmp[1], u32tmp[2]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0xc04/ 0xd04/ 0x90c", u32tmp[0], u32tmp[1], u32tmp[2]);
 
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x778);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x", "0x778",
-		 u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x", "0x778", u8tmp[0]);
 
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x92c);
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x930);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x92c/ 0x930", (u8tmp[0]), u32tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x92c/ 0x930", (u8tmp[0]), u32tmp[0]);
 
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x40);
 	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x4f);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x40/ 0x4f", u8tmp[0], u8tmp[1]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x40/ 0x4f", u8tmp[0], u8tmp[1]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x550);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x522);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x550(bcn ctrl)/0x522", u32tmp[0], u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x550(bcn ctrl)/0x522", u32tmp[0], u8tmp[0]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xc50);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x", "0xc50(dig)",
-		 u32tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x", "0xc50(dig)",
+		   u32tmp[0]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6c0);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x6c4);
 	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x6c8);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x6cc);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
-		 "0x6c0/0x6c4/0x6c8/0x6cc(coexTable)",
-		 u32tmp[0], u32tmp[1], u32tmp[2], u8tmp[0]);
-
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "0x770(hp rx[31:16]/tx[15:0])",
-		 coex_sta->high_priority_rx, coex_sta->high_priority_tx);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "0x774(lp rx[31:16]/tx[15:0])",
-		 coex_sta->low_priority_rx, coex_sta->low_priority_tx);
+	seq_printf(m,
+		   "\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "0x6c0/0x6c4/0x6c8/0x6cc(coexTable)",
+		   u32tmp[0], u32tmp[1], u32tmp[2], u8tmp[0]);
+
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "0x770(hp rx[31:16]/tx[15:0])",
+		   coex_sta->high_priority_rx, coex_sta->high_priority_tx);
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "0x774(lp rx[31:16]/tx[15:0])",
+		   coex_sta->low_priority_rx, coex_sta->low_priority_tx);
 	if (btcoexist->auto_report_2ant)
 		btc8192e2ant_monitor_bt_ctr(btcoexist);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS, m);
 }
 
 void ex_btc8192e2ant_ips_notify(struct btc_coexist *btcoexist, u8 type)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.h
index 65502acee52c..b8c95c7afc06 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8192e2ant.h
@@ -180,4 +180,5 @@ void ex_btc8192e2ant_stack_operation_notify(struct btc_coexist *btcoexist,
 					    u8 type);
 void ex_btc8192e2ant_halt_notify(struct btc_coexist *btcoexist);
 void ex_btc8192e2ant_periodical(struct btc_coexist *btcoexist);
-void ex_btc8192e2ant_display_coex_info(struct btc_coexist *btcoexist);
+void ex_btc8192e2ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c
index 5f726f6d3567..fd3b1fb35dff 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.c
@@ -2474,12 +2474,12 @@ void ex_btc8723b1ant_init_coex_dm(struct btc_coexist *btcoexist)
 	halbtc8723b1ant_query_bt_info(btcoexist);
 }
 
-void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist)
+void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
 {
 	struct btc_board_info *board_info = &btcoexist->board_info;
 	struct btc_stack_info *stack_info = &btcoexist->stack_info;
 	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	u8 u8tmp[4], i, bt_info_ext, pstdmacase = 0;
 	u16 u16tmp[4];
 	u32 u32tmp[4];
@@ -2491,62 +2491,56 @@ void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist)
 	u8 wifi_dot11_chnl, wifi_hs_chnl;
 	u32 fw_ver = 0, bt_patch_ver = 0;
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n ============[BT Coexist info]============");
+	seq_puts(m, "\n ============[BT Coexist info]============");
 
 	if (btcoexist->manual_control) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ============[Under Manual Control]==========");
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ==========================================");
+		seq_puts(m, "\n ============[Under Manual Control]==========");
+		seq_puts(m, "\n ==========================================");
 	}
 	if (btcoexist->stop_coex_dm) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ============[Coex is STOPPED]============");
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ==========================================");
+		seq_puts(m, "\n ============[Coex is STOPPED]============");
+		seq_puts(m, "\n ==========================================");
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d",
-		 "Ant PG Num/ Ant Mech/ Ant Pos:",
-		 board_info->pg_ant_num, board_info->btdm_ant_num,
-		 board_info->btdm_ant_pos);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d",
+		   "Ant PG Num/ Ant Mech/ Ant Pos:",
+		   board_info->pg_ant_num, board_info->btdm_ant_num,
+		   board_info->btdm_ant_pos);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s / %d",
-		 "BT stack/ hci ext ver",
-		 ((stack_info->profile_notified) ? "Yes" : "No"),
-		 stack_info->hci_version);
+	seq_printf(m, "\n %-35s = %s / %d",
+		   "BT stack/ hci ext ver",
+		   ((stack_info->profile_notified) ? "Yes" : "No"),
+		   stack_info->hci_version);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_BT_PATCH_VER, &bt_patch_ver);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_FW_VER, &fw_ver);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d_%x/ 0x%x/ 0x%x(%d)",
-		 "CoexVer/ FwVer/ PatchVer",
-		 glcoex_ver_date_8723b_1ant, glcoex_ver_8723b_1ant,
-		 fw_ver, bt_patch_ver, bt_patch_ver);
+	seq_printf(m, "\n %-35s = %d_%x/ 0x%x/ 0x%x(%d)",
+		   "CoexVer/ FwVer/ PatchVer",
+		   glcoex_ver_date_8723b_1ant, glcoex_ver_8723b_1ant,
+		   fw_ver, bt_patch_ver, bt_patch_ver);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_DOT11_CHNL,
 			   &wifi_dot11_chnl);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_HS_CHNL, &wifi_hs_chnl);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d / %d(%d)",
-		 "Dot11 channel / HsChnl(HsMode)",
-		 wifi_dot11_chnl, wifi_hs_chnl, bt_hs_on);
+	seq_printf(m, "\n %-35s = %d / %d(%d)",
+		   "Dot11 channel / HsChnl(HsMode)",
+		   wifi_dot11_chnl, wifi_hs_chnl, bt_hs_on);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %3ph ",
-		 "H2C Wifi inform bt chnl Info",
-		 coex_dm->wifi_chnl_info);
+	seq_printf(m, "\n %-35s = %3ph ",
+		   "H2C Wifi inform bt chnl Info",
+		   coex_dm->wifi_chnl_info);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_HS_RSSI, &bt_hs_rssi);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "Wifi rssi/ HS rssi", wifi_rssi, bt_hs_rssi);
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "Wifi rssi/ HS rssi", wifi_rssi, bt_hs_rssi);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d ",
-		 "Wifi link/ roam/ scan", link, roam, scan);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ",
+		   "Wifi link/ roam/ scan", link, roam, scan);
 
 	btcoexist->btc_get(btcoexist , BTC_GET_BL_WIFI_UNDER_5G,
 			   &wifi_under_5g);
@@ -2555,110 +2549,106 @@ void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist)
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_TRAFFIC_DIRECTION,
 			   &wifi_traffic_dir);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s / %s/ %s ",
-		 "Wifi status", (wifi_under_5g ? "5G" : "2.4G"),
-		 ((wifi_bw == BTC_WIFI_BW_LEGACY) ? "Legacy" :
-		  ((wifi_bw == BTC_WIFI_BW_HT40) ? "HT40" : "HT20")),
-		  ((!wifi_busy) ? "idle" :
-		   ((wifi_traffic_dir == BTC_WIFI_TRAFFIC_TX) ?
-		   "uplink" : "downlink")));
+	seq_printf(m, "\n %-35s = %s / %s/ %s ",
+		   "Wifi status", (wifi_under_5g ? "5G" : "2.4G"),
+		   ((wifi_bw == BTC_WIFI_BW_LEGACY) ? "Legacy" :
+		    ((wifi_bw == BTC_WIFI_BW_HT40) ? "HT40" : "HT20")),
+		    ((!wifi_busy) ? "idle" :
+		     ((wifi_traffic_dir == BTC_WIFI_TRAFFIC_TX) ?
+		     "uplink" : "downlink")));
 
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_LINK_STATUS,
 			   &wifi_link_status);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d/ %d/ %d",
-		 "sta/vwifi/hs/p2pGo/p2pGc",
-		 ((wifi_link_status & WIFI_STA_CONNECTED) ? 1 : 0),
-		 ((wifi_link_status & WIFI_AP_CONNECTED) ? 1 : 0),
-		 ((wifi_link_status & WIFI_HS_CONNECTED) ? 1 : 0),
-		 ((wifi_link_status & WIFI_P2P_GO_CONNECTED) ? 1 : 0),
-		 ((wifi_link_status & WIFI_P2P_GC_CONNECTED) ? 1 : 0));
-
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = [%s/ %d/ %d] ",
-		 "BT [status/ rssi/ retryCnt]",
-		 ((coex_sta->bt_disabled) ? ("disabled") :
-		  ((coex_sta->c2h_bt_inquiry_page) ? ("inquiry/page scan") :
-		   ((BT_8723B_1ANT_BT_STATUS_NON_CONNECTED_IDLE ==
-		     coex_dm->bt_status) ?
-		    "non-connected idle" :
-		    ((BT_8723B_1ANT_BT_STATUS_CONNECTED_IDLE ==
-		      coex_dm->bt_status) ?
-		     "connected-idle" : "busy")))),
-		     coex_sta->bt_rssi, coex_sta->bt_retry_cnt);
-
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d / %d / %d / %d",
-		 "SCO/HID/PAN/A2DP", bt_link_info->sco_exist,
-		 bt_link_info->hid_exist, bt_link_info->pan_exist,
-		 bt_link_info->a2dp_exist);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d/ %d/ %d",
+		   "sta/vwifi/hs/p2pGo/p2pGc",
+		   ((wifi_link_status & WIFI_STA_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_AP_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_HS_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_P2P_GO_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_P2P_GC_CONNECTED) ? 1 : 0));
+
+	seq_printf(m, "\n %-35s = [%s/ %d/ %d] ",
+		   "BT [status/ rssi/ retryCnt]",
+		   ((coex_sta->bt_disabled) ? ("disabled") :
+		    ((coex_sta->c2h_bt_inquiry_page) ? ("inquiry/page scan") :
+		     ((BT_8723B_1ANT_BT_STATUS_NON_CONNECTED_IDLE ==
+		       coex_dm->bt_status) ?
+		      "non-connected idle" :
+		      ((BT_8723B_1ANT_BT_STATUS_CONNECTED_IDLE ==
+			coex_dm->bt_status) ?
+		       "connected-idle" : "busy")))),
+		       coex_sta->bt_rssi, coex_sta->bt_retry_cnt);
+
+	seq_printf(m, "\n %-35s = %d / %d / %d / %d",
+		   "SCO/HID/PAN/A2DP", bt_link_info->sco_exist,
+		   bt_link_info->hid_exist, bt_link_info->pan_exist,
+		   bt_link_info->a2dp_exist);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO, m);
 
 	bt_info_ext = coex_sta->bt_info_ext;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s",
-		 "BT Info A2DP rate",
-		 (bt_info_ext & BIT0) ? "Basic rate" : "EDR rate");
+	seq_printf(m, "\n %-35s = %s",
+		   "BT Info A2DP rate",
+		   (bt_info_ext & BIT0) ? "Basic rate" : "EDR rate");
 
 	for (i = 0; i < BT_INFO_SRC_8723B_1ANT_MAX; i++) {
 		if (coex_sta->bt_info_c2h_cnt[i]) {
-			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-				 "\r\n %-35s = %7ph(%d)",
-				 glbt_info_src_8723b_1ant[i],
-				 coex_sta->bt_info_c2h[i],
-				 coex_sta->bt_info_c2h_cnt[i]);
+			seq_printf(m, "\n %-35s = %7ph(%d)",
+				   glbt_info_src_8723b_1ant[i],
+				   coex_sta->bt_info_c2h[i],
+				   coex_sta->bt_info_c2h_cnt[i]);
 		}
 	}
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %s/%s, (0x%x/0x%x)",
-		 "PS state, IPS/LPS, (lps/rpwm)",
-		 ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
-		 ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")),
-		 btcoexist->bt_info.lps_val,
-		 btcoexist->bt_info.rpwm_val);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD);
+	seq_printf(m, "\n %-35s = %s/%s, (0x%x/0x%x)",
+		   "PS state, IPS/LPS, (lps/rpwm)",
+		   ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
+		   ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")),
+		   btcoexist->bt_info.lps_val,
+		   btcoexist->bt_info.rpwm_val);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD, m);
 
 	if (!btcoexist->manual_control) {
 		/* Sw mechanism	*/
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-			 "============[Sw mechanism]============");
+		seq_printf(m, "\n %-35s",
+			   "============[Sw mechanism]============");
 
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/",
-			 "SM[LowPenaltyRA]", coex_dm->cur_low_penalty_ra);
+		seq_printf(m, "\n %-35s = %d/",
+			   "SM[LowPenaltyRA]", coex_dm->cur_low_penalty_ra);
 
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s/ %s/ %d ",
-			 "DelBA/ BtCtrlAgg/ AggSize",
+		seq_printf(m, "\n %-35s = %s/ %s/ %d ",
+			   "DelBA/ BtCtrlAgg/ AggSize",
 			   (btcoexist->bt_info.reject_agg_pkt ? "Yes" : "No"),
 			   (btcoexist->bt_info.bt_ctrl_buf_size ? "Yes" : "No"),
 			   btcoexist->bt_info.agg_buf_size);
 
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x ",
-			 "Rate Mask", btcoexist->bt_info.ra_mask);
+		seq_printf(m, "\n %-35s = 0x%x ",
+			   "Rate Mask", btcoexist->bt_info.ra_mask);
 
 		/* Fw mechanism	*/
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-			 "============[Fw mechanism]============");
+		seq_printf(m, "\n %-35s",
+			   "============[Fw mechanism]============");
 
 		pstdmacase = coex_dm->cur_ps_tdma;
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %5ph case-%d (auto:%d)",
+		seq_printf(m, "\n %-35s = %5ph case-%d (auto:%d)",
 			   "PS TDMA", coex_dm->ps_tdma_para,
 			   pstdmacase, coex_dm->auto_tdma_adjust);
 
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d ",
-			 "IgnWlanAct", coex_dm->cur_ignore_wlan_act);
+		seq_printf(m, "\n %-35s = %d ",
+			   "IgnWlanAct", coex_dm->cur_ignore_wlan_act);
 
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x ",
-			 "Latest error condition(should be 0)",
+		seq_printf(m, "\n %-35s = 0x%x ",
+			   "Latest error condition(should be 0)",
 			   coex_dm->error_condition);
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d",
-		 "Coex Table Type", coex_sta->coex_table_type);
+	seq_printf(m, "\n %-35s = %d",
+		   "Coex Table Type", coex_sta->coex_table_type);
 
 	/* Hw setting */
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Hw setting]============");
+	seq_printf(m, "\n %-35s",
+		   "============[Hw setting]============");
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/0x%x/0x%x/0x%x",
-		 "backup ARFR1/ARFR2/RL/AMaxTime", coex_dm->backup_arfr_cnt1,
+	seq_printf(m, "\n %-35s = 0x%x/0x%x/0x%x/0x%x",
+		   "backup ARFR1/ARFR2/RL/AMaxTime", coex_dm->backup_arfr_cnt1,
 		   coex_dm->backup_arfr_cnt2, coex_dm->backup_retry_limit,
 		   coex_dm->backup_ampdu_max_time);
 
@@ -2666,50 +2656,49 @@ void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist)
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x434);
 	u16tmp[0] = btcoexist->btc_read_2byte(btcoexist, 0x42a);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x456);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/0x%x/0x%x/0x%x",
-		 "0x430/0x434/0x42a/0x456",
-		 u32tmp[0], u32tmp[1], u16tmp[0], u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/0x%x/0x%x/0x%x",
+		   "0x430/0x434/0x42a/0x456",
+		   u32tmp[0], u32tmp[1], u16tmp[0], u8tmp[0]);
 
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x778);
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6cc);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x880);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x778/0x6cc/0x880[29:25]", u8tmp[0], u32tmp[0],
-		 (u32tmp[1] & 0x3e000000) >> 25);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x778/0x6cc/0x880[29:25]", u8tmp[0], u32tmp[0],
+		   (u32tmp[1] & 0x3e000000) >> 25);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x948);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x67);
 	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x765);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x948/ 0x67[5] / 0x765",
-		 u32tmp[0], ((u8tmp[0] & 0x20) >> 5), u8tmp[1]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x948/ 0x67[5] / 0x765",
+		   u32tmp[0], ((u8tmp[0] & 0x20) >> 5), u8tmp[1]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x92c);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x930);
 	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x944);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x92c[1:0]/ 0x930[7:0]/0x944[1:0]",
-		 u32tmp[0] & 0x3, u32tmp[1] & 0xff, u32tmp[2] & 0x3);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x92c[1:0]/ 0x930[7:0]/0x944[1:0]",
+		   u32tmp[0] & 0x3, u32tmp[1] & 0xff, u32tmp[2] & 0x3);
 
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x39);
 	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x40);
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x4c);
 	u8tmp[2] = btcoexist->btc_read_1byte(btcoexist, 0x64);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
-		 "0x38[11]/0x40/0x4c[24:23]/0x64[0]",
-		 ((u8tmp[0] & 0x8) >> 3), u8tmp[1],
-		  ((u32tmp[0] & 0x01800000) >> 23), u8tmp[2] & 0x1);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "0x38[11]/0x40/0x4c[24:23]/0x64[0]",
+		   ((u8tmp[0] & 0x8) >> 3), u8tmp[1],
+		    ((u32tmp[0] & 0x01800000) >> 23), u8tmp[2] & 0x1);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x550);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x522);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x550(bcn ctrl)/0x522", u32tmp[0], u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x550(bcn ctrl)/0x522", u32tmp[0], u8tmp[0]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xc50);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x49c);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0xc50(dig)/0x49c(null-drop)", u32tmp[0] & 0xff, u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0xc50(dig)/0x49c(null-drop)", u32tmp[0] & 0xff, u8tmp[0]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xda0);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0xda4);
@@ -2727,26 +2716,26 @@ void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist)
 		   (u32tmp[3] & 0xffff);
 	fa_cck = (u8tmp[0] << 8) + u8tmp[1];
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "OFDM-CCA/OFDM-FA/CCK-FA",
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "OFDM-CCA/OFDM-FA/CCK-FA",
 		 u32tmp[0] & 0xffff, fa_ofdm, fa_cck);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6c0);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x6c4);
 	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x6c8);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x6c0/0x6c4/0x6c8(coexTable)",
-		 u32tmp[0], u32tmp[1], u32tmp[2]);
-
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "0x770(high-pri rx/tx)", coex_sta->high_priority_rx,
-		 coex_sta->high_priority_tx);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "0x774(low-pri rx/tx)", coex_sta->low_priority_rx,
-		 coex_sta->low_priority_tx);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x6c0/0x6c4/0x6c8(coexTable)",
+		   u32tmp[0], u32tmp[1], u32tmp[2]);
+
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "0x770(high-pri rx/tx)", coex_sta->high_priority_rx,
+		   coex_sta->high_priority_tx);
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "0x774(low-pri rx/tx)", coex_sta->low_priority_rx,
+		   coex_sta->low_priority_tx);
 	if (btcoexist->auto_report_1ant)
 		halbtc8723b1ant_monitor_bt_ctr(btcoexist);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS, m);
 }
 
 void ex_btc8723b1ant_ips_notify(struct btc_coexist *btcoexist, u8 type)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.h
index 8d4fde235e11..934f27893c16 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b1ant.h
@@ -220,5 +220,6 @@ void ex_btc8723b1ant_halt_notify(struct btc_coexist *btcoexist);
 void ex_btc8723b1ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnpstate);
 void ex_btc8723b1ant_coex_dm_reset(struct btc_coexist *btcoexist);
 void ex_btc8723b1ant_periodical(struct btc_coexist *btcoexist);
-void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist);
+void ex_btc8723b1ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m);
 void ex_btc8723b1ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
index e8f07573aed9..4907c2ffadfe 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.c
@@ -3780,12 +3780,12 @@ void ex_btc8723b2ant_init_coex_dm(struct btc_coexist *btcoexist)
 	btc8723b2ant_init_coex_dm(btcoexist);
 }
 
-void ex_btc8723b2ant_display_coex_info(struct btc_coexist *btcoexist)
+void ex_btc8723b2ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
 {
 	struct btc_board_info *board_info = &btcoexist->board_info;
 	struct btc_stack_info *stack_info = &btcoexist->stack_info;
 	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	u8 u8tmp[4], i, bt_info_ext, ps_tdma_case = 0;
 	u32 u32tmp[4];
 	bool roam = false, scan = false;
@@ -3797,173 +3797,161 @@ void ex_btc8723b2ant_display_coex_info(struct btc_coexist *btcoexist)
 	u32 fw_ver = 0, bt_patch_ver = 0;
 	u8 ap_num = 0;
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n ============[BT Coexist info]============");
+	seq_puts(m, "\n ============[BT Coexist info]============");
 
 	if (btcoexist->manual_control) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ==========[Under Manual Control]============");
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ==========================================");
-	}
-
-	if (!board_info->bt_exist) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n BT not exists !!!");
-		return;
+		seq_puts(m, "\n ==========[Under Manual Control]============");
+		seq_puts(m, "\n ==========================================");
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d ",
-		 "Ant PG number/ Ant mechanism:",
-		 board_info->pg_ant_num, board_info->btdm_ant_num);
+	seq_printf(m, "\n %-35s = %d/ %d ",
+		   "Ant PG number/ Ant mechanism:",
+		   board_info->pg_ant_num, board_info->btdm_ant_num);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s / %d",
-		 "BT stack/ hci ext ver",
-		 ((stack_info->profile_notified) ? "Yes" : "No"),
-		 stack_info->hci_version);
+	seq_printf(m, "\n %-35s = %s / %d",
+		   "BT stack/ hci ext ver",
+		   ((stack_info->profile_notified) ? "Yes" : "No"),
+		   stack_info->hci_version);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_BT_PATCH_VER, &bt_patch_ver);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_FW_VER, &fw_ver);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d_%x/ 0x%x/ 0x%x(%d)",
-		 "CoexVer/ FwVer/ PatchVer",
-		 glcoex_ver_date_8723b_2ant, glcoex_ver_8723b_2ant,
-		 fw_ver, bt_patch_ver, bt_patch_ver);
+	seq_printf(m, "\n %-35s = %d_%x/ 0x%x/ 0x%x(%d)",
+		   "CoexVer/ FwVer/ PatchVer",
+		   glcoex_ver_date_8723b_2ant, glcoex_ver_8723b_2ant,
+		   fw_ver, bt_patch_ver, bt_patch_ver);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_DOT11_CHNL,
 			   &wifi_dot11_chnl);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_HS_CHNL, &wifi_hs_chnl);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d / %d(%d)",
-		 "Dot11 channel / HsChnl(HsMode)",
-		 wifi_dot11_chnl, wifi_hs_chnl, bt_hs_on);
+	seq_printf(m, "\n %-35s = %d / %d(%d)",
+		   "Dot11 channel / HsChnl(HsMode)",
+		   wifi_dot11_chnl, wifi_hs_chnl, bt_hs_on);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %3ph ",
-		 "H2C Wifi inform bt chnl Info", coex_dm->wifi_chnl_info);
+	seq_printf(m, "\n %-35s = %3ph ",
+		   "H2C Wifi inform bt chnl Info", coex_dm->wifi_chnl_info);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_HS_RSSI, &bt_hs_rssi);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM, &ap_num);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d",
-		 "Wifi rssi/ HS rssi/ AP#", wifi_rssi, bt_hs_rssi, ap_num);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d",
+		   "Wifi rssi/ HS rssi/ AP#", wifi_rssi, bt_hs_rssi, ap_num);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d ",
-		 "Wifi link/ roam/ scan", link, roam, scan);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ",
+		   "Wifi link/ roam/ scan", link, roam, scan);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_TRAFFIC_DIRECTION,
 			   &wifi_traffic_dir);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s / %s/ %s ",
-		 "Wifi status", (wifi_under_5g ? "5G" : "2.4G"),
+	seq_printf(m, "\n %-35s = %s / %s/ %s ",
+		   "Wifi status", (wifi_under_5g ? "5G" : "2.4G"),
 		 ((wifi_bw == BTC_WIFI_BW_LEGACY) ? "Legacy" :
 		 (((wifi_bw == BTC_WIFI_BW_HT40) ? "HT40" : "HT20"))),
 		 ((!wifi_busy) ? "idle" :
 		 ((wifi_traffic_dir == BTC_WIFI_TRAFFIC_TX) ?
 		  "uplink" : "downlink")));
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d / %d / %d / %d",
-		 "SCO/HID/PAN/A2DP",
-		 bt_link_info->sco_exist, bt_link_info->hid_exist,
-		 bt_link_info->pan_exist, bt_link_info->a2dp_exist);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO);
+	seq_printf(m, "\n %-35s = %d / %d / %d / %d",
+		   "SCO/HID/PAN/A2DP",
+		   bt_link_info->sco_exist, bt_link_info->hid_exist,
+		   bt_link_info->pan_exist, bt_link_info->a2dp_exist);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO, m);
 
 	bt_info_ext = coex_sta->bt_info_ext;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s",
-		 "BT Info A2DP rate",
-		 (bt_info_ext & BIT0) ? "Basic rate" : "EDR rate");
+	seq_printf(m, "\n %-35s = %s",
+		   "BT Info A2DP rate",
+		   (bt_info_ext & BIT0) ? "Basic rate" : "EDR rate");
 
 	for (i = 0; i < BT_INFO_SRC_8723B_2ANT_MAX; i++) {
 		if (coex_sta->bt_info_c2h_cnt[i]) {
-			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-				 "\r\n %-35s = %7ph(%d)",
-				 glbt_info_src_8723b_2ant[i],
-				 coex_sta->bt_info_c2h[i],
-				 coex_sta->bt_info_c2h_cnt[i]);
+			seq_printf(m, "\n %-35s = %7ph(%d)",
+				   glbt_info_src_8723b_2ant[i],
+				   coex_sta->bt_info_c2h[i],
+				   coex_sta->bt_info_c2h_cnt[i]);
 		}
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s/%s",
-		 "PS state, IPS/LPS",
-		 ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
-		 ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")));
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD);
+	seq_printf(m, "\n %-35s = %s/%s",
+		   "PS state, IPS/LPS",
+		   ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
+		   ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")));
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD, m);
 
 	/* Sw mechanism	*/
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s", "============[Sw mechanism]============");
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d ",
-		 "SM1[ShRf/ LpRA/ LimDig]", coex_dm->cur_rf_rx_lpf_shrink,
-		 coex_dm->cur_low_penalty_ra, coex_dm->limited_dig);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d/ %d(0x%x) ",
-		 "SM2[AgcT/ AdcB/ SwDacSwing(lvl)]",
-		 coex_dm->cur_agc_table_en, coex_dm->cur_adc_back_off,
-		 coex_dm->cur_dac_swing_on, coex_dm->cur_dac_swing_lvl);
+	seq_printf(m,
+		   "\n %-35s", "============[Sw mechanism]============");
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ",
+		   "SM1[ShRf/ LpRA/ LimDig]", coex_dm->cur_rf_rx_lpf_shrink,
+		   coex_dm->cur_low_penalty_ra, coex_dm->limited_dig);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d(0x%x) ",
+		   "SM2[AgcT/ AdcB/ SwDacSwing(lvl)]",
+		   coex_dm->cur_agc_table_en, coex_dm->cur_adc_back_off,
+		   coex_dm->cur_dac_swing_on, coex_dm->cur_dac_swing_lvl);
 
 	/* Fw mechanism	*/
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Fw mechanism]============");
+	seq_printf(m, "\n %-35s",
+		   "============[Fw mechanism]============");
 
 	ps_tdma_case = coex_dm->cur_ps_tdma;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %5ph case-%d (auto:%d)",
-		 "PS TDMA", coex_dm->ps_tdma_para,
-		 ps_tdma_case, coex_dm->auto_tdma_adjust);
+	seq_printf(m, "\n %-35s = %5ph case-%d (auto:%d)",
+		   "PS TDMA", coex_dm->ps_tdma_para,
+		   ps_tdma_case, coex_dm->auto_tdma_adjust);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d ",
-		 "DecBtPwr/ IgnWlanAct", coex_dm->cur_dec_bt_pwr_lvl,
-		 coex_dm->cur_ignore_wlan_act);
+	seq_printf(m, "\n %-35s = %d/ %d ",
+		   "DecBtPwr/ IgnWlanAct", coex_dm->cur_dec_bt_pwr_lvl,
+		   coex_dm->cur_ignore_wlan_act);
 
 	/* Hw setting */
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Hw setting]============");
+	seq_printf(m, "\n %-35s",
+		   "============[Hw setting]============");
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x",
-		 "RF-A, 0x1e initVal", coex_dm->bt_rf0x1e_backup);
+	seq_printf(m, "\n %-35s = 0x%x",
+		   "RF-A, 0x1e initVal", coex_dm->bt_rf0x1e_backup);
 
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x778);
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x880);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x778/0x880[29:25]", u8tmp[0],
-		 (u32tmp[0] & 0x3e000000) >> 25);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x778/0x880[29:25]", u8tmp[0],
+		   (u32tmp[0] & 0x3e000000) >> 25);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x948);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x67);
 	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x765);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x948/ 0x67[5] / 0x765",
-		 u32tmp[0], ((u8tmp[0] & 0x20) >> 5), u8tmp[1]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x948/ 0x67[5] / 0x765",
+		   u32tmp[0], ((u8tmp[0] & 0x20) >> 5), u8tmp[1]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x92c);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x930);
 	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x944);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x92c[1:0]/ 0x930[7:0]/0x944[1:0]",
-		 u32tmp[0] & 0x3, u32tmp[1] & 0xff, u32tmp[2] & 0x3);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x92c[1:0]/ 0x930[7:0]/0x944[1:0]",
+		   u32tmp[0] & 0x3, u32tmp[1] & 0xff, u32tmp[2] & 0x3);
 
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x39);
 	u8tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x40);
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x4c);
 	u8tmp[2] = btcoexist->btc_read_1byte(btcoexist, 0x64);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
-		 "0x38[11]/0x40/0x4c[24:23]/0x64[0]",
-		 ((u8tmp[0] & 0x8) >> 3), u8tmp[1],
-		 ((u32tmp[0] & 0x01800000) >> 23), u8tmp[2] & 0x1);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "0x38[11]/0x40/0x4c[24:23]/0x64[0]",
+		   ((u8tmp[0] & 0x8) >> 3), u8tmp[1],
+		   ((u32tmp[0] & 0x01800000) >> 23), u8tmp[2] & 0x1);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x550);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x522);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x550(bcn ctrl)/0x522", u32tmp[0], u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x550(bcn ctrl)/0x522", u32tmp[0], u8tmp[0]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xc50);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x49c);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0xc50(dig)/0x49c(null-drop)", u32tmp[0] & 0xff, u8tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0xc50(dig)/0x49c(null-drop)", u32tmp[0] & 0xff, u8tmp[0]);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xda0);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0xda4);
@@ -3981,29 +3969,27 @@ void ex_btc8723b2ant_display_coex_info(struct btc_coexist *btcoexist)
 		   (u32tmp[3] & 0xffff);
 	fa_cck = (u8tmp[0] << 8) + u8tmp[1];
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "OFDM-CCA/OFDM-FA/CCK-FA",
-		 u32tmp[0] & 0xffff, fa_ofdm, fa_cck);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "OFDM-CCA/OFDM-FA/CCK-FA",
+		   u32tmp[0] & 0xffff, fa_ofdm, fa_cck);
 
 	u32tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6c0);
 	u32tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x6c4);
 	u32tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x6c8);
 	u8tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x6cc);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
-		 "0x6c0/0x6c4/0x6c8/0x6cc(coexTable)",
-		 u32tmp[0], u32tmp[1], u32tmp[2], u8tmp[0]);
-
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "0x770(high-pri rx/tx)",
-		 coex_sta->high_priority_rx, coex_sta->high_priority_tx);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "0x774(low-pri rx/tx)", coex_sta->low_priority_rx,
-		 coex_sta->low_priority_tx);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "0x6c0/0x6c4/0x6c8/0x6cc(coexTable)",
+		   u32tmp[0], u32tmp[1], u32tmp[2], u8tmp[0]);
+
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "0x770(high-pri rx/tx)",
+		   coex_sta->high_priority_rx, coex_sta->high_priority_tx);
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "0x774(low-pri rx/tx)", coex_sta->low_priority_rx,
+		   coex_sta->low_priority_tx);
 	if (btcoexist->auto_report_2ant)
 		btc8723b2ant_monitor_bt_ctr(btcoexist);
-	btcoexist->btc_disp_dbg_msg(btcoexist,
-	BTC_DBG_DISP_COEX_STATISTICS);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS, m);
 }
 
 void ex_btc8723b2ant_ips_notify(struct btc_coexist *btcoexist, u8 type)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.h
index bc1e3042e271..aa24da402fb9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8723b2ant.h
@@ -195,7 +195,8 @@ void ex_btc8723b2ant_bt_info_notify(struct btc_coexist *btcoexist,
 				    u8 *tmpbuf, u8 length);
 void ex_btc8723b2ant_halt_notify(struct btc_coexist *btcoexist);
 void ex_btc8723b2ant_periodical(struct btc_coexist *btcoexist);
-void ex_btc8723b2ant_display_coex_info(struct btc_coexist *btcoexist);
+void ex_btc8723b2ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m);
 void ex_btc8723b2ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
 void ex_btc8723b2ant_pre_load_firmware(struct btc_coexist *btcoexist);
 void ex_btc8723b2ant_power_on_setting(struct btc_coexist *btcoexist);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c
index 4efac5fe9982..0b26419881c0 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.c
@@ -2172,12 +2172,12 @@ void ex_btc8821a1ant_init_coex_dm(struct btc_coexist *btcoexist)
 	btc8821a1ant_query_bt_info(btcoexist);
 }
 
-void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist)
+void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
 {
 	struct btc_board_info *board_info = &btcoexist->board_info;
 	struct btc_stack_info *stack_info = &btcoexist->stack_info;
 	struct btc_bt_link_info *bt_link_info = &btcoexist->bt_link_info;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	u8 u1_tmp[4], i, bt_info_ext, ps_tdma_case = 0;
 	u16 u2_tmp[4];
 	u32 u4_tmp[4];
@@ -2188,49 +2188,36 @@ void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist)
 	u8 wifi_dot11_chnl, wifi_hs_chnl;
 	u32 fw_ver = 0, bt_patch_ver = 0;
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n ============[BT Coexist info]============");
+	seq_puts(m, "\n ============[BT Coexist info]============");
 
 	if (btcoexist->manual_control) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ============[Under Manual Control]============");
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ==========================================");
+		seq_puts(m, "\n ============[Under Manual Control]============");
+		seq_puts(m, "\n ==========================================");
 	}
 	if (btcoexist->stop_coex_dm) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ============[Coex is STOPPED]============");
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n ==========================================");
-	}
-
-	if (!board_info->bt_exist) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n BT not exists !!!");
-		return;
+		seq_puts(m, "\n ============[Coex is STOPPED]============");
+		seq_puts(m, "\n ==========================================");
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d/ %d",
-		 "Ant PG Num/ Ant Mech/ Ant Pos:",
-		 board_info->pg_ant_num,
-		 board_info->btdm_ant_num,
-		 board_info->btdm_ant_pos);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d",
+		   "Ant PG Num/ Ant Mech/ Ant Pos:",
+		   board_info->pg_ant_num,
+		   board_info->btdm_ant_num,
+		   board_info->btdm_ant_pos);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %s / %d", "BT stack/ hci ext ver",
-		 ((stack_info->profile_notified) ? "Yes" : "No"),
-		 stack_info->hci_version);
+	seq_printf(m, "\n %-35s = %s / %d", "BT stack/ hci ext ver",
+		   ((stack_info->profile_notified) ? "Yes" : "No"),
+		   stack_info->hci_version);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_BT_PATCH_VER,
 			   &bt_patch_ver);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_FW_VER, &fw_ver);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d_%x/ 0x%x/ 0x%x(%d)",
-		 "CoexVer/ FwVer/ PatchVer",
-		 glcoex_ver_date_8821a_1ant,
-		 glcoex_ver_8821a_1ant,
-		 fw_ver, bt_patch_ver,
-		 bt_patch_ver);
+	seq_printf(m, "\n %-35s = %d_%x/ 0x%x/ 0x%x(%d)",
+		   "CoexVer/ FwVer/ PatchVer",
+		   glcoex_ver_date_8821a_1ant,
+		   glcoex_ver_8821a_1ant,
+		   fw_ver, bt_patch_ver,
+		   bt_patch_ver);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION,
 			   &bt_hs_on);
@@ -2238,28 +2225,24 @@ void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist)
 			   &wifi_dot11_chnl);
 	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_HS_CHNL,
 			   &wifi_hs_chnl);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d / %d(%d)",
-		 "Dot11 channel / HsChnl(HsMode)",
-		 wifi_dot11_chnl, wifi_hs_chnl, bt_hs_on);
+	seq_printf(m, "\n %-35s = %d / %d(%d)",
+		   "Dot11 channel / HsChnl(HsMode)",
+		   wifi_dot11_chnl, wifi_hs_chnl, bt_hs_on);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %3ph ",
-		 "H2C Wifi inform bt chnl Info",
-		 coex_dm->wifi_chnl_info);
+	seq_printf(m, "\n %-35s = %3ph ",
+		   "H2C Wifi inform bt chnl Info",
+		   coex_dm->wifi_chnl_info);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_HS_RSSI, &bt_hs_rssi);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d", "Wifi rssi/ HS rssi",
-		 (int)wifi_rssi, (int)bt_hs_rssi);
+	seq_printf(m, "\n %-35s = %d/ %d", "Wifi rssi/ HS rssi",
+		   (int)wifi_rssi, (int)bt_hs_rssi);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d/ %d ", "Wifi link/ roam/ scan",
-		 link, roam, scan);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ", "Wifi link/ roam/ scan",
+		   link, roam, scan);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G,
 			   &wifi_under_5g);
@@ -2269,16 +2252,15 @@ void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist)
 			   &wifi_busy);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_TRAFFIC_DIRECTION,
 			   &wifi_traffic_dir);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %s / %s/ %s ", "Wifi status",
-		 (wifi_under_5g ? "5G" : "2.4G"),
-		 ((wifi_bw == BTC_WIFI_BW_LEGACY) ? "Legacy" :
-		 (((wifi_bw == BTC_WIFI_BW_HT40) ? "HT40" : "HT20"))),
-		 ((!wifi_busy) ? "idle" :
-		 ((wifi_traffic_dir == BTC_WIFI_TRAFFIC_TX) ?
-		 "uplink" : "downlink")));
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		   "\r\n %-35s = [%s/ %d/ %d] ", "BT [status/ rssi/ retryCnt]",
+	seq_printf(m, "\n %-35s = %s / %s/ %s ", "Wifi status",
+		   (wifi_under_5g ? "5G" : "2.4G"),
+		   ((wifi_bw == BTC_WIFI_BW_LEGACY) ? "Legacy" :
+		   (((wifi_bw == BTC_WIFI_BW_HT40) ? "HT40" : "HT20"))),
+		   ((!wifi_busy) ? "idle" :
+		   ((wifi_traffic_dir == BTC_WIFI_TRAFFIC_TX) ?
+		   "uplink" : "downlink")));
+	seq_printf(m, "\n %-35s = [%s/ %d/ %d] ",
+		   "BT [status/ rssi/ retryCnt]",
 		   ((coex_sta->bt_disabled) ? ("disabled") :
 		   ((coex_sta->c2h_bt_inquiry_page) ? ("inquiry/page scan") :
 		   ((BT_8821A_1ANT_BT_STATUS_NON_CONNECTED_IDLE ==
@@ -2289,166 +2271,143 @@ void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist)
 		   "connected-idle" : "busy")))),
 		   coex_sta->bt_rssi, coex_sta->bt_retry_cnt);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d / %d / %d / %d", "SCO/HID/PAN/A2DP",
-		 bt_link_info->sco_exist,
-		 bt_link_info->hid_exist,
-		 bt_link_info->pan_exist,
-		 bt_link_info->a2dp_exist);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO);
+	seq_printf(m, "\n %-35s = %d / %d / %d / %d", "SCO/HID/PAN/A2DP",
+		   bt_link_info->sco_exist,
+		   bt_link_info->hid_exist,
+		   bt_link_info->pan_exist,
+		   bt_link_info->a2dp_exist);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_BT_LINK_INFO, m);
 
 	bt_info_ext = coex_sta->bt_info_ext;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %s",
-		 "BT Info A2DP rate",
-		 (bt_info_ext & BIT0) ?
-		 "Basic rate" : "EDR rate");
+	seq_printf(m, "\n %-35s = %s",
+		   "BT Info A2DP rate",
+		   (bt_info_ext & BIT0) ?
+		   "Basic rate" : "EDR rate");
 
 	for (i = 0; i < BT_INFO_SRC_8821A_1ANT_MAX; i++) {
 		if (coex_sta->bt_info_c2h_cnt[i]) {
-			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-				 "\r\n %-35s = %7ph(%d)",
-				 glbt_info_src_8821a_1ant[i],
-				 coex_sta->bt_info_c2h[i],
-				 coex_sta->bt_info_c2h_cnt[i]);
+			seq_printf(m, "\n %-35s = %7ph(%d)",
+				   glbt_info_src_8821a_1ant[i],
+				   coex_sta->bt_info_c2h[i],
+				   coex_sta->bt_info_c2h_cnt[i]);
 		}
 	}
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %s/%s, (0x%x/0x%x)",
-		 "PS state, IPS/LPS, (lps/rpwm)",
-		 ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
-		 ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")),
-		 btcoexist->bt_info.lps_val,
-		 btcoexist->bt_info.rpwm_val);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD);
+	seq_printf(m, "\n %-35s = %s/%s, (0x%x/0x%x)",
+		   "PS state, IPS/LPS, (lps/rpwm)",
+		   ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
+		   ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")),
+		   btcoexist->bt_info.lps_val,
+		   btcoexist->bt_info.rpwm_val);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD, m);
 
 	if (!btcoexist->manual_control) {
 		/* Sw mechanism*/
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s",
-			 "============[Sw mechanism]============");
-
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %d", "SM[LowPenaltyRA]",
-			 coex_dm->cur_low_penalty_ra);
-
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %s/ %s/ %d ",
-			 "DelBA/ BtCtrlAgg/ AggSize",
-			 (btcoexist->bt_info.reject_agg_pkt ? "Yes" : "No"),
-			 (btcoexist->bt_info.bt_ctrl_buf_size ? "Yes" : "No"),
-			 btcoexist->bt_info.agg_buf_size);
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = 0x%x ", "Rate Mask",
-			 btcoexist->bt_info.ra_mask);
+		seq_printf(m, "\n %-35s",
+			   "============[Sw mechanism]============");
+
+		seq_printf(m, "\n %-35s = %d", "SM[LowPenaltyRA]",
+			   coex_dm->cur_low_penalty_ra);
+
+		seq_printf(m, "\n %-35s = %s/ %s/ %d ",
+			   "DelBA/ BtCtrlAgg/ AggSize",
+			   (btcoexist->bt_info.reject_agg_pkt ? "Yes" : "No"),
+			   (btcoexist->bt_info.bt_ctrl_buf_size ? "Yes" : "No"),
+			   btcoexist->bt_info.agg_buf_size);
+		seq_printf(m, "\n %-35s = 0x%x ", "Rate Mask",
+			   btcoexist->bt_info.ra_mask);
 
 		/* Fw mechanism */
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-			 "============[Fw mechanism]============");
+		seq_printf(m, "\n %-35s",
+			   "============[Fw mechanism]============");
 
 		ps_tdma_case = coex_dm->cur_ps_tdma;
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %5ph case-%d (auto:%d)",
-			 "PS TDMA",
-			 coex_dm->ps_tdma_para,
-			 ps_tdma_case,
-			 coex_dm->auto_tdma_adjust);
-
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = 0x%x ",
-			 "Latest error condition(should be 0)",
+		seq_printf(m, "\n %-35s = %5ph case-%d (auto:%d)",
+			   "PS TDMA",
+			   coex_dm->ps_tdma_para,
+			   ps_tdma_case,
+			   coex_dm->auto_tdma_adjust);
+
+		seq_printf(m, "\n %-35s = 0x%x ",
+			   "Latest error condition(should be 0)",
 			   coex_dm->error_condition);
 
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %d ", "IgnWlanAct",
-			 coex_dm->cur_ignore_wlan_act);
+		seq_printf(m, "\n %-35s = %d ", "IgnWlanAct",
+			   coex_dm->cur_ignore_wlan_act);
 	}
 
 	/* Hw setting */
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s", "============[Hw setting]============");
+	seq_printf(m, "\n %-35s", "============[Hw setting]============");
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/0x%x/0x%x/0x%x",
-		 "backup ARFR1/ARFR2/RL/AMaxTime",
-		 coex_dm->backup_arfr_cnt1,
-		 coex_dm->backup_arfr_cnt2,
-		 coex_dm->backup_retry_limit,
-		 coex_dm->backup_ampdu_max_time);
+	seq_printf(m, "\n %-35s = 0x%x/0x%x/0x%x/0x%x",
+		   "backup ARFR1/ARFR2/RL/AMaxTime",
+		   coex_dm->backup_arfr_cnt1,
+		   coex_dm->backup_arfr_cnt2,
+		   coex_dm->backup_retry_limit,
+		   coex_dm->backup_ampdu_max_time);
 
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x430);
 	u4_tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x434);
 	u2_tmp[0] = btcoexist->btc_read_2byte(btcoexist, 0x42a);
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x456);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/0x%x/0x%x/0x%x",
-		 "0x430/0x434/0x42a/0x456",
-		 u4_tmp[0], u4_tmp[1], u2_tmp[0], u1_tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/0x%x/0x%x/0x%x",
+		   "0x430/0x434/0x42a/0x456",
+		   u4_tmp[0], u4_tmp[1], u2_tmp[0], u1_tmp[0]);
 
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x778);
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xc58);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x", "0x778/ 0xc58[29:25]",
-		 u1_tmp[0], (u4_tmp[0] & 0x3e000000) >> 25);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x", "0x778/ 0xc58[29:25]",
+		   u1_tmp[0], (u4_tmp[0] & 0x3e000000) >> 25);
 
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x8db);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x", "0x8db[6:5]",
-		 ((u1_tmp[0] & 0x60) >> 5));
+	seq_printf(m, "\n %-35s = 0x%x", "0x8db[6:5]",
+		   ((u1_tmp[0] & 0x60) >> 5));
 
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x975);
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0xcb4[29:28]/0xcb4[7:0]/0x974[9:8]",
-		 (u4_tmp[0] & 0x30000000) >> 28,
-		  u4_tmp[0] & 0xff,
-		  u1_tmp[0] & 0x3);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0xcb4[29:28]/0xcb4[7:0]/0x974[9:8]",
+		   (u4_tmp[0] & 0x30000000) >> 28,
+		    u4_tmp[0] & 0xff,
+		    u1_tmp[0] & 0x3);
 
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x40);
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x4c);
 	u1_tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x64);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x40/0x4c[24:23]/0x64[0]",
-		 u1_tmp[0], ((u4_tmp[0] & 0x01800000) >> 23), u1_tmp[1] & 0x1);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x40/0x4c[24:23]/0x64[0]",
+		   u1_tmp[0], ((u4_tmp[0] & 0x01800000) >> 23),
+		   u1_tmp[1] & 0x1);
 
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x550);
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x522);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x", "0x550(bcn ctrl)/0x522",
-		 u4_tmp[0], u1_tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x", "0x550(bcn ctrl)/0x522",
+		   u4_tmp[0], u1_tmp[0]);
 
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xc50);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x", "0xc50(dig)",
-		 u4_tmp[0] & 0xff);
+	seq_printf(m, "\n %-35s = 0x%x", "0xc50(dig)",
+		   u4_tmp[0] & 0xff);
 
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xf48);
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0xa5d);
 	u1_tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0xa5c);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x", "OFDM-FA/ CCK-FA",
-		 u4_tmp[0], (u1_tmp[0] << 8) + u1_tmp[1]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x", "OFDM-FA/ CCK-FA",
+		   u4_tmp[0], (u1_tmp[0] << 8) + u1_tmp[1]);
 
 	u4_tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6c0);
 	u4_tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x6c4);
 	u4_tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x6c8);
 	u1_tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x6cc);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
-		 "0x6c0/0x6c4/0x6c8/0x6cc(coexTable)",
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x/ 0x%x",
+		   "0x6c0/0x6c4/0x6c8/0x6cc(coexTable)",
 		   u4_tmp[0], u4_tmp[1], u4_tmp[2], u1_tmp[0]);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d", "0x770(high-pri rx/tx)",
-		 coex_sta->high_priority_rx, coex_sta->high_priority_tx);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d", "0x774(low-pri rx/tx)",
-		 coex_sta->low_priority_rx, coex_sta->low_priority_tx);
+	seq_printf(m, "\n %-35s = %d/ %d", "0x770(high-pri rx/tx)",
+		   coex_sta->high_priority_rx, coex_sta->high_priority_tx);
+	seq_printf(m, "\n %-35s = %d/ %d", "0x774(low-pri rx/tx)",
+		   coex_sta->low_priority_rx, coex_sta->low_priority_tx);
 	if (btcoexist->auto_report_1ant)
 		btc8821a1ant_monitor_bt_ctr(btcoexist);
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS, m);
 }
 
 void ex_btc8821a1ant_ips_notify(struct btc_coexist *btcoexist, u8 type)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.h
index b0a6626fbb66..a498ff5b1b9c 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a1ant.h
@@ -186,7 +186,8 @@ void ex_btc8821a1ant_bt_info_notify(struct btc_coexist *btcoexist,
 void ex_btc8821a1ant_halt_notify(struct btc_coexist *btcoexist);
 void ex_btc8821a1ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnpstate);
 void ex_btc8821a1ant_periodical(struct btc_coexist *btcoexist);
-void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist);
+void ex_btc8821a1ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m);
 void ex_btc8821a1ant_dbg_control(struct btc_coexist *btcoexist, u8 op_code,
 				 u8 op_len, u8 *data);
 void ex_btc8821a1ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c
index 41943c34edff..d5f282cb9d24 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.c
@@ -3657,11 +3657,11 @@ void ex_btc8821a2ant_init_coex_dm(struct btc_coexist *btcoexist)
 	btc8821a2ant_init_coex_dm(btcoexist);
 }
 
-void ex_btc8821a2ant_display_coex_info(struct btc_coexist *btcoexist)
+void ex_btc8821a2ant_display_coex_info(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
 {
 	struct btc_board_info *board_info = &btcoexist->board_info;
 	struct btc_stack_info *stack_info = &btcoexist->stack_info;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	u8 u1tmp[4], i, bt_info_ext, ps_tdma_case = 0;
 	u32 u4tmp[4];
 	bool roam = false, scan = false, link = false, wifi_under_5g = false;
@@ -3671,32 +3671,22 @@ void ex_btc8821a2ant_display_coex_info(struct btc_coexist *btcoexist)
 	u8 wifi_dot_11_chnl, wifi_hs_chnl;
 	u32 fw_ver = 0, bt_patch_ver = 0;
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n ============[BT Coexist info]============");
-
-	if (!board_info->bt_exist) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n BT not exists !!!");
-		return;
-	}
+	seq_puts(m, "\n ============[BT Coexist info]============");
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d ", "Ant PG number/ Ant mechanism:",
-		 board_info->pg_ant_num, board_info->btdm_ant_num);
+	seq_printf(m, "\n %-35s = %d/ %d ", "Ant PG number/ Ant mechanism:",
+		   board_info->pg_ant_num, board_info->btdm_ant_num);
 
 	if (btcoexist->manual_control) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s", "[Action Manual control]!!");
+		seq_printf(m, "\n %-35s", "[Action Manual control]!!");
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %s / %d", "BT stack/ hci ext ver",
+	seq_printf(m, "\n %-35s = %s / %d", "BT stack/ hci ext ver",
 		   ((stack_info->profile_notified) ? "Yes" : "No"),
 		   stack_info->hci_version);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_BT_PATCH_VER, &bt_patch_ver);
 	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_FW_VER, &fw_ver);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d_%d/ 0x%x/ 0x%x(%d)",
+	seq_printf(m, "\n %-35s = %d_%d/ 0x%x/ 0x%x(%d)",
 		   "CoexVer/ FwVer/ PatchVer",
 		   glcoex_ver_date_8821a_2ant, glcoex_ver_8821a_2ant,
 		   fw_ver, bt_patch_ver, bt_patch_ver);
@@ -3707,27 +3697,23 @@ void ex_btc8821a2ant_display_coex_info(struct btc_coexist *btcoexist)
 		BTC_GET_U1_WIFI_DOT11_CHNL, &wifi_dot_11_chnl);
 	btcoexist->btc_get(btcoexist,
 		BTC_GET_U1_WIFI_HS_CHNL, &wifi_hs_chnl);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d / %d(%d)",
+	seq_printf(m, "\n %-35s = %d / %d(%d)",
 		   "Dot11 channel / HsMode(HsChnl)",
 		   wifi_dot_11_chnl, bt_hs_on, wifi_hs_chnl);
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %3ph ",
+	seq_printf(m, "\n %-35s = %3ph ",
 		   "H2C Wifi inform bt chnl Info",
 		   coex_dm->wifi_chnl_info);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
 	btcoexist->btc_get(btcoexist, BTC_GET_S4_HS_RSSI, &bt_hs_rssi);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %ld/ %ld", "Wifi rssi/ HS rssi",
+	seq_printf(m, "\n %-35s = %ld/ %ld", "Wifi rssi/ HS rssi",
 		   wifi_rssi, bt_hs_rssi);
 
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
 	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d/ %d ", "Wifi link/ roam/ scan",
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ", "Wifi link/ roam/ scan",
 		   link, roam, scan);
 
 	btcoexist->btc_get(btcoexist,
@@ -3738,8 +3724,7 @@ void ex_btc8821a2ant_display_coex_info(struct btc_coexist *btcoexist)
 		BTC_GET_BL_WIFI_BUSY, &wifi_busy);
 	btcoexist->btc_get(btcoexist,
 		BTC_GET_U4_WIFI_TRAFFIC_DIRECTION, &wifi_traffic_dir);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %s / %s/ %s ", "Wifi status",
+	seq_printf(m, "\n %-35s = %s / %s/ %s ", "Wifi status",
 		   (wifi_under_5g ? "5G" : "2.4G"),
 		   ((BTC_WIFI_BW_LEGACY == wifi_bw) ? "Legacy" :
 		    (((BTC_WIFI_BW_HT40 == wifi_bw) ? "HT40" : "HT20"))),
@@ -3748,134 +3733,128 @@ void ex_btc8821a2ant_display_coex_info(struct btc_coexist *btcoexist)
 		     "uplink" : "downlink")));
 
 	if (stack_info->profile_notified) {
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %d / %d / %d / %d", "SCO/HID/PAN/A2DP",
+		seq_printf(m, "\n %-35s = %d / %d / %d / %d",
+			   "SCO/HID/PAN/A2DP",
 			   stack_info->sco_exist, stack_info->hid_exist,
 			   stack_info->pan_exist, stack_info->a2dp_exist);
 
 		btcoexist->btc_disp_dbg_msg(btcoexist,
-					    BTC_DBG_DISP_BT_LINK_INFO);
+					    BTC_DBG_DISP_BT_LINK_INFO,
+					    m);
 	}
 
 	bt_info_ext = coex_sta->bt_info_ext;
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s",
-		 "BT Info A2DP rate",
+	seq_printf(m, "\n %-35s = %s", "BT Info A2DP rate",
 		   (bt_info_ext&BIT0) ? "Basic rate" : "EDR rate");
 
 	for (i = 0; i < BT_INFO_SRC_8821A_2ANT_MAX; i++) {
 		if (coex_sta->bt_info_c2h_cnt[i]) {
-			RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-				 "\r\n %-35s = %7ph(%d)",
-				 glbt_info_src_8821a_2ant[i],
-				 coex_sta->bt_info_c2h[i],
-				 coex_sta->bt_info_c2h_cnt[i]);
+			seq_printf(m, "\n %-35s = %7ph(%d)",
+				   glbt_info_src_8821a_2ant[i],
+				   coex_sta->bt_info_c2h[i],
+				   coex_sta->bt_info_c2h_cnt[i]);
 		}
 	}
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %s/%s",
-		 "PS state, IPS/LPS",
-		 ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
-		 ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")));
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD);
+	seq_printf(m, "\n %-35s = %s/%s",
+		   "PS state, IPS/LPS",
+		   ((coex_sta->under_ips ? "IPS ON" : "IPS OFF")),
+		   ((coex_sta->under_lps ? "LPS ON" : "LPS OFF")));
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_FW_PWR_MODE_CMD, m);
 
 	/* Sw mechanism*/
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Sw mechanism]============");
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = %d/ %d/ %d(0x%x) ",
-		 "SM2[AgcT/ AdcB/ SwDacSwing(lvl)]",
-		 coex_dm->cur_agc_table_en, coex_dm->cur_adc_back_off,
-		 coex_dm->cur_dac_swing_on, coex_dm->cur_dac_swing_lvl);
+	seq_printf(m, "\n %-35s",
+		   "============[Sw mechanism]============");
+	seq_printf(m, "\n %-35s = %d/ %d/ %d(0x%x) ",
+		   "SM2[AgcT/ AdcB/ SwDacSwing(lvl)]",
+		   coex_dm->cur_agc_table_en, coex_dm->cur_adc_back_off,
+		   coex_dm->cur_dac_swing_on, coex_dm->cur_dac_swing_lvl);
 
 	/* Fw mechanism*/
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s",
-		 "============[Fw mechanism]============");
+	seq_printf(m, "\n %-35s",
+		   "============[Fw mechanism]============");
 
 	if (!btcoexist->manual_control) {
 		ps_tdma_case = coex_dm->cur_ps_tdma;
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %5ph case-%d",
-			 "PS TDMA",
-			 coex_dm->ps_tdma_para, ps_tdma_case);
-
-		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-			 "\r\n %-35s = %d/ %d ", "DecBtPwr/ IgnWlanAct",
-			 coex_dm->cur_dec_bt_pwr_lvl,
-			 coex_dm->cur_ignore_wlan_act);
+		seq_printf(m, "\n %-35s = %5ph case-%d",
+			   "PS TDMA",
+			   coex_dm->ps_tdma_para, ps_tdma_case);
+
+		seq_printf(m, "\n %-35s = %d/ %d ", "DecBtPwr/ IgnWlanAct",
+			   coex_dm->cur_dec_bt_pwr_lvl,
+			   coex_dm->cur_ignore_wlan_act);
 	}
 
 	/* Hw setting*/
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s", "============[Hw setting]============");
+	seq_printf(m, "\n %-35s", "============[Hw setting]============");
 
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
-		 "\r\n %-35s = 0x%x", "RF-A, 0x1e initVal",
-		 coex_dm->bt_rf0x1e_backup);
+	seq_printf(m, "\n %-35s = 0x%x", "RF-A, 0x1e initVal",
+		   coex_dm->bt_rf0x1e_backup);
 
 	u1tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x778);
 	u1tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0x6cc);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x ",
-		 "0x778 (W_Act)/ 0x6cc (CoTab Sel)",
-		 u1tmp[0], u1tmp[1]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x ",
+		   "0x778 (W_Act)/ 0x6cc (CoTab Sel)",
+		   u1tmp[0], u1tmp[1]);
 
 	u1tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x8db);
 	u1tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0xc5b);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x8db(ADC)/0xc5b[29:25](DAC)",
-		 ((u1tmp[0] & 0x60) >> 5), ((u1tmp[1] & 0x3e) >> 1));
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x8db(ADC)/0xc5b[29:25](DAC)",
+		   ((u1tmp[0] & 0x60) >> 5), ((u1tmp[1] & 0x3e) >> 1));
 
 	u4tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xcb4);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0xcb4[7:0](ctrl)/ 0xcb4[29:28](val)",
-		 u4tmp[0] & 0xff, ((u4tmp[0] & 0x30000000) >> 28));
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0xcb4[7:0](ctrl)/ 0xcb4[29:28](val)",
+		   u4tmp[0] & 0xff, ((u4tmp[0] & 0x30000000) >> 28));
 
 	u1tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x40);
 	u4tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x4c);
 	u4tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x974);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x40/ 0x4c[24:23]/ 0x974",
-		 u1tmp[0], ((u4tmp[0] & 0x01800000) >> 23), u4tmp[1]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x40/ 0x4c[24:23]/ 0x974",
+		   u1tmp[0], ((u4tmp[0] & 0x01800000) >> 23), u4tmp[1]);
 
 	u4tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x550);
 	u1tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x522);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0x550(bcn ctrl)/0x522",
-		 u4tmp[0], u1tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0x550(bcn ctrl)/0x522",
+		   u4tmp[0], u1tmp[0]);
 
 	u4tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xc50);
 	u1tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0xa0a);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "0xc50(DIG)/0xa0a(CCK-TH)",
-		 u4tmp[0], u1tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "0xc50(DIG)/0xa0a(CCK-TH)",
+		   u4tmp[0], u1tmp[0]);
 
 	u4tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0xf48);
 	u1tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0xa5b);
 	u1tmp[1] = btcoexist->btc_read_1byte(btcoexist, 0xa5c);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x",
-		 "OFDM-FA/ CCK-FA",
-		 u4tmp[0], (u1tmp[0] << 8) + u1tmp[1]);
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x",
+		   "OFDM-FA/ CCK-FA",
+		   u4tmp[0], (u1tmp[0] << 8) + u1tmp[1]);
 
 	u4tmp[0] = btcoexist->btc_read_4byte(btcoexist, 0x6c0);
 	u4tmp[1] = btcoexist->btc_read_4byte(btcoexist, 0x6c4);
 	u4tmp[2] = btcoexist->btc_read_4byte(btcoexist, 0x6c8);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x/ 0x%x/ 0x%x",
-		 "0x6c0/0x6c4/0x6c8",
-		 u4tmp[0], u4tmp[1], u4tmp[2]);
-
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
-		 "0x770 (hi-pri Rx/Tx)",
-		 coex_sta->high_priority_rx, coex_sta->high_priority_tx);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = %d/ %d",
+	seq_printf(m, "\n %-35s = 0x%x/ 0x%x/ 0x%x",
+		   "0x6c0/0x6c4/0x6c8",
+		   u4tmp[0], u4tmp[1], u4tmp[2]);
+
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "0x770 (hi-pri Rx/Tx)",
+		   coex_sta->high_priority_rx, coex_sta->high_priority_tx);
+	seq_printf(m, "\n %-35s = %d/ %d",
 		   "0x774(low-pri Rx/Tx)",
 		   coex_sta->low_priority_rx, coex_sta->low_priority_tx);
 
 	/* Tx mgnt queue hang or not, 0x41b should = 0xf, ex: 0xd ==>hang*/
 	u1tmp[0] = btcoexist->btc_read_1byte(btcoexist, 0x41b);
-	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG, "\r\n %-35s = 0x%x",
-		 "0x41b (mgntQ hang chk == 0xf)",
-		 u1tmp[0]);
+	seq_printf(m, "\n %-35s = 0x%x",
+		   "0x41b (mgntQ hang chk == 0xf)",
+		   u1tmp[0]);
 
-	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS);
+	btcoexist->btc_disp_dbg_msg(btcoexist, BTC_DBG_DISP_COEX_STATISTICS, m);
 }
 
 void ex_btc8821a2ant_ips_notify(struct btc_coexist *btcoexist, u8 type)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.h
index a839d5574422..ce3e58c4e660 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtc8821a2ant.h
@@ -226,7 +226,8 @@ ex_btc8821a2ant_periodical(
 	);
 void
 ex_btc8821a2ant_display_coex_info(
-	struct btc_coexist *btcoexist
+	struct btc_coexist *btcoexist,
+	struct seq_file *m
 	);
 void ex_btc8821a2ant_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
 void ex_btc8821a2ant_pre_load_firmware(struct btc_coexist *btcoexist);
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c
index b5e9877d935c..1404729441a2 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.c
@@ -25,19 +25,11 @@
 
 #include "halbt_precomp.h"
 
-/***********************************************
- *		Global variables
- ***********************************************/
-
-struct btc_coexist gl_bt_coexist;
-
-u32 btc_dbg_type[BTC_MSG_MAX];
-
 /***************************************************
  *		Debug related function
  ***************************************************/
 
-const char *const gl_btc_wifi_bw_string[] = {
+static const char *const gl_btc_wifi_bw_string[] = {
 	"11bg",
 	"HT20",
 	"HT40",
@@ -45,7 +37,7 @@ const char *const gl_btc_wifi_bw_string[] = {
 	"HT160"
 };
 
-const char *const gl_btc_wifi_freq_string[] = {
+static const char *const gl_btc_wifi_freq_string[] = {
 	"2.4G",
 	"5G"
 };
@@ -103,21 +95,6 @@ static bool is_any_client_connect_to_ap(struct btc_coexist *btcoexist)
 		return false;
 }
 
-static bool halbtc_is_bt40(struct rtl_priv *adapter)
-{
-	struct rtl_priv *rtlpriv = adapter;
-	struct rtl_phy *rtlphy = &(rtlpriv->phy);
-	bool is_ht40 = true;
-	enum ht_channel_width bw = rtlphy->current_chan_bw;
-
-	if (bw == HT_CHANNEL_WIDTH_20)
-		is_ht40 = false;
-	else if (bw == HT_CHANNEL_WIDTH_20_40)
-		is_ht40 = true;
-
-	return is_ht40;
-}
-
 static bool halbtc_legacy(struct rtl_priv *adapter)
 {
 	struct rtl_priv *rtlpriv = adapter;
@@ -143,18 +120,26 @@ bool halbtc_is_wifi_uplink(struct rtl_priv *adapter)
 
 static u32 halbtc_get_wifi_bw(struct btc_coexist *btcoexist)
 {
-	struct rtl_priv *rtlpriv =
-		(struct rtl_priv *)btcoexist->adapter;
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	struct rtl_phy *rtlphy = &rtlpriv->phy;
 	u32 wifi_bw = BTC_WIFI_BW_HT20;
 
-	if (halbtc_is_bt40(rtlpriv)) {
-		wifi_bw = BTC_WIFI_BW_HT40;
+	if (halbtc_legacy(rtlpriv)) {
+		wifi_bw = BTC_WIFI_BW_LEGACY;
 	} else {
-		if (halbtc_legacy(rtlpriv))
-			wifi_bw = BTC_WIFI_BW_LEGACY;
-		else
+		switch (rtlphy->current_chan_bw) {
+		case HT_CHANNEL_WIDTH_20:
 			wifi_bw = BTC_WIFI_BW_HT20;
+			break;
+		case HT_CHANNEL_WIDTH_20_40:
+			wifi_bw = BTC_WIFI_BW_HT40;
+			break;
+		case HT_CHANNEL_WIDTH_80:
+			wifi_bw = BTC_WIFI_BW_HT80;
+			break;
+		}
 	}
+
 	return wifi_bw;
 }
 
@@ -171,7 +156,7 @@ static u8 halbtc_get_wifi_central_chnl(struct btc_coexist *btcoexist)
 	return chnl;
 }
 
-u8 rtl_get_hwpg_single_ant_path(struct rtl_priv *rtlpriv)
+static u8 rtl_get_hwpg_single_ant_path(struct rtl_priv *rtlpriv)
 {
 	struct rtl_mod_params *mod_params = rtlpriv->cfg->mod_params;
 
@@ -186,12 +171,12 @@ u8 rtl_get_hwpg_single_ant_path(struct rtl_priv *rtlpriv)
 	return rtlpriv->btcoexist.btc_info.single_ant_path;
 }
 
-u8 rtl_get_hwpg_bt_type(struct rtl_priv *rtlpriv)
+static u8 rtl_get_hwpg_bt_type(struct rtl_priv *rtlpriv)
 {
 	return rtlpriv->btcoexist.btc_info.bt_type;
 }
 
-u8 rtl_get_hwpg_ant_num(struct rtl_priv *rtlpriv)
+static u8 rtl_get_hwpg_ant_num(struct rtl_priv *rtlpriv)
 {
 	struct rtl_mod_params *mod_params = rtlpriv->cfg->mod_params;
 	u8 num;
@@ -208,13 +193,117 @@ u8 rtl_get_hwpg_ant_num(struct rtl_priv *rtlpriv)
 	return num;
 }
 
-u8 rtl_get_hwpg_package_type(struct rtl_priv *rtlpriv)
+static u8 rtl_get_hwpg_package_type(struct rtl_priv *rtlpriv)
 {
 	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
 
 	return rtlhal->package_type;
 }
 
+static
+u8 rtl_get_hwpg_rfe_type(struct rtl_priv *rtlpriv)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+
+	return rtlhal->rfe_type;
+}
+
+static
+bool halbtc_is_hw_mailbox_exist(struct btc_coexist *btcoexist)
+{
+	if (IS_HARDWARE_TYPE_8812(btcoexist->adapter))
+		return false;
+	else
+		return true;
+}
+
+static
+bool halbtc_send_bt_mp_operation(struct btc_coexist *btcoexist, u8 op_code,
+				 u8 *cmd, u32 len, unsigned long wait_ms)
+{
+	struct rtl_priv *rtlpriv;
+	const u8 oper_ver = 0;
+	u8 req_num;
+
+	if (!halbtc_is_hw_mailbox_exist(btcoexist))
+		return false;
+
+	if (wait_ms)	/* before h2c to avoid race condition */
+		reinit_completion(&btcoexist->bt_mp_comp);
+
+	rtlpriv = btcoexist->adapter;
+
+	/* fill req_num by op_code, and rtl_btc_btmpinfo_notify() use it
+	 * to know message type
+	 */
+	switch (op_code) {
+	case BT_OP_GET_BT_VERSION:
+		req_num = BT_SEQ_GET_BT_VERSION;
+		break;
+	case BT_OP_GET_AFH_MAP_L:
+		req_num = BT_SEQ_GET_AFH_MAP_L;
+		break;
+	case BT_OP_GET_AFH_MAP_M:
+		req_num = BT_SEQ_GET_AFH_MAP_M;
+		break;
+	case BT_OP_GET_AFH_MAP_H:
+		req_num = BT_SEQ_GET_AFH_MAP_H;
+		break;
+	case BT_OP_GET_BT_COEX_SUPPORTED_FEATURE:
+		req_num = BT_SEQ_GET_BT_COEX_SUPPORTED_FEATURE;
+		break;
+	case BT_OP_GET_BT_COEX_SUPPORTED_VERSION:
+		req_num = BT_SEQ_GET_BT_COEX_SUPPORTED_VERSION;
+		break;
+	case BT_OP_GET_BT_ANT_DET_VAL:
+		req_num = BT_SEQ_GET_BT_ANT_DET_VAL;
+		break;
+	case BT_OP_GET_BT_BLE_SCAN_PARA:
+		req_num = BT_SEQ_GET_BT_BLE_SCAN_PARA;
+		break;
+	case BT_OP_GET_BT_BLE_SCAN_TYPE:
+		req_num = BT_SEQ_GET_BT_BLE_SCAN_TYPE;
+		break;
+	case BT_OP_GET_BT_DEVICE_INFO:
+		req_num = BT_SEQ_GET_BT_DEVICE_INFO;
+		break;
+	case BT_OP_GET_BT_FORBIDDEN_SLOT_VAL:
+		req_num = BT_SEQ_GET_BT_FORB_SLOT_VAL;
+		break;
+	case BT_OP_WRITE_REG_ADDR:
+	case BT_OP_WRITE_REG_VALUE:
+	case BT_OP_READ_REG:
+	default:
+		req_num = BT_SEQ_DONT_CARE;
+		break;
+	}
+
+	cmd[0] |= (oper_ver & 0x0f);		/* Set OperVer */
+	cmd[0] |= ((req_num << 4) & 0xf0);	/* Set ReqNum */
+	cmd[1] = op_code;
+	rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, 0x67, len, cmd);
+
+	/* wait? */
+	if (!wait_ms)
+		return true;
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "btmpinfo wait req_num=%d wait=%ld\n", req_num, wait_ms);
+
+	if (in_interrupt())
+		return false;
+
+	if (wait_for_completion_timeout(&btcoexist->bt_mp_comp,
+					msecs_to_jiffies(wait_ms)) == 0) {
+		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_DMESG,
+			 "btmpinfo wait (req_num=%d) timeout\n", req_num);
+
+		return false;	/* timeout */
+	}
+
+	return true;
+}
+
 static void halbtc_leave_lps(struct btc_coexist *btcoexist)
 {
 	struct rtl_priv *rtlpriv;
@@ -342,25 +431,80 @@ static void halbtc_aggregation_check(struct btc_coexist *btcoexist)
 
 static u32 halbtc_get_bt_patch_version(struct btc_coexist *btcoexist)
 {
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	u8 cmd_buffer[4] = {0};
-	u8 oper_ver = 0;
-	u8 req_num = 0x0E;
 
 	if (btcoexist->bt_info.bt_real_fw_ver)
 		goto label_done;
 
-	cmd_buffer[0] |= (oper_ver & 0x0f);	/* Set OperVer */
-	cmd_buffer[0] |= ((req_num << 4) & 0xf0);	/* Set ReqNum */
-	cmd_buffer[1] = 0; /* BT_OP_GET_BT_VERSION = 0 */
-	rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, 0x67, 4,
-					&cmd_buffer[0]);
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_VERSION,
+				    cmd_buffer, 4, 200);
 
 label_done:
 	return btcoexist->bt_info.bt_real_fw_ver;
 }
 
-u32 halbtc_get_wifi_link_status(struct btc_coexist *btcoexist)
+static u32 halbtc_get_bt_coex_supported_feature(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	if (btcoexist->bt_info.bt_supported_feature)
+		goto label_done;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist,
+				    BT_OP_GET_BT_COEX_SUPPORTED_FEATURE,
+				    cmd_buffer, 4, 200);
+
+label_done:
+	return btcoexist->bt_info.bt_supported_feature;
+}
+
+static u32 halbtc_get_bt_coex_supported_version(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	if (btcoexist->bt_info.bt_supported_version)
+		goto label_done;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist,
+				    BT_OP_GET_BT_COEX_SUPPORTED_VERSION,
+				    cmd_buffer, 4, 200);
+
+label_done:
+	return btcoexist->bt_info.bt_supported_version;
+}
+
+static u32 halbtc_get_bt_device_info(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist,
+				    BT_OP_GET_BT_DEVICE_INFO,
+				    cmd_buffer, 4, 200);
+
+	return btcoexist->bt_info.bt_device_info;
+}
+
+static u32 halbtc_get_bt_forbidden_slot_val(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist,
+				    BT_OP_GET_BT_FORBIDDEN_SLOT_VAL,
+				    cmd_buffer, 4, 200);
+
+	return btcoexist->bt_info.bt_forb_slot_val;
+}
+
+static u32 halbtc_get_wifi_link_status(struct btc_coexist *btcoexist)
 {
 	/* return value:
 	 * [31:16] => connected port number
@@ -521,6 +665,18 @@ static bool halbtc_get(void *void_btcoexist, u8 get_type, void *out_buf)
 	case BTC_GET_U4_VENDOR:
 		*u32_tmp = BTC_VENDOR_OTHER;
 		break;
+	case BTC_GET_U4_SUPPORTED_VERSION:
+		*u32_tmp = halbtc_get_bt_coex_supported_version(btcoexist);
+		break;
+	case BTC_GET_U4_SUPPORTED_FEATURE:
+		*u32_tmp = halbtc_get_bt_coex_supported_feature(btcoexist);
+		break;
+	case BTC_GET_U4_BT_DEVICE_INFO:
+		*u32_tmp = halbtc_get_bt_device_info(btcoexist);
+		break;
+	case BTC_GET_U4_BT_FORBIDDEN_SLOT_VAL:
+		*u32_tmp = halbtc_get_bt_forbidden_slot_val(btcoexist);
+		break;
 	case BTC_GET_U1_WIFI_DOT11_CHNL:
 		*u8_tmp = rtlphy->current_channel;
 		break;
@@ -653,6 +809,104 @@ static bool halbtc_set(void *void_btcoexist, u8 set_type, void *in_buf)
 	return ret;
 }
 
+static void halbtc_display_coex_statistics(struct btc_coexist *btcoexist,
+					   struct seq_file *m)
+{
+}
+
+static void halbtc_display_bt_link_info(struct btc_coexist *btcoexist,
+					struct seq_file *m)
+{
+}
+
+static void halbtc_display_wifi_status(struct btc_coexist *btcoexist,
+				       struct seq_file *m)
+{
+	struct rtl_priv *rtlpriv = btcoexist->adapter;
+	s32 wifi_rssi = 0, bt_hs_rssi = 0;
+	bool scan = false, link = false, roam = false, wifi_busy = false;
+	bool wifi_under_b_mode = false;
+	bool wifi_under_5g = false;
+	u32 wifi_bw = BTC_WIFI_BW_HT20;
+	u32 wifi_traffic_dir = BTC_WIFI_TRAFFIC_TX;
+	u32 wifi_freq = BTC_FREQ_2_4G;
+	u32 wifi_link_status = 0x0;
+	bool bt_hs_on = false, under_ips = false, under_lps = false;
+	bool low_power = false, dc_mode = false;
+	u8 wifi_chnl = 0, wifi_hs_chnl = 0;
+	u8 ap_num = 0;
+
+	wifi_link_status = halbtc_get_wifi_link_status(btcoexist);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d/ %d/ %d",
+		   "STA/vWifi/HS/p2pGo/p2pGc",
+		   ((wifi_link_status & WIFI_STA_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_AP_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_HS_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_P2P_GO_CONNECTED) ? 1 : 0),
+		   ((wifi_link_status & WIFI_P2P_GC_CONNECTED) ? 1 : 0));
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_HS_OPERATION, &bt_hs_on);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_DOT11_CHNL, &wifi_chnl);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_WIFI_HS_CHNL, &wifi_hs_chnl);
+	seq_printf(m, "\n %-35s = %d / %d(%d)",
+		   "Dot11 channel / HsChnl(High Speed)",
+		   wifi_chnl, wifi_hs_chnl, bt_hs_on);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_WIFI_RSSI, &wifi_rssi);
+	btcoexist->btc_get(btcoexist, BTC_GET_S4_HS_RSSI, &bt_hs_rssi);
+	seq_printf(m, "\n %-35s = %d/ %d",
+		   "Wifi rssi/ HS rssi",
+		   wifi_rssi - 100, bt_hs_rssi - 100);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_SCAN, &scan);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_LINK, &link);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_ROAM, &roam);
+	seq_printf(m, "\n %-35s = %d/ %d/ %d ",
+		   "Wifi link/ roam/ scan",
+		   link, roam, scan);
+
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_5G, &wifi_under_5g);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_BW, &wifi_bw);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_BUSY, &wifi_busy);
+	btcoexist->btc_get(btcoexist, BTC_GET_U4_WIFI_TRAFFIC_DIRECTION,
+			   &wifi_traffic_dir);
+	btcoexist->btc_get(btcoexist, BTC_GET_U1_AP_NUM, &ap_num);
+	wifi_freq = (wifi_under_5g ? BTC_FREQ_5G : BTC_FREQ_2_4G);
+	btcoexist->btc_get(btcoexist, BTC_GET_BL_WIFI_UNDER_B_MODE,
+			   &wifi_under_b_mode);
+
+	seq_printf(m, "\n %-35s = %s / %s/ %s/ AP=%d ",
+		   "Wifi freq/ bw/ traffic",
+		   gl_btc_wifi_freq_string[wifi_freq],
+		   ((wifi_under_b_mode) ? "11b" :
+		    gl_btc_wifi_bw_string[wifi_bw]),
+		   ((!wifi_busy) ? "idle" : ((BTC_WIFI_TRAFFIC_TX ==
+					      wifi_traffic_dir) ? "uplink" :
+					     "downlink")),
+		   ap_num);
+
+	/* power status	 */
+	dc_mode = true;	/*TODO*/
+	under_ips = rtlpriv->psc.inactive_pwrstate == ERFOFF ? 1 : 0;
+	under_lps = rtlpriv->psc.dot11_psmode == EACTIVE ? 0 : 1;
+	low_power = 0; /*TODO*/
+	seq_printf(m, "\n %-35s = %s%s%s%s",
+		   "Power Status",
+		   (dc_mode ? "DC mode" : "AC mode"),
+		   (under_ips ? ", IPS ON" : ""),
+		   (under_lps ? ", LPS ON" : ""),
+		   (low_power ? ", 32k" : ""));
+
+	seq_printf(m,
+		   "\n %-35s = %02x %02x %02x %02x %02x %02x (0x%x/0x%x)",
+		   "Power mode cmd(lps/rpwm)",
+		   btcoexist->pwr_mode_val[0], btcoexist->pwr_mode_val[1],
+		   btcoexist->pwr_mode_val[2], btcoexist->pwr_mode_val[3],
+		   btcoexist->pwr_mode_val[4], btcoexist->pwr_mode_val[5],
+		   btcoexist->bt_info.lps_val,
+		   btcoexist->bt_info.rpwm_val);
+}
+
 /************************************************************
  *		IO related function
  ************************************************************/
@@ -726,7 +980,8 @@ static void halbtc_write_4byte(void *bt_context, u32 reg_addr, u32 data)
 	rtl_write_dword(rtlpriv, reg_addr, data);
 }
 
-void halbtc_write_local_reg_1byte(void *btc_context, u32 reg_addr, u8 data)
+static void halbtc_write_local_reg_1byte(void *btc_context, u32 reg_addr,
+					 u8 data)
 {
 	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
 	struct rtl_priv *rtlpriv = btcoexist->adapter;
@@ -739,22 +994,6 @@ void halbtc_write_local_reg_1byte(void *btc_context, u32 reg_addr, u8 data)
 		rtl_write_byte(rtlpriv, reg_addr, data);
 }
 
-void halbtc_set_macreg(void *btc_context, u32 reg_addr, u32 bit_mask, u32 data)
-{
-	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
-
-	rtl_set_bbreg(rtlpriv->mac80211.hw, reg_addr, bit_mask, data);
-}
-
-u32 halbtc_get_macreg(void *btc_context, u32 reg_addr, u32 bit_mask)
-{
-	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
-
-	return rtl_get_bbreg(rtlpriv->mac80211.hw, reg_addr, bit_mask);
-}
-
 static void halbtc_set_bbreg(void *bt_context, u32 reg_addr, u32 bit_mask,
 			     u32 data)
 {
@@ -800,38 +1039,47 @@ static void halbtc_fill_h2c_cmd(void *bt_context, u8 element_id,
 					cmd_len, cmd_buf);
 }
 
+static
 void halbtc_set_bt_reg(void *btc_context, u8 reg_type, u32 offset, u32 set_val)
 {
 	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
-	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	u8 cmd_buffer1[4] = {0};
 	u8 cmd_buffer2[4] = {0};
-	u8 *addr_to_set = (u8 *)&offset;
-	u8 *value_to_set = (u8 *)&set_val;
-	u8 oper_ver = 0;
-	u8 req_num = 0;
 
-	if (IS_HARDWARE_TYPE_8723B(btcoexist->adapter)) {
-		cmd_buffer1[0] |= (oper_ver & 0x0f);	/* Set OperVer */
-		cmd_buffer1[0] |= ((req_num << 4) & 0xf0);	/* Set ReqNum */
-		cmd_buffer1[1] = 0x0d;	/* OpCode: BT_LO_OP_WRITE_REG_VALUE */
-		cmd_buffer1[2] = value_to_set[0];	/* Set WriteRegValue */
-		rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, 0x67, 4,
-						&cmd_buffer1[0]);
-
-		msleep(200);
-		req_num++;
-
-		cmd_buffer2[0] |= (oper_ver & 0x0f);	/* Set OperVer */
-		cmd_buffer2[0] |= ((req_num << 4) & 0xf0);	/* Set ReqNum */
-		cmd_buffer2[1] = 0x0c;	/* OpCode: BT_LO_OP_WRITE_REG_ADDR */
-		cmd_buffer2[3] = addr_to_set[0];	/* Set WriteRegAddr */
-		rtlpriv->cfg->ops->fill_h2c_cmd(rtlpriv->mac80211.hw, 0x67, 4,
-						&cmd_buffer2[0]);
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	*((__le16 *)&cmd_buffer1[2]) = cpu_to_le16((u16)set_val);
+	if (!halbtc_send_bt_mp_operation(btcoexist, BT_OP_WRITE_REG_VALUE,
+					 cmd_buffer1, 4, 200))
+		return;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	cmd_buffer2[2] = reg_type;
+	*((u8 *)&cmd_buffer2[3]) = (u8)offset;
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_WRITE_REG_ADDR,
+				    cmd_buffer2, 4, 200);
+}
+
+static void halbtc_display_dbg_msg(void *bt_context, u8 disp_type,
+				   struct seq_file *m)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)bt_context;
+
+	switch (disp_type) {
+	case BTC_DBG_DISP_COEX_STATISTICS:
+		halbtc_display_coex_statistics(btcoexist, m);
+		break;
+	case BTC_DBG_DISP_BT_LINK_INFO:
+		halbtc_display_bt_link_info(btcoexist, m);
+		break;
+	case BTC_DBG_DISP_WIFI_STATUS:
+		halbtc_display_wifi_status(btcoexist, m);
+		break;
+	default:
+		break;
 	}
 }
 
-bool halbtc_under_ips(struct btc_coexist *btcoexist)
+static bool halbtc_under_ips(struct btc_coexist *btcoexist)
 {
 	struct rtl_priv *rtlpriv = btcoexist->adapter;
 	struct rtl_ps_ctl *ppsc = rtl_psc(rtlpriv);
@@ -849,12 +1097,95 @@ bool halbtc_under_ips(struct btc_coexist *btcoexist)
 	return false;
 }
 
+static u8 halbtc_get_ant_det_val_from_bt(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_ANT_DET_VAL,
+				    cmd_buffer, 4, 200);
+
+	/* need wait completion to return correct value */
+
+	return btcoexist->bt_info.bt_ant_det_val;
+}
+
+static u8 halbtc_get_ble_scan_type_from_bt(void *btc_context)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_BLE_SCAN_TYPE,
+				    cmd_buffer, 4, 200);
+
+	/* need wait completion to return correct value */
+
+	return btcoexist->bt_info.bt_ble_scan_type;
+}
+
+static u32 halbtc_get_ble_scan_para_from_bt(void *btc_context, u8 scan_type)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[4] = {0};
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_BT_BLE_SCAN_PARA,
+				    cmd_buffer, 4, 200);
+
+	/* need wait completion to return correct value */
+
+	return btcoexist->bt_info.bt_ble_scan_para;
+}
+
+static bool halbtc_get_bt_afh_map_from_bt(void *btc_context, u8 map_type,
+					  u8 *afh_map)
+{
+	struct btc_coexist *btcoexist = (struct btc_coexist *)btc_context;
+	u8 cmd_buffer[2] = {0};
+	bool ret;
+	u32 *afh_map_l = (u32 *)afh_map;
+	u32 *afh_map_m = (u32 *)(afh_map + 4);
+	u16 *afh_map_h = (u16 *)(afh_map + 8);
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	ret = halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_AFH_MAP_L,
+					  cmd_buffer, 2, 200);
+	if (!ret)
+		goto exit;
+
+	*afh_map_l = btcoexist->bt_info.afh_map_l;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	ret = halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_AFH_MAP_M,
+					  cmd_buffer, 2, 200);
+	if (!ret)
+		goto exit;
+
+	*afh_map_m = btcoexist->bt_info.afh_map_m;
+
+	/* cmd_buffer[0] and [1] is filled by halbtc_send_bt_mp_operation() */
+	ret = halbtc_send_bt_mp_operation(btcoexist, BT_OP_GET_AFH_MAP_H,
+					  cmd_buffer, 2, 200);
+	if (!ret)
+		goto exit;
+
+	*afh_map_h = btcoexist->bt_info.afh_map_h;
+
+exit:
+	return ret;
+}
+
 /*****************************************************************
  *         Extern functions called by other module
  *****************************************************************/
-bool exhalbtc_initlize_variables(void)
+bool exhalbtc_initlize_variables(struct rtl_priv *rtlpriv)
 {
-	struct btc_coexist *btcoexist = &gl_bt_coexist;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
 
 	halbtc_dbg_init();
 
@@ -874,24 +1205,76 @@ bool exhalbtc_initlize_variables(void)
 	btcoexist->btc_get_rf_reg = halbtc_get_rfreg;
 
 	btcoexist->btc_fill_h2c = halbtc_fill_h2c_cmd;
+	btcoexist->btc_disp_dbg_msg = halbtc_display_dbg_msg;
 
 	btcoexist->btc_get = halbtc_get;
 	btcoexist->btc_set = halbtc_set;
 	btcoexist->btc_set_bt_reg = halbtc_set_bt_reg;
 
-
 	btcoexist->bt_info.bt_ctrl_buf_size = false;
 	btcoexist->bt_info.agg_buf_size = 5;
 
 	btcoexist->bt_info.increase_scan_dev_num = false;
+
+	btcoexist->btc_get_bt_coex_supported_feature =
+					halbtc_get_bt_coex_supported_feature;
+	btcoexist->btc_get_bt_coex_supported_version =
+					halbtc_get_bt_coex_supported_version;
+	btcoexist->btc_get_ant_det_val_from_bt = halbtc_get_ant_det_val_from_bt;
+	btcoexist->btc_get_ble_scan_type_from_bt =
+					halbtc_get_ble_scan_type_from_bt;
+	btcoexist->btc_get_ble_scan_para_from_bt =
+					halbtc_get_ble_scan_para_from_bt;
+	btcoexist->btc_get_bt_afh_map_from_bt =
+					halbtc_get_bt_afh_map_from_bt;
+
+	init_completion(&btcoexist->bt_mp_comp);
+
+	return true;
+}
+
+bool exhalbtc_initlize_variables_wifi_only(struct rtl_priv *rtlpriv)
+{
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+	struct wifi_only_haldata *wifionly_haldata;
+
+	if (!wifionly_cfg)
+		return false;
+
+	wifionly_cfg->adapter = rtlpriv;
+
+	switch (rtlpriv->rtlhal.interface) {
+	case INTF_PCI:
+		wifionly_cfg->chip_interface = BTC_INTF_PCI;
+		break;
+	case INTF_USB:
+		wifionly_cfg->chip_interface = BTC_INTF_USB;
+		break;
+	default:
+		wifionly_cfg->chip_interface = BTC_INTF_UNKNOWN;
+		break;
+	}
+
+	wifionly_haldata = &wifionly_cfg->haldata_info;
+
+	wifionly_haldata->customer_id = CUSTOMER_NORMAL;
+	wifionly_haldata->efuse_pg_antnum = rtl_get_hwpg_ant_num(rtlpriv);
+	wifionly_haldata->efuse_pg_antpath =
+					rtl_get_hwpg_single_ant_path(rtlpriv);
+	wifionly_haldata->rfe_type = rtl_get_hwpg_rfe_type(rtlpriv);
+	wifionly_haldata->ant_div_cfg = 0;
+
 	return true;
 }
 
 bool exhalbtc_bind_bt_coex_withadapter(void *adapter)
 {
-	struct btc_coexist *btcoexist = &gl_bt_coexist;
 	struct rtl_priv *rtlpriv = adapter;
-	u8 ant_num = 2, chip_type;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+	u8 ant_num = 2, chip_type, single_ant_path = 0;
+
+	if (!btcoexist)
+		return false;
 
 	if (btcoexist->binded)
 		return false;
@@ -922,10 +1305,16 @@ bool exhalbtc_bind_bt_coex_withadapter(void *adapter)
 	btcoexist->bt_info.miracast_plus_bt = false;
 
 	chip_type = rtl_get_hwpg_bt_type(rtlpriv);
-	exhalbtc_set_chip_type(chip_type);
+	exhalbtc_set_chip_type(btcoexist, chip_type);
 	ant_num = rtl_get_hwpg_ant_num(rtlpriv);
 	exhalbtc_set_ant_num(rtlpriv, BT_COEX_ANT_TYPE_PG, ant_num);
 
+	/* set default antenna position to main  port */
+	btcoexist->board_info.btdm_ant_pos = BTC_ANTENNA_AT_MAIN_PORT;
+
+	single_ant_path = rtl_get_hwpg_single_ant_path(rtlpriv);
+	exhalbtc_set_single_ant_path(btcoexist, single_ant_path);
+
 	if (rtl_get_hwpg_package_type(rtlpriv) == 0)
 		btcoexist->board_info.tfbga_package = false;
 	else if (rtl_get_hwpg_package_type(rtlpriv) == 1)
@@ -940,6 +1329,9 @@ bool exhalbtc_bind_bt_coex_withadapter(void *adapter)
 		RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
 			 "[BTCoex], Package Type = Non-TFBGA\n");
 
+	btcoexist->board_info.rfe_type = rtl_get_hwpg_rfe_type(rtlpriv);
+	btcoexist->board_info.ant_div_cfg = 0;
+
 	return true;
 }
 
@@ -996,6 +1388,10 @@ void exhalbtc_init_hw_config(struct btc_coexist *btcoexist, bool wifi_only)
 	}
 }
 
+void exhalbtc_init_hw_config_wifi_only(struct wifi_only_cfg *wifionly_cfg)
+{
+}
+
 void exhalbtc_init_coex_dm(struct btc_coexist *btcoexist)
 {
 	if (!halbtc_is_bt_coexist_available(btcoexist))
@@ -1122,6 +1518,11 @@ void exhalbtc_scan_notify(struct btc_coexist *btcoexist, u8 type)
 	halbtc_normal_low_power(btcoexist);
 }
 
+void exhalbtc_scan_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+				    u8 is_5g)
+{
+}
+
 void exhalbtc_connect_notify(struct btc_coexist *btcoexist, u8 action)
 {
 	u8 asso_type;
@@ -1430,30 +1831,25 @@ void exhalbtc_stack_update_profile_info(void)
 {
 }
 
-void exhalbtc_update_min_bt_rssi(s8 bt_rssi)
+void exhalbtc_update_min_bt_rssi(struct btc_coexist *btcoexist, s8 bt_rssi)
 {
-	struct btc_coexist *btcoexist = &gl_bt_coexist;
-
 	if (!halbtc_is_bt_coexist_available(btcoexist))
 		return;
 
 	btcoexist->stack_info.min_bt_rssi = bt_rssi;
 }
 
-void exhalbtc_set_hci_version(u16 hci_version)
+void exhalbtc_set_hci_version(struct btc_coexist *btcoexist, u16 hci_version)
 {
-	struct btc_coexist *btcoexist = &gl_bt_coexist;
-
 	if (!halbtc_is_bt_coexist_available(btcoexist))
 		return;
 
 	btcoexist->stack_info.hci_version = hci_version;
 }
 
-void exhalbtc_set_bt_patch_version(u16 bt_hci_version, u16 bt_patch_version)
+void exhalbtc_set_bt_patch_version(struct btc_coexist *btcoexist,
+				   u16 bt_hci_version, u16 bt_patch_version)
 {
-	struct btc_coexist *btcoexist = &gl_bt_coexist;
-
 	if (!halbtc_is_bt_coexist_available(btcoexist))
 		return;
 
@@ -1461,7 +1857,7 @@ void exhalbtc_set_bt_patch_version(u16 bt_hci_version, u16 bt_patch_version)
 	btcoexist->bt_info.bt_hci_ver = bt_hci_version;
 }
 
-void exhalbtc_set_chip_type(u8 chip_type)
+void exhalbtc_set_chip_type(struct btc_coexist *btcoexist, u8 chip_type)
 {
 	switch (chip_type) {
 	default:
@@ -1469,51 +1865,58 @@ void exhalbtc_set_chip_type(u8 chip_type)
 	case BT_ISSC_3WIRE:
 	case BT_ACCEL:
 	case BT_RTL8756:
-		gl_bt_coexist.board_info.bt_chip_type = BTC_CHIP_UNDEF;
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_UNDEF;
 		break;
 	case BT_CSR_BC4:
-		gl_bt_coexist.board_info.bt_chip_type = BTC_CHIP_CSR_BC4;
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_CSR_BC4;
 		break;
 	case BT_CSR_BC8:
-		gl_bt_coexist.board_info.bt_chip_type = BTC_CHIP_CSR_BC8;
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_CSR_BC8;
 		break;
 	case BT_RTL8723A:
-		gl_bt_coexist.board_info.bt_chip_type = BTC_CHIP_RTL8723A;
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_RTL8723A;
 		break;
 	case BT_RTL8821A:
-		gl_bt_coexist.board_info.bt_chip_type = BTC_CHIP_RTL8821;
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_RTL8821;
 		break;
 	case BT_RTL8723B:
-		gl_bt_coexist.board_info.bt_chip_type = BTC_CHIP_RTL8723B;
+		btcoexist->board_info.bt_chip_type = BTC_CHIP_RTL8723B;
 		break;
 	}
 }
 
 void exhalbtc_set_ant_num(struct rtl_priv *rtlpriv, u8 type, u8 ant_num)
 {
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
 	if (BT_COEX_ANT_TYPE_PG == type) {
-		gl_bt_coexist.board_info.pg_ant_num = ant_num;
-		gl_bt_coexist.board_info.btdm_ant_num = ant_num;
+		btcoexist->board_info.pg_ant_num = ant_num;
+		btcoexist->board_info.btdm_ant_num = ant_num;
 	} else if (BT_COEX_ANT_TYPE_ANTDIV == type) {
-		gl_bt_coexist.board_info.btdm_ant_num = ant_num;
+		btcoexist->board_info.btdm_ant_num = ant_num;
 	} else if (type == BT_COEX_ANT_TYPE_DETECTED) {
-		gl_bt_coexist.board_info.btdm_ant_num = ant_num;
+		btcoexist->board_info.btdm_ant_num = ant_num;
 		if (rtlpriv->cfg->mod_params->ant_sel == 1)
-			gl_bt_coexist.board_info.btdm_ant_pos =
+			btcoexist->board_info.btdm_ant_pos =
 				BTC_ANTENNA_AT_AUX_PORT;
 		else
-			gl_bt_coexist.board_info.btdm_ant_pos =
+			btcoexist->board_info.btdm_ant_pos =
 				BTC_ANTENNA_AT_MAIN_PORT;
 	}
 }
 
 /* Currently used by 8723b only, S0 or S1 */
-void exhalbtc_set_single_ant_path(u8 single_ant_path)
+void exhalbtc_set_single_ant_path(struct btc_coexist *btcoexist,
+				  u8 single_ant_path)
 {
-	gl_bt_coexist.board_info.single_ant_path = single_ant_path;
+	btcoexist->board_info.single_ant_path = single_ant_path;
 }
 
-void exhalbtc_display_bt_coex_info(struct btc_coexist *btcoexist)
+void exhalbtc_display_bt_coex_info(struct btc_coexist *btcoexist,
+				   struct seq_file *m)
 {
 	if (!halbtc_is_bt_coexist_available(btcoexist))
 		return;
@@ -1522,18 +1925,36 @@ void exhalbtc_display_bt_coex_info(struct btc_coexist *btcoexist)
 
 	if (IS_HARDWARE_TYPE_8821(btcoexist->adapter)) {
 		if (btcoexist->board_info.btdm_ant_num == 2)
-			ex_btc8821a2ant_display_coex_info(btcoexist);
+			ex_btc8821a2ant_display_coex_info(btcoexist, m);
 		else if (btcoexist->board_info.btdm_ant_num == 1)
-			ex_btc8821a1ant_display_coex_info(btcoexist);
+			ex_btc8821a1ant_display_coex_info(btcoexist, m);
 	} else if (IS_HARDWARE_TYPE_8723B(btcoexist->adapter)) {
 		if (btcoexist->board_info.btdm_ant_num == 2)
-			ex_btc8723b2ant_display_coex_info(btcoexist);
+			ex_btc8723b2ant_display_coex_info(btcoexist, m);
 		else if (btcoexist->board_info.btdm_ant_num == 1)
-			ex_btc8723b1ant_display_coex_info(btcoexist);
+			ex_btc8723b1ant_display_coex_info(btcoexist, m);
 	} else if (IS_HARDWARE_TYPE_8192E(btcoexist->adapter)) {
 		if (btcoexist->board_info.btdm_ant_num == 2)
-			ex_btc8192e2ant_display_coex_info(btcoexist);
+			ex_btc8192e2ant_display_coex_info(btcoexist, m);
 	}
 
 	halbtc_normal_low_power(btcoexist);
 }
+
+void exhalbtc_switch_band_notify(struct btc_coexist *btcoexist, u8 type)
+{
+	if (!halbtc_is_bt_coexist_available(btcoexist))
+		return;
+
+	if (btcoexist->manual_control)
+		return;
+
+	halbtc_leave_low_power(btcoexist);
+
+	halbtc_normal_low_power(btcoexist);
+}
+
+void exhalbtc_switch_band_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+					   u8 is_5g)
+{
+}
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.h
index f9b87c12db09..8ed217656539 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/halbtcoutsrc.h
@@ -103,8 +103,6 @@ enum btc_msg_type {
 	BTC_MSG_MAX
 };
 
-extern u32 btc_dbg_type[];
-
 /* following is for BTC_MSG_INTERFACE */
 #define		INTF_INIT				BIT0
 #define		INTF_NOTIFY				BIT2
@@ -152,8 +150,10 @@ struct btc_board_info {
 	u8 btdm_ant_num;	/* ant number for btdm */
 	u8 btdm_ant_pos;
 	u8 single_ant_path; /* current used for 8723b only, 1=>s0,  0=>s1 */
-	bool bt_exist;
 	bool tfbga_package;
+
+	u8 rfe_type;
+	u8 ant_div_cfg;
 };
 
 enum btc_dbg_opcode {
@@ -181,10 +181,17 @@ enum btc_wifi_role {
 	BTC_ROLE_MAX
 };
 
+enum btc_wireless_freq {
+	BTC_FREQ_2_4G = 0x0,
+	BTC_FREQ_5G = 0x1,
+	BTC_FREQ_MAX
+};
+
 enum btc_wifi_bw_mode {
 	BTC_WIFI_BW_LEGACY = 0x0,
 	BTC_WIFI_BW_HT20 = 0x1,
 	BTC_WIFI_BW_HT40 = 0x2,
+	BTC_WIFI_BW_HT80 = 0x3,
 	BTC_WIFI_BW_MAX
 };
 
@@ -275,6 +282,8 @@ enum btc_get_type {
 	BTC_GET_U4_VENDOR,
 	BTC_GET_U4_SUPPORTED_VERSION,
 	BTC_GET_U4_SUPPORTED_FEATURE,
+	BTC_GET_U4_BT_DEVICE_INFO,
+	BTC_GET_U4_BT_FORBIDDEN_SLOT_VAL,
 	BTC_GET_U4_WIFI_IQK_TOTAL,
 	BTC_GET_U4_WIFI_IQK_OK,
 	BTC_GET_U4_WIFI_IQK_FAIL,
@@ -355,6 +364,7 @@ enum btc_dbg_disp_type {
 	BTC_DBG_DISP_BT_LINK_INFO = 0x1,
 	BTC_DBG_DISP_BT_FW_VER = 0x2,
 	BTC_DBG_DISP_FW_PWR_MODE_CMD = 0x3,
+	BTC_DBG_DISP_WIFI_STATUS = 0x04,
 	BTC_DBG_DISP_MAX
 };
 
@@ -376,6 +386,14 @@ enum btc_notify_type_scan {
 	BTC_SCAN_MAX
 };
 
+enum btc_notify_type_switchband {
+	BTC_NOT_SWITCH = 0x0,
+	BTC_SWITCH_TO_24G = 0x1,
+	BTC_SWITCH_TO_5G = 0x2,
+	BTC_SWITCH_TO_24G_NOFORSCAN = 0x3,
+	BTC_SWITCH_MAX
+};
+
 enum btc_notify_type_associate {
 	BTC_ASSOCIATE_FINISH = 0x0,
 	BTC_ASSOCIATE_START = 0x1,
@@ -417,49 +435,6 @@ enum btc_notify_type_stack_operation {
 	BTC_STACK_OP_MAX
 };
 
-typedef u8 (*bfp_btc_r1)(void *btc_context, u32 reg_addr);
-
-typedef u16 (*bfp_btc_r2)(void *btc_context, u32 reg_addr);
-
-typedef u32 (*bfp_btc_r4)(void *btc_context, u32 reg_addr);
-
-typedef void (*bfp_btc_w1)(void *btc_context, u32 reg_addr, u32 data);
-
-typedef void (*bfp_btc_w1_bit_mak)(void *btc_context, u32 reg_addr,
-				   u32 bit_mask, u8 data1b);
-
-typedef void (*bfp_btc_w2)(void *btc_context, u32 reg_addr, u16 data);
-
-typedef void (*bfp_btc_w4)(void *btc_context, u32 reg_addr, u32 data);
-
-typedef void (*bfp_btc_local_reg_w1)(void *btc_context, u32 reg_addr, u8 data);
-typedef void (*bfp_btc_wr_1byte_bit_mask)(void *btc_context, u32 reg_addr,
-					  u8 bit_mask, u8 data);
-
-typedef void (*bfp_btc_set_bb_reg)(void *btc_context, u32 reg_addr,
-				   u32 bit_mask, u32 data);
-
-typedef u32 (*bfp_btc_get_bb_reg)(void *btc_context, u32 reg_addr,
-				  u32 bit_mask);
-
-typedef void (*bfp_btc_set_rf_reg)(void *btc_context, u8 rf_path, u32 reg_addr,
-				   u32 bit_mask, u32 data);
-
-typedef u32 (*bfp_btc_get_rf_reg)(void *btc_context, u8 rf_path,
-				  u32 reg_addr, u32 bit_mask);
-
-typedef void (*bfp_btc_fill_h2c)(void *btc_context, u8 element_id,
-				 u32 cmd_len, u8 *cmd_buffer);
-
-typedef	bool (*bfp_btc_get)(void *btcoexist, u8 get_type, void *out_buf);
-
-typedef	bool (*bfp_btc_set)(void *btcoexist, u8 set_type, void *in_buf);
-
-typedef void (*bfp_btc_set_bt_reg)(void *btc_context, u8 reg_type, u32 offset,
-				   u32 value);
-
-typedef void (*bfp_btc_disp_dbg_msg)(void *btcoexist, u8 disp_type);
-
 struct btc_bt_info {
 	bool bt_disabled;
 	u8 rssi_adjust_for_agc_table_on;
@@ -491,6 +466,17 @@ struct btc_bt_info {
 	u8 lps_val;
 	u8 rpwm_val;
 	u32 ra_mask;
+
+	u32 afh_map_l;
+	u32 afh_map_m;
+	u16 afh_map_h;
+	u32 bt_supported_feature;
+	u32 bt_supported_version;
+	u32 bt_device_info;
+	u32 bt_forb_slot_val;
+	u8 bt_ant_det_val;
+	u8 bt_ble_scan_type;
+	u32 bt_ble_scan_para;
 };
 
 struct btc_stack_info {
@@ -546,6 +532,40 @@ enum btc_antenna_pos {
 	BTC_ANTENNA_AT_AUX_PORT = 0x2,
 };
 
+enum btc_mp_h2c_op_code {
+	BT_OP_GET_BT_VERSION			= 0,
+	BT_OP_WRITE_REG_ADDR			= 12,
+	BT_OP_WRITE_REG_VALUE			= 13,
+	BT_OP_READ_REG				= 17,
+	BT_OP_GET_AFH_MAP_L			= 30,
+	BT_OP_GET_AFH_MAP_M			= 31,
+	BT_OP_GET_AFH_MAP_H			= 32,
+	BT_OP_GET_BT_COEX_SUPPORTED_FEATURE	= 42,
+	BT_OP_GET_BT_COEX_SUPPORTED_VERSION	= 43,
+	BT_OP_GET_BT_ANT_DET_VAL		= 44,
+	BT_OP_GET_BT_BLE_SCAN_PARA		= 45,
+	BT_OP_GET_BT_BLE_SCAN_TYPE		= 46,
+	BT_OP_GET_BT_DEVICE_INFO		= 48,
+	BT_OP_GET_BT_FORBIDDEN_SLOT_VAL		= 49,
+	BT_OP_MAX
+};
+
+enum btc_mp_h2c_req_num {
+	/* 4 bits only */
+	BT_SEQ_DONT_CARE			= 0,
+	BT_SEQ_GET_BT_VERSION			= 0xE,
+	BT_SEQ_GET_AFH_MAP_L			= 0x5,
+	BT_SEQ_GET_AFH_MAP_M			= 0x6,
+	BT_SEQ_GET_AFH_MAP_H			= 0x9,
+	BT_SEQ_GET_BT_COEX_SUPPORTED_FEATURE	= 0x7,
+	BT_SEQ_GET_BT_COEX_SUPPORTED_VERSION	= 0x8,
+	BT_SEQ_GET_BT_ANT_DET_VAL		= 0x2,
+	BT_SEQ_GET_BT_BLE_SCAN_PARA		= 0x3,
+	BT_SEQ_GET_BT_BLE_SCAN_TYPE		= 0x4,
+	BT_SEQ_GET_BT_DEVICE_INFO		= 0xA,
+	BT_SEQ_GET_BT_FORB_SLOT_VAL		= 0xB,
+};
+
 struct btc_coexist {
 	/* make sure only one adapter can bind the data context  */
 	bool binded;
@@ -563,55 +583,86 @@ struct btc_coexist {
 	 */
 	bool auto_report_1ant;
 	bool auto_report_2ant;
+	bool dbg_mode_1ant;
+	bool dbg_mode_2ant;
 	bool initilized;
 	bool stop_coex_dm;
 	bool manual_control;
 	struct btc_statistics statistics;
 	u8 pwr_mode_val[10];
 
-	/* function pointers - io related */
-	bfp_btc_r1 btc_read_1byte;
-	bfp_btc_w1 btc_write_1byte;
-	bfp_btc_w1_bit_mak btc_write_1byte_bitmask;
-	bfp_btc_r2 btc_read_2byte;
-	bfp_btc_w2 btc_write_2byte;
-	bfp_btc_r4 btc_read_4byte;
-	bfp_btc_w4 btc_write_4byte;
-	bfp_btc_local_reg_w1 btc_write_local_reg_1byte;
-
-	bfp_btc_set_bb_reg btc_set_bb_reg;
-	bfp_btc_get_bb_reg btc_get_bb_reg;
+	struct completion bt_mp_comp;
 
-	bfp_btc_set_rf_reg btc_set_rf_reg;
-	bfp_btc_get_rf_reg btc_get_rf_reg;
-
-	bfp_btc_fill_h2c btc_fill_h2c;
-
-	bfp_btc_disp_dbg_msg btc_disp_dbg_msg;
-
-	bfp_btc_get btc_get;
-	bfp_btc_set btc_set;
-
-	bfp_btc_set_bt_reg btc_set_bt_reg;
+	/* function pointers - io related */
+	u8 (*btc_read_1byte)(void *btc_context, u32 reg_addr);
+	void (*btc_write_1byte)(void *btc_context, u32 reg_addr, u32 data);
+	void (*btc_write_1byte_bitmask)(void *btc_context, u32 reg_addr,
+					u32 bit_mask, u8 data1b);
+	u16 (*btc_read_2byte)(void *btc_context, u32 reg_addr);
+	void (*btc_write_2byte)(void *btc_context, u32 reg_addr, u16 data);
+	u32 (*btc_read_4byte)(void *btc_context, u32 reg_addr);
+	void (*btc_write_4byte)(void *btc_context, u32 reg_addr, u32 data);
+
+	void (*btc_write_local_reg_1byte)(void *btc_context, u32 reg_addr,
+					  u8 data);
+	void (*btc_set_bb_reg)(void *btc_context, u32 reg_addr,
+			       u32 bit_mask, u32 data);
+	u32 (*btc_get_bb_reg)(void *btc_context, u32 reg_addr,
+			      u32 bit_mask);
+	void (*btc_set_rf_reg)(void *btc_context, u8 rf_path, u32 reg_addr,
+			       u32 bit_mask, u32 data);
+	u32 (*btc_get_rf_reg)(void *btc_context, u8 rf_path,
+			      u32 reg_addr, u32 bit_mask);
+
+	void (*btc_fill_h2c)(void *btc_context, u8 element_id,
+			     u32 cmd_len, u8 *cmd_buffer);
+
+	void (*btc_disp_dbg_msg)(void *btcoexist, u8 disp_type,
+				 struct seq_file *m);
+
+	bool (*btc_get)(void *btcoexist, u8 get_type, void *out_buf);
+	bool (*btc_set)(void *btcoexist, u8 set_type, void *in_buf);
+
+	void (*btc_set_bt_reg)(void *btc_context, u8 reg_type, u32 offset,
+			       u32 value);
+	u32 (*btc_get_bt_coex_supported_feature)(void *btcoexist);
+	u32 (*btc_get_bt_coex_supported_version)(void *btcoexist);
+	u8 (*btc_get_ant_det_val_from_bt)(void *btcoexist);
+	u8 (*btc_get_ble_scan_type_from_bt)(void *btcoexist);
+	u32 (*btc_get_ble_scan_para_from_bt)(void *btcoexist, u8 scan_type);
+	bool (*btc_get_bt_afh_map_from_bt)(void *btcoexist, u8 map_type,
+					   u8 *afh_map);
 };
 
 bool halbtc_is_wifi_uplink(struct rtl_priv *adapter);
 
-extern struct btc_coexist gl_bt_coexist;
+#define rtl_btc_coexist(rtlpriv)				\
+	((struct btc_coexist *)((rtlpriv)->btcoexist.btc_context))
+#define rtl_btc_wifi_only(rtlpriv)				\
+	((struct wifi_only_cfg *)((rtlpriv)->btcoexist.wifi_only_context))
 
-bool exhalbtc_initlize_variables(void);
+struct wifi_only_cfg;
+
+bool exhalbtc_initlize_variables(struct rtl_priv *rtlpriv);
+bool exhalbtc_initlize_variables_wifi_only(struct rtl_priv *rtlpriv);
 bool exhalbtc_bind_bt_coex_withadapter(void *adapter);
+void exhalbtc_power_on_setting(struct btc_coexist *btcoexist);
+void exhalbtc_pre_load_firmware(struct btc_coexist *btcoexist);
 void exhalbtc_init_hw_config(struct btc_coexist *btcoexist, bool wifi_only);
+void exhalbtc_init_hw_config_wifi_only(struct wifi_only_cfg *wifionly_cfg);
 void exhalbtc_init_coex_dm(struct btc_coexist *btcoexist);
 void exhalbtc_ips_notify(struct btc_coexist *btcoexist, u8 type);
 void exhalbtc_lps_notify(struct btc_coexist *btcoexist, u8 type);
 void exhalbtc_scan_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_scan_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+				    u8 is_5g);
 void exhalbtc_connect_notify(struct btc_coexist *btcoexist, u8 action);
 void exhalbtc_mediastatus_notify(struct btc_coexist *btcoexist,
 				 enum rt_media_status media_status);
 void exhalbtc_special_packet_notify(struct btc_coexist *btcoexist, u8 pkt_type);
 void exhalbtc_bt_info_notify(struct btc_coexist *btcoexist, u8 *tmp_buf,
 			     u8 length);
+void exhalbtc_rf_status_notify(struct btc_coexist *btcoexist, u8 type);
 void exhalbtc_stack_operation_notify(struct btc_coexist *btcoexist, u8 type);
 void exhalbtc_halt_notify(struct btc_coexist *btcoexist);
 void exhalbtc_pnp_notify(struct btc_coexist *btcoexist, u8 pnp_state);
@@ -619,18 +670,63 @@ void exhalbtc_coex_dm_switch(struct btc_coexist *btcoexist);
 void exhalbtc_periodical(struct btc_coexist *btcoexist);
 void exhalbtc_dbg_control(struct btc_coexist *btcoexist, u8 code, u8 len,
 			  u8 *data);
+void exhalbtc_antenna_detection(struct btc_coexist *btcoexist, u32 cent_freq,
+				u32 offset, u32 span, u32 seconds);
 void exhalbtc_stack_update_profile_info(void);
-void exhalbtc_set_hci_version(u16 hci_version);
-void exhalbtc_set_bt_patch_version(u16 bt_hci_version, u16 bt_patch_version);
-void exhalbtc_update_min_bt_rssi(s8 bt_rssi);
-void exhalbtc_set_bt_exist(bool bt_exist);
-void exhalbtc_set_chip_type(u8 chip_type);
+void exhalbtc_set_hci_version(struct btc_coexist *btcoexist, u16 hci_version);
+void exhalbtc_set_bt_patch_version(struct btc_coexist *btcoexist,
+				   u16 bt_hci_version, u16 bt_patch_version);
+void exhalbtc_update_min_bt_rssi(struct btc_coexist *btcoexist, s8 bt_rssi);
+void exhalbtc_set_bt_exist(struct btc_coexist *btcoexist, bool bt_exist);
+void exhalbtc_set_chip_type(struct btc_coexist *btcoexist, u8 chip_type);
 void exhalbtc_set_ant_num(struct rtl_priv *rtlpriv, u8 type, u8 ant_num);
-void exhalbtc_display_bt_coex_info(struct btc_coexist *btcoexist);
+void exhalbtc_display_bt_coex_info(struct btc_coexist *btcoexist,
+				   struct seq_file *m);
+void exhalbtc_switch_band_notify(struct btc_coexist *btcoexist, u8 type);
+void exhalbtc_switch_band_notify_wifi_only(struct wifi_only_cfg *wifionly_cfg,
+					   u8 is_5g);
 void exhalbtc_signal_compensation(struct btc_coexist *btcoexist,
 				  u8 *rssi_wifi, u8 *rssi_bt);
 void exhalbtc_lps_leave(struct btc_coexist *btcoexist);
 void exhalbtc_low_wifi_traffic_notify(struct btc_coexist *btcoexist);
-void exhalbtc_set_single_ant_path(u8 single_ant_path);
+void exhalbtc_set_single_ant_path(struct btc_coexist *btcoexist,
+				  u8 single_ant_path);
+
+/* The following are used by wifi_only case */
+enum wifionly_chip_interface {
+	WIFIONLY_INTF_UNKNOWN	= 0,
+	WIFIONLY_INTF_PCI		= 1,
+	WIFIONLY_INTF_USB		= 2,
+	WIFIONLY_INTF_SDIO		= 3,
+	WIFIONLY_INTF_MAX
+};
+
+enum wifionly_customer_id {
+	CUSTOMER_NORMAL			= 0,
+	CUSTOMER_HP_1			= 1,
+};
+
+struct wifi_only_haldata {
+	u16		customer_id;
+	u8		efuse_pg_antnum;
+	u8		efuse_pg_antpath;
+	u8		rfe_type;
+	u8		ant_div_cfg;
+};
+
+struct wifi_only_cfg {
+	void				*adapter;
+	struct wifi_only_haldata	haldata_info;
+	enum wifionly_chip_interface	chip_interface;
+};
+
+static inline
+void halwifionly_phy_set_bb_reg(struct wifi_only_cfg *wifi_conly_cfg,
+				u32 regaddr, u32 bitmask, u32 data)
+{
+	struct rtl_priv *rtlpriv = (struct rtl_priv *)wifi_conly_cfg->adapter;
+
+	rtl_set_bbreg(rtlpriv->hw, regaddr, bitmask, data);
+}
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c
index 7d296a401b6f..cce4a37ea408 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.c
@@ -31,11 +31,16 @@
 
 static struct rtl_btc_ops rtl_btc_operation = {
 	.btc_init_variables = rtl_btc_init_variables,
+	.btc_init_variables_wifi_only = rtl_btc_init_variables_wifi_only,
+	.btc_deinit_variables = rtl_btc_deinit_variables,
 	.btc_init_hal_vars = rtl_btc_init_hal_vars,
+	.btc_power_on_setting = rtl_btc_power_on_setting,
 	.btc_init_hw_config = rtl_btc_init_hw_config,
+	.btc_init_hw_config_wifi_only = rtl_btc_init_hw_config_wifi_only,
 	.btc_ips_notify = rtl_btc_ips_notify,
 	.btc_lps_notify = rtl_btc_lps_notify,
 	.btc_scan_notify = rtl_btc_scan_notify,
+	.btc_scan_notify_wifi_only = rtl_btc_scan_notify_wifi_only,
 	.btc_connect_notify = rtl_btc_connect_notify,
 	.btc_mediastatus_notify = rtl_btc_mediastatus_notify,
 	.btc_periodical = rtl_btc_periodical,
@@ -46,63 +51,143 @@ static struct rtl_btc_ops rtl_btc_operation = {
 	.btc_is_disable_edca_turbo = rtl_btc_is_disable_edca_turbo,
 	.btc_is_bt_disabled = rtl_btc_is_bt_disabled,
 	.btc_special_packet_notify = rtl_btc_special_packet_notify,
+	.btc_switch_band_notify = rtl_btc_switch_band_notify,
+	.btc_switch_band_notify_wifi_only = rtl_btc_switch_band_notify_wifionly,
 	.btc_record_pwr_mode = rtl_btc_record_pwr_mode,
 	.btc_get_lps_val = rtl_btc_get_lps_val,
 	.btc_get_rpwm_val = rtl_btc_get_rpwm_val,
 	.btc_is_bt_ctrl_lps = rtl_btc_is_bt_ctrl_lps,
 	.btc_is_bt_lps_on = rtl_btc_is_bt_lps_on,
 	.btc_get_ampdu_cfg = rtl_btc_get_ampdu_cfg,
+	.btc_display_bt_coex_info = rtl_btc_display_bt_coex_info,
 };
 
+void rtl_btc_display_bt_coex_info(struct rtl_priv *rtlpriv, struct seq_file *m)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist) {
+		seq_puts(m, "btc_coexist context is NULL!\n");
+		return;
+	}
+
+	exhalbtc_display_bt_coex_info(btcoexist, m);
+}
+
 void rtl_btc_record_pwr_mode(struct rtl_priv *rtlpriv, u8 *buf, u8 len)
 {
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
 	u8 safe_len;
 
-	safe_len = sizeof(gl_bt_coexist.pwr_mode_val);
+	if (!btcoexist)
+		return;
+
+	safe_len = sizeof(btcoexist->pwr_mode_val);
 
 	if (safe_len > len)
 		safe_len = len;
 
-	memcpy(gl_bt_coexist.pwr_mode_val, buf, safe_len);
+	memcpy(btcoexist->pwr_mode_val, buf, safe_len);
 }
 
 u8 rtl_btc_get_lps_val(struct rtl_priv *rtlpriv)
 {
-	return gl_bt_coexist.bt_info.lps_val;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return 0;
+
+	return btcoexist->bt_info.lps_val;
 }
 
 u8 rtl_btc_get_rpwm_val(struct rtl_priv *rtlpriv)
 {
-	return gl_bt_coexist.bt_info.rpwm_val;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return 0;
+
+	return btcoexist->bt_info.rpwm_val;
 }
 
 bool rtl_btc_is_bt_ctrl_lps(struct rtl_priv *rtlpriv)
 {
-	return gl_bt_coexist.bt_info.bt_ctrl_lps;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
+
+	return btcoexist->bt_info.bt_ctrl_lps;
 }
 
 bool rtl_btc_is_bt_lps_on(struct rtl_priv *rtlpriv)
 {
-	return gl_bt_coexist.bt_info.bt_lps_on;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
+
+	return btcoexist->bt_info.bt_lps_on;
 }
 
 void rtl_btc_get_ampdu_cfg(struct rtl_priv *rtlpriv, u8 *reject_agg,
 			   u8 *ctrl_agg_size, u8 *agg_size)
 {
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist) {
+		*reject_agg = false;
+		*ctrl_agg_size = false;
+		return;
+	}
+
 	if (reject_agg)
-		*reject_agg = gl_bt_coexist.bt_info.reject_agg_pkt;
+		*reject_agg = btcoexist->bt_info.reject_agg_pkt;
 	if (ctrl_agg_size)
-		*ctrl_agg_size = gl_bt_coexist.bt_info.bt_ctrl_agg_buf_size;
+		*ctrl_agg_size = btcoexist->bt_info.bt_ctrl_agg_buf_size;
 	if (agg_size)
-		*agg_size = gl_bt_coexist.bt_info.agg_buf_size;
+		*agg_size = btcoexist->bt_info.agg_buf_size;
+}
+
+static void rtl_btc_alloc_variable(struct rtl_priv *rtlpriv, bool wifi_only)
+{
+	if (wifi_only)
+		rtlpriv->btcoexist.wifi_only_context =
+			kzalloc(sizeof(struct wifi_only_cfg), GFP_KERNEL);
+	else
+		rtlpriv->btcoexist.btc_context =
+			kzalloc(sizeof(struct btc_coexist), GFP_KERNEL);
+}
+
+static void rtl_btc_free_variable(struct rtl_priv *rtlpriv)
+{
+	kfree(rtlpriv->btcoexist.btc_context);
+	rtlpriv->btcoexist.btc_context = NULL;
+
+	kfree(rtlpriv->btcoexist.wifi_only_context);
+	rtlpriv->btcoexist.wifi_only_context = NULL;
 }
 
 void rtl_btc_init_variables(struct rtl_priv *rtlpriv)
 {
-	exhalbtc_initlize_variables();
+	rtl_btc_alloc_variable(rtlpriv, false);
+
+	exhalbtc_initlize_variables(rtlpriv);
 	exhalbtc_bind_bt_coex_withadapter(rtlpriv);
 }
 
+void rtl_btc_init_variables_wifi_only(struct rtl_priv *rtlpriv)
+{
+	rtl_btc_alloc_variable(rtlpriv, true);
+
+	exhalbtc_initlize_variables_wifi_only(rtlpriv);
+}
+
+void rtl_btc_deinit_variables(struct rtl_priv *rtlpriv)
+{
+	rtl_btc_free_variable(rtlpriv);
+}
+
 void rtl_btc_init_hal_vars(struct rtl_priv *rtlpriv)
 {
 	/* move ant_num, bt_type and single_ant_path to
@@ -110,67 +195,155 @@ void rtl_btc_init_hal_vars(struct rtl_priv *rtlpriv)
 	 */
 }
 
+void rtl_btc_power_on_setting(struct rtl_priv *rtlpriv)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_power_on_setting(btcoexist);
+}
+
 void rtl_btc_init_hw_config(struct rtl_priv *rtlpriv)
 {
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
 	u8 bt_exist;
 
 	bt_exist = rtl_get_hwpg_bt_exist(rtlpriv);
 	RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
 		"%s, bt_exist is %d\n", __func__, bt_exist);
 
-	exhalbtc_init_hw_config(&gl_bt_coexist, !bt_exist);
-	exhalbtc_init_coex_dm(&gl_bt_coexist);
+	if (!btcoexist)
+		return;
+
+	exhalbtc_init_hw_config(btcoexist, !bt_exist);
+	exhalbtc_init_coex_dm(btcoexist);
+}
+
+void rtl_btc_init_hw_config_wifi_only(struct rtl_priv *rtlpriv)
+{
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+
+	if (!wifionly_cfg)
+		return;
+
+	exhalbtc_init_hw_config_wifi_only(wifionly_cfg);
 }
 
 void rtl_btc_ips_notify(struct rtl_priv *rtlpriv, u8 type)
 {
-	exhalbtc_ips_notify(&gl_bt_coexist, type);
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_ips_notify(btcoexist, type);
+
+	if (type == ERFON) {
+		/* In some situation, it doesn't scan after leaving IPS, and
+		 * this will cause btcoex in wrong state.
+		 */
+		exhalbtc_scan_notify(btcoexist, 1);
+		exhalbtc_scan_notify(btcoexist, 0);
+	}
 }
 
 void rtl_btc_lps_notify(struct rtl_priv *rtlpriv, u8 type)
 {
-	exhalbtc_lps_notify(&gl_bt_coexist, type);
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_lps_notify(btcoexist, type);
 }
 
 void rtl_btc_scan_notify(struct rtl_priv *rtlpriv, u8 scantype)
 {
-	exhalbtc_scan_notify(&gl_bt_coexist, scantype);
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_scan_notify(btcoexist, scantype);
+}
+
+void rtl_btc_scan_notify_wifi_only(struct rtl_priv *rtlpriv, u8 scantype)
+{
+	struct rtl_hal *rtlhal = rtl_hal(rtlpriv);
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+	u8 is_5g = (rtlhal->current_bandtype == BAND_ON_5G);
+
+	if (!wifionly_cfg)
+		return;
+
+	exhalbtc_scan_notify_wifi_only(wifionly_cfg, is_5g);
 }
 
 void rtl_btc_connect_notify(struct rtl_priv *rtlpriv, u8 action)
 {
-	exhalbtc_connect_notify(&gl_bt_coexist, action);
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_connect_notify(btcoexist, action);
 }
 
 void rtl_btc_mediastatus_notify(struct rtl_priv *rtlpriv,
 				enum rt_media_status mstatus)
 {
-	exhalbtc_mediastatus_notify(&gl_bt_coexist, mstatus);
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_mediastatus_notify(btcoexist, mstatus);
 }
 
 void rtl_btc_periodical(struct rtl_priv *rtlpriv)
 {
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
 	/*rtl_bt_dm_monitor();*/
-	exhalbtc_periodical(&gl_bt_coexist);
+	exhalbtc_periodical(btcoexist);
 }
 
-void rtl_btc_halt_notify(void)
+void rtl_btc_halt_notify(struct rtl_priv *rtlpriv)
 {
-	struct btc_coexist *btcoexist = &gl_bt_coexist;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
 
 	exhalbtc_halt_notify(btcoexist);
 }
 
 void rtl_btc_btinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length)
 {
-	exhalbtc_bt_info_notify(&gl_bt_coexist, tmp_buf, length);
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	exhalbtc_bt_info_notify(btcoexist, tmp_buf, length);
 }
 
 void rtl_btc_btmpinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length)
 {
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
 	u8 extid, seq, len;
 	u16 bt_real_fw_ver;
 	u8 bt_fw_ver;
+	u8 *data;
+
+	if (!btcoexist)
+		return;
 
 	if ((length < 4) || (!tmp_buf))
 		return;
@@ -182,20 +355,70 @@ void rtl_btc_btmpinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length)
 
 	len = tmp_buf[1] >> 4;
 	seq = tmp_buf[2] >> 4;
+	data = &tmp_buf[3];
 
 	/* BT Firmware version response */
-	if (seq == 0x0E) {
+	switch (seq) {
+	case BT_SEQ_GET_BT_VERSION:
 		bt_real_fw_ver = tmp_buf[3] | (tmp_buf[4] << 8);
 		bt_fw_ver = tmp_buf[5];
 
-		gl_bt_coexist.bt_info.bt_real_fw_ver = bt_real_fw_ver;
-		gl_bt_coexist.bt_info.bt_fw_ver = bt_fw_ver;
+		btcoexist->bt_info.bt_real_fw_ver = bt_real_fw_ver;
+		btcoexist->bt_info.bt_fw_ver = bt_fw_ver;
+		break;
+	case BT_SEQ_GET_AFH_MAP_L:
+		btcoexist->bt_info.afh_map_l = le32_to_cpu(*(__le32 *)data);
+		break;
+	case BT_SEQ_GET_AFH_MAP_M:
+		btcoexist->bt_info.afh_map_m = le32_to_cpu(*(__le32 *)data);
+		break;
+	case BT_SEQ_GET_AFH_MAP_H:
+		btcoexist->bt_info.afh_map_h = le16_to_cpu(*(__le16 *)data);
+		break;
+	case BT_SEQ_GET_BT_COEX_SUPPORTED_FEATURE:
+		btcoexist->bt_info.bt_supported_feature = tmp_buf[3] |
+							  (tmp_buf[4] << 8);
+		break;
+	case BT_SEQ_GET_BT_COEX_SUPPORTED_VERSION:
+		btcoexist->bt_info.bt_supported_version = tmp_buf[3] |
+							  (tmp_buf[4] << 8);
+		break;
+	case BT_SEQ_GET_BT_ANT_DET_VAL:
+		btcoexist->bt_info.bt_ant_det_val = tmp_buf[3];
+		break;
+	case BT_SEQ_GET_BT_BLE_SCAN_PARA:
+		btcoexist->bt_info.bt_ble_scan_para = tmp_buf[3] |
+						      (tmp_buf[4] << 8) |
+						      (tmp_buf[5] << 16) |
+						      (tmp_buf[6] << 24);
+		break;
+	case BT_SEQ_GET_BT_BLE_SCAN_TYPE:
+		btcoexist->bt_info.bt_ble_scan_type = tmp_buf[3];
+		break;
+	case BT_SEQ_GET_BT_DEVICE_INFO:
+		btcoexist->bt_info.bt_device_info =
+						le32_to_cpu(*(__le32 *)data);
+		break;
+	case BT_OP_GET_BT_FORBIDDEN_SLOT_VAL:
+		btcoexist->bt_info.bt_forb_slot_val =
+						le32_to_cpu(*(__le32 *)data);
+		break;
 	}
+
+	RT_TRACE(rtlpriv, COMP_BT_COEXIST, DBG_LOUD,
+		 "btmpinfo complete req_num=%d\n", seq);
+
+	complete(&btcoexist->bt_mp_comp);
 }
 
 bool rtl_btc_is_limited_dig(struct rtl_priv *rtlpriv)
 {
-	return gl_bt_coexist.bt_info.limited_dig;
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return false;
+
+	return btcoexist->bt_info.limited_dig;
 }
 
 bool rtl_btc_is_disable_edca_turbo(struct rtl_priv *rtlpriv)
@@ -227,8 +450,13 @@ bool rtl_btc_is_disable_edca_turbo(struct rtl_priv *rtlpriv)
 
 bool rtl_btc_is_bt_disabled(struct rtl_priv *rtlpriv)
 {
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return true;
+
 	/* It seems 'bt_disabled' is never be initialized or set. */
-	if (gl_bt_coexist.bt_info.bt_disabled)
+	if (btcoexist->bt_info.bt_disabled)
 		return true;
 	else
 		return false;
@@ -236,7 +464,50 @@ bool rtl_btc_is_bt_disabled(struct rtl_priv *rtlpriv)
 
 void rtl_btc_special_packet_notify(struct rtl_priv *rtlpriv, u8 pkt_type)
 {
-	return exhalbtc_special_packet_notify(&gl_bt_coexist, pkt_type);
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+
+	if (!btcoexist)
+		return;
+
+	return exhalbtc_special_packet_notify(btcoexist, pkt_type);
+}
+
+void rtl_btc_switch_band_notify(struct rtl_priv *rtlpriv, u8 band_type,
+				bool scanning)
+{
+	struct btc_coexist *btcoexist = rtl_btc_coexist(rtlpriv);
+	u8 type = BTC_NOT_SWITCH;
+
+	if (!btcoexist)
+		return;
+
+	switch (band_type) {
+	case BAND_ON_2_4G:
+		if (scanning)
+			type = BTC_SWITCH_TO_24G;
+		else
+			type = BTC_SWITCH_TO_24G_NOFORSCAN;
+		break;
+
+	case BAND_ON_5G:
+		type = BTC_SWITCH_TO_5G;
+		break;
+	}
+
+	if (type != BTC_NOT_SWITCH)
+		exhalbtc_switch_band_notify(btcoexist, type);
+}
+
+void rtl_btc_switch_band_notify_wifionly(struct rtl_priv *rtlpriv, u8 band_type,
+					 bool scanning)
+{
+	struct wifi_only_cfg *wifionly_cfg = rtl_btc_wifi_only(rtlpriv);
+	u8 is_5g = (band_type == BAND_ON_5G);
+
+	if (!wifionly_cfg)
+		return;
+
+	exhalbtc_switch_band_notify_wifi_only(wifionly_cfg, is_5g);
 }
 
 struct rtl_btc_ops *rtl_btc_get_ops_pointer(void)
diff --git a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h
index ac1253c46f44..8c996055de71 100644
--- a/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h
+++ b/drivers/net/wireless/realtek/rtlwifi/btcoexist/rtl_btc.h
@@ -28,22 +28,32 @@
 #include "halbt_precomp.h"
 
 void rtl_btc_init_variables(struct rtl_priv *rtlpriv);
+void rtl_btc_init_variables_wifi_only(struct rtl_priv *rtlpriv);
+void rtl_btc_deinit_variables(struct rtl_priv *rtlpriv);
 void rtl_btc_init_hal_vars(struct rtl_priv *rtlpriv);
+void rtl_btc_power_on_setting(struct rtl_priv *rtlpriv);
 void rtl_btc_init_hw_config(struct rtl_priv *rtlpriv);
+void rtl_btc_init_hw_config_wifi_only(struct rtl_priv *rtlpriv);
 void rtl_btc_ips_notify(struct rtl_priv *rtlpriv, u8 type);
 void rtl_btc_lps_notify(struct rtl_priv *rtlpriv, u8 type);
 void rtl_btc_scan_notify(struct rtl_priv *rtlpriv, u8 scantype);
+void rtl_btc_scan_notify_wifi_only(struct rtl_priv *rtlpriv, u8 scantype);
 void rtl_btc_connect_notify(struct rtl_priv *rtlpriv, u8 action);
 void rtl_btc_mediastatus_notify(struct rtl_priv *rtlpriv,
 				enum rt_media_status mstatus);
 void rtl_btc_periodical(struct rtl_priv *rtlpriv);
-void rtl_btc_halt_notify(void);
+void rtl_btc_halt_notify(struct rtl_priv *rtlpriv);
 void rtl_btc_btinfo_notify(struct rtl_priv *rtlpriv, u8 *tmpbuf, u8 length);
 void rtl_btc_btmpinfo_notify(struct rtl_priv *rtlpriv, u8 *tmp_buf, u8 length);
 bool rtl_btc_is_limited_dig(struct rtl_priv *rtlpriv);
 bool rtl_btc_is_disable_edca_turbo(struct rtl_priv *rtlpriv);
 bool rtl_btc_is_bt_disabled(struct rtl_priv *rtlpriv);
 void rtl_btc_special_packet_notify(struct rtl_priv *rtlpriv, u8 pkt_type);
+void rtl_btc_switch_band_notify(struct rtl_priv *rtlpriv, u8 band_type,
+				bool scanning);
+void rtl_btc_switch_band_notify_wifionly(struct rtl_priv *rtlpriv, u8 band_type,
+					 bool scanning);
+void rtl_btc_display_bt_coex_info(struct rtl_priv *rtlpriv, struct seq_file *m);
 void rtl_btc_record_pwr_mode(struct rtl_priv *rtlpriv, u8 *buf, u8 len);
 u8   rtl_btc_get_lps_val(struct rtl_priv *rtlpriv);
 u8   rtl_btc_get_rpwm_val(struct rtl_priv *rtlpriv);
diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c
index 3cb88825473e..cfea57efa7f4 100644
--- a/drivers/net/wireless/realtek/rtlwifi/core.c
+++ b/drivers/net/wireless/realtek/rtlwifi/core.c
@@ -345,9 +345,9 @@ static void rtl_op_remove_interface(struct ieee80211_hw *hw,
 	mutex_lock(&rtlpriv->locks.conf_mutex);
 
 	/* Free beacon resources */
-	if ((vif->type == NL80211_IFTYPE_AP) ||
-	    (vif->type == NL80211_IFTYPE_ADHOC) ||
-	    (vif->type == NL80211_IFTYPE_MESH_POINT)) {
+	if (vif->type == NL80211_IFTYPE_AP ||
+	    vif->type == NL80211_IFTYPE_ADHOC ||
+	    vif->type == NL80211_IFTYPE_MESH_POINT) {
 		if (mac->beacon_enabled == 1) {
 			mac->beacon_enabled = 0;
 			rtlpriv->cfg->ops->update_interrupt_mask(hw, 0,
@@ -858,8 +858,8 @@ static void rtl_op_configure_filter(struct ieee80211_hw *hw,
 	 * here just used for linked scanning, & linked
 	 * and nolink check bssid is set in set network_type
 	 */
-	if ((changed_flags & FIF_BCN_PRBRESP_PROMISC) &&
-	    (mac->link_state >= MAC80211_LINKED)) {
+	if (changed_flags & FIF_BCN_PRBRESP_PROMISC &&
+	    mac->link_state >= MAC80211_LINKED) {
 		if (mac->opmode != NL80211_IFTYPE_AP &&
 		    mac->opmode != NL80211_IFTYPE_MESH_POINT) {
 			if (*new_flags & FIF_BCN_PRBRESP_PROMISC)
@@ -1044,10 +1044,10 @@ static void rtl_op_bss_info_changed(struct ieee80211_hw *hw,
 	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
 
 	mutex_lock(&rtlpriv->locks.conf_mutex);
-	if ((vif->type == NL80211_IFTYPE_ADHOC) ||
-	    (vif->type == NL80211_IFTYPE_AP) ||
-	    (vif->type == NL80211_IFTYPE_MESH_POINT)) {
-		if ((changed & BSS_CHANGED_BEACON) ||
+	if (vif->type == NL80211_IFTYPE_ADHOC ||
+	    vif->type == NL80211_IFTYPE_AP ||
+	    vif->type == NL80211_IFTYPE_MESH_POINT) {
+		if (changed & BSS_CHANGED_BEACON ||
 		    (changed & BSS_CHANGED_BEACON_ENABLED &&
 		     bss_conf->enable_beacon)) {
 			if (mac->beacon_enabled == 0) {
@@ -1162,6 +1162,8 @@ static void rtl_op_bss_info_changed(struct ieee80211_hw *hw,
 			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
 				 "BSS_CHANGED_ASSOC\n");
 		} else {
+			struct cfg80211_bss *bss = NULL;
+
 			mstatus = RT_MEDIA_DISCONNECT;
 
 			if (mac->link_state == MAC80211_LINKED)
@@ -1169,6 +1171,22 @@ static void rtl_op_bss_info_changed(struct ieee80211_hw *hw,
 			if (ppsc->p2p_ps_info.p2p_ps_mode > P2P_PS_NONE)
 				rtl_p2p_ps_cmd(hw, P2P_PS_DISABLE);
 			mac->link_state = MAC80211_NOLINK;
+
+			bss = cfg80211_get_bss(hw->wiphy, NULL,
+					       (u8 *)mac->bssid, NULL, 0,
+					       IEEE80211_BSS_TYPE_ESS,
+					       IEEE80211_PRIVACY_OFF);
+
+			RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+				 "bssid = %pMF\n", mac->bssid);
+
+			if (bss) {
+				cfg80211_unlink_bss(hw->wiphy, bss);
+				cfg80211_put_bss(hw->wiphy, bss);
+				RT_TRACE(rtlpriv, COMP_MAC80211, DBG_DMESG,
+					 "cfg80211_unlink !!\n");
+			}
+
 			eth_zero_addr(mac->bssid);
 			mac->vendor = PEER_UNKNOWN;
 			mac->mode = 0;
@@ -1435,6 +1453,9 @@ static void rtl_op_sw_scan_start(struct ieee80211_hw *hw,
 
 	if (rtlpriv->cfg->ops->get_btc_status())
 		rtlpriv->btcoexist.btc_ops->btc_scan_notify(rtlpriv, 1);
+	else if (rtlpriv->btcoexist.btc_ops)
+		rtlpriv->btcoexist.btc_ops->btc_scan_notify_wifi_only(rtlpriv,
+								      1);
 
 	if (rtlpriv->dm.supp_phymode_switch) {
 		if (rtlpriv->cfg->ops->chk_switch_dmdp)
@@ -1490,6 +1511,9 @@ static void rtl_op_sw_scan_complete(struct ieee80211_hw *hw,
 	rtlpriv->cfg->ops->scan_operation_backup(hw, SCAN_OPT_RESTORE);
 	if (rtlpriv->cfg->ops->get_btc_status())
 		rtlpriv->btcoexist.btc_ops->btc_scan_notify(rtlpriv, 0);
+	else if (rtlpriv->btcoexist.btc_ops)
+		rtlpriv->btcoexist.btc_ops->btc_scan_notify_wifi_only(rtlpriv,
+								      0);
 }
 
 static int rtl_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
@@ -1513,9 +1537,9 @@ static int rtl_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 		return -ENOSPC;	/*User disabled HW-crypto */
 	}
 	/* To support IBSS, use sw-crypto for GTK */
-	if (((vif->type == NL80211_IFTYPE_ADHOC) ||
-	    (vif->type == NL80211_IFTYPE_MESH_POINT)) &&
-	   !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
+	if ((vif->type == NL80211_IFTYPE_ADHOC ||
+	     vif->type == NL80211_IFTYPE_MESH_POINT) &&
+	    !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
 		return -ENOSPC;
 	RT_TRACE(rtlpriv, COMP_SEC, DBG_DMESG,
 		 "%s hardware based encryption for keyidx: %d, mac: %pM\n",
@@ -1588,7 +1612,7 @@ static int rtl_op_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
 			rtlpriv->cfg->ops->enable_hw_sec(hw);
 		}
 	} else {
-		if ((!group_key) || (vif->type == NL80211_IFTYPE_ADHOC) ||
+		if (!group_key || vif->type == NL80211_IFTYPE_ADHOC ||
 		    rtlpriv->sec.pairwise_enc_algorithm == NO_ENCRYPTION) {
 			if (rtlpriv->sec.pairwise_enc_algorithm ==
 			    NO_ENCRYPTION &&
@@ -1775,7 +1799,7 @@ bool rtl_hal_pwrseqcmdparsing(struct rtl_priv *rtlpriv, u8 cut_version,
 				break;
 			case PWR_CMD_WRITE:
 				RT_TRACE(rtlpriv, COMP_INIT, DBG_TRACE,
-					"rtl_hal_pwrseqcmdparsing(): PWR_CMD_WRITE\n");
+					 "%s(): PWR_CMD_WRITE\n", __func__);
 				offset = GET_PWR_CFG_OFFSET(cfg_cmd);
 
 				/*Read the value from system register*/
diff --git a/drivers/net/wireless/realtek/rtlwifi/debug.c b/drivers/net/wireless/realtek/rtlwifi/debug.c
index 38fef6dbb44b..d70385be9976 100644
--- a/drivers/net/wireless/realtek/rtlwifi/debug.c
+++ b/drivers/net/wireless/realtek/rtlwifi/debug.c
@@ -23,15 +23,17 @@
  *****************************************************************************/
 
 #include "wifi.h"
+#include "cam.h"
 
 #include <linux/moduleparam.h>
+#include <linux/vmalloc.h>
 
 #ifdef CONFIG_RTLWIFI_DEBUG
 void _rtl_dbg_trace(struct rtl_priv *rtlpriv, u64 comp, int level,
 		    const char *fmt, ...)
 {
 	if (unlikely((comp & rtlpriv->cfg->mod_params->debug_mask) &&
-		     (level <= rtlpriv->cfg->mod_params->debug_level))) {
+		     level <= rtlpriv->cfg->mod_params->debug_level)) {
 		struct va_format vaf;
 		va_list args;
 
@@ -51,7 +53,7 @@ void _rtl_dbg_print(struct rtl_priv *rtlpriv, u64 comp, int level,
 		    const char *fmt, ...)
 {
 	if (unlikely((comp & rtlpriv->cfg->mod_params->debug_mask) &&
-		     (level <= rtlpriv->cfg->mod_params->debug_level))) {
+		     level <= rtlpriv->cfg->mod_params->debug_level)) {
 		struct va_format vaf;
 		va_list args;
 
@@ -81,4 +83,481 @@ void _rtl_dbg_print_data(struct rtl_priv *rtlpriv, u64 comp, int level,
 }
 EXPORT_SYMBOL_GPL(_rtl_dbg_print_data);
 
+struct rtl_debugfs_priv {
+	struct rtl_priv *rtlpriv;
+	int (*cb_read)(struct seq_file *m, void *v);
+	ssize_t (*cb_write)(struct file *filp, const char __user *buffer,
+			    size_t count, loff_t *loff);
+	u32 cb_data;
+};
+
+static struct dentry *debugfs_topdir;
+
+static int rtl_debug_get_common(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+
+	return debugfs_priv->cb_read(m, v);
+}
+
+static int dl_debug_open_common(struct inode *inode, struct file *file)
+{
+	return single_open(file, rtl_debug_get_common, inode->i_private);
+}
+
+static const struct file_operations file_ops_common = {
+	.open = dl_debug_open_common,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = seq_release,
+};
+
+static int rtl_debug_get_mac_page(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	u32 page = debugfs_priv->cb_data;
+	int i, n;
+	int max = 0xff;
+
+	for (n = 0; n <= max; ) {
+		seq_printf(m, "\n%8.8x  ", n + page);
+		for (i = 0; i < 4 && n <= max; i++, n += 4)
+			seq_printf(m, "%8.8x    ",
+				   rtl_read_dword(rtlpriv, (page | n)));
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_MAC_SERIES(page, addr)			\
+static struct rtl_debugfs_priv rtl_debug_priv_mac_ ##page = {	\
+	.cb_read = rtl_debug_get_mac_page,			\
+	.cb_data = addr,					\
+}
+
+RTL_DEBUG_IMPL_MAC_SERIES(0, 0x0000);
+RTL_DEBUG_IMPL_MAC_SERIES(1, 0x0100);
+RTL_DEBUG_IMPL_MAC_SERIES(2, 0x0200);
+RTL_DEBUG_IMPL_MAC_SERIES(3, 0x0300);
+RTL_DEBUG_IMPL_MAC_SERIES(4, 0x0400);
+RTL_DEBUG_IMPL_MAC_SERIES(5, 0x0500);
+RTL_DEBUG_IMPL_MAC_SERIES(6, 0x0600);
+RTL_DEBUG_IMPL_MAC_SERIES(7, 0x0700);
+RTL_DEBUG_IMPL_MAC_SERIES(10, 0x1000);
+RTL_DEBUG_IMPL_MAC_SERIES(11, 0x1100);
+RTL_DEBUG_IMPL_MAC_SERIES(12, 0x1200);
+RTL_DEBUG_IMPL_MAC_SERIES(13, 0x1300);
+RTL_DEBUG_IMPL_MAC_SERIES(14, 0x1400);
+RTL_DEBUG_IMPL_MAC_SERIES(15, 0x1500);
+RTL_DEBUG_IMPL_MAC_SERIES(16, 0x1600);
+RTL_DEBUG_IMPL_MAC_SERIES(17, 0x1700);
+
+static int rtl_debug_get_bb_page(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	u32 page = debugfs_priv->cb_data;
+	int i, n;
+	int max = 0xff;
+
+	for (n = 0; n <= max; ) {
+		seq_printf(m, "\n%8.8x  ", n + page);
+		for (i = 0; i < 4 && n <= max; i++, n += 4)
+			seq_printf(m, "%8.8x    ",
+				   rtl_get_bbreg(hw, (page | n), 0xffffffff));
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_BB_SERIES(page, addr)			\
+static struct rtl_debugfs_priv rtl_debug_priv_bb_ ##page = {	\
+	.cb_read = rtl_debug_get_bb_page,			\
+	.cb_data = addr,					\
+}
+
+RTL_DEBUG_IMPL_BB_SERIES(8, 0x0800);
+RTL_DEBUG_IMPL_BB_SERIES(9, 0x0900);
+RTL_DEBUG_IMPL_BB_SERIES(a, 0x0a00);
+RTL_DEBUG_IMPL_BB_SERIES(b, 0x0b00);
+RTL_DEBUG_IMPL_BB_SERIES(c, 0x0c00);
+RTL_DEBUG_IMPL_BB_SERIES(d, 0x0d00);
+RTL_DEBUG_IMPL_BB_SERIES(e, 0x0e00);
+RTL_DEBUG_IMPL_BB_SERIES(f, 0x0f00);
+RTL_DEBUG_IMPL_BB_SERIES(18, 0x1800);
+RTL_DEBUG_IMPL_BB_SERIES(19, 0x1900);
+RTL_DEBUG_IMPL_BB_SERIES(1a, 0x1a00);
+RTL_DEBUG_IMPL_BB_SERIES(1b, 0x1b00);
+RTL_DEBUG_IMPL_BB_SERIES(1c, 0x1c00);
+RTL_DEBUG_IMPL_BB_SERIES(1d, 0x1d00);
+RTL_DEBUG_IMPL_BB_SERIES(1e, 0x1e00);
+RTL_DEBUG_IMPL_BB_SERIES(1f, 0x1f00);
+
+static int rtl_debug_get_reg_rf(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	enum radio_path rfpath = debugfs_priv->cb_data;
+	int i, n;
+	int max = 0x40;
+
+	if (IS_HARDWARE_TYPE_8822B(rtlpriv))
+		max = 0xff;
+
+	seq_printf(m, "\nPATH(%d)", rfpath);
+
+	for (n = 0; n <= max; ) {
+		seq_printf(m, "\n%8.8x  ", n);
+		for (i = 0; i < 4 && n <= max; n += 1, i++)
+			seq_printf(m, "%8.8x    ",
+				   rtl_get_rfreg(hw, rfpath, n, 0xffffffff));
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_RF_SERIES(page, addr)			\
+static struct rtl_debugfs_priv rtl_debug_priv_rf_ ##page = {	\
+	.cb_read = rtl_debug_get_reg_rf,			\
+	.cb_data = addr,					\
+}
+
+RTL_DEBUG_IMPL_RF_SERIES(a, RF90_PATH_A);
+RTL_DEBUG_IMPL_RF_SERIES(b, RF90_PATH_B);
+
+static int rtl_debug_get_cam_register(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	int start = debugfs_priv->cb_data;
+	u32 target_cmd = 0;
+	u32 target_val = 0;
+	u8 entry_i = 0;
+	u32 ulstatus;
+	int i = 100, j = 0;
+	int end = (start + 11 > TOTAL_CAM_ENTRY ? TOTAL_CAM_ENTRY : start + 11);
+
+	/* This dump the current register page */
+	seq_printf(m,
+		   "\n#################### SECURITY CAM (%d-%d) ##################\n",
+		   start, end - 1);
+
+	for (j = start; j < end; j++) {
+		seq_printf(m, "\nD:  %2x > ", j);
+		for (entry_i = 0; entry_i < CAM_CONTENT_COUNT; entry_i++) {
+			/* polling bit, and No Write enable, and address  */
+			target_cmd = entry_i + CAM_CONTENT_COUNT * j;
+			target_cmd = target_cmd | BIT(31);
+
+			/* Check polling bit is clear */
+			while ((i--) >= 0) {
+				ulstatus =
+				    rtl_read_dword(rtlpriv,
+						   rtlpriv->cfg->maps[RWCAM]);
+				if (ulstatus & BIT(31))
+					continue;
+				else
+					break;
+			}
+
+			rtl_write_dword(rtlpriv, rtlpriv->cfg->maps[RWCAM],
+					target_cmd);
+			target_val = rtl_read_dword(rtlpriv,
+						    rtlpriv->cfg->maps[RCAMO]);
+			seq_printf(m, "%8.8x ", target_val);
+		}
+	}
+	seq_puts(m, "\n");
+	return 0;
+}
+
+#define RTL_DEBUG_IMPL_CAM_SERIES(page, addr)			\
+static struct rtl_debugfs_priv rtl_debug_priv_cam_ ##page = {	\
+	.cb_read = rtl_debug_get_cam_register,			\
+	.cb_data = addr,					\
+}
+
+RTL_DEBUG_IMPL_CAM_SERIES(1, 0);
+RTL_DEBUG_IMPL_CAM_SERIES(2, 11);
+RTL_DEBUG_IMPL_CAM_SERIES(3, 22);
+
+static int rtl_debug_get_btcoex(struct seq_file *m, void *v)
+{
+	struct rtl_debugfs_priv *debugfs_priv = m->private;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+
+	if (rtlpriv->cfg->ops->get_btc_status())
+		rtlpriv->btcoexist.btc_ops->btc_display_bt_coex_info(rtlpriv,
+								     m);
+
+	seq_puts(m, "\n");
+
+	return 0;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_btcoex = {
+	.cb_read = rtl_debug_get_btcoex,
+	.cb_data = 0,
+};
+
+static ssize_t rtl_debugfs_set_write_reg(struct file *filp,
+					 const char __user *buffer,
+					 size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	char tmp[32 + 1];
+	int tmp_len;
+	u32 addr, val, len;
+	int num;
+
+	if (count < 3)
+		return -EFAULT;
+
+	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
+
+	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
+		return count;
+
+	tmp[tmp_len] = '\0';
+
+	/* write BB/MAC register */
+	num = sscanf(tmp, "%x %x %x", &addr, &val, &len);
+
+	if (num !=  3)
+		return count;
+
+	switch (len) {
+	case 1:
+		rtl_write_byte(rtlpriv, addr, (u8)val);
+		break;
+	case 2:
+		rtl_write_word(rtlpriv, addr, (u16)val);
+		break;
+	case 4:
+		rtl_write_dword(rtlpriv, addr, val);
+		break;
+	default:
+		/*printk("error write length=%d", len);*/
+		break;
+	}
+
+	return count;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_write_reg = {
+	.cb_write = rtl_debugfs_set_write_reg,
+};
+
+static ssize_t rtl_debugfs_set_write_h2c(struct file *filp,
+					 const char __user *buffer,
+					 size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	char tmp[32 + 1];
+	int tmp_len;
+	u8 h2c_len, h2c_data_packed[8];
+	int h2c_data[8];	/* idx 0: cmd */
+	int i;
+
+	if (count < 3)
+		return -EFAULT;
+
+	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
+
+	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
+		return count;
+
+	tmp[tmp_len] = '\0';
+
+	h2c_len = sscanf(tmp, "%X %X %X %X %X %X %X %X",
+			 &h2c_data[0], &h2c_data[1],
+			 &h2c_data[2], &h2c_data[3],
+			 &h2c_data[4], &h2c_data[5],
+			 &h2c_data[6], &h2c_data[7]);
+
+	if (h2c_len <= 0)
+		return count;
+
+	for (i = 0; i < h2c_len; i++)
+		h2c_data_packed[i] = (u8)h2c_data[i];
+
+	rtlpriv->cfg->ops->fill_h2c_cmd(hw, h2c_data_packed[0],
+					h2c_len - 1,
+					&h2c_data_packed[1]);
+
+	return count;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_write_h2c = {
+	.cb_write = rtl_debugfs_set_write_h2c,
+};
+
+static ssize_t rtl_debugfs_set_write_rfreg(struct file *filp,
+					   const char __user *buffer,
+					    size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+	struct rtl_priv *rtlpriv = debugfs_priv->rtlpriv;
+	struct ieee80211_hw *hw = rtlpriv->hw;
+	char tmp[32 + 1];
+	int tmp_len;
+	int num;
+	int path;
+	u32 addr, bitmask, data;
+
+	if (count < 3)
+		return -EFAULT;
+
+	tmp_len = (count > sizeof(tmp) - 1 ? sizeof(tmp) - 1 : count);
+
+	if (!buffer || copy_from_user(tmp, buffer, tmp_len))
+		return count;
+
+	tmp[tmp_len] = '\0';
+
+	num = sscanf(tmp, "%X %X %X %X",
+		     &path, &addr, &bitmask, &data);
+
+	if (num != 4) {
+		RT_TRACE(rtlpriv, COMP_ERR, DBG_DMESG,
+			 "Format is <path> <addr> <mask> <data>\n");
+		return count;
+	}
+
+	rtl_set_rfreg(hw, path, addr, bitmask, data);
+
+	return count;
+}
+
+static struct rtl_debugfs_priv rtl_debug_priv_write_rfreg = {
+	.cb_write = rtl_debugfs_set_write_rfreg,
+};
+
+static int rtl_debugfs_close(struct inode *inode, struct file *filp)
+{
+	return 0;
+}
+
+static ssize_t rtl_debugfs_common_write(struct file *filp,
+					const char __user *buffer,
+					size_t count, loff_t *loff)
+{
+	struct rtl_debugfs_priv *debugfs_priv = filp->private_data;
+
+	return debugfs_priv->cb_write(filp, buffer, count, loff);
+}
+
+static const struct file_operations file_ops_common_write = {
+	.owner = THIS_MODULE,
+	.write = rtl_debugfs_common_write,
+	.open = simple_open,
+	.release = rtl_debugfs_close,
+};
+
+#define RTL_DEBUGFS_ADD_CORE(name, mode, fopname)			   \
+	do {								   \
+		rtl_debug_priv_ ##name.rtlpriv = rtlpriv;		   \
+		if (!debugfs_create_file(#name, mode,			   \
+					 parent, &rtl_debug_priv_ ##name,  \
+					 &file_ops_ ##fopname))		   \
+			pr_err("Unable to initialize debugfs:%s/%s\n",	   \
+			       rtlpriv->dbg.debugfs_name,		   \
+			       #name);					   \
+	} while (0)
+
+#define RTL_DEBUGFS_ADD(name)						   \
+		RTL_DEBUGFS_ADD_CORE(name, S_IFREG | 0444, common)
+#define RTL_DEBUGFS_ADD_W(name)						   \
+		RTL_DEBUGFS_ADD_CORE(name, S_IFREG | 0222, common_write)
+
+void rtl_debug_add_one(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+	struct rtl_efuse *rtlefuse = rtl_efuse(rtl_priv(hw));
+	struct dentry *parent;
+
+	snprintf(rtlpriv->dbg.debugfs_name, 18, "%pMF", rtlefuse->dev_addr);
+
+	rtlpriv->dbg.debugfs_dir =
+		debugfs_create_dir(rtlpriv->dbg.debugfs_name, debugfs_topdir);
+	if (!rtlpriv->dbg.debugfs_dir) {
+		pr_err("Unable to init debugfs:/%s/%s\n", rtlpriv->cfg->name,
+		       rtlpriv->dbg.debugfs_name);
+		return;
+	}
+
+	parent = rtlpriv->dbg.debugfs_dir;
+
+	RTL_DEBUGFS_ADD(mac_0);
+	RTL_DEBUGFS_ADD(mac_1);
+	RTL_DEBUGFS_ADD(mac_2);
+	RTL_DEBUGFS_ADD(mac_3);
+	RTL_DEBUGFS_ADD(mac_4);
+	RTL_DEBUGFS_ADD(mac_5);
+	RTL_DEBUGFS_ADD(mac_6);
+	RTL_DEBUGFS_ADD(mac_7);
+	RTL_DEBUGFS_ADD(bb_8);
+	RTL_DEBUGFS_ADD(bb_9);
+	RTL_DEBUGFS_ADD(bb_a);
+	RTL_DEBUGFS_ADD(bb_b);
+	RTL_DEBUGFS_ADD(bb_c);
+	RTL_DEBUGFS_ADD(bb_d);
+	RTL_DEBUGFS_ADD(bb_e);
+	RTL_DEBUGFS_ADD(bb_f);
+	RTL_DEBUGFS_ADD(mac_10);
+	RTL_DEBUGFS_ADD(mac_11);
+	RTL_DEBUGFS_ADD(mac_12);
+	RTL_DEBUGFS_ADD(mac_13);
+	RTL_DEBUGFS_ADD(mac_14);
+	RTL_DEBUGFS_ADD(mac_15);
+	RTL_DEBUGFS_ADD(mac_16);
+	RTL_DEBUGFS_ADD(mac_17);
+	RTL_DEBUGFS_ADD(bb_18);
+	RTL_DEBUGFS_ADD(bb_19);
+	RTL_DEBUGFS_ADD(bb_1a);
+	RTL_DEBUGFS_ADD(bb_1b);
+	RTL_DEBUGFS_ADD(bb_1c);
+	RTL_DEBUGFS_ADD(bb_1d);
+	RTL_DEBUGFS_ADD(bb_1e);
+	RTL_DEBUGFS_ADD(bb_1f);
+	RTL_DEBUGFS_ADD(rf_a);
+	RTL_DEBUGFS_ADD(rf_b);
+
+	RTL_DEBUGFS_ADD(cam_1);
+	RTL_DEBUGFS_ADD(cam_2);
+	RTL_DEBUGFS_ADD(cam_3);
+
+	RTL_DEBUGFS_ADD(btcoex);
+
+	RTL_DEBUGFS_ADD_W(write_reg);
+	RTL_DEBUGFS_ADD_W(write_h2c);
+	RTL_DEBUGFS_ADD_W(write_rfreg);
+}
+EXPORT_SYMBOL_GPL(rtl_debug_add_one);
+
+void rtl_debug_remove_one(struct ieee80211_hw *hw)
+{
+	struct rtl_priv *rtlpriv = rtl_priv(hw);
+
+	debugfs_remove_recursive(rtlpriv->dbg.debugfs_dir);
+	rtlpriv->dbg.debugfs_dir = NULL;
+}
+EXPORT_SYMBOL_GPL(rtl_debug_remove_one);
+
+void rtl_debugfs_add_topdir(void)
+{
+	debugfs_topdir = debugfs_create_dir("rtlwifi", NULL);
+}
+
+void rtl_debugfs_remove_topdir(void)
+{
+	debugfs_remove_recursive(debugfs_topdir);
+}
+
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/debug.h b/drivers/net/wireless/realtek/rtlwifi/debug.h
index 947718001457..ad6834af618b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/debug.h
+++ b/drivers/net/wireless/realtek/rtlwifi/debug.h
@@ -218,5 +218,17 @@ static inline void RT_PRINT_DATA(struct rtl_priv *rtlpriv,
 {
 }
 
+#endif
+
+#ifdef CONFIG_RTLWIFI_DEBUG
+void rtl_debug_add_one(struct ieee80211_hw *hw);
+void rtl_debug_remove_one(struct ieee80211_hw *hw);
+void rtl_debugfs_add_topdir(void);
+void rtl_debugfs_remove_topdir(void);
+#else
+#define rtl_debug_add_one(hw)
+#define rtl_debug_remove_one(hw)
+#define rtl_debugfs_add_topdir()
+#define rtl_debugfs_remove_topdir()
 #endif
 #endif
diff --git a/drivers/net/wireless/realtek/rtlwifi/efuse.c b/drivers/net/wireless/realtek/rtlwifi/efuse.c
index ef9acd466cca..35b50be633f1 100644
--- a/drivers/net/wireless/realtek/rtlwifi/efuse.c
+++ b/drivers/net/wireless/realtek/rtlwifi/efuse.c
@@ -257,11 +257,11 @@ void read_efuse(struct ieee80211_hw *hw, u16 _offset, u16 _size_byte, u8 *pbuf)
 			    sizeof(u8), GFP_ATOMIC);
 	if (!efuse_tbl)
 		return;
-	efuse_word = kzalloc(EFUSE_MAX_WORD_UNIT * sizeof(u16 *), GFP_ATOMIC);
+	efuse_word = kcalloc(EFUSE_MAX_WORD_UNIT, sizeof(u16 *), GFP_ATOMIC);
 	if (!efuse_word)
 		goto out;
 	for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
-		efuse_word[i] = kzalloc(efuse_max_section * sizeof(u16),
+		efuse_word[i] = kcalloc(efuse_max_section, sizeof(u16),
 					GFP_ATOMIC);
 		if (!efuse_word[i])
 			goto done;
diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c
index c2575b0b9440..01ccf8884831 100644
--- a/drivers/net/wireless/realtek/rtlwifi/pci.c
+++ b/drivers/net/wireless/realtek/rtlwifi/pci.c
@@ -59,6 +59,7 @@ static u8 _rtl_mac_to_hwqueue(struct ieee80211_hw *hw, struct sk_buff *skb)
 	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
 	__le16 fc = rtl_get_fc(skb);
 	u8 queue_index = skb_get_queue_mapping(skb);
+	struct ieee80211_hdr *hdr;
 
 	if (unlikely(ieee80211_is_beacon(fc)))
 		return BEACON_QUEUE;
@@ -67,6 +68,13 @@ static u8 _rtl_mac_to_hwqueue(struct ieee80211_hw *hw, struct sk_buff *skb)
 	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE)
 		if (ieee80211_is_nullfunc(fc))
 			return HIGH_QUEUE;
+	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8822BE) {
+		hdr = rtl_get_hdr(skb);
+
+		if (is_multicast_ether_addr(hdr->addr1) ||
+		    is_broadcast_ether_addr(hdr->addr1))
+			return HIGH_QUEUE;
+	}
 
 	return ac_to_hwq[queue_index];
 }
@@ -557,13 +565,6 @@ static void _rtl_pci_tx_isr(struct ieee80211_hw *hw, int prio)
 		else
 			entry = (u8 *)(&ring->desc[ring->idx]);
 
-		if (rtlpriv->cfg->ops->get_available_desc &&
-		    rtlpriv->cfg->ops->get_available_desc(hw, prio) <= 1) {
-			RT_TRACE(rtlpriv, (COMP_INTR | COMP_SEND), DBG_DMESG,
-				 "no available desc!\n");
-			return;
-		}
-
 		if (!rtlpriv->cfg->ops->is_tx_desc_closed(hw, prio, ring->idx))
 			return;
 		ring->idx = (ring->idx + 1) % ring->entries;
@@ -747,7 +748,7 @@ static void _rtl_pci_rx_interrupt(struct ieee80211_hw *hw)
 	u8 tmp_one;
 	bool unicast = false;
 	u8 hw_queue = 0;
-	unsigned int rx_remained_cnt;
+	unsigned int rx_remained_cnt = 0;
 	struct rtl_stats stats = {
 		.signal = 0,
 		.rate = 0,
@@ -768,7 +769,8 @@ static void _rtl_pci_rx_interrupt(struct ieee80211_hw *hw)
 		struct sk_buff *new_skb;
 
 		if (rtlpriv->use_new_trx_flow) {
-			rx_remained_cnt =
+			if (rx_remained_cnt == 0)
+				rx_remained_cnt =
 				rtlpriv->cfg->ops->rx_desc_buff_remained_cnt(hw,
 								      hw_queue);
 			if (rx_remained_cnt == 0)
@@ -924,10 +926,8 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
 	unsigned long flags;
-	u32 inta = 0;
-	u32 intb = 0;
-	u32 intc = 0;
-	u32 intd = 0;
+	struct rtl_int intvec = {0};
+
 	irqreturn_t ret = IRQ_HANDLED;
 
 	if (rtlpci->irq_enabled == 0)
@@ -937,47 +937,47 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 	rtlpriv->cfg->ops->disable_interrupt(hw);
 
 	/*read ISR: 4/8bytes */
-	rtlpriv->cfg->ops->interrupt_recognized(hw, &inta, &intb, &intc, &intd);
+	rtlpriv->cfg->ops->interrupt_recognized(hw, &intvec);
 
 	/*Shared IRQ or HW disappeared */
-	if (!inta || inta == 0xffff)
+	if (!intvec.inta || intvec.inta == 0xffff)
 		goto done;
 
 	/*<1> beacon related */
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_TBDOK])
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_TBDOK])
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
 			 "beacon ok interrupt!\n");
 
-	if (unlikely(inta & rtlpriv->cfg->maps[RTL_IMR_TBDER]))
+	if (unlikely(intvec.inta & rtlpriv->cfg->maps[RTL_IMR_TBDER]))
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
 			 "beacon err interrupt!\n");
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_BDOK])
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_BDOK])
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE, "beacon interrupt!\n");
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_BCNINT]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_BCNINT]) {
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
 			 "prepare beacon for interrupt!\n");
 		tasklet_schedule(&rtlpriv->works.irq_prepare_bcn_tasklet);
 	}
 
 	/*<2> Tx related */
-	if (unlikely(intb & rtlpriv->cfg->maps[RTL_IMR_TXFOVW]))
+	if (unlikely(intvec.intb & rtlpriv->cfg->maps[RTL_IMR_TXFOVW]))
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING, "IMR_TXFOVW!\n");
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_MGNTDOK]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_MGNTDOK]) {
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
 			 "Manage ok interrupt!\n");
 		_rtl_pci_tx_isr(hw, MGNT_QUEUE);
 	}
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_HIGHDOK]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_HIGHDOK]) {
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
 			 "HIGH_QUEUE ok interrupt!\n");
 		_rtl_pci_tx_isr(hw, HIGH_QUEUE);
 	}
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_BKDOK]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_BKDOK]) {
 		rtlpriv->link_info.num_tx_inperiod++;
 
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
@@ -985,7 +985,7 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 		_rtl_pci_tx_isr(hw, BK_QUEUE);
 	}
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_BEDOK]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_BEDOK]) {
 		rtlpriv->link_info.num_tx_inperiod++;
 
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
@@ -993,7 +993,7 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 		_rtl_pci_tx_isr(hw, BE_QUEUE);
 	}
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_VIDOK]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_VIDOK]) {
 		rtlpriv->link_info.num_tx_inperiod++;
 
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
@@ -1001,7 +1001,7 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 		_rtl_pci_tx_isr(hw, VI_QUEUE);
 	}
 
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_VODOK]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_VODOK]) {
 		rtlpriv->link_info.num_tx_inperiod++;
 
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
@@ -1010,7 +1010,7 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 	}
 
 	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8822BE) {
-		if (intd & rtlpriv->cfg->maps[RTL_IMR_H2CDOK]) {
+		if (intvec.intd & rtlpriv->cfg->maps[RTL_IMR_H2CDOK]) {
 			rtlpriv->link_info.num_tx_inperiod++;
 
 			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
@@ -1020,7 +1020,7 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 	}
 
 	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8192SE) {
-		if (inta & rtlpriv->cfg->maps[RTL_IMR_COMDOK]) {
+		if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_COMDOK]) {
 			rtlpriv->link_info.num_tx_inperiod++;
 
 			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
@@ -1030,25 +1030,25 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 	}
 
 	/*<3> Rx related */
-	if (inta & rtlpriv->cfg->maps[RTL_IMR_ROK]) {
+	if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_ROK]) {
 		RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE, "Rx ok interrupt!\n");
 		_rtl_pci_rx_interrupt(hw);
 	}
 
-	if (unlikely(inta & rtlpriv->cfg->maps[RTL_IMR_RDU])) {
+	if (unlikely(intvec.inta & rtlpriv->cfg->maps[RTL_IMR_RDU])) {
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING,
 			 "rx descriptor unavailable!\n");
 		_rtl_pci_rx_interrupt(hw);
 	}
 
-	if (unlikely(intb & rtlpriv->cfg->maps[RTL_IMR_RXFOVW])) {
+	if (unlikely(intvec.intb & rtlpriv->cfg->maps[RTL_IMR_RXFOVW])) {
 		RT_TRACE(rtlpriv, COMP_ERR, DBG_WARNING, "rx overflow !\n");
 		_rtl_pci_rx_interrupt(hw);
 	}
 
 	/*<4> fw related*/
 	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8723AE) {
-		if (inta & rtlpriv->cfg->maps[RTL_IMR_C2HCMD]) {
+		if (intvec.inta & rtlpriv->cfg->maps[RTL_IMR_C2HCMD]) {
 			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
 				 "firmware interrupt!\n");
 			queue_delayed_work(rtlpriv->works.rtl_wq,
@@ -1064,7 +1064,8 @@ static irqreturn_t _rtl_pci_interrupt(int irq, void *dev_id)
 	 */
 	if (rtlhal->hw_type == HARDWARE_TYPE_RTL8188EE ||
 	    rtlhal->hw_type == HARDWARE_TYPE_RTL8723BE) {
-		if (unlikely(inta & rtlpriv->cfg->maps[RTL_IMR_HSISR_IND])) {
+		if (unlikely(intvec.inta &
+		    rtlpriv->cfg->maps[RTL_IMR_HSISR_IND])) {
 			RT_TRACE(rtlpriv, COMP_INTR, DBG_TRACE,
 				 "hsisr interrupt!\n");
 			_rtl_pci_hs_interrupt(hw);
@@ -1250,7 +1251,6 @@ static int _rtl_pci_init_tx_ring(struct ieee80211_hw *hw,
 
 		rtlpci->tx_ring[prio].cur_tx_rp = 0;
 		rtlpci->tx_ring[prio].cur_tx_wp = 0;
-		rtlpci->tx_ring[prio].avl_desc = entries;
 	}
 
 	/* alloc dma for this ring */
@@ -1555,7 +1555,14 @@ int rtl_pci_reset_trx_ring(struct ieee80211_hw *hw)
 				dev_kfree_skb_irq(skb);
 				ring->idx = (ring->idx + 1) % ring->entries;
 			}
+
+			if (rtlpriv->use_new_trx_flow) {
+				rtlpci->tx_ring[i].cur_tx_rp = 0;
+				rtlpci->tx_ring[i].cur_tx_wp = 0;
+			}
+
 			ring->idx = 0;
+			ring->entries = rtlpci->txringcount[i];
 		}
 	}
 	spin_unlock_irqrestore(&rtlpriv->locks.irq_th_lock, flags);
@@ -1787,6 +1794,7 @@ static int rtl_pci_start(struct ieee80211_hw *hw)
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
 	struct rtl_mac *rtlmac = rtl_mac(rtl_priv(hw));
+	struct rtl_btc_ops *btc_ops = rtlpriv->btcoexist.btc_ops;
 
 	int err;
 
@@ -1796,9 +1804,12 @@ static int rtl_pci_start(struct ieee80211_hw *hw)
 	if (rtlpriv->cfg->ops->get_btc_status &&
 	    rtlpriv->cfg->ops->get_btc_status()) {
 		rtlpriv->btcoexist.btc_info.ap_num = 36;
-		rtlpriv->btcoexist.btc_ops->btc_init_variables(rtlpriv);
-		rtlpriv->btcoexist.btc_ops->btc_init_hal_vars(rtlpriv);
+		btc_ops->btc_init_variables(rtlpriv);
+		btc_ops->btc_init_hal_vars(rtlpriv);
+	} else if (btc_ops) {
+		btc_ops->btc_init_variables_wifi_only(rtlpriv);
 	}
+
 	err = rtlpriv->cfg->ops->hw_init(hw);
 	if (err) {
 		RT_TRACE(rtlpriv, COMP_INIT, DBG_DMESG,
@@ -1834,7 +1845,10 @@ static void rtl_pci_stop(struct ieee80211_hw *hw)
 	u8 rf_timeout = 0;
 
 	if (rtlpriv->cfg->ops->get_btc_status())
-		rtlpriv->btcoexist.btc_ops->btc_halt_notify();
+		rtlpriv->btcoexist.btc_ops->btc_halt_notify(rtlpriv);
+
+	if (rtlpriv->btcoexist.btc_ops)
+		rtlpriv->btcoexist.btc_ops->btc_deinit_variables(rtlpriv);
 
 	/*should be before disable interrupt&adapter
 	 *and will do it immediately.
@@ -2302,6 +2316,9 @@ int rtl_pci_probe(struct pci_dev *pdev,
 	}
 	rtlpriv->mac80211.mac80211_registered = 1;
 
+	/* add for debug */
+	rtl_debug_add_one(hw);
+
 	/*init rfkill */
 	rtl_init_rfkill(hw);	/* Init PCI sw */
 
@@ -2350,6 +2367,9 @@ void rtl_pci_disconnect(struct pci_dev *pdev)
 	wait_for_completion(&rtlpriv->firmware_loading_complete);
 	clear_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status);
 
+	/* remove form debug */
+	rtl_debug_remove_one(hw);
+
 	/*ieee80211_unregister_hw will call ops_stop */
 	if (rtlmac->mac80211_registered == 1) {
 		ieee80211_unregister_hw(hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.h b/drivers/net/wireless/realtek/rtlwifi/pci.h
index e7d070e8da2d..3fb56c845a61 100644
--- a/drivers/net/wireless/realtek/rtlwifi/pci.h
+++ b/drivers/net/wireless/realtek/rtlwifi/pci.h
@@ -173,7 +173,6 @@ struct rtl8192_tx_ring {
 	/*add for new trx flow*/
 	struct rtl_tx_buffer_desc *buffer_desc; /*tx buffer descriptor*/
 	dma_addr_t buffer_desc_dma; /*tx bufferd desc dma memory*/
-	u16 avl_desc; /* available_desc_to_write */
 	u16 cur_tx_wp; /* current_tx_write_point */
 	u16 cur_tx_rp; /* current_tx_read_point */
 };
@@ -320,10 +319,10 @@ static inline void pci_write32_async(struct rtl_priv *rtlpriv,
 	writel(val, (u8 __iomem *)rtlpriv->io.pci_mem_start + addr);
 }
 
-static inline u16 calc_fifo_space(u16 rp, u16 wp)
+static inline u16 calc_fifo_space(u16 rp, u16 wp, u16 size)
 {
 	if (rp <= wp)
-		return RTL_PCI_MAX_RX_COUNT - 1 + rp - wp;
+		return size - 1 + rp - wp;
 	return rp - wp - 1;
 }
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/ps.c b/drivers/net/wireless/realtek/rtlwifi/ps.c
index 24c87fae5382..71af24e2e051 100644
--- a/drivers/net/wireless/realtek/rtlwifi/ps.c
+++ b/drivers/net/wireless/realtek/rtlwifi/ps.c
@@ -51,6 +51,11 @@ bool rtl_ps_enable_nic(struct ieee80211_hw *hw)
 			&rtlmac->retry_long);
 	RT_CLEAR_PS_LEVEL(ppsc, RT_RF_OFF_LEVL_HALT_NIC);
 
+	rtlpriv->cfg->ops->switch_channel(hw);
+	rtlpriv->cfg->ops->set_channel_access(hw);
+	rtlpriv->cfg->ops->set_bw_mode(hw,
+			cfg80211_get_chandef_type(&hw->conf.chandef));
+
 	/*<3> Enable Interrupt */
 	rtlpriv->cfg->ops->enable_interrupt(hw);
 
@@ -289,7 +294,7 @@ void rtl_ips_nic_on(struct ieee80211_hw *hw)
 
 	cancel_delayed_work(&rtlpriv->works.ips_nic_off_wq);
 
-	spin_lock(&rtlpriv->locks.ips_lock);
+	mutex_lock(&rtlpriv->locks.ips_mutex);
 	if (ppsc->inactiveps) {
 		rtstate = ppsc->rfpwr_state;
 
@@ -306,7 +311,7 @@ void rtl_ips_nic_on(struct ieee80211_hw *hw)
 									ppsc->inactive_pwrstate);
 		}
 	}
-	spin_unlock(&rtlpriv->locks.ips_lock);
+	mutex_unlock(&rtlpriv->locks.ips_mutex);
 }
 EXPORT_SYMBOL_GPL(rtl_ips_nic_on);
 
@@ -415,7 +420,6 @@ static void rtl_lps_enter_core(struct ieee80211_hw *hw)
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
 	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	unsigned long flag;
 
 	if (!ppsc->fwctrl_lps)
 		return;
@@ -436,7 +440,7 @@ static void rtl_lps_enter_core(struct ieee80211_hw *hw)
 	if (mac->link_state != MAC80211_LINKED)
 		return;
 
-	spin_lock_irqsave(&rtlpriv->locks.lps_lock, flag);
+	mutex_lock(&rtlpriv->locks.lps_mutex);
 
 	/* Don't need to check (ppsc->dot11_psmode == EACTIVE), because
 	 * bt_ccoexist may ask to enter lps.
@@ -446,7 +450,7 @@ static void rtl_lps_enter_core(struct ieee80211_hw *hw)
 		 "Enter 802.11 power save mode...\n");
 	rtl_lps_set_psmode(hw, EAUTOPS);
 
-	spin_unlock_irqrestore(&rtlpriv->locks.lps_lock, flag);
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
 }
 
 /* Interrupt safe routine to leave the leisure power save mode.*/
@@ -455,9 +459,8 @@ static void rtl_lps_leave_core(struct ieee80211_hw *hw)
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
 	struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw));
-	unsigned long flag;
 
-	spin_lock_irqsave(&rtlpriv->locks.lps_lock, flag);
+	mutex_lock(&rtlpriv->locks.lps_mutex);
 
 	if (ppsc->fwctrl_lps) {
 		if (ppsc->dot11_psmode != EACTIVE) {
@@ -478,7 +481,7 @@ static void rtl_lps_leave_core(struct ieee80211_hw *hw)
 			rtl_lps_set_psmode(hw, EACTIVE);
 		}
 	}
-	spin_unlock_irqrestore(&rtlpriv->locks.lps_lock, flag);
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
 }
 
 /* For sw LPS*/
@@ -568,7 +571,6 @@ void rtl_swlps_rf_awake(struct ieee80211_hw *hw)
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
-	unsigned long flag;
 
 	if (!rtlpriv->psc.swctrl_lps)
 		return;
@@ -581,9 +583,9 @@ void rtl_swlps_rf_awake(struct ieee80211_hw *hw)
 		RT_CLEAR_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM);
 	}
 
-	spin_lock_irqsave(&rtlpriv->locks.lps_lock, flag);
+	mutex_lock(&rtlpriv->locks.lps_mutex);
 	rtl_ps_set_rf_state(hw, ERFON, RF_CHANGE_BY_PS);
-	spin_unlock_irqrestore(&rtlpriv->locks.lps_lock, flag);
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
 }
 
 void rtl_swlps_rfon_wq_callback(void *data)
@@ -600,7 +602,6 @@ void rtl_swlps_rf_sleep(struct ieee80211_hw *hw)
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_mac *mac = rtl_mac(rtl_priv(hw));
 	struct rtl_ps_ctl *ppsc = rtl_psc(rtl_priv(hw));
-	unsigned long flag;
 	u8 sleep_intv;
 
 	if (!rtlpriv->psc.sw_ps_enabled)
@@ -624,9 +625,9 @@ void rtl_swlps_rf_sleep(struct ieee80211_hw *hw)
 	}
 	spin_unlock(&rtlpriv->locks.rf_ps_lock);
 
-	spin_lock_irqsave(&rtlpriv->locks.lps_lock, flag);
+	mutex_lock(&rtlpriv->locks.lps_mutex);
 	rtl_ps_set_rf_state(hw, ERFSLEEP, RF_CHANGE_BY_PS);
-	spin_unlock_irqrestore(&rtlpriv->locks.lps_lock, flag);
+	mutex_unlock(&rtlpriv->locks.lps_mutex);
 
 	if (ppsc->reg_rfps_level & RT_RF_OFF_LEVL_ASPM &&
 	    !RT_IN_PS_LEVEL(ppsc, RT_PS_LEVEL_ASPM)) {
diff --git a/drivers/net/wireless/realtek/rtlwifi/rc.c b/drivers/net/wireless/realtek/rtlwifi/rc.c
index 02811eda57cd..d1cb7d405618 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rc.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rc.c
@@ -123,7 +123,7 @@ static void _rtl_rc_rate_set_series(struct rtl_priv *rtlpriv,
 			if (sta && (sta->ht_cap.cap &
 				    IEEE80211_HT_CAP_SUP_WIDTH_20_40))
 				rate->flags |= IEEE80211_TX_RC_40_MHZ_WIDTH;
-			if (sta && (sta->vht_cap.vht_supported))
+			if (sta && sta->vht_cap.vht_supported)
 				rate->flags |= IEEE80211_TX_RC_80_MHZ_WIDTH;
 		} else {
 			if (mac->bw_40)
@@ -135,8 +135,8 @@ static void _rtl_rc_rate_set_series(struct rtl_priv *rtlpriv,
 		if (sgi_20 || sgi_40 || sgi_80)
 			rate->flags |= IEEE80211_TX_RC_SHORT_GI;
 		if (sta && sta->ht_cap.ht_supported &&
-		    ((wireless_mode == WIRELESS_MODE_N_5G) ||
-		     (wireless_mode == WIRELESS_MODE_N_24G)))
+		    (wireless_mode == WIRELESS_MODE_N_5G ||
+		     wireless_mode == WIRELESS_MODE_N_24G))
 			rate->flags |= IEEE80211_TX_RC_MCS;
 		if (sta && sta->vht_cap.vht_supported &&
 		    (wireless_mode == WIRELESS_MODE_AC_5G ||
@@ -216,8 +216,8 @@ static void rtl_tx_status(void *ppriv,
 
 	if (sta) {
 		/* Check if aggregation has to be enabled for this tid */
-		sta_entry = (struct rtl_sta_info *) sta->drv_priv;
-		if ((sta->ht_cap.ht_supported) &&
+		sta_entry = (struct rtl_sta_info *)sta->drv_priv;
+		if (sta->ht_cap.ht_supported &&
 		    !(skb->protocol == cpu_to_be16(ETH_P_PAE))) {
 			if (ieee80211_is_data_qos(fc)) {
 				u8 tid = rtl_get_tid(skb);
@@ -265,11 +265,9 @@ static void *rtl_rate_alloc_sta(void *ppriv,
 	struct rtl_priv *rtlpriv = ppriv;
 	struct rtl_rate_priv *rate_priv;
 
-	rate_priv = kzalloc(sizeof(struct rtl_rate_priv), gfp);
-	if (!rate_priv) {
-		pr_err("Unable to allocate private rc structure\n");
+	rate_priv = kzalloc(sizeof(*rate_priv), gfp);
+	if (!rate_priv)
 		return NULL;
-	}
 
 	rtlpriv->rate_priv = rate_priv;
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c
index a2eca669873b..63874512598b 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/fw.c
@@ -141,6 +141,8 @@ int rtl88e_download_fw(struct ieee80211_hw *hw,
 		return 1;
 
 	pfwheader = (struct rtlwifi_firmware_header *)rtlhal->pfirmware;
+	rtlhal->fw_version = le16_to_cpu(pfwheader->version);
+	rtlhal->fw_subversion = pfwheader->subversion;
 	pfwdata = rtlhal->pfirmware;
 	fwsize = rtlhal->fwsize;
 	RT_TRACE(rtlpriv, COMP_FW, DBG_DMESG,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
index e30a18e64ff5..988d5ac57d02 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.c
@@ -1472,17 +1472,16 @@ void rtl88ee_card_disable(struct ieee80211_hw *hw)
 }
 
 void rtl88ee_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd)
+				  struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, ISR, *p_inta);
+	intvec->inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, ISR, intvec->inta);
 
-	*p_intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
-	rtl_write_dword(rtlpriv, REG_HISRE, *p_intb);
+	intvec->intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
+	rtl_write_dword(rtlpriv, REG_HISRE, intvec->intb);
 
 }
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.h
index cdf49de1e6ed..214cd2a32018 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/hw.h
@@ -29,8 +29,7 @@
 void rtl88ee_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
 void rtl88ee_read_eeprom_info(struct ieee80211_hw *hw);
 void rtl88ee_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd);
+				  struct rtl_int *int_vec);
 int rtl88ee_hw_init(struct ieee80211_hw *hw);
 void rtl88ee_card_disable(struct ieee80211_hw *hw);
 void rtl88ee_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
index 0f4c86a28716..4a81e0ef4b8e 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.c
@@ -1375,19 +1375,13 @@ void rtl92ce_card_disable(struct ieee80211_hw *hw)
 }
 
 void rtl92ce_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd)
+				  struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, ISR, *p_inta);
-
-	/*
-	 * *p_intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
-	 * rtl_write_dword(rtlpriv, ISR + 4, *p_intb);
-	 */
+	intvec->inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, ISR, intvec->inta);
 }
 
 void rtl92ce_set_beacon_related_registers(struct ieee80211_hw *hw)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.h
index b5c8e2fc1ba2..6711ea1a75d9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/hw.h
@@ -42,8 +42,7 @@ static inline u8 rtl92c_get_chnl_group(u8 chnl)
 void rtl92ce_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
 void rtl92ce_read_eeprom_info(struct ieee80211_hw *hw);
 void rtl92ce_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd);
+				  struct rtl_int *int_vec);
 int rtl92ce_hw_init(struct ieee80211_hw *hw);
 void rtl92ce_card_disable(struct ieee80211_hw *hw);
 void rtl92ce_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
index 0da6c0136857..80123fd97221 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.c
@@ -1356,19 +1356,13 @@ void rtl92de_card_disable(struct ieee80211_hw *hw)
 }
 
 void rtl92de_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd)
+				  struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, ISR, *p_inta);
-
-	/*
-	 * *p_intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
-	 * rtl_write_dword(rtlpriv, ISR + 4, *p_intb);
-	 */
+	intvec->inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, ISR, intvec->inta);
 }
 
 void rtl92de_set_beacon_related_registers(struct ieee80211_hw *hw)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.h
index 9236aa91273d..e6c702e69ecf 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/hw.h
@@ -29,8 +29,7 @@
 void rtl92de_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
 void rtl92de_read_eeprom_info(struct ieee80211_hw *hw);
 void rtl92de_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd);
+				  struct rtl_int *int_vec);
 int rtl92de_hw_init(struct ieee80211_hw *hw);
 void rtl92de_card_disable(struct ieee80211_hw *hw);
 void rtl92de_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
index fe5da637e77a..fd7928fdbd1a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.c
@@ -1694,17 +1694,16 @@ void rtl92ee_card_disable(struct ieee80211_hw *hw)
 }
 
 void rtl92ee_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd)
+				  struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, ISR, *p_inta);
+	intvec->inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, ISR, intvec->inta);
 
-	*p_intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
-	rtl_write_dword(rtlpriv, REG_HISRE, *p_intb);
+	intvec->intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
+	rtl_write_dword(rtlpriv, REG_HISRE, intvec->intb);
 }
 
 void rtl92ee_set_beacon_related_registers(struct ieee80211_hw *hw)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.h
index cd6d3322f033..3a63bec9b0cc 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/hw.h
@@ -29,8 +29,7 @@
 void rtl92ee_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
 void rtl92ee_read_eeprom_info(struct ieee80211_hw *hw);
 void rtl92ee_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd);
+				  struct rtl_int *int_vec);
 int rtl92ee_hw_init(struct ieee80211_hw *hw);
 void rtl92ee_card_disable(struct ieee80211_hw *hw);
 void rtl92ee_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
index 12255682e890..4f7444331b07 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/trx.c
@@ -498,7 +498,8 @@ u16 rtl92ee_rx_desc_buff_remained_cnt(struct ieee80211_hw *hw, u8 queue_index)
 	if (!start_rx)
 		return 0;
 
-	remind_cnt = calc_fifo_space(read_point, write_point);
+	remind_cnt = calc_fifo_space(read_point, write_point,
+				     RTL_PCI_MAX_RX_COUNT);
 
 	if (remind_cnt == 0)
 		return 0;
@@ -548,7 +549,6 @@ static u16 get_desc_addr_fr_q_idx(u16 queue_index)
 
 u16 rtl92ee_get_available_desc(struct ieee80211_hw *hw, u8 q_idx)
 {
-	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	u16 point_diff = 0;
 	u16 current_tx_read_point = 0, current_tx_write_point = 0;
@@ -560,9 +560,9 @@ u16 rtl92ee_get_available_desc(struct ieee80211_hw *hw, u8 q_idx)
 	current_tx_write_point = (u16)((tmp_4byte) & 0x0fff);
 
 	point_diff = calc_fifo_space(current_tx_read_point,
-				     current_tx_write_point);
+				     current_tx_write_point,
+				     TX_DESC_NUM_92E);
 
-	rtlpci->tx_ring[q_idx].avl_desc = point_diff;
 	return point_diff;
 }
 
@@ -907,10 +907,6 @@ void rtl92ee_set_desc(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
 		      u8 desc_name, u8 *val)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	u16 cur_tx_rp = 0;
-	u16 cur_tx_wp = 0;
-	static bool over_run;
-	u32 tmp = 0;
 	u8 q_idx = *val;
 	bool dma64 = rtlpriv->cfg->mod_params->dma64;
 
@@ -931,38 +927,12 @@ void rtl92ee_set_desc(struct ieee80211_hw *hw, u8 *pdesc, bool istx,
 				return;
 			}
 
+			/* make sure tx desc is available by caller */
 			ring->cur_tx_wp = ((ring->cur_tx_wp + 1) % max_tx_desc);
 
-			if (over_run) {
-				ring->cur_tx_wp = 0;
-				over_run = false;
-			}
-			if (ring->avl_desc > 1) {
-				ring->avl_desc--;
-
-				rtl_write_word(rtlpriv,
-					       get_desc_addr_fr_q_idx(q_idx),
-					       ring->cur_tx_wp);
-			}
-
-			if (ring->avl_desc < (max_tx_desc - 15)) {
-				u16 point_diff = 0;
-
-				tmp =
-				  rtl_read_dword(rtlpriv,
-						 get_desc_addr_fr_q_idx(q_idx));
-				cur_tx_rp = (u16)((tmp >> 16) & 0x0fff);
-				cur_tx_wp = (u16)(tmp & 0x0fff);
-
-				ring->cur_tx_wp = cur_tx_wp;
-				ring->cur_tx_rp = cur_tx_rp;
-				point_diff = ((cur_tx_rp > cur_tx_wp) ?
-					      (cur_tx_rp - cur_tx_wp) :
-					      (TX_DESC_NUM_92E - 1 -
-					       cur_tx_wp + cur_tx_rp));
-
-				ring->avl_desc = point_diff;
-			}
+			rtl_write_word(rtlpriv,
+				       get_desc_addr_fr_q_idx(q_idx),
+				       ring->cur_tx_wp);
 		}
 		break;
 		}
@@ -1044,13 +1014,12 @@ bool rtl92ee_is_tx_desc_closed(struct ieee80211_hw *hw, u8 hw_queue, u16 index)
 {
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
-	u16 read_point, write_point, available_desc_num;
+	u16 read_point, write_point;
 	bool ret = false;
 	static u8 stop_report_cnt;
 	struct rtl8192_tx_ring *ring = &rtlpci->tx_ring[hw_queue];
 
 	{
-		u16 point_diff = 0;
 		u16 cur_tx_rp, cur_tx_wp;
 		u32 tmpu32 = 0;
 
@@ -1060,18 +1029,12 @@ bool rtl92ee_is_tx_desc_closed(struct ieee80211_hw *hw, u8 hw_queue, u16 index)
 		cur_tx_rp = (u16)((tmpu32 >> 16) & 0x0fff);
 		cur_tx_wp = (u16)(tmpu32 & 0x0fff);
 
-		ring->cur_tx_wp = cur_tx_wp;
+		/* don't need to update ring->cur_tx_wp */
 		ring->cur_tx_rp = cur_tx_rp;
-		point_diff = ((cur_tx_rp > cur_tx_wp) ?
-			      (cur_tx_rp - cur_tx_wp) :
-			      (TX_DESC_NUM_92E - cur_tx_wp + cur_tx_rp));
-
-		ring->avl_desc = point_diff;
 	}
 
 	read_point = ring->cur_tx_rp;
 	write_point = ring->cur_tx_wp;
-	available_desc_num = ring->avl_desc;
 
 	if (write_point > read_point) {
 		if (index < write_point && index >= read_point)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
index 76bf089cced4..30dea7b9bc17 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.c
@@ -1558,17 +1558,17 @@ void rtl92se_card_disable(struct ieee80211_hw *hw)
 	udelay(100);
 }
 
-void rtl92se_interrupt_recognized(struct ieee80211_hw *hw, u32 *p_inta,
-			     u32 *p_intb, u32 *p_intc, u32 *p_intd)
+void rtl92se_interrupt_recognized(struct ieee80211_hw *hw,
+				  struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, ISR, *p_inta);
+	intvec->inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, ISR, intvec->inta);
 
-	*p_intb = rtl_read_dword(rtlpriv, ISR + 4) & rtlpci->irq_mask[1];
-	rtl_write_dword(rtlpriv, ISR + 4, *p_intb);
+	intvec->intb = rtl_read_dword(rtlpriv, ISR + 4) & rtlpci->irq_mask[1];
+	rtl_write_dword(rtlpriv, ISR + 4, intvec->intb);
 }
 
 void rtl92se_set_beacon_related_registers(struct ieee80211_hw *hw)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.h
index 607056010974..fa836ceefc8f 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/hw.h
@@ -42,8 +42,7 @@ void rtl92se_get_hw_reg(struct ieee80211_hw *hw,
 			u8 variable, u8 *val);
 void rtl92se_read_eeprom_info(struct ieee80211_hw *hw);
 void rtl92se_interrupt_recognized(struct ieee80211_hw *hw,
-				  u32 *p_inta, u32 *p_intb,
-				  u32 *p_intc, u32 *p_intd);
+				  struct rtl_int *int_vec);
 int rtl92se_hw_init(struct ieee80211_hw *hw);
 void rtl92se_card_disable(struct ieee80211_hw *hw);
 void rtl92se_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
index c3f98d58124c..545115db507e 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.c
@@ -1340,14 +1340,13 @@ void rtl8723e_card_disable(struct ieee80211_hw *hw)
 }
 
 void rtl8723e_interrupt_recognized(struct ieee80211_hw *hw,
-				   u32 *p_inta, u32 *p_intb,
-				   u32 *p_intc, u32 *p_intd)
+				   struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, 0x3a0) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, 0x3a0, *p_inta);
+	intvec->inta = rtl_read_dword(rtlpriv, 0x3a0) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, 0x3a0, intvec->inta);
 }
 
 void rtl8723e_set_beacon_related_registers(struct ieee80211_hw *hw)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.h
index 19e467a37c72..c76e453f4f43 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/hw.h
@@ -34,8 +34,7 @@ void rtl8723e_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
 void rtl8723e_read_eeprom_info(struct ieee80211_hw *hw);
 
 void rtl8723e_interrupt_recognized(struct ieee80211_hw *hw,
-				   u32 *p_inta, u32 *p_intb,
-				   u32 *p_intc, u32 *p_intd);
+				   struct rtl_int *int_vec);
 int rtl8723e_hw_init(struct ieee80211_hw *hw);
 void rtl8723e_card_disable(struct ieee80211_hw *hw);
 void rtl8723e_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
index 7cd1ffa7d4a7..e7bbbc95cdb1 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.c
@@ -43,6 +43,7 @@
 #include "../pwrseqcmd.h"
 #include "pwrseq.h"
 #include "../btcoexist/rtl_btc.h"
+#include <linux/kernel.h>
 
 #define LLT_CONFIG	5
 
@@ -1124,7 +1125,8 @@ static void _rtl8723be_enable_aspm_back_door(struct ieee80211_hw *hw)
 
 	/* Configuration Space offset 0x70f BIT7 is used to control L0S */
 	tmp8 = _rtl8723be_dbi_read(rtlpriv, 0x70f);
-	_rtl8723be_dbi_write(rtlpriv, 0x70f, tmp8 | BIT(7));
+	_rtl8723be_dbi_write(rtlpriv, 0x70f, tmp8 | BIT(7) |
+			     ASPM_L1_LATENCY << 3);
 
 	/* Configuration Space offset 0x719 Bit3 is for L1
 	 * BIT4 is for clock request
@@ -1682,18 +1684,17 @@ void rtl8723be_card_disable(struct ieee80211_hw *hw)
 }
 
 void rtl8723be_interrupt_recognized(struct ieee80211_hw *hw,
-				    u32 *p_inta, u32 *p_intb,
-				    u32 *p_intc, u32 *p_intd)
+				    struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, ISR, *p_inta);
+	intvec->inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, ISR, intvec->inta);
 
-	*p_intb = rtl_read_dword(rtlpriv, REG_HISRE) &
-					rtlpci->irq_mask[1];
-	rtl_write_dword(rtlpriv, REG_HISRE, *p_intb);
+	intvec->intb = rtl_read_dword(rtlpriv, REG_HISRE) &
+				      rtlpci->irq_mask[1];
+	rtl_write_dword(rtlpriv, REG_HISRE, intvec->intb);
 }
 
 void rtl8723be_set_beacon_related_registers(struct ieee80211_hw *hw)
@@ -2127,28 +2128,28 @@ static void _rtl8723be_read_adapter_info(struct ieee80211_hw *hw,
 
 	if (rtlhal->oem_id == RT_CID_DEFAULT) {
 		/* Does this one have a Toshiba SMID from group 1? */
-		for (i = 0; i < sizeof(toshiba_smid1) / sizeof(u16); i++) {
+		for (i = 0; i < ARRAY_SIZE(toshiba_smid1); i++) {
 			if (rtlefuse->eeprom_smid == toshiba_smid1[i]) {
 				is_toshiba_smid1 = true;
 				break;
 			}
 		}
 		/* Does this one have a Toshiba SMID from group 2? */
-		for (i = 0; i < sizeof(toshiba_smid2) / sizeof(u16); i++) {
+		for (i = 0; i < ARRAY_SIZE(toshiba_smid2); i++) {
 			if (rtlefuse->eeprom_smid == toshiba_smid2[i]) {
 				is_toshiba_smid2 = true;
 				break;
 			}
 		}
 		/* Does this one have a Samsung SMID? */
-		for (i = 0; i < sizeof(samsung_smid) / sizeof(u16); i++) {
+		for (i = 0; i < ARRAY_SIZE(samsung_smid); i++) {
 			if (rtlefuse->eeprom_smid == samsung_smid[i]) {
 				is_samsung_smid = true;
 				break;
 			}
 		}
 		/* Does this one have a Lenovo SMID? */
-		for (i = 0; i < sizeof(lenovo_smid) / sizeof(u16); i++) {
+		for (i = 0; i < ARRAY_SIZE(lenovo_smid); i++) {
 			if (rtlefuse->eeprom_smid == lenovo_smid[i]) {
 				is_lenovo_smid = true;
 				break;
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.h
index 2215a792f6bf..ae856a19e81a 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/hw.h
@@ -30,8 +30,7 @@ void rtl8723be_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
 void rtl8723be_read_eeprom_info(struct ieee80211_hw *hw);
 
 void rtl8723be_interrupt_recognized(struct ieee80211_hw *hw,
-				    u32 *p_inta, u32 *p_intb,
-				    u32 *p_intc, u32 *p_intd);
+				    struct rtl_int *int_vec);
 int rtl8723be_hw_init(struct ieee80211_hw *hw);
 void rtl8723be_card_disable(struct ieee80211_hw *hw);
 void rtl8723be_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
index 9606641519e7..1263b12db5dc 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/phy.c
@@ -35,6 +35,7 @@
 #include "../rtl8723com/dm_common.h"
 #include "table.h"
 #include "trx.h"
+#include <linux/kernel.h>
 
 static bool _rtl8723be_phy_bb8723b_config_parafile(struct ieee80211_hw *hw);
 static bool _rtl8723be_phy_config_mac_with_headerfile(struct ieee80211_hw *hw);
@@ -1143,14 +1144,13 @@ void rtl8723be_phy_set_txpower_level(struct ieee80211_hw *hw, u8 channel)
 			     DESC92C_RATEMCS2, DESC92C_RATEMCS3,
 			     DESC92C_RATEMCS4, DESC92C_RATEMCS5,
 			     DESC92C_RATEMCS6, DESC92C_RATEMCS7};
-	u8 i, size;
+	u8 i;
 	u8 power_index;
 
 	if (!rtlefuse->txpwr_fromeprom)
 		return;
 
-	size = sizeof(cck_rates) / sizeof(u8);
-	for (i = 0; i < size; i++) {
+	for (i = 0; i < ARRAY_SIZE(cck_rates); i++) {
 		power_index = _rtl8723be_get_txpower_index(hw, RF90_PATH_A,
 					cck_rates[i],
 					rtl_priv(hw)->phy.current_chan_bw,
@@ -1158,8 +1158,7 @@ void rtl8723be_phy_set_txpower_level(struct ieee80211_hw *hw, u8 channel)
 		_rtl8723be_phy_set_txpower_index(hw, power_index, RF90_PATH_A,
 						 cck_rates[i]);
 	}
-	size = sizeof(ofdm_rates) / sizeof(u8);
-	for (i = 0; i < size; i++) {
+	for (i = 0; i < ARRAY_SIZE(ofdm_rates); i++) {
 		power_index = _rtl8723be_get_txpower_index(hw, RF90_PATH_A,
 					ofdm_rates[i],
 					rtl_priv(hw)->phy.current_chan_bw,
@@ -1167,8 +1166,7 @@ void rtl8723be_phy_set_txpower_level(struct ieee80211_hw *hw, u8 channel)
 		_rtl8723be_phy_set_txpower_index(hw, power_index, RF90_PATH_A,
 						 ofdm_rates[i]);
 	}
-	size = sizeof(ht_rates_1t) / sizeof(u8);
-	for (i = 0; i < size; i++) {
+	for (i = 0; i < ARRAY_SIZE(ht_rates_1t); i++) {
 		power_index = _rtl8723be_get_txpower_index(hw, RF90_PATH_A,
 					ht_rates_1t[i],
 					rtl_priv(hw)->phy.current_chan_bw,
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c
index 381c16b9b3a9..160fee8333ae 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/table.c
@@ -25,6 +25,7 @@
  *
  *****************************************************************************/
 
+#include <linux/kernel.h>
 #include "table.h"
 
 u32 RTL8723BEPHY_REG_1TARRAY[] = {
@@ -224,8 +225,7 @@ u32 RTL8723BEPHY_REG_1TARRAY[] = {
 
 };
 
-u32 RTL8723BEPHY_REG_1TARRAYLEN =
-	sizeof(RTL8723BEPHY_REG_1TARRAY) / sizeof(u32);
+u32 RTL8723BEPHY_REG_1TARRAYLEN = ARRAY_SIZE(RTL8723BEPHY_REG_1TARRAY);
 
 u32 RTL8723BEPHY_REG_ARRAY_PG[] = {
 	0, 0, 0, 0x00000e08, 0x0000ff00, 0x00003800,
@@ -236,8 +236,7 @@ u32 RTL8723BEPHY_REG_ARRAY_PG[] = {
 	0, 0, 0, 0x00000e14, 0xffffffff, 0x26303436
 };
 
-u32 RTL8723BEPHY_REG_ARRAY_PGLEN =
-		sizeof(RTL8723BEPHY_REG_ARRAY_PG) / sizeof(u32);
+u32 RTL8723BEPHY_REG_ARRAY_PGLEN = ARRAY_SIZE(RTL8723BEPHY_REG_ARRAY_PG);
 
 u32 RTL8723BE_RADIOA_1TARRAY[] = {
 		0x000, 0x00010000,
@@ -373,8 +372,7 @@ u32 RTL8723BE_RADIOA_1TARRAY[] = {
 
 };
 
-u32 RTL8723BE_RADIOA_1TARRAYLEN =
-	sizeof(RTL8723BE_RADIOA_1TARRAY) / sizeof(u32);
+u32 RTL8723BE_RADIOA_1TARRAYLEN = ARRAY_SIZE(RTL8723BE_RADIOA_1TARRAY);
 
 u32 RTL8723BEMAC_1T_ARRAY[] = {
 		0x02F, 0x00000030,
@@ -483,7 +481,7 @@ u32 RTL8723BEMAC_1T_ARRAY[] = {
 
 };
 
-u32 RTL8723BEMAC_1T_ARRAYLEN = sizeof(RTL8723BEMAC_1T_ARRAY) / sizeof(u32);
+u32 RTL8723BEMAC_1T_ARRAYLEN = ARRAY_SIZE(RTL8723BEMAC_1T_ARRAY);
 
 u32 RTL8723BEAGCTAB_1TARRAY[] = {
 		0xC78, 0xFD000001,
@@ -620,4 +618,4 @@ u32 RTL8723BEAGCTAB_1TARRAY[] = {
 
 };
 
-u32 RTL8723BEAGCTAB_1TARRAYLEN = sizeof(RTL8723BEAGCTAB_1TARRAY) / sizeof(u32);
+u32 RTL8723BEAGCTAB_1TARRAYLEN = ARRAY_SIZE(RTL8723BEAGCTAB_1TARRAY);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723com/fw_common.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723com/fw_common.c
index efa7e1262461..521039c5d4ce 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723com/fw_common.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723com/fw_common.c
@@ -196,10 +196,12 @@ int rtl8723_download_fw(struct ieee80211_hw *hw,
 	enum version_8723e version = rtlhal->version;
 	int max_page;
 
-	if (!rtlhal->pfirmware)
+	if (rtlpriv->max_fw_size == 0 || !rtlhal->pfirmware)
 		return 1;
 
 	pfwheader = (struct rtlwifi_firmware_header *)rtlhal->pfirmware;
+	rtlhal->fw_version = le16_to_cpu(pfwheader->version);
+	rtlhal->fw_subversion = pfwheader->subversion;
 	pfwdata = rtlhal->pfirmware;
 	fwsize = rtlhal->fwsize;
 
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
index 43e18c4c1e68..317c1b3101da 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.c
@@ -1123,7 +1123,7 @@ static u8 _rtl8821ae_dbi_read(struct rtl_priv *rtlpriv, u16 addr)
 	}
 	if (0 == tmp) {
 		read_addr = REG_DBI_RDATA + addr % 4;
-		ret = rtl_read_word(rtlpriv, read_addr);
+		ret = rtl_read_byte(rtlpriv, read_addr);
 	}
 	return ret;
 }
@@ -1165,7 +1165,8 @@ static void _rtl8821ae_enable_aspm_back_door(struct ieee80211_hw *hw)
 	}
 
 	tmp = _rtl8821ae_dbi_read(rtlpriv, 0x70f);
-	_rtl8821ae_dbi_write(rtlpriv, 0x70f, tmp | BIT(7));
+	_rtl8821ae_dbi_write(rtlpriv, 0x70f, tmp | BIT(7) |
+			     ASPM_L1_LATENCY << 3);
 
 	tmp = _rtl8821ae_dbi_read(rtlpriv, 0x719);
 	_rtl8821ae_dbi_write(rtlpriv, 0x719, tmp | BIT(3) | BIT(4));
@@ -2483,17 +2484,16 @@ void rtl8821ae_card_disable(struct ieee80211_hw *hw)
 }
 
 void rtl8821ae_interrupt_recognized(struct ieee80211_hw *hw,
-				    u32 *p_inta, u32 *p_intb,
-				    u32 *p_intc, u32 *p_intd)
+				    struct rtl_int *intvec)
 {
 	struct rtl_priv *rtlpriv = rtl_priv(hw);
 	struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw));
 
-	*p_inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
-	rtl_write_dword(rtlpriv, ISR, *p_inta);
+	intvec->inta = rtl_read_dword(rtlpriv, ISR) & rtlpci->irq_mask[0];
+	rtl_write_dword(rtlpriv, ISR, intvec->inta);
 
-	*p_intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
-	rtl_write_dword(rtlpriv, REG_HISRE, *p_intb);
+	intvec->intb = rtl_read_dword(rtlpriv, REG_HISRE) & rtlpci->irq_mask[1];
+	rtl_write_dword(rtlpriv, REG_HISRE, intvec->intb);
 }
 
 void rtl8821ae_set_beacon_related_registers(struct ieee80211_hw *hw)
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.h b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.h
index 284d259fe557..e2ab783a2ad9 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.h
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/hw.h
@@ -30,8 +30,7 @@ void rtl8821ae_get_hw_reg(struct ieee80211_hw *hw, u8 variable, u8 *val);
 void rtl8821ae_read_eeprom_info(struct ieee80211_hw *hw);
 
 void rtl8821ae_interrupt_recognized(struct ieee80211_hw *hw,
-				    u32 *p_inta, u32 *p_intb,
-				    u32 *p_intc, u32 *p_intd);
+				    struct rtl_int *int_vec);
 int rtl8821ae_hw_init(struct ieee80211_hw *hw);
 void rtl8821ae_card_disable(struct ieee80211_hw *hw);
 void rtl8821ae_enable_interrupt(struct ieee80211_hw *hw);
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c
index 408c4611e5de..f87f9d03b9fa 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/table.c
@@ -24,7 +24,7 @@
  * Larry Finger <Larry.Finger@lwfinger.net>
  *
  *****************************************************************************/
-
+#include <linux/kernel.h>
 #include "table.h"
 u32 RTL8812AE_PHY_REG_ARRAY[] = {
 		0x800, 0x8020D010,
@@ -258,8 +258,7 @@ u32 RTL8812AE_PHY_REG_ARRAY[] = {
 		0xEB8, 0x00508242,
 };
 
-u32 RTL8812AE_PHY_REG_1TARRAYLEN =
-	sizeof(RTL8812AE_PHY_REG_ARRAY) / sizeof(u32);
+u32 RTL8812AE_PHY_REG_1TARRAYLEN = ARRAY_SIZE(RTL8812AE_PHY_REG_ARRAY);
 
 u32 RTL8821AE_PHY_REG_ARRAY[] = {
 	0x800, 0x0020D090,
@@ -436,8 +435,7 @@ u32 RTL8821AE_PHY_REG_ARRAY[] = {
 	0xCB8, 0x00508240,
 };
 
-u32 RTL8821AE_PHY_REG_1TARRAYLEN =
-	sizeof(RTL8821AE_PHY_REG_ARRAY) / sizeof(u32);
+u32 RTL8821AE_PHY_REG_1TARRAYLEN = ARRAY_SIZE(RTL8821AE_PHY_REG_ARRAY);
 
 u32 RTL8812AE_PHY_REG_ARRAY_PG[] = {
 	0, 0, 0, 0x00000c20, 0xffffffff, 0x34363840,
@@ -488,8 +486,7 @@ u32 RTL8812AE_PHY_REG_ARRAY_PG[] = {
 	1, 1, 1, 0x00000e4c, 0xffffffff, 0x22242628
 };
 
-u32 RTL8812AE_PHY_REG_ARRAY_PGLEN =
-		sizeof(RTL8812AE_PHY_REG_ARRAY_PG) / sizeof(u32);
+u32 RTL8812AE_PHY_REG_ARRAY_PGLEN = ARRAY_SIZE(RTL8812AE_PHY_REG_ARRAY_PG);
 
 u32 RTL8821AE_PHY_REG_ARRAY_PG[] = {
 	0, 0, 0, 0x00000c20, 0xffffffff, 0x32343638,
@@ -509,8 +506,7 @@ u32 RTL8821AE_PHY_REG_ARRAY_PG[] = {
 	1, 0, 0, 0x00000c44, 0x0000ffff, 0x00002022
 };
 
-u32 RTL8821AE_PHY_REG_ARRAY_PGLEN =
-		sizeof(RTL8821AE_PHY_REG_ARRAY_PG) / sizeof(u32);
+u32 RTL8821AE_PHY_REG_ARRAY_PGLEN = ARRAY_SIZE(RTL8821AE_PHY_REG_ARRAY_PG);
 
 u32 RTL8812AE_RADIOA_ARRAY[] = {
 		0x000, 0x00010000,
@@ -927,7 +923,7 @@ u32 RTL8812AE_RADIOA_ARRAY[] = {
 		0x018, 0x0001712A,
 };
 
-u32 RTL8812AE_RADIOA_1TARRAYLEN = sizeof(RTL8812AE_RADIOA_ARRAY) / sizeof(u32);
+u32 RTL8812AE_RADIOA_1TARRAYLEN = ARRAY_SIZE(RTL8812AE_RADIOA_ARRAY);
 
 u32 RTL8812AE_RADIOB_ARRAY[] = {
 		0x056, 0x00051CF2,
@@ -1335,7 +1331,7 @@ u32 RTL8812AE_RADIOB_ARRAY[] = {
 		0x008, 0x00008400,
 };
 
-u32 RTL8812AE_RADIOB_1TARRAYLEN = sizeof(RTL8812AE_RADIOB_ARRAY) / sizeof(u32);
+u32 RTL8812AE_RADIOB_1TARRAYLEN = ARRAY_SIZE(RTL8812AE_RADIOB_ARRAY);
 
 u32 RTL8821AE_RADIOA_ARRAY[] = {
 		0x018, 0x0001712A,
@@ -1929,7 +1925,7 @@ u32 RTL8821AE_RADIOA_ARRAY[] = {
 
 };
 
-u32 RTL8821AE_RADIOA_1TARRAYLEN = sizeof(RTL8821AE_RADIOA_ARRAY) / sizeof(u32);
+u32 RTL8821AE_RADIOA_1TARRAYLEN = ARRAY_SIZE(RTL8821AE_RADIOA_ARRAY);
 
 u32 RTL8812AE_MAC_REG_ARRAY[] = {
 		0x010, 0x0000000C,
@@ -2041,7 +2037,7 @@ u32 RTL8812AE_MAC_REG_ARRAY[] = {
 		0x718, 0x00000040,
 };
 
-u32 RTL8812AE_MAC_1T_ARRAYLEN = sizeof(RTL8812AE_MAC_REG_ARRAY) / sizeof(u32);
+u32 RTL8812AE_MAC_1T_ARRAYLEN = ARRAY_SIZE(RTL8812AE_MAC_REG_ARRAY);
 
 u32 RTL8821AE_MAC_REG_ARRAY[] = {
 		0x428, 0x0000000A,
@@ -2143,7 +2139,7 @@ u32 RTL8821AE_MAC_REG_ARRAY[] = {
 		0x718, 0x00000040,
 };
 
-u32 RTL8821AE_MAC_1T_ARRAYLEN = sizeof(RTL8821AE_MAC_REG_ARRAY) / sizeof(u32);
+u32 RTL8821AE_MAC_1T_ARRAYLEN = ARRAY_SIZE(RTL8821AE_MAC_REG_ARRAY);
 
 u32 RTL8812AE_AGC_TAB_ARRAY[] = {
 	0x80000001, 0x00000000, 0x40000000, 0x00000000,
@@ -2479,8 +2475,7 @@ u32 RTL8812AE_AGC_TAB_ARRAY[] = {
 		0xE50, 0x00000020,
 };
 
-u32 RTL8812AE_AGC_TAB_1TARRAYLEN =
-	sizeof(RTL8812AE_AGC_TAB_ARRAY) / sizeof(u32);
+u32 RTL8812AE_AGC_TAB_1TARRAYLEN = ARRAY_SIZE(RTL8812AE_AGC_TAB_ARRAY);
 
 u32 RTL8821AE_AGC_TAB_ARRAY[] = {
 		0x81C, 0xBF000001,
@@ -2676,8 +2671,7 @@ u32 RTL8821AE_AGC_TAB_ARRAY[] = {
 		0xC50, 0x00000020,
 };
 
-u32 RTL8821AE_AGC_TAB_1TARRAYLEN =
-	sizeof(RTL8821AE_AGC_TAB_ARRAY) / sizeof(u32);
+u32 RTL8821AE_AGC_TAB_1TARRAYLEN = ARRAY_SIZE(RTL8821AE_AGC_TAB_ARRAY);
 
 /******************************************************************************
 *                           TXPWR_LMT.TXT
@@ -3250,7 +3244,7 @@ u8 *RTL8812AE_TXPWR_LMT[] = {
 	"MKK", "5G", "80M", "VHT", "2T", "155", "63"
 };
 
-u32 RTL8812AE_TXPWR_LMT_ARRAY_LEN = sizeof(RTL8812AE_TXPWR_LMT) / sizeof(u8 *);
+u32 RTL8812AE_TXPWR_LMT_ARRAY_LEN = ARRAY_SIZE(RTL8812AE_TXPWR_LMT);
 
 u8 *RTL8821AE_TXPWR_LMT[] = {
 	"FCC", "2.4G", "20M", "CCK", "1T", "01", "32",
@@ -3819,4 +3813,4 @@ u8 *RTL8821AE_TXPWR_LMT[] = {
 	"MKK", "5G", "80M", "VHT", "2T", "155", "63"
 };
 
-u32 RTL8821AE_TXPWR_LMT_ARRAY_LEN = sizeof(RTL8821AE_TXPWR_LMT) / sizeof(u8 *);
+u32 RTL8821AE_TXPWR_LMT_ARRAY_LEN = ARRAY_SIZE(RTL8821AE_TXPWR_LMT);
diff --git a/drivers/net/wireless/realtek/rtlwifi/usb.c b/drivers/net/wireless/realtek/rtlwifi/usb.c
index 39b033b3b53a..ce3103bb8ebb 100644
--- a/drivers/net/wireless/realtek/rtlwifi/usb.c
+++ b/drivers/net/wireless/realtek/rtlwifi/usb.c
@@ -962,7 +962,6 @@ static void _rtl_usb_tx_preprocess(struct ieee80211_hw *hw,
 	memset(&tcb_desc, 0, sizeof(struct rtl_tcb_desc));
 	if (ieee80211_is_auth(fc)) {
 		RT_TRACE(rtlpriv, COMP_SEND, DBG_DMESG, "MAC80211_LINKING\n");
-		rtl_ips_nic_on(hw);
 	}
 
 	if (rtlpriv->psc.sw_ps_enabled) {
diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h
index 92d4859ec906..46dcb7fef195 100644
--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h
+++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h
@@ -99,6 +99,7 @@
 #define RTL_USB_MAX_RX_COUNT			100
 #define QBSS_LOAD_SIZE				5
 #define MAX_WMMELE_LENGTH			64
+#define ASPM_L1_LATENCY				7
 
 #define TOTAL_CAM_ENTRY				32
 
@@ -873,6 +874,24 @@ enum ratr_table_mode {
 	RATR_INX_WIRELESS_AC_24N = 9,
 };
 
+enum ratr_table_mode_new {
+	RATEID_IDX_BGN_40M_2SS = 0,
+	RATEID_IDX_BGN_40M_1SS = 1,
+	RATEID_IDX_BGN_20M_2SS_BN = 2,
+	RATEID_IDX_BGN_20M_1SS_BN = 3,
+	RATEID_IDX_GN_N2SS = 4,
+	RATEID_IDX_GN_N1SS = 5,
+	RATEID_IDX_BG = 6,
+	RATEID_IDX_G = 7,
+	RATEID_IDX_B = 8,
+	RATEID_IDX_VHT_2SS = 9,
+	RATEID_IDX_VHT_1SS = 10,
+	RATEID_IDX_MIX1 = 11,
+	RATEID_IDX_MIX2 = 12,
+	RATEID_IDX_VHT_3SS = 13,
+	RATEID_IDX_BGN_3SS = 14,
+};
+
 enum rtl_link_state {
 	MAC80211_NOLINK = 0,
 	MAC80211_LINKING = 1,
@@ -931,6 +950,10 @@ enum package_type {
 	PACKAGE_TFBGA79
 };
 
+enum rtl_spec_ver {
+	RTL_SPEC_NEW_RATEID = BIT(0),	/* use ratr_table_mode_new */
+};
+
 struct octet_string {
 	u8 *octet;
 	u16 length;
@@ -2093,14 +2116,21 @@ struct rtl_wow_pattern {
 	u32 mask[4];
 };
 
+/* struct to store contents of interrupt vectors */
+struct rtl_int {
+	u32 inta;
+	u32 intb;
+	u32 intc;
+	u32 intd;
+};
+
 struct rtl_hal_ops {
 	int (*init_sw_vars) (struct ieee80211_hw *hw);
 	void (*deinit_sw_vars) (struct ieee80211_hw *hw);
 	void (*read_chip_version)(struct ieee80211_hw *hw);
 	void (*read_eeprom_info) (struct ieee80211_hw *hw);
 	void (*interrupt_recognized) (struct ieee80211_hw *hw,
-				      u32 *p_inta, u32 *p_intb,
-				      u32 *p_intc, u32 *p_intd);
+				      struct rtl_int *intvec);
 	int (*hw_init) (struct ieee80211_hw *hw);
 	void (*hw_disable) (struct ieee80211_hw *hw);
 	void (*hw_suspend) (struct ieee80211_hw *hw);
@@ -2308,6 +2338,7 @@ struct rtl_hal_cfg {
 	struct rtl_hal_ops *ops;
 	struct rtl_mod_params *mod_params;
 	struct rtl_hal_usbint_cfg *usb_interface_cfg;
+	enum rtl_spec_ver spec_ver;
 
 	/*this map used for some registers or vars
 	   defined int HAL but used in MAIN */
@@ -2318,17 +2349,14 @@ struct rtl_hal_cfg {
 struct rtl_locks {
 	/* mutex */
 	struct mutex conf_mutex;
-	struct mutex ps_mutex;
+	struct mutex ips_mutex;	/* mutex for enter/leave IPS */
+	struct mutex lps_mutex;	/* mutex for enter/leave LPS */
 
 	/*spin lock */
-	spinlock_t ips_lock;
 	spinlock_t irq_th_lock;
-	spinlock_t irq_pci_lock;
-	spinlock_t tx_lock;
 	spinlock_t h2c_lock;
 	spinlock_t rf_ps_lock;
 	spinlock_t rf_lock;
-	spinlock_t lps_lock;
 	spinlock_t waitq_lock;
 	spinlock_t entry_list_lock;
 	spinlock_t usb_lock;
@@ -2341,9 +2369,6 @@ struct rtl_locks {
 	/*Dual mac*/
 	spinlock_t cck_and_rw_pagea_lock;
 
-	/*Easy concurrent*/
-	spinlock_t check_sendpkt_lock;
-
 	spinlock_t iqk_lock;
 };
 
@@ -2374,6 +2399,12 @@ struct rtl_works {
 	struct work_struct fill_h2c_cmd;
 };
 
+struct rtl_debug {
+	/* add for debug */
+	struct dentry *debugfs_dir;
+	char debugfs_name[20];
+};
+
 #define MIMO_PS_STATIC			0
 #define MIMO_PS_DYNAMIC			1
 #define MIMO_PS_NOLIMIT			3
@@ -2493,6 +2524,9 @@ struct rtl_btc_info {
 struct bt_coexist_info {
 	struct rtl_btc_ops *btc_ops;
 	struct rtl_btc_info btc_info;
+	/* btc context */
+	void *btc_context;
+	void *wifi_only_context;
 	/* EEPROM BT info. */
 	u8 eeprom_bt_coexist;
 	u8 eeprom_bt_type;
@@ -2549,16 +2583,22 @@ struct bt_coexist_info {
 
 struct rtl_btc_ops {
 	void (*btc_init_variables) (struct rtl_priv *rtlpriv);
+	void (*btc_init_variables_wifi_only)(struct rtl_priv *rtlpriv);
+	void (*btc_deinit_variables)(struct rtl_priv *rtlpriv);
 	void (*btc_init_hal_vars) (struct rtl_priv *rtlpriv);
+	void (*btc_power_on_setting)(struct rtl_priv *rtlpriv);
 	void (*btc_init_hw_config) (struct rtl_priv *rtlpriv);
+	void (*btc_init_hw_config_wifi_only)(struct rtl_priv *rtlpriv);
 	void (*btc_ips_notify) (struct rtl_priv *rtlpriv, u8 type);
 	void (*btc_lps_notify)(struct rtl_priv *rtlpriv, u8 type);
 	void (*btc_scan_notify) (struct rtl_priv *rtlpriv, u8 scantype);
+	void (*btc_scan_notify_wifi_only)(struct rtl_priv *rtlpriv,
+					  u8 scantype);
 	void (*btc_connect_notify) (struct rtl_priv *rtlpriv, u8 action);
 	void (*btc_mediastatus_notify) (struct rtl_priv *rtlpriv,
 					enum rt_media_status mstatus);
 	void (*btc_periodical) (struct rtl_priv *rtlpriv);
-	void (*btc_halt_notify) (void);
+	void (*btc_halt_notify)(struct rtl_priv *rtlpriv);
 	void (*btc_btinfo_notify) (struct rtl_priv *rtlpriv,
 				   u8 *tmp_buf, u8 length);
 	void (*btc_btmpinfo_notify)(struct rtl_priv *rtlpriv,
@@ -2568,6 +2608,12 @@ struct rtl_btc_ops {
 	bool (*btc_is_bt_disabled) (struct rtl_priv *rtlpriv);
 	void (*btc_special_packet_notify)(struct rtl_priv *rtlpriv,
 					  u8 pkt_type);
+	void (*btc_switch_band_notify)(struct rtl_priv *rtlpriv, u8 type,
+				       bool scanning);
+	void (*btc_switch_band_notify_wifi_only)(struct rtl_priv *rtlpriv,
+						 u8 type, bool scanning);
+	void (*btc_display_bt_coex_info)(struct rtl_priv *rtlpriv,
+					 struct seq_file *m);
 	void (*btc_record_pwr_mode)(struct rtl_priv *rtlpriv, u8 *buf, u8 len);
 	u8   (*btc_get_lps_val)(struct rtl_priv *rtlpriv);
 	u8   (*btc_get_rpwm_val)(struct rtl_priv *rtlpriv);
@@ -2642,6 +2688,7 @@ struct rtl_priv {
 	/* c2hcmd list for kthread level access */
 	struct list_head c2hcmd_list;
 
+	struct rtl_debug dbg;
 	int max_fw_size;
 
 	/*
diff --git a/drivers/net/wireless/ti/wl1251/init.c b/drivers/net/wireless/ti/wl1251/init.c
index 1d799bffaa9f..e7d77acdf18c 100644
--- a/drivers/net/wireless/ti/wl1251/init.c
+++ b/drivers/net/wireless/ti/wl1251/init.c
@@ -310,10 +310,8 @@ static int wl1251_hw_init_data_path_config(struct wl1251 *wl)
 	/* asking for the data path parameters */
 	wl->data_path = kzalloc(sizeof(struct acx_data_path_params_resp),
 				GFP_KERNEL);
-	if (!wl->data_path) {
-		wl1251_error("Couldnt allocate data path parameters");
+	if (!wl->data_path)
 		return -ENOMEM;
-	}
 
 	ret = wl1251_acx_data_path_params(wl, wl->data_path);
 	if (ret < 0) {
diff --git a/drivers/net/wireless/ti/wl1251/main.c b/drivers/net/wireless/ti/wl1251/main.c
index 6d02c660b4ab..037defd10b91 100644
--- a/drivers/net/wireless/ti/wl1251/main.c
+++ b/drivers/net/wireless/ti/wl1251/main.c
@@ -1200,8 +1200,7 @@ static void wl1251_op_bss_info_changed(struct ieee80211_hw *hw,
 		WARN_ON(wl->bss_type != BSS_TYPE_STA_BSS);
 
 		enable = bss_conf->arp_addr_cnt == 1 && bss_conf->assoc;
-		wl1251_acx_arp_ip_filter(wl, enable, addr);
-
+		ret = wl1251_acx_arp_ip_filter(wl, enable, addr);
 		if (ret < 0)
 			goto out_sleep;
 	}
diff --git a/drivers/net/wireless/ti/wlcore/acx.c b/drivers/net/wireless/ti/wlcore/acx.c
index a4859993db3c..3ca9167d6146 100644
--- a/drivers/net/wireless/ti/wlcore/acx.c
+++ b/drivers/net/wireless/ti/wlcore/acx.c
@@ -146,7 +146,7 @@ int wl1271_acx_feature_cfg(struct wl1271 *wl, struct wl12xx_vif *wlvif)
 	ret = wl1271_cmd_configure(wl, ACX_FEATURE_CFG,
 				   feature, sizeof(*feature));
 	if (ret < 0) {
-		wl1271_error("Couldnt set HW encryption");
+		wl1271_error("Couldn't set HW encryption");
 		goto out;
 	}
 
diff --git a/drivers/net/wireless/ti/wlcore/acx.h b/drivers/net/wireless/ti/wlcore/acx.h
index f46d7fdf9a00..7011c5d9599f 100644
--- a/drivers/net/wireless/ti/wlcore/acx.h
+++ b/drivers/net/wireless/ti/wlcore/acx.h
@@ -1129,10 +1129,8 @@ int wl12xx_acx_config_hangover(struct wl1271 *wl);
 int wlcore_acx_average_rssi(struct wl1271 *wl, struct wl12xx_vif *wlvif,
 			    s8 *avg_rssi);
 
-#ifdef CONFIG_PM
 int wl1271_acx_default_rx_filter_enable(struct wl1271 *wl, bool enable,
 					enum rx_filter_action action);
 int wl1271_acx_set_rx_filter(struct wl1271 *wl, u8 index, bool enable,
 			     struct wl12xx_rx_filter *filter);
-#endif /* CONFIG_PM */
 #endif /* __WL1271_ACX_H__ */
diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c
index d47921a84509..09714034dbf1 100644
--- a/drivers/net/wireless/ti/wlcore/main.c
+++ b/drivers/net/wireless/ti/wlcore/main.c
@@ -42,6 +42,7 @@
 #include "sysfs.h"
 
 #define WL1271_BOOT_RETRIES 3
+#define WL1271_SUSPEND_SLEEP 100
 
 static char *fwlog_param;
 static int fwlog_mem_blocks = -1;
@@ -388,7 +389,6 @@ static void wl12xx_irq_update_links_status(struct wl1271 *wl,
 static int wlcore_fw_status(struct wl1271 *wl, struct wl_fw_status *status)
 {
 	struct wl12xx_vif *wlvif;
-	struct timespec ts;
 	u32 old_tx_blk_count = wl->tx_blocks_available;
 	int avail, freed_blocks;
 	int i;
@@ -485,8 +485,7 @@ static int wlcore_fw_status(struct wl1271 *wl, struct wl_fw_status *status)
 	}
 
 	/* update the host-chipset time offset */
-	getnstimeofday(&ts);
-	wl->time_offset = (timespec_to_ns(&ts) >> 10) -
+	wl->time_offset = (ktime_get_boot_ns() >> 10) -
 		(s64)(status->fw_localtime);
 
 	wl->fw_fast_lnk_map = status->link_fast_bitmap;
@@ -979,6 +978,24 @@ static int wlcore_fw_wakeup(struct wl1271 *wl)
 	return wlcore_raw_write32(wl, HW_ACCESS_ELP_CTRL_REG, ELPCTRL_WAKE_UP);
 }
 
+static int wlcore_fw_sleep(struct wl1271 *wl)
+{
+	int ret;
+
+	mutex_lock(&wl->mutex);
+	ret = wlcore_raw_write32(wl, HW_ACCESS_ELP_CTRL_REG, ELPCTRL_SLEEP);
+	if (ret < 0) {
+		wl12xx_queue_recovery_work(wl);
+		goto out;
+	}
+	set_bit(WL1271_FLAG_IN_ELP, &wl->flags);
+out:
+	mutex_unlock(&wl->mutex);
+	mdelay(WL1271_SUSPEND_SLEEP);
+
+	return 0;
+}
+
 static int wl1271_setup(struct wl1271 *wl)
 {
 	wl->raw_fw_status = kzalloc(wl->fw_status_len, GFP_KERNEL);
@@ -1326,7 +1343,6 @@ static struct sk_buff *wl12xx_alloc_dummy_packet(struct wl1271 *wl)
 }
 
 
-#ifdef CONFIG_PM
 static int
 wl1271_validate_wowlan_pattern(struct cfg80211_pkt_pattern *p)
 {
@@ -1698,8 +1714,8 @@ static void wl1271_configure_resume(struct wl1271 *wl, struct wl12xx_vif *wlvif)
 	}
 }
 
-static int wl1271_op_suspend(struct ieee80211_hw *hw,
-			    struct cfg80211_wowlan *wow)
+static int __maybe_unused wl1271_op_suspend(struct ieee80211_hw *hw,
+					    struct cfg80211_wowlan *wow)
 {
 	struct wl1271 *wl = hw->priv;
 	struct wl12xx_vif *wlvif;
@@ -1749,7 +1765,6 @@ static int wl1271_op_suspend(struct ieee80211_hw *hw,
 		goto out_sleep;
 
 out_sleep:
-	wl1271_ps_elp_sleep(wl);
 	mutex_unlock(&wl->mutex);
 
 	if (ret < 0) {
@@ -1782,10 +1797,19 @@ static int wl1271_op_suspend(struct ieee80211_hw *hw,
 	 */
 	cancel_delayed_work(&wl->tx_watchdog_work);
 
+	/*
+	 * Use an immediate call for allowing the firmware to go into power
+	 * save during suspend.
+	 * Using a workque for this last write was only hapenning on resume
+	 * leaving the firmware with power save disabled during suspend,
+	 * while consuming full power during wowlan suspend.
+	 */
+	wlcore_fw_sleep(wl);
+
 	return 0;
 }
 
-static int wl1271_op_resume(struct ieee80211_hw *hw)
+static int __maybe_unused wl1271_op_resume(struct ieee80211_hw *hw)
 {
 	struct wl1271 *wl = hw->priv;
 	struct wl12xx_vif *wlvif;
@@ -1869,7 +1893,6 @@ static int wl1271_op_resume(struct ieee80211_hw *hw)
 
 	return 0;
 }
-#endif
 
 static int wl1271_op_start(struct ieee80211_hw *hw)
 {
diff --git a/drivers/net/wireless/ti/wlcore/tx.c b/drivers/net/wireless/ti/wlcore/tx.c
index a3f5e9ca492a..00e9b4624dcf 100644
--- a/drivers/net/wireless/ti/wlcore/tx.c
+++ b/drivers/net/wireless/ti/wlcore/tx.c
@@ -264,7 +264,6 @@ static void wl1271_tx_fill_hdr(struct wl1271 *wl, struct wl12xx_vif *wlvif,
 			       struct sk_buff *skb, u32 extra,
 			       struct ieee80211_tx_info *control, u8 hlid)
 {
-	struct timespec ts;
 	struct wl1271_tx_hw_descr *desc;
 	int ac, rate_idx;
 	s64 hosttime;
@@ -287,8 +286,7 @@ static void wl1271_tx_fill_hdr(struct wl1271 *wl, struct wl12xx_vif *wlvif,
 	}
 
 	/* configure packet life time */
-	getnstimeofday(&ts);
-	hosttime = (timespec_to_ns(&ts) >> 10);
+	hosttime = (ktime_get_boot_ns() >> 10);
 	desc->start_time = cpu_to_le32(hosttime - wl->time_offset);
 
 	is_dummy = wl12xx_is_dummy_packet(wl, skb);
diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index 9bd7ddeeb6a5..3127bc8633ca 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -351,6 +351,9 @@ static int xennet_open(struct net_device *dev)
 	unsigned int i = 0;
 	struct netfront_queue *queue = NULL;
 
+	if (!np->queues)
+		return -ENODEV;
+
 	for (i = 0; i < num_queues; ++i) {
 		queue = &np->queues[i];
 		napi_enable(&queue->napi);
@@ -1358,18 +1361,8 @@ static int netfront_probe(struct xenbus_device *dev,
 #ifdef CONFIG_SYSFS
 	info->netdev->sysfs_groups[0] = &xennet_dev_group;
 #endif
-	err = register_netdev(info->netdev);
-	if (err) {
-		pr_warn("%s: register_netdev err=%d\n", __func__, err);
-		goto fail;
-	}
 
 	return 0;
-
- fail:
-	xennet_free_netdev(netdev);
-	dev_set_drvdata(&dev->dev, NULL);
-	return err;
 }
 
 static void xennet_end_access(int ref, void *page)
@@ -1737,8 +1730,6 @@ static void xennet_destroy_queues(struct netfront_info *info)
 {
 	unsigned int i;
 
-	rtnl_lock();
-
 	for (i = 0; i < info->netdev->real_num_tx_queues; i++) {
 		struct netfront_queue *queue = &info->queues[i];
 
@@ -1747,8 +1738,6 @@ static void xennet_destroy_queues(struct netfront_info *info)
 		netif_napi_del(&queue->napi);
 	}
 
-	rtnl_unlock();
-
 	kfree(info->queues);
 	info->queues = NULL;
 }
@@ -1764,8 +1753,6 @@ static int xennet_create_queues(struct netfront_info *info,
 	if (!info->queues)
 		return -ENOMEM;
 
-	rtnl_lock();
-
 	for (i = 0; i < *num_queues; i++) {
 		struct netfront_queue *queue = &info->queues[i];
 
@@ -1774,7 +1761,7 @@ static int xennet_create_queues(struct netfront_info *info,
 
 		ret = xennet_init_queue(queue);
 		if (ret < 0) {
-			dev_warn(&info->netdev->dev,
+			dev_warn(&info->xbdev->dev,
 				 "only created %d queues\n", i);
 			*num_queues = i;
 			break;
@@ -1788,10 +1775,8 @@ static int xennet_create_queues(struct netfront_info *info,
 
 	netif_set_real_num_tx_queues(info->netdev, *num_queues);
 
-	rtnl_unlock();
-
 	if (*num_queues == 0) {
-		dev_err(&info->netdev->dev, "no queues\n");
+		dev_err(&info->xbdev->dev, "no queues\n");
 		return -EINVAL;
 	}
 	return 0;
@@ -1828,6 +1813,7 @@ static int talk_to_netback(struct xenbus_device *dev,
 		goto out;
 	}
 
+	rtnl_lock();
 	if (info->queues)
 		xennet_destroy_queues(info);
 
@@ -1838,6 +1824,7 @@ static int talk_to_netback(struct xenbus_device *dev,
 		info->queues = NULL;
 		goto out;
 	}
+	rtnl_unlock();
 
 	/* Create shared ring, alloc event channel -- for each queue */
 	for (i = 0; i < num_queues; ++i) {
@@ -1934,8 +1921,10 @@ static int talk_to_netback(struct xenbus_device *dev,
 	xenbus_transaction_end(xbt, 1);
  destroy_ring:
 	xennet_disconnect_backend(info);
+	rtnl_lock();
 	xennet_destroy_queues(info);
  out:
+	rtnl_unlock();
 	device_unregister(&dev->dev);
 	return err;
 }
@@ -1965,6 +1954,15 @@ static int xennet_connect(struct net_device *dev)
 	netdev_update_features(dev);
 	rtnl_unlock();
 
+	if (dev->reg_state == NETREG_UNINITIALIZED) {
+		err = register_netdev(dev);
+		if (err) {
+			pr_warn("%s: register_netdev err=%d\n", __func__, err);
+			device_unregister(&np->xbdev->dev);
+			return err;
+		}
+	}
+
 	/*
 	 * All public and private state should now be sane.  Get
 	 * ready to start sending and receiving packets and give the driver
@@ -2007,7 +2005,10 @@ static void netback_changed(struct xenbus_device *dev,
 	case XenbusStateInitialised:
 	case XenbusStateReconfiguring:
 	case XenbusStateReconfigured:
+		break;
+
 	case XenbusStateUnknown:
+		wake_up_all(&module_unload_q);
 		break;
 
 	case XenbusStateInitWait:
@@ -2138,7 +2139,9 @@ static int xennet_remove(struct xenbus_device *dev)
 		xenbus_switch_state(dev, XenbusStateClosing);
 		wait_event(module_unload_q,
 			   xenbus_read_driver_state(dev->otherend) ==
-			   XenbusStateClosing);
+			   XenbusStateClosing ||
+			   xenbus_read_driver_state(dev->otherend) ==
+			   XenbusStateUnknown);
 
 		xenbus_switch_state(dev, XenbusStateClosed);
 		wait_event(module_unload_q,
@@ -2150,10 +2153,14 @@ static int xennet_remove(struct xenbus_device *dev)
 
 	xennet_disconnect_backend(info);
 
-	unregister_netdev(info->netdev);
+	if (info->netdev->reg_state == NETREG_REGISTERED)
+		unregister_netdev(info->netdev);
 
-	if (info->queues)
+	if (info->queues) {
+		rtnl_lock();
 		xennet_destroy_queues(info);
+		rtnl_unlock();
+	}
 	xennet_free_netdev(info->netdev);
 
 	return 0;
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index f0788aae05c9..3cfa46876239 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -1020,6 +1020,10 @@ static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
 			goto err_dma_mask;
 		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
 	}
+	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
+					  dma_get_mask(&pdev->dev));
+	if (rc)
+		goto err_dma_mask;
 
 	ndev->self_mmio = pci_iomap(pdev, 0, 0);
 	if (!ndev->self_mmio) {
diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c
index 0cd79f367f7c..8d98872d0983 100644
--- a/drivers/ntb/hw/idt/ntb_hw_idt.c
+++ b/drivers/ntb/hw/idt/ntb_hw_idt.c
@@ -1744,20 +1744,19 @@ static int idt_ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits)
  * idt_ntb_msg_read() - read message register with specified index
  *			(NTB API callback)
  * @ntb:	NTB device context.
- * @midx:	Message register index
  * @pidx:	OUT - Port index of peer device a message retrieved from
- * @msg:	OUT - Data
+ * @midx:	Message register index
  *
  * Read data from the specified message register and source register.
  *
- * Return: zero on success, negative error if invalid argument passed.
+ * Return: inbound message register value.
  */
-static int idt_ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg)
+static u32 idt_ntb_msg_read(struct ntb_dev *ntb, int *pidx, int midx)
 {
 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 
 	if (midx < 0 || IDT_MSG_CNT <= midx)
-		return -EINVAL;
+		return ~(u32)0;
 
 	/* Retrieve source port index of the message */
 	if (pidx != NULL) {
@@ -1772,18 +1771,15 @@ static int idt_ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg)
 	}
 
 	/* Retrieve data of the corresponding message register */
-	if (msg != NULL)
-		*msg = idt_nt_read(ndev, ntdata_tbl.msgs[midx].in);
-
-	return 0;
+	return idt_nt_read(ndev, ntdata_tbl.msgs[midx].in);
 }
 
 /*
- * idt_ntb_msg_write() - write data to the specified message register
- *			 (NTB API callback)
+ * idt_ntb_peer_msg_write() - write data to the specified message register
+ *			      (NTB API callback)
  * @ntb:	NTB device context.
- * @midx:	Message register index
  * @pidx:	Port index of peer device a message being sent to
+ * @midx:	Message register index
  * @msg:	Data to send
  *
  * Just try to send data to a peer. Message status register should be
@@ -1791,7 +1787,8 @@ static int idt_ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg)
  *
  * Return: zero on success, negative error if invalid argument passed.
  */
-static int idt_ntb_msg_write(struct ntb_dev *ntb, int midx, int pidx, u32 msg)
+static int idt_ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx,
+				  u32 msg)
 {
 	struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
 	unsigned long irqflags;
@@ -2058,7 +2055,7 @@ static const struct ntb_dev_ops idt_ntb_ops = {
 	.msg_set_mask		= idt_ntb_msg_set_mask,
 	.msg_clear_mask		= idt_ntb_msg_clear_mask,
 	.msg_read		= idt_ntb_msg_read,
-	.msg_write		= idt_ntb_msg_write
+	.peer_msg_write		= idt_ntb_peer_msg_write
 };
 
 /*
@@ -2073,7 +2070,7 @@ static int idt_register_device(struct idt_ntb_dev *ndev)
 
 	/* Initialize the rest of NTB device structure and register it */
 	ndev->ntb.ops = &idt_ntb_ops;
-	ndev->ntb.topo = NTB_TOPO_PRI;
+	ndev->ntb.topo = NTB_TOPO_SWITCH;
 
 	ret = ntb_register_device(&ndev->ntb);
 	if (ret != 0) {
@@ -2269,7 +2266,7 @@ static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf,
 		 "Message data:\n");
 	for (idx = 0; idx < IDT_MSG_CNT; idx++) {
 		int src;
-		(void)idt_ntb_msg_read(&ndev->ntb, idx, &src, &data);
+		data = idt_ntb_msg_read(&ndev->ntb, &src, idx);
 		off += scnprintf(strbuf + off, size - off,
 			"\t%hhu. 0x%08x from peer %hhu (Port %hhu)\n",
 			idx, data, src, ndev->peers[src].port);
@@ -2429,7 +2426,7 @@ static int idt_init_pci(struct idt_ntb_dev *ndev)
 	struct pci_dev *pdev = ndev->ntb.pdev;
 	int ret;
 
-	/* Initialize the bit mask of DMA */
+	/* Initialize the bit mask of PCI/NTB DMA */
 	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
 	if (ret != 0) {
 		ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
@@ -2450,6 +2447,12 @@ static int idt_init_pci(struct idt_ntb_dev *ndev)
 		dev_warn(&pdev->dev,
 			"Cannot set consistent DMA highmem bit mask\n");
 	}
+	ret = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
+					   dma_get_mask(&pdev->dev));
+	if (ret != 0) {
+		dev_err(&pdev->dev, "Failed to set NTB device DMA bit mask\n");
+		return ret;
+	}
 
 	/*
 	 * Enable the device advanced error reporting. It's not critical to
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 4de074a86073..156b45cd4a19 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -74,12 +74,6 @@ MODULE_AUTHOR("Intel Corporation");
 #define bar0_off(base, bar) ((base) + ((bar) << 2))
 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
 
-static const struct intel_ntb_reg atom_reg;
-static const struct intel_ntb_alt_reg atom_pri_reg;
-static const struct intel_ntb_alt_reg atom_sec_reg;
-static const struct intel_ntb_alt_reg atom_b2b_reg;
-static const struct intel_ntb_xlat_reg atom_pri_xlat;
-static const struct intel_ntb_xlat_reg atom_sec_xlat;
 static const struct intel_ntb_reg xeon_reg;
 static const struct intel_ntb_alt_reg xeon_pri_reg;
 static const struct intel_ntb_alt_reg xeon_sec_reg;
@@ -184,15 +178,6 @@ static inline void _iowrite64(u64 val, void __iomem *mmio)
 #endif
 #endif
 
-static inline int pdev_is_atom(struct pci_dev *pdev)
-{
-	switch (pdev->device) {
-	case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
-		return 1;
-	}
-	return 0;
-}
-
 static inline int pdev_is_xeon(struct pci_dev *pdev)
 {
 	switch (pdev->device) {
@@ -1006,8 +991,7 @@ static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
 {
 	struct intel_ntb_dev *ndev = filp->private_data;
 
-	if (pdev_is_xeon(ndev->ntb.pdev) ||
-	    pdev_is_atom(ndev->ntb.pdev))
+	if (pdev_is_xeon(ndev->ntb.pdev))
 		return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
 	else if (pdev_is_skx_xeon(ndev->ntb.pdev))
 		return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
@@ -1439,242 +1423,6 @@ static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
 			       ndev->peer_reg->spad);
 }
 
-/* ATOM */
-
-static u64 atom_db_ioread(void __iomem *mmio)
-{
-	return ioread64(mmio);
-}
-
-static void atom_db_iowrite(u64 bits, void __iomem *mmio)
-{
-	iowrite64(bits, mmio);
-}
-
-static int atom_poll_link(struct intel_ntb_dev *ndev)
-{
-	u32 ntb_ctl;
-
-	ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
-
-	if (ntb_ctl == ndev->ntb_ctl)
-		return 0;
-
-	ndev->ntb_ctl = ntb_ctl;
-
-	ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
-
-	return 1;
-}
-
-static int atom_link_is_up(struct intel_ntb_dev *ndev)
-{
-	return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
-}
-
-static int atom_link_is_err(struct intel_ntb_dev *ndev)
-{
-	if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
-	    & ATOM_LTSSMSTATEJMP_FORCEDETECT)
-		return 1;
-
-	if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
-	    & ATOM_IBIST_ERR_OFLOW)
-		return 1;
-
-	return 0;
-}
-
-static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
-{
-	struct device *dev = &ndev->ntb.pdev->dev;
-
-	switch (ppd & ATOM_PPD_TOPO_MASK) {
-	case ATOM_PPD_TOPO_B2B_USD:
-		dev_dbg(dev, "PPD %d B2B USD\n", ppd);
-		return NTB_TOPO_B2B_USD;
-
-	case ATOM_PPD_TOPO_B2B_DSD:
-		dev_dbg(dev, "PPD %d B2B DSD\n", ppd);
-		return NTB_TOPO_B2B_DSD;
-
-	case ATOM_PPD_TOPO_PRI_USD:
-	case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
-	case ATOM_PPD_TOPO_SEC_USD:
-	case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
-		dev_dbg(dev, "PPD %d non B2B disabled\n", ppd);
-		return NTB_TOPO_NONE;
-	}
-
-	dev_dbg(dev, "PPD %d invalid\n", ppd);
-	return NTB_TOPO_NONE;
-}
-
-static void atom_link_hb(struct work_struct *work)
-{
-	struct intel_ntb_dev *ndev = hb_ndev(work);
-	struct device *dev = &ndev->ntb.pdev->dev;
-	unsigned long poll_ts;
-	void __iomem *mmio;
-	u32 status32;
-
-	poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
-
-	/* Delay polling the link status if an interrupt was received,
-	 * unless the cached link status says the link is down.
-	 */
-	if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
-		schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
-		return;
-	}
-
-	if (atom_poll_link(ndev))
-		ntb_link_event(&ndev->ntb);
-
-	if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
-		schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
-		return;
-	}
-
-	/* Link is down with error: recover the link! */
-
-	mmio = ndev->self_mmio;
-
-	/* Driver resets the NTB ModPhy lanes - magic! */
-	iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
-	iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
-	iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
-	iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
-
-	/* Driver waits 100ms to allow the NTB ModPhy to settle */
-	msleep(100);
-
-	/* Clear AER Errors, write to clear */
-	status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
-	dev_dbg(dev, "ERRCORSTS = %x\n", status32);
-	status32 &= PCI_ERR_COR_REP_ROLL;
-	iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
-
-	/* Clear unexpected electrical idle event in LTSSM, write to clear */
-	status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
-	dev_dbg(dev, "LTSSMERRSTS0 = %x\n", status32);
-	status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
-	iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
-
-	/* Clear DeSkew Buffer error, write to clear */
-	status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
-	dev_dbg(dev, "DESKEWSTS = %x\n", status32);
-	status32 |= ATOM_DESKEWSTS_DBERR;
-	iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
-
-	status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
-	dev_dbg(dev, "IBSTERRRCRVSTS0 = %x\n", status32);
-	status32 &= ATOM_IBIST_ERR_OFLOW;
-	iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
-
-	/* Releases the NTB state machine to allow the link to retrain */
-	status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
-	dev_dbg(dev, "LTSSMSTATEJMP = %x\n", status32);
-	status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
-	iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
-
-	/* There is a potential race between the 2 NTB devices recovering at the
-	 * same time.  If the times are the same, the link will not recover and
-	 * the driver will be stuck in this loop forever.  Add a random interval
-	 * to the recovery time to prevent this race.
-	 */
-	schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
-			      + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
-}
-
-static int atom_init_isr(struct intel_ntb_dev *ndev)
-{
-	int rc;
-
-	rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
-			   ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
-	if (rc)
-		return rc;
-
-	/* ATOM doesn't have link status interrupt, poll on that platform */
-	ndev->last_ts = jiffies;
-	INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
-	schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
-
-	return 0;
-}
-
-static void atom_deinit_isr(struct intel_ntb_dev *ndev)
-{
-	cancel_delayed_work_sync(&ndev->hb_timer);
-	ndev_deinit_isr(ndev);
-}
-
-static int atom_init_ntb(struct intel_ntb_dev *ndev)
-{
-	ndev->mw_count = ATOM_MW_COUNT;
-	ndev->spad_count = ATOM_SPAD_COUNT;
-	ndev->db_count = ATOM_DB_COUNT;
-
-	switch (ndev->ntb.topo) {
-	case NTB_TOPO_B2B_USD:
-	case NTB_TOPO_B2B_DSD:
-		ndev->self_reg = &atom_pri_reg;
-		ndev->peer_reg = &atom_b2b_reg;
-		ndev->xlat_reg = &atom_sec_xlat;
-
-		/* Enable Bus Master and Memory Space on the secondary side */
-		iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
-			  ndev->self_mmio + ATOM_SPCICMD_OFFSET);
-
-		break;
-
-	default:
-		return -EINVAL;
-	}
-
-	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
-
-	return 0;
-}
-
-static int atom_init_dev(struct intel_ntb_dev *ndev)
-{
-	u32 ppd;
-	int rc;
-
-	rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
-	if (rc)
-		return -EIO;
-
-	ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
-	if (ndev->ntb.topo == NTB_TOPO_NONE)
-		return -EINVAL;
-
-	rc = atom_init_ntb(ndev);
-	if (rc)
-		return rc;
-
-	rc = atom_init_isr(ndev);
-	if (rc)
-		return rc;
-
-	if (ndev->ntb.topo != NTB_TOPO_SEC) {
-		/* Initiate PCI-E link training */
-		rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
-					    ppd | ATOM_PPD_INIT_LINK);
-		if (rc)
-			return rc;
-	}
-
-	return 0;
-}
-
-static void atom_deinit_dev(struct intel_ntb_dev *ndev)
-{
-	atom_deinit_isr(ndev);
-}
-
 /* Skylake Xeon NTB */
 
 static int skx_poll_link(struct intel_ntb_dev *ndev)
@@ -2586,6 +2334,10 @@ static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
 			goto err_dma_mask;
 		dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
 	}
+	rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
+					  dma_get_mask(&pdev->dev));
+	if (rc)
+		goto err_dma_mask;
 
 	ndev->self_mmio = pci_iomap(pdev, 0, 0);
 	if (!ndev->self_mmio) {
@@ -2658,24 +2410,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
 
 	node = dev_to_node(&pdev->dev);
 
-	if (pdev_is_atom(pdev)) {
-		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
-		if (!ndev) {
-			rc = -ENOMEM;
-			goto err_ndev;
-		}
-
-		ndev_init_struct(ndev, pdev);
-
-		rc = intel_ntb_init_pci(ndev, pdev);
-		if (rc)
-			goto err_init_pci;
-
-		rc = atom_init_dev(ndev);
-		if (rc)
-			goto err_init_dev;
-
-	} else if (pdev_is_xeon(pdev)) {
+	if (pdev_is_xeon(pdev)) {
 		ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
 		if (!ndev) {
 			rc = -ENOMEM;
@@ -2731,9 +2466,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
 
 err_register:
 	ndev_deinit_debugfs(ndev);
-	if (pdev_is_atom(pdev))
-		atom_deinit_dev(ndev);
-	else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
+	if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
 		xeon_deinit_dev(ndev);
 err_init_dev:
 	intel_ntb_deinit_pci(ndev);
@@ -2749,41 +2482,12 @@ static void intel_ntb_pci_remove(struct pci_dev *pdev)
 
 	ntb_unregister_device(&ndev->ntb);
 	ndev_deinit_debugfs(ndev);
-	if (pdev_is_atom(pdev))
-		atom_deinit_dev(ndev);
-	else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
+	if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
 		xeon_deinit_dev(ndev);
 	intel_ntb_deinit_pci(ndev);
 	kfree(ndev);
 }
 
-static const struct intel_ntb_reg atom_reg = {
-	.poll_link		= atom_poll_link,
-	.link_is_up		= atom_link_is_up,
-	.db_ioread		= atom_db_ioread,
-	.db_iowrite		= atom_db_iowrite,
-	.db_size		= sizeof(u64),
-	.ntb_ctl		= ATOM_NTBCNTL_OFFSET,
-	.mw_bar			= {2, 4},
-};
-
-static const struct intel_ntb_alt_reg atom_pri_reg = {
-	.db_bell		= ATOM_PDOORBELL_OFFSET,
-	.db_mask		= ATOM_PDBMSK_OFFSET,
-	.spad			= ATOM_SPAD_OFFSET,
-};
-
-static const struct intel_ntb_alt_reg atom_b2b_reg = {
-	.db_bell		= ATOM_B2B_DOORBELL_OFFSET,
-	.spad			= ATOM_B2B_SPAD_OFFSET,
-};
-
-static const struct intel_ntb_xlat_reg atom_sec_xlat = {
-	/* FIXME : .bar0_base	= ATOM_SBAR0BASE_OFFSET, */
-	/* FIXME : .bar2_limit	= ATOM_SBAR2LMT_OFFSET, */
-	.bar2_xlat		= ATOM_SBAR2XLAT_OFFSET,
-};
-
 static const struct intel_ntb_reg xeon_reg = {
 	.poll_link		= xeon_poll_link,
 	.link_is_up		= xeon_link_is_up,
@@ -2940,7 +2644,6 @@ static const struct file_operations intel_ntb_debugfs_info = {
 };
 
 static const struct pci_device_id intel_ntb_pci_tbl[] = {
-	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
 	{PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h
index 2d6c38afb128..4415aa7ea775 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -66,7 +66,6 @@
 #define PCI_DEVICE_ID_INTEL_NTB_B2B_HSX	0x2F0D
 #define PCI_DEVICE_ID_INTEL_NTB_PS_HSX	0x2F0E
 #define PCI_DEVICE_ID_INTEL_NTB_SS_HSX	0x2F0F
-#define PCI_DEVICE_ID_INTEL_NTB_B2B_BWD	0x0C4E
 #define PCI_DEVICE_ID_INTEL_NTB_B2B_BDX	0x6F0D
 #define PCI_DEVICE_ID_INTEL_NTB_PS_BDX	0x6F0E
 #define PCI_DEVICE_ID_INTEL_NTB_SS_BDX	0x6F0F
@@ -196,63 +195,6 @@
 #define SKX_DB_TOTAL_SHIFT		33
 #define SKX_SPAD_COUNT			16
 
-/* Intel Atom hardware */
-
-#define ATOM_SBAR2XLAT_OFFSET		0x0008
-#define ATOM_PDOORBELL_OFFSET		0x0020
-#define ATOM_PDBMSK_OFFSET		0x0028
-#define ATOM_NTBCNTL_OFFSET		0x0060
-#define ATOM_SPAD_OFFSET			0x0080
-#define ATOM_PPD_OFFSET			0x00d4
-#define ATOM_PBAR2XLAT_OFFSET		0x8008
-#define ATOM_B2B_DOORBELL_OFFSET		0x8020
-#define ATOM_B2B_SPAD_OFFSET		0x8080
-#define ATOM_SPCICMD_OFFSET		0xb004
-#define ATOM_LINK_STATUS_OFFSET		0xb052
-#define ATOM_ERRCORSTS_OFFSET		0xb110
-#define ATOM_IP_BASE			0xc000
-#define ATOM_DESKEWSTS_OFFSET		(ATOM_IP_BASE + 0x3024)
-#define ATOM_LTSSMERRSTS0_OFFSET		(ATOM_IP_BASE + 0x3180)
-#define ATOM_LTSSMSTATEJMP_OFFSET	(ATOM_IP_BASE + 0x3040)
-#define ATOM_IBSTERRRCRVSTS0_OFFSET	(ATOM_IP_BASE + 0x3324)
-#define ATOM_MODPHY_PCSREG4		0x1c004
-#define ATOM_MODPHY_PCSREG6		0x1c006
-
-#define ATOM_PPD_INIT_LINK		0x0008
-#define ATOM_PPD_CONN_MASK		0x0300
-#define ATOM_PPD_CONN_TRANSPARENT	0x0000
-#define ATOM_PPD_CONN_B2B		0x0100
-#define ATOM_PPD_CONN_RP			0x0200
-#define ATOM_PPD_DEV_MASK		0x1000
-#define ATOM_PPD_DEV_USD			0x0000
-#define ATOM_PPD_DEV_DSD			0x1000
-#define ATOM_PPD_TOPO_MASK	(ATOM_PPD_CONN_MASK | ATOM_PPD_DEV_MASK)
-#define ATOM_PPD_TOPO_PRI_USD	(ATOM_PPD_CONN_TRANSPARENT | ATOM_PPD_DEV_USD)
-#define ATOM_PPD_TOPO_PRI_DSD	(ATOM_PPD_CONN_TRANSPARENT | ATOM_PPD_DEV_DSD)
-#define ATOM_PPD_TOPO_SEC_USD	(ATOM_PPD_CONN_RP | ATOM_PPD_DEV_USD)
-#define ATOM_PPD_TOPO_SEC_DSD	(ATOM_PPD_CONN_RP | ATOM_PPD_DEV_DSD)
-#define ATOM_PPD_TOPO_B2B_USD	(ATOM_PPD_CONN_B2B | ATOM_PPD_DEV_USD)
-#define ATOM_PPD_TOPO_B2B_DSD	(ATOM_PPD_CONN_B2B | ATOM_PPD_DEV_DSD)
-
-#define ATOM_MW_COUNT			2
-#define ATOM_DB_COUNT			34
-#define ATOM_DB_VALID_MASK		(BIT_ULL(ATOM_DB_COUNT) - 1)
-#define ATOM_DB_MSIX_VECTOR_COUNT	34
-#define ATOM_DB_MSIX_VECTOR_SHIFT	1
-#define ATOM_DB_TOTAL_SHIFT		34
-#define ATOM_SPAD_COUNT			16
-
-#define ATOM_NTB_CTL_DOWN_BIT		BIT(16)
-#define ATOM_NTB_CTL_ACTIVE(x)		!(x & ATOM_NTB_CTL_DOWN_BIT)
-
-#define ATOM_DESKEWSTS_DBERR		BIT(15)
-#define ATOM_LTSSMERRSTS0_UNEXPECTEDEI	BIT(20)
-#define ATOM_LTSSMSTATEJMP_FORCEDETECT	BIT(2)
-#define ATOM_IBIST_ERR_OFLOW		0x7FFF7FFF
-
-#define ATOM_LINK_HB_TIMEOUT		msecs_to_jiffies(1000)
-#define ATOM_LINK_RECOVERY_TIME		msecs_to_jiffies(500)
-
 /* Ntb control and link status */
 
 #define NTB_CTL_CFG_LOCK		BIT(0)
diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
index afe8ed6f3b23..f624ae27eabe 100644
--- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
+++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
@@ -94,6 +94,9 @@ struct switchtec_ntb {
 	struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
 	struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
 	struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
+	struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
+
+	void __iomem *mmio_xlink_win;
 
 	struct shared_mw *self_shared;
 	struct shared_mw __iomem *peer_shared;
@@ -109,6 +112,7 @@ struct switchtec_ntb {
 
 	int nr_direct_mw;
 	int nr_lut_mw;
+	int nr_rsvd_luts;
 	int direct_mw_to_bar[MAX_DIRECT_MW];
 
 	int peer_nr_direct_mw;
@@ -118,6 +122,7 @@ struct switchtec_ntb {
 	bool link_is_up;
 	enum ntb_speed link_speed;
 	enum ntb_width link_width;
+	struct work_struct link_reinit_work;
 };
 
 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
@@ -172,7 +177,7 @@ static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
 
 	if (ps == status) {
 		dev_err(&sndev->stdev->dev,
-			"Timed out while peforming %s (%d). (%08x)",
+			"Timed out while performing %s (%d). (%08x)\n",
 			op_text[op], op,
 			ioread32(&ctl->partition_status));
 
@@ -185,10 +190,10 @@ static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
 				  u32 val)
 {
-	if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_self_dbmsg->omsg))
+	if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
 		return -EINVAL;
 
-	iowrite32(val, &sndev->mmio_self_dbmsg->omsg[idx].msg);
+	iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
 
 	return 0;
 }
@@ -197,7 +202,7 @@ static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 {
 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 	int nr_direct_mw = sndev->peer_nr_direct_mw;
-	int nr_lut_mw = sndev->peer_nr_lut_mw - 1;
+	int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
 
 	if (pidx != NTB_DEF_PEER_IDX)
 		return -EINVAL;
@@ -210,12 +215,12 @@ static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 
 static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
 {
-	return mw_idx - sndev->nr_direct_mw + 1;
+	return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
 }
 
 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
 {
-	return mw_idx - sndev->peer_nr_direct_mw + 1;
+	return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
 }
 
 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
@@ -306,7 +311,7 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
 	if (pidx != NTB_DEF_PEER_IDX)
 		return -EINVAL;
 
-	dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap",
+	dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
 		widx, pidx, &addr, &size);
 
 	if (widx >= switchtec_ntb_mw_count(ntb, pidx))
@@ -315,6 +320,19 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
 	if (xlate_pos < 12)
 		return -EINVAL;
 
+	if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
+		/*
+		 * In certain circumstances we can get a buffer that is
+		 * not aligned to its size. (Most of the time
+		 * dma_alloc_coherent ensures this). This can happen when
+		 * using large buffers allocated by the CMA
+		 * (see CMA_CONFIG_ALIGNMENT)
+		 */
+		dev_err(&sndev->stdev->dev,
+			"ERROR: Memory window address is not aligned to it's size!\n");
+		return -EINVAL;
+	}
+
 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
 				   NTB_CTRL_PART_STATUS_LOCKED);
 	if (rc)
@@ -337,7 +355,7 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
 
 	if (rc == -EIO) {
 		dev_err(&sndev->stdev->dev,
-			"Hardware reported an error configuring mw %d: %08x",
+			"Hardware reported an error configuring mw %d: %08x\n",
 			widx, ioread32(&ctl->bar_error));
 
 		if (widx < nr_direct_mw)
@@ -355,8 +373,9 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
 {
 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
+	int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
 
-	return sndev->nr_direct_mw + (use_lut_mws ? sndev->nr_lut_mw - 1 : 0);
+	return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
 }
 
 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
@@ -463,18 +482,69 @@ static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
 	sndev->link_width = min(self_width, peer_width);
 }
 
-enum {
+static int crosslink_is_enabled(struct switchtec_ntb *sndev)
+{
+	struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
+
+	return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
+}
+
+static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
+{
+	int i;
+	u32 msg_map = 0;
+
+	if (!crosslink_is_enabled(sndev))
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
+		int m = i | sndev->self_partition << 2;
+
+		msg_map |= m << i * 8;
+	}
+
+	iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
+	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
+		  &sndev->mmio_peer_dbmsg->odb_mask);
+}
+
+enum switchtec_msg {
 	LINK_MESSAGE = 0,
 	MSG_LINK_UP = 1,
 	MSG_LINK_DOWN = 2,
 	MSG_CHECK_LINK = 3,
+	MSG_LINK_FORCE_DOWN = 4,
 };
 
-static void switchtec_ntb_check_link(struct switchtec_ntb *sndev)
+static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
+
+static void link_reinit_work(struct work_struct *work)
+{
+	struct switchtec_ntb *sndev;
+
+	sndev = container_of(work, struct switchtec_ntb, link_reinit_work);
+
+	switchtec_ntb_reinit_peer(sndev);
+}
+
+static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
+				     enum switchtec_msg msg)
 {
 	int link_sta;
 	int old = sndev->link_is_up;
 
+	if (msg == MSG_LINK_FORCE_DOWN) {
+		schedule_work(&sndev->link_reinit_work);
+
+		if (sndev->link_is_up) {
+			sndev->link_is_up = 0;
+			ntb_link_event(&sndev->ntb);
+			dev_info(&sndev->stdev->dev, "ntb link forced down\n");
+		}
+
+		return;
+	}
+
 	link_sta = sndev->self_shared->link_sta;
 	if (link_sta) {
 		u64 peer = ioread64(&sndev->peer_shared->magic);
@@ -491,8 +561,11 @@ static void switchtec_ntb_check_link(struct switchtec_ntb *sndev)
 	if (link_sta != old) {
 		switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
 		ntb_link_event(&sndev->ntb);
-		dev_info(&sndev->stdev->dev, "ntb link %s",
+		dev_info(&sndev->stdev->dev, "ntb link %s\n",
 			 link_sta ? "up" : "down");
+
+		if (link_sta)
+			crosslink_init_dbmsgs(sndev);
 	}
 }
 
@@ -500,7 +573,7 @@ static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
 {
 	struct switchtec_ntb *sndev = stdev->sndev;
 
-	switchtec_ntb_check_link(sndev);
+	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 }
 
 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
@@ -523,12 +596,12 @@ static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
 {
 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 
-	dev_dbg(&sndev->stdev->dev, "enabling link");
+	dev_dbg(&sndev->stdev->dev, "enabling link\n");
 
 	sndev->self_shared->link_sta = 1;
 	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
 
-	switchtec_ntb_check_link(sndev);
+	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 
 	return 0;
 }
@@ -537,12 +610,12 @@ static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
 {
 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 
-	dev_dbg(&sndev->stdev->dev, "disabling link");
+	dev_dbg(&sndev->stdev->dev, "disabling link\n");
 
 	sndev->self_shared->link_sta = 0;
-	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
+	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
 
-	switchtec_ntb_check_link(sndev);
+	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 
 	return 0;
 }
@@ -638,7 +711,7 @@ static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 	unsigned long offset;
 
-	offset = (unsigned long)sndev->mmio_self_dbmsg->odb -
+	offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
 		(unsigned long)sndev->stdev->mmio;
 
 	offset += sndev->db_shift / 8;
@@ -656,7 +729,7 @@ static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 	struct switchtec_ntb *sndev = ntb_sndev(ntb);
 
 	iowrite64(db_bits << sndev->db_peer_shift,
-		  &sndev->mmio_self_dbmsg->odb);
+		  &sndev->mmio_peer_dbmsg->odb);
 
 	return 0;
 }
@@ -777,24 +850,63 @@ static const struct ntb_dev_ops switchtec_ntb_ops = {
 	.peer_spad_addr		= switchtec_ntb_peer_spad_addr,
 };
 
-static void switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
+static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
 {
+	u64 tpart_vec;
+	int self;
 	u64 part_map;
+	int bit;
 
 	sndev->ntb.pdev = sndev->stdev->pdev;
 	sndev->ntb.topo = NTB_TOPO_SWITCH;
 	sndev->ntb.ops = &switchtec_ntb_ops;
 
+	INIT_WORK(&sndev->link_reinit_work, link_reinit_work);
+
 	sndev->self_partition = sndev->stdev->partition;
 
 	sndev->mmio_ntb = sndev->stdev->mmio_ntb;
+
+	self = sndev->self_partition;
+	tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
+	tpart_vec <<= 32;
+	tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
+
 	part_map = ioread64(&sndev->mmio_ntb->ep_map);
 	part_map &= ~(1 << sndev->self_partition);
-	sndev->peer_partition = ffs(part_map) - 1;
 
-	dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d (%llx)",
-		sndev->self_partition, sndev->stdev->partition_count,
-		part_map);
+	if (!ffs(tpart_vec)) {
+		if (sndev->stdev->partition_count != 2) {
+			dev_err(&sndev->stdev->dev,
+				"ntb target partition not defined\n");
+			return -ENODEV;
+		}
+
+		bit = ffs(part_map);
+		if (!bit) {
+			dev_err(&sndev->stdev->dev,
+				"peer partition is not NT partition\n");
+			return -ENODEV;
+		}
+
+		sndev->peer_partition = bit - 1;
+	} else {
+		if (ffs(tpart_vec) != fls(tpart_vec)) {
+			dev_err(&sndev->stdev->dev,
+				"ntb driver only supports 1 pair of 1-1 ntb mapping\n");
+			return -ENODEV;
+		}
+
+		sndev->peer_partition = ffs(tpart_vec) - 1;
+		if (!(part_map & (1 << sndev->peer_partition))) {
+			dev_err(&sndev->stdev->dev,
+				"ntb target partition is not NT partition\n");
+			return -ENODEV;
+		}
+	}
+
+	dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
+		sndev->self_partition, sndev->stdev->partition_count);
 
 	sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
 		SWITCHTEC_NTB_REG_CTRL_OFFSET;
@@ -804,6 +916,283 @@ static void switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
 	sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
 	sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
 	sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
+	sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
+
+	return 0;
+}
+
+static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
+			       struct ntb_ctrl_regs __iomem *ctl,
+			       int lut_idx, int partition, u64 addr)
+{
+	int peer_bar = sndev->peer_direct_mw_to_bar[0];
+	u32 ctl_val;
+	int rc;
+
+	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
+				   NTB_CTRL_PART_STATUS_LOCKED);
+	if (rc)
+		return rc;
+
+	ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
+	ctl_val &= 0xFF;
+	ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
+	ctl_val |= ilog2(LUT_SIZE) << 8;
+	ctl_val |= (sndev->nr_lut_mw - 1) << 14;
+	iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
+
+	iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
+		  &ctl->lut_entry[lut_idx]);
+
+	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
+				   NTB_CTRL_PART_STATUS_NORMAL);
+	if (rc) {
+		u32 bar_error, lut_error;
+
+		bar_error = ioread32(&ctl->bar_error);
+		lut_error = ioread32(&ctl->lut_error);
+		dev_err(&sndev->stdev->dev,
+			"Error setting up reserved lut window: %08x / %08x\n",
+			bar_error, lut_error);
+		return rc;
+	}
+
+	return 0;
+}
+
+static int config_req_id_table(struct switchtec_ntb *sndev,
+			       struct ntb_ctrl_regs __iomem *mmio_ctrl,
+			       int *req_ids, int count)
+{
+	int i, rc = 0;
+	u32 error;
+	u32 proxy_id;
+
+	if (ioread32(&mmio_ctrl->req_id_table_size) < count) {
+		dev_err(&sndev->stdev->dev,
+			"Not enough requester IDs available.\n");
+		return -EFAULT;
+	}
+
+	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
+				   NTB_CTRL_PART_OP_LOCK,
+				   NTB_CTRL_PART_STATUS_LOCKED);
+	if (rc)
+		return rc;
+
+	iowrite32(NTB_PART_CTRL_ID_PROT_DIS,
+		  &mmio_ctrl->partition_ctrl);
+
+	for (i = 0; i < count; i++) {
+		iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
+			  &mmio_ctrl->req_id_table[i]);
+
+		proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
+		dev_dbg(&sndev->stdev->dev,
+			"Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
+			req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
+			req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
+			(proxy_id >> 1) & 0x7);
+	}
+
+	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
+				   NTB_CTRL_PART_OP_CFG,
+				   NTB_CTRL_PART_STATUS_NORMAL);
+
+	if (rc == -EIO) {
+		error = ioread32(&mmio_ctrl->req_id_error);
+		dev_err(&sndev->stdev->dev,
+			"Error setting up the requester ID table: %08x\n",
+			error);
+	}
+
+	return 0;
+}
+
+static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
+			       u64 *mw_addrs, int mw_count)
+{
+	int rc, i;
+	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
+	u64 addr;
+	size_t size, offset;
+	int bar;
+	int xlate_pos;
+	u32 ctl_val;
+
+	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
+				   NTB_CTRL_PART_STATUS_LOCKED);
+	if (rc)
+		return rc;
+
+	for (i = 0; i < sndev->nr_lut_mw; i++) {
+		if (i == ntb_lut_idx)
+			continue;
+
+		addr = mw_addrs[0] + LUT_SIZE * i;
+
+		iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
+			   addr),
+			  &ctl->lut_entry[i]);
+	}
+
+	sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
+
+	for (i = 0; i < sndev->nr_direct_mw; i++) {
+		bar = sndev->direct_mw_to_bar[i];
+		offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
+		addr = mw_addrs[i] + offset;
+		size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
+		xlate_pos = ilog2(size);
+
+		if (offset && size > offset)
+			size = offset;
+
+		ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
+		ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
+
+		iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
+		iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size);
+		iowrite64(sndev->peer_partition | addr,
+			  &ctl->bar_entry[bar].xlate_addr);
+	}
+
+	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
+				   NTB_CTRL_PART_STATUS_NORMAL);
+	if (rc) {
+		u32 bar_error, lut_error;
+
+		bar_error = ioread32(&ctl->bar_error);
+		lut_error = ioread32(&ctl->lut_error);
+		dev_err(&sndev->stdev->dev,
+			"Error setting up cross link windows: %08x / %08x\n",
+			bar_error, lut_error);
+		return rc;
+	}
+
+	return 0;
+}
+
+static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
+	struct ntb_ctrl_regs __iomem *mmio_ctrl)
+{
+	int req_ids[16];
+	int i;
+	u32 proxy_id;
+
+	for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
+		proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
+
+		if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
+			break;
+
+		req_ids[i] = ((proxy_id >> 1) & 0xFF);
+	}
+
+	return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
+}
+
+/*
+ * In crosslink configuration there is a virtual partition in the
+ * middle of the two switches. The BARs in this partition have to be
+ * enumerated and assigned addresses.
+ */
+static int crosslink_enum_partition(struct switchtec_ntb *sndev,
+				    u64 *bar_addrs)
+{
+	struct part_cfg_regs __iomem *part_cfg =
+		&sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
+	u32 pff = ioread32(&part_cfg->vep_pff_inst_id);
+	struct pff_csr_regs __iomem *mmio_pff =
+		&sndev->stdev->mmio_pff_csr[pff];
+	const u64 bar_space = 0x1000000000LL;
+	u64 bar_addr;
+	int bar_cnt = 0;
+	int i;
+
+	iowrite16(0x6, &mmio_pff->pcicmd);
+
+	for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
+		iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
+		bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
+		bar_addr &= ~0xf;
+
+		dev_dbg(&sndev->stdev->dev,
+			"Crosslink BAR%d addr: %llx\n",
+			i, bar_addr);
+
+		if (bar_addr != bar_space * i)
+			continue;
+
+		bar_addrs[bar_cnt++] = bar_addr;
+	}
+
+	return bar_cnt;
+}
+
+static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
+{
+	int rc;
+	int bar = sndev->direct_mw_to_bar[0];
+	const int ntb_lut_idx = 1;
+	u64 bar_addrs[6];
+	u64 addr;
+	int offset;
+	int bar_cnt;
+
+	if (!crosslink_is_enabled(sndev))
+		return 0;
+
+	dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
+	sndev->ntb.topo = NTB_TOPO_CROSSLINK;
+
+	bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
+	if (bar_cnt < sndev->nr_direct_mw + 1) {
+		dev_err(&sndev->stdev->dev,
+			"Error enumerating crosslink partition\n");
+		return -EINVAL;
+	}
+
+	addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
+		SWITCHTEC_NTB_REG_DBMSG_OFFSET +
+		sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
+
+	offset = addr & (LUT_SIZE - 1);
+	addr -= offset;
+
+	rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
+				 sndev->peer_partition, addr);
+	if (rc)
+		return rc;
+
+	rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
+				 bar_cnt - 1);
+	if (rc)
+		return rc;
+
+	rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
+	if (rc)
+		return rc;
+
+	sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
+						LUT_SIZE, LUT_SIZE);
+	if (!sndev->mmio_xlink_win) {
+		rc = -ENOMEM;
+		return rc;
+	}
+
+	sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
+	sndev->nr_rsvd_luts++;
+
+	crosslink_init_dbmsgs(sndev);
+
+	return 0;
+}
+
+static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
+{
+	if (sndev->mmio_xlink_win)
+		pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
 }
 
 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
@@ -829,7 +1218,7 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
 	sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
 	sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
 
-	dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut",
+	dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
 		sndev->nr_direct_mw, sndev->nr_lut_mw);
 
 	sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
@@ -839,7 +1228,7 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
 		ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
 	sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
 
-	dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut",
+	dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
 		sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
 
 }
@@ -849,24 +1238,35 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
  * shared among all partitions. So we must split them in half
  * (32 for each partition). However, the message interrupts are
  * also shared with the top 4 doorbells so we just limit this to
- * 28 doorbells per partition
+ * 28 doorbells per partition.
+ *
+ * In crosslink mode, each side has it's own dbmsg register so
+ * they can each use all 60 of the available doorbells.
  */
 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
 {
-	sndev->db_valid_mask = 0x0FFFFFFF;
+	sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
 
-	if (sndev->self_partition < sndev->peer_partition) {
+	if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
+		sndev->db_shift = 0;
+		sndev->db_peer_shift = 0;
+		sndev->db_valid_mask = sndev->db_mask;
+	} else if (sndev->self_partition < sndev->peer_partition) {
 		sndev->db_shift = 0;
 		sndev->db_peer_shift = 32;
+		sndev->db_valid_mask = 0x0FFFFFFF;
 	} else {
 		sndev->db_shift = 32;
 		sndev->db_peer_shift = 0;
+		sndev->db_valid_mask = 0x0FFFFFFF;
 	}
 
-	sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
 	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
 	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
-		  &sndev->mmio_self_dbmsg->odb_mask);
+		  &sndev->mmio_peer_dbmsg->odb_mask);
+
+	dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
+		sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
 }
 
 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
@@ -887,52 +1287,23 @@ static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
 			  &sndev->mmio_self_dbmsg->imsg[i]);
 }
 
-static int switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
+static int
+switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
 {
-	int rc = 0;
-	u16 req_id;
-	u32 error;
-
-	req_id = ioread16(&sndev->mmio_ntb->requester_id);
-
-	if (ioread32(&sndev->mmio_self_ctrl->req_id_table_size) < 2) {
-		dev_err(&sndev->stdev->dev,
-			"Not enough requester IDs available.");
-		return -EFAULT;
-	}
-
-	rc = switchtec_ntb_part_op(sndev, sndev->mmio_self_ctrl,
-				   NTB_CTRL_PART_OP_LOCK,
-				   NTB_CTRL_PART_STATUS_LOCKED);
-	if (rc)
-		return rc;
-
-	iowrite32(NTB_PART_CTRL_ID_PROT_DIS,
-		  &sndev->mmio_self_ctrl->partition_ctrl);
+	int req_ids[2];
 
 	/*
 	 * Root Complex Requester ID (which is 0:00.0)
 	 */
-	iowrite32(0 << 16 | NTB_CTRL_REQ_ID_EN,
-		  &sndev->mmio_self_ctrl->req_id_table[0]);
+	req_ids[0] = 0;
 
 	/*
 	 * Host Bridge Requester ID (as read from the mmap address)
 	 */
-	iowrite32(req_id << 16 | NTB_CTRL_REQ_ID_EN,
-		  &sndev->mmio_self_ctrl->req_id_table[1]);
-
-	rc = switchtec_ntb_part_op(sndev, sndev->mmio_self_ctrl,
-				   NTB_CTRL_PART_OP_CFG,
-				   NTB_CTRL_PART_STATUS_NORMAL);
-	if (rc == -EIO) {
-		error = ioread32(&sndev->mmio_self_ctrl->req_id_error);
-		dev_err(&sndev->stdev->dev,
-			"Error setting up the requester ID table: %08x",
-			error);
-	}
+	req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
 
-	return rc;
+	return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
+				   ARRAY_SIZE(req_ids));
 }
 
 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
@@ -963,59 +1334,35 @@ static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
 
 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
 {
-	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
-	int bar = sndev->direct_mw_to_bar[0];
-	u32 ctl_val;
+	int self_bar = sndev->direct_mw_to_bar[0];
 	int rc;
 
+	sndev->nr_rsvd_luts++;
 	sndev->self_shared = dma_zalloc_coherent(&sndev->stdev->pdev->dev,
 						 LUT_SIZE,
 						 &sndev->self_shared_dma,
 						 GFP_KERNEL);
 	if (!sndev->self_shared) {
 		dev_err(&sndev->stdev->dev,
-			"unable to allocate memory for shared mw");
+			"unable to allocate memory for shared mw\n");
 		return -ENOMEM;
 	}
 
 	switchtec_ntb_init_shared(sndev);
 
-	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
-				   NTB_CTRL_PART_STATUS_LOCKED);
+	rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
+				 sndev->self_partition,
+				 sndev->self_shared_dma);
 	if (rc)
 		goto unalloc_and_exit;
 
-	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
-	ctl_val &= 0xFF;
-	ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
-	ctl_val |= ilog2(LUT_SIZE) << 8;
-	ctl_val |= (sndev->nr_lut_mw - 1) << 14;
-	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
-
-	iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) |
-		   sndev->self_shared_dma),
-		  &ctl->lut_entry[0]);
-
-	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
-				   NTB_CTRL_PART_STATUS_NORMAL);
-	if (rc) {
-		u32 bar_error, lut_error;
-
-		bar_error = ioread32(&ctl->bar_error);
-		lut_error = ioread32(&ctl->lut_error);
-		dev_err(&sndev->stdev->dev,
-			"Error setting up shared MW: %08x / %08x",
-			bar_error, lut_error);
-		goto unalloc_and_exit;
-	}
-
-	sndev->peer_shared = pci_iomap(sndev->stdev->pdev, bar, LUT_SIZE);
+	sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
 	if (!sndev->peer_shared) {
 		rc = -ENOMEM;
 		goto unalloc_and_exit;
 	}
 
-	dev_dbg(&sndev->stdev->dev, "Shared MW Ready");
+	dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
 	return 0;
 
 unalloc_and_exit:
@@ -1034,6 +1381,7 @@ static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
 		dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
 				  sndev->self_shared,
 				  sndev->self_shared_dma);
+	sndev->nr_rsvd_luts--;
 }
 
 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
@@ -1056,12 +1404,12 @@ static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
 		u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
 
 		if (msg & NTB_DBMSG_IMSG_STATUS) {
-			dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", i,
-				(u32)msg);
+			dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
+				i, (u32)msg);
 			iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
 
 			if (i == LINK_MESSAGE)
-				switchtec_ntb_check_link(sndev);
+				switchtec_ntb_check_link(sndev, msg);
 		}
 	}
 
@@ -1085,7 +1433,7 @@ static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
 	       message_irq == event_irq)
 		message_irq++;
 
-	dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d",
+	dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
 		event_irq, doorbell_irq, message_irq);
 
 	for (i = 0; i < idb_vecs - 4; i++)
@@ -1122,6 +1470,14 @@ static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
 	free_irq(sndev->message_irq, sndev);
 }
 
+static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
+{
+	dev_info(&sndev->stdev->dev, "peer reinitialized\n");
+	switchtec_ntb_deinit_shared_mw(sndev);
+	switchtec_ntb_init_mw(sndev);
+	return switchtec_ntb_init_shared_mw(sndev);
+}
+
 static int switchtec_ntb_add(struct device *dev,
 			     struct class_interface *class_intf)
 {
@@ -1134,38 +1490,50 @@ static int switchtec_ntb_add(struct device *dev,
 	if (stdev->pdev->class != MICROSEMI_NTB_CLASSCODE)
 		return -ENODEV;
 
-	if (stdev->partition_count != 2)
-		dev_warn(dev, "ntb driver only supports 2 partitions");
-
 	sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
 	if (!sndev)
 		return -ENOMEM;
 
 	sndev->stdev = stdev;
-	switchtec_ntb_init_sndev(sndev);
+	rc = switchtec_ntb_init_sndev(sndev);
+	if (rc)
+		goto free_and_exit;
+
 	switchtec_ntb_init_mw(sndev);
-	switchtec_ntb_init_db(sndev);
-	switchtec_ntb_init_msgs(sndev);
 
 	rc = switchtec_ntb_init_req_id_table(sndev);
 	if (rc)
 		goto free_and_exit;
 
-	rc = switchtec_ntb_init_shared_mw(sndev);
+	rc = switchtec_ntb_init_crosslink(sndev);
 	if (rc)
 		goto free_and_exit;
 
+	switchtec_ntb_init_db(sndev);
+	switchtec_ntb_init_msgs(sndev);
+
+	rc = switchtec_ntb_init_shared_mw(sndev);
+	if (rc)
+		goto deinit_crosslink;
+
 	rc = switchtec_ntb_init_db_msg_irq(sndev);
 	if (rc)
 		goto deinit_shared_and_exit;
 
+	/*
+	 * If this host crashed, the other host may think the link is
+	 * still up. Tell them to force it down (it will go back up
+	 * once we register the ntb device).
+	 */
+	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
+
 	rc = ntb_register_device(&sndev->ntb);
 	if (rc)
 		goto deinit_and_exit;
 
 	stdev->sndev = sndev;
 	stdev->link_notifier = switchtec_ntb_link_notification;
-	dev_info(dev, "NTB device registered");
+	dev_info(dev, "NTB device registered\n");
 
 	return 0;
 
@@ -1173,14 +1541,16 @@ static int switchtec_ntb_add(struct device *dev,
 	switchtec_ntb_deinit_db_msg_irq(sndev);
 deinit_shared_and_exit:
 	switchtec_ntb_deinit_shared_mw(sndev);
+deinit_crosslink:
+	switchtec_ntb_deinit_crosslink(sndev);
 free_and_exit:
 	kfree(sndev);
-	dev_err(dev, "failed to register ntb device: %d", rc);
+	dev_err(dev, "failed to register ntb device: %d\n", rc);
 	return rc;
 }
 
-void switchtec_ntb_remove(struct device *dev,
-			  struct class_interface *class_intf)
+static void switchtec_ntb_remove(struct device *dev,
+				 struct class_interface *class_intf)
 {
 	struct switchtec_dev *stdev = to_stdev(dev);
 	struct switchtec_ntb *sndev = stdev->sndev;
@@ -1193,8 +1563,9 @@ void switchtec_ntb_remove(struct device *dev,
 	ntb_unregister_device(&sndev->ntb);
 	switchtec_ntb_deinit_db_msg_irq(sndev);
 	switchtec_ntb_deinit_shared_mw(sndev);
+	switchtec_ntb_deinit_crosslink(sndev);
 	kfree(sndev);
-	dev_info(dev, "ntb device unregistered");
+	dev_info(dev, "ntb device unregistered\n");
 }
 
 static struct class_interface switchtec_interface  = {
diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 03b80d89b980..2581ab724c34 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -63,12 +63,11 @@
 #define DRIVER_NAME			"ntb"
 #define DRIVER_DESCRIPTION		"PCIe NTB Driver Framework"
 
-#define DRIVER_LICENSE			"Dual BSD/GPL"
 #define DRIVER_VERSION			"1.0"
 #define DRIVER_RELDATE			"24 March 2015"
 #define DRIVER_AUTHOR			"Allen Hubbe <Allen.Hubbe@emc.com>"
 
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
@@ -112,7 +111,6 @@ int ntb_register_device(struct ntb_dev *ntb)
 
 	init_completion(&ntb->released);
 
-	memset(&ntb->dev, 0, sizeof(ntb->dev));
 	ntb->dev.bus = &ntb_bus;
 	ntb->dev.parent = &ntb->pdev->dev;
 	ntb->dev.release = ntb_dev_release;
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 045e3dd4750e..9878c48826e3 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -1003,6 +1003,9 @@ static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
 	mw_base = nt->mw_vec[mw_num].phys_addr;
 	mw_size = nt->mw_vec[mw_num].phys_size;
 
+	if (max_mw_size && mw_size > max_mw_size)
+		mw_size = max_mw_size;
+
 	tx_size = (unsigned int)mw_size / num_qps_mw;
 	qp_offset = tx_size * (qp_num / mw_count);
 
diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index 427112cf101a..2a9d6b0d1f19 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2017 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
  *   BSD LICENSE
  *
  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2017 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -40,860 +42,1474 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- *   PCIe NTB Perf Linux driver
+ * PCIe NTB Perf Linux driver
+ */
+
+/*
+ * How to use this tool, by example.
+ *
+ * Assuming $DBG_DIR is something like:
+ * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
+ * Suppose aside from local device there is at least one remote device
+ * connected to NTB with index 0.
+ *-----------------------------------------------------------------------------
+ * Eg: install driver with specified chunk/total orders and dma-enabled flag
+ *
+ * root@self# insmod ntb_perf.ko chunk_order=19 total_order=28 use_dma
+ *-----------------------------------------------------------------------------
+ * Eg: check NTB ports (index) and MW mapping information
+ *
+ * root@self# cat $DBG_DIR/info
+ *-----------------------------------------------------------------------------
+ * Eg: start performance test with peer (index 0) and get the test metrics
+ *
+ * root@self# echo 0 > $DBG_DIR/run
+ * root@self# cat $DBG_DIR/run
  */
 
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
-#include <linux/kthread.h>
-#include <linux/time.h>
-#include <linux/timer.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
 #include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
 #include <linux/pci.h>
+#include <linux/ktime.h>
 #include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <linux/debugfs.h>
-#include <linux/dmaengine.h>
 #include <linux/delay.h>
 #include <linux/sizes.h>
+#include <linux/workqueue.h>
+#include <linux/debugfs.h>
+#include <linux/random.h>
 #include <linux/ntb.h>
-#include <linux/mutex.h>
 
 #define DRIVER_NAME		"ntb_perf"
-#define DRIVER_DESCRIPTION	"PCIe NTB Performance Measurement Tool"
-
-#define DRIVER_LICENSE		"Dual BSD/GPL"
-#define DRIVER_VERSION		"1.0"
-#define DRIVER_AUTHOR		"Dave Jiang <dave.jiang@intel.com>"
-
-#define PERF_LINK_DOWN_TIMEOUT	10
-#define PERF_VERSION		0xffff0001
-#define MAX_THREADS		32
-#define MAX_TEST_SIZE		SZ_1M
-#define MAX_SRCS		32
-#define DMA_OUT_RESOURCE_TO	msecs_to_jiffies(50)
-#define DMA_RETRIES		20
-#define SZ_4G			(1ULL << 32)
-#define MAX_SEG_ORDER		20 /* no larger than 1M for kmalloc buffer */
-#define PIDX			NTB_DEF_PEER_IDX
-
-MODULE_LICENSE(DRIVER_LICENSE);
+#define DRIVER_VERSION		"2.0"
+
+MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(DRIVER_VERSION);
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
+MODULE_AUTHOR("Dave Jiang <dave.jiang@intel.com>");
+MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
+
+#define MAX_THREADS_CNT		32
+#define DEF_THREADS_CNT		1
+#define MAX_CHUNK_SIZE		SZ_1M
+#define MAX_CHUNK_ORDER		20 /* no larger than 1M */
 
-static struct dentry *perf_debugfs_dir;
+#define DMA_TRIES		100
+#define DMA_MDELAY		10
+
+#define MSG_TRIES		500
+#define MSG_UDELAY_LOW		1000
+#define MSG_UDELAY_HIGH		2000
+
+#define PERF_BUF_LEN 1024
 
 static unsigned long max_mw_size;
 module_param(max_mw_size, ulong, 0644);
-MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
+MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size");
 
-static unsigned int seg_order = 19; /* 512K */
-module_param(seg_order, uint, 0644);
-MODULE_PARM_DESC(seg_order, "size order [2^n] of buffer segment for testing");
+static unsigned char chunk_order = 19; /* 512K */
+module_param(chunk_order, byte, 0644);
+MODULE_PARM_DESC(chunk_order, "Data chunk order [2^n] to transfer");
 
-static unsigned int run_order = 32; /* 4G */
-module_param(run_order, uint, 0644);
-MODULE_PARM_DESC(run_order, "size order [2^n] of total data to transfer");
+static unsigned char total_order = 30; /* 1G */
+module_param(total_order, byte, 0644);
+MODULE_PARM_DESC(total_order, "Total data order [2^n] to transfer");
 
 static bool use_dma; /* default to 0 */
 module_param(use_dma, bool, 0644);
-MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");
-
-static bool on_node = true; /* default to 1 */
-module_param(on_node, bool, 0644);
-MODULE_PARM_DESC(on_node, "Run threads only on NTB device node (default: true)");
-
-struct perf_mw {
-	phys_addr_t	phys_addr;
-	resource_size_t	phys_size;
-	void __iomem	*vbase;
-	size_t		xlat_size;
-	size_t		buf_size;
-	void		*virt_addr;
-	dma_addr_t	dma_addr;
+MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance");
+
+/*==============================================================================
+ *                         Perf driver data definition
+ *==============================================================================
+ */
+
+enum perf_cmd {
+	PERF_CMD_INVAL = -1,/* invalid spad command */
+	PERF_CMD_SSIZE = 0, /* send out buffer size */
+	PERF_CMD_RSIZE = 1, /* recv in  buffer size */
+	PERF_CMD_SXLAT = 2, /* send in  buffer xlat */
+	PERF_CMD_RXLAT = 3, /* recv out buffer xlat */
+	PERF_CMD_CLEAR = 4, /* clear allocated memory */
+	PERF_STS_DONE  = 5, /* init is done */
+	PERF_STS_LNKUP = 6, /* link up state flag */
 };
 
 struct perf_ctx;
 
-struct pthr_ctx {
-	struct task_struct	*thread;
-	struct perf_ctx		*perf;
-	atomic_t		dma_sync;
-	struct dma_chan		*dma_chan;
-	int			dma_prep_err;
-	int			src_idx;
-	void			*srcs[MAX_SRCS];
-	wait_queue_head_t       *wq;
-	int			status;
-	u64			copied;
-	u64			diff_us;
+struct perf_peer {
+	struct perf_ctx	*perf;
+	int pidx;
+	int gidx;
+
+	/* Outbound MW params */
+	u64 outbuf_xlat;
+	resource_size_t outbuf_size;
+	void __iomem *outbuf;
+
+	/* Inbound MW params */
+	dma_addr_t inbuf_xlat;
+	resource_size_t inbuf_size;
+	void		*inbuf;
+
+	/* NTB connection setup service */
+	struct work_struct	service;
+	unsigned long		sts;
 };
+#define to_peer_service(__work) \
+	container_of(__work, struct perf_peer, service)
 
-struct perf_ctx {
-	struct ntb_dev		*ntb;
-	spinlock_t		db_lock;
-	struct perf_mw		mw;
-	bool			link_is_up;
-	struct delayed_work	link_work;
-	wait_queue_head_t	link_wq;
-	u8			perf_threads;
-	/* mutex ensures only one set of threads run at once */
-	struct mutex		run_mutex;
-	struct pthr_ctx		pthr_ctx[MAX_THREADS];
-	atomic_t		tsync;
-	atomic_t                tdone;
+struct perf_thread {
+	struct perf_ctx *perf;
+	int tidx;
+
+	/* DMA-based test sync parameters */
+	atomic_t dma_sync;
+	wait_queue_head_t dma_wait;
+	struct dma_chan *dma_chan;
+
+	/* Data source and measured statistics */
+	void *src;
+	u64 copied;
+	ktime_t duration;
+	int status;
+	struct work_struct work;
 };
+#define to_thread_work(__work) \
+	container_of(__work, struct perf_thread, work)
 
-enum {
-	VERSION = 0,
-	MW_SZ_HIGH,
-	MW_SZ_LOW,
-	MAX_SPAD
+struct perf_ctx {
+	struct ntb_dev *ntb;
+
+	/* Global device index and peers descriptors */
+	int gidx;
+	int pcnt;
+	struct perf_peer *peers;
+
+	/* Performance measuring work-threads interface */
+	unsigned long busy_flag;
+	wait_queue_head_t twait;
+	atomic_t tsync;
+	u8 tcnt;
+	struct perf_peer *test_peer;
+	struct perf_thread threads[MAX_THREADS_CNT];
+
+	/* Scratchpad/Message IO operations */
+	int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
+	int (*cmd_recv)(struct perf_ctx *perf, int *pidx, enum perf_cmd *cmd,
+			u64 *data);
+
+	struct dentry *dbgfs_dir;
 };
 
+/*
+ * Scratchpads-base commands interface
+ */
+#define PERF_SPAD_CNT(_pcnt) \
+	(3*((_pcnt) + 1))
+#define PERF_SPAD_CMD(_gidx) \
+	(3*(_gidx))
+#define PERF_SPAD_LDATA(_gidx) \
+	(3*(_gidx) + 1)
+#define PERF_SPAD_HDATA(_gidx) \
+	(3*(_gidx) + 2)
+#define PERF_SPAD_NOTIFY(_gidx) \
+	(BIT_ULL(_gidx))
+
+/*
+ * Messages-base commands interface
+ */
+#define PERF_MSG_CNT		3
+#define PERF_MSG_CMD		0
+#define PERF_MSG_LDATA		1
+#define PERF_MSG_HDATA		2
+
+/*==============================================================================
+ *                           Static data declarations
+ *==============================================================================
+ */
+
+static struct dentry *perf_dbgfs_topdir;
+
+static struct workqueue_struct *perf_wq __read_mostly;
+
+/*==============================================================================
+ *                  NTB cross-link commands execution service
+ *==============================================================================
+ */
+
+static void perf_terminate_test(struct perf_ctx *perf);
+
+static inline bool perf_link_is_up(struct perf_peer *peer)
+{
+	u64 link;
+
+	link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);
+	return !!(link & BIT_ULL_MASK(peer->pidx));
+}
+
+static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
+			      u64 data)
+{
+	struct perf_ctx *perf = peer->perf;
+	int try;
+	u32 sts;
+
+	dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
+
+	/*
+	 * Perform predefined number of attempts before give up.
+	 * We are sending the data to the port specific scratchpad, so
+	 * to prevent a multi-port access race-condition. Additionally
+	 * there is no need in local locking since only thread-safe
+	 * service work is using this method.
+	 */
+	for (try = 0; try < MSG_TRIES; try++) {
+		if (!perf_link_is_up(peer))
+			return -ENOLINK;
+
+		sts = ntb_peer_spad_read(perf->ntb, peer->pidx,
+					 PERF_SPAD_CMD(perf->gidx));
+		if (sts != PERF_CMD_INVAL) {
+			usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
+			continue;
+		}
+
+		ntb_peer_spad_write(perf->ntb, peer->pidx,
+				    PERF_SPAD_LDATA(perf->gidx),
+				    lower_32_bits(data));
+		ntb_peer_spad_write(perf->ntb, peer->pidx,
+				    PERF_SPAD_HDATA(perf->gidx),
+				    upper_32_bits(data));
+		mmiowb();
+		ntb_peer_spad_write(perf->ntb, peer->pidx,
+				    PERF_SPAD_CMD(perf->gidx),
+				    cmd);
+		mmiowb();
+		ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));
+
+		dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",
+			PERF_SPAD_NOTIFY(peer->gidx));
+
+		break;
+	}
+
+	return try < MSG_TRIES ? 0 : -EAGAIN;
+}
+
+static int perf_spad_cmd_recv(struct perf_ctx *perf, int *pidx,
+			      enum perf_cmd *cmd, u64 *data)
+{
+	struct perf_peer *peer;
+	u32 val;
+
+	ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
+
+	/*
+	 * We start scanning all over, since cleared DB may have been set
+	 * by any peer. Yes, it makes peer with smaller index being
+	 * serviced with greater priority, but it's convenient for spad
+	 * and message code unification and simplicity.
+	 */
+	for (*pidx = 0; *pidx < perf->pcnt; (*pidx)++) {
+		peer = &perf->peers[*pidx];
+
+		if (!perf_link_is_up(peer))
+			continue;
+
+		val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));
+		if (val == PERF_CMD_INVAL)
+			continue;
+
+		*cmd = val;
+
+		val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));
+		*data = val;
+
+		val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));
+		*data |= (u64)val << 32;
+
+		/* Next command can be retrieved from now */
+		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),
+			       PERF_CMD_INVAL);
+
+		dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
+
+		return 0;
+	}
+
+	return -ENODATA;
+}
+
+static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
+			     u64 data)
+{
+	struct perf_ctx *perf = peer->perf;
+	int try, ret;
+	u64 outbits;
+
+	dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
+
+	/*
+	 * Perform predefined number of attempts before give up. Message
+	 * registers are free of race-condition problem when accessed
+	 * from different ports, so we don't need splitting registers
+	 * by global device index. We also won't have local locking,
+	 * since the method is used from service work only.
+	 */
+	outbits = ntb_msg_outbits(perf->ntb);
+	for (try = 0; try < MSG_TRIES; try++) {
+		if (!perf_link_is_up(peer))
+			return -ENOLINK;
+
+		ret = ntb_msg_clear_sts(perf->ntb, outbits);
+		if (ret)
+			return ret;
+
+		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,
+				   lower_32_bits(data));
+
+		if (ntb_msg_read_sts(perf->ntb) & outbits) {
+			usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
+			continue;
+		}
+
+		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,
+				   upper_32_bits(data));
+		mmiowb();
+
+		/* This call shall trigger peer message event */
+		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);
+
+		break;
+	}
+
+	return try < MSG_TRIES ? 0 : -EAGAIN;
+}
+
+static int perf_msg_cmd_recv(struct perf_ctx *perf, int *pidx,
+			     enum perf_cmd *cmd, u64 *data)
+{
+	u64 inbits;
+	u32 val;
+
+	inbits = ntb_msg_inbits(perf->ntb);
+
+	if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)
+		return -ENODATA;
+
+	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);
+	*cmd = val;
+
+	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);
+	*data = val;
+
+	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);
+	*data |= (u64)val << 32;
+
+	/* Next command can be retrieved from now */
+	ntb_msg_clear_sts(perf->ntb, inbits);
+
+	dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
+
+	return 0;
+}
+
+static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)
+{
+	struct perf_ctx *perf = peer->perf;
+
+	if (cmd == PERF_CMD_SSIZE || cmd == PERF_CMD_SXLAT)
+		return perf->cmd_send(peer, cmd, data);
+
+	dev_err(&perf->ntb->dev, "Send invalid command\n");
+	return -EINVAL;
+}
+
+static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)
+{
+	switch (cmd) {
+	case PERF_CMD_SSIZE:
+	case PERF_CMD_RSIZE:
+	case PERF_CMD_SXLAT:
+	case PERF_CMD_RXLAT:
+	case PERF_CMD_CLEAR:
+		break;
+	default:
+		dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");
+		return -EINVAL;
+	}
+
+	/* No need of memory barrier, since bit ops have invernal lock */
+	set_bit(cmd, &peer->sts);
+
+	dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);
+
+	(void)queue_work(system_highpri_wq, &peer->service);
+
+	return 0;
+}
+
+static int perf_cmd_recv(struct perf_ctx *perf)
+{
+	struct perf_peer *peer;
+	int ret, pidx, cmd;
+	u64 data;
+
+	while (!(ret = perf->cmd_recv(perf, &pidx, &cmd, &data))) {
+		peer = &perf->peers[pidx];
+
+		switch (cmd) {
+		case PERF_CMD_SSIZE:
+			peer->inbuf_size = data;
+			return perf_cmd_exec(peer, PERF_CMD_RSIZE);
+		case PERF_CMD_SXLAT:
+			peer->outbuf_xlat = data;
+			return perf_cmd_exec(peer, PERF_CMD_RXLAT);
+		default:
+			dev_err(&perf->ntb->dev, "Recv invalid command\n");
+			return -EINVAL;
+		}
+	}
+
+	/* Return 0 if no data left to process, otherwise an error */
+	return ret == -ENODATA ? 0 : ret;
+}
+
 static void perf_link_event(void *ctx)
 {
 	struct perf_ctx *perf = ctx;
+	struct perf_peer *peer;
+	bool lnk_up;
+	int pidx;
 
-	if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) {
-		schedule_delayed_work(&perf->link_work, 2*HZ);
-	} else {
-		dev_dbg(&perf->ntb->pdev->dev, "link down\n");
+	for (pidx = 0; pidx < perf->pcnt; pidx++) {
+		peer = &perf->peers[pidx];
 
-		if (!perf->link_is_up)
-			cancel_delayed_work_sync(&perf->link_work);
+		lnk_up = perf_link_is_up(peer);
 
-		perf->link_is_up = false;
+		if (lnk_up &&
+		    !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {
+			perf_cmd_exec(peer, PERF_CMD_SSIZE);
+		} else if (!lnk_up &&
+			   test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {
+			perf_cmd_exec(peer, PERF_CMD_CLEAR);
+		}
 	}
 }
 
 static void perf_db_event(void *ctx, int vec)
 {
 	struct perf_ctx *perf = ctx;
-	u64 db_bits, db_mask;
 
-	db_mask = ntb_db_vector_mask(perf->ntb, vec);
-	db_bits = ntb_db_read(perf->ntb);
+	dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,
+		ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));
+
+	/* Just receive all available commands */
+	(void)perf_cmd_recv(perf);
+}
+
+static void perf_msg_event(void *ctx)
+{
+	struct perf_ctx *perf = ctx;
+
+	dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",
+		ntb_msg_read_sts(perf->ntb));
 
-	dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
-		vec, db_mask, db_bits);
+	/* Messages are only sent one-by-one */
+	(void)perf_cmd_recv(perf);
 }
 
 static const struct ntb_ctx_ops perf_ops = {
 	.link_event = perf_link_event,
 	.db_event = perf_db_event,
+	.msg_event = perf_msg_event
 };
 
-static void perf_copy_callback(void *data)
+static void perf_free_outbuf(struct perf_peer *peer)
+{
+	(void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
+}
+
+static int perf_setup_outbuf(struct perf_peer *peer)
 {
-	struct pthr_ctx *pctx = data;
+	struct perf_ctx *perf = peer->perf;
+	int ret;
+
+	/* Outbuf size can be unaligned due to custom max_mw_size */
+	ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
+				    peer->outbuf_xlat, peer->outbuf_size);
+	if (ret) {
+		dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");
+		return ret;
+	}
 
-	atomic_dec(&pctx->dma_sync);
+	/* Initialization is finally done */
+	set_bit(PERF_STS_DONE, &peer->sts);
+
+	return 0;
 }
 
-static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst,
-			 char *src, size_t size)
+static void perf_free_inbuf(struct perf_peer *peer)
 {
-	struct perf_ctx *perf = pctx->perf;
-	struct dma_async_tx_descriptor *txd;
-	struct dma_chan *chan = pctx->dma_chan;
-	struct dma_device *device;
-	struct dmaengine_unmap_data *unmap;
-	dma_cookie_t cookie;
-	size_t src_off, dst_off;
-	struct perf_mw *mw = &perf->mw;
-	void __iomem *vbase;
-	void __iomem *dst_vaddr;
-	dma_addr_t dst_phys;
-	int retries = 0;
+	if (!peer->inbuf)
+		return;
 
-	if (!use_dma) {
-		memcpy_toio(dst, src, size);
-		return size;
+	(void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
+	dma_free_coherent(&peer->perf->ntb->dev, peer->inbuf_size,
+			  peer->inbuf, peer->inbuf_xlat);
+	peer->inbuf = NULL;
+}
+
+static int perf_setup_inbuf(struct perf_peer *peer)
+{
+	resource_size_t xlat_align, size_align, size_max;
+	struct perf_ctx *perf = peer->perf;
+	int ret;
+
+	/* Get inbound MW parameters */
+	ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,
+			       &xlat_align, &size_align, &size_max);
+	if (ret) {
+		dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");
+		return ret;
 	}
 
-	if (!chan) {
-		dev_err(&perf->ntb->dev, "DMA engine does not exist\n");
+	if (peer->inbuf_size > size_max) {
+		dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",
+			&peer->inbuf_size, &size_max);
 		return -EINVAL;
 	}
 
-	device = chan->device;
-	src_off = (uintptr_t)src & ~PAGE_MASK;
-	dst_off = (uintptr_t __force)dst & ~PAGE_MASK;
-
-	if (!is_dma_copy_aligned(device, src_off, dst_off, size))
-		return -ENODEV;
+	peer->inbuf_size = round_up(peer->inbuf_size, size_align);
 
-	vbase = mw->vbase;
-	dst_vaddr = dst;
-	dst_phys = mw->phys_addr + (dst_vaddr - vbase);
+	perf_free_inbuf(peer);
 
-	unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
-	if (!unmap)
+	peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,
+					 &peer->inbuf_xlat, GFP_KERNEL);
+	if (!peer->inbuf) {
+		dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
+			&peer->inbuf_size);
 		return -ENOMEM;
+	}
+	if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {
+		dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");
+		goto err_free_inbuf;
+	}
 
-	unmap->len = size;
-	unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src),
-				      src_off, size, DMA_TO_DEVICE);
-	if (dma_mapping_error(device->dev, unmap->addr[0]))
-		goto err_get_unmap;
+	ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
+			       peer->inbuf_xlat, peer->inbuf_size);
+	if (ret) {
+		dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");
+		goto err_free_inbuf;
+	}
 
-	unmap->to_cnt = 1;
+	/*
+	 * We submit inbuf xlat transmission cmd for execution here to follow
+	 * the code architecture, even though this method is called from service
+	 * work itself so the command will be executed right after it returns.
+	 */
+	(void)perf_cmd_exec(peer, PERF_CMD_SXLAT);
 
-	do {
-		txd = device->device_prep_dma_memcpy(chan, dst_phys,
-						     unmap->addr[0],
-						     size, DMA_PREP_INTERRUPT);
-		if (!txd) {
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(DMA_OUT_RESOURCE_TO);
-		}
-	} while (!txd && (++retries < DMA_RETRIES));
+	return 0;
 
-	if (!txd) {
-		pctx->dma_prep_err++;
-		goto err_get_unmap;
-	}
+err_free_inbuf:
+	perf_free_inbuf(peer);
 
-	txd->callback = perf_copy_callback;
-	txd->callback_param = pctx;
-	dma_set_unmap(txd, unmap);
+	return ret;
+}
 
-	cookie = dmaengine_submit(txd);
-	if (dma_submit_error(cookie))
-		goto err_set_unmap;
+static void perf_service_work(struct work_struct *work)
+{
+	struct perf_peer *peer = to_peer_service(work);
 
-	dmaengine_unmap_put(unmap);
+	if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))
+		perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);
 
-	atomic_inc(&pctx->dma_sync);
-	dma_async_issue_pending(chan);
+	if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))
+		perf_setup_inbuf(peer);
 
-	return size;
+	if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))
+		perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);
 
-err_set_unmap:
-	dmaengine_unmap_put(unmap);
-err_get_unmap:
-	dmaengine_unmap_put(unmap);
-	return 0;
-}
+	if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))
+		perf_setup_outbuf(peer);
 
-static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src,
-			  u64 buf_size, u64 win_size, u64 total)
-{
-	int chunks, total_chunks, i;
-	int copied_chunks = 0;
-	u64 copied = 0, result;
-	char __iomem *tmp = dst;
-	u64 perf, diff_us;
-	ktime_t kstart, kstop, kdiff;
-	unsigned long last_sleep = jiffies;
-
-	chunks = div64_u64(win_size, buf_size);
-	total_chunks = div64_u64(total, buf_size);
-	kstart = ktime_get();
-
-	for (i = 0; i < total_chunks; i++) {
-		result = perf_copy(pctx, tmp, src, buf_size);
-		copied += result;
-		copied_chunks++;
-		if (copied_chunks == chunks) {
-			tmp = dst;
-			copied_chunks = 0;
-		} else
-			tmp += buf_size;
-
-		/* Probably should schedule every 5s to prevent soft hang. */
-		if (unlikely((jiffies - last_sleep) > 5 * HZ)) {
-			last_sleep = jiffies;
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(1);
+	if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {
+		clear_bit(PERF_STS_DONE, &peer->sts);
+		if (test_bit(0, &peer->perf->busy_flag) &&
+		    peer == peer->perf->test_peer) {
+			dev_warn(&peer->perf->ntb->dev,
+				"Freeing while test on-fly\n");
+			perf_terminate_test(peer->perf);
 		}
+		perf_free_outbuf(peer);
+		perf_free_inbuf(peer);
+	}
+}
+
+static int perf_init_service(struct perf_ctx *perf)
+{
+	u64 mask;
 
-		if (unlikely(kthread_should_stop()))
-			break;
+	if (ntb_peer_mw_count(perf->ntb) < perf->pcnt + 1) {
+		dev_err(&perf->ntb->dev, "Not enough memory windows\n");
+		return -EINVAL;
 	}
 
-	if (use_dma) {
-		pr_debug("%s: All DMA descriptors submitted\n", current->comm);
-		while (atomic_read(&pctx->dma_sync) != 0) {
-			if (kthread_should_stop())
-				break;
-			msleep(20);
-		}
+	if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {
+		perf->cmd_send = perf_msg_cmd_send;
+		perf->cmd_recv = perf_msg_cmd_recv;
+
+		dev_dbg(&perf->ntb->dev, "Message service initialized\n");
+
+		return 0;
 	}
 
-	kstop = ktime_get();
-	kdiff = ktime_sub(kstop, kstart);
-	diff_us = ktime_to_us(kdiff);
+	dev_dbg(&perf->ntb->dev, "Message service unsupported\n");
 
-	pr_debug("%s: copied %llu bytes\n", current->comm, copied);
+	mask = GENMASK_ULL(perf->pcnt, 0);
+	if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&
+	    (ntb_db_valid_mask(perf->ntb) & mask) == mask) {
+		perf->cmd_send = perf_spad_cmd_send;
+		perf->cmd_recv = perf_spad_cmd_recv;
 
-	pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us);
+		dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");
 
-	perf = div64_u64(copied, diff_us);
+		return 0;
+	}
 
-	pr_debug("%s: MBytes/s: %llu\n", current->comm, perf);
+	dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");
 
-	pctx->copied = copied;
-	pctx->diff_us = diff_us;
+	dev_err(&perf->ntb->dev, "Command services unsupported\n");
 
-	return 0;
+	return -EINVAL;
 }
 
-static bool perf_dma_filter_fn(struct dma_chan *chan, void *node)
+static int perf_enable_service(struct perf_ctx *perf)
 {
-	/* Is the channel required to be on the same node as the device? */
-	if (!on_node)
-		return true;
+	u64 mask, incmd_bit;
+	int ret, sidx, scnt;
 
-	return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
-}
+	mask = ntb_db_valid_mask(perf->ntb);
+	(void)ntb_db_set_mask(perf->ntb, mask);
 
-static int ntb_perf_thread(void *data)
-{
-	struct pthr_ctx *pctx = data;
-	struct perf_ctx *perf = pctx->perf;
-	struct pci_dev *pdev = perf->ntb->pdev;
-	struct perf_mw *mw = &perf->mw;
-	char __iomem *dst;
-	u64 win_size, buf_size, total;
-	void *src;
-	int rc, node, i;
-	struct dma_chan *dma_chan = NULL;
+	ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);
+	if (ret)
+		return ret;
 
-	pr_debug("kthread %s starting...\n", current->comm);
+	if (perf->cmd_send == perf_msg_cmd_send) {
+		u64 inbits, outbits;
 
-	node = on_node ? dev_to_node(&pdev->dev) : NUMA_NO_NODE;
+		inbits = ntb_msg_inbits(perf->ntb);
+		outbits = ntb_msg_outbits(perf->ntb);
+		(void)ntb_msg_set_mask(perf->ntb, inbits | outbits);
 
-	if (use_dma && !pctx->dma_chan) {
-		dma_cap_mask_t dma_mask;
+		incmd_bit = BIT_ULL(__ffs64(inbits));
+		ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);
 
-		dma_cap_zero(dma_mask);
-		dma_cap_set(DMA_MEMCPY, dma_mask);
-		dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn,
-					       (void *)(unsigned long)node);
-		if (!dma_chan) {
-			pr_warn("%s: cannot acquire DMA channel, quitting\n",
-				current->comm);
-			return -ENODEV;
-		}
-		pctx->dma_chan = dma_chan;
+		dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);
+	} else {
+		scnt = ntb_spad_count(perf->ntb);
+		for (sidx = 0; sidx < scnt; sidx++)
+			ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);
+		incmd_bit = PERF_SPAD_NOTIFY(perf->gidx);
+		ret = ntb_db_clear_mask(perf->ntb, incmd_bit);
+
+		dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);
+	}
+	if (ret) {
+		ntb_clear_ctx(perf->ntb);
+		return ret;
 	}
 
-	for (i = 0; i < MAX_SRCS; i++) {
-		pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node);
-		if (!pctx->srcs[i]) {
-			rc = -ENOMEM;
-			goto err;
-		}
+	ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
+	/* Might be not necessary */
+	ntb_link_event(perf->ntb);
+
+	return 0;
+}
+
+static void perf_disable_service(struct perf_ctx *perf)
+{
+	int pidx;
+
+	ntb_link_disable(perf->ntb);
+
+	if (perf->cmd_send == perf_msg_cmd_send) {
+		u64 inbits;
+
+		inbits = ntb_msg_inbits(perf->ntb);
+		(void)ntb_msg_set_mask(perf->ntb, inbits);
+	} else {
+		(void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
 	}
 
-	win_size = mw->phys_size;
-	buf_size = 1ULL << seg_order;
-	total = 1ULL << run_order;
+	ntb_clear_ctx(perf->ntb);
 
-	if (buf_size > MAX_TEST_SIZE)
-		buf_size = MAX_TEST_SIZE;
+	for (pidx = 0; pidx < perf->pcnt; pidx++)
+		perf_cmd_exec(&perf->peers[pidx], PERF_CMD_CLEAR);
 
-	dst = (char __iomem *)mw->vbase;
+	for (pidx = 0; pidx < perf->pcnt; pidx++)
+		flush_work(&perf->peers[pidx].service);
+}
 
-	atomic_inc(&perf->tsync);
-	while (atomic_read(&perf->tsync) != perf->perf_threads)
-		schedule();
+/*==============================================================================
+ *                      Performance measuring work-thread
+ *==============================================================================
+ */
 
-	src = pctx->srcs[pctx->src_idx];
-	pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1);
+static void perf_dma_copy_callback(void *data)
+{
+	struct perf_thread *pthr = data;
 
-	rc = perf_move_data(pctx, dst, src, buf_size, win_size, total);
+	atomic_dec(&pthr->dma_sync);
+	wake_up(&pthr->dma_wait);
+}
 
-	atomic_dec(&perf->tsync);
+static int perf_copy_chunk(struct perf_thread *pthr,
+			   void __iomem *dst, void *src, size_t len)
+{
+	struct dma_async_tx_descriptor *tx;
+	struct dmaengine_unmap_data *unmap;
+	struct device *dma_dev;
+	int try = 0, ret = 0;
 
-	if (rc < 0) {
-		pr_err("%s: failed\n", current->comm);
-		rc = -ENXIO;
-		goto err;
+	if (!use_dma) {
+		memcpy_toio(dst, src, len);
+		goto ret_check_tsync;
 	}
 
-	for (i = 0; i < MAX_SRCS; i++) {
-		kfree(pctx->srcs[i]);
-		pctx->srcs[i] = NULL;
+	dma_dev = pthr->dma_chan->device->dev;
+
+	if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),
+				 offset_in_page(dst), len))
+		return -EIO;
+
+	unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
+	if (!unmap)
+		return -ENOMEM;
+
+	unmap->len = len;
+	unmap->addr[0] = dma_map_page(dma_dev, virt_to_page(src),
+		offset_in_page(src), len, DMA_TO_DEVICE);
+	if (dma_mapping_error(dma_dev, unmap->addr[0])) {
+		ret = -EIO;
+		goto err_free_resource;
 	}
+	unmap->to_cnt = 1;
 
-	atomic_inc(&perf->tdone);
-	wake_up(pctx->wq);
-	rc = 0;
-	goto done;
+	unmap->addr[1] = dma_map_page(dma_dev, virt_to_page(dst),
+		offset_in_page(dst), len, DMA_FROM_DEVICE);
+	if (dma_mapping_error(dma_dev, unmap->addr[1])) {
+		ret = -EIO;
+		goto err_free_resource;
+	}
+	unmap->from_cnt = 1;
 
-err:
-	for (i = 0; i < MAX_SRCS; i++) {
-		kfree(pctx->srcs[i]);
-		pctx->srcs[i] = NULL;
+	do {
+		tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, unmap->addr[1],
+			unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+		if (!tx)
+			msleep(DMA_MDELAY);
+	} while (!tx && (try++ < DMA_TRIES));
+
+	if (!tx) {
+		ret = -EIO;
+		goto err_free_resource;
 	}
 
-	if (dma_chan) {
-		dma_release_channel(dma_chan);
-		pctx->dma_chan = NULL;
+	tx->callback = perf_dma_copy_callback;
+	tx->callback_param = pthr;
+	dma_set_unmap(tx, unmap);
+
+	ret = dma_submit_error(dmaengine_submit(tx));
+	if (ret) {
+		dmaengine_unmap_put(unmap);
+		goto err_free_resource;
 	}
 
-done:
-	/* Wait until we are told to stop */
-	for (;;) {
-		set_current_state(TASK_INTERRUPTIBLE);
-		if (kthread_should_stop())
-			break;
-		schedule();
+	dmaengine_unmap_put(unmap);
+
+	atomic_inc(&pthr->dma_sync);
+	dma_async_issue_pending(pthr->dma_chan);
+
+ret_check_tsync:
+	return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
+
+err_free_resource:
+	dmaengine_unmap_put(unmap);
+
+	return ret;
+}
+
+static bool perf_dma_filter(struct dma_chan *chan, void *data)
+{
+	struct perf_ctx *perf = data;
+	int node;
+
+	node = dev_to_node(&perf->ntb->dev);
+
+	return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
+}
+
+static int perf_init_test(struct perf_thread *pthr)
+{
+	struct perf_ctx *perf = pthr->perf;
+	dma_cap_mask_t dma_mask;
+
+	pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
+				 dev_to_node(&perf->ntb->dev));
+	if (!pthr->src)
+		return -ENOMEM;
+
+	get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
+
+	if (!use_dma)
+		return 0;
+
+	dma_cap_zero(dma_mask);
+	dma_cap_set(DMA_MEMCPY, dma_mask);
+	pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
+	if (!pthr->dma_chan) {
+		dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
+			pthr->tidx);
+		atomic_dec(&perf->tsync);
+		wake_up(&perf->twait);
+		kfree(pthr->src);
+		return -ENODEV;
 	}
-	__set_current_state(TASK_RUNNING);
 
-	return rc;
+	atomic_set(&pthr->dma_sync, 0);
+
+	return 0;
 }
 
-static void perf_free_mw(struct perf_ctx *perf)
+static int perf_run_test(struct perf_thread *pthr)
 {
-	struct perf_mw *mw = &perf->mw;
-	struct pci_dev *pdev = perf->ntb->pdev;
+	struct perf_peer *peer = pthr->perf->test_peer;
+	struct perf_ctx *perf = pthr->perf;
+	void __iomem *flt_dst, *bnd_dst;
+	u64 total_size, chunk_size;
+	void *flt_src;
+	int ret = 0;
+
+	total_size = 1ULL << total_order;
+	chunk_size = 1ULL << chunk_order;
+	chunk_size = min_t(u64, peer->outbuf_size, chunk_size);
+
+	flt_src = pthr->src;
+	bnd_dst = peer->outbuf + peer->outbuf_size;
+	flt_dst = peer->outbuf;
+
+	pthr->duration = ktime_get();
+
+	/* Copied field is cleared on test launch stage */
+	while (pthr->copied < total_size) {
+		ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size);
+		if (ret) {
+			dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",
+				pthr->tidx, ret);
+			return ret;
+		}
 
-	if (!mw->virt_addr)
-		return;
+		pthr->copied += chunk_size;
+
+		flt_dst += chunk_size;
+		flt_src += chunk_size;
+		if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {
+			flt_dst = peer->outbuf;
+			flt_src = pthr->src;
+		}
 
-	ntb_mw_clear_trans(perf->ntb, PIDX, 0);
-	dma_free_coherent(&pdev->dev, mw->buf_size,
-			  mw->virt_addr, mw->dma_addr);
-	mw->xlat_size = 0;
-	mw->buf_size = 0;
-	mw->virt_addr = NULL;
+		/* Give up CPU to give a chance for other threads to use it */
+		schedule();
+	}
+
+	return 0;
 }
 
-static int perf_set_mw(struct perf_ctx *perf, resource_size_t size)
+static int perf_sync_test(struct perf_thread *pthr)
 {
-	struct perf_mw *mw = &perf->mw;
-	size_t xlat_size, buf_size;
-	resource_size_t	xlat_align;
-	resource_size_t	xlat_align_size;
-	int rc;
+	struct perf_ctx *perf = pthr->perf;
 
-	if (!size)
-		return -EINVAL;
+	if (!use_dma)
+		goto no_dma_ret;
 
-	rc = ntb_mw_get_align(perf->ntb, PIDX, 0, &xlat_align,
-			      &xlat_align_size, NULL);
-	if (rc)
-		return rc;
+	wait_event(pthr->dma_wait,
+		   (atomic_read(&pthr->dma_sync) == 0 ||
+		    atomic_read(&perf->tsync) < 0));
 
-	xlat_size = round_up(size, xlat_align_size);
-	buf_size = round_up(size, xlat_align);
+	if (atomic_read(&perf->tsync) < 0)
+		return -EINTR;
 
-	if (mw->xlat_size == xlat_size)
-		return 0;
+no_dma_ret:
+	pthr->duration = ktime_sub(ktime_get(), pthr->duration);
 
-	if (mw->buf_size)
-		perf_free_mw(perf);
+	dev_dbg(&perf->ntb->dev, "%d: copied %llu bytes\n",
+		pthr->tidx, pthr->copied);
 
-	mw->xlat_size = xlat_size;
-	mw->buf_size = buf_size;
+	dev_dbg(&perf->ntb->dev, "%d: lasted %llu usecs\n",
+		pthr->tidx, ktime_to_us(pthr->duration));
+
+	dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,
+		div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
+
+	return 0;
+}
+
+static void perf_clear_test(struct perf_thread *pthr)
+{
+	struct perf_ctx *perf = pthr->perf;
+
+	if (!use_dma)
+		goto no_dma_notify;
+
+	/*
+	 * If test finished without errors, termination isn't needed.
+	 * We call it anyway just to be sure of the transfers completion.
+	 */
+	(void)dmaengine_terminate_sync(pthr->dma_chan);
+
+	dma_release_channel(pthr->dma_chan);
+
+no_dma_notify:
+	atomic_dec(&perf->tsync);
+	wake_up(&perf->twait);
+	kfree(pthr->src);
+}
 
-	mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size,
-					   &mw->dma_addr, GFP_KERNEL);
-	if (!mw->virt_addr) {
-		mw->xlat_size = 0;
-		mw->buf_size = 0;
+static void perf_thread_work(struct work_struct *work)
+{
+	struct perf_thread *pthr = to_thread_work(work);
+	int ret;
+
+	/*
+	 * Perform stages in compliance with use_dma flag value.
+	 * Test status is changed only if error happened, otherwise
+	 * status -ENODATA is kept while test is on-fly. Results
+	 * synchronization is performed only if test fininshed
+	 * without an error or interruption.
+	 */
+	ret = perf_init_test(pthr);
+	if (ret) {
+		pthr->status = ret;
+		return;
 	}
 
-	rc = ntb_mw_set_trans(perf->ntb, PIDX, 0, mw->dma_addr, mw->xlat_size);
-	if (rc) {
-		dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n");
-		perf_free_mw(perf);
-		return -EIO;
+	ret = perf_run_test(pthr);
+	if (ret) {
+		pthr->status = ret;
+		goto err_clear_test;
 	}
 
-	return 0;
+	pthr->status = perf_sync_test(pthr);
+
+err_clear_test:
+	perf_clear_test(pthr);
 }
 
-static void perf_link_work(struct work_struct *work)
+static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
 {
-	struct perf_ctx *perf =
-		container_of(work, struct perf_ctx, link_work.work);
-	struct ntb_dev *ndev = perf->ntb;
-	struct pci_dev *pdev = ndev->pdev;
-	u32 val;
-	u64 size;
-	int rc;
+	if (tcnt == 0 || tcnt > MAX_THREADS_CNT)
+		return -EINVAL;
 
-	dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);
+	if (test_and_set_bit_lock(0, &perf->busy_flag))
+		return -EBUSY;
+
+	perf->tcnt = tcnt;
+
+	clear_bit_unlock(0, &perf->busy_flag);
 
-	size = perf->mw.phys_size;
+	return 0;
+}
 
-	if (max_mw_size && size > max_mw_size)
-		size = max_mw_size;
+static void perf_terminate_test(struct perf_ctx *perf)
+{
+	int tidx;
 
-	ntb_peer_spad_write(ndev, PIDX, MW_SZ_HIGH, upper_32_bits(size));
-	ntb_peer_spad_write(ndev, PIDX, MW_SZ_LOW, lower_32_bits(size));
-	ntb_peer_spad_write(ndev, PIDX, VERSION, PERF_VERSION);
+	atomic_set(&perf->tsync, -1);
+	wake_up(&perf->twait);
 
-	/* now read what peer wrote */
-	val = ntb_spad_read(ndev, VERSION);
-	if (val != PERF_VERSION) {
-		dev_dbg(&pdev->dev, "Remote version = %#x\n", val);
-		goto out;
+	for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
+		wake_up(&perf->threads[tidx].dma_wait);
+		cancel_work_sync(&perf->threads[tidx].work);
 	}
+}
+
+static int perf_submit_test(struct perf_peer *peer)
+{
+	struct perf_ctx *perf = peer->perf;
+	struct perf_thread *pthr;
+	int tidx, ret;
 
-	val = ntb_spad_read(ndev, MW_SZ_HIGH);
-	size = (u64)val << 32;
+	if (!test_bit(PERF_STS_DONE, &peer->sts))
+		return -ENOLINK;
 
-	val = ntb_spad_read(ndev, MW_SZ_LOW);
-	size |= val;
+	if (test_and_set_bit_lock(0, &perf->busy_flag))
+		return -EBUSY;
 
-	dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size);
+	perf->test_peer = peer;
+	atomic_set(&perf->tsync, perf->tcnt);
 
-	rc = perf_set_mw(perf, size);
-	if (rc)
-		goto out1;
+	for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
+		pthr = &perf->threads[tidx];
 
-	perf->link_is_up = true;
-	wake_up(&perf->link_wq);
+		pthr->status = -ENODATA;
+		pthr->copied = 0;
+		pthr->duration = ktime_set(0, 0);
+		if (tidx < perf->tcnt)
+			(void)queue_work(perf_wq, &pthr->work);
+	}
 
-	return;
+	ret = wait_event_interruptible(perf->twait,
+				       atomic_read(&perf->tsync) <= 0);
+	if (ret == -ERESTARTSYS) {
+		perf_terminate_test(perf);
+		ret = -EINTR;
+	}
 
-out1:
-	perf_free_mw(perf);
+	clear_bit_unlock(0, &perf->busy_flag);
 
-out:
-	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
-		schedule_delayed_work(&perf->link_work,
-				      msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT));
+	return ret;
 }
 
-static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf)
+static int perf_read_stats(struct perf_ctx *perf, char *buf,
+			   size_t size, ssize_t *pos)
 {
-	struct perf_mw *mw;
-	int rc;
+	struct perf_thread *pthr;
+	int tidx;
+
+	if (test_and_set_bit_lock(0, &perf->busy_flag))
+		return -EBUSY;
 
-	mw = &perf->mw;
+	(*pos) += scnprintf(buf + *pos, size - *pos,
+		"    Peer %d test statistics:\n", perf->test_peer->pidx);
 
-	rc = ntb_peer_mw_get_addr(ntb, 0, &mw->phys_addr, &mw->phys_size);
-	if (rc)
-		return rc;
+	for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
+		pthr = &perf->threads[tidx];
 
-	perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
-	if (!mw->vbase)
-		return -ENOMEM;
+		if (pthr->status == -ENODATA)
+			continue;
+
+		if (pthr->status) {
+			(*pos) += scnprintf(buf + *pos, size - *pos,
+				"%d: error status %d\n", tidx, pthr->status);
+			continue;
+		}
+
+		(*pos) += scnprintf(buf + *pos, size - *pos,
+			"%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
+			tidx, pthr->copied, ktime_to_us(pthr->duration),
+			div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
+	}
+
+	clear_bit_unlock(0, &perf->busy_flag);
 
 	return 0;
 }
 
-static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf,
-				size_t count, loff_t *offp)
+static void perf_init_threads(struct perf_ctx *perf)
 {
-	struct perf_ctx *perf = filp->private_data;
+	struct perf_thread *pthr;
+	int tidx;
+
+	perf->tcnt = DEF_THREADS_CNT;
+	perf->test_peer = &perf->peers[0];
+	init_waitqueue_head(&perf->twait);
+
+	for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
+		pthr = &perf->threads[tidx];
+
+		pthr->perf = perf;
+		pthr->tidx = tidx;
+		pthr->status = -ENODATA;
+		init_waitqueue_head(&pthr->dma_wait);
+		INIT_WORK(&pthr->work, perf_thread_work);
+	}
+}
+
+static void perf_clear_threads(struct perf_ctx *perf)
+{
+	perf_terminate_test(perf);
+}
+
+/*==============================================================================
+ *                               DebugFS nodes
+ *==============================================================================
+ */
+
+static ssize_t perf_dbgfs_read_info(struct file *filep, char __user *ubuf,
+				    size_t size, loff_t *offp)
+{
+	struct perf_ctx *perf = filep->private_data;
+	struct perf_peer *peer;
+	size_t buf_size;
+	ssize_t pos = 0;
+	int ret, pidx;
 	char *buf;
-	ssize_t ret, out_off = 0;
-	struct pthr_ctx *pctx;
-	int i;
-	u64 rate;
 
-	if (!perf)
-		return 0;
+	buf_size = min_t(size_t, size, 0x1000U);
 
-	buf = kmalloc(1024, GFP_KERNEL);
+	buf = kmalloc(buf_size, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
-	if (mutex_is_locked(&perf->run_mutex)) {
-		out_off = scnprintf(buf, 64, "running\n");
-		goto read_from_buf;
+	pos += scnprintf(buf + pos, buf_size - pos,
+		"    Performance measuring tool info:\n\n");
+
+	pos += scnprintf(buf + pos, buf_size - pos,
+		"Local port %d, Global index %d\n", ntb_port_number(perf->ntb),
+		perf->gidx);
+	pos += scnprintf(buf + pos, buf_size - pos, "Test status: ");
+	if (test_bit(0, &perf->busy_flag)) {
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"on-fly with port %d (%d)\n",
+			ntb_peer_port_number(perf->ntb, perf->test_peer->pidx),
+			perf->test_peer->pidx);
+	} else {
+		pos += scnprintf(buf + pos, buf_size - pos, "idle\n");
 	}
 
-	for (i = 0; i < MAX_THREADS; i++) {
-		pctx = &perf->pthr_ctx[i];
+	for (pidx = 0; pidx < perf->pcnt; pidx++) {
+		peer = &perf->peers[pidx];
+
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"Port %d (%d), Global index %d:\n",
+			ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,
+			peer->gidx);
+
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tLink status: %s\n",
+			test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");
+
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tOut buffer addr 0x%pK\n", peer->outbuf);
 
-		if (pctx->status == -ENODATA)
-			break;
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tOut buffer size %pa\n", &peer->outbuf_size);
 
-		if (pctx->status) {
-			out_off += scnprintf(buf + out_off, 1024 - out_off,
-					    "%d: error %d\n", i,
-					    pctx->status);
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);
+
+		if (!peer->inbuf) {
+			pos += scnprintf(buf + pos, buf_size - pos,
+				"\tIn buffer addr: unallocated\n");
 			continue;
 		}
 
-		rate = div64_u64(pctx->copied, pctx->diff_us);
-		out_off += scnprintf(buf + out_off, 1024 - out_off,
-			"%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
-			i, pctx->copied, pctx->diff_us, rate);
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tIn buffer addr 0x%pK\n", peer->inbuf);
+
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tIn buffer size %pa\n", &peer->inbuf_size);
+
+		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);
 	}
 
-read_from_buf:
-	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_off);
+	ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
 	kfree(buf);
 
 	return ret;
 }
 
-static void threads_cleanup(struct perf_ctx *perf)
+static const struct file_operations perf_dbgfs_info = {
+	.open = simple_open,
+	.read = perf_dbgfs_read_info
+};
+
+static ssize_t perf_dbgfs_read_run(struct file *filep, char __user *ubuf,
+				   size_t size, loff_t *offp)
 {
-	struct pthr_ctx *pctx;
-	int i;
+	struct perf_ctx *perf = filep->private_data;
+	ssize_t ret, pos = 0;
+	char *buf;
 
-	for (i = 0; i < MAX_THREADS; i++) {
-		pctx = &perf->pthr_ctx[i];
-		if (pctx->thread) {
-			pctx->status = kthread_stop(pctx->thread);
-			pctx->thread = NULL;
-		}
-	}
-}
+	buf = kmalloc(PERF_BUF_LEN, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
 
-static void perf_clear_thread_status(struct perf_ctx *perf)
-{
-	int i;
+	ret = perf_read_stats(perf, buf, PERF_BUF_LEN, &pos);
+	if (ret)
+		goto err_free;
+
+	ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
+err_free:
+	kfree(buf);
 
-	for (i = 0; i < MAX_THREADS; i++)
-		perf->pthr_ctx[i].status = -ENODATA;
+	return ret;
 }
 
-static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf,
-				 size_t count, loff_t *offp)
+static ssize_t perf_dbgfs_write_run(struct file *filep, const char __user *ubuf,
+				    size_t size, loff_t *offp)
 {
-	struct perf_ctx *perf = filp->private_data;
-	int node, i;
-	DECLARE_WAIT_QUEUE_HEAD(wq);
+	struct perf_ctx *perf = filep->private_data;
+	struct perf_peer *peer;
+	int pidx, ret;
 
-	if (wait_event_interruptible(perf->link_wq, perf->link_is_up))
-		return -ENOLINK;
+	ret = kstrtoint_from_user(ubuf, size, 0, &pidx);
+	if (ret)
+		return ret;
 
-	if (perf->perf_threads == 0)
+	if (pidx < 0 || pidx >= perf->pcnt)
 		return -EINVAL;
 
-	if (!mutex_trylock(&perf->run_mutex))
-		return -EBUSY;
+	peer = &perf->peers[pidx];
 
-	perf_clear_thread_status(perf);
+	ret = perf_submit_test(peer);
+	if (ret)
+		return ret;
 
-	if (perf->perf_threads > MAX_THREADS) {
-		perf->perf_threads = MAX_THREADS;
-		pr_info("Reset total threads to: %u\n", MAX_THREADS);
-	}
+	return size;
+}
 
-	/* no greater than 1M */
-	if (seg_order > MAX_SEG_ORDER) {
-		seg_order = MAX_SEG_ORDER;
-		pr_info("Fix seg_order to %u\n", seg_order);
-	}
+static const struct file_operations perf_dbgfs_run = {
+	.open = simple_open,
+	.read = perf_dbgfs_read_run,
+	.write = perf_dbgfs_write_run
+};
 
-	if (run_order < seg_order) {
-		run_order = seg_order;
-		pr_info("Fix run_order to %u\n", run_order);
-	}
+static ssize_t perf_dbgfs_read_tcnt(struct file *filep, char __user *ubuf,
+				    size_t size, loff_t *offp)
+{
+	struct perf_ctx *perf = filep->private_data;
+	char buf[8];
+	ssize_t pos;
 
-	node = on_node ? dev_to_node(&perf->ntb->pdev->dev)
-		       : NUMA_NO_NODE;
-	atomic_set(&perf->tdone, 0);
+	pos = scnprintf(buf, sizeof(buf), "%hhu\n", perf->tcnt);
 
-	/* launch kernel thread */
-	for (i = 0; i < perf->perf_threads; i++) {
-		struct pthr_ctx *pctx;
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
+}
 
-		pctx = &perf->pthr_ctx[i];
-		atomic_set(&pctx->dma_sync, 0);
-		pctx->perf = perf;
-		pctx->wq = &wq;
-		pctx->thread =
-			kthread_create_on_node(ntb_perf_thread,
-					       (void *)pctx,
-					       node, "ntb_perf %d", i);
-		if (IS_ERR(pctx->thread)) {
-			pctx->thread = NULL;
-			goto err;
-		} else {
-			wake_up_process(pctx->thread);
-		}
-	}
+static ssize_t perf_dbgfs_write_tcnt(struct file *filep,
+				     const char __user *ubuf,
+				     size_t size, loff_t *offp)
+{
+	struct perf_ctx *perf = filep->private_data;
+	int ret;
+	u8 val;
 
-	wait_event_interruptible(wq,
-		atomic_read(&perf->tdone) == perf->perf_threads);
+	ret = kstrtou8_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
 
-	threads_cleanup(perf);
-	mutex_unlock(&perf->run_mutex);
-	return count;
+	ret = perf_set_tcnt(perf, val);
+	if (ret)
+		return ret;
 
-err:
-	threads_cleanup(perf);
-	mutex_unlock(&perf->run_mutex);
-	return -ENXIO;
+	return size;
 }
 
-static const struct file_operations ntb_perf_debugfs_run = {
-	.owner = THIS_MODULE,
+static const struct file_operations perf_dbgfs_tcnt = {
 	.open = simple_open,
-	.read = debugfs_run_read,
-	.write = debugfs_run_write,
+	.read = perf_dbgfs_read_tcnt,
+	.write = perf_dbgfs_write_tcnt
 };
 
-static int perf_debugfs_setup(struct perf_ctx *perf)
+static void perf_setup_dbgfs(struct perf_ctx *perf)
 {
 	struct pci_dev *pdev = perf->ntb->pdev;
-	struct dentry *debugfs_node_dir;
-	struct dentry *debugfs_run;
-	struct dentry *debugfs_threads;
-	struct dentry *debugfs_seg_order;
-	struct dentry *debugfs_run_order;
-	struct dentry *debugfs_use_dma;
-	struct dentry *debugfs_on_node;
-
-	if (!debugfs_initialized())
-		return -ENODEV;
 
-	/* Assumpion: only one NTB device in the system */
-	if (!perf_debugfs_dir) {
-		perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
-		if (!perf_debugfs_dir)
-			return -ENODEV;
-	}
-
-	debugfs_node_dir = debugfs_create_dir(pci_name(pdev),
-					      perf_debugfs_dir);
-	if (!debugfs_node_dir)
-		goto err;
-
-	debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR,
-					  debugfs_node_dir, perf,
-					  &ntb_perf_debugfs_run);
-	if (!debugfs_run)
-		goto err;
-
-	debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR,
-					    debugfs_node_dir,
-					    &perf->perf_threads);
-	if (!debugfs_threads)
-		goto err;
-
-	debugfs_seg_order = debugfs_create_u32("seg_order", 0600,
-					       debugfs_node_dir,
-					       &seg_order);
-	if (!debugfs_seg_order)
-		goto err;
-
-	debugfs_run_order = debugfs_create_u32("run_order", 0600,
-					       debugfs_node_dir,
-					       &run_order);
-	if (!debugfs_run_order)
-		goto err;
-
-	debugfs_use_dma = debugfs_create_bool("use_dma", 0600,
-					       debugfs_node_dir,
-					       &use_dma);
-	if (!debugfs_use_dma)
-		goto err;
-
-	debugfs_on_node = debugfs_create_bool("on_node", 0600,
-					      debugfs_node_dir,
-					      &on_node);
-	if (!debugfs_on_node)
-		goto err;
+	perf->dbgfs_dir = debugfs_create_dir(pci_name(pdev), perf_dbgfs_topdir);
+	if (!perf->dbgfs_dir) {
+		dev_warn(&perf->ntb->dev, "DebugFS unsupported\n");
+		return;
+	}
+
+	debugfs_create_file("info", 0600, perf->dbgfs_dir, perf,
+			    &perf_dbgfs_info);
 
-	return 0;
+	debugfs_create_file("run", 0600, perf->dbgfs_dir, perf,
+			    &perf_dbgfs_run);
 
-err:
-	debugfs_remove_recursive(perf_debugfs_dir);
-	perf_debugfs_dir = NULL;
-	return -ENODEV;
+	debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf,
+			    &perf_dbgfs_tcnt);
+
+	/* They are made read-only for test exec safety and integrity */
+	debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order);
+
+	debugfs_create_u8("total_order", 0500, perf->dbgfs_dir, &total_order);
+
+	debugfs_create_bool("use_dma", 0500, perf->dbgfs_dir, &use_dma);
 }
 
-static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
+static void perf_clear_dbgfs(struct perf_ctx *perf)
+{
+	debugfs_remove_recursive(perf->dbgfs_dir);
+}
+
+/*==============================================================================
+ *                        Basic driver initialization
+ *==============================================================================
+ */
+
+static struct perf_ctx *perf_create_data(struct ntb_dev *ntb)
 {
-	struct pci_dev *pdev = ntb->pdev;
 	struct perf_ctx *perf;
-	int node;
-	int rc = 0;
 
-	if (ntb_spad_count(ntb) < MAX_SPAD) {
-		dev_err(&ntb->dev, "Not enough scratch pad registers for %s",
-			DRIVER_NAME);
-		return -EIO;
-	}
+	perf = devm_kzalloc(&ntb->dev, sizeof(*perf), GFP_KERNEL);
+	if (!perf)
+		return ERR_PTR(-ENOMEM);
 
-	if (!ntb->ops->mw_set_trans) {
-		dev_err(&ntb->dev, "Need inbound MW based NTB API\n");
-		return -EINVAL;
+	perf->pcnt = ntb_peer_port_count(ntb);
+	perf->peers = devm_kcalloc(&ntb->dev, perf->pcnt, sizeof(*perf->peers),
+				  GFP_KERNEL);
+	if (!perf->peers)
+		return ERR_PTR(-ENOMEM);
+
+	perf->ntb = ntb;
+
+	return perf;
+}
+
+static int perf_setup_peer_mw(struct perf_peer *peer)
+{
+	struct perf_ctx *perf = peer->perf;
+	phys_addr_t phys_addr;
+	int ret;
+
+	/* Get outbound MW parameters and map it */
+	ret = ntb_peer_mw_get_addr(perf->ntb, peer->gidx, &phys_addr,
+				   &peer->outbuf_size);
+	if (ret)
+		return ret;
+
+	peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,
+					peer->outbuf_size);
+	if (!peer->outbuf)
+		return -ENOMEM;
+
+	if (max_mw_size && peer->outbuf_size > max_mw_size) {
+		peer->outbuf_size = max_mw_size;
+		dev_warn(&peer->perf->ntb->dev,
+			"Peer %d outbuf reduced to %pa\n", peer->pidx,
+			&peer->outbuf_size);
 	}
 
-	if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT)
-		dev_warn(&ntb->dev, "Multi-port NTB devices unsupported\n");
+	return 0;
+}
 
-	node = on_node ? dev_to_node(&pdev->dev) : NUMA_NO_NODE;
-	perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
-	if (!perf) {
-		rc = -ENOMEM;
-		goto err_perf;
+static int perf_init_peers(struct perf_ctx *perf)
+{
+	struct perf_peer *peer;
+	int pidx, lport, ret;
+
+	lport = ntb_port_number(perf->ntb);
+	perf->gidx = -1;
+	for (pidx = 0; pidx < perf->pcnt; pidx++) {
+		peer = &perf->peers[pidx];
+
+		peer->perf = perf;
+		peer->pidx = pidx;
+		if (lport < ntb_peer_port_number(perf->ntb, pidx)) {
+			if (perf->gidx == -1)
+				perf->gidx = pidx;
+			peer->gidx = pidx + 1;
+		} else {
+			peer->gidx = pidx;
+		}
+		INIT_WORK(&peer->service, perf_service_work);
 	}
+	if (perf->gidx == -1)
+		perf->gidx = pidx;
 
-	perf->ntb = ntb;
-	perf->perf_threads = 1;
-	atomic_set(&perf->tsync, 0);
-	mutex_init(&perf->run_mutex);
-	spin_lock_init(&perf->db_lock);
-	perf_setup_mw(ntb, perf);
-	init_waitqueue_head(&perf->link_wq);
-	INIT_DELAYED_WORK(&perf->link_work, perf_link_work);
+	for (pidx = 0; pidx < perf->pcnt; pidx++) {
+		ret = perf_setup_peer_mw(&perf->peers[pidx]);
+		if (ret)
+			return ret;
+	}
+
+	dev_dbg(&perf->ntb->dev, "Global port index %d\n", perf->gidx);
+
+	return 0;
+}
 
-	rc = ntb_set_ctx(ntb, perf, &perf_ops);
-	if (rc)
-		goto err_ctx;
+static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
+{
+	struct perf_ctx *perf;
+	int ret;
 
-	perf->link_is_up = false;
-	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
-	ntb_link_event(ntb);
+	perf = perf_create_data(ntb);
+	if (IS_ERR(perf))
+		return PTR_ERR(perf);
 
-	rc = perf_debugfs_setup(perf);
-	if (rc)
-		goto err_ctx;
+	ret = perf_init_peers(perf);
+	if (ret)
+		return ret;
 
-	perf_clear_thread_status(perf);
+	perf_init_threads(perf);
 
-	return 0;
+	ret = perf_init_service(perf);
+	if (ret)
+		return ret;
 
-err_ctx:
-	cancel_delayed_work_sync(&perf->link_work);
-	kfree(perf);
-err_perf:
-	return rc;
+	ret = perf_enable_service(perf);
+	if (ret)
+		return ret;
+
+	perf_setup_dbgfs(perf);
+
+	return 0;
 }
 
 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
 {
 	struct perf_ctx *perf = ntb->ctx;
-	int i;
 
-	dev_dbg(&perf->ntb->dev, "%s called\n", __func__);
+	perf_clear_dbgfs(perf);
 
-	mutex_lock(&perf->run_mutex);
+	perf_disable_service(perf);
 
-	cancel_delayed_work_sync(&perf->link_work);
+	perf_clear_threads(perf);
+}
 
-	ntb_clear_ctx(ntb);
-	ntb_link_disable(ntb);
+static struct ntb_client perf_client = {
+	.ops = {
+		.probe = perf_probe,
+		.remove = perf_remove
+	}
+};
 
-	debugfs_remove_recursive(perf_debugfs_dir);
-	perf_debugfs_dir = NULL;
+static int __init perf_init(void)
+{
+	int ret;
 
-	if (use_dma) {
-		for (i = 0; i < MAX_THREADS; i++) {
-			struct pthr_ctx *pctx = &perf->pthr_ctx[i];
+	if (chunk_order > MAX_CHUNK_ORDER) {
+		chunk_order = MAX_CHUNK_ORDER;
+		pr_info("Chunk order reduced to %hhu\n", chunk_order);
+	}
 
-			if (pctx->dma_chan)
-				dma_release_channel(pctx->dma_chan);
-		}
+	if (total_order < chunk_order) {
+		total_order = chunk_order;
+		pr_info("Total data order reduced to %hhu\n", total_order);
 	}
 
-	kfree(perf);
+	perf_wq = alloc_workqueue("perf_wq", WQ_UNBOUND | WQ_SYSFS, 0);
+	if (!perf_wq)
+		return -ENOMEM;
+
+	if (debugfs_initialized())
+		perf_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+
+	ret = ntb_register_client(&perf_client);
+	if (ret) {
+		debugfs_remove_recursive(perf_dbgfs_topdir);
+		destroy_workqueue(perf_wq);
+	}
+
+	return ret;
 }
+module_init(perf_init);
+
+static void __exit perf_exit(void)
+{
+	ntb_unregister_client(&perf_client);
+	debugfs_remove_recursive(perf_dbgfs_topdir);
+	destroy_workqueue(perf_wq);
+}
+module_exit(perf_exit);
 
-static struct ntb_client perf_client = {
-	.ops = {
-		.probe = perf_probe,
-		.remove = perf_remove,
-	},
-};
-module_ntb_client(perf_client);
diff --git a/drivers/ntb/test/ntb_pingpong.c b/drivers/ntb/test/ntb_pingpong.c
index 3f5a92bae6f8..65865e460ab8 100644
--- a/drivers/ntb/test/ntb_pingpong.c
+++ b/drivers/ntb/test/ntb_pingpong.c
@@ -1,10 +1,11 @@
 /*
- * This file is provided under a dual BSD/GPLv2 license.  When using or
+ *   This file is provided under a dual BSD/GPLv2 license.  When using or
  *   redistributing this file, you may do so under either license.
  *
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2017 T-Platforms. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -18,6 +19,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2017 T-Platforms. All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -46,37 +48,45 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * PCIe NTB Pingpong Linux driver
- *
- * Contact Information:
- * Allen Hubbe <Allen.Hubbe@emc.com>
  */
 
-/* Note: load this module with option 'dyndbg=+p' */
+/*
+ * How to use this tool, by example.
+ *
+ * Assuming $DBG_DIR is something like:
+ * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
+ * Suppose aside from local device there is at least one remote device
+ * connected to NTB with index 0.
+ *-----------------------------------------------------------------------------
+ * Eg: install driver with specified delay between doorbell event and response
+ *
+ * root@self# insmod ntb_pingpong.ko delay_ms=1000
+ *-----------------------------------------------------------------------------
+ * Eg: get number of ping-pong cycles performed
+ *
+ * root@self# cat $DBG_DIR/count
+ */
 
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/device.h>
+#include <linux/bitops.h>
 
-#include <linux/dma-mapping.h>
 #include <linux/pci.h>
 #include <linux/slab.h>
-#include <linux/spinlock.h>
+#include <linux/hrtimer.h>
 #include <linux/debugfs.h>
 
 #include <linux/ntb.h>
 
-#define DRIVER_NAME			"ntb_pingpong"
-#define DRIVER_DESCRIPTION		"PCIe NTB Simple Pingpong Client"
-
-#define DRIVER_LICENSE			"Dual BSD/GPL"
-#define DRIVER_VERSION			"1.0"
-#define DRIVER_RELDATE			"24 March 2015"
-#define DRIVER_AUTHOR			"Allen Hubbe <Allen.Hubbe@emc.com>"
+#define DRIVER_NAME		"ntb_pingpong"
+#define DRIVER_VERSION		"2.0"
 
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(DRIVER_VERSION);
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
+MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
+MODULE_DESCRIPTION("PCIe NTB Simple Pingpong Client");
 
 static unsigned int unsafe;
 module_param(unsafe, uint, 0644);
@@ -86,237 +96,343 @@ static unsigned int delay_ms = 1000;
 module_param(delay_ms, uint, 0644);
 MODULE_PARM_DESC(delay_ms, "Milliseconds to delay the response to peer");
 
-static unsigned long db_init = 0x7;
-module_param(db_init, ulong, 0644);
-MODULE_PARM_DESC(db_init, "Initial doorbell bits to ring on the peer");
-
-/* Only two-ports NTB devices are supported */
-#define PIDX		NTB_DEF_PEER_IDX
-
 struct pp_ctx {
-	struct ntb_dev			*ntb;
-	u64				db_bits;
-	/* synchronize access to db_bits by ping and pong */
-	spinlock_t			db_lock;
-	struct timer_list		db_timer;
-	unsigned long			db_delay;
-	struct dentry			*debugfs_node_dir;
-	struct dentry			*debugfs_count;
-	atomic_t			count;
+	struct ntb_dev *ntb;
+	struct hrtimer timer;
+	u64 in_db;
+	u64 out_db;
+	int out_pidx;
+	u64 nmask;
+	u64 pmask;
+	atomic_t count;
+	spinlock_t lock;
+	struct dentry *dbgfs_dir;
 };
+#define to_pp_timer(__timer) \
+	container_of(__timer, struct pp_ctx, timer)
 
-static struct dentry *pp_debugfs_dir;
+static struct dentry *pp_dbgfs_topdir;
 
-static void pp_ping(struct timer_list *t)
+static int pp_find_next_peer(struct pp_ctx *pp)
 {
-	struct pp_ctx *pp = from_timer(pp, t, db_timer);
-	unsigned long irqflags;
-	u64 db_bits, db_mask;
-	u32 spad_rd, spad_wr;
+	u64 link, out_db;
+	int pidx;
+
+	link = ntb_link_is_up(pp->ntb, NULL, NULL);
+
+	/* Find next available peer */
+	if (link & pp->nmask) {
+		pidx = __ffs64(link & pp->nmask);
+		out_db = BIT_ULL(pidx + 1);
+	} else if (link & pp->pmask) {
+		pidx = __ffs64(link & pp->pmask);
+		out_db = BIT_ULL(pidx);
+	} else {
+		return -ENODEV;
+	}
 
-	spin_lock_irqsave(&pp->db_lock, irqflags);
-	{
-		db_mask = ntb_db_valid_mask(pp->ntb);
-		db_bits = ntb_db_read(pp->ntb);
+	spin_lock(&pp->lock);
+	pp->out_pidx = pidx;
+	pp->out_db = out_db;
+	spin_unlock(&pp->lock);
 
-		if (db_bits) {
-			dev_dbg(&pp->ntb->dev,
-				"Masked pongs %#llx\n",
-				db_bits);
-			ntb_db_clear(pp->ntb, db_bits);
-		}
+	return 0;
+}
 
-		db_bits = ((pp->db_bits | db_bits) << 1) & db_mask;
+static void pp_setup(struct pp_ctx *pp)
+{
+	int ret;
 
-		if (!db_bits)
-			db_bits = db_init;
+	ntb_db_set_mask(pp->ntb, pp->in_db);
 
-		spad_rd = ntb_spad_read(pp->ntb, 0);
-		spad_wr = spad_rd + 1;
+	hrtimer_cancel(&pp->timer);
 
-		dev_dbg(&pp->ntb->dev,
-			"Ping bits %#llx read %#x write %#x\n",
-			db_bits, spad_rd, spad_wr);
+	ret = pp_find_next_peer(pp);
+	if (ret == -ENODEV) {
+		dev_dbg(&pp->ntb->dev, "Got no peers, so cancel\n");
+		return;
+	}
 
-		ntb_peer_spad_write(pp->ntb, PIDX, 0, spad_wr);
-		ntb_peer_db_set(pp->ntb, db_bits);
-		ntb_db_clear_mask(pp->ntb, db_mask);
+	dev_dbg(&pp->ntb->dev, "Ping-pong started with port %d, db %#llx\n",
+		ntb_peer_port_number(pp->ntb, pp->out_pidx), pp->out_db);
 
-		pp->db_bits = 0;
-	}
-	spin_unlock_irqrestore(&pp->db_lock, irqflags);
+	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);
 }
 
-static void pp_link_event(void *ctx)
+static void pp_clear(struct pp_ctx *pp)
 {
-	struct pp_ctx *pp = ctx;
+	hrtimer_cancel(&pp->timer);
 
-	if (ntb_link_is_up(pp->ntb, NULL, NULL) == 1) {
-		dev_dbg(&pp->ntb->dev, "link is up\n");
-		pp_ping(&pp->db_timer);
-	} else {
-		dev_dbg(&pp->ntb->dev, "link is down\n");
-		del_timer(&pp->db_timer);
-	}
+	ntb_db_set_mask(pp->ntb, pp->in_db);
+
+	dev_dbg(&pp->ntb->dev, "Ping-pong cancelled\n");
 }
 
-static void pp_db_event(void *ctx, int vec)
+static void pp_ping(struct pp_ctx *pp)
 {
-	struct pp_ctx *pp = ctx;
-	u64 db_bits, db_mask;
-	unsigned long irqflags;
+	u32 count;
 
-	spin_lock_irqsave(&pp->db_lock, irqflags);
-	{
-		db_mask = ntb_db_vector_mask(pp->ntb, vec);
-		db_bits = db_mask & ntb_db_read(pp->ntb);
-		ntb_db_set_mask(pp->ntb, db_mask);
-		ntb_db_clear(pp->ntb, db_bits);
+	count = atomic_read(&pp->count);
 
-		pp->db_bits |= db_bits;
+	spin_lock(&pp->lock);
+	ntb_peer_spad_write(pp->ntb, pp->out_pidx, 0, count);
+	ntb_peer_msg_write(pp->ntb, pp->out_pidx, 0, count);
 
-		mod_timer(&pp->db_timer, jiffies + pp->db_delay);
+	dev_dbg(&pp->ntb->dev, "Ping port %d spad %#x, msg %#x\n",
+		ntb_peer_port_number(pp->ntb, pp->out_pidx), count, count);
 
-		dev_dbg(&pp->ntb->dev,
-			"Pong vec %d bits %#llx\n",
-			vec, db_bits);
-		atomic_inc(&pp->count);
-	}
-	spin_unlock_irqrestore(&pp->db_lock, irqflags);
+	ntb_peer_db_set(pp->ntb, pp->out_db);
+	ntb_db_clear_mask(pp->ntb, pp->in_db);
+	spin_unlock(&pp->lock);
 }
 
-static int pp_debugfs_setup(struct pp_ctx *pp)
+static void pp_pong(struct pp_ctx *pp)
 {
-	struct pci_dev *pdev = pp->ntb->pdev;
+	u32 msg_data = -1, spad_data = -1;
+	int pidx = 0;
 
-	if (!pp_debugfs_dir)
-		return -ENODEV;
+	/* Read pong data */
+	spad_data = ntb_spad_read(pp->ntb, 0);
+	msg_data = ntb_msg_read(pp->ntb, &pidx, 0);
+	ntb_msg_clear_sts(pp->ntb, -1);
 
-	pp->debugfs_node_dir = debugfs_create_dir(pci_name(pdev),
-						  pp_debugfs_dir);
-	if (!pp->debugfs_node_dir)
-		return -ENODEV;
+	/*
+	 * Scratchpad and message data may differ, since message register can't
+	 * be rewritten unless status is cleared. Additionally either of them
+	 * might be unsupported
+	 */
+	dev_dbg(&pp->ntb->dev, "Pong spad %#x, msg %#x (port %d)\n",
+		spad_data, msg_data, ntb_peer_port_number(pp->ntb, pidx));
 
-	pp->debugfs_count = debugfs_create_atomic_t("count", S_IRUSR | S_IWUSR,
-						    pp->debugfs_node_dir,
-						    &pp->count);
-	if (!pp->debugfs_count)
-		return -ENODEV;
+	atomic_inc(&pp->count);
 
-	return 0;
+	ntb_db_set_mask(pp->ntb, pp->in_db);
+	ntb_db_clear(pp->ntb, pp->in_db);
+
+	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);
+}
+
+static enum hrtimer_restart pp_timer_func(struct hrtimer *t)
+{
+	struct pp_ctx *pp = to_pp_timer(t);
+
+	pp_ping(pp);
+
+	return HRTIMER_NORESTART;
+}
+
+static void pp_link_event(void *ctx)
+{
+	struct pp_ctx *pp = ctx;
+
+	pp_setup(pp);
+}
+
+static void pp_db_event(void *ctx, int vec)
+{
+	struct pp_ctx *pp = ctx;
+
+	pp_pong(pp);
 }
 
 static const struct ntb_ctx_ops pp_ops = {
 	.link_event = pp_link_event,
-	.db_event = pp_db_event,
+	.db_event = pp_db_event
 };
 
-static int pp_probe(struct ntb_client *client,
-		    struct ntb_dev *ntb)
+static int pp_check_ntb(struct ntb_dev *ntb)
 {
-	struct pp_ctx *pp;
-	int rc;
+	u64 pmask;
 
 	if (ntb_db_is_unsafe(ntb)) {
-		dev_dbg(&ntb->dev, "doorbell is unsafe\n");
-		if (!unsafe) {
-			rc = -EINVAL;
-			goto err_pp;
-		}
-	}
-
-	if (ntb_spad_count(ntb) < 1) {
-		dev_dbg(&ntb->dev, "no enough scratchpads\n");
-		rc = -EINVAL;
-		goto err_pp;
+		dev_dbg(&ntb->dev, "Doorbell is unsafe\n");
+		if (!unsafe)
+			return -EINVAL;
 	}
 
 	if (ntb_spad_is_unsafe(ntb)) {
-		dev_dbg(&ntb->dev, "scratchpad is unsafe\n");
-		if (!unsafe) {
-			rc = -EINVAL;
-			goto err_pp;
-		}
+		dev_dbg(&ntb->dev, "Scratchpad is unsafe\n");
+		if (!unsafe)
+			return -EINVAL;
 	}
 
-	if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT)
-		dev_warn(&ntb->dev, "multi-port NTB is unsupported\n");
+	pmask = GENMASK_ULL(ntb_peer_port_count(ntb), 0);
+	if ((ntb_db_valid_mask(ntb) & pmask) != pmask) {
+		dev_err(&ntb->dev, "Unsupported DB configuration\n");
+		return -EINVAL;
+	}
 
-	pp = kmalloc(sizeof(*pp), GFP_KERNEL);
-	if (!pp) {
-		rc = -ENOMEM;
-		goto err_pp;
+	if (ntb_spad_count(ntb) < 1 && ntb_msg_count(ntb) < 1) {
+		dev_err(&ntb->dev, "Scratchpads and messages unsupported\n");
+		return -EINVAL;
+	} else if (ntb_spad_count(ntb) < 1) {
+		dev_dbg(&ntb->dev, "Scratchpads unsupported\n");
+	} else if (ntb_msg_count(ntb) < 1) {
+		dev_dbg(&ntb->dev, "Messages unsupported\n");
 	}
 
+	return 0;
+}
+
+static struct pp_ctx *pp_create_data(struct ntb_dev *ntb)
+{
+	struct pp_ctx *pp;
+
+	pp = devm_kzalloc(&ntb->dev, sizeof(*pp), GFP_KERNEL);
+	if (!pp)
+		return ERR_PTR(-ENOMEM);
+
 	pp->ntb = ntb;
-	pp->db_bits = 0;
 	atomic_set(&pp->count, 0);
-	spin_lock_init(&pp->db_lock);
-	timer_setup(&pp->db_timer, pp_ping, 0);
-	pp->db_delay = msecs_to_jiffies(delay_ms);
+	spin_lock_init(&pp->lock);
+	hrtimer_init(&pp->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	pp->timer.function = pp_timer_func;
+
+	return pp;
+}
+
+static void pp_init_flds(struct pp_ctx *pp)
+{
+	int pidx, lport, pcnt;
+
+	/* Find global port index */
+	lport = ntb_port_number(pp->ntb);
+	pcnt = ntb_peer_port_count(pp->ntb);
+	for (pidx = 0; pidx < pcnt; pidx++) {
+		if (lport < ntb_peer_port_number(pp->ntb, pidx))
+			break;
+	}
 
-	rc = ntb_set_ctx(ntb, pp, &pp_ops);
-	if (rc)
-		goto err_ctx;
+	pp->in_db = BIT_ULL(pidx);
+	pp->pmask = GENMASK_ULL(pidx, 0) >> 1;
+	pp->nmask = GENMASK_ULL(pcnt - 1, pidx);
 
-	rc = pp_debugfs_setup(pp);
-	if (rc)
-		goto err_ctx;
+	dev_dbg(&pp->ntb->dev, "Inbound db %#llx, prev %#llx, next %#llx\n",
+		pp->in_db, pp->pmask, pp->nmask);
+}
+
+static int pp_mask_events(struct pp_ctx *pp)
+{
+	u64 db_mask, msg_mask;
+	int ret;
+
+	db_mask = ntb_db_valid_mask(pp->ntb);
+	ret = ntb_db_set_mask(pp->ntb, db_mask);
+	if (ret)
+		return ret;
 
-	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
-	ntb_link_event(ntb);
+	/* Skip message events masking if unsupported */
+	if (ntb_msg_count(pp->ntb) < 1)
+		return 0;
+
+	msg_mask = ntb_msg_outbits(pp->ntb) | ntb_msg_inbits(pp->ntb);
+	return ntb_msg_set_mask(pp->ntb, msg_mask);
+}
+
+static int pp_setup_ctx(struct pp_ctx *pp)
+{
+	int ret;
+
+	ret = ntb_set_ctx(pp->ntb, pp, &pp_ops);
+	if (ret)
+		return ret;
+
+	ntb_link_enable(pp->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
+	/* Might be not necessary */
+	ntb_link_event(pp->ntb);
 
 	return 0;
+}
+
+static void pp_clear_ctx(struct pp_ctx *pp)
+{
+	ntb_link_disable(pp->ntb);
 
-err_ctx:
-	kfree(pp);
-err_pp:
-	return rc;
+	ntb_clear_ctx(pp->ntb);
 }
 
-static void pp_remove(struct ntb_client *client,
-		      struct ntb_dev *ntb)
+static void pp_setup_dbgfs(struct pp_ctx *pp)
+{
+	struct pci_dev *pdev = pp->ntb->pdev;
+	void *ret;
+
+	pp->dbgfs_dir = debugfs_create_dir(pci_name(pdev), pp_dbgfs_topdir);
+
+	ret = debugfs_create_atomic_t("count", 0600, pp->dbgfs_dir, &pp->count);
+	if (!ret)
+		dev_warn(&pp->ntb->dev, "DebugFS unsupported\n");
+}
+
+static void pp_clear_dbgfs(struct pp_ctx *pp)
+{
+	debugfs_remove_recursive(pp->dbgfs_dir);
+}
+
+static int pp_probe(struct ntb_client *client, struct ntb_dev *ntb)
+{
+	struct pp_ctx *pp;
+	int ret;
+
+	ret = pp_check_ntb(ntb);
+	if (ret)
+		return ret;
+
+	pp = pp_create_data(ntb);
+	if (IS_ERR(pp))
+		return PTR_ERR(pp);
+
+	pp_init_flds(pp);
+
+	ret = pp_mask_events(pp);
+	if (ret)
+		return ret;
+
+	ret = pp_setup_ctx(pp);
+	if (ret)
+		return ret;
+
+	pp_setup_dbgfs(pp);
+
+	return 0;
+}
+
+static void pp_remove(struct ntb_client *client, struct ntb_dev *ntb)
 {
 	struct pp_ctx *pp = ntb->ctx;
 
-	debugfs_remove_recursive(pp->debugfs_node_dir);
+	pp_clear_dbgfs(pp);
 
-	ntb_clear_ctx(ntb);
-	del_timer_sync(&pp->db_timer);
-	ntb_link_disable(ntb);
+	pp_clear_ctx(pp);
 
-	kfree(pp);
+	pp_clear(pp);
 }
 
 static struct ntb_client pp_client = {
 	.ops = {
 		.probe = pp_probe,
-		.remove = pp_remove,
-	},
+		.remove = pp_remove
+	}
 };
 
 static int __init pp_init(void)
 {
-	int rc;
+	int ret;
 
 	if (debugfs_initialized())
-		pp_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+		pp_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
 
-	rc = ntb_register_client(&pp_client);
-	if (rc)
-		goto err_client;
+	ret = ntb_register_client(&pp_client);
+	if (ret)
+		debugfs_remove_recursive(pp_dbgfs_topdir);
 
-	return 0;
-
-err_client:
-	debugfs_remove_recursive(pp_debugfs_dir);
-	return rc;
+	return ret;
 }
 module_init(pp_init);
 
 static void __exit pp_exit(void)
 {
 	ntb_unregister_client(&pp_client);
-	debugfs_remove_recursive(pp_debugfs_dir);
+	debugfs_remove_recursive(pp_dbgfs_topdir);
 }
 module_exit(pp_exit);
+
diff --git a/drivers/ntb/test/ntb_tool.c b/drivers/ntb/test/ntb_tool.c
index 91526a986caa..d592c0ffbd19 100644
--- a/drivers/ntb/test/ntb_tool.c
+++ b/drivers/ntb/test/ntb_tool.c
@@ -5,6 +5,7 @@
  *   GPL LICENSE SUMMARY
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2017 T-Platforms All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or modify
  *   it under the terms of version 2 of the GNU General Public License as
@@ -18,6 +19,7 @@
  *   BSD LICENSE
  *
  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
+ *   Copyright (C) 2017 T-Platforms All Rights Reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -46,9 +48,6 @@
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * PCIe NTB Debugging Tool Linux driver
- *
- * Contact Information:
- * Allen Hubbe <Allen.Hubbe@emc.com>
  */
 
 /*
@@ -56,42 +55,125 @@
  *
  * Assuming $DBG_DIR is something like:
  * '/sys/kernel/debug/ntb_tool/0000:00:03.0'
+ * Suppose aside from local device there is at least one remote device
+ * connected to NTB with index 0.
+ *-----------------------------------------------------------------------------
+ * Eg: check local/peer device information.
+ *
+ * # Get local device port number
+ * root@self# cat $DBG_DIR/port
+ *
+ * # Check local device functionality
+ * root@self# ls $DBG_DIR
+ * db            msg1          msg_sts     peer4/        port
+ * db_event      msg2          peer0/      peer5/        spad0
+ * db_mask       msg3          peer1/      peer_db       spad1
+ * link          msg_event     peer2/      peer_db_mask  spad2
+ * msg0          msg_mask      peer3/      peer_spad     spad3
+ * # As one can see it supports:
+ * # 1) four inbound message registers
+ * # 2) four inbound scratchpads
+ * # 3) up to six peer devices
+ *
+ * # Check peer device port number
+ * root@self# cat $DBG_DIR/peer0/port
  *
- * Eg: check if clearing the doorbell mask generates an interrupt.
+ * # Check peer device(s) functionality to be used
+ * root@self# ls $DBG_DIR/peer0
+ * link             mw_trans0       mw_trans6        port
+ * link_event       mw_trans1       mw_trans7        spad0
+ * msg0             mw_trans2       peer_mw_trans0   spad1
+ * msg1             mw_trans3       peer_mw_trans1   spad2
+ * msg2             mw_trans4       peer_mw_trans2   spad3
+ * msg3             mw_trans5       peer_mw_trans3
+ * # As one can see we got:
+ * # 1) four outbound message registers
+ * # 2) four outbound scratchpads
+ * # 3) eight inbound memory windows
+ * # 4) four outbound memory windows
+ *-----------------------------------------------------------------------------
+ * Eg: NTB link tests
  *
- * # Check the link status
- * root@self# cat $DBG_DIR/link
+ * # Set local link up/down
+ * root@self# echo Y > $DBG_DIR/link
+ * root@self# echo N > $DBG_DIR/link
  *
- * # Block until the link is up
- * root@self# echo Y > $DBG_DIR/link_event
+ * # Check if link with peer device is up/down:
+ * root@self# cat $DBG_DIR/peer0/link
  *
- * # Set the doorbell mask
- * root@self# echo 's 1' > $DBG_DIR/mask
+ * # Block until the link is up/down
+ * root@self# echo Y > $DBG_DIR/peer0/link_event
+ * root@self# echo N > $DBG_DIR/peer0/link_event
+ *-----------------------------------------------------------------------------
+ * Eg: Doorbell registers tests (some functionality might be absent)
  *
- * # Ring the doorbell from the peer
+ * # Set/clear/get local doorbell
+ * root@self# echo 's 1' > $DBG_DIR/db
+ * root@self# echo 'c 1' > $DBG_DIR/db
+ * root@self# cat  $DBG_DIR/db
+ *
+ * # Set/clear/get local doorbell mask
+ * root@self# echo 's 1' > $DBG_DIR/db_mask
+ * root@self# echo 'c 1' > $DBG_DIR/db_mask
+ * root@self# cat $DBG_DIR/db_mask
+ *
+ * # Ring/clear/get peer doorbell
  * root@peer# echo 's 1' > $DBG_DIR/peer_db
+ * root@peer# echo 'c 1' > $DBG_DIR/peer_db
+ * root@peer# cat $DBG_DIR/peer_db
+ *
+ * # Set/clear/get peer doorbell mask
+ * root@self# echo 's 1' > $DBG_DIR/peer_db_mask
+ * root@self# echo 'c 1' > $DBG_DIR/peer_db_mask
+ * root@self# cat $DBG_DIR/peer_db_mask
+ *
+ * # Block until local doorbell is set with specified value
+ * root@self# echo 1 > $DBG_DIR/db_event
+ *-----------------------------------------------------------------------------
+ * Eg: Message registers tests (functionality might be absent)
  *
- * # Clear the doorbell mask
- * root@self# echo 'c 1' > $DBG_DIR/mask
+ * # Set/clear/get in/out message registers status
+ * root@self# echo 's 1' > $DBG_DIR/msg_sts
+ * root@self# echo 'c 1' > $DBG_DIR/msg_sts
+ * root@self# cat $DBG_DIR/msg_sts
  *
- * Observe debugging output in dmesg or your console.  You should see a
- * doorbell event triggered by clearing the mask.  If not, this may indicate an
- * issue with the hardware that needs to be worked around in the driver.
+ * # Set/clear in/out message registers mask
+ * root@self# echo 's 1' > $DBG_DIR/msg_mask
+ * root@self# echo 'c 1' > $DBG_DIR/msg_mask
  *
- * Eg: read and write scratchpad registers
+ * # Get inbound message register #0 value and source of port index
+ * root@self# cat  $DBG_DIR/msg0
  *
- * root@peer# echo '0 0x01010101 1 0x7f7f7f7f' > $DBG_DIR/peer_spad
+ * # Send some data to peer over outbound message register #0
+ * root@self# echo 0x01020304 > $DBG_DIR/peer0/msg0
+ *-----------------------------------------------------------------------------
+ * Eg: Scratchpad registers tests (functionality might be absent)
  *
- * root@self# cat $DBG_DIR/spad
+ * # Write/read to/from local scratchpad register #0
+ * root@peer# echo 0x01020304 > $DBG_DIR/spad0
+ * root@peer# cat $DBG_DIR/spad0
  *
- * Observe that spad 0 and 1 have the values set by the peer.
+ * # Write/read to/from peer scratchpad register #0
+ * root@peer# echo 0x01020304 > $DBG_DIR/peer0/spad0
+ * root@peer# cat $DBG_DIR/peer0/spad0
+ *-----------------------------------------------------------------------------
+ * Eg: Memory windows tests
  *
- * # Check the memory window translation info
- * cat $DBG_DIR/peer_trans0
+ * # Create inbound memory window buffer of specified size/get its base address
+ * root@peer# echo 16384 > $DBG_DIR/peer0/mw_trans0
+ * root@peer# cat $DBG_DIR/peer0/mw_trans0
  *
- * # Setup a 16k memory window buffer
- * echo 16384 > $DBG_DIR/peer_trans0
+ * # Write/read data to/from inbound memory window
+ * root@peer# echo Hello > $DBG_DIR/peer0/mw0
+ * root@peer# head -c 7 $DBG_DIR/peer0/mw0
  *
+ * # Map outbound memory window/check it settings (on peer device)
+ * root@peer# echo 0xADD0BA5E:16384 > $DBG_DIR/peer0/peer_mw_trans0
+ * root@peer# cat $DBG_DIR/peer0/peer_mw_trans0
+ *
+ * # Write/read data to/from outbound memory window (on peer device)
+ * root@peer# echo olleH > $DBG_DIR/peer0/peer_mw0
+ * root@peer# head -c 7 $DBG_DIR/peer0/peer_mw0
  */
 
 #include <linux/init.h>
@@ -106,49 +188,87 @@
 
 #include <linux/ntb.h>
 
-#define DRIVER_NAME			"ntb_tool"
-#define DRIVER_DESCRIPTION		"PCIe NTB Debugging Tool"
-
-#define DRIVER_LICENSE			"Dual BSD/GPL"
-#define DRIVER_VERSION			"1.0"
-#define DRIVER_RELDATE			"22 April 2015"
-#define DRIVER_AUTHOR			"Allen Hubbe <Allen.Hubbe@emc.com>"
+#define DRIVER_NAME		"ntb_tool"
+#define DRIVER_VERSION		"2.0"
 
-MODULE_LICENSE(DRIVER_LICENSE);
+MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(DRIVER_VERSION);
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
-
-/* It is rare to have hadrware with greater than six MWs */
-#define MAX_MWS	6
-/* Only two-ports devices are supported */
-#define PIDX	NTB_DEF_PEER_IDX
-
-static struct dentry *tool_dbgfs;
+MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
+MODULE_DESCRIPTION("PCIe NTB Debugging Tool");
 
+/*
+ * Inbound and outbound memory windows descriptor. Union members selection
+ * depends on the MW type the structure describes. mm_base/dma_base are the
+ * virtual and DMA address of an inbound MW. io_base/tr_base are the MMIO
+ * mapped virtual and xlat addresses of an outbound MW respectively.
+ */
 struct tool_mw {
-	int idx;
+	int widx;
+	int pidx;
 	struct tool_ctx *tc;
-	resource_size_t win_size;
+	union {
+		u8 *mm_base;
+		u8 __iomem *io_base;
+	};
+	union {
+		dma_addr_t dma_base;
+		u64 tr_base;
+	};
 	resource_size_t size;
-	u8 __iomem *local;
-	u8 *peer;
-	dma_addr_t peer_dma;
-	struct dentry *peer_dbg_file;
+	struct dentry *dbgfs_file;
+};
+
+/*
+ * Wrapper structure is used to distinguish the outbound MW peers reference
+ * within the corresponding DebugFS directory IO operation.
+ */
+struct tool_mw_wrap {
+	int pidx;
+	struct tool_mw *mw;
+};
+
+struct tool_msg {
+	int midx;
+	int pidx;
+	struct tool_ctx *tc;
+};
+
+struct tool_spad {
+	int sidx;
+	int pidx;
+	struct tool_ctx *tc;
+};
+
+struct tool_peer {
+	int pidx;
+	struct tool_ctx *tc;
+	int inmw_cnt;
+	struct tool_mw *inmws;
+	int outmw_cnt;
+	struct tool_mw_wrap *outmws;
+	int outmsg_cnt;
+	struct tool_msg *outmsgs;
+	int outspad_cnt;
+	struct tool_spad *outspads;
+	struct dentry *dbgfs_dir;
 };
 
 struct tool_ctx {
 	struct ntb_dev *ntb;
-	struct dentry *dbgfs;
 	wait_queue_head_t link_wq;
-	int mw_count;
-	struct tool_mw mws[MAX_MWS];
+	wait_queue_head_t db_wq;
+	wait_queue_head_t msg_wq;
+	int outmw_cnt;
+	struct tool_mw *outmws;
+	int peer_cnt;
+	struct tool_peer *peers;
+	int inmsg_cnt;
+	struct tool_msg *inmsgs;
+	int inspad_cnt;
+	struct tool_spad *inspads;
+	struct dentry *dbgfs_dir;
 };
 
-#define SPAD_FNAME_SIZE 0x10
-#define INT_PTR(x) ((void *)(unsigned long)x)
-#define PTR_INT(x) ((int)(unsigned long)x)
-
 #define TOOL_FOPS_RDWR(__name, __read, __write) \
 	const struct file_operations __name = {	\
 		.owner = THIS_MODULE,		\
@@ -157,6 +277,15 @@ struct tool_ctx {
 		.write = __write,		\
 	}
 
+#define TOOL_BUF_LEN 32
+
+static struct dentry *tool_dbgfs_topdir;
+
+/*==============================================================================
+ *                               NTB events handlers
+ *==============================================================================
+ */
+
 static void tool_link_event(void *ctx)
 {
 	struct tool_ctx *tc = ctx;
@@ -182,580 +311,578 @@ static void tool_db_event(void *ctx, int vec)
 
 	dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
 		vec, db_mask, db_bits);
+
+	wake_up(&tc->db_wq);
+}
+
+static void tool_msg_event(void *ctx)
+{
+	struct tool_ctx *tc = ctx;
+	u64 msg_sts;
+
+	msg_sts = ntb_msg_read_sts(tc->ntb);
+
+	dev_dbg(&tc->ntb->dev, "message bits %#llx\n", msg_sts);
+
+	wake_up(&tc->msg_wq);
 }
 
 static const struct ntb_ctx_ops tool_ops = {
 	.link_event = tool_link_event,
 	.db_event = tool_db_event,
+	.msg_event = tool_msg_event
 };
 
-static ssize_t tool_dbfn_read(struct tool_ctx *tc, char __user *ubuf,
-			      size_t size, loff_t *offp,
-			      u64 (*db_read_fn)(struct ntb_dev *))
+/*==============================================================================
+ *                        Common read/write methods
+ *==============================================================================
+ */
+
+static ssize_t tool_fn_read(struct tool_ctx *tc, char __user *ubuf,
+			    size_t size, loff_t *offp,
+			    u64 (*fn_read)(struct ntb_dev *))
 {
 	size_t buf_size;
-	char *buf;
-	ssize_t pos, rc;
+	char buf[TOOL_BUF_LEN];
+	ssize_t pos;
 
-	if (!db_read_fn)
+	if (!fn_read)
 		return -EINVAL;
 
-	buf_size = min_t(size_t, size, 0x20);
-
-	buf = kmalloc(buf_size, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	pos = scnprintf(buf, buf_size, "%#llx\n",
-			db_read_fn(tc->ntb));
+	buf_size = min(size, sizeof(buf));
 
-	rc = simple_read_from_buffer(ubuf, size, offp, buf, pos);
+	pos = scnprintf(buf, buf_size, "%#llx\n", fn_read(tc->ntb));
 
-	kfree(buf);
-
-	return rc;
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 }
 
-static ssize_t tool_dbfn_write(struct tool_ctx *tc,
-			       const char __user *ubuf,
-			       size_t size, loff_t *offp,
-			       int (*db_set_fn)(struct ntb_dev *, u64),
-			       int (*db_clear_fn)(struct ntb_dev *, u64))
+static ssize_t tool_fn_write(struct tool_ctx *tc,
+			     const char __user *ubuf,
+			     size_t size, loff_t *offp,
+			     int (*fn_set)(struct ntb_dev *, u64),
+			     int (*fn_clear)(struct ntb_dev *, u64))
 {
-	u64 db_bits;
 	char *buf, cmd;
-	ssize_t rc;
+	ssize_t ret;
+	u64 bits;
 	int n;
 
 	buf = kmalloc(size + 1, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
-	rc = simple_write_to_buffer(buf, size, offp, ubuf, size);
-	if (rc < 0) {
+	ret = simple_write_to_buffer(buf, size, offp, ubuf, size);
+	if (ret < 0) {
 		kfree(buf);
-		return rc;
+		return ret;
 	}
 
 	buf[size] = 0;
 
-	n = sscanf(buf, "%c %lli", &cmd, &db_bits);
+	n = sscanf(buf, "%c %lli", &cmd, &bits);
 
 	kfree(buf);
 
 	if (n != 2) {
-		rc = -EINVAL;
+		ret = -EINVAL;
 	} else if (cmd == 's') {
-		if (!db_set_fn)
-			rc = -EINVAL;
+		if (!fn_set)
+			ret = -EINVAL;
 		else
-			rc = db_set_fn(tc->ntb, db_bits);
+			ret = fn_set(tc->ntb, bits);
 	} else if (cmd == 'c') {
-		if (!db_clear_fn)
-			rc = -EINVAL;
+		if (!fn_clear)
+			ret = -EINVAL;
 		else
-			rc = db_clear_fn(tc->ntb, db_bits);
+			ret = fn_clear(tc->ntb, bits);
 	} else {
-		rc = -EINVAL;
+		ret = -EINVAL;
 	}
 
-	return rc ? : size;
+	return ret ? : size;
 }
 
-static ssize_t tool_spadfn_read(struct tool_ctx *tc, char __user *ubuf,
-				size_t size, loff_t *offp,
-				u32 (*spad_read_fn)(struct ntb_dev *, int))
-{
-	size_t buf_size;
-	char *buf;
-	ssize_t pos, rc;
-	int i, spad_count;
-
-	if (!spad_read_fn)
-		return -EINVAL;
-
-	spad_count = ntb_spad_count(tc->ntb);
+/*==============================================================================
+ *                            Port read/write methods
+ *==============================================================================
+ */
 
-	/*
-	 * We multiply the number of spads by 15 to get the buffer size
-	 * this is from 3 for the %d, 10 for the largest hex value
-	 * (0x00000000) and 2 for the tab and line feed.
-	 */
-	buf_size = min_t(size_t, size, spad_count * 15);
+static ssize_t tool_port_read(struct file *filep, char __user *ubuf,
+			      size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+	char buf[TOOL_BUF_LEN];
+	int pos;
 
-	buf = kmalloc(buf_size, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
+	pos = scnprintf(buf, sizeof(buf), "%d\n", ntb_port_number(tc->ntb));
 
-	pos = 0;
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
+}
 
-	for (i = 0; i < spad_count; ++i) {
-		pos += scnprintf(buf + pos, buf_size - pos, "%d\t%#x\n",
-				 i, spad_read_fn(tc->ntb, i));
-	}
+static TOOL_FOPS_RDWR(tool_port_fops,
+		      tool_port_read,
+		      NULL);
 
-	rc = simple_read_from_buffer(ubuf, size, offp, buf, pos);
+static ssize_t tool_peer_port_read(struct file *filep, char __user *ubuf,
+				   size_t size, loff_t *offp)
+{
+	struct tool_peer *peer = filep->private_data;
+	struct tool_ctx *tc = peer->tc;
+	char buf[TOOL_BUF_LEN];
+	int pos;
 
-	kfree(buf);
+	pos = scnprintf(buf, sizeof(buf), "%d\n",
+		ntb_peer_port_number(tc->ntb, peer->pidx));
 
-	return rc;
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 }
 
-static ssize_t tool_spadfn_write(struct tool_ctx *tc,
-				 const char __user *ubuf,
-				 size_t size, loff_t *offp,
-				 int (*spad_write_fn)(struct ntb_dev *,
-						      int, u32))
+static TOOL_FOPS_RDWR(tool_peer_port_fops,
+		      tool_peer_port_read,
+		      NULL);
+
+static int tool_init_peers(struct tool_ctx *tc)
 {
-	int spad_idx;
-	u32 spad_val;
-	char *buf, *buf_ptr;
-	int pos, n;
-	ssize_t rc;
-
-	if (!spad_write_fn) {
-		dev_dbg(&tc->ntb->dev, "no spad write fn\n");
-		return -EINVAL;
-	}
+	int pidx;
 
-	buf = kmalloc(size + 1, GFP_KERNEL);
-	if (!buf)
+	tc->peer_cnt = ntb_peer_port_count(tc->ntb);
+	tc->peers = devm_kcalloc(&tc->ntb->dev, tc->peer_cnt,
+				 sizeof(*tc->peers), GFP_KERNEL);
+	if (tc->peers == NULL)
 		return -ENOMEM;
 
-	rc = simple_write_to_buffer(buf, size, offp, ubuf, size);
-	if (rc < 0) {
-		kfree(buf);
-		return rc;
-	}
-
-	buf[size] = 0;
-	buf_ptr = buf;
-	n = sscanf(buf_ptr, "%d %i%n", &spad_idx, &spad_val, &pos);
-	while (n == 2) {
-		buf_ptr += pos;
-		rc = spad_write_fn(tc->ntb, spad_idx, spad_val);
-		if (rc)
-			break;
-
-		n = sscanf(buf_ptr, "%d %i%n", &spad_idx, &spad_val, &pos);
+	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
+		tc->peers[pidx].pidx = pidx;
+		tc->peers[pidx].tc = tc;
 	}
 
-	if (n < 0)
-		rc = n;
-
-	kfree(buf);
-
-	return rc ? : size;
+	return 0;
 }
 
-static ssize_t tool_db_read(struct file *filep, char __user *ubuf,
-			    size_t size, loff_t *offp)
-{
-	struct tool_ctx *tc = filep->private_data;
-
-	return tool_dbfn_read(tc, ubuf, size, offp,
-			      tc->ntb->ops->db_read);
-}
+/*==============================================================================
+ *                       Link state read/write methods
+ *==============================================================================
+ */
 
-static ssize_t tool_db_write(struct file *filep, const char __user *ubuf,
-			     size_t size, loff_t *offp)
+static ssize_t tool_link_write(struct file *filep, const char __user *ubuf,
+			       size_t size, loff_t *offp)
 {
 	struct tool_ctx *tc = filep->private_data;
+	bool val;
+	int ret;
 
-	return tool_dbfn_write(tc, ubuf, size, offp,
-			       tc->ntb->ops->db_set,
-			       tc->ntb->ops->db_clear);
-}
+	ret = kstrtobool_from_user(ubuf, size, &val);
+	if (ret)
+		return ret;
 
-static TOOL_FOPS_RDWR(tool_db_fops,
-		      tool_db_read,
-		      tool_db_write);
+	if (val)
+		ret = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
+	else
+		ret = ntb_link_disable(tc->ntb);
 
-static ssize_t tool_mask_read(struct file *filep, char __user *ubuf,
-			      size_t size, loff_t *offp)
-{
-	struct tool_ctx *tc = filep->private_data;
+	if (ret)
+		return ret;
 
-	return tool_dbfn_read(tc, ubuf, size, offp,
-			      tc->ntb->ops->db_read_mask);
+	return size;
 }
 
-static ssize_t tool_mask_write(struct file *filep, const char __user *ubuf,
-			       size_t size, loff_t *offp)
+static TOOL_FOPS_RDWR(tool_link_fops,
+		      NULL,
+		      tool_link_write);
+
+static ssize_t tool_peer_link_read(struct file *filep, char __user *ubuf,
+				   size_t size, loff_t *offp)
 {
-	struct tool_ctx *tc = filep->private_data;
+	struct tool_peer *peer = filep->private_data;
+	struct tool_ctx *tc = peer->tc;
+	char buf[3];
 
-	return tool_dbfn_write(tc, ubuf, size, offp,
-			       tc->ntb->ops->db_set_mask,
-			       tc->ntb->ops->db_clear_mask);
+	if (ntb_link_is_up(tc->ntb, NULL, NULL) & BIT(peer->pidx))
+		buf[0] = 'Y';
+	else
+		buf[0] = 'N';
+	buf[1] = '\n';
+	buf[2] = '\0';
+
+	return simple_read_from_buffer(ubuf, size, offp, buf, 3);
 }
 
-static TOOL_FOPS_RDWR(tool_mask_fops,
-		      tool_mask_read,
-		      tool_mask_write);
+static TOOL_FOPS_RDWR(tool_peer_link_fops,
+		      tool_peer_link_read,
+		      NULL);
 
-static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf,
-				 size_t size, loff_t *offp)
+static ssize_t tool_peer_link_event_write(struct file *filep,
+					  const char __user *ubuf,
+					  size_t size, loff_t *offp)
 {
-	struct tool_ctx *tc = filep->private_data;
+	struct tool_peer *peer = filep->private_data;
+	struct tool_ctx *tc = peer->tc;
+	u64 link_msk;
+	bool val;
+	int ret;
 
-	return tool_dbfn_read(tc, ubuf, size, offp,
-			      tc->ntb->ops->peer_db_read);
-}
+	ret = kstrtobool_from_user(ubuf, size, &val);
+	if (ret)
+		return ret;
 
-static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf,
-				  size_t size, loff_t *offp)
-{
-	struct tool_ctx *tc = filep->private_data;
+	link_msk = BIT_ULL_MASK(peer->pidx);
+
+	if (wait_event_interruptible(tc->link_wq,
+		!!(ntb_link_is_up(tc->ntb, NULL, NULL) & link_msk) == val))
+		return -ERESTART;
 
-	return tool_dbfn_write(tc, ubuf, size, offp,
-			       tc->ntb->ops->peer_db_set,
-			       tc->ntb->ops->peer_db_clear);
+	return size;
 }
 
-static TOOL_FOPS_RDWR(tool_peer_db_fops,
-		      tool_peer_db_read,
-		      tool_peer_db_write);
+static TOOL_FOPS_RDWR(tool_peer_link_event_fops,
+		      NULL,
+		      tool_peer_link_event_write);
 
-static ssize_t tool_peer_mask_read(struct file *filep, char __user *ubuf,
-				   size_t size, loff_t *offp)
+/*==============================================================================
+ *                  Memory windows read/write/setting methods
+ *==============================================================================
+ */
+
+static ssize_t tool_mw_read(struct file *filep, char __user *ubuf,
+			    size_t size, loff_t *offp)
 {
-	struct tool_ctx *tc = filep->private_data;
+	struct tool_mw *inmw = filep->private_data;
+
+	if (inmw->mm_base == NULL)
+		return -ENXIO;
 
-	return tool_dbfn_read(tc, ubuf, size, offp,
-			      tc->ntb->ops->peer_db_read_mask);
+	return simple_read_from_buffer(ubuf, size, offp,
+				       inmw->mm_base, inmw->size);
 }
 
-static ssize_t tool_peer_mask_write(struct file *filep, const char __user *ubuf,
-				    size_t size, loff_t *offp)
+static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf,
+			     size_t size, loff_t *offp)
 {
-	struct tool_ctx *tc = filep->private_data;
+	struct tool_mw *inmw = filep->private_data;
+
+	if (inmw->mm_base == NULL)
+		return -ENXIO;
 
-	return tool_dbfn_write(tc, ubuf, size, offp,
-			       tc->ntb->ops->peer_db_set_mask,
-			       tc->ntb->ops->peer_db_clear_mask);
+	return simple_write_to_buffer(inmw->mm_base, inmw->size, offp,
+				      ubuf, size);
 }
 
-static TOOL_FOPS_RDWR(tool_peer_mask_fops,
-		      tool_peer_mask_read,
-		      tool_peer_mask_write);
+static TOOL_FOPS_RDWR(tool_mw_fops,
+		      tool_mw_read,
+		      tool_mw_write);
 
-static ssize_t tool_spad_read(struct file *filep, char __user *ubuf,
-			      size_t size, loff_t *offp)
+static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
+			 size_t req_size)
 {
-	struct tool_ctx *tc = filep->private_data;
+	resource_size_t size, addr_align, size_align;
+	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
+	char buf[TOOL_BUF_LEN];
+	int ret;
 
-	return tool_spadfn_read(tc, ubuf, size, offp,
-				tc->ntb->ops->spad_read);
-}
+	if (inmw->mm_base != NULL)
+		return 0;
 
-static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
-			       size_t size, loff_t *offp)
-{
-	struct tool_ctx *tc = filep->private_data;
+	ret = ntb_mw_get_align(tc->ntb, pidx, widx, &addr_align,
+				&size_align, &size);
+	if (ret)
+		return ret;
+
+	inmw->size = min_t(resource_size_t, req_size, size);
+	inmw->size = round_up(inmw->size, addr_align);
+	inmw->size = round_up(inmw->size, size_align);
+	inmw->mm_base = dma_alloc_coherent(&tc->ntb->dev, inmw->size,
+					   &inmw->dma_base, GFP_KERNEL);
+	if (!inmw->mm_base)
+		return -ENOMEM;
 
-	return tool_spadfn_write(tc, ubuf, size, offp,
-				 tc->ntb->ops->spad_write);
-}
+	if (!IS_ALIGNED(inmw->dma_base, addr_align)) {
+		ret = -ENOMEM;
+		goto err_free_dma;
+	}
 
-static TOOL_FOPS_RDWR(tool_spad_fops,
-		      tool_spad_read,
-		      tool_spad_write);
+	ret = ntb_mw_set_trans(tc->ntb, pidx, widx, inmw->dma_base, inmw->size);
+	if (ret)
+		goto err_free_dma;
 
-static u32 ntb_tool_peer_spad_read(struct ntb_dev *ntb, int sidx)
-{
-	return ntb_peer_spad_read(ntb, PIDX, sidx);
-}
+	snprintf(buf, sizeof(buf), "mw%d", widx);
+	inmw->dbgfs_file = debugfs_create_file(buf, 0600,
+					       tc->peers[pidx].dbgfs_dir, inmw,
+					       &tool_mw_fops);
 
-static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf,
-				   size_t size, loff_t *offp)
-{
-	struct tool_ctx *tc = filep->private_data;
+	return 0;
 
-	return tool_spadfn_read(tc, ubuf, size, offp, ntb_tool_peer_spad_read);
-}
+err_free_dma:
+	dma_free_coherent(&tc->ntb->dev, inmw->size, inmw->mm_base,
+			  inmw->dma_base);
+	inmw->mm_base = NULL;
+	inmw->dma_base = 0;
+	inmw->size = 0;
 
-static int ntb_tool_peer_spad_write(struct ntb_dev *ntb, int sidx, u32 val)
-{
-	return ntb_peer_spad_write(ntb, PIDX, sidx, val);
+	return ret;
 }
 
-static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf,
-				    size_t size, loff_t *offp)
+static void tool_free_mw(struct tool_ctx *tc, int pidx, int widx)
 {
-	struct tool_ctx *tc = filep->private_data;
-
-	return tool_spadfn_write(tc, ubuf, size, offp,
-				 ntb_tool_peer_spad_write);
-}
-
-static TOOL_FOPS_RDWR(tool_peer_spad_fops,
-		      tool_peer_spad_read,
-		      tool_peer_spad_write);
+	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
 
-static ssize_t tool_link_read(struct file *filep, char __user *ubuf,
-			      size_t size, loff_t *offp)
-{
-	struct tool_ctx *tc = filep->private_data;
-	char buf[3];
+	debugfs_remove(inmw->dbgfs_file);
 
-	buf[0] = ntb_link_is_up(tc->ntb, NULL, NULL) ? 'Y' : 'N';
-	buf[1] = '\n';
-	buf[2] = '\0';
+	if (inmw->mm_base != NULL) {
+		ntb_mw_clear_trans(tc->ntb, pidx, widx);
+		dma_free_coherent(&tc->ntb->dev, inmw->size,
+				  inmw->mm_base, inmw->dma_base);
+	}
 
-	return simple_read_from_buffer(ubuf, size, offp, buf, 2);
+	inmw->mm_base = NULL;
+	inmw->dma_base = 0;
+	inmw->size = 0;
+	inmw->dbgfs_file = NULL;
 }
 
-static ssize_t tool_link_write(struct file *filep, const char __user *ubuf,
-			       size_t size, loff_t *offp)
+static ssize_t tool_mw_trans_read(struct file *filep, char __user *ubuf,
+				  size_t size, loff_t *offp)
 {
-	struct tool_ctx *tc = filep->private_data;
-	char buf[32];
+	struct tool_mw *inmw = filep->private_data;
+	resource_size_t addr_align;
+	resource_size_t size_align;
+	resource_size_t size_max;
+	ssize_t ret, off = 0;
 	size_t buf_size;
-	bool val;
-	int rc;
+	char *buf;
 
-	buf_size = min(size, (sizeof(buf) - 1));
-	if (copy_from_user(buf, ubuf, buf_size))
-		return -EFAULT;
+	buf_size = min_t(size_t, size, 512);
 
-	buf[buf_size] = '\0';
+	buf = kmalloc(buf_size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
 
-	rc = strtobool(buf, &val);
-	if (rc)
-		return rc;
+	ret = ntb_mw_get_align(inmw->tc->ntb, inmw->pidx, inmw->widx,
+			       &addr_align, &size_align, &size_max);
+	if (ret)
+		goto err;
 
-	if (val)
-		rc = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
-	else
-		rc = ntb_link_disable(tc->ntb);
+	off += scnprintf(buf + off, buf_size - off,
+			 "Inbound MW     \t%d\n",
+			 inmw->widx);
 
-	if (rc)
-		return rc;
+	off += scnprintf(buf + off, buf_size - off,
+			 "Port           \t%d (%d)\n",
+			 ntb_peer_port_number(inmw->tc->ntb, inmw->pidx),
+			 inmw->pidx);
 
-	return size;
-}
+	off += scnprintf(buf + off, buf_size - off,
+			 "Window Address \t0x%pK\n", inmw->mm_base);
 
-static TOOL_FOPS_RDWR(tool_link_fops,
-		      tool_link_read,
-		      tool_link_write);
+	off += scnprintf(buf + off, buf_size - off,
+			 "DMA Address    \t%pad\n",
+			 &inmw->dma_base);
 
-static ssize_t tool_link_event_write(struct file *filep,
-				     const char __user *ubuf,
-				     size_t size, loff_t *offp)
-{
-	struct tool_ctx *tc = filep->private_data;
-	char buf[32];
-	size_t buf_size;
-	bool val;
-	int rc;
+	off += scnprintf(buf + off, buf_size - off,
+			 "Window Size    \t%pa[p]\n",
+			 &inmw->size);
 
-	buf_size = min(size, (sizeof(buf) - 1));
-	if (copy_from_user(buf, ubuf, buf_size))
-		return -EFAULT;
+	off += scnprintf(buf + off, buf_size - off,
+			 "Alignment      \t%pa[p]\n",
+			 &addr_align);
 
-	buf[buf_size] = '\0';
+	off += scnprintf(buf + off, buf_size - off,
+			 "Size Alignment \t%pa[p]\n",
+			 &size_align);
+
+	off += scnprintf(buf + off, buf_size - off,
+			 "Size Max       \t%pa[p]\n",
+			 &size_max);
 
-	rc = strtobool(buf, &val);
-	if (rc)
-		return rc;
+	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 
-	if (wait_event_interruptible(tc->link_wq,
-		ntb_link_is_up(tc->ntb, NULL, NULL) == val))
-		return -ERESTART;
+err:
+	kfree(buf);
+
+	return ret;
+}
+
+static ssize_t tool_mw_trans_write(struct file *filep, const char __user *ubuf,
+				   size_t size, loff_t *offp)
+{
+	struct tool_mw *inmw = filep->private_data;
+	unsigned int val;
+	int ret;
+
+	ret = kstrtouint_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
+
+	tool_free_mw(inmw->tc, inmw->pidx, inmw->widx);
+	if (val) {
+		ret = tool_setup_mw(inmw->tc, inmw->pidx, inmw->widx, val);
+		if (ret)
+			return ret;
+	}
 
 	return size;
 }
 
-static TOOL_FOPS_RDWR(tool_link_event_fops,
-		      NULL,
-		      tool_link_event_write);
+static TOOL_FOPS_RDWR(tool_mw_trans_fops,
+		      tool_mw_trans_read,
+		      tool_mw_trans_write);
 
-static ssize_t tool_mw_read(struct file *filep, char __user *ubuf,
-			    size_t size, loff_t *offp)
+static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
+				 size_t size, loff_t *offp)
 {
-	struct tool_mw *mw = filep->private_data;
-	ssize_t rc;
+	struct tool_mw *outmw = filep->private_data;
 	loff_t pos = *offp;
+	ssize_t ret;
 	void *buf;
 
-	if (mw->local == NULL)
+	if (outmw->io_base == NULL)
 		return -EIO;
-	if (pos < 0)
-		return -EINVAL;
-	if (pos >= mw->win_size || !size)
+
+	if (pos >= outmw->size || !size)
 		return 0;
-	if (size > mw->win_size - pos)
-		size = mw->win_size - pos;
+
+	if (size > outmw->size - pos)
+		size = outmw->size - pos;
 
 	buf = kmalloc(size, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
-	memcpy_fromio(buf, mw->local + pos, size);
-	rc = copy_to_user(ubuf, buf, size);
-	if (rc == size) {
-		rc = -EFAULT;
+	memcpy_fromio(buf, outmw->io_base + pos, size);
+	ret = copy_to_user(ubuf, buf, size);
+	if (ret == size) {
+		ret = -EFAULT;
 		goto err_free;
 	}
 
-	size -= rc;
+	size -= ret;
 	*offp = pos + size;
-	rc = size;
+	ret = size;
 
 err_free:
 	kfree(buf);
 
-	return rc;
+	return ret;
 }
 
-static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf,
-			     size_t size, loff_t *offp)
-{
-	struct tool_mw *mw = filep->private_data;
-	ssize_t rc;
+static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
+				  size_t size, loff_t *offp)
+{
+	struct tool_mw *outmw = filep->private_data;
+	ssize_t ret;
 	loff_t pos = *offp;
 	void *buf;
 
-	if (pos < 0)
-		return -EINVAL;
-	if (pos >= mw->win_size || !size)
+	if (outmw->io_base == NULL)
+		return -EIO;
+
+	if (pos >= outmw->size || !size)
 		return 0;
-	if (size > mw->win_size - pos)
-		size = mw->win_size - pos;
+	if (size > outmw->size - pos)
+		size = outmw->size - pos;
 
 	buf = kmalloc(size, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
-	rc = copy_from_user(buf, ubuf, size);
-	if (rc == size) {
-		rc = -EFAULT;
+	ret = copy_from_user(buf, ubuf, size);
+	if (ret == size) {
+		ret = -EFAULT;
 		goto err_free;
 	}
 
-	size -= rc;
+	size -= ret;
 	*offp = pos + size;
-	rc = size;
+	ret = size;
 
-	memcpy_toio(mw->local + pos, buf, size);
+	memcpy_toio(outmw->io_base + pos, buf, size);
 
 err_free:
 	kfree(buf);
 
-	return rc;
-}
-
-static TOOL_FOPS_RDWR(tool_mw_fops,
-		      tool_mw_read,
-		      tool_mw_write);
-
-static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
-				 size_t size, loff_t *offp)
-{
-	struct tool_mw *mw = filep->private_data;
-
-	if (!mw->peer)
-		return -ENXIO;
-
-	return simple_read_from_buffer(ubuf, size, offp, mw->peer, mw->size);
-}
-
-static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
-				  size_t size, loff_t *offp)
-{
-	struct tool_mw *mw = filep->private_data;
-
-	if (!mw->peer)
-		return -ENXIO;
-
-	return simple_write_to_buffer(mw->peer, mw->size, offp, ubuf, size);
+	return ret;
 }
 
 static TOOL_FOPS_RDWR(tool_peer_mw_fops,
 		      tool_peer_mw_read,
 		      tool_peer_mw_write);
 
-static int tool_setup_mw(struct tool_ctx *tc, int idx, size_t req_size)
+static int tool_setup_peer_mw(struct tool_ctx *tc, int pidx, int widx,
+			      u64 req_addr, size_t req_size)
 {
-	int rc;
-	struct tool_mw *mw = &tc->mws[idx];
-	resource_size_t size, align_addr, align_size;
-	char buf[16];
+	struct tool_mw *outmw = &tc->outmws[widx];
+	resource_size_t map_size;
+	phys_addr_t map_base;
+	char buf[TOOL_BUF_LEN];
+	int ret;
 
-	if (mw->peer)
+	if (outmw->io_base != NULL)
 		return 0;
 
-	rc = ntb_mw_get_align(tc->ntb, PIDX, idx, &align_addr,
-				&align_size, &size);
-	if (rc)
-		return rc;
+	ret = ntb_peer_mw_get_addr(tc->ntb, widx, &map_base, &map_size);
+	if (ret)
+		return ret;
 
-	mw->size = min_t(resource_size_t, req_size, size);
-	mw->size = round_up(mw->size, align_addr);
-	mw->size = round_up(mw->size, align_size);
-	mw->peer = dma_alloc_coherent(&tc->ntb->pdev->dev, mw->size,
-				      &mw->peer_dma, GFP_KERNEL);
+	ret = ntb_peer_mw_set_trans(tc->ntb, pidx, widx, req_addr, req_size);
+	if (ret)
+		return ret;
 
-	if (!mw->peer || !IS_ALIGNED(mw->peer_dma, align_addr))
-		return -ENOMEM;
+	outmw->io_base = ioremap_wc(map_base, map_size);
+	if (outmw->io_base == NULL) {
+		ret = -EFAULT;
+		goto err_clear_trans;
+	}
 
-	rc = ntb_mw_set_trans(tc->ntb, PIDX, idx, mw->peer_dma, mw->size);
-	if (rc)
-		goto err_free_dma;
+	outmw->tr_base = req_addr;
+	outmw->size = req_size;
+	outmw->pidx = pidx;
 
-	snprintf(buf, sizeof(buf), "peer_mw%d", idx);
-	mw->peer_dbg_file = debugfs_create_file(buf, S_IRUSR | S_IWUSR,
-						mw->tc->dbgfs, mw,
-						&tool_peer_mw_fops);
+	snprintf(buf, sizeof(buf), "peer_mw%d", widx);
+	outmw->dbgfs_file = debugfs_create_file(buf, 0600,
+					       tc->peers[pidx].dbgfs_dir, outmw,
+					       &tool_peer_mw_fops);
 
 	return 0;
 
-err_free_dma:
-	dma_free_coherent(&tc->ntb->pdev->dev, mw->size,
-			  mw->peer,
-			  mw->peer_dma);
-	mw->peer = NULL;
-	mw->peer_dma = 0;
-	mw->size = 0;
-
-	return rc;
+err_clear_trans:
+	ntb_peer_mw_clear_trans(tc->ntb, pidx, widx);
+
+	return ret;
 }
 
-static void tool_free_mw(struct tool_ctx *tc, int idx)
+static void tool_free_peer_mw(struct tool_ctx *tc, int widx)
 {
-	struct tool_mw *mw = &tc->mws[idx];
+	struct tool_mw *outmw = &tc->outmws[widx];
 
-	if (mw->peer) {
-		ntb_mw_clear_trans(tc->ntb, PIDX, idx);
-		dma_free_coherent(&tc->ntb->pdev->dev, mw->size,
-				  mw->peer,
-				  mw->peer_dma);
-	}
-
-	mw->peer = NULL;
-	mw->peer_dma = 0;
+	debugfs_remove(outmw->dbgfs_file);
 
-	debugfs_remove(mw->peer_dbg_file);
+	if (outmw->io_base != NULL) {
+		iounmap(tc->outmws[widx].io_base);
+		ntb_peer_mw_clear_trans(tc->ntb, outmw->pidx, widx);
+	}
 
-	mw->peer_dbg_file = NULL;
+	outmw->io_base = NULL;
+	outmw->tr_base = 0;
+	outmw->size = 0;
+	outmw->pidx = -1;
+	outmw->dbgfs_file = NULL;
 }
 
-static ssize_t tool_peer_mw_trans_read(struct file *filep,
-				       char __user *ubuf,
-				       size_t size, loff_t *offp)
+static ssize_t tool_peer_mw_trans_read(struct file *filep, char __user *ubuf,
+					size_t size, loff_t *offp)
 {
-	struct tool_mw *mw = filep->private_data;
-
-	char *buf;
+	struct tool_mw_wrap *outmw_wrap = filep->private_data;
+	struct tool_mw *outmw = outmw_wrap->mw;
+	resource_size_t map_size;
+	phys_addr_t map_base;
+	ssize_t off = 0;
 	size_t buf_size;
-	ssize_t ret, off = 0;
+	char *buf;
+	int ret;
 
-	phys_addr_t base;
-	resource_size_t mw_size;
-	resource_size_t align_addr = 0;
-	resource_size_t align_size = 0;
-	resource_size_t max_size = 0;
+	ret = ntb_peer_mw_get_addr(outmw->tc->ntb, outmw->widx,
+				  &map_base, &map_size);
+	if (ret)
+		return ret;
 
 	buf_size = min_t(size_t, size, 512);
 
@@ -763,43 +890,37 @@ static ssize_t tool_peer_mw_trans_read(struct file *filep,
 	if (!buf)
 		return -ENOMEM;
 
-	ntb_mw_get_align(mw->tc->ntb, PIDX, mw->idx,
-			 &align_addr, &align_size, &max_size);
-	ntb_peer_mw_get_addr(mw->tc->ntb, mw->idx, &base, &mw_size);
-
 	off += scnprintf(buf + off, buf_size - off,
-			 "Peer MW %d Information:\n", mw->idx);
+			 "Outbound MW:        \t%d\n", outmw->widx);
 
-	off += scnprintf(buf + off, buf_size - off,
-			 "Physical Address      \t%pa[p]\n",
-			 &base);
-
-	off += scnprintf(buf + off, buf_size - off,
-			 "Window Size           \t%lld\n",
-			 (unsigned long long)mw_size);
+	if (outmw->io_base != NULL) {
+		off += scnprintf(buf + off, buf_size - off,
+			"Port attached       \t%d (%d)\n",
+			ntb_peer_port_number(outmw->tc->ntb, outmw->pidx),
+			outmw->pidx);
+	} else {
+		off += scnprintf(buf + off, buf_size - off,
+				 "Port attached       \t-1 (-1)\n");
+	}
 
 	off += scnprintf(buf + off, buf_size - off,
-			 "Alignment             \t%lld\n",
-			 (unsigned long long)align_addr);
+			 "Virtual address     \t0x%pK\n", outmw->io_base);
 
 	off += scnprintf(buf + off, buf_size - off,
-			 "Size Alignment        \t%lld\n",
-			 (unsigned long long)align_size);
+			 "Phys Address        \t%pa[p]\n", &map_base);
 
 	off += scnprintf(buf + off, buf_size - off,
-			 "Size Max              \t%lld\n",
-			 (unsigned long long)max_size);
+			 "Mapping Size        \t%pa[p]\n", &map_size);
 
 	off += scnprintf(buf + off, buf_size - off,
-			 "Ready                 \t%c\n",
-			 (mw->peer) ? 'Y' : 'N');
+			 "Translation Address \t0x%016llx\n", outmw->tr_base);
 
 	off += scnprintf(buf + off, buf_size - off,
-			 "Allocated Size       \t%zd\n",
-			 (mw->peer) ? (size_t)mw->size : 0);
+			 "Window Size         \t%pa[p]\n", &outmw->size);
 
 	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 	kfree(buf);
+
 	return ret;
 }
 
@@ -807,12 +928,12 @@ static ssize_t tool_peer_mw_trans_write(struct file *filep,
 					const char __user *ubuf,
 					size_t size, loff_t *offp)
 {
-	struct tool_mw *mw = filep->private_data;
-
-	char buf[32];
-	size_t buf_size;
-	unsigned long long val;
-	int rc;
+	struct tool_mw_wrap *outmw_wrap = filep->private_data;
+	struct tool_mw *outmw = outmw_wrap->mw;
+	size_t buf_size, wsize;
+	char buf[TOOL_BUF_LEN];
+	int ret, n;
+	u64 addr;
 
 	buf_size = min(size, (sizeof(buf) - 1));
 	if (copy_from_user(buf, ubuf, buf_size))
@@ -820,16 +941,17 @@ static ssize_t tool_peer_mw_trans_write(struct file *filep,
 
 	buf[buf_size] = '\0';
 
-	rc = kstrtoull(buf, 0, &val);
-	if (rc)
-		return rc;
-
-	tool_free_mw(mw->tc, mw->idx);
-	if (val)
-		rc = tool_setup_mw(mw->tc, mw->idx, val);
+	n = sscanf(buf, "%lli:%zi", &addr, &wsize);
+	if (n != 2)
+		return -EINVAL;
 
-	if (rc)
-		return rc;
+	tool_free_peer_mw(outmw->tc, outmw->widx);
+	if (wsize) {
+		ret = tool_setup_peer_mw(outmw->tc, outmw_wrap->pidx,
+					 outmw->widx, addr, wsize);
+		if (ret)
+			return ret;
+	}
 
 	return size;
 }
@@ -838,195 +960,734 @@ static TOOL_FOPS_RDWR(tool_peer_mw_trans_fops,
 		      tool_peer_mw_trans_read,
 		      tool_peer_mw_trans_write);
 
-static int tool_init_mw(struct tool_ctx *tc, int idx)
+static int tool_init_mws(struct tool_ctx *tc)
 {
-	struct tool_mw *mw = &tc->mws[idx];
-	phys_addr_t base;
-	int rc;
-
-	rc = ntb_peer_mw_get_addr(tc->ntb, idx, &base, &mw->win_size);
-	if (rc)
-		return rc;
-
-	mw->tc = tc;
-	mw->idx = idx;
-	mw->local = ioremap_wc(base, mw->win_size);
-	if (!mw->local)
-		return -EFAULT;
+	int widx, pidx;
+
+	/* Initialize outbound memory windows */
+	tc->outmw_cnt = ntb_peer_mw_count(tc->ntb);
+	tc->outmws = devm_kcalloc(&tc->ntb->dev, tc->outmw_cnt,
+				  sizeof(*tc->outmws), GFP_KERNEL);
+	if (tc->outmws == NULL)
+		return -ENOMEM;
+
+	for (widx = 0; widx < tc->outmw_cnt; widx++) {
+		tc->outmws[widx].widx = widx;
+		tc->outmws[widx].pidx = -1;
+		tc->outmws[widx].tc = tc;
+	}
+
+	/* Initialize inbound memory windows and outbound MWs wrapper */
+	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
+		tc->peers[pidx].inmw_cnt = ntb_mw_count(tc->ntb, pidx);
+		tc->peers[pidx].inmws =
+			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].inmw_cnt,
+				    sizeof(*tc->peers[pidx].inmws), GFP_KERNEL);
+		if (tc->peers[pidx].inmws == NULL)
+			return -ENOMEM;
+
+		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
+			tc->peers[pidx].inmws[widx].widx = widx;
+			tc->peers[pidx].inmws[widx].pidx = pidx;
+			tc->peers[pidx].inmws[widx].tc = tc;
+		}
+
+		tc->peers[pidx].outmw_cnt = ntb_peer_mw_count(tc->ntb);
+		tc->peers[pidx].outmws =
+			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmw_cnt,
+				   sizeof(*tc->peers[pidx].outmws), GFP_KERNEL);
+
+		for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
+			tc->peers[pidx].outmws[widx].pidx = pidx;
+			tc->peers[pidx].outmws[widx].mw = &tc->outmws[widx];
+		}
+	}
 
 	return 0;
 }
 
-static void tool_free_mws(struct tool_ctx *tc)
+static void tool_clear_mws(struct tool_ctx *tc)
 {
-	int i;
+	int widx, pidx;
 
-	for (i = 0; i < tc->mw_count; i++) {
-		tool_free_mw(tc, i);
+	/* Free outbound memory windows */
+	for (widx = 0; widx < tc->outmw_cnt; widx++)
+		tool_free_peer_mw(tc, widx);
 
-		if (tc->mws[i].local)
-			iounmap(tc->mws[i].local);
+	/* Free outbound memory windows */
+	for (pidx = 0; pidx < tc->peer_cnt; pidx++)
+		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++)
+			tool_free_mw(tc, pidx, widx);
+}
 
-		tc->mws[i].local = NULL;
-	}
+/*==============================================================================
+ *                       Doorbell read/write methods
+ *==============================================================================
+ */
+
+static ssize_t tool_db_read(struct file *filep, char __user *ubuf,
+			    size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read);
 }
 
-static void tool_setup_dbgfs(struct tool_ctx *tc)
+static ssize_t tool_db_write(struct file *filep, const char __user *ubuf,
+			     size_t size, loff_t *offp)
 {
-	int i;
+	struct tool_ctx *tc = filep->private_data;
 
-	/* This modules is useless without dbgfs... */
-	if (!tool_dbgfs) {
-		tc->dbgfs = NULL;
-		return;
+	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set,
+			     tc->ntb->ops->db_clear);
+}
+
+static TOOL_FOPS_RDWR(tool_db_fops,
+		      tool_db_read,
+		      tool_db_write);
+
+static ssize_t tool_db_valid_mask_read(struct file *filep, char __user *ubuf,
+				       size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_valid_mask);
+}
+
+static TOOL_FOPS_RDWR(tool_db_valid_mask_fops,
+		      tool_db_valid_mask_read,
+		      NULL);
+
+static ssize_t tool_db_mask_read(struct file *filep, char __user *ubuf,
+				 size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read_mask);
+}
+
+static ssize_t tool_db_mask_write(struct file *filep, const char __user *ubuf,
+			       size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set_mask,
+			     tc->ntb->ops->db_clear_mask);
+}
+
+static TOOL_FOPS_RDWR(tool_db_mask_fops,
+		      tool_db_mask_read,
+		      tool_db_mask_write);
+
+static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf,
+				 size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->peer_db_read);
+}
+
+static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf,
+				  size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->peer_db_set,
+			     tc->ntb->ops->peer_db_clear);
+}
+
+static TOOL_FOPS_RDWR(tool_peer_db_fops,
+		      tool_peer_db_read,
+		      tool_peer_db_write);
+
+static ssize_t tool_peer_db_mask_read(struct file *filep, char __user *ubuf,
+				   size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp,
+			    tc->ntb->ops->peer_db_read_mask);
+}
+
+static ssize_t tool_peer_db_mask_write(struct file *filep,
+				       const char __user *ubuf,
+				       size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_write(tc, ubuf, size, offp,
+			     tc->ntb->ops->peer_db_set_mask,
+			     tc->ntb->ops->peer_db_clear_mask);
+}
+
+static TOOL_FOPS_RDWR(tool_peer_db_mask_fops,
+		      tool_peer_db_mask_read,
+		      tool_peer_db_mask_write);
+
+static ssize_t tool_db_event_write(struct file *filep,
+				   const char __user *ubuf,
+				   size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+	u64 val;
+	int ret;
+
+	ret = kstrtou64_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
+
+	if (wait_event_interruptible(tc->db_wq, ntb_db_read(tc->ntb) == val))
+		return -ERESTART;
+
+	return size;
+}
+
+static TOOL_FOPS_RDWR(tool_db_event_fops,
+		      NULL,
+		      tool_db_event_write);
+
+/*==============================================================================
+ *                       Scratchpads read/write methods
+ *==============================================================================
+ */
+
+static ssize_t tool_spad_read(struct file *filep, char __user *ubuf,
+			      size_t size, loff_t *offp)
+{
+	struct tool_spad *spad = filep->private_data;
+	char buf[TOOL_BUF_LEN];
+	ssize_t pos;
+
+	if (!spad->tc->ntb->ops->spad_read)
+		return -EINVAL;
+
+	pos = scnprintf(buf, sizeof(buf), "%#x\n",
+		ntb_spad_read(spad->tc->ntb, spad->sidx));
+
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
+}
+
+static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
+			       size_t size, loff_t *offp)
+{
+	struct tool_spad *spad = filep->private_data;
+	u32 val;
+	int ret;
+
+	if (!spad->tc->ntb->ops->spad_write) {
+		dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
+		return -EINVAL;
 	}
 
-	tc->dbgfs = debugfs_create_dir(dev_name(&tc->ntb->dev),
-				       tool_dbgfs);
-	if (!tc->dbgfs)
-		return;
+	ret = kstrtou32_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
 
-	debugfs_create_file("db", S_IRUSR | S_IWUSR, tc->dbgfs,
-			    tc, &tool_db_fops);
+	ret = ntb_spad_write(spad->tc->ntb, spad->sidx, val);
 
-	debugfs_create_file("mask", S_IRUSR | S_IWUSR, tc->dbgfs,
-			    tc, &tool_mask_fops);
+	return ret ?: size;
+}
 
-	debugfs_create_file("peer_db", S_IRUSR | S_IWUSR, tc->dbgfs,
-			    tc, &tool_peer_db_fops);
+static TOOL_FOPS_RDWR(tool_spad_fops,
+		      tool_spad_read,
+		      tool_spad_write);
 
-	debugfs_create_file("peer_mask", S_IRUSR | S_IWUSR, tc->dbgfs,
-			    tc, &tool_peer_mask_fops);
+static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf,
+				   size_t size, loff_t *offp)
+{
+	struct tool_spad *spad = filep->private_data;
+	char buf[TOOL_BUF_LEN];
+	ssize_t pos;
 
-	debugfs_create_file("spad", S_IRUSR | S_IWUSR, tc->dbgfs,
-			    tc, &tool_spad_fops);
+	if (!spad->tc->ntb->ops->peer_spad_read)
+		return -EINVAL;
 
-	debugfs_create_file("peer_spad", S_IRUSR | S_IWUSR, tc->dbgfs,
-			    tc, &tool_peer_spad_fops);
+	pos = scnprintf(buf, sizeof(buf), "%#x\n",
+		ntb_peer_spad_read(spad->tc->ntb, spad->pidx, spad->sidx));
 
-	debugfs_create_file("link", S_IRUSR | S_IWUSR, tc->dbgfs,
-			    tc, &tool_link_fops);
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
+}
+
+static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf,
+				    size_t size, loff_t *offp)
+{
+	struct tool_spad *spad = filep->private_data;
+	u32 val;
+	int ret;
+
+	if (!spad->tc->ntb->ops->peer_spad_write) {
+		dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
+		return -EINVAL;
+	}
 
-	debugfs_create_file("link_event", S_IWUSR, tc->dbgfs,
-			    tc, &tool_link_event_fops);
+	ret = kstrtou32_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
 
-	for (i = 0; i < tc->mw_count; i++) {
-		char buf[30];
+	ret = ntb_peer_spad_write(spad->tc->ntb, spad->pidx, spad->sidx, val);
 
-		snprintf(buf, sizeof(buf), "mw%d", i);
-		debugfs_create_file(buf, S_IRUSR | S_IWUSR, tc->dbgfs,
-				    &tc->mws[i], &tool_mw_fops);
+	return ret ?: size;
+}
+
+static TOOL_FOPS_RDWR(tool_peer_spad_fops,
+		      tool_peer_spad_read,
+		      tool_peer_spad_write);
+
+static int tool_init_spads(struct tool_ctx *tc)
+{
+	int sidx, pidx;
+
+	/* Initialize inbound scratchpad structures */
+	tc->inspad_cnt = ntb_spad_count(tc->ntb);
+	tc->inspads = devm_kcalloc(&tc->ntb->dev, tc->inspad_cnt,
+				   sizeof(*tc->inspads), GFP_KERNEL);
+	if (tc->inspads == NULL)
+		return -ENOMEM;
+
+	for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
+		tc->inspads[sidx].sidx = sidx;
+		tc->inspads[sidx].pidx = -1;
+		tc->inspads[sidx].tc = tc;
+	}
 
-		snprintf(buf, sizeof(buf), "peer_trans%d", i);
-		debugfs_create_file(buf, S_IRUSR | S_IWUSR, tc->dbgfs,
-				    &tc->mws[i], &tool_peer_mw_trans_fops);
+	/* Initialize outbound scratchpad structures */
+	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
+		tc->peers[pidx].outspad_cnt = ntb_spad_count(tc->ntb);
+		tc->peers[pidx].outspads =
+			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outspad_cnt,
+				sizeof(*tc->peers[pidx].outspads), GFP_KERNEL);
+		if (tc->peers[pidx].outspads == NULL)
+			return -ENOMEM;
+
+		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
+			tc->peers[pidx].outspads[sidx].sidx = sidx;
+			tc->peers[pidx].outspads[sidx].pidx = pidx;
+			tc->peers[pidx].outspads[sidx].tc = tc;
+		}
 	}
+
+	return 0;
 }
 
-static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
+/*==============================================================================
+ *                       Messages read/write methods
+ *==============================================================================
+ */
+
+static ssize_t tool_inmsg_read(struct file *filep, char __user *ubuf,
+			       size_t size, loff_t *offp)
 {
-	struct tool_ctx *tc;
-	int rc;
-	int i;
+	struct tool_msg *msg = filep->private_data;
+	char buf[TOOL_BUF_LEN];
+	ssize_t pos;
+	u32 data;
+	int pidx;
+
+	data = ntb_msg_read(msg->tc->ntb, &pidx, msg->midx);
+
+	pos = scnprintf(buf, sizeof(buf), "0x%08x<-%d\n", data, pidx);
+
+	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
+}
+
+static TOOL_FOPS_RDWR(tool_inmsg_fops,
+		      tool_inmsg_read,
+		      NULL);
+
+static ssize_t tool_outmsg_write(struct file *filep,
+				 const char __user *ubuf,
+				 size_t size, loff_t *offp)
+{
+	struct tool_msg *msg = filep->private_data;
+	u32 val;
+	int ret;
+
+	ret = kstrtou32_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
+
+	ret = ntb_peer_msg_write(msg->tc->ntb, msg->pidx, msg->midx, val);
+
+	return ret ? : size;
+}
+
+static TOOL_FOPS_RDWR(tool_outmsg_fops,
+		      NULL,
+		      tool_outmsg_write);
+
+static ssize_t tool_msg_sts_read(struct file *filep, char __user *ubuf,
+				 size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_read_sts);
+}
+
+static ssize_t tool_msg_sts_write(struct file *filep, const char __user *ubuf,
+				  size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_write(tc, ubuf, size, offp, NULL,
+			     tc->ntb->ops->msg_clear_sts);
+}
+
+static TOOL_FOPS_RDWR(tool_msg_sts_fops,
+		      tool_msg_sts_read,
+		      tool_msg_sts_write);
+
+static ssize_t tool_msg_inbits_read(struct file *filep, char __user *ubuf,
+				    size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_inbits);
+}
+
+static TOOL_FOPS_RDWR(tool_msg_inbits_fops,
+		      tool_msg_inbits_read,
+		      NULL);
+
+static ssize_t tool_msg_outbits_read(struct file *filep, char __user *ubuf,
+				     size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_outbits);
+}
+
+static TOOL_FOPS_RDWR(tool_msg_outbits_fops,
+		      tool_msg_outbits_read,
+		      NULL);
+
+static ssize_t tool_msg_mask_write(struct file *filep, const char __user *ubuf,
+				   size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+
+	return tool_fn_write(tc, ubuf, size, offp,
+			     tc->ntb->ops->msg_set_mask,
+			     tc->ntb->ops->msg_clear_mask);
+}
+
+static TOOL_FOPS_RDWR(tool_msg_mask_fops,
+		      NULL,
+		      tool_msg_mask_write);
+
+static ssize_t tool_msg_event_write(struct file *filep,
+				    const char __user *ubuf,
+				    size_t size, loff_t *offp)
+{
+	struct tool_ctx *tc = filep->private_data;
+	u64 val;
+	int ret;
+
+	ret = kstrtou64_from_user(ubuf, size, 0, &val);
+	if (ret)
+		return ret;
+
+	if (wait_event_interruptible(tc->msg_wq,
+		ntb_msg_read_sts(tc->ntb) == val))
+		return -ERESTART;
+
+	return size;
+}
+
+static TOOL_FOPS_RDWR(tool_msg_event_fops,
+		      NULL,
+		      tool_msg_event_write);
 
-	if (!ntb->ops->mw_set_trans) {
-		dev_dbg(&ntb->dev, "need inbound MW based NTB API\n");
-		rc = -EINVAL;
-		goto err_tc;
+static int tool_init_msgs(struct tool_ctx *tc)
+{
+	int midx, pidx;
+
+	/* Initialize inbound message structures */
+	tc->inmsg_cnt = ntb_msg_count(tc->ntb);
+	tc->inmsgs = devm_kcalloc(&tc->ntb->dev, tc->inmsg_cnt,
+				   sizeof(*tc->inmsgs), GFP_KERNEL);
+	if (tc->inmsgs == NULL)
+		return -ENOMEM;
+
+	for (midx = 0; midx < tc->inmsg_cnt; midx++) {
+		tc->inmsgs[midx].midx = midx;
+		tc->inmsgs[midx].pidx = -1;
+		tc->inmsgs[midx].tc = tc;
 	}
 
-	if (ntb_spad_count(ntb) < 1) {
-		dev_dbg(&ntb->dev, "no enough scratchpads\n");
-		rc = -EINVAL;
-		goto err_tc;
+	/* Initialize outbound message structures */
+	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
+		tc->peers[pidx].outmsg_cnt = ntb_msg_count(tc->ntb);
+		tc->peers[pidx].outmsgs =
+			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmsg_cnt,
+				sizeof(*tc->peers[pidx].outmsgs), GFP_KERNEL);
+		if (tc->peers[pidx].outmsgs == NULL)
+			return -ENOMEM;
+
+		for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
+			tc->peers[pidx].outmsgs[midx].midx = midx;
+			tc->peers[pidx].outmsgs[midx].pidx = pidx;
+			tc->peers[pidx].outmsgs[midx].tc = tc;
+		}
 	}
 
+	return 0;
+}
+
+/*==============================================================================
+ *                          Initialization methods
+ *==============================================================================
+ */
+
+static struct tool_ctx *tool_create_data(struct ntb_dev *ntb)
+{
+	struct tool_ctx *tc;
+
+	tc = devm_kzalloc(&ntb->dev, sizeof(*tc), GFP_KERNEL);
+	if (tc == NULL)
+		return ERR_PTR(-ENOMEM);
+
+	tc->ntb = ntb;
+	init_waitqueue_head(&tc->link_wq);
+	init_waitqueue_head(&tc->db_wq);
+	init_waitqueue_head(&tc->msg_wq);
+
 	if (ntb_db_is_unsafe(ntb))
 		dev_dbg(&ntb->dev, "doorbell is unsafe\n");
 
 	if (ntb_spad_is_unsafe(ntb))
 		dev_dbg(&ntb->dev, "scratchpad is unsafe\n");
 
-	if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT)
-		dev_warn(&ntb->dev, "multi-port NTB is unsupported\n");
+	return tc;
+}
+
+static void tool_clear_data(struct tool_ctx *tc)
+{
+	wake_up(&tc->link_wq);
+	wake_up(&tc->db_wq);
+	wake_up(&tc->msg_wq);
+}
+
+static int tool_init_ntb(struct tool_ctx *tc)
+{
+	return ntb_set_ctx(tc->ntb, tc, &tool_ops);
+}
+
+static void tool_clear_ntb(struct tool_ctx *tc)
+{
+	ntb_clear_ctx(tc->ntb);
+	ntb_link_disable(tc->ntb);
+}
 
-	tc = kzalloc(sizeof(*tc), GFP_KERNEL);
-	if (!tc) {
-		rc = -ENOMEM;
-		goto err_tc;
+static void tool_setup_dbgfs(struct tool_ctx *tc)
+{
+	int pidx, widx, sidx, midx;
+	char buf[TOOL_BUF_LEN];
+
+	/* This modules is useless without dbgfs... */
+	if (!tool_dbgfs_topdir) {
+		tc->dbgfs_dir = NULL;
+		return;
 	}
 
-	tc->ntb = ntb;
-	init_waitqueue_head(&tc->link_wq);
+	tc->dbgfs_dir = debugfs_create_dir(dev_name(&tc->ntb->dev),
+					   tool_dbgfs_topdir);
+	if (!tc->dbgfs_dir)
+		return;
 
-	tc->mw_count = min(ntb_peer_mw_count(tc->ntb), MAX_MWS);
-	for (i = 0; i < tc->mw_count; i++) {
-		rc = tool_init_mw(tc, i);
-		if (rc)
-			goto err_ctx;
+	debugfs_create_file("port", 0600, tc->dbgfs_dir,
+			    tc, &tool_port_fops);
+
+	debugfs_create_file("link", 0600, tc->dbgfs_dir,
+			    tc, &tool_link_fops);
+
+	debugfs_create_file("db", 0600, tc->dbgfs_dir,
+			    tc, &tool_db_fops);
+
+	debugfs_create_file("db_valid_mask", 0600, tc->dbgfs_dir,
+			    tc, &tool_db_valid_mask_fops);
+
+	debugfs_create_file("db_mask", 0600, tc->dbgfs_dir,
+			    tc, &tool_db_mask_fops);
+
+	debugfs_create_file("db_event", 0600, tc->dbgfs_dir,
+			    tc, &tool_db_event_fops);
+
+	debugfs_create_file("peer_db", 0600, tc->dbgfs_dir,
+			    tc, &tool_peer_db_fops);
+
+	debugfs_create_file("peer_db_mask", 0600, tc->dbgfs_dir,
+			    tc, &tool_peer_db_mask_fops);
+
+	if (tc->inspad_cnt != 0) {
+		for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
+			snprintf(buf, sizeof(buf), "spad%d", sidx);
+
+			debugfs_create_file(buf, 0600, tc->dbgfs_dir,
+					   &tc->inspads[sidx], &tool_spad_fops);
+		}
 	}
 
-	tool_setup_dbgfs(tc);
+	if (tc->inmsg_cnt != 0) {
+		for (midx = 0; midx < tc->inmsg_cnt; midx++) {
+			snprintf(buf, sizeof(buf), "msg%d", midx);
+			debugfs_create_file(buf, 0600, tc->dbgfs_dir,
+					   &tc->inmsgs[midx], &tool_inmsg_fops);
+		}
+
+		debugfs_create_file("msg_sts", 0600, tc->dbgfs_dir,
+				    tc, &tool_msg_sts_fops);
+
+		debugfs_create_file("msg_inbits", 0600, tc->dbgfs_dir,
+				    tc, &tool_msg_inbits_fops);
+
+		debugfs_create_file("msg_outbits", 0600, tc->dbgfs_dir,
+				    tc, &tool_msg_outbits_fops);
+
+		debugfs_create_file("msg_mask", 0600, tc->dbgfs_dir,
+				    tc, &tool_msg_mask_fops);
+
+		debugfs_create_file("msg_event", 0600, tc->dbgfs_dir,
+				    tc, &tool_msg_event_fops);
+	}
+
+	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
+		snprintf(buf, sizeof(buf), "peer%d", pidx);
+		tc->peers[pidx].dbgfs_dir =
+			debugfs_create_dir(buf, tc->dbgfs_dir);
+
+		debugfs_create_file("port", 0600,
+				    tc->peers[pidx].dbgfs_dir,
+				    &tc->peers[pidx], &tool_peer_port_fops);
+
+		debugfs_create_file("link", 0200,
+				    tc->peers[pidx].dbgfs_dir,
+				    &tc->peers[pidx], &tool_peer_link_fops);
+
+		debugfs_create_file("link_event", 0200,
+				  tc->peers[pidx].dbgfs_dir,
+				  &tc->peers[pidx], &tool_peer_link_event_fops);
+
+		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
+			snprintf(buf, sizeof(buf), "mw_trans%d", widx);
+			debugfs_create_file(buf, 0600,
+					    tc->peers[pidx].dbgfs_dir,
+					    &tc->peers[pidx].inmws[widx],
+					    &tool_mw_trans_fops);
+		}
+
+		for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
+			snprintf(buf, sizeof(buf), "peer_mw_trans%d", widx);
+			debugfs_create_file(buf, 0600,
+					    tc->peers[pidx].dbgfs_dir,
+					    &tc->peers[pidx].outmws[widx],
+					    &tool_peer_mw_trans_fops);
+		}
+
+		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
+			snprintf(buf, sizeof(buf), "spad%d", sidx);
+
+			debugfs_create_file(buf, 0600,
+					    tc->peers[pidx].dbgfs_dir,
+					    &tc->peers[pidx].outspads[sidx],
+					    &tool_peer_spad_fops);
+		}
+
+		for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
+			snprintf(buf, sizeof(buf), "msg%d", midx);
+			debugfs_create_file(buf, 0600,
+					    tc->peers[pidx].dbgfs_dir,
+					    &tc->peers[pidx].outmsgs[midx],
+					    &tool_outmsg_fops);
+		}
+	}
+}
+
+static void tool_clear_dbgfs(struct tool_ctx *tc)
+{
+	debugfs_remove_recursive(tc->dbgfs_dir);
+}
+
+static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
+{
+	struct tool_ctx *tc;
+	int ret;
 
-	rc = ntb_set_ctx(ntb, tc, &tool_ops);
-	if (rc)
-		goto err_ctx;
+	tc = tool_create_data(ntb);
+	if (IS_ERR(tc))
+		return PTR_ERR(tc);
 
-	ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
-	ntb_link_event(ntb);
+	ret = tool_init_peers(tc);
+	if (ret != 0)
+		goto err_clear_data;
+
+	ret = tool_init_mws(tc);
+	if (ret != 0)
+		goto err_clear_data;
+
+	ret = tool_init_spads(tc);
+	if (ret != 0)
+		goto err_clear_mws;
+
+	ret = tool_init_msgs(tc);
+	if (ret != 0)
+		goto err_clear_mws;
+
+	ret = tool_init_ntb(tc);
+	if (ret != 0)
+		goto err_clear_mws;
+
+	tool_setup_dbgfs(tc);
 
 	return 0;
 
-err_ctx:
-	tool_free_mws(tc);
-	debugfs_remove_recursive(tc->dbgfs);
-	kfree(tc);
-err_tc:
-	return rc;
+err_clear_mws:
+	tool_clear_mws(tc);
+
+err_clear_data:
+	tool_clear_data(tc);
+
+	return ret;
 }
 
 static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb)
 {
 	struct tool_ctx *tc = ntb->ctx;
 
-	tool_free_mws(tc);
+	tool_clear_dbgfs(tc);
+
+	tool_clear_ntb(tc);
 
-	ntb_clear_ctx(ntb);
-	ntb_link_disable(ntb);
+	tool_clear_mws(tc);
 
-	debugfs_remove_recursive(tc->dbgfs);
-	kfree(tc);
+	tool_clear_data(tc);
 }
 
 static struct ntb_client tool_client = {
 	.ops = {
 		.probe = tool_probe,
 		.remove = tool_remove,
-	},
+	}
 };
 
 static int __init tool_init(void)
 {
-	int rc;
+	int ret;
 
 	if (debugfs_initialized())
-		tool_dbgfs = debugfs_create_dir(KBUILD_MODNAME, NULL);
-
-	rc = ntb_register_client(&tool_client);
-	if (rc)
-		goto err_client;
+		tool_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
 
-	return 0;
+	ret = ntb_register_client(&tool_client);
+	if (ret)
+		debugfs_remove_recursive(tool_dbgfs_topdir);
 
-err_client:
-	debugfs_remove_recursive(tool_dbgfs);
-	return rc;
+	return ret;
 }
 module_init(tool_init);
 
 static void __exit tool_exit(void)
 {
 	ntb_unregister_client(&tool_client);
-	debugfs_remove_recursive(tool_dbgfs);
+	debugfs_remove_recursive(tool_dbgfs_topdir);
 }
 module_exit(tool_exit);
+
diff --git a/drivers/nubus/Makefile b/drivers/nubus/Makefile
index 21bda2031e7e..6d063cde39d1 100644
--- a/drivers/nubus/Makefile
+++ b/drivers/nubus/Makefile
@@ -2,6 +2,6 @@
 # Makefile for the nubus specific drivers.
 #
 
-obj-y   := nubus.o
+obj-y := nubus.o bus.o
 
 obj-$(CONFIG_PROC_FS) += proc.o
diff --git a/drivers/nubus/bus.c b/drivers/nubus/bus.c
new file mode 100644
index 000000000000..d306c348c857
--- /dev/null
+++ b/drivers/nubus/bus.c
@@ -0,0 +1,117 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Bus implementation for the NuBus subsystem.
+//
+// Copyright (C) 2017 Finn Thain
+
+#include <linux/device.h>
+#include <linux/list.h>
+#include <linux/nubus.h>
+#include <linux/seq_file.h>
+#include <linux/slab.h>
+
+#define to_nubus_board(d)       container_of(d, struct nubus_board, dev)
+#define to_nubus_driver(d)      container_of(d, struct nubus_driver, driver)
+
+static int nubus_bus_match(struct device *dev, struct device_driver *driver)
+{
+	return 1;
+}
+
+static int nubus_device_probe(struct device *dev)
+{
+	struct nubus_driver *ndrv = to_nubus_driver(dev->driver);
+	int err = -ENODEV;
+
+	if (ndrv->probe)
+		err = ndrv->probe(to_nubus_board(dev));
+	return err;
+}
+
+static int nubus_device_remove(struct device *dev)
+{
+	struct nubus_driver *ndrv = to_nubus_driver(dev->driver);
+	int err = -ENODEV;
+
+	if (dev->driver && ndrv->remove)
+		err = ndrv->remove(to_nubus_board(dev));
+	return err;
+}
+
+struct bus_type nubus_bus_type = {
+	.name		= "nubus",
+	.match		= nubus_bus_match,
+	.probe		= nubus_device_probe,
+	.remove		= nubus_device_remove,
+};
+EXPORT_SYMBOL(nubus_bus_type);
+
+int nubus_driver_register(struct nubus_driver *ndrv)
+{
+	ndrv->driver.bus = &nubus_bus_type;
+	return driver_register(&ndrv->driver);
+}
+EXPORT_SYMBOL(nubus_driver_register);
+
+void nubus_driver_unregister(struct nubus_driver *ndrv)
+{
+	driver_unregister(&ndrv->driver);
+}
+EXPORT_SYMBOL(nubus_driver_unregister);
+
+static struct device nubus_parent = {
+	.init_name	= "nubus",
+};
+
+int __init nubus_bus_register(void)
+{
+	int err;
+
+	err = device_register(&nubus_parent);
+	if (err)
+		return err;
+
+	err = bus_register(&nubus_bus_type);
+	if (!err)
+		return 0;
+
+	device_unregister(&nubus_parent);
+	return err;
+}
+
+static void nubus_device_release(struct device *dev)
+{
+	struct nubus_board *board = to_nubus_board(dev);
+	struct nubus_rsrc *fres, *tmp;
+
+	list_for_each_entry_safe(fres, tmp, &nubus_func_rsrcs, list)
+		if (fres->board == board) {
+			list_del(&fres->list);
+			kfree(fres);
+		}
+	kfree(board);
+}
+
+int nubus_device_register(struct nubus_board *board)
+{
+	board->dev.parent = &nubus_parent;
+	board->dev.release = nubus_device_release;
+	board->dev.bus = &nubus_bus_type;
+	dev_set_name(&board->dev, "slot.%X", board->slot);
+	return device_register(&board->dev);
+}
+
+static int nubus_print_device_name_fn(struct device *dev, void *data)
+{
+	struct nubus_board *board = to_nubus_board(dev);
+	struct seq_file *m = data;
+
+	seq_printf(m, "Slot %X: %s\n", board->slot, board->name);
+	return 0;
+}
+
+int nubus_proc_show(struct seq_file *m, void *data)
+{
+	return bus_for_each_dev(&nubus_bus_type, NULL, m,
+				nubus_print_device_name_fn);
+}
diff --git a/drivers/nubus/nubus.c b/drivers/nubus/nubus.c
index b793727cd4f7..4621ff98138c 100644
--- a/drivers/nubus/nubus.c
+++ b/drivers/nubus/nubus.c
@@ -15,6 +15,7 @@
 #include <linux/errno.h>
 #include <linux/init.h>
 #include <linux/module.h>
+#include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <asm/setup.h>
 #include <asm/page.h>
@@ -31,8 +32,7 @@
 
 /* Globals */
 
-struct nubus_dev *nubus_devices;
-struct nubus_board *nubus_boards;
+LIST_HEAD(nubus_func_rsrcs);
 
 /* Meaning of "bytelanes":
 
@@ -146,7 +146,7 @@ static inline void *nubus_rom_addr(int slot)
 	return (void *)(0xF1000000 + (slot << 24));
 }
 
-static unsigned char *nubus_dirptr(const struct nubus_dirent *nd)
+unsigned char *nubus_dirptr(const struct nubus_dirent *nd)
 {
 	unsigned char *p = nd->base;
 
@@ -161,7 +161,7 @@ static unsigned char *nubus_dirptr(const struct nubus_dirent *nd)
    pointed to with offsets) out of the card ROM. */
 
 void nubus_get_rsrc_mem(void *dest, const struct nubus_dirent *dirent,
-			int len)
+			unsigned int len)
 {
 	unsigned char *t = (unsigned char *)dest;
 	unsigned char *p = nubus_dirptr(dirent);
@@ -173,21 +173,49 @@ void nubus_get_rsrc_mem(void *dest, const struct nubus_dirent *dirent,
 }
 EXPORT_SYMBOL(nubus_get_rsrc_mem);
 
-void nubus_get_rsrc_str(void *dest, const struct nubus_dirent *dirent,
-			int len)
+unsigned int nubus_get_rsrc_str(char *dest, const struct nubus_dirent *dirent,
+				unsigned int len)
 {
-	unsigned char *t = (unsigned char *)dest;
+	char *t = dest;
 	unsigned char *p = nubus_dirptr(dirent);
 
-	while (len) {
-		*t = nubus_get_rom(&p, 1, dirent->mask);
-		if (!*t++)
+	while (len > 1) {
+		unsigned char c = nubus_get_rom(&p, 1, dirent->mask);
+
+		if (!c)
 			break;
+		*t++ = c;
 		len--;
 	}
+	if (len > 0)
+		*t = '\0';
+	return t - dest;
 }
 EXPORT_SYMBOL(nubus_get_rsrc_str);
 
+void nubus_seq_write_rsrc_mem(struct seq_file *m,
+			      const struct nubus_dirent *dirent,
+			      unsigned int len)
+{
+	unsigned long buf[32];
+	unsigned int buf_size = sizeof(buf);
+	unsigned char *p = nubus_dirptr(dirent);
+
+	/* If possible, write out full buffers */
+	while (len >= buf_size) {
+		unsigned int i;
+
+		for (i = 0; i < ARRAY_SIZE(buf); i++)
+			buf[i] = nubus_get_rom(&p, sizeof(buf[0]),
+					       dirent->mask);
+		seq_write(m, buf, buf_size);
+		len -= buf_size;
+	}
+	/* If not, write out individual bytes */
+	while (len--)
+		seq_putc(m, nubus_get_rom(&p, 1, dirent->mask));
+}
+
 int nubus_get_root_dir(const struct nubus_board *board,
 		       struct nubus_dir *dir)
 {
@@ -199,12 +227,11 @@ int nubus_get_root_dir(const struct nubus_board *board,
 EXPORT_SYMBOL(nubus_get_root_dir);
 
 /* This is a slyly renamed version of the above */
-int nubus_get_func_dir(const struct nubus_dev *dev,
-		       struct nubus_dir *dir)
+int nubus_get_func_dir(const struct nubus_rsrc *fres, struct nubus_dir *dir)
 {
-	dir->ptr = dir->base = dev->directory;
+	dir->ptr = dir->base = fres->directory;
 	dir->done = 0;
-	dir->mask = dev->board->lanes;
+	dir->mask = fres->board->lanes;
 	return 0;
 }
 EXPORT_SYMBOL(nubus_get_func_dir);
@@ -277,51 +304,20 @@ EXPORT_SYMBOL(nubus_rewinddir);
 
 /* Driver interface functions, more or less like in pci.c */
 
-struct nubus_dev*
-nubus_find_device(unsigned short category, unsigned short type,
-		  unsigned short dr_hw, unsigned short dr_sw,
-		  const struct nubus_dev *from)
-{
-	struct nubus_dev *itor = from ? from->next : nubus_devices;
-
-	while (itor) {
-		if (itor->category == category && itor->type == type &&
-		    itor->dr_hw == dr_hw && itor->dr_sw == dr_sw)
-			return itor;
-		itor = itor->next;
-	}
-	return NULL;
-}
-EXPORT_SYMBOL(nubus_find_device);
-
-struct nubus_dev*
-nubus_find_type(unsigned short category, unsigned short type,
-		const struct nubus_dev *from)
+struct nubus_rsrc *nubus_first_rsrc_or_null(void)
 {
-	struct nubus_dev *itor = from ? from->next : nubus_devices;
-
-	while (itor) {
-		if (itor->category == category && itor->type == type)
-			return itor;
-		itor = itor->next;
-	}
-	return NULL;
+	return list_first_entry_or_null(&nubus_func_rsrcs, struct nubus_rsrc,
+					list);
 }
-EXPORT_SYMBOL(nubus_find_type);
+EXPORT_SYMBOL(nubus_first_rsrc_or_null);
 
-struct nubus_dev*
-nubus_find_slot(unsigned int slot, const struct nubus_dev *from)
+struct nubus_rsrc *nubus_next_rsrc_or_null(struct nubus_rsrc *from)
 {
-	struct nubus_dev *itor = from ? from->next : nubus_devices;
-
-	while (itor) {
-		if (itor->board->slot == slot)
-			return itor;
-		itor = itor->next;
-	}
-	return NULL;
+	if (list_is_last(&from->list, &nubus_func_rsrcs))
+		return NULL;
+	return list_next_entry(from, list);
 }
-EXPORT_SYMBOL(nubus_find_slot);
+EXPORT_SYMBOL(nubus_next_rsrc_or_null);
 
 int
 nubus_find_rsrc(struct nubus_dir *dir, unsigned char rsrc_type,
@@ -339,31 +335,83 @@ EXPORT_SYMBOL(nubus_find_rsrc);
    looking at, and print out lots and lots of information from the
    resource blocks. */
 
-/* FIXME: A lot of this stuff will eventually be useful after
-   initialization, for intelligently probing Ethernet and video chips,
-   among other things.  The rest of it should go in the /proc code.
-   For now, we just use it to give verbose boot logs. */
+static int __init nubus_get_block_rsrc_dir(struct nubus_board *board,
+					   struct proc_dir_entry *procdir,
+					   const struct nubus_dirent *parent)
+{
+	struct nubus_dir dir;
+	struct nubus_dirent ent;
+
+	nubus_get_subdir(parent, &dir);
+	dir.procdir = nubus_proc_add_rsrc_dir(procdir, parent, board);
 
-static int __init nubus_show_display_resource(struct nubus_dev *dev,
-					      const struct nubus_dirent *ent)
+	while (nubus_readdir(&dir, &ent) != -1) {
+		u32 size;
+
+		nubus_get_rsrc_mem(&size, &ent, 4);
+		pr_debug("        block (0x%x), size %d\n", ent.type, size);
+		nubus_proc_add_rsrc_mem(dir.procdir, &ent, size);
+	}
+	return 0;
+}
+
+static int __init nubus_get_display_vidmode(struct nubus_board *board,
+					    struct proc_dir_entry *procdir,
+					    const struct nubus_dirent *parent)
+{
+	struct nubus_dir dir;
+	struct nubus_dirent ent;
+
+	nubus_get_subdir(parent, &dir);
+	dir.procdir = nubus_proc_add_rsrc_dir(procdir, parent, board);
+
+	while (nubus_readdir(&dir, &ent) != -1) {
+		switch (ent.type) {
+		case 1: /* mVidParams */
+		case 2: /* mTable */
+		{
+			u32 size;
+
+			nubus_get_rsrc_mem(&size, &ent, 4);
+			pr_debug("        block (0x%x), size %d\n", ent.type,
+				size);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, size);
+			break;
+		}
+		default:
+			pr_debug("        unknown resource 0x%02x, data 0x%06x\n",
+				ent.type, ent.data);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, 0);
+		}
+	}
+	return 0;
+}
+
+static int __init nubus_get_display_resource(struct nubus_rsrc *fres,
+					     struct proc_dir_entry *procdir,
+					     const struct nubus_dirent *ent)
 {
 	switch (ent->type) {
 	case NUBUS_RESID_GAMMADIR:
-		pr_info("    gamma directory offset: 0x%06x\n", ent->data);
+		pr_debug("    gamma directory offset: 0x%06x\n", ent->data);
+		nubus_get_block_rsrc_dir(fres->board, procdir, ent);
 		break;
 	case 0x0080 ... 0x0085:
-		pr_info("    mode %02X info offset: 0x%06x\n",
-		       ent->type, ent->data);
+		pr_debug("    mode 0x%02x info offset: 0x%06x\n",
+			ent->type, ent->data);
+		nubus_get_display_vidmode(fres->board, procdir, ent);
 		break;
 	default:
-		pr_info("    unknown resource %02X, data 0x%06x\n",
-		       ent->type, ent->data);
+		pr_debug("    unknown resource 0x%02x, data 0x%06x\n",
+			ent->type, ent->data);
+		nubus_proc_add_rsrc_mem(procdir, ent, 0);
 	}
 	return 0;
 }
 
-static int __init nubus_show_network_resource(struct nubus_dev *dev,
-					      const struct nubus_dirent *ent)
+static int __init nubus_get_network_resource(struct nubus_rsrc *fres,
+					     struct proc_dir_entry *procdir,
+					     const struct nubus_dirent *ent)
 {
 	switch (ent->type) {
 	case NUBUS_RESID_MAC_ADDRESS:
@@ -371,18 +419,21 @@ static int __init nubus_show_network_resource(struct nubus_dev *dev,
 		char addr[6];
 
 		nubus_get_rsrc_mem(addr, ent, 6);
-		pr_info("    MAC address: %pM\n", addr);
+		pr_debug("    MAC address: %pM\n", addr);
+		nubus_proc_add_rsrc_mem(procdir, ent, 6);
 		break;
 	}
 	default:
-		pr_info("    unknown resource %02X, data 0x%06x\n",
-		       ent->type, ent->data);
+		pr_debug("    unknown resource 0x%02x, data 0x%06x\n",
+			ent->type, ent->data);
+		nubus_proc_add_rsrc_mem(procdir, ent, 0);
 	}
 	return 0;
 }
 
-static int __init nubus_show_cpu_resource(struct nubus_dev *dev,
-					  const struct nubus_dirent *ent)
+static int __init nubus_get_cpu_resource(struct nubus_rsrc *fres,
+					 struct proc_dir_entry *procdir,
+					 const struct nubus_dirent *ent)
 {
 	switch (ent->type) {
 	case NUBUS_RESID_MEMINFO:
@@ -390,8 +441,9 @@ static int __init nubus_show_cpu_resource(struct nubus_dev *dev,
 		unsigned long meminfo[2];
 
 		nubus_get_rsrc_mem(&meminfo, ent, 8);
-		pr_info("    memory: [ 0x%08lx 0x%08lx ]\n",
-		       meminfo[0], meminfo[1]);
+		pr_debug("    memory: [ 0x%08lx 0x%08lx ]\n",
+			meminfo[0], meminfo[1]);
+		nubus_proc_add_rsrc_mem(procdir, ent, 8);
 		break;
 	}
 	case NUBUS_RESID_ROMINFO:
@@ -399,57 +451,60 @@ static int __init nubus_show_cpu_resource(struct nubus_dev *dev,
 		unsigned long rominfo[2];
 
 		nubus_get_rsrc_mem(&rominfo, ent, 8);
-		pr_info("    ROM:    [ 0x%08lx 0x%08lx ]\n",
-		       rominfo[0], rominfo[1]);
+		pr_debug("    ROM:    [ 0x%08lx 0x%08lx ]\n",
+			rominfo[0], rominfo[1]);
+		nubus_proc_add_rsrc_mem(procdir, ent, 8);
 		break;
 	}
 	default:
-		pr_info("    unknown resource %02X, data 0x%06x\n",
-		       ent->type, ent->data);
+		pr_debug("    unknown resource 0x%02x, data 0x%06x\n",
+			ent->type, ent->data);
+		nubus_proc_add_rsrc_mem(procdir, ent, 0);
 	}
 	return 0;
 }
 
-static int __init nubus_show_private_resource(struct nubus_dev *dev,
-					      const struct nubus_dirent *ent)
+static int __init nubus_get_private_resource(struct nubus_rsrc *fres,
+					     struct proc_dir_entry *procdir,
+					     const struct nubus_dirent *ent)
 {
-	switch (dev->category) {
+	switch (fres->category) {
 	case NUBUS_CAT_DISPLAY:
-		nubus_show_display_resource(dev, ent);
+		nubus_get_display_resource(fres, procdir, ent);
 		break;
 	case NUBUS_CAT_NETWORK:
-		nubus_show_network_resource(dev, ent);
+		nubus_get_network_resource(fres, procdir, ent);
 		break;
 	case NUBUS_CAT_CPU:
-		nubus_show_cpu_resource(dev, ent);
+		nubus_get_cpu_resource(fres, procdir, ent);
 		break;
 	default:
-		pr_info("    unknown resource %02X, data 0x%06x\n",
-		       ent->type, ent->data);
+		pr_debug("    unknown resource 0x%02x, data 0x%06x\n",
+			ent->type, ent->data);
+		nubus_proc_add_rsrc_mem(procdir, ent, 0);
 	}
 	return 0;
 }
 
-static struct nubus_dev * __init
+static struct nubus_rsrc * __init
 nubus_get_functional_resource(struct nubus_board *board, int slot,
 			      const struct nubus_dirent *parent)
 {
 	struct nubus_dir dir;
 	struct nubus_dirent ent;
-	struct nubus_dev *dev;
+	struct nubus_rsrc *fres;
 
-	pr_info("  Function 0x%02x:\n", parent->type);
+	pr_debug("  Functional resource 0x%02x:\n", parent->type);
 	nubus_get_subdir(parent, &dir);
-
-	pr_debug("%s: parent is 0x%p, dir is 0x%p\n",
-	         __func__, parent->base, dir.base);
+	dir.procdir = nubus_proc_add_rsrc_dir(board->procdir, parent, board);
 
 	/* Actually we should probably panic if this fails */
-	if ((dev = kzalloc(sizeof(*dev), GFP_ATOMIC)) == NULL)
+	fres = kzalloc(sizeof(*fres), GFP_ATOMIC);
+	if (!fres)
 		return NULL;
-	dev->resid = parent->type;
-	dev->directory = dir.base;
-	dev->board = board;
+	fres->resid = parent->type;
+	fres->directory = dir.base;
+	fres->board = board;
 
 	while (nubus_readdir(&dir, &ent) != -1) {
 		switch (ent.type) {
@@ -458,130 +513,96 @@ nubus_get_functional_resource(struct nubus_board *board, int slot,
 			unsigned short nbtdata[4];
 
 			nubus_get_rsrc_mem(nbtdata, &ent, 8);
-			dev->category = nbtdata[0];
-			dev->type     = nbtdata[1];
-			dev->dr_sw    = nbtdata[2];
-			dev->dr_hw    = nbtdata[3];
-			pr_info("    type: [cat 0x%x type 0x%x sw 0x%x hw 0x%x]\n",
-			        nbtdata[0], nbtdata[1], nbtdata[2], nbtdata[3]);
+			fres->category = nbtdata[0];
+			fres->type     = nbtdata[1];
+			fres->dr_sw    = nbtdata[2];
+			fres->dr_hw    = nbtdata[3];
+			pr_debug("    type: [cat 0x%x type 0x%x sw 0x%x hw 0x%x]\n",
+				nbtdata[0], nbtdata[1], nbtdata[2], nbtdata[3]);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, 8);
 			break;
 		}
 		case NUBUS_RESID_NAME:
 		{
-			nubus_get_rsrc_str(dev->name, &ent, 64);
-			pr_info("    name: %s\n", dev->name);
+			char name[64];
+			unsigned int len;
+
+			len = nubus_get_rsrc_str(name, &ent, sizeof(name));
+			pr_debug("    name: %s\n", name);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, len + 1);
 			break;
 		}
 		case NUBUS_RESID_DRVRDIR:
 		{
 			/* MacOS driver.  If we were NetBSD we might
 			   use this :-) */
-			struct nubus_dir drvr_dir;
-			struct nubus_dirent drvr_ent;
-
-			nubus_get_subdir(&ent, &drvr_dir);
-			nubus_readdir(&drvr_dir, &drvr_ent);
-			dev->driver = nubus_dirptr(&drvr_ent);
-			pr_info("    driver at: 0x%p\n", dev->driver);
+			pr_debug("    driver directory offset: 0x%06x\n",
+				ent.data);
+			nubus_get_block_rsrc_dir(board, dir.procdir, &ent);
 			break;
 		}
 		case NUBUS_RESID_MINOR_BASEOS:
+		{
 			/* We will need this in order to support
 			   multiple framebuffers.  It might be handy
 			   for Ethernet as well */
-			nubus_get_rsrc_mem(&dev->iobase, &ent, 4);
-			pr_info("    memory offset: 0x%08lx\n", dev->iobase);
+			u32 base_offset;
+
+			nubus_get_rsrc_mem(&base_offset, &ent, 4);
+			pr_debug("    memory offset: 0x%08x\n", base_offset);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, 4);
 			break;
+		}
 		case NUBUS_RESID_MINOR_LENGTH:
+		{
 			/* Ditto */
-			nubus_get_rsrc_mem(&dev->iosize, &ent, 4);
-			pr_info("    memory length: 0x%08lx\n", dev->iosize);
+			u32 length;
+
+			nubus_get_rsrc_mem(&length, &ent, 4);
+			pr_debug("    memory length: 0x%08x\n", length);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, 4);
 			break;
+		}
 		case NUBUS_RESID_FLAGS:
-			dev->flags = ent.data;
-			pr_info("    flags: 0x%06x\n", dev->flags);
+			pr_debug("    flags: 0x%06x\n", ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 		case NUBUS_RESID_HWDEVID:
-			dev->hwdevid = ent.data;
-			pr_info("    hwdevid: 0x%06x\n", dev->hwdevid);
+			pr_debug("    hwdevid: 0x%06x\n", ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 		default:
 			/* Local/Private resources have their own
 			   function */
-			nubus_show_private_resource(dev, &ent);
+			nubus_get_private_resource(fres, dir.procdir, &ent);
 		}
 	}
 
-	return dev;
-}
-
-/* This is cool. */
-static int __init nubus_get_vidnames(struct nubus_board *board,
-				     const struct nubus_dirent *parent)
-{
-	struct nubus_dir dir;
-	struct nubus_dirent ent;
-
-	/* FIXME: obviously we want to put this in a header file soon */
-	struct vidmode {
-		u32 size;
-		/* Don't know what this is yet */
-		u16 id;
-		/* Longest one I've seen so far is 26 characters */
-		char name[32];
-	};
-
-	pr_info("    video modes supported:\n");
-	nubus_get_subdir(parent, &dir);
-	pr_debug("%s: parent is 0x%p, dir is 0x%p\n",
-	         __func__, parent->base, dir.base);
-
-	while (nubus_readdir(&dir, &ent) != -1) {
-		struct vidmode mode;
-		u32 size;
-
-		/* First get the length */
-		nubus_get_rsrc_mem(&size, &ent, 4);
-
-		/* Now clobber the whole thing */
-		if (size > sizeof(mode) - 1)
-			size = sizeof(mode) - 1;
-		memset(&mode, 0, sizeof(mode));
-		nubus_get_rsrc_mem(&mode, &ent, size);
-		pr_info("      %02X: (%02X) %s\n", ent.type,
-			mode.id, mode.name);
-	}
-	return 0;
+	return fres;
 }
 
 /* This is *really* cool. */
 static int __init nubus_get_icon(struct nubus_board *board,
+				 struct proc_dir_entry *procdir,
 				 const struct nubus_dirent *ent)
 {
 	/* Should be 32x32 if my memory serves me correctly */
-	unsigned char icon[128];
-	int x, y;
+	u32 icon[32];
+	int i;
 
 	nubus_get_rsrc_mem(&icon, ent, 128);
-	pr_info("    icon:\n");
-
-	/* We should actually plot these somewhere in the framebuffer
-	   init.  This is just to demonstrate that they do, in fact,
-	   exist */
-	for (y = 0; y < 32; y++) {
-		pr_info("      ");
-		for (x = 0; x < 32; x++) {
-			if (icon[y * 4 + x / 8] & (0x80 >> (x % 8)))
-				pr_cont("*");
-			else
-				pr_cont(" ");
-		}
-		pr_cont("\n");
-	}
+	pr_debug("    icon:\n");
+	for (i = 0; i < 8; i++)
+		pr_debug("        %08x %08x %08x %08x\n",
+			icon[i * 4 + 0], icon[i * 4 + 1],
+			icon[i * 4 + 2], icon[i * 4 + 3]);
+	nubus_proc_add_rsrc_mem(procdir, ent, 128);
+
 	return 0;
 }
 
 static int __init nubus_get_vendorinfo(struct nubus_board *board,
+				       struct proc_dir_entry *procdir,
 				       const struct nubus_dirent *parent)
 {
 	struct nubus_dir dir;
@@ -589,19 +610,20 @@ static int __init nubus_get_vendorinfo(struct nubus_board *board,
 	static char *vendor_fields[6] = { "ID", "serial", "revision",
 	                                  "part", "date", "unknown field" };
 
-	pr_info("    vendor info:\n");
+	pr_debug("    vendor info:\n");
 	nubus_get_subdir(parent, &dir);
-	pr_debug("%s: parent is 0x%p, dir is 0x%p\n",
-	         __func__, parent->base, dir.base);
+	dir.procdir = nubus_proc_add_rsrc_dir(procdir, parent, board);
 
 	while (nubus_readdir(&dir, &ent) != -1) {
 		char name[64];
+		unsigned int len;
 
 		/* These are all strings, we think */
-		nubus_get_rsrc_str(name, &ent, 64);
-		if (ent.type > 5)
+		len = nubus_get_rsrc_str(name, &ent, sizeof(name));
+		if (ent.type < 1 || ent.type > 5)
 			ent.type = 5;
-		pr_info("    %s: %s\n", vendor_fields[ent.type - 1], name);
+		pr_debug("    %s: %s\n", vendor_fields[ent.type - 1], name);
+		nubus_proc_add_rsrc_mem(dir.procdir, &ent, len + 1);
 	}
 	return 0;
 }
@@ -612,9 +634,9 @@ static int __init nubus_get_board_resource(struct nubus_board *board, int slot,
 	struct nubus_dir dir;
 	struct nubus_dirent ent;
 
+	pr_debug("  Board resource 0x%02x:\n", parent->type);
 	nubus_get_subdir(parent, &dir);
-	pr_debug("%s: parent is 0x%p, dir is 0x%p\n",
-	         __func__, parent->base, dir.base);
+	dir.procdir = nubus_proc_add_rsrc_dir(board->procdir, parent, board);
 
 	while (nubus_readdir(&dir, &ent) != -1) {
 		switch (ent.type) {
@@ -625,64 +647,81 @@ static int __init nubus_get_board_resource(struct nubus_board *board, int slot,
 			   useful except insofar as it tells us that
 			   we really are looking at a board resource. */
 			nubus_get_rsrc_mem(nbtdata, &ent, 8);
-			pr_info("    type: [cat 0x%x type 0x%x sw 0x%x hw 0x%x]\n",
-			        nbtdata[0], nbtdata[1], nbtdata[2], nbtdata[3]);
+			pr_debug("    type: [cat 0x%x type 0x%x sw 0x%x hw 0x%x]\n",
+				nbtdata[0], nbtdata[1], nbtdata[2], nbtdata[3]);
 			if (nbtdata[0] != 1 || nbtdata[1] != 0 ||
 			    nbtdata[2] != 0 || nbtdata[3] != 0)
-				pr_err("this sResource is not a board resource!\n");
+				pr_err("Slot %X: sResource is not a board resource!\n",
+				       slot);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, 8);
 			break;
 		}
 		case NUBUS_RESID_NAME:
-			nubus_get_rsrc_str(board->name, &ent, 64);
-			pr_info("    name: %s\n", board->name);
+		{
+			unsigned int len;
+
+			len = nubus_get_rsrc_str(board->name, &ent,
+						 sizeof(board->name));
+			pr_debug("    name: %s\n", board->name);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, len + 1);
 			break;
+		}
 		case NUBUS_RESID_ICON:
-			nubus_get_icon(board, &ent);
+			nubus_get_icon(board, dir.procdir, &ent);
 			break;
 		case NUBUS_RESID_BOARDID:
-			pr_info("    board id: 0x%x\n", ent.data);
+			pr_debug("    board id: 0x%x\n", ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 		case NUBUS_RESID_PRIMARYINIT:
-			pr_info("    primary init offset: 0x%06x\n", ent.data);
+			pr_debug("    primary init offset: 0x%06x\n", ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 		case NUBUS_RESID_VENDORINFO:
-			nubus_get_vendorinfo(board, &ent);
+			nubus_get_vendorinfo(board, dir.procdir, &ent);
 			break;
 		case NUBUS_RESID_FLAGS:
-			pr_info("    flags: 0x%06x\n", ent.data);
+			pr_debug("    flags: 0x%06x\n", ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 		case NUBUS_RESID_HWDEVID:
-			pr_info("    hwdevid: 0x%06x\n", ent.data);
+			pr_debug("    hwdevid: 0x%06x\n", ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 		case NUBUS_RESID_SECONDINIT:
-			pr_info("    secondary init offset: 0x%06x\n", ent.data);
+			pr_debug("    secondary init offset: 0x%06x\n",
+				 ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 			/* WTF isn't this in the functional resources? */
 		case NUBUS_RESID_VIDNAMES:
-			nubus_get_vidnames(board, &ent);
+			pr_debug("    vidnames directory offset: 0x%06x\n",
+				ent.data);
+			nubus_get_block_rsrc_dir(board, dir.procdir, &ent);
 			break;
 			/* Same goes for this */
 		case NUBUS_RESID_VIDMODES:
-			pr_info("    video mode parameter directory offset: 0x%06x\n",
-			       ent.data);
+			pr_debug("    video mode parameter directory offset: 0x%06x\n",
+				ent.data);
+			nubus_proc_add_rsrc(dir.procdir, &ent);
 			break;
 		default:
-			pr_info("    unknown resource %02X, data 0x%06x\n",
-			       ent.type, ent.data);
+			pr_debug("    unknown resource 0x%02x, data 0x%06x\n",
+				ent.type, ent.data);
+			nubus_proc_add_rsrc_mem(dir.procdir, &ent, 0);
 		}
 	}
 	return 0;
 }
 
-/* Add a board (might be many devices) to the list */
-static struct nubus_board * __init nubus_add_board(int slot, int bytelanes)
+static void __init nubus_add_board(int slot, int bytelanes)
 {
 	struct nubus_board *board;
-	struct nubus_board **boardp;
 	unsigned char *rp;
 	unsigned long dpat;
 	struct nubus_dir dir;
 	struct nubus_dirent ent;
+	int prev_resid = -1;
 
 	/* Move to the start of the format block */
 	rp = nubus_rom_addr(slot);
@@ -690,19 +729,19 @@ static struct nubus_board * __init nubus_add_board(int slot, int bytelanes)
 
 	/* Actually we should probably panic if this fails */
 	if ((board = kzalloc(sizeof(*board), GFP_ATOMIC)) == NULL)
-		return NULL;
+		return;
 	board->fblock = rp;
 
 	/* Dump the format block for debugging purposes */
 	pr_debug("Slot %X, format block at 0x%p:\n", slot, rp);
+	pr_debug("%08lx\n", nubus_get_rom(&rp, 4, bytelanes));
+	pr_debug("%08lx\n", nubus_get_rom(&rp, 4, bytelanes));
+	pr_debug("%08lx\n", nubus_get_rom(&rp, 4, bytelanes));
 	pr_debug("%02lx\n", nubus_get_rom(&rp, 1, bytelanes));
 	pr_debug("%02lx\n", nubus_get_rom(&rp, 1, bytelanes));
 	pr_debug("%08lx\n", nubus_get_rom(&rp, 4, bytelanes));
 	pr_debug("%02lx\n", nubus_get_rom(&rp, 1, bytelanes));
 	pr_debug("%02lx\n", nubus_get_rom(&rp, 1, bytelanes));
-	pr_debug("%08lx\n", nubus_get_rom(&rp, 4, bytelanes));
-	pr_debug("%08lx\n", nubus_get_rom(&rp, 4, bytelanes));
-	pr_debug("%08lx\n", nubus_get_rom(&rp, 4, bytelanes));
 	rp = board->fblock;
 
 	board->slot = slot;
@@ -722,10 +761,10 @@ static struct nubus_board * __init nubus_add_board(int slot, int bytelanes)
 
 	/* Directory offset should be small and negative... */
 	if (!(board->doffset & 0x00FF0000))
-		pr_warn("Dodgy doffset!\n");
+		pr_warn("Slot %X: Dodgy doffset!\n", slot);
 	dpat = nubus_get_rom(&rp, 4, bytelanes);
 	if (dpat != NUBUS_TEST_PATTERN)
-		pr_warn("Wrong test pattern %08lx!\n", dpat);
+		pr_warn("Slot %X: Wrong test pattern %08lx!\n", slot, dpat);
 
 	/*
 	 *	I wonder how the CRC is meant to work -
@@ -742,53 +781,52 @@ static struct nubus_board * __init nubus_add_board(int slot, int bytelanes)
 	nubus_get_root_dir(board, &dir);
 
 	/* We're ready to rock */
-	pr_info("Slot %X:\n", slot);
+	pr_debug("Slot %X resources:\n", slot);
 
 	/* Each slot should have one board resource and any number of
-	   functional resources.  So we'll fill in some fields in the
-	   struct nubus_board from the board resource, then walk down
-	   the list of functional resources, spinning out a nubus_dev
-	   for each of them. */
+	 * functional resources.  So we'll fill in some fields in the
+	 * struct nubus_board from the board resource, then walk down
+	 * the list of functional resources, spinning out a nubus_rsrc
+	 * for each of them.
+	 */
 	if (nubus_readdir(&dir, &ent) == -1) {
 		/* We can't have this! */
-		pr_err("Board resource not found!\n");
-		return NULL;
-	} else {
-		pr_info("  Board resource:\n");
-		nubus_get_board_resource(board, slot, &ent);
+		pr_err("Slot %X: Board resource not found!\n", slot);
+		kfree(board);
+		return;
 	}
 
+	if (ent.type < 1 || ent.type > 127)
+		pr_warn("Slot %X: Board resource ID is invalid!\n", slot);
+
+	board->procdir = nubus_proc_add_board(board);
+
+	nubus_get_board_resource(board, slot, &ent);
+
 	while (nubus_readdir(&dir, &ent) != -1) {
-		struct nubus_dev *dev;
-		struct nubus_dev **devp;
+		struct nubus_rsrc *fres;
 
-		dev = nubus_get_functional_resource(board, slot, &ent);
-		if (dev == NULL)
+		fres = nubus_get_functional_resource(board, slot, &ent);
+		if (fres == NULL)
 			continue;
 
-		/* We zeroed this out above */
-		if (board->first_dev == NULL)
-			board->first_dev = dev;
+		/* Resources should appear in ascending ID order. This sanity
+		 * check prevents duplicate resource IDs.
+		 */
+		if (fres->resid <= prev_resid) {
+			kfree(fres);
+			continue;
+		}
+		prev_resid = fres->resid;
 
-		/* Put it on the global NuBus device chain. Keep entries in order. */
-		for (devp = &nubus_devices; *devp != NULL;
-		     devp = &((*devp)->next))
-			/* spin */;
-		*devp = dev;
-		dev->next = NULL;
+		list_add_tail(&fres->list, &nubus_func_rsrcs);
 	}
 
-	/* Put it on the global NuBus board chain. Keep entries in order. */
-	for (boardp = &nubus_boards; *boardp != NULL;
-	     boardp = &((*boardp)->next))
-		/* spin */;
-	*boardp = board;
-	board->next = NULL;
-
-	return board;
+	if (nubus_device_register(board))
+		put_device(&board->dev);
 }
 
-void __init nubus_probe_slot(int slot)
+static void __init nubus_probe_slot(int slot)
 {
 	unsigned char dp;
 	unsigned char *rp;
@@ -796,11 +834,8 @@ void __init nubus_probe_slot(int slot)
 
 	rp = nubus_rom_addr(slot);
 	for (i = 4; i; i--) {
-		int card_present;
-
 		rp--;
-		card_present = hwreg_present(rp);
-		if (!card_present)
+		if (!hwreg_present(rp))
 			continue;
 
 		dp = *rp;
@@ -822,10 +857,11 @@ void __init nubus_probe_slot(int slot)
 	}
 }
 
-void __init nubus_scan_bus(void)
+static void __init nubus_scan_bus(void)
 {
 	int slot;
 
+	pr_info("NuBus: Scanning NuBus slots.\n");
 	for (slot = 9; slot < 15; slot++) {
 		nubus_probe_slot(slot);
 	}
@@ -833,14 +869,16 @@ void __init nubus_scan_bus(void)
 
 static int __init nubus_init(void)
 {
+	int err;
+
 	if (!MACH_IS_MAC)
 		return 0;
 
-	pr_info("NuBus: Scanning NuBus slots.\n");
-	nubus_devices = NULL;
-	nubus_boards = NULL;
-	nubus_scan_bus();
 	nubus_proc_init();
+	err = nubus_bus_register();
+	if (err)
+		return err;
+	nubus_scan_bus();
 	return 0;
 }
 
diff --git a/drivers/nubus/proc.c b/drivers/nubus/proc.c
index 004a122ac0ff..c2e5a7e6bd3e 100644
--- a/drivers/nubus/proc.c
+++ b/drivers/nubus/proc.c
@@ -11,39 +11,37 @@
    structure in /proc analogous to the structure of the NuBus ROM
    resources.
 
-   Therefore each NuBus device is in fact a directory, which may in
-   turn contain subdirectories.  The "files" correspond to NuBus
-   resource records.  For those types of records which we know how to
-   convert to formats that are meaningful to userspace (mostly just
-   icons) these files will provide "cooked" data.  Otherwise they will
-   simply provide raw access (read-only of course) to the ROM.  */
+   Therefore each board function gets a directory, which may in turn
+   contain subdirectories.  Each slot resource is a file.  Unrecognized
+   resources are empty files, since every resource ID requires a special
+   case (e.g. if the resource ID implies a directory or block, then its
+   value has to be interpreted as a slot ROM pointer etc.).
+ */
 
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/nubus.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
+#include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/module.h>
-
 #include <linux/uaccess.h>
 #include <asm/byteorder.h>
 
+/*
+ * /proc/bus/nubus/devices stuff
+ */
+
 static int
 nubus_devices_proc_show(struct seq_file *m, void *v)
 {
-	struct nubus_dev *dev = nubus_devices;
-
-	while (dev) {
-		seq_printf(m, "%x\t%04x %04x %04x %04x",
-			      dev->board->slot,
-			      dev->category,
-			      dev->type,
-			      dev->dr_sw,
-			      dev->dr_hw);
-		seq_printf(m, "\t%08lx\n", dev->board->slot_addr);
-		dev = dev->next;
-	}
+	struct nubus_rsrc *fres;
+
+	for_each_func_rsrc(fres)
+		seq_printf(m, "%x\t%04x %04x %04x %04x\t%08lx\n",
+			   fres->board->slot, fres->category, fres->type,
+			   fres->dr_sw, fres->dr_hw, fres->board->slot_addr);
 	return 0;
 }
 
@@ -61,174 +59,163 @@ static const struct file_operations nubus_devices_proc_fops = {
 
 static struct proc_dir_entry *proc_bus_nubus_dir;
 
-static const struct file_operations nubus_proc_subdir_fops = {
-#warning Need to set some I/O handlers here
-};
+/*
+ * /proc/bus/nubus/x/ stuff
+ */
 
-static void nubus_proc_subdir(struct nubus_dev* dev,
-			      struct proc_dir_entry* parent,
-			      struct nubus_dir* dir)
+struct proc_dir_entry *nubus_proc_add_board(struct nubus_board *board)
 {
-	struct nubus_dirent ent;
-
-	/* Some of these are directories, others aren't */
-	while (nubus_readdir(dir, &ent) != -1) {
-		char name[8];
-		struct proc_dir_entry* e;
-		
-		sprintf(name, "%x", ent.type);
-		e = proc_create(name, S_IFREG | S_IRUGO | S_IWUSR, parent,
-				&nubus_proc_subdir_fops);
-		if (!e)
-			return;
-	}
+	char name[2];
+
+	if (!proc_bus_nubus_dir)
+		return NULL;
+	snprintf(name, sizeof(name), "%x", board->slot);
+	return proc_mkdir(name, proc_bus_nubus_dir);
 }
 
-/* Can't do this recursively since the root directory is structured
-   somewhat differently from the subdirectories */
-static void nubus_proc_populate(struct nubus_dev* dev,
-				struct proc_dir_entry* parent,
-				struct nubus_dir* root)
+/* The PDE private data for any directory under /proc/bus/nubus/x/
+ * is the bytelanes value for the board in slot x.
+ */
+
+struct proc_dir_entry *nubus_proc_add_rsrc_dir(struct proc_dir_entry *procdir,
+					       const struct nubus_dirent *ent,
+					       struct nubus_board *board)
 {
-	struct nubus_dirent ent;
-
-	/* We know these are all directories (board resource + one or
-	   more functional resources) */
-	while (nubus_readdir(root, &ent) != -1) {
-		char name[8];
-		struct proc_dir_entry* e;
-		struct nubus_dir dir;
-		
-		sprintf(name, "%x", ent.type);
-		e = proc_mkdir(name, parent);
-		if (!e) return;
-
-		/* And descend */
-		if (nubus_get_subdir(&ent, &dir) == -1) {
-			/* This shouldn't happen */
-			printk(KERN_ERR "NuBus root directory node %x:%x has no subdir!\n",
-			       dev->board->slot, ent.type);
-			continue;
-		} else {
-			nubus_proc_subdir(dev, e, &dir);
-		}
-	}
+	char name[9];
+	int lanes = board->lanes;
+
+	if (!procdir)
+		return NULL;
+	snprintf(name, sizeof(name), "%x", ent->type);
+	return proc_mkdir_data(name, 0555, procdir, (void *)lanes);
 }
 
-int nubus_proc_attach_device(struct nubus_dev *dev)
+/* The PDE private data for a file under /proc/bus/nubus/x/ is a pointer to
+ * an instance of the following structure, which gives the location and size
+ * of the resource data in the slot ROM. For slot resources which hold only a
+ * small integer, this integer value is stored directly and size is set to 0.
+ * A NULL private data pointer indicates an unrecognized resource.
+ */
+
+struct nubus_proc_pde_data {
+	unsigned char *res_ptr;
+	unsigned int res_size;
+};
+
+static struct nubus_proc_pde_data *
+nubus_proc_alloc_pde_data(unsigned char *ptr, unsigned int size)
 {
-	struct proc_dir_entry *e;
-	struct nubus_dir root;
-	char name[8];
-
-	if (dev == NULL) {
-		printk(KERN_ERR
-		       "NULL pointer in nubus_proc_attach_device, shoot the programmer!\n");
-		return -1;
-	}
-		
-	if (dev->board == NULL) {
-		printk(KERN_ERR
-		       "NULL pointer in nubus_proc_attach_device, shoot the programmer!\n");
-		printk("dev = %p, dev->board = %p\n", dev, dev->board);
-		return -1;
-	}
-		
-	/* Create a directory */
-	sprintf(name, "%x", dev->board->slot);
-	e = dev->procdir = proc_mkdir(name, proc_bus_nubus_dir);
-	if (!e)
-		return -ENOMEM;
+	struct nubus_proc_pde_data *pde_data;
 
-	/* Now recursively populate it with files */
-	nubus_get_root_dir(dev->board, &root);
-	nubus_proc_populate(dev, e, &root);
+	pde_data = kmalloc(sizeof(*pde_data), GFP_KERNEL);
+	if (!pde_data)
+		return NULL;
 
-	return 0;
+	pde_data->res_ptr = ptr;
+	pde_data->res_size = size;
+	return pde_data;
 }
-EXPORT_SYMBOL(nubus_proc_attach_device);
 
-/*
- * /proc/nubus stuff
- */
-static int nubus_proc_show(struct seq_file *m, void *v)
+static int nubus_proc_rsrc_show(struct seq_file *m, void *v)
 {
-	const struct nubus_board *board = v;
+	struct inode *inode = m->private;
+	struct nubus_proc_pde_data *pde_data;
 
-	/* Display header on line 1 */
-	if (v == SEQ_START_TOKEN)
-		seq_puts(m, "Nubus devices found:\n");
-	else
-		seq_printf(m, "Slot %X: %s\n", board->slot, board->name);
+	pde_data = PDE_DATA(inode);
+	if (!pde_data)
+		return 0;
+
+	if (pde_data->res_size > m->size)
+		return -EFBIG;
+
+	if (pde_data->res_size) {
+		int lanes = (int)proc_get_parent_data(inode);
+		struct nubus_dirent ent;
+
+		if (!lanes)
+			return 0;
+
+		ent.mask = lanes;
+		ent.base = pde_data->res_ptr;
+		ent.data = 0;
+		nubus_seq_write_rsrc_mem(m, &ent, pde_data->res_size);
+	} else {
+		unsigned int data = (unsigned int)pde_data->res_ptr;
+
+		seq_putc(m, data >> 16);
+		seq_putc(m, data >> 8);
+		seq_putc(m, data >> 0);
+	}
 	return 0;
 }
 
-static void *nubus_proc_start(struct seq_file *m, loff_t *_pos)
+static int nubus_proc_rsrc_open(struct inode *inode, struct file *file)
 {
-	struct nubus_board *board;
-	unsigned pos;
-
-	if (*_pos > LONG_MAX)
-		return NULL;
-	pos = *_pos;
-	if (pos == 0)
-		return SEQ_START_TOKEN;
-	for (board = nubus_boards; board; board = board->next)
-		if (--pos == 0)
-			break;
-	return board;
+	return single_open(file, nubus_proc_rsrc_show, inode);
 }
 
-static void *nubus_proc_next(struct seq_file *p, void *v, loff_t *_pos)
+static const struct file_operations nubus_proc_rsrc_fops = {
+	.open		= nubus_proc_rsrc_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+void nubus_proc_add_rsrc_mem(struct proc_dir_entry *procdir,
+			     const struct nubus_dirent *ent,
+			     unsigned int size)
 {
-	/* Walk the list of NuBus boards */
-	struct nubus_board *board = v;
-
-	++*_pos;
-	if (v == SEQ_START_TOKEN)
-		board = nubus_boards;
-	else if (board)
-		board = board->next;
-	return board;
+	char name[9];
+	struct nubus_proc_pde_data *pde_data;
+
+	if (!procdir)
+		return;
+
+	snprintf(name, sizeof(name), "%x", ent->type);
+	if (size)
+		pde_data = nubus_proc_alloc_pde_data(nubus_dirptr(ent), size);
+	else
+		pde_data = NULL;
+	proc_create_data(name, S_IFREG | 0444, procdir,
+			 &nubus_proc_rsrc_fops, pde_data);
 }
 
-static void nubus_proc_stop(struct seq_file *p, void *v)
+void nubus_proc_add_rsrc(struct proc_dir_entry *procdir,
+			 const struct nubus_dirent *ent)
 {
+	char name[9];
+	unsigned char *data = (unsigned char *)ent->data;
+
+	if (!procdir)
+		return;
+
+	snprintf(name, sizeof(name), "%x", ent->type);
+	proc_create_data(name, S_IFREG | 0444, procdir,
+			 &nubus_proc_rsrc_fops,
+			 nubus_proc_alloc_pde_data(data, 0));
 }
 
-static const struct seq_operations nubus_proc_seqops = {
-	.start	= nubus_proc_start,
-	.next	= nubus_proc_next,
-	.stop	= nubus_proc_stop,
-	.show	= nubus_proc_show,
-};
+/*
+ * /proc/nubus stuff
+ */
 
 static int nubus_proc_open(struct inode *inode, struct file *file)
 {
-	return seq_open(file, &nubus_proc_seqops);
+	return single_open(file, nubus_proc_show, NULL);
 }
 
 static const struct file_operations nubus_proc_fops = {
 	.open		= nubus_proc_open,
 	.read		= seq_read,
 	.llseek		= seq_lseek,
-	.release	= seq_release,
+	.release	= single_release,
 };
 
-void __init proc_bus_nubus_add_devices(void)
-{
-	struct nubus_dev *dev;
-	
-	for(dev = nubus_devices; dev; dev = dev->next)
-		nubus_proc_attach_device(dev);
-}
-
 void __init nubus_proc_init(void)
 {
 	proc_create("nubus", 0, NULL, &nubus_proc_fops);
-	if (!MACH_IS_MAC)
-		return;
 	proc_bus_nubus_dir = proc_mkdir("bus/nubus", NULL);
+	if (!proc_bus_nubus_dir)
+		return;
 	proc_create("devices", 0, proc_bus_nubus_dir, &nubus_devices_proc_fops);
-	proc_bus_nubus_add_devices();
 }
diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c
index 345acca576b3..1bd7b3734751 100644
--- a/drivers/nvdimm/blk.c
+++ b/drivers/nvdimm/blk.c
@@ -278,8 +278,6 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk)
 	disk->queue		= q;
 	disk->flags		= GENHD_FL_EXT_DEVT;
 	nvdimm_namespace_disk_name(&nsblk->common, disk->disk_name);
-	set_capacity(disk, 0);
-	device_add_disk(dev, disk);
 
 	if (devm_add_action_or_reset(dev, nd_blk_release_disk, disk))
 		return -ENOMEM;
@@ -292,6 +290,7 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk)
 	}
 
 	set_capacity(disk, available_disk_size >> SECTOR_SHIFT);
+	device_add_disk(dev, disk);
 	revalidate_disk(disk);
 	return 0;
 }
diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index c586bcdb5190..4b95ac513de2 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -753,6 +753,7 @@ static struct arena_info *alloc_arena(struct btt *btt, size_t size,
 		return NULL;
 	arena->nd_btt = btt->nd_btt;
 	arena->sector_size = btt->sector_size;
+	mutex_init(&arena->err_lock);
 
 	if (!size)
 		return arena;
@@ -891,7 +892,6 @@ static int discover_arenas(struct btt *btt)
 			goto out;
 		}
 
-		mutex_init(&arena->err_lock);
 		ret = btt_freelist_init(arena);
 		if (ret)
 			goto out;
@@ -1545,8 +1545,6 @@ static int btt_blk_init(struct btt *btt)
 	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, btt->btt_queue);
 	btt->btt_queue->queuedata = btt;
 
-	set_capacity(btt->btt_disk, 0);
-	device_add_disk(&btt->nd_btt->dev, btt->btt_disk);
 	if (btt_meta_size(btt)) {
 		int rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt));
 
@@ -1558,6 +1556,7 @@ static int btt_blk_init(struct btt *btt)
 		}
 	}
 	set_capacity(btt->btt_disk, btt->nlba * btt->sector_size >> 9);
+	device_add_disk(&btt->nd_btt->dev, btt->btt_disk);
 	btt->nd_btt->size = btt->nlba * (u64)btt->sector_size;
 	revalidate_disk(btt->btt_disk);
 
diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c
index 0a5e6cd758fe..78eabc3a1ab1 100644
--- a/drivers/nvdimm/bus.c
+++ b/drivers/nvdimm/bus.c
@@ -1142,9 +1142,6 @@ int __init nvdimm_bus_init(void)
 {
 	int rc;
 
-	BUILD_BUG_ON(sizeof(struct nd_smart_payload) != 128);
-	BUILD_BUG_ON(sizeof(struct nd_smart_threshold_payload) != 8);
-
 	rc = bus_register(&nvdimm_bus_type);
 	if (rc)
 		return rc;
diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c
index bb3ba8cf24d4..658ada497be0 100644
--- a/drivers/nvdimm/namespace_devs.c
+++ b/drivers/nvdimm/namespace_devs.c
@@ -2408,7 +2408,7 @@ static struct device **scan_labels(struct nd_region *nd_region)
 
 static struct device **create_namespaces(struct nd_region *nd_region)
 {
-	struct nd_mapping *nd_mapping = &nd_region->mapping[0];
+	struct nd_mapping *nd_mapping;
 	struct device **devs;
 	int i;
 
diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h
index e958f3724c41..8d6375ee0fda 100644
--- a/drivers/nvdimm/nd.h
+++ b/drivers/nvdimm/nd.h
@@ -368,15 +368,14 @@ unsigned int pmem_sector_size(struct nd_namespace_common *ndns);
 void nvdimm_badblocks_populate(struct nd_region *nd_region,
 		struct badblocks *bb, const struct resource *res);
 #if IS_ENABLED(CONFIG_ND_CLAIM)
-struct vmem_altmap *nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
-		struct resource *res, struct vmem_altmap *altmap);
+int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap);
 int devm_nsio_enable(struct device *dev, struct nd_namespace_io *nsio);
 void devm_nsio_disable(struct device *dev, struct nd_namespace_io *nsio);
 #else
-static inline struct vmem_altmap *nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
-		struct resource *res, struct vmem_altmap *altmap)
+static inline int nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
+				   struct dev_pagemap *pgmap)
 {
-	return ERR_PTR(-ENXIO);
+	return -ENXIO;
 }
 static inline int devm_nsio_enable(struct device *dev,
 		struct nd_namespace_io *nsio)
diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c
index 2adada1a5855..2f4d18752c97 100644
--- a/drivers/nvdimm/pfn_devs.c
+++ b/drivers/nvdimm/pfn_devs.c
@@ -304,7 +304,7 @@ static const struct attribute_group *nd_pfn_attribute_groups[] = {
 struct device *nd_pfn_devinit(struct nd_pfn *nd_pfn,
 		struct nd_namespace_common *ndns)
 {
-	struct device *dev = &nd_pfn->dev;
+	struct device *dev;
 
 	if (!nd_pfn)
 		return NULL;
@@ -542,9 +542,10 @@ static unsigned long init_altmap_reserve(resource_size_t base)
 	return reserve;
 }
 
-static struct vmem_altmap *__nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
-		struct resource *res, struct vmem_altmap *altmap)
+static int __nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap)
 {
+	struct resource *res = &pgmap->res;
+	struct vmem_altmap *altmap = &pgmap->altmap;
 	struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;
 	u64 offset = le64_to_cpu(pfn_sb->dataoff);
 	u32 start_pad = __le32_to_cpu(pfn_sb->start_pad);
@@ -561,11 +562,13 @@ static struct vmem_altmap *__nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
 	res->start += start_pad;
 	res->end -= end_trunc;
 
+	pgmap->type = MEMORY_DEVICE_HOST;
+
 	if (nd_pfn->mode == PFN_MODE_RAM) {
 		if (offset < SZ_8K)
-			return ERR_PTR(-EINVAL);
+			return -EINVAL;
 		nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
-		altmap = NULL;
+		pgmap->altmap_valid = false;
 	} else if (nd_pfn->mode == PFN_MODE_PMEM) {
 		nd_pfn->npfns = PFN_SECTION_ALIGN_UP((resource_size(res)
 					- offset) / PAGE_SIZE);
@@ -577,10 +580,11 @@ static struct vmem_altmap *__nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
 		memcpy(altmap, &__altmap, sizeof(*altmap));
 		altmap->free = PHYS_PFN(offset - SZ_8K);
 		altmap->alloc = 0;
+		pgmap->altmap_valid = true;
 	} else
-		return ERR_PTR(-ENXIO);
+		return -ENXIO;
 
-	return altmap;
+	return 0;
 }
 
 static u64 phys_pmem_align_down(struct nd_pfn *nd_pfn, u64 phys)
@@ -708,19 +712,18 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn)
  * Determine the effective resource range and vmem_altmap from an nd_pfn
  * instance.
  */
-struct vmem_altmap *nvdimm_setup_pfn(struct nd_pfn *nd_pfn,
-		struct resource *res, struct vmem_altmap *altmap)
+int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap)
 {
 	int rc;
 
 	if (!nd_pfn->uuid || !nd_pfn->ndns)
-		return ERR_PTR(-ENODEV);
+		return -ENODEV;
 
 	rc = nd_pfn_init(nd_pfn);
 	if (rc)
-		return ERR_PTR(rc);
+		return rc;
 
-	/* we need a valid pfn_sb before we can init a vmem_altmap */
-	return __nvdimm_setup_pfn(nd_pfn, res, altmap);
+	/* we need a valid pfn_sb before we can init a dev_pagemap */
+	return __nvdimm_setup_pfn(nd_pfn, pgmap);
 }
 EXPORT_SYMBOL_GPL(nvdimm_setup_pfn);
diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c
index 7fbc5c5dc8e1..06f8dcc52ca6 100644
--- a/drivers/nvdimm/pmem.c
+++ b/drivers/nvdimm/pmem.c
@@ -35,6 +35,7 @@
 #include "pmem.h"
 #include "pfn.h"
 #include "nd.h"
+#include "nd-core.h"
 
 static struct device *to_dev(struct pmem_device *pmem)
 {
@@ -298,34 +299,34 @@ static int pmem_attach_disk(struct device *dev,
 {
 	struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
 	struct nd_region *nd_region = to_nd_region(dev->parent);
-	struct vmem_altmap __altmap, *altmap = NULL;
 	int nid = dev_to_node(dev), fua, wbc;
 	struct resource *res = &nsio->res;
+	struct resource bb_res;
 	struct nd_pfn *nd_pfn = NULL;
 	struct dax_device *dax_dev;
 	struct nd_pfn_sb *pfn_sb;
 	struct pmem_device *pmem;
-	struct resource pfn_res;
 	struct request_queue *q;
 	struct device *gendev;
 	struct gendisk *disk;
 	void *addr;
+	int rc;
+
+	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
+	if (!pmem)
+		return -ENOMEM;
 
 	/* while nsio_rw_bytes is active, parse a pfn info block if present */
 	if (is_nd_pfn(dev)) {
 		nd_pfn = to_nd_pfn(dev);
-		altmap = nvdimm_setup_pfn(nd_pfn, &pfn_res, &__altmap);
-		if (IS_ERR(altmap))
-			return PTR_ERR(altmap);
+		rc = nvdimm_setup_pfn(nd_pfn, &pmem->pgmap);
+		if (rc)
+			return rc;
 	}
 
 	/* we're attaching a block device, disable raw namespace access */
 	devm_nsio_disable(dev, nsio);
 
-	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
-	if (!pmem)
-		return -ENOMEM;
-
 	dev_set_drvdata(dev, pmem);
 	pmem->phys_addr = res->start;
 	pmem->size = resource_size(res);
@@ -350,19 +351,22 @@ static int pmem_attach_disk(struct device *dev,
 		return -ENOMEM;
 
 	pmem->pfn_flags = PFN_DEV;
+	pmem->pgmap.ref = &q->q_usage_counter;
 	if (is_nd_pfn(dev)) {
-		addr = devm_memremap_pages(dev, &pfn_res, &q->q_usage_counter,
-				altmap);
+		addr = devm_memremap_pages(dev, &pmem->pgmap);
 		pfn_sb = nd_pfn->pfn_sb;
 		pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
-		pmem->pfn_pad = resource_size(res) - resource_size(&pfn_res);
+		pmem->pfn_pad = resource_size(res) -
+			resource_size(&pmem->pgmap.res);
 		pmem->pfn_flags |= PFN_MAP;
-		res = &pfn_res; /* for badblocks populate */
-		res->start += pmem->data_offset;
+		memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
+		bb_res.start += pmem->data_offset;
 	} else if (pmem_should_map_pages(dev)) {
-		addr = devm_memremap_pages(dev, &nsio->res,
-				&q->q_usage_counter, NULL);
+		memcpy(&pmem->pgmap.res, &nsio->res, sizeof(pmem->pgmap.res));
+		pmem->pgmap.altmap_valid = false;
+		addr = devm_memremap_pages(dev, &pmem->pgmap);
 		pmem->pfn_flags |= PFN_MAP;
+		memcpy(&bb_res, &pmem->pgmap.res, sizeof(bb_res));
 	} else
 		addr = devm_memremap(dev, pmem->phys_addr,
 				pmem->size, ARCH_MEMREMAP_PMEM);
@@ -401,7 +405,7 @@ static int pmem_attach_disk(struct device *dev,
 			/ 512);
 	if (devm_init_badblocks(dev, &pmem->bb))
 		return -ENOMEM;
-	nvdimm_badblocks_populate(nd_region, &pmem->bb, res);
+	nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_res);
 	disk->bb = &pmem->bb;
 
 	dax_dev = alloc_dax(pmem, disk->disk_name, &pmem_dax_ops);
diff --git a/drivers/nvdimm/pmem.h b/drivers/nvdimm/pmem.h
index 6a3cd2a10db6..a64ebc78b5df 100644
--- a/drivers/nvdimm/pmem.h
+++ b/drivers/nvdimm/pmem.h
@@ -22,6 +22,7 @@ struct pmem_device {
 	struct badblocks	bb;
 	struct dax_device	*dax_dev;
 	struct gendisk		*disk;
+	struct dev_pagemap	pgmap;
 };
 
 long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c
index abaf38c61220..1593e1806b16 100644
--- a/drivers/nvdimm/region_devs.c
+++ b/drivers/nvdimm/region_devs.c
@@ -528,6 +528,20 @@ static ssize_t resource_show(struct device *dev,
 }
 static DEVICE_ATTR_RO(resource);
 
+static ssize_t persistence_domain_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct nd_region *nd_region = to_nd_region(dev);
+
+	if (test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags))
+		return sprintf(buf, "cpu_cache\n");
+	else if (test_bit(ND_REGION_PERSIST_MEMCTRL, &nd_region->flags))
+		return sprintf(buf, "memory_controller\n");
+	else
+		return sprintf(buf, "\n");
+}
+static DEVICE_ATTR_RO(persistence_domain);
+
 static struct attribute *nd_region_attributes[] = {
 	&dev_attr_size.attr,
 	&dev_attr_nstype.attr,
@@ -543,6 +557,7 @@ static struct attribute *nd_region_attributes[] = {
 	&dev_attr_init_namespaces.attr,
 	&dev_attr_badblocks.attr,
 	&dev_attr_resource.attr,
+	&dev_attr_persistence_domain.attr,
 	NULL,
 };
 
@@ -580,6 +595,13 @@ static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n)
 			return 0;
 	}
 
+	if (a == &dev_attr_persistence_domain.attr) {
+		if ((nd_region->flags & (BIT(ND_REGION_PERSIST_CACHE)
+					| BIT(ND_REGION_PERSIST_MEMCTRL))) == 0)
+			return 0;
+		return a->mode;
+	}
+
 	if (a != &dev_attr_set_cookie.attr
 			&& a != &dev_attr_available_size.attr)
 		return a->mode;
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
index a25fd43650ad..441e67e3a9d7 100644
--- a/drivers/nvme/host/Makefile
+++ b/drivers/nvme/host/Makefile
@@ -1,4 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
+
+ccflags-y				+= -I$(src)
+
 obj-$(CONFIG_NVME_CORE)			+= nvme-core.o
 obj-$(CONFIG_BLK_DEV_NVME)		+= nvme.o
 obj-$(CONFIG_NVME_FABRICS)		+= nvme-fabrics.o
@@ -6,6 +9,7 @@ obj-$(CONFIG_NVME_RDMA)			+= nvme-rdma.o
 obj-$(CONFIG_NVME_FC)			+= nvme-fc.o
 
 nvme-core-y				:= core.o
+nvme-core-$(CONFIG_TRACING)		+= trace.o
 nvme-core-$(CONFIG_NVME_MULTIPATH)	+= multipath.o
 nvme-core-$(CONFIG_NVM)			+= lightnvm.o
 
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 839650e0926a..7aeca5db7916 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -29,6 +29,9 @@
 #include <linux/pm_qos.h>
 #include <asm/unaligned.h>
 
+#define CREATE_TRACE_POINTS
+#include "trace.h"
+
 #include "nvme.h"
 #include "fabrics.h"
 
@@ -65,9 +68,26 @@ static bool streams;
 module_param(streams, bool, 0644);
 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
 
+/*
+ * nvme_wq - hosts nvme related works that are not reset or delete
+ * nvme_reset_wq - hosts nvme reset works
+ * nvme_delete_wq - hosts nvme delete works
+ *
+ * nvme_wq will host works such are scan, aen handling, fw activation,
+ * keep-alive error recovery, periodic reconnects etc. nvme_reset_wq
+ * runs reset works which also flush works hosted on nvme_wq for
+ * serialization purposes. nvme_delete_wq host controller deletion
+ * works which flush reset works for serialization.
+ */
 struct workqueue_struct *nvme_wq;
 EXPORT_SYMBOL_GPL(nvme_wq);
 
+struct workqueue_struct *nvme_reset_wq;
+EXPORT_SYMBOL_GPL(nvme_reset_wq);
+
+struct workqueue_struct *nvme_delete_wq;
+EXPORT_SYMBOL_GPL(nvme_delete_wq);
+
 static DEFINE_IDA(nvme_subsystems_ida);
 static LIST_HEAD(nvme_subsystems);
 static DEFINE_MUTEX(nvme_subsystems_lock);
@@ -89,21 +109,26 @@ int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 {
 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 		return -EBUSY;
-	if (!queue_work(nvme_wq, &ctrl->reset_work))
+	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 		return -EBUSY;
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 
-static int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
+int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 {
 	int ret;
 
 	ret = nvme_reset_ctrl(ctrl);
-	if (!ret)
+	if (!ret) {
 		flush_work(&ctrl->reset_work);
+		if (ctrl->state != NVME_CTRL_LIVE)
+			ret = -ENETRESET;
+	}
+
 	return ret;
 }
+EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
 
 static void nvme_delete_ctrl_work(struct work_struct *work)
 {
@@ -122,7 +147,7 @@ int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 {
 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 		return -EBUSY;
-	if (!queue_work(nvme_wq, &ctrl->delete_work))
+	if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 		return -EBUSY;
 	return 0;
 }
@@ -157,13 +182,20 @@ static blk_status_t nvme_error_status(struct request *req)
 		return BLK_STS_OK;
 	case NVME_SC_CAP_EXCEEDED:
 		return BLK_STS_NOSPC;
+	case NVME_SC_LBA_RANGE:
+		return BLK_STS_TARGET;
+	case NVME_SC_BAD_ATTRIBUTES:
 	case NVME_SC_ONCS_NOT_SUPPORTED:
+	case NVME_SC_INVALID_OPCODE:
+	case NVME_SC_INVALID_FIELD:
+	case NVME_SC_INVALID_NS:
 		return BLK_STS_NOTSUPP;
 	case NVME_SC_WRITE_FAULT:
 	case NVME_SC_READ_ERROR:
 	case NVME_SC_UNWRITTEN_BLOCK:
 	case NVME_SC_ACCESS_DENIED:
 	case NVME_SC_READ_ONLY:
+	case NVME_SC_COMPARE_FAILED:
 		return BLK_STS_MEDIUM;
 	case NVME_SC_GUARD_CHECK:
 	case NVME_SC_APPTAG_CHECK:
@@ -190,8 +222,12 @@ static inline bool nvme_req_needs_retry(struct request *req)
 
 void nvme_complete_rq(struct request *req)
 {
-	if (unlikely(nvme_req(req)->status && nvme_req_needs_retry(req))) {
-		if (nvme_req_needs_failover(req)) {
+	blk_status_t status = nvme_error_status(req);
+
+	trace_nvme_complete_rq(req);
+
+	if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
+		if (nvme_req_needs_failover(req, status)) {
 			nvme_failover_req(req);
 			return;
 		}
@@ -202,8 +238,7 @@ void nvme_complete_rq(struct request *req)
 			return;
 		}
 	}
-
-	blk_mq_end_request(req, nvme_error_status(req));
+	blk_mq_end_request(req, status);
 }
 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 
@@ -232,11 +267,20 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 
 	old_state = ctrl->state;
 	switch (new_state) {
+	case NVME_CTRL_ADMIN_ONLY:
+		switch (old_state) {
+		case NVME_CTRL_CONNECTING:
+			changed = true;
+			/* FALLTHRU */
+		default:
+			break;
+		}
+		break;
 	case NVME_CTRL_LIVE:
 		switch (old_state) {
 		case NVME_CTRL_NEW:
 		case NVME_CTRL_RESETTING:
-		case NVME_CTRL_RECONNECTING:
+		case NVME_CTRL_CONNECTING:
 			changed = true;
 			/* FALLTHRU */
 		default:
@@ -247,15 +291,16 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 		switch (old_state) {
 		case NVME_CTRL_NEW:
 		case NVME_CTRL_LIVE:
+		case NVME_CTRL_ADMIN_ONLY:
 			changed = true;
 			/* FALLTHRU */
 		default:
 			break;
 		}
 		break;
-	case NVME_CTRL_RECONNECTING:
+	case NVME_CTRL_CONNECTING:
 		switch (old_state) {
-		case NVME_CTRL_LIVE:
+		case NVME_CTRL_NEW:
 		case NVME_CTRL_RESETTING:
 			changed = true;
 			/* FALLTHRU */
@@ -266,8 +311,9 @@ bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 	case NVME_CTRL_DELETING:
 		switch (old_state) {
 		case NVME_CTRL_LIVE:
+		case NVME_CTRL_ADMIN_ONLY:
 		case NVME_CTRL_RESETTING:
-		case NVME_CTRL_RECONNECTING:
+		case NVME_CTRL_CONNECTING:
 			changed = true;
 			/* FALLTHRU */
 		default:
@@ -476,9 +522,11 @@ static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 		u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
 		u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 
-		range[n].cattr = cpu_to_le32(0);
-		range[n].nlb = cpu_to_le32(nlb);
-		range[n].slba = cpu_to_le64(slba);
+		if (n < segments) {
+			range[n].cattr = cpu_to_le32(0);
+			range[n].nlb = cpu_to_le32(nlb);
+			range[n].slba = cpu_to_le64(slba);
+		}
 		n++;
 	}
 
@@ -591,6 +639,10 @@ blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 	}
 
 	cmd->common.command_id = req->tag;
+	if (ns)
+		trace_nvme_setup_nvm_cmd(req->q->id, cmd);
+	else
+		trace_nvme_setup_admin_cmd(cmd);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
@@ -748,13 +800,9 @@ static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
 
 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
 {
-	struct nvme_command c;
 	struct request *rq;
 
-	memset(&c, 0, sizeof(c));
-	c.common.opcode = nvme_admin_keep_alive;
-
-	rq = nvme_alloc_request(ctrl->admin_q, &c, BLK_MQ_REQ_RESERVED,
+	rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
 			NVME_QID_ANY);
 	if (IS_ERR(rq))
 		return PTR_ERR(rq);
@@ -786,6 +834,8 @@ void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
 		return;
 
 	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
+	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
+	ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
 	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
 }
 EXPORT_SYMBOL_GPL(nvme_start_keep_alive);
@@ -1071,14 +1121,19 @@ static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 
 static void nvme_update_formats(struct nvme_ctrl *ctrl)
 {
-	struct nvme_ns *ns;
+	struct nvme_ns *ns, *next;
+	LIST_HEAD(rm_list);
 
 	mutex_lock(&ctrl->namespaces_mutex);
 	list_for_each_entry(ns, &ctrl->namespaces, list) {
-		if (ns->disk && nvme_revalidate_disk(ns->disk))
-			nvme_ns_remove(ns);
+		if (ns->disk && nvme_revalidate_disk(ns->disk)) {
+			list_move_tail(&ns->list, &rm_list);
+		}
 	}
 	mutex_unlock(&ctrl->namespaces_mutex);
+
+	list_for_each_entry_safe(ns, next, &rm_list, list)
+		nvme_ns_remove(ns);
 }
 
 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
@@ -1217,16 +1272,27 @@ static int nvme_open(struct block_device *bdev, fmode_t mode)
 #ifdef CONFIG_NVME_MULTIPATH
 	/* should never be called due to GENHD_FL_HIDDEN */
 	if (WARN_ON_ONCE(ns->head->disk))
-		return -ENXIO;
+		goto fail;
 #endif
 	if (!kref_get_unless_zero(&ns->kref))
-		return -ENXIO;
+		goto fail;
+	if (!try_module_get(ns->ctrl->ops->module))
+		goto fail_put_ns;
+
 	return 0;
+
+fail_put_ns:
+	nvme_put_ns(ns);
+fail:
+	return -ENXIO;
 }
 
 static void nvme_release(struct gendisk *disk, fmode_t mode)
 {
-	nvme_put_ns(disk->private_data);
+	struct nvme_ns *ns = disk->private_data;
+
+	module_put(ns->ctrl->ops->module);
+	nvme_put_ns(ns);
 }
 
 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
@@ -2052,6 +2118,22 @@ static const struct attribute_group *nvme_subsys_attrs_groups[] = {
 	NULL,
 };
 
+static int nvme_active_ctrls(struct nvme_subsystem *subsys)
+{
+	int count = 0;
+	struct nvme_ctrl *ctrl;
+
+	mutex_lock(&subsys->lock);
+	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
+		if (ctrl->state != NVME_CTRL_DELETING &&
+		    ctrl->state != NVME_CTRL_DEAD)
+			count++;
+	}
+	mutex_unlock(&subsys->lock);
+
+	return count;
+}
+
 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 {
 	struct nvme_subsystem *subsys, *found;
@@ -2090,7 +2172,7 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
 		 * Verify that the subsystem actually supports multiple
 		 * controllers, else bail out.
 		 */
-		if (!(id->cmic & (1 << 1))) {
+		if (nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {
 			dev_err(ctrl->device,
 				"ignoring ctrl due to duplicate subnqn (%s).\n",
 				found->subnqn);
@@ -2257,7 +2339,7 @@ int nvme_init_identify(struct nvme_ctrl *ctrl)
 						 shutdown_timeout, 60);
 
 		if (ctrl->shutdown_timeout != shutdown_timeout)
-			dev_warn(ctrl->device,
+			dev_info(ctrl->device,
 				 "Shutdown timeout set to %u seconds\n",
 				 ctrl->shutdown_timeout);
 	} else
@@ -2341,8 +2423,14 @@ static int nvme_dev_open(struct inode *inode, struct file *file)
 	struct nvme_ctrl *ctrl =
 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
 
-	if (ctrl->state != NVME_CTRL_LIVE)
+	switch (ctrl->state) {
+	case NVME_CTRL_LIVE:
+	case NVME_CTRL_ADMIN_ONLY:
+		break;
+	default:
 		return -EWOULDBLOCK;
+	}
+
 	file->private_data = ctrl;
 	return 0;
 }
@@ -2475,14 +2563,14 @@ static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
 		serial_len, subsys->serial, model_len, subsys->model,
 		head->ns_id);
 }
-static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);
+static DEVICE_ATTR_RO(wwid);
 
 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
 		char *buf)
 {
 	return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
 }
-static DEVICE_ATTR(nguid, S_IRUGO, nguid_show, NULL);
+static DEVICE_ATTR_RO(nguid);
 
 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
 		char *buf)
@@ -2499,21 +2587,21 @@ static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
 	}
 	return sprintf(buf, "%pU\n", &ids->uuid);
 }
-static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
+static DEVICE_ATTR_RO(uuid);
 
 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
 		char *buf)
 {
 	return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
 }
-static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
+static DEVICE_ATTR_RO(eui);
 
 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
 		char *buf)
 {
 	return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
 }
-static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);
+static DEVICE_ATTR_RO(nsid);
 
 static struct attribute *nvme_ns_id_attrs[] = {
 	&dev_attr_wwid.attr,
@@ -2606,8 +2694,9 @@ static ssize_t nvme_sysfs_show_state(struct device *dev,
 	static const char *const state_name[] = {
 		[NVME_CTRL_NEW]		= "new",
 		[NVME_CTRL_LIVE]	= "live",
+		[NVME_CTRL_ADMIN_ONLY]	= "only-admin",
 		[NVME_CTRL_RESETTING]	= "resetting",
-		[NVME_CTRL_RECONNECTING]= "reconnecting",
+		[NVME_CTRL_CONNECTING]	= "connecting",
 		[NVME_CTRL_DELETING]	= "deleting",
 		[NVME_CTRL_DEAD]	= "dead",
 	};
@@ -2755,7 +2844,7 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
 }
 
 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
-		struct nvme_id_ns *id, bool *new)
+		struct nvme_id_ns *id)
 {
 	struct nvme_ctrl *ctrl = ns->ctrl;
 	bool is_shared = id->nmic & (1 << 0);
@@ -2771,8 +2860,6 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
 			ret = PTR_ERR(head);
 			goto out_unlock;
 		}
-
-		*new = true;
 	} else {
 		struct nvme_ns_ids ids;
 
@@ -2784,8 +2871,6 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
 			ret = -EINVAL;
 			goto out_unlock;
 		}
-
-		*new = false;
 	}
 
 	list_add_tail(&ns->siblings, &head->list);
@@ -2856,7 +2941,6 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 	struct nvme_id_ns *id;
 	char disk_name[DISK_NAME_LEN];
 	int node = dev_to_node(ctrl->dev), flags = GENHD_FL_EXT_DEVT;
-	bool new = true;
 
 	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
 	if (!ns)
@@ -2882,7 +2966,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 	if (id->ncap == 0)
 		goto out_free_id;
 
-	if (nvme_init_ns_head(ns, nsid, id, &new))
+	if (nvme_init_ns_head(ns, nsid, id))
 		goto out_free_id;
 	nvme_setup_streams_ns(ctrl, ns);
 	
@@ -2948,9 +3032,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
 		pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
 			ns->disk->disk_name);
 
-	if (new)
-		nvme_mpath_add_disk(ns->head);
-	nvme_mpath_add_disk_links(ns);
+	nvme_mpath_add_disk(ns->head);
 	return;
  out_unlink_ns:
 	mutex_lock(&ctrl->subsys->lock);
@@ -2970,7 +3052,6 @@ static void nvme_ns_remove(struct nvme_ns *ns)
 		return;
 
 	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
-		nvme_mpath_remove_disk_links(ns);
 		sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
 					&nvme_ns_id_attr_group);
 		if (ns->ndev)
@@ -3079,6 +3160,8 @@ static void nvme_scan_work(struct work_struct *work)
 	if (ctrl->state != NVME_CTRL_LIVE)
 		return;
 
+	WARN_ON_ONCE(!ctrl->tagset);
+
 	if (nvme_identify_ctrl(ctrl, &id))
 		return;
 
@@ -3099,8 +3182,7 @@ static void nvme_scan_work(struct work_struct *work)
 void nvme_queue_scan(struct nvme_ctrl *ctrl)
 {
 	/*
-	 * Do not queue new scan work when a controller is reset during
-	 * removal.
+	 * Only new queue scan work when admin and IO queues are both alive
 	 */
 	if (ctrl->state == NVME_CTRL_LIVE)
 		queue_work(nvme_wq, &ctrl->scan_work);
@@ -3477,16 +3559,26 @@ EXPORT_SYMBOL_GPL(nvme_reinit_tagset);
 
 int __init nvme_core_init(void)
 {
-	int result;
+	int result = -ENOMEM;
 
 	nvme_wq = alloc_workqueue("nvme-wq",
 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
 	if (!nvme_wq)
-		return -ENOMEM;
+		goto out;
+
+	nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
+			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
+	if (!nvme_reset_wq)
+		goto destroy_wq;
+
+	nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
+			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
+	if (!nvme_delete_wq)
+		goto destroy_reset_wq;
 
 	result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
 	if (result < 0)
-		goto destroy_wq;
+		goto destroy_delete_wq;
 
 	nvme_class = class_create(THIS_MODULE, "nvme");
 	if (IS_ERR(nvme_class)) {
@@ -3505,8 +3597,13 @@ int __init nvme_core_init(void)
 	class_destroy(nvme_class);
 unregister_chrdev:
 	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
+destroy_delete_wq:
+	destroy_workqueue(nvme_delete_wq);
+destroy_reset_wq:
+	destroy_workqueue(nvme_reset_wq);
 destroy_wq:
 	destroy_workqueue(nvme_wq);
+out:
 	return result;
 }
 
@@ -3516,6 +3613,8 @@ void nvme_core_exit(void)
 	class_destroy(nvme_subsys_class);
 	class_destroy(nvme_class);
 	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
+	destroy_workqueue(nvme_delete_wq);
+	destroy_workqueue(nvme_reset_wq);
 	destroy_workqueue(nvme_wq);
 }
 
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 894c2ccb3891..8f0f34d06d46 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -650,6 +650,11 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 				ret = -EINVAL;
 				goto out;
 			}
+			if (opts->discovery_nqn) {
+				pr_debug("Ignoring nr_io_queues value for discovery controller\n");
+				break;
+			}
+
 			opts->nr_io_queues = min_t(unsigned int,
 					num_online_cpus(), token);
 			break;
@@ -739,11 +744,14 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
 				ret = -ENOMEM;
 				goto out;
 			}
-			if (uuid_parse(p, &hostid)) {
+			ret = uuid_parse(p, &hostid);
+			if (ret) {
 				pr_err("Invalid hostid %s\n", p);
 				ret = -EINVAL;
+				kfree(p);
 				goto out;
 			}
+			kfree(p);
 			break;
 		case NVMF_OPT_DUP_CONNECT:
 			opts->duplicate_connect = true;
@@ -869,32 +877,41 @@ nvmf_create_ctrl(struct device *dev, const char *buf, size_t count)
 		goto out_unlock;
 	}
 
+	if (!try_module_get(ops->module)) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
 	ret = nvmf_check_required_opts(opts, ops->required_opts);
 	if (ret)
-		goto out_unlock;
+		goto out_module_put;
 	ret = nvmf_check_allowed_opts(opts, NVMF_ALLOWED_OPTS |
 				ops->allowed_opts | ops->required_opts);
 	if (ret)
-		goto out_unlock;
+		goto out_module_put;
 
 	ctrl = ops->create_ctrl(dev, opts);
 	if (IS_ERR(ctrl)) {
 		ret = PTR_ERR(ctrl);
-		goto out_unlock;
+		goto out_module_put;
 	}
 
 	if (strcmp(ctrl->subsys->subnqn, opts->subsysnqn)) {
 		dev_warn(ctrl->device,
 			"controller returned incorrect NQN: \"%s\".\n",
 			ctrl->subsys->subnqn);
+		module_put(ops->module);
 		up_read(&nvmf_transports_rwsem);
 		nvme_delete_ctrl_sync(ctrl);
 		return ERR_PTR(-EINVAL);
 	}
 
+	module_put(ops->module);
 	up_read(&nvmf_transports_rwsem);
 	return ctrl;
 
+out_module_put:
+	module_put(ops->module);
 out_unlock:
 	up_read(&nvmf_transports_rwsem);
 out_free_opts:
diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h
index 9ba614953607..a3145d90c1d2 100644
--- a/drivers/nvme/host/fabrics.h
+++ b/drivers/nvme/host/fabrics.h
@@ -108,6 +108,7 @@ struct nvmf_ctrl_options {
  *			       fabric implementation of NVMe fabrics.
  * @entry:		Used by the fabrics library to add the new
  *			registration entry to its linked-list internal tree.
+ * @module:             Transport module reference
  * @name:		Name of the NVMe fabric driver implementation.
  * @required_opts:	sysfs command-line options that must be specified
  *			when adding a new NVMe controller.
@@ -126,6 +127,7 @@ struct nvmf_ctrl_options {
  */
 struct nvmf_transport_ops {
 	struct list_head	entry;
+	struct module		*module;
 	const char		*name;
 	int			required_opts;
 	int			allowed_opts;
@@ -169,13 +171,14 @@ static inline blk_status_t nvmf_check_init_req(struct nvme_ctrl *ctrl,
 	    cmd->common.opcode != nvme_fabrics_command ||
 	    cmd->fabrics.fctype != nvme_fabrics_type_connect) {
 		/*
-		 * Reconnecting state means transport disruption, which can take
-		 * a long time and even might fail permanently, fail fast to
-		 * give upper layers a chance to failover.
+		 * Connecting state means transport disruption or initial
+		 * establishment, which can take a long time and even might
+		 * fail permanently, fail fast to give upper layers a chance
+		 * to failover.
 		 * Deleting state means that the ctrl will never accept commands
 		 * again, fail it permanently.
 		 */
-		if (ctrl->state == NVME_CTRL_RECONNECTING ||
+		if (ctrl->state == NVME_CTRL_CONNECTING ||
 		    ctrl->state == NVME_CTRL_DELETING) {
 			nvme_req(rq)->status = NVME_SC_ABORT_REQ;
 			return BLK_STS_IOERR;
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 794e66e4aa20..1dc1387b7134 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -35,8 +35,6 @@ enum nvme_fc_queue_flags {
 	NVME_FC_Q_LIVE,
 };
 
-#define NVMEFC_QUEUE_DELAY	3		/* ms units */
-
 #define NVME_FC_DEFAULT_DEV_LOSS_TMO	60	/* seconds */
 
 struct nvme_fc_queue {
@@ -57,9 +55,7 @@ struct nvme_fc_queue {
 
 enum nvme_fcop_flags {
 	FCOP_FLAGS_TERMIO	= (1 << 0),
-	FCOP_FLAGS_RELEASED	= (1 << 1),
-	FCOP_FLAGS_COMPLETE	= (1 << 2),
-	FCOP_FLAGS_AEN		= (1 << 3),
+	FCOP_FLAGS_AEN		= (1 << 1),
 };
 
 struct nvmefc_ls_req_op {
@@ -534,7 +530,7 @@ nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
 {
 	switch (ctrl->ctrl.state) {
 	case NVME_CTRL_NEW:
-	case NVME_CTRL_RECONNECTING:
+	case NVME_CTRL_CONNECTING:
 		/*
 		 * As all reconnects were suppressed, schedule a
 		 * connect.
@@ -779,7 +775,7 @@ nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
 		}
 		break;
 
-	case NVME_CTRL_RECONNECTING:
+	case NVME_CTRL_CONNECTING:
 		/*
 		 * The association has already been terminated and the
 		 * controller is attempting reconnects.  No need to do anything
@@ -1210,7 +1206,7 @@ nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
 				sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
 
 	assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
-	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize);
+	assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
 	/* Linux supports only Dynamic controllers */
 	assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
 	uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
@@ -1325,7 +1321,7 @@ nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
 				sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
 	conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
 	conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
-	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize);
+	conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
 
 	lsop->queue = queue;
 	lsreq->rqstaddr = conn_rqst;
@@ -1472,7 +1468,6 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
 
 /* *********************** NVME Ctrl Routines **************************** */
 
-static void __nvme_fc_final_op_cleanup(struct request *rq);
 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
 
 static int
@@ -1514,13 +1509,19 @@ nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
 static int
 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
 {
-	int state;
+	unsigned long flags;
+	int opstate;
+
+	spin_lock_irqsave(&ctrl->lock, flags);
+	opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
+	if (opstate != FCPOP_STATE_ACTIVE)
+		atomic_set(&op->state, opstate);
+	else if (ctrl->flags & FCCTRL_TERMIO)
+		ctrl->iocnt++;
+	spin_unlock_irqrestore(&ctrl->lock, flags);
 
-	state = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
-	if (state != FCPOP_STATE_ACTIVE) {
-		atomic_set(&op->state, state);
+	if (opstate != FCPOP_STATE_ACTIVE)
 		return -ECANCELED;
-	}
 
 	ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
 					&ctrl->rport->remoteport,
@@ -1534,60 +1535,26 @@ static void
 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
 {
 	struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
-	unsigned long flags;
-	int i, ret;
-
-	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
-		if (atomic_read(&aen_op->state) != FCPOP_STATE_ACTIVE)
-			continue;
-
-		spin_lock_irqsave(&ctrl->lock, flags);
-		if (ctrl->flags & FCCTRL_TERMIO) {
-			ctrl->iocnt++;
-			aen_op->flags |= FCOP_FLAGS_TERMIO;
-		}
-		spin_unlock_irqrestore(&ctrl->lock, flags);
-
-		ret = __nvme_fc_abort_op(ctrl, aen_op);
-		if (ret) {
-			/*
-			 * if __nvme_fc_abort_op failed the io wasn't
-			 * active. Thus this call path is running in
-			 * parallel to the io complete. Treat as non-error.
-			 */
+	int i;
 
-			/* back out the flags/counters */
-			spin_lock_irqsave(&ctrl->lock, flags);
-			if (ctrl->flags & FCCTRL_TERMIO)
-				ctrl->iocnt--;
-			aen_op->flags &= ~FCOP_FLAGS_TERMIO;
-			spin_unlock_irqrestore(&ctrl->lock, flags);
-			return;
-		}
-	}
+	for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
+		__nvme_fc_abort_op(ctrl, aen_op);
 }
 
-static inline int
+static inline void
 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
-		struct nvme_fc_fcp_op *op)
+		struct nvme_fc_fcp_op *op, int opstate)
 {
 	unsigned long flags;
-	bool complete_rq = false;
 
-	spin_lock_irqsave(&ctrl->lock, flags);
-	if (unlikely(op->flags & FCOP_FLAGS_TERMIO)) {
+	if (opstate == FCPOP_STATE_ABORTED) {
+		spin_lock_irqsave(&ctrl->lock, flags);
 		if (ctrl->flags & FCCTRL_TERMIO) {
 			if (!--ctrl->iocnt)
 				wake_up(&ctrl->ioabort_wait);
 		}
+		spin_unlock_irqrestore(&ctrl->lock, flags);
 	}
-	if (op->flags & FCOP_FLAGS_RELEASED)
-		complete_rq = true;
-	else
-		op->flags |= FCOP_FLAGS_COMPLETE;
-	spin_unlock_irqrestore(&ctrl->lock, flags);
-
-	return complete_rq;
 }
 
 static void
@@ -1603,6 +1570,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
 	__le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
 	union nvme_result result;
 	bool terminate_assoc = true;
+	int opstate;
 
 	/*
 	 * WARNING:
@@ -1641,11 +1609,12 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
 	 * association to be terminated.
 	 */
 
+	opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
+
 	fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
 				sizeof(op->rsp_iu), DMA_FROM_DEVICE);
 
-	if (atomic_read(&op->state) == FCPOP_STATE_ABORTED ||
-			op->flags & FCOP_FLAGS_TERMIO)
+	if (opstate == FCPOP_STATE_ABORTED)
 		status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
 	else if (freq->status)
 		status = cpu_to_le16(NVME_SC_INTERNAL << 1);
@@ -1710,7 +1679,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
 done:
 	if (op->flags & FCOP_FLAGS_AEN) {
 		nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
-		__nvme_fc_fcpop_chk_teardowns(ctrl, op);
+		__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
 		atomic_set(&op->state, FCPOP_STATE_IDLE);
 		op->flags = FCOP_FLAGS_AEN;	/* clear other flags */
 		nvme_fc_ctrl_put(ctrl);
@@ -1724,13 +1693,11 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
 	if (status &&
 	    (blk_queue_dying(rq->q) ||
 	     ctrl->ctrl.state == NVME_CTRL_NEW ||
-	     ctrl->ctrl.state == NVME_CTRL_RECONNECTING))
+	     ctrl->ctrl.state == NVME_CTRL_CONNECTING))
 		status |= cpu_to_le16(NVME_SC_DNR << 1);
 
-	if (__nvme_fc_fcpop_chk_teardowns(ctrl, op))
-		__nvme_fc_final_op_cleanup(rq);
-	else
-		nvme_end_request(rq, status, result);
+	__nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
+	nvme_end_request(rq, status, result);
 
 check_error:
 	if (terminate_assoc)
@@ -2231,7 +2198,7 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
 	 * the target device is present
 	 */
 	if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
-		goto busy;
+		return BLK_STS_RESOURCE;
 
 	if (!nvme_fc_ctrl_get(ctrl))
 		return BLK_STS_IOERR;
@@ -2311,16 +2278,10 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
 				ret != -EBUSY)
 			return BLK_STS_IOERR;
 
-		goto busy;
+		return BLK_STS_RESOURCE;
 	}
 
 	return BLK_STS_OK;
-
-busy:
-	if (!(op->flags & FCOP_FLAGS_AEN) && queue->hctx)
-		blk_mq_delay_run_hw_queue(queue->hctx, NVMEFC_QUEUE_DELAY);
-
-	return BLK_STS_RESOURCE;
 }
 
 static inline blk_status_t nvme_fc_is_ready(struct nvme_fc_queue *queue,
@@ -2423,46 +2384,16 @@ nvme_fc_submit_async_event(struct nvme_ctrl *arg)
 }
 
 static void
-__nvme_fc_final_op_cleanup(struct request *rq)
+nvme_fc_complete_rq(struct request *rq)
 {
 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
 	struct nvme_fc_ctrl *ctrl = op->ctrl;
 
 	atomic_set(&op->state, FCPOP_STATE_IDLE);
-	op->flags &= ~(FCOP_FLAGS_TERMIO | FCOP_FLAGS_RELEASED |
-			FCOP_FLAGS_COMPLETE);
 
 	nvme_fc_unmap_data(ctrl, rq, op);
 	nvme_complete_rq(rq);
 	nvme_fc_ctrl_put(ctrl);
-
-}
-
-static void
-nvme_fc_complete_rq(struct request *rq)
-{
-	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
-	struct nvme_fc_ctrl *ctrl = op->ctrl;
-	unsigned long flags;
-	bool completed = false;
-
-	/*
-	 * the core layer, on controller resets after calling
-	 * nvme_shutdown_ctrl(), calls complete_rq without our
-	 * calling blk_mq_complete_request(), thus there may still
-	 * be live i/o outstanding with the LLDD. Means transport has
-	 * to track complete calls vs fcpio_done calls to know what
-	 * path to take on completes and dones.
-	 */
-	spin_lock_irqsave(&ctrl->lock, flags);
-	if (op->flags & FCOP_FLAGS_COMPLETE)
-		completed = true;
-	else
-		op->flags |= FCOP_FLAGS_RELEASED;
-	spin_unlock_irqrestore(&ctrl->lock, flags);
-
-	if (completed)
-		__nvme_fc_final_op_cleanup(rq);
 }
 
 /*
@@ -2484,35 +2415,11 @@ nvme_fc_terminate_exchange(struct request *req, void *data, bool reserved)
 	struct nvme_ctrl *nctrl = data;
 	struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
 	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
-	unsigned long flags;
-	int status;
 
 	if (!blk_mq_request_started(req))
 		return;
 
-	spin_lock_irqsave(&ctrl->lock, flags);
-	if (ctrl->flags & FCCTRL_TERMIO) {
-		ctrl->iocnt++;
-		op->flags |= FCOP_FLAGS_TERMIO;
-	}
-	spin_unlock_irqrestore(&ctrl->lock, flags);
-
-	status = __nvme_fc_abort_op(ctrl, op);
-	if (status) {
-		/*
-		 * if __nvme_fc_abort_op failed the io wasn't
-		 * active. Thus this call path is running in
-		 * parallel to the io complete. Treat as non-error.
-		 */
-
-		/* back out the flags/counters */
-		spin_lock_irqsave(&ctrl->lock, flags);
-		if (ctrl->flags & FCCTRL_TERMIO)
-			ctrl->iocnt--;
-		op->flags &= ~FCOP_FLAGS_TERMIO;
-		spin_unlock_irqrestore(&ctrl->lock, flags);
-		return;
-	}
+	__nvme_fc_abort_op(ctrl, op);
 }
 
 
@@ -2574,11 +2481,11 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
 		goto out_free_tag_set;
 	}
 
-	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.opts->queue_size);
+	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
 	if (ret)
 		goto out_cleanup_blk_queue;
 
-	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.opts->queue_size);
+	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
 	if (ret)
 		goto out_delete_hw_queues;
 
@@ -2625,11 +2532,11 @@ nvme_fc_reinit_io_queues(struct nvme_fc_ctrl *ctrl)
 	if (ret)
 		goto out_free_io_queues;
 
-	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.opts->queue_size);
+	ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
 	if (ret)
 		goto out_free_io_queues;
 
-	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.opts->queue_size);
+	ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
 	if (ret)
 		goto out_delete_hw_queues;
 
@@ -2725,13 +2632,12 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
 	nvme_fc_init_queue(ctrl, 0);
 
 	ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
-				NVME_AQ_BLK_MQ_DEPTH);
+				NVME_AQ_DEPTH);
 	if (ret)
 		goto out_free_queue;
 
 	ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
-				NVME_AQ_BLK_MQ_DEPTH,
-				(NVME_AQ_BLK_MQ_DEPTH / 4));
+				NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
 	if (ret)
 		goto out_delete_hw_queue;
 
@@ -2759,7 +2665,7 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
 	}
 
 	ctrl->ctrl.sqsize =
-		min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap) + 1, ctrl->ctrl.sqsize);
+		min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
 
 	ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
 	if (ret)
@@ -2792,6 +2698,14 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
 		opts->queue_size = ctrl->ctrl.maxcmd;
 	}
 
+	if (opts->queue_size > ctrl->ctrl.sqsize + 1) {
+		/* warn if sqsize is lower than queue_size */
+		dev_warn(ctrl->ctrl.device,
+			"queue_size %zu > ctrl sqsize %u, clamping down\n",
+			opts->queue_size, ctrl->ctrl.sqsize + 1);
+		opts->queue_size = ctrl->ctrl.sqsize + 1;
+	}
+
 	ret = nvme_fc_init_aen_ops(ctrl);
 	if (ret)
 		goto out_term_aen_ops;
@@ -2921,6 +2835,9 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
 	__nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
 	nvme_fc_free_queue(&ctrl->queues[0]);
 
+	/* re-enable the admin_q so anything new can fast fail */
+	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
+
 	nvme_fc_ctlr_inactive_on_rport(ctrl);
 }
 
@@ -2935,6 +2852,9 @@ nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
 	 * waiting for io to terminate
 	 */
 	nvme_fc_delete_association(ctrl);
+
+	/* resume the io queues so that things will fast fail */
+	nvme_start_queues(nctrl);
 }
 
 static void
@@ -2945,7 +2865,7 @@ nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
 	unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
 	bool recon = true;
 
-	if (ctrl->ctrl.state != NVME_CTRL_RECONNECTING)
+	if (ctrl->ctrl.state != NVME_CTRL_CONNECTING)
 		return;
 
 	if (portptr->port_state == FC_OBJSTATE_ONLINE)
@@ -2993,10 +2913,10 @@ nvme_fc_reset_ctrl_work(struct work_struct *work)
 	/* will block will waiting for io to terminate */
 	nvme_fc_delete_association(ctrl);
 
-	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RECONNECTING)) {
+	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
 		dev_err(ctrl->ctrl.device,
 			"NVME-FC{%d}: error_recovery: Couldn't change state "
-			"to RECONNECTING\n", ctrl->cnum);
+			"to CONNECTING\n", ctrl->cnum);
 		return;
 	}
 
@@ -3197,7 +3117,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
 	 * transport errors (frame drop, LS failure) inherently must kill
 	 * the association. The transport is coded so that any command used
 	 * to create the association (prior to a LIVE state transition
-	 * while NEW or RECONNECTING) will fail if it completes in error or
+	 * while NEW or CONNECTING) will fail if it completes in error or
 	 * times out.
 	 *
 	 * As such: as the connect request was mostly likely due to a
@@ -3380,6 +3300,7 @@ nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
 
 static struct nvmf_transport_ops nvme_fc_transport = {
 	.name		= "fc",
+	.module		= THIS_MODULE,
 	.required_opts	= NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
 	.allowed_opts	= NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
 	.create_ctrl	= nvme_fc_create_ctrl,
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
index ba3d7f3349e5..50ef71ee3d86 100644
--- a/drivers/nvme/host/lightnvm.c
+++ b/drivers/nvme/host/lightnvm.c
@@ -31,27 +31,10 @@
 
 enum nvme_nvm_admin_opcode {
 	nvme_nvm_admin_identity		= 0xe2,
-	nvme_nvm_admin_get_l2p_tbl	= 0xea,
 	nvme_nvm_admin_get_bb_tbl	= 0xf2,
 	nvme_nvm_admin_set_bb_tbl	= 0xf1,
 };
 
-struct nvme_nvm_hb_rw {
-	__u8			opcode;
-	__u8			flags;
-	__u16			command_id;
-	__le32			nsid;
-	__u64			rsvd2;
-	__le64			metadata;
-	__le64			prp1;
-	__le64			prp2;
-	__le64			spba;
-	__le16			length;
-	__le16			control;
-	__le32			dsmgmt;
-	__le64			slba;
-};
-
 struct nvme_nvm_ph_rw {
 	__u8			opcode;
 	__u8			flags;
@@ -80,19 +63,6 @@ struct nvme_nvm_identity {
 	__u32			rsvd11[5];
 };
 
-struct nvme_nvm_l2ptbl {
-	__u8			opcode;
-	__u8			flags;
-	__u16			command_id;
-	__le32			nsid;
-	__le32			cdw2[4];
-	__le64			prp1;
-	__le64			prp2;
-	__le64			slba;
-	__le32			nlb;
-	__le16			cdw14[6];
-};
-
 struct nvme_nvm_getbbtbl {
 	__u8			opcode;
 	__u8			flags;
@@ -139,9 +109,7 @@ struct nvme_nvm_command {
 	union {
 		struct nvme_common_command common;
 		struct nvme_nvm_identity identity;
-		struct nvme_nvm_hb_rw hb_rw;
 		struct nvme_nvm_ph_rw ph_rw;
-		struct nvme_nvm_l2ptbl l2p;
 		struct nvme_nvm_getbbtbl get_bb;
 		struct nvme_nvm_setbbtbl set_bb;
 		struct nvme_nvm_erase_blk erase;
@@ -167,7 +135,7 @@ struct nvme_nvm_id_group {
 	__u8			num_lun;
 	__u8			num_pln;
 	__u8			rsvd1;
-	__le16			num_blk;
+	__le16			num_chk;
 	__le16			num_pg;
 	__le16			fpg_sz;
 	__le16			csecs;
@@ -234,11 +202,9 @@ struct nvme_nvm_bb_tbl {
 static inline void _nvme_nvm_check_size(void)
 {
 	BUILD_BUG_ON(sizeof(struct nvme_nvm_identity) != 64);
-	BUILD_BUG_ON(sizeof(struct nvme_nvm_hb_rw) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_nvm_ph_rw) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_nvm_getbbtbl) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_nvm_setbbtbl) != 64);
-	BUILD_BUG_ON(sizeof(struct nvme_nvm_l2ptbl) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_nvm_erase_blk) != 64);
 	BUILD_BUG_ON(sizeof(struct nvme_nvm_id_group) != 960);
 	BUILD_BUG_ON(sizeof(struct nvme_nvm_addr_format) != 16);
@@ -249,51 +215,58 @@ static inline void _nvme_nvm_check_size(void)
 static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id)
 {
 	struct nvme_nvm_id_group *src;
-	struct nvm_id_group *dst;
+	struct nvm_id_group *grp;
+	int sec_per_pg, sec_per_pl, pg_per_blk;
 
 	if (nvme_nvm_id->cgrps != 1)
 		return -EINVAL;
 
 	src = &nvme_nvm_id->groups[0];
-	dst = &nvm_id->grp;
-
-	dst->mtype = src->mtype;
-	dst->fmtype = src->fmtype;
-	dst->num_ch = src->num_ch;
-	dst->num_lun = src->num_lun;
-	dst->num_pln = src->num_pln;
-
-	dst->num_pg = le16_to_cpu(src->num_pg);
-	dst->num_blk = le16_to_cpu(src->num_blk);
-	dst->fpg_sz = le16_to_cpu(src->fpg_sz);
-	dst->csecs = le16_to_cpu(src->csecs);
-	dst->sos = le16_to_cpu(src->sos);
-
-	dst->trdt = le32_to_cpu(src->trdt);
-	dst->trdm = le32_to_cpu(src->trdm);
-	dst->tprt = le32_to_cpu(src->tprt);
-	dst->tprm = le32_to_cpu(src->tprm);
-	dst->tbet = le32_to_cpu(src->tbet);
-	dst->tbem = le32_to_cpu(src->tbem);
-	dst->mpos = le32_to_cpu(src->mpos);
-	dst->mccap = le32_to_cpu(src->mccap);
-
-	dst->cpar = le16_to_cpu(src->cpar);
-
-	if (dst->fmtype == NVM_ID_FMTYPE_MLC) {
-		memcpy(dst->lptbl.id, src->lptbl.id, 8);
-		dst->lptbl.mlc.num_pairs =
-				le16_to_cpu(src->lptbl.mlc.num_pairs);
-
-		if (dst->lptbl.mlc.num_pairs > NVME_NVM_LP_MLC_PAIRS) {
-			pr_err("nvm: number of MLC pairs not supported\n");
-			return -EINVAL;
-		}
+	grp = &nvm_id->grp;
+
+	grp->mtype = src->mtype;
+	grp->fmtype = src->fmtype;
+
+	grp->num_ch = src->num_ch;
+	grp->num_lun = src->num_lun;
+
+	grp->num_chk = le16_to_cpu(src->num_chk);
+	grp->csecs = le16_to_cpu(src->csecs);
+	grp->sos = le16_to_cpu(src->sos);
+
+	pg_per_blk = le16_to_cpu(src->num_pg);
+	sec_per_pg = le16_to_cpu(src->fpg_sz) / grp->csecs;
+	sec_per_pl = sec_per_pg * src->num_pln;
+	grp->clba = sec_per_pl * pg_per_blk;
+	grp->ws_per_chk = pg_per_blk;
 
-		memcpy(dst->lptbl.mlc.pairs, src->lptbl.mlc.pairs,
-					dst->lptbl.mlc.num_pairs);
+	grp->mpos = le32_to_cpu(src->mpos);
+	grp->cpar = le16_to_cpu(src->cpar);
+	grp->mccap = le32_to_cpu(src->mccap);
+
+	grp->ws_opt = grp->ws_min = sec_per_pg;
+	grp->ws_seq = NVM_IO_SNGL_ACCESS;
+
+	if (grp->mpos & 0x020202) {
+		grp->ws_seq = NVM_IO_DUAL_ACCESS;
+		grp->ws_opt <<= 1;
+	} else if (grp->mpos & 0x040404) {
+		grp->ws_seq = NVM_IO_QUAD_ACCESS;
+		grp->ws_opt <<= 2;
 	}
 
+	grp->trdt = le32_to_cpu(src->trdt);
+	grp->trdm = le32_to_cpu(src->trdm);
+	grp->tprt = le32_to_cpu(src->tprt);
+	grp->tprm = le32_to_cpu(src->tprm);
+	grp->tbet = le32_to_cpu(src->tbet);
+	grp->tbem = le32_to_cpu(src->tbem);
+
+	/* 1.2 compatibility */
+	grp->num_pln = src->num_pln;
+	grp->num_pg = le16_to_cpu(src->num_pg);
+	grp->fpg_sz = le16_to_cpu(src->fpg_sz);
+
 	return 0;
 }
 
@@ -332,62 +305,6 @@ static int nvme_nvm_identity(struct nvm_dev *nvmdev, struct nvm_id *nvm_id)
 	return ret;
 }
 
-static int nvme_nvm_get_l2p_tbl(struct nvm_dev *nvmdev, u64 slba, u32 nlb,
-				nvm_l2p_update_fn *update_l2p, void *priv)
-{
-	struct nvme_ns *ns = nvmdev->q->queuedata;
-	struct nvme_nvm_command c = {};
-	u32 len = queue_max_hw_sectors(ns->ctrl->admin_q) << 9;
-	u32 nlb_pr_rq = len / sizeof(u64);
-	u64 cmd_slba = slba;
-	void *entries;
-	int ret = 0;
-
-	c.l2p.opcode = nvme_nvm_admin_get_l2p_tbl;
-	c.l2p.nsid = cpu_to_le32(ns->head->ns_id);
-	entries = kmalloc(len, GFP_KERNEL);
-	if (!entries)
-		return -ENOMEM;
-
-	while (nlb) {
-		u32 cmd_nlb = min(nlb_pr_rq, nlb);
-		u64 elba = slba + cmd_nlb;
-
-		c.l2p.slba = cpu_to_le64(cmd_slba);
-		c.l2p.nlb = cpu_to_le32(cmd_nlb);
-
-		ret = nvme_submit_sync_cmd(ns->ctrl->admin_q,
-				(struct nvme_command *)&c, entries, len);
-		if (ret) {
-			dev_err(ns->ctrl->device,
-				"L2P table transfer failed (%d)\n", ret);
-			ret = -EIO;
-			goto out;
-		}
-
-		if (unlikely(elba > nvmdev->total_secs)) {
-			pr_err("nvm: L2P data from device is out of bounds!\n");
-			ret = -EINVAL;
-			goto out;
-		}
-
-		/* Transform physical address to target address space */
-		nvm_part_to_tgt(nvmdev, entries, cmd_nlb);
-
-		if (update_l2p(cmd_slba, cmd_nlb, entries, priv)) {
-			ret = -EINTR;
-			goto out;
-		}
-
-		cmd_slba += cmd_nlb;
-		nlb -= cmd_nlb;
-	}
-
-out:
-	kfree(entries);
-	return ret;
-}
-
 static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
 								u8 *blks)
 {
@@ -397,7 +314,7 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
 	struct nvme_ctrl *ctrl = ns->ctrl;
 	struct nvme_nvm_command c = {};
 	struct nvme_nvm_bb_tbl *bb_tbl;
-	int nr_blks = geo->blks_per_lun * geo->plane_mode;
+	int nr_blks = geo->nr_chks * geo->plane_mode;
 	int tblsz = sizeof(struct nvme_nvm_bb_tbl) + nr_blks;
 	int ret = 0;
 
@@ -438,7 +355,7 @@ static int nvme_nvm_get_bb_tbl(struct nvm_dev *nvmdev, struct ppa_addr ppa,
 		goto out;
 	}
 
-	memcpy(blks, bb_tbl->blk, geo->blks_per_lun * geo->plane_mode);
+	memcpy(blks, bb_tbl->blk, geo->nr_chks * geo->plane_mode);
 out:
 	kfree(bb_tbl);
 	return ret;
@@ -474,10 +391,6 @@ static inline void nvme_nvm_rqtocmd(struct nvm_rq *rqd, struct nvme_ns *ns,
 	c->ph_rw.metadata = cpu_to_le64(rqd->dma_meta_list);
 	c->ph_rw.control = cpu_to_le16(rqd->flags);
 	c->ph_rw.length = cpu_to_le16(rqd->nr_ppas - 1);
-
-	if (rqd->opcode == NVM_OP_HBWRITE || rqd->opcode == NVM_OP_HBREAD)
-		c->hb_rw.slba = cpu_to_le64(nvme_block_nr(ns,
-					rqd->bio->bi_iter.bi_sector));
 }
 
 static void nvme_nvm_end_io(struct request *rq, blk_status_t status)
@@ -597,8 +510,6 @@ static void nvme_nvm_dev_dma_free(void *pool, void *addr,
 static struct nvm_dev_ops nvme_nvm_dev_ops = {
 	.identity		= nvme_nvm_identity,
 
-	.get_l2p_tbl		= nvme_nvm_get_l2p_tbl,
-
 	.get_bb_tbl		= nvme_nvm_get_bb_tbl,
 	.set_bb_tbl		= nvme_nvm_set_bb_tbl,
 
@@ -883,7 +794,7 @@ static ssize_t nvm_dev_attr_show(struct device *dev,
 	} else if (strcmp(attr->name, "num_planes") == 0) {
 		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pln);
 	} else if (strcmp(attr->name, "num_blocks") == 0) {	/* u16 */
-		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_blk);
+		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_chk);
 	} else if (strcmp(attr->name, "num_pages") == 0) {
 		return scnprintf(page, PAGE_SIZE, "%u\n", grp->num_pg);
 	} else if (strcmp(attr->name, "page_size") == 0) {
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 1218a9fca846..060f69e03427 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -33,51 +33,11 @@ void nvme_failover_req(struct request *req)
 	kblockd_schedule_work(&ns->head->requeue_work);
 }
 
-bool nvme_req_needs_failover(struct request *req)
+bool nvme_req_needs_failover(struct request *req, blk_status_t error)
 {
 	if (!(req->cmd_flags & REQ_NVME_MPATH))
 		return false;
-
-	switch (nvme_req(req)->status & 0x7ff) {
-	/*
-	 * Generic command status:
-	 */
-	case NVME_SC_INVALID_OPCODE:
-	case NVME_SC_INVALID_FIELD:
-	case NVME_SC_INVALID_NS:
-	case NVME_SC_LBA_RANGE:
-	case NVME_SC_CAP_EXCEEDED:
-	case NVME_SC_RESERVATION_CONFLICT:
-		return false;
-
-	/*
-	 * I/O command set specific error.  Unfortunately these values are
-	 * reused for fabrics commands, but those should never get here.
-	 */
-	case NVME_SC_BAD_ATTRIBUTES:
-	case NVME_SC_INVALID_PI:
-	case NVME_SC_READ_ONLY:
-	case NVME_SC_ONCS_NOT_SUPPORTED:
-		WARN_ON_ONCE(nvme_req(req)->cmd->common.opcode ==
-			nvme_fabrics_command);
-		return false;
-
-	/*
-	 * Media and Data Integrity Errors:
-	 */
-	case NVME_SC_WRITE_FAULT:
-	case NVME_SC_READ_ERROR:
-	case NVME_SC_GUARD_CHECK:
-	case NVME_SC_APPTAG_CHECK:
-	case NVME_SC_REFTAG_CHECK:
-	case NVME_SC_COMPARE_FAILED:
-	case NVME_SC_ACCESS_DENIED:
-	case NVME_SC_UNWRITTEN_BLOCK:
-		return false;
-	}
-
-	/* Everything else could be a path failure, so should be retried */
-	return true;
+	return blk_path_error(error);
 }
 
 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
@@ -238,30 +198,16 @@ void nvme_mpath_add_disk(struct nvme_ns_head *head)
 {
 	if (!head->disk)
 		return;
-	device_add_disk(&head->subsys->dev, head->disk);
-	if (sysfs_create_group(&disk_to_dev(head->disk)->kobj,
-			&nvme_ns_id_attr_group))
-		pr_warn("%s: failed to create sysfs group for identification\n",
-			head->disk->disk_name);
-}
-
-void nvme_mpath_add_disk_links(struct nvme_ns *ns)
-{
-	struct kobject *slave_disk_kobj, *holder_disk_kobj;
-
-	if (!ns->head->disk)
-		return;
-
-	slave_disk_kobj = &disk_to_dev(ns->disk)->kobj;
-	if (sysfs_create_link(ns->head->disk->slave_dir, slave_disk_kobj,
-			kobject_name(slave_disk_kobj)))
-		return;
 
-	holder_disk_kobj = &disk_to_dev(ns->head->disk)->kobj;
-	if (sysfs_create_link(ns->disk->part0.holder_dir, holder_disk_kobj,
-			kobject_name(holder_disk_kobj)))
-		sysfs_remove_link(ns->head->disk->slave_dir,
-			kobject_name(slave_disk_kobj));
+	mutex_lock(&head->subsys->lock);
+	if (!(head->disk->flags & GENHD_FL_UP)) {
+		device_add_disk(&head->subsys->dev, head->disk);
+		if (sysfs_create_group(&disk_to_dev(head->disk)->kobj,
+				&nvme_ns_id_attr_group))
+			pr_warn("%s: failed to create sysfs group for identification\n",
+				head->disk->disk_name);
+	}
+	mutex_unlock(&head->subsys->lock);
 }
 
 void nvme_mpath_remove_disk(struct nvme_ns_head *head)
@@ -278,14 +224,3 @@ void nvme_mpath_remove_disk(struct nvme_ns_head *head)
 	blk_cleanup_queue(head->disk->queue);
 	put_disk(head->disk);
 }
-
-void nvme_mpath_remove_disk_links(struct nvme_ns *ns)
-{
-	if (!ns->head->disk)
-		return;
-
-	sysfs_remove_link(ns->disk->part0.holder_dir,
-			kobject_name(&disk_to_dev(ns->head->disk)->kobj));
-	sysfs_remove_link(ns->head->disk->slave_dir,
-			kobject_name(&disk_to_dev(ns->disk)->kobj));
-}
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index a00eabd06427..d733b14ede9d 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -32,6 +32,8 @@ extern unsigned int admin_timeout;
 #define NVME_KATO_GRACE		10
 
 extern struct workqueue_struct *nvme_wq;
+extern struct workqueue_struct *nvme_reset_wq;
+extern struct workqueue_struct *nvme_delete_wq;
 
 enum {
 	NVME_NS_LBA		= 0,
@@ -119,8 +121,9 @@ static inline struct n